diff --git a/.clang-format b/.clang-format
index 5ec78a2f0665..21f2150afcec 100644
--- a/.clang-format
+++ b/.clang-format
@@ -23,4 +23,3 @@ AlignAfterOpenBracket: true
 BinPackArguments : false
 AlignOperands : true
 BreakBeforeTernaryOperators : true
-AllowAllParametersOfDeclarationOnNextLine : false
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
new file mode 100644
index 000000000000..84ef7023736f
--- /dev/null
+++ b/.devcontainer/Dockerfile
@@ -0,0 +1,40 @@
+FROM fluxrm/testenv:focal
+
+LABEL maintainer="Vanessasaurus <@vsoch>"
+
+# Match the default user id for a single system so we aren't root
+ARG USERNAME=vscode
+ARG USER_UID=1000
+ARG USER_GID=1000
+ENV USERNAME=${USERNAME}
+ENV USER_UID=${USER_UID}
+ENV USER_GID=${USER_GID}
+
+# Pip not provided in this version
+RUN apt-get update
+COPY scripts/requirements-dev.txt /requirements.txt
+
+# For easier Python development.
+RUN python3 -m pip install IPython && \
+    python3 -m pip install -r /requirements.txt
+
+# Assuming installing to /usr/local
+ENV LD_LIBRARY_PATH=/usr/local/lib
+
+# Assuming installing to /usr/local
+ENV LD_LIBRARY_PATH=/usr/local/lib
+
+# extra interactive utilities and compilation_database generation
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+    bear \
+    fd-find \
+    gdb \
+    less \
+    ripgrep
+
+# Add the group and user that match our ids
+RUN groupadd -g ${USER_GID} ${USERNAME} && \
+    adduser --disabled-password --uid ${USER_UID} --gid ${USER_GID} --gecos "" ${USERNAME} && \
+    echo "${USERNAME} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers
+USER $USERNAME
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 000000000000..f3a64d03f367
--- /dev/null
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,39 @@
+{
+    "name": "Flux Core Python 3.6",
+    "dockerFile": "Dockerfile",
+    "context": "../",
+
+    "customizations": {
+      "vscode": {
+        "settings": {
+          "terminal.integrated.defaultProfile.linux": "bash",
+
+          // Ensure that Python autocomplete works out of the box
+          "python.autoComplete.extraPaths": [
+            "/usr/local/lib/flux/python3.8",
+            "/usr/local/lib/python3.8/site-packages",
+            "/workspaces/flux-core/src/bindings/python"
+          ],
+          "python.analysis.extraPaths": [
+            "/usr/local/lib/flux/python3.8",
+            "/usr/local/lib/python3.8/site-packages",
+            "/workspaces/flux-core/src/bindings/python"
+          ]
+        },
+        // Note to Flux Developers! We can add extensions here that you like
+        "extensions": [
+          "ms-vscode.cpptools", // C and C++ support
+          "sumneko.lua", // Lua support
+          "ms-python.python", // Python support
+          "GitHub.vscode-pull-request-github", // manage and review PRs
+        ]
+      }
+    },
+  "features": {
+    "ghcr.io/devcontainers/features/github-cli:1": {
+      "version": "latest"
+    }
+  },
+  // Needed for git security feature (this assumes you locally cloned to flux-core)
+  "postStartCommand": "git config --global --add safe.directory /workspaces/flux-core"
+}
diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml
new file mode 100644
index 000000000000..51805017aab0
--- /dev/null
+++ b/.github/codeql/codeql-config.yml
@@ -0,0 +1,9 @@
+Name: "flux codeql defaults"
+queries:
+  - uses: security-and-quality
+paths:
+  - src
+paths-ignore:
+  - "**/flux/utils/**/*.py"
+  - "src/**/parsedatetime/**/*.py"
+  - "**/_flux/**/*.py"
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 000000000000..62020476bbf5
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,59 @@
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 365
+daysUntilClose: 14
+
+# Only these labels will be considered stale. Set to `[]` to disable:
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered
+#  stale. Set to `[]` to disable
+#
+exemptLabels:
+  - pinned
+  - security
+  - design
+  - confirmed
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: true
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: true
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: wontfix
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  activity for 365 days. It will be closed if no further activity occurs
+  within 14 days.  Thank you for your contributions.
+
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 20
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed
diff --git a/.github/workflows/asan.yml b/.github/workflows/asan.yml
new file mode 100644
index 000000000000..b5274cf65023
--- /dev/null
+++ b/.github/workflows/asan.yml
@@ -0,0 +1,32 @@
+on: [push, pull_request]
+name: asan
+jobs:
+  check-asan:
+    name: address-sanitizer check
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+    - run: git fetch --tags || true
+    - name: disable ASLR
+      run: |
+        sudo sysctl -w kernel.randomize_va_space=0
+    - name: docker-run-checks with ASan
+      timeout-minutes: 40
+      env:
+        PRELOAD: /usr/lib64/libasan.so.8
+        ASAN_OPTIONS: detect_leaks=0,start_deactivated=true,replace_str=true,verify_asan_link_order=false
+        FLUX_TEST_TIMEOUT: 300
+        TAP_DRIVER_QUIET: t
+      run: >
+        src/test/docker/docker-run-checks.sh \
+          --image=fedora36 --unit-test-only -j4 \
+          -- --with-flux-security --enable-sanitizer=address
+
+    - name: after failure
+      if: failure() || cancelled()
+      run: src/test/checks-annotate.sh
+
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index 000000000000..7799db67b1c7
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,88 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "master" ]
+  schedule:
+    - cron: '42 0 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'cpp', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        queries: security-and-quality
+        config-file: ./.github/codeql/codeql-config.yml
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+    - name: Install ubuntu dependencies
+      run: |
+        sudo apt update
+        sudo scripts/install-deps-deb.sh
+    - name: Install python dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install -r scripts/requirements-dev.txt
+        # Set the `CODEQL-PYTHON` environment variable to the Python executable
+        # that includes the dependencies
+        echo "CODEQL_PYTHON=$(which python3)" >> $GITHUB_ENV
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    # - name: Autobuild
+    #   uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    - run: |
+        export FLUX_VERSION=0.0.0
+        ./autogen.sh
+        ./configure
+        make -j 4
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 16b30d20513f..125843d3c4fd 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -1,30 +1,277 @@
-on: pull_request
+on: [pull_request, push, merge_group]
+name: ci
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 jobs:
   check-pr:
     name: validate commits
     runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
         fetch-depth: 0
     - run: git fetch origin master
     - uses: flux-framework/pr-validator@master
 
+  spelling:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v4
+    - name: Check Spelling
+      uses: crate-ci/typos@bcafd462cb07ef7ba57e34abf458fe20767e808b # v1.19.0
+
+  python-lint:
+    name: python linting
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/setup-python@v5
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+    - name: install linting and formatting deps
+      run: pip install -r scripts/requirements-dev.txt
+    - name: format and linting checks
+      run: pre-commit run --all-files
+
   check-sched:
+    needs: [python-lint]
     name: flux-sched check
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
         fetch-depth: 0
-    - run: git fetch --tags
+    - run: git fetch --tags || true
     - run: >
-        src/test/docker/docker-run-checks.sh --install-only
-        --tag=fluxrm/flux-core:bionic
+        src/test/docker/docker-run-checks.sh --install-only \
+          --tag=fluxrm/flux-core:bookworm
     - run: >
-        cd .. &&
-        git clone https://github.com/flux-framework/flux-sched &&
+        cd .. && git clone https://github.com/flux-framework/flux-sched &&
         cd flux-sched &&
-        src/test/docker/docker-run-checks.sh -j 4 -i bionic
+        src/test/docker/docker-run-checks.sh -j 4 -i bookworm -- CXXFLAGS=-Wno-error=maybe-uninitialized
+
+  check-accounting:
+    needs: [python-lint]
+    name: flux-accounting check
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+    - run: >
+        src/test/docker/docker-run-checks.sh --image=el8 --install-only \
+          --tag=fluxrm/flux-core:el8
+    - run: >
+        cd .. && git clone https://github.com/flux-framework/flux-accounting &&
+        cd flux-accounting && src/test/docker/docker-run-checks.sh -j 4
+
+  check-pmix:
+    needs: [python-lint]
+    name: flux-pmix check
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+    - run: >
+        src/test/docker/docker-run-checks.sh
+        --image=el8
+        --install-only
+        --tag=fluxrm/flux-core:el8
+    - run: >
+        cd .. &&
+        git clone https://github.com/flux-framework/flux-pmix &&
+        cd flux-pmix &&
+        src/test/docker/docker-run-checks.sh -j 4 -i el8
+        --build-arg OMPI_BRANCH=v5.0.0rc12
+        --build-arg OPENPMIX_BRANCH=v4.2.3
+
+  check-pam:
+    needs: [python-lint]
+    name: flux-pam check
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+    - run: >
+        src/test/docker/docker-run-checks.sh
+        --image=el8
+        --install-only
+        --tag=fluxrm/flux-core:el8
+    - run: >
+        cd .. &&
+        git clone https://github.com/flux-framework/flux-pam &&
+        cd flux-pam &&
+        src/test/docker/docker-run-checks.sh -j 4 -i el8
+
+  build-macos:
+    needs: [python-lint]
+    name: macos build only
+    runs-on: macos-14
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+    - name: Install dependencies
+      run: scripts/install-deps-macos.sh
+    - name: autogen, configure
+      run: scripts/configure-macos.sh
+    - name: make, including test programs
+      run: make check -j4 TESTS=
+
+  generate-matrix:
+    # https://stackoverflow.com/questions/59977364
+    name: Generate build matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+    - id: set-matrix
+      run: echo "matrix=$(src/test/generate-matrix.py)" >> $GITHUB_OUTPUT
+    - run: src/test/generate-matrix.py | jq -S .
+    - run: echo "GITHUB_BRANCH=${GITHUB_REF#refs/heads}" >> $GITHUB_OUTPUT
+    - run: echo "GITHUB_TAG=${GITHUB_REF#refs/tags}" >> $GITHUB_OUTPUT
+    - run: echo "EVENT_NAME=${{ github.event_name }}" >> $GITHUB_OUTPUT
+
+  ci-checks:
+    needs: [generate-matrix]
+    runs-on: ubuntu-latest
+    env:
+      TAP_DRIVER_QUIET: 1
+      FLUX_TEST_TIMEOUT: 300
+      DOCKER_REPO: fluxrm/flux-core
+      DOCKER_USERNAME: travisflux
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_HUB_TRAVISFLUX_TOKEN }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}}
+      fail-fast: false
+    name: ${{matrix.name}}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+
+    - name: fetch annotated tag
+      if: >
+        (matrix.create_release || matrix.docker_tag) &&
+        github.ref != 'refs/heads/master'
+      run: |
+        # Ensure git-describe works on a tag.
+        #  (checkout@v4 action may have left current tag as
+        #   lightweight instead of annotated. See
+        #   https://github.com/actions/checkout/issues/290)
+        #
+        echo github.ref == ${{ github.ref }} ;
+        git fetch -f origin ${{ github.ref }}:${{ github.ref }} ;
+        echo git describe now reports $(git describe --always)
+
+    - name: coverage setup
+      env: ${{matrix.env}}
+      if: matrix.coverage
+      run: |
+        # Use python3 coverage to match version in flux docker image
+        sudo apt update ; \
+        sudo apt install -yy python3-pip ; \
+        pip3 install --upgrade pip ;
+        pip3 install --upgrade --force-reinstall coverage ;
+
+    - name: s3 setup
+      env: ${{matrix.env}}
+      if: matrix.test_s3
+      run: |
+        docker run -d -p 9000:9000 minio/minio server /data; \
+
+    - name: generate dumpfile from most recent flux-core tag
+      if: (matrix.create_release != true)
+      run: |
+        src/test/create-kvs-dumpfile.sh -d /tmp/dumpfile &&
+        if test -f /tmp/dumpfile/*.bz2; then
+            cp /tmp/dumpfile/*.tar.bz2 $(pwd)/t/job-manager/dumps/valid
+        fi
+
+    - name: docker buildx
+      uses: docker/setup-buildx-action@v3
+      if: matrix.needs_buildx
+
+    - name: setup qemu-user-static
+      run: |
+        docker run --rm --privileged aptman/qus -s -- -p --credential aarch64
+
+    - name: docker-run-checks
+      timeout-minutes: ${{matrix.timeout_minutes}}
+      env: ${{matrix.env}}
+      run: ${{matrix.command}}
+
+    - name: annotate errors
+      if: failure() || cancelled()
+      env: ${{matrix.env}}
+      run: src/test/checks-annotate.sh
+
+    - name: coverage report
+      if: success() && matrix.coverage
+      env:
+        DOCKER_REPO:
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        flags: ${{matrix.coverage_flags}}
+
+    - name: docker deploy
+      if: success() && matrix.docker_tag
+      env: ${{matrix.env}}
+      run: src/test/docker-deploy.sh
+
+    - name: create release
+      id: create_release
+      if: |
+        success()
+        && matrix.create_release
+        && github.repository == 'flux-framework/flux-core'
+      env: ${{matrix.env}}
+      uses: softprops/action-gh-release@v1
+      with:
+        tag_name: ${{ matrix.tag }}
+        name: flux-core ${{ matrix.tag }}
+        prerelease: true
+        files: flux-core*.tar.gz
+        body: |
+          View [Release Notes](https://github.com/${{ github.repository }}/blob/${{ matrix.tag }}/NEWS.md) for flux-core ${{ matrix.tag }}
+
+  generate-manifest:
+    name: Generate docker manifest
+    runs-on: ubuntu-latest
+    needs: [ci-checks]
+    env:
+      DOCKER_REPO: fluxrm/flux-core
+      DOCKER_USERNAME: travisflux
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_HUB_TRAVISFLUX_TOKEN }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+    - name: make and push manifest as fluxrm/flux-core
+      if: >
+        (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/master')
+      run: |
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker manifest create fluxrm/flux-core:bookworm fluxrm/flux-core:bookworm-amd64 fluxrm/flux-core:bookworm-386 fluxrm/flux-core:bookworm-arm64
+        docker manifest push fluxrm/flux-core:bookworm
+        for d in el9 noble alpine ; do
+          docker manifest create fluxrm/flux-core:$d fluxrm/flux-core:$d-amd64 fluxrm/flux-core:$d-arm64
+          docker manifest push fluxrm/flux-core:$d
+        done
+
diff --git a/.gitignore b/.gitignore
index 8f7432c9db1a..92926972936d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ libltdl/
 
 # docs intermediate files
 /doc/man*/*.xml
+/doc/_build
 
 # Object files
 *.o
@@ -86,6 +87,18 @@ Makefile
 *.log
 .dirstamp
 
+# ignore mypy generated cache directory
+.mypy_cache
+
+# ignore local, maybe generated tooling files
+compile_commands.json
+compile_flags.txt
+
+# local editor config dirs
+.idea
+.clangd
+.cache
+
 # Rules to ignore auto-generated test harness scripts
 test_*.t
 !/src/bindings/python/test_commands/test_runner.t
diff --git a/.mergify.yml b/.mergify.yml
index 774fe44b9441..f54c993d0f57 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -1,18 +1,28 @@
+queue_rules:
+  - name: default
+    update_method: rebase
+    merge_method: merge
+
 pull_request_rules:
   - name: rebase and merge when passing all checks
     conditions:
       - base=master
-      - status-success=continuous-integration/travis-ci/pr
-      - status-success="validate commits"
-      - status-success="flux-sched check"
       - label="merge-when-passing"
       - label!="work-in-progress"
+      - -title~=^\[*[Ww][Ii][Pp]
       - "approved-reviews-by=@flux-framework/core"
       - "#approved-reviews-by>0"
       - "#changes-requested-reviews-by=0"
-      - -title~=^\[*[Ww][Ii][Pp]
     actions:
-      merge:
-        method: merge
-        strict: smart
-        strict_method: rebase
+      queue:
+        name: default
+  - name: remove outdated approved reviews
+    conditions:
+      - author!=@core
+    actions:
+      dismiss_reviews:
+        approved: true
+        changes_requested: false
+        message: |
+          Approving reviews have been dismissed because this pull request
+          was updated.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000000..1e16d5bcbb66
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,44 @@
+files: '^(src/bindings/python/flux|src/cmd|t/python/.*\.py|t/scripts/.*\.py)'
+exclude: "^(src/bindings/python/_flux/|src/bindings/python/flux/utils/|t/python/tap)"
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-shebang-scripts-are-executable
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: mixed-line-ending
+
+  - repo: local
+    hooks:
+      - id: black
+        name: black
+        language: python
+        types: [python]
+        entry: black
+
+      - id: isort
+        name: isort
+        args: [--filter-files]
+        language: python
+        types: [python]
+        entry: isort
+
+      - id: flake8
+        name: flake8
+        language: python
+        types: [python]
+        entry: flake8
+
+      - id: mypy
+        name: mypy
+        language: python
+        types: [python]
+        entry: ./scripts/run_mypy.sh
+  - repo: https://github.com/netromdk/vermin
+    rev: v1.5.1
+    hooks:
+      - id: vermin
+        args: ['-t=3.6-', '--violations']
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 000000000000..3803a410de62
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,22 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: doc/conf.py
+
+# We recommend specifying your dependencies to enable reproducible builds:
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: doc/requirements.txt
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index ff4e89b0c7fa..000000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,195 +0,0 @@
-sudo: required
-services: docker
-dist: trusty
-language: c
-
-jobs:
-  include:
-    - stage: 'style checks'
-      name: 'python format'
-      language: 'python'
-      python: '3.6'
-      install: pip install --upgrade black
-      script: ./scripts/check-format
-      before_deploy: skip
-      deploy: skip
-      after_success: skip
-      after_failure: skip
-    - stage: 'style checks'
-      name: 'python lint'
-      language: 'python'
-      python: '3.6'
-      install: pip install 'pylint==2.2.2' --force-reinstall
-      script: pylint --rcfile=src/bindings/python/.pylintrc src/bindings/python/flux
-      before_deploy: skip
-      deploy: skip
-      after_success: skip
-      after_failure: skip
-    - name: "Ubuntu: no configure flags"
-      stage: test
-      compiler: gcc
-      env:
-       - PYTHON_VERSION=2.7
-    - name: "Ubuntu: py3.6 distcheck"
-      stage: test
-      compiler: gcc
-      env:
-       - DISTCHECK=t
-       - PYTHON_VERSION=3.6
-       - GITHUB_RELEASES_DEPLOY=t
-    - name: "Ubuntu: gcc-8 address-sanitizer --with-flux-security"
-      stage: test
-      compiler: gcc-8
-      env:
-       - CC=gcc-8
-       - CXX=g++-8
-       - ARGS="--with-flux-security --enable-sanitizer=address"
-       - PRELOAD=/usr/lib/gcc/x86_64-linux-gnu/8/libasan.so
-       - ASAN_OPTIONS=detect_leaks=0 # TODO: fix/suppress tests so we can turn this on
-       - PYTHON_VERSION=3.6
-    - name: "Ubuntu: gcc-8 --with-flux-security/caliper, distcheck"
-      stage: test
-      compiler: gcc-8
-      env:
-       - CC=gcc-8
-       - CXX=g++-8
-       - ARGS="--with-flux-security --enable-caliper"
-       - DISTCHECK=t
-       - PYTHON_VERSION=2.7
-    - name: "Ubuntu: clang-6.0 chain-lint --with-flux-security"
-      stage: test
-      compiler: clang-6.0
-      env:
-       - CC=clang-6.0
-       - CXX=clang++-6.0
-       - chain_lint=t
-       - ARGS="--with-flux-security"
-       - PYTHON_VERSION=2.7
-    - name: "Ubuntu: COVERAGE=t, --with-flux-security --enable-caliper"
-      stage: test
-      compiler: gcc
-      env:
-       - COVERAGE=t
-       - ARGS="--with-flux-security --enable-caliper"
-       - PYTHON_VERSION=2.7
-    - name: "Ubuntu: TEST_INSTALL docker-deploy"
-      stage: test
-      compiler: gcc
-      env:
-       - ARGS="--with-flux-security --enable-caliper"
-       - TEST_INSTALL=t
-       - DOCKER_TAG=t
-       - PYTHON_VERSION=2.7
-    - name: "Centos 7: security,caliper,docker-deploy"
-      stage: test
-      compiler: gcc
-      env:
-       - ARGS="--with-flux-security --enable-caliper --prefix=/usr"
-       - IMG=centos7
-       - DOCKER_TAG=t
-       - PYTHON_VERSION=2.7
-    - name: "Centos 8: py3.6,security,caliper,docker-deploy"
-      stage: test
-      compiler: gcc
-      env:
-       - ARGS="--with-flux-security --enable-caliper --prefix=/usr"
-       - IMG=centos8
-       - DOCKER_TAG=t
-       - PYTHON_VERSION=3.6
-
-stages:
-  - 'style checks'
-  - test
-
-env:
-  global:
-   - TAP_DRIVER_QUIET=1
-   - DOCKERREPO=fluxrm/flux-core
-   - DOCKER_USERNAME=travisflux
-   - secure: "Uga2i1Yu0PvWMFzOYvM9yxnAMDTgY17ZqeFlIN8MV3uoTCy6y61GULrMkKuhuI1sUfyugpFWVKIJo5jwTpsfG84f3o9lUTRgLPpTA2Xls8A/rmurF/QacVv6hZ2Zs2LQVlrM8BkT36TpT2NfWW2D2238kovqz3l5gIZKMClMvyk="
-
-cache:
-  directories:
-    - $HOME/.ccache
-
-before_install:
- # work around persistent write error bug from make in travis
- # see https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959
- - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
- # die if non-blocking is still enabled
- - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); exit(flags&os.O_NONBLOCK);'
- # coveralls-lcov required only for coveralls upload:
- - if test "$COVERAGE" = "t" ; then gem install coveralls-lcov; fi
- - if test -z "${IMG}"; then IMG="bionic"; fi
- #
- #  Tag image if this build is on master or result of a tag:
- - |
-  if test "$DOCKER_TAG" = "t" \
-    -a "$TRAVIS_REPO_SLUG" = "flux-framework/flux-core" \
-    -a "$TRAVIS_PULL_REQUEST" = "false" \
-    -a \( "$TRAVIS_BRANCH" = "master" -o -n "$TRAVIS_TAG" \); then
-     export TAGNAME="${DOCKERREPO}:${IMG}${TRAVIS_TAG:+-${TRAVIS_TAG}}"
-     echo "Tagging new image $TAGNAME"
-  fi
-
-script:
- # Unshallow repository so git describe works.
- # (The one inside docker-run-checks may fail if git version is too old)
- - git fetch --unshallow --tags
- - |
-  src/test/docker/docker-run-checks.sh -j2 \
-    --image=${IMG} \
-    ${TAGNAME:+--tag=${TAGNAME}} \
-    -- --enable-docs ${ARGS}
-
-after_success:
- - ccache -s
- # Upload coverage results for COVERAGE run
- - |
-  if test "$COVERAGE" = "t"; \
-     then coveralls-lcov flux*-coverage.info; \
-     bash <(curl -s https://codecov.io/bash); \
-  fi
- #  Deploy resulting docker image to Docker Hub with appropriate tag
- - |
-  if test -n "$TAGNAME"; then
-     echo "$DOCKER_PASSWORD" | \
-       docker login -u "$DOCKER_USERNAME" --password-stdin && \
-     docker push ${TAGNAME}
-     # If this is the bionic build, then also tag without image name:
-     if echo "$TAGNAME" | grep -q "bionic"; then
-       t="${DOCKERREPO}:${TRAVIS_TAG:-latest}"
-       docker tag "$TAGNAME" ${t} && \
-       docker push ${t}
-     fi
-  fi
-
-after_failure:
- - find . -name test-suite.log | xargs -i sh -c 'printf "===XXX {} XXX===";cat {}'
- - find . -name t[0-9]*.output | xargs -i sh -c 'printf "\033[31mFound {}\033[39m\n";cat {}'
- - find . -name *.broker.log | xargs -i sh -c 'printf "\033[31mFound {}\033[39m\n";cat {}'
- - find . -name *.asan.* | xargs -i sh -c 'printf "\033[31mFound {}\033[39m\n";cat {}'
- - src/test/backtrace-all.sh
- - grep -q 'configure. exit 1' config.log && cat config.log
-
-before_deploy:
-  # Get anchor (formatted properly) and base URI for latest tag in NEWS file
-  - export ANCHOR=$(sed -n '/^flux-core version/{s/\.//g; s/\s/-/gp;Q}' NEWS.md)
-  - export TAG_URI="https://github.com/${TRAVIS_REPO_SLUG}/blob/${TRAVIS_TAG}"
-  - export TARBALL=$(echo flux-core*.tar.gz)
-  - ls -l $TARBALL
-  - echo "Deploying tag ${TRAVIS_TAG} as $TARBALL"
-
-deploy:
-  provider: releases
-  skip_cleanup: true
-  file: $TARBALL
-  prerelease: true
-  body: "View [Release Notes](${TAG_URI}/NEWS.md#${ANCHOR}) for flux-core ${TRAVIS_TAG}"
-  api_key:
-    secure: I7ckZ7Ei9oLIe8WZ8OH3EgZz81IFCIekx+v/+g3sJa6q15URlfZhVVFtiUpsJRktHcb39AflWZiEIX+HdUZyXtuTt9IES1XBIKH7x/zUL0x6f1DZKAhBx9ktYzdO/M+SpmDUg6RYxcdjVmSHZ9u935TDo104U+dY0990ZSFrpco=
-  on:
-    # Only deploy from travis builder with GITHUB_RELEASES_DEPLOY set
-    condition: $GITHUB_RELEASES_DEPLOY = "t"
-    tags: true
-    repo: flux-framework/flux-core
diff --git a/.typos.toml b/.typos.toml
new file mode 100644
index 000000000000..f177081a81fe
--- /dev/null
+++ b/.typos.toml
@@ -0,0 +1,37 @@
+# do not check code copied into the project
+# do not check sha code, lots of hashing false positives
+# do not check testing data
+[files]
+extend-exclude = [
+  "config/*",
+  "src/common/libev/*",
+  "src/common/libccan/*",
+  "src/common/liblsd/*",
+  "src/common/libtomlc99/*",
+  "src/common/libczmqcontainers/*",
+  "src/bindings/python/flux/utils/parsedatetime/*",
+  "t/sharness.sh",
+  "src/common/libutil/sha1.c",
+  "src/common/libutil/test/sha1.c",
+  "src/common/libutil/test/blobref.c",
+  "src/common/libmissing/*.[ch]",
+  "t/hwloc-data/*",
+  "doc/test/spell.en.pws",
+  "t/resource/resource.eventlog.*",
+  "t/sharness",
+]
+
+[default.extend-words]
+# in hwloc output
+hsi = "hsi"
+# commonly used names/variables that aren't typos
+fo = "fo"
+ba = "ba"
+inout = "inout"
+trun = "trun"
+fullset = "fullset"
+mone = "mone"
+requestor = "requestor"
+requestors = "requestors"
+clen = "clen"
+WRONLY = "WRONLY"
diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json
new file mode 100644
index 000000000000..eef35b49d3b2
--- /dev/null
+++ b/.vscode/c_cpp_properties.json
@@ -0,0 +1,29 @@
+{
+    "configurations": [
+        {
+            "name": "Linux arm64",
+            "includePath": [
+                "${workspaceFolder}/**"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/bin/gcc",
+            "cStandard": "c99",
+            "cppStandard": "c++14",
+            "intelliSenseMode": "linux-gcc-arm64",
+            "compileCommands": "",
+            "mergeConfigurations": false
+        },
+        {
+            "name": "Linux amd64",
+            "includePath": [
+                "${workspaceFolder}/**"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/bin/gcc",
+            "cStandard": "c99",
+            "cppStandard": "c++14",
+            "intelliSenseMode": "linux-gcc-x64",
+        }
+    ],
+    "version": 4
+}
\ No newline at end of file
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
new file mode 100644
index 000000000000..515b018832c3
--- /dev/null
+++ b/.vscode/extensions.json
@@ -0,0 +1,8 @@
+{
+    "recommendations": [
+        "ms-vscode.cpptools",
+        "github.vscode-pull-request-github",
+        "sumneko.lua",
+        "ms-python.python"
+    ]
+}
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 000000000000..eb2d64a75b77
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,34 @@
+{
+    "python.defaultInterpreterPath": "python3",
+    "python.autoComplete.extraPaths": [
+        "./build/src/bindings/python",
+        "./src/bindings/python",
+        "./t/python/tap",
+    ],
+    "python.analysis.extraPaths": [
+        "./build/src/bindings/python",
+        "./src/bindings/python",
+        "./t/python/tap",
+    ],
+    "Lua.runtime.path": [
+        "?.lua",
+        "?/init.lua",
+        "${workspaceFolder}/src/bindings/lua/?.lua",
+        "${workspaceFolder}/src/bindings/lua/?/init.lua"
+    ],
+    "files.associations": {
+        "**/lua/**/*.t": "lua",
+        "*.t": "shellscript"
+    },
+    "C_Cpp.autoAddFileAssociations": false,
+    "Lua.completion.autoRequire": true,
+    "Lua.diagnostics.globals": [
+        "shell",
+        "plugin",
+        "task"
+    ],
+    "python.formatting.provider": "black",
+    "python.linting.flake8Enabled": true,
+    "python.linting.enabled": true,
+    "python.linting.mypyEnabled": true,
+}
\ No newline at end of file
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
new file mode 100644
index 000000000000..0508bf959cd7
--- /dev/null
+++ b/.vscode/tasks.json
@@ -0,0 +1,54 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "configure",
+            "type": "shell",
+            "command": "libtoolize --automake --copy && autoreconf --verbose --install && ./configure --prefix=$(pwd)/install",
+            "group": {
+                "kind": "build",
+                "isDefault": false
+            }
+        },
+        {
+            "label": "make",
+            "type": "shell",
+            "command": "env SHELL=/bin/sh make -j 8 all V=1",
+            "problemMatcher": [
+                "$gcc"
+            ],
+            "group": {
+                "kind": "build",
+                "isDefault": false
+            }
+        },
+        {
+            "label": "check",
+            "type": "shell",
+            "command": "env SHELL=/bin/sh make -j 8 check V=1 VERBOSE=1",
+            "problemMatcher": [
+                "$gcc",
+                {
+                    "owner": "automake",
+                    "fileLocation": [
+                        "relative",
+                        "${workspaceFolder}/t/"
+                    ],
+                    "pattern": [{
+                        "kind": "file",
+                        "regexp": "^([FE][^\\s:]*):\\s+([^\\s]+)\\s+(\\d+)\\s+(.*)$",
+                        "severity": 1,
+                        "file": 2,
+                        "message": 4
+                    }]
+                }
+            ],
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            }
+        }
+    ]
+}
\ No newline at end of file
diff --git a/AUTHORS b/AUTHORS
deleted file mode 100644
index aac4dcedd02c..000000000000
--- a/AUTHORS
+++ /dev/null
@@ -1,21 +0,0 @@
-Dong Ahn
-Jon Bringhurst
-Al Chu
-James Corbett
-Chris Dunlap
-Todd Gamblin
-Jim Garlick
-Mark Grondona
-Matthieu Hautreux
-Sam Heinz
-Stephen Herbein
-Don Lipari
-Andre Merzky
-Dan Milroy
-Chris Morrone
-Chris Moussa
-Tapasya Patki
-Suraj Prabhakaran
-Barry Rountree
-Tom Scogland
-Becky Springmeyer
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9310be85cf18..8fa92ab686f7 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -26,6 +26,6 @@ in the following RFCs from the [flux-framework/rfc][1] project:
     to Flux projects.
 
 [1]: https://github.com/flux-framework/rfc
-[2]: https://github.com/flux-framework/rfc/blob/master/spec_1.adoc
-[3]: https://github.com/flux-framework/rfc/blob/master/spec_2.adoc
-[4]: https://github.com/flux-framework/rfc/blob/master/spec_7.adoc
+[2]: https://flux-framework.rtfd.io/projects/flux-rfc/en/latest/spec_1.html
+[3]: https://flux-framework.rtfd.io/projects/flux-rfc/en/latest/spec_2.html
+[4]: https://flux-framework.rtfd.io/projects/flux-rfc/en/latest/spec_7.html
diff --git a/COPYING b/COPYING
deleted file mode 100644
index 0a041280bd00..000000000000
--- a/COPYING
+++ /dev/null
@@ -1,165 +0,0 @@
-                   GNU LESSER GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-
-  This version of the GNU Lesser General Public License incorporates
-the terms and conditions of version 3 of the GNU General Public
-License, supplemented by the additional permissions listed below.
-
-  0. Additional Definitions.
-
-  As used herein, "this License" refers to version 3 of the GNU Lesser
-General Public License, and the "GNU GPL" refers to version 3 of the GNU
-General Public License.
-
-  "The Library" refers to a covered work governed by this License,
-other than an Application or a Combined Work as defined below.
-
-  An "Application" is any work that makes use of an interface provided
-by the Library, but which is not otherwise based on the Library.
-Defining a subclass of a class defined by the Library is deemed a mode
-of using an interface provided by the Library.
-
-  A "Combined Work" is a work produced by combining or linking an
-Application with the Library.  The particular version of the Library
-with which the Combined Work was made is also called the "Linked
-Version".
-
-  The "Minimal Corresponding Source" for a Combined Work means the
-Corresponding Source for the Combined Work, excluding any source code
-for portions of the Combined Work that, considered in isolation, are
-based on the Application, and not on the Linked Version.
-
-  The "Corresponding Application Code" for a Combined Work means the
-object code and/or source code for the Application, including any data
-and utility programs needed for reproducing the Combined Work from the
-Application, but excluding the System Libraries of the Combined Work.
-
-  1. Exception to Section 3 of the GNU GPL.
-
-  You may convey a covered work under sections 3 and 4 of this License
-without being bound by section 3 of the GNU GPL.
-
-  2. Conveying Modified Versions.
-
-  If you modify a copy of the Library, and, in your modifications, a
-facility refers to a function or data to be supplied by an Application
-that uses the facility (other than as an argument passed when the
-facility is invoked), then you may convey a copy of the modified
-version:
-
-   a) under this License, provided that you make a good faith effort to
-   ensure that, in the event an Application does not supply the
-   function or data, the facility still operates, and performs
-   whatever part of its purpose remains meaningful, or
-
-   b) under the GNU GPL, with none of the additional permissions of
-   this License applicable to that copy.
-
-  3. Object Code Incorporating Material from Library Header Files.
-
-  The object code form of an Application may incorporate material from
-a header file that is part of the Library.  You may convey such object
-code under terms of your choice, provided that, if the incorporated
-material is not limited to numerical parameters, data structure
-layouts and accessors, or small macros, inline functions and templates
-(ten or fewer lines in length), you do both of the following:
-
-   a) Give prominent notice with each copy of the object code that the
-   Library is used in it and that the Library and its use are
-   covered by this License.
-
-   b) Accompany the object code with a copy of the GNU GPL and this license
-   document.
-
-  4. Combined Works.
-
-  You may convey a Combined Work under terms of your choice that,
-taken together, effectively do not restrict modification of the
-portions of the Library contained in the Combined Work and reverse
-engineering for debugging such modifications, if you also do each of
-the following:
-
-   a) Give prominent notice with each copy of the Combined Work that
-   the Library is used in it and that the Library and its use are
-   covered by this License.
-
-   b) Accompany the Combined Work with a copy of the GNU GPL and this license
-   document.
-
-   c) For a Combined Work that displays copyright notices during
-   execution, include the copyright notice for the Library among
-   these notices, as well as a reference directing the user to the
-   copies of the GNU GPL and this license document.
-
-   d) Do one of the following:
-
-       0) Convey the Minimal Corresponding Source under the terms of this
-       License, and the Corresponding Application Code in a form
-       suitable for, and under terms that permit, the user to
-       recombine or relink the Application with a modified version of
-       the Linked Version to produce a modified Combined Work, in the
-       manner specified by section 6 of the GNU GPL for conveying
-       Corresponding Source.
-
-       1) Use a suitable shared library mechanism for linking with the
-       Library.  A suitable mechanism is one that (a) uses at run time
-       a copy of the Library already present on the user's computer
-       system, and (b) will operate properly with a modified version
-       of the Library that is interface-compatible with the Linked
-       Version.
-
-   e) Provide Installation Information, but only if you would otherwise
-   be required to provide such information under section 6 of the
-   GNU GPL, and only to the extent that such information is
-   necessary to install and execute a modified version of the
-   Combined Work produced by recombining or relinking the
-   Application with a modified version of the Linked Version. (If
-   you use option 4d0, the Installation Information must accompany
-   the Minimal Corresponding Source and Corresponding Application
-   Code. If you use option 4d1, you must provide the Installation
-   Information in the manner specified by section 6 of the GNU GPL
-   for conveying Corresponding Source.)
-
-  5. Combined Libraries.
-
-  You may place library facilities that are a work based on the
-Library side by side in a single library together with other library
-facilities that are not Applications and are not covered by this
-License, and convey such a combined library under terms of your
-choice, if you do both of the following:
-
-   a) Accompany the combined library with a copy of the same work based
-   on the Library, uncombined with any other library facilities,
-   conveyed under the terms of this License.
-
-   b) Give prominent notice with the combined library that part of it
-   is a work based on the Library, and explaining where to find the
-   accompanying uncombined form of the same work.
-
-  6. Revised Versions of the GNU Lesser General Public License.
-
-  The Free Software Foundation may publish revised and/or new versions
-of the GNU Lesser General Public License from time to time. Such new
-versions will be similar in spirit to the present version, but may
-differ in detail to address new problems or concerns.
-
-  Each version is given a distinguishing version number. If the
-Library as you received it specifies that a certain numbered version
-of the GNU Lesser General Public License "or any later version"
-applies to it, you have the option of following the terms and
-conditions either of that published version or of any later version
-published by the Free Software Foundation. If the Library as you
-received it does not specify a version number of the GNU Lesser
-General Public License, you may choose any version of the GNU Lesser
-General Public License ever published by the Free Software Foundation.
-
-  If the Library as you received it specifies that a proxy can decide
-whether future versions of the GNU Lesser General Public License shall
-apply, that proxy's public statement of acceptance of any version is
-permanent authorization for you to choose that version for the
-Library.
diff --git a/INSTALL b/INSTALL
deleted file mode 100644
index 7d1c323beae7..000000000000
--- a/INSTALL
+++ /dev/null
@@ -1,365 +0,0 @@
-Installation Instructions
-*************************
-
-Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005,
-2006, 2007, 2008, 2009 Free Software Foundation, Inc.
-
-   Copying and distribution of this file, with or without modification,
-are permitted in any medium without royalty provided the copyright
-notice and this notice are preserved.  This file is offered as-is,
-without warranty of any kind.
-
-Basic Installation
-==================
-
-   Briefly, the shell commands `./configure; make; make install' should
-configure, build, and install this package.  The following
-more-detailed instructions are generic; see the `README' file for
-instructions specific to this package.  Some packages provide this
-`INSTALL' file but do not implement all of the features documented
-below.  The lack of an optional feature in a given package is not
-necessarily a bug.  More recommendations for GNU packages can be found
-in *note Makefile Conventions: (standards)Makefile Conventions.
-
-   The `configure' shell script attempts to guess correct values for
-various system-dependent variables used during compilation.  It uses
-those values to create a `Makefile' in each directory of the package.
-It may also create one or more `.h' files containing system-dependent
-definitions.  Finally, it creates a shell script `config.status' that
-you can run in the future to recreate the current configuration, and a
-file `config.log' containing compiler output (useful mainly for
-debugging `configure').
-
-   It can also use an optional file (typically called `config.cache'
-and enabled with `--cache-file=config.cache' or simply `-C') that saves
-the results of its tests to speed up reconfiguring.  Caching is
-disabled by default to prevent problems with accidental use of stale
-cache files.
-
-   If you need to do unusual things to compile the package, please try
-to figure out how `configure' could check whether to do them, and mail
-diffs or instructions to the address given in the `README' so they can
-be considered for the next release.  If you are using the cache, and at
-some point `config.cache' contains results you don't want to keep, you
-may remove or edit it.
-
-   The file `configure.ac' (or `configure.in') is used to create
-`configure' by a program called `autoconf'.  You need `configure.ac' if
-you want to change it or regenerate `configure' using a newer version
-of `autoconf'.
-
-   The simplest way to compile this package is:
-
-  1. `cd' to the directory containing the package's source code and type
-     `./configure' to configure the package for your system.
-
-     Running `configure' might take a while.  While running, it prints
-     some messages telling which features it is checking for.
-
-  2. Type `make' to compile the package.
-
-  3. Optionally, type `make check' to run any self-tests that come with
-     the package, generally using the just-built uninstalled binaries.
-
-  4. Type `make install' to install the programs and any data files and
-     documentation.  When installing into a prefix owned by root, it is
-     recommended that the package be configured and built as a regular
-     user, and only the `make install' phase executed with root
-     privileges.
-
-  5. Optionally, type `make installcheck' to repeat any self-tests, but
-     this time using the binaries in their final installed location.
-     This target does not install anything.  Running this target as a
-     regular user, particularly if the prior `make install' required
-     root privileges, verifies that the installation completed
-     correctly.
-
-  6. You can remove the program binaries and object files from the
-     source code directory by typing `make clean'.  To also remove the
-     files that `configure' created (so you can compile the package for
-     a different kind of computer), type `make distclean'.  There is
-     also a `make maintainer-clean' target, but that is intended mainly
-     for the package's developers.  If you use it, you may have to get
-     all sorts of other programs in order to regenerate files that came
-     with the distribution.
-
-  7. Often, you can also type `make uninstall' to remove the installed
-     files again.  In practice, not all packages have tested that
-     uninstallation works correctly, even though it is required by the
-     GNU Coding Standards.
-
-  8. Some packages, particularly those that use Automake, provide `make
-     distcheck', which can by used by developers to test that all other
-     targets like `make install' and `make uninstall' work correctly.
-     This target is generally not run by end users.
-
-Compilers and Options
-=====================
-
-   Some systems require unusual options for compilation or linking that
-the `configure' script does not know about.  Run `./configure --help'
-for details on some of the pertinent environment variables.
-
-   You can give `configure' initial values for configuration parameters
-by setting variables in the command line or in the environment.  Here
-is an example:
-
-     ./configure CC=c99 CFLAGS=-g LIBS=-lposix
-
-   *Note Defining Variables::, for more details.
-
-Compiling For Multiple Architectures
-====================================
-
-   You can compile the package for more than one kind of computer at the
-same time, by placing the object files for each architecture in their
-own directory.  To do this, you can use GNU `make'.  `cd' to the
-directory where you want the object files and executables to go and run
-the `configure' script.  `configure' automatically checks for the
-source code in the directory that `configure' is in and in `..'.  This
-is known as a "VPATH" build.
-
-   With a non-GNU `make', it is safer to compile the package for one
-architecture at a time in the source code directory.  After you have
-installed the package for one architecture, use `make distclean' before
-reconfiguring for another architecture.
-
-   On MacOS X 10.5 and later systems, you can create libraries and
-executables that work on multiple system types--known as "fat" or
-"universal" binaries--by specifying multiple `-arch' options to the
-compiler but only a single `-arch' option to the preprocessor.  Like
-this:
-
-     ./configure CC="gcc -arch i386 -arch x86_64 -arch ppc -arch ppc64" \
-                 CXX="g++ -arch i386 -arch x86_64 -arch ppc -arch ppc64" \
-                 CPP="gcc -E" CXXCPP="g++ -E"
-
-   This is not guaranteed to produce working output in all cases, you
-may have to build one architecture at a time and combine the results
-using the `lipo' tool if you have problems.
-
-Installation Names
-==================
-
-   By default, `make install' installs the package's commands under
-`/usr/local/bin', include files under `/usr/local/include', etc.  You
-can specify an installation prefix other than `/usr/local' by giving
-`configure' the option `--prefix=PREFIX', where PREFIX must be an
-absolute file name.
-
-   You can specify separate installation prefixes for
-architecture-specific files and architecture-independent files.  If you
-pass the option `--exec-prefix=PREFIX' to `configure', the package uses
-PREFIX as the prefix for installing programs and libraries.
-Documentation and other data files still use the regular prefix.
-
-   In addition, if you use an unusual directory layout you can give
-options like `--bindir=DIR' to specify different values for particular
-kinds of files.  Run `configure --help' for a list of the directories
-you can set and what kinds of files go in them.  In general, the
-default for these options is expressed in terms of `${prefix}', so that
-specifying just `--prefix' will affect all of the other directory
-specifications that were not explicitly provided.
-
-   The most portable way to affect installation locations is to pass the
-correct locations to `configure'; however, many packages provide one or
-both of the following shortcuts of passing variable assignments to the
-`make install' command line to change installation locations without
-having to reconfigure or recompile.
-
-   The first method involves providing an override variable for each
-affected directory.  For example, `make install
-prefix=/alternate/directory' will choose an alternate location for all
-directory configuration variables that were expressed in terms of
-`${prefix}'.  Any directories that were specified during `configure',
-but not in terms of `${prefix}', must each be overridden at install
-time for the entire installation to be relocated.  The approach of
-makefile variable overrides for each directory variable is required by
-the GNU Coding Standards, and ideally causes no recompilation.
-However, some platforms have known limitations with the semantics of
-shared libraries that end up requiring recompilation when using this
-method, particularly noticeable in packages that use GNU Libtool.
-
-   The second method involves providing the `DESTDIR' variable.  For
-example, `make install DESTDIR=/alternate/directory' will prepend
-`/alternate/directory' before all installation names.  The approach of
-`DESTDIR' overrides is not required by the GNU Coding Standards, and
-does not work on platforms that have drive letters.  On the other hand,
-it does better at avoiding recompilation issues, and works well even
-when some directory options were not specified in terms of `${prefix}'
-at `configure' time.
-
-Optional Features
-=================
-
-   If the package supports it, you can cause programs to be installed
-with an extra prefix or suffix on their names by giving `configure' the
-option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'.
-
-   Some packages pay attention to `--enable-FEATURE' options to
-`configure', where FEATURE indicates an optional part of the package.
-They may also pay attention to `--with-PACKAGE' options, where PACKAGE
-is something like `gnu-as' or `x' (for the X Window System).  The
-`README' should mention any `--enable-' and `--with-' options that the
-package recognizes.
-
-   For packages that use the X Window System, `configure' can usually
-find the X include and library files automatically, but if it doesn't,
-you can use the `configure' options `--x-includes=DIR' and
-`--x-libraries=DIR' to specify their locations.
-
-   Some packages offer the ability to configure how verbose the
-execution of `make' will be.  For these packages, running `./configure
---enable-silent-rules' sets the default to minimal output, which can be
-overridden with `make V=1'; while running `./configure
---disable-silent-rules' sets the default to verbose, which can be
-overridden with `make V=0'.
-
-Particular systems
-==================
-
-   On HP-UX, the default C compiler is not ANSI C compatible.  If GNU
-CC is not installed, it is recommended to use the following options in
-order to use an ANSI C compiler:
-
-     ./configure CC="cc -Ae -D_XOPEN_SOURCE=500"
-
-and if that doesn't work, install pre-built binaries of GCC for HP-UX.
-
-   On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot
-parse its `<wchar.h>' header file.  The option `-nodtk' can be used as
-a workaround.  If GNU CC is not installed, it is therefore recommended
-to try
-
-     ./configure CC="cc"
-
-and if that doesn't work, try
-
-     ./configure CC="cc -nodtk"
-
-   On Solaris, don't put `/usr/ucb' early in your `PATH'.  This
-directory contains several dysfunctional programs; working variants of
-these programs are available in `/usr/bin'.  So, if you need `/usr/ucb'
-in your `PATH', put it _after_ `/usr/bin'.
-
-   On Haiku, software installed for all users goes in `/boot/common',
-not `/usr/local'.  It is recommended to use the following options:
-
-     ./configure --prefix=/boot/common
-
-Specifying the System Type
-==========================
-
-   There may be some features `configure' cannot figure out
-automatically, but needs to determine by the type of machine the package
-will run on.  Usually, assuming the package is built to be run on the
-_same_ architectures, `configure' can figure that out, but if it prints
-a message saying it cannot guess the machine type, give it the
-`--build=TYPE' option.  TYPE can either be a short name for the system
-type, such as `sun4', or a canonical name which has the form:
-
-     CPU-COMPANY-SYSTEM
-
-where SYSTEM can have one of these forms:
-
-     OS
-     KERNEL-OS
-
-   See the file `config.sub' for the possible values of each field.  If
-`config.sub' isn't included in this package, then this package doesn't
-need to know the machine type.
-
-   If you are _building_ compiler tools for cross-compiling, you should
-use the option `--target=TYPE' to select the type of system they will
-produce code for.
-
-   If you want to _use_ a cross compiler, that generates code for a
-platform different from the build platform, you should specify the
-"host" platform (i.e., that on which the generated programs will
-eventually be run) with `--host=TYPE'.
-
-Sharing Defaults
-================
-
-   If you want to set default values for `configure' scripts to share,
-you can create a site shell script called `config.site' that gives
-default values for variables like `CC', `cache_file', and `prefix'.
-`configure' looks for `PREFIX/share/config.site' if it exists, then
-`PREFIX/etc/config.site' if it exists.  Or, you can set the
-`CONFIG_SITE' environment variable to the location of the site script.
-A warning: not all `configure' scripts look for a site script.
-
-Defining Variables
-==================
-
-   Variables not defined in a site shell script can be set in the
-environment passed to `configure'.  However, some packages may run
-configure again during the build, and the customized values of these
-variables may be lost.  In order to avoid this problem, you should set
-them in the `configure' command line, using `VAR=value'.  For example:
-
-     ./configure CC=/usr/local2/bin/gcc
-
-causes the specified `gcc' to be used as the C compiler (unless it is
-overridden in the site shell script).
-
-Unfortunately, this technique does not work for `CONFIG_SHELL' due to
-an Autoconf bug.  Until the bug is fixed you can use this workaround:
-
-     CONFIG_SHELL=/bin/bash /bin/bash ./configure CONFIG_SHELL=/bin/bash
-
-`configure' Invocation
-======================
-
-   `configure' recognizes the following options to control how it
-operates.
-
-`--help'
-`-h'
-     Print a summary of all of the options to `configure', and exit.
-
-`--help=short'
-`--help=recursive'
-     Print a summary of the options unique to this package's
-     `configure', and exit.  The `short' variant lists options used
-     only in the top level, while the `recursive' variant lists options
-     also present in any nested packages.
-
-`--version'
-`-V'
-     Print the version of Autoconf used to generate the `configure'
-     script, and exit.
-
-`--cache-file=FILE'
-     Enable the cache: use and save the results of the tests in FILE,
-     traditionally `config.cache'.  FILE defaults to `/dev/null' to
-     disable caching.
-
-`--config-cache'
-`-C'
-     Alias for `--cache-file=config.cache'.
-
-`--quiet'
-`--silent'
-`-q'
-     Do not print messages saying which checks are being made.  To
-     suppress all normal output, redirect it to `/dev/null' (any error
-     messages will still be shown).
-
-`--srcdir=DIR'
-     Look for the package's source code in directory DIR.  Usually
-     `configure' can determine that directory automatically.
-
-`--prefix=DIR'
-     Use DIR as the installation prefix.  *note Installation Names::
-     for more details, including other options available for fine-tuning
-     the installation locations.
-
-`--no-create'
-`-n'
-     Run the configure checks, but stop before creating any output
-     files.
-
-`configure' also accepts some other, not widely useful, options.  Run
-`configure --help' for more details.
-
diff --git a/LICENSE b/LICENSE
deleted file mode 120000
index d24842f3cdcf..000000000000
--- a/LICENSE
+++ /dev/null
@@ -1 +0,0 @@
-COPYING
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 000000000000..0a041280bd00
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/Makefile.am b/Makefile.am
index 0233470630a6..361e819f8e7d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -6,7 +6,15 @@ EXTRA_DIST = \
 	config/tap-driver.sh \
 	config/tap-driver.py \
 	NOTICE.LLNS \
-	README.md
+	LICENSE \
+	README.md \
+	vscode.md \
+	NEWS.md \
+	scripts/requirements-dev.txt \
+	scripts/install-deps-deb.sh \
+	scripts/install-deps-rpm.sh \
+	scripts/install-deps-macos.sh \
+	scripts/configure-macos.sh
 
 ACLOCAL_AMFLAGS = -I config
 
@@ -23,11 +31,17 @@ CODE_COVERAGE_IGNORE_PATTERN = \
 	"/usr/lib*" \
 	"*/bindings/python/*" \
 	"*/common/liblsd/*" \
-	"*/common/libutil/sds.*" \
 	"*/common/liboptparse/getopt*" \
 	"*/common/libtestutil/*" \
-	"*/common/libyuarel/*"
+	"*/common/libyuarel/*" \
+	"*/common/libczmqcontainers/*" \
+	"*/common/libccan/*" \
+	"*/common/libmissing/*"
 
+# ignore lcov errors to avoid merge mismatch issue since lcov < 2 doesn't offer
+# an option to just ignore this error, we use this env var to ignore all, see:
+# https://github.com/flux-framework/flux-core/issues/6078
+export GCOV_ERROR_FILE=/dev/null
 CODE_COVERAGE_LCOV_OPTIONS =
 @CODE_COVERAGE_RULES@
 
@@ -41,3 +55,15 @@ CODE_COVERAGE_LCOV_OPTIONS =
 # file's SUBDIRS, ensures that "all" is built before any of the
 # recursive checks.
 check-local: all
+
+check-prep: all
+	cd src && $(MAKE) check
+	cd doc && $(MAKE) check
+	cd t && $(MAKE) check-prep
+
+export DEB_BUILD_OPTIONS ?= nocheck terse
+deb: debian scripts/debbuild.sh
+	+@$(top_srcdir)/scripts/debbuild.sh $(abs_top_srcdir)
+
+clean-local:
+	@rm -rf debbuild
diff --git a/NEWS b/NEWS
deleted file mode 100644
index 968af700b073..000000000000
--- a/NEWS
+++ /dev/null
@@ -1 +0,0 @@
-Full Release Notes for flux-core can be found in the NEWS.md file.
diff --git a/NEWS.md b/NEWS.md
index 7b43ae2095a1..f6e9c03cc364 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,2814 @@
+flux-core version 0.69.0 - 2024-12-03
+-------------------------------------
+
+## New Features
+ * add flux module stats `--rusage=[self|children|thread]` optional argument
+   (#6471)
+ * add `-S, --setattr` and `-c, --config-path` options directly to `flux
+   start` (#6452)
+ * job-ingest: improve cleanup and stats output (#6438)
+ * libflux: add `flux_watcher_is_active()` (#6436)
+ * indicate held jobs in the` INFO` column of `flux jobs` output (#6430)
+ * recursively update instead of replacing tables when loading Flux
+   configuration (#6424)
+ * convert ISO timestamp output from UTC to local time + offset in `flux
+   dmesg` and eventlog commands (#6423)
+ * support flux uri --wait JOBID (#6443)
+ * skip "empty" lines of output in `flux resource list` with `--skip-empty`
+   or `--include` (#6460)
+
+## Fixes
+ * libfileref: fix segfault for files >2G (#6462)
+ * fix macos portability issues (#6454, #6468)
+ * fix multiple issues in the `flux job attach` statusline (#6442)
+ * librlist: avoid unnecessary hwloc dependencies (#6450)
+ * python: call shutdown() on executor in job validator (#6435)
+ * increase default prolog kill-timeout from 10s to 1m (#6431)
+ * job-manager/history: optimize list insertion (#6422)
+
+## Cleanup
+ * kvs-watch: misc cleanup (#6458)
+ * build: misc cleanup (#6451)
+ * job-manager: clean up queue code (#6448)
+ * remove `flux-perilog-run` (#6447)
+
+## CI/Testsuite/Documentation
+ * doc: improve `--include` documentation in flux-resource(1) (#6459)
+ * doc: improve housekeeping documentation (#6425)
+ * doc: launch jobs with systemd in the admin guide (#6427)
+
+
+flux-core version 0.68.0 - 2024-11-06
+-------------------------------------
+
+This release requires flux-security >= 0.13.0 as the IMP signal handling
+strategy has changed per RFC 15.
+
+## New Features
+ * update multi-user signaling to track flux-security 0.13.0 IMP changes
+   (#6408)
+ * add cleanup timeout for systemctl stop flux on rank 0 (#6397)
+ * flux-exec: use subprocess credits to avoid overflowing stdin buffers
+   (#6370)
+ * libsubprocess: support flow control on writes via credits (#6353)
+ * python: usability improvements for `JournalConsumer` class (#6390)
+ * python: add `flux.job.JournalConsumer` class with a simplified interface
+   for job manager journal consumers (#6386)
+ * support `sort:` prefix for format strings and `--sort` option to `flux
+   jobs` (#6380)
+ * flux-housekeeping: add `-i, --include=TARGETS` option to `flux
+   housekeeping list` (#6382)
+ * show response result in message traces (#6359)
+ * libsubprocess: invert SETPGRP flag logic (#6082)
+ * add --full option to display payloads in message tracing programs (#6347)
+
+
+## Fixes
+ * libsubprocess: close extra file descriptors (#6416)
+ * resolve perilog plugin issue that lets a job start after prolog timeout
+   when cancellation fails (#6412)
+ * frobnicator: allow queue-specific defaults to override global defaults
+   (#6403)
+ * sdexec: set KillMode=process SendSIGKILL=no for multi-user jobs (#6402)
+ * broker: detect mismatched bootstrap.hosts configuration (#6393)
+ * libsubprocess: take reference on callbacks (#6384)
+ * python: cleanup, fixes, and unit tests for flux.util.OutputFormat (#6374)
+ * libsubprocess: misc fixes (#6379)
+ * sched-simple: improve unsupported resource type exception (#6372)
+ * libsubprocess: ensure bulk-exec output is terminated (#6368)
+ * libsubprocess: check bufsize is > 0 (#6365)
+ * kvs: fix whitespace issues (#6356)
+ * allow project to be built with NDEBUG (#6355)
+ * systemd: make scripts fail if systemctl start does (#6346)
+ * improve policy/queues config error messages (#6339)
+ * make flux resource drain -o long reason expandable (#6338)
+
+## Cleanup
+ * job-archive: remove module (#6378)
+
+## CI/Testsuite/Documentation
+ * broker/test: avoid race in zeromq cleanup (#6405)
+ * docker: add missing tag of flux-core el8 image (#6401)
+ * doc: add debugging notes (#6369)
+ * doc: update link to flux-accounting guide (#6373)
+ * flux-jobs(1): document unlimited --count value (#6364)
+ * testsuite: add --raw-response opt to rpc test program (#6342)
+ * testsuite: improve test_must_fail_or_be_killed function (#6343)
+ * docs: rfc flux-config-bootstrap diagram (#6411)
+
+
+flux-core version 0.67.0 - 2024-10-01
+-------------------------------------
+
+## New Features
+ * include hostnames in flux resource undrain error message (#6335)
+ * libsubprocess: increase output watcher priority (#6317)
+ * libflux: support modifying watcher priority via
+   `flux_watcher_set_priority()` (#6316)
+ * add `--force` option to `flux resource undrain` (#6312)
+ * autoconf: support python 3.12 (#6303)
+ * flux-bulksubmit: support `{}`  in more options like `--cwd=`, `--signal=`,
+   `--taskmap=`, etc. (#6299)
+ * add flux-lptest (#6301)
+ * broker: use enclosing instance tbon.interface-hint unless overridden
+   (#6283)
+ * shell/oom: log more detail when tasks are killed (#6289)
+ * support expandable width output formats and use them in `flux resource
+   list` to avoid truncation of queue field (#6284)
+ * python: update Flux handle location in validator plugin API (#6282)
+ * broker: call `PMI_Abort()` if something goes wrong during PMI bootstrap
+   (#6279)
+ * add tbon.interface-hint broker attribute / configuration key with CIDR
+   network support (#6277)
+ * support configuration of require-instance and other job
+   validator/frobnicator plugins in broker config TOML (#6305)
+ * validator: allow configurable minimum job size in the require-instance
+   validator plugin (#6258)
+ * display nodes in housekeeping in `flux resource status` (#6263)
+ * shell: output stdio output size warning (#6274)
+ * python: add `conf_get()` convenience method to `flux.Flux` handle (#6267)
+ * limit output to 1G in single user instances (#6268)
+
+## Fixes
+ * systemd: improve housekeeping drain message (#6334)
+ * perilog: never send SIGKILL to prolog/epilog, drain active nodes after
+   kill-timeout instead (#6330)
+ * perilog: fix kill of prolog/epilog when using IMP (#6324)
+ * perilog: fix `FLUX_JOB_USERID` in epilog after canceled prolog (#6320)
+ * flux job info: improve error messages (#6331)
+ * libsubprocess: fix bulk-exec reporting of active ranks (#6326)
+ * libsubprocess: do not spin on large lines (#6281)
+ * configure: add check for valid version (#6276)
+ * etc: minor improvements for bash completions (#6332)
+ * perilog: ensure default prolog timeout matches documentation (#6270)
+ * hostlist: remove allocations in `hostrange_find` (#6259)
+
+## CI/Testsuite/Documentation
+ * libsubprocess: add extra documentation (#6307)
+ * format: fix clang-format file (#6280)
+ * doc: python: fix JobList default documentation (#6309)
+ * doc: add dash to flux-job(1) manpage (#6313)
+ * doc: add warning about Python argparse in flux-jobs(1) (#6285)
+ * doc: fix typo in `FLUX_IPADDR_INTERFACE` entry in `flux-environment(7)`
+   (#6271)
+ * doc: update admin guide for systemd perilog (#6261)
+ * doc: add warning about stdio paths in submission cli man pages (#6333)
+
+
+flux-core version 0.66.0 - 2024-09-03
+-------------------------------------
+
+## New Features
+ * support display of `allocated` nodes in `flux resource status` (#6253)
+ * support `resource.scheduling` in config TOML to amend configured R with
+   a scheduling key (#6252)
+ * perilog: support direct execution of prolog/epilog on job ranks for
+   better performance (#6235)
+ * flux-hostlist: change default source from `local` to `stdin` (#6246)
+ * support short option `-S` for `--setattr=` in job submission commands
+   (#6238)
+ * content-sqlite: set sqlite pragmas with flux-config (#6222)
+ * content-sqlite: add sqlite config to module stats (#6221)
+ * content: allow content.flush to return errors (#6175)
+ * flux-jobs: add `-i, --include=HOSTS|RANKS` option (#6209)
+ * add flux module trace (#6203, #6206)
+ * include offline nodes in flux overlay errors output (#6201)
+
+## Fixes
+ * run epilog even if job prolog is failed or is canceled (#6249)
+ * progress: ensure we don't access outside style (#6250)
+ * housekeeping: deprecate use-systemd-config option (#6241)
+ * Fix handling of `--` used to separate options from command in `flux
+   submit`, `run`, and `batch` (#6234)
+ * libsubprocess: minor improvements to bulk-exec (#6233)
+ * update feasibility RPCs to conform to RFC 27 (#6223)
+ * libev: fix memory fence on i386 under valgrind (#6224)
+ * job-manager: skip housekeeping for alloc-bypass jobs (#6219)
+ * cron: use `posix_spawn()` for cron tasks when possible (#6214)
+ * kvs: correct code logic about what is an append (#6210)
+ * avoid idset leak when nodes leave the broker.online group (#6198)
+ * kvs: correct transaction-merge option parsing (#6204)
+
+## Cleanup
+ * content-sqlite: misc cleanup (#6220)
+
+## CI/Testsuite/Documentation
+ * doc: add dependency example (#6226)
+ * mergify: use updated keys before deprecation bites (#6225)
+ * testsuite: fix racy flux job attach test (#6212)
+ * matrix: remove fedora40 arm again (#6200)
+
+
+flux-core version 0.65.0 - 2024-08-06
+-------------------------------------
+
+## New Features
+ * job-manager: add final flag to sched.free (#6197)
+ * add `-B, --bank=` option to submission commands (#6195)
+ * add flux overlay trace (#6174)
+ * enhance `flux queue list` with `-q, --queue=` option and abillity to
+   report queue enabled/started status (#6185)
+ * support multiple queues in `flux jobs`, `pgrep`, and `pkill` `-q,
+   --queue` option (#6149)
+ * accept `-q` for `--queue` in `flux-jobs`, `pgrep`, and `pkill`
+   (#6142)
+ * resource: mark torpid nodes down for scheduling instead of draining them
+   (#6131)
+ * job-exec: add timeout for job start barrier (#6140)
+ * job-exec: fix per-job `active_ranks` module stats (#6138)
+ * job-exec: fix job refcount leaks (#6134)
+ * drain nodes with unkillable tasks after configurable retries (#6101)
+ * move bad KVS job entries to a lost+found directory (#6139)
+ * add error text to down nodes in flux overlay status (#6111)
+ * job-manager: ignore bad KVS eventlogs (#6129)
+ * add message queue counts to flux module list (#6120)
+ * flux-job: add `-F, --follow` option to `flux job eventlog` (#6102)
+ * wait for job `clean` event by default in `flux run`  and `flux job attach`
+   (#6065)
+ * systemd: add prolog/epilog service units (#6040)
+ * broker: work around problem with launch by some versions of mpiexec.hydra
+   (#6081)
+ * reject jobs submitted as user root in a multi-user instance (#6194)
+
+## Fixes
+ * improve housekeeping logging and list management (#6191)
+ * add configure time check for zeromq crypto and improve broker error
+   handling at cert creation (#6189)
+ * resource: error message cleanup, add test for invalid exclude (#6186)
+ * flux-resource: fix missing queue in `flux resource list` output for
+   states with no nodes (#6180)
+ * flux-resource: suppress empty lines in output (#6096)
+ * kvs: return ENOSYS on unfinished requests (#6049)
+ * cmd/startlog: initialize enum to non zero (#6170)
+ * address RSS growth when there are excessive annotation events (#6115)
+ * resource: cache resource status information to reduce broker load (#6105)
+ * flux-start: accept --recovery with --test-size (#6108)
+ * doc: update flux admin guide URL (#6163)
+ * doc: update flux-shell(1) manpage (#6094)
+ * doc: add overlay network to overview section (#6092)
+
+## Cleanup
+ * kvs: minor cleanup (#6172)
+ * job-manager: silence housekeeping debug logs (#6113)
+ * kvs: replace list of message with msglist (#6047)
+
+## CI/Testsuite
+ * matrix/actions: add per-target timeout-minutes (#6192)
+ * t: add initial ENOSPC tests (#6127)
+ * matrix: avoid overwriting env elements (#6190)
+ * docker: Demote fedora40 to `x86_64` only (#6188)
+ * ci: ignore merge errors and increase parallelism (#6183)
+ * testsuite: fix racy test in t2406-job-exec-cleanup.t (#6187)
+ * add support for new wrap options in tests and heredoc support for sharness
+   (#6150)
+ * actions: increase ci timeout from 90 to 120 minutes (#6181)
+ * matrix: fix matrix generation for arm64 and remove redundant fedora
+   (#6156)
+ * Update containers: el9, fedora40, noble, add arm64 for fedora40, el9,
+   noble (#6128)
+ * t: add job environment to tests (#6106)
+
+
+flux-core version 0.64.0 - 2024-07-02
+-------------------------------------
+
+## Experimental Features
+ * job-manager: add support for housekeeping scripts with partial release
+   of resources (#5818)
+
+## New Features
+ * sdexec: add stats-get RPC handler (#6071)
+ * job-exec: add `active_ranks` per-job stats (#6070)
+ * job-list: add support for `ranks` constraint (#6048)
+ * job-list: support "hostlist" constraint to allow jobs to be filtered by
+   nodes (#5656)
+ * add environment variable to avoid `RTLD_DEEPBIND` when loading modules
+   (#6063)
+
+## Fixes
+ * job-manager: improve sched.alloc request tracking (#6076)
+ * flux-job: fix prolog refcount in attach status message #6074 
+ * job-exec: fix `active_shells` count in per-job stats (#6070)
+ * job-manager/prioritize: update priority before sending (#6062)
+ * job-exec: do not streq() an unbuffered string (#6058)
+ * python: Update default Jobspec setattr() path (#6053)
+ * shell: enhance exit-timeout and exit-on-error exceptions with hostname
+   (#6056)
+ * libidset: improve efficiency of idset intersection (#6045)
+ * etc: update bash completions (#6039, #6060)
+ * flux-python: add posix-compatible shebang wrapper (#6041)
+ * libsubprocess: use matchtag instead of pid for flux_subprocess_write()
+   (#6013)
+ * sdexec: fix use after free during teardown (#6037)
+ * broker: avoid 60s delay on follower shutdown (#6034)
+
+## CI/Testsuite
+ * t: suppress hwloc 2.10 memleaks (#6061)
+ * ci: convert `system` test container to run under `podman` for better
+   `systemd` support (#6043)
+
+
+flux-core version 0.63.0 - 2024-06-04
+-------------------------------------
+
+## New Features
+ * broker: add timezone abbrev to log message timestamps (#6027)
+ * add output buffering to sdexec (#6023)
+ * libsubprocess: support `FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF` (#5975)
+ * job-exec: add `kill-signal` and `term-signal` module parameters (#6017)
+ * broker: add tunable parameters for extreme fanout configurations (#6006)
+ * bypass KVS for job stdin with `--input=FILE` (#6005)
+ * flux-dump: add --ignore-failed-read option (#5989)
+ * libsubprocess: improve read API (#5965)
+ * job-manager: journal jobspec with submit event (#5955)
+
+## Fixes
+ * improve broker messages when nodes die (#6032)
+ * sdexec: don't return ENODATA without start/finish (#6033)
+ * speed up `flux overlay status` on large systems (#6030)
+ * librlist: ensure properties is an array when parsing resource.config entries
+   (#6029)
+ * resource: stop scheduler updates during shutdown (#6028)
+ * libsubprocess: reduce remote input prep/check (#6002)
+ * job-exec: retry kill of job tasks and job shell during job termination
+   (#6014)
+ * libsubprocess: protect against double EOF (#6025)
+ * sdexec: avoid double EOF notification (#6022)
+ * job-exec: cmdline options for signals should override config settings
+   (#6020)
+ * job-manager: don't log expiration update when unnecessary (#6016)
+ * libsubprocess: handle remote protocol errors (#6003)
+ * shell: do not fail on oom init failure (#6004)
+ * shell: split up and improve efficiency of input plugin (#5998)
+ * suppress confusing log messages when broker is signaled during shutdown
+   (#5996)
+ * broker: don't post quorum-full in non-QUORUM state (#5994)
+ * shell: fix repeated output truncation warning messages (#5993)
+ * don't free alloc-bypass resources when scheduler reloads (#5980)
+ * flux-job: improve EPERM error message (#5971)
+ * libsubprocess: demote assert to warning (#5959)
+ * job-exec: do not allow guest access to the testexec implementation by
+   default (#5961)
+
+## Documentation
+ * doc: misc man page cleanup (#6008)
+ * doc: fix typo in flux-jobs(1) (#6001)
+ * doc: add PRIORITY state to flux-jobs(1) (#5988)
+
+## Cleanup
+ * libsubprocess: minor cleanup (#5974)
+ * job-manager: misc cleanup in scheduler interface (#5969)
+ * libsubprocess: remove unused len parameter from `flux_subprocess_read()`
+   (#5950)
+ * job-exec: remove layer of stats indirection (#5947)
+
+## CI/Testsuite
+ * testsuite: fix sdexec-memlimit config update test (#6007)
+
+
+flux-core version 0.62.0 - 2024-05-07
+-------------------------------------
+
+## New Features
+ * job-exec: support config reload (#5913)
+ * job-exec: support module stats to see current bulk-exec configuration
+   (#5943)
+ * do not filter caches when loading hwloc topology (#5945)
+ * shell: add `hwloc.restrict` option to restrict `HWLOC_XMLFILE` to assigned
+   resources (#5944)
+ * cleanup resource.eventlog and handle remapped ranks (#5914)
+ * resource: upgrade KVS resource.eventlog if needed (#5936)
+ * flux-resource: add `-q, --queue=QUEUE` option (#5935)
+ * change default tbon.topo to kary:32 (#5930)
+ * flux-job: add `flux job taskmap --to=hosts` (#5941)
+ * notify user of stopped queue in `flux job attach` (#5911)
+ * add options for offline config file parsing (#5907)
+ * flux-exec: set up systemd environment to support sdexec debugging (#5903)
+ * job-list: limit constraint comparisons to avoid DoS (#5681)
+ * broker/module.c: name threads (#5895)
+ * suppress epilog on jobs canceled during shutdown (try no. 2) (#5894)
+
+## Fixes
+ * libsubprocess: reduce remote output prep/check (#5932)
+ * job-exec: fix potential use-after-free in bulk-exec implementation (#5937)
+ * broker: avoid LOST due to EHOSTUNREACH messages during shutdown (#5928)
+ * fix possible use-after-free in chained future implementation (#5927)
+ * job-manager: perilog: do not set working directory on subprocesses (#5922)
+ * cron: fix race if sync event changed or disabled (#5908)
+ * job-manager: improve handling of offline ranks in job prolog (#5910)
+ * Support optional arg in -L (--color) short option (#5890)
+ * alloc-check: account for resources in scheduler hello failure (#5897)
+ * librlist: sort rlist before generating R (#5887)
+ * properly report signaled tasks in `flux job wait` and `flux job attach`
+   (#5886)
+
+## Documentation
+ * doc: convert internals docs to readthedocs (#5939)
+ * doc: enhance core dump instructions in admin guide (#5926)
+ * doc: add pre-flight checklist to admin guide (#5899)
+ * doc: various minor flux-cron(1) fixes and improvements (#5905)
+
+## Cleanup
+ * job-exec: silence nuisance logging (#5948)
+ * assorted minor source cleanup (#5918, #5924)
+
+## CI/Testsuite
+ * docker-run-checks: ensure we match user's home (#5938)
+ * deps/docker: add 'time' as a dependency for tests (#5933)
+ * fix potential segfault in `test_terminus.t` (#5888)
+ * docker: add script help message for macOS (#5779)
+
+
+flux-core version 0.61.2 - 2024-04-12
+-------------------------------------
+
+## Fixes:
+
+ * broker: improve handling of overlay network during shutdown (#5883)
+ * job-manager: canceled job need not wait for sched (#5877)
+ * broker: allow patch versions to interoperate (#5879)
+
+## Testsuite:
+
+ * testsuite: fix `t3203-instance-recovery.t` under `fakeroot` (#5875)
+
+flux-core version 0.61.1 - 2024-04-09
+-------------------------------------
+
+## Fixes:
+
+ * broker: reduce log noise (#5869)
+ * shutdown: unload perilog plugin during shutdown (#5871)
+ * fix broker crash in `resource.status` RPC handling when excluded ranks
+   are also down (#5870)
+ * broker: avoid slow offline child UUID lookup (#5867)
+ * resource: round timestamp of drained ranks (#5866)
+ * fix more performance issues in `flux resource` (#5865)
+ * resource: improve `resource.status` response time with many drained ranks
+   (#5863)
+
+## Cleanup
+
+ * minor cleanup in job-manager journal and job-list (mostly inline docs)
+   (#5850)
+ * job-list: remove jobspec/R update dead code (#5853)
+
+## Documentation
+
+ * doc: add path to 'manpages' to conf.py (#5855)
+ * doc: fix whitespace issues in admin guide (#5854)
+
+flux-core version 0.61.0 - 2024-04-02
+-------------------------------------
+
+## New Features
+
+ * add utility for posting manual job events if required (#5848)
+ * add `--taskmap=hostfile:FILE` support (#5844)
+ * make KVS garbage collection automatic (#5840)
+ * add a faster way to get resource allocation status than
+   sched.resource-status RPC (#5796, #5834)
+ * job-manager: drop sched.free response requirement (#5786)
+ * job-manager: include R in sched.free request (#5783)
+
+## Fixes
+
+ * flux-job: fix `wait-event -m, --match-context`(#5846)
+ * eliminate duplicate KVS restart in job-list and job-manager (#5837)
+ * python: return empty string on epoch time for D conversion flag (#5841)
+ * configure: add missing check for python ply >= 3.9 (#5839)
+ * fix handling of `Ctrl-SPACE` (`NUL`) over job ptys (e.g. under `flux
+   job attach`) (#5833)
+ * job-ingest: fix handling of size > 16K and reserve some FLUID generator
+   IDs for future use (#5831)
+ * fix `flux submit` and `bulksubmit` handling of mustache templates in
+   command and args (#5817)
+ * flux-resource: improve performance of `flux resource list` (#5823)
+ * optimize key librlist functions to improve `flux resource list`
+   performance (#5824)
+ * python: fix handle barrier method to return a Future (#5825)
+ * job-manager: fix infinite loop when loading builtin jobtap plugin (#5822)
+ * flux-top: skip poorly formed job list records (#5821)
+ * job-exec: raise fatal job exception if rank 0 job shell is lost due to
+   signal (#5814)
+ * job-exec: improve cleanup after lost shell events (#5813)
+ * job-exec: fix potential leak of job KVS namespaces (#5805)
+ * job-exec: fix rare segfault at scale due to libsubprocess use-after-free
+   (#5803)
+ * kvs: remove excessive logging (#5804)
+ * modules/content: drop incorrect assertion (#5781)
+
+## Documentation
+ * add doc/requirements.txt to dist (#5852)
+ * doc: add missing R documentation to flux-jobtap-plugins(7) (#5838)
+
+## CI/Testsuite
+
+ * testsuite: fix potential hang in t2812-flux-job-last.t (#5835)
+ * testsuite: fix test racing with exit-timeout (#5810)
+ * python: update required versions of black and mypy (#5808)
+ * testsuite: fix lingering processes after `make check` (#5769)
+ * github: set `kernel.randomize_va_space=0` for asan build (#5802)
+ * github: run asan build on fedora36 (#5800)
+ * testsuite: fix test that kills the wrong processes (#5792)
+ * improve handling of lost job shells (#5780)
+
+
+flux-core version 0.60.0 - 2024-03-05
+-------------------------------------
+
+Note: This release replaces the flux-filemap(1) command with an enhanced new
+command called flux-archive(1).  Simple invocations of flux-filemap(1) will
+continue to work for a while to enable migration to flux-archive(1).
+
+## New Features
+ * job-manager: support suppression of prolog/epilog output with
+   `perilog.log-ignore` pattern list (#5772)
+ * flux-job: attach: support MPIR tool launch extension (#5758)
+ * add `flux job hostpids` command (#5765)
+ * shell: support opening output files with `-o output.mode=append` (#5766)
+ * shell: add rexec plugin (#5605)
+ * shell: unset `HWLOC_COMPONENTS` with `hwloc.xmlfile` (#5759)
+ * flux-job: add shorthand paths for `flux job eventlog` and `wait-event`
+   (#5749)
+ * flux-archive: add new command for file broadcast (#5701)
+ * completions: support completion of jobids with plain `f`, support
+   flux-hostlist(1) (#5745)
+ * libtaskmap: support decode of raw (semicolon-delimited) taskmaps (#5735)
+ * shell: make kvs output limit configurable and default single-user jobs
+   to unlimited (#5732)
+ * add flux-hostlist(1) (#5724)
+
+## Fixes
+ * broker: catch an improper use of groups and handle it gracefully (#5762)
+ * shell: fix incorrect values returned from `flux_shell_get_rank_info(3)`
+   (#5756)
+ * shell: generate fatal error if `block` taskmap scheme has an argument
+   (#5730)
+ * do not drain ranks when job is canceled during prolog (#5742)
+ * optparse: fix segfault when subcommand registration fails due to invalid
+   options table (#5740)
+ * fix various typos (#5761)
+
+## Documentation
+ * doc: pull in admin guide (#5763)
+
+## Cleanup
+ * fix various typos (#5761)
+ * libsubprocess: minor API cleanup (#5699)
+ * cmd: split flux-job into separate source files (#5747)
+ * job-info: support lookup of updated jobspec, remove manual construction
+   of updated R / jobspec (#5635)
+ * job-info: add `JSON_DECODE` & `CURRENT` flags, deprecate
+   job-info.update-lookup (#5633)
+
+## Build/Testsuite
+ * github: update deprecated actions (#5768)
+ * testsuite: fix t2617 to utilize /dev/urandom (#5757)
+ * testsuite: fix random-generation of kvs archive files in t0021 (#5751)
+
+
+flux-core version 0.59.0 - 2024-02-06
+-------------------------------------
+
+## New Features
+ * broker: add `FLUX_IPADDR_INTERFACE` to select broker network interface
+   (#5707)
+ * python: support interface to perform KVS watch (#5389)
+ * python: support Python 3.12 (#5691)
+ * broker: allow single-user rexec to rank 0 (#5677)
+ * add -x option to flux-alloc and flux-batch (#5665)
+ * add flux filemap get --overwrite and change the default overwrite behavior
+   (#5662)
+ * shell: add shell.post-init plugin calllback topic between shell.init
+   and first task.init (#5179)
+ * pmi: prepend Flux PMI directory to `LD_LIBRARY_PATH` (#5715)
+ * shell: write hwloc XML to a file with `HWLOC_XMLFILE` set with
+   `-o hwloc.xmlfile` (#5721)
+
+## Fixes
+ * job-list: initialize queue stats (#5712)
+ * job-ingest: fix FLUID initialization error handling to allow scaling
+   beyond 16K brokers (#5710)
+ * python: fix `flux-watch: TypeError: Object of type 'bytes' is not JSON
+   serializable` (#5704)
+ * enable encode of pty data as base64 and make `flux alloc vi` test more
+   reliable (#5703)
+ * librlist: workaround xml buffer size issue in some hwloc versions (#5690)
+ * librlist: fix segfault when initializing topology from XML in later
+   hwloc versions (#5682)
+ * fix broker hang under `flux proxy` (#5680)
+ * set userid to instance owner in job manager exceptions (#5675)
+ * job-manager: fix duplicate posting of jobspec-update event from plugins
+   (#5673)
+ * broker: only set parent-uri when instance is a job (#5663)
+ * kvs: store correct namespace after getroot lookup (#5661)
+
+## Documentation
+ * docs: add `flux_msg_incref` manpage (#5624)
+ * doc: correct typo in `flux_requeue(3)` (#5660)
+
+## Cleanup
+ * libsubprocess: make `flux_buffer` class private (#5683)
+ * job-list: misc cleanup (#5687)
+ * drop the flux-mini command (#5666)
+ * libsubprocess:  minor clean up (#5667)
+
+## Build/Testsuite
+ * test: add some scaling test support (#5717)
+ * github: update checkout action to v4 to silence warnings (#5716)
+ * docker: add Dockerfile for fedora39 (#5713)
+ * ci: add fedora39 build (#5698)
+ * testsuite: fix testsuite errors discovered in conda-forge build
+   environment (#5685)
+ * drop jsonschema requirement (#5678)
+ * libpmi: add `JANSSON_CFLAGS` to Makefile.am (#5672)
+
+
+flux-core version 0.58.0 - 2024-01-04
+-------------------------------------
+
+## New Features
+ * flux-batch: support `--quiet` option (#5645)
+ * resource: get local hwloc XML from parent when possible (#5636)
+ * python: Add `kvs.commit_async()` (#5627)
+ * python: add `decode` method to Flux `Message` class (#5653)
+ * python: add rolemask parameter to `msg_handler_create()` (#5650)
+
+## Fixes
+ * libflux: respond to denied, matchtagless requests (#5652)
+ * minor updates and improvements to bash completions (#5647)
+ * broker: improve handling of interactive initial programs (#5646)
+ * resource: fix initialization of multiple brokers per node when fallback
+   to job-info.lookup is used (#5639)
+ * libflux: add include for `ssize_t` to message.h (#5638)
+ * libflux: don't accept `FLUX_MSGTYPE_ANY` in `flux_msg_create()` (#5631)
+ * job-list: fix json object mem-leak (#5626)
+ * job-list: fix `flux job list-ids --wait-state` (#5620)
+
+## Documentation
+ * doc: add missing standard-io options to flux-batch(1) (#5648)
+ * doc: expand glossary (#5634)
+ * doc: reference flux-environment(7) in job submission cli man pages (#5629)
+ * misc doc fixes (#5604)
+
+## Cleanup
+ * libflux: drop `flux_msg_frames()` (#5632)
+ * libflux: deprecate some message flag accessors and add new and improved
+   ones (#5630)
+ * maint: remove flux spack docker (#5628)
+
+## Testsuite
+ * testsuite: fix fileref unit test failure on tmpfs (#5643)
+ * testsuite: avoid artificial corefile in test (#5641)
+ * testsuite: fixes for Lassen (#5551)
+ * testsuite: fix test corner case on stderr (#5625)
+ * testsuite: more reliability improvements in tests (#5621)
+ * testsuite: fix t2260-job-list.t with `-d -v` and run inception tests
+   with these args to prevent future similar errors (#5618)
+ * libkvs: fix test failure in aarch64 CI (#5617)
+ * testsuite: fix t2233-job-info-update.t with debug=t (#5616)
+
+
+flux-core version 0.57.0 - 2023-12-07
+-------------------------------------
+
+## New Features
+ * support colorized, human-readable output from eventlog commands (#5602)
+ * python: add KVSTxn class to KVS module (#5374)
+ * libidset: implement API for integer allocator use case (#5580)
+ * port to Alpine linux (#5568)
+ * job-ingest: make worker input buffer configurable with a default of 10MB
+   (#5550)
+
+## Fixes
+ * kvs: limit the content of guest commits (#5612)
+ * history: track root jobs (#5608)
+ * improve ssh connector reliability with different installation paths (#5591)
+ * flux-terminus: fix potential hang in terminus client commands (#5607)
+ * support start under older versions of Flux without the
+   job-info.update-watch RPC (#5589)
+ * kvs-watch: improve performance of kvs-watch disconnect/cleanup handling
+   (#5585)
+ * cli: avoid KeyError when PATH isn't set in environment (#5590)
+ * broker: eliminate some message copies (#5559)
+ * libidset: improve decoding functions (#5584)
+ * fix improper include directives in source files impacting portability
+   (#5567)
+ * make flux Python commands more resilient to changes in PYTHONPATH (#5553)
+ * job-ingest: fix cleanup when a pipeline worker process fails (#5549)
+ * libsubprocess: do not allow short writes with `flux_subprocess_write()`
+   (#5548)
+ * flux-submit: fix substitution of `{cc}` when cc=0 (#5541)
+
+## Documentation
+ * doc: use common format for commands with sub-commands (#5597)
+ * flux-kvs(1): improve man page formatting (#5588)
+ * clean up idset man pages (#5578)
+ * doc: improve --urgency option description in job submission commands
+   (#5571)
+ * doc: improve RFC references in man pages (#5573)
+ * man3: add Link with... instruction to SYNOPSIS (#5574)
+ * flux-shell(1): improve option descriptions and x-ref (#5557)
+ * doc: remove options from flux-alloc(1) et al that don't work (#5555)
+ * flux-pmi(1): add new manual page (#5554)
+ * flux-start(1): add more description and troubleshooting info (#5552)
+
+## Build
+ * build: reduce minimum jansson version to 2.9 (#5546)
+ * build: add libmissing, a convenience library for replacements for missing
+   functions (#5560)
+
+## Cleanup
+ * deprecate flux job cancel and improve flux-job(1) documentation (#5587)
+ * job-info: misc cleanup (#5586)
+ * broker: cleanup up attribute initialization code (#5543)
+
+## Testsuite
+ * testsuite: fix some test races and improve debugging (#5609)
+ * testsuite: fix race in job info update lookups (#5598)
+ * testsuite: improve reliability of a couple job signal/cancel tests (#5599)
+ * testsuite: fix fancy f grep inconsistency (#5576)
+ * get sharness tests working on alpine linux (#5564)
+ * testsuite: add multiple key job-info lookup tests (#5575)
+ * ci: add alpine Dockerfile and CI build (#5565)
+
+
+flux-core version 0.56.0 - 2023-11-07
+-------------------------------------
+
+## New Features
+
+ * support duration update for running jobs (#5522)
+ * job-list: support resource-update event (#5463)
+ * flux-job: get updated version of R (#5464)
+ * cache R in the job manager (#5472)
+ * job-info: support new update-lookup and update-watch service (#5467)
+ * libflux: make local connector built-in (#5504)
+ * make the loop:// connector built-in rather than a DSO (#5486)
+ * libflux: add `flux_send_new()` (#5499)
+ * add interthread connector that does not require zeromq (#5495)
+ * broker: use interthread connector between broker modules (#5496)
+
+## Fixes
+
+ * fix job submission to a multi-user instance over flux-proxy (#5533)
+ * job-manager: fix error message on duplicate plugin load (#5537)
+ * set `FLUX_PMI_LIBRARY_PATH` only in the job environment when simple pmi
+   is active (#5535)
+ * job-exec: fix potential hang after exec kill error (#5534)
+ * flux-proxy: fix double-free on lost connection (#5529)
+ * cron: handle ENOMEM without asserting (#5515)
+ * connectors: avoid future ABI issues with `_pad[]` (#5505)
+ * python: return more precise result from `flux.util.parse_datetime("now")`
+   (#5507)
+ * python: fix handling of JobInfo properties and `to_dict()` method with
+   missing attributes (#5493)
+
+## Documentation
+
+ * flux-environment(7): add new man page (#5527)
+ * man1: improve HTML formatting of man pages (#5521)
+ * man3: improve HTML formatting of man pages (#5517)
+ * man3: improve HTML formatting of SYNOPSIS sections and C examples (#5503)
+ * python: add flux.job.list docstrings (#5500)
+ * doc: add interacting with flux section (#5492)
+
+## Cleanup
+
+ * drop broker `conf.module_path`, `conf.connector_path`, `conf.exec_path`
+   attributes (#5536)
+ * flux job info: drop multiple key support, clean up code, add man page
+   entry (#5520)
+ * build: reduce junk content in DSOs (#5516)
+ * shell: drop job shell standalone mode (#5512)
+ * misc build system cleanup (#5511)
+ * libflux: refactor internal message queues (#5508)
+
+## Testsuite
+
+ * testsuite: trivial test fixes (#5498)
+ * ci: add flux-pmix, flux-pam builds to CI (#5489)
+
+
+flux-core version 0.55.0 - 2023-10-03
+-------------------------------------
+
+## New Features
+
+ * drop libzcmq dependency (#5468)
+ * allow hwloc topology to be loaded from a file with `FLUX_HWLOC_XMLFILE`
+   (#5462)
+ * improve begin-time representation in `flux-jobs(1)` output (#5473)
+ * support update of job queue (#5424)
+ * job-list: support getting job project and bank (#5413)
+ * flux-job: get updated version of jobspec (#5428)
+ * flux-top: use streaming job-stats RPC (#5432)
+ * job-list: support streaming job-stats RPC (#5430)
+
+## Fixes
+
+ * libzmqutil: fix portability to libzmq-4.1.5 (#5481)
+ * broker: move policy config check out to the modules that rely on it
+   (#5478)
+ * libzmqutil: add cert class and use it instead of CZMQ `zcert_t` (#5461)
+ * broker: stop managing 0MQ sockets with czmq (#5454)
+ * use zeromq interfaces directly where possible instead of via czmq (#5458)
+ * rc: fix `flux start` failure when multiple scripts are present in `rc1.d`
+   (#5453)
+ * rc: avoid startup problems when `BASH_ENV` is set (#5448)
+ * flux-keygen: drop libsodium requirement (#5446)
+ * content: make the content cache a broker module (#5435)
+ * job-manager: correct fsd output in error message (#5437)
+ * modules: consistently return error on invalid module arguments (#5442)
+
+## Documentation
+
+ * doc: add a page on starting Flux (#5477)
+ * doc: add build instructions and support info (#5476)
+ * doc: add content to landing page and group man pages (#5470)
+
+## Cleanup
+
+ * libflux: drop `flux_panic()` (#5439)
+ * job-manager: update: cleanup, small fixes, and documentation (#5434)
+ * job-manager: stop publishing job-state event messages (#5433)
+
+## Build/Testsuite/CI
+
+ * switch qemu-user-static setup to fix setuid (#5469)
+ * etc: remove ubuntu build-container action (#5474)
+ * configure: use distutils if available to avoid extra python module
+   dependency (#5459)
+ * configure: avoid use of deprecated python distutils module (#5456)
+ * testsuite: handle job signal race in more tests (#5438)
+ * testsuite: increase sleep time in tests (#5431)
+
+
+flux-core version 0.54.0 - 2023-09-05
+-------------------------------------
+
+## New Features
+ * flux-perilog-run: support prolog and epilog timeouts (default 30m) (#5416)
+ * cmd: add --with-imp options to flux-exec(1) and flux-perilog-run(1)
+   (#5422)
+ * shell/pmi: warn if application might be using slurm's libpmi2.so (#5420)
+ * job-list: allow updates to all of jobspec (#5418)
+ * job-manager: support jobspec update to all fields (#5419)
+ * python: add namespace support to KVS module (#5373)
+ * add job update service and new job-update(1) command (#5409)
+ * job-list: support jobspec-update event (#5408)
+ * job-manager: prevent jobspec-update events after a job has resources
+   (#5406)
+ * job-manager: add flux_jobtap_jobspec_update_pack(3) (#5386)
+
+## Fixes
+ * cmd: flux-perilog-run: avoid running prolog/epilog on offline ranks (#5417)
+ * job-manager: fix duration limit check for jobs with unset/unlimited
+   duration (#5405)
+ * flux-top: fix title when connected to instances that are not jobs (#5394)
+ * do not search for module and connector DSOs recursively (#5390)
+ * python: fix __delitem__ in KVSDir with initial paths (#5376)
+ * python: have kvs.isdir() return False if key missing (#5371)
+ * python: clear kvs txn after all commits (#5369)
+
+## Cleanup
+ * libpmi: cleanup old code and optimize client reads (#5423)
+ * job-list: misc cleanup (#5407)
+ * broker: refactor broker module loading code and fix minor bugs (#5385)
+
+## Build/Testsuite/CI
+ * ci: fix failure detection builds with coverage and remove obsolete system
+   tests (#5421)
+ * actions: update typo checker version (#5410)
+ * extend ci-checks timeout for arm64 build (#5402)
+ * testsuite: handle job signal race in more tests (#5401)
+ * matrix: add arm64 install-only builder (#5396)
+ * testsuite: relax systemctl output parsing (#5388)
+ * testsuite: fix race in t0005-exec.t signal test (#5383)
+ * actions: add merge_group trigger (#5379)
+ * mergify: remove status check conditions from config (#5381)
+ * docker-deploy: only push latest when arch is 64-bit (#5377)
+ * docker: drop bionic, el7 and fedora35 for bookworm and fedora38 (#5370)
+
+flux-core version 0.53.0 - 2023-08-01
+-------------------------------------
+
+## New Features
+ * add capability to run jobs in systemd (#5197)
+ * job-exec: allow job memory limits to be set (#5359)
+ * python: add API for job output (#5332)
+ * python: add JobWatcher class and use for `submit` and `bulksubmit`
+   `--watch` functionality (#5357)
+ * cmd: add flux-watch(1) (#5360)
+ * python: add FutureExt class for extensible futures from python (#5330)
+ * shell: support `-o gpu-affinity=map:LIST` (#5356)
+ * job-info: support WAITCREATE on eventlog watch  (#5358)
+ * job-list: support job list constraints (#5126)
+
+## Fixes
+ * job-list: support older RPC protocol (#5364)
+ * job-manager: prevent jobs with outstanding epilog-start events from
+   becoming inactive (#5353)
+ * ensure flux utilities agree that a job with a fatal exception has failed
+   (#5355)
+ * flux-job: suppress the `attach` status line in more situations (#5354)
+ * flux-jobs: correct several filtering corner cases (#5164)
+ * sdexec: fix memory leaks (#5349)
+ * python: fix potential gc of Future ffi handle before Future destruction
+   (#5346)
+ * resource: fix problem with exclusions when R is dynamically discovered
+   (#5339)
+ * python: clear KVS txns on commit error (#5335)
+ * python: do not return int status in kvs functions (#5329)
+ * python: fix exists in KVSDir with initial paths (#5331)
+ * python: fix writes in KVSDir with initial paths (#5322)
+ * flux-job: fix invalid --original info output (#5318)
+ * flux-job: fix `flux job status` handling of nonfatal exceptions (#5320)
+ * job-manager: fix prolog/epilog exception handling (#5321)
+ * job-info: ignore duplicate lookup keys (#5317)
+
+## Cleanup
+ * job-exec: remove systemd exec prototype (#5348)
+ * job-manager: make exception note a requirement (#5336)
+ * resource: ignore live resource.exclude changes (#5341)
+ * python: add extra documentation to kvs.py module (#5328)
+
+## Build/Testsuite/CI
+ * testsuite: remove get-xml-test.py (#5340)
+
+
+flux-core version 0.52.0 - 2023-07-06
+-------------------------------------
+
+## New Features
+
+ * libjob: export `flux_unwrap_string(3)` function (#5312)
+ * job-manager: add alloc-check plugin (#5304)
+ * add f58plain job encoding (#5297)
+ * libsubprocess: support user friendly error string  (#5294)
+ * python: support convenience API for `job-info.lookup` RPC
+   and `flux job info` (#5265, #5311)
+ * support `[kKMG]` suffixes in command options that take a bytes argument
+   (#5277)
+ * libutil: add `parse_size()` (#5262)
+ * flux-resource: add `R` subcommand (#5246)
+ * job-exec: always use stdio for exec barrier (#5267)
+ * sdbus: make debug logging configurable (#5264)
+
+## Fixes
+
+ * job-manager: publish event on jobtap exception (#5310)
+ * librlist: fix RFC31 corner cases (#5137)
+ * testsuite: workaround job start signal race (#5302)
+ * shell: document signal race (#5299)
+ * testsuite: fix occasional broker kill error (#5291)
+ * do not suppress job shell and broker errors with `flux alloc` (#5274)
+ * allow guests to use flux module list, flux module stats (#5280)
+ * broker: load module with DSO version extension (#5283)
+ * shell: ensure captured pty data does not come after stdout EOF in output
+   eventlog (#5282)
+ * Parse jobspec attributes.system optionally  (#5279)
+ * broker: avoid spurious overlay peer warning (#5275)
+ * flux-resource: fix `-i, --include=HOSTS` in `list` command when some
+   hosts are excluded (#5268)
+ * python: allow JobID to take a JobID (#5259)
+ * flux-top: fix formatting with ASCII jobids (#5263)
+ * shell: set correct HOSTNAME in job environment if necessary (#5261)
+ * Ignore errors when starting flux from a restart dump containing giant
+   blobs (#5254)
+ * support utf-8 in broker logs (#5253)
+ * flux-config-bootstrap(5): fix TOML error (#5252)
+ * libjob: return on error in `unwrap_string()` (#5251)
+ * libjob: fix leak in `sign_unwrap()` (#5248)
+ * flux-job: fix attach notification with multiple prolog-start events (#5315)
+
+## Cleanup
+
+ * switch from decimal to f58 jobid encoding in most log messages and shell
+   service name (#5256)
+ * flux-job: add missing include of signal.h (#5247)
+ * testsuite: improve alloc-check test (#5309)
+ * fix assorted typos and adjust whitespace per project norms (#5298)
+
+## Build/Testsuite/CI
+
+ * build: require flux-security >= 0.9.0 (#5270)
+
+flux-core version 0.51.0 - 2023-06-09
+-------------------------------------
+
+This release adds the flux-batch(1) and flux-alloc(1) `--conf` option,
+which makes subinstance configuration significantly easier.  Sys admins
+may note that _named configurations_ can be added to `/etc/xdg/flux/config`
+for configurations that are anticipated to be commonly needed such as
+node-exclusive scheduling.
+
+For example, `/etc/xdg/flux/config/node-exclusive.toml` could contain:
+```toml
+[sched-fluxion-resource]
+match-policy = "lonodex"
+```
+And then users could request this in a batch subinstance with:
+```
+flux batch --conf=node-exclusive ...
+```
+
+## New Features
+
+ * add `--conf` option for registering instance config in `flux-batch` and
+   `flux-alloc` (#5232)
+ * support RFC 14 `files` file archive in jobspec, add `--add-file` option
+   to cli submission commands (#5228, #5239)
+ * support option to signal a job a configurable time before expiration (#5212)
+ * flux-resource: add `-i, --include` option to filter hosts for `status`
+   and `drain` commands (#5219)
+ * flux-resource: add `-i, --include=TARGETS` option to `list`  and `info`
+   subcommands (#5236)
+ * broker: forward nonfatal signals to all running jobs (#5202)
+ * broker: unset `SLURM_*` in initial program environment (#5237)
+ * sdbus: add subscription filtering (#5227)
+ * job-exec: provide IMP exec helper when possible and use stdin/out for
+   shell exec protocol (#5186)
+ * support emoji encoding for Flux jobids (#5174)
+ * job-list: support retrieving current working directory for jobs (#5156)
+ * job-list: allow list-id to wait for job state (#5213)
+ * flux-jobs: support new `inactive_reason` output field and `endreason`
+   format (#5055)
+ * broker: redefine broker.quorum as a size (#5153)
+ * broker: signal readiness to systemd in JOIN state (#5152)
+
+## Fixes
+
+ * completions: update bash completions with recent command changes (#5241)
+ * shell/pmi: accept pmi1 and pmi2 as aliases for "simple" pmi (#5242)
+ * `zhash_insert()`/`zhashx_insert()` return EEXIST  (#5217)
+ * restrict remote access to sdbus on rank 0 (#5162)
+ * flux-job: submit: strip unnecessary whitespace from pre-signed jobspec
+   (#5185)
+ * libsubprocess: fail with ESRCH when pid cannot be found (#5229)
+ * flux-jobs: reduce likelihood of leaving bad terminal color state on
+   empty output (#5211)
+ * python: support non-JSON formatted data in KVS (#5204)
+ * Add missing include of config.h (#5182)
+ * liboptparse: correct message on missing argument (#5180)
+ * python: improve handling of `!d` conversion specifier in output formats
+   (#5169)
+ * libioencode: fix memleaks on error paths (#5159)
+ * shell: add missing `flux_` prefix to `shell_mustache_render(3)` (#5161)
+ * shell: truncate output to KVS after 10MB (#5155)
+ * python: open UtilConfig files as binary (#5157)
+ * job-manager: make some replay errors non-fatal (#5150)
+ * doc: fix python example (#5191)
+ * flux-job: ignore stdin instead of aborting when unable to create stdin
+   watcher (#5183)
+
+## Cleanup
+ * cleanup: use ccan/str in place of strcmp(), strncmp() (#5163)
+ * job-list: misc cleanup (#5144)
+
+## Testsuite/CI
+ * testsuite: fix potential flux-top race (#5224)
+ * testsuite: fix tests when run in debug mode (#5231)
+ * testsuite: increase test expiration (#5222)
+ * testsuite: fix race in flux top tests (#5194)
+ * testsuite: skip tests that expect COLUMNS to be inherited across calls
+   to flux(1) when that isn't true (#5166)
+ * pre-commit: increase vermin version (#5173)
+ * ci: fix pre-commit config to lint python files in testsuite (#5221)
+ * ci: add fedora38 builder, update flux-security default to 0.8.0 (#5160)
+
+flux-core version 0.50.0 - 2023-05-03
+-------------------------------------
+
+## New Features
+ * broker: make `tbon.connect_timeout` configurable (#5140)
+ * flux-job: support job ids to purge (#5047)
+ * sdbus: enable flux to communicate with a systemd user instance (#5070, #5131)
+ * shell: support `{{name}}` tag in output file templates (#5128)
+ * flux-top: support ability to flip through queues via left/right arrow keys
+   (#5052)
+ * flux-ping: output header before output of main ping output (#5034)
+ * broker: improve systemd integration with `sd_notify(3)` (#5078)
+
+## Fixes
+ * `flux(1)`: avoid prepending to PATH when unnecessary (#5138)
+ * python: make `SchedResourceList` optional in `flux.job.info` (#5141)
+ * fix parent-uri attribute under remote `flux-proxy(1)` (#5133)
+ * job-list: make job stats consistent to job results (#5048)
+ * fileref: fix compile on systems without `SEEK_DATA`/`SEEK_HOLE` (#5114)
+ * fixes for build/test on Fedora 36 (GCC 12) (#5107)
+ * shell: fix improper encoding of some hostnames in MPIR proctable (#5117)
+ * python: fix parsing of special characters in submission directives (#5125)
+ * job-validator: fix empty plugins list when one plugin fails to import
+   (#5124)
+ * broker: use human readable timestamp in local time when logging to stderr
+   (#5129)
+ * improve error on plugin load if `flux_plugin_init()` returns an error
+   (#5135)
+ * librlist: fix memleak + misc cleanup (#5110)
+ * sched-simple: avoid assertion failure when trying to load scheduler twice
+   (#5109)
+ * job-manager: improve errors from jobtap initialization (#5099)
+ * libsubprocess: avoid segfault on error path (#5096)
+ * job-exec: improve error message when job shell/imp execution fails (#5088)
+ * systemd: avoid leaving unit in failed state after flux-shutdown(1) (#5077)
+
+## Cleanup
+ * libjob: deprecate `flux_job_list()` and `flux_job_list_inactive()` (#4855)
+ * broker: clean up module infrastructure (#5085)
+ * libsubprocess: remove use of assert(0) (#5084)
+
+## Testsuite/CI/Development
+ * ensure license and other informational files appear in distribution
+   tarball (#5113)
+ * mergify: add spelling check to required checks (#5112)
+ * Add false positives typos config (#5106)
+ * Fix minor typos and formatting (#5019)
+ * testsuite: fix test issues under nix (#5015)
+ * testsuite: fix column width output corner case (#5103)
+ * testsuite: fix setup error in system tests (#5102)
+ * build: add `make deb` target to build test debian package (#5101)
+ * build: applicable build and test fixes for conda (#5093)
+ * testsuite: skip failing test on RHEL7 (#5090)
+ * add spell check for news, readme, and user facing code (#5074)
+
+
+flux-core version 0.49.0 - 2023-04-05
+-------------------------------------
+
+## New Features
+ * libpmi: improve error messages from upmi plugins (#5066)
+ * shell: support -o pmi=LIST (#5069)
+ * flux-jobs: add --json option (#5054)
+ * flux-job: add special exit code to flux job wait when not waitable (#5049)
+ * libpmi: enable flux to bootstrap with cray libpmi2.so (#5051)
+ * libpmi: improve tracing of dlopened PMI libraries (#5053)
+ * resource: do not allow ranks to be both drained and excluded (#5039)
+ * Support environment variable to override default output formats (#5028)
+ * improve broker debugging on overlay connect failure (#5014)
+ * rewrite flux-resource status (#4997)
+ * flux-resource: support overwrite of drain timestamp with `--force --force`
+   (#5000)
+ * python: improve Hostlist class indexing (#4993)
+
+## Fixes
+ * openmpi: don't force flux MCA plugins (#5067)
+ * PMI: ensure fallthrough when PMI library fails to initialize (#5058)
+ * flux-top: fix queue specific output display (#5032)
+ * flux-pgrep: fix warning about `sre_constants` on Python 3.11+ (#5043)
+ * prevent orphaned job processes when terminating jobs due to exception
+   (#4990)
+ * python: recognize local timezone epoch adjustment (#5025)
+ * fix rare `Unable to connect to JOBID` error from `flux alloc --bg` (#5012)
+ * job-manager: ensure epilog-start event prevents resource release for
+   job that never started (#5011)
+ * librlist: drop V1 flag from hwloc XML generation (#5007)
+ * fix: warning message to user should be actual command (#5002)
+ * flux-mini: improve deprecation warning (#4989)
+
+## Cleanup
+ * job-list: minor code consistency cleanup (#5031)
+ * mpi: drop mvapich.lua plugin (#5045)
+ * libflux: remove extraneous +1s used in buffers (#5020)
+ * cleanup: improve interface for subprocess logging (#5006)
+ * cleanup: simplify remote subprocess protocol (#5004)
+ * cleanup: allow subprocess service name to be configured (#5003)
+ * cleanup: improve reusability of common message handlers (#5001)
+
+## Documentation
+ * flux-job(1): update WAIT section (#5042)
+ * doc: document job environment variables (#5024)
+ * doc: document `FLUX_URI_RESOLVE_LOCAL` (#5023)
+ * cli: adjust description of `--begin-time` submission option (#5018)
+
+## Testsuite/CI/Development
+ * testsuite: increase test timeouts (#5017)
+ * testsuite: speed up t4000-issues-test-driver.t (#5010)
+ * testsuite: require jq(1) (#4995)
+
+
+flux-core version 0.48.0 - 2023-03-07
+-------------------------------------
+
+This release adds submission directives ("see flux help batch") and
+shortens the the job submission commands to "flux batch", "flux run",
+etc.  The flux-mini(1) command is deprecated.
+
+## New Features
+
+ * support RFC 36 submission directives in `flux mini batch` (#4942)
+ * make all flux-mini subcommands available as top level flux commands (#4961)
+ * add flux-cancel(1) (#4983)
+ * add flux-fortune (#4966)
+ * flux-run: allow stdin to be directed to a subset of tasks (#4977)
+ * cmd: add -u, --unbuffered option to submission commands (#4973)
+ * allow flux-core to be configured in ascii-only mode (#4968)
+ * Support {{tmpdir}} in shell mustache templates and simplify batch jobspec
+   construction (#4951)
+ * broker: allow a file argument to `-c, --config-path` in TOML or JSON
+   (#4949)
+
+## Fixes
+
+ * completions: remove flux-mini and other updates (#4984)
+ * flux-top: initialize f character before drawing panes (#4982)
+ * libsubprocess:  don't abort remote processes that receive SIGSTOP (#4981)
+ * shell: fix memory leak in doom plugin (#4979)
+ * flux-resource: suppress NGPUS on systems without GPUs (#4959)
+ * job-ingest: handle worker channel overflow (#4948)
+ * improve error message in Python frontend utilities when broker is not
+   running (#4950)
+ * job-manager: fix for job priority not reset after a duplicate urgency
+   update (#4941)
+ * job-manager/history: track inactive jobs over purge/restart (#4932)
+
+## Cleanup
+
+ * flux-filemap: update to RFC 37 internally (#4974)
+ * libsubprocess: rework internal logging (#4960)
+ * libsubprocess: drop `FLUX_SUBPROCESS_EXEC_FAILED` state (#4955)
+ * libsubprocess: fix bugs and clean up subprocess server (#4944)
+
+## Documentation
+
+ * Fix minor documentation errors and typos (#4934)
+ * flux-batch(1) and flux-alloc(1): improve DESCRIPTION section (#4963)
+ * README: fix a couple typos (#4970)
+ * README: trim it down now that we have readthedocs (#4969)
+ * divide flux(1) help output into sections (#4967)
+ * doc: add shell help on flux-jobs(1) formatting (#4939)
+ * doc: document attach in flux-job(1) (#4936)
+
+## Testsuite/CI/Development
+
+ * testsuite: unset `FLUX_F58_FORCE_ASCII` in some tests (#4976)
+ * .devcontainer permissions fix (#4964)
+ * Add/developer doc on commands (#4965)
+
+flux-core version 0.47.0 - 2023-02-07
+-------------------------------------
+
+## New Features
+
+ * add `flux job last` (#4908)
+ * add `flux-pgrep` and `flux-pkill` (#4867, #4903)
+ * add `flux-keygen --meta KEY=VAL` option (#4882)
+ * add tools for querying job remaining time: `flux_job_timeleft(3)`, python
+   `flux.job.timeleft()` and `flux-job timeleft` (#4845)
+ * flux-shell: add `-opmi=off` option (#4841)
+ * suggest use of `--force` in `flux resource drain` when target is already
+   drained (#4924)
+ * automatically provide job status for pending interactive jobs (#4916)
+ * flux-resource: mark drained+offline nodes with asterisk (#4913)
+ * support flux mini batch,alloc `--dump[=FILE]` (#4881)
+ * flux-queue: support flux queue list (#4896, #4929)
+ * support RFC 31 `hostlist` and `rank` job constraints (#4895, #4919)
+ * python: add flux.constraint.parser for RFC 35 Constraint Query Syntax
+   (#4871, #4925)
+ * Support RFC 35 constraint syntax in `flux mini --requires` (#4897, #4923)
+ * flux-top: limit jobs and summary to specific queue (#4847)
+ * enable broker bootstrap methods to be provided by dso plugins, and drop
+   compiled-in pmix support (#4865)
+ * flux-resource: support QUEUE output in resource list (#4859)
+ * flux-top: Support --color option (#4840)
+ * libutil: support "infinity" in FSD (#4846)
+ * add internal universal PMI client library (#4829)
+ * job-manager: default queues to enabled and stopped (#4857)
+ * libtaskmap: add `TASKMAP_ENCODE_RAW_DERANGED` (#4838)
+
+## Fixes
+
+ * job-list: do not assume alloc event context always contains annotations
+   (#4907)
+ * job-manager: fix alloc-bypass plugin (#4901)
+ * flux-resource: increase width of queue field (#4905)
+ * eliminate "safe mode" after improper shutdown (#4898)
+ * flux-resource: handle queues with no configured constraints (#4893)
+ * fix message encoding problem introduced in v0.46.1 (#4890)
+ * flux-shell: truncate long log messages (#4878)
+ * job-manager: switch to timer watcher in perilog plugin (#4864)
+ * job-manager: do not checkpoint on every queue state change (#4856)
+ * job-list: separate `t_submit`/`t_depend` calculation (#4853)
+ * flux-top: honor `FLUX_F58_FORCE_ASCII` (#4842)
+ * flux-job: fix potential segfault (#4827)
+ * work around fluxion inbability to recover running jobs (#4894)
+ * etc: update bash completions (#4928)
+
+## Documentation
+
+ * doc: document `--job-name` in flux-mini(1) (#4879)
+ * doc: document format fields in flux-resource(1)  (#4850)
+ * doc: document subcommand `wait` in flux-job(1) (#4851)
+
+## Testsuite/CI/Development
+
+ * clean up little used broker attribute functionality (#4870)
+ * flux-queue: rewrite in python (#4889)
+ * job-list: add jobspec and R parsing unit tests (#4883)
+ * flux-top: add extra test coverage (#4833)
+ * testsuite: increase `flux job wait-event` timeout (#4888)
+ * testsuite: drop fragile broker.boot-method test (#4876)
+ * docker: add site-packages to default python3 path (#4880)
+ * ci: speed up coverage builds (#4828)
+
+
+flux-core version 0.46.1 - 2022-12-11
+-------------------------------------
+
+## Fixes
+ * build: fix installation of libpmi.so (#4824)
+ * testsuite: fix failure on a system with fully-qualified hostname (#4825)
+
+## Cleanup
+ * libflux/message: cleanup with macros and CCAN pushpull class (#4823)
+
+flux-core version 0.46.0 - 2022-12-10
+-------------------------------------
+
+## New Features
+ * job-manager: support start / stop of individual queues (#4776)
+ * add file broadcast toolset (#4789, #4818)
+ * flux-pmi: add installed PMI test tool (#4817)
+ * flux-mini: add --cwd option (#4811)
+ * add flux start --recovery (#4783)
+ * support cyclic, manual job taskmaps and `flux job taskmap` command (#4772)
+ * shell: remove `pmi.clique` option, add `pmi.nomap` (#4785)
+ * libtaskmap: support RFC 34 unknown task maps (#4788)
+ * python: add `flux.job.get_job` (#4761)
+ * allow guests to access flux configuration (#4766)
+ * job-list: add inactive purge count to job-list stats (#4756)
+ * shell: allow users to map specific cpus to tasks with `map` cpu-affinity
+   option (#4819)
+
+## Fixes
+ * broker: fix startup failure when broker.mapping is not set (#4781)
+ * encode broker.mapping as RFC 34 taskmap, drop unused pmi clique helper
+   code (#4787)
+ * add missing taskmap pkg-config file (#4782)
+ * broker: fix potential crash after sending SIGKILL to job-manager prolog
+   (#4793)
+ * broker: improve detection of interactive shells (#4795)
+ * flux-job: remove `finish_mpir_interface()` (#4808)
+ * libhostlist: fix 8K limit on encoded hostlists (#4803)
+ * add flux.taskmap to PMI kvs for better cyclic task distribution
+   scalability (#4798)
+ * job-manager: fix memory corruption due to json reference counting bug
+   (#4773)
+ * python: by default return all attrs in job listing functions (#4762)
+ * shell: rlimit: improve handling of limits over hard limit (#4752)
+
+## Cleanup
+ * cleanup: remove some unused functions (#4806)
+ * job-manager: use allocation terms consistently (#4775)
+ * content: cleanup and minor tooling updates (#4770)
+
+## Testsuite/CI/Development
+ * testsuite: add flux job raise/cancel/kill tests (#4747)
+ * github: fix automated tag release action (#4755)
+ * Vscode setup (#4683)
+ * ci: revert to ubuntu-20.04 for workflow jobs that fail on ubuntu-latest
+   (#4794)
+ * testsuite: fix several corner cases in t0029-filemap-cmd (#4812)
+
+
+flux-core version 0.45.0 - 2022-11-01
+-------------------------------------
+
+## New Features
+ * propagate process resource limits to jobs (#4745)
+ * flux-job: support multiple jobids for `cancel`, `raise`, and `kill` (#4721)
+ * flux-resource: unify output format options, support config files and
+   named formats (#4710)
+ * broker: support binomial tree topology (#4730)
+ * broker: allow custom topology to be configured (#4675)
+ * flux-mini: add -x short option for --exclusive (#4726)
+ * flux-jobs: support emoji output formats (#4687)
+ * flux config: add load subcommand (#4695)
+ * broker: ignore ENOSYS from parent job-exec.critical-ranks RPC (#4680)
+ * job-list: support retrieving job's core count (#4664)
+ * job-list: add successful job count stat (#4739)
+ * job-list: support queue specific stats (#4684)
+ * etc: add functional bash completions (#4661, #4742)
+
+## Fixes
+ * job-list: ensure purged jobs are inactive (#4738)
+ * flux-proxy: require owner == proxy user (#4712, #4735)
+ * support mpi=none shell option and make it the default for `flux mini
+   batch` and `flux mini alloc` (#4731)
+ * unset job environment variables in initial program (#4717)
+ * flux-resource: fix scalability issues with large sets of resources (#4713)
+ * build: fix use of system bash-completion dir (#4667)
+ * rc1: reload configuration on rank > 0 (#4665)
+ * broker/test: Fix runat race on older glibc versions (#4660)
+ * broker: launch non-interactive shells in a new process group (#4653)
+
+## Cleanup
+ * job-list: cleanup error logging, remove excess logging (#4744)
+ * README: update flux help output (#4688)
+ * python: indicate truncation for some fields in flux-jobs and flux-resource
+   (#4670)
+ * python: move and rename some classes for reusability (#4669)
+ * job-list: refactor to abstract "idsync" logic better (#4644)
+ * broker: don't log failed `CONTROL_DISCONNECT` (#4656)
+
+## Testsuite/CI/Development
+ * fix github action syntax for output and yaml formatting (#4733)
+ * add devcontainer autocomplete (#4709)
+ * lint: update devcontainer to work with pre-commit (#4690)
+ * codeql: fix some critical issues found by security scanning (#4729)
+ * Create codeql.yml (#4705)
+ * ci: add isort to pre-commit and linting (#4691)
+ * ci: update setup-python/buildx actions to v4/v2 (#4693)
+ * Pre-commit extensions and multi-version setup (#4689)
+ * ci: consolidate python linting and formatting (#4636)
+ * Add devcontainer environment for vscode (#4674)
+ * test: job-ingest: skip guest tests when default sign-type fails (#4655)
+
+flux-core version 0.44.0 - 2022-10-04
+-------------------------------------
+
+This release includes initial support for job queues, which can be
+configured to partition resources with different limits and defaults.
+See `flux-config-queues(5)` for details.
+
+Other highlights include:
+
+ * Add ability to modify jobspec at the time of ingest with a new
+   job frobnicator framework similar to the existing validators.
+ * A new alternative to `system.R` for configuration of resources
+   (See `flux-config-resource(5)` for details)
+ * All child Flux instances are resilient to non-critical node failures
+ * Updates to `flux jobs` including better default output, a set of
+   new default named formats (See `flux jobs --format=help`), and support
+   for config files to add more named formats.
+
+## New Features
+ * ingest: set configured queue constraints (#4587)
+ * ingest: enable frobnicator when needed by [queues] or [policy] (#4608)
+ * reject jobs submitted to a named queue when none are configured (#4627)
+ * make queue state persist across instance restart (#4640)
+ * flux-mini: add `-q, --queue=NAME` option (#4599)
+ * flux-top: add minimal support for job queues (#4605)
+ * flux-jobs: support getting job queue and filtering by job queue (#4579)
+ * flux-jobs: support collapsible format fields (#4591)
+ * flux-jobs: add configurable named formats (#4595)
+ * flux-jobs: add queue to builtin format strings (#4607)
+ * flux-jobs: add `--format=long` (#4642)
+ * flux-jobs: support width and alignment with `!d` conversion specifier (#4597)
+ * python: add `contextual_time` jobinfo field (#4641)
+ * python: add `contextual_info` jobinfo field (#4626)
+ * broker: reduce `tbon.tcp_user_timeout` (#4632)
+ * make child instances resilient to non-critical node failures (#4615)
+ * resource: support configuration of resources in TOML config (#4566)
+ * drop environment from KVS jobspec (#4637)
+ * docker: add namespaced builds for spack/ubuntu (#4577)
+
+## Fixes
+ * flux-mini: change default unit for `--time-limit` to minutes (#4565)
+ * job-list: handle per-resource "cores" special cases (#4630)
+ * job-list: handle per-resource ntasks special case (#4555)
+ * job-list: use libjj to parse jobspec (#4611)
+ * job-list: parse tasks with total count in jobspec (#4651)
+ * job-info: return error on invalid eventlog append (#4624)
+ * flux-shell: always use `pmi=pershell` by default (#4621)
+ * ingest: require job queue if [queues] are configured (#4616)
+ * job-ingest: assign jobspec defaults before scheduler feasibility check
+   (#4529)
+ * configure: remove `--without-python` (#4584)
+ * configure: fix obsolete autotools warnings (#4588)
+ * configure.ac: fix error message when running autogen.sh (#4590)
+ * job-manager: print better errors on inactive move (#4586)
+ * broker: fix use-after-free segfault (#4570)
+ * python: uri: use path to current flux executable in lsf resolver (#4559)
+ * spack: add flux-core container build (#4561)
+ * improve signal/noise ratio in systemd journal logs (#4560)
+ * flux-mini: improve an error message and documentation for per-resource
+   options (#4549)
+ * doc: document `{id.dec}` in `flux-jobs(1)` (#4548)
+ * doc: add note about flux `--parent` option in `flux-mini(1)` (#4650)
+ * job-manager: do not ignore failure to load configured plugins (#4647)
+
+## Cleanup
+ * job-list: remove circular header dependencies (#4639)
+ * job-list: split out job information into new files (#4575)
+ * job-list: misc cleanup (#4563)
+ * Container base: remove view copy (#4551)
+
+## Testsuite
+ * testsuite: document `FLUX_TEST_VALGRIND` (#4643)
+ * testsuite: remove errant `test_done` call (#4609)
+ * testsuite: fix spurious resource norestrict test failures on some version
+   of hwloc (#4550)
+
+
+flux-core version 0.43.0 - 2022-09-06
+-------------------------------------
+
+This release includes changes after several weeks of Flux running as the
+primary resource manager on two small production systems at LLNL.  Some
+noteworthy changes are:  new options to flux-jobs(1) and flux-mini(1), show
+detailed job state in flux-jobs(1) output, and automatic KVS garbage
+collection.  Also included:  bug fixes for for cpu affinity, tcsh users,
+users with non-UTF-8 compliant terminals, and a rank 0 broker segfault when
+inactive job purging is enabled.
+
+## New Features
+ * cmd: add "per-resource" allocation options to flux-mini run and submit
+   (#4544)
+ * job-list: return nnodes if jobspec specifies nodes (#4542)
+ * resource: add norestrict option to avoid restricting hwloc topology XML
+   (#4538)
+ * flux-mini: add --bg option to flux-mini alloc (#4531)
+ * kvs: support gc-threshold config (#4528)
+ * etc: support content.backing-module=none (#4492)
+ * fetch J instead of jobspec in the job shell, support flux job info
+   --original jobspec (#4523)
+ * flux-jobs: add --since=WHEN and --name=NAME options (#4517)
+ * add flux jobtap query subcommand (#4507)
+ * libkvs: Support `KVS_CHECKPOINT_FLAG_CACHE_BYPASS` flag (#4477)
+ * flux-mini: --setattr: place keys in `attributes.system` by default
+   and default value to 1 (#4483)
+ * kvs: add root sequence number to checkpoint object (#4475)
+
+## Fixes
+ * shell: inherit `FLUX_F58_FORCE_ASCII` from job environment (#4541)
+ * shell: fix cpu-affinity=per-task (#4537)
+ * flux-mini: fix bulksubmit help message (#4539)
+ * fix ssh connector with csh/tcsh shells (#4532)
+ * broker: log content store errors to `LOG_CRIT` (#4526)
+ * broker: forward content.checkpoint-{get,put} RPCs to rank 0 (#4519)
+ * cmd/flux-jobs: include job state in status output (#4515)
+ * flux-jobs: improve bad username error message (#4503)
+ * update fluid check to check explicitly for utf-8 (#4505)
+ * doc: add TIMEOUT result to flux-jobs(1) (#4500)
+ * fix formatting issues with large UIDs (#4489)
+ * broker: fix content-cache flush list corruption (#4484)
+ * top: fix detailed report in summary pane (#4479)
+ * content-{sqlite,files,s3}: route checkpoint-get and checkpoint-put
+   through broker (#4463)
+ * job-list: avoid segfault after job purge (#4470)
+
+## Cleanup
+ * job-list: remove job-list.list-inactive RPC (#4513)
+ * flux-job: point users to flux-jobs(1) (#4499)
+ * docker: typo in path to Dockerfile (#4490)
+ * add start of spack base image for flux-core (#4471)
+ * docker: add pam development package to images (#4473)
+ * refactor broker overlay for topology flexibility (#4474)
+ * github: fixes for issues found when pushing a tag (#4462)
+
+## Testsuite
+ * testsuite: fix non-bourne shell test failure (#4543)
+ * testsuite: add more checkpoint sequence tests (#4518)
+ * testsuite: use flux jobs in valgrind workload (#4512)
+ * testsuite: unset `FLUX_F58_FORCE_ASCII` during testsuite (#4509)
+ * testsuite: add timeout job tests (#4501)
+ * testsuite: misc valgrind cleanup (#4480)
+
+
+flux-core version 0.42.0 - 2022-08-02
+-------------------------------------
+
+## New Features
+
+ * add flux_open_ex(3) (#4450)
+ * flux-top: support split of inactive jobs into completed and failed (#4449)
+ * job-manager: add limits plugins for duration and job size (#4415)
+ * kvs: add defensive checkpoint and kvs.checkpoint-period TOML config (#4383)
+ * python: add LSF URI resolver plugin (#4385)
+ * allow configurable defaults for jobspec system attributes (#4386)
+ * jobtap: add conf.update callback (#4411)
+ * Add a posix_spawn() implementation to libsubprocess and use it to launch
+   job shells (#4395)
+ * jobtap: add job.create, job.destroy callbacks (#4392)
+ * job-manager: allow dependencies on inactive jobs (#4388)
+
+## Fixes
+
+ * content-sqlite,files,s3: register with cache after setup complete (#4458)
+ * flux-overlay: add man page, open to guest users (#4459)
+ * flux-relay: initialize log prefix to hostname when possible (#4454)
+ * flux-top: avoid premature exit on recursive top error (#4452)
+ * job-manager: improve robustness of max job id recovery on restart (#4443)
+ * flux-config-bootstrap(5): improve hosts description (#4444)
+ * libflux: handle flux_respond_error (errnum=0) (#4427)
+ * flux-queue(1): add man page (#4426)
+ * sched-simple: fix allocation of down nodes when using constraints (#4425)
+ * job-archive: improve logging on parse job error (#4422)
+ * job-info: handle invalid eventlog entry errors more carefully (#4416)
+ * flux-dump: fix handling of empty blobref value (#4418)
+ * job-manager: fix race in job eventlog commit and job shell start (#4412)
+ * job-manager: fix dependency-add from job.state.depend callback (#4406)
+ * job-manager: ensure job aux items are destroyed safely (#4397)
+ * job-manager: fix restart code to handle jobs from earlier releases (#4399)
+
+## Cleanup
+
+ * use ccan ARRAY_SIZE() macro (#4445)
+ * kvs: rename kvs.sync target to kvs.wait-version (#4410)
+ * Use flux_error_t and errprintf() over char buf and snprintf() (#4407)
+ * content-sqlite: fix double free (#4391)
+ * kvs: misc cleanups (#4389)
+
+## Testsuite
+
+ * ci: create kvs dumpfile from previous tag for testing (#4402)
+
+flux-core version 0.41.0 - 2022-07-04
+-------------------------------------
+
+## New Features
+
+ * job-manager: transition NEW to DEPEND on "validate" event (#4366)
+ * kvs: support FLUX_KVS_SYNC (#4348)
+ * shell: obtain hwloc XML from enclosing instance (#4373)
+ * libkvs: add `flux_kvs_commit_get_rootref()`  (#4374)
+
+## Fixes
+
+ * job-manager: fix case where a job manager epilog can't be configured
+   without a prolog (#4382)
+ * broker: return error in content.flush if no backing store exists  (#4376)
+ * broker: content cache corner case corrections (#4380)
+ * job-manager: transition back to PRIORITY state on urgency update (#4364)
+ * wait for 'finish' instead of 'clean' event in flux-mini run and flux-job
+   attach (#4361)
+
+## Cleanup
+
+ * kvs: remove excess logging of ENOTSUP (#4381)
+ * job-manager: misc cleanup (#4362)
+
+## Testsuite
+
+ * testsuite: fix perilog sanity test (#4363)
+ * t2201-job-cmd.t: fix bug in UTF-8 test (#4360)
+
+flux-core version 0.40.0 - 2022-06-07
+-------------------------------------
+
+## New Features
+
+ * content-sqlite: verify database integrity during module load (#4340)
+ * job-exec: support new sdexec job launch plugin  (#4070)
+ * job-manager: post submit event, instead of job-ingest (#4346)
+ * shell: execute job tasks in their own process group (#4355)
+
+## Fixes
+
+ * shell: improve handling of TMPDIR (#4330)
+ * job-manager: do not send purged events (#4334)
+ * job-list: consistently return job attributes (#4327)
+ * python: fix confusing error message from pid URI resolver (#4335)
+ * improve logging of overlay peer authentication (#4342)
+ * libflux: return better errno in future wait path (#4345)
+ * shell: fix reconnect hang (#4293)
+ * libsubprocess: avoid segfault on empty argv (#4350)
+ * docs: add python resource_list docstrings (#4353)
+
+## Cleanup
+
+ * job-list: misc cleanup (#4332)
+ * job-manager: misc cleanup (#4352)
+
+## Testsuite
+
+ * docker: update default version of flux-security to v0.7.0 (#4356)
+
+
+flux-core version 0.39.0 - 2022-05-06
+-------------------------------------
+
+## New Features
+
+ * job-list: support new "all" attribute to get all job attributes (#4324)
+ * flux-overlay: replace --no-color with --color=WHEN (#4322)
+ * flux-overlay: add -H, --highlight option (#4322)
+ * flux-shutdown: add --gc garbage collection option (#4303)
+ * content: track RFC 10 protocol changes (#4299)
+ * flux-dmesg: colorize output and add -H, --human option (#4289)
+ * job-manager: add inactive job purge capability (#4286)
+ * libutil: support "ms" suffix for Flux Standard Duration (#4284)
+ * add flux-resource info subcommand (#4272)
+
+## Fixes
+
+ * python: uri: fix intermittent failure of fallback pid resolver (#4320)
+ * flux-job: fix purge usage message (#4318)
+ * use correct type in content-sqlite, misc. content test cleanup (#4315)
+ * job-archive: use safer pragmas (#4307)
+ * select safer content.sqlite consistency pragmas (#4294)
+ * sched-simple: do not allocate down nodes with --exclusive (#4292)
+ * python: fix return from `flux.importer.import_plugins()` when no plugins
+   found (#4288)
+ * fix hang when job with an interactive pty fails early (#4283)
+ * broker: prevent downstream peers from connecting during shutdown (#4277)
+
+## Cleanup
+
+ * flux-shutdown(1): document new options (#4323)
+ * README: add libarchive prerequisite (#4319)
+ * content-s3: cosmetic cleanup (#4314)
+ * job-list: misc cleanups (#4297)
+ * broker: suppress online message with no members (#4298)
+ * job-manager: introduce config callback (#4279)
+ * libev: update to version 4.33 (#4282)
+ * libflux: convert `flux_conf_error_t` to `flux_error_t` (#4278)
+ * resource: stop collecting/reducing hwloc XML  (#4263)
+ * flux-hwloc: remove command (#4262)
+ * flux-resource: remove ranks from default status output (#4271)
+ * libsubprocess: remove prefix on server setup (#4268)
+
+## Testsuite
+
+ * testsuite: increase test timeout (#4321)
+ * teststuite: document and fixup LONGTEST tests (#4305)
+ * testsuite: minor README fixes / updates (#4291)
+ * docker: update default flux-security version to v0.6.0 (#4274)
+ * testsuite: fix failing tests on parallel testsuite runs (#4275)
+ * ci: add build for Fedora 35 (#4270)
+
+flux-core version 0.38.0 - 2022-04-04
+-------------------------------------
+
+This release makes a few improvements that are visible in the flux-mini(1)
+command:
+
+ * The `-N,--nnodes` option may be used without the `-n,--nprocs` option.
+ * The `--exclusive` option requests the allocation of whole nodes.
+ * The `--requires` option requests resources with generic properties.
+
+Additionally, Flux system administrators should be aware of these changes:
+
+ * Named properties may be assigned to resources in the configured R.
+ * flux-shutdown(1) is now the preferred way to stop a Flux system instance.
+ * The default `archive.dbpath` is now `/var/lib/flux/job-archive.sqlite`.
+ * systemd-coredump(8) can now capture a Flux broker core dump.  Consider
+   enabling this on the management nodes of early access systems to help
+   gather information on critical rank 0 broker failures, should they occur.
+ * `flux resource drain` now shows nodes as "draining" if they are still
+   running work.
+ * Flux may be configured to reject jobs that do not run in a Flux sub-instance.
+
+For more information, see the Flux Administrator's Guide:
+
+https://flux-framework.readthedocs.io/en/latest/adminguide.html
+
+## New Features
+
+ * add flux-shutdown command (#4250)
+ * add flux-dump and flux-restore (#4208, #4225)
+ * support for node exclusive allocations (#4245)
+ * add support for resource properties (#4236)
+ * flux-resource: support properties when listing resources (#4249)
+ * job-ingest: add TOML config  (#4238)
+ * flux-dmesg: add --new option, plus logging cleanup (#4237)
+ * add 'require-instance' job validator plugin (#4239)
+ * job-manager: add builtin job duration validator plugin (#4224)
+ * sched-simple: set expiration of jobs with no duration to instance lifetime
+   (#4223)
+ * flux-resource: differentiate drained vs draining ranks (#4205)
+ * librlist: support hwloc discovery of AMD RSMI GPUs (#4203)
+ * broker: reject remote exec requests on rank 0 (#4258)
+ * python: allow resource count of 0 in jobspec v1 (#4259)
+ * job-archive: use statedir path if dbpath is not set (#4260)
+
+## Fixes
+
+ * content-sqlite: ensure that module load fails if initialization fails (#4265)
+ * job-archive: use statedir path if dbpath not set (#4260)
+ * broker: emit error when running interactive shell without tty (#4253)
+ * broker: add statedir attribute, drop content.backing-path (#4248)
+ * broker: prevent systemd restart if rc1 fails (#4246)
+ * flux.service: use StateDirectory for content.sqlite (#4244)
+ * rc3: ensure exit code reflects any errors (#4243)
+ * broker: don't leave shutdown state prematurely (#4241)
+ * libjob: improve `flux_job_statetostr()`, `flux_job_resulttostr()`
+   interface (#4235)
+ * job-list: fix bugs in some error paths (#4233)
+ * broker: fine tune logging and enable core dump on SIGSEGV (#4231)
+ * kvs: always store empty directory object to content store (#4229)
+ * restrict access to content service used as KVS blob layer (#4216)
+ * content-sqlite: check that file has rw permission (#4215)
+ * broker: block SIGPIPE (#4211)
+ * shell: add hostname to a couple key log messages (#4200)
+ * python: add missing methods and improve efficiency of IDset class (#4209)
+ * systemd: set SyslogIdentifier to flux (#4206)
+ * misc minor fixes and cleanup (#4197)
+ * job-exec: fix more potential hangs after early shell failure (#4199)
+ * sched-simple: fix counting bug that can cause scheduler to fail after
+   a restart (#4196)
+ * flux-top: add man page, minor bug fixes (#4194)
+
+## Cleanup
+
+ * broker: clean up shutdown logs (#4257)
+ * libsdprocess: minor fixups (#4252)
+ * job-manager: misc cleanup (#4232)
+
+## Testsuite
+
+ * testsuite: fix a couple intermittent test failures (#4247)
+ * ci: run 32bit build under linux32 personality (#4240)
+ * testsuite: ensure tests can run concurrently with `--root=$FLUX_JOB_TMPDIR`
+   (#4212)
+
+
+flux-core version 0.37.0 - 2022-03-04
+-------------------------------------
+
+This release disables resource verification of GPUs by default to
+workaround issues with GPU detection with system versions of hwloc.
+
+### Fixes
+
+ * resource: restrict resource verification to cores/hostnames only (#4192)
+ * resource: assign ranks in R based on hostlist attribute (#4188)
+ * add assertions that rank, size, hostlist broker attributes are cacheable
+   (#4187)
+
+### Testsuite
+
+ * testsuite: fix racy tests in t0005-rexec (#4179)
+
+### Cleanup
+
+ * build: ensure autogen.sh updates package version (#4174)
+
+
+flux-core version 0.36.0 - 2022-03-01
+-------------------------------------
+
+This release adds support for restarting a Flux system instance in safe
+mode after a failure to shut down properly -- for example in the case of a
+broker crash.  New `flux-startlog(1)` and `flux-uptime(1)` commands are
+also introduced to give a quick review of the start and stop times and
+status of the current Flux instance.
+
+System instance users will want to update their configuration files to
+set `tbon.tcp_user_timeout` and remove `tbon.keepalive_*`, if present.
+For more information, see the Flux Admin Guide:
+
+https://flux-framework.readthedocs.io/en/latest/adminguide.html
+
+### Fixes
+
+ * job-exec: fix job hang after early IMP/shell exit (#4155)
+ * broker: allow `tbon.zmqdebug` to be set in config file and make sure it's
+   really off if set to 0 (#4127)
+ * broker: handle network partition (#4130)
+ * shell: capture job shell error messages in designated output file (#4125)
+ * resource: emit a more specific error when `rlist_rerank()` fails (#4126)
+ * flux-overlay: fix timeout error message (#4131)
+ * README: add libc development packages in requirements (#4133)
+ * libflux/future: set missing errno in `flux_future_wait_for()`  (#4162)
+ * flux-config-archive(5): fix TOML example (#4164)
+ * shell: fix delay in completion of jobs with a single shell rank (#4159)
+
+### New Features
+
+ * flux-uptime: provide useful output for slow/stuck broker state (#4172)
+ * improve KVS checkpoint protocol to allow for future changes (#4149)
+ * add `flux config get` (#4166)
+ * broker: use RPC not control message for broker module sync/status (#4110)
+ * docs: add Python overview documentation (#4104)
+ * Support new libsdprocess to launch processes under systemd (#3864)
+ * rename keepalive messages to control messages (#4112)
+ * resource: enhance resource.drain RPC with "update" and "overwrite" modes
+   (#4121)
+ * broker: replace keepalive tunables with `tcp_user_timeout` (#4118)
+ * kvs: add date to kvs-primary checkpoint (#4136)
+ * libpmi2: implement bits needed for Cray MPI (#4142)
+ * add `flux-uptime` command (#4148)
+ * add `flux-startlog` and enter safe mode after crash (#4153)
+ * libflux: add `flux_hostmap_lookup(3)` (#4157)
+
+### Cleanup
+
+ * drop unused project meta files (#4170)
+ * doc: update flux-broker-attributes(7) (#4119)
+ * python: return `JobID` from flux.job.submit, not `int` (#4134)
+ * consolidate multiple `*_error_t` structures into a common `flux_error_t`
+   (#4165)
+ * drop unused project meta files (#4170)
+
+### Testsuite
+
+ * testsuite: remove unportable cshism (#4115)
+ * codecov: minor improvements for coverage reporting (#4147)
+ * testsuite: add clarification comments (#4167)
+
+
+flux-core version 0.35.0 - 2022-02-05
+-------------------------------------
+
+This release fixes a broker crash when a job receives an exception after
+running a job prolog. Users of the prolog/epilog feature should update
+to this version as soon as possible.
+
+In addition, TCP keepalive support was added for detection of powered off
+compute nodes.  For configuration info, see the Flux Admin Guide:
+
+https://flux-framework.readthedocs.io/en/latest/adminguide.html
+
+
+### Fixes
+
+ * flux-ping: support hostnames in TARGET #4105
+ * Fix broker segfault when an exception is raised on a job after prolog (#4096)
+ * flux-overlay: improve timeouts, hostname handling (#4095)
+ * flux resource: allow limited guest access (#4094)
+ * shell: fix duplicate logging after evlog plugin is initialized (#4085)
+ * shell: do not allow instance owner to use guest shell services (#4101)
+ * shell: fix race in attach to interactive job pty (#4102)
+ * libterminus: fix leak in pty client code (#4098)
+
+### New Features
+
+ * broker: use TCP keepalives (#4099)
+ * systemd: set restart=always (#4100)
+ * flux-mini: add --wait-event option to submit/bulksubmit (#4078)
+
+### Testsuite
+
+ * testsuite: fix spellcheck (#4082)
+ * ci: rename centos images to el, and use rockylinux for el8 image (#4080)
+
+
+flux-core version 0.34.0 - 2022-01-28
+-------------------------------------
+
+This release features the automatic draining of "torpid" (unresponsive)
+nodes, to prevent new work from being scheduled on them until the instance
+owner investigates and runs `flux resource undrain`.
+
+### Fixes
+
+ * libsubprocess: fix excess logging and logging corner cases (#4060)
+ * doc: fix cross-references (#4063)
+ * flux-proxy: improve experience when proxied Flux instance terminates
+   (#4058)
+ * flux-perilog-run: improve usefulness of logging when prolog/epilog fails
+   (#4054)
+ * Fix issues found on Cray Shasta (perlmutter) (#4050)
+ * env: fix prepend of colon-separated paths in reverse order (#4045)
+ * python: fix ImportError for collections.abc.Mapping (#4042)
+ * job-list: fix "duplicate event" errors (#4043)
+ * systemd: set linger on flux user (#4035)
+
+### New Features
+
+ * shell: enhance pty support (#4075)
+ * add broker.starttime; add uptime to flux-top, flux-pstree (#4076)
+ * libflux: add `flux_reconnect()`, revamp flux fatal error callback (#4016)
+ * doc: add/improve man pages for config files (#4057, #4069)
+ * resource: drain torpid nodes (#4052)
+
+### Cleanup
+
+ * broker/content: misc cleanup (#4074)
+ * improve error message from flux-proxy and flux-jobs for invalid and
+   unknown jobids (#4062)
+ * cmd/flux-ping: make help output clearer (#4061)
+ * configure: Add python docutils check, do not require doc utils to build
+   flux help (#4056)
+
+### Test
+
+ * testsuite: fix non-bourne shell test failure (#4064)
+ * sharness: unset `FLUX_CONF_DIR` for all tests (#4059)
+ * ci: fix use of poison-libflux.sh and add poison `flux-*` commands (#4046)
+
+
+flux-core version 0.33.0 - 2022-01-08
+-------------------------------------
+
+This release includes several improvements in the recursive tooling
+in Flux to enhance the user experience when dealing with nested jobs.
+
+Highlights include:
+
+ * Improved interface for job URI discovery allows `flux proxy` and
+   `flux top` to target jobids directly.
+ * Addition of a `-R, --recursive` option to `flux jobs`
+ * Support in `flux top` for selecting and recursing into sub-instances
+ * A new  `flux pstree` command for displaying job hierarchies in a tree
+
+### Fixes
+
+ * systemd: fix typo in flux.service unit file (#3996)
+ * libflux: check reactor flags (#4014)
+ * fix uninterruptible hang when attached to terminated jobs with -o pty
+   (#4010)
+ * cmd/flux-jobs: re-work -A option and clarify -a option (#4012)
+ * broker: avoid inappropriate quorum.timeout (#4027)
+ * add workaround for invalid job timeouts when system is busy (#4037)
+
+### New Features
+
+ * add FluxURIResolver Python class and flux-uri command for job URI
+   discovery (#3999)
+ * cmd: support high-level URIs and JOBID arguments in flux-top and
+   flux-proxy (#4004, #4015)
+ * flux-top: allow top to recursively call itself (#4011)
+ * flux-jobs: add --recursive option (#4019, #4024)
+ * flux-jobs: support instance-specific fields in output (#4022)
+ * add flux-pstree command (#4026)
+
+### Cleanup
+
+ * doc: add flux-resource(1),  clean up help output (#4021)
+ * doc: audit / cleanup SEE ALSO and RESOURCES, add cross references (#4007)
+ * doc: misc updates and fixes (#4009)
+ * connector cleanup (#4013)
+ * connectors: avoid embedded synchronous RPC for subscribe/unsubscribe
+   (#3997)
+
+### Test
+
+ * testsuite: minor testsuite fixes (#4023)
+ * ci: add ability to run tests under system instance (#3844)
+ * fluxorama: allow user to sudo to flux user, add common systemd environment
+   vars to flux user's bashrc (#4031)
+
+
+flux-core version 0.32.0 - 2021-12-05
+-------------------------------------
+
+This release adds early groundwork for recovering running jobs across a
+Flux restart.  It also includes improved log messages based on user feedback
+about Flux failures on real workflow runs, a first draft of a new `flux top`
+tool, and a critical fix for system deployments of Flux (#3958).
+
+### Fixes
+
+ * python: fix reference counting for Python Message objects (#3983)
+ * python: avoid early garbage collection of Watcher objects (#3975)
+ * libflux: improve safety against refcounting bugs in message functions (#3985)
+ * shell: reject PMI clients that request v2 (#3953)
+ * resource: don't abort if topo-reduce is received more than once (#3958)
+
+### New Features
+
+ * systemd: start flux systemd user service (#3872)
+ * broker: record child instance URIs as job memo (#3986)
+ * Support job memo events (#3984)
+ * job-exec: checkpoint/restore KVS namespaces of running jobs (#3947)
+ * set hostlist broker attribute when bootstrapped by PMI (#3966)
+ * add `flux_get_hostbyrank()` and improve broker attribute caching (#3971)
+ * broker: log slow nodes during startup (#3980)
+ * add flux-top command (#3979)
+
+### Cleanup
+
+ * flux-overlay: improve default status output, rework options (#3974)
+ * job-exec: improve job exception message/logging on broker disconnect (#3962)
+ * drop flux-jobspec command (#3951)
+ * improve flux-mini bulksubmit --dry-run output with --cc (#3956)
+ * README: update LLNL-CODE (#3954)
+ * broker/overlay: misc cleanup (#3948)
+ * bring README.md up to date (#3990)
+ * docker: fix and update ci dockerfiles (#3991)
+
+### Test
+
+ * testsuite: sanitize environment, fix hang in t2607-job-shell-input.t (#3968)
+
+flux-core version 0.31.0 - 2021-11-05
+-------------------------------------
+
+This release includes two noteworthy system instance improvements:
+crashed/offline nodes now marked offline for scheduling, and support
+has been added for prolog/epilog scripts that run as root.
+
+For prolog/epilog setup info, see the Flux Admin Guide:
+
+https://flux-framework.readthedocs.io/en/latest/adminguide.html
+
+### Fixes
+ * build: allow python 3.10.0 to satisfy version check (#3939)
+ * resource: avoid scheduling on nodes that have crashed (#3930)
+ * broker: fail gracefully when rundir or local-uri exceed `AF_UNIX` path
+   limits (#3932)
+ * broker: ignore missing ldconfig when searching for libpmi.so (#3926)
+ * job-manager: fix running job count underflow and use-after-free when an
+   event is emitted in CLEANUP state (#3922)
+ * fix problems building flux when 0MQ is not installed as a system package
+   (#3917)
+ * python: do not auto-stop ProgressBar by default (#3914)
+
+### New Features
+ * support job prolog/epilog commands (#3934)
+ * job-manager: add prolog/epilog support for jobtap plugins (#3924)
+ * libidset: add high level set functions (#3915)
+ * kvs: optionally initialize namespace to a root reference (#3941)
+ * rc: load job-archive module in default rc (#3942)
+
+### Cleanup
+ * improve broker overlay logging and debugging capability (#3913)
+ * man: Add note about shell quoting/escaping (#3918)
+
+### Test
+ * mergify: set queue method to merge, not rebase (#3916)
+ * mergify: replace strict merge with queue+rebase (#3907)
+ * testsuite: fix non-bourne shell test failure (#3937)
+
+flux-core version 0.30.0 - 2021-10-06
+-------------------------------------
+
+### Fixes
+
+ * job-manager: small fixes for the alloc-bypass plugin (#3889)
+ * job-manager: release after:JOBID dependencies after "start" instead of
+   "alloc" event (#3865)
+ * shell: avoid dropping stderr after a PMI abort (#3898)
+ * shell: require `FLUX_SHELL_PLUGIN_NAME` in plugins to fix logging component
+   discovery (#3879)
+ * libflux: deactivate RPC message handlers after final response (#3853)
+ * remove duplicate directories from `FLUX_RC_EXTRA`, `FLUX_SHELL_RC_PATH`
+   (#3878)
+ * t: fix incorrect method call in test-terminal.perl (#3888)
+ * Fix a couple build and test issues on ppc64le with clang 6.0+ (#3875)
+
+### New Features
+
+ * jobtap: allow jobtap plugins to query posted events for jobs (#3863)
+ * jobtap: allow jobtap plugins to subscribe to job events (#3861)
+ * job-exec: enable manual override option for mock execution jobs (#3868)
+ * shell: improve initrc extensibility, support version specific mpi plugin
+   loading (#3890)
+ * shell: fixes and enhancements for plugin loading (#3859)
+ * shell: allow default rc path to be extended via `FLUX_SHELL_RC_PATH` (#3869)
+ * shell: add taskids idset to `flux_shell_get_rank_info(3)` (#3873)
+ * shell: add library of Lua convenience functions for use in plugins (#3856)
+ * resource: fail get-xml request on quorum subset (#3885)
+
+### Cleanup
+
+ * libflux/future: fix comment typo (#3860)
+ * NEWS.md: Fix typo for v0.29.0 (#3857)
+
+### Testsuite
+
+ * docker: add --build-arg to docker-run-checks, systemd-devel to centos8
+   (#3871)
+ * ci: add fedora 34 build and fix compiler errors from gcc 11.2 (#3854)
+
+
+flux-core version 0.29.0 - 2021-09-03
+-------------------------------------
+
+This release of Flux includes a new fault mechanism which ensures that
+unanswered RPCs receive error responses when the overlay network is
+disrupted. Also included is a new `flux overlay` utility which can be
+used to manage and report the status of the overlay network.
+
+### Fixes
+ * shell: fix in-tree pluginpath, add `shell_plugindir` (#3841)
+ * python: fix bug in FluxExecutor.attach method (#3839)
+ * rlist: fix erroneous collapse of nodes with different resource children
+   when emitting R (#3814)
+ * libkvs: compact only if ops len > 1 (#3807)
+ * python: allow executor to attach to jobs (#3790)
+ * python: fix version requirement in jobspec validator plugin (#3784)
+ * broker: ensure subtree restart upon loss of router node (#3845)
+ * broker: drop -k-ary option, rename tbon.arity attr to tbon.fanout (#3796)
+ * add flux-job(1) manual page, plus minor fixes (#3763)
+ * libjob: improve method for determining instance owner (#3761)
+
+### New Features
+ * add flux overlay status command (#3816)
+ * broker: improve logging of 0MQ socket events (#3846)
+ * broker: fail pending RPCs when TBON parent goes down (#3843)
+ * broker: fail pending RPCs when TBON child goes down (#3822)
+ * add `bootstrap.ipv6_enable = true` config option (#3827)
+ * shell: add functions to access jobspec summary information (#3835)
+ * add stats api for internal metric collection (#3806, #3824)
+ * support io encode/decode of binary data (#3778)
+ * add flag to bypass jobspec validation (#3766)
+ * libflux: add time stamp to message trace (#3765)
+
+### Cleanup
+ * libzmqutil: generalize the zeromq authentication protocol server (#3847)
+ * libflux: use iovec-like array over zmsg (#3773)
+ * libflux: update flux msg route functions (#3746)
+ * libflux: message API fixes and cleanup (#3771)
+ * libjob: break up job.c (#3768)
+ * build: consistently use CFLAGS / LIBS in Makefiles (#3785)
+ * use CCAN base64 library over libsodium base64 library (#3789)
+ * drop unnecessary 0MQ includes (#3782)
+ * various other cleanup (#3762)
+
+### Testsuite
+ * update to flux-security v0.5.0 in docker images (#3849)
+ * make valgrind test opt-in (#3840)
+ * add valgrind suppression for opencl and libev on aarch64 (#3794, #3809)
+
+flux-core version 0.28.0 - 2021-06-30
+-------------------------------------
+
+This release adds simple job dependencies - see the `flux_mini(1)`
+DEPENDENCIES section.
+
+### Fixes
+ * shell: fix segfault when more slots are allocated than requested (#3749)
+ * testsuite: avoid long `ipc://` paths in system test personality (#3739)
+ * cron: fix race in task timeouts (#3728)
+ * Python/FluxExecutor bug fixes (#3714)
+ * flux-python: fix use of virtualenv python (#3713)
+ * optparse: make `optional_argument` apply to long options only (#3706)
+ * librlist: skip loading hwloc 'gl' plugin (#3693)
+
+### New Features
+ * allow jobs to bypass scheduler with alloc-bypass jobtap plugin (#3740)
+ * libjob: add a library for constructing V1 jobspecs (#3662, #3734, #3748)
+ * python: validate dependencies in Jobspec constructor (#3727)
+ * libflux: make `flux_plugin_handler` topic member const (#3720)
+ * job-manager: add builtin begin-time dependency plugin (#3704)
+ * broker: send offline responses while broker is initializing (#3712)
+ * python: add `flux.util.parse_datetime` (#3711)
+ * job-manager: support simple `after*` job dependencies (#3696)
+ * jobtap: fixes and api enhancements to support dependency plugins (#3698)
+ * shell: add exit-on-error option (#3692)
+
+### Cleanup/Testing/Build System
+ * job-manager: minor cleanup and improvements for event handling (#3759)
+ * libflux: make `flux_msg_fprint()` output clearer (#3742)
+ * libflux: store fully decoded message in `flux_msg_t` (#3701, #3758)
+ * libflux: msg API cleanup, test cleanup, and test enhancement  (#3745, #3699)
+ * testsuite: generalize valgrind suppressions (#3743)
+ * ci: use long path for builddir in test build (#3738)
+ * job-list: cleanup & testsuite modernization & consistency updates (#3733)
+ * testsuite: fix several tests on slower systems (#3730)
+ * testsuite: fix intermittent test, speed up others (#3725)
+ * broker: misc cleanup (#3721)
+ * github: fix ci builds on master (#3716, #3717)
+ * testsuite: add tests for late joining broker (#3709)
+ * flux-start: build system instance test features (#3700)
+ * ci: minor coverage testing fixes (#3703)
+ * libflux: test: fix segfault of `test_plugin.t` under rpmbuild (#3695)
+
+flux-core version 0.27.0 - 2021-05-28
+-------------------------------------
+
+This release features additional performance improvements that affect
+job throughput over time (see issue #3583).
+
+### Fixes
+ * shell/pmi: always populate `PMI_process_mapping` to avoid mvapich2
+   `MPI_Init` invalid tag error (#3673)
+ * openmpi: ensure that shmem segments for co-located jobs don't conflict
+   (#3672)
+ * python: fix FluxExecutorFuture cancellation bug (#3655)
+ * job-info, kvs-watch: support guest disconnect & credential checks (#3627)
+ * libflux: plugin: make `FLUX_PLUGIN_ARG_UPDATE` the default (#3685)
+
+### Performance
+ * kvs: reduce cache expiration overhead (#3664)
+ * kvs: remove client disconnect bottleneck (#3663)
+ * kvs: use json object to find duplicate keys (#3658)
+ * kvs: improve performance of transaction prep/check (#3654)
+ * content-cache: avoid linear search for dirty blobs (#3639)
+ * content-cache: make LRU purge more effective (#3632)
+ * flux-shell: add minor optimizations for single node jobs (#3626)
+ * libczmqcontainers: include zlistx, zhash, zlist, and convert internal
+   users (#3620)
+
+### New Features
+ * shell: add plugin to detect first task exit (#3681)
+ * job-manager: multiple jobtap plugin enhancements (#3687)
+ * job-manager: support a list of loaded jobtap plugins (#3667)
+ * shell: add tmpdir plugin to manage `FLUX_JOB_TMPDIR` (#3661)
+ * jobtap: support for `job.dependency.*` callbacks (#3660)
+ * flux-mini: avoid substitution without --cc/bcc, allow --setattr value
+   to be read from file (#3659)
+ * flux-start: add embedded server (#3650)
+ * flux-proxy: add flux-core version check (#3653)
+ * libflux: `msg_handler`: capture duplicate non-glob request handlers in a
+   stack (#3616)
+
+### Cleanup/Testing/Build System
+ * testsuite: add mvapich2 to centos8 CI image (#3686)
+ * testsuite: improve in-tree MPI testing (#3678)
+ * libflux: `flux_modfind`: ignore DSOs with no `mod_name` symbol (#3675)
+ * kvs: misc cleanup (#3671)
+ * flux-start: rename `--scratchdir` to `--rundir` (#3670)
+ * shell: misc environment-related fixes (#3669)
+ * testsuite: modify jobid capture logic (#3657)
+ * testsuite: handle hwloc issues and improve config file bootstrap test
+   (#3648)
+ * build: add and use autoconf variable for Flux plugin LDFLAGS (#3647)
+ * libutil: replace hand written hex conversion code with libccan (#3646)
+ * github: fixes for auto-release deployment (#3638)
+ * content-cache: general cleanup, small bug fixes, and test improvement
+   (#3645)
+ * kvs: add errmsg on namespace create failure (#3644)
+ * Use internal functions instead of zfile / zdigest (#3634)
+ * libutil: avoid `zmq_strerror()` (#3628)
+ * ci/test: switch to bulksubmit for inception tests, add throughput test,
+   dismiss reviews after PR updates (#3621)
+ * expand broker internal documentation to cover bootstrap phase (#3618)
+
+flux-core version 0.26.0 - 2021-04-22
+-------------------------------------
+
+This release features several performance improvements that affect
+job throughput over time (see issue #3583).
+
+### Fixes
+
+ * avoid mvapich segfault under flux start singleton (#3603)
+ * python: avoid throwing 2nd exception on unknown errno (#3588)
+ * avoid routing stale responses to restarted brokers (#3601)
+
+### Performance
+
+ * fix aggressive zhashx resize by importing fixed source (#3596, #3598)
+ * use zhashx, LRU in content-cache (#3589, #3593)
+ * drop root directory object from KVS setroot event (#3581)
+ * add minor optimizations to aux container (#3586)
+ * drop extra code in `flux_matchtag_free()` (#3590)
+ * libkvs: save KVS copy/move aux data in future not handle (#3585)
+
+### New Features
+
+ * libjob: add `flux_job_result(3)` (#3582)
+ * python: add explicit `service_(un)register` method (#3602)
+ * add overlay network version/config check (#3597)
+ * job-manager: enable job dependency management (#3563)
+
+### Cleanup/Testing
+
+ * flux-start: rename --size to --test-size, drop --bootstrap (#3605)
+
+flux-core version 0.25.0 - 2021-04-01
+-------------------------------------
+
+### Fixes
+
+ * kvs: fix assert due to busy KVS (#3560)
+ * systemd: configure weak dependency on munge (#3577)
+ * Fix various memleaks discovered by ASAN (#3568)
+ * README: add missing dependency - pkgconfig (#3570)
+ * fix `PMI_process_mapping` for multiple brokers per node (#3553)
+ * Python: fix "no such file or directory" job exception resulting from
+   bad jobspec (#3534)
+
+### New Features
+
+ * libflux: add `flux_plugin_aux_delete()` (#3565)
+ * job-info: support LRU cache mapping job id -> owner (#3548)
+ * python: expand FluxExecutor.submit parameters (#3562)
+ * broker: add support for PMIx bootstrap (#3537)
+ * job-ingest: add new plugin-based job validator (#3533)
+
+### Cleanup/Testing
+
+ * README.md: remove python3-six dependency (#3579)
+ * clean up disconnect, cancel handlers (#3569)
+ * broker: drop broker.rundir, select ipc vs tcp using broker.mapping (#3554)
+ * broker: refactor overlay network send/receive interfaces (#3547)
+ * github: add a stale issues and PR bot for flux-core (#3544)
+ * build/test: remove stale heartbeat references (#3535)
+ * job-info: consolidate watch RPC targets (#3525)
+ * enhance testsuite reliability on RHEL8/TOSS4 (#3540)
+
+
+flux-core version 0.24.0 - 2021-02-22
+-------------------------------------
+
+This release features multiple performance enhancements, including the
+addition of the FluxExecutor Python class which allows rapid, asynchronous
+submission of jobs.
+
+### Fixes
+
+ * broker: fix segfault/perf issues when hitting file descriptor limit (#3513)
+ * module: reduce keepalive message traffic (#3516)
+ * flux-kvs: fix --help output when not in an instance (#3500)
+ * flux-kvs: fix help output in nested subcommands (#3497)
+ * flux-mini: fix --progress counters with job exceptions (#3514)
+ * portability: fix 32-bit issues (#3507)
+ * portability: cross compilation fixes for Julia bindings (#3503)
+ * libflux: restart continuation timeout in `flux_future_reset()` (#3518)
+
+### New Features
+
+ * python: add concurrent.futures executor (#3468)
+ * libflux: add `flux_sync_create()` (#3524)
+ * job-manager: allow jobtap plugins to reject jobs (#3494)
+ * job-manager: support mode=limited (#3473)
+ * flux-mini: support `--urgency` values "default", "hold", "expedite" (#3499)
+ * broker: improve IP address heuristic in PMI bootstrap (#3489)
+ * flux-mini: add --log and --log-stderr options (#3509)
+ * use reactor time instead of heartbeats for internal time management (#3519)
+ * heartbeat: convert to loadable module (#3512)
+
+### Cleanup/Testing
+
+ * job-info: split into two modules, job-info and job-list (#3510)
+ * libflux: remove unnecessary `flux_future_then()` calls (#3520)
+ * testsuite: cleanup job-manager tests (#3488)
+ * testsuite: update hwloc-calc usage (#3523)
+ * ci: add fedora33 docker image for testing (#3498)
+ * ci: add 32 bit build to github ci checks (#3511)
+ * ci: explicitly checkout tag if creating a release (#3531)
+
+
+flux-core version 0.23.1 - 2021-01-27
+-------------------------------------
+
+### Fixes
+
+ * flux resource: allow drain, undrain, and status to work on any rank (#3486)
+ * job-manager: fix compilation error on gcc-10 (#3485)
+ * job-manager: fix uninitialized variable warning in jobtap.c (#3481)
+
+flux-core version 0.23.0 - 2021-01-25
+-------------------------------------
+
+This release adds a job priority plugin framework, enabling the
+flux-accounting project to set job priorities with a fair share
+algorithm.
+
+The scheduler protocol (RFC 27) and libschedutil convenience API
+have changed, therefore users of flux-sched must upgrade to 0.15.0.
+
+### New features
+
+ * jobtap: prototype job-manager plugin support (#3464)
+ * flux-mini: add bulk job submission capabilities (#3426, #3478)
+ * job-manager: send updated priorities to schedulers (#3442)
+ * job-manager: support job hold and expedite (#3428)
+
+### Fixes
+
+ * connectors/ssh: forward `LD_LIBRARY_PATH` over ssh when set (#3458)
+ * python: fix use of `Flux.reactor_run()` from multiple threads (#3471)
+ * python: misc. fixes to docstrings and argument names in bindings (#3451)
+ * python: fix circular reference in `check_future_error` decorator (#3437)
+ * python: fix ctrl-c, re-throw unhandled exceptions in `reactor_run()` (#3435)
+ * shell: fix dropped stdout from shell plugins in task.exec callback (#3446)
+
+### Cleanup/Testing
+
+ * ci: limit asan build to unit tests only (#3479)
+ * libschedutil: API improvements and priority integration (#3447)
+ * configure: add switch to allow flux to be built without python (#3459)
+ * testsuite: remove sched-dummy, migrate testing to sched-simple (#3462)
+ * testsuite: add debug, workarounds for failures in github actions (#3467)
+ * test: fix test for installing poison libflux (#3461)
+ * cleanup: update outdated terminology (#3456)
+ * Globally standardize spelling of "canceled" (#3443)
+ * ci: better script portability and other small updates (#3438)
+ * testsuite: fix invalid tests, cleanup list-jobs, avoid hard-coding (#3436)
+ * fix github actions on tag push (#3430)
+
+flux-core version 0.22.0 - 2020-12-16
+-------------------------------------
+
+This release resolves an issue introduced in 0.20.0 where Flux would
+occasionally hang during tear-down on RHEL/CentOS 7.  This release
+should be suitable for use with production workflows on those systems.
+
+System instance development and testing at < 256 node scale is on-going.
+The system limitations of this release are documented in the Flux Admin
+Guide:
+
+https://flux-framework.readthedocs.io/en/latest/adminguide.html
+
+### New features
+
+ * flux-keygen is no longer necessary before starting Flux (#3409)
+ * Add waitstatus and returncode JobID class and flux-jobs (#3414)
+ * New `flux resource status` command (#3351)
+ * Rename "administrative priority" to "urgency" (#3394)
+ * Prepare for fair share priority plugin (#3371, #3339, #3350, #3402,
+   #3405, #3404, #3410)
+ * job-manager: cache jobspec for scheduler, exec (#3393, #3396, #3399)
+ * python: add bindings for libflux-idset,hostlist (#3341)
+ * resource: support hostnames for drain and exclude (#3318)
+ * flux-jobs: Support nodelist in flux-jobs output (#3332)
+ * flux-jobs: add flux-jobs --stats,--stats-only options (#3419)
+ * flux-job: Add flux job attach --read-only option (#3320)
+ * python: add ResourceSet python class (#3406)
+ * python: allow future.then() variable and keyword args in callbacks (#3366)
+
+### Fixes
+
+ * Fix job shell segfault when jobspec contains JSON null (#3421)
+ * job-manager: Fix annotation clear corner case #3418
+ * broker: fix intermittent hang during instance tear-down on Centos7 (#3398)
+ * job-exec: log early shell/imp errors (#3397)
+ * shell: ensure TMPDIR exists for all jobs (#3389)
+ * misc cleanups & fixups (#3392)
+ * small fixes: resource memory leak, improve errors, check int size (#3388)
+ * affinity: use comma separated list format for `CUDA_VISIBLE_DEVICES` (#3376)
+ * libjob: repair interoperability with flux-security (#3356)
+ * job-exec: fixes for multiuser mode (#3353)
+ * shell: fix issues with `CUDA_VISIBLE_DEVICES` value (#3317)
+ * job-manager: handle scheduler disconnect (#3304)
+ * libjob: always sign owner jobs with the 'none' signing mechanism (#3306)
+ * libsubprocess: do not allow ref/unref in hooks (#3303)
+
+### Cleanup/Testing
+
+ * doc: autogenerate python binding docs with Sphinx (#3412)
+ * testsuite: support level N inception of flux testsuite (#3413)
+ * github: fix missing docker tag in matrix builds (#3387)
+ * github: fixes for workflow scripts (#3383)
+ * ci: move from Travis-CI to GitHub Workflows (#3379)
+ * docs: add explicit link to man pages section (#3365)
+ * testsuite: replace loop in t2300-sched-simple.t with helper (#3367)
+ * docker: install poison flux-core libs, cmds before build and test (#3369)
+ * libflux: drop `flux_dmesg()` from public API (#3362)
+ * testsuite: fix shed-simple test races (#3358)
+ * build: allow Lua 5.4, drop -lutil, and improve sphinx warning (#3357)
+ * testsuite: increase resource.monitor-waitup timeout (#3348)
+ * broker: update log.dmesg to use rpc streaming (#3307)
+ * testsuite: fix core idsets in resource module tests (#3314)
+ * t/t2205-hwloc-basic: only use lstopo-no-graphics (#3309)
+
+flux-core version 0.21.0 - 2020-11-04
+-------------------------------------
+
+This release enables resources to be configured in advance when Flux is
+the native resource manager for a cluster, in lieu of dynamic discovery.
+For details, refer to the Flux Admin Guide:
+
+https://flux-framework.readthedocs.io/en/latest/adminguide.html
+
+### New features
+
+ * optparse: don't sort options/subcommands by default (#3298)
+ * flux-job: Output key options for job info (#3210)
+ * resource: load resources from config or R, and rework topo discovery (#3265)
+ * add internal librlist library and flux-R utility for R version 1 (#3276)
+ * job-info: use job manager journal to track job state (#3254)
+ * job-manager: support events journal (#3261)
+ * shell: support stdio buffering options (default stderr: unbuffered) (#3272)
+ * flux-kvs: Add 'flux kvs eventlog wait-event' subcommand (#3200)
+ * job-manager: send job annotations to journal instead of publishing (#3236)
+ * add hostlist library for encoding/decoding RFC29 hostlists (#3247)
+
+### Fixes
+
+ * broker: convert broker [bootstrap] config to use libhostlist (#3283)
+ * libflux: Add missing C++ header guards (#3280)
+ * cmd: display jobid with flux-mini alloc -v, --verbose (#3279)
+ * python: fix signal handler management in threads (#3266)
+ * rc1: fix local connector retries (#3301)
+
+### Cleanup
+
+ * remove flux-hwloc reload command and aggregator module (#3296)
+ * doc: add flux-jobs(1) examples (#3295)
+ * job-manager / job-info: misc cleanup (#3246)
+ * build: increase minimum version of jansson to 2.10 (#3240)
+ * ci: ensure pylint script fails when lint warnings are produced (#3269)
+
+
+flux-core version 0.20.0 - 2020-09-30
+-------------------------------------
+
+This release features changes to support Flux as the native resource
+manager on small (<= 256 node) clusters, for testing only.  A draft system
+administration guide is available at:
+
+https://flux-framework.readthedocs.io/en/latest/adminguide.html
+
+### New features
+
+ * hwloc: add printing of num GPUs to `flux hwloc info` (#3217)
+ * resource: mark nodes down when they are stopped (#3207)
+ * broker:  allow late-joining brokers, execute rc1/rc3 on all ranks (#3168)
+ * shell/pmi: add improved PMI key exchange mechanism (#3219)
+
+### Fixes
+
+ * job-manager: communicate job priority changes to job-info (#3208)
+ * job-info: handle annotations race (#3196)
+ * python/job: Update `state_single` default header (#3227)
+ * libidset: reject idset strings that don't conform to RFC 22 (#3237)
+ * job-info: handle job-priority changes (#3208)
+ * doc: list sphinx as a doc dependency in README.md (#3225)
+ * testsuite: fix race in python SIGINT test (#3224)
+ * job-manager: fix segfault changing priority of a running job (#3220)
+ * shell: allow multiple resources per level in jobspec (#3175)
+ * python: allow Ctrl-C interrupt of `Future.get()` and `wait_for()` (#3215)
+ * shell: use F58/alternate encodings in output file template {{id}} (#3206)
+ * fallback to ASCII for F58 FLUIDs with `FLUX_F58_FORCE_ASCII` (#3204)
+ * rc: load sched-simple only if no other scheduler is loaded (#3177)
+ * docker: do not install Sphinx via pip in Centos8 image (#3195)
+ * flux-jobs / python bindings: handle empty string conversions (#3183)
+
+### Cleanup
+
+ * reduce log noise (#3226)
+ * flux-comms: remove obsolete command (#3211)
+
+
+flux-core version 0.19.0 - 2020-08-31
+-------------------------------------
+
+Notable features and improvements in this release include enhanced
+support for tools/debuggers (e.g. STAT, LaunchMON and TotalView), a
+new set of `--env` environment manipulation options for flux-mini(1),
+better support for listing jobs through the Python API, and a fix
+for an annoying usability issue with F58 encoded jobids in non-UTF-8
+locales.
+
+
+### New features
+
+ * switch to utf-8 for subprocess and job io encoding (#3086)
+ * improve support for shell plugin developers (#3159, #3132)
+ * flux-mini: add environment manipulation options (#3150)
+ * flux-mini: add --debug option for tools support (#3130)
+ * bash: top level command completions for flux (#2755)
+ * add fluxorama system instance docker image sources (#3031, #3128)
+ * content-s3: add configuration, support for libs3 variants (#3067, #3115)
+ * Use F58 JOBIDs in most user-facing commands (#3111)
+ * broker: state machine refactoring (#3107)
+ * broker: restore client-side PMI logging (#3105)
+ * libflux: add `flux_module_set_running()` (#3104)
+ * python: Add JobInfo, JobInfoFormat, and JobList classes (#3174)
+
+### Fixes
+
+ * Fix F58 encoding in non-multibyte locales (#3144)
+ * job-info,job-shell: allow non-V1 jobspec (#3160)
+ * build: fix innocuous configure error (#3129)
+ * travis-ci: fix ARGS when `DOCKER_TAG` set (#3125)
+ * doc: fix flux-help(1) output and rendering of NODESET.rst (#3119)
+ * flux-job: add `totalview_jobid` support and misc. fixes (#3130)
+ * small build/test/doc fixes (#3100)
+ * fix GitHub project license detection (#3089)
+ * shell/lua.d/openmpi: set env vars to force the use of flux plugins (#3099)
+ * job-info: do not fail on invalid jobspec / R / eventlog (#3171)
+ * flux-module: extend first column of flux-module list output (#3178)
+
+### Cleanup
+
+ * python: split flux.job module into multiple files (#3162)
+ * python: reformat with latest black formatter, pin black version (#3169)
+ * libflux: fix comment in module.h to reference readthedocs (#3138)
+ * Update rfc links to RTD site (#3137)
+ * remove the simple dynamic string (sds) code from libutil (#3135)
+ * Doc Cleanup (#3117)
+ * AUTHORS: remove (#3090)
+
+flux-core version 0.18.0 - 2020-07-29
+-------------------------------------
+
+This release features a more compact default representation for Flux JOBIDs,
+manual pages converted to ReST format and published on
+[readthedocs.io](https://flux-framework.readthedocs.io/projects/flux-core/),
+and the ability for schedulers to add data to jobs which can be displayed
+with `flux jobs`.
+
+### New features
+
+ * doc: man pages converted to ReST for publication on readthedocs.io
+   (#3033, #3078, #3085)
+ * Add support for RFC19 F58 encoded JOBIDs (#3045)
+ * Support user and scheduler job annotations (#3065, #3062, #2960)
+ * add content-s3, content-files alternate backing stores (#3025, #2992)
+ * Python interface to 'mini batch' (#3020)
+
+### Fixes
+
+ * shell: fix bug in cpu-affinity=per-task (#3080)
+ * flux-hwloc: remove ignore of `HWLOC_OBJ_GROUP` (#3046)
+ * cmd: Make label io options consistent (#3068)
+ * flux-resource list: Allow null/missing key to designate empty set (#3047)
+ * flux-jobs: small functionality and testing updates (#3060)
+ * job-manager: avoid segfault on priority change with pending alloc (#3072)
+
+### Cleanup
+
+ * doc: adjust dependency table to reflect hwloc v2.0+ support (#3053)
+ * Update terminology to use more inclusive words (#3040)
+
+### Testsuite enhancements
+
+ * testsuite: remove use of -S option in `run_timeout` (#3079)
+ * testsuite: minor valgrind test cleanup (#3077)
+ * docker: small updates for testenv images, travis builds (#3058)
+ * travis-ci: add python coverage (#3056)
+ * travis-ci: Add `--localstatedir=/var` to docker tag builds (#3050)
+ * pylint: Update pylint to 2.4.4 (#3035)
+ * Fix testsuite for Lua 5.3 on Ubuntu 20.04 (#3028)
+ * docker: really actually fix Ubuntu 20.04 (focal) docker tags (#3027)
+ * travis-ci: enforce correct configure ARGS for docker tags (#3023)
+ * travis: tag a docker image for ubuntu 20.04 (#3022)
+ * python: add stdio properties to Jobspec class (#3019)
+ * build and test fixes (#3016)
+
+
+flux-core version 0.17.0 - 2020-06-18
+-------------------------------------
+
+*NOTE*: Support has been removed for Python 2.
+
+### New features
+
+ * Improved interface for batch jobs: `flux mini batch` and `flux mini alloc`
+   (#2962)
+ * Pty support for Flux jobs via `-o pty` shell option (#2894)
+ * New resource module for monitoring and control of resources,
+   including ability to exclude and drain/undrain ranks. (#2918, #2949)
+ * New `flux resource` utility to drain and list resources. (#2949)
+ * Multiple improvements for `flux jobs`: colorize output, add "status"
+   and "exception" fields, allow jobids as positional arguments, and
+   add a custom conversion type `h` for "-" (#2798, #2858, #2902, #2910,
+   #2940, #2926, #2865)
+ * Support for hwloc v2.0+ (#2944)
+ * Support for MPIR debugging of jobs (#2654)
+ * New job-archive module optionally stores job data in sqlite. (#2880)
+ * single-broker system instance support, including minimal
+   support for restart (archived job information is saved) (#2783, #2820,
+   #2813, #2809)
+ * Add support for multi-user execution (#2822, #2813)
+ * Add support for enforcing job time limits (#2995)
+ * python: Add bindings for job cancel and kill (#2976)
+ * python: Add bindings for watching job eventlog events (#2986)
+
+### Improvements
+
+ * support systemctl reload flux (#2879)
+ * enhance job throughput (#2777, #2792)
+ * sched-simple: schedule cores instead of PUs by default (#2966)
+ * broker: send service.disconnect requests on module unload (#2913)
+ * broker: add interface for monitoring broker liveness (#2914)
+ * broker: add cleanup phase (#2971)
+ * broker: only allow userid- services to be registered by guests (#2813)
+ * libflux: add `flux_msg_last_json_error(3)` (#2905)
+ * flux-job: Use common attrs for list cmds (#2901)
+ * doc: add flux job shell API manpages (#2793)
+ * job-info: Support "exception" and "success" list attributes (#2831, #2858)
+ * job-info: improve error responses from various list RPCs (#3010)
+ * rc: load job-info on rank 0 only (#3009)
+ * python: remove support for Python 2 (#2805)
+ * python: cache python wrappers in the class (#2878)
+ * python: tweaks in preparation for flux-tree-helper (#2804)
+ * python: add 'flux_job_list_inactive' Python binding (#2790)
+ * python: allow reactor_run() to be interrupted (#2974)
+ * config: parse TOML once in broker, share with modules (#2866)
+ * config: use config file for access policy (#2871)
+ * docker: add default PS1 that includes flux instance size, depth (#2925)
+ * docker: start munge in published docker images (#2922)
+
+### Fixes
+
+ * Fix compilation under GCC 10.1.0 (#2954)
+ * librouter: avoid dropping messages on EPIPE (#2934)
+ * README: update documentation link (#2929)
+ * README.md: fix required Lua version (#2923)
+ * README: add missing dependencies: aspell-en and make (#2889)
+ * shell: make registered services secure by default (#2877)
+ * cmd/flux-kvs: Fix segfault in dir -R (#2847)
+ * job-exec: drop use of broker attrs, use conf file or cmdline instead
+   (#2821)
+ * broker: clean shutdown on SIGTERM (#2794)
+ * flux-ping: fix problems with route string (#2811)
+ * libsubprocess:  don't clobber errno in destructors, handle ENOMEM (#2808)
+ * Fix flux-job status for jobs with exceptions before start (#2784)
+ * shell: Add missing R member to shell info JSON object (#2989)
+ * job-ingest: fix validation of V1 jobspec (duration required) (#2994)
+ * doc: fixes and updates for idset manpages (#3012)
+
+### Cleanup
+
+ * removed outdated pymod module (#3008)
+ * broker and flux-comms cleanup (#2907)
+ * cmd/flux-kvs: Remove legacy --json options and json output (#2807)
+ * doc: Fix typos in man pages (#2725)
+ * libutil: improve out of memory handling, conform to RFC 7 (#2785)
+ * content-sqlite, content-cache: cleanup and refactoring (#2786)
+
+### Testsuite enhancements
+
+ * Fix skipped tests in t2205-hwloc-basic.t (#2998)
+ * t2204-job-info: Split up tests into new files (#2957)
+ * t/t2800-jobs-cmd: Fix racy test (#2951)
+ * t: add `HAVE_JQ` prereq to tests that use `jq` (#2936)
+ * sharness: fix TEST_CHECK_PREREQS for tests using $jq (#2939)
+ * job-info: module & test cleanup (#2932)
+ * testsuite: add ability to ensure programs are used under appropriate
+   prereqs (#2937)
+ * ensure unit tests do not link against installed flux libraries (#2917)
+ * t2204-job-info: Fix racy tests (#2862)
+ * test rehab: new flexible run_timeout, speeding up asan, and many more
+   timeouts and test repairs (#2849)
+ * Mypy: add static type checking for python to travis (#2836)
+ * testsuite: minor fixes and slight improvements (#2842)
+ * README: update Travis CI badge after transition to travis-ci.com (#2843)
+ * tests: timeout in automake harness (#2840)
+ * t/t0005-exec: Increase timeout lengths (#2828)
+
+
 flux-core version 0.16.0 - 2020-02-24
 -------------------------------------
 
@@ -662,7 +3473,7 @@ flux-core version 0.6.0 - 2016-11-29
  * Fix for possible unconstrained memory growth in modules/libjsc (#891)
  * Fix error message on flux-help failure (#887)
  * Issue fatal error in wrexecd for invalid tasks on node (#901)
- * Fix barrier protocol incompatability with older jansson versions (#889)
+ * Fix barrier protocol incompatibility with older jansson versions (#889)
 
 #### New Features
 
@@ -701,7 +3512,7 @@ flux-core version 0.5.0 - 2016-10-27
 * Add hierarchical lwj directory support in kvs (#811)
 * doc/man1/flux-start.adoc: Fix example option usage (#852)
 * add dlopen RTLD_DEEPBIND flag (#849)
-* src/broker/broker.c: Fix typo flux_repond -> flux_respond (#851)
+* src/broker/broker.c: Fix typo (#851)
 * doc/man1/flux-module.adoc: Fix environment variable error (#850)
 * Pull in json-c, allowing internals to link against alternate json libraries. (#835)
 * Add enhanced flux_rpc functions using libjansson json_pack/unpack functions
@@ -794,7 +3605,7 @@ flux-core version 0.4.0 - 2016-08-11
 
 * Sophia content backing store module (#727)
 
-* mrpc KVS based muti-RPC interface (#689)
+* mrpc KVS based multi-RPC interface (#689)
 
 * ZPL config file (#674)
 
@@ -834,7 +3645,7 @@ flux-core version 0.3.0 - 2016-04-26
 
 * Add module status reporting via keepalive messages.
   `flux module list` now reports live module status:
-  - I = intializing
+  - I = initializing
   - S = sleeping
   - X = exited
   - R = running
diff --git a/NOTICE b/NOTICE
deleted file mode 120000
index a45239c629bb..000000000000
--- a/NOTICE
+++ /dev/null
@@ -1 +0,0 @@
-NOTICE.LLNS
\ No newline at end of file
diff --git a/README b/README
deleted file mode 100644
index 69b3a83d6de7..000000000000
--- a/README
+++ /dev/null
@@ -1 +0,0 @@
-See the README.md file.
diff --git a/README.md b/README.md
index fd238a789db7..3355fc3b9816 100644
--- a/README.md
+++ b/README.md
@@ -1,194 +1,64 @@
-[![Build Status](https://travis-ci.org/flux-framework/flux-core.svg?branch=master)](https://travis-ci.org/flux-framework/flux-core)
-[![Coverage Status](https://coveralls.io/repos/flux-framework/flux-core/badge.svg?branch=master&service=github)](https://coveralls.io/github/flux-framework/flux-core?branch=master)
+[![ci](https://github.com/flux-framework/flux-core/workflows/ci/badge.svg)](https://github.com/flux-framework/flux-core/actions?query=workflow%3A.github%2Fworkflows%2Fmain.yml)
+[![codecov](https://codecov.io/gh/flux-framework/flux-core/branch/master/graph/badge.svg)](https://codecov.io/gh/flux-framework/flux-core)
 
-_NOTE: The interfaces of flux-core are being actively developed
-and are not yet stable._ The github issue tracker is the primary
-way to communicate with the developers.
+See our [Online Documentation](https://flux-framework.readthedocs.io)!
 
-See also the flux-framework.org [Online Documentation](http://flux-framework.org/docs/home/).
+_NOTE: the github issue tracker is the primary way to communicate
+with Flux developers._
 
-### flux-core
 
-flux-core implements the communication layer and lowest level
-services and interfaces for the Flux resource manager framework.
-It consists of a distributed message broker, plug-in _comms modules_
-that implement various distributed services, and an API and set
-of utilities to utilize these services.
+### flux-core
 
-flux-core is intended to be the first building block used in the
-construction of a site-composed Flux resource manager.  Other building
-blocks are also in development under the
+flux-core implements the lowest level services and interfaces for the Flux
+resource manager framework.  It is intended to be the first building block
+used in the construction of a site-composed Flux resource manager.  Other
+building blocks are also in development under the
 [flux-framework github organization](https://github.com/flux-framework),
-including a fully functional workload
-[scheduler](https://github.com/flux-framework/flux-sched).
+including a workload [scheduler](https://github.com/flux-framework/flux-sched).
 
 Framework projects use the C4 development model pioneered in
 the ZeroMQ project and forked as
-[Flux RFC 1](https://github.com/flux-framework/rfc/blob/master/spec_1.adoc).
+[Flux RFC 1](https://flux-framework.rtfd.io/projects/flux-rfc/en/latest/spec_1.html).
 Flux licensing and collaboration plans are described in
-[Flux RFC 2](https://github.com/flux-framework/rfc/blob/master/spec_2.adoc).
+[Flux RFC 2](https://flux-framework.rtfd.io/projects/flux-rfc/en/latest/spec_2.html).
 Protocols and API's used in Flux will be documented as Flux RFC's.
 
 #### Build Requirements
 
-flux-core requires the following packages to build:
-
-**redhat**        | **ubuntu**        | **version**       | **note**
-----------        | ----------        | -----------       | --------
-autoconf          | autoconf          |                   |
-automake          | automake          |                   |
-libtool           | libtool           |                   |
-libsodium-devel   | libsodium-dev     | >= 1.0.14         |
-zeromq4-devel     | libzmq3-dev       | >= 4.0.4          |
-czmq-devel        | libczmq-dev       | >= 3.0.1          |
-jansson-devel     | libjansson-dev    | >= 2.6            |
-libuuid-devel     | uuid-dev          |                   |
-lz4-devel         | liblz4-dev        |                   |
-hwloc-devel       | libhwloc-dev      | >= v1.11.1, < 2.0 |
-sqlite-devel      | libsqlite3-dev    | >= 3.0.0          |
-lua               | lua5.1            | >= 5.1, < 5.3     |
-lua-devel         | liblua5.1-dev     | >= 5.1, < 5.3     |
-lua-posix         | lua-posix         |                   | *1*
-python36-devel    | python3-dev       | >= 2.7            |
-python36-cffi     | python3-cffi      | >= 1.1            |
-python36-six      | python3-six       | >= 1.9            |
-python36-yaml     | python3-yaml      | >= 3.10.0         |
-python36-jsonschema | python3-jsonschema | >= 2.3.0       |
-asciidoc          | asciidoc          |                   | *2*
-asciidoctor       | asciidoctor       | >= 1.5.7          | *2*
-aspell            | aspell            |                   | *3*
-valgrind          | valgrind          |                   | *3*
-mpich             | mpich             |                   | *3*
-jq                | jq                |                   | *3*
-
-*Note 1 - Due to a packaging issue, Ubuntu lua-posix may need the
-following symlink (true for version 33.4.0-2):*
-```
-$ sudo ln -s posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.1/posix.so
-```
-
-*Note 2 - only needed if optional man pages are to be created.  Only one
-of asciidoc or asciidoctor is needed.  Asciidoc is used if both are installed.*
-
-*Note 3 - optional, for enabling additional tests*.
-
-##### Installing RedHat/CentOS Packages
-```
-yum install autoconf automake libtool libsodium-devel zeromq4-devel czmq-devel libuuid-devel jansson-devel lz4-devel hwloc-devel sqlite-devel lua lua-devel lua-posix python36-devel python36-cffi python36-six python36-yaml python36-jsonschema asciidoc asciidoctor aspell valgrind mpich jq
-```
-
-##### Installing Ubuntu Packages
-```
-apt install autoconf automake libtool libsodium-dev libzmq3-dev libczmq-dev uuid-dev libjansson-dev liblz4-dev libhwloc-dev libsqlite3-dev lua5.1 liblua5.1-dev lua-posix python3-dev python3-cffi python3-six python3-yaml python3-jsonschema asciidoc asciidoctor aspell valgrind mpich jq
-```
+For convenience, scripts that install flux-core's build dependencies
+are available for [redhat](scripts/install-deps-rpm.sh) and
+[debian](scripts/install-deps-deb.sh) distros.
 
 ##### Building from Source
 ```
 ./autogen.sh   # skip if building from a release tarball
-PYTHON_VERSION=3.6 ./configure
+./configure
 make
 make check
 ```
-Note: the `PYTHON_VERSION` environment variable adds a suffix to the
-python interpreter executable.  Configure would look for `python3.6` in
-the example above.  If unset, `python` is used, which is often Python 2.
-
-If you want Flux to use Python 2 and generate (only) Python 2 bindings,
-alter the prerequisite package names above, and set (or don't set)
-`PYTHON_VERSION` accordingly.  Python 2 support should be considered
-deprecated, although it continues to work for now.
-
-#### Bootstrapping a Flux instance
 
-A Flux instance is composed of a set of `flux-broker` processes
-that bootstrap via PMI (e.g. under another resource manager), or locally
-via the `flux start` command.
+##### VSCode Dev Containers
 
-No administrator privilege is required to start a Flux instance
-as described below.
+If you use VSCode we have a dev container and [instructions](vscode.md).
 
-##### Single node session
+#### Starting Flux
 
-To start a Flux instance (size = 8) on the local node for testing:
+A Flux instance is composed of a set of `flux-broker` processes running as
+a parallel job and can be started by most launchers that can start MPI jobs.
+Doing so for a single user does not require administrator privilege.
+To start a Flux instance (size = 8) on the local node for testing, use
+flux's built-in test launcher:
 ```
-src/cmd/flux start --size 8
+src/cmd/flux start --test-size=8
 ```
-A shell is spawned that has its environment set up so that Flux
-commands can find the message broker socket.  When the shell exits,
-the session exits.
-
-##### SLURM session
-
-To start a Flux instance (size = 64) on a cluster using SLURM,
-first ensure that MUNGE is set up on your cluster, then:
-```
-srun --pty --mpi=none -N64 src/cmd/flux start
-```
-The srun --pty option is used to connect to the rank 0 shell.
-When you exit this shell, the session terminates.
-
-#### Flux commands
-
-Within a session, the path to the `flux` command associated with the
-session broker will be prepended to `PATH`, so use of a relative or
-absolute path is no longer necessary.
-
-To see a list of commonly used commands run `flux` with no arguments,
-`flux help`, or `flux --help`
-```
-$ flux help
-Usage: flux [OPTIONS] COMMAND ARGS
-  -h, --help             Display this message.
-  -v, --verbose          Be verbose about environment and command search
-
-Common commands from flux-core:
-   broker             Invoke Flux comms message broker daemon
-   content            Access instance content storage
-   cron               Schedule tasks on timers and events
-   dmesg              manipulate broker log ring buffer
-   env                Print or run inside a Flux environment
-   event              Send and receive Flux events
-   exec               Execute processes across flux ranks
-   get,set,lsattr     Access, modify, and list broker attributes
-   hwloc              Control/query resource-hwloc service
-   keygen             generate keys for Flux security
-   kvs                Flux key-value store utility
-   logger             create a Flux log entry
-   module             manage Flux extension modules
-   ping               measure round-trip latency to Flux services
-   proxy              Create proxy environment for Flux instance
-   ps                 List subprocesses managed by brokers
-   start              bootstrap a local Flux instance
-   submit             submit job requests to a scheduler
-   user               Flux user database client
-```
-
-Most of these have UNIX manual pages as `flux-<sub-command>(1)`,
-which can also be accessed using `./flux help <sub-command>`.
-
-#### A note about PMI
-
-When flux is launched, it requires PMI-1 in order to bootstrap.
-It can use PMI-1 in one of two ways, by inheriting a file descriptor
-via the `PMI_FD` environment variable, or by dlopening a PMI library.
-The library name is `libpmi.so`, unless overridden by the `PMI_LIBRARY`
-environment variable.  If a PMI library is not found, flux falls back
-to "singleton" operation, where each broker is an independent flux instance.
-The PMI bootstrap may be traced by setting the `FLUX_PMI_DEBUG` environment
-variable.
-
-When flux launches flux or an MPI job, it provides PMI-1 to bootstrap the
-MPI's runtime.  It offers a PMI server and sets the `PMI_FD` environment
-variable to point to an open file descriptor connected to it.  It also offers
-a `libpmi.so` library that can be dlopened.
+A shell is spawned in which Flux commands can be executed.  When the shell
+exits, Flux exits.
 
-If your system process manager uses PMIx, the `libpmi.so` compatibility library
-provided by the PMIx project should be sufficient to bootstrap flux.
-If your version of PMIx was not built with the compatibility libraries
-installed, you may build libpmix as a separate package to get them installed.
+For more information on starting Flux in various environments and using it,
+please refer to our [docs](https://flux-framework.readthedocs.io) pages.
 
 #### Release
 
 SPDX-License-Identifier: LGPL-3.0
 
-LLNL-CODE-76440
+LLNL-CODE-764420
diff --git a/autogen.sh b/autogen.sh
index 15729ebd1ca3..d04e30e459ca 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -6,6 +6,6 @@
 #
 echo "Running libtoolize --automake --copy ... "
 libtoolize --automake --copy || exit
-echo "Running autoreconf --verbose --install"
-autoreconf --verbose --install || exit
+echo "Running autoreconf --force --verbose --install"
+autoreconf --force --verbose --install || exit
 echo "Now run ./configure."
diff --git a/codecov.yml b/codecov.yml
index af4b65899159..d6dd3d6e1da8 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -9,10 +9,11 @@ coverage:
  - ".*/common/libtap/.*"
  - ".*/common/liblsd/.*"
  - ".*/common/libev/.*"
+ - ".*/common/libtomlc99/.*"
  - ".*/common/liboptparse/getopt*"
- - ".*/bindings/python/.*"
- - ".*/common/libutil/sds.*"
  - ".*/common/libminilzo/.*"
+ - "src/bindings/python/_flux"
+ - "src/bindings/python/flux/utils"
  - "/usr/include/.*"
  - "/usr/lib/.*"
 
@@ -25,3 +26,8 @@ coverage:
 comment:
  layout: "header, diff, changes, tree"
  behavior: new
+ after_n_builds: 2
+
+codecov:
+ notify:
+  after_n_builds: 2
diff --git a/config/ac_pkgconfig.m4 b/config/ac_pkgconfig.m4
index a18523b77c62..da2211f5598b 100644
--- a/config/ac_pkgconfig.m4
+++ b/config/ac_pkgconfig.m4
@@ -5,7 +5,7 @@ AC_DEFUN([AC_PKGCONFIG],
     pkgconfigdir='${libdir}/pkgconfig'
     AC_MSG_CHECKING(whether to install pkg-config *.pc files)
     AC_ARG_WITH(pkgconfig-dir,
-     AC_HELP_STRING([--with-pkgconfig-dir=PATH], [where to install pkg-config *.pc files (EPREFIX/lib/pkgconfig)]),
+     AS_HELP_STRING([--with-pkgconfig-dir=PATH],[where to install pkg-config *.pc files (EPREFIX/lib/pkgconfig)]),
      [
         case "${withval}" in
             yes|auto)
diff --git a/config/am_check_pymod.m4 b/config/am_check_pymod.m4
index bcf59cfa7fdb..33c5d1cfb3c2 100644
--- a/config/am_check_pymod.m4
+++ b/config/am_check_pymod.m4
@@ -15,12 +15,15 @@ except:
         sys.exit(0)
 sys.exit(0)"], [prog="
 import sys
-from distutils.version import LooseVersion, StrictVersion
+try:
+    from distutils.version import StrictVersion as Version
+except ModuleNotFoundError:
+    from packaging.version import Version
 import $1
 if not $2:
     sys.exit(1)
 "])
-if $PYTHON -c "$prog" 1>&AC_FD_CC 2>&AC_FD_CC
+if $PYTHON -c "$prog" 1>&AS_MESSAGE_LOG_FD 2>&AS_MESSAGE_LOG_FD
   then
     eval "py_cv_mod_$py_mod_var=yes"
   else
diff --git a/config/ax_code_coverage.m4 b/config/ax_code_coverage.m4
index 6484f0332435..dbdaf72e537c 100644
--- a/config/ax_code_coverage.m4
+++ b/config/ax_code_coverage.m4
@@ -125,10 +125,10 @@ AC_DEFUN([AX_CODE_COVERAGE],[
 		dnl Build the code coverage flags
 		dnl Define CODE_COVERAGE_LDFLAGS for backwards compatibility
 		CODE_COVERAGE_CPPFLAGS="-DNDEBUG"
-		CODE_COVERAGE_CFLAGS="-O0 -g -fprofile-arcs -ftest-coverage"
-		CODE_COVERAGE_CXXFLAGS="-O0 -g -fprofile-arcs -ftest-coverage"
+		CODE_COVERAGE_CFLAGS="-O0 -g -fprofile-arcs -ftest-coverage -fprofile-update=atomic"
+		CODE_COVERAGE_CXXFLAGS="-O0 -g -fprofile-arcs -ftest-coverage -fprofile-update=atomic"
 		CODE_COVERAGE_LIBS="-lgcov"
-		CODE_COVERAGE_LDFLAGS="$CODE_COVERAGE_LIBS"
+		CODE_COVERAGE_LDFLAGS="$CODE_COVERAGE_LIBS -fprofile-update=atomic"
 
 		AC_SUBST([CODE_COVERAGE_CPPFLAGS])
 		AC_SUBST([CODE_COVERAGE_CFLAGS])
diff --git a/config/ax_compile_check_sizeof.m4 b/config/ax_compile_check_sizeof.m4
index 00d0bcc254ec..f834df6346c9 100644
--- a/config/ax_compile_check_sizeof.m4
+++ b/config/ax_compile_check_sizeof.m4
@@ -1,6 +1,6 @@
-# ===========================================================================
-#  http://www.gnu.org/software/autoconf-archive/ax_compile_check_sizeof.html
-# ===========================================================================
+# ============================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_compile_check_sizeof.html
+# ============================================================================
 #
 # SYNOPSIS
 #
@@ -24,8 +24,8 @@
 #
 #     switch (0) case 0: case 0:;
 #
-#   Thus, the AC_TRY_COMPILE will fail if the currently tried size does not
-#   match.
+#   Thus, the AC_COMPILE_IFELSE will fail if the currently tried size does
+#   not match.
 #
 #   Here is an example skeleton configure.in script, demonstrating the
 #   macro's usage:
@@ -57,6 +57,7 @@
 # LICENSE
 #
 #   Copyright (c) 2008 Kaveh Ghazi <ghazi@caip.rutgers.edu>
+#   Copyright (c) 2017 Reini Urban <rurban@cpan.org>
 #
 #   This program is free software: you can redistribute it and/or modify it
 #   under the terms of the GNU General Public License as published by the
@@ -69,7 +70,7 @@
 #   Public License for more details.
 #
 #   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see <http://www.gnu.org/licenses/>.
+#   with this program. If not, see <https://www.gnu.org/licenses/>.
 #
 #   As a special exception, the respective Autoconf Macro's copyright owner
 #   gives unlimited permission to copy, distribute and modify the configure
@@ -84,7 +85,7 @@
 #   modified version of the Autoconf Macro, you may extend this special
 #   exception to the GPL to apply to your modified version as well.
 
-#serial 5
+#serial 8
 
 AU_ALIAS([AC_COMPILE_CHECK_SIZEOF], [AX_COMPILE_CHECK_SIZEOF])
 AC_DEFUN([AX_COMPILE_CHECK_SIZEOF],
@@ -97,10 +98,10 @@ changequote([, ])dnl
 AC_MSG_CHECKING(size of $1)
 AC_CACHE_VAL(AC_CV_NAME,
 [for ac_size in 4 8 1 2 16 $3 ; do # List sizes in rough order of prevalence.
-  AC_TRY_COMPILE([#include "confdefs.h"
+  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <sys/types.h>
 $2
-], [switch (0) case 0: case (sizeof ($1) == $ac_size):;], AC_CV_NAME=$ac_size)
+]], [[switch (0) case 0: case (sizeof ($1) == $ac_size):;]])], [AC_CV_NAME=$ac_size])
   if test x$AC_CV_NAME != x ; then break; fi
 done
 ])
diff --git a/config/ax_python_devel.m4 b/config/ax_python_devel.m4
index 59a2ff090302..1553101afb69 100644
--- a/config/ax_python_devel.m4
+++ b/config/ax_python_devel.m4
@@ -1,10 +1,10 @@
 # ===========================================================================
-#      http://www.gnu.org/software/autoconf-archive/ax_python_devel.html
+#     https://www.gnu.org/software/autoconf-archive/ax_python_devel.html
 # ===========================================================================
 #
 # SYNOPSIS
 #
-#   AX_PYTHON_DEVEL([version])
+#   AX_PYTHON_DEVEL([version[,optional]])
 #
 # DESCRIPTION
 #
@@ -12,8 +12,8 @@
 #   in your configure.ac.
 #
 #   This macro checks for Python and tries to get the include path to
-#   'Python.h'. It provides the $(PYTHON_CPPFLAGS) and $(PYTHON_LDFLAGS)
-#   output variables. It also exports $(PYTHON_EXTRA_LIBS) and
+#   'Python.h'. It provides the $(PYTHON_CPPFLAGS) and $(PYTHON_LIBS) output
+#   variables. It also exports $(PYTHON_EXTRA_LIBS) and
 #   $(PYTHON_EXTRA_LDFLAGS) for embedding Python in your code.
 #
 #   You can search for some particular version of Python by passing a
@@ -23,6 +23,11 @@
 #   version number. Don't use "PYTHON_VERSION" for this: that environment
 #   variable is declared as precious and thus reserved for the end-user.
 #
+#   By default this will fail if it does not detect a development version of
+#   python.  If you want it to continue, set optional to true, like
+#   AX_PYTHON_DEVEL([], [true]).  The ax_python_devel_found variable will be
+#   "no" if it fails.
+#
 #   This macro should work for all versions of Python >= 2.1.0. As an end
 #   user, you can disable the check for the python version by setting the
 #   PYTHON_NOVERSIONCHECK environment variable to something else than the
@@ -52,7 +57,7 @@
 #   Public License for more details.
 #
 #   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see <http://www.gnu.org/licenses/>.
+#   with this program. If not, see <https://www.gnu.org/licenses/>.
 #
 #   As a special exception, the respective Autoconf Macro's copyright owner
 #   gives unlimited permission to copy, distribute and modify the configure
@@ -67,10 +72,18 @@
 #   modified version of the Autoconf Macro, you may extend this special
 #   exception to the GPL to apply to your modified version as well.
 
-#serial 17
+#serial 36
 
 AU_ALIAS([AC_PYTHON_DEVEL], [AX_PYTHON_DEVEL])
 AC_DEFUN([AX_PYTHON_DEVEL],[
+	# Get whether it's optional
+	if test -z "$2"; then
+	   ax_python_devel_optional=false
+	else
+	   ax_python_devel_optional=$2
+	fi
+	ax_python_devel_found=yes
+
 	#
 	# Allow the use of a (user set) custom python version
 	#
@@ -81,81 +94,148 @@ AC_DEFUN([AX_PYTHON_DEVEL],[
 
 	AC_PATH_PROG([PYTHON],[python[$PYTHON_VERSION]])
 	if test -z "$PYTHON"; then
-	   AC_MSG_ERROR([Cannot find python$PYTHON_VERSION in your system path])
+	   AC_MSG_WARN([Cannot find python$PYTHON_VERSION in your system path])
+	   if ! $ax_python_devel_optional; then
+	      AC_MSG_ERROR([Giving up, python development not available])
+	   fi
+	   ax_python_devel_found=no
 	   PYTHON_VERSION=""
 	fi
 
-	#
-	# Check for a version of Python >= 2.1.0
-	#
-	AC_MSG_CHECKING([for a version of Python >= '2.1.0'])
-	ac_supports_python_ver=`$PYTHON -c "import sys; \
+	if test $ax_python_devel_found = yes; then
+	   #
+	   # Check for a version of Python >= 2.1.0
+	   #
+	   AC_MSG_CHECKING([for a version of Python >= '2.1.0'])
+	   ac_supports_python_ver=`$PYTHON -c "import sys;import setuptools; \
 		ver = sys.version.split ()[[0]]; \
 		print (ver >= '2.1.0')"`
-	if test "$ac_supports_python_ver" != "True"; then
+	   if test "$ac_supports_python_ver" != "True"; then
 		if test -z "$PYTHON_NOVERSIONCHECK"; then
 			AC_MSG_RESULT([no])
-			AC_MSG_FAILURE([
+			AC_MSG_WARN([
 This version of the AC@&t@_PYTHON_DEVEL macro
 doesn't work properly with versions of Python before
 2.1.0. You may need to re-run configure, setting the
-variables PYTHON_CPPFLAGS, PYTHON_LDFLAGS, PYTHON_SITE_PKG,
+variables PYTHON_CPPFLAGS, PYTHON_LIBS, PYTHON_SITE_PKG,
 PYTHON_EXTRA_LIBS and PYTHON_EXTRA_LDFLAGS by hand.
 Moreover, to disable this check, set PYTHON_NOVERSIONCHECK
 to something else than an empty string.
 ])
+			if ! $ax_python_devel_optional; then
+			   AC_MSG_FAILURE([Giving up])
+			fi
+			ax_python_devel_found=no
+			PYTHON_VERSION=""
 		else
 			AC_MSG_RESULT([skip at user request])
 		fi
-	else
+	   else
 		AC_MSG_RESULT([yes])
+	   fi
 	fi
 
-	#
-	# if the macro parameter ``version'' is set, honour it
-	#
-	if test -n "$1"; then
+	if test $ax_python_devel_found = yes; then
+	   #
+	   # If the macro parameter ``version'' is set, honour it.
+	   # A Python shim class, VPy, is used to implement correct version comparisons via
+	   # string expressions, since e.g. a naive textual ">= 2.7.3" won't work for
+	   # Python 2.7.10 (the ".1" being evaluated as less than ".3").
+	   #
+	   if test -n "$1"; then
 		AC_MSG_CHECKING([for a version of Python $1])
-		ac_supports_python_ver=`$PYTHON -c "import sys; \
-			ver = sys.version.split ()[[0]]; \
+                cat << EOF > ax_python_devel_vpy.py
+class VPy:
+    def vtup(self, s):
+        return tuple(map(int, s.strip().replace("rc", ".").split(".")))
+    def __init__(self):
+        import sys
+        import setuptools
+        self.vpy = tuple(sys.version_info)[[:3]]
+    def __eq__(self, s):
+        return self.vpy == self.vtup(s)
+    def __ne__(self, s):
+        return self.vpy != self.vtup(s)
+    def __lt__(self, s):
+        return self.vpy < self.vtup(s)
+    def __gt__(self, s):
+        return self.vpy > self.vtup(s)
+    def __le__(self, s):
+        return self.vpy <= self.vtup(s)
+    def __ge__(self, s):
+        return self.vpy >= self.vtup(s)
+EOF
+		ac_supports_python_ver=`$PYTHON -c "import ax_python_devel_vpy; \
+                        ver = ax_python_devel_vpy.VPy(); \
 			print (ver $1)"`
+                rm -rf ax_python_devel_vpy*.py* __pycache__/ax_python_devel_vpy*.py*
 		if test "$ac_supports_python_ver" = "True"; then
-		   AC_MSG_RESULT([yes])
+			AC_MSG_RESULT([yes])
 		else
 			AC_MSG_RESULT([no])
-			AC_MSG_ERROR([this package requires Python $1.
+			AC_MSG_WARN([this package requires Python $1.
 If you have it installed, but it isn't the default Python
 interpreter in your system path, please pass the PYTHON_VERSION
 variable to configure. See ``configure --help'' for reference.
 ])
+			if ! $ax_python_devel_optional; then
+			   AC_MSG_ERROR([Giving up])
+			fi
+			ax_python_devel_found=no
 			PYTHON_VERSION=""
 		fi
+	   fi
 	fi
 
-	#
-	# Check if you have distutils, else fail
-	#
-	AC_MSG_CHECKING([for the distutils Python package])
-	ac_distutils_result=`$PYTHON -c "import distutils" 2>&1`
-	if test -z "$ac_distutils_result"; then
+	if test $ax_python_devel_found = yes; then
+	   #
+	   # Check if you have distutils, else fail
+	   #
+	   AC_MSG_CHECKING([for the sysconfig Python package])
+	   ac_sysconfig_result=`$PYTHON -c "import sysconfig" 2>&1`
+	   if test $? -eq 0; then
 		AC_MSG_RESULT([yes])
-	else
+		IMPORT_SYSCONFIG="import sysconfig"
+	   else
 		AC_MSG_RESULT([no])
-		AC_MSG_ERROR([cannot import Python module "distutils".
+
+		AC_MSG_CHECKING([for the distutils Python package])
+		ac_sysconfig_result=`$PYTHON -c "from distutils import sysconfig" 2>&1`
+		if test $? -eq 0; then
+			AC_MSG_RESULT([yes])
+			IMPORT_SYSCONFIG="from distutils import sysconfig"
+		else
+			AC_MSG_WARN([cannot import Python module "distutils".
 Please check your Python installation. The error was:
-$ac_distutils_result])
-		PYTHON_VERSION=""
+$ac_sysconfig_result])
+			if ! $ax_python_devel_optional; then
+			   AC_MSG_ERROR([Giving up])
+			fi
+			ax_python_devel_found=no
+			PYTHON_VERSION=""
+		fi
+	   fi
 	fi
 
-	#
-	# Check for Python include path
-	#
-	AC_MSG_CHECKING([for Python include path])
-	if test -z "$PYTHON_CPPFLAGS"; then
-		python_path=`$PYTHON -c "import distutils.sysconfig; \
-			print (distutils.sysconfig.get_python_inc ());"`
-		plat_python_path=`$PYTHON -c "import distutils.sysconfig; \
-			print (distutils.sysconfig.get_python_inc (plat_specific=1));"`
+	if test $ax_python_devel_found = yes; then
+	   #
+	   # Check for Python include path
+	   #
+	   AC_MSG_CHECKING([for Python include path])
+	   if test -z "$PYTHON_CPPFLAGS"; then
+		if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then
+			# sysconfig module has different functions
+			python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_path ('include'));"`
+			plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_path ('platinclude'));"`
+		else
+			# old distutils way
+			python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_python_inc ());"`
+			plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_python_inc (plat_specific=1));"`
+		fi
 		if test -n "${python_path}"; then
 			if test "${plat_python_path}" != "${python_path}"; then
 				python_path="-I$python_path -I$plat_python_path"
@@ -164,22 +244,22 @@ $ac_distutils_result])
 			fi
 		fi
 		PYTHON_CPPFLAGS=$python_path
-	fi
-	AC_MSG_RESULT([$PYTHON_CPPFLAGS])
-	AC_SUBST([PYTHON_CPPFLAGS])
+	   fi
+	   AC_MSG_RESULT([$PYTHON_CPPFLAGS])
+	   AC_SUBST([PYTHON_CPPFLAGS])
 
-	#
-	# Check for Python library path
-	#
-	AC_MSG_CHECKING([for Python library path])
-	if test -z "$PYTHON_LDFLAGS"; then
+	   #
+	   # Check for Python library path
+	   #
+	   AC_MSG_CHECKING([for Python library path])
+	   if test -z "$PYTHON_LIBS"; then
 		# (makes two attempts to ensure we've got a version number
 		# from the interpreter)
 		ac_python_version=`cat<<EOD | $PYTHON -
 
 # join all versioning strings, on some systems
 # major/minor numbers could be in different list elements
-from distutils.sysconfig import *
+from sysconfig import *
 e = get_config_var('VERSION')
 if e is not None:
 	print(e)
@@ -190,7 +270,7 @@ EOD`
 				ac_python_version=$PYTHON_VERSION
 			else
 				ac_python_version=`$PYTHON -c "import sys; \
-					print (sys.version[[:3]])"`
+					print ("%d.%d" % sys.version_info[[:2]])"`
 			fi
 		fi
 
@@ -202,8 +282,8 @@ EOD`
 		ac_python_libdir=`cat<<EOD | $PYTHON -
 
 # There should be only one
-import distutils.sysconfig
-e = distutils.sysconfig.get_config_var('LIBDIR')
+$IMPORT_SYSCONFIG
+e = sysconfig.get_config_var('LIBDIR')
 if e is not None:
 	print (e)
 EOD`
@@ -211,8 +291,8 @@ EOD`
 		# Now, for the library:
 		ac_python_library=`cat<<EOD | $PYTHON -
 
-import distutils.sysconfig
-c = distutils.sysconfig.get_config_vars()
+$IMPORT_SYSCONFIG
+c = sysconfig.get_config_vars()
 if 'LDVERSION' in c:
 	print ('python'+c[['LDVERSION']])
 else:
@@ -227,95 +307,160 @@ EOD`
 		then
 			# use the official shared library
 			ac_python_library=`echo "$ac_python_library" | sed "s/^lib//"`
-			PYTHON_LDFLAGS="-L$ac_python_libdir -l$ac_python_library"
+			PYTHON_LIBS="-L$ac_python_libdir -l$ac_python_library"
 		else
 			# old way: use libpython from python_configdir
 			ac_python_libdir=`$PYTHON -c \
-			  "from distutils.sysconfig import get_python_lib as f; \
+			  "from sysconfig import get_python_lib as f; \
 			  import os; \
 			  print (os.path.join(f(plat_specific=1, standard_lib=1), 'config'));"`
-			PYTHON_LDFLAGS="-L$ac_python_libdir -lpython$ac_python_version"
+			PYTHON_LIBS="-L$ac_python_libdir -lpython$ac_python_version"
 		fi
 
-		if test -z "PYTHON_LDFLAGS"; then
-			AC_MSG_ERROR([
+		if test -z "PYTHON_LIBS"; then
+			AC_MSG_WARN([
   Cannot determine location of your Python DSO. Please check it was installed with
-  dynamic libraries enabled, or try setting PYTHON_LDFLAGS by hand.
+  dynamic libraries enabled, or try setting PYTHON_LIBS by hand.
 			])
+			if ! $ax_python_devel_optional; then
+			   AC_MSG_ERROR([Giving up])
+			fi
+			ax_python_devel_found=no
+			PYTHON_VERSION=""
 		fi
+	   fi
 	fi
-	AC_MSG_RESULT([$PYTHON_LDFLAGS])
-	AC_SUBST([PYTHON_LDFLAGS])
 
-	#
-	# Check for site packages
-	#
-	AC_MSG_CHECKING([for Python site-packages path])
-	if test -z "$PYTHON_SITE_PKG"; then
-		PYTHON_SITE_PKG=`$PYTHON -c "import distutils.sysconfig; \
-			print (distutils.sysconfig.get_python_lib(0,0));"`
-	fi
-	AC_MSG_RESULT([$PYTHON_SITE_PKG])
-	AC_SUBST([PYTHON_SITE_PKG])
+	if test $ax_python_devel_found = yes; then
+	   AC_MSG_RESULT([$PYTHON_LIBS])
+	   AC_SUBST([PYTHON_LIBS])
 
-	#
-	# libraries which must be linked in when embedding
-	#
-	AC_MSG_CHECKING(python extra libraries)
-	if test -z "$PYTHON_EXTRA_LIBS"; then
-	   PYTHON_EXTRA_LIBS=`$PYTHON -c "import distutils.sysconfig; \
-                conf = distutils.sysconfig.get_config_var; \
+	   #
+	   # Check for site packages
+	   #
+	   AC_MSG_CHECKING([for Python site-packages path])
+	   if test -z "$PYTHON_SITE_PKG"; then
+		if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then
+			PYTHON_SITE_PKG=`$PYTHON -c "
+$IMPORT_SYSCONFIG;
+if hasattr(sysconfig, 'get_default_scheme'):
+    scheme = sysconfig.get_default_scheme()
+else:
+    scheme = sysconfig._get_default_scheme()
+if scheme == 'posix_local':
+    # Debian's default scheme installs to /usr/local/ but we want to find headers in /usr/
+    scheme = 'posix_prefix'
+prefix = '$prefix'
+if prefix == 'NONE':
+    prefix = '$ac_default_prefix'
+sitedir = sysconfig.get_path('purelib', scheme, vars={'base': prefix})
+print(sitedir)"`
+		else
+			# distutils.sysconfig way
+			PYTHON_SITE_PKG=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_python_lib(0,0));"`
+		fi
+	   fi
+	   AC_MSG_RESULT([$PYTHON_SITE_PKG])
+	   AC_SUBST([PYTHON_SITE_PKG])
+
+	   #
+	   # Check for platform-specific site packages
+	   #
+	   AC_MSG_CHECKING([for Python platform specific site-packages path])
+	   if test -z "$PYTHON_PLATFORM_SITE_PKG"; then
+		if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then
+			PYTHON_PLATFORM_SITE_PKG=`$PYTHON -c "
+$IMPORT_SYSCONFIG;
+if hasattr(sysconfig, 'get_default_scheme'):
+    scheme = sysconfig.get_default_scheme()
+else:
+    scheme = sysconfig._get_default_scheme()
+if scheme == 'posix_local':
+    # Debian's default scheme installs to /usr/local/ but we want to find headers in /usr/
+    scheme = 'posix_prefix'
+prefix = '$prefix'
+if prefix == 'NONE':
+    prefix = '$ac_default_prefix'
+sitedir = sysconfig.get_path('platlib', scheme, vars={'platbase': prefix})
+print(sitedir)"`
+		else
+			# distutils.sysconfig way
+			PYTHON_PLATFORM_SITE_PKG=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_python_lib(1,0));"`
+		fi
+	   fi
+	   AC_MSG_RESULT([$PYTHON_PLATFORM_SITE_PKG])
+	   AC_SUBST([PYTHON_PLATFORM_SITE_PKG])
+
+	   #
+	   # libraries which must be linked in when embedding
+	   #
+	   AC_MSG_CHECKING(python extra libraries)
+	   if test -z "$PYTHON_EXTRA_LIBS"; then
+	      PYTHON_EXTRA_LIBS=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+                conf = sysconfig.get_config_var; \
                 print (conf('LIBS') + ' ' + conf('SYSLIBS'))"`
-	fi
-	AC_MSG_RESULT([$PYTHON_EXTRA_LIBS])
-	AC_SUBST(PYTHON_EXTRA_LIBS)
+	   fi
+	   AC_MSG_RESULT([$PYTHON_EXTRA_LIBS])
+	   AC_SUBST(PYTHON_EXTRA_LIBS)
 
-	#
-	# linking flags needed when embedding
-	#
-	AC_MSG_CHECKING(python extra linking flags)
-	if test -z "$PYTHON_EXTRA_LDFLAGS"; then
-		PYTHON_EXTRA_LDFLAGS=`$PYTHON -c "import distutils.sysconfig; \
-			conf = distutils.sysconfig.get_config_var; \
+	   #
+	   # linking flags needed when embedding
+	   #
+	   AC_MSG_CHECKING(python extra linking flags)
+	   if test -z "$PYTHON_EXTRA_LDFLAGS"; then
+		PYTHON_EXTRA_LDFLAGS=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+			conf = sysconfig.get_config_var; \
 			print (conf('LINKFORSHARED'))"`
-	fi
-	AC_MSG_RESULT([$PYTHON_EXTRA_LDFLAGS])
-	AC_SUBST(PYTHON_EXTRA_LDFLAGS)
+		# Hack for macos, it sticks this in here.
+		PYTHON_EXTRA_LDFLAGS=`echo $PYTHON_EXTRA_LDFLAGS | sed 's/CoreFoundation.*$/CoreFoundation/'`
+	   fi
+	   AC_MSG_RESULT([$PYTHON_EXTRA_LDFLAGS])
+	   AC_SUBST(PYTHON_EXTRA_LDFLAGS)
 
-	#
-	# final check to see if everything compiles alright
-	#
-	AC_MSG_CHECKING([consistency of all components of python development environment])
-	# save current global flags
-	ac_save_LIBS="$LIBS"
-	ac_save_CPPFLAGS="$CPPFLAGS"
-	LIBS="$ac_save_LIBS $PYTHON_LDFLAGS $PYTHON_EXTRA_LDFLAGS $PYTHON_EXTRA_LIBS"
-	CPPFLAGS="$ac_save_CPPFLAGS $PYTHON_CPPFLAGS"
-	AC_LANG_PUSH([C])
-	AC_LINK_IFELSE([
+	   #
+	   # final check to see if everything compiles alright
+	   #
+	   AC_MSG_CHECKING([consistency of all components of python development environment])
+	   # save current global flags
+	   ac_save_LIBS="$LIBS"
+	   ac_save_LDFLAGS="$LDFLAGS"
+	   ac_save_CPPFLAGS="$CPPFLAGS"
+	   LIBS="$ac_save_LIBS $PYTHON_LIBS $PYTHON_EXTRA_LIBS"
+	   LDFLAGS="$ac_save_LDFLAGS $PYTHON_EXTRA_LDFLAGS"
+	   CPPFLAGS="$ac_save_CPPFLAGS $PYTHON_CPPFLAGS"
+	   AC_LANG_PUSH([C])
+	   AC_LINK_IFELSE([
 		AC_LANG_PROGRAM([[#include <Python.h>]],
 				[[Py_Initialize();]])
 		],[pythonexists=yes],[pythonexists=no])
-	AC_LANG_POP([C])
-	# turn back to default flags
-	CPPFLAGS="$ac_save_CPPFLAGS"
-	LIBS="$ac_save_LIBS"
+	   AC_LANG_POP([C])
+	   # turn back to default flags
+	   CPPFLAGS="$ac_save_CPPFLAGS"
+	   LIBS="$ac_save_LIBS"
+	   LDFLAGS="$ac_save_LDFLAGS"
 
-	AC_MSG_RESULT([$pythonexists])
+	   AC_MSG_RESULT([$pythonexists])
 
-        if test ! "x$pythonexists" = "xyes"; then
-	   AC_MSG_FAILURE([
+	   if test ! "x$pythonexists" = "xyes"; then
+	      AC_MSG_WARN([
   Could not link test program to Python. Maybe the main Python library has been
   installed in some non-standard library path. If so, pass it to configure,
-  via the LDFLAGS environment variable.
-  Example: ./configure LDFLAGS="-L/usr/non-standard-path/python/lib"
+  via the LIBS environment variable.
+  Example: ./configure LIBS="-L/usr/non-standard-path/python/lib"
   ============================================================================
    ERROR!
    You probably have to install the development version of the Python package
    for your distribution.  The exact name of this package varies among them.
   ============================================================================
-	   ])
-	  PYTHON_VERSION=""
+	      ])
+	      if ! $ax_python_devel_optional; then
+		 AC_MSG_ERROR([Giving up])
+	      fi
+	      ax_python_devel_found=no
+	      PYTHON_VERSION=""
+	   fi
 	fi
 
 	#
diff --git a/config/lx_find_mpi.m4 b/config/lx_find_mpi.m4
index fbca314b4e9b..587864d405e4 100644
--- a/config/lx_find_mpi.m4
+++ b/config/lx_find_mpi.m4
@@ -177,17 +177,17 @@ AC_DEFUN([LX_QUERY_MPI_COMPILER],
          CPPFLAGS=$MPI_$3FLAGS
          LIBS=$MPI_$3LDFLAGS
 
-         AC_TRY_LINK([#include <mpi.h>],
-                     [int rank, size;
-                      MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-                      MPI_Comm_size(MPI_COMM_WORLD, &size);],
-                     [# Add a define for testing at compile time.
-                      AC_DEFINE([HAVE_MPI], [1], [Define to 1 if you have MPI libs and headers.])
-                      have_$3_mpi='yes'],
-                     [# zero out mpi flags so we don't link against the faulty library.
-                      MPI_$3FLAGS=""
-                      MPI_$3LDFLAGS=""
-                      have_$3_mpi='no'])
+         AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <mpi.h>]],
+	 				 [[int rank, size;
+					   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+					   MPI_Comm_size(MPI_COMM_WORLD, &size);]])],
+			[# Add a define for testing at compile time.
+                         AC_DEFINE([HAVE_MPI], [1], [Define to 1 if you have MPI libs and headers.])
+                         have_$3_mpi='yes'],
+			[# zero out mpi flags so we don't link against the faulty library.
+			 MPI_$3FLAGS=""
+			 MPI_$3LDFLAGS=""
+			 have_$3_mpi='no'])
 
          # AC_SUBST everything.
          AC_SUBST($1)
diff --git a/config/tap-driver.py b/config/tap-driver.py
index 02fd5ae02aad..967cf63e5b7a 100644
--- a/config/tap-driver.py
+++ b/config/tap-driver.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 
 import sys
 import os
diff --git a/config/tap-driver.sh b/config/tap-driver.sh
index 4a687e2d02bb..90e3fce2204b 100755
--- a/config/tap-driver.sh
+++ b/config/tap-driver.sh
@@ -144,7 +144,20 @@ fi
     else
       exec 2>&3
     fi
-    "$@"
+    if test -n "${FLUX_TEST_TIMEOUT:-}" ; then
+      if test -z "${FLUX_SOURCE_DIR:-}"; then
+        if test -n "${top_srcdir:-}" ; then
+          FLUX_SOURCE_DIR="$top_srcdir"
+        else
+          SCRIPT=$(readlink -f "$0")
+          SPATH=$(dirname "$SCRIPT")
+          FLUX_SOURCE_DIR="$SPATH"/..
+        fi
+      fi
+      "${PYTHON:-python3}" -S "${FLUX_SOURCE_DIR}/t/scripts/run_timeout.py" "$FLUX_TEST_TIMEOUT" "$@"
+    else
+       "$@"
+    fi
     echo $?
   ) | LC_ALL=C ${AM_TAP_AWK-awk} \
         -v me="$me" \
diff --git a/config/x_ac_jansson.m4 b/config/x_ac_jansson.m4
index c076d21429a2..8bcade9efc70 100644
--- a/config/x_ac_jansson.m4
+++ b/config/x_ac_jansson.m4
@@ -1,6 +1,17 @@
+# Versions of jansson shipped by distros:
+# sles15-sp5    ships 2.9  (flux-framework/flux-core#5544)
+# centos7/TOSS3 ships 2.10 (flux-framework/flux-core#3239)
+# centos8/TOSS4 ships 2.11
+# Ubuntu 18.04  ships 2.11
+# Ubuntu 20.04  ships 2.12
+#
+# Some modern jansson features used in flux-core:
+# - json_pack ("O?")     from 2.8
+# - json_string_length() from 2.7
+#
 AC_DEFUN([X_AC_JANSSON], [
 
-    PKG_CHECK_MODULES([JANSSON], [jansson >= 2.6], [], [])
+    PKG_CHECK_MODULES([JANSSON], [jansson >= 2.9], [], [])
 
     ac_save_LIBS="$LIBS"
     LIBS="$LIBS $JANSSON_LIBS"
@@ -13,6 +24,14 @@ AC_DEFUN([X_AC_JANSSON], [
         AC_MSG_ERROR([json_int_t must be 64 bits for flux to be built])
     ])
 
+    AX_COMPILE_CHECK_SIZEOF(int)
+
+    AS_VAR_IF([ac_cv_sizeof_int],[2],[
+        AC_MSG_ERROR([flux cannot be built on a system with 16 bit ints])
+    ])
+
+    AC_REPLACE_FUNCS(json_object_update_recursive)
+
     LIBS="$ac_save_LIBS"
     CFLAGS="$ac_save_CFLAGS"
   ]
diff --git a/config/x_ac_sanitize.m4 b/config/x_ac_sanitize.m4
index e07ef2cecdbd..452c7c567a18 100644
--- a/config/x_ac_sanitize.m4
+++ b/config/x_ac_sanitize.m4
@@ -27,7 +27,7 @@ AC_DEFUN([X_AC_ENABLE_SANITIZER], [
         AS_VAR_SET(san_ld_zdef_flag, [])
         AC_SUBST(san_ld_zdef_flag)
     elif test "x$san_enabled" = "xno" ; then
-        AS_VAR_SET(san_ld_zdef_flag, [-Wl,--no-undefined])
+        AS_VAR_SET(san_ld_zdef_flag, [-no-undefined])
         AC_SUBST(san_ld_zdef_flag)
     else
         AC_MSG_ERROR($san_enabled is a unsupported option.)
diff --git a/config/x_ac_zeromq.m4 b/config/x_ac_zeromq.m4
index c42655c5113d..ec2e3996562b 100644
--- a/config/x_ac_zeromq.m4
+++ b/config/x_ac_zeromq.m4
@@ -1,39 +1,35 @@
+# N.B oldest in CI are focal=libzmq-4.3.2-2, centos7=zeromq-4.1.4
+
 AC_DEFUN([X_AC_ZEROMQ], [
 
-    PKG_CHECK_MODULES([ZMQ], [libczmq >= 3.0.0 libzmq >= 4.0.4])
+    PKG_CHECK_MODULES([ZMQ], [libzmq >= 4.0.4])
 
-    ac_save_LIBS="$LIBS"
-    LIBS="$LIBS $ZMQ_LIBS"
-    ac_save_CFLAGS="$CFLAGS"
+    old_CFLAGS=$CFLAGS
     CFLAGS="$CFLAGS $ZMQ_CFLAGS"
 
-    AC_MSG_CHECKING([For CURVE encryption support in libzmq])
-    AC_CACHE_VAL(ac_cv_curve_support,
-      AC_RUN_IFELSE([AC_LANG_SOURCE([[
-#include <zmq.h>
-#include <zmq_utils.h>
-  main()
-  {
-    char x[[41]], y[[41]];
-    /*
-     * Check for CURVE support in current version of libzmq
-     */
-    int rc = zmq_curve_keypair (x, y);
-    exit (rc >= 0 ? 0 : 1);
-  }
-]])],
-    ac_cv_curve_support=yes,
-    ac_cv_curve_support=no,
-    ac_cv_curve_support=no))
+    old_LIBS=$LIBS
+    LIBS="$LIBS $ZMQ_LIBS"
+
+    AC_MSG_CHECKING([whether zeromq has CURVE support])
+    AC_RUN_IFELSE([
+        AC_LANG_SOURCE([[
+            #include <zmq.h>
+	    int main () {
+	        char pub[41];
+	        char sec[41];
+	        if (zmq_curve_keypair (pub, sec) < 0) return 1;
+	        return 0;
+	    }
+	]])],
+	[have_zmq_curve_support=yes; AC_MSG_RESULT([yes])],
+	[AC_MSG_RESULT([no])]
+    )
 
-    AC_MSG_RESULT($ac_cv_curve_support)
-    if test "x$ac_cv_curve_support" = "xno"; then
-      AC_MSG_ERROR([
- Failed to detect CURVE support in libzmq!
- Perhaps you need to compile with libsodium?])
-    fi
-    LIBS="$ac_save_LIBS"
-    CFLAGS="$ac_save_CFLAGS"
+    CFLAGS=$old_CFLAGS
+    LIBS=$old_LIBS
 
+    AS_IF([test "x$have_zmq_curve_support" != "xyes"],
+        [AC_MSG_ERROR([zeromq CURVE/libsodium support is required])]
+    )
   ]
 )
diff --git a/configure.ac b/configure.ac
index ba3a4132b5d5..f70c3c573c34 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,11 +2,11 @@
 # Prologue
 ##
 AC_INIT([flux-core],
-        m4_esyscmd([git describe --always | awk '/.*/ {sub(/^v/, ""); printf "%s",$1; exit}']))
+        m4_esyscmd([test -n "$FLUX_VERSION" && printf $FLUX_VERSION || git describe --always | awk '/.*/ {sub(/^v/, ""); printf "%s",$1; exit}']))
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([config])
-AC_CONFIG_SRCDIR([NEWS])
-AC_CANONICAL_SYSTEM
+AC_CONFIG_SRCDIR([NEWS.md])
+AC_CANONICAL_TARGET
 ##
 # If runstatedir not explicitly set on command line, use '/run' as default
 # N.B. runstatedir is not set at all in autoconf < 2.70.
@@ -19,9 +19,9 @@ X_AC_EXPAND_INSTALL_DIRS
 ##
 # Automake support
 ##
-AM_INIT_AUTOMAKE([subdir-objects tar-pax])
+AM_INIT_AUTOMAKE([subdir-objects tar-pax foreign])
 AM_SILENT_RULES([yes])
-AM_CONFIG_HEADER([config/config.h])
+AC_CONFIG_HEADERS([config/config.h])
 AM_MAINTAINER_MODE([enable])
 
 AC_DEFINE([_GNU_SOURCE], 1,
@@ -36,6 +36,28 @@ AC_SUBST([AX_MAJOR_VERSION])
 AC_SUBST([AX_MINOR_VERSION])
 AC_SUBST([AX_POINT_VERSION])
 
+AC_MSG_CHECKING([whether version number is sane])
+AS_IF([printf "%d.%d.%d" ${AX_MAJOR_VERSION} ${AX_MINOR_VERSION} ${AX_POINT_VERSION} >/dev/null 2>&1], [
+    AC_MSG_RESULT([yes])
+],[
+    AC_MSG_RESULT([no])
+    version_err_msg="
+    VERSION ${VERSION} is invalid.
+    Try the following to remedy this:
+    
+      1. Run \`git fetch --tags\` before building. Versions in 
+         flux-core are derived from \`git describe\` which uses
+	 the most recent tag.
+      2. If you are running remote CI in a fork of the main repository, 
+         try pushing the upstream tags to your fork with
+	 \`git push --tags <your_remote>\` to make sure tags are 
+	 synchronized in your fork.
+      3. Set the variable manually, with FLUX_VERSION=<version>
+         in your environment.
+    "
+    AC_MSG_ERROR(["${version_err_msg}"])
+])
+
 ##
 # Initialize pkg-config for PKG_CHECK_MODULES to avoid conditional issues
 ##
@@ -85,10 +107,22 @@ LIBFLUX_SCHEDUTIL_AGE=0
 LIBFLUX_SCHEDUTIL_VERSION_INFO=$LIBFLUX_SCHEDUTIL_CURRENT:$LIBFLUX_SCHEDUTIL_REVISION:$LIBFLUX_SCHEDUTIL_AGE
 AC_SUBST([LIBFLUX_SCHEDUTIL_VERSION_INFO])
 
+LIBFLUX_HOSTLIST_CURRENT=1
+LIBFLUX_HOSTLIST_REVISION=0
+LIBFLUX_HOSTLIST_AGE=0
+LIBFLUX_HOSTLIST_VERSION_INFO=$LIBFLUX_HOSTLIST_CURRENT:$LIBFLUX_HOSTLIST_REVISION:$LIBFLUX_HOSTLIST_AGE
+AC_SUBST([LIBFLUX_HOSTLIST_VERSION_INFO])
+
+LIBFLUX_TASKMAP_CURRENT=1
+LIBFLUX_TASKMAP_REVISION=0
+LIBFLUX_TASKMAP_AGE=0
+LIBFLUX_TASKMAP_VERSION_INFO=$LIBFLUX_TASKMAP_CURRENT:$LIBFLUX_TASKMAP_REVISION:$LIBFLUX_TASKMAP_AGE
+AC_SUBST([LIBFLUX_TASKMAP_VERSION_INFO])
+
 ##
 # Checks for programs
 ##
-AC_PROG_CC_C99
+m4_version_prereq(2.70, [AC_PROG_CC], [AC_PROG_CC_C99])
 AM_PROG_CC_C_O
 AX_COMPILER_VENDOR
 AX_COMPILER_VERSION
@@ -132,48 +166,26 @@ AC_PATH_PROGS(SSH, [rsh ssh], [/usr/bin/rsh])
 AC_DEFINE_UNQUOTED([PATH_SSH], "$SSH",
           [Define remote shell program to be used by the ssh:// connector])
 
+# macos linker doesn't support --gc-sections
+saved_LDFLAGS="$LDFLAGS"
+LDFLAGS="-Wl,--gc-sections $LDFLAGS"
+AC_MSG_CHECKING([whether ld supports --gc-sections])
+AC_LINK_IFELSE([AC_LANG_PROGRAM([])], [
+  AC_MSG_RESULT([yes])
+  ld_gc_sections="-Wl,--gc-sections"
+  AS_VAR_SET(ld_gc_sections, $ld_gc_sections)
+  AC_SUBST(ld_gc_sections)
+  ],[
+  AC_MSG_RESULT([no])
+])
+LDFLAGS=$saved_LDFLAGS
+
 LT_INIT
 AC_PROG_AWK
 
-AC_ARG_ENABLE([docs],
-	      AS_HELP_STRING([--disable-docs], [disable building docs]))
-AS_IF([test "x$enable_docs" != "xno"], [
-  AC_CHECK_PROGS(ADOC, [a2x asciidoctor])
-  AS_IF([test "$ADOC" == "a2x"], [
-        ADOC_FORMAT_OPT="--format"
-        AC_SUBST([ADOC_FORMAT_OPT])
-  ])
-  AS_IF([test "$ADOC" == "asciidoctor"], [
-        AC_MSG_CHECKING([Asciidoctor version])
-        ASCIIDOCTOR_VERSION=$(asciidoctor --version|sed -n 's/Asciidoctor \(.*\) .*/\1/p')
-        AC_MSG_RESULT([$ASCIIDOCTOR_VERSION])
-        AS_IF([test -z "$ASCIIDOCTOR_VERSION"],[
-              AC_MSG_ERROR([Failed to read asciidoctor version])
-        ])
-        AX_COMPARE_VERSION([$ASCIIDOCTOR_VERSION],[ge],[1.5.7], [
-          ADOC_FORMAT_OPT="--backend"
-          AC_SUBST([ADOC_FORMAT_OPT])
-        ],[
-          ADOC_ERROR_MSG="asciidoctor version >= 1.5.7 required, got $ASCIIDOCTOR_VERSION"
-          AC_MSG_WARN([$ADOC_ERROR_MSG])
-          ADOC=""
-        ])
-  ])
-])
-#
-#  If --enable-docs=yes, but no doc generator found,
-#   then error immediately:
-#
-AS_IF([test "x$enable_docs" = "xyes" -a -z "$ADOC"],[
-  AC_MSG_ERROR([--enable-docs used but no document generator found!])
-])
-AM_CONDITIONAL([ENABLE_DOCS], [test -n "$ADOC"])
-AC_CHECK_PROG(ASPELL,[aspell],[aspell])
-
 ##
 # Checks for header files.
 ##
-AC_HEADER_STDC
 AC_CHECK_HEADERS( \
   pthread.h \
   getopt.h \
@@ -189,18 +201,25 @@ AC_CHECK_HEADERS( \
   xlocale.h \
   endian.h \
   inttypes.h \
+  link.h \
+  [sys/ucred.h] \
+  [sys/prctl.h] \
 )
 
 ##
 # Checks for typedefs, structures, and compiler characteristics
-## 
-AC_C_BIGENDIAN
+##
+AC_C_BIGENDIAN(
+  [AC_DEFINE([HAVE_BIG_ENDIAN], [1], [big endian])],
+  [AC_DEFINE([HAVE_LITTLE_ENDIAN], [1], [little endian])]
+)
 AC_C_CONST
 AC_TYPE_SIZE_T
 AX_COMPILE_CHECK_SIZEOF(int)
 AX_COMPILE_CHECK_SIZEOF(long)
 AX_COMPILE_CHECK_SIZEOF(long long)
 AX_COMPILE_CHECK_SIZEOF(uintptr_t, [#include <stdint.h>])
+AX_COMPILE_CHECK_SIZEOF(ptrdiff_t, [#include <stddef.h>])
 AX_COMPILE_CHECK_SIZEOF(size_t, [#include <stdint.h>])
 
 ##
@@ -222,14 +241,40 @@ AC_CHECK_FUNCS( \
   strncasecmp \
   setlocale \
   uselocale \
+  inotify_init1 \
+)
+# See src/common/libmissing/Makefile.am
+AC_REPLACE_FUNCS( \
+  strlcpy \
+  strlcat \
+  argz_add \
+  envz_add \
+  strerrorname_np \
+  pipe2 \
+  mempcpy \
+  get_current_dir_name \
 )
 X_AC_CHECK_PTHREADS
-X_AC_CHECK_COND_LIB(util, forkpty)
 X_AC_CHECK_COND_LIB(rt, clock_gettime)
 X_AC_CHECK_COND_LIB(dl, dlerror)
 X_AC_MALLOC
 AC_CHECK_LIB(m, floor)
+AC_SEARCH_LIBS(epoll_create1, epoll-shim)
+AM_CONDITIONAL([HAVE_INOTIFY], [test "x$ac_cv_func_inotify_init1" = xyes])
+
+AC_MSG_CHECKING([for pthread_setname_np with tid parameter])
+AC_COMPILE_IFELSE(
+  [AC_LANG_PROGRAM(
+    [#include <pthread.h>],
+    [pthread_setname_np(pthread_self(), "example")])],
+  [AC_MSG_RESULT(yes)
+    AC_DEFINE(HAVE_PTHREAD_SETNAME_NP_WITH_TID, 1,
+      [Have pthread_setname_np() with tid])],
+  [AC_MSG_RESULT(no)]
+)
 
+AC_ARG_ENABLE([docs],
+	      AS_HELP_STRING([--disable-docs], [disable building docs]))
 
 #  Edit PATH to remove $PWD/src/cmd so that AM_PATH_PYTHON doesn't find
 #  flux python script (thus creating a link to itself.) This needs to be
@@ -238,14 +283,21 @@ AC_CHECK_LIB(m, floor)
 saved_PATH=$PATH
 export PATH=$(echo $PATH | sed "s|$(pwd)/src/cmd:*||")
 
-AX_PYTHON_DEVEL([>='2.7'])
+if test "X$PYTHON_VERSION" = "X" ; then
+  if test "X$PYTHON" = "X" ; then
+    # if the user hasn't specified, try for python 3
+    PYTHON_VERSION=3
+  fi
+fi
+
+# Do not let AX_PYTHON_DEVEL set PYTHON_SITE_PKG
+saved_PYTHON_SITE_PKG=$PYTHON_SITE_PKG
+AX_PYTHON_DEVEL([>='3.6'])
+PYTHON_SITE_PKG=$saved_PYTHON_SITE_PKG
 
 AM_PATH_PYTHON([$ac_python_version])
 if test "X$PYTHON" = "X"; then
-  AC_MSG_ERROR([could not find python])
-fi
-if test "X$PYTHON" = "X"; then
-  AC_MSG_ERROR([could not find python])
+    AC_MSG_ERROR([could not find python])
 fi
 #  Restore original PATH:
 export PATH=${saved_PATH}
@@ -260,28 +312,48 @@ AM_CHECK_PYMOD(cffi,
                ,
                [AC_MSG_ERROR([could not find python module cffi, version 1.1+ required])]
                )
-AM_CHECK_PYMOD(six,
-               [StrictVersion(six.__version__) >= StrictVersion('1.9.0')],
-               ,
-               [AC_MSG_ERROR([could not find python module six, version 1.9.0+ required])]
-               )
 AM_CHECK_PYMOD(yaml,
-               [StrictVersion(yaml.__version__) >= StrictVersion ('3.10.0')],
+               [Version(yaml.__version__) >= Version ('3.10.0')],
                ,
                [AC_MSG_ERROR([could not find python module yaml, version 3.10+ required])]
                )
-AM_CHECK_PYMOD(jsonschema,
-               [StrictVersion(jsonschema.__version__) >= StrictVersion ('2.3.0')],
+AM_CHECK_PYMOD(ply,
+               [Version(ply.__version__) >= Version ('3.9')],
                ,
-               [AC_MSG_ERROR([could not find python module jsonschema, version 2.3.0+ required])]
+               [AC_MSG_ERROR([could not find python module ply, version 3.9+ required])]
                )
 
+AS_IF([test "x$enable_docs" != "xno"], [
+            AM_CHECK_PYMOD(sphinx,
+                           [Version(sphinx.__version__) >= Version ('1.6.7')],
+                           [sphinx=true],
+                           [sphinx=false; AC_MSG_WARN([could not find sphinx to generate docs, version 1.6.7+ required])]
+                           )
+            AM_CHECK_PYMOD(docutils,
+                           [Version(docutils.__version__) >= Version ('0.11.0')],
+                           [docutils=true],
+                           [docutils=false; AC_MSG_WARN([could not find docutils to generate docs, version 0.11.0+ required])]
+                           )
+])
+#  If --enable-docs=yes, but no doc generator found,
+#   then error immediately:
+#
+AS_IF([test "x$enable_docs" = "xyes" -a "x$sphinx" = "xfalse"],[
+  AC_MSG_ERROR([--enable-docs used but no document generator found!])
+])
+AS_IF([test "x$enable_docs" = "xyes" -a "x$docutils" = "xfalse"],[
+  AC_MSG_ERROR([--enable-docs used but docutils not found!])
+])
+AM_CONDITIONAL([ENABLE_DOCS], [test "x$sphinx" = "xtrue" -a "x$docutils" = "xtrue"])
+AC_CHECK_PROG(ASPELL,[aspell],[aspell])
+
+
 # Remove -L<path> from PYTHON_LDFLAGS if it is in a standard path
 # (e.g. /usr/lib64).  Placing a standard path earlier in the linker
 # search can lead to linking problems.
 #
 # Logic below assumes only newer Python versions, protected by
-# above check for atleast Python 2.7.
+# above check for atleast Python 3.6.
 if test "$ac_python_ldflags_set_by_user" != "true"; then
   AC_CHECK_LIB([$ac_python_library], [PyArg_ParseTuple],
                [ac_python_in_ld_path=true])
@@ -294,8 +366,6 @@ AS_VAR_SET(fluxpydir, $pyexecdir/flux)
 AC_SUBST(fluxpydir)
 AS_VAR_SET(fluxpysodir, $pyexecdir/_flux)
 AC_SUBST(fluxpysodir)
-AS_VAR_SET(fluxpymoddir, $pyexecdir/flux/modules)
-AC_SUBST(fluxpymoddir)
 AC_SUBST(PYTHON_LIBRARY, lib${ac_python_library}.so)
 
 AC_DEFINE_UNQUOTED([PYTHON_INTERPRETER], ["$PYTHON"], [The python interpreter flux is configured with])
@@ -310,23 +380,49 @@ AS_IF([test "x$enable_pylint" = "xyes"], [
   AC_CHECK_PROG(PYLINT,[pylint],[pylint])
   AS_IF([test "x$PYLINT" != "xpylint"], [AC_MSG_ERROR([No pylint found in PATH])])
   AM_CHECK_PYMOD(pylint,
-                   [StrictVersion(pylint.__version__) >= StrictVersion('1.4.5')],
+                   [Version(pylint.__version__) >= Version('1.8.4')],
                    ,
-                   [AC_MSG_ERROR([could not find python module pylint, version 1.4.5+ required])]
+                   [AC_MSG_ERROR([could not find python module pylint, version 1.8.4+ required])]
                  )
 ])
 AM_CONDITIONAL([ENABLE_PYLINT], [test "x$PYLINT" = "xpylint"])
 
-AX_PROG_LUA([5.1],[5.4])
+AX_PROG_LUA([5.1],[5.5])
 AX_LUA_HEADERS
 AX_LUA_LIBS
 X_AC_ZEROMQ
 X_AC_JANSSON
+PKG_CHECK_MODULES([LIBSYSTEMD], [libsystemd >= 0.23],
+                                [have_libsystemd=yes], [have_libsystemd=no])
+AC_CHECK_HEADER([systemd/sd-bus.h], [have_sd_bus_h=yes], [have_sd_bus_h=no])
+AM_CONDITIONAL([HAVE_LIBSYSTEMD],
+        [test "$have_libsystemd" = yes -a "$have_sd_bus_h" = yes])
+if test "$have_libsystemd" = yes -a "$have_sd_bus_h" = yes; then
+    AC_DEFINE([HAVE_LIBSYSTEMD], [1], [Define if you have libsystemd])
+    AC_CHECK_LIB(systemd, sd_bus_message_dump,
+                 AC_DEFINE([HAVE_SD_BUS_MESSAGE_DUMP], [1],
+                           [Define if you have sd_bus_message_dump]))
+fi
 PKG_CHECK_MODULES([HWLOC], [hwloc >= 1.11.1], [], [])
 PKG_CHECK_MODULES([LZ4], [liblz4], [], [])
 PKG_CHECK_MODULES([SQLITE], [sqlite3], [], [])
-PKG_CHECK_MODULES([LIBSODIUM], [libsodium >= 1.0.14], [], [])
 PKG_CHECK_MODULES([LIBUUID], [uuid], [], [])
+PKG_CHECK_MODULES([CURSES], [ncursesw], [], [])
+PKG_CHECK_MODULES([LIBARCHIVE], [libarchive], [], [])
+
+AC_ARG_WITH([system-bash-completion-dir],
+    AS_HELP_STRING([--with-system-bash-completion-dir],
+                   [Build with system bash-completion dir]))
+AS_IF([test "x$with_system_bash_completion_dir" = "xyes"], [
+    PKG_CHECK_EXISTS([bash-completion],
+      [bashcompdir=`$PKG_CONFIG --variable=completionsdir bash-completion`],
+      [bashcompdir="${sysconfdir}/bash_completion.d"])
+  ], [
+    bashcompdir="${sysconfdir}/bash_completion.d"
+  ]
+)
+AC_SUBST(bashcompdir)
+
 LX_FIND_MPI
 AM_CONDITIONAL([HAVE_MPI], [test "$have_C_mpi" = yes])
 AX_VALGRIND_H
@@ -337,27 +433,47 @@ AS_IF([test x$enable_code_coverage = xyes], [
 AC_ARG_WITH([flux-security], AS_HELP_STRING([--with-flux-security],
              [Build with flux-security]))
 AS_IF([test "x$with_flux_security" = "xyes"], [
-    PKG_CHECK_MODULES([FLUX_SECURITY], [flux-security],
-                      [flux_sec_incdir=`$PKG_CONFIG --variable=includedir flux-security`],
-                      [flux_sec_incdir=;])
+    PKG_CHECK_MODULES([FLUX_SECURITY], [flux-security >= 0.13.0],
+      [flux_sec_incdir=`$PKG_CONFIG --variable=includedir flux-security`])
     AS_IF([test "x$flux_sec_incdir" = x],
-          [AC_MSG_ERROR([couldn't find flux-security or include directory])])
+          [AC_MSG_ERROR([couldn't find flux-security include directory])])
     AC_CHECK_HEADERS([flux/security/version.h])
     AC_DEFINE([HAVE_FLUX_SECURITY], [1], [Define flux-security is available])
     AC_SUBST(FLUX_SECURITY_INCDIR, $flux_sec_incdir)
 ])
 AM_CONDITIONAL([HAVE_FLUX_SECURITY], [test "x$with_flux_security" = "xyes"])
 
-AC_ARG_ENABLE(caliper,
-	[  --enable-caliper[=OPTS]   Use caliper for profiling. [default=no] [OPTS=no/yes]], ,
-	[enable_caliper="no"])
+AC_ARG_ENABLE([content-s3],
+    AS_HELP_STRING([--enable-content-s3], [Enable S3 storage backend]))
+
+AS_IF([test "x$enable_content_s3" = "xyes"], [
+    X_AC_CHECK_COND_LIB(s3, S3_initialize)
+    AS_IF([test "x$ac_cv_lib_s3_S3_initialize" != "xyes"], [
+      AC_MSG_ERROR([configured with --enable-content-s3, but libs3 not found])
+    ])
+
+    AC_COMPILE_IFELSE(
+        [AC_LANG_PROGRAM([#include <libs3.h>],
+                         [S3_create_bucket (0,0,0,0,0,0,0,0,0,0,0);])],
+        AC_DEFINE([HAVE_S3_AUTH_REGION], [1], [S3_create_bucket has 11 args]),
+        AC_COMPILE_IFELSE(
+            [AC_LANG_PROGRAM([#include <libs3.h>],
+                             [S3_create_bucket (0,0,0,0,0,0,0,0,0,0,0,0,0);])],
+            AC_DEFINE([HAVE_S3_TIMEOUT_ARG], [1], [S3_create_bucket has 13 args])
+        )
+    )
+])
+
+AM_CONDITIONAL([ENABLE_CONTENT_S3], [test "x$enable_content_s3" = "xyes"])
 
-if test "$enable_caliper" = "yes"; then
-    PKG_CHECK_MODULES([CALIPER], [caliper], [], [])
-    CFLAGS="${CFLAGS} ${CALIPER_CFLAGS} "
-    # Do not use CALIPER_LIBS, only link to libcaliper-stub
-    LIBS="${LIBS} $(pkg-config --libs-only-L caliper) -lcaliper-stub -lrt "
-    AC_DEFINE([HAVE_CALIPER], [1], [Define if you have libcaliper])
+AC_ARG_ENABLE(
+    [broken-locale-mode],
+    AS_HELP_STRING([--enable-broken-locale-mode],
+                   [Assume broken locale config and use ascii only]))
+
+if test "$enable_broken_locale_mode" = "yes"; then
+  AC_DEFINE([ASSUME_BROKEN_LOCALE], [1],
+    [assume broken locale configuration and disable non-ascii characters])
 fi
 
 ##
@@ -376,38 +492,53 @@ AC_PKGCONFIG
 ##
 # Project directories
 ##
-AS_VAR_SET(fluxrcdir, $sysconfdir/flux)
-AC_SUBST(fluxrcdir)
-
-AS_VAR_SET(fluxrc1dir, $sysconfdir/flux/rc1.d)
-AC_SUBST(fluxrc1dir)
-
-AS_VAR_SET(fluxrc3dir, $sysconfdir/flux/rc3.d)
-AC_SUBST(fluxrc3dir)
-
-AS_VAR_SET(fluxcfdir, $sysconfdir/flux/conf.d)
-AC_SUBST(fluxcfdir)
-
-AS_VAR_SET(fluxlibexecdir, $libexecdir/flux)
+adl_RECURSIVE_EVAL(["$sysconfdir/flux"], [fluxconfdir])
+AC_DEFINE_UNQUOTED([FLUXCONFDIR], ["$fluxconfdir"],
+  [The expansion of "$sysconfdir/flux"])
+AC_SUBST(fluxconfdir)
+
+adl_RECURSIVE_EVAL(["$libexecdir/flux"], [fluxlibexecdir])
+AC_DEFINE_UNQUOTED([FLUXLIBEXECDIR], ["$fluxlibexecdir"],
+  [The expansion of "$libexecdir/flux"])
 AC_SUBST(fluxlibexecdir)
 
-AS_VAR_SET(fluxcmddir, $libexecdir/flux/cmd)
+adl_RECURSIVE_EVAL(["$libexecdir/flux/cmd"], [fluxcmddir])
+AC_DEFINE_UNQUOTED([FLUXCMDDIR], ["$fluxcmddir"],
+  [The expansion of "$libexecdir/flux/cmd"])
 AC_SUBST(fluxcmddir)
 
-AS_VAR_SET(fluxlibdir, $libdir/flux)
+adl_RECURSIVE_EVAL(["$libdir/flux"], [fluxlibdir])
+AC_DEFINE_UNQUOTED([FLUXLIBDIR], ["$fluxlibdir"],
+  [The expansion of "$libdir/flux"])
 AC_SUBST(fluxlibdir)
 
 # Target of PYTHONPATH set by flux(1) cmddriver, so flux(1)
 # doesn't inadvertently insert system python paths (or any
 # other python path for that matter) first in PYTHONPATH.
 #
-AS_VAR_SET(fluxpylinkdir, $fluxlibdir/python$PYTHON_VERSION)
+adl_RECURSIVE_EVAL(["$fluxlibdir/python$PYTHON_VERSION"], [fluxpylinkdir])
+AC_DEFINE_UNQUOTED([FLUXPYLINKDIR], ["$fluxpylinkdir"],
+  [The expansion of "$fluxlibdir/python$PYTHON_VERSION"])
 AC_SUBST(fluxpylinkdir)
 
-AS_VAR_SET(fluxmoddir, $libdir/flux/modules)
+adl_RECURSIVE_EVAL(["$libdir/flux/modules"], [fluxmoddir])
+AC_DEFINE_UNQUOTED([FLUXMODDIR], ["$fluxmoddir"],
+  [The expansion of "$libdir/flux/modules"])
 AC_SUBST(fluxmoddir)
 
-AS_VAR_SET(fluxconnectordir, $libdir/flux/connectors)
+adl_RECURSIVE_EVAL(["$libdir/flux/job-manager/plugins"], [jobtap_plugindir])
+AC_DEFINE_UNQUOTED([JOBTAP_PLUGINDIR], ["$jobtap_plugindir"],
+  [The expansion of "$libdir/flux/job-manager/plugins"])
+AC_SUBST(jobtap_plugindir)
+
+adl_RECURSIVE_EVAL(["$libdir/flux/shell/plugins"], [shell_plugindir])
+AC_DEFINE_UNQUOTED([SHELL_PLUGINDIR], ["$shell_plugindir"],
+  [The expansion of "$libdir/flux/shell/plugins"])
+AC_SUBST(shell_plugindir)
+
+adl_RECURSIVE_EVAL(["$libdir/flux/connectors"], [fluxconnectordir])
+AC_DEFINE_UNQUOTED([FLUXCONNECTORDIR], ["$fluxconnectordir"],
+  [The expansion of "$libdir/flux/connectors"])
 AC_SUBST(fluxconnectordir)
 
 AS_VAR_SET(fluxincludedir, $includedir/flux)
@@ -427,13 +558,20 @@ adl_RECURSIVE_EVAL([$luadir], fluxluadir)
 AS_VAR_SET(fluxluadir, $fluxluadir)
 AC_SUBST(fluxluadir)
 
+AS_VAR_SET(fluxbindingincludedir, $includedir/flux/_binding)
+AC_SUBST(fluxbindingincludedir)
+
+
 ##
 # Macros to avoid repetition in Makefiles.am's
 ##
-fluxmod_ldflags="$san_ld_zdef_flag -avoid-version -export-symbols-regex '^mod_(main|name|service)\$\$' --disable-static -shared -export-dynamic"
+fluxmod_ldflags="$san_ld_zdef_flag -avoid-version -export-symbols-regex '^mod_(main|name)\$\$' --disable-static -shared -export-dynamic $ld_gc_sections"
 AC_SUBST(fluxmod_ldflags)
 
-fluxlib_ldflags="-shared -export-dynamic --disable-static $san_ld_zdef_flag"
+fluxplugin_ldflags="-avoid-version -export-symbols-regex '^flux_plugin_init\$\$' --disable-static -shared -export-dynamic $ld_gc_sections"
+AC_SUBST(fluxplugin_ldflags)
+
+fluxlib_ldflags="-shared -export-dynamic --disable-static $san_ld_zdef_flag $ld_gc_sections"
 AC_SUBST(fluxlib_ldflags)
 
 ##
@@ -450,48 +588,50 @@ AC_CONFIG_FILES( \
   src/common/libpmi/Makefile \
   src/common/libflux/Makefile \
   src/common/libflux/version.h \
+  src/common/libfluxutil/Makefile \
   src/common/libtestutil/Makefile \
   src/common/libkvs/Makefile \
+  src/common/libcontent/Makefile \
   src/common/libjob/Makefile \
   src/common/libsubprocess/Makefile \
   src/common/liboptparse/Makefile \
   src/common/libidset/Makefile \
   src/common/libtomlc99/Makefile \
-  src/common/libaggregate/Makefile \
   src/common/libschedutil/Makefile \
   src/common/libeventlog/Makefile \
   src/common/libioencode/Makefile \
   src/common/librouter/Makefile \
   src/common/libyuarel/Makefile \
+  src/common/libdebugged/Makefile \
+  src/common/libterminus/Makefile \
+  src/common/libhostlist/Makefile \
+  src/common/librlist/Makefile \
+  src/common/libczmqcontainers/Makefile \
+  src/common/libccan/Makefile \
+  src/common/libzmqutil/Makefile \
+  src/common/libtaskmap/Makefile \
+  src/common/libfilemap/Makefile \
+  src/common/libsdexec/Makefile \
+  src/common/libmissing/Makefile \
   src/bindings/Makefile \
   src/bindings/lua/Makefile \
   src/bindings/python/Makefile \
   src/bindings/python/flux/Makefile \
-  src/bindings/python/flux/core/Makefile \
   src/bindings/python/_flux/Makefile \
   src/broker/Makefile \
   src/cmd/Makefile \
   src/shell/Makefile \
   src/connectors/Makefile \
-  src/connectors/local/Makefile \
-  src/connectors/shmem/Makefile \
-  src/connectors/loop/Makefile \
-  src/connectors/ssh/Makefile \
   src/modules/Makefile \
-  src/modules/connector-local/Makefile \
   src/modules/kvs/Makefile \
-  src/modules/kvs-watch/Makefile \
-  src/modules/content-sqlite/Makefile \
-  src/modules/barrier/Makefile \
-  src/modules/cron/Makefile \
-  src/modules/aggregator/Makefile \
-  src/modules/pymod/Makefile \
-  src/modules/userdb/Makefile \
+  src/modules/content-files/Makefile \
+  src/modules/content-s3/Makefile \
   src/modules/job-ingest/Makefile \
   src/modules/job-manager/Makefile \
-  src/modules/job-info/Makefile \
+  src/modules/job-list/Makefile \
   src/modules/job-exec/Makefile \
-  src/modules/sched-simple/Makefile \
+  src/modules/resource/Makefile \
+  src/modules/sdbus/Makefile \
   src/test/Makefile \
   etc/Makefile \
   etc/flux-core.pc \
@@ -499,16 +639,21 @@ AC_CONFIG_FILES( \
   etc/flux-optparse.pc \
   etc/flux-idset.pc \
   etc/flux-schedutil.pc \
+  etc/flux-hostlist.pc \
+  etc/flux-taskmap.pc \
   etc/flux.service \
+  etc/flux-housekeeping@.service \
+  src/cmd/flux-run-housekeeping \
+  etc/flux-prolog@.service \
+  etc/flux-epilog@.service \
+  src/cmd/flux-run-prolog \
+  src/cmd/flux-run-epilog \
   doc/Makefile \
-  doc/man1/Makefile \
-  doc/man3/Makefile \
-  doc/man5/Makefile \
-  doc/man7/Makefile \
   doc/test/Makefile \
   t/Makefile \
   t/fluxometer/conf.lua \
   t/fluxometer/conf.lua.installed \
+  src/test/docker/poison-libflux.sh
 )
 
 AC_CONFIG_LINKS([ \
@@ -518,7 +663,10 @@ AC_CONFIG_LINKS([ \
 AC_OUTPUT
 
 AS_IF([test "x$enable_docs" != "xno"], [
-  if test -z "$ADOC"; then
-    AC_MSG_WARN([${ADOC_ERROR_MSG:-No asciidoc formatter found}. Manual pages will not be generated.])
+  if test "x$sphinx" = "xfalse"; then
+    AC_MSG_WARN([Python Sphinx not found. Manual pages will not be generated.])
+  elif test "x$docutils" = "xfalse"; then
+    AC_MSG_WARN([Python Docutils not found. Manual pages will not be generated.])
   fi
 ])
+
diff --git a/ChangeLog b/debian/README.Debian
similarity index 100%
rename from ChangeLog
rename to debian/README.Debian
diff --git a/src/modules/pymod/__init__.py b/debian/README.source
similarity index 100%
rename from src/modules/pymod/__init__.py
rename to debian/README.source
diff --git a/debian/compat b/debian/compat
new file mode 100644
index 000000000000..f599e28b8ab0
--- /dev/null
+++ b/debian/compat
@@ -0,0 +1 @@
+10
diff --git a/debian/control b/debian/control
new file mode 100644
index 000000000000..cb6ec63494c8
--- /dev/null
+++ b/debian/control
@@ -0,0 +1,37 @@
+Source: flux-core
+Section: devel
+Priority: optional
+Maintainer: Mark A. Grondona <mark.grondona@gmail.com>
+Standards-Version: 4.1.2
+Build-Depends:
+  debhelper (>= 10),
+  flux-security (>= 0.13.0),
+  libarchive-dev,
+  uuid-dev,
+  libzmq3-dev,
+  libjansson-dev,
+  liblz4-dev,
+  libhwloc-dev,
+  libsqlite3-dev,
+  lua5.1,
+  liblua5.1-dev,
+  lua-posix,
+  python3-dev,
+  python3-cffi,
+  python3-yaml,
+  python3-ply,
+  python3-sphinx,
+  jq
+
+Homepage: https://github.com/flux-framework/flux-core
+Package: flux-core
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Description:  Core support for the Flux resource manager framework
+ flux-core implements the communication layer and lowest level
+ services and interfaces for the Flux resource manager framework.
+ It consists of a distributed message broker, plug-in modules
+ that implement various distributed services, and an API and set
+ of utilities to utilize these services.
+ Note: This package is for testing purposes only and may not comply
+ with all Debian packaging guidelines.
diff --git a/debian/copyright b/debian/copyright
new file mode 100644
index 000000000000..1724bb2dcaa7
--- /dev/null
+++ b/debian/copyright
@@ -0,0 +1,77 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: flux-core
+Source: <https://github.com/flux-framework/flux-core>
+
+Files: *
+Copyright: 2014-2023 Lawrence Livermore National Security
+License: LGPL-3+
+
+Files: src/common/libev/*
+Copyright: Copyright (c) 2007-2013 Marc Alexander Lehmann.
+License: BSD-2
+
+Files: src/common/libczmqcontainers/*
+Copyright: Copyright (c) the Contributors as noted in the AUTHORS file.
+License: Mozilla Public License v. 2.0
+
+Files: src/common/libtap/*
+Copyright: Copyright (c) the Contributors as noted in the AUTHORS file.
+License: LGPL
+
+Files: src/common/libtomlc99/*
+Copyright: Copyright (c) 2017 CK Tan
+License: MIT
+
+Files: src/common/libyuarel/*
+Copyright: Copyright (C) 2016,2017 Jack Engqvist Johansson
+License: MIT
+
+Files: src/common/libccan/*
+Copyright: Rusty Russell <rusty@rustcorp.com.au>
+License: CC0
+
+Files: src/common/libccan/bitmap/*
+Copyright: David Gibson <david@gibson.dropbear.id.au>
+License: LGPL-2.1+
+
+Files: src/common/libccan/list/*
+Copyright: Rusty Russell <rusty@rustcorp.com.au>
+License: BSD-MIT
+
+Files: src/common/libccan/base64/*
+Copyright: Rusty Russell <rusty@rustcorp.com.au>
+License: BSD-MIT
+
+Files: t/sharness.sh
+Copyright: Copyright (c) 2011-2012 Mathias Lafeldt
+ Copyright (c) 2005-2012 Git project
+ Copyright (c) 2005-2012 Junio C Hamano
+ Copyright (c) 2019-2023 Felipe Contreras
+License: GPL-2.0
+
+Files: src/bindings/python/flux/utils/parsedatetime/*
+Copyright: Copyright 2004-2021 Mike Taylor
+License: Apache-2.0
+
+Files: src/bindings/python/flux/utils/tomli/*
+Copyright: Copyright (c) 2021 Taneli Hukkinen
+License: MIT
+
+Files: debian/*
+Copyright: 2023 Mark A. Grondona <grondo@unknown>
+License: GPL-2+
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License version 2 can be found in "/usr/share/common-licenses/GPL-2".
diff --git a/debian/flux-core-docs.docs b/debian/flux-core-docs.docs
new file mode 100644
index 000000000000..86ca00fceb06
--- /dev/null
+++ b/debian/flux-core-docs.docs
@@ -0,0 +1 @@
+README.Debian
diff --git a/debian/rules b/debian/rules
new file mode 100755
index 000000000000..67cb7c0f3455
--- /dev/null
+++ b/debian/rules
@@ -0,0 +1,26 @@
+#!/usr/bin/make -f
+# See debhelper(7) (uncomment to enable)
+# output every command that modifies files on the build system.
+#export DH_VERBOSE = 1
+
+# see FEATURE AREAS in dpkg-buildflags(1)
+#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+# see ENVIRONMENT in dpkg-buildflags(1)
+# package maintainers to append CFLAGS
+#export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
+# package maintainers to append LDFLAGS
+#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
+
+%:
+	dh $@
+
+override_dh_auto_configure:
+	PYTHON_VERSION=3 dh_auto_configure -- --with-systemdsystemunitdir=/lib/systemd/system --with-flux-security
+
+override_dh_autoreconf:
+	@echo not running autogen.sh on dist product
+
+override_dh_auto_install:
+	dh_auto_install
+	find . -name '*.la' -delete
diff --git a/debian/source/format b/debian/source/format
new file mode 100644
index 000000000000..163aaf8d82b6
--- /dev/null
+++ b/debian/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
diff --git a/doc/Makefile.am b/doc/Makefile.am
index 18751b9cb017..be4447ca1a7c 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1 +1,550 @@
-SUBDIRS = man1 man3 man5 man7 test
+.NOTPARALLEL:
+
+SUBDIRS = . test
+
+MAN_FILES = $(MAN1_FILES) $(MAN3_FILES) $(MAN5_FILES) $(MAN7_FILES)
+
+#######
+# Man 1
+#######
+
+MAN1_FILES = $(MAN1_FILES_PRIMARY) $(MAN1_FILES_SECONDARY)
+
+MAN1_FILES_PRIMARY = \
+	man1/flux.1 \
+	man1/flux-broker.1 \
+	man1/flux-kvs.1 \
+	man1/flux-keygen.1 \
+	man1/flux-logger.1 \
+	man1/flux-overlay.1 \
+	man1/flux-ping.1 \
+	man1/flux-start.1 \
+	man1/flux-startlog.1 \
+	man1/flux-module.1 \
+	man1/flux-exec.1 \
+	man1/flux-env.1 \
+	man1/flux-getattr.1 \
+	man1/flux-dmesg.1 \
+	man1/flux-dump.1 \
+	man1/flux-archive.1 \
+	man1/flux-content.1 \
+	man1/flux-config.1 \
+	man1/flux-proxy.1 \
+	man1/flux-queue.1 \
+	man1/flux-cron.1 \
+	man1/flux-event.1 \
+	man1/flux-submit.1 \
+	man1/flux-run.1 \
+	man1/flux-bulksubmit.1 \
+	man1/flux-alloc.1 \
+	man1/flux-batch.1 \
+	man1/flux-job.1 \
+	man1/flux-version.1 \
+	man1/flux-jobs.1 \
+	man1/flux-pmi.1 \
+	man1/flux-pstree.1 \
+	man1/flux-restore.1 \
+	man1/flux-shell.1 \
+	man1/flux-top.1 \
+	man1/flux-jobtap.1 \
+	man1/flux-shutdown.1 \
+	man1/flux-uptime.1 \
+	man1/flux-uri.1 \
+	man1/flux-resource.1 \
+	man1/flux-pgrep.1 \
+	man1/flux-cancel.1 \
+	man1/flux-watch.1 \
+	man1/flux-update.1 \
+	man1/flux-hostlist.1 \
+	man1/flux-housekeeping.1
+
+# These files are generated as clones of a primary page.
+# Sphinx handles this automatically if declared in the conf.py
+MAN1_FILES_SECONDARY = \
+	man1/flux-setattr.1 \
+	man1/flux-lsattr.1 \
+	man1/flux-pkill.1
+
+
+MAN3_FILES = $(MAN3_FILES_PRIMARY) $(MAN3_FILES_SECONDARY)
+
+MAN3_FILES_PRIMARY = \
+	man3/flux_shell_task_channel_subscribe.3 \
+	man3/flux_shell_add_completion_ref.3 \
+	man3/flux_shell_add_event_context.3 \
+	man3/flux_shell_add_event_handler.3 \
+	man3/flux_shell_aux_set.3 \
+	man3/flux_shell_current_task.3 \
+	man3/flux_shell_get_flux.3 \
+	man3/flux_shell_get_hwloc_xml.3 \
+	man3/flux_shell_get_info.3 \
+	man3/flux_shell_get_jobspec_info.3 \
+	man3/flux_shell_get_taskmap.3 \
+	man3/flux_shell_get_hostlist.3 \
+	man3/flux_shell_getenv.3 \
+	man3/flux_shell_getopt.3 \
+	man3/flux_shell_killall.3 \
+	man3/flux_shell_log.3 \
+	man3/flux_shell_plugstack_call.3 \
+	man3/flux_shell_rpc_pack.3 \
+	man3/flux_shell_service_register.3 \
+	man3/flux_shell_task_get_info.3 \
+	man3/flux_shell_task_subprocess.3 \
+	man3/flux_service_register.3 \
+	man3/flux_open.3 \
+	man3/flux_send.3 \
+	man3/flux_recv.3 \
+	man3/flux_requeue.3 \
+	man3/flux_aux_set.3 \
+	man3/flux_flags_set.3 \
+	man3/flux_comms_error_set.3 \
+	man3/flux_event_subscribe.3 \
+	man3/flux_event_publish.3 \
+	man3/flux_pollevents.3 \
+	man3/flux_msg_encode.3 \
+	man3/flux_msg_has_flag.3 \
+	man3/flux_rpc.3 \
+	man3/flux_get_rank.3 \
+	man3/flux_attr_get.3 \
+	man3/flux_get_reactor.3 \
+	man3/flux_reactor_create.3 \
+	man3/flux_fd_watcher_create.3 \
+	man3/flux_watcher_start.3 \
+	man3/flux_handle_watcher_create.3 \
+	man3/flux_timer_watcher_create.3 \
+	man3/flux_periodic_watcher_create.3 \
+	man3/flux_idle_watcher_create.3 \
+	man3/flux_watcher_set_priority.3 \
+	man3/flux_msg_handler_create.3 \
+	man3/flux_msg_cmp.3 \
+	man3/flux_msg_create.3 \
+	man3/flux_msg_handler_addvec.3 \
+	man3/flux_child_watcher_create.3 \
+	man3/flux_signal_watcher_create.3 \
+	man3/flux_stat_watcher_create.3 \
+	man3/flux_respond.3 \
+	man3/flux_reactor_now.3 \
+	man3/flux_request_decode.3 \
+	man3/flux_event_decode.3 \
+	man3/flux_response_decode.3 \
+	man3/flux_request_encode.3 \
+	man3/flux_log.3 \
+	man3/flux_future_get.3 \
+	man3/flux_future_create.3 \
+	man3/flux_future_wait_all_create.3 \
+	man3/flux_future_and_then.3 \
+	man3/flux_kvs_lookup.3 \
+	man3/flux_kvs_commit.3 \
+	man3/flux_kvs_txn_create.3 \
+	man3/flux_kvs_namespace_create.3 \
+	man3/flux_kvs_getroot.3 \
+	man3/flux_kvs_copy.3 \
+	man3/flux_core_version.3 \
+	man3/hostlist_create.3 \
+	man3/idset_create.3 \
+	man3/idset_encode.3 \
+	man3/idset_decode.3 \
+	man3/idset_add.3 \
+	man3/idset_alloc.3 \
+	man3/flux_jobtap_get_flux.3 \
+	man3/flux_sync_create.3 \
+	man3/flux_job_timeleft.3
+
+
+# These files are generated as clones of a primary page.
+# Sphinx handles this automatically if declared in the conf.py
+MAN3_FILES_SECONDARY = \
+	man3/flux_shell_remove_completion_ref.3 \
+	man3/flux_shell_aux_get.3 \
+	man3/flux_shell_task_first.3 \
+	man3/flux_shell_task_next.3 \
+	man3/flux_shell_info_unpack.3 \
+	man3/flux_shell_jobspec_info_unpack.3 \
+	man3/flux_shell_get_rank_info.3 \
+	man3/flux_shell_rank_info_unpack.3 \
+	man3/flux_shell_get_environ.3 \
+	man3/flux_shell_setenvf.3 \
+	man3/flux_shell_unsetenv.3 \
+	man3/flux_shell_getopt_unpack.3 \
+	man3/flux_shell_setopt.3 \
+	man3/flux_shell_setopt_pack.3 \
+	man3/flux_shell_err.3 \
+	man3/flux_shell_fatal.3 \
+	man3/flux_shell_raise.3 \
+	man3/flux_shell_log_setlevel.3 \
+	man3/flux_shell_task_info_unpack.3 \
+	man3/flux_shell_task_cmd.3 \
+	man3/flux_service_unregister.3 \
+	man3/flux_send_new.3 \
+	man3/flux_clone.3 \
+	man3/flux_close.3 \
+	man3/flux_reconnect.3 \
+	man3/flux_open_ex.3 \
+	man3/flux_aux_get.3 \
+	man3/flux_flags_unset.3 \
+	man3/flux_flags_get.3 \
+	man3/flux_event_unsubscribe.3 \
+	man3/flux_event_publish_pack.3 \
+	man3/flux_event_publish_raw.3 \
+	man3/flux_event_publish_get_seq.3 \
+	man3/flux_pollfd.3 \
+	man3/flux_msg_decode.3 \
+	man3/flux_msg_set_flag.3 \
+	man3/flux_msg_clear_flag.3 \
+	man3/flux_msg_is_private.3 \
+	man3/flux_msg_set_private.3 \
+	man3/flux_msg_is_streaming.3 \
+	man3/flux_msg_set_streaming.3 \
+	man3/flux_msg_is_noresponse.3 \
+	man3/flux_msg_set_noresponse.3 \
+	man3/flux_get_size.3 \
+	man3/flux_attr_set.3 \
+	man3/flux_set_reactor.3 \
+	man3/flux_reactor_destroy.3 \
+	man3/flux_reactor_run.3 \
+	man3/flux_reactor_stop.3 \
+	man3/flux_reactor_stop_error.3 \
+	man3/flux_reactor_active_incref.3 \
+	man3/flux_reactor_active_decref.3 \
+	man3/flux_fd_watcher_get_fd.3 \
+	man3/flux_watcher_stop.3 \
+	man3/flux_watcher_is_active.3 \
+	man3/flux_watcher_destroy.3 \
+	man3/flux_watcher_next_wakeup.3 \
+	man3/flux_handle_watcher_get_flux.3 \
+	man3/flux_timer_watcher_reset.3 \
+	man3/flux_periodic_watcher_reset.3 \
+	man3/flux_prepare_watcher_create.3 \
+	man3/flux_check_watcher_create.3 \
+	man3/flux_msg_handler_destroy.3 \
+	man3/flux_msg_handler_start.3 \
+	man3/flux_msg_handler_stop.3 \
+	man3/flux_msg_handler_delvec.3 \
+	man3/flux_child_watcher_get_rpid.3 \
+	man3/flux_child_watcher_get_rstatus.3 \
+	man3/flux_signal_watcher_get_signum.3 \
+	man3/flux_stat_watcher_get_rstat.3 \
+	man3/flux_respond_raw.3 \
+	man3/flux_respond_pack.3 \
+	man3/flux_respond_error.3 \
+	man3/flux_reactor_now_update.3 \
+	man3/flux_request_unpack.3 \
+	man3/flux_request_decode_raw.3 \
+	man3/flux_event_unpack.3 \
+	man3/flux_event_encode.3 \
+	man3/flux_event_pack.3 \
+	man3/flux_event_encode_raw.3 \
+	man3/flux_event_decode_raw.3 \
+	man3/flux_response_decode_raw.3 \
+	man3/flux_response_decode_error.3 \
+	man3/flux_request_encode_raw.3 \
+	man3/flux_vlog.3 \
+	man3/flux_log_set_appname.3 \
+	man3/flux_log_set_procid.3 \
+	man3/flux_future_then.3 \
+	man3/flux_future_wait_for.3 \
+	man3/flux_future_reset.3 \
+	man3/flux_future_destroy.3 \
+	man3/flux_future_fulfill.3 \
+	man3/flux_future_fulfill_error.3 \
+	man3/flux_future_fulfill_with.3 \
+	man3/flux_future_aux_set.3 \
+	man3/flux_future_aux_get.3 \
+	man3/flux_future_set_flux.3 \
+	man3/flux_future_get_flux.3 \
+	man3/flux_future_wait_any_create.3 \
+	man3/flux_future_push.3 \
+	man3/flux_future_first_child.3 \
+	man3/flux_future_next_child.3 \
+	man3/flux_future_get_child.3 \
+	man3/flux_future_or_then.3 \
+	man3/flux_future_continue.3 \
+	man3/flux_future_continue_error.3 \
+	man3/flux_msg_copy.3 \
+	man3/flux_msg_incref.3 \
+	man3/flux_msg_decref.3 \
+	man3/flux_msg_destroy.3 \
+	man3/flux_rpc_pack.3 \
+	man3/flux_rpc_raw.3 \
+	man3/flux_rpc_message.3 \
+	man3/flux_rpc_get.3 \
+	man3/flux_rpc_get_unpack.3 \
+	man3/flux_rpc_get_raw.3 \
+	man3/flux_rpc_get_matchtag.3 \
+	man3/flux_rpc_get_nodeid.3 \
+	man3/flux_kvs_lookupat.3 \
+	man3/flux_kvs_lookup_get.3 \
+	man3/flux_kvs_lookup_get_unpack.3 \
+	man3/flux_kvs_lookup_get_raw.3 \
+	man3/flux_kvs_lookup_get_dir.3 \
+	man3/flux_kvs_lookup_get_treeobj.3 \
+	man3/flux_kvs_lookup_get_symlink.3 \
+	man3/flux_kvs_getroot_get_treeobj.3 \
+	man3/flux_kvs_getroot_get_blobref.3 \
+	man3/flux_kvs_getroot_get_sequence.3 \
+	man3/flux_kvs_getroot_get_owner.3 \
+	man3/flux_kvs_getroot_cancel.3 \
+	man3/flux_kvs_fence.3 \
+	man3/flux_kvs_commit_get_treeobj.3 \
+	man3/flux_kvs_commit_get_sequence.3 \
+	man3/flux_kvs_txn_destroy.3 \
+	man3/flux_kvs_txn_put.3 \
+	man3/flux_kvs_txn_pack.3 \
+	man3/flux_kvs_txn_vpack.3 \
+	man3/flux_kvs_txn_mkdir.3 \
+	man3/flux_kvs_txn_unlink.3 \
+	man3/flux_kvs_txn_symlink.3 \
+	man3/flux_kvs_txn_put_raw.3 \
+	man3/flux_kvs_txn_put_treeobj.3 \
+	man3/flux_kvs_namespace_remove.3 \
+	man3/flux_kvs_move.3 \
+	man3/flux_core_version_string.3 \
+	man3/hostlist_destroy.3 \
+	man3/hostlist_decode.3 \
+	man3/hostlist_encode.3 \
+	man3/hostlist_copy.3 \
+	man3/hostlist_append.3 \
+	man3/hostlist_append_list.3 \
+	man3/hostlist_nth.3 \
+	man3/hostlist_find.3 \
+	man3/hostlist_delete.3 \
+	man3/hostlist_count.3 \
+	man3/hostlist_sort.3 \
+	man3/hostlist_uniq.3 \
+	man3/hostlist_first.3 \
+	man3/hostlist_last.3 \
+	man3/hostlist_next.3 \
+	man3/hostlist_current.3 \
+	man3/hostlist_remove_current.3 \
+	man3/idset_destroy.3 \
+	man3/idset_decode_ex.3 \
+	man3/idset_decode_empty.3 \
+	man3/idset_decode_info.3 \
+	man3/idset_decode_add.3 \
+	man3/idset_decode_subtract.3 \
+	man3/idset_copy.3 \
+	man3/idset_test.3 \
+	man3/idset_set.3 \
+	man3/idset_range_set.3 \
+	man3/idset_clear.3 \
+	man3/idset_range_clear.3 \
+	man3/idset_first.3 \
+	man3/idset_last.3 \
+	man3/idset_next.3 \
+	man3/idset_prev.3 \
+	man3/idset_empty.3 \
+	man3/idset_universe_size.3 \
+	man3/idset_count.3 \
+	man3/idset_equal.3 \
+	man3/idset_subtract.3 \
+	man3/idset_union.3 \
+	man3/idset_difference.3 \
+	man3/idset_intersect.3 \
+	man3/idset_has_intersection.3 \
+	man3/idset_clear_all.3 \
+	man3/idset_free.3 \
+	man3/idset_free_check.3 \
+	man3/flux_jobtap_service_register.3 \
+	man3/flux_jobtap_reprioritize_all.3 \
+	man3/flux_jobtap_reprioritize_job.3 \
+	man3/flux_jobtap_priority_unavail.3 \
+	man3/flux_jobtap_reject_job.3
+
+MAN5_FILES = $(MAN5_FILES_PRIMARY)
+
+MAN5_FILES_PRIMARY = \
+	man5/flux-config.5 \
+	man5/flux-config-access.5 \
+	man5/flux-config-bootstrap.5 \
+	man5/flux-config-tbon.5 \
+	man5/flux-config-exec.5 \
+	man5/flux-config-systemd.5 \
+	man5/flux-config-resource.5 \
+	man5/flux-config-job-manager.5 \
+	man5/flux-config-ingest.5 \
+	man5/flux-config-kvs.5 \
+	man5/flux-config-policy.5 \
+	man5/flux-config-queues.5
+
+
+MAN7_FILES = $(MAN7_FILES_PRIMARY)
+
+MAN7_FILES_PRIMARY = \
+	man7/flux-broker-attributes.7 \
+	man7/flux-jobtap-plugins.7 \
+	man7/flux-environment.7
+
+
+RST_FILES  = \
+	$(MAN1_FILES_PRIMARY:.1=.rst) \
+	$(MAN3_FILES_PRIMARY:.3=.rst) \
+	$(MAN5_FILES_PRIMARY:.5=.rst) \
+	$(MAN7_FILES_PRIMARY:.7=.rst)
+
+if ENABLE_DOCS
+man_MANS = $(MAN1_FILES) $(MAN3_FILES) $(MAN5_FILES) $(MAN7_FILES)
+$(RST_FILES): \
+	man1/common/resources.rst \
+	man1/common/experimental.rst \
+	man1/common/job-param-additional.rst \
+	man1/common/job-param-batch.rst \
+	man1/common/job-param-common.rst \
+	man1/common/job-param-pertask.rst \
+	man1/common/job-param-perres.rst \
+	man1/common/job-standard-io.rst \
+	man1/common/job-constraints.rst \
+	man1/common/job-dependencies.rst \
+	man1/common/job-environment.rst \
+	man1/common/job-env-rules.rst \
+	man1/common/job-environment-variables.rst \
+	man1/common/job-process-resource-limits.rst \
+	man1/common/job-other-options.rst \
+	man1/common/job-other-batch.rst \
+	man1/common/job-other-run.rst \
+	man1/common/job-shell-options.rst \
+	man3/common/resources.rst \
+	man3/common/experimental.rst \
+	man3/common/json_pack.rst \
+	man3/common/json_unpack.rst \
+	man5/common/resources.rst \
+	man5/common/experimental.rst \
+	man7/common/resources.rst \
+	man7/common/experimental.rst
+endif
+
+SUFFIXES = .rst .1 .3 .5 .7
+
+sphinx_man = $(sphinx_man_$(V))
+sphinx_man_ = $(sphinx_man_$(AM_DEFAULT_VERBOSITY))
+sphinx_man_0 = @echo "  BUILD     manpages";
+
+sphinx_html = $(sphinx_html_$(V))
+sphinx_html_ = $(sphinx_html_$(AM_DEFAULT_VERBOSITY))
+sphinx_html_0 = @echo "  BUILD     html";
+
+sphinx_verbose_flags = $(sphinx_verbose_flags_$(V))
+sphinx_verbose_flags_ = $(sphinx_verbose_flags_$(AM_DEFAULT_VERBOSITY))
+sphinx_verbose_flags_0 =
+sphinx_verbose_flags_1 = -v
+sphinx_verbose_flags_2 = -vv
+
+STDERR_DEVNULL = $(stderr_devnull_$(V))
+stderr_devnull_ =  $(stderr_devnull_$(AM_DEFAULT_VERBOSITY))
+stderr_devnull_0 = >/dev/null 2>&1
+
+$(MAN_FILES): manpages.py conf.py $(RST_FILES)
+	$(sphinx_man) \
+	SPHINX_BUILDDIR=$(abs_builddir) $(PYTHON) \
+		-m sphinx $(sphinx_verbose_flags) -b man $(srcdir) ./man \
+		$(STDERR_DEVNULL)
+	@echo "  MV        manpages"; \
+	for sec in 1 3 5 7; do \
+	  $(MKDIR_P) man$$sec && \
+	  mv -f $(abs_builddir)/man/*.$$sec man$$sec/; \
+	done
+
+.PHONY: html
+html: conf.py $(RST_FILES)
+	$(sphinx_html) \
+	SPHINX_BUILDDIR=$(abs_builddir) $(PYTHON) \
+		-m sphinx $(sphinx_verbose_flags) -b html $(srcdir) ./html \
+		$(STDERR_DEVNULL)
+
+EXTRA_DIST = \
+	conf.py \
+	manpages.py \
+	domainrefs.py \
+	index.rst \
+	index_man.rst \
+	requirements.txt \
+	guide/build.rst \
+	guide/support.rst \
+	guide/images/lgplv3-147x51.png \
+	guide/images/adminarch.dia \
+	guide/images/adminarch.png \
+	guide/images/fox-standing.svg \
+	guide/images/Frog-1.svg \
+	guide/images/states.dot \
+	guide/images/states.png \
+	guide/images/states_norm.dot \
+	guide/images/states_norm.png \
+	guide/start.rst \
+	guide/interact.rst \
+	guide/admin.rst \
+	guide/glossary.rst \
+	guide/internals.rst \
+	guide/debug.rst \
+	guide/kvs.rst \
+	guide/broker.rst \
+	$(RST_FILES) \
+	man1/index.rst \
+	man1/common/resources.rst \
+	man1/common/experimental.rst \
+	man1/common/job-param-additional.rst \
+	man1/common/job-param-batch.rst \
+	man1/common/job-param-common.rst \
+	man1/common/job-param-pertask.rst \
+	man1/common/job-param-perres.rst \
+	man1/common/job-standard-io.rst \
+	man1/common/job-constraints.rst \
+	man1/common/job-dependencies.rst \
+	man1/common/job-environment.rst \
+	man1/common/job-env-rules.rst \
+	man1/common/job-environment-variables.rst \
+	man1/common/job-process-resource-limits.rst \
+	man1/common/job-other-options.rst \
+	man1/common/job-other-batch.rst \
+	man1/common/job-other-run.rst \
+	man1/common/job-shell-options.rst \
+	man3/common/resources.rst \
+	man3/common/experimental.rst \
+	man3/index.rst \
+	man3/common/json_pack.rst \
+	man3/common/json_unpack.rst \
+	man5/common/resources.rst \
+	man5/common/experimental.rst \
+	man5/index.rst \
+	man7/index.rst \
+	man7/common/resources.rst \
+	man7/common/experimental.rst \
+	man7/flux-undocumented.rst
+
+CLEANFILES = \
+	$(MAN_FILES)
+
+clean-local:
+	-rm -rf man python/autogenerated
+
+distclean-local:
+	-rm -rf doctrees
+
+# test programs from man3
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS) \
+	-no-install
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_builddir)/src/common/libflux
+
+LDADD = \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(LIBPTHREAD)
+
+check_PROGRAMS = \
+	man3/example/open \
+	man3/example/send \
+	man3/example/recv \
+	man3/example/event \
+	man3/example/sync \
+	man3/example/rpc \
+	man3/example/rpc_then \
+	man3/example/info
+
+check_HEADERS = man3/example/die.h
diff --git a/doc/README.md b/doc/README.md
new file mode 100644
index 000000000000..a4c3cb2647c6
--- /dev/null
+++ b/doc/README.md
@@ -0,0 +1,64 @@
+# Flux Core Documentation
+
+Flux-core documentation currently consists of man pages written in ReStructured Text (rst). The man pages are generated using sphinx and are also hosted at flux-framework.readthedocs.io.
+
+##  Build Instructions
+
+To get setup with a virtual environment run:
+
+```bash
+virtualenv -p python3 sphinx-rtd
+source sphinx-rtd/bin/activate
+git clone git@github.com:flux-framework/flux-core
+cd flux-core/doc
+pip install -r requirements.txt
+```
+
+Users can then build the documentation, either as man pages or html web pages.
+Users can also run spellcheck.
+
+```
+sphinx-build -M man ./ _build/
+sphinx-build -M html ./ _build/
+sphinx-build -W -b spelling ./ _build/
+```
+
+## Adding a New Man Page
+
+There are 2 steps to adding a man page:
+- creating the man page documentation file
+- configuring the generation of the man page(s)
+
+### Creating the Man Page
+
+Man pages are written as [ReStructured Text](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html) (`.rst`) files.
+We use [Sphinx](https://www.sphinx-doc.org/en/master/) to process the documentation files and turn them into man pages (troff) and web pages (html).
+
+Sphinx automatically adds the following sections to the generated man page (so do not include them in the `.rst` file):
+
+- `NAME` (first section)
+- `AUTHOR` (penultimate section)
+- `COPYRIGHT` (final section)
+
+Each section title should be underlined with `=`
+
+### Configuring Generation
+
+Generating a man pages is done via the `man_pages` variable in `conf.py`. For example:
+
+```
+man_pages = [
+    ('man1/flux', 'flux', 'the Flux resource management framework', [author], 1),
+]
+```
+
+The tuple entry in the `man_pages` list specifies:
+- File name (relative path, without the `.rst` extension)
+- Name of man page
+- Description of man page
+- Author (use `[author]` as in the example)
+- Manual section for the generated man page
+
+It is possible for multiple man pages to be generated from a single source file.
+Simply create an entry for each man page you want generated.
+These entries can have the same file path, but different man page names.
diff --git a/doc/conf.py b/doc/conf.py
new file mode 100644
index 000000000000..74504fa721f9
--- /dev/null
+++ b/doc/conf.py
@@ -0,0 +1,187 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+# add `manpages` directory to sys.path
+import pathlib
+sys.path.append(str(pathlib.Path(__file__).absolute().parent))
+
+from manpages import man_pages
+import docutils.nodes
+
+# -- Project information -----------------------------------------------------
+
+project = 'flux-core'
+copyright = '''Copyright 2014 Lawrence Livermore National Security, LLC and Flux developers.
+
+SPDX-License-Identifier: LGPL-3.0'''
+
+# -- General configuration ---------------------------------------------------
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+master_doc = 'index'
+source_suffix = '.rst'
+
+extensions = [
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.napoleon',
+    'domainrefs'
+]
+
+domainrefs = {
+    'rfc': {
+        'text': 'RFC %s',
+        'url': 'https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_%s.html'
+    },
+    'linux:man1': {
+        'text': '%s(1)',
+        'url': 'https://linux.die.net/man/1/%s',
+        'url': 'http://man7.org/linux/man-pages/man1/%s.1.html'
+    },
+    'linux:man2': {
+        'text': '%s(2)',
+        'url': 'http://man7.org/linux/man-pages/man2/%s.2.html'
+    },
+    'linux:man3': {
+        'text': '%s(3)',
+        'url': 'http://man7.org/linux/man-pages/man3/%s.3.html'
+    },
+    'linux:man5': {
+        'text': '%s(5)',
+        'url': 'http://man7.org/linux/man-pages/man5/%s.5.html'
+    },
+    'linux:man7': {
+        'text': '%s(7)',
+        'url': 'http://man7.org/linux/man-pages/man7/%s.7.html'
+    },
+    'linux:man8': {
+        'text': '%s(8)',
+        'url': 'http://man7.org/linux/man-pages/man8/%s.8.html'
+    },
+    'security:man3': {
+        'text': '%s(3)',
+        'url': 'https://flux-framework.readthedocs.io/projects/flux-security/en/latest/man3/%s.html'
+    },
+    'security:man5': {
+        'text': '%s(5)',
+        'url': 'https://flux-framework.readthedocs.io/projects/flux-security/en/latest/man5/%s.html'
+    },
+    'security:man8': {
+        'text': '%s(8)',
+        'url': 'https://flux-framework.readthedocs.io/projects/flux-security/en/latest/man8/%s.html'
+    },
+}
+
+# Disable "smartquotes" to avoid things such as turning long-options
+#  "--" into en-dash in html output, which won't make much sense for
+#  manpages.
+smartquotes = False
+
+# -- Setup for Sphinx API Docs -----------------------------------------------
+
+# Workaround since sphinx does not automatically run apidoc before a build
+# Copied from https://github.com/readthedocs/readthedocs.org/issues/1139
+
+script_dir = os.path.normpath(os.path.dirname(__file__))
+py_bindings_dir = os.path.normpath(os.path.join(script_dir, "../src/bindings/python/"))
+
+# Make sure that the python bindings are in PYTHONPATH for autodoc
+sys.path.insert(0, py_bindings_dir)
+
+# run api doc
+def run_apidoc(_):
+    # Move import inside so that `gen-cmdhelp.py` can exec this file in LGTM.com
+    # without sphinx installed
+    # pylint: disable=import-outside-toplevel
+    from sphinx.ext.apidoc import main
+
+    try:
+        # Check if running under `make`
+        build_dir = os.path.normpath(os.environ.get('SPHINX_BUILDDIR'))
+    except:
+        build_dir = script_dir
+    output_path = os.path.join(build_dir, 'python', 'autogenerated')
+    exclusions = [os.path.join(py_bindings_dir, 'setup.py'),]
+    main(['-e', '-f', '-M', '-T', '-o', output_path, py_bindings_dir] + exclusions)
+
+def man_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
+    section = int(name[-1])
+    page = None
+    for man in man_pages:
+        if man[1] == text and man[4] == section:
+            page = man[0]
+            break
+    if page == None:
+        page = "man7/flux-undocumented"
+        section = 7
+
+    node = docutils.nodes.reference(
+        rawsource=rawtext,
+        text=f"{text}({section})",
+        refuri=f"../{page}.html",
+        **options,
+    )
+    return [node], []
+
+# launch setup
+def setup(app):
+    app.connect('builder-inited', run_apidoc)
+    for section in [ 1, 3, 5, 7 ]:
+        app.add_role(f"man{section}", man_role)
+
+# ReadTheDocs runs sphinx without first building Flux, so the cffi modules in
+# `_flux` will not exist, causing import errors.  Mock the imports to prevent
+# these errors.
+
+autodoc_mock_imports = ["_flux", "flux.constants", "yaml"]
+
+napoleon_google_docstring = True
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']
+
+# -- Options for Intersphinx -------------------------------------------------
+
+intersphinx_mapping = {
+    "rfc": (
+        "https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/",
+        None,
+    ),
+}
diff --git a/doc/domainrefs.py b/doc/domainrefs.py
new file mode 100644
index 000000000000..5662809e964b
--- /dev/null
+++ b/doc/domainrefs.py
@@ -0,0 +1,72 @@
+"""
+Originally from:
+
+  https://github.com/mitogen-hq/mitogen/blob/master/docs/domainrefs.py
+
+Copyright 2021, the Mitogen authors
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+import functools
+import re
+
+import docutils.nodes
+import docutils.utils
+
+
+CUSTOM_RE = re.compile('(.*) <(.*)>')
+
+
+def role(config, role, rawtext, text, lineno, inliner, options={}, content=[]):
+    template = 'https://docs.ansible.com/ansible/latest/modules/%s_module.html'
+
+    match = CUSTOM_RE.match(text)
+    if match:  # "custom text <real link>"
+        title = match.group(1)
+        text = match.group(2)
+    elif text.startswith('~'):  # brief
+        text = text[1:]
+        title = config.get('brief', '%s') % (
+            docutils.utils.unescape(text),
+        )
+    else:
+        title = config.get('text', '%s') % (
+            docutils.utils.unescape(text),
+        )
+
+    node = docutils.nodes.reference(
+        rawsource=rawtext,
+        text=title,
+        refuri=config['url'] % (text,),
+        **options
+    )
+
+    return [node], []
+
+
+def setup(app):
+    for name, info in app.config._raw_config['domainrefs'].items():
+        app.add_role(name, functools.partial(role, info))
diff --git a/doc/guide/admin.rst b/doc/guide/admin.rst
new file mode 100644
index 000000000000..b77d38bf2c13
--- /dev/null
+++ b/doc/guide/admin.rst
@@ -0,0 +1,1706 @@
+.. _admin-guide:
+
+##########################
+Flux Administrator's Guide
+##########################
+
+The *Flux Administrator's Guide* documents relevant information for
+installation, configuration, and management of Flux as the native
+resource manager on a cluster.
+
+.. note::
+    Flux is still beta software and many of the interfaces documented
+    in this guide may change with regularity.
+
+***********************
+Overview and Background
+***********************
+
+:doc:`start` and :doc:`interact` provide recipes for starting Flux and
+navigating a hierarchy of Flux instance that do not require administrator
+privilege or configuration.  It may be helpful to develop some perspective
+on Flux in these contexts before configuring a Flux system instance.
+
+Flux Architecture
+=================
+
+A *Flux instance* consists of one or more Flux brokers communicating over a
+tree-based overlay network.  Most of Flux's distributed systems and services
+that aren't directly associated with a running job are embedded in the
+:man1:`flux-broker` executable or its dynamically loaded plugins.
+
+Flux may be used in *single-user mode*, where a Flux instance is launched as
+a parallel job, and the *instance owner* (the user that submitted the parallel
+job) has control of, and exclusive access to, the Flux instance and its
+assigned resources.  On a system running Flux natively, batch jobs and
+allocations are examples of single user Flux instances.
+
+When Flux is deployed as the *system instance*, or native resource manager on
+a cluster, its brokers still run with the credentials of a non-privileged
+system user, typically ``flux``.  However, to support multiple users and
+act as a long running service, it must be configured to behave differently:
+
+- The Flux broker is started directly by systemd on each node instead of
+  being launched as a process in a parallel job.
+- The systemd unit file passes arguments to the broker that tell it to use
+  system paths for various files, and to ingest TOML files from a system
+  configuration directory.
+- A single security certificate is used for the entire cluster instead of
+  each broker generating one on the fly and exchanging public keys with PMI.
+- The Flux overlay network endpoints are statically configured from files
+  instead of being generated on on the fly and exchanged via PMI.
+- The instance owner is a system account that does not correspond to an
+  actual user.
+- Users other than the instance owner (*guests*) are permitted to connect
+  to the Flux broker, and are granted limited access to Flux services.
+- Users connect to the Flux broker's AF_UNIX socket via a well known system URI
+  if FLUX_URI is not set in the environment.
+- Job processes (including the Flux job shell) are launched as the submitting
+  user with the assistance of a setuid root helper on each node called the IMP.
+- Job requests are signed with MUNGE, and this signature is verified by the IMP.
+- The content of the Flux KVS, containing system state such as the set of
+  drained nodes and the job queue, is preserved across a full Flux restart.
+- The system instance functions with some nodes offline.
+- The system instance has no *initial program*.
+
+The same Flux executables are used in both single user and system modes,
+with operation differentiated only by configuration.
+
+.. figure:: images/adminarch.png
+   :alt: Flux system instance architecture
+   :align: center
+
+   Fox prevents Frog from submitting jobs on a cluster with Flux
+   as the system resource manager.
+
+Software Components
+===================
+
+Flux was conceived as a resource manager toolkit rather than a monolithic
+project, with the idea to make components like the scheduler replaceable.
+In addition, several parts of flux can be extended with plugins.  At this
+time the primary component types are
+
+broker modules
+  Each broker module runs in its own thread as part of the broker executable,
+  communicating with other components using messages.  Broker modules are
+  dynamically loadable with the :man1:`flux-module` command.  Core
+  services like the KVS, job manager, and scheduler are implemented using
+  broker modules.
+
+jobtap plugins
+  The job manager orchestrates a job's life cycle.  Jobtap plugins extend the
+  job manager, arranging for callbacks at different points in the job life
+  cycle.  Jobtap plugins may be dynamically loaded with the
+  :man1:`flux-jobtap` command.  An example of a jobtap plugin is the Flux
+  accounting multi-factor priority plugin, which updates a job's priority value
+  when it enters the PRIORITY state.
+
+shell plugins
+  When a job is started, the :man1:`flux-shell` is the process parent
+  of job tasks on each node.  Shell plugins extend the job environment and
+  can be configured on a per-job basis using the ``--setopt`` option of
+  :man1:`flux-run` and related job submission commands.  ``affinity``,
+  ``pmi``, and ``pty`` are examples of Flux shell plugins.
+
+connectors
+  Flux commands open a connection to a particular Flux instance by specifying
+  a URI.  The *scheme* portion of the URI may refer to a *native* connection
+  method such as ``local`` or ``ssh``.  Native connection methods are
+  implemented as plugins called *connectors*.  See :man3:`flux_open`.
+
+URI resolver plugins
+  Other URI schemes must be *resolved* to a native form before they can be used.
+  Resolvers for new schemes may be added as plugins.  For example, the ``lsf``
+  resolver plugin enables LSF users to connect to Flux instances running as LSF
+  jobs by specifying a ``lsf:JOBID`` URI.  See :man1:`flux-uri`.
+
+validator plugins
+  Jobs may be rejected at ingest if their jobspec fails one of a set of
+  configured validator plugins.  The basic validator ensures the jobspec
+  conforms to the jobspec specification.  The ``feasibility`` plugin rejects
+  job that the scheduler determines would be unable to run given the instance's
+  resource set.  The ``require-instance`` plugin rejects jobs that do not run
+  in a new Flux instance.  See :man5:`flux-config-ingest`.
+
+frobnicator plugins
+  The frobnicator allows a set of configured plugins to modify jobspec at
+  submission time.  For example the ``defaults`` plugin sets configured default
+  values for jobspec attributes such as *duration* and *queue*.  See
+  :man5:`flux-config-ingest`.
+
+Independently developed Flux components are generally packaged and versioned
+separately.  Each package may provide one or more of the above components
+as well as man pages and :man1:`flux` subcommands.  At this stage of Flux
+development, it is good practice to combine only contemporaneously released
+components as the interfaces are not stable yet.
+
+File Formats and Data Types
+===========================
+
+Since some parts of Flux are developed independently, some effort has been
+made to standardize file formats and data types to ensure components work
+together and provide a consistent user experience.  System administrators may
+find it useful to be aware of some of them.
+
+hostlist
+  A compact way of representing an ordered list of hostnames, compatible with
+  legacy tools in use at LLNL and defined by
+  `RFC 29 <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_29.html>`_.
+
+idset
+  A compact way of representing an unordered set of integers, defined by
+  `RFC 22 <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_22.html>`_.
+
+TOML
+  `Tom's Oblivious Minimal Language <https://github.com/toml-lang/toml>`_
+  is the file format used in Flux configuration files.
+
+JSON
+  `Javascript Object Notation <https://json-spec.readthedocs.io/reference.html>`_
+  is used throughout Flux in messages and other file formats.
+
+eventlog
+  An ordered log of timestamped events, stored in the Flux KVS and defined by
+  `RFC 18 <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_18.html>`_.
+  Eventlogs are used to record job events, capture standard I/O streams,
+  and record resource status changes.
+
+FSD
+  Flux Standard Duration, a string format used to represent a length of time,
+  defined by
+  `RFC 23 <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_23.html>`_.
+
+jobspec
+  A job request (JSON or YAML), defined by
+  `RFC 25 <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_25.html>`_ and
+  `RFC 14 <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_14.html>`_.
+
+R
+  A resource set (JSON), defined by
+  `RFC 20 <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_20.html>`_.
+
+FLUID
+  Flux Locally Unique ID, used for Flux job IDs, defined by
+  `RFC 19 <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_19.html>`_.
+
+Security
+========
+
+The Flux brokers that make up a system instance are started on each node by
+systemd.  The brokers run as an unprivileged system user, typically ``flux``.
+This user is termed the *instance owner*.  The instance owner has complete
+control of the Flux instance.
+
+A tree-based overlay network is established among brokers, rooted at the
+management node.  This network is secured and encrypted using the
+`ZeroMQ CURVE <https://rfc.zeromq.org/spec:25>`_ mechanism.  This requires
+a single CURVE certificate to be generated and installed on all nodes,
+typically ``/etc/flux/system/curve.cert``, before Flux begins operation.
+The certificate must be readable by the instance owner but should be carefully
+protected from access by other users since disclosure could allow overlay
+network security to be bypassed.
+
+On each node, users and tools may connect to the local system instance broker
+via a UNIX domain socket at a well known location, usually ``/run/flux/local``.
+Users are authenticated on this socket using the SO_PEERCRED socket option.
+Once connected, a user may inject messages into the overlay network.  Messages
+are stamped by the broker at ingress with the user's authenticated userid,
+and a *role mask* that identifies any special capabilities granted to the user.
+Messages that are sent by the ``flux`` user are stamped with the instance owner
+role, while other users, or *guests*, are stamped with a role that grants
+minimal access.  Note that the ``root`` user is considered a guest user with
+no special privilege in Flux, but sites can choose to grant ``root`` the owner
+role by configuration if desired.  See :security:man5:`flux-config-security`.
+
+Messages are used for remote procedure calls.  A Flux service may allow or deny
+an RPC request depending on its message rolemask or userid.  For example,
+only the instance owner can drain a node because the drain service only allows
+drain request messages that have the owner role.  Similarly, any job can be
+canceled by a cancel request message with the owner role, but in addition, jobs
+can be canceled by guests whose message userid matches the target job userid.
+
+A Flux job is launched when brokers launch one :man1:`flux-shell` per
+node with the credentials of the user that submitted the job.  When that is a
+guest user, Flux employs a setuid helper called the :security:man8:`flux-imp`
+to launch the shells with the guest credentials.  The shells in turn launch
+one or more user processes that compose the parallel job.
+
+The IMP is restricted by configuration to only allow the ``flux`` user to run
+it, and to only launch the system Flux job shell executable.  In addition, job
+requests are signed by the submitting user with
+`MUNGE <https://github.com/dun/munge>`_, and the IMP verifies this signature
+before starting the shells.  The current working directory of the job, the
+environment, and the executable command line are examples of job request data
+protected by the MUNGE signature.
+
+When Flux starts a batch job or allocation, it starts an independent,
+single-user Flux instance with brokers running as the submitting user.  The new
+instance owner has complete control over this Flux instance, which cannot use
+the IMP to launch jobs as guests, and does not permit guests to connect to
+its UNIX domain sockets.  Its overlay network is also secured with the ZeroMQ
+CURVE mechanism, but instead of starting with a shared certificate read from
+disk, each broker generates a certificate in memory on the fly, then exchanges
+public keys and socket endpoints with peer brokers using the PMI service
+offered by the Flux shells of the enclosing instance.  In other words, the
+single-user Flux instance bootstraps like an MPI parallel program.
+
+See also:
+`RFC 12 <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_12.html>`_,
+`RFC 15 <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_15.html>`_.
+
+Overlay Network
+===============
+
+As described above, a Flux instance consists of one or more Flux brokers
+communicating over a tree-based overlay network.  A Flux system instance
+on a cluster runs one broker per node.  The brokers connect to each other
+using TCP in a static tree topology, which is selected by configuration files.
+The broker at the tree root is the "leader".  The others are "followers".
+
+The leader is critical.  If it goes down, the entire Flux instance must
+restart.  Moreover, an unclean shutdown could result in lost job data.
+Therefore, it is desirable to arrange for the leader to run on a management
+node or other protected server that does not run user workloads.
+
+To a lesser degree, non-leaf follower (internal) nodes are also critical.
+If they are shut down or crash, the subtree rooted at that node must restart.
+However, the Flux instance continues and no data should be lost as long as
+the leader is unaffected.
+
+.. note::
+  At this time, when a node's broker restarts, any jobs running on the node
+  receive a fatal exception.  This will be addressed in a future release of
+  Flux that allows job shells to reconnect to the broker after it restarts.
+  For now, it means that restarting the leader affects all running jobs,
+  and restarting a non-leaf follower affects all jobs running on the subtree.
+
+The network used for the overlay network should be chosen for stability,
+as network partitions that persist long enough can cause downstream nodes
+to be declared lost.  This has the same effect as crashing.  Shorter
+partitions may cause nodes to be marked "torpid" and taken offline temporarily.
+On a cluster, the conservative choice is usually a commodity Ethernet rather
+than a high speed interconnect.  Note, however, that partition tolerance can
+be tuned when the network has known issues.  See :man5:`flux-config-tbon`.
+
+Topology for Small Clusters
+---------------------------
+
+The overlay topology can be configured in any tree shape.  Because of the
+criticality of internal nodes, the *flat* tree with no internal nodes has
+appeal for smaller clusters up to a few hundred nodes.  The downsides of
+a *flat* configuration, as the cluster size increases are:
+
+- The leader must directly manage all follower TCP connections.  For example,
+  it must iterate over all of them to publish (broadcast) a message.
+
+- The memory footprint of the leader node may grow large due to per-peer
+  message queues.
+
+- The advantages of hierarchical KVS caching are lost.  For example, when
+  starting a large job, the leader node must directly service each peer
+  lookup request for the same job data.
+
+- These extra overheads may affect the responsiveness of services that are
+  only present on the leader node, such as the job manager and the scheduler.
+
+The second example in :man5:`flux-config-bootstrap` is a *flat* topology.
+
+Topology for Large Clusters
+---------------------------
+
+To avoid the above downsides, larger clusters should use a custom topology
+with tree height of 2 and internal brokers assigned to stable, well connected,
+non-busy nodes.  The downside of this topology is, obviously:
+
+- More brokers are critical
+
+The third example in :man5:`flux-config-bootstrap` is a topology with a tree
+height of 2.
+
+************
+Installation
+************
+
+System Prerequisites
+====================
+
+`MUNGE <https://github.com/dun/munge>`_ is used to sign job requests
+submitted to Flux, so the MUNGE daemon should be installed on all
+nodes running Flux with the same MUNGE key used across the cluster.
+
+System clocks must be synchronized across the cluster, e.g. with
+`Network Time Protocol <https://en.wikipedia.org/wiki/Network_Time_Protocol>`_.
+
+Flux assumes a shared UID namespace across the cluster.
+
+A system user named ``flux`` is required.  This user need not have a valid
+home directory or shell.
+
+Flux uses `hwloc <https://www.open-mpi.org/projects/hwloc/>`_ to verify that
+configured resources are present on nodes.  Ensure that the system installed
+version includes any plugins needed for the hardware, especially GPUs.
+
+A Word about Core Dumps
+-----------------------
+
+It is helpful to enable core dumps from the system instance ``flux-broker``
+(especially rank 0) so that useful bug reports can be filed should the broker
+crash.  Usually :linux:man8:`systemd-coredump` handles this, which makes core
+files and stack traces accessible with :linux:man1:`coredumpctl`.
+
+Some sites choose instead to configure the ``kernel.core_pattern``
+:linux:man8:`sysctl` parameter to a relative file path, which directs core
+files to the program's current working directory.  Please note that the system
+instance broker runs as the ``flux`` user with a working directory of ``/``
+and thus would not have write permission on its current working directory.
+This can be worked around by installing a systemd override file, e.g.
+
+.. code-block::
+
+  # /etc/systemd/system/flux.service.d/override.conf
+  [Service]
+  WorkingDirectory=/var/lib/flux
+  LimitCORE=infinity:infinity
+
+.. note::
+  If you do observe a ``flux-broker`` crash, please open a github issue at
+  https://github.com/flux-framework/flux-core/issues and include the Flux
+  version, relevant log messages from ``journalctl -u flux``, and a stack
+  trace, if available.
+
+Installing Software Packages
+============================
+
+The following Flux framework packages are needed for a Flux system instance
+and should be installed from your Linux distribution package manager.
+
+flux-security
+  APIs for job signing, and the IMP, a privileged program for starting
+  processes as multiple users. Install on all nodes (required).  If building
+  flux-security from source, be sure to configure with ``--enable-pam`` to
+  include Pluggable Authentication Modules (PAM) support.
+
+flux-core
+  All of the core components of Flux, including the Flux broker.
+  flux-core is functional on its own, but cannot run jobs as multiple users,
+  has a simple FIFO scheduler, and does not implement accounting-based job
+  prioritization. If building flux-core from source, be sure to configure with
+  ``--with-flux-security``. Install on all nodes (required).
+
+flux-sched
+  The Fluxion graph-based scheduler.
+
+flux-accounting (optional)
+  Management of limits for individual users/projects, banks, and prioritization
+  based on fair-share accounting.  For more information on how to configure
+  run flux-accounting, please refer to the `Flux Accounting Guide <https://flux-framework.readthedocs.io/en/latest/guides/accounting-guide.html>`_.
+
+flux-pam (optional)
+  A PAM module that can enable users to login to compute nodes that are
+  running their jobs.
+
+.. note::
+    Flux packages are currently maintained only for the
+    `TOSS <https://computing.llnl.gov/projects/toss-speeding-commodity-cluster-computing>`_
+    Red Hat Enterprise Linux based Linux distribution, which is not publicly
+    distributed.  Open an issue in `flux-core <https://github.com/flux-framework/flux-core>`_
+    if you would like to become a maintainer of Flux packages for another Linux
+    distribution so we can share packaging tips and avoid duplicating effort.
+
+
+*************
+Configuration
+*************
+
+Much of Flux configuration occurs via
+`TOML <https://github.com/toml-lang/toml>`_ configuration files found in a
+hierarchy under ``/etc/flux``.  There are three separate TOML configuration
+spaces:  one for flux-security, one for the IMP (an independent component of
+flux-security), and one for Flux running as the system instance.  Each
+configuration space has a separate directory, from which all files matching
+the glob ``*.toml`` are read.  System administrators have the option of using
+one file for each configuration space, or breaking up each configuration space
+into multiple files.  In the examples below, one file per configuration space
+is used.
+
+For more information on the three configuration spaces, please refer to
+:man5:`flux-config`, :security:man5:`flux-config-security`, and
+:security:man5:`flux-config-security-imp`.
+
+Configuring flux-security
+=========================
+
+When Flux is built to support multi-user workloads, job requests are signed
+using a library provided by the flux-security project.  This library reads
+a static configuration from ``/etc/flux/security/conf.d/*.toml``. Note
+that for security, these files and their parent directory should be owned
+by ``root`` without write access to other users, so adjust permissions
+accordingly.
+
+Example file installed path: ``/etc/flux/security/conf.d/security.toml``
+
+.. code-block:: toml
+
+ # Job requests should be valid for 2 weeks
+ # Use munge as the job request signing mechanism
+ [sign]
+ max-ttl = 1209600  # 2 weeks
+ default-type = "munge"
+ allowed-types = [ "munge" ]
+
+See also: :security:man5:`flux-config-security-sign`.
+
+Configuring the IMP
+===================
+
+The Independent Minister of Privilege (IMP) is the only program that runs
+as root, by way of the setuid mode bit.  To enhance security, it has a
+private configuration space in ``/etc/flux/imp/conf.d/*.toml``. Note that
+the IMP will verify that files in this path and their parent directories
+are owned by ``root`` without write access from other users, so adjust
+permissions and ownership accordingly.
+
+Example file installed path: ``/etc/flux/imp/conf.d/imp.toml``
+
+.. code-block:: toml
+
+ # Only allow access to the IMP exec method by the 'flux' user.
+ # Only allow the installed version of flux-shell(1) to be executed.
+ [exec]
+ allowed-users = [ "flux" ]
+ allowed-shells = [ "/usr/libexec/flux/flux-shell" ]
+
+ # Enable the "flux" PAM stack (requires PAM configuration file)
+ pam-support = true
+
+See also: :security:man5:`flux-config-security-imp`.
+
+Configuring the Flux PAM Stack
+------------------------------
+
+If PAM support is enabled in the IMP config, the ``flux`` PAM stack must
+exist and have at least one ``auth`` and one ``session`` module.
+
+Example file installed path: ``/etc/pam.d/flux``
+
+.. code-block:: console
+
+  auth    required pam_localuser.so
+  session required pam_limits.so
+
+The ``pam_limits.so`` module is useful for setting default job resource
+limits.  If it is not used, jobs run in the system instance may inherit
+inappropriate limits from ``flux-broker``.
+
+.. note::
+  The linux kernel employs a heuristic when assigning initial limits to
+  pid 1.  For example, the max user processes and max pending signals
+  are scaled by the amount of system RAM.  The Flux system broker inherits
+  these limits and passes them on to jobs if PAM limits are not configured.
+  This may result in rlimit warning messages similar to
+
+  .. code-block:: console
+
+    flux-shell[0]:  WARN: rlimit: nproc exceeds current max, raising value to hard limit
+
+.. _config_cert:
+
+Configuring the Network Certificate
+===================================
+
+Overlay network security requires a certificate to be distributed to all nodes.
+It should be readable only by the ``flux`` user.  To create a new certificate,
+run :man1:`flux-keygen` as the ``flux`` user, then copy the result to
+``/etc/flux/system`` since the ``flux`` user will not have write access to
+this location:
+
+.. code-block:: console
+
+ $ sudo -u flux flux keygen /tmp/curve.cert
+ $ sudo mv /tmp/curve.cert /etc/flux/system/curve.cert
+
+Do this once and then copy the certificate to the same location on
+the other nodes, preserving owner and mode.
+
+.. note::
+    The ``flux`` user only needs read access to the certificate and
+    other files and directories under ``/etc/flux``. Keeping these files
+    and directories non-writable by user ``flux`` adds an extra layer of
+    security for the system instance configuration.
+
+Systemd and cgroup unified hierarchy
+====================================
+
+The flux systemd unit launches a systemd user instance as the flux user.
+It is recommended to use this to run user jobs, as it provides cgroups
+containment and the ability to enforce memory limits.  To do this, Flux
+requires the cgroup version 2 unified hierarchy:
+
+- The cgroup2 file system must be mounted on  ``/sys/fs/cgroup``
+
+- On some systems, add ``systemd.unified_cgroup_hierarchy=1`` to the
+  kernel command line (RHEL 8).
+
+- On some systems, add ``cgroup_enable=memory`` to the kernel command line
+  (debian 12).
+
+The configuration that follows presumes jobs will be launched through systemd,
+although it is not strictly required if your system cannot meet these
+prerequisites.
+
+.. _config-flux:
+
+Configuring the Flux System Instance
+====================================
+
+Although the security components need to be isolated, most Flux components
+share a common configuration space, which for the system instance is located
+in ``/etc/flux/system/conf.d/*.toml``.  The Flux broker for the system instance
+is pointed to this configuration by the systemd unit file.
+
+Example file installed path: ``/etc/flux/system/conf.d/system.toml``
+
+.. code-block:: toml
+
+ # Enable the sdbus and sdexec broker modules
+ [systemd]
+ enable = true
+
+ # Flux needs to know the path to the IMP executable
+ [exec]
+ imp = "/usr/libexec/flux/flux-imp"
+
+ # Run jobs in a systemd user instance
+ service = "sdexec"
+
+ # Limit jobs to a percentage of physical memory
+ [exec.sdexec-properties]
+ MemoryMax = "95%"
+
+ # Allow users other than the instance owner (guests) to connect to Flux
+ # Optionally, root may be given "owner privileges" for convenience
+ [access]
+ allow-guest-user = true
+ allow-root-owner = true
+
+ # Point to shared network certificate generated flux-keygen(1).
+ # Define the network endpoints for Flux's tree based overlay network
+ # and inform Flux of the hostnames that will start flux-broker(1).
+ [bootstrap]
+ curve_cert = "/etc/flux/system/curve.cert"
+
+ default_port = 8050
+ default_bind = "tcp://eth0:%p"
+ default_connect = "tcp://%h:%p"
+
+ # Rank 0 is the TBON parent of all brokers unless explicitly set with
+ # parent directives.
+ hosts = [
+    { host = "test[1-16]" },
+ ]
+
+ # Speed up detection of crashed network peers (system default is around 20m)
+ [tbon]
+ tcp_user_timeout = "2m"
+
+ # Uncomment 'norestrict' if flux broker is constrained to system cores by
+ # systemd or other site policy.  This allows jobs to run on assigned cores.
+ # Uncomment 'exclude' to avoid scheduling jobs on certain nodes (e.g. login,
+ # management, or service nodes).
+ [resource]
+ #norestrict = true
+ #exclude = "test[1-2]"
+
+ [[resource.config]]
+ hosts = "test[1-15]"
+ cores = "0-7"
+ gpus = "0"
+
+ [[resource.config]]
+ hosts = "test16"
+ cores = "0-63"
+ gpus = "0-1"
+ properties = ["fatnode"]
+
+ # Store the kvs root hash in sqlite periodically in case of broker crash.
+ # Recommend offline KVS garbage collection when commit threshold is reached.
+ [kvs]
+ checkpoint-period = "30m"
+ gc-threshold = 100000
+
+ # Immediately reject jobs with invalid jobspec or unsatisfiable resources
+ [ingest.validator]
+ plugins = [ "jobspec", "feasibility" ]
+
+ # Remove inactive jobs from the KVS after one week.
+ [job-manager]
+ inactive-age-limit = "7d"
+
+ # Jobs submitted without duration get a very short one
+ [policy.jobspec.defaults.system]
+ duration = "1m"
+
+ # Jobs that explicitly request more than the following limits are rejected
+ [policy.limits]
+ duration = "2h"
+ job-size.max.nnodes = 8
+ job-size.max.ncores = 32
+
+ # Configure the flux-sched (fluxion) scheduler policies
+ # The 'lonodex' match policy selects node-exclusive scheduling, and can be
+ # commented out if jobs may share nodes.
+ [sched-fluxion-qmanager]
+ queue-policy = "easy"
+ [sched-fluxion-resource]
+ match-policy = "lonodex"
+ match-format = "rv1_nosched"
+
+See also: :man5:`flux-config-exec`, :man5:`flux-config-access`
+:man5:`flux-config-bootstrap`, :man5:`flux-config-tbon`,
+:man5:`flux-config-resource`, :man5:`flux-config-ingest`,
+:man5:`flux-config-job-manager`,
+:man5:`flux-config-policy`, :man5:`flux-config-kvs`,
+:man5:`flux-config-systemd`,
+:sched:man5:`flux-config-sched-fluxion-qmanager`,
+:sched:man5:`flux-config-sched-fluxion-resource`.
+
+
+Configuring Resources
+=====================
+
+The Flux system instance must be configured with a static resource set.
+The ``resource.config`` TOML array in the example above is the preferred
+way to configure clusters with a resource set consisting of only nodes,
+cores, and GPUs.
+
+More complex resource sets may be represented by generating a file in
+RFC 20 (R version 1) form with scheduler extensions using a combination of
+``flux R encode`` and ``flux ion-R encode`` and then configuring
+``resource.path`` to its fully-qualified file path.  The details of this
+method are beyond the scope of this document.
+
+When Flux is running, ``flux resource list`` shows the configured resource
+set and any resource properties.
+
+Persistent Storage on Rank 0
+============================
+
+Flux is prolific in its use of disk space to back up its key value store,
+proportional to the number of jobs run and the quantity of standard I/O.
+On your rank 0 node, ensure that the ``statedir`` directory (normally
+``/var/lib/flux``) has plenty of space and is preserved across Flux instance
+restarts.
+
+The ``statedir`` directory is used for the ``content.sqlite`` file that
+contains content addressable storage backing the Flux key value store (KVS).
+
+Adding Prolog/Epilog/Housekeeping Scripts
+=========================================
+
+Flux can execute site-defined scripts as root on compute nodes before and
+after each job.
+
+prolog
+  The prolog runs as soon as the job enters RUN state.  Job shells are not
+  launched until all prolog tasks have completed.  If the prolog fails on
+  any nodes, or if any node takes longer than a fail-safe timeout (default
+  30m), those nodes are drained and a fatal exception is raised on the job.
+  If the job is canceled or reaches its time limit during the prolog, the
+  prolog is simply aborted and the job enters COMPLETING state.
+
+epilog
+  The epilog runs after job shell exits on all nodes, with the job held
+  in COMPLETING state until all epilog tasks have terminated.  If the epilog
+  fails on any nodes, those nodes are drained and a fatal exception is raised
+  on the job.  There is no default epilog timeout.
+
+housekeeping
+  Housekeeping runs after the job has reached the INACTIVE state.  It is not
+  recorded in the job eventlog and does not affect the job result.  If
+  housekeeping fails on any nodes, those nodes are drained.  Housekeeping
+  releases resources to the scheduler as they complete.
+
+The configuration of prolog, epilog, and housekeeping requires the following
+steps:
+
+ 1. Create executable scripts named ``prolog``, ``epilog``, and
+    ``housekeeping`` in ``/etc/flux/system``.  A suggested approach is to have
+    these scripts run all executables found in ``prolog.d``, ``epilog.d``,
+    and ``housekeeping.d`` respectively.  For example:
+
+    .. code-block:: sh
+
+       #!/bin/sh
+
+       exit_rc=0
+       periname=prolog
+       peridir=/etc/flux/system/${periname}.d
+
+       # This script may be run in test with 'sudo flux run-prolog'
+       test $FLUX_JOB_USERID && userid=$(id -n -u $FLUX_JOB_USERID 2>/dev/null)
+       echo Running $periname for ${FLUX_JOB_ID:-unknown}/${userid:-unknown}
+
+       for file in ${peridir}/*; do
+           test -e $file || continue
+           name=$(basename $file)
+           echo running $name >&2
+           $file
+           rc=$?
+           test $rc -ne 0 && echo "$name exit $rc" >&2
+           test $rc -gt $exit_rc && exit_rc=$rc
+       done
+
+       exit $exit_rc
+
+    Scripts may use :envvar:`FLUX_JOB_ID` and :envvar:`FLUX_JOB_USERID`
+    to take job or user specific actions.  Flux commands can be run from
+    the scripts with instance owner credentials if the system is configured
+    for root access as suggested in :ref:`config-flux`.
+
+    The IMP sets :envvar:`PATH` to a safe ``/usr/sbin:/usr/bin:/sbin:/bin``.
+
+ 2. Flux provides systemd *oneshot* units ``flux-prolog@``, ``flux-epilog@``,
+    and ``flux-housekeeping@`` templated by jobid, which run the user-provided
+    scripts installed in the previous step.  Configure the IMP to allow the
+    system instance user to start these units as root via the provided
+    provided wrapper scripts:
+
+    .. code-block:: toml
+
+       [run.prolog]
+       allowed-users = [ "flux" ]
+       allowed-environment = [ "FLUX_*" ]
+       path = "/usr/libexec/flux/cmd/flux-run-prolog"
+
+       [run.epilog]
+       allowed-users = [ "flux" ]
+       allowed-environment = [ "FLUX_*" ]
+       path = "/usr/libexec/flux/cmd/flux-run-epilog"
+
+       [run.housekeeping]
+       allowed-users = [ "flux" ]
+       allowed-environment = [ "FLUX_*" ]
+       path = "/usr/libexec/flux/cmd/flux-run-housekeeping"
+
+
+ 3. Configure the Flux system instance to run prolog, epilog, and housekeeping:
+
+    .. code-block:: toml
+
+       [job-manager]
+       plugins = [
+         { load = "perilog.so" }
+       ]
+
+       [job-manager.prolog]
+       per-rank = true
+       # timeout = "30m"
+
+       [job-manager.epilog]
+       per-rank = true
+       # timeout = "0"
+
+       [job-manager.housekeeping]
+       release-after = "30s"
+
+Standard output and standard error of the prolog, epilog, and housekeeping
+units are captured by the systemd journal.  Standard systemd tools like
+:linux:man1:`systemctl` and :linux:man1:`journalctl` can be used to
+observe and manipulate the prolog, epilog, and housekeeping systemd units.
+
+See also:
+:man1:`flux-housekeeping`.
+:man5:`flux-config-job-manager`,
+:security:man5:`flux-config-security-imp`,
+
+Adding Job Request Validation
+=============================
+
+Jobs are submitted to Flux via a job-ingest service. This service
+validates all jobs before they are assigned a jobid and announced to
+the job manager. By default, only basic validation is done, but the
+validator supports plugins so that job ingest validation is configurable.
+
+The list of available plugins can be queried via
+``flux job-validator --list-plugins``. The current list of plugins
+distributed with Flux is shown below:
+
+.. code-block:: console
+
+  $ flux job-validator --list-plugins
+  Available plugins:
+  feasibility           Use feasibility service to validate job
+  jobspec               Python bindings based jobspec validator
+  require-instance      Require that all jobs are new instances of Flux
+  schema                Validate jobspec using jsonschema
+
+Only the ``jobspec`` plugin is enabled by default.
+
+In a system instance, it may be useful to also enable the ``feasibility`` and
+``require-instance`` validators.  This can be done by configuring the Flux
+system instance via the ``ingest`` TOML table, as shown in the example below:
+
+.. code-block:: toml
+
+  [ingest.validator]
+  plugins = [ "jobspec", "feasibility", "require-instance" ]
+
+The ``feasibility`` plugin will allow the scheduler to reject jobs that
+are not feasible given the current resource configuration. Otherwise, these
+jobs are enqueued, but will have a job exception raised once the job is
+considered for scheduling.
+
+The ``require-instance`` plugin rejects jobs that do not start another
+instance of Flux. That is, jobs are required to be submitted via tools
+like :man1:`flux-batch` and :man1:`flux-alloc`, or the equivalent.
+For example, with this plugin enabled, a user running :man1:`flux-run`
+will have their job rejected with the message:
+
+.. code-block:: console
+
+  $ flux run -n 1000 myapp
+  flux-run: ERROR: [Errno 22] Direct job submission is disabled for this instance. Please use the flux-batch(1) or flux-alloc(1) commands.
+
+See also: :man5:`flux-config-ingest`.
+
+Adding Queues
+=============
+
+It may be useful to configure a Flux system instance with multiple queues.
+Each queue should be associated with a non-overlapping resource subset,
+identified by property name. It is good practice for queues to create a
+new property that has the same name as the queue. (There is no requirement
+that queue properties match the queue name, but this will cause extra entries
+in the PROPERTIES column for these queues. When queue names match property
+names, :command:`flux resource list` suppresses these matching properties
+in its output.)
+
+When queues are defined, all jobs must specify a queue at submission time.
+If that is inconvenient, then ``policy.jobspec.defaults.system.queue`` may
+define a default queue.
+
+Finally, queues can override the ``[policy]`` table on a per queue basis.
+This is useful for setting queue-specific limits.
+
+Here is an example that puts these concepts together:
+
+.. code-block:: toml
+
+ [policy]
+ jobspec.defaults.system.duration = "1m"
+ jobspec.defaults.system.queue = "debug"
+
+ [[resource.config]]
+ hosts = "test[1-4]"
+ properties = ["debug"]
+
+ [[resource.config]]
+ hosts = "test[5-16]"
+ properties = ["batch"]
+
+ [queues.debug]
+ requires = ["debug"]
+ policy.limits.duration = "30m"
+
+ [queues.batch]
+ requires = ["batch"]
+ policy.limits.duration = "4h"
+
+When named queues are configured, :man1:`flux-queue` may be used to
+list them:
+
+.. code-block:: console
+
+ $ flux queue status
+ batch: Job submission is enabled
+ debug: Job submission is enabled
+ Scheduling is enabled
+
+See also: :man5:`flux-config-policy`, :man5:`flux-config-queues`,
+:man5:`flux-config-resource`, :man1:`flux-queue`.
+
+Policy Limits
+=============
+
+Job duration and size are unlimited by default, or limited by the scheduler
+feasibility check discussed above, if configured.  When policy limits are
+configured, the job request is compared against them *after* any configured
+jobspec defaults are set, and *before* the scheduler feasibility check.
+If the job would exceed a duration or job size policy limit, the job submission
+is rejected.
+
+.. warning::
+  flux-sched 0.25.0 limitation: jobs that specify nodes but not cores may
+  escape flux-core's ``ncores`` policy limit, and jobs that specify cores but
+  not nodes may escape the ``nnodes`` policy limit.  The flux-sched feasibility
+  check will eventually cover this case.  Until then, be sure to set both
+  ``nnodes`` *and* ``ncores`` limits when configuring job size policy limits.
+
+Limits are global when set in the top level ``[policy]`` table.  Global limits
+may be overridden by a ``policy`` table within a ``[queues]`` entry.  Here is
+an example which implements duration and job size limits for two queues:
+
+.. code-block:: toml
+
+ # Global defaults
+ [policy]
+ jobspec.defaults.system.duration = "1m"
+ jobspec.defaults.system.queue = "debug"
+
+ [queues.debug]
+ requires = ["debug"]
+ policy.limits.duration = "30m"
+ policy.limits.job-size.max.nnodes = 2
+ policy.limits.job-size.max.ncores = 16
+
+ [queues.batch]
+ requires = ["batch"]
+ policy.limits.duration = "8h"
+ policy.limits.job-size.max.nnodes = 16
+ policy.limits.job-size.max.ncores = 128
+
+See also: :man5:`flux-config-policy`.
+
+Use PAM to Restrict Access to Compute Nodes
+===========================================
+
+If Pluggable Authentication Modules (PAM) are in use within a cluster, it may
+be convenient to use the ``pam_flux.so`` *account* module to configure a PAM
+stack that denies users access to compute nodes unless they have a job running
+there.
+
+Install the ``flux-pam`` package to make the ``pam_flux.so`` module available
+to be added to one or more PAM stacks, e.g.
+
+.. code-block:: console
+
+  account  sufficient pam_flux.so
+
+********************
+Pre-flight Checklist
+********************
+
+Here are some things to check before going live with a new Flux system
+instance.
+
+Do I have all the right packages installed?
+===========================================
+
+Flux packages should be installed on all nodes.
+
+.. list-table::
+   :header-rows: 1
+
+   * - Package name
+   * - flux-core
+   * - flux-security
+   * - flux-sched
+   * - flux-pam (optional)
+   * - flux-accounting (optional)
+
+Does /var/lib/flux have plenty of space on the leader node?
+===========================================================
+
+Flux stores its databases and KVS dumps in the ``statedir`` on the
+leader (management) node.  Storage consumption depends on usage, the
+size of the cluster, and other factors but be generous as running out
+of space on the leader node is catastrophic to Flux.
+
+The ``statedir`` is created automatically by systemd if it does not
+exist when Flux is started.  If you are creating it, it should be owned
+by the ``flux`` user and private to that user.
+
+Is Munge working?
+=================
+
+Munge daemons must be running on every node, clocks must be synchronized,
+and all nodes should be using the same pre-shared munge key.
+
+.. code-block:: console
+
+  $ pdsh -a systemctl is-active munge | dshbak -c
+  ----------------
+  test[0-7]
+  ----------------
+  active
+
+  $ pdsh -a "timedatectl | grep synchronized:" | dshbak -c
+  ----------------
+  test[0-7]
+  ----------------
+  System clock synchronized: yes
+
+  # spot check
+  $ echo xyz | ssh test1 munge | ssh test2 unmunge
+  STATUS:          Success (0)
+  ENCODE_HOST:     test1 (192.168.88.246)
+  ENCODE_TIME:     2024-04-18 09:41:21 -0700 (1713458481)
+  DECODE_TIME:     2024-04-18 09:41:21 -0700 (1713458481)
+  TTL:             300
+  CIPHER:          aes128 (4)
+  MAC:             sha256 (5)
+  ZIP:             none (0)
+  UID:             testuser (100)
+  GID:             testuser (100)
+  LENGTH:          4
+
+Are users set up properly?
+==========================
+
+Flux requires that the ``flux`` user and all other users that will be
+using Flux have the a consistent UID assignment across the cluster.
+
+.. code-block:: console
+
+  $ pdsh -a id flux | dshbak -c
+  ----------------
+  test[0-7]
+  ----------------
+  uid=500(flux) gid=500(flux) groups=500(flux)
+
+Is the Flux network certificate synced?
+=======================================
+
+The network certificate should be identical on all nodes and should
+only be readable by the ``flux`` user:
+
+.. code-block:: console
+
+  $ sudo pdsh -a md5sum /etc/flux/system/curve.cert | dshbak -c
+  ----------------
+  test[0-7]
+  ----------------
+  1b3c226159b9041d357a924841845cec  /etc/flux/system/curve.cert
+
+  $ pdsh -a stat -c '"%U %A"' /etc/flux/system/curve.cert | dshbak -c
+  ----------------
+  test[0-7]
+  ----------------
+  flux -r--------
+
+See :ref:`config_cert`.
+
+Is the Flux configuration synced?
+=================================
+
+The Flux configurations for system, security, and imp should
+be identical on all nodes, owned by root, and publicly readable:
+
+
+.. code-block:: console
+
+  $ pdsh -a "flux config get --config-path=system | md5sum" | dshbak -c
+  ----------------
+  test[1-7]
+  ----------------
+  432378ee4f210a879162e1ac66465c0e  -
+  $ pdsh -a "flux config get --config-path=security | md5sum" |dshbak -c
+  ----------------
+  test[1-7]
+  ----------------
+  1c53f68eea714a1b0641f201130e0d29  -
+  $ pdsh -a "flux config get --config-path=imp | md5sum" |dshbak -c
+  ----------------
+  test[0-7]
+  ----------------
+  e69c9d49356f4f1ecb76befdac727ef4  -
+
+  $ pdsh -a stat -c '"%U %A"' /etc/flux/system/conf.d /etc/flux/security/conf.d /etc/flux/imp/conf.d |dshbak -c
+  ----------------
+  test[0-7]
+  ----------------
+  root drwxr-xr-x
+  root drwxr-xr-x
+  root drwxr-xr-x
+
+Will the network be able to wire up?
+====================================
+
+Check your firewall rules and DNS/hosts configuration to ensure that each
+broker will be able to look up and connect to its configured parent in the
+tree based overlay network using TCP.
+
+Will the network stay up?
+=========================
+
+Although TCP is a reliable transport, the network used by the Flux overlay
+should be stable, otherwise:
+
+- Nodes can be temporarily marked offline for scheduling if the Flux broker
+  remains connected but cannot get messages through promptly.  This may be
+  tuned with ``tbon.torpid_max``.
+
+- Nodes can be disconnected (and running jobs lost) when TCP acknowledgements
+  cannot get through in time.  For example, this may happen during a network
+  partition.  This may be tuned with ``tbon.tcp_user_timeout``.
+
+If the network is expected to be unstable (e.g. while the bugs are worked
+out of new hardware), then the above values may need to be temporarily
+increased to avoid nuisance failures.  See :man5:`flux-config-tbon`.
+
+Is the Flux resource configuration correct?
+===========================================
+
+Ensure all nodes have the same resource configuration and that the summary
+looks sane:
+
+.. code-block:: console
+
+  $ pdsh -a "flux R parse-config /etc/flux/system/conf.d | flux R decode --short" | dshbak -c
+  ----------------
+  test[0-7]
+  ----------------
+  rank[0-7]/core[0-3]
+
+  $ pdsh -a "flux R parse-config /etc/flux/system/conf.d | flux R decode --nodelist" | dshbak -c
+  ----------------
+  test[0-7]
+  ----------------
+  test[0-7]
+
+Does the leader broker start?
+=============================
+
+Try to start the leader (rank 0) broker on the management node.
+
+.. code-block:: console
+
+    $ sudo systemctl start flux
+    $ flux uptime
+     07:42:52 run 3.8s,  owner flux,  depth 0,  size 8,  7 offline
+    $ systemctl status flux
+      ● flux.service - Flux message broker
+       Loaded: loaded (/lib/systemd/system/flux.service; enabled; vendor preset: enabled)
+       Active: active (running) since Tue 2024-04-23 07:36:44 PDT; 37s ago
+      Process: 287736 ExecStartPre=/usr/bin/loginctl enable-linger flux (code=exited, status=0/SUCCESS)
+      Process: 287737 ExecStartPre=bash -c systemctl start user@$(id -u flux).service (code=exited, status=0/SUCCESS)
+     Main PID: 287739 (flux-broker-0)
+       Status: "Running as leader of 8 node Flux instance"
+        Tasks: 22 (limit: 8755)
+       Memory: 26.6M
+          CPU: 3.506s
+       CGroup: /system.slice/flux.service
+               └─287739 broker --config-path=/etc/flux/system/conf.d -Scron.directory=/etc/flux/system/cron.d -Srundir=/run/flux -Sstatedir=/var/lib/flux -Slocal-uri=local:///run/flux/local -Slog-stderr-level=6 -Slog-stderr-mode=local -Sbroker.rc2_none -Sbroker.quorum=1 -Sbroker.quorum-timeout=none -Sbroker.exit-norestart=42 -Sbroker.sd-notify=1 -Scontent.dump=auto -Scontent.restore=auto
+
+  Apr 23 07:36:46 test0 flux[287739]: sched-fluxion-resource.info[0]: version 0.33.1-40-g24255b38
+  Apr 23 07:36:46 test0 flux[287739]: sched-fluxion-qmanager.info[0]: version 0.33.1-40-g24255b38
+  Apr 23 07:36:46 test0 flux[287739]: broker.info[0]: rc1.0: running /etc/flux/rc1.d/02-cron
+  Apr 23 07:36:47 test0 flux[287739]: broker.info[0]: rc1.0: /etc/flux/rc1 Exited (rc=0) 2.6s
+  Apr 23 07:36:47 test0 flux[287739]: broker.info[0]: rc1-success: init->quorum 2.65475s
+  Apr 23 07:36:47 test0 flux[287739]: broker.info[0]: online: test0 (ranks 0)
+  Apr 23 07:36:47 test0 flux[287739]: broker.info[0]: quorum-full: quorum->run 0.102056s
+
+Do other nodes join?
+====================
+
+Bring up a follower node that is configured with the leader as its parent
+in the tree based overlay network:
+
+.. code-block:: console
+
+  $ ssh test1
+  $ sudo systemctl start flux
+  $ flux uptime
+   07:47:58 run 4.3m,  owner flux,  depth 0,  size 8,  6 offline
+  $ flux overlay status
+  0 test0: partial
+  ├─ 1 test1: partial
+  │  ├─ 3 test3: offline
+  │  ├─ 4 test4: offline
+  │  └─ 5 test5: offline
+  └─ 2 test2: offline
+     ├─ 6 test6: offline
+     └─ 7 test7: offline
+  $ flux resource status
+       STATE UP NNODES NODELIST
+     avail  ✔      2 test[0-1]
+    avail*  ✗      6 test[2-7]
+
+If all goes well, bring up the remaining nodes:
+
+.. code-block:: console
+
+  $ sudo pdsh -a systemctl start flux
+  $ flux overlay status
+  0 test0: full
+  ├─ 1 test1: full
+  │  ├─ 3 test3: full
+  │  ├─ 4 test4: full
+  │  └─ 5 test5: full
+  └─ 2 test2: full
+     ├─ 6 test6: full
+     └─ 7 test7: full
+  $ flux resource status
+     STATE UP NNODES NODELIST
+     avail  ✔      8 test[0-7]
+
+Are my queues started?
+======================
+
+If named queues are configured, they will be initially stopped, meaning
+jobs can be submitted but won't run.  Enable all queues with
+
+.. code-block:: console
+
+  $ sudo flux queue start --all
+  debug: Scheduling is started
+  batch: Scheduling is started
+
+Can I run a job as a regular user?
+==================================
+
+Flux should be able to run jobs as an unprivileged user:
+
+.. code-block:: console
+
+  $ id
+  uid=1000(pi) gid=1000(pi) groups=1000(pi),27(sudo),114(netdev)
+  $ flux run -N8 id
+  uid=1000(pi) gid=1000(pi) groups=1000(pi),27(sudo),117(netdev)
+  uid=1000(pi) gid=1000(pi) groups=1000(pi),27(sudo),117(netdev)
+  uid=1000(pi) gid=1000(pi) groups=1000(pi),27(sudo),117(netdev)
+  uid=1000(pi) gid=1000(pi) groups=1000(pi),27(sudo),117(netdev)
+  uid=1000(pi) gid=1000(pi) groups=1000(pi),27(sudo),117(netdev)
+  uid=1000(pi) gid=1000(pi) groups=1000(pi),27(sudo),117(netdev)
+  uid=1000(pi) gid=1000(pi) groups=1000(pi),27(sudo),117(netdev)
+  uid=1000(pi) gid=1000(pi) groups=1000(pi),27(sudo),117(netdev)
+
+*************************
+Day to day administration
+*************************
+
+Starting Flux
+=============
+
+Systemd may be configured to start Flux automatically at boot time,
+as long as the network that carries its overlay network will be
+available at that time.  Alternatively, Flux may be started manually, e.g.
+
+.. code-block:: console
+
+ $ sudo pdsh -w fluke[3,108,6-103] sudo systemctl start flux
+
+Flux brokers may be started in any order, but they won't come online
+until their parent in the tree based overlay network is available.
+
+If Flux was not shut down properly, for example if the rank 0 broker
+crashed or was killed, then Flux starts in a safe mode with job submission
+and scheduling disabled.  :man1:`flux-uptime` shows the general state
+of Flux, and :man1:`flux-startlog` prints a record of Flux starts and
+stops, including any crashes.
+
+Stopping Flux
+=============
+
+The full Flux system instance may be temporarily stopped by running
+the following on the rank 0 node:
+
+.. code-block:: console
+
+ $ sudo flux shutdown
+
+This kills any running jobs, but preserves job history and the queue of
+jobs that have been submitted but have not yet allocated resources.
+This state is held in the ``content.sqlite`` that was configured above.
+See also :man1:`flux-shutdown`.
+
+.. note::
+    ``flux-shutdown --gc`` should be used from time to time to perform offline
+    KVS garbage collection.  This, in conjunction with configuring inactive
+    job purging, keeps the size of the ``content.sqlite`` database in check
+    and improves Flux startup time.
+
+The brokers on other nodes will automatically shut down in response,
+then respawn, awaiting the return of the rank 0 broker.
+
+To shut down a single node running Flux, simply run
+
+.. code-block:: console
+
+ $ sudo systemctl stop flux
+
+on that node.
+
+Configuration update
+====================
+
+After changing flux broker or module specific configuration in the TOML
+files under ``/etc/flux``, the configuration may be reloaded with
+
+.. code-block:: console
+
+ $ sudo systemctl reload flux
+
+on each rank where the configuration needs to be updated. The broker will
+reread all configuration files and notify modules that configuration has
+been updated.
+
+Configuration which applies to the ``flux-imp`` or job shell will be reread
+at the time of the next job execution, since these components are executed
+at job launch.
+
+.. warning::
+    Many configuration changes have no effect until the Flux broker restarts.
+    This should be assumed unless otherwise noted.  See :man5:`flux-config`
+    for more information.
+
+Viewing resource status
+=======================
+
+Flux offers two different utilities to query the current resource state.
+
+``flux resource status`` is an administrative command which lists ranks
+which are available, online, offline, excluded, or drained along with
+their corresponding node names. By default, sets which have 0 members
+are not displayed, e.g.
+
+.. code-block:: console
+
+ $ flux resource status
+      STATE UP NNODES NODELIST
+      avail  ✔     78 fluke[6-16,19-23,25-60,62-63,68,71-73,77-78,80,82-86,88,90-91,93,95-101,103]
+     avail*  ✗      6 fluke[17,24,61,79,92,102]
+    exclude  ✔      3 fluke[1,3,108]
+    drained  ✔     13 fluke[18,64-65,67,69-70,74-76,81,87,89,94]
+   drained*  ✗      1 fluke66
+
+To list a set of states explicitly, use the ``--states`` option:
+(Run ``--states=help`` to get a list of valid states)
+
+.. code-block:: console
+
+ $ flux resource status --states=drained,exclude
+     STATE UP NNODES NODELIST
+   exclude  ✔      3 fluke[1,3,108]
+   drained  ✔     13 fluke[18,64-65,67,69-70,74-76,81,87,89,94]
+  drained*  ✗      1 fluke66
+
+This option is useful to get a list of ranks or hostnames in a given
+state. For example, the following command fetches the hostlist
+for all resources configured in a Flux instance:
+
+.. code-block:: console
+
+ $ flux resource status -s all -no {nodelist}
+ fluke[1,3,6-103,108]
+
+In contrast to ``flux resource status``, the ``flux resource list``
+command lists the *scheduler*'s view of available resources. This
+command shows the free, allocated, and unavailable (down) resources,
+and includes nodes, cores, and gpus at this time:
+
+.. code-block:: console
+
+ $ flux resource list
+     STATE QUEUE      PROPERTIES NNODES   NCORES NODELIST
+      free batch                     71      284 fluke[6-16,19-23,25-60,62-63,68,71-73,77-78,80,82-86,88,90-91,93,95]
+      free debug                      6       24 fluke[96-101]
+      free debug      testprop        1        4 fluke103
+ allocated                            0        0 
+      down batch                     19       76 fluke[17-18,24,61,64-67,69-70,74-76,79,81,87,89,92,94]
+      down debug      testprop        1        4 fluke102
+
+With ``--o rlist``, ``flux resource list`` will show a finer grained list
+of resources in each state, instead of a nodelist:
+
+.. code-block:: console
+
+ $ flux resource list -o rlist
+     STATE QUEUE    PROPERTIES NNODES   NCORES    NGPUS LIST
+      free batch                   71      284        0 rank[3-13,16-20,22-57,59-60,65,68-70,74-75,77,79-83,85,87-88,90,92]/core[0-3]
+      free debug                    6       24        0 rank[93-98]/core[0-3]
+      free debug    testprop        1        4        0 rank100/core[0-3]
+ allocated                          0        0        0
+      down batch                   19       76        0 rank[14-15,21,58,61-64,66-67,71-73,76,78,84,86,89,91]/core[0-3]
+      down debug    testprop        1        4        0 rank99/core[0-3]
+
+
+Draining resources
+==================
+
+Resources may be temporarily removed from scheduling via the
+``flux resource drain`` command. Currently, resources may only be drained
+at the granularity of a node, represented by its hostname or broker rank,
+for example:
+
+.. code-block:: console
+
+ $ sudo flux resource drain fluke7 node is fubar
+ $ sudo flux resource drain
+ TIMESTAMP            STATE    RANK     REASON                         NODELIST
+ 2020-12-16T09:00:25  draining 2        node is fubar                  fluke7
+
+Any work running on the "draining" node is allowed to complete normally.
+Once there is nothing running on the node its state changes to "drained":
+
+.. code-block:: console
+
+ $ sudo flux resource drain
+ TIMESTAMP            STATE    RANK     REASON                         NODELIST
+ 2020-12-16T09:00:25  drained  2        node is fubar                  fluke7
+
+To return drained resources use ``flux resource undrain``:
+
+.. code-block:: console
+
+ $ sudo flux resource undrain fluke7
+ $ sudo flux resource drain
+ TIMESTAMP            STATE    RANK     REASON                         NODELIST
+
+
+Managing the Flux queue
+=======================
+
+The queue of jobs is managed by the flux job-manager, which in turn
+makes allocation requests for jobs in priority order to the scheduler.
+This queue can be managed using the ``flux-queue`` command.
+
+.. code-block:: console
+
+ Usage: flux-queue [OPTIONS] COMMAND ARGS
+   -h, --help             Display this message.
+
+ Common commands from flux-queue:
+    enable          Enable job submission
+    disable         Disable job submission
+    start           Start scheduling
+    stop            Stop scheduling
+    status          Get queue status
+    drain           Wait for queue to become empty.
+    idle            Wait for queue to become idle.
+
+The queue may be listed with the :man1:`flux-jobs` command.
+
+Disabling job submission
+------------------------
+
+By default, the queue is *enabled*, meaning that jobs can be submitted
+into the system. To disable job submission, e..g to prepare the system
+for a shutdown, use ``flux queue disable``. To restore queue access
+use ``flux queue enable``.
+
+Stopping resource allocation
+----------------------------
+
+The queue may also be stopped with ``flux queue stop``, which disables
+further allocation requests from the job-manager to the scheduler. This
+allows jobs to be submitted, but stops new jobs from being scheduled.
+To restore scheduling use ``flux queue start``.
+
+Flux queue idle and drain
+-------------------------
+
+The ``flux queue drain`` and ``flux queue idle`` commands can be used
+to wait for the queue to enter a given state. This may be useful when
+preparing the system for a downtime.
+
+The queue is considered *drained* when there are no more active jobs.
+That is, all jobs have completed and there are no pending jobs.
+``flux queue drain`` is most useful when the queue is *disabled* .
+
+The queue is "idle" when there are no jobs in the RUN or CLEANUP state.
+In the *idle* state, jobs may still be pending. ``flux queue idle``
+is most useful when the queue is *stopped*.
+
+To query the current status of the queue use the ``flux queue status``
+command:
+
+.. code-block:: console
+
+ $ flux queue status -v
+ batch: Job submission is enabled
+ batch: Scheduling is started
+ debug: Job submission is enabled
+ debug: Scheduling is started
+ 0 alloc requests queued
+ 0 alloc requests pending to scheduler
+ 0 free requests pending to scheduler
+ 0 running jobs
+
+Managing Flux jobs
+==================
+
+Expediting/Holding jobs
+-----------------------
+
+To expedite or hold a job, set its *urgency* to the special values
+EXPEDITE or HOLD.
+
+.. code-block:: console
+
+ $ flux job urgency ƒAiVi2Sj EXPEDITE
+
+.. code-block:: console
+
+ $ flux job urgency ƒAiVi2Sj HOLD
+
+Canceling jobs
+--------------
+
+An active job may be canceled via the ``flux cancel`` command. An
+instance owner may cancel any job, while a guest may only cancel their
+own jobs.
+
+All active jobs may be canceled with ``flux cancel --user=all``.
+
+.. code-block:: console
+
+ $ flux cancel --user=all --dry-run
+ flux-cancel: Would cancel 3 jobs
+ $ flux cancel --user=all
+ flux-cancel: Canceled 3 jobs (0 errors)
+
+The set of jobs matched by the ``cancel`` command may also be restricted
+via the ``-s, --states=STATES`` and ``-u, --user=USER`` options.
+
+Software update
+===============
+
+Flux will eventually support rolling software upgrades, but prior to
+major release 1, Flux software release versions should not be assumed
+to inter-operate.  Furthermore, at this early stage, Flux software
+components (e.g. ``flux-core``, ``flux-sched``, ``flux-security``,
+and ``flux-accounting``)  should only only be installed in recommended
+combinations.
+
+.. note::
+    Mismatched broker versions are detected as brokers attempt to join
+    the instance.  The version is currently required to match exactly.
+
+.. warning::
+    Ensure that flux is completely stopped before beginning a software
+    update.  If this is not observed, Flux may fail to shut down cleanly.
+
+***************
+Troubleshooting
+***************
+
+Overlay network
+===============
+
+The tree-based overlay network interconnects brokers of the system instance.
+The current status of the overlay subtree at any rank can be shown with:
+
+.. code-block:: console
+
+ $ flux overlay status -r RANK
+
+The possible status values are:
+
+**Full**
+  Node is online and no children are in partial, offline, degraded, or lost
+  state.
+
+**Partial**
+  Node is online, and some children are in partial or offline state; no
+  children are in degraded or lost state.
+
+**Degraded**
+  Node is online, and some children are in degraded or lost state.
+
+**Lost**
+  Node has gone missing, from the parent perspective.
+
+**Offline**
+  Node has not yet joined the instance, or has been cleanly shut down.
+
+Note that the RANK argument is where the request will be sent, not necessarily
+the rank whose status is of interest.  Parents track the status of their
+children, so a good approach when something is wrong to start with rank 0
+(the default).  The following options can be used to ask rank 0 for a detailed
+listing:
+
+.. code-block:: console
+
+ $ flux overlay status
+ 0 fluke62: degraded
+ ├─ 1 fluke63: full
+ │  ├─ 3 fluke65: full
+ │  │  ├─ 7 fluke70: full
+ │  │  └─ 8 fluke71: full
+ │  └─ 4 fluke67: full
+ │     ├─ 9 fluke72: full
+ │     └─ 10 fluke73: full
+ └─ 2 fluke64: degraded
+    ├─ 5 fluke68: full
+    │  ├─ 11 fluke74: full
+    │  └─ 12 fluke75: full
+    └─ 6 fluke69: degraded
+       ├─ 13 fluke76: full
+       └─ 14 fluke77: lost
+
+To determine if a broker is reachable from the current rank, use:
+
+.. code-block:: console
+
+ $ flux ping RANK
+
+A broker that is not responding but is not shown as lost or offline
+by ``flux overlay status`` may be forcibly detached from the overlay
+network with:
+
+.. code-block:: console
+
+ $ flux overlay disconnect RANK
+
+However, before doing that, it may be useful to see if a broker acting
+as a router to that node is actually the problem.  The overlay parent
+of RANK may be listed with
+
+.. code-block:: console
+
+ $ flux overlay parentof RANK
+
+Using ``flux ping`` and ``flux overlay parentof`` iteratively, one should
+be able to isolate the problem rank.
+
+See also :man1:`flux-overlay`, :man1:`flux-ping`.
+
+Systemd journal
+===============
+
+Flux brokers log information to standard error, which is normally captured
+by the systemd journal.  It may be useful to look at this log when diagnosing
+a problem on a particular node:
+
+.. code-block:: console
+
+ $ journalctl -u flux
+ Sep 14 09:53:12 sun1 systemd[1]: Starting Flux message broker...
+ Sep 14 09:53:12 sun1 systemd[1]: Started Flux message broker.
+ Sep 14 09:53:12 sun1 flux[23182]: broker.info[2]: start: none->join 0.0162958s
+ Sep 14 09:53:54 sun1 flux[23182]: broker.info[2]: parent-ready: join->init 41.8603s
+ Sep 14 09:53:54 sun1 flux[23182]: broker.info[2]: rc1.0: running /etc/flux/rc1.d/01-enclosing-instance
+ Sep 14 09:53:54 sun1 flux[23182]: broker.info[2]: rc1.0: /bin/sh -c /etc/flux/rc1 Exited (rc=0) 0.4s
+ Sep 14 09:53:54 sun1 flux[23182]: broker.info[2]: rc1-success: init->quorum 0.414207s
+ Sep 14 09:53:54 sun1 flux[23182]: broker.info[2]: quorum-full: quorum->run 9.3847e-05s
+
+Broker log buffer
+=================
+
+The rank 0 broker accumulates log information for the full instance in a
+circular buffer.  For some problems, it may be useful to view this log:
+
+.. code-block:: console
+
+ $ sudo flux dmesg -H |tail
+
+ [May02 14:51] sched-fluxion-qmanager[0]: feasibility_request_cb: feasibility succeeded
+ [  +0.039371] sched-fluxion-qmanager[0]: alloc success (queue=debug id=184120855100391424)
+ [  +0.816587] sched-fluxion-qmanager[0]: feasibility_request_cb: feasibility succeeded
+ [  +0.857458] sched-fluxion-qmanager[0]: alloc success (queue=debug id=184120868807376896)
+ [  +1.364430] sched-fluxion-qmanager[0]: feasibility_request_cb: feasibility succeeded
+ [  +6.361275] job-ingest[0]: job-frobnicator[0]: inactivity timeout
+ [  +6.367837] job-ingest[0]: job-validator[0]: inactivity timeout
+ [ +24.778929] job-exec[0]: exec aborted: id=184120855100391424
+ [ +24.779019] job-exec[0]: exec_kill: 184120855100391424: signal 15
+ [ +24.779557] job-exec[0]: exec aborted: id=184120868807376896
+ [ +24.779632] job-exec[0]: exec_kill: 184120868807376896: signal 15
+ [ +24.779910] sched-fluxion-qmanager[0]: alloc canceled (id=184120878001291264 queue=debug)
+ [ +25.155578] job-list[0]: purged 1 inactive jobs
+ [ +25.162650] job-manager[0]: purged 1 inactive jobs
+ [ +25.512050] sched-fluxion-qmanager[0]: free succeeded (queue=debug id=184120855100391424)
+ [ +25.647542] sched-fluxion-qmanager[0]: free succeeded (queue=debug id=184120868807376896)
+ [ +27.155103] job-list[0]: purged 2 inactive jobs
+ [ +27.159820] job-manager[0]: purged 2 inactive jobs
diff --git a/doc/guide/broker.rst b/doc/guide/broker.rst
new file mode 100644
index 000000000000..6e7d711522b1
--- /dev/null
+++ b/doc/guide/broker.rst
@@ -0,0 +1,506 @@
+.. _broker:
+
+######
+Broker
+######
+
+The :man1:`flux-broker` provides an overlay network and a framework for
+implementing distributed, message-based services.
+
+.. note::
+  This document is incomplete.  Please open an issue on
+  `github <https://github.com/flux-framework/flux-core>`_
+  if you have a need for broker information that is missing and we will
+  do our best to answer questions and fill gaps in the documentation.
+
+
+*************************
+Broker Bootstrap Sequence
+*************************
+
+Each broker executes a bootstrap sequence to
+
+1. get the size of the Flux instance
+2. get its rank within the Flux instance
+3. get its level within the hierarchy of Flux instances
+4. get the mapping of broker ranks to nodes
+5. compute the broker ranks of its peers (TBON parent and children)
+6. get URI(s) of peers
+7. get public key(s) of peers to initialize secure communication
+
+An instance bootstraps using one of two mechanisms:  PMI or Config File.
+
+PMI
+===
+
+When Flux is launched by Flux, by another resource manager, or as a
+standalone test instance, PMI is used for bootstrap.
+
+The broker PMI client uses a simple subset of PMI capabilities, abstracted for
+different server implementations in the plugin-based UPMI subsystem defined
+in `upmi.h <https://github.com/flux-framework/flux-core/blob/master/src/common/libpmi/upmi.h_`.  The broker PMI bootstrap sequence roughly follows the
+steps outlined above.
+
+Steps 1-4 involve accessing parameters directly provided by the PMI server.
+
+In step 5, the fixed tree topology is used to compute the ranks of the TBON
+parent and TBON children, if any.
+
+If a broker has TBON children, it binds to a 0MQ URI that the children will
+connect to.  If step 4 indicates that all brokers mapped to a single node,
+the socket is bound to a local IPC path in the broker rundir.  If brokers are
+mapped to multiple nodes, the socket is bound to a TCP address on the interface
+hosting the default route and a randomly assigned port.  These URIs cannot
+be predicted by peers, so they must be exchanged via PMI.
+
+In addition, each broker generates a unique public, private CURVE keypair.
+The public keys must be shared with peers to enable secure communication.
+
+In step 6-7, each broker rank stores a "business card" containing its 0MQ URI
+and public key to the PMI KVS under its rank.  A PMI barrier is executed.
+Finally, the business cards for any peers are loaded from the PMI KVS.
+The bootstrap process is complete and overlay initialization may commence.
+
+For debugging, set :envvar:`FLUX_PMI_DEBUG=1` in the broker's environment
+to get a trace of the broker's client PMI calls on stderr.  Some PMI options
+may be tweaked by setting the :option:`broker.boot-method` attribute as
+described in :man7:`flux-broker-attributes`.  When there are broker bootstrap
+issues, sometimes it can be helpful to try to boot :man1:`flux-pmi`, a
+standalone PMI client.
+
+Flux booting Flux as a job
+--------------------------
+
+When Flux launches Flux in the job environment, Flux is providing both the
+PMI server via the Flux shell *pmi* plugin, and the PMI client in the broker.
+
+The shell *pmi* plugin offers the PMI-1 wire protocol to the broker, by
+passing an open file descriptor to each broker via the :envvar:`PMI_FD`
+environment variable.  The broker client uses the PMI-1 wire protocol client
+to execute the bootstrap sequence.
+
+Debugging: set the shell option :option:`verbose=2` for a server side trace
+on stderr from the shell *pmi* plugin.  The shell plugin options are further
+described in :man1:`flux-shell`.
+
+booting Flux as a standalone test instance
+------------------------------------------
+
+An instance of size N may be launched on a single node using
+
+.. code-block:: console
+
+  flux-start --test-size=N
+
+In this case, the PMI server is embedded in the start command, and the PMI-1
+wire protocol is used as described above.
+
+Debugging: use the :man1:`flux start` :option:`--verbose=2` option for a
+server side trace on stderr.
+
+booting Flux as a job in a foreign resource manager
+---------------------------------------------------
+
+The UPMI client used by the broker attempts to adapt to different situations
+if the PMI-1 wire protocol is not available.  It tries the following in order
+(unless configured otherwise):
+
+1. simple PMI wire protocol
+2. find a ``libpmi2.so`` library
+3. find a ``libpmi.so`` library
+4. assume singleton (rank = 0, size = 1)
+
+Config File
+===========
+
+When Flux is launched by systemd, the brokers go through a similar bootstrap
+process as under PMI, except that information is read from a set of identical
+TOML configuration files replicated across the cluster.  The TOML configuration
+contains a host array.
+
+In step 1-2, the broker scans the host array for an entry with a matching
+hostname.  The array index of the matching entry is the broker's rank,
+and also contains its URI.  The array size is the instance size.
+
+Steps 3-4 are satisfied by assuming that in this mode, there is one broker
+per node, and the instance is at the top (level 0) of the instance hierarchy.
+
+In step 5, as above, the tree topology is used to compute the ranks of the
+TBON parent and TBON children, if any.
+
+Instead of generating a unique CURVE keypair per broker, an instance
+bootstrapped in this way shares a single CURVE keypair replicated across
+the cluster.
+
+So steps 6-7 are satisfied by simply accessing the CURVE key certificate
+on disk, and looking up the peer rank indices in the hosts array.
+The bootstrap process is complete and overlay initialization may commence.
+
+
+********************
+Broker State Machine
+********************
+
+After bootstrap, each broker comprising a Flux instance begins executing an
+identical state machine.  Although there is synchronization between brokers
+in some states, there is no distributed agreement on a global state for the
+instance.
+
+*Events* drive the state machine.
+
+Entry to a new state triggers an *action*.
+
+Actions may differ across broker ranks.  For example, entering CLEANUP state
+on rank 0 launches a process and an event is generated upon process
+termination, while on rank > 0, entering CLEANUP does not launch a process,
+and immediately generates an event.
+
+.. image:: images/states.png
+  :scale: 100 %
+  :alt: broker state machine
+
+.. list-table::
+  :header-rows: 1
+
+  * - abbrev
+    - state
+    - action when transitioning into state
+
+  * - J
+    - JOIN
+    - wait for parent to enter QUORUM state
+
+  * - 1
+    - INIT
+    - run rc1 script
+
+  * - B
+    - QUORUM
+    - wait for quorum of brokers to reach this point
+
+  * - 2
+    - RUN
+    - run initial program (rank 0)
+
+  * - C
+    - CLEANUP
+    - run cleanup (rank 0)
+
+  * - S
+    - SHUTDOWN
+    - wait for children to finalize and exit
+
+  * - 3
+    - FINALIZE
+    - run rc3 script
+
+  * - G
+    - GOODBYE
+    - wait for flux-shutdown, if any
+
+  * - E
+    - EXIT
+    - exit broker
+
+Normal State Transitions
+========================
+
+It may be helpful to walk through the state transitions that occur when
+a Flux instance runs to completion without encountering exceptional conditions.
+
+.. image:: images/states_norm.png
+  :scale: 100 %
+  :alt: broker state machine
+
+green = common path; blue = rank 0 deviation from common path; red = leaf
+node deviation from common path
+
+startup
+-------
+
+The broker ranks > 0 wait for the parent to enter QUORUM state (*parent-ready*)
+then enters INIT state.  Rank 0 immediately enters INIT (*parent-none*).
+Upon entering INIT, the rc1 script is executed, then on completion, QUORUM
+state is entered (*rc1-success*).  Because each TBON tree level waits for the
+upstream level to enter QUORUM state before entering INIT state, rc1 executes
+in upstream-to-downstream order.  This ensures upstream service leaders are
+loaded before downstream followers.
+
+Once a configured number of brokers have reached QUORUM state (default all),
+RUN state is entered (*quorum-full*).  Rank 0 then starts the initial program.
+
+All ranks remain in RUN state until the initial program completes.
+
+shutdown
+--------
+
+When the initial program completes, rank 0 transitions to CLEANUP state
+(*rc2-success*) and runs any cleanup script(s).  Cleanups execute while the
+other broker ranks remain in RUN state.  Upon completion of cleanups, rank 0
+enters SHUTDOWN state (*cleanup-success*).
+
+The broker ranks > 0 monitor parent state transitions.  The parent
+transitioning to SHUTDOWN causes a transition from RUN to CLEANUP
+(*shutdown*).  They immediately transition through CLEANUP (*cleanup-none*)
+to SHUTDOWN state.
+
+All brokers with children remain in SHUTDOWN until their children disconnect
+(*children-complete*).  If they have no children (leaf node), they transition
+out of SHUTDOWN immediately (*children-none*). The next state is FINALIZE,
+where the rc3 script is executed.  Upon completion of rc3 (*rc3-success*),
+brokers transition to EXIT and disconnect from the parent.
+
+Because each TBON tree level waits for the downstream level to disconnect
+before entering FINALIZE state, rc3 executes in downstream-to-upstream order.
+This ensures downstream service followers are unloaded before upstream leaders.
+
+The rank 0 broker is the last to exit.
+
+variation: no rc2 script (initial program)
+------------------------------------------
+
+A system instance does not define an initial program.  Brokers transition to
+RUN state as above, and remain there until the *shutdown* event is
+posted.  That may occur if:
+- the broker receives a signal
+- the broker's TBON parent enters SHUTDOWN state
+- (rank 0 only) :man1:`flux-shutdown` requests instance shutdown
+
+variation: no rc1, rc3, or cleanup scripts
+------------------------------------------
+
+In test sometimes we eliminate the rc1, cleanup, and/or rc3 scripts to simplify
+or speed up a test environment.  In these cases, entry into INIT, CLEANUP,
+and FINALIZE states generates a *rc1-none*, *cleanup-none*, or *rc3-none*
+event, which causes an immediate transition to the next state.
+
+Events
+======
+
+.. list-table::
+  :header-rows: 1
+
+  * - event
+    - description
+
+  * - parent-ready
+    - parent has entered BARRIER state
+
+  * - parent-none
+    - this broker has no parent
+
+  * - parent-fail
+    - parent has ended communication with this broker
+
+  * - parent-timeout
+    - parent has not responded within timeout period
+
+  * - rc1-none
+    - rc1 script is defined on this broker
+
+  * - rc1-success
+    - rc1 script completed successfully
+
+  * - rc1-fail
+    - rc1 script completed with errors
+
+  * - quorum-full
+    - configured quorum of brokers reached
+
+  * - quorum-timeout
+    - configured quorum not reached within timeout period
+
+  * - rc2-none
+    - no rc2 script (initial program) is defined on this broker
+
+  * - rc2-success
+    - rc2 script completed successfully
+
+  * - rc2-fail
+    - rc2 script completed with errors
+
+  * - shutdown
+    - broker received an external cue to begin shutting down
+
+  * - signal-abort
+    - broker received terminating signal
+
+  * - cleanup-none
+    - no cleanup script is defined on this broker
+
+  * - cleanup-success
+    - cleanup script completed successfully
+
+  * - cleanup-fail
+    - cleanup script completed with errors
+
+  * - children-complete
+    - all children have disconnected from this broker
+
+  * - children-none
+    - this broker has no children
+
+  * - children-timeout
+    - children did not disconnected within timeout period
+
+  * - rc3-none
+    - no rc3 script is defined on this broker
+
+  * - rc3-success
+    - rc3 script completed successfully
+
+  * - rc3-fail
+    - rc3 script completed with errors
+
+  * - goodbye
+    - any flux-shutdown commands have completed
+
+**************************
+System Instance Resiliency
+**************************
+
+The Flux system instance has to deal with the usual challenges faced by cluster
+system software, such as node crashes and network outages.  Although Flux's
+design attempts to meet these challenges with minimal human intervention and
+lost work, there are caveats that need to be understood by Flux developers.
+This page describes Flux's current design for resiliency.
+
+NOTE: some of this is aspirational at the time of this writing, for our L2
+resiliency planning goal to be demonstrated in early 2022.
+
+Disordered bring-up
+===================
+
+The broker state machine ensures that a starting broker pauses until its TBON
+parent completes the rc1 script before starting its own rc1 script, so that
+upstream services are online before downstream ones start.  As a result, it
+is feasible to configure Flux to start automatically, then power on the entire
+cluster at once and let Flux sort itself out.
+
+If some nodes take longer to start up, or don't start at all, then those nodes
+and their TBON children, if any, will remain offline until they do start.
+The TBON has a fixed topology determined by configuration, and brokers do not
+adapt to route around down nodes.  In addition, Flux must be completely stopped
+to alter the topology configuration - it cannot be changed on the fly.
+
+See the `Flux Administrator's Guide <https://flux-framework.readthedocs.io/projects/flux-core/en/latest/guide/admin.html>`_
+for a discussion on draining nodes and excluding nodes from scheduling via
+configuration.  Scheduling is somewhat orthogonal to this topic.
+
+Subtree shut down
+=================
+
+Flux is stopped administratively with `systemctl stop flux`.  This is may
+be run on any broker, and will affect the broker's TBON subtree.  If run on
+the rank 0 broker, the entire Flux instance is shut down.
+
+Upon receiving SIGTERM from systemd, the broker informs its TBON children that
+it is shutting down, and waits for them to complete rc3 in leaves-to-root
+order, thus ensuring that the instance captures any state held on those
+brokers, such as KVS content.
+
+A broker that is cleanly shut down withdraws itself as a peer from its TBON
+parent.  Future RPCs to the down broker automatically receive an EHOSTUNREACH
+error response.
+
+Node crash
+==========
+
+If a node crashes without shutting down its flux broker, state held by that
+broker and its TBON subtree is lost if it was not saved to its TBON parent.
+
+The TBON parent of the lost node detects that its child has stopped sending
+messages.  The parent marks the child lost and future RPCs directed to
+(or through) the crashed node receive an EHOSTUNREACH error response.  In
+addition, RPCs are tracked at the broker overlay level, and any requests that
+were directed to (or through) the lost node that are unterminated, as
+defined by :doc:`RFC 6 <rfc:spec_6>` receive an EHOSTUNREACH error response.
+
+The TBON children of the lost node similarly detect the lack of parent
+messages.  The child marks the parent offline and future RPCs, as well as
+existing unterminated RPCs to that node receive an EHOSTUNREACH error response.
+These nodes then proceed as though they were shut down, however since they are
+cut off from upstream, any RPCs executed in rc3 to save state will fail
+immediately.  Effectively a *subtree panic* results and data may be lost.
+
+Node returning to Service
+=========================
+
+When a lost node comes back up, or when an administratively shut down node
+is restarted with
+
+.. code-block:: console
+
+  systemctl start flux
+
+the freshly started broker attempts to join the instance via its TBON parent,
+just as if it were joining for the first time, and carrying no state from
+the previous incarnation.
+
+The broker peer is identified for response routing purposes by its UUID, which
+changes upon restart.  In-flight responses directed to (or through) the
+old UUID are dropped.  This is desirable behavior because matchtags from the
+previous broker incarnation(s) might become confused with matchtags from the
+current one, and the old responses are answering requests that the current
+broker didn't send.
+
+:linux:man1:`systemd` is configured to aggressively restart brokers that stop
+on their own, so TBON children of the returning broker should also be
+attempting to join the instance and may do so once the returning broker has
+completed the rc1 script.
+
+Network outage
+==============
+
+Network outages that persist long enough are promoted to hard failures.
+
+Case 1:  A TBON parent marks its child lost due to a temporary network
+interruption, and the child has not yet marked the parent lost when
+communication is restored.  In this case, the parent sees messages from the
+lost UUID, and sends a kiss of death message to the child, causing a subtree
+panic at the child.  The subtree restarts and rejoins the instance.
+
+Case 2:  The TBON child marks its parent lost due to a temporary network
+interruption, and the parent has not yet marked the child lost when
+communication is restored.  Assuming the child subtree has restarted,
+the child introduces itself to the parent with a new UUID.  Before allowing
+the child to join, it marks the old UUID lost and fails unterminated RPCs
+as described above.  It then accepts the child's introduction and allows
+the subtree to join.
+
+Diagnostic: subtree health check
+================================
+
+Each broker maintains a subtree health state that depends on the health
+state reported by its TBON children.  The states are as follows:
+
+.. list-table::
+  :header-rows: 1
+
+  * - name
+    - description
+
+  * - full
+    - online and no children partial/offline/degraded/lost
+
+  * - partial
+    - online, some children partial/offline; no children degraded/lost
+
+  * - degraded
+    - online, some children degraded/lost
+
+  * - lost
+    - gone missing (according to parent)
+
+  * - offline
+    - not yet seen, or cleanly shut down (according to parent)
+
+A user may quickly assess the overall health of the overlay network by
+requesting the subtree health at rank 0.  If the state is reported as
+*partial* or *degraded*, then the TBON may be probed further for details
+using the following algorithm:
+
+1. Request state of TBON children from target rank
+2. List TBON children in *lost* or *offline* state
+3. For each child in *partial* or *degraded* state, apply this algorithm
+   on child rank
diff --git a/doc/guide/build.rst b/doc/guide/build.rst
new file mode 100644
index 000000000000..558b42decc1e
--- /dev/null
+++ b/doc/guide/build.rst
@@ -0,0 +1,130 @@
+Building Releases
+=================
+
+Version Numbering
+-----------------
+
+Flux-core releases are numbered with
+`Semantic Versioning <https://semver.org/>`_, where MAJOR.MINOR.PATCH release
+numbers communicate that:
+
+- major releases may break API
+
+- minor releases add functionality in a backward compatible manner
+
+- patch releases make backward compatible bug fixes
+
+At this time, the project is at major version zero, indicating interfaces
+are not yet stable.  That said, flux-core developers try to minimize
+disruption to early adopters and announce any necessary breaking changes
+in the release notes.
+
+Obtaining Releases
+------------------
+
+Tarballs and release notes are available on the
+`Github releases page <https://github.com/flux-framework/flux-core/releases>`_.
+
+Releases for all Flux framework projects are also announced on the
+`Flux Framework releases page <https://flux-framework.org/releases/>`_.
+
+Installing Dependencies
+-----------------------
+
+Several external software packages are prerequisites for building Flux.
+Scripts that install these packages for debian and redhat based distros are
+located in the flux-core source tree ``scripts`` sub-directory.
+
+The following packages are optional and may be omitted if you do not require
+the associated functionality:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Package
+     - Functionality
+
+   * - `flux-security <https://github.com/flux-framework/flux-security>`_
+     - Launching work as multiple users.
+
+       For example when Flux is to be the native resource manager on a cluster.
+
+   * - Sphinx
+     - Building man pages and documentation
+
+   * - MPI
+     - Test only: sanity tests that Flux can launch MPI programs
+
+   * - valgrind
+     - Test only: checks for memory errors
+
+Configuring and Building a Release
+----------------------------------
+
+flux-core uses GNU autotools internally, so it supports the usual
+`Use Cases for the GNU Build System <https://www.gnu.org/software/automake/manual/html_node/Use-Cases.html>`_.  A standard build follows this pattern:
+
+.. code-block:: console
+
+  $ tar xzf flux-core-X.Y.Z.tar.gz
+  $ cd flux-core-X.Y.Z
+  $ ./configure --with-flux-security
+  $ make
+  $ make check
+  $ make install
+
+Configure *should* abort if any required build dependencies are missing or
+insufficient.  Configure options options may be listed with ``./configure
+--help``.
+
+``make -j N`` may be used to speed up the build and check targets by
+increasing parallelism.
+
+All checks are expected to pass, although some timing related test defects
+may cause tests to sporadically fail on slow systems or when run with too much
+parallelism.  ``make recheck`` re-runs any failing checks.
+
+Packages
+--------
+
+RPM packages for TOSS 4 (RHEL 8 based) are produced by the TOSS build system
+and can be made available externally on request.  When requested, these are
+manually added to the release assets on github.
+
+deb packages for Debian or Ubuntu can be built from a release tarball with
+``make deb``, producing debs in the ``debbuild`` sub-directory.  This target
+is used by some Flux team members to build packages for test clusters running
+the `Raspberry Pi OS <https://www.raspberrypi.com/software/>`_ (Debian/GNU 11).
+
+Docker Images
+-------------
+
+Docker images for tagged releases as well as a current development snapshot
+are available in the `fluxrm/flux-core Dockerhub
+<https://hub.docker.com/r/fluxrm/flux-core/tags>`_.  For example, the following
+downloads a debian bookworm image containing flux-core 0.54.0 and starts a
+flux instance within it:
+
+.. code-block:: console
+
+  $ docker pull fluxrm/flux-core:bookworm-v0.54.0-amd64
+  $ docker run -ti fluxrm/flux-core:bookworm-v0.54.0-amd64
+  ƒ(s=1,d=0) fluxuser@080d84548cc4:~$
+
+Spack
+-----
+
+Flux-core and its dependencies can also be built using `spack
+<https://spack-tutorial.readthedocs.io/en/latest/>`_, for example:
+
+.. code-block:: console
+
+  $ git clone --depth=100 https://github.com/spack/spack.git
+  $ cd spack
+  $ . share/spack/setup-env.sh
+  $ spack install flux-core@0.54.0 %gcc@11.4.0
+  $ spack find flux-core
+  -- linux-ubuntu22.04-zen2 / gcc@11.4.0 --------------------------
+  flux-core@0.54.0
+  ==> 1 installed package
+  $ spack load flux-core
diff --git a/doc/guide/debug.rst b/doc/guide/debug.rst
new file mode 100644
index 000000000000..b2dad8e6fb18
--- /dev/null
+++ b/doc/guide/debug.rst
@@ -0,0 +1,62 @@
+.. _debug:
+
+###############
+Debugging Notes
+###############
+
+***********************
+source tree executables
+***********************
+
+`libtool <https://www.gnu.org/software/libtool/manual/libtool.html>`_ is
+in use, so :option:`libtool e` trickery is needed to launch a tool against
+an actual compiled executable.  Command front ends further complicate this.
+
+.. note::
+  :option:`libtool e` is shorthand for :option:`libtool --mode=execute`.
+
+Example: run a built-in sub-command under GDB
+
+.. code-block::
+
+  $ libtool e gdb --ex run --args src/cmd/flux version
+
+Example: run an external sub-command under GDB
+
+.. code-block::
+
+  $ src/cmd/flux /usr/bin/libtool e gdb --ex run --args src/cmd/flux-keygen
+
+Example: run the broker under GDB
+
+.. code-block::
+
+  $ src/cmd/flux start --wrap=libtool,e,gdb,--ex,run
+
+Example: run the broker under valgrind
+
+.. code-block::
+
+  $ src/cmd/flux start --wrap=libtool,e,valgrind
+
+***************
+message tracing
+***************
+
+Example: trace messages sent/received by a command
+
+.. code-block::
+
+  $ FLUX_HANDLE_TRACE=t flux kvs get foo
+
+Example: trace messages sent/received by two broker modules
+
+.. code-block::
+
+  $ flux module trace --full content kvs
+
+Example: trace messages sent/received by this broker on the overlay network
+
+.. code-block::
+
+  $ flux overlay trace --full
diff --git a/doc/guide/glossary.rst b/doc/guide/glossary.rst
new file mode 100644
index 000000000000..cd518047055e
--- /dev/null
+++ b/doc/guide/glossary.rst
@@ -0,0 +1,112 @@
+Glossary
+========
+
+Here we define Flux-specific and general HPC and workload management terms
+used in our documentation that may not be familiar to all readers.
+
+.. glossary::
+
+  enclosing instance
+    The Flux instance that a process naturally interacts with.  It is
+    the instance referred to by the :envvar:`FLUX_URI` environment variable,
+    or if that is not set, it is the :term:`system instance`.
+
+  expedited
+    A job is said to be expedited if its :term:`urgency` is set to the
+    maximum value of 31.  An expedited job's :term:`priority` is always set
+    to the maximum value.
+
+  FSD
+    A common string representation of time duration, defined by
+    :doc:`rfc:spec_23`.  Example: ``2.5h``.
+
+  guest
+    A Flux user that is not the :term:`instance owner`.  Guests are only
+    allowed to run in a Flux instance configured for multi-user support,
+    normally a :term:`system instance`.
+
+  held
+    A job is said to be held if its :term:`urgency` is set to zero.  This
+    prevents it from being considered for scheduling until the urgency is
+    raised.
+
+  hostlist
+    A compact string representation of a list of hostnames, defined by
+    :doc:`rfc:spec_29`.  Example: ``fluke[0-127,130]``.
+
+  idset
+    A compact string representation of a set of non-negative integers,
+    defined by :doc:`rfc:spec_22`.  Example: ``2,4,6,1-100``.
+
+  IMP
+    The Independent Minister of Privilege.  The simple setuid root component
+    of Flux, from the flux-security project, that allows an
+    :term:`instance owner` to perform a limited set of tasks on behalf of a
+    :term:`guest` user in a multi-user Flux instance.
+
+  initial program
+    A user-defined program, such as a batch script, launched on the first
+    node of a Flux instance.  Its purpose is to launch and monitor a
+    workload.  Once it is complete, the instance exits.
+
+  instance owner
+    The user that started the Flux instance.  The instance owner has control
+    over all aspects of the Flux instance's operation.
+
+  job
+    The smallest unit of work that can be allocated resources and run by Flux.
+    A job can be a Flux instance which in turn can run more jobs.
+
+  jobspec
+    The JSON or YAML object representing a Flux job request, defined by
+    :doc:`rfc:spec_14` (the general specification) and :doc:`rfc:spec_25`
+    (the current version).  It includes the abstract resource requirements
+    of the job and instructions for job execution.
+
+  priority
+    The order in which the scheduler considers jobs.  By default, priority
+    is derived from the :term:`urgency` and submit time, but a priority plugin
+    can be used to override this calculation.
+
+  R
+    The JSON object used by Flux to represent a concrete resource set.
+    See :doc:`rfc:spec_20`.
+
+  resource inventory
+    The concrete set of resources managed by a given Flux instance.
+
+  scheduler
+    The Flux component that fulfills resource allocation requests from the
+    :term:`resource inventory`.  Abstract resource requirements are extracted
+    from the user-provided :term:`jobspec`, and fulfilled with a resource set
+    expressed as :term:`R`. In addition to fitting concrete resources to
+    abstract requests, the scheduler must balance goals such as fairness
+    and resource utilization when it decides upon a schedule for fulfilling
+    competing requests.
+
+  slot
+    The abstract resource requirements of one task.
+
+  step
+    In other workload managers, a job step is a unit of work within a job.
+    Flux, which has a robust recursive definition of a :term:`job`, does not
+    use this term.
+
+  system instance
+    A multi-user Flux instance running as the primary resource manager
+    on a cluster.  The system instance typically runs as an unprivileged
+    system user like ``flux``, is started by :linux:man1:`systemd`, and
+    allows :term:`guest` users to run jobs.
+
+  taskmap
+    A compact mapping between job task ranks and node IDs, defined by
+    :doc:`rfc:spec_34`.
+
+  TBON
+    Tree based overlay network.  Flux brokers are interconnected with one.
+
+  urgency
+    A job attribute that the user sets to indicate how urgent the work is.
+    The range is 0 to 31, with a default value of 16.  Urgency is defined
+    by :doc:`rfc:spec_30`.
+
diff --git a/doc/guide/images/Frog-1.svg b/doc/guide/images/Frog-1.svg
new file mode 100644
index 000000000000..ff7b4a7fbebb
--- /dev/null
+++ b/doc/guide/images/Frog-1.svg
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generator: Adobe Illustrator 16.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" width="500px" height="500px" viewBox="0 0 500 500" enable-background="new 0 0 500 500" xml:space="preserve">
+<g>
+	<path fill="#73BF44" d="M308.754,213.385c-20.492,32.787-57.775,56.026-92.202,69.959c-32.787,13.115-64.355,14.467-98.782,14.467   l1.571-2.172c1.639,30.328-11.476,59.836-9.017,90.164c1.64,18.032,11.476,37.705,28.688,44.262   c16.394,6.558,35.246,6.558,53.279,6.558c10.656,0,82.774-23.091,92.61-29.648c14.262-4.559,97.022-56.471,91.816-53.139   c28.688-11.476,51.64-64.754,32.787-91.803c-14.754-19.673-42.623-27.049-64.754-35.246c-12.296-4.918-25.41-4.918-36.886-10.656"/>
+	<g>
+		<path d="M306.596,212.124c-18.178,28.659-47.222,48.649-77.429,62.992c-17.003,8.074-34.769,13.689-53.343,16.747    c-19.15,3.151-38.691,3.435-58.053,3.449c0.72,1.254,1.439,2.508,2.159,3.762c0.524-0.724,1.047-1.448,1.571-2.172    c-1.553-0.421-3.106-0.841-4.659-1.262c1.355,30.465-11.182,59.654-9.017,90.164c0.786,11.066,4.816,22.112,11.144,31.189    c6.833,9.803,16.458,15.104,27.767,18.246c14.135,3.926,29.204,3.87,43.751,3.884c5.251,0.005,10.341-1.309,15.409-2.584    c17.107-4.304,33.974-9.77,50.647-15.506c21.911-7.539,41.771-17.302,61.791-28.949c20.155-11.727,39.62-24.721,59.879-36.241    c-1.141-0.3-2.28-0.599-3.421-0.897l0.001,0.001c0.299-1.14,0.598-2.28,0.896-3.42c-0.076,0.052-0.154,0.101-0.232,0.151    c-2.271,1.457-0.796,5.704,1.927,4.569c15.21-6.341,25.931-20.259,32.936-34.734c7.389-15.269,11.337-33.775,6.346-50.367    c-5.393-17.923-24.021-28.435-40.146-35.239c-9.775-4.125-19.854-7.435-29.819-11.058c-12.249-4.454-25.771-5.271-37.571-10.876    c-2.896-1.375-5.435,2.935-2.523,4.317c9.745,4.628,20.385,5.885,30.71,8.649c9.552,2.557,19.012,6.529,28.282,9.974    c16.687,6.202,36.97,14.898,44.944,32.055c6.865,14.768,2.692,33.469-3.471,47.691c-6.071,14.012-16.472,28.703-31.017,34.766    c0.643,1.523,1.285,3.047,1.927,4.569c0.078-0.05,0.156-0.099,0.232-0.15c1.05-0.721,1.721-2.246,0.897-3.421l-0.001-0.001    c-0.788-1.125-2.161-1.612-3.42-0.896c-22.545,12.82-44.179,27.28-66.698,40.18c-9.271,5.31-18.739,10.044-28.299,14.722    c-14.096,6.898-29.709,11.37-44.619,16.111c-12.698,4.037-25.53,8.101-38.586,10.832c-7.038,1.473-14.422,0.782-21.557,0.576    c-12.896-0.373-26.296-1.755-38.181-7.119c-20.708-9.347-26.065-35.197-25.032-55.524c1.294-25.477,10.268-49.96,9.125-75.663    c-0.116-2.609-3.222-3.248-4.659-1.262c-0.524,0.724-1.047,1.448-1.571,2.172c-1.144,1.581,0.403,3.763,2.159,3.762    c19.282-0.015,38.541-0.526,57.643-3.356c19.139-2.835,37.35-8.79,54.884-16.87c31.526-14.528,61.78-35.741,80.615-65.438    C312.646,211.916,308.316,209.41,306.596,212.124L306.596,212.124z"/>
+	</g>
+</g>
+<g>
+	<g>
+		<g>
+			<path fill="#73BF44" d="M139.833,438.549c7.239-54.288-18.93-99.885-51.639-104.918c-18.853-3.278-22.131,22.951-22.131,37.705     c0.82,17.213,9.836,35.246,18.033,50.819c4.099,7.377,12.295,20.492,8.197,29.509c-4.098,7.377-16.394,10.655-23.771,12.295     c-5.738,1.639-27.049,1.639-28.688,8.196c-4.918,15.574,27.869,6.558,33.606,5.738c-1.639,7.377-0.819,11.475,8.197,11.475     c9.017,0,13.115-5.737,18.853-11.475C111.965,467.237,139.014,455.762,139.833,438.549"/>
+			<g>
+				<path d="M142.333,438.549c3.123-24.555,0.107-49.97-11.18-72.191c-8.324-16.387-22.636-31.357-41.274-34.967      C70.22,327.584,64.928,349.316,63.8,364.43c-1.959,26.23,13.783,47.813,24.209,70.485c1.727,3.754,2.911,7.797,2.884,11.944      c-0.029,4.42-3.495,7.122-6.954,9.186c-5.299,3.162-11.907,4.794-17.94,5.904c-7.917,1.458-23.549,0.069-28.129,8.383      c-4.591,8.33,3.911,12.926,10.839,13.358c8.606,0.538,16.987-2.083,25.396-3.386c-1.025-1.025-2.05-2.051-3.075-3.076      c-1.688,8.116,0.208,13.813,9.18,14.602c11.168,0.983,16.74-7.844,24.321-14.137      C117.603,466.843,140.792,458.074,142.333,438.549c0.254-3.217-4.748-3.192-5,0c-1.132,14.343-19.081,23.313-29.49,30.518      c-5.899,4.084-10.535,8.98-15.834,13.746c-3.756,3.378-8.605,4.619-13.56,3.808c-4.237-0.694-3.247-4.948-2.599-8.062      c0.416-2.001-1.16-3.373-3.075-3.076c-6.285,0.975-12.4,2.588-18.766,3.104c-3.274,0.267-6.667,0.316-9.825-0.687      c-3.034-0.963-3.055-4.862-0.577-6.49c6.62-4.351,16.858-3.376,24.412-4.767c8.501-1.565,19.651-4.556,25.121-11.806      c6.476-8.583,0.434-20.562-3.939-28.676c-6.748-12.521-13.485-25.024-17.678-38.688c-3.368-10.976-3.613-22.068-1.352-33.263      c1.702-8.423,5.714-18.557,15.792-18.338c18.389,0.399,33.121,18.271,40.873,33.007c11.132,21.162,13.476,46.248,10.498,69.668      C136.928,441.741,141.933,441.702,142.333,438.549L142.333,438.549z"/>
+			</g>
+		</g>
+	</g>
+	<g>
+		<g>
+			<path fill="#73BF44" d="M234.72,32.677c1.695,1.695,2.966,2.301,3.813,4.843c5.085,8.475,7.627,14.407,9.322,23.729     c3.39,16.949,11.017-2.542,16.102-9.322c16.949-19.492,48.305-33.898,66.949-9.322c8.475,10.169,15.254,22.034,16.102,35.593     c0.848,22.034-4.237,44.916-7.627,66.949c-5.085,35.593-23.097,82.818-52.758,104.853     c-30.509,23.729-63.344,40.063-102.327,47.689c-29.661,5.085-58.475,1.695-88.135-3.39     c-24.577-5.085-54.237-11.864-74.577-27.118c-26.271-19.492,0.848-44.916,15.254-61.018     c-8.475-27.966-38.136-93.22-9.322-112.711c17.796-12.712,70.339-12.712,80.508,11.017l0.729-0.633     c3.67-6.587,14.667-11.539,20.492-16.394l2.508,0.925c-9.322-9.322-1.695-31.356,3.39-40.678     c6.779-11.017,15.254-20.339,25.424-27.966c12.712-8.475,26.271-11.017,40.678-5.932c12.712,4.237,30.509,11.017,37.288,23.729"/>
+			<g>
+				<path d="M232.952,34.445c4.504,4.245,7.429,10.93,9.639,16.684c2.084,5.424,2.057,11.801,4.979,16.846      c4.297,7.417,11.065-2.656,13.24-6.205c2.373-3.872,4.406-7.643,7.562-10.956c4.174-4.383,8.969-8.119,14.106-11.3      c9.419-5.833,20.892-9.278,31.82-6.15c12.367,3.54,20.967,17.597,26.055,28.521c6.617,14.208,4.344,31.017,2.438,46.046      c-3.363,26.523-7.47,53.661-16.669,78.861c-8.323,22.802-21.07,46.154-40.762,61.05c-22.221,16.809-45.931,30.731-72.409,39.714      c-14.281,4.846-29.029,8.275-44.064,9.554c-15.904,1.352-31.932,0.334-47.746-1.558c-27.824-3.328-57.139-8.904-82.58-21.081      c-10.713-5.127-25.152-12.785-25.494-26.254c-0.286-11.284,9.159-21.91,16.076-29.833c3.051-3.495,7.509-6.959,9.851-10.958      c1.396-2.382-1.772-8.215-2.583-10.588c-3.128-9.149-6.417-18.24-9.368-27.449c-5.871-18.32-12.749-40.034-7.88-59.321      c4.55-18.021,24.455-21.911,40.532-22.537c15.51-0.604,38.83,2.043,46.168,18.2c0.71,1.563,2.822,1.466,3.926,0.506      c1.162-1.009,1.83-2.319,2.833-3.492c1.911-2.234,4.429-3.943,6.829-5.597c3.856-2.658,7.896-5.005,11.559-7.938      c-0.811,0.214-1.622,0.429-2.432,0.643c0.836,0.309,1.672,0.617,2.508,0.925c2.498,0.921,3.998-2.47,2.432-4.178      c-7.758-8.469-2.334-23.994,1.567-33.097c5.389-12.573,16-23.429,26.741-31.621c12.687-9.677,27.993-9.431,42.332-4.459      c11.967,4.149,25.663,9.81,32.213,21.359c1.59,2.804,5.91,0.285,4.317-2.523c-7.05-12.431-21.113-18.568-33.964-23.219      c-7.64-2.765-15.126-4.711-23.313-4.234c-8.697,0.506-16.886,4.086-24.109,8.76c-12.653,8.186-23.995,22.283-29.684,36.153      c-4.426,10.79-8.485,26.756,0.365,36.417c0.811-1.393,1.622-2.786,2.432-4.178c-1.149-0.424-2.571-1.273-3.837-0.925      c-2.622,0.721-5.744,3.943-8.078,5.451c-2.948,1.904-5.832,3.864-8.573,6.057c-2.241,1.792-3.579,4.342-5.673,6.162      c1.309,0.168,2.618,0.337,3.926,0.506c-7.055-15.533-27.117-19.723-42.428-20.604c-15.273-0.879-35.044,1.079-46.282,12.74      c-12.198,12.659-9.319,33.934-6.212,49.485c3.906,19.553,11.277,38.158,17.571,57.009c0.554,1.657,1.083,3.321,1.595,4.992      c0.214-0.811,0.429-1.622,0.643-2.432c-11.926,13.263-33.367,32.456-25.24,52.527c4.311,10.647,16.483,17.054,26.209,21.864      c11.667,5.771,24.245,9.605,36.818,12.823c28.636,7.33,58.761,11.938,88.368,10.948c28.104-0.939,55.667-9.417,80.872-21.573      c12.273-5.92,23.937-13.045,35.077-20.875c9.826-6.906,19.127-14.201,26.89-23.422c15.411-18.308,25.308-41.161,31.915-63.973      c3.439-11.875,5.28-23.962,7.303-36.13c2.509-15.089,5.218-30.158,6.104-45.452c0.823-14.218-1.246-26.709-8.568-39.058      c-5.729-9.662-13.288-19.062-24.029-23.15c-18.935-7.207-40.26,6.073-52.856,19.301c-2.849,2.991-4.785,6.548-6.876,10.08      c-1.097,1.853-2.219,3.717-3.553,5.41c-1.557,1.974-1.68,2.148-2.553-0.372c-1.942-5.599-2.446-11.428-4.902-16.929      c-2.472-5.536-5.126-11.331-9.561-15.509C234.143,28.701,230.601,32.23,232.952,34.445L232.952,34.445z"/>
+			</g>
+		</g>
+		<g>
+			<path fill="#73BF44" d="M129.211,89.215"/>
+			<g>
+				<path d="M129.211,91.715c3.224,0,3.224-5,0-5S125.987,91.715,129.211,91.715L129.211,91.715z"/>
+			</g>
+		</g>
+		<g>
+			<path fill="#73BF44" d="M36.838,206.164"/>
+			<g>
+				<path d="M36.838,208.664c3.224,0,3.224-5,0-5S33.614,208.664,36.838,208.664L36.838,208.664z"/>
+			</g>
+		</g>
+		<g>
+			<path fill="#73BF44" d="M36.838,206.164"/>
+			<g>
+				<path d="M36.838,208.664c3.224,0,3.224-5,0-5S33.614,208.664,36.838,208.664L36.838,208.664z"/>
+			</g>
+		</g>
+	</g>
+	<g>
+		<g>
+			<path fill="#73BF44" d="M344.752,226.787c-9.836-15.574,33.674-44.263,49.248-45.083c15.573-0.82,29.508,20.492,35.246,32.787     c9.836,22.131,8.196,46.721,9.016,70.492c0.82,19.672,4.099,37.705,21.312,48.36c8.197,4.099,18.033,4.099,25.41,9.017     c10.656,7.377,4.099,13.935-4.918,17.213c-6.558,1.64-13.114,1.64-19.672,2.459c0,9.017,20.491,17.214,4.099,23.771     c-12.296,4.918-20.492-5.737-25.41-13.935c-10.656-16.393-29.508-28.012-33.606-46.864"/>
+			<g>
+				<path d="M346.91,225.525c-3.702-6.358,2.923-13.817,7.031-18.143c6.463-6.803,14.437-12.22,22.62-16.728      c6.662-3.67,14.745-7.807,22.524-5.871c10.95,2.726,18.929,14.777,24.271,23.853c14.978,25.449,10.627,55.507,12.85,83.635      c0.855,10.833,3.145,21.702,9.168,30.906c2.767,4.226,6.431,7.672,10.458,10.671c5.299,3.946,12.165,5.051,18.408,6.719      c5.637,1.505,19.4,7.254,10.633,13.811c-6.735,5.039-16.516,4.24-24.48,5.153c-1.281,0.146-2.596,1.027-2.5,2.5      c0.448,6.883,6.678,10.662,9.935,16.155c2.728,4.602-7.607,6.366-10.229,6.342c-5.455-0.052-9.965-4.858-13.008-8.854      c-2.846-3.737-5.157-7.822-8.077-11.522c-3.38-4.282-7.194-8.221-10.917-12.201c-7.585-8.108-15.065-16.542-17.711-27.613      c-0.75-3.135-5.571-1.805-4.822,1.33c2.969,12.419,11.521,21.827,20.015,30.906c4.928,5.268,9.83,10.537,13.845,16.555      c3.011,4.515,5.87,9.14,10.335,12.406c3.939,2.881,8.708,4.66,13.647,3.806c4.235-0.732,10.16-2.726,11.932-7.126      c3.356-8.331-9.461-12.763-9.944-20.184c-0.833,0.833-1.667,1.667-2.5,2.5c10.392-1.191,25.584-0.161,31.667-10.767      c5.866-10.229-11.271-16.625-18.033-18.432c-6.882-1.837-12.979-3.327-18.511-8.088c-4.76-4.098-8.096-9.555-10.336-15.366      c-4.567-11.85-4.456-25.138-4.669-37.638c-0.247-14.546-0.469-29.104-3.412-43.407c-2.435-11.837-7.484-22.675-14.451-32.493      c-5.754-8.11-13.4-16.18-23.367-18.661c-7.039-1.752-14.316,1.366-20.578,4.281c-10.679,4.972-20.887,12.084-29.006,20.632      c-5.686,5.985-11.915,15.194-7.104,23.455C344.215,230.833,348.536,228.317,346.91,225.525L346.91,225.525z"/>
+			</g>
+		</g>
+	</g>
+	<g>
+		<path fill="#73BF44" d="M303.768,300.024c7.377,5.737,11.476,27.049,14.755,36.065c4.918,11.476,10.655,22.951,14.754,35.246    c2.459,9.836,4.099,19.672,2.459,30.328c-0.82,6.557-6.558,18.032-4.918,24.59c3.278,12.295,18.032-4.099,19.672-11.476    c0.819-5.737-3.278-21.312,2.459-24.59c6.558-4.099,16.394,18.032,25.41,18.853c15.573,0.819-2.459-22.951-7.377-27.869    c-2.459-2.459-5.738-3.278-7.378-5.738c46.057-7.083,36.954,4.632,40.648-7.316c-5.737-10.656-23.435-8.257-32.451-12.355    c-10.656-4.098-12.295-12.295-16.394-22.131c-6.558-16.394-20.492-29.508-24.59-45.901"/>
+		<g>
+			<path d="M302,301.792c7.42,6.306,9.402,19.172,11.961,28.073c3.673,12.777,10.339,24.501,15.055,36.914     c5.17,13.607,6.519,28.53,2.168,42.49c-1.864,5.982-7.064,19.948,1.642,23.431c6.751,2.701,13.529-5.756,16.808-10.446     c5.206-7.449,2.637-15.158,2.707-23.646c0.017-1.929,0.087-5.009,1.87-6.261c1.058-0.743,2.89,1.064,3.581,1.659     c3.429,2.949,6.148,6.762,9.17,10.107c4.101,4.539,11.49,10.704,17.713,5.813c5.579-4.384,0.348-13.364-2.327-17.848     c-2.062-3.453-4.498-6.695-7.042-9.807c-2.856-3.495-7.154-5.244-9.934-8.607c-0.367,1.393-0.735,2.785-1.103,4.179     c12.515-1.886,24.563-2.737,37.114-1.545c1.983,0.188,3.376-0.942,4.022-2.747c1.188-3.319,1.862-6.225-0.64-9.108     c-5.103-5.881-13.039-7.17-20.345-8.312c-9.41-1.469-17.569-3.584-22.274-12.684c-2.078-4.021-3.354-8.433-5.196-12.562     c-2.194-4.914-4.881-9.524-7.764-14.062c-6.06-9.542-12.974-18.683-15.958-29.762c-0.838-3.109-5.662-1.788-4.822,1.33     c2.526,9.376,7.665,17.517,12.944,25.56c5.662,8.627,10.403,17.154,14.194,26.765c2.826,7.167,6.475,13.129,13.596,16.618     c6.979,3.418,15.006,3.411,22.441,5.23c3.811,0.933,7.557,2.562,9.919,5.82c0.237-0.604,0.258-0.602,0.061,0.007     c-0.117,0.404-0.223,0.812-0.327,1.219c-0.134,0.519-0.253,1.041-0.374,1.563c-0.305,1.306-0.613,1.079,0.389,0.318     c0.69-0.523,0.313-0.154-0.454-0.261c-12.593-1.753-25.371-0.092-37.856,1.789c-2.048,0.309-2.151,2.909-1.103,4.179     c2.659,3.22,6.683,4.8,9.438,8.017c2.729,3.184,5.128,6.661,7.274,10.258c1.563,2.618,4.001,6.513,3.478,9.729     c-0.719,4.416-8.16-1.246-9.31-2.327c-5.033-4.736-9.603-14-16.867-15.569c-8.606-1.858-8.673,10.934-8.378,16.069     c0.219,3.812,1.442,8.424,0.187,12.125c-1.401,4.13-4.9,7.87-8.158,10.623c-4.561,3.854-6.75,0.923-6.38-4.317     c0.418-5.908,3.255-11.657,4.645-17.374c6.765-27.827-12.045-51.761-19.314-77.052c-2.708-9.42-5.038-22.432-12.915-29.125     C303.094,296.182,299.539,299.701,302,301.792L302,301.792z"/>
+		</g>
+	</g>
+	<g>
+		<path fill="#73BF44" d="M195.571,355.762c0,15.574,9.836,32.787,17.213,45.902c6.558,13.114,12.295,25.409,21.312,36.885    c6.558,7.377,6.558,7.377,2.459,15.574c-2.459,4.098-10.656,13.934-3.278,17.213c9.836,4.098,21.312-23.771,27.049-27.869    c4.099,4.099,13.115,25.41,22.951,17.213c8.196-7.377-8.197-25.409-9.836-31.147c5.737-1.639,31.967,3.279,31.967-8.196    c0-10.656-36.065-10.656-44.263-13.935c-10.655-3.278-11.475-9.017-15.573-18.033c-6.558-12.295-16.394-23.771-21.312-36.065"/>
+		<g>
+			<path d="M193.071,355.762c0.409,19.77,12.288,36.577,20.784,53.653c4.731,9.508,9.68,18.872,15.894,27.51     c1.907,2.65,6.884,6.5,7.125,9.842c0.215,2.975-2.886,6.759-4.472,9.144c-2.645,3.976-6.969,10.7-3.234,15.316     c3.165,3.912,8.077,3.53,11.941,0.981c4.779-3.152,8.152-8.554,11.243-13.234c2.924-4.427,5.493-10.4,9.74-13.739     c-1.179,0-2.356,0-3.535,0c6.139,6.618,13.885,26.713,25.98,17.604c11.587-8.726-5.825-25.412-8.687-33.971     c-0.582,1.025-1.164,2.051-1.746,3.076c9.604-2.307,22.21,3.061,30.646-3.989c3.214-2.686,4.236-7.413,1.689-10.907     c-2.849-3.909-8.378-5.373-12.812-6.499c-11.695-2.972-25.514-1.694-36.332-7.452c-3.251-1.729-5.041-4.836-6.552-8.045     c-2.099-4.455-3.96-8.797-6.53-13.031c-5.916-9.748-13.124-18.789-17.545-29.382c-1.223-2.93-6.063-1.646-4.821,1.33     c3.677,8.809,8.987,16.64,14.217,24.58c2.596,3.94,5.086,7.938,7.347,12.082c1.726,3.163,2.742,6.684,4.495,9.851     c5.62,10.156,18.551,10.801,28.732,12.174c3.195,0.431,31.134,3.686,25.595,10.822c-2.974,3.832-11.034,3.238-15.268,3.269     c-4.623,0.032-9.646-0.715-14.191,0.376c-1.271,0.306-2.168,1.814-1.746,3.076c1.345,4.02,4.119,7.57,6.318,11.147     c2.479,4.031,7.422,11.081,5.3,16.081c-4.625,10.897-17.861-12.823-20.555-15.727c-1.007-1.086-2.498-0.815-3.535,0     c-2.987,2.348-5.005,6.137-7.016,9.282c-3.152,4.929-6.194,9.985-10.144,14.331c-1.527,1.68-3.298,3.281-5.53,3.888     c-4.67,1.268-2.424-5.131-1.495-6.931c2.437-4.726,6.947-9.19,7.434-14.731c0.536-6.089-6.712-11.609-9.806-16.058     c-5.374-7.727-9.633-16.188-13.828-24.59c-8.087-16.196-19.713-32.37-20.101-51.129     C198.005,352.542,193.004,352.535,193.071,355.762L193.071,355.762z"/>
+		</g>
+	</g>
+</g>
+<ellipse transform="matrix(0.9397 -0.342 0.342 0.9397 -15.4814 71.265)" fill="#FFFFFF" stroke="#000000" stroke-width="4" stroke-miterlimit="10" cx="194.341" cy="79.532" rx="33.418" ry="44.197"/>
+<g>
+	<g>
+		<path d="M80.152,249.156c28.657,6.843,60.165,14.622,89.644,9.212c24.712-4.534,50.117-16.275,69.843-31.775    c9.609-7.55,17.733-16.79,24.484-26.947c8.15-12.26,16.035-24.621,22.844-37.689c6.665-12.792,11.299-26.061,16.134-39.593    c2.157-6.039,4.389-11.885,5.341-18.265c1.104-7.401,0.102-14.924,1.485-22.361c0.117-0.514,0.233-1.028,0.351-1.542    c0.713-3.14-4.107-4.473-4.822-1.329c-1.313,5.787-1.615,11.67-1.708,17.588c-0.113,7.228-1.418,13.488-3.877,20.269    c-4.865,13.414-9.272,26.736-15.624,39.567c-5.997,12.114-13.077,23.67-20.502,34.952c-7.049,10.711-14.562,20.576-24.271,29.029    c-18.708,16.286-43.75,27.044-67.722,32.592c-14.634,3.386-29.25,2.313-44.004,0.435c-15.587-1.985-30.993-5.318-46.265-8.964    C78.347,243.586,77.017,248.407,80.152,249.156L80.152,249.156z"/>
+	</g>
+</g>
+<g>
+	<g>
+		<path d="M344.752,229.287c20.978,1.681,41.015,14.449,54.581,30.037c15.91,18.28,17.331,44.28,3.983,64.418    c-1.787,2.696,2.546,5.197,4.318,2.523c13.635-20.572,13.097-47.529-2.07-67.131c-14.289-18.466-37.245-32.959-60.813-34.848    C341.536,224.029,341.561,229.031,344.752,229.287L344.752,229.287z"/>
+	</g>
+</g>
+<ellipse transform="matrix(0.903 -0.4296 0.4296 0.903 -58.5981 44.9486)" fill="#FFFFFF" stroke="#000000" stroke-width="4" stroke-miterlimit="10" cx="70.256" cy="152.262" rx="33.418" ry="44.197"/>
+<g>
+	<g>
+		<path d="M75.926,238.262c0.646,5.794-0.973,12.728-5.005,17.084c-2.187,2.361,1.341,5.906,3.535,3.535    c4.889-5.28,7.262-13.512,6.47-20.62C80.573,235.095,75.569,235.06,75.926,238.262L75.926,238.262z"/>
+	</g>
+</g>
+<g>
+	<g>
+		<path d="M295.119,73.95c5.387,7.642,12.896,9.389,21.832,9.435c3.225,0.017,3.224-4.984,0-5    c-6.964-0.035-13.231-0.882-17.514-6.958C297.596,68.814,293.257,71.308,295.119,73.95L295.119,73.95z"/>
+	</g>
+</g>
+<ellipse transform="matrix(0.903 -0.4296 0.4296 0.903 -64.208 52.8228)" stroke="#000000" stroke-width="4" stroke-miterlimit="10" cx="84.893" cy="168.625" rx="17.637" ry="22.602"/>
+<ellipse transform="matrix(0.903 -0.4296 0.4296 0.903 -24.0602 96.0336)" stroke="#000000" stroke-width="4" stroke-miterlimit="10" cx="200.68" cy="101.309" rx="17.637" ry="22.603"/>
+</svg>
\ No newline at end of file
diff --git a/doc/guide/images/adminarch.dia b/doc/guide/images/adminarch.dia
new file mode 100644
index 000000000000..05f8be68c16f
Binary files /dev/null and b/doc/guide/images/adminarch.dia differ
diff --git a/doc/guide/images/adminarch.png b/doc/guide/images/adminarch.png
new file mode 100644
index 000000000000..25f55105aafc
Binary files /dev/null and b/doc/guide/images/adminarch.png differ
diff --git a/doc/guide/images/fox-standing.svg b/doc/guide/images/fox-standing.svg
new file mode 100644
index 000000000000..7d5d064b589e
--- /dev/null
+++ b/doc/guide/images/fox-standing.svg
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+<svg xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns:svg="http://www.w3.org/2000/svg" xmlns:ns1="http://sozi.baierouge.fr" xmlns:xlink="http://www.w3.org/1999/xlink" id="svg4674" sodipodi:docname="fox_standing.svg" inkscape:export-filename="/Users/jamiely/Documents/code/BunnyMatcher/BunnyMatcher/Assets/inkscape/fox_standing.png" viewBox="0 0 800 450" inkscape:export-xdpi="90" version="1.1" inkscape:export-ydpi="90" inkscape:version="0.48.2 r9819">
+  <sodipodi:namedview id="base" bordercolor="#666666" inkscape:pageshadow="2" inkscape:window-y="0" pagecolor="#ffffff" inkscape:window-height="852" inkscape:window-maximized="1" inkscape:zoom="0.75562114" inkscape:window-x="0" showgrid="false" borderopacity="1.0" inkscape:current-layer="layer1" inkscape:cx="540.88332" inkscape:cy="204.15786" inkscape:window-width="1440" inkscape:pageopacity="0.0" inkscape:document-units="px"/>
+  <g id="layer1" inkscape:label="Layer 1" inkscape:groupmode="layer" transform="translate(0 -602.36)">
+    <path id="path4694" sodipodi:rx="85.360237" sodipodi:ry="107.85828" style="fill:#ff6600" sodipodi:type="arc" d="m412.91 381.14a85.36 107.86 0 1 1 -170.72 0 85.36 107.86 0 1 1 170.72 0z" transform="matrix(.64125 -.50945 .48018 .82897 -212.41 621.39)" sodipodi:cy="381.13931" sodipodi:cx="327.5451"/>
+    <g id="g4929" transform="translate(-167.66 99.107)">
+      <path id="path4931" sodipodi:rx="43.67268" sodipodi:ry="74.111214" style="fill:#ff6600" sodipodi:type="arc" d="m674.94 475.76c0 40.93-19.553 74.111-43.673 74.111s-43.673-33.181-43.673-74.111 19.553-74.111 43.673-74.111 43.673 33.181 43.673 74.111z" transform="matrix(1.1047 .26566 -.29293 1.0019 57.371 59.945)" sodipodi:cy="475.76343" sodipodi:cx="631.26874"/>
+      <path id="path4933" sodipodi:rx="39.702435" sodipodi:ry="56.245117" style="fill:#ff6600" sodipodi:type="arc" d="m647.15 543.92c0 31.063-17.775 56.245-39.702 56.245s-39.702-25.182-39.702-56.245 17.775-56.245 39.702-56.245 39.702 25.182 39.702 56.245z" transform="matrix(.88629 -.46312 .46312 .88629 -176.21 557.15)" sodipodi:cy="543.91931" sodipodi:cx="607.44727"/>
+      <path id="path4935" sodipodi:rx="17.20439" sodipodi:ry="44.996094" style="fill:#ff6600" sodipodi:type="arc" d="m651.12 586.93c0 24.851-7.7027 44.996-17.204 44.996-9.5017 0-17.204-20.145-17.204-44.996s7.7027-44.996 17.204-44.996c9.5017 0 17.204 20.145 17.204 44.996z" transform="matrix(1.2192 -.46991 .64906 .88272 -516.63 591.29)" sodipodi:cy="586.93024" sodipodi:cx="633.91559"/>
+      <path id="path4937" sodipodi:rx="18.527803" sodipodi:ry="47.642921" style="fill:#ff6600" sodipodi:type="arc" d="m667 646.48c0 26.312-8.2952 47.643-18.528 47.643s-18.528-21.33-18.528-47.643c0-26.312 8.2952-47.643 18.528-47.643s18.528 21.33 18.528 47.643z" transform="matrix(.93268 .36071 -.36071 .93268 278.17 34.184)" sodipodi:cy="646.48389" sodipodi:cx="648.47314"/>
+    </g>
+    <path id="path4696" style="fill:#ff6600" inkscape:connector-curvature="0" d="m411.12 710.72c-3.5149 0.0239-7.0491 0.10459-10.625 0.28125-18.288 0.90347-35.641 3.7846-51.438 8.1875-22.637-2.4723-48.229-1.425-74.781 3.7812-79.255 15.54-136.85 62.135-128.62 104.09 8.2272 41.959 79.151 63.384 158.41 47.844 16.482-3.2317 32.009-7.8083 46.219-13.406 17.734 3.6312 37.316 5.1704 57.844 4.1562 76.286-3.7687 136.42-41.419 134.31-84.125-2.0109-40.704-59.842-71.298-131.31-70.812z"/>
+    <path id="path4780" style="fill:#b3b3b3" inkscape:connector-curvature="0" d="m541.34 773.75c-8.5867 37.445-64.316 68.502-133.22 71.906-20.527 1.0141-40.109-0.525-57.844-4.1562-14.21 5.5979-29.737 10.175-46.219 13.406-78.135 15.321-148.15-5.3034-158-46.094-1.4016 6.1093-1.5874 12.226-0.40625 18.25 8.2272 41.959 79.151 63.384 158.41 47.844 16.482-3.2317 32.009-7.8083 46.219-13.406 17.734 3.6312 37.316 5.1704 57.844 4.1562 76.286-3.7687 136.42-41.419 134.31-84.125-0.13046-2.6407-0.50494-5.2314-1.0938-7.7812z"/>
+    <g id="g5268" transform="matrix(.64374 .76524 -.76524 .64374 585.22 -145.87)">
+      <path id="path4709" sodipodi:rx="75.434631" sodipodi:ry="45.657803" style="fill:#ff6600" sodipodi:type="arc" d="m800.67 411.58c0 25.216-33.773 45.658-75.435 45.658-41.661 0-75.435-20.442-75.435-45.658s33.773-45.658 75.435-45.658c41.661 0 75.435 20.442 75.435 45.658z" transform="matrix(.98761 .15692 -.15692 .98761 56.364 133.07)" sodipodi:cy="411.57785" sodipodi:cx="725.23114"/>
+      <path id="path4747" style="fill:#ff6600" inkscape:connector-curvature="0" d="m845.41 626.42c-10.972-3.3887-23.398-5.5-36.812-5.5-42.392 0-76.75 19.246-76.75 43s34.358 43.031 76.75 43.031c13.921 0 26.805-2.2446 38.062-5.875-2.3004-1.5344-4.5152-3.1849-6.5-5.0625-2.2153-2.0956-4.2088-4.3795-5.9688-6.8438-1.7599-2.4642-3.2923-5.1042-4.5312-7.875s-2.1915-5.6722-2.8438-8.6875c-0.32614-1.5076-0.58283-3.0366-0.75-4.5938-0.16717-1.5571-0.25-3.1512-0.25-4.75s0.0828-3.1616 0.25-4.7188c0.16717-1.5571 0.42386-3.0861 0.75-4.5938 0.65228-3.0153 1.6048-5.948 2.8438-8.7188 1.2389-2.7708 2.7713-5.3795 4.5312-7.8438 1.7599-2.4642 3.7535-4.7794 5.9688-6.875 1.6039-1.5172 3.4371-2.7979 5.25-4.0938z"/>
+      <path id="path4711" style="fill:#b3b3b3" inkscape:connector-curvature="0" d="m845.41 626.42c-1.8129 1.2959-3.6461 2.5766-5.25 4.0938-2.2153 2.0956-4.2088 4.4108-5.9688 6.875-1.7599 2.4642-3.2923 5.073-4.5312 7.8438s-2.1915 5.7035-2.8438 8.7188c-0.32614 1.5076-0.58283 3.0366-0.75 4.5938-0.16717 1.5571-0.25 3.1199-0.25 4.7188 0 1.5988 0.0828 3.1929 0.25 4.75 0.16717 1.5571 0.42386 3.0861 0.75 4.5938 0.65228 3.0153 1.6048 5.9167 2.8438 8.6875 1.2389 2.7708 2.7713 5.4108 4.5312 7.875s3.7535 4.7482 5.9688 6.8438c1.9848 1.8776 4.1996 3.5282 6.5 5.0625 23.023-7.4251 38.719-21.202 38.719-37.156 0-16.238-16.266-30.18-39.969-37.5z"/>
+    </g>
+    <path id="path4794" style="fill:#b3b3b3" inkscape:connector-curvature="0" d="m146.27 672.44c-6.2157 1.5493-12.076 4.3903-17.406 8.625-30.23 24.017-31.572 83.526-2.9688 132.91 28.604 49.38 76.301 69.923 106.53 45.906 1.2904-1.0252 2.5338-2.0991 3.7188-3.25-29.032 7.2148-65.511-13.904-89.062-54.562-27.372-47.254-27.356-103.8-0.8125-129.62z"/>
+    <path id="path4704" style="fill:#ff6600" inkscape:connector-curvature="0" d="m137.65 643.53c-14.612 0-27.388 6.6948-34.281 16.688-5.0593 1.043-9.9489 2.3219-14.656 3.8125 1.683 4.8219 2.5938 9.8811 2.5938 15.125 0 27.901-25.449 51.109-59.031 55.969 10.503 25.639 50.844 40.899 96.25 34.844 50.352-6.7154 87.857-37.116 83.75-67.906-2.2868-17.147-17-30.774-38.281-38.344-6.1396-11.893-20.085-20.188-36.344-20.188z"/>
+    <g id="g4806" transform="translate(-133.66 119.11)">
+      <path id="path4717" d="m222.98 549.46-11.911-62.2s43.673 26.468 48.966 54.26c5.2937 27.792-37.056 7.9405-37.056 7.9405z" style="fill:#ff6600" inkscape:connector-curvature="0"/>
+      <path id="path4803" d="m211.06 234.91 3.6562 19.156c8.4829 1.6065 17.844 10.707 23.312 23.688 3.8592 9.161 4.9801 18.348 3.625 25.5 10.577 2 21.185 0.65651 18.375-14.094-5.2937-27.792-48.969-54.25-48.969-54.25z" style="fill:#d45500" inkscape:connector-curvature="0" transform="translate(0 252.36)"/>
+    </g>
+    <path id="path4844" sodipodi:rx="8.6021948" sodipodi:ry="8.6021948" style="fill:#000000" sodipodi:type="arc" d="m244.83 340.11a8.6022 8.6022 0 1 1 -17.204 0 8.6022 8.6022 0 1 1 17.204 0z" transform="translate(-133.66 371.47)" sodipodi:cy="340.11346" sodipodi:cx="236.22949"/>
+    <path id="path4848" style="fill:#b3b3b3" inkscape:connector-curvature="0" d="m86.679 726.81c-18.951 0-35.187 10.215-42.094 24.75 17.767 15.269 49.374 23.016 83.937 18.406 1.2594-0.16796 2.5071-0.36527 3.75-0.5625 0.0457-0.73983 0.0625-1.5001 0.0625-2.25 0-22.292-20.44-40.344-45.656-40.344z"/>
+    <path id="path4842" sodipodi:rx="13.234145" sodipodi:ry="13.234145" style="fill:#000000" sodipodi:type="arc" d="m189.25 365.92a13.234 13.234 0 1 1 -26.468 0 13.234 13.234 0 1 1 26.468 0z" transform="translate(-133.66 371.47)" sodipodi:cy="365.92004" sodipodi:cx="176.01413"/>
+    <path id="path4872" sodipodi:rx="26.46829" sodipodi:ry="47.642921" style="fill:#ff6600" sodipodi:type="arc" d="m415.55 569.73a26.468 47.643 0 1 1 -52.937 0 26.468 47.643 0 1 1 52.937 0z" transform="matrix(.99637 -.38043 .29870 1.3387 -287.62 305.19)" sodipodi:cy="569.72589" sodipodi:cx="389.08386"/>
+    <path id="path4874" sodipodi:rx="23.821461" sodipodi:ry="43.67268" style="fill:#ff6600" sodipodi:type="arc" d="m428.79 585.61a23.821 43.673 0 1 1 -47.643 0 23.821 43.673 0 1 1 47.643 0z" transform="matrix(.81472 -.31058 .20860 1.3565 -161.71 322.62)" sodipodi:cy="585.60687" sodipodi:cx="404.96484"/>
+    <path id="path4876" sodipodi:rx="25.144876" sodipodi:ry="9.2639017" style="fill:#280b0b" sodipodi:type="arc" d="m428.79 637.22a25.145 9.2639 0 1 1 -50.29 0 25.145 9.2639 0 1 1 50.29 0z" transform="matrix(.73235 -.17672 .14234 .98463 -94.792 486.95)" sodipodi:cy="637.22003" sodipodi:cx="403.64142"/>
+    <path id="path4892" sodipodi:rx="29.11512" sodipodi:ry="37.055607" style="fill:#ff6600" sodipodi:type="arc" d="m363.94 518.11a29.115 37.056 0 1 1 -58.23 0 29.115 37.056 0 1 1 58.23 0z" transform="matrix(.96980 .047093 .048865 1.3156 -140.14 190.74)" sodipodi:cy="518.11273" sodipodi:cx="334.82388"/>
+    <path id="path4894" sodipodi:rx="20.512926" sodipodi:ry="33.74707" style="fill:#ff6600" sodipodi:type="arc" d="m357.32 558.48a20.513 33.747 0 1 1 -41.026 0 20.513 33.747 0 1 1 41.026 0z" transform="matrix(.96980 .047093 .048865 1.3156 -140.14 190.74)" sodipodi:cy="558.47687" sodipodi:cx="336.80899"/>
+    <path id="path4896" sodipodi:rx="16.542681" sodipodi:ry="23.821461" style="fill:#ff6600" sodipodi:type="arc" d="m356 600.16a16.543 23.821 0 1 1 -33.085 0 16.543 23.821 0 1 1 33.085 0z" transform="matrix(.96980 .047093 .048865 1.3156 -140.14 190.74)" sodipodi:cy="600.16443" sodipodi:cx="339.45584"/>
+    <path id="path4898" sodipodi:rx="21.83634" sodipodi:ry="9.2639017" style="fill:#280b0b" sodipodi:type="arc" d="m349.38 621.34a21.836 9.2639 0 1 1 -43.673 0 21.836 9.2639 0 1 1 43.673 0z" transform="matrix(.69714 .031808 .035126 .88861 -36.336 457.11)" sodipodi:cy="621.33905" sodipodi:cx="327.5451"/>
+    <path id="path4867" style="fill:#ff6600" inkscape:connector-curvature="0" d="m152.39 844.07c3.4513 5.4467 7.3688 10.83 11.772 16.016 33.128 39.023 80.286 53.401 105.36 32.115s18.564-70.171-14.563-109.19c-4.4027-5.1861-9.0788-9.9256-13.893-14.215l-88.675 75.279z"/>
+    <path id="path4883" style="fill:#b3b3b3" inkscape:connector-curvature="0" d="m169.18 829.81-16.781 14.25c3.4513 5.4467 7.3473 10.845 11.75 16.031 8.7743 10.336 18.553 18.921 28.625 25.594 0.80221-8.8566-0.73803-19.251-4.875-29.594-4.5151-11.288-11.302-20.436-18.719-26.281z"/>
+    <g id="g4923" transform="translate(-133.66 119.11)">
+      <path id="path4913" sodipodi:rx="43.67268" sodipodi:ry="74.111214" style="fill:#ff6600" sodipodi:type="arc" d="m674.94 475.76c0 40.93-19.553 74.111-43.673 74.111s-43.673-33.181-43.673-74.111 19.553-74.111 43.673-74.111 43.673 33.181 43.673 74.111z" transform="matrix(1.1047 .26566 -.29293 1.0019 57.371 59.945)" sodipodi:cy="475.76343" sodipodi:cx="631.26874"/>
+      <path id="path4915" sodipodi:rx="39.702435" sodipodi:ry="56.245117" style="fill:#ff6600" sodipodi:type="arc" d="m647.15 543.92c0 31.063-17.775 56.245-39.702 56.245s-39.702-25.182-39.702-56.245 17.775-56.245 39.702-56.245 39.702 25.182 39.702 56.245z" transform="matrix(.88629 -.46312 .46312 .88629 -176.21 557.15)" sodipodi:cy="543.91931" sodipodi:cx="607.44727"/>
+      <path id="path4917" sodipodi:rx="17.20439" sodipodi:ry="44.996094" style="fill:#ff6600" sodipodi:type="arc" d="m651.12 586.93c0 24.851-7.7027 44.996-17.204 44.996-9.5017 0-17.204-20.145-17.204-44.996s7.7027-44.996 17.204-44.996c9.5017 0 17.204 20.145 17.204 44.996z" transform="matrix(1.2192 -.46991 .64906 .88272 -516.63 591.29)" sodipodi:cy="586.93024" sodipodi:cx="633.91559"/>
+      <path id="path4919" sodipodi:rx="18.527803" sodipodi:ry="47.642921" style="fill:#ff6600" sodipodi:type="arc" d="m667 646.48c0 26.312-8.2952 47.643-18.528 47.643s-18.528-21.33-18.528-47.643c0-26.312 8.2952-47.643 18.528-47.643s18.528 21.33 18.528 47.643z" transform="matrix(.93268 .36071 -.36071 .93268 278.17 34.184)" sodipodi:cy="646.48389" sodipodi:cx="648.47314"/>
+    </g>
+    <path id="path4921" sodipodi:rx="17.20439" sodipodi:ry="7.27878" style="fill:#000000" sodipodi:type="arc" d="m648.47 661.7a17.204 7.2788 0 1 1 -34.409 0 17.204 7.2788 0 1 1 34.409 0z" transform="matrix(1.1923 0 0 1.3636 -258.37 128.2)" sodipodi:cy="661.70319" sodipodi:cx="631.26874"/>
+    <path id="path4939" sodipodi:rx="17.20439" sodipodi:ry="7.27878" style="fill:#000000" sodipodi:type="arc" d="m648.47 661.7a17.204 7.2788 0 1 1 -34.409 0 17.204 7.2788 0 1 1 34.409 0z" transform="matrix(1.0769 0 0 1.0909 -217.96 286.83)" sodipodi:cy="661.70319" sodipodi:cx="631.26874"/>
+  </g>
+  <metadata>
+    <rdf:RDF>
+      <cc:Work>
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
+        <cc:license rdf:resource="http://creativecommons.org/licenses/publicdomain/"/>
+        <dc:publisher>
+          <cc:Agent rdf:about="http://openclipart.org/">
+            <dc:title>Openclipart</dc:title>
+          </cc:Agent>
+        </dc:publisher>
+        <dc:title>Fox</dc:title>
+        <dc:date>2012-12-27T21:32:43</dc:date>
+        <dc:description/>
+        <dc:source>https://openclipart.org/detail/173937/fox-by-jamiely-173937</dc:source>
+        <dc:creator>
+          <cc:Agent>
+            <dc:title>jamiely</dc:title>
+          </cc:Agent>
+        </dc:creator>
+        <dc:subject>
+          <rdf:Bag>
+            <rdf:li>fox</rdf:li>
+          </rdf:Bag>
+        </dc:subject>
+      </cc:Work>
+      <cc:License rdf:about="http://creativecommons.org/licenses/publicdomain/">
+        <cc:permits rdf:resource="http://creativecommons.org/ns#Reproduction"/>
+        <cc:permits rdf:resource="http://creativecommons.org/ns#Distribution"/>
+        <cc:permits rdf:resource="http://creativecommons.org/ns#DerivativeWorks"/>
+      </cc:License>
+    </rdf:RDF>
+  </metadata>
+</svg>
\ No newline at end of file
diff --git a/doc/guide/images/lgplv3-147x51.png b/doc/guide/images/lgplv3-147x51.png
new file mode 100644
index 000000000000..012f011becd9
Binary files /dev/null and b/doc/guide/images/lgplv3-147x51.png differ
diff --git a/doc/guide/images/states.dot b/doc/guide/images/states.dot
new file mode 100644
index 000000000000..1688df302f7e
--- /dev/null
+++ b/doc/guide/images/states.dot
@@ -0,0 +1,36 @@
+// Usage: dot -Tpng file.dot -o file.png
+
+digraph finite_state_machine {
+    rankdir=LR;
+    size="8,5"
+
+    node [shape = point ]; N
+    node [shape = doublecircle ]; E
+
+    node [shape = circle];
+    N -> J;
+    J -> 1 [ label = "parent-ready" ];
+    J -> 1 [ label = "parent-none" ];
+    J -> S [ label = "parent-fail" ];
+    J -> S [ label = "parent-timeout" ];
+    1 -> Q [ label = "rc1-none" ];
+    1 -> Q [ label = "rc1-success" ];
+    1 -> S [ label = "rc1-fail" ];
+    Q -> 2 [ label = "quorum-full" ];
+    Q -> S [ label = "quorum-timeout" ];
+    2 -> 2 [ label = "rc2-none" ];
+    2 -> C [ label = "rc2-success" ];
+    2 -> C [ label = "rc2-fail" ];
+    2 -> C [ label = "shutdown" ];
+    2 -> C [ label = "signal-abort" ];
+    C -> S [ label = "cleanup-none" ];
+    C -> S [ label = "cleanup-success" ];
+    C -> S [ label = "cleanup-fail" ];
+    S -> 3 [ label = "children-complete" ];
+    S -> 3 [ label = "children-none" ];
+    S -> 3 [ label = "children-timeout" ];
+    3 -> G [ label = "goodbye" ];
+    G -> E [ label = "rc3-none" ];
+    G -> E [ label = "rc3-success" ];
+    G -> E [ label = "rc3-fail" ];
+}
diff --git a/doc/guide/images/states.png b/doc/guide/images/states.png
new file mode 100644
index 000000000000..28a9e115c51d
Binary files /dev/null and b/doc/guide/images/states.png differ
diff --git a/doc/guide/images/states_norm.dot b/doc/guide/images/states_norm.dot
new file mode 100644
index 000000000000..146898f31c3b
--- /dev/null
+++ b/doc/guide/images/states_norm.dot
@@ -0,0 +1,36 @@
+// Usage: dot -Tpng file.dot -o file.png
+
+digraph finite_state_machine {
+    rankdir=LR;
+    size="8,5"
+
+    node [shape = point ]; N
+    node [shape = doublecircle ]; E
+
+    node [shape = circle];
+    N -> J [ color= "green" ];
+    J -> 1 [ label = "parent-ready", color = "green" ];
+    J -> 1 [ label = "parent-none", color = "blue" ];
+    J -> S [ label = "parent-fail" ];
+    J -> S [ label = "parent-timeout" ];
+    1 -> Q [ label = "rc1-none" ];
+    1 -> Q [ label = "rc1-success", color = "green" ];
+    Q -> 2 [ label = "quorum-full", color = "green" ];
+    Q -> S [ label = "quorum-timeout" ];
+    1 -> S [ label = "rc1-fail" ];
+    2 -> 2 [ label = "rc2-none", color = "green" ];
+    2 -> C [ label = "rc2-success", color = "blue" ];
+    2 -> C [ label = "rc2-fail" ];
+    2 -> C [ label = "shutdown", color = "green" ];
+    2 -> C [ label = "signal-abort" ];
+    C -> S [ label = "cleanup-none", color = "green" ];
+    C -> S [ label = "cleanup-success", color = "blue" ];
+    C -> S [ label = "cleanup-fail" ];
+    S -> 3 [ label = "children-complete", color = "green" ];
+    S -> 3 [ label = "children-none", color = "red" ];
+    S -> 3 [ label = "children-timeout" ];
+    3 -> G [ label = "goodbye" ];
+    G -> E [ label = "rc3-none" ];
+    G -> E [ label = "rc3-success", color = "green" ];
+    G -> E [ label = "rc3-fail" ];
+}
diff --git a/doc/guide/images/states_norm.png b/doc/guide/images/states_norm.png
new file mode 100644
index 000000000000..d9e4ffd52fc5
Binary files /dev/null and b/doc/guide/images/states_norm.png differ
diff --git a/doc/guide/interact.rst b/doc/guide/interact.rst
new file mode 100644
index 000000000000..adac43e9ff3e
--- /dev/null
+++ b/doc/guide/interact.rst
@@ -0,0 +1,354 @@
+Interacting with Flux
+=====================
+
+.. _command_summary:
+
+Command Summary
+---------------
+
+Here is an abbreviated list of Flux commands, to get you started while exploring
+Flux.  If commands fail to connect to a Flux instance, refer to
+:ref:`connect_to_flux` below.
+
+To learn more, help is available within most commands, and many have
+:ref:`Manual Pages <man-commands>`.
+
+.. list-table::
+   :header-rows: 1
+
+   * - Example
+     - Description
+
+   * - :command:`flux help`
+
+       :command:`flux help run`
+
+       :command:`flux run --help`
+
+       :command:`flux job kill --help`
+
+     - Print a brief command summary.
+
+       Display :man1:`flux-run`.
+
+       Summarize run options.
+
+       List usage and options for :option:`flux job kill` sub-command.
+
+   * - :command:`flux start -s16`
+
+     -  Start a test instance that mocks 16 nodes.  See :man1:`flux-start`.
+
+   * - :command:`flux version`
+
+       :command:`flux uptime`
+
+     - Print the Flux version.  See :man1:`flux-version`.
+
+       Show brief Flux instance info.  See :man1:`flux-uptime`.
+
+   * - :command:`flux ping 15`
+
+       :command:`flux exec -x 0 cmd`
+
+     - Bounce a message off broker rank 15.  See :man1:`flux-ping`.
+
+       Run cmd on all ranks except rank 0.  Not a job. See :man1:`flux-exec`.
+
+   * - :command:`flux resource info`
+
+       :command:`flux resource list`
+
+       :command:`flux resource status`
+
+     - Show a single line summary of scheduler view of resources.
+
+       Show longer scheduler view of resources.
+
+       Show system view of resources.  See :man1:`flux-resource`.
+
+   * - :command:`flux run sleep 5`
+
+       :command:`flux run -N8 -n16 hostname`
+
+       :command:`flux run -n64 hostname`
+
+     - Run a job with 1 :linux:man1:`sleep` command.  Blocks until done.
+
+       Run a job with 16 :linux:man1:`hostname` commands, two per node.
+
+       Run a job with 64 tasks with 1 cpu per task.  See :man1:`flux-run`.
+
+   * - :command:`flux submit -n64 -c2 hostname`
+
+       :command:`flux submit --cc 1-5 sleep 30`
+
+       :command:`flux watch --all`
+
+     - Submit a job with 64 tasks, 2 cpus per task.  See :man1:`flux-submit`.
+
+       Submit 5 jobs, each consisting of one :linux:man1:`sleep` task.
+
+       Watch all job output and wait for completion.  See :man1:`flux-watch`.
+
+   * - :command:`flux alloc -N4`
+
+       :command:`flux bulksubmit sleep {} ::: 8 9`
+
+       :command:`flux top`
+
+     - Start an interactive 4 node instance.  See :man1:`flux-alloc`.
+
+       Submit 2 jobs that sleep different times.  See :man1:`flux-bulksubmit`.
+
+       View the progress of running jobs.  See :man1:`flux-top`.
+
+   * - :command:`flux batch -N4 script.sh`
+
+       :command:`flux batch -N1 --wrap sleep 60`
+
+       :command:`flux pstree`
+
+     - Submit job to run :command:`script.sh` in a 4 node instance.
+
+       Submit job to run :linux:man1:`sleep` in a 1 node instance.  See :man1:`flux-batch`.
+
+       Display tree of running jobs by name.  See :man1:`flux-pstree`.
+
+   * - :command:`flux jobs -A`
+
+       :command:`flux jobs -a`
+
+       :command:`flux jobs -o endreason ƒuAsjAo`
+
+       :command:`flux job last`
+
+     - List active jobs for all users.  See :man1:`flux-jobs`.
+
+       List all my jobs (inactive too).
+
+       Show info about the specified job including why it ended.
+
+       Print my most recently submitted jobid.  See :man1:`flux-job`.
+
+   * - :command:`flux cancel ƒuAsjAo`
+
+       :command:`flux job kill -s HUP ƒuAsjAo`
+
+       :command:`flux pgrep -f pending .`
+
+       :command:`flux pkill sl..p`
+
+     - Cancel specified job.  See :man1:`flux-cancel`.
+
+       Send specified job a SIGHUP.  See :man1:`flux-job`.
+
+       List ids of all pending jobs.  See :man1:`flux-pgrep`.
+
+       Cancel all jobs named sleep or slurp.  See :man1:`flux-pkill`.
+
+.. _connect_to_flux:
+
+Connecting to Flux
+------------------
+
+Flux commands need a Flux instance to talk to.  Which one?  Remember that batch
+jobs are Flux instances, allocations are Flux instances, and Slurm jobs can
+even be Flux instances.  Complicating matters, Flux instances can be launched
+recursively.
+
+local URI
+^^^^^^^^^
+
+Each instance, or more properly each Flux broker within an instance, can
+be contacted via a unique local URI.  The URI corresponds to a UNIX domain
+socket and looks something like::
+
+  local:///tmp/flux-lMDa6Z/local-0
+
+In the :term:`initial program` (batch script, interactive alloc shell, or
+whatever), the FLUX_URI environment variable is set to the local URI of the rank
+0 broker.  Flux commands in the initial program, which also runs on rank 0,
+read FLUX_URI and reference the instance that started them.
+
+When running outside of an instance, FLUX_URI will not be set.  In this case,
+commands fall back to the compiled-in URI of the Flux :term:`system instance`.
+When there isn't a broker of the system instance running on the local node,
+commands fail with an error like::
+
+  ERROR: Unable to connect to Flux: broker socket /run/flux/local was not found
+
+remote URI
+^^^^^^^^^^
+
+A Flux instance also has a remote URI that looks like::
+
+  ssh://test3/tmp/flux-lMDacZ/local-0
+
+This is the local URI above with the scheme changed to "ssh" and the hostname
+"test3" prepended to the path.  Given a job ID, :man1:`flux-uri` can look up
+the remote URI:
+
+.. code-block:: console
+
+  $ flux batch -N2 --wrap sleep 120
+  ƒcbUvuHDCiB
+  $ flux uri ƒcbUvuHDCiB
+  ssh://test3/tmp/flux-gMypIR/local-0
+  $
+
+Which can be used as follows:
+
+.. _sleep_example:
+
+.. code-block:: console
+
+  $ flux batch -N2 --wrap sleep 120
+  ƒcbUvuHDCiB
+  $ FLUX_URI=$(flux uri $(flux job last)) flux submit --cc 1-5 -N2 sleep 60
+  ƒ3croSDd
+  ƒ3ctHRVy
+  ƒ3ctHRVz
+  ƒ3cumQnK
+  ƒ3cwFQ4f
+  $ FLUX_URI=$(flux uri $(flux job last)) flux jobs
+       JOBID USER     NAME       ST NTASKS NNODES     TIME INFO
+    ƒ3ctHRVy alice    sleep       S      2      2        -
+    ƒ3ctHRVz alice    sleep       S      2      2        -
+    ƒ3cumQnK alice    sleep       S      2      2        -
+    ƒ3cwFQ4f alice    sleep       S      2      2        -
+    ƒ3croSDd alice    sleep       R      2      2   6.593s test[3-4]
+
+That started a batch job with a lifetime of 120s, then submitted 5 "sleep 60"
+jobs to it, then listed the batch job's active jobs.
+
+parent URI
+^^^^^^^^^^
+
+Sometimes it's handy to direct a Flux command at the enclosing or parent
+instance of the current Flux instance.  The :man1:`flux` command driver has
+a ``--parent`` option which alters FLUX_URI to refer to the enclosing instance
+in its sub-command's environment.
+
+How would a batch job submit a cleanup job to run upon its completion?  The
+cleanup job would be submitted to the enclosing instance rather than the
+batch instance.  The batch script might do this:
+
+.. code-block:: sh
+
+  #!/bin/sh
+  batch_jobid=$(flux getattr jobid)
+  flux --parent submit --dependency afterany:$batch_jobid cleanup.sh
+
+URI resolver
+^^^^^^^^^^^^
+
+:man1:`flux-uri` and some Flux commands employ an internal URI resolver class
+that can use various tricks to find a usable remote URI for a Flux instance.
+The input is an "unresolved URI" whose scheme selects the resolver method.
+If no scheme is specified, the default is ``jobid``, thus the following commands
+are equivalent::
+
+  flux uri ƒcbUvuHDCiB
+  flux uri jobid:ƒcbUvuHDCiB
+
+A ``slurm`` scheme enables a Slurm job id to be resolved:
+
+.. code-block:: console
+
+  $ sbatch -N2 --wrap "srun flux start sleep 120"
+  Submitted batch job 1533009
+  $ flux uri slurm:1533009
+  ssh://quartz17/var/tmp/bob/flux-xBj7Cg/local-0
+
+Other schemes are available like ``pid`` and ``lsf``.
+
+flux proxy
+^^^^^^^^^^
+
+It gets a bit tedious setting FLUX_URI for every command, plus each command
+has to initiate a new connection to the remote broker which could be slow.
+:man1:`flux-proxy` establishes a connection once, and spawns a shell with a
+proxy FLUX_URI setting so that commands run within it work seamlessly with the
+remote instance.  When the shell exits, the connection is dropped.
+:man1:`flux-proxy` uses the URI resolver so its job ID argument can be an
+unresolved URI.
+
+The :ref:`example above <sleep_example>` can be simplified as follows:
+
+.. code-block:: console
+
+  $ flux batch -N2 --wrap sleep 120
+  ƒcTzRVhnrW3
+  $ flux proxy $(flux job last)
+  ƒ(s=2,d=1) $ flux submit --cc 1-5 -N2 sleep 60
+  ƒABfkxas
+  ƒABfkxat
+  ƒABhEwsD
+  ƒABiiw9Z
+  ƒABkCvRu
+  ƒ(s=2,d=1) $ flux jobs
+       JOBID USER     NAME       ST NTASKS NNODES     TIME INFO
+    ƒABfkxat bob      sleep       S      2      2        -
+    ƒABhEwsD bob      sleep       S      2      2        -
+    ƒABiiw9Z bob      sleep       S      2      2        -
+    ƒABkCvRu bob      sleep       S      2      2        -
+    ƒABfkxas bob      sleep       R      2      2   2.028s test[3-4]
+  ƒ(s=2,d=1) $ exit
+  $
+
+.. tip::
+
+  This customized bash shell prompt is neat way to maintain your bearings
+  in a Flux instance hierarchy. Add this to your ``.bashrc``:
+
+  .. code-block:: shell
+
+    if ! echo "$PS1" | grep -q FLUX; then
+      PS1=$'${FLUX_URI+\u0192(s=$(flux getattr size),d=$(flux getattr instance-level)$(which flux|grep -q src/cmd && echo ,builddir))} '${PS1}
+    fi
+
+  ``ƒ(s=2,d=1)`` says you're in a Flux instance of size 2 at instance depth 1.
+
+a proxy use case with Hydra
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:man1:`flux-proxy` can provide interactive access to Flux when the start method
+doesn't support it.  A few hints are in order for this use case:
+
+- Make the initial program print the remote URI and then sleep indefinitely.
+
+- Stop the instance with :man1:`flux-shutdown` when it is no longer needed.
+
+- Beware that the interactive proxy shell will get a SIGHUP if the instance
+  terminates while the proxy is in progress.  To avoid this, stop the instance
+  *after* exiting the proxy shell.
+
+- Note that unlike :man1:`flux-alloc`, the proxy shell runs locally, not on
+  the first node of the instance.
+
+
+With those points in mind, we can revisit the :ref:`start_hydra` example
+and tweak it to be used interactively:
+
+.. code-block:: console
+
+  $ mpiexec.hydra -f hosts -launcher ssh flux start "flux uri --remote \$FLUX_URI; sleep inf"
+  ssh://test0/tmp/flux-NCPWYE/local-0
+
+Now in another window:
+
+.. code-block:: console
+
+  $ flux proxy ssh://test0/tmp/flux-NCPWYE/local-0
+  ƒ(s=8,d=0) $ flux uptime
+   09:41:03 run 42s,  owner bob,  depth 0,  size 8
+  ƒ(s=8,d=0) $ exit
+  exit
+  $ flux shutdown --quiet ssh://test0/tmp/flux-NCPWYE/local-0
+  broker.err[0]: rc2.0: flux uri --remote $FLUX_URI; sleep inf Hangup (rc=129) 52.2s
+  $
+
+The rc2 hangup error indicates that the initial program had to be terminated
+by the shutdown sequence.  Normally that would be concerning, but it is expected
+in this situation.
diff --git a/doc/guide/internals.rst b/doc/guide/internals.rst
new file mode 100644
index 000000000000..7ad9ed75e2dc
--- /dev/null
+++ b/doc/guide/internals.rst
@@ -0,0 +1,10 @@
+Resources for Flux Developers
+=============================
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Contents:
+
+   debug
+   kvs
+   broker
diff --git a/doc/guide/kvs.rst b/doc/guide/kvs.rst
new file mode 100644
index 000000000000..3474e7af0ae3
--- /dev/null
+++ b/doc/guide/kvs.rst
@@ -0,0 +1,320 @@
+.. _kvs:
+
+###############
+Key Value Store
+###############
+
+The Flux key value store is an essential building block for Flux services.
+This page describes the current KVS design.
+
+As described in
+`SRMPDS 2014 <https://flux-framework.org/publications/Flux-SRMPDS-final.pdf>`_,
+the Flux KVS is a general purpose data store used by other Flux components.
+It stores values under a hierarchical key space with a single leader
+node and multiple caching followers. The weak consistency of its follower
+caches has the following properties, using the taxonomy from
+`Vogel 2007 <http://www.allthingsdistributed.com/2007/12/eventually_consistent.html>`_:
+
+Causal consistency
+  If process A communicates with process B that it has updated a data item
+  (passing a store version in that message), a subsequent access by process
+  B will return the updated value.
+
+Read-your-writes consistency
+  A process having updated a data item, never accesses an older value.
+
+Monotonic read consistency
+  If a process has seen a particular value for an object, any subsequent
+  accesses will never return previous values.
+
+These properties are achieved with hash trees and content-addressable storage,
+borrowing ideas from
+`ZFS <https://www.cs.hmc.edu/~rhodes/cs134/readings/The%20Zettabyte%20File%20System.pdf>`_,
+`camlistore <https://camlistore.org/>`_, and
+`git <https://git-scm.com/book/en/v2/Git-Internals-Git-Objects>`_.
+KVS values and metadata are placed in a content-addressable store,
+indexed by their hash digests, or *blobrefs*.  The content store is described
+in :doc:`RFC 10 <rfc:spec_10>` and briefly below.
+
+KVS metadata consist of directories, symbolic links, and value references,
+formatted as JSON objects.  These objects are described in
+:doc:`RFC 11 <rfc:spec_11>` and briefly below.
+
+**********
+key lookup
+**********
+
+Path components of a key each reference a directory entry in a different
+directory.  The first path component is looked up by following a root
+blobref to the root directory, finding a matching entry, and following
+that blobref to a new entry, and so on until the terminal name is reached
+and the resulting object is returned to the user.
+
+For example, if the root blobref is ``sha1-1c002dde`` and we have stored
+*a.b.c = 42*, we would look it up as follows:
+
+1. load root directory from ``sha1-1c002dde``, find *a* is at
+   ``sha1-3f2243ef``
+2. load *a* from ``sha1-3f2243ef``, find *b* is at ``sha1-023e9b2d``
+3. load *b* from ``sha1-023e9b2d``, find *c* is at ``sha1-7ff234a8``
+4. load *c* from ``sha1-7ff234a8``, and return the value: *42*.
+
+It's actually slightly more complicated than that since large directories
+and values can span multiple blobs.  Instead of always a single blobref,
+directory entries contain a *dirref* or *valref* object that contain
+arrays of blobrefs.  Following a *dirref* to the next *dir* object
+may require fetching multiple blobs to reconstruct the next object.
+Retrieving a value from a *valref* may similarly require fetching multiple
+blobs to reconstruct the final value.  Finally, as an optimization,
+small values may be included directly in a directory entry in a *val* object.
+
+The basic KVS API for looking up a value by key is described in
+:man3:`flux_kvs_lookup`.
+
+
+**********
+key update
+**********
+
+An important property of the hash tree structure is that any update
+results in a new root blobref .  Continuing the example,
+to update *a.b.c = 43*, we:
+
+1. store the value *43* to ``sha1-62302aff``
+2. update *b* to associate *c* with ``sha1-62302aff``, and
+   store *b* to ``sha1-8fe9b2c3``
+3. update *a* to associate *b* with ``sha1-8fe9b2c3``, and
+   store *a* to ``sha1-aacc76b4``
+4. update root to associate *a* with ``sha1-aacc76b4``, and
+   store root to ``sha1-033fbe92``
+5. the new root blobref is ``sha1-033fbe92``.
+
+All updates are applied first on the leader node at the root of the TBON,
+which then publishes a new root reference as a *setroot* event.  Followers keep
+consistent with the leader by switching their root reference in response to
+this event, so that all new lookups begin at the new root directory.
+
+Updates are transactional in that multiple changes can be batched up into
+a single *commit*, which either succeeds or fails entirely, without making
+intermediate states visible to users.
+
+The basic KVS API for updating keys is described in :man3:`flux_kvs_txn_create`
+and :man3:`flux_kvs_commit`.
+
+*************
+content cache
+*************
+
+As mentioned above, a distinct distributed service, used by the KVS,
+maps blobrefs to blobs, and implements a temporal cache on
+each broker rank, backed ultimately by a sqlite database on rank 0.
+
+When the KVS is performing a key lookup, tree objects are retrieved from
+the content cache by blobref.  Blobs not present in the local
+broker's content cache are faulted in from the TBON parent, recursing
+up the tree until the request can be fulfilled.  Unused content cache
+entries are expired from memory after a period of disuse.  They persist
+in the sqlite database, which does not allow blobs to be deleted.
+
+A store to the content cache is allowed to return once the entry is
+present in the rank 0 memory cache, since this ensures it can be read
+by any rank.  It must be written to sqlite before it can be expired
+from the rank 0 memory cache.
+
+The content store can be used independently, e.g. through the command line
+as described in :man1:`flux-content` or or by sending the RPCs described
+in :doc:`RFC 10 <rfc:spec_10>`.
+
+**********
+namespaces
+**********
+
+The KVS supports multiple namespaces.  The *primary namespace* is
+always available and can only be accessed by the :term:`instance owner`.
+
+The instance owner can create and destroy additional namespaces,
+and assign each an *owner* who can access the namespace, in addition
+to the instance owner which can access all namespaces.  In Flux, each job
+is set up with its own namespace, owned by the job owner, who may be a
+:term:`guest` user.
+
+Although commits are serialized on a given namespace, commits on
+distinct namespaces can progress in parallel.
+
+***********
+consistency
+***********
+
+Flux event messages are sequenced and guaranteed to be delivered in order.
+This property, and the serialization of commits on the leader node, ensure
+that monotonic read consistency is achieved.
+
+Read-your-writes consistency is obtained by piggy-backing on the commit
+response the new root blobref and its version number, a monotonic sequence.
+As the response propagates from the leader node to the sender of the commit
+through a sequence of followers, the followers update their root reference to
+at least the version in the response.  Thus, once a commit response is
+received, a lookup initiated by the caller will always return that version
+of the store or newer.  Races with the setroot event are avoided by minding
+the sequence number so root updates are never applied out of sequence.
+
+Causal consistency is available programmatically.  After a commit, the root
+blobref version can be read via :man3:`flux_kvs_commit_get_sequence`
+or :func:`flux_kvs_get_version` and passed to another rank, which can use
+:func:`flux_kvs_wait_version` to block until the follower root blobref
+reaches that version, after which the data from the commit (or newer) can
+be retrieved.  In summary:
+
+1. Process A commits data, then gets the store version :math:`V` and sends
+   it to B.
+2. Process B waits for the store version to be :math:`>= V`, then reads data.
+
+**********************
+the secret other cache
+**********************
+
+In the current KVS implementation, there is a cache of content blobs
+in the KVS module that sits in front of the content cache service.
+Cached blobs are accessible directly rather than through RPC to the
+local content cache service, except when there is a "fault",
+and then access must be suspended while the RPC completes.  This local
+cache is temporally expired like the content cache service.
+
+When a request to look up a name is handled and a needed tree object
+(for example a directory in the middle of a path) is not in cache, the
+request message is queued on a cache entry that is waiting to be filled.
+Once the cache entry has been filled, the request message queued on it is
+"restarted" meaning handled like a new request.  Since all the tree objects
+for the lookup up to that point should be in cache, it won't block until it
+reaches the next missing tree object.  Multiple requests can be queued on
+each cache entry, but only one content request is triggered regardless of
+how many consumers there are for it.  This design was an optimization for
+many requests looking up the same thing such as during a PMI exchange.
+
+***********************
+treeobj metadata format
+***********************
+
+:doc:`RFC 11 <rfc:spec_11>` defines the treeobj format, but here is a brief
+summary:
+
+A *valref* refers to opaque data in the content store (the actual data,
+not a *val* object).
+
+.. code-block:: json
+
+  { "ver":1,
+    "type":"valref",
+    "data":["sha1-aaa...","sha1-bbb..."],
+  }
+
+A *val* represents opaque data directly, base64-encoded.
+
+.. code-block:: json
+
+  { "ver":1,
+    "type":"val",
+    "data":"NDIyCg==",
+  }
+
+A *dirref* refers to a *dir* or *hdir* object that was serialized and
+stored in the content store.
+
+.. code-block:: json
+
+  { "ver":1,
+    "type":"dirref",
+    "data":["sha1-aaa...","sha1-bbb..."],
+  }
+
+A *dir* is a dictionary mapping keys to any of the tree object types.
+
+.. code-block:: json
+
+  { "ver":1,
+    "type":"dir",
+    "data":{
+       "a":{"ver":1,"type":"dirref","data":["sha1-aaa"]},
+       "b":{"ver":1,"type":"val","data":"NDIyCg=="},
+       "c":{"ver":1,"type":"valref","data":["sha1-aaa","sha1-bbb"]},
+       "d":{"ver":1,"type":"dir","data":{},
+    }
+  }
+
+A *symlink* is a symbolic pointer to a another KVS key, which may
+or may not be fully qualified.
+
+.. code-block:: json
+
+  { "ver":1,
+    "type":"symlink",
+    "data":"a.aa",
+  }
+
+*****
+watch
+*****
+
+Any key (including the root directory) can be "watched", such that a user
+receives responses for each key change.
+
+Watch is implemented in a separate ``kvs-watch`` module.
+
+Each time a setroot event indicates that something has changed, the
+``kvs-watch`` module looks up the watched keys to see if they have changed.
+
+Because watched keys have to be looked up from the root on *every* KVS
+commit, watching a key has a high overhead, but some relief is given by
+the optimization of including a list of changed keys in the setroot event,
+described below.
+
+The API for watching KVS keys is described in :man3:`flux_kvs_lookup`.
+Basically a special flag is added and responses are received each time
+the value changes or the request is canceled or the caller disconnects.
+
+*************
+optimizations
+*************
+
+piggyback root directory on setroot event
+=========================================
+
+The KVS design calls for publication of a setroot event each time
+the root blobref changes.
+
+Since most lookups will need to start with the root directory
+(the object pointed to by the root blobref), the new root directory will
+most likely not be in cache, and any watched keys will be looked up on
+every commit, the root directory object is made part of the setroot event
+so this lookup is avoided.
+
+commit merging
+==============
+
+Since all updates are serialized through the rank 0 commit process,
+this area of the code is an obvious target for optimization.
+One such optimization is commit merging.
+
+*commit merging* is implemented by queuing parsed commit requests
+during the main part of the reactor loop, and processing them only
+when the reactor has no events (request messages) pending, using
+the prepare/check/idle watcher pattern.
+
+This minimizes the work that is duplicated for each commit:  creating
+a temporary in-memory json object to work on, and updating the root
+reference and top level directories when the commit is finalized.
+This saves time, but also reduces the number of intermediate objects
+written to the content cache.
+
+A problem arises when intermediate values for a key are overwritten
+in merged commits, and a "watcher" needs to see each value to function
+properly, such as to synchronize a state machine.  To solve this,
+a ``KVS_NO_MERGE`` flag may be added to :func:`flux_kvs_commit`, which
+indicates that the merge should not be subject to this optimization.
+
+piggyback list of changed keys on setroot event
+===============================================
+
+To streamline the KVS watch implementation, a list of changed keys is
+included in the kvs setroot event.  That way the entire list of watched keys
+does not need to be looked up every time there is a new root.
diff --git a/doc/guide/start.rst b/doc/guide/start.rst
new file mode 100644
index 000000000000..2a2c04ffd41d
--- /dev/null
+++ b/doc/guide/start.rst
@@ -0,0 +1,496 @@
+Starting a Flux Instance
+========================
+
+A *Flux instance* is a self-contained workload manager.  It is so easy to
+start a new Flux instance that we often ask ourselves "can we just start a
+new instance?" before we consider extending Flux to solve a new problem.
+
+In a broad sense, Flux is a workload manager on a par with Slurm, Torque,
+LSF, or similar systems.  It distinguishes itself by being able to be started
+standalone, as a job in another system, or a job in *itself* (recursively).
+It doesn't require elevated privileges to do so for one user at a time, so
+anyone can download Flux and immediately use it productively.  This opens up
+possibilities for using Flux as a portability layer or "step scheduler" in
+workflow systems, and for cooking up divide and conquer solutions to
+resource and workload problems.
+
+A Flux instance consists of a set of services running on top of a distributed
+message broker.  To start Flux, we are just starting one or more brokers in
+parallel, as a *parallel program* if you will.
+
+:man1:`flux-start` starts a Flux instance, whose life cycle consists of three
+phases:
+
+#. Initialize
+#. The rank 0 broker runs the :term:`initial program` to completion
+#. Finalize
+
+By default, the initial program is an interactive shell.  If :man1:`flux-start`
+is passed free arguments, they are interpreted as commands to run
+non-interactively instead.  Either way, when the initial program terminates,
+the instance terminates.
+
+.. _test_instance:
+
+Starting a Test Instance
+------------------------
+
+A standalone Flux instance can be started right away, say on your laptop, or
+on a login node of your institution's big cluster.  This is a convenient way to
+begin experimenting.
+
+.. code-block:: console
+
+  $ flux start --test-size=3
+  $ flux uptime
+   23:50:28 run 1.3m,  owner alice,  depth 0,  size 3
+  $ flux resource info
+  3 Nodes, 12 Cores, 0 GPUs
+  $ flux run --label-io -N3 hostname
+  2: test0
+  1: test0
+  0: test0
+  $ exit
+  $
+
+What just happened?  Hint: the process tree looks like this:
+
+.. code-block:: console
+
+  flux-start─┬─flux-broker-0───bash
+             └─flux-broker-1
+             └─flux-broker-2
+
+Let's break it down:
+
+#. :man1:`flux-start` launches three Flux brokers on the local system,
+   ranked 0, 1, and 2.
+
+#. The brokers find their rank and exchange peer network addresses and public
+   keys using a `PMI server <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_13.html>`_
+   offered by the start command, which is the process parent of the brokers.
+   The PMI server is only offered by :man1:`flux-start` when the
+   ``--test-size=N`` option is used.
+
+#. After the brokers are connected and have completed initialization, the
+   rank 0 broker spawns the initial program, an interactive shell that has its
+   environment set up so that Flux commands run from it will connect to the
+   new Flux instance.  The prompt right after the start command is from this
+   new shell.
+
+#. We run :man1:`flux-uptime` to get some basic info about the system and note
+   that it consists of three brokers, and that it has no parent instance, hence
+   a "depth" of zero.
+
+#. We run :man1:`flux-resource` *list* to show the instance's resource
+   inventory.  In this contrived test instance, we have three brokers running
+   on one node, but each broker assumes that it is discovering the resources
+   of a separate node, so while there is really 1 node, 4 cores, the
+   resource inventory contains 3 nodes, 12 cores.  See below for more on that.
+
+#. :man1:`flux-run` launches the :linux:man1:`hostname` command in
+   parallel across the three "nodes" and waits for it to complete.
+   We observe that each reports the same hostname.
+
+#. Finally, we exit the interactive shell.  Since that was the initial program,
+   the instance exits.  The next prompt is the original shell, before we ran
+   the start command.
+
+It is convenient to be able to start a Flux instance like this in pretty much
+any environment, but to reinforce what was alluded to above, the main
+caveat of a test instance is that it doesn't actually have exclusive access
+to resources.  `HWLOC <https://www.open-mpi.org/projects/hwloc/>`_ is used to
+discover what hardware is out there on each broker, but the broker has no way
+to claim it exclusively.  A test instance on a shared login node provides the
+illusion of exclusive access to jobs run within it, but someone else running
+a test instance on the same node may be doing the same thing.  Furthermore,
+when the test size is greater than one, the actual hardware resources may be
+oversubscribed within the single instance.
+
+Let's create a script to run as the initial program in place of the interactive
+shell.  Call this script ``workload.sh``.  It runs the same commands as were
+run above, in a way that works on any size instance.
+
+.. code-block:: shell
+
+  #!/bin/sh
+  flux resource info
+  flux uptime
+  NNODES=$(flux resource list -no {nnodes})
+  flux run --label-io -N $NNODES hostname
+
+When we run this way, we get the same result as above, but the user doesn't
+have to provide any input while the instance is running:
+
+.. code-block:: console
+
+  $ flux start --test-size=3 ./workload.sh
+  3 Nodes, 12 Cores, 0 GPUs
+   10:00:30 run 2.9s,  owner alice,  depth 0,  size 3
+  0: test0
+  2: test0
+  1: test0
+  $
+
+This is how many of Flux-core's tests work.  The initial program is a test
+script run under a test instance started by the test suite.
+
+Starting with Slurm
+-------------------
+
+When :man1:`flux-start` is run without the ``--test-size=N`` option, it
+simply execs the broker rather than sticking around to provide PMI service.
+The broker uses its PMI client to determine its place in a parallel program,
+or if PMI is not found, it runs as a singleton.
+
+So to start a Flux instance in a foreign resource manager, we just run
+:man1:`flux-start` in parallel as though it were an MPI job, usually with
+one broker per node in the foreign allocation.
+
+.. note::
+  Slurm has a few different options that control the parallel environment
+  for jobs.  If your site has not configured ``MpiDefault=pmi2``, then it may
+  be necessary to run flux with the srun ``--mpi=pmi2`` option.  If a parallel
+  launch of Flux results in multiple singletons, e.g. reporting 1 node when
+  more were expected, this may help.
+
+In Slurm, we can obtain an allocation with `salloc(1)
+<https://slurm.schedmd.com/salloc.html>`_.  From the interactive shell
+it spawns, we use `srun(1) <https://slurm.schedmd.com/srun.html>`_
+to start a Flux instance.  The ``--pty`` option gives Flux's interactive
+initial program a proper terminal.
+
+.. code-block:: console
+
+  $ salloc -N2
+  salloc: Pending job allocation 1505790
+  salloc: job 1505790 queued and waiting for resources
+  salloc: job 1505790 has been allocated resources
+  salloc: Granted job allocation 1505790
+  salloc: Waiting for resource configuration
+  salloc: Nodes quartz[2257-2258] are ready for job
+  $ srun -N2 --pty flux start
+  $ flux uptime
+   10:04:05 run 6.6s,  owner alice,  depth 0,  size 2
+  $ flux resource info
+  2 Nodes, 72 Cores, 0 GPUs
+  $ flux run -N2 hostname
+  quartz2257
+  quartz2258
+  $ exit
+  $ exit
+  salloc: Relinquishing job allocation 1505790
+  salloc: Job allocation 1505790 has been revoked.
+  $
+
+After typing the same two Flux commands that were demonstrated in
+:ref:`test_instance`, we exit the Flux interactive shell, then the
+Slurm one, which gives up the Slurm allocation.
+
+Conceptually, what's going on here is Slurm has granted a resource
+allocation to Flux, which then parcels it out to the jobs submitted to it.
+
+The above can be accomplished a little more succinctly by just running
+srun directly.  Then the session looks like this:
+
+.. code-block:: console
+
+  $ srun -N2 --pty flux start
+  srun: job 1505791 queued and waiting for resources
+  srun: job 1505791 has been allocated resources
+  $ flux uptime
+   10:05:51 run 4.7s,  owner alice,  depth 0,  size 2
+  $ flux resource info
+  2 Nodes, 72 Cores, 0 GPUs
+  $ flux run --label-io -N2 hostname
+  0: quartz2257
+  1: quartz2258
+  $ exit
+  $
+
+To run non-interactively, we can drop the ``--pty`` option and use our
+workload script for the initial program:
+
+.. code-block:: console
+
+  $ srun -N2 flux start ./workload.sh
+  srun: job 1505795 queued and waiting for resources
+  srun: job 1505795 has been allocated resources
+   10:07:12 run 4.6s,  owner alice,  depth 0,  size 2
+  2 Nodes, 72 Cores, 0 GPUs
+  0: quartz25
+  1: quartz26
+
+Finally, Slurm batch execution of a Flux workload can be accomplished
+by wrapping the srun command in a batch script we will name ``batch.sh``:
+
+.. code-block:: shell
+
+  #!/bin/sh
+  #SBATCH -N2
+  srun flux start ./workload.sh
+
+Whee!
+
+.. code-block:: console
+
+  $ sbatch ./batch.sh
+  Submitted batch job 1505846
+  $ cat slurm-1505846.out
+   10:09:57 run 4.8s,  owner alice,  depth 0,  size 2
+  2 Nodes, 72 Cores, 0 GPUs
+  0: quartz47
+  1: quartz48
+  $
+
+Inserting Flux between Slurm and a real workload script or workflow executor
+could have some advantages, such as:
+
+- Workload scripts that use Flux commands or the Flux python API can be made
+  portable to other systems that don't use Slurm as their primary resource
+  manager, as long as they can launch a Flux instance.
+
+- High throughput workloads are less of a burden on the Slurm controller
+  and may run faster compared to the same workload run with job steps, since
+  `Step Allocation <https://slurm.schedmd.com/job_launch.html#step_allocation>`_
+  involves an RPC to the Slurm controller.
+
+- When combined with the `flux-sched project
+  <https://github.com/flux-framework/flux-sched>`_, Flux offers more
+  sophisticated step scheduling.
+
+- A Flux instance may be tailored by the instance owner to meet the specific
+  demands of the workload.  For example, the scheduler algorithm may be
+  changed.
+
+To be clear, Flux does not have "steps" per se, only jobs.   We don't need them
+since Flux batch jobs are just regular Flux jobs that happen to be independent
+Flux instances, and Flux instances have the capability to run more Flux jobs.
+
+Since Slurm runs on many of the largest HPC systems, the capability to launch
+Flux instances under Slurm provided early opportunities to work on Flux
+portability and scalability on those systems without requiring much buy-in from
+the system owners, other than permission to run there.
+
+Starting with Flux
+------------------
+
+Flux can run parallel programs, and a Flux instance can be run as one, so Flux
+can run Flux.  No surprise there.  What does that look like and how is it
+leveraged to provide expected workload manager abstractions?
+
+First let's try our workload commands in the "outer" Flux instance, in this
+case a standalone :term:`system instance`, although this section applies to all
+Flux instances.
+
+.. code-block:: console
+
+  $ flux uptime
+   10:13:27 run 23d,  owner flux,  depth 0,  size 101,  7 drained,  8 offline
+  $ flux resource info
+  98 Nodes, 392 Cores, 0 GPUs
+  $ flux run -N2 hostname
+  $ flux run --label-io -N2 hostname
+  0: fluke6
+  1: fluke7
+
+The instance depth is zero since this is the system instance.  We also get a
+heads up about some unavailable nodes, and note that the instance owner is
+the ``flux`` user, but otherwise this looks like before.  Now, an interactive
+Flux allocation:
+
+.. code-block:: console
+
+  $ flux alloc -N2
+  $ flux uptime
+   10:16:58 run 7.3s,  owner alice,  depth 1,  size 2
+  $ flux resource info
+  2 Nodes, 8 Cores, 0 GPUs
+  $ flux run -N2 hostname
+  fluke6
+  fluke7
+  $ exit
+  $
+
+The :man1:`flux-alloc` command spawns a new Flux instance in the allocation
+automatically.  The instance depth is now 1.  This is an example of how Flux's
+recursive launch property simplifies its design:  instead of requiring
+specialized code to support interactive allocations, Flux just transfers the
+requested resources to a new instance that runs for the life of its initial
+program, the interactive shell.
+
+As a side note, while :man1:`flux-alloc` provides an interface that is
+convenient as well as familiar to Slurm users, we could have accomplished
+the same thing with :man1:`flux-run` and :man1:`flux-start`:
+
+.. code-block:: console
+
+  $ flux run -N2 -o pty.interactive flux start
+  $ flux uptime
+   10:19:27 run 8.3s,  owner alice,  depth 1,  size 2
+  $ flux resource info
+  2 Nodes, 8 Cores, 0 GPUs
+  $ flux run --label-io -N2 hostname
+  0: fluke6
+  1: fluke7
+  $ exit
+  [detached: session exiting]
+  $
+
+Now let's try batch:
+
+.. code-block:: console
+
+  $ flux batch -N2 ./workload.sh
+  f2SoPsfdA7Ub
+  $ cat flux-f2SoPsfdA7Ub.out
+   10:25:05 run 4.8s,  owner alice,  depth 1,  size 2
+  2 Nodes, 8 Cores, 0 GPUs
+  0: fluke6
+  1: fluke7
+
+Like the allocation example, the :man1:`flux-batch` command spawns a new Flux
+instance that runs for the duration of its initial program, the batch script
+``workload.sh``.  The Flux design does not require job steps or a specialized
+step scheduler because of the recursive launch property.
+
+Although :man1:`flux-batch` has other nice features like support for batch
+directives and copying of the batch script at submission, a similar result
+can be obtained for this simple example with :man1:`flux-submit` and
+:man1:`flux-start`:
+
+.. code-block:: console
+
+  $ flux submit --output=flux-{{id}}.out -N2 flux start ./workload.sh
+  f2SoiFq3N5Jo
+  $ cat flux-f2SoiFq3N5Jo.out
+   11:05:24 run 4.7s,  owner alice,  depth 1,  size 2
+  2 Nodes, 8 Cores, 0 GPUs
+  0: fluke6
+  1: fluke7
+
+.. _start_hydra:
+
+Starting with Hydra
+-------------------
+
+MPICH `hydra <https://github.com/pmodels/mpich/blob/main/doc/wiki/how_to/Using_the_Hydra_Process_Manager.md>`_
+is an MPI launcher with PMI support that can start a Flux instance across
+multiple nodes using ssh.  It is sometimes just the right tool needed to get
+a Flux instance going when nothing else will work.
+
+Given a ``hosts`` file containing:
+
+::
+
+  test0
+  test1
+  test2
+  test3
+  test4
+  test5
+  test6
+  test7
+
+And assuming passwordless ssh works for those nodes, one can run:
+
+.. code-block:: console
+
+  $ mpiexec.hydra -launcher ssh -f hosts flux start ./workload.sh
+   23:23:14 run 3.9s,  owner alice,  depth 0,  size 8
+  8 Nodes, 32 Cores, 0 GPUs
+  0: test0
+  2: test2
+  1: test1
+  5: test5
+  6: test6
+  7: test7
+  4: test4
+  3: test3
+
+Starting with Static Configuration
+----------------------------------
+
+In the methods described above, Flux brokers determine their ranks and
+exchange peer network addresses and public keys using PMI.  In situations
+where PMI is unavailable or inappropriate, the same information can be loaded
+from static configuration files.  As an example, we'll start Flux on the same
+eight nodes from :ref:`start_hydra`.
+
+Generate a shared CURVE certificate that will be used to secure the overlay
+network:
+
+.. code-block:: console
+
+  $ mkdir test
+  $ cd test
+  $ flux keygen test.cert
+  $
+
+Create a TOML config file as described in :man5:`flux-config-bootstrap`.
+We'll call it ``test.toml``.  For this experiment, we can place it in the
+same directory as ``test.cert``:
+
+.. code-block:: toml
+
+  [bootstrap]
+  curve_cert = "/home/bob/test/test.cert"
+
+  hosts = [
+    { host="test0", bind="tcp://eth0:8060", connect="tcp://test0:8060" },
+    { host="test1", bind="tcp://eth0:8060", connect="tcp://test1:8060" },
+    { host="test2", bind="tcp://eth0:8060", connect="tcp://test2:8060" },
+    { host="test3", bind="tcp://eth0:8060", connect="tcp://test3:8060" },
+    { host="test4", bind="tcp://eth0:8060", connect="tcp://test4:8060" },
+    { host="test5", bind="tcp://eth0:8060", connect="tcp://test5:8060" },
+    { host="test6", bind="tcp://eth0:8060", connect="tcp://test6:8060" },
+    { host="test7", bind="tcp://eth0:8060", connect="tcp://test7:8060" },
+  ]
+
+If necessary, replicate the ``test`` directory across the eight nodes.
+
+Finally, start a broker on each node.  You can use any technique you like,
+but keep in mind the brokers will not exit until the initial program completes,
+and the initial program will not start until all the brokers are online.
+We use :linux:man1:`pdsh` to start the brokers in parallel and capture any
+errors they print to the standard error stream:
+
+.. code-block:: console
+
+  $ pdsh -N -w test[0-7] flux start -o,--config-path=/home/bob/test/test.toml ./workload.sh
+  8 Nodes, 32 Cores, 0 GPUs
+   00:03:31 run 2.5s,  owner bob,  depth 0,  size 8
+  0: test0
+  6: test6
+  7: test7
+  4: test4
+  5: test5
+  1: test1
+  3: test3
+  2: test2
+
+This method of starting Flux is used when `systemd <https://systemd.io/>`_
+starts brokers belonging to a Flux system instance.  This is described in the
+`Flux Administration Guide
+<https://flux-framework.readthedocs.io/en/latest/guides/admin-guide.html>`_.
+Another example is the `Flux Operator
+<https://flux-framework.org/flux-operator/>`_, which uses dynamically generated
+configuration to bootstrap Flux in a Kubernetes cluster.
+
+When compared with PMI bootstrap, this method of starting Flux has the
+following drawbacks:
+
+- Managing and sharing the instance-specific files is an extra step.
+
+- The CURVE certificate must be protected from disclosure to parties other
+  than the Flux instance owner, or an attacker may attach and impersonate
+  the owner.
+
+- The use of a fixed port number in the configuration raises the possibility
+  that the port may already be in use on one of the nodes, especially if the
+  same configuration file is reused.  If a broker cannot bind to its port,
+  it will fail to start.
+
+- If a broker fails to start, the ``pdsh`` command line above may hang.
+  Options are available to deal with this like the ``broker.quorum`` setting
+  described in :man7:`flux-broker-attributes`.
diff --git a/doc/guide/support.rst b/doc/guide/support.rst
new file mode 100644
index 000000000000..8b99605d1097
--- /dev/null
+++ b/doc/guide/support.rst
@@ -0,0 +1,57 @@
+License and Support
+===================
+
+License
+-------
+
+.. figure:: images/lgplv3-147x51.png
+   :alt: LGPL-3.0 logo
+   :align: right
+
+Flux-core is licensed under `LGPL-3.0
+<https://www.gnu.org/licenses/lgpl-3.0.en.html>`_ which means project forks
+must retain the same license and follow the license's requirements for making
+source code available to end users.
+
+Software that merely uses flux-core's APIs may be independently licensed.
+
+Support
+-------
+
+All flux-core issues and changes are proposed and discussed on `Github
+<https://github.com/flux-framework/flux-core>`_.  The team values
+openness and inclusivity and encourages collaboration.  Engage with us!
+
+If you have a problem that might be a flux-core bug, please first search
+existing issues, then open a new `Github issue
+<https://github.com/flux-framework/flux-core/issues>`_ if necessary.
+Be sure to include relevant context and especially the flux version that
+is exhibiting the problem:
+
+.. code-block:: console
+
+  $ flux version
+  commands:    		0.54.0
+  libflux-core:		0.54.0
+  libflux-security:	0.10.0
+  build-options:	+systemd+hwloc==2.4.0+zmq==4.3.4
+
+`Github discussions <https://github.com/flux-framework/flux-core/discussions>`_
+are sometimes useful too.
+
+Development Commitment
+----------------------
+
+At this time, Lawrence Livermore National Laboratory (LLNL) has a team of
+four full time employees working on flux-core, and several additional employees
+and collaborators working on the broader Flux Framework and related research
+areas.  LLNL is committed to making Flux a production quality tool for HPC
+workload management, and has sustained its investment for over a decade thus
+far.
+
+Flux is part of the `TOSS <https://hpc.llnl.gov/documentation/toss>`_
+operating system and thus is supported across multiple U.S. national laboratory
+facilities.
+
+The Flux team welcomes all users and contributors to the Flux community and
+will do its best enable the community to grow and thrive.
diff --git a/doc/index.rst b/doc/index.rst
new file mode 100644
index 000000000000..b323c14ed859
--- /dev/null
+++ b/doc/index.rst
@@ -0,0 +1,108 @@
+Flux Core
+=========
+
+The flux-core project provides the distributed messaging capability and core
+services and APIs of the Flux resource manager framework.  Flux-core has been
+under active development since 2012, primarily at Lawrence Livermore National
+Laboratory.  The flux-core project implements several of Flux's innovations:
+
+Recursive launch
+  A Flux instance can be launched standalone for testing, by systemd as a
+  system service, or as a parallel job under most HPC resource managers and
+  launchers, including Flux itself.  A Flux batch job or interactive allocation
+  is an independent Flux instance running on a subset of the parent's
+  resources.  Resources can be recursively subdivided ad infinitum for
+  performance or isolation.
+
+Small Security Footprint
+  Flux-core does not contain any code that runs as root.  A Flux instance
+  only requires a setuid helper (provided by the flux-security project) in
+  multi-user configurations.  Flux can be deployed for single user use without
+  administrator access.
+
+Reactive Messaging
+  Flux components are primarily single threads that communicate only by
+  exchanging messages through message brokers.  C and Python APIs enable
+  the creation of robust, distributed services that use this paradigm.
+
+Related Information
+===================
+
+Although able to function on its own, flux-core's capability is enhanced when
+combined with other framework projects.  Those wishing to deploy Flux as a
+full-featured resource management solution or understand Flux in a broader
+context may benefit from the following material:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Description
+     - Links
+
+   * - Main Flux Documentation Pages
+     - `Docs <https://flux-framework.readthedocs.io/en/latest/index.html>`_
+
+   * - Flux as the primary resource manager on an HPC cluster
+     - `Flux Administrator's Guide <https://flux-framework.readthedocs.io/projects/flux-core/en/latest/guide/admin.html>`_
+
+   * - Flux Request for Comments Specifications
+     - `RFC <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/index.html>`_
+
+For high performance computing users, the key framework projects are
+currently:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Framework Project
+     - Links
+
+   * - flux-core - Base Flux framework
+     - `Github <https://github.com/flux-framework/flux-core>`_
+
+       `Docs <https://flux-framework.readthedocs.io/projects/flux-core>`_ - You are already here!
+
+   * - flux-sched - Fluxion graph-based scheduler
+
+       Required for complex resource types and scheduling beyond FIFO
+
+     - `Github <https://github.com/flux-framework/flux-sched>`_
+
+       `Docs <https://flux-framework.readthedocs.io/projects/flux-sched>`_
+
+   * - flux-security - Job request signatures and setuid helper
+
+       Required for multi-user Flux instances
+
+     - `Github <https://github.com/flux-framework/flux-security>`_
+
+       `Docs <https://flux-framework.readthedocs.io/projects/flux-security>`_
+
+   * - flux-accounting - Bank accounting and fair share priority
+     - `Github <https://github.com/flux-framework/flux-accounting>`_
+
+       `Flux Accounting Guide <https://flux-framework.readthedocs.io/projects/flux-accounting/en/latest/guide/accounting-guide.html>`_
+
+   * - flux-pmix - OpenMPI support
+     - `Github <https://github.com/flux-framework/flux-pmix>`_
+
+   * - flux-coral2 - Cray MPI and rabbit storage support
+     - `Github <https://github.com/flux-framework/flux-coral2>`_
+
+       `CORAL2: Flux on Cray Shasta <https://flux-framework.readthedocs.io/en/latest/tutorials/lab/coral2.html>`_
+
+Table of Contents
+=================
+
+.. toctree::
+   :maxdepth: 2
+
+   guide/build
+   guide/support
+   guide/start
+   guide/interact
+   guide/admin
+   guide/glossary
+   index_man
+   python/index
+   guide/internals
diff --git a/doc/index_man.rst b/doc/index_man.rst
new file mode 100644
index 000000000000..3e5eb89162c4
--- /dev/null
+++ b/doc/index_man.rst
@@ -0,0 +1,12 @@
+.. _man-pages:
+
+Manual Pages
+============
+
+.. toctree::
+   :maxdepth: 1
+
+   man1/index
+   man3/index
+   man5/index
+   man7/index
diff --git a/doc/man1/COPYRIGHT.adoc b/doc/man1/COPYRIGHT.adoc
deleted file mode 100644
index 6b2304f95801..000000000000
--- a/doc/man1/COPYRIGHT.adoc
+++ /dev/null
@@ -1,4 +0,0 @@
-Copyright 2014 Lawrence Livermore National Security, LLC
-and Flux developers.
-
-SPDX-License-Identifier: LGPL-3.0
diff --git a/doc/man1/Makefile.am b/doc/man1/Makefile.am
deleted file mode 100644
index c4a1d66ba462..000000000000
--- a/doc/man1/Makefile.am
+++ /dev/null
@@ -1,63 +0,0 @@
-MAN1_FILES = $(MAN1_FILES_PRIMARY) $(MAN1_FILES_SECONDARY)
-
-
-MAN1_FILES_PRIMARY = \
-	flux.1 \
-	flux-broker.1 \
-	flux-kvs.1 \
-	flux-keygen.1 \
-	flux-logger.1 \
-	flux-ping.1 \
-	flux-start.1 \
-	flux-module.1 \
-	flux-exec.1 \
-	flux-env.1 \
-	flux-getattr.1 \
-	flux-dmesg.1 \
-	flux-content.1 \
-	flux-hwloc.1 \
-	flux-proxy.1 \
-	flux-cron.1 \
-	flux-user.1 \
-	flux-event.1 \
-	flux-mini.1 \
-	flux-version.1 \
-	flux-jobs.1
-
-# These files are generated as roff .so includes of a primary page.
-# A2X handles this automatically if mentioned in NAME section
-MAN1_FILES_SECONDARY = \
-	flux-setattr.1 \
-	flux-lsattr.1
-
-
-ADOC_FILES  = $(MAN1_FILES_PRIMARY:%.1=%.adoc)
-XML_FILES   = $(MAN1_FILES_PRIMARY:%.1=%.xml)
-
-if ENABLE_DOCS
-dist_man_MANS = $(MAN1_FILES)
-$(MAN1_FILES): COPYRIGHT.adoc NODESET.adoc
-endif
-
-SUFFIXES = .adoc .1
-
-flux-setattr.1: flux-getattr.1
-flux-lsattr.1: flux-getattr.1
-
-STDERR_DEVNULL = $(stderr_devnull_$(V))
-stderr_devnull_ =  $(stderr_devnull_$(AM_DEFAULT_VERBOSITY))
-stderr_devnull_0 = 2>/dev/null
-
-.adoc.1:
-	$(AM_V_GEN)$(ADOC) --attribute mansource=$(PACKAGE_NAME) \
-	--attribute manversion=$(PACKAGE_VERSION) \
-	--attribute manmanual="Flux Command Reference" \
-	--destination-dir $(builddir) \
-	--doctype manpage $(ADOC_FORMAT_OPT) manpage $< $(STDERR_DEVNULL)
-
-EXTRA_DIST = \
-	$(ADOC_FILES) \
-	COPYRIGHT.adoc \
-	NODESET.adoc
-
-CLEANFILES = $(MAN1_FILES) $(XML_FILES)
diff --git a/doc/man1/NODESET.adoc b/doc/man1/NODESET.adoc
deleted file mode 100644
index a86142172e2c..000000000000
--- a/doc/man1/NODESET.adoc
+++ /dev/null
@@ -1,14 +0,0 @@
-A NODESET is a comma separated list of integer ranks.  Ranks may be
-listed individually or as a range in the form _l-k_ where l < k.
-
-Some examples of nodesets. 
-
-[horizontal]
-``1'':: rank 1
-``0-3'':: ranks 0, 1, 2, and 3 listed in a range
-``0,1,2,3'':: ranks 0, 1, 2, and 3 listed individually
-``2,5'':: ranks 2 and 5
-``2,4-5'':: ranks 2, 4, and 5
-
-As a special case, the string ``_all_'' can be specified to indicate every
-rank available in the flux instance.
diff --git a/doc/man1/common/experimental.rst b/doc/man1/common/experimental.rst
new file mode 100644
index 000000000000..00be899898a2
--- /dev/null
+++ b/doc/man1/common/experimental.rst
@@ -0,0 +1,4 @@
+Experimental Flux features and interfaces are made available for evaluation
+only and may change or be removed without notice.
+
+Feedback is welcome.  Please use the flux-core project Github issue tracker.
diff --git a/doc/man1/common/job-constraints.rst b/doc/man1/common/job-constraints.rst
new file mode 100644
index 000000000000..0f8c280feb0b
--- /dev/null
+++ b/doc/man1/common/job-constraints.rst
@@ -0,0 +1,71 @@
+.. option:: --requires=CONSTRAINT
+
+   Specify a set of allowable properties and other attributes to consider
+   when matching resources for a job. The **CONSTRAINT** is expressed in
+   a simple syntax described in RFC 35 (Constraint Query Syntax) which is
+   then converted into a JSON object compliant with RFC 31 (Job Constraints
+   Specification).
+
+   A constraint query string is formed by a series of terms.
+
+   A term has the form ``operator:operand``, e.g. ``hosts:compute[1-10]``.
+
+   Terms may optionally be joined with boolean operators and parenthesis to
+   allow the formation of more complex constraints or queries.
+
+   Boolean operators include logical AND (``&``, ``&&``, or ``and``), logical OR
+   (``|``, ``||``, or ``or``), and logical negation (``not``).
+
+   Terms separated by whitespace are joined with logical AND by default.
+
+   Quoting of terms is supported to allow whitespace and other reserved
+   characters in operand, e.g. ``foo:'this is args'``.
+
+   Negation is supported in front of terms such that ``-op:operand`` is
+   equivalent to ``not op:operand``. Negation is not supported in front of
+   general expressions, e.g. ``-(a|b)`` is a syntax error.
+
+   The full specification of Constraint Query Syntax can be found in RFC 35.
+
+   Currently, :option:`--requires` supports the following operators:
+
+   properties
+     Require the set of specified properties. Properties may be
+     comma-separated, in which case all specified properties are required.
+     As a convenience, if a property starts with ``^`` then a matching
+     resource must not have the specified property. In these commands,
+     the ``properties`` operator is the default, so that ``a,b`` is equivalent
+     to ``properties:a,b``.
+
+   hostlist
+     Require matching resources to  be in the specified hostlist (in RFC
+     29 format). ``host`` or ``hosts`` is also accepted.
+
+   ranks
+     Require matching resources to be on the specified broker ranks in
+     RFC 22 Idset String Representation.
+
+   Examples:
+
+   ``a b c``, ``a&b&c``, or ``a,b,c``
+      Require properties a and b and c.
+
+   ``a|b|c``, or ``a or b or c``
+      Require property a or b or c.
+
+   ``(a and b) or (b and c)``
+      Require properties a and b or b and c.
+
+   ``b|-c``, ``b or not c``
+      Require property b or not c.
+
+   ``host:fluke[1-5]``
+      Require host in fluke1 through fluke5.
+
+   ``-host:fluke7``
+      Exclude host fluke7.
+
+   ``rank:0``
+      Require broker rank 0.
+
+
diff --git a/doc/man1/common/job-dependencies.rst b/doc/man1/common/job-dependencies.rst
new file mode 100644
index 000000000000..09007c68e3e9
--- /dev/null
+++ b/doc/man1/common/job-dependencies.rst
@@ -0,0 +1,71 @@
+.. note::
+   Flux supports a simple but powerful job dependency specification in jobspec.
+   See Flux Framework RFC 26 for more detailed information about the generic
+   dependency specification.
+
+Dependencies may be specified on the command line using the following options:
+
+.. option:: --dependency=URI
+
+   Specify a dependency of the submitted job using RFC 26 dependency URI
+   format. The URI format is **SCHEME:VALUE[?key=val[&key=val...]]**.
+   The URI will be converted into RFC 26 JSON object form and appended to
+   the jobspec ``attributes.system.dependencies`` array. If the current
+   Flux instance does not support dependency scheme *SCHEME*, then the
+   submitted job will be rejected with an error message indicating this
+   fact.
+
+   The :option:`--dependency` option may be specified multiple times. Each use
+   appends a new dependency object to the ``attributes.system.dependencies``
+   array.
+
+The following dependency schemes are built-in:
+
+.. note::
+   The ``after*`` dependency schemes listed below all require that the
+   target JOBID be currently active or in the job manager's inactive job
+   cache. If a target JOBID has been purged by the time the dependent job
+   has been submitted, then the submission will be rejected with an error
+   that the target job cannot be found.
+
+after:JOBID
+   This dependency is satisfied after JOBID starts.
+
+afterany:JOBID
+   This dependency is satisfied after JOBID enters the INACTIVE state,
+   regardless of the result
+
+afterok:JOBID
+   This dependency is satisfied after JOBID enters the INACTIVE state
+   with a successful result.
+
+afternotok:JOBID
+   This dependency is satisfied after JOBID enters the INACTIVE state
+   with an unsuccessful result.
+
+begin-time:TIMESTAMP
+   This dependency is satisfied after TIMESTAMP, which is specified in
+   floating point seconds since the UNIX epoch. See the ``--begin-time``
+   option below for a more user-friendly interface to the ``begin-time``
+   dependency.
+
+In any of the above ``after*`` cases, if it is determined that the
+dependency cannot be satisfied (e.g. a job fails due to an exception
+with afterok), then a fatal exception of type=dependency is raised
+on the current job. An example of submitting a job, getting the Flux
+job identifier, and then submitting a job that depends on it (meaning
+it will wait for completion before starting) is provided below:
+
+.. code-block:: console
+  
+    # Do some long work
+    jobid=$(flux submit -N2 sleep 200)
+
+    # Submit the dependent job
+    flux submit --dependency=afterany:$jobid /bin/bash my-script.sh
+
+    # Look at your queue
+    flux jobs -a 
+
+    # Block and watch output
+    flux job attach $(flux job last)
diff --git a/doc/man1/common/job-env-rules.rst b/doc/man1/common/job-env-rules.rst
new file mode 100644
index 000000000000..acfa0209e50e
--- /dev/null
+++ b/doc/man1/common/job-env-rules.rst
@@ -0,0 +1,86 @@
+The `ENVIRONMENT`_ options allow control of the environment exported to jobs
+via a set of *RULE* expressions. The currently supported rules are
+
+ * If a rule begins with ``-``, then the rest of the rule is a pattern
+   which removes matching environment variables. If the pattern starts
+   with ``/``, it is a :linux:man7:`regex`, optionally ending with
+   ``/``, otherwise the pattern is considered a shell
+   :linux:man7:`glob` expression.
+
+   Examples:
+      ``-*`` or ``-/.*/`` filter all environment variables creating an
+      empty environment.
+
+ * If a rule begins with ``^`` then the rest of the rule is a filename
+   from which to read more rules, one per line. The ``~`` character is
+   expanded to the user's home directory.
+
+   Examples:
+      ``~/envfile`` reads rules from file ``$HOME/envfile``
+
+ * If a rule is of the form ``VAR=VAL``, the variable ``VAR`` is set
+   to ``VAL``. Before being set, however, ``VAL`` will undergo simple
+   variable substitution using the Python ``string.Template`` class. This
+   simple substitution supports the following syntax:
+
+     * ``$$`` is an escape; it is replaced with ``$``
+     * ``$var`` will substitute :envvar:`var` from the current environment,
+       falling back to the process environment. An error will be thrown
+       if environment variable :envvar:`var` is not set.
+     * ``${var}`` is equivalent to ``$var``
+     * Advanced parameter substitution is not allowed, e.g. ``${var:-foo}``
+       will raise an error.
+
+   Examples:
+       ``PATH=/bin``, ``PATH=$PATH:/bin``, ``FOO=${BAR}something``
+
+ * Otherwise, the rule is considered a pattern from which to match
+   variables from the process environment if they do not exist in
+   the generated environment. E.g. ``PATH`` will export :envvar:`PATH` from the
+   current environment (if it has not already been set in the generated
+   environment), and ``OMP*`` would copy all environment variables that
+   start with :envvar:`OMP` and are not already set in the generated
+   environment.  It is important to note that if the pattern does not match
+   any variables, then the rule is a no-op, i.e. an error is *not* generated.
+
+   Examples:
+       ``PATH``, ``FLUX_*_PATH``, ``/^OMP.*/``
+
+Since we always starts with a copy of the current environment,
+the default implicit rule is ``*`` (or :option:`--env=*`). To start with an
+empty environment instead, the ``-*`` rule or :option:`--env-remove=*` option
+should be used. For example, the following will only export the current
+:envvar:`PATH` to a job:
+
+::
+
+    flux run --env-remove=* --env=PATH ...
+
+
+Since variables can be expanded from the currently built environment, and
+:option:`--env` options are applied in the order they are used, variables can
+be composed on the command line by multiple invocations of :option:`--env`,
+e.g.:
+
+::
+
+    flux run --env-remove=* \
+                  --env=PATH=/bin --env='PATH=$PATH:/usr/bin' ...
+
+Note that care must be taken to quote arguments so that ``$PATH`` is not
+expanded by the shell.
+
+
+This works particularly well when specifying rules in a file:
+
+::
+
+    -*
+    OMP*
+    FOO=bar
+    BAR=${FOO}/baz
+
+The above file would first clear the environment, then copy all variables
+starting with :envvar:`OMP` from the current environment, set ``FOO=bar``,
+and then set ``BAR=bar/baz``.
+
diff --git a/doc/man1/common/job-environment-variables.rst b/doc/man1/common/job-environment-variables.rst
new file mode 100644
index 000000000000..2fabf241fb92
--- /dev/null
+++ b/doc/man1/common/job-environment-variables.rst
@@ -0,0 +1,26 @@
+.. list-table::
+   :header-rows: 1
+
+   * - Name
+     - Description
+
+   * - :envvar:`FLUX_JOB_ID`
+     - the current jobid
+
+   * - :envvar:`FLUX_JOB_SIZE`
+     - the number of tasks in the current job
+
+   * - :envvar:`FLUX_JOB_NNODES`
+     - the total number of nodes hosting tasks on behalf of the job
+
+   * - :envvar:`FLUX_TASK_RANK`
+     - the global task rank (zero origin)
+
+   * - :envvar:`FLUX_TASK_LOCAL_ID`
+     - the local task rank (zero origin)
+
+   * - :envvar:`FLUX_JOB_TMPDIR`
+     - path to temporary, per-job directory, usually in ``/tmp``
+
+   * - :envvar:`FLUX_URI`
+     - the URI of the enclosing Flux instance
diff --git a/doc/man1/common/job-environment.rst b/doc/man1/common/job-environment.rst
new file mode 100644
index 000000000000..f04a0482ff46
--- /dev/null
+++ b/doc/man1/common/job-environment.rst
@@ -0,0 +1,20 @@
+.. option:: --env=RULE
+
+   Control how environment variables are exported with *RULE*. See
+   the `ENV RULES`_ section below for more information. Rules are
+   applied in the order in which they are used on the command line.
+   This option may be specified multiple times.
+
+.. option:: --env-remove=PATTERN
+
+   Remove all environment variables matching *PATTERN* from the current
+   generated environment. If *PATTERN* starts with a ``/`` character,
+   then it is considered a :linux:man7:`regex`, otherwise *PATTERN* is
+   treated as a shell :linux:man7:`glob`. This option is equivalent to
+   :option:`--env=-PATTERN` and may be used multiple times.
+
+.. option:: --env-file=FILE
+
+   Read a set of environment *RULES* from a *FILE*. This option is
+   equivalent to :option:`--env=^FILE` and may be used multiple times.
+
diff --git a/doc/man1/common/job-other-batch.rst b/doc/man1/common/job-other-batch.rst
new file mode 100644
index 000000000000..502428e5121f
--- /dev/null
+++ b/doc/man1/common/job-other-batch.rst
@@ -0,0 +1,73 @@
+.. option:: --conf=FILE|KEY=VAL|STRING|NAME
+
+   The :option:`--conf`` option allows configuration for a Flux instance
+   started via ``flux-batch(1)`` or ``flux-alloc(1)`` to be iteratively built
+   on the command line. On first use, a ``conf.json`` entry is added to the
+   internal jobspec file archive, and ``-c{{tmpdir}}/conf.json`` is added
+   to the flux broker command line. Each subsequent use of the :option:`--conf`
+   option updates this configuration.
+
+   The argument to :option:`--conf` may be in one of several forms:
+
+   * A multiline string, e.g. from a batch directive. In this case the string
+     is parsed as JSON or TOML::
+
+      # flux: --conf="""
+      # flux: [resource]
+      # flux: exclude = "0"
+      # flux: """
+
+   * A string containing a ``=`` character, in which case the argument is
+     parsed as ``KEY=VAL``, where ``VAL`` is parsed as JSON, e.g.::
+
+      --conf=resource.exclude=\"0\"
+
+   * A string ending in ``.json`` or ``.toml``, in which case configuration
+     is loaded from a JSON or TOML file.
+
+   * If none of the above conditions match, then the argument ``NAME`` is
+     assumed to refer to a "named" config file ``NAME.toml`` or ``NAME.json``
+     within the following search path, in order of precedence:
+
+     - ``XDG_CONFIG_HOME/flux/config`` or ``$HOME/.config/flux/config`` if
+       :envvar:`XDG_CONFIG_HOME` is not set
+
+     - ``$XDG_CONFIG_DIRS/flux/config`` or ``/etc/xdg/flux/config`` if
+       :envvar:`XDG_CONFIG_DIRS` is not set. Note that :envvar:`XDG_CONFIG_DIRS`
+       may be a colon-separated path.
+
+.. option:: --bg
+
+   *(alloc only)* Do not interactively attach to the instance. Instead,
+   print jobid on stdout once the instance is ready to accept jobs. The
+   instance will run indefinitely until a time limit is reached, the
+   job is canceled, or it is shutdown with ``flux shutdown JOBID``
+   (preferred). If a COMMAND is given then the job will run until COMMAND
+   completes. Note that ``flux job attach JOBID`` cannot be used to
+   interactively attach to the job (though it will print any errors or
+   output).
+
+.. option:: -B, --broker-opts=OPT
+
+   Pass specified options to the Flux brokers of the new instance. This
+   option may be specified multiple times.
+
+.. option:: --wrap
+
+   *(batch only)* The :option:`--wrap` option wraps the specified COMMAND and
+   ARGS in a shell script, by prefixing with ``#!/bin/sh``. If no COMMAND is
+   present, then a SCRIPT is read on stdin and wrapped in a /bin/sh script.
+
+.. option:: --dump=[FILE]
+
+   When the job script is complete, archive the Flux
+   instance's KVS content to ``FILE``, which should have a suffix known
+   to :linux:man3:`libarchive`, and may be a mustache template as described
+   above for :option:`--output`.  The content may be unarchived directly or
+   examined within a test instance started with the
+   :option:`flux-start --recovery` option.  If ``FILE`` is unspecified,
+   ``flux-{{jobid}}-dump.tgz`` is used.
+
+.. option:: --quiet
+
+   Suppress logging of jobids to stdout
diff --git a/doc/man1/common/job-other-options.rst b/doc/man1/common/job-other-options.rst
new file mode 100644
index 000000000000..a5298fc13413
--- /dev/null
+++ b/doc/man1/common/job-other-options.rst
@@ -0,0 +1,120 @@
+.. option:: --cwd=DIRECTORY
+
+   Set job working directory.
+
+.. option:: --urgency=N
+
+   Specify job urgency.  *N* has a range of 0 to 16 for guest users, 0 to 31
+   for instance owners, and a default value of 16.  In addition to numerical
+   values, the following special names are accepted:
+
+   hold (0)
+      Hold the job until the urgency is raised with :option:`flux job urgency`.
+
+   default (16)
+      The default urgency for all users.
+
+   expedite (31)
+      Assign the highest possible priority to the job (restricted to instance
+      owner).
+
+   Urgency is one factor used to calculate job priority, which affects the
+   order in which the scheduler considers jobs.  By default, priority is
+   calculated from the urgency and the time elapsed since job submission.
+   This calculation may be overridden by configuration.  For example, in a
+   multi-user Flux instance with the Flux accounting priority plugin loaded,
+   the calculation includes other factors such as past usage and bank
+   allocations.
+
+   A job with an urgency value of 0 is treated specially:  it is never
+   considered by the scheduler and is effectively *held*.  Similarly, a job
+   with an urgency of 31 is always assigned the maximum priority, regardless
+   of other factors and is considered *expedited*.
+
+   :option:`flux jobs -o deps` lists jobs with urgency and priority fields.
+
+.. option:: -v, --verbose
+
+   Increase verbosity on stderr. For example,
+   currently :option:`flux run -v` displays jobid, :option:`-vv` displays job
+   events, and :option:`-vvv` displays exec events. :option:`flux alloc -v`
+   forces the command to print the submitted jobid on stderr.  The specific
+   output may change in the future.
+
+.. option:: -o, --setopt=KEY[=VAL]
+
+   Set shell option. Keys may include periods to denote hierarchy.
+   VAL is optional and may be valid JSON (bare values, objects, or arrays),
+   otherwise VAL is interpreted as a string. If VAL is not set, then the
+   default value is 1. See `SHELL OPTIONS`_ below.
+
+.. option:: -S, --setattr=KEY[=VAL]
+
+   Set jobspec attribute. Keys may include periods to denote hierarchy.
+   If KEY does not begin with ``system.``, ``user.``, or ``.``, then
+   ``system.`` is assumed.  VAL is optional and may be valid JSON (bare
+   values, objects, or arrays), otherwise VAL is interpreted as a string. If
+   VAL is not set, then the default value is 1.  If KEY starts with a ``^``
+   character, then VAL is interpreted as a file, which must be valid JSON,
+   to use as the attribute value.
+
+.. option:: --add-file=[NAME[:PERMS]=]ARG
+
+   Add a file to the RFC 37 file archive in jobspec before submission. Both
+   the file metadata and content are stored in the archive, so modification
+   or deletion of a file after being processed by this option will have no
+   effect on the job. If no ``NAME`` is provided, then ``ARG`` is assumed
+   to be the path to a local file and the basename of the file will be
+   used as ``NAME``.  Otherwise, if ``ARG`` contains a newline, then it
+   is assumed to be the raw file data to encode. If ``PERMS`` is provided
+   and is a valid octal integer, then this will override the default file
+   permissions of 0600.  The file will be extracted by the job shell into
+   the job temporary directory and may be referenced as ``{{tmpdir}}/NAME``
+   on the command line, or ``$FLUX_JOB_TMPDIR/NAME`` in a batch script.
+   This option may be specified multiple times to encode multiple files.
+   Note: As documented in RFC 14, the file names ``script`` and ``conf.json``
+   are both reserved.
+
+   .. note::
+      This option should only be used for small files such as program input
+      parameters, configuration, scripts, and so on. For broadcast of large
+      files, binaries, and directories, the :man1:`flux-shell` ``stage-in``
+      plugin will be more appropriate.
+
+.. option:: --begin-time=+FSD|DATETIME
+
+   Convenience option for setting a ``begin-time`` dependency for a job.
+   The job is guaranteed to start after the specified date and time.
+   If argument begins with a ``+`` character, then the remainder is
+   considered to be an offset in Flux standard duration (RFC 23), otherwise,
+   any datetime expression accepted by the Python 
+   `parsedatetime <https://github.com/bear/parsedatetime>`_ module
+   is accepted, e.g. ``2021-06-21 8am``, ``in an hour``,
+   ``tomorrow morning``, etc.
+
+.. option:: --signal=SIG@TIME
+
+   Send signal ``SIG`` to job ``TIME`` before the job time limit. ``SIG``
+   can specify either an integer signal number or a full or abbreviated
+   signal name, e.g. ``SIGUSR1`` or ``USR1`` or ``10``. ``TIME`` is
+   specified in Flux Standard Duration, e.g. ``30`` for 30s or ``1h`` for
+   1 hour. Either parameter may be omitted, with defaults of ``SIGUSR1``
+   and 60s.  For example, :option:`--signal=USR2` will send ``SIGUSR2`` to
+   the job 60 seconds before expiration, and :option:`--signal=@3m` will send
+   ``SIGUSR1`` 3 minutes before expiration. Note that if ``TIME`` is
+   greater than the remaining time of a job as it starts, the job will
+   be signaled immediately.
+
+   The default behavior is to not send any warning signal to jobs.
+
+.. option:: --dry-run
+
+   Don't actually submit job. Just emit jobspec on stdout and exit for
+   ``run``, ``submit``, ``alloc``, and ``batch``. For ``bulksubmit``,
+   emit a line of output including relevant options for each job which
+   would have been submitted,
+
+.. option:: --debug
+
+   Enable job debug events, primarily for debugging Flux itself.
+   The specific effects of this option may change in the future.
diff --git a/doc/man1/common/job-other-run.rst b/doc/man1/common/job-other-run.rst
new file mode 100644
index 000000000000..57c92ea20c49
--- /dev/null
+++ b/doc/man1/common/job-other-run.rst
@@ -0,0 +1,111 @@
+.. option:: --taskmap=SCHEME[:VALUE]
+
+   Choose an alternate method for mapping job task IDs to nodes of the
+   job. The job shell maps tasks using a "block" distribution scheme by
+   default (consecutive tasks share nodes) This option allows the
+   activation of alternate schemes by name, including an optional *VALUE*.
+   Supported schemes which are built in to the job shell include
+
+   cyclic[:N]
+    Tasks are distributed over consecutive nodes with a stride of *N*
+    (where N=1 by default).
+
+   manual:TASKMAP
+    An explicit RFC 34 taskmap is provided and used to manually map
+    task ids to nodes. The provided *TASKMAP* must match the total number
+    of tasks in the job and the number of tasks per node assigned by
+    the job shell, so this option is not useful unless the total number
+    of nodes and tasks per node are known at job submission time.
+
+   hostfile:FILE
+    Assign tasks in order to hosts as they appear in FILE. FILE should
+    have one or more lines each of which contains a host name or RFC
+    29 Hostlist string. Each host assigned to the job must appear in
+    the hostfile and be assigned the same number of tasks as the default
+    taskmap from the shell.  If there are less hosts in the hostfile than
+    tasks in the job, then the list of hosts will be reused.
+
+   However, shell plugins may provide other task mapping schemes, so
+   check the current job shell configuration for a full list of supported
+   taskmap schemes.
+
+.. option:: --cc=IDSET
+
+   *(submit,bulksubmit)* Replicate the job for each ``id`` in ``IDSET``.
+   :envvar:`FLUX_JOB_CC` will be set to ``id`` in the environment of each
+   submitted job to allow the job to alter its execution based on the
+   submission index.  (e.g. for reading from a different input file). When
+   using :option:`--cc`, the substitution string ``{cc}`` may be used in
+   options and commands and will be replaced by the current ``id``.
+
+.. option:: --bcc=IDSET
+
+   *(submit,bulksubmit)* Identical to :option:`--cc`, but do not set
+   :envvar:`FLUX_JOB_CC` in each job. All jobs will be identical copies.
+   As with :option:`--cc`, ``{cc}`` in option arguments and commands will be
+   replaced with the current ``id``.
+
+.. option:: --quiet
+
+   Suppress logging of jobids to stdout
+
+.. option:: --log=FILE
+
+   *(submit,bulksubmit)* Log command output and stderr to ``FILE``
+   instead of the terminal. If a replacement (e.g. ``{}`` or ``{cc}``)
+   appears in ``FILE``, then one or more output files may be opened.
+   For example, to save all submitted jobids into separate files, use::
+
+      flux submit --cc=1-4 --log=job{cc}.id hostname
+
+.. option:: --log-stderr=FILE
+
+   *(submit,bulksubmit)* Separate stderr into ``FILE`` instead of sending
+   it to the terminal or a ``FILE`` specified by :option:`--log`.
+
+.. option:: --wait
+
+   *(submit,bulksubmit)* Wait on completion of all jobs before exiting.
+   This is equivalent to :option:`--wait-event=clean`.
+
+.. option:: --wait-event=NAME
+
+   Wait until job or jobs have received event ``NAME``
+   before exiting. E.g. to submit a job and block until the job begins
+   running, use :option:`--wait-event=start`. *(submit,bulksubmit only)* If
+   ``NAME`` begins with ``exec.``, then wait for an event in the exec eventlog,
+   e.g.  ``exec.shell.init``. For ``flux run`` the argument to this option
+   when used is passed directly to ``flux job attach``.
+
+.. option:: --watch
+
+   *(submit,bulksubmit)* Display output from all jobs. Implies :option:`--wait`.
+
+.. option:: --progress
+
+   *(submit,bulksubmit)* With :option:`--wait`, display a progress bar showing
+   the progress of job completion. Without :option:`--wait`, the progress bar
+   will show progress of job submission.
+
+.. option:: --jps
+
+   *(submit,bulksubmit)* With :option:`--progress`, display throughput
+   statistics (jobs/s) in the progress bar.
+
+.. option:: --define=NAME=CODE
+
+   *(bulksubmit)* Define a named method that will be made available as an
+   attribute during command and option replacement. The string being
+   processed is available as ``x``. For example::
+
+   $ seq 1 8 | flux bulksubmit --define=pow="2**int(x)" -n {.pow} ...
+
+.. option:: --shuffle
+
+   *(bulksubmit)* Shuffle the list of commands before submission.
+
+.. option:: --sep=STRING
+
+   *(bulksubmit)* Change the separator for file input. The default is
+   to separate files (including stdin) by newline. To separate by
+   consecutive whitespace, specify :option:`--sep=none`.
diff --git a/doc/man1/common/job-param-additional.rst b/doc/man1/common/job-param-additional.rst
new file mode 100644
index 000000000000..30815fe28900
--- /dev/null
+++ b/doc/man1/common/job-param-additional.rst
@@ -0,0 +1,42 @@
+.. option:: -q, --queue=NAME
+
+   Submit a job to a specific named queue. If a queue is not specified
+   and queues are configured, then the jobspec will be modified at ingest
+   to specify the default queue. If queues are not configured, then this
+   option is ignored, though :man1:`flux-jobs` may display the queue
+   name in its rendering of the ``{queue}`` attribute.
+
+.. option:: -B, --bank=NAME
+
+   Set the bank name for this job to ``NAME``. This option is equivalent
+   to `--setattr=bank=NAME`, and results in the ``bank`` attribute being
+   set to ``NAME`` in the submitted jobspec. However, besides the bank
+   name appearing in job listing output, this option may have no effect
+   if no plugin or package that supports it (such as flux-accounting)
+   is installed and configured.
+
+.. option:: -t, --time-limit=MINUTES|FSD
+
+   Set a time limit for the job in either minutes or Flux standard duration
+   (RFC 23). FSD is a floating point number with a single character units
+   suffix ("s", "m", "h", or "d"). The default unit for the
+   :option:`--time-limit` option is minutes when no units are otherwise
+   specified. If the time limit is unspecified, the job is subject to the
+   system default time limit.
+
+.. option:: --job-name=NAME
+
+   Set an alternate job name for the job.  If not specified, the job name
+   will default to the command or script executed for the job.
+
+.. option:: --flags=FLAGS
+
+   Set comma separated list of job submission flags.  The possible flags are
+   ``waitable``, ``novalidate``, and ``debug``.  The ``waitable`` flag will
+   allow the job to be waited on via ``flux job wait`` and similar API calls.
+   The ``novalidate`` flag will inform flux to skip validation of a job's
+   specification.  This may be useful for high throughput ingest of a large
+   number of jobs.  Both ``waitable`` and ``novalidate`` require instance
+   owner privileges.  ``debug`` will output additional debugging into the job
+   eventlog.
+
diff --git a/doc/man1/common/job-param-batch.rst b/doc/man1/common/job-param-batch.rst
new file mode 100644
index 000000000000..04fa228b6a9a
--- /dev/null
+++ b/doc/man1/common/job-param-batch.rst
@@ -0,0 +1,20 @@
+.. option:: -n, --nslots=N
+
+   Set the number of slots requested. This parameter is required unless
+   :option:`--nodes` is specified.
+
+.. option:: -c, --cores-per-slot=N
+
+   Set the number of cores to assign to each slot (default 1).
+
+.. option:: -g, --gpus-per-slot=N
+
+   Set the number of GPU devices to assign to each slot (default none).
+
+.. option:: -N, --nodes=N
+
+   Distribute allocated resource slots across *N* individual nodes.
+
+.. option:: -x, --exclusive
+
+   With :option:`--nodes`, allocate nodes exclusively.
diff --git a/doc/man1/common/job-param-common.rst b/doc/man1/common/job-param-common.rst
new file mode 100644
index 000000000000..fd1c182dfb8f
--- /dev/null
+++ b/doc/man1/common/job-param-common.rst
@@ -0,0 +1,16 @@
+.. option:: -N, --nodes=N
+
+   Set the number of nodes to assign to the job. Tasks will be distributed
+   evenly across the allocated nodes, unless the per-resource options
+   (noted below) are used with *submit*, *run*, or *bulksubmit*. It is
+   an error to request more nodes than there are tasks. If unspecified,
+   the number of nodes will be chosen by the scheduler.
+
+.. option:: -x, --exclusive
+
+   Indicate to the scheduler that nodes should be exclusively allocated to
+   this job. It is an error to specify this option without also using
+   :option:`--nodes`. If :option:`--nodes` is specified without
+   :option:`--nslots` or :option:`--ntasks`, then this option will be enabled
+   by default and the number of tasks or slots will be set to the number of
+   requested nodes.
diff --git a/doc/man1/common/job-param-perres.rst b/doc/man1/common/job-param-perres.rst
new file mode 100644
index 000000000000..cab18659406d
--- /dev/null
+++ b/doc/man1/common/job-param-perres.rst
@@ -0,0 +1,20 @@
+.. option:: --cores=N
+
+   Set the total number of cores.
+
+.. option:: --tasks-per-node=N
+
+   Set the number of tasks per node to run.
+
+.. option:: --gpus-per-node=N
+
+   With :option:`--nodes`, request a specific number of GPUs per node.
+
+.. option:: --tasks-per-core=N
+
+   Force a number of tasks per core. Note that this will run *N* tasks per
+   *allocated* core. If nodes are exclusively scheduled by configuration or
+   use of the :option:`--exclusive` flag, then this option could result in many
+   more tasks than expected. The default for this option is effectively 1,
+   so it is useful only for oversubscribing tasks to cores for testing
+   purposes. You probably don't want to use this option.
diff --git a/doc/man1/common/job-param-pertask.rst b/doc/man1/common/job-param-pertask.rst
new file mode 100644
index 000000000000..153e75e7ead1
--- /dev/null
+++ b/doc/man1/common/job-param-pertask.rst
@@ -0,0 +1,11 @@
+.. option:: -n, --ntasks=N
+
+   Set the number of tasks to launch (default 1).
+
+.. option:: -c, --cores-per-task=N
+
+   Set the number of cores to assign to each task (default 1).
+
+.. option:: -g, --gpus-per-task=N
+
+   Set the number of GPU devices to assign to each task (default none).
diff --git a/doc/man1/common/job-process-resource-limits.rst b/doc/man1/common/job-process-resource-limits.rst
new file mode 100644
index 000000000000..d3ac3e6dda72
--- /dev/null
+++ b/doc/man1/common/job-process-resource-limits.rst
@@ -0,0 +1,41 @@
+.. option:: --rlimit=RULE
+
+    Control how process resource limits are propagated with *RULE*. Rules
+    are applied in the order in which they are used on the command line.
+    This option may be used multiple times.
+
+The :option:`--rlimit` rules work similar to the :option:`--env` option rules:
+
+ * If a rule begins with ``-``, then the rest of the rule is a name or
+   :linux:man7:`glob` pattern which removes matching resource limits from
+   the set to propagate.
+
+   Example:
+     ``-*`` disables propagation of all resource limits.
+
+ * If a rule is of the form ``LIMIT=VALUE`` then *LIMIT* is explicitly
+   set to *VALUE*. If *VALUE* is ``unlimited``, ``infinity`` or ``inf``,
+   then the value is set to ``RLIM_INFINITY`` (no limit).
+
+   Example:
+     ``nofile=1024`` overrides the current ``RLIMIT_NOFILE`` limit to 1024.
+
+ * Otherwise, *RULE* is considered a pattern from which to match resource
+   limits and propagate the current limit to the job, e.g.
+
+      ``--rlimit=memlock``
+
+   will propagate ``RLIMIT_MEMLOCK`` (which is not in the list of limits
+   that are propagated by default).
+
+We start with a default list of resource limits to propagate,
+then applies all rules specified via :option:`--rlimit` on the command line.
+Therefore, to propagate only one limit, ``-*`` should first be used to
+start with an empty set, e.g. :option:`--rlimit=-*,core` will only propagate
+the ``core`` resource limit.
+
+The set of resource limits propagated by default includes all those except
+``memlock``, ``ofile``, ``msgqueue``, ``nice``, ``rtprio``, ``rttime``,
+and ``sigpending``. To propagate all possible resource limits, use
+:option:`--rlimit=*`.
+
diff --git a/doc/man1/common/job-shell-options.rst b/doc/man1/common/job-shell-options.rst
new file mode 100644
index 000000000000..1efb6e5f55ef
--- /dev/null
+++ b/doc/man1/common/job-shell-options.rst
@@ -0,0 +1,58 @@
+.. Once we advance to sphinx 5.3+, :option: will x-ref with arguments
+.. e.g. :option:`cpu-affinity=OFF`.  For now, leave options off to get x-ref.
+
+.. list-table::
+   :header-rows: 1
+
+   * - Name
+     - Description
+
+   * - :option:`cpu-affinity`
+     - Set task affinity to cores (:option:`off|per-task|map:LIST|on`)
+
+   * - :option:`gpu-affinity`
+     - Set task affinity to GPUs (:option:`off|per-task|map:LIST|on`)
+
+   * - :option:`verbose`
+     - Increase shell log verbosity (1 or 2).
+
+   * - :option:`nosetpgrp`
+     - Don't run each task in its own process group.
+
+   * - :option:`pmi`
+     - Set PMI service(s) for launched programs (:option:`off|simple|LIST`)
+
+   * - :option:`stage-in`
+     - Copy files previously mapped with :man1:`flux-archive` to
+       :envvar:`FLUX_JOB_TMPDIR`.
+
+   * - :option:`pty.interactive`
+     - Enable a pty on rank 0 for :program:`flux job attach`.
+
+   * - :option:`exit-timeout`
+     - Start fatal job exception timer after first task exits
+       (:option:`none|FSD`)
+
+   * - :option:`exit-on-error`
+     - Raise a fatal job exception immediately if first task exits with
+       nonzero exit code.
+
+   * - :option:`hwloc.xmlfile`
+     - Write hwloc XML gathered by job to a file and set ``HWLOC_XMLFILE``.
+       Note that this option will also unset ``HWLOC_COMPONENTS`` if set, since
+       presence of this environment variable may cause hwloc to ignore
+       ``HWLOC_XMLFILE``.
+
+   * - :option:`hwloc.restrict`
+     - With :option:`hwloc.xmlfile`, restrict the exported topology XML to only
+       those resources assigned to the current job. By default, the XML is
+       not restricted.
+
+   * - :option:`output.limit`
+     - Set KVS output limit to SIZE bytes, where SIZE may be a floating point
+       value including optional SI units: k, K, M, G. This value is ignored
+       if output is directed to a file with :option:`--output`.
+
+   * - :option:`output.mode`
+     - Set the open mode for output files to either "truncate" or "append".
+       The default is "truncate".
diff --git a/doc/man1/common/job-standard-io.rst b/doc/man1/common/job-standard-io.rst
new file mode 100644
index 000000000000..0650e5a06e71
--- /dev/null
+++ b/doc/man1/common/job-standard-io.rst
@@ -0,0 +1,56 @@
+.. note::
+   Paths specified in the options :option:`--output`, :option:`--error`,
+   and :option:`--input` will be opened relative to the working directory
+   on the first node of the job allocation, not on the node from which
+   the job is submitted.
+
+.. option:: --input=FILENAME|RANKS
+
+   Redirect stdin to the specified filename, bypassing the KVS.
+   As a special case for ``flux run``, the argument may specify
+   an idset of task ranks in to which to direct standard input.
+
+.. option:: --output=TEMPLATE
+
+   Specify the filename *TEMPLATE* for stdout redirection, bypassing
+   the KVS.  *TEMPLATE* may be a mustache template which supports the
+   following tags:
+
+   *{{id}}* or *{{jobid}}*
+     Expands to the current jobid in the F58 encoding. If needed, an
+     alternate encoding may be selected by using a subkey with the name
+     of the desired encoding, e.g. *{{id.dec}}*. Supported encodings
+     include *f58* (the default), *dec*, *hex*, *dothex*, and *words*.
+
+   *{{name}}*
+     Expands to the current job name. If a name is not set for the job,
+     then the basename of the command will be used.
+
+.. option:: --error=TEMPLATE
+
+   Redirect stderr to the specified filename *TEMPLATE*, bypassing the KVS.
+   *TEMPLATE* is expanded as described above.
+
+.. option:: -u, --unbuffered
+
+   Disable buffering of standard input and output as much as practical.
+   Normally, stdout from job tasks is line buffered, as is stdin when
+   running a job in the foreground via :man1:`flux-run`. Additionally,
+   job output may experience a delay due to batching of output
+   events by the job shell. With the :option:`--unbuffered` option,
+   ``output.*.buffer.type=none`` is set in jobspec to request no buffering
+   of output, and the default output batch period is reduced greatly,
+   to make output appear in the KVS and printed to the standard output
+   of :man1:`flux-run` as soon as possible. The :option:`--unbuffered` option
+   is also passed to ``flux job attach``, which makes stdin likewise
+   unbuffered. Note that the application and/or terminal may have
+   additional input and output buffering which this option will not
+   affect. For instance, by default an interactive terminal in canonical
+   input mode will process input by line only. Likewise, stdout of a
+   process run without a terminal may be fully buffered when using
+   libc standard I/O streams (See NOTES in :linux:man3:`stdout`).
+
+.. option:: -l, --label-io
+
+   Add task rank prefixes to each line of output.
+
diff --git a/doc/man1/common/resources.rst b/doc/man1/common/resources.rst
new file mode 100644
index 000000000000..dcc16e8954b0
--- /dev/null
+++ b/doc/man1/common/resources.rst
@@ -0,0 +1,5 @@
+Flux: http://flux-framework.org
+
+Flux RFC: https://flux-framework.readthedocs.io/projects/flux-rfc
+
+Issue Tracker: https://github.com/flux-framework/flux-core/issues
diff --git a/doc/man1/flux-alloc.rst b/doc/man1/flux-alloc.rst
new file mode 100644
index 000000000000..b055416e04ed
--- /dev/null
+++ b/doc/man1/flux-alloc.rst
@@ -0,0 +1,145 @@
+.. flux-help-include: true
+.. flux-help-section: submission
+
+=============
+flux-alloc(1)
+=============
+
+SYNOPSIS
+========
+
+**flux** **alloc** [OPTIONS] [COMMAND...]
+
+DESCRIPTION
+===========
+
+.. program:: flux alloc
+
+:program:`flux alloc` runs a Flux subinstance with *COMMAND* as the initial
+program.  Once resources are allocated, *COMMAND* executes on the first node of
+the allocation with any free arguments supplied as *COMMAND* arguments.  When
+*COMMAND* exits, the Flux subinstance exits, resources are released to the
+enclosing Flux instance, and :program:`flux alloc` returns.
+
+If no *COMMAND* is specified, an interactive shell is spawned as the initial
+program, and the subinstance runs until the shell is exited.
+
+If the :option:`--bg` option is specified, the subinstance runs without an
+initial program.  :program:`flux alloc` prints the jobid and returns as soon as
+the subinstance is ready to accept jobs.  The subinstance runs until it exceeds
+its time limit, is canceled, or is shut down with :man1:`flux-shutdown`.
+
+Flux commands that are run from the subinstance (e.g. from the interactive
+shell) refer to the subinstance. For example, :man1:`flux-run` would launch
+work there.  A Flux command run from the subinstance can be forced to refer
+to the enclosing instance by supplying the :option:`flux --parent` option.
+
+Flux commands outside of the subinstance refer to their enclosing instance,
+often a system instance. :man1:`flux-proxy` establishes a connection to a
+running subinstance by jobid, then spawns a shell in which Flux commands
+refer to the subinstance, for example
+
+::
+
+   $ flux alloc --bg -N 2 --queue=batch
+   ƒM7Zq9AKHno
+   $ flux proxy ƒM7Zq9AKHno
+   $ flux run -n16 ./testprog
+   ...
+   $ flux shutdown
+   ...
+   $
+
+The available OPTIONS are detailed below.
+
+JOB PARAMETERS
+==============
+
+:program:`flux batch` and :program:`flux alloc` do not launch tasks directly,
+and therefore job parameters are normally specified in terms of resource slot
+size and number of slots. A resource slot can be thought of as the minimal
+resources required for a virtual task. The default slot size is 1 core.
+
+.. include:: common/job-param-batch.rst
+
+.. include:: common/job-param-additional.rst
+
+CONSTRAINTS
+===========
+
+.. include:: common/job-constraints.rst
+
+DEPENDENCIES
+============
+
+.. include:: common/job-dependencies.rst
+
+ENVIRONMENT
+===========
+
+By default, :man1:`flux-alloc` duplicates the current environment when
+submitting jobs. However, a set of environment manipulation options are
+provided to give fine control over the requested environment submitted
+with the job.
+
+.. note::
+   The actual environment of the initial program is subject to the caveats
+   described in the :ref:`initial_program_environment` section of
+   :man7:`flux-environment`.
+
+.. include:: common/job-environment.rst
+
+ENV RULES
+---------
+
+.. include:: common/job-env-rules.rst
+
+PROCESS RESOURCE LIMITS
+=======================
+
+By default these commands propagate some common resource limits (as described
+in :linux:man2:`getrlimit`) to the job by setting the ``rlimit`` job shell
+option in jobspec.  The set of resource limits propagated can be controlled
+via the :option:`--rlimit=RULE` option:
+
+.. include:: common/job-process-resource-limits.rst
+
+EXIT STATUS
+===========
+
+The job exit status is normally the batch script exit status.
+This result is stored in the KVS.
+
+OTHER OPTIONS
+=============
+
+.. include:: common/job-other-options.rst
+
+.. include:: common/job-other-batch.rst
+
+SHELL OPTIONS
+=============
+
+Some options that affect the parallel runtime environment of the Flux instance
+started by :program:`flux alloc` are provided by the Flux shell.
+These options are described in detail in the
+:ref:`SHELL OPTIONS <flux_shell_options>` section of :man1:`flux-shell`.
+A list of the most commonly needed options follows.
+
+Usage: :option:`flux alloc -o NAME[=ARG]`.
+
+.. make :option: references in the included table x-ref to flux-shell(1)
+.. program:: flux shell
+.. include:: common/job-shell-options.rst
+.. program:: flux alloc
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+SEE ALSO
+========
+
+:man1:`flux-run`, :man1:`flux-submit`, :man1:`flux-batch`,
+:man1:`flux-bulksubmit`, :man7:`flux-environment`
diff --git a/doc/man1/flux-archive.rst b/doc/man1/flux-archive.rst
new file mode 100644
index 000000000000..35f8422ac6f5
--- /dev/null
+++ b/doc/man1/flux-archive.rst
@@ -0,0 +1,317 @@
+===============
+flux-archive(1)
+===============
+
+
+SYNOPSIS
+========
+
+| **flux** **archive** **create** [*-n NAME*] [*-C DIR*] *PATH* ...
+| **flux** **archive** **list** [*-n NAME*] [*--long*] [*PATTERN*]
+| **flux** **archive** **extract** [*-n NAME*] [*-C DIR*] [*PATTERN*]
+| **flux** **archive** **remove** [*-n NAME*] [*-f*]
+
+
+DESCRIPTION
+===========
+
+.. program:: flux archive
+
+:program:`flux archive` stores multiple files in an RFC 37 archive
+under a single KVS key.  The archive can be efficiently extracted in
+parallel across the Flux instance, leveraging the scalability properties
+of the KVS and its content addressable data store.
+
+Sparse files such as file system images for virtual machines are archived
+efficiently.
+
+File discretionary access permissions are preserved, but file attributes,
+ACLs, and group ownership are not.
+
+The ``stage-in`` shell plugin described in :man1:`flux-shell` may be used to
+extract previously archived files into the directory referred to by
+:envvar:`FLUX_JOB_TMPDIR` or another directory.
+
+Due to the potential impact on Flux's storage footprint on rank 0, this
+command is limited to instance owners, e.g. it works in batch jobs and
+allocations but not at the system level.
+
+
+COMMANDS
+========
+
+create
+------
+
+.. program:: flux archive create
+
+:program:`flux archive create` archives one or more file *PATH* arguments.
+If a *PATH* refers to a directory, the directory is recursively archived.
+If a file is encountered that is not readable, or has a type other than
+regular file, directory, or symbolic link, a fatal error occurs.
+
+.. option:: -n, --name=NAME
+
+   Specify the archive name.  If a name is not specified, ``main`` is used.
+
+   The archive will be committed to the KVS as ``archive.NAME``.
+
+.. option:: --overwrite
+
+   Unlink ``archive.NAME`` and ``archive.NAME_blobs`` from the KVS, and
+   unmap all files associated with *NAME* that were previously archived
+   with :option:`--mmap` before creating the archive.
+
+   Without :option:`--overwrite` or :option:`--append`, it is a fatal error
+   if *NAME* exists.
+
+.. option:: --append
+
+   If *NAME* exists, append new content to the existing archive.
+   Otherwise create it from scratch.
+
+   Due to the way the KVS handles key changes, appending :math:`M` bytes to
+   to a key of size :math:`N` consumes roughly :math:`2N + M` bytes in the
+   backing store, while separate keys consume :math:`N + M`.  As a consequence,
+   creating multiple archives may be cheaper than building one iteratively
+   with :option:`--append`.
+
+.. option:: -C, --directory=DIR
+
+   Change to the specified directory before performing the operation.
+
+.. option:: --no-force-primary
+
+   Create the archive in the default KVS namespace, honoring
+   :envvar:`FLUX_KVS_NAMESPACE`, if set.  By default, the primary KVS
+   namespace is used.
+
+.. option:: --preserve
+
+   Write additional KVS metadata so that the archive remains intact across
+   a Flux restart with garbage collection.
+
+   The metadata will be committed to the KVS as ``archive.NAME_blobs``.
+
+.. option:: -v, --verbose=[LEVEL]
+
+   List files on standard error as the archive is created.
+
+.. option:: --chunksize=N
+
+   Limit the content blob size to N bytes.  Set to 0 for unlimited.
+   N may be specified as a floating point number with multiplicative suffix
+   k,K=1024, M=1024\*1024, or G=1024\*1024\*1024 up to ``INT_MAX``.
+   The default is 1M.
+
+.. option:: --small-file-threshold=N
+
+   Set the threshold in bytes for a small file.  A small file is represented
+   directly in the archive, as opposed to the content store.  Set to 0 to
+   always use the content store.  N may be specified as a floating point
+   number with multiplicative suffix k,K=1024, M=1024\*1024, or
+   G=1024\*1024\*1024 up to ``INT_MAX``.  The default is 1K.
+
+.. option:: --mmap
+
+   For large files, use :linux:man2:`mmap` to map file data into the content
+   store rather than copying it.  This only works on rank 0, and does not work
+   with :option:`--preserve` or :option:`--no-force-primary`.  Furthermore,
+   the files must remain available and unchanged while the archive exists.
+   This is most appropriate for truly large files such as VM images.
+
+   .. warning::
+
+      The rank 0 Flux broker may die with a SIGBUS error if a mapped file is
+      removed or truncated, and subsequently accessed, since that renders
+      pages mapped into the brokers address space invalid.
+
+      If mapped file content changes, access may fail if the original data
+      is not cached, but under no circumstances will the new content be
+      returned.
+
+list
+----
+
+.. program:: flux archive list
+
+:program:`flux archive list` shows the archive contents on standard output.
+If *PATTERN* is specified, only the files that match the :linux:man7:`glob`
+pattern are listed.
+
+.. option:: -l, --long
+
+   List the archive in long form, including file type, mode, and size.
+
+.. option:: --raw
+
+   List the RFC 37 file objects in JSON form, without decoding.
+
+.. option:: -n, --name=NAME
+
+   Specify the archive name.  If a name is not specified, ``main`` is used.
+
+.. option:: --no-force-primary
+
+   List the archive in the default KVS namespace, honoring
+   :envvar:`FLUX_KVS_NAMESPACE`, if set.  By default, the primary KVS
+   namespace is used.
+
+remove
+------
+
+.. program:: flux archive remove
+
+:program:`flux archive remove` expunges an archive.  The archive's KVS keys
+are unlinked, and any files previously mapped with :option:`--mmap` are
+unmapped.
+
+.. option:: -n, --name=NAME
+
+   Specify the archive name.  If a name is not specified, ``main`` is used.
+
+.. option:: --no-force-primary
+
+   Remove the archive in the default KVS namespace, honoring
+   :envvar:`FLUX_KVS_NAMESPACE`, if set.  By default, the primary KVS
+   namespace is used.
+
+.. option:: -f, --force
+
+   Don't fail if the archive does not exist.
+
+
+extract
+-------
+
+.. program:: flux archive extract
+
+:program:`flux archive extract` extracts files from a KVS archive.
+If *PATTERN* is specified, only the files that match the :linux:man7:`glob`
+pattern are extracted.
+
+.. option:: -t, --list-only
+
+   List files in the archive without extracting.
+
+.. option:: -n, --name=NAME
+
+   Specify the archive name.  If a name is not specified, ``main`` is used.
+
+.. option:: -C, --directory=DIR
+
+   Change to the specified directory before performing the operation.
+
+   When extracting files in parallel, take care when specifying *DIR*:
+
+   - It should have enough space to hold the extracted files.
+
+   - It should not be a fragile network file system such that parallel
+     writes could cause a distributed denial of service.
+
+   - It should not already be shared among the nodes of your job.
+
+.. option:: -v, --verbose=[LEVEL]
+
+   List files on standard error as the archive is extracted.
+
+.. option:: --overwrite
+
+   Overwrite existing files when extracting.  :program:`flux archive extract`
+   normally refuses to do this and treats it as a fatal error.
+
+.. option:: --waitcreate[=FSD]
+
+   Wait for the archive key to appear in the KVS if it doesn't exist.
+   This may be necessary in some circumstances as noted in `CAVEATS`_
+   below.
+
+   If *FSD* is specified, it is interpreted as a timeout value in RFC 23
+   Flux Standard Duration format.
+
+.. option:: --no-force-primary
+
+   Extract from the archive in the default KVS namespace, honoring
+   :envvar:`FLUX_KVS_NAMESPACE`, if set.  By default, the primary KVS
+   namespace is used.
+
+CAVEATS
+=======
+
+The KVS employs an "eventually consistent" cache update model, which
+means one has to be careful when writing a key on one broker rank and
+reading it on other broker ranks.  Without some form of synchronization,
+there is a short period of time where the KVS cache on the other ranks
+may not yet have the new data.
+
+This is not an issue for Example 2 below, where a batch script creates
+an archive, then submits jobs that read the archive because job
+submission and execution already include KVS synchronization.
+In other situations such as Example 1, it is advisable to use
+:option:`--waitcreate` or to explicitly synchronize between writing
+the archive and reading it, e.g.
+
+.. code-block:: console
+
+    flux exec -r all flux kvs wait $(flux kvs version)
+
+
+EXAMPLES
+========
+
+Example 1:  a batch script that archives data from ``/project/dataset1``, then
+replicates it in a temporary directory on each node of the batch allocation
+where it can be used by multiple jobs.
+
+.. code-block:: console
+
+  #!/bin/bash
+
+  flux archive create -C /project dataset1
+  flux exec -r all mkdir -p /tmp/project
+  flux exec -r all flux archive extract --waitcreate -C /tmp/project
+
+  # app1 and app2 have access to local copy of dataset1
+  flux run -N1024 app1 --input=/tmp/project/dataset1
+  flux run -N1024 app2 --input=/tmp/project/dataset1
+
+  # clean up
+  flux exec -r all rm -rf /tmp/project
+  flux archive remove
+
+Example 2: a batch script that archives a large executable and a data set,
+then uses the ``stage-in`` shell plugin to copy them to
+:envvar:`FLUX_JOB_TMPDIR` which is automatically cleaned up after each job.
+
+.. code-block:: console
+
+  #!/bin/bash
+
+  flux archive create --name=dataset1 -C /project dataset1
+  flux archive create --name=app --mmap -C /home/fred app
+
+  flux run -N1024 -o stage-in.names=app,dataset1 \
+      {{tmpdir}}/app --input={{tmpdir}}/dataset1
+
+  # clean up
+  flux archive remove --name=dataset1
+  flux archive remove --name=app
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+FLUX RFC
+========
+
+| :doc:`rfc:spec_16`
+| :doc:`rfc:spec_23`
+| :doc:`rfc:spec_37`
+
+
+SEE ALSO
+========
+
+:man1:`flux-shell`, :man1:`flux-kvs`, :man1:`flux-exec`
diff --git a/doc/man1/flux-batch.rst b/doc/man1/flux-batch.rst
new file mode 100644
index 000000000000..092314235358
--- /dev/null
+++ b/doc/man1/flux-batch.rst
@@ -0,0 +1,227 @@
+.. flux-help-include: true
+.. flux-help-section: submission
+
+=============
+flux-batch(1)
+=============
+
+
+SYNOPSIS
+========
+
+**flux** **batch** [OPTIONS] SCRIPT ...
+
+**flux** **batch** [OPTIONS] --wrap COMMAND ...
+
+
+DESCRIPTION
+===========
+
+.. program:: flux batch
+
+:program:`flux-batch` submits *SCRIPT* to run as the initial program of a Flux
+subinstance.  *SCRIPT* refers to a file that is copied at the time of
+submission.  Once resources are allocated, *SCRIPT* executes on the first
+node of the allocation, with any remaining free arguments supplied as *SCRIPT*
+arguments.  Once *SCRIPT* exits, the Flux subinstance exits and resources are
+released to the enclosing Flux instance.
+
+If there are no free arguments, the script is read from standard input.
+
+If the :option:`--wrap` option is used, the script is created by wrapping the
+free arguments or standard input in a shell script prefixed with ``#!/bin/sh``.
+
+If the job request is accepted, its jobid is printed on standard output and the
+command returns.  The job runs when the Flux scheduler fulfills its resource
+allocation request.  :man1:`flux-jobs` may be used to display the job status.
+
+Flux commands that are run from the batch script refer to the subinstance.
+For example, :man1:`flux-run` would launch work there.  A Flux command run
+from the script can be forced to refer to the enclosing instance by supplying
+the :option:`flux --parent` option.
+
+Flux commands outside of the batch script refer to their enclosing instance,
+often a system instance.  :man1:`flux-proxy` establishes a connection to a
+running subinstance by jobid, then spawns a shell in which Flux commands refer
+to the subinstance.  For example:
+
+::
+
+   $ flux uptime
+    07:48:42 run 2.1d,  owner flux,  depth 0,  size 8
+   $ flux batch -N 2 --queue=batch mybatch.sh
+   ƒM7Zq9AKHno
+   $ flux proxy ƒM7Zq9AKHno
+   $ flux uptime
+    07:47:18 run 1.6m,  owner user42,  depth 1,  size 2
+   $ flux top
+   ...
+   $ exit
+   $
+
+Other commands accept a jobid argument and establish the connection
+automatically.  For example:
+
+::
+
+   $ flux batch -N 2 --queue=batch mybatch.sh
+   ƒM7Zq9AKHno
+   $ flux top ƒM7Zq9AKHno
+   ...
+   $
+
+Batch scripts may contain submission directives denoted by ``flux:``
+as described in RFC 36.  See `SUBMISSION DIRECTIVES`_ below.
+
+The available OPTIONS are detailed below.
+
+JOB PARAMETERS
+==============
+
+:man1:`flux-batch` and :man1:`flux-alloc` do not launch tasks directly, and
+therefore job parameters are specified in terms of resource slot size
+and number of slots. A resource slot can be thought of as the minimal
+resources required for a virtual task. The default slot size is 1 core.
+
+.. include:: common/job-param-batch.rst
+
+.. include:: common/job-param-additional.rst
+
+STANDARD I/O
+============
+
+For :man1:`flux-batch` the default :option:`--output` *TEMPLATE*
+is *flux-{{id}}.out*.  To force output to KVS so it is available with
+``flux job attach`` or :man1:`flux-watch` , set the :option:`--output`
+*TEMPLATE* to *none* or *kvs*.
+
+.. include:: common/job-standard-io.rst
+
+CONSTRAINTS
+===========
+
+.. include:: common/job-constraints.rst
+
+DEPENDENCIES
+============
+
+.. include:: common/job-dependencies.rst
+
+ENVIRONMENT
+===========
+
+By default, :man1:`flux-batch` duplicates the current environment when
+submitting jobs. However, a set of environment manipulation options
+are provided to give fine control over the requested environment
+submitted with the job.
+
+.. note::
+   The actual environment of the initial program is subject to the caveats
+   described in the :ref:`initial_program_environment` section of
+   :man7:`flux-environment`.
+
+.. include:: common/job-environment.rst
+
+ENV RULES
+---------
+
+.. include:: common/job-env-rules.rst
+
+PROCESS RESOURCE LIMITS
+=======================
+
+By default these commands propagate some common resource limits (as described
+in :linux:man2:`getrlimit`) to the job by setting the ``rlimit`` job shell
+option in jobspec.  The set of resource limits propagated can be controlled
+via the :option:`--rlimit=RULE` option:
+
+.. include:: common/job-process-resource-limits.rst
+
+EXIT STATUS
+===========
+
+The job exit status is normally the batch script exit status.
+This result is stored in the KVS.
+
+OTHER OPTIONS
+=============
+
+.. include:: common/job-other-options.rst
+
+.. include:: common/job-other-batch.rst
+
+SHELL OPTIONS
+=============
+
+Some options that affect the parallel runtime environment of the Flux instance
+started by :program:`flux batch` are provided by the Flux shell.
+These options are described in detail in the
+:ref:`SHELL OPTIONS <flux_shell_options>` section of :man1:`flux-shell`.
+A list of the most commonly needed options follows.
+
+Usage: :option:`flux batch -o NAME[=ARG]`.
+
+.. make :option: references in the included table x-ref to flux-shell(1)
+.. program:: flux shell
+.. include:: common/job-shell-options.rst
+.. program:: flux batch
+
+SUBMISSION DIRECTIVES
+=====================
+
+The :program:`flux batch` command supports submission directives
+mixed within the submission script. The submission directive specification
+is fully detailed in RFC 36, but is summarized here for convenience:
+
+ * A submission directive is indicated by a line that starts with
+   a prefix of non-alphanumeric characters followed by a tag ``FLUX:`` or
+   ``flux:``. The prefix plus tag is called the *directive sentinel*. E.g.,
+   in the example below the sentinel is ``# flux:``: ::
+
+     #!/bin/sh
+     # flux: -N4 -n16
+     flux run -n16 hostname
+
+ * All directives in a file must use the same sentinel pattern, otherwise
+   an error will be raised.
+ * Directives must be grouped together - it is an error to include a
+   directive after any non-blank line that doesn't start with the common
+   prefix.
+ * The directive starts after the sentinel to the end of the line.
+ * The ``#`` character is supported as a comment character in directives.
+ * UNIX shell quoting is supported in directives.
+ * Triple quoted strings can be used to include newlines and quotes without
+   further escaping. If a triple quoted string is used across multiple lines,
+   then the opening and closing triple quotes must appear at the end of the
+   line. For example ::
+
+     # flux: --setattr=user.conf="""
+     # flux: [config]
+     # flux:   item = "foo"
+     # flux: """
+
+Submission directives may be used to set default command line options for
+:program:`flux batch` for a given script. Options given on the command line
+override those in the submission script, e.g.: ::
+
+   $ flux batch --job-name=test-name --wrap <<-EOF
+   > #flux: -N4
+   > #flux: --job-name=name
+   > flux run -N4 hostname
+   > EOF
+   ƒ112345
+   $ flux jobs -no {name} ƒ112345
+   test-name
+
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+SEE ALSO
+========
+
+:man1:`flux-submit`, :man1:`flux-run`, :man1:`flux-alloc`,
+:man1:`flux-bulksubmit` :man7:`flux-environment`
diff --git a/doc/man1/flux-broker.adoc b/doc/man1/flux-broker.adoc
deleted file mode 100644
index 8dca5f3d153f..000000000000
--- a/doc/man1/flux-broker.adoc
+++ /dev/null
@@ -1,94 +0,0 @@
-// flux-help-description: Invoke Flux comms message broker daemon
-FLUX-BROKER(1)
-==============
-:doctype: manpage
-
-
-NAME
-----
-flux-broker - Flux comms message broker daemon
-
-
-SYNOPSIS
---------
-*flux-broker* ['OPTIONS'] ['initial-program' ['args...']]
-
-
-DESCRIPTION
------------
-flux-broker(1) is a distributed message broker daemon that provides
-communications services within a Flux comms session.  It may be
-launched as a parallel program under Flux or other resource managers
-that support PMI.
-
-Resource manager services are implemented as dynamically loadable
-plugins to flux-broker(1), termed "comms modules".
-
-flux-broker(1) instances within a comms session are interconnected using
-ZeroMQ sockets, and each is assigned a rank from 0 to size - 1.
-The rank 0 node is the root of a tree-based overlay network.
-In addition, there is a multicast pub/sub network for events, and
-a ring network for debugging.  These networks may be accessed by
-Flux commands and comms modules using Flux API services.
-
-A periodic heartbeat event generated by rank 0 is used to synchronize
-resource manager activities across the session.
-
-A logging service aggregates Flux log messages across the session and
-emits them to a configured destination on rank 0.
-
-After its overlay networks have completed wire-up, flux-broker(1)
-starts the initial program on rank 0.  If none is specified on
-the broker command line, an interactive shell is launched.
-
-
-OPTIONS
--------
-*-h, --help*::
-Summarize available options.
-
-*-v, --verbose*::
-Be annoyingly chatty.
-
-*-S, --setattr*='ATTR=VAL'::
-Set initial value for broker attribute.
-
-*-k, --k-ary*='N'::
-Set the branching factor of this comms session's tree based overlay
-network (default: 2).
-
-*-H, --heartrate*='N.N'::
-Set the session heartrate in seconds.  The valid range is 0.01 to 30.0
-(default: 2.0).
-
-*-X, --module-path*='PATH'::
-Override the compiled-in module search path (colon-separated).
-The default is to search install paths.
-
-*-s, --security*='MODE'::
-Set the security mode.  The mode may be 'none', 'plain', or 'curve'
-(default: curve).  See flux-keygen(1) for more information.
-
-*-g, --shutdown-grace*='SECS'::
-Specify the shutdown grace period, in seconds (default: guess based
-on session size).
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-flux-broker-attributes(7)
diff --git a/doc/man1/flux-broker.rst b/doc/man1/flux-broker.rst
new file mode 100644
index 000000000000..b3d435f7b630
--- /dev/null
+++ b/doc/man1/flux-broker.rst
@@ -0,0 +1,70 @@
+==============
+flux-broker(1)
+==============
+
+
+SYNOPSIS
+========
+
+**flux-broker** [*OPTIONS*] [*initial-program* [*args...*]]
+
+DESCRIPTION
+===========
+
+.. program:: flux broker
+
+:program:`flux broker` is a distributed message broker daemon that provides
+communications services within a Flux instance. It may be
+launched as a parallel program under Flux or other resource managers
+that support PMI.
+
+Resource manager services are implemented as dynamically loadable
+modules.
+
+Brokers within a Flux instance are interconnected using
+ZeroMQ sockets, and each is assigned a rank from 0 to size - 1.
+The rank 0 node is the root of a tree-based overlay network.
+This network may be accessed by Flux commands and modules
+using Flux API services.
+
+A logging service aggregates Flux log messages across the instance and
+emits them to a configured destination on rank 0.
+
+After its overlay network has completed wire-up, :program:`flux broker`
+starts the initial program on rank 0. If none is specified on
+the broker command line, an interactive shell is launched.
+
+
+OPTIONS
+=======
+
+.. option:: -h, --help
+
+   Summarize available options.
+
+.. option:: -v, --verbose
+
+   Be annoyingly chatty.
+
+.. option:: -S, --setattr=ATTR=VAL
+
+   Set initial value for broker attribute.
+
+.. option:: -c, --config-path=PATH
+
+   Set the PATH to broker configuration. If PATH is a directory, then
+   read all TOML files from that directory. If PATH is a file, then load
+   configuration as JSON if the file extension is ``.json``, otherwise
+   load the file as TOML.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man7:`flux-broker-attributes`
diff --git a/doc/man1/flux-bulksubmit.rst b/doc/man1/flux-bulksubmit.rst
new file mode 100644
index 000000000000..f530ee1a256b
--- /dev/null
+++ b/doc/man1/flux-bulksubmit.rst
@@ -0,0 +1,252 @@
+.. flux-help-include: true
+.. flux-help-section: submission
+
+==================
+flux-bulksubmit(1)
+==================
+
+
+SYNOPSIS
+========
+
+**flux** **bulksubmit** [OPTIONS] COMMAND...
+
+
+DESCRIPTION
+===========
+
+.. program:: flux bulksubmit
+
+:program:`flux bulksubmit` allows rapid bulk submission of jobs using
+an interface similar to GNU parallel or ``xargs``. The command takes
+inputs on stdin or the command line (separated by ``:::``), and submits
+the supplied command template and options as one job per input combination.
+
+The replacement is done using Python's ``string.format()``, which is
+supplied a list of inputs on each iteration. Therefore, in the common case
+of a single input list, ``{}`` will work as the substitution string, e.g.::
+
+    $ seq 1 4 | flux bulksubmit echo {}
+    bulksubmit: submit echo 1
+    bulksubmit: submit echo 2
+    bulksubmit: submit echo 3
+    bulksubmit: submit echo 4
+
+With :option:`--dry-run` :program:`flux bulksubmit` will print the args and
+command which would have been submitted, but will not perform any job
+submission.
+
+The :program:`flux bulksubmit` command can also take input lists on the command
+line.  The inputs are separated from each other and the command  with the
+special delimiter ``:::``::
+
+    $ flux bulksubmit echo {} ::: 1 2 3 4
+    bulksubmit: submit echo 1
+    bulksubmit: submit echo 2
+    bulksubmit: submit echo 3
+    bulksubmit: submit echo 4
+
+Multiple inputs are combined, in which case each input is passed as a
+positional parameter to the underlying ``format()``, so should be accessed
+by index::
+
+    $ flux bulksubmit --dry-run echo {1} {0} ::: 1 2 ::: 3 4
+    bulksubmit: submit echo 3 1
+    bulksubmit: submit echo 4 1
+    bulksubmit: submit echo 3 2
+    bulksubmit: submit echo 4 2
+
+If the generation of all combinations of an  input list with other inputs is not
+desired, the special input delimited ``:::+`` may be used to "link" the input,
+so that only one argument from this source will be used per other input,
+e.g.::
+
+    $ flux bulksubmit --dry-run echo {0} {1} ::: 1 2 :::+ 3 4
+    bulksubmit: submit 1 3
+    bulksubmit: submit 2 4
+
+The linked input will be cycled through if it is shorter than other inputs.
+
+An input list can be read from a file with ``::::``::
+
+    $ seq 0 3 >inputs
+    $ flux bulksubmit --dry-run :::: inputs
+    bulksubmit: submit 0
+    bulksubmit: submit 1
+    bulksubmit: submit 2
+    bulksubmit: submit 3
+
+If the filename is ``-`` then ``stdin`` will be used. This is useful
+for including ``stdin`` when reading other inputs.
+
+The delimiter ``::::+`` indicates that the next file is to be linked to
+the inputs instead of combined with them, as with ``:::+``.
+
+There are several predefined attributes for input substitution.
+These include:
+
+ - ``{.%}`` returns the input string with any extension removed.
+ - ``{./}`` returns the basename of the input string.
+ - ``{./%}`` returns the basename of the input string with any
+   extension removed.
+ - ``{.//}`` returns the dirname of the input string
+ - ``{seq}`` returns the input sequence number (0 origin)
+ - ``{seq1}`` returns the input sequence number (1 origin)
+ - ``{cc}`` returns the current ``id`` from use of :option:`--cc` or
+   :option:`--bcc`.  Note that replacement of ``{cc}`` is done in a second
+   pass, since the :option:`--cc` option argument may itself be replaced in
+   the first substitution pass. If :option:`--cc`/:option:`--bcc` were not
+   used, then ``{cc}`` is replaced with an empty string. This is the only
+   substitution supported with :man1:`flux-submit`.
+
+Note that besides ``{seq}``, ``{seq1}``, and ``{cc}`` these attributes
+can also take the input index, e.g. ``{0.%}`` or ``{1.//}``, when multiple
+inputs are used.
+
+Additional attributes may be defined with the :option:`--define` option, e.g.::
+
+    $ flux bulksubmit --dry-run --define=p2='2**int(x)' -n {.p2} hostname \
+       ::: $(seq 0 4)
+    bulksubmit: submit -n1 hostname
+    bulksubmit: submit -n2 hostname
+    bulksubmit: submit -n4 hostname
+    bulksubmit: submit -n8 hostname
+    bulksubmit: submit -n16 hostname
+
+The input string being indexed is passed to defined attributes via the
+local ``x`` as seen above.
+
+The available OPTIONS are detailed below.
+
+JOB PARAMETERS
+==============
+
+These commands accept only the simplest parameters for expressing
+the size of the parallel program and the geometry of its task slots:
+
+Common resource options
+-----------------------
+
+These commands take the following common resource allocation options:
+
+.. include:: common/job-param-common.rst
+
+Per-task options
+----------------
+
+:man1:`flux-run`, :man1:`flux-submit` and :man1:`flux-bulksubmit` take two
+sets of mutually exclusive options to specify the size of the job request.
+The most common form uses the total number of tasks to run along with
+the amount of resources required per task to specify the resources for
+the entire job:
+
+.. include:: common/job-param-pertask.rst
+
+Per-resource options
+--------------------
+
+The second set of options allows an amount of resources to be specified
+with the number of tasks per core or node set on the command line. It is
+an error to specify any of these options when using any per-task option
+listed above:
+
+.. include:: common/job-param-perres.rst
+
+Additional job options
+----------------------
+
+These commands also take following job parameters:
+
+.. include:: common/job-param-additional.rst
+
+STANDARD I/O
+============
+
+By default, task stdout and stderr streams are redirected to the
+KVS, where they may be accessed with the ``flux job attach`` command.
+
+In addition, :man1:`flux-run` processes standard I/O in real time,
+emitting the job's I/O to its stdout and stderr.
+
+.. include:: common/job-standard-io.rst
+
+CONSTRAINTS
+===========
+
+.. include:: common/job-constraints.rst
+
+DEPENDENCIES
+============
+
+.. include:: common/job-dependencies.rst
+
+ENVIRONMENT
+===========
+
+By default, these commands duplicate the current environment when submitting
+jobs. However, a set of environment manipulation options are provided to
+give fine control over the requested environment submitted with the job.
+
+.. include:: common/job-environment.rst
+
+ENV RULES
+---------
+
+.. include:: common/job-env-rules.rst
+
+PROCESS RESOURCE LIMITS
+=======================
+
+By default these commands propagate some common resource limits (as described
+in :linux:man2:`getrlimit`) to the job by setting the ``rlimit`` job shell
+option in jobspec.  The set of resource limits propagated can be controlled
+via the :option:`--rlimit=RULE` option:
+
+.. include:: common/job-process-resource-limits.rst
+
+EXIT STATUS
+===========
+
+The job exit status, normally the largest task exit status, is stored
+in the KVS. If one or more tasks are terminated with a signal,
+the job exit status is 128+signo.
+
+The ``flux-job attach`` command exits with the job exit status.
+
+In addition, :man1:`flux-run` runs until the job completes and exits
+with the job exit status.
+
+OTHER OPTIONS
+=============
+
+.. include:: common/job-other-options.rst
+
+.. include:: common/job-other-run.rst
+
+SHELL OPTIONS
+=============
+
+Some options that affect the parallel runtime environment are provided by the
+Flux shell.  These options are described in detail in the
+:ref:`SHELL OPTIONS <flux_shell_options>` section of :man1:`flux-shell`.
+A list of the most commonly needed options follows.
+
+Usage: :option:`flux bulksubmit -o NAME[=ARG]`.
+
+.. make :option: references in the included table x-ref to flux-shell(1)
+.. program:: flux shell
+.. include:: common/job-shell-options.rst
+.. program:: flux bulksubmit
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-run`, :man1:`flux-submit`, :man1:`flux-alloc`,
+:man1:`flux-batch`, :man7:`flux-environment`
+
diff --git a/doc/man1/flux-cancel.rst b/doc/man1/flux-cancel.rst
new file mode 100644
index 000000000000..8cf8dba402f7
--- /dev/null
+++ b/doc/man1/flux-cancel.rst
@@ -0,0 +1,63 @@
+.. flux-help-description: cancel one or more jobs
+.. flux-help-section: jobs
+
+==============
+flux-cancel(1)
+==============
+
+
+SYNOPSIS
+========
+
+**flux** **cancel** [*OPTIONS*] [*JOBID...*]
+
+DESCRIPTION
+===========
+
+.. program:: flux cancel
+
+:program:`flux cancel` cancels one or more jobs by raising a job exception of
+type=cancel. An optional message included with the cancel exception may be
+provided via the :option:`--message` option. Canceled jobs are immediately
+sent SIGTERM followed by SIGKILL after a configurable timeout (default=5s).
+
+:program:`flux cancel` can target multiple jobids by either taking them on the
+command line, or via the selection options :option:`--all`, :option:`--user`,
+or :option:`--states`. It is an error to provide jobids on the command line
+and use one or more of the selection options.
+
+By default :option:`--all` will target all jobs for the current user. To
+target all jobs for all users, use :option:`--user=all` (only the instance
+owner is allowed to use :option:`--user=all`). To see how many jobs
+:program:`flux cancel` would kill, use the :option:`--dry-run` option.
+
+OPTIONS
+=======
+
+.. option:: -n, --dry-run
+
+   Do not cancel any jobs, but print a message indicating how many jobs
+   would have been canceled.
+
+.. option:: -m, --message=NOTE
+
+   Set an optional exception note.
+
+.. option:: -u, --user=USER
+
+   Set target user.  The instance owner may specify *all* for all users.
+
+.. option:: -S, --states=STATES
+
+   Set target job states (default: active). Valid states include
+   depend, priority, sched, run, pending, running, active.
+
+.. option:: -q, --quiet
+
+   Suppress output if no jobs match
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
diff --git a/doc/man1/flux-config.rst b/doc/man1/flux-config.rst
new file mode 100644
index 000000000000..4c5ec92f9f72
--- /dev/null
+++ b/doc/man1/flux-config.rst
@@ -0,0 +1,215 @@
+.. flux-help-description: Manage/query Flux configuration
+
+==============
+flux-config(1)
+==============
+
+
+SYNOPSIS
+========
+
+| **flux** **config** **get** [*--default=VALUE*] [*--type=TYPE*] [*NAME*]
+| **flux** **config** **builtin** [*NAME*]
+| **flux** **config** **load** [*PATH*]
+| **flux** **config** **reload**
+
+
+DESCRIPTION
+===========
+
+The :program:`flux config` manipulates the configuration of the local broker.
+
+COMMANDS
+========
+
+get
+---
+
+.. program:: flux config get
+
+:program:`flux config get` queries the TOML configuration for a given Flux
+broker.  if *NAME* is unspecified, it dumps the entire configuration object.
+Otherwise, *NAME* is expected to be a period-delimited path name representing
+a TOML key.  Return values are printed in string-encoded JSON form, except for
+string values, which are printed without quotes to simplify their use in shell
+scripts.
+
+.. option:: -d, --default=VALUE
+
+   Substitute *VALUE* if *NAME* is not set in the configuration, and exit
+   with a return code of zero.
+
+.. option:: -q, --quiet
+
+   Suppress printing of errors if *NAME* is not set and :option:`--default` was
+   not specified.  This may be convenient to avoid needing to redirect standard
+   error in a shell script.
+
+.. option:: -t, --type=TYPE
+
+   Require that the value has the specified type, or exit with a nonzero exit
+   code.  Valid types are *string*, *integer*, *real*, *boolean*, *object*, and
+   *array*.  In addition, types of *fsd*, *fsd-integer*, and *fsd-real* ensure
+   that a value is a both a string and valid Flux Standard Duration.
+   *fsd* prints the value in its human-readable, string form. *fsd-integer*
+   and *fsd-real* print the value in integer and real seconds, respectively.
+
+.. option:: -c, --config-path=PATH
+
+   Read configuration from PATH instead of fetching configuration from local
+   broker.  If PATH is a directory, then read all TOML files from that
+   directory. If PATH is a file, then load configuration as JSON if the file
+   extension is ``.json``, otherwise load the file as TOML.  As a special case,
+   ``system``, ``security``, and ``imp`` may be used as shorthand for the
+   compiled-in paths to system configuration objects.
+
+
+builtin
+-------
+
+.. program:: flux config builtin
+
+:program:`flux config builtin` prints compiled-in Flux configuration values.
+See `BUILTIN VALUES`_ below for a list of builtin
+configuration key names.  This command is available to all users.
+
+.. note::
+   :program:`flux config get` and :program:`flux config builtin` refer to
+   disjoint key namespaces.  Flux behavior is determined by a combination of
+   these values, :man7:`flux-broker-attributes`, and other factors.  This
+   disjoint configuration scheme is subject to change in future releases of
+   Flux.
+
+.. note::
+   :program:`flux config builtin` uses a heuristic to determine if :man1:`flux`
+   was run from the flux-core source tree, and substitutes source tree
+   specific values if found to be in tree.  This enables Flux testing without
+   requiring installation.
+
+.. option:: --intree
+   Force :program:`flux config builtin` to return in-tree paths.
+
+.. option:: --installed
+   Force :program:`flux config builtin` to return installed paths.
+
+load
+----
+
+.. program:: flux config load
+
+:program:`flux config load` replaces the current config with an object read
+from standard input (JSON or TOML), or from ``*.toml`` in *PATH*, if specified.
+
+reload
+------
+
+.. program:: flux config reload
+
+:program:`flux config reload` tells :man1:`flux-broker` to reload its TOML
+configuration after it has been modified.
+
+On Flux instances started with :linux:man1:`systemd`,
+:program:`systemctl reload flux` invokes this command.
+This command is restricted to the instance owner.
+
+This command does not have an immediate effect in all cases.  For more
+information, refer to the :ref:`flux_config_caveats` section of
+:man5:`flux-config`.
+
+
+BUILTIN VALUES
+==============
+
+The following configuration keys may be printed with
+:program:`flux config builtin`:
+
+**rc1_path**
+   The rc1 script path used by :man1:`flux-broker`, unless overridden by
+   the ``broker.rc1_path`` broker attribute.
+
+**rc3_path**
+   The rc3 script path used by :man1:`flux-broker`, unless overridden by
+   the ``broker.rc1_path`` broker attribute.
+
+**shell_path**
+   The path to the :man1:`flux-shell` executable used by the exec service.
+
+**shell_pluginpath**
+   The search path used by :man1:`flux-shell` to locate plugins, unless
+   overridden by setting the ``conf.shell_pluginpath`` broker attribute.
+
+**shell_initrc**
+   The initrc script path used by :man1:`flux-shell`, unless overridden by
+   setting the ``conf.shell_initrc`` broker attribute.
+
+**jobtap_pluginpath**
+   The search path used by the job manager to locate
+   :man7:`flux-jobtap-plugins`.
+
+**rundir**
+   The configured ``${runstatedir}/flux`` directory.
+
+**lua_cpath_add**
+   Consulted by :man1:`flux` when setting the :envvar:`LUA_CPATH` environment
+   variable.
+
+**lua_path_add**
+   Consulted by :man1:`flux` when setting the :envvar:`LUA_PATH` environment
+   variable.
+
+**python_path**
+   Consulted by :man1:`flux` when setting the :envvar:`PYTHONPATH` environment
+   variable.
+
+**man_path**
+   Consulted by :man1:`flux` when setting the :envvar:`MANPATH` environment
+   variable.
+
+**exec_path**
+   Consulted by :man1:`flux` when setting the :envvar:`FLUX_EXEC_PATH`
+   environment variable.
+
+**connector_path**
+   Consulted by :man1:`flux` when setting the :envvar:`FLUX_CONNECTOR_PATH`
+   environment variable.
+
+**module_path**
+   Consulted by :man1:`flux` when setting the :envvar:`FLUX_MODULE_PATH`
+   environment variable.
+
+**pmi_library_path**
+   Consulted by the :man1:`flux-shell` pmi plugin when setting the
+   :envvar:`FLUX_PMI_LIBRARY_PATH` environment variable.
+
+**cmdhelp_pattern**
+   Used by :man1:`flux` to generate a list of common commands when run without
+   arguments.
+
+**no_docs_path**
+
+
+EXAMPLES
+========
+
+::
+
+   $ flux config get --type=fsd-integer tbon.tcp_user_timeout
+   60
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_23`
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`
diff --git a/doc/man1/flux-content.adoc b/doc/man1/flux-content.adoc
deleted file mode 100644
index 653b3ab3dca8..000000000000
--- a/doc/man1/flux-content.adoc
+++ /dev/null
@@ -1,158 +0,0 @@
-// flux-help-command: content
-// flux-help-description: Access instance content storage
-FLUX-CONTENT(1)
-===============
-:doctype: manpage
-
-
-NAME
-----
-flux-content - access content service
-
-
-SYNOPSIS
---------
-*flux* *content* *load* ['--bypass-cache'] 'blobref'
-
-*flux* *content* *store* ['--bypass-cache']
-
-*flux* *content* *flush*
-
-*flux* *content* *dropcache*
-
-
-DESCRIPTION
------------
-Each Flux instance implements an append-only, content addressable
-storage service, which stores blobs of arbitrary content under
-message digest keys termed "blobrefs".
-
-*flux content store* accepts a blob on standard input, stores it,
-and prints the blobref on standard output.
-
-*flux content load* accepts a blobref argument, retrieves the
-corresponding blob, and writes it to standard output.
-
-After a store operation completes on any rank, the blob may be
-retrieved from any other rank.
-
-The content service includes a cache on each broker which improves
-scalability.   The *flux content flush* command initiates store requests
-for any dirty entries in the local cache and waits for them to complete.
-This is mainly used in testing.  The *flux content dropcache* command
-drops all non-essential entries in the local cache; that is, entries
-which can be removed without data loss.
-
-
-OPTIONS
--------
-*-b, --bypass-cache*::
-Bypass the in-memory cache, and directly access the backing store,
-if available (see below).
-
-BACKING STORE
--------------
-The rank 0 cache retains all content until a module providing
-the "content.backing" service is loaded which can offload content
-to some other place.  The *content-sqlite* module provides this
-service, and is loaded by default.
-
-Content database files are stored persistently on rank 0 if the
-persist-directory broker attribute is set to a directory name for
-the session.  Otherwise they are stored in the directory defined
-by the scratch-directory attribute and are cleaned up when the
-instance terminates.
-
-When one of these modules is loaded, it informs the rank 0
-cache of its availability, which triggers the cache to begin
-offloading entries.  Once entries are offloaded, they are eligible
-for expiration from the rank 0 cache.
-
-To avoid data loss, once a content backing module is loaded,
-do not unload it unless the content cache on rank 0 has been flushed
-and the system is shutting down.
-
-
-CACHE EXPIRATION
-----------------
-The parameters affecting local cache expiration may be tuned with
-flux-setattr(1):
-
-*content.purge-target-entries*::
-The cache is purged to bring the number of cache entries less than
-or equal to this value
-(default 1048576).
-
-*content.purge-target-size*::
-The cache is purged to bring the sum of the size of cached blobs less
-than or equal to this value
-(default 16777216)
-
-*content.purge-old-entry*::
-Only entries that have not been accessed in *old-entry* heartbeat epochs
-are eligible for purge (default 5).
-
-*content.purge-large-entry*::
-Only entries with blob size greater than or equal to *large-entry* are
-purged to reach the size target (default 256).
-
-Expiration becomes active on every heartbeat, when the cache exceeds one
-or both of the targets configured above.  Dirty or invalid entries are
-not eligible for purge.
-
-
-CACHE ACCOUNTING
-----------------
-Some accounting info for the local cache can be viewed with flux-getattr(1):
-
-*content.acct-entries*::
-The total number of cache entries.
-
-*content.acct-size*::
-The sum of the size of cached blobs.
-
-*content.acct-dirty*::
-The number of dirty cache entries.
-
-*content.acct-valid*::
-The number of valid cache entries.
-
-
-CACHE SEMANTICS
----------------
-The cache is write-through with respect to the rank 0 cache;
-that is, a store operation does not receive a response until it
-is valid in the rank 0 cache.
-
-The cache on rank 0 is write-back with respect to the backing store,
-if any;  that is, a store operation may receive a response before
-it has been stored on the backing store.
-
-The cache is hierarchical.  Rank 0 (the root of the tree based
-overlay network) holds all blobs stored in the instance.
-Other ranks keep only what a they heuristically determine to
-be of benefit.  On ranks > 0, a load operation that cannot be fulfilled
-from the local cache is "faulted" in from the level above it.
-A store operation that reaches a level that has already cached the
-same content is "squashed"; that is, it receives a response without
-traveling further up the tree.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-https://github.com/flux-framework/rfc/blob/master/spec_10.adoc[RFC 10: Content Store]
diff --git a/doc/man1/flux-content.rst b/doc/man1/flux-content.rst
new file mode 100644
index 000000000000..758952c4d056
--- /dev/null
+++ b/doc/man1/flux-content.rst
@@ -0,0 +1,136 @@
+===============
+flux-content(1)
+===============
+
+
+SYNOPSIS
+========
+
+| **flux** **content** **load** [*--bypass-cache*] [*blobref* ...]
+| **flux** **content** **store** [*--bypass-cache*] [*--chunksize=N*]
+| **flux** **content** **flush**
+| **flux** **content** **dropcache**
+
+
+DESCRIPTION
+===========
+
+Each Flux instance implements an append-only, content addressable storage
+service.  The content service stores blobs of arbitrary data under
+"blobref" keys.  Blobrefs are derived from a hash of the data and thus can
+be computed in advance and always refer to the same blob.
+
+The leader broker (rank 0) holds the full data set, and normally offloads
+blobs to a sqlite database on disk.  The database usually resides in the
+broker ``rundir`` which is created anew when Flux starts, and is cleaned
+up when Flux terminates.  However if the ``statedir`` broker attribute is
+set, the database resides there and can persist across Flux restarts, but
+see `CAVEATS`_ below.
+
+The content service was designed for, and is primarily used by, the Flux KVS.
+Access is restricted to the instance owner.
+
+
+COMMANDS
+========
+
+store
+-----
+
+.. program:: flux content store
+
+:program:`flux content store` reads data from standard input to EOF, stores it
+(possibly splitting into multiple blobs), and prints blobref(s) on
+standard output, one per line.
+
+After a store operation completes on any rank, the blobs may be
+retrieved from any other rank.
+
+.. option:: -b, --bypass-cache
+
+   Bypass the in-memory cache, and directly access the backing store,
+   if available.
+
+.. option:: --chunksize=N
+
+   Split a blob into chunks of *N* bytes.
+
+load
+----
+
+.. program:: flux content load
+
+:program:`flux content load` reads blobrefs from standard input, one per line,
+or parses blobrefs on the command line (but not both).  It then loads the
+corresponding blob(s), and concatenates them on standard output.
+
+.. option:: -b, --bypass-cache
+
+   Bypass the in-memory cache, and directly access the backing store,
+   if available.
+
+flush
+-----
+
+.. program:: flux content flush
+
+The content service includes a cache on each broker which improves
+scalability. The :program:`flux content flush` command initiates store requests
+for any dirty entries in the local cache and waits for them to complete.
+This is mainly used in testing.
+
+dropcache
+---------
+
+.. program:: flux content dropcache
+
+The :program:`flux content dropcache` command drops all non-essential entries
+in the local cache; that is, entries which can be removed without data loss.
+
+
+CAVEATS
+=======
+
+The KVS implements its hierarchical key space using a hash tree, where
+the hashes refer to content entries.  As the KVS is used, the append-only
+nature of the content service results in an accumulation of unreferenced
+data.  In restartable Flux instances, this is mitigated by
+:option:`flux shutdown --gc` offline garbage collection, where a dump of
+the current KVS root snapshot is created at shutdown, and the content
+database is removed and recreated from the dump at restart.  This presents
+a problem for other users of the content service.  If content needs to be
+preserved in this situation, the best recourse is to ensure it is linked
+into the KVS hash tree before the instance is shut down. The
+:option:`flux kvs put --treeobj` option is available for this purpose.
+
+A large or long-running Flux instance might generate a lot of content
+that is offloaded to ``rundir`` on the leader broker.  If the file system
+(usually ``/tmp``) containing ``rundir`` is a ramdisk, this can lead to less
+memory available for applications on the leader broker, or to catastrophic
+failures if the file system fills up.  Some workarounds for batch jobs are::
+
+  # exclude the leader (rank 0) broker from scheduling
+  flux batch --conf=resource.exclude=\"0\"
+
+  # redirect storage to a global file system (pre-create empty)
+  flux batch --broker-opts=--setattr=statedir=/path/to/directory
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_10`
+
+:doc:`rfc:spec_11`
+
+
+SEE ALSO
+========
+
+:man1:`flux-kvs`, :man7:`flux-broker-attributes`
diff --git a/doc/man1/flux-cron.adoc b/doc/man1/flux-cron.adoc
deleted file mode 100644
index 91c96c70b52f..000000000000
--- a/doc/man1/flux-cron.adoc
+++ /dev/null
@@ -1,286 +0,0 @@
-// flux-help-description: Schedule tasks on timers and events
-FLUX-CRON(1)
-===========
-:doctype: manpage
-
-NAME
-----
-flux-cron - Cron-like utility for Flux
-
-SYNOPSIS
---------
-*flux* *cron* 'COMMAND' ['OPTIONS']
-
-DESCRIPTION
------------
-The Flux cron service offers an interface for executing commands on
-triggers such as a time interval or Flux events. The service is
-implemented as a Flux extension module which, when loaded, manages
-a set of cron entries and uses the built-in 'cmb.exec' service to run
-a command associated with the entry each time the defined trigger is
-reached. As with 'flux-exec(1)', these tasks run as direct children
-of the flux-broker and run outside of the control of any loaded
-job scheduling service.
-
-The flux-cron(1) utility offers an interface to create, stop, start,
-query, and destroy these entries in the Flux cron service.
-
-For a detailed description of the cron service operation and how
-it executes tasks, see the OPERATION and TASK EXECUTION sections
-below.
-
-COMMANDS
---------
-*help* 'cmd'::
-Print help. If 'cmd' is provided, print help for that sub-command.
-
-*sync* [--epsilon='delay'] ['topic']::
-Query and modify the current *sync-event* behavior for the cron module.
-If a sync-event is set, the cron module will defer all task execution
-until an event matching the sync-event 'topic' is received.  With '--epsilon'
-the cron module will *not* delay task execution if the task is normally
-scheduled to run within 'delay' of the matching event. Without any
-'topic' supplied on command line, 'flux cron sync' displays the current
-setting for sync. If a task is deferred due to sync-event, the
-'stats.deferred' statistic is incremented.
-
-*interval* [OPTIONS] 'interval' 'command'::
-Create a cron entry to execute 'command' every 'interval', where 'interval'
-is an arbitrary floating point duration with optional suffix 's' for
-seconds, 'm' for minutes, 'h' for hours and 'd' for days.
-Options:
-
- --name='STRING':::
- -n 'STRING':::
- Set a name for this cron entry to 'STRING'.
-
- --after='TIME':::
- -a 'TIME':::
- The first task will run after a delay of 'TIME' instead of 'interval'.
- After the first task the entry will continue to execute every 'interval'.
-
- --count='N':::
- -c 'N':::
- The entry will be run a total of 'N' times, then stopped.
-
- --options='LIST':::
- -o 'LIST':::
- The '--options' option allows a comma separated list of extra options to be
- passed to the flux-cron service. See EXTRA OPTIONS below.
-
- --preserve-env:::
- -E:::
- The '--preserve-env' option allows the current environment to be exported
- and used for the command being executed as part of the cron job. Normally,
- the broker environment is used.
-
- --working-dir='DIR':::
- -d 'DIR':::
- The '--working-dir' option allows the working directory to be set for the command
- being executed as part of the cron job. Normally, the working directory of
- the broker is used.
-
-*event* [OPTIONS] 'topic' 'command'::
-
-Create a cron entry to execute 'command' after every event matching 'topic'.
-
- --name='STRING':::
- -n 'STRING':::
- Set a name for this cron entry to 'STRING'.
-
- --nth='N':::
- -n 'N':::
- If '--nth' is given then 'command' will be run after each 'N' events.
-
- --count='N':::
- -c 'N':::
- With '--count', the entry is run 'N' times then stopped.
-
- --after='N':::
- -a 'N':::
- Run the first task only after 'N' matching events. Then run every event
- or 'N' events with '--nth'.
-
- --min-interval='T':::
- -i 'T':::
- Set the minimum interval at which two cron jobs for this event will be run.
- For example, with --min-interval of 1s, the cron job will be at most run
- every 1s, even if events are generated more quickly.
-
- --options='LIST':::
- -o 'LIST':::
- Set comma separated EXTRA OPTIONS for this cron entry.
-
- --preserve-env:::
- -E:::
- The '--preserve-env' option allows the current environment to be exported
- and used for the command being executed as part of the cron job. Normally,
- the broker environment is used.
-
- --working-dir='DIR':::
- -d 'DIR':::
- The '--working-dir' option allows the working directory to be set for the command
- being executed as part of the cron job. Normally, the working directory of
- the broker is used.
-
-*tab* [OPTIONS] ['file'] ::
-Process one or more lines containing crontab expressions from 'file'
-(stdin by default) Each valid crontab line will result in a new cron
-entry registered with the flux-cron service. The cron expression format
-supported by `flux cron tab` has 5 fields: 'minutes' (0-59), 'hours'
-(0-23), 'day of month' (1-31), 'month' (0-11), and 'day of week' (0-6).
-Everything after the day of week is considered a command to be run.
-
- --options='LIST':::
- -o 'LIST':::
- Set comma separated EXTRA OPTIONS for all cron entries.
-
-*at* [OPTIONS] 'string' 'command'
-Run 'command' at specific date and time described by 'string'
-
- --options='LIST':::
- -o 'LIST':::
- Set comma separated EXTRA OPTIONS for all cron entries.
-
- --preserve-env:::
- -E:::
- The '--preserve-env' option allows the current environment to be exported
- and used for the command being executed as part of the cron job. Normally,
- the broker environment is used.
-
- --working-dir='DIR':::
- -d 'DIR':::
- The '--working-dir' option allows the working directory to be set for the command
- being executed as part of the cron job. Normally, the working directory of
- the broker is used.
-*list*::
-Display a list of current entries registered with the cron module and
-their current state, last run time, etc.
-
-*stop* 'id'::
-Stop cron entry 'id'. The entry will remain in the cron entry list until
-deleted.
-
-*start* 'id'::
-Start a stopped cron entry 'id'.
-
-*delete* [--kill] 'id'::
-Purge cron entry 'id' from the flux-cron entry list. If '--kill' is used,
-kill any running task associated with entry 'id'.
-
-*dump* [--key=KEY] 'id'::
-Dump all information for cron entry 'id'. With '--key' print only the value
-for key 'KEY'. For a list of keys run 'flux cron dump ID'.
-
-EXTRA OPTIONS
--------------
-
-For `flux-cron` commands allowing `--options`, the following EXTRA OPTIONS
-are supported:
-
-timeout='N'::
-Set a timeout for tasks invoked for this cron entry to 'N' seconds, where
-N can be a floating point number. Default is no timeout.
-
-rank='R'::
-Set the rank on which to execute the cron command to 'R'. Default is rank 0.
-
-task-history-count='N'::
-Keep history for the last 'N' tasks invoked by this cron entry. Default is 1.
-
-stop-on-failure='N'::
-Automatically stop a cron entry if the failure count exceeds 'N'. If 'N' is
-zero (the default) then the cron entry will not be stopped on failure.
-
-
-OPERATION
----------
-The Flux cron module manages the set of currently configured cron
-jobs as a set of common entries, each with a unique ID supplied by
-a global sequence number and set of common attributes, options, and
-statistics. Basic attributes of a cron job include an optional 'name',
-the 'command' to execute on the entry's trigger, the current 'state' of
-the cron entry (stopped or not stopped), a 'repeat' count indicating the
-total number of times to execute the cron job before stopping, and the
-'type' of entry.
-
-All cron entries also support a less common list of options, which may
-be set at creation time via a comma-separated list of 'option=value'
-parameters passed to the '-o', '--option=OPTS'. These options are described
-in the EXTRA OPTIONS section at the end of this document.
-
-Currently, flux-cron supports only two types of entries. The 'interval'
-entry supports executing a command once every configured duration,
-optionally starting after a different time period. More detailed
-information about the interval type can be found in the documentation for
-the flux-cron 'interval' command above. The 'event' type entry supports
-running a command once every N events matching the configured event topic.
-More information about this type can be found in the documentation for
-'flux cron event'.
-
-The Flux cron module additionally keeps a common set of statistics for
-each entry, regardless of type . These include the creation time, last
-run time, and last time the cron entry was "started", as well a count of
-total number of times the command was executed and a count of successful
-and failed runs. Currently, the stats for a cron entry may be viewed via
-the 'flux cron dump' subcommand 'stats.*' output.
-
-When registered, cron entries are automatically 'started', meaning they
-are eligible to run the configured command when the trigger condition
-is met. Entries may be 'stopped', either by use of the 'flux cron stop'
-command, or if a 'stop-on-failure' value is set. Stopped entries are
-restarted using 'flux cron start', at which point counters used for
-repeat and stop-on-failure are reset.
-
-Stopped entries are kept in the flux cron until deleted with 'flux
-cron delete'. Active cron entries may also be deleted, with currently
-executing tasks optionally killed if the '--kill' option is provided.
-
-
-TASK EXECUTION
---------------
-
-As related above, cron entry commands are executed via the 'cmb.exec'
-service, which is a low level execution service offered outside of any
-scheduler control, described in more detail in the 'flux-exec(1)' man
-page.
-
-Standard output and error from tasks executed by the cron service are
-logged and may be viewed with 'flux-dmesg(1)'. If a cron task exits
-with non-zero status, or fails to launch under the 'cmb.exec' service,
-a message is logged and the failure is added to the failure stats.
-On task failure, the cron job is stopped if 'stop-on-failure' is set, and
-the current failure count exceeds the configured value. By default,
-'stop-on-failure' is not set.
-
-By default, flux-cron module keeps information for the last task executed
-for each cron entry. This information can be viewed either via the
-'flux cron list' or 'flux cron dump ID' subcommands. Data such as
-start and end time, exit status, rank, and PID for the task is available.
-The number of tasks kept for each cron entry may be individually tuned
-via the 'task-history-count' option, described in the EXTRA OPTIONS section.
-
-Commands are normally executed immediately on the interval or event
-trigger for which they are configured. However, if the 'sync-event'
-option is active on the cron module, tasks execution will be deferred
-until the next synchronization event. See the documentation above
-for 'flux cron sync' for more information.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-SEE ALSO
---------
-flux-exec(1), flux-dmesg(1)
diff --git a/doc/man1/flux-cron.rst b/doc/man1/flux-cron.rst
new file mode 100644
index 000000000000..8017fce1e9ff
--- /dev/null
+++ b/doc/man1/flux-cron.rst
@@ -0,0 +1,404 @@
+============
+flux-cron(1)
+============
+
+
+SYNOPSIS
+========
+
+| **flux** **cron** **tab** [*-E*] [*-d* *DIR*] [*-o* *OPT...*] [*file*]
+| **flux** **cron** **at** [*-E*] [*-d* *DIR*] [*-o* *OPT...*] *time* *command*
+| **flux** **cron** **event** [*-E*] [*-d* *DIR*] [*-o* *OPT...*] *topic* *command*
+| **flux** **cron** **interval** [*-E*] [*-d* *DIR*] [*-o* *OPT...*] *interval* *command*
+
+| **flux** **cron** **list**
+| **flux** **cron** **stop** *ids...*
+| **flux** **cron** **start** *ids...*
+| **flux** **cron** **delete** [*--kill*] *ids...*
+| **flux** **cron** **dump** [*--key=KEY*] *ids...*
+| **flux** **cron** **sync** [*--disable*] [*--epsilon=TIME*] *topic*
+
+
+DESCRIPTION
+===========
+
+The Flux cron service offers an interface for executing commands on
+triggers such as a time interval or Flux events. The service is
+implemented as a Flux broker module which, when loaded, manages
+a set of cron entries and uses the built-in *broker.exec* service to run
+a command associated with the entry each time the defined trigger is
+reached. As with :man1:`flux-exec`, these tasks run as direct children
+of the :man1:`flux-broker` and run outside of the control of any loaded
+job scheduling service.
+
+The :program:`flux cron` utility offers an interface to create, stop, start,
+query, and destroy these entries in the Flux cron service.
+
+For a detailed description of the cron service operation and how
+it executes tasks, see the `OPERATION`_ and `TASK EXECUTION`_ sections
+below.
+
+
+COMMANDS
+========
+
+The following sub-commands are available:
+
+tab
+---
+
+.. program:: flux cron tab
+
+Process one or more lines containing crontab expressions from *file*
+(stdin by default).  Each valid crontab line will result in a new cron
+entry registered with the Flux cron service.
+
+Crontab lines have five standard fields, similar to :linux:man5:`crontab`:
+
+.. list-table::
+
+   * - minutes
+     - 0-59
+
+   * - hours
+     - 0-23
+
+   * - day of month
+     - 0-23
+
+   * - month
+     - 0-11
+
+   * - day of week
+     - 0-6
+
+Everything after the day of week is considered a command to be run.
+
+.. option:: -o, options=LIST
+
+  Set comma separated `EXTRA OPTIONS`_ for all cron entries.
+
+.. option:: -E, --preserve-env
+
+  Export the current environment to be used for the command being executed
+  as part of the cron job.  Normally, the broker environment is used.
+
+.. option:: -d, --working-dir=DIR
+
+  Set the working directory for commands being executed as part
+  of the cron job. Normally, the working directory of the broker is used.
+
+at
+--
+
+.. program:: flux cron at
+
+Run *command* at specific date and time described by *time*.  Any time
+string that can be parsed by :linux:man1:`date` is acceptable.
+
+.. option:: -o, options=LIST
+
+  Set comma separated `EXTRA OPTIONS`_ for all cron entries.
+
+.. option:: -E, --preserve-env
+
+  Export the current environment to be used for the command being executed
+  as part of the cron job.  Normally, the broker environment is used.
+
+.. option:: -d, --working-dir=DIR
+
+  Set the working directory for the command being executed as part
+  of the cron job. Normally, the working directory of the broker is used.
+
+event
+-----
+
+.. program:: flux cron event
+
+Create a cron entry to execute *command* after every event matching *topic*.
+
+.. option:: -N, --name=STRING
+
+  Set a name for this cron entry to *STRING*.
+
+.. option:: -n, --nth=N
+
+  If :option:`--nth` is given then *command* will be run after each *N* events.
+
+.. option:: -c, --count=N
+
+  With :option:`--count`, the entry is run *N* times then stopped.
+
+.. option:: -a, --after=N
+
+  Run the first task only after *N* matching events. Then run every event
+  or *N* events with :option:`--nth`.
+
+.. option:: -i, --min-interval=T
+
+  Set the minimum interval at which two cron jobs for this event will be run.
+  For example, with :option:`--min-interval` of 1s, the cron job will be
+  at most run every 1s, even if events are generated more quickly.
+
+.. option:: -o, --options=LIST
+
+  Set comma separated `EXTRA OPTIONS`_ for this cron entry.
+
+.. option:: -E, --preserve-env
+
+  Export the current environment to be used for the command being executed
+  as part of the cron job.  Normally, the broker environment is used.
+
+.. option:: -d, --working-dir=DIR
+
+  Set the working directory for the command being executed as part
+  of the cron job. Normally, the working directory of the broker is used.
+
+interval
+--------
+
+.. program:: flux cron interval
+
+Create a cron entry to execute *command* every *interval*, where *interval*
+is an arbitrary floating point duration with optional suffix *s* for
+seconds, *m* for minutes, *h* for hours and *d* for days.  If no suffix is
+specified, seconds is assumed.
+
+.. option:: -n, --name=STRING
+
+  Set a name for this cron entry to *STRING*.
+
+.. option:: -a, --after=TIME
+
+  The first task will run after a delay of *TIME* instead of *interval*, where
+  *TIME* is an arbitrary floating point duration specified in the same format
+  as *interval*.  After the first task the entry will continue to execute
+  every *interval*.
+
+.. option:: -c, --count=N
+
+  The entry will be run a total of *N* times, then stopped.
+
+.. option:: -o, --options=LIST
+
+  Set comma separated `EXTRA OPTIONS`_ for this cron entry.
+
+.. option:: -E, --preserve-env
+
+  Export the current environment to be used for the command being executed
+  as part of the cron job.  Normally, the broker environment is used.
+
+.. option:: -d, --working-dir=DIR
+
+  Set the working directory set for the command being executed as part
+  of the cron job. Normally, the working directory of the broker is used.
+
+
+list
+----
+
+.. program:: flux cron list
+
+Display a list of current entries registered with the cron module and
+their current state, last run time, etc.
+
+stop
+----
+
+.. program:: flux cron stop
+
+Stop cron entry *id*. The entry will remain in the cron entry list until
+deleted.
+
+start
+-----
+
+.. program:: flux cron start
+
+Start a stopped cron entry *id*.
+
+delete
+------
+
+.. program:: flux cron delete
+
+Purge cron entry *id* from the cron entry list.
+
+.. option:: -k, --kill
+
+Kill any running task associated with entry *id*.
+
+dump
+----
+
+.. program:: flux cron dump
+
+Dump all information for cron entry *id*.
+
+.. option:: -k, --key=KEY
+
+Print only the value for key *KEY*.
+
+For a list of keys run :option:`flux cron dump ID`.
+
+sync
+----
+
+.. program flux cron sync
+
+Query and modify the current **sync-event** behavior for the cron module.
+If a sync-event is set, the cron module will defer all task execution
+until an event matching the sync-event *topic* is received.
+
+Without any *topic* supplied on command line, :program:`flux cron sync`
+displays the current setting for sync.
+
+If a task is deferred due to sync-event, the *stats.deferred* statistic
+is incremented.
+
+.. option:: -e, --epsilon=TIME
+
+  Set amount of time after a *sync-event* that jobs are still allowed to
+  be run.  With this option, the cron module will **not** delay task execution
+  if the task is normally scheduled to run within *delay* of the matching
+  event.
+
+.. option:: -d, --disable
+
+  Disable the cron *sync-event*.
+
+
+EXTRA OPTIONS
+=============
+
+.. program:: flux cron tab
+
+For :program:`flux cron` commands allowing :option:`--options`, the following
+extra options are supported:
+
+.. option:: -o timeout=N
+
+   Set a timeout for tasks invoked for this cron entry to *N* seconds, where
+   N can be a floating point number. Default is no timeout.
+
+.. option:: -o rank=R
+
+   Set the rank on which to execute the cron command to *R*. Default is rank 0.
+
+.. option:: -o task-history-count=N
+
+   Keep history for the last *N* tasks invoked by this cron entry. Default is 1.
+
+.. option:: -o stop-on-failure=N
+
+   Automatically stop a cron entry if the failure count exceeds *N*. If *N* is
+   zero (the default) then the cron entry will not be stopped on failure.
+
+OPERATION
+=========
+
+The Flux cron module manages the set of currently configured cron
+jobs as a set of common entries, each with a unique ID supplied by
+a global sequence number and set of common attributes, options, and
+statistics. Basic attributes of a cron job include an optional *name*,
+the *command* to execute on the entry's trigger, the current *state* of
+the cron entry (stopped or not stopped), a *repeat* count indicating the
+total number of times to execute the cron job before stopping, and the
+*type* of entry.
+
+All cron entries also support a less common list of options, which may
+be set at creation time via a comma-separated list of *option=value*
+parameters passed to :option:`-o, --option=OPTS`. These options are described
+in the EXTRA OPTIONS section at the end of this document.
+
+Currently, Flux cron supports only two types of entries. The *interval*
+entry supports executing a command once every configured duration,
+optionally starting after a different time period. More detailed
+information about the interval type can be found in the documentation for
+the :program:`flux cron interval` command above. The *event* type entry supports
+running a command once every N events matching the configured event topic.
+More information about this type can be found in the documentation for
+:program:`flux cron event`.
+
+The Flux cron module additionally keeps a common set of statistics for
+each entry, regardless of type . These include the creation time, last
+run time, and last time the cron entry was "started", as well a count of
+total number of times the command was executed and a count of successful
+and failed runs. Currently, the stats for a cron entry may be viewed via
+the *flux cron dump* subcommand *stats.\** output.
+
+When registered, cron entries are automatically *started*, meaning they
+are eligible to run the configured command when the trigger condition
+is met. Entries may be *stopped*, either by use of the :program:`flux cron stop`
+command, or if a *stop-on-failure* value is set. Stopped entries are
+restarted using :program:`flux cron start`, at which point counters used for
+repeat and stop-on-failure are reset.
+
+Stopped entries are kept in the flux cron until deleted with
+:program:`flux cron delete`. Active cron entries may also be deleted, with
+currently executing tasks optionally killed if the :option:`--kill` option is
+provided.
+
+
+TASK EXECUTION
+==============
+
+As related above, cron entry commands are executed via the *broker.exec*
+service, which is a low level execution service offered outside of any
+scheduler control, described in more detail in the *flux-exec(1)* man
+page.
+
+Standard output and error from tasks executed by the cron service are
+logged and may be viewed with :man1:`flux-dmesg`. If a cron task exits
+with non-zero status, or fails to launch under the *broker.exec* service,
+a message is logged and the failure is added to the failure stats.
+On task failure, the cron job is stopped if *stop-on-failure* is set, and
+the current failure count exceeds the configured value. By default,
+*stop-on-failure* is not set.
+
+By default, the Flux cron module keeps information for the last task executed
+for each cron entry. This information can be viewed either via the
+:program:`flux cron list` or :program:`flux cron dump ID` subcommands. Data
+such as start and end time, exit status, rank, and PID for the task is
+available.  The number of tasks kept for each cron entry may be individually
+tuned via the :option:`--task-history-count` option, described in the
+EXTRA OPTIONS section.
+
+Commands are normally executed immediately on the interval or event
+trigger for which they are configured. However, if the :option:`sync`
+option is active on the cron module, tasks execution will be deferred
+until the next synchronization event. See the documentation above
+for :program:`flux cron sync` for more information.
+
+
+EXAMPLES
+========
+
+Run a script every hour
+
+::
+
+   $ flux cron interval 1h my_script.sh
+   interval: cron-1 created
+
+Run a script only when the event "cron.trigger" is published
+
+::
+
+   $ flux cron event cron.trigger my_script.sh
+   event: cron-2 created
+   ...
+   $ flux event pub cron.trigger
+
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-exec`, :man1:`flux-dmesg`
diff --git a/doc/man1/flux-dmesg.adoc b/doc/man1/flux-dmesg.adoc
deleted file mode 100644
index fb6f8bd2fe4e..000000000000
--- a/doc/man1/flux-dmesg.adoc
+++ /dev/null
@@ -1,63 +0,0 @@
-// flux-help-description: manipulate broker log ring buffer
-FLUX-DMESG(1)
-=============
-:doctype: manpage
-
-
-NAME
-----
-flux-dmesg - access broker ring buffer
-
-
-SYNOPSIS
---------
-*flux* *dmesg* ['OPTIONS']
-
-
-DESCRIPTION
------------
-
-Each broker rank maintains a circular buffer of log entries
-which can be printed using flux-dmesg(1).
-
-
-OPTIONS
--------
-
-*-C, --clear*::
-Clear the ring buffer.
-
-*-c, --read-clear*::
-Clear the ring buffer after printing its contents.
-
-*-f, --follow*::
-After printing the contents of the ring buffer, wait for new entries
-and print them as they arrive.
-
-
-EXAMPLES
---------
-
-To dump the ring buffer on all ranks
-
-  $ flux exec flux dmesg | sort
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-flux-setattr(1), flux-broker-attributes(7)
diff --git a/doc/man1/flux-dmesg.rst b/doc/man1/flux-dmesg.rst
new file mode 100644
index 000000000000..9872013cf0ad
--- /dev/null
+++ b/doc/man1/flux-dmesg.rst
@@ -0,0 +1,76 @@
+=============
+flux-dmesg(1)
+=============
+
+
+SYNOPSIS
+========
+
+**flux** **dmesg** [*OPTIONS*]
+
+
+DESCRIPTION
+===========
+
+.. program:: flux dmesg
+
+Each broker rank maintains a circular buffer of log entries
+which can be printed using :program:`flux dmesg`.
+
+
+OPTIONS
+=======
+
+.. option:: -C, --clear
+
+   Clear the ring buffer.
+
+.. option:: -c, --read-clear
+
+   Clear the ring buffer after printing its contents.
+
+.. option:: -f, --follow
+
+   After printing the contents of the ring buffer, wait for new entries
+   and print them as they arrive.
+
+.. option:: -n, --new
+
+   Follow only new log entries.
+
+.. option:: -H, --human
+
+   Display human-readable output. See also :option:`--color` and
+   :option:`--delta`.
+
+.. option:: -d, --delta
+
+   With :option:`--human`, display the time delta between messages instead
+   of a relative offset since the last absolute timestamp.
+
+.. option:: -L, --color[=WHEN]
+
+   Colorize output. The optional argument *WHEN* can be *auto*, *never*,
+   or *always*. If *WHEN* is omitted, it defaults to *always*. The default
+   value when the :option:`--color` option is not used is *auto*.
+
+EXAMPLES
+========
+
+To dump the ring buffer on all ranks
+
+::
+
+   $ flux exec flux dmesg | sort
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-setattr`, :man7:`flux-broker-attributes`
diff --git a/doc/man1/flux-dump.rst b/doc/man1/flux-dump.rst
new file mode 100644
index 000000000000..501771e437c5
--- /dev/null
+++ b/doc/man1/flux-dump.rst
@@ -0,0 +1,136 @@
+============
+flux-dump(1)
+============
+
+
+SYNOPSIS
+========
+
+**flux** **dump** [*OPTIONS*] *OUTFILE*
+
+
+DESCRIPTION
+===========
+
+.. program flux dump
+
+The :program:`flux dump` command writes a KVS snapshot to a portable archive
+format, usually read by :man1:`flux-restore`.
+
+The snapshot source is the primary namespace of the current KVS root by default.
+If :option:`--checkpoint` is specified, the snapshot source is the last KVS
+checkpoint written to the content backing store.
+
+The archive is a file path or *-* for standard output.  If standard output,
+the format is POSIX *ustar* with no compression.  Otherwise the format is
+determined by the file extension.  The list of valid extensions depends on the
+version of :linux:man3:`libarchive` used to build Flux, but modern versions
+support:
+
+.tar
+   POSIX *ustar* format, compatible with :linux:man1:`tar`.
+
+.tgz, .tar.gz
+   POSIX *ustar* format, compressed with :linux:man1:`gzip`.
+
+.tar.bz2
+   POSIX *ustar* format, compressed with :linux:man1:`bzip2`.
+
+.tar.xz
+   POSIX *ustar* format, compressed with :linux:man1:`xz`.
+
+.zip
+   ZIP archive, compatible with :linux:man1:`unzip`.
+
+.cpio
+   POSIX CPIO format, compatible with :linux:man1:`cpio`.
+
+.iso
+   ISO9660 CD image
+
+
+OPTIONS
+=======
+
+.. option:: -h, --help
+
+   Summarize available options.
+
+.. option:: -v, --verbose
+
+   List keys on stderr as they are dumped instead of a periodic count of
+   dumped keys.
+
+.. option:: -q, --quiet
+
+   Don't show periodic count of dumped keys on stderr.
+
+.. option:: --checkpoint
+
+   Generate snapshot from the latest checkpoint written to the content
+   backing store, instead of from the current KVS root.
+
+.. option:: --no-cache
+
+   Bypass the broker content cache and interact directly with the backing
+   store.  This may be slightly faster, depending on how frequently the same
+   content blobs are referenced by multiple keys.
+
+.. option:: --ignore-failed-read
+
+   If KVS metadata is encountered that references nonexistent blobrefs
+   (for example after a disk full event), print an error but skip over the
+   KVS key and treat it as a warning.  Without this option, content load
+   failures are treated as immediate fatal errors.
+
+
+OTHER NOTES
+===========
+
+KVS commits are atomic and propagate to the root of the namespace.  Because of
+this, when :program:`flux dump` archives a snapshot of a live system, it
+reflects one point in time, and does not include any changes committed while
+the dump is in progress.
+
+Since :program:`flux dump` generates the archive by interacting directly with
+the content store, the :option:`--checkpoint` option may be used to dump the
+most recent state of the KVS when the KVS module is not loaded.
+
+Only regular values and symbolic links are dumped to the archive.  Directories
+are not dumped as independent objects, so empty directories are omitted from
+the archive.
+
+KVS symbolic links represent the optional namespace component in the target
+as a *NAME::* prefix.
+
+The KVS path separator is converted to the UNIX-compatible slash so that the
+archive can be unpacked into a file system if desired.
+
+The modification time of files in the archive is set to the time that
+:program:`flux dump` is started if dumping the current KVS root, or to the
+timestamp of the checkpoint if :option:`--checkpoint` is used.
+
+The owner and group of files in the archive are set to the credentials of the
+user that ran :program:`flux-dump`.
+
+The mode of files in the archive is set to 0644.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_10`
+
+:doc:`rfc:spec_11`
+
+
+SEE ALSO
+========
+
+:man1:`flux-restore`, :man1:`flux-kvs`
diff --git a/doc/man1/flux-env.adoc b/doc/man1/flux-env.adoc
deleted file mode 100644
index 2c899afb35f5..000000000000
--- a/doc/man1/flux-env.adoc
+++ /dev/null
@@ -1,40 +0,0 @@
-// flux-help-description : Print or run inside a Flux environment
-FLUX-ENV(1)
-===========
-:doctype: manpage
-
-
-NAME
-----
-flux-env - Print the flux environment or execute a command inside it
-
-
-SYNOPSIS
---------
-*flux* *env* [COMMAND]
-
-
-DESCRIPTION
------------
-flux-env(1) dumps a list of all environment variables as set by flux if run
-without a command, when run with a command the environment is set and the
-command is run as it would be by the ENV(1) utility.
-
-//OPTIONS
-//-------
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
diff --git a/doc/man1/flux-env.rst b/doc/man1/flux-env.rst
new file mode 100644
index 000000000000..4c9ce14764ce
--- /dev/null
+++ b/doc/man1/flux-env.rst
@@ -0,0 +1,26 @@
+.. flux-help-description : Print or run inside a Flux environment
+
+===========
+flux-env(1)
+===========
+
+
+SYNOPSIS
+========
+
+**flux** **env** [COMMAND]
+
+
+DESCRIPTION
+===========
+
+:program:`flux env` dumps a list of all environment variables as set by
+:man1:`flux` if run without a command, when run with a command the
+environment is set and the command is run as it would be by the
+:linux:man1:`env` utility.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man1/flux-event.adoc b/doc/man1/flux-event.adoc
deleted file mode 100644
index 832fb50c38c8..000000000000
--- a/doc/man1/flux-event.adoc
+++ /dev/null
@@ -1,63 +0,0 @@
-// flux-help-include: true
-FLUX-EVENT(1)
-=============
-:doctype: manpage
-
-
-NAME
-----
-flux-event - Send and receive Flux events
-
-
-SYNOPSIS
---------
-*flux* *event* 'COMMAND' ['OPTIONS']
-
-
-DESCRIPTION
------------
-Flux events are messages that are broadcast throughout the Flux instance
-with publish/subscribe semantics.  Each event message has a _topic string_
-and an optional _payload_.
-
-Subscriptions are by topic string.  A subscription topic of length _N_
-matches an event if the first _N_ characters of the event topic
-are identical to that of the subscription.  For example the event topic
-'a.b.c' is matched by the subscription topic 'a.b.c', 'a.b', or 'a'.
-A subscription to the empty string matches all events.
-
-COMMANDS
---------
-*pub*  [-r] [-l] [-s] [-p] 'topic' ['payload']::
-Publish an event with optional payload.  If payload is specified,
-it is interpreted as raw if the '-r' option is used, otherwise it is
-interpreted as JSON.  If the payload spans multiple arguments,
-the arguments are concatenated with one space between them.
-If '-s' is specified, wait for the event's sequence number to be
-assigned before exiting.
-If '-l' is specified, subscribe to the published event and wait for
-it to be received before exiting.  '-p' causes the privacy flag to
-be set on the published event.
-
-*sub* '[-c N]' ['topic'] ['topic'...]::
-Subscribe to events matching the topic string(s) provided on the
-command line.  If none are specified, subscribe to all events.
-If '-c N' is specified, print the first 'N' events on stdout and exit;
-otherwise continue printing events until a signal is received.
-Events are displayed one per line:  the topic string, followed by a tab,
-followed by the payload, if any.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
diff --git a/doc/man1/flux-event.rst b/doc/man1/flux-event.rst
new file mode 100644
index 000000000000..14c8b3fc104e
--- /dev/null
+++ b/doc/man1/flux-event.rst
@@ -0,0 +1,90 @@
+=============
+flux-event(1)
+=============
+
+
+SYNOPSIS
+========
+
+| **flux** **event** **pub** [*--raw*] [*--synchronous*] [*--private*] *topic* [*payload*]
+| **flux** **event** **sub** [*--count=N*] *topic...*
+
+
+DESCRIPTION
+===========
+
+Flux events are messages that are broadcast throughout the Flux instance
+with publish/subscribe semantics. Each event message has a *topic string*
+and an optional *payload*.
+
+Subscriptions are by topic string. A subscription topic of length *N*
+matches an event if the first *N* characters of the event topic
+are identical to that of the subscription. For example the event topic
+*a.b.c* is matched by the subscription topic *a.b.c*, *a.b*, or *a*.
+A subscription to the empty string matches all events.
+
+
+COMMANDS
+========
+
+pub
+---
+
+.. program:: flux event pub
+
+Publish an event on *topic* with optional *payload*. If payload is specified,
+it is interpreted as JSON unless other options are selected.  If the payload
+spans multiple arguments, the arguments are concatenated with one space
+between them.
+
+.. option:: -r, --raw
+
+  Interpret event payload as raw instead of JSON.
+
+.. option:: -s, --synchronous
+
+  Wait for the event's sequence number to be assigned before exiting.
+
+.. option:: -l, --loopback
+
+  Subscribe to the published event and wait for it to be received before
+  exiting.
+
+.. option:: -p, --private
+
+  Set the privacy flag on the published event.
+
+Example: publish an event with topic ``foo.hello`` and no payload::
+
+  flux event pub foo.hello
+
+sub
+---
+
+.. program:: flux event sub
+
+Subscribe to events matching the topic string(s) provided on the
+command line. If none are specified, subscribe to all events.
+Events are displayed one per line: the topic string, followed by a tab,
+followed by the payload, if any.
+
+.. option:: -c, --count=N
+
+ Print the first *N* events on stdout and exit.  Otherwise events are
+ processed until a signal is received.
+
+Example: subscribe to all events with topic prefix of ``foo.``::
+
+  flux event sub foo.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_3`
diff --git a/doc/man1/flux-exec.adoc b/doc/man1/flux-exec.adoc
deleted file mode 100644
index 743b441a8874..000000000000
--- a/doc/man1/flux-exec.adoc
+++ /dev/null
@@ -1,85 +0,0 @@
-// flux-help-include: true
-FLUX-EXEC(1)
-============
-:doctype: manpage
-
-
-NAME
-----
-flux-exec - Execute processes across flux ranks
-
-
-SYNOPSIS
---------
-*flux* *exec* [--noinput] ['--labelio] ['--dir=DIR'] ['--rank=NODESET'] ['--verbose'] COMMANDS...
-
-
-DESCRIPTION
------------
-flux-exec(1) runs commands across one or more flux-broker ranks using
-the 'cmb.exec' service. The commands are executed as direct children
-of the broker, and the broker handles buffering stdout and stderr and
-sends the output back to flux-exec(1) which copies output to its own
-stdout and stderr.
-
-On receipt of SIGINT and SIGTERM signals, flux-exec(1) shall forward
-the received signal to all currently running remote processes.
-
-In the event subprocesses are hanging or ignoring SIGINT, two SIGINT
-signals (typically sent via Ctrl+C) in short succession can force
-flux-exec(1) to exit.
-
-flux-exec(1) is meant as an administrative and test utility, and cannot
-be used to launch Flux jobs.
-
-EXIT STATUS
------------
-In the case that all processes are successfully launched, the exit status
-of flux-exec(1) is the largest of the remote process exit codes.
-
-If a non-existent rank is targeted, flux-exec(1) will return with
-code 68 (EX_NOHOST from sysexits.h).
-
-If one or more remote commands are terminated by a signal, then flux-exec(1)
-exits with exit code 128+signo.
-
-OPTIONS
--------
-
-*-l, --labelio*::
-Label lines of output with the source RANK.
-
-*-n, --noinput*::
-Do not attempt to forward stdin.  Send EOF to remote process stdin.
-
-*-d, --dir*'=DIR'::
-Set the working directory of remote 'COMMANDS' to 'DIR'. The default is to
-propagate the current working directory of flux-exec(1).
-
-*-r, --rank*'=NODESET'::
-Target specific ranks in 'NODESET'. Default is to target "all" ranks.
-See NODESET FORMAT below for more information.
-
-*-v, --verbose*::
-Run with more verbosity.
-
-
-NODESET FORMAT
---------------
-include::NODESET.adoc[]
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
diff --git a/doc/man1/flux-exec.rst b/doc/man1/flux-exec.rst
new file mode 100644
index 000000000000..ec42202fca5e
--- /dev/null
+++ b/doc/man1/flux-exec.rst
@@ -0,0 +1,128 @@
+============
+flux-exec(1)
+============
+
+SYNOPSIS
+========
+
+**flux** **exec** [*--noinput*] [*--label-io*] [*—dir=DIR*] [*--rank=IDSET*] [*--verbose*] *COMMAND...*
+
+DESCRIPTION
+===========
+
+.. program:: flux exec
+
+:program:`flux exec` remotely executes one or more copies of *COMMAND*,
+similar to :linux:man1:`pdsh`.  It bypasses the scheduler and is intended
+for launching administrative commands or tool daemons, not for launching
+parallel jobs.  For that, see :man1:`flux-run`.
+
+By default, *COMMAND* runs across all :man1:`flux-broker` processes.  If the
+:option:`--jobid` option is specified, the commands are run across a job's
+:man1:`flux-shell` processes.  Normally there is only one broker process per
+node, and one job shell per broker, meaning that one copy of *COMMAND* is
+is executed per node, but in unusual cases it could mean more (e.g. if the
+Flux instance was started with multiple brokers per node).
+
+Standard output and standard error of the remote commands are captured
+and combined on the :program:`flux exec` standard output and standard error.
+Standard input of :program:`flux exec` is captured and broadcast to standard
+input of the remote commands.
+
+On receipt of SIGINT and SIGTERM signals, :program:`flux exec` forwards
+the received signal to the remote processes.  When standard input of
+:program:`flux exec` is a terminal, :kbd:`Control-C` may be used to send
+SIGINT.  Two of those in short succession can force :program:`flux exec`
+to exit in the event that remote processes are hanging.
+
+OPTIONS
+=======
+
+.. option:: -l, --label-io
+
+   Label lines of output with the source broker RANK.  This option is not
+   affected by :option:`--jobid`.
+
+.. option:: -n, --noinput
+
+   Do not attempt to forward stdin. Send EOF to remote process stdin.
+
+.. option:: -d, --dir=DIR
+
+   Set the working directory of remote *COMMAND* to *DIR*. The default is to
+   propagate the current working directory of flux-exec(1).
+
+.. option:: -r, --rank=IDSET
+
+   Target specific ranks, where *IDSET* is a set of zero-origin node ranks in
+   RFC 22 format.  If :option:`--jobid` is specified, the ranks are interpreted
+   as an index into the list of nodes assigned to the job.  Otherwise, they
+   refer to the nodes assigned to the Flux instance.
+
+   The default is to target all ranks.  As a special case, :option:`--rank=all`
+   is accepted and behaves the same as the default.
+
+.. option:: -x, --exclude=IDSET
+
+   Exclude specific ranks.  *IDSET* is as described in :option:`--rank`.
+
+.. option:: -j, --jobid=JOBID
+
+   Run *COMMAND* on the nodes allocated to *JOBID* instead of the nodes
+   assigned to the Flux instance.
+
+   This uses the exec service embedded in :man1:`flux-shell` rather than
+   :man1:`flux-broker`.
+
+   The interpretation of :option:`--rank` and :option:`--exclude` is adjusted
+   as noted in their descriptions.  For example, :option:`flux exec -j ID -r 0`
+   will run only on the first node assigned to *JOBID*, and
+   :option:`flux exec -j ID -x 0` will run on all nodes assigned to *JOBID*
+   except the first node.
+
+   This option is only available when the job owner is the same as the Flux
+   instance owner.
+
+.. option:: -v, --verbose
+
+   Run with more verbosity.
+
+.. option:: -q, --quiet
+
+   Suppress extraneous output (e.g. per-rank error exit status).
+
+.. option:: --with-imp
+
+   Prepend the full path to :program:`flux-imp run` to *COMMAND*. This option
+   is mostly meant for testing or as a convenience to execute a configured
+   ``prolog`` or ``epilog`` command under the IMP.
+
+CAVEATS
+=======
+
+In a multi-user flux instance, access to the rank 0 broker execution
+service is restricted to requests that originate from the local broker.
+Therefore, :program:`flux exec` (without :option:`--jobid`) must be run
+from the rank 0 broker if rank 0 is included in the target *IDSET*.
+
+EXIT STATUS
+===========
+
+In the case that all processes are successfully launched, the exit status
+of :program:`flux exec` is the largest of the remote process exit codes.
+
+If a non-existent rank is targeted, :program:`flux exec` will return with
+code 68 (EX_NOHOST from sysexits.h).
+
+If one or more remote commands are terminated by a signal, then
+:program:`flux exec` exits with exit code 128+signo.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_22`
diff --git a/doc/man1/flux-getattr.adoc b/doc/man1/flux-getattr.adoc
deleted file mode 100644
index 13c9e8afa677..000000000000
--- a/doc/man1/flux-getattr.adoc
+++ /dev/null
@@ -1,57 +0,0 @@
-// flux-help-command: get,set,lsattr
-// flux-help-description: Access, modify, and list broker attributes
-FLUX-GETATTR(1)
-===============
-:doctype: manpage
-
-
-NAME
-----
-flux-getattr, flux-setattr, flux-lsattr - access broker attributes
-
-
-SYNOPSIS
---------
-*flux* *getattr* 'name'
-
-*flux* *setattr* 'name' 'value'
-
-*flux* *setattr* ['--expunge'] 'name'
-
-*flux* *lsattr* ['--values']
-
-
-DESCRIPTION
------------
-
-Flux broker attributes are both a simple, general-purpose key-value
-store with scope limited to the local broker rank, and a method for the
-broker to export information needed by Flux comms modules and
-utilities.
-
-flux-getattr(1) retrieves the value of an attribute.
-
-flux-setattr(1) assigns a new value to an attribute, or optionally
-removes an attribute.
-
-flux-lsattr(1) lists attribute names, optionally with their values.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-flux_attr_get(3), flux-broker-attributes(7)
diff --git a/doc/man1/flux-getattr.rst b/doc/man1/flux-getattr.rst
new file mode 100644
index 000000000000..099632ff1eab
--- /dev/null
+++ b/doc/man1/flux-getattr.rst
@@ -0,0 +1,67 @@
+===============
+flux-getattr(1)
+===============
+
+
+SYNOPSIS
+========
+
+| **flux** **getattr** *name*
+| **flux** **setattr** *name* *value*
+| **flux** **lsattr** [*--values*]
+
+
+DESCRIPTION
+===========
+
+The Flux broker attribute subsystem provides a primitive key-value
+configuration mechanism for the broker.  Attributes can be set on the
+broker command line with :option:`flux broker --setattr`, then read,
+written, or listed using :program:`flux getattr`, :program:`flux setattr`,
+or :program:`flux lsattr` after the broker is running.
+
+Attribute scope is local to an individual broker.  That is, broker ranks
+may have different values for a given attribute.
+
+:man7:`flux-broker-attributes` provides a catalog of attributes.
+
+COMMANDS
+========
+
+getattr
+-------
+
+.. program:: flux getattr
+
+:program:`flux getattr` retrieves the value of an attribute.
+
+setattr
+-------
+
+.. program:: flux setattr
+
+:program:`flux setattr` assigns a value to an attribute.  If the attribute
+does not exist, it is created.
+
+lsattr
+------
+
+.. program:: flux lsattr
+
+:program:`flux lsattr` lists attributes.
+
+.. option:: -v, --values
+
+  List the attribute values too.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_attr_get`, :man7:`flux-broker-attributes`
diff --git a/doc/man1/flux-hostlist.rst b/doc/man1/flux-hostlist.rst
new file mode 100644
index 000000000000..29a5dfe78c4a
--- /dev/null
+++ b/doc/man1/flux-hostlist.rst
@@ -0,0 +1,235 @@
+.. flux-help-section: other
+
+================
+flux-hostlist(1)
+================
+
+SYNOPSIS
+========
+
+**flux** **hostlist** [*OPTIONS*] [*SOURCES*]
+
+DESCRIPTION
+===========
+
+.. program:: flux hostlist
+
+:program:`flux hostlist` takes zero or more *SOURCES* of host lists on the
+command line and concatenates them by default into a single RFC 29 Hostlist.
+
+*SOURCES* can optionally be combined by various set operations, for example
+to find the intersection, difference, or to subtract hostlists.
+
+SOURCES
+=======
+
+Valid *SOURCES* of hostlist information include:
+
+instance
+  hosts from the broker ``hostlist`` attribute
+
+jobid
+  hosts assigned to a job.
+
+local
+  *jobid* from ``FLUX_JOB_ID`` environment variable if set, otherwise
+  *instance*
+
+avail[able]
+  *instance* hostlist minus those nodes down or drained
+
+stdin or ``-``
+  read a list of hosts on stdin
+
+hosts
+  a literal RFC 29 Hostlist
+
+The default source is *stdin*.
+
+OPTIONS
+=======
+
+.. option:: -e, --expand
+
+  Expand hostlist result using the defined output delimiter. Default is
+  space-delimited.
+
+.. option:: -d, --delimiter=S
+
+  Set the delimiter for :option:`--expand` to string *S*.
+
+.. option:: -c, --count
+
+  Emit the number of hosts in the result hostlist instead of the hostlist
+  itself.
+
+.. option:: -n, --nth=IDS
+
+  Output only the hosts at indices *IDS* (*-IDS* to index from the end),
+  where *IDS* is a valid RFC 22 idset (e.g. '0' will return the first host,
+  '0-1' will return the first and second, '-1' returns the last host).  The
+  command will fail if any id in *IDS* is not a valid index.
+
+.. option:: -L, --limit=N
+
+  Output at most *N* hosts (*-N* to output the last *N* hosts).
+
+.. option:: -S, --sort
+
+  Display sorted result.
+
+.. option:: -u, --union, --unique
+
+  Return only unique hosts. This implies :option:`--sort`. Without any
+  other manipulation options, this is equivalent to returning the set
+  union of all provided hosts. (By default, all inputs are concatenated).
+
+.. option:: -x, --exclude=HOSTS|IDS
+
+  Exclude all hosts in  *HOSTS* or indices in idset *IDS* from the result.
+  It is not an error if any hosts or indices do not exist in the target
+  hostlist.
+
+.. option:: -i, --intersect
+
+  Return the set intersection of all hostlists.
+
+.. option:: -m, --minus
+
+  Subtract all hostlists from the first.
+
+.. option:: -X, --xor
+
+  Return the symmetric difference of all hostlists.
+
+.. option:: -f, --fallback
+
+  If an argument to :command:`flux-hostlist` is a single hostname, and the
+  hostname can be interpreted as a valid Flux jobid (e.g. starts with ``f``
+  and otherwise contains valid base58 characters like ``fuzzy`` or ``foo1``),
+  then the command may fail with::
+
+    flux-hostlist: ERROR: job foo1 not found
+
+  With the :option:`--fallback` option arguments that appear to be jobids that
+  are not found are treated as hostnames, e.g.::
+ 
+    $ flux hostlist --fallback foo1 foo2
+    foo[1-2]
+
+.. option:: -l, --local
+
+  Change the default input source to "local". This is a shorter way to
+  specify ``flux hostlist local``.
+  
+.. option:: -q, --quiet
+
+  Suppress output and exit with a nonzero exit code if the hostlist is empty.
+
+EXAMPLES
+========
+
+Create host file for the current job or instance if running in an initial
+program:
+
+::
+
+  $ flux hostlist -led'\n' >hostfile
+
+Launch an MPI program using :program:`mpiexec.hydra` from within a batch
+script:
+
+::
+
+  #!/bin/sh
+  mpiexec.hydra -launcher ssh -hosts "$(flux hostlist -le)" mpi_hello
+
+List the hosts for one job: (Note: this is the same as
+:command:`flux jobs -no {nodelist} JOBID`)
+
+::
+
+  $ flux hostlist JOBID
+  host[1-2]
+
+List the hosts for one job, excluding the first node:
+
+::
+
+  $ flux hostlist -x 0 JOBID
+
+List the unordered, unique hosts for multiple jobs:
+
+::
+
+  $ flux hostlist -u JOBID1 JOBID2 JOBID3
+  host[1-2,4]
+
+Determine if any failed jobs shared common nodes:
+
+::
+
+  $ flux hostlist --intersect $(flux jobs -f failed -no {id})
+  host4
+
+Determine if a given host appeared the last submitted job:
+
+::
+
+  if flux hostlist -q -i $(flux job last) host1; then
+      echo host1 was part of your last job
+  fi
+
+
+Count the number of currently available hosts:
+
+::
+
+  $ flux hostlist --count avail
+  4
+
+List all the hosts on which a job named 'myapp' ran:
+
+::
+
+  $ flux hostlist --union $(flux pgrep myapp)
+  host[2,4-5]
+
+List all hosts in the current instance which haven't had a job assigned
+in the last 100 jobs:
+
+::
+
+  $ flux hostlist --minus instance $(flux jobs -c 100 -ano {id})
+  host0
+
+EXIT STATUS
+===========
+
+0
+  Successful operation
+
+1
+  One or more *SOURCES* were invalid, an invalid index was specified to
+  :option:`--nth`, or :option:`--quiet` was used and the result hostlist
+  was empty.
+
+2
+  Invalid option specified or other command line error
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+  
+FLUX RFC
+========
+
+:doc:`rfc:spec_29`
+:doc:`rfc:spec_22`
+
+
+SEE ALSO
+========
+
+:man1:`flux-getattr`, :man1:`flux-jobs`, :man7:`flux-broker-attributes`
diff --git a/doc/man1/flux-housekeeping.rst b/doc/man1/flux-housekeeping.rst
new file mode 100644
index 000000000000..1db5df23c16c
--- /dev/null
+++ b/doc/man1/flux-housekeeping.rst
@@ -0,0 +1,185 @@
+====================
+flux-housekeeping(1)
+====================
+
+
+SYNOPSIS
+========
+
+| **flux** **housekeeping** **list** [*-n*] [*-o FORMAT*]
+| **flux** **housekeeping** **kill** [*--all*] [*-j JOBID*] [*-t HOSTS|RANKS*] [*-s SIGNUM*]
+
+
+DESCRIPTION
+===========
+
+.. program:: flux housekeeping
+
+The housekeeping service provides similar functionality to
+a job epilog, with a few advantages
+
+ - Housekeeping runs after the job, which is then allowed to exit CLEANUP
+   state and become inactive once resources are released.
+ - While housekeeping is running, the scheduler still thinks resources are
+   allocated to the job, and will not allocate resources to other jobs.
+ - Housekeeping supports partial release of resources back to the scheduler,
+   such that a subset of stuck nodes do not hold up other nodes from
+   being returned to service.
+
+The :program:`flux housekeeping` command is used to interact with the
+housekeeping service. It supports listing the resources currently executing
+housekeeping actions and a command to forcibly terminate actions on a per-job
+or per-node basis.
+
+In a Flux system instance, housekeeping is configured by default to run as a
+one-shot :linux:man5:`systemd.unit`.  See :ref:`troubleshooting` below.
+
+COMMANDS
+========
+
+list
+----
+
+.. program:: flux housekeeping list
+
+:program:`flux housekeeping list` lists active housekeeping tasks by jobid.
+
+.. option:: -i, --include=TARGETS
+
+  Filter results to only include resources matching *TARGETS*, which may
+  be specified either as an idset of broker ranks or a list of hosts in
+  hostlist form. It is not an error to specify ranks or hosts that do not
+  exist.
+
+.. option:: -o, --format=FORMAT
+
+  Customize the output format (See the `OUTPUT FORMAT`_ section below).
+
+.. option:: -n, --no-header
+
+  Suppress header from output.
+
+kill
+----
+
+.. program:: flux housekeeping kill
+
+:program:`flux housekeeping kill` can be used to terminate active housekeeping
+tasks. Housekeeping may be terminated by jobid, a set of targets such as
+broker ranks or hostnames, or all housekeeping may be terminated via the
+:option:`--all` option.
+
+.. option:: -s, --signal=SIGNUM
+
+  Send signal SIGNUM instead of SIGTERM.
+
+.. option:: -t, --targets=RANK|HOSTS
+
+  Target a specific set of ranks or hosts.
+
+.. option:: -j, --jobid=JOBID
+
+  Target a specific job by JOBID. Without ``--targets`` this will kill all
+  housekeeping tasks for the specified job.
+
+.. option:: --all
+
+  Target all housekeeping tasks for all jobs.
+
+OUTPUT FORMAT
+=============
+
+The :option:`--format` option can be used to specify an output format using
+Python's string format syntax or a defined format by name. For a list of
+built-in and configured formats use :option:`-o help`.
+
+The following field names can be specified for
+:command:`flux housekeeping list`:
+
+**id**
+   The jobid that triggered housekeeping
+
+**runtime**
+   The time since this housekeeping task started
+
+**nnodes**
+   A synonym for **allocated.nnodes**
+
+**ranks**
+   A synonym for **allocated.ranks**
+
+**nodelist**
+   A synonym for **allocated.nodelist**
+
+**allocated.nnodes**
+   The number of nodes still allocated to this housekeeping task.
+
+**allocated.ranks**
+   The list of broker ranks still allocated to this housekeeping task.
+
+**allocated.ranks**
+   The list of nodes still allocated to this housekeeping task.
+
+**pending.nnodes**
+   The number of nodes that still need to complete housekeeping.
+
+**pending.ranks**
+   The list of broker ranks that still need to complete housekeeping.
+
+**pending.ranks**
+   The list of nodes that still need to complete housekeeping.
+
+.. _troubleshooting:
+
+TROUBLESHOOTING
+===============
+
+In a Flux system instance, housekeeping is configured by default to run as a
+:linux:man5:`systemd.unit` named ``flux-housekeeping@JOBID``.
+
+:linux:man1:`systemctl` can show the status of housekeeping units running
+on the local node::
+
+  $ systemctl status flux-housekeeping@*
+
+:linux:man1:`journalctl` shows standard output and error of a housekeeping
+run::
+
+  $ journalctl -u flux-housekeeping@f4aTGTz2SN3
+
+When housekeeping fails, the systemd unit script drains the failing nodes
+with the reason obtained from systemd.  For example, housekeeping runs
+that failed due to a nonzero exit code are distinguished from those that
+were aborted early due to a signal.  In addition, a failure message is
+logged to Flux and can be accessed with :man1:`flux-dmesg`.
+
+When housekeeping hangs, no automated action is taken by Flux.  Sending
+housekeeping a signal with :program:`flux housekeeping kill` causes
+:program:`systemctl stop` to be run on the housekeeping unit.  Generally,
+it is best to let systemd take over from there.  Its default action is to
+send SIGTERM to all processes in the control group, then SIGKILL if any
+processes have not terminated after a 90s delay.
+
+.. note::
+
+  On systems with scheduler configurations that permit jobs to share nodes,
+  multiple housekeeping units may execute concurrently on a single node.
+  Housekeeping scripts must be crafted with that in mind on such systems.
+
+CAVEATS
+=======
+
+The ``flux-housekeeping@`` systemd unit is responsible for draining nodes
+when housekeeping fails.  Therefore if the system is configured to bypass
+the systemd unit file, or if housekeeping is misconfigured such that the
+the systemd unit file is not started, this draining does not occur.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+SEE ALSO
+========
+
+:man5:`flux-config-job-manager`
diff --git a/doc/man1/flux-hwloc.adoc b/doc/man1/flux-hwloc.adoc
deleted file mode 100644
index dcc57a2f2b16..000000000000
--- a/doc/man1/flux-hwloc.adoc
+++ /dev/null
@@ -1,84 +0,0 @@
-// flux-help-command: hwloc
-// flux-help-description: Control/query resource-hwloc service
-FLUX-HWLOC(1)
-=============
-:doctype: manpage
-
-
-NAME
-----
-flux-hwloc - Control/query resource-hwloc service
-
-
-SYNOPSIS
---------
-*flux* *hwloc* *info* ['OPTIONS']
-
-*flux* *hwloc* *lstopo* ['lstopo-OPTIONS']
-
-*flux* *hwloc* *reload* ['OPTIONS'] ['DIR']
-
-*flux* *hwloc* *topology* ['OPTIONS']
-
-
-DESCRIPTION
------------
-The *flux-hwloc* utility provides a mechanism to collect
-system topology from each flux-broker using the Portable Hardware
-Locality (hwloc) library, and to query the resulting data
-stored in the Flux Key Value Store (KVS).
-
-COMMANDS
---------
-
-*flux hwloc* requires a 'COMMAND' argument. The supported commands
-are
-
-*info* ['-l,--local'|'-r,--rank=NODESET']::
-Dump a short-form summary of the total number of Machines, Cores,
-and Processing Units (PUs) available across all flux-brokers
-in the current instance. With '--ranks', dump information for
-only the specified ranks. With '--local' dump local system information.
-
-*lstopo*::
-Run `lstopo(1)` against the full hardware hierarchy configured in the
-current Flux instance. Extra `OPTIONS` are passed along to the system
-`lstopo(1)`. + 
-By default, *flux hwloc lstopo* generates console output.
-For graphical output, try:  *flux hwloc lstopo --of graphical*.
-
-*reload* ['-r,--rank=NODESET'] ['-v,--verbose] ['DIR']::
-Reload hwloc topology information, optionally loading hwloc XML files
-from `DIR/<rank>.xml` files. With '--rank' only reload XML on specified
-ranks. With '--verbose' this command runs with extra debugging and
-timing information.
-
-*topology* ['-l,--local'|'-r,--rank=NODESET']::
-Dump current aggregate topology XML for the current session to stdout.
-With '--rank' only dump aggregate topology for specified ranks. With
-'--local' dump topology XML for the local system.
-
-
-NODESET FORMAT
---------------
-include::NODESET.adoc[]
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-lstopo(1), hwloc: https://www.open-mpi.org/projects/hwloc/
diff --git a/doc/man1/flux-job.rst b/doc/man1/flux-job.rst
new file mode 100644
index 000000000000..693c2c7d0423
--- /dev/null
+++ b/doc/man1/flux-job.rst
@@ -0,0 +1,511 @@
+.. flux-help-description: get job status, info, etc (see: flux help job)
+.. flux-help-section: jobs
+
+===========
+flux-job(1)
+===========
+
+
+SYNOPSIS
+========
+
+| **flux** **job** **attach** [*--label-io*] [*-E*] [*--wait-event=EVENT*] *id*
+| **flux** **job** **status** [*-v*] [*--json*] [-e CODE] *id [*id...*]
+| **flux** **job** **last** [*N* | *SLICE*]
+| **flux** **job** **urgency** [*-v*] *id* *N*
+| **flux** **job** **wait** [*-v*] [*--all*] [*id*]
+| **flux** **job** **kill** [*--signal=SIG*] *ids...*
+| **flux** **job** **killall** [*-f*] [*--user=USER*] [*--signal=SIG*]
+| **flux** **job** **raise** [*--severity=N*] [*--type=TYPE*] *ids...* [*--*] [*message...*]
+| **flux** **job** **raiseall** [*--severity=N*] [*--user=USER*] [*--states=STATES*] *type* [ [*--*] [*message...*]
+| **flux** **job** **taskmap** [*OPTIONS*] *id* | *taskmap*
+| **flux** **job** **timeleft** [*-H*] [*id*]
+| **flux** **job** **purge** [*-f*] [*--age-limit=FSD*] [*--num-limit=N*] [*ids...*]
+| **flux** **job** **info** [*--original*] [*--base*] *id* *key*
+| **flux** **job** **hostpids** [*OPTIONS*] *id*
+
+
+DESCRIPTION
+===========
+
+:program:`flux job` performs various job related housekeeping functions.
+
+
+OPTIONS
+=======
+
+.. program:: flux job
+
+.. option:: -h, --help
+
+   Display a list of :program:`flux job` sub-commands.
+
+
+COMMANDS
+========
+
+Several subcommands are available to perform various operations on jobs.
+
+attach
+------
+
+.. program:: flux job attach
+
+A job can be interactively attached to via :program:`flux job attach`.  This is
+typically used to watch stdout/stderr while a job is running or after it has
+completed.  It can also be used to feed stdin to a job.
+
+When :program:`flux job attach` is run interactively -- that is all of
+``stdout``, ``stderr`` and ``stdin`` are attached to a tty -- the command may
+display a status line while the job is pending, e.g
+
+::
+
+    flux-job: ƒJqUHUCzX9 waiting for resources                 00:00:08
+
+This status line may be suppressed by setting
+:envvar:`FLUX_ATTACH_NONINTERACTIVE` in the environment.
+
+.. option:: -l, --label-io
+
+   Label output by rank
+
+.. option:: -u, --unbuffered
+
+   Do not buffer stdin. Note that when ``flux job attach`` is used in a
+   terminal, the terminal itself may line buffer stdin.
+
+.. option:: -i, --stdin-ranks=RANKS
+
+   Send stdin to only those ranks in the **RANKS** idset. The standard input
+   for tasks not in **RANKS** will be closed. The default is to broadcast
+   stdin to all ranks.
+
+.. option:: --read-only
+
+   Operate in read-only mode. Disable reading of stdin and capturing of
+   signals.
+
+.. option:: -v, --verbose
+
+   Increase verbosity.
+
+.. option:: -w, --wait-event=EVENT
+
+   Wait for event *EVENT* before detaching from eventlog. The default is
+   ``finish``.
+
+.. option:: -E, --show-events
+
+   Show job events on stderr. This option also suppresses the status line
+   if enabled.
+
+.. option:: -X, --show-exec
+
+   Show exec eventlog events on stderr.
+
+.. option:: --show-status
+
+   Force immediate display of the status line.
+
+.. option:: --debug
+
+   Enable parallel debugger attach.
+
+status
+------
+
+.. program:: flux job status
+
+Wait for job(s) to complete and exit with the largest exit code.
+
+.. option:: -e, --exception-exit-code=N
+
+   Set the exit code for any jobs that terminate with an exception
+   (e.g. canceled jobs) to ``N``.
+
+.. option:: -j, --json
+
+   Dump job result information from job eventlog.
+
+.. option:: -v, --verbose
+
+   Increase verbosity of output.
+
+last
+-----
+
+.. program:: flux job last
+
+Print the most recently submitted jobid for the current user.
+
+If the optional argument is specified as a number *N*, print the *N* most
+recently submitted jobids in reverse submission order, one per line.  If it
+is enclosed in brackets, the argument is interpreted as a `python-style slice
+<https://python-reference.readthedocs.io/en/latest/docs/brackets/slicing.html>`_
+in :option:`[start:stop[:step]]` form which slices the job history array,
+where index 0 is the most recently submitted job.
+
+Examples:
+
+:command:`flux job last 4`
+  List the last four jobids in reverse submission order
+
+:command:`flux job last [0:4]`
+  Same as above
+
+:command:`flux job last [-1:]`
+  List the least recently submitted jobid
+
+:command:`flux job last [:]`
+  List all jobids in reverse submission order
+
+:command:`flux job last [::-1]`
+  List all jobids in submission order
+
+urgency
+-------
+
+.. program:: flux job wait
+
+:program:`flux job urgency` changes a job's urgency value.  The urgency
+may also be specified at job submission time.  The argument *N* has a range
+of 0 to 16 for guest users, or 0 to 31 for instance owners.  In lieu of a
+numerical value, the following special names are also accepted:
+
+hold (0)
+  Hold the job until the urgency is raised with :option:`flux job urgency`.
+
+default (16)
+  The default urgency for all users.
+
+expedite (31)
+  Assign the highest possible priority to the job (restricted to instance
+  owner).
+
+Urgency is one factor used to calculate job priority, which affects the
+order in which the scheduler considers jobs.  For more information, refer
+to :man1:`flux-submit` description of the :option:`flux submit --urgency`
+option.
+
+wait
+----
+
+.. program:: flux job wait
+
+:program:`flux job wait` behaves like the UNIX :linux:man2:`wait` system call,
+for jobs submitted with the ``waitable`` flag.  Compared to other methods
+of synchronizing on job completion and obtaining results, it is very
+lightweight.
+
+The result of a waitable job may only be consumed once.  This is a design
+feature that makes it possible to call :program:`flux job wait` in a loop
+until all results are consumed.
+
+.. note::
+  Only the instance owner is permitted to submit jobs with the ``waitable``
+  flag.
+
+When run with a jobid argument, :program:`flux job wait` blocks until the
+specified job completes.  If the job was successful, it silently exits with a
+code of zero.  If the job has failed, an error is printed on stderr, and it
+exits with a code of one.  If the jobid is invalid or the job is not waitable,
+:program:`flux job wait` exits with a code of two.  This special exit code of
+two is used to differentiate between a failed job and not being able to wait
+on the job.
+
+When run without arguments, :program:`flux job wait` blocks until the next
+waitable job completes and behaves as above except that the jobid is printed
+to stdout.  When there are no more waitable jobs, it exits with a code of two.
+The exit code of two can be used to determine when no more jobs are waitable
+when using :program:`flux job wait` in a loop.
+
+:option:`flux job wait --all` loops through all the waitable jobs as they
+complete, printing their jobids.  If all jobs are successful, it exits with a
+code of zero.  If any jobs have failed, it exits with a code of one.
+
+.. option:: -a, --all
+
+   Wait for all waitable jobs and exit with error if any jobs are
+   not successful.
+
+.. option:: -v, --verbose
+
+   Emit a line of output for all jobs, not just failing ones.
+
+kill
+----
+
+.. program:: flux job kill
+
+One or more running jobs may be signaled by jobid with :program:`flux job kill`.
+
+.. option:: -s, --signal=SIG
+
+   Send signal SIG (default: SIGTERM).
+
+killall
+-------
+
+.. program:: flux job killall
+
+Running jobs may be signaled in bulk with :program:`flux job killall`.
+
+.. option:: -u, --user=USER
+
+   Set target user.  The instance owner may specify *all* for all users.
+
+.. option:: -f, --force
+
+   Confirm the command.
+
+.. option:: -s, --signal=SIG
+
+   Send signal SIG (default: SIGTERM).
+
+raise
+-----
+
+.. program:: flux job raise
+
+An exception may raised on one or more jobids with :program:`flux job raise`.
+An optional message included with the job exception may be provided via
+the :option:`--message=NOTE` option or after the list of jobids. The special
+argument *"--"* forces the end of jobid processing and can be used to
+separate the exception message from the jobids when necessary.
+
+.. option:: -m, --message=NOTE
+
+   Set the optional exception note. It is an error to specify the message
+   via this option and on the command line after the jobid list.
+
+.. option:: -s, --severity=N
+
+   Set exception severity.  The severity may range from 0=fatal to
+   7=least severe (default: 0).
+
+.. option:: -t, --type=TYPE
+
+   Set exception type (default: cancel).
+
+raiseall
+--------
+
+Exceptions may be raised in bulk with :program:`flux job raiseall`, which
+requires a type (positional argument) and accepts the following options:
+
+.. program:: flux job raiseall
+
+.. option:: -s, --severity=N
+
+   Set exception severity.  The severity may range from 0=fatal to
+   7=least severe (default: 7).
+
+.. option:: -u, --user=USER
+
+   Set target user.  The instance owner may specify *all* for all users.
+
+.. option:: -S, --states=STATES
+
+   Set target job states (default: ACTIVE)
+
+.. option:: -f, --force
+
+   Confirm the command.
+
+taskmap
+-------
+
+.. program:: flux job taskmap
+
+The mapping between job task ranks to node IDs is encoded in the RFC 34
+Flux Task Map format and posted to the job's ``shell.start`` event in the
+exec eventlog. The :program:`flux job taskmap` utility is provided to assist in
+working with these task maps.
+
+When executed with a jobid argument and no options, the taskmap for the job
+is printed after the ``shell.start`` event has been posted.
+
+With one of the following arguments, the job taskmap may be used to convert
+a nodeid to a list of tasks, or to query on which node or host a given
+taskid ran. The command may also be used to convert between different
+support task mapping formats:
+
+.. option:: --taskids=NODEID
+
+   Print an idset of tasks which ran on node  *NODEID*
+
+.. option:: --ntasks=NODEID
+
+   Print the number of tasks  which ran on node *NODEID*
+
+.. option:: --nodeid=TASKID
+
+   Print the node ID that ran task *TASKID*
+
+.. option:: --hostname=TASKID
+
+   Print the hostname of the node that rank task *TASKID*
+
+.. option:: --to=raw|pmi|multiline|hosts
+
+   Convert the taskmap to *raw* or *pmi* formats (described in RFC 34),
+   *multiline* which prints the node ID of each task, one per line,
+   or *hosts* which prints a list of taskids for each host. The default
+   behavior is to print the RFC 34 taskmap. This option can be useful
+   to convert between mapping forms, since :program:`flux job taskmap`
+   can take a raw, pmi, or RFC 34 task map on the command line.
+
+Only one of the above options may be used per call.
+
+timeleft
+--------
+
+.. program:: flux job timeleft
+
+The :program:`flux job timeleft` utility reports the number of whole seconds
+left in the current or specified job time limit. If the job has expired or is
+complete, then this command reports ``0``. If the job does not have a time
+limit, then a large number (``UINT_MAX``) is reported.
+
+If :program:`flux job timeleft` is called outside the context of a Flux job, or
+an invalid or pending job is targeted, then this command will exit with
+an error and diagnostic message.
+
+Options:
+
+.. option:: -H, --human
+
+  Generate human readable output. Report results in Flux Standard Duration.
+
+purge
+-----
+
+.. program:: flux job purge
+
+Inactive job data may be purged from the Flux instance with
+:program:`flux job purge`.  Specific job ids may be specified for purging.
+If no job ids are specified, the following options may be used for selection
+criteria:
+
+.. option:: --age-limit=FSD
+
+   Purge inactive jobs older than the specified Flux Standard Duration.
+
+.. option:: --num-limit=COUNT
+
+   Purge the oldest inactive jobs until there are at most COUNT left.
+
+.. option:: -f, --force
+
+   Confirm the command.
+
+Inactive jobs may also be purged automatically if the job manager is
+configured as described in :man5:`flux-config-job-manager`.
+
+
+info
+----
+
+.. program:: flux job info
+
+:program:`flux job info` retrieves the selected low level job object
+and displays it on standard output.  Object formats are described in the
+RFCs listed in `RESOURCES`_.
+
+Options:
+
+.. option:: -o, --original
+
+  For :option:`jobspec`, return the original submitted jobspec, prior
+  to any modifications made at ingest, such as setting defaults.
+
+.. option:: -b, --base
+
+  For :option:`jobspec` or :option:`R`, return the base version, prior
+  to any updates posted to the job eventlog.
+
+The following keys are valid:
+
+eventlog
+   The primary job eventlog, consisting of timestamped events that drive the
+   job through various states.  For example, a job that is pending resource
+   allocation in SCHED state transitions to RUN state on the *alloc* event.
+
+guest.exec.eventlog
+   The execution eventlog, consisting of timestamped events posted by the
+   execution system while the job is running.
+
+guest.input, guest.output
+   The job input and output eventlogs, consisting of timestamped chunks of
+   input/output data.
+
+jobspec
+   The job specification.  Three versions are available:
+
+   - default: the *current* jobspec, which may reflect updates,
+     for example if the job duration was extended
+
+   - with :option:`--original`: the original jobspec submitted by the user
+
+   - with :option:`--base`: the jobspec as initially ingested to the KVS, after
+     the frobnicator filled in any default values, but before updates
+
+R
+   The resource set allocated to the job.  Two versions are available:
+
+   - default: the *current* R, which may reflect updates, for example if the job
+     expiration time was extended (default)
+
+   - with :option:`--base`: the initial R allocated by the scheduler
+
+hostpids
+--------
+
+.. program:: flux job hostpids
+
+:program:`flux job hostpids` prints a comma-delimited list of
+``hostname:PID`` pairs for all tasks in a running job. If the job is
+pending, :program:`flux job hostpids` will block until all tasks in the
+job are running.
+
+Options:
+
+.. option:: -d, --delimiter=STRING
+
+  Set the output delimiter to STRING (default=``,``).
+
+.. option:: -r, --ranks=IDSET
+
+  Restrict output to the task ranks in IDSET. The default is to display
+  all ranks.
+
+.. option:: -t, --timeout=DURATION
+
+  Timeout the command after DURATION, which is specified in FSD.
+  (a floating point value with optional suffix ``s`` for seconds,
+   ``m`` for minutes, ``h`` for hours, or ``d`` for days).
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_14`
+
+:doc:`rfc:spec_18`
+
+:doc:`rfc:spec_20`
+
+:doc:`rfc:spec_21`
+
+:doc:`rfc:spec_24`
+
+:doc:`rfc:spec_25`
+
+:doc:`rfc:spec_34`
diff --git a/doc/man1/flux-jobs.adoc b/doc/man1/flux-jobs.adoc
deleted file mode 100644
index 9c8bdb09d628..000000000000
--- a/doc/man1/flux-jobs.adoc
+++ /dev/null
@@ -1,107 +0,0 @@
-// flux-help-include: true
-FLUX-JOBS(1)
-============
-:doctype: manpage
-
-
-NAME
-----
-flux-jobs - list jobs submitted to Flux
-
-
-SYNOPSIS
---------
-*flux* *jobs* ['OPTIONS']
-
-
-DESCRIPTION
------------
-flux-jobs(1) is used to list jobs run under Flux.  By default only
-pending and running jobs for the current user are listed.  Additional
-jobs and information can be listed using options listed below.
-
-
-OPTIONS
--------
-*-a*::
-List all jobs of the current user, including inactive jobs.
-Equivalent to specifying '--state=pending,running,inactive'.
-
-*-A*::
-List all jobs from all users, including inactive jobs.  Equivalent to
-specifying '--state=pending,running,inactive --user=all'.
-
-*-n, --suppress-header*::
-For default output, do not output column headers.
-
-*-u, --user*'=[USERNAME|UID]'::
-List jobs for a specific username or userid.  Specify 'all' for all users.
-
-*-c, --count*'=N'::
-Limit output to N jobs (default 1000)
-
-*-s, --states*'=STATES'::
-List jobs in specific job states or virtual job states.  Multiple
-states can be listed separated by comma.  See JOB STATES below for
-additional information.  Defaults to 'pending,running'.
-
-*-o, --format*'=FORMAT'::
-Specify output format using Python's string format syntax.  See OUTPUT
-FORMAT below for field names.
-
-JOB STATES
-----------
-Jobs may be observed to pass through five job states in Flux: DEPEND,
-SCHED, RUN, CLEANUP, and INACTIVE (see Flux RFC 21). For convenience
-and clarity, some options accept the following virtual job states:
-"pending", an alias for DEPEND,SCHED; "running", an alias for
-RUN,CLEANUP; "active", an alias for "pending,running".
-
-OUTPUT FORMAT
--------------
-
-The '--format' option can be used to specify an output format to
-flux-jobs(1) using Python's string format syntax.  For example, the
-following is the format used for the default format:
-
-  {id:>18} {username:<8.8} {name:<10.10} {state:<8.8} {ntasks:>6} {nnodes_hyphen:>6} {runtime_fsd_hyphen:>8} {ranks_hyphen}
-
-The field names that can be specified are:
-
-[horizontal]
-id:: job ID
-userid:: job submitter's userid
-username:: job submitter's username
-priority:: job priority
-state:: job state
-state_single:: job state as a single character
-name:: job name
-ntasks:: job task count
-nnodes:: job node count (if job ran / is running), empty string otherwise
-nnodes_hyphen:: same as nnodes, but '-' if job has not run yet / never ran
-ranks:: job ranks (if job ran / is running), empty string otherwise
-ranks_hyphen:: same as ranks, but '-' if job has not run yet / never ran
-t_submit:: time job was submitted
-t_depend:: time job entered depend state
-t_sched:: time job entered sched state
-t_run:: time job entered run state
-t_cleanup:: time job entered cleanup state
-t_inactive:: time job entered inactive state
-runtime:: job runtime
-runtime_fsd:: job runtime in Flux standard duration format
-runtime_fsd_hyphen:: same as runtime_fsd, but '-' if runtime is 0s
-runtime_hms:: job runtime in H:M:S format
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
diff --git a/doc/man1/flux-jobs.rst b/doc/man1/flux-jobs.rst
new file mode 100644
index 000000000000..cb614bd1cc28
--- /dev/null
+++ b/doc/man1/flux-jobs.rst
@@ -0,0 +1,674 @@
+.. flux-help-section: jobs
+
+============
+flux-jobs(1)
+============
+
+
+SYNOPSIS
+========
+
+**flux** **jobs** [*OPTIONS*] [JOBID ...]
+
+DESCRIPTION
+===========
+
+.. program:: flux jobs
+
+:program:`flux jobs` is used to list jobs run under Flux. By default only
+pending and running jobs for the current user are listed. Additional
+jobs and information can be listed using options listed below.
+Alternately, specific job ids can be listed on the command line to
+only list those job IDs.
+
+
+OPTIONS
+=======
+
+.. option:: -a
+
+   List jobs in all states, including inactive jobs.
+   This is shorthand for :option:`--filter=pending,running,inactive`.
+
+.. option:: -A
+
+   List jobs of all users. This is shorthand for :option:`--user=all`.
+
+.. option:: -n, --no-header
+
+   For default output, do not output column headers.
+
+.. option:: -u, --user=[USERNAME|UID]
+
+   List jobs for a specific username or userid. Specify *all* for all users.
+
+.. option:: --name=[JOB NAME]
+
+   List jobs with a specific job name.
+
+.. option:: -q, --queue=QUEUE[,...]
+
+   List jobs in a specific queue or queues. Multiple queues may be separated
+   by a comma or by using the :option:`-q, --queue` option multiple times.
+
+.. option:: -i, --include=HOSTS|RANKS
+
+   List only jobs where the assigned resources intersect with the supplied
+   argument, which may be specified either as an RFC 22 idset of broker ranks
+   or an RFC 29 hostlist of host names. It is not an error to specify ranks or
+   hosts which do not exist.
+
+.. option:: -c, --count=N
+
+   Limit output to N jobs. N=0 means unlimited. (default 1000)
+
+.. option:: --since=WHEN
+
+   Limit output to jobs that have been active since a given timestamp.  In other
+   words, jobs that are currently pending, currently running, or became inactive
+   since the given timestamp.  This option implies :option:`-a` if no other
+   :option:`--filter` options are specified.  If *WHEN* begins with ``-``
+   character, then the remainder is considered to be a an offset in Flux
+   standard duration (RFC 23). Otherwise, any datetime expression accepted by
+   the Python `parsedatetime <https://github.com/bear/parsedatetime>`_ module
+   is accepted. Examples: "-6h", "-1d", "yesterday", "2021-06-21 6am",
+   "last Monday", etc. It is assumed to be an error if a timestamp in
+   the future is supplied.
+
+   .. note::
+      Due to a quirk in the Python argument parsing implementation,
+      it is suggested to always use ``=`` between the :option:`--since`
+      option and its argument, e.g. ``--since=-1d`` rather than ``--since
+      -1d``. In the second case Python mistakenly considers the option
+      argument an unknown option and will raise an error about a missing
+      argument to :option:`--since`.
+
+
+.. option:: -f, --filter=STATE|RESULT
+
+   List jobs with specific job state or result. Multiple states or
+   results can be listed separated by comma. See `JOB STATUS`_ below for
+   additional information. Defaults to *pending,running*.
+
+.. option:: -o, --format=NAME|FORMAT
+
+   Specify a named output format *NAME* or a format string using Python's
+   format syntax. See `OUTPUT FORMAT`_ below for field names. Named formats
+   may be listed via :option:`--format=help`.  An alternate default format can
+   be set via the :envvar:`FLUX_JOBS_FORMAT_DEFAULT` environment variable.
+   Additional named formats may be registered with :program:`flux jobs` via
+   configuration.  See the `CONFIGURATION`_ section for more details. A
+   configuration snippet for an existing named format may be generated with
+   :option:`--format=get-config=NAME`.
+
+.. option:: --sort=[-]KEY,..
+
+   Sort jobs based on a list of comma separated keys. If a KEY is preceded
+   by a dash ``-``, then the sort order is reversed. Supported keys match
+   output field names, e.g. ``id``, ``t_run``, etc. This option overrides
+   any ``sort:`` prefix specified in the current format.
+
+.. option:: --json
+
+   Emit data for selected jobs in JSON format. The data for multiple
+   matching jobs is contained in a ``jobs`` array in the emitted JSON
+   object, unless a single job was selected by jobid on the command
+   line, in which case a JSON object representing that job is emitted on
+   success. With :option:`--recursive`, each job which is also an instance
+   of Flux will will have any recursively listed jobs in a ``jobs`` array,
+   and so on for each sub-child.
+
+   Only the attributes which are available at the time of the
+   :program:`flux jobs` query will be present in the returned JSON object for
+   a job. For instance a pending job will not have ``runtime``, ``waitstatus``
+   or ``result`` keys, among others. A missing key should be considered
+   unavailable.
+
+   The :option:`--json` option is incompatible with :option:`--stats` and
+   :option:`--stats-only`, and any :option:`--format` is ignored.
+
+.. option:: --color[=WHEN]
+
+   Control output coloring.  The optional argument *WHEN* can be
+   *auto*, *never*, or *always*.  If *WHEN* is omitted, it defaults to
+   *always*.  Otherwise the default is *auto*.
+
+.. option:: --stats
+
+   Output a summary of job statistics before the header.  By default
+   shows global statistics.  If :option:`--queue` is specified, shows
+   statistics for the specified queue.  May be useful in conjunction
+   with utilities like :linux:man1:`watch`, e.g.::
+
+      $ watch -n 2 flux jobs --stats -f running -c 25
+
+   will display a summary of statistics along with the top 25
+   running jobs, updated every 2 seconds.
+
+   Note that all job failures, including canceled and timeout jobs,
+   are collectively counted as "failed" in :option:`--stats`.
+
+.. option:: --stats-only
+
+   Output a summary of job statistics and exit.  By default shows
+   global statistics.  If :option:`--queue` is specified, shows statistics
+   for the specified queue.  :program:`flux jobs` will exit with non-zero
+   exit status with :option:`--stats-only` if there are no active jobs. This
+   allows the following loop to work::
+
+       $ while flux jobs --stats-only; do sleep 2; done
+
+   All options other than :option:`--queue` are ignored when
+   :option:`--stats-only` is used.
+
+   Note that all job failures, including canceled and timeout jobs,
+   are collectively counted as "failed" in :option:`--stats-only`.
+
+.. option:: -R, --recursive
+
+   List jobs recursively. Each child job which is also an instance of
+   Flux is prefixed by its jobid "path" followed by the list of jobs,
+   recursively up to any defined :option:`--level`. If the :option:`--stats`
+   option is used, then each child instance in the hierarchy is listed
+   with its stats.
+
+.. option:: --recurse-all
+
+   By default, jobs not owned by the user running :program:`flux jobs` are
+   skipped with :option:`--recursive`, because normally Flux instances
+   only permit the instance owner to connect. This option forces the
+   command to attempt to recurse into the jobs of other users.  Implies
+   :option:`--recursive`.
+
+.. option:: -L, --level=N
+
+   With :option:`--recursive`, stop recursive job listing at level **N**.
+   Levels are counted starting at 0, so :option:`flux jobs -R --level=0` is
+   equivalent to :program:`flux jobs` without :option:`-R`, and
+   :option:`--level=1` would limit recursive job listing to child jobs of the
+   current instance.
+
+.. option:: --threads=N
+
+   When :program:`flux jobs` recursively queries job lists (with
+   :option:`--recursive`) or fetches info for jobs that are also instances
+   (see ``instance.*`` fields), a pool of threads is used to parallelize
+   the required RPCs. Normally, the default number of ThreadPoolExecutor
+   threads is used, but by using the :option:`--threads`, a specific number
+   of threads can be chosen.
+
+.. _flux_jobs_job_status:
+
+JOB STATUS
+==========
+
+Jobs may be observed to pass through five job states in Flux: DEPEND,
+PRIORITY, SCHED, RUN, CLEANUP, and INACTIVE (see Flux RFC 21). Under the
+*state_single* field name, these are abbreviated as D, S, P, R, C, and I
+respectively. For convenience and clarity, the following virtual job
+states also exist: "pending", an alias for DEPEND,PRIORITY,SCHED; "running",
+an alias for RUN,CLEANUP; "active", an alias for "pending,running".
+
+After a job has finished and is in the INACTIVE state, it can be
+marked with one of the possible results: COMPLETED, FAILED,
+CANCELED, TIMEOUT. Under the *result_abbrev* field name, these are
+abbreviated as CD, F, CA, and TO respectively.
+
+The job status is a user friendly mix of both, a job is always in one
+of the following statuses: DEPEND, PRIORITY, SCHED, RUN, CLEANUP, COMPLETED,
+FAILED, CANCELED, or TIMEOUT. Under the *status_abbrev* field name,
+these are abbreviated as D, P, S, R, C, CD, F, CA, and TO respectively.
+
+
+.. _flux_jobs_output_format:
+
+OUTPUT FORMAT
+=============
+
+The :option:`--format` option can be used to specify an output format to
+:program:`flux jobs` using Python's string format syntax. For example, the
+following is the format used for the default format:
+
+::
+
+   {id.f58:>12} ?:{queue:<8.8} {username:<8.8} {name:<10.10+} \
+   {status_abbrev:>2.2} {ntasks:>6} {nnodes:>6h} \
+   {contextual_time!F:>8h} {contextual_info}
+
+If the format string begins with ``sort:k1[,k2,...]``, then ``k1[,k2,...]``
+will be taken to be a comma-separated list of keys on which to sort
+the displayed output. If a sort key starts with ``-``, then the key
+will be sorted in reverse order. The sort order embedded in the format
+ may be overridden on the command line by the :option:`--sort` option.
+
+If a format field is preceded by the special string ``?:`` this will
+cause the field to be removed entirely from output if the result would
+be an empty string or zero value for all jobs in the listing. E.g.::
+
+   {id.f58:>12} ?:{exception.type}
+
+would eliminate the EXCEPTION-TYPE column if no jobs in the list received
+an exception. (Thus the job queue is only displayed if at least one job
+has a queue assigned in the default format shown above).
+
+If a format field is preceded by the special string ``+:`` this will
+cause the field width to be set to the maximum width such that no entry
+will be truncated. If the field already has a width, then this will be
+the minimum width of that field. For example::
+
+  {id.f58:>12} +:{queue:>5}
+
+would set the width of the ``QUEUE`` field to the maximum of 5 and the
+actual width of the largest presented queue.
+
+If a format field is preceded by the string ``?+:``, then the field is
+eliminated if empty, or set the maximum item width.
+
+As a reminder to the reader, some shells will interpret braces
+(``{`` and ``}``) in the format string.  They may need to be quoted.
+
+The special presentation type *h* can be used to convert an empty
+string, "0s", "0.0", "0:00:00", or epoch time to a hyphen. For example, normally
+"{nodelist}" would output an empty string if the job has not yet run.
+By specifying, "{nodelist:h}", a hyphen would be presented instead.
+
+The special suffix *+* can be used to indicate if a string was truncated
+by including a ``+`` character when truncation occurs. If both *h* and
+*+* are being used, then the *+* must appear after the *h*.
+
+Additionally, the custom job formatter supports a set of special
+conversion flags. Conversion flags follow the format field and are
+used to transform the value before formatting takes place. Currently,
+the following conversion flags are supported by :program:`flux jobs`:
+
+**!D**
+   convert a timestamp field to ISO8601 date and time (e.g. 2020-01-07T13:31:00).
+   Defaults to empty string if timestamp field does not exist or the timestamp
+   is 0 (i.e epoch time).
+
+**!d**
+   convert a timestamp to a Python datetime object. This allows datetime
+   specific format to be used, e.g. *{t_inactive!d:%H:%M:%S}*. Additionally,
+   width and alignment can be specified after the time format by using
+   two colons (``::``), e.g. *{t_inactive!d:%H:%M:%S::>20}*. Returns an
+   empty string (or "-" if the *h* suffix is used) for an unset timestamp.
+
+**!F**
+   convert a time duration in floating point seconds to Flux Standard
+   Duration (FSD) string (e.g. *{runtime!F}*).  Defaults to empty string if
+   field does not exist.
+
+**!H**
+   convert a time duration in floating point seconds to
+   hours:minutes:seconds form (e.g. *{runtime!H}*).  Defaults to empty
+   string if time duration field does not exist.
+
+**!P**
+   convert a floating point number into a percentage fitting in 5 characters
+   including the "%" character. E.g. 0.5 becomes "50%" 0.015 becomes 1.5%,
+   and 0.0005 becomes 0.05% etc.
+
+As a reminder to the reader, some shells will interpret the exclamation
+point (``!``) when using a conversion flag.  The exclamation point may
+need to be escaped (``\!``).
+
+Annotations can be retrieved via the *annotations* field name.
+Specific keys and sub-object keys can be retrieved separated by a
+period (".").  For example, if the scheduler has annotated the job
+with a reason pending status, it can be retrieved via
+"{annotations.sched.reason_pending}".
+
+As a convenience, the field names *sched* and *user* can be used as
+substitutions for *annotations.sched* and *annotations.user*.  For
+example, a reason pending status can be retrieved via
+"{sched.reason_pending}".
+
+The field names that can be specified are:
+
+**id**
+   job ID
+
+**id.f58**
+  job ID in RFC 19 F58 (base58) encoding
+
+**id.f58plain**
+  job ID in RFC 19 F58 encoding with ascii ``f``
+
+**id.dec**
+  job ID in decimal representation
+
+**id.hex**
+   job ID in ``0x`` prefix hexadecimal representation
+
+**id.dothex**
+   job ID in dotted hexadecimal representation (``xx.xx.xx.xx``)
+
+**id.words**
+  job ID in mnemonic encoding
+
+**id.emoji**
+  job ID in emoji encoding
+
+**userid**
+   job submitter's userid
+
+**username**
+   job submitter's username
+
+**urgency**
+   job urgency
+
+**priority**
+   job priority
+
+**dependencies**
+   list of any currently outstanding job dependencies
+
+**status**
+   job status (DEPEND, SCHED, RUN, CLEANUP, COMPLETED, FAILED,
+   CANCELED, or TIMEOUT)
+
+**status_abbrev**
+   status but in a max 2 character abbreviation
+
+**status_emoji**
+   status but an appropriate emoji instead of job state / result
+
+**name**
+   job name
+
+**cwd**
+   job current working directory
+
+**queue**
+   job queue
+
+**project**
+   job accounting project
+
+**bank**
+   job accounting bank
+
+**ntasks**
+   job task count
+
+**ncores**
+   job core count
+
+**duration**
+   job duration in seconds
+
+**nnodes**
+   job node count (if job ran / is running), empty string otherwise
+
+**ranks**
+   job ranks (if job ran / is running), empty string otherwise
+
+**nodelist**
+   job nodelist (if job ran / is running), empty string otherwise
+
+**state**
+   job state (DEPEND, SCHED, RUN, CLEANUP, INACTIVE)
+
+**state_single**
+   job state as a single character
+
+**state_emoji**
+   job state but an appropriate emoji instead of DEPEND, SCHED, RUN,
+   CLEANUP, or INACTIVE
+
+**result**
+   job result if job is inactive (COMPLETED, FAILED, CANCELED, TIMEOUT),
+   empty string otherwise
+
+**result_abbrev**
+   result but in a max 2 character abbreviation
+
+**result_emoji**
+   result but an appropriate emoji instead of COMPLETED, FAILED,
+   CANCELED, or TIMEOUT
+
+**success**
+   True of False if job completed successfully, empty string otherwise
+
+**waitstatus**
+   The raw status of the job as returned by :linux:man2:`waitpid` if the job
+   exited, otherwise an empty string. Note: *waitstatus* is the maximum
+   wait status returned by all job shells in a job, which may not necessarily
+   indicate the highest *task* wait status. (The job shell exits with the
+   maximum task exit status, unless a task died due to a signal, in which
+   case the shell exits with 128+signo)
+
+**returncode**
+   The job return code if the job has exited, or an empty string if the
+   job is still active. The return code of a job is the highest job shell
+   exit code, or negative signal number if the job shell was terminated by
+   a signal. If the job was canceled before it started, then the returncode
+   is set to the special value -128.
+
+**exception.occurred**
+   True of False if job had an exception, empty string otherwise
+
+**exception.severity**
+   If exception.occurred True, the highest severity, empty string otherwise
+
+**exception.type**
+   If exception.occurred True, the highest severity exception type, empty string otherwise
+
+**exception.note**
+   If exception.occurred True, the highest severity exception note, empty string otherwise
+
+**t_submit**
+   time job was submitted
+
+**t_depend**
+   time job entered depend state
+
+**t_run**
+   time job entered run state
+
+**t_cleanup**
+   time job entered cleanup state
+
+**t_inactive**
+   time job entered inactive state
+
+**runtime**
+   job runtime
+
+**expiration**
+   time at which job allocation was marked to expire
+
+**t_remaining**
+   If job is running, amount of time remaining before expiration
+
+**annotations**
+   annotations metadata, use "." to get specific keys
+
+**sched**
+   short hand for *annotations.sched*
+
+**user**
+   short hand for *annotations.user*
+
+
+Field names which are specific to jobs which are also instances of Flux
+include:
+
+**instance.stats**
+   a short string describing current job statistics for the instance of
+   the form ``PD:{pending} R:{running} CD:{successful} F:{failed}``
+
+**instance.stats.total**
+   total number of jobs in any state in the instance.
+
+**instance.utilization**
+   number of cores currently allocated divided by the total number of cores.
+   Can be formatted as a percentage with ``!P``, e.g.
+   ``{instance.utilization!P:>4}``.
+
+**instance.gpu_utilization**
+   same as ``instance.utilization`` but for gpu resources
+
+**instance.progress**
+   number of inactive jobs divided by the total number of jobs.
+   Can be formatted as a percentage with ``{instance.progress!P:>4}``
+
+**instance.resources.<state>.{ncores,ngpus}**
+   number of cores, gpus in state ``state``, where ``state`` can be
+   ``all``, ``up``, ``down``, ``allocated``, or ``free``, e.g.
+   ``{instance.resources.all.ncores}``
+
+The following fields may return different information depending on
+the state of the job or other context:
+
+**contextual_info**
+   Returns selected information based on the job's current state.  If the
+   job is in PRIORITY state, then the string ``priority-wait`` is returned.
+   If the job is in DEPEND state, then a list of outstanding dependencies
+   is returned. If the job is in SCHED state and its priority is currently
+   0, then one of ``held`` or ``priority-hold`` will be printed depending
+   on if urgency is also 0, otherwise an estimated time the job will run is
+   returned (if supported by the scheduler). In other states, the assigned
+   nodelist is returned (if resources were assigned).
+
+**contextual_info**
+   Returns the job runtime for jobs in RUN state or later, otherwise the
+   job duration (if set) is returned.
+
+**inactive_reason**
+   If the job is inactive, returns the reason that the job is no
+   longer active.  Generally speaking, will output "Exit", "Timeout",
+   "Canceled", or signal.  If available, other contextual information
+   will also be provided such as the exit ``returncode`` or
+   cancellation message.
+
+.. _flux_jobs_configuration:
+
+CONFIGURATION
+=============
+
+The :program:`flux jobs` command supports registration of named output formats
+in configuration files. The command loads configuration files from
+``flux-jobs.EXT`` from the following paths in order of increasing precedence:
+
+ * ``$XDG_CONFIG_DIRS/flux`` or ``/etc/xdg/flux`` if :envvar:`XDG_CONFIG_DIRS`
+   is not set. Note that :envvar:`XDG_CONFIG_DIRS` is traversed in reverse
+   order such that entries first in the colon separated path are highest
+   priority.
+
+ * ``$XDG_CONFIG_HOME/flux`` or ``$HOME/.config/flux`` if
+   :envvar:`XDG_CONFIG_HOME` is not set
+
+where ``EXT`` can be one of ``toml``, ``yaml``, or ``json``.
+
+If there are multiple ``flux-jobs.*`` files found in a directory, then
+they are loaded in lexical order (i.e. ``.json`` first, then ``.toml``,
+then ``.yaml``)
+
+Named formats are registered in a ``formats`` table or dictionary with a
+key per format pointing to a table or dictionary with the keys:
+
+**format**
+   (required) The format string
+
+**description**
+   (optional) A short description of the named format, displayed with
+   :option:`flux jobs --format=help`
+
+If a format name is specified in more than one config file, then the last
+one loaded is used. Due to the order that :program:`flux jobs` loads config
+files, this allows user configuration to override system configuration. It is
+an error to override any internally defined formats (such as ``default``).
+
+If a format name or string is not specified on the command line the
+internally defined format ``default`` is used.
+
+Example::
+
+  # $HOME/.config/flux/flux-jobs.toml
+
+  [formats.myformat]
+  description = "My useful format"
+  format = """\
+  {id.f58:>12} {name:>8.8} {t_submit!D:<19} \
+  {t_run!D:<19} {t_remaining!F}\
+  """
+
+It may be helpful to start with an existing named format by using the
+:option:`--format=get-config=NAME` option, e.g.::
+
+  $ flux jobs --format=get-config=default >> ~/.config/flux/flux-jobs.toml
+
+Be sure to change the name of the format string from ``default``. It is an
+error to redefine the default format string.
+
+
+EXAMPLES
+========
+
+The default output of :program:`flux jobs` will list the pending and running
+jobs of the current user.  It is equivalent to:
+
+::
+
+    $ flux jobs --filter=pending,running
+
+To list all pending, running, and inactive jobs, of the current user,
+you can use :option:`--filter` option or the :option:`-a` option:
+
+::
+
+    $ flux jobs -a
+
+    OR
+
+    $ flux jobs --filter=pending,running,inactive
+
+To alter which user's jobs are listed, specify the user with :option:`--user`:
+
+::
+
+    $ flux jobs --user=flux
+
+Jobs that have finished may be filtered further by specifying if they
+have completed, failed, or were canceled.  For example, the following
+will list the jobs that have failed or were canceled:
+
+::
+
+    $ flux jobs --filter=failed,canceled
+
+The :option:`--format` option can be used to alter the output format or output
+additional information.  For example, the following would output all
+jobids for the user in decimal form, and output any annotations the
+scheduler attached to each job:
+
+::
+
+   $ flux jobs -a --format="{id} {annotations.sched}"
+
+The following would output the job id and exception information, so a
+user can learn why a job failed.
+
+::
+
+   $ flux jobs --filter=failed --format="{id} {exception.type} {exception.note}"
+
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+FLUX RFC
+========
+
+| :doc:`rfc:spec_22`
+| :doc:`rfc:spec_29`
+
+SEE ALSO
+========
+
+:man1:`flux-pstree`
diff --git a/doc/man1/flux-jobtap.rst b/doc/man1/flux-jobtap.rst
new file mode 100644
index 000000000000..a22df091cfd4
--- /dev/null
+++ b/doc/man1/flux-jobtap.rst
@@ -0,0 +1,79 @@
+==============
+flux-jobtap(1)
+==============
+
+
+SYNOPSIS
+========
+
+| **flux** **jobtap** **load** [*--remove=NAME*] *plugin* [*key=val...*]
+| **flux** **jobtap** **remove** *plugin*
+| **flux** **jobtap** **list** [*--all*]
+| **flux** **jobtap** **query** *plugin*
+
+DESCRIPTION
+===========
+
+The :program:`flux jobtap` command is used to query, load, and remove *jobtap*
+plugins from the Flux job-manager module at runtime.
+
+COMMANDS
+========
+
+load
+----
+
+.. program:: flux jobtap load
+
+Load a new plugin into the job-manager.  Optional *key=val* arguments
+occurring after *plugin*  will set config *key* to *val* for *plugin*.
+
+.. option:: --remove=NAME
+
+  Remove plugin *NAME* before loading *plugin*.  *NAME* may be a
+  :linux:man7:`glob` pattern match.
+
+remove
+------
+
+.. program:: flux jobtap remove
+
+Remove *plugin*. *plugin* may be a :linux:man7:`glob` pattern in
+which case all matching, non-builtin plugins are removed. The
+special value ``all`` may be used to remove all loaded jobtap
+plugins. Builtin plugins (those starting with a leading ``.``) must
+be removed explicitly or by preceding *NAME* with ``.``,
+e.g. ``.*``.
+
+list
+----
+
+.. program:: flux jobtap list
+
+Print the currently loaded list of plugins.  Plugins built in to the job
+manager have a leading ``.`` in the name, e.g. ``.priority-default``.
+They are not displayed by default.
+
+.. option:: -a, --all
+
+  List builtin plugins too.
+
+query
+-----
+
+.. program:: flux jobtap query
+
+Print a JSON object with extended information about *plugin*. This
+includes at least the plugin name and path (or "builtin" if the plugin
+was loaded internally), but may contain plugin-specific data if the plugin
+supports the ``plugin.query`` callback topic.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+SEE ALSO
+========
+
+:man7:`flux-jobtap-plugins`
diff --git a/doc/man1/flux-keygen.adoc b/doc/man1/flux-keygen.adoc
deleted file mode 100644
index b15cfdf29f42..000000000000
--- a/doc/man1/flux-keygen.adoc
+++ /dev/null
@@ -1,77 +0,0 @@
-// flux-help-include: true
-FLUX-KEYGEN(1)
-==============
-:doctype: manpage
-
-
-NAME
-----
-flux-keygen - generate keys for Flux security
-
-
-SYNOPSIS
---------
-*flux* *keygen* ['--force'] ['--plain']
-
-
-DESCRIPTION
------------
-flux-keygen(1) generates long-term keys for Flux security.
-Keys are written to files in '$HOME/.flux'.
-
-Flux comms sessions implement cryptographic privacy and data integrity
-when data is sent over a network.  Point to point ZeroMQ TCP connections
-are protected with the CURVE security mechanism built into ZeroMQ
-version 4, based on curve25519 and a CurveCP-like protocol.
-
-It is possible to start a Flux comms session with security
-disabled or using the toy PLAIN ZeroMQ security mechanism.
-This is intended for testing performance overhead of security only.
-By default, *flux-keygen* generates both CURVE and PLAIN keys.
-PLAIN keys are merely RFC 4122 uuids stored in the clear that are
-used as passwords.
-
-All instances of Flux message brokers launched in a comms session
-need to access your keys, therefore we assume that your '$HOME/.flux'
-directory is globally accessible.  Of course if keys are being transferred
-in the clear across public networks to make this happen, you have
-the same sort of problem as you might have with ssh private keys stored
-in your home directory.
-
-The Flux security design is not yet complete, and these measures
-are probably not the final ones.
-
-OPTIONS
--------
-*-f, --force*::
-Generate new keys, even if they already exist.
-
-*-p, --plain*::
-Generate PLAIN key.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-ZAP: <http://rfc.zeromq.org/spec:27>
-
-CurveZMQ: <http://curvezmq.org/page:read-the-docs>
-
-ZMTP/3.0: <http://rfc.zeromq.org/spec:23>
-
-Using ZeroMQ Security:
-<http://hintjens.com/blog:48> and
-<http://hintjens.com/blog:49>
diff --git a/doc/man1/flux-keygen.rst b/doc/man1/flux-keygen.rst
new file mode 100644
index 000000000000..673531be37c2
--- /dev/null
+++ b/doc/man1/flux-keygen.rst
@@ -0,0 +1,62 @@
+==============
+flux-keygen(1)
+==============
+
+
+SYNOPSIS
+========
+
+**flux** **keygen** [*--name=NAME*] [*--meta=KEY=VAL...*] *PATH*
+
+
+DESCRIPTION
+===========
+
+.. program:: flux keygen
+
+:program:`flux keygen` generates a long-term CURVE certificate used to secure
+the overlay network of a Flux system instance.
+
+The Flux overlay network implements cryptographic privacy and data integrity
+when data is sent over a network.  Point to point ZeroMQ TCP connections
+are protected with the CURVE security mechanism built into ZeroMQ
+version 4, based on curve25519 and a CurveCP-like protocol.
+
+All brokers participating in the system instance must use the same
+certificate.  The certificate is part of the bootstrap configuration.
+
+Flux instances that bootstrap with PMI do not require a configured certificate.
+In that case, each broker self-generates a unique certificate and the
+public keys are exchanged with PMI.
+
+
+OPTIONS
+=======
+
+:program:`flux keygen` accepts the following options:
+
+.. option:: -n, --name=NAME
+
+   Set the certificate metadata ``name`` field.  The value is logged when
+   :man1:`flux-broker` authenticates a peer that presents this certificate.
+   A cluster name might be appropriate here.  Default: the local hostname.
+
+.. option:: --meta=KEY=VAL
+
+   Set arbitrary certificate metadata.  Multiple key-value pairs may be
+   specified, separated by commas.  This option may be specified multiple
+   times.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+ZAP: http://rfc.zeromq.org/spec:27
+
+CurveZMQ: http://curvezmq.org/page:read-the-docs
+
+ZMTP/3.0: http://rfc.zeromq.org/spec:23
+
+Using ZeroMQ Security: http://hintjens.com/blog:48, http://hintjens.com/blog:49
diff --git a/doc/man1/flux-kvs.adoc b/doc/man1/flux-kvs.adoc
deleted file mode 100644
index 8871bb3222c2..000000000000
--- a/doc/man1/flux-kvs.adoc
+++ /dev/null
@@ -1,208 +0,0 @@
-// flux-help-include: true
-FLUX-KVS(1)
-===========
-:doctype: manpage
-
-
-NAME
-----
-flux-kvs - Flux key-value store utility
-
-
-SYNOPSIS
---------
-*flux* *kvs* 'COMMAND' ['OPTIONS']
-
-
-DESCRIPTION
------------
-The Flux key-value store (KVS) is a simple, distributed data storage
-service used a building block by other Flux components.
-flux-kvs(1) is a command line utility that operates on the KVS.
-It is a very thin layer on top of a C API.
-
-The Flux KVS stores values under string keys.  The keys are
-hierarchical, using "." as a path separator, analogous to "/"
-separated UNIX file paths.  A single "." represents the root directory
-of the KVS.
-
-The KVS is distributed among the ranks of a comms session.  Rank 0
-is the leader, and other ranks are caching followers.  All writes are flushed
-to the leader during a commit operation.  Data is stored in a hash tree
-such that every commit results in a new root hash.  Each new root hash
-is multicast across the session.  When followers update their root hash,
-they atomically update their view to match the leader.  There may be a
-delay after a commit while old data is served on a follower that has not yet
-updated its root hash, thus the Flux KVS consistency model is "eventually
-consistent".  Followers cache data temporally and fault in new data through
-their parent in the overlay network.
-
-Different KVS namespaces can be created in which kvs values can be
-read from/written to.  By default, all KVS operations operate on the
-default KVS namespace "primary".  An alternate namespace can be
-specified in most kvs commands via the '--namespace' option, or by
-setting the namespace in the environment variable FLUX_KVS_NAMESPACE.
-
-flux-kvs(1) runs a KVS 'COMMAND'.  The possible commands and their
-arguments are described below.
-
-COMMANDS
---------
-*namespace create* [-o owner] 'name' ['name...']::
-Create a new kvs namespace.  User may specify an alternate userid of a
-user that owns the namespace via '-o'.  Specifying an alternate owner
-would allow a non-instance owner to read/write to a namespace.
-
-*namespace remove* 'name' ['name...']::
-Remove a kvs namespace.
-
-*namespace list*::
-List all current namespaces and info on each namespace.
-
-*get* [-N ns] [-r|-t] [-a treeobj] [-l] [-W] [-w] [-u] [-A] [-f] [-c count] 'key' ['key...']::
-Retrieve the value stored under 'key'.  If nothing has been stored
-under 'key', display an error message.  Specify an alternate namespace
-to retrieve 'key' from via '-N'.  If no options, value is displayed
-with a newline appended (if value length is nonzero).  If '-l', a
-'key=' prefix is added. If '-r', value is displayed without a newline.
-If '-t', the RFC 11 object is displayed.  '-a treeobj' causes the
-lookup to be relative to an RFC 11 snapshot reference.  If '-W' is
-specified and a key does not exist, wait until the key has been
-created.  If '-w', after the initial value, display the new value each
-time the key is written to until interrupted, or if '-c count' is
-specified, until 'count' values have been displayed.  If '-u' is
-specified, only writes that change the key value will be displayed.
-If '-A' is specified, only display appends that occur on a key.  By
-default, only a direct write to a key is monitored, which may miss
-several unique situations, such as the replacement of an entire parent
-directory.  The '-f' option can be specified to monitor for many of
-these special situations.
-
-*put* [-N ns] [-O|-s] [-r|-t] [-n] [-A] 'key=value' ['key=value...']::
-Store 'value' under 'key' and commit it.  Specify an alternate
-namespace to commit value(s) via '-N'.  If it already has a value,
-overwrite it.  If no options, value is stored directly.  If '-r' or
-'-t', the value may optionally be read from standard input if
-specified as "-".  If '-r', the value may include embedded NULL bytes.
-If '-t', value is stored as a RFC 11 object.  '-n' prevents the commit
-from being merged with with other contemporaneous commits.  '-A'
-appends the value to a key instead of overwriting the value.  Append
-is incompatible with the -j option.  After a successful put, '-O' or
-'-s' can be specified to output the RFC11 treeobj or root sequence
-number of the root containing the put(s).
-
-*ls* [-N ns] [-R] [-d] [-F] [-w COLS] [-1] ['key' ...]::
-Display directory referred to by _key_, or "." (root) if unspecified.
-Specify an alternate namespace to display via '-N'.  Remaining options are
-roughly equivalent to a subset of ls(1) options.  '-R' lists directory
-recursively.  '-d' displays directory not its contents.  '-F'
-classifies files with one character suffix (. is directory, @ is
-symlink).  '-w COLS' sets the terminal width in characters.  '-1'
-causes output to be displayed in one column.
-
-*dir* [-N ns] [-R] [-d] [-w COLS] [-a treeobj] ['key']::
-Display all keys and their values under the directory 'key'.  Specify
-an alternate namespace to display via '-N'.  If 'key' does not exist
-or is not a directory, display an error message.  If 'key' is not
-provided, "." (root of the namespace) is assumed.  If '-R' is
-specified, recursively display keys under subdirectories.  If '-d' is
-specified, do not output key values.  Output is truncated to fit the
-terminal width.  '-w COLS' sets the terminal width (0=unlimited).  '-a
-treeobj' causes the lookup to be relative to an RFC 11 snapshot
-reference.
-
-*unlink* [-N ns] [-O|-s] [-R] [-f] 'key' ['key...']::
-Remove 'key' from the KVS and commit the change.  Specify an alternate
-namespace to commit to via '-N'.  If 'key' represents a directory,
-specify '-R' to remove all keys underneath it.  If '-f' is specified,
-ignore nonexistent files.  After a successful unlink, '-O' or '-s' can
-be specified to output the RFC11 treeobj or root sequence number of
-the root containing the unlink(s).
-
-*link* [-N ns] [-T ns] [-O|-s] 'target' 'linkname'::
-Create a new name for 'target', similar to a symbolic link, and commit
-the change.  'target' does not have to exist.  If 'linkname' exists,
-it is overwritten.  Specify an alternate namespace to commit linkname
-to via '-N'.  Specify the target's namespace via '-T'.  After a
-successfully created link, '-O' or '-s' can be specified to output the
-RFC11 treeobj or root sequence number of the root containing the link.
-
-*readlink* [-N ns] [-a treeobj] [ -o | -k ] 'key' ['key...']::
-Retrieve the key a link refers to rather than its value, as would be
-returned by *get*.  Specify an alternate namespace to retrieve from
-via '-N'.  '-a treeobj' causes the lookup to be relative to an RFC 11
-snapshot reference.  If the link points to a namespace, the namespace
-and key will be output in the format '<namespace>::<key>'.  The '-o'
-can be used to only output namespaces and the '-k' can be used to only
-output keys.
-
-*mkdir* [-N ns] [-O|-s] 'key' ['key...']::
-Create an empty directory and commit the change.  If 'key' exists,
-it is overwritten.  Specify an alternate namespace to commit to via
-'-N'.  After a successful mkdir, '-O' or '-s' can be specified to
-output the RFC11 treeobj or root sequence number of the root
-containing the new directory.
-
-*copy* [-S src-ns] [-D dst-ns] 'source' 'destination'::
-Copy 'source' key to 'destination' key.  Optionally, specify a source
-and/or destination namespace for the 'source' and/or 'destination'
-respectively.  If a directory is copied, a new reference is created;
-it is unnecessary for *copy* to recurse into 'source'.
-
-*move* [-S src-ns] [-D dst-ns] 'source' 'destination'::
-Like *copy*, but 'source' is unlinked after the copy.
-
-*dropcache* [--all]::
-Tell the local KVS to drop any cache it is holding.  If '--all' is
-specified, send an event across the comms session instructing all KVS
-instances to drop their caches.
-
-*version* [-N ns]::
-Display the current KVS version, an integer value.  The version starts
-at zero and is incremented on each KVS commit.  Note that some commits
-may be aggregated for performance and the version will be incremented
-once for the aggregation, so it cannot be used as a direct count of
-commit requests.  Specify an alternate namespace to retrieve the
-version from via '-N'.
-
-*wait* [-N ns] 'version'::
-Block until the KVS version reaches 'version' or greater.  A simple form
-of synchronization between peers is:  node A puts a value, commits it,
-reads version, sends version to node B.  Node B waits for version, gets
-value.
-
-*getroot* [-N ns] [-s | -o]::
-Retrieve the current KVS root, displaying it as an RFC 11 dirref object.
-Specify an alternate namespace to retrieve from via '-N'.  If '-o' is
-specified, display the namespace owner.  If '-s' is specified, display
-the root sequence number.
-
-*eventlog get* [-N ns] [-w] [-c count] [-u] 'key'::
-Display the contents of an RFC 18 KVS eventlog referred to by 'key'.
-If '-u' is specified, display the log in raw form.  If '-w' is
-specified, after the existing contents have been displayed, the
-eventlog is monitored and updates are displayed as they are committed.
-This runs until the program is interrupted or an error occurs, unless
-the number of events is limited with the '-c' option.  Specify an
-alternate namespace to display from via '-N'.
-
-*eventlog append* [-N ns] [-t SECONDS] 'key' 'name' ['context ...']::
-Append an event to an RFC 18 KVS eventlog referred to by 'key'.
-The event 'name' and optional 'context' are specified on the command line.
-The timestamp may optionally be specified with '-t' as decimal seconds since
-the UNIX epoch (UTC), otherwise the current wall clock is used.
-Specify an alternate namespace to append to via '-N'.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
diff --git a/doc/man1/flux-kvs.rst b/doc/man1/flux-kvs.rst
new file mode 100644
index 000000000000..e91975bdc7c2
--- /dev/null
+++ b/doc/man1/flux-kvs.rst
@@ -0,0 +1,574 @@
+===========
+flux-kvs(1)
+===========
+
+
+SYNOPSIS
+========
+
+| **flux** **kvs** **get** [*--waitcreate*] [*--watch*] [*--raw*] *key...*
+| **flux** **kvs** **put** [*--append*] [*--raw*] *key=value...*
+| **flux** **kvs** **dir** [*-R*] [*-d*] [*key*]
+| **flux** **kvs** **ls** [*-R*] [*-d*] [*-1*] [*-F*]  *key...*
+| **flux** **kvs** **unlink** [*-R*] [*-f*] *key...*
+| **flux** **kvs** **link** *target* *linkname*
+| **flux** **kvs** **readlink** *key...*
+| **flux** **kvs** **mkdir** *key...*
+| **flux** **kvs** **dropcache**
+
+| **flux** **kvs** **copy** *source* *destination*
+| **flux** **kvs** **move** *source* *destination*
+
+| **flux** **kvs** **getroot**
+| **flux** **kvs** **version**
+| **flux** **kvs** **wait** *version*
+
+| **flux** **kvs** **namespace** **create** [*-o owner*] *name...*
+| **flux** **kvs** **namespace** **remove** *name...*
+| **flux** **kvs** **namespace** **list**
+
+| **flux** **kvs** **eventlog** **append** *key* *name* [*context...*]
+| **flux** **kvs** **eventlog** **get** [*--waitcreate*] [*--watch*] [*-u*] *key*
+| **flux** **kvs** **eventlog** **wait-event** [*-v*] [*--waitcreate*] *key* *event*
+
+
+DESCRIPTION
+===========
+
+The Flux key-value store (KVS) is a simple, distributed data storage
+service used a building block by other Flux components.
+:program:`flux kvs` is a command line utility that operates on the KVS.
+
+The Flux KVS stores values under string keys. The keys are
+hierarchical, using "." as a path separator, analogous to "/"
+separated UNIX file paths. A single "." represents the root directory
+of the KVS.
+
+The KVS is distributed among the broker ranks of a Flux instance. Rank 0
+is the leader, and other ranks are caching followers. All writes are flushed
+to the leader during a commit operation. Data is stored in a hash tree
+such that every commit results in a new root hash. Each new root hash
+is multicast across the Flux instance. When followers update their root hash,
+they atomically update their view to match the leader. There may be a
+delay after a commit while old data is served on a follower that has not yet
+updated its root hash, thus the Flux KVS cache is "eventually consistent".
+Followers expire cache data after a period of disuse, and fault in new data
+through their parent in the overlay network.
+
+The primary KVS namespace is only accessible to the Flux instance owner.
+Other namespaces may be created and assigned to guest users.  Although the
+cache is shared across namespaces, each has an independent root directory,
+which permits commits in multiple namespaces to complete in parallel.
+
+
+COMMANDS
+========
+
+.. program:: flux kvs
+
+The :program:`flux kvs` sub-commands and their arguments are described below.
+
+The following options are common to most sub-commands:
+
+.. option:: -h, --help
+
+  Display help on this sub-command.
+
+.. option:: -N, --namespace=NAME
+
+  Specify an alternate namespace.  By default, the primary KVS namespace
+  or the value of :envvar:`FLUX_KVS_NAMESPACE` is used.
+
+get
+---
+
+.. program:: flux kvs get
+
+Retrieve the value stored under *key* and print it on standard output.
+It is an error if *key* does not exist, unless :option:``--waitcreate`` is
+specified.  It is an error if *key* is a directory.
+
+If multiple *key* arguments are specified, their values are concatenated.
+A newline is appended to each value, unless :option:`--raw` is specified or
+the value is zero length.
+
+.. option:: -r, --raw
+
+  Display value without a newline.
+
+.. option:: -t, --treeobj
+
+  Display RFC 11 tree object.
+
+.. option:: -a, --at=TREEOBJ
+
+  Perform the lookup relative to a directory reference in RFC 11 tree object
+  format.
+
+.. option:: -l, --label
+
+  Add *key=* prefix to output.
+
+.. option:: -W, --waitcreate
+
+  If the key does not exist, wait until it does, then return its value.
+
+.. option:: -w, --watch
+
+  Display the initial value for *key*, then watch for changes and display
+  each new value until interrupted or :option:`--count=N` is reached.
+
+.. option:: -c, --count=N
+
+  With :option:`--watch`, display *N* values then exit.
+
+.. option:: -u, --uniq
+
+  With :option:`--watch`, suppress output when value is the same, even
+  if the watched key was the target of a KVS commit.
+
+.. option:: -A, --append
+
+  With :option:`--watch`, display only the new data when the watched
+  value is appended to.
+
+.. option:: -f, --full
+
+  With :option:`--watch`, monitor key changes with more complete accuracy.
+
+  By default, only a direct write to a key is monitored, thus changes that
+  occur indirectly could be missed, such as when the parent directory is
+  replaced.  The :option:`--full` option ensures these changes are reported
+  as well, at greater overhead.
+
+put
+---
+
+.. program:: flux kvs put
+
+Set *key* to *value*.  If *key* exists, the current value is overwritten.
+If multiple *key=value* pairs are specified, they are sent as one
+commit and succeed or fail atomically.
+
+.. option:: -O, --treeobj-root
+
+   After the commit has completed, display the new root directory reference
+   in RFC 11 tree object format.
+
+.. option:: -b, --blobref
+
+   After the commit has completed, display the new root directory reference
+   as a single blobref.
+
+.. option:: -s, --sequence
+
+   After the commit has completed, display the new root sequence number
+   or "version".
+
+.. option:: -r, --raw
+
+   Store *value* as-is, without adding NUL termination.
+   If *value* is ``-``, read it from standard input.
+   *value* may include embedded NUL bytes.
+
+.. option:: -t, --treeobj
+
+   Interpret *value* as an RFC 11 tree object to be stored directly.
+   If *value* is ``-``, read it from standard input.
+
+.. option:: -n, --no-merge
+
+   Set the ``NO_MERGE`` flag on the commit to ensure it is not combined with
+   other commits.  The KVS normally combines contemporaneous commits to save
+   work, but since commits succeed or fail atomically, this a commit could
+   fail due to collateral damage.  This option prevents that from happening.
+
+.. option:: -A, --append
+
+   Append *value* to key's existing value, if any, instead of overwriting it.
+
+.. option:: -S, --sync
+
+   After the commit has completed, flush pending content and checkpoints
+   to disk.  Commits normally complete with data in memory, which ensures a
+   subsequent :command:`flux kvs get` would receive the updated value, but
+   does not ensure it persists across a Flux instance crash.  This can be
+   used to ensure critical data has been written to non-volatile storage.
+
+dir
+---
+
+.. program:: flux kvs dir
+
+Display all keys and their values under the directory *key*.  Values that
+are too long to fit the terminal width are truncated with "..." appended.
+This command fails if *key* does not exist or is not a directory.
+If *key* is not provided, ``.`` (root of the namespace) is assumed.
+
+.. option:: -R, --recursive
+
+  Recursively display keys under subdirectories.
+
+.. option:: -d, --directory
+
+  List directory entries but not values.
+
+.. option:: -w, --width=N
+
+  Truncate values to fit in *N* columns instead of the terminal width.
+  Specify 0 to avoid truncation entirely.
+
+.. option:: -a, --at=TREEOBJ
+
+  Perform the directory lookup relative to a directory reference in
+  RFC 11 tree object format.
+
+ls
+--
+
+.. program:: flux kvs ls
+
+Display directory referred to by *key*, or "." (root) if unspecified.  This
+sub-command is intended to mimic :linux:man1:`ls` behavior in a limited way.
+
+.. option:: -R, --recursive
+
+  List directory recursively.
+
+.. option:: -d, --directory
+
+  List directory instead of contents.
+
+.. option:: -w, --width=N
+
+  Limit output width to *N* columns.  Specify 0 for unlimited output width.
+
+.. option:: -1, --1
+
+  Force one entry per line.
+
+.. option:: -F, --classify
+
+  Append key type indicator to key: ``.`` for directory.  ``@`` for
+  symbolic link.
+
+unlink
+------
+
+.. program:: flux kvs unlink
+
+Remove key from the KVS.
+
+.. option:: -O, --treeobj-root
+
+   After the commit has completed, display the new root directory reference
+   in RFC 11 tree object format.
+
+.. option:: -b, --blobref
+
+   After the commit has completed, display the new root directory reference
+   as a single blobref.
+
+.. option:: -s, --sequence
+
+   After the commit has completed, display the new root sequence number
+   or "version".
+
+.. option:: -R, --recursive
+
+   Specify recursive removal of a directory.
+
+.. option:: -f, --force
+
+   Ignore nonexistent keys.
+
+link
+----
+
+.. program:: flux kvs link
+
+Create a new name for *target*, similar to a symbolic link.  *target* does not
+have to exist.  If *linkname* exists, it is overwritten.
+
+.. option:: -T, --target-namespace=NAME
+
+  Specify an alternate namespace for *target*.  By default, *target* is
+  in the same namespace as *linkname*.
+
+.. option:: -O, --treeobj-root
+
+   After the commit has completed, display the new root directory reference
+   in RFC 11 tree object format.
+
+.. option:: -b, --blobref
+
+   After the commit has completed, display the new root directory reference
+   as a single blobref.
+
+.. option:: -s, --sequence
+
+   After the commit has completed, display the new root sequence number
+   or "version".
+
+readlink
+---------
+
+.. program:: flux kvs readlink
+
+Print the symbolic link target of *key*.  The target may be another key name,
+or a :option:`namespace::key` tuple.  It is an error if *key* is not a
+symbolic link.
+
+.. option:: -a, --at=TREEOBJ
+
+  Perform the lookup relative to a directory reference in RFC 11 tree object
+  format.
+
+.. option:: -o, --namespace-only
+
+  Print only the namespace name if the target has a namespace prefix.
+
+.. option:: -k, --key-only
+
+  Print only the key if the target has a namespace prefix.
+
+mkdir
+-----
+
+.. program:: flux kvs mkdir
+
+Create an empty directory. If *key* exists, it is overwritten.
+
+.. option:: -O, --treeobj-root
+
+   After the commit has completed, display the new root directory reference
+   in RFC 11 tree object format.
+
+.. option:: -b, --blobref
+
+   After the commit has completed, display the new root directory reference
+   as a single blobref.
+
+.. option:: -s, --sequence
+
+   After the commit has completed, display the new root sequence number
+   or "version".
+
+dropcache
+---------
+
+.. program:: flux kvs dropcache
+
+Tell the local KVS to drop any cache it is holding.
+
+.. option:: -a, --all
+
+  Publish an event across the Flux instance instructing the KVS module on
+  all ranks to drop their caches.
+
+copy
+----
+
+.. program:: flux kvs copy
+
+Copy *source* key to *destination* key.  If a directory is copied, a new
+reference is created.
+
+.. option:: -S, --src-namespace=NAME
+
+  Specify the source namespace.  By default, the primary KVS namespace
+  or the value of :envvar:`FLUX_KVS_NAMESPACE` is used.
+
+.. option:: -D, --dst-namespace=NAME
+
+  Specify the destination namespace.  By default, the primary KVS namespace
+  or the value of :envvar:`FLUX_KVS_NAMESPACE` is used.
+
+move
+----
+
+.. program:: flux kvs move
+
+Copy *source* key to *destination* key and unlink *source*.
+
+.. option:: -S, --src-namespace=NAME
+
+  Specify the source namespace.  By default, the primary KVS namespace
+  or the value of :envvar:`FLUX_KVS_NAMESPACE` is used.
+
+.. option:: -D, --dst-namespace=NAME
+
+  Specify the destination namespace.  By default, the primary KVS namespace
+  or the value of :envvar:`FLUX_KVS_NAMESPACE` is used.
+
+getroot
+-------
+
+.. program:: flux kvs getroot
+
+Retrieve the current KVS root directory reference, displaying it as an
+RFC 11 dirref object unless otherwise specified.
+
+.. option:: -o, --owner
+
+   Display the numerical user ID that owns the target namespace.
+
+.. option:: -b, --blobref
+
+   Display the root directory reference as a single blobref.
+
+.. option:: -s, --sequence
+
+   Display the root sequence number or "version".
+
+version
+-------
+
+.. program:: flux kvs version
+
+Display the current KVS version, an integer value. The version starts
+at zero and is incremented on each KVS commit. Note that some commits
+may be aggregated for performance and the version will be incremented
+once for the aggregation, so it cannot be used as a direct count of
+commit requests.
+
+wait
+----
+
+.. program:: flux kvs wait
+
+Block until the KVS version reaches *version* or greater. A simple form
+of synchronization between peers is: node A puts a value, commits it,
+reads version, sends version to node B. Node B waits for version, gets value.
+
+namespace create
+----------------
+
+.. program:: flux kvs namespace create
+
+Create a new KVS namespace.
+
+.. option:: -o, --owner=UID
+
+  Set the owner of the namespace to *UID*.  If unspecified, the owner is
+  set to the Flux instance owner.
+
+.. option:: -r, --rootref=TREEOBJ
+
+  Initialize namespace with specific root directory reference
+  If unspecified, an empty directory is referenced.
+
+namespace remove
+----------------
+
+.. program:: flux kvs namespace remove
+
+Remove a KVS namespace.  Removal is not guaranteed to be complete when
+the command returns.
+
+namespace list
+--------------
+
+.. program:: flux kvs namespace list
+
+List all current namespaces, with owner and flags.
+
+eventlog get
+------------
+
+.. program:: flux kvs eventlog get
+
+Display the contents of an RFC 18 KVS eventlog referred to by *key*.
+
+.. option:: -W, --waitcreate
+
+  If the key does not exist, wait until it does.
+
+.. option:: -w, --watch
+
+  Monitor the eventlog, displaying new events as they are appended.
+
+.. option:: -c, --count=N
+
+  With :option:`--watch`, exit once *N* events have been displayed.
+
+.. option:: -u, --unformatted
+
+  Display the eventlog in raw RFC 18 form.
+
+.. option:: -H, --human
+
+  Display the eventlog in human-readable form.
+
+.. option:: -L, --color[=WHEN]
+
+  Control output colorization. The optional argument *WHEN* can be one of
+  'auto', 'never', or 'always'. The default value of *WHEN* if omitted is
+  'always'. The default is 'auto' if the option is unused.
+
+eventlog append
+---------------
+
+.. program:: flux kvs eventlog append
+
+Append an event to an RFC 18 KVS eventlog referred to by *key*.
+The event *name* and optional *context* are specified on the command line.
+
+.. option:: -t, --timestamp=SEC
+
+  Specify timestamp in decimal seconds since the UNIX epoch (UTC), otherwise
+  the current wall clock is used.
+
+
+eventlog wait-event
+-------------------
+
+.. program:: flux kvs eventlog wait-event
+
+Wait for a specific *event* to occur in an RFC 18 KVS eventlog
+referred to by *key*.
+
+.. option:: -W, --waitcreate
+
+  If the key does not exist, wait until it does.
+
+.. option:: -t, --timeout=SEC
+
+   Timeout after *SEC* if the specified event has not occurred by then.
+
+.. option:: -u, --unformatted
+
+  Display the event(s) in raw RFC 18 form.
+
+.. option:: -q, --quiet
+
+  Do not display the matched event.
+
+.. option:: -v, --verbose
+
+  Display events prior to the matched event.
+
+.. option:: -H, --human
+
+  Display eventlog events in human-readable form.
+
+.. option:: -L, --color[=WHEN]
+
+  Control output colorization. The optional argument *WHEN* can be one of
+  'auto', 'never', or 'always'. The default value of *WHEN* if omitted is
+  'always'. The default is 'auto' if the option is unused.
+
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_11`
+
+:doc:`rfc:spec_18`
diff --git a/doc/man1/flux-logger.adoc b/doc/man1/flux-logger.adoc
deleted file mode 100644
index fc182da0e35a..000000000000
--- a/doc/man1/flux-logger.adoc
+++ /dev/null
@@ -1,59 +0,0 @@
-// flux-help-include: true
-FLUX-LOGGER(1)
-==============
-:doctype: manpage
-
-
-NAME
-----
-flux-logger - create a Flux log entry 
-
-
-SYNOPSIS
---------
-*flux* *logger* ['--severity SEVERITY'] ['--appname NAME'] 'message' '...'
-
-
-DESCRIPTION
------------
-flux-logger(1) appends Flux log entries to the local Flux
-broker's circular buffer.
-
-Log entries are associated with a syslog(3) style severity.
-Valid severity names are 'emerg', 'alert', 'crit', 'err',
-'warning', 'notice', 'info', 'debug'.
-
-Log entries may also have a user-defined application name.
-This is different than the syslog 'facility', which is always set
-to LOG_USER in Flux log messages.
-
-The wall clock time (UTC) and the broker rank are added to the log
-message when it is created.
-
-OPTIONS
--------
-*-s, --severity*='SEVERITY'::
-Specify the log message severity.  The default severity is 'info'.
-
-*-n, --appname*='NAME'::
-Specify a user-defined application name to associate with the log message.
-The default appname is 'logger'.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-flux-dmesg(1), flux_log(3), syslog(3)
diff --git a/doc/man1/flux-logger.rst b/doc/man1/flux-logger.rst
new file mode 100644
index 000000000000..f69136d62816
--- /dev/null
+++ b/doc/man1/flux-logger.rst
@@ -0,0 +1,53 @@
+==============
+flux-logger(1)
+==============
+
+
+SYNOPSIS
+========
+
+**flux** **logger** [*--severity SEVERITY*] [*--appname NAME*] *message* *...*
+
+DESCRIPTION
+===========
+
+.. program:: flux logger
+
+:program:`flux logger` appends Flux log entries to the local Flux
+broker's circular buffer.
+
+Log entries are associated with a :linux:man3:`syslog` style severity.
+Valid severity names are *emerg*, *alert*, *crit*, *err*,
+*warning*, *notice*, *info*, *debug*.
+
+Log entries may also have a user-defined application name.
+This is different than the syslog *facility*, which is always set
+to LOG_USER in Flux log messages.
+
+The wall clock time (UTC) and the broker rank are added to the log
+message when it is created.
+
+
+OPTIONS
+=======
+
+.. option:: -s, --severity=SEVERITY
+
+   Specify the log message severity. The default severity is *info*.
+
+.. option:: -n, --appname=NAME
+
+   Specify a user-defined application name to associate with the log message.
+   The default appname is *logger*.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-dmesg`, :man3:`flux_log`, :linux:man3:`syslog`
diff --git a/doc/man1/flux-mini.adoc b/doc/man1/flux-mini.adoc
deleted file mode 100644
index 2844b02a2e17..000000000000
--- a/doc/man1/flux-mini.adoc
+++ /dev/null
@@ -1,167 +0,0 @@
-// flux-help-include: true
-FLUX-MINI(1)
-============
-:doctype: manpage
-
-
-NAME
-----
-flux-mini - Minimal Job Submission Tool
-
-
-SYNOPSIS
---------
-*flux* *mini* *submit* [OPTIONS] ['--ntasks=N'] COMMAND...
-
-*flux* *mini* *run* [OPTIONS] ['--ntasks=N'] COMMAND...
-
-
-DESCRIPTION
------------
-flux-mini(1) submits jobs to run under Flux.  The job consists of
-'N' copies of COMMAND launched together as a parallel job.
-If '--ntasks' is unspecified, a value of 'N=1' is assumed.
-
-The *submit* command enqueues the job and prints its numerical Job ID
-on standard output.
-
-The *run* command does the same interactively, blocking until the job
-has completed.
-
-The intent is for the "mini" commands to remain simple with stable interfaces
-over time, making them suitable for use in scripts.  For advanced usage,
-see flux-run(1) and flux-submit(1).
-
-The available OPTIONS are detailed below.
-
-
-JOB PARAMETERS
---------------
-These commands accept only the simplest parameters for expressing
-the size of the parallel program and the geometry of its task slots:
-
-*-n, --ntasks=N*::
-Set the number of tasks to launch (default 1).
-
-*-c, --cores-per-task=N*::
-Set the number of cores to assign to each task (default 1).
-
-*-g, --gpus-per-task=N*::
-Set the number of GPU devices to assign to each task (default none).
-
-*-N, --nodes=N*::
-Set the number of nodes to assign to the job.  Tasks will be distributed
-evenly across the allocated nodes.  It is an error to request more nodes
-than there are tasks.  If unspecified, the number of nodes will be chosen
-by the scheduler.
-
-*-t, --time-limit=FSD*::
-Set a time limit for the job in Flux standard duration (RFC 23).
-FSD is a floating point number with a single character units suffix
-("s", "m", "h", or "d").  If unspecified, the job is subject to the
-system default time limit.
-
-
-STANDARD I/O
-------------
-By default, task stdout and stderr streams are redirected to the
-KVS, where they may be accessed with the `flux job attach` command.
-
-In addition, `flux-mini run` processes standard I/O in real time,
-emitting the job's I/O to its stdout and stderr.
-
-*--output=FILENAME*::
-Redirect stdout to the specified FILENAME, bypassing the KVS.
-The mustache template '{{id}}' is expanded to the numerical Job ID,
-useful to ensure FILENAME is unique across multiple jobs.
-
-*--error=FILENAME*::
-Redirect stderr to the specified FILENAME, bypassing the KVS.
-The mustache template '{{id}}' is expanded as above.
-
-*--label-io*::
-Add task rank prefixes to each line of output.
-
-
-EXIT STATUS
------------
-The job exit status, normally the largest task exit status, is stored
-in the KVS.  If one or more tasks are terminated with a signal,
-the job exit status is 128+signo.
-
-The `flux-job attach` command exits with the job exit status.
-
-In addition, `flux-mini run` runs until the job completes and exits
-with the job exit status.
-
-
-OTHER OPTIONS
--------------
-*--priority=N*::
-Specify job priority, which affects queue order.  Numerically higher priority
-jobs are considered by the scheduler first.  Guests may submit jobs with
-priority in the range of 0 to 16, while instance owners may submit jobs
-with priority in the range of 0 to 31 (default 16).
-
-*-v, --verbose*::
-_(run only)_ Increase verbosity on stderr.  For example, currently `-v`
-displays jobid, `-vv` displays job events, and `-vvv` displays exec events.
-The specific output may change in the future.
-
-*-o, --setopt=KEY[=VAL]*::
-Set shell option.  Keys may include periods to denote hierarchy.
-VAL is optional and may be valid JSON (bare values, objects, or arrays),
-otherwise VAL is interpreted as a string. If VAL is not set, then the
-default value is 1. See SHELL OPTIONS below.
-
-*--setattr=KEY=VAL*::
-Set jobspec attribute.  Keys may include periods to denote hierarchy.
-VAL may be valid JSON (bare values, objects, or arrays), otherwise VAL
-is interpreted as a string.
-
-*--dry-run*::
-Don't actually submit the job.  Just emit jobspec on stdout and exit.
-
-*--debug*::
-Enable job debug events, primarily for debugging Flux itself.
-The specific effects of this option may change in the future.
-
-
-SHELL OPTIONS
--------------
-These options are provided by built-in shell plugins that may be
-overridden in some cases:
-
-*mpi=spectrum*::
-Load the MPI personality plugin for IBM Spectrum MPI.  All other MPI
-plugins are loaded by default.
-
-*affinity=per-task*::
-Tasks are distributed across the assigned resources.
-
-*affinity=off*::
-Disable task affinity plugin.
-
-*gpu-affinity=per-task*::
-GPU devices are distributed evenly among local tasks.  Otherwise,
-GPU device affinity is to the job.
-
-*gpu-affinity=off*::
-Disable GPU affinity for this job.
-
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
diff --git a/doc/man1/flux-module.adoc b/doc/man1/flux-module.adoc
deleted file mode 100644
index bdc7095c48cf..000000000000
--- a/doc/man1/flux-module.adoc
+++ /dev/null
@@ -1,158 +0,0 @@
-// flux-help-include: true
-FLUX-MODULE(1)
-==============
-:doctype: manpage
-
-
-NAME
-----
-flux-module - manage Flux extension modules
-
-
-SYNOPSIS
---------
-*flux* *module* 'COMMAND' ['OPTIONS']
-
-
-DESCRIPTION
------------
-flux-module(1) manages dynamically loadable Flux modules.
-It can load/remove/list modules for the flux-broker(1), and for other
-Flux services that support dynamic module extensions.
-
-
-COMMANDS
---------
-*info* ['name']::
-Display information about module 'name'.
-If 'name' includes a slash '/' character, it is interpreted as a
-file path, and the module name is then determined by reading the
-*mod_name* symbol.  Otherwise, FLUX_MODULE_PATH is searched for a module
-with *mod_name* equal to 'name'.
-
-*load* 'name' ['module-arguments' ...]::
-Load module 'name',  interpreted as described above.
-The service that will load the module is inferred
-from the module name.  When the load command completes successfully,
-the new module is ready to accept messages on all targeted ranks.
-
-*remove* [--force] 'name'::
-Remove module 'name'.  The service that will unload the module is
-inferred from the name specified on the command line. If '-f, --force'
-is used, then do not error if module 'name' is not loaded.
-
-*reload* [--force] 'name' ['module-arguments' ...]::
-Reload module 'name'. This is equivalent to running 'flux module remove'
-followed by 'flux module load'. It is a fatal error if module 'name' is
-not loaded during removal unless the `-f, --force` option is specified.
-
-*list* ['service']::
-List modules loaded by 'service', or by flux-broker(1) if 'service' is unspecified.
-
-*stats* ['OPTIONS'] ['name']::
-Request statistics from module 'name'.  A JSON object containing a set of
-counters for each type of Flux message is returned by default, however
-the object may be customized on a module basis.
-
-*debug* ['OPTIONS'] ['name']::
-Manipulate debug flags in module 'name'.  The interpretation of debug
-flag bits is private to the module and its test drivers.
-
-STATS OPTIONS
--------------
-*-p, --parse*'=OBJNAME'::
-OBJNAME is a period delimited list of field names that should be walked
-to obtain a specific value or object in the returned JSON.
-
-*-t, --type*'=int|double'::
-Force the returned value to be converted to int or double.
-
-*-s, --scale*'=N'::
-Multiply the returned (int or double) value by the specified
-floating point value.
-
-*-R, --rusage*::
-Return a JSON object representing an 'rusage' structure
-returned by getrusage(2).
-
-*-c, --clear*::
-Send a request message to clear statistics in the target module.
-
-*-C, --clear-all*::
-Broadcast an event message to clear statistics in the target module
-on all ranks.
-
-DEBUG OPTIONS
--------------
-
-*-c, --clear*::
-Set debug flags to zero.
-
-*-S, --set*'=MASK'::
-Set debug flags to MASK.
-The value may be prefixed with 0x to indicate hexadecimal or 0
-to indicate octal, otherwise the value is interpreted as decimal.
-
-*-c, --clearbit*'=MASK'::
-Clear the debug bits specified in MASK without disturbing other bits.
-The value is interpreted as above.
-
-*-s, --setbit*'=MASK'::
-Set the debug bits specified in MASK without disturbing other bits.
-The value is interpreted as above.
-
-LIST OUTPUT
------------
-
-The 'list' command displays one line for each unique (as determined by
-SHA1 hash) module loaded by a service.
-
-*Module*::
-The value of the *mod_name* symbol for this module.
-
-*Size*::
-The size in bytes of the module .so file.
-
-*Digest*::
-The last 7 characters of the SHA1 digest of the contents of
-the module .so file.
-
-*Idle*::
-Idle times are defined for flux-broker(1) comms modules as the number of
-heartbeats since the module last sent a request or response message.
-The idle time may be defined differently for other services, or have no
-meaning.
-
-
-MODULE SYMBOLS
---------------
-All Flux modules define the following global symbols:
-
-*const char *mod_name;*::
-A null-terminated string defining the module name.
-Module names are words delimited by periods, with the service that
-will load the module indicated by the words that prefix the final one.
-If there is no prefix, the module is loaded by flux-broker(1).
-
-*int mod_main (void *context, int argc, char **argv);*::
-An entry function.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-syslog(3)
diff --git a/doc/man1/flux-module.rst b/doc/man1/flux-module.rst
new file mode 100644
index 000000000000..e8c7d8ed33b0
--- /dev/null
+++ b/doc/man1/flux-module.rst
@@ -0,0 +1,268 @@
+==============
+flux-module(1)
+==============
+
+
+SYNOPSIS
+========
+
+| **flux** **module** **load** [*--name*] *module* [*args...*]
+| **flux** **module** **reload** [*--name*] [*--force*] *module* [*args...*]
+| **flux** **module** **remove** [*--force*] *name*
+| **flux** **module** **list** [*-l*]
+| **flux** **module** **stats** [*-R*] [*--clear*] *name*
+| **flux** **module** **debug** [*--setbit=VAL*] [*--clearbit=VAL*] [*--set=MASK*] [*--clear=MASK*] *name*
+| **flux** **module** **trace** [-f] [*-t TYPE,...*] [-T *topic-glob*] *name...*
+
+
+
+DESCRIPTION
+===========
+
+.. program:: flux module
+
+:program:`flux module` manages dynamically loadable :man1:`flux-broker` modules.
+
+
+COMMANDS
+========
+
+load
+----
+
+.. program:: flux module load
+
+Load *module*, which may be either the path to a shared object file, including
+the ``.so`` suffix, or the basename of a shared object file on the
+:envvar:`FLUX_MODULE_PATH`, without the suffix.
+
+When :program:`flux module load` completes successfully, the new module has
+entered the running state (see LIST OUTPUT below).
+
+.. option:: -n, --name=NAME
+
+  Override the default module name.  A single shared object file may be
+  loaded multiple times under different names.
+
+reload
+------
+
+.. program:: flux module reload
+
+Reload module *name*. This is equivalent to running
+:program:`flux module remove` followed by :program:`flux module load`.
+
+.. option:: -f, --force
+
+  Suppress failure if *module* is not loaded and proceed with loading.
+
+.. option:: -n, --name=NAME
+
+  Override the default module name.
+
+remove
+------
+
+.. program:: flux module reload
+
+Remove module *name*.
+
+.. option:: -f, --force
+
+  Suppress failure if module *name* is not loaded.
+
+list
+----
+
+.. program:: flux module list
+
+List the loaded modules.
+
+.. option:: -l, --long
+
+  Include the full DSO path for each module.
+
+stats
+-----
+
+.. program:: flux module stats
+
+Request statistics from module *name*. A JSON object containing a set of
+counters for each type of Flux message is returned by default, however
+the object may be customized on a module basis.
+
+.. option:: -p, --parse=OBJNAME
+
+  *OBJNAME* is a period delimited list of field names that should be walked
+  to obtain a specific value or object in the returned JSON.
+
+.. option:: -t, --type=int|double
+
+  Force the returned value to be converted to int or double.
+
+.. option:: -s, --scale=N
+
+  Multiply the returned (int or double) value by the specified
+  floating point value.
+
+.. option:: -R, --rusage=[self|children|thread]
+
+  Return a JSON object representing an *rusage* structure
+  returned by :linux:man2:`getrusage`.  If specified, the optional argument
+  specifies the query target (default: self).
+
+.. option:: -c, --clear
+
+  Send a request message to clear statistics in the target module.
+
+.. option:: -C, --clear-all
+
+  Broadcast an event message to clear statistics in the target module
+  on all ranks.
+
+debug
+-----
+
+.. program:: flux module debug
+
+Manipulate debug flags in module *name*. The interpretation of debug
+flag bits is private to the module and its test drivers.
+
+.. option:: -C, --clear
+
+  Set all debug flags to 0.
+
+.. option:: -S, --set=MASK
+
+  Set debug flags to *MASK*.
+
+.. option:: -s, --setbit=VAL
+
+  Set one debug flag *VAL* to 1.
+
+.. option:: -c, --clearbit=VAL
+
+  Set one debug flag *VAL* to 0.
+
+trace
+-----
+
+.. program:: flux module trace
+
+Display message summaries for messages transmitted and received by the
+named modules.
+
+.. option:: -f, --full
+
+   Include JSON payload in output, if any.  Payloads that are not JSON are
+   not displayed.
+
+.. option:: -T, --topic=GLOB
+
+   Filter output by topic string.
+
+.. option:: -t, --type=TYPE,...
+
+   Filter output by message type, a comma-separated list.  Valid types are
+   ``request``, ``response``, ``event``, or ``control``.
+
+.. option:: -L, --color=WHEN
+
+   Colorize output when supported; WHEN can be ``always`` (default if omitted),
+   ``never``, or ``auto`` (default).
+
+.. option:: -H, --human
+
+   Display human-readable output. See also :option:`--color` and
+   :option:`--delta`.
+
+.. option:: -d, --delta
+
+   With :option:`--human`, display the time delta between messages instead
+   of a relative offset since the last absolute timestamp.
+
+
+DEBUG OPTIONS
+=============
+
+.. program:: flux module debug
+
+.. option:: -c, --clear
+
+   Set debug flags to zero.
+
+.. option:: -S, --set=MASK
+
+   Set debug flags to MASK.
+   The value may be prefixed with 0x to indicate hexadecimal or 0
+   to indicate octal, otherwise the value is interpreted as decimal.
+
+.. option:: -c, --clearbit=MASK
+
+   Clear the debug bits specified in MASK without disturbing other bits.
+   The value is interpreted as above.
+
+.. option:: -s, --setbit=MASK
+
+   Set the debug bits specified in MASK without disturbing other bits.
+   The value is interpreted as above.
+
+
+LIST OUTPUT
+===========
+
+.. program:: flux module list
+
+The *list* command displays one line for each unique (as determined by
+SHA1 hash) loaded module.
+
+**Module**
+   The value of the **mod_name** symbol for this module.
+
+**Idle**
+   Idle times are defined as the number of seconds since the module last sent
+   a request or response message.
+
+**State**
+   The state of the module is shown as a single character: *I* initializing,
+   *R* running, *F* finalizing, *E* exited.  A module automatically enters
+   running state when it calls :man3:`flux_reactor_run`.  It can transition
+   earlier by calling `flux_module_set_running()`.
+
+**Service**
+   If the module has registered additional services, the service names are
+   displayed in a comma-separated list.
+
+**Path**
+   The full path to the broker module shared object file (only shown with
+   the **-l, --long** option).
+
+
+MODULE SYMBOLS
+==============
+
+All Flux modules define the following global symbols:
+
+**const char \*mod_name;**
+   A null-terminated string defining the module name.
+
+**int mod_main (void \*context, int argc, char \**argv);**
+   An entry function.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_5`
+
+
+SEE ALSO
+========
+
+:linux:man3:`syslog`
diff --git a/doc/man1/flux-overlay.rst b/doc/man1/flux-overlay.rst
new file mode 100644
index 000000000000..1702b99cbfec
--- /dev/null
+++ b/doc/man1/flux-overlay.rst
@@ -0,0 +1,286 @@
+.. flux-help-description: Show flux overlay network status
+.. flux-help-section: instance
+
+===============
+flux-overlay(1)
+===============
+
+
+SYNOPSIS
+========
+
+| **flux** **overlay** **status** [*-v*] [*--timeout=FSD*] [*--rank=N*]
+| **flux** **overlay** **errors** [*--timeout=FSD*]
+| **flux** **overlay** **lookup** *target*
+| **flux** **overlay** **parentof** *rank*
+| **flux** **overlay** **disconnect** [*--parent=RANK*] *target*
+| **flux** **overlay** **trace** [-f] [*-r rank*] [*-t TYPE,...*] [*topic-glob*]
+
+
+DESCRIPTION
+===========
+
+.. program:: flux overlay
+
+:program:`flux overlay` is a utility for the Flux tree based overlay network.
+
+COMMANDS
+========
+
+status
+------
+
+.. program:: flux overlay status
+
+:program:`flux overlay status` reports the current status of the tree based
+overlay network.  The possible status values are shown in `SUBTREE STATUS`_
+below.
+
+.. option:: -r, --rank=[RANK]
+
+   Check health of sub-tree rooted at NODEID (default 0).
+
+.. option:: -v, --verbose=[LEVEL]
+
+   Increase reporting detail: 1=show time since current state was entered,
+   2=show round-trip RPC times.
+
+.. option:: -t, --timeout=FSD
+
+   Set RPC timeout, 0=disable (default 0.5s)
+
+.. option:: --summary
+
+   Show only the root sub-tree status.
+
+.. option:: --down
+
+   Show only the partial/degraded sub-trees.
+
+.. option:: --no-pretty
+
+   Do not indent entries and use line drawing characters to show overlay
+   tree structure
+
+.. option:: --no-ghost
+
+   Do not fill in presumed state of nodes that are inaccessible behind
+   offline/lost overlay parents.
+
+.. option:: -L, --color=WHEN
+
+   Colorize output when supported; WHEN can be 'always' (default if omitted),
+   'never', or 'auto' (default).
+
+.. option:: -H, --highlight=TARGET
+
+   Highlight one or more targets and their ancestors.
+
+.. option:: -w, --wait=STATE
+
+   Wait until sub-tree enters *STATE* before reporting (full, partial, offline,
+   degraded, lost).
+
+errors
+------
+
+.. program:: flux overlay errors
+
+:program:`flux overlay errors` summarizes any errors recorded for lost or
+offline nodes.  The output consists of one line per unique error with a
+hostlist prefix.
+
+.. option:: -t, --timeout=FSD
+
+   Set RPC timeout, 0=disable (default 0.5s)
+
+lookup
+------
+
+.. program:: flux overlay lookup
+
+Translate a hostlist *target* to a rank idset or a rank idset *target* to
+hostlist.
+
+parentof
+--------
+
+.. program:: flux overlay parentof
+
+Show the parent of *rank*.
+
+disconnect
+----------
+
+.. program:: flux overlay disconnect
+
+Disconnect a subtree rooted at *target* (hostname or rank).
+
+.. option:: -r, --parent=NODEID
+
+  Set parent rank to *NODEID*.  By default, the parent is determined from
+  the topology.
+
+trace
+-----
+
+.. program:: flux overlay trace
+
+Display message summaries for messages transmitted and received on the
+overlay network.  A topic string glob pattern may be supplied as a positional
+argument.
+
+.. option:: -f, --full
+
+   Include JSON payload in output, if any.  Payloads that are not JSON are
+   not displayed.
+
+.. option:: -r, --rank=NODEID
+
+   Filter output by overlay network peer rank.  Note that this rank is not
+   necessarily the same as the message source or destination.
+
+.. option:: -t, --type=TYPE,...
+
+   Filter output by message type, a comma-separated list.  Valid types are
+   ``request``, ``response``, ``event``, or ``control``.
+
+.. option:: -L, --color=WHEN
+
+   Colorize output when supported; WHEN can be ``always`` (default if omitted),
+   ``never``, or ``auto`` (default).
+
+.. option:: -H, --human
+
+   Display human-readable output. See also :option:`--color` and
+   :option:`--delta`.
+
+.. option:: -d, --delta
+
+   With :option:`--human`, display the time delta between messages instead
+   of a relative offset since the last absolute timestamp.
+
+
+
+EXAMPLES
+========
+
+By default, :program:`flux overlay status` shows the status of the full
+Flux instance in graphical form, e.g.
+
+::
+
+  $ flux overlay status
+  0 test0: partial
+  ├─ 1 test1: offline
+  ├─ 2 test2: full
+  ├─ 3 test3: full
+  ├─ 4 test4: full
+  ├─ 5 test5: offline
+  ├─ 6 test6: full
+  └─ 7 test7: offline
+
+The time in the current state is reported if ``-v`` is added, e.g.
+
+::
+
+  $ flux overlay status -v
+  0 test0: partial for 2.55448m
+  ├─ 1 test1: offline for 12.7273h
+  ├─ 2 test2: full for 2.55484m
+  ├─ 3 test3: full for 18.2725h
+  ├─ 4 test4: full for 18.2777h
+  ├─ 5 test5: offline for 12.7273h
+  ├─ 6 test6: full for 18.2784h
+  └─ 7 test7: offline for 12.7273h
+
+Round trip RPC times are shown with ``-vv``, e.g.
+
+::
+
+  0 test0: partial for 4.15692m (2.982 ms)
+  ├─ 1 test1: offline for 12.754h
+  ├─ 2 test2: full for 4.15755m (2.161 ms)
+  ├─ 3 test3: full for 18.2992h (2.332 ms)
+  ├─ 4 test4: full for 18.3045h (2.182 ms)
+  ├─ 5 test5: offline for 12.754h
+  ├─ 6 test6: full for 18.3052h (2.131 ms)
+  └─ 7 test7: offline for 12.754h
+
+
+
+::
+
+At times an error summary for *lost* nodes may be useful, e.g.
+
+::
+
+  $ flux overlay errors
+  test[2,5]: lost connection
+  test[6-7]: lost parent
+
+A broker that is not responding but is not shown as *lost* or *offline* may
+be forcibly disconnected from the overlay network with
+
+::
+
+  $ flux overlay disconnect 2
+  flux-overlay: asking test0 (rank 0) to disconnect child test2 (rank 2)
+
+However, before doing that it may be useful to see if a broker acting as a
+router to that node is actually the problem.  The parent of a broker rank may
+be listed with
+
+::
+
+  $ flux overlay parentof 2
+  0
+
+Finally, translation between hostnames and broker ranks is accomplished with
+
+::
+
+  $ flux overlay lookup 2
+  test2
+  $ flux overlay lookup test2
+  2
+
+
+SUBTREE STATUS
+==============
+
+The possible overlay subtree status values are:
+
+full
+    Node is online and no children are in *partial*, *offline*, *degraded*, or
+    *lost* state.
+
+partial
+    Node is online, and some children are in *partial* or *offline* state; no
+    children are in *degraded* or *lost* state.
+
+degraded
+    Node is online, and some children are in *degraded* or *lost* state.
+
+lost
+    Node has gone missing, from the parent perspective.
+
+offline
+    Node has not yet joined the instance, or has been cleanly shut down.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_3`
+
+
+SEE ALSO
+========
+
+:man1:`flux-ping`
diff --git a/doc/man1/flux-pgrep.rst b/doc/man1/flux-pgrep.rst
new file mode 100644
index 000000000000..9005a084f197
--- /dev/null
+++ b/doc/man1/flux-pgrep.rst
@@ -0,0 +1,113 @@
+.. flux-help-include: true
+.. flux-help-section: jobs
+.. flux-help-command: pgrep/pkill
+
+==============
+flux-pgrep(1)
+==============
+
+
+SYNOPSIS
+========
+
+**flux** **pgrep** [*OPTIONS*] expression..
+
+**flux** **pkill** [*OPTIONS*] expression..
+
+DESCRIPTION
+===========
+
+.. program:: flux pgrep
+
+:program:`flux pgrep` lists jobids that match a supplied expression. The
+expression may contain a pattern which matches the job name, or
+a range of jobids in the form ``jobid1..jobid2``. If both a pattern
+and jobid range are supplied then both must match.
+
+In rare cases, a pattern may appear to be a jobid range, when the
+pattern ``..`` is used and the strings on both sides area also valid
+Flux jobids. In that case, a name pattern match can be forced by
+prefixing the pattern with ``name:``, e.g. ``name:fr..123``.
+
+By default, only active jobs for the current user are considered.
+
+:program:`flux pkill` cancels matching jobs instead of listing them.
+
+OPTIONS
+=======
+
+.. option:: -a
+
+   Include jobs in all states, including inactive jobs.
+   This is shorthand for :option:`--filter=pending,running,inactive`.
+   (pgrep only)
+
+.. option:: -A
+
+   Include jobs for all users. This is shorthand for :option:`--user=-all`.
+
+.. option:: -u, --user=USER
+
+   Fetch jobs only for the given user, instead of the current UID.
+
+.. option:: -f, --filter=STATE|RESULT
+
+   Include jobs with specific job state or result. Multiple states or
+   results can be listed separated by comma. See the JOB STATUS section
+   of the :man1:`flux-jobs` manual for more detail.
+
+.. option:: -q, --queue=QUEUE[,...]
+
+   Only include jobs in the named queue *QUEUE*. Multiple queues may be
+   specified as a comma-separated list, or by using the :option:`--queue`
+   option multiple times.
+
+.. option:: -c, --count=N
+
+   Limit output to the first *N* matches (default 1000).
+
+.. option:: --max-entries=N
+
+   Limit the number of jobs to consider to *N* entries (default 1000).
+
+.. option:: -o, --format=NAME|FORMAT
+
+   Specify a named output format *NAME* or a format string using Python's
+   format syntax.  An alternate default format can be set via the
+   :envvar:`FLUX_PGREP_FORMAT_DEFAULT` environment variable.  For full
+   documentation of this option, including supported field names and format
+   configuration options, see :man1:flux-jobs. This command shares configured
+   named formats with *flux-jobs* by reading *flux-jobs* configuration files.
+   Supported builtin named formats include *default*, *full*, *long*, and
+   *deps*. The default format emits the matched jobids only. (pgrep only)
+
+.. option:: -n, --no-header
+
+   Suppress printing of the header line. (pgrep only)
+
+.. option:: -w, --wait
+
+   Wait for jobs to finish after cancel. (pkill only)
+
+EXIT STATUS
+===========
+
+0
+   One or more jobs matched the supplied expression. For *pkill* the
+   process have also been successfully canceled.
+
+1
+   No jobs matched or there was an error canceling them.
+
+2
+   Syntax or other command line error.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+SEE ALSO
+========
+
+:man1:`flux-jobs`
diff --git a/doc/man1/flux-ping.adoc b/doc/man1/flux-ping.adoc
deleted file mode 100644
index f2a91766c705..000000000000
--- a/doc/man1/flux-ping.adoc
+++ /dev/null
@@ -1,96 +0,0 @@
-// flux-help-include: true
-FLUX-PING(1)
-============
-:doctype: manpage
-
-
-NAME
-----
-flux-ping - measure round-trip latency to Flux services
-
-
-SYNOPSIS
---------
-*flux* *ping* ['OPTIONS'] target
-
-
-DESCRIPTION
------------
-flux-ping(1) measures round-trip latency to a Flux service implementing
-the "ping" method in a manner analogous to ping(8).  The ping response is
-essentially an echo of the request, with the route taken to the service
-added by the service.  This route is displayed in the output and can
-give insight into how various addresses are routed.
-
-'target' may be the name of a comms module service, e.g. "kvs".
-flux-ping(1) will send a request to "kvs.ping".  As a shorthand,
-'target' can include a rank prefix delimited by an exclamation point.
-"flux ping 4!kvs" is equivalent to "flux ping --rank 4 kvs" (see --rank
-option below).  Don't forget to quote the exclamation point if it is
-interpreted by your shell.
-
-As a shorthand, 'target' may also simply be a rank by itself
-indicating that the broker on that rank or ranks, rather than a comms
-module, is to be pinged.  "flux ping 1" is equivalent to
-"flux ping --rank 1 cmb".
-
-OPTIONS
--------
-*-r, --rank*'=N'::
-Find target on a specific broker rank.  Special case strings ``_any_''
-and ``_upstream_'' available to ping FLUX_NODEID_ANY and FLUX_NODEID_UPSTREAM
-respectively.  Default: send to ``_any_''.
-
-*-p, --pad*'=N'::
-Include in the payload a string of length 'N' bytes.  The payload will be
-echoed back in the response.  This option can be used to explore the
-effect of message size on latency.  Default: no padding.
-
-*-i, --interval*'=N'::
-Specify the delay, in seconds, between successive requests.
-A value of zero is valid and indicates that there should be no delay.
-Requests are sent without waiting for responses.  Default: 1.0 seconds.
-
-*-c, --count*'=N'::
-Specify the number of requests to send, and terminate the command once
-responses have been received for all the requests.  Default: unlimited.
-
-*-b, --batch*::
-Begin processing responses after all requests are sent.  Requires --count.
-
-*-u, --userid*::
-Include userid and rolemask of original request, which are echoed back
-in ping response, in ping output.
-
-EXAMPLES
---------
-
-One can ping a service by name, e.g.
-
-  $ flux ping kvs
-  kvs.ping pad=0 seq=0 time=0.774 ms (0EB02!A3368!0!382A6)
-  kvs.ping pad=0 seq=1 time=0.686 ms (0EB02!A3368!0!382A6)
-  ...
-
-This tells you that the local "kvs" comms module is alive and the
-round-trip latency is a bit over half a millisecond.  The route hops are:
-
-  0EB02: UUID of the ping command
-  A3368: UUID of the API module
-  0:     rank of the local broker
-  382A6: UUID of the KVS module.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
diff --git a/doc/man1/flux-ping.rst b/doc/man1/flux-ping.rst
new file mode 100644
index 000000000000..c714174a3393
--- /dev/null
+++ b/doc/man1/flux-ping.rst
@@ -0,0 +1,100 @@
+============
+flux-ping(1)
+============
+
+
+SYNOPSIS
+========
+
+**flux** **ping** [*OPTIONS*] target
+
+
+DESCRIPTION
+===========
+
+.. program:: flux ping
+
+:program:`flux ping` measures round-trip latency to a Flux service implementing
+the "ping" method in a manner analogous to ping(8). The ping response is
+essentially an echo of the request, with the route taken to the service
+added by the service. This route is displayed in the output and can
+give insight into how various addresses are routed.
+
+*target* may be the name of a Flux service, e.g. "kvs".
+:program:`flux ping` will send a request to "kvs.ping". As a shorthand,
+*target* can include a rank or host prefix delimited by an exclamation point.
+:program:`flux ping 4!kvs` is equivalent to :option:`flux ping --rank 4 kvs`
+(see :option:`--rank` option below). Don't forget to quote the exclamation
+point if it is interpreted by your shell.
+
+As a shorthand, *target* may also simply be a rank or host by itself
+indicating that the broker on that rank/host, rather than a Flux
+service, is to be pinged. :command:`flux ping 1` is equivalent to
+:option:`flux ping --rank 1 broker`.
+
+
+OPTIONS
+=======
+
+.. option:: -r, --rank=N
+
+   Find target on a specific broker rank. Special case strings “*any*”
+   and “*upstream*” available to ping FLUX_NODEID_ANY and FLUX_NODEID_UPSTREAM
+   respectively. Default: send to “*any*”.
+
+.. option:: -p, --pad=N
+
+   Include in the payload a string of length *N* bytes. *N* may be a
+   floating point number with optional multiplicative suffix k,K=1024,
+   M=1024\*1024, or G=1024\*1024\*1024. The payload will be echoed back in
+   the response. This option can be used to explore the effect of message
+   size on latency. Default: no padding.
+
+.. option:: -i, --interval=Ns
+
+   Specify the delay, in seconds, between successive requests.
+   A value of zero is valid and indicates that there should be no delay.
+   Requests are sent without waiting for responses. Default: 1.0 seconds.
+
+.. option:: -c, --count=N
+
+   Specify the number of requests to send, and terminate the command once
+   responses have been received for all the requests. Default: unlimited.
+
+.. option:: -b, --batch
+
+   Begin processing responses after all requests are sent. Requires --count.
+
+.. option:: -u, --userid
+
+   Include userid and rolemask of original request, which are echoed back
+   in ping response, in ping output.
+
+
+EXAMPLES
+========
+
+One can ping a service by name, e.g.
+
+::
+
+   $ flux ping kvs
+   kvs.ping pad=0 seq=0 time=0.774 ms (0EB02!A3368!0!382A6)
+   kvs.ping pad=0 seq=1 time=0.686 ms (0EB02!A3368!0!382A6)
+   ...
+
+This tells you that the local "kvs" service is alive and the
+round-trip latency is a bit over half a millisecond. The route hops are:
+
+::
+
+   0EB02: UUID of the ping command
+   A3368: UUID of the API module
+   0:     rank of the local broker
+   382A6: UUID of the KVS module.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man1/flux-pmi.rst b/doc/man1/flux-pmi.rst
new file mode 100644
index 000000000000..481f643d10d0
--- /dev/null
+++ b/doc/man1/flux-pmi.rst
@@ -0,0 +1,141 @@
+===========
+flux-pmi(1)
+===========
+
+
+SYNOPSIS
+========
+
+| [**launcher**] **flux-pmi** [*-v*] [*--method=URI*] barrier [*--count=N*]
+| [**launcher**] **flux-pmi** [*-v*] [*--method=URI*] exchange [*--count=N*]
+| [**launcher**] **flux-pmi** [*-v*] [*--method=URI*] get [*--ranks=IDSET*]
+
+
+DESCRIPTION
+===========
+
+.. program:: flux pmi
+
+:program:`flux pmi` is a standalone Process Management Interface (PMI) client
+that embeds the same PMI client plugins as :man1:`flux-broker`.  It can be
+used to test the PMI service offered by :man1:`flux-shell` to parallel
+programs launched by Flux, or to probe the PMI services of an external
+launcher like Slurm or Hydra in isolation, without the complications of
+starting a Flux instance.
+
+:program:`flux pmi` tries a sequence of PMI plugins until one successfully
+initializes.  Alternatively, the specific plugin can be forced with the
+:option:`--method` option.  Initialization is followed by a subcommand-specific
+sequence of PMI operations that mimics a common pattern, and then PMI
+finalization.
+
+OPTIONS
+=======
+
+.. option:: -v, --verbose[=LEVEL]
+
+   Trace PMI operations.  This is equivalent to setting
+   :envvar:`FLUX_PMI_DEBUG` in the broker environment.
+
+.. option:: --method=URI
+
+   Specify the PMI method to use, where the scheme portion of the URI specifies
+   a plugin and the path portion specifies plugin-specific options.  The
+   builtin plugins are
+
+   simple
+     Use the simple PMI-1 wire protocol.
+
+   libpmi2[:PATH]
+     :func:`dlopen` ``libpmi2.so`` and use the PMI-2 API, optionally
+     at a specific *PATH*.
+
+   libpmi[:PATH]
+     :func:`dlopen` ``libpmi.so`` and use the PMI-1 API, optionally
+     at a specific *PATH*.
+
+   single
+     Become a singleton.
+
+.. option:: --libpmi-noflux
+
+   Fail if the libpmi or libpmi2 methods find the Flux ``libpmi.so``.
+
+.. option:: --libpmi2-cray
+
+   Force the libpmi2 Cray workarounds to be enabled for testing.  Normally
+   they are enabled only if a heuristic detects that Cray libpmi2 is in use.
+   The workarounds are
+
+   - Encode all KVS values with base64.
+   - Immediately fail an attempt to fetch any KVS with a ``flux.`` prefix.
+
+COMMANDS
+========
+
+barrier
+-------
+
+.. program:: flux pmi barrier
+
+:program:`flux pmi barrier` does the following:
+
+  #. Execute PMI barrier
+  #. Execute PMI barrier
+  #. Print elapsed time of (2)
+
+.. option:: --count=N
+
+  Execute N barrier (step 2) operations (default 1).
+
+.. option:: --abort=RANK
+
+  Instead of entering the barrier, arrange for RANK to call the PMI
+  abort function.
+
+exchange
+--------
+
+.. program:: flux pmi exchange
+
+:program:`flux pmi exchange` does the following:
+
+  #. Execute PMI barrier
+  #. Put rank specific key to PMI KVS
+  #. Execute PMI barrier
+  #. Get rank specific key from PMI KVS for all other ranks
+  #. Execute PMI barrier
+  #. Print elapsed time of (2-5)
+
+.. option:: --count=N
+
+  Execute N exchange (step 2-5) operations (default 1).
+
+get
+---
+
+.. program:: flux pmi get
+
+:program:`flux pmi get` fetches a pre-set key from the PMI KVS.
+
+.. option:: --ranks=RANKS
+
+   Print the value on specified *RANKS*, an RFC 22 idset or ``all`` (default 0).
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_13`
+
+
+SEE ALSO
+========
+
+:man7:`flux-broker-attributes`
diff --git a/doc/man1/flux-proxy.adoc b/doc/man1/flux-proxy.adoc
deleted file mode 100644
index 8ae1fb0ec420..000000000000
--- a/doc/man1/flux-proxy.adoc
+++ /dev/null
@@ -1,58 +0,0 @@
-// flux-help-command: proxy
-// flux-help-description: Create proxy environment for Flux instance
-FLUX-PROXY(1)
-=============
-:doctype: manpage
-
-
-NAME
-----
-flux-proxy - create proxy environment for Flux instance
-
-
-SYNOPSIS
---------
-*flux* *proxy* URI [command [args...]]
-
-
-DESCRIPTION
------------
-*flux proxy* connects to the Flux instance identified by _URI_,
-then spawns a shell with FLUX_URI pointing to a local:// socket
-managed by the proxy program.  As long as the shell is running,
-the proxy program routes messages between the instance and the
-local:// socket.  Once the shell terminates, the proxy program
-terminates and removes the socket.
-
-The purpose of *flux proxy* is to allow a connection to be reused,
-for example where connection establishment has high latency or
-requires authentication.
-
-EXAMPLES
---------
-
-Connect to a job running on the localhost which has a FLUX_URI
-of local:///tmp/flux-123456-abcdef/0/local and spawn an interactive
-shell:
-
-  $ flux proxy local:///tmp/flux-123456-abcdef/0/local
-
-Connect to the same job remotely on host foo.com:
-
-  $ flux proxy ssh://foo.com/tmp/flux-123456-abcdef/0/local
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
diff --git a/doc/man1/flux-proxy.rst b/doc/man1/flux-proxy.rst
new file mode 100644
index 000000000000..bcc19e2b62e4
--- /dev/null
+++ b/doc/man1/flux-proxy.rst
@@ -0,0 +1,119 @@
+.. flux-help-command: proxy
+.. flux-help-description: proxy connections to Flux jobs and instances
+.. flux-help-section: jobs
+
+=============
+flux-proxy(1)
+=============
+
+
+SYNOPSIS
+========
+
+**flux** **proxy** [*OPTIONS*] TARGET [command [args...]]
+
+DESCRIPTION
+===========
+
+.. program:: flux proxy
+
+:program:`flux proxy` connects to the Flux instance identified by *TARGET*,
+then spawns a shell with :envvar:`FLUX_URI` pointing to a local:// socket
+managed by the proxy program. As long as the shell is running,
+the proxy program routes messages between the instance and the
+local:// socket. Once the shell terminates, the proxy program
+terminates and removes the socket.
+
+The *TARGET* argument is a URI which can be resolved by ``flux uri``,
+including a Flux jobid, a fully-resolved native ``ssh`` or ``local``
+URI, or a resolvable URI with a scheme supported by a ``flux uri``
+plugin.  See :man1:`flux-uri` for details.
+
+If the connection to the Flux instance is lost, for example when the
+target instance terminates, :program:`flux proxy` will emit an error message,
+send ``SIGHUP`` and ``SIGCONT`` to the spawned shell or other process,
+and wait for it to terminate before exiting.  The delivery of signals
+can be disabled with the :option:`--nohup` option, but be aware that Flux
+commands running under a **flux proxy** which has lost its connection
+will likely result in errors.
+
+The purpose of :program:`flux proxy` is to allow a connection to be reused,
+for example where connection establishment has high latency or
+requires authentication.
+
+
+OPTIONS
+=======
+
+.. option:: -f, --force
+
+   Allow the proxy command to connect to a broker running a different
+   version of Flux with a warning message instead of a fatal error.
+
+.. option:: -n, --nohup
+
+   When an error occurs in the proxy connection, :program:`flux proxy` will
+   normally shut down the proxy and send ``SIGHUP`` and ``SIGCONT`` to
+   the spawned shell or command. If the :option:`--nohup` option is used,
+   the ``SIGHUP`` and ``SIGCONT`` signals will not be sent.
+   :program:`flux proxy` will still wait for the spawned shell or command to
+   exit before terminating to avoid having the child process reparented
+   and possibly lose its controlling tty.
+
+.. option:: --reconnect
+
+   If broker communication fails, drop the current connection and try to
+   reconnect every 2 seconds until the connection succeeds.  Any event
+   subscriptions and service registrations that were made on behalf of
+   clients are re-established, and in-flight RPCs receive an ECONNRESET
+   error responses.
+
+
+EXAMPLES
+========
+
+Connect to a job running on the localhost which has a :envvar:`FLUX_URI`
+of ``local:///tmp/flux-123456-abcdef/0/local`` and spawn an interactive
+shell:
+
+::
+
+   $ flux proxy local:///tmp/flux-123456-abcdef/0/local
+
+Connect to the same job remotely on host foo.com:
+
+::
+
+   $ flux proxy ssh://foo.com/tmp/flux-123456-abcdef/0/local
+
+Connect to a Flux instance running as job ƒQBfmbm in the current instance:
+
+::
+
+   $ flux proxy ƒQBfmbm
+
+or
+
+::
+
+   $ flux proxy jobid:ƒQBfmbm
+
+
+Connect to a Flux instance running as job ƒQ8ho35 in ƒQBfmbm
+
+::
+
+  $ flux proxy jobid:ƒQBfmbm/ƒQ8ho35
+
+
+Connect to a Flux instance started in Slurm job 1234
+
+::
+
+  $ flux proxy slurm:1234
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man1/flux-pstree.rst b/doc/man1/flux-pstree.rst
new file mode 100644
index 000000000000..ed1db274933d
--- /dev/null
+++ b/doc/man1/flux-pstree.rst
@@ -0,0 +1,252 @@
+.. flux-help-include: true
+.. flux-help-section: jobs
+
+==============
+flux-pstree(1)
+==============
+
+
+SYNOPSIS
+========
+
+**flux** **pstree** [*OPTIONS*] [JOBID ...]
+
+DESCRIPTION
+===========
+
+.. program:: flux pstree
+
+:program:`flux pstree` displays a tree of running jobs by job name, similar to
+what the :linux:man1:`pstree` command does for system processes.
+
+Like :command:`pstree`, identical leaves of the job tree are combined, which
+results in a more compact output when many jobs within a Flux instance
+share the same job name.
+
+The :program:`flux pstree` command supports custom labels for jobs, including
+separately labeling parent jobs, using the same format string syntax
+supported by :man1:`flux-jobs`.
+
+The command lists actively running jobs by default, but a :option:`--all`
+option lists all jobs in all states for the current user. In the case
+that :option:`-all` is used, the job labels will automatically be amended to
+include the job status (i.e. ``{name}:{status_abbrev}``), though this
+can be overridden on the command line.
+
+The :program:`flux pstree` command additionally supports listing extended
+job information before the tree display with the :option:`--extended`,
+:option:`--details=NAME`, or :option:`--detail-format=FORMAT`  options, e.g.
+
+::
+
+  $ flux pstree -x
+       JOBID USER     ST NTASKS NNODES  RUNTIME
+  ƒJqUHUCzX9 user1     R      2      2   10.68s flux
+     ƒe1j54L user1     R      1      1   8.539s ├── flux
+     ƒuusNLo user1     R      1      1   5.729s │   ├── sleep
+     ƒutPP4T user1     R      1      1   5.731s │   └── sleep
+     ƒe1j54K user1     R      1      1   8.539s └── flux
+    ƒ2MYrwzf user1     R      1      1   4.736s     └── sleep
+
+Several detail formats are available via the :option:`-d, --details=NAME`
+option, including progress, resources, and stats. For example, the
+``progress`` display attempts to show the overall progress and
+utilization of all Flux instances in a hierarchy by displaying the
+total number of jobs in that instance (``NJOBS``), the "progress"
+(``PROG`` - inactive/finished jobs divided by total jobs), and
+core and GPU utilization (``CORE%`` and ``GPU%`` - number of used
+resources divided by total available resources):
+
+::
+
+  $ flux pstree --details=progress
+
+       JOBID NTASKS  NJOBS  PROG CORE%  GPU%    RUNTIME
+  ƒJqUHUCzX9      2      3    0% 37.5%          0:02:15 flux
+     ƒe1j54L      1   1000   23%  100%          0:02:13 ├── flux
+    ƒ2HmSnHd      1                             0:00:01 │   ├── sleep
+    ƒ2Hjxo1K      1                             0:00:01 │   └── sleep
+     ƒe1j54K      1   1000 11.5%  100%          0:02:13 └── flux
+    ƒ2b6cPMS      1                             0:00:01     └── sleep
+
+
+By default, :program:`flux pstree` truncates lines that exceed the current
+value of the ``COLUMNS`` environment variable or the terminal width
+if ``COLUMNS`` is not set. To disable truncation, use the :option:`--long`
+option.
+
+
+By default, the enclosing Flux instance, or root of the tree, is included
+in output, unless extended details are displayed as when any of the
+:option:`--extended`, :option:`--details=NAME` or
+:option:`--detail-format=FORMAT` options are used, or if one or more jobids
+are directly targeted with a ``JOBID`` argument. This behavior can be changed
+via the :option:`--skip-root=[yes|no]` option.
+
+
+OPTIONS
+=======
+
+.. option:: -a, --all
+
+   Include jobs in all states, including inactive jobs.
+   This is shorthand for :option:`--filter=pending,running,inactive`.
+
+.. option:: -c, --count=N
+
+   Limit output to N jobs at every level (default 1000).
+
+.. option:: -f, --filter=STATE|RESULT
+
+   Include jobs with specific job state or result. Multiple states or
+   results can be listed separated by comma. See the :ref:`flux_jobs_job_status`
+   section of the :man1:`flux-jobs` manual for more detail.
+
+.. option:: -l, --long
+
+   Do not truncate long lines at ``COLUMNS`` characters.
+
+.. option:: -p, --parent-ids
+
+   Prepend jobid to parent labels.
+
+.. option:: -L, --level=N
+
+   Only descend *N* levels of the job hierarchy.
+
+.. option:: -x, --extended
+
+   Print extended details before tree output. This is the same as
+   :option:`--details=default`.
+
+.. option:: -d, --detail=NAME
+
+   Select a named extended details format. The list of supported names
+   can be seen in :option:`flux pstree --help` output.
+
+.. option:: -n, --no-header
+
+   For output with extended details, do not print header row.
+
+.. option:: -X, --no-combine
+
+   Typically, identical child jobs that are leaves in the tree display
+   are combined as ``n*[label]``. With this option, the combination of
+   like jobs is disabled.
+
+.. option:: -o, --label=FORMAT
+
+   Specify output format for node labels using Python format strings.
+   Supports all format fields supported by :man1:`flux-jobs`.
+
+.. option:: --parent-label=FORMAT
+
+   Label tree parents with a different format than child jobs.
+
+.. option:: --detail-format=FORMAT
+
+   Specify an explicit details format to display before the tree part.
+   Care should be taken that each line of the format is the same width
+   to ensure that the tree display is rendered correctly (i.e. by judicious
+   use of format field widths, e.g. ``{id.f58:>12}`` instead of just
+   ``{id.f58}``.
+
+.. option:: --skip-root=yes|no
+
+   Explicitly skip (yes)  or force (no) display of the enclosing instance,
+   or root of the tree, in output.
+
+.. option:: -C, --compact
+
+   Use compact tree connectors. Usefully for deep hierarchies.
+
+.. option:: --ascii
+
+   Use ascii tree connectors.
+
+
+EXAMPLES
+========
+
+The default output of :program:`flux pstree` shows all running jobs for the
+current user by name, including any running sub-jobs. If there are
+currently no running jobs for the current user, only the enclosing
+instance is displayed as a ``.``, to indicate the root of the tree:
+
+::
+
+  $ flux pstree
+  .
+
+
+If there is a running job, it is displayed under the root instance,
+and includes all child jobs. Identical children are combined:
+
+::
+
+  $ flux pstree
+  .
+  └── flux
+      ├── flux
+      │   └── 2*[sleep]
+      └── flux
+          └── sleep
+  
+
+Extra information can be added to parents, which are instances of
+flux. For example, summary job stats can be easily added:
+
+::
+
+  $ flux pstree --skip-root=yes --parent-label='{name} {instance.stats}'
+  flux PD:1 R:2 CD:0 F:0
+  ├── flux PD:592 R:2 CD:406 F:0
+  │   └── 2*[sleep]
+  └── flux PD:794 R:1 CD:205 F:0
+      └── sleep
+  
+Or utilization:
+
+::
+
+  $ flux pstree --skip-root=yes \
+    --parent-label='cores={instance.resources.all.ncores} {instance.utilization!P}' \
+  cores=8 37.5%
+  ├── cores=2 100%
+  │   └── 2*[sleep]
+  └── cores=1 100%
+
+Displaying jobs in all states automatically adds the job *status* to the
+display, which offers a compact representation of the state of jobs
+throughout a hierarchy:
+
+::
+
+  $ flux pstree -a
+  .
+  ├── flux
+  │   ├── flux:PD
+  │   ├── flux
+  │   │   ├── 824*[sleep:PD]
+  │   │   ├── 2*[sleep:R]
+  │   │   └── 174*[sleep:CD]
+  │   └── flux
+  │       ├── 914*[sleep:PD]
+  │       ├── sleep:R
+  │       └── 85*[sleep:CD]
+  ├── flux:CA
+  ├── 36*[flux:CD]
+  ├── hostname:CA
+  └── hostname:CD
+  
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+SEE ALSO
+========
+
+:man1:`flux-jobs`
diff --git a/doc/man1/flux-queue.rst b/doc/man1/flux-queue.rst
new file mode 100644
index 000000000000..8584eac9a6b1
--- /dev/null
+++ b/doc/man1/flux-queue.rst
@@ -0,0 +1,294 @@
+.. flux-help-description: list and manipulate flux queues
+.. flux-help-section: instance
+
+=============
+flux-queue(1)
+=============
+
+
+SYNOPSIS
+========
+
+| **flux** **queue** **list** [-n] [-o FORMAT]
+| **flux** **queue** **status** [*-q* *NAME* | *-a*] [*-v*]
+
+| **flux** **queue** **disable** [*-q* *NAME* | *-a*] [*-v*] [*--quiet*] [*--nocheckpoint*] *reason*
+| **flux** **queue** **enable** [*-q* *NAME* | *-a*] [*-v*] [*--quiet*]
+
+| **flux** **queue** **stop** [*-q* *NAME* | *-a*] [*-v*] [*--quiet*] [*--nocheckpoint*] *reason*
+| **flux** **queue** **start** [*-q* *NAME* | *-a*] [*-v*] [*--quiet*]
+
+| **flux** **queue** **drain** [*--timeout=DURATION*]
+| **flux** **queue** **idle** [*--timeout=DURATION*]
+
+
+DESCRIPTION
+===========
+
+.. program:: flux queue
+
+The :program:`flux queue` command operates on Flux job queue(s).
+
+By default, Flux has one anonymous queue.  Multiple named queues may be
+configured - see :man5:`flux-config-queues`.
+
+COMMANDS
+========
+
+:program:`flux queue` has the following subcommands:
+
+list
+----
+
+.. program:: flux queue list
+
+List queue status, defaults, and limits.
+
+.. option:: -q, --queue=QUEUE,...
+
+   Limit output to specified queues
+
+.. option:: -n, --no-header
+
+   Do not output column headers in ``list`` output.
+
+.. option:: -o, --format=FORMAT
+
+   Specify output format in ``list`` using Python's string format syntax.
+   See `OUTPUT FORMAT`_ below for field names.
+
+status
+------
+
+.. program:: flux queue status
+
+Report the current queue status.
+
+.. option:: -q, --queue=NAME
+
+   Select a queue by name.
+
+.. option:: -v, --verbose
+
+   Display more detail about internal job manager state.
+
+disable
+-------
+
+.. program:: flux queue disable
+
+Prevent jobs from being submitted to the queue, with a reason that is
+shown to submitting users.
+
+.. option:: -q, --queue=NAME
+
+   Select a queue by name.
+
+.. option:: -a, --all
+
+   Select all queues.
+
+.. option:: -v, --verbose
+
+   Display more detail about internal job manager state.
+
+.. option:: --quiet
+
+   Display only errors.
+
+.. option:: --nocheckpoint
+
+   Do not preserve the new queue stop state across a Flux instance restart.
+
+enable
+------
+
+.. program:: flux queue enable
+
+Allow jobs to be submitted to the queue.
+
+.. option:: -q, --queue=NAME
+
+   Select a queue by name.
+
+.. option:: -a, --all
+
+   Select all queues.
+
+.. option:: -v, --verbose
+
+   Display more detail about internal job manager state.
+
+.. option:: --quiet
+
+   Display only errors.
+
+stop
+----
+
+.. program:: flux queue stop
+
+Stop allocating resources to jobs.  Pending jobs remain enqueued, and
+running jobs continue to run, but no new jobs are allocated
+resources.
+
+.. option:: -q, --queue=NAME
+
+   Select a queue by name.
+
+.. option:: -a, --all
+
+   Select all queues.
+
+.. option:: -v, --verbose
+
+   Display more detail about internal job manager state.
+
+.. option:: --quiet
+
+   Display only errors.
+
+.. option:: --nocheckpoint
+
+   Do not preserve the new queue stop state across a Flux instance restart.
+
+start
+-----
+
+.. program:: flux queue start
+
+Start allocating resources to jobs.
+
+.. option:: -q, --queue=NAME
+
+   Select a queue by name.
+
+.. option:: -a, --all
+
+   Select all queues.
+
+.. option:: -v, --verbose
+
+   Display more detail about internal job manager state.
+
+.. option:: --quiet
+
+   Display only errors.
+
+drain
+-----
+
+.. program:: flux queue drain
+
+Block until all queues become empty.  It is sometimes useful to run after
+:program:`flux queue disable`, to wait until the system is quiescent and can
+be taken down for maintenance.
+
+.. option:: --timeout=FSD
+
+   Limit the time that the command` will block.
+
+idle
+----
+
+.. program:: flux queue idle
+
+Block until all queues become `idle` (no jobs in RUN or CLEANUP state,
+and no outstanding alloc requests to the scheduler).  It may be useful to run
+after :program:`flux queue stop` to wait until the scheduler and execution
+system are quiescent before maintenance involving them.
+
+.. option:: --timeout=FSD
+
+   Limit the time that the command` will block.
+
+
+OUTPUT FORMAT
+=============
+
+The :option:`flux queue list --format` option can be used to specify an
+output format using Python's string format syntax or a defined format by
+name. For a list of built-in and configured formats use :option:`-o help`.
+An alternate default format can be set via the
+:envvar:`FLUX_QUEUE_LIST_FORMAT_DEFAULT` environment variable.
+A configuration snippet for an existing named format may be
+generated with :option:`--format=get-config=NAME`.  See :man1:`flux-jobs`
+*OUTPUT FORMAT* section for a detailed description of this syntax.
+
+The following field names can be specified:
+
+**queue**
+   queue name
+
+**queuem**
+   queue name, but default queue is marked up with an asterisk
+
+**submission**
+   Description of queue submission status: ``enabled`` or ``disabled``
+
+**scheduling**
+   Description of queue scheduling status: ``started`` or ``stopped``
+
+**enabled**
+   Single character submission status: ``✔`` if enabled, ``✗`` if disabled.
+
+**started**
+   Single character scheduling status: ``✔`` if started, ``✗`` if stopped.
+
+**enabled.ascii**
+   Single character submission status: ``y`` if enabled, ``n`` if disabled.
+
+**started.ascii**
+   Single character scheduling status: ``y`` if started, ``n`` if stopped.
+
+**defaults.timelimit**
+   default timelimit for jobs submitted to the queue
+
+**limits.timelimit**
+   max timelimit for jobs submitted to the queue
+
+**limits.range.nnodes**
+   range of nodes that can be requested for this queue
+
+**limits.range.ncores**
+   range of cores that can be requested for this queue
+
+**limits.range.ngpus**
+   range of gpus that can be requested for this queue
+
+**limits.min.nnodes**
+   minimum number of nodes that must be requested for this queue
+
+**limits.max.nnodes**
+   maximum number of nodes that can be requested for this queue
+
+**limits.min.ncores**
+   minimum number of cores that must be requested for this queue
+
+**limits.max.ncores**
+   maximum number of cores that can be requested for this queue
+
+**limits.min.ngpus**
+   minimum number of gpus that must be requested for this queue
+
+**limits.max.ngpus**
+   maximum number of gpus that can be requested for this queue
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+| :doc:`rfc:spec_23`
+| :doc:`rfc:spec_33`
+
+
+SEE ALSO
+========
+
+:man1:`flux-jobs`, :man1:`flux-submit`
diff --git a/doc/man1/flux-resource.rst b/doc/man1/flux-resource.rst
new file mode 100644
index 000000000000..fab5c39a6b94
--- /dev/null
+++ b/doc/man1/flux-resource.rst
@@ -0,0 +1,522 @@
+.. flux-help-include: true
+.. flux-help-section: instance
+
+================
+flux-resource(1)
+================
+
+
+SYNOPSIS
+========
+
+| **flux** **resource** **list** [*-n*] [*-o* *FORMAT*] [*-s* *STATES*] [*-q* *QUEUE*] [*-i* *TARGETS*]
+| **flux** **resource** **info** [*-s* *STATES*] [*-q* *QUEUE*] [*-i* *TARGETS*]
+| **flux** **resource** **R** [*-s* *STATES*] [*-q* *QUEUE*] [*-i* *TARGETS*]
+
+| **flux** **resource** **status** [*-n*] [*-o* *FORMAT*] [*-s* *STATES*] [*-q* *QUEUE*] [*-i* *TARGETS*]
+
+| **flux** **resource** **drain** [*-n*] [*-o* *FORMAT*] [*-i* *TARGETS*]
+| **flux** **resource** **drain** [*-f*] [*-u*] [*targets*] [*reason*]
+| **flux** **resource** **undrain** [*-f*] *targets*
+
+| **flux** **resource** **reload** [-f] [--xml] *path*
+
+| **flux** **resource** **acquire-mute**
+
+DESCRIPTION
+===========
+
+.. program:: flux resource
+
+:program:`flux resource` lists and manipulates Flux resources.  The resource
+inventory is maintained and monitored by the resource service.  The scheduler
+acquires a subset of resources from the resource service to allocate to jobs,
+and relies on the resource service to inform it of status changes that affect
+the usability of resources by jobs as described in RFC 27.
+
+The :program:`flux resource list` subcommand queries the resource module
+for the scheduler view of resources, including allocated/free status.
+
+The other :program:`flux resource` subcommands operate on the resource service
+and are primarily of interest to system administrators of a Flux system
+instance.  For example, they can show whether or not a node is booted, and may
+be used to administratively drain and undrain nodes.
+
+A few notes on drained nodes:
+
+- While a node is drained, the scheduler will not allocate it to jobs.
+- The act of draining a node does not affect running jobs.
+- When an instance is restarted, drained nodes remain drained.
+- The scheduler may determine that a job request is *feasible* if the total
+  resource set, including drained nodes, would allow it to run.
+- In :program:`flux resource status` and :program:`flux resource drain`, the
+  drain state of a node will be presented as "drained" if the node has no job
+  allocations, and "draining" if there are still jobs running on the node.
+- If a node is drained and offline, then "drained*" will be displayed.
+
+Some further background on resource service operation may be found in the
+`RESOURCE INVENTORY`_ section below.
+
+
+COMMANDS
+========
+
+list
+----
+
+.. program:: flux resource list
+
+Show the scheduler view of resources.
+
+In the scheduler view, excluded resources are always omitted, and
+unavailable resources are shown in the "down" state regardless of the
+reason (drained, offline, torpid, etc).  Valid states in the scheduler view are:
+
+up
+  Available for use.
+
+down
+  Unavailable for use.
+
+allocated
+  Allocated to jobs.
+
+free
+  Not allocated to jobs.
+
+all
+  Wildcard matching all resources.
+
+.. option:: -s, --states=STATE,...
+
+  Restrict displayed resource states to a comma separated list of
+  the states listed above.
+
+  If unspecified, :option:`free,allocated,down` is used.
+
+.. option:: -q, --queue=QUEUE,...
+
+  Filter results to only include resources in the specified *QUEUE*. Multiple
+  queues may be separated by a comma.
+
+.. option:: -i, --include=TARGETS
+
+  Filter results to only include resources matching *TARGETS*, which may be
+  specified either as an idset of broker ranks or list of hosts or nodes in
+  hostlist form. It is not an error to specify ranks, nodes, or hosts which
+  do not exist.
+
+.. option:: --skip-empty
+
+  Skip lines with empty resource sets in output. This is the default when
+  `-i, --include` is used.
+
+.. options:: --no-skip-empty
+
+  Do not skip empty resource sets in output, even with `-i, --include`.
+
+.. option:: -o, --format=FORMAT
+
+  Customize the output format (See the `OUTPUT FORMAT`_ section below).
+
+.. option:: -n, --no-header
+
+  Suppress header from output,
+
+info
+----
+
+.. program:: flux resource info
+
+Show a brief, single line summary of scheduler view of resources, for
+example::
+
+  8 Nodes, 32 Cores, 0 GPUs
+
+.. option:: -s, --states=STATE,...
+
+  Limit the output to specified resource states as described above for
+  the `list`_ command.
+
+  If unspecified, :option:`all` is used.
+
+.. option:: -q, --queue=QUEUE,...
+
+  Filter results to only include resources in the specified *QUEUE*. Multiple
+  queues may be separated by a comma.
+
+.. option:: -i, --include=TARGETS
+
+  Filter results to only include resources matching *TARGETS*, which may be
+  specified either as an idset of broker ranks or list of hosts or nodes in
+  hostlist form. It is not an error to specify ranks, nodes, or hosts which
+  do not exist.
+
+R
+-
+
+.. program:: flux resource R
+
+Emit an RFC 20 Resource Set based on the scheduler view of resources.
+
+.. option:: -s, --states=STATE,...
+
+  Limit the output to specified resource states as described above for
+  the `list`_ command.
+
+  If unspecified, :option:`all` is used.
+
+.. option:: -q, --queue=QUEUE,...
+
+  Filter results to only include resources in the specified *QUEUE*. Multiple
+  queues may be separated by a comma.
+
+.. option:: -i, --include=TARGETS
+
+  Filter results to only include resources matching *TARGETS*, which may be
+  specified either as an idset of broker ranks or list of hosts or nodes in
+  hostlist form. It is not an error to specify ranks, nodes, or hosts which
+  do not exist.
+
+status
+------
+
+.. program:: flux resource status
+
+Show system view of resources.  Valid states in the system view are:
+
+avail
+  available for scheduling when up. This includes all nodes that are
+  not excluded, drained, or torpid.
+
+exclude
+  excluded by configuration
+
+draining
+  drained but still allocated
+
+drained
+  drained and unallocated
+
+drain
+  shorthand for :option:`drained,draining`
+
+allocated
+  node is currently allocated to a job or housekeeping
+
+torpid
+  node has been unresponsive for a period of time and is temporarily
+  unavailable for scheduling
+
+housekeeping
+  node is currently running housekeeping
+
+offline
+  node has not joined the Flux instance (e.g. turned off or has not
+  started the flux broker).
+
+online
+  node has joined the Flux instance
+
+:program:`flux resource status` displays a line of output for each set of
+resources that share a state and online/offline state.
+
+.. note::
+  :program:`flux resource status` queries both the administrative and
+  scheduler view of resources to identify resources that are available,
+  excluded by configuration, torpid, administratively drained or draining,
+  or currently executing housekeeping.
+
+.. option:: -s, --states=STATE,...
+
+  Restrict the set of resource states a comma-separated list.
+
+  If unspecified, :option:`avail,exclude,draining,drained,torpid,housekeeping`
+  is used.
+
+
+.. option:: -q, --queue=QUEUE,...
+
+  Filter results to only include resources in the specified *QUEUE*. Multiple
+  queues may be separated by a comma.
+
+.. option:: -i, --include=TARGETS
+
+  Filter results to only include resources matching *TARGETS*, which may be
+  specified either as an idset of broker ranks or list of hosts or nodes in
+  hostlist form. It is not an error to specify ranks, nodes, or hosts which
+  do not exist.
+
+.. option:: -o, --format=FORMAT
+
+  Customize output formatting.  See the `OUTPUT FORMAT`_ section below for
+  details.
+
+.. option:: -n,--no-header
+
+  Suppress header from output,
+
+.. option:: --skip-empty
+
+  Force suppression of empty lines.
+
+  Normally, :program:`flux resource status` skips lines with no resources,
+  unless the :option:`-s, --states` option is used.
+
+drain
+-----
+
+.. program:: flux resource drain
+
+If specified without *targets*, list the drained nodes. In this mode, the
+following options are available:
+
+.. option:: -o, --format=FORMAT
+
+  Customize output formatting.  See the `OUTPUT FORMAT`_ section below for
+  details.
+
+.. option:: -n,--no-header
+
+  Suppress header from output,
+
+.. option:: -q, --queue=QUEUE,...
+
+  Filter results to only include resources in the specified *QUEUE*. Multiple
+  queues may be separated by a comma.
+
+.. option:: -i, --include=TARGETS
+
+  Filter results to only include resources matching *TARGETS*, which may be
+  specified either as an idset of broker ranks or list of hosts or nodes in
+  hostlist form. It is not an error to specify ranks, nodes, or hosts which
+  do not exist.
+
+If specified with *targets* (IDSET or HOSTLIST), drain the specified nodes.
+Any remaining free arguments are recorded as a reason for the drain event.
+By default, :program:`flux resource drain` fails if any of the *targets*
+are already drained.
+
+Resources cannot be both excluded and drained, so
+:program:`flux resource drain` will also fail if any *targets* are
+currently excluded by configuration.  There is no option to force an
+excluded node into the drain state.
+
+This command, when run with arguments, is restricted to the Flux instance
+owner.
+
+.. option:: -f, --force
+
+  If any of *targets* are already drained, do not fail.  Overwrite the
+  original drain reason.  When :option:`--force` is specified twice,
+  the original drain timestamp is also overwritten.
+
+.. option:: -u, --update
+
+  If any of *targets* are already drained, do not fail and do not overwrite
+  the existing drain reason or timestamp.
+
+undrain
+-------
+
+.. program:: flux resource undrain
+
+Undrain the nodes specified by the *targets* argument (IDSET or HOSTLIST).
+
+This command is restricted to the Flux instance owner.
+
+.. option:: -f, --force
+
+  Do not fail if any of the *targets* are not drained.
+
+reload
+------
+
+.. program:: flux resource reload
+
+Reload the resource inventory from *path*.  By default, *path* refers to a
+file in RFC 20 format.
+
+This command is primarily used in test.
+
+.. option:: -x, --xml
+
+  Interpret *path* as a directory of hwloc ``<rank>.xml`` files.
+
+.. option:: -f, --force
+
+  Do not fail if resource contain invalid ranks.
+
+acquire-mute
+------------
+
+.. program:: flux resource acquire-mute
+
+Tell the resource module to stop sending RFC 28 ``resource.acquire`` responses
+to the scheduler.  This is used during Flux instance shutdown to avoid asking
+the scheduler to needlessly process OFFLINE updates.
+
+OUTPUT FORMAT
+=============
+
+The :option:`--format` option can be used to specify an output format using
+Python's string format syntax or a defined format by name. For a list of
+built-in and configured formats use :option:`-o help`.  An alternate default
+format can be set via the :envvar:`FLUX_RESOURCE_STATUS_FORMAT_DEFAULT`,
+:envvar:`FLUX_RESOURCE_DRAIN_FORMAT_DEFAULT`, and
+:envvar:`FLUX_RESOURCE_LIST_FORMAT_DEFAULT` environment variables (for
+:program:`flux resource status`, :program:`flux resource drain`, and
+:program:`flux resource list` respectively).  A configuration snippet for an
+existing named format may be generated with :option:`--format=get-config=NAME`.
+See :man1:`flux-jobs` :ref:`flux_jobs_output_format` section for a detailed
+description of this syntax.
+
+Resources are combined into a single line of output when possible depending on
+the supplied output format.  Resource counts are not included in the
+determination of uniqueness.  Therefore, certain output formats will alter the
+number of lines of output.  For example:
+
+::
+
+   $ flux resource list -no {nnodes}
+
+Would simply output a single of output containing the total number of nodes.
+The actual state of the nodes would not matter in the output.
+
+The following field names can be specified for the **status** and **drain**
+subcommands:
+
+**state**
+   State of node(s): "avail", "exclude", "drain", "draining", "drained",
+   "torpid", "allocated". If the set of resources is offline, an asterisk
+   suffix is appended to the state, e.g. "avail*".
+
+**statex**
+   Like **state**, but exclude the asterisk for offline resources.
+
+**status**
+   Current online/offline status of nodes(s): "online", "offline"
+
+**up**
+   Displays a *✔* if the node is online, or *✗* if offline. An ascii *y*
+   or *n* may be used instead with **up.ascii**.
+
+**nnodes**
+   number of nodes
+
+**ranks**
+   ranks of nodes
+
+**nodelist**
+   node names
+
+**timestamp**
+   If node(s) in drain/draining/drained state, timestamp of node(s)
+   set to drain.
+
+**reason**
+   If node(s) in drain/draining/drained state, reason node(s) set to
+   drain.
+
+The following field names can be specified for the **list** subcommand:
+
+**state**
+   State of node(s): "up", "down", "allocated", "free", "all"
+
+**queue**
+   queue(s) associated with resources.
+
+**properties**
+   Properties associated with resources.
+
+**propertiesx**
+   Properties associated with resources, but with queue names removed.
+
+**nnodes**
+   number of nodes
+
+**ncores**
+   number of cores
+
+**ngpus**
+   number of gpus
+
+**ranks**
+   ranks of nodes
+
+**nodelist**
+   node names
+
+**rlist**
+   Short form string of all resources.
+
+
+CONFIGURATION
+=============
+
+Similar to :man1:`flux-jobs`, the :program:`flux resource` command supports
+loading a set of config files for customizing utility output formats. Currently
+this can be used to register named format strings for the ``status``,
+``list``, and ``drain`` subcommands.
+
+Configuration for each :program:`flux resource` subcommand is defined in a
+separate table, so to add a new format ``myformat`` for ``flux resource list``,
+the following config file could be used::
+
+  # $HOME/.config/flux/flux-resource.toml
+  [list.formats.myformat]
+  description = "My flux resource list format"
+  format = "{state} {nodelist}"
+
+See :man1:`flux-jobs` :ref:`flux_jobs_configuration` section for more
+information about the order of precedence for loading these config files.
+
+RESOURCE INVENTORY
+==================
+
+The Flux instance's inventory of resources is managed by the resource service,
+which determines the set of available resources through one of three
+mechanisms:
+
+configuration
+   Resources are read from a config file in RFC 20 (R version 1) format.
+   This mechanism is typically used in a system instance of Flux.
+
+enclosing instance
+   Resources are assigned by the enclosing Flux instance.  The assigned
+   resources are read from the job's ``R`` key in the enclosing instance KVS.
+
+dynamic discovery
+   Resources are aggregated from the set of resources reported by hwloc
+   on each broker.
+
+Once the inventory has been determined, it is stored the KVS ``resource.R``
+key, in RFC 20 (R version 1) format.
+
+Events that affect the availability of resources and should persist across
+a Flux restart are posted to the KVS *resource.eventlog*.  Such events include:
+
+resource-define
+   The instance resource set is known (posted each time the resource module
+   is loaded).
+
+drain
+   One or more nodes are administratively removed from scheduling.
+
+undrain
+   One or more nodes are no longer drained.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+| :doc:`rfc:spec_20`
+| :doc:`rfc:spec_22`
+| :doc:`rfc:spec_27`
+| :doc:`rfc:spec_29`
diff --git a/doc/man1/flux-restore.rst b/doc/man1/flux-restore.rst
new file mode 100644
index 000000000000..11c24439dd2c
--- /dev/null
+++ b/doc/man1/flux-restore.rst
@@ -0,0 +1,88 @@
+===============
+flux-restore(1)
+===============
+
+
+SYNOPSIS
+========
+
+**flux** **restore** [*OPTIONS*] *INFILE*
+
+
+DESCRIPTION
+===========
+
+.. program:: flux restore
+
+The :program:`flux restore` command reads a KVS snapshot from a portable
+archive format, usually written by :man1:`flux-dump`.
+
+The archive source may be specified as a file path or *-* for standard input.
+The format of the archive may be any of the formats supported by
+:linux:man3:`libarchive` and is determined on the fly based on the archive
+content.
+
+The snapshot may be restored to a KVS key if :option:`--key=NAME` is used and
+the KVS service is running, or as a checkpoint in the content backing store
+if :option:`--checkpoint` is used, without the KVS running.  One of those two
+options is required.
+
+
+OPTIONS
+=======
+
+.. option:: -h, --help
+
+   Summarize available options.
+
+.. option:: -v, --verbose
+
+   List keys on stderr as they are restored instead of a periodic count of
+   restored keys.
+
+.. option:: -q, --quiet
+
+   Don't show a periodic count of restored keys on stderr.
+
+.. option:: --checkpoint
+
+   After restoring the archived content, write the final root blobref
+   to the KVS checkpoint area in the content backing store.  The checkpoint
+   is used as the initial KVS root when the KVS module is loaded.  Unload
+   the KVS module before restoring with this option.
+
+.. option:: --key=NAME
+
+   After restoring the archived content, write the final root blobref
+   to a KVS key, so the key becomes the restored root directory.
+
+.. option:: --no-cache
+
+   Bypass the broker content cache and interact directly with the backing
+   store.  Performance will vary depending on the content of the archive.
+
+.. option:: --size-limit=SIZE
+
+   Skip restoring keys that exceed SIZE bytes (default: no limit). SIZE may
+   be specified as a floating point number with an optional multiplicative
+   suffix k or K=1024, M=1024\*1024, or G=1024\*1024\*1024 (up to
+   ``INT_MAX``).
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_10`
+
+:doc:`rfc:spec_11`
+
+
+SEE ALSO
+========
+
+:man1:`flux-dump`, :man1:`flux-kvs`
diff --git a/doc/man1/flux-run.rst b/doc/man1/flux-run.rst
new file mode 100644
index 000000000000..f440538fc714
--- /dev/null
+++ b/doc/man1/flux-run.rst
@@ -0,0 +1,167 @@
+.. flux-help-include: true
+.. flux-help-section: submission
+
+===========
+flux-run(1)
+===========
+
+
+SYNOPSIS
+========
+
+**flux** **run** [OPTIONS] [*--ntasks=N*] COMMAND...
+
+
+DESCRIPTION
+===========
+
+.. program:: flux run
+
+:program:`flux run` submits a job to run interactively under Flux, blocking
+until the job has completed.  The job consists of *N* copies of COMMAND
+launched together as a parallel job.
+
+If :option:`--ntasks` is unspecified, a value of *N=1* is assumed.
+
+The available OPTIONS are detailed below.
+
+JOB PARAMETERS
+==============
+
+These commands accept only the simplest parameters for expressing
+the size of the parallel program and the geometry of its task slots:
+
+Common resource options
+-----------------------
+
+These commands take the following common resource allocation options:
+
+.. include:: common/job-param-common.rst
+
+Per-task options
+----------------
+
+:man1:`flux-run`, :man1:`flux-submit` and :man1:`flux-bulksubmit` take two
+sets of mutually exclusive options to specify the size of the job request.
+The most common form uses the total number of tasks to run along with
+the amount of resources required per task to specify the resources for
+the entire job:
+
+.. include:: common/job-param-pertask.rst
+
+Per-resource options
+--------------------
+
+The second set of options allows an amount of resources to be specified
+with the number of tasks per core or node set on the command line. It is
+an error to specify any of these options when using any per-task option
+listed above:
+
+.. include:: common/job-param-perres.rst
+
+Additional job options
+----------------------
+
+These commands also take following job parameters:
+
+.. include:: common/job-param-additional.rst
+
+STANDARD I/O
+============
+
+By default, task stdout and stderr streams are redirected to the
+KVS, where they may be accessed with the ``flux job attach`` command.
+
+In addition, :man1:`flux-run` processes standard I/O in real time,
+emitting the job's I/O to its stdout and stderr.
+
+.. include:: common/job-standard-io.rst
+
+CONSTRAINTS
+===========
+
+.. include:: common/job-constraints.rst
+
+DEPENDENCIES
+============
+
+.. include:: common/job-dependencies.rst
+
+ENVIRONMENT
+===========
+
+By default, these commands duplicate the current environment when submitting
+jobs. However, a set of environment manipulation options are provided to
+give fine control over the requested environment submitted with the job.
+
+.. include:: common/job-environment.rst
+
+ENV RULES
+---------
+
+.. include:: common/job-env-rules.rst
+
+PROCESS RESOURCE LIMITS
+=======================
+
+By default these commands propagate some common resource limits (as described
+in :linux:man2:`getrlimit`) to the job by setting the ``rlimit`` job shell
+option in jobspec.  The set of resource limits propagated can be controlled
+via the :option:`--rlimit=RULE` option:
+
+.. include:: common/job-process-resource-limits.rst
+
+JOB ENVIRONMENT VARIABLES
+=========================
+
+The job environment is described in more detail in the :man7:`flux-environment`
+:ref:`job_environment` section.  A summary of the most commonly used variables
+is provided below:
+
+.. include:: common/job-environment-variables.rst
+
+EXIT STATUS
+===========
+
+The job exit status, normally the largest task exit status, is stored
+in the KVS. If one or more tasks are terminated with a signal,
+the job exit status is 128+signo.
+
+The ``flux-job attach`` command exits with the job exit status.
+
+In addition, :man1:`flux-run` runs until the job completes and exits
+with the job exit status.
+
+OTHER OPTIONS
+=============
+
+.. include:: common/job-other-options.rst
+
+.. include:: common/job-other-run.rst
+
+SHELL OPTIONS
+=============
+
+Some options that affect the parallel runtime environment are provided by the
+Flux shell.  These options are described in detail in the
+:ref:`SHELL OPTIONS <flux_shell_options>` section of :man1:`flux-shell`.
+A list of the most commonly needed options follows.
+
+Usage: :option:`flux run -o NAME[=ARG]`.
+
+.. make :option: references in the included table x-ref to flux-shell(1)
+.. program:: flux shell
+.. include:: common/job-shell-options.rst
+.. program:: flux run
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-submit`, :man1:`flux-alloc`, :man1:`flux-batch`,
+:man1:`flux-bulksubmit`, :man7:`flux-environment`
diff --git a/doc/man1/flux-shell.rst b/doc/man1/flux-shell.rst
new file mode 100644
index 000000000000..b1fa7f165056
--- /dev/null
+++ b/doc/man1/flux-shell.rst
@@ -0,0 +1,680 @@
+==============
+flux-shell(1)
+==============
+
+
+SYNOPSIS
+========
+
+**flux-shell** [*OPTIONS*] *JOBID*
+
+DESCRIPTION
+===========
+
+.. program:: flux shell
+
+:program:`flux shell`, the Flux job shell, is the component of Flux which
+manages the startup and execution of user jobs.  :program:`flux shell` runs as
+the job user, reads the jobspec and assigned resource set R for the job from
+the KVS, and using this data determines what local job tasks to execute. While
+job tasks are running, the job shell acts as the interface between the
+Flux instance and the job by handling standard I/O, signals, and finally
+collecting the exit status of tasks as they complete.
+
+The design of the Flux job shell allows customization through a set of
+builtin and runtime loadable shell plugins. These plugins are used to
+handle standard I/O redirection, PMI, CPU and GPU affinity, debugger
+support and more. Details of the :program:`flux shell` plugin capabilities and
+design can be found in the `PLUGINS`_ section below.
+
+:program:`flux shell` also supports configuration via a Lua-based configuration
+file, called the shell ``initrc``, from which shell plugins may be loaded
+or shell options and data examined or set. The :program:`flux shell` initrc may
+even extend the shell itself via simple shell plugins developed directly
+in Lua. See the `SHELL INITRC`_ section below for details of the ``initrc``
+format and features.
+
+OPTIONS
+=======
+
+.. option:: -h, --help
+
+   Summarize available options.
+
+.. option:: --reconnect
+
+   Attempt to reconnect if broker connection is lost.
+
+OPERATION
+=========
+
+When a job has been granted resources by a Flux instance, a
+:program:`flux shell` process is invoked on each broker rank involved in the
+job. The job shell runs as the job user, and will always have
+:envvar:`FLUX_KVS_NAMESPACE` set such that the root of the job shell's
+KVS accesses will be the guest namespace for the job.
+
+Each :program:`flux shell` connects to the local broker, fetches the jobspec
+and resource set **R** for the job from the job-info module, and uses this
+information to plan which tasks to locally execute.
+
+Once the job shell has successfully gathered job information, the
+:program:`flux shell` then goes through the following general steps to manage
+execution of the job:
+
+ * register service endpoint specific to the job and userid,
+   typically ``<userid>-shell-<jobid>``
+ * load the system default ``initrc.lua``
+   (``$sysconfdir/flux/shell/initrc.lua``), unless overridden by
+   configuration (See `SHELL OPTIONS`_ and `SHELL INITRC`_ sections below)
+ * call ``shell.init`` plugin callbacks
+ * change working directory to the cwd of the job
+ * enter a barrier to ensure shell initialization is complete on all shells
+ * emit ``shell.init`` event to exec.eventlog
+ * call ``shell.post-init`` plugin callbacks
+ * create all local tasks. For each task, the following procedure is used
+
+   - call ``task.init`` plugin callback
+   - launch task, call ``task.exec`` plugin callback just before :linux:man2:`execve`
+   - call ``task.fork`` plugin callback
+
+ * once all tasks have started, call ``shell.start`` plugin callback
+ * enter shell "start" barrier
+ * emit ``shell.start`` event, after which all tasks are known running
+ * for each exiting task:
+
+   - call ``task.exit`` plugin callback
+   - collect exit status
+
+ * call ``shell.exit`` plugin callback when all tasks have exited.
+ * exit with max task exit code
+
+PLUGINS
+=======
+
+The job shell supports external and builtin plugins which implement most
+of the advanced job shell features. Job shell plugins are loaded into
+a plugin stack by name, where the last loaded name wins. Therefore, to
+override a builtin plugin, an alternate plugin which registers the same
+name may be loaded at runtime.
+
+.. note::
+   Job shell plugins should be written with the assumption their access
+   to Flux services may be restricted as a guest.
+
+C plugins are defined using the Flux standard plugin format. A shell C
+plugin should therefore export a single symbol ``flux_plugin_init()``, in
+which calls to ``flux_plugin_add_handler(3)`` should be used to register
+functions which will be invoked at defined points during shell execution.
+These callbacks are defined by "topic strings" to which plugins can
+"subscribe" by calling ``flux_plugin_add_handler(3)`` and/or
+``flux_plugin_register(3)`` with topic :linux:man7:`glob` strings.
+
+.. note::
+   ``flux_plugin_init(3)`` is not called for builtin shell plugins. If
+   a dynamically loaded plugin wishes to set shell options to influence
+   a shell builtin plugin (e.g. to disable its operation), it should
+   therefore do so in ``flux_plugin_init()`` in order to guarantee that
+   the shell option is set before the builtin attempts to read them.
+
+Simple plugins may also be developed directly in the shell ``initrc.lua``
+file itself (see `SHELL INITRC`_ section, ``plugin.register()`` below)
+
+By default, :program:`flux shell` supports the following plugin callback
+topics:
+
+**taskmap.SCHEME**
+  Called when a taskmap scheme *SCHEME* is requested via the taskmap
+  shell option or corresponding :option:`flux submit --taskmap` option.
+  Plugins that want to offer a different taskmap scheme than the defaults
+  of ``block``, ``cyclic``, ``hostfile``, and ``manual`` can register a
+  ``taskmap.*`` plugin callback and then users can request this mapping
+  with the appropriate :option:`flux submit --taskmap=name`` option.
+  The default block taskmap is passed to the plugin as "taskmap" in the
+  plugin input arguments, and the plugin should return the new taskmap as a
+  string in the output args.  This callback is called before ``shell.init``.
+
+**shell.connect**
+  Called just after the shell connects to the local Flux broker. (Only
+  available to builtin shell plugins.)
+
+**shell.init**
+  Called after the shell has finished fetching and parsing the
+  **jobspec** and **R** from the KVS, but before any tasks
+  are started.
+
+**shell.post-init**
+  Called after the shell initialization barrier has completed, but
+  before starting any tasks.
+
+**task.init**
+  Called for each task after the task info has been constructed
+  but before the task is executed.
+
+**task.exec**
+  Called for each task after the task has been forked just before
+  :linux:man2:`execve` is called. This callback is made from within the
+  task process.
+
+**task.fork**
+  Called for each task after the task if forked from the parent
+  process (:program:`flux shell` process)
+
+**task.exit**
+  Called for each task after it exits and wait_status is available.
+
+**shell.start**
+  Called after all local tasks have been started. The shell "start"
+  barrier is called just after this callback returns.
+
+**shell.log**
+  Called by the shell logging facility when a shell component
+  posts a log message.
+
+**shell.log-setlevel**
+  Called by the shell logging facility when a request to set the
+  shell loglevel is made.
+
+
+Note however, that plugins may also call into the plugin stack to create
+new callbacks at runtime, so more topics than those listed above may be
+available in a given shell instance.
+
+.. _flux_shell_options:
+
+SHELL OPTIONS
+=============
+
+On startup, :program:`flux shell` will examine the jobspec for any shell
+specific options under the ``attributes.system.shell.options`` key.  These
+options may be set by the :option:`flux submit -o, --setopt=OPT` option,
+or explicitly added to the jobspec by other means.
+
+Job shell options may be switches to enable or disable a shell feature or
+plugin, or they may take an argument. Because jobspec is a JSON document,
+job shell options in jobspec may take arguments that are themselves
+JSON objects. This allows maximum flexibility in runtime configuration
+of optional job shell behavior. In the list below, if an option doesn't
+include a ``=``, then it is a simple boolean option or switch and may be
+specified simply with :option:`flux submit -o OPTION`.
+
+Job shell plugins may also support configuration via shell options in
+the jobspec. For specific information about runtime-loaded plugins,
+see the documentation for the specific plugin in question.
+
+Shell options supported by :program:`flux shell` itself and its built-in
+plugins include:
+
+.. option:: verbose[=INT]
+
+  Set the shell verbosity to *INT*. A larger value indicates increased
+  verbosity, though setting this value larger than 2 currently has no
+  effect.
+
+.. option:: nosetpgrp
+
+  Disable the use of :linux:man2:`setpgrp` to launch each
+  job task in its own process group. This will cause signals to be
+  delivered only to direct children of the shell.
+
+.. option:: initrc=FILE
+
+  Load :program:`flux shell` initrc.lua file from *FILE* instead of the default
+  initrc path. For details of the job shell initrc.lua file format,
+  see the `SHELL INITRC`_ section below.
+
+.. option:: userrc=FILE
+
+  Load another initrc.lua file after the system one.  For details of the
+  job shell initrc.lua file format, see the `SHELL INITRC`_ section below.
+
+.. option:: pty
+
+  Allocate a pty to all task ranks for non-interactive use. Output
+  from all ranks will be captured to the same location as ``stdout``.
+  This is the same as setting :option:`pty.ranks=all` and :option:`pty.capture`.
+  (see below).
+
+.. option:: pty.ranks=OPT
+
+  Set the task ranks for which to allocate a pty. *OPT* may be either
+  an RFC 22 IDset of target ranks, an integer rank, or the string "all"
+  to indicate all ranks.
+
+.. option:: pty.capture
+
+  Enable capture of pty output to the same location as stdout. This is
+  the default unless :option:`pty.interactive` is set.
+
+.. option:: pty.interactive
+
+  Enable a a pty on rank 0 that is set up for interactive attach by
+  a front-end program (i.e. :program:`flux job attach`). With no other
+  :option:`pty` options, only rank 0 will be assigned a pty and output will not
+  be captured. These defaults can be changed by setting other
+  :option:`pty` options after :option:`pty.interactive`, e.g.
+
+  .. code-block:: console
+
+    $  flux run -o pty.interactive -o pty.capture ...
+
+  would allocate an interactive pty on rank 0 and also capture the
+  pty session to the KVS (so it can be displayed after the job exits
+  with ``flux job attach``).
+
+.. option:: cpu-affinity=OPT
+
+  Adjust the operation of the builtin shell ``affinity`` plugin.  If the
+  option is unspecified, ``on`` is assumed.  *OPT* may be set to:
+
+  on
+    Bind each task to the full set of cores allocated to the job.
+
+  off
+    Disable the affinity plugin.  This may be useful if using another plugin
+    such as `mpibind <https://github.com/LLNL/mpibind>`_ to manage CPU
+    affinity.
+
+  per-task
+    Bind each task to an evenly populated subset of the cores allocated to
+    the job.  Tasks share cores only if there are more tasks than cores.
+
+  map:LIST
+    Bind each task to *LIST*, a semicolon-delimited list of cores.
+    Each entry in the list can be in one of the :linux:man7:`hwloc`
+    *list*, *bitmask*, or *taskset* formats. See `hwlocality_bitmap(3)
+    <https://www.open-mpi.org/projects/hwloc/doc/v2.9.0/a00181.php>`_,
+    especially the :func:`hwloc_bitmap_list_snprintf`,
+    :func:`hwloc_bitmap_snprintf` and :func:`hwloc_bitmap_taskset_snprintf`
+    functions.
+
+.. option:: gpu-affinity=OPT
+
+  Adjust operation of the builtin shell ``gpubind`` plugin.  This plugin
+  sets :envvar:`CUDA_VISIBLE_DEVICES` to the GPU IDs allocated to the job.
+  If the option is unspecified, ``on`` is assumed.  *OPT* may be set to:
+
+  on
+    Constrain each task to the full set of GPUs allocated to the job.
+
+  off
+    Disable the gpu-affinity plugin.
+
+  per-task
+    Constrain each task to an evenly populated subset of the GPUs allocated
+    to the job.  Tasks share GPUs only if there are more tasks than GPUs.
+
+  map:LIST
+    Constrain each task to *LIST*, a semicolon-delimited list of GPUs.
+    See :option:`cpu-affinity` above for a description of *LIST* format.
+
+.. option:: stop-tasks-in-exec
+
+  Stops tasks in ``exec()`` using ``PTRACE_TRACEME``. Used for debugging
+  parallel jobs. Users should not need to set this option directly.
+
+.. option:: output.{stdout,stderr}.type=TYPE
+
+  Set job output to for **stderr** or **stdout** to *TYPE*. *TYPE* may
+  be one of ``term``, ``kvs`` or ``file`` (Default: ``kvs``). If only
+  ``output.stdout.type`` is set, then this option applies to both
+  ``stdout`` and ``stderr``. If set to ``file``, then ``output.<stream>.path``
+  must also be set for the stream. Most users will not need to set
+  this option directly, as it will be set automatically by options
+  of higher level commands such as :man1:`flux-submit`.
+
+.. option:: output.limit=SIZE
+
+   Truncate KVS output after SIZE bytes have been written. SIZE may
+   be a floating point value with optional SI units k, K, M, G.  The maximum
+   value is 1G.  The default KVS output limit is 10M for jobs
+   in a multi-user instance or 1G for single-user instance jobs.
+   This value is ignored if output is directed to a file.
+
+.. option:: output.{stdout,stderr}.path=PATH
+
+  Set job stderr/out file output to PATH.
+
+.. option:: output.mode=truncate|append
+
+  Set the mode in which output files are opened to either truncate or
+  append. The default is to truncate.
+
+.. option:: input.stdin.type=TYPE
+
+  Set job input for **stdin** to *TYPE*. *TYPE* may be either ``service``
+  or ``file``. Users should not need to set this option directly as it
+  will be handled by options of higher level commands like :man1:`flux-submit`.
+
+.. option:: exit-timeout=VALUE
+
+  A fatal exception is raised on the job 30s after the first task exits.
+  The timeout period may be altered by providing a different value in
+  Flux Standard Duration form.  A value of ``none`` disables generation of
+  the exception.
+
+.. option:: exit-on-error
+
+  If the first task to exit was signaled or exited with a nonzero status,
+  raise a fatal exception on the job immediately.
+
+.. option:: rlimit
+
+  A dictionary of soft process resource limits to apply to the job before
+  starting tasks. Resource limits are set to integer values by lowercase
+  name without the ``RLIMIT_`` prefix, e.g. ``core`` or ``nofile``. Users
+  should not need to set this shell option as it is handled by commands
+  like :man1:`flux-submit`.
+
+.. option:: taskmap
+
+  Request an alternate job task mapping. This option is an object
+  consisting of required key ``scheme`` and optional key ``value``. The
+  shell will attempt to call a ``taskmap.scheme`` plugin callback in the
+  shell to invoke the alternate requested mapping. If ``value`` is set,
+  this will also be passed to the invoked plugin. Normally, this option will
+  be set by the :man1:`flux-submit` and related commands :option:`--taskmap`
+  option.
+
+.. option:: pmi=LIST
+
+  Specify a comma-separated list of PMI implementations to enable.  If the
+  option is unspecified, ``simple`` is assumed.  To disable, set *LIST* to
+  ``off``.  Available implementations include
+
+  simple
+    The simple PMI-1 wire protocol.  This implementation works by passing an
+    open file descriptor to clients via the :envvar:`PMI_FD` environment
+    variable.  It must be enabled when Flux's ``libpmi.so`` or ``libpmi2.so``
+    libraries are used, and is preferred by :man1:`flux-broker`
+    when Flux launches Flux, e.g. by means of :man1:`flux-batch` or
+    :man1:`flux-alloc`.
+
+  pmi1, pmi2
+    Aliases for ``simple``.
+
+  cray-pals
+    Provided via external plugin from the
+    `flux-coral2 <https://github.com/flux-framework/flux-coral2>`_ project.
+
+  pmix
+    Provided via external plugin from the
+    `flux-pmix <https://github.com/flux-framework/flux-pmix>`_ project.
+
+.. option:: pmi-simple.nomap
+
+  Skip pre-populating the ``flux.taskmap`` and ``PMI_process_mapping`` keys
+  in the ``simple`` implementation.
+
+.. option:: pmi-simple.kvs=native
+
+  Use the native Flux KVS instead of the PMI plugin's built-in key exchange
+  algorithm in the ``simple`` implementation.
+
+.. option:: pmi-simple.exchange.k=N
+
+  Configure the PMI plugin's built-in key exchange algorithm to use a
+  virtual tree fanout of ``N`` for key gather/broadcast in the ``simple``
+  implementation.  The default is 2.
+
+.. option:: stage-in
+
+  Copy files to the directory referenced by :envvar:`FLUX_JOB_TMPDIR` that
+  were previously archived with :man1:`flux-archive`.
+
+.. option:: stage-in.names=LIST
+
+  Select archives to extract by specifying a comma-separated list of names
+  If no names are specified, ``main`` is assumed.
+
+.. option:: stage-in.pattern=PATTERN
+
+  Further filter the selected files to copy using a :man7:`glob` pattern.
+
+.. option:: stage-in.destination=[SCOPE:]PATH
+
+  Copy files to the specified destination instead of the directory referenced
+  by :envvar:`FLUX_JOB_TMPDIR`.  The argument is a directory with optional
+  *scope* prefix.  A scope of ``local`` denotes a local file system (the
+  default), and a scope of ``global`` denotes a global file system.  The copy
+  takes place on all the job's nodes if the scope is local, versus only the
+  first node of the job if the scope is global.
+
+.. option:: signal=OPTION
+
+  Deliver signal ``SIGUSR1`` to the job 60s before job expiration.
+  To customize the signal number or amount of time before expiration to
+  deliver the signal, the ``signal`` option may be an object with one
+  or both of the keys ``signum`` or ``timeleft``. (See below)
+
+.. option:: signal.signum=NUMBER
+
+  Send signal *NUMBER* to the job :option:`signal.timeleft` seconds before
+  the time limit.
+
+.. option:: signal.timeleft=TIME
+
+  Send signal :option:`signal.signum` *TIME* seconds before job expiration.
+
+.. option:: hwloc.xmlfile
+
+  Write the job shell's copy of hwloc XML to a file and set ``HWLOC_XMLFILE``.
+  Note that this option will also unset ``HWLOC_COMPONENTS`` since presence
+  of this environment variable may cause hwloc to ignore ``HWLOC_XMLFILE``.
+
+.. option:: hwloc.restrict
+
+  With :option:`hwloc.xmlfile`, restrict the exported topology XML to only
+  the resources assigned to the current job. By default the XML is not
+  restricted.
+
+.. warning::
+  The directory referenced by :envvar:`FLUX_JOB_TMPDIR` is cleaned up when the
+  job ends, is guaranteed to be unique, and is generally on fast local storage
+  such as a *tmpfs*.  If a destination is explicitly specified, use the
+  ``global:`` prefix where appropriate to avoid overwhelming a shared file
+  system, and be sure to clean up.
+
+.. _flux_shell_initrc:
+
+SHELL INITRC
+============
+
+At initialization, :program:`flux shell` reads a Lua initrc file which can be
+used to customize the shell operation. The initrc is loaded by default from
+``$sysconfdir/flux/shell/initrc.lua`` (or ``/etc/flux/shell/initrc.lua``
+for a "standard" install), but a different path may be specified when
+launching a job via the ``initrc`` shell option.  Alternatively, the ``userrc``
+shell option can specify an initrc file to load after the system one.
+
+A job shell initrc file may be used to adjust the shell plugin searchpath,
+load specific plugins, read and set shell options, and even extend the
+shell itself using Lua.
+
+Since the job shell ``initrc`` is a Lua file, any Lua syntax is
+supported. Job shell specific functions and tables are described below:
+
+**plugin.searchpath**
+  The current plugin searchpath. This value can be queried, set,
+  or appended. E.g. to add a new path to the plugin search path:
+  ``plugin.searchpath = plugin.searchpath .. ':' .. path``
+
+**plugin.load({file=glob, [conf=table]})**
+  Explicitly load one more shell plugins. This function takes a table
+  argument with ``file`` and ``conf`` arguments. The ``file`` argument
+  is a glob of one or more plugins to load. If an absolute path is not
+  specified, then the glob will be relative to ``plugin.searchpath``.
+  E.g. ``plugin.load { file = "*.so" }`` will load all ``.so`` plugins in
+  the current search path. The ``conf`` option allows static configuration
+  values to be passed to plugin initialization functions when supported.
+
+  For example a plugin ``test.so`` may be explicitly loaded with
+  configuration via:
+
+  .. code-block:: lua
+
+    plugin.load { file = "test.so", conf = { value = "foo" } }
+
+**plugin.register({name=plugin_name, handlers=handlers_table)**
+  Register a Lua plugin. Requires a table argument with the plugin ``name``
+  and a set of ``handlers``. ``handlers_table`` is an array of tables, each
+  of which must define ``topic``, a topic glob of shell plugin callbacks to
+  which to subscribe, and ``fn`` a handler function to call for each match
+
+  For example, the following plugin would log the topic string for
+  every possible plugin callback (except for callbacks which are made
+  before the shell logging facility is initialized)
+
+  .. code-block:: lua
+
+    plugin.register {
+      name = "test",
+      handlers = {
+         { topic = "*",
+           fn = function (topic) shell.log ("topic="..topic) end
+         },
+      }
+    }
+
+**source(glob)**
+  Source another Lua file or files. Supports specification of a glob,
+  e.g. ``source ("*.lua")``.  This function fails if a non-glob argument
+  specifies a file that does not exist, or there is an error loading or
+  compiling the Lua chunk.
+
+**source_if_exists(glob)**
+  Same as ``source()``, but do not throw an error if the target file does
+  not exist.
+
+**shell.rcpath**
+  The directory in which the current initrc file resides.
+
+**shell.getenv([name])**
+  Return the job environment (not the job shell environment). This is
+  the environment which will be inherited by the job tasks. If called
+  with no arguments, then the entire environment is copied to a table
+  and returned. Otherwise, acts as :man3:`flux_shell_getenv` and returns
+  the value for the environment variable name, or ``nil`` if not set.
+
+**shell.setenv(var, val, [overwrite])**
+  Set environment variable ``var`` to value ``val`` in the job environment.
+  If ``overwrite`` is set and is ``0`` or ``false`` then do not overwrite
+  existing environment variable value.
+
+**shell.unsetenv(var)**
+  Unset environment variable ``var`` in job environment.
+
+**shell.options**
+  A virtual index into currently set shell options, including those
+  set in jobspec. This table can be used to check jobspec options,
+  and even to force certain options to a value by default e.g.
+  ``shell.options['cpu-affinity'] = "per-task"``, would force
+  ``cpu-affinity`` shell option to ``per-task``.
+
+**shell.options.verbose**
+  Current :program:`flux shell` verbosity. This value may be changed at
+  runtime, e.g. ``shell.options.verbose = 2`` to set maximum verbosity.
+
+**shell.info**
+  Returns a Lua table of shell information obtained via
+  :man3:`flux_shell_get_info`. This table includes
+
+  **jobid**
+    The current jobid.
+  **rank**
+    The rank of the current shell within the job.
+  **size**
+    The number of :program:`flux shell` processes participating in this job.
+  **ntasks**
+    The total number of tasks in this job.
+  **service**
+    The service string advertised by the shell.
+  **options.verbose**
+    True if the shell is running in verbose mode.
+  **jobspec**
+    The jobspec of the current job
+  **R**
+    The resource set **R** of the current job
+
+**shell.rankinfo**
+  Returns a Lua table of rank-specific shell information for the
+  current shell rank. See `shell.get_rankinfo()` for a description
+  of the members of this table.
+
+**shell.get_rankinfo(shell_rank)**
+  Query rank-specific shell info as in the function call
+  :man3:`flux_shell_get_rank_info`.  If ``shell_rank`` is not provided
+  then the current rank is used.  Returns a table of rank-specific
+  information including:
+
+  **broker_rank**
+    The broker rank on which ``shell_rank`` is running.
+  **ntasks**
+    The number of local tasks assigned to ``shell_rank``.
+  **resources**
+    A table of resources by name (e.g. "core", "gpu") assigned to
+    ``shell_rank``, e.g. ``{ core = "0-1", gpu = "0" }``.
+
+**shell.log(msg)**, **shell.debug(msg)**, **shell.log_error(msg)**
+  Log messages to the shell log facility at INFO, DEBUG, and ERROR
+  levels respectively.
+
+**shell.die(msg)**
+  Log a FATAL message to the shell log facility. This generates a
+  job exception and will terminate the job.
+
+The following task-specific initrc data and functions are available
+only in one of the ``task.*`` plugin callbacks. An error will be
+generated if they are accessed from any other context.
+
+**task.info**
+  Returns a Lua table of task specific information for the "current"
+  task (see :man3:`flux_shell_task_get_info`). Included members of
+  the ``task.info`` table include:
+
+  **localid**
+    The local task rank (i.e. within this shell)
+  **rank**
+    The global task rank (i.e. within this job)
+  **state**
+    The current task state name
+  **pid**
+    The process id of the current task (if task has been started)
+  **wait_status**
+    (Only in ``task.exit``) The status returned by
+    :linux:man2:`waitpid` for this task.
+  **exitcode**
+    (Only in ``task.exit``) The exit code if ``WIFEXTED()`` is true.
+  **signaled**
+    (Only in ``task.exit``) If task was signaled, this member will be
+    non-zero integer signal number that caused the task to exit.
+
+**task.getenv(var)**
+  Get the value of environment variable ``var`` if set in the current
+  task's environment. This function reads the environment from the
+  underlying ``flux_cmd_t`` for a shell task, and thus only makes sense
+  before a task is executed, e.g. in ``task.init`` and ``task.exec``
+  callbacks.
+
+**task.unsetenv(var)**
+  Unset environment variable ``var`` for the current task. As with
+  ``task.getenv()`` this function is only valid before a task has
+  been started.
+
+**task.setenv(var, value, [overwrite])**
+  Set environment variable ``var`` to ``val`` for the current task.
+  If ``overwrite`` is set to ``0`` or ``false``, then do not overwrite
+  any current value. As with ``task.getenv()`` and ``task.unsetenv()``,
+  this function only has an effect before the task is started.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-submit`
diff --git a/doc/man1/flux-shutdown.rst b/doc/man1/flux-shutdown.rst
new file mode 100644
index 000000000000..a427ef871ecf
--- /dev/null
+++ b/doc/man1/flux-shutdown.rst
@@ -0,0 +1,111 @@
+================
+flux-shutdown(1)
+================
+
+
+SYNOPSIS
+========
+
+**flux** **shutdown** [*OPTIONS*] [*TARGET*]
+
+
+DESCRIPTION
+===========
+
+.. program:: flux shutdown
+
+The :program:`flux shutdown` command causes the default Flux instance, or the
+instance specified by *TARGET*, to exit RUN state and begin the process
+of shutting down.  *TARGET* may be either a native Flux URI or a high level
+URI, as described in :man1:`flux-uri`.
+
+Only the rank 0 broker in RUN state may be targeted for shutdown.
+The current broker state may be viewed with :man1:`flux-uptime`.
+
+If the instance is running an initial program, that program is terminated
+with SIGHUP.  Note that the broker exit value normally reflects the
+exit code of the initial program, so if it is terminated by this signal,
+the broker exits with 128 + 1 = 129.
+
+If the broker was launched by systemd, an exit code is used that informs
+systemd not to restart the broker.
+
+Broker log messages that are posted during shutdown are displayed by
+the shutdown command on stderr, until the broker completes executing its
+``rc3`` script.  By default, log messages with severity level <= LOG_INFO
+are printed.
+
+A Flux system instance requires offline KVS garbage collection to remove
+deleted KVS content and purged job directories, which accrue over time and
+increase storage overhead and restart time.  It is recommended that the
+:option:`--gc` option be used on a routine basis to optimize Flux.
+
+
+OPTIONS
+=======
+
+:program:`flux shutdown` accepts the following options:
+
+.. option:: -h, --help
+
+   Display options and exit
+
+.. option:: --background
+
+   Start the shutdown and exit immediately, without monitoring the process
+   and displaying log messages.
+
+.. option:: --quiet
+
+   Show only error log messages (severity level <= LOG_WARNING level).
+
+.. option:: --verbose=[LEVEL]
+
+   Increase output verbosity.  Level 1 shows all log messages.  Higher
+   verbosity levels are reserved for future use.
+
+.. option:: --dump=PATH
+
+   Dump a checkpoint of KVS content to *PATH* using :man1:`flux-dump` after the
+   KVS has been unloaded.  The dump may be restored into a new Flux instance
+   using :man1:`flux-restore`.  Dump creation adds time to the shutdown
+   sequence, proportional to the amount of data in the KVS.
+   :option:`--dump=auto` is a special case equivalent to :option:`--gc`.
+
+.. option:: --gc
+
+   Prepare for offline KVS garbage collection by dumping a checkpoint of KVS
+   content to ``dump/<date>.tgz`` in *statedir*, if defined, otherwise in
+   the broker's current working directory.  Create a symbolic link named
+   ``dump/RESTORE`` pointing to the dump file.  When this link is discovered
+   on instance startup, the content database is truncated and recreated from
+   the dump, and the link is removed.  :linux:man8:`systemd-tmpfiles`
+   automatically cleans up dump files in ``/var/lib/flux/dump`` after 30 days.
+
+.. option:: --skip-gc
+
+   When garbage collection has been enabled automatically, as indicated
+   by the ``content.dump`` broker attribute, this option disables it
+   during shutdown.  Otherwise it is a preemptive "no" answer to the garbage
+   collection prompt.
+
+.. option:: -y, --yes
+
+   Answer yes to any yes/no questions.
+
+.. option:: -n, --no
+
+   Answer no to any yes/no questions.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-start`, :man1:`flux-uptime`, :man1:`flux-uri`, :man1:`flux-dump`,
+:man5:`flux-config-kvs`,:linux:man8:`systemd-tmpfiles`
diff --git a/doc/man1/flux-start.adoc b/doc/man1/flux-start.adoc
deleted file mode 100644
index e77853e25c4e..000000000000
--- a/doc/man1/flux-start.adoc
+++ /dev/null
@@ -1,104 +0,0 @@
-// flux-help-include: true
-FLUX-START(1)
-=============
-:doctype: manpage
-
-
-NAME
-----
-flux-start - bootstrap a local Flux instance
-
-
-SYNOPSIS
---------
-*flux* *start* ['OPTIONS'] [initial-program [args...]]
-
-
-DESCRIPTION
------------
-flux-start(1) launches a new Flux instance.  By default, flux-start
-execs a single flux-broker(1) directly.  By default it will attempt to use
-PMI to fetch job information and bootstrap a flux instance.
-
-If a size is specified via '--size', an instance of that size is to be
-started on the local host with flux-start as the parent. (Mostly used for testing
-purposes.)
-
-A failure of the initial program (such as non-zero exit code)
-causes flux-start to exit with a non-zero exit code.
-
-Note: in order to launch a Flux instance, you must have generated
-long-term CURVE keys using flux-keygen(1).
-
-OPTIONS
--------
-*-s, --size*='N'::
-Launch an instance of size 'N' on the local host.  Only works with
-'--bootstrap=selfpmi'.  Automatically sets '--bootstrap=selfpmi' and prints
-a warning to stderr if no '--bootstrap' option is specified.
-
-*-b, --bootstrap*='METHOD'::
-Select the flux bootstrap method.  Possible values of 'METHOD' are:
-  * pmi - Use PMI (Process Management Interface)
-  * selfpmi - The flux-start process will activate its own internal PMI server,
-    and then it will launch flux-brokers that will bootstrap using said PMI server.
-
-*-o, --broker-opts*='option_string'::
-Add options to the message broker daemon, separated by commas.
-
-*-v, --verbose*::
-Display commands before executing them.
-
-*-X, --noexec*::
-Don't execute anything.  This option is most useful with -v.
-
-*--caliper-profile*='PROFILE'::
-Run brokers with Caliper profiling enabled, using a Caliper
-configuration profile named 'PROFILE'. Requires a version of Flux
-built with --enable-caliper. Unless CALI_LOG_VERBOSITY is already
-set in the environment, it will default to 0 for all brokers.
-
-*--scratchdir*='DIR'::
-For selfpmi bootstrap mode, set the directory that will be
-used as the rundir directory for the instance. If the directory
-does not exist then it will be created during instance startup.
-If a DIR is not set with this option, a unique temporary directory
-will be created. Unless DIR was pre-existing, it will be removed
-when the instance is destroyed.
-
-*--wrap*='ARGS,...'::
-Wrap broker execution in a comma-separated list of arguments. This is
-useful for running flux-broker directly under debuggers or valgrind.
-
-
-EXAMPLES
---------
-
-Launch an 8-way local Flux instance with an interactive shell as the
-initial program and all logs output to stderr:
-
-  flux start -s8 -o,--setattr=log-stderr-level=7
-
-Launch an 8-way Flux instance within a slurm job, with an interactive
-shell as the initial program:
-
-  srun --pty -N8 flux start
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-flux-broker(1) flux-keygen(1)
diff --git a/doc/man1/flux-start.rst b/doc/man1/flux-start.rst
new file mode 100644
index 000000000000..a7c4acda847b
--- /dev/null
+++ b/doc/man1/flux-start.rst
@@ -0,0 +1,312 @@
+.. flux-help-include: true
+
+=============
+flux-start(1)
+=============
+
+
+SYNOPSIS
+========
+
+[**launcher**] **flux** **start** [*OPTIONS*] [initial-program [args...]]
+
+**flux** **start** *--test-size=N* [*OPTIONS*] [initial-program [args...]]
+
+DESCRIPTION
+===========
+
+.. program:: flux start
+
+:program:`flux start` assists with launching a new Flux instance, which
+consists of one or more :man1:`flux-broker` processes functioning as a
+distributed system.  It is primarily useful in environments that don't run
+Flux natively, or when a standalone Flux instance is required for test,
+development, or post-mortem debugging of another Flux instance.
+
+When already running under Flux, single-user Flux instances can be more
+conveniently started with :man1:`flux-batch` and :man1:`flux-alloc`.
+The `Flux Administration Guide
+<https://flux-framework.readthedocs.io/en/latest/guides/admin-guide.html>`_
+covers setting up a multi-user Flux "system instance", where Flux natively
+manages a cluster's resources and those commands work ab initio for its users.
+
+:program:`flux start` operates in two modes.  In `NORMAL MODE`_, it does not
+launch broker processes; it *becomes* a single broker which joins an externally
+bootstrapped parallel program.  In `TEST MODE`_, it starts one or more brokers
+locally, provides their bootstrap environment, and then cleans up when the
+instance terminates.
+
+NORMAL MODE
+===========
+
+Normal mode is used when an external launcher like Slurm or Hydra starts
+the broker processes and provides the bootstrap environment.  It is selected
+when the :option:`--test-size` option is *not* specified.
+
+In normal mode, :program:`flux start` replaces itself with a broker process
+by calling :linux:man2:`execvp`.  The brokers bootstrap as a parallel program
+and establish overlay network connections.  The usual bootstrap method is
+some variant of the Process Management Interface (PMI) provided by the
+launcher.
+
+For example, Hydra provides a simple PMI server.  The following command
+starts brokers on the hosts listed in a file called ``hosts``.  The
+instance's initial program prints a URI that can be used with
+:man1:`flux-proxy` and then sleeps forever::
+
+  mpiexec.hydra -f hosts -launcher ssh \
+    flux start "flux uri --remote \$FLUX_URI; sleep inf"
+
+Slurm has a PMI-2 server plugin with backwards compatibility to the simple
+PMI-1 wire protocol that Flux prefers.  The following command starts a two
+node Flux instance in a Slurm allocation, with an interactive shell as the
+initial program (the default if none is specified)::
+
+  srun -N2 --pty --mpi=pmi2 flux start
+
+When Flux is started by a launcher that is not Flux, resources are probed
+using `HWLOC <https://www.open-mpi.org/projects/hwloc/>`_.  If all goes well,
+when Slurm launches Flux :option:`flux resource info` in Flux should show all
+the nodes, cores, and GPUs that Slurm allocated to the job.
+
+TEST MODE
+=========
+
+Test mode, selected by specifying the :option:`--test-size` option, launches
+a single node Flux instance that is independent of any configured resource
+management on the node.  In test mode, :program:`flux start` provides the
+bootstrap environment and launches the broker process(es).  It remains running
+as long as the Flux instance is running.  It covers the following use cases:
+
+- Start an interactive Flux instance on one node such as a developer system
+  ::
+
+    flux start --test-size=1
+
+  Jobs can be submitted from the interactive shell started as the initial
+  program, similar to the experience of running on a one node cluster.
+
+- Mock a multi-node (multi-broker) Flux instance on one node
+  ::
+
+    flux start --test-size=64
+
+  When the test size is greater than one, the actual resource inventory is
+  multiplied by the test size, since each broker thinks it
+  is running on a different node and re-discovers the same resources.
+
+- Start a Flux instance to run a continuous integration test.  A test
+  that runs jobs in Flux can be structured as::
+
+    flux start --test-size=1 test.sh
+
+  where ``test.sh`` (the initial program) runs work under Flux.  The exit
+  status of :program:`flux start` reflects the exit status of ``test.sh``.
+  This is how many of Flux's own tests work.
+
+- Start a Flux instance to access job data from an inactive batch job that
+  was configured to leave a dump file::
+
+   flux start --test-size=1 --recovery=dump.tar
+
+- Start a Flux instance to repair the on-disk state of a crashed system
+  instance (experts only)::
+
+   sudo -u flux flux start --test-size=1 --recovery
+
+- Run the broker under :linux:man1:`gdb` from the source tree::
+
+   ${top_builddir}/src/cmd/flux start --test-size=1 \
+      --wrap=libtool,e,gdb
+
+
+OPTIONS
+=======
+.. option:: -S, --setattr=ATTR=VAL
+
+   Set broker attribute *ATTR* to *VAL*. This is equivalent to
+   :option:`-o,-SATTR=VAL`.
+
+.. option:: -c, --config-path=PATH
+
+   Set the *PATH* for broker configuration. See :man1:`flux-broker` for
+   option details. This is equivalent to :option:`-o,-cPATH`.
+
+.. option:: -o, --broker-opts=OPTIONS
+
+   Add options to the message broker daemon, separated by commas.
+
+.. option:: -v, --verbose=[LEVEL]
+
+   This option may be specified multiple times, or with a value, to
+   set a verbosity level (1: display commands before executing them,
+   2: trace PMI server requests in `TEST MODE`_ only).
+
+.. option:: -X, --noexec
+
+   Don't execute anything. This option is most useful with -v.
+
+.. option:: --rundir=DIR
+
+   (only with :option:`--test-size`) Set the directory that will be
+   used as the rundir directory for the instance. If the directory
+   does not exist then it will be created during instance startup.
+   If a DIR is not set with this option, a unique temporary directory
+   will be created. Unless DIR was pre-existing, it will be removed
+   when the instance is destroyed.
+
+.. option:: --wrap=ARGS
+
+   Wrap broker execution in a comma-separated list of arguments. This is
+   useful for running flux-broker directly under debuggers or valgrind.
+
+.. option:: -s, --test-size=N
+
+   Launch an instance of size *N* on the local host.
+
+.. option:: --test-hosts=HOSTLIST
+
+   Set :envvar:`FLUX_FAKE_HOSTNAME` in the environment of each broker so that
+   the broker can bootstrap from a config file instead of PMI.  HOSTLIST is
+   assumed to be in rank order.  The broker will use the fake hostname to
+   find its entry in the configured bootstrap host array.
+
+.. option:: --test-exit-timeout=FSD
+
+   After a broker exits, kill the other brokers after a timeout (default 20s).
+
+.. option:: --test-exit-mode=MODE
+
+   Set the mode for the exit timeout.  If set to ``leader``, the exit timeout
+   is only triggered upon exit of the leader broker, and the
+   :program:`flux start` exit code is that of the leader broker.  If set to
+   ``any``, the exit timeout is triggered upon exit of any broker, and the
+   :program:`flux start` exit code is the highest exit code of all brokers.
+   Default: ``any``.
+
+.. option:: --test-start-mode=MODE
+
+   Set the start mode.  If set to ``all``, all brokers are started immediately.
+   If set to ``leader``, only the leader is started.  Hint: in ``leader`` mode,
+   use :option:`--setattr=broker.quorum=1` to let the initial program start
+   before the other brokers are online.  Default: ``all``.
+
+.. option:: --test-rundir=PATH
+
+   Set the directory to be used as the broker rundir instead of creating a
+   temporary one.  The directory must exist, and is not cleaned up unless
+   :option:`--test-rundir-cleanup` is also specified.
+
+.. option:: --test-rundir-cleanup
+
+   Recursively remove the directory specified with :option:`--test-rundir` upon
+   completion of :program:`flux start`.
+
+.. option:: --test-pmi-clique=MODE
+
+   Set the pmi clique mode, which determines how ``PMI_process_mapping`` is set
+   in the PMI server used to bootstrap the brokers.  If ``none``, the mapping
+   is not created.  If ``single``, all brokers are placed in one clique. If
+   ``per-broker``, each broker is placed in its own clique.
+   Default: ``single``.
+
+.. option:: -r, --recovery=[TARGET]
+
+   Start the rank 0 broker of an instance in recovery mode.  If *TARGET*
+   is a directory, treat it as a *statedir* from a previous instance.
+   If *TARGET* is a file, treat it as an archive file from :man1:`flux-dump`.
+   If *TARGET* is unspecified, assume the system instance is to be recovered.
+   In recovery mode, any rc1 errors are ignored, broker peers are not allowed
+   to connect, and resources are offline.
+
+.. option:: --sysconfig
+
+   Run the broker with :option:`--config-path` set to the default system
+   instance configuration directory.  This option is unnecessary if
+   :option:`--recovery` is specified without its optional argument.  It may
+   be required if recovering a dump from a system instance.
+
+
+TROUBLESHOOTING
+===============
+
+`NORMAL MODE`_ requires Flux, the launcher, and the network to cooperate.
+If :program:`flux start` appears to hang, the following tips may be helpful:
+
+#. Reduce the size of the Flux instance to at most two nodes.  This reduces the
+   volume of log data to look at and may be easier to allocate on a busy
+   system.  Rule out the simple problems that can be reproduced with a small
+   allocation first.
+
+#. Use an initial program that prints something and exits rather than the
+   default interactive shell, in case there are problems with the launcher's
+   pty setup.  Something like::
+
+     [launcher] flux start [options] echo hello world
+
+#. Ensure that standard output and error are being captured and add launcher
+   options to add rank prefixes to the output.
+
+   .. list-table::
+
+     * - Slurm
+       - :option:`--label`
+
+     * - Hydra
+       - :option:`-prepend-rank`
+
+     * - :man1:`flux-run`
+       - :option:`--label-io`
+
+#. Tell the broker to print its rank, size, and network endpoint by adding
+   the :option:`flux start -o,-v` option.  If this doesn't happen, most likely
+   the PMI bootstrap is getting stuck.
+
+#. Trace Flux's PMI client on stderr by setting the FLUX_PMI_DEBUG environment
+   variable::
+
+     FLUX_PMI_DEBUG=1 [launcher] flux start ...
+
+#. Consider altering :envvar:`FLUX_PMI_CLIENT_METHODS` to better match the
+   launcher's PMI offerings.  See :man7:`flux-environment`.
+
+#. A launcher's PMI capabilities can also be explored in a simplified way
+   using the :man1:`flux-pmi` client.
+
+#. If PMI is successful but the initial program fails to run, the brokers
+   may not be able to reach each other over the network.  After one minute,
+   the rank 0 broker should log a "quorum delayed" message if this is true.
+
+#. Examine the network endpoints in the output above.  Flux preferentially
+   binds to the IPv4 network address associated with the default route and
+   a random port.  The address choice can be modified by setting the
+   :envvar:`FLUX_IPADDR_HOSTNAME` and/or :envvar:`FLUX_IPADDR_V6`.
+   See :man7:`flux-environment`.
+
+#. More logging can be enabled by adding the
+   :option:`flux start -Slog-stderr-level=7` option, which instructs the
+   broker to forward its internal log buffer to stderr.  See
+   :man7:`flux-broker-attributes`.
+
+Another common failure mode is getting a single node instance when multiple
+nodes were expected.  This can occur if no viable PMI server was found and the
+brokers fell back to singleton operation.  It may be helpful to enable PMI
+tracing, check into launcher PMI options, and possibly adjust the order of
+options that Flux tries using :envvar:`FLUX_PMI_CLIENT_METHODS` as described
+above.
+
+Finally, if Flux starts but GPUs are missing from :option:`flux resource info`
+output, verify that the version of HWLOC that Flux is using was built with
+the appropriate GPU plugins.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-broker`
diff --git a/doc/man1/flux-startlog.rst b/doc/man1/flux-startlog.rst
new file mode 100644
index 000000000000..7ce5271eca42
--- /dev/null
+++ b/doc/man1/flux-startlog.rst
@@ -0,0 +1,71 @@
+================
+flux-startlog(1)
+================
+
+
+SYNOPSIS
+========
+
+**flux** **startlog**
+
+
+DESCRIPTION
+===========
+
+.. program:: flux startlog
+
+List the Flux instance's start and stop times, by interpreting the contents
+of the KVS ``admin.eventlog``.
+
+A ``start`` event is posted to the eventlog at startup, and a ``finish`` event
+is posted to the eventlog at finalization.  The timestamps on the two events
+are used to calculate the instance run time, which is shown in the listing
+in Flux Standard Duration format.
+
+If the current ``start`` event is not immediately preceded by a ``finish``
+event (unless it is the first entry in the eventlog), then the Flux instance
+may have crashed and data may have been lost.  If this is detected on instance
+startup, it is logged by the broker's ``rc1`` script on the next reboot.
+
+This command is not available to guest users.
+
+
+OPTIONS
+=======
+
+.. option:: -h, --help
+
+   Summarize available options.
+
+.. option:: --check
+
+   If the instance has most recently restarted from a crash, exit with a
+   return code of 1, otherwise 0.
+
+.. option:: --quiet
+
+   Suppress non-error output.
+
+.. option:: -v, --show-version
+
+   Show the flux-core software version associated with each start event.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_18`
+
+:doc:`rfc:spec_23`
+
+
+SEE ALSO
+========
+
+:man1:`flux-uptime`, :man1:`flux-kvs`
diff --git a/doc/man1/flux-submit.rst b/doc/man1/flux-submit.rst
new file mode 100644
index 000000000000..4bade399689f
--- /dev/null
+++ b/doc/man1/flux-submit.rst
@@ -0,0 +1,167 @@
+.. flux-help-include: true
+.. flux-help-section: submission
+
+==============
+flux-submit(1)
+==============
+
+
+SYNOPSIS
+========
+
+**flux** **submit** [OPTIONS] [*--ntasks=N*] COMMAND...
+
+
+DESCRIPTION
+===========
+
+.. program:: flux submit
+
+:program:`flux submit` enqueues a job to run under Flux and prints its
+numerical jobid on standard output.  The job consists of *N* copies of COMMAND
+launched together as a parallel job.
+
+If :option:`--ntasks` is unspecified, a value of *N=1* is assumed.
+
+The available OPTIONS are detailed below.
+
+JOB PARAMETERS
+==============
+
+These commands accept only the simplest parameters for expressing
+the size of the parallel program and the geometry of its task slots:
+
+Common resource options
+-----------------------
+
+These commands take the following common resource allocation options:
+
+.. include:: common/job-param-common.rst
+
+Per-task options
+----------------
+
+:man1:`flux-run`, :man1:`flux-submit` and :man1:`flux-bulksubmit` take two
+sets of mutually exclusive options to specify the size of the job request.
+The most common form uses the total number of tasks to run along with
+the amount of resources required per task to specify the resources for
+the entire job:
+
+.. include:: common/job-param-pertask.rst
+
+Per-resource options
+--------------------
+
+The second set of options allows an amount of resources to be specified
+with the number of tasks per core or node set on the command line. It is
+an error to specify any of these options when using any per-task option
+listed above:
+
+.. include:: common/job-param-perres.rst
+
+Additional job options
+----------------------
+
+These commands also take following job parameters:
+
+.. include:: common/job-param-additional.rst
+
+STANDARD I/O
+============
+
+By default, task stdout and stderr streams are redirected to the
+KVS, where they may be accessed with the ``flux job attach`` command.
+
+In addition, :man1:`flux-run` processes standard I/O in real time,
+emitting the job's I/O to its stdout and stderr.
+
+.. include:: common/job-standard-io.rst
+
+CONSTRAINTS
+===========
+
+.. include:: common/job-constraints.rst
+
+DEPENDENCIES
+============
+
+.. include:: common/job-dependencies.rst
+
+ENVIRONMENT
+===========
+
+By default, these commands duplicate the current environment when submitting
+jobs. However, a set of environment manipulation options are provided to
+give fine control over the requested environment submitted with the job.
+
+.. include:: common/job-environment.rst
+
+ENV RULES
+---------
+
+.. include:: common/job-env-rules.rst
+
+PROCESS RESOURCE LIMITS
+=======================
+
+By default these commands propagate some common resource limits (as described
+in :linux:man2:`getrlimit`) to the job by setting the ``rlimit`` job shell
+option in jobspec.  The set of resource limits propagated can be controlled
+via the :option:`--rlimit=RULE` option:
+
+.. include:: common/job-process-resource-limits.rst
+
+JOB ENVIRONMENT VARIABLES
+=========================
+
+The job environment is described in more detail in the :man7:`flux-environment`
+:ref:`job_environment` section.  A summary of the most commonly used variables
+is provided below:
+
+.. include:: common/job-environment-variables.rst
+
+EXIT STATUS
+===========
+
+The job exit status, normally the largest task exit status, is stored
+in the KVS. If one or more tasks are terminated with a signal,
+the job exit status is 128+signo.
+
+The ``flux-job attach`` command exits with the job exit status.
+
+In addition, :man1:`flux-run` runs until the job completes and exits
+with the job exit status.
+
+OTHER OPTIONS
+=============
+
+.. include:: common/job-other-options.rst
+
+.. include:: common/job-other-run.rst
+
+SHELL OPTIONS
+=============
+
+Some options that affect the parallel runtime environment are provided by the
+Flux shell.  These options are described in detail in the
+:ref:`SHELL OPTIONS <flux_shell_options>` section of :man1:`flux-shell`.
+A list of the most commonly needed options follows.
+
+Usage: :option:`flux submit -o NAME[=ARG]`.
+
+.. make :option: references in the included table x-ref to flux-shell(1)
+.. program:: flux shell
+.. include:: common/job-shell-options.rst
+.. program:: flux submit
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-run`, :man1:`flux-alloc`, :man1:`flux-batch`,
+:man1:`flux-bulksubmit`, :man7:`flux-environment`
diff --git a/doc/man1/flux-top.rst b/doc/man1/flux-top.rst
new file mode 100644
index 000000000000..d1c1b086edad
--- /dev/null
+++ b/doc/man1/flux-top.rst
@@ -0,0 +1,166 @@
+.. flux-help-description: display running Flux jobs
+.. flux-help-section: jobs
+
+===========
+flux-top(1)
+===========
+
+
+SYNOPSIS
+========
+
+**flux** **top** [*OPTIONS*] [*TARGET*]
+
+
+DESCRIPTION
+===========
+
+.. program:: flux top
+
+The :program:`flux top` command provides a dynamic view of Flux instance status
+and running jobs.  *TARGET*, if specified, selects a Flux instance other
+than the default, and may be either a native Flux URI or a high level URI,
+as described in :man1:`flux-uri`.
+
+The :program:`flux top` display window is divided into two parts:  the summary
+pane, and the job listing pane, which are described in detail below.
+
+
+OPTIONS
+=======
+
+.. option:: -h, --help
+
+   Summarize available options.
+
+.. option:: --color[=WHEN]
+
+   Colorize output.  The optional argument *WHEN* can be *auto*, *never*,
+   or *always*.  If *WHEN* is omitted, it defaults to *always*. The default
+   value when the :option:`--color` option is not used is *auto*.
+
+.. option:: -q, --queue=NAME
+
+   Limit status and jobs to specific queue.
+
+
+KEYS
+====
+
+:program:`flux top` responds to the following key presses:
+
+j, down-arrow
+   Move cursor down in the job listing.
+
+k, up-arrow
+   Move cursor up in the job listing.
+
+h/l, left-arrow/right-arrow
+   Rotate through all queues on the system.
+
+d
+   Toggle display of inactive job details (failed vs successful jobs).
+
+enter
+   Open the job at the current cursor position.  Only Flux instances (colored
+   blue in the job listing) owned by the user running :program:`flux top` may
+   be opened.  The display changes to show a new Flux instance, with its jobid
+   added to the path in the summary pane.  Nothing happens if the selected
+   job cannot be opened.
+
+q
+   Quit the current Flux instance, popping back to the previous Flux instance,
+   if any.  If the original Flux instance is being displayed, quit the program.
+
+control-l
+   Force a redraw of the :program:`flux top` window.
+
+
+SUMMARY PANE
+============
+
+The summary pane shows the following information:
+
+- The path of nested job ID's, if navigating the job hierarchy with the *up*,
+  *down*, *enter*, and *q* keys.
+
+- The amount of time until the job's expiration time, in Flux Standard Duration
+  format.  If the expiration time is unknown, the infinity symbol is
+  displayed (see `CAVEATS`_ below).
+
+- The nodes bargraph, which shows the fraction of used and down/excluded nodes
+  vs total nodes.  The graph of used nodes is colored yellow and extends from
+  left to right.  The graph of down/excluded nodes is red and extends from
+  right to left.
+
+- The cores bargraph, with the same layout as the nodes bargraph.
+
+- The gpus bargraph, with the same layout as the nodes bargraph.
+
+- The number of pending, running, and inactive jobs. When executed as the
+  instance owner, inactive jobs are split into completed (successful) and
+  failed (unsuccessful and canceled) jobs. This display can be toggled with
+  the ``d`` key.
+
+- A heart icon that appears each time the instance heartbeat event is
+  published.
+
+- The instance size.  This is the total number of brokers, which is usually
+  also the number of nodes.
+
+- The Flux instance depth, if greater than zero.  If the Flux instance was
+  not launched as a job within another Flux instance, the depth is zero.
+
+- The elapsed time the Flux instance has been running, in RFC 23 Flux Standard
+  Duration format.
+
+- The flux-core software version.
+
+
+JOB LIST PANE
+=============
+
+The job listing pane shows running jobs, that is, jobs in *RUN* (R) or
+*CLEANUP* (C) state.  Jobs that are Flux instances are shown in blue and
+may be selected for display, as described in the KEYS section.
+
+The columns of output are currently fixed, and use the same naming convention
+as :man1:`flux-jobs`.
+
+The newest jobs are shown at the top of the display.
+
+:program:`flux top` subscribes to job state update events, and tries to update
+its display within 2s of receiving new job information.
+
+
+CAVEATS
+=======
+
+:program:`flux top` employs a few UTF-8 characters to maximize cuteness.  If
+your heart emoji looks like a cartoon expletive, consult your system
+administrator.
+
+The infinity symbol in the expiration field does not really mean the Flux
+instance will run forever.  The field width of the timestamp portion of the
+Flux Locally Unique IDs (RFC 19) used for job IDs places an outer bound on
+a Flux instance's lifetime of about 35 years.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_19`
+
+:doc:`rfc:spec_23`
+
+
+SEE ALSO
+========
+
+:man1:`flux-resource`, :man1:`flux-uptime`, :man1:`flux-jobs`, :man1:`flux-uri`
diff --git a/doc/man1/flux-update.rst b/doc/man1/flux-update.rst
new file mode 100644
index 000000000000..bb6022aeb991
--- /dev/null
+++ b/doc/man1/flux-update.rst
@@ -0,0 +1,152 @@
+.. flux-help-section: jobs
+
+==============
+flux-update(1)
+==============
+
+SYNOPSIS
+========
+
+**flux** **update** [*OPTIONS*] JOBID KEY=VALUE [KEY=VALUE...]
+
+DESCRIPTION
+===========
+
+.. program:: flux update
+
+:program:`flux update` requests an update of one or more attributes for an
+active (pending or running) job. Updates are permitted and validated by the job
+manager before being applied to the job.
+
+Keys are expressed as period-delimited strings corresponding to an attribute
+path in jobspec. If a key does not start with ``attributes.``, ``tasks.``,
+or ``.resources`` (the top-level jobspec keys allowed by RFC 14), then
+the key is assumed to be prefixed with ``attributes.system.``, such that::
+
+  $ flux update f12345 myattr="value"
+
+would request an update of ``attributes.system.myattr`` to the string value
+``"value"``.
+
+The :program:`flux update` command may also support other convenient key
+aliases.  Key aliases are listed in the `SPECIAL KEYS`_ section below.
+
+Updates will be sent to the job manager update service, which checks that
+the current user is permitted to apply all updates, and that all updates
+are valid. If multiple updates are specified on the command line, then
+all updates are either applied or the request fails.
+
+.. note::
+   Job updates are allowed in the job manager by special plugins on
+   a case by case basis. At this time, the set of keys that can actually
+   be updated for a job may be very limited.
+
+The instance owner may be allowed to update specific attributes of jobs
+and bypass validation checks. For example, the duration of a guest job may
+be increased beyond currently configured limits if the update request is
+performed by the instance owner. When a job is modified in this way, future
+updates to the job by the guest user are denied with an error message::
+
+   job is immutable due to previous instance owner update
+
+This is necessary to prevent possible unintended bypass of limits or
+other checks on a job by a guest.
+
+The :program:`flux update` command may also support special handling of values
+for specific keys. Those special cases are documented in the SPECIAL KEYS
+section below.
+
+OPTIONS
+=======
+
+.. option:: -n, --dry-run
+
+  Do not send update to job manager, but print the updates in JSON to
+  stdout.
+
+.. option:: -v, --verbose
+
+  Print updated keys on success.
+
+SPECIAL KEYS
+============
+
+*attributes.system.duration*, *duration*
+  Updates of the job ``duration`` can take the form of of *[+-]FSD*, where
+  ``+`` or ``-`` indicate an adjustment of the existing duration, and *FSD*
+  is any string or number in RFC 23 Flux Standard Duration. Examples include
+  ``60``, ``1m``, ``1.5h``, ``+10m``, ``-1h``. Updates to the duration of
+  a running job may be allowed (instance owner only) and are handled as a
+  special case. For details see `DURATION UPDATE OF A RUNNING JOB`_ below.
+
+*attributes.system.queue*, *queue*
+  Updates of a pending job's ``queue`` to another enabled queue may
+  be allowed. The update could be rejected if the new job exceeds the
+  destination queue limits or if the job would not be feasible in the
+  new queue.
+
+*name*
+  Alias for job name, i.e. ``attributes.system.job.name``
+
+
+DURATION UPDATE OF A RUNNING JOB
+================================
+
+As a special case, an update of the duration of a running job may be allowed
+when performed by the instance owner and validated by the scheduler. When
+a running job's duration is updated, this triggers a ``resource-update``
+event in the main eventlog which contains the updated resource set
+(R) expiration time. This event then triggers the following actions to
+propagate the updated job expiration through the system:
+
+ - The updated expiration is forwarded to the job execution system which
+   modifies its expiration timer for the job.
+ - If the user enabled the job shell option to signal a job before the
+   time limit, then the job shell receives the updated R and resets this
+   timer.
+ - If the job is a subinstance of Flux (e.g. started with :man1:`flux-alloc`
+   or :man1:`flux-batch`), the instance resource module also receives
+   notification of the updated R, replaces its internal copy, writes
+   the update to ``resource.R`` in the KVS, and issues a ``resource-update``
+   event to the ``resource.eventlog``.
+ - The ``resource-update`` event in the eventlog results in a new
+   RFC 28 ``resource.acquire`` response to the scheduler containing the
+   updated expiration.
+ - The instance scheduler accepts the new response and updates its
+   internal resource set representation.
+ - The update service in the job-manager is notified of the R update,
+   and if the expiration has been increased, it walks the list of running
+   jobs and issues a ``resource-update`` event for any job which had
+   its expiration set based on the previous instance expiration.
+ - For any job that has its expiration extended due to an instance expiration
+   update, all these steps are repeated.
+
+EXIT STATUS
+===========
+
+0
+  All updates were successful
+
+1
+  Updates were invalid or not permitted, or the provided JOBID was invalid
+  or inactive, or the user does not have permission to modify the job
+
+2
+  Syntax or other command line error
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_14`
+
+
+SEE ALSO
+========
+
+:man1:`flux-jobs`, :man1:`flux-submit`, :man1:`flux-bulksubmit`
diff --git a/doc/man1/flux-uptime.rst b/doc/man1/flux-uptime.rst
new file mode 100644
index 000000000000..d558fdf5ea5b
--- /dev/null
+++ b/doc/man1/flux-uptime.rst
@@ -0,0 +1,95 @@
+.. flux-help-description: Tell how long Flux has been up and running
+.. flux-help-section: instance
+
+==============
+flux-uptime(1)
+==============
+
+
+SYNOPSIS
+========
+
+**flux** **uptime**
+
+
+DESCRIPTION
+===========
+
+The :program:`flux uptime` command displays the following information about the
+current Flux instance, on one or two lines:
+
+- The current wall clock time.
+
+- The broker state.  See BROKER STATES.
+
+- The elapsed time the Flux instance has been running, in RFC 23 Flux Standard
+  Duration format.  If the local broker is not in **run** state, the elapsed
+  time in the current state is reported instead.
+
+- The Flux instance owner.  On a system instance, this is probably the
+  ``flux`` user.
+
+- The Flux instance depth.  If the Flux instance was not launched as a job
+  within another Flux instance, the depth is zero.
+
+- The instance size.  This is the total number of brokers, which is usually
+  also the number of nodes.
+
+- The number of drained nodes, if greater than zero.  Drained nodes are
+  not eligible to run new jobs, although they may be online, and may currently
+  be running a job.
+
+- The number of offline nodes, if greater than zero.  A node is offline if
+  its broker is not connected to the instance overlay network.
+
+- A notice if job submission is disabled on all queues.
+
+- A notice if scheduling is disabled.
+
+
+BROKER STATES
+=============
+
+join
+   The local broker is trying to connect to its overlay network parent,
+   or is waiting for the parent to complete initialization and reach
+   **quorum** state.
+
+init
+   The local broker is waiting for the ``rc1`` script to complete locally.
+
+quorum
+   All brokers are waiting for a configured number of brokers to reach
+   **quorum** state.  The default quorum is the instance size.  A Flux
+   system instance typically defines the quorum size to 1.
+
+run
+   Flux is fully up and running.
+
+cleanup
+   Cleanup scripts are executing.  This state appears on the rank 0 broker only.
+
+shutdown
+   The local broker is waiting for its overlay network children to finalize
+   and disconnect.
+
+finalize
+   The local broker is waiting for the ``rc3`` script to complete locally.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_23`
+
+
+SEE ALSO
+========
+
+:man1:`flux-resource`, :man1:`flux-getattr`, :man7:`flux-broker-attributes`
diff --git a/doc/man1/flux-uri.rst b/doc/man1/flux-uri.rst
new file mode 100644
index 000000000000..c12dd9854b08
--- /dev/null
+++ b/doc/man1/flux-uri.rst
@@ -0,0 +1,180 @@
+===========
+flux-uri(1)
+===========
+
+
+SYNOPSIS
+========
+
+**flux** *uri* [OPTIONS] *TARGET*
+
+DESCRIPTION
+===========
+
+.. program:: flux uri
+
+Connections to Flux are established via a Uniform Resource Identifier
+(URI) which is passed to the :man3:`flux_open` API call. These *native*
+URIs indicate the "connector" which will be used to establish the
+connection, and are typically either *local*, with a  ``local`` URI
+scheme, or *remote*, with a ``ssh`` URI scheme. These URIs are considered
+fully-resolved, native Flux URIs.
+
+Processes running within a Flux instance will have the :envvar:`FLUX_URI`
+environment variable set to a native URI which :man3:`flux_open` will
+use by default, with fallback to a compiled-in native URI for the system
+instance of Flux. Therefore, there is usually no need to specify a URI when
+connecting to the enclosing instance. However, connecting to a *different*
+Flux instance will require discovery of the fully-resolved URI for that
+instance.
+
+:program:`flux uri` attempts to resolve its *TARGET* argument to a native local
+or remote URI. The *TARGET* is itself a URI which specifies the method
+to use in URI resolution via the scheme part, with the path and query
+parts passed to a plugin which implements the resolution method.
+
+As a convenience, if *TARGET* is specified with no scheme, then the scheme
+is assumed to be ``jobid``.  This allows :program:`flux uri` to be used to look
+up the URI for a Flux instance running as a job in the current enclosing
+instance with:
+
+::
+
+   $ flux uri JOBID
+
+Depending on the *TARGET* URI scheme and corresponding plugin, specific
+query arguments in *TARGET* may be supported. However, as a convenience,
+all *TARGET* URIs support the special query arguments ``local`` or
+``remote`` to force the resulting URI into a local (``local://``) or remote
+(``ssh://``) form. For example:
+
+::
+
+   $ flux uri JOBID?local
+
+would return the ``local://`` URI for *JOBID* (if the URI can be resolved).
+
+A special environment variable :envvar:`FLUX_URI_RESOLVE_LOCAL` will force
+:program:`flux uri` to always resolve URIs to local form.  This is often useful
+if the local connector is known to be on the local system (i.e. within a test
+Flux instance), and ssh to localhost does not work.
+
+A list of supported URI schemes will be listed at the bottom of
+:option:`flux uri --help` message. For a description of the URI resolver
+schemes included with Flux, see the URI SCHEMES and EXAMPLES sections below.
+
+OPTIONS
+=======
+
+.. option:: --remote
+
+   Return the *remote* (``ssh://``)  equivalent of the resolved URI.
+
+.. option:: --local
+
+   Return the *local* (``local://``) equivalent of the resolved URI.
+   Warning: the resulting URI may be invalid for the current system
+   if the network host specified by an :program:`ssh` URI is not the current
+   host.
+
+.. option:: --wait
+
+   Wait for the URI to become available if the resolver scheme supports it.
+   This is the same as specifying :command:`flux uri TARGET?wait`. Currently
+   only supported by the ``jobid`` resolver. It will be ignored by other
+   schemes.
+
+URI SCHEMES
+===========
+
+The following URI schemes are included by default:
+
+jobid:ID[/ID...]
+   This scheme attempts to get the URI for a Flux instance running as a
+   job in the current enclosing instance. This is the assumed scheme if no
+   ``scheme:`` is provided in *TARGET* passed to :program:`flux uri`, so the
+   ``jobid:`` prefix is optional. A hierarchy of Flux jobids is supported,
+   so ``f1234/f3456`` will resolve the URI for job ``f3456`` running in
+   job ``f1234`` in the current instance. This scheme will raise an error
+   if the target job is not running.
+
+   The ``jobid`` scheme supports the optional query parameter ``?wait``, which
+   causes the resolver to wait until a URI has been posted to the job eventlog
+   for the target jobs(s), if the job is in RUN state or any previous state.
+   Note that the resolver could wait until the job is inactive if the ``?wait``
+   query parameter is used on a job that is not an instance of Flux.
+
+pid:PID
+  This scheme attempts to read the :envvar:`FLUX_URI` value from the
+  process id *PID* using ``/proc/PID/environ``. If *PID* refers to a
+  :program:`flux-broker`, then the scheme reads :envvar:`FLUX_URI` from the
+  broker's initial program or another child process since :envvar:`FLUX_URI`
+  in the broker's environment would refer to *its* parent (or may not be
+  set at all in the case of a test instance started with :option:`flux
+  start --test-size=N`).
+
+slurm:JOBID
+  This scheme makes a best-effort to resolve the URI of a Flux instance
+  launched under Slurm. It invokes :program:`srun` to run :program:`scontrol
+  listpids` on the first node of the job, and then uses the ``pid``
+  resolver until it finds a valid :envvar:`FLUX_URI`.
+
+
+EXAMPLES
+========
+
+To get the URI of a job in the current instance in its ``local://`` form:
+
+::
+
+   $ flux uri --local ƒN8Pz2xVu
+   local:///tmp/flux-zbVtVg/jobtmp-0-ƒN8Pz2xVu/flux-59uf5w/local-0
+
+or
+
+::
+
+   $ flux uri ƒN8Pz2xVu?local
+   local:///tmp/flux-zbVtVg/jobtmp-0-ƒN8Pz2xVu/flux-59uf5w/local-0
+
+
+Get the URI of a nested job:
+
+::
+
+   $ flux uri ƒqxxTiZBM/ƒr2XFWP?local
+   local:///tmp/flux-zbVtVg/jobtmp-0-ƒqxxTiZBM/flux-EPgSwk/local-0
+
+.. note::
+   With  the ``jobid`` resolver, ``?local`` only needs to be placed on
+   the last component of the jobid "path" or hierarchy. This will resolve
+   each URI in turn as a local URI.
+
+Get the URI of a local flux-broker
+
+::
+
+   $ flux uri pid:$(pidof -s flux-broker)
+   local:///tmp/flux-sLuBkZ/local-0
+
+The following submits a batch job and returns the instance URI once it is
+available. Without the :option:`--wait` flag, :command:`flux uri` would fail
+immediately with "job is not running":
+
+::
+
+   $ flux uri --wait $(flux batch -n1 --wrap flux run myprogram)
+
+
+Get the URI for a Flux instance running as a Slurm job:
+
+::
+
+   $ flux uri slurm:7843494
+   ssh://cluster42/var/tmp/user/flux-MpnytT/local-0
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man1/flux-user.adoc b/doc/man1/flux-user.adoc
deleted file mode 100644
index f7fa32253e38..000000000000
--- a/doc/man1/flux-user.adoc
+++ /dev/null
@@ -1,76 +0,0 @@
-// flux-help-include: true
-FLUX-USER(1)
-============
-:doctype: manpage
-
-
-NAME
-----
-flux-user - Flux user database client
-
-
-SYNOPSIS
---------
-*flux* *user* 'COMMAND' ['OPTIONS']
-
-
-DESCRIPTION
------------
-The Flux user database is used to authorize users to access the
-Flux instance with assigned roles, as described in Flux RFC 12.
-
-This command line client can be used to manipulate the database.
-
-If the 'userdb' module is loaded with the '--default-rolemask'
-option, then users are added to the user database when they successfully
-authenticate to a Flux connector.  Otherwise, users that are not
-already present in the database are denied access.
-
-COMMANDS
---------
-*list*::
-List the contents of the user database.  Each entry is listed with
-fields separated by colons.  The first field is the 32-bit userid
-in decimal.  The second field is the 32-bit rolemask, represented as
-a comma-delimited list of strings.
-
-*lookup* 'USERID'::
-Look up 'USERID' in the user database and display its entry,
-as described above.  If any part of 'USERID' is non-numeric,
-an attempt is made to look up 'USERID' as a name in the password file.
-If successful, the resulting UID is used in the query.
-
-*addrole* 'USERID' 'ROLEMASK'::
-Request that the database add the roles in 'ROLEMASK' to the existing
-roles held by 'USERID'.  If 'USERID' has no entry in the database,
-one is added.  If any part of 'USERID' is non-numeric, an attempt is
-made to look up 'USERID' as a name in the password file and use the UID
-from the result. 'ROLEMASK' may be numeric (any base - use strtoul base
-prefix rules), or may be a comma-separated list of roles, e.g.
-"owner", "user", etc..
-
-*delrole* 'USERID' 'ROLEMASK'::
-Request that the database remove the roles in 'ROLEMASK' from the existing
-roles held by 'USERID'.  If after removing those roles, 'USERID' has no
-roles, the entry is removed from the database.  'USERID' and 'ROLEMASK'
-may be specified as described above.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-https://github.com/flux-framework/rfc/blob/master/spec_12.adoc[RFC 12: Flux Security Architecture]
-
diff --git a/doc/man1/flux-version.adoc b/doc/man1/flux-version.adoc
deleted file mode 100644
index d68e1e268760..000000000000
--- a/doc/man1/flux-version.adoc
+++ /dev/null
@@ -1,39 +0,0 @@
-// flux-help-description : Display flux version information
-FLUX-VERSION(1)
-===============
-:doctype: manpage
-
-
-NAME
-----
-flux-version - Display flux version information
-
-
-SYNOPSIS
---------
-*flux* *version*
-
-
-DESCRIPTION
------------
-flux-version(1) prints version information for flux components.
-At a minimum, the version of flux commands and the currently linked
-libflux-core.so library is displayed. If running within an instance,
-the version of the flux-broker found and FLUX_URI are also included.
-Finally, if flux is compiled against flux-security, then the version
-of the currently linked libflux-security is included.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
diff --git a/doc/man1/flux-version.rst b/doc/man1/flux-version.rst
new file mode 100644
index 000000000000..ec5a5372362d
--- /dev/null
+++ b/doc/man1/flux-version.rst
@@ -0,0 +1,28 @@
+.. flux-help-description : Display flux version information
+
+===============
+flux-version(1)
+===============
+
+
+SYNOPSIS
+========
+
+**flux** **version**
+
+
+DESCRIPTION
+===========
+
+:program:`flux version` prints version information for flux components.
+At a minimum, the version of flux commands and the currently linked
+libflux-core.so library is displayed. If running within an instance,
+the version of the flux-broker found and :envvar:`FLUX_URI` are also included.
+Finally, if flux is compiled against flux-security, then the version
+of the currently linked libflux-security is included.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man1/flux-watch.rst b/doc/man1/flux-watch.rst
new file mode 100644
index 000000000000..e1283024c623
--- /dev/null
+++ b/doc/man1/flux-watch.rst
@@ -0,0 +1,101 @@
+.. flux-help-section: jobs
+
+=============
+flux-watch(1)
+=============
+
+
+SYNOPSIS
+========
+
+**flux** **watch** [*OPTIONS*] [JOBID ...]
+
+DESCRIPTION
+===========
+
+.. program:: flux watch
+
+The :program:`flux watch` command is used to monitor the output and state of
+one or more Flux jobs. The command works similarly to the
+:option:`flux submit --watch` option, but can be used to monitor even inactive
+jobs. For example, to copy all job output to the terminal after submitting a
+series of jobs with :man1:`flux-submit` or :man1:`flux-bulksubmit`, use
+
+::
+
+  flux watch --all
+
+This command can also be used at the end of a batch script to wait for all
+submitted jobs to complete and copy all output to the same location as the
+batch job.
+
+OPTIONS
+=======
+
+.. option:: -a, --active
+
+   Watch all active jobs.
+   This is equivalent to  *--filter=pending,running*.
+
+.. option:: -A, --all
+
+   Watch all jobs. This is equivalent to *--filter=pending,running,inactive*.
+
+.. option:: -c, --count=N
+
+   Limit output to N jobs (default 1000). This is a safety measure to
+   protect against watching too many jobs with the :option:`--all` option. The
+   limit can be disabled with :option:`--count=0`.
+
+.. option:: --since=WHEN
+
+   Limit output to jobs that have been active since a given timestamp.
+   This option implies :option:`-a` if no other :option:`--filter` options are
+   specified.  If *WHEN* begins with ``-`` character, then the remainder is
+   considered to be a an offset in Flux standard duration (RFC 23). Otherwise,
+   any datetime expression accepted by the Python `parsedatetime
+   <https://github.com/bear/parsedatetime>`_ module is accepted. Examples:
+   "-6h", "-1d", "yesterday", "2021-06-21 6am", "last Monday", etc. It is
+   assumed to be an error if a timestamp in the future is supplied.
+
+.. option:: -f, --filter=STATE|RESULT
+
+   Watch jobs with specific job state or result. Multiple states or results
+   can be listed separated by comma. See the JOB STATUS section in the
+   :man1:`flux-jobs` manual for additional information.
+
+.. option:: --progress
+
+   Display a progress bar showing the completion progress of monitored
+   jobs.  Jobs that are already inactive will immediately have their
+   progress updated in the progress bar, with output later copied to the
+   terminal. The progress bar by default includes a count of pending,
+   running, complete and failed jobs, and an elapsed timer. The elapsed
+   timer is initialized at the submit time of the earliest job, or the
+   starttime of the instance with :option:`--all`, in order to reflect the real
+   elapsed time for the jobs being monitored.
+
+.. option:: --jps
+
+   With :option:`--progress`, display throughput statistics (job/s) in the
+   progress bar instead of an elapsed timer. Note: The throughput will be
+   calculated based on the elapsed time as described in the description
+   of the :option:`-progress` option.
+
+EXIT STATUS
+===========
+
+The exit status of :program:`flux watch` is 0 if no jobs match the job
+selection options or if all jobs complete with success. Otherwise, the command
+exits with the largest exit status of all monitored jobs, or 2 if there is an
+error during option processing.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+SEE ALSO
+========
+
+:man1:`flux-jobs`, :man1:`flux-submit`, :man1:`flux-bulksubmit`
diff --git a/doc/man1/flux.adoc b/doc/man1/flux.adoc
deleted file mode 100644
index 580ca470c981..000000000000
--- a/doc/man1/flux.adoc
+++ /dev/null
@@ -1,102 +0,0 @@
-FLUX(1)
-=======
-:doctype: manpage
-
-
-NAME
-----
-flux - the Flux resource management framework
-
-
-SYNOPSIS
---------
-*flux* ['OPTIONS'] 'CMD' ['CMD-OPTIONS']
-
-
-DESCRIPTION
------------
-Flux is a modular framework for resource management.
-
-flux(1) is a front end for Flux sub-commands.
-"flux -h" summarizes the core Flux commands.
-"flux help 'CMD'" displays the manual page for 'CMD'.
-
-If 'CMD' contains a slash "/" character, it is executed directly,
-bypassing the sub-command search path.
-
-
-OPTIONS
--------
-*-h, --help*::
-Display help on options, and a list of the core Flux sub-commands.
-
-*-p, --parent*::
-If current instance is a child, connect to parent instead. Also sets
-'FLUX_KVS_NAMEPSACE' if current instance is confined to a KVS namespace
-in the parent. This option may be specified multiple times.
-
-*-v, --verbose*::
-Display command environment, and the path search for 'CMD'.
-
-*-V, --version*::
-Convenience option to run flux-version(1).
-
-
-SUB-COMMAND ENVIRONMENT
------------------------
-flux(1) uses compiled-in install paths and its environment
-to construct the environment for sub-commands.
-
-Sub-command search path::
-Look for "flux-'CMD'" executable by searching a path constructed
-with the following prototype:
-
-  [getenv FLUX_EXEC_PATH_PREPEND]:install-path:\
-    [getenv FLUX_EXEC_PATH]
-
-setenv FLUX_MODULE_PATH::
-Set up broker comms module search path according to:
-
-  [getenv FLUX_MODULE_PATH_PREPEND]:install-path:\
-    [getenv FLUX_MODULE_PATH]
-
-setenv FLUX_CONNECTOR_PATH::
-Set up search path for connector modules used by libflux to open a connection
-to the broker
-
-  [getenv FLUX_CONNECTOR_PATH_PREPEND]:install-path:\
-    [getenv FLUX_CONNECTOR_PATH]
-
-setenv FLUX_SEC_DIRECTORY::
-Set directory for Flux CURVE keys.  This is not a search path.
-If unset, flux(1) sets it to $HOME/flux.
-
-setenv LUA_PATH::
-Set Lua module search path:
-
-  [getenv FLUX_LUA_PATH_PREPEND];[getenv LUA_PATH];install-path;;;
-
-setenv LUA_CPATH::
-Set Lua binary module search path:
-
-  [getenv FLUX_LUA_CPATH_PREPEND];[getenv LUA_CPATH];install-path;;;
-
-setenv PYTHONPATH::
-Set Python module search path:
-
-  [getenv FLUX_PYTHONPATH_PREPEND]:[getenv PYTHONPATH];install-path
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
diff --git a/doc/man1/flux.rst b/doc/man1/flux.rst
new file mode 100644
index 000000000000..dd6fd831e168
--- /dev/null
+++ b/doc/man1/flux.rst
@@ -0,0 +1,86 @@
+=======
+flux(1)
+=======
+
+
+SYNOPSIS
+========
+
+**flux** [*OPTIONS*] *CMD* [*CMD-OPTIONS*]
+
+
+DESCRIPTION
+===========
+
+.. program:: flux
+
+Flux is a modular framework for resource management.
+
+:program:`flux` is a front end for Flux sub-commands.
+:option:`flux -h` summarizes the core Flux commands.
+:program:`flux help CMD` displays the manual page for *CMD*.
+
+If *CMD* contains a slash "/" character, it is executed directly,
+bypassing the sub-command search path.
+
+
+OPTIONS
+=======
+
+.. option:: -h, --help
+
+   Display help on options, and a list of the core Flux sub-commands.
+
+.. option:: -p, --parent
+
+   If current instance is a child, connect to parent instead. Also sets
+   :envvar:`FLUX_KVS_NAMESPACE` if current instance is confined to a KVS
+   namespace in the parent. This option may be specified multiple times.
+
+.. option:: -v, --verbose
+
+   Display command environment, and the path search for *CMD*.
+
+.. option:: -V, --version
+
+   Convenience option to run :man1:`flux-version`.
+
+
+SUB-COMMAND ENVIRONMENT
+=======================
+
+:program:`flux` uses compiled-in install paths and its environment
+to construct the environment for sub-commands.  More detail is available in the
+:man7:`flux-environment` :ref:`sub_command_environment` section.  A summary
+is provided below:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Name
+     - Description
+
+   * - :envvar:`FLUX_EXEC_PATH`
+     - where to look for "flux-*CMD*" executables
+
+   * - :envvar:`FLUX_MODULE_PATH`
+     - directories to look for broker modules
+
+   * - :envvar:`FLUX_CONNECTOR_PATH`
+     - directories to search for connector modules
+
+   * - :envvar:`LUA_PATH`
+     - Lua module search path
+
+   * - :envvar:`LUA_CPATH`
+     - Lua binary module search path
+
+   * - :envvar:`PYTHONPATH`
+     - Python module search path:
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
diff --git a/doc/man1/index.rst b/doc/man1/index.rst
new file mode 100644
index 000000000000..f2f7986077af
--- /dev/null
+++ b/doc/man1/index.rst
@@ -0,0 +1,10 @@
+.. _man-commands:
+
+Section 1 - Flux Commands
+=========================
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   *
diff --git a/doc/man3/COPYRIGHT.adoc b/doc/man3/COPYRIGHT.adoc
deleted file mode 100644
index 6b2304f95801..000000000000
--- a/doc/man3/COPYRIGHT.adoc
+++ /dev/null
@@ -1,4 +0,0 @@
-Copyright 2014 Lawrence Livermore National Security, LLC
-and Flux developers.
-
-SPDX-License-Identifier: LGPL-3.0
diff --git a/doc/man3/JSON_PACK.adoc b/doc/man3/JSON_PACK.adoc
deleted file mode 100644
index 2860e3bccd2d..000000000000
--- a/doc/man3/JSON_PACK.adoc
+++ /dev/null
@@ -1,73 +0,0 @@
-ENCODING JSON PAYLOADS
-----------------------
-
-Flux API functions that are based on Jansson's json_pack()
-accept the following tokens in their format string.
-The type in parenthesis denotes the resulting JSON type, and
-the type in brackets (if any) denotes the C type that is expected as
-the corresponding argument or arguments.
-
-*s* (string)['const char *']::
-Convert a null terminated UTF-8 string to a JSON string.
-
-*s#* (string)['const char *', 'int']::
-Convert a UTF-8 buffer of a given length to a JSON string.
-
-*s%* (string)['const char *', 'size_t']::
-Like *s#* but the length argument is of type size_t.
-
-*+* ['const char *']::
-Like *s*, but concatenate to the previous string.
-Only valid after a string.
-
-*+#* ['const char *', 'int']::
-Like *s#*, but concatenate to the previous string.
-Only valid after a string.
-
-*+%* ['const char *', 'size_t']::
-Like *+#*, but the length argument is of type size_t.
-
-*n* (null)::
-Output a JSON null value.  No argument is consumed.
-
-*b* (boolean)['int']::
-Convert a C int to JSON boolean value.  Zero is converted to
-_false_ and non-zero to _true_.
-
-*i* (integer)['int']::
-Convert a C int to JSON integer.
-
-*I* (integer)['int64_t']::
-Convert a C int64_t to JSON integer.
-Note: Jansson expects a json_int_t here without committing to a size,
-but Flux guarantees that this is a 64-bit integer.
-
-*f* (real)['double']::
-Convert a C double to JSON real.
-
-*o* (any value)['json_t *']::
-Output any given JSON value as-is.  If the value is added to an array
-or object, the reference to the value passed to *o* is stolen by the
-container.
-
-*O* (any value)['json_t *']::
-Like *o*, but the argument's reference count is incremented.  This
-is useful if you pack into an array or object and want to keep the reference
-for the JSON value consumed by *O* to yourself.
-
-*[fmt]* (array)::
-Build an array with contents from the inner format string.  *fmt* may
-contain objects and arrays, i.e. recursive value building is supported.
-
-*\{fmt\}* (object)::
-Build an object with contents from the inner format string *fmt*.
-The first, third, etc. format specifier represent a key, and must be a
-string as object keys are always strings.  The second, fourth, etc.
-format specifier represent a value.  Any value may be an object or array,
-i.e. recursive value building is supported.
-
-Whitespace, *:* (colon) and *,* (comma) are ignored.
-
-These descriptions came from the Jansson 2.6 manual.
-
-See also: http://jansson.readthedocs.io/en/2.6/apiref.html#building-values[Jansson API: Building Values]
diff --git a/doc/man3/JSON_UNPACK.adoc b/doc/man3/JSON_UNPACK.adoc
deleted file mode 100644
index b6b9fe54fd4e..000000000000
--- a/doc/man3/JSON_UNPACK.adoc
+++ /dev/null
@@ -1,70 +0,0 @@
-DECODING JSON PAYLOADS
-----------------------
-
-Flux API functions that are based on Jansson's json_unpack()
-accept the following tokens in their format string.
-The type in parenthesis denotes the resulting JSON type, and
-the type in brackets (if any) denotes the C type that is expected as
-the corresponding argument or arguments.
-
-*s* (string)['const char *']::
-Convert a JSON string to a pointer to a null terminated UTF-8 string.
-The resulting string is extracted by using 'json_string_value()'
-internally, so it exists as long as there are still references to the
-corresponding JSON string.
-
-*n* (null)::
-Expect a JSON null value.  Nothing is extracted.
-
-*b* (boolean)['int']::
-Convert a JSON boolean value to a C int, so that _true_ is converted to 1
-and _false_ to 0.
-
-*i* (integer)['int']::
-Convert a JSON integer to a C int.
-
-*I* (integer)['int64_t']::
-Convert a JSON integer to a C int64_t.
-Note: Jansson expects a json_int_t here without committing to a size,
-but Flux guarantees that this is a 64-bit integer.
-
-*f* (real)['double']::
-Convert JSON real to a C double.
-
-*F* (real)['double']::
-Convert JSON number (integer or real) to a C double.
-
-*o* (any value)['json_t *']::
-Store a JSON value, with no conversion, to a json_t pointer.
-
-*O* (any value)['json_t *']::
-Like *o*, but the JSON value's reference count is incremented.
-
-*[fmt]* (array)::
-Convert each item in the JSON array according to the inner format
-string.  *fmt* may contain objects and arrays, i.e. recursive value
-extraction is supported.
-
-*\{fmt\}* (object)::
-Convert each item in the JSON object according to the inner format
-string *fmt*.    The first, third, etc. format specifier represent a
-key, and must by *s*.  The corresponding argument to unpack functions
-is read as the object key.  The second, fourth, etc. format specifier
-represent a value and is written to the address given as the corresponding
-argument.  Note that every other argument is read from and every other
-is written to.  *fmt* may contain objects and arrays as values, i.e.
-recursive value extraction is supported.  Any *s* representing a key
-may be suffixed with *?* to make the key optional.  If the key is not
-found, nothing is extracted.
-
-*!*::
-This special format specifier is used to enable the check that all
-object and array items are accessed, on a per-value basis. It must
-appear inside an array or object as the last format specifier before
-the closing bracket or brace.
-
-Whitespace, *:* (colon) and *,* (comma) are ignored.
-
-These descriptions came from the Jansson 2.6 manual.
-
-See also:  http://jansson.readthedocs.io/en/2.6/apiref.html#parsing-and-validating-values[Jansson API: Parsing and Validating Values]
diff --git a/doc/man3/Makefile.am b/doc/man3/Makefile.am
deleted file mode 100644
index 042a85a8e921..000000000000
--- a/doc/man3/Makefile.am
+++ /dev/null
@@ -1,356 +0,0 @@
-MAN3_FILES = $(MAN3_FILES_PRIMARY) $(MAN3_FILES_SECONDARY)
-
-MAN3_FILES_PRIMARY = \
-	flux_open.3 \
-	flux_send.3 \
-	flux_recv.3 \
-	flux_requeue.3 \
-	flux_aux_set.3 \
-	flux_flags_set.3 \
-	flux_fatal_set.3 \
-	flux_event_subscribe.3 \
-	flux_event_publish.3 \
-	flux_pollevents.3 \
-	flux_msg_encode.3 \
-	flux_rpc.3 \
-	flux_get_rank.3 \
-	flux_attr_get.3 \
-	flux_get_reactor.3 \
-	flux_reactor_create.3 \
-	flux_fd_watcher_create.3 \
-	flux_watcher_start.3 \
-	flux_zmq_watcher_create.3 \
-	flux_handle_watcher_create.3 \
-	flux_timer_watcher_create.3 \
-	flux_periodic_watcher_create.3 \
-	flux_idle_watcher_create.3 \
-	flux_msg_handler_create.3 \
-	flux_msg_cmp.3 \
-	flux_msg_handler_addvec.3 \
-	flux_child_watcher_create.3 \
-	flux_signal_watcher_create.3 \
-	flux_stat_watcher_create.3 \
-	flux_respond.3 \
-	flux_reactor_now.3 \
-	flux_request_decode.3 \
-	flux_event_decode.3 \
-	flux_response_decode.3 \
-	flux_request_encode.3 \
-	flux_content_load.3 \
-	flux_log.3 \
-	flux_future_get.3 \
-	flux_future_create.3 \
-	flux_future_wait_all_create.3 \
-	flux_future_and_then.3 \
-	flux_kvs_lookup.3 \
-	flux_kvs_commit.3 \
-	flux_kvs_txn_create.3 \
-	flux_kvs_namespace_create.3 \
-	flux_kvs_getroot.3 \
-	flux_kvs_copy.3 \
-	flux_core_version.3 \
-	idset_create.3
-
-# These files are generated as roff .so includes of a primary page.
-# A2X handles this automatically if mentioned in NAME section
-MAN3_FILES_SECONDARY = \
-	flux_clone.3 \
-	flux_close.3 \
-	flux_requeue_nocopy.3 \
-	flux_aux_get.3 \
-	flux_flags_unset.3 \
-	flux_flags_get.3 \
-	flux_fatal_error.3 \
-	FLUX_FATAL.3 \
-	flux_event_unsubscribe.3 \
-	flux_event_publish_pack.3 \
-	flux_event_publish_raw.3 \
-	flux_event_publish_get_seq.3 \
-	flux_pollfd.3 \
-	flux_msg_decode.3 \
-	flux_get_size.3 \
-	flux_attr_set.3 \
-	flux_set_reactor.3 \
-	flux_reactor_destroy.3 \
-	flux_reactor_run.3 \
-	flux_reactor_stop.3 \
-	flux_reactor_stop_error.3 \
-	flux_reactor_active_incref.3 \
-	flux_reactor_active_decref.3 \
-	flux_fd_watcher_get_fd.3 \
-	flux_watcher_stop.3 \
-	flux_watcher_destroy.3 \
-	flux_watcher_next_wakeup.3 \
-	flux_zmq_watcher_get_zsock.3 \
-	flux_handle_watcher_get_flux.3 \
-	flux_timer_watcher_reset.3 \
-	flux_periodic_watcher_reset.3 \
-	flux_prepare_watcher_create.3 \
-	flux_check_watcher_create.3 \
-	flux_msg_handler_destroy.3 \
-	flux_msg_handler_start.3 \
-	flux_msg_handler_stop.3 \
-	flux_msg_handler_delvec.3 \
-	flux_child_watcher_get_rpid.3 \
-	flux_child_watcher_get_rstatus.3 \
-	flux_signal_watcher_get_signum.3 \
-	flux_stat_watcher_get_rstat.3 \
-	flux_respond_raw.3 \
-	flux_respond_pack.3 \
-	flux_respond_error.3 \
-	flux_reactor_now_update.3 \
-	flux_request_unpack.3 \
-	flux_request_decode_raw.3 \
-	flux_event_unpack.3 \
-	flux_event_encode.3 \
-	flux_event_pack.3 \
-	flux_event_encode_raw.3 \
-	flux_event_decode_raw.3 \
-	flux_response_decode_raw.3 \
-	flux_response_decode_error.3 \
-	flux_request_encode_raw.3 \
-	flux_content_load_get.3 \
-	flux_content_store.3 \
-	flux_content_store_get.3 \
-	flux_vlog.3 \
-	flux_log_set_appname.3 \
-	flux_log_set_procid.3 \
-	flux_future_then.3 \
-	flux_future_wait_for.3 \
-	flux_future_reset.3 \
-	flux_future_destroy.3 \
-	flux_future_get.3 \
-	flux_future_fulfill.3 \
-	flux_future_fulfill_error.3 \
-	flux_future_fulfill_with.3 \
-	flux_future_aux_set.3 \
-	flux_future_aux_get.3 \
-	flux_future_set_flux.3 \
-	flux_future_get_flux.3 \
-	flux_future_wait_all_create.3 \
-	flux_future_wait_any_create.3 \
-	flux_future_push.3 \
-	flux_future_first_child.3 \
-	flux_future_next_child.3 \
-	flux_future_get_child.3 \
-	flux_future_and_then.3 \
-	flux_future_or_then.3 \
-	flux_future_continue.3 \
-	flux_future_continue_error.3 \
-	flux_rpc_pack.3 \
-	flux_rpc_raw.3 \
-	flux_rpc_message.3 \
-	flux_rpc_get.3 \
-	flux_rpc_get_unpack.3 \
-	flux_rpc_get_raw.3 \
-	flux_kvs_lookupat.3 \
-	flux_kvs_lookup_get.3 \
-	flux_kvs_lookup_get_unpack.3 \
-	flux_kvs_lookup_get_raw.3 \
-	flux_kvs_lookup_get_dir.3 \
-	flux_kvs_lookup_get_treeobj.3 \
-	flux_kvs_lookup_get_symlink.3 \
-	flux_kvs_getroot_get_treeobj.3 \
-	flux_kvs_getroot_get_blobref.3 \
-	flux_kvs_getroot_get_sequence.3 \
-	flux_kvs_getroot_get_owner.3 \
-	flux_kvs_getroot_cancel.3 \
-	flux_kvs_fence.3 \
-	flux_kvs_commit_get_treeobj.3 \
-	flux_kvs_commit_get_sequence.3 \
-	flux_kvs_txn_destroy.3 \
-	flux_kvs_txn_put.3 \
-	flux_kvs_txn_pack.3 \
-	flux_kvs_txn_vpack.3 \
-	flux_kvs_txn_mkdir.3 \
-	flux_kvs_txn_unlink.3 \
-	flux_kvs_txn_symlink.3 \
-	flux_kvs_txn_put_raw.3 \
-	flux_kvs_namespace_remove.3 \
-	flux_kvs_move.3 \
-	flux_core_version_string.3 \
-	idset_destroy.3 \
-	idset_encode.3 \
-	idset_decode.3 \
-	idset_set.3 \
-	idset_clear.3 \
-	idset_first.3 \
-	idset_next.3 \
-	idset_count.3
-
-ADOC_FILES  = $(MAN3_FILES_PRIMARY:%.3=%.adoc)
-XML_FILES   = $(MAN3_FILES_PRIMARY:%.3=%.xml)
-
-if ENABLE_DOCS
-dist_man_MANS = $(MAN3_FILES)
-$(MAN3_FILES): COPYRIGHT.adoc JSON_PACK.adoc JSON_UNPACK.adoc
-endif
-
-SUFFIXES = .adoc .3
-
-STDERR_DEVNULL = $(stderr_devnull_$(V))
-stderr_devnull_ =  $(stderr_devnull_$(AM_DEFAULT_VERBOSITY))
-stderr_devnull_0 = 2>/dev/null
-
-.adoc.3:
-	$(AM_V_GEN)$(ADOC) --attribute mansource=$(PACKAGE_NAME) \
-	    --attribute manversion=$(PACKAGE_VERSION) \
-	    --attribute manmanual="Flux Programming Reference" \
-	    --destination-dir $(builddir) \
-	    --doctype manpage $(ADOC_FORMAT_OPT) manpage $< $(STDERR_DEVNULL)
-
-flux_clone.3: flux_open.3
-flux_close.3: flux_open.3
-flux_requeue_nocopy.3: flux_requeue.3
-flux_aux_get.3: flux_aux_set.3
-flux_flags_unset.3: flux_flags_set.3
-flux_flags_get.3: flux_flags_set.3
-flux_fatal_error.3: flux_fatal_set.3
-FLUX_FATAL.3: flux_fatal_set.3
-flux_event_unsubscribe.3: flux_event_subscribe.3
-flux_event_publish_pack.3: flux_event_publish.3
-flux_event_publish_raw.3: flux_event_publish.3
-flux_event_publish_get_seq.3: flux_event_publish.3
-flux_pollfd.3: flux_pollevents.3
-flux_msg_decode.3: flux_msg_encode.3
-flux_get_size.3: flux_get_rank.3
-flux_attr_set.3: flux_attr_get.3
-flux_set_reactor.3: flux_get_reactor.3
-flux_reactor_destroy.3: flux_reactor_create.3
-flux_reactor_run.3: flux_reactor_create.3
-flux_reactor_stop.3: flux_reactor_create.3
-flux_reactor_error.3: flux_reactor_create.3
-flux_reactor_active_incref.3: flux_reactor_create.3
-flux_reactor_active_decref.3: flux_reactor_create.3
-flux_fd_watcher_get_fd.3: flux_fd_watcher_create.3
-flux_watcher_stop.3: flux_watcher_start.3
-flux_watcher_destroy.3: flux_watcher_start.3
-flux_watcher_next_wakeup.3: flux_watcher_start.3
-flux_zmq_watcher_get_zsock.3: flux_zmq_watcher_create.3
-flux_handle_watcher_get_flux.3: flux_handle_watcher_create.3
-flux_timer_watcher_reset.3: flux_timer_watcher_create.3
-flux_periodic_watcher_reset.3: flux_periodic_watcher_create.3
-flux_prepare_watcher_create.3: flux_idle_watcher_create.3
-flux_check_watcher_create.3: flux_idle_watcher_create.3
-flux_msg_handler_destroy.3: flux_msg_handler_create.3
-flux_msg_handler_start.3: flux_msg_handler_create.3
-flux_msg_handler_stop.3: flux_msg_handler_create.3
-flux_msg_handler_delvec.3: flux_msg_handler_addvec.3
-flux_child_watcher_get_rpid.3: flux_child_watcher_create.3
-flux_child_watcher_get_rstatus.3: flux_child_watcher_create.3
-flux_signal_watcher_get_signum.3: flux_signal_watcher_create.3
-flux_stat_watcher_get_rstat.3: flux_stat_watcher_create.3
-flux_respond_raw.3: flux_respond.3
-flux_respond_pack.3: flux_respond.3
-flux_respond_error.3: flux_respond.3
-flux_reactor_now_update.3: flux_reactor_now.3
-flux_request_unpack.3: flux_request_decode.3
-flux_request_decode_raw.3: flux_request_decode.3
-flux_event_unpack.3: flux_event_decode.3
-flux_event_encode.3: flux_event_decode.3
-flux_event_pack.3: flux_event_decode.3
-flux_event_encode_raw.3: flux_event_decode.3
-flux_event_decode_raw.3: flux_event_decode.3
-flux_response_decode_raw.3: flux_response_decode.3
-flux_response_decode_error.3: flux_response_decode.3
-flux_request_encode_raw.3: flux_request_encode.3
-flux_content_load_get.3: flux_content_load.3
-flux_content_store.3: flux_content_load.3
-flux_content_store_get.3: flux_content_load.3
-flux_vlog.3: flux_log.3
-flux_log_set_appname.3: flux_log.3
-flux_log_set_procid.3: flux_log.3
-flux_future_reset.3: flux_future_get.3
-flux_future_destroy.3: flux_future_get.3
-flux_future_wait_for.3: flux_future_get.3
-flux_future_then.3: flux_future_get.3
-flux_future_fulfill.3: flux_future_create.3
-flux_future_fulfill_error.3: flux_future_create.3
-flux_future_fulfill_with.3: flux_future_create.3
-flux_future_aux_set.3: flux_future_create.3
-flux_future_aux_get.3: flux_future_create.3
-flux_future_set_flux.3: flux_future_create.3
-flux_future_get_flux.3: flux_future_create.3
-flux_rpc_pack.3: flux_rpc.3
-flux_rpc_raw.3: flux_rpc.3
-flux_rpc_message.3: flux_rpc.3
-flux_rpc_get.3: flux_rpc.3
-flux_rpc_get_unpack.3: flux_rpc.3
-flux_rpc_get_raw.3: flux_rpc.3
-flux_kvs_lookupat.3: flux_kvs_lookup.3
-flux_kvs_lookup_get.3: flux_kvs_lookup.3
-flux_kvs_lookup_get_unpack.3: flux_kvs_lookup.3
-flux_kvs_lookup_get_raw.3: flux_kvs_lookup.3
-flux_kvs_lookup_get_dir.3: flux_kvs_lookup.3
-flux_kvs_lookup_treeobj.3: flux_kvs_lookup.3
-flux_kvs_lookup_symlink.3: flux_kvs_lookup.3
-flux_kvs_getroot_get_treeobj.3: flux_kvs_getroot.3
-flux_kvs_getroot_get_blobref.3: flux_kvs_getroot.3
-flux_kvs_getroot_get_sequence.3: flux_kvs_getroot.3
-flux_kvs_getroot_get_owner.3: flux_kvs_getroot.3
-flux_kvs_getroot_cancel.3: flux_kvs_getroot.3
-flux_kvs_fence.3: flux_kvs_commit.3
-flux_kvs_commit_get_treeobj.3: flux_kvs_commit.3
-flux_kvs_commit_get_sequence.3: flux_kvs_commit.3
-flux_kvs_txn_destroy.3: flux_kvs_txn_create.3
-flux_kvs_txn_put.3: flux_kvs_txn_create.3
-flux_kvs_txn_pack.3: flux_kvs_txn_create.3
-flux_kvs_txn_vpack.3: flux_kvs_txn_create.3
-flux_kvs_txn_mkdir.3: flux_kvs_txn_create.3
-flux_kvs_txn_unlink.3: flux_kvs_txn_create.3
-flux_kvs_txn_symlink.3: flux_kvs_txn_create.3
-flux_kvs_txn_put_raw.3: flux_kvs_txn_create.3
-# N.B. exceeds max 8 stubs  flux_kvs_txn_put_treeobj.3: flux_kvs_txn_create.3
-flux_kvs_namespace_remove.3: flux_kvs_namespace_create.3
-flux_kvs_move.3: flux_kvs_copy.3
-flux_core_version_string.3: flux_core_version.3
-idset_destroy.3: idset_create.3
-idset_encode.3: idset_create.3
-idset_decode.3: idset_create.3
-idset_set.3: idset_create.3
-idset_clear.3: idset_create.3
-idset_first.3: idset_create.3
-idset_next.3: idset_create.3
-idset_count.3: idset_create.3
-# N.B. exceeds max 8 stubs  idset_equal.3
-
-flux_open.3: topen.c
-flux_send.3: tsend.c
-flux_recv.3: trecv.c
-flux_event_subscribe.3: tevent.c
-flux_rpc.3: trpc.c
-flux_rpc.3: trpc_then.c
-flux_get_rank.3: tinfo.c
-
-
-
-EXTRA_DIST = $(ADOC_FILES) COPYRIGHT.adoc JSON_PACK.adoc JSON_UNPACK.adoc
-
-CLEANFILES = $(MAN3_FILES) $(XML_FILES)
-
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-        -I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-        $(ZMQ_CFLAGS) $(LIBUUID_CFLAGS)
-
-LDADD = \
-        $(top_builddir)/src/common/libflux-internal.la \
-        $(top_builddir)/src/common/libflux-core.la \
-        $(ZMQ_LIBS) $(LIBUUID_CFLAGS) $(LIBPTHREAD)
-
-check_PROGRAMS = \
-	topen \
-	tsend \
-	trecv \
-	tevent \
-	trpc \
-	trpc_then \
-	tinfo
diff --git a/doc/man3/common/experimental.rst b/doc/man3/common/experimental.rst
new file mode 100644
index 000000000000..00be899898a2
--- /dev/null
+++ b/doc/man3/common/experimental.rst
@@ -0,0 +1,4 @@
+Experimental Flux features and interfaces are made available for evaluation
+only and may change or be removed without notice.
+
+Feedback is welcome.  Please use the flux-core project Github issue tracker.
diff --git a/doc/man3/common/json_pack.rst b/doc/man3/common/json_pack.rst
new file mode 100644
index 000000000000..b233b4d5bca9
--- /dev/null
+++ b/doc/man3/common/json_pack.rst
@@ -0,0 +1,77 @@
+Flux API functions that are based on Jansson's :func:`json_pack`
+accept the following tokens in their format string.
+The type in parenthesis denotes the resulting JSON type, and
+the type in brackets (if any) denotes the C type that is expected as
+the corresponding argument or arguments.
+
+**s** (string)['const char \*']
+   Convert a null terminated UTF-8 string to a JSON string.
+
+**s?** (string)['const char \*']
+   Like **s**, but if the argument is NULL, outputs a JSON null value.
+
+**s#** (string)['const char \*', 'int']
+   Convert a UTF-8 buffer of a given length to a JSON string.
+
+**s%** (string)['const char \*', 'size_t']
+   Like **s#** but the length argument is of type size_t.
+
+**+** ['const char \*']
+   Like **s**, but concatenate to the previous string.
+   Only valid after a string.
+
+**+#** ['const char \*', 'int']
+   Like **s#**, but concatenate to the previous string.
+   Only valid after a string.
+
+**+%** ['const char \*', 'size_t']
+   Like **+#**, but the length argument is of type size_t.
+
+**n** (null)
+   Output a JSON null value. No argument is consumed.
+
+**b** (boolean)['int']
+   Convert a C int to JSON boolean value. Zero is converted to
+   *false* and non-zero to *true*.
+
+**i** (integer)['int']
+   Convert a C int to JSON integer.
+
+**I** (integer)['int64_t']
+   Convert a C int64_t to JSON integer.
+   Note: Jansson expects a json_int_t here without committing to a size,
+   but Flux guarantees that this is a 64-bit integer.
+
+**f** (real)['double']
+   Convert a C double to JSON real.
+
+**o** (any value)['json_t \*']
+   Output any given JSON value as-is. If the value is added to an array
+   or object, the reference to the value passed to **o** is stolen by the
+   container.
+
+**O** (any value)['json_t \*']
+   Like **o**, but the argument's reference count is incremented. This
+   is useful if you pack into an array or object and want to keep the reference
+   for the JSON value consumed by **O** to yourself.
+
+**o?**, **O?** (any value)['json_t \*']
+   Like **o** and **O**, respectively, but if the argument is NULL,
+   output a JSON null value.
+
+**[fmt]** (array)
+   Build an array with contents from the inner format string. **fmt** may
+   contain objects and arrays, i.e. recursive value building is supported.
+
+**{fmt}** (object)
+   Build an object with contents from the inner format string **fmt**.
+   The first, third, etc. format specifier represent a key, and must be a
+   string as object keys are always strings. The second, fourth, etc.
+   format specifier represent a value. Any value may be an object or array,
+   i.e. recursive value building is supported.
+
+Whitespace, **:** (colon) and **,** (comma) are ignored.
+
+These descriptions came from the Jansson 2.9 manual.
+
+See also: Jansson API: Building Values: http://jansson.readthedocs.io/en/2.9/apiref.html#building-values
diff --git a/doc/man3/common/json_unpack.rst b/doc/man3/common/json_unpack.rst
new file mode 100644
index 000000000000..f3afe8b5e5cb
--- /dev/null
+++ b/doc/man3/common/json_unpack.rst
@@ -0,0 +1,71 @@
+Flux API functions that are based on Jansson's :func:`json_unpack`
+accept the following tokens in their format string.
+The type in parenthesis denotes the resulting JSON type, and
+the type in brackets (if any) denotes the C type that is expected as
+the corresponding argument or arguments.
+
+**s** (string)['const char \*']
+   Convert a JSON string to a pointer to a null terminated UTF-8 string.
+   The resulting string is extracted by using 'json_string_value()'
+   internally, so it exists as long as there are still references to the
+   corresponding JSON string.
+
+**s%** (string)['const char \*', 'size_t']
+   Convert a JSON string to a pointer to a null terminated UTF-8
+   string and its length.
+
+**n** (null)
+   Expect a JSON null value. Nothing is extracted.
+
+**b** (boolean)['int']
+   Convert a JSON boolean value to a C int, so that *true* is converted to 1
+   and *false* to 0.
+
+**i** (integer)['int']
+   Convert a JSON integer to a C int.
+
+**I** (integer)['int64_t']
+   Convert a JSON integer to a C int64_t.
+   Note: Jansson expects a json_int_t here without committing to a size,
+   but Flux guarantees that this is a 64-bit integer.
+
+**f** (real)['double']
+   Convert JSON real to a C double.
+
+**F** (real)['double']
+   Convert JSON number (integer or real) to a C double.
+
+**o** (any value)['json_t \*']
+   Store a JSON value, with no conversion, to a json_t pointer.
+
+**O** (any value)['json_t \*']
+   Like **o**, but the JSON value's reference count is incremented.
+
+**[fmt]** (array)
+   Convert each item in the JSON array according to the inner format
+   string. **fmt** may contain objects and arrays, i.e. recursive value
+   extraction is supported.
+
+**{fmt}** (object)
+   Convert each item in the JSON object according to the inner format
+   string **fmt**. The first, third, etc. format specifier represent a
+   key, and must by **s**. The corresponding argument to unpack functions
+   is read as the object key. The second, fourth, etc. format specifier
+   represent a value and is written to the address given as the corresponding
+   argument. Note that every other argument is read from and every other
+   is written to. **fmt** may contain objects and arrays as values, i.e.
+   recursive value extraction is supported. Any **s** representing a key
+   may be suffixed with **?** to make the key optional. If the key is not
+   found, nothing is extracted.
+
+**!**
+   This special format specifier is used to enable the check that all
+   object and array items are accessed, on a per-value basis. It must
+   appear inside an array or object as the last format specifier before
+   the closing bracket or brace.
+
+Whitespace, **:** (colon) and **,** (comma) are ignored.
+
+These descriptions came from the Jansson 2.9 manual.
+
+See also: Jansson API: Parsing and Validating Values: http://jansson.readthedocs.io/en/2.9/apiref.html#parsing-and-validating-values
diff --git a/doc/man3/common/resources.rst b/doc/man3/common/resources.rst
new file mode 100644
index 000000000000..dcc16e8954b0
--- /dev/null
+++ b/doc/man3/common/resources.rst
@@ -0,0 +1,5 @@
+Flux: http://flux-framework.org
+
+Flux RFC: https://flux-framework.readthedocs.io/projects/flux-rfc
+
+Issue Tracker: https://github.com/flux-framework/flux-core/issues
diff --git a/doc/man3/example/die.h b/doc/man3/example/die.h
new file mode 100644
index 000000000000..177ad83cff62
--- /dev/null
+++ b/doc/man3/example/die.h
@@ -0,0 +1,14 @@
+#ifndef _EXAMPLE_DIE
+#define _EXAMPLE_DIE
+
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+
+#define die(fmt, ...) do { \
+    int _e = errno; \
+    fprintf (stderr, (fmt), ##__VA_ARGS__); \
+    fprintf (stderr, ": %s\n", strerror (_e)); \
+    exit (1); \
+} while (0);
+#endif // _EXAMPLE_DIE
diff --git a/doc/man3/example/event.c b/doc/man3/example/event.c
new file mode 100644
index 000000000000..36a29d393b2d
--- /dev/null
+++ b/doc/man3/example/event.c
@@ -0,0 +1,24 @@
+#include <flux/core.h>
+#include "die.h"
+
+int main (int argc, char **argv)
+{
+    flux_t *h;
+    flux_msg_t *msg;
+    const char *topic;
+
+    if (!(h = flux_open (NULL, 0)))
+        die ("could not connect to broker");
+    if (flux_event_subscribe (h, "heartbeat.pulse") < 0)
+        die ("error subscribing to heartbeat");
+    if (!(msg = flux_recv (h, FLUX_MATCH_EVENT, 0)))
+        die ("message receive error");
+    if (flux_msg_get_topic (msg, &topic) < 0)
+        die ("error decoding message");
+    printf ("Event: %s\n", topic);
+
+    (void)flux_event_unsubscribe (h, "heartbeat.pulse");
+    flux_msg_destroy (msg);
+    flux_close (h);
+    return (0);
+}
diff --git a/doc/man3/example/info.c b/doc/man3/example/info.c
new file mode 100644
index 000000000000..2b3ed1e12a1f
--- /dev/null
+++ b/doc/man3/example/info.c
@@ -0,0 +1,17 @@
+#include <flux/core.h>
+#include "die.h"
+
+int main (int argc, char **argv)
+{
+    flux_t *h;
+    uint32_t rank, n;
+
+    if (!(h = flux_open (NULL, 0)))
+        die ("could not connect to broker");
+    if (flux_get_rank (h, &rank) < 0)
+        die ("error getting rank");
+    if (flux_get_size (h, &n) < 0)
+        die ("error getting size");
+    flux_close (h);
+    return (0);
+}
diff --git a/doc/man3/example/open.c b/doc/man3/example/open.c
new file mode 100644
index 000000000000..a49d6555a37a
--- /dev/null
+++ b/doc/man3/example/open.c
@@ -0,0 +1,16 @@
+#include <flux/core.h>
+#include "die.h"
+
+int main (int argc, char **argv)
+{
+    flux_t *h;
+    uint32_t rank;
+
+    if (!(h = flux_open (NULL, 0)))
+        die ("could not connect to broker");
+    if (flux_get_rank (h, &rank) < 0)
+        die ("could not get rank");
+    printf ("My rank is %d\n", (int)rank);
+    flux_close (h);
+    return (0);
+}
diff --git a/doc/man3/example/recv.c b/doc/man3/example/recv.c
new file mode 100644
index 000000000000..fcf4b3ea7645
--- /dev/null
+++ b/doc/man3/example/recv.c
@@ -0,0 +1,24 @@
+#include <flux/core.h>
+#include "die.h"
+
+int main (int argc, char **argv)
+{
+    flux_t *h;
+    flux_msg_t *msg;
+    const char *topic;
+
+    if (!(h = flux_open (NULL, 0)))
+        die ("could not connect to broker");
+    if (flux_event_subscribe (h, "") < 0)
+        die ("could not subscribe to all events");
+    for (;;) {
+        if ((msg = flux_recv (h, FLUX_MATCH_EVENT, 0)))
+            die ("receive error");
+        if (flux_msg_get_topic (msg, &topic) < 0)
+            die ("message decode error");
+        printf ("Event: %s\n", topic);
+        flux_msg_destroy (msg);
+    }
+    flux_close (h);
+    return (0);
+}
diff --git a/doc/man3/example/rpc.c b/doc/man3/example/rpc.c
new file mode 100644
index 000000000000..26f032352baa
--- /dev/null
+++ b/doc/man3/example/rpc.c
@@ -0,0 +1,31 @@
+#include <flux/core.h>
+#include "die.h"
+
+int main (int argc, char **argv)
+{
+    flux_t *h;
+    flux_future_t *f;
+    const char *rankstr;
+
+    if (!(h = flux_open (NULL, 0)))
+        die ("could not connect to broker");
+
+    if (!(f = flux_rpc_pack (h,
+                             "attr.get",
+			     FLUX_NODEID_ANY,
+			     0,
+		             "{s:s}",
+			     "name", "rank")))
+        die ("error sending attr.get request");
+
+    if (flux_rpc_get_unpack (f,
+			     "{s:s}",
+			     "value", &rankstr) < 0)
+        die ("error fetching rank");
+
+    printf ("rank is %s\n", rankstr);
+
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
diff --git a/doc/man3/example/rpc_then.c b/doc/man3/example/rpc_then.c
new file mode 100644
index 000000000000..1a562c341935
--- /dev/null
+++ b/doc/man3/example/rpc_then.c
@@ -0,0 +1,37 @@
+#include <flux/core.h>
+#include "die.h"
+
+void continuation (flux_future_t *f, void *arg)
+{
+    const char *rankstr;
+
+    if (flux_rpc_get_unpack (f, "{s:s}", "value", &rankstr) < 0)
+        die ("error getting rank");
+
+    printf ("rank is %s\n", rankstr);
+    flux_future_destroy (f);
+}
+
+int main (int argc, char **argv)
+{
+    flux_t *h;
+    flux_future_t *f;
+
+    if (!(h = flux_open (NULL, 0)))
+        die ("could not connect to broker");
+
+    if (!(f = flux_rpc_pack (h,
+                             "attr.get",
+			     FLUX_NODEID_ANY,
+			     0,
+		             "{s:s}",
+			     "name", "rank"))
+    	|| flux_future_then (f, -1., continuation, NULL) < 0)
+        die ("error sending attr.get request");
+
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        die ("reactor meltdown");
+
+    flux_close (h);
+    return (0);
+}
diff --git a/doc/man3/example/send.c b/doc/man3/example/send.c
new file mode 100644
index 000000000000..9e3cc4e867ba
--- /dev/null
+++ b/doc/man3/example/send.c
@@ -0,0 +1,18 @@
+#include <flux/core.h>
+#include "die.h"
+
+int main (int argc, char **argv)
+{
+    flux_t *h;
+    flux_msg_t *msg;
+
+    if (!(h = flux_open (NULL, 0)))
+        die ("flux open %s", "NULL");
+    if (!(msg = flux_event_encode ("snack.bar.closing", NULL)))
+        die ("flux_event_encode");
+    if (flux_send (h, msg, 0) < 0)
+        die ("flux_send");
+    flux_msg_destroy (msg);
+    flux_close (h);
+    return (0);
+}
diff --git a/doc/man3/example/sync.c b/doc/man3/example/sync.c
new file mode 100644
index 000000000000..5f5c0872a389
--- /dev/null
+++ b/doc/man3/example/sync.c
@@ -0,0 +1,34 @@
+#include <flux/core.h>
+#include "die.h"
+
+const double sync_min = 1.0;
+const double sync_max = 60.0;
+
+void sync_continuation (flux_future_t *f, void *arg)
+{
+    // do work here
+    flux_future_reset (f);
+}
+
+int main (int argc, char **argv)
+{
+    flux_t *h;
+    flux_future_t *f;
+
+    if (!(h = flux_open (NULL, 0)))
+        die ("could not connect to broker");
+
+    if (!(f = flux_sync_create (h, sync_max)))
+        die ("error creating future");
+
+    if (flux_future_then (f, sync_min, sync_continuation, NULL) < 0)
+        die ("error registering continuation");
+
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        die ("reactor returned with error");
+
+    flux_future_destroy (f);
+
+    flux_close (h);
+    return (0);
+}
diff --git a/doc/man3/flux_attr_get.adoc b/doc/man3/flux_attr_get.adoc
deleted file mode 100644
index 6886530dc8b9..000000000000
--- a/doc/man3/flux_attr_get.adoc
+++ /dev/null
@@ -1,81 +0,0 @@
-flux_attr_get(3)
-================
-:doctype: manpage
-
-
-NAME
-----
-flux_attr_get, flux_attr_set - get/set Flux broker attributes
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- const char *flux_attr_get (flux_t *h, const char *name);
-
- int flux_attr_set (flux_t *h, const char *name, const char *val);
-
-
-DESCRIPTION
------------
-
-Flux broker attributes are both a simple, general-purpose key-value
-store with scope limited to the local broker rank, and a method for the
-broker to export information needed by Flux comms modules and utilities.
-
-`flux_attr_get()` retrieves the value of the attribute _name_.
-
-Attributes that have the broker tags as _immutable_ are cached automatically
-in the flux_t handle.  For example, the "rank" attribute is a frequently
-accessed attribute whose value never changes.  It will be cached on the first
-access and thereafter does not require an RPC to the broker to access.
-
-`flux_attr_set()` updates the value of attribute _name_ to _val_.
-If _name_ is not currently a valid attribute, it is created.
-If _val_ is NULL, the attribute is unset.
-
-
-RETURN VALUE
-------------
-
-`flux_attr_get()` returns the requested value on success.  On error, NULL
-is returned and errno is set appropriately.
-
-`flux_attr_set()` returns zero on success.  On error, -1 is returned
-and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOENT::
-The requested attribute is invalid or has a NULL value on the broker.
-
-EINVAL::
-Some arguments were invalid.
-
-EPERM::
-Set was attempted on an attribute that is not writable with the
-user's credentials.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-flux-lsattr(1), flux-getattr(1), flux-setattr(1), flux-broker-attributes(7),
-https://github.com/flux-framework/rfc/blob/master/spec_3.adoc[RFC 3: CMB1 - Flux Comms Message Broker Protocol]
diff --git a/doc/man3/flux_attr_get.rst b/doc/man3/flux_attr_get.rst
new file mode 100644
index 000000000000..c073b397fb90
--- /dev/null
+++ b/doc/man3/flux_attr_get.rst
@@ -0,0 +1,72 @@
+================
+flux_attr_get(3)
+================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   const char *flux_attr_get (flux_t *h, const char *name);
+
+   int flux_attr_set (flux_t *h, const char *name, const char *val);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+Flux broker attributes are both a simple, general-purpose key-value
+store with scope limited to the local broker rank, and a method for the
+broker to export information needed by Flux services and utilities.
+
+:func:`flux_attr_get` retrieves the value of the attribute :var:`name`.
+
+Attributes that have the broker tags as *immutable* are cached automatically
+in the :type:`flux_t` handle. For example, the "rank" attribute is a frequently
+accessed attribute whose value never changes. It will be cached on the first
+access and thereafter does not require an RPC to the broker to access.
+
+:func:`flux_attr_set` updates the value of attribute :var:`name` to :var:`val`.
+If :var:`name` is not currently a valid attribute, it is created.
+If :var:`val` is NULL, the attribute is unset.
+
+
+RETURN VALUE
+============
+
+:func:`flux_attr_get` returns the requested value on success. On error, NULL
+is returned and :var:`errno` is set appropriately.
+
+:func:`flux_attr_set` returns zero on success. On error, -1 is returned
+and errno is set appropriately.
+
+
+ERRORS
+======
+
+ENOENT
+   The requested attribute is invalid or has a NULL value on the broker.
+
+EINVAL
+   Some arguments were invalid.
+
+EPERM
+   Set was attempted on an attribute that is not writable with the
+   user's credentials.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-getattr`, :man7:`flux-broker-attributes`,
diff --git a/doc/man3/flux_aux_set.adoc b/doc/man3/flux_aux_set.adoc
deleted file mode 100644
index 3c7822e43ea3..000000000000
--- a/doc/man3/flux_aux_set.adoc
+++ /dev/null
@@ -1,79 +0,0 @@
-flux_aux_set(3)
-===============
-:doctype: manpage
-
-
-NAME
-----
-flux_aux_set, flux_aux_get - get/set auxiliary handle data
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_free_f)(void *arg);
-
- void *flux_aux_get (flux_t *h, const char *name);
-
- int flux_aux_set (flux_t *h, const char *name,
-                   void *aux, flux_free_f destroy);
-
-
-DESCRIPTION
------------
-
-`flux_aux_set()` attaches application-specific data
-to the parent object _h_.  It stores data _aux_ by key _name_,
-with optional destructor _destroy_.  The destructor, if non-NULL,
-is called when the parent object is destroyed, or when
-_key_ is overwritten by a new value.  If _aux_ is NULL,
-the destructor for a previous value, if any is called,
- but no new value is stored.  If _name_ is NULL,
-_aux_ is stored anonymously.
-
-`flux_aux_get()` retrieves application-specific data
-by _name_.  If the data was stored anonymously, it
-cannot be retrieved.
-
-Names beginning with "flux::" are reserved for internal use.
-
-RETURN VALUE
-------------
-
-`flux_aux_get()` returns data on success, or NULL on failure, with errno set.
-
-`flux_aux_set()` returns 0 on success, or -1 on failure, with errno set.
-
-
-ERRORS
-------
-
-EINVAL::
-Some arguments were invalid.
-
-ENOMEM::
-Out of memory.
-
-ENOENT::
-`flux_aux_get()` could not find an entry for _key_.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_open(3)
diff --git a/doc/man3/flux_aux_set.rst b/doc/man3/flux_aux_set.rst
new file mode 100644
index 000000000000..d56625eea9bb
--- /dev/null
+++ b/doc/man3/flux_aux_set.rst
@@ -0,0 +1,77 @@
+===============
+flux_aux_set(3)
+===============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_free_f)(void *arg);
+
+   void *flux_aux_get (flux_t *h, const char *name);
+
+   int flux_aux_set (flux_t *h,
+                     const char *name,
+                     void *aux,
+                     flux_free_f destroy);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_aux_set` attaches application-specific data
+to the parent object :var:`h`. It stores data :var:`aux` by key :var:`name`,
+with optional destructor :var:`destroy`. The destructor, if non-NULL,
+is called when the parent object is destroyed, or when
+:var:`key` is overwritten by a new value. If :var:`aux` is NULL,
+the destructor for a previous value, if any is called,
+but no new value is stored. If :var:`name` is NULL,
+:var:`aux` is stored anonymously.
+
+:func:`flux_aux_get` retrieves application-specific data
+by :var:`name`. If the data was stored anonymously, it
+cannot be retrieved.  Note that :func:`flux_aux_get` does not scale to a
+large number of items, and flux module handles may persist for a long
+time.
+
+Names beginning with "flux::" are reserved for internal use.
+
+
+RETURN VALUE
+============
+
+:func:`flux_aux_get` returns data on success, or NULL on failure,
+with :var:`errno` set.
+
+:func:`flux_aux_set` returns 0 on success, or -1 on failure, with errno set.
+
+
+ERRORS
+======
+
+EINVAL
+   Some arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+ENOENT
+   :func:`flux_aux_get` could not find an entry for *key*.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_open`
diff --git a/doc/man3/flux_child_watcher_create.adoc b/doc/man3/flux_child_watcher_create.adoc
deleted file mode 100644
index 50dd0f193d0d..000000000000
--- a/doc/man3/flux_child_watcher_create.adoc
+++ /dev/null
@@ -1,85 +0,0 @@
-flux_child_watcher_create(3)
-============================
-:doctype: manpage
-
-
-NAME
-----
-flux_child_watcher_create, flux_child_watcher_get_rpid, flux_child_watcher_get_rstatus - create child watcher
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_watcher_f)(flux_reactor_t *r,
-                                flux_watcher_t *w,
-                                int revents, void *arg);
-
- flux_watcher_t *flux_child_watcher_create (flux_reactor_t *r,
-                                            int pid, bool trace,
-                                            flux_watcher_f cb, void *arg);
-
- int flux_child_watcher_get_rpid (flux_watcher_t *w);
-
- int flux_child_watcher_get_rstatus (flux_watcher_t *w);
-
-
-DESCRIPTION
------------
-
-`flux_child_watcher_create()` creates a reactor watcher that
-monitors state transitions of child processes.  If _trace_ is false,
-only child termination will trigger an event; otherwise, stop and start
-events may be generated.
-
-The callback _revents_ argument should be ignored.
-
-The process id that had a transition may be obtained by calling
-`flux_child_watcher_get_rpid()`.
-
-The status value returned by waitpid(2) may be obtained by calling
-`flux_child_watcher_get_rstatus()`.
-
-Only a Flux reactor created with the FLUX_REACTOR_SIGCHLD flag can
-be used with child watchers, as the reactor must register a SIGCHLD
-signal watcher before any processes are spawned.  Only one reactor instance
-per program may be created with this capability.
-
-
-RETURN VALUE
-------------
-
-flux_child_watcher_create() returns a flux_watcher_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-EINVAL::
-Reactor was not created with FLUX_REACTOR_SIGCHLD.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_watcher_start(3), flux_reactor_start(3)
-
-http://software.schmorp.de/pkg/libev.html[libev home page]
diff --git a/doc/man3/flux_child_watcher_create.rst b/doc/man3/flux_child_watcher_create.rst
new file mode 100644
index 000000000000..a1d2d8478508
--- /dev/null
+++ b/doc/man3/flux_child_watcher_create.rst
@@ -0,0 +1,81 @@
+============================
+flux_child_watcher_create(3)
+============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_watcher_f)(flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg);
+
+   flux_watcher_t *flux_child_watcher_create (flux_reactor_t *r,
+                                              int pid,
+                                              bool trace,
+                                              flux_watcher_f cb,
+                                              void *arg);
+
+   int flux_child_watcher_get_rpid (flux_watcher_t *w);
+
+   int flux_child_watcher_get_rstatus (flux_watcher_t *w);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_child_watcher_create` creates a reactor watcher that
+monitors state transitions of child processes. If :var:`trace` is false,
+only child termination will trigger an event; otherwise, stop and start
+events may be generated.
+
+The callback :var:`revents` argument should be ignored.
+
+The process id that had a transition may be obtained by calling
+:func:`flux_child_watcher_get_rpid`.
+
+The status value returned by :linux:man2:`waitpid` may be obtained by calling
+:func:`flux_child_watcher_get_rstatus`.
+
+Only a Flux reactor created with the FLUX_REACTOR_SIGCHLD flag can
+be used with child watchers, as the reactor must register a SIGCHLD
+signal watcher before any processes are spawned. Only one reactor instance
+per program may be created with this capability.
+
+
+RETURN VALUE
+============
+
+:func:`flux_child_watcher_create` returns a :type:`flux_watcher_t` object on
+success.  On error, NULL is returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+EINVAL
+   Reactor was not created with FLUX_REACTOR_SIGCHLD.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+libev: http://software.schmorp.de/pkg/libev.html
+
+
+SEE ALSO
+========
+
+:man3:`flux_watcher_start`, :man3:`flux_reactor_run`
diff --git a/doc/man3/flux_comms_error_set.rst b/doc/man3/flux_comms_error_set.rst
new file mode 100644
index 000000000000..5bbe343fba1d
--- /dev/null
+++ b/doc/man3/flux_comms_error_set.rst
@@ -0,0 +1,52 @@
+=======================
+flux_comms_error_set(3)
+=======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef int (*flux_comms_error_f)(flux_t *h, void *arg);
+
+   void flux_comms_error_set (flux_t *h,
+                              flux_comms_error_f fun,
+                              void *arg);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_comms_error_set` configures an optional callback :var:`fun` to
+be called internally by ``libflux_core`` if an error occurs when sending
+or receiving messages on the handle :var:`h`.
+
+:var:`arg` is an optional argument passed through to the callback function.
+
+The callback may assume that :var:`errno` is valid.  A typical callback in an
+application might log the error and then exit.
+
+If a comms error function is not registered, or if the function returns -1,
+error handling proceeds as normal.  Be aware that further access attempts
+to :var:`h` are likely to fail and the callback may be invoked again.
+
+In advanced use cases, the callback may resolve the error and return 0,
+in which case the errant low level message send or receive call is retried.
+This mode should be considered experimental at this time.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_open`
diff --git a/doc/man3/flux_content_load.adoc b/doc/man3/flux_content_load.adoc
deleted file mode 100644
index b5c98bf81858..000000000000
--- a/doc/man3/flux_content_load.adoc
+++ /dev/null
@@ -1,137 +0,0 @@
-flux_content_load(3)
-====================
-:doctype: manpage
-
-
-NAME
-----
-flux_content_load, flux_content_load_get, flux_content_store, flux_content_store_get - load/store content
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_future_t *flux_content_load (flux_t *h,
-                                   const char *blobref,
-                                   int flags);
-
- int flux_content_load_get (flux_future_t *f,
-                            const void **buf,
-                            size_t *len);
-
-
- flux_future_t *flux_content_store (flux_t *h,
-                                    const void *buf,
-                                    size_t len,
-                                    int flags);
-
- int flux_content_store_get (flux_future_t *f,
-                             const char **ref);
-
-
-DESCRIPTION
------------
-
-The Flux content service is an append-only, immutable, content addressed
-data storage service unique to each Flux instance, described in RFC 10.
-All functions described below are idempotent.
-
-`flux_content_load()` sends a request to the content service
-to load a data blob by _blobref_, a hash digest whose format
-is described in RFC 10.  A `flux_future_t` object which encapsulates the
-remote procedure call state is returned.  _flags_ is a mask that may
-include the values described below.
-
-`flux_request_load_get()` completes a load operation, blocking on
-response(s) if needed, parsing the result, and returning the requested
-blob in _buf_ and its length in _len_.  _buf_ is valid until
-`flux_future_destroy()` is called.
-
-`flux_content_store()` sends a request to the content service
-to store a data blob _buf_ of length _len_.   A `flux_future_t`
-object which encapsulates the remote procedure call state is returned.
-_flags_ is a mask that may include the values described below.
-
-`flux_content_store_get()` completes a store operation, blocking on
-response(s) if needed, and returning a blobref that can be used to
-retrieve the stored blob.  The blobref string is valid until
-`flux_future_destroy()` is called.
-
-These functions may be used asynchronously.
-See `flux_future_then(3)` for details.
-
-
-FLAGS
------
-
-The following are valid bits in a _flags_ mask passed as an argument
-to `flux_content_load()` or `flux_content_store()`.
-
-CONTENT_FLAG_CACHE_BYPASS::
-Send the request directly to the backing store (default sqlite),
-bypassing the cache.
-
-CONTENT_FLAG_UPSTREAM::
-Direct the request to the next broker upstream on the TBON rather
-than to the local broker.
-
-
-RETURN VALUE
-------------
-
-`flux_content_load()` and `flux_content_store()` return a
-`flux_future_t` on success, or NULL on failure with errno set appropriately.
-
-`flux_content_load_get()` and `flux_content_store_get()`
-return 0 on success, or -1 on failure with errno set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-One of the arguments was invalid.
-
-ENOMEM::
-Out of memory.
-
-ENOENT::
-An unknown blob was requested.
-
-EPROTO::
-A request was malformed.
-
-EFBIG::
-A blob larger than the configured maximum blob size
-could not be stored.  See flux-broker-attributes(7).
-
-ENOSYS::
-The CONTENT_FLAG_CACHE_BYPASS flag was set in a request, but no
-backing store module is loaded.
-
-EHOSTUNREACH::
-The CONTENT_FLAG_UPSTREAM flag was set in a request received by
-the rank 0 broker.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_rpc(3), flux_future_get(3)
-
-https://github.com/flux-framework/rfc/blob/master/spec_10.adoc[RFC 10: Content Storage Service]
diff --git a/doc/man3/flux_core_version.adoc b/doc/man3/flux_core_version.adoc
deleted file mode 100644
index b07bfd00611a..000000000000
--- a/doc/man3/flux_core_version.adoc
+++ /dev/null
@@ -1,94 +0,0 @@
-flux_core_version(3)
-====================
-:doctype: manpage
-
-
-NAME
-----
-flux_core_version, flux_core_version_string - get flux-core version
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- int flux_core_version (int *major, int *minor, int *patch);
-
- const char *flux_core_version_string (void);
-
-
-DESCRIPTION
------------
-
-flux-core defines several macros and functions to let API users determine
-the version they are working with.  A version has three components
-(_major_, _minor_, _patch_), accessible with the following macros:
-
-FLUX_CORE_VERSION_MAJOR::
-(integer) incremented when there are incompatible API changes
-
-FLUX_CORE_VERSION_MINOR::
-(integer) incremented when functionality is added in a backwards-compatible
-manner
-
-FLUX_CORE_VERSION_PATCH::
-(integer) incremented when bug fixes are added in a backwards-compatible manner
-
-These definitions conform to the _semantic versioning_ standard (see below).
-In addition, the following convenience macros are available:
-
-FLUX_CORE_VERSION_HEX::
-(hex) the three versions combined into a three-byte integer value,
-useful for comparing versions with '<', '=', and '>' operators.
-
-FLUX_CORE_VERSION_STRING::
-(string) the three versions above separated by periods, with optional
-`git-describe(1)` suffix preceded by a hyphen, if the version is a
-development snapshot.
-
-Note that major version zero (0.y.z) is for initial development.
-Under version zero, the public API should not be considered stable.
-
-Functions are also available to access the same values.  While the header
-macros tell what version of flux-core your program was compiled against,
-the functions tell what version your program is dynamically linked with.
-
-`flux_core_version()` sets _major_, _minor_, and _patch_ to the values of
-the macros above.  If any parameters are NULL, no assignment is attempted.
-
-`flux_core_version_string()` returns the string value.
-
-
-RETURN VALUE
-------------
-
-`flux_core_version ()` returns the hex version.
-
-`flux_core_version_string ()` returns the version string
-
-
-
-ERRORS
-------
-
-These functions cannot fail.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-semver.org[Semantic Versioning 2.0.0]
diff --git a/doc/man3/flux_core_version.rst b/doc/man3/flux_core_version.rst
new file mode 100644
index 000000000000..39000334f3bf
--- /dev/null
+++ b/doc/man3/flux_core_version.rst
@@ -0,0 +1,82 @@
+====================
+flux_core_version(3)
+====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   int flux_core_version (int *major, int *minor, int *patch);
+
+   const char *flux_core_version_string (void);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+flux-core defines several macros and functions to let API users determine
+the version they are working with. A version has three components
+(*major*, *minor*, *patch*), accessible with the following macros:
+
+FLUX_CORE_VERSION_MAJOR
+   (integer) incremented when there are incompatible API changes
+
+FLUX_CORE_VERSION_MINOR
+   (integer) incremented when functionality is added in a backwards-compatible
+   manner
+
+FLUX_CORE_VERSION_PATCH
+   (integer) incremented when bug fixes are added in a backwards-compatible manner
+
+These definitions conform to the *semantic versioning* standard (see below).
+In addition, the following convenience macros are available:
+
+FLUX_CORE_VERSION_HEX
+   (hex) the three versions combined into a three-byte integer value,
+   useful for comparing versions with *<*, *=*, and *>* operators.
+
+FLUX_CORE_VERSION_STRING
+   (string) the three versions above separated by periods, with optional
+   :linux:man1:`git-describe` suffix preceded by a hyphen, if the version is a
+   development snapshot.
+
+Note that major version zero (0.y.z) is for initial development.
+Under version zero, the public API should not be considered stable.
+
+Functions are also available to access the same values. While the header
+macros tell what version of flux-core your program was compiled against,
+the functions tell what version your program is dynamically linked with.
+
+:func:`flux_core_version` sets :var:`major`, :var:`minor`, and :var:`patch`
+to the values of the macros above. If any parameters are NULL, no assignment
+is attempted.
+
+:func:`flux_core_version_string` returns the string value.
+
+
+RETURN VALUE
+============
+
+:func:`flux_core_version` returns the hex version.
+
+:func:`flux_core_version_string` returns the version string
+
+
+ERRORS
+======
+
+These functions cannot fail.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+Semantic Versioning 2.0.0: http://semver.org
diff --git a/doc/man3/flux_event_decode.adoc b/doc/man3/flux_event_decode.adoc
deleted file mode 100644
index 9fda00cf0875..000000000000
--- a/doc/man3/flux_event_decode.adoc
+++ /dev/null
@@ -1,121 +0,0 @@
-flux_event_decode(3)
-====================
-:doctype: manpage
-
-
-NAME
-----
-flux_event_decode, flux_event_decode_raw, flux_event_unpack, flux_event_encode, flux_event_encode_raw, flux_event_pack - encode/decode a Flux event message
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- int flux_event_decode (const flux_msg_t *msg,
-                        const char **topic,
-                        const char **s);
-
- int flux_event_decode_raw (const flux_msg_t *msg,
-                            const char **topic,
-                            const void **data, int *len);
-
- int flux_event_unpack (const flux_msg_t *msg,
-                        const char **topic,
-                        const char *fmt, ...);
-
- flux_msg_t *flux_event_encode (const char *topic,
-                                const char *s);
-
- flux_msg_t *flux_event_encode_raw (const char *topic,
-                                    const void *data, int len);
-
- flux_msg_t *flux_event_pack (const char *topic,
-                              const char *fmt, ...);
-
-
-DESCRIPTION
------------
-
-`flux_event_decode()` decodes a Flux event message _msg_.
-
-_topic_, if non-NULL, will be set to the message's topic string. The storage
-for this string belongs to _msg_ and should not be freed.
-
-_s_, if non-NULL, will be set to the message's NULL-terminated string payload.
-If no payload exists, it is set to NULL.  The storage for this string belongs
-to _msg_ and should not be freed.
-
-`flux_event_decode_raw()` decodes an event message with a raw payload,
-setting _data_ and _len_ to the payload data and length.  The storage for
-the raw payload belongs to _msg_ and should not be freed.
-
-`flux_event_unpack()` decodes a Flux event message with a JSON payload as
-above, parsing the payload using variable arguments with a format string
-in the style of jansson's `json_unpack()` (used internally). Decoding fails
-if the message doesn't have a JSON payload.
-
-`flux_event_encode()` encodes a Flux event message with topic string _topic_
-and optional NULL-terminated string payload _s_.  The newly constructed
-message that is returned must be destroyed with `flux_msg_destroy()`.
-
-`flux_event_encode_raw()` encodes a Flux event message with topic
-string _topic_.  If _data_ is non-NULL, its contents will be used as
-the message payload, and the payload type set to raw.
-
-`flux_event_pack()` encodes a Flux event message with a JSON payload as
-above, encoding the payload using variable arguments with a format string
-in the style of jansson's `json_pack()` (used internally). Decoding fails
-if the message doesn't have a JSON payload.
-
-Events propagated to all subscribers.  Events will not be received
-without a matching subscription established using `flux_event_subscribe()`.
-
-include::JSON_PACK.adoc[]
-
-
-include::JSON_UNPACK.adoc[]
-
-
-RETURN VALUE
-------------
-
-Decoding functions return 0 on success.  On error, -1 is returned, and
-errno is set appropriately.
-
-Encoding functions return a message on success.  On error, NULL is returned,
-and errno is set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-The _msg_ argument was NULL or there was a problem encoding.
-
-ENOMEM::
-Memory was unavailable.
-
-EPROTO::
-Message decoding failed, such as due to incorrect message type,
-missing topic string, etc..
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_event_subscribe(3)
diff --git a/doc/man3/flux_event_decode.rst b/doc/man3/flux_event_decode.rst
new file mode 100644
index 000000000000..591ce35de57c
--- /dev/null
+++ b/doc/man3/flux_event_decode.rst
@@ -0,0 +1,122 @@
+====================
+flux_event_decode(3)
+====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   int flux_event_decode (const flux_msg_t *msg,
+                          const char **topic,
+                          const char **s);
+
+   int flux_event_decode_raw (const flux_msg_t *msg,
+                              const char **topic,
+                              const void **data,
+                              size_t *len);
+
+   int flux_event_unpack (const flux_msg_t *msg,
+                          const char **topic,
+                          const char *fmt,
+                          ...);
+
+   flux_msg_t *flux_event_encode (const char *topic, const char *s);
+
+   flux_msg_t *flux_event_encode_raw (const char *topic,
+                                      const void *data,
+                                      size_t len);
+
+   flux_msg_t *flux_event_pack (const char *topic,
+                                const char *fmt,
+                                ...);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_event_decode` decodes a Flux event message :var:`msg`.
+
+:var:`topic`, if non-NULL, will be set to the message's topic string. The
+storage for this string belongs to :var:`msg` and should not be freed.
+
+:var:`s`, if non-NULL, will be set to the message's NULL-terminated string
+payload.  If no payload exists, it is set to NULL. The storage for this string
+belongs to :var:`msg` and should not be freed.
+
+:func:`flux_event_decode_raw` decodes an event message with a raw payload,
+setting :var:`data` and :var:`len` to the payload data and length. The storage
+for the raw payload belongs to :var:`msg` and should not be freed.
+
+:func:`flux_event_unpack` decodes a Flux event message with a JSON payload as
+above, parsing the payload using variable arguments with a format string
+in the style of jansson's :func:`json_unpack` (used internally). Decoding fails
+if the message doesn't have a JSON payload.
+
+:func:`flux_event_encode` encodes a Flux event message with topic string
+:var:`topic` and optional NULL-terminated string payload :var:`s`. The newly
+constructed message that is returned must be destroyed with
+:func:`flux_msg_destroy()`.
+
+:func:`flux_event_encode_raw` encodes a Flux event message with topic
+string :var:`topic`. If :var:`data` is non-NULL, its contents will be used as
+the message payload, and the payload type set to raw.
+
+:func:`flux_event_pack` encodes a Flux event message with a JSON payload as
+above, encoding the payload using variable arguments with a format string
+in the style of jansson's :func:`json_pack` (used internally). Decoding fails
+if the message doesn't have a JSON payload.
+
+Events propagated to all subscribers. Events will not be received
+without a matching subscription established using :func:`flux_event_subscribe`.
+
+ENCODING JSON PAYLOADS
+======================
+
+.. include:: common/json_pack.rst
+
+DECODING JSON PAYLOADS
+======================
+
+.. include:: common/json_unpack.rst
+
+
+RETURN VALUE
+============
+
+Decoding functions return 0 on success. On error, -1 is returned, and
+:var:`errno` is set appropriately.
+
+Encoding functions return a message on success. On error, NULL is returned,
+and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   The :var:`msg` argument was NULL or there was a problem encoding.
+
+ENOMEM
+   Memory was unavailable.
+
+EPROTO
+   Message decoding failed, such as due to incorrect message type,
+   missing topic string, etc.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_event_subscribe`
diff --git a/doc/man3/flux_event_publish.adoc b/doc/man3/flux_event_publish.adoc
deleted file mode 100644
index 1555b5bfc115..000000000000
--- a/doc/man3/flux_event_publish.adoc
+++ /dev/null
@@ -1,118 +0,0 @@
-flux_event_publish(3)
-=====================
-:doctype: manpage
-
-
-NAME
-----
-flux_event_publish, flux_event_publish_pack, flux_event_publish_raw, flux_event_publish_get_seq - publish events
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_future_t *flux_event_publish (flux_t *h,
-                                    const char *topic, int flags,
-                                    const char *s);
-
- flux_future_t *flux_event_publish_pack (flux_t *h,
-                                         const char *topic, int flags,
-                                         const char *fmt, ...);
-
- flux_future_t *flux_event_publish_raw (flux_t *h,
-                                        const char *topic, int flags,
-                                        const void *data, int len);
-
- int flux_event_publish_get_seq (flux_future_t *f, int *seq);
-
-DESCRIPTION
------------
-
-`flux_event_publish()` sends an event message with topic string _topic_,
-_flags_ as described below, and optional payload _s_, a NULL-terminated
-string, or NULL indicating no payload.  The returned future is
-fulfilled once the event is accepted by the broker and assigned a
-global sequence number.
-
-`flux_event_publish_pack()` is similar, except the JSON payload
-is constructed using `json_pack()` style arguments (see below).
-
-`flux_event_publish_raw()` is similar, except the payload is raw _data_
-of length _len_.
-
-`flux_event_publish_get_seq()` may be used to retrieve the sequence
-number assigned to the message once the future is fulfilled.
-
-
-CONFIRMATION SEMANTICS
-----------------------
-
-All Flux events are "open loop" in the sense that publishers get no
-confirmation that subscribers have received a message.  However,
-the above functions do confirm, upon fulfillment of the returned future,
-that the published event has been received by the broker and assigned
-a global sequence number.
-
-Gaps in the sequence trigger the logging of errors currently, and in
-the future will trigger recovery of lost events, so these confirmations
-do indicate that Flux's best effort at event propagation is under way.
-
-If this level of confirmation is not required, one may encode
-an event message directly using `flux_event_encode(3)` and related
-functions and send it directly with `flux_send(3)`.
-
-
-FLAGS
------
-
-The _flags_ argument in the above functions must be zero, or the
-logical OR of the following values:
-
-FLUX_MSGFLAG_PRIVATE::
-Indicates that the event should only be visible to the instance owner
-and the sender.
-
-
-
-include::JSON_PACK.adoc[]
-
-
-RETURN VALUE
-------------
-
-These functions return a future on success.  On error, NULL is returned,
-and errno is set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-Some arguments were invalid.
-
-ENOMEM::
-Out of memory.
-
-EPROTO::
-A protocol error was encountered.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-flux_event_decode(3), flux_event_subscribe(3)
diff --git a/doc/man3/flux_event_publish.rst b/doc/man3/flux_event_publish.rst
new file mode 100644
index 000000000000..239f81cd1be9
--- /dev/null
+++ b/doc/man3/flux_event_publish.rst
@@ -0,0 +1,117 @@
+=====================
+flux_event_publish(3)
+=====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_event_publish (flux_t *h,
+                                      const char *topic,
+                                      int flags,
+                                      const char *s);
+
+   flux_future_t *flux_event_publish_pack (flux_t *h,
+                                           const char *topic,
+                                           int flags,
+                                           const char *fmt,
+                                           ...);
+
+   flux_future_t *flux_event_publish_raw (flux_t *h,
+                                          const char *topic,
+                                          int flags,
+                                          const void *data,
+                                          size_t len);
+
+   int flux_event_publish_get_seq (flux_future_t *f, int *seq);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_event_publish` sends an event message with topic string
+:var:`topic`, :var:`flags` as described below, and optional payload :var:`s`,
+a NULL-terminated string, or NULL indicating no payload. The returned future
+is fulfilled once the event is accepted by the broker and assigned a
+global sequence number.
+
+:func:`flux_event_publish_pack` is similar, except the JSON payload
+is constructed using :func:`json_pack` style arguments (see below).
+
+:func:`flux_event_publish_raw` is similar, except the payload is raw
+:var:`data` of length :var:`len`.
+
+:func:`flux_event_publish_get_seq` may be used to retrieve the sequence
+number assigned to the message once the future is fulfilled.
+
+
+CONFIRMATION SEMANTICS
+======================
+
+All Flux events are "open loop" in the sense that publishers get no
+confirmation that subscribers have received a message. However,
+the above functions do confirm, upon fulfillment of the returned future,
+that the published event has been received by the broker and assigned
+a global sequence number.
+
+Gaps in the sequence trigger the logging of errors currently, and in
+the future will trigger recovery of lost events, so these confirmations
+do indicate that Flux's best effort at event propagation is under way.
+
+If this level of confirmation is not required, one may encode
+an event message directly using :man3:`flux_event_encode` and related
+functions and send it directly with :man3:`flux_send`.
+
+
+FLAGS
+=====
+
+The :var:`flags` argument in the above functions must be zero, or the
+logical OR of the following values:
+
+FLUX_MSGFLAG_PRIVATE
+   Indicates that the event should only be visible to the instance owner
+   and the sender.
+
+ENCODING JSON PAYLOADS
+======================
+
+.. include:: common/json_pack.rst
+
+
+RETURN VALUE
+============
+
+These functions return a future on success. On error, NULL is returned,
+and errno is set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   Some arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+EPROTO
+   A protocol error was encountered.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_event_decode`, :man3:`flux_event_subscribe`
diff --git a/doc/man3/flux_event_subscribe.adoc b/doc/man3/flux_event_subscribe.adoc
deleted file mode 100644
index 545d01d22b95..000000000000
--- a/doc/man3/flux_event_subscribe.adoc
+++ /dev/null
@@ -1,87 +0,0 @@
-flux_event_subscribe(3)
-=======================
-:doctype: manpage
-
-
-NAME
-----
-flux_event_subscribe, flux_event_unsubscribe - manage subscriptions
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-int flux_event_subscribe (flux_t *h, const char *topic);
-
-int flux_event_unsubscribe (flux_t *h, const char *topic);
-
-
-DESCRIPTION
------------
-
-Flux events are broadcast across the session, but are only delivered
-to handles that subscribe to them by topic.  Topic strings consist of
-one or more words separated by periods, interpreted as a hierarchical
-name space.
-
-`flux_event_subscribe()` requests that event messages matching _topic_
-be delivered via `flux_recv(3)`.  A match consists of a string comparison
-of the event topic and the subscription topic, up to the length of the
-subscription topic.  Thus "foo." matches events with topics "foo.bar"
-and "foo.baz", and "" matches all events.  This matching algorithm
-is inherited from ZeroMQ.  Globs or regular expressions are not allowed
-in subscriptions, and the period delimiter is included in the comparison.
-
-`flux_event_unsubscribe()` unsubscribes to a topic.  The _topic_
-argument must exactly match that provided to `flux_event_subscribe()`.
-
-Duplicate subscriptions are allowed in the subscription list but
-will not result in multiple deliveries of a given message.  Each
-duplicate subscription requires a separate unsubscribe.
-
-It is not necessary to remove subscriptions with `flux_event_unsubscribe()`
-prior to calling `flux_close(3)`.
-
-RETURN VALUE
-------------
-
-These functions return 0 on success.  On error, -1 is returned,
-and errno is set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-Some arguments were invalid.
-
-ENOMEM::
-Out of memory.
-
-
-EXAMPLES
---------
-
-This example opens the Flux broker, subscribes to heartbeat messages,
-displays one, then quits.
-
-....
-include::tevent.c[]
-....
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
diff --git a/doc/man3/flux_event_subscribe.rst b/doc/man3/flux_event_subscribe.rst
new file mode 100644
index 000000000000..8ef2f92a2558
--- /dev/null
+++ b/doc/man3/flux_event_subscribe.rst
@@ -0,0 +1,77 @@
+=======================
+flux_event_subscribe(3)
+=======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  int flux_event_subscribe (flux_t *h, const char *topic);
+
+  int flux_event_unsubscribe (flux_t *h, const char *topic);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+Flux events are broadcast across the session, but are only delivered
+to handles that subscribe to them by topic. Topic strings consist of
+one or more words separated by periods, interpreted as a hierarchical
+name space.
+
+:func:`flux_event_subscribe` requests that event messages matching :var:`topic`
+be delivered via :man3:`flux_recv`. A match consists of a string comparison
+of the event topic and the subscription topic, up to the length of the
+subscription topic. Thus "foo." matches events with topics "foo.bar"
+and "foo.baz", and "" matches all events. This matching algorithm
+is inherited from ZeroMQ. Globs or regular expressions are not allowed
+in subscriptions, and the period delimiter is included in the comparison.
+
+:func:`flux_event_unsubscribe` unsubscribes to a topic. The :var:`topic`
+argument must exactly match that provided to :func:`flux_event_subscribe`.
+
+Duplicate subscriptions are allowed in the subscription list but
+will not result in multiple deliveries of a given message. Each
+duplicate subscription requires a separate unsubscribe.
+
+It is not necessary to remove subscriptions with :func:`flux_event_unsubscribe`
+prior to calling :man3:`flux_close`.
+
+
+RETURN VALUE
+============
+
+These functions return 0 on success. On error, -1 is returned,
+and errno is set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   Some arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+
+EXAMPLES
+========
+
+This example opens the Flux broker, subscribes to heartbeat messages,
+displays one, then quits.
+
+.. literalinclude:: example/event.c
+  :language: c
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_fatal_set.adoc b/doc/man3/flux_fatal_set.adoc
deleted file mode 100644
index 5e408623019a..000000000000
--- a/doc/man3/flux_fatal_set.adoc
+++ /dev/null
@@ -1,67 +0,0 @@
-flux_fatal_set(3)
-=================
-:doctype: manpage
-
-
-NAME
-----
-flux_fatal_set, flux_fatal_error, FLUX_FATAL - register/call fatal error
-function
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-typedef void (*flux_fatal_f)(const char *msg, void *arg);
-
-void flux_fatal_set (flux_t *h, flux_fatal_f fun, void *arg);
-
-void flux_fatal_error (flux_t *h, const char *fun, const char *msg);
-
-FLUX_FATAL (flux_t *h);
-
-
-DESCRIPTION
------------
-
-`flux_fatal_set()` configures an optional fatal error function _fun_ to
-be called internally by `libflux_core` if an error occurs that is fatal
-to the handle _h_.  A fatal error is any error that renders the handle
-unusable.  The function may log _msg_, terminate the program,
-or take other action appropriate to the application.
-
-If a fatal error function is not registered, or if the fatal error
-function returns, error handling proceeds as normal.
-
-The fatal error function will only be called once, for the first
-fatal error encountered.
-
-_arg_ is an optional argument passed through to the fatal error function.
-
-`FLUX_FATAL()` is a macro that calls
-....
-flux_fatal_error (h, __FUNCTION__, strerror (errno))
-....
-which translates to a fatal error function called with _msg_ set to
-"function name: error string".
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_open(3)
diff --git a/doc/man3/flux_fd_watcher_create.adoc b/doc/man3/flux_fd_watcher_create.adoc
deleted file mode 100644
index 63839a12795e..000000000000
--- a/doc/man3/flux_fd_watcher_create.adoc
+++ /dev/null
@@ -1,91 +0,0 @@
-flux_fd_watcher_create(3)
-=========================
-:doctype: manpage
-
-
-NAME
-----
-flux_fd_watcher_create, flux_fd_watcher_get_fd -  create file descriptor watcher
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_watcher_f)(flux_reactor_t *r,
-                                flux_watcher_t *w,
-                                int revents, void *arg);
-
- flux_watcher_t *flux_fd_watcher_create (flux_reactor_t *r,
-                                         int fd, int events,
-                                         flux_watcher_f callback,
-                                         void *arg);
-
- int flux_fd_watcher_get_fd (flux_watcher_t *w);
-
-
-DESCRIPTION
------------
-
-`flux_fd_watcher_create()` creates a flux_watcher_t object which can be used
-to monitor for events on a file descriptor _fd_.  When events occur,
-the user-supplied _callback_ is invoked.
-
-The _events_ and _revents_ arguments are a bitmask containing a logical
-``or'' of the following bits.  If a bit is set in _events_, it indicates
-interest in this type of event.  If a bit is set in _revents_, it
-indicates that this event has occurred.
-
-FLUX_POLLIN::
-The file descriptor is ready for reading.
-
-FLUX_POLLOUT::
-The file descriptor is ready for writing.
-
-FLUX_POLLERR::
-The file descriptor has encountered an error.
-This bit is ignored if it is set in the create _events_ argument.
-
-Events are processed in a level-triggered manner.  That is, the callback
-will continue to be invoked as long as the event has not been
-fully consumed or cleared, and the watcher has not been stopped.
-
-`flux_fd_watcher_get_fd()` is used to obtain the file descriptor from
-within the flux_watcher_f callback.
-
-
-RETURN VALUE
-------------
-
-`flux_fd_watcher_create()` returns a flux_watcher_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-`flux_fd_watcher_get_fd()` returns the file descriptor associated with
-the watcher.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_watcher_start(3), flux_reactor_start(3).
diff --git a/doc/man3/flux_fd_watcher_create.rst b/doc/man3/flux_fd_watcher_create.rst
new file mode 100644
index 000000000000..09c98db1048a
--- /dev/null
+++ b/doc/man3/flux_fd_watcher_create.rst
@@ -0,0 +1,85 @@
+=========================
+flux_fd_watcher_create(3)
+=========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_watcher_f)(flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg);
+
+   flux_watcher_t *flux_fd_watcher_create (flux_reactor_t *r,
+                                           int fd,
+                                           int events,
+                                           flux_watcher_f callback,
+                                           void *arg);
+
+   int flux_fd_watcher_get_fd (flux_watcher_t *w);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_fd_watcher_create()` creates a :type:`flux_watcher_t` object which
+can be used to monitor for events on a file descriptor :var:`fd`. When events
+occur, the user-supplied :var:`callback` is invoked.
+
+The :var:`events` and :var:`revents` arguments are a bitmask containing a
+logical OR of the following bits. If a bit is set in :var:`events`, it
+indicates interest in this type of event. If a bit is set in :var:`revents`, it
+indicates that this event has occurred.
+
+FLUX_POLLIN
+   The file descriptor is ready for reading.
+
+FLUX_POLLOUT
+   The file descriptor is ready for writing.
+
+FLUX_POLLERR
+   The file descriptor has encountered an error.
+   This bit is ignored if it is set in the create *events* argument.
+
+Events are processed in a level-triggered manner. That is, the callback
+will continue to be invoked as long as the event has not been
+fully consumed or cleared, and the watcher has not been stopped.
+
+:func:`flux_fd_watcher_get_fd` is used to obtain the file descriptor from
+within the :type:`flux_watcher_f callback`.
+
+
+RETURN VALUE
+============
+
+:func:`flux_fd_watcher_create` returns a :type:`flux_watcher_t` object on
+success.  On error, NULL is returned, and :var:`errno` is set appropriately.
+
+:func:`flux_fd_watcher_get_fd` returns the file descriptor associated with
+the watcher.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_watcher_start`, :man3:`flux_reactor_run`
diff --git a/doc/man3/flux_flags_set.adoc b/doc/man3/flux_flags_set.adoc
deleted file mode 100644
index 07def1a8979f..000000000000
--- a/doc/man3/flux_flags_set.adoc
+++ /dev/null
@@ -1,66 +0,0 @@
-flux_flags_set(3)
-=================
-:doctype: manpage
-
-
-NAME
-----
-flux_flags_set, flux_flags_unset, flux_flags_get - manipulate Flux handle flags
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-void flux_flags_set (flux_t *h, int flags);
-
-void flux_flags_unset (flux_t *h, int flags);
-
-int flux_flags_get (flux_t *h);
-
-
-DESCRIPTION
------------
-
-`flux_flags_set()` sets new open _flags_ in handle _h_.  The resulting
-handle flags will be a logical or of the old flags and the new.
-
-`flux_flags_unset()` clears open _flags_ in handle _h_.  The resulting
-handle flags will be a logical and of the old flags and the inverse of the new.
-
-`flux_flags_get()` can be used to retrieve the current open flags from
-handle _h_.
-
-The valid flags are described in `flux_open(3)`.
-
-
-RETURN VALUE
-------------
-
-`flux_flags_get()` returns the current flags.
-
-
-ERRORS
-------
-
-These functions never fail.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_open(3)
diff --git a/doc/man3/flux_flags_set.rst b/doc/man3/flux_flags_set.rst
new file mode 100644
index 000000000000..880335c34796
--- /dev/null
+++ b/doc/man3/flux_flags_set.rst
@@ -0,0 +1,59 @@
+=================
+flux_flags_set(3)
+=================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  void flux_flags_set (flux_t *h, int flags);
+
+  void flux_flags_unset (flux_t *h, int flags);
+
+  int flux_flags_get (flux_t *h);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_flags_set` sets new open :var:`flags` in handle :var:`h`. The
+resulting handle flags will be a logical or of the old flags and the new.
+
+:func:`flux_flags_unset` clears open :var:`flags` in handle :var:`h`. The
+resulting handle flags will be a logical and of the old flags and the
+inverse of the new.
+
+:func:`flux_flags_get` can be used to retrieve the current open flags from
+handle :var:`h`.
+
+The valid flags are described in :man3:`flux_open`.
+
+
+RETURN VALUE
+============
+
+:func:`flux_flags_get` returns the current flags.
+
+
+ERRORS
+======
+
+These functions never fail.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_open`
diff --git a/doc/man3/flux_future_and_then.adoc b/doc/man3/flux_future_and_then.adoc
deleted file mode 100644
index 97a16c8eb0eb..000000000000
--- a/doc/man3/flux_future_and_then.adoc
+++ /dev/null
@@ -1,125 +0,0 @@
-flux_future_and_then(3)
-=======================
-:doctype: manpage
-
-
-NAME
-----
-flux_future_and_then, flux_future_or_then, flux_future_continue, flux_future_continue_error - functions for sequential composition of futures
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_future_t *flux_future_and_then (flux_future_t *f,
-                                      flux_continuation_f cb, void *arg);
- flux_future_t *flux_future_or_then (flux_future_t *f,
-                                     flux_continuation_f cb, void *arg);
-
- int flux_future_continue (flux_future_t *prev, flux_future_t *f);
- void flux_future_continue_error (flux_future_t *prev, int errnum,
-                                  const char *errstr);
-
-
-
-DESCRIPTION
------------
-
-See `flux_future_get(3)` for general functions that operate on futures,
-and `flux_future_create(3)` for a description of the `flux_future_t`
-base type. This page covers functions for the sequential composition of
-futures, i.e. chains of dependent futures.
-
-`flux_future_and_then(3)` is similar to `flux_future_then(3)`, but
-returns a future that may later be "continued" from the continuation
-callback `cb`. The provided continuation callback `cb` is only 
-executed when the future argument `f` is fulfilled successfully. On
-error, the error from `f` is automatically propagated to the "next"
-future in the chain (returned by the function).
-
-`flux_future_and_then()` is useful when a series of asynchronous
-operations, each returning a `flux_future_t`, depend on the result
-of a previous operation. That is, `flux_future_and_then()` returns a
-placeholder future for an eventual future that can't be created until
-the continuation `cb` is run. The returned future can then be
-used as a synchronization handle or even passed to another
-`flux_future_and_then()` in the chain. By default, the next future
-in the chain will be fulfilled immediately using the result of the
-previous future after return from the callback `cb`. Most callbacks,
-however, should use either `flux_future_continue(3)` or
-`flux_future_continue_error(3)` to pass an intermediate future
-to use in fulfillment of the next future in the chain.
-
-`flux_future_or_then(3)` is like `flux_future_and_then()`, except
-the continuation callback `cb` is run when the future `f` is fulfilled
-with an error. This function is useful for recovery or other error
-handling (other than the default behavior of propagating an error
-down the chain to the final result). The `flux_future_or_then()`
-callback offers a chance to successfully fulfill the "next" future
-in the chain, even when the "previous" future was fulfilled with
-an error.
-
-As with `flux_future_and_then()` the continuation
-`cb` function for `flux_future_or_then()` should call
-`flux_future_continue()` or `flux_future_continue_error()`, or
-the result of the previous future will be propagated immediately
-to the next future in the chain.
-
-`flux_future_continue(3)` continues the next future embedded in `prev`
-(created by `flux_future_and_then()` or `flux_future_or_then()`) with
-the eventual result of the provided future `f`. This allows a future
-that was not created until the context of the callback to continue
-a sequential chain of futures created earlier. After the call to
-`flux_future_continue(3)` completes, the future `prev` may safely be
-destroyed. `flux_future_continue(3)` may be called with `f` equal
-to `NULL` if the caller desires the next future in the chain to
-*not* be fulfilled, in order to disable the automatic fulfillment
-that normally occurs for non-continued futures after the callback
-completes.
-
-`flux_future_continue_error(3)` is like `flux_future_continue()`
-but immediately fulfills the next future in the chain with an error and
-an optional error string.  Once `flux_future_continue_error(3)`
-completes, the future `prev` may safely be destroyed.
-
-RETURN VALUE
-------------
-
-`flux_future_and_then()` and `flux_future_or_then()` return a `flux_future_t`
-on success, or NULL on error. If both functions are called on the same
-future, the returned `flux_future_t` from each will be the same object.
-
-`flux_future_continue()` returns 0 on success, or -1 on error with errno
-set.
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-EINVAL::
-Invalid argument.
-
-ENOENT::
-The requested object is not found.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_future_get(3), flux_future_create(3)
diff --git a/doc/man3/flux_future_and_then.rst b/doc/man3/flux_future_and_then.rst
new file mode 100644
index 000000000000..437d7a9f5c8e
--- /dev/null
+++ b/doc/man3/flux_future_and_then.rst
@@ -0,0 +1,141 @@
+=======================
+flux_future_and_then(3)
+=======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_future_and_then (flux_future_t *f,
+                                        flux_continuation_f cb,
+                                        void *arg);
+
+   flux_future_t *flux_future_or_then (flux_future_t *f,
+                                       flux_continuation_f cb,
+                                       void *arg);
+
+   int flux_future_continue (flux_future_t *prev,
+                             flux_future_t *f);
+
+   void flux_future_continue_error (flux_future_t *prev,
+                                    int errnum,
+                                    const char *errstr);
+
+   int flux_future_fulfill_next (flux_future_t *f,
+                                 void *result,
+                                 flux_free_f free_fn);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+See :man3:`flux_future_get` for general functions that operate on futures,
+and :man3:`flux_future_create` for a description of the :type:`flux_future_t`
+base type. This page covers functions for the sequential composition of
+futures, i.e. chains of dependent futures.
+
+:func:`flux_future_and_then` is similar to :man3:`flux_future_then`, but
+returns a future that may later be "continued" from the continuation
+callback :var:`cb`. The provided continuation callback :var:`cb` is only
+executed when the future argument :var:`f` is fulfilled successfully. On
+error, the error from :var:`f` is automatically propagated to the "next"
+future in the chain (returned by the function).
+
+:func:`flux_future_and_then` is useful when a series of asynchronous
+operations, each returning a :type:`flux_future_t`, depend on the result
+of a previous operation. That is, :func:`flux_future_and_then` returns a
+placeholder future for an eventual future that can't be created until
+the continuation :var:`cb` is run. The returned future can then be
+used as a synchronization handle or even passed to another
+:func:`flux_future_and_then` in the chain. By default, the next future
+in the chain will be fulfilled immediately using the result of the
+previous future after return from the callback :var:`cb`. Most callbacks,
+however, should use either :func:`flux_future_continue` or
+:func:`flux_future_continue_error` to pass an intermediate future
+to use in fulfillment of the next future in the chain.
+
+:func:`flux_future_or_then` is like :func:`flux_future_and_then`, except
+the continuation callback :var:`cb` is run when the future :var:`f` is fulfilled
+with an error. This function is useful for recovery or other error
+handling (other than the default behavior of propagating an error
+down the chain to the final result). The :func:`flux_future_or_then`
+callback offers a chance to successfully fulfill the "next" future
+in the chain, even when the "previous" future was fulfilled with
+an error.
+
+As with :func:`flux_future_and_then` the continuation
+:var:`cb` function for :func:`flux_future_or_then` should call
+:func:`flux_future_continue` or :func:`flux_future_continue_error`, or
+the result of the previous future will be propagated immediately
+to the next future in the chain.
+
+:func:`flux_future_continue` continues the next future embedded in :var:`prev`
+(created by :func:`flux_future_and_then` or :func:`flux_future_or_then`) with
+the eventual result of the provided future :var:`f`. This allows a future
+that was not created until the context of the callback to continue
+a sequential chain of futures created earlier. After the call to
+:func:`flux_future_continue` completes, the future :var:`prev` may safely be
+destroyed. :func:`flux_future_continue` may be called with :var:`f` equal
+to ``NULL`` if the caller desires the next future in the chain to
+**not** be fulfilled, in order to disable the automatic fulfillment
+that normally occurs for non-continued futures after the callback
+completes.
+
+:func:`flux_future_continue_error` is like :func:`flux_future_continue`
+but immediately fulfills the next future in the chain with an error and
+an optional error string. Once :func:`flux_future_continue_error`
+completes, the future :var:`prev` may safely be destroyed.
+
+:func:`flux_future_fulfill_next` is like :man3:`flux_future_fulfill`, but
+fulfills the next future in the chain instead of the current future (which
+is presumably already fulfilled). This call is useful when a chained future
+is being used for post-processing a result from intermediate future-based
+calls, as it allows the next future to be fulfilled with a custom result,
+instead of with the value of another future as in
+:func:`flux_future_continue`.
+
+
+RETURN VALUE
+============
+
+:func:`flux_future_and_then` and :func:`flux_future_or_then` return a
+:type:`flux_future_t` on success, or NULL on error. If both functions are
+called on the same future, the returned :type:`flux_future_t` from each will
+be the same object.
+
+:func:`flux_future_continue` returns 0 on success, or -1 on error with errno
+set.
+
+:func:`flux_future_fulfill_next` returns 0 on success, or -1 with errno set
+to ``EINVAL`` if the target future does not have a next future to fulfill.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+EINVAL
+   Invalid argument.
+
+ENOENT
+   The requested object is not found.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_future_get`, :man3:`flux_future_create`
diff --git a/doc/man3/flux_future_create.adoc b/doc/man3/flux_future_create.adoc
deleted file mode 100644
index c06338b8fa90..000000000000
--- a/doc/man3/flux_future_create.adoc
+++ /dev/null
@@ -1,240 +0,0 @@
-flux_future_create(3)
-=====================
-:doctype: manpage
-
-
-NAME
-----
-flux_future_create, flux_future_fulfill, flux_future_fulfill_error, flux_future_fulfill_with, flux_future_aux_get, flux_future_aux_set, flux_future_set_flux, flux_future_get_flux - support methods for classes that return futures
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_future_init_f)(flux_future_t *f,
-                                    flux_reactor_t *r, void *arg);
-
- flux_future_t *flux_future_create (flux_future_init_f cb, void *arg);
-
- void flux_future_fulfill (flux_future_t *f,
-                           void *result, flux_free_f free_fn);
-
- void flux_future_fulfill_error (flux_future_t *f, int errnum,
-                                 const char *errstr);
-
- void flux_future_fulfill_with (flux_future_t *f, flux_future_t *p);
-
- void flux_future_fatal_error (flux_future_t *f, int errnum,
-                               const char *errstr);
-
- void *flux_future_aux_get (flux_future_t *f, const char *name);
-
- int flux_future_aux_set (flux_future_t *f, const char *name,
-                          void *aux, flux_free_f destroy);
-
- void flux_future_set_reactor (flux_future_t *f, flux_t *h);
-
- flux_reactor_t *flux_future_get_reactor (flux_future_t *f);
-
- void flux_future_set_flux (flux_future_t *f, flux_t *h);
-
- flux_t *flux_future_get_flux (flux_future_t *f);
-
-
-DESCRIPTION
------------
-
-See `flux_future_get(3)` for general functions that operate on futures.
-This page covers functions primarily used when building classes that
-return futures.
-
-A Flux future represents some activity that may be completed with reactor
-watchers and/or message handlers.  It is intended to be returned by other
-classes as a handle for synchronization and a container for results.
-This page describes the future interfaces used by such classes.
-
-A class that returns a future usually provides a creation function
-that internally calls `flux_future_create()`, and may provide functions
-to access class-specific result(s), that internally call `flux_future_get()`.
-The create function internally registers a _flux_future_init_f_
-function that is called lazily by the future implementation to perform
-class-specific reactor setup, such as installing watchers and message
-handlers.
-
-`flux_future_create()` creates a future and registers the
-class-specific initialization callback _cb_, and an opaque argument
-_arg_ that will be passed to _cb_.  The purpose of the initialization
-callback is to set up class-specific watchers on a reactor obtained
-with `flux_future_get_reactor()`, or message handlers on a flux_t
-handle obtained with `flux_future_get_flux()`, or both.
-`flux_future_get_reactor()` and `flux_future_get_flux()` return
-different results depending on whether the initialization callback is
-triggered by a user calling `flux_future_then()` or
-`flux_future_wait_for()`.  The function may be triggered in one or
-both contexts, at most once for each.  The watchers or message
-handlers must eventually call `flux_future_fulfill()`,
-`flux_future_fulfill_error()`, or `flux_future_fatal_error()` to
-fulfill the future.  See REACTOR CONTEXTS below for more information.
-
-`flux_future_fulfill()` fulfills the future, assigning an opaque
-_result_ value with optional destructor _free_fn_ to the future.
-A NULL _result_ is valid and also fulfills the future.  The _result_
-is contained within the future and can be accessed with `flux_future_get()`
-as needed until the future is destroyed.
-
-`flux_future_fulfill_error()` fulfills the future, assigning an
-_errnum_ value and an optional error string.  After the future is
-fulfilled with an error, `flux_future_get()` will return -1 with errno
-set to _errnum_.
-
-`flux_future_fulfill_with()` fulfills the target future _f_ using a
-fulfilled future _p_. This function copies the pending result or error
-from _p_ into _f_, and adds read-only access to the _aux_ items for _p_
-from _f_. This ensures that any `get` method which requires _aux_ items
-for _p_ will work with _f_. This function takes a reference to the source
-future _p_, so it safe to call `flux_future_destroy (p)` after this call.
-`flux_future_fulfill_with()` returns -1 on error with _errno_
-set on failure.
-
-`flux_future_fulfill()`, `flux_future_fulfill_with()`, and
-`flux_future_fulfill_error()` can be called multiple times to queue
-multiple results or errors.  When callers access future results via
-`flux_future_get()`, results or errors will be returned in FIFO order.
-It is an error to call `flux_future_fulfill_with()` multiple times on
-the same target future _f_ with a different source future _p_.
-
-`flux_future_fatal_error()` fulfills the future, assigning an _errnum_
-value and an optional error string.  Unlike
-`flux_future_fulfill_error()` this fulfillment can only be called once
-and takes precedence over all other fulfillments.  It is used for
-catastrophic error paths in future fulfillment.
-
-`flux_future_aux_set()` attaches application-specific data
-to the parent object _f_.  It stores data _aux_ by key _name_,
-with optional destructor _destroy_.  The destructor, if non-NULL,
-is called when the parent object is destroyed, or when
-_key_ is overwritten by a new value.  If _aux_ is NULL,
-the destructor for a previous value, if any is called,
- but no new value is stored.  If _name_ is NULL,
-_aux_ is stored anonymously.
-
-`flux_future_aux_get()` retrieves application-specific data
-by _name_.  If the data was stored anonymously, it
-cannot be retrieved.
-
-Names beginning with "flux::" are reserved for internal use.
-
-`flux_future_set_reactor()` may be used to associate a Flux reactor
-with a future.  The reactor (or a temporary one, depending on the context)
-may be retrieved using `flux_future_get_reactor()`.
-
-`flux_future_set_flux()` may be used to associate a Flux broker handle
-with a future.  The handle (or a clone associated with a temporary reactor,
-depending on the context) may be retrieved using `flux_future_get_flux()`.
-
-Futures may "contain" other futures, to arbitrary depth.  That is, an
-init callback may create futures and use their continuations to fulfill
-the containing future in the same manner as reactor watchers and message
-handlers.
-
-
-REACTOR CONTEXTS
-----------------
-
-Internally, a future can operate in two reactor contexts.  The initialization
-callback may be called in either or both contexts, depending on which
-synchronization functions are called by the user.  `flux_future_get_reactor()`
-and `flux_future_get_flux()` return a result that depends on which context
-they are called from.
-
-When the user calls `flux_future_then()`, this triggers a call to the
-initialization callback.  The callback would typically call
-`flux_future_get_reactor()` and/or `flux_future_get_flux()` to obtain the 
-reactor or flux_t handle to be used to set up watchers or message handlers.
-In this context, the reactor or flux_t handle are exactly the ones passed
-to `flux_future_set_reactor()` and `flux_future_set_flux()`.
-
-When the user calls `flux_future_wait_for()`, this triggers the creation
-of a temporary reactor, then a call to the initialization callback.
-The temporary reactor allows these functions to wait _only_ for the future's
-events, without allowing unrelated watchers registered in the main reactor
-to run, which might complicate the application's control flow.  In this
-context, `flux_future_get_reactor()` returns the temporary reactor, not
-the one passed in with `flux_future_set_reactor()`.  `flux_future_get_flux()`
-returns a temporary flux_t handle cloned from the one passed to
-`flux_future_set_flux()`, and associated with the temporary reactor.
-After the internal reactor returns, any messages unmatched by the dispatcher
-on the cloned handle are requeued in the main flux_t handle with
-`flux_dispatch_requeue()`.
-
-Since the init callback may be made in either reactor context (at most once
-each), and is unaware of which context that is, it should take care when
-managing any context-specific state not to overwrite the state from a prior
-call.  The ability to attach objects with destructors anonymously to the future
-with `flux_future_aux_set()` may be useful for managing the life cycle
-of reactor watchers and message handlers created by init callbacks.
-
-
-RETURN VALUE
-------------
-
-`flux_future_create()` returns a future on success.  On error, NULL is
-returned and errno is set appropriately.
-
-`flux_future_aux_set()` returns zero on success.  On error, -1 is
-returned and errno is set appropriately.
-
-`flux_future_aux_get()` returns the requested object on success.  On
-error, NULL is returned and errno is set appropriately.
-
-`flux_future_get_flux()` returns a flux_t handle on success.  On error,
-NULL is returned and errno is set appropriately.
-
-`flux_future_get_reactor()` returns a flux_reactor_t on success.  On error,
-NULL is returned and errno is set appropriately.
-
-`flux_future_fulfill_with()` returns zero on success. On error, -1 is
-returned with errno set to EINVAL if either _f_ or _p_  is NULL, or
-_f_ and _p_ are the same, EAGAIN if the future _p_ is not ready, or
-EEXIST if the function is called multiple times with different _p_.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-EINVAL::
-Invalid argument.
-
-ENOENT::
-The requested object is not found.
-
-EAGAIN::
-The requested operation is not ready. For `flux_future_fulfill_with()`,
-the target future _p_ is not fulfilled.
-
-EEXIST::
-`flux_future_fulfill_with()` was called multiple times with a different
-target future _p_.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_future_get(3), flux_clone(3)
diff --git a/doc/man3/flux_future_create.rst b/doc/man3/flux_future_create.rst
new file mode 100644
index 000000000000..59f40a97a054
--- /dev/null
+++ b/doc/man3/flux_future_create.rst
@@ -0,0 +1,243 @@
+=====================
+flux_future_create(3)
+=====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_future_init_f)(flux_future_t *f,
+                                      flux_reactor_t *r,
+                                      void *arg);
+
+   flux_future_t *flux_future_create (flux_future_init_f cb, void *arg);
+
+   void flux_future_fulfill (flux_future_t *f,
+                             void *result,
+                             flux_free_f free_fn);
+
+   void flux_future_fulfill_error (flux_future_t *f,
+                                   int errnum,
+                                   const char *errstr);
+
+   void flux_future_fulfill_with (flux_future_t *f, flux_future_t *p);
+
+   void flux_future_fatal_error (flux_future_t *f,
+                                 int errnum,
+                                 const char *errstr);
+
+   void *flux_future_aux_get (flux_future_t *f, const char *name);
+
+   int flux_future_aux_set (flux_future_t *f,
+                            const char *name,
+                            void *aux,
+                            flux_free_f destroy);
+
+   void flux_future_set_reactor (flux_future_t *f, flux_t *h);
+
+   flux_reactor_t *flux_future_get_reactor (flux_future_t *f);
+
+   void flux_future_set_flux (flux_future_t *f, flux_t *h);
+
+   flux_t *flux_future_get_flux (flux_future_t *f);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+See :man3:`flux_future_get` for general functions that operate on futures.
+This page covers functions primarily used when building classes that
+return futures.
+
+A Flux future represents some activity that may be completed with reactor
+watchers and/or message handlers. It is intended to be returned by other
+classes as a handle for synchronization and a container for results.
+This page describes the future interfaces used by such classes.
+
+A class that returns a future usually provides a creation function that
+internally calls :func:`flux_future_create`, and may provide functions to
+access class-specific result(s), that internally call :man3:`flux_future_get`.
+The create function internally registers a :type:`flux_future_init_f`
+function that is called lazily by the future implementation to perform
+class-specific reactor setup, such as installing watchers and message
+handlers.
+
+:func:`flux_future_create` creates a future and registers the
+class-specific initialization callback :var:`cb`, and an opaque argument
+:var:`arg` that will be passed to :var:`cb`. The purpose of the initialization
+callback is to set up class-specific watchers on a reactor obtained
+with :func:`flux_future_get_reactor`, or message handlers on a :type:`flux_t`
+handle obtained with :func:`flux_future_get_flux`, or both.
+:func:`flux_future_get_reactor` and :func:`flux_future_get_flux` return
+different results depending on whether the initialization callback is
+triggered by a user calling :man3:`flux_future_then` or
+:man3:`flux_future_wait_for`. The function may be triggered in one or
+both contexts, at most once for each. The watchers or message
+handlers must eventually call :func:`flux_future_fulfill`,
+:func:`flux_future_fulfill_error`, or :func:`flux_future_fatal_error` to
+fulfill the future. See REACTOR CONTEXTS below for more information.
+
+:func:`flux_future_fulfill` fulfills the future, assigning an opaque
+:var:`result` value with optional destructor :var:`free_fn` to the future.
+A NULL :var:`result` is valid and also fulfills the future. The :var:`result`
+is contained within the future and can be accessed with :man3:`flux_future_get`
+as needed until the future is destroyed.
+
+:func:`flux_future_fulfill_error` fulfills the future, assigning an
+:var:`errnum` value and an optional error string. After the future is
+fulfilled with an error, :man3:`flux_future_get` will return -1 with
+:var:`errno` set to :var:`errnum`.
+
+:func:`flux_future_fulfill_with` fulfills the target future :var:`f` using a
+fulfilled future :var:`p`. This function copies the pending result or error
+from :var:`p` into :var:`f`, and adds read-only access to the :var:`aux` items
+for :var:`p` from :var:`f`. This ensures that any ``get`` method which requires
+:var:`aux` items for :var:`p` will work with :var:`f`. This function takes a
+reference to the source future :var:`p`, so it safe to call
+:man3:`flux_future_destroy` on :var:`p` after this call.
+:func:`flux_future_fulfill_with` returns -1 on error with :var:`errno`
+set on failure.
+
+:func:`flux_future_fulfill`, :func:`flux_future_fulfill_with`, and
+:func:`flux_future_fulfill_error` can be called multiple times to queue
+multiple results or errors. When callers access future results via
+:man3:`flux_future_get`, results or errors will be returned in FIFO order.
+It is an error to call :func:`flux_future_fulfill_with` multiple times on
+the same target future :var:`f` with a different source future :var:`p`.
+
+:func:`flux_future_fatal_error` fulfills the future, assigning an :var:`errnum`
+value and an optional error string. Unlike
+:func:`flux_future_fulfill_error` this fulfillment can only be called once
+and takes precedence over all other fulfillments. It is used for
+catastrophic error paths in future fulfillment.
+
+:func:`flux_future_aux_set` attaches application-specific data
+to the parent object :var:`f`. It stores data :var:`aux` by key :var:`name`,
+with optional destructor *destroy*. The destructor, if non-NULL,
+is called when the parent object is destroyed, or when
+:var:`key` is overwritten by a new value. If :var:`aux` is NULL,
+the destructor for a previous value, if any is called,
+but no new value is stored. If :var:`name` is NULL,
+:var:`aux` is stored anonymously.
+
+:func:`flux_future_aux_get` retrieves application-specific data
+by :var:`name`. If the data was stored anonymously, it
+cannot be retrieved.
+
+Names beginning with "flux::" are reserved for internal use.
+
+:func:`flux_future_set_reactor` may be used to associate a Flux reactor
+with a future. The reactor (or a temporary one, depending on the context)
+may be retrieved using :func:`flux_future_get_reactor`.
+
+:func:`flux_future_set_flux` may be used to associate a Flux broker handle
+with a future. The handle (or a clone associated with a temporary reactor,
+depending on the context) may be retrieved using :func:`flux_future_get_flux`.
+
+Futures may "contain" other futures, to arbitrary depth. That is, an
+init callback may create futures and use their continuations to fulfill
+the containing future in the same manner as reactor watchers and message
+handlers.
+
+
+REACTOR CONTEXTS
+================
+
+Internally, a future can operate in two reactor contexts. The initialization
+callback may be called in either or both contexts, depending on which
+synchronization functions are called by the user.
+:func:`flux_future_get_reactor` and :func:`flux_future_get_flux` return a
+result that depends on which context they are called from.
+
+When the user calls :man3:`flux_future_then`, this triggers a call to the
+initialization callback. The callback would typically call
+:func:`flux_future_get_reactor` and/or :func:`flux_future_get_flux()` to obtain
+the reactor or :type:`flux_t` handle to be used to set up watchers or message
+handlers.  In this context, the reactor or :type:`flux_t` handle are exactly
+the ones passed to :func:`flux_future_set_reactor` and
+:func:`flux_future_set_flux`.
+
+When the user calls :man3:`flux_future_wait_for`, this triggers the creation
+of a temporary reactor, then a call to the initialization callback.
+The temporary reactor allows these functions to wait *only* for the future's
+events, without allowing unrelated watchers registered in the main reactor
+to run, which might complicate the application's control flow. In this
+context, :func:`flux_future_get_reactor` returns the temporary reactor, not
+the one passed in with :func:`flux_future_set_reactor`.
+:func:`flux_future_get_flux` returns a temporary :type:`flux_t` handle cloned
+from the one passed to :func:`flux_future_set_flux`, and associated with the
+temporary reactor.
+After the internal reactor returns, any messages unmatched by the dispatcher
+on the cloned handle are requeued in the main :type:`flux_t` handle with
+:func:`flux_dispatch_requeue`.
+
+Since the init callback may be made in either reactor context (at most once
+each), and is unaware of which context that is, it should take care when
+managing any context-specific state not to overwrite the state from a prior
+call. The ability to attach objects with destructors anonymously to the future
+with :func:`flux_future_aux_set` may be useful for managing the life cycle
+of reactor watchers and message handlers created by init callbacks.
+
+
+RETURN VALUE
+============
+
+:func:`flux_future_create` returns a future on success. On error, NULL is
+returned and :var:`errno` is set appropriately.
+
+:func:`flux_future_aux_set` returns zero on success. On error, -1 is
+returned and :var:`errno` is set appropriately.
+
+:func:`flux_future_aux_get` returns the requested object on success. On
+error, NULL is returned and :var:`errno` is set appropriately.
+
+:func:`flux_future_get_flux` returns a :type:`flux_t` handle on success.
+On error, NULL is returned and :var:`errno` is set appropriately.
+
+:func:`flux_future_get_reactor` returns a :type:`flux_reactor_t` on success.
+On error, NULL is returned and :var:`errno` is set appropriately.
+
+:func:`flux_future_fulfill_with` returns zero on success. On error, -1 is
+returned with :var:`errno` set to EINVAL if either :var:`f` or :var:`p` is
+NULL, or :var:`f` and :var:`p` are the same, EAGAIN if the future :var:`p` is
+not ready, or EEXIST if the function is called multiple times with different
+:var:`p`.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+EINVAL
+   Invalid argument.
+
+ENOENT
+   The requested object is not found.
+
+EAGAIN
+   The requested operation is not ready. For :func:`flux_future_fulfill_with`,
+   the target future :var:`p` is not fulfilled.
+
+EEXIST
+   :func:`flux_future_fulfill_with` was called multiple times with a different
+   target future :var:`p`.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_future_get`, :man3:`flux_clone`
diff --git a/doc/man3/flux_future_get.adoc b/doc/man3/flux_future_get.adoc
deleted file mode 100644
index 7a2bbc3bdeec..000000000000
--- a/doc/man3/flux_future_get.adoc
+++ /dev/null
@@ -1,140 +0,0 @@
-flux_future_get(3)
-==================
-:doctype: manpage
-
-
-NAME
-----
-flux_future_get, flux_future_then, flux_future_wait_for, flux_future_reset, flux_future_destroy - synchronize an activity
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_continuation_f)(flux_future_t *f, void *arg);
-
- int flux_future_get (flux_future_t *f, const void **result);
-
- int flux_future_then (flux_future_t *f, double timeout,
-                       flux_continuation_f cb, void *arg);
-
- int flux_future_wait_for (flux_future_t *f, double timeout);
-
- void flux_future_reset (flux_future_t *f);
-
- void flux_future_destroy (flux_future_t *f);
-
- bool flux_future_has_error (flux_future_t *f);
-
- const char *flux_future_error_string (flux_future_t *f);
-
-OVERVIEW
---------
-
-A Flux future represents some activity that may be completed with reactor
-watchers and/or message handlers.  It is both a handle for synchronization
-and a container for the result.  A Flux future is said to be "fulfilled"
-when a result is available in the future container, or a fatal error has
-occurred.  Flux futures were inspired by similar constructs in other
-programming environments mentioned in RESOURCES, but are not a faithful
-implementation of any particular one.
-
-Generally other Flux classes return futures, and may provide class-specific
-access function for results. The functions described in this page can be
-used to access, synchronize, and destroy futures returned from any such class.
-Authors of classes that return futures are referred to `flux_future_create(3)`.
-
-DESCRIPTION
------------
-
-`flux_future_get()` accesses the result of a fulfilled future.  If the
-future is not yet fulfilled, it calls `flux_future_wait_for()` internally
-with a negative _timeout_, causing it to block until the future is fulfilled.
-A pointer to the result is assigned to _result_ (caller must NOT free),
-or -1 is returned if the future was fulfilled with an error.
-
-`flux_future_then()` sets up a continuation callback _cb_ that is called
-with opaque argument _arg_ once the future is fulfilled.  The continuation
-will normally use `flux_future_get()` or a class-specific access function
-to obtain the result from the future container without blocking.  The
-continuation may call `flux_future_destroy()` or `flux_future_reset()`.
-If _timeout_ is non-negative, the future must be fulfilled within the
-specified amount of time or the timeout fulfills it with an error (errno
-set to ETIMEDOUT).
-
-`flux_future_wait_for()` blocks until the future is fulfilled, or _timeout_
-(if non-negative) expires.  This function may be called multiple times,
-with different values for _timeout_.  If the timeout expires before
-the future is fulfilled, an error is returned (errno set to ETIMEDOUT)
-but the future remains unfulfilled.  If _timeout_ is zero, function times
-out immediately if the future has not already been fulfilled.
-
-`flux_future_reset()` unfulfills a future, invalidating any result stored
-in the container, and preparing it to be fulfilled once again.  If a
-continuation was registered, it remains in effect for the next fulfillment,
-however any timeout will have been cleared by the current fulfillment
-and must be re-established by following the `flux_future_reset()` with
-another `flux_future_then()`, if desired. 
-
-`flux_future_destroy()` destroys a future, including any result contained
-within.
-
-`flux_future_has_error()` tests if an error exists in the future or not.
-It can be useful for determining if an error exists in a future or in
-other parts of code that may wrap around a future.  It is commonly
-called before calling `flux_future_error_string()`.
-
-`flux_future_error_string()` returns the error string stored in a
-future.  If the future was fulfilled with an optional error string,
-`flux_future_error_string()` will return that string.  Otherwise, it
-will return the string associated with the error number set in a
-future.  If the future is a NULL pointer, not fulfilled, or fulfilled
-with a non-error, NULL is returned.
-
-RETURN VALUE
-------------
-
-`flux_future_then()`, `flux_future_get()`, and `flux_future_wait_for()`
-return zero on success.  On error, -1 is returned, and errno is set
-appropriately.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-EINVAL::
-Invalid argument.
-
-ETIMEDOUT::
-A timeout passed to `flux_future_wait_for()` expired before the future
-was fulfilled.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-C++ std::future: <http://en.cppreference.com/w/cpp/thread/future>
-
-Java util.concurrent.Future: <https://docs.oracle.com/javase/7/docs/api/java/util/concurrent/Future.html>
-
-Python3 concurrent.futures: <https://docs.python.org/3/library/concurrent.futures.html>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_future_create (3)
diff --git a/doc/man3/flux_future_get.rst b/doc/man3/flux_future_get.rst
new file mode 100644
index 000000000000..9253fe9fd047
--- /dev/null
+++ b/doc/man3/flux_future_get.rst
@@ -0,0 +1,140 @@
+==================
+flux_future_get(3)
+==================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_continuation_f)(flux_future_t *f, void *arg);
+
+   int flux_future_get (flux_future_t *f, const void **result);
+
+   int flux_future_then (flux_future_t *f,
+                         double timeout,
+                         flux_continuation_f cb,
+                         void *arg);
+
+   int flux_future_wait_for (flux_future_t *f, double timeout);
+
+   void flux_future_reset (flux_future_t *f);
+
+   void flux_future_destroy (flux_future_t *f);
+
+   bool flux_future_has_error (flux_future_t *f);
+
+   const char *flux_future_error_string (flux_future_t *f);
+
+Link with :command:`-lflux-core`.
+
+OVERVIEW
+========
+
+A Flux future represents some activity that may be completed with reactor
+watchers and/or message handlers. It is both a handle for synchronization
+and a container for the result. A Flux future is said to be "fulfilled"
+when a result is available in the future container, or a fatal error has
+occurred. Flux futures were inspired by similar constructs in other
+programming environments mentioned in RESOURCES, but are not a faithful
+implementation of any particular one.
+
+Generally other Flux classes return futures, and may provide class-specific
+access function for results. The functions described in this page can be
+used to access, synchronize, and destroy futures returned from any such class.
+Authors of classes that return futures are referred to :man3:`flux_future_create`.
+
+
+DESCRIPTION
+===========
+
+:func:`flux_future_get` accesses the result of a fulfilled future. If the
+future is not yet fulfilled, it calls :func:`flux_future_wait_for` internally
+with a negative :var:`timeout`, causing it to block until the future is
+fulfilled.  A pointer to the result is assigned to :var:`result` (caller must
+NOT free), or -1 is returned if the future was fulfilled with an error.
+
+:func:`flux_future_then` sets up a continuation callback :var:`cb` that is
+called with opaque argument :var:`arg` once the future is fulfilled. The
+continuation will normally use :func:`flux_future_get` or a class-specific
+access function to obtain the result from the future container without
+blocking. The continuation may call :func:`flux_future_destroy` or
+:func:`flux_future_reset`.
+If :var:`timeout` is non-negative, the future must be fulfilled within the
+specified amount of time or the timeout fulfills it with an error (:var:`errno`
+set to ETIMEDOUT).
+
+:func:`flux_future_wait_for` blocks until the future is fulfilled, or
+:var:`timeout` (if non-negative) expires. This function may be called multiple
+times, with different values for :var:`timeout`. If the timeout expires before
+the future is fulfilled, an error is returned (:var:`errno` set to ETIMEDOUT)
+but the future remains unfulfilled. If :var:`timeout` is zero, function times
+out immediately if the future has not already been fulfilled.
+
+:func:`flux_future_reset` unfulfills a future, invalidating any result stored
+in the container, and preparing it to be fulfilled once again. If a
+continuation was registered, it remains in effect for the next fulfillment.
+If a timeout was specified when the continuation was registered, it is
+restarted.
+
+:func:`flux_future_destroy` destroys a future, including any result contained
+within.
+
+:func:`flux_future_has_error` tests if an error exists in the future or not.
+It can be useful for determining if an error exists in a future or in
+other parts of code that may wrap around a future. It is commonly
+called before calling :func:`flux_future_error_string`.
+
+:func:`flux_future_error_string` returns the error string stored in a
+future. If the future was fulfilled with an optional error string,
+:func:`flux_future_error_string` will return that string. Otherwise, it
+will return the string associated with the error number set in a
+future. If the future is a NULL pointer, not fulfilled, or fulfilled
+with a non-error, NULL is returned.
+
+
+RETURN VALUE
+============
+
+:func:`flux_future_then`, :func:`flux_future_get`, and
+:func:`flux_future_wait_for` return zero on success. On error, -1 is returned,
+and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+EINVAL
+   Invalid argument.
+
+ETIMEDOUT
+   A timeout passed to :func:`flux_future_wait_for` expired before the future
+   was fulfilled.
+
+EDEADLOCK (or EDEADLK on BSD systems)
+   :func:`flux_future_wait_for` would likely deadlock due to an
+   improperly initialized future.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+C++ std::future: http://en.cppreference.com/w/cpp/thread/future
+
+Java ``util.concurrent.Future``: https://docs.oracle.com/javase/7/docs/api/java/util/concurrent/Future.html
+
+Python3 concurrent.futures: https://docs.python.org/3/library/concurrent.futures.html
+
+
+SEE ALSO
+========
+
+:man3:`flux_future_create`
diff --git a/doc/man3/flux_future_wait_all_create.adoc b/doc/man3/flux_future_wait_all_create.adoc
deleted file mode 100644
index 1e2f4fe02cab..000000000000
--- a/doc/man3/flux_future_wait_all_create.adoc
+++ /dev/null
@@ -1,126 +0,0 @@
-flux_future_wait_all_create(3)
-==============================
-:doctype: manpage
-
-
-NAME
-----
-flux_future_wait_all_create, flux_future_wait_any_create, flux_future_push, flux_future_first_child, flux_future_next_child, flux_future_get_child - functions for future composition
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_future_t *flux_future_wait_all_create (void);
- flux_future_t *flux_future_wait_any_create (void);
- 
- int flux_future_push (flux_future_t *cf, const char *name, flux_future_t *f);
-
- const char *flux_future_first_child (flux_future_t *cf);
- const char *flux_future_next_child (flux_future_t *cf);
- flux_future_t *flux_future_get_child (flux_future_t *cf, const char *name);
-
-
-DESCRIPTION
------------
-
-See `flux_future_get(3)` for general functions that operate on futures,
-and `flux_future_create(3)` for a description of the `flux_future_t`
-base type. This page covers functions used for composing futures into
-composite types using containers that allow waiting on all or any of a
-set of child futures.
-
-`flux_future_wait_all_create(3)` creates a future that is an empty
-container for other futures, which can subsequently be pushed into
-the container using `flux_future_push(3)`. The returned future will
-be automatically fulfilled when *all* children futures have been
-fulfilled.  The caller may then use `flux_future_first_child(3)`,
-`flux_future_next_child(3)`, and/or `flux_future_get_child(3)` and
-expect that `flux_future_get(3)` will not block for any of these child
-futures. This function is useful to synchronize on a series of futures
-that may be run in parallel.
-
-`flux_future_wait_any_create(3)` creates a composite future that will be
-fulfilled once *any* one of its children are fulfilled. Once the composite
-future is fulfilled, the caller will need to traverse the child futures
-to determine which was fulfilled. This function is useful to synchronize
-on work where any one of several results is sufficient to continue.
-
-`flux_future_push(3)` places a new child future `f` into a future
-composite created by either `flux_future_wait_all_create(3)` or
-`flux_future_wait_any_create(3)`. A `name` is provided for the child so
-that the child future can be easily differentiated from other futures
-inside the container once the composite future is fulfilled.
-
-Once a `flux_future_t` is pushed onto a composite future with
-`flux_future_push(3)`, the memory for the child future is "adopted" by
-the new parent. Thus, calling `flux_future_destroy(3)` on the parent
-composite will destroy all children. Therefore, child futures that
-have been the target of `flux_future_push(3)` should *not* have
-flux_future_destroy(3)` called upon them to avoid double-free.
-
-`flux_future_first_child(3)` and `flux_future_next_child(3)` are used to
-iterate over child future names in a composite future created with either
-`flux_future_wait_all_create(3)` or `flux_future_wait_any_create(3)`. The
-`flux_future_t` corresponding to the returned _name_ can be then
-fetched with `flux_future_get_child(3)`. `flux_future_next_child` will
-return a `NULL` once all children have been iterated.
-
-
-`flux_future_get_child(3)` retrieves a child future from a composite
-by name.
-
-
-
-RETURN VALUE
-------------
-
-`flux_future_wait_any_create()` and `flux_future_wait_all_create()` return
-a future on success.  On error, NULL is returned and errno is set appropriately.
-
-`flux_future_push()` returns zero on success.  On error, -1 is
-returned and errno is set appropriately.
-
-`flux_future_first_child()` returns the name of the first child future in
-the targeted composite in no given order. If the composite is empty,
-a NULL is returned.
-
-`flux_future_next_child()` returns the name of the next child future in the
-targeted composite in no given order. If the last child has already been
-returned then this function returns NULL.
-
-`flux_future_get_child()` returns a `flux_future_t` corresponding to the
-child future with the supplied string `name` parameter. If no future with
-that name is a child of the composite, then the function returns NULL.
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-EINVAL::
-Invalid argument.
-
-ENOENT::
-The requested object is not found.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_future_get(3), flux_future_create(3)
diff --git a/doc/man3/flux_future_wait_all_create.rst b/doc/man3/flux_future_wait_all_create.rst
new file mode 100644
index 000000000000..647ff94fc746
--- /dev/null
+++ b/doc/man3/flux_future_wait_all_create.rst
@@ -0,0 +1,125 @@
+==============================
+flux_future_wait_all_create(3)
+==============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_future_wait_all_create (void);
+
+   flux_future_t *flux_future_wait_any_create (void);
+
+   int flux_future_push (flux_future_t *cf,
+                         const char *name,
+                         flux_future_t *f);
+
+   const char *flux_future_first_child (flux_future_t *cf);
+
+   const char *flux_future_next_child (flux_future_t *cf);
+
+   flux_future_t *flux_future_get_child (flux_future_t *cf,
+                                         const char *name);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+See :man3:`flux_future_get` for general functions that operate on futures,
+and :man3:`flux_future_create` for a description of the :type:`flux_future_t`
+base type. This page covers functions used for composing futures into
+composite types using containers that allow waiting on all or any of a
+set of child futures.
+
+:func:`flux_future_wait_all_create` creates a future that is an empty
+container for other futures, which can subsequently be pushed into
+the container using :func:`flux_future_push`. The returned future will
+be automatically fulfilled when **all** children futures have been
+fulfilled. The caller may then use :func:`flux_future_first_child`,
+:func:`flux_future_next_child`, and/or :func:`flux_future_get_child` and
+expect that :man3:`flux_future_get` will not block for any of these child
+futures. This function is useful to synchronize on a series of futures
+that may be run in parallel.
+
+:func:`flux_future_wait_any_create` creates a composite future that will be
+fulfilled once **any** one of its children are fulfilled. Once the composite
+future is fulfilled, the caller will need to traverse the child futures
+to determine which was fulfilled. This function is useful to synchronize
+on work where any one of several results is sufficient to continue.
+
+:func:`flux_future_push` places a new child future :var:`f` into a future
+composite created by either :func:`flux_future_wait_all_create` or
+:func:`flux_future_wait_any_create`. A :var:`name` is provided for the child so
+that the child future can be easily differentiated from other futures
+inside the container once the composite future is fulfilled.
+
+Once a :type:`flux_future_t` is pushed onto a composite future with
+:func:`flux_future_push`, the memory for the child future is "adopted" by
+the new parent. Thus, calling :man3:`flux_future_destroy` on the parent
+composite will destroy all children. Therefore, child futures that
+have been the target of :func:`flux_future_push` should **not** have
+:man3:`flux_future_destroy` called upon them to avoid double-free.
+
+:func:`flux_future_first_child` and :func:`flux_future_next_child` are used to
+iterate over child future names in a composite future created with either
+:func:`flux_future_wait_all_create` or :func:`flux_future_wait_any_create`. The
+:type:`flux_future_t` corresponding to the returned :var:`name` can be then
+fetched with :func:`flux_future_get_child`. :func:`flux_future_next_child` will
+return a ``NULL`` once all children have been iterated.
+
+:func:`flux_future_get_child` retrieves a child future from a composite
+by name.
+
+
+RETURN VALUE
+============
+
+:func:`flux_future_wait_any_create` and :func:`flux_future_wait_all_create`
+return a future on success. On error, NULL is returned and :var:`errno` is set
+appropriately.
+
+:func:`flux_future_push` returns zero on success. On error, -1 is
+returned and :var:`errno` is set appropriately.
+
+:func:`flux_future_first_child` returns the name of the first child future in
+the targeted composite in no given order. If the composite is empty,
+a NULL is returned.
+
+:func:`flux_future_next_child` returns the name of the next child future in the
+targeted composite in no given order. If the last child has already been
+returned then this function returns NULL.
+
+:func:`flux_future_get_child` returns a :type:`flux_future_t` corresponding to
+the child future with the supplied string :var:`name` parameter. If no future
+with that name is a child of the composite, then the function returns NULL.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+EINVAL
+   Invalid argument.
+
+ENOENT
+   The requested object is not found.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_future_get`, :man3:`flux_future_create`
diff --git a/doc/man3/flux_get_rank.adoc b/doc/man3/flux_get_rank.adoc
deleted file mode 100644
index 273bb666357e..000000000000
--- a/doc/man3/flux_get_rank.adoc
+++ /dev/null
@@ -1,69 +0,0 @@
-flux_get_rank(3)
-================
-:doctype: manpage
-
-
-NAME
-----
-flux_get_rank, flux_get_size - query Flux broker comms info
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-int flux_get_rank (flux_t *h, uint32_t *rank);
-
-int flux_get_size (flux_t *h, uint32_t *size);
-
-
-DESCRIPTION
------------
-
-`flux_get_rank()` and `flux_get_size()` ask the
-Flux broker for its rank in the comms session, and the size of the comms
-session.
-
-Session ranks are numbered 0 through size - 1.
-
-
-RETURN VALUE
-------------
-
-These functions return zero on success.  On error, -1 is returned, and errno
-is set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-Some arguments were invalid.
-
-EXAMPLES
---------
-
-Example:
-....
-include::tinfo.c[]
-....
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-https://github.com/flux-framework/rfc/blob/master/spec_3.adoc[RFC 3: CMB1 - Flux Comms Message Broker Protocol]
diff --git a/doc/man3/flux_get_rank.rst b/doc/man3/flux_get_rank.rst
new file mode 100644
index 000000000000..7b6d45c0ab95
--- /dev/null
+++ b/doc/man3/flux_get_rank.rst
@@ -0,0 +1,56 @@
+================
+flux_get_rank(3)
+================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  int flux_get_rank (flux_t *h, uint32_t *rank);
+
+  int flux_get_size (flux_t *h, uint32_t *size);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_get_rank` and :func:`flux_get_size` ask the
+Flux broker for its rank in the Flux instance, and the size of the Flux
+instance.
+
+Ranks are numbered 0 through size - 1.
+
+
+RETURN VALUE
+============
+
+These functions return zero on success. On error, -1 is returned, and errno
+is set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   Some arguments were invalid.
+
+
+EXAMPLES
+========
+
+Example:
+
+.. literalinclude:: example/info.c
+  :language: c
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_get_reactor.adoc b/doc/man3/flux_get_reactor.adoc
deleted file mode 100644
index 9db0910e25f4..000000000000
--- a/doc/man3/flux_get_reactor.adoc
+++ /dev/null
@@ -1,76 +0,0 @@
-flux_get_reactor(3)
-===================
-:doctype: manpage
-
-
-NAME
-----
-flux_get_reactor, flux_set_reactor - get/set reactor associated with broker handle
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-flux_reactor_t *flux_get_reactor (flux_t *h);
-
-int flux_set_reactor (flux_t *h, flux_reactor_t *r);
-
-
-DESCRIPTION
------------
-
-`flux_get_reactor()` retrieves a flux_reactor_t object previously
-associated with the broker handle _h_ by a call to `flux_set_reactor()`.
-If one has not been previously associated, a flux_reactor_t object is created
-on demand.  If the flux_reactor_t object is created on demand, it will be
-destroyed when the handle is destroyed, otherwise it is the responsibility
-of the owner to destroy it after the handle is destroyed.
-
-`flux_set_reactor()` associates a flux_reactor_t object _r_ with a broker
-handle _h_.  A flux_reactor_t object may be obtained from another handle,
-for example when events from multiple handles are to be managed using
-a common flux_reactor_t, or one may be created directly with
-`flux_reactor_create(3)`.  `flux_set_reactor()` should be called
-immediately after `flux_open(3)` to avoid conflict with other API calls
-which may internally call `flux_get_reactor()`.
-
-
-RETURN VALUE
-------------
-
-`flux_get_reactor()` returns a flux_reactor_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-`flux_set_reactor()` returns 0 on success, or -1 on failure with
-errno set appropriately.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-EEXIST::
-Handle already has a reactor association.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_reactor_create(3), flux_reactor_destroy(3)
diff --git a/doc/man3/flux_get_reactor.rst b/doc/man3/flux_get_reactor.rst
new file mode 100644
index 000000000000..79e21a3600f2
--- /dev/null
+++ b/doc/man3/flux_get_reactor.rst
@@ -0,0 +1,69 @@
+===================
+flux_get_reactor(3)
+===================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  flux_reactor_t *flux_get_reactor (flux_t *h);
+
+  int flux_set_reactor (flux_t *h, flux_reactor_t *r);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_get_reactor` retrieves a :type:`flux_reactor_t` object previously
+associated with the broker handle :var:`h` by a call to
+:func:`flux_set_reactor`.  If one has not been previously associated,
+a :type:`flux_reactor_t` object is created on demand. If the
+:type:`flux_reactor_t` object is created on demand, it will be destroyed when
+the handle is destroyed, otherwise it is the responsibility of the owner to
+destroy it after the handle is destroyed.
+
+:func:`flux_set_reactor` associates a :type:`flux_reactor_t` object :var:`r`
+with a broker handle :var:`h`. A :type:`flux_reactor_t` object may be obtained
+from another handle, for example when events from multiple handles are to be
+managed using a common :type:`flux_reactor_t`, or one may be created directly
+with :man3:`flux_reactor_create`. :func:`flux_set_reactor` should be called
+immediately after :man3:`flux_open` to avoid conflict with other API calls
+which may internally call :func:`flux_get_reactor`.
+
+
+RETURN VALUE
+============
+
+:func:`flux_get_reactor` returns a :type:`flux_reactor_t` object on success.
+On error, NULL is returned, and :var:`errno` is set appropriately.
+
+:func:`flux_set_reactor` returns 0 on success, or -1 on failure with
+:var:`errno` set appropriately.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+EEXIST
+   Handle already has a reactor association.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_future_create`, :man3:`flux_reactor_destroy`
diff --git a/doc/man3/flux_handle_watcher_create.adoc b/doc/man3/flux_handle_watcher_create.adoc
deleted file mode 100644
index 537546aad460..000000000000
--- a/doc/man3/flux_handle_watcher_create.adoc
+++ /dev/null
@@ -1,91 +0,0 @@
-flux_handle_watcher_create(3)
-=============================
-:doctype: manpage
-
-
-NAME
-----
-flux_handle_watcher_create, flux_handle_watcher_get_flux -  create broker handle watcher
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_watcher_f)(flux_reactor_t *r,
-                                flux_watcher_t *w,
-                                int revents, void *arg);
-
- flux_watcher_t *flux_handle_watcher_create (flux_reactor_t *r,
-                                             flux_t *h, int events,
-                                             flux_watcher_f callback,
-                                             void *arg);
-
- flux_t *flux_handle_watcher_get_flux (flux_watcher_t *w);
-
-
-DESCRIPTION
------------
-
-`flux_handle_watcher_create()` creates a flux_watcher_t object which
-monitors for events on a Flux broker handle _h_.  When events occur,
-the user-supplied _callback_ is invoked.
-
-The _events_ and _revents_ arguments are a bitmask containing a
-logical ``or'' of the following bits.  If a bit is set in _events_,
-it indicates interest in this type of event.  If a bit is set in _revents_,
-it indicates that this event has occurred.
-
-FLUX_POLLIN::
-The handle is ready for reading.
-
-FLUX_POLLOUT::
-The handle is ready for writing.
-
-FLUX_POLLERR::
-The handle has encountered an error.
-This bit is ignored if it is set in _events_.
-
-Events are processed in a level-triggered manner.  That is, the
-callback will continue to be invoked as long as the event has not been
-fully consumed or cleared, and the watcher has not been stopped.
-
-`flux_handle_watcher_get_flux()` is used to obtain the handle from
-within the callback.
-
-
-RETURN VALUE
-------------
-
-`flux_handle_watcher_create()` returns a flux_watcher_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-`flux_handle_watcher_get_flux()` returns the handle associated with
-the watcher.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_watcher_start(3), flux_reactor_start(3), flux_recv(3), flux_send(3).
diff --git a/doc/man3/flux_handle_watcher_create.rst b/doc/man3/flux_handle_watcher_create.rst
new file mode 100644
index 000000000000..365a33d1c49f
--- /dev/null
+++ b/doc/man3/flux_handle_watcher_create.rst
@@ -0,0 +1,86 @@
+=============================
+flux_handle_watcher_create(3)
+=============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_watcher_f)(flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg);
+
+   flux_watcher_t *flux_handle_watcher_create (flux_reactor_t *r,
+                                               flux_t *h,
+                                               int events,
+                                               flux_watcher_f callback,
+                                               void *arg);
+
+   flux_t *flux_handle_watcher_get_flux (flux_watcher_t *w);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_handle_watcher_create` creates a :type:`flux_watcher_t` object
+which monitors for events on a Flux broker handle :var:`h`. When events occur,
+the user-supplied :var:`callback` is invoked.
+
+The :var:`events` and :var:`revents` arguments are a bitmask containing a
+logical OR of the following bits. If a bit is set in :var:`events`,
+it indicates interest in this type of event. If a bit is set in :var:`revents`,
+it indicates that this event has occurred.
+
+FLUX_POLLIN
+   The handle is ready for reading.
+
+FLUX_POLLOUT
+   The handle is ready for writing.
+
+FLUX_POLLERR
+   The handle has encountered an error.
+   This bit is ignored if it is set in :var:`events`.
+
+Events are processed in a level-triggered manner. That is, the
+callback will continue to be invoked as long as the event has not been
+fully consumed or cleared, and the watcher has not been stopped.
+
+:func:`flux_handle_watcher_get_flux` is used to obtain the handle from
+within the callback.
+
+
+RETURN VALUE
+============
+
+:func:`flux_handle_watcher_create` returns a :type:`flux_watcher_t` object
+on success.  On error, NULL is returned, and errno is set appropriately.
+
+:func:`flux_handle_watcher_get_flux` returns the handle associated with
+the watcher.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_watcher_start`, :man3:`flux_reactor_run`,
+:man3:`flux_recv`, :man3:`flux_send`
diff --git a/doc/man3/flux_idle_watcher_create.adoc b/doc/man3/flux_idle_watcher_create.adoc
deleted file mode 100644
index 8a589d4cadbe..000000000000
--- a/doc/man3/flux_idle_watcher_create.adoc
+++ /dev/null
@@ -1,88 +0,0 @@
-flux_idle_watcher_create(3)
-===========================
-:doctype: manpage
-
-
-NAME
-----
-flux_idle_watcher_create, flux_prepare_watcher_create, flux_check_watcher_create - create prepare/check/idle watchers
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_watcher_f)(flux_reactor_t *r,
-                                flux_watcher_t *w,
-                                int revents, void *arg);
-
- flux_watcher_t *flux_prepare_watcher_create (flux_reactor_t *r,
-                                              flux_watcher_f callback,
-                                              void *arg);
-
- flux_watcher_t *flux_check_watcher_create (flux_reactor_t *r,
-                                            flux_watcher_f callback,
-                                            void *arg);
-
- flux_watcher_t *flux_idle_watcher_create (flux_reactor_t *r,
-                                           flux_watcher_f callback,
-                                           void *arg);
-
-
-DESCRIPTION
------------
-
-`flux_prepare_watcher_create()`, `flux_check_watcher_create()`, and
-`flux_idle_watcher_create()` create specialized reactor watchers with
-the following properties:
-
-The prepare watcher is called by the reactor loop immediately before
-blocking, while the check watcher is called by the reactor loop
-immediately after blocking.
-
-The idle watcher is always run when no other events are pending,
-excluding other idle watchers, prepare and check watchers.
-While it is active, the reactor loop does not block waiting for
-new events.
-
-The callback _revents_ argument should be ignored.
-
-Note: the Flux reactor is based on libev.  For additional information
-on the behavior of these watchers, refer to the libev documentation on
-`ev_idle`, `ev_prepare`, and `ev_check`.
-
-
-RETURN VALUE
-------------
-
-These functions return a flux_watcher_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_watcher_start(3), flux_reactor_start(3)
-
-http://software.schmorp.de/pkg/libev.html[libev home page]
diff --git a/doc/man3/flux_idle_watcher_create.rst b/doc/man3/flux_idle_watcher_create.rst
new file mode 100644
index 000000000000..1ea5b23ddc9b
--- /dev/null
+++ b/doc/man3/flux_idle_watcher_create.rst
@@ -0,0 +1,81 @@
+===========================
+flux_idle_watcher_create(3)
+===========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_watcher_f)(flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg);
+
+   flux_watcher_t *flux_prepare_watcher_create (flux_reactor_t *r,
+                                                flux_watcher_f callback,
+                                                void *arg);
+
+   flux_watcher_t *flux_check_watcher_create (flux_reactor_t *r,
+                                              flux_watcher_f callback,
+                                              void *arg);
+
+   flux_watcher_t *flux_idle_watcher_create (flux_reactor_t *r,
+                                             flux_watcher_f callback,
+                                             void *arg);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_prepare_watcher_create`, :func:`flux_check_watcher_create`, and
+:func:`flux_idle_watcher_create` create specialized reactor watchers with
+the following properties:
+
+The prepare watcher is called by the reactor loop immediately before
+blocking, while the check watcher is called by the reactor loop
+immediately after blocking.
+
+The idle watcher is always run when no other events are pending,
+excluding other idle watchers, prepare and check watchers.
+While it is active, the reactor loop does not block waiting for
+new events.
+
+The callback :var:`revents` argument should be ignored.
+
+Note: the Flux reactor is based on libev. For additional information
+on the behavior of these watchers, refer to the libev documentation on
+``ev_idle``, ``ev_prepare``, and ``ev_check``.
+
+
+RETURN VALUE
+============
+
+These functions return a :type:`flux_watcher_t` object on success.
+On error, NULL is returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+libev: http://software.schmorp.de/pkg/libev.html
+
+
+SEE ALSO
+========
+
+:man3:`flux_watcher_start`, :man3:`flux_reactor_run`, :man3:`flux_check_watcher_create`
diff --git a/doc/man3/flux_job_timeleft.rst b/doc/man3/flux_job_timeleft.rst
new file mode 100644
index 000000000000..e8cb972ebf56
--- /dev/null
+++ b/doc/man3/flux_job_timeleft.rst
@@ -0,0 +1,44 @@
+====================
+flux_job_timeleft(3)
+====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   int flux_job_timeleft (flux_t *h,
+                          flux_error_t *error,
+                          double *timeleft);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+The :func:`flux_job_timeleft` function determines if the calling process
+is executing within the context of a Flux job (either a parallel job or
+a Flux instance running as a job), then handles querying the appropriate
+service for the remaining time in the job.
+
+RETURN VALUE
+============
+
+:func:`flux_job_timeleft` returns 0 on success with the remaining time in
+floating point seconds stored in :var:`timeleft`. If the job does not have
+an established time limit, then :var:`timeleft` is set to ``inf``. If the job
+time limit has expired or the job is no longer running, then :var:`timeleft`
+is set to ``0``.
+
+If the current process is not part of an active job or instance, or another
+error occurs, then this function returns ``-1`` with an error string set in
+``error->text``.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_jobtap_get_flux.rst b/doc/man3/flux_jobtap_get_flux.rst
new file mode 100644
index 000000000000..b7bd8a71087a
--- /dev/null
+++ b/doc/man3/flux_jobtap_get_flux.rst
@@ -0,0 +1,102 @@
+=======================
+flux_jobtap_get_flux(3)
+=======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+   #include <flux/jobtap.h>
+
+   flux_t *flux_jobtap_get_flux (flux_plugin_t *p);
+
+   int flux_jobtap_service_register (flux_plugin_t *p,
+                                     const char *method,
+                                     flux_msg_handler_f cb,
+                                     void *arg);
+
+   int flux_jobtap_reprioritize_all (flux_plugin_t *p);
+
+   int flux_jobtap_reprioritize_job (flux_plugin_t *p,
+                                     flux_jobid_t id,
+                                     unsigned int priority);
+
+   int flux_jobtap_priority_unavail (flux_plugin_t *p,
+                                     flux_plugin_arg_t *args);
+
+   int flux_jobtap_reject_job (flux_plugin_t *p,
+                               flux_plugin_arg_t *args,
+                               const char *fmt,
+                               ...);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+These interfaces are used by Flux *jobtap* plugins which are used to
+extend the job manager broker module.
+
+:func:`flux_jobtap_get_flux` returns the job manager's Flux handle given
+the plugin's :type:`flux_plugin_t`. This can be used by a *jobtap* plugin
+to send RPCs, schedule timer watchers, or other asynchronous work.
+
+:func:`flux_jobtap_service_register` registers a service name :var:`method`
+under the job manager which will be handled by the provided message
+handler :var:`cb`.  The constructed service name will be
+``job-manager.<name>.<method>`` where ``name`` is the name of the plugin
+as returned by :func:`flux_plugin_get_name`. As such, this call may
+fail if the *jobtap* plugin has not yet set a name for itself using
+:func:`flux_plugin_set_name`.
+
+:func:`flux_jobtap_reprioritize_all` requests that the job manager begin
+reprioritization of all pending jobs, i.e. jobs in the PRIORITY and
+SCHED states. This will result on each job having a ``job.priority.get``
+callback invoked on it.
+
+:func:`flux_jobtap_reprioritize_job` allows a *jobtap* plugin to asynchronously
+assign the priority of a job.
+
+:func:`flux_jobtap_priority_unavail` is a convenience function which may
+be used by a plugin in the ``job.state.priority`` priority callback to
+indicate that a priority for the job is not yet available. It can be
+called as::
+
+   return flux_jobtap_priority_unavail (p, args);
+
+:func:`flux_jobtap_reject_job` is a convenience function which may be used
+by a plugin from the ``job.validate`` callback to reject a job before its
+submission is fully complete. The error and optional message supplied in
+:var:`fmt` will be returned to the originating job submission request. This
+function returns ``-1`` so that it may be conveniently called as::
+
+  return flux_jobtap_reject_job (p, args,
+                                 "User exceeded %d jobs",
+                                 limit);
+
+RETURN VALUE
+============
+
+:func:`flux_jobtap_get_flux` returns a :type:`flux_t` handle on success.
+``NULL`` is returned with errno set to ``EINVAL`` if the supplied
+:type:`flux_plugin_t` argument is not a jobtap plugin handle.
+
+:func:`flux_jobtap_reject_job` always returns ``-1`` so that it may be used
+to exit the ``job.validate`` callback.
+
+The remaining functions return 0 on success, -1 on failure.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man7:`flux-jobtap-plugins`
diff --git a/doc/man3/flux_kvs_commit.adoc b/doc/man3/flux_kvs_commit.adoc
deleted file mode 100644
index 999226a4e692..000000000000
--- a/doc/man3/flux_kvs_commit.adoc
+++ /dev/null
@@ -1,137 +0,0 @@
-flux_kvs_commit(3)
-==================
-:doctype: manpage
-
-
-NAME
-----
-flux_kvs_commit, flux_kvs_fence, flux_kvs_commit_get_treeobj, flux_kvs_commit_get_sequence - commit a KVS transaction
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_future_t *flux_kvs_commit (flux_t *h,
-                                 const char *ns,
-                                 int flags,
-                                 flux_kvs_txn_t *txn);
-
- flux_future_t *flux_kvs_fence (flux_t *h,
-                                const char *ns,
-                                int flags,
-                                const char *name,
-                                int nprocs,
-                                flux_kvs_txn_t *txn);
-
- int flux_kvs_commit_get_treeobj (flux_future_t *f,
-                                  const char **treeobj);
-
- int flux_kvs_commit_get_sequence (flux_future_t *f,
-                                   int *seq);
-
-DESCRIPTION
------------
-
-`flux_kvs_commit()` sends a request via handle _h_ to the KVS service
-to commit a transaction _txn_.  _txn_ is created with
-`flux_kvs_txn_create(3)` and after commit completion, is destroyed
-with `flux_kvs_txn_destroy()`.  A `flux_future_t` object is returned,
-which acts as handle for synchronization and container for the
-response.  The _txn_ will operate in the namespace specified by _ns_.
-If _ns_ is NULL, `flux_kvs_commit()` will operate on the default
-namespace, or if set, the namespace from the FLUX_KVS_NAMESPACE
-environment variable.  Note that all transactions operate on the same
-namespace.
-
-`flux_kvs_fence()` is a "collective" version of `flux_kvs_commit()` that
-supports multiple callers.  Each caller uses the same _flags_, _name_,
-and _nprocs_ arguments.  Once _nprocs_ requests are received by the KVS
-service for the named operation, the transactions are combined and committed
-together as one transaction.  _name_ must be unique across the Flux session
-and should not be reused, even after the fence is complete.
-
-`flux_future_then(3)` may be used to register a reactor callback
-(continuation) to be called once the response to the commit/fence
-request has been received.  `flux_future_wait_for(3)` may be used to
-block until the response has been received.  Both accept an optional timeout.
-
-`flux_future_get()`, `flux_kvs_commit_get_treeobj()`, or
-`flux_kvs_commit_get_sequence()` can decode the response.  A return of
-0 indicates success and the entire transaction was committed.  A
-return of -1 indicates failure, none of the transaction was committed.
-All can be used on the `flux_future_t` returned by `flux_kvs_commit()`
-or `flux_kvs_fence()`.
-
-In addition to checking for success or failure,
-`flux_kvs_commit_get_treeobj()` and `flux_kvs_commit_get_sequence()`
-can return information about the root snapshot that the commit or
-fence has completed its transaction on.
-
-`flux_kvs_commit_get_treeobj()` obtains the root hash in the form of
-an RFC 11 _dirref_ treeobj, suitable to be passed to
-`flux_kvs_lookupat(3)`.
-
-`flux_kvs_commit_get_sequence()` retrieves the monotonic sequence number
-for the root.
-
-FLAGS
------
-
-The following are valid bits in a _flags_ mask passed as an argument
-to `flux_kvs_commit()` or `flux_kvs_fence()`.
-
-FLUX_KVS_NO_MERGE::
-The KVS service may merge contemporaneous commit transactions as an
-optimization.  However, if the combined transactions modify the same key,
-a watch on that key may only be notified of the last-in value.  This flag
-can be used to disable that optimization for this transaction.
-
-
-RETURN VALUE
-------------
-
-`flux_kvs_commit()` and `flux_kvs_fence()` return a `flux_future_t` on
-success, or NULL on failure with errno set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-One of the arguments was invalid.
-
-ENOMEM::
-Out of memory.
-
-EPROTO::
-A request was malformed.
-
-ENOSYS::
-The KVS module is not loaded.
-
-ENOTSUP::
-An unknown namespace was requested.
-
-EOVERFLOW::
-`flux_kvs_fence()` has been called too many times and _nprocs_ has
-been exceeded.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_future_get(3), flux_kvs_txn_create(3), flux_kvs_set_namespace(3)
diff --git a/doc/man3/flux_kvs_commit.rst b/doc/man3/flux_kvs_commit.rst
new file mode 100644
index 000000000000..59b3e216c493
--- /dev/null
+++ b/doc/man3/flux_kvs_commit.rst
@@ -0,0 +1,133 @@
+==================
+flux_kvs_commit(3)
+==================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_kvs_commit (flux_t *h,
+                                   const char *ns,
+                                   int flags,
+                                   flux_kvs_txn_t *txn);
+
+   flux_future_t *flux_kvs_fence (flux_t *h,
+                                  const char *ns,
+                                  int flags,
+                                  const char *name,
+                                  int nprocs,
+                                  flux_kvs_txn_t *txn);
+
+   int flux_kvs_commit_get_treeobj (flux_future_t *f,
+                                    const char **treeobj);
+
+   int flux_kvs_commit_get_sequence (flux_future_t *f, int *seq);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_kvs_commit` sends a request via handle :var:`h` to the KVS service
+to commit a transaction :var:`txn`. :var:`txn` is created with
+:man3:`flux_kvs_txn_create` and after commit completion, is destroyed
+with :man3:`flux_kvs_txn_destroy`. A :type:`flux_future_t` object is returned,
+which acts as handle for synchronization and container for the
+response. The :var:`txn` will operate in the namespace specified by :var:`ns`.
+If :var:`ns` is NULL, :func:`flux_kvs_commit` will operate on the default
+namespace, or if set, the namespace from the FLUX_KVS_NAMESPACE
+environment variable. Note that all transactions operate on the same
+namespace.
+
+:func:`flux_kvs_fence` is a "collective" version of :func:`flux_kvs_commit`
+that supports multiple callers. Each caller uses the same :var:`flags`,
+:var:`name`, and :var:`nprocs` arguments. Once :var:`nprocs` requests are
+received by the KVS service for the named operation, the transactions are
+combined and committed together as one transaction. :var:`name` must be unique
+across the Flux session and should not be reused, even after the fence is
+complete.
+
+:man3:`flux_future_then` may be used to register a reactor callback
+(continuation) to be called once the response to the commit/fence
+request has been received. :man3:`flux_future_wait_for` may be used to
+block until the response has been received. Both accept an optional timeout.
+
+:man3:`flux_future_get`, :func:`flux_kvs_commit_get_treeobj`, or
+:func:`flux_kvs_commit_get_sequence` can decode the response. A return of
+0 indicates success and the entire transaction was committed. A
+return of -1 indicates failure, none of the transaction was committed.
+All can be used on the :type:`flux_future_t` returned by :func:`flux_kvs_commit`
+or :func:`flux_kvs_fence`.
+
+In addition to checking for success or failure,
+:func:`flux_kvs_commit_get_treeobj` and :func:`flux_kvs_commit_get_sequence`
+can return information about the root snapshot that the commit or
+fence has completed its transaction on.
+
+:func:`flux_kvs_commit_get_treeobj` obtains the root hash in the form of
+an RFC 11 *dirref* treeobj, suitable to be passed to
+:man3:`flux_kvs_lookupat`.
+
+:func:`flux_kvs_commit_get_sequence` retrieves the monotonic sequence number
+for the root.
+
+
+FLAGS
+=====
+
+The following are valid bits in a :var:`flags` mask passed as an argument
+to :func:`flux_kvs_commit` or :func:`flux_kvs_fence`.
+
+FLUX_KVS_NO_MERGE
+   The KVS service may merge contemporaneous commit transactions as an
+   optimization. However, if the combined transactions modify the same key,
+   a watch on that key may only be notified of the last-in value. This flag
+   can be used to disable that optimization for this transaction.
+
+
+RETURN VALUE
+============
+
+:func:`flux_kvs_commit` and :func:`flux_kvs_fence` return a
+:type:`flux_future_t` on success, or NULL on failure with :var:`errno` set
+appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   One of the arguments was invalid.
+
+ENOMEM
+   Out of memory.
+
+EPROTO
+   A request was malformed.
+
+ENOSYS
+   The KVS module is not loaded.
+
+ENOTSUP
+   An unknown namespace was requested.
+
+EOVERFLOW
+   :func:`flux_kvs_fence` has been called too many times and :var:`nprocs` has
+   been exceeded.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_future_get`, :man3:`flux_kvs_txn_create`
diff --git a/doc/man3/flux_kvs_copy.adoc b/doc/man3/flux_kvs_copy.adoc
deleted file mode 100644
index 55e32a074069..000000000000
--- a/doc/man3/flux_kvs_copy.adoc
+++ /dev/null
@@ -1,108 +0,0 @@
-flux_kvs_copy(3)
-================
-:doctype: manpage
-
-
-NAME
-----
-flux_kvs_copy, flux_kvs_move - copy/move a KVS key
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_future_t *flux_kvs_copy (flux_t *h,
-                               const char *srckey,
-                               const char *dstkey,
-                               int commit_flags);
-
- flux_future_t *flux_kvs_move (flux_t *h,
-                               const char *srckey,
-                               const char *dstkey,
-                               int commit_flags);
-
-
-DESCRIPTION
------------
-
-`flux_kvs_copy()` sends a request via handle _h_ to the KVS service
-to look up the directory entry of _srckey_.  Upon receipt of the response,
-it then sends another request to commit a duplicate at _dstkey_.
-_commit_flags_ are passed through to the commit operation.
-See the FLAGS section of flux_kvs_commit(3).
-
-The net effect is that all content below _srckey_ is copied to _dstkey_.
-Due to the hash tree organization of the KVS name space, only the
-directory entry needs to be duplicated to create a new, fully independent
-deep copy of the original data.
-
-`flux_kvs_move()` first performs a `flux_kvs_copy()`, then sends a
-commit request to unlink _srckey_.  _commit_flags_ are passed through to
-the commit within `flux_kvs_copy()`, and to the commit which performs
-the unlink.
-
-`flux_kvs_copy()` and `flux_kvs_move()` are capable of working across
-namespaces.  See `flux_kvs_commit(3)` for info on how to select a
-namespace other than the default.
-
-
-CAVEATS
--------
-
-`flux_kvs_copy()` and `flux_kvs_commit()` are implemented as aggregates
-of multiple KVS operations.  As such they do not have the "all or nothing"
-guarantee of a being carried out within a single KVS transaction.
-
-In the unlikely event that the copy phase of a `flux_kvs_move()`
-succeeds but the unlink phase fails, `flux_kvs_move()` may return failure
-without cleaning up the new copy.  Since the copy phase already validated
-that the unlink target key exists by copying from it, the source of such a
-failure would be a transient error such as out of memory or communication
-failure.
-
-
-RETURN VALUE
-------------
-
-`flux_kvs_copy ()` and `flux_kvs_move ()` return a `flux_future_t` on
-success, or NULL on failure with errno set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-One of the arguments was invalid.
-
-ENOMEM::
-Out of memory.
-
-EPROTO::
-A request was malformed.
-
-ENOSYS::
-The KVS module is not loaded.
-
-ENOTSUP::
-An unknown namespace was requested.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_future_get(3), flux_kvs_commit(3)
diff --git a/doc/man3/flux_kvs_copy.rst b/doc/man3/flux_kvs_copy.rst
new file mode 100644
index 000000000000..ce46b59b3f10
--- /dev/null
+++ b/doc/man3/flux_kvs_copy.rst
@@ -0,0 +1,100 @@
+================
+flux_kvs_copy(3)
+================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_kvs_copy (flux_t *h,
+                                 const char *srckey,
+                                 const char *dstkey,
+                                 int commit_flags);
+
+   flux_future_t *flux_kvs_move (flux_t *h,
+                                 const char *srckey,
+                                 const char *dstkey,
+                                 int commit_flags);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_kvs_copy` sends a request via handle :var:`h` to the KVS service
+to look up the directory entry of :var:`srckey`. Upon receipt of the response,
+it then sends another request to commit a duplicate at :var:`dstkey`.
+:var:`commit_flags` are passed through to the commit operation.
+See the FLAGS section of :man3:`flux_kvs_commit`.
+
+The net effect is that all content below :var:`srckey` is copied to
+:var:`dstkey`.  Due to the hash tree organization of the KVS name space, only
+the directory entry needs to be duplicated to create a new, fully independent
+deep copy of the original data.
+
+:func:`flux_kvs_move` first performs a :func:`flux_kvs_copy`, then sends a
+commit request to unlink :var:`srckey`. :var:`commit_flags` are passed through
+to the commit within :func:`flux_kvs_copy`, and to the commit which performs
+the unlink.
+
+:func:`flux_kvs_copy` and :func:`flux_kvs_move` are capable of working across
+namespaces. See :man3:`flux_kvs_commit` for info on how to select a
+namespace other than the default.
+
+
+CAVEATS
+=======
+
+:func:`flux_kvs_copy` and :func:`flux_kvs_commit` are implemented as aggregates
+of multiple KVS operations. As such they do not have the "all or nothing"
+guarantee of a being carried out within a single KVS transaction.
+
+In the unlikely event that the copy phase of a :func:`flux_kvs_move`
+succeeds but the unlink phase fails, :func:`flux_kvs_move` may return failure
+without cleaning up the new copy. Since the copy phase already validated
+that the unlink target key exists by copying from it, the source of such a
+failure would be a transient error such as out of memory or communication
+failure.
+
+
+RETURN VALUE
+============
+
+:func:`flux_kvs_copy` and :func:`flux_kvs_move` return a :type:`flux_future_t`
+on success, or NULL on failure with errno set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   One of the arguments was invalid.
+
+ENOMEM
+   Out of memory.
+
+EPROTO
+   A request was malformed.
+
+ENOSYS
+   The KVS module is not loaded.
+
+ENOTSUP
+   An unknown namespace was requested.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_future_get`, :man3:`flux_kvs_commit`
diff --git a/doc/man3/flux_kvs_getroot.adoc b/doc/man3/flux_kvs_getroot.adoc
deleted file mode 100644
index 4b854594ec9e..000000000000
--- a/doc/man3/flux_kvs_getroot.adoc
+++ /dev/null
@@ -1,113 +0,0 @@
-flux_kvs_getroot(3)
-===================
-:doctype: manpage
-
-
-NAME
-----
-flux_kvs_getroot, flux_kvs_getroot_get_treeobj, flux_kvs_getroot_get_blobref, flux_kvs_getroot_get_sequence, flux_kvs_getroot_get_owner, flux_kvs_getroot_cancel - look up KVS root hash
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_future_t *flux_kvs_getroot (flux_t *h,
-                                  const char *ns,
-                                  int flags);
-
- int flux_kvs_getroot_get_treeobj (flux_future_t *f,
-                                   const char **treeobj);
-
- int flux_kvs_getroot_get_blobref (flux_future_t *f,
-                                   const char **blobref);
-
- int flux_kvs_getroot_get_sequence (flux_future_t *f,
-                                    int *seq);
-
- int flux_kvs_getroot_get_owner (flux_future_t *f,
-                                 uint32_t *owner);
-
-
-DESCRIPTION
------------
-
-`flux_kvs_getroot()` sends a request via handle _h_ to the `kvs`
-service to look up the current root hash for namespace _ns_.  A `flux_future_t`
-object is returned, which acts as handle for synchronization and container
-for the response.  _flags_ is currently unused and should be set to 0.
-
-Upon future fulfillment, these functions can decode the result:
-
-`flux_kvs_getroot_get_treeobj()` obtains the root hash in the form
-of an RFC 11 _dirref_ treeobj, suitable to be passed to `flux_kvs_lookupat(3)`.
-
-`flux_kvs_getroot_get_blobref()` obtains the RFC 10 blobref, suitable to
-be passed to `flux_content_load(3)`.
-
-`flux_kvs_getroot_get_sequence()` retrieves the monotonic sequence number
-for the root.
-
-`flux_kvs_getroot_get_owner()` retrieves the namespace owner.
-
-
-FLAGS
------
-
-The _flags_ mask is currently unused and should be set to 0.
-
-
-RETURN VALUE
-------------
-
-`flux_kvs_getroot()` returns a `flux_future_t` on success, or NULL on
-failure with errno set appropriately.
-
-The other functions return zero on success, or -1 on failure with errno
-set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-One of the arguments was invalid.
-
-ENOMEM::
-Out of memory.
-
-EPROTO::
-A request was malformed.
-
-ENOSYS::
-The kvs module is not loaded.
-
-ENOTSUP::
-An unknown namespace was requested or namespace was deleted.
-
-EPERM::
-The requesting user is not permitted to access the requested namespace.
-
-ENODATA::
-A stream of responses has been terminated by a call to
-`flux_kvs_getroot_cancel()`.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_kvs_lookup (3), flux_future_get (3), flux_content_load (3).
diff --git a/doc/man3/flux_kvs_getroot.rst b/doc/man3/flux_kvs_getroot.rst
new file mode 100644
index 000000000000..caf38ea71cf7
--- /dev/null
+++ b/doc/man3/flux_kvs_getroot.rst
@@ -0,0 +1,104 @@
+===================
+flux_kvs_getroot(3)
+===================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_kvs_getroot (flux_t *h,
+                                    const char *ns,
+                                    int flags);
+
+   int flux_kvs_getroot_get_treeobj (flux_future_t *f,
+                                     const char **treeobj);
+
+   int flux_kvs_getroot_get_blobref (flux_future_t *f,
+                                     const char **blobref);
+
+   int flux_kvs_getroot_get_sequence (flux_future_t *f, int *seq);
+
+   int flux_kvs_getroot_get_owner (flux_future_t *f, uint32_t *owner);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_kvs_getroot` sends a request via handle :var:`h` to the ``kvs``
+service to look up the current root hash for namespace :var:`ns`. A
+:type:`flux_future_t` object is returned, which acts as handle for
+synchronization and container for the response. :var:`flags` is currently
+unused and should be set to 0.
+
+Upon future fulfillment, these functions can decode the result:
+
+:func:`flux_kvs_getroot_get_treeobj` obtains the root hash in the form
+of an RFC 11 *dirref* treeobj, suitable to be passed to :man3:`flux_kvs_lookupat`.
+
+:func:`flux_kvs_getroot_get_blobref` obtains the RFC 10 blobref, suitable to
+be passed to :man3:`flux_content_load`.
+
+:func:`flux_kvs_getroot_get_sequence` retrieves the monotonic sequence number
+for the root.
+
+:func:`flux_kvs_getroot_get_owner` retrieves the namespace owner.
+
+
+FLAGS
+=====
+
+The :var:`flags` mask is currently unused and should be set to 0.
+
+
+RETURN VALUE
+============
+
+:func:`flux_kvs_getroot` returns a :type:`flux_future_t` on success, or NULL
+on failure with :var:`errno` set appropriately.
+
+The other functions return zero on success, or -1 on failure with :var:`errno`
+set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   One of the arguments was invalid.
+
+ENOMEM
+   Out of memory.
+
+EPROTO
+   A request was malformed.
+
+ENOSYS
+   The kvs module is not loaded.
+
+ENOTSUP
+   An unknown namespace was requested or namespace was deleted.
+
+EPERM
+   The requesting user is not permitted to access the requested namespace.
+
+ENODATA
+   A stream of responses has been terminated by a call to
+   :func:`flux_kvs_getroot_cancel`.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_kvs_lookup`, :man3:`flux_future_get`, :man3:`flux_content_load`
diff --git a/doc/man3/flux_kvs_lookup.adoc b/doc/man3/flux_kvs_lookup.adoc
deleted file mode 100644
index 449ff353a41c..000000000000
--- a/doc/man3/flux_kvs_lookup.adoc
+++ /dev/null
@@ -1,239 +0,0 @@
-flux_kvs_lookup(3)
-==================
-:doctype: manpage
-
-
-NAME
-----
-flux_kvs_lookup, flux_kvs_lookupat, flux_kvs_lookup_get, flux_kvs_lookup_get_unpack, flux_kvs_lookup_get_raw, flux_kvs_lookup_get_dir, flux_kvs_lookup_get_treeobj, flux_kvs_lookup_get_symlink - look up KVS key
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_future_t *flux_kvs_lookup (flux_t *h, const char *ns, int flags,
-                                 const char *key);
-
- flux_future_t *flux_kvs_lookupat (flux_t *h, int flags,
-                                   const char *key, const char *treeobj);
-
- int flux_kvs_lookup_get (flux_future_t *f, const char **value);
-
- int flux_kvs_lookup_get_unpack (flux_future_t *f, const char *fmt, ...);
-
- int flux_kvs_lookup_get_raw (flux_future_t *f,
-                              const void **data, int *len);
-
- int flux_kvs_lookup_get_dir (flux_future_t *f,
-                              const flux_kvsdir_t **dir);
-
- int flux_kvs_lookup_get_treeobj (flux_future_t *f, const char **treeobj);
-
- int flux_kvs_lookup_get_symlink (flux_future_t *f, const char **ns,
-                                  const char **target);
-
- const char *flux_kvs_lookup_get_key (flux_future_t *f);
-
- int flux_kvs_lookup_cancel (flux_future_t *f);
-
-
-DESCRIPTION
------------
-
-The Flux Key Value Store is a general purpose distributed storage
-service used by Flux services.
-
-`flux_kvs_lookup()` sends a request to the KVS service to look up
-_key_ in namespace _ns_.  It returns a `flux_future_t` object which
-acts as handle for synchronization and container for the result.  The
-namespace _ns_ is optional.  If set to NULL, `flux_kvs_lookup()` uses
-the default namespace, or if set, the namespace from the
-FLUX_KVS_NAMESPACE environment variable.  _flags_ modifies the request
-as described below.
-
-`flux_kvs_lookupat()` is identical to `flux_kvs_lookup()` except
-_treeobj_ is a serialized RFC 11 object that references a particular
-static set of content within the KVS, effectively a snapshot.
-See `flux_kvs_lookup_get_treeobj()` below.
-
-All the functions below are variations on a common theme.  First they
-complete the lookup RPC by blocking on the response, if not already received.
-Then they interpret the result in different ways.  They may be called more
-than once on the same future, and they may be intermixed to probe a result
-or interpret it in different ways.  Results remain valid until
-`flux_future_destroy()` is called.
-
-`flux_kvs_lookup_get()` interprets the result as a value.  If the value
-has length greater than zero, a NULL is appended and it is assigned
-to _value_, otherwise NULL is assigned to _value_.
-
-`flux_kvs_lookup_get_unpack()` interprets the result as a value, which
-it decodes as JSON according to variable arguments in Jansson
-`json_unpack()` format.
-
-`flux_kvs_lookup_get_raw()` interprets the result as a value.  If the value
-has length greater than zero, the value and its length are assigned to
-_buf_ and _len_, respectively.  Otherwise NULL and zero are assigned.
-
-`flux_kvs_lookup_get_dir()` interprets the result as a directory,
-e.g. in response to a lookup with the FLUX_KVS_READDIR flag set.
-The directory object is assigned to _dir_.
-
-`flux_kvs_lookup_get_treeobj()` interprets the result as any RFC 11 object.
-The object in JSON-encoded form is assigned to _treeobj_.  Since all
-lookup requests return an RFC 11 object of one type or another, this
-function should work on all.
-
-`flux_kvs_lookup_get_symlink()` interprets the result as a symlink target,
-e.g. in response to a lookup with the FLUX_KVS_READLINK flag set.
-The result is parsed and symlink namespace is assigned to _ns_ and
-the symlink target is assigned to _target_.  If a namespace was not assigned
-to the symlink, _ns_ is set to NULL.
-
-`flux_kvs_lookup_get_key()` accesses the key argument from the original
-lookup.
-
-`flux_kvs_lookup_cancel()` cancels a stream of lookup responses
-requested with FLUX_KVS_WATCH or a waiting lookup response with
-FLUX_KVS_WAITCREATE.  See FLAGS below for additional information.
-
-These functions may be used asynchronously.  See `flux_future_then(3)` for
-details.
-
-
-FLAGS
------
-
-The following are valid bits in a _flags_ mask passed as an argument
-to `flux_kvs_lookup()` or `flux_kvs_lookupat()`.
-
-FLUX_KVS_READDIR::
-Look up a directory, not a value.  The lookup fails if the key does
-not refer to a directory object.
-
-FLUX_KVS_READLINK::
-If key is a symlink, read the link value.  The lookup fails if the key
-does not refer to a symlink object.
-
-FLUX_KVS_TREEOBJ::
-All KVS lookups return an RFC 11 tree object.  This flag requests that
-they be returned without conversion.  That is, a "valref" will not
-be converted to a "val" object, and a "dirref" will not be converted
-to a "dir" object.  This is useful for obtaining a snapshot reference
-that can be passed to `flux_kvs_lookupat()`.
-
-FLUX_KVS_WATCH::
-After the initial response, continue to send responses to the lookup
-request each time _key_ is mentioned verbatim in a committed transaction.
-After receiving a response, `flux_future_reset()` should be used to
-consume a response and prepare for the next one.  Responses continue
-until the namespace is removed, the key is removed, the lookup is
-canceled with `flux_kvs_lookup_cancel()`, or an error occurs.  After
-calling `flux_kvs_lookup_cancel()`, callers should wait for the future
-to be fulfilled with an ENODATA error to ensure the cancel request has
-been received and processed.
-
-FLUX_KVS_WATCH_UNIQ::
-Specified along with FLUX_KVS_WATCH, this flag will alter watch
-behavior to only respond when _key_ is mentioned verbatim in a
-committed transaction and the value of the key has changed.
-
-FLUX_KVS_WATCH_APPEND::
-Specified along with FLUX_KVS_WATCH, this flag will alter watch
-behavior to only respond when _key_ is mentioned verbatim in a
-committed transaction and the key has been appended to.  The response
-will only contain the additional appended data.  Note that only data
-length is considered for appends and no guarantee is made that prior
-data hasn't been overwritten.
-
-FLUX_KVS_WATCH_FULL::
-Specified along with FLUX_KVS_WATCH, this flag will alter watch
-behavior to respond when the value of the key being watched has
-changed.  Unlike FLUX_KVS_WATCH_UNIQ, the key being watched need not
-be mentioned in a transaction.  This may occur under several
-scenarios, such as a parent directory being altered.
-
-FLUX_KVS_WAITCREATE::
-If a KVS key does not exist, wait for it to exist before returning.
-This flag can be specified with or without FLUX_KVS_WATCH.  The lookup
-can be canceled with `flux_kvs_lookup_cancel()`.  After calling
-`flux_kvs_lookup_cancel()`, callers should wait for the future to be
-fulfilled with an ENODATA error to ensure the cancel request has been
-received and processed.
-
-RETURN VALUE
-------------
-
-`flux_kvs_lookup()` and `flux_kvs_lookupat()` return a
-`flux_future_t` on success, or NULL on failure with errno set appropriately.
-
-`flux_kvs_lookup_get()`, `flux_kvs_lookup_get_unpack()`,
-`flux_kvs_lookup_get_raw()`, `flux_kvs_lookup_get_dir()`,
-`flux_kvs_lookup_get_treeobj()`, `flux_kvs_lookup_get_symlink()`,
-and `flux_kvs_lookup_cancel()` return 0 on success, or -1 on failure with
-errno set appropriately.
-
-`flux_kvs_lookup_get_key()` returns key on success, or NULL with errno
-set to EINVAL if its future argument did not come from a KVS lookup.
-
-
-ERRORS
-------
-
-EINVAL::
-One of the arguments was invalid, or FLUX_KVS_READLINK was used but
-the key does not refer to a symlink.
-
-ENOMEM::
-Out of memory.
-
-ENOENT::
-An unknown key was requested.
-
-ENOTDIR::
-FLUX_KVS_READDIR flag was set and key does NOT point to a directory.
-
-EISDIR::
-FLUX_KVS_READDIR flag was NOT set and key points to a directory.
-
-EPROTO::
-A request or response was malformed.
-
-EFBIG::
-
-ENOSYS::
-The KVS module is not loaded.
-
-ENOTSUP::
-An unknown namespace was requested or namespace was deleted.
-
-ENODATA::
-A stream of responses requested with FLUX_KVS_WATCH was terminated
-with `flux_kvs_lookup_cancel()`.
-
-EPERM::
-The user does not have instance owner capability, and a lookup was attempted
-against a KVS namespace owned by another user.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_rpc(3), flux_future_then(3), flux_kvs_set_namespace(3)
-
-https://github.com/flux-framework/rfc/blob/master/spec_11.adoc[RFC 11: Key Value Store Tree Object Format v1]
diff --git a/doc/man3/flux_kvs_lookup.rst b/doc/man3/flux_kvs_lookup.rst
new file mode 100644
index 000000000000..c30cfc88443b
--- /dev/null
+++ b/doc/man3/flux_kvs_lookup.rst
@@ -0,0 +1,237 @@
+==================
+flux_kvs_lookup(3)
+==================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_kvs_lookup (flux_t *h,
+                                   const char *ns,
+                                   int flags,
+                                   const char *key);
+
+   flux_future_t *flux_kvs_lookupat (flux_t *h,
+                                     int flags,
+                                     const char *key,
+                                     const char *treeobj);
+
+   int flux_kvs_lookup_get (flux_future_t *f, const char **value);
+
+   int flux_kvs_lookup_get_unpack (flux_future_t *f,
+                                   const char *fmt,
+                                   ...);
+
+   int flux_kvs_lookup_get_raw (flux_future_t *f,
+                                const void **data
+                                size_t *len);
+
+   int flux_kvs_lookup_get_dir (flux_future_t *f,
+                                const flux_kvsdir_t **dir);
+
+   int flux_kvs_lookup_get_treeobj (flux_future_t *f,
+                                    const char **treeobj);
+
+   int flux_kvs_lookup_get_symlink (flux_future_t *f,
+                                    const char **ns,
+                                    const char **target);
+
+   const char *flux_kvs_lookup_get_key (flux_future_t *f);
+
+   int flux_kvs_lookup_cancel (flux_future_t *f);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+The Flux Key Value Store is a general purpose distributed storage
+service used by Flux services.
+
+:func:`flux_kvs_lookup` sends a request to the KVS service to look up
+:var:`key` in namespace :var:`ns`. It returns a :type:`flux_future_t` object
+which acts as handle for synchronization and container for the result. The
+namespace :var:`ns` is optional. If set to NULL, :func:`flux_kvs_lookup` uses
+the default namespace, or if set, the namespace from the
+:envvar:`FLUX_KVS_NAMESPACE` environment variable. :var:`flags` modifies the
+request as described below.
+
+:func:`flux_kvs_lookupat` is identical to :func:`flux_kvs_lookup` except
+:var:`treeobj` is a serialized RFC 11 object that references a particular
+static set of content within the KVS, effectively a snapshot.
+See :func:`flux_kvs_lookup_get_treeobj` below.
+
+All the functions below are variations on a common theme. First they
+complete the lookup RPC by blocking on the response, if not already received.
+Then they interpret the result in different ways. They may be called more
+than once on the same future, and they may be intermixed to probe a result
+or interpret it in different ways. Results remain valid until
+:man3:`flux_future_destroy` is called.
+
+:func:`flux_kvs_lookup_get` interprets the result as a value. If the value
+has length greater than zero, a NULL is appended and it is assigned
+to :var:`value`, otherwise NULL is assigned to *value*.
+
+:func:`flux_kvs_lookup_get_unpack` interprets the result as a value, which
+it decodes as JSON according to variable arguments in Jansson
+:func:`json_unpack` format.
+
+:func:`flux_kvs_lookup_get_raw` interprets the result as a value. If the value
+has length greater than zero, the value and its length are assigned to
+:var:`buf` and :var:`len`, respectively. Otherwise NULL and zero are assigned.
+
+:func:`flux_kvs_lookup_get_dir` interprets the result as a directory,
+e.g. in response to a lookup with the FLUX_KVS_READDIR flag set.
+The directory object is assigned to :var:`dir`.
+
+:func:`flux_kvs_lookup_get_treeobj` interprets the result as any RFC 11 object.
+The object in JSON-encoded form is assigned to :var:`treeobj`. Since all
+lookup requests return an RFC 11 object of one type or another, this
+function should work on all.
+
+:func:`flux_kvs_lookup_get_symlink` interprets the result as a symlink target,
+e.g. in response to a lookup with the FLUX_KVS_READLINK flag set.
+The result is parsed and symlink namespace is assigned to :var:`ns` and
+the symlink target is assigned to :var:`target`. If a namespace was not assigned
+to the symlink, :var:`ns` is set to NULL.
+
+:func:`flux_kvs_lookup_get_key` accesses the key argument from the original
+lookup.
+
+:func:`flux_kvs_lookup_cancel` cancels a stream of lookup responses
+requested with FLUX_KVS_WATCH or a waiting lookup response with
+FLUX_KVS_WAITCREATE. See FLAGS below for additional information.
+
+These functions may be used asynchronously. See :man3:`flux_future_then` for
+details.
+
+
+FLAGS
+=====
+
+The following are valid bits in a :var:`flags` mask passed as an argument
+to :func:`flux_kvs_lookup` or :func:`flux_kvs_lookupat`.
+
+FLUX_KVS_READDIR
+   Look up a directory, not a value. The lookup fails if :var:`key` does
+   not refer to a directory object.
+
+FLUX_KVS_READLINK
+   If :var:`key` is a symlink, read the link value. The lookup fails if the key
+   does not refer to a symlink object.
+
+FLUX_KVS_TREEOBJ
+   All KVS lookups return an RFC 11 tree object. This flag requests that
+   they be returned without conversion. That is, a "valref" will not
+   be converted to a "val" object, and a "dirref" will not be converted
+   to a "dir" object. This is useful for obtaining a snapshot reference
+   that can be passed to :func:`flux_kvs_lookupat`.
+
+FLUX_KVS_WATCH
+   After the initial response, continue to send responses to the lookup
+   request each time :var:`key` is mentioned verbatim in a committed
+   transaction.  After receiving a response, :man3:`flux_future_reset` should
+   be used to consume a response and prepare for the next one. Responses
+   continue until the namespace is removed, the key is removed, the lookup is
+   canceled with :func:`flux_kvs_lookup_cancel`, or an error occurs. After
+   calling :func:`flux_kvs_lookup_cancel`, callers should wait for the future
+   to be fulfilled with an ENODATA error to ensure the cancel request has
+   been received and processed.
+
+FLUX_KVS_WATCH_UNIQ
+   Specified along with FLUX_KVS_WATCH, this flag will alter watch
+   behavior to only respond when :var:`key` is mentioned verbatim in a
+   committed transaction and the value of the key has changed.
+
+FLUX_KVS_WATCH_APPEND
+   Specified along with FLUX_KVS_WATCH, this flag will alter watch
+   behavior to only respond when :var:`key` is mentioned verbatim in a
+   committed transaction and the key has been appended to. The
+   response will only contain the additional appended data.  If the
+   value is overwritten, the lookup fails with EINVAL.
+
+FLUX_KVS_WATCH_FULL
+   Specified along with FLUX_KVS_WATCH, this flag will alter watch
+   behavior to respond when the value of the key being watched has
+   changed. Unlike FLUX_KVS_WATCH_UNIQ, the key being watched need not
+   be mentioned in a transaction. This may occur under several
+   scenarios, such as a parent directory being altered.
+
+FLUX_KVS_WAITCREATE
+   If a KVS key does not exist, wait for it to exist before returning.
+   This flag can be specified with or without FLUX_KVS_WATCH. The lookup
+   can be canceled with :func:`flux_kvs_lookup_cancel`. After calling
+   :func:`flux_kvs_lookup_cancel`, callers should wait for the future to be
+   fulfilled with an ENODATA error to ensure the cancel request has been
+   received and processed.
+
+
+RETURN VALUE
+============
+
+:func:`flux_kvs_lookup` and :func:`flux_kvs_lookupat` return a
+:type:`flux_future_t` on success, or NULL on failure with errno set
+appropriately.
+
+:func:`flux_kvs_lookup_get`, :func:`flux_kvs_lookup_get_unpack`,
+:func:`flux_kvs_lookup_get_raw`, :func:`flux_kvs_lookup_get_dir`,
+:func:`flux_kvs_lookup_get_treeobj`, :func:`flux_kvs_lookup_get_symlink`,
+and :func:`flux_kvs_lookup_cancel` return 0 on success, or -1 on failure with
+:var:`errno` set appropriately.
+
+:func:`flux_kvs_lookup_get_key` returns key on success, or NULL with
+:var:`errno` set to EINVAL if its future argument did not come from a KVS
+lookup.
+
+
+ERRORS
+======
+
+EINVAL
+   One of the arguments was invalid, or FLUX_KVS_READLINK was used but
+   the key does not refer to a symlink.
+
+ENOMEM
+   Out of memory.
+
+ENOENT
+   An unknown key was requested.
+
+ENOTDIR
+   FLUX_KVS_READDIR flag was set and key does NOT point to a directory.
+
+EISDIR
+   FLUX_KVS_READDIR flag was NOT set and key points to a directory.
+
+EPROTO
+   A request or response was malformed.
+
+EFBIG; ENOSYS
+   The KVS module is not loaded.
+
+ENOTSUP
+   An unknown namespace was requested or namespace was deleted.
+
+ENODATA
+   A stream of responses requested with FLUX_KVS_WATCH was terminated
+   with :func:`flux_kvs_lookup_cancel`.
+
+EPERM
+   The user does not have instance owner capability, and a lookup was attempted
+   against a KVS namespace owned by another user.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_11`
diff --git a/doc/man3/flux_kvs_namespace_create.adoc b/doc/man3/flux_kvs_namespace_create.adoc
deleted file mode 100644
index 4d93fcecde6e..000000000000
--- a/doc/man3/flux_kvs_namespace_create.adoc
+++ /dev/null
@@ -1,86 +0,0 @@
-flux_kvs_namespace_create(3)
-============================
-:doctype: manpage
-
-
-NAME
-----
-flux_kvs_namespace_create, flux_kvs_namespace_remove - create/remove a KVS namespace
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_future_t *flux_kvs_namespace_create (flux_t *h,
-                                           const char *namespace,
-                                           uint32_t owner,
-                                           int flags);
-
- flux_future_t *flux_kvs_namespace_remove (flux_t *h,
-                                           const char *namespace);
-
-DESCRIPTION
------------
-
-`flux_kvs_namespace_create()` creates a KVS namespace.  Within a
-namespace, users can get/put KVS values completely independent of
-other KVS namespaces.  An owner of the namespace other than the
-instance owner can be chosen by setting _owner_.  Otherwise, _owner_
-can be set to FLUX_USERID_UNKNOWN.
-
-`flux_kvs_namespace_remove()` removes a KVS namespace.
-
-FLAGS
------
-
-The _flags_ mask is currently unused and should be set to 0.
-
-
-RETURN VALUE
-------------
-
-`flux_kvs_namespace_create()` and `flux_kvs_namespace_remove()` return
-a `flux_future_t` on success, or NULL on failure with errno set
-appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-One of the arguments was invalid.
-
-ENOMEM::
-Out of memory.
-
-EPROTO::
-A request was malformed.
-
-ENOSYS::
-The KVS module is not loaded.
-
-EEXIST::
-The namespace already exists.
-
-ENOTSUP::
-Attempt to remove illegal namespace.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_kvs_lookup(3), flux_kvs_commit(3)
diff --git a/doc/man3/flux_kvs_namespace_create.rst b/doc/man3/flux_kvs_namespace_create.rst
new file mode 100644
index 000000000000..ab59b3389aff
--- /dev/null
+++ b/doc/man3/flux_kvs_namespace_create.rst
@@ -0,0 +1,92 @@
+============================
+flux_kvs_namespace_create(3)
+============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_kvs_namespace_create (flux_t *h,
+                                             const char *namespace,
+                                             uint32_t owner,
+                                             int flags);
+
+   flux_future_t *flux_kvs_namespace_create_with (flux_t *h,
+                                                  const char *namespace,
+                                                  const char *rootref,
+                                                  uint32_t owner,
+                                                  int flags);
+
+   flux_future_t *flux_kvs_namespace_remove (flux_t *h,
+                                             const char *namespace);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_kvs_namespace_create` creates a KVS namespace. Within a
+namespace, users can get/put KVS values completely independent of
+other KVS namespaces. An owner of the namespace other than the
+instance owner can be chosen by setting :var:`owner`. Otherwise, :var:`owner`
+can be set to FLUX_USERID_UNKNOWN.
+
+:func:`flux_kvs_namespace_create_with` is identical to
+:func:`flux_kvs_namespace_create` but will initialize the namespace to
+the specified :var:`rootref`.  This may be useful in several circumstances,
+such as initializing a namespace to an earlier checkpoint.
+
+:func:`flux_kvs_namespace_remove` removes a KVS namespace.
+
+
+FLAGS
+=====
+
+The :var:`flags` mask is currently unused and should be set to 0.
+
+
+RETURN VALUE
+============
+
+:func:`flux_kvs_namespace_create` and :func:`flux_kvs_namespace_remove` return
+a :type:`flux_future_t` on success, or NULL on failure with :var:`errno` set
+appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   One of the arguments was invalid.
+
+ENOMEM
+   Out of memory.
+
+EPROTO
+   A request was malformed.
+
+ENOSYS
+   The KVS module is not loaded.
+
+EEXIST
+   The namespace already exists.
+
+ENOTSUP
+   Attempt to remove illegal namespace.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_kvs_lookup`, :man3:`flux_kvs_commit`
diff --git a/doc/man3/flux_kvs_txn_create.adoc b/doc/man3/flux_kvs_txn_create.adoc
deleted file mode 100644
index 62daee20458b..000000000000
--- a/doc/man3/flux_kvs_txn_create.adoc
+++ /dev/null
@@ -1,140 +0,0 @@
-flux_kvs_txn_create(3)
-======================
-:doctype: manpage
-
-
-NAME
-----
-flux_kvs_txn_create, flux_kvs_txn_destroy, flux_kvs_txn_put, flux_kvs_txn_pack, flux_kvs_txn_vpack, flux_kvs_txn_mkdir, flux_kvs_txn_unlink, flux_kvs_txn_symlink, flux_kvs_txn_put_raw, flux_kvs_txn_put_treeobj - operate on a KVS transaction object
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_kvs_txn_t *flux_kvs_txn_create (void);
-
- void flux_kvs_txn_destroy (flux_kvs_txn_t *txn);
-
- int flux_kvs_txn_put (flux_kvs_txn_t *txn, int flags,
-                       const char *key, const char *value);
-
- int flux_kvs_txn_pack (flux_kvs_txn_t *txn, int flags,
-                        const char *key, const char *fmt, ...);
-
- int flux_kvs_txn_vpack (flux_kvs_txn_t *txn, int flags,
-                         const char *key, const char *fmt, va_list ap);
-
- int flux_kvs_txn_mkdir (flux_kvs_txn_t *txn, int flags,
-                         const char *key);
-
- int flux_kvs_txn_unlink (flux_kvs_txn_t *txn, int flags,
-                          const char *key);
-
- int flux_kvs_txn_symlink (flux_kvs_txn_t *txn, int flags,
-                           const char *key, const char *ns,
-                           const char *target);
-
- int flux_kvs_txn_put_raw (flux_kvs_txn_t *txn, int flags,
-                           const char *key, const void *data, int len);
-
- int flux_kvs_txn_put_treeobj (flux_kvs_txn_t *txn, int flags,
-                               const char *key, const char *treeobj);
-
-
-DESCRIPTION
------------
-
-The Flux Key Value Store is a general purpose distributed storage
-service used by Flux services.
-
-`flux_kvs_txn_create()` creates a KVS transaction object that may be
-passed to `flux_kvs_commit(3)` or `flux_kvs_fence(3)`.  The transaction
-consists of a list of operations that are applied to the KVS together,
-in order.  The entire transaction either succeeds or fails.  After commit
-or fence, the object must be destroyed with `flux_kvs_txn_destroy()`.
-
-Each function below adds a single operation to _txn_.  _key_ is a
-hierarchical path name with period (".") used as path separator.
-When the transaction is committed, any existing keys or path components
-that are in conflict with the requested operation are overwritten.
-_flags_ can modify the request as described below.
-
-`flux_kvs_txn_put()` sets _key_ to a NULL terminated string _value_.
-_value_ may be NULL indicating that an empty value should be stored.
-
-`flux_kvs_txn_pack()` sets _key_ to a NULL terminated string encoded
-from a JSON object built with `json_pack()` style arguments (see below).
-`flux_kvs_txn_vpack()` is a variant that accepts a _va_list_ argument.
-
-`flux_kvs_txn_mkdir()` sets _key_ to an empty directory.
-
-`flux_kvs_txn_unlink()` removes _key_.  If _key_ is a directory,
-all its contents are removed as well.
-
-`flux_kvs_txn_symlink()` sets _key_ to a symbolic link pointing to a
-namespace _ns_ and a _target_ key within that namespace.  Neither _ns_
-nor _target_ must exist.  The namespace _ns_ is optional, if set to
-NULL the _target_ is assumed to be in the key's current namespace.
-
-`flux_kvs_txn_put_raw()` sets _key_ to a value containing raw data
-referred to by _data_ of length _len_.
-
-`flux_kvs_txn_put_treeobj()` sets _key_ to an RFC 11 object, encoded
-as a JSON string.
-
-
-FLAGS
------
-
-The following are valid bits in a _flags_ mask passed as an argument
-to `flux_kvs_txn_put()` or `flux_kvs_txn_put_raw()`.
-
-FLUX_KVS_APPEND::
-Append value instead of overwriting it.  If the key does not exist,
-it will be created with the value as the initial value.
-
-
-include::JSON_PACK.adoc[]
-
-
-RETURN VALUE
-------------
-
-`flux_kvs_txn_create()` returns a `flux_kvs_txn_t` object on success,
-or NULL on failure with errno set appropriately.
-
-`flux_kvs_txn_put()`, `flux_kvs_txn_pack()`, `flux_kvs_txn_mkdir()`,
-`flux_kvs_txn_unlink()`, `flux_kvs_txn_symlink()`, and `flux_kvs_txn_put_raw()`
-returns 0 on success, or -1 on failure with errno set appropriately.
-
-ERRORS
-------
-
-EINVAL::
-One of the arguments was invalid.
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_kvs_commit(3)
-
-https://github.com/flux-framework/rfc/blob/master/spec_11.adoc[RFC 11: Key Value Store Tree Object Format v1]
diff --git a/doc/man3/flux_kvs_txn_create.rst b/doc/man3/flux_kvs_txn_create.rst
new file mode 100644
index 000000000000..b83f08dc5a79
--- /dev/null
+++ b/doc/man3/flux_kvs_txn_create.rst
@@ -0,0 +1,159 @@
+======================
+flux_kvs_txn_create(3)
+======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_kvs_txn_t *flux_kvs_txn_create (void);
+
+   void flux_kvs_txn_destroy (flux_kvs_txn_t *txn);
+
+   int flux_kvs_txn_put (flux_kvs_txn_t *txn,
+                         int flags,
+                         const char *key,
+                         const char *value);
+
+   int flux_kvs_txn_pack (flux_kvs_txn_t *txn,
+                          int flags,
+                          const char *key,
+                          const char *fmt,
+                          ...);
+
+   int flux_kvs_txn_vpack (flux_kvs_txn_t *txn,
+                           int flags,
+                           const char *key,
+                           const char *fmt,
+                           va_list ap);
+
+   int flux_kvs_txn_mkdir (flux_kvs_txn_t *txn,
+                           int flags,
+                           const char *key);
+
+   int flux_kvs_txn_unlink (flux_kvs_txn_t *txn,
+                            int flags,
+                            const char *key);
+
+   int flux_kvs_txn_symlink (flux_kvs_txn_t *txn,
+                             int flags,
+                             const char *key,
+                             const char *ns,
+                             const char *target);
+
+   int flux_kvs_txn_put_raw (flux_kvs_txn_t *txn,
+                             int flags,
+                             const char *key,
+                             const void *data,
+                             size_t len);
+
+   int flux_kvs_txn_put_treeobj (flux_kvs_txn_t *txn,
+                                 int flags,
+                                 const char *key,
+                                 const char *treeobj);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+The Flux Key Value Store is a general purpose distributed storage
+service used by Flux services.
+
+:func:`flux_kvs_txn_create` creates a KVS transaction object that may be
+passed to :man3:`flux_kvs_commit` or :man3:`flux_kvs_fence`. The transaction
+consists of a list of operations that are applied to the KVS together,
+in order. The entire transaction either succeeds or fails. After commit
+or fence, the object must be destroyed with :func:`flux_kvs_txn_destroy`.
+
+Each function below adds a single operation to :var:`txn`. :var:`key` is a
+hierarchical path name with period (".") used as path separator.
+When the transaction is committed, any existing keys or path components
+that are in conflict with the requested operation are overwritten.
+:var:`flags` can modify the request as described below.
+
+:func:`flux_kvs_txn_put` sets :var:`key` to a NULL terminated string
+:var:`value`.  :var:`value` may be NULL indicating that an empty value should
+be stored.
+
+:func:`flux_kvs_txn_pack` sets :var:`key` to a NULL terminated string encoded
+from a JSON object built with :func:`json_pack` style arguments (see below).
+:func:`flux_kvs_txn_vpack` is a variant that accepts a :type:`va_list` argument.
+
+:func:`flux_kvs_txn_mkdir` sets :var:`key` to an empty directory.
+
+:func:`flux_kvs_txn_unlink` removes :var:`key`. If :var:`key` is a directory,
+all its contents are removed as well.
+
+:func:`flux_kvs_txn_symlink` sets :var:`key` to a symbolic link pointing to a
+namespace :var:`ns` and a :var:`target` key within that namespace. Neither
+:var:`ns` nor :var:`target` must exist. The namespace :var:`ns` is optional,
+if set to NULL the :var:`target` is assumed to be in the key's current
+namespace.
+
+:func:`flux_kvs_txn_put_raw` sets :var:`key` to a value containing raw data
+referred to by :var:`data` of length :var:`len`.
+
+:func:`flux_kvs_txn_put_treeobj` sets :var:`key` to an RFC 11 object, encoded
+as a JSON string.
+
+
+FLAGS
+=====
+
+The following are valid bits in a :var:`flags` mask passed as an argument
+to :func:`flux_kvs_txn_put` or :func:`flux_kvs_txn_put_raw`.
+
+FLUX_KVS_APPEND
+   Append value instead of overwriting it. If the key does not exist,
+   it will be created with the value as the initial value.
+
+
+ENCODING JSON PAYLOADS
+======================
+
+.. include:: common/json_pack.rst
+
+
+RETURN VALUE
+============
+
+:func:`flux_kvs_txn_create` returns a :type:`flux_kvs_txn_t` object on success,
+or NULL on failure with :var:`errno` set appropriately.
+
+:func:`flux_kvs_txn_put`, :func:`flux_kvs_txn_pack`, :func:`flux_kvs_txn_mkdir`,
+:func:`flux_kvs_txn_unlink`, :func:`flux_kvs_txn_symlink`, and
+:func:`flux_kvs_txn_put_raw` returns 0 on success, or -1 on failure with
+:var:`errno` set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   One of the arguments was invalid.
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_11`
+
+
+SEE ALSO
+========
+
+:man3:`flux_kvs_commit`
diff --git a/doc/man3/flux_log.adoc b/doc/man3/flux_log.adoc
deleted file mode 100644
index 1e7810637ba6..000000000000
--- a/doc/man3/flux_log.adoc
+++ /dev/null
@@ -1,128 +0,0 @@
-flux_log(3)
-===========
-:doctype: manpage
-
-
-NAME
-----
-flux_log, flux_vlog, flux_log_set_appname, flux_log_set_procid - Log messages to the Flux Message Broker
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-int flux_vlog (flux_t *h, int level, const char *fmt, va_list ap);
-
-int flux_log (flux_t *h, int level, const char *fmt, ...);
-
-void flux_log_set_appname (flux_t *h, const char *s);
-
-void flux_log_set_procid (flux_t *h, const char *s);
-
-DESCRIPTION
------------
-
-`flux_log()` creates RFC 5424 format log messages.  The log messages
-are sent to the Flux message broker on 'h' for handling if it is
-specified.  If 'h' is NULL, the log message is output to stderr.
-
-The 'level' parameter should be set to one of the syslog(3) severity
-levels, which are, in order of decreasing importance:
-
-'LOG_EMERG'::      system is unusable
-'LOG_ALERT'::      action must be taken immediately
-'LOG_CRIT'::       critical conditions
-'LOG_ERR'::        error conditions
-'LOG_WARNING'::    warning conditions
-'LOG_NOTICE'::     normal, but significant, condition
-'LOG_INFO'::       informational message
-'LOG_DEBUG'::      debug-level message
-
-When 'h' is specified, log messages are are added to the broker's
-circular buffer which can be accessed with flux-dmesg(3).  From there,
-a message's disposition is up to the broker's log configuration.
-
-`flux_log_set_procid()` may be used to override the default procid,
-which is initialized to the calling process's PID.
-
-`flux_log_set_appname()` may be used to override the default
-application name, which is initialized to the value of the '__progname'
-symbol (normally the argv[0] program name).
-
-MAPPING TO SYSLOG
------------------
-
-A Flux log message is formatted as a Flux request with a "raw" payload,
-as defined by Flux RFC 3.  The raw payload is formatted according to
-Internet RFC 5424.
-
-If the Flux handle 'h' is specified, the following Syslog header
-fields are set in a Flux log messages when it is created within
-`flux_log()`:
-
-PRI::
-Set to the user-specified severity level combined with the facility,
-which is hardwired to 'LOG_USER' in Flux log messages.
-
-VERSION::
-Set to 1.
-
-TIMESTAMP::
-Set to the current UTC wallclock time.
-
-HOSTNAME::
-Set to the broker rank associated with 'h'.
-
-APP-NAME::
-Set to the user-defined application name, truncated to 48 characters,
-excluding terminating NULL.
-
-PROCID::
-Set to the PID of the calling process.
-
-MSGID::
-Set to the NIL string "-".
-
-The STRUCTURED-DATA portion of the message is empty, and reserved for
-future use by Flux.
-
-The MSG portion is post-processed to ensure it contains no NULL's or non-ASCII
-characters.  At this time non-ASCII UTF-8 is not supported by `flux_log()`.
-
-RETURN VALUE
-------------
-
-`flux_log()` normally returns 0 on success, or -1 if there was
-a problem building or sending the log message, with errno set.
-
-
-ERRORS
-------
-
-EPERM::
-The user does not have permission to log messages to this Flux instance.
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-flux-dmesg(1), flux-logger(1),
-https://tools.ietf.org/html/rfc5424[RFC 5424 The Syslog Protocol]
diff --git a/doc/man3/flux_log.rst b/doc/man3/flux_log.rst
new file mode 100644
index 000000000000..769aac272417
--- /dev/null
+++ b/doc/man3/flux_log.rst
@@ -0,0 +1,139 @@
+===========
+flux_log(3)
+===========
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  int flux_vlog (flux_t *h, int level, const char *fmt, va_list ap);
+
+  int flux_log (flux_t *h, int level, const char *fmt, ...);
+
+  void flux_log_set_appname (flux_t *h, const char *s);
+
+  void flux_log_set_procid (flux_t *h, const char *s);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_log` creates RFC 5424 format log messages. The log messages
+are sent to the Flux message broker on :var:`h` for handling if it is
+specified. If :var:`h` is NULL, the log message is output to stderr.
+
+The :var:`level` parameter should be set to one of the :linux:man3:`syslog`
+severity levels, which are, in order of decreasing importance:
+
+*LOG_EMERG*
+   system is unusable
+
+*LOG_ALERT*
+   action must be taken immediately
+
+*LOG_CRIT*
+   critical conditions
+
+*LOG_ERR*
+   error conditions
+
+*LOG_WARNING*
+   warning conditions
+
+*LOG_NOTICE*
+   normal, but significant, condition
+
+*LOG_INFO*
+   informational message
+
+*LOG_DEBUG*
+   debug-level message
+
+When :var:`h` is specified, log messages are are added to the broker's
+circular buffer which can be accessed with :man1:`flux-dmesg`. From there,
+a message's disposition is up to the broker's log configuration.
+
+:func:`flux_log_set_procid` may be used to override the default procid,
+which is initialized to the calling process's PID.
+
+:func:`flux_log_set_appname` may be used to override the default
+application name, which is initialized to the value of the :var:`__progname`
+symbol (normally the :var:`argv[0]` program name).
+
+
+MAPPING TO SYSLOG
+=================
+
+A Flux log message is formatted as a Flux request with a "raw" payload,
+as defined by Flux RFC 3. The raw payload is formatted according to
+Internet RFC 5424.
+
+If the Flux handle :var:`h` is specified, the following Syslog header
+fields are set in a Flux log messages when it is created within
+:func:`flux_log`:
+
+PRI
+   Set to the user-specified severity level combined with the facility,
+   which is hardwired to *LOG_USER* in Flux log messages.
+
+VERSION
+   Set to 1.
+
+TIMESTAMP
+   Set to the current UTC wallclock time.
+
+HOSTNAME
+   Set to the broker rank associated with *h*.
+
+APP-NAME
+   Set to the user-defined application name, truncated to 48 characters,
+   excluding terminating NULL.
+
+PROCID
+   Set to the PID of the calling process.
+
+MSGID
+   Set to the NIL string "-".
+
+The STRUCTURED-DATA portion of the message is empty, and reserved for
+future use by Flux.
+
+The MSG portion is post-processed to ensure it contains no NULL's or non-ASCII
+characters. At this time non-ASCII UTF-8 is not supported by :func:`flux_log`.
+
+
+RETURN VALUE
+============
+
+:func:`flux_log` normally returns 0 on success, or -1 if there was
+a problem building or sending the log message, with :var:`errno` set.
+
+
+ERRORS
+======
+
+EPERM
+   The user does not have permission to log messages to this Flux instance.
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+RFC 5424 The Syslog Protocol: https://tools.ietf.org/html/rfc5424
+
+
+SEE ALSO
+========
+
+:man1:`flux-dmesg`, :man1:`flux-logger`,
diff --git a/doc/man3/flux_msg_cmp.adoc b/doc/man3/flux_msg_cmp.adoc
deleted file mode 100644
index 474ffa065443..000000000000
--- a/doc/man3/flux_msg_cmp.adoc
+++ /dev/null
@@ -1,62 +0,0 @@
-flux_msg_cmp(3)
-===============
-:doctype: manpage
-
-
-NAME
-----
-flux_msg_cmp - match a message
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- struct flux_match {
-     int typemask;
-     uint32_t matchtag;
-     char *topic_glob;
- };
-
- bool flux_msg_cmp (const flux_msg_t *msg, struct flux_match match);
-
-DESCRIPTION
------------
-
-`flux_msg_cmp()` compares _msg_ to _match_ criteria.
-
-If _match.typemask_ is nonzero, the type of the message must match
-one of the types in the mask.
-
-If _match.matchtag_ is not FLUX_MATCHTAG_NONE, the message matchtag
-must match _match.matchtag_.
-
-If _match.topic_glob_ is not NULL or an empty string, then the message topic
-string must match _match.topic_glob_ according to the rules of shell wildcards.
-
-
-RETURN VALUE
-------------
-
-`flux_msg_cmp()` returns true on a match, otherwise false.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-
-SEE ALSO
---------
-fnmatch(3)
diff --git a/doc/man3/flux_msg_cmp.rst b/doc/man3/flux_msg_cmp.rst
new file mode 100644
index 000000000000..ea1c50fdc01f
--- /dev/null
+++ b/doc/man3/flux_msg_cmp.rst
@@ -0,0 +1,55 @@
+===============
+flux_msg_cmp(3)
+===============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   struct flux_match {
+       int typemask;
+       uint32_t matchtag;
+       char *topic_glob;
+   };
+
+   bool flux_msg_cmp (const flux_msg_t *msg, struct flux_match match);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_msg_cmp` compares :var:`msg` to :var:`match` criteria.
+
+If :var:`match.typemask` is nonzero, the type of the message must match
+one of the types in the mask.
+
+If :var:`match.matchtag` is not FLUX_MATCHTAG_NONE, the message matchtag
+must match :var:`match.matchtag`.
+
+If :var:`match.topic_glob` is not NULL or an empty string, then the message
+topic string must match :var:`match.topic_glob` according to the rules of
+shell wildcards.
+
+
+RETURN VALUE
+============
+
+:func:`flux_msg_cmp` returns true on a match, otherwise false.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:linux:man3:`fnmatch`
diff --git a/doc/man3/flux_msg_create.rst b/doc/man3/flux_msg_create.rst
new file mode 100644
index 000000000000..0f476ad05347
--- /dev/null
+++ b/doc/man3/flux_msg_create.rst
@@ -0,0 +1,73 @@
+==========================
+flux_msg_create(3)
+==========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_msg_t *flux_msg_create (int type)
+
+   flux_msg_t *flux_msg_copy (const flux_msg_t *msg, bool payload)
+
+   const flux_msg_t *flux_msg_incref (const flux_msg_t *msg)
+  
+   void flux_msg_decref (const flux_msg_t *msg)
+
+   void flux_msg_destroy (flux_msg_t *msg)
+
+
+DESCRIPTION
+===========
+
+:func:`flux_msg_create` creates a :type:`flux_msg_t` of :var:`type`.
+Different types of Flux messages are defined in RFC :doc:`rfc:spec_3`. All
+messages have a starting reference count of 1.
+
+:func:`flux_msg_copy` duplicates :var:`msg`. The payload is omitted unless
+:var:`payload` is true. The initial reference count of the new message is 1.
+
+:func:`flux_msg_incref` increments the reference count of :var:`msg`
+by 1.
+
+:func:`flux_msg_decref` decrements the reference count of :var:`msg`
+by 1. When the reference count reaches 0, the message is destroyed.
+
+:func:`flux_msg_destroy` is an alias for :func:`flux_msg_decref`.
+
+RETURN VALUE
+============
+
+:func:`flux_msg_create` and :func:`flux_msg_copy` return a 
+:type:`flux_msg_t` type on success. On failure, NULL is returned and
+:var:`errno` is set.
+
+:func:`flux_msg_incref` returns a constant pointer to :var:`msg` for
+convenience. On failure, NULL is returned and :var:`errno` is set.
+
+:func:`flux_msg_decref` and :func:`flux_msg_destroy` have no return value.
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+EINVAL
+   Invalid message or message type.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_send`, :man3:`flux_respond`
diff --git a/doc/man3/flux_msg_encode.adoc b/doc/man3/flux_msg_encode.adoc
deleted file mode 100644
index 96e242be7537..000000000000
--- a/doc/man3/flux_msg_encode.adoc
+++ /dev/null
@@ -1,62 +0,0 @@
-flux_msg_encode(3)
-==================
-:doctype: manpage
-
-
-NAME
-----
-flux_msg_encode, flux_msg_decode - convert a Flux message to buffer and back again
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-int flux_msg_encode (const flux_msg_t *msg, void **buf, size_t *size);
-
-flux_msg_t *flux_msg_decode (void *buf, size_t size);
-
-
-DESCRIPTION
------------
-
-`flux_msg_encode()` converts _msg_ to a serialized representation,
-allocated internally and assigned to _buf_, number of bytes to _size_.
-The caller must release _buf_ with free(3).
-
-`flux_msg_decode()` performs the inverse, creating _msg_ from _buf_ and _size_.
-The caller must destroy _msg_ with flux_msg_destroy().
-
-RETURN VALUE
-------------
-
-`flux_msg_encode()` returns 0 on success.  On error, -1 is returned,
-and errno is set appropriately.
-
-`flux_msg_decode()` the decoded message on success.  On error, NULL
-is returned, and errno is set appropriately.
-
-ERRORS
-------
-
-EINVAL::
-Some arguments were invalid.
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
diff --git a/doc/man3/flux_msg_encode.rst b/doc/man3/flux_msg_encode.rst
new file mode 100644
index 000000000000..57c8d8d4359b
--- /dev/null
+++ b/doc/man3/flux_msg_encode.rst
@@ -0,0 +1,57 @@
+==================
+flux_msg_encode(3)
+==================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  int flux_msg_encode (const flux_msg_t *msg,
+                       void **buf,
+                       size_t *size);
+
+  flux_msg_t *flux_msg_decode (void *buf, size_t size);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_msg_encode` converts :var:`msg` to a serialized representation,
+allocated internally and assigned to :var:`buf`, number of bytes to :var:`size`.
+The caller must release :var:`buf` with :linux:man3:`free`.
+
+:func:`flux_msg_decode` performs the inverse, creating :var:`msg` from
+:var:`buf` and :var:`size`.  The caller must destroy :var:`msg` with
+:func:`flux_msg_destroy`.
+
+
+RETURN VALUE
+============
+
+:func:`flux_msg_encode` returns 0 on success. On error, -1 is returned,
+and :var:`errno` is set appropriately.
+
+:func:`flux_msg_decode` the decoded message on success. On error, NULL
+is returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   Some arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_msg_handler_addvec.adoc b/doc/man3/flux_msg_handler_addvec.adoc
deleted file mode 100644
index c1cd64dbfa55..000000000000
--- a/doc/man3/flux_msg_handler_addvec.adoc
+++ /dev/null
@@ -1,83 +0,0 @@
-flux_msg_handler_addvec(3)
-==========================
-:doctype: manpage
-
-
-NAME
-----
-flux_msg_handler_addvec,
-flux_msg_handler_delvec - bulk add/remove message handlers
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- struct flux_msg_handler_spec {
-     int typemask;
-     const char *topic_glob;
-     flux_msg_handler_f cb;
-     uint32_t rolemask;
- };
-
- int flux_msg_handler_addvec (flux_t *h,
-                              const struct flux_msg_handler_spec tab[],
-                              void *arg,
-                              flux_msg_handler_t **handlers[]);
-
- void flux_msg_handler_delvec (flux_msg_handler_t *handlers[]);
-
-
-DESCRIPTION
------------
-
-`flux_msg_handler_addvec()` creates and starts an array of message handlers,
-terminated by FLUX_MSGHANDLER_TABLE_END.  The new message handler objects
-are assigned to an internally allocated array, returned in _handlers_.
-The last entry in the array is set to NULL.
-
-`flux_msg_handler_delvec()` stops and destroys an array of message handlers
-returned from `flux_msg_handler_addvec()`.
-
-These functions are convenience functions which call
-`flux_msg_handler_create(3)`, `flux_msg_handler_start(3)`; and
-`flux_msg_handler_stop(3)`, `flux_msg_handler_destroy(3)` on each element
-of the array, respectively.
-
-If `flux_msg_handler_addvec()` encounters an error creating a message
-handler, all previously created message handlers in the array are destroyed
-before an error is returned.
-
-
-RETURN VALUE
-------------
-
-`flux_msg_handler_addvec()` returns zero on success.
-On error, -1 is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_msg_handler_create(3)
diff --git a/doc/man3/flux_msg_handler_addvec.rst b/doc/man3/flux_msg_handler_addvec.rst
new file mode 100644
index 000000000000..6d28f8100312
--- /dev/null
+++ b/doc/man3/flux_msg_handler_addvec.rst
@@ -0,0 +1,74 @@
+==========================
+flux_msg_handler_addvec(3)
+==========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   struct flux_msg_handler_spec {
+       int typemask;
+       const char *topic_glob;
+       flux_msg_handler_f cb;
+       uint32_t rolemask;
+   };
+
+   int flux_msg_handler_addvec (flux_t *h,
+                                const struct flux_msg_handler_spec tab[],
+                                void *arg,
+                                flux_msg_handler_t **handlers[]);
+
+   void flux_msg_handler_delvec (flux_msg_handler_t *handlers[]);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_msg_handler_addvec` creates and starts an array of message handlers,
+terminated by FLUX_MSGHANDLER_TABLE_END. The new message handler objects
+are assigned to an internally allocated array, returned in :var:`handlers`.
+The last entry in the array is set to NULL.
+
+:func:`flux_msg_handler_delvec` stops and destroys an array of message handlers
+returned from :func:`flux_msg_handler_addvec`.
+
+These functions are convenience functions which call
+:man3:`flux_msg_handler_create`, :man3:`flux_msg_handler_start`; and
+:man3:`flux_msg_handler_stop`, :man3:`flux_msg_handler_destroy` on each element
+of the array, respectively.
+
+If :func:`flux_msg_handler_addvec` encounters an error creating a message
+handler, all previously created message handlers in the array are destroyed
+before an error is returned.
+
+
+RETURN VALUE
+============
+
+:func:`flux_msg_handler_addvec` returns zero on success.
+On error, -1 is returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_msg_handler_create`
diff --git a/doc/man3/flux_msg_handler_create.adoc b/doc/man3/flux_msg_handler_create.adoc
deleted file mode 100644
index c42dd9ee98b5..000000000000
--- a/doc/man3/flux_msg_handler_create.adoc
+++ /dev/null
@@ -1,120 +0,0 @@
-flux_msg_handler_create(3)
-==========================
-:doctype: manpage
-
-
-NAME
-----
-flux_msg_handler_create, flux_msg_handler_destroy,
-flux_msg_handler_start, flux_msg_handler_stop -  manage message handlers
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_msg_handler_f)(flux_t *h,
-                                    flux_msg_handler_t *mh,
-                                    const flux_msg_t *msg,
-                                    void *arg);
-
- flux_msg_handler_t *
- flux_msg_handler_create (flux_t *h,
-                          const struct flux_match match,
-                          flux_msg_handler_f callback,
-                          void *arg);
-
- void flux_msg_handler_destroy (flux_msg_handler_t *mh);
-
- void flux_msg_handler_start (flux_msg_handler_t *mh);
-
- void flux_msg_handler_stop (flux_msg_handler_t *mh);
-
-
-DESCRIPTION
------------
-
-`flux_msg_handler_create()` registers _callback_ to be invoked when
-a message meeting _match_ criteria, as described in `flux_msg_cmp(3)`,
-is received on Flux broker handle _h_.
-
-The message handler must be started with `flux_msg_handler_start()` in
-order to receive messages.   Conversely, `flux_msg_handler_stop()` causes
-the message handler to stop receiving messages.  Starting and stopping
-are idempotent operations.
-
-The handle _h_ is monitored for FLUX_POLLIN events on the flux_reactor_t
-associated with the handle as described in `flux_set_reactor(3)`.
-This internal "handle watcher" is started when the first message handler
-is started, and stopped when the last message handler is stopped.
-
-Messages arriving on _h_ are internally read and dispatched to matching
-message handlers.  If multiple handlers match the message, the following
-rules apply:
-
-FLUX_MSGTYPE_REQUEST::
-Requests are first delivered to a message handler whose match.topic_glob
-is set to an exact string match of the message topic glob.  The most recently
-registered of these takes precedence.  If an exact match is unavailable,
-the message is delivered to the most recently registered message handler
-for which `flux_msg_cmp()` returns true.  If there is no match, an ENOSYS
-response is automatically generated by the dispatcher.
-
-FLUX_MSGTYPE_RESPONSE::
-Responses are first delivered to a matching RPC response handler
-(match.matchtag != FLUX_MATCHTAG_NONE).  If an RPC response handler
-does not match, responses are delivered to the most recently registered
-message handler for which `flux_msg_cmp()` returns true.  If there is no
-match, the response is discarded.
-
-FLUX_MSGTYPE_EVENT::
-Events are delivered to _all_ matching message handlers.
-
-`flux_msg_handler_destroy()` destroys a handler, after internally
-stopping it.
-
-
-CAVEATS
--------
-
-FLUX_MSGTYPE_EVENT messages are received on the handle only as
-requested by `flux_event_subscribe(3)`.
-
-`flux-broker(1)` only routes FLUX_MSGTYPE_REQUEST messages to comms
-modules according to their registered service name, which is the same as
-the module name.  Other handle instances such as those on the local connector
-cannot yet receive requests.
-
-
-RETURN VALUE
-------------
-
-`flux_msg_handler_create()` returns a flux_msg_handler_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_get_reactor(3), flux_reactor_start(3), flux_msg_cmp(3)
diff --git a/doc/man3/flux_msg_handler_create.rst b/doc/man3/flux_msg_handler_create.rst
new file mode 100644
index 000000000000..c32e00897d8c
--- /dev/null
+++ b/doc/man3/flux_msg_handler_create.rst
@@ -0,0 +1,116 @@
+==========================
+flux_msg_handler_create(3)
+==========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_msg_handler_f)(flux_t *h,
+                                      flux_msg_handler_t *mh,
+                                      const flux_msg_t *msg,
+                                      void *arg);
+
+   flux_msg_handler_t *flux_msg_handler_create (
+                                      flux_t *h,
+                                      const struct flux_match match,
+                                      flux_msg_handler_f callback,
+                                      void *arg);
+
+   void flux_msg_handler_destroy (flux_msg_handler_t *mh);
+
+   void flux_msg_handler_start (flux_msg_handler_t *mh);
+
+   void flux_msg_handler_stop (flux_msg_handler_t *mh);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_msg_handler_create` registers :var:`callback` to be invoked when
+a message meeting :var:`match` criteria, as described in :man3:`flux_msg_cmp`,
+is received on Flux broker handle :var:`h`.
+
+The message handler must be started with :func:`flux_msg_handler_start` in
+order to receive messages. Conversely, :func:`flux_msg_handler_stop` causes
+the message handler to stop receiving messages. Starting and stopping
+are idempotent operations.
+
+The handle :var:`h` is monitored for FLUX_POLLIN events on the
+:type:`flux_reactor_t` associated with the handle as described in
+:man3:`flux_set_reactor`.  This internal "handle watcher" is started when the
+first message handler is started, and stopped when the last message handler
+is stopped.
+
+Messages arriving on :var:`h` are internally read and dispatched to matching
+message handlers. If multiple handlers match the message, the following
+rules apply:
+
+FLUX_MSGTYPE_REQUEST
+   Requests are first delivered to a message handler whose
+   :var:`match.topic_glob` is set to an exact string match of the message
+   topic glob. The most recently registered of these takes precedence. If an
+   exact match is unavailable, the message is delivered to the most recently
+   registered message handler for which :man3:`flux_msg_cmp` returns true. If
+   there is no match, an ENOSYS response is automatically generated by the
+   dispatcher.
+
+FLUX_MSGTYPE_RESPONSE
+   Responses are first delivered to a matching RPC response handler
+   (:var:`match.matchtag` != FLUX_MATCHTAG_NONE). If an RPC response handler
+   does not match, responses are delivered to the most recently registered
+   message handler for which :man3:`flux_msg_cmp` returns true. If there is no
+   match, the response is discarded.
+
+FLUX_MSGTYPE_EVENT
+   Events are delivered to *all* matching message handlers.
+
+:func:`flux_msg_handler_destroy` destroys a handler, after internally
+stopping it.
+
+
+CAVEATS
+=======
+
+Although it is possible to register a message handler in a given :type:`flux_t`
+handle for any topic string, :man1:`flux-broker` does not automatically route
+matching requests or events to the handle.
+
+Requests are only routed if the handle has registered a matching service
+with :man3:`flux_service_register`, or for broker modules only, the service
+matches the module name.
+
+Events are only routed if the topic matches a subscription registered
+with :man3:`flux_event_subscribe`.
+
+
+RETURN VALUE
+============
+
+:func:`flux_msg_handler_create` returns a :type:`flux_msg_handler_t` object on
+success.  On error, NULL is returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_get_reactor`, :man3:`flux_reactor_run`, :man3:`flux_msg_cmp`
diff --git a/doc/man3/flux_msg_has_flag.rst b/doc/man3/flux_msg_has_flag.rst
new file mode 100644
index 000000000000..d9be92288a7c
--- /dev/null
+++ b/doc/man3/flux_msg_has_flag.rst
@@ -0,0 +1,134 @@
+====================
+flux_msg_has_flag(3)
+====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  bool flux_msg_has_flag (const flux_msg_t *msg, int flag);
+  int flux_msg_set_flag (flux_msg_t *msg, int flag);
+  int flux_msg_clear_flag (flux_msg_t *msg, int flag);
+
+  int flux_msg_set_private (flux_msg_t *msg);
+  bool flux_msg_is_private (const flux_msg_t *msg);
+
+  int flux_msg_set_streaming (flux_msg_t *msg);
+  bool flux_msg_is_streaming (const flux_msg_t *msg);
+
+  int flux_msg_set_noresponse (flux_msg_t *msg);
+  bool flux_msg_is_noresponse (const flux_msg_t *msg);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+These functions manipulate the Flux `MESSAGE FLAGS`_.
+
+:func:`flux_msg_has_flag` returns true if any flags in the :var:`flag` bitmask
+are set in :var:`msg`.
+
+:func:`flux_msg_set_flag` sets all flags in the :var:`flag` bitmask in
+:var:`msg`.
+
+:func:`flux_msg_clear_flag` clears all flags in the :var:`flag` bitmask in
+:var:`msg`.
+
+:func:`flux_msg_is_private` returns true if FLUX_MSGFLAG_PRIVATE is set
+in :var:`msg`.
+
+:func:`flux_msg_set_private` sets FLUX_MSGFLAG_PRIVATE in :var:`msg`.
+
+:func:`flux_msg_is_streaming` returns true if FLUX_MSGFLAG_STREAMING is set
+in :var:`msg`.
+
+:func:`flux_msg_set_streaming` sets FLUX_MSGFLAG_STREAMING in :var:`msg`.
+
+:func:`flux_msg_is_noresponse` returns true if FLUX_MSGFLAG_NORESPONSE is set
+in :var:`msg`.
+
+:func:`flux_msg_set_noresponse` sets FLUX_MSGFLAG_NORESPONSE in :var:`msg`.
+
+MESSAGE FLAGS
+=============
+
+The following message flags are defined by :doc:`RFC 3 <rfc:spec_3>`:
+
+FLUX_MSGFLAG_TOPIC
+  The message has a topic string.  This flag is updated by
+  :func:`flux_msg_set_topic`.
+
+FLUX_MSGFLAG_PAYLOAD
+  The message has a payload.  This flag is updated by
+  :func:`flux_msg_set_payload`, :func:`flux_msg_pack`, etc.
+
+FLUX_MSGFLAG_NORESPONSE
+  The request message should not be sent a response.  This flag is set
+  by :func:`flux_rpc` when the FLUX_RPC_NORESPONSE flag is set.
+
+FLUX_MSGFLAG_ROUTE
+  The request or response message has a route stack, although it may be empty.
+  This flag is updated by :func:`flux_msg_route_enable` and
+  :func:`flux_msg_route_disable`.
+
+FLUX_MSGFLAG_UPSTREAM
+  Force the broker to route a request upstream (towards the root on the tree
+  based overlay network) relative to the sending rank.  In other words,
+  prevent the request from being handled locally.  The message :var:`nodeid`
+  is interpreted as the *sending* rank when this flag is set.
+
+FLUX_MSGFLAG_PRIVATE
+  The event message should only be forwarded to connections authenticated
+  as the instance owner or the message :var:`userid`.
+
+FLUX_MSGFLAG_STREAMING
+  The request or response message is part of a streaming RPC, as defined
+  by  :doc:`RFC 6 <rfc:spec_6>`.  This flag is set by :func:`flux_rpc` when
+  the FLUX_RPC_STREAMING flag is set.
+
+FLUX_MSGFLAG_USER1
+  This flag is opaque to Flux's message handling semantics and may be assigned
+  application-specific meaning in the same way as the message payload.
+
+RETURN VALUE
+============
+
+:func:`flux_msg_has_flag`, :func:`flux_msg_is_private`,
+:func:`flux_msg_is_streaming`, and :func:`flux_msg_is_noresponse` return
+true if the specified flag is set, or false if if the flag is not set or
+the arguments are invalid.
+
+The remaining functions return 0 on success. On error, -1 is returned,
+and :var:`errno` is set appropriately.
+
+ERRORS
+======
+
+EINVAL
+   Some arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_3`
+
+:doc:`rfc:spec_6`
+
+SEE ALSO
+========
+
+:man3:`flux_rpc`
diff --git a/doc/man3/flux_open.adoc b/doc/man3/flux_open.adoc
deleted file mode 100644
index bab342c0487e..000000000000
--- a/doc/man3/flux_open.adoc
+++ /dev/null
@@ -1,98 +0,0 @@
-flux_open(3)
-============
-:doctype: manpage
-
-
-NAME
-----
-flux_open, flux_clone, flux_close - open/close connection to Flux Message Broker
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-flux_t *flux_open (const char *uri, int flags);
-
-void flux_close (flux_t *h);
-
-flux_t *flux_clone (flux_t *orig);
-
-
-DESCRIPTION
------------
-
-`flux_open()` creates a `flux_t` handle, used to communicate with the
-Flux message broker.
-
-The _uri_ scheme (before "://") specifies the "connector"
-that will be used to establish the connection.  The _uri_ path
-(after "://") is parsed by the connector.  If _uri_ is NULL,
-the value of $FLUX_URI is used, if set.
-
-_flags_ is the logical "or" of zero or more of the following flags:
-
-FLUX_O_TRACE::
-Dumps message trace to stderr.
-
-FLUX_O_MATCHDEBUG::
-Prints diagnostic to stderr when matchtags are leaked, for example when
-a streaming RPC is destroyed without receiving a error response as
-end-of-stream, or a regular RPC is destroyed without being fulfilled at all.
-
-FLUX_O_NONBLOCK::
-The `flux_send()` and `flux_recv()` functions should never block.
-
-`flux_clone()` creates another reference to a `flux_t` handle that is
-identical to the original in all respects except that it does not inherit
-a copy of the original handle's "aux" hash, or its reactor and message
-dispatcher references.  By creating a clone, and calling `flux_set_reactor()`
-on it, one can create message handlers on the cloned handle that run on a
-different reactor than the one associated with the original handle.
-
-`flux_close()` destroys a `flux_t` handle, closing its connection with
-the Flux message broker.
-
-
-RETURN VALUE
-------------
-
-`flux_open()` and `flux_clone()` return a `flux_t` handle on success.
-On error, NULL is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-_uri_ was NULL and $FLUX_URI was not set, or other arguments were invalid.
-
-ENOMEM::
-Out of memory.
-
-
-EXAMPLES
---------
-
-This example opens the Flux broker using the default connector
-and path, requests the broker rank, and finally closes the broker handle.
-
-....
-include::topen.c[]
-....
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
diff --git a/doc/man3/flux_open.rst b/doc/man3/flux_open.rst
new file mode 100644
index 000000000000..724367d5e180
--- /dev/null
+++ b/doc/man3/flux_open.rst
@@ -0,0 +1,128 @@
+============
+flux_open(3)
+============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_t *flux_open (const char *uri, int flags);
+
+   flux_t *flux_open_ex (const char *uri,
+                         int flags,
+                         flux_error_t *error);
+
+   void flux_close (flux_t *h);
+
+   flux_t *flux_clone (flux_t *h);
+
+   int flux_reconnect (flux_t *h);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_open` and :func:`flux_open_ex` create a :type:`flux_t` handle, used
+to communicate with the Flux message broker. :func:`flux_open_ex` takes an
+optional pointer to a :type:`flux_error_t` structure which, when non-NULL, will
+be used to store any errors which may have otherwise gone to :var:`stderr`.
+
+The :var:`uri` scheme (before "://") specifies the "connector" that will be used
+to establish the connection. The :var:`uri` path (after "://") is parsed by the
+connector. If :var:`uri` is NULL, the value of :envvar:`FLUX_URI` is used.  If
+:envvar:`FLUX_URI` is not set, a compiled-in default URI is used.
+
+*flags* is the logical "or" of zero or more of the following flags:
+
+FLUX_O_TRACE
+   Dumps message trace to stderr.
+
+FLUX_O_CLONE
+   Used internally by :func:`flux_clone` (see below).
+
+FLUX_O_MATCHDEBUG
+   Prints diagnostic to stderr when matchtags are leaked, for example when
+   a streaming RPC is destroyed without receiving a error response as
+   end-of-stream, or a regular RPC is destroyed without being fulfilled at all.
+
+FLUX_O_NONBLOCK
+   The :man3:`flux_send` and :man3:`flux_recv` functions should never block.
+
+FLUX_O_TEST_NOSUB
+   Make :man3:`flux_event_subscribe` and :man3:`flux_event_unsubscribe` no-ops.
+   This may be useful in specialized situations with the ``loop://`` connector,
+   where no message handler is available to service subscription RPCs.
+
+FLUX_O_RPCTRACK
+   Track pending RPCs so that they can receive automatic ECONNRESET failure
+   responses if the broker connection is re-established with
+   :func:`flux_reconnect`.  Tracking incurs a small overhead.  This flag can
+   only be specified with :func:`flux_open`, not :man3:`flux_flags_set`.
+
+:func:`flux_reconnect` may be called from a communications error callback
+registered with :man3:`flux_comms_error_set`.  The current connection is
+closed and a new one is established with a new UUID.  Since responses addressed
+to the old UUID will not be routed to the new connection, RPCs that are pending
+before :func:`flux_reconnect` remain pending indefinitely without
+FLUX_O_RPCTRACK.  After a successful reconnect, the following additional steps
+may be needed before a client can continue normal operation:
+
+- Wait until the broker has entered RUN state by making an RPC to ``state_machine.wait``
+- Restore service registrations.
+- Restore event subscriptions.
+
+:func:`flux_clone` creates another reference to a :type:`flux_t` handle that is
+identical to the original in all respects except that it does not inherit
+a copy of the original handle's "aux" hash, or its reactor and message
+dispatcher references. By creating a clone, and calling
+:man3:`flux_set_reactor` on it, one can create message handlers on the cloned
+handle that run on a different reactor than the one associated with the
+original handle.
+
+:func:`flux_close` destroys a :type:`flux_t` handle, closing its connection with
+the Flux message broker.
+
+
+RETURN VALUE
+============
+
+:func:`flux_open` and :func:`flux_clone` return a :type:`flux_t`` handle on
+success.  On error, NULL is returned, with :var:`errno` set.
+
+
+ERRORS
+======
+
+EINVAL
+   One or more arguments was invalid.
+
+ENOMEM
+   Out of memory.
+
+
+EXAMPLES
+========
+
+This example opens the Flux broker using the default connector
+and path, requests the broker rank, and finally closes the broker handle.
+
+.. literalinclude:: example/open.c
+  :language: c
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-uri`, :man3:`flux_comms_error_set`
diff --git a/doc/man3/flux_periodic_watcher_create.adoc b/doc/man3/flux_periodic_watcher_create.adoc
deleted file mode 100644
index cb4f8f11969d..000000000000
--- a/doc/man3/flux_periodic_watcher_create.adoc
+++ /dev/null
@@ -1,92 +0,0 @@
-flux_periodic_watcher_create(3)
-===============================
-:doctype: manpage
-
-
-NAME
-----
-flux_periodic_watcher_create, flux_periodic_watcher_reset -  set/reset a timer
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_watcher_f)(flux_reactor_t *r,
-                                flux_watcher_t *w,
-                                int revents, void *arg);
-
- typedef double (*flux_reschedule_f) (flux_watcher_t *w, double now, void *arg);
-
- flux_watcher_t *flux_periodic_watcher_create (flux_reactor_t *r,
-                                               double offset, double interval,
-                                               flux_reschedule_f reschedule_cb, 
-                                               flux_watcher_f callback,
-                                               void *arg);
-
- void flux_periodic_watcher_reset (flux_watcher_t *w,
-                                   double offset, double interval);
-
-
-DESCRIPTION
------------
-
-`flux_periodic_watcher_create()` creates a flux_watcher_t object which
-monitors for periodic events.  The periodic watcher will trigger when the
-wall clock time _offset_ has elapsed, and optionally again ever _interval_
-of wall clock time thereafter. If the _reschedule_cb_ parameter is used,
-then _offset_ and _interval_ are ignored, and instead each time the
-periodic watcher is scheduled the reschedule callback will be called
-with the current time, and is expected to return the next absolute time
-at which the watcher should be scheduled.
-
-Unlike timer events, a periodic watcher monitors wall clock or system time,
-not the actual time that passes. Thus, a periodic watcher can be used
-to run a callback when system time reaches a certain point. For example,
-if a periodic watcher is set to run with an _offset_ of 10 seconds, and
-then system time is set back by 1 hour, it will take approximately 1 hour,
-10 seconds for the watcher to execute.
-
-If a periodic watcher is running in manual reschedule mode (_reschedule_cb_
-is non-NULL), and the user-provided reschedule callback returns a time
-that is before the current time, the watcher will be silently stopped.
-
-Note: the Flux reactor is based on libev.  For additional important
-information on the behavior of periodic, refer to the libev documentation
-on `ev_periodic`.
-
-
-RETURN VALUE
-------------
-
-`flux_periodic_watcher_create()` returns a flux_watcher_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_watcher_start(3), flux_reactor_start(3), flux_timer_watcher_create(3)
-
-http://software.schmorp.de/pkg/libev.html[libev home page]
diff --git a/doc/man3/flux_periodic_watcher_create.rst b/doc/man3/flux_periodic_watcher_create.rst
new file mode 100644
index 000000000000..cf49b21502ee
--- /dev/null
+++ b/doc/man3/flux_periodic_watcher_create.rst
@@ -0,0 +1,90 @@
+===============================
+flux_periodic_watcher_create(3)
+===============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_watcher_f)(flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg);
+
+   typedef double (*flux_reschedule_f)(flux_watcher_t *w,
+                                       double now,
+                                       void *arg);
+
+   flux_watcher_t *flux_periodic_watcher_create (
+                                         flux_reactor_t *r,
+                                         double offset,
+                                         double interval,
+                                         flux_reschedule_f resched_cb,
+                                         flux_watcher_f callback,
+                                         void *arg);
+
+   void flux_periodic_watcher_reset (flux_watcher_t *w,
+                                     double offset,
+                                     double interval);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_periodic_watcher_create` creates a :type:`flux_watcher_t` object
+which monitors for periodic events. The periodic watcher will trigger when the
+wall clock time :var:`offset` has elapsed, and optionally again every
+:var:`interval` of wall clock time thereafter. If the :var:`reschedule_cb`
+parameter is used, then :var:`offset` and :var:`interval` are ignored, and
+instead each time the periodic watcher is scheduled the reschedule callback
+will be called with the current time, and is expected to return the next
+absolute time at which the watcher should be scheduled.
+
+Unlike timer events, a periodic watcher monitors wall clock or system time,
+not the actual time that passes. Thus, a periodic watcher can be used
+to run a callback when system time reaches a certain point. For example,
+if a periodic watcher is set to run with an :var:`offset` of 10 seconds, and
+then system time is set back by 1 hour, it will take approximately 1 hour,
+10 seconds for the watcher to execute.
+
+If a periodic watcher is running in manual reschedule mode (:var:`reschedule_cb`
+is non-NULL), and the user-provided reschedule callback returns a time
+that is before the current time, the watcher will be silently stopped.
+
+Note: the Flux reactor is based on libev. For additional important
+information on the behavior of periodic, refer to the libev documentation
+on ``ev_periodic``.
+
+
+RETURN VALUE
+============
+
+:func:`flux_periodic_watcher_create` returns a :type:`flux_watcher_t` object
+on success.  On error, NULL is returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+libev: http://software.schmorp.de/pkg/libev.html
+
+
+SEE ALSO
+========
+
+:man3:`flux_watcher_start`, :man3:`flux_reactor_run`, :man3:`flux_timer_watcher_create`
diff --git a/doc/man3/flux_pollevents.adoc b/doc/man3/flux_pollevents.adoc
deleted file mode 100644
index 7570e3452e61..000000000000
--- a/doc/man3/flux_pollevents.adoc
+++ /dev/null
@@ -1,211 +0,0 @@
-flux_pollevents(3)
-==================
-:doctype: manpage
-
-
-NAME
-----
-flux_pollevents, flux_pollfd - poll Flux broker handle
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-int flux_pollevents (flux_t *h);
-
-int flux_pollfd (flux_t *h);
-
-
-DESCRIPTION
------------
-
-`flux_pollevents()` returns a bitmask of poll flags for handle _h_.
-
-`flux_pollfd()` obtains a file descriptor that becomes readable, in
-an edge triggered fashion, when `flux_pollevents()` has poll flags
-raised.
-
-Valid poll flags are:
-
-FLUX_POLLIN::
-Handle is ready for reading.
-
-FLUX_POLLOUT::
-Handle is ready for writing.
-
-FLUX_POLLERR::
-Handle has experienced an error.
-
-These functions can be used to integrate a `flux_t` handle into an
-external event loop.  They are analogous to the ZMQ_FD and ZMQ_EVENTS
-socket options provided by ZeroMQ.
-
-
-RETURN VALUE
-------------
-
-`flux_pollevents()` returns flags on success.  On error, -1 is returned,
-and errno is set appropriately.
-
-`flux_pollfd()` returns a file descriptor on success.  On error, -1 is
-returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-Some arguments were invalid.
-
-
-EXAMPLES
---------
-
-Here is an example of a libev "composite watcher" for a Flux handle
-using the hooks provided above.  This code, more or less, is used internally
-to integrate flux handles into the Flux reactor, which is based on libev.
-Refer to the libev documentation for background on how libev works.
-
-There are a total of four different types of libev watcher in the
-composite watcher.  libev "prepare" and "check" callbacks are executed just
-before and just after libev blocks internally in the poll(2) system call.
-Here they are used to test `flux_pollevents()`, make user callbacks,
-and enable/disable no-op "io" and "idle" watchers.   The io watcher
-watches for EV_READ on `flux_pollfd()` file descriptor.  The idle watcher,
-if enabled, is always ready and thus causes the event loop to spin.
-
-When `flux_pollevents()` has poll flags asserted, the idle watcher is enabled.
-When `flux_pollevents()` has no poll flags asserted, the idle watcher is
-disabled and the io watcher is enabled.  While the idle and io watchers
-have no callbacks, if either is enabled and ready, the event loop must
-execute the prepare and check callbacks.
-
-The net results are 1) the edge-triggered notification provided by
-`flux_pollfd()` is integrated with libev's level-triggered watcher
-processing;  2) the handle is able to give control back to the event
-loop between handle event callbacks to preserve fairness--in other words
-it doesn't have to consume events until they they are gone in one
-callback; and 3) the event loop is able to sleep when there are no
-handle events pending.
-
-....
-// ev_flux.h
-#include <ev.h>
-
-struct ev_flux;
-
-typedef void (*ev_flux_f)(struct ev_loop *loop,
-                          struct ev_flux *w, int revents);
-
-struct ev_flux {
-    ev_io       io_w;
-    ev_prepare  prepare_w;
-    ev_idle     idle_w;
-    ev_check    check_w;
-    flux_t      *h;
-    int         pollfd;
-    int         events;
-    ev_flux_f   cb;
-    void        *data;
-};
-....
-
-
-....
-// ev_flux.c
-static int get_pollevents (flux_t *h)
-{
-    int e = flux_pollevents (h);
-    int events = 0;
-    if (e < 0 || (e & FLUX_POLLERR))
-        events |= EV_ERROR;
-    if ((e & FLUX_POLLIN))
-        events |= EV_READ;
-    if ((e & FLUX_POLLOUT))
-        events |= EV_WRITE;
-    return events;
-}
-
-static void prepare_cb (struct ev_loop *loop, ev_prepare *w,
-                        int revents)
-{
-    struct ev_flux *fw = (struct ev_flux *)((char *)w
-                       - offsetof (struct ev_flux, prepare_w));
-    int events = get_pollevents (fw->h);
-
-    if ((events & fw->events) || (events & EV_ERROR))
-        ev_idle_start (loop, &fw->idle_w);
-    else
-        ev_io_start (loop, &fw->io_w);
-}
-
-static void check_cb (struct ev_loop *loop, ev_check *w,
-                      int revents)
-{
-    struct ev_flux *fw = (struct ev_flux *)((char *)w
-                       - offsetof (struct ev_flux, check_w));
-    int events = get_pollevents (fw->h);
-
-    ev_io_stop (loop, &fw->io_w);
-    ev_idle_stop (loop, &fw->idle_w);
-
-    if ((events & fw->events) || (events & EV_ERROR))
-        fw->cb (loop, fw, events);
-}
-
-int ev_flux_init (struct ev_flux *w, ev_flux_f cb,
-                  flux_t *h, int events)
-{
-    w->cb = cb;
-    w->h = h;
-    w->events = events;
-    if ((w->pollfd = flux_pollfd (h)) < 0)
-        return -1;
-
-    ev_prepare_init (&w->prepare_w, prepare_cb);
-    ev_check_init (&w->check_w, check_cb);
-    ev_idle_init (&w->idle_w, NULL);
-    ev_io_init (&w->io_w, NULL, w->pollfd, EV_READ);
-
-    return 0;
-}
-
-void ev_flux_start (struct ev_loop *loop, struct ev_flux *w)
-{
-    ev_prepare_start (loop, &w->prepare_w);
-    ev_check_start (loop, &w->check_w);
-}
-
-void ev_flux_stop (struct ev_loop *loop, struct ev_flux *w)
-{
-    ev_prepare_stop (loop, &w->prepare_w);
-    ev_check_stop (loop, &w->check_w);
-    ev_io_stop (loop, &w->io_w);
-    ev_idle_stop (loop, &w->idle_w);
-}
-....
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod
-
-http://api.zeromq.org/4-0:zmq-getsockopt
-
-http://funcptr.net/2013/04/20/embedding-zeromq-in-the-libev-event-loop
diff --git a/doc/man3/flux_pollevents.rst b/doc/man3/flux_pollevents.rst
new file mode 100644
index 000000000000..da8171298841
--- /dev/null
+++ b/doc/man3/flux_pollevents.rst
@@ -0,0 +1,200 @@
+==================
+flux_pollevents(3)
+==================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  int flux_pollevents (flux_t *h);
+
+  int flux_pollfd (flux_t *h);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_pollevents` returns a bitmask of poll flags for handle :var:`h`.
+
+:func:`flux_pollfd` obtains a file descriptor that becomes readable, in
+an edge triggered fashion, when :func:`flux_pollevents` has poll flags
+raised.
+
+Valid poll flags are:
+
+FLUX_POLLIN
+   Handle is ready for reading.
+
+FLUX_POLLOUT
+   Handle is ready for writing.
+
+FLUX_POLLERR
+   Handle has experienced an error.
+
+These functions can be used to integrate a :type:`flux_t` handle into an
+external event loop. They are analogous to the ZMQ_FD and ZMQ_EVENTS
+socket options provided by ZeroMQ.
+
+
+RETURN VALUE
+============
+
+:func:`flux_pollevents` returns flags on success. On error, -1 is returned,
+and :var:`errno` is set appropriately.
+
+:func:`flux_pollfd` returns a file descriptor on success. On error, -1 is
+returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   Some arguments were invalid.
+
+
+EXAMPLES
+========
+
+Here is an example of a libev "composite watcher" for a Flux handle
+using the hooks provided above. This code, more or less, is used internally
+to integrate flux handles into the Flux reactor, which is based on libev.
+Refer to the libev documentation for background on how libev works.
+
+There are a total of four different types of libev watcher in the
+composite watcher. libev "prepare" and "check" callbacks are executed
+just before and just after libev blocks internally in the
+:linux:man2:`poll` system call.  Here they are used to test
+:func:`flux_pollevents`, make user callbacks, and enable/disable no-op
+"io" and "idle" watchers. The io watcher watches for EV_READ on
+:func:`flux_pollfd` file descriptor. The idle watcher, if enabled, is
+always ready and thus causes the event loop to spin.
+
+When :func:`flux_pollevents` has poll flags asserted, the idle watcher is
+enabled.  When :func:`flux_pollevents` has no poll flags asserted, the idle
+watcher is disabled and the io watcher is enabled. While the idle and io
+watchers have no callbacks, if either is enabled and ready, the event loop
+must execute the prepare and check callbacks.
+
+The net results are 1) the edge-triggered notification provided by
+:func:`flux_pollfd` is integrated with libev's level-triggered watcher
+processing; 2) the handle is able to give control back to the event
+loop between handle event callbacks to preserve fairness, i.e.
+it doesn't have to consume events until they they are gone in one
+callback; and 3) the event loop is able to sleep when there are no
+handle events pending.
+
+::
+
+   // ev_flux.h
+   #include <ev.h>
+
+   struct ev_flux;
+
+   typedef void (*ev_flux_f)(struct ev_loop *loop,
+                             struct ev_flux *w, int revents);
+
+   struct ev_flux {
+       ev_io       io_w;
+       ev_prepare  prepare_w;
+       ev_idle     idle_w;
+       ev_check    check_w;
+       flux_t      *h;
+       int         pollfd;
+       int         events;
+       ev_flux_f   cb;
+       void        *data;
+   };
+
+::
+
+   // ev_flux.c
+   static int get_pollevents (flux_t *h)
+   {
+       int e = flux_pollevents (h);
+       int events = 0;
+       if (e < 0 || (e & FLUX_POLLERR))
+           events |= EV_ERROR;
+       if ((e & FLUX_POLLIN))
+           events |= EV_READ;
+       if ((e & FLUX_POLLOUT))
+           events |= EV_WRITE;
+       return events;
+   }
+
+   static void prepare_cb (struct ev_loop *loop, ev_prepare *w,
+                           int revents)
+   {
+       struct ev_flux *fw = (struct ev_flux *)((char *)w
+                          - offsetof (struct ev_flux, prepare_w));
+       int events = get_pollevents (fw->h);
+
+       if ((events & fw->events) || (events & EV_ERROR))
+           ev_idle_start (loop, &fw->idle_w);
+       else
+           ev_io_start (loop, &fw->io_w);
+   }
+
+   static void check_cb (struct ev_loop *loop, ev_check *w,
+                         int revents)
+   {
+       struct ev_flux *fw = (struct ev_flux *)((char *)w
+                          - offsetof (struct ev_flux, check_w));
+       int events = get_pollevents (fw->h);
+
+       ev_io_stop (loop, &fw->io_w);
+       ev_idle_stop (loop, &fw->idle_w);
+
+       if ((events & fw->events) || (events & EV_ERROR))
+           fw->cb (loop, fw, events);
+   }
+
+   int ev_flux_init (struct ev_flux *w, ev_flux_f cb,
+                     flux_t *h, int events)
+   {
+       w->cb = cb;
+       w->h = h;
+       w->events = events;
+       if ((w->pollfd = flux_pollfd (h)) < 0)
+           return -1;
+
+       ev_prepare_init (&w->prepare_w, prepare_cb);
+       ev_check_init (&w->check_w, check_cb);
+       ev_idle_init (&w->idle_w, NULL);
+       ev_io_init (&w->io_w, NULL, w->pollfd, EV_READ);
+
+       return 0;
+   }
+
+   void ev_flux_start (struct ev_loop *loop, struct ev_flux *w)
+   {
+       ev_prepare_start (loop, &w->prepare_w);
+       ev_check_start (loop, &w->check_w);
+   }
+
+   void ev_flux_stop (struct ev_loop *loop, struct ev_flux *w)
+   {
+       ev_prepare_stop (loop, &w->prepare_w);
+       ev_check_stop (loop, &w->check_w);
+       ev_io_stop (loop, &w->io_w);
+       ev_idle_stop (loop, &w->idle_w);
+   }
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+libev API: http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod
+
+zmq_getsockopt(3): http://api.zeromq.org/4-0:zmq-getsockopt
+
+Embedding ZeroMQ in the libev event loop:
+http://funcptr.net/2013/04/20/embedding-zeromq-in-the-libev-event-loop
diff --git a/doc/man3/flux_reactor_create.adoc b/doc/man3/flux_reactor_create.adoc
deleted file mode 100644
index 8a09352602b9..000000000000
--- a/doc/man3/flux_reactor_create.adoc
+++ /dev/null
@@ -1,129 +0,0 @@
-flux_reactor_create(3)
-======================
-:doctype: manpage
-
-
-NAME
-----
-flux_reactor_create, flux_reactor_destroy, flux_reactor_run, flux_reactor_stop, flux_reactor_stop_error, flux_reactor_active_incref, flux_reactor_active_decref - create/destroy/control event reactor object
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-flux_reactor_t *flux_reactor_create (int flags);
-
-void flux_reactor_destroy (flux_reactor_t *r);
-
-int flux_reactor_run (flux_reactor_t *r, int flags);
-
-void flux_reactor_stop (flux_reactor_t *r);
-
-void flux_reactor_stop_error (flux_reactor_t *r);
-
-void flux_reactor_active_incref (flux_reactor_t *r);
-
-void flux_reactor_active_decref (flux_reactor_t *r);
-
-DESCRIPTION
------------
-
-`flux_reactor_create()` creates a flux_reactor_t object which can be used
-to monitor for events on file descriptors, ZeroMQ sockets, timers, and
-flux_t broker handles.
-
-There is currently only one possible flag for reactor creation:
-
-FLUX_REACTOR_SIGCHLD::
-The reactor will internally register a SIGCHLD handler and be capable
-of handling flux child watchers (see flux_child_watcher_create(3)).
-
-For each event source and type that is to be monitored, a flux_watcher_t
-object is created using a type-specific create function, and started
-with flux_watcher_start(3).
-
-For each event source and type that is to be monitored, a flux_watcher_t
-object is created and associated with a specific reactor using a type-specific
-create function, and started with flux_watcher_start(3). To receive events,
-control must be transferred to the reactor event loop by calling
-`flux_reactor_run()`.
-
-The full list of flux reactor run flags is as follows:
-
-FLUX_REACTOR_NOWAIT::
-Run one reactor loop iteration without blocking.
-
-FLUX_REACTOR_ONCE::
-Run one reactor loop iteration, blocking until at least one event is handled.
-
-flux_reactor_run() processes events until one of the following conditions
-is met:
-
-* There are no more active watchers.
-* The `flux_reactor_stop()` or `flux_reactor_stop_error()` functions
-  are called by one of the watchers.
-* Flags include FLUX_REACTOR_NOWAIT and one reactor loop iteration
-  has been completed.
-* Flags include FLUX_REACTOR_ONCE, at least one event has been handled,
-  and one reactor loop iteration has been completed.
-
-If `flux_reactor_stop_error()` is called, this will cause
-`flux_reactor_run()` to return -1 indicating that an error has occurred.
-The caller should ensure that a valid error code has been assigned to
-errno(3) before calling this function.
-
-`flux_reactor_destroy()` releases an internal reference taken at
-`flux_reactor_create()` time.  Freeing of the underlying resources will
-be deferred if there are any remaining watchers associated with the reactor.
-
-`flux_reactor_active_decref()` and `flux_reactor_active_incref()` manipulate
-the reactor's internal count of active watchers.  Each active watcher takes
-a reference count on the reactor, and the reactor returns when this count
-reaches zero.  It is useful sometimes to have a watcher that can remain
-active without preventing the reactor from exiting.  To achieve this,
-call `flux_reactor_active_decref()` after the watcher is started, and
-`flux_reactor_active_incref()` before the watcher is stopped.
-Remember that destroying an active reactor internally stops it,
-so be sure to stop/incref such a watcher first.
-
-RETURN VALUE
-------------
-
-`flux_reactor_create()` returns a flux_reactor_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-`flux_reactor_run()` returns the number of active watchers on success.
-On failure, it returns -1 with errno set.  A failure return is triggered
-when the application sets errno and calls `flux_reactor_stop_error()`.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_fd_watcher_create(3), flux_handle_watcher_create(3),
-flux_zmq_watcher_create(3), flux_timer_watcher_create(3),
-flux_watcher_start(3)
-
-http://software.schmorp.de/pkg/libev.html[libev home page]
diff --git a/doc/man3/flux_reactor_create.rst b/doc/man3/flux_reactor_create.rst
new file mode 100644
index 000000000000..5ab2aafcf63b
--- /dev/null
+++ b/doc/man3/flux_reactor_create.rst
@@ -0,0 +1,126 @@
+======================
+flux_reactor_create(3)
+======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  flux_reactor_t *flux_reactor_create (int flags);
+
+  void flux_reactor_destroy (flux_reactor_t *r);
+
+  int flux_reactor_run (flux_reactor_t *r, int flags);
+
+  void flux_reactor_stop (flux_reactor_t *r);
+
+  void flux_reactor_stop_error (flux_reactor_t *r);
+
+  void flux_reactor_active_incref (flux_reactor_t *r);
+
+  void flux_reactor_active_decref (flux_reactor_t *r);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_reactor_create` creates a :type:`flux_reactor_t` object which can
+be used to monitor for events on file descriptors, ZeroMQ sockets, timers, and
+:type:`flux_t` broker handles.
+
+There is currently only one possible flag for reactor creation:
+
+FLUX_REACTOR_SIGCHLD
+   The reactor will internally register a SIGCHLD handler and be capable
+   of handling flux child watchers (see :man3:`flux_child_watcher_create`).
+
+For each event source and type that is to be monitored, a :type:`flux_watcher_t`
+object is created using a type-specific create function, and started
+with :man3:`flux_watcher_start`.
+
+For each event source and type that is to be monitored, a
+:type:`flux_watcher_t` object is created and associated with a specific
+reactor using a type-specific create function, and started with
+:man3:`flux_watcher_start`. To receive events, control must be transferred
+to the reactor event loop by calling :func:`flux_reactor_run`.
+
+The full list of flux reactor run flags is as follows:
+
+FLUX_REACTOR_NOWAIT
+   Run one reactor loop iteration without blocking.
+
+FLUX_REACTOR_ONCE
+   Run one reactor loop iteration, blocking until at least one event is handled.
+
+:func:`flux_reactor_run` processes events until one of the following conditions
+is met:
+
+-  There are no more active watchers.
+
+-  The :func:`flux_reactor_stop` or :func:`flux_reactor_stop_error` functions
+   are called by one of the watchers.
+
+-  Flags include FLUX_REACTOR_NOWAIT and one reactor loop iteration
+   has been completed.
+
+-  Flags include FLUX_REACTOR_ONCE, at least one event has been handled,
+   and one reactor loop iteration has been completed.
+
+If :func:`flux_reactor_stop_error` is called, this will cause
+:func:`flux_reactor_run` to return -1 indicating that an error has occurred.
+The caller should ensure that a valid error code has been assigned to
+:linux:man3:`errno` before calling this function.
+
+:func:`flux_reactor_destroy` releases an internal reference taken at
+:func:`flux_reactor_create` time. Freeing of the underlying resources will
+be deferred if there are any remaining watchers associated with the reactor.
+
+:func:`flux_reactor_active_decref` and :func:`flux_reactor_active_incref`
+manipulate the reactor's internal count of active watchers. Each active
+watcher takes a reference count on the reactor, and the reactor returns
+when this count reaches zero. It is useful sometimes to have a watcher that
+can remain active without preventing the reactor from exiting. To achieve this,
+call :func:`flux_reactor_active_decref` after the watcher is started, and
+:func:`flux_reactor_active_incref` before the watcher is stopped.
+Remember that destroying an active reactor internally stops it,
+so be sure to stop/incref such a watcher first.
+
+
+RETURN VALUE
+============
+
+:func:`flux_reactor_create` returns a :type:`flux_reactor_t` object on success.
+On error, NULL is returned, and :var:`errno` is set appropriately.
+
+:func:`flux_reactor_run` returns the number of active watchers on success.
+On failure, it returns -1 with :var:`errno` set. A failure return is triggered
+when the application sets :var:`errno` and calls
+:func:`flux_reactor_stop_error`.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+libev: http://software.schmorp.de/pkg/libev.html
+
+
+SEE ALSO
+========
+
+:man3:`flux_fd_watcher_create`, :man3:`flux_handle_watcher_create`,
+:man3:`flux_timer_watcher_create`, :man3:`flux_watcher_start`
diff --git a/doc/man3/flux_reactor_now.adoc b/doc/man3/flux_reactor_now.adoc
deleted file mode 100644
index 4e9baff6fa18..000000000000
--- a/doc/man3/flux_reactor_now.adoc
+++ /dev/null
@@ -1,63 +0,0 @@
-flux_watcher_now(3)
-===================
-:doctype: manpage
-
-
-NAME
-----
-flux_reactor_now, flux_reactor_now_update - get/update reactor time
-
-
-SYNOPSIS
---------
-
-double flux_reactor_now (flux_reactor_t *r);
-
-void flux_reactor_now_update (flux_reactor_t *r);
-
-double flux_reactor_time (void);
-
-
-DESCRIPTION
------------
-
-`flux_reactor_now()` returns the current reactor time, which is the time
-the reactor began processing events.  The time will not be updated until
-the reactor runs out of events and wakes up again.  This is a lighter
-weight alternative to system calls when only coarse event timing is needed,
-e.g. when all events processed in a given wakeup can be considered
-simultaneous.
-
-`flux_reactor_now_update()` forces an update to reactor time.
-This may be useful when the reactor has not run for a while and timing
-calculations relative to reactor time need to be made, for example when
-creating timer watchers.
-
-`flux_reactor_time()` returns the system time as a double.
-Reactor time is a snapshot of `flux_reactor_time()`.
-
-Note: the Flux reactor is based on libev.  For additional information
-on the behavior of reactor time, refer to the libev documentation on
-`ev_now` and `ev_now_update()`.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_reactor_create (3)
-
-http://software.schmorp.de/pkg/libev.html[libev home page]
diff --git a/doc/man3/flux_reactor_now.rst b/doc/man3/flux_reactor_now.rst
new file mode 100644
index 000000000000..158515915b13
--- /dev/null
+++ b/doc/man3/flux_reactor_now.rst
@@ -0,0 +1,54 @@
+===================
+flux_watcher_now(3)
+===================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  double flux_reactor_now (flux_reactor_t *r);
+
+  void flux_reactor_now_update (flux_reactor_t *r);
+
+  double flux_reactor_time (void);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_reactor_now` returns the current reactor time, which is the time
+the reactor began processing events. The time will not be updated until
+the reactor runs out of events and wakes up again. This is a lighter
+weight alternative to system calls when only coarse event timing is needed,
+e.g. when all events processed in a given wakeup can be considered
+simultaneous.
+
+:func:`flux_reactor_now_update` forces an update to reactor time.
+This may be useful when the reactor has not run for a while and timing
+calculations relative to reactor time need to be made, for example when
+creating timer watchers.
+
+:func:`flux_reactor_time` returns the system time as a double.
+Reactor time is a snapshot of :func:`flux_reactor_time`.
+
+Note: the Flux reactor is based on libev. For additional information
+on the behavior of reactor time, refer to the libev documentation on
+``ev_now`` and :func:`ev_now_update`.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+libev: http://software.schmorp.de/pkg/libev.html
+
+
+SEE ALSO
+========
+
+:man3:`flux_reactor_create`
diff --git a/doc/man3/flux_recv.adoc b/doc/man3/flux_recv.adoc
deleted file mode 100644
index 4b73f94b6224..000000000000
--- a/doc/man3/flux_recv.adoc
+++ /dev/null
@@ -1,107 +0,0 @@
-flux_recv(3)
-============
-:doctype: manpage
-
-
-NAME
-----
-flux_recv - receive message using Flux Message Broker
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-flux_msg_t *flux_recv (flux_t *h, struct flux_match match, int flags);
-
-
-DESCRIPTION
------------
-
-`flux_recv()` receives a message using the Flux Message broker,
-previously opened with `flux_open()` on handle _h_.
-The message should eventually be destroyed with `flux_msg_destroy()`.
-
-_match_ is a message match structure which limits which messages
-can be received.
-....
-struct flux_match {
-    int typemask;      // bitmask of matching message types
-    uint32_t matchtag; // matchtag
-    char *topic_glob;  // glob matching topic string
-};
-....
-The following initializers are available for _match_:
-
-FLUX_MATCH_ANY::
-Match any message.
-
-FLUX_MATCH_EVENT::
-Match any event message.
-
-For additional details on how to use _match_, see flux_msg_cmp(3).
-
-_flags_ is the logical "or" of zero or more of the following flags:
-
-FLUX_O_TRACE::
-Dumps _msg_ to stderr.
-
-FLUX_O_NONBLOCK::
-If unable to receive a matching message, return an error rather than block.
-
-Internally, flags are the logical "or" of _flags_ and the flags provided
-to `flux_open()` when the handle was created.
-
-Messages that do not meet _match_ criteria, are requeued with
-`flux_requeue()` for later consumption.
-
-
-RETURN VALUE
-------------
-
-`flux_recv()` returns a message on success.  On error, NULL is returned,
-and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOSYS::
-Handle has no recv operation.
-
-EINVAL::
-Some arguments were invalid.
-
-EAGAIN::
-`FLUX_O_NONBLOCK` was selected and `flux_send()` would block.
-
-
-EXAMPLES
---------
-
-This example opens the Flux broker and displays event messages
-as they arrive.
-
-....
-include::trecv.c[]
-....
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_open(3), flux_send(3), flux_requeue(3), flux_msg_cmp(3)
diff --git a/doc/man3/flux_recv.rst b/doc/man3/flux_recv.rst
new file mode 100644
index 000000000000..ae70590aa52b
--- /dev/null
+++ b/doc/man3/flux_recv.rst
@@ -0,0 +1,102 @@
+============
+flux_recv(3)
+============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  flux_msg_t *flux_recv (flux_t *h,
+                         struct flux_match match,
+                         int flags);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_recv` receives a message using the Flux Message broker,
+previously opened with :man3:`flux_open` on handle :var:`h`.
+The message should eventually be destroyed with :man3:`flux_msg_destroy`.
+
+:var:`match` is a message match structure which limits which messages
+can be received.
+
+::
+
+   struct flux_match {
+       int typemask;      // bitmask of matching message types
+       uint32_t matchtag; // matchtag
+       char *topic_glob;  // glob matching topic string
+   };
+
+The following initializers are available for :var:`match`:
+
+FLUX_MATCH_ANY
+   Match any message.
+
+FLUX_MATCH_EVENT
+   Match any event message.
+
+For additional details on how to use :var:`match`, see :man3:`flux_msg_cmp`.
+
+:var:`flags` is the logical "or" of zero or more of the following flags:
+
+FLUX_O_TRACE
+   Dumps :var:`msg` to stderr.
+
+FLUX_O_NONBLOCK
+   If unable to receive a matching message, return an error rather than block.
+
+Internally, flags are the logical "or" of :var:`flags` and the flags provided
+to :man3:`flux_open` when the handle was created.
+
+Messages that do not meet :var:`match` criteria, are requeued with
+:man3:`flux_requeue` for later consumption.
+
+
+RETURN VALUE
+============
+
+:func:`flux_recv` returns a message on success. On error, NULL is returned,
+and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOSYS
+   Handle has no recv operation.
+
+EINVAL
+   Some arguments were invalid.
+
+EAGAIN
+   ``FLUX_O_NONBLOCK`` was selected and :func:`flux_recv` would block.
+
+
+EXAMPLES
+========
+
+This example opens the Flux broker and displays event messages
+as they arrive.
+
+.. literalinclude:: example/recv.c
+  :language: c
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_open`, :man3:`flux_send`, :man3:`flux_requeue`, :man3:`flux_msg_cmp`
diff --git a/doc/man3/flux_request_decode.adoc b/doc/man3/flux_request_decode.adoc
deleted file mode 100644
index 72c4797d2cfe..000000000000
--- a/doc/man3/flux_request_decode.adoc
+++ /dev/null
@@ -1,87 +0,0 @@
-flux_request_decode(3)
-======================
-:doctype: manpage
-
-
-NAME
-----
-flux_request_decode, flux_request_unpack, flux_request_decode_raw - decode a Flux request message
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- int flux_request_decode (const flux_msg_t *msg,
-                          const char **topic,
-                          const char **s);
-
- int flux_request_unpack (const flux_msg_t *msg,
-                          const char **topic,
-                          const char *fmt, ...);
-
- int flux_request_decode_raw (const flux_msg_t *msg,
-                              const char **topic,
-                              const void **data, int *len);
-
-DESCRIPTION
------------
-
-`flux_request_decode()` decodes a request message _msg_.
-
-_topic_, if non-NULL, will be set the message's topic string. The storage
-for this string belongs to _msg_ and should not be freed.
-
-_s_, if non-NULL, will be set to the message's NULL-terminated string payload.
-If no payload exists, it is set to NULL.  The storage for this string belongs
-to _msg_ and should not be freed.
-
-`flux_request_unpack()` decodes a request message with a JSON payload as
-above, parsing the payload using variable arguments with a format string
-in the style of jansson's `json_unpack()` (used internally). Decoding fails
-if the message doesn't have a JSON payload.
-
-`flux_request_decode_raw()` decodes a request message with a raw payload,
-setting _data_ and _len_ to the payload data and length. The storage for
-the raw payload belongs to _msg_ and should not be freed.
-
-
-include::JSON_UNPACK.adoc[]
-
-
-RETURN VALUE
-------------
-
-These functions return 0 on success.  On error, -1 is returned, and
-errno is set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-The _msg_ argument was NULL.
-
-EPROTO::
-Message decoding failed, such as due to incorrect message type,
-missing topic string, etc..
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_respond(3), flux_rpc(3)
diff --git a/doc/man3/flux_request_decode.rst b/doc/man3/flux_request_decode.rst
new file mode 100644
index 000000000000..1a4befb94244
--- /dev/null
+++ b/doc/man3/flux_request_decode.rst
@@ -0,0 +1,84 @@
+======================
+flux_request_decode(3)
+======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   int flux_request_decode (const flux_msg_t *msg,
+                            const char **topic,
+                            const char **s);
+
+   int flux_request_unpack (const flux_msg_t *msg,
+                            const char **topic,
+                            const char *fmt,
+                            ...);
+
+   int flux_request_decode_raw (const flux_msg_t *msg,
+                                const char **topic,
+                                const void **data,
+                                size_t *len);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_request_decode` decodes a request message :var:`msg`.
+
+:var:`topic`, if non-NULL, will be set the message's topic string. The storage
+for this string belongs to :var:`msg` and should not be freed.
+
+:var:`s`, if non-NULL, will be set to the message's NULL-terminated string
+payload.  If no payload exists, it is set to NULL. The storage for this string
+belongs to :var:`msg` and should not be freed.
+
+:func:`flux_request_unpack` decodes a request message with a JSON payload as
+above, parsing the payload using variable arguments with a format string
+in the style of jansson's :func:`json_unpack` (used internally). Decoding fails
+if the message doesn't have a JSON payload.
+
+:func:`flux_request_decode_raw` decodes a request message with a raw payload,
+setting :var:`data` and :var:`len` to the payload data and length. The storage
+for the raw payload belongs to :var:`msg` and should not be freed.
+
+DECODING JSON PAYLOADS
+======================
+
+.. include:: common/json_unpack.rst
+
+
+RETURN VALUE
+============
+
+These functions return 0 on success. On error, -1 is returned, and
+:var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   The *msg* argument was NULL.
+
+EPROTO
+   Message decoding failed, such as due to incorrect message type,
+   missing topic string, etc.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_respond`, :man3:`flux_rpc`
diff --git a/doc/man3/flux_request_encode.adoc b/doc/man3/flux_request_encode.adoc
deleted file mode 100644
index 94573e651e3f..000000000000
--- a/doc/man3/flux_request_encode.adoc
+++ /dev/null
@@ -1,67 +0,0 @@
-flux_request_encode(3)
-======================
-:doctype: manpage
-
-
-NAME
-----
-flux_request_encode, flux_request_encode_raw - encode a Flux request message
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_msg_t *flux_request_encode (const char *topic,
-                                  const char *s);
-
- flux_msg_t *flux_request_encode_raw (const char *topic,
-                                      void *data, int len);
-
-DESCRIPTION
------------
-
-`flux_request_encode()` encodes a request message with topic string
-_topic_ and optional NULL terminated string payload _s_.  The newly constructed
-message that is returned must be destroyed with `flux_msg_destroy()`.
-
-`flux_request_encode_raw()` encodes a request message with topic
-string _topic_.  If _data_ is non-NULL its contents will be used
-as the message payload, and the payload type set to raw.
-
-
-RETURN VALUE
-------------
-
-These functions return a message on success.  On error, NULL is
-returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-The _topic_ argument was NULL or _s_ is not NULL terminated.
-
-ENOMEM::
-Memory was unavailable.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_response_decode(3), flux_rpc(3)
diff --git a/doc/man3/flux_request_encode.rst b/doc/man3/flux_request_encode.rst
new file mode 100644
index 000000000000..7f2c7fc25124
--- /dev/null
+++ b/doc/man3/flux_request_encode.rst
@@ -0,0 +1,61 @@
+======================
+flux_request_encode(3)
+======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_msg_t *flux_request_encode (const char *topic, const char *s);
+
+   flux_msg_t *flux_request_encode_raw (const char *topic,
+                                        void *data,
+                                        size_t len);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_request_encode` encodes a request message with topic string
+:var:`topic` and optional NULL terminated string payload :var:`s`. The newly
+constructed message that is returned must be destroyed with
+:func:`flux_msg_destroy`.
+
+:func:`flux_request_encode_raw` encodes a request message with topic
+string :var:`topic`. If :var:`data` is non-NULL its contents will be used
+as the message payload, and the payload type set to raw.
+
+
+RETURN VALUE
+============
+
+These functions return a message on success. On error, NULL is
+returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   The *topic* argument was NULL or *s* is not NULL terminated.
+
+ENOMEM
+   Memory was unavailable.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_response_decode`, :man3:`flux_rpc`
diff --git a/doc/man3/flux_requeue.adoc b/doc/man3/flux_requeue.adoc
deleted file mode 100644
index 34791d4bc4ae..000000000000
--- a/doc/man3/flux_requeue.adoc
+++ /dev/null
@@ -1,73 +0,0 @@
-flux_requeue(3)
-===============
-:doctype: manpage
-
-
-NAME
-----
-flux_requeue, flux_requeue_nocopy - requeue a message
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-int flux_requeue (flux_t *h, const flux_msg_t *msg, int flags);
-
-int flux_requeue_nocopy (flux_t *h, flux_msg_t *msg, int flags);
-
-
-DESCRIPTION
------------
-
-`flux_requeue()` requeues a _msg_ in handle _h_.  The message
-can be received with `flux_recv()` as though it arrived from the broker.
-
-`flux_requeue_nocopy()` is identical to `flux_requeue()`, but does not
-copy the message_.  It instead takes ownership of the _msg_ passed in
-by the caller.
-
-_flags_ must be set to one of the following values:
-
-FLUX_RQ_TAIL::
-_msg_ is placed at the tail of the message queue.
-
-FLUX_RQ_TAIL::
-_msg_ is placed at the head of the message queue.
-
-
-RETURN VALUE
-------------
-
-`flux_requeue()` and `flux_requeue_nocopy()` return zero on success.
-On error, -1 is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-Some arguments were invalid.
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_open(3), flux_recv(3), flux_send(3)
diff --git a/doc/man3/flux_requeue.rst b/doc/man3/flux_requeue.rst
new file mode 100644
index 000000000000..ba0988e538ac
--- /dev/null
+++ b/doc/man3/flux_requeue.rst
@@ -0,0 +1,59 @@
+===============
+flux_requeue(3)
+===============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/core.h>
+
+  int flux_requeue (flux_t *h, const flux_msg_t *msg, int flags);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_requeue` requeues a :var:`msg` in handle :var:`h`. The message
+can be received with :man3:`flux_recv` as though it arrived from the broker.
+
+:var:`flags` must be set to one of the following values:
+
+FLUX_RQ_TAIL
+   :var:`msg` is placed at the tail of the message queue.
+
+FLUX_RQ_HEAD
+   :var:`msg` is placed at the head of the message queue.
+
+
+RETURN VALUE
+============
+
+:func:`flux_requeue` return zero on success.
+On error, -1 is returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   Some arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_open`, :man3:`flux_recv`, :man3:`flux_send`
diff --git a/doc/man3/flux_respond.adoc b/doc/man3/flux_respond.adoc
deleted file mode 100644
index 3069e633c252..000000000000
--- a/doc/man3/flux_respond.adoc
+++ /dev/null
@@ -1,112 +0,0 @@
-flux_respond(3)
-===============
-:doctype: manpage
-
-
-NAME
-----
-flux_respond, flux_respond_pack, flux_respond_raw, flux_respond_error - respond to a request
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- int flux_respond (flux_t *h, const flux_msg_t *request,
-                   const char *s);
-
- int flux_respond_pack (flux_t *h, const flux_msg_t *request,
-                        const char *fmt, ...);
-
- int flux_respond_raw (flux_t *h, const flux_msg_t *request,
-                       const void *data, int length);
-
- int flux_respond_error (flux_t *h, const flux_msg_t *request,
-                         int errnum, const char *errmsg);
-
-DESCRIPTION
------------
-
-`flux_respond()`, `flux_respond_pack()`, `flux_respond_raw()`, and
-`flux_respond_error()`  encode and send a response message on handle _h_,
-deriving topic string, matchtag, and route stack from the provided
-_request_.
-
-`flux_respond()` sends a response to _request_.  If _s_ is non-NULL,
-`flux_respond()` will send it as the response payload, otherwise there
-will be no payload.
-
-`flux_respond_raw()` is identical except if _data_ is non-NULL,
-`flux_respond_raw()` will send it as the response payload.
-
-`flux_respond_pack()` encodes a response message with a JSON payload,
-building the payload using variable arguments with a format string in
-the style of jansson's `json_pack()` (used internally).
-
-`flux_respond_error()` returns an error response to the sender.
-_errnum_ must be non-zero.  If _errmsg_ is non-NULL, an error string
-payload is included in the response.  The error string may be used to
-provide a more detailed error message than can be conveyed via _errnum_.
-
-
-STREAMING SERVICES
-------------------
-
-Per RFC 6, a "streaming" service must return zero or more non-error
-responses to a request and a final error response.  If the requested
-operation was successful, the final error response may use ENODATA as
-the error number.  Clients should interpret ENODATA as a non-error
-end-of-stream marker.
-
-It is essential that services which return multiple responses verify that
-requests were made with the FLUX_RPC_STREAMING flag by testing the
-FLUX_MSGFLAG_STREAMING flag, e.g. using `flux_msg_is_streaming()`.
-If the flag is not set, the service must return an immediate EPROTO error.
-
-
-include::JSON_PACK.adoc[]
-
-
-RETURN VALUE
-------------
-
-These functions return zero on success.  On error, -1 is returned,
-and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOSYS::
-Handle has no send operation.
-
-EINVAL::
-Some arguments were invalid.
-
-EPROTO::
-A protocol error was encountered.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_rpc(3), flux_rpc_raw(3)
-
-https://github.com/flux-framework/rfc/blob/master/spec_6.adoc[RFC 6: Flux
-Remote Procedure Call Protocol]
-
-https://github.com/flux-framework/rfc/blob/master/spec_3.adoc[RFC 3: CMB1 - Flux Comms Message Broker Protocol]
diff --git a/doc/man3/flux_respond.rst b/doc/man3/flux_respond.rst
new file mode 100644
index 000000000000..18aadedc8f7f
--- /dev/null
+++ b/doc/man3/flux_respond.rst
@@ -0,0 +1,118 @@
+===============
+flux_respond(3)
+===============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   int flux_respond (flux_t *h,
+                     const flux_msg_t *request,
+                     const char *s);
+
+   int flux_respond_pack (flux_t *h,
+                          const flux_msg_t *request,
+                          const char *fmt,
+                          ...);
+
+   int flux_respond_raw (flux_t *h,
+                         const flux_msg_t *request,
+                         const void *data,
+                         size_t length);
+
+   int flux_respond_error (flux_t *h,
+                           const flux_msg_t *request,
+                           int errnum,
+                           const char *errmsg);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_respond`, :func:`flux_respond_pack`, :func:`flux_respond_raw`, and
+:func:`flux_respond_error` encode and send a response message on handle
+:var:`h`, deriving topic string, matchtag, and route stack from the provided
+:var:`request`.
+
+:func:`flux_respond` sends a response to :var:`request`. If :var:`s` is
+non-NULL, :func:`flux_respond` will send it as the response payload, otherwise
+there will be no payload.
+
+:func:`flux_respond_raw` is identical except if :var:`data` is non-NULL,
+:func:`flux_respond_raw` will send it as the response payload.
+
+:func:`flux_respond_pack` encodes a response message with a JSON payload,
+building the payload using variable arguments with a format string in
+the style of jansson's :func:`json_pack` (used internally).
+
+:func:`flux_respond_error` returns an error response to the sender.
+If :var:`errnum` is zero, EINVAL is used.  If :var:`errmsg` is non-NULL,
+an error string payload is included in the response. The error string may be
+used to provide a more detailed error message than can be conveyed via
+:var:`errnum`.
+
+
+STREAMING SERVICES
+==================
+
+Per RFC 6, a "streaming" service must return zero or more non-error
+responses to a request and a final error response. If the requested
+operation was successful, the final error response may use ENODATA as
+the error number. Clients should interpret ENODATA as a non-error
+end-of-stream marker.
+
+It is essential that services which return multiple responses verify that
+requests were made with the FLUX_RPC_STREAMING flag by testing the
+FLUX_MSGFLAG_STREAMING flag, e.g. using :man3:`flux_msg_is_streaming`.
+If the flag is not set, the service must return an immediate EPROTO error.
+
+ENCODING JSON PAYLOADS
+======================
+
+.. include:: common/json_pack.rst
+
+
+RETURN VALUE
+============
+
+These functions return zero on success. On error, -1 is returned,
+and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOSYS
+   Handle has no send operation.
+
+EINVAL
+   Some arguments were invalid.
+
+EPROTO
+   A protocol error was encountered.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX_RFC
+========
+
+:doc:`rfc:spec_6`
+
+:doc:`rfc:spec_3`
+
+
+SEE ALSO
+========
+
+:man3:`flux_rpc`, :man3:`flux_rpc_raw`
diff --git a/doc/man3/flux_response_decode.adoc b/doc/man3/flux_response_decode.adoc
deleted file mode 100644
index 16b20cbd2a09..000000000000
--- a/doc/man3/flux_response_decode.adoc
+++ /dev/null
@@ -1,87 +0,0 @@
-flux_response_decode(3)
-=======================
-:doctype: manpage
-
-
-NAME
-----
-flux_response_decode, flux_response_decode_raw, flux_response_decode_error - decode a Flux response message
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- int flux_response_decode (const flux_msg_t *msg,
-                           const char **topic,
-                           const char **s);
-
- int flux_response_decode_raw (const flux_msg_t *msg,
-                               const char **topic,
-                               const void **data, int *len);
-
- int flux_response_decode_error (const flux_msg_t *msg,
-                                 const char *errstr);
-
-
-DESCRIPTION
------------
-
-`flux_response_decode()` decodes a response message _msg_.
-
-_topic_, if non-NULL, will be set to the message's topic string. The
-storage for this string belongs to _msg_ and should not be freed.
-
-_s_, if non-NULL, will be set to the message's NULL-terminated string payload.
-If no payload exists, it is set to NULL.  The storage for this
-string belongs to _msg_ and should not be freed.
-
-`flux_response_decode_raw()` decodes a response message with a raw payload,
-setting _data_ and _len_ to the payload data and length. The storage for
-the raw payload belongs to _msg_ and should not be freed.
-
-`flux_response_decode_error()` decodes an optional error string included
-with an error response.  This fails if the response is not an error,
-or does not include an error string payload.
-
-
-RETURN VALUE
-------------
-
-These functions return 0 on success.  On error, -1 is returned, and
-errno is set appropriately.
-
-
-ERRORS
-------
-
-EINVAL::
-The _msg_ argument was NULL.
-
-EPROTO::
-Message decoding failed, such as due to incorrect message type,
-missing topic string, etc..
-
-ENOENT::
-`flux_response_decode_error()` was called on a message with no
-error response payload.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_request_encode(3), flux_rpc(3)
diff --git a/doc/man3/flux_response_decode.rst b/doc/man3/flux_response_decode.rst
new file mode 100644
index 000000000000..11b5d877d1ea
--- /dev/null
+++ b/doc/man3/flux_response_decode.rst
@@ -0,0 +1,80 @@
+=======================
+flux_response_decode(3)
+=======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   int flux_response_decode (const flux_msg_t *msg,
+                             const char **topic,
+                             const char **s);
+
+   int flux_response_decode_raw (const flux_msg_t *msg,
+                                 const char **topic,
+                                 const void **data,
+                                 size_t *len);
+
+   int flux_response_decode_error (const flux_msg_t *msg,
+                                   const char *errstr);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_response_decode` decodes a response message :var:`msg`.
+
+:var:`topic`, if non-NULL, will be set to the message's topic string. The
+storage for this string belongs to :var:`msg` and should not be freed.
+
+:var:`s`, if non-NULL, will be set to the message's NULL-terminated string
+payload.  If no payload exists, it is set to NULL. The storage for this
+string belongs to :var:`msg` and should not be freed.
+
+:func:`flux_response_decode_raw` decodes a response message with a raw payload,
+setting :var:`data` and :var:`len` to the payload data and length. The storage
+for the raw payload belongs to :var:`msg` and should not be freed.
+
+:func:`flux_response_decode_error` decodes an optional error string included
+with an error response. This fails if the response is not an error,
+or does not include an error string payload.
+
+
+RETURN VALUE
+============
+
+These functions return 0 on success. On error, -1 is returned, and
+:var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   The :var:`msg` argument was NULL.
+
+EPROTO
+   Message decoding failed, such as due to incorrect message type,
+   missing topic string, etc.
+
+ENOENT
+   :func:`flux_response_decode_error` was called on a message with no
+   error response payload.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_request_encode`, :man3:`flux_rpc`
diff --git a/doc/man3/flux_rpc.adoc b/doc/man3/flux_rpc.adoc
deleted file mode 100644
index ddf7dccb70ca..000000000000
--- a/doc/man3/flux_rpc.adoc
+++ /dev/null
@@ -1,224 +0,0 @@
-flux_rpc(3)
-===========
-:doctype: manpage
-
-
-NAME
-----
-flux_rpc, flux_rpc_pack, flux_rpc_raw, flux_rpc_message, flux_rpc_get, flux_rpc_get_unpack, flux_rpc_get_raw - perform a remote procedure call to a Flux service
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- flux_future_t *flux_rpc (flux_t *h, const char *topic,
-                          const char *s,
-                          uint32_t nodeid, int flags);
-
- flux_future_t *flux_rpc_pack (flux_t *h, const char *topic,
-                               uint32_t nodeid, int flags,
-                               const char *fmt, ...);
-
- flux_future_t *flux_rpc_raw (flux_t *h, const char *topic,
-                              const void *data, int len,
-                              uint32_t nodeid, int flags);
-
- flux_future_t *flux_rpc_message (flux_t *h,
-                                  const flux_msg_t *msg,
-                                  uint32_t nodeid, int flags);
-
- int flux_rpc_get (flux_future_t *f, const char **s);
-
- int flux_rpc_get_unpack (flux_future_t *f, const char *fmt, ...);
-
- int flux_rpc_get_raw (flux_future_t *f,
-                       const void **data, int *len);
-
-DESCRIPTION
------------
-
-A remote procedure call (RPC) consists of a matched request and
-response message exchanged with a Flux service.  `flux_rpc()`,
-`flux_rpc_pack()`, and `flux_rpc_raw()` encode and send a request message
-via Flux broker handle _h_ to a Flux service identified by _topic_
-and _nodeid_.  A `flux_future_t` object is returned which acts as a handle
-for synchronization and a container for the response message which in
-turn contains the RPC result.
-
-A lower-level variant of `flux_rpc()`, `flux_rpc_message()` accepts a
-pre-created request message, assigning _nodeid_ and matchtag according
-to _flags_.
-
-`flux_future_then(3)` may be used to register a reactor callback
-(continuation) to be called once the response has been received.
-`flux_future_wait_for(3)` may be used to block until the
-response has been received.  Both accept an optional timeout.
-
-`flux_rpc_get()`, `flux_rpc_get_unpack()`, and `flux_rpc_get_raw()`
-decode the RPC result.  Internally, they call `flux_future_get()`
-to access the response message stored in the future.  If the response
-message has not yet been received, these functions block until it is,
-or an error occurs.
-
-
-REQUEST OPTIONS
----------------
-
-The request message is encoded and sent with or without a payload
-using one of the three `flux_rpc()` variants.
-
-`flux_rpc()` attaches _s_, a NULL terminated string, as request
-payload.  If NULL, the request is encoded without a payload.
-
-`flux_rpc_pack()` attaches a JSON payload encoded as a NULL terminated
-string using Jansson `json_pack()` style arguments (see below).
-
-`flux_rpc_raw()` attaches a raw payload _data_ of length _len_, in bytes.
-If _data_ is NULL, the request is encoded without a payload.
-
-_nodeid_ affects request routing, and must be set to one of the following
-values:
-
-FLUX_NODEID_ANY::
-The request is routed to the first matching service instance.
-
-FLUX_NODEID_UPSTREAM::
-The request is routed to the first matching service instance,
-skipping over the sending rank.
-
-integer::
-The request is routed to a specific rank.
-
-_flags_ may be zero or:
-
-FLUX_RPC_NORESPONSE::
-No response is expected.  The request will not be assigned a matchtag,
-and the returned flux_future_t is immediately fulfilled, and may simply
-be destroyed.
-
-FLUX_RPC_STREAMING::
-The RPC is for a service that may send zero or more non-error responses,
-and a final error response.  ENODATA should be interpreted as a non-error
-end-of-stream sentinel.
-
-
-RESPONSE OPTIONS
-----------------
-
-The response message is stored in the future when the future is fulfilled.
-At that time it is decoded with `flux_response_decode(3)`.  If it cannot
-be decoded, or if the service returned an error, the future is fulfilled
-with an error.  Otherwise it is fulfilled with the response message.
-If there was an error, `flux_future_get()` or the `flux_rpc_get()` variants
-return an error.
-
-`flux_rpc_get()` sets _s_ (if non-NULL) to the NULL-terminated string
-payload contained in the RPC response.  If there was no payload, _s_
-is set to NULL.
-
-`flux_rpc_get_unpack()` decodes the NULL-terminated string payload as JSON
-using Jansson `json_unpack()` style arguments (see below).  It is an error
-if there is no payload, or if the payload is not JSON.
-
-`flux_rpc_get_raw()` assigns the raw payload of the RPC response message
-to _data_ and its length to _len_.  If there is no payload, this function
-will fail.
-
-
-PREMATURE DESTRUCTION
----------------------
-
-If a regular RPC future is destroyed before its response is received,
-the matchtag allocated to it is not immediately returned to the pool
-for reuse.  If an unclaimed response subsequently arrives with that
-matchtag, it is returned to the pool then.
-
-If a *streaming* RPC future is destroyed before its terminating response
-is received, its matchtag is only returned to the pool when an unclaimed
-*error* response is received.  Non-error responses are ignored.
-
-It is essential that services which return multiple responses verify that
-requests were made with the FLUX_RPC_STREAMING flag and return an immediate
-EPROTO error if they were not.  See flux_respond(3).
-
-
-CANCELLATION
--------------
- 
-Flux RFC 6 does not currently specify a cancellation protocol for an
-individual RPC, but does stipulate that an RPC may be canceled if a disconnect
-message is received, as is automatically generated by the local connector
-upon client disconnection.
-
-
-include::JSON_PACK.adoc[]
-
-
-include::JSON_UNPACK.adoc[]
-
-
-RETURN VALUE
-------------
-
-`flux_rpc()`, `flux_rpc_pack()`, and `flux_rpc_raw()` return a flux_future_t
-object on success.  On error, NULL is returned, and errno is set appropriately.
-
-`flux_rpc_get()`, `flux_rpc_get_unpack()`, and `flux_rpc_get_raw()` return
-zero on success.  On error, -1 is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOSYS::
-Service is not available (misspelled topic string, module not loaded, etc),
-or flux_t handle has no send operation.
-
-EINVAL::
-Some arguments were invalid.
-
-EPROTO::
-A request was malformed, the FLUX_RPC_STREAMING flag was
-omitted on a request to a service that may send multiple responses,
-or other protocol error occurred.
-
-
-EXAMPLES
---------
-
-This example performs a synchronous RPC with the broker's "attr.get"
-service to obtain the broker's rank.
-
-....
-include::trpc.c[]
-....
-
-This example registers a continuation to do the same thing asynchronously.
-
-....
-include::trpc_then.c[]
-....
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_future_get(3), flux_respond(3)
-
-https://github.com/flux-framework/rfc/blob/master/spec_6.adoc[RFC 6: Flux
-Remote Procedure Call Protocol]
diff --git a/doc/man3/flux_rpc.rst b/doc/man3/flux_rpc.rst
new file mode 100644
index 000000000000..76a43b97830d
--- /dev/null
+++ b/doc/man3/flux_rpc.rst
@@ -0,0 +1,249 @@
+===========
+flux_rpc(3)
+===========
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_rpc (flux_t *h,
+                            const char *topic,
+                            const char *s,
+                            uint32_t nodeid,
+                            int flags);
+
+   flux_future_t *flux_rpc_pack (flux_t *h,
+                                 const char *topic,
+                                 uint32_t nodeid,
+                                 int flags,
+                                 const char *fmt,
+                                 ...);
+
+   flux_future_t *flux_rpc_raw (flux_t *h,
+                                const char *topic,
+                                const void *data,
+                                size_t len,
+                                uint32_t nodeid,
+                                int flags);
+
+   flux_future_t *flux_rpc_message (flux_t *h,
+                                    const flux_msg_t *msg,
+                                    uint32_t nodeid,
+                                    int flags);
+
+   int flux_rpc_get (flux_future_t *f, const char **s);
+
+   int flux_rpc_get_unpack (flux_future_t *f, const char *fmt, ...);
+
+   int flux_rpc_get_raw (flux_future_t *f,
+                         const void **data,
+                         size_t *len);
+
+   uint32_t flux_rpc_get_matchtag (flux_future_t *f);
+
+   uint32_t flux_rpc_get_nodeid (flux_future_t *f);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+A remote procedure call (RPC) consists of a matched request and
+response message exchanged with a Flux service. :func:`flux_rpc`,
+:func:`flux_rpc_pack`, and :func:`flux_rpc_raw` encode and send a request
+message via Flux broker handle :var:`h` to a Flux service identified by
+:var:`topic` and :var:`nodeid`. A :type:`flux_future_t` object is returned
+which acts as a handle for synchronization and a container for the response
+message which in turn contains the RPC result.
+
+A lower-level variant of :func:`flux_rpc`, :func:`flux_rpc_message` accepts a
+pre-created request message, assigning :var:`nodeid` and matchtag according
+to :var:`flags`.
+
+:man3:`flux_future_then` may be used to register a reactor callback
+(continuation) to be called once the response has been received.
+:man3:`flux_future_wait_for` may be used to block until the
+response has been received. Both accept an optional timeout.
+
+:func:`flux_rpc_get`, :func:`flux_rpc_get_unpack`, and :func:`flux_rpc_get_raw`
+decode the RPC result. Internally, they call :man3:`flux_future_get`
+to access the response message stored in the future. If the response
+message has not yet been received, these functions block until it is,
+or an error occurs.
+
+:func:`flux_rpc_get_matchtag` and :func:`flux_rpc_get_nodeid` are accessors
+which allow access to the RPC matchtag and target nodeid from the
+future returned from :func:`flux_rpc`.
+
+
+REQUEST OPTIONS
+===============
+
+The request message is encoded and sent with or without a payload
+using one of the three :func:`flux_rpc` variants.
+
+:func:`flux_rpc` attaches :var:`s`, a NULL terminated string, as request
+payload. If NULL, the request is encoded without a payload.
+
+:func:`flux_rpc_pack` attaches a JSON payload encoded as a NULL terminated
+string using Jansson :func:`json_pack` style arguments (see below).
+
+:func:`flux_rpc_raw` attaches a raw payload :var:`data` of length :var:`len`,
+in bytes.  If :var:`data` is NULL, the request is encoded without a payload.
+
+:var:`nodeid` affects request routing, and must be set to one of the following
+values:
+
+FLUX_NODEID_ANY
+   The request is routed to the first matching service instance.
+
+FLUX_NODEID_UPSTREAM
+   The request is routed to the first matching service instance,
+   skipping over the sending rank.
+
+integer
+   The request is routed to a specific rank.
+
+:var:`flags` may be zero or:
+
+FLUX_RPC_NORESPONSE
+   No response is expected. The request will not be assigned a matchtag,
+   and the returned :type:`flux_future_t` is immediately fulfilled, and may
+   simply be destroyed.
+
+FLUX_RPC_STREAMING
+   The RPC is for a service that may send zero or more non-error responses,
+   and a final error response. ENODATA should be interpreted as a non-error
+   end-of-stream sentinel.
+
+
+RESPONSE OPTIONS
+================
+
+The response message is stored in the future when the future is fulfilled.
+At that time it is decoded with :man3:`flux_response_decode`. If it cannot
+be decoded, or if the service returned an error, the future is fulfilled
+with an error. Otherwise it is fulfilled with the response message.
+If there was an error, :man3:`flux_future_get` or the :func:`flux_rpc_get`
+variants return an error.
+
+:func:`flux_rpc_get` sets :var:`s` (if non-NULL) to the NULL-terminated string
+payload contained in the RPC response. If there was no payload, :var:`s`
+is set to NULL.
+
+:func:`flux_rpc_get_unpack` decodes the NULL-terminated string payload as JSON
+using Jansson :func:`json_unpack` style arguments (see below). It is an error
+if there is no payload, or if the payload is not JSON.
+
+:func:`flux_rpc_get_raw` assigns the raw payload of the RPC response message
+to :var:`data` and its length to :var:`len`. If there is no payload, this
+function will fail.
+
+
+PREMATURE DESTRUCTION
+=====================
+
+If a regular RPC future is destroyed before its response is received,
+the matchtag allocated to it is not immediately returned to the pool
+for reuse. If an unclaimed response subsequently arrives with that
+matchtag, it is returned to the pool then.
+
+If a **streaming** RPC future is destroyed before its terminating response
+is received, its matchtag is only returned to the pool when an unclaimed
+**error** response is received. Non-error responses are ignored.
+
+It is essential that services which return multiple responses verify that
+requests were made with the FLUX_RPC_STREAMING flag and return an immediate
+EPROTO error if they were not. See :man3:`flux_respond`.
+
+
+CANCELLATION
+============
+
+Flux RFC 6 does not currently specify a cancellation protocol for an
+individual RPC, but does stipulate that an RPC may be canceled if a disconnect
+message is received, as is automatically generated by the local connector
+upon client disconnection.
+
+ENCODING JSON PAYLOADS
+======================
+
+.. include:: common/json_pack.rst
+
+DECODING JSON PAYLOADS
+======================
+
+.. include:: common/json_unpack.rst
+
+
+RETURN VALUE
+============
+
+:func:`flux_rpc`, :func:`flux_rpc_pack`, and :func:`flux_rpc_raw` return a
+:type:`flux_future_t` object on success. On error, NULL is returned, and
+:var:`errno` is set appropriately.
+
+:func:`flux_rpc_get`, :func:`flux_rpc_get_unpack`, and :func:`flux_rpc_get_raw`
+return zero on success. On error, -1 is returned, and :var:`errno` is set
+appropriately.
+
+:func:`flux_rpc_get_matchtag` returns the matchtag allocated to the particular
+RPC request, or ``FLUX_MATCHTAG_NONE`` if no matchtag was allocated (e.g. no
+response is expected), or the future argument does not correspond to an RPC.
+
+:func:`flux_rpc_get_nodeid` returns the original ``nodeid`` target of the
+:func:`flux_rpc` request, including if the RPC was targeted to
+``FLUX_NODEID_ANY`` or ``FLUX_NODEID_UPSTREAM``.
+
+ERRORS
+======
+
+ENOSYS
+   Service is not available (misspelled topic string, module not loaded, etc),
+   or :type:`flux_t` handle has no send operation.
+
+EINVAL
+   Some arguments were invalid.
+
+EPROTO
+   A request was malformed, the FLUX_RPC_STREAMING flag was
+   omitted on a request to a service that may send multiple responses,
+   or other protocol error occurred.
+
+
+EXAMPLES
+========
+
+This example performs a synchronous RPC with the broker's "attr.get"
+service to obtain the broker's rank.
+
+.. literalinclude:: example/rpc.c
+  :language: c
+
+This example registers a continuation to do the same thing asynchronously.
+
+.. literalinclude:: example/rpc_then.c
+  :language: c
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_6`
+
+
+SEE ALSO
+========
+
+:man3:`flux_future_get`, :man3:`flux_respond`
diff --git a/doc/man3/flux_send.adoc b/doc/man3/flux_send.adoc
deleted file mode 100644
index 9727e830e103..000000000000
--- a/doc/man3/flux_send.adoc
+++ /dev/null
@@ -1,85 +0,0 @@
-flux_send(3)
-============
-:doctype: manpage
-
-
-NAME
-----
-flux_send - send message using Flux Message Broker
-
-
-SYNOPSIS
---------
-#include <flux/core.h>
-
-int flux_send (flux_t *h, const flux_msg_t *msg, int flags);
-
-
-DESCRIPTION
------------
-
-`flux_send()` sends _msg_ using the Flux Message broker,
-previously opened with `flux_open()` on handle _h_.
-
-_flags_ is the logical "or" of zero or more of the following flags:
-
-FLUX_O_TRACE::
-Dumps _msg_ to stderr.
-
-FLUX_O_NONBLOCK::
-If unable to send, return an error rather than block.
-
-Internally, flags are the logical "or" of _flags_ and the flags provided
-to `flux_open()` when the handle was created.
-
-The message type, topic string, and nodeid affect how the message
-will be routed by the broker.  These attributes are pre-set in the message.
-
-RETURN VALUE
-------------
-
-`flux_send()` returns  zero on success.  On error, -1 is returned, and errno
-is set appropriately.
-
-
-ERRORS
-------
-
-ENOSYS::
-Handle has no send operation.
-
-EINVAL::
-Some arguments were invalid.
-
-EAGAIN::
-`FLUX_O_NONBLOCK` was selected and `flux_send()` would block.
-
-
-EXAMPLES
---------
-
-This example opens the Flux broker and publishes an event message.
-
-....
-include::tsend.c[]
-....
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_open(3), flux_recv(3), flux_requeue(3)
diff --git a/doc/man3/flux_send.rst b/doc/man3/flux_send.rst
new file mode 100644
index 000000000000..82ebd64db5cd
--- /dev/null
+++ b/doc/man3/flux_send.rst
@@ -0,0 +1,84 @@
+============
+flux_send(3)
+============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   int flux_send (flux_t *h, const flux_msg_t *msg, int flags);
+
+   int flux_send_new (flux_t *h, flux_msg_t **msg, int flags);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_send` sends :var:`msg` using the Flux Message broker,
+previously opened with :man3:`flux_open` on handle :var:`h`.
+
+:var:`flags` is the logical "or" of zero or more of the following flags:
+
+FLUX_O_TRACE
+   Dumps :var:`msg` to stderr.
+
+FLUX_O_NONBLOCK
+   If unable to send, return an error rather than block.
+
+Internally, flags are the logical "or" of :var:`flags` and the flags provided
+to :man3:`flux_open` when the handle was created.
+
+The message type, topic string, and nodeid affect how the message
+will be routed by the broker. These attributes are pre-set in the message.
+
+:func:`flux_send_new` is the same, except message ownership is transferred
+to the handle :var:`h`.  The double pointer :var:`msg` points to a NULL value if
+the message is successfully transferred.  The send fails if the message
+reference count is greater than one.
+
+
+RETURN VALUE
+============
+
+:func:`flux_send` returns zero on success. On error, -1 is returned, and
+:var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOSYS
+   Handle has no send operation.
+
+EINVAL
+   Some arguments were invalid.
+
+EAGAIN
+   ``FLUX_O_NONBLOCK`` was selected and :func:`flux_send` would block.
+
+
+EXAMPLES
+========
+
+This example opens the Flux broker and publishes an event message.
+
+.. literalinclude:: example/send.c
+  :language: c
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_open`, :man3:`flux_recv`, :man3:`flux_requeue`
diff --git a/doc/man3/flux_service_register.rst b/doc/man3/flux_service_register.rst
new file mode 100644
index 000000000000..27256f10a059
--- /dev/null
+++ b/doc/man3/flux_service_register.rst
@@ -0,0 +1,61 @@
+============================
+flux_service_register(3)
+============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_service_register (flux_t *h, const char *name);
+
+   flux_future_t *flux_service_unregister (flux_t *h, const char *name);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_service_register` enables a new service :var:`name` to be registered
+with the flux broker.  On success, request message sent to "name.*" will
+be routed to this handle until :func:`flux_service_unregister` is called
+for :var:`name`.
+
+While :func:`flux_service_register` registers :var:`name`, the user must
+still setup a handler for the service.  One can be setup through
+:man3:`flux_msg_handler_addvec`.
+
+
+RETURN VALUE
+============
+
+:func:`flux_service_register` and :func:`flux_service_unregister` return a
+:type:`flux_future_t` on success, or NULL on failure with :var:`errno` set
+appropriately.
+
+
+ERRORS
+======
+
+EINVAL
+   One of the arguments was invalid.
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+flux_future_get(3), flux_msg_handler_addvec(3)
+
diff --git a/doc/man3/flux_shell_add_completion_ref.rst b/doc/man3/flux_shell_add_completion_ref.rst
new file mode 100644
index 000000000000..8e755b8f3afa
--- /dev/null
+++ b/doc/man3/flux_shell_add_completion_ref.rst
@@ -0,0 +1,61 @@
+================================
+flux_shell_add_completion_ref(3)
+================================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   int flux_shell_add_completion_ref (flux_shell_t *shell,
+                                      const char *fmt,
+                                      ...)
+
+   int flux_shell_remove_completion_ref (flux_shell_t *shell,
+                                         const char *fmt,
+                                         ...)
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_add_completion_ref` creates a named "completion
+reference" on the shell object :var:`shell` so that the shell will
+not consider a job "complete" until the reference is released with
+:func:`flux_shell_remove_completion_ref`. Once all references have been
+removed, the shell's reactor :var:`shell->r` is stopped with
+:man3:`flux_reactor_stop`.
+
+
+RETURN VALUE
+============
+
+:func:`flux_shell_add_completion_ref` returns the reference count for the
+particular name, or -1 on error.
+
+:func:`flux_shell_remove_completion_ref` returns 0 on success, -1 on failure.
+
+
+ERRORS
+======
+
+EINVAL
+   Either :var:`shell` or :var:`fmt` are NULL.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_reactor_stop`
diff --git a/doc/man3/flux_shell_add_event_context.rst b/doc/man3/flux_shell_add_event_context.rst
new file mode 100644
index 000000000000..5d7cb2bcdc96
--- /dev/null
+++ b/doc/man3/flux_shell_add_event_context.rst
@@ -0,0 +1,46 @@
+===============================
+flux_shell_add_event_context(3)
+===============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   int flux_shell_add_event_context (flux_shell_t *shell,
+                                     const char *name,
+                                     int flags,
+                                     const char *fmt,
+                                     ...);
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+Add extra context that will be emitted with shell standard event
+:var:`name` using Jansson :func:`json_pack` style arguments. The :var:`flags`
+parameter is currently unused.
+
+
+RETURN VALUE
+============
+
+Returns 0 on success, -1 if :var:`shell`, :var:`name` or :var:`fmt` are NULL.
+
+
+ERRORS
+======
+
+EINVAL
+   :var:`shell`, :var:`name` or :var:`fmt` are NULL.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_add_event_handler.rst b/doc/man3/flux_shell_add_event_handler.rst
new file mode 100644
index 000000000000..00cf1014be96
--- /dev/null
+++ b/doc/man3/flux_shell_add_event_handler.rst
@@ -0,0 +1,51 @@
+===============================
+flux_shell_add_event_handler(3)
+===============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   int flux_shell_add_event_handler (flux_shell_t *shell,
+                                     const char *subtopic,
+                                     flux_msg_handler_f cb,
+                                     void *arg);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+When the shell initializes, it subscribes to all events with the
+substring ``shell-JOBID.``, where ``JOBID`` is the jobid under which the
+shell is running. :func:`flux_shell_add_event_handler` registers a handler
+to be run for a :var:`subtopic` within the shell's event namespace, e.g.
+registering a handler for :var:`subtopic` ``"kill"`` will invoke the handler
+:var:`cb` whenever an event named ``shell-JOBID.kill`` is generated.
+
+
+RETURN VALUE
+============
+
+Returns -1 if :var:`shell`, :var:`shell->h`, :var:`subtopic` or :var:`cb` are
+NULL, or if underlying calls to :linux:man3:`asprintf` or
+:man3:`flux_msg_handler_create` fail.
+
+
+ERRORS
+======
+
+EINVAL
+   :var:`shell`, :var:`shell->h`, :var:`subtopic` or :var:`cb` are NULL.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_aux_set.rst b/doc/man3/flux_shell_aux_set.rst
new file mode 100644
index 000000000000..6fb7f52be63a
--- /dev/null
+++ b/doc/man3/flux_shell_aux_set.rst
@@ -0,0 +1,96 @@
+=====================
+flux_shell_aux_set(3)
+=====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   typedef void (*flux_free_f)(void *arg);
+
+   int flux_shell_aux_set (flux_shell_t *shell,
+                           const char *name,
+                           void *aux,
+                           flux_free_f free_fn);
+
+   void * flux_shell_aux_get (flux_shell_t *shell,
+                              const char *key);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_aux_set` attaches application-specific data to the parent
+object. It stores data :var:`aux` by key :var:`name`, with optional destructor
+:var:`destroy`. The destructor, if non-NULL, is called when the parent
+object is destroyed, or when :var:`name` is overwritten by a new value. If
+:var:`aux` is NULL, the destructor for a previous value, if any is called,
+but no new value is stored. If :var:`name` is NULL, :var:`aux` is stored
+anonymously.
+
+:func:`flux_shell_aux_get` retrieves application-specific data by name. If
+the data was stored anonymously, it cannot be retrieved.
+
+The implementation (as opposed to the header file) uses the variable
+names :var:`shell`, :var:`key`, :var:`val` and :var:`free_fn`, which may be
+more intuitive.
+
+In most cases the :var:`key`, :var:`value` and :var:`free` function will be
+non-null.  Several exceptions are supported.
+
+First, if :var:`key` and :var:`val` are non-NULL but :var:`free_fn` is null,
+the caller is responsible for memory management associated with the value.
+
+Second, if :var:`key` is NULL but :var:`val` and :var:`free_fun` are not NULL,
+the lifetime of the object is tied to the lifetime of the underlying
+aux object; the object will be destroyed during the destruction
+of the aux. The value cannot be retrieved.
+
+Third, a non-null :var:`key` and a null :var:`val` deletes the value previously
+associated with the key by calling its previously-associated :var:`free_fn`,
+if the destructor exists.
+
+
+RETURN VALUE
+============
+
+:func:`flux_aux_set` returns 0 on success, or -1 on failure, with :var:`errno`
+set.
+
+:func:`flux_shell_aux_get` returns data on success, or NULL on failure,
+with :var:`errno` set.
+
+
+ERRORS
+======
+
+EINVAL
+   | :var:`shell` is null; or
+   | both :var:`name` (aka :var:`key`) and :var:`aux` (aka :var:`val`) are null; or
+   | :var:`free_fn` is not null but :var:`aux` is; or
+   | :var:`free_fn` and :var:`name` are both null.
+
+ENOMEM
+   Out of memory.
+
+ENOENT
+   :func:`flux_aux_get` could not find an entry for :var:`key`.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_aux_get`, :man3:`flux_aux_set`
diff --git a/doc/man3/flux_shell_current_task.rst b/doc/man3/flux_shell_current_task.rst
new file mode 100644
index 000000000000..90c9f7c6e83b
--- /dev/null
+++ b/doc/man3/flux_shell_current_task.rst
@@ -0,0 +1,56 @@
+==========================
+flux_shell_current_task(3)
+==========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   flux_shell_task_t *flux_shell_current_task (flux_shell_t *shell);
+
+   flux_shell_task_t *flux_shell_task_first (flux_shell_t *shell);
+
+   flux_shell_task_t *flux_shell_task_next (flux_shell_t *shell);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_task_first` and :func:`flux_shell_task_next` are used to
+iterate over all current tasks known to the shell.
+
+:func:`flux_shell_current_task` returns the current task for ``task_init``,
+``task_exec`` and ``task_exec`` callbacks and NULL in any other
+context.
+
+:func:`flux_shell_task_first` and :func:`flux_shell_task_next` return the first
+and next tasks, respectively.
+
+
+RETURN VALUE
+============
+
+The relevant :type:`flux_shell_task_t` value, or NULL on error.
+
+
+ERRORS
+======
+
+EINVAL
+   ``shell`` is NULL.
+
+EAGAIN
+   There are no tasks.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_get_flux.rst b/doc/man3/flux_shell_get_flux.rst
new file mode 100644
index 000000000000..9b824c50adc2
--- /dev/null
+++ b/doc/man3/flux_shell_get_flux.rst
@@ -0,0 +1,75 @@
+======================
+flux_shell_get_flux(3)
+======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+
+   flux_t *flux_shell_get_flux (flux_shell_t *shell);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+Returns the Flux handle.
+
+
+RETURN VALUE
+============
+
+Returns the Flux handle.
+
+
+ERRORS
+======
+
+No error conditions are possible.
+
+
+EXAMPLE
+=======
+
+::
+
+   // Set a timer in flux_plugin_init().
+
+::
+
+   void flux_plugin_init (flux_plugin_t *p){
+
+::
+
+   // Get the shell handle,
+   flux_shell_t *shell = flux_plugin_get_shell( p );
+
+::
+
+   // use that to get the flux handle,
+   flux_t *flux = flux_shell_get_flux( shell );
+
+::
+
+   // and use that to get the reactor handle.
+   flux_reactor_t *reactor = flux_get_reactor( flux );
+
+::
+
+   flux_watcher_t* timer   = flux_timer_watcher_create( reactor, 0.1, 0.1, timer_cb, NULL );
+   flux_watcher_start(timer);
+
+::
+
+   ....
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_get_hostlist.rst b/doc/man3/flux_shell_get_hostlist.rst
new file mode 100644
index 000000000000..082dace1f3ab
--- /dev/null
+++ b/doc/man3/flux_shell_get_hostlist.rst
@@ -0,0 +1,52 @@
+==========================
+flux_shell_get_hostname(3)
+==========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   const struct hostlist * flux_shell_get_hostlist (flux_shell_t *shell);
+
+Link with :command:`-lflux-core -lflux-hostlist`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_get_hostlist` returns the list of hosts assigned to the
+current job in ``struct hostlist`` form. This hostlist can be used to
+map job node IDs or job shell ranks to hostnames using the interfaces
+exported in ``libflux-hostlist.so``.
+
+
+RETURN VALUE
+============
+
+This function returns a pointer to the shell's internal :type:`struct hostlist`
+or ``NULL`` on failure with :var:`errno` set.
+
+
+ERRORS
+======
+
+EINVAL
+   if :var:`shell` is NULL or the function is called before the hostlist is
+   available to the job shell.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_34`
diff --git a/doc/man3/flux_shell_get_hwloc_xml.rst b/doc/man3/flux_shell_get_hwloc_xml.rst
new file mode 100644
index 000000000000..3fca44de9f0c
--- /dev/null
+++ b/doc/man3/flux_shell_get_hwloc_xml.rst
@@ -0,0 +1,47 @@
+===========================
+flux_shell_get_hwloc_xml(3)
+===========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   int flux_shell_get_hwloc_xml (flux_shell_t *shell,
+                                 const char **hwloc_xml);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_get_hwloc_xml` returns an hwloc XML string which has
+been cached by the job shell. This XML string can be used to load an
+hwloc topology via :func:`hwloc_topology_load` without requiring shell
+components to rediscover the entire topology by probing the local
+system. This can make loading hwloc topology much more efficient.
+
+RETURN VALUE
+============
+
+:func:`flux_shell_get_hwloc_xml` returns 0 on success and -1 on error.
+
+
+ERRORS
+======
+
+EINVAL
+   :var:`shell` or :var:`hwloc_xml` are NULL, or the current :var:`shell`
+   object is being used uninitialized.
+    
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_get_info.rst b/doc/man3/flux_shell_get_info.rst
new file mode 100644
index 000000000000..cd1146d9b4df
--- /dev/null
+++ b/doc/man3/flux_shell_get_info.rst
@@ -0,0 +1,92 @@
+======================
+flux_shell_get_info(3)
+======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   int flux_shell_get_info (flux_shell_t *shell,
+                           char **json_str);
+
+   int flux_shell_info_unpack (flux_shell_t *shell,
+                              const char *fmt,
+                              ...);
+
+   int flux_shell_get_rank_info (flux_shell_t *shell,
+                                 int shell_rank,
+                                 char **json_str);
+
+   int flux_shell_rank_info_unpack (flux_shell_t *shell,
+                                    int shell_rank,
+                                    const char *fmt,
+                                    ...);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_get_info` returns shell information as a json string
+with the following layout:
+
+::
+
+   "jobid":I,
+   "instance_owner":i,
+   "rank":i,
+   "size":i,
+   "ntasks";i,
+   "service";s,
+   "options": { "verbose":b, "standalone":b },
+   "jobspec":o,
+   "R":o
+
+:func:`flux_shell_get_rank_info` returns shell rank information as a json
+string with the following layout:
+
+::
+
+   "broker_rank":i,
+   "ntasks":i
+   "taskids":s
+   "resources": { "cores":s, ... }
+
+where :var:`broker_rank` is the broker rank on which the target shell rank
+of the query is running, :var:`ntasks` is the number of tasks running under
+that shell rank, :var:`taskids` is a list of task id assignments for those
+tasks (an RFC 22 idset string), and :var:`resources` is a dictionary of
+resource name to resource ids assigned to the shell rank.
+
+:func:`flux_shell_info_unpack` and :func:`flux_shell_rank_info_unpack`
+accomplished the same thing with Jansson-style formatting arguments.
+
+If :var:`shell_rank` is set to -1, the current shell rank is used.
+
+
+RETURN VALUE
+============
+
+All functions return 0 on success and -1 on error.
+
+
+ERRORS
+======
+
+EINVAL
+   if :var:`shell` is NULL, or either :var:`json_str` or :var:`fmt` are NULL,
+   or if :var:`shell_rank` is less than -1.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+Jansson: https://jansson.readthedocs.io/en/2.9/apiref.html
diff --git a/doc/man3/flux_shell_get_jobspec_info.rst b/doc/man3/flux_shell_get_jobspec_info.rst
new file mode 100644
index 000000000000..87246e6e3cb5
--- /dev/null
+++ b/doc/man3/flux_shell_get_jobspec_info.rst
@@ -0,0 +1,75 @@
+==============================
+flux_shell_get_jobspec_info(3)
+==============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   int flux_shell_get_jobspec_info (flux_shell_t *shell,
+                                    char **json_str);
+
+   int flux_shell_jobspec_info_unpack (flux_shell_t *shell,
+                                       const char *fmt,
+                                       ...);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_get_jobspec_info` returns jobspec summary information
+from the flux job shell as a json string. The only key guaranteed to
+be in the returned JSON object is the jobspec :var:`version`, e.g.
+
+::
+   {"version": 1}
+
+
+For jobspec version 1, the following keys are also available:
+
+::
+
+   {
+     "ntasks":i,         # number of tasks requested
+     "nslots":i,         # number of task slots
+     "cores_per_slot":i  # number of cores per task slot
+     "nnodes":i          # number of nodes requested, -1 if unset
+     "slots_per_node":i  # number of slots per node, -1 if unavailable
+    }
+
+This summary information is derived from the jobspec by the shell and
+is shared with plugins in order to avoid duplication of effort.
+
+Currently only version 1 jobspec is supported.
+
+:func:`flux_shell_jobspec_info_unpack` accomplishes the same thing with
+Jansson-style formatting arguments.
+
+
+RETURN VALUE
+============
+
+All functions return 0 on success and -1 on error.
+
+
+ERRORS
+======
+
+EINVAL
+   if :var:`shell` is NULL, or either :var:`json_str` or :var:`fmt` are NULL,
+   or if :var:`shell_rank` is less than -1.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+Jansson: https://jansson.readthedocs.io/en/2.9/apiref.html
diff --git a/doc/man3/flux_shell_get_taskmap.rst b/doc/man3/flux_shell_get_taskmap.rst
new file mode 100644
index 000000000000..4ed31d6a59e2
--- /dev/null
+++ b/doc/man3/flux_shell_get_taskmap.rst
@@ -0,0 +1,52 @@
+=========================
+flux_shell_get_taskmap(3)
+=========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   const struct taskmap * flux_shell_get_taskmap (flux_shell_t *shell);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_get_taskmap` returns the current shell task map. The
+task map can be used to map job task ranks to node IDs and get the set
+of tasks assigned to any node ID. The :type:`struct taskmap` object
+can be queried via the functions exported in ``libflux-taskmap.so``.
+
+
+RETURN VALUE
+============
+
+This function returns a pointer to the shell's internal :type:`struct taskmap`
+or ``NULL`` on failure with :var:`errno` set.
+
+
+ERRORS
+======
+
+EINVAL
+   if :var:`shell` is NULL or the function is called before the initial taskmap
+   is set by the shell.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_34`
diff --git a/doc/man3/flux_shell_getenv.rst b/doc/man3/flux_shell_getenv.rst
new file mode 100644
index 000000000000..a204cac286f5
--- /dev/null
+++ b/doc/man3/flux_shell_getenv.rst
@@ -0,0 +1,73 @@
+====================
+flux_shell_getenv(3)
+====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   const char * flux_shell_getenv (flux_shell_t *shell,
+                                   const char *name);
+
+   int flux_shell_get_environ (flux_shell_t *shell,
+                               char **json_str);
+
+   int flux_shell_setenvf (flux_shell_t *shell,
+                           int overwrite,
+                           const char *name,
+                           const char *fmt,
+                           ...);
+
+   int flux_shell_unsetenv (flux_shell_t *shell,
+                            const char *name);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_getenv` returns the value of an environment variable from
+the global job environment.  :func:`flux_shell_get_environ` returns 0 on
+success with :var:`json_str` set to an allocated JSON string, or -1 on failure
+with :var:`errno` set.  :func:`flux_shell_setenvf` sets an environment variable
+in the global job environment using :linux:man3:`printf` style format
+arguments.  :func:`flux_shell_unsetenv` unsets the specified environment
+variable in the global job environment.
+
+
+RETURN VALUE
+============
+
+:func:`flux_shell_getenv` returns NULL if either :var:`shell` or :var:`name`
+is NULL, or if the variable is not found.
+
+:func:`flux_shell_get_environ` returns a json string on success or NULL on
+failure.
+
+:func:`flux_shell_setenvf` and :func:`flux_shell_unsetenv` return 0 on
+success and -1 on failure.
+
+
+ERRORS
+======
+
+EINVAL
+   :var:`shell`, :var:`name` or :var:`fmt` is NULL.
+
+EEXIST
+   The variable already exists and :var:`overwrite` was not non-zero
+   (func:`flux_shell_setenvf`).
+
+ENOENT
+   With :func:`flux_shell_unsetenv`, the target variable does not exist.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_getopt.rst b/doc/man3/flux_shell_getopt.rst
new file mode 100644
index 000000000000..eda7c75d294a
--- /dev/null
+++ b/doc/man3/flux_shell_getopt.rst
@@ -0,0 +1,72 @@
+====================
+flux_shell_getopt(3)
+====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   int flux_shell_getopt (flux_shell_t *shell,
+                          const char *name,
+                          char **json_str);
+
+   int flux_shell_getopt_unpack (flux_shell_t *shell,
+                                 const char *name,
+                                 const char *fmt,
+                                 ...);
+
+   int flux_shell_setopt (flux_shell_t *shell,
+                          const char *name,
+                          const char *json_str);
+
+   int flux_shell_setopt_pack (flux_shell_t *shell,
+                               const char *name,
+                               const char *fmt,
+                               ...);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_getopt` gets shell option :var:`name` as a JSON string from
+jobspec ``attributes.system.shell.options.name``.
+
+:func:`flux_shell_setopt` sets shell option :var:`name`, making it available to
+subsequent calls from :func:`flux_shell_getopt`. If :var:`json_str` is NULL,
+the option is unset.
+
+:func:`flux_shell_getopt_unpack` and :func:`flux_shell_setopt_unpack` use
+Jansson format strings to accomplish the same functionality.
+
+
+RETURN VALUE
+============
+
+:func:`flux_shell_getopt` and :func:`flux_shell_getopt_unpack` return 1 on
+success, 0 if :var:`name` was not set, and -1 on error,
+
+:func:`flux_shell_setopt` and :func:`flux_shell_setopt_pack` return 0 on
+success and -1 on error.
+
+
+ERRORS
+======
+
+EINVAL
+   :var:`name` or :var:`shell` is NULL.
+
+ENOMEM
+   The process has exhausted its memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_killall.rst b/doc/man3/flux_shell_killall.rst
new file mode 100644
index 000000000000..c7f1ad79226a
--- /dev/null
+++ b/doc/man3/flux_shell_killall.rst
@@ -0,0 +1,41 @@
+=====================
+flux_shell_killall(3)
+=====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+
+   void flux_shell_killall (flux_shell_t *shell, int sig);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+Sends the signal :var:`sig` to all processes running in :var:`shell`. No
+errors are set, but the call returns immediately if :var:`shell` is NULL
+or if :var:`sig` is zero or negative.
+
+
+RETURN VALUE
+============
+
+None.
+
+
+ERRORS
+======
+
+None.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_log.rst b/doc/man3/flux_shell_log.rst
new file mode 100644
index 000000000000..75f2109d6306
--- /dev/null
+++ b/doc/man3/flux_shell_log.rst
@@ -0,0 +1,165 @@
+=================
+flux_shell_log(3)
+=================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   void flux_shell_log (const char *component,
+                        int level,
+                        const char *file,
+                        int line,
+                        const char *fmt,
+                        ...);
+
+   int flux_shell_err (const char *component,
+                       const char *file,
+                       int line,
+                       int errnum,
+                       const char *fmt,
+                       ...);
+
+   void flux_shell_fatal (const char *component,
+                          const char *file,
+                          int line,
+                          int errnum,
+                          int exit_code,
+                          const char *fmt,
+                          ...);
+
+   void flux_shell_raise (const char *type,
+                          int severity,
+                          const char *fmt,
+                          ...);
+
+   int flux_shell_log_setlevel (int level, const char *dest);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_log` logs a message at for shell component or plugin
+:var:`component` at :var:`level` to all loggers registered to receive messages
+at that severity or greater. See :man3:`flux_log` for a list of supported
+levels.
+
+
+The following macros handle common levels. For external shell plugins,
+the required macro ``FLUX_SHELL_PLUGIN_NAME`` is automatically substituted
+for the :var:`component` in all macros.
+
+
+::
+
+   #define shell_trace(...) \
+
+::
+
+   #define shell_debug(...) \
+
+::
+
+   #define shell_log(...) \
+
+::
+
+   #define shell_warn(...) \
+
+::
+
+   #define shell_log_error(...) \
+
+:func:`flux_shell_err` logs a message at FLUX_SHELL_ERROR level,
+additionally appending the result of strerror(``errnum``) for
+convenience. Macros include:
+
+::
+
+   #define shell_log_errn(errn, ...) \
+
+::
+
+   #define shell_log_errno(...) \
+
+Note that :var:`errno` is the standard global value defined in ``errno.h``
+and :var:`errn` is a user-provided error code.
+
+func:`flux_shell_fatal` logs a message at FLUX_SHELL_FATAL level and
+schedules termination of the job shell. This may generate an
+exception if tasks are already running. Exits with :var:`exit_code`.
+While the macro names are similar to those using :func:`flux_shell_err`,
+note that the choices of :var:`errnum` are either 0 or :var:`errno`.
+
+::
+
+   #define shell_die(code,...) \
+
+::
+
+   #define shell_die_errno(code,...) \
+
+:func:`flux_shell_raise` explicitly raises an exception for the current
+job of the given :var:`type` and :var:`severity`. Exceptions of severity 0
+will result in termination of the job by the execution system.
+
+:func:`flux_shell_log_setlevel` sets default severity of logging
+destination :var:`dest` to :var:`level`. If :var:`dest` is NULL then the
+internal log dispatch level is set (i.e. no messages above severity level will
+be logged to any log destination). Macros include:
+
+::
+
+   #define shell_set_verbose(n) \
+   flux_shell_log_setlevel(FLUX_SHELL_NOTICE+n, NULL)
+
+::
+
+   #define shell_set_quiet(n) \
+   flux_shell_log_setlevel(FLUX_SHELL_NOTICE-n, NULL)
+
+As a special case, if :var:`level` is set to ``FLUX_SHELL_QUIET``, then
+logging will be completely disabled to :var:`dest`. For example, to disable
+logging to :var:`stderr`, use:
+
+::
+
+   flux_shell_log_setlevel (FLUX_SHELL_QUIET, "stderr");
+
+
+RETURN VALUE
+============
+
+:func:`flux_shell_err` returns -1 with :var:`errno` = :var:`errnum`, so that the
+function can be used as:
+return flux_shell_err(…​);
+
+:func:`flux_shell_log_setlevel` will return -1 and set :var:`errno` to EINVAL
+if the requested :var:`level` is not valid or if :var:`dest` is not a valid
+pointer to a logger shell.
+
+
+ERRORS:
+=======
+
+EINVAL
+   :var:`level` or :var:`dest` is not valid.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_log`
diff --git a/doc/man3/flux_shell_plugstack_call.rst b/doc/man3/flux_shell_plugstack_call.rst
new file mode 100644
index 000000000000..d3c11813588f
--- /dev/null
+++ b/doc/man3/flux_shell_plugstack_call.rst
@@ -0,0 +1,47 @@
+============================
+flux_shell_plugstack_call(3)
+============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+
+   int flux_shell_plugstack_call (flux_shell_t *shell,
+                                  const char *topic,
+                                  flux_plugin_arg_t *args);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+The job shell implements a flexible plugin architecture which allows
+registration of one or more callback functions on arbitrary topic
+names. The stack of functions "listening" on a given topic string is
+called the "plugin stack". :func:`flux_shell_plugstack_call` exports the
+ability to call into the plugin stack so that plugins can invoke
+callbacks from other plugins.
+
+
+RETURN VALUE
+============
+
+Returns 0 on success and -1 on failure, setting :var:`errno`.
+
+
+ERRORS:
+=======
+
+EINVAL
+   :var:`shell` or :var:`topic` are NULL.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_rpc_pack.rst b/doc/man3/flux_shell_rpc_pack.rst
new file mode 100644
index 000000000000..d3d6829b5654
--- /dev/null
+++ b/doc/man3/flux_shell_rpc_pack.rst
@@ -0,0 +1,48 @@
+======================
+flux_shell_rpc_pack(3)
+======================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+   #include <errno.h>
+
+   flux_future_t *flux_shell_rpc_pack (flux_shell_t *shell,
+                                       const char *method,
+                                       int shell_rank,
+                                       int flags,
+                                       const char *fmt,
+                                       ...);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+Send a remote procedure call :var:`method` to another shell in the same
+job at shell rank :var:`shell_rank`.
+
+
+RETURN VALUE
+============
+
+Returns NULL on failure.
+
+
+ERRORS
+======
+
+EINVAL
+   :var:`shell`, :var:`method` or :var:`fmt` are NULL, or if :var:`rank` is
+   less than 0.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_service_register.rst b/doc/man3/flux_shell_service_register.rst
new file mode 100644
index 000000000000..3c2c226f2dc0
--- /dev/null
+++ b/doc/man3/flux_shell_service_register.rst
@@ -0,0 +1,54 @@
+==============================
+flux_shell_service_register(3)
+==============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+
+   int flux_shell_service_register (flux_shell_t *shell,
+                                    const char *method,
+                                    flux_msg_handler_f cb,
+                                    void *arg);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+The job shell registers a unique service name with the flux broker on
+startup, and posts the topic string for this service in the context of
+the ``shell.init`` event. :func:`flux_shell_service_register` allows
+registration of a request handler :var:`cb` for subtopic :var:`method` on this
+service endpoint, allowing other job shells and/or flux commands to
+interact with arbitrary services within a job.
+
+
+RETURN VALUE
+============
+
+Returns -1 on failure, 0 on success.
+
+
+ERRORS
+======
+
+EINVAL
+   :var:`shell`, :var:`method` or :var:`cb` is NULL.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_msg_handler_create`
diff --git a/doc/man3/flux_shell_task_channel_subscribe.rst b/doc/man3/flux_shell_task_channel_subscribe.rst
new file mode 100644
index 000000000000..989ef17566b3
--- /dev/null
+++ b/doc/man3/flux_shell_task_channel_subscribe.rst
@@ -0,0 +1,50 @@
+====================================
+flux_shell_task_channel_subscribe(3)
+====================================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+
+   int flux_shell_task_channel_subscribe (flux_shell_task_t *task,
+                                          const char *channel,
+                                          flux_shell_task_io_f cb,
+                                          void *arg);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+Call :var:`cb` when shell task output channel :var:`name` is ready for reading.
+
+Callback can then call :man3:`flux_shell_task_get_subprocess` and use
+:man3:`flux_subprocess_read` or :man3:`flux_subprocess_getline` on the
+result to get available data. Only one subscriber per stream is allowed.
+
+
+RETURN VALUE
+============
+
+Returns 0 on success and -1 on error.
+
+Not yet implemented.
+
+
+ERRORS
+======
+
+EEXIST
+   :func:`flux_shell_task_channel_subscribe` is called on a stream with an
+   existing subscriber
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_task_get_info.rst b/doc/man3/flux_shell_task_get_info.rst
new file mode 100644
index 000000000000..13cdf873fd56
--- /dev/null
+++ b/doc/man3/flux_shell_task_get_info.rst
@@ -0,0 +1,57 @@
+===========================
+flux_shell_task_get_info(3)
+===========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+
+   int flux_shell_task_get_info (flux_shell_task_t *task,
+                                 char **json_str);
+
+   int flux_shell_task_info_unpack (flux_shell_task_t *task,
+                                    const char *fmt,
+                                    ...);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+Returns task info either as a json string (specified below) or
+using Jansson-style parameters. The structure of the former is:
+
+::
+
+   "localid":i,
+   "rank":i,
+   "state":s,
+   "pid":I,
+   "wait_status":i,
+   "exitcode":i,
+   "signaled":i
+
+
+RETURN VALUE
+============
+
+Returns 0 on success and -1 on failure. A failure will not
+necessarily set errno.
+
+
+ERRORS
+======
+
+EINVAL
+   If :var:`task` or :var:`json_str` is NULL.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_shell_task_subprocess.rst b/doc/man3/flux_shell_task_subprocess.rst
new file mode 100644
index 000000000000..c3428e328f40
--- /dev/null
+++ b/doc/man3/flux_shell_task_subprocess.rst
@@ -0,0 +1,47 @@
+=============================
+flux_shell_task_subprocess(3)
+=============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/shell.h>
+
+   flux_subprocess_t *flux_shell_task_subprocess (flux_shell_task_t *task);
+
+   flux_cmd_t *flux_shell_task_cmd (flux_shell_task_t *task);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_shell_task_subprocess` returns the subprocess for a shell
+task in var:`task_fork` and :var:`task_exit` callbacks.
+
+:func:`flux_shell_task_cmd` returns the cmd structure for a shell task.
+
+
+RETURN VALUE
+============
+
+:func:`flux_shell_task_subprocess` returns the :var:`proc` field of the
+:var:`task`, and :func:`flux_shell_task_cmd` returns the :var:`cmd` field,
+or NULL on error.
+
+
+ERRORS
+======
+
+EINVAL
+   :var:`task` is NULL.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
diff --git a/doc/man3/flux_signal_watcher_create.adoc b/doc/man3/flux_signal_watcher_create.adoc
deleted file mode 100644
index 7e436783a7d2..000000000000
--- a/doc/man3/flux_signal_watcher_create.adoc
+++ /dev/null
@@ -1,71 +0,0 @@
-flux_signal_watcher_create(3)
-=============================
-:doctype: manpage
-
-
-NAME
-----
-flux_signal_watcher_create, flux_signal_watcher_get_signum - create signal watcher
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_watcher_f)(flux_reactor_t *r,
-                                flux_watcher_t *w,
-                                int revents, void *arg);
-
- flux_watcher_t *flux_signal_watcher_create (flux_reactor_t *r,
-                                             int signum,
-                                             flux_watcher_f callback,
-                                             void *arg);
-
- int flux_signal_watcher_get_signum (flux_watcher_t *w);
-
-DESCRIPTION
------------
-
-`flux_signal_watcher_create()` creates a reactor watcher that
-monitors for receipt of signal _signum_.
-
-The callback _revents_ argument should be ignored.
-
-When one _callback_ is shared by multiple watchers, the signal number that
-triggered the event can be obtained with `flux_signal_watcher_get_signum()`.
-
-
-RETURN VALUE
-------------
-
-flux_signal_watcher_create() returns a flux_watcher_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_watcher_start(3), flux_reactor_start(3)
-
-http://software.schmorp.de/pkg/libev.html[libev home page]
diff --git a/doc/man3/flux_signal_watcher_create.rst b/doc/man3/flux_signal_watcher_create.rst
new file mode 100644
index 000000000000..0baf073dbe7d
--- /dev/null
+++ b/doc/man3/flux_signal_watcher_create.rst
@@ -0,0 +1,66 @@
+=============================
+flux_signal_watcher_create(3)
+=============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_watcher_f)(flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg);
+
+   flux_watcher_t *flux_signal_watcher_create (flux_reactor_t *r,
+                                               int signum,
+                                               flux_watcher_f callback,
+                                               void *arg);
+
+   int flux_signal_watcher_get_signum (flux_watcher_t *w);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_signal_watcher_create` creates a reactor watcher that
+monitors for receipt of signal :var:`signum`.
+
+The callback :var:`revents` argument should be ignored.
+
+When one :var:`callback` is shared by multiple watchers, the signal number that
+triggered the event can be obtained with
+:func:`flux_signal_watcher_get_signum`.
+
+
+RETURN VALUE
+============
+
+:func:`flux_signal_watcher_create` returns a :type:`flux_watcher_t` object
+on success.  On error, NULL is returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+libev: http://software.schmorp.de/pkg/libev.html
+
+
+SEE ALSO
+========
+
+:man3:`flux_watcher_start`, :man3:`flux_reactor_run`
diff --git a/doc/man3/flux_stat_watcher_create.adoc b/doc/man3/flux_stat_watcher_create.adoc
deleted file mode 100644
index 56023c5f32ff..000000000000
--- a/doc/man3/flux_stat_watcher_create.adoc
+++ /dev/null
@@ -1,83 +0,0 @@
-flux_stat_watcher_create(3)
-===========================
-:doctype: manpage
-
-
-NAME
-----
-flux_stat_watcher_create, flux_stat_watcher_get_rstat - create stat watcher
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_watcher_f)(flux_reactor_t *r,
-                                flux_watcher_t *w,
-                                int revents, void *arg);
-
- flux_watcher_t *flux_stat_watcher_create (flux_reactor_t *r,
-                                           const char *path,
-                                           double interval,
-                                           flux_watcher_f callback,
-                                           void *arg);
-
- void flux_stat_watcher_get_rstat (flux_watcher_t *w,
-                                   struct stat *stat,
-                                   struct stat *prev);
-
-DESCRIPTION
------------
-
-`flux_stat_watcher_create()` creates a reactor watcher that
-monitors for changes in the status of the file system object
-represented by _path_.  If the file system object exists,
-inotify(2) is used, if available; otherwise the reactor polls
-the file every _interval_ seconds.  A value of zero selects a
-conservative default (currently five seconds).
-
-The callback _revents_ argument should be ignored.
-
-`flux_stat_watcher_get_rstat ()` may be used to obtain the status
-within _callback_.  If non-NULL, _stat_ receives the current status.
-If non-NULL, _prev_ receives the previous status.
-
-If the object does not exist, stat->st_nlink will be zero and other
-status fields are undefined.  The appearance/disappearance of a file
-is considered a status change like any other.
-
-
-RETURN VALUE
-------------
-
-flux_stat_watcher_create() returns a flux_watcher_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_watcher_start(3), flux_reactor_start(3), stat(2)
-
-http://software.schmorp.de/pkg/libev.html[libev home page]
diff --git a/doc/man3/flux_stat_watcher_create.rst b/doc/man3/flux_stat_watcher_create.rst
new file mode 100644
index 000000000000..dae76688f106
--- /dev/null
+++ b/doc/man3/flux_stat_watcher_create.rst
@@ -0,0 +1,78 @@
+===========================
+flux_stat_watcher_create(3)
+===========================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_watcher_f)(flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg);
+
+   flux_watcher_t *flux_stat_watcher_create (flux_reactor_t *r,
+                                             const char *path,
+                                             double interval,
+                                             flux_watcher_f callback,
+                                             void *arg);
+
+   void flux_stat_watcher_get_rstat (flux_watcher_t *w,
+                                     struct stat *stat,
+                                     struct stat *prev);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_stat_watcher_create` creates a reactor watcher that
+monitors for changes in the status of the file system object
+represented by :var:`path`. If the file system object exists,
+:linux:man7:`inotify` is used, if available; otherwise the reactor polls
+the file every :var:`interval` seconds. A value of zero selects a
+conservative default (currently five seconds).
+
+The callback :var:`revents` argument should be ignored.
+
+:func:`flux_stat_watcher_get_rstat` may be used to obtain the status
+within :var:`callback`. If non-NULL, :var:`stat` receives the current status.
+If non-NULL, :var:`prev` receives the previous status.
+
+If the object does not exist, :var:`stat->st_nlink` will be zero and other
+status fields are undefined. The appearance/disappearance of a file
+is considered a status change like any other.
+
+
+RETURN VALUE
+============
+
+:func:`flux_stat_watcher_create` returns a :type:`flux_watcher_t` object
+on success.  On error, NULL is returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+libev: http://software.schmorp.de/pkg/libev.html
+
+
+SEE ALSO
+========
+
+:man3:`flux_watcher_start`, :man3:`flux_reactor_run`,
+:linux:man2:`stat`
diff --git a/doc/man3/flux_sync_create.rst b/doc/man3/flux_sync_create.rst
new file mode 100644
index 000000000000..b2d224e62a13
--- /dev/null
+++ b/doc/man3/flux_sync_create.rst
@@ -0,0 +1,79 @@
+===================
+flux_sync_create(3)
+===================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_future_t *flux_sync_create (flux_t *h, double minimum);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_sync_create` creates a future that is fulfilled when
+the system heartbeat message is received.  System heartbeats are
+event messages published periodically at a configurable interval.
+Synchronizing Flux internal overhead to the heartbeat can, in theory,
+reduce disruption to bulk synchronous applications.
+
+If :var:`minimum` is greater than zero, it establishes a minimum time in seconds
+between fulfillments.  Heartbeats that arrive too soon after the last one
+are ignored.  This may be used to protect from thrashing if the heartbeat
+period is set too fast, or if heartbeats arrive close to one another in time
+due to overlay network congestion.
+
+A maximum time between fulfillments may be established by specifying a
+continuation timeout with :man3:`flux_future_then`.  If the timeout expires,
+the future is fulfilled with an error (ETIMEDOUT), as usual.
+
+On each fulfillment, :man3:`flux_future_reset` should be called to enable
+the future to be fulfilled again, and to re-start any timeout.
+
+
+RETURN VALUE
+============
+
+:func:`flux_sync_create` returns a future, or NULL on failure with
+:var:`errno` set.
+
+
+ERRORS
+======
+
+EINVAL
+   One or more arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+
+EXAMPLE
+=======
+
+Set up a continuation callback for each heartbeat that arrives at least
+:var:`sync_min` seconds from the last, with a timeout of :var:`sync_max`
+seconds:
+
+
+.. literalinclude:: example/sync.c
+  :language: c
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_future_then`, :man3:`flux_future_get`, :man3:`flux_future_reset`
diff --git a/doc/man3/flux_timer_watcher_create.adoc b/doc/man3/flux_timer_watcher_create.adoc
deleted file mode 100644
index fc510df34407..000000000000
--- a/doc/man3/flux_timer_watcher_create.adoc
+++ /dev/null
@@ -1,90 +0,0 @@
-flux_timer_watcher_create(3)
-============================
-:doctype: manpage
-
-
-NAME
-----
-flux_timer_watcher_create, flux_timer_watcher_reset -  set/reset a timer
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_watcher_f)(flux_reactor_t *r,
-                                flux_watcher_t *w,
-                                int revents, void *arg);
-
- flux_watcher_t *flux_timer_watcher_create (flux_reactor_t *r,
-                                            double after, double repeat,
-                                            flux_watcher_f callback,
-                                            void *arg);
-
- void flux_timer_watcher_reset (flux_watcher_t *w,
-                                double after, double repeat);
-
-
-DESCRIPTION
------------
-
-`flux_timer_watcher_create()` creates a flux_watcher_t object which
-monitors for timer events.  A timer event occurs when _after_ seconds
-have elapsed, and optionally again every _repeat_ seconds.
-When events occur, the user-supplied _callback_ is invoked.
-
-If _after_ is 0., the flux_watcher_t will be immediately ready 
-when the reactor is started.  If _repeat_ is 0., the flux_watcher_t
-will automatically be stopped when _after_ seconds have elapsed.
-
-Note that _after_ is internally referenced to reactor time, which is
-only updated when the reactor is run/created, and therefore
-can be out of date.  Use `flux_reactor_now_update(3)` to manually
-update reactor time before creating timer watchers in such cases.
-Refer to "The special problem of time updates" in the libev manual
-for more information.
-
-To restart a timer that has been automatically stopped, you must reset
-the _after_ and _repeat_ values with `flux_timer_watcher_reset()` before
-calling `flux_watcher_start()`.
-
-The callback _revents_ argument should be ignored.
-
-Note: the Flux reactor is based on libev.  For additional information
-on the behavior of timers, refer to the libev documentation on `ev_timer`.
-
-
-RETURN VALUE
-------------
-
-`flux_timer_watcher_create()` returns a flux_watcher_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_watcher_start(3), flux_reactor_start(3), flux_reactor_now(3)
-
-http://software.schmorp.de/pkg/libev.html[libev home page]
diff --git a/doc/man3/flux_timer_watcher_create.rst b/doc/man3/flux_timer_watcher_create.rst
new file mode 100644
index 000000000000..6f0ff59f8984
--- /dev/null
+++ b/doc/man3/flux_timer_watcher_create.rst
@@ -0,0 +1,85 @@
+============================
+flux_timer_watcher_create(3)
+============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   typedef void (*flux_watcher_f)(flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg);
+
+   flux_watcher_t *flux_timer_watcher_create (flux_reactor_t *r,
+                                              double after,
+                                              double repeat,
+                                              flux_watcher_f callback,
+                                              void *arg);
+
+   void flux_timer_watcher_reset (flux_watcher_t *w,
+                                  double after,
+                                  double repeat);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_timer_watcher_create` creates a :type:`flux_watcher_t` object which
+monitors for timer events. A timer event occurs when :var:`after` seconds
+have elapsed, and optionally again every :var:`repeat` seconds.
+When events occur, the user-supplied :var:`callback` is invoked.
+
+If :var:`after` is 0., the :type:`flux_watcher_t` will be immediately ready
+when the reactor is started. If :var:`repeat` is 0., the :type:`flux_watcher_t`
+will automatically be stopped when :var:`after` seconds have elapsed.
+
+Note that :var:`after` is internally referenced to reactor time, which is
+only updated when the reactor is run/created, and therefore
+can be out of date. Use :man3:`flux_reactor_now_update` to manually
+update reactor time before creating timer watchers in such cases.
+Refer to "The special problem of time updates" in the libev manual
+for more information.
+
+To restart a timer that has been automatically stopped, you must reset
+the :var:`after` and :var:`repeat` values with :func:`flux_timer_watcher_reset`
+before calling :man3:`flux_watcher_start`.
+
+The callback :var:`revents` argument should be ignored.
+
+Note: the Flux reactor is based on libev. For additional information
+on the behavior of timers, refer to the libev documentation on ``ev_timer``.
+
+
+RETURN VALUE
+============
+
+:func:`flux_timer_watcher_create` returns a :type:`flux_watcher_t` object
+on success.  On error, NULL is returned, and :var:`errno` is set appropriately.
+
+
+ERRORS
+======
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+libev: http://software.schmorp.de/pkg/libev.html
+
+
+SEE ALSO
+========
+
+:man3:`flux_watcher_start`, :man3:`flux_reactor_run`, :man3:`flux_reactor_now`
diff --git a/doc/man3/flux_watcher_set_priority.rst b/doc/man3/flux_watcher_set_priority.rst
new file mode 100644
index 000000000000..dfc507e10d72
--- /dev/null
+++ b/doc/man3/flux_watcher_set_priority.rst
@@ -0,0 +1,43 @@
+============================
+flux_watcher_set_priority(3)
+============================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/core.h>
+
+   flux_watcher_t *flux_watcher_set_priority (flux_watcher_t *w,
+                                              int priority);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_watcher_set_priority` sets the priority on the watcher.
+Higher priority watchers run first.  The range of priorities is from
+-2 to 2, with the default being 0.  If the priority is out
+of range, the max or min value is set.  The priority should only be
+set when the watcher is stopped.
+
+This function is a no-op if the underlying watcher does not support priorities.
+Currently only the check watcher supports priorities.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+libev: http://software.schmorp.de/pkg/libev.html
+
+
+SEE ALSO
+========
+
+:man3:`flux_check_watcher_create`
diff --git a/doc/man3/flux_watcher_start.adoc b/doc/man3/flux_watcher_start.adoc
deleted file mode 100644
index 3d820cee3bf0..000000000000
--- a/doc/man3/flux_watcher_start.adoc
+++ /dev/null
@@ -1,61 +0,0 @@
-flux_watcher_start(3)
-=====================
-:doctype: manpage
-
-
-NAME
-----
-flux_watcher_start, flux_watcher_stop, flux_watcher_destroy, flux_watcher_next_wakeup -  start/stop/destroy/query reactor watcher
-
-
-SYNOPSIS
---------
-
-void flux_watcher_start (flux_watcher_t *w);
-
-void flux_watcher_stop (flux_watcher_t *w);
-
-void flux_watcher_destroy (flux_watcher_t *w);
-
-double flux_watcher_next_wakeup (flux_watcher_t *w);
-
-
-DESCRIPTION
------------
-
-`flux_watcher_start()` activates a flux_watcher_t object _w_ so that it
-can receive events.  If _w_ is already active, the call has no effect.
-This may be called from within a flux_watcher_f callback.
-
-`flux_watcher_stop()` deactivates a flux_watcher_t object _w_ so that it
-stops receiving events.  If _w_ is already inactive, the call has no effect.
-This may be called from within a flux_watcher_f callback.
-
-`flux_watcher_destroy()` destroys a flux_watcher_t object _w_,
-after stopping it.  It is not safe to destroy a watcher object within a
-flux_watcher_f callback.
-
-`flux_watcher_next_wakeup()` returns the absolute time that the watcher
-is supposed to trigger next. This function only works for _timer_ and
-_periodic_ watchers, and will return a value less than zero with errno
-set to `EINVAL` otherwise.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_reactor_create (3)
diff --git a/doc/man3/flux_watcher_start.rst b/doc/man3/flux_watcher_start.rst
new file mode 100644
index 000000000000..a902021e3625
--- /dev/null
+++ b/doc/man3/flux_watcher_start.rst
@@ -0,0 +1,58 @@
+=====================
+flux_watcher_start(3)
+=====================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  void flux_watcher_start (flux_watcher_t *w);
+
+  void flux_watcher_stop (flux_watcher_t *w);
+
+  bool flux_watcher_is_active (flux_watcher_t *w);
+
+  void flux_watcher_destroy (flux_watcher_t *w);
+
+  double flux_watcher_next_wakeup (flux_watcher_t *w);
+
+Link with :command:`-lflux-core`.
+
+DESCRIPTION
+===========
+
+:func:`flux_watcher_start` activates a :type:`flux_watcher_t` object :var:`w`
+so that it can receive events. If :var:`w` is already active, the call has no
+effect.  This may be called from within a :type:`flux_watcher_f` callback.
+
+:func:`flux_watcher_stop` deactivates a :type:`flux_watcher_t` object :var:`w`
+so that it stops receiving events. If :var:`w` is already inactive, the call
+has no effect.  This may be called from within a :type:`flux_watcher_f`
+callback.
+
+:func:`flux_watcher_is_active` returns a true value if the watcher is active
+(i.e. it has been started and not yet stopped) and false otherwise.
+
+:func:`flux_watcher_destroy` destroys a :type:`flux_watcher_t` object :var:`w`,
+after stopping it. It is not safe to destroy a watcher object within a
+:type:`flux_watcher_f` callback.
+
+:func:`flux_watcher_next_wakeup` returns the absolute time that the watcher
+is supposed to trigger next. This function only works for :var:`timer` and
+:var:`periodic` watchers, and will return a value less than zero with
+:var:`errno` set to ``EINVAL`` otherwise.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man3:`flux_reactor_create`
diff --git a/doc/man3/flux_zmq_watcher_create.adoc b/doc/man3/flux_zmq_watcher_create.adoc
deleted file mode 100644
index 26f36d894b8f..000000000000
--- a/doc/man3/flux_zmq_watcher_create.adoc
+++ /dev/null
@@ -1,91 +0,0 @@
-flux_zmq_watcher_create(3)
-==========================
-:doctype: manpage
-
-
-NAME
-----
-flux_zmq_watcher_create, flux_zmq_watcher_get_zsock -  create ZeroMQ watcher
-
-
-SYNOPSIS
---------
- #include <flux/core.h>
-
- typedef void (*flux_watcher_f)(flux_reactor_t *r,
-                                flux_watcher_t *w,
-                                int revents, void *arg);
-
- flux_watcher_t *flux_zmq_watcher_create (flux_reactor_t *r,
-                                          void *zsock, int events,
-                                          flux_watcher_f callback,
-                                          void *arg);
-
- void *flux_zmq_watcher_get_zsock (flux_watcher_t *w);
-
-
-DESCRIPTION
------------
-
-`flux_zmq_watcher_create()` creates a flux_watcher_t object which
-monitors for events on a ZeroMQ socket _zsock_.  When events occur,
-the user-supplied _callback_ is invoked.
-
-The _events_ and _revents_ arguments are a bitmask containing a
-logical ``or'' of the following bits.  If a bit is set in _events_,
-it indicates interest in this type of event.  If a bit is set in the
-_revents_, it indicates that this event has occurred.
-
-FLUX_POLLIN::
-The socket is ready for reading.
-
-FLUX_POLLOUT::
-The socket is ready for writing.
-
-FLUX_POLLERR::
-The socket has encountered an error.
-This bit is ignored if it is set in _events_.
-
-Events are processed in a level-triggered manner.  That is, the
-callback will continue to be invoked as long as the event has not been
-fully consumed or cleared, and the watcher has not been stopped.
-
-`flux_zmq_watcher_get_zsock()` is used to obtain the socket from
-within the callback.
-
-
-RETURN VALUE
-------------
-
-`flux_zmq_watcher_create()` returns a flux_watcher_t object on success.
-On error, NULL is returned, and errno is set appropriately.
-
-`flux_zmq_watcher_get_zsock()` returns the socket associated with
-the watcher.
-
-
-ERRORS
-------
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
----------
-flux_watcher_start(3), flux_reactor_start(3).
diff --git a/doc/man3/hostlist_create.rst b/doc/man3/hostlist_create.rst
new file mode 100644
index 000000000000..d613362c798c
--- /dev/null
+++ b/doc/man3/hostlist_create.rst
@@ -0,0 +1,161 @@
+==================
+hostlist_create(3)
+==================
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+  #include <flux/hostlist.h>
+
+  struct hostlist *hostlist_create (void);
+
+  void hostlist_destroy (struct hostlist *hl);
+
+  struct hostlist *hostlist_decode (const char *s);
+
+  char *hostlist_encode (struct hostlist *hl);
+
+  struct hostlist *hostlist_copy (const struct hostlist *hl);
+
+  int hostlist_count (struct hostlist *hl);
+
+  int hostlist_append (struct hostlist *hl, const char *hosts);
+
+  int hostlist_append_list (struct hostlist *hl1, struct hostlist *hl2);
+
+  const char *hostlist_nth (struct hostlist *hl, int n);
+
+  int hostlist_find (struct hostlist *hl, const char *hostname);
+
+  int hostlist_delete (struct hostlist *hl, const char *hosts);
+
+  void hostlist_sort (struct hostlist *hl);
+
+  void hostlist_uniq (struct hostlist *hl);
+
+  const char *hostlist_first (struct hostlist *hl);
+
+  const char *hostlist_last (struct hostlist *hl);
+
+  const char *hostlist_next (struct hostlist *hl);
+
+  const char *hostlist_current (struct hostlist *hl);
+
+  int hostlist_remove_current (struct hostlist *hl);
+
+Link with :command:`-lflux-hostlist`.
+
+DESCRIPTION
+===========
+
+A hostlist is an ordered list of hostnames that can be encoded as a
+compact string when the hostnames contain numerical indices.  For example,
+the hostnames ``test0``, ``test1``, ... ``test127`` may be encoded to
+``test[0-127]``.  Hostlists are further described in
+:doc:`Flux RFC 29 <rfc:spec_29>`.
+
+The hostlist contains an internal cursor that is used for iteration.
+For the functions below that return a :type:`const char *` hostname,
+the returned value may be assumed to remain valid only until the next
+call to a function that updates the cursor.
+
+:func:`hostlist_create` creates an empty hostlist.
+
+:func:`hostlist_destroy` destroys a hostlist.
+
+:func:`hostlist_decode` converts an RFC 29 hostlist string into a hostlist.
+The caller must free the result with :func:`hostlist_destroy`.
+
+:func:`hostlist_encode` converts a hostlist into an RFC 29 hostlist string.
+The caller must free the result with :linux:man3:`free`.
+
+:func:`hostlist_copy` makes a copy of a hostlist.
+The caller must free the result with :func:`hostlist_destroy`.
+
+:func:`hostlist_count` returns the number of hostnames in a hostlist.
+
+:func:`hostlist_append` decodes an RFC 29 hostlist string and appends its
+hostnames to another hostlist.
+
+:func:`hostlist_append_list` appends the hostnames of a hostlist to another
+hostlist.
+
+:func:`hostlist_nth` sets the cursor to the hostname at index :var:`n`
+(zero origin) and returns it.
+
+:func:`hostlist_find` sets the cursor to the position of :var:`hostname`
+and returns its zero-origin index.
+
+:func:`hostlist_delete` deletes an RFC 29 hostlist string from a hostlist.
+If the cursor hostname is deleted, the cursor is advanced to the next valid
+hostname.
+
+:func:`hostlist_sort` sorts a hostlist object.  The cursor may be updated.
+
+:func:`hostlist_uniq` sorts a hostlist object, then removes duplicate
+hostnames.  The cursor may be updated.
+
+:func:`hostlist_first` sets the cursor to the first hostname and returns it.
+
+:func:`hostlist_last` sets the cursor to the last hostname and returns it.
+
+:func:`hostlist_next` sets the cursor to next hostname and returns it.
+
+:func:`hostlist_current` returns the hostname at the cursor.
+
+:func:`hostlist_remove_current` removes the hostname at the cursor and sets
+the cursor to the next hostname.
+
+RETURN VALUE
+============
+
+:func:`hostlist_create`, :func:`hostlist_decode`, and :func:`hostlist_copy`
+return a hostlist on success which must be freed with :func:`hostlist_destroy`.
+On failure, NULL is returned with :var:`errno` set.
+
+:func:`hostlist_encode` returns a string on success that must be freed.
+On failure, NULL is returned with :var:`errno` set.
+
+:func:`hostlist_append`, :func:`hostlist_append_list`, :func:`hostlist_delete`,
+and :func:`hostlist_remove_current` return a count on success.
+On failure, -1 is returned with :var:`errno` set.
+
+:func:`hostlist_count` returns a count.  If the hostlist is invalid, zero
+is returned.
+
+:func:`hostlist_find` returns an index on success.
+On failure, -1 is returned with :var:`errno` set.
+
+:func:`hostlist_sort` and :func:`hostlist_uniq` return nothing.
+
+Other functions return a string on success, or NULL on failure with
+:var:`errno` set.
+
+ERRORS
+======
+
+EINVAL
+  One or more arguments were invalid.
+
+ENOMEM
+  Out of memory.
+
+ERANGE
+  Internal maximum numerical range span was exceeded.
+
+ENOENT
+  Index or hostname was not found.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_29`
diff --git a/doc/man3/idset_add.rst b/doc/man3/idset_add.rst
new file mode 100644
index 000000000000..e68eec02aefa
--- /dev/null
+++ b/doc/man3/idset_add.rst
@@ -0,0 +1,103 @@
+============
+idset_add(3)
+============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/idset.h>
+
+   bool idset_equal (const struct idset *a,
+                     const struct idset *b);
+
+   struct idset *idset_union (const struct idset *a,
+                              const struct idset *b);
+
+   struct idset *idset_difference (const struct idset *a,
+                                   const struct idset *b);
+
+   struct idset *idset_intersect (const struct idset *a,
+                                  const struct idset *b);
+
+   int idset_add (struct idset *a, const struct idset *b);
+
+   int idset_subtract (struct idset *a, const struct idset *b);
+
+   bool idset_has_intersection (const struct idset *a,
+                                const struct idset *b);
+
+   #define idset_clear_all (x) idset_subtract (x, x)
+
+Link with :command:`-lflux-idset`.
+
+DESCRIPTION
+===========
+
+Refer to :man3:`idset_create` for a general description of idsets.
+
+:func:`idset_equal` returns true if the two idset objects :var:`a` and
+:var:`b` are equal sets, i.e. the sets contain the same set of integers.
+
+:func:`idset_union` creates a new idset that is the union of :var:`a` and
+:var:`b`.
+
+:func:`idset_difference` creates a new idset that is :var:`a` with the members
+of :var:`b` removed.
+
+:func:`idset_intersect` creates a new idset containing only members of :var:`a`
+and :var:`b` that are in both sets.
+
+:func:`idset_add` adds the members of :var:`b` to :var:`a`.
+
+:func:`idset_subtract` removes the members of :var:`b` from :var:`a`.
+
+:func:`idset_has_intersection` tests whether :var:`a` and :var:`b` have any
+members in common.
+
+:func:`idset_clear_all` removes all members of :var:`x`.
+
+
+RETURN VALUE
+============
+
+:func:`idset_union`, :func:`idset_difference`, and :func:`idset_intersect`
+return an idset on success which must be freed with :man3:`idset_destroy`.
+On error, NULL is returned with :var:`errno` set.
+
+:func:`idset_add`, :func:`idset_subtract`, and :func:`idset_clear_all`
+return 0 on success.  On error, -1 is returned with :var:`errno` set.
+
+:func:`idset_equal` and :func:`idset_has_intersection` return true or false.
+
+
+ERRORS
+======
+
+EINVAL
+   One or more arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_22`
+
+
+
+SEE ALSO
+========
+
+:man3:`idset_create`, :man3:`idset_encode`
diff --git a/doc/man3/idset_alloc.rst b/doc/man3/idset_alloc.rst
new file mode 100644
index 000000000000..7bd17d901308
--- /dev/null
+++ b/doc/man3/idset_alloc.rst
@@ -0,0 +1,85 @@
+==============
+idset_alloc(3)
+==============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/idset.h>
+
+   int idset_alloc (struct idset *idset, unsigned int *val);
+
+   void idset_free (struct idset *idset, unsigned int val);
+
+   int idset_free_check (struct idset *idset, unsigned int val);
+
+Link with :command:`-lflux-idset`.
+
+
+DESCRIPTION
+===========
+
+Refer to :man3:`idset_create` for a general description of idsets.
+
+These functions are useful when using an idset as an integer allocator.
+The idset must have been created with IDSET_FLAG_INITFULL.
+
+.. note::
+  Unallocated is defined as "in the set" so that allocation can use the
+  constant-time *first* operation to find the next available id.  Defining
+  unallocated as "not in the set" would mean that iteration would
+  be required to find the next available id.
+
+:func:`idset_alloc` takes the next available id out of the set.
+This is implemented as :func:`idset_first` and :func:`idset_clear` internally.
+If there are no more ids available and the set was created with
+IDSET_FLAG_AUTOGROW, the set is expanded in order to fulfill the request.
+
+:func:`idset_free` puts an id back in the set.  This is implemented
+as :func:`idset_set` internally.
+
+:func:`idset_free_check` is identical to the above, except it fails
+if the id is already in the set.  This is implemented as :func:`idset_test`
+and :func:`idset_set` internally.
+
+RETURN VALUE
+============
+
+:func:`idset_alloc` and :func:`idset_free_check` return 0 on success, or
+-1 on error with :var:`errno` set.
+
+
+ERRORS
+======
+
+EINVAL
+   One or more arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+EEXIST
+   :func:`idset_free_check` was called on an id that is already in the
+   idset.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_22`
+
+
+SEE ALSO
+========
+
+:man3:`idset_create`, :man3:`idset_encode`, :man3:`idset_add`
diff --git a/doc/man3/idset_create.adoc b/doc/man3/idset_create.adoc
deleted file mode 100644
index 97f41495277e..000000000000
--- a/doc/man3/idset_create.adoc
+++ /dev/null
@@ -1,171 +0,0 @@
-idset_create(3)
-===============
-:doctype: manpage
-
-
-NAME
-----
-idset_create, idset_destroy, idset_encode, idset_decode, idset_set, idset_clear, idset_first, idset_next, idset_count, idset_equal - Manipulate numerically sorted sets of non-negative integers
-
-SYNOPSIS
---------
- #include <flux/idset.h>
-
- struct idset *idset_create (size_t slots, int flags);
-
- void idset_destroy (struct idset *idset);
-
- struct idset *idset_copy (const struct idset *idset);
-
- struct idset *idset_decode (const char *s);
-
- char *idset_encode (const struct idset *idset, int flags);
-
- int idset_set (struct idset *idset, unsigned int id);
-
- int idset_range_set (struct idset *idset,
-                      unsigned int lo, unsigned int hi);
-
- int idset_clear (struct idset *idset, unsigned int id);
-
- int idset_range_clear (struct idset *idset,
-                        unsigned int lo, unsigned int hi)
-
- bool idset_test (const struct idset *idset, unsigned int id);
-
- unsigned int idset_first (const struct idset *idset);
-
- unsigned int idset_next (const struct idset *idset, unsigned int prev);
-
- unsigned int idset_last (const struct idset *idset)
-
- size_t idset_count (const struct idset *idset);
-
- bool idset_equal (const struct idset *set1, const struct idset *set2);
-
-
-USAGE
------
-
-cc [flags] files -lflux-idset [libraries]
-
-DESCRIPTION
------------
-
-An idset is a set of numerically sorted, non-negative integers.
-It is internally represented as a van Embde Boas (or vEB) tree.
-Functionally it behaves like a bitmap, and has space efficiency
-comparable to a bitmap, but performs operations (insert, delete,
-lookup, findNext, findPrevious) in O(log(m)) time, where pow (2,m)
-is the number of slots in the idset.
-
-`idset_create()` creates an idset.  'slots' specifies the highest
-numbered 'id' it can hold, plus one.  The size is fixed unless
-'flags' specify otherwise (see FLAGS below).
-
-`idset_destroy()` destroys an idset.
-
-`idset_copy()` copies an idset.
-
-`idset_decode ()` creates an idset from a string 's'.  The string may
-have been produced by `idset_encode()`.  It must consist of comma-separated
-non-negative integer ids, and may also contain hyphenated ranges.
-If enclosed in square brackets, the brackets are ignored.  Some examples
-of valid input strings are:
-
-  1,2,5,4
-
-  1-4,7,9-10
-
-  42
-
-  [99-101]
-
-`idset_encode()` creates a string from 'idset'.  The string contains
-a comma-separated list of ids, potentially modified by 'flags'
-(see FLAGS below).
-
-`idset_set()` and `idset_clear()` set or clear 'id'.
-
-`idset_range_set()` and `idset_range_clear()` set or clear an inclusive
-range of ids, from 'lo' to 'hi'.
-
-`idset_test()` returns true if 'id' is set, false if not.
-
-`idset_first()` and `idset_next()` can be used to iterate over ids
-in the set, returning IDSET_INVALID_ID at the end.  `idset_last()`
-returns the last (highest) id, or IDSET_INVALID_ID if the set is
-empty.
-
-`idset_count()` returns the number of ids in the set.
-
-`idset_equal()` returns true if the two idset objects 'set1' and 'set2'
-are equal sets, i.e. the sets contain the same set of integers.
-
-
-FLAGS
------
-
-IDSET_FLAG_AUTOGROW::
-Valid for `idset_create()` only.  If set, the idset will grow to
-accommodate any id inserted into it. The internal vEB tree is doubled
-in size until until the new id can be inserted.  Resizing is a costly
-operation that requires all ids in the old tree to be inserted into
-the new one.
-
-IDSET_FLAG_BRACKETS::
-Valid for `idset_encode()` only.  If set, the encoded string will be
-enclosed in brackets, unless the idset is a singleton (contains only
-one id).
-
-IDSET_FLAG_RANGE::
-Valid for `idset_encode()` only.  If set, any consecutive ids are
-compressed into hyphenated ranges in the encoded string.
-
-RETURN VALUE
-------------
-
-`idset_create()`, `idset_encode()`, and `idset_copy()` return an
-idset on success which must be freed with `idset_destroy()`.
-On error, NULL is returned with errno set.
-
-`idset_decode()` returns a string on success which must be freed
-with `free()`.  On error, NULL is returned with errno set.
-
-`idset_first()`, `idset_next()`, and `idset_last()` return an id,
-or IDSET_INVALID_ID if no id is available.
-
-`idset_equal()` returns true if 'set1' and 'set2' are equal sets,
-or false if they are not equal, or either argument is 'NULL'.
-
-Other functions return 0 on success, or -1 on error with errno set.
-
-ERRORS
-------
-
-EINVAL::
-One or more arguments were invalid.
-
-ENOMEM::
-Out of memory.
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-
-https://github.com/flux-framework/rfc/blob/master/spec_22.adoc[RFC 22: Idset String Representation]
diff --git a/doc/man3/idset_create.rst b/doc/man3/idset_create.rst
new file mode 100644
index 000000000000..cbb1987255bc
--- /dev/null
+++ b/doc/man3/idset_create.rst
@@ -0,0 +1,159 @@
+===============
+idset_create(3)
+===============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/idset.h>
+
+   struct idset *idset_create (size_t size, int flags);
+
+   void idset_destroy (struct idset *idset);
+
+   struct idset *idset_copy (const struct idset *idset);
+
+   int idset_set (struct idset *idset, unsigned int id);
+
+   int idset_range_set (struct idset *idset,
+                        unsigned int lo,
+                        unsigned int hi);
+
+   int idset_clear (struct idset *idset, unsigned int id);
+
+   int idset_range_clear (struct idset *idset,
+                          unsigned int lo,
+                          unsigned int hi);
+
+   bool idset_test (const struct idset *idset, unsigned int id);
+
+   unsigned int idset_first (const struct idset *idset);
+
+   unsigned int idset_next (const struct idset *idset,
+                            unsigned int id);
+
+   unsigned int idset_last (const struct idset *idset);
+
+   unsigned int idset_prev (const struct idset *idset,
+                            unsigned int id);
+
+   size_t idset_count (const struct idset *idset);
+
+   bool idset_empty (const struct idset *idset);
+
+   size_t idset_universe_size (const struct idset *idset);
+
+Link with :command:`-lflux-idset`.
+
+DESCRIPTION
+===========
+
+An idset is a set of numerically sorted, non-negative integers.
+It is internally represented as a van Embde Boas (or vEB) tree.
+Functionally it behaves like a bitmap, and has space efficiency
+comparable to a bitmap, but performs *test*, *set*, *clear*, *next*,
+and *prev* operations in :math:`O(log(m))` time (where :math:`2^m` is the
+universe size); and performs *first* and *last* operations in constant time.
+
+:func:`idset_create` creates an idset. :var:`size` specifies the universe
+size, which is the maximum *id* it can hold, plus one. The universe size is
+fixed unless :var:`flags` specify otherwise (see FLAGS below).
+
+:func:`idset_destroy` destroys an idset.
+
+:func:`idset_copy` copies an idset.
+
+:func:`idset_set` and :func:`idset_clear` set or clear :var:`id`.
+
+:func:`idset_range_set` and :func:`idset_range_clear` set or clear an inclusive
+range of ids, from :var:`lo` to :var:`hi`.
+
+:func:`idset_test` returns true if :var:`id` is set, false if not.
+
+:func:`idset_first` and :func:`idset_next` can be used to iterate forward
+over ids in the set, returning IDSET_INVALID_ID at the end.
+
+:func:`idset_last` and :func:`idset_prev` can be used to iterate backward
+over ids in the set, returning IDSET_INVALID_ID at the end.
+
+:func:`idset_count` returns the number of ids in the set.  A running count
+is kept so this function runs in constant time.
+
+:func:`idset_empty` returns true if the set is empty.  This function runs
+in constant time.
+
+:func:`idset_universe_size` returns the current set universe size.
+This is normally the size specified at creation, or a multiple of it if
+IDSET_FLAG_AUTOGROW was specified.
+
+FLAGS
+=====
+
+The following flags are valid for :func:`idset_create`:
+
+IDSET_FLAG_AUTOGROW
+   The idset will grow to accommodate any id that is the target of a set, or
+   if IDSET_FLAG_INITFULL is set, a clear operation.  The universe size is
+   doubled until until the new id can be accessed.  Resizing is a costly
+   operation that requires all ids in the old tree to be inserted into the
+   new one.
+
+IDSET_FLAG_INITFULL
+   The idset is created full instead of empty.  If specified with
+   IDSET_FLAG_AUTOGROW, new portions that are added are also filled.
+
+IDSET_FLAG_COUNT_LAZY
+   The running count is not maintained and :func:`idset_count` uses a slower
+   iteration method.  Not maintaining the count makes set/clear operations
+   slightly faster, an acceptable trade-off for some use cases.  This flag does
+   not affect :func:`idset_empty`.
+
+
+RETURN VALUE
+============
+
+:func:`idset_create` and :func:`idset_copy` return an idset on success which
+must be freed with :func:`idset_destroy`. On error, NULL is returned with
+:var:`errno` set.
+
+:func:`idset_first`, :func:`idset_next`, :func:`idset_prev`, and
+:func:`idset_last` return an id, or IDSET_INVALID_ID if no id is available.
+
+:func:`idset_count` and :func:`idset_universe_size` return 0 if the argument
+is invalid.
+
+:func:`idset_empty` returns true for the empty set or invalid arguments.
+
+Other functions return 0 on success, or -1 on error with :var:`errno` set.
+
+
+ERRORS
+======
+
+EINVAL
+   One or more arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_22`
+
+
+SEE ALSO
+========
+
+:man3:`idset_encode`, :man3:`idset_add`, :man3:`idset_alloc`
diff --git a/doc/man3/idset_decode.rst b/doc/man3/idset_decode.rst
new file mode 100644
index 000000000000..9b775cd0e331
--- /dev/null
+++ b/doc/man3/idset_decode.rst
@@ -0,0 +1,148 @@
+===============
+idset_decode(3)
+===============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/idset.h>
+
+   typedef struct {
+      char text[160];
+   } idset_error_t;
+
+   struct idset *idset_decode (const char *s);
+
+   struct idset *idset_decode_ex (const char *s,
+                                  ssize_t len,
+                                  ssize_t size,
+                                  int flags,
+                                  idset_error_t *error);
+
+   bool idset_decode_empty (const char *s, ssize_t len);
+
+   int idset_decode_info (const char *s,
+                          ssize_t len,
+                          size_t *count,
+                          unsigned int *maxid,
+                          idset_error_t *error);
+
+   int idset_decode_add (struct idset *idset,
+                         const char *s,
+                         ssize_t len,
+                         idset_error_t *error);
+
+   int idset_decode_subtract (struct idset *idset,
+                              const char *s,
+                              ssize_t len,
+                              idset_error_t *error);
+
+Link with :command:`-lflux-idset`.
+
+DESCRIPTION
+===========
+
+Refer to :man3:`idset_create` for a general description of idsets.
+
+:func:`idset_decode` creates an idset from a string :var:`s`. The string may
+have been produced by :func:`idset_encode`. It must consist of comma-separated
+non-negative integer ids, and may also contain hyphenated ranges.
+If enclosed in square brackets, the brackets are ignored. Some examples
+of valid input strings are:
+
+::
+
+   1,2,5,4
+   1-4,7,9-10
+   42
+   [99-101]
+
+:func:`idset_decode_ex` creates an idset from a string :var:`s` optionally
+truncated at :var:`len` bytes.  If :var:`len` is -1, the full, NUL-terminated
+string length is parsed.  The idset is created using the specified :var:`flags`
+and :var:`size`.  If :var:`size` is -1, the idset is made to exactly fit the
+largest id specified in :var:`s`.  If :var:`size` is 0, a default size is used.
+
+The following functions parse an idset string without creating an idset:
+
+:func:`idset_decode_empty` parses :var:`s` optionally truncated at :var:`len`
+and returns true if it is the empty set, or false if it is not the empty set
+or cannot be parsed.
+
+:func:`idset_decode_info` parses :var:`s` optionally truncated at :var:`len`
+bytes to determine the idset :var:`count` and :var:`maxid`.  Either of the
+two output parameters may be set to NULL to suppress assignment.
+
+:func:`idset_decode_add` parses :var:`s` optionally truncated at :var:`len`
+bytes and adds members of the resulting set to :var:`idset`.
+
+:func:`idset_decode_subtract` parses :var:`s` optionally truncated at
+:var:`len` bytes and removes members of the resulting set from :var:`idset`.
+
+
+CAVEATS
+=======
+
+:func:`idset_decode_ex` without IDSET_FLAGS_AUTOGROW fails when presented
+with an empty set because an idset cannot be created with zero slots.
+In situations where the set should not grow automatically and the empty
+set is valid, it is recommended to represent the empty set with NULL,
+and use :func:`idset_decode_empty` to distinguish it from a parse error, e.g.
+
+.. code-block:: c
+
+    struct idset *idset;
+    idset_error_t error;
+
+    if (idset_decode_empty (input, -1))
+        idset = NULL;
+    else if (!(idset = idset_decode_ex (input, -1, -1, 0, &error)))
+        // handle parse error
+
+
+RETURN VALUE
+============
+
+:func:`idset_decode_empty` returns a boolean value.
+
+:func:`idset_decode` and :func:`idset_decode_ex` return an idset on success
+which must be freed with :man3:`idset_destroy`.  On failure, they return NULL
+with :var:`errno` set.  In addition, :func:`idset_decode_ex` sets :var:`error`
+on failure, if non-NULL.
+
+:func:`idset_decode_empty` returns a boolean value.
+
+:func:`idset_decode_add` and :func:`idset_decode_subtract` return 0 on success,
+or -1 on failure with :var:`errno` and :var:`error` (if non-NULL) set.
+
+
+ERRORS
+======
+
+EINVAL
+   One or more arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_22`
+
+
+SEE ALSO
+========
+
+:man3:`idset_create`
diff --git a/doc/man3/idset_encode.rst b/doc/man3/idset_encode.rst
new file mode 100644
index 000000000000..56a4a0d165a3
--- /dev/null
+++ b/doc/man3/idset_encode.rst
@@ -0,0 +1,72 @@
+===============
+idset_encode(3)
+===============
+
+.. default-domain:: c
+
+SYNOPSIS
+========
+
+.. code-block:: c
+
+   #include <flux/idset.h>
+
+   char *idset_encode (const struct idset *idset, int flags);
+
+Link with :command:`-lflux-idset`.
+
+DESCRIPTION
+===========
+
+Refer to :man3:`idset_create` for a general description of idsets.
+
+:func:`idset_encode` creates a string from :var:`idset`. The string contains
+a comma-separated list of ids, potentially modified by :var:`flags`
+(see FLAGS below).
+
+
+FLAGS
+=====
+
+IDSET_FLAG_BRACKETS
+   If set, the encoded string will be enclosed in brackets, unless the idset
+   is a singleton (contains only one id).
+
+IDSET_FLAG_RANGE
+   If set, any consecutive ids are compressed into hyphenated ranges in the
+   encoded string.
+
+
+RETURN VALUE
+============
+
+:func:`idset_encode` returns a string on success which must be freed
+with :linux:man3:`free`. On error, NULL is returned with :var:`errno` set.
+
+
+ERRORS
+======
+
+EINVAL
+   One or more arguments were invalid.
+
+ENOMEM
+   Out of memory.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_22`
+
+
+SEE ALSO
+========
+
+:man3:`idset_create`
diff --git a/doc/man3/index.rst b/doc/man3/index.rst
new file mode 100644
index 000000000000..11890b20e6b1
--- /dev/null
+++ b/doc/man3/index.rst
@@ -0,0 +1,8 @@
+Section 3 - Flux C API
+======================
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   *
diff --git a/doc/man3/tevent.c b/doc/man3/tevent.c
deleted file mode 100644
index 14c97639cdab..000000000000
--- a/doc/man3/tevent.c
+++ /dev/null
@@ -1,24 +0,0 @@
-#include <flux/core.h>
-#include "src/common/libutil/log.h"
-
-int main (int argc, char **argv)
-{
-    flux_t *h;
-    flux_msg_t *msg;
-    const char *topic;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (flux_event_subscribe (h, "hb") < 0)
-        log_err_exit ("flux_event_subscribe");
-    if (!(msg = flux_recv (h, FLUX_MATCH_EVENT, 0)))
-        log_err_exit ("flux_recv");
-    if (flux_msg_get_topic (msg, &topic) < 0)
-        log_err_exit ("flux_msg_get_topic");
-    printf ("Event: %s\n", topic);
-    if (flux_event_unsubscribe (h, "hb") < 0)
-        log_err_exit ("flux_event_unsubscribe");
-    flux_msg_destroy (msg);
-    flux_close (h);
-    return (0);
-}
diff --git a/doc/man3/tinfo.c b/doc/man3/tinfo.c
deleted file mode 100644
index dbc22acf414f..000000000000
--- a/doc/man3/tinfo.c
+++ /dev/null
@@ -1,19 +0,0 @@
-#include <math.h>
-#include <flux/core.h>
-#include <inttypes.h>
-#include "src/common/libutil/log.h"
-
-int main (int argc, char **argv)
-{
-    flux_t *h;
-    uint32_t rank, n;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (flux_get_rank (h, &rank) < 0)
-        log_err_exit ("flux_get_rank");
-    if (flux_get_size (h, &n) < 0)
-        log_err_exit ("flux_get_size");
-    flux_close (h);
-    return (0);
-}
diff --git a/doc/man3/topen.c b/doc/man3/topen.c
deleted file mode 100644
index 4e7653f6511a..000000000000
--- a/doc/man3/topen.c
+++ /dev/null
@@ -1,17 +0,0 @@
-#include <inttypes.h>
-#include <flux/core.h>
-#include "src/common/libutil/log.h"
-
-int main (int argc, char **argv)
-{
-    flux_t *h;
-    uint32_t rank;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (flux_get_rank (h, &rank) < 0)
-        log_err_exit ("flux_get_rank");
-    printf ("My rank is %"PRIu32"\n", rank);
-    flux_close (h);
-    return (0);
-}
diff --git a/doc/man3/trecv.c b/doc/man3/trecv.c
deleted file mode 100644
index 0f690667e0e0..000000000000
--- a/doc/man3/trecv.c
+++ /dev/null
@@ -1,24 +0,0 @@
-#include <flux/core.h>
-#include "src/common/libutil/log.h"
-
-int main (int argc, char **argv)
-{
-    flux_t *h;
-    flux_msg_t *msg;
-    const char *topic;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (flux_event_subscribe (h, "") < 0)
-        log_err_exit ("flux_event_subscribe");
-    for (;;) {
-        if ((msg = flux_recv (h, FLUX_MATCH_EVENT, 0)))
-            log_err_exit ("flux_recv");
-        if (flux_msg_get_topic (msg, &topic) < 0)
-            log_err_exit ("flux_msg_get_topic");
-        printf ("Event: %s\n", topic);
-        flux_msg_destroy (msg);
-    }
-    flux_close (h);
-    return (0);
-}
diff --git a/doc/man3/trpc.c b/doc/man3/trpc.c
deleted file mode 100644
index 14aa47bb0d63..000000000000
--- a/doc/man3/trpc.c
+++ /dev/null
@@ -1,25 +0,0 @@
-#include <flux/core.h>
-#include "src/common/libutil/log.h"
-
-int main (int argc, char **argv)
-{
-    flux_t *h;
-    flux_future_t *f;
-    const char *rankstr;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (!(f = flux_rpc_pack (h, "attr.get", FLUX_NODEID_ANY, 0,
-		             "{s:s}", "name", "rank")))
-        log_err_exit ("flux_rpc_pack");
-
-    if (flux_rpc_get_unpack (f, "{s:s}", "value", &rankstr) < 0)
-        log_err_exit ("flux_rpc_get_unpack");
-
-    printf ("rank is %s\n", rankstr);
-    flux_future_destroy (f);
-
-    flux_close (h);
-    return (0);
-}
diff --git a/doc/man3/trpc_then.c b/doc/man3/trpc_then.c
deleted file mode 100644
index 9aae4dd65ea4..000000000000
--- a/doc/man3/trpc_then.c
+++ /dev/null
@@ -1,35 +0,0 @@
-#include <flux/core.h>
-#include "src/common/libutil/log.h"
-
-void continuation (flux_future_t *f, void *arg)
-{
-    const char *rankstr;
-
-    if (flux_rpc_get_unpack (f, "{s:s}", "value", &rankstr) < 0)
-        log_err_exit ("flux_rpc_get_unpack");
-
-    printf ("rank is %s\n", rankstr);
-    flux_future_destroy (f);
-}
-
-int main (int argc, char **argv)
-{
-    flux_t *h;
-    flux_future_t *f;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (!(f = flux_rpc_pack (h, "attr.get", FLUX_NODEID_ANY, 0,
-		             "{s:s}", "name", "rank")))
-        log_err_exit ("flux_rpc_pack");
-
-    if (flux_future_then (f, -1., continuation, NULL) < 0)
-        log_err_exit ("flux_future_then");
-
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
-        log_err_exit ("flux_reactor_run");
-
-    flux_close (h);
-    return (0);
-}
diff --git a/doc/man3/tsend.c b/doc/man3/tsend.c
deleted file mode 100644
index 633847007552..000000000000
--- a/doc/man3/tsend.c
+++ /dev/null
@@ -1,18 +0,0 @@
-#include <flux/core.h>
-#include "src/common/libutil/log.h"
-
-int main (int argc, char **argv)
-{
-    flux_t *h;
-    flux_msg_t *msg;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (!(msg = flux_event_encode ("snack.bar.closing", NULL)))
-        log_err_exit ("flux_event_encode");
-    if (flux_send (h, msg, 0) < 0)
-        log_err_exit ("flux_send");
-    flux_msg_destroy (msg);
-    flux_close (h);
-    return (0);
-}
diff --git a/doc/man5/COPYRIGHT.adoc b/doc/man5/COPYRIGHT.adoc
deleted file mode 100644
index 6b2304f95801..000000000000
--- a/doc/man5/COPYRIGHT.adoc
+++ /dev/null
@@ -1,4 +0,0 @@
-Copyright 2014 Lawrence Livermore National Security, LLC
-and Flux developers.
-
-SPDX-License-Identifier: LGPL-3.0
diff --git a/doc/man5/Makefile.am b/doc/man5/Makefile.am
deleted file mode 100644
index b9bedbef294e..000000000000
--- a/doc/man5/Makefile.am
+++ /dev/null
@@ -1,27 +0,0 @@
-MAN5_FILES = \
-	flux-config-bootstrap.5
-
-ADOC_FILES  = $(MAN5_FILES:%.5=%.adoc)
-XML_FILES   = $(MAN5_FILES:%.5=%.xml)
-
-if ENABLE_DOCS
-dist_man_MANS = $(MAN5_FILES)
-$(MAN5_FILES): COPYRIGHT.adoc
-endif
-
-SUFFIXES = .adoc .5
-
-STDERR_DEVNULL = $(stderr_devnull_$(V))
-stderr_devnull_ =  $(stderr_devnull_$(AM_DEFAULT_VERBOSITY))
-stderr_devnull_0 = 2>/dev/null
-
-.adoc.5:
-	$(AM_V_GEN)$(ADOC) --attribute mansource=$(PACKAGE_NAME) \
-	    --attribute manversion=$(PACKAGE_VERSION) \
-	    --attribute manmanual="Flux Miscellaneous Reference" \
-	    --destination-dir $(builddir) \
-	    --doctype manpage $(ADOC_FORMAT_OPT) manpage $< $(STDERR_DEVNULL)
-
-EXTRA_DIST = $(ADOC_FILES) COPYRIGHT.adoc
-
-CLEANFILES = $(MAN5_FILES) $(XML_FILES)
diff --git a/doc/man5/common/experimental.rst b/doc/man5/common/experimental.rst
new file mode 100644
index 000000000000..00be899898a2
--- /dev/null
+++ b/doc/man5/common/experimental.rst
@@ -0,0 +1,4 @@
+Experimental Flux features and interfaces are made available for evaluation
+only and may change or be removed without notice.
+
+Feedback is welcome.  Please use the flux-core project Github issue tracker.
diff --git a/doc/man5/common/resources.rst b/doc/man5/common/resources.rst
new file mode 100644
index 000000000000..dcc16e8954b0
--- /dev/null
+++ b/doc/man5/common/resources.rst
@@ -0,0 +1,5 @@
+Flux: http://flux-framework.org
+
+Flux RFC: https://flux-framework.readthedocs.io/projects/flux-rfc
+
+Issue Tracker: https://github.com/flux-framework/flux-core/issues
diff --git a/doc/man5/flux-config-access.rst b/doc/man5/flux-config-access.rst
new file mode 100644
index 000000000000..6b7526e43d0c
--- /dev/null
+++ b/doc/man5/flux-config-access.rst
@@ -0,0 +1,50 @@
+=====================
+flux-config-access(5)
+=====================
+
+
+DESCRIPTION
+===========
+
+Flux normally denies access to all users except the instance owner (the user
+running the Flux instance).  The system instance, however, runs as the
+``flux`` user and permits limited access to guests, such as submitting work
+and manipulating their own jobs.
+
+The ``access`` table is required for a multi-user Flux system instance and may
+contain the following keys:
+
+
+KEYS
+====
+
+allow-guest-user
+   (optional) Boolean value to allow guest users to connect and assigns them
+   the ``user`` role.  If set to false or not present, connection attempts
+   by guests fail with EPERM.
+
+allow-root-owner
+   (optional) Boolean value to assign ``owner`` role to root user.  If set to false
+   or not present, root is treated like any other guest.
+
+
+EXAMPLE
+=======
+
+::
+
+   [access]
+   allow-guest-user = true
+   allow-root-owner = true
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`
diff --git a/doc/man5/flux-config-bootstrap.adoc b/doc/man5/flux-config-bootstrap.adoc
deleted file mode 100644
index 6d7bf54bc23d..000000000000
--- a/doc/man5/flux-config-bootstrap.adoc
+++ /dev/null
@@ -1,121 +0,0 @@
-flux-config-bootstrap(5)
-========================
-:doctype: manpage
-
-
-NAME
-----
-flux-config-bootstrap - configure Flux instance bootstrap
-
-
-DESCRIPTION
------------
-
-The broker discovers the size of the Flux instance, the broker's rank,
-and overlay network wireup information either dynamically using a PMI
-service, such as when being launched by Flux or another resource manager,
-or statically using the `bootstrap` section of the Flux configuration,
-such as when being launched by systemd.
-
-The default bootstrap mode is PMI.  The Flux systemd unit file forces the
-broker to use the config method by specifying `--setattr=boot.method=config`
-on the broker command line.
-
-CONFIG FILES
-------------
-
-Flux uses the TOML configuration file format.  The broker normally
-parses `/etc/flux/conf.d/*.toml`.  The actual path is dependent on
-`configure` command line options used to build flux, and can be
-overridden with the FLUX_CONF_DIR environment variable.
-
-The `bootstrap` section is a TOML table containing the following
-keys.  Each node in a cluster is expected to bootstrap from an
-identical config file.
-
-KEYWORDS
---------
-
-default_port::
-(optional) The value is an integer port number that is substituted
-for the token `%p` in the other keys.
-
-default_bind::
-(optional) The value is a ZeroMQ endpoint URI that is used for host
-entries that do not explicitly set a bind address.  The tokens
-`%p` and `%h` are replaced with the default port and the host
-for the current host entry.
-
-default_connect::
-(optional) The value is a ZeroMQ endpoint URI that is used for host
-entries that do not explicitly set a connect address.  The tokens
-`%p` and `%h` are replaced with the default port and the host
-for the current host entry.
-
-hosts::
-(optional) A rank-ordered array of host entries.  Each host entry is
-a TOML table containing at minimum the `host` key.  The broker determines
-its rank by matching its hostname in the hosts array and taking the array
-index.  An empty or missing hosts array implies a standalone (single
-broker) instance.  The entry for a broker with downstream peers must
-either assign the `bind` key to a ZeroMQ endpoint URI, or the `default_bind`
-URI described above is used.  The entry for a broker with downstream peers
-must also either assign the `connect` key to a ZeroMQ endpoint URI, or
-the `default_connect` URI described above is used.  The same `%h` and `%p`
-substitutions work here as well.
-
-COMPACT HOSTS
--------------
-
-Since it would be tedious to repeat host entries for every compute node in
-a large cluster, the `hosts` array may be abbreviated using bracketed
-"idset" notation in `host` keys.
-
-An idset is an unordered set of non-negative integers that may be expressed
-as a comma-separated list including hyphenated ranges.  For example
-the set 0, 1, 2, 3, 4, 18, 20 may be represented as "0-4,18,20".
-
-A `host` key may include one or more bracketed idsets.  For example,
-"foo[0-1023]" represents the hosts "foo0, foo1, ..., foo1023", or
-"rack[0-1]node[0-1]" represents the hosts "rack0node0, rack0node1,
-rack1node0, rack1node1".
-
-EXAMPLE
--------
-
-....
-[bootstrap]
-
-default_port = 8050
-default_bind = "tcp://en0:%p"
-default_connect = "tcp://e%h:%p"
-
-hosts = [
-    {
-        host="fluke0",
-        bind="tcp://en4:9001",
-        connect="tcp://fluke-mgmt:9001"
-    },
-    { host = "fluke[1-1023]" },
-]
-....
-
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-flux-getattr(1), flux_attr_get(3)
diff --git a/doc/man5/flux-config-bootstrap.rst b/doc/man5/flux-config-bootstrap.rst
new file mode 100644
index 000000000000..f12dd2c3c301
--- /dev/null
+++ b/doc/man5/flux-config-bootstrap.rst
@@ -0,0 +1,249 @@
+========================
+flux-config-bootstrap(5)
+========================
+
+
+DESCRIPTION
+===========
+
+The system instance requires that the overlay network configuration be
+statically configured.  The ``bootstrap`` TOML table defines these details
+for a given cluster.
+
+.. warning::
+   Although ``flux config reload`` works on a live system, bootstrap
+   settings do not take effect until the next broker restart.  As such, they
+   must only be changed in conjunction with a full system instance restart in
+   order to avoid brokers becoming desynchronized if they are independently
+   restarted before the next instance restart.
+
+Flux brokers are interconnected in a tree topology.  A broker has zero or one
+upstream peer (towards the root, rank 0) and zero or more downstream peers
+(towards the leaves).  A broker passively accepts connections from its
+downstream peers on its ZeroMQ `bind` endpoint , and actively connects to its
+upstream peer on that broker's ZeroMQ `connect` endpoint, which is the
+remote version of the peer's `bind` endpoint.
+
+A broker determines the ranks of its peers based on the topology.  When
+bootstrapping from configuration files, the default topology is ``custom``.
+In a ``custom`` topology, rank 0 is the upstream peer of all other ranks,
+unless ``parent`` keys are present in the ``hosts`` array to define a
+different tree shape.  The default topology may be altered from ``custom``
+by configuring ``tbon.topo`` as described in :man5:`flux-config-tbon`.
+
+A broker determines its own rank by looking for its hostname in the ``hosts``
+array.  The index of the first matching entry is the broker's rank.  The
+information in the ``hosts`` array also provides
+ - the `bind` endpoint that the broker configures to accept connections
+   from downstream peers
+ - the `connect` endpoint it uses to connect to its parent
+
+Each point to point connection between brokers is authenticated and encrypted
+using ZeroMQ native CURVE cryptography.  This requires a shared certificate
+to bootstrap.  This certificate is stored on disk and must be protected from
+access by users other than the Flux instance owner.
+
+The ``bootstrap`` table contains the following keys:
+
+
+KEYS
+====
+
+enable_ipv6
+   (optional) Boolean value for enabling IPv6.  By default only IPv4 is
+   enabled.  Note that setting this to true prevents binding to a named
+   interface that only supports IPv4.
+
+curve_cert
+   (optional) Path to a CURVE certificate generated with
+   :man1:`flux-keygen`.  The certificate should be identical on all
+   broker ranks.  It is required for instance sizes > 1.  The file should
+   be owned by the instance owner (e.g. `flux` user) and only readable by
+   that user.
+
+default_port
+   (optional) The value is an integer port number that is substituted
+   for the token ``%p`` in the other keys.
+
+default_bind
+   (optional) The value is a ZeroMQ endpoint URI that is used for host
+   entries that do not explicitly set a bind address. The tokens
+   ``%p`` and ``%h`` are replaced with the default port and the host
+   for the current host entry.
+
+default_connect
+   (optional) The value is a ZeroMQ endpoint URI that is used for host
+   entries that do not explicitly set a connect address. The tokens
+   ``%p`` and ``%h`` are replaced with the default port and the host
+   for the current host entry.
+
+hosts
+   (optional) A rank-ordered array of host entries. Each entry is a TOML
+   table containing, at minimum, a ``host`` key and optionally, ``connect``
+   and ``bind`` keys.  These keys are described in detail below. The array
+   is required to be defined for instance sizes > 1.
+
+HOSTS ENTRY
+===========
+
+Each ``hosts`` array entry contains the following keys:
+
+host
+   (required) The name of the host that corresponds to this entry.  The value
+   must exactly match the output of the :linux:man1:`hostname` command.
+
+connect
+   (optional) The ZeroMQ endpoint that downstream peers use to connect to this
+   broker.  The value may contain ``%h`` and ``%p`` tokens as described under
+   ``default_connect`` above.  If the key is omitted, ``default_connect``
+   is used.
+
+bind
+   (optional) The ZeroMQ endpoint that this broker uses to accept connections
+   from downstream peers.  The value may contain ``%h`` and ``%p`` tokens as
+   described under ``default_bind`` above.  If the key is omitted,
+   ``default_bind`` is used.
+
+parent
+    (optional) The name of the host that is the upstream peer of this entry
+    in the overlay network tree topology.
+
+ZEROMQ ENDPOINTS
+================
+
+Brokers use the :linux:man7:`zmq_tcp` transport, consisting of the transport
+name ``tcp://`` followed by an address.
+
+`Bind` addresses specify interface followed by a colon and the TCP port.
+The interface may be one of:
+
+- the wild-card ``*`` meaning all available interfaces
+
+- the primary IP address assigned to the interface (numeric only)
+
+- the interface name
+
+The port should be an explicit numerical port number.
+
+`Connect` addresses specify a peer address followed by a colon and the TCP port.
+The peer address may be one of:
+
+- the DNS name of the peer
+
+- the IP address of the peer in its numeric representation
+
+When specifying the ``bind`` and ``connect`` URIs for a hosts entry, ensure
+that another host can use the ``connect`` URI to reach the Flux service bound
+to the ``bind`` address on the host.
+
+
+COMPACT HOSTS
+=============
+
+Since it would be tedious to repeat host entries for every compute
+node in a large cluster, the ``hosts`` array may be abbreviated using
+RFC 29 hostlists.  For example, the list of hosts foo0, foo1, foo2,
+foo3, foo18, foo4, foo20 can be represented as "foo[0-3,18,4,20]".
+
+
+EXAMPLE
+=======
+
+The following example is a simple, two node cluster with a fully specified
+``hosts`` array.
+
+::
+
+   [bootstrap]
+
+   curve_cert = "/etc/flux/system/curve.cert"
+
+   hosts = [
+       { host="foo", bind="tcp://eth0:9001", connect="tcp://10.0.1.1:9001" },
+       { host = "bar" },
+   ]
+
+
+Host ``foo`` is assigned rank 0, and binds to the interface ``eth0`` port 9001.
+
+Host ``bar`` is assigned rank 1, and connects to ``10.0.1.1`` port 9001.
+
+The following example represents a 256 node cluster.  The management node has
+a different network interface configuration compared to its peers.
+
+::
+
+   [bootstrap]
+
+   curve_cert = "/etc/flux/system/curve.cert"
+
+   default_port = 8050
+   default_bind = "tcp://en0:%p"
+   default_connect = "tcp://e%h:%p"
+
+   hosts = [
+       # Management requires non-default config
+       { host="test0", bind="tcp://en4:9001", connect="tcp://test-mgmt:9001" },
+
+       # Other nodes use defaults
+       { host = "test[1-255]" },
+   ]
+
+The following example is a 256 node cluster that uses the ``parent`` key to
+create a tree topology with three levels.
+
+::
+
+   [bootstrap]
+
+   curve_cert = "/etc/flux/system/curve.cert"
+
+   default_port = 8050
+   default_bind = "tcp://en0:%p"
+   default_connect = "tcp://e%h:%p"
+
+   [[bootstrap.hosts]]
+   host = "test[0-255]"
+   [[bootstrap.hosts]]
+   host = "test[1,128]"
+   parent = "test0"
+   [[bootstrap.hosts]]
+   host = "test[2-127]"
+   parent = "test1"
+   [[bootstrap.hosts]]
+   host = "test[129-255]"
+   parent = "test128"
+
+
+Note that the first block of hosts defines the entire tree, with nodes test0 through test255. The second block has a comma instead of a dash, indicating a group of two members (1 and 128) and not a range. The resulting tree looks like this. The numbers on the left indicate the level of the tree.
+
+::
+
+        ┌───────┐                          
+    1)  │ test0 │                          
+        └───┬───┴──────────┐               
+            │              │               
+        ┌───┴───┐     ┌────┴────┐          
+    2)  │ test1 │     │ test128 │          
+        └───┬───┘     └────┬────┘          
+            │              │               
+        ┌───┴─────────┐   ┌┴──────────────┐
+    3)  │ test[2-127] │   │ test[129-255] │
+        └─────────────┘   └───────────────┘
+    
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_29`
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`
diff --git a/doc/man5/flux-config-exec.rst b/doc/man5/flux-config-exec.rst
new file mode 100644
index 000000000000..fc1320f4aafd
--- /dev/null
+++ b/doc/man5/flux-config-exec.rst
@@ -0,0 +1,137 @@
+===================
+flux-config-exec(5)
+===================
+
+
+DESCRIPTION
+===========
+
+The Flux system instance **job-exec** service requires additional
+configuration via the ``exec`` table, for example to enlist the services
+of a setuid helper to launch jobs as guests.
+
+The ``exec`` table may contain the following keys:
+
+
+KEYS
+====
+
+imp
+   (optional) Set the path to the IMP (Independent Minister of Privilege)
+   helper program, as described in RFC 15, so that jobs may be launched with
+   the credentials of the guest user that submitted them.  If unset, only
+   jobs submitted by the instance owner may be executed.
+
+service
+   (optional) Set the remote subprocess service name. (Default: ``rexec``).
+   Note that ``systemd.enable`` must be set to ``true`` if ``sdexec`` is
+   configured.  See :man5:`flux-config-systemd`.
+
+service-override
+   (optional) Allow ``service`` to be overridden on a per-job basis with
+   ``--setattr system.exec.bulkexec.service=NAME``.  (Default: ``false``).
+
+job-shell
+   (optional) Override the compiled-in default job shell path.
+
+sdexec-properties
+   (optional) A table of systemd properties to set for all jobs.  All values
+   must be strings.  See SDEXEC PROPERTIES below.
+
+kill-timeout
+   (optional) The amount of time to wait after ``SIGTERM`` is sent to a job
+   before sending ``SIGKILL``.
+
+term-signal
+   (optional) Specify an alternate signal to ``SIGTERM`` when terminating
+   job tasks. Mainly used for testing.
+
+kill-signal
+   (optional) Specify an alternate signal to ``SIGKILL`` when killing tasks
+   and the job shell. Mainly used for testing.
+
+barrier-timeout
+   (optional) Specify the default job shell start barrier timeout in FSD.
+   All multi-node jobs enter a barrier at startup once the Flux job shell
+   completes initialization tasks such as changing the working directory
+   and processing the initrc file. Once the first node enters this barrier,
+   the job execution system starts a timer, and if the timer expires
+   before the barrier is complete, raises a job exception and drains the
+   nodes on which the barrier is waiting.  To disable the barrier timeout,
+   set this value to ``"0"``. (Default: ``30m``).
+
+testexec
+   (options) A table of keys (see :ref:`testexec`) for configuring the
+   **job-exec** test execution implementation (used in mainly for testing).
+
+
+SDEXEC PROPERTIES
+=================
+
+When the sdexec service is selected, The following systemd unit properties may
+be set by adding them to the ``sdexec-properties`` table:
+
+MemoryMax
+   Specify the absolute limit on memory used by the job, in bytes. The value
+   may be suffixed with K, M, G or T, to multiply by Kilobytes, Megabytes,
+   Gigabytes, or Terabytes (base 1024), respectively. Alternatively, a
+   percentage of physical memory may be specified.  If assigned the special
+   value "infinity", no memory limit is applied.
+
+MemoryHigh
+   Specify the throttling limit on memory used by the job.  Values are
+   formatted as described above.
+
+MemoryMin, MemoryLow
+   Specify the memory usage protection of the job.  Values are formatted as
+   described above.
+
+
+.. _testexec:
+
+TESTEXEC
+========
+
+allow-guests
+  Boolean value enables access to the testexec implementation from guest
+  users. By default, guests cannot use this implementation.
+
+EXAMPLES
+========
+
+::
+
+   [exec]
+   imp = "/usr/libexec/flux/flux-imp"
+   job-shell = "/usr/libexec/flux/flux-shell-special"
+
+::
+
+   [exec]
+   service = "sdexec"
+   [exec.sdexec-properties]
+   MemoryMax = "90%"
+
+::
+
+   [exec.testexec]
+   allow-guests = true
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_15`
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`,
+`systemd.resource-control(5) <https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html>`_
diff --git a/doc/man5/flux-config-ingest.rst b/doc/man5/flux-config-ingest.rst
new file mode 100644
index 000000000000..9bbedf76cb54
--- /dev/null
+++ b/doc/man5/flux-config-ingest.rst
@@ -0,0 +1,138 @@
+=====================
+flux-config-ingest(5)
+=====================
+
+
+DESCRIPTION
+===========
+
+The Flux **job-ingest** service optionally modifies and validates job requests
+before announcing new jobs to the **job-manager**. Configuration of the
+**job-ingest** module can be accomplished via the ``ingest`` TOML table.
+See the KEYS section below for supported ``ingest`` table keys.
+
+The **job-ingest** module implements a two stage pipeline for job requests.
+The first stage modifies jobspec and is implemented as a work crew of
+``flux job-frobnicator`` processes.  The second stage validates the modified
+requests and is implemented as a work crew of ``flux job-validator`` processes.
+The frobnicator is disabled by default, and the validator is enabled by default.
+
+The frobnicator and validator each supports a set of plugins, and each plugin
+may consume additional arguments from the command line for specific
+configuration.  The plugins and any arguments are configured in the
+``ingest.frobnicator`` and ``ingest.validator`` TOML tables, respectively.
+See the FROBNICATOR KEYS and VALIDATOR KEYS sections for supported keys.
+
+KEYS
+====
+
+batch-count
+   (optional) The job-ingest module batches sets of jobs together
+   for efficiency. Normally this is done using a timer, but if the
+   ``batch-count`` key is nonzero then jobs are batched based on a counter
+   instead. This is mostly useful for testing.
+
+buffer-size
+   (optional) Set the input buffer size for job-ingest module workers.
+   The value is string indicating the buffer size with optional SI units
+   (e.g. "102400", "4.5M", "1024K") The default value is ``10M``.
+
+FROBNICATOR KEYS
+================
+
+disable
+   (optional) A boolean indicating whether to disable job frobnication,
+   usually for testing purposes.
+
+plugins
+   (optional) An array of frobnicator plugins to use.  The default value is
+   ``[ "defaults", "constraints" ]`` which are needed for assigning configured
+   jobspec defaults, and adding queue constraints, respectively.
+   For a list of supported plugins on your system run
+   ``flux job-frobnicator --list-plugins``
+
+args
+   (optional) An array of extra arguments to pass on the frobnicator
+   command line. Valid arguments can be found by running
+   ``flux job-frobnicator --plugins=LIST --help``
+
+FROBNICATOR PLUGIN CONFIGURATION
+--------------------------------
+
+Job frobnicator plugins optionally support plugin-specific configuration
+via keys based on the plugin name in the ``[ingest.frobnicator]``
+table. Frobnicator plugin keys match the plugin name, and define a table
+of keys specific to each plugin.
+
+No frobnicator plugins distributed with Flux support configuration at
+this time.
+
+
+VALIDATOR KEYS
+==============
+
+disable
+   (optional) A boolean indicating whether to disable job validation.
+   Disabling the job validator is not recommended, but may be useful
+   for testing or high job throughput scenarios.
+
+plugins
+   (optional) An array of validator plugins to use. The default
+   value is ``[ "jobspec" ]``, which uses the Python Jobspec class as
+   a validator.  For a list of supported plugins on your system run
+   ``flux job-validator --list-plugins``
+
+args
+   (optional) An array of extra arguments to pass on the validator
+   command line. Valid arguments can be found by running
+   ``flux job-validator --plugins=LIST --help``
+
+VALIDATOR PLUGIN CONFIGURATION
+------------------------------
+
+Job validator plugins optionally support plugin-specific configuration via
+keys based on the plugin name in the ``[ingest.validator]`` table. Validator
+plugin keys match the plugin name, and define a table of keys specific to
+each plugin.
+
+Plugins distributed with Flux and support configuration are described below.
+
+require-instance
+----------------
+
+The ``[ingest.validator.require-instance]`` table supports the following
+keys:
+
+minnodes
+   (optional) The minimum number of nodes at which to reject jobs that
+   are not instances of Flux. Default is 0. This is an alternative to passing
+   the ``--require-instance-minnodes=`` option on the validator command line.
+
+mincores
+   (optional) The minimum number of cores at which to reject jobs that
+   are not instances of Flux. Default is 0. This is an alternative to passing
+   the ``--require-instance-mincores=`` option on the validator command line.
+
+EXAMPLE
+=======
+
+::
+
+   [ingest.frobnicator]
+   plugins = [ "defaults" ]
+
+   [ingest.validator]
+   plugins = [ "jobspec", "feasibility" ]
+   args =  [ "--require-version=1" ]
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`
diff --git a/doc/man5/flux-config-job-manager.rst b/doc/man5/flux-config-job-manager.rst
new file mode 100644
index 000000000000..7a9da275a74e
--- /dev/null
+++ b/doc/man5/flux-config-job-manager.rst
@@ -0,0 +1,211 @@
+==========================
+flux-config-job-manager(5)
+==========================
+
+
+DESCRIPTION
+===========
+
+The Flux **job-manager** service may be configured via the ``job-manager``
+table, which may contain the following keys:
+
+Note that this set of keys may be extended by loaded jobtap plugins.
+For details pertaining to some plugins distributed with flux see
+:ref:`plugin_specific_keys` below.
+
+
+KEYS
+====
+
+inactive-age-limit
+   (optional) String (in RFC 23 Flux Standard Duration format) that specifies
+   the maximum age of inactive jobs retained in the KVS.  The age is computed
+   since the job became inactive.  Once a job is removed from the KVS, its job
+   data is not longer available.  Inactive jobs can also be manually purged
+   with :man1:`flux-job` ``purge``.
+
+inactive-num-limit
+   (optional) Integer maximum number of inactive jobs retained in the KVS.
+
+plugins
+   (optional) An array of objects defining a list of jobtap plugin directives.
+   Each directive follows the format defined in the :ref:`plugin_directive`
+   section.
+
+housekeeping
+   (optional) Table of configuration for the job-manager housekeeping
+   service. The housekeeping service is an alternative for
+   handling administrative job epilog workloads. If enabled, resources are
+   released by jobs to housekeeping, which runs a command or a systemd unit
+   and releases resources to the scheduler on completion. See configuration
+   details in the :ref:`housekeeping` section.
+
+   Note: The housekeeping script runs as the instance owner (e.g. "flux").
+   On a real system, "command" is configured to "imp run housekeeping",
+   and the IMP is configured to launch the flux-housekeeping systemd
+   service as root. (See :man5:`flux-config-security-imp` for details
+   on configuring :command:`flux imp run`).
+
+
+.. _plugin_directive:
+
+PLUGIN DIRECTIVE
+================
+
+load
+   (optional) A string instructing the job manager to load a plugin matching
+   the given filename into the job-manager.  If the path is not absolute,
+   then the first plugin matching the job-manager searchpath will be loaded.
+
+remove
+   (optional) A string instructing the job manager to remove all plugins
+   matching  the  value.  The  value may be a :linux:man7:`glob`. If ``remove``
+   appears with ``load``, plugin removal is always handled first.  The special
+   value ``all`` is a synonym for ``*``, but will not fail when no plugins
+   match.
+
+conf
+   (optional) An object, valid with ``load`` only, that defines a configuration
+   table to pass to the loaded plugin.
+
+.. _housekeeping:
+
+HOUSEKEEPING
+============
+
+command
+  (optional) An array of strings specifying the housekeeping command.
+  If unspecified but the housekeeping table exists, then assume the command is
+  ``imp run housekeeping``.
+
+release-after
+  (optional) A string specified in Flux Standard Duration (FSD). If unset,
+  resources for a given job are not released until all execution targets for
+  a given job have completed housekeeping. If set to ``0``, resources are
+  released as each target completes. Otherwise, a timer is started when the
+  first execution target for a given job completes, and all resources that
+  have completed housekeeping when the timer fires are released. Following
+  that, resources are released as each execution target completes.
+
+.. _plugin_specific_keys:
+
+PLUGIN SPECIFIC KEYS
+====================
+
+The following configuration keys are supported by included jobtap plugins.
+The documented config will have no effect if the associated plugin is not
+loaded into the job-manager.
+
+perilog.so
+----------
+
+The ``perilog.so`` plugin supports configuration and execution of job-manager
+prolog and/or epilog. The following keys are supported when the ``perilog.so``
+plugin is loaded:
+
+prolog
+   (optional) Table of configuration for a job-manager prolog. If enabled,
+   the prolog is initiated when the job enters the RUN state before any
+   job shells (i.e. user processes) are started. The following keys are
+   supported for the ``[job-manager.prolog]`` table:
+
+   command
+      (optional) An array of strings specifying the command to run. If
+      ``exec.imp`` is set, the the default command is ``["flux-imp",
+      "run", "prolog"]``, otherwise it is an error if command is not set.
+   per-rank
+      (optional) By default the job-manager prolog only runs ``command``
+      on rank 0. With ``per-rank=true``, the command will be run on each
+      rank assigned to the job.
+   timeout
+      (optional) A string value in Flux Standard Duration specifying a
+      timeout for the prolog, after which it is terminated (and a job
+      exception raised). The default prolog timeout is 30m. To disable
+      the timeout use ``0`` or ``infinity``.
+   kill-timeout
+      (optional) If a job exception is raised during the job prolog and
+      ``cancel-on-exception`` is true,  the prolog will be canceled by
+      sending it a SIGTERM signal. ``kill-timeout`` is the number of
+      seconds to wait until any nodes with prolog tasks that are still
+      active will be drained. The drain reason will include the string
+      "canceled then timed out". The default is 60.
+   cancel-on-exception
+      (optional) A boolean indicating whether a fatal job exception raised
+      while the prolog is active terminates the prolog. The default is true.
+
+epilog
+   (optional) Table of configuration for a job-manager epilog. If
+   configured, the epilog is started at the job ``finish`` event,
+   i.e. after all user processes and job shells have terminated, or after
+   prolog failure (in which case there will not be a job ``finish`` event.)
+   The ``[job-manager.epilog]`` table supports the following keys:
+
+   command
+      (optional) An array of strings specifying the command to run. If
+      ``exec.imp`` is set, the the default command is ``["flux-imp",
+      "run", "prolog"]``, otherwise it is an error if command is not set.
+   per-rank
+      (optional) By default the job-manager epilog only runs ``command``
+      on rank 0. With ``per-rank=true``, the command will be run on each
+      rank assigned to the job.
+   timeout
+      (optional) A string value in Flux Standard Duration specifying a
+      timeout for the epilog, after which it is terminated (and a job
+      exception raised). By default, the epilog timeout is disabled.
+   kill-timeout
+      (optional) If a job exception is raised during the job epilog and
+      ``cancel-on-exception`` is ``true``, then the epilog will be canceled
+      by sending it a SIGTERM signal. ``kill-timeout`` is the number of
+      seconds to wait until any nodes with prolog tasks that are still
+      active will be drained. The drain reason will include the string
+      "canceled then timed out". The default is 10. (``kill-timeout`` with
+      the job epilog should only be used for testing purposes)
+   cancel-on-exception
+      (optional) A boolean indicating whether a fatal job exception raised
+      while the epilog is active terminates the epilog. The default is true.
+      (cancel-on-exception is only used with the epilog for testing purposes)
+
+perilog
+  (optional) Common prolog/epilog configuration keys:
+
+   log-ignore
+      (optional) An array of regular expression strings to ignore in the
+      stdout and stderr of prolog and epilog processes.
+
+
+EXAMPLE
+=======
+
+::
+
+   [job-manager]
+
+   journal-size-limit = 10000
+
+   inactive-age-limit = "7d"
+   inactive-num-limit = 10000
+
+   plugins = [
+      {
+        load = "priority-custom.so",
+        conf = {
+           job-limit = 100,
+           size-limit = 128
+        }
+      }
+   ]
+
+   [job-manager.housekeeping]
+   release-after = "1m"
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`, :man1:`flux-jobtap`, :man7:`flux-jobtap-plugins`
diff --git a/doc/man5/flux-config-kvs.rst b/doc/man5/flux-config-kvs.rst
new file mode 100644
index 000000000000..203863f5e8a7
--- /dev/null
+++ b/doc/man5/flux-config-kvs.rst
@@ -0,0 +1,57 @@
+==================
+flux-config-kvs(5)
+==================
+
+
+DESCRIPTION
+===========
+
+The Flux system instance **kvs** service provides the primary key value
+store (i.e. "the KVS") for a large number of Flux services.  For
+example, job eventlogs are stored in the KVS.
+
+The ``kvs`` table may contain the following keys:
+
+
+KEYS
+====
+
+checkpoint-period
+   (optional) Sets a period of time (in RFC 23 Flux Standard Duration
+   format) that the KVS will regularly checkpoint a reference to its
+   primary namespace.  The checkpoint is used to protect against data
+   loss in the event of a Flux broker crash.
+
+gc-threshold
+   (optional) Sets the number of KVS commits (distinct root snapshots)
+   after which offline garbage collection is performed by
+   :man1:`flux-shutdown`. A value of 100000 may be a good starting
+   point. (Default: garbage collection must be manually requested with
+   `flux-shutdown --gc`).
+
+
+EXAMPLE
+=======
+
+::
+
+   [kvs]
+   checkpoint-period = "30m"
+   gc-threshold = 100000
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_23`
+
+
+SEE ALSO
+========
+
+:man1:`flux-shutdown`,:man5:`flux-config`
diff --git a/doc/man5/flux-config-policy.rst b/doc/man5/flux-config-policy.rst
new file mode 100644
index 000000000000..c076c308f6c5
--- /dev/null
+++ b/doc/man5/flux-config-policy.rst
@@ -0,0 +1,114 @@
+=====================
+flux-config-policy(5)
+=====================
+
+
+DESCRIPTION
+===========
+
+The ``policy`` table captures general site preferences for job request defaults
+and limits, as described in RFC 33.
+
+Each queue defined in the ``queues`` table described in
+:man5:`flux-config-queues` may have a ``policy`` sub-table that follows the
+same rules as the main table.  The per-queue table overrides the general table
+for jobs submitted to that queue.
+
+DEFAULTS
+========
+
+Default values for job requests may be configured in the
+``policy.jobspec.defaults.system`` table.  If a job request does not specify
+a value for a system attribute that is configured in the table, the configured
+value is substituted.  Currently the following values are allowed:
+
+policy.jobspec.defaults.system.duration (FSD string)
+   (optional) If a job is submitted without specifying a job duration,
+   this value is substituted.
+
+policy.jobspec.defaults.system.queue (string)
+   (optional) If a job is submitted without specifying a queue name,
+   this value is substituted.
+
+LIMITS
+======
+
+Limits may be configured in the ``policy.limits`` table.  Job requests are
+rejected at submission time if they violate configured limits.
+
+.. note::
+   It is possible to set a queue-specific limit to an `unlimited` value,
+   and thereby defeat a configured general limit for jobs submitted to that
+   queue.  The actual value used to indicate `unlimited` varies by limit
+   type and is noted below.
+
+policy.limits.duration (FSD string)
+   (optional) Maximum duration that a job may request ("0" = unlimited).
+
+.. note::
+   Limit checks take place before the scheduler sees the request, so it is
+   possible to bypass a node limit by requesting only cores, or the core limit
+   by requesting only nodes (exclusively) since this part of the system does
+   not have detailed resource information.  Generally node and core limits
+   should be configured in tandem to be effective on resource sets with
+   uniform cores per node.  Flux does not yet have a solution for node/core
+   limits on heterogeneous resources.
+
+policy.limits.job-size.max.nnodes (integer)
+   (optional) Maximum number of nodes that may be requested (-1 = unlimited).
+
+policy.limits.job-size.max.ncores (integer)
+   (optional) Maximum number of cores that may be requested (-1 = unlimited).
+
+policy.limits.job-size.max.ngpus (integer)
+   (optional) Maximum number of GPUs that may be requested (-1 = unlimited).
+
+policy.limits.job-size.min.nnodes (integer)
+   (optional) Minimum number of nodes that may be requested (-1 = unlimited).
+
+policy.limits.job-size.min.ncores (integer)
+   (optional) Minimum number of cores that may be requested (-1 = unlimited).
+
+policy.limits.job-size.min.ngpus (integer)
+   (optional) Minimum number of GPUs that may be requested (-1 = unlimited).
+
+
+
+EXAMPLE
+=======
+
+::
+
+   [policy.jobspec.defaults.system]
+   duration = "1h"
+   queue = "batch"
+
+   [policy.limits]
+   duration = "4h"
+   job-size.max.nnodes = 8
+   job-size.max.ngpus = 4
+
+   [queues.debug.policy.limits]
+   duration = "30m"
+   job-size.max.ngpus = -1  # unlimited
+
+   [queues.batch]
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_23`
+
+:doc:`rfc:spec_33`
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`
diff --git a/doc/man5/flux-config-queues.rst b/doc/man5/flux-config-queues.rst
new file mode 100644
index 000000000000..01e9945e9e7f
--- /dev/null
+++ b/doc/man5/flux-config-queues.rst
@@ -0,0 +1,89 @@
+=====================
+flux-config-queues(5)
+=====================
+
+
+DESCRIPTION
+===========
+
+The ``queues`` table configures job queues, as described in RFC 33.
+Normally, Flux has a single anonymous queue, but when queues are configured,
+all queues are named, and a job submitted without a queue name is rejected
+unless a default queue is configured.
+
+Each key in the ``queues`` table is a queue name, whose value is a table
+consisting of the following optional keys:
+
+requires (array)
+   A list of resource property names that selects a subset of the total
+   configured resources for the queue.  Each property name is a string, as
+   defined in RFC 20.  If multiple properties are listed, they are combined
+   in a logical *and* operation.  Properties are attached to resources in the
+   ``resource`` table as described in :man5:`flux-config-resource`.
+
+policy (table)
+   A policy table as described in :man5:`flux-config-policy` that overrides
+   the general system policy for jobs submitted to this queue.
+
+A default queue name may be configured by setting
+``policy.jobspec.defaults.system.queue`` as described in
+:man5:`flux-config-policy`.
+
+
+EXAMPLE
+=======
+
+::
+
+   [[resource.config]]
+   hosts = test[0-7]
+   properties = ["debug"]
+
+   [[resource.config]]
+   hosts = test[8-127]
+   properties = ["batch"]
+
+   [queues.debug]
+   policy.limits.duration = "30m"
+   requires = [ "debug" ]
+
+   [queues.batch]
+   policy.limits.duration = "8h"
+   requires = [ "batch" ]
+
+   [policy.jobspec.defaults.system]
+   queue = "batch"
+
+   [sched-fluxion-qmanager]
+   queues = "batch debug"
+   queue-policy-per-queue="batch:easy debug:fcfs"
+
+   [sched-fluxion-resource]
+   match-policy = "lonodex"
+   match-format = "rv1_nosched"
+
+
+CAVEATS
+=======
+
+Queue resources should not overlap.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_20`
+
+:doc:`rfc:spec_33`
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`, :man5:`flux-config-policy`, :man5:`flux-config-resource`
diff --git a/doc/man5/flux-config-resource.rst b/doc/man5/flux-config-resource.rst
new file mode 100644
index 000000000000..b1edf7084706
--- /dev/null
+++ b/doc/man5/flux-config-resource.rst
@@ -0,0 +1,109 @@
+=======================
+flux-config-resource(5)
+=======================
+
+
+DESCRIPTION
+===========
+
+The Flux **resource** service provides an interface to the scheduler
+for acquiring instance resources.  It accepts some configuration, especially
+useful in the Flux system instance where resources must be known before
+all brokers are online.
+
+The ``resource`` table may contain the following keys:
+
+
+KEYS
+====
+
+path
+   (optional) Set the path to an RFC 20 (R version 1) resource description
+   which defines the resources available to the system.  If undefined,
+   resources are determined either by dynamic discovery, or by information
+   passed from the enclosing Flux instance.  The ``flux R`` utility may be
+   used to generate this file.
+
+config
+   (optional) An array of resource config entries used as an alternative to
+   a R object configured by ``resource.path``. Each array entry must contain
+   a ``hosts`` key in RFC 29 Hostlist Format which configures the list of
+   hosts to which the rest of the entry applies. The entry may also contain
+   ``cores`` and/or ``gpus`` keys which configure the set of core ids and
+   GPU ids (in RFC 22 idset form) available on the targeted hosts, or a
+   ``properties`` key which is an array of property strings to assign to
+   ``hosts``. It is not an error to list a host multiple times, instead
+   each entry updates ``hosts``. If the ``config`` array exists, then any
+   ``path`` is ignored.
+
+   Example::
+
+     [[resource.config]]
+     hosts = "test[1-100]"
+     cores = "0-7"
+
+     [[resource.config]]
+     hosts = "test[1,2]"
+     gpus = "0-1"
+
+     [[resource.config]]
+     hosts = "test[1-89]"
+     properties = ["batch"]
+
+     [[resource.config]]
+     hosts = "test[90-100]"
+     properties = ["debug"]
+
+scheduling
+   (optional) Set the path to a file stored as JSON which will be used
+   to amend the configured R with a RFC 20 ``scheduling`` key.
+
+exclude
+   (optional) A string value that defines one or more nodes to withhold
+   from scheduling, either in RFC 22 idset form, or in RFC 29 hostlist form.
+
+   If a drained node is subsequently excluded, the drain state of the node
+   is cleared since nodes cannot be both excluded and drained.
+
+norestrict
+   (optional) Disable restricting of the loaded HWLOC topology XML to the
+   current cpu affinity mask of the Flux broker. This option should be used
+   when the Flux system instance is constrained to a subset of cores,
+   but jobs run within this instance should have access to all cores.
+
+noverify
+   (optional) If true, disable the draining of nodes when there is a
+   discrepancy between configured resources and HWLOC-probed resources.
+
+Note that updates to the resource table are ignored until the next Flux
+restart.
+
+EXAMPLE
+=======
+
+::
+
+   [resource]
+   path = "/etc/flux/system/R"
+   exclude = "test[3,108]"
+   norestrict = true
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_22`
+
+:doc:`rfc:spec_29`
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`
diff --git a/doc/man5/flux-config-systemd.rst b/doc/man5/flux-config-systemd.rst
new file mode 100644
index 000000000000..6ddd49234a3e
--- /dev/null
+++ b/doc/man5/flux-config-systemd.rst
@@ -0,0 +1,46 @@
+======================
+flux-config-systemd(5)
+======================
+
+
+DESCRIPTION
+===========
+
+Flux can optionally launch jobs using systemd when :man5:`flux-config-exec`
+selects the ``sdexec`` service.  The ``systemd`` table must also be configured.
+
+KEYS
+====
+
+enable (optional)
+   Boolean value enables the ``sdbus`` and ``sdexec`` modules to be loaded.
+   (Default: ``false``).
+
+sdbus-debug (optional)
+   Boolean value enables debug logging by the ``sdbus`` module.
+
+sdexec-debug (optional)
+   Boolean value enables debug logging by the ``sdexec`` module.
+
+
+EXAMPLE
+=======
+
+::
+
+   [systemd]
+   enable = true
+   sdbus-debug = true
+   sdexec-debug = true
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`
diff --git a/doc/man5/flux-config-tbon.rst b/doc/man5/flux-config-tbon.rst
new file mode 100644
index 000000000000..2cb5ae4b3bd0
--- /dev/null
+++ b/doc/man5/flux-config-tbon.rst
@@ -0,0 +1,125 @@
+===================
+flux-config-tbon(5)
+===================
+
+
+DESCRIPTION
+===========
+
+The ``tbon`` table may be used to tune the configuration of the Flux tree-based
+overlay network (TBON).
+
+It may contain the following keys:
+
+
+KEYS
+====
+
+topo
+   (optional) A URI that selects a specific tree topology.  The default value
+   is ``kary:32`` when bootstrapping from PMI, and ``custom`` when bootstrapping
+   from configuration, as described in :man5:`flux-config-bootstrap`.
+   The configured value may be overridden by setting the ``tbon.topo`` broker
+   attribute.
+
+torpid_min
+   (optional) The amount of time (in RFC 23 Flux Standard Duration format) that
+   a broker will allow the connection to its TBON parent to remain idle before
+   sending a control message to create activity.  The default value of
+   ``5s`` should be reasonable in most circumstances.  This configured value
+   may be overridden by setting the ``tbon.torpid_min`` broker attribute.
+
+torpid_max
+   (optional) The amount of time (in RFC 23 Flux Standard Duration format) that
+   a broker will wait for an idle TBON child connection to send messages before
+   declaring it torpid  (unresponsive). A value of 0 disables torpid node
+   checking.  New work is not scheduled on a node while torpid, but a job
+   running on a node when it becomes torpid is allowed to complete.  This
+   configured value may be overridden by setting the ``tbon.torpid_max``
+   broker attribute.
+
+tcp_user_timeout
+   (optional) The amount of time (in RFC 23 Flux Standard Duration format) that
+   a broker waits for a TBON child connection to acknowledge transmitted TCP
+   data before forcibly closing the connection.  A value of 0 means use the
+   system default.  This value affects how Flux responds to an abruptly turned
+   off node.  The configured value may be overridden by setting the
+   ``tbon.tcp_user_timeout`` broker attribute.  See also: :linux:man7:`tcp`,
+   TCP_USER_TIMEOUT socket option.  Default: 20s.
+
+connect_timeout
+   (optional) The amount of time (in RFC 23 Flux Standard Duration format)
+   that a broker waits for a :linux:man1:`connect` attempt to its TBON parent
+   to succeed before before retrying.  A value of 0 means use the system
+   default.  The configured value may be overridden by setting the
+   ``tbon.connect_timeout`` broker attribute.  Default: 30s.
+
+zmqdebug
+   (optional) Integer value indicating whether ZeroMQ socket debug logging
+   should be enabled: 0=disabled, 1=enabled.  Default: ``0``.  This configured
+   value may be overridden by setting the ``tbon.zmqdebug`` broker attribute.
+
+zmq_io_threads
+   (optional) Integer value to set the number of I/O threads libzmq will start
+   on the leader node.  The default is 1.  This configured value may be
+   overridden by setting the ``tbon.zmq_io_threads`` broker attribute.
+
+child_rcvhwm
+   (optional) Integer value that limits the number of messages stored locally
+   on behalf of each downstream TBON peer.  When the limit is reached, messages
+   are queued on the peer instead.  Setting this reduces memory usage for
+   nodes with a large number of downstream peers, at the expense of message
+   latency.  The value should be 0 (unlimited) or >= 2.  The default is 0.
+   This configured value may be overridden by setting the ``tbon.child_rcvhwm``
+   broker attribute.
+
+interface-hint
+   When the broker's bind address is not explicitly configured via
+   :man5:`flux-config-bootstrap`, it is chosen dynamically, influenced by
+   one of the following hints:
+
+   default-route
+     The address associated with the default route (the default hint).
+   hostname
+     The address associated with the system hostname.
+   *interface*
+     The address associated with the named network interface, e.g. ``enp4s0``
+   *network*
+     The address associated with the first interface that matches the
+     network address in CIDR form, e.g. ``10.0.2.0/24``.  NOTE: IPv6
+     network addresses are not supported at this time.
+
+   This configured value may be overridden by setting the
+   ``tbon.interface-hint`` broker attribute on the command line.
+   It is inherited by sub-instances spawned for batch jobs and allocations.
+   Refer to :man7:`flux-broker-attributes` for more information.
+
+
+EXAMPLE
+=======
+
+::
+
+   [tbon]
+   torpid_min = "10s"
+   torpid_max = "1m"
+
+   tcp_user_timeout = "2m"
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_23`
+
+
+SEE ALSO
+========
+
+:man5:`flux-config`, :man7:`flux-broker-attributes`
diff --git a/doc/man5/flux-config.rst b/doc/man5/flux-config.rst
new file mode 100644
index 000000000000..ee487c6856a7
--- /dev/null
+++ b/doc/man5/flux-config.rst
@@ -0,0 +1,88 @@
+==============
+flux-config(5)
+==============
+
+DESCRIPTION
+===========
+
+Flux normally operates without configuration files.  If configuration is
+needed, the :man1:`flux-broker` **--config-path=PATH** option may be used
+to instruct Flux to parse a config file or, if **PATH** is a directory, all
+files matching the glob **PATH/*.toml**.  Alternatively, the
+:envvar:`FLUX_CONF_DIR` environment variable may be used to set the
+configuration file or directory path. If both are set, the command line
+argument takes precedence.
+
+The Flux systemd unit file starts the system instance broker with
+``--config-path=${sysconfdir}/flux/system/conf.d``.  Further discussion of the
+system instance configuration as a whole may be found in the Flux
+Administrator's Guide.
+
+Flux configuration files typically follow the TOML file format,
+with configuration subdivided by function into separate TOML tables.
+The tables may all appear in a single ``.toml`` file or be fragmented in
+multiple files.  For example, the Flux Administrator's Guide suggests one
+file per top level TOML table.
+
+If the **PATH** value is a regular file, the configuration may optionally be
+provided as JSON. This allows the output of ``flux config get`` to be read
+by another Flux instance. A JSON config file must have the file extension
+``.json``, otherwise it is assumed to be TOML.
+
+Each Flux broker parses configuration files independently, however it is
+assumed that config files are identical across the brokers of a given instance.
+System administrators managing Flux configuration files should ensure that
+this is the case, and be mindful that old ``.toml`` files must be removed from
+the configuration directory or renamed so they no longer match the glob.
+
+The configuration may be altered at runtime by changing the files, then running
+``flux config reload`` on each broker, or ``systemctl reload flux`` on each
+node of the system instance.  However see CAVEATS below.
+
+.. _flux_config_caveats:
+
+CAVEATS
+=======
+
+Although most Flux framework projects such as **fluxion** and
+**flux-accounting** are built upon the **flux-core** configuration mechanism
+described above, the **flux-security** project use a similar, but independent
+TOML configuration.  The security components unconditionally parse ``*.toml``
+from compiled-in directories: ``${sysconfdir}/flux/security/conf.d`` for the
+signing library, and ``${sysconfdir}/flux/imp/conf.d`` for the IMP.
+``flux config reload`` does not affect the security components.  Refer to the
+Flux Administrator's Guide for more information on configuring security.
+
+Some portions of the configuration use formats other than TOML:
+
+- The ``resource.path`` key optionally points to a file in RFC 20 (R version 1) JSON format.
+- The ``bootstrap.curve_cert`` key optionally points to a CURVE certificate in a ZeroMQ certificate file format.
+For clarity, these files should be located outside of the TOML configuration
+directory.
+
+Although configuration can be reloaded on a live system, it should be assumed
+that configuration parameters take effect on the next Flux broker restart
+unless otherwise noted in their documentation.  This means that some
+parameters, such as the system instance network topology, must only be
+changed in conjunction with a full system instance restart in order to avoid
+brokers becoming out of sync if they are independently restarted before the
+next instance restart.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+Flux Administrator's Guide: https://flux-framework.readthedocs.io/projects/flux-core/en/latest/guide/admin.html
+
+TOML: Tom's Obvious Minimal Language: https://toml.io/en/
+
+
+SEE ALSO
+========
+
+:man1:`flux-broker`, :man5:`flux-config-access`, :man5:`flux-config-bootstrap`,
+:man5:`flux-config-tbon`, :man5:`flux-config-exec`, :man5:`flux-config-ingest`,
+:man5:`flux-config-resource`,
+:man5:`flux-config-job-manager`, :man5:`flux-config-kvs`
diff --git a/doc/man5/index.rst b/doc/man5/index.rst
new file mode 100644
index 000000000000..577beb7fdaf3
--- /dev/null
+++ b/doc/man5/index.rst
@@ -0,0 +1,8 @@
+Section 5 - Flux Configuration File
+===================================
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   *
diff --git a/doc/man7/COPYRIGHT.adoc b/doc/man7/COPYRIGHT.adoc
deleted file mode 100644
index 6b2304f95801..000000000000
--- a/doc/man7/COPYRIGHT.adoc
+++ /dev/null
@@ -1,4 +0,0 @@
-Copyright 2014 Lawrence Livermore National Security, LLC
-and Flux developers.
-
-SPDX-License-Identifier: LGPL-3.0
diff --git a/doc/man7/Makefile.am b/doc/man7/Makefile.am
deleted file mode 100644
index 58b6eb204824..000000000000
--- a/doc/man7/Makefile.am
+++ /dev/null
@@ -1,27 +0,0 @@
-MAN7_FILES = \
-	flux-broker-attributes.7
-
-ADOC_FILES  = $(MAN7_FILES:%.7=%.adoc)
-XML_FILES   = $(MAN7_FILES:%.7=%.xml)
-
-if ENABLE_DOCS
-dist_man_MANS = $(MAN7_FILES)
-$(MAN7_FILES): COPYRIGHT.adoc
-endif
-
-SUFFIXES = .adoc .7
-
-STDERR_DEVNULL = $(stderr_devnull_$(V))
-stderr_devnull_ =  $(stderr_devnull_$(AM_DEFAULT_VERBOSITY))
-stderr_devnull_0 = 2>/dev/null
-
-.adoc.7:
-	$(AM_V_GEN)$(ADOC) --attribute mansource=$(PACKAGE_NAME) \
-	    --attribute manversion=$(PACKAGE_VERSION) \
-	    --attribute manmanual="Flux Miscellaneous Reference" \
-	    --destination-dir $(builddir) \
-	    --doctype manpage $(ADOC_FORMAT_OPT) manpage $< $(STDERR_DEVNULL)
-
-EXTRA_DIST = $(ADOC_FILES) COPYRIGHT.adoc
-
-CLEANFILES = $(MAN7_FILES) $(XML_FILES)
diff --git a/doc/man7/common/experimental.rst b/doc/man7/common/experimental.rst
new file mode 100644
index 000000000000..00be899898a2
--- /dev/null
+++ b/doc/man7/common/experimental.rst
@@ -0,0 +1,4 @@
+Experimental Flux features and interfaces are made available for evaluation
+only and may change or be removed without notice.
+
+Feedback is welcome.  Please use the flux-core project Github issue tracker.
diff --git a/doc/man7/common/resources.rst b/doc/man7/common/resources.rst
new file mode 100644
index 000000000000..dcc16e8954b0
--- /dev/null
+++ b/doc/man7/common/resources.rst
@@ -0,0 +1,5 @@
+Flux: http://flux-framework.org
+
+Flux RFC: https://flux-framework.readthedocs.io/projects/flux-rfc
+
+Issue Tracker: https://github.com/flux-framework/flux-core/issues
diff --git a/doc/man7/flux-broker-attributes.adoc b/doc/man7/flux-broker-attributes.adoc
deleted file mode 100644
index c0fc0b5476ef..000000000000
--- a/doc/man7/flux-broker-attributes.adoc
+++ /dev/null
@@ -1,207 +0,0 @@
-flux-broker-attributes(7)
-=========================
-:doctype: manpage
-
-
-NAME
-----
-flux-broker-attributes - overview Flux broker attributes
-
-
-DESCRIPTION
------------
-
-Flux broker attributes are parameters that affect how different
-broker systems behave.  Attributes can be listed and manipulated
-with flux-getattr(1), flux-setattr(1), and flux-lsattr(1).
-
-The broker currently exports the following attributes:
-
-SESSION ATTRIBUTES
-------------------
-
-rank::
-The rank of the local broker.
-
-size::
-The number of ranks in the comms session.
-
-rundir::
-A global, shared temporary directory available for scratch storage
-within the session. By default, a temporary directory is created
-for each broker rank, but if rundir is set on the command line, this
-directory may be shared by all broker ranks (typically only within
-a node).  If rundir does not exist, it is created and subsequently
-removed during session exit. An existing rundir is not removed at exit.
-
-broker.rundir::
-A temporary directory available for per-rank scratch storage within
-the session. By default broker.rundir is set to "${rundir}/${rank}",
-which guarantees a unique directory per rank.  It is not advisable
-to override this attribute on the command line. Use rundir instead.
-
-content.backing-path::
-The path to the content backing store file(s).  If this is set on the
-broker command line, the backing store uses this path instead of
-a temporary one, and content is preserved on instance exit.
-If file exists, its content is imported into the instance.
-If it doesn't exist, it is created.
-
-TOPOLOGY ATTRIBUTES
--------------------
-tbon.arity::
-Branching factor of the tree based overlay network.
-
-tbon.descendants::
-Number of descendants "below" this node of the tree based
-overlay network, not including this node.
-
-tbon.level::
-The level of this node in the tree based overlay network.
-Root is level 0.
-
-tbon.maxlevel::
-The maximum level number in the tree based overlay network.
-Maxlevel is 0 for a size=1 instance.
-
-tbon.endpoint::
-The endpoint for the tree based overlay network to communicate over.
-Format specifier "%h" can be used to specify the IP address of the
-host and is useful when configuring an IP endpoint.  Format specifier
-"%B" can be used to specify the value of the attribute broker.rundir.
-It is useful when configuring an IPC endpoint.  Defaults to
-"tcp://%h:*".
-
-SOCKET ATTRIBUTES
------------------
-
-tbon.parent-endpoint::
-The URI of the ZeroMQ endpoint this rank is connected to in the tree
-based overlay network.  This attribute will not be set on rank zero.
-
-local-uri::
-The Flux URI that should be passed to flux_open(1) to establish
-a connection to the local broker rank. By default, local-uri is
-created as "local://<broker.rank>/local".
-
-parent-uri::
-The Flux URI that should be passed to flux_open(1) to establish
-a connection to the enclosing instance.
-
-
-LOGGING ATTRIBUTES
-------------------
-
-log-ring-used::
-The number of log entries currently stored in the ring buffer.
-
-log-ring-size::
-The maximum number of log entries that can be stored in the ring buffer.
-
-log-count::
-The number of log entries ever stored in the ring buffer.
-
-log-forward-level::
-Log entries at syslog(3) level at or below this value are forwarded
-to rank zero for permanent capture.
-
-log-critical-level::
-Log entries at syslog(3) level at or below this value are copied
-to stderr on the logging rank, for capture by the enclosing instance.
-
-log-filename::
-(rank zero only) If set, session log entries, as filtered by log-forward-level,
-are directed to this file.
-
-log-stderr-level::
-(rank zero only) Session log entries at syslog(3) level at or below this
-value, and as filtered by log-forward-level, are copied to stderr of the
-rank zero broker.
-
-log-level::
-Log entries at syslog(3) level at or below this value are stored
-in the ring buffer.
-
-
-CONTENT ATTRIBUTES
-------------------
-content.acct-dirty::
-The number of dirty cache entries on this rank.
-
-content.acct-entries::
-The total number of cache entries on this rank.
-
-content.acct-size::
-The estimated total size in bytes consumed by cache entries on
-this rank, excluding overhead.
-
-content.acct-valid::
-The number of valid cache entries on this rank.
-
-content.backing::
-The selected backing store, if any.  This attribute is only
-set on rank 0 where the content backing store is active.
-
-content.blob-size-limit::
-The maximum size of a blob, the basic unit of content storage.
-
-content.flush-batch-count::
-The current number of outstanding store requests, either to the
-backing store (rank 0) or upstream (rank > 0).
-
-content.flush-batch-limit::
-The maximum number of outstanding store requests that will be
-initiated when handling a flush or backing store load operation.
-
-content.hash::
-The selected hash algorithm, default sha1.
-
-content.purge-large-entry::
-When the cache size footprint needs to be reduced, first consider
-purging entries of this size or greater.
-
-content.purge-old-entry::
-When the cache size footprint needs to be reduced, only consider
-purging entries that are older than this number of heartbeats.
-
-content.purge-target-entries::
-If possible, the cache size purged periodically so that the total
-number of entries stays at or below this value.
-
-content.purge-target-size::
-If possible, the cache size purged periodically so that the total
-size of the cache stays at or below this value.
-
-
-WIREUP ATTRIBUTES
------------------
-hello.timeout::
-The reduction timeout (in seconds) for the broker wireup protocol.
-Before the timeout, a topology-based high water mark is applied
-at each node of the tree based overlay network.  After the timeout,
-new wireup information is forwarded upstream without delay.
-Set to 0 to disable the timeout.
-
-hello.hwm::
-The reduction high water mark for the broker wireup protocol,
-normally calculated based on the topology.
-Set to 0 to disable the high water mark.
-
-AUTHOR
-------
-This page is maintained by the Flux community.
-
-
-RESOURCES
----------
-Github: <http://github.com/flux-framework>
-
-
-COPYRIGHT
----------
-include::COPYRIGHT.adoc[]
-
-
-SEE ALSO
---------
-flux-getattr(1), flux_attr_get(3)
diff --git a/doc/man7/flux-broker-attributes.rst b/doc/man7/flux-broker-attributes.rst
new file mode 100644
index 000000000000..1202cca09ae5
--- /dev/null
+++ b/doc/man7/flux-broker-attributes.rst
@@ -0,0 +1,321 @@
+=========================
+flux-broker-attributes(7)
+=========================
+
+
+DESCRIPTION
+===========
+
+Flux broker attributes are broker parameters with a scope of a single broker
+rank.  They may be listed with :man1:`flux-lsattr` and queried with
+:man1:`flux-getattr`.
+
+Attributes should be considered read-only, unless annotated below with:
+
+C
+   The attribute may be set on the :man1:`flux-broker` command line with
+   ``--setattr=NAME=VALUE``.
+
+R
+   The attribute may be updated on a running broker with :man1:`flux-setattr`.
+
+
+GENERAL
+=======
+
+rank
+   The rank of the local broker.
+
+size
+   The number of broker ranks in the flux instance
+
+version
+   The version of flux-core that was used to build this broker.
+
+rundir [Updates: C]
+   A temporary directory where the broker's UNIX domain sockets are located.
+   In addition, if ``statedir`` is not set, this directory is used by the
+   content backing store (if applicable).  By default, each broker rank creates
+   a unique ``rundir`` in ``$TMPDIR`` and removes it on exit.  If ``rundir`` is
+   set on the command line, beware exceeding the UNIX domain socket path limit
+   described in :linux:man7:`unix`, as low as 92 bytes on some systems.  To
+   support the :man1:`flux-start` ``--test-size`` option where multiple brokers
+   share a ``rundir``, if ``rundir`` is set to a pre-existing directory, the
+   directory is not removed by the broker on exit.  In most cases this
+   attribute should not be set by users.
+
+statedir [Updates: C]
+   A directory in which persistent state is stored by the Flux broker.  For
+   example, content backing store data is stored here to facilitate restarts.
+   If unset, this data goes to ``rundir`` where it is cleaned up on instance
+   shutdown.  If set, this directory must exist and be owned by the instance
+   owner.  Default: unset.
+
+security.owner
+   The numeric userid of the owner of this Flux instance.
+
+local-uri [Updates: C]
+   The Flux URI that the local connector binds to for accepting connections
+   from local Flux clients.  The name must begin with ``local://``
+   and the path must refer to a :linux:man7:`unix` domain socket in an
+   existing directory.
+
+parent-uri
+   The URI that should be passed to :man3:`flux_open` to establish a connection
+   to the enclosing instance.
+
+instance-level
+   The nesting level of this Flux instance, or ``0`` if there is no enclosing
+   Flux instance.
+
+jobid
+   The Flux job ID of this Flux instance, if it was launched by Flux as a job.
+   The value is obtained from ``PMI_KVS_Get_my_name()`` which may be something
+   other than a Flux job ID if Flux was started by another means.
+
+parent-kvs-namespace
+   The value of the broker's :envvar:`FLUX_KVS_NAMESPACE` environment variable.
+   This is the KVS namespace assigned to this Flux instance by its enclosing
+   instance, if it was launched by Flux as a job.
+
+hostlist
+   An RFC 29 hostlist in broker rank order.  This value may be used to
+   translate between broker ranks and hostnames.
+
+broker.mapping
+   A string representing the process mapping of broker ranks in the Flux
+   Task Map format described in RFC 34.  For example, ``[[0,16,1,1]]`` means
+   the instance has one broker per node on 16 nodes, and ``[[0,1,16,1]]``
+   means it has 16 brokers on one node.
+
+broker.critical-ranks [Updates: C]
+   An RFC 22 idset representing broker ranks that are considered critical
+   to instance operation. The broker notifies the job execution system in
+   the parent instance of these ranks such that a fatal job exception
+   is raised when a failing node or other error occurs affecting any rank
+   in this set. Default: rank 0 plus any other overlay network routers.
+
+broker.boot-method [Updates: C]
+   A URI representing the method used to bootstrap Flux.  Valid values are
+   ``config`` (boot via TOML config file), ``simple`` (use the PMI-1 simple
+   wire protocol), ``libpmi[:path]`` (use a PMI-1 shared library), or
+   ``single`` (standalone size=1).  Additional boot methods may be provided
+   by plugins.
+
+broker.pid
+   The process id of the local broker.
+
+broker.quorum [Updates: C]
+   The number of brokers that are required to be online before the rank 0
+   broker enters the RUN state and starts the initial program, if any.
+   Default: instance size.
+
+broker.quorum-timeout [Updates: C]
+   The amount of time (in RFC 23 Flux Standard Duration format) that the
+   rank 0 broker waits for the ``broker.quorum`` set to come online before
+   aborting the Flux instance.   Default: ``60s``.
+
+broker.cleanup-timeout [Updates: C]
+   The amount of time (in RFC 23 Flux Standard Duration format) that the
+   rank 0 broker waits for cleanup actions to complete when the broker has
+   received a terminating signal.  Default: ``none``.
+
+broker.rc1_path [Updates: C]
+   The path to the broker's rc1 script.  Default: ``${prefix}/etc/flux/rc1``.
+
+broker.rc3_path [Updates: C]
+   The path to the broker's rc3 script.  Default: ``${prefix}/etc/flux/rc3``.
+
+broker.exit-restart [Updates: C, R]
+   A numeric exit code that the broker uses to indicate that it should not be
+   restarted.  This is set by the systemd unit file.  Default: unset.
+
+broker.starttime
+   Timestamp of broker startup from :man3:`flux_reactor_now`.
+
+conf.shell_initrc [Updates: C, R]
+   The path to the :man1:`flux-shell` initrc script.  Default:
+   ``${prefix}/etc/flux/shell/initrc.lua``.
+
+conf.shell_pluginpath [Updates: C, R]
+   The list of colon-separated directories to be searched by :man1:`flux-shell`
+   for shell plugins.  Default: ``${prefix}/lib/flux/shell/plugins``.
+
+config.path [Updates: see below]
+   A config file or directory (containing ``*.toml`` config files) for
+   this Flux instance. This attribute may be set via the :envvar:`FLUX_CONF_DIR`
+   environment variable, or the :man1:`flux-broker` ``--config-path``
+   command line argument.  Default: none.  See also :man5:`flux-config`.
+
+
+TREE BASED OVERLAY NETWORK
+==========================
+
+tbon.topo [Updates: C]
+   URI describing the TBON tree topology such as ``kary:16``.  The ``kary``
+   scheme selects a complete, k-ary tree with fanout *k*, with ``kary:0``
+   meaning that rank 0 is the parent of all other ranks by convention.  The
+   ``binomial`` scheme selects a binomial tree topology of the minimum order
+   that fits the instance size.  Default: ``kary:32``, unless bootstrapping by
+   TOML configuration, then see :man5:`flux-config-bootstrap`.
+
+tbon.descendants
+   Number of descendants "below" this node of the tree based
+   overlay network, not including this node.
+
+tbon.level
+   The level of this node in the tree based overlay network.
+   Root is level 0.
+
+tbon.maxlevel
+   The maximum level number in the tree based overlay network.
+   Maxlevel is 0 for a size=1 instance.
+
+tbon.parent-endpoint
+   The ZeroMQ endpoint of this broker's TBON parent.
+
+tbon.zmqdebug [Updates: C]
+   If set to an non-zero integer value, 0MQ socket event logging is enabled,
+   if available.  This is potentially useful for debugging overlay
+   connectivity problems.  Default: ``0``.
+
+tbon.zmq_io_threads [Updates: C]
+   Set the number of I/O threads libzmq will start on the leader node.
+   Default: ``1``.
+
+tbon.child_rcvhwm [Updates: C]
+   Limit the number of messages stored locally on behalf of each downstream
+   TBON peer.  When the limit is reached, messages are queued on the peer
+   instead.  Default: ``0`` (unlimited).
+
+tbon.prefertcp [Updates: C]
+   If set to an integer value other than zero, and the broker is bootstrapping
+   with PMI, tcp:// endpoints will be used instead of ipc://, even if all
+   brokers are on a single node.  Default: ``0``.
+
+tbon.interface-hint [Updates: C, R]
+   When bootstrapping with PMI, tcp endpoints are chosen heuristically
+   using one of the following methods:
+
+   default-route
+      The address associated with the default route (default, but see below).
+   hostname
+      The address associated with the system hostname.
+   *interface*
+     The address associated with the named network interface, e.g. ``enp4s0``
+   *network*
+     The address associated with the first interface that matches the
+     network address in CIDR form, e.g. ``10.0.2.0/24``.  NOTE: IPv6
+     network addresses are not supported at this time.
+
+   If the attribute is not explicitly set, its value is assigned from
+   (in descending precedence):
+
+   1. the TOML configuration key of the same name
+   2. the :envvar:`FLUX_IPADDR_INTERFACE` or :envvar:`FLUX_IPADDR_HOSTNAME`
+      environment variables (these should be considered deprecated)
+   3. the enclosing Flux instance's value (via the PMI KVS)
+   4. the compiled-in default of default-route
+
+tbon.torpid_min [Updates: C, R]
+   The amount of time (in RFC 23 Flux Standard Duration format) that a broker
+   will allow the connection to its TBON parent to remain idle before sending a
+   control message to indicate create activity.  Default: ``5s``.
+
+tbon.torpid_max [Updates: C, R]
+   The amount of time (in RFC 23 Flux Standard Duration format) that a broker
+   will wait for an idle TBON child connection to send messages before
+   declaring it torpid (unresponsive).  A value of 0 disables torpid node
+   checking.  New work is not scheduled on a node while torpid, but a job
+   running on a node when it becomes torpid is allowed to complete.
+   Default: ``30s``.
+
+tbon.tcp_user_timeout
+   The amount of time (in RFC 23 Flux Standard Duration format) that a broker
+   waits for a TBON child connection to acknowledge transmitted TCP data
+   before forcibly closing the connection.  A value of 0 means use the system
+   default.  This value affects how Flux responds to an abruptly turned off
+   node, which could take up to 20m if this value is not set.  This attribute
+   may not be changed during runtime.  The broker attribute overrides
+   the :man5:`flux-config-tbon` ``tcp_user_timeout`` value, if configured.
+   See also: :linux:man7:`tcp`, TCP_USER_TIMEOUT socket option.
+
+tbon.connect_timeout
+   The amount of time (in RFC 23 Flux Standard Duration format) that a broker
+   waits for a :linux:man2:`connect` attempt to its TBON parent to succeed
+   before retrying.  A value of 0 means use the system default.  This
+   attribute may not be changed during runtime.  The broker attribute
+   overrides the :man5:`flux-config-tbon` ``connect_timeout`` value, if
+   configured.
+
+LOGGING
+=======
+
+log-ring-size [Updates: C, R]
+   The maximum number of log entries that can be stored in the ring buffer.
+   Default: ``1024``.
+
+log-forward-level [Updates: C, R]
+   Log entries at :linux:man3:`syslog` level at or below this value
+   are forwarded to rank zero for permanent capture.  Default ``7``
+   (LOG_DEBUG).
+
+log-critical-level [Updates: C, R]
+   Log entries at :linux:man3:`syslog` level at or below this value
+   are copied to stderr on the logging rank, for capture by the
+   enclosing instance.  Default ``2`` (LOG_CRIT).
+
+log-filename [Updates: C, R]
+   (rank zero only) If set, session log entries, as filtered by
+   ``log-forward-level``, are directed to this file.  Default: none.
+
+log-stderr-mode [Updates: C, R]
+   If set to "leader" (default), broker rank 0 emits forwarded logs from
+   other ranks to stderr, subject to the constraints of log-forward-level
+   and log-stderr-level.  If set to "local", each broker emits its own
+   logs to stderr, subject to the constraints of log-stderr-level.
+   Default: ``leader``.
+
+log-stderr-level (Updates: C, R)
+   Log entries at :linux:man3:`syslog` level at or below this value to
+   stderr, subject to log-stderr-mode.  Default: ``3`` (LOG_ERR).
+
+log-level (Updates: C, R)
+   Log entries at :linux:man3:`syslog` level at or below this value
+   are stored in the ring buffer.  Default: ``7`` (LOG_DEBUG).
+
+
+CONTENT
+=======
+
+content.backing-module (Updates: C)
+   The selected backing store module, if any. This attribute is only set
+   on rank 0 where the content backing store is active.  Default:
+   ``content-sqlite``.
+
+content.hash (Updates: C)
+   The selected hash algorithm.  Default ``sha1``.  Other options: ``sha256``.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+FLUX RFC
+========
+
+:doc:`rfc:spec_13`
+
+:doc:`rfc:spec_22`
+
+:doc:`rfc:spec_23`
+
+:doc:`rfc:spec_29`
+
+
+SEE ALSO
+========
+
+:man1:`flux-broker`, :man1:`flux-getattr`, :man3:`flux_attr_get`
diff --git a/doc/man7/flux-environment.rst b/doc/man7/flux-environment.rst
new file mode 100644
index 000000000000..5a727529cc30
--- /dev/null
+++ b/doc/man7/flux-environment.rst
@@ -0,0 +1,550 @@
+===================
+flux-environment(7)
+===================
+
+
+DESCRIPTION
+===========
+
+The following environment variables are set by Flux or influence Flux.
+
+.. _job_environment:
+
+JOB ENVIRONMENT
+===============
+
+The following are set in the environment of each task spawned by
+:man1:`flux-shell` as part of a Flux job.
+
+.. envvar:: FLUX_JOB_ID
+
+   The current jobid in F58 form.  F58 is a compact, non-numeric representation
+   of Flux's 64-bit integer jobid.  If the numeric form is required, use e.g.:
+
+   .. code-block:: shell
+
+      NUMERIC_JOB_ID=$(flux job id $FLUX_JOB_ID)
+
+.. envvar:: FLUX_JOB_SIZE
+
+   The number of tasks in the current job.
+
+.. envvar:: FLUX_JOB_NNODES
+
+   The total number of nodes hosting tasks on behalf of the current job.
+
+.. note::
+
+   :envvar:`FLUX_JOB_NNODES` is more precisely defined as the total number of
+   :man1:`flux-shell` processes running tasks on behalf of the current job.
+   Normally one shell is started per broker, and one broker is started per
+   node.  However, in rare test setups, a large Flux instance is mocked by
+   running multiple brokers per node.  In that case, this variable may not
+   represent the physical node count.
+
+.. envvar:: FLUX_TASK_RANK
+
+   The zero-origin, global rank for this task.  Tasks are assigned ranks using
+   a "block" algorithm by default, although :option:`flux submit --taskmap` may
+   select other mapping algorithms.
+
+   Example: 8 tasks on 2 nodes with block and cyclic task mapping:
+
+   .. list-table::
+
+      * - Mapping
+        - Node 0
+        - Node 1
+
+      * - block
+        - 0, 1, 2, 3
+	- 4, 5, 6, 7
+
+      * - cyclic
+        - 0, 2, 4, 6
+	- 1, 3, 5, 7
+
+
+.. envvar:: FLUX_TASK_LOCAL_ID
+
+   The zero-origin, local (to the node) rank for this task.
+
+   Example: 8 tasks on 2 nodes:
+
+   .. list-table::
+
+      * - Node 0
+        - Node 1
+
+      * - 0, 1, 2, 3
+	- 0, 1, 2, 3
+
+.. envvar:: FLUX_JOB_CC
+
+   When :option:`flux submit --cc` or :option:`flux bulksubmit --cc` is used
+   to submit a set of jobs, :envvar:`FLUX_JOB_CC` is set to the the integer
+   id of the current job in the set.
+
+.. envvar:: FLUX_JOB_TMPDIR
+
+   The path of a per-job temporary directory that is created on each host node
+   before any tasks are started, and cleaned up after all tasks have exited.
+   All a job's tasks on a given node share the same directory.
+
+.. envvar:: FLUX_KVS_NAMESPACE
+
+   Each job is assigned a unique, job-owner-writable Flux KVS key space that
+   is independent of the default (primary) one and persists as such while the
+   job is in the RUNNING state.  This environment variable is interpreted by
+   the Flux KVS API and therefore :man1:`flux-kvs` as a directive to treat
+   all operations as rooted in that space.  The job exec service and the job
+   shell record the job's input, output, and a log of events in this space.
+
+   After the job completes, the job's namespace is added to the primary
+   namespace and becomes part of the read-only job record.
+
+.. envvar:: PMI_RANK
+	    PMI_SIZE
+	    PMI_FD
+	    PMI_SPAWNED
+	    FLUX_PMI_LIBRARY_PATH
+
+   The ``pmi`` shell plugin sets these variables in the job environment to
+   aid in the bootstrap of parallel programs.  They are not set when the simple
+   PMI server is disabled, e.g.  with :option:`flux run -opmi=none`.
+
+   The :envvar:`PMI_*` variables are standard for PMI-1 and are described
+   in Flux RFC 13.
+
+   :envvar:`FLUX_PMI_LIBRARY_PATH` is set to the full path of Flux's
+   ``libpmi.so`` shared library, which is normally not installed to standard
+   system paths.  This exists as an aid to the pre-v5 OpenMPI Flux MCA plugins
+   so that an MPI program running under Flux knows where to :func:`dlopen`
+   the library for bootstrap.
+
+.. envvar:: CUDA_VISIBLE_DEVICES
+            CUDA_DEVICE_ORDER
+
+   The ``gpubind`` shell plugin sets these variables in the job environment
+   to assign GPU devices to tasks.  They are not set when GPU affinity is
+   disabled with :option:`flux run -ogpu-affinity=off`.
+
+.. envvar:: FLUX_URI
+
+   :envvar:`FLUX_URI` overrides the default, compiled-in broker socket path
+   in the Flux API, and by extension all the Flux commands.  In the job
+   environment, it points to the local broker responsible for the job.
+
+
+.. _initial_program_environment:
+
+INITIAL PROGRAM ENVIRONMENT
+===========================
+
+The :man1:`flux-alloc` interactive shell and the :man1:`flux-batch` batch
+script are examples of Flux initial programs.  Flux does not set many
+environment variables for the initial program.  In fact, the following
+are actively unset to avoid confusion when they are set by the *enclosing
+instance*:
+
+ - :envvar:`FLUX_JOB_ID`
+ - :envvar:`FLUX_JOB_SIZE`
+ - :envvar:`FLUX_JOB_NNODES`
+ - :envvar:`FLUX_JOB_TMPDIR`
+ - :envvar:`FLUX_TASK_RANK`
+ - :envvar:`FLUX_TASK_LOCAL_ID`
+ - :envvar:`FLUX_KVS_NAMESPACE`
+ - :envvar:`FLUX_PROXY_REMOTE`
+ - :envvar:`FLUX_PMI_LIBRARY_PATH`
+ - :envvar:`I_MPI_PMI_LIBRARY`
+ - :envvar:`PMI_*`
+ - :envvar:`SLURM_*`
+
+The :envvar:`FLUX_URI` variable is set, however, so Flux commands can be used
+as needed from the initial program to obtain information they might get via the
+environment in other workload managers, for example:
+
+.. code-block:: shell
+
+   BATCH_NNODES=$(flux resource list -n -o {nnodes})
+   BATCH_NCORES=$(flux resource list -n -o {ncores})
+   BATCH_NGPUS=$(flux resource list -n -o {ngpus})
+   BATCH_HOSTLIST=$(flux getattr hostlist)
+   BATCH_JOBID=$(flux getattr jobid)
+
+
+PMI CLIENT
+==========
+
+The :man1:`flux-broker` is capable of bootstrapping from configuration or
+using a PMI client, similar to the way an MPI program bootstraps.  The broker's
+PMI *client* is separate from the :man1:`flux-shell` PMI *server* offered to
+parallel programs launched by Flux.  The following environment variables
+affect the broker's PMI client.
+
+.. envvar:: FLUX_PMI_DEBUG
+
+   When set (to any value) in the broker's environment, PMI client tracing
+   is enabled, causing PMI operations that occur during broker bootstrap to
+   be logged to standard error.
+
+.. envvar:: FLUX_PMI_CLIENT_METHODS
+
+   Flux iterates through a list of PMI client implementations to find one that
+   works.  By default the list is ``simple libpmi2 libpmi single``.  The
+   sequence can be altered by setting this variable to a space-delimited list
+   of client implementations.  The built-in ones are:
+
+   simple
+      Use the PMI-1 simple wire protocol.
+
+   libpmi2[:PATH]
+      :func:`dlopen` ``libpmi2.so`` and use the PMI-2 API, optionally at
+      a specific *PATH*.
+
+   libpmi[:PATH]
+      :func:`dlopen` ``libpmi.so`` and use the PMI-1 API, optionally at
+      a specific *PATH*.
+
+   single
+      Become a singleton.  This always succeeds so should be the last method.
+
+.. envvar:: FLUX_PMI_CLIENT_SEARCHPATH
+
+   A colon-separated list of directories to search for PMI client plugins.
+   Client plugins can be packaged separately from flux-core.
+
+.. envvar:: FLUX_IPADDR_HOSTNAME
+
+   When bootstrapping with PMI, the broker dynamically selects an TCP address
+   to bind to for overlay network communication, which it then exchanges with
+   peers using PMI.  By default, it tries to use the address associated with
+   the default route.  Setting this variable to any value in the broker's
+   environment directs it to prefer the address associated with the system
+   :linux:man1:`hostname` instead.
+
+.. envvar:: FLUX_IPADDR_V6
+
+   When dynamically selecting an address to use with PMI, the broker prefers IP
+   version 4 addresses.  Setting this variable to any value in the broker's
+   environment causes it to prefer version 6 addresses.
+
+.. envvar:: FLUX_IPADDR_INTERFACE
+
+   Force PMI bootstrap to assign the broker an address associated with a
+   particular network interface, like ``eth0``.  Alternatively, the interface
+   may be specified by its IPv4 network address in CIDR form, e.g.
+   "192.168.1.1/24".
+
+
+CUSTOM OUTPUT FORMATS
+=====================
+
+Sites and individual users may create custom output formats for some Flux
+commands.  The formats are expressed in configuration files with a base name
+of the command name plus a ``.toml``, ``.yaml``, or ``.json`` extension,
+stored in directories that follow the `XDG Base Directory Specification
+<https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html>`_.
+
+Named formats are merged and/or overridden in the following order:
+
+   #. internal defaults
+   #. config files found in a ``flux`` sub-directory of the
+      :envvar:`XDG_CONFIG_DIRS` directories
+   #. config files found in a ``flux`` sub-directory of
+      :envvar:`XDG_CONFIG_HOME`
+
+For more information on named formats see the individual command documentation
+and the :ref:`flux_jobs_configuration` section of :man1:`flux-jobs`.
+
+.. envvar:: XDG_CONFIG_DIRS
+
+   A colon-separated, preference-ordered list of base directories to search for
+   configuration files in addition to the :envvar:`XDG_CONFIG_HOME` base
+   directory.  If unset, ``/etc/xdg`` is used.
+
+.. envvar:: XDG_CONFIG_HOME
+
+   The base directory for user-specific configuration files.  If unset,
+   ``$HOME/.config`` is used.
+
+.. envvar:: FLUX_JOBS_FORMAT_DEFAULT
+            FLUX_RESOURCE_STATUS_FORMAT
+            FLUX_RESOURCE_LIST_FORMAT_DEFAULT
+            FLUX_QUEUE_LIST_FORMAT_DEFAULT
+            FLUX_PGREP_FORMAT_DEFAULT
+
+   In addition to registering custom named formats, users and sites can change
+   the default output format to one of the named formats by setting an
+   environment variable to the format name.  The above variables affect the
+   default output of :man1:`flux-jobs`, :man1:`flux-resource`,
+   :man1:`flux-queue`, and :man1:`flux-pgrep`.
+
+
+TESTING
+=======
+
+The following environment variables are primarily useful when debugging Flux
+components or writing tests.
+
+.. envvar:: FLUX_HANDLE_TRACE
+
+   If set in the environment of a Flux component, the ``FLUX_O_TRACE`` flag
+   is automatically set in any call to :man3:`flux_open`.  This causes decoded
+   messages passed over the :c:type:`flux_t` handle to be decoded and printed
+   on standard error.
+
+.. envvar:: FLUX_HANDLE_MATCHDEBUG
+
+   If set in the environment of a Flux component, the ``FLUX_O_MATCHDEBUG``
+   flag is automatically set in any call to :man3:`flux_open`.  This causes a
+   diagnostic to be printed to standard error if any matchtags are leaked when
+   the broker connection is closed.
+
+.. envvar:: FLUX_HANDLE_USERID
+
+   Mock a user.  If set to a numerical user ID in the environment of a Flux
+   component, all messages sent by the component appear to have been sent by
+   this user.  This is useful for testing code that authorizes actions based on
+   the identity of the requesting user.  This is restricted to the instance
+   owner.
+
+.. envvar:: FLUX_HANDLE_ROLEMASK
+
+   Mock a rolemask (capability set).  If set to a decimal or hex (``0x``
+   prefixed) value in the environment of a Flux component, all messages sent
+   by the component are stamped with this rolemask. This is useful for testing
+   code that authorizes actions based on the possession of particular roles.
+   This is restricted to the instance owner.
+
+.. envvar:: FLUX_FAKE_HOSTNAME
+
+   When Flux bootstraps from a configuration file as described in
+   :man5:`flux-config-bootstrap`, a :man1:`flux-broker` determines its rank
+   by looking up its own hostname in a ``hosts`` array and using the array
+   index as its rank.  To allow this to be tested on a single node,
+   :envvar:`FLUX_FAKE_HOSTNAME` may be set in the broker's environment to use
+   the specified name instead of the result of :linux:man3:`gethostname`.  Use
+   of this capability in test is simplified by the
+   :option:`flux start --test-hosts` option.
+
+.. envvar:: FLUX_HWLOC_XMLFILE
+
+   Flux discovers available resources dynamically using `HWLOC
+   <https://www.open-mpi.org/projects/hwloc/>`_.  In some cases dynamic
+   discovery is not desired, such as when it causes poor performance in
+   parallel testing.  Flux may be directed to read topology from an XML file
+   instead by setting :envvar:`FLUX_HWLOC_XMLFILE` to the file path.
+
+   :program:`flux resource reload` offers a related mechanism for loading a
+   set of HWLOC xml files directly into the instance resource inventory
+   for test scenarios.
+
+.. envvar:: FLUX_URI_RESOLVE_LOCAL
+
+   If set, force :man1:`flux-uri` and the URI resolver embedded in other
+   commands to resolve URIs to local form.  This is useful in test environments
+   where the remote connector does not work.
+
+.. envvar:: FLUX_RESOURCE_LIST_RPC
+
+   If set, :man1:`flux-resource` uses the specified RPC topic string instead
+   of ``resource.sched-status``.  This is used in test to verify that the
+   ``sched.resource-status`` RPC used in earlier releases still works for
+   backwards compatibility.
+
+.. envvar:: FLUX_LOAD_WITH_DEEPBIND
+
+   By default flux loads all modules, plugins and dlopened libraries of any kind
+   with RTLD_DEEPBIND to avoid symbol conflicts.  If this environment variable
+   is set to 0 that flag will be cleared from the flags of all dlopen
+   invocations. This is mainly useful to override the allocator or otherwise
+   interpose a tool or library with LD_PRELOAD.  Be aware that this can cause
+   symbol conflicts with plugins, and is not recommended for production.
+
+.. envvar:: FLUX_HOSTLIST_STDIN_TIMEOUT
+
+   The :command:`flux-hostlist` command reads from stdin by default. If no
+   data is available within 15s, the command times out to prevent a permanent
+   hang. This environment variable can be used to modify or disable (set to 0)
+   the timeout.
+
+MISCELLANEOUS
+=============
+
+.. envvar:: FLUX_F58_FORCE_ASCII
+
+   A locale or terminal misconfiguration can cause the ``ƒ`` character used in
+   Flux jobids to be rendered incorrectly.  As a workaround, set this variable
+   and ASCII ``f`` is used instead.
+
+.. envvar:: FLUX_CONF_DIR
+
+   If set in in the :man1:`flux-broker` environment, configuration files
+   matching ``*.toml`` are loaded from the specified directory.  The
+   :option:`flux broker --config-path` option does that too, and is more
+   flexible in that it can also load single files in TOML or JSON format.
+
+.. envvar:: FLUX_ATTACH_NONINTERACTIVE
+
+   If set, never show the status line in :program:`flux job attach` output.
+
+.. envvar:: FLUX_PROXY_REMOTE
+
+   When :man1:`flux-proxy` connects to a remote instance, it sets this variable
+   to the authority part of the remote URI.  This serves as a hint to
+   :man3:`flux_attr_get` to transform the value of the ``parent-uri`` broker
+   attribute into a remote URI so it can work from the remote proxy environment.
+   For example:
+
+   .. code-block:: shell
+
+      $ flux alloc -N1
+      f(s=1,d=1) $ flux getattr parent-uri
+      local:///run/flux/local
+
+   .. code-block:: shell
+
+      $ flux proxy $(flux job last)
+      ƒ(s=1,d=1) $ printenv FLUX_PROXY_REMOTE
+      test0
+      ƒ(s=1,d=1) $ flux getattr parent-uri
+      ssh://test0/run/flux/local
+
+.. envvar:: FLUX_TERMINUS_SESSION
+
+   The current terminus session ID.  A terminus session is started when the
+   job has an interactive pseudo-terminal, which occurs when a job is run with
+   :option:`flux run -o pty.interactive`, or when a Flux instance is started
+   with :man1:`flux-alloc`.
+
+.. envvar:: FLUX_RC_EXTRA
+
+   If set to a colon-separated list of directories, the installed
+   :man1:`flux-broker` rc scripts search these directories for additional
+   scripts to run during broker initialization and finalization.
+
+   Specifically the ``rc1`` script runs ``rc1.d/*`` in each directory
+   and the ``rc3`` script runs ``rc3.d/*`` in each directory.
+
+.. envvar:: FLUX_SHELL_RC_PATH
+
+   Set to a colon-separated list of directories to be added to the directories
+   that :man1:`flux-shell` searches for lua scripts to extend its initrc.
+
+.. envvar:: FLUX_SSH
+
+   Override the compiled-in path to the :program:`ssh` executable used by the
+   ssh connector.  The ssh connector is invoked when attempting to open a
+   connection to Flux with a URI that begins with ``ssh://``.
+
+.. envvar:: FLUX_SSH_RCMD
+
+   Override the heuristically-determined remote path to the :man1:`flux`
+   command front end executable used by the ssh connector to start
+   :program:`flux relay` on the remote system.
+
+.. envvar:: DBUS_SESSION_BUS_ADDRESS
+
+   :man1:`flux-exec` sets this to point to the Flux instance owner's
+   D-Bus instance, to ensure that a remote invocation of
+   :option:`systemctl --user` accesses the service manager for the Flux
+   instance owner.  This is helpful when debugging a system instance
+   configured to launch jobs with systemd, as described in
+   :man5:`flux-config-exec`.
+
+
+.. _sub_command_environment:
+
+SUB-COMMAND ENVIRONMENT
+=======================
+
+:man1:`flux` sets up the environment for sub-commands using a combination
+of compiled-in install paths and the environment.
+
+.. note::
+   The PREPEND versions of environment variables below may
+   be necessary when developing and testing a new version
+   of a Flux command (:envvar:`FLUX_EXEC_PATH_PREPEND`),
+   module (:envvar:`FLUX_MODULE_PATH_PREPEND`), connector
+   (:envvar:`FLUX_CONNECTOR_PATH_PREPEND`), or Python module
+   (:envvar:`FLUX_PYTHONPATH_PREPEND`) when an existing version of that
+   component is already installed in the system default paths. Otherwise,
+   the installed component would always be used by the system Flux, since
+   the installed paths are always placed first in the subcommand environment
+   created by :man1:`flux`.
+
+.. envvar:: FLUX_EXEC_PATH
+            FLUX_EXEC_PATH_PREPEND
+
+   :man1:`flux` finds sub-command executables by searching:
+
+      $FLUX_EXEC_PATH_PREPEND : install-path : $FLUX_EXEC_PATH
+
+   Values may include multiple directories separated by colons.
+
+.. envvar:: FLUX_MODULE_PATH
+            FLUX_MODULE_PATH_PREPEND
+
+   :envvar:`FLUX_MODULE_PATH` is set in the environment of the broker
+   so that broker modules can be found and loaded when requested by
+   :man1:`flux-module`:
+
+      $FLUX_MODULE_PATH_PREPEND : install-path : $FLUX_MODULE_PATH
+
+   Values may include multiple directories separated by colons.
+
+.. envvar:: FLUX_CONNECTOR_PATH
+            FLUX_CONNECTOR_PATH_PREPEND
+
+   :envvar:`FLUX_CONNECTOR_PATH` is set in the environment of sub-commands so
+   that :man3:`flux_open` can find the connector corresponding to the URI
+   scheme:
+
+      $FLUX_CONNECTOR_PATH_PREPEND : install-path : $FLUX_CONNECTOR_PATH
+
+   Values may include multiple directories separated by colons.
+
+.. envvar:: PYTHONPATH
+            FLUX_PYTHONPATH_PREPEND
+
+   :envvar:`PYTHONPATH` is set so that sub-commands can find required Python
+   libraries:
+
+      $FLUX_PYTHONPATH_PREPEND : install-path : $PYTHONPATH
+
+   Values may include multiple directories separated by colons.
+
+.. note::
+   Flux commands written in Python further modify Python's
+   `sys.path <https://docs.python.org/3/library/sys.html#sys.path>`_
+   to ensure that interpreter default paths appear before any custom values
+   set in :envvar:`PYTHONPATH`. This is an attempt to  avoid incompatible
+   modules interfering with the operation of Flux commands. If it becomes
+   necessary to force a non-standard module first in the search path (e.g.
+   for testing, instrumentation, etc.) then :envvar:`FLUX_PYTHONPATH_PREPEND`
+   should be used.
+
+.. envvar:: LUA_PATH
+            LUA_CPATH
+            FLUX_LUA_PATH_PREPEND
+            FLUX_LUA_CPATH_PREPEND
+
+   :envvar:`LUA_PATH` and :envvar:`LUA_CPATH` are set so that sub-commands can
+   find required Lua libraries.  They are set, respectively, to
+
+      $FLUX_LUA_PATH_PREPEND ; install-path ; $LUA_PATH ;;
+
+      $FLUX_LUA_CPATH_PREPEND ; install-path ; $LUA_CPATH ;;
+
+   Values may include multiple directories separated by semicolons.
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+SEE ALSO
+========
+
+:man1:`flux-env`
diff --git a/doc/man7/flux-jobtap-plugins.rst b/doc/man7/flux-jobtap-plugins.rst
new file mode 100644
index 000000000000..40edbf5c6381
--- /dev/null
+++ b/doc/man7/flux-jobtap-plugins.rst
@@ -0,0 +1,446 @@
+======================
+flux-jobtap-plugins(7)
+======================
+
+
+DESCRIPTION
+===========
+
+The *jobtap* interface supports loading of builtin and external
+plugins into the job manager broker module. These plugins can be used
+to assign job priorities using algorithms other than the default,
+assign job dependencies, aid in debugging of the flow of job states,
+or generically extend the functionality of the job manager.
+
+Jobtap plugins are defined using the Flux standard plugin format. Therefore
+a jobtap plugin should export the single symbol: ``flux_plugin_init()``,
+from which calls to ``flux_plugin_add_handler(3)`` should be used to
+register functions which will be called for the callback topic strings
+described in the :ref:`callback_topics` section below.
+
+Each callback function uses the Flux standard plugin callback form, e.g.::
+
+   int callback (flux_plugin_t *p,
+                 const char *topic,
+                 flux_plugin_arg_t *args,
+                 void *arg);
+
+where ``p`` is the handle for the current *jobtap* plugin, ``topic`` is
+the *topic string* for the currently invoked callback, ``args`` contains
+a set of plugin arguments which may be unpacked with the
+``flux_plugin_arg_unpack(3)`` call, and ``arg`` is any opaque argument
+passed along when registering the handler.
+
+Multiple plugins may be loaded in the job-manager simultaneously. In this
+case, all matching handlers are called in all loaded plugins in the order
+in which they were loaded. For more information about loading plugins
+see the :man5:`flux-conf-job-manager` or :man1:`flux-jobtap` manpage.
+
+JOBTAP PLUGIN NAMES
+===================
+
+Jobtap plugins are loaded into the job-manager and referenced in the
+output of ``flux jobtap list`` by file name. If a plugin is loaded by
+a fully qualified path, the plugin name is shortened to the basename,
+such that all dynamically loaded plugins have names such as
+``plugin-name.so``.
+
+Builtin plugins, on the other hand, are named with a leading ``.``,
+and are hidden in ``flux jobtap list``, do not match the
+:linux:man7:`glob` ``*`` or "all" keyword, etc. (similar to hidden
+filesystem files).  To list builtin plugins, use the ``-a, --all``
+option to ``flux jobtap list``, and to remove them use the name
+explicitly or include the leading ``.`` in any pattern.
+
+A plugin may optionally assign a name with ``flux_plugin_set_name(3)``,
+however this name is not displayed in ``flux jobtap list`` or used in
+matching. The internal plugin name is only used as part of the service
+name generated by ``flux_jobtap_service_register()``, i.e. the service
+name will be ``job-manager.<name>.<method>``. If a plugin does not
+set a name with ``flux_plugin_set_name(3)``, then the basename of the
+plugin file will be used with the trailing ``.so`` removed.
+
+JOBTAP PLUGIN ARGUMENTS
+=======================
+
+For job-specific callbacks, all job data is passed to the plugin via
+the ``flux_plugin_arg_t *args``, and return data is sent back to the
+job manager via the same ``args``. Incoming arguments may be unpacked
+using ``flux_plugin_arg_unpack(3)``, e.g.::
+
+   rc = flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                                "{s{s:o}, s:I}",
+                                "jobspec", "resources", &resources,
+                                "id", &id);
+
+will unpack the ``resources`` section of jobspec and the jobid into
+``resources`` and ``id`` respectively.
+
+The full list of available args includes the following:
+
+========== ==== ==========================================
+name       type description
+========== ==== ==========================================
+jobspec    o    jobspec with environment redacted
+R          o    R with scheduling key redacted (RUN state or later)
+id         I    jobid
+state      i    current job state
+prev_state i    previous state (``job.state.*`` callbacks)
+userid     i    userid
+urgency    i    current urgency
+priority   I    current priority
+t_submit   f    submit timestamp in floating point seconds
+entry      o    posted eventlog entry, including context
+========== ==== ==========================================
+
+Return arguments can be packed using the ``FLUX_PLUGIN_ARG_OUT`` and
+optionally ``FLUX_PLUGIN_ARG_REPLACE`` flags. For example to return
+a priority::
+
+   rc = flux_plugin_arg_pack (args, FLUX_PLUGIN_ARG_OUT,
+                              "{s:I}",
+                              "priority", (int64_t) priority);
+
+While a job is pending, *jobtap* plugin callbacks may also add job
+annotations by returning a value for the ``annotations`` key::
+
+   flux_plugin_arg_pack (args, FLUX_PLUGIN_ARG_OUT,
+                         "{s:{s:s}}",
+                         "annotations", "test", value);
+
+.. _callback_topics:
+
+JOB CALLBACK TOPICS
+===================
+
+The following job callback "topic strings" are currently provided by the
+*jobtap* interface:
+
+job.create
+  The ``job.create`` topic notifies a jobtap plugin about a newly introduced
+  job. This call may be made in three different situations:
+
+    1. on job submission
+    2. when the job manager is restarted and has reloaded a job from the KVS
+    3. when a new jobtap plugin is loaded
+
+  In case 1 above, the job state will always be ``FLUX_JOB_STATE_NEW``, while
+  jobs in cases 2 and 3 can be in any state except ``FLUX_JOB_STATE_INACTIVE``.
+
+  In case 1, the job is not yet validated.  If necessary, ``job.create`` may
+  reject the job in the same manner as ``job.validate`` using
+  :man3:`flux_jobtap_reject_job` and a negative return code from the callback.
+
+  In cases 2 and 3, fatal errors may be handled by raising a fatal job
+  exception, as usual.
+
+  It is safe to post events from a ``job.create`` handler in all cases.
+
+job.destroy
+  The ``job.destroy`` topic is called after a job is rejected or becomes
+  inactive.
+
+job.validate
+  The ``job.validate`` topic allows a plugin to reject a job before
+  it is introduced to the job manager. A rejected job will result in
+  a job submission error in the submitting client, and any job data in
+  the KVS will be purged. No further callbacks except ``job.destroy``
+  will be made for rejected jobs. Note: If a job is not rejected, then
+  the ``job.new`` callback will be invoked immediately after ``job.validate``.
+  This allows limits or other checks to be implemented in the ``job.validate``
+  callback, but accounting for those limits should be confined to the
+  ``job.new`` callback, since ``job.new`` may also be called during job-manager
+  restart or plugin reload.
+
+job.dependency.*
+  The ``job.dependency.*`` topic allows a dependency plugin to notify the
+  job-manager that it handles a given dependency _scheme_. The job-manager
+  will scan the ``attributes.system.dependencies`` array, if provided, and
+  issue a ``job.dependency.SCHEME`` callback for each listed dependency.
+  If no plugin has registered for ``SCHEME``, then the job is rejected.
+  The plugin should then call ``flux_jobtap_dependency_add(3)`` to add
+  a new named dependency to the job (if necessary). Jobs with dependencies
+  will remain in the ``DEPEND`` state until all dependencies are removed
+  with a corresponding call to ``flux_jobtap_dependency_remove(3)``. See
+  ``job.state.depend`` below for more information about dependencies.
+  If there is an error in the dependency specification, the job may be
+  rejected with :man3:`flux_jobtap_reject_job` and a negative return code
+  from the callback.
+
+job.new
+  The ``job.new`` topic announces a new valid job.  It may be called in the
+  same three situations listed for ``job.create``,
+
+job.state.*
+  The ``job.state.*`` callbacks are made just after a job state transition.
+  The callback is made after the state has been published to the job's
+  eventlog, but before any action has been taken on that state (since the
+  action could involve immediately transitioning to a new state)
+
+job.event.*
+  The ``job.event.*`` callbacks are only made for plugins that have explicitly
+  subscribed to a job with ``flux_jobtap_job_subscribe()``. In this case,
+  all job events result in this callback being invoked on all subscribed
+  plugins. This may be useful for plugins to get notification of events
+  that do not necessarily result in a state transition, e.g. the ``start``
+  event or a non-fatal ``exception``.
+
+job.state.depend
+  The callback for ``FLUX_JOB_STATE_DEPEND`` is the final place from which
+  a plugin may add dependencies to a job. Dependencies are added via
+  the ``flux_jobtap_dependency_add()`` function. This function allows a
+  named dependency to be attached to a job. Jobs with dependencies will
+  remain in the ``DEPEND`` state until all dependencies are removed with
+  a corresponding call the ``flux_jobtap_dependency_remove()``. A dependency
+  may only be used once. A second call to ``flux_jobtap_dependency_add()``
+  with the same dependency description will return ``EEXIST``, even if
+  the dependency was subsequently removed. (This allows idempotent operation
+  of plugin-managed dependencies for job-manager or plugin restart).
+
+job.state.priority
+  The callback for ``FLUX_JOB_STATE_PRIORITY`` is special, in that a plugin
+  must return a priority at the end of the callback (if the plugin is
+  a priority-managing plugin). If the job priority is not available, the
+  plugin should use ``flux_jobtap_priority_unavail()`` to indicate
+  that the priority cannot be set. Jobs that do not have a priority due
+  to unavailable priority or when no current priority plugin is loaded will
+  remain in the PRIORITY state until a priority is assigned. Therefore,
+  a plugin should arrange for the priority to be set asynchronously using 
+  ``flux_jobtap_reprioritize_job()``. See the :ref:`priority` section
+  for more detailed information about plugin management of job priority.
+
+job.state.sched
+  In the callback for ``FLUX_JOB_STATE_SCHED`` a plugin may set ``R``
+  in output args. In this case, if an ``R`` is not already assigned, then
+  this will force ``R`` for the current job and bypass the scheduler.
+
+job.priority.get
+  The job manager calls the ``job.priority.get`` topic whenever it wants
+  to update the job priority of a single job. The plugin should return a
+  priority immediately, but if one is not available when a job is in
+  the PRIORITY state, the plugin may use ``flux_jobtap_priority_unavail()``
+  to indicate the priority is not available. Returning an unavailable
+  priority in the SCHED state is an error and it will be logged, but
+  otherwise ignored. A call of ``job.priority.get`` can be requested for
+  all jobs by calling ``flux_jobtap_reprioritize_all()``. See the
+  :ref:`priority` section for more information about plugin management
+  of job priority.
+
+job.inactive-add
+  The job has transitioned to INACTIVE state and has been added to the
+  inactive hash.
+
+job.inactive-remove
+  The job has been purged from the inactive hash.
+
+job.update
+  The job has been updated with an RFC 21 ``jobspec-update`` event.
+
+
+CONFIGURATION CALLBACK TOPIC
+============================
+
+Jobtap plugins may register a ``conf.update`` callback.  The current/proposed
+configuration object is present in the input arguments under the ``conf`` key.
+The callback is invoked in the following circumstances:
+
+  - When the plugin is first loaded.  If the callback returns failure,
+    the plugin load fails.
+
+  - Each time the configuration changes.  If the callback returns failure,
+    ``flux config reload`` fails.
+
+The callback should return 0 on success, and -1 on failure.  On failure,
+it may optionally set a human readable error string in the ``errstr`` output
+argument.  The ``flux_jobtap_error()`` convenience function may be useful here.
+
+JOB UPDATE CALLBACKS
+====================
+
+The job manager allows updates of select job attributes through a
+plugin-based scheme. Plugins may register a callback topic matching
+``job.update.KEY``, where ``KEY`` is a period-delimited jobspec attribute,
+e.g. ``job.update.attributes.system.duration``. The requested updates are
+passed as an additional argument to the plugin in the ``updates`` key.
+
+The purpose of ``job.update.*`` callbacks to enable plugins to allow or
+deny the update of specific job attributes. Updates are denied by default
+unless a callback exists for the updated attribute and the plugin returns 0
+from the callback. Plugins deny an attribute update by returning -1 from
+the callback, and may optionally set an error message to return to the
+user with ``flux_jobtap_error(3)``.
+
+After all updates in a request are allowed by plugins, then the updated
+jobspec is passed through the ``job.validate`` plugin stack to ensure the
+result is valid. Plugins can note that an update is already validated by
+setting a ``validated`` flag in the ``FLUX_PLUGIN_OUT_ARGS``. If all updated
+attributes have this flag then this validation step is skipped. This can
+be useful to allow an instance owner to update a job attribute beyond limits
+for example.
+
+Some updates may benefit from a job feasibility check before the updates
+are applied. This prevents a user from inadvertently causing a job that
+was feasible at the time of submission to become infeasible through an
+update.  Because the update plugin is in the best position to determine
+if a feasibility check should be completed for an update, feasibility
+checks are only done if a ``feasibility`` flag in ``FLUX_PLUGIN_OUT_ARGS``
+is set. If any plugin for a set of updates requires a feasibility check,
+then feasibility of the updated jobspec as a whole will be checked. If
+the updated job is determined to be infeasible, then the update is aborted
+and an error returned to the user.
+
+The update of one attribute may require modification of other attributes.
+For example, an update of ``attributes.system.queue`` may require
+modification of ``attributes.system.constraints`` to apply the constraints
+of the new queue. To support this use case, plugins may additionally push
+an ``updates`` object onto ``FLUX_PLUGIN_OUT_ARGS``. This object has the
+same form as the ``jobspec-update`` context defined in RFC 21. For example,
+if a plugin wishes to update ``attributes.system.foo`` to 1, it can set ::
+
+   {"updates": {"attributes.system.foo": 1}}
+
+in the ``FLUX_PLUGIN_OUT_ARGS`` before returning. Updates are applied by
+updating the requested updates, so this method could overwrite other user-
+requested updates and caution is advised.
+
+PLUGIN CALLBACK TOPICS
+======================
+
+plugin.query
+  The job manager calls the ``plugin.query`` callback topic to give a plugin
+  the opportunity to provide extra data in response to a ``jobtap-query``
+  request (as used by the ``flux jobtap query PLUGIN`` command). This can be
+  used by a plugin to export internal plugin state for inspection by an admin
+  or user by placing the data in the output arguments of the callback, e.g.::
+
+    flux_plugin_arg_pack (p, FLUX_PLUGIN_ARG_OUT,
+                          "{s:O}"
+                          "data", internal_data);
+
+.. _priority:
+
+PRIORITY
+========
+
+Custom assignment of job priority values is one of the core
+features supported by the jobtap plugin interface. A builtin
+``.priority-default`` plugin is always loaded in the job-manager to
+ensure that jobs move past the PRIORITY state when no other priority
+plugin is loaded. The default plugin simply assigns the priority to
+the same value as the current job urgency.
+
+When loading a new jobtap plugin that assigns priority, it is important
+to be cognizant of the fact that the ``.priority-default`` plugin may
+still be loaded. This will result in the ``priority`` set in the return
+arguments to always be initialized to the job urgency. However, since
+plugin ``job.state.priority`` and ``job.priority.get`` callbacks are
+run in order, any subsequently loaded plugin that assigns a priority
+will overwrite the returned default ``priority`` and thus the last
+loaded priority plugin will be active.
+
+To ensure the default priority is always overridden priority plugins
+should therefore make sure to always set a priority, or use
+``flux_jobtap_priority_unavail()`` if the priority is not available,
+in any callback in which a priority is expected to be returned, i.e.
+``job.state.priority`` and ``job.priority.get``.
+
+To fully ensure priority plugins do not conflict, the builtin priority
+plugin may explicitly be removed with
+
+::
+
+    flux jobtap remove .priority-default
+
+or via configuration (See :man5:`flux-conf-job-manager`)
+
+::
+
+   [job-manager]
+   plugins = [
+     { remove = ".priority-default",
+       load = "complex-priority.so"
+     },
+   ]
+
+
+.. _perilogs:
+
+PROLOG AND EPILOG ACTIONS
+=========================
+
+Plugins that need to perform asynchronous tasks for jobs after an ``alloc``
+event but before the job is running, or after a ``finish`` event but before
+resources are freed to the scheduler can make use of job manager prolog or
+epilog actions.
+
+Prolog and epilog actions are delineated by the following functions:
+
+::
+
+   int flux_jobtap_prolog_start (flux_plugin_t *p,
+                                 const char *description);
+
+   int flux_jobtap_prolog_finish (flux_plugin_t *p,
+                                  flux_jobid_t id,
+                                  const char *description,
+                                  int status);
+
+   int flux_jobtap_epilog_start (flux_plugin_t *p,
+                                 const char *description);
+
+   int flux_jobtap_epilog_finish (flux_plugin_t *p,
+                                  flux_jobid_t id,
+                                  const char *description,
+                                  int status);
+
+To initiate a prolog action, a plugin should call the function
+``flux_jobtap_prolog_start()``. This will block the job from starting
+even after resources have been assigned until a corresponding call to
+``flux_jobtap_prolog_finish()`` has been called. While the status of the
+prolog action is passed to ``flux_jobtap_prolog_finish()`` so it can be
+captured in the eventlog, the action itself is responsible for raising
+a job exception or taking other action on failure. That is, a non-zero
+prolog finish status does not cause any automated behavior on the part of
+the job manager. Similarly, the prolog ``description`` is used for
+informational purposes only, so that multiple actions in an eventlog
+may be differentiated.
+
+Similarly, an epilog action is initiated with ``flux_jobtap_epilog_start()``,
+and prevents resources from being released to the scheduler until a
+corresponding call to ``flux_jobtap_epilog_finish()``. The same caveats
+described for prolog actions regarding description and completion status
+of epilog actions apply.
+
+The ``flux_jobtap_prolog_start()`` function may be initiated anytime
+before the ``start`` request is made to the execution system, though most
+often from the ``job.state.run`` or ``job.event.alloc`` callbacks,
+since this is the point at which a job has been allocated resources.
+(Note: plugins will only receive the ``job.event.*`` callbacks for
+jobs to which they have subscribed with a call to
+``flux_jobtap_job_subscribe()``). A prolog action cannot be started
+after a job enters the CLEANUP state.
+
+The ``flux_jobtap_epilog_start()`` function may only be called after a
+job is in the CLEANUP state, but before the ``free`` request has been
+sent to the scheduler, for example from the ``job.state.cleanup``
+or ``job.event.finish`` callbacks.
+
+If ``flux_jobtap_prolog_start()``, ``flux_jobtap_prolog_finish()``,
+``flux_jobtap_epilog_start()`` or ``flux_jobtap_epilog_finish()`` are
+called for a job in an invalid state, these function will return -1 with
+``errno`` set to ``EINVAL``.
+
+Multiple prolog or epilog actions can be active at the same time.
+
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+
+SEE ALSO
+========
+
+:man1:`flux-jobtap`, :man5:`flux-conf-job-manager`
+
diff --git a/doc/man7/flux-undocumented.rst b/doc/man7/flux-undocumented.rst
new file mode 100644
index 000000000000..24b94b27a77e
--- /dev/null
+++ b/doc/man7/flux-undocumented.rst
@@ -0,0 +1,15 @@
+====================
+flux-undocumented(7)
+====================
+
+
+DESCRIPTION
+===========
+
+This documentation does not exist
+Apparently something was missed
+We might need reminding
+Of the page you're not finding
+So sorry, please don't be pissed
+
+Issue or PR to http://github.com/flux-framework/flux-core welcome.
diff --git a/doc/man7/index.rst b/doc/man7/index.rst
new file mode 100644
index 000000000000..e0faf6996c2c
--- /dev/null
+++ b/doc/man7/index.rst
@@ -0,0 +1,8 @@
+Section 7 - Flux Miscellany
+===========================
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   *
diff --git a/doc/manpages.py b/doc/manpages.py
new file mode 100644
index 000000000000..b5f784fb7c19
--- /dev/null
+++ b/doc/manpages.py
@@ -0,0 +1,365 @@
+###############################################################
+# Copyright 2022 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+author = 'This page is maintained by the Flux community.'
+
+# Add man page entries with the following information:
+# - Relative file path (without .rst extension)
+# - Man page name
+# - Man page description
+# - Author (use [author])
+# - Manual section
+#
+# Note: the relative order of commands in this list affects the order
+# in which commands appear within a section in the output of flux help.
+# Therefore, keep these commands in relative order of importance.
+#
+man_pages = [
+    ('man1/flux-broker', 'flux-broker', 'Flux message broker daemon', [author], 1),
+    ('man1/flux-start', 'flux-start', 'bootstrap a local Flux instance', [author], 1),
+    ('man1/flux-version', 'flux-version', 'Display flux version information', [author], 1),
+    ('man1/flux-config', 'flux-config', 'Manage/query Flux configuration', [author], 1),
+    ('man1/flux-content', 'flux-content', 'access content service', [author], 1),
+    ('man1/flux-cron', 'flux-cron', 'Cron-like utility for Flux', [author], 1),
+    ('man1/flux-dmesg', 'flux-dmesg', 'access broker ring buffer', [author], 1),
+    ('man1/flux-dump', 'flux-dump', 'Write KVS snapshot to portable archive', [author], 1),
+    ('man1/flux-env', 'flux-env', 'Print the flux environment or execute a command inside it', [author], 1),
+    ('man1/flux-event', 'flux-event', 'Send and receive Flux events', [author], 1),
+    ('man1/flux-exec', 'flux-exec', 'Execute processes across flux ranks', [author], 1),
+    ('man1/flux-archive', 'flux-archive', 'KVS file archive utility', [author], 1),
+    ('man1/flux-getattr', 'flux-setattr', 'access broker attributes', [author], 1),
+    ('man1/flux-getattr', 'flux-lsattr', 'access broker attributes', [author], 1),
+    ('man1/flux-getattr', 'flux-getattr', 'access broker attributes', [author], 1),
+    ('man1/flux-jobs', 'flux-jobs', 'list jobs submitted to Flux', [author], 1),
+    ('man1/flux-top', 'flux-top', 'Display running Flux jobs', [author], 1),
+    ('man1/flux-pstree', 'flux-pstree', 'display job hierarchies', [author], 1),
+    ('man1/flux-cancel', 'flux-cancel', 'cancel one or more jobs', [author], 1),
+    ('man1/flux-pgrep', 'flux-pgrep', 'search or cancel matching jobs', [author], 1),
+    ('man1/flux-pgrep', 'flux-pkill', 'search or cancel matching jobs', [author], 1),
+    ('man1/flux-pmi', 'flux-pmi', 'PMI client test tool', [author], 1),
+    ('man1/flux-jobtap', 'flux-jobtap', 'List, remove, and load job-manager plugins', [author], 1),
+    ('man1/flux-shutdown', 'flux-shutdown', 'Shut down a Flux instance', [author], 1),
+    ('man1/flux-uri', 'flux-uri', 'resolve Flux URIs', [author], 1),
+    ('man1/flux-resource', 'flux-resource', 'list/manipulate Flux resource status', [author], 1),
+    ('man1/flux-queue', 'flux-queue', 'manage the job queue', [author], 1),
+    ('man1/flux-restore', 'flux-restore', 'Read KVS snapshot from portable archive', [author], 1),
+    ('man1/flux-keygen', 'flux-keygen', 'generate keys for Flux security', [author], 1),
+    ('man1/flux-kvs', 'flux-kvs', 'Flux key-value store utility', [author], 1),
+    ('man1/flux-logger', 'flux-logger', 'create a Flux log entry', [author], 1),
+    ('man1/flux-submit', 'flux-submit', 'submit a job to a Flux instance', [author], 1),
+    ('man1/flux-run', 'flux-run', 'run a Flux job interactively', [author], 1),
+    ('man1/flux-bulksubmit', 'flux-bulksubmit', 'submit jobs in bulk to a Flux instance', [author], 1),
+    ('man1/flux-alloc', 'flux-alloc', 'allocate a new Flux instance for interactive use', [author], 1),
+    ('man1/flux-batch', 'flux-batch', 'submit a batch script to Flux', [author], 1),
+    ('man1/flux-job', 'flux-job', 'Job Housekeeping Tool', [author], 1),
+    ('man1/flux-module', 'flux-module', 'manage Flux extension modules', [author], 1),
+    ('man1/flux-overlay', 'flux-overlay', 'Show flux overlay network status', [author], 1),
+    ('man1/flux-uptime', 'flux-uptime', 'Tell how long Flux has been up and running', [author], 1),
+    ('man1/flux-ping', 'flux-ping', 'measure round-trip latency to Flux services', [author], 1),
+    ('man1/flux-proxy', 'flux-proxy', 'create proxy environment for Flux instance', [author], 1),
+    ('man1/flux-startlog', 'flux-startlog', 'Show Flux instance start and stop times', [author], 1),
+    ('man1/flux', 'flux', 'the Flux resource management framework', [author], 1),
+    ('man1/flux-shell', 'flux-shell', 'the Flux job shell', [author], 1),
+    ('man1/flux-watch', 'flux-watch', 'monitor one or more Flux jobs', [author], 1),
+    ('man1/flux-update', 'flux-update', 'update active Flux jobs', [author], 1),
+    ('man1/flux-hostlist', 'flux-hostlist', 'fetch, combine, and manipulate Flux hostlists', [author], 1),
+    ('man1/flux-housekeeping', 'flux-housekeeping', 'list and terminate housekeeping tasks', [author], 1),
+    ('man3/flux_attr_get', 'flux_attr_set', 'get/set Flux broker attributes', [author], 3),
+    ('man3/flux_attr_get', 'flux_attr_get', 'get/set Flux broker attributes', [author], 3),
+    ('man3/flux_aux_set', 'flux_aux_get', 'get/set auxiliary handle data', [author], 3),
+    ('man3/flux_aux_set', 'flux_aux_set', 'get/set auxiliary handle data', [author], 3),
+    ('man3/flux_child_watcher_create', 'flux_child_watcher_get_rpid', 'create child watcher', [author], 3),
+    ('man3/flux_child_watcher_create', 'flux_child_watcher_get_rstatus', 'create child watcher', [author], 3),
+    ('man3/flux_child_watcher_create', 'flux_child_watcher_create', 'create child watcher', [author], 3),
+    ('man3/flux_core_version', 'flux_core_version_string', 'get flux-core version', [author], 3),
+    ('man3/flux_core_version', 'flux_core_version', 'get flux-core version', [author], 3),
+    ('man3/flux_event_decode', 'flux_event_decode_raw', 'encode/decode a Flux event message', [author], 3),
+    ('man3/flux_event_decode', 'flux_event_unpack', 'encode/decode a Flux event message', [author], 3),
+    ('man3/flux_event_decode', 'flux_event_encode', 'encode/decode a Flux event message', [author], 3),
+    ('man3/flux_event_decode', 'flux_event_encode_raw', 'encode/decode a Flux event message', [author], 3),
+    ('man3/flux_event_decode', 'flux_event_pack', 'encode/decode a Flux event message', [author], 3),
+    ('man3/flux_event_decode', 'flux_event_decode', 'encode/decode a Flux event message', [author], 3),
+    ('man3/flux_event_publish', 'flux_event_publish_pack', 'publish events', [author], 3),
+    ('man3/flux_event_publish', 'flux_event_publish_raw', 'publish events', [author], 3),
+    ('man3/flux_event_publish', 'flux_event_publish_get_seq', 'publish events', [author], 3),
+    ('man3/flux_event_publish', 'flux_event_publish', 'publish events', [author], 3),
+    ('man3/flux_event_subscribe', 'flux_event_unsubscribe', 'manage subscriptions', [author], 3),
+    ('man3/flux_event_subscribe', 'flux_event_subscribe', 'manage subscriptions', [author], 3),
+    ('man3/flux_comms_error_set', 'flux_comms_error_set', 'register callback for communications errors', [author], 3),
+    ('man3/flux_fd_watcher_create', 'flux_fd_watcher_get_fd', 'create file descriptor watcher', [author], 3),
+    ('man3/flux_fd_watcher_create', 'flux_fd_watcher_create', 'create file descriptor watcher', [author], 3),
+    ('man3/flux_flags_set', 'flux_flags_unset', 'manipulate Flux handle flags', [author], 3),
+    ('man3/flux_flags_set', 'flux_flags_get', 'manipulate Flux handle flags', [author], 3),
+    ('man3/flux_flags_set', 'flux_flags_set', 'manipulate Flux handle flags', [author], 3),
+    ('man3/flux_future_and_then', 'flux_future_or_then', 'functions for sequential composition of futures', [author], 3),
+    ('man3/flux_future_and_then', 'flux_future_continue', 'functions for sequential composition of futures', [author], 3),
+    ('man3/flux_future_and_then', 'flux_future_continue_error', 'functions for sequential composition of futures', [author], 3),
+    ('man3/flux_future_and_then', 'flux_future_and_then', 'functions for sequential composition of futures', [author], 3),
+    ('man3/flux_future_create', 'flux_future_fulfill', 'support methods for classes that return futures', [author], 3),
+    ('man3/flux_future_create', 'flux_future_fulfill_error', 'support methods for classes that return futures', [author], 3),
+    ('man3/flux_future_create', 'flux_future_fulfill_with', 'support methods for classes that return futures', [author], 3),
+    ('man3/flux_future_create', 'flux_future_aux_get', 'support methods for classes that return futures', [author], 3),
+    ('man3/flux_future_create', 'flux_future_aux_set', 'support methods for classes that return futures', [author], 3),
+    ('man3/flux_future_create', 'flux_future_set_flux', 'support methods for classes that return futures', [author], 3),
+    ('man3/flux_future_create', 'flux_future_get_flux', 'support methods for classes that return futures', [author], 3),
+    ('man3/flux_future_create', 'flux_future_create', 'support methods for classes that return futures', [author], 3),
+    ('man3/flux_future_get', 'flux_future_then', 'synchronize an activity', [author], 3),
+    ('man3/flux_future_get', 'flux_future_wait_for', 'synchronize an activity', [author], 3),
+    ('man3/flux_future_get', 'flux_future_reset', 'synchronize an activity', [author], 3),
+    ('man3/flux_future_get', 'flux_future_destroy', 'synchronize an activity', [author], 3),
+    ('man3/flux_future_get', 'flux_future_get', 'synchronize an activity', [author], 3),
+    ('man3/flux_future_wait_all_create', 'flux_future_wait_any_create', 'functions for future composition', [author], 3),
+    ('man3/flux_future_wait_all_create', 'flux_future_push', 'functions for future composition', [author], 3),
+    ('man3/flux_future_wait_all_create', 'flux_future_first_child', 'functions for future composition', [author], 3),
+    ('man3/flux_future_wait_all_create', 'flux_future_next_child', 'functions for future composition', [author], 3),
+    ('man3/flux_future_wait_all_create', 'flux_future_get_child', 'functions for future composition', [author], 3),
+    ('man3/flux_future_wait_all_create', 'flux_future_wait_all_create', 'functions for future composition', [author], 3),
+    ('man3/flux_get_rank', 'flux_get_size', 'query Flux broker info', [author], 3),
+    ('man3/flux_get_rank', 'flux_get_rank', 'query Flux broker info', [author], 3),
+    ('man3/flux_get_reactor', 'flux_set_reactor', 'get/set reactor associated with broker handle', [author], 3),
+    ('man3/flux_get_reactor', 'flux_get_reactor', 'get/set reactor associated with broker handle', [author], 3),
+    ('man3/flux_handle_watcher_create', 'flux_handle_watcher_get_flux', 'create broker handle watcher', [author], 3),
+    ('man3/flux_handle_watcher_create', 'flux_handle_watcher_create', 'create broker handle watcher', [author], 3),
+    ('man3/flux_idle_watcher_create', 'flux_prepare_watcher_create', 'create prepare/check/idle watchers', [author], 3),
+    ('man3/flux_idle_watcher_create', 'flux_check_watcher_create', 'create prepare/check/idle watchers', [author], 3),
+    ('man3/flux_idle_watcher_create', 'flux_idle_watcher_create', 'create prepare/check/idle watchers', [author], 3),
+    ('man3/flux_kvs_commit', 'flux_kvs_fence', 'commit a KVS transaction', [author], 3),
+    ('man3/flux_kvs_commit', 'flux_kvs_commit_get_treeobj', 'commit a KVS transaction', [author], 3),
+    ('man3/flux_kvs_commit', 'flux_kvs_commit_get_sequence', 'commit a KVS transaction', [author], 3),
+    ('man3/flux_kvs_commit', 'flux_kvs_commit', 'commit a KVS transaction', [author], 3),
+    ('man3/flux_kvs_copy', 'flux_kvs_move', 'copy/move a KVS key', [author], 3),
+    ('man3/flux_kvs_copy', 'flux_kvs_copy', 'copy/move a KVS key', [author], 3),
+    ('man3/flux_kvs_getroot', 'flux_kvs_getroot_get_treeobj', 'look up KVS root hash', [author], 3),
+    ('man3/flux_kvs_getroot', 'flux_kvs_getroot_get_blobref', 'look up KVS root hash', [author], 3),
+    ('man3/flux_kvs_getroot', 'flux_kvs_getroot_get_sequence', 'look up KVS root hash', [author], 3),
+    ('man3/flux_kvs_getroot', 'flux_kvs_getroot_get_owner', 'look up KVS root hash', [author], 3),
+    ('man3/flux_kvs_getroot', 'flux_kvs_getroot_cancel', 'look up KVS root hash', [author], 3),
+    ('man3/flux_kvs_getroot', 'flux_kvs_getroot', 'look up KVS root hash', [author], 3),
+    ('man3/flux_kvs_lookup', 'flux_kvs_lookupat', 'look up KVS key', [author], 3),
+    ('man3/flux_kvs_lookup', 'flux_kvs_lookup_get', 'look up KVS key', [author], 3),
+    ('man3/flux_kvs_lookup', 'flux_kvs_lookup_get_unpack', 'look up KVS key', [author], 3),
+    ('man3/flux_kvs_lookup', 'flux_kvs_lookup_get_raw', 'look up KVS key', [author], 3),
+    ('man3/flux_kvs_lookup', 'flux_kvs_lookup_get_dir', 'look up KVS key', [author], 3),
+    ('man3/flux_kvs_lookup', 'flux_kvs_lookup_get_treeobj', 'look up KVS key', [author], 3),
+    ('man3/flux_kvs_lookup', 'flux_kvs_lookup_get_symlink', 'look up KVS key', [author], 3),
+    ('man3/flux_kvs_lookup', 'flux_kvs_lookup', 'look up KVS key', [author], 3),
+    ('man3/flux_kvs_namespace_create', 'flux_kvs_namespace_create', 'create/remove a KVS namespace', [author], 3),
+    ('man3/flux_kvs_namespace_create', 'flux_kvs_namespace_remove', 'create/remove a KVS namespace', [author], 3),
+    ('man3/flux_kvs_txn_create', 'flux_kvs_txn_destroy', 'operate on a KVS transaction object', [author], 3),
+    ('man3/flux_kvs_txn_create', 'flux_kvs_txn_put', 'operate on a KVS transaction object', [author], 3),
+    ('man3/flux_kvs_txn_create', 'flux_kvs_txn_pack', 'operate on a KVS transaction object', [author], 3),
+    ('man3/flux_kvs_txn_create', 'flux_kvs_txn_vpack', 'operate on a KVS transaction object', [author], 3),
+    ('man3/flux_kvs_txn_create', 'flux_kvs_txn_mkdir', 'operate on a KVS transaction object', [author], 3),
+    ('man3/flux_kvs_txn_create', 'flux_kvs_txn_unlink', 'operate on a KVS transaction object', [author], 3),
+    ('man3/flux_kvs_txn_create', 'flux_kvs_txn_symlink', 'operate on a KVS transaction object', [author], 3),
+    ('man3/flux_kvs_txn_create', 'flux_kvs_txn_put_raw', 'operate on a KVS transaction object', [author], 3),
+    ('man3/flux_kvs_txn_create', 'flux_kvs_txn_put_treeobj', 'operate on a KVS transaction object', [author], 3),
+    ('man3/flux_kvs_txn_create', 'flux_kvs_txn_create', 'operate on a KVS transaction object', [author], 3),
+    ('man3/flux_log', 'flux_vlog', 'Log messages to the Flux Message Broker', [author], 3),
+    ('man3/flux_log', 'flux_log_set_appname', 'Log messages to the Flux Message Broker', [author], 3),
+    ('man3/flux_log', 'flux_log_set_procid', 'Log messages to the Flux Message Broker', [author], 3),
+    ('man3/flux_log', 'flux_log', 'Log messages to the Flux Message Broker', [author], 3),
+    ('man3/flux_msg_cmp', 'flux_msg_cmp', 'match a message', [author], 3),
+    ('man3/flux_msg_create', 'flux_msg_create', 'functions for Flux messages', [author], 3),
+    ('man3/flux_msg_create', 'flux_msg_copy', 'functions for Flux messages', [author], 3),
+    ('man3/flux_msg_create', 'flux_msg_incref', 'functions for Flux messages', [author], 3),
+    ('man3/flux_msg_create', 'flux_msg_decref', 'functions for Flux messages', [author], 3),
+    ('man3/flux_msg_create', 'flux_msg_destroy', 'functions for Flux messages', [author], 3),
+    ('man3/flux_msg_encode', 'flux_msg_decode', 'convert a Flux message to buffer and back again', [author], 3),
+    ('man3/flux_msg_encode', 'flux_msg_encode', 'convert a Flux message to buffer and back again', [author], 3),
+    ('man3/flux_msg_has_flag', 'flux_msg_has_flag', 'test/set Flux message flags', [author], 3),
+    ('man3/flux_msg_has_flag', 'flux_msg_set_flag', 'test/set Flux message flags', [author], 3),
+    ('man3/flux_msg_has_flag', 'flux_msg_clear_flag', 'test/set Flux message flags', [author], 3),
+    ('man3/flux_msg_has_flag', 'flux_msg_is_streaming', 'test/set Flux message flags', [author], 3),
+    ('man3/flux_msg_has_flag', 'flux_msg_set_streaming', 'test/set Flux message flags', [author], 3),
+    ('man3/flux_msg_has_flag', 'flux_msg_is_noresponse', 'test/set Flux message flags', [author], 3),
+    ('man3/flux_msg_has_flag', 'flux_msg_set_noresponse', 'test/set Flux message flags', [author], 3),
+    ('man3/flux_msg_has_flag', 'flux_msg_is_private', 'test/set Flux message flags', [author], 3),
+    ('man3/flux_msg_has_flag', 'flux_msg_set_private', 'test/set Flux message flags', [author], 3),
+    ('man3/flux_msg_handler_addvec', 'flux_msg_handler_delvec', 'bulk add/remove message handlers', [author], 3),
+    ('man3/flux_msg_handler_addvec', 'flux_msg_handler_addvec', 'bulk add/remove message handlers', [author], 3),
+    ('man3/flux_msg_handler_create', 'flux_msg_handler_destroy', 'manage message handlers', [author], 3),
+    ('man3/flux_msg_handler_create', 'flux_msg_handler_start', 'manage message handlers', [author], 3),
+    ('man3/flux_msg_handler_create', 'flux_msg_handler_stop', 'manage message handlers', [author], 3),
+    ('man3/flux_msg_handler_create', 'flux_msg_handler_create', 'manage message handlers', [author], 3),
+    ('man3/flux_open', 'flux_clone', 'open/close connection to Flux Message Broker', [author], 3),
+    ('man3/flux_open', 'flux_close', 'open/close connection to Flux Message Broker', [author], 3),
+    ('man3/flux_open', 'flux_open', 'open/close connection to Flux Message Broker', [author], 3),
+    ('man3/flux_open', 'flux_open_ex', 'open/close connection to Flux Message Broker', [author], 3),
+    ('man3/flux_open', 'flux_reconnect', 'open/close connection to Flux Message Broker', [author], 3),
+    ('man3/flux_periodic_watcher_create', 'flux_periodic_watcher_reset', 'set/reset a timer', [author], 3),
+    ('man3/flux_periodic_watcher_create', 'flux_periodic_watcher_create', 'set/reset a timer', [author], 3),
+    ('man3/flux_pollevents', 'flux_pollfd', 'poll Flux broker handle', [author], 3),
+    ('man3/flux_pollevents', 'flux_pollevents', 'poll Flux broker handle', [author], 3),
+    ('man3/flux_reactor_create', 'flux_reactor_destroy', 'create/destroy/control event reactor object', [author], 3),
+    ('man3/flux_reactor_create', 'flux_reactor_run', 'create/destroy/control event reactor object', [author], 3),
+    ('man3/flux_reactor_create', 'flux_reactor_stop', 'create/destroy/control event reactor object', [author], 3),
+    ('man3/flux_reactor_create', 'flux_reactor_stop_error', 'create/destroy/control event reactor object', [author], 3),
+    ('man3/flux_reactor_create', 'flux_reactor_active_incref', 'create/destroy/control event reactor object', [author], 3),
+    ('man3/flux_reactor_create', 'flux_reactor_active_decref', 'create/destroy/control event reactor object', [author], 3),
+    ('man3/flux_reactor_create', 'flux_reactor_create', 'create/destroy/control event reactor object', [author], 3),
+    ('man3/flux_reactor_now', 'flux_reactor_now_update', 'get/update reactor time', [author], 3),
+    ('man3/flux_reactor_now', 'flux_reactor_now', 'get/update reactor time', [author], 3),
+    ('man3/flux_recv', 'flux_recv', 'receive message using Flux Message Broker', [author], 3),
+    ('man3/flux_request_decode', 'flux_request_unpack', 'decode a Flux request message', [author], 3),
+    ('man3/flux_request_decode', 'flux_request_decode_raw', 'decode a Flux request message', [author], 3),
+    ('man3/flux_request_decode', 'flux_request_decode', 'decode a Flux request message', [author], 3),
+    ('man3/flux_request_encode', 'flux_request_encode_raw', 'encode a Flux request message', [author], 3),
+    ('man3/flux_request_encode', 'flux_request_encode', 'encode a Flux request message', [author], 3),
+    ('man3/flux_requeue', 'flux_requeue', 'requeue a message', [author], 3),
+    ('man3/flux_respond', 'flux_respond_pack', 'respond to a request', [author], 3),
+    ('man3/flux_respond', 'flux_respond_raw', 'respond to a request', [author], 3),
+    ('man3/flux_respond', 'flux_respond_error', 'respond to a request', [author], 3),
+    ('man3/flux_respond', 'flux_respond', 'respond to a request', [author], 3),
+    ('man3/flux_response_decode', 'flux_response_decode_raw', 'decode a Flux response message', [author], 3),
+    ('man3/flux_response_decode', 'flux_response_decode_error', 'decode a Flux response message', [author], 3),
+    ('man3/flux_response_decode', 'flux_response_decode', 'decode a Flux response message', [author], 3),
+    ('man3/flux_rpc', 'flux_rpc_pack', 'perform a remote procedure call to a Flux service', [author], 3),
+    ('man3/flux_rpc', 'flux_rpc_raw', 'perform a remote procedure call to a Flux service', [author], 3),
+    ('man3/flux_rpc', 'flux_rpc_message', 'perform a remote procedure call to a Flux service', [author], 3),
+    ('man3/flux_rpc', 'flux_rpc_get', 'perform a remote procedure call to a Flux service', [author], 3),
+    ('man3/flux_rpc', 'flux_rpc_get_unpack', 'perform a remote procedure call to a Flux service', [author], 3),
+    ('man3/flux_rpc', 'flux_rpc_get_raw', 'perform a remote procedure call to a Flux service', [author], 3),
+    ('man3/flux_rpc', 'flux_rpc', 'perform a remote procedure call to a Flux service', [author], 3),
+    ('man3/flux_rpc', 'flux_rpc_get_matchtag', 'perform a remote procedure call to a Flux service', [author], 3),
+    ('man3/flux_rpc', 'flux_rpc_get_nodeid', 'perform a remote procedure call to a Flux service', [author], 3),
+    ('man3/flux_send', 'flux_send', 'send message using Flux Message Broker', [author], 3),
+    ('man3/flux_send', 'flux_send_new', 'send message using Flux Message Broker', [author], 3),
+    ('man3/flux_service_register', 'flux_service_register', 'Register service with flux broker', [author], 3),
+    ('man3/flux_service_register', 'flux_service_unregister', 'Unregister service with flux broker', [author], 3),
+    ('man3/flux_shell_add_completion_ref', 'flux_shell_remove_completion_ref', 'Manipulate conditions for job completion.', [author], 3),
+    ('man3/flux_shell_add_completion_ref', 'flux_shell_add_completion_ref', 'Manipulate conditions for job completion.', [author], 3),
+    ('man3/flux_shell_add_event_context', 'flux_shell_add_event_context', 'Add context information for standard shell events', [author], 3),
+    ('man3/flux_shell_add_event_handler', 'flux_shell_add_event_handler', 'Add an event handler for a shell event', [author], 3),
+    ('man3/flux_shell_aux_set', 'flux_shell_aux_get', 'get/set auxiliary handle data', [author], 3),
+    ('man3/flux_shell_aux_set', 'flux_shell_aux_set', 'get/set auxiliary handle data', [author], 3),
+    ('man3/flux_shell_current_task', 'flux_shell_task_first', 'Get and iterate over shell tasks', [author], 3),
+    ('man3/flux_shell_current_task', 'flux_shell_task_next', 'Get and iterate over shell tasks', [author], 3),
+    ('man3/flux_shell_current_task', 'flux_shell_current_task', 'Get and iterate over shell tasks', [author], 3),
+    ('man3/flux_shell_get_flux', 'flux_shell_get_flux', 'Get a flux_t\* object from flux shell handle', [author], 3),
+    ('man3/flux_shell_get_hwloc_xml', 'flux_shell_get_hwloc_xml', 'Access the shell cached copy of local hwloc xml', [author], 3),
+    ('man3/flux_shell_get_info', 'flux_shell_info_unpack', 'Manage shell info and rank info', [author], 3),
+    ('man3/flux_shell_get_info', 'flux_shell_get_rank_info', 'Manage shell info and rank info', [author], 3),
+    ('man3/flux_shell_get_info', 'flux_shell_get_rank_info', 'Manage shell info and rank info', [author], 3),
+    ('man3/flux_shell_get_info', 'flux_shell_rank_info_unpack', 'Manage shell info and rank info', [author], 3),
+    ('man3/flux_shell_get_info', 'flux_shell_get_info', 'Manage shell info and rank info', [author], 3),
+    ('man3/flux_shell_get_jobspec_info', 'flux_shell_jobspec_info_unpack', 'Manage shell jobspec summary information', [author], 3),
+    ('man3/flux_shell_get_jobspec_info', 'flux_shell_get_jobspec_info', 'Manage shell jobspec summary information', [author], 3),
+    ('man3/flux_shell_get_taskmap', 'flux_shell_get_taskmap', 'Get shell task mapping', [author], 3),
+    ('man3/flux_shell_get_hostlist', 'flux_shell_get_hostlist', 'Get the list of hosts in the current job', [author], 3),
+    ('man3/flux_shell_getenv', 'flux_shell_get_environ', 'Get and set global environment variables', [author], 3),
+    ('man3/flux_shell_getenv', 'flux_shell_setenvf', 'Get and set global environment variables', [author], 3),
+    ('man3/flux_shell_getenv', 'flux_shell_unsetenv', 'Get and set global environment variables', [author], 3),
+    ('man3/flux_shell_getenv', 'flux_shell_getenv', 'Get and set global environment variables', [author], 3),
+    ('man3/flux_shell_getopt', 'flux_shell_getopt_unpack', 'Get and set shell options', [author], 3),
+    ('man3/flux_shell_getopt', 'flux_shell_setopt', 'Get and set shell options', [author], 3),
+    ('man3/flux_shell_getopt', 'flux_shell_setopt_pack', 'Get and set shell options', [author], 3),
+    ('man3/flux_shell_getopt', 'flux_shell_getopt', 'Get and set shell options', [author], 3),
+    ('man3/flux_shell_killall', 'flux_shell_killall', 'Send the specified signal to all processes in the shell', [author], 3),
+    ('man3/flux_shell_log', 'flux_shell_err', 'Log shell plugin messages to registered shell loggers', [author], 3),
+    ('man3/flux_shell_log', 'flux_shell_fatal', 'Log shell plugin messages to registered shell loggers', [author], 3),
+    ('man3/flux_shell_log', 'flux_shell_raise', 'Log shell plugin messages to registered shell loggers', [author], 3),
+    ('man3/flux_shell_log', 'flux_shell_log_setlevel', 'Log shell plugin messages to registered shell loggers', [author], 3),
+    ('man3/flux_shell_log', 'flux_shell_log', 'Log shell plugin messages to registered shell loggers', [author], 3),
+    ('man3/flux_shell_plugstack_call', 'flux_shell_plugstack_call', 'Calls the function referenced by topic.', [author], 3),
+    ('man3/flux_shell_rpc_pack', 'flux_shell_rpc_pack', 'perform an rpc to a running flux shell using Jansson style pack arguments', [author], 3),
+    ('man3/flux_shell_service_register', 'flux_shell_service_register', r'Register a service handler for \`method\` in the shell.', [author], 3),
+    ('man3/flux_shell_task_channel_subscribe', 'flux_shell_task_channel_subscribe', r'Call \`cb\` when \`name\` is ready for reading.', [author], 3),
+    ('man3/flux_shell_task_get_info', 'flux_shell_task_info_unpack', 'interfaces for fetching task info', [author], 3),
+    ('man3/flux_shell_task_get_info', 'flux_shell_task_get_info', 'interfaces for fetching task info', [author], 3),
+    ('man3/flux_shell_task_subprocess', 'flux_shell_task_cmd', 'return the subprocess and cmd structure of a shell task, respectively', [author], 3),
+    ('man3/flux_shell_task_subprocess', 'flux_shell_task_subprocess', 'return the subprocess and cmd structure of a shell task, respectively', [author], 3),
+    ('man3/flux_signal_watcher_create', 'flux_signal_watcher_get_signum', 'create signal watcher', [author], 3),
+    ('man3/flux_signal_watcher_create', 'flux_signal_watcher_create', 'create signal watcher', [author], 3),
+    ('man3/flux_stat_watcher_create', 'flux_stat_watcher_get_rstat', 'create stat watcher', [author], 3),
+    ('man3/flux_stat_watcher_create', 'flux_stat_watcher_create', 'create stat watcher', [author], 3),
+    ('man3/flux_timer_watcher_create', 'flux_timer_watcher_reset', 'set/reset a timer', [author], 3),
+    ('man3/flux_timer_watcher_create', 'flux_timer_watcher_create', 'set/reset a timer', [author], 3),
+    ('man3/flux_watcher_start', 'flux_watcher_stop', 'start/stop/destroy/query reactor watcher', [author], 3),
+    ('man3/flux_watcher_start', 'flux_watcher_is_active', 'start/stop/destroy/query reactor watcher', [author], 3),
+    ('man3/flux_watcher_start', 'flux_watcher_destroy', 'start/stop/destroy/query reactor watcher', [author], 3),
+    ('man3/flux_watcher_start', 'flux_watcher_next_wakeup', 'start/stop/destroy/query reactor watcher', [author], 3),
+    ('man3/flux_watcher_start', 'flux_watcher_start', 'start/stop/destroy/query reactor watcher', [author], 3),
+    ('man3/flux_watcher_set_priority', 'flux_watcher_set_priority', 'set watcher priority', [author], 3),
+    ('man3/hostlist_create', 'hostlist_create', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_destroy', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_decode', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_encode', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_copy', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_append', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_append_list', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_nth', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_find', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_delete', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_count', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_sort', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_uniq', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_first', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_last', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_next', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_current', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/hostlist_create', 'hostlist_remove_current', 'Manipulate lists of hostnames', [author], 3),
+    ('man3/idset_create', 'idset_create', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_destroy', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_copy', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_set', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_range_set', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_clear', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_range_clear', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_test', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_first', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_next', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_prev', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_empty', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_universe_size', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_last', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_create', 'idset_count', 'Manipulate numerically sorted sets of non-negative integers', [author], 3),
+    ('man3/idset_encode','idset_encode', 'Convert idset to string', [author], 3),
+    ('man3/idset_decode','idset_decode', 'Convert string to idset', [author], 3),
+    ('man3/idset_decode','idset_decode_ex', 'Convert string to idset', [author], 3),
+    ('man3/idset_decode','idset_decode_empty', 'Convert string to idset', [author], 3),
+    ('man3/idset_decode','idset_decode_info', 'Convert string to idset', [author], 3),
+    ('man3/idset_decode','idset_decode_add', 'Convert string to idset', [author], 3),
+    ('man3/idset_decode','idset_decode_subtract', 'Convert string to idset', [author], 3),
+    ('man3/idset_add', 'idset_equal', 'Perform set operations on idsets', [author], 3),
+    ('man3/idset_add','idset_union', 'Perform set operations on idsets', [author], 3),
+    ('man3/idset_add','idset_add', 'Perform set operations on idsets', [author], 3),
+    ('man3/idset_add','idset_difference', 'Perform set operations on idsets', [author], 3),
+    ('man3/idset_add','idset_subtract', 'Perform set operations on idsets', [author], 3),
+    ('man3/idset_add','idset_intersect', 'Perform set operations on idsets', [author], 3),
+    ('man3/idset_add','idset_has_intersection', 'Perform set operations on idsets', [author], 3),
+    ('man3/idset_add','idset_clear_all', 'Perform set operations on idsets', [author], 3),
+    ('man3/idset_alloc','idset_alloc', 'Allocate an id from an idset', [author], 3),
+    ('man3/idset_alloc','idset_free', 'Allocate an id from an idset', [author], 3),
+    ('man3/idset_alloc','idset_free_check', 'Allocate an id from an idset', [author], 3),
+    ('man3/flux_jobtap_get_flux','flux_jobtap_get_flux', 'Flux jobtap plugin interfaces', [author], 3),
+    ('man3/flux_jobtap_get_flux','flux_jobtap_service_register', 'Flux jobtap plugin interfaces', [author], 3),
+    ('man3/flux_jobtap_get_flux','flux_jobtap_reprioritize_all', 'Flux jobtap plugin interfaces', [author], 3),
+    ('man3/flux_jobtap_get_flux','flux_jobtap_reprioritize_job', 'Flux jobtap plugin interfaces', [author], 3),
+    ('man3/flux_jobtap_get_flux','flux_jobtap_priority_unavail', 'Flux jobtap plugin interfaces', [author], 3),
+    ('man3/flux_jobtap_get_flux','flux_jobtap_reject_job', 'Flux jobtap plugin interfaces', [author], 3),
+    ('man3/flux_sync_create','flux_sync_create', 'Synchronize on system heartbeat', [author], 3),
+    ('man3/flux_job_timeleft','flux_job_timeleft', 'Get remaining time for a job', [author], 3),
+    ('man5/flux-config', 'flux-config', 'Flux configuration files', [author], 5),
+    ('man5/flux-config-access', 'flux-config-access', 'configure Flux instance access', [author], 5),
+    ('man5/flux-config-bootstrap', 'flux-config-bootstrap', 'configure Flux instance bootstrap', [author], 5),
+    ('man5/flux-config-tbon', 'flux-config-tbon', 'configure Flux overlay network', [author], 5),
+    ('man5/flux-config-exec', 'flux-config-exec', 'configure Flux job execution service', [author], 5),
+    ('man5/flux-config-systemd', 'flux-config-systemd', 'configure Flux systemd support', [author], 5),
+    ('man5/flux-config-ingest', 'flux-config-ingest', 'configure Flux job ingest service', [author], 5),
+    ('man5/flux-config-resource', 'flux-config-resource', 'configure Flux resource service', [author], 5),
+    ('man5/flux-config-policy', 'flux-config-policy', 'configure Flux job policy', [author], 5),
+    ('man5/flux-config-queues', 'flux-config-queues', 'configure Flux job queues', [author], 5),
+    ('man5/flux-config-job-manager', 'flux-config-job-manager', 'configure Flux job manager service', [author], 5),
+    ('man5/flux-config-kvs', 'flux-config-kvs', 'configure Flux kvs service', [author], 5),
+    ('man7/flux-broker-attributes', 'flux-broker-attributes', 'overview Flux broker attributes', [author], 7),
+    ('man7/flux-jobtap-plugins', 'flux-jobtap-plugins', 'overview Flux jobtap plugin API', [author], 7),
+    ('man7/flux-environment', 'flux-environment', 'Flux environment overview', [author], 7),
+]
diff --git a/doc/python/.gitignore b/doc/python/.gitignore
new file mode 100644
index 000000000000..ac3b6ede8913
--- /dev/null
+++ b/doc/python/.gitignore
@@ -0,0 +1 @@
+autogenerated/
diff --git a/doc/python/basics.rst b/doc/python/basics.rst
new file mode 100644
index 000000000000..a997b005625d
--- /dev/null
+++ b/doc/python/basics.rst
@@ -0,0 +1,51 @@
+Flux Python Basics
+==================
+
+
+Importing the ``flux`` Python package
+-------------------------------------
+
+.. note:: The ``flux`` package which is used to interact with Flux *cannot* be
+   installed into a virtual environment with pip or conda.
+
+Flux's Python bindings are available with any installation of Flux.
+When running in a Flux instance, Flux will export the
+`PYTHONPATH <https://docs.python.org/3/using/cmdline.html#envvar-PYTHONPATH>`_
+environment variable so that Python processes can import the flux package
+by the usual import mechanism (``import flux``).
+
+If you want to import the package from outside of a Flux instance,
+running ``/path/to/flux env | grep PYTHONPATH`` in your shell will show you
+what PYTHONPATH would be set to if you were running in a Flux instance
+built/installed at ``/path/to/flux``.
+
+Note however that if you import the ``flux`` package from outside a Flux
+instance, you will need to specify a Flux instance to communicate with,
+or most of the package's operations will fail.
+
+
+The Flux class
+--------------
+
+Almost all of the functionality of the ``flux`` package revolves around
+``flux.Flux`` objects, often called "Flux handles", which represent a
+connection to a Flux instance.
+It is possible to simultaneously have multiple connections open
+to the same Flux instance, or to multiple
+Flux instances.
+
+.. note:: Flux handles are not thread-safe and should
+	not be shared between threads.
+
+In addition to the methods defined on ``flux.Flux`` objects, the ``flux``
+package also provides a number of functions which accept
+``flux.Flux`` objects as arguments, such as the functions described
+:ref:`here <python_job_submission>`.
+
+.. autoclass:: flux.core.handle.Flux
+	:members:
+
+
+The Flux reactor
+----------------
+Content coming soon.
diff --git a/doc/python/complete_api.rst b/doc/python/complete_api.rst
new file mode 100644
index 000000000000..77441663195b
--- /dev/null
+++ b/doc/python/complete_api.rst
@@ -0,0 +1,11 @@
+Complete API Reference
+======================
+
+Note that some of the items documented here
+may not be intended or designed for public use.
+
+
+.. toctree::
+   :maxdepth: 4
+
+   autogenerated/flux
diff --git a/doc/python/developer.rst b/doc/python/developer.rst
new file mode 100644
index 000000000000..3cbaa3b2ec67
--- /dev/null
+++ b/doc/python/developer.rst
@@ -0,0 +1,126 @@
+Flux Python Development
+=======================
+
+These are notes for Python developers of Flux!
+
+Development Environment
+-----------------------
+
+Note that we have a provided Dev Containers (VSCode) environment that you can
+use to develop in. Instructions for using it are `documented in the repository README <https://github.com/flux-framework/flux-core/#vscode-dev-containers>`_. We recommend you use this environment (or the same container it 
+is built from in ``.devcontainer`` for consistent results.
+
+Linting
+-------
+
+We have added support for basic automation of linting tasks, which you are
+free to use or not. If you choose to not use these tools locally, they
+will be checked in the continuous integration (and you can make changes 
+accordingly). Python 3.6 is required for the project, as a standard. You can
+also continue to use the previous older custom linting and formatting 
+scripts in the ``scripts`` folder of the repository.
+
+.. note:: The ``flux`` package which is used to interact with Flux *cannot* yet be installed into a virtual environment with pip or conda.
+
+Install the dependencies for linting (note these will already be installed in 
+the dev containers environment and you can skip this step):
+
+.. code-block:: console
+
+    $ pip install --force-reinstall -r scripts/requirements-dev.txt
+
+This includes linting tools black, isort, mypy, and flake8, and pre-commit
+to automate running them. You can then (optionally) install the hook:
+
+.. code-block:: console
+
+    $ pre-commit install --hook-type pre-commit
+
+You can optionally run pre-commit at any time like so:
+
+.. code-block:: console
+
+    $ pre-commit run --all-files
+    
+You'll see (on the first run) a bunch of errors, but likely the tools will
+fix them for you, and a new run will be much cleaner, with only a few manual
+fixes necessary:
+
+.. code-block:: console
+
+    Check for added large files..............................................Passed
+    Check for case conflicts.................................................Passed
+    Check docstring is first.................................................Passed
+    Check that scripts with shebangs are executable..........................Passed
+    Fix End of Files.........................................................Passed
+    Trim Trailing Whitespace.................................................Passed
+    Mixed line ending........................................................Passed
+    black....................................................................Passed
+    mypy.....................................................................Passed
+
+And that's it! The check run in the CI is equivalent to here, so you should
+always be able to reproduce failures. And if you install the pre-commit hook,
+you can largely prevent them by always catching them before commit.
+
+Writing Python Commands
+-----------------------
+
+The Flux command design is done in a way that some commands are generated
+from the C code, and others are helpers that come from Python. 
+If you need to add a new command (and want it to show up) there are some tricks you need to know.
+This short guide will help you on this journey! 🏔️
+
+Locations
+~~~~~~~~~
+
+Flux commands can be found under ``src/cmd``. You'll notice many files with
+a prefix "flux-". By default, any file with this prefix will be discovered
+and available to the user as a flux command. As an example, ``flux-run.py`` will
+be available as ``flux run`` and ``flux-job.c`` will be available as ``flux job``. 
+This shows that we have a mix of C derived and Python derived commands. if you are
+interested in how this works, look at ``src/cmd/cmdhelp.c``.
+
+Let's now say that we add a new command ``flux-foo.py``. We would expect to compile
+Flux and see it available in ``flux help``, right? Wrong!
+While the command will technically work as ``flux foo``, it won't show up in the help,
+and there are a few tricks to getting that working, discussed next.
+
+Adding a New Command
+~~~~~~~~~~~~~~~~~~~~
+
+The command generation works as follows:
+
+ - A new command should be a Python file added to ``src/cmd`` named like ``flux-<command-name>.py``.
+ - The command can use (and write new) shared base classes that are found under ``src/bindings/python/flux/cli`` 
+ - The script ``etc/gen-cmdhelp.py`` is run during build, and it looks for entries in ``doc/manpages.py``
+ - This means you need to add line entries for your new commands including the .rst file path relative to doc, e.g.:
+ 
+  .. code-block:: python
+
+    ('man1/flux-submit', 'flux-submit', 'submit a job to a Flux instance', [author], 1),
+    ('man1/flux-run', 'flux-run', 'run a Flux job interactively', [author], 1),
+    ('man1/flux-bulksubmit', 'flux-bulksubmit', 'submit jobs in bulk to a Flux instance', [author], 1),
+    ('man1/flux-alloc', 'flux-alloc', 'allocate a new Flux instance for interactive use', [author], 1),
+    ('man1/flux-batch', 'flux-batch', 'submit a batch script to Flux', [author], 1),
+
+ - And of course, the dependency to that means actually writing the file! Python commands are in "man1."
+ - Update ``doc/Makefile.am`` (MAN1_FILES_PRIMARY) and ``src/cmd/Makefile.am`` (dist_fluxcmd_SCRIPTS) with your new file.
+ - Try to use shared logic whenever possible! E.g., ``doc/man1/common`` has common snippets.
+ - The script generates ``etc/flux/help.d/core.json`` where you can sanity check the output.
+ 
+Since these changes need to be compiled into the source code, be careful that if you re-generate that json
+file, you do a ``make clean`` to ensure that the cmd and etc directories are rebuilt. It's best to
+start from scratch with ``make clean`` as (in this writer's experience), partial cleans don't always work.
+
+Updating a Command
+~~~~~~~~~~~~~~~~~~
+
+Updating a command is much easier, as the documentation .rst file will already
+exist! This means that (depending on your updates) you should update this file,
+do a build from scratch, and check that:
+
+ - The command functions as you would expect.
+ - It shows up in help, and the help documentation is comprehensive and correct.
+ - Tests for the command are updated or added.
+
+Happy Developing!
diff --git a/doc/python/index.rst b/doc/python/index.rst
new file mode 100644
index 000000000000..c03a8d6ebf4c
--- /dev/null
+++ b/doc/python/index.rst
@@ -0,0 +1,24 @@
+Python API
+==========
+
+Flux provides a rich Python interface. With few exceptions,
+any Flux interactions that are possible in C or on the
+command line are possible in Python as well via the ``flux`` package.
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   basics
+   job_submission
+   complete_api
+   developer
+
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/doc/python/job_submission.rst b/doc/python/job_submission.rst
new file mode 100644
index 000000000000..1d11771f9a08
--- /dev/null
+++ b/doc/python/job_submission.rst
@@ -0,0 +1,244 @@
+.. _python_job_submission:
+
+Flux Job Submission and Monitoring
+==================================
+
+The Flux Python bindings provide synchronous and asynchronous
+functions for:
+
+* Submitting jobs with arbitrary attributes and resources
+* Waiting for jobs to complete
+* Cancelling jobs
+* Viewing job info and events
+
+
+Job submission
+--------------
+
+Job submission is performed by creating a ``flux.job.Jobspec`` object,
+populating it with attributes, and then passing it to one of the submission
+functions, e.g. :ref:`flux.job.submit <python_flux_job_submit_func>`. Jobspec
+objects define everything
+about a job, including the job's resources, executable, working directory,
+environment, and stdio streams.
+
+Basic Jobspec creation is generally done with the
+``JobspecV1.from_command`` class method and its
+variants ``from_batch_command`` and ``from_nest_command``, which are helper
+methods replicating the jobspecs created by the
+job submission command-line utilities.
+
+
+.. autoclass:: flux.job.Jobspec
+	:members:
+
+.. autoclass:: flux.job.JobspecV1
+	:show-inheritance:
+	:members:
+
+
+
+Job manipulation
+----------------
+
+After a job has been submitted, it will be assigned an ID. That ID
+can then be used for getting information about the job or
+for manipulating it---see the synchronous and asynchronous sections below.
+
+To translate job ID representations, use the ``flux.job.JobID`` class:
+
+.. autoclass:: flux.job.JobID
+	:members:
+
+
+Synchronous interface
+---------------------
+
+
+The simplest way to interact with Flux is with the synchronous functions listed
+below.
+However, these functions introduce a lot of overhead (e.g. any function that
+waits for a job to reach a certain state, such as ``result``
+may block for an indeterminate amount of time) and may not be suitable for
+interacting with large numbers of jobs in time-sensitive applications.
+
+To spend less time blocking in the Flux reactor, consider using one
+of the asynchronous interfaces.
+
+
+.. _python_flux_job_submit_func:
+.. autofunction:: flux.job.submit
+
+.. autofunction:: flux.job.event_watch
+
+.. autofunction:: flux.job.event_wait
+
+.. autofunction:: flux.job.kill
+
+.. autofunction:: flux.job.cancel
+
+.. autofunction:: flux.job.result
+
+.. autofunction:: flux.job.wait
+
+.. autofunction:: flux.job.get_job
+
+
+``get_job``
+~~~~~~~~~~~
+
+After submitting a job, if you quickly want to see information for it,
+you can use ``flux.job.get_job``. Here is an example:
+
+.. code:: python
+
+    import flux
+    import flux.job
+
+    # It's encouraged to create a handle to use across commands
+    handle = flux.Flux()
+
+    jobspec = flux.job.JobspecV1.from_command(["sleep", "60"])
+    jobid = flux.job.submit(handle, jobspec)
+    job_meta = flux.job.get_job(handle, jobid)
+
+    {
+        "job": {
+            "id": 676292747853824,
+            "userid": 0,
+            "urgency": 16,
+            "priority": 16,
+            "t_submit": 1667760398.4034982,
+            "t_depend": 1667760398.4034982,
+            "state": "SCHED",
+            "name": "sleep",
+            "ntasks": 1,
+            "ncores": 1,
+            "duration": 0.0
+        }
+    }
+
+If the jobid you are asking for does not exist, ``None`` will be returned.
+For the interested user, this is a courtesy function that wraps using the identifier
+to create an RPC object, serializing that to string, and loading as JSON. 
+Since it is likely you, as the user, will be interacting with ``flux.job``, it
+is also logical you would look for this function to retrieve the job on the same
+module.
+
+``result`` vs ``wait``
+~~~~~~~~~~~~~~~~~~~~~~
+
+Both ``flux.job.result`` and ``flux.job.wait`` return when a job has completed.
+However, ``wait`` only works on jobs which have been submitted
+with the ``waitable`` flag, and the ability to set that flag is
+restricted to instance owners.
+
+
+Asynchronous interfaces
+-----------------------
+
+There are two primary asynchronous interfaces to job manipulations. The first is
+an event-loop interface, which is closer to the native C interface, and consists of
+functions like ``flux.job.submit_async`` and ``flux.job.result_async`` (note the
+functions are the same as in the synchronous interface, only with an "_async"
+suffix). The second is an interface which is almost identical to the
+`concurrent.futures <https://docs.python.org/3/library/concurrent.futures.html>`_
+interface in Python's standard library, and it consists of the
+``flux.job.FluxExecutor`` class. Both interfaces deal in callbacks and
+futures, the difference being that the ``FluxExecutor`` is designed so that
+all futures fulfill in the background, and there is no need for user code
+to enter the Flux event loop, while the event-loop-based interface
+requires the user to call into the Flux event loop in order for futures
+to fulfill and for callbacks to trigger.
+
+Our general recommendation is that you use the ``FluxExecutor`` interface
+unless you are familiar with event-loop based programming.
+
+
+The ``FluxExecutor`` interface
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Basic ``FluxExecutor`` usage consists of creating an executor,
+submitting jobspecs to it, and then attaching callbacks to those
+futures or waiting for them to complete. Executors must have
+``.shutdown()`` called when they are no longer to be used. However,
+they also support the context-manager protocol (i.e. ``with executor ...:``)
+which will call ``shutdown`` upon leaving the ``with`` block.
+
+Example usage:
+
+.. code:: python
+
+	import concurrent.futures
+	import flux.job
+
+	jobspec = flux.job.JobspecV1.from_command(["/bin/true"])
+	with flux.job.FluxExecutor() as executor:
+		futs = [executor.submit(jobspec) for _ in range(5)]
+		for f in concurrent.futures.as_completed(futs):
+			print(f.result())
+
+
+.. autoclass:: flux.job.FluxExecutor
+	:members:
+
+Futures should not be created directly by user code. Since
+the futures are subclasses of ``concurrent.executors.Future``,
+you can invoke ``concurrent.futures`` functions like ``wait`` or
+``as_completed`` on them.
+
+.. autoclass:: flux.job.FluxExecutorFuture
+	:members:
+
+
+Asynchronous event-loop interface
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+General usage consists of initiating some events,
+then entering the Flux reactor and writing all code
+thereafter as callbacks.
+
+For instance, the below example submits five jobs,
+then enters the reactor and fires off callbacks
+as futures complete.
+
+.. code:: python
+
+	import flux
+	import flux.job
+
+	def submit_cb(fut, flux_handle):
+		# when this callback fires, the jobid will be ready
+		jobid = fut.get_id()
+		# Create a future representing the result of the job
+		result_fut = flux.job.result_async(flux_handle, jobid)
+		# attach a callback to fire when the job finishes
+		result_fut.then(result_cb)
+
+	def result_cb(fut):
+		job = fut.get_info()
+		result = job.result.lower()
+		print(f"{job.id}: {result} with returncode {job.returncode}")
+
+	flux_handle = flux.Flux()
+	jobspec = flux.job.JobspecV1.from_command(["/bin/true"])
+	for _ in range(5):
+		# submit 5 futures and attach callbacks to each one
+		submit_future = flux.job.submit_async(flux_handle, jobspec)
+		submit_future.then(submit_cb, flux_handle)
+	# enter the flux event loop (the 'reactor') to trigger the callbacks
+	# once the futures complete
+	flux_handle.reactor_run()
+
+
+.. autofunction:: flux.job.submit_async
+
+.. autofunction:: flux.job.event_watch_async
+
+.. autofunction:: flux.job.cancel_async
+
+.. autofunction:: flux.job.kill_async
+
+.. autofunction:: flux.job.result_async
+
+.. autofunction:: flux.job.wait_async
diff --git a/doc/requirements.txt b/doc/requirements.txt
new file mode 120000
index 000000000000..2b46c0f1f9eb
--- /dev/null
+++ b/doc/requirements.txt
@@ -0,0 +1 @@
+../scripts/requirements-doc.txt
\ No newline at end of file
diff --git a/doc/test/spell.en.pws b/doc/test/spell.en.pws
index 456670af4e77..a6163e2b6129 100644
--- a/doc/test/spell.en.pws
+++ b/doc/test/spell.en.pws
@@ -1,4 +1,23 @@
 personal_ws-1.1 en 0
+maxdepth
+literalinclude
+toctree
+rst
+proc
+localid
+exitcode
+getline
+plugstack
+setlevel
+errn
+dest
+killall
+sig
+unsets
+setenvf
+environ
+substring
+asprintf
 KVS
 kvs
 api
@@ -19,7 +38,6 @@ readlink
 tokvs
 adoc
 apisock
-cmbd
 cmd
 ZMTP
 stderr
@@ -46,9 +64,7 @@ syslog
 graphviz
 prepended
 multipart
-hb
 Tpng
-nodesets
 keepalives
 emerg
 gettimeofday
@@ -81,8 +97,7 @@ hiWSLCPRzqFYHNubktbCQ
 hostname
 io
 jobid
-knowlege
-labelio
+jobids
 libpmi
 lwj
 mpi
@@ -90,6 +105,7 @@ mpicc
 nnodes
 noinput
 ntasks
+ncores
 pmi
 resrc
 runtime
@@ -100,7 +116,6 @@ ary
 baz
 EPGM
 modopts
-nodeset
 noexec
 pre
 slurm
@@ -121,7 +136,6 @@ repo
 lua
 cpath
 DEST
-heartrate
 logdest
 modopt
 noshell
@@ -200,7 +214,6 @@ requeues
 RQ
 libev
 baz
-hb
 unsubscribe
 unsubscribes
 zeromq
@@ -229,6 +242,7 @@ topen
 tevent
 tsend
 trecv
+tsync
 scalability
 env
 ENV
@@ -244,7 +258,6 @@ tinfo
 hwloc
 epgm
 args
-arity
 attr
 ATTRFLAG
 ENOENT
@@ -306,6 +319,7 @@ cron
 lifecycle
 rexec
 subcommand
+subcommands
 stddev
 RTT
 wakeup
@@ -356,13 +370,10 @@ setbit
 rolemask
 userid
 selfpmi
-aggregator
 UTC
 addrole
 delrole
 strtoul
-userdb
-scratchdir
 EHOSTUNREACH
 ap
 MSGID
@@ -415,6 +426,8 @@ lflux
 resizing
 HOSTLIST
 hostlist
+hostlists
+nodelist
 hostnames
 unfulfills
 MSGFLAG
@@ -471,3 +484,458 @@ fsd
 hms
 username
 submitter's
+enqueue
+nslots
+alloc
+datetime
+formatter
+ndecode
+dothex
+cwd
+execve
+initrc
+INITRC
+loglevel
+overwrite
+PTRACE
+rankinfo
+rc
+rcpath
+searchpath
+sysconfdir
+TRACEME
+WIFEXTED
+builtin
+OMP
+envfile
+regex
+encodings
+dec
+subkey
+waitstatus
+returncode
+bulksubmit
+basename
+bcc
+dirname
+jps
+xargs
+sep
+RPCs
+jobtap
+unavail
+reprioritize
+reprioritization
+afterany
+afterok
+afternotok
+parsedatetime
+cancelall
+raiseall
+keepalive
+linux
+starttime
+undraining
+unix
+zmqdebug
+perilogs
+prolog
+pstree
+ThreadPoolExecutor
+rootref
+ECONNRESET
+SIGCONT
+SIGHUP
+nohup
+parallelize
+reparented
+cPMS
+HmSnHd
+Hjxo
+JqUHUCzX
+MYrwzf
+NJOBS
+PROG
+utPP
+uusNLo
+hwloc
+undrain
+QBfmbm
+ascii
+listpids
+pidof
+Pz
+qxxTiZBM
+scontrol
+URIs
+XFWP
+xVu
+NOSUB
+RPCTRACK
+getsockopt
+accessor
+accessors
+unregister
+IPv
+ipv
+maxlen
+mgmt
+tcp
+URIs
+DNS
+etest
+eth
+setuid
+IDset
+busytimeout
+dbpath
+pragma
+satisfiability
+fluxion
+systemctl
+MQ
+prefertcp
+desynchronized
+multi
+sysctl
+pluginpath
+uptime
+startlog
+timestamps
+runstatedir
+libexecdir
+bindir
+cmdhelp
+MANPATH
+bargraph
+emoji
+bzip
+cpio
+gzip
+libarchive
+OUTFILE
+INFILE
+POSIX
+ustar
+xz
+validator
+statedir
+num
+gc
+tgz
+tmpfiles
+EDEADLOCK
+EDEADLK
+BSD
+setpgrp
+nosetpgrp
+checkpointed
+pbatch
+pdebug
+parentof
+bg
+norestrict
+frobnicator
+frobnication
+fcfs
+lonodex
+nosched
+qmanager
+rv
+DIRS
+myformat
+xdg
+XDG
+yaml
+enqueued
+kary
+getrlimit
+memlock
+MEMLOCK
+msgqueue
+nofile
+NOFILE
+ofile
+RLIM
+rlimit
+RLIMIT
+rtprio
+rttime
+sigpending
+chunksize
+taskmap
+multiline
+sudo
+sysconfig
+destdir
+filemap
+dataset
+fileref
+mmap
+unmap
+unmaps
+ds
+rf
+ACLs
+SIGBUS
+tmpfs
+nocheckpoint
+CPUs
+cpuset
+taskset
+nomap
+timeleft
+waitable
+novalidate
+ngpus
+rlist
+propertiesx
+pgrep
+pkill
+deps
+queuem
+timelimit
+unarchived
+subinstance
+AKHno
+mybatch
+Zq
+testprog
+libc
+unbuffered
+statex
+pmix
+SIGUSR
+iteratively
+noverify
+sdbus
+sdexec
+NONINTERACTIVE
+hwlocality
+snprintf
+MemoryHigh
+MemoryMax
+MemoryLow
+MemoryMin
+myattr
+autotools
+cd
+debbuild
+debian
+distros
+dockerhub
+fluxrm
+LLNL
+redhat
+RHEL
+spack
+ubuntu
+xzf
+APIs
+HPC
+infinitum
+OpenMPI
+inclusivity
+lgplv
+png
+alice
+LSF
+sbatch
+salloc
+MpiDefault
+se
+SoiFq
+SoPsfdA
+Ub
+jsrun
+Kubernetes
+MPICH
+mpiexec
+passwordless
+pdsh
+bashrc
+endreason
+HUP
+fi
+lsf
+sl
+ABfkxas
+ABfkxat
+ABhEwsD
+ABiiw
+ABkCvRu
+cbUvuHDCiB
+croSDd
+ctHRVy
+ctHRVz
+cTzRVhnrW
+cumQnK
+cwFQ
+uAsjAo
+resched
+func
+rfc
+envvar
+dlopen
+executables
+gethostname
+IPADDR
+MCA
+misconfiguration
+ogpu
+opmi
+printenv
+XMLFILE
+initio
+mortem
+natively
+execvp
+gdb
+libtool
+prepend
+noflux
+param
+perres
+pertask
+mpibind
+sys
+allocator
+INITFULL
+ITER
+maxid
+unallocated
+NUL
+ssize
+waitcreate
+unformatted
+uniq
+unreferenced
+restartable
+ramdisk
+loopback
+DSO
+subtree
+ERANGE
+hl
+noresponse
+moldable
+aspirational
+infeasible
+login
+workflow
+xmlfile
+hostfile
+ano
+myapp
+unlinks
+VM
+fred
+unmapped
+kbd
+adminarch
+AiVi
+auth
+coredump
+coredumpctl
+fatnode
+fubar
+javascript
+journalctl
+jsonschema
+localuser
+munge
+mv
+myapp
+nproc
+pam
+PEERCRED
+perilog
+pluggable
+resolvers
+respawn
+Sj
+testprop
+ttl
+uncomment
+unsatisfiable
+validators
+hostpids
+Xauth
+dbus
+intree
+aes
+dshbak
+gid
+testuser
+timedatectl
+uid
+unmunge
+xyz
+befdac
+cec
+CGroup
+drwxr
+ecb
+eea
+ef
+le
+ExecStartPre
+loginctl
+md
+norestart
+Sbroker
+Scontent
+sd
+Srundir
+Sstatedir
+xr
+acknowledgements
+netdev
+LimitCORE
+WorkingDirectory
+aa
+aaa
+aacc
+aff
+bbb
+camlistore
+dde
+fbe
+fe
+hdir
+NDIyCg
+recursing
+setroot
+SRMPDS
+Vogel
+ZFS
+programmatically
+transactional
+keypair
+unterminated
+upmi
+testexec
+rcvhwm
+libzmq
+userrc
+DEEPBIND
+PRELOAD
+RTLD
+dlopened
+ne
+oneshot
+peridir
+periname
+templated
+CIDR
+enp
+mincores
+minnodes
+cgroup
+cgroups
+fs
+misconfigured
+aTGTz
+cPATH
+SATTR
+myprogram
diff --git a/doc/test/spellcheck b/doc/test/spellcheck
index 3d6d34fb4192..ca895cdc054a 100755
--- a/doc/test/spellcheck
+++ b/doc/test/spellcheck
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if test $man_base_dir; then
-    set ${man_base_dir}/man*/*.adoc
+    set ${man_base_dir}/*.rst ${man_base_dir}/guide/*.rst ${man_base_dir}/man*/*.rst ${man_base_dir}/man*/common/*.rst
 fi
 
 if test $# == 0; then
@@ -22,7 +22,7 @@ if test  -z "$ASPELL"; then
    echo "1..0 # skip because aspell is not installed"
    exit 0
 fi
-if ! $ASPELL -n list </dev/null >/dev/null; then
+if ! $ASPELL -p $dict -n list </dev/null >/dev/null; then
    echo "1..$# # skip because aspell dry run failed"
    exit 0
 fi
@@ -34,7 +34,8 @@ for f in $*; do
     filename=$(basename $f)
     dir=$(basename $(dirname $f))
     tmpfile=$(mktemp)
-    rc=$(cat $f | $ASPELL -p $dict -n list | sort | uniq | tee $tmpfile | wc -l)
+    rc=$(cat $f | $ASPELL -p $dict --encoding='utf-8' -n list \
+        | sort | uniq | tee $tmpfile | wc -l)
     if test $rc == 0; then
         echo "ok $count - spell check $dir/$filename"
     else
diff --git a/etc/Makefile.am b/etc/Makefile.am
index 41e58780e82a..546a94814228 100644
--- a/etc/Makefile.am
+++ b/etc/Makefile.am
@@ -1,28 +1,29 @@
 #if HAVE_SYSTEMD
-systemdsystemunit_DATA = flux.service
+systemdsystemunit_DATA = \
+	flux.service \
+	flux-housekeeping@.service \
+	flux-prolog@.service \
+	flux-epilog@.service
 #endif
 
-dist_fluxcf_DATA = \
-	conf.d/bootstrap.toml
+tmpfilesdir = $(prefix)/lib/tmpfiles.d
+fluxrc1dir = $(fluxconfdir)/rc1.d
 
-noinst_DATA = \
-	flux/curve
+tmpfiles_DATA = flux.conf
 
-dist_fluxrc_SCRIPTS = \
+
+dist_fluxconf_SCRIPTS = \
         rc1 \
         rc3
 
 dist_fluxrc1_SCRIPTS = \
-        rc1.d/01-enclosing-instance
-
-flux/curve:
-	$(AM_V_GEN)$(top_builddir)/src/cmd/flux keygen --force
+        rc1.d/02-cron
 
 fluxhelpdir = $(datadir)/flux/help.d
 fluxhelp_DATA = flux/help.d/core.json
-flux/help.d/core.json: $(top_srcdir)/doc/man1
+flux/help.d/core.json: $(top_srcdir)/doc/manpages.py
 	$(AM_V_GEN)umask 077 && $(MKDIR_P) flux/help.d && \
-	$(srcdir)/gen-cmdhelp.pl --category=core $^/*.adoc >$@
+	$(PYTHON) $(srcdir)/gen-cmdhelp.py $< > $@
 
 if !ENABLE_DOCS
 fluxnodocsdir = $(datadir)/flux/.nodocs
@@ -35,13 +36,32 @@ endif
 clean-local:
 	-rm -rf flux
 
+CLEANFILES=completions/flux
+
 if WITH_PKG_CONFIG
 pkgconfig_DATA = flux-core.pc \
 	flux-pmi.pc \
 	flux-optparse.pc \
 	flux-idset.pc \
-	flux-schedutil.pc
+	flux-schedutil.pc \
+	flux-hostlist.pc \
+	flux-taskmap.pc
 endif
 
+noinst_SCRIPTS = \
+	completions/get_builtins.sh \
+	completions/flux.pre
+
 EXTRA_DIST = \
-	gen-cmdhelp.pl
+	gen-cmdhelp.py \
+	flux.conf \
+	$(noinst_SCRIPTS)
+
+completions/flux: completions/flux.pre completions/get_builtins.sh
+	$(AM_V_GEN)test -d completions || mkdir completions && \
+	cp $< $@ && chmod +w $@ && \
+	$(srcdir)/completions/get_builtins.sh \
+	$(top_srcdir)/src/cmd/builtin >> $@
+
+bashcomp_SCRIPTS = \
+	completions/flux
diff --git a/etc/completions/flux.pre b/etc/completions/flux.pre
new file mode 100644
index 000000000000..5faca2b7d044
--- /dev/null
+++ b/etc/completions/flux.pre
@@ -0,0 +1,2239 @@
+#
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+#
+shopt -s extglob
+
+# Lifted from bash_completions:
+# For a longopt --foo=bar, set $prev=--foo and $cur=bar.
+_flux_split_longopt() {
+    if [[ $cur == --?*=* ]]; then
+        # Cut also backslash before '=' in case it ended up there
+        # for some reason
+        prev="${cur%%?(\\)=*}"
+        cur="${cur#*=}"
+        return 0
+    fi
+    return 1
+}
+
+# return success if first argument is in remaining args
+_flux_contains_word() {
+    local w word=$1; shift
+    for w in "$@"; do
+        [[ $w = "$word" ]] && return
+    done
+}
+
+# Determines the first non-option word of the command line. This
+# is usually the command. Returns empty string if first non-option
+# word is $cur, since that means we're still completing the command.
+#
+_flux_get_cmd() {
+    local firstword i
+
+    firstword=
+    for ((i = 1; i < ${#COMP_WORDS[@]}; ++i)); do
+        if [[ ${COMP_WORDS[i]} != -* && ${COMP_WORDS[i]} != $cur ]]; then
+            firstword=${COMP_WORDS[i]}
+            break
+        fi
+    done
+
+    echo $firstword
+}
+
+# Determines the first non-option word after another given word,
+# This is usually a sub-command
+# Returns the word if result = $cur since that means we're still
+# completing the current subcommand
+#
+_flux_get_subcmd() {
+    local firstword i word=$1; shift
+
+    firstword=$word
+    found=0
+    for ((i = 1; i < ${#COMP_WORDS[@]} - 1; ++i)); do
+        local w="${COMP_WORDS[i]}"
+        if [[ $w != -*  && -n $w && $w != $cur ]]; then
+            if ((found == 1)); then
+                firstword=$w
+                break
+            fi
+            if [[ $w == $word ]]; then
+                found=1
+            fi
+        fi
+    done
+
+    echo $firstword
+}
+
+# Autocomplete current queues
+_flux_complete_queue() {
+    local queues=$(flux queue status | sed -n 's/^\(.*\): .*$/\1/p')
+    COMPREPLY=( $(compgen -W "$queues" -- "$cur") )
+}
+
+# Autocomplete format names for commands that take -o, --format
+# usage _flux_complete_format_name COMMAND SUBCOMMAND [ARGS...]
+_flux_complete_format_name() {
+    local formats=$($@ --format=help | grep -v ^Conf | awk 'NF {print $1}')
+    COMPREPLY=( $(compgen -W "$formats" -- "$cur") )
+}
+
+#  Get the list of subcommands from FLUX_EXEC_PATH and hard-coded builtins
+__get_flux_subcommands() {
+    local subcommands
+    if [ -z "$FLUX_EXEC_PATH" ]; then
+        FLUX_EXEC_PATH=`flux env printenv FLUX_EXEC_PATH`
+    fi
+
+    local IFS=":"
+    for dir in $FLUX_EXEC_PATH; do
+        for op in $dir/flux-*; do
+            if [[ -x $op  ]]; then
+                op="${op##*flux-}"
+                subcommands+="${op%.*} "
+            fi
+        done
+    done
+
+    for builtin in $FLUX_BUILTINS; do
+        subcommands+="$builtin "
+    done
+
+    echo "$subcommands"
+}
+
+# Helper to emit id.f58plain when match begins with `f`
+_flux_id_fmt()
+{
+    local cur=$1
+    local fmt="{id}"
+    if [[ $cur == f* ]]; then
+        fmt="{id.f58plain}"
+    fi
+    echo "${fmt}"
+}
+
+_flux_active_instances()
+{
+    flux jobs -no "$(_flux_id_fmt $1) {uri}" | awk '!($2 == "None") {print $1}'
+}
+
+#  flux-proxy(1) completions
+_flux_proxy()
+{
+    local $cmd=$1
+    OPTS="\
+        -f --force \
+        -n --nohup \
+        --reconnect \
+    "
+    if [[ $cur != -* ]]; then
+        #  Attempt to substitute an active jobid
+        active_jobs=$(_flux_active_instances $cur)
+        COMPREPLY=( $(compgen -W "${active_jobs}" -- "$cur") )
+        return 0
+    fi
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    return 0
+}
+
+#  flux-cancel(1) completions
+_flux_cancel()
+{
+    local cmd=$1
+    local split=false
+    OPTS="\
+        --all \
+        -n --dry-run \
+        -q --quiet \
+        -u --user= \
+        -S --states= \
+        -m --message= \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --user | -!(-*)u)
+            users=$(flux jobs -Ano {username} | uniq)
+            COMPREPLY=( $(compgen -W "$users" -- "$cur") )
+            return
+            ;;
+        --states | -!(-*)S)
+            states="active depend priority sched run pending running"
+            COMPREPLY=( $(compgen -W "$states" -- "$cur") )
+            return
+            ;;
+        --message | -!(-*)m)
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cur != -* ]]; then
+        #  Attempt to substitute an active jobid
+        active_jobs=$(flux jobs -no $(_flux_id_fmt $cur))
+        COMPREPLY=( $(compgen -W "${active_jobs}" -- "$cur") )
+        return 0
+    fi
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+#  flux-update(1) completions
+_flux_update()
+{
+    local cmd=$1
+    OPTS="\
+        -v --verbose \
+        -n --dry-run \
+    "
+    if [[ $cur != -* ]]; then
+        #  Attempt to substitute an active jobid
+        active_jobs="$(flux jobs -no $(_flux_id_fmt $cur))"
+        COMPREPLY=( $(compgen -W "${active_jobs}" -- "$cur") )
+        return 0
+    fi
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+#  flux-hostlist(1) completions
+_flux_hostlist()
+{
+    local cmd=$1 split=false
+    OPTS="\
+        -e --expand \
+        -d --delimiter= \
+        -c --count \
+        -n --nth= \
+        -L --limit= \
+        -S --sort \
+        -x --exclude= \
+        -u --union --unique \
+        -i --intersect \
+        -m --minus \
+        -X --xor \
+        -f --fallback \
+        -q --quiet \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --limit | --exclude | --nth | --delimiter | -!(-*)[lmxn])
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cur != -* ]]; then
+        # Substitute source name or recent jobid
+        jobs="$(flux jobs -ano $(_flux_id_fmt $cur))"
+        sources="local instance avail stdin $jobs"
+        COMPREPLY=( $(compgen -W "$sources $jobs" -- "$cur") )
+        return 0
+    fi
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+#  flux-shutdown(1) completions
+_flux_shutdown()
+{
+    local subcmd=$1 split=false
+    OPTS="\
+        --skip-gc \
+        --gc \
+        --dump= \
+        --background \
+        --quiet \
+        -v --verbose= \
+        -y --yes \
+        -n --no \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --dump)
+            COMPREPLY=( $(compgen -f -- "$cur") )
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cur != -* ]]; then
+        # Substitute recent alloc/batch jobid
+        jobs=$(_flux_active_instances $cur)
+        COMPREPLY=( $(compgen -W "$jobs" -- "$cur") )
+        return 0
+    fi
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+#  flux-archive(1) completions
+_flux_archive()
+{
+    local cmd=$1
+    local subcmds="create remove extract list"
+    local split=false
+
+    local create_OPTS="\
+        -h --help \
+        -n --name= \
+        --no-force-primary \
+        -C --directory= \
+        -v --verbose= \
+        --overwrite \
+        --append \
+        --preserve \
+        --mmap \
+    "
+    local remove_OPTS="\
+        -h --help \
+        -n --name= \
+        --no-force-primary \
+        -f --force \
+    "
+    local extract_OPTS="\
+        -h --help \
+        -n --name= \
+        -v --verbose= \
+        -C --directory= \
+        --overwrite \
+        --waitcreate= \
+        --no-force-primary \
+        -t --list-only \
+    "
+    local list_OPTS="\
+        -h --help \
+        -n --name= \
+        --no-force-primary \
+        -l --long \
+        --raw \
+    "
+    if [[ $cmd != "archive" ]]; then
+
+        _flux_split_longopt && split=true
+        case $prev in
+            --directory | -!(-*)C)
+                compopt -o filenames
+                COMPREPLY=( $(compgen -d -- "$cur") )
+                return
+                ;;
+            -!(-*)[n])
+                return
+                ;;
+        esac
+        $split && return
+
+        if [[ $cmd == "create" && $cur != -* ]]; then
+            compopt -o default -o bashdefault -o filenames
+            COMPREPLY=( $(compgen -f -c -- "$cur") )
+            return
+        fi
+
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var}" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+    return 0
+}
+
+#  flux-{run,submit,batch,alloc,bulksubmit}(1) completions
+_flux_submit_commands()
+{
+    local subcmds="run submit batch alloc bulksubmit"
+    local cmd=$1
+    local split=false
+
+    local COMMON_OPTIONS="\
+        -q --queue= \
+        -B --bank= \
+        -t --time-limit= \
+        -o --setopt= \
+        -S --setattr= \
+        --urgency= \
+        --job-name= \
+        --dependency= \
+        --requires= \
+        --begin-time= \
+        --env= \
+        --env-remove= \
+        --env-file \
+        --rlimit= \
+        --input= \
+        --output= \
+        --error= \
+        -u --unbuffered \
+        -l --label-io \
+        --flags= \
+        --dry-run \
+        -h --help \
+        --cwd= \
+        --signal= \
+        --add-file= \
+    "
+    local SUBMIT_OPTIONS="\
+        -N --nodes= \
+        -x --exclusive \
+        -n --ntasks= \
+        -c --cores-per-task= \
+        -g --gpus-per-task= \
+        --cores= \
+        --tasks-per-node= \
+        --tasks-per-core= \
+        --gpus-per-node= \
+        -v --verbose \
+        --taskmap= \
+    "
+    local BATCH_ALLOC_OPTIONS="\
+        -n --nslots= \
+        -c --cores-per-slot= \
+        -g --gpus-per-slot= \
+        -N --nodes= \
+        --dump= \
+        --conf= \
+        -x --exclusive \
+    "
+    local SUBMITBULK_OPTIONS="\
+        --quiet \
+        --cc= \
+        --bcc= \
+        --wait \
+        --wait-event= \
+        --watch \
+        --progress \
+        --log= \
+        --log-stderr= \
+        --jps \
+    "
+    local BULKSUBMIT_OPTIONS="\
+        --shuffle \
+        --sep= \
+        --define= \
+    "
+    local RUN_OPTIONS="\
+        --wait-event= \
+    "
+    local BATCH_OPTIONS="\
+        --wrap \
+        --quiet \
+    "
+    local ALLOC_OPTIONS="\
+        -v --verbose \
+        --bg \
+    "
+    local bulksubmit_OPTS="\
+        $COMMON_OPTIONS \
+        $SUBMIT_OPTIONS \
+        $SUBMITBULK_OPTIONS \
+        $BULKSUBMIT_OPTIONS \
+    "
+    local run_OPTS="\
+        $COMMON_OPTIONS \
+        $SUBMIT_OPTIONS \
+        $RUN_OPTIONS \
+    "
+    local submit_OPTS="\
+        $COMMON_OPTIONS \
+        $SUBMIT_OPTIONS \
+        $SUBMITBULK_OPTIONS \
+    "
+    local batch_OPTS="\
+        $COMMON_OPTIONS \
+        $BATCH_OPTIONS \
+    "
+    local alloc_OPTS="\
+        $COMMON_OPTIONS \
+        $ALLOC_OPTIONS \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --queue | -!(-*)q)
+             _flux_complete_queue
+            return
+            ;;
+        --urgency)
+            COMPREPLY=( $(compgen -W "{0..32} hold default expedite" -- "$cur") )
+            return
+            ;;
+        --add-file | --env-file | --input | --output | --error | \
+            --log | --log-stderr)
+            compopt -o filenames
+            COMPREPLY=( $(compgen -f -- "$cur") $(compgen -d -- "$cur") )
+            return
+            ;;
+        --cwd)
+            compopt -o filenames
+            COMPREPLY=( $(compgen -d -- "$cur") )
+            return
+            ;;
+        --flags)
+            COMPREPLY=( $(compgen -W 'waitable debug novalidate' -- "$cur") )
+            return
+            ;;
+        -!(-*)[tNoncg])
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cur != -* ]]; then
+        compopt -o bashdefault -o default
+    fi
+
+    var="${cmd}_OPTS"
+    COMPREPLY=( $(compgen -W "${!var}" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # no space if suggestions ends with '='
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-resource(1) completions
+_flux_resource()
+{
+    local subcmds="drain undrain status list R info reload"
+    local cmd=$1
+    local split=false
+    local states
+
+    local reload_OPTS="\
+        -h --help \
+        -x --xml \
+        -f --force \
+    "
+    local info_OPTS="\
+        -h --help \
+        -s --states= \
+        -i --include= \
+    "
+    local list_OPTS="\
+        -h --help \
+        -o --format= \
+        -s --states= \
+        -n --no-header \
+        -q --queue= \
+        -i --include= \
+        --skip-empty \
+        --no-skip-empty \
+    "
+    local status_OPTS="\
+        ${list_OPTS} \
+    "
+    local R_OPTS="\
+        -h --help \
+        -s --states= \
+        -q --queue= \
+        -i --include= \
+    "
+    local undrain_OPTS="\
+        -h --help \
+        -f --force \
+    "
+    local drain_OPTS="\
+        -h --help \
+        -f --force \
+        -u --update \
+        -o --format= \
+        -n --no-header \
+        -q --queue= \
+        -i --include= \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --queue | -!(-*)q)
+            _flux_complete_queue
+            return
+            ;;
+        --format | -!(-*)o)
+            _flux_complete_format_name flux resource $cmd
+            return
+            ;;
+        --states | -!(-*)s)
+            case $cmd in
+                list | info | R)
+                    states="up down allocated free all"
+                    ;;
+                status | drain)
+                    states="avail exclude draining drained drain offline online"
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "$states" -- "$cur") )
+            return
+            ;;
+        -!(-*)i)
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cmd != "resource" ]]; then
+        if [[ $cur != -* ]]; then
+            if [[ $cmd == "reload" ]]; then
+                compopt -o filenames
+                COMPREPLY=( $(compgen -f -- "$cur") $(compgen -d -- "$cur") )
+                return
+            fi
+        fi
+        var="${cmd}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var}" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+    return 0
+}
+
+# flux-job(1) completions
+_flux_job()
+{
+    local cmd=$1
+    local split=false
+
+    local subcmds="\
+        urgency \
+        cancel \
+        raise \
+        kill \
+        killall \
+        attach \
+        status \
+        id \
+        eventlog \
+        wait-event \
+        info \
+        namespace \
+        wait \
+        memo \
+        taskmap \
+        timeleft \
+        last \
+        hostpids \
+    "
+    local nojob_subcmds="\
+        cancelall \
+        raiseall \
+        stats \
+        submit \
+        purge \
+    "
+    local all_OPTS="\
+        -h --help \
+    "
+    local urgency_OPTS="\
+        -v --verbose \
+    "
+    local cancel_OPTS="\
+        -m --message= \
+    "
+    local cancelall_OPTS="\
+        -u --user= \
+        -S --states= \
+        -f --force \
+        -q --quiet \
+    "
+    local raise_OPTS="\
+        -s --severity= \
+        -t --type= \
+        -m --message= \
+    "
+    local raiseall_OPTS="\
+        -s --severity= \
+        -u --user= \
+        -S --states= \
+        -f --force \
+    "
+    local kill_OPTS="\
+        -s --signal= \
+    "
+    local killall_OPTS="\
+        -s --signal= \
+        -u --user= \
+        -f --force \
+    "
+    local attach_OPTS="\
+        -E --show-events \
+        -X --show-exec \
+        -w --wait-event= \
+        -l --label-io \
+        -v --verbose \
+        -q --quiet \
+        -r --read-only \
+        -i --stdin-ranks= \
+        -u --unbuffered \
+        --debug \
+    "
+    local status_OPTS="\
+        -v --verbose \
+        -j --json \
+        -e --exception-exit-code= \
+    "
+    local submit_OPTS="\
+        -u --urgency= \
+        -f --flags= \
+    "
+    local id_OPTS="\
+        -t --to= \
+    "
+    local eventlog_OPTS="\
+        -f --format= \
+        -T --time-format= \
+        -p --path= \
+        -H --human \
+        -L --color= \
+    "
+    local wait_event_OPTS="\
+        -f --format= \
+        -T --time-format= \
+        -t --timeout= \
+        -m --match-context= \
+        -c --count= \
+        -q --quiet \
+        -v --verbose \
+        -p --path= \
+        -W --waitcreate \
+        -H --human \
+        -L --color= \
+    "
+    local info_OPTS="\
+        -o --original \
+        -b --base \
+    "
+    local wait_OPTS="\
+        -a --all \
+        -v --verbose \
+    "
+    local memo_OPTS="\
+        --volatile
+    "
+    local taskmap_OPTS="\
+        --taskids= \
+        --ntasks= \
+        --nodeid= \
+        --hostname= \
+        --to= \
+    "
+    local purge_OPTS="\
+        --age-limit= \
+        --num-limit= \
+        -f --force \
+        --batch= \
+    "
+    local timeleft_OPTS="\
+        -H --human
+    "
+    local hostpids_OPTS="\
+        -d --delimiter= \
+        -t --timeout= \
+        -r --ranks= \
+    "
+    local last_OPTS=""
+
+    #  Handle options for all subcommands in one case statement
+    _flux_split_longopt && split=true
+    case $prev in
+        --path | -!(-*)p)
+            # wait-event, eventlog -p, --path=
+            COMPREPLY=( $(compgen -W "exec output input" -- "$cur") )
+            return
+            ;;
+        --format | -!(-*)f)
+            # wait-event, eventlog -f, --format=
+            COMPREPLY=( $(compgen -W "text json" -- "$cur") )
+            return
+            ;;
+        --time-format | -!(-*)T)
+            # wait-event, eventlog -T, --time-format=
+            COMPREPLY=( $(compgen -W "raw iso offset human" -- "$cur") )
+            return
+            ;;
+        --to | -!(-*)t)
+            case $cmd in
+                id)
+                    values="dec kvs hex dothex words f58 f58plain"
+                    ;;
+                taskmap)
+                    values="pmi raw multiline hosts"
+                    ;;
+                *)
+                    return
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "$values" -- "$cur") )
+            return
+            ;;
+        --user | -!(-*)u)
+            active_users=$(flux jobs -Ano {username} | uniq)
+            COMPREPLY=( $(compgen -W "${active_users}" -- "$cur") )
+            return
+            ;;
+        --flags | -!(-*)f)
+            COMPREPLY=( $(compgen -W "debug waitable novalidate" -- "$cur") )
+            return
+            ;;
+        -!(-*)[cmsSwiudr])
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cmd != "job" ]]; then
+        if [[ $cur != -* ]]; then
+           if _flux_contains_word ${cmd} ${subcmds}; then
+                #  These commands take active jobids by default:
+                active_jobs=$(flux jobs -no $(_flux_id_fmt $cur))
+                COMPREPLY=( $(compgen -W "${active_jobs}" -- "$cur") )
+                return 0
+            fi
+        fi
+        # Don't substitute longopts if already fully completed
+        if [[ $cur == --*= ]]; then
+            COMPREPLY=()
+            return 0
+        fi
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var}" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # no space if suggestions ends with '='
+            compopt -o nospace
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds} ${nojob_subcmds}" -- "$cur") )
+    fi
+    return 0
+}
+
+# flux-queue(1) completions
+_flux_queue()
+{
+    local cmd=$1 split=false
+    local subcmds="\
+        enable \
+        disable \
+        start \
+        stop \
+        status \
+        drain \
+        list \
+        idle
+    "
+    local enable_OPTS="\
+        -h --help \
+        -q --queue= \
+        -a --all \
+    "
+    local disable_OPTS="\
+        -h --help \
+        -q --queue= \
+        -a --all \
+        --quiet \
+        -v --verbose \
+        --nocheckpoint \
+    "
+    local start_OPTS="\
+        -h --help \
+        -q --queue= \
+        -v --verbose \
+        --quiet \
+    "
+    local stop_OPTS="\
+        -h --help \
+        -q --queue= \
+        -v --verbose \
+        --quiet \
+    "
+    local status_OPTS="\
+        -h --help \
+        -v --verbose \
+        -q --queue= \
+    "
+    local drain_OPTS="\
+        -h --help \
+        -t --timeout= \
+    "
+    local list_OPTS="\
+        -q --queue= \
+        -o --format= \
+        -n --no-header \
+    "
+
+    _flux_split_longopt && split=true
+    case $prev in
+        --queue | -!(-*)q)
+            _flux_complete_queue
+            return
+            ;;
+        --format | -!(-*)o)
+            _flux_complete_format_name flux queue $cmd
+            return
+            ;;
+        --timeout | -!(-*)t)
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cmd != "queue" ]]; then
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var}" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+        # Don't complete longopt if fully completed
+        if [[ $cur == --*= ]]; then
+            COMPREPLY=()
+            return 0
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+    return 0
+}
+
+_flux_complete_namespace() {
+    namespaces=$(flux kvs namespace list 2>/dev/null \
+                 | awk '!/^NAMESPACE/ {print $1}')
+    COMPREPLY=( $(compgen -W "$namespaces" -- "$cur") )
+    return
+}
+
+# flux-kvs(1) eventlog completions
+_flux_kvs_eventlog()
+{
+    local cmd=$1
+    local subcmds="append get wait-event"
+    local split=false
+
+    local append_OPTS="\
+        -N --namespace= \
+        -t --timestamp= \
+    "
+    local get_OPTS="\
+        -N --namespace= \
+        -W --waitcreate \
+        -w --watch \
+        -c --count= \
+        -u --unformatted \
+        -H --human \
+        -L --color= \
+    "
+    local wait_event_OPTS="\
+        -N --namespace= \
+        -W --waitcreate \
+        -u --unformatted \
+        -q --quiet \
+        -v --verbose \
+        -H --human \
+        -L --color= \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --namespace | -!(-*)N)
+            _flux_complete_namespace
+            return
+            ;;
+        -!(-*)[tc])
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cmd != "eventlog" ]]; then
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var}" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+        if [[ "$cur" != -* ]]; then
+            compopt -o default
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+    return 0
+}
+
+# flux-kvs(1) completions
+_flux_kvs()
+{
+    local cmd=$1
+    local split=false
+    local subcmds="\
+        namespace
+        get
+        put
+        dir
+        ls
+        unlink
+        link
+        readlink
+        mkdir
+        copy
+        move
+        dropcache
+        version
+        wait
+        getroot
+        eventlog
+    "
+    get_OPTS="\
+        -N --namespace= \
+        -r --raw \
+        -t --treeobj \
+        -a --at= \
+        -l --label \
+        -W --waitcreate \
+        -w --watch \
+        -u --uniq \
+        -A --append \
+        -f --full \
+        -c --count= \
+    "
+    put_OPTS="\
+        -N --namespace= \
+        -O --treeobj-root \
+        -b --blobref \
+        -s --sequence \
+        -r --raw \
+        -t --treeobj \
+        -n --no-merge \
+        -A --append \
+        -S --sync \
+    "
+    dir_OPTS="\
+        -N --namespace= \
+        -R --recursive \
+        -d --directory \
+        -w --width= \
+        -a --at \
+    "
+    ls_OPTS="\
+        -N --namespace= \
+        -R --recursive \
+        -d --directory \
+        -w --width= \
+        -1 --1 \
+        -F --classify \
+    "
+    unlink_OPTS="\
+        -N --namespace= \
+        -O --treeobj-root \
+        -b --blobref \
+        -s --sequence \
+        -R --recursive \
+        -f --force \
+    "
+    link_OPTS="\
+        -N --namespace= \
+        -O --treeobj-root \
+        -b --blobref \
+        -s --sequence \
+    "
+    readlink_OPTS="\
+        -N --namespace= \
+        -a --at= \
+        -o --namespace-only \
+        -k --key-only \
+    "
+    mkdir_OPTS="\
+        -N --namespace= \
+        -O --treeobj-root \
+        -b --blobref \
+        -s --sequence \
+    "
+    copy_OPTS="\
+        -S --src-namespace= \
+        -D --dst-namespace= \
+    "
+    move_OPTS="\
+        ${copy_OPTS}
+    "
+    dropcache_OPTS="\
+        -a --all
+    "
+    version_OPTS="\
+        -N --namespace= \
+    "
+    wait_OPTS="\
+        -N --namespace= \
+    "
+    getroot_OPTS="\
+        -N --namespace= \
+        -s --sequence \
+        -o --owner \
+        -b --blobref \
+    "
+
+    if [[ $cmd != "kvs" ]]; then
+
+        if _flux_contains_word "eventlog" ${COMP_WORDS[*]}; then
+            _flux_kvs_eventlog $(_flux_get_subcmd $cmd)
+            return 0
+        fi
+
+        _flux_split_longopt && split=true
+        case $prev in
+            --src-namespace | --dst-namespace | --namespace | -!(-*)[SDN])
+                _flux_complete_namespace
+                return
+                ;;
+            -!(-*)[awc])
+                return
+                ;;
+        esac
+        $split && return
+
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var}" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+        # Don't complete longopt if fully completed
+        if [[ $cur == --*= ]]; then
+            COMPREPLY=()
+            return 0
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+}
+
+# flux-overlay(1) completions
+_flux_overlay()
+{
+    local cmd=$1
+    local subcmds="status lookup parentof disconnect trace"
+    local split=false
+
+    local status_OPTS="\
+        -h --help \
+        -r --rank= \
+        -v --verbose= \
+        -t --timeout= \
+        --summary \
+        --down \
+        --no-pretty \
+        --no-ghost \
+        -L --color= \
+        -H --highlight= \
+        -w --wait= \
+    "
+    local lookup_OPTS="\
+        -h --help \
+    "
+    local parentof_OPTS="\
+        -h --help \
+    "
+    local disconnect_OPTS="\
+        -h --help \
+        -r --parent= \
+    "
+    local trace_OPTS="\
+        -h --help \
+        -r --rank= \
+        -t --type= \
+        -L --color= \
+        -H --human \
+        -d --delta \
+        -f --full \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        -!(-*)[rtwH])
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cmd != "overlay" ]]; then
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var}" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+        # Don't complete longopt if fully completed
+        if [[ $cur == --*= ]]; then
+            COMPREPLY=()
+            return 0
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+    return 0
+}
+
+# flux-config(1) completions
+_flux_config()
+{
+    local cmd=$1
+    local subcmds="reload get builtin load"
+    local split=false
+
+    local get_OPTS="\
+        -t --type= \
+        -q --quiet \
+        -d --default= \
+        -c --config-path \
+    "
+
+    if [[ $cmd != "config" ]]; then
+
+        if _flux_contains_word "load" ${COMP_WORDS[*]}; then
+            COMPREPLY=( $(compgen -f -- "$cur") $(compgen -d -- "$cur") )
+            return 0
+        fi
+
+        _flux_split_longopt && split=true
+        case $prev in
+            --config-path | -!(-*)c)
+                compopt -o filenames
+                COMPREPLY=( $(compgen -f -- "$cur") $(compgen -d -- "$cur") )
+                return
+                ;;
+            --type | -!(-*)t)
+                values="string integer real boolean object array fsd fsd-integer fsd-real"
+                COMPREPLY=( $(compgen -W "$values" -- "$cur") )
+                return
+                ;;
+            -!(-*)d)
+                return
+                ;;
+        esac
+        $split && return
+
+        COMPREPLY=( $(compgen -W "${!var} -h --help" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+    return 0
+}
+
+# flux-admin(8) completions
+_flux_admin()
+{
+    local cmd=$1
+    local subcmds="cleanup-push"
+
+    if [[ $cmd != "admin" ]]; then
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var} -h --help" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+    return 0
+}
+
+# flux-module(1) completions
+_flux_module()
+{
+    local cmd=$1
+    local subcmds_module_arg="remove reload stats debug trace"
+    local subcmds="list load ${subcmds_module_arg}"
+    local split=false
+
+    local load_OPTS="\
+        --name= \
+    "
+    local remove_OPTS="\
+        -f --force \
+    "
+    local reload_OPTS="\
+        ${remove_OPTS} \
+    "
+    local stats_OPTS="\
+        -p --parse= \
+        -s --scale=N \
+        -t --type= \
+        -R --rusage \
+        -c --clear \
+        -C --clear-all \
+    "
+    local debug_OPTS="\
+        -C --clear \
+        -S --set \
+        -s --setbit \
+        -c --clearbit \
+    "
+    local list_OPTS="\
+        -l --long \
+    "
+    local trace_OPTS="\
+        -h --help \
+        -T --topic= \
+        -t --type= \
+        -L --color= \
+        -H --human \
+        -d --delta \
+        -f --full \
+    "
+    if [[ $cmd != "module" ]]; then
+
+        _flux_split_longopt && split=true
+        case $prev in
+            -!(-*)[pst])
+                return
+                ;;
+        esac
+        $split && return
+
+        if [[ $cur != -* ]]; then
+            if _flux_contains_word ${cmd} ${subcmds_module_arg}; then
+                #  These commands take loaded module as args
+                modules=$(flux module list | grep -v Module | awk '{print $1}')
+                COMPREPLY=( $(compgen -W "${modules}" -- "$cur") )
+                return 0
+            elif _flux_contains_word ${cmd} "load"; then
+                compopt -o default -o bashdefault
+            fi
+        fi
+
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var} -h --help" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+        # Don't complete longopt if fully completed
+        if [[ $cur == --*= ]]; then
+            COMPREPLY=()
+            return 0
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+    return 0
+}
+
+# flux-jobtap(1) completions
+_flux_jobtap()
+{
+    local cmd=$1
+    local subcmds_plugin_arg="remove query"
+    local subcmds="load list ${subcmds_plugin_arg}"
+
+    load_OPTS="\
+        -r --remove= \
+    "
+    list_OPTS="\
+        -a --all \
+    "
+    if [[ $cmd != "jobtap" ]]; then
+
+        if [[ $cur != -* ]]; then
+            if _flux_contains_word ${cmd} ${subcmds_plugin_arg}; then
+                #  These commands take loaded plugins as args
+                plugins=$(flux jobtap list 2>/dev/null)
+                COMPREPLY=( $(compgen -W "${plugins}" -- "$cur") )
+                return 0
+            elif _flux_contains_word ${cmd} "load"; then
+                compopt -o bashdefault -o default
+            fi
+        fi
+
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var} -h --help" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+        # Don't complete longopt if fully completed
+        if [[ $cur == --*= ]]; then
+            COMPREPLY=()
+            return 0
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+    return 0
+}
+
+# flux-jobs(1) completions
+_flux_jobs()
+{
+    local cmd=$1
+    local split=false
+    local OPTS="\
+        -a \
+        -A \
+        -c --count=N \
+        -f --filter= \
+        -i --include= \
+        --since=WHEN \
+        -n --no-header \
+        -u --user= \
+        --name= \
+        -q --queue= \
+        -o --format= \
+        --color= \
+        -R --recursive \
+        -L --level= \
+        --recurse-all \
+        --threads= \
+        --stats \
+        --stats-only \
+    "
+
+    _flux_split_longopt && split=true
+    case $prev in
+        --queue | -!(-*)q)
+            _flux_complete_queue
+            return
+            ;;
+        --format | -!(-*)o)
+            _flux_complete_format_name flux jobs
+            return
+            ;;
+        -!(-*)[Lcf])
+            return
+            ;;
+    esac
+    $split && return
+
+    # Don't complete longopt if fully completed
+    if [[ $cur == --*= ]]; then
+        return
+    fi
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-pgrep(1) completions
+_flux_pgrep()
+{
+    local cmd=$1
+    local split=false
+    local OPTS="\
+        -a \
+        -A \
+        -c --count= \
+        --max-entries= \
+        -f --filter= \
+        -n --no-header \
+        -u --user= \
+        -q --queue= \
+        -o --format= \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --queue | -!(-*)q)
+            _flux_complete_queue
+            return
+            ;;
+        --format | -!(-*)o)
+            # flux-prgrep uses flux-jobs named formats
+            _flux_complete_format_name flux jobs
+            return
+            ;;
+        --user | -!(-*)u)
+            active_users=$(flux jobs -Ano {username} | uniq)
+            COMPREPLY=( $(compgen -W "$active_users" -- "$cur") )
+            return
+            ;;
+        -!(-*)[cf])
+            return
+            ;;
+    esac
+    $split && return
+
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-pkill(1) completions
+_flux_pkill()
+{
+    local cmd=$1
+    local split=false
+    local OPTS="\
+        -A \
+        -c --count= \
+        --max-entries= \
+        -f --filter= \
+        -u --user= \
+        -q --queue= \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --queue | -!(-*)q)
+            _flux_complete_queue
+            return
+            ;;
+        --user | -!(-*)u)
+            active_users=$(flux jobs -Ano {username} | uniq)
+            COMPREPLY=( $(compgen -W "$active_users" -- "$cur") )
+            return
+            ;;
+        -!(-*)[cf])
+            return
+            ;;
+    esac
+    $split && return
+
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-pstree(1) completions
+_flux_pstree()
+{
+    local cmd=$1
+    local split=false
+
+    local OPTS="\
+        -a --all \
+        -c --count= \
+        -f --filter= \
+        -x --extended \
+        -l --long \
+        -L --level= \
+        -p --parent-ids \
+        -n --no-header \
+        -X --no-combine \
+        -o --label= \
+        --parent-label= \
+        --detail-format= \
+        -d --details= \
+        -C --compact \
+        --ascii \
+        --skip-root \
+    "
+
+    _flux_split_longopt && split=true
+    case $prev in
+        --details | -!(-*)d)
+            COMPREPLY=( $(compgen -W 'default resources progress stats' -- "$cur") )
+            return
+            ;;
+        -!(-*)[cfLo])
+            return
+            ;;
+    esac
+    $split && return
+
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-watch(1) completions
+_flux_watch()
+{
+    local cmd=$1
+    local split=false
+    local OPTS="\
+        -a --active \
+        -A --all \
+        -c --count= \
+        -f --filter= \
+        -l --label-io \
+        --since= \
+        --progress \
+        --jps \
+        --verbose
+    "
+    if [[ $cur != -* ]]; then
+        #  Attempt to substitute an active jobid
+        active_jobs=$(flux jobs -no $(_flux_id_fmt $cur))
+        COMPREPLY=( $(compgen -W "${active_jobs}" -- "$cur") )
+        return 0
+    fi
+
+    _flux_split_longopt && split=true
+    case $prev in
+        -!(-*)[cf])
+            return
+            ;;
+    esac
+    $split && return
+
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+#  flux-dump(1) completions
+_flux_dump()
+{
+    local cmd=$1
+    local OPTS="\
+        -v --verbose \
+        -q --quiet \
+        --checkpoint \
+        --no-cache \
+        --ignore-failed-read \
+    "
+    compopt -o default -o bashdefault
+
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+#  flux-restore(1) completions
+_flux_restore()
+{
+    local cmd=$1
+    local OPTS="\
+        -v --verbose \
+        -q --quiet \
+        --checkpoint \
+        --key= \
+        --no-cache \
+    "
+    compopt -o default -o bashdefault
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-top(1) completions
+_flux_top()
+{
+    local cmd=$1
+    local split=false
+    local OPTS="\
+        --color= \
+        -q --queue= \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --queue | -!(-*)q)
+            _flux_complete_queue
+            return
+            ;;
+    esac
+    $split && return
+
+    #  flux-top(1) can target jobids that are also instances
+    local jobs=$(flux jobs -no {uri}:$(_flux_id_fmt $cur) | grep -v ^None: \
+                 | sed -n 's/.*://p')
+    COMPREPLY=( $(compgen -W "${OPTS} $jobs" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-dmesg(1) completions
+_flux_dmesg()
+{
+    local cmd=$1
+    OPTS="\
+        -C --clear \
+        -c --read-clear \
+        -f --follow \
+        -n --new \
+        -H --human \
+        -d --delta \
+        -L --color= \
+    "
+    if [[ $cur == --*= ]]; then
+        return
+    fi
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-cron(1) completions
+_flux_cron()
+{
+    local cmd=$1
+    local split=false
+    local subcmds="\
+        help \
+        interval \
+        event \
+        tab \
+        at \
+        list \
+    "
+    local cron_entry_subcmds="\
+        dump \
+        delete \
+        stop \
+        start \
+        sync \
+    "
+    local interval_OPTS="\
+        -c --count= \
+        -N --name= \
+        -a --after= \
+        -o --options= \
+        -E --preserve-env \
+        -d --working-dir= \
+    "
+    local event_OPTS="\
+        -n --nth= \
+        -a --after= \
+        -i --min-interval= \
+        -c --count= \
+        -o --options= \
+        -N --name= \
+        -E --preserve-env \
+        -d --working-dir= \
+    "
+    local tab_OPTS="\
+        -o --options= \
+        -E --preserve-env \
+        -d --working-dir= \
+    "
+    local at_OPTS="\
+        ${tab_OPTS} \
+    "
+    local dump_OPTS="\
+        -k --key= \
+    "
+    local delete_OPTS="\
+        -k --kill \
+    "
+    local sync_OPTS="\
+        -d --disable \
+        -e --epsilon= \
+    "
+    if [[ $cmd != "cron" ]]; then
+
+        if [[ $cur != -* ]]; then
+            if _flux_contains_word ${cmd} ${cron_entry_subcmds}; then
+                #  These commands take cron entries as args
+                entries=$(flux cron list 2>/dev/null \
+                          | grep -v ID | awk '{print $1}')
+                COMPREPLY=( $(compgen -W "${entries}" -- "$cur") )
+                return 0
+            fi
+        fi
+
+        _flux_split_longopt && split=true
+        case $prev in
+            --working-dir | -!(-*)d)
+                compopt -o filenames
+                COMPREPLY=( $(compgen -d -- "$cur") )
+                return
+                ;;
+            -!(-*)k)
+                [[ $cmd == "dump" ]] && return
+                ;;
+            -!(-*)[cNaonie])
+                return
+                ;;
+        esac
+        $split && return
+
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var} -h --help" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds} ${cron_entry_subcmds}" -- "$cur") )
+    fi
+}
+
+# flux-R(1) completions
+_flux_R()
+{
+    local cmd=$1
+    local split=false
+
+    local subcmds="\
+        encode \
+        append \
+        diff \
+        intersect \
+        remap \
+        rerank \
+        decode \
+        verify \
+        set-property \
+        parse-config \
+    "
+    local encode_OPTS="\
+        -r --ranks= \
+        -c --cores= \
+        -g --gpus= \
+        -H --hosts= \
+        -l --local \
+        -f --xml \
+    "
+    local decode_OPTS="\
+        -s --short \
+        -n --nodelist \
+        -r --ranks \
+        -c --count= \
+        -i --include= \
+        -x --exclude= \
+        -p --properties= \
+    "
+    if [[ $cmd != "R" ]]; then
+        # Don't complete longopt if fully completed
+        if [[ $cur == --*= ]]; then
+            COMPREPLY=()
+            return 0
+        fi
+
+        _flux_split_longopt && split=true
+        case $prev in
+            -!(-*)[ixcgH])
+                return
+                ;;
+        esac
+        $split && return
+
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var} -h --help" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # Add space if there is not a '=' in suggestions
+            compopt -o nospace
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+}
+
+#  flux-ping(1) completions
+_flux_ping()
+{
+    local cmd=$1
+    local split=false
+    OPTS="\
+        -r --rank= \
+        -p --pad= \
+        -i --interval= \
+        -c --count= \
+        -b --batch \
+        -u --userid \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        -!(-*)[rpic])
+            return
+            ;;
+    esac
+    $split && return
+
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-exec(1) completions
+_flux_exec()
+{
+    local cmd=$1
+    local split=false
+    OPTS="\
+        -r --rank= \
+        -x --exclude= \
+        -d --dir= \
+        -l --label-io \
+        -n --noinput \
+        -v --verbose \
+        -q --quiet \
+    "
+
+    _flux_split_longopt && split=true
+    case $prev in
+        --dir | -!(-*)d)
+            compopt -o filenames
+            COMPREPLY=( $(compgen -d -- "$cur") )
+            return
+            ;;
+        -!(-*)[rx])
+            return
+            ;;
+    esac
+    $split && return
+
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    else
+        # Reset to default completions for path/executable
+        compopt -o bashdefault -o default
+    fi
+    return 0
+}
+
+# flux-getattr(1), flux-setattr(1), flux-lsattr(1) completions
+_flux_attr()
+{
+    local cmd=$1
+    local lsattr_OPTS="\
+        -v --values \
+    "
+    if [[ $cmd != "flux" ]]; then
+        if [[ $cur != -* && $cmd == "getattr" ]]; then
+            COMPREPLY=( $(compgen -W "$(flux lsattr)" -- "$cur") )
+            return 0
+        fi
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var} -h --help" -- "$cur") )
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+}
+
+# flux-content(1) completions
+_flux_content()
+{
+    local cmd=$1
+    local subcmds="load store flush dropcache"
+    load_OPTS="\
+        -b --bypass-cache \
+    "
+    store_OPTS="\
+        -b --bypass-cache \
+        --chunksize=
+    "
+    if [[ $cmd != "content" ]]; then
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var} -h --help" -- "$cur") )
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+}
+
+# flux-start(1) completions
+_flux_start()
+{
+    local cmd=$1
+    local split=false
+
+    #  Most test-only options left off on purpose here
+    OPTS="\
+        -v --verbose \
+        -X --noexec \
+        -o --broker-opts= \
+        --wrap= \
+        -s --test-size= \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        -!(-*)[os])
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cur != -* ]]; then
+        compopt -o default -o bashdefault -o filenames
+        COMPREPLY=( $(compgen -f -c -- "$cur") )
+        return
+    fi
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-broker(1) completions
+_flux_broker()
+{
+    local cmd=$1
+    local split=false
+
+    OPTS="\
+        -h --help \
+        -v --verbose= \
+        -S --setattr= \
+        -c --config-path= \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --config-path | -!(-*)c)
+            compopt -o filenames
+            COMPREPLY=( $(compgen -f -- "$cur") $(compgen -d -- "$cur") )
+            return
+            ;;
+        -!(-*)[S])
+            return
+            ;;
+    esac
+    $split && return
+
+    if [[ $cur != -* ]]; then
+        compopt -o default -o bashdefault -o filenames
+        COMPREPLY=( $(compgen -f -c -- "$cur") )
+        return
+    fi
+    COMPREPLY=( $(compgen -W "${OPTS} -h --help" -- "$cur") )
+    if [[ "${COMPREPLY[@]}" == *= ]]; then
+        # Add space if there is not a '=' in suggestions
+        compopt -o nospace
+    fi
+    return 0
+}
+
+# flux-housekeeping(1) completions
+_flux_housekeeping()
+{
+    local cmd=$1
+    local subcmds="list kill"
+    local split=false
+    list_OPTS="\
+        -i --include= \
+        -o --format= \
+        -n --no-header \
+    "
+    kill_OPTS="\
+        -s --signal= \
+        -t --targets= \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --format | -!(-*)o)
+            _flux_complete_format_name flux housekeeping list
+            return
+            ;;
+    esac
+    if [[ $cmd != "housekeeping" ]]; then
+        if [[ $cur != -* ]]; then
+            if _flux_contains_word ${cmd} "kill"; then
+                hk_active=$(flux housekeeping list -no $(_flux_id_fmt $cur))
+                COMPREPLY=( $(compgen -W "${hk_active}" -- "$cur") )
+                return 0
+            fi
+        fi
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var}" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # no space if suggestions ends with '='
+            compopt -o nospace
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+}
+
+_flux_core()
+{
+    local cur prev cmd subcmd matched
+    local cmds=$(__get_flux_subcommands)
+    cmd=$(_flux_get_cmd)
+    subcmd=$(_flux_get_subcmd $cmd)
+
+    #  If available, use bash-completion _get_comp_words_by_ref()
+    #  This better handles not splitting on `=` for long options.
+    if type _get_comp_words_by_ref >/dev/null 2>&1; then
+        _get_comp_words_by_ref -n = cur prev
+    else
+        cur=${COMP_WORDS[COMP_CWORD]}
+        prev=${COMP_WORDS[COMP_CWORD-1]}
+    fi
+
+    matched=t
+    case "${cmd}" in
+    submit|run|alloc|batch|bulksubmit)
+        _flux_submit_commands $cmd
+        ;;
+    proxy)
+        _flux_proxy $subcmd
+        ;;
+    cancel)
+        _flux_cancel $subcmd
+        ;;
+    resource)
+        _flux_resource $subcmd
+        ;;
+    job)
+        _flux_job $subcmd
+        ;;
+    kvs)
+        _flux_kvs $subcmd
+        ;;
+    queue)
+        _flux_queue $subcmd
+        ;;
+    overlay)
+        _flux_overlay $subcmd
+        ;;
+    config)
+        _flux_config $subcmd
+        ;;
+    admin)
+        _flux_admin $subcmd
+        ;;
+    module)
+        _flux_module $subcmd
+        ;;
+    jobtap)
+        _flux_jobtap $subcmd
+        ;;
+    jobs)
+        _flux_jobs $subcmd
+        ;;
+    pgrep)
+        _flux_pgrep $subcmd
+        ;;
+    pkill)
+        _flux_pkill $subcmd
+        ;;
+    pstree)
+        _flux_pstree $subcmd
+        ;;
+    dump)
+        _flux_dump $subcmd
+        ;;
+    restore)
+        _flux_restore $subcmd
+        ;;
+    top)
+        _flux_top $subcmd
+        ;;
+    dmesg)
+        _flux_dmesg $subcmd
+        ;;
+    ping)
+        _flux_ping $subcmd
+        ;;
+    exec)
+        _flux_exec $subcmd
+        ;;
+    R)
+        _flux_R $subcmd
+        ;;
+    cron)
+        _flux_cron $subcmd
+        ;;
+    *attr)
+        _flux_attr $subcmd
+        ;;
+    content)
+        _flux_content $subcmd
+        ;;
+    start)
+        _flux_start $subcmd
+        ;;
+    watch)
+        _flux_watch $subcmd
+        ;;
+    update)
+        _flux_update $subcmd
+        ;;
+    hostlist)
+        _flux_hostlist $subcmd
+        ;;
+    shutdown)
+        _flux_shutdown $subcmd
+        ;;
+    archive)
+        _flux_archive $subcmd
+        ;;
+    broker)
+        _flux_broker $subcmd
+        ;;
+    python)
+        # reset to defaults and let readline take over
+        compopt -o default
+        COMPREPLY=()
+        ;;
+    housekeeping)
+        _flux_housekeeping $subcmd
+        ;;
+    -*)
+        COMPREPLY=( $(compgen -W "${FLUX_OPTS}" -- "$cur") )
+        ;;
+    help)
+        COMPREPLY=( $(compgen -W "${cmds}" -- "$cur") )
+        ;;
+    *)
+        # return with no suggestions for unknown subcommands:
+        [[ "$prev" != "flux" ]] && return
+
+        matched=f
+        COMPREPLY=( $(compgen -W "${cmds}" -- "$cur") )
+        ;;
+    esac
+
+    # If there was an exact command match, and current == command and
+    # there are no other completion possibilities, return it so it gets
+    # completed by a tab, e.g. `flux top<tab>` adds a space: `flux top `.
+    if test "$matched" = "t" -a -z "$COMPREPLY" -a "$cur" = "$cmd"; then
+        COMPREPLY="$cur"
+    fi
+
+    return 0
+}
+
+complete -F _flux_core flux
+
+# vi: ts=4 sw=4 expandtab
diff --git a/etc/completions/get_builtins.sh b/etc/completions/get_builtins.sh
new file mode 100755
index 000000000000..9ed307660b7a
--- /dev/null
+++ b/etc/completions/get_builtins.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+#
+
+get_builtins() {
+    local FLUX_BUILTINS
+
+    builtin_path=$1
+
+    for builtin in $builtin_path/*; do
+        if [[ $builtin == *".c" && \
+              $builtin != *"attr"* ]]; then
+            builtin="${builtin%.*}"
+            FLUX_BUILTINS+="${builtin##*/}:"
+        fi
+    done
+
+    while read line; do
+        if [[ $line == *"optparse_reg_subcommand"* \
+              && $line == *"attr"* ]]; then
+             line="${line##*_}"
+            FLUX_BUILTINS+="${line%,*}:"
+        fi
+    done < $builtin_path/attr.c
+
+    echo FLUX_BUILTINS="$FLUX_BUILTINS"
+}
+
+get_builtins $1
diff --git a/etc/conf.d/bootstrap.toml b/etc/conf.d/bootstrap.toml
deleted file mode 100644
index 03391c2d9561..000000000000
--- a/etc/conf.d/bootstrap.toml
+++ /dev/null
@@ -1,9 +0,0 @@
-#
-# This bootstrap configuration is for a single-node
-# (i.e. workstation) Flux system configuration.
-#
-# See flux-config-bootstrap(5) for more information on this
-# configuration section.
-#
-
-[bootstrap]
diff --git a/etc/flux-core.pc.in b/etc/flux-core.pc.in
index 7265193c35d7..75761ed618a6 100644
--- a/etc/flux-core.pc.in
+++ b/etc/flux-core.pc.in
@@ -2,6 +2,15 @@ prefix=@prefix@
 exec_prefix=@exec_prefix@
 libdir=@libdir@
 includedir=@includedir@
+datarootdir=@datarootdir@
+mandir=@mandir@
+fluxconfdir=@fluxconfdir@
+fluxcmddir=@fluxcmddir@
+fluxlibdir=@fluxlibdir@
+fluxmoddir=@fluxmoddir@
+fluxcmdhelpdir=@datadir@/flux/help.d
+fluxshellrcdir=@fluxconfdir@/shell
+fluxshellpluginpath=@fluxlibdir@/shell/plugins
 
 Name: flux-core
 Description: Flux Resource Manager Framework Core
diff --git a/etc/flux-epilog@.service.in b/etc/flux-epilog@.service.in
new file mode 100644
index 000000000000..15c229dd84d8
--- /dev/null
+++ b/etc/flux-epilog@.service.in
@@ -0,0 +1,9 @@
+[Unit]
+Description=Epilog for Flux job %I
+CollectMode=inactive-or-failed
+
+[Service]
+Type=oneshot
+EnvironmentFile=-@X_RUNSTATEDIR@/flux-epilog@%I.env
+ExecStart=@X_SYSCONFDIR@/flux/system/epilog
+ExecStopPost=-rm -f @X_RUNSTATEDIR@/flux-epilog@%I.env
diff --git a/etc/flux-hostlist.pc.in b/etc/flux-hostlist.pc.in
new file mode 100644
index 000000000000..bbc3bc119629
--- /dev/null
+++ b/etc/flux-hostlist.pc.in
@@ -0,0 +1,10 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: flux-hostlist
+Description: Flux Resource Manager Hostlist Library
+Version: @PACKAGE_VERSION@
+Libs: -L${libdir} -lflux-hostlist
+Cflags: -I${includedir}
diff --git a/etc/flux-housekeeping@.service.in b/etc/flux-housekeeping@.service.in
new file mode 100644
index 000000000000..cd6012351faf
--- /dev/null
+++ b/etc/flux-housekeeping@.service.in
@@ -0,0 +1,18 @@
+[Unit]
+Description=Housekeeping for Flux job %I
+CollectMode=inactive-or-failed
+
+[Service]
+Type=oneshot
+EnvironmentFile=-@X_RUNSTATEDIR@/flux-housekeeping@%I.env
+ExecStart=@X_SYSCONFDIR@/flux/system/housekeeping
+ExecStopPost=-rm -f @X_RUNSTATEDIR@/flux-housekeeping@%I.env
+ExecStopPost=-sh -c '\
+    if test "$SERVICE_RESULT" != "success"; then \
+        message="housekeeping@%I ${SERVICE_RESULT:-failure}"; \
+        if test "${EXIT_CODE}${EXIT_STATUS}"; then \
+            message="$message: $EXIT_CODE $EXIT_STATUS"; \
+        fi; \
+        flux resource drain $(flux getattr rank) $message; \
+    fi \
+'
diff --git a/etc/flux-prolog@.service.in b/etc/flux-prolog@.service.in
new file mode 100644
index 000000000000..14d6a1a92ca3
--- /dev/null
+++ b/etc/flux-prolog@.service.in
@@ -0,0 +1,9 @@
+[Unit]
+Description=Prolog for Flux job %I
+CollectMode=inactive-or-failed
+
+[Service]
+Type=oneshot
+EnvironmentFile=-@X_RUNSTATEDIR@/flux-prolog@%I.env
+ExecStart=@X_SYSCONFDIR@/flux/system/prolog
+ExecStopPost=-rm -f @X_RUNSTATEDIR@/flux-prolog@%I.env
diff --git a/etc/flux-taskmap.pc.in b/etc/flux-taskmap.pc.in
new file mode 100644
index 000000000000..8ae3e7debab8
--- /dev/null
+++ b/etc/flux-taskmap.pc.in
@@ -0,0 +1,10 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: flux-taskmap
+Description: Flux Resource Manager Taskmap Library
+Version: @PACKAGE_VERSION@
+Libs: -L${libdir} -lflux-taskmap
+Cflags: -I${includedir}
diff --git a/etc/flux.conf b/etc/flux.conf
new file mode 100644
index 000000000000..d6feb54bbbfe
--- /dev/null
+++ b/etc/flux.conf
@@ -0,0 +1,4 @@
+# See tmpfiles.d(5)
+# remove Flux dump files older than 30 days
+
+e /var/lib/flux/dump - - - 30d
diff --git a/etc/flux.service.in b/etc/flux.service.in
index 3cabc88b7a81..8203b4976f92 100644
--- a/etc/flux.service.in
+++ b/etc/flux.service.in
@@ -1,25 +1,48 @@
 [Unit]
 Description=Flux message broker
+Wants=munge.service
 
 [Service]
-Environment=FLUX_USERDB_OPTIONS=--default-rolemask=user
+Type=notify
+NotifyAccess=all
 TimeoutStopSec=90
 KillMode=mixed
-ExecStart=@X_BINDIR@/flux broker \
+ExecStart=/bin/bash -c '\
+  XDG_RUNTIME_DIR=/run/user/$UID \
+  DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$UID/bus \
+  @X_BINDIR@/flux broker \
+  --config-path=@X_SYSCONFDIR@/flux/system/conf.d \
+  -Scron.directory=@X_SYSCONFDIR@/flux/system/cron.d \
   -Srundir=@X_RUNSTATEDIR@/flux \
+  -Sstatedir=${STATE_DIRECTORY:-/var/lib/flux} \
   -Slocal-uri=local://@X_RUNSTATEDIR@/flux/local \
-  -Sboot.method=config \
   -Slog-stderr-level=6 \
-  -Scontent.backing-path=@X_LOCALSTATEDIR@/lib/flux/content.sqlite \
-  -Sbroker.rc2_none
+  -Slog-stderr-mode=local \
+  -Sbroker.rc2_none \
+  -Sbroker.quorum=1 \
+  -Sbroker.quorum-timeout=none \
+  -Sbroker.cleanup-timeout=45 \
+  -Sbroker.exit-norestart=42 \
+  -Sbroker.sd-notify=1 \
+  -Scontent.dump=auto \
+  -Scontent.restore=auto \
+'
+SyslogIdentifier=flux
+ExecReload=@X_BINDIR@/flux config reload
+Restart=always
+RestartSec=30s
+RestartPreventExitStatus=42
+SuccessExitStatus=42
 User=flux
 Group=flux
 RuntimeDirectory=flux
 RuntimeDirectoryMode=0755
+StateDirectory=flux
+StateDirectoryMode=0700
 PermissionsStartOnly=true
-ExecStartPre=-/bin/mkdir -p @X_LOCALSTATEDIR@/lib/flux
-ExecStartPre=/bin/chown flux:flux @X_LOCALSTATEDIR@/lib/flux
-ExecStartPre=/bin/chmod 0700 @X_LOCALSTATEDIR@/lib/flux
+ExecStartPre=/usr/bin/loginctl enable-linger flux
+ExecStartPre=bash -c 'systemctl start user@$(id -u flux).service'
+
 #
 # Delegate cgroup control to user flux, so that systemd doesn't reset
 #  cgroups for flux initiated processes, and to allow (some) cgroup
diff --git a/etc/gen-cmdhelp.pl b/etc/gen-cmdhelp.pl
deleted file mode 100755
index 601daf9b013c..000000000000
--- a/etc/gen-cmdhelp.pl
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/perl
-#
-#  Process .adoc manpage files and generate JSON file for use by
-#   flux-help(1) command default list of commands. By default, any lines:
-#
-#   // flux-help-[key]: value
-#
-#   are output directly into JSON object as `"key": "value"` for the
-#   current command entry with the exception of `flux-help-include:`
-#   which is used to explicitly include the command in `flux-help(1)`
-#   output, but use autogenerated data for other keys.
-#
-#   The keys:
-#     "command": shortname 
-#     "description": short description
-#
-#   are autogenerated from the asciidoc manpage. If the name/description
-#   is to be overridden, then `flux-help-command` and `flux-help-description`
-#   can be used.
-#
-# If no `flux-help-*` tags appear in an adoc file, it is ignored.
-#
-use strict;
-use warnings;
-use Getopt::Long qw/ :config gnu_getopt /;
-
-my $opts = {};
-my $rc = GetOptions ($opts, 'category|c=s');
-
-die "Usage: $0 --category=[label] FILES...\n" if !$#ARGV or !$rc;
-
-print ("[\n");
-my $filecount = 0;
-
-for my $file (@ARGV) {
-    my %h = ();
-    my %data = ();
-
-    $data{category} = $opts->{category} if ($opts->{category});
-
-    ($data{basename}) = $file =~ m/flux-(.*).adoc/;
-
-    open (FILE, "$file") or die ("can't open $file: $@\n");
-
-    while (<FILE>) {
-        if (m|// *flux-help-(\S+)\s*: *(.*)\s*$|) {
-            $h{$1} = $2;
-        }
-        if (m|^NAME| && <FILE> =~ m|^----|) {
-            <FILE> =~ m/^\s*flux-(.+)\s+-\s*(.+\S)/;
-            $data{"command"} = $1;
-            $data{"description"} = $2;
-        }
-    }
-    next unless keys %h;
-
-    $h{command} = $data{command} if not $h{command};
-    $h{description} = $data{description} if not $h{description};
-    $h{category} = $data{category} if $data{category} and not $h{cateegory};
-
-    print (",\n") if ($filecount++);
-    print ("  {\n");
-    my $n = 0;
-
-    for my $key (sort keys %h) {
-        next if $key eq "include";
-        print (",\n") if $n++;
-        printf ("    \"%s\": \"%s\"", $key, $h{$key});
-    }
-    printf ("\n  }");
-}
-
-print ("\n]");
-print ("\n");
-
-# vi: ts=4 sw=4 expandtab
diff --git a/etc/gen-cmdhelp.py b/etc/gen-cmdhelp.py
new file mode 100644
index 000000000000..d2758fec5059
--- /dev/null
+++ b/etc/gen-cmdhelp.py
@@ -0,0 +1,92 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+# Usage: python3 gen-cmdhelp.py path/to/sphinx/conf.py
+#
+# Generate flux-core cmdhelp from rst comments in manpages
+# Reads sphinx conf.py to get list of section 1 manpages and
+# their command names/descriptions
+#
+
+import os
+import sys
+import json
+from os import path
+
+
+class HelpEntries:
+    def __init__(self):
+        self.sections = [
+            {
+                "name": "submission",
+                "description": "run and submit jobs, allocate resources",
+            },
+            {
+                "name": "jobs",
+                "description": "list and interact with jobs",
+            },
+            {
+                "name": "instance",
+                "description": "get resource, queue and other instance information",
+            },
+            {
+                "name": "other",
+                "description": "other useful commands",
+            },
+        ]
+        for entry in self.sections:
+            entry["commands"] = []
+
+    def add_entry(self, name, description, section="other"):
+        for entry in self.sections:
+            if entry["name"] == section:
+                info = dict(name=name, description=description)
+                entry["commands"].append(info)
+
+
+if len(sys.argv) < 2:
+    print(f"Usage: {sys.argv[0]} sphinxconf.py", file=sys.stderr)
+
+sphinxconf = sys.argv[1]
+docsdir = os.path.abspath(path.dirname(sphinxconf))
+
+# Don't exec the sphinxconf, instead import what we need
+sys.path.insert(0, docsdir)
+from manpages import man_pages
+
+visited = dict()
+
+entries = HelpEntries()
+for (path, cmd, descr, author, sec) in man_pages:
+    if sec != 1 or path in visited:
+        continue
+    visited[path] = True
+    rst_file = os.path.join(docsdir, f"{path}.rst")
+    with open(rst_file, "r", encoding="utf-8") as f:
+        include_flag = False
+        section = "other"
+        for line in f:
+            line = line.rstrip("\n")
+            if ".. flux-help-" in line:
+                include_flag = True
+                if cmd.startswith("flux-"):
+                    cmd = cmd[5:]
+                if ".. flux-help-description: " in line:
+                    descr = " ".join(line.split(" ")[2:])
+                if ".. flux-help-command: " in line:
+                    cmd = " ".join(line.split(" ")[2:])
+                if ".. flux-help-section: " in line:
+                    section = " ".join(line.split(" ")[2:])
+
+        if include_flag:
+            entries.add_entry(cmd, descr, section)
+
+print(json.dumps(entries.sections, indent=2, sort_keys=True), file=sys.stdout)
+sys.path.pop(0)
diff --git a/etc/rc1 b/etc/rc1
index 800f15bd2531..32fbaac55300 100755
--- a/etc/rc1
+++ b/etc/rc1
@@ -1,52 +1,118 @@
-#!/bin/bash -e
+#!/bin/sh -e
 
-# Usage: wait_check pid [pid ...]
-wait_check() {
-    local pid
-    for pid in $*; do wait $pid; done
+# Allow connector-local more time to start listening on socket
+RANK=$(FLUX_LOCAL_CONNECTOR_RETRY_COUNT=30 flux getattr rank)
+
+# Usage: modload {all|<rank>} modname [args ...]
+modload() {
+    local where=$1; shift
+    if test "$where" = "all" || test $where -eq $RANK; then
+        flux module load $*
+    fi
+}
+
+backing_module() {
+    local backingmod=$(flux getattr content.backing-module 2>/dev/null) || :
+    echo ${backingmod:-content-sqlite}
 }
 
-# Allow connector-local more time to start listening on socket in rc1 only
-export FLUX_LOCAL_CONNECTOR_RETRY_COUNT=10
-
-declare -a pids
-flux exec -r all flux module load barrier & pids+=($!)
-flux module load content-sqlite & pids+=($!)
-flux exec -r all flux module load aggregator & pids+=($!)
-wait_check ${pids[@]}
-unset pids
-
-declare -a pids
-flux module load kvs
-flux exec -r all -x 0 flux module load kvs & pids+=($!)
-flux exec -r all flux module load kvs-watch & pids+=($!)
-wait_check ${pids[@]}
-unset pids
-
-declare -a pids
-flux hwloc reload & pids+=($!)
-flux exec -r all flux module load job-info & pids+=($!)
-flux exec -r all flux module load job-ingest & pids+=($!)
-flux module load cron sync=hb & pids+=($!)
-flux module load userdb ${FLUX_USERDB_OPTIONS} & pids+=($!)
-flux module load job-manager & pids+=($!)
-wait_check ${pids[@]}
-unset pids
-
-declare -a pids
-flux module load job-exec &  pids+=($!)
-flux module load sched-simple & pids+=($!)
-wait_check ${pids[@]}
-unset pids
+# Get the latest config in case it changed while upstream broker was down.
+# See also: flux-framework/flux-core#4663
+if test $RANK -gt 0; then
+    flux config reload
+fi
+
+modload all content
+modload all barrier
+if test "$(flux config get --default=false systemd.enable)" = "true"; then
+    modload all sdbus
+    modload all sdexec
+fi
+
+if test $RANK -eq 0; then
+    backingmod=$(backing_module)
+    dumpfile=$(flux getattr content.restore 2>/dev/null) || :
+    if test -n "${dumpfile}"; then
+        if test "${dumpfile}" = "auto"; then
+            statedir=$(flux getattr statedir 2>/dev/null) || :
+            dumplink="${statedir:-.}/dump/RESTORE"
+            if test -h "${dumplink}"; then
+                dumpfile=$(readlink -f ${dumplink}) || :
+            else
+                dumpfile=""
+                dumplink=""
+            fi
+        fi
+    fi
+    if test -n "${dumpfile}"; then
+        if test "${backingmod}" != "none"; then
+            flux module load ${backingmod} truncate
+        fi
+        echo "restoring content from ${dumpfile}"
+        flux restore --quiet --checkpoint --size-limit=100M ${dumpfile}
+        if test -n "${dumplink}"; then
+            rm -f ${dumplink}
+        fi
+    elif test "${backingmod}" != "none"; then
+        flux module load ${backingmod}
+    fi
+fi
+
+modload all kvs
+modload all kvs-watch
+
+if test $RANK -eq 0; then
+    if test "$(backing_module)" != "none"; then
+        flux startlog --post-start-event
+    fi
+fi
+
+modload all resource
+modload 0 cron sync=heartbeat.pulse
+modload 0 job-manager
+modload all job-info
+modload 0 job-list
+
+if test $RANK -eq 0; then
+    if test "$(backing_module)" != "none"; then
+        if ! flux startlog --check --quiet; then
+	    echo "Flux was not shut down properly.  Data may have been lost."
+        fi
+    fi
+fi
+
+modload all job-ingest
+modload 0 job-exec
+modload 0 heartbeat
 
 core_dir=$(cd ${0%/*} && pwd -P)
 all_dirs=$core_dir${FLUX_RC_EXTRA:+":$FLUX_RC_EXTRA"}
 IFS=:
-shopt -s nullglob
 for rcdir in $all_dirs; do
     for rcfile in $rcdir/rc1.d/*; do
+	[ -e $rcfile ] || continue
 	echo running $rcfile
         $rcfile
     done
 done
-shopt -u nullglob
+
+# Print module that has registered 'sched' service, if any
+lookup_sched_module() {
+    flux module list | awk '$NF ~ /^\s*(\w+,)?sched(,\w+)?\s*$/ { print $1 }'
+}
+
+if test $RANK -eq 0 -a "${FLUX_SCHED_MODULE}" != "none" \
+                    -a -z "$(lookup_sched_module)"; then
+    flux module load ${FLUX_SCHED_MODULE:-sched-simple}
+fi
+
+if test $RANK -eq 0; then
+    if test -z "${FLUX_DISABLE_JOB_CLEANUP}"; then
+	flux admin cleanup-push <<-EOT
+	flux queue stop --quiet --all --nocheckpoint
+	flux resource acquire-mute
+	flux cancel --user=all --quiet --states RUN
+	flux queue idle --quiet
+	EOT
+    fi
+fi
diff --git a/etc/rc1.d/01-enclosing-instance b/etc/rc1.d/01-enclosing-instance
deleted file mode 100755
index 7640647fc522..000000000000
--- a/etc/rc1.d/01-enclosing-instance
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-# Inform the enclosing instance (if any) of the URI's for this instance
-
-if parent_uri=$(flux getattr parent-uri 2>/dev/null) \
-	&& parent_ns=$(flux getattr parent-kvs-namespace 2>/dev/null); then
-    key_prefix=flux
-    local_uri=${FLUX_URI}
-    remote_uri="ssh://$(hostname)/$(echo $local_uri|sed 's,^.*://,,')"
-
-    FLUX_URI=${parent_uri} FLUX_KVS_NAMESPACE=${parent_ns} \
-        flux kvs put ${key_prefix}.local_uri=${local_uri}
-    FLUX_URI=${parent_uri} FLUX_KVS_NAMESPACE=${parent_ns} \
-        flux kvs put ${key_prefix}.remote_uri=${remote_uri}
-fi
diff --git a/etc/rc1.d/02-cron b/etc/rc1.d/02-cron
new file mode 100755
index 000000000000..2539c6893c17
--- /dev/null
+++ b/etc/rc1.d/02-cron
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Load crontabs from directory if cron.directory attribute is set
+
+if test $(flux getattr rank) -eq 0 \
+	&& cron_dir=$(flux getattr cron.directory 2>/dev/null) \
+	&& test -d "$cron_dir"; then
+	shopt -s nullglob
+	for file in $cron_dir/*; do
+		if test -f $file; then
+			if ! flux cron tab <$file; then
+				echo "could not load crontab: $file" >&2
+				exit 1
+			fi
+		fi
+	done
+	shopt -u nullglob
+fi
diff --git a/etc/rc3 b/etc/rc3
index c2ec001ede90..e4d68ad0df0d 100755
--- a/etc/rc3
+++ b/etc/rc3
@@ -1,36 +1,80 @@
-#!/bin/bash
+#!/bin/sh
 
-flux queue stop --quiet
-flux job cancelall --user=all --quiet -f --states RUN
-flux queue idle --quiet
+RANK=$(flux getattr rank)
+exit_rc=0
+
+# Usage: modrm {all|<rank>} modname
+modrm() {
+    local where=$1; shift
+    if test "$where" = "all" || test $where -eq $RANK; then
+        flux module remove -f $* || exit_rc=1
+    fi
+}
+
+backing_module() {
+    local backingmod=$(flux getattr content.backing-module 2>/dev/null) || :
+    echo ${backingmod:-content-sqlite}
+}
 
 core_dir=$(cd ${0%/*} && pwd -P)
 all_dirs=$core_dir${FLUX_RC_EXTRA:+":$FLUX_RC_EXTRA"}
 IFS=:
-shopt -s nullglob
 for rcdir in $all_dirs; do
     for rcfile in $rcdir/rc3.d/*; do
+	[ -e $rcfile ] || continue
         echo running $rcfile
-        $rcfile
+        $rcfile || exit_rc=1
     done
 done
-shopt -u nullglob
 
-flux module remove -f sched-simple
-flux module remove -f job-exec
-flux module remove -f job-manager
-flux exec -r all flux module remove -f job-ingest
+modrm 0 heartbeat
+modrm 0 sched-simple
+modrm all resource
+modrm 0 job-exec
+modrm 0 job-list
+modrm all job-info
+modrm 0 job-manager
+modrm all job-ingest
+
+modrm 0 cron
+modrm all sdexec
+modrm all sdbus
+modrm all barrier
+
+if test $RANK -eq 0; then
+    if test "$(backing_module)" != "none"; then
+        flux startlog --post-finish-event || exit_rc=1
+    fi
+fi
 
-flux module remove -f userdb
+modrm all kvs-watch
+modrm all kvs
 
-flux module remove -f cron
-flux exec -r all flux module remove -f aggregator
-flux exec -r all flux module remove -f barrier
+if test "$(backing_module)" != "none"; then
+    flux content flush || exit_rc=1
+fi
 
-flux exec -r all flux module remove -f job-info
-flux exec -r all flux module remove -f kvs-watch
-flux exec -r all -x 0 flux module remove -f kvs
+if test $RANK -eq 0; then
+    backingmod=$(backing_module)
+    dumpfile=$(flux getattr content.dump 2>/dev/null)
+    if test $exit_rc -eq 0 -a -n "${dumpfile}"; then
+        if test "${dumpfile}" = "auto"; then
+            statedir=$(flux getattr statedir 2>/dev/null)
+            mkdir -p "${statedir:-.}/dump"
+            dumpfile="${statedir:-.}/dump/$(date +%Y%m%d_%H%M%S).tgz"
+            dumplink="${statedir:-.}/dump/RESTORE"
+        fi
+        echo "dumping content to ${dumpfile}"
+        if flux dump --quiet --ignore-failed-read --checkpoint ${dumpfile}; then
+            test -n "$dumplink" && ln -s $(basename ${dumpfile}) ${dumplink}
+        else
+            exit_rc=1
+        fi
+    fi
+    if test "${backingmod}" != "none"; then
+        flux module remove ${backingmod} || exit_rc=1
+    fi
+fi
+modrm all content
 
-flux module remove -f kvs
-flux content flush
-flux module remove -f content-sqlite
+exit $exit_rc
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000000..a07f288fc976
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,54 @@
+[tool.black]
+target-version = ['py36']
+force-exclude = "/^(env/|src/bindings/python/flux/utils)/"
+
+[tool.isort]
+profile = "black" # needed for black/isort compatibility
+skip = [
+  "src/bindings/python/flux/job/kvs.py",
+  "src/bindings/python/flux/job/__init__.py",
+  "src/bindings/python/flux/resource/__init__.py"]
+
+[tool.mypy]
+python_version = "3.6"
+files = ["src/cmd/**/*.py", "src/bindings/python/flux", "t/python/*.py"]
+mypy_path = ["src/bindings/python", "t/python/tap", "t/python"]
+allow_redefinition = true
+exclude = ["src/bindings/python/flux/utils/parsedatetime", "env/", "src/bindings/python/flux/utils/tomli"]
+
+# Having the cache makes spurious errors about looking up handle
+# It's slightly slower, but more correct to not have it
+cache_dir="/dev/null"
+namespace_packages = true
+ignore_errors = true
+ignore_missing_imports = true
+
+  [[tool.mypy.overrides]]
+  module = "subflux"
+  ignore_errors = true
+  ignore_missing_imports = true
+  follow_imports = 'skip'
+
+  [[tool.mypy.overrides]]
+  module = [
+    "_flux._core",
+    "_flux._security",
+    "_flux._hostlist",
+    "_flux._idset",
+    "_flux._rlist",
+    "_flux.cffi",
+    "setuptools",
+    "flux.utils",
+    "flux.utils.parsedatetime",
+    "flux.utils.tomli",
+    "tomllib",
+    "pycotap",
+
+    # These are temporary while we find a way to generate stubs for flux.constants
+    "flux.job.list",
+    "flux.core.handle",
+    "python.t1000-service-add-remove"
+  ]
+  ignore_errors = true
+  ignore_missing_imports = true
+  follow_imports = 'skip'
diff --git a/src/bindings/python/.pylintrc b/scripts/.pylintrc
similarity index 96%
rename from src/bindings/python/.pylintrc
rename to scripts/.pylintrc
index 90bd8a3af015..d8c14207fe63 100644
--- a/src/bindings/python/.pylintrc
+++ b/scripts/.pylintrc
@@ -44,7 +44,7 @@ symbols=no
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=missing-docstring, too-few-public-methods, too-many-arguments, star-args, arguments-differ, no-member, import-error, no-name-in-module, useless-object-inheritance, fixme, bad-continuation
+disable=missing-docstring, too-few-public-methods, too-many-arguments, star-args, arguments-differ, no-member, import-error, no-name-in-module, useless-object-inheritance, fixme, bad-continuation, too-many-lines, duplicate-code
 
 
 [REPORTS]
@@ -91,7 +91,7 @@ logging-modules=logging
 bad-functions=map,filter,apply,input,file
 
 # Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_
+good-names=i,j,k,x,y,z,ex,io,rc,Run,_,ts,t0
 
 # Bad variable names which should always be refused, separated by a comma
 bad-names=foo,bar,baz,toto,tutu,tata
@@ -110,10 +110,10 @@ function-rgx=[a-z_][a-z0-9_]{2,30}$
 function-name-hint=[a-z_][a-z0-9_]{2,30}$
 
 # Regular expression matching correct variable names
-variable-rgx=[a-z_][a-z0-9_]{2,30}$
+variable-rgx=[a-zA-Z_][a-zA-Z0-9_]{2,30}$
 
 # Naming hint for variable names
-variable-name-hint=[a-z_][a-z0-9_]{2,30}$
+variable-name-hint=[a-zA-Z_][a-zA-Z0-9_]{2,30}$
 
 # Regular expression matching correct constant names
 const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
@@ -122,16 +122,16 @@ const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
 const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
 
 # Regular expression matching correct attribute names
-attr-rgx=[a-z_][a-z0-9_]{2,30}$
+attr-rgx=[a-zA-Z_][a-zA-Z0-9_]{2,30}$
 
 # Naming hint for attribute names
-attr-name-hint=[a-z_][a-z0-9_]{2,30}$
+attr-name-hint=[a-zA-Z_][a-zA-Z0-9_]{2,30}$
 
 # Regular expression matching correct argument names
-argument-rgx=[a-z_][a-z0-9_]{2,30}$
+argument-rgx=[a-zA-Z_][a-zA-Z0-9_]{2,30}$
 
 # Naming hint for argument names
-argument-name-hint=[a-z_][a-z0-9_]{2,30}$
+argument-name-hint=[a-zA-Z_][a-zA-Z0-9_]{2,30}$
 
 # Regular expression matching correct class attribute names
 class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
diff --git a/scripts/check-format b/scripts/check-format
index 125e198c652b..59b1ea68b5f6 100755
--- a/scripts/check-format
+++ b/scripts/check-format
@@ -14,9 +14,11 @@ if command -v black 2>&1 > /dev/null ; then
     if [[ $# -eq 0 ]]; then
         # awk cmd copied from:
         # https://unix.stackexchange.com/questions/66097/find-all-files-with-a-python-shebang
-        find src t -path "src/bindings/python/_flux" -prune -o \
-             -type f \( -name "*.py" -print -o \
-             -exec awk ' /^#!.*python/{print FILENAME} {nextfile}' {} + \) \
+        find src t \
+	    -path "src/bindings/python/_flux" -prune -o \
+	    -path "src/bindings/python/flux/utils" -prune -o \
+            -type f \( -name "*.py" -print -o \
+            -exec awk ' /^#!.*python/{print FILENAME} {nextfile}' {} + \) \
             | xargs black --check --diff
     else
         filter_pyfiles "$@" | xargs black --check --diff
diff --git a/scripts/configure-macos.sh b/scripts/configure-macos.sh
new file mode 100755
index 000000000000..3fde1e3ea0ce
--- /dev/null
+++ b/scripts/configure-macos.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+die() {
+    echo "$(basename $0): $@" >&2
+    exit 1
+}
+
+DEPS_SCRIPT=scripts/install-deps-macos.sh
+
+test -f $DEPS_SCRIPT || die "please run from the top level of the source tree"
+test -d macos-venv || die "please run $DEPS_SCRIPT first"
+
+eval "$(/opt/homebrew/bin/brew shellenv)"
+
+CPPFLAGS=-I${HOMEBREW_PREFIX}/include/lua
+CPPFLAGS="-I$(brew --prefix epoll-shim)/include/libepoll-shim ${CPPFLAGS}"
+LDFLAGS=-L${HOMEBREW_PREFIX}/lib
+
+PKG_CONFIG_PATH=$(pkg-config --variable pc_path pkg-config)
+PKG_CONFIG_PATH=$(brew --prefix libarchive)/lib/pkgconfig:${PKG_CONFIG_PATH}
+
+PATH=$(brew --prefix libtool)/libexec/gnubin:$PATH
+
+source macos-venv/bin/activate
+
+./autogen.sh
+
+CPPFLAGS=$CPPFLAGS LDFLAGS=$LDFLAGS PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
+  ./configure
diff --git a/scripts/debbuild.sh b/scripts/debbuild.sh
new file mode 100755
index 000000000000..9a21ea542e7c
--- /dev/null
+++ b/scripts/debbuild.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+PACKAGE=flux-core
+USER=$(git config --get user.name)
+DEBFULLNAME=$USER
+EMAIL=$(git config --get user.email)
+DEBEMAIL=$EMAIL
+
+SRCDIR=${1:-$(pwd)}
+
+die() { echo "debbuild: $@" >&2; exit 1; }
+log() { echo "debbuild: $@"; }
+
+test -z "$USER" && die "User name not set in git-config"
+test -z "$EMAIL" && die "User email not set in git-config"
+
+log "Running make dist"
+make dist >/dev/null || exit 1
+
+log "Building package from latest dist tarball"
+tarball=$(ls -tr *.tar.gz | tail -1)
+version=$(echo $tarball | sed "s/${PACKAGE}-\(.*\)\.tar\.gz/\1/")
+
+rm -rf debbuild
+mkdir -p debbuild && cd debbuild
+
+mv ../$tarball .
+
+log "Unpacking $tarball"
+tar xvfz $tarball >/dev/null
+
+log "Creating debian directory and files"
+cd ${PACKAGE}-${version}
+cp -a ${SRCDIR}/debian . || die "failed to copy debian dir"
+
+export DEBEMAIL DEBFULLNAME
+log "Creating debian/changelog"
+dch --create --package=$PACKAGE --newversion $version build tree release
+
+log "Running debian-buildpackage -b"
+dpkg-buildpackage -b
+log "Check debbuild directory for results"
diff --git a/scripts/fetch-and-build-catch.sh b/scripts/fetch-and-build-catch.sh
new file mode 100755
index 000000000000..f6399d5ff94e
--- /dev/null
+++ b/scripts/fetch-and-build-catch.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mkdir catch2
+pushd catch2
+wget -O - https://github.com/catchorg/Catch2/archive/refs/tags/v3.6.0.tar.gz | tar xvz --strip-components 1
+cmake -B build -DCMAKE_INSTALL_PREFIX=/usr
+cmake --build build -j 4 -t install
+popd
+rm -rf catch2
diff --git a/scripts/fetch-and-build-mpich.sh b/scripts/fetch-and-build-mpich.sh
new file mode 100755
index 000000000000..0f7ff00c1aa0
--- /dev/null
+++ b/scripts/fetch-and-build-mpich.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mkdir catch2
+pushd catch2
+wget -O - https://www.mpich.org/static/downloads/4.2.2/mpich-4.2.2.tar.gz | tar xvz --strip-components 1
+mkdir -p build
+pushd build
+../configure --prefix=/usr --without-pmix
+make -j 4 install
+popd
+popd
+rm -rf catch2
+
diff --git a/scripts/format b/scripts/format
index e8111de28df9..d9b30836cc74 100755
--- a/scripts/format
+++ b/scripts/format
@@ -3,9 +3,11 @@
 if command -v black 2>&1 > /dev/null ; then
     # awk cmd copied from:
     # https://unix.stackexchange.com/questions/66097/find-all-files-with-a-python-shebang
-    find src t -path "src/bindings/python/_flux" -prune -o \
-         -type f \( -name "*.py" -print -o \
-         -exec awk ' /^#!.*python/{print FILENAME} {nextfile}' {} + \) \
+    find src t \
+        -path "src/bindings/python/_flux" -prune -o \
+        -path "src/bindings/python/flux/utils" -prune -o \
+        -type f \( -name "*.py" -print -o \
+        -exec awk ' /^#!.*python/{print FILENAME} {nextfile}' {} + \) \
         | xargs black
 else
   echo "black not found, python left unformatted"
diff --git a/scripts/generate_compile_commands b/scripts/generate_compile_commands
new file mode 100755
index 000000000000..fcadc60dd8d1
--- /dev/null
+++ b/scripts/generate_compile_commands
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+: ${OUTPUT:=$(pwd)/compile_commands.json}
+: ${APPEND:=--append}
+: ${TARGET:=all}
+while test $# -gt 0 ; do
+    case $1 in
+    -f)
+    # overwrite existing, do not append
+    APPEND=''
+    shift
+    ;;
+    -t)
+    # include tests, this means running all of check, prepare to wait
+    TARGET=check
+    shift
+    ;;
+    *)
+    break
+    ;;
+    esac
+done
+
+bear -o "${OUTPUT}" $APPEND make "$TARGET" "$@"
\ No newline at end of file
diff --git a/scripts/install-deps-deb.sh b/scripts/install-deps-deb.sh
new file mode 100755
index 000000000000..63c13fd4e9ad
--- /dev/null
+++ b/scripts/install-deps-deb.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+apt install \
+  autoconf \
+  automake \
+  libtool \
+  make \
+  pkg-config \
+  libc6-dev \
+  libzmq3-dev \
+  uuid-dev \
+  libjansson-dev \
+  liblz4-dev \
+  libarchive-dev \
+  libhwloc-dev \
+  libsqlite3-dev \
+  lua5.1 \
+  liblua5.1-dev \
+  lua-posix \
+  python3-dev \
+  python3-cffi \
+  python3-ply \
+  python3-setuptools \
+  python3-yaml \
+  python3-sphinx \
+  aspell \
+  aspell-en \
+  time \
+  valgrind \
+  libmpich-dev \
+  jq
+
diff --git a/scripts/install-deps-macos.sh b/scripts/install-deps-macos.sh
new file mode 100755
index 000000000000..b7b4262d3562
--- /dev/null
+++ b/scripts/install-deps-macos.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+die() {
+    echo "$(basename $0): $@" >&2
+    exit 1
+}
+
+test -f scripts/requirements-dev.txt || die "Please run from top of source tree"
+
+eval "$(/opt/homebrew/bin/brew shellenv)"
+
+brew install \
+  autoconf \
+  automake \
+  libtool \
+  make \
+  pkg-config \
+  epoll-shim \
+  zeromq \
+  jansson \
+  lz4 \
+  libarchive \
+  hwloc \
+  sqlite \
+  lua@5.3 \
+  python3 \
+  cffi \
+  libyaml \
+  jq
+
+brew link lua@5.3
+
+python3 -m venv macos-venv
+source macos-venv/bin/activate
+
+pip3 install setuptools
+pip3 install -r scripts/requirements-dev.txt
+
+echo "Now run scripts/configure-macos.sh"
diff --git a/scripts/install-deps-rpm.sh b/scripts/install-deps-rpm.sh
new file mode 100755
index 000000000000..2268f2e674fe
--- /dev/null
+++ b/scripts/install-deps-rpm.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+yum install \
+  autoconf \
+  automake \
+  libtool \
+  make \
+  pkgconfig \
+  glibc-devel \
+  zeromq4-devel \
+  libuuid-devel \
+  jansson-devel \
+  lz4-devel \
+  libarchive-devel \
+  hwloc-devel \
+  sqlite-devel \
+  lua \
+  lua-devel \
+  lua-posix \
+  python36-devel \
+  python36-cffi \
+  python36-ply \
+  python36-yaml \
+  python3-sphinx \
+  python3-setuptools \
+  aspell \
+  aspell-en \
+  time \
+  valgrind-devel \
+  mpich-devel \
+  jq
diff --git a/scripts/pylint b/scripts/pylint
new file mode 100755
index 000000000000..a431a0dddbf2
--- /dev/null
+++ b/scripts/pylint
@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+pylint=$(command -v pylint3 pylint | head -1)
+if test -n "$pylint"; then
+    $pylint --rcfile=scripts/.pylintrc src/bindings/python/flux
+
+    # awk cmd copied from:
+    # https://unix.stackexchange.com/questions/66097/find-all-files-with-a-python-shebang
+    find src/cmd -type f \( -name "*.py" -print -o \
+         -exec awk ' /^#!.*python/{print FILENAME} {nextfile}' {} + \) \
+        | xargs $pylint --rcfile=scripts/.pylintrc --disable=missing-docstring,no-member --module-rgx='[a-z-]+'
+else
+  echo "pylint not found, python left unlinted"
+fi
diff --git a/scripts/requirements-ci.txt b/scripts/requirements-ci.txt
new file mode 100644
index 000000000000..c868fc0a2c6c
--- /dev/null
+++ b/scripts/requirements-ci.txt
@@ -0,0 +1,9 @@
+markupsafe==2.0.0
+coverage
+cffi
+ply
+six
+pyyaml
+jsonschema>=2.6,<4.0
+sphinxcontrib-spelling
+-r requirements-doc.txt
diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt
new file mode 100644
index 000000000000..bd3800c44a1a
--- /dev/null
+++ b/scripts/requirements-dev.txt
@@ -0,0 +1,9 @@
+cffi>=1.1
+ply>=3.9
+pyyaml>=3.10.0
+black==24.3.0
+flake8>=5.0.4
+isort>=5.9.3
+mypy==1.9.0
+pre-commit>=2.9.2
+types-PyYAML
diff --git a/scripts/requirements-doc.txt b/scripts/requirements-doc.txt
new file mode 100644
index 000000000000..e91a94e049fc
--- /dev/null
+++ b/scripts/requirements-doc.txt
@@ -0,0 +1,5 @@
+sphinx<6.0.0
+sphinx-rtd-theme>=0.5.2
+docutils>=0.14,<0.18
+urllib3<2
+jinja2<3.1.0
diff --git a/scripts/run_mypy.sh b/scripts/run_mypy.sh
new file mode 100755
index 000000000000..ae578d7d3ea3
--- /dev/null
+++ b/scripts/run_mypy.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+# This seems to be needed to use our own version of mypy?
+mypy
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 000000000000..9696b37b4d4c
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,25 @@
+[flake8]
+exclude = benchmarks docs
+max-line-length = 100
+ignore = E1 E2 E5 W5
+per-file-ignores =
+    src/bindings/python/flux/resource/__init__.py:F401
+    src/bindings/python/flux/job/frobnicator/__init__.py:F401
+    src/bindings/python/flux/job/__init__.py:F401
+    src/bindings/python/flux/job/validator/__init__.py:F401
+    src/bindings/python/flux/uri/__init__.py:F401
+    src/bindings/python/flux/utils/parsedatetime/pdt_locales/*:F405
+    src/bindings/python/flux/job/frobnicator/frobnicator.py:E722
+    src/bindings/python/flux/job/validator/validator.py:E722
+    src/bindings/python/flux/util.py:E722
+    src/bindings/python/flux/uri/resolvers/jobid.py:F841
+    src/bindings/python/flux/job/info.py:E713
+    src/bindings/python/flux/util.py:E713
+    src/bindings/python/flux/uri/resolvers/lsf.py:F541
+    src/bindings/python/flux/__init__.py:E302
+    src/bindings/python/flux/resource/Rlist.py:F541
+    src/bindings/python/flux/uri/resolvers/pid.py:F841
+    src/bindings/python/flux/uri/resolvers/pid.py:E713
+    src/bindings/python/flux/job/validator/plugins/jobspec.py:F541
+    src/bindings/python/flux/util.py:E713
+    src/bindings/python/flux/uri/resolvers/pid.py:E713
diff --git a/src/Makefile.am b/src/Makefile.am
index 88883e5676e9..7ebc171ee2a1 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -11,4 +11,7 @@ noinst_HEADERS = \
 	include/flux/optparse.h \
 	include/flux/idset.h \
 	include/flux/schedutil.h \
-	include/flux/shell.h
+	include/flux/shell.h \
+	include/flux/hostlist.h \
+	include/flux/jobtap.h \
+	include/flux/taskmap.h
diff --git a/src/bindings/lua/Makefile.am b/src/bindings/lua/Makefile.am
index e3a106356fe7..8e3cc40429d3 100644
--- a/src/bindings/lua/Makefile.am
+++ b/src/bindings/lua/Makefile.am
@@ -1,11 +1,13 @@
 AM_CFLAGS =	$(WARNING_CFLAGS) $(CODE_COVERAGE_CFLAGS) \
-		$(ZMQ_CFLAGS) $(LIBUUID_CFLAGS) \
 		-Wno-parentheses -Wno-error=parentheses
 AM_LDFLAGS =	$(CODE_COVERAGE_LIBS)
 AM_CPPFLAGS =	\
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
+	$(JANSSON_CFLAGS) \
 	$(LUA_INCLUDE)
 
 fluxluadir =     $(luadir)/flux
@@ -16,6 +18,7 @@ nobase_dist_lua_SCRIPTS = \
 	flux/timer.lua \
 	flux/alt_getopt.lua \
 	flux/posix.lua \
+	flux/shell.lua \
 	flux/Subcommander.lua
 
 nobase_dist_fluxometer_SCRIPTS = \
@@ -39,12 +42,13 @@ noinst_LTLIBRARIES = \
 
 luamod_ldflags = \
 	-avoid-version -module -shared --disable-static \
+	$(ld_gc_sections) \
 	$(san_ld_zdef_flag)
 
 luamod_libadd = \
-	$(top_builddir)/src/common/libflux-internal.la \
 	$(top_builddir)/src/common/libflux-core.la \
-	$(LUA_LIB) $(ZMQ_LIBS) $(LIBUUID_LIBS)
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(LUA_LIB)
 
 flux_la_LDFLAGS = \
 	$(luamod_ldflags)
diff --git a/src/bindings/lua/Test/Builder/Tester.lua b/src/bindings/lua/Test/Builder/Tester.lua
index 862451e5ee28..511caa6ce0c8 100644
--- a/src/bindings/lua/Test/Builder/Tester.lua
+++ b/src/bindings/lua/Test/Builder/Tester.lua
@@ -41,7 +41,7 @@ local function _start_testing ()
     out:reset()
     err:reset()
 
-    -- remeber that we're testing
+    -- remember that we're testing
     testing = true
     testing_num = tb:current_test()
     tb:current_test(0)
diff --git a/src/bindings/lua/Test/More.lua b/src/bindings/lua/Test/More.lua
index 89da8946f9bb..19c750720fb2 100644
--- a/src/bindings/lua/Test/More.lua
+++ b/src/bindings/lua/Test/More.lua
@@ -1,4 +1,4 @@
-
+--- @diagnostic disable
 --
 -- lua-TestMore : <http://fperrad.github.com/lua-TestMore/>
 --
@@ -15,7 +15,7 @@ local _G = _G
 
 local tb = require 'Test.Builder'.new()
 
-_ENV = nil
+-- _ENV = nil
 local m = {}
 
 function m.plan (arg)
@@ -373,9 +373,38 @@ function m.cleanup (func)
     tb:cleanup(func)
 end
 
-for k, v in pairs(m) do  -- injection
-    _G[k] = v
-end
+-- replaced by explicit below to fix test warnings
+-- for k, v in pairs(m) do  -- injection
+--     _G[k] = v
+-- end
+plan = m.plan
+done_testing = m.done_testing
+skip_all = m.skip_all
+BAIL_OUT = m.BAIL_OUT
+ok = m.ok
+nok = m.nok
+is = m.is
+isnt = m.isnt
+like = m.like
+unlike = m.unlike
+cmp_ok = m.cmp_ok
+type_ok = m.type_ok
+subtest = m.subtest
+pass = m.pass
+fail = m.fail
+require_ok = m.require_ok
+eq_array = m.eq_array
+is_deeply = m.is_deeply
+error_is = m.error_is
+error_like = m.error_like
+lives_ok = m.lives_ok
+diag = m.diag
+note = m.note
+skip = m.skip
+todo_skip = m.todo_skip
+skip_rest = m.skip_rest
+todo = m.todo
+cleanup = m.cleanup
 
 m._VERSION = "0.3.1"
 m._DESCRIPTION = "lua-TestMore : an Unit Testing Framework"
diff --git a/src/bindings/lua/flux-lua.c b/src/bindings/lua/flux-lua.c
index 39a70c799c64..237c891f41b9 100644
--- a/src/bindings/lua/flux-lua.c
+++ b/src/bindings/lua/flux-lua.c
@@ -25,6 +25,8 @@
 
 #include "flux/core.h"
 
+#include "ccan/str/str.h"
+
 #include "jansson-lua.h"
 #include "zmsg-lua.h"
 #include "lutil.h"
@@ -152,7 +154,7 @@ static int lua_push_flux_handle (lua_State *L, flux_t *f)
     *fp = f;
 
     /*
-     *  2. Set metatable for Lua "flux" object so it inherets the right
+     *  2. Set metatable for Lua "flux" object so it inherits the right
      *     methods:
      */
     luaL_getmetatable (L, "FLUX.handle");
@@ -213,7 +215,7 @@ static int l_flux_new (lua_State *L)
         s = lua_tostring (L, 1);
     f = flux_open (s, 0);
     if (f == NULL)
-        return lua_pusherror (L, (char *)flux_strerror (errno));
+        return lua_pusherror (L, "%s", (char *)flux_strerror (errno));
     return (lua_push_flux_handle (L, f));
 }
 
@@ -235,18 +237,6 @@ static int l_flux_size (lua_State *L)
     return (l_pushresult (L, size));
 }
 
-static int l_flux_arity (lua_State *L)
-{
-    flux_t *f = lua_get_flux (L, 1);
-    const char *s;
-    int arity;
-
-    if (!(s = flux_attr_get (f, "tbon.arity")))
-        return lua_pusherror (L, "flux_attr_get tbon.arity error");
-    arity = strtoul (s, NULL, 10);
-    return (l_pushresult (L, arity));
-}
-
 static int l_flux_index (lua_State *L)
 {
     const char *key = lua_tostring (L, 2);
@@ -254,12 +244,10 @@ static int l_flux_index (lua_State *L)
     if (key == NULL)
         return luaL_error (L, "flux: invalid index");
 
-    if (strcmp (key, "size") == 0)
+    if (streq (key, "size"))
         return l_flux_size (L);
-    if (strcmp (key, "rank") == 0)
+    if (streq (key, "rank"))
         return l_flux_rank (L);
-    if (strcmp (key, "arity") == 0)
-        return l_flux_arity (L);
 
     lua_getmetatable (L, 1);
     lua_getfield (L, -1, key);
@@ -277,10 +265,7 @@ static int send_json_request (flux_t *h, uint32_t nodeid, uint32_t matchtag,
     if (flux_msg_set_matchtag (msg, matchtag) < 0)
         goto done;
     if (nodeid == FLUX_NODEID_UPSTREAM) {
-        uint8_t flags;
-        if (flux_msg_get_flags (msg, &flags) < 0)
-            goto done;
-        if (flux_msg_set_flags (msg, flags | FLUX_MSGFLAG_UPSTREAM) < 0)
+        if (flux_msg_set_flag (msg, FLUX_MSGFLAG_UPSTREAM) < 0)
             goto done;
         if (flux_get_rank (h, &nodeid) < 0)
             goto done;
@@ -316,12 +301,12 @@ static int l_flux_send (lua_State *L)
 
     matchtag = flux_matchtag_alloc (f);
     if (matchtag == FLUX_MATCHTAG_NONE)
-        return lua_pusherror (L, (char *)flux_strerror (errno));
+        return lua_pusherror (L, "%s", (char *)flux_strerror (errno));
 
     rc = send_json_request (f, nodeid, matchtag, tag, json_str);
     free (json_str);
     if (rc < 0)
-        return lua_pusherror (L, (char *)flux_strerror (errno));
+        return lua_pusherror (L, "%s", (char *)flux_strerror (errno));
 
     return l_pushresult (L, matchtag);
 }
@@ -377,7 +362,7 @@ static int l_flux_recv (lua_State *L)
         flux_msg_destroy (msg);
         errno = saved_errno;
     }
-    return lua_pusherror (L, (char *)flux_strerror (errno));
+    return lua_pusherror (L, "%s", (char *)flux_strerror (errno));
 }
 
 static int l_flux_rpc (lua_State *L)
@@ -410,13 +395,13 @@ static int l_flux_rpc (lua_State *L)
      */
     if (json_str[0] != '{' || json_str[strlen (json_str) - 1] != '}') {
         errno = EINVAL;
-        rc = lua_pusherror (L, (char *)flux_strerror (errno));
+        rc = lua_pusherror (L, "%s", (char *)flux_strerror (errno));
         goto done;
     }
     fut = flux_rpc (f, tag, json_str, nodeid, 0);
     free (json_str);
     if (!fut || flux_rpc_get (fut, &s) < 0) {
-        rc = lua_pusherror (L, (char *)flux_strerror (errno));
+        rc = lua_pusherror (L, "%s", (char *)flux_strerror (errno));
         goto done;
     }
     if (json_object_string_to_lua (L, s ? : "{}") < 0) {
@@ -435,7 +420,7 @@ static int l_flux_getattr (lua_State *L)
     const char *name = luaL_checkstring (L, 2);
     const char *val = flux_attr_get (f, name);
     if (val == NULL)
-        return lua_pusherror (L, (char *)flux_strerror (errno));
+        return lua_pusherror (L, "%s", (char *)flux_strerror (errno));
     lua_pushstring (L, val);
     return (1);
 }
@@ -505,11 +490,11 @@ static int l_flux_recv_event (lua_State *L)
     flux_msg_t *msg = NULL;
 
     if (!(msg = flux_recv (f, match, 0)))
-        return lua_pusherror (L, (char *)flux_strerror (errno));
+        return lua_pusherror (L, "%s", (char *)flux_strerror (errno));
 
     if (flux_event_decode (msg, &topic, &json_str) < 0) {
         flux_msg_destroy (msg);
-        return lua_pusherror (L, (char *)flux_strerror (errno));
+        return lua_pusherror (L, "%s", (char *)flux_strerror (errno));
     }
 
     /* FIXME: create empty JSON object if message payload was empty,
@@ -668,7 +653,7 @@ static int l_flux_recvmsg (lua_State *L)
         match.matchtag = lua_tointeger (L, 2);
 
     if (!(msg = flux_recv (f, match, 0)))
-        return lua_pusherror (L, (char *)flux_strerror (errno));
+        return lua_pusherror (L, "%s", (char *)flux_strerror (errno));
 
     if (flux_msg_get_type (msg, &type) < 0)
         type = FLUX_MSGTYPE_ANY;
@@ -747,7 +732,7 @@ static int l_msghandler_remove (lua_State *L)
 
 static int l_msghandler_add (lua_State *L)
 {
-    struct flux_match match;
+    struct flux_match match = FLUX_MATCH_ANY;
     struct l_flux_ref *mh = NULL;
     flux_msg_handler_t *fmh = NULL;
     flux_t *f = lua_get_flux (L, 1);
@@ -800,7 +785,7 @@ static int l_msghandler_index (lua_State *L)
     /*
      *  Check for method names
      */
-    if (strcmp (key, "remove") == 0) {
+    if (streq (key, "remove")) {
         lua_getmetatable (L, 1);
         lua_getfield (L, -1, "remove");
         return (1);
@@ -951,7 +936,7 @@ static int l_watcher_index (lua_State *L)
     /*
      *  Check for method names
      */
-    if (strcmp (key, "remove") == 0) {
+    if (streq (key, "remove")) {
         lua_getmetatable (L, 1);
         lua_getfield (L, -1, "remove");
         return (1);
@@ -1060,9 +1045,9 @@ static int l_flux_reactor_start (lua_State *L)
     flux_t *h;
     int mode = 0;
     if ((lua_gettop (L) > 1) && (arg = lua_tostring (L, 2))) {
-        if (strcmp (arg, "once") == 0)
+        if (streq (arg, "once"))
             mode = FLUX_REACTOR_ONCE;
-        else if (strcmp (arg, "nowait") == 0)
+        else if (streq (arg, "nowait"))
             mode = FLUX_REACTOR_NOWAIT;
         else
             return lua_pusherror (L, "flux_reactor: Invalid argument");
diff --git a/src/bindings/lua/flux/Subcommander.lua b/src/bindings/lua/flux/Subcommander.lua
index b93174966ae5..61fa69db9747 100644
--- a/src/bindings/lua/flux/Subcommander.lua
+++ b/src/bindings/lua/flux/Subcommander.lua
@@ -54,7 +54,7 @@ function Command:fullname ()
 end
 
 --
--- Split line in `s` at `columns` colums assuming left pad `pad`
+-- Split line in `s` at `columns` columns assuming left pad `pad`
 --
 local function linesplit (s, columns, pad)
     local width = columns - pad
@@ -170,7 +170,7 @@ local DefaultHelp = {
         end
         local cmd = self.parent:lookup (arg[1])
         if not cmd then
-            self:die ("Unkown command %s", arg[1])
+            self:die ("Unknown command %s", arg[1])
         end
         cmd:help (0)
     end
diff --git a/src/bindings/lua/flux/shell.lua b/src/bindings/lua/flux/shell.lua
new file mode 100644
index 000000000000..d04f04e72c28
--- /dev/null
+++ b/src/bindings/lua/flux/shell.lua
@@ -0,0 +1,105 @@
+-------------------------------------------------------------
+-- Copyright 2021 Lawrence Livermore National Security, LLC
+-- (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+--
+-- This file is part of the Flux resource manager framework.
+-- For details, see https://github.com/flux-framework.
+--
+-- SPDX-License-Identifier: LGPL-3.0
+-------------------------------------------------------------
+
+if shell == nil then
+    error ("This module can only be imported from the Flux job shell")
+end
+
+--  Useful functions shared by all subsequently loaded
+--   shell Lua plugins
+
+--- Get a top level option from the shell.options table which includes
+---  an optional "@version" specification.
+--
+--  @param name The name of the option to get
+--
+--  Returns val, version
+--   `val` will be nil if shell option was not set
+--   `version` will be nil if no version specified.
+--
+function shell.getopt_with_version (name)
+    if not name then return nil end
+    local opt = shell.options[name]
+    if opt then
+        return opt:match("^[^@]+"), opt:match("@(.+)$")
+    end
+    return nil
+end
+
+--- Prepend `path` to colon-separated path environment variable `env_var`
+--
+--  @param env_var The environment variable to which to prepend
+--  @param path    The path to prepend
+--
+function shell.prepend_path (env_var, path)
+    local val = shell.getenv (env_var)
+
+    -- If path is already in env_var, do nothing. We stick ":" on both
+    -- ends of the existing value so we can easily match exact paths
+    -- instead of possibly matching substrings of paths when trying
+    -- to match "zero or more" colons.
+    --
+    if val and ((":"..val..":"):match (":"..path..":")) then return end
+
+    if val == nil then
+       suffix = ''
+    else
+       suffix = ':'..val
+    end
+    shell.setenv (env_var, path..suffix)
+end
+
+--- Strip all environment variables from job environment that match one
+--   or more pattern arguments.
+--
+function shell.env_strip (...)
+    local env = shell.getenv ()
+    for k,v in pairs (env) do
+        for _,pattern in pairs ({...}) do
+            if k:match(pattern) then
+                shell.unsetenv (k)
+            end
+        end
+    end
+end
+
+--- Set rcpath to the path to the loaded initrc.lua (shell.rcpath) with
+--   "/lua.d" appended, plus `FLUX_SHELL_RC_PATH` if set in environment
+--   of the  job.
+--
+local rcpath = shell.rcpath .. "/lua.d"
+               .. ":"
+               .. (shell.getenv ("FLUX_SHELL_RC_PATH") or "")
+
+--- Source all files matching pattern from rcpath
+--
+function shell.source_rcpath (pattern)
+    for path in rcpath:gmatch ("[^:]+") do
+        source (path .. "/" .. pattern)
+    end
+end
+
+--- Source all files matching value[@version].lua from rcpath
+--   for option `opt`
+--
+function shell.source_rcpath_option (opt)
+    local name, version = shell.getopt_with_version (opt)
+    if name and name ~= "none" then
+        for path in rcpath:gmatch ("[^:]+") do
+            local basename = path .. "/" .. opt .. "/" .. name
+            source_if_exists (basename .. ".lua")
+            if version then
+                source_if_exists (basename .. "@" .. version .. ".lua")
+            end
+        end
+    end
+end
+-- vi: ts=4 sw=4 expandtab
+-- 
diff --git a/src/bindings/lua/jansson-lua.c b/src/bindings/lua/jansson-lua.c
index cc61644e8a09..30f7de4feef5 100644
--- a/src/bindings/lua/jansson-lua.c
+++ b/src/bindings/lua/jansson-lua.c
@@ -11,6 +11,7 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <stdint.h>
 #include <math.h>
 #include <errno.h>
 #include <lua.h>
@@ -34,11 +35,12 @@ int lua_is_json_null (lua_State *L, int index)
 
 int json_object_to_lua (lua_State *L, json_t *o)
 {
-        if (o == NULL) {
-            lua_pushnil (L);
-            return (1);
-        }
-        switch (json_typeof (o)) {
+    uint64_t val;
+    if (o == NULL) {
+        lua_pushnil (L);
+        return (1);
+    }
+    switch (json_typeof (o)) {
         case JSON_OBJECT:
             json_object_to_lua_table (L, o);
             break;
@@ -49,7 +51,12 @@ int json_object_to_lua (lua_State *L, json_t *o)
             lua_pushstring (L, json_string_value (o));
             break;
         case JSON_INTEGER:
-            lua_pushinteger (L, json_integer_value (o));
+            val = json_integer_value (o);
+#if LUA_VERSION_NUM < 530 && SIZEOF_PTRDIFF_T == 4
+            lua_pushnumber (L, (double) val);
+#else
+            lua_pushinteger (L, (lua_Integer) val);
+#endif
             break;
         case JSON_REAL:
             lua_pushnumber (L, json_real_value (o));
@@ -61,10 +68,10 @@ int json_object_to_lua (lua_State *L, json_t *o)
             lua_pushboolean (L, 0);
             break;
         case JSON_NULL:
-            /* XXX: crap. */
+            lua_pushnil (L);
             break;
-        }
-        return (1);
+    }
+    return (1);
 }
 
 int json_object_string_to_lua (lua_State *L, const char *json_str)
@@ -132,8 +139,13 @@ int lua_value_to_json (lua_State *L, int i, json_t **valp)
 
     switch (lua_type (L, index)) {
         case LUA_TNUMBER:
-            if (lua_is_integer (L, index))
+            if (lua_is_integer (L, index)) {
+#if LUA_VERSION_NUM < 530 && SIZEOF_PTRDIFF_T == 4
+                o = json_integer ((json_int_t) lua_tonumber (L, index));
+#else
                 o = json_integer (lua_tointeger (L, index));
+#endif
+            }
             else
                 o = json_real (lua_tonumber (L, index));
             break;
diff --git a/src/bindings/lua/lutil.c b/src/bindings/lua/lutil.c
index a40acab8492f..52c4b259cfb2 100644
--- a/src/bindings/lua/lutil.c
+++ b/src/bindings/lua/lutil.c
@@ -43,7 +43,7 @@ int lua_pusherror (lua_State *L, char *fmt, ...)
 int l_pushresult (lua_State *L, int rc)
 {
     if (rc < 0)
-        return lua_pusherror (L, strerror (errno));
+        return lua_pusherror (L, "%s", strerror (errno));
     lua_pushnumber (L, rc);
     return (1);
 }
diff --git a/src/bindings/lua/ping.lua b/src/bindings/lua/ping.lua
index 665be6ede4ad..4e7922b6f44b 100755
--- a/src/bindings/lua/ping.lua
+++ b/src/bindings/lua/ping.lua
@@ -11,7 +11,7 @@
 -------------------------------------------------------------
 
  --
- -- ping.lua: Example script implementing cmb 'ping' client using
+ -- ping.lua: Example script implementing broker 'ping' client using
  --  Lua bindings for Flux.
  --
 local sleep = require 'flux.posix'.sleep
diff --git a/src/bindings/lua/tests/zmsg-test.c b/src/bindings/lua/tests/zmsg-test.c
index 0174bb788669..d86e36e572cd 100644
--- a/src/bindings/lua/tests/zmsg-test.c
+++ b/src/bindings/lua/tests/zmsg-test.c
@@ -13,18 +13,18 @@
 #endif
 #include <lua.h>
 #include <lauxlib.h>
-#include <zmq.h>
-#include <czmq.h>
 #include <errno.h>
 #include <string.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
 #include "flux/core.h"
 #include "jansson-lua.h"
 #include "zmsg-lua.h"
 #include "lutil.h"
 
 
-flux_msg_t *l_cmb_zmsg_encode (lua_State *L)
+flux_msg_t *l_broker_zmsg_encode (lua_State *L)
 {
 	const char *tag = lua_tostring (L, 1);
     char *json_str = NULL;
@@ -54,11 +54,11 @@ static int l_zi_resp_cb (lua_State *L,
     return lua_push_zmsg_info (L, zmsg_info_create (msg, FLUX_MSGTYPE_RESPONSE));
 }
 
-static int l_cmb_zmsg_create_type (lua_State *L, int type)
+static int l_broker_zmsg_create_type (lua_State *L, int type)
 {
     struct zmsg_info *zi;
 	flux_msg_t **msg = malloc (sizeof (*msg));
-	if ((*msg = l_cmb_zmsg_encode (L)) == NULL) {
+	if ((*msg = l_broker_zmsg_encode (L)) == NULL) {
         free (msg);
         return luaL_error (L, "Failed to encode zmsg");
     }
@@ -68,15 +68,15 @@ static int l_cmb_zmsg_create_type (lua_State *L, int type)
 	return lua_push_zmsg_info (L, zi);
 }
 
-static int l_cmb_zmsg_create_response (lua_State *L)
+static int l_broker_zmsg_create_response (lua_State *L)
 {
-    return l_cmb_zmsg_create_type (L, FLUX_MSGTYPE_RESPONSE);
+    return l_broker_zmsg_create_type (L, FLUX_MSGTYPE_RESPONSE);
 }
 
 /*
  *  Send new-API-style response with errnum.
  */
-static int l_cmb_zmsg_create_response_with_error (lua_State *L)
+static int l_broker_zmsg_create_response_with_error (lua_State *L)
 {
     struct zmsg_info *zi;
     int errnum;
@@ -104,21 +104,21 @@ static int l_cmb_zmsg_create_response_with_error (lua_State *L)
     return lua_push_zmsg_info (L, zi);
 }
 
-static int l_cmb_zmsg_create_request (lua_State *L)
+static int l_broker_zmsg_create_request (lua_State *L)
 {
-    return l_cmb_zmsg_create_type (L, FLUX_MSGTYPE_REQUEST);
+    return l_broker_zmsg_create_type (L, FLUX_MSGTYPE_REQUEST);
 }
 
-static int l_cmb_zmsg_create_event (lua_State *L)
+static int l_broker_zmsg_create_event (lua_State *L)
 {
-    return l_cmb_zmsg_create_type (L, FLUX_MSGTYPE_EVENT);
+    return l_broker_zmsg_create_type (L, FLUX_MSGTYPE_EVENT);
 }
 
 static const struct luaL_Reg zmsg_info_test_functions [] = {
-	{ "req",       l_cmb_zmsg_create_request   },
-	{ "resp",      l_cmb_zmsg_create_response  },
-	{ "resp_err",  l_cmb_zmsg_create_response_with_error },
-	{ "event",     l_cmb_zmsg_create_event     },
+	{ "req",       l_broker_zmsg_create_request   },
+	{ "resp",      l_broker_zmsg_create_response  },
+	{ "resp_err",  l_broker_zmsg_create_response_with_error },
+	{ "event",     l_broker_zmsg_create_event     },
 	{ NULL,        NULL              }
 };
 
diff --git a/src/bindings/lua/zmsg-lua.c b/src/bindings/lua/zmsg-lua.c
index e73dc500a679..feaaad1ed6dd 100644
--- a/src/bindings/lua/zmsg-lua.c
+++ b/src/bindings/lua/zmsg-lua.c
@@ -13,8 +13,9 @@
 #endif
 #include <lua.h>
 #include <lauxlib.h>
-#include <zmq.h>
-#include <czmq.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
 
 #include "flux/core.h"
 #include "lutil.h"
@@ -22,7 +23,7 @@
 #include "zmsg-lua.h"
 
 /*
- *  cmb ZMSG lua binding
+ *  broker ZMSG lua binding
  */
 
 struct zmsg_info {
@@ -121,32 +122,32 @@ static int l_zmsg_info_index (lua_State *L)
     if (key == NULL)
         return lua_pusherror (L, "zmsg: invalid member");
 
-    if (strcmp (key, "type") == 0) {
+    if (streq (key, "type")) {
         lua_pushstring (L, zmsg_type_string (zi->typemask));
         return (1);
     }
-    if (strcmp (key, "tag") == 0) {
+    if (streq (key, "tag")) {
         if (zi->tag)
             lua_pushstring (L, zi->tag);
         else
             lua_pushnil (L);
         return (1);
     }
-    if (strcmp (key, "data") == 0) {
+    if (streq (key, "data")) {
         if (!zi->o || json_object_to_lua (L, zi->o) < 0)
             lua_pushnil (L);
         return (1);
     }
-    if (strcmp (key, "errnum") == 0) {
+    if (streq (key, "errnum")) {
         int errnum;
         if (!(zi->typemask & FLUX_MSGTYPE_RESPONSE))
             return lua_pusherror (L,
-                "zmsg: errnum requested for non-respose msg");
+                "zmsg: errnum requested for non-response msg");
         flux_msg_get_errnum (zi->msg, &errnum);
         lua_pushnumber (L, errnum);
         return (1);
     }
-    if (strcmp (key, "matchtag") == 0) {
+    if (streq (key, "matchtag")) {
         uint32_t matchtag;
         if (flux_msg_get_matchtag (zi->msg, &matchtag) < 0)
             return lua_pusherror (L, "zmsg: matchtag: %s",
diff --git a/src/bindings/lua/zmsg-lua.h b/src/bindings/lua/zmsg-lua.h
index 21c6b597d441..c753c2111da7 100644
--- a/src/bindings/lua/zmsg-lua.h
+++ b/src/bindings/lua/zmsg-lua.h
@@ -12,7 +12,6 @@
 #define HAVE_ZMSG_LUA_H
 
 #include <lua.h>
-#include <zmq.h>
 #include <jansson.h>
 
 #include "flux/core.h"
diff --git a/src/bindings/python/.style.yapf b/src/bindings/python/.style.yapf
deleted file mode 100644
index 41381948ea4d..000000000000
--- a/src/bindings/python/.style.yapf
+++ /dev/null
@@ -1,5 +0,0 @@
-[style]
-based_on_style = pep8
-spaces_before_comment = 4
-split_before_logical_operator = true
-align_closing_bracket_with_visual_indent = true
diff --git a/src/bindings/python/Makefile.am b/src/bindings/python/Makefile.am
index 55f64737c9d4..e073051e7a79 100644
--- a/src/bindings/python/Makefile.am
+++ b/src/bindings/python/Makefile.am
@@ -1,7 +1,4 @@
-
 SUBDIRS=_flux flux
 
-EXTRA_DIST = make_binding.py .pylintrc
-
 clean-local:
 	-rm -f .coverage*
diff --git a/src/bindings/python/_flux/.gitignore b/src/bindings/python/_flux/.gitignore
index 67f6451a6bd3..064a8d8ef55d 100644
--- a/src/bindings/python/_flux/.gitignore
+++ b/src/bindings/python/_flux/.gitignore
@@ -1,2 +1 @@
-*build.py
 *.c
diff --git a/src/bindings/python/_flux/Makefile.am b/src/bindings/python/_flux/Makefile.am
index adb36491d1b2..c3767250db86 100644
--- a/src/bindings/python/_flux/Makefile.am
+++ b/src/bindings/python/_flux/Makefile.am
@@ -1,72 +1,178 @@
 AM_CPPFLAGS = \
-	$(WARNING_CFLAGS) -Wno-missing-field-initializers \
+	$(WARNING_CFLAGS) \
+	-Wno-missing-field-initializers \
 	-I$(top_srcdir) -I$(top_srcdir)/src/include \
 	-I$(top_srcdir)/src/common/libflux \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) $(LIBUUID_CFLAGS) $(PYTHON_CPPFLAGS) \
+	$(PYTHON_CPPFLAGS) \
+	$(JANSSON_CFLAGS) \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	$(CODE_COVERAGE_CFLAGS)
 
 AM_LDFLAGS = \
-	-avoid-version -module $(san_ld_zdef_flag) \
+	-avoid-version \
+	-module \
+	$(san_ld_zdef_flag) \
 	-Wl,-rpath,$(PYTHON_PREFIX)/lib \
+	$(ld_gc_sections) \
 	$(CODE_COVERAGE_LIBS)
 
-MAKE_BINDING=$(top_srcdir)/src/bindings/python/make_binding.py
-SUFFIXES = _build.py
+common_libs = \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libdebugged/libdebugged.la \
+	$(PYTHON_LIBS)
+
+PREPROC_FLAGS = \
+	-E '-D__attribute__(...)=' \
+	-D "FLUX_DEPRECATED(...)="
+
+fluxpyso_LTLIBRARIES = \
+	_core.la \
+	_hostlist.la \
+	_idset.la \
+	_rlist.la
+
+fluxpyso_PYTHON = \
+	__init__.py
+
+nodist_fluxbindinginclude_HEADERS = \
+	_core_preproc.h \
+	_hostlist_preproc.h \
+	_idset_preproc.h \
+	_rlist_preproc.h
+
+EXTRA_DIST = \
+	_core_build.py \
+	_security_build.py \
+	_hostlist_build.py \
+	_idset_build.py \
+	_rlist_build.py \
+	make_clean_header.py
+
+STDERR_DEVNULL = $(stderr_devnull_$(V))
+stderr_devnull_ =  $(stderr_devnull_$(AM_DEFAULT_VERBOSITY))
+stderr_devnull_0 = >/dev/null 2>&1
+
+_core.c: $(srcdir)/_core_build.py _core_preproc.h
+	$(AM_V_GEN)$(PYTHON) $< $(STDERR_DEVNULL)
+
+_core_clean.h: Makefile
+	$(AM_V_GEN)$(PYTHON) $(srcdir)/make_clean_header.py \
+	  --path $(top_srcdir) \
+	  --search $(top_builddir)/src/common/libflux \
+	  --additional_headers \
+	      src/bindings/python/_flux/callbacks.h \
+	      src/common/libdebugged/debugged.h \
+	  --output _core_clean.h \
+	  src/include/flux/core.h
+
+_core_preproc.h: _core_clean.h
+	$(AM_V_GEN)$(CC) $(PREPROC_FLAGS) _core_clean.h \
+	  | sed -e '/^# [0-9]*.*/d' > $@
+
+_hostlist.c: $(srcdir)/_hostlist_build.py _hostlist_preproc.h
+	$(AM_V_GEN)$(PYTHON) $^ $(STDERR_DEVNULL)
+
+_hostlist_clean.h: Makefile
+	$(AM_V_GEN)$(PYTHON) $(srcdir)/make_clean_header.py \
+	  --path $(top_srcdir)/src/common/libhostlist \
+	  --output $@ \
+	  hostlist.h
+
+_hostlist_preproc.h: _hostlist_clean.h
+	$(AM_V_GEN)$(CC) $(PREPROC_FLAGS) $< \
+	  | sed -e '/^# [0-9]*.*/d' > $@
+
+_idset.c: $(srcdir)/_idset_build.py _idset_preproc.h
+	$(AM_V_GEN)$(PYTHON) $^ $(STDERR_DEVNULL)
+
+_idset_clean.h: Makefile
+	$(AM_V_GEN)$(PYTHON) $(srcdir)/make_clean_header.py \
+	  --path $(top_srcdir)/src/common/libidset \
+	  --output $@ \
+	  idset.h
+
+_idset_preproc.h: _idset_clean.h
+	$(AM_V_GEN)$(CC) $(PREPROC_FLAGS) $< \
+	  | sed -e '/^# [0-9]*.*/d' > $@
+
+
+_rlist.c: $(srcdir)/_rlist_build.py _rlist_preproc.h
+	$(AM_V_GEN)$(PYTHON) $^ $(STDERR_DEVNULL)
+
+_rlist_clean.h: Makefile
+	$(AM_V_GEN)$(PYTHON) $(srcdir)/make_clean_header.py \
+	  --path $(top_srcdir)/src/common/librlist \
+	  --search $(top_builddir)/config \
+	  --search /usr/include \
+	  --search $(top_srcdir) \
+	  --ignore_header czmq_containers \
+	  --output $@ \
+	  rlist.h
+
+_rlist_preproc.h: _rlist_clean.h
+	$(AM_V_GEN)$(CC) $(PREPROC_FLAGS) $< \
+	  | sed -e '/^# [0-9]*.*/d' > $@
 
-common_libs = $(top_builddir)/src/common/libflux-core.la \
-	      $(top_builddir)/src/common/libflux-internal.la \
-	      $(ZMQ_LIBS) $(LIBUUID_LIBS) \
-	      $(PYTHON_LDFLAGS)
-
-_build.py.c:
-	$(PYTHON) $*_build.py
-
-
-_core_build.py: $(MAKE_BINDING)
-	$(PYTHON) $(MAKE_BINDING) --path $(top_srcdir) \
-		--search $(top_builddir)/src/common/libflux \
-		--package _flux \
-		--modname _core \
-		--add_sub '.*va_list.*|||' \
-		--additional_headers src/bindings/python/_flux/callbacks.h \
-		--ignore_header 'macros' \
-		src/include/flux/core.h
-
-BUILT_SOURCES= _core.c
-fluxpyso_LTLIBRARIES = _core.la
-fluxpyso_PYTHON = __init__.py
 
 dist__core_la_SOURCES = callbacks.h
 nodist__core_la_SOURCES = _core.c
 _core_la_LIBADD = $(common_libs)
-_core_la_DEPENDENCIES = _core_build.py
+
+nodist__hostlist_la_SOURCES = _hostlist.c
+_hostlist_la_LIBADD = $(common_libs)
+
+nodist__idset_la_SOURCES = _idset.c
+_idset_la_LIBADD = $(common_libs)
+
+nodist__rlist_la_SOURCES = _rlist.c
+_rlist_la_LIBADD = \
+	$(top_builddir)/src/common/librlist/librlist.la \
+	$(common_libs)
 
 if HAVE_FLUX_SECURITY
-BUILT_SOURCES += _security.c
-fluxpyso_LTLIBRARIES += _security.la
-_security_build.py: $(MAKE_BINDING)
-	$(PYTHON) $(MAKE_BINDING) --path $(FLUX_SECURITY_INCDIR)/flux/security \
-		--package _flux \
-		--modname _security \
-		--include_header flux/security/sign.h \
-		--add_sub '.*va_list.*|||' \
-		sign.h
-
-nodist__security_la_SOURCES = _security.c
-_security_la_CPPFLAGS = $(AM_CPPFLAGS) $(FLUX_SECURITY_CFLAGS)
-_security_la_LIBADD = $(common_libs) $(FLUX_SECURITY_LIBS)
-_security_la_DEPENDENCIES = _security_build.py
+
+fluxpyso_LTLIBRARIES += \
+	_security.la
+
+nodist_fluxbindinginclude_HEADERS +=\
+	_security_preproc.h
+
+nodist__security_la_SOURCES = \
+	_security.c
+
+_security.c: $(srcdir)/_security_build.py _security_preproc.h
+	$(AM_V_GEN)$(PYTHON) $^ $(STDERR_DEVNULL)
+_security_clean.h: Makefile
+	$(AM_V_GEN)$(PYTHON) $(srcdir)/make_clean_header.py \
+	  --path $(FLUX_SECURITY_INCDIR)/flux/security \
+	  --output _security_clean.h \
+	  sign.h $(STDERR_DEVNULL)
+_security_preproc.h: _security_clean.h
+	$(AM_V_GEN)$(CC) $(PREPROC_FLAGS) _security_clean.h \
+	  | sed -e '/^# [0-9]*.*/d' > $@
+
+_security_la_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(FLUX_SECURITY_CFLAGS)
+_security_la_LIBADD = \
+	$(common_libs) \
+	$(FLUX_SECURITY_LIBS)
 endif
 
 .PHONY: lib-copy
 
+cp_verbose = $(cp_verbose_$(V))
+cp_verbose_ = $(cp_verbose_$(AM_DEFAULT_VERBOSITY))
+cp_verbose_0 = @echo "  COPY     python DSOs" ;
+
+# Copy libraries to where they can be used by python in-tree
 lib-copy-vpath: ${fluxpyso_PYTHON} ${fluxpyso_LTLIBRARIES}
-	-echo Copying libraries to where they can be used by python in-tree
-	[ "$(top_srcdir)" != "$(top_builddir)" ] && cp $(top_srcdir)/src/bindings/python/_flux/__init__.py ./; \
+	$(cp_verbose)[ "$(top_srcdir)" != "$(top_builddir)" ] && cp $(top_srcdir)/src/bindings/python/_flux/__init__.py ./; \
 	for LIB in ${fluxpyso_LTLIBRARIES:la=so} ; do \
 		test -e .libs/$$LIB && \
-		$(LN_S) .libs/$$LIB ./ || true; \
+		$(LN_S) .libs/$$LIB ./ $(STDERR_DEVNULL) || true; \
 	done
 
 all-local: lib-copy-vpath
@@ -76,7 +182,7 @@ clean-local-vpath:
 	[ "$(top_srcdir)" != "$(top_builddir)" ] && rm -f $(top_builddir)/src/bindings/python/_flux/*.py || true
 
 clean-local: clean-local-vpath
-	-rm -f *.c *_build.py *.so *.pyc *.pyo
+	-rm -f *.c *.so *.pyc *.pyo *_clean.h *_preproc.h
 	-rm -rf __pycache__
 
 install-data-hook:
diff --git a/src/bindings/python/_flux/_core_build.py b/src/bindings/python/_flux/_core_build.py
new file mode 100644
index 000000000000..161af875f5e4
--- /dev/null
+++ b/src/bindings/python/_flux/_core_build.py
@@ -0,0 +1,43 @@
+from pathlib import Path
+
+from cffi import FFI
+
+ffi = FFI()
+
+
+ffi.set_source(
+    "_flux._core",
+    """
+#include <src/include/flux/core.h>
+#include <src/common/libdebugged/debugged.h>
+
+
+void * unpack_long(ptrdiff_t num){
+  return (void*)num;
+}
+// TODO: remove this when we can use cffi 1.10
+#ifdef __GNUC__
+#pragma GCC visibility push(default)
+#endif
+            """,
+    libraries=["flux-core"],
+)
+
+cdefs = """
+typedef int... ptrdiff_t;
+typedef int... pid_t;
+typedef ... va_list;
+void * unpack_long(ptrdiff_t num);
+void free(void *ptr);
+#define FLUX_JOBID_ANY 0xFFFFFFFFFFFFFFFF
+
+    """
+
+with open("_core_preproc.h") as h:
+    cdefs = cdefs + h.read()
+
+ffi.cdef(cdefs)
+if __name__ == "__main__":
+    ffi.emit_c_code("_core.c")
+    # ensure mtime of target is updated
+    Path("_core.c").touch()
diff --git a/src/bindings/python/_flux/_hostlist_build.py b/src/bindings/python/_flux/_hostlist_build.py
new file mode 100644
index 000000000000..41123dfd1d5e
--- /dev/null
+++ b/src/bindings/python/_flux/_hostlist_build.py
@@ -0,0 +1,31 @@
+from pathlib import Path
+
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source(
+    "_flux._hostlist",
+    """
+#include <flux/hostlist.h>
+
+
+// TODO: remove this when we can use cffi 1.10
+#ifdef __GNUC__
+#pragma GCC visibility push(default)
+#endif
+            """,
+)
+
+cdefs = """
+    void free (void *);
+"""
+
+with open("_hostlist_preproc.h") as h:
+    cdefs = cdefs + h.read()
+
+ffi.cdef(cdefs)
+if __name__ == "__main__":
+    ffi.emit_c_code("_hostlist.c")
+    # Ensure target mtime is updated, emit_c_code() may not do it
+    Path("_hostlist.c").touch()
diff --git a/src/bindings/python/_flux/_idset_build.py b/src/bindings/python/_flux/_idset_build.py
new file mode 100644
index 000000000000..1d2c8ab19e9b
--- /dev/null
+++ b/src/bindings/python/_flux/_idset_build.py
@@ -0,0 +1,32 @@
+from pathlib import Path
+
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source(
+    "_flux._idset",
+    """
+#include <flux/idset.h>
+
+
+// TODO: remove this when we can use cffi 1.10
+#ifdef __GNUC__
+#pragma GCC visibility push(default)
+#endif
+            """,
+)
+
+cdefs = """
+static const unsigned int IDSET_INVALID_ID;
+void free (void *);
+"""
+
+with open("_idset_preproc.h") as h:
+    cdefs = cdefs + h.read()
+
+ffi.cdef(cdefs)
+if __name__ == "__main__":
+    ffi.emit_c_code("_idset.c")
+    # Ensure target mtime is updated
+    Path("_idset.c").touch()
diff --git a/src/bindings/python/_flux/_rlist_build.py b/src/bindings/python/_flux/_rlist_build.py
new file mode 100644
index 000000000000..75a2cd3e180b
--- /dev/null
+++ b/src/bindings/python/_flux/_rlist_build.py
@@ -0,0 +1,47 @@
+from pathlib import Path
+
+from _hostlist_build import ffi as hostlist_ffi
+from _idset_build import ffi as idset_ffi
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.include(hostlist_ffi)
+ffi.include(idset_ffi)
+
+ffi.set_source(
+    "_flux._rlist",
+    """
+#include <jansson.h>
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libflux/types.h"
+#include "src/common/librlist/rlist.h"
+
+
+// TODO: remove this when we can use cffi 1.10
+#ifdef __GNUC__
+#pragma GCC visibility push(default)
+#endif
+            """,
+)
+
+cdefs = """
+typedef struct _zlistx_t zlistx_t;
+typedef struct _zhashx_t zhashx_t;
+typedef int... json_type;
+typedef struct json_t json_t;
+typedef struct json_error_t json_error_t;
+
+
+void free (void *);
+
+"""
+
+with open("_rlist_preproc.h") as h:
+    cdefs = cdefs + h.read()
+
+ffi.cdef(cdefs)
+if __name__ == "__main__":
+    ffi.emit_c_code("_rlist.c")
+    # Ensure target mtime is updated
+    Path("_rlist.c").touch()
diff --git a/src/bindings/python/_flux/_security_build.py b/src/bindings/python/_flux/_security_build.py
new file mode 100644
index 000000000000..c70d86626c09
--- /dev/null
+++ b/src/bindings/python/_flux/_security_build.py
@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from cffi import FFI
+
+ffi = FFI()
+
+
+ffi.set_source(
+    "_flux._security",
+    """
+#include <flux/security/sign.h>
+
+
+// TODO: remove this when we can use cffi 1.10
+#ifdef __GNUC__
+#pragma GCC visibility push(default)
+#endif
+            """,
+)
+
+cdefs = """
+typedef int... ptrdiff_t;
+typedef int... pid_t;
+typedef ... va_list;
+void free(void *ptr);
+
+    """
+
+with open("_security_preproc.h") as h:
+    cdefs = cdefs + h.read()
+
+ffi.cdef(cdefs)
+if __name__ == "__main__":
+    ffi.emit_c_code("_security.c")
+    # ensure target mtime is updated
+    Path("_security.c").touch()
diff --git a/src/bindings/python/_flux/callbacks.h b/src/bindings/python/_flux/callbacks.h
index f241fcaea58a..3378ee7438cf 100644
--- a/src/bindings/python/_flux/callbacks.h
+++ b/src/bindings/python/_flux/callbacks.h
@@ -4,6 +4,10 @@ extern "Python" void message_handler_wrapper(flux_t *, flux_msg_handler_t *, con
 //flux_watcher_f
 extern "Python" void timeout_handler_wrapper(flux_reactor_t *, flux_watcher_t *, int, void *);
 extern "Python" void fd_handler_wrapper(flux_reactor_t *, flux_watcher_t *, int, void *);
+extern "Python" void signal_handler_wrapper(flux_reactor_t *, flux_watcher_t *, int, void *);
 
 //flux_continuation_f
 extern "Python" void continuation_callback(flux_future_t *, void *);
+
+//flux_future_init_f
+extern "Python" void init_callback(flux_future_t *, void *);
diff --git a/src/bindings/python/_flux/make_clean_header.py b/src/bindings/python/_flux/make_clean_header.py
new file mode 100755
index 000000000000..a8bddb555e73
--- /dev/null
+++ b/src/bindings/python/_flux/make_clean_header.py
@@ -0,0 +1,93 @@
+###############################################################
+# Copyright 2014 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import argparse
+import os
+import re
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("header", help="C header file to parse", type=str)
+parser.add_argument("--include_header", help="Include base path", type=str, default="")
+parser.add_argument(
+    "--additional_headers", help="Additional headers to parse", nargs="*", default=[]
+)
+parser.add_argument("--output", help="Output path", default=".")
+parser.add_argument("--path", help="Include base path", default=".")
+parser.add_argument(
+    "--search", help="append to header search path", action="append", default=[]
+)
+parser.add_argument(
+    "--ignore_header",
+    help="Pattern to ignore when searching header files",
+    default=[],
+    action="append",
+)
+
+args = parser.parse_args()
+
+absolute_head = os.path.abspath(os.path.join(args.path, args.header))
+
+# Prepend 'path' option to search list:
+args.search.insert(0, args.path)
+args.search = [os.path.abspath(f) for f in args.search]
+
+
+def find_first(path, name, extra=None):
+    for dirname in path:
+        filename = os.path.join(dirname, name)
+        if os.path.isfile(filename):
+            return filename
+    if extra is not None:
+        filename = os.path.join(extra, name)
+        if os.path.isfile(filename):
+            return filename
+    raise IOError(name)
+
+
+mega_header=""
+def process_header(f, including_path="."):
+    global mega_header
+    if not os.path.isfile(f):
+        f = os.path.join(including_path, f)
+    f = os.path.abspath(f)
+    if f not in checked_heads:
+        for p in args.ignore_header:
+            if re.search(p, f):
+                checked_heads[f] = 1
+                return
+        with open(f, "r") as header:
+            for l in header.readlines():
+                m = re.search(r'#include\s*"([^"]*)"', l)
+                if m:
+                    nf = find_first(args.search, m.group(1), including_path)
+                    process_header(nf, os.path.dirname(os.path.abspath(nf)))
+                if not re.match(r"#\s*include", l):
+                    mega_header += l
+        checked_heads[f] = 1
+
+
+orig_wd = os.getcwd()
+os.chdir(args.path)
+
+checked_heads = {}
+
+process_header(absolute_head)
+
+for header in args.additional_headers:
+    process_header(header)
+
+include_head = args.header
+if args.include_header != "":
+    include_head = args.include_header
+
+os.chdir(orig_wd)
+with open(args.output, "w") as clean_header:
+    clean_header.write(mega_header)
diff --git a/src/bindings/python/flux/.gitignore b/src/bindings/python/flux/.gitignore
index c3d09153ff07..f1a32f874edd 100644
--- a/src/bindings/python/flux/.gitignore
+++ b/src/bindings/python/flux/.gitignore
@@ -1,4 +1,3 @@
 # Intermediate autogenerated files
 _*_build.py
 _*.c
-
diff --git a/src/bindings/python/flux/Makefile.am b/src/bindings/python/flux/Makefile.am
index 907ed63318db..1cfe05137bec 100644
--- a/src/bindings/python/flux/Makefile.am
+++ b/src/bindings/python/flux/Makefile.am
@@ -1,31 +1,129 @@
-fluxpy_PYTHON=\
-	      __init__.py\
-	      kvs.py\
-	      wrapper.py\
-	      rpc.py\
-	      message.py\
-	      constants.py\
-	      job.py \
-	      util.py \
-	      future.py \
-	      memoized_property.py
+nobase_fluxpy_PYTHON = \
+	__init__.py \
+	kvs.py \
+	wrapper.py \
+	rpc.py \
+	message.py \
+	constants.py \
+	util.py \
+	compat36.py \
+	future.py \
+	memoized_property.py \
+	debugged.py \
+	importer.py \
+	conf_builtin.py \
+	cli/__init__.py \
+	cli/base.py \
+	cli/alloc.py \
+	cli/batch.py \
+	cli/bulksubmit.py \
+	cli/run.py \
+	cli/submit.py \
+	cli/fortune.py \
+	core/__init__.py \
+	core/watchers.py \
+	core/inner.py \
+	core/handle.py \
+	core/trampoline.py \
+	job/__init__.py \
+	job/JobID.py \
+	job/Jobspec.py \
+	job/event.py \
+	job/kill.py \
+	job/kvs.py \
+	job/list.py \
+	job/kvslookup.py \
+	job/info.py \
+	job/wait.py \
+	job/submit.py \
+	job/timeleft.py \
+	job/stats.py \
+	job/output.py \
+	job/watcher.py \
+	job/_wrapper.py \
+	job/executor.py \
+	job/directives.py \
+	job/journal.py \
+	job/validator/__init__.py \
+	job/validator/validator.py \
+	job/validator/plugins/jobspec.py \
+	job/validator/plugins/feasibility.py \
+	job/validator/plugins/require-instance.py \
+	job/frobnicator/__init__.py \
+	job/frobnicator/frobnicator.py \
+	job/frobnicator/plugins/defaults.py \
+	job/frobnicator/plugins/constraints.py \
+	resource/Rlist.py \
+	resource/__init__.py \
+	resource/ResourceSetImplementation.py \
+	resource/ResourceSet.py \
+	resource/list.py \
+	resource/status.py \
+	hostlist.py \
+	idset.py \
+	progress.py \
+	uri/uri.py \
+	uri/__init__.py \
+	uri/resolvers/jobid.py \
+	uri/resolvers/pid.py \
+	uri/resolvers/slurm.py \
+	uri/resolvers/lsf.py \
+	constraint/parser.py \
+	constraint/parsetab.py \
+	constraint/__init__.py \
+	utils/parsedatetime/__init__.py \
+	utils/parsedatetime/parsedatetime.py \
+	utils/parsedatetime/warns.py \
+	utils/parsedatetime/context.py \
+	utils/parsedatetime/pdt_locales/__init__.py \
+	utils/parsedatetime/pdt_locales/base.py \
+	utils/parsedatetime/pdt_locales/icu.py \
+	utils/parsedatetime/pdt_locales/de_DE.py \
+	utils/parsedatetime/pdt_locales/en_US.py \
+	utils/parsedatetime/pdt_locales/en_AU.py \
+	utils/parsedatetime/pdt_locales/es.py \
+	utils/parsedatetime/pdt_locales/fr_FR.py \
+	utils/parsedatetime/pdt_locales/nl_NL.py \
+	utils/parsedatetime/pdt_locales/pt_BR.py \
+	utils/parsedatetime/pdt_locales/ru_RU.py \
+	utils/tomli/_re.py \
+	utils/tomli/_parser.py \
+	utils/tomli/__init__.py \
+	utils/tomli/py.typed \
+	utils/tomli/LICENSE \
+	utils/tomli/_types.py
 
 if HAVE_FLUX_SECURITY
-fluxpy_PYTHON += security.py
+nobase_fluxpy_PYTHON += security.py
 endif
 
-if ENABLE_PYLINT
-#TODO: there must be a better way to do this
-# scan flux bindings with pylint, currently fails on any exit but 0
-check-local:
-	if [ -x "$$( which pylint )" ] ; then  pylint --rcfile=$(top_srcdir)/src/bindings/python/.pylintrc $(top_srcdir)/src/bindings/python/flux; fi
-endif
 
-clean-local:
-	-rm -f *.pyc *.pyo
-	-rm -rf __pycache__
+BUILT_SOURCES = \
+	constraint/parsetab.py
+
+CLEANFILES = \
+	constraint/parser.out \
+	$(BUILT_SOURCES)
 
-SUBDIRS = core
+STDERR_DEVNULL = $(stderr_devnull_$(V))
+stderr_devnull_ =  $(stderr_devnull_$(AM_DEFAULT_VERBOSITY))
+stderr_devnull_0 = >/dev/null 2>&1
+
+#  Ensure that python-ply's parsetab.py is generated. Create
+#  parent directory if necessary in builddir. Note: this also
+#  creates a parser.out, which is explicitly removed in
+#  CLEANFILES above. If in builddir, constraint directory
+#  should be removed. That is done below in clean-local.
+#
+constraint/parsetab.py: constraint/parser.py
+	$(AM_V_GEN)$(MKDIR_P) constraint && \
+	  $(PYTHON) $< --outputdir $(builddir)/constraint $(STDERR_DEVNULL) && \
+	  touch $@
+
+clean-local:
+	-rmdir constraint 2>/dev/null || :
+	-rm -f *.pyc */*.pyc *.pyo */*.pyo
+	-rm -rf __pycache__ */__pycache__
 
 install-data-hook:
 	$(AM_V_at)echo Linking python modules in non-standard location... && \
diff --git a/src/bindings/python/flux/__init__.py b/src/bindings/python/flux/__init__.py
index 6ebaa04a5dfe..5595d04c4c1e 100644
--- a/src/bindings/python/flux/__init__.py
+++ b/src/bindings/python/flux/__init__.py
@@ -11,11 +11,12 @@
 """
 python bindings to flux-core, the main core of the flux resource manager
 """
+import flux.core.handle
+
+
 # Manually lazy
 # pylint: disable=invalid-name
 def Flux(*args, **kwargs):
-    import flux.core.handle
-
     return flux.core.handle.Flux(*args, **kwargs)
 
 
diff --git a/src/bindings/python/flux/cli/__init__.py b/src/bindings/python/flux/cli/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/bindings/python/flux/cli/alloc.py b/src/bindings/python/flux/cli/alloc.py
new file mode 100644
index 000000000000..9cf1c5678dc4
--- /dev/null
+++ b/src/bindings/python/flux/cli/alloc.py
@@ -0,0 +1,177 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import os
+import sys
+import time
+
+import flux
+import flux.job
+from flux.cli import base
+from flux.uri import JobURI
+
+
+class AllocCmd(base.MiniCmd):
+    def __init__(self, prog, usage=None, description=None):
+        self.t0 = None
+        super().__init__(prog, usage, description, exclude_io=True)
+        base.add_batch_alloc_args(self.parser)
+        self.parser.add_argument(
+            "-v",
+            "--verbose",
+            action="count",
+            default=0,
+            help="Increase verbosity on stderr (multiple use OK)",
+        )
+        self.parser.add_argument(
+            "--bg",
+            action="store_true",
+            help="Wait for new instance to start, but do not attach to it.",
+        )
+        self.parser.add_argument(
+            "COMMAND",
+            nargs=argparse.REMAINDER,
+            help="Set the initial COMMAND of new Flux instance."
+            + "(default is an interactive shell)",
+        )
+
+    def init_jobspec(self, args):
+        #  If number of slots not specified, then set it to node count
+        #   if set, otherwise raise an error.
+        if not args.nslots:
+            if not args.nodes:
+                raise ValueError("Number of slots to allocate must be specified")
+            args.nslots = args.nodes
+            args.exclusive = True
+
+        #  For --bg, do not run an rc2 (initial program) unless
+        #    the user explicitly specified COMMAND:
+        if args.bg and not args.COMMAND:
+            args.broker_opts = args.broker_opts or []
+            args.broker_opts.append("-Sbroker.rc2_none=1")
+
+        if args.dump:
+            args.broker_opts = args.broker_opts or []
+            args.broker_opts.append("-Scontent.dump=" + args.dump)
+
+        jobspec = flux.job.JobspecV1.from_nest_command(
+            command=args.COMMAND,
+            num_slots=args.nslots,
+            cores_per_slot=args.cores_per_slot,
+            gpus_per_slot=args.gpus_per_slot,
+            num_nodes=args.nodes,
+            broker_opts=base.list_split(args.broker_opts),
+            exclusive=args.exclusive,
+            conf=args.conf.config,
+        )
+
+        #  For --bg, always allocate a pty, but not interactive,
+        #   since an interactive pty causes the job shell to hang around
+        #   until a pty client attaches, which may never happen.
+        #
+        #  O/w, allocate an interactive pty only if stdin is a tty
+        #
+        if args.bg:
+            jobspec.setattr_shell_option("pty.capture", 1)
+        elif sys.stdin.isatty():
+            jobspec.setattr_shell_option("pty.interactive", 1)
+        return jobspec
+
+    @staticmethod
+    def log(jobid, ts, msg):
+        print(f"{jobid}: {ts:6.3f}s: {msg}", file=sys.stderr, flush=True)
+
+    def bg_wait_cb(self, future, args, jobid):
+        """
+        Wait for memo event, connect to child instance, and finally wait
+        for rc1 to complete
+        """
+        event = future.get_event()
+        if not event:
+            #  The job has unexpectedly exited since we're at the end
+            #   of the eventlog. Run `flux job attach` since this will dump
+            #   any errors or output, then raise an exception.
+            os.system(f"flux job attach {jobid} >&2")
+            raise OSError(f"{jobid}: unexpectedly exited")
+
+        if not self.t0:
+            self.t0 = event.timestamp
+        ts = event.timestamp - self.t0
+
+        if args.verbose and event.name == "alloc":
+            self.log(jobid, ts, "resources allocated")
+        if event.name == "memo" and "uri" in event.context:
+            if args.verbose:
+                self.log(jobid, ts, "waiting for instance")
+
+            #  Wait for child instance to finish rc1 using state-machine.wait,
+            #   then stop the reactor to return to caller.
+            uri = str(JobURI(event.context["uri"]))
+            try:
+                child_handle = flux.Flux(uri)
+            except OSError as exc:
+                raise OSError(f"Unable to connect to {jobid}: {exc}")
+            try:
+                child_handle.rpc("state-machine.wait").get()
+            except OSError:
+                raise OSError(f"{jobid}: instance startup failed")
+
+            if args.verbose:
+                self.log(jobid, time.time() - self.t0, "ready")
+            self.flux_handle.reactor_stop()
+
+    def background(self, args, jobid):
+        """Handle the --bg option
+
+        Wait for child instance to be ready to accept jobs before returning.
+        Print jobid to stdout once the job is ready.
+        """
+        jobid = flux.job.JobID(jobid)
+
+        flux.job.event_watch_async(self.flux_handle, jobid).then(
+            self.bg_wait_cb, args, jobid
+        )
+        if args.verbose:
+            self.log(jobid, 0.0, "waiting for resources")
+        try:
+            self.flux_handle.reactor_run()
+        except KeyboardInterrupt:
+            print(f"\r{jobid}: Interrupt: canceling job", file=sys.stderr)
+            flux.job.cancel(self.flux_handle, jobid)
+            sys.exit(1)
+
+        print(jobid)
+
+    def main(self, args):
+        jobid = self.submit(args)
+
+        if args.bg:
+            self.background(args, jobid)
+            sys.exit(0)
+
+        # Display job id on stderr if -v
+        # N.B. we must flush sys.stderr due to the fact that it is buffered
+        # when it points to a file, and os.execvp leaves it unflushed
+        if args.verbose > 0:
+            print("jobid:", jobid, file=sys.stderr)
+            sys.stderr.flush()
+
+        # Build args for flux job attach
+        attach_args = ["flux-job", "attach"]
+        attach_args.append(jobid.f58.encode("utf-8", errors="surrogateescape"))
+
+        # Exec flux-job attach, searching for it in FLUX_EXEC_PATH.
+        old_path = os.environ.get("PATH")
+        os.environ["PATH"] = os.environ["FLUX_EXEC_PATH"]
+        if old_path:
+            os.environ["PATH"] += f":{old_path}"
+
+        os.execvp("flux-job", attach_args)
diff --git a/src/bindings/python/flux/cli/base.py b/src/bindings/python/flux/cli/base.py
new file mode 100644
index 000000000000..bab650ed43ae
--- /dev/null
+++ b/src/bindings/python/flux/cli/base.py
@@ -0,0 +1,1706 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+# A command base for all user facing commands for different
+# kinds of submission. This used to be a base in flux-mini
+# and now is used as a module with shared logic.
+
+import argparse
+import atexit
+import fnmatch
+import json
+import logging
+import os
+import pathlib
+import re
+import resource
+import signal
+import sys
+from collections import ChainMap
+from itertools import chain
+from os.path import basename
+from string import Template
+from urllib.parse import parse_qs, urlparse
+
+try:
+    import tomllib  # novermin
+except ModuleNotFoundError:
+    from flux.utils import tomli as tomllib
+
+import flux
+from flux import debugged, job, util
+from flux.constraint.parser import ConstraintParser, ConstraintSyntaxError
+from flux.idset import IDset
+from flux.job import JobspecV1, JobWatcher
+from flux.progress import ProgressBar
+from flux.util import dict_merge, set_treedict
+
+LOGGER = logging.getLogger("flux")
+
+
+def decode_signal(val):
+    """
+    Decode a signal as string or number
+    A string can be of the form 'SIGUSR1' or just 'USR1'
+    """
+    if isinstance(val, int):
+        return val
+    try:
+        return int(val)
+    except ValueError:
+        pass  # Fall back to signal name
+    try:
+        return getattr(signal, val)
+    except AttributeError:
+        pass  # Fall back SIG{name}
+    try:
+        return getattr(signal, f"SIG{val}")
+    except AttributeError:
+        pass
+    raise ValueError(f"signal '{val}' is invalid")
+
+
+def decode_duration(val):
+    """
+    Decode a duration as a number or string in FSD
+    """
+    if isinstance(val, (float, int)):
+        return val
+    try:
+        return float(val)
+    except ValueError:
+        pass  # Fall back to fsd
+    return util.parse_fsd(val)
+
+
+def parse_signal_option(arg):
+    """
+    Parse the --signal= option argument of the form SIG@TIME, where
+    both signal and time are optional.
+
+    Returns a dict with signal and timeleft members
+    """
+    signo = signal.SIGUSR1
+    tleft = 60
+    if arg is not None:
+        sig, _, time = arg.partition("@")
+        if time:
+            tleft = time
+        if sig:
+            signo = sig
+    try:
+        signum = decode_signal(signo)
+        if signum <= 0:
+            raise ValueError("signal must be > 0")
+    except ValueError as exc:
+        raise ValueError(f"--signal={arg}: {exc}") from None
+
+    try:
+        timeleft = decode_duration(tleft)
+    except ValueError as exc:
+        raise ValueError(f"--signal={arg}: {exc}") from None
+
+    return {"signum": signum, "timeleft": timeleft}
+
+
+class MiniConstraintParser(ConstraintParser):
+    operator_map = {
+        None: "properties",
+        "host": "hostlist",
+        "hosts": "hostlist",
+        "rank": "ranks",
+    }
+    split_values = {"properties": ","}
+    combined_terms = {"properties"}
+
+
+class URIArg:
+    """Convenience class for handling dependencies
+
+    Splits a dependency URI into fields and returns an RFC 26 dependency
+    entry via the entry attribute.
+    """
+
+    def __init__(self, uri, name):
+        # append `:` if missing in uri so that a plain string is treated as
+        # a scheme with no path.
+        if ":" not in uri:
+            uri += ":"
+
+        # replace first ':' with ':FXX' to work around urlparse refusal
+        # to treat integer only path as a scheme:path.
+        self.uri = urlparse(uri.replace(":", ":FXX", 1))
+
+        if not self.uri.scheme or not self.uri.path:
+            raise ValueError(f'Invalid {name} URI "{uri}"')
+
+        self.path = self.uri.path.replace("FXX", "", 1)
+        self.scheme = self.uri.scheme
+
+    @staticmethod
+    def _try_number(value):
+        """Convert value to an int or a float if possible"""
+        for _type in (int, float):
+            try:
+                return _type(value)
+            except ValueError:
+                continue
+        return value
+
+    @property
+    def entry(self):
+        entry = {
+            "scheme": self.scheme,
+            "value": self.path,
+        }
+        if self.uri.query:
+            for key, val in parse_qs(self.uri.query).items():
+                #  val is always a list, but convert to single value
+                #   if it only contains a single item:
+                if len(val) > 1:
+                    entry[key] = [self._try_number(x) for x in val]
+                else:
+                    entry[key] = self._try_number(val[0])
+        return entry
+
+
+def dependency_array_create(uris):
+    dependencies = []
+    for uri in uris:
+        dependencies.append(URIArg(uri, "dependency").entry)
+    return dependencies
+
+
+class BeginTimeAction(argparse.Action):
+    """Convenience class to handle --begin-time file option
+
+    Append --begin-time options to the "dependency" list in namespace
+    """
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        uri = "begin-time:" + str(util.parse_datetime(values).timestamp())
+        items = getattr(namespace, "dependency", [])
+        if items is None:
+            items = []
+        items.append(uri)
+        setattr(namespace, "dependency", items)
+
+
+def filter_dict(env, pattern, reverseMatch=True):
+    """
+    Filter out all keys that match "pattern" from dict 'env'
+
+    Pattern is assumed to be a shell glob(7) pattern, unless it begins
+    with '/', in which case the pattern is a regex.
+    """
+    if pattern.startswith("/"):
+        pattern = pattern[1::].rstrip("/")
+    else:
+        pattern = fnmatch.translate(pattern)
+    regex = re.compile(pattern)
+    if reverseMatch:
+        return dict(filter(lambda x: not regex.match(x[0]), env.items()))
+    return dict(filter(lambda x: regex.match(x[0]), env.items()))
+
+
+def get_rlimits(name="*"):
+    """
+    Return set of rlimits matching `name` in a dict
+    """
+    rlimits = {}
+    pattern = f"RLIMIT_{name}".upper()
+    for limit in fnmatch.filter(resource.__dict__.keys(), pattern):
+        soft, hard = resource.getrlimit(getattr(resource, limit))
+        #  Remove RLIMIT_ prefix and lowercase the result for compatibility
+        #  with rlimit shell plugin:
+        rlimits[limit[7::].lower()] = soft
+    if not rlimits:
+        raise ValueError(f'No corresponding rlimit matching "{name}"')
+    return rlimits
+
+
+def list_split(opts):
+    """
+    Return a list by splitting each member of opts on ','
+    """
+    if opts:
+        x = chain.from_iterable([x.split(",") for x in opts])
+        return list(x)
+    return []
+
+
+def get_filtered_rlimits(user_rules=None):
+    """
+    Get a filtered set of rlimits based on user rules and defaults
+    """
+    #  We start with the entire set of available rlimits and then apply
+    #  rules to remove or override them. Therefore, the set of default
+    #  rules excludes limits *not* to propagate by default.
+    rules = [
+        "-memlock",
+        "-ofile",
+        "-msgqueue",
+        "-nice",
+        "-rtprio",
+        "-rttime",
+        "-sigpending",
+    ]
+
+    if user_rules:
+        rules.extend(list_split(user_rules))
+
+    rlimits = get_rlimits()
+    for rule in rules:
+        if rule.startswith("-"):
+            # -name removes RLIMIT_{pattern} from propagated rlimits
+            rlimits = filter_dict(rlimits, rule[1::].lower())
+        else:
+            name, *rest = rule.split("=", 1)
+            if not rest:
+                #  limit with no value pulls in current limit(s)
+                limits = get_rlimits(name.lower())
+                for key, value in limits.items():
+                    rlimits[key] = value
+            else:
+                if not hasattr(resource, f"RLIMIT_{name}".upper()):
+                    raise ValueError(f'Invalid rlimit "{name}"')
+                #  limit with value sets limit to that value
+                if rest[0] in ["unlimited", "infinity", "inf"]:
+                    value = resource.RLIM_INFINITY
+                else:
+                    try:
+                        value = int(rest[0])
+                    except ValueError:
+                        raise ValueError(f"Invalid value in {name}={rest[0]}")
+                rlimits[name.lower()] = value
+    return rlimits
+
+
+def get_filtered_environment(rules, environ=None):
+    """
+    Filter environment dictionary 'environ' given a list of rules.
+    Each rule can filter, set, or modify the existing environment.
+    """
+    if environ is None:
+        environ = dict(os.environ)
+    if rules is None:
+        return environ
+    for rule in rules:
+        #
+        #  If rule starts with '-' then the rest of the rule is a pattern
+        #   which filters matching environment variables from the
+        #   generated environment.
+        #
+        if rule.startswith("-"):
+            environ = filter_dict(environ, rule[1::])
+        #
+        #  If rule starts with '^', then the result of the rule is a filename
+        #   from which to read more rules.
+        #
+        elif rule.startswith("^"):
+            filename = os.path.expanduser(rule[1::])
+            with open(filename) as envfile:
+                lines = [line.strip() for line in envfile]
+                environ = get_filtered_environment(lines, environ=environ)
+        #
+        #  Otherwise, the rule is an explicit variable assignment
+        #   VAR=VAL. If =VAL is not provided then VAL refers to the
+        #   value for VAR in the current environment of this process.
+        #
+        #  Quoted shell variables are expanded using values from the
+        #   built environment, not the process environment. So
+        #   --env=PATH=/bin --env=PATH='$PATH:/foo' results in
+        #   PATH=/bin:/foo.
+        #
+        else:
+            var, *rest = rule.split("=", 1)
+            if not rest:
+                #
+                #  VAR alone with no set value pulls in all matching
+                #   variables from current environment that are not already
+                #   in the generated environment.
+                env = filter_dict(os.environ, var, reverseMatch=False)
+                for key, value in env.items():
+                    if key not in environ:
+                        environ[key] = value
+            else:
+                #
+                #  Template lookup: use jobspec environment first, fallback
+                #   to current process environment using ChainMap:
+                lookup = ChainMap(environ, os.environ)
+                try:
+                    environ[var] = Template(rest[0]).substitute(lookup)
+                except ValueError:
+                    LOGGER.error("--env: Unable to substitute %s", rule)
+                    raise
+                except KeyError as ex:
+                    raise Exception(f"--env: Variable {ex} not found in {rule}")
+    return environ
+
+
+class EnvFileAction(argparse.Action):
+    """Convenience class to handle --env-file option
+
+    Append --env-file options to the "env" list in namespace, with "^"
+    prepended to the rule to indicate further rules are to be read
+    from the indicated file.
+
+    This is required to preserve ordering between the --env and --env-file
+    and --env-remove options.
+    """
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = getattr(namespace, "env", [])
+        if items is None:
+            items = []
+        items.append("^" + values)
+        setattr(namespace, "env", items)
+
+
+class EnvFilterAction(argparse.Action):
+    """Convenience class to handle --env-remove option
+
+    Append --env-remove options to the "env" list in namespace, with "-"
+    prepended to the option argument.
+
+    This is required to preserve ordering between the --env and --env-remove
+    options.
+    """
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = getattr(namespace, "env", [])
+        if items is None:
+            items = []
+        items.append("-" + values)
+        setattr(namespace, "env", items)
+
+
+class BatchConfig:
+    """Convenience class for handling a --conf=[FILE|KEY=VAL] option
+
+    Iteratively build a "config" dict from successive updates.
+    """
+
+    loaders = {".toml": tomllib.load, ".json": json.load}
+
+    def __init__(self):
+        self.config = None
+
+    def update_string(self, value):
+        # Update config with JSON or TOML string
+        try:
+            conf = json.loads(value)
+        except json.decoder.JSONDecodeError:
+            # Try TOML
+            try:
+                conf = tomllib.loads(value)
+            except tomllib.TOMLDecodeError:
+                raise ValueError(
+                    "--conf: failed to parse multiline as TOML or JSON"
+                ) from None
+        self.config = dict_merge(self.config, conf)
+        return self
+
+    def update_keyval(self, keyval):
+        # dotted key (e.g. resource.noverify=true)
+        key, _, value = keyval.partition("=")
+        try:
+            value = json.loads(value)
+        except json.decoder.JSONDecodeError:
+            value = str(value)
+        set_treedict(self.config, key, value)
+        return self
+
+    def update_file(self, path, extension=".toml"):
+        # Update from file in the filesystem
+        try:
+            loader = self.loaders[extension]
+        except KeyError:
+            raise ValueError("--conf: {path} must end in .toml or .json")
+        try:
+            with open(path, "rb") as fp:
+                conf = loader(fp)
+        except OSError as exc:
+            raise ValueError(f"--conf: {exc}") from None
+        except (json.decoder.JSONDecodeError, tomllib.TOMLDecodeError) as exc:
+            raise ValueError(f"--conf: parse error: {path}: {exc}") from None
+        self.config = dict_merge(self.config, conf)
+        return self
+
+    def _find_config(self, name):
+        # Find a named config as either TOML or JSON in XDG search path
+        for path in util.xdg_searchpath(subdir="config"):
+            # Take the first matching filename preferring TOML:
+            for ext in (".toml", ".json"):
+                filename = f"{path}/{name}{ext}"
+                if os.path.exists(filename):
+                    return filename, self.loaders[ext]
+        return None, None
+
+    def update_named_config(self, name):
+        # Update from a named configuration file in a standard path or paths.
+        filename, loader = self._find_config(name)
+        if filename is not None:
+            try:
+                with open(filename, "rb") as fp:
+                    self.config = dict_merge(self.config, loader(fp))
+                    return self
+            except (
+                OSError,
+                tomllib.TOMLDecodeError,
+                json.decoder.JSONDecodeError,
+            ) as exc:
+                raise ValueError(f"--conf={name}: {filename}: {exc}") from None
+        raise ValueError(f"--conf: named config '{name}' not found")
+
+    def update(self, value):
+        """
+        Update current config with value using the following rules:
+        - If value contains one or more newlines, parse it as a JSON or
+          TOML string.
+        - Otherwise, if value contains an ``=``, then parse it as a dotted
+          key and value, e.g. ``resource.noverify=true``. The value (part
+          after the ``=``) will be parsed as JSON.
+        - Otherwise, if value ends in ``.toml`` or ``.json`` treat value as
+          a path and attempt to parse contents of file as TOML or JSON.
+        - Otherwise, read a named config from a standard config search path.
+
+        Configuration can be updated iteratively. The end result is available
+        in the ``config`` attribute.
+        """
+        if self.config is None:
+            self.config = {}
+        if "\n" in value:
+            return self.update_string(value)
+        if "=" in value:
+            return self.update_keyval(value)
+        extension = pathlib.Path(value).suffix
+        if extension in (".toml", ".json"):
+            return self.update_file(value, extension)
+        return self.update_named_config(value)
+
+
+class ConfAction(argparse.Action):
+    """Handle batch/alloc --conf option"""
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        conf = getattr(namespace, "conf", None)
+        if conf is None:
+            conf = BatchConfig()
+            setattr(namespace, "conf", conf)
+        conf.update(values)
+
+
+class Xcmd:
+    """Represent a Flux job with mutable command and option args"""
+
+    # dict of mutable argparse args. The values are used in
+    #  the string representation of an Xcmd object.
+    mutable_args = {
+        "queue": "-q",
+        "bank": "-B",
+        "ntasks": "-n",
+        "nodes": "-N",
+        "cores_per_task": "-c",
+        "gpus_per_task": "-g",
+        "cores": "--cores=",
+        "tasks_per_node": "--tasks-per-node=",
+        "tasks_per_core": "--tasks-per-core=",
+        "gpus_per_node": "--gpus-per-node=",
+        "time_limit": "-t",
+        "env": "--env=",
+        "env_file": "--env-file=",
+        "env_remove": "--env-remove=",
+        "urgency": "--urgency=",
+        "setopt": "-o ",
+        "setattr": "--setattr=",
+        "job_name": "--job-name=",
+        "input": "--input=",
+        "output": "--output=",
+        "error": "--error=",
+        "cc": "--cc=",
+        "bcc": "--bcc=",
+        "log": "--log=",
+        "log_stderr": "--log-stderr=",
+        "dependency": "--dependency=",
+        "taskmap": "--taskmap=",
+        "requires": "--requires=",
+        "wait": "--wait-event=",
+        "cwd": "--cwd=",
+        "flags": "--flags=",
+        "begin_time": "--begin-time=",
+        "signal": "--signal=",
+        "taskmap": "--taskmap=",
+    }
+
+    class Xinput:
+        """A string class with convenient attributes for formatting args
+
+        This class represents a string with special attributes specific
+        for use on the bulksubmit command line, e.g.::
+
+            {0.%}    : the argument without filename extension
+            {0./}    : the argument basename
+            {0.//}   : the argument dirname
+            {0./%}   : the basename without filename extension
+            {0.name} : the result of dynamically assigned method "name"
+
+        """
+
+        def __init__(self, arg, methods):
+            self.methods = methods
+            self.string = arg
+
+        def __str__(self):
+            return self.string
+
+        def __getattr__(self, attr):
+            if attr == "%":
+                return os.path.splitext(self.string)[0]
+            if attr == "/":
+                return os.path.basename(self.string)
+            if attr == "//":
+                return os.path.dirname(self.string)
+            if attr == "/%":
+                return os.path.basename(os.path.splitext(self.string)[0])
+            if attr in self.methods:
+                #  Note: combine list return values with the special
+                #   sentinel ::list::: so they can be split up again
+                #   after .format() converts them to strings. This allows
+                #   user-provided methods to return lists as well as
+                #   single values, where each list element can become
+                #   a new argument in a command
+                #
+                # pylint: disable=eval-used
+                result = eval(self.methods[attr], globals(), dict(x=self.string))
+                if isinstance(result, list):
+                    return "::list::".join(result)
+                return result
+            raise ValueError(f"Unknown input string method '.{attr}'")
+
+    @staticmethod
+    def preserve_mustache(val):
+        """Preserve any mustache template in value 'val'"""
+
+        def subst(val):
+            return val.replace("{{", "=stache=").replace("}}", "=/stache=")
+
+        if isinstance(val, str):
+            return subst(val)
+        if isinstance(val, list):
+            return [subst(x) for x in val]
+        return val
+
+    @staticmethod
+    def restore_mustache(val):
+        """Restore any mustache template in value 'val'"""
+
+        def restore(val):
+            return val.replace("=stache=", "{{").replace("=/stache=", "}}")
+
+        if isinstance(val, str):
+            return restore(val)
+        if isinstance(val, list):
+            return [restore(x) for x in val]
+        return val
+
+    def __init__(self, args, inputs=None, **kwargs):
+        """Initialize and Xcmd (eXtensible Command) object
+
+        Given BulkSubmit `args` and `inputs`, substitute all inputs
+        in command and applicable options using string.format().
+
+        """
+        if inputs is None:
+            inputs = []
+
+        #  Save reference to original args:
+        self._orig_args = args
+
+        #  Convert all inputs to Xinputs so special attributes are
+        #   available during .format() processing:
+        #
+        inputs = [self.Xinput(x, args.methods) for x in inputs]
+
+        #  Format each argument in args.command, splitting on the
+        #   special "::list::" sentinel to handle the case where
+        #   custom input methods return a list (See Xinput.__getattr__)
+        #
+        self.command = []
+        for arg in args.command:
+            try:
+                val = self.preserve_mustache(arg)
+                result = val.format(*inputs, **kwargs).split("::list::")
+                newval = self.restore_mustache(result)
+            except (IndexError, KeyError):
+                LOGGER.error("Invalid replacement string in command: '%s'", arg)
+                sys.exit(1)
+            if newval:
+                self.command.extend(newval)
+
+        #  Format all supported mutable options defined in `mutable_args`
+        #  Note: only list and string options are supported.
+        #
+        self.modified = {}
+        for attr in self.mutable_args:
+            val = getattr(args, attr)
+            if val is None:
+                continue
+
+            val = self.preserve_mustache(val)
+
+            try:
+                if isinstance(val, str):
+                    newval = val.format(*inputs, **kwargs)
+                elif isinstance(val, list):
+                    newval = [x.format(*inputs, **kwargs) for x in val]
+                else:
+                    newval = val
+            except IndexError:
+                LOGGER.error(
+                    "Invalid replacement index in %s%s'",
+                    self.mutable_args[attr],
+                    val,
+                )
+                sys.exit(1)
+            except KeyError as exc:
+                LOGGER.error(
+                    "Replacement key %s not found in '%s%s'",
+                    exc,
+                    self.mutable_args[attr],
+                    val,
+                )
+                sys.exit(1)
+
+            newval = self.restore_mustache(newval)
+
+            setattr(self, attr, newval)
+
+            #  For better verbose and dry-run output, capture mutable
+            #   args that were actually changed:
+            if val != newval or attr == "cc":
+                self.modified[attr] = True
+
+    def __getattr__(self, attr):
+        """
+        Fall back to original args if attribute not found.
+        This allows an Xcmd object to used in place of an argparse Namespace.
+        """
+        return getattr(self._orig_args, attr)
+
+    def __str__(self):
+        """String representation of an Xcmd for debugging output"""
+        result = []
+        for attr in self.mutable_args:
+            value = getattr(self, attr)
+            if attr in self.modified and value:
+                opt = self.mutable_args[attr]
+                result.append(f"{opt}{value}")
+        result.extend(self.command)
+        return " ".join(result)
+
+
+def parse_jobspec_keyval(label, keyval):
+    """Parse a key[=value] option as used with --setopt and --setattr
+
+    Supports ^key=filename to load JSON object from a file
+    """
+    # Split into key, val with a default of 1 if no val given:
+    key, val = (keyval.split("=", 1) + [1])[:2]
+
+    # Support key prefix of ^ to load value from a file
+    if key.startswith("^"):
+        key = key.lstrip("^")
+        with open(val) as filep:
+            try:
+                val = json.load(filep)
+            except (json.JSONDecodeError, TypeError) as exc:
+                raise ValueError(f"{label}: {val}: {exc}") from exc
+    else:
+        try:
+            val = json.loads(val)
+        except (json.JSONDecodeError, TypeError):
+            pass
+    return key, val
+
+
+class MiniCmd:
+    """
+    MiniCmd is the base class for all flux submission subcommands
+    """
+
+    def __init__(self, prog, usage=None, description=None, exclude_io=False):
+        self.flux_handle = None
+        self.exitcode = 0
+        self.progress = None
+        self.watcher = None
+        self.parser = self.create_parser(prog, usage, description, exclude_io)
+
+    @staticmethod
+    def create_parser(
+        prog, usage=None, description=None, exclude_io=False, add_help=True
+    ):
+        """
+        Create default parser with args for submission subcommands
+        Args:
+            prog (str): program name in usage output
+            usage (str, optional): usage string, by default
+                ``{prog} [OPTIONS...] COMMAND [ARGS...]``
+            description (str, optional): short description of command to
+                follow usage. May be multiple lines.
+        """
+        if usage is None:
+            usage = f"{prog} [OPTIONS...] COMMAND [ARGS...]"
+        parser = argparse.ArgumentParser(
+            prog=prog,
+            usage=usage,
+            description=description,
+            formatter_class=flux.util.help_formatter(),
+        )
+        parser.add_argument(
+            "-B",
+            "--bank",
+            type=str,
+            metavar="BANK",
+            help="Submit a job to a specific named bank",
+        )
+        parser.add_argument(
+            "-q",
+            "--queue",
+            type=str,
+            metavar="NAME",
+            help="Submit a job to a specific named queue",
+        )
+        parser.add_argument(
+            "-t",
+            "--time-limit",
+            type=str,
+            metavar="MIN|FSD",
+            help="Time limit in minutes when no units provided, otherwise "
+            + "in Flux standard duration, e.g. 30s, 2d, 1.5h",
+        )
+        parser.add_argument(
+            "--urgency",
+            help="Set job urgency (0-31), hold=0, default=16, expedite=31",
+            metavar="N",
+            default="16",
+        )
+        parser.add_argument(
+            "--job-name",
+            type=str,
+            help="Set an optional name for job to NAME",
+            metavar="NAME",
+        )
+        parser.add_argument(
+            "-o",
+            "--setopt",
+            action="append",
+            help="Set shell option OPT. An optional value is supported with"
+            + " OPT=VAL (default VAL=1) (multiple use OK)",
+            metavar="OPT",
+        )
+        parser.add_argument(
+            "-S",
+            "--setattr",
+            action="append",
+            help="Set job attribute ATTR. An optional value is supported "
+            + " with ATTR=VAL (default VAL=1). If ATTR starts with ^, "
+            + "then VAL is a file containing valid JSON which will be used "
+            + "as the value of the attribute. (multiple use OK)",
+            metavar="ATTR",
+        )
+        parser.add_argument(
+            "--add-file",
+            action="append",
+            help="Add a file at PATH with optional NAME to jobspec. The "
+            + "file will be extracted to {{tmpdir}}/NAME. If NAME is not "
+            + "specified, then the basename of PATH will be used. If "
+            + "necessary, permissions may be specified via NAME:PERMS. "
+            + "(multiple use OK)",
+            metavar="[NAME=]PATH",
+        )
+        parser.add_argument(
+            "--dependency",
+            action="append",
+            help="Set an RFC 26 dependency URI for this job",
+            metavar="URI",
+        )
+        parser.add_argument(
+            "--requires",
+            action="append",
+            help="Specify job constraints in RFC 35 syntax",
+            metavar="CONSTRAINT",
+        )
+        parser.add_argument(
+            "--begin-time",
+            action=BeginTimeAction,
+            metavar="+FSD|TIME",
+            help="Set minimum start time as offset in FSD (e.g. +1h) or "
+            + 'an absolute TIME (e.g. "3pm") for job',
+        )
+        parser.add_argument(
+            "--env",
+            action="append",
+            help="Control how environment variables are exported. If RULE "
+            + "starts with '-' apply rest of RULE as a remove filter (see "
+            + "--env-remove), if '^' then read rules from a file "
+            + "(see --env-file). Otherwise, set matching environment variables "
+            + "from the current environment (--env=PATTERN) or set a value "
+            + "explicitly (--env=VAR=VALUE). Rules are applied in the order "
+            + "they are used on the command line. (multiple use OK)",
+            metavar="RULE",
+        )
+        parser.add_argument(
+            "--env-remove",
+            action=EnvFilterAction,
+            help="Remove environment variables matching PATTERN. "
+            + "If PATTERN starts with a '/', then it is matched "
+            + "as a regular expression, otherwise PATTERN is a shell "
+            + "glob expression. (multiple use OK)",
+            metavar="PATTERN",
+        )
+        parser.add_argument(
+            "--env-file",
+            action=EnvFileAction,
+            help="Read a set of environment rules from FILE. (multiple use OK)",
+            metavar="FILE",
+        )
+        parser.add_argument(
+            "--rlimit",
+            action="append",
+            help="Control how soft resource limits are propagated to the job. "
+            + "If RULE starts with a '-', then do not propagate matching "
+            + "resource limits (e.g. '-*' propagates nothing). Otherwise, "
+            + "propagate the current limit or a specific value, e.g. "
+            + "--rlimit=core or --rlimit=core=16. The option may be used "
+            + "multiple times to build a reduced set of propagated limits, "
+            + "e.g. --rlimit=-*,core will only propagate RLIMIT_CORE.",
+            metavar="RULE",
+        )
+        parser.add_argument(
+            "--input",
+            type=str,
+            help=(
+                "Redirect job stdin from FILENAME, bypassing KVS"
+                if not exclude_io
+                else argparse.SUPPRESS
+            ),
+            metavar="FILENAME",
+        )
+        parser.add_argument(
+            "--output",
+            type=str,
+            help=(
+                "Redirect job stdout to FILENAME, bypassing KVS"
+                if not exclude_io
+                else argparse.SUPPRESS
+            ),
+            metavar="FILENAME",
+        )
+        parser.add_argument(
+            "--error",
+            type=str,
+            help=(
+                "Redirect job stderr to FILENAME, bypassing KVS"
+                if not exclude_io
+                else argparse.SUPPRESS
+            ),
+            metavar="FILENAME",
+        )
+        parser.add_argument(
+            "-u",
+            "--unbuffered",
+            action="store_true",
+            help="Disable buffering of input and output",
+        )
+        parser.add_argument(
+            "-l",
+            "--label-io",
+            action="store_true",
+            help=(
+                "Add rank labels to stdout, stderr lines"
+                if not exclude_io
+                else argparse.SUPPRESS
+            ),
+        )
+        parser.add_argument(
+            "--cwd", help="Set job working directory", metavar="DIRECTORY"
+        )
+        parser.add_argument(
+            "--flags",
+            action="append",
+            help="Set comma separated list of job submission flags. Possible "
+            + "flags:  debug, waitable, novalidate",
+            metavar="FLAGS",
+        )
+        parser.add_argument(
+            "--signal",
+            help="Schedule delivery of signal SIG at a defined TIME before "
+            + "job expiration. Default SIG is SIGUSR1, default TIME is 60s.",
+            metavar="[SIG][@TIME]",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Don't actually submit job, just emit jobspec",
+        )
+        parser.add_argument(
+            "--quiet",
+            action="store_true",
+            help="Do not print jobid to stdout on submission",
+        )
+        parser.add_argument(
+            "--debug-emulate", action="store_true", help=argparse.SUPPRESS
+        )
+        return parser
+
+    def init_jobspec(self, args):
+        """
+        Return initialized jobspec. This is an abstract method which must
+        be provided by each base class
+        """
+        raise NotImplementedError()
+
+    def handle_add_file_arg(self, jobspec, arg):
+        """Process a single argument to --add-file=ARG."""
+        perms = None
+        #  Note: Replace any newline escaped by the shell with literal '\n'
+        #  so that newline detection below works for file data passed on
+        #  on the command line:
+        name, _, data = arg.replace("\\n", "\n").partition("=")
+        if not data:
+            # No '=' implies path-only argument (no multiline allowed)
+            if "\n" in name:
+                raise ValueError("--add-file: file name missing")
+            data = name
+            name = basename(data)
+        else:
+            # Check if name specifies permissions after ':'
+            tmpname, _, permstr = name.partition(":")
+            try:
+                perms = int(permstr, base=8)
+                name = tmpname
+            except ValueError:
+                # assume ':' was part of name
+                pass
+        try:
+            jobspec.add_file(name, data, perms=perms)
+        except (TypeError, ValueError, OSError) as exc:
+            raise ValueError(f"--add-file={arg}: {exc}") from None
+
+    # pylint: disable=too-many-branches,too-many-statements
+    def jobspec_create(self, args):
+        """
+        Create a jobspec from args and return it to caller
+        """
+        jobspec = self.init_jobspec(args)
+        jobspec.environment = get_filtered_environment(args.env)
+        jobspec.cwd = args.cwd if args.cwd is not None else os.getcwd()
+        rlimits = get_filtered_rlimits(args.rlimit)
+        if rlimits:
+            jobspec.setattr_shell_option("rlimit", rlimits)
+        if args.signal:
+            entry = parse_signal_option(args.signal)
+            jobspec.setattr_shell_option("signal", entry)
+
+        # --taskmap is only defined for run/submit, but we check
+        # for it in the base jobspec_create() for convenience
+        if hasattr(args, "taskmap") and args.taskmap is not None:
+            jobspec.setattr_shell_option(
+                "taskmap", URIArg(args.taskmap, "taskmap").entry
+            )
+
+        if args.dependency is not None:
+            jobspec.setattr(
+                "system.dependencies", dependency_array_create(args.dependency)
+            )
+        if args.requires is not None:
+            constraint = " ".join(args.requires)
+            try:
+                jobspec.setattr(
+                    "system.constraints", MiniConstraintParser().parse(constraint)
+                )
+            except ConstraintSyntaxError as exc:
+                raise ValueError(f"--requires='{constraint}': {exc}")
+        if args.time_limit is not None:
+            #  With no units, time_limit is in minutes, but jobspec.duration
+            #  takes seconds or FSD by default, so convert here if necessary.
+            try:
+                limit = float(args.time_limit)
+                args.time_limit = limit * 60
+            except ValueError:
+                pass
+            jobspec.duration = args.time_limit
+
+        if args.job_name is not None:
+            jobspec.setattr("system.job.name", args.job_name)
+
+        if args.input is not None:
+            #  Note: temporary stopgap until per-task input is supported
+            #  by the job shell:
+            #  Check if --input specified an IDset. If not, then assume
+            #  a file, otherwise, do not modify jobspec, input will be
+            #  handled by `flux job attach`.
+            try:
+                IDset(args.input)
+            except (ValueError, OSError):
+                jobspec.stdin = args.input
+
+        if args.output is not None and args.output not in ["none", "kvs"]:
+            jobspec.stdout = args.output
+            if args.label_io:
+                jobspec.setattr_shell_option("output.stdout.label", True)
+
+        if args.error is not None:
+            jobspec.stderr = args.error
+            if args.label_io:
+                jobspec.setattr_shell_option("output.stderr.label", True)
+
+        if args.unbuffered:
+            #  For output, set the buffer.type to none and reduce the configured
+            #  event batch-timeout to something very small.
+            jobspec.setattr_shell_option("output.stdout.buffer.type", "none")
+            jobspec.setattr_shell_option("output.stderr.buffer.type", "none")
+            jobspec.setattr_shell_option("output.batch-timeout", 0.05)
+
+        if args.setopt is not None:
+            for keyval in args.setopt:
+                key, val = parse_jobspec_keyval("--setopt", keyval)
+                jobspec.setattr_shell_option(key, val)
+
+        if args.debug_emulate:
+            debugged.set_mpir_being_debugged(1)
+
+        if debugged.get_mpir_being_debugged() == 1:
+            # if stop-tasks-in-exec is present, overwrite
+            jobspec.setattr_shell_option("stop-tasks-in-exec", json.loads("1"))
+
+        if args.queue is not None:
+            jobspec.setattr("system.queue", args.queue)
+
+        if args.bank is not None:
+            jobspec.setattr("system.bank", args.bank)
+
+        if args.setattr is not None:
+            for keyval in args.setattr:
+                key, val = parse_jobspec_keyval("--setattr", keyval)
+
+                #  If key does not explicitly start with ".", "attributes.", "system."
+                #   or "user.", then "system." is implied. This is a
+                #   meant to be a usability enhancement since almost all
+                #   uses of --setattr will target attributes.system.
+                #
+                #  Allow users to set keys at the top level by
+                #   specifying "." before the key name, e.g. the key
+                #   ".foo" sets "attributes.foo".
+                if not key.startswith((".", "attributes.", "user.", "system.")):
+                    key = "system." + key
+                elif key.startswith("."):
+                    key = "attributes" + key
+
+                jobspec.setattr(key, val)
+
+        if args.add_file is not None:
+            for arg in args.add_file:
+                self.handle_add_file_arg(jobspec, arg)
+
+        return jobspec
+
+    def submit_async(self, args, jobspec=None):
+        """
+        Submit job, constructing jobspec from args unless jobspec is not None.
+        Returns a SubmitFuture.
+        """
+        if jobspec is None:
+            jobspec = self.jobspec_create(args)
+
+        if args.dry_run:
+            print(jobspec.dumps(), file=sys.stdout)
+            sys.exit(0)
+
+        arg_debug = False
+        arg_waitable = False
+        arg_novalidate = False
+        if args.flags is not None:
+            for tmp in args.flags:
+                for flag in tmp.split(","):
+                    if flag == "debug":
+                        arg_debug = True
+                    elif flag == "waitable":
+                        arg_waitable = True
+                    elif flag == "novalidate":
+                        arg_novalidate = True
+                    else:
+                        raise ValueError("--flags: Unknown flag " + flag)
+
+        if not self.flux_handle:
+            self.flux_handle = flux.Flux()
+
+        if args.urgency == "default":
+            urgency = flux.constants.FLUX_JOB_URGENCY_DEFAULT
+        elif args.urgency == "hold":
+            urgency = flux.constants.FLUX_JOB_URGENCY_HOLD
+        elif args.urgency == "expedite":
+            urgency = flux.constants.FLUX_JOB_URGENCY_EXPEDITE
+        else:
+            urgency = int(args.urgency)
+
+        return job.submit_async(
+            self.flux_handle,
+            jobspec.dumps(),
+            urgency=urgency,
+            waitable=arg_waitable,
+            debug=arg_debug,
+            novalidate=arg_novalidate,
+        )
+
+    def submit(self, args, jobspec=None):
+        return self.submit_async(args, jobspec).get_id()
+
+    def get_parser(self):
+        return self.parser
+
+
+class SubmitBaseCmd(MiniCmd):
+    """
+    SubmitBaseCmd is an abstract class with shared code for job submission
+    """
+
+    def __init__(self, prog, usage=None, description=None):
+        super().__init__(prog, usage, description)
+        self.parser.add_argument(
+            "--taskmap",
+            type=str,
+            help="Select the scheme for mapping task ids to nodes as a URI "
+            + "(i.e. SCHEME[:VALUE]). Value options include block, cyclic, "
+            + "cyclic:N, or manual:TASKMAP (default: block)",
+            metavar="URI",
+        )
+        group = self.parser.add_argument_group("Common resource options")
+        group.add_argument(
+            "-N", "--nodes", metavar="N", help="Number of nodes to allocate"
+        )
+        group.add_argument(
+            "-x",
+            "--exclusive",
+            action="store_true",
+            help="With -N, --nodes, allocate nodes exclusively",
+        )
+        group = self.parser.add_argument_group(
+            "Per task options",
+            "The following options allow per-task specification of resources, "
+            + "and should not be combined with per-resource options.",
+        )
+        group.add_argument(
+            "-n",
+            "--ntasks",
+            metavar="N",
+            help="Number of tasks to start",
+        )
+        group.add_argument(
+            "-c",
+            "--cores-per-task",
+            metavar="N",
+            help="Number of cores to allocate per task",
+        )
+        group.add_argument(
+            "-g",
+            "--gpus-per-task",
+            metavar="N",
+            help="Number of GPUs to allocate per task",
+        )
+        group = self.parser.add_argument_group(
+            "Per resource options",
+            "The following options allow per-resource specification of "
+            + "tasks, and should not be used with per-task options above",
+        )
+        group.add_argument(
+            "--cores",
+            metavar="N",
+            help="Request a total number of cores",
+        )
+        group.add_argument(
+            "--tasks-per-node",
+            metavar="N",
+            help="Force number of tasks per node",
+        )
+        group.add_argument(
+            "--tasks-per-core",
+            metavar="N",
+            help="Force number of tasks per core",
+        )
+        group.add_argument(
+            "--gpus-per-node",
+            metavar="N",
+            help="Request a number of GPUs per node with --nodes",
+        )
+        self.parser.add_argument(
+            "-v",
+            "--verbose",
+            action="count",
+            default=0,
+            help="Increase verbosity on stderr (multiple use OK)",
+        )
+
+    # pylint: disable=too-many-branches
+    def init_jobspec(self, args):
+        per_resource_type = None
+        per_resource_count = None
+
+        if not args.command:
+            raise ValueError("job command and arguments are missing")
+
+        #  Remove first -- from command in case user used it to separate
+        #  Flux cli options from command and options
+        if args.command[0] == "--":
+            args.command.pop(0)
+
+        #  Ensure integer args are converted to int() here.
+        #  This is done because we do not use type=int in argparse in order
+        #   to allow these options to be mutable for bulksubmit:
+        #
+        for arg in [
+            "ntasks",
+            "nodes",
+            "cores",
+            "cores_per_task",
+            "gpus_per_task",
+            "tasks_per_node",
+            "tasks_per_core",
+            "gpus_per_node",
+        ]:
+            value = getattr(args, arg)
+            if value:
+                try:
+                    setattr(args, arg, int(value))
+                except ValueError:
+                    opt = arg.replace("_", "-")
+                    raise ValueError(f"--{opt}: invalid int value '{value}'")
+
+        if args.tasks_per_node is not None and args.tasks_per_core is not None:
+            raise ValueError(
+                "Do not specify both the number of tasks per node and per core"
+            )
+
+        #  Handle --tasks-per-node or --tasks-per-core (it is an error to
+        #   specify both). Check options for validity and assign the
+        #   per_resource variable when valid.
+        #
+        if args.tasks_per_node is not None or args.tasks_per_core is not None:
+            if args.tasks_per_node is not None:
+                if args.tasks_per_node < 1:
+                    raise ValueError("--tasks-per-node must be >= 1")
+
+                per_resource_type = "node"
+                per_resource_count = args.tasks_per_node
+            elif args.tasks_per_core is not None:
+                if args.tasks_per_core < 1:
+                    raise ValueError("--tasks-per-core must be >= 1")
+                per_resource_type = "core"
+                per_resource_count = args.tasks_per_core
+
+        if args.gpus_per_node:
+            if not args.nodes:
+                raise ValueError("--gpus-per-node requires --nodes")
+
+        #  If any of --tasks-per-node, --tasks-per-core, --cores, or
+        #   --gpus-per-node is used, then use the per_resource constructor:
+        #
+        if (
+            per_resource_type is not None
+            or args.gpus_per_node is not None
+            or args.cores is not None
+        ):
+            #  If any of the per-task options was also specified, raise an
+            #   error here instead of silently ignoring those options:
+            if (
+                args.ntasks is not None
+                or args.cores_per_task is not None
+                or args.gpus_per_task
+            ):
+                raise ValueError(
+                    "Per-resource options can't be used with per-task options."
+                    + " (See --help for details)"
+                )
+
+            #  In per-resource mode, set the exclusive flag if nodes is
+            #   specified without cores. This preserves the default behavior
+            #   of requesting nodes exclusively when only -N is used:
+            if args.nodes and args.cores is None:
+                args.exclusive = True
+
+            return JobspecV1.per_resource(
+                args.command,
+                ncores=args.cores,
+                nnodes=args.nodes,
+                per_resource_type=per_resource_type,
+                per_resource_count=per_resource_count,
+                gpus_per_node=args.gpus_per_node,
+                exclusive=args.exclusive,
+            )
+
+        #  If ntasks not set, then set it to node count, with
+        #   exclusive flag enabled
+        if not args.ntasks and args.nodes:
+            args.ntasks = args.nodes
+            args.exclusive = True
+
+        # O/w default ntasks for from_command() is 1:
+        if not args.ntasks:
+            args.ntasks = 1
+
+        # default cores_per_task for from_command() is 1:
+        if not args.cores_per_task:
+            args.cores_per_task = 1
+
+        return JobspecV1.from_command(
+            args.command,
+            num_tasks=args.ntasks,
+            cores_per_task=args.cores_per_task,
+            gpus_per_task=args.gpus_per_task,
+            num_nodes=args.nodes,
+            exclusive=args.exclusive,
+        )
+
+    def run_and_exit(self):
+        self.flux_handle.reactor_run()
+        if self.watcher:
+            self.exitcode = max(self.watcher.exitcode, self.exitcode)
+        sys.exit(self.exitcode)
+
+
+class SubmitBulkCmd(SubmitBaseCmd):
+    """
+    SubmitBulkCmd adds options for submitting copies of jobs,
+    watching progress of submission, and waiting for job completion
+    to the SubmitBaseCmd class
+    """
+
+    def __init__(self, prog, usage=None, description=None):
+        #  dictionary of open logfiles for --log, --log-stderr:
+        self._logfiles = {}
+        self.t0 = None
+
+        super().__init__(prog, usage, description)
+        self.parser.add_argument(
+            "--cc",
+            metavar="IDSET",
+            default=None,
+            help="Replicate job for each ID in IDSET. "
+            "(FLUX_JOB_CC=ID will be set for each job submitted)",
+        )
+        self.parser.add_argument(
+            "--bcc",
+            metavar="IDSET",
+            default=None,
+            help="Like --cc, but FLUX_JOB_CC is not set",
+        )
+        self.parser.add_argument(
+            "--wait-event",
+            metavar="NAME",
+            dest="wait",
+            help="Wait for event NAME for all jobs after submission",
+        )
+        self.parser.add_argument(
+            "--wait",
+            action="store_const",
+            const="clean",
+            help="Wait for all jobs to complete after submission "
+            "(same as --wait-event=clean)",
+        )
+        self.parser.add_argument(
+            "--watch",
+            action="store_true",
+            help="Watch all job output (implies --wait)",
+        )
+        self.parser.add_argument(
+            "--log",
+            metavar="FILE",
+            help="Print program log messages (e.g. submitted jobid) to FILE "
+            "instead of terminal",
+        )
+        self.parser.add_argument(
+            "--log-stderr",
+            metavar="FILE",
+            help="Separate stderr messages into FILE instead of terminal or "
+            "logfile destination",
+        )
+        self.parser.add_argument(
+            "--progress",
+            action="store_true",
+            help="Show progress of job submission or completion (with --wait)",
+        )
+        self.parser.add_argument(
+            "--jps",
+            action="store_true",
+            help="With --progress, show job throughput",
+        )
+
+    def submit_cb(self, future, args, label=""):
+        try:
+            jobid = future.get_id()
+            if not args.quiet:
+                print(jobid, file=args.stdout)
+        except OSError as exc:
+            print(f"{label}{exc}", file=args.stderr)
+            self.exitcode = 1
+            self.progress_update(submit_failed=True)
+            return
+
+        if self.watcher:
+            self.watcher.add_jobid(jobid, args.stdout, args.stderr, args.wait)
+        elif self.progress:
+            #  Update progress of submission only
+            self.progress.update(jps=self.jobs_per_sec())
+
+    def _progress_check(self, args):
+        if args.progress and not self.progress and not sys.stdout.isatty():
+            LOGGER.warning("stdout is not a tty. Ignoring --progress option")
+            args.progress = None
+
+    def watcher_start(self, args):
+        if not self.watcher:
+            #  Need to open self.flux_handle if it isn't already in order
+            #  to start the watcher
+            if not self.flux_handle:
+                self.flux_handle = flux.Flux()
+
+            self._progress_check(args)
+
+            self.watcher = JobWatcher(
+                self.flux_handle,
+                progress=args.progress,
+                jps=args.jps,
+                log_events=(args.verbose > 1),
+                log_status=(args.verbose > 0),
+                labelio=args.label_io,
+                wait=args.wait,
+                watch=args.watch,
+            ).start()
+
+    def jobs_per_sec(self):
+        return (self.progress.count + 1) / self.progress.elapsed
+
+    def progress_start(self, args, total):
+        """
+        Initialize job submission progress bar if user requested --progress
+        without --wait or --watch
+        """
+        self._progress_check(args)
+        if not args.progress or self.progress:
+            # progress bar not requested or already started
+            return
+
+        if args.wait or args.watch:
+            # progress handled in JobWatcher class
+            return
+
+        before = "Submitting {total} jobs: "
+        after = "{percent:5.1f}% {elapsed.dt}"
+        if args.jps:
+            after = "{percent:5.1f}% {jps:4.1f} job/s"
+        self.progress = ProgressBar(
+            timer=False,
+            total=total,
+            width=len(str(total)),
+            before=before,
+            after=after,
+            pending=0,
+            fail=0,
+            jps=0,
+        ).start()
+
+    def progress_update(self, jobinfo=None, submit=False, submit_failed=False):
+        """
+        Update submission progress bar if one was requested
+        """
+        if not self.progress:
+            return
+
+        if not self.progress.timer:
+            #  Start a timer to update progress bar without other events
+            #  (we have to do it here since a flux handle does not exist
+            #   in progress_start). We use 250ms to make the progress bar
+            #   more fluid.
+            timer = self.flux_handle.timer_watcher_create(
+                0, lambda *x: self.progress.redraw(), repeat=0.25
+            ).start()
+            self.progress.update(advance=0, timer=timer)
+
+            #  Don't let this timer watcher contribute to the reactor's
+            #   "active" reference count:
+            self.flux_handle.reactor_decref()
+
+        if submit:
+            self.progress.update(
+                advance=0,
+                pending=self.progress.pending + 1,
+            )
+        elif submit_failed:
+            self.progress.update(
+                advance=1,
+                pending=self.progress.pending - 1,
+                fail=self.progress.fail + 1,
+                jps=self.jobs_per_sec(),
+            )
+
+    @staticmethod
+    def cc_list(args):
+        """
+        Return a list of values representing job copies given by --cc/--bcc
+        """
+        cclist = [""]
+        if args.cc and args.bcc:
+            raise ValueError("specify only one of --cc or --bcc")
+        if args.cc:
+            cclist = IDset(args.cc)
+        elif args.bcc:
+            cclist = IDset(args.bcc)
+        return cclist
+
+    def openlog(self, filename):
+        if filename not in self._logfiles:
+            filep = open(filename, "w", buffering=1)
+            atexit.register(lambda x: x.close(), filep)
+            self._logfiles[filename] = filep
+        return self._logfiles[filename]
+
+    def submit_async_with_cc(self, args, cclist=None):
+        """
+        Asynchronously submit jobs, optionally submitting a copy of
+        each job for each member of a cc-list. If the cclist is not
+        passed in to the method, then one is created from either
+        --cc or --bcc options.
+        """
+        if not cclist:
+            cclist = self.cc_list(args)
+        label = ""
+
+        #  Save default stdout/err location in args so it can be overridden
+        #   by --log and --log-stderr and the correct location is available
+        #   in each job's callback chain:
+        #
+        args.stdout = sys.stdout
+        args.stderr = sys.stderr
+
+        if args.watch or args.wait:
+            self.watcher_start(args)
+
+        elif args.progress:
+            self.progress_start(args, len(cclist))
+
+        for i in cclist:
+            #  substitute any {cc} in args (only if --cc or --bcc):
+            xargs = Xcmd(args, cc=i) if isinstance(i, int) else args
+            jobspec = self.jobspec_create(xargs)
+
+            #  For now, an idset argument to args.input is not supported
+            #  in submit:
+            if xargs.input:
+                try:
+                    IDset(xargs.input)
+                    LOGGER.error("per-task input not supported for submit")
+                    sys.exit(1)
+                except (ValueError, OSError):
+                    # --input was not an idset, just continue:
+                    pass
+
+            if args.cc or args.bcc:
+                label = f"cc={i}: "
+                if not args.bcc:
+                    jobspec.environment["FLUX_JOB_CC"] = str(i)
+
+            #  Check for request to redirect program stdout/err
+            #  By default, --log redirects both stdout and stderr
+            #  (We explicitly don't want these attributes defined in
+            #   __init__, o/w we won't fall back to parent args, so
+            #   disable pylint warning)
+            #  pylint: disable=attribute-defined-outside-init
+            if xargs.log:
+                xargs.stdout = self.openlog(xargs.log)
+                xargs.stderr = xargs.stdout
+            if xargs.log_stderr:
+                xargs.stderr = self.openlog(xargs.log_stderr)
+
+            self.submit_async(xargs, jobspec).then(self.submit_cb, xargs, label)
+
+    def main(self, args):
+        self.submit_async_with_cc(args)
+        self.run_and_exit()
+
+
+def add_batch_alloc_args(parser):
+    """
+    Add "batch"-specific resource allocation arguments to parser object
+    which deal in slots instead of tasks.
+    """
+    parser.add_argument(
+        "--conf",
+        metavar="CONF",
+        default=BatchConfig(),
+        action=ConfAction,
+        help="Set configuration for a child Flux instance. CONF may be a "
+        + "multiline string in JSON or TOML, a configuration key=value, a "
+        + "path to a JSON or TOML file, or a configuration loaded by name "
+        + "from a standard search path. This option may specified multiple "
+        + "times, in which case the config is iteratively updated.",
+    )
+    parser.add_argument(
+        "--broker-opts",
+        metavar="OPTS",
+        default=None,
+        action="append",
+        help="Pass options to flux brokers",
+    )
+    parser.add_argument(
+        "--dump",
+        nargs="?",
+        const="flux-{{jobid}}-dump.tgz",
+        metavar="FILE",
+        help="Archive KVS on exit",
+    )
+    parser.add_argument(
+        "-n",
+        "--nslots",
+        type=int,
+        metavar="N",
+        help="Number of total resource slots requested."
+        + " The size of a resource slot may be specified via the"
+        + " -c, --cores-per-slot and -g, --gpus-per-slot options."
+        + " The default slot size is 1 core.",
+    )
+    parser.add_argument(
+        "-c",
+        "--cores-per-slot",
+        type=int,
+        metavar="N",
+        default=1,
+        help="Number of cores to allocate per slot",
+    )
+    parser.add_argument(
+        "-g",
+        "--gpus-per-slot",
+        type=int,
+        metavar="N",
+        help="Number of GPUs to allocate per slot",
+    )
+    parser.add_argument(
+        "-N",
+        "--nodes",
+        type=int,
+        metavar="N",
+        help="Distribute allocated resource slots across N individual nodes",
+    )
+    parser.add_argument(
+        "-x",
+        "--exclusive",
+        action="store_true",
+        help="With -N, --nodes, allocate nodes exclusively",
+    )
diff --git a/src/bindings/python/flux/cli/batch.py b/src/bindings/python/flux/cli/batch.py
new file mode 100644
index 000000000000..2405b3364f9f
--- /dev/null
+++ b/src/bindings/python/flux/cli/batch.py
@@ -0,0 +1,149 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+import sys
+
+import flux
+import flux.job
+from flux.cli import base
+from flux.job.directives import DirectiveParser
+
+LOGGER = logging.getLogger("flux-batch")
+
+
+class BatchCmd(base.MiniCmd):
+    def __init__(self, prog, usage=None, description=None):
+        super().__init__(prog, usage, description)
+        self.parser.add_argument(
+            "--wrap",
+            action="store_true",
+            help="Wrap arguments or stdin in a /bin/sh script",
+        )
+        base.add_batch_alloc_args(self.parser)
+        self.parser.add_argument(
+            "SCRIPT",
+            nargs=argparse.REMAINDER,
+            help="Batch script and arguments to submit",
+        )
+
+    def parse_directive_args(self, name, batchscript):
+        """
+        Parse any directives in batchscript.directives, then apply
+        command line arguments in self.argv. This allows command line
+        to override file directives
+        """
+        args = None
+        for item in batchscript.directives:
+            try:
+                if item.action == "SETARGS":
+                    args = self.parser.parse_args(item.args, namespace=args)
+            except SystemExit:
+                #  Argparse exits on error. Give the user a clue
+                #  about which line failed in the source file:
+                LOGGER.error(f"argument parsing failed at {name} line {item.lineno}")
+                sys.exit(2)
+        args = self.parser.parse_args(self.argv, namespace=args)
+        return batchscript.script, args
+
+    def process_script(self, args):
+        """
+        Process a batch script that may contain RFC 36 directives.
+        Returns the ingested script and new argparse args Namespace.
+        """
+        if args.SCRIPT:
+            # Remove leading "--" in case it was used to separate flux-batch(1)
+            # options from user script and options:
+            if args.SCRIPT[0] == "--":
+                args.SCRIPT.pop(0)
+
+            if args.wrap:
+                #  Return script which will be wrapped by caller
+                return " ".join(args.SCRIPT) + "\n", args
+
+            # O/w, open script for reading
+            name = open_arg = args.SCRIPT[0]
+        else:
+            name = "stdin"
+            open_arg = 0  # when passed to `open`, 0 gives the `stdin` stream
+        with open(open_arg, "r", encoding="utf-8") as filep:
+            try:
+                batchscript = DirectiveParser(filep)
+            except UnicodeError:
+                raise ValueError(
+                    f"{name} does not appear to be a script, "
+                    "or failed to encode as utf-8"
+                )
+            except ValueError as exc:
+                raise ValueError(f"{name}: {exc}") from None
+        return self.parse_directive_args(name, batchscript)
+
+    def init_jobspec(self, args):
+        if args.wrap:
+            self.script = f"#!/bin/sh\n{self.script}"
+
+        #  If number of slots not specified, then set it to node count
+        #   if set, otherwise raise an error.
+        if not args.nslots:
+            if not args.nodes:
+                raise ValueError("Number of slots to allocate must be specified")
+            args.nslots = args.nodes
+            args.exclusive = True
+
+        if args.dump:
+            args.broker_opts = args.broker_opts or []
+            args.broker_opts.append("-Scontent.dump=" + args.dump)
+
+        #  If job name is not explicitly set in args, use the script name
+        #   if a script was provided, else the string "batch" to
+        #   indicate the script was set on flux batch stdin.
+        if args.job_name is None:
+            if args.SCRIPT:
+                args.job_name = args.SCRIPT[0]
+            else:
+                args.job_name = "batch"
+
+        jobspec = flux.job.JobspecV1.from_batch_command(
+            script=self.script,
+            jobname=args.job_name,
+            args=args.SCRIPT[1:],
+            num_slots=args.nslots,
+            cores_per_slot=args.cores_per_slot,
+            gpus_per_slot=args.gpus_per_slot,
+            num_nodes=args.nodes,
+            broker_opts=base.list_split(args.broker_opts),
+            exclusive=args.exclusive,
+            conf=args.conf.config,
+        )
+
+        # Default output is flux-{{jobid}}.out
+        # overridden by either --output=none or --output=kvs
+        if not args.output:
+            jobspec.stdout = "flux-{{id}}.out"
+        return jobspec
+
+    def main(self, args):
+        #  Save cmdline argv to flux-batch in case it must be reprocessed
+        #  after applying directive options.
+        #  self.argv is sys.argv without flux-batch:
+        self.argv = sys.argv[1:]
+        if self.argv and self.argv[0] == "batch":
+            self.argv.pop(0)
+
+        #  Process file with possible submission directives, returning
+        #  script and new argparse args Namespace as a result.
+        #  This must be done before calling self.submit() so that SETARGS
+        #  directives are available in jobspec_create():
+        self.script, args = self.process_script(args)
+
+        jobid = self.submit(args)
+        if not args.quiet:
+            print(jobid, file=sys.stdout)
diff --git a/src/bindings/python/flux/cli/bulksubmit.py b/src/bindings/python/flux/cli/bulksubmit.py
new file mode 100644
index 000000000000..97e29e22cbd3
--- /dev/null
+++ b/src/bindings/python/flux/cli/bulksubmit.py
@@ -0,0 +1,211 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import itertools
+import random
+import sys
+
+from flux.cli import base
+
+
+class BulkSubmitCmd(base.SubmitBulkCmd):
+    """
+    BulkSubmitCmd is like xargs for job submission. It takes a series of
+    inputs on stdin (or the cmdline separated by :::), and substitutes them
+    into the initial arguments, e.g::
+
+       $ echo 1 2 3 | flux bulksubmit echo {}
+
+    """
+
+    def __init__(self, prog, usage=None, description=None):
+        super().__init__(prog, usage, description)
+        self.parser.add_argument(
+            "--shuffle",
+            action="store_true",
+            help="Shuffle list of commands before submission",
+        )
+        self.parser.add_argument(
+            "--sep",
+            type=str,
+            metavar="STRING",
+            default="\n",
+            help="Set the input argument separator. To split on whitespace, "
+            "use --sep=none. The default is newline.",
+        )
+        self.parser.add_argument(
+            "--define",
+            action="append",
+            type=lambda kv: kv.split("="),
+            dest="methods",
+            default=[],
+            help="Define a named method for transforming any input, "
+            "accessible via e.g. '{0.NAME}'. (local variable 'x' "
+            "will contain the input string to be transformed)",
+            metavar="NAME=CODE",
+        )
+        self.parser.add_argument(
+            "command",
+            nargs=argparse.REMAINDER,
+            help="Job command and initial arguments",
+        )
+
+    @staticmethod
+    def input_file(filep, sep):
+        """Read set of inputs from file object filep, using separator sep"""
+        return list(filter(None, filep.read().split(sep)))
+
+    @staticmethod
+    def split_before(iterable, pred):
+        """
+        Like more_itertools.split_before, but if predicate returns
+        True on first element, then return an empty list
+        """
+        buf = []
+        for item in iter(iterable):
+            if pred(item):
+                yield buf
+                buf = []
+            buf.append(item)
+        yield buf
+
+    def split_command_inputs(self, command, sep="\n", delim=":::"):
+        """Generate a list of inputs from command list
+
+        Splits the command list on the input delimiter ``delim``,
+        and returns 3 lists:
+
+            - the initial command list (everything before the first delim)
+            - a list of normal "input lists"
+            - a list of "linked" input lists, (delim + "+")
+
+        Special case delimiter values are handled here,
+        e.g. ":::+" and ::::".
+
+        """
+        links = []
+
+        #  Split command array into commands and inputs on ":::":
+        command, *input_lists = self.split_before(
+            command, lambda x: x.startswith(delim)
+        )
+
+        #  Remove ':::' separators from each input, allowing GNU parallel
+        #   like alternate separators e.g. '::::' and ':::+':
+        #
+        for i, lst in enumerate(input_lists):
+            first = lst.pop(0)
+            if first == delim:
+                #  Normal input
+                continue
+            if first == delim + ":":
+                #
+                #  Read input from file
+                if len(lst) > 1:
+                    raise ValueError("Multiple args not allowed after ::::")
+                if lst[0] == "-":
+                    input_lists[i] = self.input_file(sys.stdin, sep)
+                else:
+                    with open(lst[0]) as filep:
+                        input_lists[i] = self.input_file(filep, sep)
+            if first in (delim + "+", delim + ":+"):
+                #
+                #  "Link" input to previous, similar to GNU parallel:
+                #  Clear list so this entry can be removed below after
+                #   iteration is complete:
+                links.append({"index": i, "list": lst.copy()})
+                lst.clear()
+
+        #  Remove empty lists (which are now links)
+        input_lists = [lst for lst in input_lists if lst]
+
+        return command, input_lists, links
+
+    def create_commands(self, args):
+        """Create bulksubmit commands list"""
+
+        #  Expand any escape sequences in args.sep, and replace "none"
+        #   with literal None:
+        sep = bytes(args.sep, "utf-8").decode("unicode_escape")
+        if sep.lower() == "none":
+            sep = None
+
+        #  Ensure any provided methods can compile
+        args.methods = {
+            name: compile(code, name, "eval")
+            for name, code in dict(args.methods).items()
+        }
+
+        #  Split command into command template and input lists + links:
+        args.command, input_list, links = self.split_command_inputs(
+            args.command, sep, delim=":::"
+        )
+
+        #  If no command provided then the default is "{}"
+        if not args.command:
+            args.command = ["{}"]
+
+        #  If no inputs on commandline, read from stdin:
+        if not input_list:
+            input_list = [self.input_file(sys.stdin, sep)]
+
+        #  Take the product of all inputs in input_list
+        inputs = [list(x) for x in list(itertools.product(*input_list))]
+
+        #  Now cycle over linked inputs and insert them in result:
+        for link in links:
+            cycle = itertools.cycle(link["list"])
+            for lst in inputs:
+                lst.insert(link["index"], next(cycle))
+
+        #  For each set of generated input lists, append a command
+        #   to run. Keep a sequence counter so that {seq} can be used
+        #   in the format expansion.
+        return [
+            base.Xcmd(args, inp, seq=i, seq1=i + 1, cc="{cc}")
+            for i, inp in enumerate(inputs)
+        ]
+
+    def main(self, args):
+        if not args.command:
+            args.command = ["{}"]
+
+        #  Create one "command" to be run for each combination of
+        #   "inputs" on stdin or the command line:
+        #
+        commands = self.create_commands(args)
+
+        if args.shuffle:
+            random.shuffle(commands)
+
+        #  Calculate total number of commands for use with progress:
+        total = 0
+        for xargs in commands:
+            total += len(self.cc_list(xargs))
+
+        if total == 0:
+            raise ValueError("no jobs provided for bulk submission")
+        #  Initialize progress bar if requested:
+        if args.progress:
+            if not args.dry_run:
+                self.progress_start(args, total)
+            else:
+                print(f"bulksubmit: submitting a total of {total} jobs")
+
+        #  Loop through commands and asynchronously submit them:
+        for xargs in commands:
+            if args.verbose or args.dry_run:
+                print(f"bulksubmit: submit {xargs}")
+            if not args.dry_run:
+                self.submit_async_with_cc(xargs)
+
+        if not args.dry_run:
+            self.run_and_exit()
diff --git a/src/bindings/python/flux/cli/fortune.py b/src/bindings/python/flux/cli/fortune.py
new file mode 100644
index 000000000000..b34b12852430
--- /dev/null
+++ b/src/bindings/python/flux/cli/fortune.py
@@ -0,0 +1,366 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import random
+import sys
+from datetime import datetime
+
+import flux.util
+from flux.cli import base
+
+# Choice of decorating symbols
+symbols = ["@", "*", "**", "!", "$", "%", "^", "O", "o", "|", "x", "8", "*", "{*}", "-"]
+
+# Choices of colors to print
+colors = [
+    "\033[91m %s %s %s\033[00m",  # red
+    "\033[92m %s %s %s\033[00m",  # green
+    "\033[93m %s %s %s\033[00m",  # yellow
+    "\033[95m %s %s %s\033[00m",  # magenta
+    "\033[94m %s %s %s\033[00m",  # blue
+    "\033[96m %s %s %s\033[00m",  # cyan
+    "\033[97m %s %s %s\033[00m",
+]  # gray
+
+
+class FortuneCmd(base.MiniCmd):
+    """
+    Surprise the user with some beautiful, hidden Flux fortunes and art!
+
+    Usage: flux fortune
+
+    flux fortune -c all            # this is the default
+    flux fortune -c valentines     # show valentine fortune
+    flux fortune -c art            # show art
+    flux fortune -c facts          # show learning facts / tidbits
+    flux fortune -c fun            # show fun fortune
+    """
+
+    @staticmethod
+    def create_parser(
+        prog, usage=None, description=None, exclude_io=False, add_help=True
+    ):
+        """
+        Create a largely empty parser for flux fortune (no arguments or exposed)
+        """
+        if usage is None:
+            usage = f"{prog} [OPTIONS...] COMMAND [ARGS...]"
+
+        parser = argparse.ArgumentParser(
+            prog=prog,
+            usage=usage,
+            description=description,
+            formatter_class=flux.util.help_formatter(),
+        )
+        parser.add_argument(
+            "-c",
+            "--category",
+            choices=["all", "valentines", "fun", "facts", "art"],
+            default="all",
+            help="Choose the category of fortunes to display.",
+        )
+        return parser
+
+    def generate_fortune(self, args):
+        """
+        Generate the fortune, meaning:
+
+        1. Choose to print a fortune (a) or the (rare) ascii art.
+        2. If a, choose a color and print.
+        3. If b, print the ascii and exit.
+        """
+        # Derive fortune based on category
+        if args.category == "all":
+            return self.random_fortune()
+
+        # Request for facts
+        if args.category == "facts":
+            return self.show_fortune(facts)
+
+        # Request for ascii art
+        if args.category == "art":
+            return self.show_art()
+
+        # Request for a valentine
+        if args.category == "valentines":
+            return self.show_fortune(valentines)
+
+        # Otherwise show a fun fortune
+        self.show_fortune(fortunes)
+
+    def show_art(self):
+        """
+        Show ascii art.
+        """
+        print(random.choice(art))
+
+    def random_fortune(self):
+        """
+        A random fortune can be art, fun, valenties, or factoid.
+        """
+        # 1% chance to print ascii art, no matter what
+        if random.uniform(0, 1) <= 0.01:
+            return self.show_art()
+
+        # Beyond that, 80% of the time is fact
+        if random.uniform(0, 1) <= 0.80:
+            return self.show_fortune(facts)
+
+        # Otherwise we show fun or valentines
+        # If it's within 3 weeks of Valentines...
+        self.check_valentines()
+        self.show_fortune(fortunes)
+
+    def show_fortune(self, listing):
+        """
+        Randomly select a fortune from a list and colorize.
+        """
+        # Otherwise, choose a color and a fortune...
+        color = random.choice(colors)
+        s = random.choice(symbols)
+        fortune = random.choice(listing)
+        print(color % (s, fortune, s))
+
+    def check_valentines(self):
+        """
+        Check if we are within a few weeks of Valentine's Day
+        """
+        global fortunes
+        global valentines
+        now = datetime.now()
+
+        # End of January or start of February
+        is_soon = (now.month == 1 and now.day > 29) or (
+            now.month == 2 and now.day <= 14
+        )
+        if not is_soon:
+            return
+
+        fortunes += valentines
+
+    def main(self, args):
+        self.generate_fortune(args)
+        sys.exit(self.exitcode)
+
+
+# Valentines fortunes (2 weeks up to Valentine's day)
+valentines = [
+    "Roses are red, violets are blue, if you want graph-based scheduling, Flux is for you! <3",
+    "Roses are red, violets are blue, all of my jobs, submit to you! <3",
+    "Roses are red, violets are blue, you are my favorite job manager queue! <3",
+    """
+Sung to the tune of Monty Python's "Bruce's Philosopher Song":
+
+Slurm, Slurm, on this I'm firm:
+your interface resembles the excretions of a worm
+Loadleveler took forever-er
+but waiting for P-O-E gave you time to pee
+jsrun is no j-s-fun
+but we paid a lot for a system and so they gave us one
+And of Condor we couldn't be fonder -
+when run right, it kept the professor's office warm all night
+""",
+    """
+Oh Flux, my darling, my heart doth flutter
+With every submit you help me utter
+Your unidirectional flow is so sublime
+It simplifies my code and saves me time
+
+My love for you is like an unchanging store
+Always at the ready, forevermore
+Your Python bindings make me swoon
+Together, we make such a lovely tune
+
+Oh Flux, you make my programming heart sing
+Your architecture is a beautiful thing
+With you by my side, I can conquer all
+Happy Valentine's Day, my sweet, Flux, my all.
+
+Citation:
+OpenAI, 2023, Feb. 13, ChatGPT response to the prompt
+"Write a valentine's day poem about flux framework that rhymes".
+https://chat.openai.com
+""",
+]
+
+# Facts about Flux
+facts = [
+    """
+Flux can be started as a parallel job.  This is how flux-batch(1) and
+flux-alloc(1) work!  Within one of these "subinstances" of Flux, you
+(or your batch script) have access all the features of Flux without bothering
+the system or parent Flux instance.  Do your worst - it is your personal
+sandbox!
+""",
+    """
+Other resource managers and MPI launchers can start Flux instances the
+same way they launch MPI.  This is why workflows "coded to Flux" are
+portable to many environments.
+""",
+    """
+flux-submit(1) and flux-run(1) both start one parallel program.  The difference
+is flux-submit(1) prints the job ID and exits immediately, while flux-run(1)
+doesn't exit until the job has completed.
+""",
+    """
+All jobs run within a Flux subinstance started by flux-batch(1) or
+flux-alloc(1) look like *one* job to the Flux accounting system.
+""",
+    """
+The system prolog/epilog do not run between jobs in a Flux subinstance
+started by flux-batch(1) or flux-alloc(1).
+""",
+    """
+A Flux subinstance started by flux-batch(1) or flux-alloc(1) has the same
+capabilities as the system level Flux instance.  Unlike legacy systems that
+offer a simpler "step scheduler" in batch jobs, the Flux subinstance is an
+identical copy of Flux running on a resource subset.  This has been called
+"fractal scheduling".
+""",
+    """
+So Flux can start Flux, and the second Flux is identical to the first Flux,
+so can *that* Flux start Flux?
+
+"It's turtles all the way down." --Dong Ahn
+""",
+    """
+flux-top(1) lets you navigate with arrow keys to your batch jobs or allocations
+and display the jobs running within them.  Since you can nest Flux as deep
+as you like, you can think of flux-top(1) as a browser for the job hieararchy.
+""",
+    """
+Flux commands look for an environment variable FLUX_URI to determine which
+Flux instance to connect to.  If it's not set, they try to connect to the
+system instance of Flux.
+""",
+    """
+flux-proxy(1) establishes a connection to a remote Flux instance, then
+starts a shell in which Flux commands refer to that instance.  The connection
+is dropped when the shell exits.
+""",
+    """
+flux-uri(1) can resolve a jobid to its remote URI.  It can even resolve
+Slurm and LSF job IDs when Flux is running in a non-native environment.
+""",
+    """
+flux-overlay(1) can pretty-print a view of the Flux overlay network in your
+batch job or allocation. Try this:
+
+$ flux alloc -N16 --broker-opts=-Stbon.topo=kary:4
+$ flux overlay status
+$ flux alloc -N16 --broker-opts=-Stbon.topo=binomial
+$ flux overlay status
+$ flux alloc -N16 --broker-opts=-Stbon.topo=kary:1
+$ flux overlay status
+$ exit
+$ exit
+$ exit
+
+You just ran three Flux instances nested like Matryoshka dolls, each with a
+unique overlay network topology!
+""",
+    """
+To get a quick summary of the resources available within a Flux instance:
+   $ flux resource info
+   16 Nodes, 96 Cores, 16 GPUs
+""",
+    """
+To peruse Flux documentation, visit https://flux-framework.readthedocs.io
+""",
+    """
+flux-version(1) reports the version of flux-core that you're running.
+Visit https://flux-framework.org/releases/ for release notes.
+For each release notes item, you'll find a link to the corresponding pull
+request (change proposal) on github.
+""",
+]
+
+# Fun fortunes
+fortunes = [
+    "I refuse to have modem speeds without the sweet modem sounds",
+    "A yawn is a silent scream for coffee.",
+    "Due to a shortage of robots, our staff is composed of humans and may react unpredictably when abused.",
+    "Dear <software>, stop doing <that>. Thank you. Best, <developer>",
+    "Dear Flux, I don't need your bad jokes. Thanks. Best, <flux-user>",
+    "The best kind of HPC system I can imagine would smell like cookies. Forever.",
+    "From __future__ import fluxisthebest",
+    "Help me, I'm trapped in a container!",
+    "You don’t need to worry about getting older when you’re a robot… it’s just a one digit progression in your time-stamp.",
+    "I reached for my mouse... grabbed an avocado instead.",
+    "One could predict supercomputer age based on bug accumulation.",
+    "Yo dawg I heard you liked flux instances, so here is a flux instance to run in your flux instance!",
+    "One does not simply run an HPC job on the cloud... without Flux!",
+    "Flux submit, flux run... get the job done!",
+    "Job in pending? Could be... a ghost in the machine! ...or you forgot to update your accounting database.",
+    "A completed job is worth two in the queue.",
+    "An analysis of 1000 tasks begins with one batch.",
+    "A flux alloc a day keeps the sysadmins away.",
+    "The cluster is always less busy on the other side.",
+    "Don't count your jobs completed before they're done.",
+    "The early flux user catches the queue!",
+    "The early bird gets the worm, but the early user gets the supercomputer!",
+    "No use crying over failed jobs... ask for help!",
+    "The cluster is shining, the weather is sweet. Submit your job, to complete!",
+    "If you have a Flux nightmare, you might wake up sweating in parallel.",
+    "A cycle saved is a cycle earned",
+    "You can't judge a program by it's source code, but you can judge the developer!",
+    "Don't panic! That's the kernel's job.",
+    "Keep calm and carry on - it's the kernel's job to panic.",
+    "All work and no computer games makes your GPU idle.",
+    "If you want to go fast, go alone. If you want to go far, go distributed.",
+    "A stitch in time saves 9TB of backups.",
+    "If at first you don't succeed, reboot and try again.",
+    "To err is to human. To really foul things up requires a computer - Paul Ehrlich",
+    "Give a scientist a program, frustrate him for a day. Teach a scientist to program, frustrate him for a lifetime!",
+    "Why did the Flux instance go to the gym? To beef up its processing power!",
+    "Did you know 'fuzzybunny' is a valid Flux jobid?",
+]
+
+# This can be appended with new art as desired
+art = [
+    """
+                                                                 
+                                       .'                        
+                                       o,    .........           
+                                      .k...','....,''''.         
+                .......               ll''''.     ,'''',         
+           .,:::;,''',,:::;.         ';'','       ..'''.         
+        .:o:.      .....  .:o;      .''''.                       
+       ,o,     ,:cc::;;::c:'.,o,   .'''''                        
+     .ll    .ll'.         .;o:.oc .'''',.  .,:::::::c:,.         
+    .d,   .l:.    ':::::::c;.,oco:''''',;cc;,.....   ..;cc'      
+   'x'   ;d.   .cl,       .,lc;lc,''''',;'',;,,;;:cc:'   .:o,    
+  .d'   ,d.   ,d'        ''''',,'''''',,,:ccccc;.   .;l:   .co.  
+  o:    o.   ;d.              .;'''''':lc;'....;cl,    ,o.   ,d. 
+ cO.   ::   'k.                '''''',c.          ;o.   .d'   ;d 
+;kk:  cO;  .xl                .'''''';.            .o:   .d.   d;
+ o:  .lk: .xOx.               '''''',.              .k.   cc   ;d
+ d:   ;d   :k,.              .'''''',                o:   :l   'x
+ oc   ,x   'k.               '''''',.                l:   :c   .x
+ :d   .k.   o:              .,''''',                .x'   o:   ,o
+ .k;   ;l   .d:            .'''''',.               .ok.  ,O,   o;
+  ;k.   c:    co'        .;l,''''':.              .xOk. ;kO'  'k'
+   :x'   :o.   .;cc:;;;:cld;''''''o:              ,k:.  cxo, ckO;
+    ,x:   .cl;.     ...,:ol,''''':cd:           .cx,   .o.   ;k;.
+      ;l;.   .,::::::::;:c;''''':od:,c:'.   ..,cl,    'o.   ,d.  
+        .:c:,'......',::,.''''',,x:cl. .;:cc:,.     'l:    co.   
+            .',;;;,,'.   .'',,,. .d:.:l;.       .'cl;.   'd:     
+                         '''''.    ;o' .,::ccccc:'.    .ll.      
+             ...        .'',,        ,::,.         .':c:.        
+           .'''''.     .'';l.           .;c::cccccc;'.           
+           .'''''.  ..''..ll                                     
+             ....'....  .lkl.                                    
+                        .kk:                                     
+                         c'                                      
+
+Surprise! Thank you for using Flux Framework.
+"""
+]
diff --git a/src/bindings/python/flux/cli/run.py b/src/bindings/python/flux/cli/run.py
new file mode 100644
index 000000000000..ab385a812cc7
--- /dev/null
+++ b/src/bindings/python/flux/cli/run.py
@@ -0,0 +1,80 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import os
+import sys
+
+from flux.cli import base
+from flux.idset import IDset
+
+
+class RunCmd(base.SubmitBaseCmd):
+    """
+    RunCmd is identical to SubmitCmd, except it attaches the the job
+    after submission.  Some additional options are added to modify the
+    attach behavior.
+
+    Usage: flux run [OPTIONS] cmd ...
+    """
+
+    def __init__(self, prog, usage=None, description=None):
+        super().__init__(prog, usage, description)
+        self.parser.add_argument(
+            "--wait-event",
+            metavar="NAME",
+            help="Pass --wait-event=NAME to flux-job attach",
+        )
+        self.parser.add_argument(
+            "command", nargs=argparse.REMAINDER, help="Job command and arguments"
+        )
+
+    def main(self, args):
+        jobid = self.submit(args)
+
+        # Display job id on stderr if -v
+        # N.B. we must flush sys.stderr due to the fact that it is buffered
+        # when it points to a file, and os.execvp leaves it unflushed
+        if args.verbose > 0:
+            print("jobid:", jobid, file=sys.stderr)
+            sys.stderr.flush()
+
+        # Build args for flux job attach
+        attach_args = ["flux-job", "attach"]
+        if args.label_io:
+            attach_args.append("--label-io")
+        if args.verbose > 1:
+            attach_args.append("--show-events")
+        if args.verbose > 2:
+            attach_args.append("--show-exec")
+        if args.debug_emulate:
+            attach_args.append("--debug-emulate")
+        if args.wait_event:
+            attach_args.append(f"--wait-event={args.wait_event}")
+        if args.unbuffered:
+            attach_args.append("--unbuffered")
+        # If args.input is an idset, then pass along to attach:
+        if args.input:
+            try:
+                in_ranks = IDset(args.input)
+                attach_args.append(f"--stdin-ranks={in_ranks}")
+            except (ValueError, EnvironmentError):
+                #  Do nothing if ranks were not an idset, file input is
+                #  handled in jobspec.
+                pass
+        attach_args.append(jobid.f58.encode("utf-8", errors="surrogateescape"))
+
+        # Exec flux-job attach, searching for it in FLUX_EXEC_PATH.
+        old_path = os.environ.get("PATH")
+        os.environ["PATH"] = os.environ["FLUX_EXEC_PATH"]
+        if old_path:
+            os.environ["PATH"] += f":{old_path}"
+
+        os.execvp("flux-job", attach_args)
diff --git a/src/bindings/python/flux/cli/submit.py b/src/bindings/python/flux/cli/submit.py
new file mode 100644
index 000000000000..a1b5c6415fce
--- /dev/null
+++ b/src/bindings/python/flux/cli/submit.py
@@ -0,0 +1,27 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+
+from flux.cli import base
+
+
+class SubmitCmd(base.SubmitBulkCmd):
+    """
+    SubmitCmd submits a job, displays the jobid on stdout, and returns.
+
+    Usage: flux submit [OPTIONS] cmd ...
+    """
+
+    def __init__(self, prog, usage=None, description=None):
+        super().__init__(prog, usage, description)
+        self.parser.add_argument(
+            "command", nargs=argparse.REMAINDER, help="Job command and arguments"
+        )
diff --git a/src/bindings/python/flux/compat36.py b/src/bindings/python/flux/compat36.py
new file mode 100644
index 000000000000..f8b3b89d083e
--- /dev/null
+++ b/src/bindings/python/flux/compat36.py
@@ -0,0 +1,79 @@
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import signal
+
+
+# strsignal() is only available on Python 3.8 and up
+def strsignal(signum):
+    if signum == signal.SIGHUP:
+        return "Hangup"
+    elif signum == signal.SIGINT:
+        return "Interrupt"
+    elif signum == signal.SIGQUIT:
+        return "Quit"
+    elif signum == signal.SIGILL:
+        return "Illegal instruction"
+    elif signum == signal.SIGTRAP:
+        return "Trace/breakpoint trap"
+    elif signum == signal.SIGABRT or signum == signal.SIGIOT:
+        return "Aborted"
+    elif signum == signal.SIGBUS:
+        return "Bus error"
+    elif signum == signal.SIGFPE:
+        return "Floating point exception"
+    elif signum == signal.SIGKILL:
+        return "Killed"
+    elif signum == signal.SIGUSR1:
+        return "User defined signal 1"
+    elif signum == signal.SIGSEGV:
+        return "Segmentation Fault"
+    elif signum == signal.SIGUSR2:
+        return "User defined signal 2"
+    elif signum == signal.SIGPIPE:
+        return "Broken pipe"
+    elif signum == signal.SIGALRM:
+        return "Alarm clock"
+    elif signum == signal.SIGTERM:
+        return "Terminated"
+    # N.B. signal.SIGSTKFLT not defined until Python 3.11
+    elif "SIGSTKFLT" in dir(signal) and signum == signal.SIGSTKFLT:  # novermin
+        return "Stack fault"
+    elif signum == signal.SIGCHLD:
+        return "Child exited"
+    elif signum == signal.SIGCONT:
+        return "Continued"
+    elif signum == signal.SIGSTOP:
+        return "Stopped (signal)"
+    elif signum == signal.SIGTSTP:
+        return "Stopped"
+    elif signum == signal.SIGTTIN:
+        return "Stopped (tty input)"
+    elif signum == signal.SIGTTOU:
+        return "Stopped (tty output)"
+    elif signum == signal.SIGURG:
+        return "Urgent I/O condition"
+    elif signum == signal.SIGXCPU:
+        return "CPU time limit exceeded"
+    elif signum == signal.SIGXFSZ:
+        return "File size limit exceeded"
+    elif signum == signal.SIGVTALRM:
+        return "Virtual timer expired"
+    elif signum == signal.SIGPROF:
+        return "Profiling timer expired"
+    elif signum == signal.SIGWINCH:
+        return "Window changed"
+    elif signum == signal.SIGIO or signum == signal.SIGPOLL:
+        return "I/O possible"
+    elif signum == signal.SIGPWR:
+        return "Power failure"
+    elif signum == signal.SIGSYS:
+        return "Bad system call"
+    raise ValueError
diff --git a/src/bindings/python/flux/conf_builtin.py b/src/bindings/python/flux/conf_builtin.py
new file mode 100644
index 000000000000..84ededa0f2c0
--- /dev/null
+++ b/src/bindings/python/flux/conf_builtin.py
@@ -0,0 +1,64 @@
+###############################################################
+# Copyright 2024 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import threading
+from pathlib import Path
+
+from flux.core.inner import raw
+
+tls = threading.local()
+tls.FLUX_CONF_AUTO_FLAG = None
+
+
+def _conf_builtin_get_flag():
+    """Simulate the use of FLUX_CONF_AUTO for Python
+
+    FLUX_CONF_AUTO will not work from Python since the executable will
+    be python and not something part of the Flux build tree or installed
+    by Flux. This function simulates FLUX_CONF_AUTO by returning the
+    correct FLUX_CONF_FLAG based on whether this module is under the
+    in tree PYTHONPATH or not.
+    """
+    if tls.FLUX_CONF_AUTO_FLAG is None:
+        # Resolve builtin installed python path:
+        pythonpath = conf_builtin_get("python_path", which="intree").split(":")
+        for path in pythonpath:
+            if Path(path).resolve() in Path(__file__).resolve().parents:
+                # If path is one of this module's parents,
+                # then this module is in tree:
+                tls.FLUX_CONF_AUTO_FLAG = raw.FLUX_CONF_INTREE
+                return tls.FLUX_CONF_AUTO_FLAG
+        # O/w, assume we're installed
+        tls.FLUX_CONF_AUTO_FLAG = raw.FLUX_CONF_INSTALLED
+    return tls.FLUX_CONF_AUTO_FLAG
+
+
+def conf_builtin_get(name, which="auto"):
+    """Get builtin (compiled-in) configuration values from libflux
+
+    Args:
+        name (str): name of config value
+        which (str): one of "installed", "intree", or "auto" to return
+            the installed path, in tree path, or automatically determine
+            which to use. default=auto.
+    """
+    if which == "auto":
+        flag = _conf_builtin_get_flag()
+    elif which == "installed":
+        flag = raw.FLUX_CONF_INSTALLED
+    elif which == "intree":
+        flag = raw.FLUX_CONF_INTREE
+    else:
+        raise ValueError("which must be one of auto, installed, or intree")
+
+    try:
+        return raw.flux_conf_builtin_get(name, flag).decode("utf-8")
+    except OSError:
+        raise ValueError(f"No builtin config value for '{name}'")
diff --git a/src/bindings/python/flux/constants.py b/src/bindings/python/flux/constants.py
index 283d910f64fa..6a80de391775 100644
--- a/src/bindings/python/flux/constants.py
+++ b/src/bindings/python/flux/constants.py
@@ -10,8 +10,9 @@
 
 """Global constants for the flux interface"""
 
-import sys
 import re
+import sys
+
 from _flux._core import lib
 
 MOD = sys.modules[__name__]
diff --git a/src/bindings/python/flux/constraint/__init__.py b/src/bindings/python/flux/constraint/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/bindings/python/flux/constraint/parser.py b/src/bindings/python/flux/constraint/parser.py
new file mode 100644
index 000000000000..ca299dd524c7
--- /dev/null
+++ b/src/bindings/python/flux/constraint/parser.py
@@ -0,0 +1,480 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import json
+import re
+
+import ply.yacc as yacc
+from ply import lex
+
+
+class ConstraintSyntaxError(Exception):
+    """
+    Specialized SyntaxError exception to allow ConstraintParser to throw
+    a SyntaxError without PLY trying to force recovery.
+
+    """
+
+    pass
+
+
+class ConstraintLexer(object):
+    """
+    Simple constraint query syntax lexical analyzer based on RFC 35.
+    Used mainly as the lexer for BaseConstraintParser
+    """
+
+    #  Different quoting states for single vs double quotes:
+    states = (
+        ("squoting", "exclusive"),
+        ("dquoting", "exclusive"),
+    )
+
+    tokens = (
+        "NOT",
+        "AND",
+        "OR",
+        "NEGATE",
+        "LPAREN",
+        "RPAREN",
+        "TOKEN",
+        "QUOTE",
+    )
+
+    # Ignore whitespace in default state
+    t_ignore = " \t\r\n\f\v"
+
+    # Tokens in 'quoting' state
+    t_squoting_ignore = ""
+    t_dquoting_ignore = ""
+
+    def __init__(self, **kw_args):
+        super().__init__()
+        self.lexer = lex.lex(module=self, **kw_args)
+        self.parens_level = 0
+        self.last_lparens = 0
+        self.last_rparens = 0
+        self.last_quote = None
+        self.quote_start = None
+        self.pending_token = None
+
+    def input(self, data):
+        self.lexer.push_state("INITIAL")
+        self.parens_level = 0
+        self.last_lparens = 0
+        self.last_rparens = 0
+        self.last_quote = None
+        self.quote_start = None
+        self.lexer.input(data)
+
+    def __getattr__(self, attr):
+        return getattr(self.lexer, attr)
+
+    def t_ANY_error(self, t):
+        raise ConstraintSyntaxError(
+            f"Illegal character '{t.value[0]}' at position {t.lexer.lexpos}"
+        )
+
+    #  Define special tokens as functions before t_TOKEN so they are
+    #  guaranteed to take precedence.
+    #  c.f. http://www.dabeaz.com/ply/ply.html#ply_nn6
+
+    def t_NEGATE(self, t):
+        r"-"
+        return t
+
+    def t_NOT(self, t):
+        r"not\b"
+        return t
+
+    def t_AND(self, t):
+        r"&{1,2}|and\b"
+        return t
+
+    def t_OR(self, t):
+        r"\|{1,2}|or\b"
+        return t
+
+    def t_LPAREN(self, t):
+        r"\("
+        self.parens_level += 1
+        self.last_lparens = t.lexer.lexpos - 1
+        return t
+
+    def t_RPAREN(self, t):
+        r"\)"
+        self.parens_level -= 1
+        self.last_rparens = t.lexer.lexpos - 1
+        return t
+
+    def t_TOKEN(self, t):
+        r"[^()|&\s\"\']+"
+        if self.pending_token is not None:
+            t.value = self.pending_token.value + t.value
+            self.pending_token = None
+        elif t.value.endswith(":"):
+            #  Save a token that ends with ':' to possibly combine with
+            #  any following token. This allows op:"quoted string"
+            self.pending_token = t
+            return None
+        return t
+
+    def t_eof(self, t):
+        if self.pending_token is not None:
+            val = self.pending_token.value
+            raise ConstraintSyntaxError(f"Missing argument to token '{val}'")
+        return None
+
+    def t_QUOTE(self, t):
+        r"'|\""  # fmt: skip
+        self.quote_start = t.lexer.lexpos
+        self.last_quote = t.lexer.lexpos - 1
+        if t.value == "'":
+            t.lexer.begin("squoting")
+        else:
+            t.lexer.begin("dquoting")
+
+    #  quoting state:
+    def t_squoting_TOKEN(self, t):
+        r"([^'])+"
+        return self.t_TOKEN(t)
+
+    def t_squoting_QUOTE(self, t):
+        r"'"
+        self.last_quote = None
+        t.lexer.begin("INITIAL")
+
+    def t_squoting_eof(self, t):
+        pos = self.quote_start
+        raise ConstraintSyntaxError(f'Unclosed quote "\'" at position {pos}')
+
+    def t_dquoting_TOKEN(self, t):
+        r"([^\"])+"
+        return self.t_TOKEN(t)
+
+    def t_dquoting_QUOTE(self, t):
+        r"\""  # fmt: skip
+        self.last_quote = None
+        t.lexer.begin("INITIAL")
+
+    def t_dquoting_eof(self, t):
+        pos = self.quote_start
+        raise ConstraintSyntaxError(f"Unclosed quote '\"' at position {pos}")
+
+
+class ConstraintParser:
+    r"""
+    Base constraint query syntax parser class.
+
+    This class implements an RFC 35 compliant constraint query syntax
+    parser with the following simplified grammar:
+    ::
+
+       expr : expr expr
+        | expr and expr
+        | expr or expr
+        | not expr
+        | '(' expr ')'
+        | '-' term
+        | term
+
+       and : &{1,2}|and|AND
+
+       or : \|{1,2}|or|OR
+
+       not : not|NOT
+
+       term : \w*:?.+  # i.e. [operator:]operand
+
+    Where a term is a constraint operation which has the form
+    '[operator:]operand'. If the token does not include a `:`, then a
+    class default operator may optionally be substituted, e.g. "operand"
+    becomes "default:operand".
+
+    By default, ``operand`` is included as a single entry in the RFC 31
+    values array for the operator, i.e. ``{"operator":["operand"]}``.
+    However, if ``operator`` appears in the self.split_values dict
+    of the parser object, then the corresponding string will be used
+    to split ``value`` into multiple entries. E.g. if
+    ::
+
+        split_values = { "op": "," }
+
+    Then ``op:foo,bar`` would result in ``{"op":["foo","bar"]}``.
+
+    Terms are joined by AND unless OR is specified, such that ``a b c``
+    is the same as ``a && b && c``. A term can be negated with ``-``
+    (e.g. ``-a b c`` is equivlant to ``(not a)&b&c``), but to negate a
+    whole expression, NOT must be used (e.g. ``-(a|b)`` is a syntax error,
+    use ``not (a|b)`` instead).
+
+    As a result of parsing, an RFC 31 constraint object is returned as
+    a Python dict.
+
+    Attributes:
+        operator_map (dict): A mapping of operator names, used to specify
+            default and shorthand operator names. To configura a default
+            operator, specify ``None`` as a key, e.g.
+            ::
+
+               operator_map = { None: "name" }
+
+        split_values (dict): A mapping of operator name to string on which
+            to split values of that operator. For instance ``{"op": ","}``
+            would autosplit operator ``op`` values on comma.
+
+        combined_terms (set): A set of operator terms whose values can be
+            combined when joined with the AND logical operator. E.g. if
+            "test" is in ``combined_terms``, then
+            ::
+
+                test:a test:b
+
+            would produce
+            ::
+
+                {"test": ["a", "b"]}
+
+            instead of
+            ::
+
+                {"and": [{"test": ["a"]}, {"test": ["b"]}]}
+
+    Subclasses can set the values of the above attributes to create a
+    custom parser with default operators, split value handling, and
+    set of combined terms.
+
+    E.g.:
+    ::
+
+      class MyConstraintParser(ConstraintParser):
+          operator_map = { None: "default", "foo": "long_foo" }
+          split_values = { "default": "," }
+          combined_terms = set("default")
+
+    By default there is no mapping for ``None`` which means each term
+    requires an operator.
+
+    """
+
+    precedence = (
+        ("left", "OR"),
+        ("left", "AND"),
+        ("right", "NOT"),
+        ("right", "NEGATE"),
+    )
+
+    # Mapping of operator shorthand names to full names
+    # Subclasses should provide this mapping
+    operator_map = {}
+
+    # Mapping of operator to a string on which to split the operator's
+    # value. The value is typically stored as one entry in an array,
+    # but if set, the split string can be used to return multiple entries.
+    split_values = {}
+
+    # Combined terms
+    combined_terms = set()
+
+    def __init__(
+        self, lexer=None, optimize=True, debug=False, write_tables=False, **kw_args
+    ):
+        super().__init__()
+        self.lexer = ConstraintLexer() if lexer is None else lexer
+        self.tokens = self.lexer.tokens
+        self.query = None
+        self.parser = yacc.yacc(
+            module=self,
+            optimize=optimize,
+            debug=debug,
+            write_tables=write_tables,
+            **kw_args,
+        )
+
+    def parse(self, query, **kw_args):
+        self.query = query
+        return self.parser.parse(query, lexer=self.lexer, debug=0, **kw_args)
+
+    def p_error(self, p):
+        if p is None:
+            if self.lexer.parens_level > 0:
+                pos = self.lexer.last_lparens
+                raise ConstraintSyntaxError(f"Unclosed parenthesis in position {pos}")
+            raise ConstraintSyntaxError(f"Unexpected end of input in '{self.query}'")
+        if self.lexer.parens_level < 0:
+            raise ConstraintSyntaxError(
+                "Mismatched parentheses starting at position {self.lexer.last_rparens}."
+            )
+        raise ConstraintSyntaxError(f"Invalid token '{p.value}' at position {p.lexpos}")
+
+    def combine_like_terms(self, p1, p2):
+        combined_terms = {}
+        terms = []
+        entries = [p1, p2]
+
+        # First, attempt to combine any "and" terms
+        if "and" in p1:
+            p1["and"].append(p2)
+            entries = [p1]
+
+        # Then, combine any requested combined terms
+        for entry in entries:
+            key = list(entry)[0]
+            if key not in self.combined_terms:
+                terms.append(entry)
+            elif key not in combined_terms:
+                combined_terms[key] = entry[key]
+                terms.append(entry)
+            else:
+                combined_terms[key].extend(entry[key])
+        if len(terms) == 1:
+            return terms[0]
+        else:
+            return {"and": terms}
+
+    @staticmethod
+    def invalid_operator(op):
+        match = re.search(r"[^\w.+@-]", op)
+        if not match:
+            return None
+        return match[0]
+
+    def p_expression_space(self, p):
+        """
+        expression : expression expression %prec AND
+        """
+        p[0] = self.combine_like_terms(p[1], p[2])
+
+    def p_expression_and(self, p):
+        """
+        expression : expression AND expression
+        """
+        p[0] = self.combine_like_terms(p[1], p[3])
+
+    def p_expression_or(self, p):
+        """
+        expression : expression OR expression
+        """
+        if "or" in p[1]:
+            # Combine this `or` with a previous one
+            p[1]["or"].append(p[3])
+            p[0] = p[1]
+        else:
+            p[0] = {"or": [p[1], p[3]]}
+
+    def p_expression_unot(self, p):
+        """
+        expression : NOT expression
+                   | NEGATE token
+        """
+        p[0] = {"not": [p[2]]}
+
+    def p_expression_parens(self, p):
+        """
+        expression : LPAREN expression RPAREN
+        """
+        p[0] = p[2]
+
+    def p_token(self, p):
+        """
+        expression : token
+        """
+        p[0] = p[1]
+
+    def p_expression_token(self, p):
+        """
+        token : TOKEN
+        """
+        op, colon, value = p[1].partition(":")
+        if not colon:
+            if None not in self.operator_map:
+                raise ConstraintSyntaxError(f'Missing required operator in "{p[1]}"')
+            op = self.operator_map[None]
+            value = p[1]
+        elif op in self.operator_map:
+            op = self.operator_map[op]
+
+        invalid = self.invalid_operator(op)
+        if invalid:
+            raise ConstraintSyntaxError(
+                f"invalid character '{invalid}' in operator '{op}:'"
+            )
+
+        if op in self.split_values:
+            p[0] = {op: value.split(self.split_values[op])}
+        else:
+            p[0] = {op: [value]}
+
+    def p_quoted_token(self, p):
+        """
+        token : QUOTE TOKEN QUOTE
+        """
+        p[0] = p[2]
+
+
+if __name__ == "__main__":
+    """
+    Test command which can be run as flux python -m flux.constraint.parser
+    Also used to generate ply's parsetab.py in defined outputdir.
+    """
+
+    argparser = argparse.ArgumentParser(prog="constraint.parser")
+    argparser.add_argument(
+        "--outputdir",
+        metavar="DIR",
+        type=str,
+        help="Set outputdir for parsetab.py generation",
+    )
+    argparser.add_argument(
+        "--default-op",
+        metavar="NAME",
+        type=str,
+        help="Set a default operator to substitute for bare terms",
+    )
+    argparser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Emit lexer debug information before parsing expression",
+    )
+    argparser.add_argument(
+        "expression",
+        metavar="EXPRESSION",
+        type=str,
+        nargs="*",
+        help="Expression to parse",
+    )
+    args = argparser.parse_args()
+
+    if args.outputdir:
+        print(f"Generating constraint parsetab.py in {args.outputdir}")
+
+    class TConstraintParser(ConstraintParser):
+        if args.default_op:
+            operator_map = {None: args.default_op}
+
+    parser = TConstraintParser(
+        optimize=False, debug=True, write_tables=True, outputdir=args.outputdir
+    )
+    if args.expression:
+
+        s = " ".join(args.expression)
+        if args.debug:
+            print(f"parsing expression `{s}'")
+        if args.debug:
+            lexer = ConstraintLexer()
+            lexer.input(s)
+            while True:
+                tok = lexer.token()
+                if not tok:
+                    break
+                print(tok)
+
+        print(json.dumps(parser.parse(s)))
diff --git a/src/bindings/python/flux/core/Makefile.am b/src/bindings/python/flux/core/Makefile.am
deleted file mode 100644
index d0c4c3a22072..000000000000
--- a/src/bindings/python/flux/core/Makefile.am
+++ /dev/null
@@ -1,11 +0,0 @@
-fluxpycoredir = $(fluxpydir)/core
-fluxpycore_PYTHON=\
-	      __init__.py\
-	      watchers.py\
-	      inner.py\
-	      handle.py\
-	      trampoline.py
-
-clean-local:
-	-rm -f *.pyc *.pyo
-	-rm -rf __pycache__
diff --git a/src/bindings/python/flux/core/handle.py b/src/bindings/python/flux/core/handle.py
index 0e6f0bf61966..85b30d7a69e8 100644
--- a/src/bindings/python/flux/core/handle.py
+++ b/src/bindings/python/flux/core/handle.py
@@ -8,23 +8,38 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-import six
+import signal
+import threading
+from contextlib import contextmanager
 
-from flux.wrapper import Wrapper
-from flux.rpc import RPC
-from flux.message import Message
-from flux.util import encode_topic, encode_payload
-from flux.core.inner import raw
 from _flux._core import ffi, lib
+from flux.constants import FLUX_POLLERR, FLUX_POLLIN, FLUX_POLLOUT
+from flux.core.inner import raw
+from flux.core.watchers import FDWatcher, SignalWatcher, TimerWatcher
+from flux.future import Future
+from flux.message import Message, MessageWatcher
+from flux.rpc import RPC
+from flux.util import encode_payload, encode_topic, get_treedict
+from flux.wrapper import Wrapper
 
 
+# pylint: disable=too-many-public-methods
 class Flux(Wrapper):
     """
-  The general Flux handle class, create one of these to connect to the
-  nearest enclosing flux instance
-  >>> flux.Flux() #doctest: +ELLIPSIS
-  <flux.core.Flux object at 0x...>
-  """
+    The general Flux handle class, create one of these to connect to the
+    nearest enclosing flux instance
+
+    Example:
+        >>> flux.Flux()
+        <flux.core.Flux object at 0x...>
+    """
+
+    #  Thread local storage to hold a reactor_running boolean, indicating
+    #  when the current thread is running under a Flux reactor.
+    #
+    tls = threading.local()
+    tls.reactor_depth = 0
+    tls.exception = None
 
     def __init__(self, url=ffi.NULL, flags=0, handle=None):
         super(Flux, self).__init__(
@@ -37,15 +52,86 @@ def __init__(self, url=ffi.NULL, flags=0, handle=None):
             destructor=raw.flux_close,
         )
 
+        # Ensure reactor_depth is initialized for this thread
+        if "reactor_depth" not in self.tls.__dict__:
+            self.tls.reactor_depth = 0
+            self.tls.exception = None
+
         if handle is None:
+            error = ffi.new("flux_error_t[1]")
             try:
-                self.handle = raw.flux_open(url, flags)
-            except EnvironmentError as err:
-                raise EnvironmentError(
-                    err.errno, "Unable to connect to Flux: {}".format(err.strerror)
-                )
+                self.handle = raw.flux_open_ex(url, flags, error)
+            except OSError as err:
+                msg = "Unable to connect to Flux"
+                errmsg = ffi.string(error[0].text).decode("utf-8")
+                if errmsg:
+                    msg += f": {errmsg}"
+                raise OSError(err.errno, msg)
 
         self.aux_txn = None
+        self._active_watchers = set()
+        self._cached_config = None
+
+    @classmethod
+    def reactor_running(cls):
+        """Return True if this thread is running the Flux reactor"""
+        return cls.tls.reactor_depth > 0
+
+    @classmethod
+    def reactor_enter(cls):
+        cls.tls.reactor_depth += 1
+
+    @classmethod
+    def reactor_exit(cls):
+        cls.tls.reactor_depth -= 1
+
+    @contextmanager
+    def in_reactor(self):
+        self.reactor_enter()
+        try:
+            yield
+        finally:
+            self.reactor_exit()
+
+    @classmethod
+    def set_exception(cls, exception):
+        """Set a global, per-thread exception for Flux
+
+        This class method allows Python callbacks called from the Flux
+        reactor to set a global exception which can be re-thrown after
+        the return to Python (when reactor_run() returns). This is
+        implemented as a class attribute since the Flux handle object
+        which is available in a Python callback from C will be a
+        different instantiation than the Flux handle object which
+        started the reactor (with the same underlying flux_t however)
+
+        Args:
+            exception (Exception): A reference to the exception thrown.
+
+        Returns:
+            Exception: The previously set exception, or None
+        """
+        prev = cls.tls.exception
+        cls.tls.exception = exception
+        return prev
+
+    @classmethod
+    def raise_if_exception(cls):
+        """Re-raise any class global exception if set
+
+        If a global exception is currently set for the Flux handle class,
+        re-raise it and reset the exception state to None.
+
+        The exception is raised ``from None`` to preserve the original
+        stack trace.
+        """
+        #  Note: for unknown reason, an AttributeError is occasionally raised
+        #  here: '_thread._local' object has no attribute 'exception'
+        #  This should not be possible since the attribute is initialized
+        #  in the class, but workaround it anyway through use of getattr():
+        #
+        if getattr(cls.tls, "exception", None) is not None:
+            raise cls.set_exception(None) from None
 
     # pylint: disable=no-self-use
     def close(self):
@@ -69,7 +155,7 @@ def log(self, level, fstring):
         :param fstring: A string to log, C-style formatting is *not* supported
         """
         # Short-circuited because variadics can't be wrapped cleanly
-        if isinstance(fstring, six.text_type):
+        if isinstance(fstring, str):
             fstring = fstring.encode("utf-8")
         lib.flux_log(self.handle, level, fstring)
 
@@ -116,11 +202,11 @@ def recv(
         return None
 
     def rpc(self, topic, payload=None, nodeid=raw.FLUX_NODEID_ANY, flags=0):
-        """ Create a new RPC object """
+        """Create a new RPC object"""
         return RPC(self, topic, payload, nodeid, flags)
 
     def event_create(self, topic, payload=None):
-        """ Create a new event message.
+        """Create a new event message.
 
         :param topic: A string, the event's topic
         :param payload: If a string, the payload is used unmodified, if it is
@@ -130,7 +216,7 @@ def event_create(self, topic, payload=None):
         return Message.from_event_encode(topic, payload)
 
     def event_send(self, topic, payload=None):
-        """ Create and send a new event in one step """
+        """Create and send a new event in one step"""
         return self.send(self.event_create(topic, payload))
 
     def event_recv(self, topic=None):
@@ -146,6 +232,15 @@ def event_subscribe(self, topic):
         """
         return self.flux_event_subscribe(encode_topic(topic))
 
+    def add_watcher(self, watcher):
+        """Add a reference to a watcher so it avoids garbage collection"""
+        self._active_watchers.add(watcher)
+        return watcher
+
+    def del_watcher(self, watcher):
+        """Remove ref to ``watcher`` so it is eligible for garbage collection"""
+        self._active_watchers.discard(watcher)
+
     def msg_watcher_create(
         self,
         callback,
@@ -153,20 +248,134 @@ def msg_watcher_create(
         topic_glob="*",
         args=None,
         match_tag=raw.FLUX_MATCHTAG_NONE,
+        rolemask=None,
     ):
-        from flux.message import MessageWatcher
-
-        return MessageWatcher(self, type_mask, callback, topic_glob, match_tag, args)
+        return MessageWatcher(
+            self, type_mask, callback, topic_glob, match_tag, rolemask, args
+        )
 
     def timer_watcher_create(self, after, callback, repeat=0.0, args=None):
-        from flux.core.watchers import TimerWatcher
-
         return TimerWatcher(self, after, callback, repeat=repeat, args=args)
 
+    def signal_watcher_create(self, signum, callback, args=None):
+        return SignalWatcher(self, signum, callback, args)
+
+    def fd_watcher_create(self, fd_int, callback, events=None, args=None):
+        if events is None:
+            # TODO add mypy type stubs for constants so this passes vermin without
+            # comment
+            events = FLUX_POLLIN | FLUX_POLLOUT | FLUX_POLLERR  # novm
+        return FDWatcher(self, fd_int, events, callback, args=args)
+
     def barrier(self, name, nprocs):
-        self.flux_barrier(name, nprocs)
+        return Future(self.flux_barrier(name, nprocs))
 
     def get_rank(self):
         rank = ffi.new("uint32_t [1]")
         self.flux_get_rank(rank)
         return rank[0]
+
+    def attr_get(self, attr_name):
+        return self.flux_attr_get(attr_name).decode("utf-8")
+
+    def conf_get(self, key=None, default=None, update=False):
+        """
+        Access Flux configuration via this handle. On first use, the
+        configuration is fetched synchronously from the broker, then
+        cached in this :obj:`Flux` object. To force the configuration
+        to be updated, pass ``update=True``.
+
+        Example:
+        >>> print(handle.conf_get("tbon.topo", default="kary:32"))
+
+        Args:
+            key (str): key to get from configuration in dotted-string
+                form, e.g. ``tbon.topo``. If None, then the entire
+                config will be returned. Default is None.
+            default (obj): value to return if ``key`` is not set.
+                (default=None)
+            update (bool): Force an update of the internal conf dict.
+                (default=False)
+        """
+        if update or self._cached_config is None:
+            self._cached_config = self.rpc("config.get").get()
+        if key is not None:
+            return get_treedict(self._cached_config, key, default)
+        return self._cached_config
+
+    def reactor_run(self, reactor=None, flags=0):
+        """
+        Run reactor associated with this Flux handle or reactor argument
+        if it is provided. Sets a signal watcher for SIGINT to return
+        from the reactor on Ctrl-C, and raise KeyboardInterrupt.
+        """
+        rc = 0
+        if reactor is None:
+            reactor = self.get_reactor()
+
+        #
+        #  Only do the whole signals rigamarole below if we're in the
+        #   the main thread: libev don't take kindly to registration
+        #   of signal watcher from multiple threads.
+        #
+        if threading.current_thread() != threading.main_thread():
+            rc = self.flux_reactor_run(reactor, flags)
+            Flux.raise_if_exception()
+            return rc
+
+        reactor_interrupted = False
+
+        def reactor_interrupt(handle, *_args):
+            #  ensure reactor_interrupted from enclosing scope:
+            nonlocal reactor_interrupted
+            reactor_interrupted = True
+            handle.reactor_stop(reactor)
+
+        with self.signal_watcher_create(signal.SIGINT, reactor_interrupt):
+            with self.in_reactor():
+                # This signal watcher should not take a reference on reactor
+                #  o/w the reactor may not exit as expected when all other
+                #  active watchers and msghandlers are complete.
+                #
+                self.reactor_active_decref(reactor)
+                rc = self.flux_reactor_run(reactor, flags)
+                #  Re-establish signal watcher reference so reactor refcount
+                #   doesn't underflow when signal watcher is destroyed
+                #
+                self.reactor_active_incref(reactor)
+            if reactor_interrupted:
+                raise KeyboardInterrupt
+            Flux.raise_if_exception()
+
+        # If rc > 0, we need to subtract our added SIGINT watcher, which
+        # will now be destroyed since it has left scope
+        return rc if rc <= 0 else rc - 1
+
+    def reactor_stop(self, reactor=None):
+        if reactor is None:
+            reactor = self.get_reactor()
+        self.flux_reactor_stop(reactor)
+
+    def reactor_stop_error(self, reactor=None):
+        if reactor is None:
+            reactor = self.get_reactor()
+        self.flux_reactor_stop_error(reactor)
+
+    def reactor_incref(self, reactor=None):
+        if reactor is None:
+            reactor = self.get_reactor()
+        self.reactor_active_incref(reactor)
+
+    def reactor_decref(self, reactor=None):
+        if reactor is None:
+            reactor = self.get_reactor()
+        self.reactor_active_decref(reactor)
+
+    def service_register(self, name):
+        return Future(self.flux_service_register(name))
+
+    def service_unregister(self, name):
+        return Future(self.flux_service_unregister(name))
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/bindings/python/flux/core/trampoline.py b/src/bindings/python/flux/core/trampoline.py
index 5588f4d415ad..946f290e9fd5 100644
--- a/src/bindings/python/flux/core/trampoline.py
+++ b/src/bindings/python/flux/core/trampoline.py
@@ -9,6 +9,7 @@
 ###############################################################
 
 import importlib
+
 from _flux._core import lib
 from flux.core.handle import Flux
 
diff --git a/src/bindings/python/flux/core/watchers.py b/src/bindings/python/flux/core/watchers.py
index e01a8b69cbcc..3df6f19e7729 100644
--- a/src/bindings/python/flux/core/watchers.py
+++ b/src/bindings/python/flux/core/watchers.py
@@ -9,15 +9,20 @@
 ###############################################################
 
 import abc
-from flux.core.inner import raw, lib, ffi
+import errno
+import signal
 
-__all__ = ["TimerWatcher", "FDWatcher"]
+from flux.core.inner import ffi, lib, raw
+
+__all__ = ["TimerWatcher", "FDWatcher", "SignalWatcher"]
 
 
 class Watcher(object):
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self, handle=None):
+    def __init__(self, flux_handle, handle=None):
+        self.flux_handle = flux_handle
+        self.flux_handle.add_watcher(self)
         self.handle = handle
 
     def __enter__(self):
@@ -31,16 +36,25 @@ def __exit__(self, type_arg, value, _):
         return False
 
     def __del__(self):
-        if self.handle is not None:
-            self.destroy()
+        self.destroy()
 
     def start(self):
         raw.flux_watcher_start(self.handle)
+        return self
 
     def stop(self):
         raw.flux_watcher_stop(self.handle)
+        return self
 
     def destroy(self):
+        #  Remove this watcher from its owning Flux handle
+        #  if the handle is still around. A try/except block
+        #  used here in case the Watcher got an exception
+        #  before __init__ (e.g. in subclass __init__)
+        try:
+            self.flux_handle.del_watcher(self)
+        except AttributeError:
+            pass
         if self.handle is not None:
             raw.flux_watcher_destroy(self.handle)
             self.handle = None
@@ -50,12 +64,16 @@ def destroy(self):
 def timeout_handler_wrapper(unused1, unused2, revents, opaque_handle):
     del unused1, unused2  # unused arguments
     watcher = ffi.from_handle(opaque_handle)
-    watcher.callback(watcher.flux_handle, watcher, revents, watcher.args)
+    try:
+        watcher.callback(watcher.flux_handle, watcher, revents, watcher.args)
+    # pylint: disable=broad-except
+    except Exception as exc:
+        type(watcher.flux_handle).set_exception(exc)
+        watcher.flux_handle.reactor_stop_error()
 
 
 class TimerWatcher(Watcher):
     def __init__(self, flux_handle, after, callback, repeat=0, args=None):
-        self.flux_handle = flux_handle
         self.after = after
         self.repeat = repeat
         self.callback = callback
@@ -63,13 +81,14 @@ def __init__(self, flux_handle, after, callback, repeat=0, args=None):
         self.handle = None
         self.wargs = ffi.new_handle(self)
         super(TimerWatcher, self).__init__(
+            flux_handle,
             raw.flux_timer_watcher_create(
                 raw.flux_get_reactor(flux_handle),
                 float(after),
                 float(repeat),
                 lib.timeout_handler_wrapper,
                 self.wargs,
-            )
+            ),
         )
 
 
@@ -77,13 +96,17 @@ def __init__(self, flux_handle, after, callback, repeat=0, args=None):
 def fd_handler_wrapper(unused1, unused2, revents, opaque_handle):
     del unused1, unused2  # unused arguments
     watcher = ffi.from_handle(opaque_handle)
-    fd_int = raw.fd_watcher_get_fd(watcher.handle)
-    watcher.callback(watcher.flux_handle, watcher, fd_int, revents, watcher.args)
+    try:
+        fd_int = raw.fd_watcher_get_fd(watcher.handle)
+        watcher.callback(watcher.flux_handle, watcher, fd_int, revents, watcher.args)
+    # pylint: disable=broad-except
+    except Exception as exc:
+        type(watcher.flux_handle).set_exception(exc)
+        watcher.flux_handle.reactor_stop_error()
 
 
 class FDWatcher(Watcher):
     def __init__(self, flux_handle, fd_int, events, callback, args=None):
-        self.flux_handle = flux_handle
         self.fd_int = fd_int
         self.events = events
         self.callback = callback
@@ -91,11 +114,50 @@ def __init__(self, flux_handle, fd_int, events, callback, args=None):
         self.handle = None
         self.wargs = ffi.new_handle(self)
         super(FDWatcher, self).__init__(
+            flux_handle,
             raw.flux_fd_watcher_create(
                 raw.flux_get_reactor(flux_handle),
                 self.fd_int,
                 self.events,
                 lib.fd_handler_wrapper,
                 self.wargs,
-            )
+            ),
+        )
+
+
+@ffi.def_extern()
+def signal_handler_wrapper(_unused1, _unused2, _unused3, opaque_handle):
+    watcher = ffi.from_handle(opaque_handle)
+    try:
+        signal_int = raw.signal_watcher_get_signum(watcher.handle)
+        watcher.callback(watcher.flux_handle, watcher, signal_int, watcher.args)
+    # pylint: disable=broad-except
+    except Exception as exc:
+        type(watcher.flux_handle).set_exception(exc)
+        watcher.flux_handle.reactor_stop_error()
+
+
+class SignalWatcher(Watcher):
+    def __init__(self, flux_handle, signal_int, callback, args=None):
+        self.signal_int = signal_int
+        self.callback = callback
+        self.args = args
+        self.handle = None
+        self.wargs = ffi.new_handle(self)
+        super(SignalWatcher, self).__init__(
+            flux_handle,
+            raw.flux_signal_watcher_create(
+                raw.flux_get_reactor(flux_handle),
+                self.signal_int,
+                lib.signal_handler_wrapper,
+                self.wargs,
+            ),
         )
+        # N.B.: check for error only after SignalWatcher object fully
+        #  initialized to avoid 'no attribute self.handle' in __del__
+        #  method.
+        if signal_int < 1 or signal_int >= signal.NSIG:
+            raise OSError(errno.EINVAL, "invalid signal number")
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/bindings/python/flux/debugged.py b/src/bindings/python/flux/debugged.py
new file mode 100644
index 000000000000..589764450a75
--- /dev/null
+++ b/src/bindings/python/flux/debugged.py
@@ -0,0 +1,19 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+from _flux._core import lib
+
+
+def get_mpir_being_debugged():
+    return lib.get_mpir_being_debugged()
+
+
+def set_mpir_being_debugged(value):
+    return lib.set_mpir_being_debugged(value)
diff --git a/src/bindings/python/flux/future.py b/src/bindings/python/flux/future.py
index b3f7af056f29..1f0b8fe17fdc 100644
--- a/src/bindings/python/flux/future.py
+++ b/src/bindings/python/flux/future.py
@@ -9,11 +9,12 @@
 ###############################################################
 
 import errno
+import json
+from typing import Dict
 
-from flux.util import check_future_error
-from flux.wrapper import Wrapper, WrapperPimpl
 from flux.core.inner import ffi, lib, raw
-
+from flux.util import check_future_error, interruptible
+from flux.wrapper import Wrapper, WrapperPimpl
 
 # Reference count dictionary to keep futures with a pending `then` callback
 # alive, even if there are no remaining references to the future in the user's
@@ -22,22 +23,76 @@
 # future's callback is run, decrement the count in the dictionary for that
 # future, and whenever reset is called on a future, increment the counter. If
 # the counter hits 0, delete the reference to the future from the dictionary.
-_THEN_HANDLES = {}
+_THEN_HANDLES: Dict["Future", int] = {}
+
+# Same as above, but for FutreExt init callbacks:
+_INIT_HANDLES: Dict["Future", int] = {}
 
 
 @ffi.def_extern()
 def continuation_callback(c_future, opaque_handle):
     try:
-        py_future = ffi.from_handle(opaque_handle)
+        py_future: "Future" = ffi.from_handle(opaque_handle)
         assert c_future == py_future.pimpl.handle
-        py_future.then_cb(py_future, py_future.then_arg)
+        if not py_future.stopped:
+            py_future.then_cb(py_future, *py_future.then_args, **py_future.then_kwargs)
+    # pylint: disable=broad-except
+    except Exception as exc:
+        #
+        # Uncaught exceptions stop reactor with error.
+        # Setting the exception in the global Flux handle class will cause
+        #  the handle to re-throw the exception after leaving the reactor:
+        #
+        flux_handle = py_future.get_flux()
+        type(flux_handle).set_exception(exc)
+        flux_handle.reactor_stop_error()
+        #
+        # Reset this future so it doesn't immediately call the callback
+        # again (and fail) if the reactor is restarted. (and bypass
+        # future.reset() to avoid unnecessarily incrementing _THEN_HANDLES
+        # count).
+        #
+        py_future.pimpl.reset()
+        py_future.stop()
+
     finally:
-        _THEN_HANDLES[py_future] -= 1
-        if _THEN_HANDLES[py_future] <= 0:
+        if py_future in _THEN_HANDLES:
+            _THEN_HANDLES[py_future] -= 1
+            if _THEN_HANDLES[py_future] <= 0:
+                #
+                #  Note, a multiply fulfilled future which is not reset
+                #  before leaving the then_cb() will end up here, since a
+                #  call to reset() is the only thing that increments the
+                #  _THEN_HANDLES counter. If py_future.cb_handle is set
+                #  to None at this point, then the handle could be garbage
+                #  collected immediately. If the Future is not also collected,
+                #  then continuation_callback() might be called again with
+                #  the same cb_handle, which is now invalid, and Python will
+                #  abort. Therefore, leave cb_handle defined for the
+                #  lifetime of the Future.
+                #
+                del _THEN_HANDLES[py_future]
+
+
+@ffi.def_extern()
+def init_callback(c_future, opaque_handle):
+    try:
+        handle_ptr: "list" = ffi.from_handle(opaque_handle)
+        future: "Future" = ffi.from_handle(handle_ptr[0])
+        assert c_future == future.pimpl.handle
+        future.init_cb(future, *future.init_args, **future.init_kwargs)
+    # pylint: disable=broad-except
+    except Exception as exc:
+        flux_handle = future.get_flux()
+        type(flux_handle).set_exception(exc)
+        flux_handle.reactor_stop_error()
+    finally:
+        _INIT_HANDLES[future] -= 1
+        if _INIT_HANDLES[future] <= 0:
             # allow future object to be garbage collected now that all
             # registered callbacks have completed
-            py_future.cb_handle = None
-            del _THEN_HANDLES[py_future]
+            future.init_handle = None
+            del _INIT_HANDLES[future]
 
 
 class Future(WrapperPimpl):
@@ -59,6 +114,13 @@ def __init__(
             if prefixes is None:
                 prefixes = ["flux_future_"]
 
+            # enable Future(fut_obj) to work by just re-wrapping the
+            # underlying future object and increments its reference to avoid
+            # pre-mature destruction when the original obj gets GC'd
+            if isinstance(handle, Future):
+                handle.incref()
+                handle = handle.handle
+
             super(Future.InnerWrapper, self).__init__(
                 ffi, lib, handle, match, filter_match, prefixes, destructor
             )
@@ -67,12 +129,22 @@ def check_wrap(self, fun, name):
             func = super(Future.InnerWrapper, self).check_wrap(fun, name)
             return check_future_error(func)
 
-    def __init__(self, future_handle, prefixes=None):
+    def __init__(self, future_handle, prefixes=None, pimpl_t=None):
         super(Future, self).__init__()
-        self.pimpl = self.InnerWrapper(handle=future_handle, prefixes=prefixes)
+        if pimpl_t is None:
+            pimpl_t = self.InnerWrapper
+        self.pimpl = pimpl_t(handle=future_handle, prefixes=prefixes)
         self.then_cb = None
-        self.then_arg = None
+        self.then_args = []
+        self.then_kwargs = {}
         self.cb_handle = None
+        self.stopped = False
+
+    def stop(self):
+        """Stop a future from calling the user callback.
+        Useful for streaming futures given lack of destroy.
+        """
+        self.stopped = True
 
     def error_string(self):
         try:
@@ -82,10 +154,18 @@ def error_string(self):
         return errmsg.decode("utf-8") if errmsg else None
 
     def get_flux(self):
-        # pylint: disable=cyclic-import
+        # importing within a function to break a cyclic import
+        # pylint: disable=cyclic-import, import-outside-toplevel
         import flux.core.handle
 
-        flux_handle = self.pimpl.get_flux()
+        try:
+            flux_handle = self.pimpl.get_flux()
+        except OSError as exc:
+            #  get_flux() throws OSError of EINVAL if !f->h, but this should
+            #   a valid return (i.e. no flux handle set yet)
+            if exc.errno == errno.EINVAL:
+                return None
+            raise
         if flux_handle == ffi.NULL:
             return None
         handle = flux.core.handle.Flux(handle=flux_handle)
@@ -94,10 +174,13 @@ def get_flux(self):
         handle.incref()
         return handle
 
+    def set_flux(self, flux_handle):
+        self.pimpl.set_flux(flux_handle)
+
     def get_reactor(self):
         return self.pimpl.get_reactor()
 
-    def then(self, callback, arg=None, timeout=-1.0):
+    def then(self, callback, *args, timeout=-1.0, **kwargs):
         if self in _THEN_HANDLES:
             raise EnvironmentError(
                 errno.EEXIST, "then callback already exists for this future"
@@ -106,7 +189,8 @@ def then(self, callback, arg=None, timeout=-1.0):
             raise ValueError("Callback cannot be None")
 
         self.then_cb = callback
-        self.then_arg = arg
+        self.then_args = args
+        self.then_kwargs = kwargs
         self.cb_handle = ffi.new_handle(self)
         self.pimpl.then(timeout, lib.continuation_callback, self.cb_handle)
 
@@ -119,10 +203,13 @@ def then(self, callback, arg=None, timeout=-1.0):
         # For example `f.rpc('topic').then(cb).wait_for(-1)
         return self
 
+    def fulfill_error(self, errnum=2, errstr=ffi.NULL):
+        self.pimpl.fulfill_error(errnum, errstr)
+
     def reset(self):
         self.pimpl.reset()
 
-        if self.cb_handle is not None:
+        if self in _THEN_HANDLES:
             # ensure that this future object is not garbage collected with a
             # callback outstanding. Particularly important for streaming RPCs.
             _THEN_HANDLES[self] += 1
@@ -130,5 +217,182 @@ def reset(self):
     def is_ready(self):
         return self.pimpl.is_ready()
 
+    def raise_if_handle_exception(self):
+        #  Helper function to raise any pending exception on this Future's
+        #  handle instead of raising an OSError returned from calling
+        #  flux_future_get(3) or flux_future_wait_for(3). The reason for
+        #  doing this is that these functions can fail if a Python callback
+        #  raises an exception, since the callback could then return without
+        #  fulfilling the target future, thus the low-level future calls
+        #  can return EDEADLOCK. In other situations we also want to raise any
+        #  pending exceptions since they will be more valuable than just
+        #  the equivalent of an errno. (If the future was really fulfilled
+        #  with an error, then no pending exception is expected, and the
+        #  caller should go on to raise the original OSError exception.)
+        #
+        flux_handle = self.get_flux()
+        if flux_handle is not None:
+            type(flux_handle).raise_if_exception()
+
+    @interruptible
     def wait_for(self, timeout=-1.0):
-        self.pimpl.wait_for(timeout)
+        try:
+            self.pimpl.wait_for(timeout)
+        except OSError:
+            self.raise_if_handle_exception()
+            raise
+
+        return self
+
+    @interruptible
+    def get(self):
+        """
+        Base Future.get() method. Does not return a result, just blocks
+        until future is fulfilled and throws OSError on failure.
+        """
+        try:
+            self.pimpl.flux_future_get(ffi.NULL)
+        except OSError:
+            self.raise_if_handle_exception()
+            raise
+
+    def incref(self):
+        self.pimpl.flux_future_incref()
+
+
+class WaitAllFuture(Future):
+    """Create a composite future which waits for all children to be fulfilled"""
+
+    def __init__(self, children=None):
+        self.children = children
+        if self.children is None:
+            self.children = []
+        future = raw.flux_future_wait_all_create()
+        super(WaitAllFuture, self).__init__(future)
+
+    def push(self, child, name=None):
+        if name is None:
+            name = ffi.NULL
+        #
+        #  if this future does not have a flux handle yet, attempt
+        #   to grab from the first pushed "child" future.
+        if self.get_flux() is None:
+            self.pimpl.set_flux(child.get_flux())
+
+        self.pimpl.push(name, child)
+        #
+        #  flux_future_push(3) "adopts" memory for child, so call
+        #   incref on child to avoid  calling flux_future_destroy(3) when
+        #   child goes out of scope in caller context.
+        child.pimpl.incref()
+        self.children.append(child)
+
+
+class FutureExt(Future):
+    """
+    Extensible Future for use directly from Python.
+
+    This class allows creation of an "empty" Future which can then
+    be fulfilled with a value or other Future directly from Python.
+
+    The purpose is to allow multiple actions, RPCs, etc to be abstracted
+    behind a single interface, with one "Future" object used to signal
+    completion and/or availability of a result to the caller.
+    """
+
+    def __init__(self, init_cb, *args, flux_handle=None, **kw_args):
+        """
+        Create an empty Future with initialization callback init_cb(),
+        with optional arguments args and kwargs.
+
+        The initialization callback will be called either when a then()
+        callback is registered on the returned Future or a blocking get()
+        is used on the Future.
+
+        The initialization callback should take care to use
+        ``future.get_flux()`` to get the correct Flux handle for the
+        context in which the initialization is running. This is because
+        the underlying reactor will be different depending on whether
+        the Future is being initialized in a blocking ``future.get()``
+        or by a call to ``future.then()``. See the documentation for
+        ``flux_future_create(3)`` for more information.
+
+        If ``flux_handle`` is not None, then the given Flux handle will
+        be associated with the Future during initialization. Otherwise,
+        Future.set_flux() must be called before .then() or .get() are
+        called, or an OSError with errno.EINVAL exception may be raised.
+
+        Args:
+            init_cb (Callable): Future initialization callback
+            flux_handle (flux.Flux): Flux handle to associate with this Future
+            args, kw_args: args and keyword args to pass along to init_cb
+        """
+        # The ffi handle for the Future object is not available until we
+        # create it, but the ffi handle needs to be passed to
+        # flux_future_create(3). Therefore, create a list object we can pass
+        # to create(), then add the Future ffi handle to the list.
+        #
+        ffi_handle_list = []
+        handle = ffi.new_handle(ffi_handle_list)
+
+        # Setup initialization callback:
+        self.init_handle = handle
+        self.init_cb = init_cb
+        self.init_args = args
+        self.init_kwargs = kw_args
+
+        # When this Future is fulfilled with a Python value, the memory for
+        # the C copy of this value (or the string representation of it) must
+        # have a reference held by this Future object, so it is not released
+        # until this Future is garbage-collected. Use a _results list for this
+        # purpose:
+        self._results = []
+
+        # Create a Future from flux_future_create(3)
+        future = raw.flux_future_create(lib.init_callback, handle)
+
+        # Place ffi handle for future into ffi_handle_list and set this
+        # future in _INIT_HANDLES to avoid garbage collection
+        ffi_handle_list.append(ffi.new_handle(self))
+        _INIT_HANDLES[self] = 1
+        super().__init__(future)
+
+        if flux_handle is not None:
+            self.set_flux(flux_handle)
+
+    def fulfill(self, result=None):
+        """
+        Fulfill a future with a result. The ``result`` can be any object
+        or value that is JSON serializable by json.dumps()
+        """
+        # Note: result=None handled since json.dumps(None) => 'null'
+        result = json.dumps(result).encode("utf-8", errors="surrogateescape")
+        payload = ffi.new("char[]", result)
+        self.pimpl.fulfill(payload, ffi.NULL)
+
+        # Python will free memory for the payload once it goes out of scope.
+        # Therefore append payload to the internal results list to keep
+        # a reference until the future is garbage collected.
+        self._results.append(payload)
+
+    @interruptible
+    def get(self):
+        """
+        Convenience method to return a value stored by ``fulfill()``
+
+        Note: will return garbage if future does not contain a string, i.e.
+        if this Future was fulfilled outside of Python with a C object.
+
+        Returns string payload or None if future payload is NULL.
+        """
+        payload = ffi.new("void **")
+        try:
+            self.pimpl.get(payload)
+        except OSError:
+            self.raise_if_handle_exception()
+            raise
+
+        if payload[0] == ffi.NULL:
+            return None
+        value = ffi.string(ffi.cast("char *", payload[0])).decode("utf-8")
+        return json.loads(value)
diff --git a/src/bindings/python/flux/hostlist.py b/src/bindings/python/flux/hostlist.py
new file mode 100644
index 000000000000..5e5157c76ca2
--- /dev/null
+++ b/src/bindings/python/flux/hostlist.py
@@ -0,0 +1,255 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import collections
+import numbers
+
+from _flux._hostlist import ffi, lib
+from flux.wrapper import Wrapper, WrapperPimpl
+
+
+class HostlistIterator:
+    def __init__(self, hostlist):
+        self._hostlist = hostlist
+        self._index = 0
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self._index < len(self._hostlist):
+            result = self._hostlist[self._index]
+            self._index += 1
+            return result
+        raise StopIteration
+
+
+class Hostlist(WrapperPimpl):
+    """A Flux hostlist object
+
+    The Hostlist class wraps libflux-hostlist to implement a list
+    of hosts which can be converted to and from the RFC 29 hostlist
+    encoding.
+    """
+
+    class InnerWrapper(Wrapper):
+        def __init__(
+            self,
+            handle=None,
+        ):
+            super().__init__(
+                ffi,
+                lib,
+                handle=handle,
+                match=ffi.typeof("struct hostlist *"),
+                prefixes=["hostlist_"],
+                destructor=lib.hostlist_destroy,
+            )
+
+    def __init__(self, arg="", handle=None):
+        """
+        Create a new Hostlist object from RFC 29 Hostlist string or an
+        Iterable containing a set of RFC 29 Hostlist strings:
+
+        :param arg: A string or Iterable containing one or more hosts
+                    encoded in RFC 29 hostlist format
+
+        E.g.
+
+        >>> hl = Hostlist()
+        >>> hl = Hostlist("host")
+        >>> hl = Hostlist("host[0-10]")
+        >>> hl = Hostlist([ "foo1", "foo2" ])
+
+        """
+        #  If no `struct hostlist *` handle passed in, then create
+        #   a new handle from string argument
+        if handle is None:
+            if isinstance(arg, str):
+                handle = lib.hostlist_decode(arg.encode("utf-8"))
+                if handle == ffi.NULL:
+                    raise ValueError(f"Invalid hostlist: '{arg}'")
+            elif isinstance(arg, collections.abc.Iterable):
+                handle = lib.hostlist_create()
+                for hosts in arg:
+                    if not isinstance(hosts, str):
+                        typestr = type(hosts)
+                        raise TypeError(
+                            f"Hostlist(): expected string or Iterable, got {typestr}"
+                        )
+                    result = lib.hostlist_append(handle, hosts.encode("utf-8"))
+                    if result < 0:
+                        raise ValueError(f"Invalid hostlist: '{hosts}'")
+            else:
+                typestr = type(arg)
+                raise TypeError(
+                    f"Hostlist(): expected string or Iterable, got {typestr}"
+                )
+        super().__init__()
+        self.pimpl = self.InnerWrapper(handle)
+
+    def __str__(self):
+        return self.encode()
+
+    def __repr__(self):
+        return f"Hostlist('{self}')"
+
+    def __len__(self):
+        return self.pimpl.count()
+
+    def __getitem__(self, index):
+        """Index and slice a hostlist
+
+        Works like normal Python list indexing, including slices.
+        Any iterable is also supported as long as the iterable contains
+        only integers.
+
+        Slices and iterables return a new Hostlist object.
+
+        >>> hl = Hostlist("foo[0-9]")
+        >>> hl[0]
+        'foo0'
+        >>> hl[9]
+        'foo9'
+        >>> hl[-1]
+        'foo9'
+        >>> hl[8:]
+        Hostlist('foo[8-9]']
+        >>> hl[1:3]
+        Hostlist('foo[1-2]']
+        >>> hl[1,3]
+        Hostlist('foo[1,3]']
+
+        """
+        if isinstance(index, numbers.Integral):
+            if index < 0:
+                index = len(self) + index
+            if 0 <= index < len(self):
+                # N.B. wrapper class already calls ffi.string() on result:
+                return self.pimpl.nth(index).decode("utf-8")
+            raise IndexError("Hostlist index out of range")
+
+        if isinstance(index, slice):
+            hl = Hostlist()
+            for n in range(len(self))[index]:
+                hl.append(self[n])
+            return hl
+
+        if isinstance(index, collections.abc.Iterable):
+            hl = Hostlist()
+            for n in index:
+                # Avoid infinite recursion by catching non-integer indices
+                if not isinstance(n, numbers.Integral):
+                    raise TypeError(f"Invalid Hostlist index '{n}'")
+                hl.append(self[n])
+            return hl
+
+        raise TypeError("Hostlist index must be integer or slice")
+
+    def __iter__(self):
+        """Return a Hostlist iterator"""
+        return HostlistIterator(self)
+
+    def __contains__(self, name):
+        """Test if a hostname is in a Hostlist"""
+        try:
+            self.find(name)
+        except FileNotFoundError:
+            return False
+        return True
+
+    def encode(self):
+        """Encode a Hostlist to an RFC 29 hostlist string"""
+        #
+        #  N.B. Do not use automatic wrapper call here to avoid leaking
+        #   `char *` result. Instead explicitly call free() after copying
+        #   the returned string to Python
+        #
+        val = lib.hostlist_encode(self.handle)
+        result = ffi.string(val)
+        lib.free(val)
+        return result.decode("utf-8")
+
+    def count(self):
+        """Return the number of hosts in Hostlist"""
+        return self.pimpl.count()
+
+    def append(self, *args):
+        """Append one or more arguments to a Hostlist
+
+        Args may be either a Hostlist or any valid argument to Hostlist()
+        """
+        count = 0
+        for arg in args:
+            if not isinstance(arg, Hostlist):
+                arg = Hostlist(arg)
+            count += self.pimpl.append_list(arg)
+        return count
+
+    def delete(self, hosts):
+        """Delete host or hosts from Hostlist
+
+        param: hosts: A Hostlist or string in RFC 29 hostlist encoding
+        """
+        return self.pimpl.delete(str(hosts))
+
+    def sort(self):
+        """Sort a Hostlist"""
+        self.pimpl.sort()
+        return self
+
+    def uniq(self):
+        """Sort and remove duplicate hostnames from Hostlist"""
+        self.pimpl.uniq()
+        return self
+
+    def expand(self):
+        """Convert a Hostlist to a Python list"""
+        return list(self)
+
+    def copy(self):
+        """Copy a Hostlist object"""
+        return Hostlist(handle=self.pimpl.copy())
+
+    def find(self, host):
+        """Return the position of a host in a Hostlist"""
+        return self.pimpl.find(host)
+
+    def index(self, hosts, ignore_nomatch=False):
+        """
+        Return a list of integers corresponding to the indices of ``hosts``
+        in the current Hostlist.
+        Args:
+            hosts (str, Hostlist): List of hosts to find
+            ignore_nomatch (bool): Ignore hosts in ``hosts`` that are not
+             present in Hostlist. Otherwise, FileNotFound error is raised
+             with the missing hosts.
+        """
+        if not isinstance(hosts, Hostlist):
+            hosts = Hostlist(hosts)
+        ids = []
+        notfound = Hostlist()
+        for host in hosts:
+            try:
+                ids.append(self.find(host))
+            except FileNotFoundError:
+                notfound.append(host)
+        if notfound and not ignore_nomatch:
+            suffix = "s" if len(notfound) > 1 else ""
+            raise FileNotFoundError(f"host{suffix} '{notfound}' not found")
+        return ids
+
+
+def decode(arg):
+    """
+    Decode a string or iterable of strings in RFC 29 hostlist format
+    to a Hostlist object
+    """
+    return Hostlist(arg)
diff --git a/src/bindings/python/flux/idset.py b/src/bindings/python/flux/idset.py
new file mode 100644
index 000000000000..fc142336a679
--- /dev/null
+++ b/src/bindings/python/flux/idset.py
@@ -0,0 +1,319 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import collections
+import numbers
+
+from _flux._idset import ffi, lib
+from flux.wrapper import Wrapper, WrapperPimpl
+
+IDSET_INVALID_ID = lib.IDSET_INVALID_ID
+IDSET_FLAG_RANGE = lib.IDSET_FLAG_RANGE
+IDSET_FLAG_BRACKETS = lib.IDSET_FLAG_BRACKETS
+
+
+class IDsetIterator:
+    def __init__(self, idset):
+        self._idset = idset
+        self._next = idset.pimpl.first()
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        result = self._next
+        self._next = self._idset.pimpl.next(result)
+        if result == IDSET_INVALID_ID:
+            raise StopIteration
+        return result
+
+
+class IDset(WrapperPimpl):
+    """A Flux idset object
+
+    The IDset class wraps libflux-idset, and encapsulates a set of
+    unordered non-negative integers. See idset_create(3).
+
+    A Python IDset object may be created from a valid RFC22 idset string,
+    e.g. "0", "0-3", "0,5,7", or any Python iterable type as long as the
+    iterable contains only non-negative integers. For example:
+
+    >>> ids = IDset("0-3")
+    >>> ids2 = IDset([0, 1, 2, 3])
+    >>> ids3 = IDset({0, 1, 2, 3})
+
+    """
+
+    class InnerWrapper(Wrapper):
+        def __init__(
+            self,
+            arg="",
+            handle=None,
+        ):
+            if handle is None:
+                handle = lib.idset_decode(arg.encode("utf-8"))
+                if handle == ffi.NULL:
+                    raise ValueError(f"IDset(): Invalid argument: {arg}")
+            super().__init__(
+                ffi,
+                lib,
+                match=ffi.typeof("struct idset *"),
+                prefixes=["idset_", "IDSET_"],
+                destructor=lib.idset_destroy,
+                handle=handle,
+            )
+
+    def __init__(self, arg="", flags=IDSET_FLAG_RANGE, handle=None):
+        super().__init__()
+        if isinstance(arg, IDset):
+            #  Encode incoming IDset to string to generate a copy.
+            #  Note: Trying to use handle=arg.copy() here failed for some
+            #   reason and conversion to string just works, though it is
+            #   slightly less efficient.
+            arg = str(arg)
+        elif isinstance(arg, collections.abc.Iterable) and not isinstance(arg, str):
+            arg = ",".join(str(i) for i in sorted(arg))
+        elif isinstance(arg, int):
+            arg = str(arg)
+
+        self.default_flags = flags
+        try:
+            self.pimpl = self.InnerWrapper(arg=arg, handle=handle)
+        except (TypeError, AttributeError):
+            raise TypeError(
+                "IDset() expected an idset string or iterable, got " + type(arg)
+            )
+
+    def __str__(self):
+        return self.encode()
+
+    def __repr__(self):
+        # Always encode repr with ranges
+        ids = self.encode(flags=IDSET_FLAG_RANGE)
+        return f"IDset('{ids}')"
+
+    def __len__(self):
+        return self.pimpl.count()
+
+    def __iter__(self):
+        return IDsetIterator(self)
+
+    def __contains__(self, i):
+        return self.test(i)
+
+    def __getitem__(self, i):
+        return self.test(i)
+
+    def __setitem__(self, i, value):
+        if value in (0, 1):
+            value = bool(value)
+        if not isinstance(value, bool):
+            typestr = type(value)
+            raise TypeError(f"IDset setitem must be bool or 0,1 not {typestr}")
+        if value:
+            self.set(i)
+        else:
+            self.clear(i)
+
+    def __eq__(self, idset):
+        """Test if two idsets are equal"""
+        return self.equal(idset)
+
+    def __add__(self, arg):
+        """Returns the union of an idset and argument"""
+        return self.union(arg)
+
+    def __sub__(self, arg):
+        """Returns the difference of an idset and argument"""
+        return self.difference(arg)
+
+    def __and__(self, arg):
+        """Returns the set intersection of an idset and argument"""
+        return self.intersect(arg)
+
+    def __or__(self, arg):
+        """Returns the union of an idset and argument"""
+        return self.union(arg)
+
+    def __iadd__(self, arg):
+        """Adds the provided argument to an idset in-place"""
+        self.add(arg)
+        return self
+
+    def __isub__(self, arg):
+        """Subtracts the provided argument to an idset in-place"""
+        self.subtract(arg)
+        return self
+
+    def equal(self, idset):
+        if not isinstance(idset, IDset):
+            raise TypeError
+        return self.pimpl.equal(idset)
+
+    def set_flags(self, flags):
+        """Set default flags for IDset encoding:
+        valid flags are IDSET_FLAG_RANGE and IDSET_FLAG_BRACKETS
+        """
+        self.default_flags = flags
+
+    @staticmethod
+    def _check_index(i, name):
+        if not isinstance(i, numbers.Integral):
+            typestr = type(i)
+            raise TypeError(f"IDset.{name} supports integers, not {typestr}")
+        if i < 0:
+            raise ValueError(f"negative index passed to IDset.{name}")
+
+    def test(self, i):
+        """Test if an id is set in an IDset
+        :param: i: the id to test
+        """
+        self._check_index(i, "test")
+        return self.pimpl.test(i)
+
+    def set(self, start, end=None):
+        """Set an id or range of ids in an IDset
+        :param: start: The first id to set
+        :param: end: (optional) The last id in a range to set
+
+        Returns a copy of self so that this will work:
+        >>> print(IDset().set(0,3))
+
+        """
+        self._check_index(start, "set")
+        if end is None:
+            self.pimpl.set(start)
+        else:
+            self._check_index(end, "set")
+            if end < start:
+                raise ValueError(f"can't set range with {start} > {end}")
+            self.pimpl.range_set(start, end)
+        return self
+
+    def clear(self, start, end=None):
+        """Clear an id or range of ids in an IDset
+        :param: start: The first id to clear
+        :param: end: (optional) The last id in a range to clear
+
+        Returns a copy of self so that this will work:
+        >>> print(IDset("0-9").clear(0,3))
+
+        """
+        self._check_index(start, "clear")
+        if end is None:
+            self.pimpl.clear(start)
+        else:
+            self._check_index(end, "clear")
+            if end < start:
+                raise ValueError(f"can't set range with {start} > {end}")
+            self.pimpl.range_clear(start, end)
+        return self
+
+    def expand(self):
+        """Expand an IDset into a list of integers"""
+        return list(self)
+
+    def count(self):
+        """Return the number of integers in an IDset"""
+        return self.pimpl.count()
+
+    def copy(self):
+        return IDset(handle=self.pimpl.copy())
+
+    def encode(self, flags=None):
+        """Encode an IDset to a string.
+        :param: flags: (optional) flags to influence encoding
+        """
+        if flags is None:
+            flags = self.default_flags
+        #
+        #  N.B. Do not use automatic wrapper call here to avoid leaking
+        #  `char *` result. Instead, explicitly call free() after copying
+        #  the returned string to Python
+        #
+        val = lib.idset_encode(self.handle, flags)
+        result = ffi.string(val)
+        lib.free(val)
+        return result.decode("utf-8")
+
+    def first(self):
+        """Return the first id set in an IDset"""
+        return self.pimpl.first()
+
+    def next(self, i):
+        """Return the next id set in an IDset after value i"""
+        self._check_index(i, "next")
+        return self.pimpl.next(i)
+
+    def last(self):
+        """Return the greatest id set in an IDset"""
+        return self.pimpl.last()
+
+    def add(self, arg):
+        """Add all ids or values in arg to IDset
+        :param: arg: IDset, string, or iterable of integers to add
+        """
+        if not isinstance(arg, IDset):
+            arg = IDset(arg)
+        self.pimpl.add(arg)
+        return self
+
+    def subtract(self, arg):
+        """subtract all ids or values in arg from IDset
+        :param: arg: IDset, string, or iterable of integers to subtract
+        """
+        if not isinstance(arg, IDset):
+            arg = IDset(arg)
+        self.pimpl.subtract(arg)
+        return self
+
+    def union(self, *args):
+        """Return the union of the current IDset and all args
+
+        All args will be converted to IDsets if possible, i.e. any IDset,
+        valid idset string, or iterable composed of integers will work.
+        """
+        result = self.copy()
+        for idset in args:
+            if not isinstance(idset, IDset):
+                idset = IDset(idset)
+            result = IDset(handle=result.pimpl.union(idset))
+        return result
+
+    def intersect(self, *args):
+        """Return the set intersection of the target IDset and all args
+
+        All args will be converted to IDsets if possible, i.e. any IDset,
+        valid idset string, or iterable composed of integers will work.
+        """
+        result = self.copy()
+        for idset in args:
+            if not isinstance(idset, IDset):
+                idset = IDset(idset)
+            result = IDset(handle=result.pimpl.intersect(idset))
+        return result
+
+    def difference(self, *args):
+        """Return the set difference of the target IDset and all args
+
+        All args will be converted to IDsets if possible, i.e. any IDset,
+        valid idset string, or iterable composted of integers will work.
+        """
+        result = self.copy()
+        for idset in args:
+            if not isinstance(idset, IDset):
+                idset = IDset(idset)
+            result = IDset(handle=result.pimpl.difference(idset))
+        return result
+
+
+def decode(string):
+    """Decode an idset string and return IDset object"""
+    return IDset(string)
diff --git a/src/bindings/python/flux/importer.py b/src/bindings/python/flux/importer.py
new file mode 100644
index 000000000000..3df439e68a9d
--- /dev/null
+++ b/src/bindings/python/flux/importer.py
@@ -0,0 +1,65 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import importlib
+import os
+import pkgutil
+import sys
+
+
+def import_plugins_pkg(ns_pkg):
+    """Import all modules found in the namespace package ``ns_pkg``"""
+    result = {}
+    for finder, name, ispkg in pkgutil.iter_modules(ns_pkg.__path__):
+        try:
+            result[name] = importlib.import_module(f"{ns_pkg.__name__}.{name}")
+        except ImportError as exc:
+            raise ImportError(f"failed to import {name} plugin: {exc}")
+    return result
+
+
+def import_plugins(pkg_name, pluginpath=None):
+    """Load plugins from a namespace package and optional additional paths
+
+    A plugin in pluginpath with the same name as an existing plugin will
+    take precedence
+    """
+    if pluginpath is not None:
+        sys.path[1:1] = pluginpath
+
+    try:
+        #  Load 'pkg_name' as a namespace plugin
+        #
+        #  Note: importlib.import_module() will raise ModuleNotFoundError
+        #  if pkg_name points to an empty directory, but we just want to
+        #  return an empty list in this case:
+        #
+        pkg = importlib.import_module(pkg_name)
+    except ModuleNotFoundError:
+        return {}
+
+    plugins = import_plugins_pkg(pkg)
+
+    if pluginpath is not None:
+        #  Undo any added pluginpath elements.
+        for path in pluginpath:
+            sys.path.remove(path)
+
+    return plugins
+
+
+def import_path(file_path):
+    """Import a module directly from file_path"""
+
+    module_name = os.path.basename(file_path).rstrip(".py")
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
diff --git a/src/bindings/python/flux/job.py b/src/bindings/python/flux/job.py
deleted file mode 100644
index 539c2f221c3f..000000000000
--- a/src/bindings/python/flux/job.py
+++ /dev/null
@@ -1,761 +0,0 @@
-###############################################################
-# Copyright 2014 Lawrence Livermore National Security, LLC
-# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
-#
-# This file is part of the Flux resource manager framework.
-# For details, see https://github.com/flux-framework.
-#
-# SPDX-License-Identifier: LGPL-3.0
-###############################################################
-from __future__ import print_function
-
-import os
-import math
-import json
-import errno
-import datetime
-import collections
-
-import six
-import yaml
-
-from flux import constants
-from flux.wrapper import Wrapper
-from flux.util import check_future_error, parse_fsd
-from flux.future import Future
-from flux.rpc import RPC
-from _flux._core import ffi, lib
-
-try:
-    # pylint: disable=invalid-name
-    collectionsAbc = collections.abc
-except AttributeError:
-    # pylint: disable=invalid-name
-    collectionsAbc = collections
-
-
-class JobWrapper(Wrapper):
-    def __init__(self):
-        super(JobWrapper, self).__init__(ffi, lib, prefixes=["flux_job_"])
-
-
-RAW = JobWrapper()
-
-
-def _convert_jobspec_arg_to_string(jobspec):
-    """
-    Convert a jobspec argument into a string.  A valid jobspec argument is:
-    - An instance of the Jobspec class
-    - A string (i.e., bytes, str, or unicode)
-
-    :raises EnvironmentError: jobspec is None or NULL
-    :raises TypeError: jobspec is neither a Jobspec nor a string
-    """
-    if isinstance(jobspec, Jobspec):
-        jobspec = jobspec.dumps()
-    elif isinstance(jobspec, six.text_type):
-        jobspec = jobspec.encode("utf-8")
-    elif jobspec is None or jobspec == ffi.NULL:
-        # catch this here rather than in C for a better error message
-        raise EnvironmentError(errno.EINVAL, "jobspec must not be None/NULL")
-    elif not isinstance(jobspec, six.binary_type):
-        raise TypeError(
-            "jobpsec must be a Jobspec or string (either binary or unicode)"
-        )
-    return jobspec
-
-
-class SubmitFuture(Future):
-    def get_id(self):
-        return submit_get_id(self)
-
-
-def submit_async(
-    flux_handle,
-    jobspec,
-    priority=lib.FLUX_JOB_PRIORITY_DEFAULT,
-    waitable=False,
-    debug=False,
-):
-    """Ask Flux to run a job, without waiting for a response
-
-    Submit a job to Flux.  This method returns immediately with a
-    Flux Future, which can be used obtain the job ID later.
-
-    :param flux_handle: handle for Flux broker from flux.Flux()
-    :type flux_handle: Flux
-    :param jobspec: jobspec defining the job request
-    :type jobspec: Jobspec or its string encoding
-    :param priority: job priority 0 (lowest) through 31 (highest)
-        (default is 16).  Priorities 0 through 15 are restricted to
-        the instance owner.
-    :type priority: int
-    :param waitable: allow result to be fetched with job.wait()
-        (default is False).  Waitable=True is restricted to the
-        instance owner.
-    :type waitable: bool
-    :param debug: enable job manager debugging events to job eventlog
-        (default is False)
-    :type debug: bool
-    :returns: a Flux Future object for obtaining the assigned jobid
-    :rtype: Future
-    """
-    jobspec = _convert_jobspec_arg_to_string(jobspec)
-    flags = 0
-    if waitable:
-        flags |= constants.FLUX_JOB_WAITABLE
-    if debug:
-        flags |= constants.FLUX_JOB_DEBUG
-    future_handle = RAW.submit(flux_handle, jobspec, priority, flags)
-    return SubmitFuture(future_handle)
-
-
-@check_future_error
-def submit_get_id(future):
-    """Get job ID from a Future returned by job.submit_async()
-
-    Process a response to a Flux job submit request.  This method blocks
-    until the response is received, then decodes the result to obtain
-    the assigned job ID.
-
-    :param future: a Flux future object returned by job.submit_async()
-    :type future: Future
-    :returns: job ID
-    :rtype: int
-    """
-    if future is None or future == ffi.NULL:
-        raise EnvironmentError(errno.EINVAL, "future must not be None/NULL")
-    future.wait_for()  # ensure the future is fulfilled
-    jobid = ffi.new("flux_jobid_t[1]")
-    RAW.submit_get_id(future, jobid)
-    return int(jobid[0])
-
-
-def submit(
-    flux_handle,
-    jobspec,
-    priority=lib.FLUX_JOB_PRIORITY_DEFAULT,
-    waitable=False,
-    debug=False,
-):
-    """Submit a job to Flux
-
-    Ask Flux to run a job, blocking until a job ID is assigned.
-
-    :param flux_handle: handle for Flux broker from flux.Flux()
-    :type flux_handle: Flux
-    :param jobspec: jobspec defining the job request
-    :type jobspec: Jobspec or its string encoding
-    :param priority: job priority 0 (lowest) through 31 (highest)
-        (default is 16).  Priorities 0 through 15 are restricted to
-        the instance owner.
-    :type priority: int
-    :param waitable: allow result to be fetched with job.wait()
-        (default is False).  Waitable=true is restricted to the
-        instance owner.
-    :type waitable: bool
-    :param debug: enable job manager debugging events to job eventlog
-        (default is False)
-    :type debug: bool
-    :returns: job ID
-    :rtype: int
-    """
-    future = submit_async(flux_handle, jobspec, priority, waitable, debug)
-    return future.get_id()
-
-
-class JobWaitFuture(Future):
-    def get_status(self):
-        return wait_get_status(self)
-
-
-def wait_async(flux_handle, jobid=lib.FLUX_JOBID_ANY):
-    """Wait for a job to complete, asynchronously
-
-    Submit a request to wait for job completion.  This method returns
-    immediately with a Flux Future, which can be used to process
-    the result later.
-
-    Only jobs submitted with waitable=True can be waited for.
-
-    :param flux_handle: handle for Flux broker from flux.Flux()
-    :type flux_handle: Flux
-    :param jobid: the job ID to wait for (default is any waitable job)
-    :returns: a Flux Future object for obtaining the job result
-    :rtype: Future
-    """
-    future_handle = RAW.wait(flux_handle, jobid)
-    return JobWaitFuture(future_handle)
-
-
-JobWaitResult = collections.namedtuple("JobWaitResult", "jobid, success, errstr")
-
-
-@check_future_error
-def wait_get_status(future):
-    """Get job status from a Future returned by job.wait_async()
-
-    Process a response to a Flux job wait request.  This method blocks
-    until the response is received, then decodes the result to obtain
-    the job status.
-
-    :param future: a Flux future object returned by job.wait_async()
-    :type future: Future
-    :returns: job status, a tuple of: Job ID (int), success (bool),
-        and an error (string) if success=False
-    :rtype: tuple
-    """
-    if future is None or future == ffi.NULL:
-        raise EnvironmentError(errno.EINVAL, "future must not be None/NULL")
-    future.wait_for()  # ensure the future is fulfilled
-    success = ffi.new("bool[1]")
-    errstr = ffi.new("const char *[1]")
-    jobid = ffi.new("flux_jobid_t[1]")
-    RAW.wait_get_id(future, jobid)
-    RAW.wait_get_status(future, success, errstr)
-    return JobWaitResult(int(jobid[0]), bool(success[0]), ffi.string(errstr[0]))
-
-
-def wait(flux_handle, jobid=lib.FLUX_JOBID_ANY):
-    """Wait for a job to complete
-
-    Submit a request to wait for job completion, blocking until a
-    response is received, then return the job status.
-
-    Only jobs submitted with waitable=True can be waited for.
-
-    :param flux_handle: handle for Flux broker from flux.Flux()
-    :type flux_handle: Flux
-    :param jobid: the job ID to wait for (default is any waitable job)
-    :returns: job status, a tuple of: Job ID (int), success (bool),
-        and an error (string) if success=False
-    :rtype: tuple
-    """
-    future = wait_async(flux_handle, jobid)
-    return future.get_status()
-
-
-class JobListRPC(RPC):
-    def get_jobs(self):
-        return self.get()["jobs"]
-
-
-# Due to subtleties in the python bindings and this call, this binding
-# is more of a reimplementation of flux_job_list() instead of calling
-# the flux_job_list() C function directly.  Some reasons:
-#
-# - Desire to return a Python RPC class and use its get() method
-# - Desired return value is json array, not a single value
-#
-# pylint: disable=dangerous-default-value
-def job_list(flux_handle, max_entries=1000, attrs=[], userid=os.geteuid(), states=0):
-    payload = {
-        "max_entries": max_entries,
-        "attrs": attrs,
-        "userid": userid,
-        "states": states,
-    }
-    return JobListRPC(flux_handle, "job-info.list", payload)
-
-
-def job_list_inactive(flux_handle, since=0.0, max_entries=1000, attrs=[]):
-    payload = {"since": since, "max_entries": max_entries, "attrs": attrs}
-    return JobListRPC(flux_handle, "job-info.list-inactive", payload)
-
-
-def _validate_keys(expected, given, keys_optional=False, allow_additional=False):
-    if not isinstance(expected, set):
-        expected = set(expected)
-    if not isinstance(given, set):
-        given = set(given)
-    if not keys_optional:
-        for req_key in expected.difference(given):
-            raise ValueError("Missing key ({})".format(req_key))
-    if not allow_additional:
-        for extraneous_key in given.difference(expected):
-            raise ValueError("Extraneous key ({})".format(extraneous_key))
-
-
-def validate_jobspec(jobspec, require_version=None):
-    """
-    Validates the jobspec by attempting to construct a Jobspec object.  If no
-    exceptions are thrown during construction, then the jobspec is assumed to be
-    valid and this function returns True.  If the jobspec is invalid, the
-    relevant exception is thrown (i.e., TypeError, ValueError, EnvironmentError)
-
-    By default, the validation function will read the `version` key in the
-    jobspec to determine which Jobspec object to instantiate. An optional
-    `require_version` is included to override this behavior and force a
-    particular class to be used.
-
-    :param jobspec: a Jobspec object or JSON string
-    :param require_version: jobspec version to use, if not provided,
-                            the value of jobspec['version'] is used
-    :raises ValueError:
-    :raises TypeError:
-    :raises EnvironmentError:
-    """
-    jobspec_str = _convert_jobspec_arg_to_string(jobspec)
-    jobspec_obj = json.loads(jobspec_str)
-    if jobspec_obj is None:
-        return (1, "Unable to parse JSON")
-    _validate_keys(Jobspec.top_level_keys, jobspec_obj.keys())
-    if require_version == 1 or jobspec_obj.get("version", 0) == 1:
-        JobspecV1(**jobspec_obj)
-    else:
-        Jobspec(**jobspec_obj)
-    return True
-
-
-class Jobspec(object):
-    top_level_keys = set(["resources", "tasks", "version", "attributes"])
-
-    def __init__(self, resources, tasks, **kwargs):
-        """
-        Constructor for Canonical Jobspec, as described in RFC 14
-
-        :param resources:  dictionary following the specification in RFC14 for
-                           the `resources` top-level key
-        :param tasks: dictionary following the specification in RFC14 for the
-                      `tasks` top-level key
-        :param attributes: dictionary following the specification in RFC14 for
-                           the `attributes` top-level key
-        :param version: included to allow for usage like JobspecV1(**jobspec)
-        :raises ValueError:
-        :raises TypeError:
-        """
-
-        # ensure that no unknown keyword arguments are used
-        _validate_keys(
-            ["attributes", "version"],
-            kwargs,
-            keys_optional=False,
-            allow_additional=False,
-        )
-
-        if "version" not in kwargs:
-            raise ValueError("version must be set")
-        version = kwargs["version"]
-        attributes = kwargs.get("attributes", None)
-
-        if not isinstance(resources, collectionsAbc.Sequence):
-            raise TypeError("resources must be a sequence")
-        if not isinstance(tasks, collectionsAbc.Sequence):
-            raise TypeError("tasks must be a sequence")
-        if not isinstance(version, int):
-            raise TypeError("version must be an integer")
-        if attributes is not None and not isinstance(
-            attributes, collectionsAbc.Mapping
-        ):
-            raise TypeError("attributes must be a mapping")
-        elif version < 1:
-            raise ValueError("version must be >= 1")
-
-        self.jobspec = {
-            "resources": resources,
-            "tasks": tasks,
-            "attributes": attributes,
-            "version": version,
-        }
-
-        for res in self:
-            self._validate_resource(res)
-
-        for task in tasks:
-            self._validate_task(task)
-
-        if attributes is not None:
-            self._validate_attributes(attributes)
-
-    @classmethod
-    def from_yaml_stream(cls, yaml_stream):
-        jobspec = yaml.safe_load(yaml_stream)
-        _validate_keys(cls.top_level_keys, jobspec.keys())
-        return cls(**jobspec)
-
-    @classmethod
-    def from_yaml_file(cls, filename):
-        with open(filename, "rb") as infile:
-            return cls.from_yaml_stream(infile)
-
-    @staticmethod
-    def _validate_complex_range(range_dict):
-        if "min" not in range_dict:
-            raise ValueError("min must be in range")
-        if len(range_dict) > 1:
-            _validate_keys(["min", "max", "operator", "operand"], range_dict.keys())
-        for key in ["min", "max", "operand"]:
-            if key not in range_dict:
-                continue
-            if not isinstance(range_dict[key], six.integer_types):
-                raise TypeError("{} must be an int".format(key))
-            elif range_dict[key] < 1:
-                raise ValueError("{} must be > 0".format(key))
-        valid_operator_values = ["+", "*", "^"]
-        if (
-            "operator" in range_dict
-            and range_dict["operator"] not in valid_operator_values
-        ):
-            raise ValueError("operator must be one of {}".format(valid_operator_values))
-
-    @classmethod
-    def _validate_resource(cls, res):
-        if not isinstance(res, collectionsAbc.Mapping):
-            raise TypeError("resource must be a mapping")
-
-        # validate the 'type' key
-        if "type" not in res:
-            raise ValueError("type is a required key for resources")
-        if not isinstance(res["type"], six.string_types):
-            raise TypeError("type must be a string")
-
-        # validate the 'count' key
-        if "count" not in res:
-            raise ValueError("count is a required key for resources")
-        count = res["count"]
-        if isinstance(count, collectionsAbc.Mapping):
-            cls._validate_complex_range(count)
-        elif not isinstance(count, six.integer_types):
-            raise TypeError("count must be an int or mapping")
-        elif count < 1:
-            raise ValueError("count must be > 0")
-
-        # validate the string keys
-        for key in ["id", "unit", "label"]:
-            if key in res and not isinstance(res[key], six.string_types):
-                raise TypeError("{} must be a string".format(key))
-
-        # validate the 'exclusive' key
-        if "exclusive" in res:
-            if res["exclusive"] not in [True, False]:
-                raise TypeError("exclusive must be a boolean")
-
-        # validate that slots have a 'label'
-        if res["type"] == "slot" and "label" not in res:
-            raise ValueError("slots must have labels")
-
-    @staticmethod
-    def _validate_task(task):
-        if not isinstance(task, collectionsAbc.Mapping):
-            raise TypeError("task must be a mapping")
-
-        _validate_keys(["command", "slot", "count"], task.keys(), allow_additional=True)
-
-        if not isinstance(task["count"], collectionsAbc.Mapping):
-            raise TypeError("count must be a mapping")
-
-        if not isinstance(task["slot"], six.string_types):
-            raise TypeError("slot must be a string")
-
-        if "attributes" in task and not isinstance(
-            task["attributes"], collectionsAbc.Mapping
-        ):
-            raise TypeError("count must be a mapping")
-
-        command = task["command"]
-        if not (
-            (  # sequence of strings - N.B. also true for just a plain string
-                isinstance(command, collectionsAbc.Sequence)
-                and all(isinstance(x, six.string_types) for x in command)
-            )
-        ) or isinstance(command, six.string_types):
-            raise TypeError("command must be a list of strings")
-
-    @staticmethod
-    def _validate_attributes(attributes):
-        _validate_keys(["system", "user"], attributes.keys(), keys_optional=True)
-
-    @staticmethod
-    def _create_resource(res_type, count, with_child=None):
-        if with_child is not None and not isinstance(
-            with_child, collectionsAbc.Sequence
-        ):
-            raise TypeError("child resource must None or a sequence")
-        elif with_child is not None and isinstance(with_child, six.string_types):
-            raise TypeError("child resource must not be a string")
-        if not count > 0:
-            raise ValueError("resource count must be > 0")
-
-        res = {"type": res_type, "count": count}
-
-        if with_child:
-            res["with"] = with_child
-        return res
-
-    @classmethod
-    def _create_slot(cls, label, count, with_child):
-        slot = cls._create_resource("slot", count, with_child)
-        slot["label"] = label
-        return slot
-
-    @property
-    def duration(self):
-        try:
-            return self.jobspec["attributes"]["system"]["duration"]
-        except KeyError:
-            return None
-
-    @duration.setter
-    def duration(self, duration):
-        """
-        Assign a time limit to the job.  The duration may be:
-        - an int or float in seconds
-        - a string in Flux Standard Duration
-        - a python datetime.timedelta
-        A duration of zero is interpreted as "not set".
-        """
-        if isinstance(duration, six.string_types):
-            time = parse_fsd(duration)
-        elif isinstance(duration, datetime.timedelta):
-            time = duration.total_seconds()
-        elif isinstance(duration, (float, int)):
-            time = float(duration)
-        else:
-            raise TypeError("duration must be an int, float, string, or timedelta")
-        if time < 0:
-            raise ValueError("duration must not be negative")
-        if math.isnan(time) or math.isinf(time):
-            raise ValueError("duration must be a normal, finite value")
-        self.setattr("system.duration", time)
-
-    @property
-    def cwd(self):
-        """
-        Get working directory of job.
-        """
-        try:
-            return self.jobspec["attributes"]["system"]["cwd"]
-        except KeyError:
-            return None
-
-    @cwd.setter
-    def cwd(self, cwd):
-        """
-        Set working directory of job. The cwd may be:
-        - a pathlib object (if py 3.6+)
-        - a string
-        """
-        if six.PY3 and isinstance(cwd, os.PathLike):
-            cwd = os.fspath(cwd)
-        if not isinstance(cwd, six.string_types):
-            raise ValueError("cwd must be a string")
-        self.setattr("system.cwd", cwd)
-
-    @property
-    def environment(self):
-        """
-        Get (entire) environment of job.
-        """
-        try:
-            return self.jobspec["attributes"]["system"]["environment"]
-        except KeyError:
-            return None
-
-    @environment.setter
-    def environment(self, environ):
-        """
-        Set (entire) environment of job.
-
-        Does a direct assignment (i.e., no deep copy), so future modifications
-        to the `environ` will be reflected in the jobspec.
-        """
-        if not isinstance(environ, collectionsAbc.Mapping):
-            raise ValueError("environment must be a mapping")
-        self.setattr("system.environment", environ)
-
-    def _set_treedict(self, in_dict, key, val):
-        """
-        _set_treedict(d, "a.b.c", 42) is like d[a][b][c] = 42
-        but levels are created on demand.
-        """
-        path = key.split(".", 1)
-        if len(path) == 2:
-            self._set_treedict(in_dict.setdefault(path[0], {}), path[1], val)
-        else:
-            in_dict[key] = val
-
-    def setattr(self, key, val):
-        """
-        set job attribute
-        """
-        self._set_treedict(self.jobspec, "attributes." + key, val)
-
-    def setattr_shell_option(self, key, val):
-        """
-        set job attribute: shell option
-        """
-        self.setattr("system.shell.options." + key, val)
-
-    def dumps(self, **kwargs):
-        return json.dumps(self.jobspec, **kwargs)
-
-    @property
-    def resources(self):
-        return self.jobspec.get("resources", None)
-
-    @property
-    def tasks(self):
-        return self.jobspec.get("tasks", None)
-
-    @property
-    def attributes(self):
-        return self.jobspec.get("attributes", None)
-
-    @property
-    def version(self):
-        return self.jobspec.get("version", None)
-
-    def __iter__(self):
-        """
-        Iterate over the resources in the `resources` section of the jobspec.
-
-        Performs a depth-first, pre-order traversal.
-        """
-
-        def iter_helper(res_list):
-            for resource in res_list:
-                yield resource
-                children = resource.get("with", [])
-                # PY2: convert to `yield from` after dropping 2.7
-                for res in iter_helper(children):
-                    yield res
-
-        return iter_helper(self.resources)
-
-    def resource_walk(self):
-        """
-        Traverse the resources in the `resources` section of the jobspec.
-
-        Performs a depth-first, pre-order traversal. Yields a tuple containing
-        (parent, resource, count).  `parent` is None when `resource` is a
-        top-level resource. `count` is the number of that resource including the
-        multiplicative effects of the `with` clause in ancestor resources.  For
-        example, the following resource section, will yield a count of 2 for the
-        `slot` and a count of 8 for the `core` resource.
-
-        ```yaml
-        - type: slot
-          count: 2
-          with:
-            - type: core
-              count: 4
-        ```
-        """
-
-        def walk_helper(res_list, parent, count):
-            for resource in res_list:
-                res_count = count * resource["count"]
-                yield (parent, resource, res_count)
-                children = resource.get("with", [])
-                # PY2: convert to `yield from` after dropping 2.7
-                for walk_tuple in walk_helper(children, resource, res_count):
-                    yield walk_tuple
-
-        return walk_helper(self.resources, None, 1)
-
-    def resource_counts(self):
-        """
-        Compute the counts of each resource type in the jobspec
-
-        The following jobspec would return
-        `{ "slot": 12, "core": 18, "memory": 242 }`
-
-        ```yaml
-        - type: slot
-          count: 2
-          with:
-            - type: core
-              count: 4
-            - type: memory
-              count: 1
-              unit: GB
-        - type: slot
-          count: 10
-          with:
-            - type: core
-              count: 1
-            - type: memory
-              count: 24
-              unit: GB
-        ```
-
-        Note: the current implementation ignores the `unit` label and assumes
-        they are consist across resources
-        """
-        count_dict = collections.defaultdict(lambda: 0)
-        for _, resource, count in self.resource_walk():
-            count_dict[resource["type"]] += count
-        return count_dict
-
-
-class JobspecV1(Jobspec):
-    def __init__(self, resources, tasks, **kwargs):
-        """
-        Constructor for Version 1 of the Jobspec
-
-        :param resources:  dictionary following the specification in RFC14 for
-                           the `resources` top-level key
-        :param tasks: dictionary following the specification in RFC14 for the
-                      `tasks` top-level key
-        :param attributes: dictionary following the specification in RFC14 for
-                           the `attributes` top-level key
-        :param version: must be 1, included to allow for usage like
-                        JobspecV1(**jobspec)
-        """
-
-        # ensure that no unknown keyword arguments are used
-        _validate_keys(
-            ["attributes", "version"],
-            kwargs,
-            keys_optional=True,
-            allow_additional=False,
-        )
-
-        if "version" not in kwargs:
-            kwargs["version"] = 1
-        elif kwargs["version"] != 1:
-            raise ValueError("version must be 1")
-        super(JobspecV1, self).__init__(resources, tasks, **kwargs)
-
-    @classmethod
-    def from_command(
-        cls, command, num_tasks=1, cores_per_task=1, gpus_per_task=None, num_nodes=None
-    ):
-        """
-        Factory function that builds the minimum legal v1 jobspec.
-
-        Use setters to assign additional properties.
-        """
-        if not isinstance(num_tasks, int) or num_tasks < 1:
-            raise ValueError("task count must be a integer >= 1")
-        if not isinstance(cores_per_task, int) or cores_per_task < 1:
-            raise ValueError("cores per task must be an integer >= 1")
-        if gpus_per_task is not None:
-            if not isinstance(gpus_per_task, int) or gpus_per_task < 0:
-                raise ValueError("gpus per task must be an integer >= 0")
-        if num_nodes is not None:
-            if not isinstance(num_nodes, int) or num_nodes < 1:
-                raise ValueError("node count must be an integer >= 1 (if set)")
-            if num_nodes > num_tasks:
-                raise ValueError("node count must not be greater than task count")
-        children = [cls._create_resource("core", cores_per_task)]
-        if gpus_per_task not in (None, 0):
-            children.append(cls._create_resource("gpu", gpus_per_task))
-        if num_nodes is not None:
-            num_slots = int(math.ceil(num_tasks / float(num_nodes)))
-            if num_tasks % num_nodes != 0:
-                # N.B. uneven distribution results in wasted task slots
-                task_count_dict = {"total": num_tasks}
-            else:
-                task_count_dict = {"per_slot": 1}
-            slot = cls._create_slot("task", num_slots, children)
-            resource_section = cls._create_resource("node", num_nodes, [slot])
-        else:
-            task_count_dict = {"per_slot": 1}
-            slot = cls._create_slot("task", num_tasks, children)
-            resource_section = slot
-
-        resources = [resource_section]
-        tasks = [{"command": command, "slot": "task", "count": task_count_dict}]
-        attributes = {"system": {"duration": 0}}
-        return cls(resources, tasks, attributes=attributes)
diff --git a/src/bindings/python/flux/job/JobID.py b/src/bindings/python/flux/job/JobID.py
new file mode 100644
index 000000000000..869b86904a17
--- /dev/null
+++ b/src/bindings/python/flux/job/JobID.py
@@ -0,0 +1,122 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+from _flux._core import ffi
+from flux.job._wrapper import _RAW as RAW
+
+
+def id_parse(jobid_str):
+    """
+    returns: An integer jobid
+    :rtype int
+    """
+    jobid = ffi.new("flux_jobid_t[1]")
+    RAW.id_parse(jobid_str, jobid)
+    return int(jobid[0])
+
+
+def id_encode(jobid, encoding="f58"):
+    """
+    returns: Jobid encoded in encoding
+    :rtype str
+    """
+    buflen = 128
+    buf = ffi.new("char[]", buflen)
+    RAW.id_encode(int(jobid), encoding, buf, buflen)
+    return ffi.string(buf, buflen).decode("utf-8")
+
+
+class JobID(int):
+    """Class used to represent a Flux JOBID
+
+    JobID is a subclass of `int`, so may be used in place of integer.
+    However, a JobID may be created from any valid RFC 19 FLUID
+    encoding, including:
+
+     - decimal integer (no prefix)
+     - hexadecimal integer (prefix 0x)
+     - dotted hex (dothex) (xxxx.xxxx.xxxx.xxxx)
+     - kvs dir (dotted hex with `job.` prefix)
+     - RFC19 F58: (Base58 encoding with prefix `ƒ` or `f`)
+     - basemoji (emoji encoding)
+
+    A JobID object also has properties for encoding a JOBID into each
+    of the above representations, e.g. jobid.f85, jobid.words, jobid.dothex...
+
+    """
+
+    def __new__(cls, value):
+        if isinstance(value, JobID):
+            return cls(value.orig_str)
+        if isinstance(value, int):
+            jobid = value
+        else:
+            try:
+                jobid = id_parse(value)
+            except OSError:
+                raise ValueError(f"{value} is not a valid Flux jobid")
+        self = super(cls, cls).__new__(cls, jobid)
+        self.orig_str = str(value)
+        return self
+
+    def encode(self, encoding="dec"):
+        """Encode a JobID to alternate supported format"""
+        return id_encode(self, encoding)
+
+    @property
+    def dec(self):
+        """Return decimal integer representation of a JobID"""
+        return self.encode()
+
+    @property
+    def f58(self):
+        """Return RFC19 F58 representation of a JobID"""
+        return self.encode("f58")
+
+    @property
+    def f58plain(self):
+        """Return RFC19 F58 representation of a JobID with ASCII prefix"""
+        return self.encode("f58plain")
+
+    @property
+    def hex(self):
+        """Return 0x-prefixed hexadecimal representation of a JobID"""
+        return self.encode("hex")
+
+    @property
+    def dothex(self):
+        """Return dotted hexadecimal representation of a JobID"""
+        return self.encode("dothex")
+
+    @property
+    def words(self):
+        """Return words (mnemonic) representation of a JobID"""
+        return self.encode("words")
+
+    @property
+    def emoji(self):
+        """Return emoji representation of a JobID"""
+        return self.encode("emoji")
+
+    @property
+    def kvs(self):
+        """Return KVS directory path of a JobID"""
+        return self.encode("kvs")
+
+    @property
+    def orig(self):
+        """Return the original string used to create the JobID"""
+        return self.orig_str
+
+    def __str__(self):
+        return self.encode("f58")
+
+    def __repr__(self):
+        return f"JobID({self.dec})"
diff --git a/src/bindings/python/flux/job/Jobspec.py b/src/bindings/python/flux/job/Jobspec.py
new file mode 100644
index 000000000000..3ba2c5073690
--- /dev/null
+++ b/src/bindings/python/flux/job/Jobspec.py
@@ -0,0 +1,1035 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+import collections
+import collections.abc as abc
+import datetime
+import errno
+import json
+import math
+import numbers
+import os
+
+import yaml
+from _flux._core import ffi
+from flux import hostlist, idset
+from flux.util import Fileref, parse_fsd, set_treedict
+
+
+def _convert_jobspec_arg_to_string(jobspec):
+    """
+    Convert a jobspec argument into a string.  A valid jobspec argument is:
+    - An instance of the Jobspec class
+    - A string (i.e., bytes, str, or unicode)
+
+    :raises EnvironmentError: jobspec is None or NULL
+    :raises TypeError: jobspec is neither a Jobspec nor a string
+    """
+    if isinstance(jobspec, Jobspec):
+        jobspec = jobspec.dumps()
+    elif isinstance(jobspec, str):
+        jobspec = jobspec.encode("utf-8", errors="surrogateescape")
+    elif jobspec is None or jobspec == ffi.NULL:
+        # catch this here rather than in C for a better error message
+        raise EnvironmentError(errno.EINVAL, "jobspec must not be None/NULL")
+    elif not isinstance(jobspec, bytes):
+        raise TypeError(
+            "jobspec must be a Jobspec or string (either binary or unicode)"
+        )
+    return jobspec
+
+
+def _validate_keys(expected, given, keys_optional=False, allow_additional=False):
+    if not isinstance(expected, set):
+        expected = set(expected)
+    if not isinstance(given, set):
+        given = set(given)
+    if not keys_optional:
+        for req_key in expected.difference(given):
+            raise ValueError("Missing key ({})".format(req_key))
+    if not allow_additional:
+        for extraneous_key in given.difference(expected):
+            raise ValueError("Extraneous key ({})".format(extraneous_key))
+
+
+def _validate_dependency(dep):
+    _validate_keys(["scheme", "value"], dep.keys(), allow_additional=True)
+    if not isinstance(dep["scheme"], str):
+        raise TypeError("dependency scheme must be a string")
+    if not isinstance(dep["value"], str):
+        raise TypeError("dependency value must be a string")
+
+
+def _validate_property_query(name):
+    invalid_chars = set("&'\"`|()")
+    if any((x in invalid_chars) for x in name):
+        raise TypeError(f"invalid character in property '{name}'")
+
+
+def _validate_constraint_op(operator, args):
+    if not isinstance(operator, str):
+        raise TypeError(f"constraint operation {operator} is not a string")
+    if not isinstance(args, abc.Sequence):
+        raise TypeError(f"argument to constraint {operator} must be a sequence")
+    if operator in ["and", "or", "not"]:
+        for constraint in args:
+            _validate_constraint(constraint)
+    elif operator in ["properties"]:
+        for name in args:
+            _validate_property_query(name)
+    elif operator in ["hostlist"]:
+        for hosts in args:
+            hostlist.decode(hosts)
+    elif operator in ["ranks"]:
+        for ranks in args:
+            idset.decode(ranks)
+    else:
+        raise TypeError(f"unknown constraint operator '{operator}'")
+
+
+def _validate_constraint(constraints):
+    """Validate RFC 31 Constraint object"""
+    if not isinstance(constraints, abc.Mapping):
+        raise TypeError("constraints must be a mapping")
+    for operator, arg in constraints.items():
+        _validate_constraint_op(operator, arg)
+
+
+def validate_jobspec(jobspec, require_version=None):
+    """
+    Validates the jobspec by attempting to construct a Jobspec object.  If no
+    exceptions are thrown during construction, then the jobspec is assumed to be
+    valid and this function returns True.  If the jobspec is invalid, the
+    relevant exception is thrown (i.e., TypeError, ValueError, EnvironmentError)
+
+    By default, the validation function will read the `version` key in the
+    jobspec to determine which Jobspec object to instantiate. An optional
+    `require_version` is included to override this behavior and force a
+    particular class to be used.
+
+    :param jobspec: a Jobspec object or JSON string
+    :param require_version: jobspec version to use, if not provided,
+                            the value of jobspec['version'] is used
+    :raises ValueError:
+    :raises TypeError:
+    :raises EnvironmentError:
+    """
+    jobspec_str = _convert_jobspec_arg_to_string(jobspec)
+    jobspec_obj = json.loads(jobspec_str)
+    if jobspec_obj is None:
+        return (1, "Unable to parse JSON")
+    _validate_keys(Jobspec.top_level_keys, jobspec_obj.keys())
+    if require_version == 1 or jobspec_obj.get("version", 0) == 1:
+        JobspecV1(**jobspec_obj)
+    else:
+        Jobspec(**jobspec_obj)
+    return True
+
+
+class Jobspec(object):
+    top_level_keys = set(["resources", "tasks", "version", "attributes"])
+
+    def __init__(self, resources, tasks, **kwargs):
+        """
+        Constructor for Canonical Jobspec, as described in RFC 14
+
+        :param resources:  dictionary following the specification in RFC14 for
+                           the `resources` top-level key
+        :param tasks: dictionary following the specification in RFC14 for the
+                      `tasks` top-level key
+        :param attributes: dictionary following the specification in RFC14 for
+                           the `attributes` top-level key
+        :param version: included to allow for usage like JobspecV1(**jobspec)
+        :raises ValueError:
+        :raises TypeError:
+        """
+
+        # ensure that no unknown keyword arguments are used
+        _validate_keys(
+            ["attributes", "version"],
+            kwargs,
+            keys_optional=False,
+            allow_additional=False,
+        )
+
+        if "version" not in kwargs:
+            raise ValueError("version must be set")
+        version = kwargs["version"]
+        attributes = kwargs.get("attributes", None)
+
+        if not isinstance(resources, abc.Sequence):
+            raise TypeError("resources must be a sequence")
+        if not isinstance(tasks, abc.Sequence):
+            raise TypeError("tasks must be a sequence")
+        if not isinstance(version, int):
+            raise TypeError("version must be an integer")
+        if not isinstance(attributes, abc.Mapping):
+            raise TypeError("attributes must be a mapping")
+        if version < 1:
+            raise ValueError("version must be >= 1")
+
+        self.jobspec = {
+            "resources": resources,
+            "tasks": tasks,
+            "attributes": attributes,
+            "version": version,
+        }
+
+        for res in self:
+            self._validate_resource(res)
+
+        for task in tasks:
+            self._validate_task(task)
+
+        if attributes is not None:
+            self._validate_attributes(attributes)
+
+            if "system" in attributes:
+                self._validate_system_attributes(attributes["system"])
+
+    @classmethod
+    def from_yaml_stream(cls, yaml_stream):
+        """Create a jobspec from a YAML file-like object."""
+        jobspec = yaml.safe_load(yaml_stream)
+        _validate_keys(cls.top_level_keys, jobspec.keys())
+        return cls(**jobspec)
+
+    @classmethod
+    def from_yaml_file(cls, filename):
+        """Create a jobspec from a path to a YAML file."""
+        with open(filename, "rb") as infile:
+            return cls.from_yaml_stream(infile)
+
+    @staticmethod
+    def _validate_complex_range(range_dict):
+        if "min" not in range_dict:
+            raise ValueError("min must be in range")
+        if len(range_dict) > 1:
+            _validate_keys(["min", "max", "operator", "operand"], range_dict.keys())
+        for key in ["min", "max", "operand"]:
+            if key not in range_dict:
+                continue
+            if not isinstance(range_dict[key], int):
+                raise TypeError("{} must be an int".format(key))
+            if range_dict[key] < 1:
+                raise ValueError("{} must be > 0".format(key))
+        valid_operator_values = ["+", "*", "^"]
+        if (
+            "operator" in range_dict
+            and range_dict["operator"] not in valid_operator_values
+        ):
+            raise ValueError("operator must be one of {}".format(valid_operator_values))
+
+    @classmethod
+    # pylint: disable=too-many-branches
+    def _validate_resource(cls, res):
+        if not isinstance(res, abc.Mapping):
+            raise TypeError("resource must be a mapping")
+
+        # validate the 'type' key
+        if "type" not in res:
+            raise ValueError("type is a required key for resources")
+        if not isinstance(res["type"], str):
+            raise TypeError("type must be a string")
+
+        # validate the 'count' key
+        if "count" not in res:
+            raise ValueError("count is a required key for resources")
+        count = res["count"]
+        if isinstance(count, abc.Mapping):
+            cls._validate_complex_range(count)
+        elif not isinstance(count, int):
+            raise TypeError("count must be an int or mapping")
+        else:
+            # node, slot, and core must have count > 0, but allow 0 for
+            # any other resource type.
+            if res["type"] in ["node", "slot", "core"] and count < 1:
+                raise ValueError("node or slot count must be > 0")
+            if count < 0:
+                raise ValueError("count must be >= 0")
+
+        # validate the string keys
+        for key in ["id", "unit", "label"]:
+            if key in res and not isinstance(res[key], str):
+                raise TypeError("{} must be a string".format(key))
+
+        # validate the 'exclusive' key
+        if "exclusive" in res:
+            if res["exclusive"] not in [True, False]:
+                raise TypeError("exclusive must be a boolean")
+
+        # validate that slots have a 'label'
+        if res["type"] == "slot" and "label" not in res:
+            raise ValueError("slots must have labels")
+
+    @staticmethod
+    def _validate_task(task):
+        if not isinstance(task, abc.Mapping):
+            raise TypeError("task must be a mapping")
+
+        _validate_keys(["command", "slot", "count"], task.keys(), allow_additional=True)
+
+        if not isinstance(task["count"], abc.Mapping):
+            raise TypeError("count must be a mapping")
+
+        count = task["count"]
+        if "total" in count:
+            if not isinstance(count["total"], int):
+                raise TypeError("count total must be an int")
+            if count["total"] <= 0:
+                raise ValueError("count total must be > 0")
+        if "per_slot" in count:
+            if not isinstance(count["per_slot"], int):
+                raise TypeError("count per_slot must be an int")
+            if count["per_slot"] <= 0:
+                raise ValueError("count per_slot must be > 0")
+
+        if not isinstance(task["slot"], str):
+            raise TypeError("slot must be a string")
+
+        if "attributes" in task and not isinstance(task["attributes"], abc.Mapping):
+            raise TypeError("count must be a mapping")
+
+        command = task["command"]
+        if len(command) == 0:
+            raise TypeError("command array cannot have length of zero")
+        if not (
+            (  # sequence of strings - N.B. also true for just a plain string
+                isinstance(command, abc.Sequence)
+                and all(isinstance(x, str) for x in command)
+            )
+        ) or isinstance(command, str):
+            raise TypeError("command must be a list of strings")
+
+    @staticmethod
+    def _validate_attributes(attributes):
+        _validate_keys(["system", "user"], attributes.keys(), keys_optional=True)
+
+    @staticmethod
+    def _validate_system_attributes(system):
+        if "dependencies" in system:
+            if not isinstance(system["dependencies"], abc.Sequence):
+                raise TypeError("attributes.system.dependencies must be a list")
+            for dependency in system["dependencies"]:
+                _validate_dependency(dependency)
+        if "constraints" in system:
+            _validate_constraint(system["constraints"])
+
+    @staticmethod
+    def _create_resource(res_type, count, with_child=None, exclusive=False):
+        if with_child is not None and not isinstance(with_child, abc.Sequence):
+            raise TypeError("child resource must None or a sequence")
+        if with_child is not None and isinstance(with_child, str):
+            raise TypeError("child resource must not be a string")
+        if not count > 0:
+            raise ValueError("resource count must be > 0")
+
+        res = {"type": res_type, "count": count}
+
+        if exclusive:
+            res["exclusive"] = True
+
+        if with_child:
+            res["with"] = with_child
+        return res
+
+    @classmethod
+    def _create_slot(cls, label, count, with_child):
+        slot = cls._create_resource("slot", count, with_child)
+        slot["label"] = label
+        return slot
+
+    @property
+    def duration(self):
+        """Job's time limit.
+
+        The duration may be:
+
+        * an int or float in seconds
+        * a string in Flux Standard Duration (see RFC 23)
+        * a python ``datetime.timedelta``
+
+        A duration of zero is interpreted as "not set".
+        """
+        try:
+            return self.jobspec["attributes"]["system"]["duration"]
+        except KeyError:
+            return None
+
+    @duration.setter
+    def duration(self, duration):
+        """Assign a time limit to the job."""
+        if isinstance(duration, str):
+            time = parse_fsd(duration)
+        elif isinstance(duration, datetime.timedelta):
+            time = duration.total_seconds()
+        elif isinstance(duration, (float, int)):
+            time = float(duration)
+        else:
+            raise TypeError("duration must be an int, float, string, or timedelta")
+        if time < 0:
+            raise ValueError("duration must not be negative")
+        if math.isnan(time) or math.isinf(time):
+            raise ValueError("duration must be a normal, finite value")
+        self.setattr("system.duration", time)
+
+    @property
+    def queue(self):
+        """
+        Target queue of job submission
+        """
+        try:
+            return self.jobspec["attributes"]["system"]["queue"]
+        except KeyError:
+            return None
+
+    @queue.setter
+    def queue(self, queue):
+        """
+        Set target submission queue
+        """
+        if not isinstance(queue, str):
+            raise TypeError("queue must be a string")
+        self.setattr("system.queue", queue)
+
+    @property
+    def cwd(self):
+        """
+        Working directory of job.
+        """
+        try:
+            return self.jobspec["attributes"]["system"]["cwd"]
+        except KeyError:
+            return None
+
+    @cwd.setter
+    def cwd(self, cwd):
+        """
+        Set working directory of job. The cwd may be:
+        - a pathlib object (if py 3.6+)
+        - a string
+        """
+        if isinstance(cwd, os.PathLike):
+            cwd = os.fspath(cwd)
+        if not isinstance(cwd, str):
+            raise ValueError("cwd must be a string")
+        self.setattr("system.cwd", cwd)
+
+    @property
+    def environment(self):
+        """
+        Environment of job. Defaults to ``None``.
+        """
+        try:
+            return self.jobspec["attributes"]["system"]["environment"]
+        except KeyError:
+            return None
+
+    @environment.setter
+    def environment(self, environ):
+        """
+        Set (entire) environment of job.
+
+        Does a direct assignment (i.e., no deep copy), so future modifications
+        to the `environ` will be reflected in the jobspec.
+        """
+        if not isinstance(environ, abc.Mapping):
+            raise ValueError("environment must be a mapping")
+        self.setattr("system.environment", environ)
+
+    @property
+    def stdin(self):
+        """Path to use for stdin."""
+        return self._get_io_path("input", "stdin")
+
+    @stdin.setter
+    def stdin(self, path):
+        """Redirect stdin to a file given by `path`, a string or pathlib object."""
+        self._set_io_path("input", "stdin", path)
+
+    @property
+    def stdout(self):
+        """Path to use for stdout."""
+        return self._get_io_path("output", "stdout")
+
+    @stdout.setter
+    def stdout(self, path):
+        """Redirect stdout to a file given by `path`, a string or pathlib object."""
+        self._set_io_path("output", "stdout", path)
+
+    @property
+    def stderr(self):
+        """Path to use for stderr."""
+        return self._get_io_path("output", "stderr")
+
+    @stderr.setter
+    def stderr(self, path):
+        """Redirect stderr to a file given by `path`, a string or pathlib object."""
+        self._set_io_path("output", "stderr", path)
+
+    def _get_io_path(self, iotype, stream_name):
+        """Get the path of a stdio stream, if set.
+
+        :param iotype: the stream type, one of `"input"` or `"output"`
+        :param stream_name: the name of the io stream
+        """
+        try:
+            return self.jobspec["attributes"]["system"]["shell"]["options"][iotype][
+                stream_name
+            ]["path"]
+        except KeyError:
+            return None
+
+    def _set_io_path(self, iotype, stream_name, path):
+        """Set the path of a stdio stream.
+
+        :param iotype: the stream type, one of `"input"` or `"output"`
+        :param stream_name: the name of the io stream
+        :param path: the path to redirect the stream
+        """
+        if isinstance(path, os.PathLike):
+            path = os.fspath(path)
+        if not isinstance(path, str):
+            raise TypeError(
+                "The path must be a string or pathlib object, "
+                f"got {type(path).__name__}"
+            )
+        self.setattr_shell_option("{}.{}.type".format(iotype, stream_name), "file")
+        self.setattr_shell_option("{}.{}.path".format(iotype, stream_name), path)
+
+    def add_file(self, path, data, perms=None, encoding=None):
+        """
+        Add a file to the RFC 14 "files" dictionary in Jobspec. If
+        ``data`` contains newlines or an encoding is explicitly provided,
+        then it is presumed to be the file content. Otherwise, ``data``
+        is a local filesystem path, the contents of which are to be loaded
+        into jobspec. For filesystem
+
+        Args:
+            path (str): path or file name to encode ``data`` as in Jobspec
+            data (dict, str): content of file or a local path to load
+            perms (int): file pemissions, default 0o0600 (octal). If ``data``
+                is a file system path, then permissions of the local file
+                system object will be used.
+            encoding (str): RFC 37 compatible encoding for ``data``. None
+                if ``data`` is a dict or to determine encoding from a file
+                when ``data`` specifies a filesystem path.  O/w, if encoding
+                set, data is a string encoded in specified ``encoding``.
+        """
+        if not (isinstance(data, abc.Mapping) or isinstance(data, str)):
+            raise TypeError("data must be a Mapping or string")
+
+        if perms is None:
+            perms = 0o600
+
+        files = self.jobspec["attributes"]["system"].get("files", {})
+        if encoding is None and "\n" in data:
+            #  Use default encoding of utf-8 if data contains newlines,
+            #  since this is presumed to be file content.
+            encoding = "utf-8"
+        files[path] = Fileref(data, perms=perms, encoding=encoding)
+        self.jobspec["attributes"]["system"]["files"] = files
+
+    def getattr(self, key):
+        """
+        get attribute from jobspec using dotted key notation, e.g.
+        system.duration or optionally attributes.system.duration.
+
+        Raises KeyError if a component of key does not exist.
+        """
+        if not key.startswith("attributes."):
+            key = "attributes." + key
+        value = self.jobspec
+        for attr in key.split("."):
+            value = value.get(attr)
+            if value is None:
+                raise KeyError
+        return value
+
+    def setattr(self, key, val):
+        """
+        set job attribute
+        """
+        if not key.startswith("attributes."):
+            if not key.startswith(("user.", "system.")):
+                key = "system." + key
+            key = "attributes." + key
+        set_treedict(self.jobspec, key, val)
+
+    def setattr_shell_option(self, key, val):
+        """
+        set job attribute: shell option
+        """
+        self.setattr("system.shell.options." + key, val)
+
+    def dumps(self, **kwargs):
+        return json.dumps(self.jobspec, ensure_ascii=False, **kwargs)
+
+    @property
+    def resources(self):
+        """Jobspec resources section"""
+        return self.jobspec.get("resources", None)
+
+    @property
+    def tasks(self):
+        """Jobspec tasks section"""
+        return self.jobspec.get("tasks", None)
+
+    @property
+    def attributes(self):
+        """Jobspec attributes section"""
+        return self.jobspec.get("attributes", None)
+
+    @property
+    def version(self):
+        """Jobspec version section"""
+        return self.jobspec.get("version", None)
+
+    def __iter__(self):
+        """
+        Iterate over the resources in the `resources` section of the jobspec.
+
+        Performs a depth-first, pre-order traversal.
+        """
+
+        def iter_helper(res_list):
+            for resource in res_list:
+                yield resource
+                children = resource.get("with", [])
+                # PY2: convert to `yield from` after dropping 2.7
+                for res in iter_helper(children):
+                    yield res
+
+        return iter_helper(self.resources)
+
+    def resource_walk(self):
+        """
+        Traverse the resources in the `resources` section of the jobspec.
+
+        Performs a depth-first, pre-order traversal. Yields a tuple containing
+        (parent, resource, count).  `parent` is None when `resource` is a
+        top-level resource. `count` is the number of that resource including the
+        multiplicative effects of the `with` clause in ancestor resources.  For
+        example, the following resource section, will yield a count of 2 for the
+        `slot` and a count of 8 for the `core` resource:
+
+        .. code-block:: yaml
+
+            - type: slot
+              count: 2
+              with:
+                - type: core
+                  count: 4
+        """
+
+        def walk_helper(res_list, parent, count):
+            for resource in res_list:
+                res_count = count * resource["count"]
+                yield (parent, resource, res_count)
+                children = resource.get("with", [])
+                # PY2: convert to `yield from` after dropping 2.7
+                for walk_tuple in walk_helper(children, resource, res_count):
+                    yield walk_tuple
+
+        return walk_helper(self.resources, None, 1)
+
+    def resource_counts(self):
+        """
+        Compute the counts of each resource type in the jobspec
+
+        The following jobspec would return
+        ``{ "slot": 12, "core": 18, "memory": 242 }``
+
+        .. code-block:: yaml
+
+            - type: slot
+              count: 2
+              with:
+                - type: core
+                  count: 4
+                - type: memory
+                  count: 1
+                  unit: GB
+            - type: slot
+              count: 10
+              with:
+                - type: core
+                  count: 1
+                - type: memory
+                  count: 24
+                  unit: GB
+
+        Note:
+            the current implementation ignores the `unit` label and assumes
+            they are consist across resources
+        """
+        count_dict = collections.defaultdict(lambda: 0)
+        for _, resource, count in self.resource_walk():
+            count_dict[resource["type"]] += count
+        return count_dict
+
+
+class JobspecV1(Jobspec):
+    def __init__(self, resources, tasks, **kwargs):
+        """
+        Constructor for Version 1 of the Jobspec
+
+        :param resources:  dictionary following the specification in RFC14 for
+                           the `resources` top-level key
+        :param tasks: dictionary following the specification in RFC14 for the
+                      `tasks` top-level key
+        :param attributes: dictionary following the specification in RFC14 for
+                           the `attributes` top-level key
+        :param version: must be 1, included to allow for usage like
+                        JobspecV1(**jobspec)
+        """
+
+        # ensure that no unknown keyword arguments are used
+        _validate_keys(
+            ["attributes", "version"],
+            kwargs,
+            keys_optional=True,
+            allow_additional=False,
+        )
+
+        if "version" not in kwargs:
+            kwargs["version"] = 1
+        elif kwargs["version"] != 1:
+            raise ValueError("version must be 1")
+
+        super(JobspecV1, self).__init__(resources, tasks, **kwargs)
+
+        # validate V1 specific requirements:
+        self._v1_validate(resources, tasks, kwargs)
+
+    @staticmethod
+    def _v1_validate(resources, tasks, kwargs):
+        # process extra V1 attributes requirements:
+
+        # attributes already required by base Jobspec validator
+        attributes = kwargs["attributes"]
+
+        # attributes.system.duration is required
+        if "system" not in attributes:
+            raise ValueError("attributes.system is a required key")
+        system = attributes["system"]
+        if not isinstance(system, abc.Mapping):
+            raise ValueError("attributes.system must be a mapping")
+        if "duration" not in system:
+            raise ValueError("attributes.system.duration is a required key")
+        if not isinstance(system["duration"], numbers.Number):
+            raise ValueError("attributes.system.duration must be a number")
+
+    @classmethod
+    # pylint: disable=too-many-branches,too-many-locals,too-many-statements
+    def per_resource(
+        cls,
+        command,
+        ncores=None,
+        nnodes=None,
+        per_resource_type=None,
+        per_resource_count=None,
+        gpus_per_node=None,
+        exclusive=False,
+    ):
+        """
+        Factory function that builds a v1 jobspec from an explicit count
+        of nodes or cores and a number of tasks per one of these resources.
+
+        Use setters to assign additional properties.
+
+        Args:
+            ncores: Total number of cores to allocate
+            nnodes: Total number of nodes to allocate
+            per_resource_type: (optional) Type of resource over which to
+                               schedule a count of tasks. Only "node" or
+                               "core" are currently supported.
+            per_resource_count: (optional) Count of tasks per
+                                `per_resource_type`
+            gpus_per_node: With nnodes, request a number of gpus per node
+            exclusive: with nnodes, request whole nodes exclusively
+        """
+
+        #  Handle per-resource specification:
+        #  It is an error to specify one of per_resource_{type,count} and
+        #   not the other:
+        #
+        per_resource = None
+        if per_resource_type is not None and per_resource_count is not None:
+            if not isinstance(per_resource_type, str):
+                raise ValueError("per_resource_type must be a string")
+            if per_resource_type not in ("node", "core"):
+                raise ValueError(
+                    f"Invalid per_resource_type='{per_resource_type}' specified"
+                )
+            if not isinstance(per_resource_count, int):
+                raise ValueError("per_resource_count must be an integer")
+            if per_resource_count < 1:
+                raise ValueError("per_resource_count must be >= 1")
+
+            per_resource = {"type": per_resource_type, "count": per_resource_count}
+        elif per_resource_type is not None:
+            raise ValueError("must specify a per_resource_count with per_resource_type")
+        elif per_resource_count is not None:
+            raise ValueError("must specify a per_resource_type with per_resource_count")
+
+        if ncores is not None:
+            if not isinstance(ncores, int) or ncores < 1:
+                raise ValueError("ncores must be an integer >= 1")
+        if gpus_per_node is not None:
+            if not isinstance(gpus_per_node, int) or gpus_per_node < 0:
+                raise ValueError("gpus_per_node must be an integer >= 0")
+            if not nnodes:
+                raise ValueError("gpus_per_node must be specified with nnodes")
+        if nnodes is not None:
+            if not isinstance(nnodes, int) or nnodes < 1:
+                raise ValueError("nnodes must be an integer >= 1")
+        elif exclusive:
+            raise ValueError("exclusive can only be set with a node count")
+
+        nslots = None
+        slot_size = 1
+        if nnodes and ncores:
+            #  Request ncores across nnodes, actually running a given
+            #   number of tasks per node or core
+            if ncores < nnodes:
+                raise ValueError("number of cores cannot be less than nnodes")
+            if ncores % nnodes != 0:
+                raise ValueError(
+                    "number of cores must be evenly divisible by node count"
+                )
+            #
+            #  With nnodes, nslots is slots/node (total_slots=slots*nodes)
+            nslots = 1
+            slot_size = int(ncores / nnodes)
+        elif ncores:
+            #  Request ncores total, actually running a given
+            #   number of tasks per node or core
+            #
+            #  Without nnodes, nslots is total number of slots:
+            nslots = ncores
+            slot_size = 1
+        elif nnodes:
+            #  Request nnodes total with a given number of tasks per node
+            #   or per core. (requires exclusive)
+            if not exclusive:
+                raise ValueError(
+                    "Specifying nnodes also requires ncores or exclusive",
+                )
+            #  With nnodes, nslots is slots/node (total_slots=slots*nodes)
+            nslots = 1
+            slot_size = 1
+        else:
+            raise ValueError("must specify node or core count with per_resource")
+
+        children = [cls._create_resource("core", slot_size)]
+        if gpus_per_node:
+            children.append(cls._create_resource("gpu", gpus_per_node))
+
+        slot = cls._create_slot("task", nslots, children)
+
+        if nnodes:
+            resources = cls._create_resource("node", nnodes, [slot], exclusive)
+        else:
+            resources = slot
+
+        resources = [resources]
+        tasks = [{"command": command, "slot": "task", "count": {"per_slot": 1}}]
+        attributes = {"system": {"duration": 0}}
+        if per_resource:
+            set_treedict(attributes, "system.shell.options.per-resource", per_resource)
+        return cls(resources, tasks, attributes=attributes)
+
+    @classmethod
+    # pylint: disable=too-many-branches
+    def from_command(
+        cls,
+        command,
+        num_tasks=1,
+        cores_per_task=1,
+        gpus_per_task=None,
+        num_nodes=None,
+        exclusive=False,
+    ):
+        """
+        Factory function that builds the minimum legal v1 jobspec.
+
+        Use setters to assign additional properties.
+
+        :param command: command to execute
+        :type command: iterable of str
+        :param num_tasks: number of MPI tasks to create
+        :param cores_per_task: number of cores to allocate per task
+        :param gpus_per_task: number of GPUs to allocate per task
+        :param num_nodes: distribute allocated tasks across N individual nodes
+        """
+        if not isinstance(num_tasks, int) or num_tasks < 1:
+            raise ValueError("task count must be a integer >= 1")
+        if not isinstance(cores_per_task, int) or cores_per_task < 1:
+            raise ValueError("cores per task must be an integer >= 1")
+        if gpus_per_task is not None:
+            if not isinstance(gpus_per_task, int) or gpus_per_task < 0:
+                raise ValueError("gpus per task must be an integer >= 0")
+        if num_nodes is not None:
+            if not isinstance(num_nodes, int) or num_nodes < 1:
+                raise ValueError("node count must be an integer >= 1 (if set)")
+            if num_nodes > num_tasks:
+                raise ValueError("node count must not be greater than task count")
+        elif exclusive:
+            raise ValueError("exclusive can only be set with a node count")
+        children = [cls._create_resource("core", cores_per_task)]
+        if gpus_per_task not in (None, 0):
+            children.append(cls._create_resource("gpu", gpus_per_task))
+        if num_nodes is not None:
+            num_slots = int(math.ceil(num_tasks / float(num_nodes)))
+            if num_tasks % num_nodes != 0:
+                # N.B. uneven distribution results in wasted task slots
+                task_count_dict = {"total": num_tasks}
+            else:
+                task_count_dict = {"per_slot": 1}
+            slot = cls._create_slot("task", num_slots, children)
+            resource_section = cls._create_resource(
+                "node", num_nodes, [slot], exclusive
+            )
+        else:
+            task_count_dict = {"per_slot": 1}
+            slot = cls._create_slot("task", num_tasks, children)
+            resource_section = slot
+
+        resources = [resource_section]
+        tasks = [{"command": command, "slot": "task", "count": task_count_dict}]
+        attributes = {"system": {"duration": 0}}
+        return cls(resources, tasks, attributes=attributes)
+
+    @classmethod
+    def from_batch_command(
+        cls,
+        script,
+        jobname,
+        args=None,
+        num_slots=1,
+        cores_per_slot=1,
+        gpus_per_slot=None,
+        num_nodes=None,
+        broker_opts=None,
+        exclusive=False,
+        conf=None,
+    ):
+        """
+        Create a Jobspec describing a nested Flux instance controlled by
+        a script.
+
+        The nested Flux instance will execute the script with the given
+        command-line arguments after copying it and setting the executable
+        bit.  Conceptually, this differs from the `from_nest_command`,
+        which also creates a nested Flux instance, in that it a) requires
+        the initial program of the new instance to be an executable text
+        file and b) creates the initial program from a string rather than
+        using an executable existing somewhere on the filesystem.
+
+        Use setters to assign additional properties.
+
+        Args:
+            script (str): contents of the script to execute, as a string. The
+                script should have a shebang (e.g. `#!/bin/sh`) at the top.
+            jobname (str): name to use for system.job.name attribute This will
+                be the default job name reported by Flux.
+            args (iterable of `str`): arguments to pass to `script`
+            num_slots (int): number of resource slots to create. Slots are an
+                abstraction, and are only used (along with `cores_per_slot`
+                and `gpus_per_slot`) to determine the nested instance's
+                allocation size and layout.
+            cores_per_slot (int): number of cores to allocate per slot
+            gpus_per_slot (int): number of GPUs to allocate per slot
+            num_nodes (int): distribute allocated resource slots across N
+                individual nodes
+            broker_opts (iterable of `str`): options to pass to the new Flux
+                broker
+            conf (dict): optional broker configuration to pass to the
+                child instance brokers. If set, `conf` will be set in the
+                jobspec 'files' (RFC 37 File Archive) attribute as `conf.json`,
+                and broker_opts will be extended to add
+                `-c{{tmpdir}}/conf.json`
+        """
+        if not script.startswith("#!"):
+            raise ValueError(f"{jobname} does not appear to start with '#!'")
+        args = () if args is None else args
+        jobspec = cls.from_nest_command(
+            command=["{{tmpdir}}/script", *args],
+            num_slots=num_slots,
+            cores_per_slot=cores_per_slot,
+            gpus_per_slot=gpus_per_slot,
+            num_nodes=num_nodes,
+            broker_opts=broker_opts,
+            exclusive=exclusive,
+            conf=conf,
+        )
+        #  Copy script contents into jobspec
+        jobspec.add_file("script", script, perms=0o700, encoding="utf-8")
+        jobspec.setattr("system.job.name", jobname)
+        return jobspec
+
+    @classmethod
+    def from_nest_command(
+        cls,
+        command,
+        num_slots=1,
+        cores_per_slot=1,
+        gpus_per_slot=None,
+        num_nodes=None,
+        broker_opts=None,
+        exclusive=False,
+        conf=None,
+    ):
+        """
+        Create a Jobspec describing a nested Flux instance controlled by
+        `command`.
+
+        Conceptually, this differs from the `from_batch_command` method
+        in that a) the initial program of the nested Flux instance can
+        be any executable on the file system, not just a text file and b)
+        the executable is not copied at submission time.
+
+        Use setters to assign additional properties.
+
+        Args:
+            command (iterable of `str`): initial program for the nested Flux
+            instance
+            num_slots (int): number of resource slots to create. Slots are
+                an abstraction, and are only used (along with `cores_per_slot`
+                and `gpus_per_slot`) to determine the nested instance's
+                allocation size and layout.
+            cores_per_slot (int): number of cores to allocate per slot
+            gpus_per_slot (int): number of GPUs to allocate per slot
+            num_nodes (int): distribute allocated resource slots across N
+                individual nodes
+            broker_opts (iterable of `str`): options to pass to the new Flux
+                broker
+            conf (dict): optional broker configuration to pass to the
+                child instance brokers. If set, `conf` will be set in the
+                jobspec 'files' (RFC 37 File Archive) attribute as `conf.json`,
+                and broker_opts will be extended to add
+                `-c{{tmpdir}}/conf.json`
+        """
+        broker_opts = [] if broker_opts is None else broker_opts
+        if conf is not None:
+            broker_opts.append("-c{{tmpdir}}/conf.json")
+        jobspec = cls.from_command(
+            command=["flux", "broker", *broker_opts, *command],
+            num_tasks=num_slots,
+            cores_per_task=cores_per_slot,
+            gpus_per_task=gpus_per_slot,
+            num_nodes=num_nodes,
+            exclusive=exclusive,
+        )
+        jobspec.setattr_shell_option("per-resource.type", "node")
+        jobspec.setattr_shell_option("mpi", "none")
+        if conf is not None:
+            jobspec.add_file("conf.json", conf)
+        return jobspec
diff --git a/src/bindings/python/flux/job/__init__.py b/src/bindings/python/flux/job/__init__.py
new file mode 100644
index 000000000000..fd2dc6b199d7
--- /dev/null
+++ b/src/bindings/python/flux/job/__init__.py
@@ -0,0 +1,45 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+from flux.job.Jobspec import Jobspec, JobspecV1, validate_jobspec
+from flux.job.JobID import id_parse, id_encode, JobID
+from flux.job.kvs import job_kvs, job_kvs_guest
+from flux.job.kill import kill_async, kill, cancel_async, cancel
+from flux.job.submit import submit_async, submit, submit_get_id
+from flux.job.info import JobInfo, JobInfoFormat, job_fields_to_attrs
+from flux.job.list import job_list, job_list_inactive, job_list_id, JobList, get_job
+from flux.job.kvslookup import job_info_lookup, JobKVSLookup, job_kvs_lookup
+from flux.job.wait import wait_async, wait, wait_get_status, result_async, result
+from flux.job.event import (
+    event_watch_async,
+    event_watch,
+    event_wait,
+    JobEventWatchFuture,
+    EventLogEvent,
+    JobException,
+    MAIN_EVENTS,
+)
+from flux.job.executor import (
+    FluxExecutor,
+    FluxExecutorFuture,
+)
+from flux.job.timeleft import timeleft
+from flux.core.inner import ffi
+from flux.job.output import (
+    job_output,
+    output_event_watch,
+    output_event_watch_async,
+    output_watch,
+    output_watch_async,
+    output_watch_lines,
+    output_watch_lines_async,
+)
+from flux.job.watcher import JobWatcher
+from flux.job.journal import JournalConsumer
diff --git a/src/bindings/python/flux/job/_wrapper.py b/src/bindings/python/flux/job/_wrapper.py
new file mode 100644
index 000000000000..da9b7254fb1d
--- /dev/null
+++ b/src/bindings/python/flux/job/_wrapper.py
@@ -0,0 +1,20 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+from _flux._core import ffi, lib
+from flux.wrapper import Wrapper
+
+
+class JobWrapper(Wrapper):
+    def __init__(self):
+        super(JobWrapper, self).__init__(ffi, lib, prefixes=["flux_job_"])
+
+
+_RAW = JobWrapper()
diff --git a/src/bindings/python/flux/job/directives.py b/src/bindings/python/flux/job/directives.py
new file mode 100644
index 000000000000..951137adfc3f
--- /dev/null
+++ b/src/bindings/python/flux/job/directives.py
@@ -0,0 +1,280 @@
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+import fileinput
+import re
+import shlex
+
+
+class Directive:
+    """
+    This class represents a single job submission directive processed
+    from an input file or batch script. Input values to the constructor
+    should have the sentinel stripped, and are then split using Python's
+    shlex module to provide familiarity UNIX shell syntax and quoting
+    for argument. The first argument after shell lexing determines the
+    Directive type or "action".
+
+    At this time the only supported actions are "SETARGS", which indicates
+    an args list to pass through to the submission utility, and NOOP,
+    which is an empty directive.
+
+    If lineno is provided, it is used in error messages to indicate to
+    the user more detail about the failing line (e.g. shell parsing error).
+
+    Args:
+        value (str): A preprocessed directive with sentinel removed
+        lineno (int, optional): The source line number of the current
+            directive
+
+    Attributes:
+        lineno (int): line number associated with directive
+        args (list): list of directive arguments
+        action (str): the directive type or instruction to the submission
+            utility.
+
+    """
+
+    def __init__(self, value, lineno=-1):
+        self.lineno = lineno
+        #
+        # split value as POSIX shell, removing comments.
+        # We specify posix and punctuation_chars to get more predictable
+        # quoting and avoid splitting on cmdline args like --foo.
+        lexer = shlex.shlex(value, posix=True, punctuation_chars=True)
+        #
+        # set whitespace_split to match shell parsing of cmdlines as
+        # closely as possible as documented in the final note here:
+        # https://docs.python.org/3/library/shlex.html
+        lexer.whitespace_split = True
+        #
+        # Add single-quote to escapedquotes. This is necessary to avoid
+        # unclosed quote ValueError due to escaped single quote. (The
+        # default in Posix mode is to escape only '"')
+        lexer.escapedquotes = "\"'"
+        try:
+            self.args = list(lexer)
+        except ValueError as exc:
+            raise ValueError(f"line {lineno}: {value}: {exc}") from None
+
+        if not self.args:
+            self.action = "NOOP"
+        elif self.args[0].startswith("-"):
+            self.action = "SETARGS"
+        else:
+            raise ValueError(f"line {lineno}: Unknown directive: {value}")
+
+    def __str__(self):
+        return f"{self.action}({self.args})"
+
+
+class MultiLine:
+    """
+    Container for multiline quoted directives.
+
+    A Multiline is opened and closed by matching triple quote at the end
+    of a line. While a multi line triple quote is open, lines are pushed
+    verbatim (minus any common indent). When the Multiline is finished
+    a string is returned with the multiline literal escaped such that it
+    may be passed to the Directive constructor.
+    """
+
+    def __init__(self):
+        self.triplequote = None
+        self.inprogress = False
+        self.startline = -1
+        self.start = None
+        self.indent = ""
+        self.lines = []
+
+    def finish(self):
+        result = self.lines[0] + shlex.quote("\n".join(self.lines[1:]))
+        if self.triplequote in result:
+            raise ValueError(
+                f"improperly terminated triple quoted string at: `{self.start}'"
+            )
+        self.triplequote = None
+        self.lines.clear()
+        self.indent = ""
+        self.inprogress = False
+        self.start = None
+        return result
+
+    def append(self, value):
+        # Remove common indent and append line
+        if value.startswith(self.indent):
+            value = value[len(self.indent) :]
+        self.lines.append(value)
+
+    def process(self, value, lineno):
+        closed = False
+        quotes = value[-3:]
+        if not self.inprogress:
+            # Strip leading whitespace and stash indent.
+            # Indent will be removed if matching from all multiline lines
+            self.triplequote = quotes
+            self.indent = value[: -len(value.lstrip())]
+            self.inprogress = True
+            self.start = value.lstrip()
+            self.startline = lineno
+        elif quotes != self.triplequote:
+            # A different kind of triplequote, append value and return
+            self.append(value)
+            return None
+        else:
+            closed = True
+
+        # Append value with quotes removed:
+        self.append(value[:-3])
+        if closed:
+            return self.finish()
+        return None
+
+
+class DirectiveParser:
+    """
+    RFC 36 submission directive parser.
+
+    The DirectiveParser looks for sentinels matching the pattern specified
+    in RFC 36 in an input stream and extracts each line or quoted multiline
+    into a Directive object. Single line strings are split into multiple
+    tokens by Python's shlex module, which allows lines to contain familiar
+    shell quoting and comments.  As a convenience, inline triple quoting
+    is also supported.
+
+    Args:
+        inputfile (:obj:`io.TextIOWrapper`):
+
+    Attributes:
+        directives (list): list of Directive objects
+        script (str): the script to submit
+    """
+
+    def __init__(self, inputfile):
+        self.directives = []
+        self.script = ""
+        self.re = re.compile(r"^([^\w]*)((?:flux|FLUX):)(.*)$")
+        self.sentinel = None
+        self.prefix = None
+        self.line = 0
+
+        multiline = MultiLine()
+        lineno = 0
+        started = False
+        directives_disabled = False
+        last_directive_line = 0
+
+        for line in inputfile:
+            lineno += 1
+            self.script += line
+
+            match = self.re.match(line)
+            if not match:
+                #  All lines in a multiline must start with the sentinel:
+                if multiline.inprogress:
+                    raise ValueError(
+                        f"line {lineno}: unterminated multi-line quote at"
+                        + f" line {multiline.startline}: `{multiline.start}'"
+                    )
+                #  Disable further directives if directives are not already
+                #  disabled, line prefix with trailing whitespace removed
+                #  doesn't match, and the line is not otherwise empty:
+                if (
+                    started
+                    and not directives_disabled
+                    and not line.startswith(self.prefix.rstrip())
+                    and line.strip()
+                ):
+                    directives_disabled = True
+                    last_directive_line = lineno - 1
+                continue
+
+            #  Get directive prefix and tag:
+            started = True
+            prefix = match.group(1)
+            tag = match.group(2)
+            sentinel = prefix + tag
+
+            #  It is an error if a directive is found after directives
+            #  have been disabled:
+            if directives_disabled:
+                #  Raise an error if a directive appears when directives
+                #  have been disabled
+                raise ValueError(
+                    f"line {lineno}: orphan '{tag}' detected: "
+                    + f"directives disabled after line {last_directive_line}"
+                )
+
+            #  Try processing paired triple quotes on this line.
+            #  Raises ValueError on unbalanced triple quotes, or a single
+            #  triple quote not at end of line:
+            try:
+                value = self.triplequote(match.group(3))
+            except ValueError as exc:
+                raise ValueError(f"line {lineno}: {exc}") from None
+
+            #  If this is the first line with a sentinel, stash the
+            #  sentinel for later comparison. Otherwise, raise an error
+            #  if it does not match:
+            if self.sentinel is None:
+                self.sentinel = sentinel
+                self.prefix = prefix
+            elif sentinel != self.sentinel:
+                raise ValueError(
+                    f"line {lineno}: sentinel changed from "
+                    + f"'{self.sentinel}' to '{sentinel}'"
+                )
+
+            if value.endswith('"""') or value.endswith("'''"):
+                #  Handle start or end of a multiline triple quoted string:
+                result = multiline.process(value, lineno)
+                if result:
+                    self.append(result, multiline.startline)
+            elif multiline.inprogress:
+                #  Multiline in progress: collect in multiline object
+                multiline.append(value)
+            elif value:
+                self.append(value, lineno)
+
+    def triplequote(self, value):
+        """
+        Escape quotes within triple quotes (single or double) so they pass
+        unmodified to shell lexing done in Directive constructor
+        """
+
+        def paired(n):
+            return n > 0 and (n % 2) == 0
+
+        if paired(value.count('"""')) or paired(value.count("'''")):
+            value = re.sub(
+                r"""((?:["']){3})(.*)\1""", lambda x: shlex.quote(x[2]), value
+            )
+
+        elif re.search(r"""((?:["']){3}).""", value):
+            #  Multiline strings must have triple quote at end of line,
+            #  so a triple quote followed by anything is an error:
+            raise ValueError(f"unclosed triple quote: {value.strip()}")
+        return value
+
+    def append(self, value, lineno):
+        """
+        Append one Directive
+        """
+        self.directives.append(Directive(value, lineno))
+
+
+if __name__ == "__main__":
+    """
+    Parse directives from any file for test/debug purposes with
+    ::
+       flux python -m flux.job.directives FILE
+    """
+    directives = DirectiveParser(fileinput.input()).directives
+    if directives:
+        print("\n".join(map(str, directives)))
diff --git a/src/bindings/python/flux/job/event.py b/src/bindings/python/flux/job/event.py
new file mode 100644
index 000000000000..2f7d75fee355
--- /dev/null
+++ b/src/bindings/python/flux/job/event.py
@@ -0,0 +1,222 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+import errno
+import json
+
+from _flux._core import ffi
+from flux.job._wrapper import _RAW as RAW
+from flux.kvs import WatchImplementation
+
+# Names of events that may appear in the main eventlog (i.e. ``eventlog="eventlog"``)
+# See Flux RFC 21 for documentation on each event.
+MAIN_EVENTS = frozenset(
+    {
+        "submit",
+        "depend",
+        "priority",
+        "flux-restart",
+        "urgency",
+        "alloc",
+        "free",
+        "start",
+        "release",
+        "finish",
+        "clean",
+        "debug",
+        "exception",
+    }
+)
+
+
+class EventLogEvent(dict):
+    """
+    wrapper class for a single KVS EventLog entry
+    """
+
+    def __init__(self, event):
+        """
+        "Initialize from a string or dict eventlog event
+        """
+        if isinstance(event, str):
+            event = json.loads(event)
+        super().__init__(event)
+        if "context" not in self:
+            self["context"] = {}
+
+    def __str__(self):
+        return "{0.timestamp:<0.5f}: {0.name} {0.context}".format(self)
+
+    @property
+    def name(self):
+        return self["name"]
+
+    @property
+    def timestamp(self):
+        return self["timestamp"]
+
+    @property
+    def context(self):
+        return self["context"]
+
+    @property
+    def context_string(self):
+        if not self.context:
+            return ""
+        return json.dumps(
+            self.context, ensure_ascii=False, separators=(",", ":"), sort_keys=True
+        )
+
+
+class JobEventWatchFuture(WatchImplementation):
+    """
+    A future returned from job.event_watch_async().
+    Adds get_event() method to return an EventLogEntry event
+    """
+
+    def __init__(self, future_handle):
+        super().__init__(future_handle)
+
+    def watch_get(self, future):
+        """
+        Implementation of watch_get() for JobEventWatchFuture.
+
+        Will be called from WatchABC.get()
+        """
+        result = ffi.new("char *[1]")
+        RAW.event_watch_get(future, result)
+        return EventLogEvent(ffi.string(result[0]).decode("utf-8"))
+
+    def watch_cancel(self, future):
+        """
+        Implementation of watch_cancel() for JobEventWatchFuture.
+
+        Will be called from WatchABC.cancel()
+        """
+        RAW.event_watch_cancel(future)
+
+    def get_event(self, autoreset=True):
+        """
+        Return the next event from a JobEventWatchFuture, or None
+        if the event stream has terminated.
+
+        The future is auto-reset unless autoreset=False, so a subsequent
+        call to get_event() will try to fetch the next event and thus
+        may block.
+        """
+        return self.get(autoreset=autoreset)
+
+
+def event_watch_async(flux_handle, jobid, eventlog="eventlog"):
+    """Asynchronously get eventlog updates for a job
+
+    Asynchronously watch the events of a job eventlog.
+
+    Returns a JobEventWatchFuture. Call .get_event() from the then
+    callback to get the currently returned event from the Future object.
+
+    .. seealso::
+
+       :doc:`rfc:spec_21`
+          Documentation for the events in the main eventlog
+
+    :param flux_handle: handle for Flux broker from flux.Flux()
+    :type flux_handle: Flux
+    :param jobid: the job ID on which to watch events
+    :param eventlog: eventlog path in job kvs directory (default: eventlog)
+    :returns: a JobEventWatchFuture object
+    :rtype: JobEventWatchFuture
+    """
+
+    future = RAW.event_watch(flux_handle, int(jobid), eventlog, 0)
+    return JobEventWatchFuture(future)
+
+
+def event_watch(flux_handle, jobid, eventlog="eventlog"):
+    """Python generator to watch all events for a job
+
+    Synchronously watch events a job eventlog via a simple generator.
+
+    Example:
+        >>> for event in job.event_watch(flux_handle, jobid):
+        ...     # do something with event
+
+    .. seealso::
+
+       :doc:`rfc:spec_21`
+          Documentation for the events in the main eventlog
+
+    :param flux_handle: handle for Flux broker from flux.Flux()
+    :type flux_handle: Flux
+    :param jobid: the job ID on which to watch events
+    :param eventlog: eventlog path in job kvs directory (default: eventlog)
+    """
+    watcher = event_watch_async(flux_handle, jobid, eventlog)
+    event = watcher.get_event()
+    while event is not None:
+        yield event
+        event = watcher.get_event()
+
+
+class JobException(Exception):
+    """Represents an 'exception' event occurring to a job.
+
+    Instances expose a few public attributes.
+
+    :var timestamp: the timestamp of the 'exception' event.
+    :var type: A string identifying the type of job exception.
+    :var note: Brief human-readable explanation of the exception.
+    :var severity: the severity of the exception. Exceptions with a severity
+        of 0 are fatal to the job; any other severity is non-fatal.
+    """
+
+    def __init__(self, event):
+        self.timestamp = event.timestamp
+        self.type = event.context["type"]
+        self.note = event.context["note"]
+        self.severity = event.context["severity"]
+        super().__init__(self)
+
+    def __str__(self):
+        return f"job.exception: type={self.type}: {self.note}"
+
+
+def event_wait(flux_handle, jobid, name, eventlog="eventlog", raiseJobException=True):
+    """Wait for a job eventlog entry 'name'
+
+    Wait synchronously for an eventlog entry named "name" and
+    return the entry to caller, raises OSError with ENODATA if
+    event never occurred
+
+    .. seealso::
+
+       :doc:`rfc:spec_21`
+          Documentation for the events in the main eventlog
+
+    :param flux_handle: handle for Flux broker from flux.Flux()
+    :type flux_handle: Flux
+    :param jobid: the job ID on which to wait for eventlog events
+    :param name: The event name for which to wait
+    :param eventlog: eventlog path in job kvs directory (default: eventlog)
+    :param raiseJobException: if True, watch for job exception events and
+      raise a JobException if one is seen before event 'name' (default=True)
+    :returns: an EventLogEvent object, or raises OSError if eventlog
+     ended before matching event was found
+    :rtype: EventLogEvent
+    """
+    for event in event_watch(flux_handle, jobid, eventlog):
+        if event.name == name:
+            return event
+        if (
+            raiseJobException
+            and event.name == "exception"
+            and event.context["severity"] == 0
+        ):
+            raise JobException(event)
+    raise OSError(errno.ENODATA, f"eventlog ended before event='{name}'")
diff --git a/src/bindings/python/flux/job/executor.py b/src/bindings/python/flux/job/executor.py
new file mode 100644
index 000000000000..4dbb76d30744
--- /dev/null
+++ b/src/bindings/python/flux/job/executor.py
@@ -0,0 +1,625 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+"""
+This module defines the ``FluxExecutor`` and ``FluxExecutorFuture`` classes.
+"""
+
+import collections
+import concurrent.futures
+import itertools
+import logging
+import os
+import threading
+import time
+import weakref
+
+import flux
+from flux.job.event import MAIN_EVENTS, JobException, event_watch_async
+from flux.job.submit import submit_async, submit_get_id
+
+_SubmitPackage = collections.namedtuple(
+    "_SubmitPackage", ["submit_args", "submit_kwargs", "future"]
+)
+
+
+class _AttachPackage:  # pylint: disable=too-few-public-methods
+    """Namedtuple-esque class. Constructor sets jobid on future."""
+
+    __slots__ = ("jobid", "future")
+
+    def __init__(self, jobid, fut):
+        self.jobid = jobid
+        self.future = fut
+        self.future._set_jobid(jobid)  # pylint: disable=protected-access
+
+
+class _FluxExecutorThread(threading.Thread):
+    """Thread that submits jobs to Flux and waits for event updates.
+
+    Completes FluxExecutorFutures as events indicate that they finish.
+
+    :param exit_event: ``threading.Event`` indicating when the associated
+        Executor has shut down.
+    :param packages_to_handle: a queue filled with Packages by the Executor
+    :param poll_interval: the interval (in seconds) to check for new jobs.
+    """
+
+    # pylint: disable=too-many-arguments
+    def __init__(
+        self,
+        broken_event,
+        exit_event,
+        packages_to_handle,
+        poll_interval,
+        handle_args,
+        handle_kwargs,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.__broken_event = broken_event
+        self.__exit_event = exit_event
+        self.__packages_to_handle = packages_to_handle
+        self.__poll_interval = poll_interval
+        self.__flux_handle = flux.Flux(*handle_args, **handle_kwargs)
+        self.__running_user_futures = set()  # unfulfilled futures
+
+    def run(self):
+        try:
+            self.__run()
+        except Exception as exc:
+            self.__broken_event.set()
+            for fut in self.__running_user_futures:
+                if not fut.done():
+                    fut.set_exception(exc)
+            while self.__packages_to_handle or not self.__exit_event.is_set():
+                try:
+                    package = self.__packages_to_handle.popleft()
+                except IndexError:
+                    time.sleep(0.01)
+                else:
+                    package.future.set_exception(exc)
+            raise
+
+    def __run(self):
+        """Loop indefinitely, submitting jobspecs and fetching jobids."""
+        self.__flux_handle.timer_watcher_create(
+            self.__poll_interval, self.__submit_new_jobs, repeat=self.__poll_interval
+        ).start()
+        while self.__work_remains():
+            self.__submit_new_jobs(reactor_run=False)
+            if self.__flux_handle.reactor_run() < 0:
+                raise RuntimeError("reactor start failed")
+
+    def __work_remains(self):
+        """Return True if and only if there is still work to be done.
+
+        Equivalently, return False if it is safe to exit.
+        """
+        return (
+            not self.__exit_event.is_set()
+            or self.__packages_to_handle
+            or self.__running_user_futures
+        )
+
+    def __submit_new_jobs(self, *_, reactor_run=True):
+        """Pull jobspecs from the queue and submit them.
+
+        Invoked on a timer, and passed several arguments, none
+        of which are used---hence the "*_"
+        """
+        if not self.__work_remains() and reactor_run:
+            self.__flux_handle.reactor_stop()
+        if not self.__running_user_futures and not self.__packages_to_handle:
+            time.sleep(self.__poll_interval)
+        while self.__packages_to_handle:
+            try:
+                package = self.__packages_to_handle.popleft()
+            except IndexError:
+                continue
+            if package.future.set_running_or_notify_cancel():
+                if isinstance(package, _SubmitPackage):
+                    self.__handle_submit(package)
+                else:
+                    self.__handle_attach(package)
+
+    def __handle_submit(self, package):
+        """Submit a _SubmitPackage and set a jobid callback."""
+        try:
+            submit_async(
+                self.__flux_handle, *package.submit_args, **package.submit_kwargs
+            ).then(self.__submission_callback, package.future)
+        except Exception as submit_exc:  # pylint: disable=broad-except
+            package.future.set_exception(submit_exc)
+        else:
+            self.__running_user_futures.add(package.future)
+
+    def __handle_attach(self, package):
+        """Submit an _AttachPackage and set an event callback."""
+        try:
+            event_watch_async(self.__flux_handle, package.jobid).then(
+                self.__event_update, package.future
+            )
+        except Exception as event_exc:  # pylint: disable=broad-except
+            package.future.set_exception(event_exc)
+        else:
+            self.__running_user_futures.add(package.future)
+
+    def __submission_callback(self, submission_future, user_future):
+        """Callback invoked when a jobid is ready for a submitted jobspec."""
+        jobid = submit_get_id(submission_future)
+        user_future._set_jobid(jobid)  # pylint: disable=protected-access
+        event_watch_async(self.__flux_handle, jobid).then(
+            self.__event_update, user_future
+        )
+
+    def __event_update(self, event_future, user_future):
+        """Callback invoked when a job has an event update."""
+        event = None
+        try:
+            event = event_future.get_event()
+        except FileNotFoundError:  # job ID was not accepted
+            user_future.set_exception(ValueError("job ID does not match any job"))
+        if event is not None:
+            if event.name in user_future.EVENTS:
+                user_future._set_event(event)  # pylint: disable=protected-access
+            # check if the event tells us that the job is done
+            if not user_future.done():
+                if event.name == "finish":
+                    exit_status = event.context["status"]
+                    if os.WIFEXITED(exit_status):
+                        user_future.set_result(os.WEXITSTATUS(exit_status))
+                    elif os.WIFSIGNALED(exit_status):
+                        user_future.set_result(-os.WTERMSIG(exit_status))
+                    else:
+                        user_future.set_exception(ValueError(exit_status))
+                elif event.name == "exception" and event.context["severity"] == 0:
+                    user_future.set_exception(JobException(event))
+        else:  # no more events
+            self.__running_user_futures.discard(user_future)
+
+
+# pylint: disable=too-many-instance-attributes
+class FluxExecutorFuture(concurrent.futures.Future):
+    """A ``concurrent.futures.Future`` subclass that represents a single Flux job.
+
+    In addition to all of the ``concurrent.futures.Future`` functionality,
+    ``FluxExecutorFuture`` instances offer:
+
+    * The ``jobid`` and ``add_jobid_callback`` methods for retrieving the
+      Flux jobid of the underlying job.
+    * The ``add_event_callback`` method to invoke callbacks when particular
+      job-state events occur.
+
+    Valid events are contained in the ``EVENTS`` class attribute.
+    """
+
+    #: A set containing the names of valid events.
+    EVENTS = MAIN_EVENTS
+
+    def __init__(self, owning_thread_id, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # Thread.ident of thread tasked with completing this future
+        self.__owning_thread_id = owning_thread_id
+        self.__jobid_condition = threading.Condition()
+        self.__jobid = None
+        self.__jobid_set = False  # True if the jobid has been set to something
+        self.__jobid_exception = None
+        self.__jobid_callbacks = []
+        self.__event_lock = threading.RLock()
+        self.__events_occurred = {state: collections.deque() for state in self.EVENTS}
+        self.__event_callbacks = {state: collections.deque() for state in self.EVENTS}
+
+    def _set_jobid(self, jobid, exc=None):
+        """Sets the Flux jobid associated with the future.
+
+        If `exc` is not None, raise `exc` instead of returning the jobid
+        in calls to `Future.jobid()`. Useful if the job ID cannot be
+        retrieved.
+
+        Should only be used by Executor implementations and unit tests.
+        """
+        if self.__jobid_set:
+            # should be InvalidStateError in 3.8+
+            raise RuntimeError("invalid state: jobid already set")
+        with self.__jobid_condition:
+            self.__jobid = jobid
+            self.__jobid_set = True
+            if exc is not None:
+                self.__jobid_exception = exc
+            self.__jobid_condition.notify_all()
+        for callback in self.__jobid_callbacks:
+            self._invoke_flux_callback(callback)
+
+    def add_done_callback(self, *args, **kwargs):  # pylint: disable=arguments-differ
+        """Attaches a callable that will be called when the future finishes.
+
+        :param fn: A callable that will be called with this future as its only
+            argument when the future completes or is cancelled. The callable
+            will always be called by a thread in the same process in which
+            it was added. If the future has already completed or been
+            cancelled then the callable will be called immediately. These
+            callables are called in the order that they were added.
+        :return: ``self``
+        """
+        super().add_done_callback(*args, **kwargs)
+        return self
+
+    def jobid(self, timeout=None):
+        """Return the jobid of the Flux job that the future represents.
+
+        :param timeout: The number of seconds to wait for the jobid.
+            If None, then there is no limit on the wait time.
+
+        :return: a positive integer jobid.
+
+        :raises concurrent.futures.TimeoutError: If the jobid is not available
+            before the given timeout.
+        :raises concurrent.futures.CancelledError: If the future was cancelled.
+        :raises RuntimeError: If the job could not be submitted (e.g. if
+            the jobspec was invalid).
+        """
+        if self.__jobid_set:
+            return self._get_jobid()
+        with self.__jobid_condition:
+            self.__jobid_condition.wait(timeout)
+            if self.__jobid_set:
+                return self._get_jobid()
+            raise concurrent.futures.TimeoutError()
+
+    def _get_jobid(self):
+        """Get the jobid, checking for cancellation and invalid job ids."""
+        if self.__jobid_exception is not None:
+            raise self.__jobid_exception
+        return self.__jobid
+
+    def add_jobid_callback(self, callback):
+        """Attaches a callable that will be called when the jobid is ready.
+
+        Added callables are called in the order that they were added and may be called
+        in another thread.  If the callable raises an ``Exception`` subclass, it will
+        be logged and ignored.  If the callable raises a ``BaseException`` subclass,
+        the behavior is undefined.
+
+        :param callback: a callable taking the future as its only argument.
+        :return: ``self``
+        """
+        with self.__jobid_condition:
+            if self.__jobid is None:
+                self.__jobid_callbacks.append(callback)
+                return self
+        self._invoke_flux_callback(callback)
+        return self
+
+    def _invoke_flux_callback(self, callback, *args):
+        try:
+            callback(self, *args)
+        except Exception:  # pylint: disable=broad-except
+            logging.getLogger(__name__).exception(
+                "exception calling callback for %r", self
+            )
+
+    def exception(self, *args, **kwargs):  # pylint: disable=arguments-differ
+        """If this method is invoked from a jobid/event callback by an executor thread,
+        it will result in deadlock, since the current thread will wait
+        for work that the same thread is meant to do.
+
+        Head off this possibility by checking the current thread.
+        """
+        if not self.done() and threading.get_ident() == self.__owning_thread_id:
+            raise RuntimeError("Cannot wait for future from inside callback")
+        return super().exception(*args, **kwargs)
+
+    exception.__doc__ = concurrent.futures.Future.exception.__doc__
+
+    def result(self, *args, **kwargs):  # pylint: disable=arguments-differ
+        """If this method is invoked from a jobid/event callback by an executor thread,
+        it will result in deadlock, since the current thread will wait
+        for work that the same thread is meant to do.
+
+        Head off this possibility by checking the current thread.
+        """
+        if not self.done() and threading.get_ident() == self.__owning_thread_id:
+            raise RuntimeError("Cannot wait for future from inside callback")
+        return super().result(*args, **kwargs)
+
+    result.__doc__ = concurrent.futures.Future.result.__doc__
+
+    def set_exception(self, exception):
+        """When setting an exception on the future, set the jobid if it hasn't
+        been set already. The jobid will already have been set unless the exception
+        was generated before the job could be successfully submitted.
+        """
+        try:
+            self.jobid(0)
+        except concurrent.futures.TimeoutError:
+            # set jobid to something
+            self._set_jobid(
+                None, RuntimeError(f"job could not be submitted due to {exception}")
+            )
+        return super().set_exception(exception)
+
+    set_exception.__doc__ = concurrent.futures.Future.set_exception.__doc__
+
+    def cancel(self, *args, **kwargs):  # pylint: disable=arguments-differ
+        """If a thread is waiting for the future's jobid, and another
+        thread cancels the future, the waiting thread would never wake up
+        because the jobid would never be set.
+
+        When cancelling, set the jobid to something invalid.
+        """
+        if self.cancelled():  # if already cancelled, return True
+            return True
+        cancelled = super().cancel(*args, **kwargs)
+        if cancelled:
+            try:
+                self.jobid(0)
+            except concurrent.futures.TimeoutError:
+                # set jobid to something
+                self._set_jobid(None, concurrent.futures.CancelledError())
+        return cancelled
+
+    cancel.__doc__ = concurrent.futures.Future.cancel.__doc__
+
+    def add_event_callback(self, event, callback):
+        """Add a callback to be invoked when an event occurs.
+
+        The callback will be invoked, with the future as the first argument and the
+        ``flux.job.EventLogEvent`` as the second, whenever the event occurs. If the
+        event occurs multiple times, the callback will be invoked with each different
+        `EventLogEvent` instance. If the event never occurs, the callback
+        will never be invoked.
+
+        Added callables are called in the order that they were added and may be called
+        in another thread.  If the callable raises an ``Exception`` subclass, it will
+        be logged and ignored.  If the callable raises a ``BaseException`` subclass,
+        the behavior is undefined.
+
+        If the event has already occurred, the callback will be called immediately.
+
+        :param event: the name of the event to add the callback to.
+        :param callback: a callable taking the future and the event as arguments.
+        :return: ``self``
+        """
+        if event not in self.EVENTS:
+            raise ValueError(event)
+        with self.__event_lock:
+            self.__event_callbacks[event].append(callback)
+            for log_entry in self.__events_occurred[event]:
+                self._invoke_flux_callback(callback, log_entry)
+        return self
+
+    def _set_event(self, log_entry):
+        """Set an event on the future.
+
+        For use by Executor implementations and unit tests.
+
+        :param log_entry: an ``EventLogEvent``.
+        """
+        event_name = log_entry.name
+        if event_name not in self.EVENTS:
+            raise ValueError(event_name)
+        with self.__event_lock:
+            self.__events_occurred[event_name].append(log_entry)
+            # make a shallow copy of callbacks --- in case a user callback
+            # tries to add another callback for the same event
+            for callback in list(self.__event_callbacks[event_name]):
+                self._invoke_flux_callback(callback, log_entry)
+
+
+class FluxExecutor:
+    """Provides a method to submit and monitor Flux jobs asynchronously.
+
+    Forks threads to complete futures and fetch event updates in the background.
+
+    Inspired by the ``concurrent.futures.Executor`` class, with the following
+    interface differences:
+
+        - the ``submit`` method takes a ``flux.job.Jobspec`` instead of a
+          callable and its arguments, and returns a ``FluxExecutorFuture``
+          representing that job.
+        - the ``map`` method is not supported, given that the executor consumes
+          Jobspecs rather than callables.
+
+    Otherwise, the FluxExecutor is faithful to its inspiration. In addition
+    to methods and behavior defined by ``concurrent.futures``, FluxExecutor
+    provides its futures with event updates and the jobid of the underlying job.
+
+    Futures returned by ``submit`` have their jobid set as soon as it is available,
+    which is always before the future completes.
+
+    The executor can also monitor existing jobs through the ``attach`` method,
+    which takes a job ID and returns a future representing the job.
+
+    Futures may receive event updates even after they complete. The names
+    of valid events are contained in the ``EVENTS`` class attribute.
+
+    The result of a future is the highest process exit status of the underlying job
+    (in which case the result is an integer greater than or equal to 0),
+    or ``-signum`` where ``signum`` is the number
+    of the signal that caused the process to terminate
+    (in which case the result is an integer less than 0).
+
+    A future is marked as "running" (and can no longer be canceled using the
+    ``.cancel()`` method) once it reaches a certain point in the Executor---a point
+    which is completely unrelated to the status of the underlying Flux job.
+    The underlying Flux job may still be canceled at any point before it terminates,
+    however, using the ``flux.job.cancel`` and ``flux.job.kill`` functions,
+    in which case a ``JobException`` will be set.
+
+    If the jobspec is invalid, an ``OSError`` is set.
+
+    :param threads: the number of worker threads to fork.
+    :param thread_name_prefix: used to control the names of ``threading.Thread``
+        objects created by the executor, for easier debugging.
+    :param poll_interval: the interval (in seconds) in which to break out of the
+        flux event loop to check for new job submissions.
+    :param handle_args: positional arguments to the ``flux.Flux`` instances used by
+        the executor.
+    :param handle_kwargs: keyword arguments to the ``flux.Flux`` instances used by
+        the executor.
+    """
+
+    # Used to assign unique thread names when thread_name_prefix is not supplied.
+    _counter = itertools.count().__next__
+    #: A set containing valid event names for attaching to futures.
+    EVENTS = MAIN_EVENTS
+
+    # pylint: disable=too-many-arguments,dangerous-default-value
+    def __init__(
+        self,
+        threads=1,
+        thread_name_prefix="",
+        poll_interval=0.1,
+        handle_args=(),
+        handle_kwargs={},
+    ):
+        if threads < 0:
+            raise ValueError("the number of threads must be > 0")
+        # split jobs equally among threads; give them each their own queue
+        self._submission_queues = [collections.deque() for i in range(threads)]
+        self._next_thread = 0  # the next thread to give a job to
+        self._shutdown_lock = threading.Lock()
+        self._broken_event = threading.Event()
+        self._shutdown_event = threading.Event()
+        thread_name_prefix = (
+            thread_name_prefix or f"{type(self).__name__}-{self._counter()}"
+        )
+        self._executor_threads = [
+            _FluxExecutorThread(
+                self._broken_event,
+                self._shutdown_event,
+                deque,
+                poll_interval,
+                handle_args,
+                handle_kwargs,
+                name=(f"{thread_name_prefix}-{i}"),
+                daemon=True,
+            )
+            for i, deque in enumerate(self._submission_queues)
+        ]
+        for thread in self._executor_threads:
+            thread.start()
+        # register a finalizer to ensure worker threads are notified to shut down
+        self._finalizer = weakref.finalize(
+            self, self._shutdown_threads, self._shutdown_event, self._executor_threads
+        )
+
+    def shutdown(self, wait=True, *, cancel_futures=False):
+        """Clean-up the resources associated with the Executor.
+
+        It is safe to call this method several times. Otherwise, no other
+        methods can be called after this one.
+
+        :param wait: If ``True``, then this method will not return until all running
+            futures have finished executing and the resources used by the
+            executor have been reclaimed.
+        :param cancel_futures: If ``True``, this method will cancel all pending
+            futures that the executor has not started running. Any futures that
+            are completed or running won't be cancelled, regardless of the value
+            of ``cancel_futures``.
+        """
+        with self._shutdown_lock:
+            self._shutdown_event.set()
+        if cancel_futures:
+            # Drain all work items from the queues, and then cancel their
+            # associated futures.
+            for deque in self._submission_queues:
+                while deque:
+                    try:
+                        package = deque.popleft()
+                    except IndexError:
+                        pass
+                    else:
+                        package.future.cancel()
+                        package.future.set_running_or_notify_cancelled()
+        if wait:
+            for thread in self._executor_threads:
+                thread.join()
+
+    def submit(self, *args, **kwargs):
+        """Submit a jobspec to Flux and return a ``FluxExecutorFuture``.
+
+        Accepts the same positional and keyword arguments as ``flux.job.submit``,
+        except for the ``flux.job.submit`` function's first argument, ``flux_handle``.
+
+        :param jobspec: jobspec defining the job request
+        :type jobspec: Jobspec or its string encoding
+        :param urgency: job urgency 0 (lowest) through 31 (highest)
+            (default is 16).  Priorities 0 through 15 are restricted to
+            the instance owner.
+        :type urgency: int
+        :param waitable: allow result to be fetched with ``flux.job.wait()``
+            (default is False).  Waitable=True is restricted to the
+            instance owner.
+        :type waitable: bool
+        :param debug: enable job manager debugging events to job eventlog
+            (default is False)
+        :type debug: bool
+        :param pre_signed: jobspec argument is already signed
+            (default is False)
+        :type pre_signed: bool
+
+        :raises RuntimeError: if ``shutdown`` has been called or if an error has
+            occurred and new jobs cannot be submitted (e.g. a remote Flux instance
+            can no longer be communicated with).
+        """
+        return self._create_future(_SubmitPackage, args, kwargs)
+
+    def attach(self, jobid):
+        """Attach a ``FluxExecutorFuture`` to an existing job ID and return it.
+
+        Returned futures will behave identically to futures returned by the
+        ``FluxExecutor.submit`` method. If the job ID is not accepted by Flux
+        an exception will be set on the future.
+
+        This method is primarily useful for monitoring jobs that have been
+        submitted through other mechanisms.
+
+        :param jobid: jobid to attach to.
+        :type jobid: int
+
+        :raises RuntimeError: if ``shutdown`` has been called or if an error has
+            occurred and new jobs cannot be submitted (e.g. a remote Flux instance
+            can no longer be communicated with).
+        """
+        return self._create_future(_AttachPackage, jobid)
+
+    def _create_future(self, factory, *factory_args):
+        if self._broken_event.is_set():
+            raise RuntimeError("Executor is broken, new futures cannot be scheduled")
+        with self._shutdown_lock:
+            if self._shutdown_event.is_set():
+                raise RuntimeError("cannot schedule new futures after shutdown")
+            future_owner_id = self._executor_threads[self._next_thread].ident
+            fut = FluxExecutorFuture(future_owner_id)
+            self._submission_queues[self._next_thread].append(
+                factory(*factory_args, fut)
+            )
+            self._next_thread = (self._next_thread + 1) % len(self._submission_queues)
+            return fut
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.shutdown(wait=True)
+        return False
+
+    @staticmethod
+    def _shutdown_threads(event, threads):
+        """Set the threading.Event and join all threads.
+
+        Not a method so as not to prevent garbage collection
+        (see `weakref.finalize` docs).
+        """
+        event.set()
+        for thread in threads:
+            thread.join()
diff --git a/src/bindings/python/flux/job/frobnicator/__init__.py b/src/bindings/python/flux/job/frobnicator/__init__.py
new file mode 100644
index 000000000000..6811d0076fa2
--- /dev/null
+++ b/src/bindings/python/flux/job/frobnicator/__init__.py
@@ -0,0 +1,11 @@
+###############################################################
+# Copyright 2022 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+from flux.job.frobnicator.frobnicator import FrobnicatorPlugin, JobFrobnicator
diff --git a/src/bindings/python/flux/job/frobnicator/frobnicator.py b/src/bindings/python/flux/job/frobnicator/frobnicator.py
new file mode 100644
index 000000000000..0ce0fd6155ae
--- /dev/null
+++ b/src/bindings/python/flux/job/frobnicator/frobnicator.py
@@ -0,0 +1,154 @@
+###############################################################
+# Copyright 2022 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import argparse
+import os
+from abc import abstractmethod
+
+import flux
+from flux.importer import import_path, import_plugins
+from flux.job import Jobspec
+
+
+class FrobnicatorPlugin:
+    """Base class for plugins which modify jobspec in place"""
+
+    def __init__(self, parser):
+        """Initialize a FrobnicatorPlugin"""
+
+    def configure(self, args, config):
+        """Configure a FrobnicatorPlugin. Run after arguments are parsed
+
+        Args:
+           args (:obj:`Namespace`): The resulting namespace after calling
+           argparse.parse_args()
+
+           config (:obj:`dict`): The current broker config, stored as a Python
+           dictionary.
+        """
+
+    @abstractmethod
+    def frob(self, jobspec, userid, urgency, flags):
+        """Modify jobspec. A FrobnicatorPlugin must implement this method.
+
+        The plugin should modify the jobspec parameter directly. Extra
+        job information (user, urgency, flags) are available in the
+        ``info`` parameter.
+
+        Args:
+            jobspec (:obj:`Jobspec`): The jobspec to modify
+
+            userid (:obj:`int`): Submitting user
+
+            urgency (:obj:`int`): Initial job urgency
+
+            flags (:obj:`int`): Job submission flags
+
+        Returns:
+            None or raises exception.
+        """
+        raise NotImplementedError
+
+
+# pylint: disable=too-many-instance-attributes
+class JobFrobnicator:
+    """A plugin-based job modification class
+
+    JobFrobnicator loads an ordered stack of plugins that implement the
+    FrobnicatorPlugin interface from the 'flux.job.frobnicator.plugins'
+    namespace.
+    """
+
+    default_frobnicators = ["defaults", "constraints"]
+    plugin_namespace = "flux.job.frobnicator.plugins"
+
+    def __init__(self, argv, pluginpath=None, parser=None):
+
+        self.frobnicators = []
+        self.config = {}
+
+        if pluginpath is None:
+            pluginpath = []
+
+        if parser is None:
+            parser = argparse.ArgumentParser(
+                formatter_class=flux.util.help_formatter(), add_help=False
+            )
+
+        self.parser = parser
+        self.parser_group = self.parser.add_argument_group("Options")
+        self.plugins_group = self.parser.add_argument_group(
+            "Options provided by plugins"
+        )
+
+        self.parser_group.add_argument("--plugins", action="append", default=[])
+
+        args, self.remaining_args = self.parser.parse_known_args(argv)
+        if not args.plugins:
+            args.plugins = self.default_frobnicators
+        else:
+            args.plugins = [x for xs in args.plugins for x in xs.split(",")]
+
+        #  Load all available frobnicator plugins
+        self.plugins = import_plugins(self.plugin_namespace, pluginpath)
+        self.args = args
+
+    def start(self):
+        """Read broker config, select and configure frobnicator plugins"""
+
+        self.config = flux.Flux().rpc("config.get").get()
+
+        for name in self.args.plugins:
+            if name not in self.plugins:
+                try:
+                    self.plugins[name] = import_path(name)
+                except:
+                    raise ValueError(f"frobnicator plugin '{name}' not found")
+            plugin = self.plugins[name].Frobnicator(parser=self.plugins_group)
+            self.frobnicators.append(plugin)
+
+        # Parse remaining args and pass result to loaded plugins
+        args = self.parser.parse_args(self.remaining_args)
+        for frobnicator in self.frobnicators:
+            frobnicator.configure(args, config=self.config)
+
+    def frob(self, jobspec, user=None, flags=None, urgency=16):
+        """Modify jobspec using stack of loaded frobnicator plugins
+
+        Args:
+            jobspec (:obj:`Jobspec`): A Jobspec or JobspecV1 object
+                    which will be modified in place
+
+            userid (:obj:`int`): Submitting user
+
+            flags (:obj:`int`): Job submission flags
+
+            urgency (:obj:`int`): Initial job urgency
+
+        Returns:
+            :obj:`dict`: A dictionary containing a result object,
+            including keys::
+
+                {
+                    'errnum': 0,
+                    'errmsg': "An error message",
+                    'data': jobspec or None
+                }
+
+        """
+        if not isinstance(jobspec, Jobspec):
+            raise ValueError("jobspec not an instance of Jobspec")
+
+        if user is None:
+            user = os.getuid()
+
+        for frob in self.frobnicators:
+            frob.frob(jobspec, user, flags, urgency)
+        return {"errnum": 0, "data": jobspec}
diff --git a/src/bindings/python/flux/job/frobnicator/plugins/constraints.py b/src/bindings/python/flux/job/frobnicator/plugins/constraints.py
new file mode 100644
index 000000000000..cdab195f248e
--- /dev/null
+++ b/src/bindings/python/flux/job/frobnicator/plugins/constraints.py
@@ -0,0 +1,79 @@
+##############################################################
+# Copyright 2022 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+"""Apply constraints to incoming jobspec based on broker config.
+
+"""
+
+from flux.job.frobnicator import FrobnicatorPlugin
+
+
+class QueueConfig:
+    """Convenience class for handling jobspec queues configuration"""
+
+    def __init__(self, config={}):
+        self.queues = {}
+        try:
+            self.queues = config["queues"]
+        except KeyError:
+            pass
+
+    def queue_properties(self, name):
+        try:
+            return self.queues[name]["requires"]
+        except KeyError:
+            return None
+
+    def apply_constraints(self, jobspec):
+        """Apply queue-specific constraints to jobspec"""
+
+        if jobspec.queue:
+            if jobspec.queue not in self.queues:
+                raise ValueError(f"Invalid queue '{jobspec.queue}' specified")
+            queue_properties = self.queue_properties(jobspec.queue)
+            if queue_properties is None:
+                return
+
+            # First try appending to existing constraints
+            try:
+                spec = jobspec.attributes["system"]["constraints"]["properties"]
+                for prop in queue_properties:
+                    if prop not in spec:
+                        spec.append(prop)
+                return
+            except KeyError:
+                #  No "properties" operator at top level, try combining
+                #  existing constraints with logical AND
+                pass
+            try:
+                jobspec.setattr(
+                    "system.constraints",
+                    {
+                        "and": [
+                            jobspec.attributes["system"]["constraints"],
+                            {"properties": queue_properties},
+                        ]
+                    },
+                )
+            except KeyError:
+                #  No existing "constraints" - set constraints to queue
+                #  constraints
+                jobspec.setattr("system.constraints", {"properties": queue_properties})
+
+
+class Frobnicator(FrobnicatorPlugin):
+    def __init__(self, parser):
+        super().__init__(parser)
+
+    def configure(self, args, config):
+        self.config = QueueConfig(config)
+
+    def frob(self, jobspec, user, urgency, flags):
+        self.config.apply_constraints(jobspec)
diff --git a/src/bindings/python/flux/job/frobnicator/plugins/defaults.py b/src/bindings/python/flux/job/frobnicator/plugins/defaults.py
new file mode 100644
index 000000000000..e9b758ca7fc3
--- /dev/null
+++ b/src/bindings/python/flux/job/frobnicator/plugins/defaults.py
@@ -0,0 +1,94 @@
+##############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+"""Apply defaults to incoming jobspec based on broker config.
+
+"""
+
+import copy
+
+from flux.job.frobnicator import FrobnicatorPlugin
+
+
+class DefaultsConfig:
+    """Convenience class for handling jobspec defaults configuration"""
+
+    def __init__(self, config={}):
+        self.defaults = {}
+        self.queues = {}
+        self.default_queue = None
+
+        try:
+            self.defaults = config["policy"]["jobspec"]["defaults"]["system"]
+            self.default_queue = self.defaults["queue"]
+        except KeyError:
+            pass
+
+        try:
+            self.queues = config["queues"]
+        except KeyError:
+            pass
+
+        self.validate_config()
+
+    def validate_config(self):
+        if self.queues and not isinstance(self.queues, dict):
+            raise ValueError("queues must be a table")
+
+        if self.default_queue and self.default_queue not in self.queues:
+            raise ValueError(
+                f"default queue '{self.default_queue}' must be in [queues]"
+            )
+
+        for queue in self.queues:
+            self.queue_defaults(queue)
+
+    def queue_defaults(self, name):
+        """Create a copy of self.defaults updated with queue-specific values"""
+        defaults = copy.deepcopy(self.defaults)
+        if name and self.queues:
+            if name not in self.queues:
+                raise ValueError(f"Invalid queue '{name}' specified")
+            qconf = self.queues[name]
+            try:
+                qdefaults = qconf["policy"]["jobspec"]["defaults"]["system"]
+                defaults.update(qdefaults)
+                return defaults
+            except KeyError:
+                return defaults
+        return defaults
+
+    def setattr_default(self, jobspec, attr, value):
+        if attr == "duration" and jobspec.duration == 0:
+            jobspec.duration = value
+        elif attr not in jobspec.attributes["system"]:
+            jobspec.setattr(f"system.{attr}", value)
+
+    def apply_defaults(self, jobspec):
+        """Apply general defaults then queue-specific defaults to jobspec"""
+
+        queue = jobspec.queue or self.defaults.get("queue")
+        if queue is None and self.queues:
+            raise ValueError("no queue specified")
+
+        for attr, value in self.queue_defaults(queue).items():
+            self.setattr_default(jobspec, attr, value)
+
+
+class Frobnicator(FrobnicatorPlugin):
+    def __init__(self, parser):
+        self.config = DefaultsConfig()
+        super().__init__(parser)
+
+    def configure(self, args, config):
+        self.config = DefaultsConfig(config)
+
+    def frob(self, jobspec, user, urgency, flags):
+        self.config.apply_defaults(jobspec)
diff --git a/src/bindings/python/flux/job/info.py b/src/bindings/python/flux/job/info.py
new file mode 100644
index 000000000000..10204e4e11c6
--- /dev/null
+++ b/src/bindings/python/flux/job/info.py
@@ -0,0 +1,862 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import json
+import os
+import pwd
+import string
+import sys
+import time
+from collections import namedtuple
+from datetime import datetime
+from itertools import chain
+
+import flux.constants
+from flux.core.inner import raw
+from flux.job.JobID import JobID
+from flux.job.stats import JobStats
+from flux.memoized_property import memoized_property
+from flux.uri import JobURI
+
+try:
+    from flux.resource import SchedResourceList
+except ImportError:
+    SchedResourceList = None
+
+# strsignal() is only available in Python 3.8 and up.
+# flux-core's minimum is 3.6.  Use compat library if not available.
+try:
+    from signal import strsignal  # novermin
+except ImportError:
+    from flux.compat36 import strsignal
+
+
+def statetostr(stateid, fmt="L"):
+    return raw.flux_job_statetostr(stateid, fmt).decode("utf-8")
+
+
+def statetoemoji(stateid):
+    statestr = raw.flux_job_statetostr(stateid, "S").decode("utf-8")
+    if statestr == "N":
+        # wrapped gift
+        emoji = "\U0001F381"
+    elif statestr == "D":
+        # stop sign
+        emoji = "\U0001F6D1"
+    elif statestr == "P":
+        # vertical traffic light
+        emoji = "\U0001F6A6"
+    elif statestr == "S":
+        # calendar
+        emoji = "\U0001F4C5"
+    elif statestr == "R":
+        # person running
+        emoji = "\U0001F3C3"
+    elif statestr == "C":
+        # wastebasket
+        emoji = "\U0001F5D1"
+    elif statestr == "I":
+        # skull
+        emoji = "\U0001F480"
+    # can we output unicode to stdout? if not, return the normal short
+    # string
+    try:
+        emoji.encode(sys.stdout.encoding)
+    except UnicodeEncodeError:
+        return statestr
+    return emoji
+
+
+def resulttostr(resultid, fmt="L"):
+    # if result not returned, just return empty string back
+    if resultid == "":
+        return ""
+    return raw.flux_job_resulttostr(resultid, fmt).decode("utf-8")
+
+
+def resulttoemoji(resultid):
+    if resultid != "":
+        resultstr = raw.flux_job_resulttostr(resultid, "S").decode("utf-8")
+        if resultstr == "CD":
+            # grinning face
+            emoji = "\U0001F600"
+            alt = ":-)"
+        elif resultstr == "F":
+            # pile of poo
+            emoji = "\U0001F4A9"
+            alt = ":'-("
+        elif resultstr == "CA":
+            # collision
+            emoji = "\U0001F4A5"
+            alt = "%-|"
+        elif resultstr == "TO":
+            # hourglass done
+            emoji = "\u231B"
+            alt = "(-_-)"
+    else:
+        # ideographic space
+        emoji = "\u3000"
+        alt = ""
+    # can we output unicode to stdout? if not, return the ascii
+    try:
+        emoji.encode(sys.stdout.encoding)
+    except UnicodeEncodeError:
+        return alt
+    return emoji
+
+
+# Status is the job state when pending/running (i.e. not inactive)
+# status is the result when inactive
+def statustostr(stateid, resultid, fmt="L"):
+    if (stateid & flux.constants.FLUX_JOB_STATE_PENDING) or (
+        stateid & flux.constants.FLUX_JOB_STATE_RUNNING
+    ):
+        statusstr = statetostr(stateid, fmt)
+    else:  # flux.constants.FLUX_JOB_STATE_INACTIVE
+        statusstr = resulttostr(resultid, fmt)
+    return statusstr
+
+
+def statustoemoji(stateid, resultid):
+    if (stateid & flux.constants.FLUX_JOB_STATE_PENDING) or (
+        stateid & flux.constants.FLUX_JOB_STATE_RUNNING
+    ):
+        emoji = statetoemoji(stateid)
+    else:  # flux.constants.FLUX_JOB_STATE_INACTIVE
+        emoji = resulttoemoji(resultid)
+    return emoji
+
+
+def get_username(userid):
+    try:
+        return pwd.getpwuid(userid).pw_name
+    except KeyError:
+        return str(userid)
+
+
+class ExceptionInfo:
+    def __init__(self, occurred, severity, _type, note):
+        self.occurred = occurred
+        self.severity = severity
+        self.type = _type
+        self.note = note
+
+
+class EmptyObject:
+    """Convenience "empty" object for use with string.format
+
+    This class can be used in place of a real class but returns
+    appropriate empty or unset value for various conversions, or
+    for string.format() calls.
+    """
+
+    def __getattr__(self, attr):
+        return EmptyObject()
+
+    def __repr__(self):
+        return ""
+
+    def __str__(self):
+        return ""
+
+    def __format__(self, spec):
+        # Strip trailing specifier (e.g. d, f)
+        spec = spec.rstrip("bcdoxXeEfFgGn%")
+        return "".__format__(spec)
+
+
+# AnnotationsInfo is a wrapper for a namedtuple.  We need this
+# object so that we can we detect when an attribute is missing and
+# ultimately return an empty string (e.g. when an attribute does not
+# exist in a namedtuple).
+#
+# recursive namedtuple trick inspired via
+# https://stackoverflow.com/questions/1305532/convert-nested-python-dict-to-object/1305663
+class AnnotationsInfo:
+    def __init__(self, annotationsDict):
+        self.annotationsDict = annotationsDict
+        self.atuple = namedtuple("X", annotationsDict.keys())(
+            *(
+                AnnotationsInfo(v) if isinstance(v, dict) else v
+                for v in annotationsDict.values()
+            )
+        )
+
+    def __repr__(self):
+        # Special case, empty dict return empty string
+        if self.annotationsDict:
+            return json.dumps(self.annotationsDict)
+        return ""
+
+    def __getattr__(self, attr):
+        try:
+            return object.__getattribute__(self.atuple, attr)
+        except AttributeError:
+            # We return an empty object so that we can recursively
+            # handle errors.  e.g. annotations.user.illegal.illegal.illegal
+            return EmptyObject()
+
+
+class StatsInfo(JobStats):
+    """Extend JobStats with default __repr__"""
+
+    def __init__(self, handle=None):
+        super().__init__(handle)
+
+    def __repr__(self):
+        return (
+            f"PD:{self.pending} R:{self.running} "
+            f"CD:{self.successful} F:{self.failed}"
+        )
+
+    def __format__(self, fmt):
+        return str(self).__format__(fmt)
+
+
+class InstanceInfo:
+    def __init__(self, uri=None):
+        self.initialized = False
+        try:
+            if not uri or SchedResourceList is None:
+                raise ValueError
+            handle = flux.Flux(str(uri))
+            future = flux.resource.resource_list(handle)
+            self.stats = StatsInfo(handle).update_sync()
+            self.resources = future.get()
+            self.initialized = True
+            return
+        except (ValueError, OSError, FileNotFoundError):
+            self.stats = EmptyObject()
+            self.resources = EmptyObject()
+
+    @memoized_property
+    def utilization(self):
+        if self.initialized and self.resources.all.ncores:
+            res = self.resources
+            return res.allocated.ncores / res.all.ncores
+        return ""
+
+    @memoized_property
+    def gpu_utilization(self):
+        if self.initialized and self.resources.all.ngpus > 0:
+            res = self.resources
+            return res.allocated.ngpus / res.all.ngpus
+        return ""
+
+    @memoized_property
+    def progress(self):
+        if self.initialized:
+            stats = self.stats
+            if stats.total == 0:
+                return ""
+            return stats.inactive / stats.total
+        return ""
+
+    def __getattr__(self, attr):
+        if not self.initialized:
+            return ""
+        return self.__getattribute__(attr)
+
+
+class InfoList(list):
+    """Extend list with string representation appropriate for JobInfo format"""
+
+    def __str__(self):
+        return ",".join(self)
+
+
+class DependencyList(InfoList):
+    """Post processed list of dependencies."""
+
+    def __init__(self, items):
+        result = []
+        for dep in items:
+            #  Loop through dependencies and handle special cases:
+            if dep.startswith("begin-time="):
+                _, eq, timestamp = dep.partition("=")
+                dt = datetime.fromtimestamp(float(timestamp)).astimezone()
+                if dt.date() == datetime.today().date():
+                    dep = f"begin@{dt:%H:%M}"
+                else:
+                    dep = f"begin@{dt:%b%d-%H:%M}"
+                # Only add seconds if nonzero:
+                if dt.second > 0:
+                    dep += f":{dt:%S}"
+            result.append(dep)
+        super().__init__(result)
+
+
+class JobInfo:
+    """
+    JobInfo class: encapsulate job-list.list response in an object
+    that implements a getattr interface to job information with
+    memoization. Better for use with output formats since results
+    are only computed as-needed.
+    """
+
+    #  Default values for job properties.
+    defaults = {
+        "t_depend": 0.0,
+        "t_run": 0.0,
+        "t_cleanup": 0.0,
+        "t_inactive": 0.0,
+        "duration": 0.0,
+        "expiration": 0.0,
+        "name": "",
+        "cwd": "",
+        "queue": "",
+        "project": "",
+        "bank": "",
+        "ntasks": "",
+        "ncores": "",
+        "nnodes": "",
+        "priority": "",
+        "ranks": "",
+        "nodelist": "",
+        "success": "",
+        "result": "",
+        "waitstatus": "",
+    }
+
+    #  Other properties (used in to_dict())
+    properties = (
+        "id",
+        "t_submit",
+        "t_remaining",
+        "state",
+        "result",
+        "username",
+        "userid",
+        "urgency",
+        "runtime",
+        "status",
+        "returncode",
+        "dependencies",
+    )
+
+    def __init__(self, info_resp):
+        #  Set defaults, then update with job-list.list response items:
+        combined_dict = self.defaults.copy()
+        combined_dict.update(info_resp)
+
+        #  Cast jobid to JobID
+        combined_dict["id"] = JobID(combined_dict["id"])
+
+        #  Rename "state" to "state_id" and "result" to "result_id"
+        #  until returned state is a string:
+        if "state" in combined_dict:
+            combined_dict["state_id"] = combined_dict.pop("state")
+
+        if "result" in combined_dict:
+            combined_dict["result_id"] = combined_dict.pop("result")
+
+        # Overwrite "exception" with our exception object
+        exc1 = combined_dict.get("exception_occurred", "")
+        exc2 = combined_dict.get("exception_severity", "")
+        exc3 = combined_dict.get("exception_type", "")
+        exc4 = combined_dict.get("exception_note", "")
+        combined_dict["exception"] = ExceptionInfo(exc1, exc2, exc3, exc4)
+
+        aDict = combined_dict.get("annotations", {})
+        combined_dict["annotations"] = AnnotationsInfo(aDict)
+        combined_dict["sched"] = combined_dict["annotations"].sched
+        combined_dict["user"] = combined_dict["annotations"].user
+
+        deps = combined_dict.get("dependencies", [])
+        combined_dict["dependencies"] = DependencyList(deps)
+
+        #  Set all keys as self._{key} to be found by getattr and
+        #   memoized_property decorator:
+        for key, value in combined_dict.items():
+            setattr(self, "_{0}".format(key), value)
+
+    #  getattr method to return all non-computed values in job-list.list
+    #   response by default. Avoids the need to wrap @property methods
+    #   that just return self._<attr>.
+    #
+    def __getattr__(self, attr):
+        if attr.startswith("_"):
+            raise AttributeError
+        try:
+            return getattr(self, "_{0}".format(attr))
+        except (KeyError, AttributeError):
+            raise AttributeError("invalid JobInfo attribute '{}'".format(attr))
+
+    def get_instance_info(self):
+        if self.uri and self.state_single == "R":  # pylint: disable=W0143
+            setattr(self, "_instance", InstanceInfo(self.uri))
+        else:
+            setattr(self, "_instance", InstanceInfo())
+        return self
+
+    def get_runtime(self):
+        if self.t_cleanup > 0 and self.t_run > 0:
+            runtime = self.t_cleanup - self.t_run
+        elif self.t_run > 0:
+            runtime = time.time() - self.t_run
+        else:
+            runtime = 0.0
+        return runtime
+
+    def get_remaining_time(self):
+        try:
+            status = str(self.status)
+            if status != "RUN":
+                return 0.0
+            tleft = self.expiration - time.time()
+        except (KeyError, AttributeError):
+            # expiration and/or status attributes may not exist, return 0.0
+            return 0.0
+        if tleft < 0.0:
+            return 0.0
+        return tleft
+
+    @memoized_property
+    def uri(self):
+        if str(self.user.uri):
+            return JobURI(self.user.uri)
+        return None
+
+    @property
+    def t_remaining(self):
+        return self.get_remaining_time()
+
+    @memoized_property
+    def state(self):
+        return statetostr(self.state_id)
+
+    @memoized_property
+    def state_single(self):
+        return statetostr(self.state_id, fmt="S")
+
+    @memoized_property
+    def state_emoji(self):
+        return statetoemoji(self.state_id)
+
+    @memoized_property
+    def result(self):
+        return resulttostr(self.result_id)
+
+    @memoized_property
+    def result_abbrev(self):
+        return resulttostr(self.result_id, "S")
+
+    @memoized_property
+    def result_emoji(self):
+        return resulttoemoji(self.result_id)
+
+    @memoized_property
+    def username(self):
+        return get_username(self.userid)
+
+    @memoized_property
+    def runtime(self):
+        return self.get_runtime()
+
+    @memoized_property
+    def status(self):
+        return statustostr(self.state_id, self.result_id)
+
+    @memoized_property
+    def status_abbrev(self):
+        return statustostr(self.state_id, self.result_id, fmt="S")
+
+    @memoized_property
+    def status_emoji(self):
+        return statustoemoji(self.state_id, self.result_id)
+
+    @memoized_property
+    def returncode(self):
+        """
+        The job return code if the job has exited, or an empty string
+        if the job is still active. The return code of a job is the
+        highest job shell exit code, or the negative signal number if the
+        job shell was terminated by a signal. For jobs that were canceled
+        before the RUN state, the return code will be set to -128.
+        """
+        status = self.waitstatus
+        code = ""
+        if not isinstance(status, int):
+            if self.result_id == flux.constants.FLUX_JOB_RESULT_CANCELED:
+                code = -128
+            elif self.result_id == flux.constants.FLUX_JOB_RESULT_FAILED:
+                #  A job with empty waitstatus could fail if it received a
+                #  fatal exception before starting. Use generic returncode
+                #  of 1.
+                code = 1
+        elif os.WIFSIGNALED(status):
+            code = -os.WTERMSIG(status)
+        elif os.WIFEXITED(status):
+            code = os.WEXITSTATUS(status)
+        return code
+
+    @memoized_property
+    def contextual_info(self):
+        """
+        Generate contextual nodelist/reason information based on job state:
+         PRIORITY: returns "priority-wait"
+         DEPEND:   returns depends:dependencies list
+         SCHED:    returns eta:sched.t_estimate if available, "held" if
+                   urgency=0, or "priority-hold" if urgency>0 and priority=0.
+         RUN+:     returns assigned nodelist
+        """
+        # Required for pylint, o/w it thinks state is a callable:
+        state = str(self.state)
+        if state == "PRIORITY":
+            return "priority-wait"
+        if state == "DEPEND":
+            return f"depends:{self.dependencies}"
+        if state == "SCHED":
+            try:
+                eta = self.sched.t_estimate - time.time()
+                if eta < 0:
+                    eta = "now"
+                else:
+                    eta = flux.util.fsd(eta)
+                return f"eta:{eta}"
+            except TypeError:
+                # No eta available. Print "held" if job is held, or
+                # priority-hold if priority == 0, otherwise nothing.
+                if self.urgency == 0:
+                    return "held"
+                elif self.priority == 0:
+                    return "priority-hold"
+                return ""
+        else:
+            return self.nodelist
+
+    @memoized_property
+    def contextual_time(self):
+        """
+        Return job duration if job is not running, otherwise runtime
+        """
+        state = str(self.state)
+        if state in ["PRIORITY", "DEPEND", "SCHED"]:
+            return self.duration
+        return self.runtime
+
+    def to_dict(self, filtered=True):
+        """
+        Return a set of job attributes as a dict
+        By default, empty or unset values are filtered from the result,
+        so these keys will be missing. Set ``filtered=False`` to get the
+        unfiltered dict, which has these uninitialized values set to
+        an empty string or 0, key dependent.
+        """
+        result = {}
+        for attr in chain(self.defaults.keys(), self.properties):
+            try:
+                val = getattr(self, attr)
+            except AttributeError:
+                val = None
+            if val is not None:
+                result[attr] = val
+
+        #  The following attributes all need special handling to
+        #  be converted to a dict:
+        result["annotations"] = self.annotations.annotationsDict
+        result["exception"] = self.exception.__dict__
+        if self.uri is not None:
+            result["uri"] = str(self.uri)
+
+        if not filtered:
+            return result
+
+        #  Now clear empty/unset values to avoid confusion:
+        #  - Remove any empty values (empty string).
+        #  - Remove any unset timestamp values (key t_*, value 0)
+        #  - Remove runtime and expiration if 0
+        def zero_remove(key):
+            return key.startswith("t_") or key in ("runtime", "expiration")
+
+        for key in list(result.keys()):
+            val = result[key]
+            if val == "" or (zero_remove(key) and val == 0):
+                del result[key]
+            if key == "exception" and not val["occurred"]:
+                result["exception"] = {"occurred": False}
+
+        #  Remove duplicate annotations.user.uri if necessary:
+        if self.uri is not None:
+            del result["annotations"]["user"]["uri"]
+            if not result["annotations"]["user"]:
+                del result["annotations"]["user"]
+
+        return result
+
+    @memoized_property
+    def inactive_reason(self):
+        """
+        Generate contextual exit reason based on how the job ended
+        """
+        state = str(self.state)
+        if state != "INACTIVE":
+            return ""
+        result = str(self.result)
+        if result == "CANCELED":
+            if (
+                self.exception.occurred
+                and self.exception.type == "cancel"
+                and self.exception.note
+            ):
+                return f"Canceled: {self.exception.note}"
+            else:
+                return "Canceled"
+        elif result == "FAILED":
+            # exception.type == "exec" is special case, handled by returncode
+            if (
+                self.exception.occurred
+                and self.exception.type != "exec"
+                and self.exception.severity == 0
+            ):
+                note = None
+                if self.exception.note:
+                    note = f" note={self.exception.note}"
+                return f'Exception: type={self.exception.type}{note or ""}'
+            elif self.returncode > 128:
+                signum = self.returncode - 128
+                try:
+                    sigdesc = strsignal(signum)
+                except ValueError:
+                    sigdesc = f"Signaled {signum}"
+                return sigdesc
+            elif self.returncode == 126:
+                return "Command invoked cannot execute"
+            elif self.returncode == 127:
+                return "command not found"
+            elif self.returncode == 128:
+                return "Invalid argument to exit"
+            else:
+                return f"Exit {self.returncode}"
+        elif result == "TIMEOUT":
+            return "Timeout"
+        else:
+            return f"Exit {self.returncode}"
+
+
+def job_fields_to_attrs(fields):
+    # Note there is no attr for "id", it is always returned
+    fields2attrs = {
+        "id": (),
+        "id.dec": (),
+        "id.hex": (),
+        "id.f58": (),
+        "id.f58plain": (),
+        "id.emoji": (),
+        "id.kvs": (),
+        "id.words": (),
+        "id.dothex": (),
+        "userid": ("userid",),
+        "username": ("userid",),
+        "urgency": ("urgency",),
+        "priority": ("priority",),
+        "state": ("state",),
+        "state_single": ("state",),
+        "state_emoji": ("state",),
+        "name": ("name",),
+        "cwd": ("cwd",),
+        "queue": ("queue",),
+        "project": ("project",),
+        "bank": ("bank",),
+        "ntasks": ("ntasks",),
+        "ncores": ("ncores",),
+        "duration": ("duration",),
+        "nnodes": ("nnodes",),
+        "ranks": ("ranks",),
+        "nodelist": ("nodelist",),
+        "success": ("success",),
+        "waitstatus": ("waitstatus",),
+        "returncode": ("waitstatus", "result"),
+        "exception.occurred": ("exception_occurred",),
+        "exception.severity": ("exception_severity",),
+        "exception.type": ("exception_type",),
+        "exception.note": ("exception_note",),
+        "result": ("result",),
+        "result_abbrev": ("result",),
+        "result_emoji": ("result",),
+        "t_submit": ("t_submit",),
+        "t_depend": ("t_depend",),
+        "t_run": ("t_run",),
+        "t_cleanup": ("t_cleanup",),
+        "t_inactive": ("t_inactive",),
+        "runtime": ("t_run", "t_cleanup"),
+        "status": ("state", "result"),
+        "status_abbrev": ("state", "result"),
+        "status_emoji": ("state", "result"),
+        "expiration": ("expiration", "state", "result"),
+        "t_remaining": ("expiration", "state", "result"),
+        "annotations": ("annotations",),
+        "dependencies": ("dependencies",),
+        "contextual_info": (
+            "state",
+            "dependencies",
+            "annotations",
+            "nodelist",
+            "priority",
+            "urgency",
+        ),
+        "contextual_time": ("state", "t_run", "t_cleanup", "duration"),
+        "inactive_reason": (
+            "state",
+            "result",
+            "waitstatus",
+            "exception_occurred",
+            "exception_severity",
+            "exception_type",
+            "exception_note",
+        ),
+        # Special cases, pointers to sub-dicts in annotations
+        "sched": ("annotations",),
+        "user": ("annotations",),
+        "uri": ("annotations",),
+        "uri.local": ("annotations",),
+    }
+
+    attrs = set()
+    for field in fields:
+        # Special case for annotations, can be arbitrary field names determined
+        # by scheduler/user.
+        if (
+            field.startswith("annotations.")
+            or field.startswith("sched.")
+            or field.startswith("user.")
+        ):
+            attrs.update(fields2attrs["annotations"])
+        elif field.startswith("instance."):
+            attrs.update(fields2attrs["annotations"])
+            attrs.update(fields2attrs["status"])
+        else:
+            attrs.update(fields2attrs[field])
+
+    return attrs
+
+
+class JobInfoFormat(flux.util.OutputFormat):
+    """
+    Store a parsed version of an output format string for JobInfo objects,
+    allowing the fields to iterated without modifiers, building
+    a new format suitable for headers display, etc...
+    """
+
+    #  List of legal format fields and their header names
+    headings = {
+        "id": "JOBID",
+        "id.dec": "JOBID",
+        "id.hex": "JOBID",
+        "id.f58": "JOBID",
+        "id.f58plain": "JOBID",
+        "id.emoji": "JOBID",
+        "id.kvs": "JOBID",
+        "id.words": "JOBID",
+        "id.dothex": "JOBID",
+        "userid": "UID",
+        "username": "USER",
+        "urgency": "URG",
+        "priority": "PRI",
+        "state": "STATE",
+        "state_single": "S",
+        "state_emoji": "STATE",
+        "name": "NAME",
+        "cwd": "CWD",
+        "queue": "QUEUE",
+        "project": "PROJECT",
+        "bank": "BANK",
+        "ntasks": "NTASKS",
+        "ncores": "NCORES",
+        "duration": "DURATION",
+        "nnodes": "NNODES",
+        "expiration": "EXPIRATION",
+        "t_remaining": "T_REMAINING",
+        "ranks": "RANKS",
+        "nodelist": "NODELIST",
+        "success": "SUCCESS",
+        "result": "RESULT",
+        "result_abbrev": "RS",
+        "result_emoji": "RESULT",
+        "t_submit": "T_SUBMIT",
+        "t_depend": "T_DEPEND",
+        "t_run": "T_RUN",
+        "t_cleanup": "T_CLEANUP",
+        "t_inactive": "T_INACTIVE",
+        "runtime": "RUNTIME",
+        "status": "STATUS",
+        "status_abbrev": "ST",
+        "status_emoji": "STATUS",
+        "waitstatus": "WSTATUS",
+        "returncode": "RC",
+        "exception.occurred": "EXCEPTION-OCCURRED",
+        "exception.severity": "EXCEPTION-SEVERITY",
+        "exception.type": "EXCEPTION-TYPE",
+        "exception.note": "EXCEPTION-NOTE",
+        "annotations": "ANNOTATIONS",
+        "dependencies": "DEPENDENCIES",
+        "contextual_info": "INFO",
+        "contextual_time": "TIME",
+        "inactive_reason": "INACTIVE-REASON",
+        # The following are special pre-defined cases per RFC27
+        "annotations.sched.t_estimate": "T_ESTIMATE",
+        "annotations.sched.reason_pending": "REASON",
+        "annotations.sched.resource_summary": "RESOURCES",
+        "sched": "SCHED",
+        "sched.t_estimate": "T_ESTIMATE",
+        "sched.reason_pending": "REASON",
+        "sched.resource_summary": "RESOURCES",
+        "user": "USER",
+        "uri": "URI",
+        "uri.local": "URI",
+        "instance.stats.total": "NJOBS",
+        "instance.utilization": "CORE%",
+        "instance.gpu_utilization": "GPU%",
+        "instance.progress": "PROG",
+        "instance.resources.all.ncores": "CORES",
+        "instance.resources.all.ngpus": "GPUS",
+        "instance.resources.all.nnodes": "NODES",
+        "instance.resources.up.ncores": "UP",
+        "instance.resources.up.ngpus": "UP",
+        "instance.resources.up.nnodes": "UP",
+        "instance.resources.down.ncores": "DOWN",
+        "instance.resources.down.ngpus": "DOWN",
+        "instance.resources.down.nnodes": "DOWN",
+        "instance.resources.allocated.ncores": "USED",
+        "instance.resources.allocated.ngpus": "USED",
+        "instance.resources.allocated.nnodes": "USED",
+        "instance.resources.free.ncores": "FREE",
+        "instance.resources.free.ngpus": "FREE",
+        "instance.resources.free.nnodes": "FREE",
+    }
+
+    def __init__(self, fmt, headings=None, prepend=None):
+        """
+        Parse the input format fmt with string.Formatter.
+        Save off the fields and list of format tokens for later use,
+        (converting None to "" in the process)
+
+        Throws an exception if any format fields do not match the allowed
+        list of headings above.
+
+        Special case for annotations, which may be arbitrary
+        creations of scheduler or user.
+        """
+        format_list = string.Formatter().parse(fmt)
+        for _, field, _, _ in format_list:
+            if field and not field in self.headings:
+                if field.startswith("annotations."):
+                    field_heading = field[len("annotations.") :].upper()
+                    self.headings[field] = field_heading
+                elif field.startswith("sched.") or field.startswith("user."):
+                    field_heading = field.upper()
+                    self.headings[field] = field_heading
+                elif field.startswith("instance."):
+                    field_heading = field[9:].upper()
+                    #  Shorten RESOURCES. headings
+                    if field_heading.startswith("RESOURCES."):
+                        field_heading = field_heading[10:]
+                    self.headings[field] = field_heading
+        super().__init__(fmt)
diff --git a/src/bindings/python/flux/job/journal.py b/src/bindings/python/flux/job/journal.py
new file mode 100644
index 000000000000..9763132da3d2
--- /dev/null
+++ b/src/bindings/python/flux/job/journal.py
@@ -0,0 +1,278 @@
+###############################################################
+# Copyright 2024 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import errno
+from collections import deque
+
+from flux.constants import FLUX_RPC_NORESPONSE, FLUX_RPC_STREAMING
+from flux.job import JobID
+from flux.job.event import EventLogEvent
+
+
+class JournalEvent(EventLogEvent):
+    """A container for an event from the job manager journal
+
+    Attributes:
+        jobid (:obj:`flux.job.JobID`): The job id for which the event applies
+        name (str): event name (See :rfc:`21` for possible event names)
+        timestamp (float): event timestamp in seconds since the epoch
+            with sub-millisecond precision.
+        context (dict): context dictionary
+            (See `RFC 21: Event Descriptions <https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_21.html#event-descriptions>`_.)
+        context_string (str): context dict converted to comma separated
+            key=value string.
+        jobspec (dict): For ``submit`` events, the job's redacted jobspec
+            (See :rfc:`25`).
+        R (dict): For alloc events, the job's assigned R (See :rfc:`20`)
+    """
+
+    def __init__(self, jobid, event, jobspec=None, R=None):
+        super().__init__(event)
+        self.jobid = JobID(jobid)
+        self.jobspec = jobspec
+        self.R = R
+
+    def is_empty(self):
+        """Return True if this event is an empty journal event"""
+        if self.jobid == -1:
+            return True
+        return False
+
+    def __str__(self):
+        if self.is_empty():
+            return "-1: End of historical data stream"
+        return " ".join(
+            [
+                f"{self.jobid.f58}:",
+                f"{self.timestamp:<0.5f}",
+                self.name,
+                self.context_string,
+            ]
+        )
+
+
+SENTINEL_EVENT = JournalEvent(-1, {})
+"""
+A constant :obj:`JournalEvent` demarcating the transition from historical
+events to current events when a :obj:`JournalConsumer` is created with
+*include_sentinel* set to True.
+"""
+
+
+class JournalConsumer:
+    """Class for consuming the job manager journal
+
+    This class is a wrapper around the ``job-manager.events-journal`` RPC,
+    which allows clients to subscribe to primary job events for all jobs
+    (See :rfc:`21`) in a single interface.
+
+    A :obj:`JournalConsumer` returns individual job events
+    as :obj:`JournalEvent` objects either synchronously, via the
+    :func:`poll` method, or asynchronously via a callback registered with
+    the :func:`set_callback` method. In the case of asynchronous mode, the
+    Flux reactor must be run via :func:`~flux.core.handle.Flux.reactor_run`.
+
+    A :obj:`JournalConsumer` is created by passing the constructor an open
+    :obj:`~flux.Flux` handle, along with any other optional parameters
+    described below. The :func:`start` method should then be called,
+    which sends the initial RPC. To stop the stream of events, call the
+    :func:`stop` method. After all events have been processed :func:`poll`
+    or the registered callback will return None.
+
+    When the consumer is first started, historical data (events in the
+    past) will be sent from the job manager unless *full* is set to False.
+    These events are stored until all historical events are processed,
+    then are time ordered before returning them via :func:`poll` or to
+    the callback registered by :func:`set_callback`.
+
+    To avoid processing previously seen events with a new instance of this
+    class, the timestamp of the newest processed event can be passed via the
+    *since* parameter. Timestamps should be unique so :func:`poll` or the
+    callback will start with the newest event after the *since* timestamp.
+
+    When *full* is True, the job manager sends a sentinel event in
+    the journal stream to demarcate the transition between history
+    and current events. If *include_sentinel* is set to True,
+    then an empty event will be returned by :func:`poll` or the
+    registered callback to represent the sentinel. An empty event can
+    be compared to :data:`flux.job.journal.SENTINEL_EVENT` or by using the
+    :func:`JournalEvent.is_empty` method. The default behavior is to
+    suppress the sentinel.
+
+    """
+
+    def __init__(self, flux_handle, full=True, since=0.0, include_sentinel=False):
+        self.handle = flux_handle
+        self.backlog = deque()
+        self.full = full
+        self.since = since
+        self.rpc = None
+        self.cb = None
+        self.cb_args = []
+        self.cb_kwargs = {}
+        self.processing_inactive = self.full
+        self.include_sentinel = include_sentinel
+
+    def __sort_backlog(self):
+        self.processing_inactive = False
+        self.backlog = deque(sorted(self.backlog, key=lambda x: x.timestamp))
+        if self.include_sentinel:
+            self.backlog.append(SENTINEL_EVENT)
+
+    def __enqueue_response(self, resp):
+        if resp is None:
+            # End of data, enqueue None:
+            self.backlog.append(None)
+            return
+
+        jobid = resp["id"]
+        jobspec = resp.get("jobspec")
+        R = resp.get("R")
+        for entry in resp["events"]:
+            event = JournalEvent(jobid, entry)
+            if event.timestamp > self.since:
+                if event.name == "submit":
+                    event.jobspec = jobspec or None
+                elif event.name == "alloc":
+                    event.R = R or None
+                self.backlog.append(event)
+
+    def __next_event(self):
+        return self.backlog.popleft()
+
+    def __set_then_cb(self):
+        if self.rpc is not None and self.cb is not None:
+            try:
+                self.rpc.then(self.__cb)
+            except OSError as exc:
+                if exc.errno == errno.EINVAL:
+                    # then callback is already set
+                    pass
+
+    def start(self):
+        """Start the stream of events by sending a request to the job manager
+
+        This function sends the job-manager.events-journal RPC to the
+        job manager. It must be called to start the stream of events.
+
+        .. note::
+            If :func:`start` is called more than once the stream of events
+            will be restarted using the original options passed to the
+            constructor. This may cause duplicate events, or missed events
+            if *full* is False since no history will be included.
+        """
+        self.rpc = self.handle.rpc(
+            "job-manager.events-journal", {"full": self.full}, 0, FLUX_RPC_STREAMING
+        )
+        #  Need to call self.rpc.then() if a user cb is registered:
+        self.__set_then_cb()
+
+        return self
+
+    def stop(self):
+        """Cancel the job-manager.events-journal RPC
+
+        Cancel the RPC. This will eventually stop the stream of events to
+        either :func:`poll` or the defined callback. After all events have
+        been processed an *event* of None will be returned by :func:`poll`
+        or the defined callback.
+        """
+        self.handle.rpc(
+            "job-manager.events-journal-cancel",
+            {"matchtag": self.rpc.pimpl.get_matchtag()},
+            0,
+            FLUX_RPC_NORESPONSE,
+        )
+
+    def poll(self, timeout=-1.0):
+        """Synchronously get the next job event
+
+        if *full* is True, then this call will not return until all
+        historical events have been processed. Historical events will sorted
+        in time order and returned once per :func:`poll` call.
+
+        :func:`start` must be called before this function.
+
+        Args:
+            timeout (float): Only wait *timeout* seconds for the next event.
+                If the timeout expires then a :exc:`TimeoutError` is raised.
+                A *timeout* of -1.0 disables any timeout.
+        Raises:
+            RuntimeError:  :func:`poll` was called before :func:`start`.
+        """
+        if self.rpc is None:
+            raise RuntimeError("poll() called before start()")
+
+        if self.processing_inactive:
+            # process backlog. Time order events once done:
+            while self.processing_inactive:
+                resp = self.rpc.wait_for(timeout).get()
+                if resp["id"] == -1:
+                    self.__sort_backlog()
+                else:
+                    self.__enqueue_response(resp)
+                self.rpc.reset()
+
+        while not self.backlog:
+            try:
+                resp = self.rpc.wait_for(timeout).get()
+            except OSError as exc:
+                if exc.errno != errno.ENODATA:
+                    raise
+                resp = None
+            self.__enqueue_response(resp)
+            self.rpc.reset()
+
+        return self.__next_event()
+
+    def __user_cb_flush(self):
+        if self.processing_inactive:
+            #  all events are accumulated in the backlog while we're still
+            #  processing inactive job events so that those events can be
+            #  sorted in timestamp order:
+            return
+        while self.backlog:
+            self.cb(self.__next_event(), *self.cb_args, **self.cb_kwargs)
+
+    def __cb(self, future):
+        try:
+            resp = future.get()
+            if self.processing_inactive and resp["id"] == -1:
+                self.__sort_backlog()
+            else:
+                self.__enqueue_response(resp)
+            self.__user_cb_flush()
+        except OSError as exc:
+            if exc.errno == errno.ENODATA:
+                self.__enqueue_response(None)
+        finally:
+            future.reset()
+
+    def set_callback(self, event_cb, *args, **kwargs):
+        """Register callback *event_cb* to be called for each job event
+
+        If provided, ``*args``, and ``**kwargs`` are passed along to
+        *event_cb*, whose only required argument is an *event*, e.g.
+
+        >>> def event_cb(event)
+        >>>     # do something with event
+
+        After a :obj:`JournalConsumer` is stopped and the final event is
+        received, *event_cb* will be called with an *event* of None, which
+        signals the end of the event stream.
+        """
+        self.cb = event_cb
+        self.cb_args = args
+        self.cb_kwargs = kwargs
+        self.__set_then_cb()
+        return self
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/bindings/python/flux/job/kill.py b/src/bindings/python/flux/job/kill.py
new file mode 100644
index 000000000000..f374f467f340
--- /dev/null
+++ b/src/bindings/python/flux/job/kill.py
@@ -0,0 +1,74 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+import signal
+from typing import Optional, Union
+
+from _flux._core import ffi
+from flux.core.handle import Flux  # for typing
+from flux.future import Future
+from flux.job._wrapper import _RAW as RAW
+from flux.job.JobID import JobID  # for typing
+
+
+def kill_async(
+    flux_handle: Flux, jobid: Union[JobID, int], signum: Optional[int] = None
+):
+    """Send a signal to a running job asynchronously
+
+    :param flux_handle: handle for Flux broker from flux.Flux()
+    :type flux_handle: Flux
+    :param jobid: the job ID of the job to kill
+    :param signum: signal to send (default SIGTERM)
+    :returns: a Future
+    :rtype: Future
+    """
+    if not signum:
+        signum = signal.SIGTERM
+    return Future(RAW.kill(flux_handle, int(jobid), signum))
+
+
+def kill(flux_handle: Flux, jobid: Union[JobID, int], signum: Optional[int] = None):
+    """Send a signal to a running job.
+
+    :param flux_handle: handle for Flux broker from flux.Flux()
+    :type flux_handle: Flux
+    :param jobid: the job ID of the job to kill
+    :param signum: signal to send (default SIGTERM)
+    """
+    return kill_async(flux_handle, jobid, signum).get()
+
+
+def cancel_async(
+    flux_handle: Flux, jobid: Union[JobID, int], reason: Optional[str] = None
+):
+    """Cancel a pending or or running job asynchronously
+
+    Arguments:
+        flux_handle: handle for Flux broker from flux.Flux()
+        jobid: the job ID of the job to cancel
+        reason: the textual reason associated with the cancelation
+
+    Returns:
+        Future: a future fulfilled when the cancelation completes
+    """
+    if not reason:
+        reason = ffi.NULL
+    return Future(RAW.cancel(flux_handle, int(jobid), reason))
+
+
+def cancel(flux_handle: Flux, jobid: Union[JobID, int], reason: Optional[str] = None):
+    """Cancel a pending or or running job
+
+    Arguments:
+        flux_handle: handle for Flux broker from flux.Flux()
+        jobid: the job ID of the job to cancel
+        reason: the textual reason associated with the cancelation
+    """
+    return cancel_async(flux_handle, jobid, reason).get()
diff --git a/src/bindings/python/flux/job/kvs.py b/src/bindings/python/flux/job/kvs.py
new file mode 100644
index 000000000000..ce3ea5a9d54b
--- /dev/null
+++ b/src/bindings/python/flux/job/kvs.py
@@ -0,0 +1,38 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+import flux.kvs
+from flux.job._wrapper import _RAW as RAW
+from _flux._core import ffi
+
+
+def job_kvs(flux_handle, jobid):
+    """
+    :returns: The KVS directory of the given job
+    :rtype: KVSDir
+    """
+
+    path_len = 1024
+    buf = ffi.new("char[]", path_len)
+    RAW.kvs_key(buf, path_len, jobid, "")
+    kvs_key = ffi.string(buf, path_len).decode("utf-8")
+    return flux.kvs.get_dir(flux_handle, kvs_key)
+
+
+def job_kvs_guest(flux_handle, jobid):
+    """
+    :returns: The KVS guest directory of the given job
+    :rtype: KVSDir
+    """
+
+    path_len = 1024
+    buf = ffi.new("char[]", path_len)
+    RAW.kvs_guest_key(buf, path_len, jobid, "")
+    kvs_key = ffi.string(buf, path_len).decode("utf-8")
+    return flux.kvs.get_dir(flux_handle, kvs_key)
diff --git a/src/bindings/python/flux/job/kvslookup.py b/src/bindings/python/flux/job/kvslookup.py
new file mode 100644
index 000000000000..2246da9af754
--- /dev/null
+++ b/src/bindings/python/flux/job/kvslookup.py
@@ -0,0 +1,240 @@
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+import errno
+import json
+
+import flux.constants
+from _flux._core import ffi, lib
+from flux.future import WaitAllFuture
+from flux.job import JobID
+from flux.rpc import RPC
+
+
+def _decode_field(data, key):
+    try:
+        tmp = json.loads(data[key])
+        data[key] = tmp
+    except json.decoder.JSONDecodeError:
+        # Ignore if can't be decoded
+        pass
+
+
+# a few keys are special, decode them into dicts if you can
+def decode_special_data(data):
+    for key in ("jobspec", "R"):
+        if key in data:
+            _decode_field(data, key)
+
+
+class JobInfoLookupRPC(RPC):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.jobid = None
+
+    def get(self):
+        return super().get()
+
+    def get_decode(self):
+        data = super().get()
+        decode_special_data(data)
+        return data
+
+
+def job_info_lookup(flux_handle, jobid, keys=["jobspec"], flags=0):
+    payload = {"id": int(jobid), "keys": keys, "flags": flags}
+    rpc = JobInfoLookupRPC(flux_handle, "job-info.lookup", payload)
+    rpc.jobid = jobid
+    return rpc
+
+
+def _setup_lookup_keys(keys, original):
+    if "jobspec" in keys:
+        if original:
+            keys.remove("jobspec")
+            if "J" not in keys:
+                keys.append("J")
+
+
+def _get_original_jobspec(job_data):
+    J = bytes(job_data["J"], encoding="utf8")
+    val = lib.flux_unwrap_string(J, False, ffi.NULL, ffi.NULL)
+    result = ffi.string(val)
+    lib.free(val)
+    return result.decode("utf-8")
+
+
+def _update_keys(job_data, decode, keys, original):
+    if "jobspec" in keys:
+        if original:
+            job_data["jobspec"] = _get_original_jobspec(job_data)
+            if decode:
+                _decode_field(job_data, "jobspec")
+            if "J" not in keys:
+                job_data.pop("J")
+
+
+# jobs_kvs_lookup simple variant for one jobid
+def job_kvs_lookup(
+    flux_handle, jobid, keys=["jobspec"], decode=True, original=False, base=False
+):
+    """
+    Lookup job kvs data based on a jobid
+
+    Some keys such as "jobspec" or "R" may be altered based on update events
+    in the eventlog.  Set 'base' to True to skip these updates and
+    read exactly what is in the KVS.
+
+    :flux_handle: A Flux handle obtained from flux.Flux()
+    :jobid: jobid to lookup info for
+    :keys: Optional list of keys to fetch. (default is "jobspec")
+    :decode: Optional flag to decode special data into Python data structures
+             currently decodes "jobspec" and "R" into dicts
+             (default True)
+    :original: For 'jobspec', return the original submitted jobspec
+    :base: For 'jobspec' or 'R', get base value, do not apply updates from eventlog
+    """
+    keyslookup = list(keys)
+    _setup_lookup_keys(keyslookup, original)
+    # N.B. JobInfoLookupRPC() has a "get_decode()" member
+    # function, so we will always get the non-decoded result from
+    # job-info.
+    flags = 0
+    if not base:
+        flags |= flux.constants.FLUX_JOB_LOOKUP_CURRENT
+    payload = {"id": int(jobid), "keys": keyslookup, "flags": flags}
+    rpc = JobInfoLookupRPC(flux_handle, "job-info.lookup", payload)
+    try:
+        if decode:
+            rsp = rpc.get_decode()
+        else:
+            rsp = rpc.get()
+        _update_keys(rsp, decode, keys, original)
+    # The job does not exist!
+    except FileNotFoundError:
+        return None
+    return rsp
+
+
+class JobKVSLookupFuture(WaitAllFuture):
+    """Wrapper Future for multiple jobids"""
+
+    def __init__(self):
+        super(JobKVSLookupFuture, self).__init__()
+        self.errors = []
+
+    def _get(self, decode=True):
+        jobs = []
+        #  Wait for all RPCs to complete
+        self.wait_for()
+
+        #  Get all successful jobs, accumulate errors in self.errors
+        for child in self.children:
+            try:
+                if decode:
+                    rsp = child.get_decode()
+                else:
+                    rsp = child.get()
+                jobs.append(rsp)
+            except EnvironmentError as err:
+                if err.errno == errno.ENOENT:
+                    msg = f"JobID {child.jobid.orig} unknown"
+                else:
+                    msg = f"rpc: {err.strerror}"
+                self.errors.append(msg)
+        return jobs
+
+    def get(self):
+        """get all successful results, appending errors into self.errors"""
+        return self._get(False)
+
+    def get_decode(self):
+        """
+        get all successful results, appending errors into self.errors.  Decode
+        special data into Python data structures
+        """
+        return self._get(True)
+
+
+class JobKVSLookup:
+    """User friendly class to lookup job KVS data
+
+    Some keys such as "jobspec" or "R" may be altered based on update events
+    in the eventlog.  Set 'base' to True to skip these updates and
+    read exactly what is in the KVS.
+
+    :flux_handle: A Flux handle obtained from flux.Flux()
+    :ids: List of jobids to get data for
+    :keys: Optional list of keys to fetch. (default is "jobspec")
+    :decode: Optional flag to decode special data into Python data structures
+             currently decodes "jobspec" and "R" into dicts
+             (default True)
+    :original: For 'jobspec', return the original submitted jobspec
+    :base: For 'jobspec' or 'R', get base value, do not apply updates from eventlog
+    """
+
+    def __init__(
+        self,
+        flux_handle,
+        ids=[],
+        keys=["jobspec"],
+        decode=True,
+        original=False,
+        base=False,
+    ):
+        self.handle = flux_handle
+        self.keys = list(keys)
+        self.keyslookup = list(keys)
+        self.ids = list(map(JobID, ids)) if ids else []
+        self.decode = decode
+        self.original = original
+        self.base = base
+        self.errors = []
+        _setup_lookup_keys(self.keyslookup, self.original)
+
+    def fetch_data(self):
+        """Initiate the job info lookup to the Flux job-info module
+
+        JobKVSLookup.fetch_data() returns a JobKVSLookupFuture,
+        which will be fulfilled when the job data is available.
+
+        Once the Future has been fulfilled, a list of objects
+        can be obtained via JobKVSLookup.data(). If
+        JobKVSLookupFuture.errors is non-empty, then it will contain a
+        list of errors returned via the query.
+        """
+        flags = 0
+        # N.B. JobInfoLookupRPC() has a "get_decode()" member
+        # function, so we will always get the non-decoded result from
+        # job-info.
+        if not self.base:
+            flags |= flux.constants.FLUX_JOB_LOOKUP_CURRENT
+        listids = JobKVSLookupFuture()
+        for jobid in self.ids:
+            listids.push(job_info_lookup(self.handle, jobid, self.keyslookup, flags))
+        return listids
+
+    def data(self):
+        """Synchronously fetch a list of data responses
+
+        If the Future object returned by JobKVSLookup.fetch_data has
+        not yet been fulfilled (e.g. is_ready() returns False), then this call
+        may block.  Otherwise, returns a list of responses for all job ids
+        returned.
+        """
+        rpc = self.fetch_data()
+        if self.decode:
+            data = rpc.get_decode()
+        else:
+            data = rpc.get()
+        if hasattr(rpc, "errors"):
+            self.errors = rpc.errors
+        for job_data in data:
+            _update_keys(job_data, self.decode, self.keys, self.original)
+        return data
diff --git a/src/bindings/python/flux/job/list.py b/src/bindings/python/flux/job/list.py
new file mode 100644
index 000000000000..1b5dab66b029
--- /dev/null
+++ b/src/bindings/python/flux/job/list.py
@@ -0,0 +1,343 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+import errno
+import os
+import pwd
+from collections.abc import Iterable
+
+import flux.constants
+from flux.future import WaitAllFuture
+from flux.job import JobID
+from flux.job.info import JobInfo
+from flux.rpc import RPC
+
+
+class JobListRPC(RPC):
+    def get_jobs(self):
+        """Returns all jobs in the RPC."""
+        return self.get()["jobs"]
+
+    def get_jobinfos(self):
+        """Yields a JobInfo object for each job in its current state.
+
+        :rtype: JobInfo
+        """
+        for job in self.get_jobs():
+            yield JobInfo(job)
+
+
+# Due to subtleties in the python bindings and this call, this binding
+# is more of a reimplementation of flux_job_list() instead of calling
+# the flux_job_list() C function directly.  Some reasons:
+#
+# - Desire to return a Python RPC class and use its get() method
+# - Desired return value is json array, not a single value
+#
+# pylint: disable=dangerous-default-value
+def job_list(
+    flux_handle,
+    max_entries=1000,
+    attrs=["all"],
+    userid=os.getuid(),
+    states=0,
+    results=0,
+    since=0.0,
+    name=None,
+    queue=None,
+    constraint=None,
+):
+    if constraint is None:
+        # N.B. an "and" operation with no values returns everything
+        constraint = {"and": []}
+    else:
+        # O/w, a provided constraint will be anded with other parameters below:
+        constraint = {"and": [constraint]}
+
+    if userid != flux.constants.FLUX_USERID_UNKNOWN:
+        constraint["and"].append({"userid": [userid]})
+    if name:
+        constraint["and"].append({"name": [name]})
+    if queue:
+        if isinstance(queue, str):
+            queue = [queue]
+        if not isinstance(queue, Iterable):
+            raise ValueError("queue parameter must be a string or iterable")
+        constraint["and"].append({"queue": list(queue)})
+    if states and results:
+        tmp = {"or": []}
+        tmp["or"].append({"states": [states]})
+        tmp["or"].append({"results": [results]})
+        constraint["and"].append(tmp)
+    elif states:
+        constraint["and"].append({"states": [states]})
+    elif results:
+        constraint["and"].append({"results": [results]})
+    payload = {
+        "max_entries": int(max_entries),
+        "attrs": attrs,
+        "since": since,
+        "constraint": constraint,
+    }
+    return JobListRPC(flux_handle, "job-list.list", payload)
+
+
+def job_list_inactive(
+    flux_handle,
+    since=0.0,
+    max_entries=1000,
+    attrs=["all"],
+    name=None,
+    queue=None,
+    constraint=None,
+):
+    """Same as ``flux.job.list.job_list``, but lists only inactive jobs."""
+    return job_list(
+        flux_handle,
+        max_entries=max_entries,
+        attrs=attrs,
+        userid=flux.constants.FLUX_USERID_UNKNOWN,
+        states=flux.constants.FLUX_JOB_STATE_INACTIVE,
+        since=since,
+        name=name,
+        queue=queue,
+        constraint=constraint,
+    )
+
+
+class JobListIdRPC(RPC):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.jobid = None
+
+    def get_job(self):
+        return self.get()["job"]
+
+    def get_jobinfo(self):
+        """Returns a ``JobInfo`` object for the job.
+
+        :rtype: JobInfo
+        """
+        return JobInfo(self.get_job())
+
+
+# list-id is not like list or list-inactive, it doesn't return an
+# array, so don't use JobListRPC
+def job_list_id(flux_handle, jobid, attrs=["all"]):
+    """Query job information for a single ``jobid``.
+
+    Sends an RPC to the job-list module to query information about the provided jobid.
+    Use the ``get_job()`` or ``get_jobinfo()`` method on the returned ``JobListIdRPC`` to
+    obtain the job data as a dict or a JobInfo object.
+
+    :rtype: JobListIdRPC
+    """
+    payload = {"id": int(jobid), "attrs": attrs}
+    rpc = JobListIdRPC(flux_handle, "job-list.list-id", payload)
+    #  save original JobId argument for error reporting
+    rpc.jobid = jobid
+    return rpc
+
+
+# get_job is the single variant of job_list_id, and returns the
+# expected data structure (dict) to describe one job (or None)
+def get_job(flux_handle, jobid):
+    """Get job information dictionary based on a jobid
+
+    This is a courtesy, blocking function for users looking for
+    details about a job after submission. The dictionary includes
+    the job identifier, userid that submit it, urgency, priority,
+    t_submit, t_depend, (and others when finished), state, name,
+    ntasks, ncores, duration, nnodes, result, runtime, returncode,
+    waitstatus, nodelist, and exception type, severity, and note.
+    """
+    payload = {"id": int(jobid), "attrs": ["all"]}
+    rpc = JobListIdRPC(flux_handle, "job-list.list-id", payload)
+    try:
+        jobinfo = rpc.get_jobinfo()
+
+    # The job does not exist!
+    except FileNotFoundError:
+        return None
+
+    return jobinfo.to_dict(filtered=False)
+
+
+class JobListIdsFuture(WaitAllFuture):
+    """Simulate interface of JobListRPC for listing multiple jobids"""
+
+    def __init__(self):
+        super(JobListIdsFuture, self).__init__()
+        self.errors = []
+
+    def get_jobs(self):
+        """get all successful results, appending errors into self.errors"""
+        jobs = []
+        #  Wait for all jobid RPCs to complete
+        self.wait_for()
+
+        #  Get all successful jobs, accumulate errors in self.errors
+        for child in self.children:
+            try:
+                jobs.append(child.get_job())
+            except EnvironmentError as err:
+                if err.errno == errno.ENOENT:
+                    msg = f"JobID {child.jobid.orig} unknown"
+                else:
+                    msg = f"rpc: {err.strerror}"
+                self.errors.append(msg)
+        return jobs
+
+    def get_jobinfos(self):
+        """get all successful results as list of JobInfo objects
+
+        Any errors are appended to self.errors.
+        """
+        return [JobInfo(job) for job in self.get_jobs()]
+
+
+class JobList:
+    """User friendly class for querying lists of jobs from Flux
+
+    By default a JobList will query the last ``max_entries`` jobs for all
+    users. Other filter parameters can be passed to the constructor or
+    the ``set_user()`` and ``add_filter()`` methods.
+
+    :flux_handle: A Flux handle obtained from flux.Flux()
+    :attrs: Optional list of job attributes to fetch. (default is all attrs)
+    :filters: List of strings defining the results or states to filter. E.g.,
+              [ "pending", "running" ].
+    :ids: List of jobids to return. Other filters are ignored if ``ids`` is
+          not empty.
+    :user: Username or userid for which to fetch jobs. Default is current user.
+    :max_entries: Maximum number of jobs to return
+    :since: Limit jobs to those that have been active since a given timestamp.
+    :name: Limit jobs to those with a specific name.
+    :queue: Limit jobs to those submitted to a specific queue or queues
+    :constraint: An RFC 31 Constraint object describing a job-list constraint
+        as documented in RFC 43 Constraint Operators section. This constraint
+        may then be joined with other constraints provided by above parameters
+        via the ``and`` operator.
+    """
+
+    # pylint: disable=too-many-instance-attributes
+
+    STATES = {
+        "depend": flux.constants.FLUX_JOB_STATE_DEPEND,
+        "priority": flux.constants.FLUX_JOB_STATE_PRIORITY,
+        "sched": flux.constants.FLUX_JOB_STATE_SCHED,
+        "run": flux.constants.FLUX_JOB_STATE_RUN,
+        "cleanup": flux.constants.FLUX_JOB_STATE_CLEANUP,
+        "inactive": flux.constants.FLUX_JOB_STATE_INACTIVE,
+        "pending": flux.constants.FLUX_JOB_STATE_PENDING,
+        "running": flux.constants.FLUX_JOB_STATE_RUNNING,
+        "active": flux.constants.FLUX_JOB_STATE_ACTIVE,
+    }
+    RESULTS = {
+        "completed": flux.constants.FLUX_JOB_RESULT_COMPLETED,
+        "failed": flux.constants.FLUX_JOB_RESULT_FAILED,
+        "canceled": flux.constants.FLUX_JOB_RESULT_CANCELED,
+        "timeout": flux.constants.FLUX_JOB_RESULT_TIMEOUT,
+    }
+
+    def __init__(
+        self,
+        flux_handle,
+        attrs=["all"],
+        filters=[],
+        ids=[],
+        user=None,
+        max_entries=1000,
+        since=0.0,
+        name=None,
+        queue=None,
+        constraint=None,
+    ):
+        self.handle = flux_handle
+        self.attrs = list(attrs)
+        self.states = 0
+        self.results = 0
+        self.max_entries = max_entries
+        self.since = since
+        self.name = name
+        self.queue = queue
+        self.ids = list(map(JobID, ids)) if ids else None
+        self.errors = []
+        for fname in filters:
+            for x in fname.split(","):
+                self.add_filter(x)
+        self.set_user(user)
+        self.constraint = constraint
+
+    def set_user(self, user):
+        """Only return jobs for user (may be a username, userid, or "all")"""
+        if user is None:
+            self.userid = os.getuid()
+        elif user == "all":
+            self.userid = flux.constants.FLUX_USERID_UNKNOWN
+        else:
+            try:
+                self.userid = pwd.getpwnam(user).pw_uid
+            except KeyError:
+                try:
+                    self.userid = int(user)
+                except ValueError:
+                    raise ValueError(f"Invalid user {user} specified")
+
+    def add_filter(self, fname):
+        """Append a state or result filter to JobList query"""
+        fname = fname.lower()
+        if fname in self.STATES:
+            self.states |= self.STATES[fname]
+        elif fname in self.RESULTS:
+            self.results |= self.RESULTS[fname]
+        else:
+            raise ValueError(f"Invalid filter specified: {fname}")
+
+    def fetch_jobs(self):
+        """Initiate the JobList query to the Flux job-info module
+
+        JobList.fetch_jobs() returns a JobListRPC or JobListIdsFuture,
+        either of which will be fulfilled when the job data is available.
+
+        Once the Future has been fulfilled, a list of JobInfo objects
+        can be obtained via JobList.jobs(). If JobList.errors is non-empty,
+        then it will contain a list of errors returned via the query.
+        """
+        if self.ids:
+            listids = JobListIdsFuture()
+            for jobid in self.ids:
+                listids.push(job_list_id(self.handle, jobid, self.attrs))
+            return listids
+        return job_list(
+            self.handle,
+            max_entries=self.max_entries,
+            attrs=self.attrs,
+            userid=self.userid,
+            states=self.states,
+            results=self.results,
+            since=self.since,
+            name=self.name,
+            queue=self.queue,
+            constraint=self.constraint,
+        )
+
+    def jobs(self):
+        """Synchronously fetch a list of JobInfo objects from JobList query
+
+        If the Future object returned by JobList.fetch_jobs has not yet been
+        fulfilled (e.g. is_ready() returns False), then this call may block.
+        Otherwise, returns a list of JobInfo objects for all jobs returned
+        from the underlying job listing RPC.
+        """
+        rpc = self.fetch_jobs()
+        jobs = rpc.get_jobs()
+        if hasattr(rpc, "errors"):
+            self.errors = rpc.errors
+        return [JobInfo(job) for job in jobs]
diff --git a/src/bindings/python/flux/job/output.py b/src/bindings/python/flux/job/output.py
new file mode 100644
index 000000000000..2781f1a559d6
--- /dev/null
+++ b/src/bindings/python/flux/job/output.py
@@ -0,0 +1,984 @@
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import base64
+import errno
+from typing import NamedTuple
+
+from flux.future import FutureExt
+from flux.idset import IDset
+from flux.job import (
+    EventLogEvent,
+    JobException,
+    JobID,
+    event_wait,
+    event_watch,
+    event_watch_async,
+    job_kvs_lookup,
+)
+
+# Log levels from shell.h for use with log_stderr_level:
+LOG_QUIET = -1
+LOG_FATAL = 0
+# Note: LOG_ALERT=1, LOG_CRIT=2 are currently reserved in shell.h
+LOG_ERROR = 3
+LOG_WARN = 4
+LOG_NOTICE = 5
+LOG_DEBUG = 6
+LOG_TRACE = 7
+
+
+class Taskset:
+    """
+    A Taskset represents a specific set of task ranks or "all" tasks.
+    Implements a minimal interface of the IDset class, mainly for
+    testing if one Taskset intersects with another.
+
+    The underlying IDset is only initialized for Taskset objects that
+    do represent all tasks.
+
+    Attributes:
+        all (bool): a Taskset representing all tasks
+        ids (IDset): If self.all is False, the underlying idset
+    """
+
+    def __init__(self, tasks):
+        self.all = False
+        if isinstance(tasks, Taskset):
+            # Copy an existing Taskset
+            self.all = tasks.all
+            self.ids = tasks.ids.copy()
+            return
+        if isinstance(tasks, IDset):
+            self.ids = tasks.copy()
+            return
+        self.ids = None
+        if tasks in ("all", "*"):
+            self.all = True
+            return
+        self.ids = IDset(tasks)
+
+    def intersect(self, tasks):
+        if not isinstance(tasks, Taskset):
+            tasks = Taskset(tasks)
+        if self.all:
+            return Taskset(tasks)
+        elif tasks.all:
+            return Taskset(self)
+        return Taskset(self.ids.intersect(tasks.ids))
+
+    def __bool__(self):
+        return self.all or len(self.ids) > 0
+
+    def __str__(self):
+        if self.all:
+            return "all"
+        return str(self.ids)
+
+
+class OutputEvent(EventLogEvent):
+    """
+    Object representing RFC 24 Job Standard I/O data events
+    Attributes:
+        timestamp (float): timestamp for this event
+        name (str): Name of this event: 'data'
+        rank (Taskset): Set of ranks to which this event applies
+        stream (str): name of output stream ("stdout", "stderr")
+        eof (bool): True if this event marks EOF for stream
+        data (str): output data
+        dict (dict): original event as dict
+    """
+
+    def __init__(self, entry, labelio=False):
+        super().__init__(entry)
+        if self.name != "data":
+            raise ValueError(f"event {self.name} is not a data event")
+
+        self.rank = Taskset(self.context["rank"])
+        self.stream = self.context["stream"]
+        self.data = None
+        self.eof = False
+
+        if "eof" in self.context:
+            self.eof = self.context["eof"]
+
+        if "data" in self.context:
+            data = self.context["data"]
+            if "encoding" in self.context:
+                if self.context["encoding"] == "base64":
+                    data = base64.b64decode(data).decode(
+                        "utf-8", errors="surrogateescape"
+                    )
+            if "repeat" in self.context:
+                data *= self.context["repeat"]
+            if labelio:
+                data = [f"{self.rank}: {x}" for x in data.splitlines()]
+                data = "\n".join(data) + "\n"
+            self.data = data
+
+    def render(self):
+        return self.data
+
+
+class LogEvent(EventLogEvent):
+    """
+    Object representing a RFC 24 Job "log" event
+    Attributes:
+        timestamp (float): timestamp for this log event
+        name (str): name of this event: 'log'
+        rank (Taskset): shell rank that produced the log message
+        level (int): log level
+        levelstr (str): log level string
+        message (str): log message
+        component (str): log component if available
+        file (str): source file if available
+        line (int); source line if available
+        dict (dict): original event as dict
+    """
+
+    log_level_string = [
+        "FATAL",
+        "FATAL",
+        "FATAL",
+        "ERROR",
+        " WARN",
+        "",
+        "DEBUG",
+        "TRACE",
+    ]
+
+    @property
+    def levelstr(self):
+        return self.log_level_string[self.level]
+
+    def __init__(self, entry):
+        super().__init__(entry)
+        # LogEvent class is only for 'log' events
+        if self.name != "log":
+            raise ValueError(f"event {self.name} is not a log event")
+        self.level = self.context["level"]
+        self.rank = Taskset(self.context["rank"])
+        self.message = self.context["message"]
+        self.component = self.context.get("component", None)
+        self.file = self.context.get("file", None)
+        self.line = self.context.get("line", -1)
+
+    def render(self, include_file_and_line=False):
+        prefix = f"flux-shell[{self.rank}]: "
+        if self.levelstr:
+            prefix += f"{self.levelstr}: "
+        if self.component:
+            prefix += f"{self.component}: "
+        if include_file_and_line and self.file:
+            prefix += f"{self.file}:{self.line}: "
+        return prefix + self.message
+
+
+class OutputHeaderEvent(EventLogEvent):
+    """
+    Object representing an RFC 24 header event
+    Attributes:
+        timestamp (float): timestamp for this event
+        name (str): Name of this event: 'header'
+        version (int): RFC 24 version
+        stdout_count (int): expected number of stdout streams
+        stderr_count (int): expected number of stderr streams
+        stdout_encoding (str): default encoding for stdout
+        stderr_encoding (str): default encoding for stderr
+        event (dict): original event as dict
+    """
+
+    def __init__(self, entry):
+        super().__init__(entry)
+        if self.name != "header":
+            raise ValueError(f"event {self.name} is not a header event")
+        self.version = self.context["version"]
+        self.stderr_count = self.context["count"]["stderr"]
+        self.stdout_count = self.context["count"]["stdout"]
+        self.stdout_encoding = self.context["encoding"]["stdout"]
+        self.stderr_encoding = self.context["encoding"]["stderr"]
+
+
+class RedirectEvent(EventLogEvent):
+    """
+    Object representing an RFC 24 redirect event
+    Attributes:
+        timestamp (float): timestamp for this event
+        name (str): Name of this event: 'redirect'
+        stream (str): the name of the stream being redirected
+        rank (Taskset): the set of ranks being redirected
+        path (str): redirect path
+        dict (dict): original event as dict
+    """
+
+    def __init__(self, entry):
+        super().__init__(entry)
+        if self.name != "redirect":
+            raise ValueError("event {self.name} is not a redirect event")
+        self.stream = self.context["stream"]
+        self.rank = Taskset(self.context["rank"])
+        self.path = self.context["path"]
+
+    def render(self):
+        return f"{self.rank}: {self.stream} redirected to {self.path}"
+
+
+class JobExceptionEvent(EventLogEvent):
+    """
+    Object representing a JobException event
+    Attributes:
+        timestamp (float): timestamp for this event
+        name (str): name of this event: 'exception'
+        severity (int): exception severity
+        exc_type (str): exception type
+        note (str): exception note
+    """
+
+    def __init__(self, entry):
+        super().__init__(entry)
+        if self.name != "exception":
+            raise ValueError("event {self.name} is not an exception event")
+        self.severity = self.context["severity"]
+        self.exc_type = self.context["type"]
+        self.note = self.context["note"]
+
+    def render(self):
+        result = f"job.exception: type={self.exc_type} severity={self.severity}"
+        if self.note:
+            result += f" {self.note}"
+        return result
+
+
+class JobOutput(NamedTuple):
+    """
+    Tuple containing job output result
+    Attributes:
+        stdout (str): stdout from all tasks
+        stderr (str): stderr from all tasks
+        log (str): log messages
+    """
+
+    stdout: str
+    stderr: str
+    log: str
+
+
+def _parse_output_eventlog_entry(entry, labelio=False):
+    """
+    Parse a single output eventlog entry, returning an object of the
+    appropriate type: OutputEvent, LogEvent, OutputHeaderEvent,
+    RedirectEvent or JobExceptionEvent.
+    """
+    if entry is None:
+        return None
+    if isinstance(entry, EventLogEvent):
+        event = entry
+    else:
+        event = EventLogEvent(entry)
+    name = event.name
+    if name == "data":
+        return OutputEvent(event, labelio)
+    elif name == "log":
+        return LogEvent(event)
+    elif name == "header":
+        return OutputHeaderEvent(event)
+    elif name == "redirect":
+        return RedirectEvent(event)
+    elif name == "exception":
+        # Not technically an output event, but handle anyway
+        return JobExceptionEvent(event)
+    raise ValueError(f"event {name} is not a supported output event type")
+
+
+def _output_eventlog_entry_decode(
+    entry, stream_dict, tasks, labelio=False, log_stderr_level=LOG_TRACE
+):
+    """
+    Decode RFC 24 output eventlog entry ``entry``, appending the result
+    to the stream_dict[stream_name] list if the entry is in the set of
+    requested ``tasks``. If stream_dict[stream_name] does not exist,
+    the entry will be created.
+    """
+    if not isinstance(tasks, Taskset):
+        raise ValueError("tasks argument must be a Taskset, got " + type(tasks))
+    event = _parse_output_eventlog_entry(entry, labelio)
+
+    #  Determine stream name of this event:
+    stream = None
+    if event.name == "data":
+        if event.rank.intersect(tasks):
+            stream = event.stream
+    elif event.name == "log":
+        if event.level <= log_stderr_level:
+            stream = "stderr"
+        else:
+            stream = "log"
+    elif event.name in "exception":
+        if event.rank.intersect(tasks):
+            stream = "stderr"
+    elif event.name == "exception":
+        stream = "stderr"
+    else:
+        return
+
+    text = event.render()
+    if text:
+        stream_dict.setdefault(stream, []).append(text)
+
+
+def _parse_output_eventlog(
+    eventlog, tasks="*", labelio=False, log_stderr_level=LOG_TRACE
+):
+    """
+    Given an eventlog, return a JobOutput tuple with stdout, stderr,
+    and log streams broken out.
+    """
+    stream_dict = {"stdout": [], "stderr": [], "log": []}
+    tasks = Taskset(tasks)
+
+    for line in eventlog.splitlines():
+        _output_eventlog_entry_decode(
+            line, stream_dict, tasks, labelio, log_stderr_level
+        )
+
+    # Join lines and return result
+    results = ["".join(stream_dict.get(k)) for k in ("stdout", "stderr", "log")]
+    return JobOutput(*results)
+
+
+def job_output(
+    flux_handle,
+    jobid,
+    tasks="*",
+    labelio=False,
+    nowait=False,
+    log_stderr_level=LOG_TRACE,
+):
+    """
+    Synchronously fetch output for a job.
+
+    If ``labelio`` is True, then each line of output will be labeled with
+    the source task rank.
+
+    If ``nowait`` is True, then currently available output is returned
+    without waiting for the output eventlog to be complete (i.e. for the
+    job to finish). In this case, a ``FileNotFound`` exception is raised
+    if the job output evenlog does not exist (job has not yet started
+    writing output), or the jobid is not valid.
+
+    If ``nowait`` is False (the default), then this function will block
+    until the job output is complete, i.e. the output eventlog has received
+    EOF on all streams. In this case a ``FileNotFound`` exception will be
+    raised only if the jobid is not valid.
+
+    Log messages at or below ``log_stderr_level`` will be sent to
+    stderr. All other messages will appear in the ``log`` stream. By
+    default, ``log_stderr_level=LOG_TRACE``, so all messages are sent to
+    ``stderr``. To separate all log messages in the ``log`` stream, set
+    ``log_stderr_level=-1`` (``LOG_QUIET``).
+
+    Args:
+        flux_handle (Flux): Flux handle
+        jobid (int, JobID, str): target jobid
+        tasks (str): idset of task ranks to include in output (default=all)
+        labelio (bool): prefix lines of output with source task rank
+        log_stderr_level (int): combine log messages at or below level with
+            stderr (default=LOG_TRACE)
+    Returns:
+        JobOutput: JobOutput tuple containing output for ``stdout``,
+            ``stderr``, and ``log`` streams.
+    Raises:
+        :py:exc:`FileNotFoundError`: jobid does not exist or no output found
+        :py:exc:`flux.job.JobException`: Job received an exception
+    """
+    jobid = JobID(jobid)
+    if nowait:
+        # Do not wait, fetch current output eventlog if available and
+        # return JobOutput object
+        eventlog = job_kvs_lookup(flux_handle, jobid, keys=["guest.output"])
+        if eventlog is None:
+            msg = f"job {jobid} does not exist or output not ready"
+            raise FileNotFoundError(msg)
+        return _parse_output_eventlog(
+            eventlog["guest.output"], tasks, labelio, log_stderr_level
+        )
+
+    stream_dict = {"stdout": [], "stderr": [], "log": []}
+    tasks = Taskset(tasks)
+
+    #  Now block until output eventlog is ready:
+    try:
+        event_wait(flux_handle, jobid, "shell.init", "guest.exec.eventlog")
+    except OSError:
+        #  event_wait() could fail due to a job exception before the 'start'
+        #  event. Check for a fatal job exception in the job eventlog
+        #  and raise it as such if so. O/w, reraise the event_wait() exception.
+        #
+        event = event_wait(flux_handle, jobid, "exception")
+        if event is not None and event.context["severity"] == 0:
+            raise JobException(event) from None
+        raise
+
+    #  Output eventlog is ready, synchronously gather all output
+    for event in event_watch(flux_handle, jobid, "guest.output"):
+        _output_eventlog_entry_decode(
+            event, stream_dict, tasks, labelio, log_stderr_level
+        )
+
+    #  Join lines and return JobOutput result
+    results = ["".join(stream_dict.get(k)) for k in ("stdout", "stderr", "log")]
+    return JobOutput(*results)
+
+
+class JobOutputEventWatch(FutureExt):
+    """
+    A class for watching job output events.
+
+    See output_watch_events_async() for full documentation.
+    """
+
+    def __init__(self, flux_handle, jobid, labelio=False, nowait=False):
+        self.labelio = labelio
+        self.nowait = nowait
+
+        #  Capture when 'start' and 'finish' events have been posted to
+        #  the main eventlog:
+        self.started = False
+        self.finished = False
+
+        #  Capture when the end of the output eventlog has been reached
+        #  (indicated by None returned from watching the eventlog)
+        self.closed = False
+        super().__init__(self._watch_init, JobID(jobid), flux_handle=flux_handle)
+
+    def get_event(self, autoreset=True):
+        event = self.get()
+        if event is None:
+            return None
+        if autoreset:
+            self.reset()
+        return _parse_output_eventlog_entry(event, labelio=self.labelio)
+
+    def _watch_output(self, future):
+        #  Watch output events and propagate to output_watch_future.
+        #
+        #  Note: handle and propagate OSError from future.get_event() for
+        #  reasons noted in wait_for_start_event() below.
+        #
+        event = None
+        try:
+            event = future.get_event(autoreset=False)
+            if event is None:
+                self.closed = True
+                #  Fulfill this future with None (indicating no more data)
+                #  if the 'finish' event is also posted in the main eventlog.
+                #  This is done to ensure any exception has been captured.
+                #
+                #  If nowait is enabled, then the main eventlog may not be
+                #  monitored, so assume job is finished. This may miss logging
+                #  job exceptions in some cases, but it is assumed that users
+                #  of the `nowait` option are using it because the main
+                #  eventlog is already being monitored.
+                #
+                if self.nowait or self.finished:
+                    self.fulfill()
+                return
+            self.fulfill(event)
+        except OSError as exc:
+            self.fulfill_error(exc.errno, exc.strerror)
+
+        if event is not None:
+            future.reset()
+
+    def _wait_for_shell_init(self, future, jobid):
+        #  Wait for the 'shell.init' event, then the output eventlog
+        #  should be available.
+        #
+        #  Note: handle and propagate OSError from future.get_event() for
+        #  reasons noted in wait_for_start_event() below.
+        #
+        event = None
+        try:
+            event = future.get_event()
+        except OSError as exc:
+            self.fulfill_error(exc.errno, exc.strerror)
+
+        if event is not None and event.name == "shell.init":
+            event_watch_async(
+                future.get_flux(), int(jobid), eventlog="guest.output"
+            ).then(self._watch_output)
+            future.cancel(stop=True)
+
+    def _wait_for_start_event(self, future, jobid):
+        #  Wait for 'start' event or another error or job exception.
+        #
+        #  Note: future.get_event() raises OSError if the future was
+        #  fulfilled with an error. An exception raised within a callback
+        #  will result in flux_reactor_stop_error(3) being called, which
+        #  is not the correct behavior in an async context. Instead,
+        #  the error should be propagated to self so that the caller can
+        #  handle it if desired.
+        #
+        try:
+            event = future.get_event()
+        except OSError as exc:
+            if exc.errno == errno.ENOENT:
+                # Translate strerror to more helpful string:
+                exc.strerror = f"job {jobid} not found"
+            self.fulfill_error(exc.errno, exc.strerror)
+            return
+        if event is None:
+            #  If the eventlog ended without a start event, then the output
+            #  eventlog can't exist, so fulfill the future with an error
+            #  indicating the job never started.
+            #
+            #  Note: we can't use ENODATA here, since that is an expected
+            #  end of data error and is ignored. Also ENOENT could be
+            #  confused with the FileNotFoundError returned for invalid
+            #  jobid. Therefore, use EIO, which is at least somewhat related
+            #  to missing output file. More important is the message that
+            #  the job never started anyway.
+            #
+            #  Note we also don't want to raise a JobException since an
+            #  unhandled exception will terminate the reactor.
+            #
+            if not self.started:
+                self.fulfill_error(
+                    errnum=errno.EIO, errstr=f"job {jobid} never started"
+                )
+            return
+        if event.name == "exception":
+            #  Emit a JobExceptionEvent in the output since expectation is
+            #  than an exception message will appear on stderr:
+            #
+            self.fulfill(event)
+        elif event.name == "start":
+            #  Note that we've seen a 'start' event and proceed to watch the
+            #  exec eventlog for the 'shell.init' event:
+            #
+            self.started = True
+            event_watch_async(
+                future.get_flux(), int(jobid), eventlog="guest.exec.eventlog"
+            ).then(self._wait_for_shell_init, jobid)
+        elif event.name == "finish":
+            self.finished = True
+
+            #  Exceptions after 'finish' are not reported, so stop watching
+            #  the main eventlog:
+            future.cancel(stop=True)
+
+            #  Fulfill this future with None (indicating no more data)
+            #  if output has closed. This is done to ensure any all output
+            #  and any exceptions before the finish event have been captured.
+            #
+            if self.closed:
+                self.fulfill()
+
+    def _watch_init(self, future, jobid):
+        if self.nowait:
+            #  If nowait == True, go straight to watching output and
+            #  skip intermediate eventlog watches:
+            event_watch_async(
+                future.get_flux(), int(jobid), eventlog="guest.output"
+            ).then(self._watch_output)
+        else:
+            #  Initialize output watcher, start with main eventlog, which is
+            #  monitored for the 'start' event and also any job exceptions.
+            #
+            event_watch_async(future.get_flux(), int(jobid)).then(
+                self._wait_for_start_event, jobid
+            )
+
+
+class JobOutputWatch(FutureExt):
+    """
+    A class for watching job output.
+
+    See output_watch_async() for full documentation.
+    """
+
+    def __init__(
+        self,
+        flux_handle,
+        jobid,
+        labelio=False,
+        log_stderr_level=LOG_TRACE,
+        nowait=False,
+    ):
+        self.labelio = labelio
+        self.log_stderr_level = log_stderr_level
+        self.nowait = nowait
+        super().__init__(self._watch_init, JobID(jobid), flux_handle=flux_handle)
+
+    def _watch_init(self, future, jobid):
+        #  Start watching job output events. The `output_event_watch`
+        #  callback will then process these events and fulfill this Future
+        #  as lines of stdout, stderr, or log output arrive.
+        #
+        flux_handle = future.get_flux()
+        JobOutputEventWatch(
+            flux_handle, jobid, labelio=self.labelio, nowait=self.nowait
+        ).then(self._output_event_watch)
+
+    def _fulfill_output(self, event):
+        if event is None:
+            self.fulfill((None, None))
+            return
+        if event.name == "data" and event.data is not None:
+            self.fulfill((event.stream, event.render()))
+        elif event.name in ("log", "redirect", "exception"):
+            stream = "stderr"
+            if event.name == "log" and event.level > self.log_stderr_level:
+                stream = "log"
+            self.fulfill((stream, event.render() + "\n"))
+
+    def _output_event_watch(self, future):
+        #   Output event watch callback.
+        #
+        #   Note: Use a try/except block here to handle the case where
+        #   the underlying output_event_watch_async() future is fulfilled
+        #   with an error (future.get_event() raises an OSError in this case)
+        #   If an OSError is raised, then error is simply propagated to the
+        #   output_watch_future (which was returned to the user)
+        #
+        try:
+            #  Get event and only propagate events that will result in one
+            #  or more lines of output. "redirect" and "exception" events
+            #  result in an informational message to stderr (see getline())
+            #  so they are propagated as well:
+            #
+            event = future.get_event()
+            if (
+                event is None
+                or (event.name == "data" and event.data is not None)
+                or (event.name == "log")
+                or (event.name == "redirect")
+                or (event.name == "exception")
+            ):
+                self._fulfill_output(event)
+        except OSError as exc:
+            self.fulfill_error(exc.errno, exc.strerror)
+
+    def get_output(self):
+        """
+        Return a tuple of (stream, data) containing the next chunk of
+        output from a JobOutputWatch Future. Possible values for stream
+        include: "stdout", "stderr", or "log."
+
+        When no more output is available, (None, None) will be returned.
+        """
+        result = self.get()
+        self.reset()
+        return result
+
+
+class JobOutputWatchLines(FutureExt):
+    """
+    A class for watching lines of job output.
+
+    See output_watch_lines_async() for full documentation.
+    """
+
+    def __init__(
+        self,
+        flux_handle,
+        jobid,
+        labelio=False,
+        log_stderr_level=LOG_TRACE,
+        nowait=False,
+        keepends=False,
+    ):
+        self.labelio = labelio
+        self.log_stderr_level = log_stderr_level
+        self.nowait = nowait
+        self.keepends = keepends
+        super().__init__(self._watch_lines_init, JobID(jobid), flux_handle=flux_handle)
+
+    def _watch_lines_init(self, future, jobid):
+        #  Start watching job output events. The `output_event_watch`
+        #  callback will then process these events and fulfill this Future
+        #  as lines of stdout, stderr, or log output arrive.
+        #
+        flux_handle = future.get_flux()
+        JobOutputWatch(
+            flux_handle,
+            jobid,
+            labelio=self.labelio,
+            log_stderr_level=self.log_stderr_level,
+            nowait=self.nowait,
+        ).then(self._output_watch_cb)
+
+    def _output_watch_cb(self, future):
+        #  Split data into multiple lines and fulfill future once per line
+        try:
+            stream, lines = future.get_output()
+        except OSError as exc:
+            self.fulfill_error(exc.errno, exc.strerror)
+            return
+        if lines is not None:
+            for line in lines.splitlines(keepends=self.keepends):
+                self.fulfill((stream, line))
+        else:
+            self.fulfill((None, None))
+
+    def getline(self):
+        """
+        Return a tuple of (stream, line) for the next line of available
+        output from a JobOutputWatch Future. Possible values for stream
+        include: "stdout", "stderr", or "log."
+
+        When no more output is available, (None, None) will be returned.
+        """
+        result = self.get()
+        self.reset()
+        return result
+
+
+def output_event_watch_async(flux_handle, jobid, labelio=False, nowait=False):
+    """
+    Asynchronously get output event updates for a job.
+
+    Returns a JobOutputEventWatch Future. Call .get_event() from the callback
+    to get the next available output event from the Future. The event will
+    be of type OutputEvent, LogEvent, OutputHeaderEvent, RedirectEvent, or
+    JobExceptionEvent (check 'name' attribute or type to get type of event)
+
+    If the job has a fatal exception before the 'start' event, then the
+    future will be fulfilled with an error with errno=EIO and message
+    'job {jobid} never started'.
+
+    Args:
+        flux_handle (Flux): Flux handle
+        jobid (int, JobID, str): jobid to watch
+        labelio (bool): label lines of output with source tasks (default: False)
+        nowait (bool): Assume output eventlog already exists and skip watching
+            precursor eventlogs.
+    Returns:
+        JobOutputEventWatch: JobOutputEventWatch Future
+    """
+    return JobOutputEventWatch(flux_handle, jobid, labelio=labelio, nowait=nowait)
+
+
+def output_watch_async(
+    flux_handle,
+    jobid,
+    labelio=False,
+    log_stderr_level=LOG_TRACE,
+    nowait=False,
+):
+    """
+    Asynchronously get output data for a job.
+
+    This function returns a JobOutputWatch Future. Use future.get_output()
+    to get the available output, which returns a (stream, data) tuple,
+    where stream is one of 'stdout', 'stderr', or 'log'.
+
+    If the job receives an exception while watching output, an appropriate
+    error message is emitted to the stderr stream. Similarly, redirect
+    events generate a "redirected to" message on stderr.
+
+    If the job has a fatal exception before the 'start' event, then the
+    future will be fulfilled with an error with errno=EIO and message
+    'job {jobid} never started'.
+
+    Args:
+        flux_handle (Flux): Flux handle
+        jobid (JobID): jobid to watch
+        labelio (bool): label lines of output with source tasks (default=False)
+        log_stderr_level (int): emit log messages at this level or below to
+            stderr instead of the "log" stream.  (default=LOG_TRACE, i.e. all
+            log messages are copied to stderr)
+        nowait (bool): Assume output eventlog already exists and skip watching
+            precursor eventlogs.
+    Returns:
+        JobOutputWatch: JobOutputWatch Future
+    """
+    return JobOutputWatch(
+        flux_handle,
+        jobid,
+        labelio=labelio,
+        log_stderr_level=log_stderr_level,
+        nowait=nowait,
+    )
+
+
+def output_watch_lines_async(
+    flux_handle,
+    jobid,
+    labelio=False,
+    log_stderr_level=LOG_TRACE,
+    nowait=False,
+    keepends=False,
+):
+    """
+    Asynchronously get lines of output for a job.
+
+    This function returns a JobOutputWatchLines Future. Use future.getline()
+    to get the next line of output, which returns a (stream, line) tuple,
+    where stream is one of 'stdout', 'stderr', or 'log'.
+
+    If the job receives an exception while watching output, an appropriate
+    error message is emitted to the stderr stream. Similarly, redirect
+    events generate a "redirected to"  message to stderr.
+
+    If the job has a fatal exception before the 'start' event, then the
+    future will be fulfilled with an error with errno=EIO and message
+    'job {jobid} never started'.
+
+    Note:
+        The current implementation may return a partial line as a full line
+        from ``getline()`` if a partial line is written into a job output
+        eventlog entry. That is, this implementation does not currently line
+        buffer and wait for full lines to fulfill the JobOutputWatchLines
+        future. (This may be fixed in a future release, at which time this
+        note will be removed).
+
+    Args:
+        flux_handle (Flux): Flux handle
+        jobid (int, JobID, str): jobid to watch
+        labelio (bool): label lines of output with source task (default=False)
+        log_stderr_level (int): emit log messages at this level or below to
+            stderr instead of the "log" stream.  (default=LOG_TRACE, i.e. all
+            log messages are copied to stderr)
+        nowait (bool): Assume output eventlog already exists and skip watching
+            precursor eventlogs.
+        keepends (bool): If True, keep line breaks in the result.
+    Returns:
+        JobOutputWatchLines: JobOutputWatchLines Future
+    """
+    return JobOutputWatchLines(
+        flux_handle,
+        jobid,
+        labelio=labelio,
+        log_stderr_level=log_stderr_level,
+        nowait=nowait,
+        keepends=keepends,
+    )
+
+
+def output_event_watch(flux_handle, jobid, labelio=False, nowait=False):
+    """
+    Synchronously watch job output events via a generator.
+
+    This function will block until the first output event is available.
+
+    Example:
+        >>> for event in output_event_watch(flux_handle, jobid):
+        ...     if event.name == "data" and event.data is not None:
+        ...         print(f"{event.stream}: {event.data}"
+
+    Args:
+        flux_handle (Flux): Flux handle
+        jobid (int, JobID, str): jobid to watch
+        labelio (bool): label lines of output with source tasks (default: False)
+        nowait (bool): If True, assume output eventlog already exists and skip
+            watching precursor eventlogs.
+    """
+    jobid = JobID(jobid)
+    watcher = output_event_watch_async(
+        flux_handle, jobid, labelio=labelio, nowait=nowait
+    )
+    event = watcher.get_event()
+    while event is not None:
+        yield event
+        event = watcher.get_event()
+
+
+def output_watch(
+    flux_handle,
+    jobid,
+    labelio=False,
+    log_stderr_level=LOG_TRACE,
+    nowait=False,
+):
+    """
+    Synchronously fetch job output via a generator.
+
+    This function will block until job output is available.
+
+    Example:
+        >>> for stream, data in output_watch(flux_handle, jobid):
+        ...     print(f"{stream}: {data}")
+
+    Args:
+        flux_handle (flux.Flux): Flux handle
+        jobid (int, JobID, str): jobid to watch
+        labelio (bool): label lines of output with source tasks (default:
+            False)
+        log_stderr_level (int): return log messages at or below this level
+            to stderr instead of the "log" stream (default=LOG_TRACE, i.e.
+            all log messages are sent to stderr)
+        nowait (bool): If True, assume output eventlog already exists and skip
+            watching precursor eventlogs.
+    """
+    jobid = JobID(jobid)
+    watcher = output_watch_async(
+        flux_handle,
+        jobid,
+        labelio=labelio,
+        log_stderr_level=log_stderr_level,
+        nowait=nowait,
+    )
+    stream, data = watcher.get_output()
+    while data is not None:
+        yield stream, data
+        stream, data = watcher.get_output()
+
+
+def output_watch_lines(
+    flux_handle,
+    jobid,
+    labelio=False,
+    log_stderr_level=LOG_TRACE,
+    nowait=False,
+    keepends=False,
+):
+    """
+    Synchronously fetch job output lines via a generator.
+
+    This function will block until the first line of output is available.
+
+    Example:
+        >>> for stream, line in output_watch(flux_handle, jobid):
+        ...     print(f"{stream}: {line}")
+
+    Note:
+        The current implementation may return a partial line as a full
+        line if a partial line is written into a job output eventlog
+        entry. That is, this implementation does not currently line buffer
+        and wait for full lines before returning the next ``stream, line``
+        pair. This may be fixed in a future release, at which time this
+        note will be removed).
+
+    Args:
+        flux_handle (flux.flux): Flux handle
+        jobid (int, JobID, str): jobid to watch
+        labelio (bool): label lines of output with source tasks (default:
+            False)
+        log_stderr_level (int): return log messages at or below this level
+            to stderr instead of the "log" stream (default=LOG_TRACE, i.e.
+            all log messages are sent to stderr)
+        nowait (bool): If True, assume output eventlog already exists and skip
+            watching precursor eventlogs.
+        keepends (bool): If True, keep line breaks in the result.
+    """
+    jobid = JobID(jobid)
+    watcher = output_watch_lines_async(
+        flux_handle,
+        jobid,
+        labelio=labelio,
+        log_stderr_level=log_stderr_level,
+        nowait=nowait,
+        keepends=keepends,
+    )
+    stream, line = watcher.getline()
+    while line is not None:
+        yield stream, line
+        stream, line = watcher.getline()
diff --git a/src/bindings/python/flux/job/stats.py b/src/bindings/python/flux/job/stats.py
new file mode 100644
index 000000000000..5fd15ffe7e81
--- /dev/null
+++ b/src/bindings/python/flux/job/stats.py
@@ -0,0 +1,149 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import itertools
+
+from flux.rpc import RPC
+
+
+# pylint: disable=too-many-instance-attributes
+class JobStats:
+    """Container for job statistics as returned by job-list.job-stats
+
+
+    Attributes:
+        depend: Count of jobs current in DEPEND state
+        priority: Count of jobs in PRIORITY state
+        sched: Count of jobs in SCHED state
+        run: Count of jobs in RUN state
+        cleanup: Count of jobs in CLEANUP state
+        inactive: Count of INACTIVE jobs
+        successful: Total number of jobs completed with zero exit code
+        failed: Total number of jobs that did not exit with zero status
+        timeout: Total number of jobs that timed out
+        canceled: Total number of jobs that were canceled
+        pending: Sum of "depend", "priority", and "sched"
+        running: Sum of "run" and "cleanup"
+        active: Total number of active jobs (all states but INACTIVE)
+
+    """
+
+    states = (
+        "depend",
+        "priority",
+        "sched",
+        "run",
+        "cleanup",
+        "inactive",
+        "total",
+    )
+    stats = (
+        "successful",
+        "failed",
+        "timeout",
+        "canceled",
+        "inactive_purged",
+    )
+    derived_stats = (
+        "pending",
+        "running",
+        "active",
+    )
+
+    class QueueStats:
+        """Container for a set of per-queue stats"""
+
+        def __init__(self, stats=None):
+            if stats is None:
+                self.queue_name = ""
+                for stat in itertools.chain(JobStats.states, JobStats.stats):
+                    setattr(self, stat, 0)
+                return
+
+            self.queue_name = stats["name"] if "name" in stats else "all"
+            #  Move all stats to top-level attributes of this object
+            for state in JobStats.states:
+                setattr(self, state, stats["job_states"][state])
+            for stat in JobStats.stats:
+                setattr(self, stat, stats[stat])
+
+        def __iadd__(self, other):
+            self.queue_name += "," + other.queue_name
+            for stat in itertools.chain(JobStats.states, JobStats.stats):
+                setattr(self, stat, getattr(self, stat) + getattr(other, stat))
+            return self
+
+    def __init__(self, handle, queue=None):
+        """Initialize a JobStats object with Flux handle ``handle``"""
+        self.handle = handle
+        self.queues = []
+        # Accept queue as str or iterable
+        if queue is not None:
+            self.queues.extend([queue] if isinstance(queue, str) else queue)
+        self.callback = None
+        self.cb_kwargs = {}
+        for attr in itertools.chain(
+            JobStats.states, JobStats.stats, JobStats.derived_stats
+        ):
+            setattr(self, attr, -1)
+
+    def _update_cb(self, rpc):
+        resp = rpc.get()
+        queues = {x["name"]: self.QueueStats(x) for x in resp["queues"]}
+        if self.queues:
+            qstat = self.QueueStats()
+            for queue in self.queues:
+                try:
+                    qstat += queues[queue]
+                except KeyError:
+                    raise ValueError(f"no stats available for queue {queue}")
+        else:
+            qstat = self.QueueStats(resp)
+
+        for attr in itertools.chain(JobStats.states, JobStats.stats):
+            setattr(self, attr, getattr(qstat, attr))
+
+        #  Compute some stats for convenience:
+        #  pylint: disable=attribute-defined-outside-init
+        self.pending = self.depend + self.priority + self.sched
+        self.running = self.run + self.cleanup
+        self.active = self.total - self.inactive
+
+        # This class reports the total number of unsuccessful jobs in
+        # the 'failed' attribute, not just the count of jobs that ran
+        # to completion with nonzero exit code
+        self.failed += self.timeout
+        self.failed += self.canceled
+
+        if self.callback:
+            self.callback(self, **self.cb_kwargs)
+
+    def _query(self):
+        return RPC(self.handle, "job-list.job-stats", {})
+
+    def update(self, callback=None, **kwargs):
+        """Asynchronously fetch job statistics and update this object.
+
+        Requires that the reactor for this handle be running in order to
+        process the result.
+
+        Args:
+            callback: Optional: a callback to call when asynchronous
+             update is complete.
+            kwargs: Optional: extra keyword arguments to pass to callback()
+        """
+        self.callback = callback
+        self.cb_kwargs = kwargs
+        self._query().then(self._update_cb)
+
+    def update_sync(self):
+        """Synchronously update job statistics"""
+        self._update_cb(self._query())
+        return self
diff --git a/src/bindings/python/flux/job/submit.py b/src/bindings/python/flux/job/submit.py
new file mode 100644
index 000000000000..3a4075733c3c
--- /dev/null
+++ b/src/bindings/python/flux/job/submit.py
@@ -0,0 +1,137 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+import errno
+
+from _flux._core import ffi, lib
+from flux import constants
+from flux.future import Future
+from flux.job import JobID
+from flux.job._wrapper import _RAW as RAW
+from flux.job.Jobspec import _convert_jobspec_arg_to_string
+from flux.util import check_future_error
+
+
+class SubmitFuture(Future):
+    """Future subclass representing job IDs."""
+
+    def get_id(self):
+        """Return the job ID represented by this future."""
+        return submit_get_id(self)
+
+
+def submit_async(
+    flux_handle,
+    jobspec,
+    urgency=lib.FLUX_JOB_URGENCY_DEFAULT,
+    waitable=False,
+    debug=False,
+    pre_signed=False,
+    novalidate=False,
+):
+    """Ask Flux to run a job, without waiting for a response
+
+    Submit a job to Flux.  This method returns immediately with a
+    Flux Future, which can be used obtain the job ID later.
+
+    :param flux_handle: handle for Flux broker from flux.Flux()
+    :type flux_handle: Flux
+    :param jobspec: jobspec defining the job request
+    :type jobspec: Jobspec or its string encoding
+    :param urgency: job urgency 0 (lowest) through 31 (highest)
+        (default is 16).  Priorities 0 through 15 are restricted to
+        the instance owner.
+    :type urgency: int
+    :param waitable: allow result to be fetched with job.wait()
+        (default is False).  Waitable=True is restricted to the
+        instance owner.
+    :type waitable: bool
+    :param debug: enable job manager debugging events to job eventlog
+        (default is False)
+    :type debug: bool
+    :param pre_signed: jobspec argument is already signed
+        (default is False)
+    :type pre_signed: bool
+    :param novalidate: jobspec does not need to be validated.
+        (default is False) novalidate=True is restricted to the
+        instance owner.
+    :type novalidate: bool
+    :returns: a Flux Future object for obtaining the assigned jobid
+    :rtype: SubmitFuture
+    """
+    jobspec = _convert_jobspec_arg_to_string(jobspec)
+    flags = 0
+    if waitable:
+        flags |= constants.FLUX_JOB_WAITABLE
+    if debug:
+        flags |= constants.FLUX_JOB_DEBUG
+    if pre_signed:
+        flags |= constants.FLUX_JOB_PRE_SIGNED
+    if novalidate:
+        flags |= constants.FLUX_JOB_NOVALIDATE
+    future_handle = RAW.submit(flux_handle, jobspec, urgency, flags)
+    return SubmitFuture(future_handle)
+
+
+@check_future_error
+def submit_get_id(future):
+    """Get job ID from a Future returned by job.submit_async()
+
+    Process a response to a Flux job submit request.  This method blocks
+    until the response is received, then decodes the result to obtain
+    the assigned job ID.
+
+    :param future: a Flux future object returned by job.submit_async()
+    :type future: Future
+    :returns: job ID
+    :rtype: JobID
+    """
+    if future is None or future == ffi.NULL:
+        raise EnvironmentError(errno.EINVAL, "future must not be None/NULL")
+    future.wait_for()  # ensure the future is fulfilled
+    jobid = ffi.new("flux_jobid_t[1]")
+    RAW.submit_get_id(future, jobid)
+    return JobID(jobid[0])
+
+
+def submit(
+    flux_handle,
+    jobspec,
+    urgency=lib.FLUX_JOB_URGENCY_DEFAULT,
+    waitable=False,
+    debug=False,
+    pre_signed=False,
+):
+    """Submit a job to Flux
+
+    Ask Flux to run a job, blocking until a job ID is assigned.
+
+    :param flux_handle: handle for Flux broker from flux.Flux()
+    :type flux_handle: Flux
+    :param jobspec: jobspec defining the job request
+    :type jobspec: Jobspec or its string encoding
+    :param urgency: job urgency 0 (lowest) through 31 (highest)
+        (default is 16).  Priorities 0 through 15 are restricted to
+        the instance owner.
+    :type urgency: int
+    :param waitable: allow result to be fetched with job.wait()
+        (default is False).  Waitable=true is restricted to the
+        instance owner.
+    :type waitable: bool
+    :param debug: enable job manager debugging events to job eventlog
+        (default is False)
+    :type debug: bool
+    :param pre_signed: jobspec argument is already signed
+        (default is False)
+    :type pre_signed: bool
+    :returns: job ID
+    :rtype: int
+    """
+    future = submit_async(flux_handle, jobspec, urgency, waitable, debug, pre_signed)
+    return future.get_id()
diff --git a/src/bindings/python/flux/job/timeleft.py b/src/bindings/python/flux/job/timeleft.py
new file mode 100644
index 000000000000..0692673915c4
--- /dev/null
+++ b/src/bindings/python/flux/job/timeleft.py
@@ -0,0 +1,37 @@
+###############################################################
+# Copyright 2022 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import flux
+from _flux._core import ffi
+from flux.job._wrapper import _RAW as RAW
+
+
+def timeleft(flux_handle=None):
+    """
+    Return the remaining time in floating point seconds for the current
+    job or enclosing instance.
+
+    If the calling process is not associated with a job, then an exception
+    will be raised. If the job associated with the current process has no
+    timelimit, then ``float(inf)`` is returned.
+
+    If a Flux handle is not provided, then this function will open a
+    handle to the enclosing instance.
+    """
+    if flux_handle is None:
+        flux_handle = flux.Flux()
+
+    error = ffi.new("flux_error_t[1]")
+    result = ffi.new("double[1]")
+    try:
+        RAW.timeleft(flux_handle, error, result)
+    except OSError as err:
+        raise OSError(err.errno, ffi.string(error[0].text).decode("utf-8")) from err
+    return result[0]
diff --git a/src/bindings/python/flux/job/validator/__init__.py b/src/bindings/python/flux/job/validator/__init__.py
new file mode 100644
index 000000000000..c613e8fe1b5a
--- /dev/null
+++ b/src/bindings/python/flux/job/validator/__init__.py
@@ -0,0 +1,11 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+from flux.job.validator.validator import JobValidator, ValidatorPlugin
diff --git a/src/bindings/python/flux/job/validator/plugins/feasibility.py b/src/bindings/python/flux/job/validator/plugins/feasibility.py
new file mode 100644
index 000000000000..5fcff5d775b5
--- /dev/null
+++ b/src/bindings/python/flux/job/validator/plugins/feasibility.py
@@ -0,0 +1,39 @@
+##############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+"""Use feasibility service to validate job
+
+This plugin validates jobs via a feasibility.check RPC.
+
+This allows jobs which are making infeasible or otherwise invalid
+requests to be rejected by the scheduler before ingest, instead of
+when the scheduler attempts to allocate resources for them.
+
+ENOSYS errors from the RPC are ignored, in case there is no feasibility
+service currently loaded.
+"""
+
+import errno
+
+from flux.job.validator import ValidatorPlugin
+
+
+class Validator(ValidatorPlugin):
+    def __init__(self, parser):
+        self.service_name = "feasibility.check"
+
+    def validate(self, args):
+        try:
+            self.flux.rpc(self.service_name, args.jobinfo).get()
+        except OSError as err:
+            if err.errno == errno.ENOSYS:
+                #  Treat ENOSYS as success
+                return (0, None)
+            return (err.errno, err.strerror)
diff --git a/src/bindings/python/flux/job/validator/plugins/jobspec.py b/src/bindings/python/flux/job/validator/plugins/jobspec.py
new file mode 100644
index 000000000000..54e57fd8d19b
--- /dev/null
+++ b/src/bindings/python/flux/job/validator/plugins/jobspec.py
@@ -0,0 +1,51 @@
+##############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+"""Python bindings based jobspec validator
+
+Uses the ``flux.job.validate_jobspec`` Python function to validate
+submitted jobspec. If the optional ``--require-version`` option is
+supplied, then only that version of jobspec is permitted.
+"""
+
+import json
+
+from flux.job import validate_jobspec
+from flux.job.validator import ValidatorPlugin
+
+
+class Validator(ValidatorPlugin):
+    def __init__(self, parser):
+        self.require_version = 1
+        parser.add_argument(
+            "--require-version",
+            metavar="V",
+            default=1,
+            help="Require jobspec version V (or any)",
+        )
+
+    def configure(self, args):
+        try:
+            self.require_version = int(args.require_version)
+            if self.require_version < 1:
+                raise ValueError(
+                    f"Required jobspec version too low: {args.require_version} is < 1"
+                )
+            elif self.require_version > 1:
+                raise ValueError(
+                    f"Required jobspec version too high: {args.require_version} is > 1"
+                )
+        except ValueError:
+            if args.require_version != "any":
+                raise ValueError(f"Invalid argument to --require-version")
+            self.require_version = None
+
+    def validate(self, args):
+        validate_jobspec(json.dumps(args.jobspec), self.require_version)
diff --git a/src/bindings/python/flux/job/validator/plugins/require-instance.py b/src/bindings/python/flux/job/validator/plugins/require-instance.py
new file mode 100644
index 000000000000..1f56f2ec1239
--- /dev/null
+++ b/src/bindings/python/flux/job/validator/plugins/require-instance.py
@@ -0,0 +1,123 @@
+##############################################################
+# Copyright 2022 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+"""Require that all jobs are new instances of Flux
+
+This plugin validates that submitted jobs are new instances of Flux.
+That is, either the `batch` system attribute is set, or the first
+2 arguments of the command are `flux broker` or `flux start`.
+
+This is not a foolproof solution. The validator could reject jobs that
+are new instances of Flux if an instance is launched by a script and
+not directly via `flux broker` or `flux start`.
+"""
+
+import errno
+from os.path import basename
+
+from flux.job import JobspecV1
+from flux.job.validator import ValidatorPlugin
+
+
+class JobSize:
+
+    def __init__(self, nnodes=0, ncores=0):
+        self.nnodes = nnodes
+        self.ncores = ncores
+
+    @classmethod
+    def from_jobspec(cls, jobspec):
+        counts = JobspecV1(**jobspec).resource_counts()
+        return JobSize(counts.get("node", None), counts.get("core", None))
+
+    def ignore_by_size(self, jobspec):
+        """
+        Return True if jobspec size is less than the currently configured
+        minimum job size required for a require-instance check
+        """
+        size = JobSize.from_jobspec(jobspec)
+        if size.nnodes is not None and size.nnodes >= self.nnodes:
+            return False
+        if size.ncores is not None and size.ncores >= self.ncores:
+            return False
+        return True
+
+    def configured(self):
+        return self.nnodes > 0 or self.ncores > 0
+
+    @property
+    def errstr(self):
+        result = "Direct job submission disabled"
+        if self.nnodes == 0 and self.ncores == 0:
+            result += " for all jobs in this instance"
+        else:
+            result += " for jobs >="
+            if self.nnodes > 0:
+                result += f" {self.nnodes} nodes"
+            if self.ncores > 0:
+                if self.nnodes > 0:
+                    result += " or"
+                result += f" {self.ncores} cores"
+        result += ". Please use flux-batch(1) or flux-alloc(1)"
+        return result
+
+
+class Validator(ValidatorPlugin):
+    def __init__(self, parser):
+        parser.add_argument(
+            "--require-instance-minnodes",
+            help="minimum node count that requires flux-batch/alloc be used."
+            + "default: 0",
+            metavar="N",
+            type=int,
+            default=-1,
+        )
+        parser.add_argument(
+            "--require-instance-mincores",
+            metavar="N",
+            help="minimum core count that requires flux-batch/alloc be used."
+            + "default: 0",
+            type=int,
+            default=-1,
+        )
+        super().__init__(parser)
+
+    def configure(self, args):
+        base = "ingest.validator.require-instance"
+        nnodes = self.flux.conf_get(f"{base}.minnodes", 0)
+        ncores = self.flux.conf_get(f"{base}.mincores", 0)
+
+        # Allow command line to override config:
+        if args.require_instance_minnodes >= 0:
+            nnodes = args.require_instance_minnodes
+        if args.require_instance_mincores >= 0:
+            ncores = args.require_instance_mincores
+        if nnodes > 0 and ncores == 0:
+            # Default ncores to a multiple of nnodes to avoid leaving an
+            # accidental hole that allows a cores-only job to be admitted
+            # by this plugin:
+            ncores = 16 * nnodes
+
+        self.minsize = JobSize(nnodes, ncores)
+
+    def validate(self, args):
+
+        #  Ignore this job if it falls below a configured minsize:
+        if self.minsize.ignore_by_size(args.jobspec):
+            return
+
+        #  Otherwise, ensure this job is a new instance of Flux:
+        if "batch" in args.jobspec["attributes"]["system"]:
+            return
+        command = args.jobspec["tasks"][0]["command"]
+        arg0 = basename(command[0])
+        if arg0 == "flux" and command[1] in ["broker", "start"]:
+            return
+        return (errno.EINVAL, self.minsize.errstr)
diff --git a/src/bindings/python/flux/job/validator/validator.py b/src/bindings/python/flux/job/validator/validator.py
new file mode 100644
index 000000000000..172c1ccefb5b
--- /dev/null
+++ b/src/bindings/python/flux/job/validator/validator.py
@@ -0,0 +1,237 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import argparse
+import concurrent.futures
+import json
+from abc import ABC, abstractmethod
+
+import flux
+from flux.importer import import_path, import_plugins
+
+
+class ValidatorResult:
+    """Container for result or results from the JobValidator validate method"""
+
+    def __init__(self):
+        self.errnum = 0
+        self.errmsgs = []
+
+    def __str__(self):
+        result = dict(errnum=self.errnum)
+        if self.errmsgs:
+            result["errstr"] = self.errmsg
+        return json.dumps(result)
+
+    def push_result(self, errnum, errmsg=None):
+        """Add a result from one validator to a ValidatorResult
+
+        Args:
+            errnum (:obj:`int`): error number (0 for success)
+            errmsg (:obj:`str`, optional): An optional error message for a
+                failed result.
+        """
+        if errnum > self.errnum:
+            self.errnum = errnum
+        if errmsg and errmsg not in self.errmsgs:
+            self.errmsgs.append(errmsg)
+
+    @property
+    def errmsg(self):
+        """str: comma-separated string list of all error messages"""
+        return ", ".join(self.errmsgs)
+
+    @property
+    def success(self):
+        """bool: True if job validated successfully, False otherwise"""
+        return self.errnum == 0
+
+
+class ValidatorJobInfo:
+    """An instance of a Flux job specification used by job validators
+
+    Attributes:
+        jobspec (dict): Submitted jobspec in Python dict form
+        userid (int): Submitting user id
+        flags (int): Job flags supplied during submission
+        urgency (int): Job urgency
+    """
+
+    def __init__(self, jobinfo):
+        self.jobinfo = jobinfo
+
+    def __getattr__(self, attr):
+        #  Return components of the validate request as attrs
+        return self.jobinfo[attr]
+
+
+class ValidatorPlugin(ABC):  # pragma: no cover
+    """Base class for Validator Plugins
+
+    Attributes:
+        flux (:obj:`flux.Flux`): on-demand per-plugin (per-thread) Flux
+            handle.
+    """
+
+    def __init__(self, parser):
+        """Initialize a ValidatorPlugin"""
+
+    @property
+    def flux(self):
+        if not hasattr(self, "_flux"):
+            self._flux = flux.Flux()
+        return self._flux
+
+    def configure(self, args):
+        """Configure a ValidatorPlugin. Run after argparse.parse_args()
+
+        Args:
+            args (:obj:`Namespace`): The resulting Namespace after calling
+            argparse.parse_args()
+        """
+
+    @abstractmethod
+    def validate(self, job):
+        """Validate a job. A ValidatorPlugin must implement this method
+
+        If a job fails validation, this method should either throw an
+        exception, which will be caught by the calling script, or a
+        ``(errnum, errmsg)`` tuple may optionally be returned, if that
+        is more convenient.
+
+        On success, this method should return nothing or explicitly::
+
+            (0, None)
+
+        Args:
+            job (:obj:`ValidatorJobInfo`): the job to validate
+
+        Returns:
+            None or (errnum, errmsg) tuple.
+        """
+        raise NotImplementedError
+
+
+# pylint: disable=too-many-instance-attributes
+class JobValidator:
+    """A plugin-based job validator class
+
+    JobValidator loads plugins that implement the ValidatorPlugin interface
+    from the 'flux.job.validator.plugins' namespace. Plugins may be configured
+    at runtime by passing in a ``--plugins=LIST`` option
+
+    """
+
+    default_validators = ["jobspec"]
+    plugin_namespace = "flux.job.validator.plugins"
+
+    def __init__(self, argv, pluginpath=None, parser=None):
+
+        self.validators = []
+        self.executor = None
+
+        if pluginpath is None:
+            pluginpath = []
+
+        #  Setup parser so we can parse --plugin and plugins can attach
+        #   their own options
+        if parser is None:
+            parser = argparse.ArgumentParser(
+                formatter_class=flux.util.help_formatter(), add_help=False
+            )
+
+        self.parser = parser
+        self.parser_group = self.parser.add_argument_group("Validator options")
+        self.plugins_group = self.parser.add_argument_group(
+            "Options provided by plugins"
+        )
+        self.parser_group.add_argument("--plugins", action="append", default=[])
+
+        #  Parse provided argv, but only parse known args, save
+        #   remaining arguments for plugin configuration once plugins
+        #   have been selected.
+        #
+        args, self.remaining_args = self.parser.parse_known_args(argv)
+        if not args.plugins:
+            args.plugins = self.default_validators
+        else:
+            args.plugins = [x for xs in args.plugins for x in xs.split(",")]
+
+        #  Load all available validator plugins:
+        self.plugins = import_plugins(self.plugin_namespace, pluginpath)
+        self.args = args
+
+    def start(self):
+        """Select and configure plugins, start executor, etc."""
+
+        #  Now configure selected plugins:
+        for name in self.args.plugins:
+            if name not in self.plugins:
+                try:
+                    self.plugins[name] = import_path(name)
+                except:
+                    raise ValueError(f"validator plugin '{name}' not found")
+            plugin = self.plugins[name].Validator(parser=self.plugins_group)
+            self.validators.append(plugin)
+
+        #  Parse remaining args and pass result to plugins now that all
+        args = self.parser.parse_args(self.remaining_args)
+        for validator in self.validators:
+            validator.configure(args)
+
+        self.executor = concurrent.futures.ThreadPoolExecutor(
+            max_workers=len(self.validators),
+        )
+        return self
+
+    def stop(self):
+        """Stop the validator."""
+        self.executor.shutdown()
+
+    def validate(self, jobinfo):
+        """Validate jobinfo using all loaded validators
+
+        Args:
+            jobinfo (:obj:`ValidatorJobInfo`): A ValidatorJobInfo object which
+                describes the job to be validated.
+
+        Returns:
+            :obj:`ValidatorResult`
+
+            If any one validator plugin fails, then result will indicate
+            failure.
+        """
+
+        # Empty jobinfo is considered success
+        if jobinfo is None:
+            return (0, None)
+
+        if isinstance(jobinfo, str):
+            jobinfo = json.loads(jobinfo)
+        job = ValidatorJobInfo(jobinfo)
+
+        futures = [
+            self.executor.submit(validator.validate, job)
+            for validator in self.validators
+        ]
+
+        result = ValidatorResult()
+        for fut in concurrent.futures.as_completed(futures):
+            try:
+                res = fut.result()
+                if res is not None:
+                    result.push_result(*res)
+            except (ValueError, TypeError, EnvironmentError) as exc:
+                result.push_result(1, str(exc))
+                for future in futures:
+                    future.cancel()
+            except concurrent.futures.CancelledError:
+                pass
+        return result
diff --git a/src/bindings/python/flux/job/wait.py b/src/bindings/python/flux/job/wait.py
new file mode 100644
index 000000000000..8a4281b539ed
--- /dev/null
+++ b/src/bindings/python/flux/job/wait.py
@@ -0,0 +1,201 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+import collections
+import errno
+import json
+
+import flux
+from _flux._core import ffi, lib
+from flux.future import Future
+from flux.job._wrapper import _RAW as RAW
+from flux.util import check_future_error, interruptible
+
+
+class JobWaitFuture(Future):
+    def get_status(self):
+        """Return the result of a job wait request.
+
+        This method blocks until the response is received,
+        then decodes the result to obtain the job status.
+
+        :returns: job status, a tuple of: Job ID (int), success (bool),
+            and an error (string) if success=False
+        :rtype: JobWaitResult
+        """
+        return wait_get_status(self)
+
+
+def wait_async(flux_handle, jobid=lib.FLUX_JOBID_ANY):
+    """Wait for a job to complete, asynchronously
+
+    Submit a request to wait for job completion.  This method returns
+    immediately with a Flux Future, which can be used to process
+    the result later.
+
+    Only jobs submitted with waitable=True can be waited for.
+
+    :param flux_handle: handle for Flux broker from flux.Flux()
+    :type flux_handle: Flux
+    :param jobid: the job ID to wait for (default is any waitable job)
+    :returns: a Flux Future object for obtaining the job result
+    :rtype: JobWaitFuture
+    """
+    future_handle = RAW.wait(flux_handle, jobid)
+    return JobWaitFuture(future_handle)
+
+
+JobWaitResult = collections.namedtuple("JobWaitResult", "jobid, success, errstr")
+
+
+@check_future_error
+def wait_get_status(future):
+    """Get job status from a Future returned by job.wait_async()
+
+    Process a response to a Flux job wait request.  This method blocks
+    until the response is received, then decodes the result to obtain
+    the job status.
+
+    :param future: a Flux future object returned by job.wait_async()
+    :type future: Future
+    :returns: job status, a tuple of: Job ID (int), success (bool),
+        and an error (string) if success=False
+    :rtype: JobWaitResult
+    """
+    if future is None or future == ffi.NULL:
+        raise EnvironmentError(errno.EINVAL, "future must not be None/NULL")
+    future.wait_for()  # ensure the future is fulfilled
+    success = ffi.new("bool[1]")
+    errstr = ffi.new("const char *[1]")
+    jobid = ffi.new("flux_jobid_t[1]")
+    RAW.wait_get_id(future, jobid)
+    RAW.wait_get_status(future, success, errstr)
+    return JobWaitResult(int(jobid[0]), bool(success[0]), ffi.string(errstr[0]))
+
+
+def wait(flux_handle, jobid=lib.FLUX_JOBID_ANY):
+    """Wait for a job to complete
+
+    Submit a request to wait for job completion, blocking until a
+    response is received, then return the job status.
+
+    Only jobs submitted with waitable=True can be waited for.
+
+    :param flux_handle: handle for Flux broker from flux.Flux()
+    :type flux_handle: Flux
+    :param jobid: the job ID to wait for (default is any waitable job)
+    :returns: job status, a tuple of: Job ID (int), success (bool),
+        and an error (string) if success=False
+    :rtype: JobWaitResult
+    """
+    future = wait_async(flux_handle, jobid)
+    return future.get_status()
+
+
+class JobResultFuture(Future):
+    """Future fulfilled with a job "result"
+
+    Supports methods to return the result as either a raw :obj:`dict` or
+    :obj:`flux.job.info.JobInfo` object.
+    """
+
+    @interruptible
+    def get_dict(self):
+        """Get the raw "result" dictionary for the job
+
+        Return the underlying "result" payload from ``flux_job_result(3)``
+        as a dictionary.
+        """
+        try:
+            result_str = ffi.new("char *[1]")
+            RAW.result_get(self.handle, result_str)
+        except OSError:
+            self.raise_if_handle_exception()
+            raise
+        if result_str[0] == ffi.NULL:
+            return None
+        return json.loads(ffi.string(result_str[0]).decode("utf-8"))
+
+    @interruptible
+    def get_info(self):
+        """Get a :obj:`flux.job.info.JobInfo` object from the job result
+
+        Note: The JobInfo object returned from this method is only capable
+        of computing a small subset of job information, including, but possibly
+        not limited to:
+            - id
+            - t_submit, t_run, t_cleanup
+            - returncode
+            - waitstatus
+            - runtime
+            - result
+            - result_id
+        """
+        return flux.job.JobInfo(self.get_dict())
+
+
+def result_async(flux_handle, jobid, flags=0):
+    """Wait for a job to reach its terminal state and return job result
+
+    This function waits for job completion by watching the eventlog.
+    Because this function must process the eventlog, it is a little more
+    heavyweight than :meth:`flux.job.wait.wait_async`. However, it may
+    be used for non-waitable jobs, jobs that have already completed,
+    and works multiple times on the same jobid.
+
+    Once the eventlog terminal state is reached, the returned Future is
+    fulfilled with a set of information gleaned from the processed events,
+    including whether the job started running (in case it was canceled
+    before starting), any exception state, and the final exit code and
+    wait(2) status.
+
+    Args:
+        flux_handle (:obj:`flux.Flux`): handle for Flux broker
+        jobid (:obj:`flux.job.JobID`): the jobid for which to fetch result
+
+    Returns:
+        JobResultFuture: A Future fulfilled with the job result.
+    """
+    future = RAW.result(flux_handle, flux.job.JobID(jobid), flags)
+    return JobResultFuture(future)
+
+
+def result(flux_handle, jobid, flags=0):
+    """Wait for a job to reach its terminal state and return job result
+
+    This function waits for job completion by watching the eventlog.
+    Because this function must process the eventlog, it is a little more
+    heavyweight than :meth:`flux.job.wait.wait`. However, it may be used
+    for non-waitable jobs, jobs that have already completed, and works
+    multiple times on the same jobid.
+
+    This function will wait until the job result is available and returns
+    a :obj:`flux.job.info.JobInfo` object filled with the available information.
+
+    Note: The JobInfo object returned from this method is only capable
+    of computing a small subset of job information, including, but possibly
+    not limited to:
+        - id
+        - t_submit, t_run, t_cleanup
+        - returncode
+        - waitstatus
+        - runtime
+        - result
+        - result_id
+
+    Args:
+        flux_handle (:obj:`flux.Flux`): handle for Flux broker
+        jobid (:obj:`flux.job.JobID`): the jobid for which to fetch result
+
+    Returns:
+        JobInfo: A limited JobInfo object which can be used to fetch
+        the final job result, returncode, etc.
+    """
+    future = result_async(flux_handle, flux.job.JobID(jobid), flags)
+    return future.get_info()
diff --git a/src/bindings/python/flux/job/watcher.py b/src/bindings/python/flux/job/watcher.py
new file mode 100644
index 000000000000..09b065254b55
--- /dev/null
+++ b/src/bindings/python/flux/job/watcher.py
@@ -0,0 +1,592 @@
+#############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import os
+import sys
+import time
+from collections import Counter
+
+import flux
+from flux.job import output_watch_async
+from flux.progress import ProgressBar
+
+
+class JobStatus:
+    """
+    Simple convenience class for caching job "state" in the JobProgressBar
+    and JobWatch classes.
+
+    Attributes:
+        id: The job id
+        status: This job's current simplified status (See below for possible
+            status values)
+        exitcode: The exit code of the job (0 for success, otherwise failure)
+
+    Current valid statuses include:
+    - pending
+    - running
+    - complete
+    - failed
+    """
+
+    def __init__(self, job):
+        """
+        Initialize a JobStatus object using an instance of flux.job.JobInfo
+        """
+        if not isinstance(job, flux.job.JobInfo):
+            raise ValueError("JobStatus requires an object of type JobInfo")
+        self.id = job.id
+        self.status, self.exitcode = self._jobinfo_get_status_and_exitcode(job)
+        self._events = Counter()
+
+    def _jobinfo_get_status_and_exitcode(self, job):
+        #  Return simplified job status and exitcode if job is inactive
+        status = job.status
+        if status in ("DEPEND", "PRIORITY", "SCHED"):
+            return "pending", 0
+        elif status in ("RUN", "CLEANUP"):
+            return "running", 0
+        elif status == "COMPLETED":
+            return "complete", 0
+        elif status in ("FAILED", "CANCELED", "TIMEOUT"):
+            return "failed", job.returncode
+        raise ValueError(f"unknown job status {status}")
+
+    @property
+    def active(self):
+        """True if the job is still active"""
+        return self.status in ("running", "pending")
+
+    def add_event(self, name):
+        """
+        Add an event to the event counter
+        """
+        self._events[name] += 1
+
+    def event_count(self, name):
+        """
+        Return the number of times event ``name`` has been seen for this job
+        """
+        return self._events[name]
+
+    def has_event(self, name):
+        """
+        Return True if this job has seen event ``name``
+        """
+        return self.event_count(name) > 0
+
+
+class JobProgressBar(ProgressBar):
+    """Progress bar for multiple jobs
+
+    The JobProgressBar class is a ProgressBar specific to monitoring
+    progress of a group of jobs. It displays a progress bar with the
+    number of pending, running, complete and failed jobs on the left,
+    and a percent complete and elapsed timer by default on the right,
+    with a progress bar in the middle.
+
+    Once a JobProgressBar object is initialized, jobs to track should
+    be added with JobProgressBar.add_job or JobProgressBar.add_jobs.
+    Subsequently, eventlog events for jobs are fed into the progress
+    bar via JobProgressBar.process_event. To advance the progress bar,
+    the ``update()`` method should be called.
+
+    Attributes:
+        jobs: list of JobStatus objects being monitored
+        starttime: start time used for the elapsed timer and jobs/s display.
+            By default this will be the minimum submission time of all jobs
+            being monitored. To choose a different starttime, manually set
+            the starttime in the initializer.
+        total: total number of jobs being monitored
+        pending: current pending job count
+        running: current running job count
+        complete: current complete job count (successful jobs)
+        failed: current failed job count (failed, canceled, timeout)
+    """
+
+    def __init__(
+        self,
+        flux_handle,
+        starttime=None,
+        jps=False,
+        counter_width=3,
+        update_interval=0.25,
+    ):
+        """
+        Initialize an instance of a JobProgressBar.
+
+        Args
+            starttime (float): if set, use this value as the start time for
+                calculation of the progress bar elapsed time of jobs/s.
+            jps (bool): show job/s on right hand side instead of elapsed time
+            counter_width (int): width reserved for pending, running, etc.
+                counters on left hand display (default=3)
+            update_interval (float): interval in floating point seconds at
+                which the progress bar is forced to be updated. For an elapsed
+                timer, this should be at least 1.0 (default=0.25)
+        """
+        before = (
+            "PD:{pending:<{width}} R:{running:<{width}} "
+            "CD:{complete:<{width}} F:{failed:<{width}} "
+        )
+        after = "{percent:5.1f}% {elapsed.dt}"
+        if jps:
+            after = "{percent:5.1f}% {jps:4.1f} job/s"
+        self.jobs = {}
+        self._finished_jobs = {}
+        self._started = False
+        self.starttime = starttime
+        self.status = Counter()
+
+        timer = flux_handle.timer_watcher_create(
+            0, lambda *x: self.redraw(), repeat=update_interval
+        )
+
+        super().__init__(
+            timer=timer,
+            total=0,
+            width=counter_width,
+            before=before,
+            after=after,
+            pending=0,
+            running=0,
+            complete=0,
+            failed=0,
+            jps=0,
+        )
+
+    def start(self):
+        """
+        Start JobProgressBar operation.
+
+        Initialize and start timer watchers, display initial progress bar,
+        and if not set, initialize the elapsed start time.
+        """
+        if self._started:
+            return
+
+        self._started = True
+        self.timer.start()
+        #  Don't let this timer watcher contribute to the reactor's
+        #   "active" reference count:
+        #
+        self.timer.flux_handle.reactor_decref()
+        super().start()
+        #  Override superclass `_t0` attribute to elapsed time is computed
+        #  from this value and not the time of super().start():
+        #
+        if self.starttime is not None:
+            self._t0 = self.starttime
+
+    def update(self):
+        """
+        Update job state counts for ProgressBar and refresh display
+        """
+        super().update(
+            advance=0,
+            pending=self.pending,
+            running=self.running,
+            complete=self.complete,
+            failed=self.failed,
+        )
+
+    def advance(self, **kwargs):
+        """
+        Advance progress bar (e.g. if one job has completed).
+        Args:
+            kwargs: keyword args passed to ProgressBar.update
+        """
+        super().update(advance=1, **kwargs)
+
+    def add_job(self, job):
+        """
+        Begin monitoring the progress of a job.
+
+        Args:
+            job (JobInfo): The job to begin monitoring
+        """
+        if not isinstance(job, flux.job.JobInfo):
+            raise ValueError("add_job takes an argument of type JobInfo")
+        if job.id in self.jobs:
+            raise ValueError(f"job {job.id} is already being monitored")
+        if self._t0 is None or job.t_submit < self._t0:
+            self._t0 = job.t_submit
+        jobstatus = JobStatus(job)
+        self.jobs[job.id] = jobstatus
+        self.total += 1
+        #  Increment current status attribute:
+        setattr(self, jobstatus.status, getattr(self, jobstatus.status) + 1)
+        #  Update counts/redraw
+        self.update()
+
+    def add_jobs(self, *jobs):
+        """
+        Add multiple jobs to a JobProgressBar instance
+        """
+        for job in jobs:
+            self.add_job(job)
+
+    def jobs_per_sec(self):
+        """
+        Return the current job throughput.
+        """
+        return (self.count + 1) / (time.time() - self._t0)
+
+    def _set_running(self, jobid):
+        job = self.jobs[jobid]
+        if job.status != "pending":
+            raise ValueError(f"set_running: {job.id} not pending")
+        job.status = "running"
+        self.pending -= 1
+        self.running += 1
+
+    def _set_complete(self, jobid):
+        job = self.jobs[jobid]
+        if job.status != "running":
+            raise ValueError(f"set_complete: {job.id} not running")
+        job.status = "complete"
+        self.running -= 1
+        self.complete += 1
+
+    def _set_failed(self, jobid):
+        job = self.jobs[jobid]
+        if job.status == "pending":
+            self.pending -= 1
+        elif job.status == "running":
+            self.running -= 1
+        self.failed += 1
+        job.status = "failed"
+
+    def process_event(self, jobid, event=None):
+        """
+        Process an event for a job, updating job's progress if necessary.
+
+        Args:
+            jobid: job id
+            event: event entry to process. If None, then the job is considered
+                complete, i.e. no more events will be received for this job.
+        """
+        job = self.jobs[jobid]
+        if event is None:
+            #
+            #  Caller should set event=None when no more events are expected
+            #  for this job. This is where we advance the progress bar instead
+            #  of at the 'finish' or 'exception' events since this allows the
+            #  caller to determine when progress should advance (e.g. if the
+            #  use case is to only wait for job 'start' events.
+            #
+            if jobid not in self._finished_jobs:
+                self._finished_jobs[jobid] = True
+                self.advance(jps=self.jobs_per_sec())
+        elif event.name == "alloc" and job.status == "pending":
+            self._set_running(jobid)
+        elif event.name == "exception" and event.context["severity"] == 0:
+            #
+            #  Exceptions only need to be specially handled in the
+            #   pending state. If the job is running and gets an exception
+            #   then a finish event will be posted.
+            #
+            if job.status == "pending":
+                self._set_failed(jobid)
+        elif event.name == "finish" and job.active:
+            job.exitcode = event.context["status"]
+            if job.exitcode == 0:
+                self._set_complete(jobid)
+            else:
+                self._set_failed(jobid)
+        self.update()
+
+
+class JobWatcher:
+    """Watch output and status for multiple jobs.
+
+    The JobWatcher class can watch the status, output, and logs for one or
+    more jobs, optionally including a progress bar for use in a tty. This
+    is the class that implements the ``--watch`` option of ``flux submit``
+    and ``flux bulksubmit``.
+
+    """
+
+    class JobWatchStatus(JobStatus):
+        """
+        JobStatus class with extra attributes for use in JobWatcher
+        """
+
+        def __init__(self, job, stdout, stderr, wait="clean"):
+            super().__init__(job)
+            self.stdout = stdout
+            self.stderr = stderr
+            self.wait = wait
+
+    def __init__(
+        self,
+        flux_handle,
+        jobs=None,
+        progress=False,
+        jps=False,
+        wait="clean",
+        watch=True,
+        log_events=False,
+        log_status=False,
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+        labelio=False,
+        starttime=None,
+    ):
+        """
+        Initialize an instance of the JobWatcher class.
+
+        Args:
+            flux_handle (Flux): Flux handle
+            jobs (list of JobInfo): initialize JobWatcher with a list of jobs
+            progress (bool): Show status and throughput progress bar
+                (default=False)
+            jps (bool): with ``progress=True`` show jobs per second instead of
+                timer on right hand side of progress bar
+            wait (str): Event to wait for before terminating watch of job.
+                (default="clean")
+            log_events (bool): Log all events on stderr (default=False)
+            log_status (bool): Log final status of jobs if applicable
+                (default=False)
+            stdout (TextIOWrapper): Default stdout location (default=sys.stdout)
+            stderr (TextIOWrapper): Default stderr location (default=sys.stderr)
+            labelio (bool): Label lines of output with jobid and taskid
+            starttime (float): If not None, start elapsed timer at this time.
+        """
+        self.flux_handle = flux_handle
+        self.progress = None
+        self.wait = wait
+        self.watch = watch
+        self.t0 = starttime
+        self.log_events = log_events
+        self.log_status = log_status
+        self.stdout = self._reopen(stdout)
+        self.stderr = self._reopen(stderr)
+        self.labelio = labelio
+        self.exitcode = 0
+        self.show_progress = progress
+        self.progress = JobProgressBar(flux_handle, starttime=self.t0, jps=jps)
+
+        self._states = Counter()
+
+        if jobs:
+            self.add_jobs(*jobs)
+
+    @staticmethod
+    def _reopen(stream):
+        """reconfigure/reopen stream with correct encoding and error handling"""
+        try:
+            # reconfigure() only available in >=3.7
+            stream.reconfigure(encoding="utf-8", errors="surrogateescape")
+            return stream
+        except AttributeError:
+            return open(
+                stream.fileno(),
+                mode="w",
+                encoding="utf-8",
+                errors="surrogateescape",
+                closefd=False,
+            )
+
+    @staticmethod
+    def _status_to_exitcode(status):
+        """Calculate exitcode from job status"""
+        if os.WIFEXITED(status):
+            status = os.WEXITSTATUS(status)
+        elif os.WIFSIGNALED(status):
+            status = 128 + os.WTERMSIG(status)
+        return status
+
+    def start(self):
+        """
+        Start JobWatcher progress bar if configured
+        """
+        if self.show_progress:
+            self.progress.start()
+        return self
+
+    def stop(self):
+        """
+        Stop JobWatcher progress bar if configured
+        """
+        if self.show_progress:
+            self.progress.stop()
+        return self
+
+    def add_jobs(self, *jobs, stdout=None, stderr=None, wait="clean"):
+        """
+        Begin monitoring the progress of a set of jobs
+
+        Args:
+            *jobs (JobID): one or more jobs to monitor
+            stdout (TextIOWrapper): destination for stdout for this job
+                or jobs. If None, use default for this JobWatcher instance.
+            stderr: (TextIOWrapper): destination for stderr for this job
+                or jobs (if None, then set to same location as ``stdout``)
+            wait: event at which to stop watching the job (default=clean)
+        """
+
+        self.progress.add_jobs(*jobs)
+
+        if stdout is None:
+            stdout = self._reopen(self.stdout)
+        if stderr is None:
+            stderr = self._reopen(self.stderr)
+
+        for job in jobs:
+            if not self.t0 or job.t_submit < self.t0:
+                self.t0 = job.t_submit
+
+            job_status = self.JobWatchStatus(job, stdout, stderr, wait=wait)
+            flux.job.event_watch_async(self.flux_handle, job.id).then(
+                self._event_watch_cb, job_status
+            )
+            if not job_status.active:
+                self._progress_update(job_status)
+        return self
+
+    def add_jobid(self, jobid, stdout=None, stderr=None, wait="clean"):
+        """
+        Begin monitoring the progress of a job by jobid
+
+        Args:
+            jobid (JobID): one or more jobs to monitor
+            stdout (TextIOWrapper): destination for stdout for this job
+                or jobs. If None, use default for this JobWatcher instance.
+            stderr: (TextIOWrapper): destination for stderr for this job
+                or jobs (if None, then set to same location as ``stdout``)
+            wait: event at which to stop watching the job (default=clean)
+        """
+        #
+        #  add_jobs() expects a JobInfo object, but caller only has a jobid.
+        #  Create a mock JobInfo object assuming the job is in SCHED state
+        #  so that this job is properly initialized for watching.
+        #
+        job = flux.job.JobInfo(
+            {
+                "id": jobid,
+                "state": flux.constants.FLUX_JOB_STATE_SCHED,
+                "t_submit": time.time(),
+            }
+        )
+        self.add_jobs(job, stdout=stdout, stderr=stderr, wait=wait)
+
+    def _progress_update(self, job, event=None):
+        self.progress.process_event(job.id, event)
+
+    def _log(self, job, timestamp, msg):
+        dt = timestamp - self.t0
+        print(f"{job.id.f58}: {dt:4.3f}s: {msg}", file=job.stderr)
+
+    def _log_event(self, job, event, event_prefix=""):
+        if self.log_events and event is not None:
+            self._log(
+                job,
+                event.timestamp,
+                f"{event_prefix}{event.name} {event.context_string}",
+            )
+
+    def _event_watch_cb(self, future, job):
+        event = future.get_event()
+
+        # Update progress meter if being used
+        self._progress_update(job, event)
+
+        # End of eventlog
+        if event is None:
+            return
+
+        job.add_event(event.name)
+
+        if event.timestamp < self.t0:
+            self.t0 = event.timestamp
+
+        self._log_event(job, event)
+
+        if event.name == "exception":
+            severity = event.context["severity"]
+            if severity == 0:
+                #  If job didn't start then it failed to execute.
+                #  Set status to failed and emit exception error on stderr:
+                if not job.has_event("start"):
+                    self.exitcode = max(self.exitcode, 1)
+                    job.status = "failed"
+
+                #  If the output eventlog is not being watched because the
+                #  shell never initialized, then print the exception error
+                #  to stderr:
+                if not job.has_event("shell.init"):
+                    print(
+                        flux.job.output.JobExceptionEvent(event).render(),
+                        file=job.stderr,
+                    )
+        elif event.name == "alloc":
+            job.status = "running"
+        elif event.name == "start":
+            if self.watch or job.wait.startswith("exec."):
+                flux.job.event_watch_async(
+                    self.flux_handle, job.id, eventlog="guest.exec.eventlog"
+                ).then(self._exec_event_cb, job, future)
+        elif event.name == "finish":
+            #
+            # job finished. Collect wait status into self.exitcode
+            #
+            status = event.context["status"]
+            self.exitcode = max(self.exitcode, self._status_to_exitcode(status))
+            job.status = "complete"
+            if status > 0:
+                job.status = "failed"
+            if self.log_status:
+                self._log(job, event.timestamp, f"{job.status}: status={status}")
+        if job.wait and job.wait == event.name:
+            #
+            # Done with this job: update progress bar and cancel future
+            #
+            self._progress_update(job)
+            future.cancel(stop=True)
+
+    def _exec_event_cb(self, future, job, main_eventlog_future):
+        event = future.get_event()
+        if event is None:
+            return
+        self._log_event(job, event, event_prefix="exec.")
+        if self.watch and event.name == "shell.init":
+            #  shell.init event indicates output eventlog is ready
+            #  (use nowait=True to avoid watching intermediate eventlogs)
+            #
+            output_watch_async(
+                self.flux_handle,
+                job.id,
+                labelio=self.labelio,
+                nowait=True,
+            ).then(self._output_watch_cb, job)
+
+            if not job.wait or not job.wait.startswith("exec."):
+                # No more events from exec eventlog are needed
+                future.cancel(stop=True)
+
+        if job.wait and job.wait == f"exec.{event.name}":
+            #  Done with this job
+            #
+            future.cancel(stop=True)
+            main_eventlog_future.cancel(stop=True)
+
+    def _output_watch_cb(self, future, job):
+        stream, data = future.get_output()
+        if stream is not None:
+            output_stream = getattr(job, stream)
+            if self.labelio:
+                for line in data.splitlines(keepends=True):
+                    output_stream.write(f"{job.id}: {line}")
+            else:
+                output_stream.write(data)
+        else:
+            for stream in ("stdout", "stderr"):
+                getattr(job, stream).flush()
diff --git a/src/bindings/python/flux/kvs.py b/src/bindings/python/flux/kvs.py
index 0f1f33d3fbbb..20eb79b5ed7d 100644
--- a/src/bindings/python/flux/kvs.py
+++ b/src/bindings/python/flux/kvs.py
@@ -8,22 +8,19 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-from __future__ import print_function
-
-import json
+import collections.abc as abc
 import errno
-import collections
+import json
+import os
+from abc import ABC, abstractmethod
+from typing import Any, Mapping
 
+import flux.constants
 from _flux._core import ffi, lib
+from flux.future import Future
+from flux.rpc import RPC
 from flux.wrapper import Wrapper, WrapperPimpl
 
-try:
-    # pylint: disable=invalid-name
-    collectionsAbc = collections.abc
-except AttributeError:
-    # pylint: disable=invalid-name
-    collectionsAbc = collections
-
 
 class KVSWrapper(Wrapper):
     # This empty class accepts new methods, preventing accidental overloading
@@ -36,21 +33,44 @@ class KVSWrapper(Wrapper):
 RAW.flux_kvsitr_next.set_error_check(lambda x: False)
 
 
-def get_key_direct(flux_handle, key):
+def _get_value(valp):
+    try:
+        ret = json.loads(ffi.string(valp[0]).decode("utf-8"))
+    except json.decoder.JSONDecodeError:
+        ret = ffi.string(valp[0]).decode("utf-8")
+    except UnicodeDecodeError:
+        ret = ffi.string(valp[0])
+    return ret
+
+
+def get_key_direct(flux_handle, key, namespace=None):
     valp = ffi.new("char *[1]")
-    future = RAW.flux_kvs_lookup(flux_handle, None, 0, key)
+    future = RAW.flux_kvs_lookup(flux_handle, namespace, 0, key)
     RAW.flux_kvs_lookup_get(future, valp)
     if valp[0] == ffi.NULL:
         return None
 
-    ret = json.loads(ffi.string(valp[0]).decode("utf-8"))
+    ret = _get_value(valp)
     RAW.flux_future_destroy(future)
     return ret
 
 
-def exists(flux_handle, key):
+def exists(flux_handle, key, namespace=None):
+    """Determine if key exists
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        key: key to check for existence
+
+    Returns:
+        bool: True if key exists, False if not
+        namespace: namespace to read from, defaults to None.  If namespace
+          is None, the namespace specified in the FLUX_KVS_NAMESPACE
+          environment variable will be used.  If FLUX_KVS_NAMESPACE is not
+          set, the primary namespace will be used.
+    """
     try:
-        get_key_direct(flux_handle, key)
+        get_key_direct(flux_handle, key, namespace=namespace)
         return True
     except EnvironmentError as err:
         if err.errno == errno.ENOENT:
@@ -60,77 +80,435 @@ def exists(flux_handle, key):
         raise err
 
 
-def isdir(flux_handle, key):
+def isdir(flux_handle, key, namespace=None):
+    """Determine if key is a directory
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        key: key to check if it is a directory
+
+    Returns:
+        bool: True if key is a directory, False if not
+        namespace: namespace to read from, defaults to None.  If namespace
+          is None, the namespace specified in the FLUX_KVS_NAMESPACE
+          environment variable will be used.  If FLUX_KVS_NAMESPACE is not
+          set, the primary namespace will be used.
+    """
     try:
-        get_key_direct(flux_handle, key)
+        get_key_direct(flux_handle, key, namespace=namespace)
     except EnvironmentError as err:
+        if err.errno == errno.ENOENT:
+            return False
         if err.errno == errno.EISDIR:
             return True
         raise err
     return False
 
 
-def get_dir(flux_handle, key="."):
-    return KVSDir(path=key, flux_handle=flux_handle)
-
-
-def get(flux_handle, key):
+def get_dir(flux_handle, key=".", namespace=None, _kvstxn=None):
+    """Get KVS directory
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        key: directory name (default ".")
+        namespace: namespace to read from, defaults to None.  If namespace
+          is None, the namespace specified in the FLUX_KVS_NAMESPACE
+          environment variable will be used.  If FLUX_KVS_NAMESPACE is not
+          set, the primary namespace will be used.
+
+    Returns:
+        KVSDir: object representing directory
+    """
+    return KVSDir(
+        path=key, flux_handle=flux_handle, namespace=namespace, _kvstxn=_kvstxn
+    )
+
+
+def get(flux_handle, key, namespace=None, _kvstxn=None):
+    """Get KVS directory
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        key: key to get
+        namespace: namespace to read from, defaults to None.  If namespace
+          is None, the namespace specified in the FLUX_KVS_NAMESPACE
+          environment variable will be used.  If FLUX_KVS_NAMESPACE is not
+          set, the primary namespace will be used.
+
+    Returns:
+        If value is decodeable by json.loads(), the decoded
+        result is returned.  If the value is a legal utf-8 decodable
+        string, it is returned as a string.  Otherwise, the value is
+        returned as a bytes array.
+    """
     try:
-        return get_key_direct(flux_handle, key)
+        return get_key_direct(flux_handle, key, namespace=namespace)
     except EnvironmentError as err:
         if err.errno == errno.EISDIR:
             pass
         else:
             raise err
-    return get_dir(flux_handle, key)
+    return get_dir(flux_handle, key, namespace=namespace, _kvstxn=_kvstxn)
+
+
+# convenience function to get RAW kvs txn object to use
+def _get_kvstxn(flux_handle, _kvstxn):
+    # If _kvstxn is None, use the default txn stored in the flux
+    # handle (create it if necessary)
+    if _kvstxn is None:
+        if flux_handle.aux_txn is None:
+            flux_handle.aux_txn = RAW.flux_kvs_txn_create()
+        _kvstxn = flux_handle.aux_txn
+    elif isinstance(_kvstxn, KVSTxn):
+        # if _kvstxn is type KVSTxn, get the RAW kvs txn
+        # stored within it
+        _kvstxn = _kvstxn.txn
+    # we don't perform this check, as it would require an
+    # import of _cffi_backend
+    # elif not isinstance(_kvstxn, _cffi_backend.FFI.CData):
+    #     raise TypeError
+    return _kvstxn
+
+
+def put(flux_handle, key, value, _kvstxn=None):
+    """Put data into the KVS
+
+    Internally will stage changes until commit() is called.
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        key: key to write to
+        value: value of the key
+    """
+    _kvstxn = _get_kvstxn(flux_handle, _kvstxn)
+    try:
+        json_str = json.dumps(value)
+        RAW.flux_kvs_txn_put(_kvstxn, 0, key, json_str)
+    except TypeError:
+        if isinstance(value, bytes):
+            RAW.flux_kvs_txn_put_raw(_kvstxn, 0, key, value, len(value))
+            return
+        raise TypeError
+
+
+def put_mkdir(flux_handle, key, _kvstxn=None):
+    """Create directory in the KVS
+
+    Internally will stage changes until commit() is called.
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        key: directory to create
+    """
+    _kvstxn = _get_kvstxn(flux_handle, _kvstxn)
+    RAW.flux_kvs_txn_mkdir(_kvstxn, 0, key)
+
+
+def put_unlink(flux_handle, key, _kvstxn=None):
+    """Unlink key in the KVS
+
+    Internally will stage changes until commit() is called.
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        key: key to delete
+    """
+    _kvstxn = _get_kvstxn(flux_handle, _kvstxn)
+    RAW.flux_kvs_txn_unlink(_kvstxn, 0, key)
+
+
+def put_symlink(flux_handle, key, target, _kvstxn=None):
+    """Create symlink in the KVS
+
+    Internally will stage changes until commit() is called.
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        key: symlink name
+        target: target symlink points to
+    """
+    _kvstxn = _get_kvstxn(flux_handle, _kvstxn)
+    RAW.flux_kvs_txn_symlink(_kvstxn, 0, key, None, target)
+
+
+def commit(flux_handle, flags: int = 0, namespace=None, _kvstxn=None):
+    """Commit changes to the KVS
+
+    Must be called after put(), put_mkdir(), put_unlink(), or
+    put_symlink() to write staged changes to the KVS.
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        flags: defaults to 0, possible flag options:
+          flux.constants.FLUX_KVS_NO_MERGE - disallow merging of different commits
+          flux.constants.FLUX_KVS_TXN_COMPACT - if possible compact changes
+          flux.constants.FLUX_KVS_SYNC - flush & checkpoint commit (only against primary KVS)
+        namespace: namespace to write to, defaults to None.  If namespace
+          is None, the namespace specified in the FLUX_KVS_NAMESPACE
+          environment variable will be used.  If FLUX_KVS_NAMESPACE is not
+          set, the primary namespace will be used.
+    """
+    if _kvstxn is None:
+        # If _kvstxn is None, use the default txn stored in the flux
+        # handle.  If aux_txn is None, there is nothing to commit.
+        if flux_handle.aux_txn is None:
+            return
+        future = RAW.flux_kvs_commit(flux_handle, namespace, flags, flux_handle.aux_txn)
+        try:
+            RAW.flux_future_get(future, None)
+        except OSError:
+            raise
+        finally:
+            RAW.flux_kvs_txn_destroy(flux_handle.aux_txn)
+            flux_handle.aux_txn = None
+            RAW.flux_future_destroy(future)
+    else:
+        # if _kvstxn is type KVSTxn, get the RAW kvs txn stored within
+        # it.  If not type KVSTxn, it is assumed to of type RAW txn.
+        if isinstance(_kvstxn, KVSTxn):
+            _kvstxn = _kvstxn.txn
+        future = RAW.flux_kvs_commit(flux_handle, namespace, flags, _kvstxn)
+        try:
+            RAW.flux_future_get(future, None)
+        except OSError:
+            raise
+        finally:
+            RAW.flux_future_destroy(future)
+
+
+def commit_async(flux_handle, flags: int = 0, namespace=None, _kvstxn=None):
+    """Commit changes to the KVS.  Identical to commit(), but returns
+    a Future to wait on for RPC to complete.
+
+    Must be called after put(), put_mkdir(), put_unlink(), or
+    put_symlink() to write staged changes to the KVS.
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        flags: defaults to 0, possible flag options:
+          flux.constants.FLUX_KVS_NO_MERGE - disallow merging of different commits
+          flux.constants.FLUX_KVS_TXN_COMPACT - if possible compact changes
+          flux.constants.FLUX_KVS_SYNC - flush & checkpoint commit (only against primary KVS)
+        namespace: namespace to write to, defaults to None.  If namespace
+          is None, the namespace specified in the FLUX_KVS_NAMESPACE
+          environment variable will be used.  If FLUX_KVS_NAMESPACE is not
+          set, the primary namespace will be used.
+
+    Returns:
+        Future: a future fulfilled when the commit RPC returns
+    """
+    if _kvstxn is None:
+        # If _kvstxn is None, use the default txn stored in the flux
+        # handle.  If aux_txn is None, make an empty txn for the
+        # commit.
+        if flux_handle.aux_txn is None:
+            flux_handle.aux_txn = RAW.flux_kvs_txn_create()
+        future = RAW.flux_kvs_commit(flux_handle, namespace, flags, flux_handle.aux_txn)
+        RAW.flux_kvs_txn_destroy(flux_handle.aux_txn)
+        flux_handle.aux_txn = None
+        return Future(future)
+    else:
+        # if _kvstxn is type KVSTxn, get the RAW kvs txn stored within
+        # it.  If not type KVSTxn, it is assumed to of type RAW txn.
+        if isinstance(_kvstxn, KVSTxn):
+            _kvstxn = _kvstxn.txn
+        future = RAW.flux_kvs_commit(flux_handle, namespace, flags, _kvstxn)
+        return Future(future)
+
+
+def namespace_create(flux_handle, namespace, owner=os.getuid(), flags: int = 0):
+    """Create KVS Namespace
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        namespace: namespace to create
+        owner: uid of namespace owner, defaults to caller uid
+        flags: currently unused, defaults to 0
+    """
+    future = RAW.flux_kvs_namespace_create(flux_handle, namespace, owner, flags)
+    try:
+        RAW.flux_future_get(future, None)
+    finally:
+        RAW.flux_future_destroy(future)
+
 
+def namespace_remove(flux_handle, namespace):
+    """Remove KVS Namespace
 
-def put(flux_handle, key, value):
-    json_str = json.dumps(value)
-    if flux_handle.aux_txn is None:
-        flux_handle.aux_txn = RAW.flux_kvs_txn_create()
-    return RAW.flux_kvs_txn_put(flux_handle.aux_txn, 0, key, json_str)
+    Namespace is removed in background.  Caller cannot be certain of its removal
+    after this function returns.
 
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        namespace: namespace to remove
+    """
+    future = RAW.flux_kvs_namespace_remove(flux_handle, namespace)
+    try:
+        RAW.flux_future_get(future, None)
+    finally:
+        RAW.flux_future_destroy(future)
 
-def put_mkdir(flux_handle, key):
-    if flux_handle.aux_txn is None:
-        flux_handle.aux_txn = RAW.flux_kvs_txn_create()
-    return RAW.flux_kvs_txn_mkdir(flux_handle.aux_txn, 0, key)
 
+def namespace_list(flux_handle):
+    """Get list of KVS Namespace
 
-def put_unlink(flux_handle, key):
-    if flux_handle.aux_txn is None:
-        flux_handle.aux_txn = RAW.flux_kvs_txn_create()
-    return RAW.flux_kvs_txn_unlink(flux_handle.aux_txn, 0, key)
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
 
+    Returns:
+        list: list of strings with names of namespaces
+    """
+    nslist = []
+    rpc = RPC(flux_handle, "kvs.namespace-list")
+    rsp = rpc.get()
+    for ns in rsp["namespaces"]:
+        nslist.append(ns["namespace"])
+    return nslist
 
-def put_symlink(flux_handle, key, target):
-    if flux_handle.aux_txn is None:
-        flux_handle.aux_txn = RAW.flux_kvs_txn_create()
-    return RAW.flux_kvs_txn_symlink(flux_handle.aux_txn, 0, key, None, target)
 
+def dropcache(flux_handle):
+    """Drop KVS cache entries
 
-def commit(flux_handle, flags=0):
-    if flux_handle.aux_txn is None:
-        return -1
-    future = RAW.flux_kvs_commit(flux_handle, None, flags, flux_handle.aux_txn)
-    RAW.flux_future_get(future, None)
-    RAW.flux_kvs_txn_destroy(flux_handle.aux_txn)
-    flux_handle.aux_txn = None
-    return 0
+    Inform KVS module to drop cache entries without a reference.
 
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+    """
+    RAW.flux_kvs_dropcache(flux_handle)
 
-def dropcache(flux_handle):
-    return RAW.flux_kvs_dropcache(flux_handle)
 
+class KVSTxn:
+    """KVS Transaction Object
+
+    Stage changes to the KVS.  When all changes have been placed
+    within the transaction, use commit() to finalize the transaction.
+    Can be used as a context manager and commits will be handled at
+    exit. e.g.
+
+    with KVSTxn(handle, "basedirectory") as kt:
+        kt.put("a", 1)
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        path: Optional base path for all writes to be relative to (default ".")
+        namespace: Optional namespace to write to, defaults to None.  If
+          namespace is None, the namespace specified in the FLUX_KVS_NAMESPACE
+          environment variable will be used.  If FLUX_KVS_NAMESPACE is not
+          set, the primary namespace will be used.
+    """
+
+    def __init__(self, flux_handle=None, path=".", namespace=None):
+        self.fhdl = flux_handle
+        self.path = path
+        # Helper var for easier concatenations
+        if not path or path == ".":
+            self._path = ""
+        else:
+            self._path = path if path[-1] == "." else path + "."
+        self.namespace = namespace
+        self.txn = RAW.flux_kvs_txn_create()
+
+    def commit(self, flags=0):
+        """Commit changes to the KVS
+
+        When keys are added, removed, or updated in the KVSTxn object, the
+        changes are only cached in memory until the commit method asks the
+        KVS service to make them permanent.
+
+        After the commit method returns, updated keys can be accessed by other
+        clients on the same broker rank. Other broker ranks are eventually
+        consistent.
+        """
+        # If no transactions stored, no need to waste an RPC on a commit call
+        if RAW.flux_kvs_txn_is_empty(self.txn):
+            return
+        try:
+            commit(self.fhdl, flags=flags, namespace=self.namespace, _kvstxn=self.txn)
+        except OSError:
+            raise
+        finally:
+            self.clear()
+
+    def commit_async(self, flags=0):
+        """Commit changes to the KVS.  Identical to commit(), but returns a
+        Future to wait on for RPC to complete.
+        """
+        future = commit_async(
+            self.fhdl, flags=flags, namespace=self.namespace, _kvstxn=self.txn
+        )
+        self.clear()
+        return Future(future)
+
+    def put(self, key, value):
+        """Put key=value in the KVS"""
+        put(self.fhdl, self._path + key, value, _kvstxn=self.txn)
+
+    def mkdir(self, key):
+        """Create a directory in the KVS"""
+        put_mkdir(self.fhdl, self._path + key, _kvstxn=self.txn)
+
+    def unlink(self, key):
+        """Unlink key in the KVS"""
+        put_unlink(self.fhdl, self._path + key, _kvstxn=self.txn)
+
+    def symlink(self, key, target):
+        """Create a symlink in the KVS"""
+        put_symlink(self.fhdl, self._path + key, target, _kvstxn=self.txn)
+
+    def clear(self):
+        RAW.flux_kvs_txn_clear(self.txn)
+
+    def __del__(self):
+        RAW.flux_kvs_txn_destroy(self.txn)
+
+    def __enter__(self):
+        """Allow this to be used as a context manager"""
+        return self
+
+    def __exit__(self, type_arg, value, tb):
+        """
+        When used as a context manager, the KVSTxn commits itself on exit
+        """
+        self.commit()
+        return False
+
+
+class KVSDir(WrapperPimpl, abc.MutableMapping):
+    """User friendly class for KVS operations
+
+    KVS values can be read or written through this class's item accessor.  e.g.
+
+    mydir = KVSDir(flux_handle)
+    print(mydir["mykey"])
+
+    mydir["newkey"] = "foo"
+    mydir.commit()
+
+    Any KVS directories accessed through the item accessor will share
+    the same internal KVS transaction, so that only a single call to
+    commit() is necessary.  e.g.
+
+    mydir = KVSDir(flux_handle)
+    subdir = mydir["subdir"]
+    subdir["anotherkey"] = "bar"
+    mydir.commit()
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        path: Optional base path for all read/write to be relative to (default ".")
+        namespace: Optional namespace to read/write from/to, defaults to None.  If
+          namespace is None, the namespace specified in the FLUX_KVS_NAMESPACE
+          environment variable will be used.  If FLUX_KVS_NAMESPACE is not
+          set, the primary namespace will be used.
+
+    """
 
-class KVSDir(WrapperPimpl, collectionsAbc.MutableMapping):
     # pylint: disable=too-many-ancestors, too-many-public-methods
 
     class InnerWrapper(Wrapper):
 
         # pylint: disable=no-value-for-parameter
-        def __init__(self, flux_handle=None, path=".", handle=None):
+        def __init__(self, flux_handle=None, path=".", handle=None, namespace=None):
             dest = RAW.flux_kvsdir_destroy
             super(KVSDir.InnerWrapper, self).__init__(
                 ffi,
@@ -149,7 +527,7 @@ def __init__(self, flux_handle=None, path=".", handle=None):
             if handle is None:
                 directory = ffi.new("flux_kvsdir_t *[1]")
                 future = RAW.flux_kvs_lookup(
-                    flux_handle, None, RAW.FLUX_KVS_READDIR, path
+                    flux_handle, namespace, RAW.FLUX_KVS_READDIR, path
                 )
                 RAW.flux_kvs_lookup_get_dir(future, directory)
                 self.handle = RAW.flux_kvsdir_copy(directory[0])
@@ -157,43 +535,106 @@ def __init__(self, flux_handle=None, path=".", handle=None):
                 if self.handle is None or self.handle == ffi.NULL:
                     raise EnvironmentError("No such file or directory")
 
-    def __init__(self, flux_handle=None, path=".", handle=None):
+    def __init__(
+        self, flux_handle=None, path=".", handle=None, namespace=None, _kvstxn=None
+    ):
         super(KVSDir, self).__init__()
         self.fhdl = flux_handle
         self.path = path
+        # Helper var for easier concatenations
+        if not path or path == ".":
+            self._path = ""
+        else:
+            self._path = path if path[-1] == "." else path + "."
+        self.namespace = namespace
         if flux_handle is None and handle is None:
             raise ValueError(
                 "flux_handle must be a valid Flux object or"
                 "handle must be a valid kvsdir cdata pointer"
             )
-        self.pimpl = self.InnerWrapper(flux_handle, path, handle)
+        self.pimpl = self.InnerWrapper(flux_handle, path, handle, namespace)
+        # See comment in __getitem__ as to why we don't set path to "." and not self.path
+        self.kvstxn = (
+            _kvstxn if _kvstxn else KVSTxn(self.fhdl, ".", namespace=self.namespace)
+        )
 
     def commit(self, flags=0):
-        return commit(self.fhdl, flags)
+        """Commit changes to the KVS
+
+        When keys are added, removed, or updated in the KVSDir object, the
+        changes are only cached in memory until the commit method asks the
+        KVS service to make them permanent. The commit method only includes
+        keys that have been explicitly updated in the KVSDir object, and the
+        contents of the KVS directory may diverge from the contents of the
+        KVSDir object if other changes are being made to the directory
+        concurrently.
+
+        After the commit method returns, updated keys can be accessed by other
+        clients on the same broker rank. Other broker ranks are eventually
+        consistent.
+        """
+        try:
+            self.kvstxn.commit(flags=flags)
+        except OSError:
+            raise
+        finally:
+            self.kvstxn.clear()
+
+    def commit_async(self, flags=0):
+        """Commit changes to the KVS.  Identical to commit(), but returns a
+        Future to wait on for RPC to complete.
+        """
+        future = self.kvstxn.commit_async(flags=flags)
+        self.kvstxn.clear()
+        return Future(future)
 
     def key_at(self, key):
+        """Get full path to KVS key"""
         p_str = self.pimpl.key_at(key)
         return p_str.decode("utf-8")
 
     def exists(self, name):
-        return exists(self.fhdl, name)
+        """Evaluate if key exists in the basedir"""
+        return exists(self.fhdl, self._path + name, namespace=self.namespace)
 
     def __getitem__(self, key):
+        # it is common for users to do something like
+        #
+        # with flux.kvs.get_dir(self.f) as kd:
+        #    kd["dir"]["subdir"]["subsubdir"]["a"] = 1
+        #
+        # the ability to get KVSDir subdirectories via the item
+        # accessor requires us to share this KVS transaction with
+        # subdirs for the final commit.  So if the user gets a KVSDir
+        # via item accessor, give it the same KVS transaction object
+        # by passing self.txn to get()..
+        #
+        # This also requires all updates to the KVSTxn object to be based
+        # on an initial path of "." in the transaction object.  i.e. use
+        # the "absolute path".  Otherwise, subdirs may write to the wrong
+        # relative location.
         try:
-            return get(self.fhdl, self.key_at(key))
+            val = get(
+                self.fhdl,
+                self.key_at(key),
+                namespace=self.namespace,
+                _kvstxn=self.kvstxn,
+            )
         except EnvironmentError:
             raise KeyError(
                 "{} not found under directory {}".format(key, self.key_at(""))
             )
+        return val
 
     def __setitem__(self, key, value):
-        if put(self.fhdl, key, value) < 0:
-            print("Error setting item in KVS")
+        # See note in __getitem__, we always write to "absolute" path
+        self.kvstxn.put(self._path + key, value)
 
     def __delitem__(self, key):
-        put_unlink(self.fhdl, key)
+        # See note in __getitem__, we always write to "absolute" path
+        self.kvstxn.unlink(self._path + key)
 
-    class KVSDirIterator(collectionsAbc.Iterator):
+    class KVSDirIterator(abc.Iterator):
         def __init__(self, kvsdir):
             super(KVSDir.KVSDirIterator, self).__init__()
             self.kvsdir = kvsdir
@@ -221,14 +662,14 @@ def __iter__(self):
     def __len__(self):
         return self.pimpl.get_size()
 
-    def fill(self, contents):
-        """
-        Populate this directory with keys specified by contents, which must
-        conform to the Mapping interface
+    def fill(self, contents: Mapping[str, Any]):
+        """Populate this directory with keys specified by contents
+
+        Args:
+            contents: A dict of keys and values to be created in the directory
+              or None, sub-directories can be created by using ``dir.file``
+              syntax, sub-dicts will be stored as json values in a single key
 
-        :param contents: A dict of keys and values to be created in the
-        directory or None, sub-directories can be created by using `dir.file`
-        syntax, sub-dicts will be stored as json values in a single key
         """
 
         if contents is None:
@@ -240,34 +681,38 @@ def fill(self, contents):
         finally:
             self.commit()
 
-    def mkdir(self, key, contents=None):
-        """
-        Create a new sub-directory, optionally pre-populated with the contents
-        of `files` as would be done with `fill(contents)`
+    def mkdir(self, key: str, contents: Mapping[str, Any] = None):
+        """Create a new sub-directory, optionally pre-populated by
+        contents, as would be done with ``fill(contents)``
 
-        :param key: Key of the directory to be created
-        :param contents: A dict of keys and values to be created in the
-          directory or None, sub-directories can be created by using `dir.file`
-          syntax, sub-dicts will be stored as json values in a single key
+        Args:
+            key: Key of the directory to be created
+            contents: A dict of keys and values to be created in the directory
+              or None, sub-directories can be created by using `dir.file`
+              syntax, sub-dicts will be stored as json values in a single key
         """
 
-        put_mkdir(self.fhdl, key)
+        # See note in __getitem__, we always write to "absolute" path
+        self.kvstxn.mkdir(self._path + key)
         self.commit()
-        new_kvsdir = KVSDir(self.fhdl, key)
         if contents is not None:
+            new_kvsdir = KVSDir(self.fhdl, key, namespace=self.namespace)
             new_kvsdir.fill(contents)
 
     def files(self):
+        """Get list of files in basedir"""
         for k in self:
             if not self.pimpl.isdir(k):
                 yield k
 
     def directories(self):
+        """Get list of directories in basedir"""
         for k in self:
             if self.pimpl.isdir(k):
                 yield k
 
     def list_all(self):
+        """Get tuple with list of files and directories in basedir"""
         files = []
         dirs = []
         for k in self:
@@ -290,27 +735,180 @@ def __exit__(self, type_arg, value, tb):
 
 
 def join(*args):
+    """Convenience function for use with walk(), similar to os.path.join()"""
     return ".".join([a for a in args if len(a) > 0])
 
 
-def inner_walk(kvsdir, curr_dir, topdown=False):
+def _inner_walk(kvsdir, curr_dir, topdown=False, namespace=None):
     if topdown:
         yield (curr_dir, kvsdir.directories(), kvsdir.files())
 
     for directory in kvsdir.directories():
         path = join(curr_dir, directory)
         key = kvsdir.key_at(directory)
-        for entry in inner_walk(get_dir(kvsdir.fhdl, key), path, topdown):
+        for entry in _inner_walk(
+            get_dir(kvsdir.fhdl, key, namespace=namespace),
+            path,
+            topdown,
+            namespace=namespace,
+        ):
             yield entry
 
     if not topdown:
         yield (curr_dir, kvsdir.directories(), kvsdir.files())
 
 
-def walk(directory, topdown=False, flux_handle=None):
-    """ Walk a directory in the style of os.walk() """
+def walk(directory, topdown=False, flux_handle=None, namespace=None):
+    """Walk a directory in the style of os.walk()
+
+    Args:
+        directory: A path or KVSDir object
+        topdown: Specify True for current directory to be
+          listed before subdirectories.
+        flux_handle: Required if "directory" is a path.
+        namespace: namespace to read from, defaults to None.  If namespace
+          is None, the namespace specified in the FLUX_KVS_NAMESPACE
+          environment variable will be used.  If FLUX_KVS_NAMESPACE is not
+          set, the primary namespace will be used.
+    """
     if not isinstance(directory, KVSDir):
         if flux_handle is None:
             raise ValueError("If directory is a key, flux_handle must be specified")
-        directory = KVSDir(flux_handle, directory)
-    return inner_walk(directory, "", topdown)
+        directory = KVSDir(flux_handle, directory, namespace=namespace)
+    return _inner_walk(directory, "", topdown, namespace=namespace)
+
+
+class WatchImplementation(Future, ABC):
+    """
+    Interface for KVS based watchers
+
+    Users to implement watch_get() and watch_cancel() functions.
+    """
+
+    def __del__(self):
+        if self.needs_cancel is not False:
+            self.cancel()
+        try:
+            super().__del__()
+        except AttributeError:
+            # not an error if super did not implement
+            pass
+
+    def __init__(self, future_handle):
+        super().__init__(future_handle)
+        self.needs_cancel = True
+
+    @abstractmethod
+    def watch_get(self, future):
+        pass
+
+    @abstractmethod
+    def watch_cancel(self, future):
+        pass
+
+    def get(self, autoreset=True):
+        """
+        Return the new value or None if the stream has terminated.
+
+        The future is auto-reset unless autoreset=False, so a subsequent
+        call to get() will try to fetch the next value and thus
+        may block.
+        """
+        try:
+            #  Block until Future is ready:
+            self.wait_for()
+            ret = self.watch_get(self.pimpl)
+        except OSError as exc:
+            if exc.errno == errno.ENODATA:
+                self.needs_cancel = False
+                return None
+            # raise handle exception if there is one
+            self.raise_if_handle_exception()
+            # re-raise all other exceptions
+            #
+            # Note: overwrite generic OSError strerror string with the
+            # EventWatch future error string to give the caller appropriate
+            # detail (e.g. instead of "No such file or directory" use
+            # "job <jobid> does not exist"
+            #
+            exc.strerror = self.error_string()
+            raise
+        if autoreset is True:
+            self.reset()
+        return ret
+
+    def cancel(self, stop=False):
+        """Cancel a streaming future
+
+        If stop=True, then deactivate the multi-response future so no
+        further callbacks are called.
+        """
+        self.watch_cancel(self.pimpl)
+        self.needs_cancel = False
+        if stop:
+            self.stop()
+
+
+class KVSWatchFuture(WatchImplementation):
+    """
+    A future returned from kvs_watch_async().
+    """
+
+    def __init__(self, future_handle):
+        super().__init__(future_handle)
+
+    def watch_get(self, future):
+        """
+        Implementation of watch_get() for KVSWatchFuture.
+
+        Will be called from WatchABC.get()
+        """
+        valp = ffi.new("char *[1]")
+        RAW.flux_kvs_lookup_get(future, valp)
+        return _get_value(valp)
+
+    def watch_cancel(self, future):
+        """
+        Implementation of watch_cancel() for KVSWatchFuture.
+
+        Will be called from WatchABC.cancel()
+        """
+        RAW.flux_kvs_lookup_cancel(future)
+
+
+def kvs_watch_async(
+    flux_handle, key, namespace=None, waitcreate=False, uniq=False, full=False
+):
+    """Asynchronously get KVS updates for a key
+
+    Args:
+        flux_handle: A Flux handle obtained from flux.Flux()
+        key: the key on which to watch
+        namespace: namespace to read from, defaults to None.  If namespace
+          is None, the namespace specified in the FLUX_KVS_NAMESPACE
+          environment variable will be used.  If FLUX_KVS_NAMESPACE is not
+          set, the primary namespace will be used.
+        waitcreate: If True and a key does not yet exist, will wait
+          for it to exit.  Defaults to False.
+        uniq: If True, only different values will be returned by
+          watch.  Defaults to False.
+        full: If True, any change that can affect the key is
+          monitored.  Typically, this is to capture when a parent directory
+          is removed or altered in some way.  Typically kvs watch will not
+          detect this as the exact key has not been changed.  Defaults to
+          False.
+
+    Returns:
+        KVSWatchFuture: a KVSWatchFuture object.  Call .get() from the then
+         callback to get the currently returned value from the Future object.
+    """
+
+    flags = flux.constants.FLUX_KVS_WATCH
+    if waitcreate:
+        flags |= flux.constants.FLUX_KVS_WAITCREATE
+    if uniq:
+        flags |= flux.constants.FLUX_KVS_WATCH_UNIQ
+    if full:
+        flags |= flux.constants.FLUX_KVS_WATCH_FULL
+    future = RAW.flux_kvs_lookup(flux_handle, namespace, flags, key)
+    return KVSWatchFuture(future)
diff --git a/src/bindings/python/flux/memoized_property.py b/src/bindings/python/flux/memoized_property.py
index 7172947456ec..50c335810a63 100644
--- a/src/bindings/python/flux/memoized_property.py
+++ b/src/bindings/python/flux/memoized_property.py
@@ -39,7 +39,8 @@ def memoized_property(fget):
     Return a property attribute for new-style classes that only calls its
     getter on the first access. The result is stored and on subsequent
     accesses is returned, preventing the need to call the getter any more.
-    Example::
+
+    Example:
         >>> class C(object):
         ...     load_name_count = 0
         ...     @memoized_property
diff --git a/src/bindings/python/flux/message.py b/src/bindings/python/flux/message.py
index 87d1b83d6cd0..33e562eb7ee8 100644
--- a/src/bindings/python/flux/message.py
+++ b/src/bindings/python/flux/message.py
@@ -11,31 +11,28 @@
 import errno
 import json
 
-import six
-
-from flux.wrapper import Wrapper, WrapperPimpl
+import flux.constants
 from flux.core.inner import ffi, lib, raw
 from flux.core.watchers import Watcher
-import flux.constants
 from flux.util import encode_payload, encode_topic
+from flux.wrapper import Wrapper, WrapperPimpl
 
 __all__ = ["Message", "MessageWatcher", "msg_typestr"]
 
 
 def msg_typestr(msg_type):
     # the returned string is guaranteed to be ascii
-    return ffi.string(raw.flux_msg_typestr(msg_type)).decode("ascii")
+    return raw.flux_msg_typestr(msg_type).decode("ascii")
 
 
 class Message(WrapperPimpl):
-    """ Flux message wrapper class. """
+    """Flux message wrapper class."""
 
     class InnerWrapper(Wrapper):
         def __init__(
             self,
             type_id=flux.constants.FLUX_MSGTYPE_REQUEST,
             handle=None,
-            destruct=False,
         ):
             super(Message.InnerWrapper, self).__init__(
                 ffi,
@@ -43,29 +40,23 @@ def __init__(
                 handle=handle,
                 match=ffi.typeof(lib.flux_msg_create).result,
                 prefixes=["flux_msg_", "FLUX_MSG"],
-                destructor=raw.flux_msg_destroy if destruct else None,
+                destructor=raw.flux_msg_destroy,
             )
-            self.destruct = destruct
             if handle is None:
                 self.handle = raw.flux_msg_create(type_id)
+            else:
+                #  Always take a reference on externally supplied flux_msg_t
+                #   so that it can't be destroyed while the Python
+                #   Message object is still active.
+                raw.flux_msg_incref(self.handle)
 
         def __del__(self):
-            if (
-                (not self.external or self.destruct)
-                and self.handle is not None
-                and self.handle != ffi.NULL
-            ):
-                raw.flux_msg_destroy(self.handle)
+            if self.handle is not None and self.handle != ffi.NULL:
+                raw.flux_msg_decref(self.handle)
 
-    def __init__(
-        self, type_id=flux.constants.FLUX_MSGTYPE_REQUEST, handle=None, destruct=False
-    ):
+    def __init__(self, type_id=flux.constants.FLUX_MSGTYPE_REQUEST, handle=None):
         super(Message, self).__init__()
-        self.pimpl = self.InnerWrapper(type_id, handle, destruct)
-
-    @property
-    def handle(self):
-        return self.pimpl.handle
+        self.pimpl = self.InnerWrapper(type_id, handle)
 
     @classmethod
     def from_event_encode(cls, topic, payload=None):
@@ -118,6 +109,37 @@ def type(self, value):
     def type_str(self):
         return msg_typestr(self.type)
 
+    def decode(self):
+        """Decode a message
+
+        Attempt to decode a message and return the message type, topic,
+        and payload string if successful.
+
+        Returns:
+            type: FLUX_MSGTYPE_REQUEST or FLUX_MSGTYPE_RESPONSE
+            topic: topic string
+            payload: payload string if one exists
+        Raises:
+            OSError: if message is an error response, raises OSError with
+                exception.errno set
+        """
+        topic_string = ffi.new("char *[1]")
+        string = ffi.new("char *[1]")
+        if self.type == flux.constants.FLUX_MSGTYPE_REQUEST:
+            raw.flux_request_decode(self.handle, topic_string, string)
+        elif self.type == flux.constants.FLUX_MSGTYPE_RESPONSE:
+            raw.flux_response_decode(self.handle, topic_string, string)
+        elif self.type == flux.constants.FLUX_MSGTYPE_EVENT:
+            raw.flux_event_decode(self.handle, topic_string, string)
+        payload_str = None
+        if string[0] != ffi.NULL:
+            payload_str = ffi.string(string[0]).decode("utf-8")
+        return (
+            self.type,
+            ffi.string(topic_string[0]).decode("utf-8"),
+            payload_str,
+        )
+
 
 # Residing here to avoid cyclic references
 
@@ -126,12 +148,17 @@ def type_str(self):
 def message_handler_wrapper(unused1, unused2, msg_handle, opaque_handle):
     del unused1, unused2  # unused arguments
     watcher = ffi.from_handle(opaque_handle)
-    watcher.callback(
-        watcher.flux_handle,
-        watcher,
-        Message(handle=msg_handle, destruct=False),
-        watcher.args,
-    )
+    try:
+        watcher.callback(
+            watcher.flux_handle,
+            watcher,
+            Message(handle=msg_handle),
+            watcher.args,
+        )
+    # pylint: disable=broad-except
+    except Exception as exc:
+        type(watcher.flux_handle).set_exception(exc)
+        watcher.flux_handle.reactor_stop_error()
 
 
 class MessageWatcher(Watcher):
@@ -142,18 +169,18 @@ def __init__(
         callback,
         topic_glob="*",
         match_tag=flux.constants.FLUX_MATCHTAG_NONE,
+        rolemask=None,
         args=None,
     ):
-        self.flux_handle = flux_handle
         self.callback = callback
         self.args = args
         self.wargs = ffi.new_handle(self)
 
         if topic_glob is None or topic_glob == ffi.NULL:
             topic_glob = ffi.NULL
-        elif isinstance(topic_glob, six.text_type):
+        elif isinstance(topic_glob, str):
             topic_glob = topic_glob.encode("UTF-8")
-        elif not isinstance(topic_glob, six.binary_type):
+        elif not isinstance(topic_glob, bytes):
             errmsg = "Topic must be a string, not {}".format(type(topic_glob))
             raise TypeError(errno.EINVAL, errmsg)
         c_topic_glob = ffi.new("char[]", topic_glob)
@@ -163,13 +190,16 @@ def __init__(
             {"typemask": type_mask, "matchtag": match_tag, "topic_glob": c_topic_glob},
         )
         super(MessageWatcher, self).__init__(
+            flux_handle,
             raw.flux_msg_handler_create(
-                self.flux_handle.handle,
+                flux_handle.handle,
                 match[0],
                 lib.message_handler_wrapper,
                 self.wargs,
-            )
+            ),
         )
+        if rolemask:
+            flux_handle.msg_handler_allow_rolemask(self.handle, rolemask)
 
     def start(self):
         raw.flux_msg_handler_start(self.handle)
diff --git a/src/bindings/python/flux/progress.py b/src/bindings/python/flux/progress.py
new file mode 100644
index 000000000000..be0811d47fe7
--- /dev/null
+++ b/src/bindings/python/flux/progress.py
@@ -0,0 +1,331 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import atexit
+import datetime
+import shutil
+import sys
+import time
+
+# Bottombar heavily borrows from
+#
+#  https://github.com/evalf/bottombar/blob/master/bottombar.py
+#
+# Copyright (c) 2020 Evalf
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+class ElapsedTime(float):
+    """
+    An ElapsedTime object is a floating point elapsed time in seconds
+    that comes with a convenient "dt" property that returns a
+    datetime.timedelta object
+    """
+
+    @property
+    # pylint: disable=invalid-name
+    def dt(self):
+        return datetime.timedelta(seconds=round(self))
+
+
+class Bottombar:
+    """Maintain a status line at bottom of terminal using vt100 escape codes
+
+    The Bottombar class implements a very simple status line which stays
+    positioned at the last line of vt100 capable terminals through the
+    use of vt100 escape codes.
+
+    This class will only work properly on vt100 compatible terminals, which
+    includes xterm, rxvt, and gnome-terminal and their derivatives on Linux,
+    as well as iTerm and Terminal on OSX, and reportedly the new Windows
+    Terminal on Windows.
+
+    Use of Bottombar requires that a ``formatter`` function be provided.
+    The ``formatter`` will be called on each update as::
+
+        formatter(bbar, width)
+
+    Where ``bbar`` is the bottombar object being formatted and ``width`` is
+    the current terminal width at the time of the update. The default
+    ``formatter`` will simply print all extra
+
+    As a convenience, the Bottombar constructor collects all extra keyword
+    arguments and presents them as attributes on the Bottombar object for
+    later access from within and outside the provided ``formatter``, e.g::
+
+        def formatter(bb, width):
+            text = f"iteration={bb.i}"
+            return text + time.ctime().rjust(width - len(text))
+
+        bb = Bottombar(formatter, i=0).start()
+        for i in range(0, 128):
+            bb.update(i=i)
+            time.sleep(.05)
+        bb.stop()
+
+    will print a statusbar with an iteration count left justified, and the
+    current time right justified.
+
+    Attributes:
+        elapsed (float): The elapsed time since ``bb.start()`` in floating
+            point seconds. As a convenience, ``bb.elapsed`` may be converted
+            to a ``datetime.timedelta`` object via the ``dt`` attribute,
+            e.g. ``bb.elapsed.dt``.
+
+    Args:
+        formatter (function): Function which returns the status string
+        kwargs: all extra keyword arguments are collected in the Bottombar
+            instance and made available as attributes for convenience
+
+    """
+
+    def __init__(self, formatter=None, **kwargs):
+        self.size = None
+        if formatter is None:
+            formatter = self._format
+        self.formatter = formatter
+        self.kwargs = kwargs
+        self._running = False
+        self._t0 = None
+
+    def __getattr__(self, attr):
+        if attr == "elapsed":
+            return ElapsedTime(time.time() - self._t0)
+        return self.kwargs[attr]
+
+    def _format(self, _bbar, _width):
+        return " ".join([f"{key}={val}" for key, val in self.kwargs.items()])
+
+    def __str__(self):
+        return self.formatter(self, self.size.columns)
+
+    def _setup_terminal(self, size=None):
+        """
+        Reset terminal scroll region and save last line for progressbar
+        """
+        if size:
+            self.size = size
+        sys.stdout.write(
+            "\0337"  # save cursor and attributes
+            "\033[r"  # reset scroll region (moves cursor)
+            "\0338"  # restore cursor and attributes
+            "\033D"  # move/scroll down
+            "\033M"  # move up
+            "\0337"  # save cursor and attributes
+            "\033[1;%dr"  # set scroll region to lines - 1
+            "\0338" % (self.size.lines - 1)  # restore cursor and attributes
+        )
+
+    def _reset_terminal(self):
+        """Reset terminal after use of progress bar
+
+        Reset terminal scroll region, print final version of progress bar.
+        Print newline.
+        """
+        sys.stdout.write(
+            "\0337"  # save cursor position
+            "\033[%d;1H"  # move cursor to bottom row, first column
+            "\033[K"  # clear entire line
+            "\033[r"  # reset scroll region
+            "\0338"  # restore cursor position
+            "%s\n" % (self.size.lines, self)  # print final bar
+        )
+        atexit.unregister(self._reset_terminal)
+
+    def redraw(self):
+        """Redraw bar without update"""
+        size = shutil.get_terminal_size()
+        if self.size != size:
+            self._setup_terminal(size)
+        sys.stdout.write(
+            "\0337"  # save cursor and attributes
+            "\033[%d;1H"  # move cursor to bottom row, first column
+            "\033[2K"  # clear entire line
+            "\033[?7l"  # disable line wrap
+            "\033[0m%s"  # clear attributes, print bar
+            "\033[?7h"  # enable line wrap
+            "\0338" % (self.size.lines, self)
+        )
+        sys.stdout.flush()
+
+    def start(self):
+        """Start drawing a Bottombar"""
+        self._running = True
+        if self._t0 is None:
+            self._t0 = time.time()
+        self.redraw()
+        atexit.register(self._reset_terminal)
+        return self
+
+    def stop(self):
+        """Reset terminal and write final bottombar state with newline"""
+        if self._running:
+            self._reset_terminal()
+            self._running = False
+
+    def update(self, **kwargs):
+        """Update keyword args and redraw a bottombar"""
+        self.kwargs.update(kwargs)
+        if self._running:
+            self.redraw()
+
+
+class ProgressBar(Bottombar):
+    """Simple progress bar that stays on last line of terminal
+
+    The ProgressBar class uses the features of Bottombar to create a
+    progress bar, plus optional other text, which stays on the last line
+    of a terminal. A vt100 compatible terminal is required.
+
+    Args:
+        total (int): The total expected number of items/units for which
+                     the progressbar is monitoring progress, default=100.
+        style (str, optional): A string progress bar style from the list
+                               "line", "bar", "dots", "steps", "vertbars".
+        before (str, optional): A string to place before the progress bar.
+        after  (str, optional): A string to place after the progress bar.
+                                default=" {percent:5.1f}%"
+        autostop (bool, optional): If True, ProgressBar instance will be
+                                   automatically stopped when count == total.
+                                   Otherwise, terminal reset will be deferred
+                                   to an atexit handler.
+        kwargs (optional): Extra keyword args are saved and passed as args
+                           when formatting the ``before`` and ``after``
+                           strings.
+
+    The ``before`` and ``after`` strings are formatted on each update to
+    the progressbar and passed all extra keyword args, plus the current
+    ``total``, ``count``, ``percent``, and ``elapsed`` time e.g.::
+
+        before_str = before.format(
+                        total=total,
+                        count=count,
+                        percent=percent,
+                        elapsed=elapsed,
+                        **kwargs
+                     )
+
+    which means that these strings are most useful when they are format
+    strings, e.g.::
+
+        ProgressBar(before="Running {total} jobs, {percent}% complete: ")
+
+    """
+
+    bar_style = {
+        "line": "─━",
+        "bar": "─█",
+        "dots": "⣀⣄⣤⣦⣶⣷⣿",
+        "steps": " ▁▂▃▄▅▆▇█",
+        "vertbars": " ▏▎▍▌▋▊▉█",
+    }
+
+    def __init__(
+        self,
+        total=100,
+        style="vertbars",
+        before="",
+        after=" {percent:5.1f}%",
+        autostop=False,
+        **kwargs,
+    ):
+        super().__init__(self._formatter, **kwargs)
+        self.count = 0
+        self.total = total
+        self.before = before
+        self.after = after
+        self.style = self.bar_style[style]
+        self.autostop = autostop
+
+    def __getattr__(self, attr):
+        if attr == "total":
+            return self.total
+        if attr == "count":
+            return self.count
+        if attr == "elapsed":
+            return super().__getattr__(attr)
+        return self.kwargs[attr]
+
+    def _formatter(self, bbar, width):
+        style = self.style
+        fraction = float(self.count / self.total) if self.total else 0
+        percent = 100 * fraction
+
+        #  Format before/after strings:
+        before = self.before.format(
+            total=self.total,
+            count=self.count,
+            elapsed=self.elapsed,
+            percent=percent,
+            **bbar.kwargs,
+        )
+        after = self.after.format(
+            total=self.total,
+            count=self.count,
+            elapsed=self.elapsed,
+            percent=percent,
+            **bbar.kwargs,
+        )
+
+        #  Calculate remaining length for progress bar:
+        length = width - len(before) - len(after) - 2
+
+        #  Calculate amount of bar fully filled and build string
+        fraction = min(fraction, 1.0)
+        fill = int(length * fraction)
+        filled = style[-1] * fill
+
+        #  Now calculate partial fill and append it to filled block
+        part = int(length * len(style) * fraction) - fill * len(style)
+        if part:
+            fill += 1
+            if part >= len(style):
+                part = -1
+            filled += style[part]
+
+        #  Build the bar string as 'filled':
+        filled = "\u2502" + filled + style[0] * (length - fill) + "\u2502"
+        return f"{before}{filled}{after}"
+
+    def update(self, advance=1, **kwargs):
+        """Update the state of a ProgressBar
+
+        Update the state of a ProgressBar and redraw if the progress bar is
+        currently running. If ``count == total`` and autostop is set, the
+        progress bar will be automatically stopped.
+
+        When the progress bar is stopped, the terminal will be reset and
+        the final state of the bar will be left on the last line of output.
+
+        Args:
+            advance (int, optional): Advance progress by ``advance`` amount.
+            kwargs: Update stored keyword arguments
+        """
+        self.count += advance
+        super().update(**kwargs)
+        if self.count == self.total and self.autostop:
+            self.stop()
diff --git a/src/bindings/python/flux/resource/ResourceSet.py b/src/bindings/python/flux/resource/ResourceSet.py
new file mode 100644
index 000000000000..c1c871c28b6e
--- /dev/null
+++ b/src/bindings/python/flux/resource/ResourceSet.py
@@ -0,0 +1,276 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import json
+from collections.abc import Mapping
+
+from flux.hostlist import Hostlist
+from flux.idset import IDset
+from flux.resource import Rlist
+from flux.resource.ResourceSetImplementation import ResourceSetImplementation
+
+
+# pylint: disable=too-many-public-methods
+class ResourceSet:
+    """
+    ResourceSet object constructor.
+
+    :param arg: Argument from which to construct a ResourceSet. `arg`
+                may be a serialized R string, a decoded Mapping of
+                an R string, or a valid ResourceSet implementation
+                (an instance of ResourceSetImplementation)
+
+    :param version: R specification version
+
+    :raises TypeError: A ResourceSet cannot be instantiated from arg
+    :raises ValueError: Invalid R version, or invalid R encoding
+    :raises KeyError: arg was a dict without a 'version' key
+    :raises json.decoder.JSONDecodeError: `arg` is an Invalid JSON string
+
+    All parameters are optional. ResourceSet() will initialize an
+    empty, version 1 ResourceSet object.
+    """
+
+    def __init__(self, arg=None, version=1):
+        self._state = None
+
+        if isinstance(arg, ResourceSetImplementation):
+            #  If argument is a resource set implementation, instantiate
+            #   from that and return immediately
+            self.impl = arg
+            self.version = arg.version
+            return
+
+        if isinstance(arg, str):
+            #  If arg is a string, assume an encoded R representation.
+            #  decode to a mapping:
+            arg = json.loads(arg)
+
+        if isinstance(arg, Mapping):
+            #  If argument is a mapping, grab version field for later use
+            version = arg["version"]
+        elif arg is not None:
+            # arg must be ResourceSetImplementation, Mapping, string or None:
+            tstr = type(arg)
+            raise TypeError(f"ResourceSet cannot be instantiated from {tstr}")
+
+        #  Instantiate implementation from version
+        #  note: only version 1 supported for now
+        if version == 1:
+            self.version = 1
+            self.impl = Rlist(arg)
+        else:
+            raise ValueError(f"R version {version} not supported")
+
+    def __str__(self):
+        return self.dumps()
+
+    def __and__(self, arg):
+        return self.intersect(arg)
+
+    def __sub__(self, arg):
+        return self.diff(arg)
+
+    def __or__(self, arg):
+        return self.union(arg)
+
+    def dumps(self):
+        """Return a short-form, human-readable string of a ResourceSet object"""
+        return self.impl.dumps()
+
+    def encode(self):
+        """Encode a ResourceSet object to its serialized string representation"""
+        return self.impl.encode()
+
+    def count(self, name):
+        """
+        Return a count of resource objects within a ResourceSet
+
+        :param name: The name of the object to count, e.g. "core"
+        """
+        return self.impl.count(name)
+
+    def append(self, *args):
+        """Append a ResourceSet to another"""
+        for arg in args:
+            if not isinstance(arg, ResourceSet):
+                arg = ResourceSet(arg, version=self.version)
+            self.impl.append(arg.impl)
+        return self
+
+    def add(self, *args):
+        """
+        Add resources to a ResourceSet that are not already members
+        """
+        for arg in args:
+            if not isinstance(arg, ResourceSet):
+                arg = ResourceSet(arg, version=self.version)
+            self.impl.add(arg.impl)
+        return self
+
+    def copy(self):
+        """Return a copy of a ResourceSet"""
+        rset = ResourceSet(self.impl.copy())
+        rset.state = self.state
+        return rset
+
+    def _run_op(self, method, *args):
+        result = self.copy()
+        for arg in args:
+            if not isinstance(arg, ResourceSet):
+                arg = ResourceSet(arg, version=self.version)
+            impl = getattr(result.impl, method)(arg.impl)
+            result = ResourceSet(impl)
+        result.state = self.state
+        return result
+
+    def union(self, *args):
+        """
+        Return a new ResourceSet with elements from this set and all others.
+
+        Equivalent to ``set | other | ...``.
+        """
+        return self._run_op("union", *args)
+
+    def diff(self, *args):
+        """
+        Return a new ResourceSet with elements in this set that are not in the others.
+
+        Equivalent to ``set - other - ...``.
+        """
+        return self._run_op("diff", *args)
+
+    def intersect(self, *args):
+        """
+        Return a new ResourceSet with elements common to this set and all others.
+
+        Equivalent to ``set & other & ...``.
+        """
+        return self._run_op("intersect", *args)
+
+    def copy_constraint(self, constraint):
+        """
+        Return a copy of a ResourceSet containing only those resources that
+        match the RFC 31 constraint object `constraint`
+
+        :param constraint: An RFC 31 constraint object in encoded string
+                           form or as Python mapping. (The mapping will be
+                           converted to a JSON string)
+        """
+        return ResourceSet(self.impl.copy_constraint(constraint))
+
+    def set_property(self, name, ranks=None):
+        """
+        Set property 'name' on optional 'ranks' (all ranks if ranks is None)
+        """
+        if ranks is None:
+            ranks = str(self.ranks)
+        self.impl.set_property(name, ranks)
+        return self
+
+    def get_properties(self):
+        """
+        Return an RFC 20 properties object for this ResourceSet
+        """
+        return self.impl.get_properties()
+
+    def remove_ranks(self, ranks):
+        """
+        Remove the rank or ranks specified from the ResourceSet
+
+        :param ranks: A flux.idset.IDset object, or number or string which
+                      can be converted into an IDset, containing the ranks
+                      to remove
+        """
+        if not isinstance(ranks, IDset):
+            ranks = IDset(ranks)
+        self.impl.remove_ranks(ranks)
+        return self
+
+    def copy_ranks(self, ranks):
+        """
+        Copy only the rank or ranks specified from the ResourceSet
+
+        :param ranks: A flux.idset.IDset object, or number or string which
+                      can be converted into an IDset, containing the ranks
+                      to copy
+        """
+        if not isinstance(ranks, IDset):
+            ranks = IDset(ranks)
+        rset = ResourceSet(self.impl.copy_ranks(ranks))
+
+        #  Preserve current state
+        rset.state = self.state
+        return rset
+
+    def host_ranks(self, hosts, ignore_nomatch=False):
+        """
+        Translate a set of hostnames to broker ranks using the current
+        ResourceSet.
+
+        Args:
+            ignore_nomatch (bool): If True, then hosts that are not in
+                the current ResourceSet are ignored, and only matching
+                hosts result in a returned rank. O/w, FileNotFound error
+                 is raised.
+        Returns:
+            list of rank ids in order of provided hosts
+        """
+        if not isinstance(hosts, Hostlist):
+            hosts = Hostlist(hosts)
+        ranks = list(self.ranks)
+        index = self.nodelist.index(hosts, ignore_nomatch=ignore_nomatch)
+        return [ranks[i] for i in index]
+
+    @property
+    def nodelist(self):
+        """
+        Return a flux.hostlist.Hostlist containing the list of hosts in
+        this ResourceSet
+        """
+        return self.impl.nodelist()
+
+    @property
+    def state(self):
+        """An optional state associated with this ResourceSet (e.g. "up")"""
+        return self._state
+
+    @state.setter
+    def state(self, value):
+        """Set an optional state for this ResourceSet"""
+        self._state = value
+
+    @property
+    def ranks(self):
+        """
+        Return a flux.idset.IDset containing the set of ranks in this
+        ResourceSet
+        """
+        return self.impl.ranks()
+
+    @property
+    def nnodes(self):
+        return self.impl.nnodes()
+
+    @property
+    def ncores(self):
+        return self.impl.count("core")
+
+    @property
+    def ngpus(self):
+        return self.impl.count("gpu")
+
+    @property
+    def rlist(self):
+        return self.impl.dumps()
+
+    @property
+    def properties(self):
+        return ",".join(json.loads(self.get_properties()).keys())
diff --git a/src/bindings/python/flux/resource/ResourceSetImplementation.py b/src/bindings/python/flux/resource/ResourceSetImplementation.py
new file mode 100644
index 000000000000..41c400a57434
--- /dev/null
+++ b/src/bindings/python/flux/resource/ResourceSetImplementation.py
@@ -0,0 +1,92 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+from abc import ABC, abstractmethod
+
+
+class ResourceSetImplementation(ABC):  # pragma: no cover
+    """
+    This abstract class defines the interface that a ResourceSet
+    implementation shall provide in order to work with the ResourceSet
+    class
+    """
+
+    @abstractmethod
+    def dumps(self):
+        """Return a short-form string representation of a resource set"""
+        raise NotImplementedError
+
+    def encode(self):
+        """Return a JSON string representation of the resource set"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def nodelist(self):
+        """Return the list of nodes in the resource set as a Hostlist"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def ranks(self):
+        """Return the set of ranks in the resource set as an IDset"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_properties(self):
+        """Return an RFC 20 properties object for this resource set"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def nnodes(self):
+        """Return the number of nodes in the resource set as an IDset"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def count(self, name):
+        """Return the total number of resources of type 'name'"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def copy(self):
+        """Return a copy of the resource set"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def copy_ranks(self, ranks):
+        """Return a copy of resource set with only 'ranks' included"""
+
+    @abstractmethod
+    def union(self, rset):
+        """Return the union of two resource sets"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def intersect(self, rset):
+        """Return the set intersection of two resource sets"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def diff(self, rset):
+        """Return the set difference of two resource sets"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def append(self, rset):
+        """Append one resource set to another"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def add(self, rset):
+        """Add resources not existing in one set to the other"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def remove_ranks(self, ranks):
+        """Remove an IDset of ranks from a resource set"""
+        raise NotImplementedError
diff --git a/src/bindings/python/flux/resource/Rlist.py b/src/bindings/python/flux/resource/Rlist.py
new file mode 100644
index 000000000000..7c5b4d73abe2
--- /dev/null
+++ b/src/bindings/python/flux/resource/Rlist.py
@@ -0,0 +1,145 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import json
+import socket
+from collections.abc import Mapping
+
+from _flux._rlist import ffi, lib
+from flux.hostlist import Hostlist
+from flux.idset import IDset
+from flux.resource.ResourceSetImplementation import ResourceSetImplementation
+from flux.wrapper import Wrapper, WrapperPimpl
+
+
+@ResourceSetImplementation.register
+class Rlist(WrapperPimpl):
+    version = 1
+
+    class InnerWrapper(Wrapper):
+        def __init__(self, rstring=None, handle=None):
+
+            if handle is None:
+                if rstring is None:
+                    handle = lib.rlist_create()
+                else:
+                    if isinstance(rstring, Mapping):
+                        rstring = json.dumps(rstring)
+                    handle = lib.rlist_from_R(rstring.encode("utf-8"))
+            if handle == ffi.NULL:
+                raise ValueError(f"Rlist: invalid argument")
+            super().__init__(
+                ffi,
+                lib,
+                match=ffi.typeof("struct rlist *"),
+                prefixes=["rlist_"],
+                destructor=lib.rlist_destroy,
+                handle=handle,
+            )
+
+    def __init__(self, rstring=None, handle=None):
+        super().__init__()
+        self.pimpl = self.InnerWrapper(rstring, handle)
+
+    def dumps(self):
+        val = lib.rlist_dumps(self.handle)
+        result = ffi.string(val).decode("utf-8")
+        lib.free(val)
+        return result
+
+    def encode(self):
+        val = lib.rlist_encode(self.handle)
+        result = ffi.string(val).decode("utf-8")
+        lib.free(val)
+        return result
+
+    def nodelist(self):
+        return Hostlist(handle=self.pimpl.nodelist())
+
+    def get_properties(self):
+        val = lib.rlist_properties_encode(self.handle)
+        result = ffi.string(val).decode("utf-8")
+        lib.free(val)
+        return result
+
+    def ranks(self, hosts=None):
+        if hosts is None:
+            return IDset(handle=self.pimpl.ranks())
+        return IDset(handle=self.pimpl.hosts_to_ranks(hosts))
+
+    def nnodes(self):
+        return self.pimpl.nnodes()
+
+    def count(self, name):
+        return self.pimpl.count(name)
+
+    def remap(self):
+        self.pimpl.remap()
+        return self
+
+    def append(self, arg):
+        self.pimpl.append(arg)
+        return self
+
+    def add(self, arg):
+        self.pimpl.add(arg)
+        return self
+
+    def copy(self):
+        return Rlist(handle=self.pimpl.copy_empty())
+
+    def union(self, arg):
+        return Rlist(handle=self.pimpl.union(arg))
+
+    def intersect(self, arg):
+        return Rlist(handle=self.pimpl.intersect(arg))
+
+    def diff(self, arg):
+        return Rlist(handle=self.pimpl.diff(arg))
+
+    def add_rank(self, rank, hostname=None, cores="0"):
+        if hostname is None:
+            hostname = socket.gethostname()
+        self.pimpl.append_rank_cores(hostname, rank, cores)
+        return self
+
+    def remove_ranks(self, ranks):
+        if not isinstance(ranks, IDset):
+            ranks = IDset(ranks)
+        self.pimpl.remove_ranks(ranks)
+        return self
+
+    def copy_ranks(self, ranks):
+        return Rlist(handle=self.pimpl.copy_ranks(ranks))
+
+    def add_child(self, rank, name, ids):
+        self.pimpl.rank_add_child(rank, name, ids)
+        return self
+
+    def set_property(self, name, ranks):
+        error = ffi.new("flux_error_t *")
+        try:
+            self.pimpl.add_property(error, name, ranks)
+        except OSError as exc:
+            raise ValueError(
+                "set_property: " + ffi.string(error.text).decode("utf-8")
+            ) from exc
+
+    def copy_constraint(self, constraint):
+        error = ffi.new("flux_error_t *")
+        if not isinstance(constraint, str):
+            constraint = json.dumps(constraint)
+        try:
+            handle = self.pimpl.copy_constraint_string(constraint, error)
+        except OSError as exc:
+            raise ValueError(
+                "copy_constraint: " + ffi.string(error.text).decode("utf-8")
+            ) from exc
+        return Rlist(handle=handle)
diff --git a/src/bindings/python/flux/resource/__init__.py b/src/bindings/python/flux/resource/__init__.py
new file mode 100644
index 000000000000..95a4be8cb116
--- /dev/null
+++ b/src/bindings/python/flux/resource/__init__.py
@@ -0,0 +1,14 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+from flux.resource.Rlist import Rlist
+from flux.resource.ResourceSet import ResourceSet
+from flux.resource.list import resource_list, SchedResourceList
+from flux.resource.status import resource_status, ResourceStatus
diff --git a/src/bindings/python/flux/resource/list.py b/src/bindings/python/flux/resource/list.py
new file mode 100644
index 000000000000..c543f377d25f
--- /dev/null
+++ b/src/bindings/python/flux/resource/list.py
@@ -0,0 +1,128 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import errno
+import json
+import os
+
+from flux.future import FutureExt
+from flux.idset import IDset
+from flux.memoized_property import memoized_property
+from flux.resource import ResourceSet
+
+
+class SchedResourceList:
+    """
+    Encapsulate response from resource.sched-status query.
+    The response will contain 3 Rv1 resource sets:
+
+    :ivar all:         all resources known to scheduler
+    :ivar down:        resources currently unavailable (drained or down)
+    :ivar allocated:   resources currently allocated to jobs
+
+    From these sets, the "up" and "free" resource sets are
+    computed on-demand.
+
+    There is generally no need to instantiate this class directly. Instead,
+    instances are returned by fetching the result of a ``resource_list()`` call.
+    """
+
+    def __init__(self, resp):
+        for state in ["all", "down", "allocated"]:
+            rset = ResourceSet(resp.get(state))
+            rset.state = state
+            setattr(self, f"_{state}", rset)
+
+    def __getattr__(self, attr):
+        if attr.startswith("_"):
+            raise AttributeError
+        try:
+            return getattr(self, f"_{attr}")
+        except KeyError:
+            raise AttributeError(f"Invalid SchedResourceList attr {attr}")
+
+    #  Make class subscriptable, e.g. resources[state]
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def filter(self, include):
+        """
+        Filter the reported resources in a ResourceList object
+        Args:
+            include(str, IDset, Hostlist): restrict the current set of
+                reported ranks to the given ranks or hosts.
+        """
+        try:
+            include_ranks = IDset(include)
+        except ValueError:
+            include_ranks = IDset(self["all"].host_ranks(include, ignore_nomatch=True))
+        for state in ["all", "down", "allocated"]:
+            setattr(self, state, self[state].copy_ranks(include_ranks))
+
+    @memoized_property
+    # pylint: disable=invalid-name
+    def up(self):
+        """All resources which are not down."""
+        res = self.all - self.down
+        res.state = "up"
+        return res
+
+    @memoized_property
+    def free(self):
+        """All resources which are neither down nor allocated."""
+        res = self.up - self.allocated
+        res.state = "free"
+        return res
+
+
+class ResourceListRPC(FutureExt):
+    def __init__(self, flux_handle, topic, nodeid=0):
+        self.topic = topic
+        super().__init__(self._init_cb, flux_handle=flux_handle)
+
+    def _init_cb(self, future):
+        future.get_flux().rpc(self.topic, nodeid=0).then(self._list_cb)
+
+    def _list_cb(self, future):
+        try:
+            self.fulfill(future.get())
+        except Exception as exc:
+            if exc.errno == errno.ENOSYS:
+                #  Fall back to sched.resource-status:
+                future.get_flux().rpc("sched.resource-status", nodeid=0).then(
+                    self._fallback_cb
+                )
+            else:
+                self.fulfill_error(exc.errno, exc.strerror)
+
+    def _fallback_cb(self, future):
+        try:
+            self.fulfill(json.loads(future.get_str()))
+        except OSError as exc:
+            self.fulfill_error(exc.errno, exc.strerror)
+
+    def get(self):
+        """Return a SchedResourceList corresponding to the request.
+
+        Blocks until the request is fulfilled."""
+        return SchedResourceList(super().get())
+
+
+def resource_list(flux_handle):
+    """Send a request for a SchedResourceList object.
+
+    Args:
+        flux_handle (flux.Flux): a Flux handle
+
+    Returns:
+        ResourceListRPC: a future representing the request.
+    """
+    topic = os.getenv("FLUX_RESOURCE_LIST_RPC") or "resource.sched-status"
+    return ResourceListRPC(flux_handle, topic, nodeid=0)
diff --git a/src/bindings/python/flux/resource/status.py b/src/bindings/python/flux/resource/status.py
new file mode 100644
index 000000000000..42039f51ff03
--- /dev/null
+++ b/src/bindings/python/flux/resource/status.py
@@ -0,0 +1,246 @@
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+from typing import NamedTuple
+
+from flux.idset import IDset
+from flux.resource import ResourceSet, resource_list
+from flux.rpc import RPC
+
+
+class DrainInfo(NamedTuple):
+    """
+    Drained resource information tuple.
+    Attributes:
+        timestamp (float): timestamp at which resource was drained
+        reason (str): message recorded when resources were drained
+        ranks (IDset): idset of drain ranks with this reason and timestamp
+
+    """
+
+    ranks: IDset
+    timestamp: float
+    reason: str
+
+
+class ResourceStatus:
+    """
+    Container for combined information from resource module and scheduler.
+
+    Attributes:
+        nodelist (Hostlist): rank ordered set of hostnames
+        all (IDset): idset of all known ranks
+        avail (IDset): idset of ranks not excluded, drained, or torpid
+        offline (IDset): idset of ranks currently offline
+        online (IDset): idset of ranks currently online
+        exclude (IDset): idset of ranks excluded by configuration
+        torpid (IDset): idset of torpid ranks
+        housekeeping (IDset): idset of ranks in housekeeping
+        allocated (IDset): idset of ranks with one or more jobs
+        drained (IDset): idset of ranks drained and not allocated
+        draining (IDset): idset of ranks drained and allocated
+        drain_info (list): list of DrainInfo object for drain ranks
+    """
+
+    def __init__(self, rstatus=None, allocated_ranks=None, housekeeping_ranks=None):
+        # Allow "empty" ResourceStatus object to be created:
+        if rstatus is None:
+            rstatus = dict(
+                R=None,
+                offline="",
+                online="",
+                exclude="",
+                torpid="",
+                housekeeping="",
+                drain={},
+            )
+        if allocated_ranks is None:
+            allocated_ranks = IDset()
+        if housekeeping_ranks is None:
+            housekeeping_ranks = IDset()
+
+        self.rstatus = rstatus
+        self.rset = ResourceSet(rstatus["R"])
+        self.nodelist = self.rset.nodelist
+        self.allocated_ranks = allocated_ranks
+        self.housekeeping_ranks = housekeeping_ranks
+
+        self._recalculate()
+
+    def filter(self, include):
+        """
+        Filter the reported resources in a ResourceStatus object
+        Args:
+            include (str, IDset, Hostlist): restrict the current set of
+             reported ranks to the given ranks or hosts.
+        """
+        try:
+            include_ranks = IDset(include)
+        except ValueError:
+            include_ranks = self.nodelist.index(include, ignore_nomatch=True)
+        self._recalculate(include_ranks)
+
+    def _recalculate(self, include_ranks=None):
+        """
+        Recalculate derived idsets and drain_info, only including ranks
+        in the IDset 'include_ranks' if given.
+        Args:
+            include_ranks (IDset): restrict the current set of reported ranks.
+        """
+        # get idset of all ranks:
+        self.all = self.rset.ranks
+
+        # offline/online
+        self.offline = IDset(self.rstatus["offline"])
+        self.online = IDset(self.rstatus["online"])
+
+        # excluded: excluded by configuration
+        self.exclude = IDset(self.rstatus["exclude"])
+
+        # torpid: unresponsive nodes
+        try:
+            self.torpid = IDset(self.rstatus["torpid"])
+        except KeyError:
+            # May be an older Flux version response missing "torpid" idset
+            self.torpid = IDset()
+
+        # allocated: online and allocated by scheduler
+        self.allocated = self.allocated_ranks
+
+        # housekeeping: ranks in housekeeping. May overlap with avail/allocated
+        self.housekeeping = self.housekeeping_ranks
+
+        # If include_ranks was provided, filter all idsets to only those
+        # that intersect the provided idset
+        if include_ranks is not None:
+            for name in ("all", "offline", "online", "exclude", "allocated", "torpid"):
+                result = getattr(self, name).intersect(include_ranks)
+                setattr(self, name, result)
+
+        # drained: free+drain
+        self.drained = IDset()
+        # draining: allocated+drain
+        self.draining = IDset()
+
+        # drain_info: ranks, timestamp, reason tuples for all drained resources
+        self.drain_info = []
+        self._drain_lookup = {}
+        for drain_ranks, entry in self.rstatus["drain"].items():
+            ranks = IDset(drain_ranks)
+            if include_ranks is not None:
+                ranks = ranks.intersect(include_ranks)
+            self.drained += ranks
+            info = DrainInfo(ranks, entry["timestamp"], entry["reason"])
+            self.drain_info.append(info)
+            for rank in ranks:
+                self._drain_lookup[rank] = info
+
+        # create the set of draining ranks as the intersection of
+        #  drained and allocated
+        self.draining = self.drained & self.allocated
+        self.drained -= self.draining
+
+        # available: all ranks not excluded, drained/draining, or currently
+        # marked as torpid
+        self.avail = self.all - self.get_idset(
+            "exclude", "drained", "draining", "torpid"
+        )
+
+    def __getitem__(self, state):
+        """
+        Allow a ResourceStatus object to be subscriptable for convenience,
+        e.g. self.drained == self["drained"]
+        """
+        return getattr(self, state)
+
+    def get_idset(self, *args):
+        """
+        Return an idset of ranks that are the union of all states in args
+        """
+        ids = IDset()
+        for state in args:
+            ids.add(self[state])
+        return ids
+
+    def get_drain_info(self, rank):
+        """
+        Find and return the DrainInfo object for rank ``rank``
+        """
+        if rank not in self.all:
+            raise ValueError("invalid rank {rank}")
+        return self._drain_lookup.get(rank)
+
+
+class ResourceStatusRPC:
+    """
+    A ResourceStatusRPC encapsulates a query to both the resource module
+    and scheduler and returns a ResourceStatus object.
+    """
+
+    def __init__(self, handle):
+        self.rpcs = [
+            RPC(handle, "resource.status", nodeid=0, flags=0),
+            resource_list(handle),
+            RPC(handle, "job-manager.stats-get", nodeid=0, flags=0),
+        ]
+        self.rlist = None
+        self.rstatus = None
+        self.allocated_ranks = None
+        self.housekeeping_ranks = None
+
+    def get_status(self):
+        if self.rstatus is None:
+            self.rstatus = self.rpcs[0].get()
+        return self.rstatus
+
+    def get_allocated_ranks(self):
+        if self.allocated_ranks is None:
+            try:
+                self.rlist = self.rpcs[1].get()
+                self.allocated_ranks = self.rlist.allocated.ranks
+            except EnvironmentError:
+                self.allocated_ranks = IDset()
+        return self.allocated_ranks
+
+    def get_housekeeping_ranks(self):
+        if self.housekeeping_ranks is None:
+            self.housekeeping_ranks = IDset()
+            try:
+                stats = self.rpcs[2].get()
+                for jobid, info in stats["housekeeping"]["running"].items():
+                    self.housekeeping_ranks.add(info["allocated"])
+            except EnvironmentError:
+                # use empty housekeeping_ranks from above
+                pass
+        return self.housekeeping_ranks
+
+    def get(self):
+        """
+        Return a ResourceStatus object corresponding to the request
+        Blocks until all RPCs are fulfilled
+        """
+        return ResourceStatus(
+            self.get_status(), self.get_allocated_ranks(), self.get_housekeeping_ranks()
+        )
+
+
+def resource_status(flux_handle):
+    """
+    Initiate RPCs to scheduler and resource module and return a ResourceStatus
+    object holding the result.
+
+    Args:
+        flux_handle (flux.Flux): a Flux handle
+
+    Returns:
+        ResourceStatusRPC: A future representing the request. Call .get()
+         to get the ResourceStatus result.
+    """
+    return ResourceStatusRPC(flux_handle)
diff --git a/src/bindings/python/flux/rpc.py b/src/bindings/python/flux/rpc.py
index b14c3cda590d..398bf1ead09d 100644
--- a/src/bindings/python/flux/rpc.py
+++ b/src/bindings/python/flux/rpc.py
@@ -10,16 +10,38 @@
 
 import json
 
-from flux.wrapper import Wrapper
-from flux.future import Future
-from flux.core.inner import ffi, raw
 import flux.constants
-from flux.util import encode_payload, encode_topic
+from flux.core.inner import ffi, lib, raw
+from flux.future import Future
+from flux.util import check_future_error, encode_payload, encode_topic, interruptible
+from flux.wrapper import Wrapper
 
 
 class RPC(Future):
     """An RPC state object"""
 
+    class RPCInnerWrapper(Wrapper):
+        def __init__(
+            self,
+            handle=None,
+            match=ffi.typeof("flux_future_t *"),
+            filter_match=True,
+            prefixes=None,
+            destructor=raw.flux_future_destroy,
+        ):
+            # avoid using a static list as a default argument
+            # pylint error 'dangerous-default-value'
+            if prefixes is None:
+                prefixes = ["flux_rpc_", "flux_future_"]
+
+            super().__init__(
+                ffi, lib, handle, match, filter_match, prefixes, destructor
+            )
+
+        def check_wrap(self, fun, name):
+            func = super().check_wrap(fun, name)
+            return check_future_error(func)
+
     def __init__(
         self,
         flux_handle,
@@ -37,11 +59,20 @@ def __init__(
         payload = encode_payload(payload)
 
         future_handle = raw.flux_rpc(flux_handle, topic, payload, nodeid, flags)
-        super(RPC, self).__init__(future_handle, prefixes=["flux_rpc_", "flux_future_"])
+        super(RPC, self).__init__(
+            future_handle,
+            prefixes=["flux_rpc_", "flux_future_"],
+            pimpl_t=self.RPCInnerWrapper,
+        )
 
+    @interruptible
     def get_str(self):
         payload_str = ffi.new("char *[1]")
-        self.pimpl.flux_rpc_get(payload_str)
+        try:
+            self.pimpl.flux_rpc_get(payload_str)
+        except OSError:
+            self.raise_if_handle_exception()
+            raise
         if payload_str[0] == ffi.NULL:
             return None
         return ffi.string(payload_str[0]).decode("utf-8")
diff --git a/src/bindings/python/flux/security.py b/src/bindings/python/flux/security.py
index 0dd9ecf31d91..b2ec73f7f979 100644
--- a/src/bindings/python/flux/security.py
+++ b/src/bindings/python/flux/security.py
@@ -8,21 +8,14 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-import six
-
 from _flux._security import ffi, lib
-from flux.wrapper import Wrapper, WrapperPimpl, FunctionWrapper
+from flux.wrapper import FunctionWrapper, Wrapper, WrapperPimpl
 
 
 class SecurityFunctionWrapper(FunctionWrapper):
-    def __init__(self, *args, **kwargs):
-        super(SecurityFunctionWrapper, self).__init__(*args, **kwargs)
-
     def __call__(self, calling_object, *args_in):
         try:
-            return super(SecurityFunctionWrapper, self).__call__(
-                calling_object, *args_in
-            )
+            return super().__call__(calling_object, *args_in)
         except EnvironmentError:
             errstr = calling_object.last_error()
             errnum = calling_object.last_errnum()
@@ -35,7 +28,7 @@ class SecurityContext(WrapperPimpl):
     class InnerWrapper(Wrapper):
         # pylint: disable=no-value-for-parameter
         def __init__(self, flags=0):
-            super(SecurityContext.InnerWrapper, self).__init__(
+            super().__init__(
                 ffi,
                 lib,
                 match=ffi.typeof("flux_security_t *"),
@@ -53,21 +46,29 @@ def check_wrap(self, fun, name):
                 fun, name, fun_type, self.ffi, add_handle=add_handle
             )
 
-    def __init__(self, config_pattern, flags=0):
-        super(SecurityContext, self).__init__()
+    def __init__(self, config_pattern=None, flags=0):
+        super().__init__()
         self.pimpl = self.InnerWrapper(flags)
         self.pimpl.configure(config_pattern)
 
+    def sign_wrap_as(self, userid, payload, mech_type=ffi.NULL, flags=0):
+        if isinstance(payload, str):
+            payload = payload.encode("utf-8", errors="surrogateescape")
+        elif not isinstance(payload, bytes):
+            errstr = "payload must be a text or binary type, not {}"
+            raise TypeError(errstr.format(type(payload)))
+        return self.pimpl.wrap_as(userid, payload, len(payload), mech_type, flags)
+
     def sign_wrap(self, payload, mech_type=ffi.NULL, flags=0):
-        if isinstance(payload, six.text_type):
+        if isinstance(payload, str):
             payload = payload.encode("utf-8")
-        elif not isinstance(payload, six.binary_type):
+        elif not isinstance(payload, bytes):
             errstr = "payload must be a text or binary type, not {}"
             raise TypeError(errstr.format(type(payload)))
         return self.pimpl.wrap(payload, len(payload), mech_type, flags)
 
     def sign_unwrap(self, signed_payload, flags=0):
-        if not isinstance(signed_payload, six.binary_type):
+        if not isinstance(signed_payload, bytes):
             errstr = "signed_payload must be a binary type, not {}"
             raise TypeError(errstr.format(type(signed_payload)))
 
diff --git a/src/bindings/python/flux/uri/__init__.py b/src/bindings/python/flux/uri/__init__.py
new file mode 100644
index 000000000000..205125ad9760
--- /dev/null
+++ b/src/bindings/python/flux/uri/__init__.py
@@ -0,0 +1,11 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+from flux.uri.uri import URI, FluxURIResolver, JobURI, URIResolverPlugin, URIResolverURI
diff --git a/src/bindings/python/flux/uri/resolvers/jobid.py b/src/bindings/python/flux/uri/resolvers/jobid.py
new file mode 100644
index 000000000000..580e38893b81
--- /dev/null
+++ b/src/bindings/python/flux/uri/resolvers/jobid.py
@@ -0,0 +1,88 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import os
+from pathlib import PurePath
+
+import flux
+from flux.job import JobID, job_list_id
+from flux.uri import JobURI, URIResolverPlugin, URIResolverURI
+
+
+def filter_slash(iterable):
+    return list(filter(lambda x: "/" not in x, iterable))
+
+
+def wait_for_uri(flux_handle, jobid):
+    """Wait for memo event containing job uri, O/w finish event"""
+    for event in flux.job.event_watch(flux_handle, jobid):
+        if event.name == "memo" and "uri" in event.context:
+            return event.context["uri"]
+        if event.name == "finish":
+            return None
+    return None
+
+
+class URIResolver(URIResolverPlugin):
+    """A URI resolver that attempts to fetch the remote_uri for a job"""
+
+    def describe(self):
+        return "Get URI for a given Flux JOBID"
+
+    def _do_resolve(self, uri, flux_handle, force_local=False, wait=False):
+        #
+        #  Convert a possible hierarchy of jobids to a list, dropping any
+        #   extraneous '/' (e.g. //id0/id1 -> [ "id0", "id1" ]
+        jobids = filter_slash(PurePath(uri.path).parts)
+
+        #  Pop the first jobid off the list, this id should be local:
+        arg = jobids.pop(0)
+        try:
+            jobid = JobID(arg)
+        except OSError as exc:
+            raise ValueError(f"{arg} is not a valid jobid")
+
+        try:
+            if wait:
+                uri = wait_for_uri(flux_handle, jobid)
+            else:
+                #  Fetch the jobinfo object for this job
+                job = job_list_id(
+                    flux_handle, jobid, attrs=["state", "annotations"]
+                ).get_jobinfo()
+                if job.state != "RUN":
+                    raise ValueError(f"jobid {arg} is not running")
+                uri = job.user.uri
+        except FileNotFoundError as exc:
+            raise ValueError(f"jobid {arg} not found") from exc
+
+        if uri is None or str(uri) == "":
+            raise ValueError(f"URI not found for job {arg}")
+
+        #  If there are more jobids in the hierarchy to resolve, resolve
+        #   them recursively
+        if jobids:
+            arg = "/".join(jobids)
+            resolver_uri = URIResolverURI(f"jobid:{arg}")
+            if force_local:
+                uri = JobURI(uri).local
+            return self._do_resolve(
+                resolver_uri, flux.Flux(uri), force_local=force_local
+            )
+        return uri
+
+    def resolve(self, uri):
+        force_local = False
+        wait = False
+        if "local" in uri.query_dict or "FLUX_URI_RESOLVE_LOCAL" in os.environ:
+            force_local = True
+        if "wait" in uri.query_dict:
+            wait = True
+        return self._do_resolve(uri, flux.Flux(), force_local=force_local, wait=wait)
diff --git a/src/bindings/python/flux/uri/resolvers/lsf.py b/src/bindings/python/flux/uri/resolvers/lsf.py
new file mode 100644
index 000000000000..3e893e761414
--- /dev/null
+++ b/src/bindings/python/flux/uri/resolvers/lsf.py
@@ -0,0 +1,124 @@
+###############################################################
+# Copyright 2022 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+"""
+This module supports the Flux URI resolver for IBM Spectrum LSF.
+
+It can be used like:
+  `flux uri lsf:<JOBID>`.
+
+"""
+
+import getpass
+import os
+import shutil
+import subprocess
+from pathlib import PurePath
+
+import flux.hostlist as hostlist
+import yaml
+from flux.uri import FluxURIResolver, URIResolverPlugin
+
+
+def lsf_find_compute_node(jobid):
+    """Figure out where the job is being run using YAML output from IBM Cluster Systems Manager."""
+
+    csm_path = os.getenv(
+        "CSM_ALLOCATION_QUERY", "/opt/ibm/csm/bin/csm_allocation_query"
+    )
+
+    sp = subprocess.run(
+        [csm_path, "-j", str(jobid)],
+        stdout=subprocess.PIPE,
+    )
+    try:
+        hosts = yaml.safe_load(sp.stdout.decode("utf-8"))["compute_nodes"]
+        return str(hostlist.decode(hosts).sort()[0])
+    except Exception as exc:
+        raise ValueError(
+            f"Unable to find a compute node attached to job {jobid}"
+        ) from exc
+
+
+def check_lsf_jobid(pid, jobid):
+    try:
+        with open(f"/proc/{pid}/environ", encoding="utf-8") as envfile:
+            if f"LSB_JOBID={jobid}" in envfile.read():
+                return True
+    except FileNotFoundError:
+        # if pid disappears while we try to read it, this is a False
+        pass
+    return False
+
+
+def lsf_job_pid(jobid):
+    """Resolve the pids of an LSF job using ssh to the compute node"""
+
+    sp = subprocess.run(
+        ["ps", "-Ho", "pid,comm", "-u", getpass.getuser()],
+        stdout=subprocess.PIPE,
+        check=True,
+    )
+    for line in sp.stdout.decode("utf-8").split("\n"):
+        if "flux-broker" in line:
+            pid = line.split()[0]
+            if check_lsf_jobid(pid, jobid):
+                return pid
+    raise ValueError(f"Unable to find a flux session attached to your job")
+
+
+def lsf_get_uri(hostname, jobid):
+    """
+    Resolve the URI of a Flux instance by running
+
+      `ssh <HOSTNAME> flux uri lsf:<PID>?is_compute`
+
+    and converting the result from bytecode to a string.
+    """
+    ssh_path = os.getenv("FLUX_SSH", "ssh")
+
+    #  Allow path to remote flux to be overridden as in ssh connector,
+    #   o/w, use path the current `flux` executable:
+    flux_cmd_path = os.getenv("FLUX_SSH_RCMD", shutil.which("flux"))
+    sp = subprocess.run(
+        [
+            ssh_path,
+            hostname,
+            flux_cmd_path,
+            "uri",
+            "--remote",
+            f"lsf:{jobid}?is_compute",
+        ],
+        stdout=subprocess.PIPE,
+        check=True,
+    )
+    return sp.stdout.decode("utf-8")
+
+
+class URIResolver(URIResolverPlugin):
+    """A URIResolver that can fetch a FLUX_URI from an LSF job id"""
+
+    def describe(self):
+        return "Get URI for a Flux instance launched under IBM Spectrum LSF"
+
+    def resolve(self, uri):
+        jobid = str(PurePath(uri.path).parts[0])
+
+        if "is_compute" in uri.query_dict:
+            uri = FluxURIResolver().resolve(f"pid:{lsf_job_pid(jobid)}")
+            return uri.remote
+
+        try:
+            hostname = lsf_find_compute_node(jobid)
+            return lsf_get_uri(hostname, jobid)
+        except OSError as exc:
+            raise ValueError(
+                f"LSF Job {jobid} doesn't seem to have a FLUX_URI"
+            ) from exc
diff --git a/src/bindings/python/flux/uri/resolvers/pid.py b/src/bindings/python/flux/uri/resolvers/pid.py
new file mode 100644
index 000000000000..7953287fc826
--- /dev/null
+++ b/src/bindings/python/flux/uri/resolvers/pid.py
@@ -0,0 +1,109 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import os
+import re
+from pathlib import PurePath
+
+from flux.uri import URIResolverPlugin
+
+
+def _get_broker_child_fallback(broker_pid):
+    broker_pid = int(broker_pid)
+    pids = [int(x) for x in os.listdir("/proc") if x.isdigit()]
+
+    #  Sort list of pids based on distance from broker_pid
+    #  (PID of broker child is likely close to broker_pid)
+    pids.sort(key=lambda pid: abs(pid - broker_pid))
+
+    #  Remove broker_pid since it definitely isn't a child
+    pids.remove(broker_pid)
+
+    #  Now iterate all processes, returning immediately when
+    #   we've found a process for which broker_pid is the parent
+    for pid in pids:
+        try:
+            with open(f"/proc/{pid}/stat") as statf:
+                data = statf.read()
+        except OSError:
+            #  /proc/PID/stat is readable by all, but there is a chance
+            #  of a TOU-TOC race here, so just ignore all errors
+            pass
+        else:
+            match = re.match(r"^[0-9]+ \(.*\) \w+ ([0-9]+)", data)
+            #  Attempt to convert match to integer. On regex match failure,
+            #   or integer conversion failure, just skip this entry
+            try:
+                ppid = int(match.group(1))
+                if ppid == broker_pid:
+                    return pid
+            except (IndexError, ValueError):
+                pass
+    raise ValueError(f"PID {broker_pid} is a flux-broker and no child found")
+
+
+def _get_broker_child(pid):
+    """Return the pid of the first child found, for use with flux-broker"""
+    for taskid in os.listdir(f"/proc/{pid}/task"):
+        try:
+            with open(f"/proc/{pid}/task/{taskid}/children") as cfile:
+                cpid = cfile.read().split(" ")[0]
+                if cpid:
+                    return cpid
+        except FileNotFoundError:
+            pass
+    raise ValueError(f"PID {pid} is a flux-broker and no child found")
+
+
+def _proc_has_task_children():
+    pid = os.getpid()
+    for taskid in os.listdir(f"/proc/{pid}/task"):
+        try:
+            with open(f"/proc/{pid}/task/{taskid}/children") as cfile:
+                return True
+        except FileNotFoundError:
+            return False
+    return False
+
+
+class URIResolver(URIResolverPlugin):
+    """A URIResolver that can fetch a FLUX_URI value from a local PID"""
+
+    #  Determine which broker_get_child method to use:
+    if (
+        _proc_has_task_children()
+        and "FLUX_FORCE_BROKER_CHILD_FALLBACK" not in os.environ
+    ):
+        get_broker_child = staticmethod(_get_broker_child)
+    else:
+        get_broker_child = staticmethod(_get_broker_child_fallback)
+
+    def describe(self):
+        return "Get FLUX_URI for a given local PID"
+
+    def resolve(self, uri):
+        """
+        Resolve a PID to a URI by grabbing the FLUX_URI environment variable
+        from the process environment, or if the process is a flux-broker,
+        then from the first child.
+        """
+        pid = PurePath(uri.path).parts[0]
+        command = ""
+        with open(f"/proc/{pid}/status") as sfile:
+            command = sfile.read().split("\t")[1]
+        if re.search("flux-broker", command):
+            pid = self.get_broker_child(pid)
+        with open(f"/proc/{pid}/environ", encoding="utf-8") as envfile:
+            for line in envfile.read().split("\0"):
+                if "=" in line:
+                    result = line.split("=")
+                    if result[0] == "FLUX_URI":
+                        return result[1]
+        raise ValueError(f"PID {pid} doesn't seem to have a FLUX_URI")
diff --git a/src/bindings/python/flux/uri/resolvers/slurm.py b/src/bindings/python/flux/uri/resolvers/slurm.py
new file mode 100644
index 000000000000..d2e9c2248e8c
--- /dev/null
+++ b/src/bindings/python/flux/uri/resolvers/slurm.py
@@ -0,0 +1,107 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import errno
+import os
+import subprocess
+from pathlib import PurePath
+
+from flux.uri import FluxURIResolver, URIResolverPlugin
+
+
+def slurm_job_pids(jobid):
+    """Read pids for Slurm jobid using scontrol listpids"""
+
+    pids = []
+    sp = subprocess.run(
+        ["scontrol", "listpids", jobid], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    )
+    if sp.returncode != 0:
+        raise OSError(errno.ENOENT)
+    for line in sp.stdout.decode("utf-8").split("\n"):
+        #  Output fields are "PID,JOBID,STEPID,LOCALID,GLOBALID"
+        #
+        #  The "main" flux-broker should have been directly launched
+        #   from slurmstepd, so its LOCALID should be 0. Thus, only
+        #   process pids with LOCALID 0 so we don't accidentally get
+        #   the FLUX_URI for a subinstance of the main Flux instance
+        #   (avoids the need to query the instance-level attribute)
+        fields = line.split()
+        try:
+            if int(fields[3]) != 0:
+                continue
+            pid = int(fields[0])
+            if pid != os.getpid():
+                pids.append(pid)
+        except (ValueError, IndexError):
+            pass
+    return pids
+
+
+def slurm_resolve_remote(jobid):
+    """
+    Attempt to resolve a Flux job URI for Slurm jobid by running
+
+      srun --overlap --jobid flux uri slurm:jobid
+
+    on the first node of the Slurm job
+    """
+
+    #  Clear FLUX_URI in srun environment so we don't confuse
+    #   ourselves and return the current FLUX_URI from flux-uri's
+    #   environment
+    env = os.environ.copy()
+    if "FLUX_URI" in env:
+        del env["FLUX_URI"]
+    sp = subprocess.run(
+        [
+            "srun",
+            "--overlap",
+            f"--jobid={jobid}",
+            "-n1",
+            "-N1",
+            "flux",
+            "uri",
+            f"slurm:{jobid}",
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=env,
+    )
+    if sp.returncode != 0:
+        raise ValueError(f"Unable to resolve Flux URI for Slurm job {jobid}")
+    return sp.stdout.decode("utf-8").rstrip()
+
+
+class URIResolver(URIResolverPlugin):
+    """A URIResolver that can fetch a FLUX_URI from a Slurm job"""
+
+    def describe(self):
+        return "Get URI for a Flux instance launched under Slurm"
+
+    def resolve(self, uri):
+        jobid = PurePath(uri.path).parts[0]
+
+        #  Get list of local Slurm job pids from scontrol.
+        #  If that fails, then the job might be running remotely, so
+        #   try using srun to run `flux uri slurm:JOBID` on the first
+        #   node of the Slurm job.
+        try:
+            pids = slurm_job_pids(jobid)
+        except OSError:
+            return slurm_resolve_remote(jobid)
+
+        resolver = FluxURIResolver()
+        for pid in pids:
+            try:
+                return resolver.resolve(f"pid:{pid}").remote
+            except (ValueError, OSError):
+                pass
+        raise ValueError(f"PID {pid} doesn't seem to have a FLUX_URI")
diff --git a/src/bindings/python/flux/uri/uri.py b/src/bindings/python/flux/uri/uri.py
new file mode 100644
index 000000000000..8247a146a4b9
--- /dev/null
+++ b/src/bindings/python/flux/uri/uri.py
@@ -0,0 +1,203 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import os
+import platform
+import re
+from abc import ABC, abstractmethod
+from urllib.parse import parse_qs, urlparse
+
+from flux.importer import import_plugins
+
+
+class URI:
+    """Simple URI class
+
+    Parse URI strings with urllib.parse.urlparse, but with mutable properties
+
+    Attributes:
+      uri: copy of original uri argument
+      scheme: addressing scheme
+      netloc: network location part
+      path: path part
+      params: parameters if present
+      query:  query component
+      query_dict: query component as dictionary
+      fragment: fragment identifier
+    """
+
+    def __init__(self, uri):
+
+        self.uri = uri
+        self.query_dict = {}
+        uri = urlparse(self.uri)
+
+        for name, value in uri._asdict().items():
+            setattr(self, name, value)
+
+        if self.query:
+            self.query_dict = parse_qs(self.query, keep_blank_values=True)
+
+
+class JobURI(URI):
+    """A Flux job/instance URI
+
+    A URI specific to a Flux instance. Same attributes as flux.uri.URI,
+    with additional attributes to convert to a ``remote`` or ``local``
+    URI.
+
+    Attributes:
+      remote: If local URI, returns a remote URI substituting current hostname.
+              If a remote URI, returns the URI.
+      local: If a remote URI, convert to a local URI. Otherwise return the URI.
+    """
+
+    force_local = os.environ.get("FLUX_URI_RESOLVE_LOCAL", False)
+
+    def __init__(self, uri):
+        super().__init__(uri)
+        if self.scheme == "":
+            raise ValueError(f"JobURI '{uri}' does not have a valid scheme")
+        self.path = re.sub("/+", "/", self.path)
+        self.remote_uri = None
+        self.local_uri = None
+
+    @property
+    def remote(self):
+        if not self.remote_uri:
+            if self.scheme == "ssh":
+                self.remote_uri = self.uri
+            elif self.scheme == "local":
+                hostname = platform.uname()[1]
+                self.remote_uri = f"ssh://{hostname}{self.path}"
+            else:
+                raise ValueError(
+                    f"Cannot convert JobURI with scheme {self.scheme} to remote"
+                )
+        return self.remote_uri
+
+    @property
+    def local(self):
+        if not self.local_uri:
+            if self.scheme == "local":
+                self.local_uri = self.uri
+            elif self.scheme == "ssh":
+                self.local_uri = f"local://{self.path}"
+            else:
+                raise ValueError(
+                    f"Cannot convert JobURI with scheme {self.scheme} to local"
+                )
+
+        return self.local_uri
+
+    def __str__(self):
+        if self.force_local:
+            return self.local
+        return self.uri
+
+
+class URIResolverURI(URI):
+    """A URI which resolves to a JobURI
+
+    A URI used with ``FluxURIResolver.resolve``.
+    Includes a workaround for ``urllib.parse.urlparse`` problems parsing
+    path components with only digits.
+    """
+
+    def __init__(self, uri):
+        #  Replace : with :FXX to allow path to be digits only
+        super().__init__(uri.replace(":", ":FXX", 1))
+        self.path = self.path.replace("FXX", "", 1)
+
+
+class URIResolverPlugin(ABC):  # pragma: no cover
+    """Abstract type for a plugin used to resolve Flux URIs"""
+
+    def __init__(self, *args):
+        """Initialize a URI resolver plugin"""
+
+    @abstractmethod
+    def describe(self):
+        """Return a short description of the support URI scheme"""
+        return NotImplementedError
+
+    @abstractmethod
+    def resolve(self, uri):
+        """Resolve a get-uri URI into a FluxJobURI"""
+        return NotImplementedError
+
+
+class FluxURIResolver:
+    """A plugin-based Flux job URI resolver class
+
+    A FluxURIResolver loads plugins which implement different _schemes_
+    for resolution simple URIs to Flux job URIs. Plugins or "resolvers"
+    are loaded from the ``flux.uri.resolvers`` namespace.
+    """
+
+    def __init__(self, pluginpath=None):
+        if pluginpath is None:
+            pluginpath = []
+        self.plugin_namespace = "flux.uri.resolvers"
+        self.resolvers = {}
+
+        plugins = import_plugins(self.plugin_namespace, pluginpath)
+        if plugins:
+            for scheme, plugin in plugins.items():
+                self.resolvers[scheme] = plugin.URIResolver()
+
+    def resolve(self, uri):
+        """Resolve ``uri`` to a Flux job URI using dynamically loaded plugins
+
+        The _scheme_ of the provided target URI determines which plugin
+        is used to satisfy the query.
+
+        If no _scheme_ is provided, then a default scheme of ``jobid``
+        is assumed.
+
+        URI "query" parameters may be supported by the underlying resolver
+        plugin, but the ``remote`` and ``local`` query strings are always
+        supported and will result in forced conversion of the returned
+        job URI to a remote or local form, respectively.
+
+        Raises NotImplementedError if no plugin for _scheme_ can be found.
+
+        Raises ValueError if ``uri`` cannot otherwise be converted to a
+        job uri by the plugin for _scheme_.
+
+        """
+        scheme = URIResolverURI(uri).scheme
+        if str(scheme) == "":
+            scheme = "jobid"
+            uri = f"jobid:{uri}"
+        if scheme in ["ssh", "local"]:
+            return JobURI(uri)
+        if scheme not in self.resolvers:
+            raise NotImplementedError(f"No plugin for URI scheme {scheme}")
+
+        resolver_uri = URIResolverURI(uri)
+
+        query = resolver_uri.query_dict
+        if "local" in query and "remote" in query:
+            raise ValueError("Only one of 'local' or 'remote' is allowed")
+
+        result = JobURI(self.resolvers[scheme].resolve(resolver_uri))
+
+        #  Special case for 'local' or 'remote' query parameters:
+        if "local" in query:
+            result = JobURI(result.local)
+        elif "remote" in query:
+            result = JobURI(result.remote)
+        return result
+
+    def plugins(self):
+        """Get a dict of loaded URI resolver plugins by {name: description}"""
+
+        return {name: x.describe() for name, x in self.resolvers.items()}
diff --git a/src/bindings/python/flux/util.py b/src/bindings/python/flux/util.py
index a3fd7e990590..19374485f318 100644
--- a/src/bindings/python/flux/util.py
+++ b/src/bindings/python/flux/util.py
@@ -8,19 +8,40 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-import re
-import sys
+import argparse
+import base64
+import copy
 import errno
+import glob
 import json
 import logging
-import os
 import math
-import argparse
-from datetime import timedelta
+import os
+import re
+import shutil
+import signal
+import stat
+import sys
+import threading
+import traceback
+from collections import namedtuple
+from datetime import datetime, timedelta
+from operator import attrgetter
+from pathlib import Path, PurePosixPath
+from string import Formatter
+from typing import Mapping
 
-import six
+import yaml
+
+# tomllib added to standard library in Python 3.11
+# flux-core minimum is Python 3.6.
+try:
+    import tomllib  # novermin
+except ModuleNotFoundError:
+    from flux.utils import tomli as tomllib
 
 from flux.core.inner import ffi, raw
+from flux.utils.parsedatetime import Calendar
 
 __all__ = [
     "check_future_error",
@@ -36,7 +57,6 @@ def func_wrapper(calling_obj, *args, **kwargs):
         try:
             return func(calling_obj, *args, **kwargs)
         except EnvironmentError as error:
-            exception_tuple = sys.exc_info()
             try:
                 future = (
                     calling_obj.handle
@@ -45,10 +65,37 @@ def func_wrapper(calling_obj, *args, **kwargs):
                 )
                 errmsg = raw.flux_future_error_string(future)
             except EnvironmentError:
-                six.reraise(*exception_tuple)
+                raise error from None
             if errmsg is None:
-                six.reraise(*exception_tuple)
-            raise EnvironmentError(error.errno, errmsg.decode("utf-8"))
+                raise error from None
+            raise EnvironmentError(error.errno, errmsg.decode("utf-8")) from None
+
+    return func_wrapper
+
+
+def interruptible(func):
+    """Make a Future method interruptible via Ctrl-C
+
+    Necessary for Future methods that may block when calling into the C API.
+    """
+
+    def func_wrapper(future, *args, **kwargs):
+        # python only allows `signal.signal` calls in the main thread
+        # only activate if the process is not in the Flux reactor
+        active = False
+        flux_handle = future.get_flux()
+        if (
+            threading.current_thread() is threading.main_thread()
+            and flux_handle is not None
+            and not flux_handle.reactor_running()
+        ):
+            handler = signal.getsignal(signal.SIGINT)
+            signal.signal(signal.SIGINT, signal.SIG_DFL)
+            active = True
+        retval = func(future, *args, **kwargs)
+        if active:
+            signal.signal(signal.SIGINT, handler)
+        return retval
 
     return func_wrapper
 
@@ -56,10 +103,12 @@ def func_wrapper(calling_obj, *args, **kwargs):
 def encode_payload(payload):
     if payload is None or payload == ffi.NULL:
         payload = ffi.NULL
-    elif isinstance(payload, six.text_type):
-        payload = payload.encode("UTF-8")
-    elif not isinstance(payload, six.binary_type):
-        payload = json.dumps(payload, ensure_ascii=False).encode("UTF-8")
+    elif isinstance(payload, str):
+        payload = payload.encode("UTF-8", errors="surrogateescape")
+    elif not isinstance(payload, bytes):
+        payload = json.dumps(payload, ensure_ascii=False).encode(
+            "UTF-8", errors="surrogateescape"
+        )
     return payload
 
 
@@ -67,15 +116,15 @@ def encode_topic(topic):
     # Convert topic to utf-8 binary string
     if topic is None or topic == ffi.NULL:
         raise EnvironmentError(errno.EINVAL, "Topic must not be None/NULL")
-    elif isinstance(topic, six.text_type):
+    if isinstance(topic, str):
         topic = topic.encode("UTF-8")
-    elif not isinstance(topic, six.binary_type):
+    elif not isinstance(topic, bytes):
         errmsg = "Topic must be a string, not {}".format(type(topic))
         raise TypeError(errno.EINVAL, errmsg)
     return topic
 
 
-def help_formatter(argwidth=40):
+def help_formatter(argwidth=40, raw_description=False):
     """
     Return our 'clean' HelpFormatter, if possible, with a wider default
      for the max width allowed for options.
@@ -100,7 +149,7 @@ def _format_action_invocation(self, action):
             optstring = ", ".join(opts)
 
             #  If only a long option is supported, then prefix with
-            #   whitepsace by the width of a short option so that all
+            #   whitespace by the width of a short option so that all
             #   long opts start in the same column:
             if len(opts) == 1 and len(opts[0]) > 2:
                 optstring = "    " + opts[0]
@@ -114,9 +163,133 @@ def _format_action_invocation(self, action):
             args_string = self._format_args(action, default)
             return optstring + "=" + args_string
 
+    class FluxRawDescriptionHelpFormatter(
+        FluxHelpFormatter, argparse.RawDescriptionHelpFormatter
+    ):
+        pass
+
+    if raw_description:
+        return lambda prog: FluxRawDescriptionHelpFormatter(
+            prog, max_help_position=argwidth
+        )
+
     return lambda prog: FluxHelpFormatter(prog, max_help_position=argwidth)
 
 
+def get_treedict(in_dict, key, default=None):
+    """
+    get_treedict(d, "a.b.c" [,default=None]) is like:
+    >>> try:
+    >>>     return d[a][b][c]
+    >>> except KeyError:
+    >>>     return default
+    """
+    if "." in key:
+        next_key, rest = key.split(".", 1)
+        try:
+            return get_treedict(in_dict[next_key], rest, default)
+        except KeyError:
+            return default
+    return in_dict.get(key, default)
+
+
+def set_treedict(in_dict, key, val):
+    """
+    set_treedict(d, "a.b.c", 42) is like d[a][b][c] = 42
+    but levels are created on demand.
+    """
+    path = key.split(".", 1)
+    if len(path) == 2:
+        set_treedict(in_dict.setdefault(path[0], {}), path[1], val)
+    else:
+        in_dict[key] = val
+
+
+class TreedictAction(argparse.Action):
+    """
+    argparse action that returns a dictionary from a set of key=value option
+    arguments. The ``flux.util.set_treedict()`` function is used to set
+    ``key`` in the dictionary, which allows ``key`` to have embedded dots.
+    e.g. ``a.b.c=42`` is like::
+
+       dest[a][b][c] = 42
+
+    but levels are created on demand.
+    """
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        result = {}
+        for arg in values:
+            tmp = arg.split("=", 1)
+            if len(tmp) != 2:
+                raise ValueError(f"Missing required value for key {arg}")
+            try:
+                val = json.loads(tmp[1])
+            except (json.JSONDecodeError, TypeError):
+                val = tmp[1]
+            set_treedict(result, tmp[0], val)
+            setattr(namespace, self.dest, result)
+
+
+class FilterAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+        setattr(namespace, "filtered", True)
+
+
+class FilterActionSetUpdate(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, "filtered", True)
+        values = values.split(",")
+        getattr(namespace, self.dest).update(values)
+
+
+# pylint: disable=redefined-builtin
+class FilterTrueAction(argparse.Action):
+    def __init__(
+        self,
+        option_strings,
+        dest,
+        const=True,
+        default=False,
+        required=False,
+        help=None,
+        metavar=None,
+    ):
+        super(FilterTrueAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            help=help,
+        )
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, self.const)
+        setattr(namespace, "filtered", True)
+
+
+class YesNoAction(argparse.Action):
+    """Simple argparse.Action for options with yes|no arguments"""
+
+    def __init__(
+        self,
+        option_strings,
+        dest,
+        help=None,
+        metavar="[yes|no]",
+    ):
+        super().__init__(
+            option_strings=option_strings, dest=dest, help=help, metavar=metavar
+        )
+
+    def __call__(self, parser, namespace, value, option_string=None):
+        if value not in ["yes", "no"]:
+            raise ValueError(f"{option_string} requires either 'yes' or 'no'")
+        setattr(namespace, self.dest, value == "yes")
+
+
 class CLIMain(object):
     def __init__(self, logger=None):
         if logger is None:
@@ -136,9 +309,13 @@ def __call__(self, main_func):
             exit_code = ex
         except Exception as ex:  # pylint: disable=broad-except
             exit_code = 1
-            self.logger.error(str(ex))
-            import traceback
-
+            # Prefer '{strerror}: {filename}' error message over default
+            # OSError string representation which includes useless
+            # `[Error N]` prefix in output.
+            errmsg = getattr(ex, "strerror", None) or str(ex)
+            if getattr(ex, "filename", None):
+                errmsg += f": '{ex.filename}'"
+            self.logger.error(errmsg)
             self.logger.debug(traceback.format_exc())
         finally:
             logging.shutdown()
@@ -146,12 +323,16 @@ def __call__(self, main_func):
 
 
 def parse_fsd(fsd_string):
-    match = re.match(r".*([smhd])$", fsd_string)
+    # Special case for RFC 23 "infinity"
+    if fsd_string in ["inf", "infinity", "INF", "INFINITY"]:
+        return float("inf")
+
+    match = re.match(r"(.*?)(s|m|h|d|ms)$", fsd_string)
     try:
-        value = float(fsd_string[:-1] if match else fsd_string)
+        value = float(match.group(1) if match else fsd_string)
     except:
         raise ValueError("invalid Flux standard duration")
-    unit = match.group(1) if match else "s"
+    unit = match.group(2) if match else "s"
 
     if unit == "m":
         seconds = timedelta(minutes=value).total_seconds()
@@ -159,8 +340,1298 @@ def parse_fsd(fsd_string):
         seconds = timedelta(hours=value).total_seconds()
     elif unit == "d":
         seconds = timedelta(days=value).total_seconds()
+    elif unit == "ms":
+        seconds = timedelta(milliseconds=value).total_seconds()
     else:
         seconds = value
     if seconds < 0 or math.isnan(seconds) or math.isinf(seconds):
         raise ValueError("invalid Flux standard duration")
     return seconds
+
+
+def parse_datetime(string, now=None):
+    """Parse a possibly human readable datetime string or offset
+
+    If string starts with `+` or `-`, then the remainder of the string
+    is assumed to be a duration to add or subtract to `now` in Flux Standard
+    Duration.
+
+    Otherwise, the parsedatetime package will be used to parse the input
+    string.
+
+    Args:
+        string: The string to parse as datetime or offset
+        now: Optional: datetime object to use as starttime of any offset
+
+    Returns:
+        A datetime object
+
+    Raises:
+        ValueError: Input string could not be converted to datetime
+
+    """
+
+    if now is None:
+        now = datetime.now().astimezone()
+
+    if string == "now":
+        return now
+
+    if string.startswith("+"):
+        timestamp = now.timestamp() + parse_fsd(string[1:])
+        return datetime.fromtimestamp(timestamp).astimezone()
+
+    if string.startswith("-"):
+        timestamp = now.timestamp() - parse_fsd(string[1:])
+        return datetime.fromtimestamp(timestamp).astimezone()
+
+    cal = Calendar()
+    cal.ptc.StartHour = 0
+    time_struct, status = cal.parse(string, sourceTime=now.timetuple())
+    if status == 0:
+        raise ValueError(f'Invalid datetime: "{string}"')
+    return datetime(*time_struct[:6]).astimezone()
+
+
+class UtilDatetime(datetime):
+    """
+    Subclass of datetime but with a __format__ method that supports
+    width and alignment specification after any time formatting via
+    two colons `::` before the Python format spec, e.g::
+
+    >>> "{0:%b%d %R::>16}".format(UtilDatetime.now())
+    '     Sep21 18:36'
+    """
+
+    def __format__(self, fmt):
+        # The string "::" is used to split the strftime() format from
+        # any Python format spec:
+        timefmt, *spec = fmt.split("::", 1)
+
+        if self == datetime.fromtimestamp(0.0):
+            result = ""
+        else:
+            # Call strftime() to get the formatted datetime as a string
+            result = self.strftime(timefmt or "%FT%T")
+
+        spec = spec[0] if spec else ""
+
+        # Handling of the 'h' suffix on spec is required here, since the
+        # UtilDatetime format() is called _after_ UtilFormatter.format_field()
+        # (where this option is handled for other types)
+        if spec.endswith("h"):
+            if not result:
+                result = "-"
+            spec = spec[:-1] + "s"
+
+        # If there was a format spec, apply it here:
+        if spec:
+            return f"{{0:{spec}}}".format(result)
+        return result
+
+
+def fsd(secs):
+    # Special case for RFC 23 "infinity"
+    # N.B. We return lower case "inf" to match Python's "math.inf".
+    if math.isinf(secs):
+        return "inf"
+
+    #  Round <1ms down to 0s for now
+    if secs < 1.0e-3:
+        strtmp = "0s"
+    elif secs < 10.0:
+        strtmp = "%.03fs" % secs
+    elif secs < 60.0:
+        strtmp = "%.4gs" % secs
+    elif secs < (60.0 * 60.0):
+        strtmp = "%.4gm" % (secs / 60.0)
+    elif secs < (60.0 * 60.0 * 24.0):
+        strtmp = "%.4gh" % (secs / (60.0 * 60.0))
+    else:
+        strtmp = "%.4gd" % (secs / (60.0 * 60.0 * 24.0))
+    return strtmp
+
+
+def empty_outputs():
+    localepoch = datetime.fromtimestamp(0.0).strftime("%FT%T")
+    empty = ("", "0s", "0.0", "0:00:00", "1970-01-01T00:00:00", localepoch)
+    return empty
+
+
+class UtilFormatter(Formatter):
+    # pylint: disable=too-many-branches
+
+    def convert_field(self, value, conv):
+        """
+        Flux utility-specific field conversions. Avoids the need
+        to create many different format field names to represent
+        different conversion types.
+        """
+        orig_value = str(value)
+        if conv == "d":
+            # convert from float seconds since epoch to a datetime.
+            # User can than use datetime specific format fields, e.g.
+            # {t_inactive!d:%H:%M:%S}.
+            try:
+                value = UtilDatetime.fromtimestamp(value)
+            except TypeError:
+                if orig_value == "":
+                    value = UtilDatetime.fromtimestamp(0.0)
+                else:
+                    raise
+        elif conv == "D":
+            # the JobInfo class initializes many timestamps to 0.0 if
+            # they are not available.  Treat epoch time as special case
+            # and just return empty string.
+            if not value:
+                return ""
+            # As above, but convert to ISO 8601 date time string.
+            try:
+                value = datetime.fromtimestamp(value).strftime("%FT%T")
+            except TypeError:
+                if orig_value == "":
+                    value = ""
+                else:
+                    raise
+        elif conv == "F":
+            # convert to Flux Standard Duration (fsd) string.
+            try:
+                value = fsd(value)
+            except TypeError:
+                if orig_value == "":
+                    value = ""
+                else:
+                    raise
+        elif conv == "H":
+            # if > 0, always round up to at least one second to
+            #  avoid presenting a nonzero timedelta as zero
+            try:
+                if 0 < value < 1:
+                    value = 1
+                value = str(timedelta(seconds=round(value)))
+            except TypeError:
+                if orig_value == "":
+                    value = ""
+                else:
+                    raise
+        elif conv == "P":
+            #  Convert a floating point to percentage
+            try:
+                value = value * 100
+                if 0 < value < 1:
+                    value = f"{value:.2f}%"
+                else:
+                    value = f"{value:.3g}%"
+            except (TypeError, ValueError):
+                if orig_value == "":
+                    value = ""
+                else:
+                    raise
+        else:
+            value = super().convert_field(value, conv)
+        return value
+
+    def format_field(self, value, spec):
+
+        denote_truncation = False
+        if spec.endswith("+"):
+            denote_truncation = True
+            spec = spec[:-1]
+
+        # Note: handling of the 'h' suffix for UtilDatetime objects
+        # must be deferred to the UtilDatetetime format() method, since
+        # that method will be called after this one:
+        if spec.endswith("h") and not isinstance(value, UtilDatetime):
+            basecases = empty_outputs()
+            value = "-" if str(value) in basecases else str(value)
+            spec = spec[:-1] + "s"
+        retval = super().format_field(value, spec)
+
+        if denote_truncation and len(retval) < len(str(value)):
+            retval = retval[:-1] + "+"
+
+        return retval
+
+
+class AltField:
+    """
+    Convenient wrapper for fields that have an ascii and non-ascii
+    representation. Allows the ascii representation to be selected with
+    {field.ascii}.
+    """
+
+    def __init__(self, default, ascii):
+        self.default = default
+        self.ascii = ascii
+
+    def __str__(self):
+        return self.default
+
+    def __format__(self, fmt):
+        return str(self).__format__(fmt)
+
+
+class OutputFormat:
+    """Extended output format container class for Flux utilities.
+
+    The OutputFormat class stores a parsed version of a program's output
+    format, allowing fields to be iterated without their corresponding
+    modifiers, automatic creation of a format string suitable for header
+    display, as well as some other features specific to Flux utility
+    output including:
+
+     - Support for format prefix "sentinels" ``?:``, ``+:`` and ``?+:``
+       which suppress fields that would be empty, expand fields automatically
+       to their maximum obsevered width, or both.
+     - Support for embedding a set of sort keys in a format via a
+       ``sort:KEY,...`` prefix.
+     - A ``print_items()`` method to handle the common code for sorting,
+       filtering and printing a set of items using the current format.
+
+    Attributes:
+        formatter (Formatter): a custom string formatter that will be used to
+            render output using its ``format()`` method. The default
+            formatter is :py:class:`flux.util.UtilFormatter`
+        headings (dict): A mapping of field name to header string. This also
+            provides detection of invalid fields, so all valid fields should
+            have an entry in headings. The headings dict can be set as an
+            argument to :py:func:`OutputFormat.__init__`, or a subclass of
+            the :py:class`flux.util.OutputFormat` class may set default
+            headings for a given tool.
+    """
+
+    formatter = UtilFormatter
+    headings: Mapping[str, str] = {}
+
+    class HeaderFormatter(UtilFormatter):
+        """Custom formatter for flux utilities header row.
+
+        Override default formatter behavior of calling getattr() on dotted
+        field names. Instead look up header literally in kwargs.
+        This greatly simplifies header name registration as well as
+        registration of "valid" fields.
+        """
+
+        def get_field(self, field_name, args, kwargs):
+            """Override get_field() so we don't do the normal gettatr thing"""
+            if field_name in kwargs:
+                return kwargs[field_name], None
+            return super().get_field(field_name, args, kwargs)
+
+    def __init__(self, fmt, headings=None, prepend="0."):
+        """
+        Parse the input format ``fmt`` with string.Formatter.
+        Save off the fields and list of format tokens for later use,
+        (converting None to "" in the process)
+
+        Args:
+            headings (dict): Set a mapping of field name to header string.
+            prepend (str): Prepend each field with a string. (Default is
+                "0.", so ``{f1} {f2}`` becomes ``{0.f1} {0.f2}``.
+        """
+        if headings is not None:
+            self.headings = headings
+        if prepend is not None:
+            self.prepend = prepend
+        #  Parse format into list of (string, field, spec, conv) tuples,
+        #   replacing any None values with empty string "" (this makes
+        #   substitution back into a format string in self.header() and
+        #   self.get_format() much simpler below)
+        format_list = Formatter().parse(fmt)
+        self.format_list = [[s or "" for s in t] for t in format_list]
+
+        #  If the first entry's text portion starts with `sort:`, then
+        #  what follows is a list of sort keys (used only in print_items())
+        #
+        #  self.sort_keys is a list of (key(str), reverse(bool)) tuples.
+        self.sort_keys = []
+        if self.format_list and self.format_list[0][0].startswith("sort:"):
+            #  Remove sort:keys, prefix from format_list, replace with
+            #   empty string:
+            sort_spec = self.format_list[0][0]
+            self.format_list[0][0] = ""
+
+            #  Store sort keys for later use
+            self.set_sort_keys(sort_spec[5:])
+
+            #  Strip sort_spec from beginning of fmt
+            fmt = fmt.lstrip(sort_spec)
+
+        #  Save format after removal of sort keys:
+        self.fmt = fmt
+        self.fmt_orig = fmt
+
+        #  Store list of requested fields in self.fields
+        #  (ignore any empty fields, which may be present due to text
+        #   at the end of a format string):
+        self._fields = [field for (_, field, _, _) in self.format_list if field]
+
+        #  Throw an exception if any requested fields are invalid:
+        for field in self._fields:
+            #  Remove any "0." prefix:
+            field = field[2:] if field.startswith("0.") else field
+            if field and field not in self.headings:
+                raise ValueError("Unknown format field: " + field)
+
+        #  Prepend arbitrary string to format fields if requested
+        if self.prepend:
+            self.fmt = self.get_format_prepended(
+                self.prepend, include_sort_prefix=False
+            )
+
+    class FormatSpec:
+        """Split Python format spec into components"""
+
+        # Note: 'hyphen' and 'truncate' are Flux extensions
+        components = (
+            "fill",
+            "align",
+            "sign",
+            "pos_zero",
+            "alt",
+            "zero_pad",
+            "width_str",
+            "grouping",
+            "decimal",
+            "precision_str",
+            "type",
+            "hyphen",
+            "truncate",
+        )
+
+        def __init__(self, spec):
+
+            # If spec contains two colons ('::') then this is a special
+            # datetime conversion spec and the format spec is after the
+            # two colons:
+            if "::" in spec:
+                spec = spec[spec.find("::") + 2 :]
+
+            # Regex taken from https://stackoverflow.com/a/78351366
+            # 'hyphen' and 'truncate' are Flux extensions
+            spec_re = re.compile(
+                r"(?:(?P<fill>[\s\S])?(?P<align>[<>=^]))?"
+                r"(?P<sign>[- +])?"
+                r"(?P<pos_zero>z)?"
+                r"(?P<alt>#)?"
+                r"(?P<zero_pad>0)?"
+                r"(?P<width_str>\d+)?"
+                r"(?P<grouping>[_,])?"
+                r"(?:(?P<decimal>\.)(?P<precision_str>\d+))?"
+                r"(?P<type>[bcdeEfFgGnosxX%])?"
+                r"(?P<hyphen>h)?"
+                r"(?P<truncate>\+)?"
+            )
+            try:
+                self._spec_dict = spec_re.fullmatch(spec).groupdict()
+            except AttributeError:
+                self._spec_dict = {}
+
+            for item in self.components:
+                value = self._spec_dict.get(item)
+                setattr(self, item, "" if value is None else value)
+
+        @property
+        def width(self):
+            return int(self.width_str) if self.width_str else ""
+
+        @width.setter
+        def width(self, val):
+            self.width_str = str(val)
+
+            # Also adjust precision if necessary (only for string type)
+            if (
+                self.type in (None, "s")
+                and self.precision
+                and self.precision < self.width
+            ):
+                self.precision = self.width
+
+        @property
+        def precision(self):
+            return int(self.precision_str) if self.precision_str else 0
+
+        @precision.setter
+        def precision(self, val):
+            self.precision_str = str(val)
+
+        def __str__(self):
+            result = ""
+            for item in self.components:
+                value = getattr(self, item)
+                if value is not None:
+                    result += str(value)
+            return result
+
+    @property
+    def fields(self):
+        # Add any sort keys to returned fields for users like flux-jobs(1),
+        # which need to translate requested fields to job-list attributes:
+        fields = set(self._fields.copy())
+        if self.sort_keys:
+            fields.update([x for x, y in self.sort_keys])
+        return list(fields)
+
+    # This should be a method, not a function since it is overridden by
+    # inheriting classes
+    # pylint: disable=no-self-use
+    def _fmt_tuple(self, text, field, spec, conv):
+        #  If field is empty string or None, then the result of the
+        #   format (besides 'text') doesn't make sense, just return 'text'
+        if not field:
+            return text
+        #  The prefix of spec and conv are stripped by formatter.parse()
+        #   replace them here if the values are not empty:
+        spec = ":" + spec if spec else ""
+        conv = "!" + conv if conv else ""
+        return "{0}{{{1}{2}{3}}}".format(text, field, conv, spec)
+
+    def header_format(self):
+        """
+        Return the current format sanitized for use in formatting a header
+        row. All format specs are adjusted to keep only the align and width,
+        and any numeric formatting types are dropped since the header row
+        is made up of only strings.
+        """
+        format_list = []
+        for text, field, spec, _ in self.format_list:
+            spec = self.FormatSpec(spec)
+
+            #  Only keep align and min width of the result.
+            #  This strips possible type-specific formatting spec that
+            #   will not apply to a heading, but keeps width and alignment.
+            spec = f"{spec.align}{spec.width}"
+
+            #  Remove any conversion, these do not make sense for headings
+            format_list.append(self._fmt_tuple(text, field, spec, None))
+        fmt = "".join(format_list)
+        return fmt
+
+    def header(self):
+        """Return a header row using the current format"""
+        formatter = self.HeaderFormatter()
+        return formatter.format(self.header_format(), **self.headings)
+
+    def get_format_prepended(
+        self, prepend, except_fields=None, include_sort_prefix=True
+    ):
+        """
+        Return the format string, ensuring that the string in "prepend"
+        is prepended to each format field
+
+        Args:
+            prepend (str): String to prepend to each field name, e.g. "0."
+            except_fields (list): List of fields to skip  when constructing
+                the format.
+            include_sort_prefix (bool): If format specifies a list of sort
+                keys, include them in a ``sort:`` prefix. Default: True
+        """
+        if except_fields is None:
+            except_fields = []
+        lst = []
+        for text, field, spec, conv in self.format_list:
+            # Skip this field if it is in except_fields
+            if field in except_fields:
+                # Preserve any format "prefix" (i.e. the text):
+                lst.append(text)
+                continue
+            # If field doesn't have 'prepend' then add it
+            if field and not field.startswith(prepend):
+                field = prepend + field
+            lst.append(self._fmt_tuple(text, field, spec, conv))
+        if include_sort_prefix:
+            prefix = self._sort_prefix()
+        else:
+            prefix = ""
+        return prefix + "".join(lst)
+
+    def _sort_prefix(self):
+        if self.sort_keys:
+            keys = [f"-{x}" if rev else x for x, rev in self.sort_keys]
+            return "sort:" + ",".join(keys) + " "
+        return ""
+
+    def get_format(self, orig=False, include_sort_prefix=True):
+        """
+        Return the format string
+        Args:
+            orig (bool): Return the format as originally specified.
+            include_sort_prefix (bool): If the format includes a list of
+                sort keys, include a ``sort:`` prefix in the result.
+        """
+        fmt = self.fmt_orig if orig else self.fmt
+        if include_sort_prefix:
+            return self._sort_prefix() + fmt
+        return fmt
+
+    def copy(self, except_fields=None):
+        """
+        Return a copy of the current formatter, optionally with some
+        fields removed
+
+        Args:
+            except_fields (list): List of fields to remove from result.
+        """
+        cls = self.__class__
+        return cls(
+            self.get_format_prepended("", except_fields),
+            headings=self.headings,
+            prepend=self.prepend,
+        )
+
+    def format(self, obj):
+        """Format object with internal format"""
+        try:
+            retval = self.formatter().format(
+                self.get_format(include_sort_prefix=False), obj
+            )
+        except KeyError as exc:
+            typestr = type(obj)
+            raise KeyError(f"Invalid format field {exc} for {typestr}")
+        return retval
+
+    def filter(self, items):
+        """
+        Check for format fields that are prefixed with `?:` (e.g. "?:{name}"),
+        and filter them out of the current format string if they result in an
+        empty value (as defined by the `empty` tuple defined below) for every
+        entry in `items`.
+
+        Also, check for fields prefixed with `+:` (e.g. "+:{name}") and expand
+        those fields to the maximum observed width.
+
+        (`?+:` requests both actions: filter out field if it is empty for all
+        items, if not expand to maximum width)
+        """
+
+        #  Build a list of all format strings that have one of the width
+        #  adjustment sentinels to determine which are subject to the test for
+        #  emptiness/maxwidth.
+        #
+        sentinels = {"?:": "filter", "+:": "maxwidth", "?+:": "both"}
+
+        def sentinel_keys():
+            #  helper function to iterate supported sentinels by longest
+            #  first to avoid matching `+:` instead of `?+:`, etc.
+            #
+            return sorted(sentinels.keys(), key=lambda x: -len(x))
+
+        lst = []
+        index = 0
+
+        #  Loop over each format entry to find entries using any one
+        #  of the sentinels above:
+        #
+        #  Note: we remove the leading `text` and the format `spec` because
+        #  these may make even empty formatted fields non-empty. E.g. a spec
+        #  of `:>8` will always make the format result 8 characters wide.
+        #
+        for text, field, spec, conv in self.format_list:
+            # Determine if field has any supported sentinel above:
+            end = next((x for x in sentinel_keys() if text.endswith(x)), False)
+
+            if end:
+                #  This entry matches one of the filtering sentinels, parse
+                #  the spec to get current width (and allow possible
+                #  reconstruction later).
+                spec = self.FormatSpec(spec)
+
+                #  Save a format without any spec to allow determination of
+                #  maximum width after formatting all items:
+                fmt = self._fmt_tuple("", "0." + field, None, conv)
+
+                #  Save the modified format, index, type, maximum width,
+                #  observed width, and broken-down spec in lst:
+                lst.append(
+                    dict(
+                        fmt=fmt,
+                        index=index,
+                        type=sentinels[end],
+                        maxwidth=spec.width or 0,
+                        width=0,
+                        spec=spec,
+                    )
+                )
+            index = index + 1
+
+        # Return immediately if no format fields need filtering:
+        if not lst:
+            return self.get_format(orig=True)
+
+        formatter = self.formatter()
+
+        #  Get a list of outputs that would qualify as "empty"
+        empty = empty_outputs()
+
+        #  Loop over all items that will be printed and capture the max
+        #  width. This will later be used to either filter out the format
+        #  field entirely, or update the width to the maximum value:
+        for item in items:
+            for entry in lst:
+                result = formatter.format(entry["fmt"], item)
+                width = 0 if result in empty else len(result)
+                if width > entry["maxwidth"]:
+                    entry["maxwidth"] = width
+                if width > entry["width"]:
+                    entry["width"] = width
+
+        #  Create two new lists from lst:
+        #
+        #  remove: These entries have 0 width and were requested to be removed
+        #
+        #  new_lst: lst without those entries in `remove`. These entries
+        #      have nonzero width and/or are requesting max width be
+        #      substituted into the format spec.
+        #
+        remove = []
+        new_lst = []
+        for entry in lst:
+            if entry["width"] == 0 and entry["type"] in ("filter", "both"):
+                remove.append(entry["index"])
+            else:
+                new_lst.append(entry)
+        lst = new_lst
+
+        #  For any remaining entries in lst, update the format spec width
+        #  to the found maxwidth:
+        #
+        format_list = self.format_list.copy()
+        for entry in lst:
+            if entry["type"] in ("maxwidth", "both"):
+                entry["spec"].width = entry["maxwidth"]
+                format_list[entry["index"]][2] = str(entry["spec"])
+
+        #  Remove any entries that were empty from self.format_list
+        #  After this line saved indices in entry["index"] will no longer
+        #  be valid, so they should not be used after this point.
+        #
+        format_list = [x for i, x in enumerate(self.format_list) if i not in remove]
+
+        #  Remove sentinels from remaining entries so they disappear in output.
+        for entry in format_list:
+            for s in sentinel_keys():
+                entry[0] = entry[0].replace(s, "")
+
+        #  Return new format string:
+        return self._sort_prefix() + "".join(self._fmt_tuple(*x) for x in format_list)
+
+    def set_sort_keys(self, sort_keys):
+        """
+        Override current sort keys used in print_items() which may have
+        been set by a `sort:` prefix.
+
+        Args:
+            sort_keys (str): Comma separated list of sort keys. Prefix a
+                key with `-` to reverse sort on that key.
+        """
+        #  Reset self.sort_keys:
+        self.sort_keys = []
+        for key in sort_keys.split(","):
+            key = key.strip()
+            reverse = False
+            if key.startswith("-"):
+                reverse = True
+                key = key[1:]
+            if key not in self.headings:
+                raise ValueError("Invalid sort key: " + key)
+            self.sort_keys.append((key, reverse))
+
+    def sort_items(self, items):
+        """
+        Sort items using sort_keys attribute.
+
+        Args:
+            items (list): list of items to sort. Note that the list is
+                sorted in place and then returned.
+        """
+        # Apply any requested sort:
+        for key, reverse in reversed(self.sort_keys):
+            items.sort(key=attrgetter(key), reverse=reverse)
+        return items
+
+    def print_items(self, items, no_header=False, pre=None, post=None):
+        """
+        Handle printing a list of items with the current format.
+
+        First pre-process format using ``items`` to remove any empty
+        fields, if requested. (The pre-processing step could be extended
+        in the future.)
+
+        Next, sort items via any sort keys as set by set_sort_keys() or
+        via a ``sort:`` prefix in the supplied format.
+
+        Then, generate a header unless no_header is True.
+
+        Finally output a formatted line for each provided item.
+
+        Args:
+            items (iterable): list of items to format
+            no_header (boolean): disable header row (default: False)
+            pre (callable): Function to call before printing each item
+            post (callable): Function to call after printing each item
+        """
+        #  Preprocess original format by processing with filter():
+        newfmt = self.filter(items)
+        #  Get the current class for creating a new formatter instance:
+        cls = self.__class__
+        #  Create new instance of the current class from filtered format:
+        formatter = cls(newfmt, headings=self.headings, prepend=self.prepend)
+
+        items = self.sort_items(items)
+
+        if not no_header:
+            print(formatter.header())
+        for item in items:
+            line = formatter.format(item)
+            if not line or line.isspace():
+                continue
+            if callable(pre):
+                pre(item)
+            try:
+                print(line)
+            except UnicodeEncodeError:
+                print(line.encode("utf-8", errors="surrogateescape").decode())
+            if callable(post):
+                post(item)
+
+
+class Deduplicator:
+    """
+    Generic helper to deduplicate a list of formatted items for objects
+    that can be aggregated.
+
+    Args:
+        formatter (OutputFormat): Formatter instance to use for deduplication
+        except_fields (list): List of fields to not consider when merging
+            like lines. These are typically fields that can be combined, such
+            as a node count, node list, ranks, etc.
+        combine (callable): A function that is used to combine matching
+            items, called as combine(existing, new).
+    """
+
+    def __init__(self, formatter, except_fields=None, combine=None):
+        self.formatter = formatter.copy(except_fields=except_fields)
+        self.combine = combine
+        self.hash = {}
+        self.items = []
+        #  Allow class to be iterable https://stackoverflow.com/a/48670014
+        self.__iter = None
+
+    def __iter__(self):
+        if self.__iter is None:
+            self.__iter = iter(self.items)
+        return self
+
+    def __next__(self):
+        try:
+            return next(self.__iter)
+        except StopIteration:  # support repeated iteration
+            self.__iter = None
+            raise
+
+    def append(self, item):
+        """
+        Append a new item to a deduplicator. Combines item with an existing
+        entry if the formatted result is identical to another entry.
+        """
+        key = self.formatter.format(item)
+        try:
+            result = self.hash[key]
+            if self.combine is not None:
+                self.combine(result, item)
+        except KeyError:
+            self.hash[key] = item
+            self.items.append(item)
+
+
+class Tree:
+    """Very simple pstree-like display for the console
+
+    Args:
+        label: Label for this node
+        prefix: Text which comes before connector and label
+        combine_children: combine like children as they are added
+    """
+
+    Connectors = namedtuple("Connectors", ["PIPE", "TEE", "ELBOW", "BLANK"])
+
+    connector_styles = {
+        "ascii": Connectors("|   ", "|-- ", "`-- ", "    "),
+        "box": Connectors("│   ", "├── ", "└── ", "    "),
+        "compact": Connectors("│ ", "├─", "└─", "  "),
+    }
+
+    def __init__(self, label, prefix="", combine_children=False):
+
+        self.label = label
+        self.prefix = prefix
+        if self.prefix:
+            self.prefix += " "
+
+        #  dictionary of prefix/label for duplicate child tracking:
+        self.duplicates = {}
+        self.duplicate_count = 1
+
+        self.children = []
+
+        #  True if attempt to combine duplicate children should be
+        #   made as they are added to the tree with self.append()
+        #   and self.append_tree():
+        self.combine_children = combine_children
+
+    def append(self, label, prefix=""):
+        """Append a new child, possibly combining duplicates
+
+        Args:
+            label: Label for this node
+            prefix: Optional prefix to display before tree part
+
+        Returns:
+            A new or existing Tree object (if duplicate)
+
+        """
+        if not self.combine_children:
+            return self.add(label, prefix, combine_children=False)
+
+        #  Create a key and check for existing duplicate children.
+        #  If found, simply increment the count of the existing node,
+        #  O/w, add a new child and new duplicate label
+        #
+        clabel = f"{prefix}{label}"
+        if clabel in self.duplicates:
+            self.duplicates[clabel].increment()
+        else:
+            tree = self.add(label, prefix, combine_children=True)
+            self.duplicates[clabel] = tree
+        return self.duplicates[clabel]
+
+    def append_tree(self, tree):
+        """Add a child Tree, combining duplicates
+
+        Args:
+            tree: the Tree object to append
+
+        Returns:
+            A new or existing Tree object (if duplicate)
+        """
+        if not self.combine_children or tree.children:
+            self.children.append(tree)
+            return tree
+        clabel = f"{tree.prefix}{tree.label}"
+        if clabel in self.duplicates:
+            self.duplicates[clabel].increment()
+        else:
+            self.children.append(tree)
+            self.duplicates[clabel] = tree
+        return self.duplicates[clabel]
+
+    def add(self, label, prefix="", combine_children=False):
+        """Add a child by prefix and label, does not combine duplicates
+
+        Args:
+            label: Label for this node
+            prefix: Optional prefix to display before tree part
+            combine_children: True if like children should be combined
+
+        Returns:
+            Tree: the new child Tree object
+        """
+        tree = Tree(label, prefix=prefix, combine_children=combine_children)
+        self.children.append(tree)
+        return tree
+
+    def increment(self):
+        self.duplicate_count = self.duplicate_count + 1
+
+    def _render(
+        self,
+        pstack=None,
+        connector="",
+        style="box",
+        max_level=None,
+        truncate=None,
+    ):
+
+        #  `pstack` is a stack of prefix characters used during
+        #   recursive walk of the Tree. This stack maintains the
+        #   extra prefix characters required before printing the
+        #   current tree connector and node label.
+        #
+        #  Note: the connector at the top of the stack is always
+        #   the connector required for the _next_ level down in the
+        #   tree. This is because we know what the prefix connector
+        #   should be at this level, not at the subsequent level.
+        #
+        if pstack is None:
+            pstack = []
+
+        if max_level is not None and len(pstack) > max_level:
+            return
+
+        if self.duplicate_count > 1:
+            label = f"{self.duplicate_count}*[{self.label}]"
+        else:
+            label = self.label
+
+        #  As noted above, we need to prefix this node with all characters
+        #   in `pstack` _except_ the character at the top of the stack,
+        #    which applies to the next level down.
+        #
+        prefix = "".join(pstack[:-1])
+
+        #  Generate the current line, truncating at 'truncate'
+        #   characters if necessary
+        #
+        result = f"{self.prefix}{prefix}{connector}{label}"
+        if truncate and len(result) > truncate:
+            result = result[: truncate - 1] + "+"
+
+        print(result)
+
+        cstyle = self.connector_styles[style]
+
+        for child in self.children:
+            if child == self.children[-1]:
+                #  If this is the last child, then push a BLANK prefix for
+                #   two levels down and an ELBLOW to connect this child
+                #   to its parent.
+                #
+                pstack.append(cstyle.BLANK)
+                connector = cstyle.ELBOW
+            else:
+                #  Otherwise, push a PIPE prefix for two levels down
+                #   and a TEE to connect this child to its parent.
+                #
+                pstack.append(cstyle.PIPE)
+                connector = cstyle.TEE
+
+            #  Finally, render this child and its children:
+            #
+            # pylint: disable=protected-access
+            child._render(
+                pstack,
+                connector=connector,
+                style=style,
+                max_level=max_level,
+                truncate=truncate,
+            )
+            pstack.pop()
+
+    def render(
+        self,
+        style="box",
+        level=None,
+        skip_root=False,
+        truncate=True,
+    ):
+        """Render a Tree to the console
+
+        Args:
+            style (str): style of connectors, "ascii" for ascii or "box" for
+                         unicode box drawing characters (default="box")
+            level (int): stop traversing at tree depth of ``level``.
+            skip_root(bool): Do not include root of tree in rendered output
+            truncate(bool): Chop long lines at current terminal width
+                            (or 132 columns if COLUMNS variable not set
+                            and current terminal width cannot be determined)
+        """
+        limit = None
+        if truncate:
+            limit = shutil.get_terminal_size((132, 25)).columns
+
+        if skip_root:
+            for child in self.children:
+                child.render(
+                    style=style,
+                    level=level,
+                    truncate=truncate,
+                )
+            return
+        self._render(
+            style=style,
+            max_level=level,
+            truncate=limit,
+        )
+
+
+#  Slightly modified from https://stackoverflow.com/a/7205107
+def dict_merge(src, new):
+    "merges dict new into dict src"
+    for key in new:
+        if key in src:
+            if isinstance(src[key], dict) and isinstance(new[key], dict):
+                dict_merge(src[key], new[key])
+            elif src[key] == new[key]:
+                pass  # same leaf value
+            else:
+                #  Default is to override:
+                src[key] = new[key]
+        else:
+            src[key] = new[key]
+    return src
+
+
+def xdg_searchpath(subdir=""):
+    """
+    Build standard Flux config search path based on XDG specification
+    """
+    #  Start with XDG_CONFIG_HOME (or ~/.config) since it is the
+    #  highest precedence:
+    confdirs = [os.getenv("XDG_CONFIG_HOME") or f"{Path.home()}/.config"]
+
+    #  Append XDG_CONFIG_DIRS as colon separated path (or /etc/xdg)
+    #  Note: colon separated paths are in order of precedence.
+    confdirs += (os.getenv("XDG_CONFIG_DIRS") or "/etc/xdg").split(":")
+
+    #  Append "/flux" (with optional subdir) to all members of
+    #  confdirs to build searchpath:
+    return [Path(directory, "flux", subdir) for directory in confdirs]
+
+
+class UtilConfig:
+    """
+    Very simple class for loading hierarchical configuration for Flux
+    Python utilities. Configuration is loaded as a dict from an optional
+    initial dict, overriding from XDG system and user base directories
+    in that order.
+
+    Config files are loaded from <name>.<ext>, where ext can be one
+    of json, yaml, or toml. If multiple files exist they are processed
+    in glob(3) order.
+
+    Args:
+        name: config name, used as the stem of config file to load
+              this is typically the name of the tool
+        toolname (optional): actual name of the tool/utility if "name" is
+                             different.  used in environment variable lookup.
+        subcommand (optional): name of subcommand. Used as name of subtable
+                               in configuration to use.
+        initial_dict: Set of default values (optional)
+
+    """
+
+    extension_handlers = {
+        ".toml": tomllib.load,
+        ".json": json.load,
+        ".yaml": yaml.safe_load,
+    }
+
+    builtin_formats = {}
+
+    def __init__(self, name, toolname=None, subcommand=None, initial_dict=None):
+        self.name = name
+        self.toolname = toolname
+        self.dict = {}
+        if initial_dict:
+            self.dict = copy.deepcopy(initial_dict)
+        self.config = self.dict
+
+        #  If not None,  use subcommand config in subtable = 'subtable'
+        self.subtable = subcommand
+
+        #  Build config search path in precedence order based on XDG
+        #  specification. Later this will be reversed since we want
+        #  to traverse the paths in _reverse_ precedence order in this
+        #  implementation.
+        self.searchpath = xdg_searchpath()
+
+        #  Reorder searchpath into reverse precedence order
+        self.searchpath.reverse()
+
+    def load(self):
+        """Load configuration from current searchpath
+
+        Returns self so that constructor and load() can be called like
+
+        >>> config = UtilConfig("myutil").load()
+
+        """
+
+        for path in self.searchpath:
+            for filepath in sorted(glob.glob(f"{path}/{self.name}.*")):
+                conf = {}
+                ppath = PurePosixPath(filepath)
+
+                # ignore files with unsupported extensions:
+                if ppath.suffix not in self.extension_handlers:
+                    continue
+
+                try:
+                    with open(filepath, "rb") as ofile:
+                        conf = self.extension_handlers[ppath.suffix](ofile)
+                except (
+                    tomllib.TOMLDecodeError,
+                    json.decoder.JSONDecodeError,
+                    yaml.scanner.ScannerError,
+                ) as exc:
+                    #  prepend file path to decode exceptions in case it
+                    #  it is missing (e.g. tomllib)
+                    raise ValueError(f"{filepath}: {exc}") from exc
+
+                self.validate(filepath, conf)
+
+                dict_merge(self.dict, conf)
+
+        # If a subtable is set, then update self.config:
+        if self.subtable and self.subtable in self.dict:
+            self.config = self.dict[self.subtable]
+
+        return self
+
+    def validate_formats(self, path, formats):
+        """
+        Convenience function to validate a set of formats in config loaded
+        from path in the common flux-* formats config schema, i.e a dictionary
+        of format names in the form:
+
+            { "name": {
+                "description": "format description",
+                "format":      "format",
+              },
+              "name2": ..
+            }
+
+        Raises ValueError on failure.
+        """
+        builtin_formats = self.builtin_formats
+        #  If we're working with a subtable, then choose the same
+        #  subtable from builtin_formats if it exists:
+        if self.subtable and self.subtable in self.builtin_formats:
+            builtin_formats = self.builtin_formats[self.subtable]
+
+        if not isinstance(formats, dict):
+            raise ValueError(f"{path}: the 'formats' key must be a mapping")
+        for name, entry in formats.items():
+            if name in builtin_formats:
+                raise ValueError(
+                    f"{path}: override of builtin format '{name}' not permitted"
+                )
+            if not isinstance(entry, dict):
+                raise ValueError(f"{path}: 'formats.{name}' must be a mapping")
+            if "format" not in entry:
+                raise ValueError(
+                    f"{path}: formats.{name}: required key 'format' missing"
+                )
+            if not isinstance(entry["format"], str):
+                raise ValueError(
+                    f"{path}: formats.{name}: 'format' key must be a string"
+                )
+
+    def validate(self, path, conf):
+        """
+        Validate config file as it is loaded before merging it with the
+        configuration. This function does nothing in the base class, but
+        subclasses may define it to implement higher level validation.
+        """
+
+    def _default_env_name(self):
+        """
+        Get tool default environment variable (name upper case, s/-/_/g)
+        """
+        name = self.toolname if self.toolname else self.name
+        prefix = name.upper().replace("-", "_")
+        if self.subtable:
+            prefix += "_" + self.subtable.upper().replace("-", "_")
+        return prefix + "_FORMAT_DEFAULT"
+
+    def get_format_string(self, format_name):
+        """
+        Convenience function to fetch a format string from the current
+        config. Assumes the current config has been loaded and contains
+        a "formats" table.
+
+        Special format names:
+            help: print a list of configured formats
+            get-config=NAME: dump the config for format "name"
+
+        Args:
+            format_name: Name of the configured format to return
+        """
+        if "{" in format_name:
+            return format_name
+
+        formats = self.formats
+        if format_name == "help":
+            print(f"\nConfigured {self.name} output formats:\n")
+            for name, entry in formats.items():
+                print(f"  {name:<12}", end="")
+                try:
+                    print(f" {entry['description']}")
+                except KeyError:
+                    print()
+            sys.exit(0)
+        elif format_name.startswith("get-config="):
+            _, name = format_name.split("=", 1)
+            try:
+                entry = formats[name]
+            except KeyError:
+                raise ValueError(f"--format: No such format {name}")
+            if self.subtable:
+                print(f"[{self.subtable}.formats.{name}]")
+            else:
+                print(f"[formats.{name}]")
+            if "description" in entry:
+                print(f"description = \"{entry['description']}\"")
+            print(f"format = \"{entry['format']}\"")
+            sys.exit(0)
+        elif format_name == "default":
+            # Default can be overridden by environment variable
+            try:
+                format_name = os.environ[self._default_env_name()]
+                if "{" in format_name:
+                    return format_name
+            except KeyError:
+                # no environment var, fallthrough
+                pass
+
+        try:
+            return formats[format_name]["format"]
+        except KeyError:
+            raise ValueError(f"--format: No such format {format_name}")
+
+    def __getattr__(self, attr):
+        return self.config[attr]
+
+
+class Fileref(dict):
+    """
+    Construct a RFC 37 data file object from a data parameter and optional
+    permissions and encoding. ``Fileref`` is a subclass of ``dict`` so it
+    may be used in place of a dict and is directly serializable to JSON.
+
+    If ``data`` is a dict, then a file object with no encoding is created
+    and the dict is stored in ``data`` for eventual encoding as JSON
+    content. Otherwise, if ``encoding`` is set, then data is presumed to
+    already be encoded and is stored verbatim.  If data is not a dict,
+    and encoding is not set, then data is assumed to be the path to file
+    in the filesystem, in which case the encoding will be 'utf-8' if the
+    file contents can be encoded as such, otherwise 'base64'.
+
+    Args:
+        data (dict, str, bytes): File data or path as explained above.
+        perms: File permissions (default 0600 octal)
+        encoding: Explicit encoding if `data` is not a dict or filename.
+    """
+
+    def __init__(self, data, perms=0o0600, encoding=None):
+        ref = {"mode": perms | stat.S_IFREG}
+        if isinstance(data, dict):
+            ref["data"] = data
+        elif encoding is not None:
+            if encoding not in ("base64", "utf-8"):
+                raise ValueError("invalid encoding: {encoding}")
+            ref["data"] = data
+            ref["encoding"] = encoding
+        else:
+            st = os.stat(data)
+            ref["size"] = st.st_size
+            ref["mode"] = st.st_mode
+            with open(data, "rb") as infp:
+                try:
+                    ref["data"] = infp.read().decode("utf-8")
+                    ref["encoding"] = "utf-8"
+                except UnicodeError:
+                    infp.seek(0)
+                    ref["data"] = base64.b64encode(infp.read()).decode("utf-8")
+                    ref["encoding"] = "base64"
+        super().__init__(**ref)
diff --git a/src/bindings/python/flux/utils/README.md b/src/bindings/python/flux/utils/README.md
new file mode 100644
index 000000000000..ceade1b214ff
--- /dev/null
+++ b/src/bindings/python/flux/utils/README.md
@@ -0,0 +1,6 @@
+Place vendored Python modules here.
+
+Record commit hash of included version for convenience.
+
+parsedatetime c55337589ee582813182b74f2d3ae80e2fcd9738
+tomi	      345bd2a224f215fbb33546b6d7e358307b783793
diff --git a/src/bindings/python/flux/utils/parsedatetime/__init__.py b/src/bindings/python/flux/utils/parsedatetime/__init__.py
new file mode 100644
index 000000000000..654476dca976
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/__init__.py
@@ -0,0 +1,2789 @@
+# -*- coding: utf-8 -*-
+#
+# vim: sw=2 ts=2 sts=2
+#
+# Copyright 2004-2019 Mike Taylor
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""parsedatetime
+
+Parse human-readable date/time text.
+
+Requires Python 2.7 or later
+"""
+
+from __future__ import with_statement, absolute_import, unicode_literals
+
+import re
+import time
+import logging
+import warnings
+import datetime
+import calendar
+import contextlib
+import email.utils
+
+from .pdt_locales import (locales as _locales,
+                          get_icu, load_locale)
+from .context import pdtContext, pdtContextStack
+from .warns import pdt20DeprecationWarning
+
+
+__author__ = 'Mike Taylor'
+__email__ = 'bear@bear.im'
+__copyright__ = 'Copyright (c) 2017 Mike Taylor'
+__license__ = 'Apache License 2.0'
+__version__ = '2.6'
+__url__ = 'https://github.com/bear/parsedatetime'
+__download_url__ = 'https://pypi.python.org/pypi/parsedatetime'
+__description__ = 'Parse human-readable date/time text.'
+
+# as a library, do *not* setup logging
+# see docs.python.org/2/howto/logging.html#configuring-logging-for-a-library
+# Set default logging handler to avoid "No handler found" warnings.
+
+try:  # Python 2.7+
+    from logging import NullHandler
+except ImportError:
+    class NullHandler(logging.Handler):
+
+        def emit(self, record):
+            pass
+
+log = logging.getLogger(__name__)
+log.addHandler(NullHandler())
+
+debug = False
+
+pdtLocales = dict([(x, load_locale(x)) for x in _locales])
+
+
+# Copied from feedparser.py
+# Universal Feedparser
+# Copyright (c) 2002-2006, Mark Pilgrim, All rights reserved.
+# Originally a def inside of _parse_date_w3dtf()
+def _extract_date(m):
+    year = int(m.group('year'))
+    if year < 100:
+        year = 100 * int(time.gmtime()[0] / 100) + int(year)
+    if year < 1000:
+        return 0, 0, 0
+    julian = m.group('julian')
+    if julian:
+        julian = int(julian)
+        month = julian / 30 + 1
+        day = julian % 30 + 1
+        jday = None
+        while jday != julian:
+            t = time.mktime((year, month, day, 0, 0, 0, 0, 0, 0))
+            jday = time.gmtime(t)[-2]
+            diff = abs(jday - julian)
+            if jday > julian:
+                if diff < day:
+                    day = day - diff
+                else:
+                    month = month - 1
+                    day = 31
+            elif jday < julian:
+                if day + diff < 28:
+                    day = day + diff
+                else:
+                    month = month + 1
+        return year, month, day
+    month = m.group('month')
+    day = 1
+    if month is None:
+        month = 1
+    else:
+        month = int(month)
+        day = m.group('day')
+        if day:
+            day = int(day)
+        else:
+            day = 1
+    return year, month, day
+
+
+# Copied from feedparser.py
+# Universal Feedparser
+# Copyright (c) 2002-2006, Mark Pilgrim, All rights reserved.
+# Originally a def inside of _parse_date_w3dtf()
+def _extract_time(m):
+    if not m:
+        return 0, 0, 0
+    hours = m.group('hours')
+    if not hours:
+        return 0, 0, 0
+    hours = int(hours)
+    minutes = int(m.group('minutes'))
+    seconds = m.group('seconds')
+    if seconds:
+        seconds = seconds.replace(',', '.').split('.', 1)[0]
+        seconds = int(seconds)
+    else:
+        seconds = 0
+    return hours, minutes, seconds
+
+
+def _pop_time_accuracy(m, ctx):
+    if not m:
+        return
+    if m.group('hours'):
+        ctx.updateAccuracy(ctx.ACU_HOUR)
+    if m.group('minutes'):
+        ctx.updateAccuracy(ctx.ACU_MIN)
+    if m.group('seconds'):
+        ctx.updateAccuracy(ctx.ACU_SEC)
+
+
+# Copied from feedparser.py
+# Universal Feedparser
+# Copyright (c) 2002-2006, Mark Pilgrim, All rights reserved.
+# Modified to return a tuple instead of mktime
+#
+# Original comment:
+#   W3DTF-style date parsing adapted from PyXML xml.utils.iso8601, written by
+#   Drake and licensed under the Python license.  Removed all range checking
+#   for month, day, hour, minute, and second, since mktime will normalize
+#   these later
+def __closure_parse_date_w3dtf():
+    # the __extract_date and __extract_time methods were
+    # copied-out so they could be used by my code --bear
+    def __extract_tzd(m):
+        '''Return the Time Zone Designator as an offset in seconds from UTC.'''
+        if not m:
+            return 0
+        tzd = m.group('tzd')
+        if not tzd:
+            return 0
+        if tzd == 'Z':
+            return 0
+        hours = int(m.group('tzdhours'))
+        minutes = m.group('tzdminutes')
+        if minutes:
+            minutes = int(minutes)
+        else:
+            minutes = 0
+        offset = (hours * 60 + minutes) * 60
+        if tzd[0] == '+':
+            return -offset
+        return offset
+
+    def _parse_date_w3dtf(dateString):
+        m = __datetime_rx.match(dateString)
+        if m is None or m.group() != dateString:
+            return
+        return _extract_date(m) + _extract_time(m) + (0, 0, 0)
+
+    __date_re = (r'(?P<year>\d\d\d\d)'
+                 r'(?:(?P<dsep>-|)'
+                 r'(?:(?P<julian>\d\d\d)'
+                 r'|(?P<month>\d\d)(?:(?P=dsep)(?P<day>\d\d))?))?')
+    __tzd_re = r'(?P<tzd>[-+](?P<tzdhours>\d\d)(?::?(?P<tzdminutes>\d\d))|Z)'
+    # __tzd_rx = re.compile(__tzd_re)
+    __time_re = (r'(?P<hours>\d\d)(?P<tsep>:|)(?P<minutes>\d\d)'
+                 r'(?:(?P=tsep)(?P<seconds>\d\d(?:[.,]\d+)?))?' + __tzd_re)
+    __datetime_re = '%s(?:T%s)?' % (__date_re, __time_re)
+    __datetime_rx = re.compile(__datetime_re)
+
+    return _parse_date_w3dtf
+
+
+_parse_date_w3dtf = __closure_parse_date_w3dtf()
+del __closure_parse_date_w3dtf
+
+_monthnames = set([
+    'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul',
+    'aug', 'sep', 'oct', 'nov', 'dec',
+    'january', 'february', 'march', 'april', 'may', 'june', 'july',
+    'august', 'september', 'october', 'november', 'december'])
+_daynames = set(['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'])
+
+
+# Copied from feedparser.py
+# Universal Feedparser
+# Copyright (c) 2002-2006, Mark Pilgrim, All rights reserved.
+# Modified to return a tuple instead of mktime
+def _parse_date_rfc822(dateString):
+    '''Parse an RFC822, RFC1123, RFC2822, or asctime-style date'''
+    data = dateString.split()
+    if data[0][-1] in (',', '.') or data[0].lower() in _daynames:
+        del data[0]
+    if len(data) == 4:
+        s = data[3]
+        s = s.split('+', 1)
+        if len(s) == 2:
+            data[3:] = s
+        else:
+            data.append('')
+        dateString = " ".join(data)
+    if len(data) < 5:
+        dateString += ' 00:00:00 GMT'
+    return email.utils.parsedate_tz(dateString)
+
+
+# rfc822.py defines several time zones, but we define some extra ones.
+# 'ET' is equivalent to 'EST', etc.
+# _additional_timezones = {'AT': -400, 'ET': -500,
+#                          'CT': -600, 'MT': -700,
+#                          'PT': -800}
+# email.utils._timezones.update(_additional_timezones)
+
+VERSION_FLAG_STYLE = 1
+VERSION_CONTEXT_STYLE = 2
+
+
+class Calendar(object):
+
+    """
+    A collection of routines to input, parse and manipulate date and times.
+    The text can either be 'normal' date values or it can be human readable.
+    """
+
+    def __init__(self, constants=None, version=VERSION_FLAG_STYLE):
+        """
+        Default constructor for the L{Calendar} class.
+
+        @type  constants: object
+        @param constants: Instance of the class L{Constants}
+        @type  version:   integer
+        @param version:   Default style version of current Calendar instance.
+                          Valid value can be 1 (L{VERSION_FLAG_STYLE}) or
+                          2 (L{VERSION_CONTEXT_STYLE}). See L{parse()}.
+
+        @rtype:  object
+        @return: L{Calendar} instance
+        """
+        # if a constants reference is not included, use default
+        if constants is None:
+            self.ptc = Constants()
+        else:
+            self.ptc = constants
+
+        self.version = version
+        if version == VERSION_FLAG_STYLE:
+            warnings.warn(
+                'Flag style will be deprecated in parsedatetime 2.0. '
+                'Instead use the context style by instantiating `Calendar()` '
+                'with argument `version=parsedatetime.VERSION_CONTEXT_STYLE`.',
+                pdt20DeprecationWarning)
+        self._ctxStack = pdtContextStack()
+
+    @contextlib.contextmanager
+    def context(self):
+        ctx = pdtContext()
+        self._ctxStack.push(ctx)
+        yield ctx
+        ctx = self._ctxStack.pop()
+        if not self._ctxStack.isEmpty():
+            self.currentContext.update(ctx)
+
+    @property
+    def currentContext(self):
+        return self._ctxStack.last()
+
+    def _convertUnitAsWords(self, unitText):
+        """
+        Converts text units into their number value.
+
+        @type  unitText: string
+        @param unitText: number text to convert
+
+        @rtype:  integer
+        @return: numerical value of unitText
+        """
+        word_list, a, b = re.split(r"[,\s-]+", unitText), 0, 0
+        for word in word_list:
+            x = self.ptc.small.get(word)
+            if x is not None:
+                a += x
+            elif word == "hundred":
+                a *= 100
+            else:
+                x = self.ptc.magnitude.get(word)
+                if x is not None:
+                    b += a * x
+                    a = 0
+                elif word in self.ptc.ignore:
+                    pass
+                else:
+                    raise Exception("Unknown number: " + word)
+        return a + b
+
+    def _buildTime(self, source, quantity, modifier, units):
+        """
+        Take C{quantity}, C{modifier} and C{unit} strings and convert them
+        into values. After converting, calcuate the time and return the
+        adjusted sourceTime.
+
+        @type  source:   time
+        @param source:   time to use as the base (or source)
+        @type  quantity: string
+        @param quantity: quantity string
+        @type  modifier: string
+        @param modifier: how quantity and units modify the source time
+        @type  units:    string
+        @param units:    unit of the quantity (i.e. hours, days, months, etc)
+
+        @rtype:  struct_time
+        @return: C{struct_time} of the calculated time
+        """
+        ctx = self.currentContext
+        debug and log.debug('_buildTime: [%s][%s][%s]',
+                            quantity, modifier, units)
+
+        if source is None:
+            source = time.localtime()
+
+        if quantity is None:
+            quantity = ''
+        else:
+            quantity = quantity.strip()
+
+        qty = self._quantityToReal(quantity)
+
+        if modifier in self.ptc.Modifiers:
+            qty = qty * self.ptc.Modifiers[modifier]
+
+            if units is None or units == '':
+                units = 'dy'
+
+        # plurals are handled by regex's (could be a bug tho)
+
+        (yr, mth, dy, hr, mn, sec, _, _, _) = source
+
+        start = datetime.datetime(yr, mth, dy, hr, mn, sec)
+        target = start
+        # realunit = next((key for key, values in self.ptc.units.items()
+        #                  if any(imap(units.__contains__, values))), None)
+        realunit = units
+        for key, values in self.ptc.units.items():
+            if units in values:
+                realunit = key
+                break
+
+        debug and log.debug('units %s --> realunit %s (qty=%s)',
+                            units, realunit, qty)
+
+        try:
+            if realunit in ('years', 'months'):
+                target = self.inc(start, **{realunit[:-1]: qty})
+            elif realunit in ('days', 'hours', 'minutes', 'seconds', 'weeks'):
+                delta = datetime.timedelta(**{realunit: qty})
+                target = start + delta
+        except OverflowError:
+            # OverflowError is raise when target.year larger than 9999
+            pass
+        else:
+            ctx.updateAccuracy(realunit)
+
+        return target.timetuple()
+
+    def parseDate(self, dateString, sourceTime=None):
+        """
+        Parse short-form date strings::
+
+            '05/28/2006' or '04.21'
+
+        @type  dateString: string
+        @param dateString: text to convert to a C{datetime}
+        @type  sourceTime:     struct_time
+        @param sourceTime:     C{struct_time} value to use as the base
+
+        @rtype:  struct_time
+        @return: calculated C{struct_time} value of dateString
+        """
+        if sourceTime is None:
+            yr, mth, dy, hr, mn, sec, wd, yd, isdst = time.localtime()
+        else:
+            yr, mth, dy, hr, mn, sec, wd, yd, isdst = sourceTime
+
+        # values pulled from regex's will be stored here and later
+        # assigned to mth, dy, yr based on information from the locale
+        # -1 is used as the marker value because we want zero values
+        # to be passed thru so they can be flagged as errors later
+        v1 = -1
+        v2 = -1
+        v3 = -1
+        accuracy = []
+
+        s = dateString
+        m = self.ptc.CRE_DATE2.search(s)
+        if m is not None:
+            index = m.start()
+            v1 = int(s[:index])
+            s = s[index + 1:]
+
+        m = self.ptc.CRE_DATE2.search(s)
+        if m is not None:
+            index = m.start()
+            v2 = int(s[:index])
+            v3 = int(s[index + 1:])
+        else:
+            v2 = int(s.strip())
+
+        v = [v1, v2, v3]
+        d = {'m': mth, 'd': dy, 'y': yr}
+
+        # yyyy/mm/dd format
+        dp_order = self.ptc.dp_order if v1 <= 31 else ['y', 'm', 'd']
+
+        for i in range(0, 3):
+            n = v[i]
+            c = dp_order[i]
+            if n >= 0:
+                d[c] = n
+                accuracy.append({'m': pdtContext.ACU_MONTH,
+                                 'd': pdtContext.ACU_DAY,
+                                 'y': pdtContext.ACU_YEAR}[c])
+
+        # if the year is not specified and the date has already
+        # passed, increment the year
+        if v3 == -1 and ((mth > d['m']) or (mth == d['m'] and dy > d['d'])):
+            yr = d['y'] + self.ptc.YearParseStyle
+        else:
+            yr = d['y']
+
+        mth = d['m']
+        dy = d['d']
+
+        # birthday epoch constraint
+        if yr < self.ptc.BirthdayEpoch:
+            yr += 2000
+        elif yr < 100:
+            yr += 1900
+
+        daysInCurrentMonth = self.ptc.daysInMonth(mth, yr)
+        debug and log.debug('parseDate: %s %s %s %s',
+                            yr, mth, dy, daysInCurrentMonth)
+
+        with self.context() as ctx:
+            if mth > 0 and mth <= 12 and dy > 0 and \
+                    dy <= daysInCurrentMonth:
+                sourceTime = (yr, mth, dy, hr, mn, sec, wd, yd, isdst)
+                ctx.updateAccuracy(*accuracy)
+            else:
+                # return current time if date string is invalid
+                sourceTime = time.localtime()
+
+        return sourceTime
+
+    def parseDateText(self, dateString, sourceTime=None):
+        """
+        Parse long-form date strings::
+
+            'May 31st, 2006'
+            'Jan 1st'
+            'July 2006'
+
+        @type  dateString: string
+        @param dateString: text to convert to a datetime
+        @type  sourceTime:     struct_time
+        @param sourceTime:     C{struct_time} value to use as the base
+
+        @rtype:  struct_time
+        @return: calculated C{struct_time} value of dateString
+        """
+        if sourceTime is None:
+            yr, mth, dy, hr, mn, sec, wd, yd, isdst = time.localtime()
+        else:
+            yr, mth, dy, hr, mn, sec, wd, yd, isdst = sourceTime
+
+        currentMth = mth
+        currentDy = dy
+        accuracy = []
+
+        debug and log.debug('parseDateText currentMth %s currentDy %s',
+                            mth, dy)
+
+        s = dateString.lower()
+        m = self.ptc.CRE_DATE3.search(s)
+        mth = m.group('mthname')
+        mth = self.ptc.MonthOffsets[mth]
+        accuracy.append('month')
+
+        if m.group('day') is not None:
+            dy = int(m.group('day'))
+            accuracy.append('day')
+        else:
+            dy = 1
+
+        if m.group('year') is not None:
+            yr = int(m.group('year'))
+            accuracy.append('year')
+
+            # birthday epoch constraint
+            if yr < self.ptc.BirthdayEpoch:
+                yr += 2000
+            elif yr < 100:
+                yr += 1900
+
+        elif (mth < currentMth) or (mth == currentMth and dy < currentDy):
+            # if that day and month have already passed in this year,
+            # then increment the year by 1
+            yr += self.ptc.YearParseStyle
+
+        with self.context() as ctx:
+            if dy > 0 and dy <= self.ptc.daysInMonth(mth, yr):
+                sourceTime = (yr, mth, dy, hr, mn, sec, wd, yd, isdst)
+                ctx.updateAccuracy(*accuracy)
+            else:
+                # Return current time if date string is invalid
+                sourceTime = time.localtime()
+
+        debug and log.debug('parseDateText returned '
+                            'mth %d dy %d yr %d sourceTime %s',
+                            mth, dy, yr, sourceTime)
+
+        return sourceTime
+
+    def evalRanges(self, datetimeString, sourceTime=None):
+        """
+        Evaluate the C{datetimeString} text and determine if
+        it represents a date or time range.
+
+        @type  datetimeString: string
+        @param datetimeString: datetime text to evaluate
+        @type  sourceTime:     struct_time
+        @param sourceTime:     C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of: start datetime, end datetime and the invalid flag
+        """
+        rangeFlag = retFlag = 0
+        startStr = endStr = ''
+
+        s = datetimeString.strip().lower()
+
+        if self.ptc.rangeSep in s:
+            s = s.replace(self.ptc.rangeSep, ' %s ' % self.ptc.rangeSep)
+            s = s.replace('  ', ' ')
+
+        for cre, rflag in [(self.ptc.CRE_TIMERNG1, 1),
+                           (self.ptc.CRE_TIMERNG2, 2),
+                           (self.ptc.CRE_TIMERNG4, 7),
+                           (self.ptc.CRE_TIMERNG3, 3),
+                           (self.ptc.CRE_DATERNG1, 4),
+                           (self.ptc.CRE_DATERNG2, 5),
+                           (self.ptc.CRE_DATERNG3, 6)]:
+            m = cre.search(s)
+            if m is not None:
+                rangeFlag = rflag
+                break
+
+        debug and log.debug('evalRanges: rangeFlag = %s [%s]', rangeFlag, s)
+
+        if m is not None:
+            if (m.group() != s):
+                # capture remaining string
+                parseStr = m.group()
+                chunk1 = s[:m.start()]
+                chunk2 = s[m.end():]
+                s = '%s %s' % (chunk1, chunk2)
+
+                sourceTime, ctx = self.parse(s, sourceTime,
+                                             VERSION_CONTEXT_STYLE)
+
+                if not ctx.hasDateOrTime:
+                    sourceTime = None
+            else:
+                parseStr = s
+
+        if rangeFlag in (1, 2):
+            m = re.search(self.ptc.rangeSep, parseStr)
+            startStr = parseStr[:m.start()]
+            endStr = parseStr[m.start() + 1:]
+            retFlag = 2
+
+        elif rangeFlag in (3, 7):
+            m = re.search(self.ptc.rangeSep, parseStr)
+            # capturing the meridian from the end time
+            if self.ptc.usesMeridian:
+                ampm = re.search(self.ptc.am[0], parseStr)
+
+                # appending the meridian to the start time
+                if ampm is not None:
+                    startStr = parseStr[:m.start()] + self.ptc.meridian[0]
+                else:
+                    startStr = parseStr[:m.start()] + self.ptc.meridian[1]
+            else:
+                startStr = parseStr[:m.start()]
+
+            endStr = parseStr[m.start() + 1:]
+            retFlag = 2
+
+        elif rangeFlag == 4:
+            m = re.search(self.ptc.rangeSep, parseStr)
+            startStr = parseStr[:m.start()]
+            endStr = parseStr[m.start() + 1:]
+            retFlag = 1
+
+        elif rangeFlag == 5:
+            m = re.search(self.ptc.rangeSep, parseStr)
+            endStr = parseStr[m.start() + 1:]
+
+            # capturing the year from the end date
+            date = self.ptc.CRE_DATE3.search(endStr)
+            endYear = date.group('year')
+
+            # appending the year to the start date if the start date
+            # does not have year information and the end date does.
+            # eg : "Aug 21 - Sep 4, 2007"
+            if endYear is not None:
+                startStr = (parseStr[:m.start()]).strip()
+                date = self.ptc.CRE_DATE3.search(startStr)
+                startYear = date.group('year')
+
+                if startYear is None:
+                    startStr = startStr + ', ' + endYear
+            else:
+                startStr = parseStr[:m.start()]
+
+            retFlag = 1
+
+        elif rangeFlag == 6:
+            m = re.search(self.ptc.rangeSep, parseStr)
+
+            startStr = parseStr[:m.start()]
+
+            # capturing the month from the start date
+            mth = self.ptc.CRE_DATE3.search(startStr)
+            mth = mth.group('mthname')
+
+            # appending the month name to the end date
+            endStr = mth + parseStr[(m.start() + 1):]
+
+            retFlag = 1
+
+        else:
+            # if range is not found
+            startDT = endDT = time.localtime()
+
+        if retFlag:
+            startDT, sctx = self.parse(startStr, sourceTime,
+                                       VERSION_CONTEXT_STYLE)
+            endDT, ectx = self.parse(endStr, sourceTime,
+                                     VERSION_CONTEXT_STYLE)
+
+            if not sctx.hasDateOrTime or not ectx.hasDateOrTime:
+                retFlag = 0
+
+        return startDT, endDT, retFlag
+
+    def _CalculateDOWDelta(self, wd, wkdy, offset, style, currentDayStyle):
+        """
+        Based on the C{style} and C{currentDayStyle} determine what
+        day-of-week value is to be returned.
+
+        @type  wd:              integer
+        @param wd:              day-of-week value for the current day
+        @type  wkdy:            integer
+        @param wkdy:            day-of-week value for the parsed day
+        @type  offset:          integer
+        @param offset:          offset direction for any modifiers (-1, 0, 1)
+        @type  style:           integer
+        @param style:           normally the value
+                                set in C{Constants.DOWParseStyle}
+        @type  currentDayStyle: integer
+        @param currentDayStyle: normally the value
+                                set in C{Constants.CurrentDOWParseStyle}
+
+        @rtype:  integer
+        @return: calculated day-of-week
+        """
+        diffBase = wkdy - wd
+        origOffset = offset
+
+        if offset == 2:
+            # no modifier is present.
+            # i.e. string to be parsed is just DOW
+            if wkdy * style > wd * style or \
+                    currentDayStyle and wkdy == wd:
+                # wkdy located in current week
+                offset = 0
+            elif style in (-1, 1):
+                # wkdy located in last (-1) or next (1) week
+                offset = style
+            else:
+                # invalid style, or should raise error?
+                offset = 0
+
+        # offset = -1 means last week
+        # offset = 0 means current week
+        # offset = 1 means next week
+        diff = diffBase + 7 * offset
+        if style == 1 and diff < -7:
+            diff += 7
+        elif style == -1 and diff > 7:
+            diff -= 7
+
+        debug and log.debug("wd %s, wkdy %s, offset %d, "
+                            "style %d, currentDayStyle %d",
+                            wd, wkdy, origOffset, style, currentDayStyle)
+
+        return diff
+
+    def _quantityToReal(self, quantity):
+        """
+        Convert a quantity, either spelled-out or numeric, to a float
+
+        @type    quantity: string
+        @param   quantity: quantity to parse to float
+        @rtype:  int
+        @return: the quantity as an float, defaulting to 0.0
+        """
+        if not quantity:
+            return 1.0
+
+        try:
+            return float(quantity.replace(',', '.'))
+        except ValueError:
+            pass
+
+        try:
+            return float(self.ptc.numbers[quantity])
+        except KeyError:
+            pass
+
+        return 0.0
+
+    def _evalModifier(self, modifier, chunk1, chunk2, sourceTime):
+        """
+        Evaluate the C{modifier} string and following text (passed in
+        as C{chunk1} and C{chunk2}) and if they match any known modifiers
+        calculate the delta and apply it to C{sourceTime}.
+
+        @type  modifier:   string
+        @param modifier:   modifier text to apply to sourceTime
+        @type  chunk1:     string
+        @param chunk1:     text chunk that preceded modifier (if any)
+        @type  chunk2:     string
+        @param chunk2:     text chunk that followed modifier (if any)
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of: remaining text and the modified sourceTime
+        """
+        ctx = self.currentContext
+        offset = self.ptc.Modifiers[modifier]
+
+        if sourceTime is not None:
+            (yr, mth, dy, hr, mn, sec, wd, yd, isdst) = sourceTime
+        else:
+            (yr, mth, dy, hr, mn, sec, wd, yd, isdst) = time.localtime()
+
+        if self.ptc.StartTimeFromSourceTime:
+            startHour = hr
+            startMinute = mn
+            startSecond = sec
+        else:
+            startHour = self.ptc.StartHour
+            startMinute = 0
+            startSecond = 0
+
+        # capture the units after the modifier and the remaining
+        # string after the unit
+        m = self.ptc.CRE_REMAINING.search(chunk2)
+        if m is not None:
+            index = m.start() + 1
+            unit = chunk2[:m.start()]
+            chunk2 = chunk2[index:]
+        else:
+            unit = chunk2
+            chunk2 = ''
+
+        debug and log.debug("modifier [%s] chunk1 [%s] "
+                            "chunk2 [%s] unit [%s]",
+                            modifier, chunk1, chunk2, unit)
+
+        if unit in self.ptc.units['months']:
+            currentDaysInMonth = self.ptc.daysInMonth(mth, yr)
+            if offset == 0:
+                dy = currentDaysInMonth
+                sourceTime = (yr, mth, dy, startHour, startMinute,
+                              startSecond, wd, yd, isdst)
+            elif offset == 2:
+                # if day is the last day of the month, calculate the last day
+                # of the next month
+                if dy == currentDaysInMonth:
+                    dy = self.ptc.daysInMonth(mth + 1, yr)
+
+                start = datetime.datetime(yr, mth, dy, startHour,
+                                          startMinute, startSecond)
+                target = self.inc(start, month=1)
+                sourceTime = target.timetuple()
+            else:
+                start = datetime.datetime(yr, mth, 1, startHour,
+                                          startMinute, startSecond)
+                target = self.inc(start, month=offset)
+                sourceTime = target.timetuple()
+            ctx.updateAccuracy(ctx.ACU_MONTH)
+
+        elif unit in self.ptc.units['weeks']:
+            if offset == 0:
+                start = datetime.datetime(yr, mth, dy, 17, 0, 0)
+                target = start + datetime.timedelta(days=(4 - wd))
+                sourceTime = target.timetuple()
+            elif offset == 2:
+                start = datetime.datetime(yr, mth, dy, startHour,
+                                          startMinute, startSecond)
+                target = start + datetime.timedelta(days=7)
+                sourceTime = target.timetuple()
+            else:
+                start = datetime.datetime(yr, mth, dy, startHour,
+                                          startMinute, startSecond)
+                target = start + offset * datetime.timedelta(weeks=1)
+                sourceTime = target.timetuple()
+            ctx.updateAccuracy(ctx.ACU_WEEK)
+
+        elif unit in self.ptc.units['days']:
+            if offset == 0:
+                sourceTime = (yr, mth, dy, 17, 0, 0, wd, yd, isdst)
+                ctx.updateAccuracy(ctx.ACU_HALFDAY)
+            elif offset == 2:
+                start = datetime.datetime(yr, mth, dy, hr, mn, sec)
+                target = start + datetime.timedelta(days=1)
+                sourceTime = target.timetuple()
+            else:
+                start = datetime.datetime(yr, mth, dy, startHour,
+                                          startMinute, startSecond)
+                target = start + datetime.timedelta(days=offset)
+                sourceTime = target.timetuple()
+            ctx.updateAccuracy(ctx.ACU_DAY)
+
+        elif unit in self.ptc.units['hours']:
+            if offset == 0:
+                sourceTime = (yr, mth, dy, hr, 0, 0, wd, yd, isdst)
+            else:
+                start = datetime.datetime(yr, mth, dy, hr, 0, 0)
+                target = start + datetime.timedelta(hours=offset)
+                sourceTime = target.timetuple()
+            ctx.updateAccuracy(ctx.ACU_HOUR)
+
+        elif unit in self.ptc.units['years']:
+            if offset == 0:
+                sourceTime = (yr, 12, 31, hr, mn, sec, wd, yd, isdst)
+            elif offset == 2:
+                sourceTime = (yr + 1, mth, dy, hr, mn, sec, wd, yd, isdst)
+            else:
+                sourceTime = (yr + offset, 1, 1, startHour, startMinute,
+                              startSecond, wd, yd, isdst)
+            ctx.updateAccuracy(ctx.ACU_YEAR)
+
+        elif modifier == 'eom':
+            dy = self.ptc.daysInMonth(mth, yr)
+            sourceTime = (yr, mth, dy, startHour, startMinute,
+                          startSecond, wd, yd, isdst)
+            ctx.updateAccuracy(ctx.ACU_DAY)
+
+        elif modifier == 'eoy':
+            mth = 12
+            dy = self.ptc.daysInMonth(mth, yr)
+            sourceTime = (yr, mth, dy, startHour, startMinute,
+                          startSecond, wd, yd, isdst)
+            ctx.updateAccuracy(ctx.ACU_MONTH)
+
+        elif self.ptc.CRE_WEEKDAY.match(unit):
+            m = self.ptc.CRE_WEEKDAY.match(unit)
+            debug and log.debug('CRE_WEEKDAY matched')
+            wkdy = m.group()
+
+            if modifier == 'eod':
+                ctx.updateAccuracy(ctx.ACU_HOUR)
+                # Calculate the upcoming weekday
+                sourceTime, subctx = self.parse(wkdy, sourceTime,
+                                                VERSION_CONTEXT_STYLE)
+                sTime = self.ptc.getSource(modifier, sourceTime)
+                if sTime is not None:
+                    sourceTime = sTime
+                    ctx.updateAccuracy(ctx.ACU_HALFDAY)
+            else:
+                # unless one of these modifiers is being applied to the
+                # day-of-week, we want to start with target as the day
+                # in the current week.
+                dowOffset = offset
+                relativeModifier = modifier not in ['this', 'next', 'last', 'prior', 'previous']
+                if relativeModifier:
+                    dowOffset = 0
+
+                wkdy = self.ptc.WeekdayOffsets[wkdy]
+                diff = self._CalculateDOWDelta(
+                    wd, wkdy, dowOffset, self.ptc.DOWParseStyle,
+                    self.ptc.CurrentDOWParseStyle)
+                start = datetime.datetime(yr, mth, dy, startHour,
+                                          startMinute, startSecond)
+                target = start + datetime.timedelta(days=diff)
+
+                if chunk1 != '' and relativeModifier:
+                    # consider "one day before thursday": we need to parse chunk1 ("one day")
+                    # and apply according to the offset ("before"), rather than allowing the
+                    # remaining parse step to apply "one day" without the offset direction.
+                    t, subctx = self.parse(chunk1, sourceTime, VERSION_CONTEXT_STYLE)
+                    if subctx.hasDateOrTime:
+                        delta = time.mktime(t) - time.mktime(sourceTime)
+                        target = start + datetime.timedelta(days=diff) + datetime.timedelta(seconds=delta * offset)
+                        chunk1 = ''
+
+                sourceTime = target.timetuple()
+            ctx.updateAccuracy(ctx.ACU_DAY)
+
+        elif chunk1 == '' and chunk2 == '' and self.ptc.CRE_TIME.match(unit):
+            m = self.ptc.CRE_TIME.match(unit)
+            debug and log.debug('CRE_TIME matched')
+            (yr, mth, dy, hr, mn, sec, wd, yd, isdst), subctx = \
+                self.parse(unit, None, VERSION_CONTEXT_STYLE)
+
+            start = datetime.datetime(yr, mth, dy, hr, mn, sec)
+            target = start + datetime.timedelta(days=offset)
+            sourceTime = target.timetuple()
+
+        else:
+            # check if the remaining text is parsable and if so,
+            # use it as the base time for the modifier source time
+
+            debug and log.debug('check for modifications '
+                                'to source time [%s] [%s]',
+                                chunk1, unit)
+
+            unit = unit.strip()
+            if unit:
+                s = '%s %s' % (unit, chunk2)
+                t, subctx = self.parse(s, sourceTime, VERSION_CONTEXT_STYLE)
+
+                if subctx.hasDate:  # working with dates
+                    u = unit.lower()
+                    if u in self.ptc.Months or \
+                            u in self.ptc.shortMonths:
+                        yr, mth, dy, hr, mn, sec, wd, yd, isdst = t
+                        start = datetime.datetime(
+                            yr, mth, dy, hr, mn, sec)
+                        t = self.inc(start, year=offset).timetuple()
+                    elif u in self.ptc.Weekdays:
+                        t = t + datetime.timedelta(weeks=offset)
+
+                if subctx.hasDateOrTime:
+                    sourceTime = t
+                    chunk2 = ''
+
+            chunk1 = chunk1.strip()
+
+            # if the word after next is a number, the string is more than
+            # likely to be "next 4 hrs" which we will have to combine the
+            # units with the rest of the string
+            if chunk1:
+                try:
+                    m = list(self.ptc.CRE_NUMBER.finditer(chunk1))[-1]
+                except IndexError:
+                    pass
+                else:
+                    qty = None
+                    debug and log.debug('CRE_NUMBER matched')
+                    qty = self._quantityToReal(m.group()) * offset
+                    chunk1 = '%s%s%s' % (chunk1[:m.start()],
+                                         qty, chunk1[m.end():])
+                t, subctx = self.parse(chunk1, sourceTime,
+                                       VERSION_CONTEXT_STYLE)
+
+                chunk1 = ''
+
+                if subctx.hasDateOrTime:
+                    sourceTime = t
+
+            debug and log.debug('looking for modifier %s', modifier)
+            sTime = self.ptc.getSource(modifier, sourceTime)
+            if sTime is not None:
+                debug and log.debug('modifier found in sources')
+                sourceTime = sTime
+                ctx.updateAccuracy(ctx.ACU_HALFDAY)
+
+        debug and log.debug('returning chunk = "%s %s" and sourceTime = %s',
+                            chunk1, chunk2, sourceTime)
+
+        return '%s %s' % (chunk1, chunk2), sourceTime
+
+    def _evalDT(self, datetimeString, sourceTime):
+        """
+        Calculate the datetime from known format like RFC822 or W3CDTF
+
+        Examples handled::
+            RFC822, W3CDTF formatted dates
+            HH:MM[:SS][ am/pm]
+            MM/DD/YYYY
+            DD MMMM YYYY
+
+        @type  datetimeString: string
+        @param datetimeString: text to try and parse as more "traditional"
+                               date/time text
+        @type  sourceTime:     struct_time
+        @param sourceTime:     C{struct_time} value to use as the base
+
+        @rtype:  datetime
+        @return: calculated C{struct_time} value or current C{struct_time}
+                 if not parsed
+        """
+        ctx = self.currentContext
+        s = datetimeString.strip()
+
+        # Given string date is a RFC822 date
+        if sourceTime is None:
+            sourceTime = _parse_date_rfc822(s)
+            debug and log.debug(
+                'attempt to parse as rfc822 - %s', str(sourceTime))
+
+            if sourceTime is not None:
+                (yr, mth, dy, hr, mn, sec, wd, yd, isdst, _) = sourceTime
+                ctx.updateAccuracy(ctx.ACU_YEAR, ctx.ACU_MONTH, ctx.ACU_DAY)
+
+                if hr != 0 and mn != 0 and sec != 0:
+                    ctx.updateAccuracy(ctx.ACU_HOUR, ctx.ACU_MIN, ctx.ACU_SEC)
+
+                sourceTime = (yr, mth, dy, hr, mn, sec, wd, yd, isdst)
+
+        # Given string date is a W3CDTF date
+        if sourceTime is None:
+            sourceTime = _parse_date_w3dtf(s)
+
+            if sourceTime is not None:
+                ctx.updateAccuracy(ctx.ACU_YEAR, ctx.ACU_MONTH, ctx.ACU_DAY,
+                                   ctx.ACU_HOUR, ctx.ACU_MIN, ctx.ACU_SEC)
+
+        if sourceTime is None:
+            sourceTime = time.localtime()
+
+        return sourceTime
+
+    def _evalUnits(self, datetimeString, sourceTime):
+        """
+        Evaluate text passed by L{_partialParseUnits()}
+        """
+        s = datetimeString.strip()
+        sourceTime = self._evalDT(datetimeString, sourceTime)
+
+        # Given string is a time string with units like "5 hrs 30 min"
+        modifier = ''  # TODO
+
+        m = self.ptc.CRE_UNITS.search(s)
+        if m is not None:
+            units = m.group('units')
+            quantity = s[:m.start('units')]
+
+        sourceTime = self._buildTime(sourceTime, quantity, modifier, units)
+        return sourceTime
+
+    def _evalQUnits(self, datetimeString, sourceTime):
+        """
+        Evaluate text passed by L{_partialParseQUnits()}
+        """
+        s = datetimeString.strip()
+        sourceTime = self._evalDT(datetimeString, sourceTime)
+
+        # Given string is a time string with single char units like "5 h 30 m"
+        modifier = ''  # TODO
+
+        m = self.ptc.CRE_QUNITS.search(s)
+        if m is not None:
+            units = m.group('qunits')
+            quantity = s[:m.start('qunits')]
+
+        sourceTime = self._buildTime(sourceTime, quantity, modifier, units)
+        return sourceTime
+
+    def _evalDateStr(self, datetimeString, sourceTime):
+        """
+        Evaluate text passed by L{_partialParseDateStr()}
+        """
+        s = datetimeString.strip()
+        sourceTime = self._evalDT(datetimeString, sourceTime)
+
+        # Given string is in the format  "May 23rd, 2005"
+        debug and log.debug('checking for MMM DD YYYY')
+        return self.parseDateText(s, sourceTime)
+
+    def _evalDateStd(self, datetimeString, sourceTime):
+        """
+        Evaluate text passed by L{_partialParseDateStd()}
+        """
+        s = datetimeString.strip()
+        sourceTime = self._evalDT(datetimeString, sourceTime)
+
+        # Given string is in the format 07/21/2006
+        return self.parseDate(s, sourceTime)
+
+    def _evalDayStr(self, datetimeString, sourceTime):
+        """
+        Evaluate text passed by L{_partialParseDaystr()}
+        """
+        s = datetimeString.strip()
+        sourceTime = self._evalDT(datetimeString, sourceTime)
+
+        # Given string is a natural language date string like today, tomorrow..
+        (yr, mth, dy, hr, mn, sec, wd, yd, isdst) = sourceTime
+
+        try:
+            offset = self.ptc.dayOffsets[s]
+        except KeyError:
+            offset = 0
+
+        if self.ptc.StartTimeFromSourceTime:
+            startHour = hr
+            startMinute = mn
+            startSecond = sec
+        else:
+            startHour = self.ptc.StartHour
+            startMinute = 0
+            startSecond = 0
+
+        self.currentContext.updateAccuracy(pdtContext.ACU_DAY)
+        start = datetime.datetime(yr, mth, dy, startHour,
+                                  startMinute, startSecond)
+        target = start + datetime.timedelta(days=offset)
+        return target.timetuple()
+
+    def _evalWeekday(self, datetimeString, sourceTime):
+        """
+        Evaluate text passed by L{_partialParseWeekday()}
+        """
+        s = datetimeString.strip()
+        sourceTime = self._evalDT(datetimeString, sourceTime)
+
+        # Given string is a weekday
+        yr, mth, dy, hr, mn, sec, wd, yd, isdst = sourceTime
+
+        start = datetime.datetime(yr, mth, dy, hr, mn, sec)
+        wkdy = self.ptc.WeekdayOffsets[s]
+
+        if wkdy > wd:
+            qty = self._CalculateDOWDelta(wd, wkdy, 2,
+                                          self.ptc.DOWParseStyle,
+                                          self.ptc.CurrentDOWParseStyle)
+        else:
+            qty = self._CalculateDOWDelta(wd, wkdy, 2,
+                                          self.ptc.DOWParseStyle,
+                                          self.ptc.CurrentDOWParseStyle)
+
+        self.currentContext.updateAccuracy(pdtContext.ACU_DAY)
+        target = start + datetime.timedelta(days=qty)
+        return target.timetuple()
+
+    def _evalTimeStr(self, datetimeString, sourceTime):
+        """
+        Evaluate text passed by L{_partialParseTimeStr()}
+        """
+        s = datetimeString.strip()
+        sourceTime = self._evalDT(datetimeString, sourceTime)
+
+        if s in self.ptc.re_values['now']:
+            self.currentContext.updateAccuracy(pdtContext.ACU_NOW)
+        else:
+            # Given string is a natural language time string like
+            # lunch, midnight, etc
+            sTime = self.ptc.getSource(s, sourceTime)
+            if sTime:
+                sourceTime = sTime
+            self.currentContext.updateAccuracy(pdtContext.ACU_HALFDAY)
+
+        return sourceTime
+
+    def _evalMeridian(self, datetimeString, sourceTime):
+        """
+        Evaluate text passed by L{_partialParseMeridian()}
+        """
+        s = datetimeString.strip()
+        sourceTime = self._evalDT(datetimeString, sourceTime)
+
+        # Given string is in the format HH:MM(:SS)(am/pm)
+        yr, mth, dy, hr, mn, sec, wd, yd, isdst = sourceTime
+
+        m = self.ptc.CRE_TIMEHMS2.search(s)
+        if m is not None:
+            dt = s[:m.start('meridian')].strip()
+            if len(dt) <= 2:
+                hr = int(dt)
+                mn = 0
+                sec = 0
+            else:
+                hr, mn, sec = _extract_time(m)
+
+            if hr == 24:
+                hr = 0
+
+            meridian = m.group('meridian').lower()
+
+            # if 'am' found and hour is 12 - force hour to 0 (midnight)
+            if (meridian in self.ptc.am) and hr == 12:
+                hr = 0
+
+            # if 'pm' found and hour < 12, add 12 to shift to evening
+            if (meridian in self.ptc.pm) and hr < 12:
+                hr += 12
+
+        # time validation
+        if hr < 24 and mn < 60 and sec < 60:
+            sourceTime = (yr, mth, dy, hr, mn, sec, wd, yd, isdst)
+            _pop_time_accuracy(m, self.currentContext)
+
+        return sourceTime
+
+    def _evalTimeStd(self, datetimeString, sourceTime):
+        """
+        Evaluate text passed by L{_partialParseTimeStd()}
+        """
+        s = datetimeString.strip()
+        sourceTime = self._evalDT(datetimeString, sourceTime)
+
+        # Given string is in the format HH:MM(:SS)
+        yr, mth, dy, hr, mn, sec, wd, yd, isdst = sourceTime
+
+        m = self.ptc.CRE_TIMEHMS.search(s)
+        if m is not None:
+            hr, mn, sec = _extract_time(m)
+        if hr == 24:
+            hr = 0
+
+        # time validation
+        if hr < 24 and mn < 60 and sec < 60:
+            sourceTime = (yr, mth, dy, hr, mn, sec, wd, yd, isdst)
+            _pop_time_accuracy(m, self.currentContext)
+
+        return sourceTime
+
+    def _UnitsTrapped(self, s, m, key):
+        # check if a day suffix got trapped by a unit match
+        # for example Dec 31st would match for 31s (aka 31 seconds)
+        # Dec 31st
+        #     ^ ^
+        #     | +-- m.start('units')
+        #     |     and also m2.start('suffix')
+        #     +---- m.start('qty')
+        #           and also m2.start('day')
+        m2 = self.ptc.CRE_DAY2.search(s)
+        if m2 is not None:
+            t = '%s%s' % (m2.group('day'), m.group(key))
+            if m.start(key) == m2.start('suffix') and \
+                    m.start('qty') == m2.start('day') and \
+                    m.group('qty') == t:
+                return True
+            else:
+                return False
+        else:
+            return False
+
+    def _partialParseModifier(self, s, sourceTime):
+        """
+        test if giving C{s} matched CRE_MODIFIER, used by L{parse()}
+
+        @type  s:          string
+        @param s:          date/time text to evaluate
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of remained date/time text, datetime object and
+                 an boolean value to describ if matched or not
+
+        """
+        parseStr = None
+        chunk1 = chunk2 = ''
+
+        # Modifier like next/prev/from/after/prior..
+        m = self.ptc.CRE_MODIFIER.search(s)
+        if m is not None:
+            if m.group() != s:
+                # capture remaining string
+                parseStr = m.group()
+                chunk1 = s[:m.start()].strip()
+                chunk2 = s[m.end():].strip()
+            else:
+                parseStr = s
+
+        if parseStr:
+            debug and log.debug('found (modifier) [%s][%s][%s]',
+                                parseStr, chunk1, chunk2)
+            s, sourceTime = self._evalModifier(parseStr, chunk1,
+                                               chunk2, sourceTime)
+
+        return s, sourceTime, bool(parseStr)
+
+    def _partialParseUnits(self, s, sourceTime):
+        """
+        test if giving C{s} matched CRE_UNITS, used by L{parse()}
+
+        @type  s:          string
+        @param s:          date/time text to evaluate
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of remained date/time text, datetime object and
+                 an boolean value to describ if matched or not
+
+        """
+        parseStr = None
+        chunk1 = chunk2 = ''
+
+        # Quantity + Units
+        m = self.ptc.CRE_UNITS.search(s)
+        if m is not None:
+            debug and log.debug('CRE_UNITS matched')
+            if self._UnitsTrapped(s, m, 'units'):
+                debug and log.debug('day suffix trapped by unit match')
+            else:
+                if (m.group('qty') != s):
+                    # capture remaining string
+                    parseStr = m.group('qty')
+                    chunk1 = s[:m.start('qty')].strip()
+                    chunk2 = s[m.end('qty'):].strip()
+
+                    if chunk1[-1:] == '-':
+                        parseStr = '-%s' % parseStr
+                        chunk1 = chunk1[:-1]
+
+                    s = '%s %s' % (chunk1, chunk2)
+                else:
+                    parseStr = s
+                    s = ''
+
+        if parseStr:
+            debug and log.debug('found (units) [%s][%s][%s]',
+                                parseStr, chunk1, chunk2)
+            sourceTime = self._evalUnits(parseStr, sourceTime)
+
+        return s, sourceTime, bool(parseStr)
+
+    def _partialParseQUnits(self, s, sourceTime):
+        """
+        test if giving C{s} matched CRE_QUNITS, used by L{parse()}
+
+        @type  s:          string
+        @param s:          date/time text to evaluate
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of remained date/time text, datetime object and
+                 an boolean value to describ if matched or not
+
+        """
+        parseStr = None
+        chunk1 = chunk2 = ''
+
+        # Quantity + Units
+        m = self.ptc.CRE_QUNITS.search(s)
+        if m is not None:
+            debug and log.debug('CRE_QUNITS matched')
+            if self._UnitsTrapped(s, m, 'qunits'):
+                debug and log.debug(
+                    'day suffix trapped by qunit match')
+            else:
+                if (m.group('qty') != s):
+                    # capture remaining string
+                    parseStr = m.group('qty')
+                    chunk1 = s[:m.start('qty')].strip()
+                    chunk2 = s[m.end('qty'):].strip()
+
+                    if chunk1[-1:] == '-':
+                        parseStr = '-%s' % parseStr
+                        chunk1 = chunk1[:-1]
+
+                    s = '%s %s' % (chunk1, chunk2)
+                else:
+                    parseStr = s
+                    s = ''
+
+        if parseStr:
+            debug and log.debug('found (qunits) [%s][%s][%s]',
+                                parseStr, chunk1, chunk2)
+            sourceTime = self._evalQUnits(parseStr, sourceTime)
+
+        return s, sourceTime, bool(parseStr)
+
+    def _partialParseDateStr(self, s, sourceTime):
+        """
+        test if giving C{s} matched CRE_DATE3, used by L{parse()}
+
+        @type  s:          string
+        @param s:          date/time text to evaluate
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of remained date/time text, datetime object and
+                 an boolean value to describ if matched or not
+
+        """
+        parseStr = None
+        chunk1 = chunk2 = ''
+
+        m = self.ptc.CRE_DATE3.search(s)
+        # NO LONGER NEEDED, THE REGEXP HANDLED MTHNAME NOW
+        # for match in self.ptc.CRE_DATE3.finditer(s):
+        # to prevent "HH:MM(:SS) time strings" expressions from
+        # triggering this regex, we checks if the month field
+        # exists in the searched expression, if it doesn't exist,
+        # the date field is not valid
+        #     if match.group('mthname'):
+        #         m = self.ptc.CRE_DATE3.search(s, match.start())
+        #         valid_date = True
+        #         break
+
+        # String date format
+        if m is not None:
+
+            if (m.group('date') != s):
+                # capture remaining string
+                mStart = m.start('date')
+                mEnd = m.end('date')
+
+                # we need to check that anything following the parsed
+                # date is a time expression because it is often picked
+                # up as a valid year if the hour is 2 digits
+                fTime = False
+                mm = self.ptc.CRE_TIMEHMS2.search(s)
+                # "February 24th 1PM" doesn't get caught
+                # "February 24th 12PM" does
+                mYear = m.group('year')
+                if mm is not None and mYear is not None:
+                    fTime = True
+                else:
+                    # "February 24th 12:00"
+                    mm = self.ptc.CRE_TIMEHMS.search(s)
+                    if mm is not None and mYear is None:
+                        fTime = True
+                if fTime:
+                    hoursStart = mm.start('hours')
+
+                    if hoursStart < m.end('year'):
+                        mEnd = hoursStart
+
+                parseStr = s[mStart:mEnd]
+                chunk1 = s[:mStart]
+                chunk2 = s[mEnd:]
+
+                s = '%s %s' % (chunk1, chunk2)
+            else:
+                parseStr = s
+                s = ''
+
+        if parseStr:
+            debug and log.debug(
+                'found (date3) [%s][%s][%s]', parseStr, chunk1, chunk2)
+            sourceTime = self._evalDateStr(parseStr, sourceTime)
+
+        return s, sourceTime, bool(parseStr)
+
+    def _partialParseDateStd(self, s, sourceTime):
+        """
+        test if giving C{s} matched CRE_DATE, used by L{parse()}
+
+        @type  s:          string
+        @param s:          date/time text to evaluate
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of remained date/time text, datetime object and
+                 an boolean value to describ if matched or not
+
+        """
+        parseStr = None
+        chunk1 = chunk2 = ''
+
+        # Standard date format
+        m = self.ptc.CRE_DATE.search(s)
+        if m is not None:
+
+            if (m.group('date') != s):
+                # capture remaining string
+                parseStr = m.group('date')
+                chunk1 = s[:m.start('date')]
+                chunk2 = s[m.end('date'):]
+                s = '%s %s' % (chunk1, chunk2)
+            else:
+                parseStr = s
+                s = ''
+
+        if parseStr:
+            debug and log.debug(
+                'found (date) [%s][%s][%s]', parseStr, chunk1, chunk2)
+            sourceTime = self._evalDateStd(parseStr, sourceTime)
+
+        return s, sourceTime, bool(parseStr)
+
+    def _partialParseDayStr(self, s, sourceTime):
+        """
+        test if giving C{s} matched CRE_DAY, used by L{parse()}
+
+        @type  s:          string
+        @param s:          date/time text to evaluate
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of remained date/time text, datetime object and
+                 an boolean value to describ if matched or not
+
+        """
+        parseStr = None
+        chunk1 = chunk2 = ''
+
+        # Natural language day strings
+        m = self.ptc.CRE_DAY.search(s)
+        if m is not None:
+
+            if (m.group() != s):
+                # capture remaining string
+                parseStr = m.group()
+                chunk1 = s[:m.start()]
+                chunk2 = s[m.end():]
+                s = '%s %s' % (chunk1, chunk2)
+            else:
+                parseStr = s
+                s = ''
+
+        if parseStr:
+            debug and log.debug(
+                'found (day) [%s][%s][%s]', parseStr, chunk1, chunk2)
+            sourceTime = self._evalDayStr(parseStr, sourceTime)
+
+        return s, sourceTime, bool(parseStr)
+
+    def _partialParseWeekday(self, s, sourceTime):
+        """
+        test if giving C{s} matched CRE_WEEKDAY, used by L{parse()}
+
+        @type  s:          string
+        @param s:          date/time text to evaluate
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of remained date/time text, datetime object and
+                 an boolean value to describ if matched or not
+
+        """
+        parseStr = None
+        chunk1 = chunk2 = ''
+
+        ctx = self.currentContext
+        log.debug('eval %s with context - %s, %s', s, ctx.hasDate, ctx.hasTime)
+
+        # Weekday
+        m = self.ptc.CRE_WEEKDAY.search(s)
+        if m is not None:
+            gv = m.group()
+            if s not in self.ptc.dayOffsets:
+
+                if (gv != s):
+                    # capture remaining string
+                    parseStr = gv
+                    chunk1 = s[:m.start()]
+                    chunk2 = s[m.end():]
+                    s = '%s %s' % (chunk1, chunk2)
+                else:
+                    parseStr = s
+                    s = ''
+
+        if parseStr and not ctx.hasDate:
+            debug and log.debug(
+                'found (weekday) [%s][%s][%s]', parseStr, chunk1, chunk2)
+            sourceTime = self._evalWeekday(parseStr, sourceTime)
+
+        return s, sourceTime, bool(parseStr)
+
+    def _partialParseTimeStr(self, s, sourceTime):
+        """
+        test if giving C{s} matched CRE_TIME, used by L{parse()}
+
+        @type  s:          string
+        @param s:          date/time text to evaluate
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of remained date/time text, datetime object and
+                 an boolean value to describ if matched or not
+
+        """
+        parseStr = None
+        chunk1 = chunk2 = ''
+
+        # Natural language time strings
+        m = self.ptc.CRE_TIME.search(s)
+        if m is not None or s in self.ptc.re_values['now']:
+
+            if (m and m.group() != s):
+                # capture remaining string
+                parseStr = m.group()
+                chunk1 = s[:m.start()]
+                chunk2 = s[m.end():]
+                s = '%s %s' % (chunk1, chunk2)
+            else:
+                parseStr = s
+                s = ''
+
+        if parseStr:
+            debug and log.debug(
+                'found (time) [%s][%s][%s]', parseStr, chunk1, chunk2)
+            sourceTime = self._evalTimeStr(parseStr, sourceTime)
+
+        return s, sourceTime, bool(parseStr)
+
+    def _partialParseMeridian(self, s, sourceTime):
+        """
+        test if giving C{s} matched CRE_TIMEHMS2, used by L{parse()}
+
+        @type  s:          string
+        @param s:          date/time text to evaluate
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of remained date/time text, datetime object and
+                 an boolean value to describ if matched or not
+
+        """
+        parseStr = None
+        chunk1 = chunk2 = ''
+
+        # HH:MM(:SS) am/pm time strings
+        m = self.ptc.CRE_TIMEHMS2.search(s)
+        if m is not None:
+
+            if m.group('minutes') is not None:
+                if m.group('seconds') is not None:
+                    parseStr = '%s:%s:%s' % (m.group('hours'),
+                                             m.group('minutes'),
+                                             m.group('seconds'))
+                else:
+                    parseStr = '%s:%s' % (m.group('hours'),
+                                          m.group('minutes'))
+            else:
+                parseStr = m.group('hours')
+            parseStr += ' ' + m.group('meridian')
+
+            chunk1 = s[:m.start()]
+            chunk2 = s[m.end():]
+
+            s = '%s %s' % (chunk1, chunk2)
+
+        if parseStr:
+            debug and log.debug('found (meridian) [%s][%s][%s]',
+                                parseStr, chunk1, chunk2)
+            sourceTime = self._evalMeridian(parseStr, sourceTime)
+
+        return s, sourceTime, bool(parseStr)
+
+    def _partialParseTimeStd(self, s, sourceTime):
+        """
+        test if giving C{s} matched CRE_TIMEHMS, used by L{parse()}
+
+        @type  s:          string
+        @param s:          date/time text to evaluate
+        @type  sourceTime: struct_time
+        @param sourceTime: C{struct_time} value to use as the base
+
+        @rtype:  tuple
+        @return: tuple of remained date/time text, datetime object and
+                 an boolean value to describ if matched or not
+
+        """
+        parseStr = None
+        chunk1 = chunk2 = ''
+
+        # HH:MM(:SS) time strings
+        m = self.ptc.CRE_TIMEHMS.search(s)
+        if m is not None:
+
+            if m.group('seconds') is not None:
+                parseStr = '%s:%s:%s' % (m.group('hours'),
+                                         m.group('minutes'),
+                                         m.group('seconds'))
+                chunk1 = s[:m.start('hours')]
+                chunk2 = s[m.end('seconds'):]
+            else:
+                parseStr = '%s:%s' % (m.group('hours'),
+                                      m.group('minutes'))
+                chunk1 = s[:m.start('hours')]
+                chunk2 = s[m.end('minutes'):]
+
+            s = '%s %s' % (chunk1, chunk2)
+
+        if parseStr:
+            debug and log.debug(
+                'found (hms) [%s][%s][%s]', parseStr, chunk1, chunk2)
+            sourceTime = self._evalTimeStd(parseStr, sourceTime)
+
+        return s, sourceTime, bool(parseStr)
+
+    def parseDT(self, datetimeString, sourceTime=None,
+                tzinfo=None, version=None):
+        """
+        C{datetimeString} is as C{.parse}, C{sourceTime} has the same semantic
+        meaning as C{.parse}, but now also accepts datetime objects.  C{tzinfo}
+        accepts a tzinfo object.  It is advisable to use pytz.
+
+
+        @type  datetimeString: string
+        @param datetimeString: date/time text to evaluate
+        @type  sourceTime:     struct_time, datetime, date, time
+        @param sourceTime:     time value to use as the base
+        @type  tzinfo:         tzinfo
+        @param tzinfo:         Timezone to apply to generated datetime objs.
+        @type  version:        integer
+        @param version:        style version, default will use L{Calendar}
+                               parameter version value
+
+        @rtype:  tuple
+        @return: tuple of: modified C{sourceTime} and the result flag/context
+
+        see .parse for return code details.
+        """
+        # if sourceTime has a timetuple method, use thet, else, just pass the
+        # entire thing to parse and prey the user knows what the hell they are
+        # doing.
+        sourceTime = getattr(sourceTime, 'timetuple', (lambda: sourceTime))()
+        # You REALLY SHOULD be using pytz.  Using localize if available,
+        # hacking if not.  Note, None is a valid tzinfo object in the case of
+        # the ugly hack.
+        localize = getattr(
+            tzinfo,
+            'localize',
+            (lambda dt: dt.replace(tzinfo=tzinfo)),  # ugly hack is ugly :(
+        )
+
+        # Punt
+        time_struct, ret_code = self.parse(
+            datetimeString,
+            sourceTime=sourceTime,
+            version=version)
+
+        # Comments from GHI indicate that it is desired to have the same return
+        # signature on this method as that one it punts to, with the exception
+        # of using datetime objects instead of time_structs.
+        dt = localize(datetime.datetime(*time_struct[:6]))
+        return dt, ret_code
+
+    def parse(self, datetimeString, sourceTime=None, version=None):
+        """
+        Splits the given C{datetimeString} into tokens, finds the regex
+        patterns that match and then calculates a C{struct_time} value from
+        the chunks.
+
+        If C{sourceTime} is given then the C{struct_time} value will be
+        calculated from that value, otherwise from the current date/time.
+
+        If the C{datetimeString} is parsed and date/time value found, then::
+
+            If C{version} equals to L{VERSION_FLAG_STYLE}, the second item of
+            the returned tuple will be a flag to let you know what kind of
+            C{struct_time} value is being returned::
+
+                0 = not parsed at all
+                1 = parsed as a C{date}
+                2 = parsed as a C{time}
+                3 = parsed as a C{datetime}
+
+            If C{version} equals to L{VERSION_CONTEXT_STYLE}, the second value
+            will be an instance of L{pdtContext}
+
+        @type  datetimeString: string
+        @param datetimeString: date/time text to evaluate
+        @type  sourceTime:     struct_time
+        @param sourceTime:     C{struct_time} value to use as the base
+        @type  version:        integer
+        @param version:        style version, default will use L{Calendar}
+                               parameter version value
+
+        @rtype:  tuple
+        @return: tuple of: modified C{sourceTime} and the result flag/context
+        """
+        debug and log.debug('parse()')
+
+        datetimeString = re.sub(r'(\w)\.(\s)', r'\1\2', datetimeString)
+        datetimeString = re.sub(r'(\w)[\'"](\s|$)', r'\1 \2', datetimeString)
+        datetimeString = re.sub(r'(\s|^)[\'"](\w)', r'\1 \2', datetimeString)
+
+        if sourceTime:
+            if isinstance(sourceTime, datetime.datetime):
+                debug and log.debug('coercing datetime to timetuple')
+                sourceTime = sourceTime.timetuple()
+            else:
+                if not isinstance(sourceTime, time.struct_time) and \
+                        not isinstance(sourceTime, tuple):
+                    raise ValueError('sourceTime is not a struct_time')
+        else:
+            sourceTime = time.localtime()
+
+        with self.context() as ctx:
+            s = datetimeString.lower().strip()
+            debug and log.debug('remainedString (before parsing): [%s]', s)
+
+            while s:
+                for parseMeth in (self._partialParseModifier,
+                                  self._partialParseUnits,
+                                  self._partialParseQUnits,
+                                  self._partialParseDateStr,
+                                  self._partialParseDateStd,
+                                  self._partialParseDayStr,
+                                  self._partialParseWeekday,
+                                  self._partialParseTimeStr,
+                                  self._partialParseMeridian,
+                                  self._partialParseTimeStd):
+                    retS, retTime, matched = parseMeth(s, sourceTime)
+                    if matched:
+                        s, sourceTime = retS.strip(), retTime
+                        break
+                else:
+                    # nothing matched
+                    s = ''
+
+                debug and log.debug('hasDate: [%s], hasTime: [%s]',
+                                    ctx.hasDate, ctx.hasTime)
+                debug and log.debug('remainedString: [%s]', s)
+
+            # String is not parsed at all
+            if sourceTime is None:
+                debug and log.debug('not parsed [%s]', str(sourceTime))
+                sourceTime = time.localtime()
+
+        if not isinstance(sourceTime, time.struct_time):
+            sourceTime = time.struct_time(sourceTime)
+
+        version = self.version if version is None else version
+        if version == VERSION_CONTEXT_STYLE:
+            return sourceTime, ctx
+        else:
+            return sourceTime, ctx.dateTimeFlag
+
+    def inc(self, source, month=None, year=None):
+        """
+        Takes the given C{source} date, or current date if none is
+        passed, and increments it according to the values passed in
+        by month and/or year.
+
+        This routine is needed because Python's C{timedelta()} function
+        does not allow for month or year increments.
+
+        @type  source: struct_time
+        @param source: C{struct_time} value to increment
+        @type  month:  float or integer
+        @param month:  optional number of months to increment
+        @type  year:   float or integer
+        @param year:   optional number of years to increment
+
+        @rtype:  datetime
+        @return: C{source} incremented by the number of months and/or years
+        """
+        yr = source.year
+        mth = source.month
+        dy = source.day
+
+        try:
+            month = float(month)
+        except (TypeError, ValueError):
+            month = 0
+
+        try:
+            year = float(year)
+        except (TypeError, ValueError):
+            year = 0
+        finally:
+            month += year * 12
+            year = 0
+
+        subMi = 0.0
+        maxDay = 0
+        if month:
+            mi = int(month)
+            subMi = month - mi
+
+            y = int(mi / 12.0)
+            m = mi - y * 12
+
+            mth = mth + m
+            if mth < 1:  # cross start-of-year?
+                y -= 1  # yes - decrement year
+                mth += 12  # and fix month
+            elif mth > 12:  # cross end-of-year?
+                y += 1  # yes - increment year
+                mth -= 12  # and fix month
+
+            yr += y
+
+            # if the day ends up past the last day of
+            # the new month, set it to the last day
+            maxDay = self.ptc.daysInMonth(mth, yr)
+            if dy > maxDay:
+                dy = maxDay
+
+        if yr > datetime.MAXYEAR or yr < datetime.MINYEAR:
+            raise OverflowError('year is out of range')
+
+        d = source.replace(year=yr, month=mth, day=dy)
+        if subMi:
+            d += datetime.timedelta(days=subMi * maxDay)
+        return source + (d - source)
+
+    def nlp(self, inputString, sourceTime=None, version=None):
+        """Utilizes parse() after making judgements about what datetime
+        information belongs together.
+
+        It makes logical groupings based on proximity and returns a parsed
+        datetime for each matched grouping of datetime text, along with
+        location info within the given inputString.
+
+        @type  inputString: string
+        @param inputString: natural language text to evaluate
+        @type  sourceTime:  struct_time
+        @param sourceTime:  C{struct_time} value to use as the base
+        @type  version:     integer
+        @param version:     style version, default will use L{Calendar}
+                            parameter version value
+
+        @rtype:  tuple or None
+        @return: tuple of tuples in the format (parsed_datetime as
+                 datetime.datetime, flags as int, start_pos as int,
+                 end_pos as int, matched_text as string) or None if there
+                 were no matches
+        """
+
+        orig_inputstring = inputString
+
+        # replace periods at the end of sentences w/ spaces
+        # opposed to removing them altogether in order to
+        # retain relative positions (identified by alpha, period, space).
+        # this is required for some of the regex patterns to match
+        inputString = re.sub(r'(\w)(\.)(\s)', r'\1 \3', inputString).lower()
+        inputString = re.sub(r'(\w)(\'|")(\s|$)', r'\1 \3', inputString)
+        inputString = re.sub(r'(\s|^)(\'|")(\w)', r'\1 \3', inputString)
+
+        startpos = 0  # the start position in the inputString during the loop
+
+        # list of lists in format:
+        # [startpos, endpos, matchedstring, flags, type]
+        matches = []
+
+        while startpos < len(inputString):
+
+            # empty match
+            leftmost_match = [0, 0, None, 0, None]
+
+            # Modifier like next\prev..
+            m = self.ptc.CRE_MODIFIER.search(inputString[startpos:])
+            if m is not None:
+                if leftmost_match[1] == 0 or \
+                        leftmost_match[0] > m.start() + startpos:
+                    leftmost_match[0] = m.start() + startpos
+                    leftmost_match[1] = m.end() + startpos
+                    leftmost_match[2] = m.group()
+                    leftmost_match[3] = 0
+                    leftmost_match[4] = 'modifier'
+
+            # Quantity + Units
+            m = self.ptc.CRE_UNITS.search(inputString[startpos:])
+            if m is not None:
+                debug and log.debug('CRE_UNITS matched')
+                if self._UnitsTrapped(inputString[startpos:], m, 'units'):
+                    debug and log.debug('day suffix trapped by unit match')
+                else:
+
+                    if leftmost_match[1] == 0 or \
+                            leftmost_match[0] > m.start('qty') + startpos:
+                        leftmost_match[0] = m.start('qty') + startpos
+                        leftmost_match[1] = m.end('qty') + startpos
+                        leftmost_match[2] = m.group('qty')
+                        leftmost_match[3] = 3
+                        leftmost_match[4] = 'units'
+
+                        if m.start('qty') > 0 and \
+                                inputString[m.start('qty') - 1] == '-':
+                            leftmost_match[0] = leftmost_match[0] - 1
+                            leftmost_match[2] = '-' + leftmost_match[2]
+
+            # Quantity + Units
+            m = self.ptc.CRE_QUNITS.search(inputString[startpos:])
+            if m is not None:
+                debug and log.debug('CRE_QUNITS matched')
+                if self._UnitsTrapped(inputString[startpos:], m, 'qunits'):
+                    debug and log.debug('day suffix trapped by qunit match')
+                else:
+                    if leftmost_match[1] == 0 or \
+                            leftmost_match[0] > m.start('qty') + startpos:
+                        leftmost_match[0] = m.start('qty') + startpos
+                        leftmost_match[1] = m.end('qty') + startpos
+                        leftmost_match[2] = m.group('qty')
+                        leftmost_match[3] = 3
+                        leftmost_match[4] = 'qunits'
+
+                        if m.start('qty') > 0 and \
+                                inputString[m.start('qty') - 1] == '-':
+                            leftmost_match[0] = leftmost_match[0] - 1
+                            leftmost_match[2] = '-' + leftmost_match[2]
+
+            m = self.ptc.CRE_DATE3.search(inputString[startpos:])
+            # NO LONGER NEEDED, THE REGEXP HANDLED MTHNAME NOW
+            # for match in self.ptc.CRE_DATE3.finditer(inputString[startpos:]):
+            # to prevent "HH:MM(:SS) time strings" expressions from
+            # triggering this regex, we checks if the month field exists
+            # in the searched expression, if it doesn't exist, the date
+            # field is not valid
+            #     if match.group('mthname'):
+            #         m = self.ptc.CRE_DATE3.search(inputString[startpos:],
+            #                                       match.start())
+            #         break
+
+            # String date format
+            if m is not None:
+                if leftmost_match[1] == 0 or \
+                        leftmost_match[0] > m.start('date') + startpos:
+                    leftmost_match[0] = m.start('date') + startpos
+                    leftmost_match[1] = m.end('date') + startpos
+                    leftmost_match[2] = m.group('date')
+                    leftmost_match[3] = 1
+                    leftmost_match[4] = 'dateStr'
+
+            # Standard date format
+            m = self.ptc.CRE_DATE.search(inputString[startpos:])
+            if m is not None:
+                if leftmost_match[1] == 0 or \
+                        leftmost_match[0] > m.start('date') + startpos:
+                    leftmost_match[0] = m.start('date') + startpos
+                    leftmost_match[1] = m.end('date') + startpos
+                    leftmost_match[2] = m.group('date')
+                    leftmost_match[3] = 1
+                    leftmost_match[4] = 'dateStd'
+
+            # Natural language day strings
+            m = self.ptc.CRE_DAY.search(inputString[startpos:])
+            if m is not None:
+                if leftmost_match[1] == 0 or \
+                        leftmost_match[0] > m.start() + startpos:
+                    leftmost_match[0] = m.start() + startpos
+                    leftmost_match[1] = m.end() + startpos
+                    leftmost_match[2] = m.group()
+                    leftmost_match[3] = 1
+                    leftmost_match[4] = 'dayStr'
+
+            # Weekday
+            m = self.ptc.CRE_WEEKDAY.search(inputString[startpos:])
+            if m is not None:
+                if inputString[startpos:] not in self.ptc.dayOffsets:
+                    if leftmost_match[1] == 0 or \
+                            leftmost_match[0] > m.start() + startpos:
+                        leftmost_match[0] = m.start() + startpos
+                        leftmost_match[1] = m.end() + startpos
+                        leftmost_match[2] = m.group()
+                        leftmost_match[3] = 1
+                        leftmost_match[4] = 'weekdy'
+
+            # Natural language time strings
+            m = self.ptc.CRE_TIME.search(inputString[startpos:])
+            if m is not None:
+                if leftmost_match[1] == 0 or \
+                        leftmost_match[0] > m.start() + startpos:
+                    leftmost_match[0] = m.start() + startpos
+                    leftmost_match[1] = m.end() + startpos
+                    leftmost_match[2] = m.group()
+                    leftmost_match[3] = 2
+                    leftmost_match[4] = 'timeStr'
+
+            # HH:MM(:SS) am/pm time strings
+            m = self.ptc.CRE_TIMEHMS2.search(inputString[startpos:])
+            if m is not None:
+                if leftmost_match[1] == 0 or \
+                        leftmost_match[0] > m.start('hours') + startpos:
+                    leftmost_match[0] = m.start('hours') + startpos
+                    leftmost_match[1] = m.end('meridian') + startpos
+                    leftmost_match[2] = inputString[leftmost_match[0]:
+                                                    leftmost_match[1]]
+                    leftmost_match[3] = 2
+                    leftmost_match[4] = 'meridian'
+
+            # HH:MM(:SS) time strings
+            m = self.ptc.CRE_TIMEHMS.search(inputString[startpos:])
+            if m is not None:
+                if leftmost_match[1] == 0 or \
+                        leftmost_match[0] > m.start('hours') + startpos:
+                    leftmost_match[0] = m.start('hours') + startpos
+                    if m.group('seconds') is not None:
+                        leftmost_match[1] = m.end('seconds') + startpos
+                    else:
+                        leftmost_match[1] = m.end('minutes') + startpos
+                    leftmost_match[2] = inputString[leftmost_match[0]:
+                                                    leftmost_match[1]]
+                    leftmost_match[3] = 2
+                    leftmost_match[4] = 'timeStd'
+
+            # Units only; must be preceded by a modifier
+            if len(matches) > 0 and matches[-1][3] == 0:
+                m = self.ptc.CRE_UNITS_ONLY.search(inputString[startpos:])
+                # Ensure that any match is immediately proceded by the
+                # modifier. "Next is the word 'month'" should not parse as a
+                # date while "next month" should
+                if m is not None and \
+                        inputString[startpos:startpos + m.start()].strip() == '':
+                    debug and log.debug('CRE_UNITS_ONLY matched [%s]',
+                                        m.group())
+                    if leftmost_match[1] == 0 or \
+                            leftmost_match[0] > m.start() + startpos:
+                        leftmost_match[0] = m.start() + startpos
+                        leftmost_match[1] = m.end() + startpos
+                        leftmost_match[2] = m.group()
+                        leftmost_match[3] = 3
+                        leftmost_match[4] = 'unitsOnly'
+
+            # set the start position to the end pos of the leftmost match
+            startpos = leftmost_match[1]
+
+            # nothing was detected
+            # so break out of the loop
+            if startpos == 0:
+                startpos = len(inputString)
+            else:
+                if leftmost_match[3] > 0:
+                    m = self.ptc.CRE_NLP_PREFIX.search(
+                        inputString[:leftmost_match[0]] + ' ' + str(leftmost_match[3]))
+                    if m is not None:
+                        leftmost_match[0] = m.start('nlp_prefix')
+                        leftmost_match[2] = inputString[leftmost_match[0]:
+                                                        leftmost_match[1]]
+                matches.append(leftmost_match)
+
+        # find matches in proximity with one another and
+        # return all the parsed values
+        proximity_matches = []
+        if len(matches) > 1:
+            combined = ''
+            from_match_index = 0
+            date = matches[0][3] == 1
+            time = matches[0][3] == 2
+            units = matches[0][3] == 3
+            for i in range(1, len(matches)):
+
+                # test proximity (are there characters between matches?)
+                endofprevious = matches[i - 1][1]
+                begofcurrent = matches[i][0]
+                if orig_inputstring[endofprevious:
+                                    begofcurrent].lower().strip() != '':
+                    # this one isn't in proximity, but maybe
+                    # we have enough to make a datetime
+                    # TODO: make sure the combination of
+                    # formats (modifier, dateStd, etc) makes logical sense
+                    # before parsing together
+                    if date or time or units:
+                        combined = orig_inputstring[matches[from_match_index]
+                                                    [0]:matches[i - 1][1]]
+                        parsed_datetime, flags = self.parse(combined,
+                                                            sourceTime,
+                                                            version)
+                        proximity_matches.append((
+                            datetime.datetime(*parsed_datetime[:6]),
+                            flags,
+                            matches[from_match_index][0],
+                            matches[i - 1][1],
+                            combined))
+                    # not in proximity, reset starting from current
+                    from_match_index = i
+                    date = matches[i][3] == 1
+                    time = matches[i][3] == 2
+                    units = matches[i][3] == 3
+                    continue
+                else:
+                    if matches[i][3] == 1:
+                        date = True
+                    if matches[i][3] == 2:
+                        time = True
+                    if matches[i][3] == 3:
+                        units = True
+
+            # check last
+            # we have enough to make a datetime
+            if date or time or units:
+                combined = orig_inputstring[matches[from_match_index][0]:
+                                            matches[len(matches) - 1][1]]
+                parsed_datetime, flags = self.parse(combined, sourceTime,
+                                                    version)
+                proximity_matches.append((
+                    datetime.datetime(*parsed_datetime[:6]),
+                    flags,
+                    matches[from_match_index][0],
+                    matches[len(matches) - 1][1],
+                    combined))
+
+        elif len(matches) == 0:
+            return None
+        else:
+            if matches[0][3] == 0:  # not enough info to parse
+                return None
+            else:
+                combined = orig_inputstring[matches[0][0]:matches[0][1]]
+                parsed_datetime, flags = self.parse(matches[0][2], sourceTime,
+                                                    version)
+                proximity_matches.append((
+                    datetime.datetime(*parsed_datetime[:6]),
+                    flags,
+                    matches[0][0],
+                    matches[0][1],
+                    combined))
+
+        return tuple(proximity_matches)
+
+
+def _initSymbols(ptc):
+    """
+    Initialize symbols and single character constants.
+    """
+    # build am and pm lists to contain
+    # original case, lowercase, first-char and dotted
+    # versions of the meridian text
+    ptc.am = ['', '']
+    ptc.pm = ['', '']
+    for idx, xm in enumerate(ptc.locale.meridian[:2]):
+        # 0: am
+        # 1: pm
+        target = ['am', 'pm'][idx]
+        setattr(ptc, target, [xm])
+        target = getattr(ptc, target)
+        if xm:
+            lxm = xm.lower()
+            target.extend((xm[0], '{0}.{1}.'.format(*xm),
+                           lxm, lxm[0], '{0}.{1}.'.format(*lxm)))
+
+
+class Constants(object):
+
+    """
+    Default set of constants for parsedatetime.
+
+    If PyICU is present, then the class will first try to get PyICU
+    to return a locale specified by C{localeID}.  If either C{localeID} is
+    None or if the locale does not exist within PyICU, then each of the
+    locales defined in C{fallbackLocales} is tried in order.
+
+    If PyICU is not present or none of the specified locales can be used,
+    then the class will initialize itself to the en_US locale.
+
+    if PyICU is not present or not requested, only the locales defined by
+    C{pdtLocales} will be searched.
+    """
+
+    def __init__(self, localeID=None, usePyICU=True,
+                 fallbackLocales=['en_US']):
+        self.localeID = localeID
+        self.fallbackLocales = fallbackLocales[:]
+
+        if 'en_US' not in self.fallbackLocales:
+            self.fallbackLocales.append('en_US')
+
+        # define non-locale specific constants
+        self.locale = None
+        self.usePyICU = usePyICU
+
+        # starting cache of leap years
+        # daysInMonth will add to this if during
+        # runtime it gets a request for a year not found
+        self._leapYears = list(range(1904, 2097, 4))
+
+        self.Second = 1
+        self.Minute = 60  # 60 * self.Second
+        self.Hour = 3600  # 60 * self.Minute
+        self.Day = 86400  # 24 * self.Hour
+        self.Week = 604800  # 7   * self.Day
+        self.Month = 2592000  # 30  * self.Day
+        self.Year = 31536000  # 365 * self.Day
+
+        self._DaysInMonthList = (31, 28, 31, 30, 31, 30,
+                                 31, 31, 30, 31, 30, 31)
+        self.rangeSep = '-'
+        self.BirthdayEpoch = 50
+
+        # When True the starting time for all relative calculations will come
+        # from the given SourceTime, otherwise it will be self.StartHour
+
+        self.StartTimeFromSourceTime = False
+
+        # The hour of the day that will be used as the starting time for all
+        # relative calculations when self.StartTimeFromSourceTime is False
+
+        self.StartHour = 9
+
+        # YearParseStyle controls how we parse "Jun 12", i.e. dates that do
+        # not have a year present.  The default is to compare the date given
+        # to the current date, and if prior, then assume the next year.
+        # Setting this to 0 will prevent that.
+
+        self.YearParseStyle = 1
+
+        # DOWParseStyle controls how we parse "Tuesday"
+        # If the current day was Thursday and the text to parse is "Tuesday"
+        # then the following table shows how each style would be returned
+        # -1, 0, +1
+        #
+        # Current day marked as ***
+        #
+        #          Sun Mon Tue Wed Thu Fri Sat
+        # week -1
+        # current         -1,0     ***
+        # week +1          +1
+        #
+        # If the current day was Monday and the text to parse is "Tuesday"
+        # then the following table shows how each style would be returned
+        # -1, 0, +1
+        #
+        #          Sun Mon Tue Wed Thu Fri Sat
+        # week -1           -1
+        # current      *** 0,+1
+        # week +1
+
+        self.DOWParseStyle = 1
+
+        # CurrentDOWParseStyle controls how we parse "Friday"
+        # If the current day was Friday and the text to parse is "Friday"
+        # then the following table shows how each style would be returned
+        # True/False. This also depends on DOWParseStyle.
+        #
+        # Current day marked as ***
+        #
+        # DOWParseStyle = 0
+        #          Sun Mon Tue Wed Thu Fri Sat
+        # week -1
+        # current                      T,F
+        # week +1
+        #
+        # DOWParseStyle = -1
+        #          Sun Mon Tue Wed Thu Fri Sat
+        # week -1                       F
+        # current                       T
+        # week +1
+        #
+        # DOWParseStyle = +1
+        #
+        #          Sun Mon Tue Wed Thu Fri Sat
+        # week -1
+        # current                       T
+        # week +1                       F
+
+        self.CurrentDOWParseStyle = False
+
+        if self.usePyICU:
+            self.locale = get_icu(self.localeID)
+
+            if self.locale.icu is None:
+                self.usePyICU = False
+                self.locale = None
+
+        if self.locale is None:
+            if self.localeID not in pdtLocales:
+                for localeId in range(0, len(self.fallbackLocales)):
+                    self.localeID = self.fallbackLocales[localeId]
+                    if self.localeID in pdtLocales:
+                        break
+
+            self.locale = pdtLocales[self.localeID]
+
+        if self.locale is not None:
+
+            def _getLocaleDataAdjusted(localeData):
+                """
+                If localeData is defined as ["mon|mnd", 'tu|tues'...] then this
+                function splits those definitions on |
+                """
+                adjusted = []
+                for d in localeData:
+                    if '|' in d:
+                        adjusted += d.split("|")
+                    else:
+                        adjusted.append(d)
+                return adjusted
+
+            def re_join(g):
+                return '|'.join(re.escape(i) for i in g)
+
+            mths = _getLocaleDataAdjusted(self.locale.Months)
+            smths = _getLocaleDataAdjusted(self.locale.shortMonths)
+            swds = _getLocaleDataAdjusted(self.locale.shortWeekdays)
+            wds = _getLocaleDataAdjusted(self.locale.Weekdays)
+
+            # escape any regex special characters that may be found
+            self.locale.re_values['months'] = re_join(mths)
+            self.locale.re_values['shortmonths'] = re_join(smths)
+            self.locale.re_values['days'] = re_join(wds)
+            self.locale.re_values['shortdays'] = re_join(swds)
+            self.locale.re_values['dayoffsets'] = \
+                re_join(self.locale.dayOffsets)
+            self.locale.re_values['numbers'] = \
+                re_join(self.locale.numbers)
+            self.locale.re_values['decimal_mark'] = \
+                re.escape(self.locale.decimal_mark)
+
+            units = [unit for units in self.locale.units.values()
+                     for unit in units]  # flatten
+            units.sort(key=len, reverse=True)  # longest first
+            self.locale.re_values['units'] = re_join(units)
+            self.locale.re_values['modifiers'] = re_join(self.locale.Modifiers)
+            self.locale.re_values['sources'] = re_join(self.locale.re_sources)
+
+            # For distinguishing numeric dates from times, look for timeSep
+            # and meridian, if specified in the locale
+            self.locale.re_values['timecomponents'] = \
+                re_join(self.locale.timeSep + self.locale.meridian)
+
+            # build weekday offsets - yes, it assumes the Weekday and
+            # shortWeekday lists are in the same order and Mon..Sun
+            # (Python style)
+            def _buildOffsets(offsetDict, localeData, indexStart):
+                o = indexStart
+                for key in localeData:
+                    if '|' in key:
+                        for k in key.split('|'):
+                            offsetDict[k] = o
+                    else:
+                        offsetDict[key] = o
+                    o += 1
+
+            _buildOffsets(self.locale.WeekdayOffsets,
+                          self.locale.Weekdays, 0)
+            _buildOffsets(self.locale.WeekdayOffsets,
+                          self.locale.shortWeekdays, 0)
+
+            # build month offsets - yes, it assumes the Months and shortMonths
+            # lists are in the same order and Jan..Dec
+            _buildOffsets(self.locale.MonthOffsets,
+                          self.locale.Months, 1)
+            _buildOffsets(self.locale.MonthOffsets,
+                          self.locale.shortMonths, 1)
+
+        _initSymbols(self)
+
+        # TODO: add code to parse the date formats and build the regexes up
+        # from sub-parts, find all hard-coded uses of date/time separators
+
+        # not being used in code, but kept in case others are manually
+        # utilizing this regex for their own purposes
+        self.RE_DATE4 = r'''(?P<date>
+                                (
+                                    (
+                                        (?P<day>\d\d?)
+                                        (?P<suffix>{daysuffix})?
+                                        (,)?
+                                        (\s)*
+                                    )
+                                    (?P<mthname>
+                                        \b({months}|{shortmonths})\b
+                                    )\s*
+                                    (?P<year>\d\d
+                                        (\d\d)?
+                                    )?
+                                )
+                            )'''.format(**self.locale.re_values)
+
+        # still not completely sure of the behavior of the regex and
+        # whether it would be best to consume all possible irrelevant
+        # characters before the option groups (but within the {1,3} repetition
+        # group or inside of each option group, as it currently does
+        # however, right now, all tests are passing that were,
+        # including fixing the bug of matching a 4-digit year as ddyy
+        # when the day is absent from the string
+        self.RE_DATE3 = r'''(?P<date>
+                                (?:
+                                    (?:^|\s+)
+                                    (?P<mthname>
+                                        {months}|{shortmonths}
+                                    )\b
+                                    |
+                                    (?:^|\s+)
+                                    (?P<day>[1-9]|[012]\d|3[01])
+                                    (?P<suffix>{daysuffix}|)\b
+                                    (?!\s*(?:{timecomponents}))
+                                    |
+                                    ,?\s+
+                                    (?P<year>\d\d(?:\d\d|))\b
+                                    (?!\s*(?:{timecomponents}))
+                                ){{1,3}}
+                                (?(mthname)|$-^)
+                            )'''.format(**self.locale.re_values)
+
+        # not being used in code, but kept in case others are manually
+        # utilizing this regex for their own purposes
+        self.RE_MONTH = r'''(\s+|^)
+                            (?P<month>
+                                (
+                                    (?P<mthname>
+                                        \b({months}|{shortmonths})\b
+                                    )
+                                    (\s*
+                                        (?P<year>(\d{{4}}))
+                                    )?
+                                )
+                            )
+                            (?=\s+|$|[^\w])'''.format(**self.locale.re_values)
+
+        self.RE_WEEKDAY = r'''\b
+                              (?:
+                                  {days}|{shortdays}
+                              )
+                              \b'''.format(**self.locale.re_values)
+
+        self.RE_NUMBER = (r'(\b(?:{numbers})\b|\d+(?:{decimal_mark}\d+|))'
+                          .format(**self.locale.re_values))
+
+        self.RE_SPECIAL = (r'(?P<special>^[{specials}]+)\s+'
+                           .format(**self.locale.re_values))
+
+        self.RE_UNITS_ONLY = (r'''\b({units})\b'''
+                              .format(**self.locale.re_values))
+
+        self.RE_UNITS = r'''\b(?P<qty>
+                                -?
+                                (?:\d+(?:{decimal_mark}\d+|)|(?:{numbers})\b)\s*
+                                (?P<units>{units})
+                            )\b'''.format(**self.locale.re_values)
+
+        self.RE_QUNITS = r'''\b(?P<qty>
+                                 -?
+                                 (?:\d+(?:{decimal_mark}\d+|)|(?:{numbers})\s+)\s*
+                                 (?P<qunits>{qunits})
+                             )\b'''.format(**self.locale.re_values)
+
+        self.RE_MODIFIER = r'''\b(?:
+                                   {modifiers}
+                               )\b'''.format(**self.locale.re_values)
+
+        self.RE_TIMEHMS = r'''([\s(\["'-]|^)
+                              (?P<hours>\d\d?)
+                              (?P<tsep>{timeseparator}|)
+                              (?P<minutes>\d\d)
+                              (?:(?P=tsep)
+                                  (?P<seconds>\d\d
+                                      (?:[\.,]\d+)?
+                                  )
+                              )?\b'''.format(**self.locale.re_values)
+
+        self.RE_TIMEHMS2 = r'''([\s(\["'-]|^)
+                               (?P<hours>\d\d?)
+                               (?:
+                                   (?P<tsep>{timeseparator}|)
+                                   (?P<minutes>\d\d?)
+                                   (?:(?P=tsep)
+                                       (?P<seconds>\d\d?
+                                           (?:[\.,]\d+)?
+                                       )
+                                   )?
+                               )?'''.format(**self.locale.re_values)
+
+        # 1, 2, and 3 here refer to the type of match date, time, or units
+        self.RE_NLP_PREFIX = r'''\b(?P<nlp_prefix>
+                                  (on)
+                                  (\s)+1
+                                  |
+                                  (at|in)
+                                  (\s)+2
+                                  |
+                                  (in)
+                                  (\s)+3
+                                 )'''
+
+        if 'meridian' in self.locale.re_values:
+            self.RE_TIMEHMS2 += (r'\s*(?P<meridian>{meridian})\b'
+                                 .format(**self.locale.re_values))
+        else:
+            self.RE_TIMEHMS2 += r'\b'
+
+        # Always support common . and - separators
+        dateSeps = ''.join(re.escape(s)
+                           for s in self.locale.dateSep + ['-', '.'])
+
+        self.RE_DATE = r'''([\s(\["'-]|^)
+                           (?P<date>
+                                \d\d?[{0}]\d\d?(?:[{0}]\d\d(?:\d\d)?)?
+                                |
+                                \d{{4}}[{0}]\d\d?[{0}]\d\d?
+                            )
+                           \b'''.format(dateSeps)
+
+        self.RE_DATE2 = r'[{0}]'.format(dateSeps)
+
+        assert 'dayoffsets' in self.locale.re_values
+
+        self.RE_DAY = r'''\b
+                          (?:
+                              {dayoffsets}
+                          )
+                          \b'''.format(**self.locale.re_values)
+
+        self.RE_DAY2 = r'''(?P<day>\d\d?)
+                           (?P<suffix>{daysuffix})?
+                       '''.format(**self.locale.re_values)
+
+        self.RE_TIME = r'''\b
+                           (?:
+                               {sources}
+                           )
+                           \b'''.format(**self.locale.re_values)
+
+        self.RE_REMAINING = r'\s+'
+
+        # Regex for date/time ranges
+        self.RE_RTIMEHMS = r'''(\s*|^)
+                               (\d\d?){timeseparator}
+                               (\d\d)
+                               ({timeseparator}(\d\d))?
+                               (\s*|$)'''.format(**self.locale.re_values)
+
+        self.RE_RTIMEHMS2 = (r'''(\s*|^)
+                                 (\d\d?)
+                                 ({timeseparator}(\d\d?))?
+                                 ({timeseparator}(\d\d?))?'''
+                             .format(**self.locale.re_values))
+
+        if 'meridian' in self.locale.re_values:
+            self.RE_RTIMEHMS2 += (r'\s*({meridian})'
+                                  .format(**self.locale.re_values))
+
+        self.RE_RDATE = r'(\d+([%s]\d+)+)' % dateSeps
+        self.RE_RDATE3 = r'''(
+                                (
+                                    (
+                                        \b({months})\b
+                                    )\s*
+                                    (
+                                        (\d\d?)
+                                        (\s?|{daysuffix}|$)+
+                                    )?
+                                    (,\s*\d{{4}})?
+                                )
+                            )'''.format(**self.locale.re_values)
+
+        # "06/07/06 - 08/09/06"
+        self.DATERNG1 = (r'{0}\s*{rangeseparator}\s*{0}'
+                         .format(self.RE_RDATE, **self.locale.re_values))
+
+        # "march 31 - june 1st, 2006"
+        self.DATERNG2 = (r'{0}\s*{rangeseparator}\s*{0}'
+                         .format(self.RE_RDATE3, **self.locale.re_values))
+
+        # "march 1rd -13th"
+        self.DATERNG3 = (r'{0}\s*{rangeseparator}\s*(\d\d?)\s*(rd|st|nd|th)?'
+                         .format(self.RE_RDATE3, **self.locale.re_values))
+
+        # "4:00:55 pm - 5:90:44 am", '4p-5p'
+        self.TIMERNG1 = (r'{0}\s*{rangeseparator}\s*{0}'
+                         .format(self.RE_RTIMEHMS2, **self.locale.re_values))
+
+        self.TIMERNG2 = (r'{0}\s*{rangeseparator}\s*{0}'
+                         .format(self.RE_RTIMEHMS, **self.locale.re_values))
+
+        # "4-5pm "
+        self.TIMERNG3 = (r'\d\d?\s*{rangeseparator}\s*{0}'
+                         .format(self.RE_RTIMEHMS2, **self.locale.re_values))
+
+        # "4:30-5pm "
+        self.TIMERNG4 = (r'{0}\s*{rangeseparator}\s*{1}'
+                         .format(self.RE_RTIMEHMS, self.RE_RTIMEHMS2,
+                                 **self.locale.re_values))
+
+        self.re_option = re.IGNORECASE + re.VERBOSE
+        self.cre_source = {'CRE_SPECIAL': self.RE_SPECIAL,
+                           'CRE_NUMBER': self.RE_NUMBER,
+                           'CRE_UNITS': self.RE_UNITS,
+                           'CRE_UNITS_ONLY': self.RE_UNITS_ONLY,
+                           'CRE_QUNITS': self.RE_QUNITS,
+                           'CRE_MODIFIER': self.RE_MODIFIER,
+                           'CRE_TIMEHMS': self.RE_TIMEHMS,
+                           'CRE_TIMEHMS2': self.RE_TIMEHMS2,
+                           'CRE_DATE': self.RE_DATE,
+                           'CRE_DATE2': self.RE_DATE2,
+                           'CRE_DATE3': self.RE_DATE3,
+                           'CRE_DATE4': self.RE_DATE4,
+                           'CRE_MONTH': self.RE_MONTH,
+                           'CRE_WEEKDAY': self.RE_WEEKDAY,
+                           'CRE_DAY': self.RE_DAY,
+                           'CRE_DAY2': self.RE_DAY2,
+                           'CRE_TIME': self.RE_TIME,
+                           'CRE_REMAINING': self.RE_REMAINING,
+                           'CRE_RTIMEHMS': self.RE_RTIMEHMS,
+                           'CRE_RTIMEHMS2': self.RE_RTIMEHMS2,
+                           'CRE_RDATE': self.RE_RDATE,
+                           'CRE_RDATE3': self.RE_RDATE3,
+                           'CRE_TIMERNG1': self.TIMERNG1,
+                           'CRE_TIMERNG2': self.TIMERNG2,
+                           'CRE_TIMERNG3': self.TIMERNG3,
+                           'CRE_TIMERNG4': self.TIMERNG4,
+                           'CRE_DATERNG1': self.DATERNG1,
+                           'CRE_DATERNG2': self.DATERNG2,
+                           'CRE_DATERNG3': self.DATERNG3,
+                           'CRE_NLP_PREFIX': self.RE_NLP_PREFIX}
+        self.cre_keys = set(self.cre_source.keys())
+
+    def __getattr__(self, name):
+        if name in self.cre_keys:
+            value = re.compile(self.cre_source[name], self.re_option)
+            setattr(self, name, value)
+            return value
+        elif name in self.locale.locale_keys:
+            return getattr(self.locale, name)
+        else:
+            raise AttributeError(name)
+
+    def daysInMonth(self, month, year):
+        """
+        Take the given month (1-12) and a given year (4 digit) return
+        the number of days in the month adjusting for leap year as needed
+        """
+        result = None
+        debug and log.debug('daysInMonth(%s, %s)', month, year)
+        if month > 0 and month <= 12:
+            result = self._DaysInMonthList[month - 1]
+
+            if month == 2:
+                if year in self._leapYears:
+                    result += 1
+                else:
+                    if calendar.isleap(year):
+                        self._leapYears.append(year)
+                        result += 1
+
+        return result
+
+    def getSource(self, sourceKey, sourceTime=None):
+        """
+        GetReturn a date/time tuple based on the giving source key
+        and the corresponding key found in self.re_sources.
+
+        The current time is used as the default and any specified
+        item found in self.re_sources is inserted into the value
+        and the generated dictionary is returned.
+        """
+        if sourceKey not in self.re_sources:
+            return None
+
+        if sourceTime is None:
+            (yr, mth, dy, hr, mn, sec, wd, yd, isdst) = time.localtime()
+        else:
+            (yr, mth, dy, hr, mn, sec, wd, yd, isdst) = sourceTime
+
+        defaults = {'yr': yr, 'mth': mth, 'dy': dy,
+                    'hr': hr, 'mn': mn, 'sec': sec}
+
+        source = self.re_sources[sourceKey]
+
+        values = {}
+
+        for key, default in defaults.items():
+            values[key] = source.get(key, default)
+
+        return (values['yr'], values['mth'], values['dy'],
+                values['hr'], values['mn'], values['sec'],
+                wd, yd, isdst)
diff --git a/src/bindings/python/flux/utils/parsedatetime/context.py b/src/bindings/python/flux/utils/parsedatetime/context.py
new file mode 100644
index 000000000000..c1cc39aeeb11
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/context.py
@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*-
+"""
+parsedatetime/context.py
+
+Context related classes
+
+"""
+
+from threading import local
+
+
+class pdtContextStack(object):
+    """
+    A thread-safe stack to store context(s)
+
+    Internally used by L{Calendar} object
+    """
+
+    def __init__(self):
+        self.__local = local()
+
+    @property
+    def __stack(self):
+        if not hasattr(self.__local, 'stack'):
+            self.__local.stack = []
+        return self.__local.stack
+
+    def push(self, ctx):
+        self.__stack.append(ctx)
+
+    def pop(self):
+        try:
+            return self.__stack.pop()
+        except IndexError:
+            return None
+
+    def last(self):
+        try:
+            return self.__stack[-1]
+        except IndexError:
+            raise RuntimeError('context stack is empty')
+
+    def isEmpty(self):
+        return not self.__stack
+
+
+class pdtContext(object):
+    """
+    Context contains accuracy flag detected by L{Calendar.parse()}
+
+    Accuracy flag uses bitwise-OR operation and is combined by:
+
+        ACU_YEAR - "next year", "2014"
+        ACU_MONTH - "March", "July 2014"
+        ACU_WEEK - "last week", "next 3 weeks"
+        ACU_DAY - "tomorrow", "July 4th 2014"
+        ACU_HALFDAY - "morning", "tonight"
+        ACU_HOUR - "18:00", "next hour"
+        ACU_MIN - "18:32", "next 10 minutes"
+        ACU_SEC - "18:32:55"
+        ACU_NOW - "now"
+
+    """
+
+    __slots__ = ('accuracy',)
+
+    ACU_YEAR = 2 ** 0
+    ACU_MONTH = 2 ** 1
+    ACU_WEEK = 2 ** 2
+    ACU_DAY = 2 ** 3
+    ACU_HALFDAY = 2 ** 4
+    ACU_HOUR = 2 ** 5
+    ACU_MIN = 2 ** 6
+    ACU_SEC = 2 ** 7
+    ACU_NOW = 2 ** 8
+
+    ACU_DATE = ACU_YEAR | ACU_MONTH | ACU_WEEK | ACU_DAY
+    ACU_TIME = ACU_HALFDAY | ACU_HOUR | ACU_MIN | ACU_SEC | ACU_NOW
+
+    _ACCURACY_MAPPING = [
+        (ACU_YEAR, 'year'),
+        (ACU_MONTH, 'month'),
+        (ACU_WEEK, 'week'),
+        (ACU_DAY, 'day'),
+        (ACU_HALFDAY, 'halfday'),
+        (ACU_HOUR, 'hour'),
+        (ACU_MIN, 'min'),
+        (ACU_SEC, 'sec'),
+        (ACU_NOW, 'now')]
+
+    _ACCURACY_REVERSE_MAPPING = {
+        'year': ACU_YEAR,
+        'years': ACU_YEAR,
+        'month': ACU_MONTH,
+        'months': ACU_MONTH,
+        'week': ACU_WEEK,
+        'weeks': ACU_WEEK,
+        'day': ACU_DAY,
+        'days': ACU_DAY,
+        'halfday': ACU_HALFDAY,
+        'morning': ACU_HALFDAY,
+        'afternoon': ACU_HALFDAY,
+        'evening': ACU_HALFDAY,
+        'night': ACU_HALFDAY,
+        'tonight': ACU_HALFDAY,
+        'midnight': ACU_HALFDAY,
+        'hour': ACU_HOUR,
+        'hours': ACU_HOUR,
+        'min': ACU_MIN,
+        'minute': ACU_MIN,
+        'mins': ACU_MIN,
+        'minutes': ACU_MIN,
+        'sec': ACU_SEC,
+        'second': ACU_SEC,
+        'secs': ACU_SEC,
+        'seconds': ACU_SEC,
+        'now': ACU_NOW}
+
+    def __init__(self, accuracy=0):
+        """
+        Default constructor of L{pdtContext} class.
+
+        @type  accuracy: integer
+        @param accuracy: Accuracy flag
+
+        @rtype:  object
+        @return: L{pdtContext} instance
+        """
+        self.accuracy = accuracy
+
+    def updateAccuracy(self, *accuracy):
+        """
+        Updates current accuracy flag
+        """
+        for acc in accuracy:
+            if not isinstance(acc, int):
+                acc = self._ACCURACY_REVERSE_MAPPING[acc]
+            self.accuracy |= acc
+
+    def update(self, context):
+        """
+        Uses another L{pdtContext} instance to update current one
+        """
+        self.updateAccuracy(context.accuracy)
+
+    @property
+    def hasDate(self):
+        """
+        Returns True if current context is accurate to date
+        """
+        return bool(self.accuracy & self.ACU_DATE)
+
+    @property
+    def hasTime(self):
+        """
+        Returns True if current context is accurate to time
+        """
+        return bool(self.accuracy & self.ACU_TIME)
+
+    @property
+    def dateTimeFlag(self):
+        """
+        Returns the old date/time flag code
+        """
+        return int(self.hasDate and 1) | int(self.hasTime and 2)
+
+    @property
+    def hasDateOrTime(self):
+        """
+        Returns True if current context is accurate to date/time
+        """
+        return bool(self.accuracy)
+
+    def __repr__(self):
+        accuracy_repr = []
+        for acc, name in self._ACCURACY_MAPPING:
+            if acc & self.accuracy:
+                accuracy_repr.append('pdtContext.ACU_%s' % name.upper())
+        if accuracy_repr:
+            accuracy_repr = 'accuracy=' + ' | '.join(accuracy_repr)
+        else:
+            accuracy_repr = ''
+
+        return 'pdtContext(%s)' % accuracy_repr
+
+    def __eq__(self, ctx):
+        return self.accuracy == ctx.accuracy
diff --git a/src/bindings/python/flux/utils/parsedatetime/parsedatetime.py b/src/bindings/python/flux/utils/parsedatetime/parsedatetime.py
new file mode 100644
index 000000000000..647eb06caa3d
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/parsedatetime.py
@@ -0,0 +1,2 @@
+# Backward compatibility fix.
+from . import *         # noqa
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/__init__.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/__init__.py
new file mode 100644
index 000000000000..cb057181e212
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/__init__.py
@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+"""
+pdt_locales
+
+All of the included locale classes shipped with pdt.
+"""
+
+from __future__ import absolute_import
+from .icu import get_icu
+
+locales = ['de_DE', 'en_AU', 'en_US', 'es', 'nl_NL', 'pt_BR', 'ru_RU', 'fr_FR']
+
+__locale_caches = {}
+
+__all__ = ['get_icu', 'load_locale']
+
+
+def load_locale(locale, icu=False):
+    """
+    Return data of locale
+    :param locale:
+    :return:
+    """
+    if locale not in locales:
+        raise NotImplementedError("The locale '%s' is not supported" % locale)
+    if locale not in __locale_caches:
+        mod = __import__(__name__, fromlist=[locale], level=0)
+        __locale_caches[locale] = getattr(mod, locale)
+    return __locale_caches[locale]
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/base.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/base.py
new file mode 100644
index 000000000000..7a4871d29fcd
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/base.py
@@ -0,0 +1,199 @@
+from __future__ import unicode_literals
+
+locale_keys = set([
+    'MonthOffsets', 'Months', 'WeekdayOffsets', 'Weekdays',
+    'dateFormats', 'dateSep', 'dayOffsets', 'dp_order',
+    'localeID', 'meridian', 'Modifiers', 're_sources', 're_values',
+    'shortMonths', 'shortWeekdays', 'timeFormats', 'timeSep', 'units',
+    'uses24', 'usesMeridian', 'numbers', 'decimal_mark', 'small',
+    'magnitude', 'ignore'])
+
+localeID = None
+
+dateSep = ['/', '.']
+timeSep = [':']
+meridian = ['AM', 'PM']
+usesMeridian = True
+uses24 = True
+WeekdayOffsets = {}
+MonthOffsets = {}
+
+# always lowercase any lookup values - helper code expects that
+Weekdays = [
+    'monday', 'tuesday', 'wednesday', 'thursday',
+    'friday', 'saturday', 'sunday',
+]
+
+shortWeekdays = [
+    'mon', 'tues|tue', 'wed', 'thu', 'fri', 'sat', 'sun',
+]
+
+Months = [
+    'january', 'february', 'march', 'april', 'may', 'june', 'july',
+    'august', 'september', 'october', 'november', 'december',
+]
+
+shortMonths = [
+    'jan', 'feb', 'mar', 'apr', 'may', 'jun',
+    'jul', 'aug', 'sep', 'oct', 'nov', 'dec',
+]
+
+# use the same formats as ICU by default
+dateFormats = {
+    'full': 'EEEE, MMMM d, yyyy',
+    'long': 'MMMM d, yyyy',
+    'medium': 'MMM d, yyyy',
+    'short': 'M/d/yy'
+}
+
+timeFormats = {
+    'full': 'h:mm:ss a z',
+    'long': 'h:mm:ss a z',
+    'medium': 'h:mm:ss a',
+    'short': 'h:mm a',
+}
+
+dp_order = ['m', 'd', 'y']
+
+# Used to parse expressions like "in 5 hours"
+numbers = {
+    'zero': 0,
+    'one': 1,
+    'a': 1,
+    'an': 1,
+    'two': 2,
+    'three': 3,
+    'four': 4,
+    'five': 5,
+    'six': 6,
+    'seven': 7,
+    'eight': 8,
+    'nine': 9,
+    'ten': 10,
+    'eleven': 11,
+    'thirteen': 13,
+    'fourteen': 14,
+    'fifteen': 15,
+    'sixteen': 16,
+    'seventeen': 17,
+    'eighteen': 18,
+    'nineteen': 19,
+    'twenty': 20,
+}
+
+decimal_mark = '.'
+
+
+# this will be added to re_values later
+units = {
+    'seconds': ['second', 'seconds', 'sec', 'secs', 's'],
+    'minutes': ['minute', 'minutes', 'min', 'mins', 'm'],
+    'hours': ['hour', 'hours', 'hr', 'h'],
+    'days': ['day', 'days', 'dy', 'd'],
+    'weeks': ['week', 'weeks', 'wk', 'w'],
+    'months': ['month', 'months', 'mth'],
+    'years': ['year', 'years', 'yr', 'y'],
+}
+
+
+# text constants to be used by later regular expressions
+re_values = {
+    'specials': 'in|on|of|at',
+    'timeseparator': ':',
+    'rangeseparator': '-',
+    'daysuffix': 'rd|st|nd|th',
+    'meridian': r'am|pm|a\.m\.|p\.m\.|a|p',
+    'qunits': 'h|m|s|d|w|y',
+    'now': ['now', 'right now'],
+}
+
+# Used to adjust the returned date before/after the source
+Modifiers = {
+    'from': 1,
+    'before': -1,
+    'after': 1,
+    'ago': -1,
+    'prior': -1,
+    'prev': -1,
+    'last': -1,
+    'next': 1,
+    'previous': -1,
+    'end of': 0,
+    'this': 0,
+    'eod': 1,
+    'eom': 1,
+    'eoy': 1,
+}
+
+dayOffsets = {
+    'tomorrow': 1,
+    'today': 0,
+    'yesterday': -1,
+}
+
+# special day and/or times, i.e. lunch, noon, evening
+# each element in the dictionary is a dictionary that is used
+# to fill in any value to be replace - the current date/time will
+# already have been populated by the method buildSources
+re_sources = {
+    'noon': {'hr': 12, 'mn': 0, 'sec': 0},
+    'afternoon': {'hr': 13, 'mn': 0, 'sec': 0},
+    'lunch': {'hr': 12, 'mn': 0, 'sec': 0},
+    'morning': {'hr': 6, 'mn': 0, 'sec': 0},
+    'breakfast': {'hr': 8, 'mn': 0, 'sec': 0},
+    'dinner': {'hr': 19, 'mn': 0, 'sec': 0},
+    'evening': {'hr': 18, 'mn': 0, 'sec': 0},
+    'midnight': {'hr': 0, 'mn': 0, 'sec': 0},
+    'night': {'hr': 21, 'mn': 0, 'sec': 0},
+    'tonight': {'hr': 21, 'mn': 0, 'sec': 0},
+    'eod': {'hr': 17, 'mn': 0, 'sec': 0},
+}
+
+small = {
+    'zero': 0,
+    'one': 1,
+    'a': 1,
+    'an': 1,
+    'two': 2,
+    'three': 3,
+    'four': 4,
+    'five': 5,
+    'six': 6,
+    'seven': 7,
+    'eight': 8,
+    'nine': 9,
+    'ten': 10,
+    'eleven': 11,
+    'twelve': 12,
+    'thirteen': 13,
+    'fourteen': 14,
+    'fifteen': 15,
+    'sixteen': 16,
+    'seventeen': 17,
+    'eighteen': 18,
+    'nineteen': 19,
+    'twenty': 20,
+    'thirty': 30,
+    'forty': 40,
+    'fifty': 50,
+    'sixty': 60,
+    'seventy': 70,
+    'eighty': 80,
+    'ninety': 90
+}
+
+magnitude = {
+    'thousand': 1000,
+    'million': 1000000,
+    'billion': 1000000000,
+    'trillion': 1000000000000,
+    'quadrillion': 1000000000000000,
+    'quintillion': 1000000000000000000,
+    'sextillion': 1000000000000000000000,
+    'septillion': 1000000000000000000000000,
+    'octillion': 1000000000000000000000000000,
+    'nonillion': 1000000000000000000000000000000,
+    'decillion': 1000000000000000000000000000000000,
+}
+
+ignore = ('and', ',')
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/de_DE.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/de_DE.py
new file mode 100644
index 000000000000..afee991c1041
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/de_DE.py
@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from .base import *  # noqa
+
+# don't use an unicode string
+localeID = 'de_DE'
+dateSep = ['.']
+timeSep = [':']
+meridian = []
+usesMeridian = False
+uses24 = True
+decimal_mark = ','
+
+Weekdays = [
+    'montag', 'dienstag', 'mittwoch',
+    'donnerstag', 'freitag', 'samstag', 'sonntag',
+]
+shortWeekdays = ['mo', 'di', 'mi', 'do', 'fr', 'sa', 'so']
+Months = [
+    'januar', 'februar', 'märz',
+    'april', 'mai', 'juni',
+    'juli', 'august', 'september',
+    'oktober', 'november', 'dezember',
+]
+shortMonths = [
+    'jan', 'feb', 'mrz', 'apr', 'mai', 'jun',
+    'jul', 'aug', 'sep', 'okt', 'nov', 'dez',
+]
+
+dateFormats = {
+    'full': 'EEEE, d. MMMM yyyy',
+    'long': 'd. MMMM yyyy',
+    'medium': 'dd.MM.yyyy',
+    'short': 'dd.MM.yy',
+}
+
+timeFormats = {
+    'full': 'HH:mm:ss v',
+    'long': 'HH:mm:ss z',
+    'medium': 'HH:mm:ss',
+    'short': 'HH:mm',
+}
+
+dp_order = ['d', 'm', 'y']
+
+# the short version would be a capital M,
+# as I understand it we can't distinguish
+# between m for minutes and M for months.
+units = {
+    'seconds': ['sekunden', 'sek', 's'],
+    'minutes': ['minuten', 'min', 'm'],
+    'hours': ['stunden', 'std', 'h'],
+    'days': ['tag', 'tage', 't'],
+    'weeks': ['wochen', 'w'],
+    'months': ['monat', 'monate'],
+    'years': ['jahr', 'jahre', 'j'],
+}
+
+re_values = re_values.copy()
+re_values.update({
+    'specials': 'am|dem|der|im|in|den|zum',
+    'timeseparator': ':',
+    'rangeseparator': '-',
+    'daysuffix': '',
+    'qunits': 'h|m|s|t|w|m|j',
+    'now': ['jetzt'],
+})
+
+# Used to adjust the returned date before/after the source
+# still looking for insight on how to translate all of them to german.
+Modifiers = {
+    'from': 1,
+    'before': -1,
+    'after': 1,
+    'vergangener': -1,
+    'vorheriger': -1,
+    'prev': -1,
+    'letzter': -1,
+    'nächster': 1,
+    'dieser': 0,
+    'previous': -1,
+    'in a': 2,
+    'end of': 0,
+    'eod': 0,
+    'eo': 0,
+}
+
+# morgen/abermorgen does not work, see
+# http://code.google.com/p/parsedatetime/issues/detail?id=19
+dayOffsets = {
+    'morgen': 1,
+    'heute': 0,
+    'gestern': -1,
+    'vorgestern': -2,
+    'übermorgen': 2,
+}
+
+# special day and/or times, i.e. lunch, noon, evening
+# each element in the dictionary is a dictionary that is used
+# to fill in any value to be replace - the current date/time will
+# already have been populated by the method buildSources
+re_sources = {
+    'mittag': {'hr': 12, 'mn': 0, 'sec': 0},
+    'mittags': {'hr': 12, 'mn': 0, 'sec': 0},
+    'mittagessen': {'hr': 12, 'mn': 0, 'sec': 0},
+    'morgen': {'hr': 6, 'mn': 0, 'sec': 0},
+    'morgens': {'hr': 6, 'mn': 0, 'sec': 0},
+    'frühstück': {'hr': 8, 'mn': 0, 'sec': 0},
+    'abendessen': {'hr': 19, 'mn': 0, 'sec': 0},
+    'abend': {'hr': 18, 'mn': 0, 'sec': 0},
+    'abends': {'hr': 18, 'mn': 0, 'sec': 0},
+    'mitternacht': {'hr': 0, 'mn': 0, 'sec': 0},
+    'nacht': {'hr': 21, 'mn': 0, 'sec': 0},
+    'nachts': {'hr': 21, 'mn': 0, 'sec': 0},
+    'heute abend': {'hr': 21, 'mn': 0, 'sec': 0},
+    'heute nacht': {'hr': 21, 'mn': 0, 'sec': 0},
+    'feierabend': {'hr': 17, 'mn': 0, 'sec': 0},
+}
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/en_AU.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/en_AU.py
new file mode 100644
index 000000000000..bff3d3f90a22
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/en_AU.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from .base import *  # noqa
+
+# don't use an unicode string
+localeID = 'en_AU'
+dateSep = ['-', '/']
+uses24 = False
+
+dateFormats = {
+    'full': 'EEEE, d MMMM yyyy',
+    'long': 'd MMMM yyyy',
+    'medium': 'dd/MM/yyyy',
+    'short': 'd/MM/yy',
+}
+
+timeFormats['long'] = timeFormats['full']
+
+dp_order = ['d', 'm', 'y']
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/en_US.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/en_US.py
new file mode 100644
index 000000000000..12584e570912
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/en_US.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from .base import *  # noqa
+
+# don't use an unicode string
+localeID = 'en_US'
+uses24 = False
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/es.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/es.py
new file mode 100644
index 000000000000..351f2544e36e
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/es.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from .base import *  # noqa
+
+# don't use an unicode string
+localeID = 'es'
+dateSep = ['/']
+usesMeridian = False
+uses24 = True
+decimal_mark = ','
+
+Weekdays = [
+    'lunes', 'martes', 'miércoles',
+    'jueves', 'viernes', 'sábado', 'domingo',
+]
+shortWeekdays = [
+    'lun', 'mar', 'mié',
+    'jue', 'vie', 'sáb', 'dom',
+]
+Months = [
+    'enero', 'febrero', 'marzo',
+    'abril', 'mayo', 'junio',
+    'julio', 'agosto', 'septiembre',
+    'octubre', 'noviembre', 'diciembre',
+]
+shortMonths = [
+    'ene', 'feb', 'mar',
+    'abr', 'may', 'jun',
+    'jul', 'ago', 'sep',
+    'oct', 'nov', 'dic',
+]
+dateFormats = {
+    'full': "EEEE d' de 'MMMM' de 'yyyy",
+    'long': "d' de 'MMMM' de 'yyyy",
+    'medium': "dd-MMM-yy",
+    'short': "d/MM/yy",
+}
+
+timeFormats = {
+    'full': "HH'H'mm' 'ss z",
+    'long': "HH:mm:ss z",
+    'medium': "HH:mm:ss",
+    'short': "HH:mm",
+}
+
+dp_order = ['d', 'm', 'y']
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/fr_FR.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/fr_FR.py
new file mode 100644
index 000000000000..237a78ac6fc7
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/fr_FR.py
@@ -0,0 +1,248 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from .base import *  # noqa
+
+# don't use an unicode string
+localeID = 'fr_FR'
+dateSep = [r'\/']
+timeSep = [':', 'h']
+meridian = ['du matin', 'du soir']
+usesMeridian = True
+uses24 = True
+WeekdayOffsets = {}
+MonthOffsets = {}
+
+# always lowercase any lookup values - helper code expects that
+Weekdays = [
+    'lundi', 'mardi', 'mercredi', 'jeudi',
+    'vendredi', 'samedi', 'dimanche',
+]
+
+shortWeekdays = [
+    'lun', 'mar', 'mer', 'jeu', 'ven', 'sam', 'dim',
+]
+
+Months = [
+    'janvier', 'février|fevrier', 'mars', 'avril', 'mai', 'juin', 'juillet',
+    'août|aout', 'septembre', 'octobre', 'novembre', 'décembre|decembre',
+]
+
+# We do not list 'mar' as a short name for 'mars' as it conflicts with
+# the 'mar' of 'mardi'
+shortMonths = [
+    'jan', 'fév|fev', 'mars', 'avr', 'mai', 'jui',
+    'juil', 'aoû|aou', 'sep', 'oct', 'nov', 'déc|dec',
+]
+
+# use the same formats as ICU by default
+dateFormats = {
+    'full': 'EEEE d MMMM yyyy',
+    'long': 'd MMMM yyyy',
+    'medium': 'd MMM yyyy',
+    'short': 'd/M/yy'
+}
+
+timeFormats = {
+    'full': 'h:mm:ss a z',
+    'long': 'h:mm:ss a z',
+    'medium': 'h:mm:ss a',
+    'short': 'h:mm a',
+}
+
+dp_order = ['d', 'm', 'y']
+
+# Used to parse expressions like "in 5 hours"
+numbers = {
+    'zéro': 0,
+    'zero': 0,
+    'un': 1,
+    'une': 1,
+    'deux': 2,
+    'trois': 3,
+    'quatre': 4,
+    'cinq': 5,
+    'six': 6,
+    'sept': 7,
+    'huit': 8,
+    'neuf': 9,
+    'dix': 10,
+    'onze': 11,
+    'douze': 12,
+    'treize': 13,
+    'quatorze': 14,
+    'quinze': 15,
+    'seize': 16,
+    'dix-sept': 17,
+    'dix sept': 17,
+    'dix-huit': 18,
+    'dix huit': 18,
+    'dix-neuf': 19,
+    'dix neuf': 19,
+    'vingt': 20,
+    'vingt-et-un': 21,
+    'vingt et un': 21,
+    'vingt-deux': 22,
+    'vingt deux': 22,
+    'vingt-trois': 23,
+    'vingt trois': 23,
+    'vingt-quatre': 24,
+    'vingt quatre': 24,
+}
+
+decimal_mark = ','
+
+# this will be added to re_values later
+units = {
+    'seconds': ['seconde', 'secondes', 'sec', 's'],
+    'minutes': ['minute', 'minutes', 'min', 'mn'],
+    'hours': ['heure', 'heures', 'h'],
+    'days': ['jour', 'jours', 'journée', 'journee', 'journées', 'journees', 'j'],
+    'weeks': ['semaine', 'semaines', 'sem'],
+    'months': ['mois', 'm'],
+    'years': ['année', 'annee', 'an', 'années', 'annees', 'ans'],
+}
+
+# text constants to be used by later regular expressions
+re_values = {
+    'specials': r'à|a|le|la|du|de',
+    'timeseparator': r'(?:\:|h|\s*heures?\s*)',
+    'rangeseparator': r'-',
+    'daysuffix': r'ième|ieme|ème|eme|ère|ere|nde',
+    'meridian': None,
+    'qunits': r'h|m|s|j|sem|a',
+    'now': [r'maintenant', r'tout de suite', r'immédiatement', r'immediatement', r'à l\'instant', r'a l\'instant'],
+}
+
+# Used to adjust the returned date before/after the source
+Modifiers = {
+    'avant': -1,
+    'il y a': -1,
+    'plus tot': -1,
+    'plus tôt': -1,
+    'y a': -1,
+    'antérieur': -1,
+    'anterieur': -1,
+    'dernier': -1,
+    'dernière': -1,
+    'derniere': -1,
+    'précédent': -1,
+    'précedent': -1,
+    'precédent': -1,
+    'precedent': -1,
+    'fin de': 0,
+    'fin du': 0,
+    'fin de la': 0,
+    'fin des': 0,
+    'fin d\'': 0,
+    'ce': 0,
+    'cette': 0,
+    'depuis': 1,
+    'dans': 1,
+    'à partir': 1,
+    'a partir': 1,
+    'après': 1,
+    'apres': 1,
+    'lendemain': 1,
+    'prochain': 1,
+    'prochaine': 1,
+    'suivant': 1,
+    'suivante': 1,
+    'plus tard': 1
+}
+
+dayOffsets = {
+    'après-demain': 2,
+    'apres-demain': 2,
+    'après demain': 2,
+    'apres demain': 2,
+    'demain': 1,
+    'aujourd\'hui': 0,
+    'hier': -1,
+    'avant-hier': -2,
+    'avant hier': -2
+}
+
+# special day and/or times, i.e. lunch, noon, evening
+# each element in the dictionary is a dictionary that is used
+# to fill in any value to be replace - the current date/time will
+# already have been populated by the method buildSources
+re_sources = {
+    'après-midi': {'hr': 13, 'mn': 0, 'sec': 0},
+    'apres-midi': {'hr': 13, 'mn': 0, 'sec': 0},
+    'après midi': {'hr': 13, 'mn': 0, 'sec': 0},
+    'apres midi': {'hr': 13, 'mn': 0, 'sec': 0},
+    'midi': {'hr': 12, 'mn': 0, 'sec': 0},
+    'déjeuner': {'hr': 12, 'mn': 0, 'sec': 0},
+    'dejeuner': {'hr': 12, 'mn': 0, 'sec': 0},
+    'matin': {'hr': 6, 'mn': 0, 'sec': 0},
+    'petit-déjeuner': {'hr': 8, 'mn': 0, 'sec': 0},
+    'petit-dejeuner': {'hr': 8, 'mn': 0, 'sec': 0},
+    'petit déjeuner': {'hr': 8, 'mn': 0, 'sec': 0},
+    'petit dejeuner': {'hr': 8, 'mn': 0, 'sec': 0},
+    'diner': {'hr': 19, 'mn': 0, 'sec': 0},
+    'dîner': {'hr': 19, 'mn': 0, 'sec': 0},
+    'soir': {'hr': 18, 'mn': 0, 'sec': 0},
+    'soirée': {'hr': 18, 'mn': 0, 'sec': 0},
+    'soiree': {'hr': 18, 'mn': 0, 'sec': 0},
+    'minuit': {'hr': 0, 'mn': 0, 'sec': 0},
+    'nuit': {'hr': 21, 'mn': 0, 'sec': 0},
+}
+
+small = {
+    'zéro': 0,
+    'zero': 0,
+    'un': 1,
+    'une': 1,
+    'deux': 2,
+    'trois': 3,
+    'quatre': 4,
+    'cinq': 5,
+    'six': 6,
+    'sept': 7,
+    'huit': 8,
+    'neuf': 9,
+    'dix': 10,
+    'onze': 11,
+    'douze': 12,
+    'treize': 13,
+    'quatorze': 14,
+    'quinze': 15,
+    'seize': 16,
+    'dix-sept': 17,
+    'dix sept': 17,
+    'dix-huit': 18,
+    'dix huit': 18,
+    'dix-neuf': 19,
+    'dix neuf': 19,
+    'vingt': 20,
+    'vingt-et-un': 21,
+    'vingt et un': 21,
+    'trente': 30,
+    'quarante': 40,
+    'cinquante': 50,
+    'soixante': 60,
+    'soixante-dix': 70,
+    'soixante dix': 70,
+    'quatre-vingt': 80,
+    'quatre vingt': 80,
+    'quatre-vingt-dix': 90,
+    'quatre vingt dix': 90
+}
+
+magnitude = {
+    'mille': 1000,
+    'millier': 1000,
+    'million': 1000000,
+    'milliard': 1000000000,
+    'trillion': 1000000000000,
+    'quadrillion': 1000000000000000,
+    'quintillion': 1000000000000000000,
+    'sextillion': 1000000000000000000000,
+    'septillion': 1000000000000000000000000,
+    'octillion': 1000000000000000000000000000,
+    'nonillion': 1000000000000000000000000000000,
+    'décillion': 1000000000000000000000000000000000,
+    'decillion': 1000000000000000000000000000000000,
+}
+
+ignore = ('et', ',')
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/icu.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/icu.py
new file mode 100644
index 000000000000..e09f51735576
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/icu.py
@@ -0,0 +1,157 @@
+# -*- encoding: utf-8 -*-
+
+"""
+pdt_locales
+
+All of the included locale classes shipped with pdt.
+"""
+import datetime
+
+try:
+    range = xrange
+except NameError:
+    pass
+
+try:
+    import icu as pyicu
+except ImportError:
+    try:
+        import PyICU as pyicu
+    except ImportError:
+        pyicu = None
+
+
+def icu_object(mapping):
+    return type('_icu', (object,), mapping)
+
+
+def merge_weekdays(base_wd, icu_wd):
+    result = []
+    for left, right in zip(base_wd, icu_wd):
+        if left == right:
+            result.append(left)
+            continue
+        left = set(left.split('|'))
+        right = set(right.split('|'))
+        result.append('|'.join(left | right))
+    return result
+
+
+def get_icu(locale):
+
+    def _sanitize_key(k):
+        import re
+        return re.sub("\\.(\\||$)", "\\1", k)
+
+    from . import base
+    result = dict([(key, getattr(base, key))
+                   for key in dir(base) if not key.startswith('_')])
+    result['icu'] = None
+
+    if pyicu is None:
+        return icu_object(result)
+
+    if locale is None:
+        locale = 'en_US'
+    result['icu'] = icu = pyicu.Locale(locale)
+
+    if icu is None:
+        return icu_object(result)
+
+    # grab spelled out format of all numbers from 0 to 100
+    rbnf = pyicu.RuleBasedNumberFormat(pyicu.URBNFRuleSetTag.SPELLOUT, icu)
+    result['numbers'].update([(rbnf.format(i), i) for i in range(0, 100)])
+
+    symbols = result['symbols'] = pyicu.DateFormatSymbols(icu)
+
+    # grab ICU list of weekdays, skipping first entry which
+    # is always blank
+    wd = [_sanitize_key(w.lower()) for w in symbols.getWeekdays()[1:]]
+    swd = [_sanitize_key(sw.lower()) for sw in symbols.getShortWeekdays()[1:]]
+
+    # store them in our list with Monday first (ICU puts Sunday first)
+    result['Weekdays'] = merge_weekdays(result['Weekdays'],
+                                        wd[1:] + wd[0:1])
+    result['shortWeekdays'] = merge_weekdays(result['shortWeekdays'],
+                                             swd[1:] + swd[0:1])
+    result['Months'] = [_sanitize_key(m.lower()) for m in symbols.getMonths()]
+    result['shortMonths'] = [_sanitize_key(sm.lower()) for sm in symbols.getShortMonths()]
+    keys = ['full', 'long', 'medium', 'short']
+
+    createDateInstance = pyicu.DateFormat.createDateInstance
+    createTimeInstance = pyicu.DateFormat.createTimeInstance
+    icu_df = result['icu_df'] = {
+        'full': createDateInstance(pyicu.DateFormat.kFull, icu),
+        'long': createDateInstance(pyicu.DateFormat.kLong, icu),
+        'medium': createDateInstance(pyicu.DateFormat.kMedium, icu),
+        'short': createDateInstance(pyicu.DateFormat.kShort, icu),
+    }
+    icu_tf = result['icu_tf'] = {
+        'full': createTimeInstance(pyicu.DateFormat.kFull, icu),
+        'long': createTimeInstance(pyicu.DateFormat.kLong, icu),
+        'medium': createTimeInstance(pyicu.DateFormat.kMedium, icu),
+        'short': createTimeInstance(pyicu.DateFormat.kShort, icu),
+    }
+
+    result['dateFormats'] = {}
+    result['timeFormats'] = {}
+    for x in keys:
+        result['dateFormats'][x] = icu_df[x].toPattern()
+        result['timeFormats'][x] = icu_tf[x].toPattern()
+
+    am = pm = ts = ''
+
+    # ICU doesn't seem to provide directly the date or time separator
+    # so we have to figure it out
+    o = result['icu_tf']['short']
+    s = result['timeFormats']['short']
+
+    result['usesMeridian'] = 'a' in s
+    result['uses24'] = 'H' in s
+
+    # '11:45 AM' or '11:45'
+    s = o.format(datetime.datetime(2003, 10, 30, 11, 45))
+
+    # ': AM' or ':'
+    s = s.replace('11', '').replace('45', '')
+
+    if len(s) > 0:
+        ts = s[0]
+
+    if result['usesMeridian']:
+        # '23:45 AM' or '23:45'
+        am = s[1:].strip()
+        s = o.format(datetime.datetime(2003, 10, 30, 23, 45))
+
+        if result['uses24']:
+            s = s.replace('23', '')
+        else:
+            s = s.replace('11', '')
+
+            # 'PM' or ''
+        pm = s.replace('45', '').replace(ts, '').strip()
+
+    result['timeSep'] = [ts]
+    result['meridian'] = [am, pm] if am and pm else []
+
+    o = result['icu_df']['short']
+    s = o.format(datetime.datetime(2003, 10, 30, 11, 45))
+    s = s.replace('10', '').replace('30', '').replace(
+        '03', '').replace('2003', '')
+
+    if len(s) > 0:
+        ds = s[0]
+    else:
+        ds = '/'
+
+    result['dateSep'] = [ds]
+    s = result['dateFormats']['short']
+    ll = s.lower().split(ds)
+    dp_order = []
+
+    for s in ll:
+        if len(s) > 0:
+            dp_order.append(s[:1])
+
+    result['dp_order'] = dp_order
+    return icu_object(result)
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/nl_NL.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/nl_NL.py
new file mode 100644
index 000000000000..41f9e7242c0e
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/nl_NL.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from .base import *  # noqa
+
+# don't use an unicode string
+localeID = 'nl_NL'
+dateSep = ['-', '/']
+timeSep = [':']
+meridian = []
+usesMeridian = False
+uses24 = True
+decimal_mark = ','
+
+Weekdays = [
+    'maandag', 'dinsdag', 'woensdag', 'donderdag',
+    'vrijdag', 'zaterdag', 'zondag',
+]
+shortWeekdays = [
+    'ma', 'di', 'wo', 'do', 'vr', 'za', 'zo',
+]
+Months = [
+    'januari', 'februari', 'maart', 'april', 'mei', 'juni', 'juli',
+    'augustus', 'september', 'oktober', 'november', 'december',
+]
+shortMonths = [
+    'jan', 'feb', 'mar', 'apr', 'mei', 'jun',
+    'jul', 'aug', 'sep', 'okt', 'nov', 'dec',
+]
+dateFormats = {
+    'full': 'EEEE, dd MMMM yyyy',
+    'long': 'dd MMMM yyyy',
+    'medium': 'dd-MM-yyyy',
+    'short': 'dd-MM-yy',
+}
+
+timeFormats = {
+    'full': 'HH:mm:ss v',
+    'long': 'HH:mm:ss z',
+    'medium': 'HH:mm:ss',
+    'short': 'HH:mm',
+}
+
+dp_order = ['d', 'm', 'y']
+
+# the short version would be a capital M,
+# as I understand it we can't distinguish
+# between m for minutes and M for months.
+units = {
+    'seconds': ['secunden', 'sec', 's'],
+    'minutes': ['minuten', 'min', 'm'],
+    'hours': ['uren', 'uur', 'h'],
+    'days': ['dagen', 'dag', 'd'],
+    'weeks': ['weken', 'w'],
+    'months': ['maanden', 'maand'],
+    'years': ['jaar', 'jaren', 'j'],
+}
+
+re_values = re_values.copy()
+re_values.update({
+    'specials': 'om',
+    'timeseparator': ':',
+    'rangeseparator': '-',
+    'daysuffix': ' |de',
+    'qunits': 'h|m|s|d|w|m|j',
+    'now': ['nu'],
+})
+
+# Used to adjust the returned date before/after the source
+# still looking for insight on how to translate all of them to german.
+Modifiers = {
+    'vanaf': 1,
+    'voor': -1,
+    'na': 1,
+    'eervorige': -1,
+    'prev': -1,
+    'laastste': -1,
+    'volgende': 1,
+    'deze': 0,
+    'vorige': -1,
+    'over': 2,
+    'eind van': 0,
+}
+
+# morgen/abermorgen does not work, see
+# http://code.google.com/p/parsedatetime/issues/detail?id=19
+dayOffsets = {
+    'morgen': 1,
+    'vandaag': 0,
+    'gisteren': -1,
+    'eergisteren': -2,
+    'overmorgen': 2,
+}
+
+# special day and/or times, i.e. lunch, noon, evening
+# each element in the dictionary is a dictionary that is used
+# to fill in any value to be replace - the current date/time will
+# already have been populated by the method buildSources
+re_sources = {
+    'middag': {'hr': 12, 'mn': 0, 'sec': 0},
+    'vanmiddag': {'hr': 12, 'mn': 0, 'sec': 0},
+    'lunch': {'hr': 12, 'mn': 0, 'sec': 0},
+    'morgen': {'hr': 6, 'mn': 0, 'sec': 0},
+    "'s morgens": {'hr': 6, 'mn': 0, 'sec': 0},
+    'ontbijt': {'hr': 8, 'mn': 0, 'sec': 0},
+    'avondeten': {'hr': 19, 'mn': 0, 'sec': 0},
+    'avond': {'hr': 18, 'mn': 0, 'sec': 0},
+    'avonds': {'hr': 18, 'mn': 0, 'sec': 0},
+    'middernacht': {'hr': 0, 'mn': 0, 'sec': 0},
+    'nacht': {'hr': 21, 'mn': 0, 'sec': 0},
+    'nachts': {'hr': 21, 'mn': 0, 'sec': 0},
+    'vanavond': {'hr': 21, 'mn': 0, 'sec': 0},
+    'vannacht': {'hr': 21, 'mn': 0, 'sec': 0},
+}
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/pt_BR.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/pt_BR.py
new file mode 100644
index 000000000000..21fdf6d1d777
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/pt_BR.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from .base import *  # noqa
+
+# don't use an unicode string
+localeID = 'pt_BR'
+dateSep = ['/']
+usesMeridian = False
+uses24 = True
+decimal_mark = ','
+
+Weekdays = [
+    'segunda-feira', 'terça-feira', 'quarta-feira',
+    'quinta-feira', 'sexta-feira', 'sábado', 'domingo',
+]
+shortWeekdays = [
+    'seg', 'ter', 'qua', 'qui', 'sex', 'sáb', 'dom',
+]
+Months = [
+    'janeiro', 'fevereiro', 'março', 'abril', 'maio', 'junho', 'julho',
+    'agosto', 'setembro', 'outubro', 'novembro', 'dezembro'
+]
+shortMonths = [
+    'jan', 'fev', 'mar', 'abr', 'mai', 'jun',
+    'jul', 'ago', 'set', 'out', 'nov', 'dez'
+]
+dateFormats = {
+    'full': "EEEE, d' de 'MMMM' de 'yyyy",
+    'long': "d' de 'MMMM' de 'yyyy",
+    'medium': "dd-MM-yy",
+    'short': "dd/MM/yyyy",
+}
+
+timeFormats = {
+    'full': "HH'H'mm' 'ss z",
+    'long': "HH:mm:ss z",
+    'medium': "HH:mm:ss",
+    'short': "HH:mm",
+}
+
+dp_order = ['d', 'm', 'y']
+
+units = {
+    'seconds': ['segundo', 'seg', 's'],
+    'minutes': ['minuto', 'min', 'm'],
+    'days': ['dia', 'dias', 'd'],
+    'months': ['mês', 'meses'],
+}
diff --git a/src/bindings/python/flux/utils/parsedatetime/pdt_locales/ru_RU.py b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/ru_RU.py
new file mode 100644
index 000000000000..2b80eb301d03
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/pdt_locales/ru_RU.py
@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from .base import *  # noqa
+
+# don't use an unicode string
+localeID = 'ru_RU'
+dateSep = ['-', '.']
+timeSep = [':']
+meridian = []
+usesMeridian = False
+uses24 = True
+
+Weekdays = [
+    'понедельник', 'вторник', 'среда', 'четверг',
+    'пятница', 'суббота', 'воскресенье',
+]
+shortWeekdays = [
+    'пн', 'вт', 'ср', 'чт', 'пт', 'сб', 'вс',
+]
+# library does not know how to conjugate words
+# библиотека не умеет спрягать слова
+Months = [
+    'января', 'февраля', 'марта', 'апреля', 'мая', 'июня', 'июля',
+    'августа', 'сентября', 'октября', 'ноября', 'декабря',
+]
+shortMonths = [
+    'янв', 'фев', 'мрт', 'апр', 'май', 'июн',
+    'июл', 'авг', 'сен', 'окт', 'нбр', 'дек',
+]
+dateFormats = {
+    'full': 'EEEE, dd MMMM yyyy',
+    'long': 'dd MMMM yyyy',
+    'medium': 'dd-MM-yyyy',
+    'short': 'dd-MM-yy',
+}
+
+timeFormats = {
+    'full': 'HH:mm:ss v',
+    'long': 'HH:mm:ss z',
+    'medium': 'HH:mm:ss',
+    'short': 'HH:mm',
+}
+
+dp_order = ['d', 'm', 'y']
+
+decimal_mark = '.'
+
+units = {
+    'seconds': ['секунда', 'секунды', 'секунд', 'сек', 'с'],
+    'minutes': ['минута', 'минуты', 'минут', 'мин', 'м'],
+    'hours': ['час', 'часов', 'часа', 'ч'],
+    'days': ['день', 'дней', 'д'],
+    'weeks': ['неделя', 'недели', 'н'],
+    'months': ['месяц', 'месяца', 'мес'],
+    'years': ['год', 'года', 'годы', 'г'],
+}
+
+re_values = re_values.copy()
+re_values.update({
+    'specials': 'om',
+    'timeseparator': ':',
+    'rangeseparator': '-',
+    'daysuffix': 'ого|ой|ий|тье',
+    'qunits': 'д|мес|г|ч|н|м|с',
+    'now': ['сейчас'],
+})
+
+Modifiers = {
+    'после': 1,
+    'назад': -1,
+    'предыдущий': -1,
+    'последний': -1,
+    'далее': 1,
+    'ранее': -1,
+}
+
+dayOffsets = {
+    'завтра': 1,
+    'сегодня': 0,
+    'вчера': -1,
+    'позавчера': -2,
+    'послезавтра': 2,
+}
+
+re_sources = {
+    'полдень': {'hr': 12, 'mn': 0, 'sec': 0},
+    'день': {'hr': 13, 'mn': 0, 'sec': 0},
+    'обед': {'hr': 12, 'mn': 0, 'sec': 0},
+    'утро': {'hr': 6, 'mn': 0, 'sec': 0},
+    'завтрак': {'hr': 8, 'mn': 0, 'sec': 0},
+    'ужин': {'hr': 19, 'mn': 0, 'sec': 0},
+    'вечер': {'hr': 18, 'mn': 0, 'sec': 0},
+    'полночь': {'hr': 0, 'mn': 0, 'sec': 0},
+    'ночь': {'hr': 21, 'mn': 0, 'sec': 0},
+}
+
+small = {
+    'ноль': 0,
+    'один': 1,
+    'два': 2,
+    'три': 3,
+    'четыре': 4,
+    'пять': 5,
+    'шесть': 6,
+    'семь': 7,
+    'восемь': 8,
+    'девять': 9,
+    'десять': 10,
+    'одиннадцать': 11,
+    'двенадцать': 12,
+    'тринадцать': 13,
+    'четырнадцать': 14,
+    'пятнадцать': 15,
+    'шестнадцать': 16,
+    'семнадцать': 17,
+    'восемнадцать': 18,
+    'девятнадцать': 19,
+    'двадцать': 20,
+    'тридцать': 30,
+    'сорок': 40,
+    'пятьдесят': 50,
+    'шестьдесят': 60,
+    'семьдесят': 70,
+    'восемьдесят': 80,
+    'девяносто': 90,
+}
+
+numbers = {
+    'ноль': 0,
+    'один': 1,
+    'два': 2,
+    'три': 3,
+    'четыре': 4,
+    'пять': 5,
+    'шесть': 6,
+    'семь': 7,
+    'восемь': 8,
+    'девять': 9,
+    'десять': 10,
+    'одиннадцать': 11,
+    'двенадцать': 12,
+    'тринадцать': 13,
+    'четырнадцать': 14,
+    'пятнадцать': 15,
+    'шестнадцать': 16,
+    'семнадцать': 17,
+    'восемнадцать': 18,
+    'девятнадцать': 19,
+    'двадцать': 20,
+}
+
+magnitude = {
+    'тысяча': 1000,
+    'миллион': 1000000,
+    'миллиард': 1000000000,
+    'триллион': 1000000000000,
+    'квадриллион': 1000000000000000,
+    'квинтиллион': 1000000000000000000,
+    'секстиллион': 1000000000000000000000,
+    'септиллион': 1000000000000000000000000,
+    'октиллион': 1000000000000000000000000000,
+    'нониллион': 1000000000000000000000000000000,
+    'дециллион': 1000000000000000000000000000000000,
+}
diff --git a/src/bindings/python/flux/utils/parsedatetime/warns.py b/src/bindings/python/flux/utils/parsedatetime/warns.py
new file mode 100644
index 000000000000..a754fc9db1da
--- /dev/null
+++ b/src/bindings/python/flux/utils/parsedatetime/warns.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+"""
+parsedatetime/warns.py
+
+All subclasses inherited from `Warning` class
+
+"""
+from __future__ import absolute_import
+
+import warnings
+
+
+class pdtDeprecationWarning(DeprecationWarning):
+    pass
+
+
+class pdtPendingDeprecationWarning(PendingDeprecationWarning):
+    pass
+
+
+class pdt20DeprecationWarning(pdtPendingDeprecationWarning):
+    pass
+
+
+warnings.simplefilter('default', pdtDeprecationWarning)
+warnings.simplefilter('ignore', pdtPendingDeprecationWarning)
diff --git a/src/bindings/python/flux/utils/tomli/LICENSE b/src/bindings/python/flux/utils/tomli/LICENSE
new file mode 100644
index 000000000000..e859590f886c
--- /dev/null
+++ b/src/bindings/python/flux/utils/tomli/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Taneli Hukkinen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/src/bindings/python/flux/utils/tomli/__init__.py b/src/bindings/python/flux/utils/tomli/__init__.py
new file mode 100644
index 000000000000..85974670c38d
--- /dev/null
+++ b/src/bindings/python/flux/utils/tomli/__init__.py
@@ -0,0 +1,9 @@
+"""A lil' TOML parser."""
+
+__all__ = ("loads", "load", "TOMLDecodeError")
+__version__ = "1.2.3"  # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT
+
+from ._parser import TOMLDecodeError, load, loads
+
+# Pretend this exception was created here.
+TOMLDecodeError.__module__ = "tomli"
diff --git a/src/bindings/python/flux/utils/tomli/_parser.py b/src/bindings/python/flux/utils/tomli/_parser.py
new file mode 100644
index 000000000000..093afe50d375
--- /dev/null
+++ b/src/bindings/python/flux/utils/tomli/_parser.py
@@ -0,0 +1,663 @@
+import string
+from types import MappingProxyType
+from typing import Any, BinaryIO, Dict, FrozenSet, Iterable, NamedTuple, Optional, Tuple
+import warnings
+
+from ._re import (
+    RE_DATETIME,
+    RE_LOCALTIME,
+    RE_NUMBER,
+    match_to_datetime,
+    match_to_localtime,
+    match_to_number,
+)
+from ._types import Key, ParseFloat, Pos
+
+ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
+
+# Neither of these sets include quotation mark or backslash. They are
+# currently handled as separate cases in the parser functions.
+ILLEGAL_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t")
+ILLEGAL_MULTILINE_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t\n")
+
+ILLEGAL_LITERAL_STR_CHARS = ILLEGAL_BASIC_STR_CHARS
+ILLEGAL_MULTILINE_LITERAL_STR_CHARS = ILLEGAL_MULTILINE_BASIC_STR_CHARS
+
+ILLEGAL_COMMENT_CHARS = ILLEGAL_BASIC_STR_CHARS
+
+TOML_WS = frozenset(" \t")
+TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n")
+BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_")
+KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'")
+HEXDIGIT_CHARS = frozenset(string.hexdigits)
+
+BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType(
+    {
+        "\\b": "\u0008",  # backspace
+        "\\t": "\u0009",  # tab
+        "\\n": "\u000A",  # linefeed
+        "\\f": "\u000C",  # form feed
+        "\\r": "\u000D",  # carriage return
+        '\\"': "\u0022",  # quote
+        "\\\\": "\u005C",  # backslash
+    }
+)
+
+
+class TOMLDecodeError(ValueError):
+    """An error raised if a document is not valid TOML."""
+
+
+def load(fp: BinaryIO, *, parse_float: ParseFloat = float) -> Dict[str, Any]:
+    """Parse TOML from a binary file object."""
+    s_bytes = fp.read()
+    try:
+        s = s_bytes.decode()
+    except AttributeError:
+        warnings.warn(
+            "Text file object support is deprecated in favor of binary file objects."
+            ' Use `open("foo.toml", "rb")` to open the file in binary mode.',
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        s = s_bytes  # type: ignore[assignment]
+    return loads(s, parse_float=parse_float)
+
+
+def loads(s: str, *, parse_float: ParseFloat = float) -> Dict[str, Any]:  # noqa: C901
+    """Parse TOML from a string."""
+
+    # The spec allows converting "\r\n" to "\n", even in string
+    # literals. Let's do so to simplify parsing.
+    src = s.replace("\r\n", "\n")
+    pos = 0
+    out = Output(NestedDict(), Flags())
+    header: Key = ()
+
+    # Parse one statement at a time
+    # (typically means one line in TOML source)
+    while True:
+        # 1. Skip line leading whitespace
+        pos = skip_chars(src, pos, TOML_WS)
+
+        # 2. Parse rules. Expect one of the following:
+        #    - end of file
+        #    - end of line
+        #    - comment
+        #    - key/value pair
+        #    - append dict to list (and move to its namespace)
+        #    - create dict (and move to its namespace)
+        # Skip trailing whitespace when applicable.
+        try:
+            char = src[pos]
+        except IndexError:
+            break
+        if char == "\n":
+            pos += 1
+            continue
+        if char in KEY_INITIAL_CHARS:
+            pos = key_value_rule(src, pos, out, header, parse_float)
+            pos = skip_chars(src, pos, TOML_WS)
+        elif char == "[":
+            try:
+                second_char: Optional[str] = src[pos + 1]
+            except IndexError:
+                second_char = None
+            if second_char == "[":
+                pos, header = create_list_rule(src, pos, out)
+            else:
+                pos, header = create_dict_rule(src, pos, out)
+            pos = skip_chars(src, pos, TOML_WS)
+        elif char != "#":
+            raise suffixed_err(src, pos, "Invalid statement")
+
+        # 3. Skip comment
+        pos = skip_comment(src, pos)
+
+        # 4. Expect end of line or end of file
+        try:
+            char = src[pos]
+        except IndexError:
+            break
+        if char != "\n":
+            raise suffixed_err(
+                src, pos, "Expected newline or end of document after a statement"
+            )
+        pos += 1
+
+    return out.data.dict
+
+
+class Flags:
+    """Flags that map to parsed keys/namespaces."""
+
+    # Marks an immutable namespace (inline array or inline table).
+    FROZEN = 0
+    # Marks a nest that has been explicitly created and can no longer
+    # be opened using the "[table]" syntax.
+    EXPLICIT_NEST = 1
+
+    def __init__(self) -> None:
+        self._flags: Dict[str, dict] = {}
+
+    def unset_all(self, key: Key) -> None:
+        cont = self._flags
+        for k in key[:-1]:
+            if k not in cont:
+                return
+            cont = cont[k]["nested"]
+        cont.pop(key[-1], None)
+
+    def set_for_relative_key(self, head_key: Key, rel_key: Key, flag: int) -> None:
+        cont = self._flags
+        for k in head_key:
+            if k not in cont:
+                cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}}
+            cont = cont[k]["nested"]
+        for k in rel_key:
+            if k in cont:
+                cont[k]["flags"].add(flag)
+            else:
+                cont[k] = {"flags": {flag}, "recursive_flags": set(), "nested": {}}
+            cont = cont[k]["nested"]
+
+    def set(self, key: Key, flag: int, *, recursive: bool) -> None:  # noqa: A003
+        cont = self._flags
+        key_parent, key_stem = key[:-1], key[-1]
+        for k in key_parent:
+            if k not in cont:
+                cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}}
+            cont = cont[k]["nested"]
+        if key_stem not in cont:
+            cont[key_stem] = {"flags": set(), "recursive_flags": set(), "nested": {}}
+        cont[key_stem]["recursive_flags" if recursive else "flags"].add(flag)
+
+    def is_(self, key: Key, flag: int) -> bool:
+        if not key:
+            return False  # document root has no flags
+        cont = self._flags
+        for k in key[:-1]:
+            if k not in cont:
+                return False
+            inner_cont = cont[k]
+            if flag in inner_cont["recursive_flags"]:
+                return True
+            cont = inner_cont["nested"]
+        key_stem = key[-1]
+        if key_stem in cont:
+            cont = cont[key_stem]
+            return flag in cont["flags"] or flag in cont["recursive_flags"]
+        return False
+
+
+class NestedDict:
+    def __init__(self) -> None:
+        # The parsed content of the TOML document
+        self.dict: Dict[str, Any] = {}
+
+    def get_or_create_nest(
+        self,
+        key: Key,
+        *,
+        access_lists: bool = True,
+    ) -> dict:
+        cont: Any = self.dict
+        for k in key:
+            if k not in cont:
+                cont[k] = {}
+            cont = cont[k]
+            if access_lists and isinstance(cont, list):
+                cont = cont[-1]
+            if not isinstance(cont, dict):
+                raise KeyError("There is no nest behind this key")
+        return cont
+
+    def append_nest_to_list(self, key: Key) -> None:
+        cont = self.get_or_create_nest(key[:-1])
+        last_key = key[-1]
+        if last_key in cont:
+            list_ = cont[last_key]
+            try:
+                list_.append({})
+            except AttributeError:
+                raise KeyError("An object other than list found behind this key")
+        else:
+            cont[last_key] = [{}]
+
+
+class Output(NamedTuple):
+    data: NestedDict
+    flags: Flags
+
+
+def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos:
+    try:
+        while src[pos] in chars:
+            pos += 1
+    except IndexError:
+        pass
+    return pos
+
+
+def skip_until(
+    src: str,
+    pos: Pos,
+    expect: str,
+    *,
+    error_on: FrozenSet[str],
+    error_on_eof: bool,
+) -> Pos:
+    try:
+        new_pos = src.index(expect, pos)
+    except ValueError:
+        new_pos = len(src)
+        if error_on_eof:
+            raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None
+
+    if not error_on.isdisjoint(src[pos:new_pos]):
+        while src[pos] not in error_on:
+            pos += 1
+        raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}")
+    return new_pos
+
+
+def skip_comment(src: str, pos: Pos) -> Pos:
+    try:
+        char: Optional[str] = src[pos]
+    except IndexError:
+        char = None
+    if char == "#":
+        return skip_until(
+            src, pos + 1, "\n", error_on=ILLEGAL_COMMENT_CHARS, error_on_eof=False
+        )
+    return pos
+
+
+def skip_comments_and_array_ws(src: str, pos: Pos) -> Pos:
+    while True:
+        pos_before_skip = pos
+        pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE)
+        pos = skip_comment(src, pos)
+        if pos == pos_before_skip:
+            return pos
+
+
+def create_dict_rule(src: str, pos: Pos, out: Output) -> Tuple[Pos, Key]:
+    pos += 1  # Skip "["
+    pos = skip_chars(src, pos, TOML_WS)
+    pos, key = parse_key(src, pos)
+
+    if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN):
+        raise suffixed_err(src, pos, f"Can not declare {key} twice")
+    out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False)
+    try:
+        out.data.get_or_create_nest(key)
+    except KeyError:
+        raise suffixed_err(src, pos, "Can not overwrite a value") from None
+
+    if not src.startswith("]", pos):
+        raise suffixed_err(src, pos, 'Expected "]" at the end of a table declaration')
+    return pos + 1, key
+
+
+def create_list_rule(src: str, pos: Pos, out: Output) -> Tuple[Pos, Key]:
+    pos += 2  # Skip "[["
+    pos = skip_chars(src, pos, TOML_WS)
+    pos, key = parse_key(src, pos)
+
+    if out.flags.is_(key, Flags.FROZEN):
+        raise suffixed_err(src, pos, f"Can not mutate immutable namespace {key}")
+    # Free the namespace now that it points to another empty list item...
+    out.flags.unset_all(key)
+    # ...but this key precisely is still prohibited from table declaration
+    out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False)
+    try:
+        out.data.append_nest_to_list(key)
+    except KeyError:
+        raise suffixed_err(src, pos, "Can not overwrite a value") from None
+
+    if not src.startswith("]]", pos):
+        raise suffixed_err(src, pos, 'Expected "]]" at the end of an array declaration')
+    return pos + 2, key
+
+
+def key_value_rule(
+    src: str, pos: Pos, out: Output, header: Key, parse_float: ParseFloat
+) -> Pos:
+    pos, key, value = parse_key_value_pair(src, pos, parse_float)
+    key_parent, key_stem = key[:-1], key[-1]
+    abs_key_parent = header + key_parent
+
+    if out.flags.is_(abs_key_parent, Flags.FROZEN):
+        raise suffixed_err(
+            src, pos, f"Can not mutate immutable namespace {abs_key_parent}"
+        )
+    # Containers in the relative path can't be opened with the table syntax after this
+    out.flags.set_for_relative_key(header, key, Flags.EXPLICIT_NEST)
+    try:
+        nest = out.data.get_or_create_nest(abs_key_parent)
+    except KeyError:
+        raise suffixed_err(src, pos, "Can not overwrite a value") from None
+    if key_stem in nest:
+        raise suffixed_err(src, pos, "Can not overwrite a value")
+    # Mark inline table and array namespaces recursively immutable
+    if isinstance(value, (dict, list)):
+        out.flags.set(header + key, Flags.FROZEN, recursive=True)
+    nest[key_stem] = value
+    return pos
+
+
+def parse_key_value_pair(
+    src: str, pos: Pos, parse_float: ParseFloat
+) -> Tuple[Pos, Key, Any]:
+    pos, key = parse_key(src, pos)
+    try:
+        char: Optional[str] = src[pos]
+    except IndexError:
+        char = None
+    if char != "=":
+        raise suffixed_err(src, pos, 'Expected "=" after a key in a key/value pair')
+    pos += 1
+    pos = skip_chars(src, pos, TOML_WS)
+    pos, value = parse_value(src, pos, parse_float)
+    return pos, key, value
+
+
+def parse_key(src: str, pos: Pos) -> Tuple[Pos, Key]:
+    pos, key_part = parse_key_part(src, pos)
+    key: Key = (key_part,)
+    pos = skip_chars(src, pos, TOML_WS)
+    while True:
+        try:
+            char: Optional[str] = src[pos]
+        except IndexError:
+            char = None
+        if char != ".":
+            return pos, key
+        pos += 1
+        pos = skip_chars(src, pos, TOML_WS)
+        pos, key_part = parse_key_part(src, pos)
+        key += (key_part,)
+        pos = skip_chars(src, pos, TOML_WS)
+
+
+def parse_key_part(src: str, pos: Pos) -> Tuple[Pos, str]:
+    try:
+        char: Optional[str] = src[pos]
+    except IndexError:
+        char = None
+    if char in BARE_KEY_CHARS:
+        start_pos = pos
+        pos = skip_chars(src, pos, BARE_KEY_CHARS)
+        return pos, src[start_pos:pos]
+    if char == "'":
+        return parse_literal_str(src, pos)
+    if char == '"':
+        return parse_one_line_basic_str(src, pos)
+    raise suffixed_err(src, pos, "Invalid initial character for a key part")
+
+
+def parse_one_line_basic_str(src: str, pos: Pos) -> Tuple[Pos, str]:
+    pos += 1
+    return parse_basic_str(src, pos, multiline=False)
+
+
+def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> Tuple[Pos, list]:
+    pos += 1
+    array: list = []
+
+    pos = skip_comments_and_array_ws(src, pos)
+    if src.startswith("]", pos):
+        return pos + 1, array
+    while True:
+        pos, val = parse_value(src, pos, parse_float)
+        array.append(val)
+        pos = skip_comments_and_array_ws(src, pos)
+
+        c = src[pos : pos + 1]
+        if c == "]":
+            return pos + 1, array
+        if c != ",":
+            raise suffixed_err(src, pos, "Unclosed array")
+        pos += 1
+
+        pos = skip_comments_and_array_ws(src, pos)
+        if src.startswith("]", pos):
+            return pos + 1, array
+
+
+def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> Tuple[Pos, dict]:
+    pos += 1
+    nested_dict = NestedDict()
+    flags = Flags()
+
+    pos = skip_chars(src, pos, TOML_WS)
+    if src.startswith("}", pos):
+        return pos + 1, nested_dict.dict
+    while True:
+        pos, key, value = parse_key_value_pair(src, pos, parse_float)
+        key_parent, key_stem = key[:-1], key[-1]
+        if flags.is_(key, Flags.FROZEN):
+            raise suffixed_err(src, pos, f"Can not mutate immutable namespace {key}")
+        try:
+            nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
+        except KeyError:
+            raise suffixed_err(src, pos, "Can not overwrite a value") from None
+        if key_stem in nest:
+            raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}")
+        nest[key_stem] = value
+        pos = skip_chars(src, pos, TOML_WS)
+        c = src[pos : pos + 1]
+        if c == "}":
+            return pos + 1, nested_dict.dict
+        if c != ",":
+            raise suffixed_err(src, pos, "Unclosed inline table")
+        if isinstance(value, (dict, list)):
+            flags.set(key, Flags.FROZEN, recursive=True)
+        pos += 1
+        pos = skip_chars(src, pos, TOML_WS)
+
+
+def parse_basic_str_escape(  # noqa: C901
+    src: str, pos: Pos, *, multiline: bool = False
+) -> Tuple[Pos, str]:
+    escape_id = src[pos : pos + 2]
+    pos += 2
+    if multiline and escape_id in {"\\ ", "\\\t", "\\\n"}:
+        # Skip whitespace until next non-whitespace character or end of
+        # the doc. Error if non-whitespace is found before newline.
+        if escape_id != "\\\n":
+            pos = skip_chars(src, pos, TOML_WS)
+            try:
+                char = src[pos]
+            except IndexError:
+                return pos, ""
+            if char != "\n":
+                raise suffixed_err(src, pos, 'Unescaped "\\" in a string')
+            pos += 1
+        pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE)
+        return pos, ""
+    if escape_id == "\\u":
+        return parse_hex_char(src, pos, 4)
+    if escape_id == "\\U":
+        return parse_hex_char(src, pos, 8)
+    try:
+        return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
+    except KeyError:
+        if len(escape_id) != 2:
+            raise suffixed_err(src, pos, "Unterminated string") from None
+        raise suffixed_err(src, pos, 'Unescaped "\\" in a string') from None
+
+
+def parse_basic_str_escape_multiline(src: str, pos: Pos) -> Tuple[Pos, str]:
+    return parse_basic_str_escape(src, pos, multiline=True)
+
+
+def parse_hex_char(src: str, pos: Pos, hex_len: int) -> Tuple[Pos, str]:
+    hex_str = src[pos : pos + hex_len]
+    if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str):
+        raise suffixed_err(src, pos, "Invalid hex value")
+    pos += hex_len
+    hex_int = int(hex_str, 16)
+    if not is_unicode_scalar_value(hex_int):
+        raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value")
+    return pos, chr(hex_int)
+
+
+def parse_literal_str(src: str, pos: Pos) -> Tuple[Pos, str]:
+    pos += 1  # Skip starting apostrophe
+    start_pos = pos
+    pos = skip_until(
+        src, pos, "'", error_on=ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True
+    )
+    return pos + 1, src[start_pos:pos]  # Skip ending apostrophe
+
+
+def parse_multiline_str(src: str, pos: Pos, *, literal: bool) -> Tuple[Pos, str]:
+    pos += 3
+    if src.startswith("\n", pos):
+        pos += 1
+
+    if literal:
+        delim = "'"
+        end_pos = skip_until(
+            src,
+            pos,
+            "'''",
+            error_on=ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
+            error_on_eof=True,
+        )
+        result = src[pos:end_pos]
+        pos = end_pos + 3
+    else:
+        delim = '"'
+        pos, result = parse_basic_str(src, pos, multiline=True)
+
+    # Add at maximum two extra apostrophes/quotes if the end sequence
+    # is 4 or 5 chars long instead of just 3.
+    if not src.startswith(delim, pos):
+        return pos, result
+    pos += 1
+    if not src.startswith(delim, pos):
+        return pos, result + delim
+    pos += 1
+    return pos, result + (delim * 2)
+
+
+def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> Tuple[Pos, str]:
+    if multiline:
+        error_on = ILLEGAL_MULTILINE_BASIC_STR_CHARS
+        parse_escapes = parse_basic_str_escape_multiline
+    else:
+        error_on = ILLEGAL_BASIC_STR_CHARS
+        parse_escapes = parse_basic_str_escape
+    result = ""
+    start_pos = pos
+    while True:
+        try:
+            char = src[pos]
+        except IndexError:
+            raise suffixed_err(src, pos, "Unterminated string") from None
+        if char == '"':
+            if not multiline:
+                return pos + 1, result + src[start_pos:pos]
+            if src.startswith('"""', pos):
+                return pos + 3, result + src[start_pos:pos]
+            pos += 1
+            continue
+        if char == "\\":
+            result += src[start_pos:pos]
+            pos, parsed_escape = parse_escapes(src, pos)
+            result += parsed_escape
+            start_pos = pos
+            continue
+        if char in error_on:
+            raise suffixed_err(src, pos, f"Illegal character {char!r}")
+        pos += 1
+
+
+def parse_value(  # noqa: C901
+    src: str, pos: Pos, parse_float: ParseFloat
+) -> Tuple[Pos, Any]:
+    try:
+        char: Optional[str] = src[pos]
+    except IndexError:
+        char = None
+
+    # Basic strings
+    if char == '"':
+        if src.startswith('"""', pos):
+            return parse_multiline_str(src, pos, literal=False)
+        return parse_one_line_basic_str(src, pos)
+
+    # Literal strings
+    if char == "'":
+        if src.startswith("'''", pos):
+            return parse_multiline_str(src, pos, literal=True)
+        return parse_literal_str(src, pos)
+
+    # Booleans
+    if char == "t":
+        if src.startswith("true", pos):
+            return pos + 4, True
+    if char == "f":
+        if src.startswith("false", pos):
+            return pos + 5, False
+
+    # Dates and times
+    datetime_match = RE_DATETIME.match(src, pos)
+    if datetime_match:
+        try:
+            datetime_obj = match_to_datetime(datetime_match)
+        except ValueError as e:
+            raise suffixed_err(src, pos, "Invalid date or datetime") from e
+        return datetime_match.end(), datetime_obj
+    localtime_match = RE_LOCALTIME.match(src, pos)
+    if localtime_match:
+        return localtime_match.end(), match_to_localtime(localtime_match)
+
+    # Integers and "normal" floats.
+    # The regex will greedily match any type starting with a decimal
+    # char, so needs to be located after handling of dates and times.
+    number_match = RE_NUMBER.match(src, pos)
+    if number_match:
+        return number_match.end(), match_to_number(number_match, parse_float)
+
+    # Arrays
+    if char == "[":
+        return parse_array(src, pos, parse_float)
+
+    # Inline tables
+    if char == "{":
+        return parse_inline_table(src, pos, parse_float)
+
+    # Special floats
+    first_three = src[pos : pos + 3]
+    if first_three in {"inf", "nan"}:
+        return pos + 3, parse_float(first_three)
+    first_four = src[pos : pos + 4]
+    if first_four in {"-inf", "+inf", "-nan", "+nan"}:
+        return pos + 4, parse_float(first_four)
+
+    raise suffixed_err(src, pos, "Invalid value")
+
+
+def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError:
+    """Return a `TOMLDecodeError` where error message is suffixed with
+    coordinates in source."""
+
+    def coord_repr(src: str, pos: Pos) -> str:
+        if pos >= len(src):
+            return "end of document"
+        line = src.count("\n", 0, pos) + 1
+        if line == 1:
+            column = pos + 1
+        else:
+            column = pos - src.rindex("\n", 0, pos)
+        return f"line {line}, column {column}"
+
+    return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})")
+
+
+def is_unicode_scalar_value(codepoint: int) -> bool:
+    return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
diff --git a/src/bindings/python/flux/utils/tomli/_re.py b/src/bindings/python/flux/utils/tomli/_re.py
new file mode 100644
index 000000000000..45e17e2cb067
--- /dev/null
+++ b/src/bindings/python/flux/utils/tomli/_re.py
@@ -0,0 +1,101 @@
+from datetime import date, datetime, time, timedelta, timezone, tzinfo
+from functools import lru_cache
+import re
+from typing import Any, Optional, Union
+
+from ._types import ParseFloat
+
+# E.g.
+# - 00:32:00.999999
+# - 00:32:00
+_TIME_RE_STR = r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?"
+
+RE_NUMBER = re.compile(
+    r"""
+0
+(?:
+    x[0-9A-Fa-f](?:_?[0-9A-Fa-f])*   # hex
+    |
+    b[01](?:_?[01])*                 # bin
+    |
+    o[0-7](?:_?[0-7])*               # oct
+)
+|
+[+-]?(?:0|[1-9](?:_?[0-9])*)         # dec, integer part
+(?P<floatpart>
+    (?:\.[0-9](?:_?[0-9])*)?         # optional fractional part
+    (?:[eE][+-]?[0-9](?:_?[0-9])*)?  # optional exponent part
+)
+""",
+    flags=re.VERBOSE,
+)
+RE_LOCALTIME = re.compile(_TIME_RE_STR)
+RE_DATETIME = re.compile(
+    fr"""
+([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])  # date, e.g. 1988-10-27
+(?:
+    [Tt ]
+    {_TIME_RE_STR}
+    (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))?  # optional time offset
+)?
+""",
+    flags=re.VERBOSE,
+)
+
+
+def match_to_datetime(match: "re.Match") -> Union[datetime, date]:
+    """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
+
+    Raises ValueError if the match does not correspond to a valid date
+    or datetime.
+    """
+    (
+        year_str,
+        month_str,
+        day_str,
+        hour_str,
+        minute_str,
+        sec_str,
+        micros_str,
+        zulu_time,
+        offset_sign_str,
+        offset_hour_str,
+        offset_minute_str,
+    ) = match.groups()
+    year, month, day = int(year_str), int(month_str), int(day_str)
+    if hour_str is None:
+        return date(year, month, day)
+    hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
+    micros = int(micros_str.ljust(6, "0")) if micros_str else 0
+    if offset_sign_str:
+        tz: Optional[tzinfo] = cached_tz(
+            offset_hour_str, offset_minute_str, offset_sign_str
+        )
+    elif zulu_time:
+        tz = timezone.utc
+    else:  # local date-time
+        tz = None
+    return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
+
+
+@lru_cache(maxsize=None)
+def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone:
+    sign = 1 if sign_str == "+" else -1
+    return timezone(
+        timedelta(
+            hours=sign * int(hour_str),
+            minutes=sign * int(minute_str),
+        )
+    )
+
+
+def match_to_localtime(match: "re.Match") -> time:
+    hour_str, minute_str, sec_str, micros_str = match.groups()
+    micros = int(micros_str.ljust(6, "0")) if micros_str else 0
+    return time(int(hour_str), int(minute_str), int(sec_str), micros)
+
+
+def match_to_number(match: "re.Match", parse_float: "ParseFloat") -> Any:
+    if match.group("floatpart"):
+        return parse_float(match.group())
+    return int(match.group(), 0)
diff --git a/src/bindings/python/flux/utils/tomli/_types.py b/src/bindings/python/flux/utils/tomli/_types.py
new file mode 100644
index 000000000000..e37cc8088fbd
--- /dev/null
+++ b/src/bindings/python/flux/utils/tomli/_types.py
@@ -0,0 +1,6 @@
+from typing import Any, Callable, Tuple
+
+# Type annotations
+ParseFloat = Callable[[str], Any]
+Key = Tuple[str, ...]
+Pos = int
diff --git a/src/bindings/python/flux/utils/tomli/py.typed b/src/bindings/python/flux/utils/tomli/py.typed
new file mode 100644
index 000000000000..7632ecf77545
--- /dev/null
+++ b/src/bindings/python/flux/utils/tomli/py.typed
@@ -0,0 +1 @@
+# Marker file for PEP 561
diff --git a/src/bindings/python/flux/wrapper.py b/src/bindings/python/flux/wrapper.py
index 64c26612ae77..6b7aefe0ddba 100644
--- a/src/bindings/python/flux/wrapper.py
+++ b/src/bindings/python/flux/wrapper.py
@@ -13,13 +13,12 @@
 This could, in principle, be used for other projects as well, but it encodes a
 number of assumptions about the error propagation and handling that flux uses.
 """
-import re
-import os
 import errno
 import inspect
-import weakref
-
-import six
+import os
+import re
+from types import MethodType
+from typing import Any, Dict
 
 
 class MissingFunctionError(Exception):
@@ -29,13 +28,13 @@ def __init__(self, name, c_name, name_list, arguments):
         caller = inspect.getframeinfo(call_stack[2][0])
 
         message = """
-A non-existant or unavailable function invocation has been detected.
+A non-existent or unavailable function invocation has been detected.
 Has this function been recently removed or renamed?
 
 Name called: {name}
 Possible C names: {name_list}
 Arguments: {arguments}
-Likely intended C invokation: {c_name}{arguments}
+Likely intended C invocation: {c_name}{arguments}
 Invocation detail: inside function {outer}
 {file}:{line}: {context}
         """.format(
@@ -188,18 +187,15 @@ def __call__(self, calling_object, *args_in):
             elif isinstance(args[i], WrapperBase):
                 # Unpack wrapper objects
                 args[i] = args[i].handle
-            elif isinstance(args[i], six.text_type):
-                args[i] = args[i].encode("utf-8")
+            elif isinstance(args[i], str):
+                args[i] = args[i].encode("utf-8", errors="surrogateescape")
 
         try:
             result = self.fun(*args)
         except TypeError as err:
-            six.raise_from(
-                InvalidArguments(
-                    self.name, self.ffi.getctype(self.function_type), args_in
-                ),
-                err,
-            )
+            raise InvalidArguments(
+                self.name, self.ffi.getctype(self.function_type), args_in
+            ) from err
 
         if result == calling_object.ffi.NULL:
             result = None
@@ -213,15 +209,26 @@ def __call__(self, calling_object, *args_in):
         # Convert errno errors into python exceptions
 
         if self.is_error(result):
-            err = calling_object.ffi.errno
-            if err != 0:
+            errnum = calling_object.ffi.errno
+            if errnum != 0:
                 raise EnvironmentError(
-                    err, "{}: {}".format(errno.errorcode[err], os.strerror(err))
+                    errnum,
+                    "{}: {}".format(
+                        (
+                            errno.errorcode[errnum]
+                            if errnum in errno.errorcode
+                            else "Errno" + str(errnum)
+                        ),
+                        os.strerror(errnum),
+                    ),
                 )
 
         return result
 
 
+SIGS_: Dict[Any, Any] = {}
+
+
 class Wrapper(WrapperBase):
     """
     Forms a wrapper around an interface that dynamically searches for undefined
@@ -249,6 +256,20 @@ def __init__(
         self.filter_match = filter_match
         self.prefixes = prefixes
         self.destructor = destructor
+        # this is an error-checking dance to ensure that the class-based caching of
+        # callables is safe by only allowing one set of prefixes, filter-matches, etc.
+        # per derived class of wrapper
+        signature = (match, filter_match, prefixes)
+        mytype = type(self)
+        if SIGS_.get(mytype, None) is None:
+            SIGS_[mytype] = signature
+        else:
+            assert (
+                signature == SIGS_[mytype]
+            ), f"""
+signatures do not match, create a new subclass to change matching parameters:
+{mytype}: mysig: {SIGS_[mytype]} sig:{signature}
+            """
 
     def check_handle(self, name, fun_type):
         if self.match is not None and self._handle is not None:
@@ -321,12 +342,16 @@ def __getattr__(self, name):
             return fun
 
         new_fun = self.check_wrap(fun, name)
-        new_method = six.create_bound_method(new_fun, weakref.proxy(self))
+        new_meth = MethodType(new_fun, self)
+
+        # wrap the class in a function so it's correctly treated as a method
+        def wrap_class(self_renamed, *args, **kwargs):
+            return new_fun(self_renamed, *args, **kwargs)
 
-        # Store the wrapper function into the instance
+        # Store the wrapper function into the class
         # to prevent a second lookup
-        setattr(self, name, new_method)
-        return new_method
+        setattr(type(self), name, wrap_class)
+        return new_meth
 
     def _clear(self):
         # avoid recursion
@@ -346,7 +371,7 @@ def handle(self):
 
     @handle.setter
     def handle(self, h):
-        """ Override handle setter to clean up old handle if requested """
+        """Override handle setter to clean up old handle if requested"""
         if h is not None and self.match is not None:
             if self.ffi.typeof(h) != self.match:
                 raise TypeError(
diff --git a/src/bindings/python/make_binding.py b/src/bindings/python/make_binding.py
deleted file mode 100644
index 22c9a3923abf..000000000000
--- a/src/bindings/python/make_binding.py
+++ /dev/null
@@ -1,233 +0,0 @@
-###############################################################
-# Copyright 2014 Lawrence Livermore National Security, LLC
-# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
-#
-# This file is part of the Flux resource manager framework.
-# For details, see https://github.com/flux-framework.
-#
-# SPDX-License-Identifier: LGPL-3.0
-###############################################################
-
-from __future__ import print_function
-import os
-import re
-
-import argparse
-
-parser = argparse.ArgumentParser()
-
-parser.add_argument("header", help="C header file to parse", type=str)
-parser.add_argument("--include_header", help="Include base path", type=str, default="")
-parser.add_argument(
-    "--additional_headers", help="Additional headers to parse", nargs="*", default=[]
-)
-parser.add_argument(
-    "--include_ffi", help="FFI module for inclusion", action="append", default=[]
-)
-parser.add_argument("--package", help="Package prefix for module import", default=None)
-parser.add_argument("--path", help="Include base path", default=".")
-parser.add_argument(
-    "--search", help="append to header search path", action="append", default=[]
-)
-parser.add_argument(
-    "--modname", help="Name for the module to be generated", default="_flux"
-)
-parser.add_argument(
-    "--library", help="Library to include in the build", default="flux-core"
-)
-parser.add_argument(
-    "--add_long_sub",
-    help="Regex filter to apply whole-file of the form <match>|||<replacement>",
-    action="append",
-    default=[],
-)
-parser.add_argument(
-    "--add_sub",
-    "-a",
-    help="Regex filter to apply in processing of the form <match>|||<replacement>",
-    action="append",
-    default=[],
-)
-parser.add_argument(
-    "--extra_source",
-    help="Source to add directly to the output, mainly for includes",
-    type=str,
-    default="",
-)
-parser.add_argument(
-    "--ignore_header",
-    help="Pattern to ignore when searching header files",
-    default=[],
-    action="append",
-)
-
-args = parser.parse_args()
-
-absolute_head = os.path.abspath(os.path.join(args.path, args.header))
-
-# Prepend 'path' option to search list:
-args.search.insert(0, args.path)
-args.search = [os.path.abspath(f) for f in args.search]
-
-
-def find_first(path, name, extra=None):
-    for dirname in path:
-        filename = os.path.join(dirname, name)
-        if os.path.isfile(filename):
-            return filename
-    if extra is not None:
-        filename = os.path.join(extra, name)
-        if os.path.isfile(filename):
-            return filename
-    raise IOError(name)
-
-
-def process_header(f, including_path="."):
-    global mega_header
-    if not os.path.isfile(f):
-        f = os.path.join(including_path, f)
-    f = os.path.abspath(f)
-    if f not in checked_heads:
-        for p in args.ignore_header:
-            if re.search(p, f):
-                checked_heads[f] = 1
-                return
-        with open(f, "r") as header:
-            s = header.read()
-            # turn lin-continuations into single lines, any line ending in backslash
-            # newline has the backslash and newline removed
-            s = re.sub(r"\\\n", "", s)
-            # remove C-style comments, especially inside function declarations
-            s = re.sub(r"\/\*([\s\S]*?)\*\/", "", s)
-            # attempt to make argument lists single-line
-            s = re.sub(r",\s*\n", ", ", s, flags=re.MULTILINE)
-            # remove gcc-style attributes
-            s = re.sub(r"__attribute__\s*(([^;]*))", "", s)
-
-            for sub in args.add_long_sub:
-                [m, r] = sub.split("|||")
-                s = re.sub(m, r, s)
-
-            lines = s.split("\n")
-
-            for sub in args.add_sub:
-                new_lines = []
-                [m, r] = sub.split("|||")
-                for l in lines:
-                    l = re.sub(m, r, l)
-                    new_lines.append(l)
-                lines = new_lines
-
-            in_ifdef = False
-            in_ifndef = False
-            for l in lines:
-                # The compiler can't handle the 'extern "C" {' directive,
-                # so we need to manually honor the '#ifdef __cplusplus'
-                # preprocessor block to make the directive disappear.
-                # Assumes no nesting of preprocessor conditionals below
-                # __cplusplus conditionals.  Can handle either #ifdef or
-                # #ifndef, and the associated #else.
-                if in_ifndef:
-                    if re.match("#\s*else", l):
-                        in_ifndef = False
-                        in_ifdef = True
-                        continue
-                    else:
-                        pass  # allow the line to be included
-                elif in_ifdef:
-                    if re.match("#\s*(endif|else)", l):
-                        in_ifdef = False
-                    continue
-                elif re.match("#\s*ifdef\s+__cplusplus", l):
-                    in_ifdef = True
-                    continue
-                elif re.match("#\s*ifndef\s+__cplusplus", l):
-                    in_ifndef = True
-                    continue
-
-                m = re.search('#include\s*"([^"]*)"', l)
-                if m:
-                    nf = find_first(args.search, m.group(1), including_path)
-                    process_header(nf, os.path.dirname(os.path.abspath(nf)))
-                if not re.match("#\s*(ifdef|ifndef|endif|include|define)", l):
-                    mega_header += l + "\n"
-        checked_heads[f] = 1
-
-
-with open("{}_build.py".format(args.modname), "w") as modfile:
-    os.chdir(args.path)
-
-    mega_header = ""
-    checked_heads = {}
-
-    process_header(absolute_head)
-
-    for header in args.additional_headers:
-        process_header(header)
-
-    include_head = args.header
-    if args.include_header != "":
-        include_head = args.include_header
-
-    mega_header = (
-        """
-  """
-        + mega_header
-    )
-
-    ffi_include_base = """
-from {module} import ffi as {module}_ffi
-ffi.include({module}_ffi)
-  """
-    ffi_include = ""
-    for inc in args.include_ffi:
-        ffi_include += ffi_include_base.format(module=inc)
-
-    print(
-        '''
-#pylint: skip-file
-# This is a generated file... linting is less than useful
-from cffi import FFI
-ffi = FFI()
-
-
-ffi.set_source('{full_mod}',
-            """
-#include <{header}>
-{extra_source}
-
-void * unpack_long(ptrdiff_t num){{
-  return (void*)num;
-}}
-// TODO: remove this when we can use cffi 1.10
-#ifdef __GNUC__
-#pragma GCC visibility push(default)
-#endif
-            """,
-            libraries=['{library}'])
-
-{includes}
-
-ffi.cdef("""
-void * unpack_long(ptrdiff_t num);
-void free(void *ptr);
-typedef int... pid_t;
-
-        {cdefs}
-
-    """)
-if __name__ == "__main__":
-  ffi.emit_c_code('{modname}.c')
-    '''.format(
-            cdefs=mega_header,
-            full_mod=args.modname
-            if args.package is None
-            else ".".join([args.package, args.modname]),
-            modname=args.modname,
-            library=args.library,
-            header=include_head,
-            extra_source=args.extra_source,
-            includes=ffi_include,
-        ),
-        file=modfile,
-    )
diff --git a/src/bindings/python/setup.py b/src/bindings/python/setup.py
index 0c79ef79222d..abfe50d2eb71 100644
--- a/src/bindings/python/setup.py
+++ b/src/bindings/python/setup.py
@@ -8,9 +8,10 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-from setuptools import setup
 import os
 
+from setuptools import setup
+
 here = os.path.abspath(os.path.dirname(__file__))
 cffi_dep = "cffi>=1.1"
 setup(
diff --git a/src/broker/.gitignore b/src/broker/.gitignore
index f98ca7bed9ea..63fd25af77f8 100644
--- a/src/broker/.gitignore
+++ b/src/broker/.gitignore
@@ -1,2 +1 @@
-/cmbd
 /flux-broker
diff --git a/src/broker/Makefile.am b/src/broker/Makefile.am
index 62785323de79..d386ab6ae753 100644
--- a/src/broker/Makefile.am
+++ b/src/broker/Makefile.am
@@ -5,11 +5,16 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
+	-DABS_TOP_BUILDDIR=\"${abs_top_builddir}\" \
 	$(ZMQ_CFLAGS) \
 	$(LIBUUID_CFLAGS) \
+	$(JANSSON_CFLAGS) \
+	$(LIBSYSTEMD_CFLAGS) \
 	$(VALGRIND_CFLAGS)
 
 fluxcmd_PROGRAMS = flux-broker
@@ -17,75 +22,89 @@ fluxcmd_PROGRAMS = flux-broker
 noinst_LTLIBRARIES = libbroker.la
 
 flux_broker_SOURCES = \
-	broker.c
+	broker.c \
+	broker.h
 
 libbroker_la_SOURCES = \
+	brokercfg.c \
+	brokercfg.h \
 	module.c \
 	module.h \
+	modhash.c \
+	modhash.h \
 	modservice.c \
 	modservice.h \
 	overlay.h \
 	overlay.c \
-	heartbeat.h \
-	heartbeat.c \
 	service.h \
 	service.c \
-	hello.h \
-	hello.c \
-	reduce.h \
-	reduce.c \
-	shutdown.h \
-	shutdown.c \
 	attr.h \
 	attr.c \
 	log.h \
 	log.c \
-	content-cache.h \
-	content-cache.c \
-	runlevel.h \
-	runlevel.c \
+	runat.h \
+	runat.c \
+	state_machine.h \
+	state_machine.c \
 	heaptrace.h \
 	heaptrace.c \
 	exec.h \
 	exec.c \
-	ping.h \
-	ping.c \
-	rusage.h \
-	rusage.c \
 	boot_config.h \
 	boot_config.c \
 	boot_pmi.h \
 	boot_pmi.c \
-	pmiutil.h \
-	pmiutil.c \
-	liblist.h \
-	liblist.c \
 	publisher.h \
-	publisher.c
+	publisher.c \
+	groups.h \
+	groups.c \
+	shutdown.h \
+	shutdown.c \
+	topology.h \
+	topology.c \
+	trace.h \
+	trace.c
 
 flux_broker_LDADD = \
 	$(builddir)/libbroker.la \
 	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libzmqutil/libzmqutil.la \
+	$(top_builddir)/src/common/libflux/libflux.la \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(top_builddir)/src/common/libpmi/libupmi.la \
 	$(top_builddir)/src/common/libpmi/libpmi_client.la \
-	$(top_builddir)/src/common/libflux-internal.la
+	$(top_builddir)/src/common/libpmi/libpmi_common.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-optparse.la \
+	$(ZMQ_LIBS) \
+	$(LIBUUID_LIBS) \
+	$(JANSSON_LIBS) \
+	$(LIBSYSTEMD_LIBS) \
+	$(LIBDL)
 
 flux_broker_LDFLAGS =
 
-TESTS = test_heartbeat.t \
-	test_hello.t \
-	test_attr.t \
+TESTS = test_attr.t \
 	test_service.t \
-	test_reduce.t \
-	test_liblist.t \
-	test_pmiutil.t \
-	test_boot_config.t
+	test_boot_config.t \
+	test_runat.t \
+	test_overlay.t \
+	test_topology.t
 
 test_ldadd = \
 	$(builddir)/libbroker.la \
+	$(top_builddir)/src/common/libtestutil/libtestutil.la \
 	$(top_builddir)/src/common/libflux-core.la \
-	$(top_builddir)/src/common/libpmi/libpmi_client.la \
+	$(top_builddir)/src/common/libzmqutil/libzmqutil.la \
+	$(top_builddir)/src/common/libflux/libflux.la \
 	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libtap/libtap.la
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(ZMQ_LIBS) \
+	$(LIBSYSTEMD_LIBS) \
+	$(JANSSON_LIBS)
+
+test_ldflags = \
+	-no-install
 
 test_cppflags = \
         -I$(top_srcdir)/src/common/libtap \
@@ -98,34 +117,34 @@ TEST_EXTENSIONS = .t
 T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
        $(top_srcdir)/config/tap-driver.sh
 
-test_heartbeat_t_SOURCES = test/heartbeat.c
-test_heartbeat_t_CPPFLAGS = $(test_cppflags)
-test_heartbeat_t_LDADD = $(test_ldadd)
-
-test_hello_t_SOURCES = test/hello.c
-test_hello_t_CPPFLAGS = $(test_cppflags)
-test_hello_t_LDADD = $(test_ldadd)
-
 test_attr_t_SOURCES = test/attr.c
 test_attr_t_CPPFLAGS = $(test_cppflags)
 test_attr_t_LDADD = $(test_ldadd)
+test_attr_t_LDFLAGS = $(test_ldflags)
 
 test_service_t_SOURCES = test/service.c
 test_service_t_CPPFLAGS = $(test_cppflags)
 test_service_t_LDADD = $(test_ldadd)
-
-test_reduce_t_SOURCES = test/reduce.c
-test_reduce_t_CPPFLAGS = $(test_cppflags)
-test_reduce_t_LDADD = $(test_ldadd)
-
-test_liblist_t_SOURCES = test/liblist.c
-test_liblist_t_CPPFLAGS = $(test_cppflags)
-test_liblist_t_LDADD = $(test_ldadd)
-
-test_pmiutil_t_SOURCES = test/pmiutil.c
-test_pmiutil_t_CPPFLAGS = $(test_cppflags)
-test_pmiutil_t_LDADD = $(test_ldadd)
+test_service_t_LDFLAGS = $(test_ldflags)
 
 test_boot_config_t_SOURCES = test/boot_config.c
 test_boot_config_t_CPPFLAGS = $(test_cppflags)
 test_boot_config_t_LDADD = $(test_ldadd)
+test_boot_config_t_LDFLAGS = $(test_ldflags)
+
+test_runat_t_SOURCES = test/runat.c
+test_runat_t_CPPFLAGS = $(test_cppflags)
+test_runat_t_LDADD = $(test_ldadd)
+test_runat_t_LDFLAGS = $(test_ldflags)
+
+test_overlay_t_SOURCES = test/overlay.c
+test_overlay_t_CPPFLAGS = $(test_cppflags)
+test_overlay_t_LDADD = $(test_ldadd)
+test_overlay_t_LDFLAGS = $(test_ldflags)
+
+test_topology_t_SOURCES = test/topology.c
+test_topology_t_CPPFLAGS = $(test_cppflags)
+test_topology_t_LDADD = $(test_ldadd)
+test_topology_t_LDFLAGS = $(test_ldflags)
+
+EXTRA_DIST = README.md
diff --git a/src/broker/README.md b/src/broker/README.md
new file mode 100644
index 000000000000..4d5867553867
--- /dev/null
+++ b/src/broker/README.md
@@ -0,0 +1,6 @@
+## Design notes
+
+Assorted design notes are available in
+[Resources for Flux Developers](https://flux-framework.readthedocs.io/projects/flux-core/en/latest/guide/internals.html)
+including for the
+[Broker](https://flux-framework.readthedocs.io/projects/flux-core/en/latest/guide/broker.html).
diff --git a/src/broker/attr.c b/src/broker/attr.c
index 7d61a8c40b2e..1ab02e7254af 100644
--- a/src/broker/attr.c
+++ b/src/broker/attr.c
@@ -11,12 +11,15 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
+#include <limits.h>
 #include <jansson.h>
+#include <inttypes.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "attr.h"
 
-struct attr_struct {
+struct broker_attr {
     zhash_t *hash;
     flux_msg_handler_t **handlers;
 };
@@ -34,33 +37,24 @@ static void entry_destroy (void *arg)
 {
     struct entry *e = arg;
     if (e) {
-        if (e->val)
-            free (e->val);
-        if (e->name)
-            free (e->name);
+        int saved_errno = errno;
+        free (e->val);
+        free (e->name);
         free (e);
+        errno = saved_errno;
     }
 }
 
 static struct entry *entry_create (const char *name, const char *val, int flags)
 {
-    struct entry *e = calloc (1, sizeof (*e));
-    if (!e) {
-        errno = ENOMEM;
+    struct entry *e;
+    if (!(e = calloc (1, sizeof (*e))))
         return NULL;
-    }
-    if (name) {
-        if (!(e->name = strdup (name))) {
-            errno = ENOMEM;
-            goto cleanup;
-        }
-    }
-    if (val) {
-        if (!(e->val = strdup (val))) {
-            errno = ENOMEM;
-            goto cleanup;
-        }
-    }
+
+    if (name && !(e->name = strdup (name)))
+        goto cleanup;
+    if (val && !(e->val = strdup (val)))
+        goto cleanup;
     e->flags = flags;
     return e;
 cleanup:
@@ -74,12 +68,7 @@ int attr_delete (attr_t *attrs, const char *name, bool force)
     int rc = -1;
 
     if ((e = zhash_lookup (attrs->hash, name))) {
-        if ((e->flags & FLUX_ATTRFLAG_IMMUTABLE)) {
-            errno = EPERM;
-            goto done;
-        }
-        if (((e->flags & FLUX_ATTRFLAG_READONLY)
-                            || (e->flags & FLUX_ATTRFLAG_ACTIVE)) && !force) {
+        if ((e->flags & ATTR_IMMUTABLE)) {
             errno = EPERM;
             goto done;
         }
@@ -94,7 +83,7 @@ int attr_add (attr_t *attrs, const char *name, const char *val, int flags)
 {
     struct entry *e;
 
-    if (name == NULL || (flags & FLUX_ATTRFLAG_ACTIVE)) {
+    if (attrs == NULL || name == NULL) {
         errno = EINVAL;
         return -1;
     }
@@ -102,7 +91,8 @@ int attr_add (attr_t *attrs, const char *name, const char *val, int flags)
         errno = EEXIST;
         return -1;
     }
-    e = entry_create (name, val, flags);
+    if (!(e = entry_create (name, val, flags)))
+        return -1;
     zhash_update (attrs->hash, name, e);
     zhash_freefn (attrs->hash, name, entry_destroy);
     return 0;
@@ -114,6 +104,10 @@ int attr_add_active (attr_t *attrs, const char *name, int flags,
     struct entry *e;
     int rc = -1;
 
+    if (!attrs) {
+        errno = EINVAL;
+        goto done;
+    }
     if ((e = zhash_lookup (attrs->hash, name))) {
         if (!set) {
             errno = EEXIST;
@@ -122,11 +116,11 @@ int attr_add_active (attr_t *attrs, const char *name, int flags,
         if (set (name, e->val, arg) < 0)
             goto done;
     }
-    e = entry_create (name, NULL, flags);
+    if (!(e = entry_create (name, NULL, flags)))
+        goto done;
     e->set = set;
     e->get = get;
     e->arg = arg;
-    e->flags |= FLUX_ATTRFLAG_ACTIVE;
     zhash_update (attrs->hash, name, e);
     zhash_freefn (attrs->hash, name, entry_destroy);
     rc = 0;
@@ -139,22 +133,24 @@ int attr_get (attr_t *attrs, const char *name, const char **val, int *flags)
     struct entry *e;
     int rc = -1;
 
+    if (!attrs || !name) {
+        errno = EINVAL;
+        goto done;
+    }
     if (!(e = zhash_lookup (attrs->hash, name))) {
         errno = ENOENT;
         goto done;
     }
     if (e->get) {
-        if (!e->val || !(e->flags & FLUX_ATTRFLAG_IMMUTABLE)) {
+        if (!e->val || !(e->flags & ATTR_IMMUTABLE)) {
             const char *tmp;
             if (e->get (name, &tmp, e->arg) < 0)
                 goto done;
             if (e->val)
                 free (e->val);
             if (tmp) {
-                if (!(e->val = strdup (tmp))) {
-                    errno = ENOMEM;
+                if (!(e->val = strdup (tmp)))
                     goto done;
-                }
             }
             else
                 e->val = NULL;
@@ -169,7 +165,7 @@ int attr_get (attr_t *attrs, const char *name, const char **val, int *flags)
     return rc;
 }
 
-int attr_set (attr_t *attrs, const char *name, const char *val, bool force)
+int attr_set (attr_t *attrs, const char *name, const char *val)
 {
     struct entry *e;
     int rc = -1;
@@ -178,11 +174,7 @@ int attr_set (attr_t *attrs, const char *name, const char *val, bool force)
         errno = ENOENT;
         goto done;
     }
-    if ((e->flags & FLUX_ATTRFLAG_IMMUTABLE)) {
-        errno = EPERM;
-        goto done;
-    }
-    if ((e->flags & FLUX_ATTRFLAG_READONLY) && !force) {
+    if ((e->flags & ATTR_IMMUTABLE)) {
         errno = EPERM;
         goto done;
     }
@@ -193,10 +185,8 @@ int attr_set (attr_t *attrs, const char *name, const char *val, bool force)
     if (e->val)
         free (e->val);
     if (val) {
-        if (!(e->val = strdup (val))) {
-            errno = ENOMEM;
+        if (!(e->val = strdup (val)))
             goto done;
-        }
     }
     else
         e->val = NULL;
@@ -224,9 +214,11 @@ static int get_int (const char *name, const char **val, void *arg)
 {
     int *i = arg;
     static char s[32];
-    int n = snprintf (s, sizeof (s), "%d", *i);
 
-    assert (n <= sizeof (s));
+    if (snprintf (s, sizeof (s), "%d", *i) >= sizeof (s)) {
+        errno = EOVERFLOW;
+        return -1;
+    }
     *val = s;
     return 0;
 }
@@ -241,13 +233,14 @@ static int set_int (const char *name, const char *val, void *arg)
         errno = EINVAL;
         return -1;
     }
+    errno = 0;
     n = strtol (val, &endptr, 0);
-    if (n <= INT_MIN || n >= INT_MAX) {
-        errno = ERANGE;
+    if (errno != 0 || *endptr != '\0') {
+        errno = EINVAL;
         return -1;
     }
-    if (*endptr != '\0') {
-        errno = EINVAL;
+    if (n <= INT_MIN || n >= INT_MAX) {
+        errno = ERANGE;
         return -1;
     }
     *i = (int)n;
@@ -256,13 +249,13 @@ static int set_int (const char *name, const char *val, void *arg)
 
 int attr_add_int (attr_t *attrs, const char *name, int val, int flags)
 {
-    char val_string[32];
-    int n;
+    char s[32];
 
-    n = snprintf (val_string, sizeof (val_string), "%d", val);
-    assert (n <= sizeof(val_string));
-
-    return attr_add (attrs, name, val_string, flags);
+    if (snprintf (s, sizeof (s), "%d", val) >= sizeof (s)) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    return attr_add (attrs, name, s, flags);
 }
 
 int attr_add_active_int (attr_t *attrs, const char *name, int *val, int flags)
@@ -274,9 +267,11 @@ static int get_uint32 (const char *name, const char **val, void *arg)
 {
     uint32_t *i = arg;
     static char s[32];
-    int n = snprintf (s, sizeof (s), "%" PRIu32, *i);
 
-    assert (n <= sizeof (s));
+    if (snprintf (s, sizeof (s), "%" PRIu32, *i) >= sizeof (s)) {
+        errno = EOVERFLOW;
+        return -1;
+    }
     *val = s;
     return 0;
 }
@@ -287,10 +282,9 @@ static int set_uint32 (const char *name, const char *val, void *arg)
     char *endptr;
     unsigned long n;
 
+    errno = 0;
     n = strtoul (val, &endptr, 0);
-    if (n == ULONG_MAX) /* ERANGE set by strtol */
-        return -1;
-    if (endptr == val || *endptr != '\0') {
+    if (errno != 0 || *endptr != '\0') {
         errno = EINVAL;
         return -1;
     }
@@ -313,6 +307,25 @@ int attr_add_active_uint32 (attr_t *attrs, const char *name, uint32_t *val,
     return attr_add_active (attrs, name, flags, get_uint32, set_uint32, val);
 }
 
+int attr_get_uint32 (attr_t *attrs, const char *name, uint32_t *value)
+{
+    const char *s;
+    uint32_t i;
+    char *endptr;
+
+    if (attr_get (attrs, name, &s, NULL) < 0)
+        return -1;
+
+    errno = 0;
+    i = strtoul (s, &endptr, 10);
+    if (errno != 0 || *endptr != '\0') {
+        errno = EINVAL;
+        return -1;
+    }
+    *value = i;
+    return 0;
+}
+
 const char *attr_first (attr_t *attrs)
 {
     struct entry *e = zhash_first (attrs->hash);
@@ -325,6 +338,21 @@ const char *attr_next (attr_t *attrs)
     return e ? e->name : NULL;
 }
 
+int attr_cache_immutables (attr_t *attrs, flux_t *h)
+{
+    struct entry *e;
+
+    e = zhash_first (attrs->hash);
+    while (e) {
+        if ((e->flags & ATTR_IMMUTABLE)) {
+            if (flux_attr_set_cacheonly (h, e->name, e->val) < 0)
+                return -1;
+        }
+        e = zhash_next (attrs->hash);
+    }
+    return 0;
+}
+
 /**
  ** Service
  **/
@@ -365,7 +393,7 @@ void setattr_request_cb (flux_t *h, flux_msg_handler_t *mh,
     if (flux_request_unpack (msg, NULL, "{s:s s:s}", "name", &name,
                                                      "value", &val) < 0)
         goto error;
-    if (attr_set (attrs, name, val, false) < 0) {
+    if (attr_set (attrs, name, val) < 0) {
         if (errno != ENOENT)
             goto error;
         if (attr_add (attrs, name, val, 0) < 0)
@@ -379,24 +407,6 @@ void setattr_request_cb (flux_t *h, flux_msg_handler_t *mh,
         FLUX_LOG_ERROR (h);
 }
 
-void rmattr_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                        const flux_msg_t *msg, void *arg)
-{
-    attr_t *attrs = arg;
-    const char *name;
-
-    if (flux_request_unpack (msg, NULL, "{s:s}", "name", &name) < 0)
-        goto error;
-    if (attr_delete (attrs, name, false) < 0)
-        goto error;
-    if (flux_respond (h, msg, NULL) < 0)
-        FLUX_LOG_ERROR (h);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        FLUX_LOG_ERROR (h);
-}
-
 void lsattr_request_cb (flux_t *h, flux_msg_handler_t *mh,
                         const flux_msg_t *msg, void *arg)
 {
@@ -412,14 +422,11 @@ void lsattr_request_cb (flux_t *h, flux_msg_handler_t *mh,
     }
     name = attr_first (attrs);
     while (name) {
-        if (!(js = json_string (name))) {
-            errno = ENOMEM;
-            goto error;
-        }
+        if (!(js = json_string (name)))
+            goto nomem;
         if (json_array_append_new (names, js) < 0) {
             json_decref (js);
-            errno = ENOMEM;
-            goto error;
+            goto nomem;
         }
         name = attr_next (attrs);
     }
@@ -427,6 +434,8 @@ void lsattr_request_cb (flux_t *h, flux_msg_handler_t *mh,
         FLUX_LOG_ERROR (h);
     json_decref (names);
     return;
+nomem:
+    errno = ENOMEM;
 error:
     if (flux_respond_error (h, msg, errno, NULL) < 0)
         FLUX_LOG_ERROR (h);
@@ -441,7 +450,6 @@ static const struct flux_msg_handler_spec handlers[] = {
     { FLUX_MSGTYPE_REQUEST, "attr.get",    getattr_request_cb, FLUX_ROLE_ALL },
     { FLUX_MSGTYPE_REQUEST, "attr.list",   lsattr_request_cb, FLUX_ROLE_ALL },
     { FLUX_MSGTYPE_REQUEST, "attr.set",    setattr_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "attr.rm",     rmattr_request_cb, 0 },
     FLUX_MSGHANDLER_TABLE_END,
 };
 
@@ -455,11 +463,10 @@ int attr_register_handlers (attr_t *attrs, flux_t *h)
 
 attr_t *attr_create (void)
 {
-    attr_t *attrs = calloc (1, sizeof (*attrs));
-    if (!attrs) {
-        errno = ENOMEM;
+    attr_t *attrs;
+
+    if (!(attrs = calloc (1, sizeof (*attrs))))
         return NULL;
-    }
     if (!(attrs->hash = zhash_new ())) {
         attr_destroy (attrs);
         errno = ENOMEM;
@@ -471,10 +478,12 @@ attr_t *attr_create (void)
 void attr_destroy (attr_t *attrs)
 {
     if (attrs) {
-        if (attrs->handlers)
-            flux_msg_handler_delvec (attrs->handlers);
+        int saved_errno = errno;
+        flux_msg_handler_delvec (attrs->handlers);
         zhash_destroy (&attrs->hash);
         free (attrs);
+        errno = saved_errno;
+
     }
 }
 
diff --git a/src/broker/attr.h b/src/broker/attr.h
index 6e6da33b10d4..e51ce1231939 100644
--- a/src/broker/attr.h
+++ b/src/broker/attr.h
@@ -15,19 +15,16 @@
 #include <flux/core.h>
 
 enum {
-    FLUX_ATTRFLAG_IMMUTABLE = 1,    /* attribute is cacheable */
-    FLUX_ATTRFLAG_READONLY = 2,     /* attribute cannot be written */
-                                    /*   but may change on broker */
-    FLUX_ATTRFLAG_ACTIVE = 4,       /* attribute has get and/or set callbacks */
+    ATTR_IMMUTABLE = 1,    /* attribute is cacheable */
 };
 
-/* Callbacks for active values.  Return 0 on succes, -1 on eror with
+/* Callbacks for active values.  Return 0 on success, -1 on error with
  * errno set.  Errors are propagated to the return of attr_set() and attr_get().
  */
 typedef int (*attr_get_f)(const char *name, const char **val, void *arg);
 typedef int (*attr_set_f)(const char *name, const char *val, void *arg);
 
-typedef struct attr_struct attr_t;
+typedef struct broker_attr attr_t;
 
 /* Create/destroy attribute cache
  */
@@ -57,7 +54,7 @@ int attr_add_uint32 (attr_t *attrs, const char *name, uint32_t val,
  */
 int attr_get (attr_t *attrs, const char *name, const char **val, int *flags);
 
-int attr_set (attr_t *attrs, const char *name, const char *val, bool force);
+int attr_set (attr_t *attrs, const char *name, const char *val);
 
 /* Set an attribute's flags.
  */
@@ -75,11 +72,19 @@ int attr_add_active_int (attr_t *attrs, const char *name, int *val,
 int attr_add_active_uint32 (attr_t *attrs, const char *name, uint32_t *val,
                             int flags);
 
+/* Get an attribute and parse it as an integer value.
+ */
+int attr_get_uint32 (attr_t *attrs, const char *name, uint32_t *value);
+
 /* Iterate over attribute names with internal cursor.
  */
 const char *attr_first (attr_t *attrs);
 const char *attr_next (attr_t *attrs);
 
+/* Cache all immutable attributes present in attrs at this point in time.
+ */
+int attr_cache_immutables (attr_t *attrs, flux_t *h);
+
 #endif /* BROKER_ATTR_H */
 
 /*
diff --git a/src/broker/boot_config.c b/src/broker/boot_config.c
index 5a9bc3cb9cdd..7289e73e8d53 100644
--- a/src/broker/boot_config.c
+++ b/src/broker/boot_config.c
@@ -13,23 +13,29 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
 #include <unistd.h>
 #include <stdbool.h>
-#include <assert.h>
 #include <jansson.h>
+#include <sys/param.h>
 #include <flux/core.h>
+#include <flux/hostlist.h>
+#include <flux/taskmap.h>
 
+#include "src/common/libyuarel/yuarel.h"
 #include "src/common/libutil/log.h"
-#include "src/common/libutil/kary.h"
 #include "src/common/libutil/errno_safe.h"
-#include "src/common/libidset/idset.h"
+#include "ccan/str/str.h"
 
 #include "attr.h"
 #include "overlay.h"
+#include "topology.h"
 #include "boot_config.h"
 
 
-/* Copy 'fmt' into 'buf', substuting the following tokens:
+/* Copy 'fmt' into 'buf', substituting the following tokens:
  * - %h  host
  * - %p  port
  * Returns 0 on success, or -1 on overflow.
@@ -93,97 +99,140 @@ int boot_config_format_uri (char *buf,
     return -1;
 }
 
-struct map_arg {
-    json_t *hosts;
-    json_t *entry;
-};
+static int set_string (json_t *o, const char *key, const char *s)
+{
+    json_t *val;
+
+    if (!(val = json_string (s))
+        || json_object_set_new (o, key, val) < 0) {
+        json_decref (val);
+        return -1;
+    }
+    return 0;
+}
 
-/* Add a host entry to maparg->hosts for the formatted hostname in 's',
- * cloned from the original host entry object in maparg->entry.
- * N.B. idset_map_f footprint
+/* Make a copy of 'entry', set host key to the specified value, and append
+ * to 'hosts' array.
  */
-static int boot_config_map_hosts (const char *s, bool *stop, void *arg)
+static int boot_config_append_host (json_t *hosts,
+                                    const char *hostname,
+                                    json_t *entry)
 {
-    struct map_arg *maparg = arg;
     json_t *nentry;
-    json_t *o;
-
-    if (!(nentry = json_deep_copy (maparg->entry)))
-        goto nomem;
-    if (!(o = json_string (s)))
-        goto nomem;
-    if (json_object_set_new (nentry, "host", o) < 0) {
-        json_decref (o);
-        goto nomem;
-    }
-    if (json_array_append_new (maparg->hosts, nentry) < 0)
-        goto nomem;
+
+    if (!(nentry = json_deep_copy (entry))
+        || set_string (nentry, "host", hostname) < 0
+        || json_array_append_new (hosts, nentry) < 0) {
+        json_decref (nentry);
+        return -1;
+    }
     return 0;
-nomem:
-    json_decref (nentry);
-    errno = ENOMEM;
-    return -1;
 }
 
-/* Build a new hosts array, expanding any entries "compressed" with
- * embedded idsets, so that json_array_size() is the number of hosts,
- * and json_array_get() can be used to fetch an entry by rank.
- * Caller must free.
+/* Build a new hosts array, expanding any RFC29 hostlists, so that
+ * json_array_size() is the number of hosts, and json_array_get() can
+ * be used to fetch an entry by rank.  Preserve the initial hosts order so
+ * that the rank mapping is deterministic, but combine the host entries of
+ * any entries that have the same host key.
+ * The caller must release the returned JSON object with json_decref().
  */
 static json_t *boot_config_expand_hosts (json_t *hosts)
 {
-    json_t *nhosts;
+    json_t *nhosts = NULL;
+    json_t *hash = NULL;
     size_t index;
     json_t *value;
 
-    if (!(nhosts = json_array ())) {
+    if (!(nhosts = json_array ())
+        || !(hash = json_object ())) {
         log_msg ("Config file error [bootstrap]: out of memory");
-        return NULL;
+        goto error;
     }
     json_array_foreach (hosts, index, value) {
-        struct map_arg maparg = { .hosts = nhosts, .entry = value};
-        const char *host;
-
-        if (json_unpack (value, "{s:s}", "host", &host) < 0) {
-            log_msg ("Config file error [bootstrap]: missing host field");
-            log_msg ("Hint: hosts entries must be table containing a host key");
+        struct hostlist *hl = NULL;
+        json_error_t error;
+        const char *host, *s;
+        const char *bind = NULL;
+        const char *connect = NULL;
+        const char *parent = NULL;
+
+        if (json_unpack_ex (value,
+                            &error,
+                            0,
+                            "{s:s s?s s?s s?s !}",
+                            "host", &host,
+                            "bind", &bind,
+                            "connect", &connect,
+                            "parent", &parent) < 0) {
+            log_msg ("Config file error [bootstrap] host entry: %s",
+                     error.text);
             goto error;
         }
-        if (idset_format_map (host, boot_config_map_hosts, &maparg) < 0) {
-            log_err ("Config file error [bootstrap]");
-            log_msg ("Hint: text enclosed by square brackets must be an idset");
+        if (!(hl = hostlist_decode (host))) {
+            log_msg ("Config file error [bootstrap]:"
+                     " host value '%s' is not a valid hostlist",
+                     host);
             goto error;
         }
+        s = hostlist_first (hl);
+        while (s) {
+            json_t *entry;
+            if (!(entry = json_object_get (hash, s))) {
+                if (boot_config_append_host (nhosts, s, value) < 0
+                    || json_object_set (hash, s, value) < 0) {
+                    log_msg ("Config file error [bootstrap]:"
+                             " error appending host %s", s);
+                    hostlist_destroy (hl);
+                    goto error;
+                }
+            }
+            else {
+                if (json_object_update (entry, value) < 0) {
+                    log_msg ("Config file error [bootstrap]:"
+                             " error merging host %s", s);
+                    hostlist_destroy (hl);
+                    goto error;
+                }
+            }
+            s = hostlist_next (hl);
+        }
+        hostlist_destroy (hl);
     }
+    json_decref (hash);
     return nhosts;
 error:
     json_decref (nhosts);
+    json_decref (hash);
+
     return NULL;
 }
 
 /* Parse the [bootstrap] configuration.
- * Capture the portion that is owned by the flux_t handle in 'conf'.
- * Post-process conf->hosts to expand any embedded idsets and return
+ * Post-process conf->hosts to expand any embedded RFC29 hostlists and return
  * a new hosts JSON array on success.  On failure, log a human readable
  * message and return NULL.
  */
-int boot_config_parse (flux_t *h, struct boot_conf *conf, json_t **hostsp)
+int boot_config_parse (const flux_conf_t *cf,
+                       struct boot_conf *conf,
+                       json_t **hostsp)
 {
-    flux_conf_error_t error;
+    flux_error_t error;
     const char *default_bind = NULL;
     const char *default_connect = NULL;
     json_t *hosts = NULL;
 
     memset (conf, 0, sizeof (*conf));
-    if (flux_conf_unpack (flux_get_conf (h, NULL),
+    if (flux_conf_unpack (cf,
                           &error,
-                          "{s:{s?:i s?:s s?:s s?:o}}",
+                          "{s:{s?s s?i s?s s?s s?o s?b}}",
                           "bootstrap",
+                            "curve_cert", &conf->curve_cert,
                             "default_port", &conf->default_port,
                             "default_bind", &default_bind,
                             "default_connect", &default_connect,
-                            "hosts", &conf->hosts) < 0) {
-        log_msg ("Config file error [bootstrap]: %s", error.errbuf);
+                            "hosts", &conf->hosts,
+                            "enable_ipv6", &conf->enable_ipv6) < 0) {
+        log_msg ("Config file error [bootstrap]: %s", error.text);
         return -1;
     }
 
@@ -213,7 +262,7 @@ int boot_config_parse (flux_t *h, struct boot_conf *conf, json_t **hostsp)
         }
     }
 
-    /* Expand any embedded idsets in hosts entries.
+    /* Expand any embedded hostlists in hosts entries.
      */
     if (conf->hosts) {
         if (!json_is_array (conf->hosts)) {
@@ -226,10 +275,96 @@ int boot_config_parse (flux_t *h, struct boot_conf *conf, json_t **hostsp)
         }
     }
 
+    /* Fail early if size > 1 and there is no CURVE certificate configured.
+     */
+    if (json_array_size (conf->hosts) > 1 && !conf->curve_cert) {
+        log_msg ("Config file error [bootstrap] %s",
+                 "curve_cert is required for size > 1");
+        return -1;
+    }
+
     *hostsp = hosts;
     return 0;
 }
 
+int boot_config_attr (attr_t *attrs, const char *hostname, json_t *hosts)
+{
+    struct hostlist *hl = NULL;
+    struct taskmap *map = NULL;
+    char *s = NULL;
+    size_t index;
+    json_t *value;
+    char *val;
+    int rv = -1;
+
+    if (!hosts || json_array_size (hosts) == 0) {
+        if (attr_add (attrs,
+                      "hostlist",
+                      hostname,
+                      ATTR_IMMUTABLE) < 0) {
+            log_err ("failed to set hostlist attribute to localhost");
+            goto error;
+        }
+        return 0;
+    }
+
+    if (!(hl = hostlist_create ()))
+        goto error;
+
+    json_array_foreach (hosts, index, value) {
+        const char *host;
+        if (json_unpack (value, "{s:s}", "host", &host) < 0) {
+            log_msg ("Internal error [bootstrap]: missing host field");
+            errno = EINVAL;
+            goto error;
+        }
+        if (hostlist_append (hl, host) < 0)
+            goto error;
+    }
+
+    if (!(s = hostlist_encode (hl)))
+        goto error;
+
+    if (attr_add (attrs,
+                  "hostlist",
+                  s,
+                  ATTR_IMMUTABLE) < 0) {
+        log_err ("failed to set hostlist attribute to config derived value");
+        goto error;
+    }
+    free (s);
+    s = NULL;
+
+    /* Generate broker.mapping.
+     * For now, set it to NULL if there are multiple brokers per node.
+     */
+    hostlist_uniq (hl);
+    if (hostlist_count (hl) < json_array_size (hosts))
+        val = NULL;
+    else {
+        if (!(map = taskmap_create ())
+            || taskmap_append (map, 0, json_array_size (hosts), 1) < 0)
+            goto error;
+        if (!(s = taskmap_encode (map, 0))) {
+            log_msg ("encoding broker.mapping");
+            errno = EOVERFLOW;
+            goto error;
+        }
+        val = s;
+    }
+    if (attr_add (attrs, "broker.mapping", val, ATTR_IMMUTABLE) < 0) {
+        log_err ("setattr broker.mapping");
+        goto error;
+    }
+
+    rv = 0;
+error:
+    taskmap_destroy (map);
+    hostlist_destroy (hl);
+    free (s);
+    return rv;
+}
+
 /* Find host 'name' in hosts array, and set 'rank' to its array index.
  * Return 0 on success, -1 on failure.
  */
@@ -241,10 +376,10 @@ int boot_config_getrankbyname (json_t *hosts, const char *name, uint32_t *rank)
     json_array_foreach (hosts, index, entry) {
         const char *host;
 
-        /* N.B. missing host key already detected by boot_config_parse().
+        /* N.B. entry already validated by boot_config_parse().
          */
         if (json_unpack (entry, "{s:s}", "host", &host) == 0
-                    && !strcmp (name, host)) {
+            && streq (name, host)) {
             *rank = index;
             return 0;
         }
@@ -267,21 +402,13 @@ static int gethostentry (json_t *hosts,
                  (unsigned int)rank);
         return -1;
     }
-    /* N.B. missing host key already detected by boot_config_parse().
+    /* N.B. entry already validated by boot_config_parse().
      */
-    if (json_unpack (entry,
-                     "{s:s s?:s s?:s}",
-                     "host",
-                     host,
-                     "bind",
-                     bind,
-                     "connect",
-                     uri) < 0) {
-        log_msg ("Config file error [bootstrap]: rank %u bad hosts entry",
-                 (unsigned int)rank);
-        log_msg ("Hint: bind and connect keys, if present, are type string");
-        return -1;
-    }
+    (void)json_unpack (entry,
+                      "{s:s s?s s?s}",
+                      "host", host,
+                      "bind", bind,
+                      "connect", uri);
     return 0;
 }
 
@@ -355,41 +482,77 @@ int boot_config_geturibyrank (json_t *hosts,
     return 0;
 }
 
-int boot_config (flux_t *h, overlay_t *overlay, attr_t *attrs, int tbon_k)
+static int set_broker_boot_method_attr (attr_t *attrs, const char *value)
+{
+    (void)attr_delete (attrs, "broker.boot-method", true);
+    if (attr_add (attrs,
+                  "broker.boot-method",
+                  value,
+                  ATTR_IMMUTABLE) < 0) {
+        log_err ("setattr broker.boot-method");
+        return -1;
+    }
+    return 0;
+}
+
+/* Zeromq treats failed hostname resolution as a transient error, and silently
+ * retries in the background, which can make config problems hard to diagnose.
+ * Parse the URI in advance, and if the host portion is invalid, log it.
+ * Ref: flux-framework/flux-core#5009
+ */
+static void warn_of_invalid_host (flux_t *h, const char *uri)
+{
+    char *cpy;
+    struct yuarel u = { 0 };
+    struct addrinfo *result;
+    int e;
+
+    if (!(cpy = strdup (uri))
+        || yuarel_parse (&u, cpy) < 0
+        || !u.scheme
+        || !u.host
+        || !streq (u.scheme, "tcp"))
+        goto done;
+    /* N.B. this URI will be used for zmq_connect(), therefore it must
+     * be a valid peer address, not an interface name or wildcard.
+     */
+    if ((e = getaddrinfo (u.host, NULL, NULL, &result)) == 0) {
+        freeaddrinfo (result);
+        goto done;
+    }
+    log_msg ("Warning: unable to resolve upstream peer %s: %s",
+             u.host,
+             gai_strerror (e));
+done:
+    free (cpy);
+}
+
+int boot_config (flux_t *h,
+                 const char *hostname,
+                 struct overlay *overlay,
+                 attr_t *attrs)
 {
     struct boot_conf conf;
     uint32_t rank;
     uint32_t size;
     json_t *hosts = NULL;
-
-    /* Throw an error if 'tbon.endpoint' attribute is already set.
-     * flux-start sets this, and it's not compatible with the
-     * config boot method as it would be overwritten below.
-     */
-    if (attr_get (attrs, "tbon.endpoint", NULL, NULL) == 0) {
-        log_msg ("attr tbon.endpoint may not be set with boot_method=config");
-        return -1;
-    }
+    struct topology *topo = NULL;
+    flux_error_t error;
+    const char *topo_uri;
 
     /* Ingest the [bootstrap] stanza.
      */
-    if (boot_config_parse (h, &conf, &hosts) < 0)
+    if (boot_config_parse (flux_get_conf (h), &conf, &hosts) < 0)
         return -1;
 
+    if (boot_config_attr (attrs, hostname, hosts) < 0)
+        goto error;
+
     /* If hosts array was specified, match hostname to determine rank,
      * and size is the length of the hosts array.  O/w rank=0, size=1.
      */
     if (hosts != NULL) {
-        const char *fakehost = getenv ("FLUX_FAKE_HOSTNAME"); // for testing;
-        char hostname[MAXHOSTNAMELEN + 1];
-
-        if (gethostname (hostname, sizeof (hostname)) < 0) {
-            log_err ("gethostbyname");
-            goto error;
-        }
-        if (boot_config_getrankbyname (hosts,
-                                      fakehost ? fakehost : hostname,
-                                      &rank) < 0)
+        if (boot_config_getrankbyname (hosts, hostname, &rank) < 0)
             goto error;
         size = json_array_size (hosts);
     }
@@ -398,34 +561,65 @@ int boot_config (flux_t *h, overlay_t *overlay, attr_t *attrs, int tbon_k)
         rank = 0;
     }
 
-    /* Tell overlay network this broker's rank, size, and branching factor.
+    // N.B. overlay_create() sets the tbon.topo attribute
+    if (attr_get (attrs, "tbon.topo", &topo_uri, NULL) < 0) {
+        log_msg ("error fetching tbon.topo attribute");
+        goto error;
+    }
+    topology_hosts_set (hosts);
+    topo = topology_create (topo_uri, size, &error);
+    topology_hosts_set (NULL);
+    if (!topo) {
+        log_msg ("Error creating %s topology: %s", topo_uri, error.text);
+        goto error;
+    }
+    if (topology_set_rank (topo, rank) < 0
+        || overlay_set_topology (overlay, topo) < 0)
+        goto error;
+
+    /* If a curve certificate was provided, load it.
+     */
+    if (conf.curve_cert) {
+        if (overlay_cert_load (overlay, conf.curve_cert) < 0)
+            goto error; // prints error
+    }
+
+    /* If user requested ipv6, enable it here.
+     * N.B. this prevents binding from interfaces that are IPv4 only (#3824).
+     */
+    overlay_set_ipv6 (overlay, conf.enable_ipv6);
+
+    /* Ensure that tbon.interface-hint is set.
      */
-    if (overlay_init (overlay, size, rank, tbon_k) < 0)
+    if (overlay_set_tbon_interface_hint (overlay, NULL) < 0) {
+        log_err ("error setting tbon.interface-hint attribute");
         goto error;
+    }
 
     /* If broker has "downstream" peers, determine the URI to bind to
      * from the config and tell overlay.  Also, set the tbon.endpoint
      * attribute to the URI peers will connect to.  If broker has no
      * downstream peers, set tbon.endpoint to NULL.
      */
-    if (kary_childof (tbon_k, size, rank, 0) != KARY_NONE) {
+    if (topology_get_child_ranks (topo, NULL, 0) > 0
+        && attr_get (attrs, "broker.recovery-mode", NULL, NULL) < 0) {
         char bind_uri[MAX_URI + 1];
         char my_uri[MAX_URI + 1];
 
-        assert (hosts != NULL);
-
+        if (!hosts)
+            log_err_exit ("internal error: hosts object is NULL");
         if (boot_config_getbindbyrank (hosts,
                                        &conf,
                                        rank,
                                        bind_uri,
                                        sizeof (bind_uri)) < 0)
             goto error;
-        if (overlay_set_child (overlay, bind_uri) < 0) {
-            log_err ("overlay_set_child %s", bind_uri);
+        if (overlay_bind (overlay, bind_uri) < 0)
             goto error;
-        }
-        if (overlay_bind (overlay) < 0) { /* idempotent */
-            log_err ("overlay_bind");
+        if (overlay_authorize (overlay,
+                               overlay_cert_name (overlay),
+                               overlay_cert_pubkey (overlay)) < 0) {
+            log_err ("overlay_authorize");
             goto error;
         }
         if (boot_config_geturibyrank (hosts,
@@ -437,7 +631,7 @@ int boot_config (flux_t *h, overlay_t *overlay, attr_t *attrs, int tbon_k)
         if (attr_add (attrs,
                       "tbon.endpoint",
                       my_uri,
-                      FLUX_ATTRFLAG_IMMUTABLE) < 0) {
+                      ATTR_IMMUTABLE) < 0) {
             log_err ("setattr tbon.endpoint %s", my_uri);
             goto error;
         }
@@ -446,7 +640,7 @@ int boot_config (flux_t *h, overlay_t *overlay, attr_t *attrs, int tbon_k)
         if (attr_add (attrs,
                       "tbon.endpoint",
                       NULL,
-                      FLUX_ATTRFLAG_IMMUTABLE) < 0) {
+                      ATTR_IMMUTABLE) < 0) {
             log_err ("setattr tbon.endpoint NULL");
             goto error;
         }
@@ -458,12 +652,18 @@ int boot_config (flux_t *h, overlay_t *overlay, attr_t *attrs, int tbon_k)
         char parent_uri[MAX_URI + 1];
         if (boot_config_geturibyrank (hosts,
                                       &conf,
-                                      kary_parentof (tbon_k, rank),
+                                      topology_get_parent (topo),
                                       parent_uri,
                                       sizeof (parent_uri)) < 0)
             goto error;
-        if (overlay_set_parent (overlay, parent_uri) < 0) {
-            log_err ("overlay_set_parent %s", parent_uri);
+        warn_of_invalid_host (h, parent_uri);
+        if (overlay_set_parent_uri (overlay, parent_uri) < 0) {
+            log_err ("overlay_set_parent_uri %s", parent_uri);
+            goto error;
+        }
+        if (overlay_set_parent_pubkey (overlay,
+                                       overlay_cert_pubkey (overlay)) < 0) {
+            log_err ("overlay_set_parent_pubkey self");
             goto error;
         }
     }
@@ -473,14 +673,18 @@ int boot_config (flux_t *h, overlay_t *overlay, attr_t *attrs, int tbon_k)
     if (attr_add (attrs,
                   "instance-level",
                   "0",
-                  FLUX_ATTRFLAG_IMMUTABLE) < 0) {
+                  ATTR_IMMUTABLE) < 0) {
         log_err ("setattr instance-level 0");
         goto error;
     }
+    if (set_broker_boot_method_attr (attrs, "config") < 0)
+        goto error;
     json_decref (hosts);
+    topology_decref (topo);
     return 0;
 error:
     ERRNO_SAFE_WRAP (json_decref, hosts);
+    ERRNO_SAFE_WRAP (topology_decref, topo);
     return -1;
 }
 
diff --git a/src/broker/boot_config.h b/src/broker/boot_config.h
index 9cfe45e33621..3008080dc1c1 100644
--- a/src/broker/boot_config.h
+++ b/src/broker/boot_config.h
@@ -20,17 +20,22 @@
  *   tbon.endpoint (w)
  *   instance-level (w)
  */
-int boot_config (flux_t *h, overlay_t *overlay, attr_t *attrs, int tbon_k);
+int boot_config (flux_t *h,
+                 const char *hostname,
+                 struct overlay *overlay,
+                 attr_t *attrs);
 
 /* The following is exported for unit testing.
  */
 #define MAX_URI 2048
 
 struct boot_conf {
+    const char *curve_cert;
     int default_port;
     char default_bind[MAX_URI + 1];
     char default_connect[MAX_URI + 1];
     json_t *hosts;
+    int enable_ipv6;
 };
 
 int boot_config_geturibyrank (json_t *hosts,
@@ -46,7 +51,10 @@ int boot_config_getbindbyrank (json_t *hosts,
 int boot_config_getrankbyname (json_t *hosts,
                                const char *name,
                                uint32_t *rank);
-int boot_config_parse (flux_t *h, struct boot_conf *conf, json_t **hosts);
+int boot_config_parse (const flux_conf_t *cf,
+                       struct boot_conf *conf,
+                       json_t **hosts);
+int boot_config_attr (attr_t *attrs, const char *hostname, json_t *hosts);
 int boot_config_format_uri (char *buf,
                             int bufsz,
                             const char *fmt,
diff --git a/src/broker/boot_pmi.c b/src/broker/boot_pmi.c
index 4a242fc46b32..0ea01f0556dd 100644
--- a/src/broker/boot_pmi.c
+++ b/src/broker/boot_pmi.c
@@ -11,312 +11,538 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <sys/param.h>
+#include <limits.h>
 #include <unistd.h>
-#include <assert.h>
+#include <jansson.h>
+#include <flux/hostlist.h>
+#include <flux/taskmap.h>
 
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/cleanup.h"
 #include "src/common/libutil/ipaddr.h"
-#include "src/common/libutil/kary.h"
-#include "src/common/libpmi/pmi.h"
-#include "src/common/libpmi/pmi_strerror.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libpmi/upmi.h"
+#include "ccan/str/str.h"
 
 #include "attr.h"
 #include "overlay.h"
+#include "topology.h"
 #include "boot_pmi.h"
-#include "pmiutil.h"
 
 
-/* Generally accepted max, although some go higher (IE is 2083) */
-#define ENDPOINT_MAX 2048
+/*  If the broker is launched via flux-shell, then the shell may opt
+ *  to set a "flux.instance-level" parameter in the PMI kvs to tell
+ *  the booting instance at what "level" it will be running, i.e. the
+ *  number of parents. If the PMI key is missing, this is not an error,
+ *  instead the level of this instance is considered to be zero.
+ */
+static int set_instance_level_attr (struct upmi *upmi,
+                                    attr_t *attrs,
+                                    bool *under_flux)
+{
+    char *val = NULL;
+    int rc = -1;
 
-/* Given a string with possible format specifiers, return string that is
- * fully expanded.
- *
- * Possible format specifiers:
- * - %h - IP address of current hostname
- * - %B - value of attribute broker.rundir
- *
- * Caller is responsible for freeing memory of returned value.
+    (void)upmi_get (upmi, "flux.instance-level", -1, &val, NULL);
+    if (attr_add (attrs, "instance-level", val ? val : "0", ATTR_IMMUTABLE) < 0)
+        goto error;
+    *under_flux = val ? true : false;
+    rc = 0;
+error:
+    ERRNO_SAFE_WRAP (free, val);
+    return rc;
+}
+
+/* If the tbon.interface-hint broker attr is not already set, set it.
+ * If running under Flux, use the value, if any, placed in PMI KVS by
+ * the enclsoing instance.  Otherwise, set a default value.
  */
-static char * format_endpoint (attr_t *attrs, const char *endpoint)
+static int set_tbon_interface_hint_attr (struct upmi *upmi,
+                                         attr_t *attrs,
+                                         struct overlay *ov,
+                                         bool under_flux)
+{
+    char *val = NULL;
+    int rc = -1;
+
+    if (attr_get (attrs, "tbon.interface-hint", NULL, NULL) == 0)
+        return 0;
+    if (under_flux)
+        (void)upmi_get (upmi, "flux.tbon-interface-hint", -1, &val, NULL);
+    if (overlay_set_tbon_interface_hint (ov, val) < 0)
+        goto error;
+    rc = 0;
+error:
+    ERRNO_SAFE_WRAP (free, val);
+    return rc;
+}
+
+static char *pmi_mapping_to_taskmap (const char *s)
 {
-    char ipaddr[HOST_NAME_MAX + 1];
-    char *ptr, *buf, *rv = NULL;
-    bool percent_flag = false;
-    unsigned int len = 0;
-    const char *rundir;
-    char error[200];
-
-    if (!(buf = calloc (1, ENDPOINT_MAX + 1))) {
-        errno = ENOMEM;
+    char *result;
+    flux_error_t error;
+    struct taskmap *map = taskmap_decode (s, &error);
+    if (!map) {
+        log_err ("failed to decode PMI_process_mapping: %s",
+                 error.text);
         return NULL;
     }
+    result = taskmap_encode (map, 0);
+    taskmap_destroy (map);
+    return result;
+}
 
-    ptr = (char *)endpoint;
-    while (*ptr) {
-        if (percent_flag) {
-            if (*ptr == 'h') {
-                if (ipaddr_getprimary (ipaddr, sizeof (ipaddr),
-                                       error, sizeof (error)) < 0) {
-                    log_msg ("%s", error);
-                    goto done;
-                }
-                if ((len + strlen (ipaddr)) > ENDPOINT_MAX) {
-                    log_msg ("ipaddr overflow max endpoint length");
-                    goto done;
-                }
-                strcat (buf, ipaddr);
-                len += strlen (ipaddr);
-            }
-            else if (*ptr == 'B') {
-                if (attr_get (attrs, "broker.rundir", &rundir, NULL) < 0) {
-                    log_msg ("broker.rundir attribute is not set");
-                    goto done;
-                }
-                if ((len + strlen (rundir)) > ENDPOINT_MAX) {
-                    log_msg ("broker.rundir overflow max endpoint length");
-                    goto done;
-                }
-                strcat (buf, rundir);
-                len += strlen (rundir);
-            }
-            else if (*ptr == '%')
-                buf[len++] = '%';
-            else {
-                buf[len++] = '%';
-                buf[len++] = *ptr;
-            }
-            percent_flag = false;
-        }
-        else {
-            if (*ptr == '%')
-                percent_flag = true;
-            else
-                buf[len++] = *ptr;
-        }
+/* Set broker.mapping attribute from enclosing instance taskmap.
+ */
+static int set_broker_mapping_attr (struct upmi *upmi,
+                                    int size,
+                                    attr_t *attrs)
+{
+    char *val = NULL;
+    int rc;
 
-        if (len >= ENDPOINT_MAX) {
-            log_msg ("overflow max endpoint length");
-            goto done;
+    if (size == 1)
+        val = strdup ("[[0,1,1,1]]");
+    else {
+        /* First attempt to get flux.taskmap, falling back to
+         * PMI_process_mapping if this key is not available.
+         */
+        char *s;
+        if (upmi_get (upmi, "flux.taskmap", -1, &s, NULL) == 0
+            || upmi_get (upmi, "PMI_process_mapping", -1, &s, NULL) == 0) {
+            val = pmi_mapping_to_taskmap (s);
+            free (s);
         }
-
-        ptr++;
     }
-
-    rv = buf;
-done:
-    if (!rv)
-        free (buf);
-    return (rv);
+    rc = attr_add (attrs, "broker.mapping", val, ATTR_IMMUTABLE);
+    free (val);
+    return rc;
 }
 
-/* Process attribute with format_endpoint(), writing it back to the
- * attribute cache, then returning it in 'value'.
- * If attribute was not initially set, start with 'default_value'.
- * Return 0 on success, -1 on failure with diagnostics to stderr.
+/* Check if IPC can be used to communicate.
+ * Currently this only goes so far as to check if the process mapping of
+ * brokers has all brokers on the same node.  We could check if all peers
+ * are on the same node, but given how the TBON maps to rank assignments,
+ * it is fairly unlikely.
  */
-static int update_endpoint_attr (attr_t *attrs, const char *name,
-                                 const char **value, const char *default_value)
+static bool use_ipc (attr_t *attrs)
 {
+    bool result = false;
+    struct taskmap *map = NULL;
     const char *val;
-    char *fmt_val = NULL;
-    int rc = -1;
 
-    if (attr_get (attrs, name, &val, NULL) < 0)
-        val = default_value;
-    if (!(fmt_val = format_endpoint (attrs, val))) {
-        log_msg ("malformed %s: %s", name, val);
-        return -1;
-    }
-    (void)attr_delete (attrs, name, true);
-    if (attr_add (attrs, name, fmt_val, FLUX_ATTRFLAG_IMMUTABLE) < 0) {
-        log_err ("setattr %s", name);
+    if (attr_get (attrs, "tbon.prefertcp", &val, NULL) == 0
+        && !streq (val, "0"))
         goto done;
-    }
-    if (attr_get (attrs, name, &val, NULL) < 0)
+    if (attr_get (attrs, "broker.mapping", &val, NULL) < 0 || !val)
         goto done;
-    *value = val;
-    rc = 0;
+    if (!(map = taskmap_decode (val, NULL)))
+        goto done;
+    if (taskmap_nnodes (map) == 1)
+        result = true;
 done:
-    free (fmt_val);
-    return rc;
+    taskmap_destroy (map);
+    return result;
 }
 
-/*  If the broker is launched via flux-shell, then the shell may opt
- *  to set a "flux.instance-level" parameter in the PMI kvs to tell
- *  the booting instance at what "level" it will be running, i.e. the
- *  number of parents. If the PMI key is missing, this is not an error,
- *  instead the level of this instance is considered to be zero.
- *  Additonally, if level > 0, the shell will have put the instance's
- *  jobid in the PMI kvsname for us as well, so populate the 'jobid' attr.
+/* Build URI for broker TBON to bind to.
+ * If IPC, use '<rundir>/tbon-<rank>' which should be unique if there are
+ * multiple brokers and/or multiple instances per node.
+ * If using TCP, choose the address to be the one associated with the default
+ * route (see src/common/libutil/ipaddr.h), and a randomly chosen port.
  */
-static int set_instance_level_attr (struct pmi_handle *pmi,
-                                    const char *kvsname,
-                                    attr_t *attrs)
+static int format_bind_uri (char *buf, int bufsz, attr_t *attrs, int rank)
 {
-    int result;
-    char val[32];
-    const char *level = "0";
-    const char *jobid = NULL;
-
-    result = broker_pmi_kvs_get (pmi,
-                                 kvsname,
-                                 "flux.instance-level",
-                                 val,
-                                 sizeof (val));
-    if (result == PMI_SUCCESS)
-        level = val;
-    if (attr_add (attrs, "instance-level", level, FLUX_ATTRFLAG_IMMUTABLE) < 0)
-        return -1;
-    if (result == PMI_SUCCESS)
-        jobid = kvsname;
-    if (attr_add (attrs, "jobid", jobid, FLUX_ATTRFLAG_IMMUTABLE) < 0)
+    if (use_ipc (attrs)) {
+        const char *rundir;
+
+        if (attr_get (attrs, "rundir", &rundir, NULL) < 0) {
+            log_err ("rundir attribute is not set");
+            return -1;
+        }
+        if (snprintf (buf, bufsz, "ipc://%s/tbon-%d", rundir, rank) >= bufsz)
+            goto overflow;
+    }
+    else {
+        char ipaddr[_POSIX_HOST_NAME_MAX + 1];
+        flux_error_t error;
+        int flags = 0;
+        const char *interface = NULL;
+        const char *hint;
+
+        if (attr_get (attrs, "tbon.interface-hint", &hint, NULL) < 0) {
+            log_err ("tbon.interface-hint attribute is not set");
+            return -1;
+        }
+        if (streq (hint, "hostname"))
+            flags |= IPADDR_HOSTNAME;
+        else if (streq (hint, "default-route"))
+            ; // default behavior
+        else
+            interface = hint;
+        if (getenv ("FLUX_IPADDR_V6"))
+            flags |= IPADDR_V6;
+        if (ipaddr_getprimary (ipaddr,
+                               sizeof (ipaddr),
+                               flags,
+                               interface,
+                               &error) < 0) {
+            log_msg ("%s", error.text);
+            return -1;
+        }
+        if (snprintf (buf, bufsz, "tcp://%s:*", ipaddr) >= bufsz)
+            goto overflow;
+    }
+    return 0;
+overflow:
+    log_msg ("buffer overflow while building bind URI");
+    return -1;
+}
+
+static int set_hostlist_attr (attr_t *attrs, struct hostlist *hl)
+{
+    const char *value;
+    char *s;
+    int rc = -1;
+
+    /*  Allow hostlist attribute to be set on command line for testing.
+     *  The value must be re-added if so, so that the IMMUTABLE flag can
+     *   be set so that the attribute is properly cached.
+     */
+    if (attr_get (attrs, "hostlist", &value, NULL) == 0) {
+        s = strdup (value);
+        (void) attr_delete (attrs, "hostlist", true);
+    }
+    else
+        s = hostlist_encode (hl);
+    if (s && attr_add (attrs, "hostlist", s, ATTR_IMMUTABLE) == 0)
+        rc = 0;
+    ERRNO_SAFE_WRAP (free, s);
+    return rc;
+}
+
+static int set_broker_boot_method_attr (attr_t *attrs, const char *value)
+{
+    (void)attr_delete (attrs, "broker.boot-method", true);
+    if (attr_add (attrs,
+                  "broker.boot-method",
+                  value,
+                  ATTR_IMMUTABLE) < 0) {
+        log_err ("setattr broker.boot-method");
         return -1;
+    }
     return 0;
 }
 
-int boot_pmi (overlay_t *overlay, attr_t *attrs, int tbon_k)
+static void trace_upmi (void *arg, const char *text)
 {
-    int parent_rank;
-    const char *child_uri;
+    fprintf (stderr, "boot_pmi: %s\n", text);
+}
+
+int boot_pmi (const char *hostname, struct overlay *overlay, attr_t *attrs)
+{
+    const char *topo_uri;
+    flux_error_t error;
+    json_error_t jerror;
     char key[64];
-    char val[1024];
-    const char *tbonendpoint = NULL;
-    struct pmi_handle *pmi;
-    struct pmi_params pmi_params;
-    int result;
-
-    memset (&pmi_params, 0, sizeof (pmi_params));
-    if (!(pmi = broker_pmi_create ())) {
-        log_err ("broker_pmi_create");
+    char *val;
+    char *bizcard = NULL;
+    struct hostlist *hl = NULL;
+    json_t *o;
+    struct upmi *upmi;
+    struct upmi_info info;
+    struct topology *topo = NULL;
+    int child_count;
+    int *child_ranks = NULL;
+    const char *uri;
+    int i;
+    int upmi_flags = UPMI_LIBPMI_NOFLUX;
+    const char *upmi_method;
+    bool under_flux;
+
+    // N.B. overlay_create() sets the tbon.topo attribute
+    if (attr_get (attrs, "tbon.topo", &topo_uri, NULL) < 0) {
+        log_msg ("error fetching tbon.topo attribute");
+        return -1;
+    }
+    if (attr_get (attrs, "broker.boot-method", &upmi_method, NULL) < 0)
+        upmi_method = NULL;
+    if (getenv ("FLUX_PMI_DEBUG"))
+        upmi_flags |= UPMI_TRACE;
+    if (!(upmi = upmi_create (upmi_method,
+                              upmi_flags,
+                              trace_upmi,
+                              NULL,
+                              &error))) {
+        log_msg ("boot_pmi: %s", error.text);
+        return -1;
+    }
+    if (upmi_initialize (upmi, &info, &error) < 0) {
+        log_msg ("%s: initialize: %s", upmi_describe (upmi), error.text);
+        upmi_destroy (upmi);
+        return -1;
+    }
+    if (set_instance_level_attr (upmi, attrs, &under_flux) < 0) {
+        log_err ("set_instance_level_attr");
         goto error;
     }
-    result = broker_pmi_init (pmi);
-    if (result != PMI_SUCCESS) {
-        log_msg ("broker_pmi_init: %s", pmi_strerror (result));
+    if (under_flux) {
+        if (attr_add (attrs, "jobid", info.name, ATTR_IMMUTABLE) < 0) {
+            log_err ("error setting jobid attribute");
+            goto error;
+        }
+    }
+    if (set_tbon_interface_hint_attr (upmi, attrs, overlay, under_flux) < 0) {
+        log_err ("error setting tbon.interface-hint attribute");
         goto error;
     }
-    result = broker_pmi_get_params (pmi, &pmi_params);
-    if (result != PMI_SUCCESS) {
-        log_msg ("broker_pmi_get_params: %s", pmi_strerror (result));
+    if (set_broker_mapping_attr (upmi, info.size, attrs) < 0) {
+        log_err ("error setting broker.mapping attribute");
         goto error;
     }
-    if (set_instance_level_attr (pmi, pmi_params.kvsname, attrs) < 0) {
-        log_err ("set_instance_level_attr");
+    if (!(topo = topology_create (topo_uri, info.size, &error))) {
+        log_msg ("error creating '%s' topology: %s", topo_uri, error.text);
         goto error;
     }
-    if (overlay_init (overlay, pmi_params.size, pmi_params.rank, tbon_k) < 0)
+    if (topology_set_rank (topo, info.rank) < 0
+        || overlay_set_topology (overlay, topo) < 0)
+        goto error;
+    if (!(hl = hostlist_create ())) {
+        log_err ("hostlist_create");
         goto error;
+    }
 
-    /* If there are to be downstream peers, then bind to socket and share the
-     * concretized URI with other ranks via PMI KVS key=cmbd.<rank>.uri.
-     * N.B. there are no downstream peers if the 0th child of this rank
-     * in k-ary tree does not exist.
+    /* A size=1 instance has no peers, so skip the PMI exchange.
      */
-    if (kary_childof (tbon_k,
-                      pmi_params.size,
-                      pmi_params.rank,
-                      0) != KARY_NONE) {
-
-        if (update_endpoint_attr (attrs,
-                                  "tbon.endpoint",
-                                  &tbonendpoint,
-                                  "tcp://%h:*") < 0) {
-            log_msg ("update_endpoint_attr failed");
+    if (info.size == 1) {
+        if (hostlist_append (hl, hostname) < 0) {
+            log_err ("hostlist_append");
             goto error;
         }
-        if (overlay_set_child (overlay, tbonendpoint) < 0) {
-            log_err ("overlay_set_child");
+        goto done;
+    }
+
+    /* Enable ipv6 for maximum flexibility in address selection.
+     */
+    overlay_set_ipv6 (overlay, 1);
+
+    child_count = topology_get_child_ranks (topo, NULL, 0);
+    if (child_count > 0) {
+        if (!(child_ranks = calloc (child_count, sizeof (child_ranks[0])))
+            || topology_get_child_ranks (topo, child_ranks, child_count) < 0)
             goto error;
-        }
-        if (overlay_bind (overlay) < 0) {
-            log_err ("overlay_bind failed");   /* function is idempotent */
+    }
+
+    /* If there are to be downstream peers, then bind to socket and extract
+     * the concretized URI for sharing with other ranks.
+     */
+    if (child_count > 0) {
+        char buf[1024];
+
+        if (format_bind_uri (buf, sizeof (buf), attrs, info.rank) < 0)
             goto error;
-        }
-        if (!(child_uri = overlay_get_child (overlay))) {
-            log_msg ("overlay_get_child returned NULL");
+        if (overlay_bind (overlay, buf) < 0)
             goto error;
-        }
-        if (snprintf (key, sizeof (key),
-                      "cmbd.%d.uri", pmi_params.rank) >= sizeof (key)) {
+        uri = overlay_get_bind_uri (overlay);
+    }
+    else {
+        uri = NULL;
+    }
+    if (attr_add (attrs, "tbon.endpoint", uri, ATTR_IMMUTABLE) < 0) {
+        log_err ("setattr tbon.endpoint");
+        goto error;
+    }
+
+    /* Each broker writes a "business card" consisting of hostname,
+     * public key, and URI (empty string for leaf node).
+     */
+    if (snprintf (key, sizeof (key), "%d", info.rank) >= sizeof (key)) {
+        log_msg ("pmi key string overflow");
+        goto error;
+    }
+    if (!(o = json_pack ("{s:s s:s s:s}",
+                         "hostname", hostname,
+                         "pubkey", overlay_cert_pubkey (overlay),
+                         "uri", uri ? uri : ""))
+        || !(bizcard = json_dumps (o, JSON_COMPACT))) {
+        log_msg ("error encoding pmi business card object");
+        json_decref (o);
+        goto error;
+    }
+    json_decref (o);
+    if (upmi_put (upmi, key, bizcard, &error) < 0) {
+        log_msg ("%s: put %s: %s", upmi_describe (upmi), key, error.text);
+        goto error;
+    }
+    if (upmi_barrier (upmi, &error) < 0) {
+        log_msg ("%s: barrier: %s", upmi_describe (upmi), error.text);
+        goto error;
+    }
+
+    /* Fetch the business card of parent and inform overlay of URI
+     * and public key.
+     */
+    if (info.rank > 0) {
+        const char *peer_pubkey;
+        const char *peer_uri;
+        int parent_rank = topology_get_parent (topo);
+
+        if (snprintf (key, sizeof (key), "%d", parent_rank) >= sizeof (key)) {
             log_msg ("pmi key string overflow");
             goto error;
         }
-        if (snprintf (val, sizeof (val), "%s", child_uri) >= sizeof (val)) {
-            log_msg ("pmi val string overflow");
+        if (upmi_get (upmi, key, parent_rank, &val, &error) < 0) {
+            log_msg ("%s: get %s: %s", upmi_describe (upmi), key, error.text);
             goto error;
         }
-        result = broker_pmi_kvs_put (pmi, pmi_params.kvsname, key, val);
-        if (result != PMI_SUCCESS) {
-            log_msg ("broker_pmi_kvs_put: %s", pmi_strerror (result));
+        if (!(o = json_loads (val, 0, &jerror))
+            || json_unpack_ex (o,
+                               &jerror,
+                               0,
+                               "{s:s s:s}",
+                               "pubkey", &peer_pubkey,
+                               "uri", &peer_uri) < 0) {
+            log_msg ("error decoding rank %d business card: %s",
+                     parent_rank,
+                     jerror.text);
+            json_decref (o);
+            free (val);
             goto error;
         }
-        result = broker_pmi_kvs_commit (pmi, pmi_params.kvsname);
-        if (result != PMI_SUCCESS) {
-            log_msg ("broker_pmi_kvs_commit: %s", pmi_strerror (result));
+        if (strlen (peer_uri) == 0) {
+            log_msg ("error decoding rank %d business card", parent_rank);
+            json_decref (o);
+            free (val);
             goto error;
         }
-    }
-    else {
-        (void)attr_delete (attrs, "tbon.endpoint", true);
-        if (attr_add (attrs,
-                     "tbon.endpoint",
-                     NULL,
-                     FLUX_ATTRFLAG_IMMUTABLE) < 0) {
-            log_err ("setattr tbon.endpoint");
+        if (overlay_set_parent_uri (overlay, peer_uri) < 0) {
+            log_err ("overlay_set_parent_uri");
+            json_decref (o);
+            free (val);
             goto error;
         }
+        if (overlay_set_parent_pubkey (overlay, peer_pubkey) < 0) {
+            log_err ("overlay_set_parent_pubkey");
+            json_decref (o);
+            free (val);
+            goto error;
+        }
+        json_decref (o);
+        free (val);
     }
 
-    /* The PMI barrier (which is implicitly over 'size' ranks) ensures that
-     * all KVS puts are complete before any PMI gets.
+    /* Fetch the business card of children and inform overlay of public keys.
      */
-    result = broker_pmi_barrier (pmi);
-    if (result != PMI_SUCCESS) {
-        log_msg ("broker_pmi_barrier: %s", pmi_strerror (result));
-        goto error;
+    for (i = 0; i < child_count; i++) {
+        const char *peer_pubkey;
+        int child_rank = child_ranks[i];
+
+        if (snprintf (key, sizeof (key), "%d", child_rank) >= sizeof (key)) {
+            log_msg ("pmi key string overflow");
+            goto error;
+        }
+        if (upmi_get (upmi, key, child_rank, &val, &error) < 0) {
+            log_msg ("%s: get %s: %s", upmi_describe (upmi), key, error.text);
+            goto error;
+        }
+        if (!(o = json_loads (val, 0, &jerror))
+            || json_unpack_ex (o,
+                               &jerror,
+                               0,
+                               "{s:s}",
+                               "pubkey", &peer_pubkey) < 0) {
+            log_msg ("error decoding rank %d business card: %s",
+                     child_rank,
+                     jerror.text);
+            json_decref (o);
+            free (val);
+            goto error;
+        }
+        if (overlay_authorize (overlay, key, peer_pubkey) < 0) {
+            log_err ("overlay_authorize %s=%s", key, peer_pubkey);
+            json_decref (o);
+            free (val);
+            goto error;
+        }
+        json_decref (o);
+        free (val);
     }
 
-    /* If there is to be an upstream peer, fetch its URI from PMI KVS.
-     * N.B. only rank 0 has no upstream peer.
+    /* Fetch the business card of all ranks and build hostlist.
+     * The hostlist is built independently (and in parallel) on all ranks.
      */
-    if (pmi_params.rank > 0) {
-        parent_rank = kary_parentof (tbon_k, pmi_params.rank);
-        if (snprintf (key, sizeof (key),
-                      "cmbd.%d.uri", parent_rank) >= sizeof (key)) {
+    for (i = 0; i < info.size; i++) {
+        const char *peer_hostname;
+
+        if (snprintf (key, sizeof (key), "%d", i) >= sizeof (key)) {
             log_msg ("pmi key string overflow");
             goto error;
         }
-        result = broker_pmi_kvs_get (pmi, pmi_params.kvsname,
-                                     key, val, sizeof (val));
-        if (result != PMI_SUCCESS) {
-            log_msg ("broker_pmi_kvs_get: %s", pmi_strerror (result));
+        if (upmi_get (upmi, key, i, &val, &error) < 0) {
+            log_msg ("%s: get %s: %s", upmi_describe (upmi), key, error.text);
             goto error;
         }
-        if (overlay_set_parent (overlay, "%s", val) < 0) {
-            log_err ("overlay_set_parent");
+        if (!(o = json_loads (val, 0, &jerror))
+            || json_unpack_ex (o,
+                               &jerror,
+                               0,
+                               "{s:s}",
+                               "hostname", &peer_hostname) < 0) {
+            log_msg ("error decoding rank %d pmi business card: %s",
+                     i,
+                     error.text);
+            json_decref (o);
+            free (val);
             goto error;
         }
+        if (hostlist_append (hl, peer_hostname) < 0) {
+            log_err ("hostlist_append");
+            json_decref (o);
+            free (val);
+            goto error;
+        }
+        json_decref (o);
+        free (val);
     }
 
-    result = broker_pmi_barrier (pmi);
-    if (result != PMI_SUCCESS) {
-        log_msg ("broker_pmi_barrier: %s", pmi_strerror (result));
+    /* One more barrier before allowing connects to commence.
+     * Need to ensure that all clients are "allowed".
+     */
+    if (upmi_barrier (upmi, &error) < 0) {
+        log_msg ("%s: barrier: %s", upmi_describe (upmi), error.text);
         goto error;
     }
 
-    result = broker_pmi_finalize (pmi);
-    if (result != PMI_SUCCESS) {
-        log_msg ("broker_pmi_finalize: %s", pmi_strerror (result));
+done:
+    if (set_hostlist_attr (attrs, hl) < 0) {
+        log_err ("setattr hostlist");
         goto error;
     }
-
-    broker_pmi_destroy (pmi);
+    if (set_broker_boot_method_attr (attrs, upmi_describe (upmi)) < 0)
+        goto error;
+    if (upmi_finalize (upmi, &error) < 0) {
+        log_msg ("%s: finalize: %s", upmi_describe (upmi), error.text);
+        goto error;
+    }
+    free (bizcard);
+    upmi_destroy (upmi);
+    hostlist_destroy (hl);
+    free (child_ranks);
+    topology_decref (topo);
     return 0;
 error:
-    broker_pmi_destroy (pmi);
+    /* We've logged error to stderr before getting here so the fatal
+     * error message passed to the PMI server does not necessarily need
+     * to be highly detailed.  Some implementations of abort may not
+     * return.
+     */
+    if (upmi_abort (upmi, "fatal bootstrap error", &error) < 0)
+        log_msg ("upmi_abort: %s", error.text);
+    free (bizcard);
+    upmi_destroy (upmi);
+    hostlist_destroy (hl);
+    free (child_ranks);
+    topology_decref (topo);
     return -1;
 }
 
diff --git a/src/broker/boot_pmi.h b/src/broker/boot_pmi.h
index 3b241982c8c7..cd1174d5be27 100644
--- a/src/broker/boot_pmi.h
+++ b/src/broker/boot_pmi.h
@@ -16,7 +16,7 @@
 #include "attr.h"
 #include "overlay.h"
 
-int boot_pmi (overlay_t *overlay, attr_t *attrs, int tbon_k);
+int boot_pmi (const char *hostname, struct overlay *overlay, attr_t *attrs);
 
 #endif /* BROKER_BOOT_PMI_H */
 
diff --git a/src/broker/broker.c b/src/broker/broker.c
index 95bb4accd48a..a401f38bde62 100644
--- a/src/broker/broker.c
+++ b/src/broker/broker.c
@@ -11,31 +11,26 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <stdio.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <getopt.h>
 #include <libgen.h>
-#include <sys/types.h>
+#include <signal.h>
+#include <locale.h>
 #include <inttypes.h>
-#include <sys/stat.h>
+#ifdef HAVE_SYS_PRCTL_H
 #include <sys/prctl.h>
-#include <sys/signalfd.h>
+#endif
 #include <sys/resource.h>
+#include <fcntl.h>
 #include <unistd.h>
-#include <sys/param.h>
-#include <stdbool.h>
-#include <dlfcn.h>
+#include <sys/un.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
 #include <flux/core.h>
-#include <czmq.h>
 #include <jansson.h>
-#if HAVE_CALIPER
-#include <caliper/cali.h>
-#include <sys/syscall.h>
-#endif
 #if HAVE_VALGRIND
 # if HAVE_VALGRIND_H
 #  include <valgrind.h>
@@ -43,242 +38,134 @@
 #  include <valgrind/valgrind.h>
 # endif
 #endif
+#include <flux/taskmap.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
-#include "src/common/libutil/oom.h"
-#include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/cleanup.h"
 #include "src/common/libidset/idset.h"
 #include "src/common/libutil/ipaddr.h"
-#include "src/common/libutil/kary.h"
-#include "src/common/libutil/monotime.h"
-#include "src/common/libutil/zsecurity.h"
-#include "src/common/libpmi/pmi.h"
-#include "src/common/libpmi/pmi_strerror.h"
 #include "src/common/libutil/fsd.h"
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/intree.h"
+#include "src/common/libutil/basename.h"
+#include "src/common/librouter/subhash.h"
+#include "src/common/libfluxutil/method.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "ccan/ptrint/ptrint.h"
+#ifndef HAVE_STRLCPY
+#include "src/common/libmissing/strlcpy.h"
+#endif
+#ifndef HAVE_STRLCAT
+#include "src/common/libmissing/strlcat.h"
+#endif
 
-#include "heartbeat.h"
 #include "module.h"
+#include "modhash.h"
+#include "brokercfg.h"
+#include "groups.h"
 #include "overlay.h"
 #include "service.h"
-#include "hello.h"
-#include "shutdown.h"
 #include "attr.h"
 #include "log.h"
-#include "content-cache.h"
-#include "runlevel.h"
+#include "runat.h"
 #include "heaptrace.h"
 #include "exec.h"
-#include "ping.h"
-#include "rusage.h"
 #include "boot_config.h"
 #include "boot_pmi.h"
 #include "publisher.h"
+#include "state_machine.h"
+#include "shutdown.h"
 
-typedef struct {
-    /* Reactor
-     */
-    flux_t *h;
-    flux_reactor_t *reactor;
+#include "broker.h"
 
-    /* Sockets.
-     */
-    overlay_t *overlay;
 
-    /* Session parameters
-     */
-    attr_t *attrs;
-    struct flux_msg_cred cred;  /* instance owner */
+static int broker_request_sendmsg_new_internal (broker_ctx_t *ctx,
+                                                flux_msg_t **msg);
 
-    /* Modules
-     */
-    modhash_t *modhash;
-    /* Misc
-     */
-    bool verbose;
-    int event_recv_seq;
-    zlist_t *sigwatchers;
-    struct service_switch *services;
-    heartbeat_t *heartbeat;
-    struct shutdown *shutdown;
-    double shutdown_grace;
-    double heartbeat_rate;
-    int sec_typemask;
-    zlist_t *subscriptions;     /* subscripts for internal services */
-    content_cache_t *cache;
-    struct publisher *publisher;
-    int tbon_k;
-    /* Bootstrap
-     */
-    hello_t *hello;
-    struct runlevel *runlevel;
-
-    char *init_shell_cmd;
-    size_t init_shell_cmd_len;
-} broker_ctx_t;
-
-static int broker_event_sendmsg (broker_ctx_t *ctx, const flux_msg_t *msg);
-static int broker_response_sendmsg (broker_ctx_t *ctx, const flux_msg_t *msg);
-static void broker_request_sendmsg (broker_ctx_t *ctx, const flux_msg_t *msg);
-static int broker_request_sendmsg_internal (broker_ctx_t *ctx,
-                                            const flux_msg_t *msg);
-
-static void parent_cb (overlay_t *ov, void *sock, void *arg);
-static void child_cb (overlay_t *ov, void *sock, void *arg);
-static void module_cb (module_t *p, void *arg);
-static void module_status_cb (module_t *p, int prev_state, void *arg);
-static void hello_update_cb (hello_t *h, void *arg);
-static void shutdown_cb (struct shutdown *s, void *arg);
-static void signal_cb (flux_reactor_t *r, flux_watcher_t *w,
-                       int revents, void *arg);
+static void h_internal_watcher (flux_reactor_t *r,
+                                flux_watcher_t *w,
+                                int revents,
+                                void *arg);
+
+static int overlay_recv_cb (flux_msg_t **msg, overlay_where_t where, void *arg);
+
+static void signal_cb (flux_reactor_t *r,
+                       flux_watcher_t *w,
+                       int revents,
+                       void *arg);
 static int broker_handle_signals (broker_ctx_t *ctx);
 
 static flux_msg_handler_t **broker_add_services (broker_ctx_t *ctx);
 static void broker_remove_services (flux_msg_handler_t *handlers[]);
 
-static int load_module_byname (broker_ctx_t *ctx, const char *name,
-                               const char *argz, size_t argz_len,
-                               const flux_msg_t *request);
-static int unload_module_byname (broker_ctx_t *ctx, const char *name,
-                                 const flux_msg_t *request);
-
 static void set_proctitle (uint32_t rank);
-static void runlevel_cb (struct runlevel *r,
-                         int level,
-                         int rc,
-                         double elapsed,
-                         const char *state,
-                         void *arg);
-static void runlevel_io_cb (struct runlevel *r,
-                            const char *name,
-                            const char *msg,
-                            void *arg);
 
 static int create_rundir (attr_t *attrs);
-static int create_broker_rundir (overlay_t *ov, void *arg);
-static int create_dummyattrs (flux_t *h, uint32_t rank, uint32_t size);
+static int check_statedir (attr_t *attrs);
+
+static int create_runat_phases (broker_ctx_t *ctx);
 
 static int handle_event (broker_ctx_t *ctx, const flux_msg_t *msg);
 
-static void init_attrs (attr_t *attrs, pid_t pid);
+static void init_attrs (attr_t *attrs, pid_t pid, struct flux_msg_cred *cred);
 
-static const struct flux_handle_ops broker_handle_ops;
+static void init_attrs_starttime (attr_t *attrs, double starttime);
 
-static int parse_config_files (flux_t *h);
+static int init_local_uri_attr (struct overlay *ov, attr_t *attrs);
 
-static int exit_rc = 1;
+static int init_critical_ranks_attr (struct overlay *ov, attr_t *attrs);
 
-#define OPTIONS "+vM:X:k:s:g:EIS:"
-static const struct option longopts[] = {
-    {"verbose",         no_argument,        0, 'v'},
-    {"security",        required_argument,  0, 's'},
-    {"module-path",     required_argument,  0, 'X'},
-    {"k-ary",           required_argument,  0, 'k'},
-    {"heartrate",       required_argument,  0, 'H'},
-    {"shutdown-grace",  required_argument,  0, 'g'},
-    {"setattr",         required_argument,  0, 'S'},
-    {0, 0, 0, 0},
-};
+static int execute_parental_notifications (struct broker *ctx);
 
-static void usage (void)
-{
-    fprintf (stderr,
-"Usage: flux-broker OPTIONS [initial-command ...]\n"
-" -v,--verbose                 Be annoyingly verbose\n"
-" -X,--module-path PATH        Set module search path (colon separated)\n"
-" -s,--security=plain|curve|none    Select security mode (default: curve)\n"
-" -k,--k-ary K                 Wire up in a k-ary tree\n"
-" -H,--heartrate SECS          Set heartrate in seconds (rank 0 only)\n"
-" -g,--shutdown-grace SECS     Set shutdown grace period in seconds\n"
-" -S,--setattr ATTR=VAL        Set broker attribute\n"
-);
-    exit (1);
-}
+static struct optparse_option opts[] = {
+    { .name = "verbose",    .key = 'v', .has_arg = 2, .arginfo = "[LEVEL]",
+      .usage = "Be annoyingly informative by degrees", },
+    { .name = "setattr",    .key = 'S', .has_arg = 1, .arginfo = "ATTR=VAL",
+      .usage = "Set broker attribute", },
+    { .name = "config-path",.key = 'c', .has_arg = 1, .arginfo = "PATH",
+      .usage = "Set broker config from PATH (default: none)", },
+    OPTPARSE_TABLE_END,
+};
 
 void parse_command_line_arguments (int argc, char *argv[], broker_ctx_t *ctx)
 {
-    int c;
-    int e;
-    char *endptr;
-
-    while ((c = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
-        switch (c) {
-        case 's':   /* --security=MODE */
-            if (!strcmp (optarg, "none")) {
-                ctx->sec_typemask = 0;
-            } else if (!strcmp (optarg, "plain")) {
-                ctx->sec_typemask |= ZSECURITY_TYPE_PLAIN;
-                ctx->sec_typemask &= ~ZSECURITY_TYPE_CURVE;
-            } else if (!strcmp (optarg, "curve")) {
-                ctx->sec_typemask |= ZSECURITY_TYPE_CURVE;
-                ctx->sec_typemask &= ~ZSECURITY_TYPE_PLAIN;
-            } else {
-                log_msg_exit ("--security arg must be none|plain|curve");
-            }
-            break;
-        case 'v':   /* --verbose */
-            ctx->verbose = true;
-            break;
-        case 'X':   /* --module-path PATH */
-            if (attr_set (ctx->attrs, "conf.module_path", optarg, true) < 0)
-                log_err_exit ("setting conf.module_path attribute");
-            break;
-        case 'k':   /* --k-ary k */
-            errno = 0;
-            ctx->tbon_k = strtoul (optarg, &endptr, 10);
-            if (errno || *endptr != '\0')
-                log_err_exit ("k-ary '%s'", optarg);
-            if (ctx->tbon_k < 1)
-                usage ();
-            break;
-        case 'H':   /* --heartrate SECS */
-            if (fsd_parse_duration (optarg, &ctx->heartbeat_rate) < 0)
-                log_err_exit ("heartrate '%s'", optarg);
-            break;
-        case 'g':   /* --shutdown-grace SECS */
-            if (fsd_parse_duration (optarg, &ctx->shutdown_grace) < 0) {
-                log_err_exit ("shutdown-grace '%s'", optarg);
-                usage ();
-            }
-            break;
-        case 'S': { /* --setattr ATTR=VAL */
-            char *val, *attr = xstrdup (optarg);
-            if ((val = strchr (attr, '=')))
-                *val++ = '\0';
-            if (attr_add (ctx->attrs, attr, val, 0) < 0)
-                if (attr_set (ctx->attrs, attr, val, true) < 0)
-                    log_err_exit ("setattr %s=%s", attr, val);
-            free (attr);
-            break;
-        }
-        default:
-            usage ();
-        }
-    }
-    if (optind < argc) {
-        if ((e = argz_create (argv + optind, &ctx->init_shell_cmd,
+    int optindex;
+    const char *arg;
+
+    if (!(ctx->opts = optparse_create ("flux-broker"))
+        || optparse_add_option_table (ctx->opts, opts) != OPTPARSE_SUCCESS)
+        log_msg_exit ("error setting up option parsing");
+    if ((optindex = optparse_parse_args (ctx->opts, argc, argv)) < 0)
+        exit (1);
+
+    ctx->verbose = optparse_get_int (ctx->opts, "verbose", 0);
+
+    optparse_get_str (ctx->opts, "config-path", NULL);
+
+    while ((arg = optparse_getopt_next (ctx->opts, "setattr"))) {
+        char *val, *attr;
+        if (!(attr = strdup (arg)))
+            log_err_exit ("out of memory duplicating optarg");
+        if ((val = strchr (attr, '=')))
+            *val++ = '\0';
+        if (attr_add (ctx->attrs, attr, val, 0) < 0)
+            if (attr_set (ctx->attrs, attr, val) < 0)
+                log_err_exit ("setattr %s=%s", attr, val);
+        free (attr);
+    }
+
+    if (optindex < argc) {
+        int e;
+        if ((e = argz_create (argv + optindex,
+                              &ctx->init_shell_cmd,
                               &ctx->init_shell_cmd_len)) != 0)
             log_errn_exit (e, "argz_create");
     }
 }
 
-static int setup_profiling (const char *program, int rank)
-{
-#if HAVE_CALIPER
-    cali_begin_string_byname ("flux.type", "main");
-    cali_begin_int_byname ("flux.tid", syscall (SYS_gettid));
-    cali_begin_string_byname ("binary", program);
-    cali_begin_int_byname ("flux.rank", rank);
-    // TODO: this is a stopgap until we have better control over
-    // instrumemtation in child processes. If we want to see what children
-    // that load libflux are up to, this should be disabled
-    unsetenv ("CALI_SERVICES_ENABLE");
-    unsetenv ("CALI_CONFIG_PROFILE");
-#endif
-    return (0);
-}
-
 static int increase_rlimits (void)
 {
     struct rlimit rlim;
@@ -305,94 +192,117 @@ int main (int argc, char *argv[])
     struct sigaction old_sigact_int;
     struct sigaction old_sigact_term;
     flux_msg_handler_t **handlers = NULL;
-    const char *boot_method;
+    const flux_conf_t *conf;
+    const char *method;
+    flux_error_t error;
+
+    setlocale (LC_ALL, "");
 
     memset (&ctx, 0, sizeof (ctx));
     log_init (argv[0]);
 
-    if (!(ctx.sigwatchers = zlist_new ()))
-        oom ();
-    if (!(ctx.modhash = modhash_create ()))
-        oom ();
-    if (!(ctx.services = service_switch_create ()))
-        oom ();
-    if (!(ctx.overlay = overlay_create ()))
-        oom ();
-    if (!(ctx.hello = hello_create ()))
-        oom ();
-    if (!(ctx.heartbeat = heartbeat_create ()))
-        oom ();
-    if (!(ctx.attrs = attr_create ()))
-        oom ();
-    if (!(ctx.subscriptions = zlist_new ()))
-        oom ();
-    if (!(ctx.cache = content_cache_create ()))
-        oom ();
-    if (!(ctx.publisher = publisher_create ()))
-        oom ();
-
-    ctx.tbon_k = 2; /* binary TBON is default */
+    ctx.exit_rc = 1;
+
+    if (!(ctx.sigwatchers = zlist_new ())
+        || !(ctx.services = service_switch_create ())
+        || !(ctx.attrs = attr_create ())
+        || !(ctx.sub = subhash_create ()))
+        log_msg_exit ("Out of memory in early initialization");
+
     /* Record the instance owner: the effective uid of the broker. */
-    ctx.cred.userid = geteuid ();
+    ctx.cred.userid = getuid ();
     /* Set default rolemask for messages sent with flux_send()
      * on the broker's internal handle. */
-    ctx.cred.rolemask = FLUX_ROLE_OWNER;
-    ctx.heartbeat_rate = 2;
-    ctx.sec_typemask = ZSECURITY_TYPE_CURVE;
+    ctx.cred.rolemask = FLUX_ROLE_OWNER | FLUX_ROLE_LOCAL;
 
-    init_attrs (ctx.attrs, getpid ());
+    init_attrs (ctx.attrs, getpid (), &ctx.cred);
+
+    const char *hostname = getenv ("FLUX_FAKE_HOSTNAME");
+    if (hostname)
+        strlcpy (ctx.hostname, hostname, sizeof (ctx.hostname));
+    else if (gethostname (ctx.hostname, sizeof (ctx.hostname)) < 0)
+        log_err_exit ("gethostname");
 
     parse_command_line_arguments (argc, argv, &ctx);
 
-    /* Block all signals, saving old mask and actions for SIGINT, SIGTERM.
+    /* Block all signals but those that we want to generate core dumps.
+     * Save old mask and actions for SIGINT, SIGTERM.
      */
     sigset_t sigmask;
     sigfillset (&sigmask);
-    if (sigprocmask (SIG_SETMASK, &sigmask, &old_sigmask) < 0)
-        log_err_exit ("sigprocmask");
-    if (sigaction (SIGINT, NULL, &old_sigact_int) < 0)
-        log_err_exit ("sigaction");
-    if (sigaction (SIGTERM, NULL, &old_sigact_term) < 0)
-        log_err_exit ("sigaction");
-
-    /* Initailize zeromq context
+    sigdelset (&sigmask, SIGSEGV);
+    sigdelset (&sigmask, SIGFPE);
+    sigdelset (&sigmask, SIGILL);
+    sigdelset (&sigmask, SIGABRT);
+    sigdelset (&sigmask, SIGFPE);
+    sigdelset (&sigmask, SIGSYS);
+    sigdelset (&sigmask, SIGTRAP);
+    sigdelset (&sigmask, SIGXCPU);
+    sigdelset (&sigmask, SIGXFSZ);
+    if (sigprocmask (SIG_SETMASK, &sigmask, &old_sigmask) < 0
+        || sigaction (SIGINT, NULL, &old_sigact_int) < 0
+        || sigaction (SIGTERM, NULL, &old_sigact_term) < 0)
+        log_err_exit ("error setting signal mask");
+
+    /* Set up two interthread flux_t handles, connected back to back.
+     * ctx.h is used conventionally within the broker for RPCs, message
+     * handlers, etc.  ctx.h_internal belongs to the broker's routing logic
+     * and is accessed using flux_send() and flux_recv() only.  Both handles
+     * share a reactor, which is created with FLUX_REACTOR_SIGCHLD in order
+     * to support libsubprocess.
+     *
+     * N.B. since both handles are in the same thread, synchronous RPCs on
+     * ctx.h will deadlock.  The main broker reactor must run in order
+     * to move messages from the interthread queue to the routing logic.
+     * Careful with flux_attr_get(), which hides a synchronous RPC if the
+     * requested value is not cached.
      */
-    if (!zsys_init ()) {
-        log_err ("zsys_init");
+    if (!(ctx.reactor = flux_reactor_create (FLUX_REACTOR_SIGCHLD))
+        || !(ctx.h = flux_open ("interthread://broker", 0))
+        || flux_set_reactor (ctx.h, ctx.reactor) < 0
+        || !(ctx.h_internal = flux_open ("interthread://broker", 0))
+        || flux_set_reactor (ctx.h_internal, ctx.reactor) < 0
+        || !(ctx.w_internal = flux_handle_watcher_create (ctx.reactor,
+                                                          ctx.h_internal,
+                                                          FLUX_POLLIN,
+                                                          h_internal_watcher,
+                                                          &ctx))) {
+        log_err ("error setting up broker reactor/flux_t handle");
         goto cleanup;
     }
-    zsys_set_logstream (stderr);
-    zsys_set_logident ("flux-broker");
-    zsys_handler_set (NULL);
-    zsys_set_linger (5);
-    zsys_set_rcvhwm (0);
-    zsys_set_sndhwm (0);
+    flux_watcher_start (ctx.w_internal);
 
-    /* Set up the flux reactor.
-     */
-    if (!(ctx.reactor = flux_reactor_create (FLUX_REACTOR_SIGCHLD))) {
-        log_err ("flux_reactor_create");
+    const char *val;
+    if (attr_get (ctx.attrs, "broker.sd-notify", &val, NULL) == 0
+        && !streq (val, "0")) {
+#if !HAVE_LIBSYSTEMD
+        log_err ("broker.sd_notify is set but Flux was not built"
+                 " with systemd support.");
         goto cleanup;
+#else
+        ctx.sd_notify = true;
+#endif
     }
 
-    /* Set up flux handle.
-     * The handle is used for simple purposes such as logging.
+    /* Initialize module infrastructure.
      */
-    if (!(ctx.h = flux_handle_create (&ctx, &broker_handle_ops, 0))) {
-        log_err ("flux_handle_create");
-        goto cleanup;
-    }
-    if (flux_set_reactor (ctx.h, ctx.reactor) < 0) {
-        log_err ("flux_set_reactor");
+    if (!(ctx.modhash = modhash_create (&ctx))) {
+        log_err ("error creating broker module hash");
         goto cleanup;
     }
 
-    if (increase_rlimits () < 0)
+    /* Parse config.
+     */
+    if (!(ctx.config = brokercfg_create (ctx.h,
+                                         optparse_get_str (ctx.opts,
+                                                           "config-path",
+                                                           NULL),
+                                         ctx.attrs,
+                                         ctx.modhash)))
         goto cleanup;
+    conf = flux_get_conf (ctx.h);
 
-    /* Parse config file(s).  The result is cached in ctx.h.
-     */
-    if (parse_config_files (ctx.h) < 0)
+    if (increase_rlimits () < 0)
         goto cleanup;
 
     /* Prepare signal handling
@@ -402,230 +312,118 @@ int main (int argc, char *argv[])
         goto cleanup;
     }
 
-    /* The first call to overlay_bind() or overlay_connect() calls
-     * zsecurity_comms_init().  Delay calling zsecurity_comms_init()
-     * so that we can defer creating the libzmq work thread until we
-     * are ready to communicate.
-     */
-    const char *keydir;
-    if (attr_get (ctx.attrs, "security.keydir", &keydir, NULL) < 0) {
-        log_err ("getattr security.keydir");
+    if (!(ctx.overlay = overlay_create (ctx.h,
+                                        ctx.hostname,
+                                        ctx.attrs,
+                                        NULL,
+                                        overlay_recv_cb,
+                                        &ctx))) {
+        log_err ("overlay_create");
         goto cleanup;
     }
-    if (overlay_set_flux (ctx.overlay, ctx.h) < 0) {
-        log_err ("overlay_set_flux");
-        goto cleanup;
-    }
-    if (overlay_setup_sec (ctx.overlay, ctx.sec_typemask, keydir) < 0) {
-        log_err ("overlay_setup_sec");
-        goto cleanup;
-    }
-
-    overlay_set_parent_cb (ctx.overlay, parent_cb, &ctx);
-    overlay_set_child_cb (ctx.overlay, child_cb, &ctx);
 
     /* Arrange for the publisher to route event messages.
-     * handle_event - local subscribers (ctx.h)
      */
-    if (publisher_set_flux (ctx.publisher, ctx.h) < 0) {
-        log_err ("publisher_set_flux");
-        goto cleanup;
-    }
-    if (publisher_set_sender (ctx.publisher, "handle_event",
-                              (publisher_send_f)handle_event, &ctx) < 0) {
-        log_err ("publisher_set_sender");
+    if (!(ctx.publisher = publisher_create (&ctx,
+                                            (publisher_send_f)handle_event,
+                                            &ctx))) {
+        log_err ("error setting up event publishing service");
         goto cleanup;
     }
 
-    if (create_rundir (ctx.attrs) < 0) {
-        log_err ("create_rundir");
+    if (create_rundir (ctx.attrs) < 0)
+        goto cleanup;
+    if (check_statedir (ctx.attrs) < 0)
         goto cleanup;
-    }
 
-    /* Set & create broker.rundir *after* overlay initialization,
-     * when broker rank is determined.
+    /* Record the broker start time.  This time will also be used to
+     * capture how long network bootstrap takes.
      */
-    overlay_set_init_callback (ctx.overlay, create_broker_rundir, ctx.attrs);
+    flux_reactor_now_update (ctx.reactor);
+    ctx.starttime = flux_reactor_now (ctx.reactor);
+    init_attrs_starttime (ctx.attrs, ctx.starttime);
 
-    /* Execute boot method selected by 'boot.method' attr.
-     * Default is pmi.
+    /* Execute broker network bootstrap.
+     * Default method is pmi.
+     * If [bootstrap] is defined in configuration, use static configuration.
      */
-    if (attr_get (ctx.attrs, "boot.method", &boot_method, NULL) < 0) {
-        boot_method = "pmi";
-        if (attr_add (ctx.attrs, "boot.method", boot_method, 0)) {
-            log_err ("setattr boot.method");
-            goto cleanup;
-        }
-    }
-    if (attr_set_flags (ctx.attrs,
-                        "boot.method",
-                        FLUX_ATTRFLAG_IMMUTABLE) < 0) {
-        log_err ("attr_set_flags boot.method");
-        goto cleanup;
+    if (attr_get (ctx.attrs, "broker.boot-method", &method, NULL) < 0) {
+        if (flux_conf_unpack (conf, NULL, "{s:{}}", "bootstrap") == 0)
+            method = "config";
+        else
+            method = NULL;
     }
-    if (!strcmp (boot_method, "config")) {
-        if (boot_config (ctx.h, ctx.overlay, ctx.attrs, ctx.tbon_k) < 0) {
+    if (!method || !streq (method, "config")) {
+        if (boot_pmi (ctx.hostname, ctx.overlay, ctx.attrs) < 0) {
             log_msg ("bootstrap failed");
             goto cleanup;
         }
     }
-    else if (!strcmp (boot_method, "pmi")) {
-        double elapsed_sec;
-        struct timespec start_time;
-        monotime (&start_time);
-        if (boot_pmi (ctx.overlay, ctx.attrs, ctx.tbon_k) < 0) {
+    else {
+        if (boot_config (ctx.h, ctx.hostname, ctx.overlay, ctx.attrs) < 0) {
             log_msg ("bootstrap failed");
             goto cleanup;
         }
-        elapsed_sec = monotime_since (start_time) / 1000;
-        flux_log (ctx.h, LOG_INFO, "pmi: bootstrap time %.1fs", elapsed_sec);
-
-    }
-    else {
-        log_err ("unknown boot method: %s", boot_method);
-        goto cleanup;
     }
-    uint32_t rank = overlay_get_rank (ctx.overlay);
-    uint32_t size = overlay_get_size (ctx.overlay);
-    char rank_str[16];
-    snprintf (rank_str, sizeof (rank_str), "%"PRIu32, rank);
 
-    assert (size > 0);
+    ctx.rank = overlay_get_rank (ctx.overlay);
+    ctx.size = overlay_get_size (ctx.overlay);
+
+    if (ctx.size == 0)
+        log_err_exit ("internal error: instance size is zero!");
 
     /* Must be called after overlay setup */
-    if (overlay_register_attrs (ctx.overlay, ctx.attrs) < 0) {
+    if (overlay_register_attrs (ctx.overlay) < 0) {
         log_err ("registering overlay attributes");
         goto cleanup;
     }
 
-    if (ctx.verbose)
-        log_msg ("boot: rank=%d size=%d", rank, size);
-
-    // Setup profiling
-    setup_profiling (argv[0], rank);
+    if (ctx.verbose) {
+        flux_reactor_now_update (ctx.reactor);
+        log_msg ("boot: rank=%d size=%d time %.3fs",
+                  ctx.rank,
+                  ctx.size,
+                  flux_reactor_now (ctx.reactor) - ctx.starttime);
+    }
 
     /* Initialize logging.
      * OK to call flux_log*() after this.
      */
-    logbuf_initialize (ctx.h, rank, ctx.attrs);
+    logbuf_initialize (ctx.h, ctx.rank, ctx.attrs);
 
-    /* Allow flux_get_rank() and flux_get_size() to work in the broker.
+    /* Allow flux_get_rank(), flux_get_size(), flux_get_hostybyrank(), etc.
+     * to work in the broker without causing a synchronous RPC to self that
+     * would deadlock.
      */
-    if (create_dummyattrs (ctx.h, rank, size) < 0) {
-        log_err ("creating dummy attributes");
+    if (attr_cache_immutables (ctx.attrs, ctx.h) < 0) {
+        log_err ("error priming broker attribute cache");
         goto cleanup;
     }
 
-    /* Registers message handlers and obtains rank.
-     */
-    if (content_cache_set_flux (ctx.cache, ctx.h) < 0) {
-        log_err ("content_cache_set_flux");
-        goto cleanup;
-    }
-    if (content_cache_register_attrs (ctx.cache, ctx.attrs) < 0) {
-        log_err ("content cache attributes");
+    if (!(ctx.groups = groups_create (&ctx))) {
+        log_err ("groups_create");
         goto cleanup;
     }
 
     if (ctx.verbose) {
-        const char *parent = overlay_get_parent (ctx.overlay);
-        const char *child = overlay_get_child (ctx.overlay);
+        const char *parent = overlay_get_parent_uri (ctx.overlay);
+        const char *child = overlay_get_bind_uri (ctx.overlay);
         log_msg ("parent: %s", parent ? parent : "none");
         log_msg ("child: %s", child ? child : "none");
     }
 
-    set_proctitle (rank);
-
-    if (rank == 0) {
-        const char *rc1, *rc3, *pmi, *uri;
-        bool rc2_none = false;
-
-        if (attr_get (ctx.attrs, "local-uri", &uri, NULL) < 0) {
-            log_err ("local-uri is not set");
-            goto cleanup;
-        }
-        if (attr_get (ctx.attrs, "broker.rc1_path", &rc1, NULL) < 0) {
-            log_err ("broker.rc1_path is not set");
-            goto cleanup;
-        }
-        if (attr_get (ctx.attrs, "broker.rc3_path", &rc3, NULL) < 0) {
-            log_err ("broker.rc3_path is not set");
-            goto cleanup;
-        }
-        if (attr_get (ctx.attrs, "broker.rc2_none", NULL, NULL) == 0)
-            rc2_none = true;
-
-        if (attr_get (ctx.attrs, "conf.pmi_library_path", &pmi, NULL) < 0) {
-            log_err ("conf.pmi_library_path is not set");
-            goto cleanup;
-        }
-
-        if (!(ctx.runlevel = runlevel_create (ctx.h, ctx.attrs))) {
-            log_err ("creating runlevel handler");
-            goto cleanup;
-        }
-
-        runlevel_set_callback (ctx.runlevel, runlevel_cb, &ctx);
-        runlevel_set_io_callback (ctx.runlevel, runlevel_io_cb, &ctx);
-
-        /* N.B. if runlevel_set_rc() is not called for run levels 1 or 3,
-         * then that level will immediately transition to the next one.
-         * One may set -Sbroker.rc1_path= -Sbroker.rc2_path= on the broker
-         * command line to set an empty rc1/rc3 and skip calling set_rc().
-         */
-        if (rc1 && strlen (rc1) > 0) {
-            if (runlevel_set_rc (ctx.runlevel,
-                                 1,
-                                 rc1,
-                                 strlen (rc1) + 1,
-                                 uri) < 0) {
-                log_err ("runlevel_set_rc 1");
-                goto cleanup;
-            }
-        }
-
-        /* N.B. initial program has the following cases:
-         * 1) if command line, ctx.init_shell_cmd will be non-NULL
-         * 2) if broker.rc2_none is set, skip calling runlevel_set_rc().
-         *    Broker must call runlevel_abort() to transition out of level.
-         * 3) if neither command line nor broker.rc2_none are set,
-         *    call runlevel_set_rc() with empty string and let it configure
-         *    its default command (interactive shell).
-         */
-        if (!rc2_none) {
-            if (runlevel_set_rc (ctx.runlevel,
-                                 2,
-                                 ctx.init_shell_cmd,
-                                 ctx.init_shell_cmd_len,
-                                 uri) < 0) {
-                log_err ("runlevel_set_rc 2");
-                goto cleanup;
-            }
-        }
+    set_proctitle (ctx.rank);
 
-        if (rc3 && strlen (rc3) > 0) {
-            if (runlevel_set_rc (ctx.runlevel,
-                                 3,
-                                 rc3,
-                                 strlen (rc3) + 1,
-                                 uri) < 0) {
-                log_err ("runlevel_set_rc 3");
-                goto cleanup;
-            }
-        }
-    }
+    if (init_local_uri_attr (ctx.overlay, ctx.attrs) < 0 // used by runat
+        || init_critical_ranks_attr (ctx.overlay, ctx.attrs) < 0)
+        goto cleanup;
 
-    /* If Flux was launched by Flux, now that PMI bootstrap and runlevel
-     * initialization is complete, unset Flux job environment variables
-     * so that they don't leak into the jobs other children of this instance.
-     */
-    unsetenv ("FLUX_JOB_ID");
-    unsetenv ("FLUX_JOB_SIZE");
-    unsetenv ("FLUX_JOB_NNODES");
+    if (create_runat_phases (&ctx) < 0)
+        goto cleanup;
 
     /* Wire up the overlay.
      */
-    if (rank > 0) {
+    if (ctx.rank > 0) {
         if (ctx.verbose)
             log_msg ("initializing overlay connect");
         if (overlay_connect (ctx.overlay) < 0) {
@@ -634,16 +432,6 @@ int main (int argc, char *argv[])
         }
     }
 
-    if (!(ctx.shutdown = shutdown_create (ctx.h,
-                                          ctx.shutdown_grace,
-                                          size,
-                                          ctx.tbon_k,
-                                          ctx.overlay))) {
-        log_err ("shutdown_create");
-        goto cleanup;
-    }
-    shutdown_set_callback (ctx.shutdown, shutdown_cb, &ctx);
-
     /* Register internal services
      */
     if (attr_register_handlers (ctx.attrs, ctx.h) < 0) {
@@ -654,182 +442,126 @@ int main (int argc, char *argv[])
         log_err ("heaptrace_initialize");
         goto cleanup;
     }
-    if (exec_initialize (ctx.h, rank, ctx.attrs) < 0) {
+    if (exec_initialize (ctx.h, ctx.rank, ctx.attrs) < 0) {
         log_err ("exec_initialize");
         goto cleanup;
     }
-    if (ping_initialize (ctx.h, "cmb", rank_str) < 0) {
-        log_err ("ping_initialize");
-        goto cleanup;
-    }
-    if (rusage_initialize (ctx.h, "cmb") < 0) {
-        log_err ("rusage_initialize");
+    if (flux_aux_set (ctx.h,
+                      "flux::uuid",
+                      (char *)overlay_get_uuid (ctx.overlay),
+                      NULL) < 0) {
+        log_err ("error adding broker uuid to aux container");
         goto cleanup;
     }
-
     if (!(handlers = broker_add_services (&ctx))) {
         log_err ("broker_add_services");
         goto cleanup;
     }
-
-    /* Initialize comms module infrastructure.
-     */
-    if (ctx.verbose)
-        log_msg ("initializing modules");
-    modhash_set_rank (ctx.modhash, rank);
-    modhash_set_flux (ctx.modhash, ctx.h);
-    modhash_set_heartbeat (ctx.modhash, ctx.heartbeat);
-
-    /* install heartbeat (including timer on rank 0)
+    /* overlay_control_start() calls flux_sync_create(), thus
+     * requires event.subscribe to have a handler before running.
      */
-    heartbeat_set_flux (ctx.heartbeat, ctx.h);
-    if (heartbeat_register_attrs (ctx.heartbeat, ctx.attrs) < 0) {
-        log_err ("initializing heartbeat attributes");
+    if (overlay_control_start (ctx.overlay) < 0) {
+        log_err ("error initializing overlay control messages");
         goto cleanup;
     }
-    if (heartbeat_set_rate (ctx.heartbeat, ctx.heartbeat_rate) < 0) {
-        log_err ("heartbeat_set_rate");
-        goto cleanup;
-    }
-    if (heartbeat_start (ctx.heartbeat) < 0) {
-        log_err ("heartbeat_start");
-        goto cleanup;
-    }
-    if (rank == 0 && ctx.verbose)
-        log_msg ("installing session heartbeat: T=%0.1fs",
-                  heartbeat_get_rate (ctx.heartbeat));
 
-    /* Send hello message to parent.
-     * N.B. uses tbon topology attributes set above.
-     * Start init once wireup is complete.
+    /* Configure broker state machine
      */
-    hello_set_flux (ctx.hello, ctx.h);
-    hello_set_callback (ctx.hello, hello_update_cb, &ctx);
-    if (hello_register_attrs (ctx.hello, ctx.attrs) < 0) {
-        log_err ("configuring hello attributes");
+    if (!(ctx.state_machine = state_machine_create (&ctx))) {
+        log_err ("error creating broker state machine");
         goto cleanup;
     }
-    if (hello_start (ctx.hello) < 0) {
-        log_err ("hello_start");
+    state_machine_post (ctx.state_machine, "start");
+
+    /* Create shutdown mechanism
+     */
+    if (!(ctx.shutdown = shutdown_create (&ctx))) {
+        log_err ("error creating shutdown mechanism");
         goto cleanup;
     }
+
     /* Load the local connector module.
      * Other modules will be loaded in rc1 using flux module,
      * which uses the local connector.
      * The shutdown protocol unloads it.
      */
-    if (ctx.verbose)
+    if (ctx.verbose > 1)
         log_msg ("loading connector-local");
-    if (load_module_byname (&ctx, "connector-local", NULL, 0, NULL) < 0) {
-        log_err ("load_module connector-local");
+    if (modhash_load (ctx.modhash,
+                      NULL,
+                      "connector-local",
+                      NULL,
+                      NULL,
+                      &error) < 0) {
+        log_err ("load_module connector-local: %s", error.text);
         goto cleanup;
     }
 
+    if (ctx.rank == 0 && execute_parental_notifications (&ctx) < 0)
+        goto cleanup;
+
     /* Event loop
      */
-    if (ctx.verbose)
+    if (ctx.verbose > 1)
         log_msg ("entering event loop");
     /* Once we enter the reactor, default exit_rc is now 0 */
-    exit_rc = 0;
+    ctx.exit_rc = 0;
     if (flux_reactor_run (ctx.reactor, 0) < 0)
         log_err ("flux_reactor_run");
-    if (ctx.verbose)
+    if (ctx.verbose > 1)
         log_msg ("exited event loop");
 
-    /* inform all lingering subprocesses we are tearing down.  Do this
-     * before any cleanup/teardown below, as this call will re-enter
-     * the reactor.
-     */
-    exec_terminate_subprocesses (ctx.h);
-
 cleanup:
-    if (ctx.verbose)
+    if (ctx.verbose > 1)
         log_msg ("cleaning up");
 
     /* Restore default sigmask and actions for SIGINT, SIGTERM
      */
-    if (sigprocmask (SIG_SETMASK, &old_sigmask, NULL) < 0)
-        log_err ("sigprocmask");
-    if (sigaction (SIGINT, &old_sigact_int, NULL) < 0)
-        log_err ("sigaction");
-    if (sigaction (SIGTERM, &old_sigact_term, NULL) < 0)
-        log_err ("sigaction");
-
-    /* remove heartbeat timer, if any
-     */
-    heartbeat_stop (ctx.heartbeat);
+    if (sigprocmask (SIG_SETMASK, &old_sigmask, NULL) < 0
+        || sigaction (SIGINT, &old_sigact_int, NULL) < 0
+        || sigaction (SIGTERM, &old_sigact_term, NULL) < 0)
+        log_err ("error restoring signal mask");
 
     /* Unregister builtin services
      */
     attr_destroy (ctx.attrs);
-    content_cache_destroy (ctx.cache);
 
-    modhash_destroy (ctx.modhash);
+    if (modhash_destroy (ctx.modhash) > 0) {
+        if (ctx.exit_rc == 0)
+            ctx.exit_rc = 1;
+    }
     zlist_destroy (&ctx.sigwatchers);
+    shutdown_destroy (ctx.shutdown);
+    state_machine_destroy (ctx.state_machine);
     overlay_destroy (ctx.overlay);
-    heartbeat_destroy (ctx.heartbeat);
+    groups_destroy (ctx.groups);
     service_switch_destroy (ctx.services);
-    hello_destroy (ctx.hello);
-    shutdown_destroy (ctx.shutdown);
     broker_remove_services (handlers);
     publisher_destroy (ctx.publisher);
+    brokercfg_destroy (ctx.config);
+    runat_destroy (ctx.runat);
+    flux_watcher_destroy (ctx.w_internal);
+    flux_close (ctx.h_internal);
     flux_close (ctx.h);
     flux_reactor_destroy (ctx.reactor);
-    zlist_destroy (&ctx.subscriptions);
-    runlevel_destroy (ctx.runlevel);
+    subhash_destroy (ctx.sub);
     free (ctx.init_shell_cmd);
+    optparse_destroy (ctx.opts);
 
-    return exit_rc;
-}
-
-struct attrmap {
-    const char *env;
-    const char *attr;
-    uint8_t required:1;
-    uint8_t sanitize:1;
-};
-
-static struct attrmap attrmap[] = {
-    { "FLUX_EXEC_PATH",         "conf.exec_path",           1, 0 },
-    { "FLUX_CONNECTOR_PATH",    "conf.connector_path",      1, 0 },
-    { "FLUX_MODULE_PATH",       "conf.module_path",         1, 0 },
-    { "FLUX_PMI_LIBRARY_PATH",  "conf.pmi_library_path",    1, 0 },
-    { "FLUX_SEC_DIRECTORY",     "security.keydir",          1, 0 },
-
-    { "FLUX_URI",               "parent-uri",               0, 1 },
-    { "FLUX_KVS_NAMESPACE",     "parent-kvs-namespace",     0, 1 },
-    { NULL, NULL, 0, 0 },
-};
-
-static void init_attrs_from_environment (attr_t *attrs)
-{
-    struct attrmap *m;
-    const char *val;
-    int flags = 0;  // XXX possibly these should be immutable?
-
-    for (m = &attrmap[0]; m->env != NULL; m++) {
-        val = getenv (m->env);
-        if (!val && m->required)
-            log_msg_exit ("required environment variable %s is not set", m->env);
-        if (attr_add (attrs, m->attr, val, flags) < 0)
-            log_err_exit ("attr_add %s", m->attr);
-        if (m->sanitize)
-            unsetenv (m->env);
-    }
+    return ctx.exit_rc;
 }
 
 static void init_attrs_broker_pid (attr_t *attrs, pid_t pid)
 {
     char *attrname = "broker.pid";
-    char *pidval;
+    char pidval[32];
 
-    pidval = xasprintf ("%u", pid);
+    snprintf (pidval, sizeof (pidval), "%u", pid);
     if (attr_add (attrs,
                   attrname,
                   pidval,
-                  FLUX_ATTRFLAG_IMMUTABLE) < 0)
+                  ATTR_IMMUTABLE) < 0)
         log_err_exit ("attr_add %s", attrname);
-    free (pidval);
 }
 
 static void init_attrs_rc_paths (attr_t *attrs)
@@ -847,190 +579,278 @@ static void init_attrs_rc_paths (attr_t *attrs)
         log_err_exit ("attr_add rc3_path");
 }
 
-static void init_attrs (attr_t *attrs, pid_t pid)
+static void init_attrs_shell_paths (attr_t *attrs)
+{
+    if (attr_add (attrs,
+                  "conf.shell_pluginpath",
+                  flux_conf_builtin_get ("shell_pluginpath", FLUX_CONF_AUTO),
+                  0) < 0)
+        log_err_exit ("attr_add conf.shell_pluginpath");
+    if (attr_add (attrs,
+                  "conf.shell_initrc",
+                  flux_conf_builtin_get ("shell_initrc", FLUX_CONF_AUTO),
+                  0) < 0)
+        log_err_exit ("attr_add conf.shell_initrc");
+}
+
+static void init_attrs_starttime (attr_t *attrs, double starttime)
+{
+    char buf[32];
+
+    snprintf (buf, sizeof (buf), "%.2f", starttime);
+    if (attr_add (attrs, "broker.starttime", buf, ATTR_IMMUTABLE) < 0)
+        log_err_exit ("error setting broker.starttime attribute");
+}
+
+static void init_attrs (attr_t *attrs, pid_t pid, struct flux_msg_cred *cred)
 {
-    /* Initialize config attrs from environment set up by flux(1)
+    const char *val;
+
+    /* Set the parent-uri attribute IFF this instance was run as a job
+     * in the enclosing instance.  "parent" in this context reflects
+     * a hierarchy of resource allocation.
      */
-    init_attrs_from_environment (attrs);
+    if (getenv ("FLUX_JOB_ID"))
+        val = getenv ("FLUX_URI");
+    else
+        val = NULL;
+    if (attr_add (attrs, "parent-uri", val, ATTR_IMMUTABLE) < 0)
+        log_err_exit ("setattr parent-uri");
+    unsetenv ("FLUX_URI");
 
-    /* Initialize other miscellaneous attrs
+    /* Unset FLUX_PROXY_REMOTE since once a new broker starts we're no
+     * longer technically running under the influence of flux-proxy(1).
      */
+    unsetenv ("FLUX_PROXY_REMOTE");
+
+    val = getenv ("FLUX_KVS_NAMESPACE");
+    if (attr_add (attrs, "parent-kvs-namespace", val, ATTR_IMMUTABLE) < 0)
+        log_err_exit ("setattr parent-kvs-namespace");
+    unsetenv ("FLUX_KVS_NAMESPACE");
+
     init_attrs_broker_pid (attrs, pid);
     init_attrs_rc_paths (attrs);
+    init_attrs_shell_paths (attrs);
 
-    if (attr_add (attrs, "version", FLUX_CORE_VERSION_STRING,
-                                            FLUX_ATTRFLAG_IMMUTABLE) < 0)
+    /* Allow version to be changed by instance owner for testing
+     */
+    if (attr_add (attrs, "version", FLUX_CORE_VERSION_STRING, 0) < 0)
         log_err_exit ("attr_add version");
+
+    char tmp[32];
+    snprintf (tmp, sizeof (tmp), "%ju", (uintmax_t)cred->userid);
+    if (attr_add (attrs, "security.owner", tmp, ATTR_IMMUTABLE) < 0)
+        log_err_exit ("attr_add owner");
 }
 
-/* Parse TOML config, emitting any parse error here.
- * This will fail if no configuration exists.
- */
-static int parse_config_files (flux_t *h)
+static void set_proctitle (uint32_t rank)
 {
-    flux_conf_error_t error;
-
-    if (flux_get_conf (h, &error) == NULL) {
-        if (error.lineno == -1)
-            log_err ("Config file error: %s%s%s",
-                     error.filename,
-                     *error.filename ? ": " : "",
-                     error.errbuf);
-        else
-            log_err ("Config file error: %s:%d: %s",
-                     error.filename,
-                     error.lineno,
-                     error.errbuf);
-        return -1;
-    }
-    return 0;
+#ifdef PR_SET_NAME
+    static char proctitle[32];
+    snprintf (proctitle, sizeof (proctitle), "flux-broker-%"PRIu32, rank);
+    (void)prctl (PR_SET_NAME, proctitle, 0, 0, 0);
+#endif
 }
 
-static void hello_update_cb (hello_t *hello, void *arg)
+static bool is_interactive_shell (const char *argz, size_t argz_len)
 {
-    broker_ctx_t *ctx = arg;
+    bool result = false;
+    /*  If no command is specified, then an interactive shell will be run
+     */
+    if (argz == NULL)
+        return true;
 
-    if (hello_complete (hello)) {
-        flux_log (ctx->h, LOG_INFO, "wireup: %d/%d (complete) %.1fs",
-                  hello_get_count (hello), overlay_get_size(ctx->overlay),
-                  hello_get_time (hello));
-        flux_log (ctx->h, LOG_INFO, "Run level %d starting", 1);
-        overlay_set_idle_warning (ctx->overlay, 3);
-        if (runlevel_set_level (ctx->runlevel, 1) < 0)
-            log_err_exit ("runlevel_set_level 1");
-        /* FIXME: shutdown hello protocol */
-    } else  {
-        flux_log (ctx->h, LOG_INFO, "wireup: %d/%d (incomplete) %.1fs",
-                  hello_get_count (hello), overlay_get_size(ctx->overlay),
-                  hello_get_time (hello));
+    /*  O/w, if command is plain "$SHELL", e.g. bash, zsh, csh, etc.
+     *   then assume interactive shell.
+     */
+    if (argz_count (argz, argz_len) == 1) {
+        char *shell;
+        char *cmd = argz_next (argz, argz_len, NULL);
+        while ((shell = getusershell ())) {
+            if (streq (cmd, shell) || streq (cmd, basename_simple (shell))) {
+                result = true;
+                break;
+            }
+        }
+        endusershell ();
     }
+    return result;
 }
 
-/* If shutdown timeout has occured, exit immediately.
- * If shutdown is beginning, start unload of connector-local module.
- * If shutdown is ending, then IFF connector-local has finished
- * unloading, stop the reactor.  Otherwise module_status_cb() will do it.
- */
-static void shutdown_cb (struct shutdown *s, void *arg)
+static int create_runat_rc2 (struct runat *r, const char *argz, size_t argz_len)
 {
-    broker_ctx_t *ctx = arg;
-
-    if (shutdown_is_expired (s)) {
-        log_msg ("shutdown timer expired on rank %"PRIu32,
-                 overlay_get_rank (ctx->overlay));
-        _exit (1);
+    if (is_interactive_shell (argz, argz_len)) { // run interactive shell
+        /*  Check if stdin is a tty and error out if not to avoid
+         *   confusing users with what appears to be a hang.
+         */
+        if (!isatty (STDIN_FILENO))
+            log_msg_exit ("stdin is not a tty - can't run interactive shell");
+        if (runat_push_shell (r, "rc2", argz, 0) < 0)
+            return -1;
     }
-    if (!shutdown_is_complete (s)) {
-        module_t *p;
-        if ((p = module_lookup_byname (ctx->modhash, "connector-local")))
-            module_stop (p);
+    else if (argz_count (argz, argz_len) == 1) { // run shell -c "command"
+        if (runat_push_shell_command (r, "rc2", argz, 0) < 0)
+            return -1;
     }
-    else {
-        if (!module_lookup_byname (ctx->modhash, "connector-local"))
-            flux_reactor_stop (flux_get_reactor (ctx->h));
+    else { // direct exec
+        if (runat_push_command (r, "rc2", argz, argz_len, 0) < 0)
+            return -1;
     }
+    return 0;
 }
 
-static void set_proctitle (uint32_t rank)
+static int create_runat_phases (broker_ctx_t *ctx)
 {
-    static char proctitle[32];
-    snprintf (proctitle, sizeof (proctitle), "flux-broker-%"PRIu32, rank);
-    (void)prctl (PR_SET_NAME, proctitle, 0, 0, 0);
+    const char *rc1, *rc3, *local_uri;
+    bool rc2_none = false;
+
+    if (attr_get (ctx->attrs, "local-uri", &local_uri, NULL) < 0) {
+        log_err ("local-uri is not set");
+        return -1;
+    }
+    if (attr_get (ctx->attrs, "broker.rc1_path", &rc1, NULL) < 0) {
+        log_err ("broker.rc1_path is not set");
+        return -1;
+    }
+    if (attr_get (ctx->attrs, "broker.rc3_path", &rc3, NULL) < 0) {
+        log_err ("broker.rc3_path is not set");
+        return -1;
+    }
+    if (attr_get (ctx->attrs, "broker.rc2_none", NULL, NULL) == 0)
+        rc2_none = true;
+
+    if (!(ctx->runat = runat_create (ctx->h, local_uri, ctx->sd_notify))) {
+        log_err ("runat_create");
+        return -1;
+    }
+
+    /* rc1 - initialization
+     */
+    if (rc1 && strlen (rc1) > 0) {
+        if (runat_push_shell_command (ctx->runat,
+                                      "rc1",
+                                      rc1,
+                                      RUNAT_FLAG_LOG_STDIO) < 0) {
+            log_err ("runat_push_shell_command rc1");
+            return -1;
+        }
+    }
+
+    /* rc2 - initial program
+     */
+    if (ctx->rank == 0 && !rc2_none) {
+        if (create_runat_rc2 (ctx->runat,
+                              ctx->init_shell_cmd,
+                              ctx->init_shell_cmd_len) < 0) {
+            log_err ("create_runat_rc2");
+            return -1;
+        }
+    }
+
+    /* rc3 - finalization
+     */
+    if (rc3 && strlen (rc3) > 0) {
+        if (runat_push_shell_command (ctx->runat,
+                                      "rc3",
+                                      rc3,
+                                      RUNAT_FLAG_LOG_STDIO) < 0) {
+            log_err ("runat_push_shell_command rc3");
+            return -1;
+        }
+    }
+    return 0;
 }
 
-/* Handle line by line output on stdout, stderr of runlevel subprocess.
- */
-static void runlevel_io_cb (struct runlevel *r,
-                            const char *name,
-                            const char *msg,
-                            void *arg)
+static int checkdir (const char *name, const char *path)
 {
-    broker_ctx_t *ctx = arg;
-    int loglevel = !strcmp (name, "stderr") ? LOG_ERR : LOG_INFO;
-    int runlevel = runlevel_get_level (r);
+    struct stat sb;
 
-    flux_log (ctx->h, loglevel, "rc%d: %s", runlevel, msg);
+    if (stat (path, &sb) < 0) {
+        log_err ("cannot stat %s %s", name, path);
+        return -1;
+    }
+    if (sb.st_uid != getuid ()) {
+        errno = EPERM;
+        log_err ("%s %s is not owned by instance owner", name, path);
+        return -1;
+    }
+    if (!S_ISDIR (sb.st_mode)) {
+        errno = ENOTDIR;
+        log_err ("%s %s", name, path);
+        return -1;
+    }
+    if ((sb.st_mode & S_IRWXU) != S_IRWXU) {
+        log_msg ("%s %s does not have owner=rwx permissions", name, path);
+        errno = EPERM;
+        return -1;
+    }
+    return 0;
 }
 
-/* Handle completion of runlevel subprocess.
+/* Validate statedir, if set.
+ * Ensure that the attribute cannot change from this point forward.
  */
-static void runlevel_cb (struct runlevel *r,
-                         int level,
-                         int rc,
-                         double elapsed,
-                         const char *exit_string,
-                         void *arg)
+static int check_statedir (attr_t *attrs)
 {
-    broker_ctx_t *ctx = arg;
-    int new_level = -1;
-
-    flux_log (ctx->h, rc == 0 ? LOG_INFO : LOG_ERR,
-              "Run level %d %s (rc=%d) %.1fs", level, exit_string, rc, elapsed);
-
-    switch (level) {
-        case 1: /* init completed */
-            if (rc != 0) {
-                exit_rc = rc;
-                new_level = 3;
-            } else
-                new_level = 2;
-            break;
-        case 2: /* initial program completed */
-            exit_rc = rc;
-            new_level = 3;
-            break;
-        case 3: /* finalization completed */
-            if (rc != 0 && exit_rc == 0)
-                exit_rc = rc;
-            shutdown_instance (ctx->shutdown); // initiate shutdown from rank 0
-            break;
+    const char *statedir;
+
+    if (attr_get (attrs, "statedir", &statedir, NULL) < 0) {
+        if (attr_add (attrs, "statedir", NULL, ATTR_IMMUTABLE) < 0) {
+            log_err ("error creating statedir broker attribute");
+            return -1;
+        }
     }
-    if (new_level != -1) {
-        flux_log (ctx->h, LOG_INFO, "Run level %d starting", new_level);
-        if (runlevel_set_level (r, new_level) < 0)
-            log_err_exit ("runlevel_set_level %d", new_level);
+    else {
+        if (checkdir ("statedir", statedir) < 0)
+            return -1;
+        if (attr_set_flags (attrs, "statedir", ATTR_IMMUTABLE) < 0) {
+            log_err ("error setting statedir broker attribute flags");
+            return -1;
+        }
     }
+    return 0;
 }
 
-static int create_dummyattrs (flux_t *h, uint32_t rank, uint32_t size)
+static int create_rundir_symlinks (const char *run_dir, flux_error_t *error)
 {
-    char *rank_str = NULL;
-    char *size_str = NULL;
-    int rc = -1;
-
-    if (asprintf (&rank_str, "%"PRIu32, rank) < 0)
-        goto cleanup;
-    if (flux_attr_set_cacheonly (h, "rank", rank_str) < 0)
-        goto cleanup;
-
-    if (asprintf (&size_str, "%"PRIu32, size) < 0)
-        goto cleanup;
-    if (flux_attr_set_cacheonly (h, "size", size_str) < 0)
-        goto cleanup;
-
-    rc = 0;
-cleanup:
-    free (rank_str);
-    free (size_str);
-    return rc;
+    char path[1024];
+    size_t size = sizeof (path);
+    const char *target;
+
+    if (strlcpy (path, run_dir, size) >= size
+        || strlcat (path, "/bin", size) >= size)
+        goto overflow;
+    if (mkdir (path, 0755) < 0) {
+        errprintf (error, "mkdir %s: %s", path, strerror (errno));
+        return -1;
+    }
+    cleanup_push_string (cleanup_directory_recursive, path);
+    if (strlcat (path, "/flux", size) >= size)
+        goto overflow;
+    if (executable_is_intree () == 1)
+        target = ABS_TOP_BUILDDIR "/src/cmd/flux";
+    else
+        target = X_BINDIR "/flux";
+    if (symlink (target, path) < 0) {
+        errprintf (error, "symlink %s: %s", path, strerror (errno));
+        return -1;
+    }
+    return 0;
+overflow:
+    errprintf (error, "buffer overflow");
+    errno = EOVERFLOW;
+    return -1;
 }
 
 /*  Handle global rundir attribute.
- *
- *  If not set, create a temporary directory and use it as the rundir.
- *  If set, attempt to create it if it doesn't exist. In either case,
- *  validate directory persmissions and set the rundir attribute
- *  immutable. If the rundir is created by this function it will be
- *  scheduled for later cleanup at broker exit. Pre-existing directories
- *  are left intact.
  */
 static int create_rundir (attr_t *attrs)
 {
-    const char *run_dir;
-    char *dir = NULL;
-    char *uri = NULL;
+    const char *tmpdir;
+    const char *run_dir = NULL;
+    char path[1024];
+    int len;
     bool do_cleanup = true;
-    struct stat sb;
     int rc = -1;
 
     /*  If rundir attribute isn't set, then create a temp directory
@@ -1039,17 +859,27 @@ static int create_rundir (attr_t *attrs)
      *   the dir for auto-cleanup at broker exit.
      */
     if (attr_get (attrs, "rundir", &run_dir, NULL) < 0) {
-        const char *tmpdir = getenv ("TMPDIR");
-        if (asprintf (&dir, "%s/flux-XXXXXX", tmpdir ? tmpdir : "/tmp") < 0)
+        if (!(tmpdir = getenv ("TMPDIR")))
+            tmpdir = "/tmp";
+        len = snprintf (path, sizeof (path), "%s/flux-XXXXXX", tmpdir);
+        if (len >= sizeof (path)) {
+            log_msg ("rundir buffer overflow");
             goto done;
-        if (!(run_dir = mkdtemp (dir)))
+        }
+        if (!(run_dir = mkdtemp (path))) {
+            log_err ("cannot create directory in %s", tmpdir);
             goto done;
-        if (attr_add (attrs, "rundir", run_dir, 0) < 0)
+        }
+        if (attr_add (attrs, "rundir", run_dir, 0) < 0) {
+            log_err ("error setting rundir broker attribute");
             goto done;
+        }
     }
     else if (mkdir (run_dir, 0700) < 0) {
-        if (errno != EEXIST)
+        if (errno != EEXIST) {
+            log_err ("error creating rundir %s ", run_dir);
             goto done;
+        }
         /* Do not cleanup directory if we did not create it here
          */
         do_cleanup = false;
@@ -1057,187 +887,312 @@ static int create_rundir (attr_t *attrs)
 
     /*  Ensure created or existing directory is writeable:
      */
-    if (stat (run_dir, &sb) < 0)
+    if (checkdir ("rundir", run_dir) < 0)
         goto done;
-    if (!S_ISDIR (sb.st_mode)) {
-        errno = ENOTDIR;
-        goto done;
-    }
-    if ((sb.st_mode & S_IRWXU) != S_IRWXU) {
-        errno = EPERM;
+
+    /*  Ensure that AF_UNIX sockets can be created in rundir - see #3925.
+     */
+    struct sockaddr_un sa;
+    size_t path_limit = sizeof (sa.sun_path) - sizeof ("/local-9999");
+    size_t path_length = strlen (run_dir);
+    if (path_length > path_limit) {
+        log_msg ("rundir length of %zu bytes exceeds max %zu"
+                 " to allow for AF_UNIX socket creation.",
+                 path_length,
+                 path_limit);
         goto done;
     }
 
     /*  rundir is now fixed, so make the attribute immutable, and
      *   schedule the dir for cleanup at exit if we created it here.
      */
-    if (attr_set_flags (attrs, "rundir", FLUX_ATTRFLAG_IMMUTABLE) < 0)
+    if (attr_set_flags (attrs, "rundir", ATTR_IMMUTABLE) < 0) {
+        log_err ("error setting rundir broker attribute flags");
         goto done;
-    if (do_cleanup)
-        cleanup_push_string (cleanup_directory_recursive, run_dir);
+    }
+
+    /*  Create $rundir/bin/flux so flux-relay can be found - see #5583.
+     */
+    flux_error_t error;
+    if (create_rundir_symlinks (run_dir, &error) < 0) {
+        if (errno != EEXIST)
+            log_err ("error creating rundir symlinks: %s", error.text);
+        // if this fails, soldier on
+    }
+
     rc = 0;
 done:
-    free (dir);
-    free (uri);
+    if (do_cleanup && run_dir != NULL)
+        cleanup_push_string (cleanup_directory_recursive, run_dir);
     return rc;
 }
 
-static int create_broker_rundir (overlay_t *ov, void *arg)
+static int init_local_uri_attr (struct overlay *ov, attr_t *attrs)
 {
-    attr_t *attrs = arg;
-    uint32_t rank;
-    const char *rundir;
-    const char *local_uri;
-    char *broker_rundir = NULL;
-    char *uri = NULL;
-    int rv = -1;
-
-    if (attr_get (attrs, "rundir", &rundir, NULL) < 0) {
-        log_msg ("create_broker_rundir: rundir attribute not set");
-        goto cleanup;
-    }
+    const char *uri;
 
-    rank = overlay_get_rank (ov);
-    if (asprintf (&broker_rundir, "%s/%u", rundir, rank) < 0) {
-        log_err ("create_broker_rundir: asprintf");
-        goto cleanup;
-    }
-    if (mkdir (broker_rundir, 0700) < 0) {
-        log_err ("create_broker_rundir: mkdir (%s)", broker_rundir);
-        goto cleanup;
-    }
-    if (attr_add (attrs, "broker.rundir", broker_rundir,
-                  FLUX_ATTRFLAG_IMMUTABLE) < 0) {
-        log_err ("create_broker_rundir: attr_add broker.rundir");
-        goto cleanup;
+    if (attr_get (attrs, "local-uri", &uri, NULL) < 0) {
+        uint32_t rank = overlay_get_rank (ov);
+        const char *rundir;
+        char buf[1024];
+
+        if (attr_get (attrs, "rundir", &rundir, NULL) < 0) {
+            log_msg ("rundir attribute is not set");
+            return -1;
+        }
+        if (snprintf (buf, sizeof (buf), "local://%s/local-%d",
+                      rundir, rank) >= sizeof (buf)) {
+            log_msg ("buffer overflow while building local-uri");
+            return -1;
+        }
+        if (attr_add (attrs, "local-uri", buf, ATTR_IMMUTABLE) < 0) {
+            log_err ("setattr local-uri");
+            return -1;
+        }
     }
+    else {
+        char path[1024];
 
-    if (attr_get (attrs, "local-uri", &local_uri, NULL) < 0) {
-        if (asprintf (&uri, "local://%s/local", broker_rundir) < 0) {
-            log_err ("create_broker_rundir: asprintf (uri)");
-            goto cleanup;
+        if (!strstarts (uri, "local://")) {
+            log_msg ("local-uri is malformed");
+            return -1;
         }
-        if (attr_add (attrs, "local-uri", uri, FLUX_ATTRFLAG_IMMUTABLE) < 0) {
-            log_err ("create_broker_rundir: attr_add (local-uri)");
-            goto cleanup;
+        if (snprintf (path, sizeof (path), "%s", uri + 8) >= sizeof (path)) {
+            log_msg ("buffer overflow while checking local-uri");
+            return -1;
+        }
+        if (checkdir ("local-uri directory", dirname (path)) < 0)
+            return -1;
+
+        /* see #3925 */
+        struct sockaddr_un sa;
+        size_t path_limit = sizeof (sa.sun_path) - 1;
+        size_t path_length = strlen (uri + 8);
+        if (path_length > path_limit) {
+            log_msg ("local-uri length of %zu bytes exceeds max %zu"
+                     " AF_UNIX socket path length",
+                     path_length,
+                     path_limit);
+            return -1;
         }
     }
-    rv = 0;
-cleanup:
-    free (uri);
-    free (broker_rundir);
-    return rv;
+    return 0;
 }
 
-static bool nodeset_member (const char *s, uint32_t rank)
+static int init_critical_ranks_attr (struct overlay *ov, attr_t *attrs)
 {
-    struct idset *ns = NULL;
-    bool member = true;
-
-    if (s) {
-        if (!(ns = idset_decode (s)))
-            log_msg_exit ("malformed nodeset: %s", s);
-        member = idset_test (ns, rank);
-        idset_destroy (ns);
+    int rc = -1;
+    const char *val;
+    char *ranks = NULL;
+    struct idset *critical_ranks = NULL;
+
+    if (attr_get (attrs, "broker.critical-ranks", &val, NULL) < 0) {
+        if (!(critical_ranks = overlay_get_default_critical_ranks (ov))
+            || !(ranks = idset_encode (critical_ranks, IDSET_FLAG_RANGE))) {
+            log_err ("unable to calculate critical-ranks attribute");
+            goto out;
+        }
+        if (attr_add (attrs,
+                      "broker.critical-ranks",
+                      ranks,
+                      ATTR_IMMUTABLE) < 0) {
+            log_err ("attr_add critical_ranks");
+            goto out;
+        }
     }
-    return member;
+    else {
+        if (!(critical_ranks = idset_decode (val))
+            || idset_last (critical_ranks) >= overlay_get_size (ov)) {
+            log_msg ("invalid value for broker.critical-ranks='%s'", val);
+            goto out;
+        }
+        /*  Need to set immutable flag when attr set on command line
+         */
+        if (attr_set_flags (attrs,
+                            "broker.critical-ranks",
+                            ATTR_IMMUTABLE) < 0) {
+            log_err ("failed to make broker.criitcal-ranks attr immutable");
+            goto out;
+        }
+    }
+    rc = 0;
+out:
+    idset_destroy (critical_ranks);
+    free (ranks);
+    return rc;
 }
 
-static int mod_svc_cb (const flux_msg_t *msg, void *arg)
+static flux_future_t *set_uri_job_memo (flux_t *h,
+                                        const char *hostname,
+                                        flux_jobid_t id,
+                                        attr_t *attrs)
 {
-    module_t *p = arg;
-    int rc = module_sendmsg (p, msg);
-    return rc;
+    const char *local_uri = NULL;
+    const char *path;
+    char uri [1024];
+
+    if (attr_get (attrs, "local-uri", &local_uri, NULL) < 0) {
+        log_err ("Unexpectedly unable to fetch local-uri attribute");
+        return NULL;
+    }
+    path = local_uri + 8; /* forward past "local://" */
+    if (snprintf (uri,
+                 sizeof (uri),
+                 "ssh://%s%s",
+                 hostname, path) >= sizeof (uri)) {
+        log_msg ("buffer overflow while checking local-uri");
+        return NULL;
+    }
+    return flux_rpc_pack (h,
+                          "job-manager.memo",
+                          FLUX_NODEID_ANY,
+                          0,
+                          "{s:I s:{s:s}}",
+                          "id", id,
+                          "memo",
+                            "uri", uri);
 }
 
-/* If a dlerror/dlsym error occurs during modfind/modname,
- * log it here.  Such messages can be helpful in diagnosing
- * dynamic binding problems for comms modules.
+/*  Encode idset of critical nodes/shell ranks, which is calculated
+ *   from broker.mapping and broker.critical-ranks.
  */
-static void module_dlerror (const char *errmsg, void *arg)
+static char *encode_critical_nodes (attr_t *attrs)
 {
-    flux_t *h = arg;
-    flux_log (h, LOG_DEBUG, "flux_modname: %s", errmsg);
-}
+    struct idset *ranks = NULL;
+    struct idset *nodeids = NULL;
+    struct taskmap *map = NULL;
+    char *s = NULL;
+    int nodeid;
+    const char *mapping;
+    const char *ranks_attr;
+    unsigned int i;
+
+
+    if (attr_get (attrs, "broker.mapping", &mapping, NULL) < 0
+        || mapping == NULL
+        || !(map = taskmap_decode (mapping, NULL))
+        || attr_get (attrs, "broker.critical-ranks", &ranks_attr, NULL) < 0
+        || !(ranks = idset_decode (ranks_attr))
+        || !(nodeids = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        goto done;
 
+    /*  Map the broker ranks from the broker.critical-ranks attr to
+     *  shell ranks/nodeids using PMI_process_mapping (this handles the
+     *  rare case where multiple brokers per node/shell were launched)
+     */
+    i = idset_first (ranks);
+    while (i != IDSET_INVALID_ID) {
+        if ((nodeid = taskmap_nodeid (map, i)) < 0
+            || idset_set (nodeids, nodeid) < 0)
+            goto done;
+        i = idset_next (ranks, i);
+    }
+    s = idset_encode (nodeids, IDSET_FLAG_RANGE);
+done:
+    taskmap_destroy (map);
+    idset_destroy (ranks);
+    idset_destroy (nodeids);
+    return s;
+}
 
-static int load_module_bypath (broker_ctx_t *ctx, const char *path,
-                               const char *argz, size_t argz_len,
-                               const flux_msg_t *request)
+static flux_future_t *set_critical_ranks (flux_t *h,
+                                          flux_jobid_t id,
+                                          attr_t *attrs)
 {
-    module_t *p = NULL;
-    char *name, *arg;
+    int saved_errno;
+    flux_future_t *f;
+    char *nodeids;
 
-    if (!(name = flux_modname (path, module_dlerror, ctx->h))) {
-        errno = ENOENT;
-        goto error;
-    }
-    if (!(p = module_add (ctx->modhash, path)))
-        goto error;
-    if (service_add (ctx->services, module_get_name (p),
-                                    module_get_uuid (p), mod_svc_cb, p) < 0)
-        goto module_remove;
-    arg = argz_next (argz, argz_len, NULL);
-    while (arg) {
-        module_add_arg (p, arg);
-        arg = argz_next (argz, argz_len, arg);
-    }
-    module_set_poller_cb (p, module_cb, ctx);
-    module_set_status_cb (p, module_status_cb, ctx);
-    if (request && module_push_insmod (p, request) < 0) // response deferred
-        goto service_remove;
-    if (module_start (p) < 0)
-        goto service_remove;
-    flux_log (ctx->h, LOG_DEBUG, "insmod %s", name);
-    free (name);
-    return 0;
-service_remove:
-    service_remove_byuuid (ctx->services, module_get_uuid (p));
-module_remove:
-    module_remove (ctx->modhash, p);
-error:
-    free (name);
-    return -1;
+    if (!(nodeids = encode_critical_nodes (attrs)))
+        return NULL;
+    f = flux_rpc_pack (h,
+                       "job-exec.critical-ranks",
+                       FLUX_NODEID_ANY, 0,
+                       "{s:I s:s}",
+                       "id", id,
+                       "ranks", nodeids);
+    saved_errno = errno;
+    free (nodeids);
+    errno = saved_errno;
+    return f;
 }
 
-static int load_module_byname (broker_ctx_t *ctx, const char *name,
-                               const char *argz, size_t argz_len,
-                               const flux_msg_t *request)
+static int execute_parental_notifications (struct broker *ctx)
 {
-    const char *modpath;
-    char *path;
+    const char *jobid = NULL;
+    const char *parent_uri = NULL;
+    flux_jobid_t id;
+    flux_t *h = NULL;
+    flux_future_t *f = NULL;
+    flux_future_t *f2 = NULL;
+    int rc = -1;
 
-    if (attr_get (ctx->attrs, "conf.module_path", &modpath, NULL) < 0) {
-        log_msg ("conf.module_path is not set");
+    /* Skip if "jobid" or "parent-uri" not set, this is probably
+     *  not a child of any Flux instance.
+     */
+    if (attr_get (ctx->attrs, "parent-uri", &parent_uri, NULL) < 0
+        || parent_uri == NULL
+        || attr_get (ctx->attrs, "jobid", &jobid, NULL) < 0
+        || jobid == NULL)
+        return 0;
+
+    if (flux_job_id_parse (jobid, &id) < 0) {
+        log_err ("Unable to parse jobid attribute '%s'", jobid);
         return -1;
     }
-    if (!(path = flux_modfind (modpath, name, module_dlerror, ctx->h))) {
-        log_msg ("%s: not found in module search path", name);
+
+    /*  Open connection to parent instance:
+     */
+    if (!(h = flux_open (parent_uri, 0))) {
+        log_err ("flux_open to parent failed");
         return -1;
     }
-    if (load_module_bypath (ctx, path, argz, argz_len, request) < 0) {
-        free (path);
-        return -1;
+
+    /*  Perform any RPCs to parent in parallel */
+    if (!(f = set_uri_job_memo (h, ctx->hostname, id, ctx->attrs)))
+        goto out;
+
+    /*  Note: not an error if rpc to set critical ranks fails, but
+     *  issue an error notifying user that no criitcal ranks are set.
+     */
+    if (!(f2 = set_critical_ranks (h, id, ctx->attrs)))
+        log_msg ("Unable to get critical ranks, all ranks will be critical");
+
+    /*  Wait for RPC results */
+    if (flux_future_get (f, NULL) < 0) {
+        log_err ("job-manager.memo uri");
+        goto out;
     }
-    free (path);
-    return 0;
+    if (f2 && flux_future_get (f2, NULL) < 0 && errno != ENOSYS) {
+        log_err ("job-exec.critical-ranks");
+        goto out;
+    }
+    rc = 0;
+out:
+    flux_close (h);
+    flux_future_destroy (f);
+    flux_future_destroy (f2);
+    return rc;
 }
 
-static int unload_module_byname (broker_ctx_t *ctx, const char *name,
-                                 const flux_msg_t *request)
+static bool nodeset_member (const char *s, uint32_t rank)
 {
-    module_t *p;
+    struct idset *ns = NULL;
+    bool member = true;
 
-    if (!(p = module_lookup_byname (ctx->modhash, name))) {
-        errno = ENOENT;
-        return -1;
+    if (s) {
+        if (!(ns = idset_decode (s)))
+            log_msg_exit ("malformed nodeset: %s", s);
+        member = idset_test (ns, rank);
+        idset_destroy (ns);
     }
-    if (module_stop (p) < 0)
-        return -1;
-    if (module_push_rmmod (p, request) < 0)
-        return -1;
-    flux_log (ctx->h, LOG_DEBUG, "rmmod %s", name);
-    return 0;
+    return member;
+}
+
+static int mod_svc_cb (flux_msg_t **msg, void *arg)
+{
+    module_t *p = arg;
+    return module_sendmsg_new (p, msg);
 }
 
 static void broker_destroy_sigwatcher (void *data)
@@ -1249,11 +1204,12 @@ static void broker_destroy_sigwatcher (void *data)
 
 static int broker_handle_signals (broker_ctx_t *ctx)
 {
-    int i, sigs[] = { SIGHUP, SIGINT, SIGQUIT, SIGTERM, SIGSEGV, SIGFPE,
-                      SIGALRM };
+    int i, sigs[] = { SIGHUP, SIGINT, SIGQUIT, SIGTERM,
+                      SIGALRM, SIGUSR1, SIGUSR2 };
+    int blocked[] = { SIGPIPE };
     flux_watcher_t *w;
 
-    for (i = 0; i < sizeof (sigs) / sizeof (sigs[0]); i++) {
+    for (i = 0; i < ARRAY_SIZE (sigs); i++) {
         w = flux_signal_watcher_create (ctx->reactor, sigs[i], signal_cb, ctx);
         if (!w) {
             log_err ("flux_signal_watcher_create");
@@ -1266,208 +1222,36 @@ static int broker_handle_signals (broker_ctx_t *ctx)
         zlist_freefn (ctx->sigwatchers, w, broker_destroy_sigwatcher, false);
         flux_watcher_start (w);
     }
-    return 0;
-}
-
-/**
- ** Built-in services
- **/
-
-/* Unload a comms module by name, asynchronously.
- * Message format is defined by RFC 5.
- * N.B. unload_module_byname() handles response, unless it fails early
- * and returns -1.
- */
-static void cmb_rmmod_cb (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg)
-{
-    broker_ctx_t *ctx = arg;
-    const char *name;
-
-    if (flux_request_unpack (msg, NULL, "{s:s}", "name", &name) < 0)
-        goto error;
-    if (unload_module_byname (ctx, name, msg) < 0)
-        goto error;
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-/* Load a comms module by name, asynchronously.
- * Message format is defined by RFC 5.
- * N.B. load_module_bypath() handles response, unless it returns -1.
- */
-static void cmb_insmod_cb (flux_t *h, flux_msg_handler_t *mh,
-                           const flux_msg_t *msg, void *arg)
-{
-    broker_ctx_t *ctx = arg;
-    const char *path;
-    json_t *args;
-    size_t index;
-    json_t *value;
-    char *argz = NULL;
-    size_t argz_len = 0;
-    error_t e;
-
-    if (flux_request_unpack (msg, NULL, "{s:s s:o}", "path", &path,
-                                                     "args", &args) < 0)
-        goto error;
-    if (!json_is_array (args))
-        goto proto;
-    json_array_foreach (args, index, value) {
-        if (!json_is_string (value))
-            goto proto;
-        if ((e = argz_add (&argz, &argz_len, json_string_value (value)))) {
-            errno = e;
-            goto error;
-        }
-    }
-    if (load_module_bypath (ctx, path, argz, argz_len, msg) < 0)
-        goto error;
-    free (argz);
-    return;
-proto:
-    errno = EPROTO;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    free (argz);
-}
-
-/* Load a comms module by name.
- * Message format is defined by RFC 5.
- */
-static void cmb_lsmod_cb (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg)
-{
-    broker_ctx_t *ctx = arg;
-    json_t *mods = NULL;
-
-    if (flux_request_decode (msg, NULL, NULL) < 0)
-        goto error;
-    if (!(mods = module_get_modlist (ctx->modhash, ctx->services)))
-        goto error;
-    if (flux_respond_pack (h, msg, "{s:O}", "mods", mods) < 0)
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-    json_decref (mods);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-static void cmb_lspeer_cb (flux_t *h, flux_msg_handler_t *mh,
-                           const flux_msg_t *msg, void *arg)
-{
-    broker_ctx_t *ctx = arg;
-    char *out;
 
-    if (!(out = overlay_lspeer_encode (ctx->overlay))) {
-        if (flux_respond_error (h, msg, errno, NULL) < 0)
-            flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-        return;
-    }
-    if (flux_respond (h, msg, out) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    free (out);
+    /*  Block the list of signals in the blocked array */
+    for (i = 0; i < ARRAY_SIZE (blocked); i++)
+        signal(blocked[i], SIG_IGN);
+    return 0;
 }
 
-#if CODE_COVERAGE_ENABLED
-void __gcov_flush (void);
-#endif
-
-static void cmb_panic_cb (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg)
-{
-    const char *reason;
-    int flags; // reserved
+/**
+ ** Built-in services
+ **/
 
-    if (flux_request_unpack (msg, NULL, "{s:s s:i}",
-                             "reason", &reason,
-                             "flags", &flags) < 0) {
-        flux_log_error (h, "malformed cmb.panic request");
-        return;
-    }
-    fprintf (stderr, "PANIC: %s\n", reason);
 #if CODE_COVERAGE_ENABLED
-    __gcov_flush ();
+void __gcov_dump (void);
+void __gcov_reset (void);
 #endif
-    _exit (1);
-    /*NOTREACHED*/
-}
-
-static void cmb_disconnect_cb (flux_t *h, flux_msg_handler_t *mh,
-                               const flux_msg_t *msg, void *arg)
-{
-    char *sender = NULL;
 
-    if (flux_msg_get_route_first (msg, &sender) == 0) {
-        exec_terminate_subprocesses_by_uuid (h, sender);
-        free (sender);
-    }
-    /* no response */
-}
-
-static void cmb_sub_cb (flux_t *h, flux_msg_handler_t *mh,
-                        const flux_msg_t *msg, void *arg)
+static void broker_disconnect_cb (flux_t *h,
+                                  flux_msg_handler_t *mh,
+                                  const flux_msg_t *msg,
+                                  void *arg)
 {
-    broker_ctx_t *ctx = arg;
-    char *uuid = NULL;
-    const char *topic;
-
-    if (flux_request_unpack (msg, NULL, "{ s:s }", "topic", &topic) < 0)
-        goto error;
-    if (flux_msg_get_route_first (msg, &uuid) < 0)
-        goto error;
-    if (!uuid) {
-        errno = EPROTO;
-        goto error;
-    }
-    if (module_subscribe (ctx->modhash, uuid, topic) < 0)
-        goto error;
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    free (uuid);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    free (uuid);
 }
 
-static void cmb_unsub_cb (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg)
+static int route_to_handle (flux_msg_t **msg, void *arg)
 {
     broker_ctx_t *ctx = arg;
-    char *uuid = NULL;
-    const char *topic;
-
-    if (flux_request_unpack (msg, NULL, "{ s:s }", "topic", &topic) < 0)
-        goto error;
-    if (flux_msg_get_route_first (msg, &uuid) < 0)
-        goto error;
-    if (!uuid) {
-        errno = EPROTO;
-        goto error;
+    if (flux_send_new (ctx->h_internal, msg, 0) < 0) {
+        flux_log_error (ctx->h, "send failed on internal broker handle");
+        return -1;
     }
-    if (module_unsubscribe (ctx->modhash, uuid, topic) < 0)
-        goto error;
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    free (uuid);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    free (uuid);
-}
-
-static int route_to_handle (const flux_msg_t *msg, void *arg)
-{
-    broker_ctx_t *ctx = arg;
-    if (flux_requeue (ctx->h, msg, FLUX_RQ_TAIL) < 0)
-        flux_log_error (ctx->h, "%s: flux_requeue\n", __FUNCTION__);
     return 0;
 }
 
@@ -1481,7 +1265,7 @@ static int service_allow (struct flux_msg_cred cred, const char *name)
     if ((cred.rolemask & FLUX_ROLE_OWNER))
         return 0;
     snprintf (prefix, sizeof (prefix), "%" PRIu32 "-", cred.userid);
-    if (!strncmp (prefix, name, strlen (prefix)))
+    if (strstarts (name, prefix))
         return 0;
     errno = EPERM;
     return -1;
@@ -1491,23 +1275,27 @@ static int service_allow (struct flux_msg_cred cred, const char *name)
  * These handlers need to appear in broker.c so that they have
  *  access to broker internals like modhash
  */
-static void service_add_cb (flux_t *h, flux_msg_handler_t *w,
-                            const flux_msg_t *msg, void *arg)
+static void service_add_cb (flux_t *h,
+                            flux_msg_handler_t *w,
+                            const flux_msg_t *msg,
+                            void *arg)
 {
     broker_ctx_t *ctx = arg;
     const char *name = NULL;
-    char *sender = NULL;
+    const char *sender;
     module_t *p;
     struct flux_msg_cred cred;
 
     if (flux_request_unpack (msg, NULL, "{ s:s }", "service", &name) < 0
-            || flux_msg_get_cred (msg, &cred) < 0)
+        || flux_msg_get_cred (msg, &cred) < 0)
         goto error;
     if (service_allow (cred, name) < 0)
         goto error;
-    if (flux_msg_get_route_first (msg, &sender) < 0)
+    if (!(sender = flux_msg_route_first (msg))) {
+        errno = EPROTO;
         goto error;
-    if (!(p = module_lookup (ctx->modhash, sender))) {
+    }
+    if (!(p = modhash_lookup (ctx->modhash, sender))) {
         errno = ENOENT;
         goto error;
     }
@@ -1515,97 +1303,67 @@ static void service_add_cb (flux_t *h, flux_msg_handler_t *w,
         goto error;
     if (flux_respond (h, msg, NULL) < 0)
         flux_log_error (h, "service_add: flux_respond");
-    free (sender);
     return;
 error:
     if (flux_respond_error (h, msg, errno, NULL) < 0)
         flux_log_error (h, "service_add: flux_respond_error");
-    free (sender);
 }
 
-static void service_remove_cb (flux_t *h, flux_msg_handler_t *w,
-                               const flux_msg_t *msg, void *arg)
+static void service_remove_cb (flux_t *h,
+                               flux_msg_handler_t *w,
+                               const flux_msg_t *msg,
+                               void *arg)
 {
     broker_ctx_t *ctx = arg;
     const char *name;
     const char *uuid;
-    char *sender = NULL;
+    const char *sender;
     struct flux_msg_cred cred;
 
     if (flux_request_unpack (msg, NULL, "{ s:s }", "service", &name) < 0
-            || flux_msg_get_cred (msg, &cred) < 0)
+        || flux_msg_get_cred (msg, &cred) < 0)
         goto error;
     if (service_allow (cred, name) < 0)
         goto error;
-    if (flux_msg_get_route_first (msg, &sender) < 0)
+    if (!(sender = flux_msg_route_first (msg))) {
+        errno = EPROTO;
         goto error;
+    }
     if (!(uuid = service_get_uuid (ctx->services, name))) {
         errno = ENOENT;
         goto error;
     }
-    if (strcmp (uuid, sender) != 0) {
+    if (!streq (uuid, sender)) {
         errno = EINVAL;
         goto error;
     }
     service_remove (ctx->services, name);
     if (flux_respond (h, msg, NULL) < 0)
         flux_log_error (h, "service_remove: flux_respond");
-    free (sender);
     return;
 error:
     if (flux_respond_error (h, msg, errno, NULL) < 0)
         flux_log_error (h, "service_remove: flux_respond_error");
-    free (sender);
 }
 
 
 static const struct flux_msg_handler_spec htab[] = {
     {
         FLUX_MSGTYPE_REQUEST,
-        "cmb.rmmod",
-        cmb_rmmod_cb,
-        0
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "cmb.insmod",
-        cmb_insmod_cb,
-        0
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "cmb.lsmod",
-        cmb_lsmod_cb,
-        0
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "cmb.lspeer",
-        cmb_lspeer_cb,
-        0
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "cmb.panic",
-        cmb_panic_cb,
-        0
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "cmb.disconnect",
-        cmb_disconnect_cb,
-        0
+        "broker.rusage",
+        method_rusage_cb,
+        FLUX_ROLE_USER
     },
     {
         FLUX_MSGTYPE_REQUEST,
-        "cmb.sub",
-        cmb_sub_cb,
-        0
+        "broker.ping",
+        method_ping_cb,
+        FLUX_ROLE_USER
     },
     {
         FLUX_MSGTYPE_REQUEST,
-        "cmb.unsub",
-        cmb_unsub_cb,
+        "broker.disconnect",
+        broker_disconnect_cb,
         0
     },
     {
@@ -1629,20 +1387,25 @@ struct internal_service {
 };
 
 static struct internal_service services[] = {
-    { "cmb",                NULL }, // kind of a catch-all, slowly deprecating
+    { "broker",             NULL }, // kind of a catch-all, slowly deprecating
     { "log",                NULL },
-    { "seq",                "[0]" },
-    { "content",            NULL },
-    { "hello",              NULL },
     { "attr",               NULL },
     { "heaptrace",          NULL },
-    { "event",              "[0]" },
+    { "event",              NULL },
     { "service",            NULL },
+    { "overlay",            NULL },
+    { "module",             NULL },
+    { "config",             NULL },
+    { "runat",              NULL },
+    { "state-machine",      NULL },
+    { "groups",             NULL },
+    { "shutdown",           NULL },
+    { "rexec",              NULL },
     { NULL, NULL, },
 };
 
 /* Register builtin services (sharing ctx->h and broker thread).
- * Register message handlers for some cmb services.  Others are registered
+ * Register message handlers for some broker services.  Others are registered
  * in their own initialization functions.
  */
 static flux_msg_handler_t **broker_add_services (broker_ctx_t *ctx)
@@ -1650,10 +1413,12 @@ static flux_msg_handler_t **broker_add_services (broker_ctx_t *ctx)
     flux_msg_handler_t **handlers;
     struct internal_service *svc;
     for (svc = &services[0]; svc->name != NULL; svc++) {
-        if (!nodeset_member (svc->nodeset, overlay_get_rank (ctx->overlay)))
+        if (!nodeset_member (svc->nodeset, ctx->rank))
             continue;
-        if (service_add (ctx->services, svc->name, NULL,
-                         route_to_handle, ctx) < 0) {
+        if (service_add (ctx->services,
+                         svc->name, NULL,
+                         route_to_handle,
+                         ctx) < 0) {
             log_err ("error registering service for %s", svc->name);
             return NULL;
         }
@@ -1677,62 +1442,68 @@ static void broker_remove_services (flux_msg_handler_t *handlers[])
  ** reactor callbacks
  **/
 
-
-/* Handle requests from overlay peers.
+/* Handle messages received from overlay peers.
  */
-static void child_cb (overlay_t *ov, void *sock, void *arg)
+static int overlay_recv_cb (flux_msg_t **msg, overlay_where_t where, void *arg)
 {
     broker_ctx_t *ctx = arg;
     int type;
-    char *uuid = NULL;
-    flux_msg_t *msg = flux_msg_recvzsock (sock);
 
-    if (!msg)
-        goto done;
-    if (flux_msg_get_type (msg, &type) < 0)
-        goto done;
-    if (flux_msg_get_route_last (msg, &uuid) < 0)
-        goto done;
-    overlay_checkin_child (ctx->overlay, uuid);
+    if (flux_msg_get_type (*msg, &type) < 0)
+        return -1;
     switch (type) {
-        case FLUX_MSGTYPE_KEEPALIVE:
-            break;
         case FLUX_MSGTYPE_REQUEST:
-            broker_request_sendmsg (ctx, msg);
+            /* broker_request_sendmsg_new() generates a response on error.
+             */
+            broker_request_sendmsg_new (ctx, msg);
             break;
         case FLUX_MSGTYPE_RESPONSE:
-            /* TRICKY:  Fix up ROUTER socket used in reverse direction.
-             * Request/response is designed for requests to travel
-             * ROUTER->DEALER (up) and responses DEALER-ROUTER (down).
-             * When used conventionally, the route stack is accumulated
-             * automatically as a request is routed up, and unwound
-             * automatically as a response is routed down.  When responses
-             * are routed up, ROUTER socket behavior must be subverted on
-             * the receiving end by popping two frames off of the stack and
-             * discarding.
-             */
-            (void)flux_msg_pop_route (msg, NULL);
-            (void)flux_msg_pop_route (msg, NULL);
-            if (broker_response_sendmsg (ctx, msg) < 0)
-                goto done;
+            if (broker_response_sendmsg_new (ctx, msg) < 0)
+                goto drop;
             break;
         case FLUX_MSGTYPE_EVENT:
-            (void)broker_event_sendmsg (ctx, msg);
+            /* If event originated from upstream peer, then it has already been
+             * published and we are to continue its distribution.
+             * Otherwise, take the next step to get the event published.
+             */
+            if (where == OVERLAY_UPSTREAM) {
+                if (handle_event (ctx, *msg) < 0)
+                    goto drop;
+            }
+            else {
+                if (broker_event_sendmsg_new (ctx, msg) < 0)
+                    goto drop;
+            }
+            break;
+        default:
             break;
     }
-done:
-    if (uuid)
-        free (uuid);
-    flux_msg_destroy (msg);
+    flux_msg_decref (*msg);
+    *msg = NULL;
+    return 0;
+drop:
+    /* Suppress logging if a response could not be sent due to ENOSYS,
+     * which happens if sending module unloads before finishing all RPCs.
+     */
+    if (type != FLUX_MSGTYPE_RESPONSE || errno != ENOSYS) {
+        const char *topic = "unknown";
+        (void)flux_msg_get_topic (*msg, &topic);
+        flux_log_error (ctx->h,
+                        "DROP %s %s topic=%s",
+                        where == OVERLAY_UPSTREAM ? "upstream" : "downstream",
+                        flux_msg_typestr (type),
+                        topic);
+    }
+    return -1;
 }
 
-/* Handle events received by parent_cb.
+/* Distribute events downstream, and to module and broker-resident subscribers.
  * On rank 0, publisher is wired to send events here also.
  */
 static int handle_event (broker_ctx_t *ctx, const flux_msg_t *msg)
 {
     uint32_t seq;
-    const char *topic, *s;
+    const char *topic;
 
     if (flux_msg_get_seq (msg, &seq) < 0
             || flux_msg_get_topic (msg, &topic) < 0) {
@@ -1755,281 +1526,123 @@ static int handle_event (broker_ctx_t *ctx, const flux_msg_t *msg)
 
     /* Forward to this rank's children.
      */
-    if (overlay_mcast_child (ctx->overlay, msg) < 0)
-        flux_log_error (ctx->h, "%s: overlay_mcast_child", __FUNCTION__);
+    overlay_sendmsg (ctx->overlay, msg, OVERLAY_DOWNSTREAM);
 
     /* Internal services may install message handlers for events.
      */
-    s = zlist_first (ctx->subscriptions);
-    while (s) {
-        if (!strncmp (s, topic, strlen (s))) {
-            if (flux_requeue (ctx->h, msg, FLUX_RQ_TAIL) < 0)
-                flux_log_error (ctx->h, "%s: flux_requeue\n", __FUNCTION__);
-            break;
-        }
-        s = zlist_next (ctx->subscriptions);
+    if (subhash_topic_match (ctx->sub, topic)) {
+        if (flux_send (ctx->h_internal, msg, 0) < 0)
+            flux_log_error (ctx->h, "send failed on internal broker handle");
     }
     /* Finally, route to local module subscribers.
      */
-    return module_event_mcast (ctx->modhash, msg);
+    return modhash_event_mcast (ctx->modhash, msg);
 }
 
-/* Handle messages from one or more parents.
- */
-static void parent_cb (overlay_t *ov, void *sock, void *arg)
+static bool signal_is_deadly (int signum)
 {
-    broker_ctx_t *ctx = arg;
-    flux_msg_t *msg = flux_msg_recvzsock (sock);
-    int type;
-
-    if (!msg)
-        goto done;
-    if (flux_msg_get_type (msg, &type) < 0)
-        goto done;
-    switch (type) {
-        case FLUX_MSGTYPE_RESPONSE:
-            if (broker_response_sendmsg (ctx, msg) < 0)
-                goto done;
-            break;
-        case FLUX_MSGTYPE_EVENT:
-            if (flux_msg_clear_route (msg) < 0) {
-                flux_log (ctx->h, LOG_ERR, "dropping malformed event");
-                goto done;
-            }
-            if (handle_event (ctx, msg) < 0)
-                goto done;
-            break;
-        case FLUX_MSGTYPE_REQUEST:
-            broker_request_sendmsg (ctx, msg);
-            break;
-        default:
-            flux_log (ctx->h, LOG_ERR, "%s: unexpected %s", __FUNCTION__,
-                      flux_msg_typestr (type));
-            break;
+    int deadly_sigs[] = { SIGHUP, SIGINT, SIGQUIT, SIGTERM, SIGALRM };
+    for (int i = 0; i < ARRAY_SIZE (deadly_sigs); i++) {
+        if (signum == deadly_sigs[i])
+            return true;
     }
-done:
-    flux_msg_destroy (msg);
+    return false;
 }
 
-/* Handle messages on the service socket of a comms module.
- */
-static void module_cb (module_t *p, void *arg)
+static void killall_cb (flux_future_t *f, void *arg)
 {
     broker_ctx_t *ctx = arg;
-    flux_msg_t *msg = module_recvmsg (p);
-    int type;
-    int ka_errnum, ka_status;
-
-    if (!msg)
-        goto done;
-    if (flux_msg_get_type (msg, &type) < 0)
-        goto done;
-    switch (type) {
-        case FLUX_MSGTYPE_RESPONSE:
-            (void)broker_response_sendmsg (ctx, msg);
-            break;
-        case FLUX_MSGTYPE_REQUEST:
-            broker_request_sendmsg (ctx, msg);
-            break;
-        case FLUX_MSGTYPE_EVENT:
-            if (broker_event_sendmsg (ctx, msg) < 0) {
-                flux_log_error (ctx->h, "%s(%s): broker_event_sendmsg %s",
-                                __FUNCTION__, module_get_name (p),
-                                flux_msg_typestr (type));
-            }
-            break;
-        case FLUX_MSGTYPE_KEEPALIVE:
-            if (flux_keepalive_decode (msg, &ka_errnum, &ka_status) < 0) {
-                flux_log_error (ctx->h, "%s: flux_keepalive_decode",
-                                module_get_name (p));
-                break;
-            }
-            if (ka_status == FLUX_MODSTATE_FINALIZING) {
-                /* Module is finalizing and doesn't want any more messages.
-                 * mute the module and respond with the same keepalive
-                 * message for synchronization (module waits to proceed)
-                 */
-                module_mute (p);
-                if (module_sendmsg (p, msg) < 0)
-                    flux_log_error (ctx->h,
-                                    "%s: reply to finalizing: module_sendmsg",
-                                    module_get_name (p));
-            }
-            if (ka_status == FLUX_MODSTATE_EXITED)
-                module_set_errnum (p, ka_errnum);
-            module_set_status (p, ka_status);
-            break;
-        default:
-            flux_log (ctx->h, LOG_ERR, "%s(%s): unexpected %s",
-                      __FUNCTION__, module_get_name (p),
-                      flux_msg_typestr (type));
-            break;
+    int count = 0;
+    if (flux_rpc_get_unpack (f, "{s:i}", "count", &count) < 0) {
+        flux_log_error (ctx->h,
+                        "job-manager.killall: %s",
+                        future_strerror (f, errno));
+    }
+    flux_future_destroy (f);
+    if (count) {
+        flux_log (ctx->h,
+                  LOG_INFO,
+                  "forwarded signal %d to %d jobs",
+                  (int) ptr2int (flux_future_aux_get (f, "signal")),
+                  count);
     }
-done:
-    flux_msg_destroy (msg);
-}
-
-static int module_insmod_respond (flux_t *h, module_t *p)
-{
-    int rc;
-    int errnum = 0;
-    int status = module_get_status (p);
-    flux_msg_t *msg = module_pop_insmod (p);
-
-    if (msg == NULL)
-        return 0;
-
-    /* If the module is EXITED, return error to insmod if mod_main() < 0
-     */
-    if (status == FLUX_MODSTATE_EXITED)
-        errnum = module_get_errnum (p);
-    if (errnum == 0)
-        rc = flux_respond (h, msg, NULL);
-    else
-        rc = flux_respond_error (h, msg, errnum, NULL);
-
-    flux_msg_destroy (msg);
-    return rc;
 }
 
-static int module_rmmod_respond (flux_t *h, module_t *p)
+static int killall_jobs (broker_ctx_t *ctx, int signum)
 {
-    flux_msg_t *msg;
-    int rc = 0;
-    while ((msg = module_pop_rmmod (p))) {
-        if (flux_respond (h, msg, NULL) < 0)
-            rc = -1;
-        flux_msg_destroy (msg);
+    flux_future_t *f = NULL;
+    if (!(f = flux_rpc_pack (ctx->h,
+                             "job-manager.killall",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:b s:i s:i}",
+                             "dry_run", 0,
+                             "userid", FLUX_USERID_UNKNOWN,
+                             "signum", signum))
+        || flux_future_then (f, -1., killall_cb, ctx) < 0) {
+        flux_future_destroy (f);
+        return -1;
     }
-    return rc;
+    if (flux_future_aux_set (f, "signum", int2ptr (signum), NULL) < 0)
+        flux_log_error (ctx->h, "killall: future_aux_set");
+    return 0;
 }
 
-static void module_status_cb (module_t *p, int prev_status, void *arg)
+static void signal_cb (flux_reactor_t *r,
+                       flux_watcher_t *w,
+                       int revents,
+                       void *arg)
 {
     broker_ctx_t *ctx = arg;
-    int status = module_get_status (p);
-    const char *name = module_get_name (p);
-
-    /* Transition from INIT 
-     * If module started normally, i.e. INIT->SLEEPING/RUNNING, then
-     * respond to insmod requests now. O/w, delay responses until
-     * EXITED, when any errnum is available.
-     */
-    if (prev_status == FLUX_MODSTATE_INIT &&
-        (status == FLUX_MODSTATE_RUNNING ||
-         status == FLUX_MODSTATE_SLEEPING)) {
-        if (module_insmod_respond (ctx->h, p) < 0)
-            flux_log_error (ctx->h, "flux_respond to insmod %s", name);
-    }
-
-    /* Transition to EXITED
-     * Remove service routes, respond to insmod & rmmod request(s), if any,
-     * and remove the module (which calls pthread_join).
-     */
-    if (status == FLUX_MODSTATE_EXITED) {
-        flux_log (ctx->h, LOG_DEBUG, "module %s exited", name);
-        service_remove_byuuid (ctx->services, module_get_uuid (p));
-
-        if (module_insmod_respond (ctx->h, p) < 0)
-            flux_log_error (ctx->h, "flux_respond to insmod %s", name);
+    int signum = flux_signal_watcher_get_signum (w);
 
-        if (module_rmmod_respond (ctx->h, p) < 0)
-            flux_log_error (ctx->h, "flux_respond to rmmod %s", name);
+    flux_log (ctx->h, LOG_INFO, "signal %d", signum);
 
-        /* Special case for connector-local removal:
-         * If shutdown is complete, stop the reactor.
+    if (ctx->rank == 0 && !signal_is_deadly (signum)) {
+        /* Attempt to forward non-deadly signals to jobs. If that fails,
+         * then fall through to state_machine_kill() so the signal is
+         * delivered somewhere.
          */
-        if (!strcmp (name, "connector-local")) {
-            if (shutdown_is_complete (ctx->shutdown))
-                flux_reactor_stop (flux_get_reactor (ctx->h));
-        }
-
-        module_remove (ctx->modhash, p);
-    }
-}
-
-static void signal_cb (flux_reactor_t *r, flux_watcher_t *w,
-                         int revents, void *arg)
-{
-    broker_ctx_t *ctx = arg;
-    int rank = overlay_get_rank (ctx->overlay);
-    int signum = flux_signal_watcher_get_signum (w);
-    int level;
-
-    if (rank > 0) {
-        flux_log (ctx->h, LOG_INFO, "signal %d ignored on rank > 0", signum);
-        return;
-    }
-    if ((level = runlevel_get_level (ctx->runlevel)) == 3) {
-        flux_log (ctx->h, LOG_INFO, "signal %d ignored in run level 3", signum);
-        return;
+        if (killall_jobs (ctx, signum) == 0)
+            return;
+        /*
+         * Note: flux_rpc(3) in the rank 0 broker to the job manager module
+         *  is expected to fail immediately if the job-manager module is not
+         *  loaded due to the broker internal flux_t handle implementation.
+         */
+        flux_log (ctx->h,
+                  LOG_INFO,
+                  "killall failed, delivering signal %d locally instead",
+                  signum);
     }
-    flux_log (ctx->h, LOG_INFO, "signal %d aborting runlevel %d", signum, level);
-    runlevel_abort (ctx->runlevel);
-}
-
-/* Send a request message down the TBON.
- * N.B. this message is going from ROUTER socket to DEALER socket.
- * Since ROUTER pops a route off the stack and uses it to select the peer,
- * we must push *two* routes on the stack: the identity of this broker,
- * then the identity the peer.  The parent_cb() can then accept the request
- * from DEALER as though it were received on ROUTER.
- */
-static int sendmsg_child_request (broker_ctx_t *ctx,
-                                  const flux_msg_t *msg,
-                                  uint32_t nodeid)
-{
-    flux_msg_t *cpy = flux_msg_copy (msg, true);
-    int saved_errno;
-    char uuid[16];
-    int rc = -1;
-
-    snprintf (uuid, sizeof (uuid), "%"PRIu32, overlay_get_rank (ctx->overlay));
-    if (flux_msg_push_route (cpy, uuid) < 0)
-        goto done;
-    snprintf (uuid, sizeof (uuid), "%"PRIu32, nodeid);
-    if (flux_msg_push_route (cpy, uuid) < 0)
-        goto done;
-    if (overlay_sendmsg_child (ctx->overlay, cpy) < 0)
-        goto done;
-    rc = 0;
-done:
-    saved_errno = errno;
-    flux_msg_destroy (cpy);
-    errno = saved_errno;
-    return rc;
+    state_machine_kill (ctx->state_machine, signum);
 }
 
 /* Route request.
  * On success, return 0.  On failure, return -1 with errno set.
  */
-static int broker_request_sendmsg_internal (broker_ctx_t *ctx,
-                                            const flux_msg_t *msg)
+static int broker_request_sendmsg_new_internal (broker_ctx_t *ctx,
+                                                flux_msg_t **msg)
 {
-    uint32_t rank = overlay_get_rank (ctx->overlay);
-    uint32_t size = overlay_get_size (ctx->overlay);
+    bool upstream = flux_msg_has_flag (*msg, FLUX_MSGFLAG_UPSTREAM);
     uint32_t nodeid;
-    uint8_t flags;
 
-    if (flux_msg_get_nodeid (msg, &nodeid) < 0)
-        return -1;
-    if (flux_msg_get_flags (msg, &flags) < 0)
+    if (flux_msg_get_nodeid (*msg, &nodeid) < 0)
         return -1;
     /* Route up TBON if destination if upstream of this broker.
      */
-    if ((flags & FLUX_MSGFLAG_UPSTREAM) && nodeid == rank) {
-        if (overlay_sendmsg_parent (ctx->overlay, msg) < 0)
+    if (upstream && nodeid == ctx->rank) {
+        if (overlay_sendmsg_new (ctx->overlay, msg, OVERLAY_UPSTREAM) < 0)
             return -1;
     }
     /* Deliver to local service if destination *could* be this broker.
      * If there is no such service locally (ENOSYS), route up TBON.
      */
-    else if (((flags & FLUX_MSGFLAG_UPSTREAM) && nodeid != rank)
-                                              || nodeid == FLUX_NODEID_ANY) {
-        if (service_send (ctx->services, msg) < 0) {
+    else if ((upstream && nodeid != ctx->rank) || nodeid == FLUX_NODEID_ANY) {
+        if (service_send_new (ctx->services, msg) < 0) {
             if (errno != ENOSYS)
                 return -1;
-            if (overlay_sendmsg_parent (ctx->overlay, msg) < 0) {
+            if (overlay_sendmsg_new (ctx->overlay, msg, OVERLAY_UPSTREAM) < 0) {
                 if (errno == EHOSTUNREACH)
                     errno = ENOSYS;
                 return -1;
@@ -2038,23 +1651,15 @@ static int broker_request_sendmsg_internal (broker_ctx_t *ctx,
     }
     /* Deliver to local service if this broker is the addressed rank.
      */
-    else if (nodeid == rank) {
-        if (service_send (ctx->services, msg) < 0)
+    else if (nodeid == ctx->rank) {
+        if (service_send_new (ctx->services, msg) < 0)
             return -1;
     }
     /* Send the request up or down TBON as addressed.
      */
     else {
-        uint32_t down_rank;
-        down_rank = kary_child_route (ctx->tbon_k, size, rank, nodeid);
-        if (down_rank == KARY_NONE) { // up
-            if (overlay_sendmsg_parent (ctx->overlay, msg) < 0)
-                return -1;
-        }
-        else { // down
-            if (sendmsg_child_request (ctx, msg, down_rank) < 0)
-                return -1;
-        }
+        if (overlay_sendmsg_new (ctx->overlay, msg, OVERLAY_ANY) < 0)
+            return -1;
     }
     return 0;
 }
@@ -2063,213 +1668,110 @@ static int broker_request_sendmsg_internal (broker_ctx_t *ctx,
  * generate an error response.  Make an extra effort to return a useful
  * error message if ENOSYS indicates an unmatched service name.
  */
-static void broker_request_sendmsg (broker_ctx_t *ctx, const flux_msg_t *msg)
+void broker_request_sendmsg_new (broker_ctx_t *ctx, flux_msg_t **msg)
 {
-    if (broker_request_sendmsg_internal (ctx, msg) < 0) {
+    if (broker_request_sendmsg_new_internal (ctx, msg) < 0) {
         const char *topic;
         char errbuf[64];
         const char *errstr = NULL;
 
-        if (errno == ENOSYS && flux_msg_get_topic (msg, &topic) == 0) {
+        if (errno == ENOSYS && flux_msg_get_topic (*msg, &topic) == 0) {
             snprintf (errbuf,
                       sizeof (errbuf),
                       "No service matching %s is registered", topic);
             errstr = errbuf;
         }
-        if (flux_respond_error (ctx->h, msg, errno, errstr) < 0)
+        if (flux_respond_error (ctx->h, *msg, errno, errstr) < 0)
             flux_log_error (ctx->h, "flux_respond");
+        flux_msg_decref (*msg);
+        *msg = NULL;
     }
 }
 
-/* Broker's use their rank in place of a UUID for message routing purposes.
- * Try to convert a UUID from a message to a rank.
- * It must be entirely numerical, and be less than 'size'.
- * If it works, assign result to 'rank' and return true.
- * If it doesn't return false.
- */
-static bool uuid_to_rank (const char *s, uint32_t size, uint32_t *rank)
-{
-    unsigned long num;
-    char *endptr;
-
-    if (!isdigit (*s))
-        return false;
-    errno = 0;
-    num = strtoul (s, &endptr, 10);
-    if (errno != 0)
-        return false;
-    if (*endptr != '\0')
-        return false;
-    if (num >= size)
-        return false;
-    *rank = num;
-    return true;
-}
-
-/* Test whether the TBON parent of this broker is 'rank'.
- */
-static bool is_my_parent (broker_ctx_t *ctx, uint32_t rank)
-{
-    if (kary_parentof (ctx->tbon_k, overlay_get_rank (ctx->overlay)) == rank)
-        return true;
-    return false;
-}
-
 /* Route a response message, determining next hop from route stack.
  * If there is no next hop, routing is complete to broker-resident service.
- * If the next hop is a rank, route up or down the TBON.
- * If not a rank, look up a comms module by uuid.
+ * If the next hop is an overlay peer, route up or down the TBON.
+ * If not a peer, look up a module by uuid.
  */
-static int broker_response_sendmsg (broker_ctx_t *ctx, const flux_msg_t *msg)
+int broker_response_sendmsg_new (broker_ctx_t *ctx, flux_msg_t **msg)
 {
-    int rc = -1;
-    char *uuid = NULL;
-    uint32_t rank;
+    const char *uuid;
 
-    if (flux_msg_get_route_last (msg, &uuid) < 0)
-        goto done;
-    if (uuid == NULL) { // broker resident service
-        if (flux_requeue (ctx->h, msg, FLUX_RQ_TAIL) < 0)
-            goto done;
+    if (!(uuid = flux_msg_route_last (*msg))) {
+        if (flux_send_new (ctx->h_internal, msg, 0) < 0)
+            return -1;
     }
-    else if (uuid_to_rank (uuid, overlay_get_size (ctx->overlay), &rank)) {
-        if (is_my_parent (ctx, rank)) {
-            /* N.B. this message is going from DEALER socket to ROUTER socket.
-             * Instead of popping a route off the stack, ROUTER pushes one
-             * on, so the upstream broker must to detect this case and pop
-             * *two* off to maintain route stack integrity.  See child_cb().
-             */
-            if (overlay_sendmsg_parent (ctx->overlay, msg) < 0)
-                goto done;
-        }
-        else {
-            if (overlay_sendmsg_child (ctx->overlay, msg) < 0) {
-                if (errno == EINVAL)
-                    errno = EHOSTUNREACH;
-                goto done;
-            }
-        }
+    else if (overlay_uuid_is_parent (ctx->overlay, uuid)) {
+        if (overlay_sendmsg_new (ctx->overlay, msg, OVERLAY_UPSTREAM) < 0)
+            return -1;
+    }
+    else if (overlay_uuid_is_child (ctx->overlay, uuid)) {
+        if (overlay_sendmsg_new (ctx->overlay, msg, OVERLAY_DOWNSTREAM) < 0)
+            return -1;
     }
     else {
-        if (module_response_sendmsg (ctx->modhash, msg) < 0)
-            goto done;
+        if (modhash_response_sendmsg_new (ctx->modhash, msg) < 0)
+            return -1;
     }
-    rc = 0;
-done:
-    ERRNO_SAFE_WRAP (free, uuid);
-    return rc;
+    return 0;
 }
 
-/* Events are forwarded up the TBON to rank 0, then published from there.
- * (This mechanism predates and is separate from the "event.pub" service).
+/* Events are forwarded up the TBON to rank 0, then published per RFC 3.
+ * An alternate publishing mechanism that allows the event sequence number
+ * to be obtained is to send an RPC to event.pub.
  */
-static int broker_event_sendmsg (broker_ctx_t *ctx, const flux_msg_t *msg)
+int broker_event_sendmsg_new (broker_ctx_t *ctx, flux_msg_t **msg)
 {
-
-    if (overlay_get_rank (ctx->overlay) > 0) {
-        flux_msg_t *cpy;
-        if (!(cpy = flux_msg_copy (msg, true)))
-            return -1;
-        if (flux_msg_enable_route (cpy) < 0) {
-            flux_msg_destroy (cpy);
-            return -1;
-        }
-        if (overlay_sendmsg_parent (ctx->overlay, cpy) < 0) {
-            flux_msg_destroy (cpy);
+    if (ctx->rank > 0) {
+        if (overlay_sendmsg_new (ctx->overlay, msg, OVERLAY_UPSTREAM) < 0)
             return -1;
-        }
-        flux_msg_destroy (cpy);
-    } else {
-        if (publisher_send (ctx->publisher, msg) < 0)
+    }
+    else {
+        if (publisher_send (ctx->publisher, *msg) < 0)
             return -1;
+        flux_msg_decref (*msg);
+        *msg = NULL;
     }
     return 0;
 }
 
-/**
- ** Broker's internal flux_t implementation
- ** N.B. recv() method is missing because messages are "received"
- ** when routing logic calls flux_requeue().
- **/
-
-static int broker_send (void *impl, const flux_msg_t *msg, int flags)
+/* Handle messages received from the "router end" of the back to back
+ * interthread handles.  Hand the message off to routing logic.
+ */
+static void h_internal_watcher (flux_reactor_t *r,
+                                flux_watcher_t *w,
+                                int revents,
+                                void *arg)
 {
-    broker_ctx_t *ctx = impl;
+    flux_t *h = flux_handle_watcher_get_flux (w);
+    broker_ctx_t *ctx = arg;
+    flux_msg_t *msg;
     int type;
-    struct flux_msg_cred cred;
-    flux_msg_t *cpy = NULL;
-    int rc = -1;
-
-    if (!(cpy = flux_msg_copy (msg, true)))
-        goto done;
-    if (flux_msg_get_type (cpy, &type) < 0)
-        goto done;
-    if (flux_msg_get_cred (cpy, &cred) < 0)
-        goto done;
-    if (cred.userid == FLUX_USERID_UNKNOWN)
-        cred.userid = ctx->cred.userid;
-    if (cred.rolemask == FLUX_ROLE_NONE)
-        cred.rolemask = ctx->cred.rolemask;
-    if (flux_msg_set_cred (cpy, cred) < 0)
-        goto done;
 
+    if (!(msg = flux_recv (h, FLUX_MATCH_ANY, 0))
+        || flux_msg_get_type (msg, &type) < 0)
+        goto error;
     switch (type) {
         case FLUX_MSGTYPE_REQUEST:
-            rc = broker_request_sendmsg_internal (ctx, cpy);
+            if (broker_request_sendmsg_new_internal (ctx, &msg) < 0)
+                goto error;
             break;
         case FLUX_MSGTYPE_RESPONSE:
-            rc = broker_response_sendmsg (ctx, cpy);
+            if (broker_response_sendmsg_new (ctx, &msg) < 0)
+                goto error;
             break;
         case FLUX_MSGTYPE_EVENT:
-            rc = broker_event_sendmsg (ctx, cpy);
+            if (broker_event_sendmsg_new (ctx, &msg) < 0)
+                goto error;
             break;
         default:
-            errno = EINVAL;
-            break;
-    }
-done:
-    flux_msg_destroy (cpy);
-    return rc;
-}
-
-static int broker_subscribe (void *impl, const char *topic)
-{
-    broker_ctx_t *ctx = impl;
-    char *cpy = NULL;
-
-    if (!(cpy = strdup (topic)))
-        goto nomem;
-    if (zlist_append (ctx->subscriptions, cpy) < 0)
-        goto nomem;
-    zlist_freefn (ctx->subscriptions, cpy, free, true);
-    return 0;
-nomem:
-    free (cpy);
-    errno = ENOMEM;
-    return -1;
-}
-
-static int broker_unsubscribe (void *impl, const char *topic)
-{
-    broker_ctx_t *ctx = impl;
-    char *s = zlist_first (ctx->subscriptions);
-    while (s) {
-        if (!strcmp (s, topic)) {
-            zlist_remove (ctx->subscriptions, s);
-            break;
-        }
-        s = zlist_next (ctx->subscriptions);
+            goto error;
     }
-    return 0;
+    return;
+error:
+    flux_msg_destroy (msg);
 }
 
-static const struct flux_handle_ops broker_handle_ops = {
-    .send = broker_send,
-    .event_subscribe = broker_subscribe,
-    .event_unsubscribe = broker_unsubscribe,
-};
-
-
 #if HAVE_VALGRIND
 /* Disable dlclose() during valgrind operation
  */
diff --git a/src/broker/broker.h b/src/broker/broker.h
new file mode 100644
index 000000000000..c51c91e9442f
--- /dev/null
+++ b/src/broker/broker.h
@@ -0,0 +1,74 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _BROKER_H
+#define _BROKER_H
+
+#include <limits.h>
+#include <flux/optparse.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+struct broker {
+    void *zctx;
+    flux_t *h;
+    flux_t *h_internal;
+    flux_watcher_t *w_internal;
+    flux_reactor_t *reactor;
+    optparse_t *opts;
+    char hostname[_POSIX_HOST_NAME_MAX + 1];
+
+    struct overlay *overlay;
+    uint32_t rank;
+    uint32_t size;
+
+    bool online;
+
+    double starttime;
+
+    struct broker_attr *attrs;
+    struct flux_msg_cred cred;  /* instance owner */
+
+    struct modhash *modhash;
+
+    int verbose;
+    int event_recv_seq;
+    zlist_t *sigwatchers;
+    struct service_switch *services;
+    struct brokercfg *config;
+    double heartbeat_rate;
+    struct subhash *sub;        /* subscriptions for internal services */
+    struct content_cache *cache;
+    struct publisher *publisher;
+    struct groups *groups;
+
+    struct runat *runat;
+    struct state_machine *state_machine;
+    struct shutdown *shutdown;
+
+    char *init_shell_cmd;
+    size_t init_shell_cmd_len;
+
+    bool sd_notify;
+
+    int exit_rc;
+};
+
+typedef struct broker broker_ctx_t;
+
+int broker_event_sendmsg_new (broker_ctx_t *ctx, flux_msg_t **msg);
+int broker_response_sendmsg_new (broker_ctx_t *ctx, flux_msg_t **msg);
+void broker_request_sendmsg_new (broker_ctx_t *ctx, flux_msg_t **msg);
+
+#endif /* !_BROKER_H */
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/broker/brokercfg.c b/src/broker/brokercfg.c
new file mode 100644
index 000000000000..69ceaece79eb
--- /dev/null
+++ b/src/broker/brokercfg.c
@@ -0,0 +1,327 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* brokercfg.c - broker configuration
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/errprintf.h"
+
+#include "attr.h"
+#include "modhash.h"
+#include "brokercfg.h"
+
+
+struct brokercfg {
+    flux_t *h;
+    char *path;
+    flux_msg_handler_t **handlers;
+    modhash_t *modhash;
+    flux_future_t *reload_f;
+};
+
+static int brokercfg_set (flux_t *h,
+                          const flux_conf_t *conf,
+                          flux_error_t *error)
+{
+    if (flux_set_conf (h, conf) < 0) {
+        errprintf (error, "Error caching config object");
+        return -1;
+    }
+    return 0;
+}
+
+/* Parse config object from TOML config files if path is set;
+ * otherwise, create an empty config object.  Store the object
+ * in ctx->h for later access by flux_get_conf().
+ */
+static int brokercfg_parse (flux_t *h,
+                            const char *path,
+                            flux_error_t *errp)
+{
+    flux_error_t error;
+    flux_conf_t *conf;
+
+    if (path) {
+        if (!(conf = flux_conf_parse (path, &error))) {
+            errprintf (errp,
+                       "Config file error: %s",
+                       error.text);
+            return -1;
+        }
+    }
+    else {
+        if (!(conf = flux_conf_create ())) {
+            errprintf (errp, "Error creating config object");
+            return -1;
+        }
+    }
+    if (brokercfg_set (h, conf, errp) < 0)
+        goto error;
+    return 0;
+error:
+    flux_conf_decref (conf);
+    return -1;
+}
+
+/* Now that all modules have responded to '<name>.config-reload' request,
+ * send a response to the original broker load/reload request.  If errors
+ * occurred (other than ENOSYS), include as much diagnostic info as possible
+ * in the response.
+ */
+static void reload_continuation (flux_future_t *cf, void *arg)
+{
+    const flux_msg_t *msg = flux_future_aux_get (cf, "flux::request");
+    struct brokercfg *cfg = arg;
+    const char *name;
+    flux_future_t *f;
+    char errbuf[4096];
+    int errnum = 0;
+    size_t offset = 0;
+
+    name = flux_future_first_child (cf);
+    while (name) {
+        f = flux_future_get_child (cf, name);
+
+        if (flux_future_get (f, NULL) < 0 && errno != ENOSYS) {
+            if (errnum == 0)
+                errnum = errno;
+            (void)snprintf (errbuf + offset,
+                            sizeof (errbuf) - offset,
+                            "%s%s: %s",
+                            offset > 0 ? "\n" : "",
+                            name,
+                            flux_future_error_string (f));
+            offset = strlen (errbuf);
+        }
+        name = flux_future_next_child (cf);
+    }
+    if (errnum != 0)
+        goto error;
+
+    if (flux_respond (cfg->h, msg, NULL) < 0)
+        flux_log_error (cfg->h, "reload: flux_respond");
+    flux_log (cfg->h, LOG_INFO, "configuration updated");
+    flux_future_destroy (cfg->reload_f);
+    cfg->reload_f = NULL;
+    return;
+
+error:
+    if (flux_respond_error (cfg->h, msg, errnum, errbuf) < 0)
+        flux_log_error (cfg->h, "reload: flux_respond_error");
+    flux_future_destroy (cfg->reload_f);
+    cfg->reload_f = NULL;
+    flux_log (cfg->h, LOG_ERR, "config reload failed");
+}
+
+/* Send a '<name>.config-reload' request to all loaded modules.
+ * On success, return a composite future that is fulfilled once all modules
+ * have responded.  On failure, return NULL with errno set.
+ */
+static flux_future_t *reload_module_configs (flux_t *h, struct brokercfg *cfg)
+{
+    flux_future_t *cf;
+    module_t *module;
+    json_t *conf;
+
+    if (flux_conf_unpack (flux_get_conf (h), NULL, "o", &conf) < 0)
+        return NULL;
+    if (!(cf = flux_future_wait_all_create ()))
+        return NULL;
+    flux_future_set_flux (cf, h);
+    module = modhash_first (cfg->modhash);
+    while (module) {
+        flux_future_t *f;
+        char topic[1024];
+
+        if (snprintf (topic,
+                      sizeof (topic),
+                      "%s.config-reload",
+                      module_get_name (module)) >= sizeof (topic)) {
+            errno = EOVERFLOW;
+            goto error;
+        }
+        if (!(f = flux_rpc_pack (h, topic, FLUX_NODEID_ANY, 0, "O", conf)))
+            goto error;
+        if (flux_future_push (cf, module_get_name (module), f) < 0) {
+            flux_future_destroy (f);
+            goto error;
+        }
+        module = modhash_next (cfg->modhash);
+    }
+    return cf;
+error:
+    flux_future_destroy (cf);
+    return NULL;
+}
+
+static int update_modules_and_respond (flux_t *h,
+                                       struct brokercfg *cfg,
+                                       const flux_msg_t *msg,
+                                       flux_error_t *error)
+{
+    flux_future_t *f;
+
+    if (cfg->reload_f) {
+        errprintf (error, "module config-reload in progress, try again later");
+        errno = EBUSY;
+        return -1;
+    }
+    if (!(f = reload_module_configs (h, cfg))
+        || flux_future_then (f, -1., reload_continuation, cfg) < 0)
+        goto error;
+    if (flux_future_aux_set (f,
+                             "flux::request",
+                             (void *)flux_msg_incref (msg),
+                             (flux_free_f)flux_msg_decref) < 0) {
+        flux_msg_decref (msg);
+        goto error;
+    }
+    cfg->reload_f = f;
+    return 0;
+error:
+    errprintf (error, "failed to set up asynchronous module config-reload");
+    flux_future_destroy (f);
+    return -1;
+}
+
+/* Handle request to re-parse config object from TOML config files.
+ * If files fail to parse, generate an immediate error response.  Otherwise,
+ * initiate reload of config in all loaded modules and respond when complete.
+ */
+static void reload_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    struct brokercfg *cfg = arg;
+    flux_error_t error;
+
+    if (brokercfg_parse (h, cfg->path, &error) < 0
+        || update_modules_and_respond (h, cfg, msg, &error) < 0)
+        goto error;
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, error.text) < 0)
+        flux_log_error (h, "error responding to config.reload request");
+}
+
+/* Handle request to replace config object with request payload.
+ * Initiate reload of config in all loaded modules.
+ */
+static void load_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    struct brokercfg *cfg = arg;
+    const flux_conf_t *conf = NULL;
+    flux_error_t error;
+
+    if (flux_conf_reload_decode (msg, &conf) < 0) {
+        errprintf (&error, "error decoding config.load request");
+        goto error;
+    }
+    if (brokercfg_set (h, flux_conf_incref (conf), &error) < 0) {
+        flux_conf_decref (conf);
+        goto error;
+    }
+    if (update_modules_and_respond (h, cfg, msg, &error) < 0)
+        goto error;
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, error.text) < 0)
+        flux_log_error (h, "error responding to config.load request");
+}
+
+static void get_cb (flux_t *h,
+                    flux_msg_handler_t *mh,
+                    const flux_msg_t *msg,
+                    void *arg)
+{
+    const char *errmsg = NULL;
+    flux_error_t error;
+    json_t *o;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (flux_conf_unpack (flux_get_conf (h), &error, "o", &o) < 0) {
+        errmsg = error.text;
+        goto error;
+    }
+    if (flux_respond_pack (h, msg, "O", o) < 0)
+        flux_log_error (h, "error responding to config.get request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to config.get request");
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,  "config.reload", reload_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST,  "config.load", load_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST,  "config.get", get_cb, FLUX_ROLE_USER },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void brokercfg_destroy (struct brokercfg *cfg)
+{
+    if (cfg) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (cfg->handlers);
+        flux_future_destroy (cfg->reload_f);
+        free (cfg->path);
+        free (cfg);
+        errno = saved_errno;
+    }
+}
+
+struct brokercfg *brokercfg_create (flux_t *h,
+                                    const char *path,
+                                    attr_t *attrs,
+                                    modhash_t *modhash)
+{
+    struct brokercfg *cfg;
+    flux_error_t error;
+
+    if (!(cfg = calloc (1, sizeof (*cfg))))
+        return NULL;
+    cfg->h = h;
+    cfg->modhash = modhash;
+    if (!path)
+        path = getenv ("FLUX_CONF_DIR");
+    if (path) {
+        if (!(cfg->path = strdup (path)))
+            goto error;
+    }
+    if (brokercfg_parse (h, path, &error) < 0) {
+        log_msg ("%s", error.text);
+        goto error;
+    }
+    if (flux_msg_handler_addvec (h, htab, cfg, &cfg->handlers) < 0)
+        goto error;
+    if (attr_add (attrs, "config.path", path, ATTR_IMMUTABLE) < 0)
+        goto error;
+    return cfg;
+error:
+    brokercfg_destroy (cfg);
+    return NULL;
+}
+
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/broker/brokercfg.h b/src/broker/brokercfg.h
new file mode 100644
index 000000000000..b449f3551c99
--- /dev/null
+++ b/src/broker/brokercfg.h
@@ -0,0 +1,30 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _BROKER_BROKERCFG_H
+#define _BROKER_BROKERCFG_H
+
+#include <flux/core.h>
+#include "attr.h"
+#include "modhash.h"
+
+struct brokercfg;
+
+struct brokercfg *brokercfg_create (flux_t *h,
+                                    const char *path,
+                                    attr_t *attr,
+                                    modhash_t *modhash);
+void brokercfg_destroy (struct brokercfg *cfg);
+
+#endif /* !_BROKER_BROKERCFG_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/broker/content-cache.c b/src/broker/content-cache.c
deleted file mode 100644
index 0069ab9573a3..000000000000
--- a/src/broker/content-cache.c
+++ /dev/null
@@ -1,1060 +0,0 @@
-/************************************************************\
- * Copyright 2015 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* See RFC 10 */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <inttypes.h>
-#include <czmq.h>
-#include <flux/core.h>
-#include "src/common/libutil/errno_safe.h"
-#include "src/common/libutil/blobref.h"
-#include "src/common/libutil/iterators.h"
-#include "src/common/libutil/log.h"
-
-#include "attr.h"
-#include "content-cache.h"
-
-static const uint32_t default_cache_purge_target_entries = 1024*1024;
-static const uint32_t default_cache_purge_target_size = 1024*1024*16;
-
-static const uint32_t default_cache_purge_old_entry = 5;
-static const uint32_t default_cache_purge_large_entry = 256;
-
-/* Raise the max blob size value to 1GB so that large KVS values
- * (including KVS directories) can be supported while the KVS transitions
- * to the RFC 11 treeobj data representation.
- */
-//static const uint32_t default_blob_size_limit = 1048576; /* RFC 10 */
-static const uint32_t default_blob_size_limit = 1048576*1024;
-
-static const uint32_t default_flush_batch_limit = 256;
-
-struct cache_entry {
-    flux_t *h;
-    void *data;
-    int len;
-    char *blobref;
-    uint8_t valid:1;                /* entry contains valid data */
-    uint8_t dirty:1;                /* entry needs to be stored upstream */
-                                    /*   or to backing store (rank 0) */
-    uint8_t load_pending:1;
-    uint8_t store_pending:1;
-    zlist_t *load_requests;
-    zlist_t *store_requests;
-    int lastused;
-};
-
-struct content_cache {
-    flux_t *h;
-    flux_msg_handler_t **handlers;
-    uint32_t rank;
-    zhash_t *entries;
-    uint8_t backing:1;              /* 'content.backing' service available */
-    char *backing_name;
-    char hash_name[BLOBREF_MAX_STRING_SIZE];
-    zlist_t *flush_requests;
-    int epoch;
-
-    uint32_t blob_size_limit;
-    uint32_t flush_batch_limit;
-    uint32_t flush_batch_count;
-
-    uint32_t purge_target_entries;
-    uint32_t purge_target_size;
-    uint32_t purge_old_entry;
-    uint32_t purge_large_entry;
-
-    uint32_t acct_size;             /* total size of all cache entries */
-    uint32_t acct_valid;            /* count of valid cache entries */
-    uint32_t acct_dirty;            /* count of dirty cache entries */
-};
-
-static void flush_respond (content_cache_t *cache);
-static int cache_flush (content_cache_t *cache);
-
-static void request_list_destroy (zlist_t **l)
-{
-    const flux_msg_t *msg;
-    if (*l) {
-        while ((msg = zlist_pop (*l)))
-            flux_msg_decref (msg);
-        zlist_destroy (l);
-    }
-}
-
-/* Respond identically to a list of requests.
- * The list is always run to completion, then destroyed.
- * On error, log at LOG_ERR level.
- */
-static void request_list_respond_raw (zlist_t **l,
-                                      flux_t *h,
-                                      const void *data,
-                                      int len,
-                                      const char *type)
-{
-    if (*l) {
-        const flux_msg_t *msg;
-        while ((msg = zlist_pop (*l))) {
-            if (flux_respond_raw (h, msg, data, len) < 0)
-                flux_log_error (h, "%s (%s):", __FUNCTION__, type);
-            flux_msg_decref (msg);
-        }
-        zlist_destroy (l);
-    }
-}
-
-/* Same as above only send errnum, errmsg response
- */
-static void request_list_respond_error (zlist_t **l,
-                                        flux_t *h,
-                                        int errnum,
-                                        const char *errmsg,
-                                        const char *type)
-{
-    if (*l) {
-        const flux_msg_t *msg;
-        while ((msg = zlist_pop (*l))) {
-            if (flux_respond_error (h, msg, errnum, errmsg) < 0)
-                flux_log_error (h, "%s (%s):", __FUNCTION__, type);
-            flux_msg_decref (msg);
-        }
-        zlist_destroy (l);
-    }
-}
-
-/* Add request message to a list, creating the list as needed.
- * Returns 0 on succes, -1 on failure with errno set.
- */
-static int request_list_add (zlist_t **l, const flux_msg_t *msg)
-{
-    if (!*l) {
-        if (!(*l = zlist_new ())) {
-            errno = ENOMEM;
-            return -1;
-        }
-    }
-    if (zlist_append (*l, (void *)flux_msg_incref (msg)) < 0) {
-        flux_msg_decref (msg);
-        errno = ENOMEM;
-        return -1;
-    }
-    return 0;
-}
-
-/* Destroy a cache entry
- */
-static void cache_entry_destroy (void *arg)
-{
-    struct cache_entry *e = arg;
-    if (e) {
-        if (e->data)
-            free (e->data);
-        if (e->blobref)
-            free (e->blobref);
-        if (e->load_requests && zlist_size (e->load_requests) > 0)
-            flux_log (e->h, LOG_ERR, "%s: load_requests not empty",
-                      __FUNCTION__);
-        if (e->store_requests && zlist_size (e->store_requests) > 0)
-            flux_log (e->h, LOG_ERR, "%s: store_requests not empty",
-                      __FUNCTION__);
-        request_list_destroy (&e->load_requests);
-        request_list_destroy (&e->store_requests);
-        free (e);
-    }
-}
-
-/* Create a cache entry.
- * Initially only the digest is filled in;  defaults for the rest (zeroed).
- * Returns entry on success, NULL with errno set on failure.
- */
-static struct cache_entry *cache_entry_create (flux_t *h, const char *blobref)
-{
-    struct cache_entry *e = malloc (sizeof (*e));
-    if (!e) {
-        errno = ENOMEM;
-        return NULL;
-    }
-    memset (e, 0, sizeof (*e));
-    e->h = h;
-    if (!(e->blobref = strdup (blobref))) {
-        free (e);
-        errno = ENOMEM;
-        return NULL;
-    }
-    return e;
-}
-
-/* Make an invalid cache entry valid, filling in its data.
- * Returns 0 on success, -1 on failure with errno set.
- */
-static int cache_entry_fill (struct cache_entry *e, const void *data, int len)
-{
-    int rc = -1;
-
-    if (!e->valid) {
-        assert (!e->data);
-        assert (e->len == 0);
-        if (len > 0 && !(e->data = malloc (len))) {
-            errno = ENOMEM;
-            goto done;
-        }
-        memcpy (e->data, data, len);
-        e->len = len;
-    }
-    rc = 0;
-done:
-    return rc;
-}
-
-/* Insert a cache entry, by blobref.
- * Returns 0 on success, -1 on failure with errno set.
- * Side effect: destroys entry on failure.
- */
-static int insert_entry (content_cache_t *cache, struct cache_entry *e)
-{
-    if (zhash_insert (cache->entries, e->blobref, e) < 0) {
-        cache_entry_destroy (e);
-        errno = ENOMEM;
-        return -1;
-    }
-    zhash_freefn (cache->entries, e->blobref, cache_entry_destroy);
-    if (e->valid) {
-        cache->acct_size += e->len;
-        cache->acct_valid++;
-    }
-    if (e->dirty)
-        cache->acct_dirty++;
-    return 0;
-}
-
-/* Look up a cache entry, by blobref.
- * Returns entry on success, NULL on failure.
- * N.B. errno is not set
- */
-static struct cache_entry *lookup_entry (content_cache_t *cache,
-                                         const char *blobref)
-{
-    return zhash_lookup (cache->entries, blobref);
-}
-
-/* Remove a cache entry.
- */
-static void remove_entry (content_cache_t *cache, struct cache_entry *e)
-{
-    assert (!e->load_requests || zlist_size (e->load_requests) == 0);
-    assert (!e->store_requests || zlist_size (e->store_requests) == 0);
-    if (e->valid) {
-        cache->acct_size -= e->len;
-        cache->acct_valid--;
-    }
-    if (e->dirty)
-        cache->acct_dirty--;
-    zhash_delete (cache->entries, e->blobref);
-}
-
-/* Load operation
- *
- * If a cache entry is already present and valid, response is immediate.
- * Otherwise request is queued on the invalid cache entry, and a new
- * request is sent to the next level of TBON, or on rank 0, to the
- * content.backing service.  At most a single request is sent per cache entry.
- * Once the response is received, identical responses are sent to all
- * parked requests, and cache entry is made valid or removed if there was
- * an error such as ENOENT.
- */
-
-static void cache_load_continuation (flux_future_t *f, void *arg)
-{
-    content_cache_t *cache = arg;
-    struct cache_entry *e = flux_future_aux_get (f, "entry");
-    const void *data = NULL;
-    int len = 0;
-
-    e->load_pending = 0;
-    if (flux_content_load_get (f, &data, &len) < 0) {
-        if (errno == ENOSYS && cache->rank == 0)
-            errno = ENOENT;
-        if (errno != ENOENT)
-            flux_log_error (cache->h, "content load");
-        goto error;
-    }
-    if (cache_entry_fill (e, data, len) < 0) {
-        flux_log_error (cache->h, "content load");
-        goto error;
-    }
-    if (!e->valid) {
-        e->valid = 1;
-        cache->acct_valid++;
-        cache->acct_size += len;
-    }
-    e->lastused = cache->epoch;
-    request_list_respond_raw (&e->load_requests,
-                              cache->h,
-                              e->data,
-                              e->len,
-                              "load");
-    flux_future_destroy (f);
-    return;
-error:
-    request_list_respond_error (&e->load_requests,
-                                cache->h,
-                                errno,
-                                NULL,
-                                "load");
-    remove_entry (cache, e);
-    flux_future_destroy (f);
-}
-
-static int cache_load (content_cache_t *cache, struct cache_entry *e)
-{
-    flux_future_t *f;
-    int saved_errno = 0;
-    int flags = CONTENT_FLAG_UPSTREAM;
-    int rc = -1;
-
-    if (e->load_pending)
-        return 0;
-    if (cache->rank == 0)
-        flags = CONTENT_FLAG_CACHE_BYPASS;
-    if (!(f = flux_content_load (cache->h, e->blobref, flags))) {
-        if (errno == ENOSYS && cache->rank == 0)
-            errno = ENOENT;
-        saved_errno = errno;
-        if (errno != ENOENT)
-            flux_log_error (cache->h, "%s: RPC", __FUNCTION__);
-        goto done;
-    }
-    if (flux_future_aux_set (f, "entry", e, NULL) < 0) {
-        flux_log_error (cache->h, "content load flux_future_aux_set");
-        goto done;
-    }
-    if (flux_future_then (f, -1., cache_load_continuation, cache) < 0) {
-        saved_errno = errno;
-        flux_log_error (cache->h, "content load");
-        flux_future_destroy (f);
-        goto done;
-    }
-    e->load_pending = 1;
-    rc = 0;
-done:
-    if (rc < 0)
-        errno = saved_errno;
-    return rc;
-}
-
-void content_load_request (flux_t *h, flux_msg_handler_t *mh,
-                           const flux_msg_t *msg, void *arg)
-{
-    content_cache_t *cache = arg;
-    const char *blobref;
-    int blobref_size;
-    void *data = NULL;
-    int len = 0;
-    struct cache_entry *e;
-
-    if (flux_request_decode_raw (msg, NULL, (const void **)&blobref,
-                                 &blobref_size) < 0)
-        goto error;
-    if (!blobref || blobref[blobref_size - 1] != '\0') {
-        errno = EPROTO;
-        goto error;
-    }
-    if (!(e = lookup_entry (cache, blobref))) {
-        if (cache->rank == 0 && !cache->backing) {
-            errno = ENOENT;
-            goto error;
-        }
-        if (!(e = cache_entry_create (h, blobref))
-                                            || insert_entry (cache, e) < 0) {
-            flux_log_error (h, "content load");
-            goto error; /* insert destroys 'e' on failure */
-        }
-    }
-    if (!e->valid) {
-        if (cache_load (cache, e) < 0)
-            goto error;
-        if (request_list_add (&e->load_requests, msg) < 0) {
-            flux_log_error (h, "content load");
-            goto error;
-        }
-        return; /* RPC continuation will respond to msg */
-    }
-    e->lastused = cache->epoch;
-    data = e->data;
-    len = e->len;
-    if (flux_respond_raw (h, msg, data, len) < 0)
-        flux_log_error (h, "content load: flux_respond_raw");
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "content load: flux_respond_error");
-}
-
-/* Store operation
- *
- * If a cache entry is already valid and not dirty, response is immediate.
- * If cache entry is invalid, it is made valid (responding to any queued
- * load requests), and then dirty.
- *
- * Dirty cache is write-through for ranks > 0;  that is, a request is queued
- * and a single store request per cache entry is sent to the next level
- * of TBON.  Once present in the rank 0 cache, requests are unwound and
- * responded to at each level.
- *
- * Dirty cache is write-back for rank 0, that is;  the response is immediate
- * even though the entry may be dirty with respect to a 'content.backing'
- * service.  This allows the cache to be updated at memory speeds,
- * while holding the invariant that after a store RPC returns, the entry may
- * be loaded from any rank.  The optional content.backing service can
- * offload rank 0 hash entries at a slower pace.
- */
-
-/* If cache has been flushed, respond to flush requests, if any.
- * If there are still dirty entries and the number of outstanding
- * store requests would not exceed the limit, flush more entries.
- * Optimization: since scanning for dirty entries is a linear search,
- * only do it when the number of outstanding store requests falls to
- * a low water mark, here hardwired to be half of the limit.
- */
-static void cache_resume_flush (content_cache_t *cache)
-{
-    if (cache->acct_dirty == 0 || (cache->rank == 0 && !cache->backing))
-        flush_respond (cache);
-    else if (cache->acct_dirty - cache->flush_batch_count > 0
-            && cache->flush_batch_count <= cache->flush_batch_limit / 2)
-        (void)cache_flush (cache); /* resume flushing */
-}
-
-static void cache_store_continuation (flux_future_t *f, void *arg)
-{
-    content_cache_t *cache = arg;
-    struct cache_entry *e = flux_future_aux_get (f, "entry");
-    const char *blobref;
-
-    e->store_pending = 0;
-    assert (cache->flush_batch_count > 0);
-    cache->flush_batch_count--;
-    if (flux_content_store_get (f, &blobref) < 0) {
-        if (cache->rank == 0 && errno == ENOSYS)
-            flux_log (cache->h, LOG_DEBUG, "content store: %s",
-                      "backing store service unavailable");
-        else
-            flux_log_error (cache->h, "content store");
-        goto error;
-    }
-    if (strcmp (blobref, e->blobref)) {
-        flux_log (cache->h, LOG_ERR, "content store: wrong blobref");
-        errno = EIO;
-        goto error;
-    }
-    if (e->dirty) {
-        cache->acct_dirty--;
-        e->dirty = 0;
-    }
-    request_list_respond_raw (&e->store_requests,
-                              cache->h,
-                              e->blobref,
-                              strlen (e->blobref) + 1,
-                              "store");
-    flux_future_destroy (f);
-    cache_resume_flush (cache);
-    return;
-error:
-    request_list_respond_error (&e->store_requests,
-                                cache->h,
-                                errno,
-                                NULL,
-                                "store");
-    flux_future_destroy (f);
-    cache_resume_flush (cache);
-}
-
-static int cache_store (content_cache_t *cache, struct cache_entry *e)
-{
-    flux_future_t *f;
-    int saved_errno = 0;
-    int flags = CONTENT_FLAG_UPSTREAM;
-    int rc = -1;
-
-    assert (e->valid);
-
-    if (e->store_pending)
-        return 0;
-    if (cache->rank == 0) {
-        if (cache->flush_batch_count >= cache->flush_batch_limit)
-            return 0;
-        flags = CONTENT_FLAG_CACHE_BYPASS;
-    }
-    if (!(f = flux_content_store (cache->h, e->data, e->len, flags))) {
-        saved_errno = errno;
-        flux_log_error (cache->h, "content store");
-        goto done;
-    }
-    if (flux_future_aux_set (f, "entry", e, NULL) < 0) {
-        flux_log_error (cache->h, "content store: flux_future_aux_set");
-        goto done;
-    }
-    if (flux_future_then (f, -1., cache_store_continuation, cache) < 0) {
-        saved_errno = errno;
-        flux_log_error (cache->h, "content store");
-        flux_future_destroy (f);
-        goto done;
-    }
-    e->store_pending = 1;
-    cache->flush_batch_count++;
-    rc = 0;
-done:
-    if (rc < 0)
-        errno = saved_errno;
-    return rc;
-}
-
-static void content_store_request (flux_t *h, flux_msg_handler_t *mh,
-                                   const flux_msg_t *msg, void *arg)
-{
-    content_cache_t *cache = arg;
-    const void *data;
-    int len;
-    struct cache_entry *e = NULL;
-    char blobref[BLOBREF_MAX_STRING_SIZE];
-
-    if (flux_request_decode_raw (msg, NULL, &data, &len) < 0)
-        goto error;
-    if (len > cache->blob_size_limit) {
-        errno = EFBIG;
-        goto error;
-    }
-    if (blobref_hash (cache->hash_name, (uint8_t *)data, len, blobref,
-                      sizeof (blobref)) < 0)
-        goto error;
-
-    if (!(e = lookup_entry (cache, blobref))) {
-        if (!(e = cache_entry_create (h, blobref)))
-            goto error;
-        if (insert_entry (cache, e) < 0)
-            goto error; /* insert destroys 'e' on failure */
-    }
-    if (!e->valid) {
-        if (cache_entry_fill (e, data, len) < 0)
-            goto error;
-        if (!e->valid) {
-            e->valid = 1;
-            cache->acct_valid++;
-            cache->acct_size += len;
-        }
-        request_list_respond_raw (&e->load_requests,
-                                  cache->h,
-                                  e->data,
-                                  e->len,
-                                  "load");
-        if (!e->dirty) {
-            e->dirty = 1;
-            cache->acct_dirty++;
-        }
-    }
-    e->lastused = cache->epoch;
-    if (e->dirty) {
-        if (cache->rank > 0 || cache->backing) {
-            if (cache_store (cache, e) < 0)
-                goto error;
-            if (cache->rank > 0) {  /* write-through */
-                if (request_list_add (&e->store_requests, msg) < 0)
-                    goto error;
-                return;
-            }
-        }
-    } else {
-        /* When a backing store module is unloaded, it will clear
-         * cache->backing then attempt to store all its blobs.  Any of
-         * those still in cache need to be marked dirty.
-         */
-        if (cache->rank == 0 && !cache->backing) {
-            e->dirty = 1;
-            cache->acct_dirty++;
-        }
-    }
-    if (flux_respond_raw (h, msg, blobref, strlen (blobref) + 1) < 0)
-        flux_log_error (h, "content store: flux_respond_raw");
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "content store: flux_respond_error");
-}
-
-/* Backing store is enabled/disabled by modules that provide the
- * 'content.backing' service.  At module load time, the backing module
- * informs the content service of its availability, and entries are
- * asynchronously duplicated on the backing store and made eligible for
- * dropping from the rank 0 cache.
- */
-
-static int cache_flush (content_cache_t *cache)
-{
-    struct cache_entry *e;
-    const char *key;
-    int saved_errno = 0;
-    int count = 0;
-    int rc = 0;
-
-    if (cache->acct_dirty - cache->flush_batch_count == 0
-            || cache->flush_batch_count >= cache->flush_batch_limit)
-        return 0;
-
-    flux_log (cache->h, LOG_DEBUG, "content flush begin");
-    FOREACH_ZHASH (cache->entries, key, e) {
-        if (!e->dirty || e->store_pending)
-            continue;
-        if (cache_store (cache, e) < 0) {
-            saved_errno = errno;
-            rc = -1;
-        }
-        count++;
-        if (cache->flush_batch_count >= cache->flush_batch_limit)
-            break;
-    }
-    flux_log (cache->h, LOG_DEBUG, "content flush +%d (dirty=%d pending=%d)",
-              count, cache->acct_dirty, cache->flush_batch_count);
-    if (rc < 0)
-        errno = saved_errno;
-    return rc;
-}
-
-static void content_register_backing_request (flux_t *h,
-                                              flux_msg_handler_t *mh,
-                                              const flux_msg_t *msg,
-                                              void *arg)
-{
-    content_cache_t *cache = arg;
-    const char *name;
-
-    if (flux_request_unpack (msg, NULL, "{s:s}", "name", &name) < 0)
-        goto error;
-    if (cache->rank != 0 || cache->backing) {
-        errno = EINVAL;
-        goto error;
-    }
-    if (!(cache->backing_name = strdup (name)))
-        goto error;
-    cache->backing = 1;
-    flux_log (h, LOG_DEBUG, "content backing store: enabled %s", name);
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "content backing");
-    (void)cache_flush (cache);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "content backing");
-};
-
-static void content_unregister_backing_request (flux_t *h,
-                                                flux_msg_handler_t *mh,
-                                                const flux_msg_t *msg,
-                                                void *arg)
-{
-    content_cache_t *cache = arg;
-
-    if (flux_request_decode (msg, NULL, NULL) < 0)
-        goto error;
-    if (!cache->backing) {
-        errno = EINVAL;
-        goto error;
-    }
-    cache->backing = 0;
-    free (cache->backing_name);
-    cache->backing_name = NULL;
-    flux_log (h, LOG_DEBUG, "content backing store: disabled");
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "flux_respond");
-    if (cache->acct_dirty > 0)
-        flux_log (h, LOG_ERR, "%d unflushables", cache->acct_dirty);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "flux_respond_error");
-}
-
-/* Forcibly drop all entries from the cache that can be dropped
- * without data loss.
- * N.B. this walks the entire cache in one go.
- */
-
-static void content_dropcache_request (flux_t *h, flux_msg_handler_t *mh,
-                                       const flux_msg_t *msg, void *arg)
-{
-    content_cache_t *cache = arg;
-    zlist_t *keys = NULL;
-    char *key;
-    struct cache_entry *e;
-    int orig_size;
-
-    if (flux_request_decode (msg, NULL, NULL) < 0)
-        goto error;
-    orig_size = zhash_size (cache->entries);
-    if (!(keys = zhash_keys (cache->entries))) {
-        errno = ENOMEM;
-        goto error;
-    }
-    while ((key = zlist_pop (keys))) {
-        e = zhash_lookup (cache->entries, key);
-        assert (e != NULL);
-        if (e->valid && !e->dirty)
-            remove_entry (cache, e);
-        free (key);
-    }
-    flux_log (h, LOG_DEBUG, "content dropcache %d/%d",
-              orig_size - (int)zhash_size (cache->entries), orig_size);
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "content dropcache");
-    zlist_destroy (&keys);
-    return;
-error:
-    flux_log (h, LOG_DEBUG, "content dropcache: %s", flux_strerror (errno));
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "content dropcache");
-    zlist_destroy (&keys);
-}
-
-/* Return stats about the cache.
- */
-
-static void content_stats_request (flux_t *h, flux_msg_handler_t *mh,
-                                   const flux_msg_t *msg, void *arg)
-{
-    content_cache_t *cache = arg;
-
-    if (flux_request_decode (msg, NULL, NULL) < 0)
-        goto error;
-    if (flux_respond_pack (h, msg, "{ s:i s:i s:i s:i}",
-                           "count", zhash_size (cache->entries),
-                           "valid", cache->acct_valid,
-                           "dirty", cache->acct_dirty,
-                           "size", cache->acct_size) < 0)
-        flux_log_error (h, "content stats");
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "content stats");
-}
-
-/* Flush all dirty entries by walking the entire cache, issuing store
- * requests for all dirty entries.  Responses are handled asynchronously
- * using RPC continuations.  A response to the flush request is not sent
- * until all the store responses are received.  If 'backing' is false on
- * rank 0, we go ahead and issue the store requests and handle the ENOSYS
- * errors that result.
- */
-
-/* This is called when outstanding store ops have completed.  */
-static void flush_respond (content_cache_t *cache)
-{
-    if (!cache->acct_dirty) {
-        request_list_respond_raw (&cache->flush_requests,
-                                  cache->h,
-                                  NULL,
-                                  0,
-                                  "flush");
-    }
-    else {
-        errno = EIO;
-        if (cache->rank == 0 && !cache->backing)
-            errno = ENOSYS;
-        request_list_respond_error (&cache->flush_requests,
-                                    cache->h,
-                                    errno,
-                                    NULL,
-                                    "flush");
-    }
-}
-
-static void content_flush_request (flux_t *h, flux_msg_handler_t *mh,
-                                   const flux_msg_t *msg, void *arg)
-{
-    content_cache_t *cache = arg;
-
-    if (flux_request_decode (msg, NULL, NULL) < 0)
-        goto error;
-    if (cache->acct_dirty != 0) {
-        if (cache_flush (cache) < 0)
-            goto error;
-        if (cache->acct_dirty > 0) {
-            if (request_list_add (&cache->flush_requests, msg) < 0)
-                goto error;
-            return;
-        }
-        if (cache->acct_dirty > 0) {
-            errno = EIO;
-            goto error;
-        }
-    }
-    flux_log (h, LOG_DEBUG, "content flush");
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "content flush");
-    return;
-error:
-    flux_log (h, LOG_DEBUG, "content flush: %s", flux_strerror (errno));
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "content flush");
-}
-
-/* Heartbeat drives periodic cache purge
- */
-
-static int cache_purge (content_cache_t *cache)
-{
-    int after_entries = zhash_size (cache->entries);
-    int after_size = cache->acct_size;
-    struct cache_entry *e;
-    zlist_t *purge = NULL;
-    int rc = -1;
-    const char *key;
-
-    if (cache->acct_dirty == zhash_size (cache->entries))
-        return 0;
-
-    FOREACH_ZHASH (cache->entries, key, e) {
-        if (after_size <= cache->purge_target_size
-                        && after_entries <= cache->purge_target_entries)
-            break;
-        if (!e->valid || e->dirty)
-            continue;
-        if (cache->epoch - e->lastused < cache->purge_old_entry)
-            continue;
-        if (after_entries <= cache->purge_target_entries
-                    && e->len < cache->purge_large_entry)
-            continue;
-        if ((!purge && !(purge = zlist_new ()))
-                    || zlist_append (purge, e) < 0) {
-            errno = ENOMEM;
-            goto done;
-        }
-        after_size -= e->len;
-        after_entries--;
-    }
-    if (purge) {
-        flux_log (cache->h, LOG_DEBUG, "content purge: %d entries",
-                  (int)zlist_size (purge));
-        while ((e = zlist_pop (purge)))
-            remove_entry (cache, e);
-    }
-    rc = 0;
-done:
-    zlist_destroy (&purge);
-    return rc;
-}
-
-static void heartbeat_event (flux_t *h, flux_msg_handler_t *mh,
-                             const flux_msg_t *msg, void *arg)
-{
-    content_cache_t *cache = arg;
-
-    if (flux_heartbeat_decode (msg, &cache->epoch) < 0)
-        return; /* ignore mangled heartbeat */
-    cache_purge (cache);
-}
-
-/* Initialization
- */
-
-static const struct flux_msg_handler_spec htab[] = {
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "content.load",
-        content_load_request,
-        FLUX_ROLE_USER
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "content.store",
-        content_store_request,
-        FLUX_ROLE_USER
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "content.unregister-backing",
-        content_unregister_backing_request,
-        0
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "content.register-backing",
-        content_register_backing_request,
-        0
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "content.dropcache",
-        content_dropcache_request,
-        0
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "content.stats.get",
-        content_stats_request,
-        0
-    },
-    {
-        FLUX_MSGTYPE_REQUEST,
-        "content.flush",
-        content_flush_request,
-        0
-    },
-    {
-        FLUX_MSGTYPE_EVENT,
-        "hb",
-        heartbeat_event,
-        0
-    },
-    FLUX_MSGHANDLER_TABLE_END,
-};
-
-int content_cache_set_flux (content_cache_t *cache, flux_t *h)
-{
-    cache->h = h;
-
-    if (flux_msg_handler_addvec (h, htab, cache, &cache->handlers) < 0)
-        return -1;
-    if (flux_get_rank (h, &cache->rank) < 0)
-        return -1;
-    if (flux_event_subscribe (h, "hb") < 0)
-        return -1;
-    return 0;
-}
-
-static int content_cache_setattr (const char *name, const char *val, void *arg)
-{
-
-    content_cache_t *cache = arg;
-    if (!strcmp (name, "content.hash")) {
-        if (blobref_validate_hashtype (val) < 0)
-            goto invalid;
-        strcpy (cache->hash_name, val);
-    } else
-        goto invalid;
-    return 0;
-invalid:
-    errno = EINVAL;
-    return -1;
-}
-
-static int content_cache_getattr (const char *name, const char **val, void *arg)
-{
-    content_cache_t *cache = arg;
-    static char s[32];
-
-    if (!strcmp (name, "content.hash"))
-        *val = cache->hash_name;
-    else if (!strcmp (name, "content.backing"))
-        *val = cache->backing_name;
-    else if (!strcmp (name, "content.acct-entries")) {
-        snprintf (s, sizeof (s), "%zd", zhash_size (cache->entries));
-        *val = s;
-    } else
-        return -1;
-    return 0;
-}
-
-int content_cache_register_attrs (content_cache_t *cache, attr_t *attr)
-{
-    /* Purge tunables
-     */
-    if (attr_add_active_uint32 (attr, "content.purge-target-entries",
-                &cache->purge_target_entries, 0) < 0)
-        return -1;
-    if (attr_add_active_uint32 (attr, "content.purge-target-size",
-                &cache->purge_target_size, 0) < 0)
-        return -1;
-    if (attr_add_active_uint32 (attr, "content.purge-old-entry",
-                &cache->purge_old_entry, 0) < 0)
-        return -1;
-    if (attr_add_active_uint32 (attr, "content.purge-large-entry",
-                &cache->purge_large_entry, 0) < 0)
-        return -1;
-    /* Accounting numbers
-     */
-    if (attr_add_active_uint32 (attr, "content.acct-size",
-                &cache->acct_size, FLUX_ATTRFLAG_READONLY) < 0)
-        return -1;
-    if (attr_add_active_uint32 (attr, "content.acct-dirty",
-                &cache->acct_dirty, FLUX_ATTRFLAG_READONLY) < 0)
-        return -1;
-    if (attr_add_active_uint32 (attr, "content.acct-valid",
-                &cache->acct_valid, FLUX_ATTRFLAG_READONLY) < 0)
-        return -1;
-    if (attr_add_active (attr, "content.acct-entries", FLUX_ATTRFLAG_READONLY,
-                content_cache_getattr, NULL, cache) < 0)
-        return -1;
-    /* Misc
-     */
-    if (attr_add_active_uint32 (attr, "content.flush-batch-limit",
-                &cache->flush_batch_limit, 0) < 0)
-        return -1;
-    if (attr_add_active_uint32 (attr, "content.blob-size-limit",
-                &cache->blob_size_limit, FLUX_ATTRFLAG_IMMUTABLE) < 0)
-        return -1;
-    if (attr_add_active (attr, "content.backing",FLUX_ATTRFLAG_READONLY,
-                 content_cache_getattr, NULL, cache) < 0)
-        return -1;
-    if (attr_add_active_uint32 (attr, "content.flush-batch-count",
-                &cache->flush_batch_count, 0) < 0)
-        return -1;
-    /* content-hash can be set on the command line
-     */
-    if (attr_add_active (attr, "content.hash", FLUX_ATTRFLAG_IMMUTABLE,
-                         content_cache_getattr,
-                         content_cache_setattr, cache) < 0)
-        return -1;
-
-    return 0;
-}
-
-void content_cache_destroy (content_cache_t *cache)
-{
-    if (cache) {
-        if (cache->h) {
-            (void)flux_event_unsubscribe (cache->h, "hb");
-            flux_msg_handler_delvec (cache->handlers);
-        }
-        if (cache->backing_name)
-            free (cache->backing_name);
-        zhash_destroy (&cache->entries);
-        request_list_destroy (&cache->flush_requests);
-        free (cache);
-    }
-}
-
-content_cache_t *content_cache_create (void)
-{
-    content_cache_t *cache = calloc (1, sizeof (*cache));
-    if (!cache) {
-        errno = ENOMEM;
-        return NULL;
-    }
-    if (!(cache->entries = zhash_new ())) {
-        content_cache_destroy (cache);
-        errno = ENOMEM;
-        return NULL;
-    }
-    cache->rank = FLUX_NODEID_ANY;
-    cache->blob_size_limit = default_blob_size_limit;
-    cache->flush_batch_limit = default_flush_batch_limit;
-    cache->purge_target_entries = default_cache_purge_target_entries;
-    cache->purge_target_size = default_cache_purge_target_size;
-    cache->purge_old_entry = default_cache_purge_old_entry;
-    cache->purge_large_entry = default_cache_purge_large_entry;
-    strcpy (cache->hash_name, "sha1");
-    return cache;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/content-cache.h b/src/broker/content-cache.h
deleted file mode 100644
index e2125daafccb..000000000000
--- a/src/broker/content-cache.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/************************************************************\
- * Copyright 2015 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef HAVE_BROKER_CONTENT_CACHE_H
-#define HAVE_BROKER_CONTENT_CACHE_H 1
-
-typedef struct content_cache content_cache_t;
-
-int content_cache_set_flux (content_cache_t *cache, flux_t *h);
-
-content_cache_t *content_cache_create (void);
-void content_cache_destroy (content_cache_t *cache);
-
-int content_cache_register_attrs (content_cache_t *cache, attr_t *attr);
-
-#endif /* !HAVE_BROKER_CONTENT_CACHE_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/exec.c b/src/broker/exec.c
index fd8a83b78319..8093606fd884 100644
--- a/src/broker/exec.c
+++ b/src/broker/exec.c
@@ -8,86 +8,72 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+/* exec.c - broker subprocess server
+ *
+ * The service is restricted to the instance owner.
+ * In addition, remote access to rank 0 is prohibited on multi-user instances.
+ * This is a precaution for system instances where rank 0 is deployed on a
+ * management node with restricted user access.
+ */
+
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <sys/param.h>
-#include <signal.h>
-#include <unistd.h>
-#include <assert.h>
-#include <jansson.h>
 #include <flux/core.h>
 
-#include "src/common/libsubprocess/command.h"
-#include "src/common/libutil/log.h"
+#include "src/common/libsubprocess/server.h"
+#include "src/common/libutil/errprintf.h"
 
 #include "attr.h"
 #include "exec.h"
+#include "overlay.h"
 
-#define EXEC_TERMINATE_TIMEOUT 5.0
-
-static void exec_finalize (void *arg)
-{
-    flux_subprocess_server_t *s = arg;
-    flux_subprocess_server_stop (s);
-}
-
-void exec_terminate_subprocesses (flux_t *h)
+/* The motivating use case for this was discussed in
+ * flux-framework/flux-core#5676
+ */
+static int is_multiuser_instance (flux_t *h)
 {
-    flux_subprocess_server_t *s = flux_aux_get (h, "flux::exec");
-
-    /* exec_initialize() never called */
-    if (!s)
-        return;
-
-    if (flux_subprocess_server_subprocesses_kill (s,
-                                                  SIGTERM,
-                                                  EXEC_TERMINATE_TIMEOUT) < 0)
-        flux_log_error (h, "flux_subprocess_server_subprocesses_kill");
-
-    /* SIGKILL is also sent in final teardown via
-     * flux_subprocess_server_stop(), but we wish to avoid any
-     * subprocesses continuing to run and potential send broker
-     * messages while we begin teardown, so we will SIGKILL the
-     * subprocesses as well.
-     */
-
-    if (flux_subprocess_server_subprocesses_kill (s,
-                                                  SIGKILL,
-                                                  EXEC_TERMINATE_TIMEOUT) < 0)
-        flux_log_error (h, "flux_subprocess_server_subprocesses_kill");
+    int allow_guest_user = 0;
+    (void)flux_conf_unpack (flux_get_conf (h),
+                            NULL,
+                            "{s:{s:b}}",
+                            "access",
+                              "allow-guest-user", &allow_guest_user);
+    return allow_guest_user;
 }
 
-int exec_terminate_subprocesses_by_uuid (flux_t *h, const char *id)
+static int reject_nonlocal (const flux_msg_t *msg,
+                            void *arg,
+                            flux_error_t *error)
 {
-    flux_subprocess_server_t *s = flux_aux_get (h, "flux::exec");
-
-    if (!s) {
-        flux_log (h, LOG_DEBUG, "no server_ctx found");
+    flux_t *h = arg;
+    if (!flux_msg_is_local (msg) && is_multiuser_instance (h)) {
+        errprintf (error,
+               "Remote rexec requests are not allowed on rank 0");
         return -1;
     }
-
-    if (flux_subprocess_server_terminate_by_uuid (s, id) < 0) {
-        flux_log_error (h, "flux_subprocess_server_terminate_by_uuid");
-        return -1;
-    }
-
     return 0;
 }
 
 int exec_initialize (flux_t *h, uint32_t rank, attr_t *attrs)
 {
-    flux_subprocess_server_t *s = NULL;
+    subprocess_server_t *s = NULL;
     const char *local_uri;
 
     if (attr_get (attrs, "local-uri", &local_uri, NULL) < 0)
         goto cleanup;
-    if (!(s = flux_subprocess_server_start (h, "cmb", local_uri, rank)))
+    if (!(s = subprocess_server_create (h, "rexec", local_uri, flux_llog, h)))
+        goto cleanup;
+    if (rank == 0)
+        subprocess_server_set_auth_cb (s, reject_nonlocal, h);
+    if (flux_aux_set (h,
+                      "flux::exec",
+                      s,
+                      (flux_free_f)subprocess_server_destroy) < 0)
         goto cleanup;
-    flux_aux_set (h, "flux::exec", s, exec_finalize);
     return 0;
 cleanup:
-    flux_subprocess_server_stop (s);
+    subprocess_server_destroy (s);
     return -1;
 }
 
diff --git a/src/broker/exec.h b/src/broker/exec.h
index df0195743ee5..b89749b14f8f 100644
--- a/src/broker/exec.h
+++ b/src/broker/exec.h
@@ -19,10 +19,6 @@
  * beginning of teardown of broker */
 void exec_terminate_subprocesses (flux_t *h);
 
-/* Kill any processes started by disconnecting client.
- */
-int exec_terminate_subprocesses_by_uuid (flux_t *h, const char *id);
-
 int exec_initialize (flux_t *h, uint32_t rank, attr_t *attrs);
 
 #endif /* BROKER_EXEC_H */
diff --git a/src/broker/groups.c b/src/broker/groups.c
new file mode 100644
index 000000000000..cfb6b399472c
--- /dev/null
+++ b/src/broker/groups.c
@@ -0,0 +1,841 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* groups.c - broker groups
+ *
+ * Track broker rank membership in multiple named groups.
+ * Each broker tracks membership for its TBON subtree, with membership
+ * for the full instance available at rank 0.
+ * Membership is updated through JOIN and LEAVE requests.
+ * An operation (join, leave, get) on an unknown group triggers its creation.
+ * Groups are is never removed.
+ *
+ * N.B. JOIN and LEAVE requests set/clear the broker rank that processed
+ * the request, therefore these requests must be sent to FLUX_NODEID_ANY
+ * so that they are processed on the same broker as the requestor.
+ *
+ * If a disconnect notification is received, a LEAVE is automatically
+ * generated for all groups that the disconnecting UUID has joined.
+ * Similarly, if the overlay subsystem notifies us that a peer subtree has
+ * become "lost", LEAVEs are automatically generated for all groups that
+ * the subtree ranks belong to.
+ *
+ * Optimization: collect contemporaneous JOIN/LEAVE requests at each
+ * rank for a short time before applying them and sending them upstream.
+ * During that time, JOINs/LEAVEs of the same key may be combined.
+ *
+ * broker.online use case:
+ * Groups are used for instance quorum detection.  The state machine calls
+ * groups.join broker.online in the QUORUM state on all ranks.  Rank 0 calls
+ * groups.get broker.online which notifies the broker as membership evolves,
+ * and when the quorum condition is satisfied, the state transitions
+ * to RUN.  The 'broker.online' group is also monitored by the resource module
+ * so that it can inform the scheduler as execution targets go up/down.
+ *
+ * broker.torpid use case:
+ * A broker.torpid group is maintained by the broker overlay (see overlay.c).
+ * The resource module also monitors broker.torpid and drains torpid nodes.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libidset/idset.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
+#include "overlay.h"
+#include "groups.h"
+
+static const double batch_timeout = 0.1;
+
+/* N.B. only one client can join a group per broker.  That client
+ * join request is cached in group->join_request so that when the client
+ * disconnects, we can identify its groups and force it to leave.
+ */
+struct group {
+    char *name; // used directly as zhashx key
+    struct idset *members;
+    const flux_msg_t *join_request;
+    struct flux_msglist *watchers;
+};
+
+struct groups {
+    struct broker *ctx;
+    flux_msg_handler_t **handlers;
+    zhashx_t *groups;
+    json_t *batch; // dict of arrays, keyed by group name
+    flux_watcher_t *batch_timer;
+    uint32_t rank;
+    struct idset *self;
+    struct idset *torpid; // current list of torpid peers at this broker rank
+};
+
+static void get_respond_all (struct groups *g, struct group *group);
+
+static void group_destroy (struct group *group)
+{
+    if (group) {
+        int saved_errno = errno;
+        free (group->name);
+        idset_destroy (group->members);
+        flux_msg_decref (group->join_request);
+        flux_msglist_destroy (group->watchers);
+        free (group);
+        errno = saved_errno;
+    }
+}
+
+// zhashx_destructor_fn footprint
+static void group_destructor (void **item)
+{
+    if (*item) {
+        struct group *group = *item;
+        group_destroy (group);
+        *item = NULL;
+    }
+}
+
+static struct group *group_create (const char *name)
+{
+    struct group *group;
+
+    if (!(group = calloc (1, sizeof (*group)))
+        || !(group->name = strdup (name))
+        || !(group->watchers = flux_msglist_create ())
+        || !(group->members = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+        group_destroy (group);
+        return NULL;
+    }
+    return group;
+}
+
+static struct group *group_lookup (struct groups *g,
+                                   const char *name,
+                                   bool create)
+{
+    struct group *group;
+
+    if (!(group = zhashx_lookup (g->groups, name))) {
+        if (!create) {
+            errno = ENOENT;
+            return NULL;
+        }
+        if (!(group = group_create (name)))
+            return NULL;
+        (void)zhashx_insert (g->groups, group->name, group);
+    }
+    return group;
+}
+
+/* Decode batch update object.
+ * Caller must idset_destroy 'ranks' on success.
+ * Returns 0 on success, -1 on failure with error set.
+ */
+static int update_decode (json_t *o, struct idset **ranksp, bool *set_flagp)
+{
+    const char *s;
+    struct idset *ranks = NULL;
+    int set_flag;
+
+    if (json_unpack (o, "{s:s s:b}", "ranks", &s, "set", &set_flag) < 0
+        || !(ranks = idset_decode (s))) {
+        errno = EPROTO;
+        return -1;
+    }
+    *ranksp = ranks;
+    *set_flagp = set_flag ? true : false;
+    return 0;
+}
+
+/* Encode batch update object.
+ * Returns object on success, NULL on failure with errno set.
+ */
+static json_t *update_encode (struct idset *ranks, bool set_flag)
+{
+    char *s;
+    json_t *o;
+
+    if (!(s = idset_encode (ranks, IDSET_FLAG_RANGE)))
+        return NULL;
+    if (!(o = json_pack ("{s:s s:b}", "ranks", s, "set", set_flag))) {
+        free (s);
+        errno = ENOMEM;
+        return NULL;
+    }
+    free (s);
+    return o;
+}
+
+/* Apply one join/leave batch update to the local hash.
+ */
+static void batch_apply_one (struct groups *g,
+                             struct group *group,
+                             json_t *entry)
+{
+    struct idset *ranks;
+    bool set_flag;
+    int rc;
+
+    if (update_decode (entry, &ranks, &set_flag) < 0) {
+        flux_log_error (g->ctx->h,
+                        "groups: error decoding batch update for group=%s",
+                        group->name);
+        return;
+    }
+    if (set_flag)
+        rc = idset_add (group->members, ranks);
+    else
+        rc = idset_subtract (group->members, ranks);
+    if (rc < 0) {
+        flux_log_error (g->ctx->h,
+                        "groups: error applying batch update for group=%s",
+                        group->name);
+    }
+    idset_destroy (ranks);
+}
+
+/* Apply all batch updates to the local hash.
+ * On rank 0, respond to any relevant groups.get requests.
+ */
+static void batch_apply (struct groups *g)
+{
+    const char *name;
+    json_t *a;
+    struct group *group;
+    size_t index;
+    json_t *entry;
+
+    json_object_foreach (g->batch, name, a) {
+        if (!(group = group_lookup (g, name, true))) {
+            flux_log_error (g->ctx->h,
+                "groups: error creating group during batch update for group=%s",
+                name);
+            continue;
+        }
+        json_array_foreach (a, index, entry) {
+            batch_apply_one (g, group, entry);
+        }
+        get_respond_all (g, group);
+    }
+    json_object_clear (g->batch);
+    flux_watcher_stop (g->batch_timer);
+}
+
+/* Add a batch update object to the batch queue, and activate timer if
+ * queue was initially empty.  Queued updates are applied on timer expiration.
+ * 'update' may be either an individual get/set operation, or an array of them.
+ * Returns 0 if update object is accepted, -1 on failure with errno set.
+ */
+static int batch_append (struct groups *g, const char *name, json_t *update)
+{
+    size_t size = json_object_size (g->batch);
+    json_t *a;
+    int rc;
+
+    if (!(a = json_object_get (g->batch, name))) {
+        if (!(a = json_array ())
+            || json_object_set_new (g->batch, name, a) < 0) {
+            json_decref (a);
+            goto nomem;
+        }
+    }
+    if (json_is_array (update))
+        rc = json_array_extend (a, update);
+    else
+        rc = json_array_append (a, update);
+    if (rc < 0)
+        goto nomem;
+    if (size == 0) {
+        flux_timer_watcher_reset (g->batch_timer, batch_timeout, 0.);
+        flux_watcher_start (g->batch_timer);
+    }
+    return 0;
+nomem:
+    errno = ENOMEM;
+    return -1;
+}
+
+/* Try to reduce like updates to a particular group into one update.
+ * Returns the one update if successful, else NULL.  This is a "best effort"
+ * operation, so NULL should not be treated as a fatal error.
+ */
+static json_t *batch_reduce_one (struct groups *g, json_t *a)
+{
+    size_t index;
+    struct idset *ids = NULL;
+    bool set_flag = false;
+    json_t *update;
+    json_t *new_update = NULL;
+
+    if (json_array_size (a) < 2)
+        return NULL;
+    json_array_foreach (a, index, update) {
+        struct idset *new_ids;
+        bool new_set_flag;
+
+        if (update_decode (update, &new_ids, &new_set_flag) < 0) {
+            flux_log_error (g->ctx->h, "groups: reduce decode update failed");
+            goto error;
+        }
+        if (index == 0) {
+            ids = new_ids;
+            set_flag = new_set_flag;
+            continue;
+        }
+        if (new_set_flag != set_flag) {
+            idset_destroy (new_ids);
+            goto error;
+        }
+        if (idset_add (ids, new_ids) < 0) {
+            flux_log_error (g->ctx->h, "groups: reduce idset update failed");
+            idset_destroy (new_ids);
+            goto error;
+        }
+        idset_destroy (new_ids);
+    }
+    if (!(new_update = update_encode (ids, set_flag)))
+        flux_log_error (g->ctx->h, "groups: reduce encode update failed");
+error:
+    idset_destroy (ids);
+    return new_update;
+}
+
+/* Try to reduce all keys in the current batch.  If a reduction is
+ * successful, replace the current array of operations with the new one.
+ * N.B. json_array_clear() + json_array_append() would be more direct,
+ * but destructive if the append fails.
+ */
+static void batch_reduce (struct groups *g)
+{
+    const char *name;
+    json_t *a;
+
+    json_object_foreach (g->batch, name, a) {
+        json_t *reduced;
+        json_t *new_a = NULL;
+
+        if (!(reduced = batch_reduce_one (g, a))
+            || !(new_a = json_array ())
+            || json_array_append (new_a, reduced) < 0
+            || json_object_set (g->batch, name, new_a) < 0)
+            goto next;
+next:
+        json_decref (new_a);
+        json_decref (reduced);
+    }
+}
+
+/* Apply all updates to local hash, and pass them upstream, if applicable.
+ * This is called when the timer expires, and may also be called from the
+ * disconnect and overlay loss handlers, which need to test group membership
+ * before generating LEAVEs.  Stop the batch timer, if running.
+ */
+static void batch_flush (struct groups *g)
+{
+    batch_reduce (g);
+    if (g->ctx->rank > 0) {
+        flux_future_t *f;
+        if (!(f = flux_rpc_pack (g->ctx->h,
+                                 "groups.update",
+                                 FLUX_NODEID_UPSTREAM,
+                                 FLUX_RPC_NORESPONSE,
+                                 "{s:O}",
+                                 "update", g->batch)))
+            flux_log_error (g->ctx->h, "error sending groups.update request");
+        flux_future_destroy (f);
+    }
+    batch_apply (g);
+    flux_watcher_stop (g->batch_timer);
+}
+
+/* Handle batch timeout.
+ */
+static void batch_timeout_cb (flux_reactor_t *r,
+                              flux_watcher_t *w,
+                              int revents,
+                              void *arg)
+{
+    struct groups *g = arg;
+    batch_flush (g);
+}
+
+/* Enqueue updates from a downstream peer.  After the batch timer expires,
+ * updates are applied to local hash and forwarded upstream.
+ * This is an internal (broker to broker) RPC which requires no response.
+ */
+static void update_request_cb (flux_t *h,
+                               flux_msg_handler_t *mh,
+                               const flux_msg_t *msg,
+                               void *arg)
+{
+    struct groups *g = arg;
+    json_t *update;
+    const char *name;
+    json_t *a;
+
+    if (flux_request_unpack (msg, NULL, "{s:o}", "update", &update) < 0) {
+        flux_log_error (h, "error decoding groups.update request");
+        return;
+    }
+    json_object_foreach (update, name, a) {
+        if (batch_append (g, name, a) < 0)
+            flux_log_error (h, "error enqueuing groups.update group=%s", name);
+    }
+}
+
+/* Add this broker rank to a group.
+ * Helper for groups.join RPC handler.
+ */
+static int groups_join (struct groups *g, const char *name)
+{
+    json_t *update;
+
+    if (!(update = update_encode (g->self, true))
+        || batch_append (g, name, update) < 0) {
+        ERRNO_SAFE_WRAP (json_decref, update);
+        return -1;
+    }
+    json_decref (update);
+    return 0;
+}
+
+/* Remove this broker rank from a group.
+ * Helper for groups.join RPC handler.
+ */
+static int groups_leave (struct groups *g, const char *name)
+{
+    json_t *update;
+
+    if (!(update = update_encode (g->self, false))
+        || batch_append (g, name, update) < 0) {
+        ERRNO_SAFE_WRAP (json_decref, update);
+        return -1;
+    }
+    json_decref (update);
+    return 0;
+}
+
+/* Process client request to JOIN a group.
+ * Create the group if needed, then add JOIN update to timed batch.
+ * Respond immediately, as opposed to when the batch is processed.
+ * The request is cached to support auto LEAVE upon client disconnect.
+ */
+static void join_request_cb (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
+{
+    struct groups *g = arg;
+    const char *name;
+    struct group *group;
+    char errbuf[256];
+    const char *errmsg = NULL;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "name", &name) < 0)
+        goto error;
+    if (!flux_msg_is_local (msg)) {
+        errno = EPROTO;
+        errmsg = "groups.join is restricted to the local broker";
+        goto error;
+    }
+    if (!(group = group_lookup (g, name, true)))
+        goto error;
+    if (group->join_request) {
+        snprintf (errbuf,
+                  sizeof (errbuf),
+                  "rank %lu is already a member of %s",
+                  (unsigned long)g->ctx->rank,
+                  name);
+        errmsg = errbuf;
+        errno = EEXIST;
+        goto error;
+    }
+    if (groups_join (g, name) < 0)
+        goto error;
+    group->join_request = flux_msg_incref (msg);
+    if (flux_respond (h, msg, NULL) < 0) {
+        if (errno != ENOSYS)
+            flux_log_error (h, "error responding to groups.leave request");
+    }
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0) {
+        if (errno != ENOSYS)
+            flux_log_error (h, "error responding to groups.leave request");
+    }
+}
+
+/* A client wishes to LEAVE a group.
+ * Drop cached JOIN request and add LEAVE update to batch.
+ * N.B. response is sent before batch updates are applied.
+ */
+static void leave_request_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct groups *g = arg;
+    const char *name;
+    struct group *group;
+    char errbuf[256];
+    const char *errmsg = NULL;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "name", &name) < 0)
+        goto error;
+    if (!flux_msg_is_local (msg)) {
+        errno = EPROTO;
+        errmsg = "groups.leave is restricted to the local broker";
+        goto error;
+    }
+    if (!(group = group_lookup (g, name, false))
+        || !group->join_request) {
+        snprintf (errbuf,
+                  sizeof (errbuf),
+                  "rank %lu is not a member of %s",
+                  (unsigned long)g->ctx->rank,
+                  name);
+        errmsg = errbuf;
+        errno = ENOENT;
+        goto error;
+    }
+    if (groups_leave (g, name) < 0)
+        goto error;
+    flux_msg_decref (group->join_request);
+    group->join_request = NULL;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to groups.join request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to groups.join request");
+}
+
+/* Respond to a request for group membership.
+ * Return 0 on success, -1 on error with errno set.
+ * A failure to respond is just logged, not treated as an error.
+ */
+static int get_respond_one (struct groups *g,
+                            struct group *group,
+                            const flux_msg_t *msg)
+{
+    char *s;
+
+    if (!(s = idset_encode (group->members, IDSET_FLAG_RANGE)))
+        return -1;
+    if (flux_respond_pack (g->ctx->h, msg, "{s:s}", "members", s) < 0) {
+        if (errno != ENOSYS) {
+            flux_log_error (g->ctx->h,
+                            "error responding to groups.get request");
+        }
+    }
+    free (s);
+    return 0;
+}
+
+/* 'group' membership has changed, respond to all pending groups.get
+ * requests.
+ */
+static void get_respond_all (struct groups *g, struct group *group)
+{
+    const flux_msg_t *request;
+
+    request = flux_msglist_first (group->watchers);
+    while (request) {
+        if (get_respond_one (g, group, request) < 0) {
+            flux_log_error (g->ctx->h,
+                            "error constructing groups.get response");
+        }
+        request = flux_msglist_next (group->watchers);
+    }
+}
+
+/* Process a groups.get request for group membership.  To avoid bugs arising
+ * from users making the request with nodeid=FLUX_NODEID_ANY on rank > 0,
+ * which would return that broker's subtree membership, reject such requests.
+ * If the group doesn't exist, it is created, empty.  This request may
+ * optionally specify the FLUX_RPC_STREAMING flag, to "watch" a group.
+ * Currently the streaming RPC is only terminated upon client disconnect.
+ */
+static void get_request_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
+{
+    struct groups *g = arg;
+    const char *name;
+    struct group *group;
+    const char *errmsg = NULL;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "name", &name) < 0)
+        goto error;
+    if (g->ctx->rank != 0) {
+        errmsg = "this RPC is only available on rank 0";
+        errno = EPROTO;
+        goto error;
+    }
+    if (!(group = group_lookup (g, name, true)))
+        goto error;
+    if (get_respond_one (g, group, msg) < 0)
+        goto error;
+    if (flux_msg_is_streaming (msg)) {
+        if (flux_msglist_append (group->watchers, msg) < 0)
+            goto error;
+    }
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0) {
+        if (errno != ENOSYS)
+            flux_log_error (h, "error responding to groups.get request");
+    }
+}
+
+/* A client has disconnected.  Find any groups with a cached JOIN request
+ * that matches the identity of the disconnecting client, and LEAVE those
+ * groups.  Also, drop streaming get requests that match the disconnecting
+ * client.
+ */
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct groups *g = arg;
+    struct group *group;
+
+    batch_flush (g); // handle any JOINs before disconnects
+
+    group = zhashx_first (g->groups);
+    while (group) {
+        if (group->join_request
+            && flux_disconnect_match (msg, group->join_request)) {
+            if (groups_leave (g, group->name) < 0) {
+                flux_log_error (h,
+                                "groups: error disconnecting from %s",
+                                group->name);
+            }
+            flux_msg_decref (group->join_request);
+            group->join_request = NULL;
+        }
+        if (flux_msglist_disconnect (group->watchers, msg) < 0) {
+            flux_log_error (h,
+                            "groups: error disconnecting watchers of group=%s",
+                            group->name);
+        }
+        group = zhashx_next (g->groups);
+    }
+}
+
+/* Recursive function to walk 'topology', adding all subtree ranks to 'ids'.
+ * Returns 0 on success, -1 on failure (errno is not set).
+ */
+static int add_subtree_ids (struct idset *ids, json_t *topology)
+{
+    int rank;
+    json_t *a;
+    size_t index;
+    json_t *entry;
+
+    if (json_unpack (topology, "{s:i s:o}", "rank", &rank, "children", &a) < 0
+        || idset_set (ids, rank) < 0)
+        return -1;
+    json_array_foreach (a, index, entry) {
+        if (add_subtree_ids (ids, entry) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+/* Generate JOIN/LEAVE for 'rank' in 'broker.torpid' group if rank becomes
+ * torpid/non-torpid.  N.B. For now, just operate on the single rank, not
+ * its entire subtree.  Although it would be straightforward to add the subtree
+ * to the group when the root becomes torpid, removing the whole subtree when
+ * responsiveness returns is less clear, since only a broker's immediate
+ * parent really knows how responsive it is.
+ */
+static void torpid_update (struct groups *g,
+                           uint32_t rank,
+                           struct idset *subtree_ids,
+                           bool torpid)
+{
+    struct idset *ids;
+    bool set_flag;
+    json_t *update = NULL;
+
+    if (torpid && !idset_test (g->torpid, rank))
+        set_flag = true;
+    else if (!torpid && idset_test (g->torpid, rank))
+        set_flag = false;
+    else
+        return; // nothing to do
+
+    if (!(ids = idset_create (0, IDSET_FLAG_AUTOGROW))
+        || idset_set (ids, rank) < 0
+        || !(update = update_encode (ids, set_flag))
+        || batch_append (g, "broker.torpid", update) < 0
+        || (set_flag ? idset_set (g->torpid, rank)
+                     : idset_clear (g->torpid, rank)) < 0) {
+        flux_log_error (g->ctx->h, "error updating broker.torpid");
+    }
+    idset_destroy (ids);
+    json_decref (update);
+}
+
+static void auto_leave (struct groups *g,
+                        const char *status,
+                        uint32_t rank,
+                        struct idset *ids)
+{
+    struct group *group;
+
+    group = zhashx_first (g->groups);
+    while (group) {
+        struct idset *x;
+        json_t *update;
+
+        if ((x = idset_intersect (group->members, ids))
+            && idset_count (x) > 0) {
+            if (!(update = update_encode (x, false))
+                || batch_append (g, group->name, update) < 0) {
+                flux_log_error (g->ctx->h,
+                        "groups: error auto-updating %s on subtree loss",
+                        group->name);
+            }
+            json_decref (update);
+        }
+        idset_destroy (x);
+        group = zhashx_next (g->groups);
+    }
+}
+
+static void overlay_monitor_cb (struct overlay *ov, uint32_t rank, void *arg)
+{
+    struct groups *g = arg;
+    const char *status = overlay_get_subtree_status (ov, rank);
+    json_t *topology;
+    struct idset *ids = NULL;
+
+    batch_flush (g); // handle any pending ops first
+
+    /* Prepare a list of ranks that are members of subtree rooted at rank.
+     */
+    if (!(topology = overlay_get_subtree_topo (ov, rank))
+        || !(ids = idset_create (0, IDSET_FLAG_AUTOGROW))
+        || add_subtree_ids (ids, topology) < 0)
+        goto done;
+
+    /* Generate LEAVEs for any groups 'rank' (and subtree) may be a member
+     * of if transitioning to lost (crashed) or offline (shutdown).
+     */
+    if (streq (status, "lost")
+        || streq (status, "offline")) {
+        auto_leave (g, status, rank, ids);
+    }
+    /* Update broker.torpid if torpidity has changed while subtree is in
+     * one of the "online" states.
+     */
+    else if (streq (status, "full")
+        || streq (status, "partial")
+        || streq (status, "degraded")) {
+        torpid_update (g, rank, ids, overlay_peer_is_torpid (ov, rank));
+    }
+
+done:
+    idset_destroy (ids);
+    json_decref (topology);
+    return;
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {   FLUX_MSGTYPE_REQUEST,
+        "groups.update",
+        update_request_cb,
+        0
+    },
+    {   FLUX_MSGTYPE_REQUEST,
+        "groups.join",
+        join_request_cb,
+        0
+    },
+    {   FLUX_MSGTYPE_REQUEST,
+        "groups.leave",
+        leave_request_cb,
+        0
+    },
+    {   FLUX_MSGTYPE_REQUEST,
+        "groups.get",
+        get_request_cb,
+        FLUX_ROLE_USER,
+    },
+    {   FLUX_MSGTYPE_REQUEST,
+        "groups.disconnect",
+        disconnect_cb,
+        FLUX_ROLE_USER,
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void groups_destroy (struct groups *g)
+{
+    if (g) {
+        int saved_errno = errno;
+        zhashx_destroy (&g->groups);
+        json_decref (g->batch);
+        idset_destroy (g->self);
+        idset_destroy (g->torpid);
+        flux_msg_handler_delvec (g->handlers);
+        flux_watcher_destroy (g->batch_timer);
+        free (g);
+        errno = saved_errno;
+    }
+}
+
+struct groups *groups_create (struct broker *ctx)
+{
+    struct groups *g;
+
+    if (!(g = calloc (1, sizeof (*g))))
+        return NULL;
+    g->ctx = ctx;
+    if (!(g->batch = json_object ())
+        || !(g->groups = zhashx_new ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    if (!(g->self = idset_create (0, IDSET_FLAG_AUTOGROW))
+        || idset_set (g->self, g->ctx->rank) < 0
+        || !(g->torpid = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        goto error;
+    zhashx_set_destructor (g->groups, group_destructor);
+    zhashx_set_key_duplicator (g->groups, NULL);
+    zhashx_set_key_destructor (g->groups, NULL);
+    if (flux_msg_handler_addvec (ctx->h, htab, g, &g->handlers) < 0)
+        goto error;
+    if (!(g->batch_timer = flux_timer_watcher_create (flux_get_reactor (ctx->h),
+                                                      0.,
+                                                      0.,
+                                                      batch_timeout_cb,
+                                                      g)))
+        goto error;
+    overlay_set_monitor_cb (ctx->overlay, overlay_monitor_cb, g);
+    return g;
+error:
+    groups_destroy (g);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/broker/groups.h b/src/broker/groups.h
new file mode 100644
index 000000000000..97455f2ee900
--- /dev/null
+++ b/src/broker/groups.h
@@ -0,0 +1,24 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _BROKER_GROUPS_H
+#define _BROKER_GROUPS_H
+
+#include <flux/core.h>
+#include "src/common/libidset/idset.h"
+
+#include "broker.h"
+
+struct groups *groups_create (struct broker *ctx);
+void groups_destroy (struct groups *g);
+
+#endif // !_BROKER_GROUPS_H
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/broker/heartbeat.c b/src/broker/heartbeat.c
deleted file mode 100644
index 756e18209990..000000000000
--- a/src/broker/heartbeat.c
+++ /dev/null
@@ -1,159 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <czmq.h>
-#include <errno.h>
-#include <flux/core.h>
-
-#include "src/common/libutil/log.h"
-
-#include "heartbeat.h"
-
-struct heartbeat_struct {
-    flux_t *h;
-    double rate;
-    flux_watcher_t *timer;
-    flux_msg_handler_t *mh;
-    int send_epoch;
-    int epoch;
-};
-
-static const double min_heartrate = 0.01;   /* min seconds */
-static const double max_heartrate = 30;     /* max seconds */
-static const double dfl_heartrate = 2;
-
-
-void heartbeat_destroy (heartbeat_t *hb)
-{
-    if (hb) {
-        flux_watcher_destroy (hb->timer);
-        flux_msg_handler_destroy (hb->mh);
-        free (hb);
-    }
-}
-
-heartbeat_t *heartbeat_create (void)
-{
-    heartbeat_t *hb = calloc (1, sizeof (*hb));
-
-    if (!hb) {
-        errno = ENOMEM;
-        return NULL;
-    }
-
-    hb->rate = dfl_heartrate;
-    return hb;
-}
-
-void heartbeat_set_flux (heartbeat_t *hb, flux_t *h)
-{
-    hb->h = h;
-}
-
-int heartbeat_register_attrs (heartbeat_t *hb, attr_t *attrs)
-{
-    if (attr_add_active_int (attrs, "heartbeat-epoch",
-                             &hb->epoch, FLUX_ATTRFLAG_READONLY) < 0)
-        return -1;
-    return 0;
-}
-
-int heartbeat_set_rate (heartbeat_t *hb, double rate)
-{
-    if (rate < min_heartrate || rate > max_heartrate)
-        goto error;
-    hb->rate = rate;
-    return 0;
-error:
-    errno = EINVAL;
-    return -1;
-}
-
-double heartbeat_get_rate (heartbeat_t *hb)
-{
-    return hb->rate;
-}
-
-int heartbeat_get_epoch (heartbeat_t *hb)
-{
-    return hb->epoch;
-}
-
-static void event_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
-{
-    heartbeat_t *hb = arg;
-    int epoch;
-
-    if (flux_heartbeat_decode (msg, &epoch) < 0)
-        return;
-    if (epoch >= hb->epoch) { /* ensure epoch remains monotonic */
-        hb->epoch = epoch;
-    }
-}
-
-static void timer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
-{
-    heartbeat_t *hb = arg;
-    flux_msg_t *msg = NULL;
-
-    if (!(msg = flux_heartbeat_encode (hb->send_epoch++))) {
-        log_err ("heartbeat_encode");
-        goto done;
-    }
-    if (flux_send (hb->h, msg, 0) < 0) {
-        log_err ("flux_send");
-        goto done;
-    }
-done:
-    flux_msg_destroy (msg);
-}
-
-int heartbeat_start (heartbeat_t *hb)
-{
-    uint32_t rank;
-    struct flux_match match = FLUX_MATCH_EVENT;
-
-    if (!hb->h) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (flux_get_rank (hb->h, &rank) < 0)
-        return -1;
-    if (rank == 0) {
-        flux_reactor_t *r = flux_get_reactor (hb->h);
-        flux_reactor_now_update (r);
-        if (!(hb->timer = flux_timer_watcher_create (r, hb->rate, hb->rate,
-                                                     timer_cb, hb)))
-            return -1;
-        flux_watcher_start (hb->timer);
-    }
-    match.topic_glob = "hb";
-    if (!(hb->mh = flux_msg_handler_create (hb->h, match, event_cb, hb)))
-        return -1;
-    flux_msg_handler_start (hb->mh);
-    return 0;
-}
-
-void heartbeat_stop (heartbeat_t *hb)
-{
-    if (hb->timer)
-        flux_watcher_stop (hb->timer);
-    if (hb->mh)
-        flux_msg_handler_stop (hb->mh);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/heartbeat.h b/src/broker/heartbeat.h
deleted file mode 100644
index 90f644148179..000000000000
--- a/src/broker/heartbeat.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _BROKER_HEARTBEAT_H
-#define _BROKER_HEARTBEAT_H
-
-#include "attr.h"
-
-/* Manage the session heartbeat.
- *
- * All ranks should call heartbeat_start() to install reactor watchers.
- * On rank 0 only, this registers a reactor timer watcher which sends
- * the reactor event message.
- *
- * The heartbeat_get_epoch() getter obtains the most recently processed epoch.
- *
- * Note: rank 0's epoch update is driven by the receipt of the
- * heartbeat event, not its generation.
- */
-
-typedef struct heartbeat_struct heartbeat_t;
-
-heartbeat_t *heartbeat_create (void);
-void heartbeat_destroy (heartbeat_t *hb);
-
-/* Returns -1, EINVAL if rate is out of range (0.1, 30). */
-int heartbeat_set_rate (heartbeat_t *hb, double rate);
-double heartbeat_get_rate (heartbeat_t *hb);
-
-void heartbeat_set_flux (heartbeat_t *hb, flux_t *h);
-int heartbeat_register_attrs (heartbeat_t *hb, attr_t *attrs);
-
-void heartbeat_set_epoch (heartbeat_t *hb, int epoch);
-int heartbeat_get_epoch (heartbeat_t *hb);
-
-int heartbeat_start (heartbeat_t *hb);
-void heartbeat_stop (heartbeat_t *hb);
-
-#endif /* !_BROKER_HEARTBEAT_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/hello.c b/src/broker/hello.c
deleted file mode 100644
index 71f1869a33c4..000000000000
--- a/src/broker/hello.c
+++ /dev/null
@@ -1,301 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <czmq.h>
-#include <flux/core.h>
-
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/fsd.h"
-
-#include "hello.h"
-#include "reduce.h"
-
-/* After this many seconds, ignore topo-based hwm.
- * Override by setting hello.timeout broker attribute.
- */
-static double default_reduction_timeout = 10.;
-
-struct hello_struct {
-    flux_t *h;
-    attr_t *attrs;
-    flux_msg_handler_t **handlers;
-    uint32_t rank;
-    uint32_t size;
-    uint32_t count;
-
-    double start;
-
-    hello_cb_f cb;
-    void *cb_arg;
-
-    flux_reduce_t *reduce;
-};
-
-static void join_request (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg);
-
-static void r_reduce (flux_reduce_t *r, int batch, void *arg);
-static void r_sink (flux_reduce_t *r, int batch, void *arg);
-static void r_forward (flux_reduce_t *r, int batch, void *arg);
-static int r_itemweight (void *item);
-
-
-struct flux_reduce_ops reduce_ops = {
-    .destroy = NULL,
-    .reduce = r_reduce,
-    .sink = r_sink,
-    .forward = r_forward,
-    .itemweight = r_itemweight,
-};
-
-hello_t *hello_create (void)
-{
-    hello_t *hello = calloc (1, sizeof (*hello));
-
-    if (!hello) {
-        errno = ENOMEM;
-        return NULL;
-    }
-
-    hello->size = 1;
-    return hello;
-}
-
-void hello_destroy (hello_t *hello)
-{
-    if (hello) {
-        flux_reduce_destroy (hello->reduce);
-        flux_msg_handler_delvec (hello->handlers);
-        free (hello);
-    }
-}
-
-static int hwm_from_topology (attr_t *attrs)
-{
-    const char *s;
-    if (attr_get (attrs, "tbon.descendants", &s, NULL) < 0) {
-        log_err ("hello: reading tbon.descendants attribute");
-        return 1;
-    }
-    return strtoul (s, NULL, 10) + 1;
-}
-
-int hello_register_attrs (hello_t *hello, attr_t *attrs)
-{
-    const char *s;
-    double timeout = default_reduction_timeout;
-    int hwm = hwm_from_topology (attrs);
-    char num[32];
-
-    hello->attrs = attrs;
-    if (attr_get (attrs, "hello.timeout", &s, NULL) == 0) {
-        if (fsd_parse_duration (s, &timeout) < 0) {
-            log_err ("hello: invalid hello.timeout: %s", s);
-            return -1;
-        }
-        if (attr_delete (attrs, "hello.timeout", true) < 0)
-            return -1;
-    }
-    snprintf (num, sizeof (num), "%.3f", timeout);
-    if (attr_add (attrs, "hello.timeout", num, FLUX_ATTRFLAG_IMMUTABLE) < 0)
-        return -1;
-    if (attr_get (attrs, "hello.hwm", &s, NULL) == 0) {
-        hwm = strtoul (s, NULL, 10);
-        if (attr_delete (attrs, "hello.hwm", true) < 0)
-            return -1;
-    }
-    snprintf (num, sizeof (num), "%d", hwm);
-    if (attr_add (attrs, "hello.hwm", num, FLUX_ATTRFLAG_IMMUTABLE) < 0)
-        return -1;
-    return 0;
-}
-
-static const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_REQUEST, "hello.join",     join_request, 0 },
-    FLUX_MSGHANDLER_TABLE_END,
-};
-
-void hello_set_flux (hello_t *hello, flux_t *h)
-{
-    hello->h = h;
-}
-
-double hello_get_time (hello_t *hello)
-{
-    if (hello->start == 0. || hello->h == NULL)
-        return 0.;
-    return flux_reactor_now (flux_get_reactor (hello->h)) - hello->start;
-}
-
-int hello_get_count (hello_t *hello)
-{
-    return hello->count;
-}
-
-void hello_set_callback (hello_t *hello, hello_cb_f cb, void *arg)
-{
-    hello->cb = cb;
-    hello->cb_arg = arg;
-}
-
-bool hello_complete (hello_t *hello)
-{
-    return (hello->size == hello->count);
-}
-
-int hello_start (hello_t *hello)
-{
-    int rc = -1;
-    int flags = 0;
-    int hwm = 1;
-    double timeout = 0.;
-    const char *s;
-
-    if (flux_get_rank (hello->h, &hello->rank) < 0
-                        || flux_get_size (hello->h, &hello->size) < 0) {
-        log_err ("hello: error getting rank/size");
-        goto done;
-    }
-    if (flux_msg_handler_addvec (hello->h, htab, hello,
-                                 &hello->handlers) < 0) {
-        log_err ("hello: adding message handlers");
-        goto done;
-    }
-    if (hello->attrs) {
-        if (attr_get (hello->attrs, "hello.hwm", &s, NULL) < 0) {
-            log_err ("hello: reading hello.hwm attribute");
-            goto done;
-        }
-        hwm = strtoul (s, NULL, 10);
-        if (attr_get (hello->attrs, "hello.timeout", &s, NULL) < 0) {
-            log_err ("hello: reading hello.timeout attribute");
-            goto done;
-        }
-        if (fsd_parse_duration (s, &timeout) < 0)
-            log_err ("hello: invalid hello.timeout attribute");
-    }
-    if (timeout > 0.)
-        flags |= FLUX_REDUCE_TIMEDFLUSH;
-    if (hwm > 0)
-        flags |= FLUX_REDUCE_HWMFLUSH;
-    if (!(hello->reduce = flux_reduce_create (hello->h, reduce_ops,
-                                              timeout, hello, flags))) {
-        log_err ("hello: creating reduction handle");
-        goto done;
-    }
-    if (flux_reduce_opt_set (hello->reduce, FLUX_REDUCE_OPT_HWM,
-                             &hwm, sizeof (hwm)) < 0) {
-        log_err ("hello: setting FLUX_REDUCE_OPT_HWM");
-        goto done;
-    }
-
-    flux_reactor_t *r = flux_get_reactor (hello->h);
-    flux_reactor_now_update (r);
-    hello->start = flux_reactor_now (r);
-    if (flux_reduce_append (hello->reduce, (void *)(uintptr_t)1, 0) < 0)
-        goto done;
-    rc = 0;
-done:
-    return rc;
-}
-
-/* handle a message sent from downstream via downstream's r_forward op.
- */
-static void join_request (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg)
-{
-    hello_t *hello = arg;
-    int count, batch;
-
-    if (flux_request_unpack (msg, NULL, "{ s:i s:i }",
-                             "count", &count,
-                             "batch", &batch) < 0)
-        log_err_exit ("hello: flux_request_unpack");
-    if (batch != 0 || count <= 0)
-        log_msg_exit ("hello: error decoding join request");
-    if (flux_reduce_append (hello->reduce, (void *)(uintptr_t)count, batch) < 0)
-        log_err_exit ("hello: flux_reduce_append");
-}
-
-/* Reduction ops
- * N.B. since we are reducing integers, we cheat and avoid memory
- * allocation by stuffing the int inside the pointer value.
- */
-
-/* Pop one or more counts, push their sum
- */
-static void r_reduce (flux_reduce_t *r, int batch, void *arg)
-{
-    int i, count = 0;
-
-    assert (batch == 0);
-
-    while ((i = (uintptr_t)flux_reduce_pop (r)) > 0)
-        count += i;
-    if (count > 0 && flux_reduce_push (r, (void *)(uintptr_t)count) < 0)
-        log_err_exit ("hello: flux_reduce_push");
-    /* Invariant for r_sink and r_forward:
-     * after reduce, handle contains exactly one item.
-     */
-}
-
-/* (called on rank 0 only) Pop exactly one count, update global count,
- * call the registered callback.
- * This may be called once the total hwm is reached on rank 0,
- * or after the timeout, as new messages arrive (after r_reduce).
- */
-static void r_sink (flux_reduce_t *r, int batch, void *arg)
-{
-    hello_t *hello = arg;
-    int count = (uintptr_t)flux_reduce_pop (r);
-
-    assert (batch == 0);
-    assert (count > 0);
-
-    hello->count += count;
-    if (hello->cb)
-        hello->cb (hello, hello->cb_arg);
-}
-
-/* (called on rank > 0 only) Pop exactly one count, forward upstream.
- * This may be called once the hwm is reached on this rank (based on topo),
- * or after the timeout, as new messages arrive (after r_reduce).
- */
-static void r_forward (flux_reduce_t *r, int batch, void *arg)
-{
-    flux_future_t *f;
-    hello_t *hello = arg;
-    int count = (uintptr_t)flux_reduce_pop (r);
-
-    assert (batch == 0);
-    assert (count > 0);
-
-    if (!(f = flux_rpc_pack (hello->h, "hello.join", FLUX_NODEID_UPSTREAM,
-                             FLUX_RPC_NORESPONSE, "{ s:i s:i }",
-                             "count", count,
-                             "batch", batch)))
-        log_err_exit ("hello: flux_rpc_pack");
-    flux_future_destroy (f);
-}
-
-/* How many original items does this item represent after reduction?
- * In this simple case it is just the value of the item (the count).
- */
-static int r_itemweight (void *item)
-{
-    return (uintptr_t)item;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/hello.h b/src/broker/hello.h
deleted file mode 100644
index d3c81c6efbd2..000000000000
--- a/src/broker/hello.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _BROKER_HELLO_H
-#define _BROKER_HELLO_H
-
-#include <stdbool.h>
-#include "attr.h"
-
-/* hello protocol is used to detect that TBON overlay has wired up.
- */
-
-typedef struct hello_struct hello_t;
-
-typedef void (*hello_cb_f)(hello_t *hello, void *arg);
-
-hello_t *hello_create (void);
-void hello_destroy (hello_t *hello);
-
-/* Register handle
- */
-void hello_set_flux (hello_t *hello, flux_t *h);
-
-/* Set up broker attributes
- */
-int hello_register_attrs (hello_t *hello, attr_t *attrs);
-
-/* Register callback for completion/progress.
- */
-void hello_set_callback (hello_t *hello, hello_cb_f cb, void *arg);
-
-/* Get time in seconds elapsed since hello_start()
- */
-double hello_get_time (hello_t *hello);
-
-/* Get number of ranks currently accounted for.
- */
-int hello_get_count (hello_t *hello);
-
-/* Get completion status
- */
-bool hello_complete (hello_t *hello);
-
-/* Start the hello protocol (call on all ranks).
- */
-int hello_start (hello_t *hello);
-
-#endif /* !_BROKER_HELLO_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/liblist.c b/src/broker/liblist.c
deleted file mode 100644
index bb7ab88cfac6..000000000000
--- a/src/broker/liblist.c
+++ /dev/null
@@ -1,142 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <czmq.h>
-#include <argz.h>
-
-#include "liblist.h"
-
-static int liblist_append_from_environment (zlist_t *libs, const char *libname)
-{
-    const char *path;
-    char *argz = NULL;
-    size_t argz_len;
-    int rc = -1;
-    char *filename, *entry = NULL;
-
-    if ((path = getenv ("LD_LIBRARY_PATH"))) {
-        if (argz_create_sep (path, ':', &argz, &argz_len) != 0)
-            goto done;
-        while ((entry = argz_next (argz, argz_len, entry))) {
-            if (asprintf (&filename, "%s/%s", entry, libname) < 0)
-                goto done;
-            if (access (filename, F_OK) < 0) {
-                free (filename);
-                continue;
-            }
-            if (zlist_append (libs, filename) < 0) {
-                free (filename);
-                goto done;
-            }
-        }
-    }
-    rc = 0;
-done:
-    if (argz)
-        free (argz);
-    return rc;
-}
-
-static int split2 (char *s, int delim, char **w1, char **w2)
-{
-    char *p = strchr (s, delim);
-    if (!p)
-        return -1;
-    *p++ = '\0';
-    *w1 = s;
-    *w2 = p;
-    return 0;
-}
-
-static void trim_end (char *s, int ch)
-{
-    int len = strlen (s);
-    while (len > 0) {
-        if (s[len - 1] != ch)
-            break;
-        s[--len] = '\0';
-    }
-}
-
-static int liblist_append_from_ldconfig (zlist_t *libs, const char *libname)
-{
-    FILE *f;
-    const char *cmd = "ldconfig -p | sed -e 's/([^(]*)[\t ]*//'" \
-                      "            | awk -F\" => \" '{print $1 \":\" $2};'";
-    char line[1024];
-    int rc = -1;
-
-    if (!(f = popen (cmd, "r")))
-        goto done;
-    while  (fgets (line, sizeof (line), f) != NULL) {
-        char *name, *path, *cpy;
-        if (split2 (line, ':', &name, &path) < 0)
-            continue;
-        while (isspace (*name))
-            name++;
-        if (strcmp (name, libname) != 0)
-            continue;
-        trim_end (path, '\n');
-        if (!(cpy = strdup (path)))
-            goto done;
-        if (zlist_append (libs, cpy) < 0) {
-            free (cpy);
-            goto done;
-        }
-    }
-    rc = 0;
-done:
-    if (f)
-        fclose (f);
-    return rc;
-}
-
-void liblist_destroy (zlist_t *libs)
-{
-    char *entry;
-    if (libs) {
-        while ((entry = zlist_pop (libs)))
-            free (entry);
-        zlist_destroy (&libs);
-    }
-}
-
-zlist_t *liblist_create (const char *libname)
-{
-    zlist_t *libs = NULL;
-
-    if (!(libs = zlist_new ()))
-        goto error;
-    if (strchr (libname, '/')) {
-        char *cpy = strdup (libname);
-        if (!cpy)
-            goto error;
-        if (zlist_append (libs, cpy) < 0) {
-            free (cpy);
-            goto error;
-        }
-    } else {
-        if (liblist_append_from_environment (libs, libname) < 0)
-            goto error;
-        if (liblist_append_from_ldconfig (libs, libname) < 0)
-            goto error;
-    }
-    return libs;
-error:
-    liblist_destroy (libs);
-    return NULL;
-}
-
-/*
- * vi:tabstop=4 shiftwidth
- */
diff --git a/src/broker/liblist.h b/src/broker/liblist.h
deleted file mode 100644
index 7651c129781e..000000000000
--- a/src/broker/liblist.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef HAVE_BROKER_LIBLIST_H
-#define HAVE_BROKER_LIBLIST_H 1
-
-/* Create a list of candidate library paths to 'libname', using directories
- * from LD_LIBRARY_PATH, if any, plus parsed 'ldconfig -p' output.
- * Purpose: search for libpmi.so with the ability to detect a special
- * symbol in flux's version and skip over it, continuing the search.
- */
-void liblist_destroy (zlist_t *libs);
-zlist_t *liblist_create (const char *libname);
-
-#endif /* !HAVE_BROKER_LIBLIST_H */
-
-/*
- * vi:tabstop=4 shiftwidth
- */
diff --git a/src/broker/log.c b/src/broker/log.c
index 93d5efbe2fd1..0200f67e8d3f 100644
--- a/src/broker/log.c
+++ b/src/broker/log.c
@@ -8,27 +8,37 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+/* CAUTION: logging errors with `flux_log()` here could result in
+ * deadlock.  Errors that need to be seen should be logged to stderr
+ * instead.
+ */
+
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
+#include <inttypes.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/wallclock.h"
 #include "src/common/libutil/stdlog.h"
+#include "src/common/libutil/timestamp.h"
+#include "ccan/str/str.h"
 
 #include "log.h"
 
+typedef enum { MODE_LEADER, MODE_LOCAL } stderr_mode_t;
+typedef enum { LOG_NO_TIMESTAMP=1 } log_flags_t;
+
 /* See descriptions in flux-broker-attributes(7) */
 static const int default_ring_size = 1024;
 static const int default_forward_level = LOG_DEBUG;
 static const int default_critical_level = LOG_CRIT;
 static const int default_stderr_level = LOG_ERR;
+static const stderr_mode_t default_stderr_mode = MODE_LEADER;
 static const int default_level = LOG_DEBUG;
 
-#define LOGBUF_MAGIC 0xe1e2e3e4
 typedef struct {
-    int magic;
     flux_t *h;
     flux_msg_handler_t **handlers;
     uint32_t rank;
@@ -37,60 +47,21 @@ typedef struct {
     int forward_level;
     int critical_level;
     int stderr_level;
+    stderr_mode_t stderr_mode;
     int level;
     zlist_t *buf;
     int ring_size;
     int seq;
-    zlist_t *sleepers;
+    struct flux_msglist *followers;
 } logbuf_t;
 
 struct logbuf_entry {
     char *buf;
-    int len;
     int seq;
 };
 
-#define SLEEPER_MAGIC 0xe4e3e2e1
-struct sleeper {
-    int magic;
-    flux_t *h;
-    flux_msg_handler_t *mh;
-    flux_msg_handler_f fun;
-    flux_msg_t *msg;
-    void *arg;
-};
-
 void logbuf_destroy (logbuf_t *logbuf);
 
-static void sleeper_destroy (struct sleeper *s)
-{
-    if (s) {
-        assert (s->magic == SLEEPER_MAGIC);
-        flux_msg_destroy (s->msg);
-        s->magic =~ SLEEPER_MAGIC;
-        free (s);
-    }
-}
-
-static struct sleeper *sleeper_create (flux_msg_handler_f fun,
-                                       flux_t *h, flux_msg_handler_t *mh,
-                                       const flux_msg_t *msg, void *arg)
-{
-    struct sleeper *s = calloc (1, sizeof (*s));
-    if (!s)
-        return NULL;
-    s->magic = SLEEPER_MAGIC;
-    s->h = h;
-    s->mh = mh;
-    s->fun = fun;
-    s->arg = arg;
-    if (!(s->msg = flux_msg_copy (msg, true))) {
-        sleeper_destroy (s);
-        return NULL;
-    }
-    return s;
-}
-
 static void logbuf_entry_destroy (struct logbuf_entry *e)
 {
     if (e) {
@@ -100,23 +71,26 @@ static void logbuf_entry_destroy (struct logbuf_entry *e)
     }
 }
 
+/* Create a logbuf entry from RFC 5424 formatted buf.
+ * Since buf may not have a terminating \0, add one so that it can be
+ * treated as a string when returned in a log.dmesg response.
+ */
 static struct logbuf_entry *logbuf_entry_create (const char *buf, int len)
 {
     struct logbuf_entry *e = calloc (1, sizeof (*e));
     if (!e)
         return NULL;
-    if (!(e->buf = calloc (1, len))) {
+    if (!(e->buf = malloc (len + 1))) {
         free (e);
         return NULL;
     }
     memcpy (e->buf, buf, len);
-    e->len = len;
+    e->buf[len] = '\0';
     return e;
 }
 
 static void logbuf_trim (logbuf_t *logbuf, int size)
 {
-    assert (logbuf->magic == LOGBUF_MAGIC);
     struct logbuf_entry *e;
     while (zlist_size (logbuf->buf) > size) {
         e = zlist_pop (logbuf->buf);
@@ -124,61 +98,10 @@ static void logbuf_trim (logbuf_t *logbuf, int size)
     }
 }
 
-static void logbuf_clear (logbuf_t *logbuf, int seq_index)
-{
-    struct logbuf_entry *e;
-
-    if (seq_index == -1)
-        logbuf_trim (logbuf, 0);
-    else {
-        while ((e = zlist_first (logbuf->buf)) && e->seq <= seq_index) {
-            e = zlist_pop (logbuf->buf);
-            logbuf_entry_destroy (e);
-        }
-    }
-}
-
-static int logbuf_get (logbuf_t *logbuf, int seq_index, int *seq,
-                       const char **buf, int *len)
-{
-    struct logbuf_entry *e = zlist_first (logbuf->buf);
-
-    while (e && e->seq <= seq_index)
-        e = zlist_next (logbuf->buf);
-    if (!e) {
-        errno = ENOENT;
-        return -1;
-    }
-    if (seq)
-        *seq = e->seq;
-    if (buf)
-        *buf = e->buf;
-    if (len)
-        *len = e->len;
-    return 0;
-}
-
-static int logbuf_sleepon (logbuf_t *logbuf, flux_msg_handler_f fun, flux_t *h,
-                           flux_msg_handler_t *mh, const flux_msg_t *msg,
-                           void *arg)
-{
-    assert (logbuf->magic == LOGBUF_MAGIC);
-    struct sleeper *s = sleeper_create (fun, h, mh, msg, arg);
-    if (!s)
-        return -1;
-    if (zlist_append (logbuf->sleepers, s) < 0) {
-        sleeper_destroy (s);
-        errno = ENOMEM;
-        return -1;
-    }
-    return 0;
-}
-
 static int append_new_entry (logbuf_t *logbuf, const char *buf, int len)
 {
-    assert (logbuf->magic == LOGBUF_MAGIC);
     struct logbuf_entry *e;
-    struct sleeper *s;
+    const flux_msg_t *msg;
 
     if (logbuf->ring_size > 0) {
         logbuf_trim (logbuf, logbuf->ring_size - 1);
@@ -192,9 +115,11 @@ static int append_new_entry (logbuf_t *logbuf, const char *buf, int len)
             errno = ENOMEM;
             return -1;
         }
-        while ((s = zlist_pop (logbuf->sleepers))) {
-            s->fun (s->h, s->mh, s->msg, s->arg);
-            sleeper_destroy (s);
+        msg = flux_msglist_first (logbuf->followers);
+        while (msg) {
+            if (flux_respond (logbuf->h, msg, e->buf) < 0)
+                log_err ("error responding to log.dmesg request");
+            msg = flux_msglist_next (logbuf->followers);
         }
     }
     return 0;
@@ -207,20 +132,18 @@ static logbuf_t *logbuf_create (void)
         errno = ENOMEM;
         goto cleanup;
     }
-    logbuf->magic = LOGBUF_MAGIC;
     logbuf->forward_level = default_forward_level;
     logbuf->critical_level = default_critical_level;
     logbuf->stderr_level = default_stderr_level;
+    logbuf->stderr_mode = default_stderr_mode;
     logbuf->level = default_level;
     logbuf->ring_size = default_ring_size;
     if (!(logbuf->buf = zlist_new ())) {
         errno = ENOMEM;
         goto cleanup;
     }
-    if (!(logbuf->sleepers = zlist_new ())) {
-        errno = ENOMEM;
+    if (!(logbuf->followers = flux_msglist_create ()))
         goto cleanup;
-    }
     return logbuf;
 cleanup:
     logbuf_destroy (logbuf);
@@ -230,22 +153,17 @@ static logbuf_t *logbuf_create (void)
 void logbuf_destroy (logbuf_t *logbuf)
 {
     if (logbuf) {
-        assert (logbuf->magic == LOGBUF_MAGIC);
         if (logbuf->buf) {
             logbuf_trim (logbuf, 0);
             zlist_destroy (&logbuf->buf);
         }
-        if (logbuf->sleepers) {
-            struct sleeper *s;
-            while ((s = zlist_pop (logbuf->sleepers)))
-                sleeper_destroy (s);
-            zlist_destroy (&logbuf->sleepers);
-        }
+        /* logbuf_destroy() would be called after local connector
+         * unloaded, so no need to send ENODATA to followers */
+        flux_msglist_destroy (logbuf->followers);
         if (logbuf->f)
             (void)fclose (logbuf->f);
         if (logbuf->filename)
             free (logbuf->filename);
-        logbuf->magic = ~LOGBUF_MAGIC;
         free (logbuf);
     }
 }
@@ -327,49 +245,36 @@ static int logbuf_set_filename (logbuf_t *logbuf, const char *destination)
     return 0;
 }
 
+static const char *int_to_string (int n)
+{
+    static char s[32]; // ample room to avoid overflow
+    (void)snprintf (s, sizeof (s), "%d", n);
+    return s;
+}
+
 static int attr_get_log (const char *name, const char **val, void *arg)
 {
     logbuf_t *logbuf = arg;
-    static char s[32];
-    int n, rc = -1;
-
-    if (!strcmp (name, "log-forward-level")) {
-        n = snprintf (s, sizeof (s), "%d", logbuf->forward_level);
-        assert (n < sizeof (s));
-        *val = s;
-    } else if (!strcmp (name, "log-critical-level")) {
-        n = snprintf (s, sizeof (s), "%d", logbuf->critical_level);
-        assert (n < sizeof (s));
-        *val = s;
-    } else if (!strcmp (name, "log-stderr-level")) {
-        n = snprintf (s, sizeof (s), "%d", logbuf->stderr_level);
-        assert (n < sizeof (s));
-        *val = s;
-    } else if (!strcmp (name, "log-ring-size")) {
-        n = snprintf (s, sizeof (s), "%d", logbuf->ring_size);
-        assert (n < sizeof (s));
-        *val = s;
-    } else if (!strcmp (name, "log-ring-used")) {
-        n = snprintf (s, sizeof (s), "%zd", zlist_size (logbuf->buf));
-        assert (n < sizeof (s));
-        *val = s;
-    } else if (!strcmp (name, "log-count")) {
-        n = snprintf (s, sizeof (s), "%d", logbuf->seq);
-        assert (n < sizeof (s));
-        *val = s;
-    } else if (!strcmp (name, "log-filename")) {
+
+    if (streq (name, "log-forward-level"))
+        *val = int_to_string (logbuf->forward_level);
+    else if (streq (name, "log-critical-level"))
+        *val = int_to_string (logbuf->critical_level);
+    else if (streq (name, "log-stderr-level"))
+        *val = int_to_string (logbuf->stderr_level);
+    else if (streq (name, "log-stderr-mode"))
+        *val = logbuf->stderr_mode == MODE_LEADER ? "leader" : "local";
+    else if (streq (name, "log-ring-size"))
+        *val = int_to_string (logbuf->ring_size);
+    else if (streq (name, "log-filename"))
         *val = logbuf->filename;
-    } else if (!strcmp (name, "log-level")) {
-        n = snprintf (s, sizeof (s), "%d", logbuf->level);
-        assert (n < sizeof (s));
-        *val = s;
-    } else {
+    else if (streq (name, "log-level"))
+        *val = int_to_string (logbuf->level);
+    else {
         errno = ENOENT;
-        goto done;
+        return -1;
     }
-    rc = 0;
-done:
-    return rc;
+    return 0;
 }
 
 static int attr_set_log (const char *name, const char *val, void *arg)
@@ -377,30 +282,46 @@ static int attr_set_log (const char *name, const char *val, void *arg)
     logbuf_t *logbuf = arg;
     int rc = -1;
 
-    if (!strcmp (name, "log-forward-level")) {
+    if (streq (name, "log-forward-level")) {
         int level = strtol (val, NULL, 10);
         if (logbuf_set_forward_level (logbuf, level) < 0)
             goto done;
-    } else if (!strcmp (name, "log-critical-level")) {
+    }
+    else if (streq (name, "log-critical-level")) {
         int level = strtol (val, NULL, 10);
         if (logbuf_set_critical_level (logbuf, level) < 0)
             goto done;
-    } else if (!strcmp (name, "log-stderr-level")) {
+    }
+    else if (streq (name, "log-stderr-level")) {
         int level = strtol (val, NULL, 10);
         if (logbuf_set_stderr_level (logbuf, level) < 0)
             goto done;
-    } else if (!strcmp (name, "log-ring-size")) {
+    }
+    else if (streq (name, "log-stderr-mode")) {
+        if (streq (val, "leader"))
+            logbuf->stderr_mode = MODE_LEADER;
+        else if (streq (val, "local"))
+            logbuf->stderr_mode = MODE_LOCAL;
+        else {
+            errno = EINVAL;
+            goto done;
+        }
+    }
+    else if (streq (name, "log-ring-size")) {
         int size = strtol (val, NULL, 10);
         if (logbuf_set_ring_size (logbuf, size) < 0)
             goto done;
-    } else if (!strcmp (name, "log-filename")) {
+    }
+    else if (streq (name, "log-filename")) {
         if (logbuf_set_filename (logbuf, val) < 0)
             goto done;
-    } else if (!strcmp (name, "log-level")) {
+    }
+    else if (streq (name, "log-level")) {
         int level = strtol (val, NULL, 10);
         if (logbuf_set_level (logbuf, level) < 0)
             goto done;
-    } else {
+    }
+    else {
         errno = ENOENT;
         goto done;
     }
@@ -417,38 +338,61 @@ static int logbuf_register_attrs (logbuf_t *logbuf, attr_t *attrs)
      * Only allowed to be set on rank 0 (ignore initial value on rank > 0).
      */
     if (logbuf->rank == 0) {
-        if (attr_add_active (attrs, "log-filename", 0,
-                             attr_get_log, attr_set_log, logbuf) < 0)
-            goto done;
-        if (attr_add_active (attrs, "log-stderr-level", 0,
-                             attr_get_log, attr_set_log, logbuf) < 0)
+        if (attr_add_active (attrs,
+                             "log-filename",
+                             0,
+                             attr_get_log,
+                             attr_set_log,
+                             logbuf) < 0)
             goto done;
-    } else {
+    }
+    else {
         (void)attr_delete (attrs, "log-filename", true);
-        if (attr_add (attrs, "log-filename", NULL, FLUX_ATTRFLAG_IMMUTABLE) < 0)
-            goto done;
-        (void)attr_delete (attrs, "log-stderr-level", true);
-        if (attr_add (attrs, "log-stderr-level", NULL, FLUX_ATTRFLAG_IMMUTABLE) < 0)
+        if (attr_add (attrs, "log-filename", NULL, ATTR_IMMUTABLE) < 0)
             goto done;
     }
 
-    if (attr_add_active (attrs, "log-level", 0,
-                         attr_get_log, attr_set_log, logbuf) < 0)
+    if (attr_add_active (attrs,
+                         "log-stderr-level",
+                         0,
+                         attr_get_log,
+                         attr_set_log,
+                         logbuf) < 0)
         goto done;
-    if (attr_add_active (attrs, "log-forward-level", 0,
-                         attr_get_log, attr_set_log, logbuf) < 0)
+    if (attr_add_active (attrs,
+                         "log-stderr-mode",
+                         0,
+                         attr_get_log,
+                         attr_set_log,
+                         logbuf) < 0)
         goto done;
-    if (attr_add_active (attrs, "log-critical-level", 0,
-                         attr_get_log, attr_set_log, logbuf) < 0)
+    if (attr_add_active (attrs,
+                         "log-level",
+                         0,
+                         attr_get_log,
+                         attr_set_log,
+                         logbuf) < 0)
         goto done;
-    if (attr_add_active (attrs, "log-ring-size", 0,
-                         attr_get_log, attr_set_log, logbuf) < 0)
+    if (attr_add_active (attrs,
+                         "log-forward-level",
+                         0,
+                         attr_get_log,
+                         attr_set_log,
+                         logbuf) < 0)
         goto done;
-    if (attr_add_active (attrs, "log-ring-used", 0,
-                         attr_get_log, NULL, logbuf) < 0)
+    if (attr_add_active (attrs,
+                         "log-critical-level",
+                         0,
+                         attr_get_log,
+                         attr_set_log,
+                         logbuf) < 0)
         goto done;
-    if (attr_add_active (attrs, "log-count", 0,
-                         attr_get_log, NULL, logbuf) < 0)
+    if (attr_add_active (attrs,
+                         "log-ring-size",
+                         0,
+                         attr_get_log,
+                         attr_set_log,
+                         logbuf) < 0)
         goto done;
     rc = 0;
 done:
@@ -457,19 +401,68 @@ static int logbuf_register_attrs (logbuf_t *logbuf, attr_t *attrs)
 
 static int logbuf_forward (logbuf_t *logbuf, const char *buf, int len)
 {
-    assert (logbuf->magic == LOGBUF_MAGIC);
-
     flux_future_t *f;
-    if (!(f = flux_rpc_raw (logbuf->h, "log.append", buf, len,
-                              FLUX_NODEID_UPSTREAM, FLUX_RPC_NORESPONSE)))
+    if (!(f = flux_rpc_raw (logbuf->h,
+                            "log.append",
+                            buf,
+                            len,
+                            FLUX_NODEID_UPSTREAM,
+                            FLUX_RPC_NORESPONSE)))
         return -1;
     flux_future_destroy (f);
     return 0;
 }
 
+static void log_timestamp (FILE *fp,
+                           struct stdlog_header *hdr,
+                           int flags)
+{
+    struct tm tm;
+    struct timeval tv;
+    char datetime[16]; /* 'MMM DD HH:MM:SS' */
+    char timezone[16]; /* TZ abbrev should be short, give 15 chars max */
+
+    if (flags & LOG_NO_TIMESTAMP)
+        return;
+    if (timestamp_parse (hdr->timestamp, &tm, &tv) < 0
+        || strftime (datetime, sizeof (datetime), "%b %d %T", &tm) == 0
+        || strftime (timezone, sizeof (timezone), "%Z", &tm) == 0)
+        fprintf (fp, "%s ", hdr->timestamp);
+    fprintf (fp, "%s.%06ld %s ", datetime, (long)tv.tv_usec, timezone);
+}
+
+/* Log a message to 'fp', if non-NULL.
+ * Set flags to LOG_NO_TIMESTAMP to suppress timestamp.
+ */
+static void log_fp (FILE *fp, int flags, const char *buf, int len)
+{
+    struct stdlog_header hdr;
+    const char *msg;
+    size_t msglen;
+    int severity;
+    uint32_t nodeid;
+
+    if (fp) {
+        if (stdlog_decode (buf, len, &hdr, NULL, NULL, &msg, &msglen) < 0)
+            fprintf (fp, "%.*s\n", len, buf);
+        else {
+            nodeid = strtoul (hdr.hostname, NULL, 10);
+            severity = STDLOG_SEVERITY (hdr.pri);
+            log_timestamp (fp, &hdr, flags);
+            fprintf (fp,
+                     "%s.%s[%" PRIu32 "]: %.*s\n",
+                     hdr.appname,
+                     stdlog_severity_to_string (severity),
+                     nodeid,
+                     (int)msglen,
+                     msg);
+        }
+    }
+    fflush (fp);
+}
+
 static int logbuf_append (logbuf_t *logbuf, const char *buf, int len)
 {
-    assert (logbuf->magic == LOGBUF_MAGIC);
     bool logged_stderr = false;
     int rc = 0;
     uint32_t rank = FLUX_NODEID_ANY;
@@ -487,21 +480,31 @@ static int logbuf_append (logbuf_t *logbuf, const char *buf, int len)
             if (append_new_entry (logbuf, buf, len) < 0)
                 rc = -1;
         }
-        if (severity <= logbuf->critical_level) {
-            flux_log_fprint (buf, len, stderr);
+        if (severity <= logbuf->critical_level
+            || (severity <= logbuf->stderr_level
+            && logbuf->stderr_mode == MODE_LOCAL)) {
+            int flags = 0;
+            if (logbuf->stderr_mode == MODE_LOCAL)
+                flags |= LOG_NO_TIMESTAMP; // avoid dup in syslog journal
+            log_fp (stderr, flags, buf, len);
             logged_stderr = true;
         }
     }
     if (severity <= logbuf->forward_level) {
         if (logbuf->rank == 0) {
-            flux_log_fprint (buf, len, logbuf->f);
-        } else {
+            log_fp (logbuf->f, 0, buf, len);
+        }
+        else {
             if (logbuf_forward (logbuf, buf, len) < 0)
                 rc = -1;
         }
     }
-    if (!logged_stderr && severity <= logbuf->stderr_level && logbuf->rank == 0)
-        flux_log_fprint (buf, len, stderr);
+    if (!logged_stderr
+        && severity <= logbuf->stderr_level
+        && logbuf->stderr_mode == MODE_LEADER
+        && logbuf->rank == 0) {
+        log_fp (stderr, 0, buf, len);
+    }
     return rc;
 }
 
@@ -517,13 +520,15 @@ static void logbuf_append_redirect (const char *buf, int len, void *arg)
 
 /* N.B. log requests have no response.
  */
-static void append_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                               const flux_msg_t *msg, void *arg)
+static void append_request_cb (flux_t *h,
+                               flux_msg_handler_t *mh,
+                               const flux_msg_t *msg,
+                               void *arg)
 {
     logbuf_t *logbuf = arg;
     uint32_t matchtag;
     const char *buf;
-    int len;
+    size_t len;
 
     if (flux_msg_get_matchtag (msg, &matchtag) < 0) {
         log_msg ("%s: malformed log request", __FUNCTION__);
@@ -545,104 +550,110 @@ static void append_request_cb (flux_t *h, flux_msg_handler_t *mh,
     }
 }
 
-static void clear_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                              const flux_msg_t *msg, void *arg)
+static void clear_request_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
 {
     logbuf_t *logbuf = arg;
-    int seq;
 
-    if (flux_request_unpack (msg, NULL, "{ s:i }", "seq", &seq) < 0)
-        goto error;
-    logbuf_clear (logbuf, seq);
+    logbuf_trim (logbuf, 0);
     flux_respond (h, msg, NULL);
     return;
-error:
-    flux_respond_error (h, msg, errno, NULL);
 }
 
-static void dmesg_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                              const flux_msg_t *msg, void *arg)
+static void dmesg_request_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
 {
     logbuf_t *logbuf = arg;
-    const char *buf;
-    int len;
-    int seq, follow;
+    struct logbuf_entry *e;
+    int follow;
+    int nobacklog = 0;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:b s?b}",
+                             "follow", &follow,
+                             "nobacklog", &nobacklog) < 0)
+        goto error;
 
-    if (flux_request_unpack (msg, NULL, "{ s:i s:b }",
-                             "seq", &seq,
-                             "follow", &follow) < 0)
+    if (!flux_msg_is_streaming (msg)) {
+        errno = EPROTO;
         goto error;
-    if (logbuf_get (logbuf, seq, &seq, &buf, &len) < 0) {
-        if (follow && errno == ENOENT) {
-            if (logbuf_sleepon (logbuf, dmesg_request_cb, h, mh, msg, arg) < 0)
+    }
+
+    if (!nobacklog) {
+        e = zlist_first (logbuf->buf);
+        while (e) {
+            if (flux_respond (h, msg, e->buf) < 0) {
+                log_err ("error responding to log.dmesg request");
                 goto error;
-            return; /* no reply */
+            }
+            e = zlist_next (logbuf->buf);
         }
+    }
+
+    if (follow) {
+        if (flux_msglist_append (logbuf->followers, msg) < 0)
+            goto error;
+    }
+    else {
+        errno = ENODATA;
         goto error;
     }
-    flux_respond_pack (h, msg, "{ s:i s:s# }",
-                               "seq", seq,
-                               "buf", buf, len);
+
     return;
 
 error:
-    flux_respond_error (h, msg, errno, NULL);
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        log_err ("error responding to log.dmesg request");
 }
 
-static int cmp_sender (flux_msg_t *msg, const char *uuid)
+static void disconnect_request_cb (flux_t *h,
+                                   flux_msg_handler_t *mh,
+                                   const flux_msg_t *msg,
+                                   void *arg)
 {
-    char *sender = NULL;
-    int rc = 0;
+    logbuf_t *logbuf = arg;
 
-    if (flux_msg_get_route_first (msg, &sender) < 0)
-        goto done;
-    if (!sender || strcmp (sender, uuid) != 0)
-        goto done;
-    rc = 1;
-done:
-    free (sender);
-    return rc;
+    flux_msglist_disconnect (logbuf->followers, msg);
 }
 
-static void disconnect_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                   const flux_msg_t *msg, void *arg)
+static void cancel_request_cb (flux_t *h,
+                                flux_msg_handler_t *mh,
+                                const flux_msg_t *msg,
+                                void *arg)
 {
     logbuf_t *logbuf = arg;
-    char *sender = NULL;
-    struct sleeper *s;
-    zlist_t *tmp = NULL;
 
-    assert (logbuf->magic == LOGBUF_MAGIC);
-    if (flux_msg_get_route_first (msg, &sender) < 0 || !sender)
-        goto done;
-    s = zlist_first (logbuf->sleepers);
-    while (s) {
-        assert (s->magic == SLEEPER_MAGIC);
-        if (cmp_sender (s->msg, sender)) {
-            if (!tmp && !(tmp = zlist_new ()))
-                goto done;
-            if (zlist_append (tmp, s) < 0)
-                goto done;
-        }
-        s = zlist_next (logbuf->sleepers);
-    }
-    if (tmp) {
-        while ((s = zlist_pop (tmp))) {
-            zlist_remove (logbuf->sleepers, s);
-            sleeper_destroy (s);
-        }
-    }
-done:
-    free (sender);
-    zlist_destroy (&tmp);
-    /* no response */
+    flux_msglist_cancel (h, logbuf->followers, msg);
 }
 
+static void stats_request_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    logbuf_t *logbuf = arg;
+
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:i s:i}",
+                           "ring-used", (int)zlist_size (logbuf->buf),
+                           "count", logbuf->seq) < 0)
+        flux_log_error (h, "error responding to log.stats-get");
+}
+
+
 static const struct flux_msg_handler_spec htab[] = {
     { FLUX_MSGTYPE_REQUEST, "log.append",         append_request_cb, 0 },
     { FLUX_MSGTYPE_REQUEST, "log.clear",          clear_request_cb, 0 },
     { FLUX_MSGTYPE_REQUEST, "log.dmesg",          dmesg_request_cb, 0 },
     { FLUX_MSGTYPE_REQUEST, "log.disconnect",     disconnect_request_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "log.cancel",         cancel_request_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "log.stats-get",      stats_request_cb, 0 },
     FLUX_MSGHANDLER_TABLE_END,
 };
 
diff --git a/src/broker/modhash.c b/src/broker/modhash.c
new file mode 100644
index 000000000000..d68080bfb5af
--- /dev/null
+++ b/src/broker/modhash.c
@@ -0,0 +1,779 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/dirwalk.h"
+#include "src/common/libutil/iterators.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+#include "module.h"
+#include "broker.h"
+#include "modhash.h"
+#include "overlay.h"
+
+struct modhash {
+    zhash_t *zh_byuuid;
+    flux_msg_handler_t **handlers;
+    struct broker *ctx;
+};
+
+static json_t *modhash_get_modlist (modhash_t *mh,
+                                    double now,
+                                    struct service_switch *sw);
+
+int modhash_response_sendmsg_new (modhash_t *mh, flux_msg_t **msg)
+{
+    const char *uuid;
+    module_t *p;
+
+    if (!*msg)
+        return 0;
+    if (!(uuid = flux_msg_route_last (*msg))) {
+        errno = EPROTO;
+        return -1;
+    }
+    if (!(p = zhash_lookup (mh->zh_byuuid, uuid))) {
+        errno = ENOSYS;
+        return -1;
+    }
+    return module_sendmsg_new (p, msg);
+}
+
+static void modhash_add (modhash_t *mh, module_t *p)
+{
+    /* always succeeds - uuids are by definition unique */
+    (void)zhash_insert (mh->zh_byuuid, module_get_uuid (p), p);
+    zhash_freefn (mh->zh_byuuid,
+                  module_get_uuid (p),
+                  (zhash_free_fn *)module_destroy);
+}
+
+static void modhash_remove (modhash_t *mh, module_t *p)
+{
+    zhash_delete (mh->zh_byuuid, module_get_uuid (p));
+}
+
+static int module_insmod_respond (flux_t *h, module_t *p)
+{
+    int rc;
+    int errnum = 0;
+    int status = module_get_status (p);
+    const flux_msg_t *msg = module_pop_insmod (p);
+
+    if (msg == NULL)
+        return 0;
+
+    /* If the module is EXITED, return error to insmod if mod_main() < 0
+     */
+    if (status == FLUX_MODSTATE_EXITED)
+        errnum = module_get_errnum (p);
+    if (errnum == 0)
+        rc = flux_respond (h, msg, NULL);
+    else
+        rc = flux_respond_error (h, msg, errnum, NULL);
+
+    flux_msg_decref (msg);
+    return rc;
+}
+
+static int module_rmmod_respond (flux_t *h, module_t *p)
+{
+    const flux_msg_t *msg;
+    int rc = 0;
+    while ((msg = module_pop_rmmod (p))) {
+        if (flux_respond (h, msg, NULL) < 0)
+            rc = -1;
+        flux_msg_decref (msg);
+    }
+    return rc;
+}
+
+/* If a message from a connector-routed client is not matched by this function,
+ * then it will fail with EAGAIN if the broker is in a pre-INIT state.
+ */
+static bool allow_early_request (const flux_msg_t *msg)
+{
+    const struct flux_match match[] = {
+        // state-machine.wait may be needed early by flux_reconnect(3) users
+        { FLUX_MSGTYPE_REQUEST, FLUX_MATCHTAG_NONE, "state-machine.wait" },
+        // let state-machine.get and attr.get work for flux-uptime(1)
+        { FLUX_MSGTYPE_REQUEST, FLUX_MATCHTAG_NONE, "state-machine.get" },
+        { FLUX_MSGTYPE_REQUEST, FLUX_MATCHTAG_NONE, "attr.get" },
+        { FLUX_MSGTYPE_REQUEST, FLUX_MATCHTAG_NONE, "log.dmesg" },
+    };
+    for (int i = 0; i < ARRAY_SIZE (match); i++)
+        if (flux_msg_cmp (msg, match[i]))
+            return true;
+    return false;
+}
+
+/* Callback to send disconnect messages on behalf of unloading module.
+ */
+static void disconnect_send_cb (const flux_msg_t *msg, void *arg)
+{
+    broker_ctx_t *ctx = arg;
+    flux_msg_t *cpy;
+    if (!(cpy = flux_msg_copy (msg, false)))
+        return;
+    broker_request_sendmsg_new (ctx, &cpy);
+}
+
+/* Handle messages on the service socket of a module.
+ */
+static void module_cb (module_t *p, void *arg)
+{
+    broker_ctx_t *ctx = arg;
+    flux_msg_t *msg = module_recvmsg (p);
+    int type;
+    int count;
+
+    if (!msg)
+        goto done;
+    if (flux_msg_get_type (msg, &type) < 0)
+        goto done;
+    switch (type) {
+        case FLUX_MSGTYPE_RESPONSE:
+            (void)broker_response_sendmsg_new (ctx, &msg);
+            break;
+        case FLUX_MSGTYPE_REQUEST:
+            count = flux_msg_route_count (msg);
+            /* Requests originated by the broker module will have a route
+             * count of 1.  Ensure that, when the module is unloaded, a
+             * disconnect message is sent to all services used by broker module.
+             */
+            if (count == 1) {
+                if (module_disconnect_arm (p, msg, disconnect_send_cb, ctx) < 0)
+                    flux_log_error (ctx->h, "error arming module disconnect");
+            }
+            /* Requests sent by the module on behalf of _its_ peers, e.g.
+             * connector-local module with connected clients, will have a
+             * route count greater than one here.  If this broker is not
+             * "online" (entered INIT state), politely rebuff these requests.
+             * Possible scenario for this message: user submitting a job on
+             * a login node before cluster reboot is complete.
+             */
+            else if (count > 1 && !ctx->online && !allow_early_request (msg)) {
+                const char *errmsg = "Upstream Flux broker is offline."
+                                     " Try again later.";
+
+                if (flux_respond_error (ctx->h, msg, EAGAIN, errmsg) < 0)
+                    flux_log_error (ctx->h, "send offline response message");
+                break;
+            }
+            broker_request_sendmsg_new (ctx, &msg);
+            break;
+        case FLUX_MSGTYPE_EVENT:
+            if (broker_event_sendmsg_new (ctx, &msg) < 0) {
+                flux_log_error (ctx->h,
+                                "%s(%s): broker_event_sendmsg_new %s",
+                                __FUNCTION__,
+                                module_get_name (p),
+                                flux_msg_typestr (type));
+            }
+            break;
+        default:
+            flux_log (ctx->h,
+                      LOG_ERR,
+                      "%s(%s): unexpected %s",
+                      __FUNCTION__,
+                      module_get_name (p),
+                      flux_msg_typestr (type));
+            break;
+    }
+done:
+    flux_msg_destroy (msg);
+}
+
+static void module_status_cb (module_t *p, int prev_status, void *arg)
+{
+    broker_ctx_t *ctx = arg;
+    int status = module_get_status (p);
+    const char *name = module_get_name (p);
+
+    /* Transition from INIT
+     * If module started normally, i.e. INIT->RUNNING, then
+     * respond to insmod requests now. O/w, delay responses until
+     * EXITED, when any errnum is available.
+     */
+    if (prev_status == FLUX_MODSTATE_INIT
+        && status == FLUX_MODSTATE_RUNNING) {
+        if (module_insmod_respond (ctx->h, p) < 0)
+            flux_log_error (ctx->h, "flux_respond to insmod %s", name);
+    }
+
+    /* Transition to EXITED
+     * Remove service routes, respond to insmod & rmmod request(s), if any,
+     * and remove the module (which calls pthread_join).
+     */
+    if (status == FLUX_MODSTATE_EXITED) {
+        flux_log (ctx->h, LOG_DEBUG, "module %s exited", name);
+        service_remove_byuuid (ctx->services, module_get_uuid (p));
+
+        if (module_insmod_respond (ctx->h, p) < 0)
+            flux_log_error (ctx->h, "flux_respond to insmod %s", name);
+
+        if (module_rmmod_respond (ctx->h, p) < 0)
+            flux_log_error (ctx->h, "flux_respond to rmmod %s", name);
+
+        modhash_remove (ctx->modhash, p);
+    }
+}
+
+static int mod_svc_cb (flux_msg_t **msg, void *arg)
+{
+    module_t *p = arg;
+    return module_sendmsg_new (p, msg);
+}
+
+/* Load broker module.
+ * 'name' is the name to use for the module (NULL = use dso basename minus ext)
+ * 'path' is either a dso path or a dso basename (e.g. "kvs" or "/a/b/kvs.so".
+ */
+int modhash_load (modhash_t *mh,
+                  const char *name,
+                  const char *path,
+                  json_t *args,
+                  const flux_msg_t *request,
+                  flux_error_t *error)
+{
+    const char *searchpath;
+    char *pattern = NULL;
+    zlist_t *files = NULL;
+    module_t *p;
+
+    if (!strchr (path, '/')) {
+        if (!(searchpath = getenv ("FLUX_MODULE_PATH"))) {
+            errprintf (error, "FLUX_MODULE_PATH is not set in the environment");
+            errno = EINVAL;
+            return -1;
+        }
+        if (asprintf (&pattern, "%s.so*", path) < 0) {
+            errprintf (error, "out of memory");
+            return -1;
+        }
+        if (!(files = dirwalk_find (searchpath,
+                                    DIRWALK_REALPATH | DIRWALK_NORECURSE,
+                                    pattern,
+                                    1,
+                                    NULL,
+                                    NULL))
+            || zlist_size (files) == 0) {
+            errprintf (error, "module not found in search path");
+            errno = ENOENT;
+            goto error;
+        }
+        path = zlist_first (files);
+    }
+    if (!(p = module_create (mh->ctx->h,
+                             overlay_get_uuid (mh->ctx->overlay),
+                             name,
+                             path,
+                             mh->ctx->rank,
+                             args,
+                             error)))
+        goto error;
+    modhash_add (mh, p);
+    if (service_add (mh->ctx->services,
+                     module_get_name (p),
+                     module_get_uuid (p),
+                     mod_svc_cb,
+                     p) < 0) {
+        errprintf (error, "error registering %s service", module_get_name (p));
+        goto module_remove;
+    }
+    module_set_poller_cb (p, module_cb, mh->ctx);
+    module_set_status_cb (p, module_status_cb, mh->ctx);
+    if (request && module_push_insmod (p, request) < 0) { // response deferred
+        errprintf (error, "error saving %s request", module_get_name (p));
+        goto service_remove;
+    }
+    if (module_start (p) < 0) {
+        errprintf (error, "error starting %s module", module_get_name (p));
+        goto service_remove;
+    }
+    flux_log (mh->ctx->h, LOG_DEBUG, "insmod %s", module_get_name (p));
+    zlist_destroy (&files);
+    free (pattern);
+    return 0;
+service_remove:
+    service_remove_byuuid (mh->ctx->services, module_get_uuid (p));
+module_remove:
+    modhash_remove (mh, p);
+error:
+    ERRNO_SAFE_WRAP (zlist_destroy, &files);
+    ERRNO_SAFE_WRAP (free, pattern);
+    return -1;
+}
+
+/* Load a module, asynchronously.
+ * N.B. modhash_load () handles response, unless it returns -1.
+ */
+static void load_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    broker_ctx_t *ctx = arg;
+    const char *name = NULL;
+    const char *path;
+    json_t *args;
+    flux_error_t error;
+    const char *errmsg = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s?s s:s s:o}",
+                             "name", &name,
+                             "path", &path,
+                             "args", &args) < 0)
+        goto error;
+    if (modhash_load (ctx->modhash, name, path, args, msg, &error) < 0) {
+        errmsg = error.text;
+        goto error;
+    }
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+
+static int unload_module (broker_ctx_t *ctx,
+                          const char *name,
+                          const flux_msg_t *request)
+{
+    module_t *p;
+
+    if (!(p = modhash_lookup_byname (ctx->modhash, name))) {
+        errno = ENOENT;
+        return -1;
+    }
+    if (module_stop (p, ctx->h) < 0)
+        return -1;
+    if (module_push_rmmod (p, request) < 0)
+        return -1;
+    flux_log (ctx->h, LOG_DEBUG, "rmmod %s", name);
+    return 0;
+}
+
+/* Unload a module, asynchronously.
+ * N.B. unload_module() handles response, unless it fails early
+ * and returns -1.
+ */
+static void remove_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    broker_ctx_t *ctx = arg;
+    const char *name;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "name", &name) < 0)
+        goto error;
+    if (unload_module (ctx, name, msg) < 0)
+        goto error;
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+/* List loaded modules
+ */
+static void list_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    broker_ctx_t *ctx = arg;
+    json_t *mods = NULL;
+    double now = flux_reactor_now (flux_get_reactor (h));
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (!(mods = modhash_get_modlist (ctx->modhash, now, ctx->services)))
+        goto error;
+    if (flux_respond_pack (h, msg, "{s:O}", "mods", mods) < 0)
+        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+    json_decref (mods);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+static void debug_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
+{
+    broker_ctx_t *ctx = arg;
+    const char *name;
+    int defer = -1;
+    module_t *p;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s?b}",
+                             "name", &name,
+                             "defer", &defer) < 0)
+        goto error;
+    if (!(p = modhash_lookup_byname (ctx->modhash, name))) {
+        errno = ENOENT;
+        goto error;
+    }
+    if (defer != -1) {
+        if (module_set_defer (p, defer) < 0)
+            goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to module.debug request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to module.debug request");
+}
+
+static void trace_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
+{
+    broker_ctx_t *ctx = arg;
+    struct flux_match match = FLUX_MATCH_ANY;
+    json_t *names = NULL;
+    size_t index;
+    json_t *entry;
+    const char *errmsg = NULL;
+    flux_error_t error;
+    zlist_t *l = NULL;
+    module_t *p;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:o s:i s:s}",
+                             "names", &names,
+                             "typemask", &match.typemask,
+                             "topic_glob", &match.topic_glob) < 0)
+        goto error;
+    if (!flux_msg_is_streaming (msg) || !json_is_array (names)) {
+        errno = EPROTO;
+        goto error;
+    }
+    /* Put modules in a list as the names are checked,
+     */
+    if (!(l = zlist_new ()))
+        goto nomem;
+    json_array_foreach (names, index, entry) {
+        const char *name = json_string_value (entry);
+        if (!(p = modhash_lookup_byname (ctx->modhash, (name)))) {
+            errprintf (&error, "%s module is not loaded", name);
+            errmsg = error.text;
+            errno = ENOENT;
+            goto error;
+        }
+        if (zlist_append (l, p) < 0)
+            goto nomem;
+    }
+    p = zlist_first (l);
+    while (p) {
+        (void)module_trace (p, msg);
+        p = zlist_next (l);
+    }
+    zlist_destroy (&l);
+    return;
+nomem:
+    errno = ENOMEM;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to module.trace");
+    zlist_destroy (&l);
+}
+
+static void status_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    broker_ctx_t *ctx = arg;
+    int status;
+    int errnum = 0;
+    const char *sender;
+    module_t *p;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s?i}",
+                             "status", &status,
+                             "errnum", &errnum) < 0
+        || !(sender = flux_msg_route_first (msg))
+        || !(p = modhash_lookup (ctx->modhash, sender))) {
+        const char *errmsg = "error decoding/finding module.status";
+        if (flux_msg_is_noresponse (msg))
+            flux_log_error (h, "%s", errmsg);
+        else if (flux_respond_error (h, msg, errno, errmsg) < 0)
+            flux_log_error (h, "error responding to module.status");
+        return;
+    }
+    switch (status) {
+        case FLUX_MODSTATE_FINALIZING:
+            module_mute (p);
+            break;
+        case FLUX_MODSTATE_EXITED:
+            module_set_errnum (p, errnum);
+            break;
+        default:
+            break;
+    }
+    /* Send a response if required.
+     * Hint: module waits for response in FINALIZING state.
+     */
+    if (!flux_msg_is_noresponse (msg)) {
+        if (flux_respond (h, msg, NULL) < 0) {
+            flux_log_error (h,
+                            "%s: error responding to module.status",
+                            module_get_name (p));
+        }
+    }
+    /* N.B. this will cause module_status_cb() to be called.
+     */
+    module_set_status (p, status);
+}
+
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    broker_ctx_t *ctx = arg;
+    module_t *p;
+
+    p = zhash_first (ctx->modhash->zh_byuuid);
+    while (p) {
+        module_trace_disconnect (p, msg);
+        p = zhash_next (ctx->modhash->zh_byuuid);
+    }
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "module.load",
+        load_cb,
+        0,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "module.remove",
+        remove_cb,
+        0,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "module.list",
+        list_cb,
+        FLUX_ROLE_USER,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "module.status",
+        status_cb,
+        0,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "module.debug",
+        debug_cb,
+        0,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "module.trace",
+        trace_cb,
+        0,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "module.disconnect",
+        disconnect_cb,
+        0,
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+modhash_t *modhash_create (struct broker *ctx)
+{
+    modhash_t *mh = calloc (1, sizeof (*mh));
+    if (!mh)
+        return NULL;
+    mh->ctx = ctx;
+    if (flux_msg_handler_addvec (ctx->h, htab, ctx, &mh->handlers) < 0)
+        goto error;
+    if (!(mh->zh_byuuid = zhash_new ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    return mh;
+error:
+    modhash_destroy (mh);
+    return NULL;
+}
+
+int modhash_destroy (modhash_t *mh)
+{
+    int saved_errno = errno;
+    const char *uuid;
+    module_t *p;
+    int count = 0;
+
+    if (mh) {
+        if (mh->zh_byuuid) {
+            FOREACH_ZHASH (mh->zh_byuuid, uuid, p) {
+                log_msg ("broker module '%s' was not properly shut down",
+                         module_get_name (p));
+                flux_error_t error;
+                if (module_cancel (p, &error) < 0)
+                    log_msg ("%s: %s", module_get_name (p), error.text);
+                count++;
+            }
+            zhash_destroy (&mh->zh_byuuid);
+        }
+        flux_msg_handler_delvec (mh->handlers);
+        free (mh);
+    }
+    errno = saved_errno;
+    return count;
+}
+
+static json_t *modhash_entry_tojson (module_t *p,
+                                     double now,
+                                     struct service_switch *sw)
+{
+    json_t *svcs;
+    json_t *entry = NULL;
+
+    if (!(svcs  = service_list_byuuid (sw, module_get_uuid (p))))
+        return NULL;
+    entry = json_pack ("{s:s s:s s:i s:i s:O s:i s:i}",
+                       "name", module_get_name (p),
+                       "path", module_get_path (p),
+                       "idle", (int)(now - module_get_lastseen (p)),
+                       "status", module_get_status (p),
+                       "services", svcs,
+                       "sendqueue", module_get_send_queue_count (p),
+                       "recvqueue", module_get_recv_queue_count (p));
+    json_decref (svcs);
+    return entry;
+}
+
+/* Prepare RFC 5 'mods' array for lsmod response.
+ */
+static json_t *modhash_get_modlist (modhash_t *mh,
+                                    double now,
+                                    struct service_switch *sw)
+{
+    json_t *mods = NULL;
+    module_t *p;
+
+    if (!(mods = json_array()))
+        goto nomem;
+    p = zhash_first (mh->zh_byuuid);
+    while (p) {
+        json_t *entry;
+
+        if (!(entry = modhash_entry_tojson (p, now, sw))
+            || json_array_append_new (mods, entry) < 0) {
+            json_decref (entry);
+            goto nomem;
+        }
+        p = zhash_next (mh->zh_byuuid);
+    }
+    return mods;
+nomem:
+    json_decref (mods);
+    errno = ENOMEM;
+    return NULL;
+}
+
+module_t *modhash_lookup (modhash_t *mh, const char *uuid)
+{
+    module_t *m;
+
+    if (!(m = zhash_lookup (mh->zh_byuuid, uuid))) {
+        errno = ENOENT;
+        return NULL;
+    }
+    return m;
+}
+
+module_t *modhash_lookup_byname (modhash_t *mh, const char *name)
+{
+    zlist_t *uuids;
+    char *uuid;
+    module_t *result = NULL;
+
+    if (!name)
+        return NULL;
+    if (!(uuids = zhash_keys (mh->zh_byuuid))) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    uuid = zlist_first (uuids);
+    while (uuid) {
+        module_t *p = zhash_lookup (mh->zh_byuuid, uuid);
+        if (p) {
+            if (streq (module_get_name (p), name)
+                || streq (module_get_path (p), name)) {
+                result = p;
+                break;
+            }
+        }
+        uuid = zlist_next (uuids);
+    }
+    zlist_destroy (&uuids);
+    return result;
+}
+
+int modhash_event_mcast (modhash_t *mh, const flux_msg_t *msg)
+{
+    module_t *p;
+
+    p = zhash_first (mh->zh_byuuid);
+    while (p) {
+        if (module_event_cast (p, msg) < 0)
+            return -1;
+        p = zhash_next (mh->zh_byuuid);
+    }
+    return 0;
+}
+
+module_t *modhash_first (modhash_t *mh)
+{
+    return zhash_first (mh->zh_byuuid);
+}
+
+module_t *modhash_next (modhash_t *mh)
+{
+    return zhash_next (mh->zh_byuuid);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/broker/modhash.h b/src/broker/modhash.h
new file mode 100644
index 000000000000..2e969783d28a
--- /dev/null
+++ b/src/broker/modhash.h
@@ -0,0 +1,66 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _BROKER_MODHASH_H
+#define _BROKER_MODHASH_H
+
+#include <jansson.h>
+
+#include "src/common/librouter/disconnect.h"
+
+#include "attr.h"
+#include "service.h"
+#include "module.h"
+#include "broker.h"
+
+typedef struct modhash modhash_t;
+
+/* Hash-o-modules, keyed by uuid
+ * Destructor returns the number of modules that had to be canceled.
+ */
+modhash_t *modhash_create (struct broker *ctx);
+int modhash_destroy (modhash_t *mh);
+
+/* Send an event message to all modules that have matching subscription.
+ */
+int modhash_event_mcast (modhash_t *mh, const flux_msg_t *msg);
+
+/* Send a response message to the module whose uuid matches the
+ * next hop in the routing stack.
+ */
+int modhash_response_sendmsg_new (modhash_t *mh, flux_msg_t **msg);
+
+/* Find a module matching 'uuid'.
+ */
+module_t *modhash_lookup (modhash_t *mh, const char *uuid);
+
+/* Find a module matching 'name'.
+ * Either the module name or the path given to module_add() works.
+ * N.B. this is a slow linear search - keep out of crit paths
+ */
+module_t *modhash_lookup_byname (modhash_t *mh, const char *name);
+
+/* Iterator
+ */
+module_t *modhash_first (modhash_t *mh);
+module_t *modhash_next (modhash_t *mh);
+
+int modhash_load (modhash_t *mh,
+                  const char *name,
+                  const char *path,
+                  json_t *args,
+                  const flux_msg_t *request,
+                  flux_error_t *error);
+
+#endif /* !_BROKER_MODHASH_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/broker/modservice.c b/src/broker/modservice.c
index efae59c9ad53..c3d5d59fd7ae 100644
--- a/src/broker/modservice.c
+++ b/src/broker/modservice.c
@@ -11,47 +11,27 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <stdio.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <getopt.h>
-#include <libgen.h>
-#include <unistd.h>
-#include <sys/param.h>
-#include <stdbool.h>
-#include <sys/time.h>
-#include <sys/resource.h>
-#include <stdarg.h>
-#include <argz.h>
-#include <czmq.h>
 #include <flux/core.h>
 
-#include "src/common/libutil/log.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libfluxutil/method.h"
+#include "ccan/str/str.h"
 
 #include "module.h"
 #include "modservice.h"
-#include "ping.h"
-#include "rusage.h"
 
 typedef struct {
     flux_t *h;
     module_t *p;
-    zlist_t *handlers;
     flux_watcher_t *w_prepare;
-    flux_watcher_t *w_check;
+    flux_msg_handler_t **handlers;
 } modservice_ctx_t;
 
 static void freectx (void *arg)
 {
     modservice_ctx_t *ctx = arg;
-    flux_msg_handler_t *mh;
-    while ((mh = zlist_pop (ctx->handlers)))
-        flux_msg_handler_destroy (mh);
-    zlist_destroy (&ctx->handlers);
+    flux_msg_handler_delvec (ctx->handlers);
     flux_watcher_destroy (ctx->w_prepare);
-    flux_watcher_destroy (ctx->w_check);
     free (ctx);
 }
 
@@ -64,11 +44,6 @@ static modservice_ctx_t *getctx (flux_t *h, module_t *p)
             errno = ENOMEM;
             return NULL;
         }
-        if (!(ctx->handlers = zlist_new ())) {
-            free (ctx);
-            errno = ENOMEM;
-            return NULL;
-        }
         ctx->h = h;
         ctx->p = p;
         flux_aux_set (h, "flux::modservice", ctx, freectx);
@@ -76,39 +51,6 @@ static modservice_ctx_t *getctx (flux_t *h, module_t *p)
     return ctx;
 }
 
-static void stats_get_cb (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg)
-{
-    flux_msgcounters_t mcs;
-
-    flux_get_msgcounters (h, &mcs);
-
-    if (flux_respond_pack (h, msg, "{ s:i s:i s:i s:i s:i s:i s:i s:i }",
-                           "#request (tx)", mcs.request_tx,
-                           "#request (rx)", mcs.request_rx,
-                           "#response (tx)", mcs.response_tx,
-                           "#response (rx)", mcs.response_rx,
-                           "#event (tx)", mcs.event_tx,
-                           "#event (rx)", mcs.event_rx,
-                           "#keepalive (tx)", mcs.keepalive_tx,
-                           "#keepalive (rx)", mcs.keepalive_rx) < 0)
-      FLUX_LOG_ERROR (h);
-}
-
-static void stats_clear_event_cb (flux_t *h, flux_msg_handler_t *mh,
-                                  const flux_msg_t *msg, void *arg)
-{
-    flux_clr_msgcounters (h);
-}
-
-static void stats_clear_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                    const flux_msg_t *msg, void *arg)
-{
-    flux_clr_msgcounters (h);
-    if (flux_respond (h, msg, NULL) < 0)
-        FLUX_LOG_ERROR (h);
-}
-
 static void shutdown_cb (flux_t *h, flux_msg_handler_t *mh,
                          const flux_msg_t *msg, void *arg)
 {
@@ -132,13 +74,13 @@ static void debug_cb (flux_t *h, flux_msg_handler_t *mh,
         }
         flux_aux_set (h, "flux::debug_flags", debug_flags, free);
     }
-    if (!strcmp (op, "setbit"))
+    if (streq (op, "setbit"))
         *debug_flags |= flags;
-    else if (!strcmp (op, "clrbit"))
+    else if (streq (op, "clrbit"))
         *debug_flags &= ~flags;
-    else if (!strcmp (op, "set"))
+    else if (streq (op, "set"))
         *debug_flags = flags;
-    else if (!strcmp (op, "clr"))
+    else if (streq (op, "clr"))
         *debug_flags = 0;
     else {
         errno = EPROTO;
@@ -153,97 +95,69 @@ static void debug_cb (flux_t *h, flux_msg_handler_t *mh,
 }
 
 /* Reactor loop is about to block.
+ * Notify broker that module is running, then disable the prepare watcher.
  */
 static void prepare_cb (flux_reactor_t *r, flux_watcher_t *w,
                         int revents, void *arg)
 {
     modservice_ctx_t *ctx = arg;
-    flux_msg_t *msg = flux_keepalive_encode (0, FLUX_MODSTATE_SLEEPING);
-    if (!msg || flux_send (ctx->h, msg, 0) < 0)
-        flux_log_error (ctx->h, "error sending keepalive");
-    flux_msg_destroy (msg);
-}
-
-/* Reactor loop just unblocked.
- */
-static void check_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
-{
-    modservice_ctx_t *ctx = arg;
-    flux_msg_t *msg = flux_keepalive_encode (0, FLUX_MODSTATE_RUNNING);
-    if (!msg || flux_send (ctx->h, msg, 0) < 0)
-        flux_log_error (ctx->h, "error sending keepalive");
-    flux_msg_destroy (msg);
-}
 
-static int register_event (modservice_ctx_t *ctx, const char *name,
-                           flux_msg_handler_f cb)
-{
-    struct flux_match match = FLUX_MATCH_EVENT;
-    flux_msg_handler_t *mh = NULL;
-    int rc = -1;
-
-    if (flux_match_asprintf (&match,
-                  "%s.%s",
-                  module_get_name (ctx->p),
-                  name) < 0) {
-        log_err ("asprintf");
-        goto cleanup;
-    }
-    if (!(mh = flux_msg_handler_create (ctx->h, match, cb, ctx->p))) {
-        log_err ("flux_msg_handler_create");
-        goto cleanup;
-    }
-    flux_msg_handler_start (mh);
-    if (zlist_append (ctx->handlers, mh) < 0) {
-        log_errn (ENOMEM, "zlist_append");
-        goto cleanup;
-    }
-    /* memory now managed by list */
-    mh = NULL;
-    if (flux_event_subscribe (ctx->h, match.topic_glob) < 0) {
-        log_err ("%s: flux_event_subscribe %s",
-                 __FUNCTION__, match.topic_glob);
-        goto cleanup;
-    }
-    rc = 0;
-cleanup:
-    flux_msg_handler_destroy (mh);
-    flux_match_free (match);
-    return rc;
+    if (flux_module_set_running (ctx->h) < 0)
+        flux_log_error (ctx->h, "error setting module status to running");
+    flux_watcher_destroy (ctx->w_prepare);
+    ctx->w_prepare = NULL;
 }
 
-static int register_request (modservice_ctx_t *ctx, const char *name,
-                             flux_msg_handler_f cb, uint32_t rolemask)
+static struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,
+      "shutdown",
+      shutdown_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "stats-get",
+      method_stats_get_cb,
+      FLUX_ROLE_ALL,
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "stats-clear",
+      method_stats_clear_cb,
+      0,
+    },
+    { FLUX_MSGTYPE_EVENT,
+      "stats-clear",
+      method_stats_clear_event_cb,
+      0,
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "debug",
+      debug_cb,
+      0,
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "rusage",
+      method_rusage_cb,
+      FLUX_ROLE_USER,
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "ping",
+      method_ping_cb,
+      FLUX_ROLE_USER,
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+static int mod_subscribe (flux_t *h, module_t *p, const char *method)
 {
-    struct flux_match match = FLUX_MATCH_REQUEST;
-    flux_msg_handler_t *mh = NULL;
-    int rc = -1;
+    char *topic;
 
-    if (flux_match_asprintf (&match,
-                  "%s.%s",
-                  module_get_name (ctx->p),
-                  name) < 0) {
-        log_err ("asprintf");
-        goto cleanup;
-    }
-    if (!(mh = flux_msg_handler_create (ctx->h, match, cb, ctx->p))) {
-        log_err ("flux_msg_handler_create");
-        goto cleanup;
-    }
-    flux_msg_handler_allow_rolemask (mh, rolemask);
-    flux_msg_handler_start (mh);
-    if (zlist_append (ctx->handlers, mh) < 0) {
-        log_errn (ENOMEM, "zlist_append");
-        goto cleanup;
+    if (asprintf (&topic, "%s.%s", module_get_name (p), method) < 0
+        || flux_event_subscribe (h, topic) < 0) {
+        ERRNO_SAFE_WRAP (free, topic);
+        return -1;
     }
-    /* memory now managed by list */
-    mh = NULL;
-    rc = 0;
-cleanup:
-    flux_msg_handler_destroy (mh);
-    flux_match_free (match);
-    return rc;
+    free (topic);
+    return 0;
 }
 
 int modservice_register (flux_t *h, module_t *p)
@@ -254,39 +168,31 @@ int modservice_register (flux_t *h, module_t *p)
     if (!ctx || !r)
         return -1;
 
-    if (register_request (ctx, "shutdown", shutdown_cb, FLUX_ROLE_OWNER) < 0)
-        return -1;
-    if (register_request (ctx, "stats.get", stats_get_cb, FLUX_ROLE_ALL) < 0)
-        return -1;
-    if (register_request (ctx, "stats.clear", stats_clear_request_cb, FLUX_ROLE_OWNER) < 0)
+    if (flux_aux_set (h,
+                      "flux::uuid",
+                      (char *)module_get_uuid (ctx->p),
+                      NULL) < 0)
         return -1;
-    if (register_request (ctx, "debug", debug_cb, FLUX_ROLE_OWNER) < 0)
+    if (flux_aux_set (h,
+                      "flux::name",
+                      (char *)module_get_name (ctx->p),
+                      NULL) < 0)
         return -1;
 
-    if (ping_initialize (h,
-                         module_get_name (ctx->p),
-                         module_get_uuid (ctx->p)) < 0) {
-        log_err ("ping_initialize");
-        return -1;
-    }
-    if (rusage_initialize (h, module_get_name (ctx->p)) < 0) {
-        log_err ("rusage_initialize");
+    if (flux_msg_handler_addvec_ex (h,
+                                    module_get_name (ctx->p),
+                                    htab,
+                                    ctx->p,
+                                    &ctx->handlers) < 0)
         return -1;
-    }
 
-    if (register_event (ctx, "stats.clear", stats_clear_event_cb) < 0)
+    if (mod_subscribe (h, ctx->p, "stats-clear") < 0)
         return -1;
 
-    if (!(ctx->w_prepare = flux_prepare_watcher_create (r, prepare_cb, ctx))) {
-        log_err ("flux_prepare_watcher_create");
-        return -1;
-    }
-    if (!(ctx->w_check = flux_check_watcher_create (r, check_cb, ctx))) {
-        log_err ("flux_check_watcher_create");
+    if (!(ctx->w_prepare = flux_prepare_watcher_create (r, prepare_cb, ctx)))
         return -1;
-    }
+
     flux_watcher_start (ctx->w_prepare);
-    flux_watcher_start (ctx->w_check);
     return 0;
 }
 
diff --git a/src/broker/module.c b/src/broker/module.c
index 289c8d2a478d..bda23d484d26 100644
--- a/src/broker/module.c
+++ b/src/broker/module.c
@@ -11,64 +11,57 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <stdio.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <libgen.h>
-#include <unistd.h>
-#include <sys/param.h>
-#include <stdbool.h>
-#include <sys/socket.h>
-#include <sys/time.h>
-#include <stdarg.h>
 #include <dlfcn.h>
-#include <inttypes.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
-#include <czmq.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+#include <signal.h>
+#include <pthread.h>
 #include <uuid.h>
+#ifndef UUID_STR_LEN
+#define UUID_STR_LEN 37     // defined in later libuuid headers
+#endif
 #include <flux/core.h>
 #include <jansson.h>
-#if HAVE_CALIPER
-#include <caliper/cali.h>
-#include <sys/syscall.h>
-#endif
 
+#include "src/common/libflux/plugin_private.h"
 #include "src/common/libutil/log.h"
-#include "src/common/libutil/iterators.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/basename.h"
+#include "src/common/librouter/subhash.h"
+#include "ccan/str/str.h"
 
-#include "heartbeat.h"
 #include "module.h"
 #include "modservice.h"
+#include "trace.h"
 
-#ifndef UUID_STR_LEN
-#define UUID_STR_LEN 37     // defined in later libuuid headers
-#endif
-
+struct broker_module {
+    flux_t *h;              /* ref to broker's internal flux_t handle */
 
-#define MODULE_MAGIC    0xfeefbe01
-struct module_struct {
-    int magic;
-
-    uint32_t rank;
-    flux_t *broker_h;
     flux_watcher_t *broker_w;
 
-    int lastseen;
-    heartbeat_t *heartbeat;
+    double lastseen;
 
-    zsock_t *sock;          /* broker end of PAIR socket */
-    struct flux_msg_cred cred; /* cred of connection */
+    flux_t *h_broker_end;   /* broker end of interthread channel */
+    char uri[128];
 
     uuid_t uuid;            /* uuid for unique request sender identity */
     char uuid_str[UUID_STR_LEN];
+    char *parent_uuid_str;
+    int rank;
+    json_t *attr_cache;     /* attrs to be cached in module flux_t */
+    flux_conf_t *conf;
     pthread_t t;            /* module thread */
     mod_main_f *main;       /* dlopened mod_main() */
     char *name;
+    char *path;             /* retain the full path as a key for lookup */
     void *dso;              /* reference on dlopened module */
-    int size;               /* size of .so file for lsmod */
-    char *digest;           /* digest of .so file for lsmod */
     size_t argz_len;
     char *argz;
     int status;
@@ -80,103 +73,127 @@ struct module_struct {
     module_status_cb_f status_cb;
     void *status_arg;
 
-    zlist_t *rmmod;
-    flux_msg_t *insmod;
-
-    flux_t *h;               /* module's handle */
+    struct disconnect *disconnect;
 
-    zlist_t *subs;          /* subscription strings */
-};
+    struct flux_msglist *rmmod_requests;
+    struct flux_msglist *insmod_requests;
+    struct flux_msglist *trace_requests;
+    struct flux_msglist *deferred_messages;
 
-struct modhash_struct {
-    zhash_t *zh_byuuid;
-    uint32_t rank;
-    flux_t *broker_h;
-    heartbeat_t *heartbeat;
+    flux_t *h_module_end;   /* module end of interthread_channel */
+    struct subhash *sub;
 };
 
 static int setup_module_profiling (module_t *p)
 {
-#if HAVE_CALIPER
-    cali_begin_string_byname ("flux.type", "module");
-    cali_begin_int_byname ("flux.tid", syscall (SYS_gettid));
-    cali_begin_int_byname ("flux.rank", p->rank);
-    cali_begin_string_byname ("flux.name", p->name);
+    size_t len = strlen (p->name);
+    // one character longer than target to pass -Wstringop-truncation
+    char local_name[17] = {0};
+    const char *name_ptr = p->name;
+    // pthread name is limited to 16 bytes including \0 on linux
+    if (len > 15) {
+        strncpy (local_name, p->name, 16);
+        local_name[15] = 0;
+        name_ptr = local_name;
+    }
+    // Set the name of each thread to its module name
+#if HAVE_PTHREAD_SETNAME_NP_WITH_TID
+    (void) pthread_setname_np (pthread_self (), name_ptr);
+#else // e.g. macos
+    (void) pthread_setname_np (name_ptr);
 #endif
     return (0);
 }
 
+static int attr_cache_to_json (flux_t *h, json_t **cachep)
+{
+    json_t *cache;
+    const char *name;
+
+    if (!(cache = json_object ()))
+        return -1;
+    name = flux_attr_cache_first (h);
+    while (name) {
+        json_t *val = json_string (flux_attr_get (h, name));
+        if (!val || json_object_set_new (cache, name, val) < 0) {
+            json_decref (val);
+            goto error;
+        }
+        name = flux_attr_cache_next (h);
+    }
+    *cachep = cache;
+    return 0;
+error:
+    json_decref (cache);
+    return -1;
+}
+
+static int attr_cache_from_json (flux_t *h, json_t *cache)
+{
+    const char *name;
+    json_t *o;
+
+    json_object_foreach (cache, name, o) {
+        const char *val = json_string_value (o);
+        if (flux_attr_set_cacheonly (h, name, val) < 0)
+            return -1;
+    }
+    return 0;
+}
+
 /*  Synchronize the FINALIZING state with the broker, so the broker
  *   can stop messages to this module until we're fully shutdown.
  */
 static int module_finalizing (module_t *p)
 {
-    int rc = -1;
-    flux_msg_t *msg = NULL;
-    struct flux_match match = {
-        .typemask = FLUX_MSGTYPE_KEEPALIVE
-    };
-    /* Notify the broker we're finalizing, which will disable new
-     *  messages
-     */
-    if (!(msg = flux_keepalive_encode (0, FLUX_MODSTATE_FINALIZING))) {
-        flux_log_error (p->h, "module_finalizing: flux_keepalive_encode");
+    flux_future_t *f;
+
+    if (!(f = flux_rpc_pack (p->h_module_end,
+                             "module.status",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:i}",
+                             "status", FLUX_MODSTATE_FINALIZING))
+        || flux_rpc_get (f, NULL)) {
+        flux_log_error (p->h_module_end, "module.status FINALIZING error");
+        flux_future_destroy (f);
         return -1;
     }
-    if (flux_send (p->h, msg, 0) < 0) {
-        flux_log_error (p->h, "module_finalizing: flux_send");
-        goto done;
-    }
-    flux_msg_destroy (msg);
-
-    /* Synchronize with the broker using a blocking recv for keepalive
-     *  message. This should be the only time the broker sends a keepalive
-     *  to a module.
-     */
-    if (!(msg = flux_recv (p->h, match, 0)))
-        flux_log_error (p->h, "module_finalizing: flux_recv");
-    rc = 0;
-done:
-    flux_msg_destroy (msg);
-    return rc;
+    flux_future_destroy (f);
+    return 0;
 }
 
 static void *module_thread (void *arg)
 {
     module_t *p = arg;
-    assert (p->magic == MODULE_MAGIC);
     sigset_t signal_set;
     int errnum;
-    char *uri = NULL;
+    char uri[128];
     char **av = NULL;
-    char *rankstr = NULL;
     int ac;
     int mod_main_errno = 0;
     flux_msg_t *msg;
+    flux_future_t *f;
 
     setup_module_profiling (p);
 
     /* Connect to broker socket, enable logging, register built-in services
      */
-    if (asprintf (&uri, "shmem://%s", p->uuid_str) < 0) {
-        log_err ("asprintf");
-        goto done;
-    }
-    if (!(p->h = flux_open (uri, 0))) {
+    if (!(p->h_module_end = flux_open (p->uri, 0))) {
         log_err ("flux_open %s", uri);
         goto done;
     }
-    if (asprintf (&rankstr, "%"PRIu32, p->rank) < 0) {
-        log_err ("asprintf");
+    if (attr_cache_from_json (p->h_module_end, p->attr_cache) < 0) {
+        log_err ("%s: error priming broker attribute cache", p->name);
         goto done;
     }
-    if (flux_attr_set_cacheonly (p->h, "rank", rankstr) < 0) {
-        log_err ("%s: error faking rank attribute", p->name);
+    flux_log_set_appname (p->h_module_end, p->name);
+    if (flux_set_conf (p->h_module_end, p->conf) < 0) {
+        log_err ("%s: error setting config object", p->name);
         goto done;
     }
-    flux_log_set_appname (p->h, p->name);
-
-    if (modservice_register (p->h, p) < 0) {
+    p->conf = NULL; // flux_set_conf() transfers ownership to p->h_module_end
+    if (modservice_register (p->h_module_end, p) < 0) {
         log_err ("%s: modservice_register", p->name);
         goto done;
     }
@@ -196,15 +213,15 @@ static void *module_thread (void *arg)
      */
     ac = argz_count (p->argz, p->argz_len);
     if (!(av = calloc (1, sizeof (av[0]) * (ac + 1)))) {
-        log_errn (ENOMEM, "calloc");
+        log_err ("calloc");
         goto done;
     }
     argz_extract (p->argz, p->argz_len, av);
-    if (p->main (p->h, ac, av) < 0) {
+    if (p->main (p->h_module_end, ac, av) < 0) {
         mod_main_errno = errno;
         if (mod_main_errno == 0)
             mod_main_errno = ECONNRESET;
-        flux_log (p->h, LOG_CRIT, "fatal error: %s", strerror (errno));
+        flux_log (p->h_module_end, LOG_CRIT, "module exiting abnormally");
     }
 
     /* Before processing unhandled requests, ensure that this module
@@ -213,227 +230,322 @@ static void *module_thread (void *arg)
      * which could cause the broker to block.
      */
     if (module_finalizing (p) < 0)
-        flux_log_error (p->h, "failed to set module state to finalizing");
+        flux_log_error (p->h_module_end,
+                        "failed to set module state to finalizing");
 
     /* If any unhandled requests were received during shutdown,
      * respond to them now with ENOSYS.
      */
-    while ((msg = flux_recv (p->h, FLUX_MATCH_REQUEST, FLUX_O_NONBLOCK))) {
+    while ((msg = flux_recv (p->h_module_end,
+                             FLUX_MATCH_REQUEST,
+                             FLUX_O_NONBLOCK))) {
         const char *topic = "unknown";
         (void)flux_msg_get_topic (msg, &topic);
-        flux_log (p->h, LOG_DEBUG, "responding to post-shutdown %s", topic);
-        if (flux_respond_error (p->h, msg, ENOSYS, NULL) < 0)
-            flux_log_error (p->h, "responding to post-shutdown %s", topic);
+        flux_log (p->h_module_end,
+                  LOG_DEBUG,
+                  "responding to post-shutdown %s",
+                  topic);
+        if (flux_respond_error (p->h_module_end, msg, ENOSYS, NULL) < 0)
+            flux_log_error (p->h_module_end,
+                            "responding to post-shutdown %s",
+                            topic);
         flux_msg_destroy (msg);
     }
-    if (!(msg = flux_keepalive_encode (mod_main_errno, FLUX_MODSTATE_EXITED))) {
-        flux_log_error (p->h, "flux_keepalive_encode");
+    if (!(f = flux_rpc_pack (p->h_module_end,
+                             "module.status",
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_NORESPONSE,
+                             "{s:i s:i}",
+                             "status", FLUX_MODSTATE_EXITED,
+                             "errnum", mod_main_errno))) {
+        flux_log_error (p->h_module_end, "module.status EXITED error");
         goto done;
     }
-    if (flux_send (p->h, msg, 0) < 0)
-        flux_log_error (p->h, "flux_send");
-    flux_msg_destroy (msg);
+    flux_future_destroy (f);
 done:
-    free (uri);
-    free (rankstr);
-    if (av)
-        free (av);
-    flux_close (p->h);
-    p->h = NULL;
+    free (av);
+    flux_close (p->h_module_end);
+    p->h_module_end = NULL;
+    return NULL;
+}
+
+static void module_cb (flux_reactor_t *r,
+                       flux_watcher_t *w,
+                       int revents,
+                       void *arg)
+{
+    module_t *p = arg;
+    p->lastseen = flux_reactor_now (r);
+    if (p->poller_cb)
+        p->poller_cb (p, p->poller_arg);
+}
+
+static char *module_name_from_path (const char *path)
+{
+    char *name;
+    char *cp;
+
+    name = basename_simple (path);
+    // if path ends in .so or .so.VERSION, strip it off
+    if ((cp = strstr (name, ".so")))
+        return strndup (name, cp - name);
+    return strdup (name);
+}
+
+module_t *module_create (flux_t *h,
+                         const char *parent_uuid,
+                         const char *name, // may be NULL
+                         const char *path,
+                         int rank,
+                         json_t *args,
+                         flux_error_t *error)
+{
+    flux_reactor_t *r = flux_get_reactor (h);
+    module_t *p;
+    void *dso;
+    const char **mod_namep;
+    mod_main_f *mod_main;
+
+    dlerror ();
+    if (!(dso = dlopen (path, RTLD_NOW | RTLD_GLOBAL | plugin_deepbind ()))) {
+        errprintf (error, "%s", dlerror ());
+        errno = ENOENT;
+        return NULL;
+    }
+    if (!(mod_main = dlsym (dso, "mod_main"))) {
+        errprintf (error, "module does not define mod_main()");
+        dlclose (dso);
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(p = calloc (1, sizeof (*p))))
+        goto nomem;
+    p->main = mod_main;
+    p->dso = dso;
+    p->rank = rank;
+    p->h = h;
+    if (!(p->conf = flux_conf_copy (flux_get_conf (h))))
+        goto cleanup;
+    if (!(p->parent_uuid_str = strdup (parent_uuid)))
+        goto nomem;
+    strncpy (p->uuid_str, parent_uuid, sizeof (p->uuid_str) - 1);
+    if (args) {
+        size_t index;
+        json_t *entry;
+
+        json_array_foreach (args, index, entry) {
+            const char *s = json_string_value (entry);
+            if (s && (argz_add (&p->argz, &p->argz_len, s) != 0))
+                goto nomem;
+        }
+    }
+    if (!(p->path = strdup (path))
+        || !(p->rmmod_requests = flux_msglist_create ())
+        || !(p->insmod_requests = flux_msglist_create ())
+        || !(p->trace_requests = flux_msglist_create ()))
+        goto nomem;
+    if (name) {
+        if (!(p->name = strdup (name)))
+            goto nomem;
+    }
+    else {
+        if (!(p->name = module_name_from_path (path)))
+            goto nomem;
+    }
+    if (!(p->sub = subhash_create ())) {
+        errprintf (error, "error creating subscription hash");
+        goto cleanup;
+    }
+    /* Handle legacy 'mod_name' symbol - not recommended for new modules
+     * but double check that it's sane if present.
+     */
+    if ((mod_namep = dlsym (p->dso, "mod_name")) && *mod_namep != NULL) {
+        if (!streq (*mod_namep, p->name)) {
+            errprintf (error, "mod_name %s != name %s", *mod_namep, name);
+            errno = EINVAL;
+            goto cleanup;
+        }
+    }
+    uuid_generate (p->uuid);
+    uuid_unparse (p->uuid, p->uuid_str);
+
+    /* Broker end of interthread pair is opened here.
+     */
+    // copying 13 + 37 + 1 = 51 bytes into 128 byte buffer cannot fail
+    (void)snprintf (p->uri, sizeof (p->uri), "interthread://%s", p->uuid_str);
+    if (!(p->h_broker_end = flux_open (p->uri, FLUX_O_NOREQUEUE))
+        || flux_opt_set (p->h_broker_end,
+                         FLUX_OPT_ROUTER_NAME,
+                         parent_uuid,
+                         strlen (parent_uuid) + 1) < 0
+        || flux_set_reactor (p->h_broker_end, r) < 0) {
+        errprintf (error, "could not create %s interthread handle", p->name);
+        goto cleanup;
+    }
+    if (!(p->broker_w = flux_handle_watcher_create (r,
+                                                    p->h_broker_end,
+                                                    FLUX_POLLIN,
+                                                    module_cb,
+                                                    p))) {
+        errprintf (error, "could not create %s flux handle watcher", p->name);
+        goto cleanup;
+    }
+    /* Optimization: create attribute cache to be primed in the module's
+     * flux_t handle.  Priming the cache avoids a synchronous RPC from
+     * flux_attr_get(3) for common attrs like rank, etc.
+     */
+    if (attr_cache_to_json (h, &p->attr_cache) < 0)
+        goto nomem;
+    return p;
+nomem:
+    errprintf (error, "out of memory");
+    errno = ENOMEM;
+cleanup:
+    module_destroy (p);
     return NULL;
 }
 
+const char *module_get_path (module_t *p)
+{
+    return p && p->path ? p->path : "unknown";
+}
+
 const char *module_get_name (module_t *p)
 {
-    assert (p->magic == MODULE_MAGIC);
-    return p->name;
+    return p && p->name ? p->name : "unknown";
 }
 
 const char *module_get_uuid (module_t *p)
 {
-    return p->uuid_str;
+    return p ? p->uuid_str : "unknown";
 }
 
-static int module_get_idle (module_t *p)
+double module_get_lastseen (module_t *p)
 {
-    return heartbeat_get_epoch (p->heartbeat) - p->lastseen;
+    return p ? p->lastseen : 0;
+}
+
+int module_get_status (module_t *p)
+{
+    return p ? p->status : 0;
 }
 
 flux_msg_t *module_recvmsg (module_t *p)
 {
-    flux_msg_t *msg = NULL;
-    int type;
-    struct flux_msg_cred cred;
-
-    assert (p->magic == MODULE_MAGIC);
-
-    if (!(msg = flux_msg_recvzsock (p->sock)))
-        goto error;
-    if (flux_msg_get_type (msg, &type) < 0)
-        goto error;
-    switch (type) {
-        case FLUX_MSGTYPE_RESPONSE:
-            if (flux_msg_pop_route (msg, NULL) < 0)
-                goto error;
-            break;
-        case FLUX_MSGTYPE_REQUEST:
-        case FLUX_MSGTYPE_EVENT:
-            if (flux_msg_push_route (msg, p->uuid_str) < 0)
-                goto error;
-            break;
-        default:
-            break;
-    }
-    /* All shmem:// connections to the broker have FLUX_ROLE_OWNER
-     * and are "authenticated" as the instance owner.
-     * Allow modules so endowed to change the userid/rolemask on messages when
-     * sending on behalf of other users.  This is necessary for connectors
-     * implemented as comms modules.
-     */
-    assert ((p->cred.rolemask & FLUX_ROLE_OWNER));
-    if (flux_msg_get_cred (msg, &cred) < 0)
-        goto error;
-    if (cred.userid == FLUX_USERID_UNKNOWN)
-        cred.userid = p->cred.userid;
-    if (cred.rolemask == FLUX_ROLE_NONE)
-        cred.rolemask = p->cred.rolemask;
-    if (flux_msg_set_cred (msg, cred) < 0)
-        goto error;
+    flux_msg_t *msg;
+    msg = flux_recv (p->h_broker_end, FLUX_MATCH_ANY, FLUX_O_NONBLOCK);
+    trace_module_msg (p->h, "tx", p->name, p->trace_requests, msg);
     return msg;
-error:
-    flux_msg_destroy (msg);
-    return NULL;
 }
 
-int module_sendmsg (module_t *p, const flux_msg_t *msg)
+int module_sendmsg_new (module_t *p, flux_msg_t **msg)
 {
-    flux_msg_t *cpy = NULL;
     int type;
-    int rc = -1;
+    const char *topic;
 
-    if (!msg)
+    if (!msg || !*msg)
         return 0;
-    if (flux_msg_get_type (msg, &type) < 0)
-        goto done;
-    if (p->muted && type != FLUX_MSGTYPE_KEEPALIVE) {
-        /* Muted modules only accept keepalive messages */
-        const char *topic;
-        (void) flux_msg_get_topic (msg, &topic);
-        errno = ENOSYS;
-        goto done;
-    }
-    switch (type) {
-        case FLUX_MSGTYPE_REQUEST: { /* simulate DEALER socket */
-            char uuid[16];
-            snprintf (uuid, sizeof (uuid), "%"PRIu32, p->rank);
-            if (!(cpy = flux_msg_copy (msg, true)))
-                goto done;
-            if (flux_msg_push_route (cpy, uuid) < 0)
-                goto done;
-            if (flux_msg_sendzsock (p->sock, cpy) < 0)
-                goto done;
-            break;
-        }
-        case FLUX_MSGTYPE_RESPONSE: { /* simulate ROUTER socket */
-            if (!(cpy = flux_msg_copy (msg, true)))
-                goto done;
-            if (flux_msg_pop_route (cpy, NULL) < 0)
-                goto done;
-            if (flux_msg_sendzsock (p->sock, cpy) < 0)
-                goto done;
-            break;
+    if (flux_msg_get_type (*msg, &type) < 0
+        || flux_msg_get_topic (*msg, &topic) < 0)
+        return -1;
+    /* Muted modules only accept response to module.status
+     */
+    if (p->muted) {
+        if (type != FLUX_MSGTYPE_RESPONSE
+            || !streq (topic, "module.status")) {
+            errno = ENOSYS;
+            return -1;
         }
-        default:
-            if (flux_msg_sendzsock (p->sock, msg) < 0)
-                goto done;
-            break;
     }
-    rc = 0;
-done:
-    flux_msg_destroy (cpy);
-    return rc;
+    if (p->deferred_messages) {
+        if (flux_msglist_append (p->deferred_messages, *msg) < 0)
+            return -1;
+        flux_msg_decref (*msg);
+        *msg = NULL;
+        return 0;
+    }
+    trace_module_msg (p->h, "rx", p->name, p->trace_requests, *msg);
+    return flux_send_new (p->h_broker_end, msg, 0);
 }
 
-int module_response_sendmsg (modhash_t *mh, const flux_msg_t *msg)
+int module_disconnect_arm (module_t *p,
+                           const flux_msg_t *msg,
+                           disconnect_send_f cb,
+                           void *arg)
 {
-    char *uuid = NULL;
-    int rc = -1;
-    module_t *p;
-
-    if (!msg)
-        return 0;
-    if (flux_msg_get_route_last (msg, &uuid) < 0)
-        goto done;
-    if (!uuid) {
-        errno = EPROTO;
-        goto done;
+    if (!p->disconnect) {
+        if (!(p->disconnect = disconnect_create (cb, arg)))
+            return -1;
     }
-    if (!(p = zhash_lookup (mh->zh_byuuid, uuid))) {
-        errno = ENOSYS;
-        goto done;
-    }
-    rc = module_sendmsg (p, msg);
-done:
-    if (uuid)
-        free (uuid);
-    return rc;
+    if (disconnect_arm (p->disconnect, msg) < 0)
+        return -1;
+    return 0;
 }
 
-static void module_destroy (module_t *p)
+void module_destroy (module_t *p)
 {
     int e;
     void *res;
+    int saved_errno = errno;
 
     if (!p)
         return;
 
-    assert (p->magic == MODULE_MAGIC);
-
     if (p->t) {
         if ((e = pthread_join (p->t, &res)) != 0)
             log_errn_exit (e, "pthread_cancel");
+        if (p->status != FLUX_MODSTATE_EXITED) {
+            /* Calls broker.c module_status_cb() => service_remove_byuuid()
+             * and releases a reference on 'p'.  Without this, disconnect
+             * requests sent when other modules are destroyed can still find
+             * this service name and trigger a use-after-free segfault.
+             * See also: flux-framework/flux-core#4564.
+             */
+            module_set_status (p, FLUX_MODSTATE_EXITED);
+        }
     }
 
+    /* Send disconnect messages to services used by this module.
+     */
+    disconnect_destroy (p->disconnect);
+
     flux_watcher_stop (p->broker_w);
     flux_watcher_destroy (p->broker_w);
-    zsock_destroy (&p->sock);
+    flux_close (p->h_broker_end);
 
 #ifndef __SANITIZE_ADDRESS__
     dlclose (p->dso);
 #endif
-    free (p->digest);
     free (p->argz);
     free (p->name);
-    if (p->rmmod) {
-        flux_msg_t *msg;
-        while ((msg = zlist_pop (p->rmmod)))
-            flux_msg_destroy (msg);
-    }
-    flux_msg_destroy (p->insmod);
-    if (p->subs) {
-        char *s;
-        while ((s = zlist_pop (p->subs)))
-            free (s);
-        zlist_destroy (&p->subs);
-    }
-    zlist_destroy (&p->rmmod);
-    p->magic = ~MODULE_MAGIC;
+    free (p->path);
+    free (p->parent_uuid_str);
+    flux_conf_decref (p->conf);
+    json_decref (p->attr_cache);
+    flux_msglist_destroy (p->rmmod_requests);
+    flux_msglist_destroy (p->insmod_requests);
+    flux_msglist_destroy (p->deferred_messages);
+    flux_msglist_destroy (p->trace_requests);
+    subhash_destroy (p->sub);
     free (p);
+    errno = saved_errno;
 }
 
 /* Send shutdown request, broker to module.
  */
-int module_stop (module_t *p)
+int module_stop (module_t *p, flux_t *h)
 {
-    assert (p->magic == MODULE_MAGIC);
     char *topic = NULL;
     flux_future_t *f = NULL;
     int rc = -1;
 
     if (asprintf (&topic, "%s.shutdown", p->name) < 0)
         goto done;
-    if (!(f = flux_rpc (p->broker_h, topic, NULL,
-                          FLUX_NODEID_ANY, FLUX_RPC_NORESPONSE)))
+    if (!(f = flux_rpc (h,
+                        topic,
+                        NULL,
+                        FLUX_NODEID_ANY,
+                        FLUX_RPC_NORESPONSE)))
         goto done;
     rc = 0;
 done:
@@ -447,19 +559,28 @@ void module_mute (module_t *p)
     p->muted = true;
 }
 
-static void module_cb (flux_reactor_t *r, flux_watcher_t *w,
-                       int revents, void *arg)
+int module_set_defer (module_t *p, bool flag)
 {
-    module_t *p = arg;
-    assert (p->magic == MODULE_MAGIC);
-    p->lastseen = heartbeat_get_epoch (p->heartbeat);
-    if (p->poller_cb)
-        p->poller_cb (p, p->poller_arg);
+    if (flag && !p->deferred_messages) {
+        if (!(p->deferred_messages = flux_msglist_create ()))
+            return -1;
+    }
+    if (!flag && p->deferred_messages) {
+        const flux_msg_t *msg;
+        while ((msg = flux_msglist_pop (p->deferred_messages))) {
+            if (flux_send_new (p->h_broker_end, (flux_msg_t **)&msg, 0) < 0) {
+                flux_msg_decref (msg);
+                return -1;
+            }
+        }
+        flux_msglist_destroy (p->deferred_messages);
+        p->deferred_messages = NULL;
+    }
+    return 0;
 }
 
 int module_start (module_t *p)
 {
-    assert (p->magic == MODULE_MAGIC);
     int errnum;
     int rc = -1;
 
@@ -473,413 +594,135 @@ int module_start (module_t *p)
     return rc;
 }
 
-void module_set_args (module_t *p, int argc, char * const argv[])
+int module_cancel (module_t *p, flux_error_t *error)
 {
-    int e;
-
-    assert (p->magic == MODULE_MAGIC);
-    if (p->argz) {
-        free (p->argz);
-        p->argz_len = 0;
+    if (p->t) {
+        int e;
+        if ((e = pthread_cancel (p->t)) != 0 && e != ESRCH) {
+            errprintf (error, "pthread_cancel: %s", strerror (e));
+            return -1;
+        }
     }
-    if (argv && (e = argz_create (argv, &p->argz, &p->argz_len)) != 0)
-        log_errn_exit (e, "argz_create");
-}
-
-void module_add_arg (module_t *p, const char *arg)
-{
-    int e;
-
-    assert (p->magic == MODULE_MAGIC);
-    if ((e = argz_add (&p->argz, &p->argz_len, arg)) != 0)
-        log_errn_exit (e, "argz_add");
+    return 0;
 }
 
 void module_set_poller_cb (module_t *p, modpoller_cb_f cb, void *arg)
 {
-    assert (p->magic == MODULE_MAGIC);
     p->poller_cb = cb;
     p->poller_arg = arg;
 }
 
 void module_set_status_cb (module_t *p, module_status_cb_f cb, void *arg)
 {
-    assert (p->magic == MODULE_MAGIC);
     p->status_cb = cb;
     p->status_arg = arg;
 }
 
 void module_set_status (module_t *p, int new_status)
 {
-    assert (p->magic == MODULE_MAGIC);
-    assert (p->status != new_status);
-    assert (new_status != FLUX_MODSTATE_INIT);  /* illegal state transition */
-    assert (p->status != FLUX_MODSTATE_EXITED); /* illegal state transition */
+    if (new_status == FLUX_MODSTATE_INIT || p->status == FLUX_MODSTATE_EXITED)
+        return; // illegal state transitions
     int prev_status = p->status;
     p->status = new_status;
     if (p->status_cb)
         p->status_cb (p, prev_status, p->status_arg);
 }
 
-int module_get_status (module_t *p)
-{
-    assert (p->magic == MODULE_MAGIC);
-    return p->status;
-}
-
 void module_set_errnum (module_t *p, int errnum)
 {
-    assert (p->magic == MODULE_MAGIC);
     p->errnum = errnum;
 }
 
 int module_get_errnum (module_t *p)
 {
-    assert (p->magic == MODULE_MAGIC);
     return p->errnum;
 }
 
 int module_push_rmmod (module_t *p, const flux_msg_t *msg)
 {
-    flux_msg_t *cpy = flux_msg_copy (msg, false);
-    if (!cpy)
-        return -1;
-    if (zlist_push (p->rmmod, cpy) < 0) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return 0;
+    return flux_msglist_push (p->rmmod_requests, msg);
 }
 
-flux_msg_t *module_pop_rmmod (module_t *p)
+const flux_msg_t *module_pop_rmmod (module_t *p)
 {
-    assert (p->magic == MODULE_MAGIC);
-    return zlist_pop (p->rmmod);
+    return flux_msglist_pop (p->rmmod_requests);
 }
 
-/* There can be only one.
+/* There can be only one insmod request.
  */
 int module_push_insmod (module_t *p, const flux_msg_t *msg)
 {
-    flux_msg_t *cpy = flux_msg_copy (msg, false);
-    if (!cpy)
+    if (flux_msglist_count (p->insmod_requests) > 0) {
+        errno = EEXIST;
         return -1;
-    if (p->insmod)
-        flux_msg_destroy (p->insmod);
-    p->insmod = cpy;
-    return 0;
-}
-
-flux_msg_t *module_pop_insmod (module_t *p)
-{
-    assert (p->magic == MODULE_MAGIC);
-    flux_msg_t *msg = p->insmod;
-    p->insmod = NULL;
-    return msg;
-}
-
-module_t *module_add (modhash_t *mh, const char *path)
-{
-    module_t *p;
-    void *dso;
-    const char **mod_namep;
-    mod_main_f *mod_main;
-    zfile_t *zf;
-    int rc;
-
-    dlerror ();
-    if (!(dso = dlopen (path, RTLD_NOW | RTLD_LOCAL | FLUX_DEEPBIND))) {
-        log_msg ("%s", dlerror ());
-        errno = ENOENT;
-        return NULL;
-    }
-    mod_main = dlsym (dso, "mod_main");
-    mod_namep = dlsym (dso, "mod_name");
-    if (!mod_main || !mod_namep || !*mod_namep) {
-        dlclose (dso);
-        errno = ENOENT;
-        return NULL;
-    }
-    if (!(p = calloc (1, sizeof (*p)))) {
-        dlclose (dso);
-        errno = ENOMEM;
-        return NULL;
-    }
-    p->magic = MODULE_MAGIC;
-    p->main = mod_main;
-    p->dso = dso;
-    if (!(p->name = strdup (*mod_namep))) {
-        errno = ENOMEM;
-        goto cleanup;
-    }
-    zf = zfile_new (NULL, path);
-    if (!(p->digest = strdup (zfile_digest (zf)))) {
-        errno = ENOMEM;
-        goto cleanup;
-    }
-    p->size = (int)zfile_cursize (zf);
-    zfile_destroy (&zf);
-    uuid_generate (p->uuid);
-    uuid_unparse (p->uuid, p->uuid_str);
-    if (!(p->rmmod = zlist_new ())) {
-        errno = ENOMEM;
-        goto cleanup;
-    }
-    if (!(p->subs = zlist_new ())) {
-        errno = ENOMEM;
-        goto cleanup;
-    }
-
-    p->rank = mh->rank;
-    p->broker_h = mh->broker_h;
-    p->heartbeat = mh->heartbeat;
-
-    /* Broker end of PAIR socket is opened here.
-     */
-    if (!(p->sock = zsock_new_pair (NULL))) {
-        log_err ("zsock_new_pair");
-        goto cleanup;
-    }
-    if (zsock_bind (p->sock, "inproc://%s", module_get_uuid (p)) < 0) {
-        log_err ("zsock_bind inproc://%s", module_get_uuid (p));
-        goto cleanup;
-    }
-    if (!(p->broker_w = flux_zmq_watcher_create (flux_get_reactor (p->broker_h),
-                                                 p->sock, FLUX_POLLIN,
-                                                 module_cb, p))) {
-        log_err ("flux_zmq_watcher_create");
-        goto cleanup;
-    }
-    /* Set creds for connection.
-     * Since this is a point to point connection between broker threads,
-     * credentials are always those of the instance owner.
-     */
-    p->cred.userid = geteuid ();
-    p->cred.rolemask = FLUX_ROLE_OWNER;
-
-    /* Update the modhash.
-     */
-    rc = zhash_insert (mh->zh_byuuid, module_get_uuid (p), p);
-    assert (rc == 0); /* uuids are by definition unique */
-    zhash_freefn (mh->zh_byuuid, module_get_uuid (p),
-                  (zhash_free_fn *)module_destroy);
-    return p;
-
-cleanup:
-    module_destroy (p);
-    return NULL;
-}
-
-void module_remove (modhash_t *mh, module_t *p)
-{
-    assert (p->magic == MODULE_MAGIC);
-    zhash_delete (mh->zh_byuuid, module_get_uuid (p));
-}
-
-modhash_t *modhash_create (void)
-{
-    modhash_t *mh = calloc (1, sizeof (*mh));
-    if (!mh) {
-        errno = ENOMEM;
-        return NULL;
-    }
-    if (!(mh->zh_byuuid = zhash_new ())) {
-        modhash_destroy (mh);
-        errno = ENOMEM;
-        return NULL;
-    }
-    return mh;
-}
-
-void modhash_destroy (modhash_t *mh)
-{
-    const char *uuid;
-    module_t *p;
-    int e;
-
-    if (mh) {
-        if (mh->zh_byuuid) {
-            FOREACH_ZHASH (mh->zh_byuuid, uuid, p) {
-                if (p->t) {
-                    if ((e = pthread_cancel (p->t)) != 0 && e != ESRCH)
-                        log_errn (e, "pthread_cancel");
-                }
-            }
-            zhash_destroy (&mh->zh_byuuid);
-        }
-        free (mh);
     }
+    return flux_msglist_push (p->insmod_requests, msg);
 }
 
-void modhash_set_rank (modhash_t *mh, uint32_t rank)
+const flux_msg_t *module_pop_insmod (module_t *p)
 {
-    mh->rank = rank;
+    return flux_msglist_pop (p->insmod_requests);
 }
 
-void modhash_set_flux (modhash_t *mh, flux_t *h)
+int module_subscribe (module_t *p, const char *topic)
 {
-    mh->broker_h = h;
+    return subhash_subscribe (p->sub, topic);
 }
 
-void modhash_set_heartbeat (modhash_t *mh, heartbeat_t *hb)
+int module_unsubscribe (module_t *p, const char *topic)
 {
-    mh->heartbeat = hb;
+    return subhash_unsubscribe (p->sub, topic);
 }
 
-json_t *module_get_modlist (modhash_t *mh, struct service_switch *sw)
+int module_event_cast (module_t *p, const flux_msg_t *msg)
 {
-    json_t *mods = NULL;
-    zlist_t *uuids = NULL;
-    char *uuid;
-    module_t *p;
-
-    if (!(mods = json_array()))
-        goto nomem;
-    if (!(uuids = zhash_keys (mh->zh_byuuid)))
-        goto nomem;
-    uuid = zlist_first (uuids);
-    while (uuid) {
-        if ((p = zhash_lookup (mh->zh_byuuid, uuid))) {
-            json_t *svcs;
-            json_t *entry;
-
-            if (!(svcs  = service_list_byuuid (sw, uuid)))
-                goto nomem;
-            if (!(entry = json_pack ("{s:s s:i s:s s:i s:i s:o}",
-                                     "name", module_get_name (p),
-                                     "size", p->size,
-                                     "digest", p->digest,
-                                      "idle", module_get_idle (p),
-                                      "status", p->status,
-                                      "services", svcs))) {
-                json_decref (svcs);
-                goto nomem;
-            }
-            if (json_array_append_new (mods, entry) < 0) {
-                json_decref (entry);
-                goto nomem;
-            }
-        }
-        uuid = zlist_next (uuids);
-    }
-    zlist_destroy (&uuids);
-    return mods;
-nomem:
-    zlist_destroy (&uuids);
-    json_decref (mods);
-    errno = ENOMEM;
-    return NULL;
-}
-
-module_t *module_lookup (modhash_t *mh, const char *uuid)
-{
-    return zhash_lookup (mh->zh_byuuid, uuid);
-}
-
-module_t *module_lookup_byname (modhash_t *mh, const char *name)
-{
-    zlist_t *uuids;
-    char *uuid;
-    module_t *result = NULL;
+    const char *topic;
 
-    if (!(uuids = zhash_keys (mh->zh_byuuid))) {
-        errno = ENOMEM;
-        return NULL;
-    }
-    uuid = zlist_first (uuids);
-    while (uuid) {
-        module_t *p = zhash_lookup (mh->zh_byuuid, uuid);
-        assert (p != NULL);
-        if (!strcmp (module_get_name (p), name)) {
-            result = p;
-            break;
+    if (flux_msg_get_topic (msg, &topic) < 0)
+        return -1;
+    if (subhash_topic_match (p->sub, topic)) {
+        flux_msg_t *cpy;
+        if (!(cpy = flux_msg_copy (msg, true))
+            || module_sendmsg_new (p, &cpy) < 0) {
+            flux_msg_decref (cpy);
+            return -1;
         }
-        uuid = zlist_next (uuids);
-        p = NULL;
     }
-    zlist_destroy (&uuids);
-    return result;
+    return 0;
 }
 
-int module_subscribe (modhash_t *mh, const char *uuid, const char *topic)
+ssize_t module_get_send_queue_count (module_t *p)
 {
-    module_t *p = zhash_lookup (mh->zh_byuuid, uuid);
-    char *cpy = NULL;
-    int rc = -1;
-
-    if (!p) {
-        errno = ENOENT;
-        goto done;
-    }
-    if (!(cpy = strdup (topic))) {
-        errno = ENOMEM;
-        goto done;
-    }
-    if (zlist_push (p->subs, cpy) < 0) {
-        free (cpy);
-        errno = ENOMEM;
-        goto done;
-    }
-    rc = 0;
-done:
-    return rc;
+    size_t count;
+    if (flux_opt_get (p->h_broker_end,
+                      FLUX_OPT_SEND_QUEUE_COUNT,
+                      &count,
+                      sizeof (count)) < 0)
+        return -1;
+    return count;
 }
 
-int module_unsubscribe (modhash_t *mh, const char *uuid, const char *topic)
+ssize_t module_get_recv_queue_count (module_t *p)
 {
-    module_t *p = zhash_lookup (mh->zh_byuuid, uuid);
-    char *s;
-    int rc = -1;
-
-    if (!p) {
-        errno = ENOENT;
-        goto done;
-    }
-    s = zlist_first (p->subs);
-    while (s) {
-        if (!strcmp (topic, s)) {
-            zlist_remove (p->subs, s);
-            free (s);
-            break;
-        }
-        s = zlist_next (p->subs);
-    }
-    rc = 0;
-done:
-    return rc;
+    size_t count;
+    if (flux_opt_get (p->h_broker_end,
+                      FLUX_OPT_RECV_QUEUE_COUNT,
+                      &count,
+                      sizeof (count)) < 0)
+        return -1;
+    return count;
 }
 
-static bool match_sub (module_t *p, const char *topic)
+int module_trace (module_t *p, const flux_msg_t *msg)
 {
-    char *s = zlist_first (p->subs);
-
-    while (s) {
-        if (!strncmp (topic, s, strlen (s)))
-            return true;
-        s = zlist_next (p->subs);
-    }
-    return false;
+    if (flux_msglist_append (p->trace_requests, msg) < 0)
+        return -1;
+    return 0;
 }
 
-int module_event_mcast (modhash_t *mh, const flux_msg_t *msg)
+void module_trace_disconnect (module_t *p, const flux_msg_t *msg)
 {
-    const char *topic;
-    module_t *p;
-    int rc = -1;
-
-    if (flux_msg_get_topic (msg, &topic) < 0)
-        goto done;
-    p = zhash_first (mh->zh_byuuid);
-    while (p) {
-        if (match_sub (p, topic)) {
-            if (module_sendmsg (p, msg) < 0)
-                goto done;
-        }
-        p = zhash_next (mh->zh_byuuid);
-    }
-    rc = 0;
-done:
-    return rc;
+    (void)flux_msglist_disconnect (p->trace_requests, msg);
 }
 
 /*
diff --git a/src/broker/module.h b/src/broker/module.h
index 28c4f5c1c16c..6943ddab6fb8 100644
--- a/src/broker/module.h
+++ b/src/broker/module.h
@@ -12,41 +12,29 @@
 #define _BROKER_MODULE_H
 
 #include <jansson.h>
+#include <flux/core.h>
 
-#include "heartbeat.h"
-#include "service.h"
+#include "src/common/librouter/disconnect.h"
 
-typedef struct module_struct module_t;
-typedef struct modhash_struct modhash_t;
+typedef struct broker_module module_t;
 typedef void (*modpoller_cb_f)(module_t *p, void *arg);
 typedef void (*module_status_cb_f)(module_t *p, int prev_status, void *arg);
 
-/* Hash-o-modules, keyed by uuid
- */
-modhash_t *modhash_create (void);
-void modhash_destroy (modhash_t *mh);
-
-void modhash_set_rank (modhash_t *mh, uint32_t rank);
-void modhash_set_flux (modhash_t *mh, flux_t *h);
-void modhash_set_heartbeat (modhash_t *mh, heartbeat_t *hb);
-
-/* Prepare module at 'path' for starting.
- */
-module_t *module_add (modhash_t *mh, const char *path);
-void module_remove (modhash_t *mh, module_t *p);
-
-/* Set arguments to module main().  Call before module_start().
- */
-void module_set_args (module_t *p, int argc, char * const argv[]);
-void module_add_arg (module_t *p, const char *arg);
+module_t *module_create (flux_t *h,
+                         const char *parent_uuid,
+                         const char *name, // may be NULL
+                         const char *path,
+                         int rank,
+                         json_t *args,
+                         flux_error_t *error);
+void module_destroy (module_t *p);
 
-/* Get module name.
+/* accessors
  */
 const char *module_get_name (module_t *p);
-
-/* Get module uuid.
- */
+const char *module_get_path (module_t *p);
 const char *module_get_uuid (module_t *p);
+double module_get_lastseen (module_t *p);
 
 /* The poller callback is called when module socket is ready for
  * reading with module_recvmsg().
@@ -56,21 +44,21 @@ void module_set_poller_cb (module_t *p, modpoller_cb_f cb, void *arg);
 /* Send/recv a message for to/from a specific module.
  */
 flux_msg_t *module_recvmsg (module_t *p);
-int module_sendmsg (module_t *p, const flux_msg_t *msg);
+int module_sendmsg_new (module_t *p, flux_msg_t **msg);
 
-/* Send an event message to all modules that have matching subscription.
+/* Pass module's requests through this function to enable disconnect
+ * messages to be sent when the module is unloaded.  The callback will
+ * be used to send those messages.
  */
-int module_event_mcast (modhash_t *mh, const flux_msg_t *msg);
-
-/* Subscribe/unsubscribe module by uuid
- */
-int module_subscribe (modhash_t *mh, const char *uuid, const char *topic);
-int module_unsubscribe (modhash_t *mh, const char *uuid, const char *topic);
+int module_disconnect_arm (module_t *p,
+                           const flux_msg_t *msg,
+                           disconnect_send_f cb,
+                           void *arg);
 
 int module_push_rmmod (module_t *p, const flux_msg_t *msg);
-flux_msg_t *module_pop_rmmod (module_t *p);
+const flux_msg_t *module_pop_rmmod (module_t *p);
 int module_push_insmod (module_t *p, const flux_msg_t *msg);
-flux_msg_t *module_pop_insmod (module_t *p);
+const flux_msg_t *module_pop_insmod (module_t *p);
 
 /* Get/set module status.
  */
@@ -81,35 +69,38 @@ void module_set_status_cb (module_t *p, module_status_cb_f cb, void *arg);
 int module_get_errnum (module_t *p);
 void module_set_errnum (module_t *p, int errnum);
 
-/* Send a response message to the module whose uuid matches the
- * next hop in the routing stack.
- */
-int module_response_sendmsg (modhash_t *mh, const flux_msg_t *msg);
-
-/* Find a module matching 'uuid'.
- */
-module_t *module_lookup (modhash_t *mh, const char *uuid);
-
-/* Find a module matching 'name'.
- * N.B. this is a slow linear search - keep out of crit paths
- */
-module_t *module_lookup_byname (modhash_t *mh, const char *name);
-
 /* Start module thread.
  */
 int module_start (module_t *p);
 
 /* Stop module thread by sending a shutdown request.
  */
-int module_stop (module_t *p);
+int module_stop (module_t *p, flux_t *h);
+
+/* Defer all messages that would be sent to module if flag=true.
+ * Stop deferring them and send backlog if flag=false.
+ */
+int module_set_defer (module_t *p, bool flag);
 
 /*  Mute module. Do not send any more messages.
  */
 void module_mute (module_t *p);
 
-/* Prepare RFC 5 'mods' array for lsmod response.
+/* Call pthread_cancel() on module.
+ */
+int module_cancel (module_t *p, flux_error_t *error);
+
+/* Manage module subscriptions.
  */
-json_t *module_get_modlist (modhash_t *mh, struct service_switch *sw);
+int module_subscribe (module_t *p, const char *topic);
+int module_unsubscribe (module_t *p, const char *topic);
+int module_event_cast (module_t *p, const flux_msg_t *msg);
+
+ssize_t module_get_send_queue_count (module_t *p);
+ssize_t module_get_recv_queue_count (module_t *p);
+
+int module_trace (module_t *p, const flux_msg_t *msg);
+void module_trace_disconnect (module_t *p, const flux_msg_t *msg);
 
 #endif /* !_BROKER_MODULE_H */
 
diff --git a/src/broker/overlay.c b/src/broker/overlay.c
index ab3467e535e5..cbdb47e103b4 100644
--- a/src/broker/overlay.c
+++ b/src/broker/overlay.c
@@ -12,656 +12,2537 @@
 #include "config.h"
 #endif
 #include <stdarg.h>
-#include <czmq.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 #include <zmq.h>
+#include <unistd.h>
 #include <flux/core.h>
 #include <inttypes.h>
 #include <jansson.h>
-
-#include "src/common/libutil/xzmalloc.h"
+#include <uuid.h>
+
+#include "src/common/libzmqutil/msg_zsock.h"
+#include "src/common/libzmqutil/sockopt.h"
+#include "src/common/libzmqutil/reactor.h"
+#include "src/common/libzmqutil/zap.h"
+#include "src/common/libzmqutil/cert.h"
+#include "src/common/libzmqutil/monitor.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
-#include "src/common/libutil/iterators.h"
-#include "src/common/libutil/kary.h"
 #include "src/common/libutil/cleanup.h"
-#include "src/common/libutil/zsecurity.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/monotime.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/librouter/rpc_track.h"
+#include "ccan/str/str.h"
 
-#include "heartbeat.h"
 #include "overlay.h"
 #include "attr.h"
+#include "trace.h"
+
+/* How long to wait (seconds) for a peer broker's TCP ACK before disconnecting.
+ * This can be configured via TOML and on the broker command line.
+ */
+static const double default_tcp_user_timeout = 20.;
+#ifdef ZMQ_TCP_MAXRT
+static bool have_tcp_maxrt = true;
+#else
+static bool have_tcp_maxrt = false;
+#endif
+
+/* How long to wait (seconds) for a connect attempt to time out before
+ * reconnecting.
+ */
+static const double default_connect_timeout = 30.;
+#ifdef ZMQ_CONNECT_TIMEOUT
+static bool have_connect_timeout = true;
+#else
+static bool have_connect_timeout = false;
+#endif
+
+#ifndef UUID_STR_LEN
+#define UUID_STR_LEN 37     // defined in later libuuid headers
+#endif
+
+#define FLUX_ZAP_DOMAIN "flux"
+
+/* Numerical values for "subtree health" so we can send them in control
+ * messages.  Textual values below will be used for communication with front
+ * end diagnostic tool.
+ */
+enum subtree_status {
+    SUBTREE_STATUS_UNKNOWN = 0,
+    SUBTREE_STATUS_FULL = 1,
+    SUBTREE_STATUS_PARTIAL = 2,
+    SUBTREE_STATUS_DEGRADED = 3,
+    SUBTREE_STATUS_LOST = 4,
+    SUBTREE_STATUS_OFFLINE = 5,
+    SUBTREE_STATUS_MAXIMUM = 5,
+};
+
+/* Names array is indexed with subtree_status enum.
+ * Convert with subtree_status_str()
+ */
+static const char *subtree_status_names[] = {
+    "unknown",
+    "full",
+    "partial",
+    "degraded",
+    "lost",
+    "offline",
+    NULL,
+};
 
-struct endpoint {
-    zsock_t *zs;
+struct child {
+    double lastseen;
+    uint32_t rank;
+    char uuid[UUID_STR_LEN];
+    enum subtree_status status;
+    struct timespec status_timestamp;
+    bool torpid;
+    struct rpc_track *tracker;
+    flux_error_t error;
+};
+
+struct parent {
+    void *zsock;            // NULL on rank 0
     char *uri;
     flux_watcher_t *w;
+    int lastsent;
+    char *pubkey;
+    uint32_t rank;
+    char uuid[UUID_STR_LEN];
+    bool hello_error;
+    bool hello_responded;
+    bool offline;           // set upon receipt of CONTROL_DISCONNECT
+    bool goodbye_sent;
+    flux_future_t *f_goodbye;
+    struct rpc_track *tracker;
+    struct zmqutil_monitor *monitor;
+};
+
+/* Wake up periodically (between 'sync_min' and 'sync_max' seconds) and:
+ * 1) send control to parent if nothing was sent in 'torpid_min' seconds
+ * 2) find children that have not been heard from in 'torpid_max' seconds
+ */
+static const double sync_min = 1.0;
+static const double sync_max = 5.0;
+
+static const double default_torpid_min = 5.0;
+static const double default_torpid_max = 30.0;
+
+static const char *default_interface_hint = "default-route";
+
+struct overlay_monitor {
+    overlay_monitor_f cb;
+    void *arg;
 };
 
-struct overlay_struct {
-    zsecurity_t *sec;             /* security context (MT-safe) */
-    bool sec_initialized;
+struct overlay {
+    void *zctx;
+    bool zctx_external;
+    struct cert *cert;
+    struct zmqutil_zap *zap;
+    int enable_ipv6;
+
     flux_t *h;
-    zhash_t *children;          /* child_t - by uuid */
-    flux_msg_handler_t *heartbeat;
-    int epoch;
+    char *hostname;
+    attr_t *attrs;
+    flux_reactor_t *reactor;
+    flux_msg_handler_t **handlers;
+    flux_future_t *f_sync;
 
+    struct topology *topo;
     uint32_t size;
     uint32_t rank;
-    int tbon_k;
-    int tbon_level;
-    int tbon_maxlevel;
-    int tbon_descendants;
-
-    struct endpoint *parent;    /* DEALER - requests to parent */
-    overlay_cb_f parent_cb;
-    void *parent_arg;
-    int parent_lastsent;
-
-    struct endpoint *child;     /* ROUTER - requests from children */
-    overlay_cb_f child_cb;
-    void *child_arg;
-
-    zsock_t *child_monitor_sock;
-    flux_watcher_t *child_monitor_w;
-    int child_peer_count;
-    overlay_monitor_cb_f child_monitor_cb;
-    void *child_monitor_arg;
-
-    overlay_init_cb_f init_cb;
-    void *init_arg;
-
-    int idle_warning;
+    char uuid[UUID_STR_LEN];
+    int version;
+    int zmqdebug;
+    int zmq_io_threads;
+    double torpid_min;
+    double torpid_max;
+    double tcp_user_timeout;
+    double connect_timeout;
+    int child_rcvhwm;
+
+    struct parent parent;
+
+    bool shutdown_in_progress;  // no new downstream connections permitted
+    void *bind_zsock;           // NULL if no downstream peers
+    char *bind_uri;
+    flux_watcher_t *bind_w;
+    struct child *children;
+    int child_count;
+    zhashx_t *child_hash;
+    enum subtree_status status;
+    struct timespec status_timestamp;
+    struct zmqutil_monitor *bind_monitor;
+
+    zlist_t *monitor_callbacks;
+
+    overlay_recv_f recv_cb;
+    void *recv_arg;
+
+    struct flux_msglist *health_requests;
+    struct flux_msglist *trace_requests;
 };
 
-typedef struct {
-    int lastseen;
-} child_t;
+static void overlay_mcast_child (struct overlay *ov, flux_msg_t *msg);
+static int overlay_sendmsg_child (struct overlay *ov, const flux_msg_t *msg);
+static int overlay_sendmsg_parent (struct overlay *ov, const flux_msg_t *msg);
+static void hello_response_handler (struct overlay *ov, const flux_msg_t *msg);
+static void hello_request_handler (struct overlay *ov, const flux_msg_t *msg);
+static int overlay_control_parent (struct overlay *ov,
+                                   enum control_type type,
+                                   int status);
+static void overlay_health_respond_all (struct overlay *ov);
+static struct child *child_lookup_byrank (struct overlay *ov, uint32_t rank);
+
+/* Convenience iterator for ov->children
+ */
+#define foreach_overlay_child(ov, child) \
+    for ((child) = &(ov)->children[0]; \
+            (child) - &(ov)->children[0] < (ov)->child_count; \
+            (child)++)
 
-static void heartbeat_handler (flux_t *h, flux_msg_handler_t *mh,
-                               const flux_msg_t *msg, void *arg);
+static const char *subtree_status_str (enum subtree_status status)
+{
+    if (status > SUBTREE_STATUS_MAXIMUM)
+        return "unknown";
+    return subtree_status_names[status];
+}
 
-static void endpoint_destroy (struct endpoint *ep)
+static bool subtree_is_online (enum subtree_status status)
 {
-    if (ep) {
-        free (ep->uri);
-        flux_watcher_destroy (ep->w);
-        zsock_destroy (&ep->zs);
-        free (ep);
+    switch (status) {
+        case SUBTREE_STATUS_FULL:
+        case SUBTREE_STATUS_PARTIAL:
+        case SUBTREE_STATUS_DEGRADED:
+            return true;
+        default:
+            return false;
     }
 }
 
-static struct endpoint *endpoint_vcreate (const char *fmt, va_list ap)
+/* Call this function after a child->status changes.
+ * It calculates a new subtree status based on the state of children,
+ * then if the status has changed, the parent is informed with a
+ * control message and any waiting health requests are processed.
+ */
+static void subtree_status_update (struct overlay *ov)
 {
-    struct endpoint *ep = calloc (1, sizeof (*ep));
-    if (vasprintf (&ep->uri, fmt, ap) < 0) {
-        free (ep);
-        errno = ENOMEM;
-        return NULL;
+    enum subtree_status status = SUBTREE_STATUS_FULL;
+    struct child *child;
+
+    foreach_overlay_child (ov, child) {
+        switch (child->status) {
+            case SUBTREE_STATUS_FULL:
+                break;
+            case SUBTREE_STATUS_PARTIAL:
+            case SUBTREE_STATUS_OFFLINE:
+                if (status == SUBTREE_STATUS_FULL)
+                    status = SUBTREE_STATUS_PARTIAL;
+                break;
+            case SUBTREE_STATUS_DEGRADED:
+            case SUBTREE_STATUS_LOST:
+                if (status != SUBTREE_STATUS_DEGRADED)
+                    status = SUBTREE_STATUS_DEGRADED;
+                break;
+            case SUBTREE_STATUS_UNKNOWN:
+                break;
+        }
+    }
+    if (ov->status != status) {
+        ov->status = status;
+        monotime (&ov->status_timestamp);
+        overlay_control_parent (ov, CONTROL_STATUS, ov->status);
+        overlay_health_respond_all (ov);
     }
-    return ep;
 }
 
-void overlay_destroy (overlay_t *ov)
+static void overlay_monitor_notify (struct overlay *ov, uint32_t rank)
 {
-    if (ov) {
-        if (ov->sec)
-            zsecurity_destroy (ov->sec);
-        if (ov->heartbeat)
-            flux_msg_handler_destroy (ov->heartbeat);
-        if (ov->h)
-            (void)flux_event_unsubscribe (ov->h, "hb");
-
-        flux_watcher_destroy (ov->child_monitor_w);
-        zsock_destroy (&ov->child_monitor_sock);
-
-        endpoint_destroy (ov->parent);
-        endpoint_destroy (ov->child);
-        zhash_destroy (&ov->children);
-        free (ov);
+    struct overlay_monitor *mon;
+
+    mon = zlist_first (ov->monitor_callbacks);
+    while (mon) {
+        mon->cb (ov, rank, mon->arg);
+        mon = zlist_next (ov->monitor_callbacks);
     }
 }
 
-overlay_t *overlay_create (void)
+int overlay_set_topology (struct overlay *ov, struct topology *topo)
 {
-    overlay_t *ov = calloc (1, sizeof (*ov) );
+    int *child_ranks = NULL;
+    ssize_t child_count;
 
-    if (!ov) {
-        errno = ENOMEM;
-        return NULL;
-    }
+    ov->topo = topology_incref (topo);
+    /* Determine which ranks, if any are direct children of this one.
+     */
+    if ((child_count = topology_get_child_ranks (topo, NULL, 0)) < 0
+        || !(child_ranks = calloc (child_count, sizeof (child_ranks[0])))
+        || topology_get_child_ranks (topo, child_ranks, child_count) < 0)
+        goto error;
 
-    ov->rank = FLUX_NODEID_ANY;
-    ov->parent_lastsent = -1;
+    ov->size = topology_get_size (topo);
+    ov->rank = topology_get_rank (topo);
+    ov->child_count = child_count;
+    if (ov->child_count > 0) {
+        int i;
 
-    if (!(ov->children = zhash_new ())) {
-        overlay_destroy (ov);
-        errno = ENOMEM;
-        return NULL;
+        if (!(ov->children = calloc (ov->child_count, sizeof (struct child))))
+            return -1;
+        if (!(ov->child_hash = zhashx_new ()))
+            return -1;
+        zhashx_set_key_duplicator (ov->child_hash, NULL);
+        zhashx_set_key_destructor (ov->child_hash, NULL);
+        for (i = 0; i < ov->child_count; i++) {
+            struct child *child = &ov->children[i];
+            child->rank = child_ranks[i];
+            child->status = SUBTREE_STATUS_OFFLINE;
+            monotime (&child->status_timestamp);
+            child->tracker = rpc_track_create (MSG_HASH_TYPE_UUID_MATCHTAG);
+            if (!child->tracker)
+                return -1;
+            if (topology_rank_aux_set (topo,
+                                       child->rank,
+                                       "child",
+                                       child,
+                                       NULL) < 0)
+                return -1;
+        }
+        ov->status = SUBTREE_STATUS_PARTIAL;
     }
-
-    return ov;
+    else
+        ov->status = SUBTREE_STATUS_FULL;
+    monotime (&ov->status_timestamp);
+    if (ov->rank > 0) {
+        ov->parent.rank = topology_get_parent (topo);
+        ov->parent.tracker = rpc_track_create (MSG_HASH_TYPE_UUID_MATCHTAG);
+    }
+    free (child_ranks);
+    return 0;
+error:
+    free (child_ranks);
+    return -1;
 }
 
-void overlay_set_init_callback (overlay_t *ov, overlay_init_cb_f cb, void *arg)
+uint32_t overlay_get_rank (struct overlay *ov)
 {
-    ov->init_cb = cb;
-    ov->init_arg = arg;
+    return ov->rank;
 }
 
-int overlay_init (overlay_t *overlay,
-                  uint32_t size, uint32_t rank, int tbon_k)
+void overlay_set_rank (struct overlay *ov, uint32_t rank)
 {
-    overlay->size = size;
-    overlay->rank = rank;
-    overlay->tbon_k = tbon_k;
-    overlay->tbon_level = kary_levelof (tbon_k, rank);
-    overlay->tbon_maxlevel = kary_levelof (tbon_k, size - 1);
-    overlay->tbon_descendants = kary_sum_descendants (tbon_k, size, rank);
-    if (overlay->init_cb)
-        return (*overlay->init_cb) (overlay, overlay->init_arg);
-    return 0;
+    ov->rank = rank;
 }
 
-uint32_t overlay_get_rank (overlay_t *ov)
+uint32_t overlay_get_size (struct overlay *ov)
 {
-    return ov->rank;
+    return ov->size;
 }
 
-uint32_t overlay_get_size (overlay_t *ov)
+const char *overlay_get_uuid (struct overlay *ov)
 {
-    return ov->size;
+    return ov->uuid;
 }
 
-int overlay_get_child_peer_count (overlay_t *ov)
+bool overlay_parent_error (struct overlay *ov)
 {
-    return ov->child_peer_count;
+    return ((ov->parent.hello_responded && ov->parent.hello_error)
+            || ov->parent.offline);
 }
 
-/* Cleanup not done in this function, responsibiility of caller to
- * call overlay_destroy() eventually */
-int overlay_set_flux (overlay_t *ov, flux_t *h)
+void overlay_set_version (struct overlay *ov, int version)
 {
-    struct flux_match match = FLUX_MATCH_EVENT;
-
-    ov->h = h;
-
-    match.topic_glob = "hb";
-    if (!(ov->heartbeat = flux_msg_handler_create (ov->h, match,
-                                                   heartbeat_handler, ov))) {
-        log_err ("flux_msg_handler_create");
-        return -1;
-    }
-    flux_msg_handler_start (ov->heartbeat);
-    if (flux_event_subscribe (ov->h, "hb") < 0) {
-        log_err ("flux_event_subscribe");
-        return -1;
-    }
-    return 0;
+    ov->version = version;
 }
 
-int overlay_setup_sec (overlay_t *ov, int sec_typemask, const char *keydir)
+int overlay_get_child_peer_count (struct overlay *ov)
 {
-    if (!keydir) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (!(ov->sec = zsecurity_create (sec_typemask, keydir))) {
-        log_err ("zsecurity_create");
-        return -1;
+    struct child *child;
+    int count = 0;
+
+    foreach_overlay_child (ov, child) {
+        if (subtree_is_online (child->status))
+            count++;
     }
-    return 0;
+    return count;
 }
 
-void overlay_set_idle_warning (overlay_t *ov, int heartbeats)
+void overlay_set_ipv6 (struct overlay *ov, int enable)
 {
-    ov->idle_warning = heartbeats;
+    ov->enable_ipv6 = enable;
 }
 
-char *overlay_lspeer_encode (overlay_t *ov)
+bool overlay_peer_is_torpid (struct overlay *ov, uint32_t rank)
 {
-    json_t *o = NULL;
-    json_t *child_o;
-    const char *uuid;
-    child_t *child;
-    char *json_str;
+    struct child *child;
 
-    if (!(o = json_object ()))
-        goto nomem;
-    FOREACH_ZHASH (ov->children, uuid, child) {
-        if (!(child_o = json_pack ("{s:i}", "idle",
-                                   ov->epoch - child->lastseen)))
-            goto nomem;
-        if (json_object_set_new (o, uuid, child_o) < 0) {
-            json_decref (child_o);
-            goto nomem;
-        }
-    }
-    if (!(json_str = json_dumps (o, 0)))
-        goto nomem;
-    json_decref (o);
-    return json_str;
-nomem:
-    json_decref (o);
-    errno = ENOMEM;
-    return NULL;
+    if (!(child = child_lookup_byrank (ov, rank)))
+        return false;
+    return child->torpid;
 }
 
-void overlay_log_idle_children (overlay_t *ov)
+static void log_torpid_child (flux_t *h,
+                               uint32_t rank,
+                               bool torpid,
+                               double duration)
 {
-    const char *uuid;
-    child_t *child;
-    int idle;
-
-    if (ov->idle_warning > 0) {
-        FOREACH_ZHASH (ov->children, uuid, child) {
-            idle = ov->epoch - child->lastseen;
-            if (idle >= ov->idle_warning)
-                flux_log (ov->h, LOG_CRIT, "child %s idle for %d heartbeats",
-                          uuid, idle);
-        }
+    if (torpid) {
+        char fsd[64] = "unknown duration";
+        (void)fsd_format_duration (fsd, sizeof (fsd), duration);
+        flux_log (h,
+                  LOG_ERR,
+                  "broker on %s (rank %lu) has been unresponsive for %s",
+                  flux_get_hostbyrank (h, rank),
+                  (unsigned long)rank,
+                  fsd);
     }
-}
-
-void overlay_checkin_child (overlay_t *ov, const char *uuid)
-{
-    child_t *child  = zhash_lookup (ov->children, uuid);
-    if (!child) {
-        child = xzmalloc (sizeof (*child));
-        zhash_update (ov->children, uuid, child);
-        zhash_freefn (ov->children, uuid, (zhash_free_fn *)free);
+    else {
+        flux_log (h,
+                  LOG_ERR,
+                  "broker on %s (rank %lu) is responsive now",
+                  flux_get_hostbyrank (h, rank),
+                  (unsigned long)rank);
     }
-    child->lastseen = ov->epoch;
 }
 
-int overlay_set_parent (overlay_t *ov, const char *fmt, ...)
+/* Find children that have not been heard from in a while.
+ * If torpid_max is set to zero it means torpid node flagging is disabled.
+ * This value may be set on the fly during runtime, so ensure that any
+ * torpid nodes are immediately transitioned to non-torpid if that occurs.
+ */
+static void update_torpid_children (struct overlay *ov)
 {
-    int rc = -1;
-    if (ov->parent)
-        endpoint_destroy (ov->parent);
-    va_list ap;
-    va_start (ap, fmt);
-    if ((ov->parent = endpoint_vcreate (fmt, ap)))
-        rc = 0;
-    va_end (ap);
-    return rc;
+    struct child *child;
+    double now = flux_reactor_now (ov->reactor);
+
+    foreach_overlay_child (ov, child) {
+        if (subtree_is_online (child->status) && child->lastseen > 0) {
+            double duration = now - child->lastseen;
+
+            if (duration >= ov->torpid_max && ov->torpid_max > 0) {
+                if (!child->torpid) {
+                    log_torpid_child (ov->h, child->rank, true, duration);
+                    child->torpid = true;
+                    overlay_monitor_notify (ov, child->rank);
+                }
+            }
+            else { // duration < torpid_max OR torpid_max == 0
+                if (child->torpid) {
+                    log_torpid_child (ov->h, child->rank, false, 0.);
+                    child->torpid = false;
+                    overlay_monitor_notify (ov, child->rank);
+                }
+            }
+        }
+    }
 }
 
-const char *overlay_get_parent (overlay_t *ov)
+/* N.B. overlay_child_status_update() ensures child_lookup_online() only
+ * succeeds for online peers.
+ */
+static struct child *child_lookup_online (struct overlay *ov, const char *id)
 {
-    if (!ov->parent)
-        return NULL;
-    return ov->parent->uri;
+    return ov->child_hash ?  zhashx_lookup (ov->child_hash, id) : NULL;
 }
 
-int overlay_sendmsg_parent (overlay_t *ov, const flux_msg_t *msg)
+/* Lookup (direct) child peer by rank.
+ * Returns NULL on lookup failure.
+ */
+static struct child *child_lookup_byrank (struct overlay *ov, uint32_t rank)
 {
-    int rc = -1;
-
-    if (!ov->parent || !ov->parent->zs) {
-        errno = EHOSTUNREACH;
-        goto done;
-    }
-    rc = flux_msg_sendzsock (ov->parent->zs, msg);
-    if (rc == 0)
-        ov->parent_lastsent = ov->epoch;
-done:
-    return rc;
+    return topology_rank_aux_get (ov->topo, rank, "child");
 }
 
-static int overlay_keepalive_parent (overlay_t *ov)
+/* Look up child that provides route to 'rank' (NULL if none).
+ */
+static struct child *child_lookup_route (struct overlay *ov, uint32_t rank)
 {
-    int idle = ov->epoch - ov->parent_lastsent;
-    flux_msg_t *msg = NULL;
-    int rc = -1;
+    int child_rank;
 
-    if (!ov->parent || !ov->parent->zs || idle <= 1)
-        return 0;
-    if (!(msg = flux_keepalive_encode (0, 0)))
-        goto done;
-    if (flux_msg_enable_route (msg) < 0)
-        goto done;
-    rc = flux_msg_sendzsock (ov->parent->zs, msg);
-done:
-    flux_msg_destroy (msg);
-    return rc;
+    child_rank = topology_get_child_route (ov->topo, rank);
+    if (child_rank < 0)
+        return NULL;
+    return child_lookup_byrank (ov, child_rank);
 }
 
-static void heartbeat_handler (flux_t *h, flux_msg_handler_t *mh,
-                               const flux_msg_t *msg, void *arg)
+bool overlay_uuid_is_child (struct overlay *ov, const char *uuid)
 {
-    overlay_t *ov = arg;
-
-    if (flux_heartbeat_decode (msg, &ov->epoch) < 0)
-        return;
-    overlay_keepalive_parent (ov);
-    overlay_log_idle_children (ov);
+    if (child_lookup_online (ov, uuid) != NULL)
+        return true;
+    return false;
 }
 
-void overlay_set_parent_cb (overlay_t *ov, overlay_cb_f cb, void *arg)
+bool overlay_uuid_is_parent (struct overlay *ov, const char *uuid)
 {
-    ov->parent_cb = cb;
-    ov->parent_arg = arg;
+    if (ov->rank > 0 && streq (uuid, ov->parent.uuid))
+        return true;
+    return false;
 }
 
-int overlay_set_child (overlay_t *ov, const char *fmt, ...)
+int overlay_set_parent_pubkey (struct overlay *ov, const char *pubkey)
 {
-    int rc = -1;
-    if (ov->child)
-        endpoint_destroy (ov->child);
-    va_list ap;
-    va_start (ap, fmt);
-    if ((ov->child = endpoint_vcreate (fmt, ap)))
-        rc = 0;
-    va_end (ap);
-    return rc;
+    if (!(ov->parent.pubkey = strdup (pubkey)))
+        return -1;
+    return 0;
 }
 
-const char *overlay_get_child (overlay_t *ov)
+int overlay_set_parent_uri (struct overlay *ov, const char *uri)
 {
-    if (!ov->child)
-        return NULL;
-    return ov->child->uri;
+    free (ov->parent.uri);
+    if (!(ov->parent.uri = strdup (uri)))
+        return -1;
+    return 0;
 }
 
-void overlay_set_child_cb (overlay_t *ov, overlay_cb_f cb, void *arg)
+const char *overlay_get_parent_uri (struct overlay *ov)
 {
-    ov->child_cb = cb;
-    ov->child_arg = arg;
+    return ov->parent.uri;
 }
 
-int overlay_sendmsg_child (overlay_t *ov, const flux_msg_t *msg)
+static int overlay_sendmsg_parent (struct overlay *ov, const flux_msg_t *msg)
 {
     int rc = -1;
 
-    if (!ov->child || !ov->child->zs) {
-        errno = EINVAL;
+    if (!ov->parent.zsock || ov->parent.offline || ov->parent.goodbye_sent) {
+        errno = EHOSTUNREACH;
         goto done;
     }
-    rc = flux_msg_sendzsock (ov->child->zs, msg);
+    rc = zmqutil_msg_send (ov->parent.zsock, msg);
+    if (rc == 0) {
+        ov->parent.lastsent = flux_reactor_now (ov->reactor);
+        trace_overlay_msg (ov->h,
+                           "tx",
+                           ov->parent.rank,
+                           ov->trace_requests,
+                           msg);
+    }
 done:
     return rc;
 }
 
-int overlay_mcast_child (overlay_t *ov, const flux_msg_t *msg)
+static int overlay_control_parent (struct overlay *ov,
+                                   enum control_type type,
+                                   int status)
 {
-    flux_msg_t *cpy = NULL;
-    const char *uuid;
-    child_t *child;
-    int rc = -1;
+    flux_msg_t *msg = NULL;
 
-    if (!ov->child || !ov->child->zs || !ov->children)
-        return 0;
-    FOREACH_ZHASH (ov->children, uuid, child) {
-        if (!(cpy = flux_msg_copy (msg, true)))
-            goto done;
-        if (flux_msg_enable_route (cpy) < 0)
-            goto done;
-        if (flux_msg_push_route (cpy, uuid) < 0)
-            goto done;
-        if (flux_msg_sendzsock (ov->child->zs, cpy) < 0)
-            goto done;
-        flux_msg_destroy (cpy);
-        cpy = NULL;
+    if (ov->parent.zsock) {
+        if (!(msg = flux_control_encode (type, status)))
+            return -1;
+        flux_msg_route_enable (msg);
+        if (overlay_sendmsg_parent (ov, msg) < 0)
+            goto error;
+        flux_msg_destroy (msg);
     }
-    rc = 0;
-done:
-    flux_msg_destroy (cpy);
-    return rc;
+    return 0;
+error:
+    flux_msg_destroy (msg);
+    return -1;
 }
 
-/* Handle notification of peer connect/disconnect on child monitor socket.
- * Maintain ov->child_peer_count, accessed via overlay_get_child_peer_count().
- * Call a callback, if any, when the count changes.
- * N.B. Assumes monitor protocol in libzmq >= 4.0 (assured by build system)
- */
-static void child_monitor_cb (flux_reactor_t *r,
-                              flux_watcher_t *w,
-                              int revents,
-                              void *arg)
+static int overlay_control_child (struct overlay *ov,
+                                  const char *uuid,
+                                  enum control_type type,
+                                  int status)
 {
-    overlay_t *ov = arg;
-    zframe_t *zf;
-    uint16_t event;
-
-    if (!(zf = zframe_recv (ov->child_monitor_sock)))
-        return; // spurious wakeup?
-    event = *(uint16_t *)zframe_data (zf);
-    zframe_destroy (&zf);
+    flux_msg_t *msg;
 
-    if (!(zf = zframe_recv (ov->child_monitor_sock))) {
-        log_msg ("zmq_socket_monitor: expected frame 2!");
-        return;
-    }
-    zframe_destroy (&zf);
-    if (event & ZMQ_EVENT_ACCEPTED)
-        ov->child_peer_count++;
-    if (event & ZMQ_EVENT_DISCONNECTED)
-        ov->child_peer_count--;
-    if (ov->child_monitor_cb)
-        ov->child_monitor_cb (ov, ov->child_monitor_arg);
+    if (!(msg = flux_control_encode (type, status)))
+        return -1;
+    flux_msg_route_enable (msg);
+    if (flux_msg_route_push (msg, uuid) < 0)
+        goto error;
+    if (overlay_sendmsg_child (ov, msg) < 0)
+        goto error;
+    flux_msg_destroy (msg);
+    return 0;
+error:
+    flux_msg_destroy (msg);
+    return -1;
 }
 
-/* Monitor child socket.
- * This creates a new socket that receives control messages when downstream
- * peers connect/disconnect.  Set up a watcher to capture these messages and
- * maintain a connected peer count.
- */
-static int child_monitor_init (overlay_t *ov, struct endpoint *ep)
+int overlay_sendmsg_new (struct overlay *ov,
+                         flux_msg_t **msg,
+                         overlay_where_t where)
 {
-    const char *uri = "inproc://monitor-child";
-    flux_reactor_t *r = flux_get_reactor (ov->h);
+    int type;
+    const char *uuid;
+    uint32_t nodeid;
+    struct child *child = NULL;
 
-    if (zmq_socket_monitor (zsock_resolve (ep->zs),
-                            uri,
-                            ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED) < 0) {
-        log_err ("zmq_socket_monitor");
+    if (flux_msg_get_type (*msg, &type) < 0)
         return -1;
+    switch (type) {
+        case FLUX_MSGTYPE_REQUEST:
+            /* If message is being routed downstream to reach 'nodeid',
+             * push the local uuid, then the next hop onto the messages's
+             * route stack so that the ROUTER socket can pop off next hop to
+             * select the peer, and our uuid remains as part of the source addr.
+             */
+            if (where == OVERLAY_ANY) {
+                if (flux_msg_get_nodeid (*msg, &nodeid) < 0)
+                    return -1;
+                if (flux_msg_has_flag (*msg, FLUX_MSGFLAG_UPSTREAM)
+                    && nodeid == ov->rank)
+                    where = OVERLAY_UPSTREAM;
+                else {
+                    if ((child = child_lookup_route (ov, nodeid))) {
+                        if (!subtree_is_online (child->status)) {
+                            errno = EHOSTUNREACH;
+                            return -1;
+                        }
+                        if (flux_msg_route_push (*msg, ov->uuid) < 0
+                            || flux_msg_route_push (*msg, child->uuid) < 0)
+                            return -1;
+                        where = OVERLAY_DOWNSTREAM;
+                    }
+                    else
+                        where = OVERLAY_UPSTREAM;
+                }
+            }
+            if (where == OVERLAY_UPSTREAM) {
+                if (overlay_sendmsg_parent (ov, *msg) < 0)
+                    return -1;
+                rpc_track_update (ov->parent.tracker, *msg);
+            }
+            else {
+                if (overlay_sendmsg_child (ov, *msg) < 0)
+                    return -1;
+                if (!child) {
+                    if ((uuid = flux_msg_route_last (*msg)))
+                        child = child_lookup_online (ov, ov->uuid);
+                }
+                if (child)
+                    rpc_track_update (child->tracker, *msg);
+            }
+            break;
+        case FLUX_MSGTYPE_RESPONSE:
+            /* Assume if next route matches parent, the message goes upstream;
+             * otherwise downstream.  The send downstream will fail with
+             * EHOSTUNREACH if uuid doesn't match an immediate peer.
+             */
+            if (where == OVERLAY_ANY) {
+                if (ov->rank > 0
+                    && (uuid = flux_msg_route_last (*msg)) != NULL
+                    && streq (uuid, ov->parent.uuid))
+                    where = OVERLAY_UPSTREAM;
+                else
+                    where = OVERLAY_DOWNSTREAM;
+            }
+            if (where == OVERLAY_UPSTREAM) {
+                if (overlay_sendmsg_parent (ov, *msg) < 0)
+                    return -1;
+            }
+            else {
+                if (overlay_sendmsg_child (ov, *msg) < 0)
+                    return -1;
+            }
+            break;
+        case FLUX_MSGTYPE_EVENT:
+            if (where == OVERLAY_DOWNSTREAM || where == OVERLAY_ANY)
+                overlay_mcast_child (ov, *msg);
+            else {
+                /* N.B. add route delimiter if needed to pass unpublished
+                 * event message upstream through router socket.
+                 */
+                if (flux_msg_route_count (*msg) < 0)
+                    flux_msg_route_enable (*msg);
+                if (overlay_sendmsg_parent (ov, *msg) < 0)
+                    return -1;
+            }
+            break;
+        default:
+            errno = EINVAL;
+            return -1;
     }
-    if (!(ov->child_monitor_sock = zsock_new_pair (uri))) {
-        log_err ("zsock_new_pair");
+    flux_msg_decref (*msg);
+    *msg = NULL;
+    return 0;
+}
+
+int overlay_sendmsg (struct overlay *ov,
+                     const flux_msg_t *msg,
+                     overlay_where_t where)
+{
+    flux_msg_t *cpy;
+
+    if (!(cpy = flux_msg_copy (msg, true)))
         return -1;
-    }
-    if (!(ov->child_monitor_w = flux_zmq_watcher_create (r,
-                                                         ov->child_monitor_sock,
-                                                         FLUX_POLLIN,
-                                                         child_monitor_cb,
-                                                         ov))) {
-        log_err ("flux_zmq_watcher_create");
+    if (overlay_sendmsg_new (ov, &cpy, where) < 0) {
+        flux_msg_destroy (cpy);
         return -1;
     }
-    flux_watcher_start (ov->child_monitor_w);
     return 0;
 }
 
-static void child_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
+static void sync_cb (flux_future_t *f, void *arg)
 {
-    void *zsock = flux_zmq_watcher_get_zsock (w);
-    overlay_t *ov = arg;
-    if (ov->child_cb)
-        ov->child_cb (ov, zsock, ov->child_arg);
+    struct overlay *ov = arg;
+    double now = flux_reactor_now (ov->reactor);
+
+    if (now - ov->parent.lastsent > ov->torpid_min)
+        overlay_control_parent (ov, CONTROL_HEARTBEAT, 0);
+    update_torpid_children (ov);
+
+    flux_future_reset (f);
 }
 
-/* Cleanup not done in this function, responsibiility of caller to
- * call endpoint_destroy() eventually */
-static int bind_child (overlay_t *ov, struct endpoint *ep)
+int overlay_control_start (struct overlay *ov)
 {
-    if (!(ep->zs = zsock_new_router (NULL))) {
-        log_err ("zsock_new_router");
-        return -1;
-    }
-    if (child_monitor_init (ov, ep) < 0)
-        return -1;
-    zsock_set_router_mandatory (ep->zs, 1);
-    if (zsecurity_ssockinit (ov->sec, ep->zs) < 0) {
-        log_msg ("zsecurity_ssockinit: %s", zsecurity_errstr (ov->sec));
-        return -1;
-    }
-    if (zsock_bind (ep->zs, "%s", ep->uri) < 0) {
-        log_err ("%s", ep->uri);
-        return -1;
-    }
-    if (strchr (ep->uri, '*')) { /* capture dynamically assigned port */
-        free (ep->uri);
-        ep->uri = zsock_last_endpoint (ep->zs);
-    }
-    if (!(ep->w = flux_zmq_watcher_create (flux_get_reactor (ov->h),
-                                           ep->zs, FLUX_POLLIN, child_cb, ov))) {
-        log_err ("flux_zmq_watcher_create");
-        return -1;
+    if (!ov->f_sync) {
+        if (!(ov->f_sync = flux_sync_create (ov->h, sync_min))
+            || flux_future_then (ov->f_sync, sync_max, sync_cb, ov) < 0)
+            return -1;
     }
-    flux_watcher_start (ep->w);
-    /* Ensure that ipc files are removed when the broker exits.
-     */
-    char *ipc_path = strstr (ep->uri, "ipc://");
-    if (ipc_path)
-        cleanup_push_string (cleanup_file, ipc_path + 6);
     return 0;
 }
 
-static void parent_cb (flux_reactor_t *r, flux_watcher_t *w,
-                       int revents, void *arg)
+const char *overlay_get_bind_uri (struct overlay *ov)
 {
-    void *zsock = flux_zmq_watcher_get_zsock (w);
-    overlay_t *ov = arg;
-    if (ov->parent_cb)
-        ov->parent_cb (ov, zsock, ov->parent_arg);
+    return ov->bind_uri;
 }
 
-static int connect_parent (overlay_t *ov, struct endpoint *ep)
+/* Log a failure to send tracker EHOSTUNREACH response.  Suppress logging
+ * ENOSYS failures, which happen if sending module unloads before completing
+ * all RPCs.
+ */
+static void log_tracker_error (flux_t *h, const flux_msg_t *msg, int errnum)
 {
-    int savederr;
-    char rankstr[16];
-
-    if (!(ep->zs = zsock_new_dealer (NULL)))
-        goto error;
-    if (zsecurity_csockinit (ov->sec, ep->zs) < 0) {
-        savederr = errno;
-        log_msg ("zsecurity_csockinit: %s", zsecurity_errstr (ov->sec));
-        errno = savederr;
-        goto error;
-    }
-    snprintf (rankstr, sizeof (rankstr), "%"PRIu32, ov->rank);
-    zsock_set_identity (ep->zs, rankstr);
-    if (zsock_connect (ep->zs, "%s", ep->uri) < 0)
-        goto error;
-    if (!(ep->w = flux_zmq_watcher_create (flux_get_reactor (ov->h),
-                                           ep->zs, FLUX_POLLIN, parent_cb, ov)))
-        goto error;
-    flux_watcher_start (ep->w);
-    return 0;
-error:
-    if (ep->zs) {
-        savederr = errno;
-        zsock_destroy (&ep->zs);
-        errno = savederr;
+    if (errnum != ENOSYS) {
+        const char *topic = "unknown";
+        (void)flux_msg_get_topic (msg, &topic);
+        flux_log_error (h,
+                        "tracker: error sending %s EHOSTUNREACH response",
+                        topic);
     }
-    return -1;
 }
 
-static int overlay_sec_init (overlay_t *ov)
+/* Child endpoint disconnected, so any pending RPCs going that way
+ * get EHOSTUNREACH responses so they can fail fast.
+ */
+static void fail_child_rpcs (const flux_msg_t *msg, void *arg)
 {
-    if (!ov->sec_initialized) {
-        if (zsecurity_comms_init (ov->sec) < 0) {
-            log_msg ("zsecurity_comms_init: %s", zsecurity_errstr (ov->sec));
-            return -1;
+    struct overlay *ov = arg;
+    flux_msg_t *rep;
+
+    if (!(rep = flux_response_derive (msg, EHOSTUNREACH))
+        || flux_msg_route_delete_last (rep) < 0
+        || flux_msg_route_delete_last (rep) < 0
+        || flux_send (ov->h, rep, 0) < 0)
+        log_tracker_error (ov->h, rep, errno);
+    flux_msg_destroy (rep);
+}
+
+static void overlay_child_status_update (struct overlay *ov,
+                                         struct child *child,
+                                         int status,
+                                         const char *reason)
+{
+    if (child->status != status) {
+        if (subtree_is_online (child->status)
+            && !subtree_is_online (status)) {
+            zhashx_delete (ov->child_hash, child->uuid);
+            rpc_track_purge (child->tracker, fail_child_rpcs, ov);
+        }
+        else if (!subtree_is_online (child->status)
+            && subtree_is_online (status)) {
+            zhashx_insert (ov->child_hash, child->uuid, child);
         }
-        ov->sec_initialized = true;
+
+        child->status = status;
+        monotime (&child->status_timestamp);
+
+        subtree_status_update (ov);
+        overlay_monitor_notify (ov, child->rank);
+        overlay_health_respond_all (ov);
     }
-    return 0;
+    errprintf (&child->error, "%s", reason ? reason : "");
+}
+
+static void log_lost_connection (struct overlay *ov,
+                                 struct child *child,
+                                 const char *why)
+{
+    int children = topology_get_descendant_count_at (ov->topo, child->rank);
+    char add[128] = { 0 };
+
+    if (children > 0) {
+        snprintf (add,
+                  sizeof (add),
+                  " and severed contact with %d other nodes",
+                  children);
+    }
+    flux_log (ov->h,
+              LOG_ERR,
+              "%s (rank %d) %s%s",
+              flux_get_hostbyrank (ov->h, child->rank),
+              (int)child->rank,
+              why,
+              add);
 }
 
-int overlay_connect (overlay_t *ov)
+static int overlay_sendmsg_child (struct overlay *ov, const flux_msg_t *msg)
 {
     int rc = -1;
 
-    if (!ov->sec || !ov->h || ov->rank == FLUX_NODEID_ANY || !ov->parent_cb) {
-        errno = EINVAL;
+    if (!ov->bind_zsock) {
+        errno = EHOSTUNREACH;
         goto done;
     }
-    if (overlay_sec_init (ov) < 0)
-        goto done;
-    if (ov->parent && !ov->parent->zs) {
-        if (connect_parent (ov, ov->parent) < 0) {
-            log_err ("%s", ov->parent->uri);
-            goto done;
+    rc = zmqutil_msg_send_ex (ov->bind_zsock, msg, true);
+    /* Since ROUTER socket has ZMQ_ROUTER_MANDATORY set, EHOSTUNREACH on a
+     * connected peer signifies a disconnect.  See zmq_setsockopt(3).
+     */
+    if (rc < 0 && errno == EHOSTUNREACH) {
+        int saved_errno = errno;
+        const char *uuid;
+        struct child *child;
+
+        if ((uuid = flux_msg_route_last (msg))
+            && (child = child_lookup_online (ov, uuid))) {
+            log_lost_connection (ov, child, "failed");
+            overlay_child_status_update (ov,
+                                         child,
+                                         SUBTREE_STATUS_LOST,
+                                         "lost connection");
+        }
+        errno = saved_errno;
+    }
+    if (rc == 0 && flux_msglist_count (ov->trace_requests) > 0) {
+        const char *uuid;
+        struct child *child = NULL;
+        int rank = -1;
+        int type = 0;
+
+        (void)flux_msg_get_type (msg, &type);
+
+        // N.B. events are traced in overlay_mcast_child()
+        if (type != FLUX_MSGTYPE_EVENT) {
+            if ((uuid = flux_msg_route_last (msg))
+                && (child = child_lookup_online (ov, uuid)))
+                rank = child->rank;
+            trace_overlay_msg (ov->h, "tx", rank, ov->trace_requests, msg);
         }
     }
-    rc = 0;
 done:
     return rc;
 }
 
-int overlay_bind (overlay_t *ov)
+/* Push child->uuid onto the message, then pop it off again after sending.
+ */
+static int overlay_mcast_child_one (struct overlay *ov,
+                                    flux_msg_t *msg,
+                                    struct child *child)
 {
-    int rc = -1;
+    if (flux_msg_route_push (msg, child->uuid) < 0)
+        return -1;
+    int rc = overlay_sendmsg_child (ov, msg);
+    (void)flux_msg_route_delete_last (msg);
+    return rc;
+}
 
-    if (!ov->sec || !ov->h || ov->rank == FLUX_NODEID_ANY || !ov->child_cb) {
-        errno = EINVAL;
-        goto done;
+static void overlay_mcast_child (struct overlay *ov, flux_msg_t *msg)
+{
+    struct child *child;
+    int count = 0;
+
+    flux_msg_route_enable (msg);
+
+    foreach_overlay_child (ov, child) {
+        if (subtree_is_online (child->status)) {
+            if (overlay_mcast_child_one (ov, msg, child) < 0) {
+                if (errno != EHOSTUNREACH) {
+                    flux_log_error (ov->h,
+                                    "mcast error to child rank %lu",
+                                    (unsigned long)child->rank);
+                }
+            }
+            else
+                count++;
+        }
     }
-    if (overlay_sec_init (ov) < 0)
-        goto done;
-
-    if (ov->child && !ov->child->zs) {
-        if (bind_child (ov, ov->child) < 0)
-            goto done;
+    if (count > 0) {
+        trace_overlay_msg (ov->h,
+                           "tx",
+                           FLUX_NODEID_ANY,
+                           ov->trace_requests,
+                           msg);
     }
-
-    rc = 0;
-done:
-    return rc;
 }
 
-/* A callback of type attr_get_f to allow retrieving some information
- * from an overlay_t through attr_get().
- */
-static int overlay_attr_get_cb (const char *name, const char **val, void *arg)
+static void logdrop (struct overlay *ov,
+                     overlay_where_t where,
+                     const flux_msg_t *msg,
+                     const char *fmt,
+                     ...)
 {
-    overlay_t *overlay = arg;
-    int rc = -1;
+    char reason[128];
+    va_list ap;
+    const char *topic = NULL;
+    int type = -1;
+    const char *child_uuid = NULL;
 
-    if (!strcmp (name, "tbon.parent-endpoint"))
-        *val = overlay_get_parent (overlay);
-    else {
-        errno = ENOENT;
-        goto done;
-    }
-    rc = 0;
-done:
-    return rc;
+    (void)flux_msg_get_type (msg, &type);
+    (void)flux_msg_get_topic (msg, &topic);
+    if (where == OVERLAY_DOWNSTREAM)
+        child_uuid = flux_msg_route_last (msg);
+
+    va_start (ap, fmt);
+    (void)vsnprintf (reason, sizeof (reason), fmt, ap);
+    va_end (ap);
+
+    flux_log (ov->h,
+              LOG_ERR,
+              "DROP %s %s topic %s %s%s: %s",
+              where == OVERLAY_UPSTREAM ? "upstream" : "downstream",
+              type != -1 ? flux_msg_typestr (type) : "message",
+              topic ? topic : "-",
+              child_uuid ? "from " : "",
+              child_uuid ? child_uuid : "",
+              reason);
 }
 
-int overlay_register_attrs (overlay_t *overlay, attr_t *attrs)
+static int clear_msg_role (flux_msg_t *msg, uint32_t role)
 {
-    if (attr_add_active (attrs, "tbon.parent-endpoint",
-                         FLUX_ATTRFLAG_READONLY,
-                         overlay_attr_get_cb, NULL, overlay) < 0)
-        return -1;
-    if (attr_add_uint32 (attrs, "rank", overlay->rank,
-                         FLUX_ATTRFLAG_IMMUTABLE) < 0)
-        return -1;
-    if (attr_add_uint32 (attrs, "size", overlay->size,
-                         FLUX_ATTRFLAG_IMMUTABLE) < 0)
-        return -1;
-    if (attr_add_int (attrs, "tbon.arity", overlay->tbon_k,
-                      FLUX_ATTRFLAG_IMMUTABLE) < 0)
-        return -1;
-    if (attr_add_int (attrs, "tbon.level", overlay->tbon_level,
-                      FLUX_ATTRFLAG_IMMUTABLE) < 0)
-        return -1;
-    if (attr_add_int (attrs, "tbon.maxlevel", overlay->tbon_maxlevel,
-                      FLUX_ATTRFLAG_IMMUTABLE) < 0)
+    uint32_t rolemask;
+
+    if (flux_msg_get_rolemask (msg, &rolemask) < 0)
         return -1;
-    if (attr_add_int (attrs, "tbon.descendants", overlay->tbon_descendants,
-                      FLUX_ATTRFLAG_IMMUTABLE) < 0)
+    rolemask &= ~role;
+    if (flux_msg_set_rolemask (msg, rolemask) < 0)
         return -1;
-
     return 0;
 }
 
-void overlay_set_monitor_cb (overlay_t *ov, overlay_monitor_cb_f cb, void *arg)
+/* Handle a message received from TBON child (downstream).
+ */
+static void child_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
 {
-    ov->child_monitor_cb = cb;
-    ov->child_monitor_arg = arg;
+    struct overlay *ov = arg;
+    flux_msg_t *msg;
+    int type = -1;
+    const char *topic = NULL;
+    const char *uuid = NULL;
+    struct child *child;
+
+    if (!(msg = zmqutil_msg_recv (ov->bind_zsock)))
+        return;
+    if (clear_msg_role (msg, FLUX_ROLE_LOCAL) < 0) {
+        logdrop (ov, OVERLAY_DOWNSTREAM, msg, "failed to clear local role");
+        goto done;
+    }
+    if (flux_msg_get_type (msg, &type) < 0
+        || !(uuid = flux_msg_route_last (msg))) {
+        logdrop (ov, OVERLAY_DOWNSTREAM, msg, "malformed message");
+        goto done;
+    }
+    if (!(child = child_lookup_online (ov, uuid))) {
+        /* This is a new peer trying to introduce itself by sending an
+         * overlay.hello request.
+         * N.B. the broker generates a new UUID on startup, and hello is only
+         * sent once on startup, in overlay_connect().  Therefore, it is
+         * assumed that a overlay.hello is always introducing a new UUID and
+         * we don't bother checking if we've seen this UUID before, which can
+         * be slow given current design.  See flux-framework/flux-core#5864.
+         */
+        if (type == FLUX_MSGTYPE_REQUEST
+            && flux_msg_get_topic (msg, &topic) == 0
+            && streq (topic, "overlay.hello")
+            && !ov->shutdown_in_progress) {
+            trace_overlay_msg (ov->h,
+                               "rx",
+                               FLUX_NODEID_ANY,
+                               ov->trace_requests,
+                               msg);
+            hello_request_handler (ov, msg);
+        }
+        /* Or one of the following cases occurred that requires (or at least
+         * will not be harmed by) a DISCONNECT message sent to the peer:
+         * 1) This is a known peer that has transitioned to offline/lost _here_
+         *    but the DISCONNECT is still in flight to the peer.
+         * 2) This is a known peer that has transitioned to offline/lost
+         *    as a result of a network partition, but the child never received
+         *    the DISCONNECT and connectivity has been restored.
+         * 3) This is a new-to-us peer because *we* restarted without getting
+         *    a message through (e.g. crash)
+         * 4) A peer said hello while shutdown is in progress
+         * Control send failures may occur, see flux-framework/flux-core#4464.
+         * Don't log here, see flux-framework/flux-core#4180.
+         */
+        else {
+            (void)overlay_control_child (ov, uuid, CONTROL_DISCONNECT, 0);
+        }
+        goto done;
+    }
+    /* N.B. We just looked up the child subtree in the online list, so
+     * it is guaranteed to be online at this point.
+     */
+
+    child->lastseen = flux_reactor_now (ov->reactor);
+    switch (type) {
+        case FLUX_MSGTYPE_CONTROL: {
+            int type, status;
+            if (flux_control_decode (msg, &type, &status) == 0
+                && type == CONTROL_STATUS) {
+                trace_overlay_msg (ov->h,
+                                   "rx",
+                                   child->rank,
+                                   ov->trace_requests,
+                                   msg);
+                overlay_child_status_update (ov, child, status, NULL);
+            }
+            goto done;
+        }
+        case FLUX_MSGTYPE_REQUEST:
+            break;
+        case FLUX_MSGTYPE_RESPONSE:
+            /* Response message traveling upstream requires special handling:
+             * ROUTER socket will have pushed peer id onto message as if it
+             * were a request, but the effect we want for responses is to have
+             * a route popped off at each router hop.
+             */
+            (void)flux_msg_route_delete_last (msg); // child id from ROUTER
+            (void)flux_msg_route_delete_last (msg); // my id
+            rpc_track_update (child->tracker, msg);
+            break;
+        case FLUX_MSGTYPE_EVENT:
+            break;
+    }
+    trace_overlay_msg (ov->h, "rx", child->rank, ov->trace_requests, msg);
+    if (ov->recv_cb (&msg, OVERLAY_DOWNSTREAM, ov->recv_arg) < 0)
+        goto done;
+    return;
+done:
+    flux_msg_decref (msg);
+}
+
+/* Parent endpoint disconnected, so any pending RPCs going that way
+ * get EHOSTUNREACH responses so they can fail fast.
+ */
+static void fail_parent_rpc (const flux_msg_t *msg, void *arg)
+{
+    struct overlay *ov = arg;
+
+    if (flux_respond_error (ov->h,
+                            msg,
+                            EHOSTUNREACH,
+                            "overlay disconnect") < 0)
+        log_tracker_error (ov->h, msg, errno);
+}
+
+static void parent_cb (flux_reactor_t *r,
+                       flux_watcher_t *w,
+                       int revents,
+                       void *arg)
+{
+    struct overlay *ov = arg;
+    flux_msg_t *msg;
+    int type;
+    const char *topic = NULL;
+
+    if (!(msg = zmqutil_msg_recv (ov->parent.zsock)))
+        return;
+    if (clear_msg_role (msg, FLUX_ROLE_LOCAL) < 0) {
+        logdrop (ov, OVERLAY_UPSTREAM, msg, "failed to clear local role");
+        goto done;
+    }
+    if (flux_msg_get_type (msg, &type) < 0) {
+        logdrop (ov, OVERLAY_UPSTREAM, msg, "malformed message");
+        goto done;
+    }
+    if (!ov->parent.hello_responded) {
+        /* process hello response */
+        if (type == FLUX_MSGTYPE_RESPONSE
+            && flux_msg_get_topic (msg, &topic) == 0
+            && streq (topic, "overlay.hello")) {
+            hello_response_handler (ov, msg);
+            goto done;
+        }
+        else if (type != FLUX_MSGTYPE_CONTROL) {
+            logdrop (ov, OVERLAY_UPSTREAM, msg,
+                     "message received before hello handshake completed");
+            goto done;
+        }
+    }
+    switch (type) {
+        case FLUX_MSGTYPE_RESPONSE:
+            rpc_track_update (ov->parent.tracker, msg);
+            break;
+        case FLUX_MSGTYPE_EVENT:
+            /* Upstream broker enables routing and pushes our uuid, then
+             * router socket pops it off, but leaves routing enabled.
+             * An event type message should not have routing enabled
+             * under normal circumstances, so turn it off here.
+             */
+            flux_msg_route_disable (msg);
+            break;
+        case FLUX_MSGTYPE_CONTROL: {
+            int ctrl_type, reason;
+            if (flux_control_decode (msg, &ctrl_type, &reason) < 0) {
+                logdrop (ov, OVERLAY_UPSTREAM, msg, "malformed control");
+            }
+            else if (ctrl_type == CONTROL_DISCONNECT) {
+                flux_log (ov->h, LOG_CRIT,
+                          "%s (rank %lu) sent disconnect control message",
+                          flux_get_hostbyrank (ov->h, ov->parent.rank),
+                          (unsigned long)ov->parent.rank);
+                (void)zmq_disconnect (ov->parent.zsock, ov->parent.uri);
+                ov->parent.offline = true;
+                rpc_track_purge (ov->parent.tracker, fail_parent_rpc, ov);
+                overlay_monitor_notify (ov, FLUX_NODEID_ANY);
+            }
+            else
+                logdrop (ov, OVERLAY_UPSTREAM, msg, "unknown control type");
+            goto done;
+        }
+        default:
+            break;
+    }
+    trace_overlay_msg (ov->h, "rx", ov->parent.rank, ov->trace_requests, msg);
+    if (ov->recv_cb (&msg, OVERLAY_UPSTREAM, ov->recv_arg) < 0)
+        goto done;
+    return;
+done:
+    flux_msg_destroy (msg);
+}
+
+
+#define V_MAJOR(v)  (((v) >> 16) & 0xff)
+#define V_MINOR(v)  (((v) >> 8) & 0xff)
+#define V_PATCH(v)  ((v) & 0xff)
+
+/* Check child flux-core version 'v1' against this broker's version 'v2'.
+ * For now we require an exact match of MAJOR.MINOR, but not PATCH.
+ * ignore any commit id appended to the version string.
+ * Return 0 on error, or -1 on failure with message for child in 'error'.
+ */
+static bool version_check (int v1, int v2, flux_error_t *error)
+{
+    if (V_MAJOR (v1) != V_MAJOR (v2) || V_MINOR (v1) != V_MINOR (v2)) {
+        errprintf (error,
+                  "client (%u.%u.%u) version mismatched with server (%u.%u.%u)",
+                  V_MAJOR (v1),
+                  V_MINOR (v1),
+                  V_PATCH (v1),
+                  V_MAJOR (v2),
+                  V_MINOR (v2),
+                  V_PATCH (v2));
+        return false;
+    }
+    return true;
+}
+
+/* Handle overlay.hello request from downstream (child) TBON peer.
+ * The peer may be rejected here if it is improperly configured.
+ * If successful the child's status is updated to reflect its online
+ * state and the state machine is notified.
+ * N.B. respond using overlay_sendmsg_child() to avoid the main message path
+ * during initialization.
+ */
+static void hello_request_handler (struct overlay *ov, const flux_msg_t *msg)
+{
+    struct child *child;
+    json_int_t rank;
+    int version;
+    const char *errmsg = NULL;
+    flux_error_t error;
+    flux_msg_t *response;
+    const char *uuid;
+    int status;
+    const char *hostname = NULL;
+    int hello_log_level = LOG_DEBUG;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:i s:s s:i s?s}",
+                             "rank", &rank,
+                             "version", &version,
+                             "uuid", &uuid,
+                             "status", &status,
+                             "hostname", &hostname) < 0)
+        goto error; // EPROTO (unlikely)
+
+    if (flux_msg_authorize (msg, FLUX_USERID_UNKNOWN) < 0) {
+        /* special handling for v0.46.1 flux-framework/flux-core#4886.
+         * The rolemask/userid are sent in the wrong byte order, so
+         * authorization silently fails. Log something helpful.
+         */
+        if (version == 0x002e01) {
+            flux_log (ov->h, LOG_ERR,
+                      "rejecting connection from %s (rank %lu): %s",
+                      flux_get_hostbyrank (ov->h, rank),
+                      (unsigned long)rank,
+                      "v0.46.1 has a message encoding bug, please upgrade");
+        }
+        goto error; // EPERM
+    }
+    // flux-framework/flux-core#6389
+    if (hostname && !streq (hostname, flux_get_hostbyrank (ov->h, rank))) {
+        errprintf (&error,
+                  "%s is configured as rank %lu: mismatched config?",
+                  flux_get_hostbyrank (ov->h, rank),
+                  (unsigned long)rank);
+        flux_log (ov->h,
+                  LOG_ERR,
+                  "rejecting connection from %s (rank %lu): %s",
+                  hostname,
+                  (unsigned long)rank,
+                  error.text);
+        errno = EINVAL;
+        goto error;
+    }
+    if (!(child = child_lookup_byrank (ov, rank))) {
+        errprintf (&error,
+                  "rank %lu is not a peer of parent %lu: mismatched config?",
+                  (unsigned long)rank,
+                  (unsigned long)ov->parent.rank);
+        flux_log (ov->h, LOG_ERR,
+                  "rejecting connection from %s (rank %lu): %s",
+                  flux_get_hostbyrank (ov->h, rank),
+                  (unsigned long)rank,
+                  error.text);
+        errmsg = error.text;
+        errno = EINVAL;
+        goto error;
+    }
+    /* Oops, child was previously online, but is now saying hello.
+     * Update the (old) child's subtree status to LOST.  If the hello
+     * request is successful, another update will immediately follow.
+     */
+    if (subtree_is_online (child->status)) { // crash
+        flux_log (ov->h, LOG_ERR,
+        "%s (rank %lu) reconnected after crash, dropping old connection state",
+                  flux_get_hostbyrank (ov->h, child->rank),
+                  (unsigned long)child->rank);
+        overlay_child_status_update (ov, child, SUBTREE_STATUS_LOST, NULL);
+        hello_log_level = LOG_ERR; // want hello log to stand out in this case
+    }
+
+    if (!version_check (version, ov->version, &error)) {
+        child->error = error; // capture this error message for health report
+        errmsg = error.text;
+        errno = EINVAL;
+        goto error;
+    }
+
+    snprintf (child->uuid, sizeof (child->uuid), "%s", uuid);
+    overlay_child_status_update (ov, child, status, NULL);
+
+    flux_log (ov->h,
+              hello_log_level,
+              "accepting connection from %s (rank %lu) status %s",
+              flux_get_hostbyrank (ov->h, child->rank),
+              (unsigned long)child->rank,
+              subtree_status_str (child->status));
+
+    if (!(response = flux_response_derive (msg, 0))
+        || flux_msg_pack (response, "{s:s}", "uuid", ov->uuid) < 0
+        || overlay_sendmsg_child (ov, response) < 0)
+        flux_log_error (ov->h, "error responding to overlay.hello request");
+    flux_msg_destroy (response);
+    return;
+error:
+    if (!(response = flux_response_derive (msg, errno))
+        || (errmsg && flux_msg_set_string (response, errmsg) < 0)
+        || overlay_sendmsg_child (ov, response) < 0)
+        flux_log_error (ov->h, "error responding to overlay.hello request");
+    flux_msg_destroy (response);
+}
+
+/* Process overlay.hello response.
+ * If the response indicates an error, set in motion a clean broker exit by
+ * printing the error message to stderr and notifying the state machine
+ * that it should check overlay_parent_error() / overlay_parent_success().
+ */
+static void hello_response_handler (struct overlay *ov, const flux_msg_t *msg)
+{
+    const char *errstr = NULL;
+    const char *uuid;
+
+    if (flux_response_decode (msg, NULL, NULL) < 0
+        || flux_msg_unpack (msg, "{s:s}", "uuid", &uuid) < 0) {
+        int saved_errno = errno;
+        (void)flux_msg_get_string (msg, &errstr);
+        errno = saved_errno;
+        goto error;
+    }
+    flux_log (ov->h,
+              LOG_DEBUG,
+              "hello parent %lu %s",
+              (unsigned long)ov->parent.rank,
+              uuid);
+    snprintf (ov->parent.uuid, sizeof (ov->parent.uuid), "%s", uuid);
+    ov->parent.hello_responded = true;
+    ov->parent.hello_error = false;
+    overlay_monitor_notify (ov, FLUX_NODEID_ANY);
+    return;
+error:
+    log_msg ("overlay.hello: %s", errstr ? errstr : flux_strerror (errno));
+    ov->parent.hello_responded = true;
+    ov->parent.hello_error = true;
+    overlay_monitor_notify (ov, FLUX_NODEID_ANY);
+}
+
+/* Send overlay.hello message to TBON parent.
+ */
+static int hello_request_send (struct overlay *ov,
+                               json_int_t rank,
+                               int version)
+{
+    flux_msg_t *msg;
+
+    if (!(msg = flux_request_encode ("overlay.hello", NULL))
+        || flux_msg_pack (msg,
+                          "{s:I s:i s:s s:i s:s}",
+                          "rank", rank,
+                          "version", ov->version,
+                          "uuid", ov->uuid,
+                          "status", ov->status,
+                          "hostname", ov->hostname) < 0
+        || flux_msg_set_rolemask (msg, FLUX_ROLE_OWNER) < 0
+        || overlay_sendmsg_parent (ov, msg) < 0) {
+        flux_msg_decref (msg);
+        return -1;
+    }
+    flux_msg_decref (msg);
+    return 0;
+}
+
+static void bind_monitor_cb (struct zmqutil_monitor *mon, void *arg)
+{
+    struct overlay *ov = arg;
+    struct monitor_event event;
+
+    if (zmqutil_monitor_get (mon, &event) == 0) {
+        flux_log (ov->h,
+                  zmqutil_monitor_iserror (&event) ? LOG_ERR : LOG_DEBUG,
+                  "child sockevent %s %s%s%s",
+                  event.endpoint,
+                  event.event_str,
+                  *event.value_str ? ": " : "",
+                  event.value_str);
+    }
+}
+static void parent_monitor_cb (struct zmqutil_monitor *mon, void *arg)
+{
+    struct overlay *ov = arg;
+    struct monitor_event event;
+
+    if (zmqutil_monitor_get (mon, &event) == 0) {
+        flux_log (ov->h,
+                  zmqutil_monitor_iserror (&event) ? LOG_ERR : LOG_DEBUG,
+                  "parent sockevent %s %s%s%s",
+                  event.endpoint,
+                  event.event_str,
+                  *event.value_str ? ": " : "",
+                  event.value_str);
+    }
+}
+
+static int overlay_zmq_init (struct overlay *ov)
+{
+    if (!ov->zctx) {
+        if (!(ov->zctx = zmq_ctx_new ()))
+            return -1;
+        /* At this point, ensure that tbon.zmq_io_threads is only increased on
+         * the leader node, on the assumption that it will be less effective on
+         * other nodes yet increases the broker's footprint.
+         * This could be removed if we decide otherwise later on.
+         */
+        if (ov->rank > 0 && ov->zmq_io_threads != 1) {
+            const char *key = "tbon.zmq_io_threads";
+            if (attr_set_flags (ov->attrs, key, 0) < 0
+                || attr_delete (ov->attrs, key, true) < 0
+                || attr_add_int (ov->attrs, key, 1, ATTR_IMMUTABLE) < 0)
+                return -1;
+            ov->zmq_io_threads = 1;
+        }
+        if (zmq_ctx_set (ov->zctx,
+                         ZMQ_IO_THREADS,
+                         ov->zmq_io_threads) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+int overlay_connect (struct overlay *ov)
+{
+    if (ov->rank > 0) {
+        if (!ov->h || ov->rank == FLUX_NODEID_ANY || !ov->parent.uri) {
+            errno = EINVAL;
+            return -1;
+        }
+        if (overlay_zmq_init (ov) < 0)
+            return -1;
+        if (!(ov->parent.zsock = zmq_socket (ov->zctx, ZMQ_DEALER))
+            || zsetsockopt_int (ov->parent.zsock, ZMQ_SNDHWM, 0) < 0
+            || zsetsockopt_int (ov->parent.zsock, ZMQ_RCVHWM, 0) < 0
+            || zsetsockopt_int (ov->parent.zsock, ZMQ_LINGER, 5) < 0
+            || zsetsockopt_int (ov->parent.zsock, ZMQ_IPV6, ov->enable_ipv6) < 0
+            || zsetsockopt_str (ov->parent.zsock, ZMQ_IDENTITY, ov->uuid) < 0
+            || zsetsockopt_str (ov->parent.zsock,
+                                ZMQ_ZAP_DOMAIN,
+                                FLUX_ZAP_DOMAIN) < 0
+            || zsetsockopt_str (ov->parent.zsock,
+                                ZMQ_CURVE_SERVERKEY,
+                                ov->parent.pubkey) < 0)
+            return -1;
+        /* The socket monitor is only used for logging.
+         * Setup may fail if libzmq is too old.
+         */
+        if (ov->zmqdebug) {
+            ov->parent.monitor = zmqutil_monitor_create (ov->zctx,
+                                                         ov->parent.zsock,
+                                                         ov->reactor,
+                                                         parent_monitor_cb,
+                                                         ov);
+        }
+#ifdef ZMQ_CONNECT_TIMEOUT
+        if (ov->connect_timeout > 0) {
+            if (zsetsockopt_int (ov->parent.zsock,
+                                 ZMQ_CONNECT_TIMEOUT,
+                                 ov->connect_timeout * 1000) < 0)
+                return -1;
+        }
+#endif
+        if (cert_apply (ov->cert, ov->parent.zsock) < 0)
+            return -1;
+        if (zmq_connect (ov->parent.zsock, ov->parent.uri) < 0)
+            return -1;
+        if (!(ov->parent.w = zmqutil_watcher_create (ov->reactor,
+                                                     ov->parent.zsock,
+                                                     FLUX_POLLIN,
+                                                     parent_cb,
+                                                     ov)))
+            return -1;
+        flux_watcher_start (ov->parent.w);
+        if (hello_request_send (ov, ov->rank, FLUX_CORE_VERSION_HEX) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static void zaplogger (int severity, const char *message, void *arg)
+{
+    struct overlay *ov = arg;
+
+    flux_log (ov->h, severity, "%s", message);
+}
+
+int overlay_bind (struct overlay *ov, const char *uri)
+{
+    if (!ov->h || ov->rank == FLUX_NODEID_ANY || ov->bind_zsock) {
+        errno = EINVAL;
+        log_err ("overlay_bind: invalid arguments");
+        return -1;
+    }
+    if (overlay_zmq_init (ov) < 0) {
+        log_err ("error creating zeromq context");
+        return -1;
+    }
+    if (ov->zap != NULL) {
+        log_err ("ZAP is already initialized!");
+        return -1;
+    }
+    if (!(ov->zap = zmqutil_zap_create (ov->zctx, ov->reactor))) {
+        log_err ("error creating ZAP server");
+        return -1;
+    }
+    zmqutil_zap_set_logger (ov->zap, zaplogger, ov);
+
+    if (!(ov->bind_zsock = zmq_socket (ov->zctx, ZMQ_ROUTER))
+        || zsetsockopt_int (ov->bind_zsock, ZMQ_SNDHWM, 0) < 0
+        || zsetsockopt_int (ov->bind_zsock, ZMQ_RCVHWM, ov->child_rcvhwm) < 0
+        || zsetsockopt_int (ov->bind_zsock, ZMQ_LINGER, 5) < 0
+        || zsetsockopt_int (ov->bind_zsock, ZMQ_ROUTER_MANDATORY, 1) < 0
+        || zsetsockopt_int (ov->bind_zsock, ZMQ_IPV6, ov->enable_ipv6) < 0
+        || zsetsockopt_str (ov->bind_zsock, ZMQ_ZAP_DOMAIN, FLUX_ZAP_DOMAIN) < 0
+        || zsetsockopt_int (ov->bind_zsock, ZMQ_CURVE_SERVER, 1) < 0) {
+        log_err ("error creating zmq ROUTER socket");
+        return -1;
+    }
+    /* The socket monitor is only used for logging.
+     * Setup may fail if libzmq is too old.
+     */
+    if (ov->zmqdebug) {
+        ov->bind_monitor = zmqutil_monitor_create (ov->zctx,
+                                                   ov->bind_zsock,
+                                                   ov->reactor,
+                                                   bind_monitor_cb,
+                                                   ov);
+    }
+#ifdef ZMQ_TCP_MAXRT
+    if (ov->tcp_user_timeout > 0) {
+        if (zsetsockopt_int (ov->bind_zsock,
+                             ZMQ_TCP_MAXRT,
+                             ov->tcp_user_timeout * 1000) < 0) {
+            log_err ("error setting TCP_MAXRT option on bind socket");
+            return -1;
+        }
+    }
+#endif
+    if (cert_apply (ov->cert, ov->bind_zsock) < 0) {
+        log_err ("error setting curve socket options");
+        return -1;
+    }
+    if (zmq_bind (ov->bind_zsock, uri) < 0) {
+        log_err ("error binding to %s", uri);
+        return -1;
+    }
+    /* Capture URI after zmq_bind() processing, so it reflects expanded
+     * wildcards and normalized addresses.
+     */
+    if (zgetsockopt_str (ov->bind_zsock,
+                         ZMQ_LAST_ENDPOINT,
+                         &ov->bind_uri) < 0) {
+        log_err ("error obtaining concretized bind URI");
+        return -1;
+    }
+    if (!(ov->bind_w = zmqutil_watcher_create (ov->reactor,
+                                               ov->bind_zsock,
+                                               FLUX_POLLIN,
+                                               child_cb,
+                                               ov))) {
+        log_err ("error creating watcher for bind socket");
+        return -1;
+    }
+    flux_watcher_start (ov->bind_w);
+    /* Ensure that ipc files are removed when the broker exits.
+     */
+    char *ipc_path = strstr (ov->bind_uri, "ipc://");
+    if (ipc_path)
+        cleanup_push_string (cleanup_file, ipc_path + 6);
+    return 0;
+}
+
+/* Don't allow downstream peers to reconnect while we are shutting down.
+ */
+void overlay_shutdown (struct overlay *overlay, bool unbind)
+{
+    overlay->shutdown_in_progress = true;
+    if (unbind) {
+        if (overlay->bind_zsock && overlay->bind_uri)
+            if (zmq_unbind (overlay->bind_zsock, overlay->bind_uri) < 0)
+                flux_log (overlay->h, LOG_ERR, "zmq_unbind failed");
+    }
+}
+
+/* Call after overlay bootstrap (bind/connect),
+ * to get concretized 0MQ endpoints.
+ */
+int overlay_register_attrs (struct overlay *overlay)
+{
+    if (attr_add (overlay->attrs,
+                  "tbon.parent-endpoint",
+                  overlay->parent.uri,
+                  ATTR_IMMUTABLE) < 0)
+        return -1;
+    if (attr_add_uint32 (overlay->attrs,
+                         "rank",
+                         overlay->rank,
+                         ATTR_IMMUTABLE) < 0)
+        return -1;
+    if (attr_add_uint32 (overlay->attrs,
+                         "size", overlay->size,
+                         ATTR_IMMUTABLE) < 0)
+        return -1;
+    if (attr_add_int (overlay->attrs,
+                      "tbon.level",
+                      topology_get_level (overlay->topo),
+                      ATTR_IMMUTABLE) < 0)
+        return -1;
+    if (attr_add_int (overlay->attrs,
+                      "tbon.maxlevel",
+                      topology_get_maxlevel (overlay->topo),
+                      ATTR_IMMUTABLE) < 0)
+        return -1;
+    if (attr_add_int (overlay->attrs,
+                      "tbon.descendants",
+                      topology_get_descendant_count (overlay->topo),
+                      ATTR_IMMUTABLE) < 0)
+        return -1;
+
+    return 0;
+}
+
+int overlay_set_monitor_cb (struct overlay *ov,
+                            overlay_monitor_f cb,
+                            void *arg)
+{
+    struct overlay_monitor *mon;
+
+    if (!(mon = calloc (1, sizeof (*mon))))
+        return -1;
+    mon->cb = cb;
+    mon->arg = arg;
+    if (zlist_append (ov->monitor_callbacks, mon) < 0) {
+        free (mon);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+/* A child has sent an overlay.goodbye request.
+ * Respond, then transition it to OFFLINE.
+ */
+static void overlay_goodbye_cb (flux_t *h,
+                                flux_msg_handler_t *mh,
+                                const flux_msg_t *msg,
+                                void *arg)
+{
+    struct overlay *ov = arg;
+    const char *uuid;
+    struct child *child;
+    flux_msg_t *response = NULL;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0
+        || !(uuid = flux_msg_route_last (msg))
+        || !(child = child_lookup_online (ov, uuid)))
+        goto error;
+    if (!(response = flux_response_derive (msg, 0)))
+        goto error;
+    if (overlay_sendmsg_child (ov, response) < 0) {
+        flux_msg_decref (response);
+        goto error;
+    }
+    overlay_child_status_update (ov,
+                                 child,
+                                 SUBTREE_STATUS_OFFLINE,
+                                 "administrative shutdown");
+    flux_msg_decref (response);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to overlay.goodbye");
+}
+
+/* The parent has responded to overlay.goodbye.  Fulfill the future
+ * returned by overlay_goodbye_parent() so the state machine can
+ * make progress.
+ */
+static void overlay_goodbye_response_cb (flux_t *h,
+                                         flux_msg_handler_t *mh,
+                                         const flux_msg_t *msg,
+                                         void *arg)
+{
+    struct overlay *ov = arg;
+    flux_future_fulfill (ov->parent.f_goodbye, NULL, NULL);
+}
+
+/* This allows the state machine to delay overlay_destroy() and its
+ * disconnection from the parent until the parent has marked this peer
+ * offline and sent an acknowledgement.  If we simply send a control status
+ * offline message and disconnect, the parent may log errors if it sends any
+ * messages to this peer (such as broadcasts) before the offline message is
+ * processed and gets an EHOSTUNREACH for an online peer.
+ * See flux-framework/flux-core#5881.
+ */
+flux_future_t *overlay_goodbye_parent (struct overlay *ov)
+{
+    flux_msg_t *msg;
+
+    /* Avoid 60s delay on shutdown of followers when upstream is down.
+     * flux-framework/flux-core#5991
+     */
+    if (!(ov->parent.hello_responded)) {
+        errno = EHOSTUNREACH;
+        return NULL;
+    }
+    if (!(msg = flux_request_encode ("overlay.goodbye", NULL))
+        || flux_msg_set_rolemask (msg, FLUX_ROLE_OWNER) < 0
+        || overlay_sendmsg_parent (ov, msg) < 0) {
+        flux_msg_decref (msg);
+        return NULL;
+    }
+    ov->parent.goodbye_sent = true; // suppress further sends to parent
+    flux_msg_decref (msg);
+    flux_future_incref (ov->parent.f_goodbye);
+    return ov->parent.f_goodbye;
+}
+
+static int child_rpc_track_count (struct overlay *ov)
+{
+    int count = 0;
+    int i;
+    for (i = 0; i < ov->child_count; i++)
+        count += rpc_track_count (ov->children[i].tracker);
+    return count;
+}
+
+static void overlay_stats_get_cb (flux_t *h,
+                                  flux_msg_handler_t *mh,
+                                  const flux_msg_t *msg,
+                                  void *arg)
+{
+    struct overlay *ov = arg;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:i s:i s:i s:i s:i}",
+                           "child-count", ov->child_count,
+                           "child-connected", overlay_get_child_peer_count (ov),
+                           "parent-count", ov->rank > 0 ? 1 : 0,
+                           "parent-rpc", rpc_track_count (ov->parent.tracker),
+                           "child-rpc", child_rpc_track_count (ov)) < 0)
+        flux_log_error (h, "error responding to overlay.stats-get");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to overlay.stats-get");
+}
+
+static int overlay_health_respond (struct overlay *ov, const flux_msg_t *msg)
+{
+    struct child *child;
+    json_t *array = NULL;
+    json_t *entry;
+    double duration;
+
+    if (!(array = json_array ()))
+        goto nomem;
+    foreach_overlay_child (ov, child) {
+        duration = monotime_since (child->status_timestamp) / 1000.0;
+        if (!(entry = json_pack ("{s:i s:s s:f}",
+                                 "rank", child->rank,
+                                 "status", subtree_status_str (child->status),
+                                 "duration", duration)))
+            goto nomem;
+        if (!subtree_is_online (child->status) && child->error.text[0]) {
+            json_t *o;
+            if (!(o = json_string (child->error.text))
+                || json_object_set_new (entry, "error", o) < 0) {
+                json_decref (o);
+                // if this fails (unlikely), soldier on
+            }
+        }
+        if (json_array_append_new (array, entry) < 0) {
+            json_decref (entry);
+            goto nomem;
+        }
+    }
+    duration = monotime_since (ov->status_timestamp) / 1000.0;
+    if (flux_respond_pack (ov->h,
+                           msg,
+                           "{s:i s:s s:f s:O}",
+                           "rank", ov->rank,
+                           "status", subtree_status_str (ov->status),
+                           "duration", duration,
+                           "children", array) < 0)
+        flux_log_error (ov->h, "error responding to overlay.health");
+    json_decref (array);
+    return 0;
+nomem:
+    errno = ENOMEM;
+    return -1;
+}
+
+static void overlay_health_respond_all (struct overlay *ov)
+{
+    const flux_msg_t *msg;
+
+    msg = flux_msglist_first (ov->health_requests);
+    while (msg) {
+        if (overlay_health_respond (ov, msg) < 0)
+            flux_log_error (ov->h, "error responding to overlay.health");
+        msg = flux_msglist_next (ov->health_requests);
+    }
+}
+
+static void overlay_health_cb (flux_t *h,
+                               flux_msg_handler_t *mh,
+                               const flux_msg_t *msg,
+                               void *arg)
+{
+    struct overlay *ov = arg;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (flux_msg_is_streaming (msg)) {
+        if (flux_msglist_append (ov->health_requests, msg) < 0)
+            goto error;
+    }
+    if (overlay_health_respond (ov, msg) < 0)
+        goto error;
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to overlay.health");
+}
+
+/* If a disconnect is received for waiting health request,
+ * drop the request.
+ */
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct overlay *ov = arg;
+    int count;
+
+    if ((count = flux_msglist_disconnect (ov->health_requests, msg)) < 0)
+        flux_log_error (h, "error handling overlay.disconnect");
+    if (count > 0)
+        flux_log (h, LOG_DEBUG, "overlay: goodbye to %d health clients", count);
+    if ((count = flux_msglist_disconnect (ov->trace_requests, msg)) < 0)
+        flux_log_error (h, "error handling overlay.disconnect");
+    if (count > 0)
+        flux_log (h, LOG_DEBUG, "overlay: goodbye to %d trace clients", count);
+}
+
+const char *overlay_get_subtree_status (struct overlay *ov, int rank)
+{
+    const char *result = "unknown";
+
+    if (rank == ov->rank)
+        result = subtree_status_str (ov->status);
+    else {
+        struct child *child;
+        if ((child = child_lookup_byrank (ov, rank)))
+            result = subtree_status_str (child->status);
+    }
+    return result;
+}
+
+struct idset *overlay_get_default_critical_ranks (struct overlay *ov)
+{
+    struct idset *ranks;
+
+    /* For now, return all internal ranks plus rank 0
+     */
+    if (!(ranks = topology_get_internal_ranks (ov->topo))
+        || idset_set (ranks, 0) < 0) {
+        idset_destroy (ranks);
+        return NULL;
+    }
+    return ranks;
+}
+
+/* Recursive function to build subtree topology object.
+ * Right now the tree is regular.  In the future support the configuration
+ * of irregular tree topologies.
+ */
+json_t *overlay_get_subtree_topo (struct overlay *ov, int rank)
+{
+    if (!ov) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return topology_get_json_subtree_at (ov->topo, rank);
+}
+
+/* Get the topology of the subtree rooted here.
+ */
+static void overlay_topology_cb (flux_t *h,
+                                 flux_msg_handler_t *mh,
+                                 const flux_msg_t *msg,
+                                 void *arg)
+{
+    struct overlay *ov = arg;
+    json_t *topo = NULL;
+    int rank;
+    const char *errstr = NULL;
+
+    if (flux_request_unpack (msg, NULL, "{s:i}", "rank", &rank) < 0)
+        goto error;
+    if (rank != ov->rank && !child_lookup_byrank (ov, rank)) {
+        errstr = "requested rank is not this broker or its direct child";
+        errno = ENOENT;
+        goto error;
+    }
+    if (!(topo = overlay_get_subtree_topo (ov, rank)))
+        goto error;
+    if (flux_respond_pack (h, msg, "O", topo) < 0)
+        flux_log_error (h, "error responding to overlay.topology");
+    json_decref (topo);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to overlay.topology");
+    json_decref (topo);
+}
+
+/* Administratively force a disconnect of subtree rooted at 'rank'.
+ */
+static void overlay_disconnect_subtree_cb (flux_t *h,
+                                           flux_msg_handler_t *mh,
+                                           const flux_msg_t *msg,
+                                           void *arg)
+{
+    struct overlay *ov = arg;
+    const char *errstr = NULL;
+    char errbuf[128];
+    int rank;
+    struct child *child;
+
+    if (flux_request_unpack (msg, NULL, "{s:i}", "rank", &rank) < 0)
+        goto error;
+    if (!(child = child_lookup_byrank (ov, rank))) {
+        errstr = "requested rank is not this broker's direct child";
+        errno = ENOENT;
+        goto error;
+    }
+    if (!subtree_is_online (child->status)) {
+        snprintf (errbuf, sizeof (errbuf), "rank %d is already %s", rank,
+                  subtree_status_str (child->status));
+        errstr = errbuf;
+        errno = EINVAL;
+        goto error;
+    }
+    if (overlay_control_child (ov,
+                               child->uuid,
+                               CONTROL_DISCONNECT, 0) < 0) {
+        errstr = "failed to send CONTROL_DISCONNECT message";
+        goto error;
+    }
+    log_lost_connection (ov, child, "disconnected by request");
+    overlay_child_status_update (ov,
+                                 child,
+                                 SUBTREE_STATUS_LOST,
+                                 "administrative disconnect");
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to overlay.disconnect-subtree");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to overlay.disconnect-subtree");
+}
+
+static void overlay_trace_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct overlay *ov = arg;
+    struct flux_match match = FLUX_MATCH_ANY;
+    int nodeid;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s:s s:i}",
+                             "typemask", &match.typemask,
+                             "topic_glob", &match.topic_glob,
+                             "nodeid", &nodeid) < 0)
+        goto error;
+    if (!flux_msg_is_streaming (msg)) {
+        errno = EPROTO;
+        goto error;
+    }
+    if (flux_msglist_append (ov->trace_requests, msg) < 0)
+        goto error;
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to overlay.trace");
+}
+
+int overlay_cert_load (struct overlay *ov, const char *path)
+{
+    struct stat sb;
+    int fd;
+    FILE *f = NULL;
+    struct cert *cert;
+
+    if ((fd = open (path, O_RDONLY)) < 0
+        || fstat (fd, &sb) < 0) {
+        goto error;
+    }
+    if ((sb.st_mode & S_IROTH) | (sb.st_mode & S_IRGRP)) {
+        log_msg ("%s: readable by group/other", path);
+        errno = EPERM;
+        goto error_quiet;
+    }
+    if (!(f = fdopen (fd, "r")))
+        goto error;
+    fd = -1; // now owned by 'f'
+    if (!(cert = cert_read (f)))
+        goto error;
+    cert_destroy (ov->cert); // replace ov->cert (if any) with this
+    ov->cert = cert;
+    (void)fclose (f);
+    return 0;
+error:
+    log_err ("%s", path);
+error_quiet:
+    if (fd >= 0)
+        (void)close (fd);
+    if (f)
+        (void)fclose (f);
+    return -1;
+}
+
+const char *overlay_cert_pubkey (struct overlay *ov)
+{
+    return cert_public_txt (ov->cert);
+}
+
+const char *overlay_cert_name (struct overlay *ov)
+{
+    return cert_meta_get (ov->cert, "name");
+}
+
+int overlay_authorize (struct overlay *ov,
+                       const char *name,
+                       const char *pubkey)
+{
+    if (!ov->zap) {
+        errno = EINVAL;
+        return -1;
+    }
+    return zmqutil_zap_authorize (ov->zap, name, pubkey);
+}
+
+static int overlay_configure_attr (attr_t *attrs,
+                                   const char *name,
+                                   const char *default_value,
+                                   const char **valuep)
+{
+    const char *val = default_value;
+    int flags;
+
+    if (attr_get (attrs, name, &val, &flags) == 0) {
+        if (!(flags & ATTR_IMMUTABLE)) {
+            flags |= ATTR_IMMUTABLE;
+            if (attr_set_flags (attrs, name, flags) < 0)
+                return -1;
+        }
+    }
+    else {
+        val = default_value;
+        if (attr_add (attrs, name, val, ATTR_IMMUTABLE) < 0)
+            return -1;
+    }
+    if (valuep)
+        *valuep = val;
+    return 0;
+}
+
+static int overlay_configure_attr_int (attr_t *attrs,
+                                       const char *name,
+                                       int default_value,
+                                       int *valuep)
+{
+    int value = default_value;
+    const char *val;
+    char *endptr;
+
+    if (attr_get (attrs, name, &val, NULL) == 0) {
+        errno = 0;
+        value = strtol (val, &endptr, 10);
+        if (errno != 0 || *endptr != '\0') {
+            log_msg ("%s value must be an integer", name);
+            errno = EINVAL;
+            return -1;
+        }
+        if (attr_delete (attrs, name, true) < 0)
+            return -1;
+    }
+    if (attr_add_int (attrs, name, value, ATTR_IMMUTABLE) < 0)
+        return -1;
+    if (valuep)
+        *valuep = value;
+    return 0;
+}
+
+static int set_torpid (const char *name, const char *val, void *arg)
+{
+    struct overlay *ov = arg;
+    double d;
+
+    if (fsd_parse_duration (val, &d) < 0)
+        return -1;
+    if (streq (name, "tbon.torpid_max"))
+        ov->torpid_max = d;
+    else if (streq (name, "tbon.torpid_min")) {
+        if (d == 0)
+            goto error;
+        ov->torpid_min = d;
+    }
+    else
+        goto error;
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+static int get_torpid (const char *name, const char **val, void *arg)
+{
+    struct overlay *ov = arg;
+    static char buf[64];
+    double d;
+
+    if (streq (name, "tbon.torpid_max"))
+        d = ov->torpid_max;
+    else if (streq (name, "tbon.torpid_min"))
+        d = ov->torpid_min;
+    else
+        goto error;
+    if (fsd_format_duration (buf, sizeof (buf), d) < 0)
+        return -1;
+    *val = buf;
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+/* This is called from boot_*.c with a default value to set if the attribute
+ * is not set already.
+ */
+int overlay_set_tbon_interface_hint (struct overlay *ov, const char *val)
+{
+    if (attr_get (ov->attrs, "tbon.interface-hint", NULL, NULL) == 0)
+        return 0;
+    if (!val)
+        val = default_interface_hint;
+    return attr_add (ov->attrs, "tbon.interface-hint", val, 0);
+}
+
+/* Set attribute with the following precedence:
+ * 1. broker attribute
+ * 2. TOML config
+ * 3. legacy environment variables
+ * Leave it unset if none of those are available.
+ * The bootstrap methods set it later, but only if not already set.
+ */
+static int overlay_configure_interface_hint (struct overlay *ov,
+                                             const char *table,
+                                             const char *name)
+{
+    char long_name[128];
+    const char *val = NULL;
+    const char *config_val = NULL;
+    const char *attr_val = NULL;
+    int flags;
+    const flux_conf_t *cf;
+    flux_error_t error;
+
+    if ((cf = flux_get_conf (ov->h))) {
+        if (flux_conf_unpack (cf,
+                              &error,
+                              "{s?{s?s}}",
+                              table,
+                                name, &config_val) < 0) {
+            log_msg ("Config file error [%s]: %s", table, error.text);
+            return -1;
+        }
+    }
+    (void)snprintf (long_name, sizeof (long_name), "%s.%s", table, name);
+    (void)attr_get (ov->attrs, long_name, &attr_val, &flags);
+
+    if (attr_val)
+        val = attr_val;
+    else if (config_val)
+        val = config_val;
+    else if ((val = getenv ("FLUX_IPADDR_INTERFACE")))
+        ;
+    else if (getenv ("FLUX_IPADDR_HOSTNAME"))
+        val = "hostname";
+
+    if (val && !attr_val) {
+         if (attr_add (ov->attrs, long_name, val, 0) < 0) {
+            log_err ("Error setting %s attribute value", long_name);
+            return -1;
+         }
+    }
+    return 0;
+}
+
+static int overlay_configure_torpid (struct overlay *ov)
+{
+    const flux_conf_t *cf;
+
+    /* Start with compiled in defaults.
+     */
+    ov->torpid_min = default_torpid_min;
+    ov->torpid_max = default_torpid_max;
+
+    /* Override with config file settings, if any.
+     */
+    if ((cf = flux_get_conf (ov->h))) {
+        flux_error_t error;
+        const char *min_fsd = NULL;
+        const char *max_fsd = NULL;
+
+        if (flux_conf_unpack (flux_get_conf (ov->h),
+                              &error,
+                              "{s?{s?s s?s}}",
+                              "tbon",
+                                "torpid_min", &min_fsd,
+                                "torpid_max", &max_fsd) < 0) {
+            log_msg ("Config file error [tbon]: %s", error.text);
+            return -1;
+        }
+        if (min_fsd) {
+            if (fsd_parse_duration (min_fsd, &ov->torpid_min) < 0
+                || ov->torpid_min == 0) {
+                log_msg ("Config file error parsing tbon.torpid_min value");
+                return -1;
+            }
+        }
+        if (max_fsd) {
+            if (fsd_parse_duration (max_fsd, &ov->torpid_max) < 0) {
+                log_msg ("Config file error parsing tbon.torpid_max value");
+                return -1;
+            }
+        }
+    }
+
+    /* Override with broker attribute (command line/runtime) settings, if any.
+     */
+    if (attr_add_active (ov->attrs,
+                         "tbon.torpid_max",
+                         0,
+                         get_torpid,
+                         set_torpid,
+                         ov) < 0)
+        return -1;
+    if (attr_add_active (ov->attrs,
+                         "tbon.torpid_min",
+                         0,
+                         get_torpid,
+                         set_torpid,
+                         ov) < 0)
+        return -1;
+
+    return 0;
+}
+
+static int overlay_configure_timeout (struct overlay *ov,
+                                      const char *table,
+                                      const char *name,
+                                      bool enabled,
+                                      double default_value,
+                                      double *valuep)
+{
+    const flux_conf_t *cf;
+    const char *fsd = NULL;
+    bool override = false;
+    double value = default_value;
+    char long_name[128];
+
+    (void)snprintf (long_name, sizeof (long_name), "%s.%s", table, name);
+
+    if ((cf = flux_get_conf (ov->h))) {
+        flux_error_t error;
+
+        if (flux_conf_unpack (cf, &error, "{s?{s?s}}", table, name, &fsd) < 0) {
+            log_msg ("Config file error [%s]: %s", table, error.text);
+            return -1;
+        }
+        if (fsd) {
+            if (fsd_parse_duration (fsd, &value) < 0) {
+                log_msg ("Config file error parsing %s", long_name);
+                return -1;
+            }
+            override = true;
+        }
+    }
+    /* Override with broker attribute (command line only) settings, if any.
+     */
+    if (attr_get (ov->attrs, long_name, &fsd, NULL) == 0) {
+        if (fsd_parse_duration (fsd, &value) < 0) {
+            log_msg ("Error parsing %s attribute", long_name);
+            return -1;
+        }
+        if (attr_delete (ov->attrs, long_name, true) < 0)
+            return -1;
+        override = true;
+    }
+    if (enabled) {
+        char buf[64];
+        if (fsd_format_duration (buf, sizeof (buf), value) < 0)
+            return -1;
+        if (attr_add (ov->attrs, long_name, buf, ATTR_IMMUTABLE) < 0)
+            return -1;
+    }
+    else {
+        if (override) {
+            log_msg ("%s unsupported by this zeromq version", long_name);
+            return -1;
+        }
+    }
+    *valuep = value;
+    return 0;
+}
+
+static int overlay_configure_tbon_int (struct overlay *ov,
+                                       const char *name,
+                                       int *value,
+                                       int default_value)
+{
+    const flux_conf_t *cf;
+    char attrname[128];
+
+    *value = default_value;
+    if ((cf = flux_get_conf (ov->h))) {
+        flux_error_t error;
+
+        if (flux_conf_unpack (cf,
+                              &error,
+                              "{s?{s?i}}",
+                              "tbon",
+                                name, value) < 0) {
+            log_msg ("Config file error [tbon]: %s", error.text);
+            return -1;
+        }
+    }
+    (void)snprintf (attrname, sizeof (attrname), "tbon.%s", name);
+    if (overlay_configure_attr_int (ov->attrs, attrname, *value, value) < 0)
+        return -1;
+    return 0;
+}
+
+/* Configure tbon.topo attribute.
+ * Ascending precedence: compiled-in default, TOML config, command line.
+ * Topology creation is deferred to bootstrap, when we know the instance size.
+ */
+static int overlay_configure_topo (struct overlay *ov)
+{
+    const char *topo_uri = "kary:32";
+    const flux_conf_t *cf;
+
+    if ((cf = flux_get_conf (ov->h))) {
+        flux_error_t error;
+
+        if (flux_conf_unpack (cf, NULL, "{s:{}}", "bootstrap") == 0)
+            topo_uri = "custom"; // adjust default for config boot
+
+        if (flux_conf_unpack (cf,
+                              &error,
+                              "{s?{s?s}}",
+                              "tbon",
+                                "topo", &topo_uri) < 0) {
+            log_msg ("Config file error [tbon]: %s", error.text);
+            return -1;
+        }
+    }
+    /* Treat tbon.fanout=K as an alias for tbon.topo=kary:K.
+     */
+    const char *fanout;
+    char buf[16];
+    if (attr_get (ov->attrs, "tbon.fanout", &fanout, NULL) == 0) {
+        snprintf (buf, sizeof (buf), "kary:%s", fanout);
+        topo_uri = buf;
+    }
+    if (overlay_configure_attr (ov->attrs,
+                                "tbon.topo",
+                                topo_uri,
+                                NULL) < 0) {
+        log_err ("Error manipulating tbon.topo attribute");
+        return -1;
+    }
+    return 0;
+}
+
+void overlay_destroy (struct overlay *ov)
+{
+    if (ov) {
+        int saved_errno = errno;
+
+        flux_msglist_destroy (ov->health_requests);
+
+        cert_destroy (ov->cert);
+        zmqutil_zap_destroy (ov->zap);
+
+        flux_future_destroy (ov->f_sync);
+        flux_msg_handler_delvec (ov->handlers);
+        ov->status = SUBTREE_STATUS_OFFLINE;
+        overlay_control_parent (ov, CONTROL_STATUS, ov->status);
+        flux_future_destroy (ov->parent.f_goodbye);
+
+        zmq_close (ov->parent.zsock);
+        free (ov->parent.uri);
+        flux_watcher_destroy (ov->parent.w);
+        free (ov->parent.pubkey);
+        zmqutil_monitor_destroy (ov->parent.monitor);
+
+        free (ov->bind_uri);
+        zmq_close (ov->bind_zsock);
+        flux_watcher_destroy (ov->bind_w);
+        zmqutil_monitor_destroy (ov->bind_monitor);
+
+        zhashx_destroy (&ov->child_hash);
+        if (ov->children) {
+            int i;
+            for (i = 0; i < ov->child_count; i++)
+                rpc_track_destroy (ov->children[i].tracker);
+            free (ov->children);
+        }
+        rpc_track_destroy (ov->parent.tracker);
+        if (ov->monitor_callbacks) {
+            struct montior *mon;
+
+            while ((mon = zlist_pop (ov->monitor_callbacks)))
+                free (mon);
+            zlist_destroy (&ov->monitor_callbacks);
+        }
+        flux_msglist_destroy (ov->trace_requests);
+        topology_decref (ov->topo);
+        if (!ov->zctx_external)
+            zmq_ctx_term (ov->zctx);
+        free (ov->hostname);
+        free (ov);
+        errno = saved_errno;
+    }
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "overlay.trace",
+        overlay_trace_cb,
+        0,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "overlay.stats-get",
+        overlay_stats_get_cb,
+        FLUX_ROLE_USER,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "overlay.health",
+        overlay_health_cb,
+        FLUX_ROLE_USER,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "overlay.disconnect", // clean up after 'flux overlay status --wait'
+        disconnect_cb,
+        FLUX_ROLE_USER,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "overlay.topology",
+        overlay_topology_cb,
+        FLUX_ROLE_USER,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "overlay.disconnect-subtree",   // handle 'flux overlay disconnect'
+        overlay_disconnect_subtree_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "overlay.goodbye",
+        overlay_goodbye_cb,
+        0,
+    },
+    {
+        FLUX_MSGTYPE_RESPONSE,
+        "overlay.goodbye",
+        overlay_goodbye_response_cb,
+        0,
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+struct overlay *overlay_create (flux_t *h,
+                                const char *hostname,
+                                attr_t *attrs,
+                                void *zctx,
+                                overlay_recv_f cb,
+                                void *arg)
+{
+    struct overlay *ov;
+    uuid_t uuid;
+
+    if (!(ov = calloc (1, sizeof (*ov))))
+        return NULL;
+    if (!(ov->hostname = strdup (hostname)))
+        goto error;
+    ov->attrs = attrs;
+    ov->rank = FLUX_NODEID_ANY;
+    ov->parent.lastsent = -1;
+    ov->h = h;
+    ov->reactor = flux_get_reactor (h);
+    ov->recv_cb = cb;
+    ov->recv_arg = arg;
+    ov->version = FLUX_CORE_VERSION_HEX;
+    uuid_generate (uuid);
+    uuid_unparse (uuid, ov->uuid);
+    if (zctx) {
+        ov->zctx = zctx;
+        ov->zctx_external = true;
+    }
+    if (!(ov->monitor_callbacks = zlist_new ()))
+        goto nomem;
+    if (overlay_configure_attr_int (ov->attrs, "tbon.prefertcp", 0, NULL) < 0)
+        goto error;
+    if (overlay_configure_interface_hint (ov, "tbon", "interface-hint") < 0)
+        goto error;
+    if (overlay_configure_torpid (ov) < 0)
+        goto error;
+    if (overlay_configure_timeout (ov,
+                                   "tbon",
+                                   "tcp_user_timeout",
+                                   have_tcp_maxrt,
+                                   default_tcp_user_timeout,
+                                   &ov->tcp_user_timeout) < 0)
+        goto error;
+    if (overlay_configure_timeout (ov,
+                                   "tbon",
+                                   "connect_timeout",
+                                   have_connect_timeout,
+                                   default_connect_timeout,
+                                   &ov->connect_timeout) < 0)
+        goto error;
+    if (overlay_configure_tbon_int (ov, "zmqdebug", &ov->zmqdebug, 0) < 0)
+        goto error;
+    if (overlay_configure_tbon_int (ov,
+                                    "child_rcvhwm",
+                                    &ov->child_rcvhwm,
+                                    0) < 0)
+        goto error;
+    if (ov->child_rcvhwm < 0 || ov->child_rcvhwm == 1) {
+        log_msg ("tbon.child_rcvhwm must be 0 (unlimited) or >= 2");
+        errno = EINVAL;
+        goto error;
+    }
+    if (overlay_configure_tbon_int (ov,
+                                    "zmq_io_threads",
+                                    &ov->zmq_io_threads,
+                                    1) < 0)
+        goto error;
+    if (ov->zmq_io_threads < 1) {
+        log_msg ("tbon.zmq_io_threads must be >= 1");
+        errno = EINVAL;
+        goto error;
+    }
+    if (overlay_configure_topo (ov) < 0)
+        goto error;
+    if (flux_msg_handler_addvec (h, htab, ov, &ov->handlers) < 0)
+        goto error;
+    if (!(ov->cert = cert_create ())) {
+        log_err ("could not create curve certificate");
+        goto error;
+    }
+    if (!(ov->health_requests = flux_msglist_create ())
+        || !(ov->trace_requests = flux_msglist_create ()))
+        goto error;
+    if (!(ov->parent.f_goodbye = flux_future_create (NULL, NULL)))
+        goto error;
+    flux_future_set_flux (ov->parent.f_goodbye, h);
+    return ov;
+nomem:
+    errno = ENOMEM;
+error:
+    overlay_destroy (ov);
+    return NULL;
 }
 
 /*
diff --git a/src/broker/overlay.h b/src/broker/overlay.h
index a60cba0150a4..dc1632664b28 100644
--- a/src/broker/overlay.h
+++ b/src/broker/overlay.h
@@ -11,93 +11,165 @@
 #ifndef _BROKER_OVERLAY_H
 #define _BROKER_OVERLAY_H
 
-#include "attr.h"
-#include "src/common/libutil/zsecurity.h"
+#include <jansson.h>
+#include <stdint.h>
 
-typedef struct overlay_struct overlay_t;
-typedef void (*overlay_cb_f)(overlay_t *ov, void *sock, void *arg);
-typedef int (*overlay_init_cb_f)(overlay_t *ov, void *arg);
-typedef void (*overlay_monitor_cb_f)(overlay_t *ov, void *arg);
+#include "attr.h"
+#include "topology.h"
 
-overlay_t *overlay_create (void);
-void overlay_destroy (overlay_t *ov);
+typedef enum {
+    OVERLAY_ANY = 0,
+    OVERLAY_UPSTREAM,
+    OVERLAY_DOWNSTREAM,
+} overlay_where_t;
 
-/* Set a callback triggered during overlay_init()
+/* Overlay control messages
  */
-void overlay_set_init_callback (overlay_t *ov,
-                                overlay_init_cb_f cb, void *arg);
-
-/* These need to be called before connect/bind.
+enum control_type {
+    CONTROL_HEARTBEAT = 0, // child sends when connection is idle
+    CONTROL_STATUS = 1,    // child tells parent of subtree status change
+    CONTROL_DISCONNECT = 2,// parent tells child to immediately disconnect
+};
+
+struct overlay;
+
+typedef void (*overlay_monitor_f)(struct overlay *ov, uint32_t rank, void *arg);
+typedef int (*overlay_recv_f)(flux_msg_t **msg,
+                              overlay_where_t from,
+                              void *arg);
+
+/* Create overlay network, registering 'cb' to be called with each
+ * received message.
+ * Note: If zctx is NULL, it is created/destroyed on demand internally.
  */
-int overlay_set_flux (overlay_t *ov, flux_t *h);
-int overlay_setup_sec (overlay_t *ov, int sec_typemask, const char *keydir);
-int overlay_init (overlay_t *ov, uint32_t size, uint32_t rank, int tbon_k);
-void overlay_set_idle_warning (overlay_t *ov, int heartbeats);
+struct overlay *overlay_create (flux_t *h,
+                                const char *hostname,
+                                attr_t *attrs,
+                                void *zctx,
+                                overlay_recv_f cb,
+                                void *arg);
+void overlay_destroy (struct overlay *ov);
+
+/* Start sending control messages to parent and monitoring peers.
+ * This registers a sync callback, and will fail if event.subscribe
+ * doesn't have a handler yet.
+ */
+int overlay_control_start (struct overlay *ov);
 
-/* Accessors
+/* Set the overlay topology.
  */
-uint32_t overlay_get_rank (overlay_t *ov);
-uint32_t overlay_get_size (overlay_t *ov);
-int overlay_get_child_peer_count (overlay_t *ov);
+int overlay_set_topology (struct overlay *ov, struct topology *topo);
 
-/* All ranks but rank 0 connect to a parent to form the main TBON.
+/* Send a message on the overlay network.
+ * 'where' determines whether the message is routed upstream or downstream.
+ */
+int overlay_sendmsg (struct overlay *ov,
+                     const flux_msg_t *msg,
+                     overlay_where_t where);
+/* same as above but steals reference to 'msg' on success.
+ */
+int overlay_sendmsg_new (struct overlay *ov,
+                         flux_msg_t **msg,
+                         overlay_where_t where);
+
+/* Each broker has a public, private CURVE key-pair.  Call overlay_authorize()
+ * with the public key of each downstream peer to authorize it to connect,
+ * and overlay_set_parent_pubkey() with the public key of the parent
+ * before calling overlay_connect().
+ * NOTE: if bootstrapping with PMI, unique public keys are generated for
+ * each broker and shared via PMI exchange.  If bootstrapping with config
+ * files, each broker loads an (assumed) identical key-pair from a file.
+ * Only the public key may be shared over the network, never the private key.
  */
-int overlay_set_parent (overlay_t *ov, const char *fmt, ...);
-const char *overlay_get_parent (overlay_t *ov);
-void overlay_set_parent_cb (overlay_t *ov, overlay_cb_f cb, void *arg);
-int overlay_sendmsg_parent (overlay_t *ov, const flux_msg_t *msg);
+int overlay_cert_load (struct overlay *ov, const char *path);
+const char *overlay_cert_pubkey (struct overlay *ov);
+const char *overlay_cert_name (struct overlay *ov);
+int overlay_authorize (struct overlay *ov,
+                       const char *name,
+                       const char *pubkey);
+int overlay_set_parent_pubkey (struct overlay *ov, const char *pubkey);
+
+/* Misc. accessors
+ */
+uint32_t overlay_get_rank (struct overlay *ov);
+void overlay_set_rank (struct overlay *ov, uint32_t rank); // test only
+uint32_t overlay_get_size (struct overlay *ov);
+int overlay_get_child_peer_count (struct overlay *ov);
+const char *overlay_get_bind_uri (struct overlay *ov);
+const char *overlay_get_parent_uri (struct overlay *ov);
+int overlay_set_parent_uri (struct overlay *ov, const char *uri);
+bool overlay_parent_error (struct overlay *ov);
+void overlay_set_version (struct overlay *ov, int version); // test only
+const char *overlay_get_uuid (struct overlay *ov);
+bool overlay_uuid_is_parent (struct overlay *ov, const char *uuid);
+bool overlay_uuid_is_child (struct overlay *ov, const char *uuid);
+void overlay_set_ipv6 (struct overlay *ov, int enable);
+int overlay_set_tbon_interface_hint (struct overlay *ov, const char *val);
+
+/* Return an idset of critical ranks, i.e. non-leaf brokers
+ */
+struct idset *overlay_get_default_critical_ranks (struct overlay *ov);
 
-/* The child is where other ranks connect to send requests.
- * This is the ROUTER side of parent sockets described above.
+/* Fetch TBON subtree topo at 'rank'.  The returned topology object has the
+ * following recursive structure, where "children" is an array of topology
+ * objects:
+ *
+ * {"rank":i, "size":i, "children":o}
+ *
+ * If rank has no children, the "children" array will be present but empty.
+ * Caller must release returned object with json_decref().
  */
-int overlay_set_child (overlay_t *ov, const char *fmt, ...);
-const char *overlay_get_child (overlay_t *ov);
-void overlay_set_child_cb (overlay_t *ov, overlay_cb_f cb, void *arg);
-int overlay_sendmsg_child (overlay_t *ov, const flux_msg_t *msg);
-/* We can "multicast" events to all child peers using mcast_child().
- * It walks the 'children' hash, finding peers and routeing them a copy of msg.
+json_t *overlay_get_subtree_topo (struct overlay *ov, int rank);
+
+/* Fetch status for TBON subtree rooted at 'rank'.  If 'rank' is not this
+ * broker's rank or one of its direct descendants, "unknown" is returned.
  */
-int overlay_mcast_child (overlay_t *ov, const flux_msg_t *msg);
+const char *overlay_get_subtree_status (struct overlay *ov, int rank);
 
-/* Call when message is received from child 'uuid'.
+/* A TBON child is "torpid" if no messages (including regular control messages)
+ * have been received from it for a while.
  */
-void overlay_checkin_child (overlay_t *ov, const char *uuid);
+bool overlay_peer_is_torpid (struct overlay *ov, uint32_t rank);
 
-/* Register callback that will be called each time a child connects/disconnects.
- * Use overlay_get_child_peer_count() to access the actual count.
+/* Broker should call overlay_bind() if there are children.  This may happen
+ * before any peers are authorized as long as they are authorized before they
+ * try to connect.
  */
-void overlay_set_monitor_cb (overlay_t *ov, overlay_monitor_cb_f cb, void *arg);
+int overlay_bind (struct overlay *ov, const char *uri);
 
-/* Encode cmb.lspeer response payload.
+/* Broker should call overlay_connect(), after overlay_set_parent_uri()
+ * and overlay_set_parent_pubkey(), if there is a parent.
+ */
+int overlay_connect (struct overlay *ov);
+
+/* Arrange for 'cb' to be called if:
+ * - error on TBON parent (rank = FLUX_NODEID_ANY)
+ * - a subtree rooted at rank (child) has changed status
+ * The following accessors may be useful in the callback:
+ * - overlay_parent_error() - test whether TBON parent connection has failed
+ * - overlay_get_child_peer_count() - number of online children
+ * - overlay_get_subtree_status (rank) - subtree status of child
+ * - overlay_get_subtree_topo (rank) - topology of subtree rooted at child
  */
-char *overlay_lspeer_encode (overlay_t *ov);
+int overlay_set_monitor_cb (struct overlay *ov,
+                            overlay_monitor_f cb,
+                            void *arg);
 
-/* Establish connections.
- * These functions are idempotent as the bind may need to be called
- * early to resolve wildcard addresses (e.g. during PMI endpoint exchange).
+/* Register overlay-related broker attributes.
  */
-int overlay_bind (overlay_t *ov);
-int overlay_connect (overlay_t *ov);
+int overlay_register_attrs (struct overlay *overlay);
 
-/* Switch parent DEALER socket to a new peer.  If the uri is already present
- * in the parent endpoint stack, reuse the existing socket ('recycled' set
- * to true).  The new parent is moved to the top of the parent stack.
+/* Stop allowing new connections from downstream peers.
+ * If unbind is false, stop all communication on the socket.
+ * Otherwise arrange to send a disconnect control message in response
+ * to all messages.
  */
-int overlay_reparent (overlay_t *ov, const char *uri, bool *recycled);
+void overlay_shutdown (struct overlay *overlay, bool unbind);
 
-/* Add attributes to 'attrs' to reveal information about the overlay network.
- * Active attrs:
- *   tbon.parent-endpoint
- * Passive attrs:
- *   rank
- *   size
- *   tbon.arity
- *   tbon.level
- *   tbon.maxlevel
- *   tbon.descendants
- * Returns 0 on success, -1 on error.
+/* Say goodbye to parent.
+ * After this, sends to parent are dropped.
  */
-int overlay_register_attrs (overlay_t *overlay, attr_t *attrs);
+flux_future_t *overlay_goodbye_parent (struct overlay *overlay);
 
 #endif /* !_BROKER_OVERLAY_H */
 
diff --git a/src/broker/ping.c b/src/broker/ping.c
deleted file mode 100644
index e17ab2b1e28d..000000000000
--- a/src/broker/ping.c
+++ /dev/null
@@ -1,150 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <jansson.h>
-#include <flux/core.h>
-
-#include "src/common/libutil/errno_safe.h"
-
-#include "ping.h"
-
-struct ping_context {
-    flux_msg_handler_t *mh;
-    char *uuid;
-};
-
-static char *make_json_response_payload (const char *request_payload,
-                                         const char *route,
-                                         struct flux_msg_cred cred)
-{
-    json_t *o = NULL;
-    json_t *add = NULL;
-    char *result = NULL;
-
-    if (!request_payload || !(o = json_loads (request_payload, 0, NULL))) {
-        errno = EPROTO;
-        goto done;
-    }
-    if (!(add = json_pack ("{s:s s:i s:i}", "route", route,
-                                            "userid", cred.userid,
-                                            "rolemask", cred.rolemask))) {
-        errno = ENOMEM;
-        goto done;
-    }
-    if (json_object_update (o, add) < 0) {
-        errno = ENOMEM;
-        goto done;
-    }
-    if (!(result = json_dumps (o, 0))) {
-        errno = ENOMEM;
-        goto done;
-    }
-done:
-    ERRNO_SAFE_WRAP (json_decref, o);
-    ERRNO_SAFE_WRAP (json_decref, add);
-    return result;
-}
-
-static void ping_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                             const flux_msg_t *msg, void *arg)
-{
-    struct ping_context *p = arg;
-    const char *json_str;
-    char *route_str = NULL;
-    char *new_str;
-    size_t new_size;
-    char *resp_str = NULL;
-    struct flux_msg_cred cred;
-
-    if (flux_request_decode (msg, NULL, &json_str) < 0)
-        goto error;
-    if (flux_msg_get_cred (msg, &cred) < 0)
-        goto error;
-
-    /* The route string as obtained from the message includes all
-     * hops but the last one, e.g. the identity of the destination.
-     * That identity is passed in to ping_initialize() as the uuid.
-     * Tack it onto the route string.
-     */
-    if (!(route_str = flux_msg_get_route_string (msg)))
-        goto error;
-    new_size = strlen (route_str) + strlen (p->uuid) + 2;
-    if (!(new_str = realloc (route_str, new_size)))
-        goto error;
-    route_str = new_str;
-    strcat (route_str, "!");
-    strcat (route_str, p->uuid);
-
-    if (!(resp_str = make_json_response_payload (json_str, route_str, cred)))
-        goto error;
-    if (flux_respond (h, msg, resp_str) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    free (route_str);
-    free (resp_str);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    free (route_str);
-    free (resp_str);
-}
-
-static void ping_finalize (void *arg)
-{
-    struct ping_context *p = arg;
-    if (p) {
-        int saved_errno = errno;
-        flux_msg_handler_stop (p->mh);
-        flux_msg_handler_destroy (p->mh);
-        free (p->uuid);
-        free (p);
-        errno = saved_errno;
-    }
-}
-
-int ping_initialize (flux_t *h, const char *service, const char *uuid)
-{
-    struct flux_match match = FLUX_MATCH_ANY;
-    struct ping_context *p = calloc (1, sizeof (*p));
-    if (!p)
-        goto error;
-    /* The uuid is tacked onto the route string constructed for
-     * ping responses.  If it contains a hyphen, truncate it there
-     * to match policy of flux_msg_get_route_str().  If it doesn't
-     * contain a hyphen, it's probably a broker rank - never truncate that.
-     */
-    if (!(p->uuid = strdup (uuid)))
-        goto error;
-    char *cp = strchr (p->uuid, '-');
-    if (cp)
-        *cp = '\0';
-
-    match.typemask = FLUX_MSGTYPE_REQUEST;
-    if (flux_match_asprintf (&match, "%s.ping", service) < 0)
-        goto error;
-    if (!(p->mh = flux_msg_handler_create (h, match, ping_request_cb, p)))
-        goto error;
-    flux_msg_handler_allow_rolemask (p->mh, FLUX_ROLE_ALL);
-    flux_msg_handler_start (p->mh);
-    flux_aux_set (h, "flux::ping", p, ping_finalize);
-    flux_match_free (match);
-    return 0;
-error:
-    flux_match_free (match);
-    ping_finalize (p);
-    return -1;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/ping.h b/src/broker/ping.h
deleted file mode 100644
index 2e0f95ca789f..000000000000
--- a/src/broker/ping.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef BROKER_PING_H
-#define BROKER_PING_H
-
-#include <flux/core.h>
-
-int ping_initialize (flux_t *h, const char *service, const char *uuid);
-
-#endif /* BROKER_PING_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/pmiutil.c b/src/broker/pmiutil.c
deleted file mode 100644
index 965b2bbf2101..000000000000
--- a/src/broker/pmiutil.c
+++ /dev/null
@@ -1,270 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <sys/param.h>
-#include <unistd.h>
-#include <dlfcn.h>
-#include <assert.h>
-#include <czmq.h>
-
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/iterators.h"
-#include "src/common/libpmi/pmi.h"
-#include "src/common/libpmi/pmi_strerror.h"
-#include "src/common/libpmi/simple_client.h"
-
-#include "pmiutil.h"
-#include "liblist.h"
-
-struct pmi_dso {
-    void *dso;
-    int (*init) (int *spawned);
-    int (*finalize) (void);
-    int (*get_size) (int *size);
-    int (*get_rank) (int *rank);
-    int (*barrier) (void);
-    int (*kvs_get_my_name) (char *kvsname, int length);
-    int (*kvs_put) (const char *kvsname, const char *key, const char *value);
-    int (*kvs_commit) (const char *kvsname);
-    int (*kvs_get) (const char *kvsname, const char *key, char *value, int len);
-};
-
-struct pmi_handle {
-    struct pmi_dso *dso;
-    struct pmi_simple_client *cli;
-    int debug;
-};
-
-static void broker_pmi_dlclose (struct pmi_dso *dso)
-{
-    if (dso) {
-#ifndef __SANITIZE_ADDRESS__
-        if (dso->dso)
-            dlclose (dso->dso);
-#endif
-        free (dso);
-    }
-}
-
-/* Notes:
- * - Use RTLD_GLOBAL due to issue #432
- */
-static struct pmi_dso *broker_pmi_dlopen (const char *pmi_library, int debug)
-{
-    struct pmi_dso *dso;
-    zlist_t *libs = NULL;
-    char *name;
-
-    if (!(dso = calloc (1, sizeof (*dso))))
-        return NULL;
-    if (!pmi_library)
-        pmi_library = "libpmi.so";
-    if (!(libs = liblist_create (pmi_library)))
-        goto error;
-    FOREACH_ZLIST (libs, name) {
-        dlerror ();
-        if (!(dso->dso = dlopen (name, RTLD_NOW | RTLD_GLOBAL))) {
-            if (debug) {
-                char *errstr = dlerror ();
-                if (errstr)
-                    log_msg ("%s", errstr);
-                else
-                    log_msg ("dlopen %s failed", name);
-            }
-        }
-        else if (dlsym (dso->dso, "flux_pmi_library")) {
-            if (debug)
-                log_msg ("skipping %s", name);
-            dlclose (dso->dso);
-            dso->dso = NULL;
-        }
-        else {
-            if (debug)
-                log_msg ("dlopen %s", name);
-        }
-    }
-    liblist_destroy (libs);
-    libs = NULL;
-    if (!dso->dso)
-        goto error;
-    dso->init = dlsym (dso->dso, "PMI_Init");
-    dso->finalize = dlsym (dso->dso, "PMI_Finalize");
-    dso->get_size = dlsym (dso->dso, "PMI_Get_size");
-    dso->get_rank = dlsym (dso->dso, "PMI_Get_rank");
-    dso->barrier = dlsym (dso->dso, "PMI_Barrier");
-    dso->kvs_get_my_name = dlsym (dso->dso, "PMI_KVS_Get_my_name");
-    dso->kvs_put = dlsym (dso->dso, "PMI_KVS_Put");
-    dso->kvs_commit = dlsym (dso->dso, "PMI_KVS_Commit");
-    dso->kvs_get = dlsym (dso->dso, "PMI_KVS_Get");
-
-    if (!dso->init || !dso->finalize || !dso->get_size || !dso->get_rank
-            || !dso->barrier || !dso->kvs_get_my_name
-            || !dso->kvs_put || !dso->kvs_commit || !dso->kvs_get) {
-        log_msg ("dlsym: %s is missing required symbols", pmi_library);
-        goto error;
-    }
-    return dso;
-error:
-    broker_pmi_dlclose (dso);
-    if (libs)
-        liblist_destroy (libs);
-    return NULL;
-}
-
-int broker_pmi_kvs_commit (struct pmi_handle *pmi, const char *kvsname)
-{
-    if (pmi->cli)
-        return PMI_SUCCESS;
-    if (pmi->dso)
-        return pmi->dso->kvs_commit (kvsname);
-    return PMI_SUCCESS;
-}
-
-int broker_pmi_kvs_put (struct pmi_handle *pmi,
-                        const char *kvsname,
-                        const char *key,
-                        const char *value)
-{
-    if (pmi->cli)
-        return pmi_simple_client_kvs_put (pmi->cli, kvsname, key, value);
-    if (pmi->dso)
-        return pmi->dso->kvs_put (kvsname, key, value);
-    return PMI_SUCCESS;
-}
-
-int broker_pmi_kvs_get (struct pmi_handle *pmi,
-                               const char *kvsname,
-                               const char *key,
-                               char *value,
-                               int len)
-{
-    if (pmi->cli)
-        return pmi_simple_client_kvs_get (pmi->cli, kvsname, key, value, len);
-    if (pmi->dso)
-        return pmi->dso->kvs_get (kvsname, key, value, len);
-    return PMI_FAIL;
-}
-
-int broker_pmi_barrier (struct pmi_handle *pmi)
-{
-    if (pmi->cli)
-        return pmi_simple_client_barrier (pmi->cli);
-    if (pmi->dso)
-        return pmi->dso->barrier();
-    return PMI_SUCCESS;
-}
-
-int broker_pmi_get_params (struct pmi_handle *pmi,
-                           struct pmi_params *params)
-{
-    int result;
-
-    if (pmi->cli) {
-        params->rank = pmi->cli->rank;
-        params->size = pmi->cli->size;
-        result = pmi_simple_client_kvs_get_my_name (pmi->cli,
-                                                    params->kvsname,
-                                                    sizeof (params->kvsname));
-        if (result != PMI_SUCCESS)
-            goto error;
-    }
-    else if (pmi->dso) {
-        result = pmi->dso->get_rank (&params->rank);
-        if (result != PMI_SUCCESS)
-            goto error;
-        result = pmi->dso->get_size (&params->size);
-        if (result != PMI_SUCCESS)
-            goto error;
-        result = pmi->dso->kvs_get_my_name (params->kvsname,
-                                            sizeof (params->kvsname));
-        if (result != PMI_SUCCESS)
-            goto error;
-    }
-    else {
-        params->rank = 0;
-        params->size = 1;
-        snprintf (params->kvsname, sizeof (params->kvsname), "singleton");
-    }
-
-    return PMI_SUCCESS;
-error:
-    return result;
-}
-
-int broker_pmi_init (struct pmi_handle *pmi)
-{
-    int spawned;
-
-    if (pmi->cli)
-        return pmi_simple_client_init (pmi->cli);
-    if (pmi->dso)
-        return pmi->dso->init(&spawned);
-    return PMI_SUCCESS;
-}
-
-int broker_pmi_finalize (struct pmi_handle *pmi)
-{
-    if (pmi->cli)
-        return pmi_simple_client_finalize (pmi->cli);
-    if (pmi->dso)
-        return pmi->dso->finalize ();
-    return PMI_SUCCESS;
-}
-
-void broker_pmi_destroy (struct pmi_handle *pmi)
-{
-    if (pmi) {
-        int saved_errno = errno;
-        if (pmi->cli)
-            pmi_simple_client_destroy (pmi->cli);
-        else if (pmi->dso)
-            broker_pmi_dlclose (pmi->dso);
-        free (pmi);
-        errno = saved_errno;
-    }
-}
-
-/* Attempt to set up PMI-1 wire protocol client.
- * If that fails, try dlopen.
- * If that fails, singleton will be used.
- */
-struct pmi_handle *broker_pmi_create (void)
-{
-    const char *pmi_debug;
-    struct pmi_handle *pmi = calloc (1, sizeof (*pmi));
-    if (!pmi)
-        return NULL;
-    pmi_debug = getenv ("FLUX_PMI_DEBUG");
-    if (pmi_debug)
-        pmi->debug = strtol (pmi_debug, NULL, 10);
-    pmi->cli = pmi_simple_client_create_fd (getenv ("PMI_FD"),
-                                            getenv ("PMI_RANK"),
-                                            getenv ("PMI_SIZE"),
-                                            NULL);
-    /* N.B. SLURM boldly installs its libpmi.so into the system libdir,
-     * so it will be found here, even if not running in a SLURM job.
-     * Fortunately it emulates singleton in that case, in lieu of failing.
-     */
-    if (!pmi->cli)
-        pmi->dso = broker_pmi_dlopen (getenv ("PMI_LIBRARY"), pmi->debug);
-    /* If neither pmi->cli nor pmi->dso is set, singleton is assumed later.
-     */
-    if (pmi->debug)
-        log_msg ("using %s", pmi->cli ? "PMI-1 wire protocol"
-                           : pmi->dso ? "dlopen" : "singleton");
-    return pmi;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/pmiutil.h b/src/broker/pmiutil.h
deleted file mode 100644
index d9cffe781cbf..000000000000
--- a/src/broker/pmiutil.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef HAVE_BROKER_PMIUTIL_H
-#define HAVE_BROKER_PMIUTIL_H 1
-
-struct pmi_params {
-    int rank;
-    int size;
-    char kvsname[1024];
-};
-
-struct pmi_handle;
-
-int broker_pmi_kvs_commit (struct pmi_handle *pmi, const char *kvsname);
-
-int broker_pmi_kvs_put (struct pmi_handle *pmi,
-                        const char *kvsname,
-                        const char *key,
-                        const char *value);
-
-int broker_pmi_kvs_get (struct pmi_handle *pmi,
-                        const char *kvsname,
-                        const char *key,
-                        char *value,
-                        int len);
-
-int broker_pmi_barrier (struct pmi_handle *pmi);
-
-int broker_pmi_get_params (struct pmi_handle *pmi, struct pmi_params *params);
-
-int broker_pmi_init (struct pmi_handle *pmi);
-
-int broker_pmi_finalize (struct pmi_handle *pmi);
-
-void broker_pmi_destroy (struct pmi_handle *pmi);
-
-struct pmi_handle *broker_pmi_create (void);
-
-#endif /* !HAVE_BROKER_PMIUTIL_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/publisher.c b/src/broker/publisher.c
index 989e34e2a4f5..11d4e8b3a7c5 100644
--- a/src/broker/publisher.c
+++ b/src/broker/publisher.c
@@ -15,64 +15,31 @@
 #endif
 #include <errno.h>
 #include <flux/core.h>
-#include <czmq.h>
-#include <sodium.h>
 
-#include "src/common/libutil/macros.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libccan/ccan/base64/base64.h"
+#include "src/common/librouter/subhash.h"
+#include "ccan/str/str.h"
 
+#include "modhash.h"
 #include "publisher.h"
 
-struct sender {
-    publisher_send_f send;
-    void *arg;
-    char name[32];
-};
 
 struct publisher {
-    flux_t *h;
+    struct broker *ctx;
     flux_msg_handler_t **handlers;
     int seq;
     zlist_t *senders;
+    publisher_send_f send;
+    void *arg;
 };
 
-void publisher_destroy (struct publisher *pub)
-{
-    if (pub) {
-        int saved_errno = errno;
-        flux_msg_handler_delvec (pub->handlers);
-        if (pub->senders) {
-            struct sender *sender;
-            while ((sender = zlist_pop (pub->senders)))
-                free (sender);
-            zlist_destroy (&pub->senders);
-        }
-        free (pub);
-        errno = saved_errno;
-    }
-}
-
-struct publisher *publisher_create (void)
-{
-    struct publisher *pub = calloc (1, sizeof (*pub));
-
-    if (!pub) {
-        errno = ENOMEM;
-        return NULL;
-    }
-    if (!(pub->senders = zlist_new ())) {
-        publisher_destroy (pub);
-        errno = ENOMEM;
-        return NULL;
-    }
-    return pub;
-}
-
 static flux_msg_t *encode_event (const char *topic, int flags,
                                  struct flux_msg_cred cred,
                                  uint32_t seq, const char *src)
 {
     flux_msg_t *msg;
-    void *dst = NULL;
+    char *dst = NULL;
     int saved_errno;
 
     if (!(msg = flux_msg_create (FLUX_MSGTYPE_EVENT)))
@@ -89,13 +56,12 @@ static flux_msg_t *encode_event (const char *topic, int flags,
     }
     if (src) { // optional payload
         int srclen = strlen (src);
-        size_t dstlen = BASE64_DECODE_SIZE (srclen);
+        size_t dstbuflen = base64_decoded_length (srclen);
+        ssize_t dstlen;
 
-        if (!(dst = malloc (dstlen)))
+        if (!(dst = malloc (dstbuflen)))
             goto error;
-        if (sodium_base642bin ((unsigned char *)dst, dstlen, src, srclen,
-                               NULL, &dstlen, NULL,
-                               sodium_base64_VARIANT_ORIGINAL) < 0) {
+        if ((dstlen = base64_decode (dst, dstbuflen, src, srclen)) < 0) {
             errno = EPROTO;
             goto error;
         }
@@ -120,19 +86,14 @@ static flux_msg_t *encode_event (const char *topic, int flags,
  */
 static void send_event (struct publisher *pub, const flux_msg_t *msg)
 {
-    struct sender *sender;
-
-    sender = zlist_first (pub->senders);
-    while (sender != NULL) {
-        if (sender->send (sender->arg, msg) < 0)
-            flux_log_error (pub->h, "%s: sender=%s",
-                            __FUNCTION__, sender->name);
-        sender = zlist_next (pub->senders);
-    }
+    if (pub->send (pub->arg, msg) < 0)
+        flux_log_error (pub->ctx->h, "error publishing event message");
 }
 
-void pub_cb (flux_t *h, flux_msg_handler_t *mh,
-             const flux_msg_t *msg, void *arg)
+static void publish_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
 {
     struct publisher *pub = arg;
     const char *topic;
@@ -140,12 +101,18 @@ void pub_cb (flux_t *h, flux_msg_handler_t *mh,
     int flags;
     struct flux_msg_cred cred;
     flux_msg_t *event = NULL;
+    const char *errmsg = NULL;
 
-    if (flux_request_unpack (msg, NULL, "{s:s s:i s?:s}",
+    if (flux_request_unpack (msg, NULL, "{s:s s:i s?s}",
                                         "topic", &topic,
                                         "flags", &flags,
                                         "payload", &payload) < 0)
         goto error;
+    if (pub->ctx->rank > 0) {
+        errno = EPROTO;
+        errmsg = "this service is only available on rank 0";
+        goto error;
+    }
     if ((flags & ~(FLUX_MSGFLAG_PRIVATE)) != 0) {
         errno = EPROTO;
         goto error;
@@ -162,7 +129,7 @@ void pub_cb (flux_t *h, flux_msg_handler_t *mh,
 error_restore_seq:
     pub->seq--;
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
     flux_msg_destroy (event);
 }
@@ -173,8 +140,7 @@ int publisher_send (struct publisher *pub, const flux_msg_t *msg)
 
     if (!(cpy = flux_msg_copy (msg, true)))
         return -1;
-    if (flux_msg_clear_route (cpy) < 0)
-        goto error;
+    flux_msg_route_disable (cpy);
     if (flux_msg_set_seq (cpy, ++pub->seq) < 0)
         goto error_restore_seq;
     send_event (pub, cpy);
@@ -182,43 +148,104 @@ int publisher_send (struct publisher *pub, const flux_msg_t *msg)
     return 0;
 error_restore_seq:
     pub->seq--;
-error:
     flux_msg_destroy (cpy);
     return -1;
 }
 
+static void subscribe_cb (flux_t *h, flux_msg_handler_t *mh,
+                          const flux_msg_t *msg, void *arg)
+{
+    struct publisher *pub = arg;
+    const char *uuid;
+    const char *topic;
+
+    if (flux_request_unpack (msg, NULL, "{ s:s }", "topic", &topic) < 0)
+        goto error;
+    if ((uuid = flux_msg_route_first (msg))) {
+        module_t *p;
+        if (!(p = modhash_lookup (pub->ctx->modhash, uuid))
+            || module_subscribe (p, topic) < 0)
+            goto error;
+    }
+    else {
+        if (subhash_subscribe (pub->ctx->sub, topic) < 0)
+            goto error;
+    }
+    if (!flux_msg_is_noresponse (msg)
+        && flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to subscribe request");
+    return;
+error:
+    if (!flux_msg_is_noresponse (msg)
+        && flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to subscribe request");
+}
+
+static void unsubscribe_cb (flux_t *h, flux_msg_handler_t *mh,
+                            const flux_msg_t *msg, void *arg)
+{
+    struct publisher *pub = arg;
+    const char *uuid;
+    const char *topic;
+
+    if (flux_request_unpack (msg, NULL, "{ s:s }", "topic", &topic) < 0)
+        goto error;
+    if ((uuid = flux_msg_route_first (msg))) {
+        module_t *p;
+        if (!(p = modhash_lookup (pub->ctx->modhash, uuid))
+            || module_unsubscribe (p, topic) < 0)
+            goto error;
+    }
+    else {
+        if (subhash_unsubscribe (pub->ctx->sub, topic) < 0)
+            goto error;
+    }
+    if (!flux_msg_is_noresponse (msg)
+        && flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to unsubscribe request");
+    return;
+error:
+    if (!flux_msg_is_noresponse (msg)
+        && flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to unsubscribe request");
+}
+
 static const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_REQUEST, "event.pub",  pub_cb, FLUX_ROLE_USER },
+    { FLUX_MSGTYPE_REQUEST, "event.publish",  publish_cb, FLUX_ROLE_USER },
+    { FLUX_MSGTYPE_REQUEST, "event.subscribe",  subscribe_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "event.unsubscribe",  unsubscribe_cb, 0 },
     FLUX_MSGHANDLER_TABLE_END,
 };
 
-
-int publisher_set_flux (struct publisher *pub, flux_t *h)
+void publisher_destroy (struct publisher *pub)
 {
-    pub->h = h;
-    if (flux_msg_handler_addvec (h, htab, pub, &pub->handlers) < 0)
-        return -1;
-    return 0;
+    if (pub) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (pub->handlers);
+        free (pub);
+        errno = saved_errno;
+    }
 }
 
-int publisher_set_sender (struct publisher *pub, const char *name,
-                          publisher_send_f cb, void *arg)
+struct publisher *publisher_create (struct broker *ctx,
+                                    publisher_send_f cb,
+                                    void *arg)
 {
-    struct sender *sender;
+    struct publisher *pub;
 
-    if (!(sender = calloc (1, sizeof (*sender))))
-        return -1;
-    sender->send = cb;
-    sender->arg = arg;
-    (void)snprintf (sender->name, sizeof (sender->name), "%s", name);
-    if (zlist_append (pub->senders, sender) < 0) {
-        free (sender);
-        errno = ENOMEM;
-        return -1;
+    if (!(pub = calloc (1, sizeof (*pub))))
+        return NULL;
+    pub->ctx = ctx;
+    pub->send = cb;
+    pub->arg = arg;
+    if (flux_msg_handler_addvec (ctx->h, htab, pub, &pub->handlers) < 0) {
+        publisher_destroy (pub);
+        return NULL;
     }
-    return 0;
+    return pub;
 }
 
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/broker/publisher.h b/src/broker/publisher.h
index 18a2bd118727..690d367b6a84 100644
--- a/src/broker/publisher.h
+++ b/src/broker/publisher.h
@@ -11,20 +11,15 @@
 #ifndef _BROKER_PUBLISHER_H
 #define _BROKER_PUBLISHER_H
 
+#include "broker.h"
+
 typedef int (*publisher_send_f)(void *arg, const flux_msg_t *msg);
 
-struct publisher *publisher_create (void);
+struct publisher *publisher_create (struct broker *ctx,
+                                    publisher_send_f cb,
+                                    void *arg);
 void publisher_destroy (struct publisher *pub);
 
-int publisher_set_flux (struct publisher *pub, flux_t *h);
-
-/* Add a sender.  All senders are called when an event is published.
- * If a sender returns -1, an error will be logged but sending will continue.
- * Senders should return 0 on success.
- */
-int publisher_set_sender (struct publisher *pub, const char *name,
-                          publisher_send_f cb, void *arg);
-
 /* Publish an encoded event message, assigning sequence number.
  */
 int publisher_send (struct publisher *pub, const flux_msg_t *msg);
diff --git a/src/broker/reduce.c b/src/broker/reduce.c
deleted file mode 100644
index 7df8089b9871..000000000000
--- a/src/broker/reduce.c
+++ /dev/null
@@ -1,303 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdbool.h>
-#include <czmq.h>
-#include <flux/core.h>
-
-#include "reduce.h"
-
-struct flux_reduce_struct {
-    struct flux_reduce_ops ops;
-    void *arg;
-
-    zlist_t *items; /* set of current items */
-    void *old_item; /* flux_reduce_pop() pops old_item if old_flag is true */
-    bool old_flag;
-
-    uint32_t rank;
-    flux_t *h;
-    flux_reactor_t *reactor;
-    int flags;
-
-    flux_watcher_t *timer;
-    double timeout;
-    bool timer_armed;
-
-    unsigned int hwm;
-    bool hwm_readonly;
-    unsigned int count; /* count of items in current batch towards hwm */
-
-    int batchnum;
-    bool flushed;
-};
-
-
-static void flush_current (flux_reduce_t *r);
-
-
-static void timer_cb (flux_reactor_t *reactor, flux_watcher_t *w,
-                      int revents, void *arg)
-{
-    flux_reduce_t *r = arg;
-    flush_current (r);
-}
-
-flux_reduce_t *flux_reduce_create (flux_t *h, struct flux_reduce_ops ops,
-                                   double timeout, void *arg, int flags)
-{
-    if (!h || ((flags & FLUX_REDUCE_HWMFLUSH) && !ops.itemweight)
-           || ((flags & FLUX_REDUCE_TIMEDFLUSH) && timeout <= 0)) {
-        errno = EINVAL;
-        return NULL;
-    }
-    flux_reduce_t *r = calloc (1, sizeof (*r));
-    if (!r)
-        return NULL;
-    r->h = h;
-    if (!(r->reactor = flux_get_reactor (h))) {
-        flux_reduce_destroy (r);
-        return NULL;
-    }
-    r->ops = ops;
-    r->rank = 0;
-    if (flux_get_rank (h, &r->rank) < 0) {
-        flux_reduce_destroy (r);
-        return NULL;
-    }
-
-    r->arg = arg;
-    r->flags = flags;
-    r->timeout = timeout;
-    if (!(r->items = zlist_new ())) {
-        flux_reduce_destroy (r);
-        return NULL;
-    }
-    if ((flags & FLUX_REDUCE_TIMEDFLUSH)) {
-        if (!(r->timer = flux_timer_watcher_create (r->reactor, 0., 0.,
-                                                    timer_cb, r))) {
-            flux_reduce_destroy (r);
-            return NULL;
-        }
-    }
-    return r;
-}
-
-void flux_reduce_destroy (flux_reduce_t *r)
-{
-    if (r) {
-        int saved_errno = errno;
-        if (r->items) {
-            void *item;
-            while ((item = zlist_pop (r->items))) {
-                if (r->ops.destroy)
-                    r->ops.destroy (item);
-            }
-            zlist_destroy (&r->items);
-        }
-        if (r->ops.destroy && r->old_item)
-            r->ops.destroy (r->old_item);
-        if (r->timer) {
-            flux_watcher_stop (r->timer);
-            flux_watcher_destroy (r->timer);
-        }
-        free (r);
-        errno = saved_errno;
-    }
-}
-
-/* Empty the queue of items.
- */
-static void flush_current (flux_reduce_t *r)
-{
-    void *item;
-
-    if (zlist_size (r->items) > 0) {
-        if (r->rank > 0) {
-            if (r->ops.forward)
-                r->ops.forward (r, r->batchnum, r->arg);
-        } else {
-            if (r->ops.sink)
-                r->ops.sink (r, r->batchnum, r->arg);
-        }
-        while ((item = zlist_pop (r->items))) {
-            if (r->ops.destroy)
-                r->ops.destroy (item);
-        }
-    }
-    if (r->timer) {
-        flux_watcher_stop (r->timer);
-        r->timer_armed = false;
-    }
-    r->flushed = true;
-}
-
-/* Flush one item that is a straggler from a previous batch.
- */
-static void flush_old (flux_reduce_t *r, void *item, int batchnum)
-{
-    assert (r->old_item == NULL);
-    r->old_item = item;
-    r->old_flag = true;
-
-    if (r->rank > 0) {
-        if (r->ops.forward)
-            r->ops.forward (r, batchnum, r->arg);
-    } else {
-        if (r->ops.sink)
-            r->ops.sink (r, batchnum, r->arg);
-    }
-    if (r->ops.destroy)
-        r->ops.destroy (r->old_item);
-    r->old_item = NULL;
-    r->old_flag = false;
-}
-
-int flux_reduce_append (flux_reduce_t *r, void *item, int batchnum)
-{
-    int rc = -1;
-    int count = 1;
-
-    if (r->ops.itemweight)
-        count = r->ops.itemweight (item);
-
-    if (batchnum < r->batchnum - 1) {
-        flush_old (r, item, batchnum);
-    } else if (batchnum == r->batchnum - 1) {
-        if (!r->hwm_readonly)
-            r->hwm += count;
-        flush_old (r, item, batchnum);
-    } else if (batchnum == r->batchnum && r->flushed) {
-        r->count += count;
-        flush_old (r, item, batchnum);
-    } else {
-        if (batchnum > r->batchnum) {
-            flush_current (r);
-            if (!r->hwm_readonly)
-                r->hwm = r->count;
-            r->count = 0;
-            r->batchnum = batchnum;
-            r->flushed = false;
-        }
-
-        assert (batchnum == r->batchnum);
-        r->count += count;
-        if (zlist_push (r->items, item) < 0)
-            goto done;
-        if (r->ops.reduce && zlist_size (r->items) > 1)
-            r->ops.reduce (r, r->batchnum, r->arg);
-
-        if ((r->flags & FLUX_REDUCE_HWMFLUSH)) {
-            if (r->count >= r->hwm)
-                flush_current (r);
-        }
-        if ((r->flags & FLUX_REDUCE_TIMEDFLUSH)) {
-            if (zlist_size (r->items) > 0 && !r->timer_armed) {
-                flux_timer_watcher_reset (r->timer, r->timeout, 0);
-                flux_watcher_start (r->timer);
-                r->timer_armed = true;
-            }
-        }
-        if (!(r->flags & FLUX_REDUCE_HWMFLUSH)
-                && !(r->flags & FLUX_REDUCE_TIMEDFLUSH)) {
-            flush_current (r);
-        }
-    }
-    rc = 0;
-done:
-    return rc;
-}
-
-void *flux_reduce_pop (flux_reduce_t *r)
-{
-    void *item = NULL;
-    if (r->old_flag) {
-        item = r->old_item;
-        r->old_item = NULL;
-    } else
-        item = zlist_pop (r->items);
-    return item;
-}
-
-int flux_reduce_push (flux_reduce_t *r, void *item)
-{
-    return zlist_push (r->items, item);
-}
-
-int flux_reduce_opt_get (flux_reduce_t *r, int option, void *val, size_t size)
-{
-    switch (option) {
-        case FLUX_REDUCE_OPT_TIMEOUT:
-            if (size != sizeof (r->timeout))
-                goto invalid;
-            memcpy (val, &r->timeout, size);
-            break;
-        case FLUX_REDUCE_OPT_HWM:
-            if (size != sizeof (r->count))
-                goto invalid;
-            memcpy (val, &r->hwm, size);
-            break;
-        case FLUX_REDUCE_OPT_COUNT : {
-            unsigned int count = zlist_size (r->items);
-            if (size != sizeof (count))
-                goto invalid;
-            memcpy (val, &count, size);
-            break;
-        }
-        case FLUX_REDUCE_OPT_WCOUNT : {
-            unsigned int count = 0;
-            void *item = zlist_first (r->items);
-            while (item) {
-                count += r->ops.itemweight ? r->ops.itemweight (item) : 1;
-                item = zlist_next (r->items);
-            }
-            if (size != sizeof (count))
-                goto invalid;
-            memcpy (val, &count, size);
-            break;
-        }
-        default:
-            goto invalid;
-    }
-    return 0;
-invalid:
-    errno = EINVAL;
-    return -1;
-}
-
-int flux_reduce_opt_set (flux_reduce_t *r, int option, void *val, size_t size)
-{
-    switch (option) {
-        case FLUX_REDUCE_OPT_TIMEOUT:
-            if (size != sizeof (r->timeout))
-                goto invalid;
-            memcpy (&r->timeout, val, size);
-            break;
-        case FLUX_REDUCE_OPT_HWM:
-            if (size != sizeof (r->hwm))
-                goto invalid;
-            memcpy (&r->hwm, val, size);
-            r->hwm_readonly = true;
-            break;
-        default:
-            goto invalid;
-    }
-    return 0;
-invalid:
-    errno = EINVAL;
-    return -1;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/reduce.h b/src/broker/reduce.h
deleted file mode 100644
index 95bb3ab58473..000000000000
--- a/src/broker/reduce.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_CORE_REDUCE_H
-#define _FLUX_CORE_REDUCE_H
-
-#include <flux/core.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct flux_reduce_struct flux_reduce_t;
-
-struct flux_reduce_ops {
-    flux_free_f destroy;
-    void   (*reduce)(flux_reduce_t *r, int batchnum, void *arg);
-    void   (*sink)(flux_reduce_t *r, int batchnum, void *arg);
-    void   (*forward)(flux_reduce_t *r, int batchnum, void *arg);
-    int    (*itemweight)(void *item);
-};
-
-enum {
-    FLUX_REDUCE_TIMEDFLUSH = 1,
-    FLUX_REDUCE_HWMFLUSH = 2,
-};
-
-enum {
-    FLUX_REDUCE_OPT_TIMEOUT = 1,
-    FLUX_REDUCE_OPT_HWM = 2,
-    FLUX_REDUCE_OPT_COUNT = 3,
-    FLUX_REDUCE_OPT_WCOUNT = 4,
-};
-
-flux_reduce_t *flux_reduce_create (flux_t *h, struct flux_reduce_ops ops,
-                                   double timeout, void *arg, int flags);
-
-void flux_reduce_destroy (flux_reduce_t *r);
-
-int flux_reduce_append (flux_reduce_t *r, void *item, int batchnum);
-
-void *flux_reduce_pop (flux_reduce_t *r);
-
-int flux_reduce_push (flux_reduce_t *r, void *item);
-
-int flux_reduce_opt_get (flux_reduce_t *r, int option, void *val, size_t size);
-
-int flux_reduce_opt_set (flux_reduce_t *r, int option, void *val, size_t size);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _FLUX_CORE_REDUCE_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/runat.c b/src/broker/runat.c
new file mode 100644
index 000000000000..c5743bf77162
--- /dev/null
+++ b/src/broker/runat.c
@@ -0,0 +1,732 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* runat.c - run named list of sequential commands
+ *
+ * Notes:
+ * - Command env is inherited from broker, minus blocklist, plus FLUX_URI.
+ * - All commands in a list are executed, even if one fails.
+ * - The exit code of the first failed command is captured.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <termios.h>
+#include <unistd.h>
+#include <signal.h>
+#ifdef HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <jansson.h>
+#if HAVE_LIBSYSTEMD
+#include <systemd/sd-daemon.h>
+#endif
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/monotime.h"
+#include "ccan/str/str.h"
+
+#include "runat.h"
+
+struct runat_command {
+    flux_subprocess_t *p;
+    flux_cmd_t *cmd;
+    int flags;
+    struct timespec t_start;
+};
+
+struct runat_entry {
+    char *name;
+    zlist_t *commands;
+    int exit_code;
+    int count;
+    bool aborted;
+    bool completed;
+    bool interactive;
+    bool foreground;
+    runat_completion_f cb;
+    void *cb_arg;
+};
+
+struct runat {
+    flux_t *h;
+    const char *local_uri;
+    zhashx_t *entries;
+    flux_msg_handler_t **handlers;
+    bool sd_notify;
+    struct termios saved_termios;
+};
+
+static void runat_command_destroy (struct runat_command *cmd);
+static void start_next_command (struct runat *r, struct runat_entry *entry);
+
+static const int abort_signal = SIGHUP;
+
+static const char *env_blocklist[] = {
+    "FLUX_JOB_ID",
+    "FLUX_JOB_SIZE",
+    "FLUX_JOB_NNODES",
+    "FLUX_JOB_TMPDIR",
+    "FLUX_TASK_RANK",
+    "FLUX_TASK_LOCAL_ID",
+    "FLUX_URI",
+    "FLUX_KVS_NAMESPACE",
+    "FLUX_PROXY_REMOTE",
+    "PMI_*",
+    "FLUX_PMI_LIBRARY_PATH",
+    "I_MPI_PMI_LIBRARY",
+    "SLURM_*",  // flux-framework/flux-core#5206
+    NULL,
+};
+
+extern char **environ;
+
+static const char *get_shell (void)
+{
+    const char *shell = getenv ("SHELL");
+    if (!shell)
+        shell = "/bin/bash";
+    return shell;
+}
+
+static char *get_cmdline (flux_cmd_t *cmd)
+{
+    char *buf = NULL;
+    size_t len = 0;
+    int i;
+    int start = 0;
+
+    /* Drop the "/bin/bash -c" from logging for brevity.
+     */
+    if (flux_cmd_argc (cmd) > 2
+        && streq (flux_cmd_arg (cmd, 0), get_shell ())
+        && streq (flux_cmd_arg (cmd, 1), "-c"))
+        start += 2;
+    for (i = start; i < flux_cmd_argc (cmd); i++) {
+        if (argz_add (&buf, &len, flux_cmd_arg (cmd, i)) != 0) {
+            free (buf);
+            return NULL;
+        }
+    }
+    argz_stringify (buf, len, ' ');
+    return buf;
+}
+
+static void log_command (flux_t *h,
+                         struct runat_entry *entry,
+                         int rc,
+                         double elapsed,
+                         const char *s)
+{
+    struct runat_command *command = zlist_head (entry->commands);
+    int command_index = entry->count - zlist_size (entry->commands);
+    char *cmdline = get_cmdline (command->cmd);
+
+    flux_log (h,
+              rc == 0 ? LOG_INFO : LOG_ERR,
+              "%s.%d: %s %s (rc=%d) %.1fs",
+              entry->name,
+              command_index,
+              cmdline ? cmdline : "???",
+              s,
+              rc,
+              elapsed);
+
+    free (cmdline);
+}
+
+/* See POSIX 2008 Volume 3 Shell and Utilities, Issue 7
+ * Section 2.8.2 Exit status for shell commands (page 2315)
+ */
+static void completion_cb (flux_subprocess_t *p)
+{
+    struct runat *r = flux_subprocess_aux_get (p, "runat");
+    struct runat_entry *entry = flux_subprocess_aux_get (p, "runat_entry");
+    struct runat_command *cmd = zlist_head (entry->commands);
+    double elapsed = monotime_since (cmd->t_start) / 1000;
+    int rc = flux_subprocess_exit_code (p);
+    int signum;
+
+    if (rc == 0 && entry->aborted) {
+        rc = 1;
+        log_command (r->h, entry, rc, elapsed, "aborted after exit with rc=0");
+    }
+    else if (rc >= 0)
+        log_command (r->h, entry, rc, elapsed, "Exited");
+    else if ((signum = flux_subprocess_signaled (p)) > 0) { // signaled
+        rc = signum + 128;
+        log_command (r->h, entry, rc, elapsed, strsignal (signum));
+    }
+    else { // ???
+        rc = 1;
+        log_command (r->h, entry, rc, elapsed, "???");
+    }
+    if (rc != 0 && entry->exit_code == 0) // capture first exit error
+        entry->exit_code = rc;
+    if (entry->foreground) {
+        /*  This entry was moved to the foreground. Now that it has exited,
+         *  restore the current process group to the foreground and
+         *  reset terminal state.
+         */
+        if (tcsetpgrp (STDIN_FILENO, getpgrp ()) < 0
+            || tcsetattr (STDIN_FILENO, TCSAFLUSH, &r->saved_termios) < 0)
+        flux_log_error (r->h, "failed to reset foreground process group");
+    }
+    runat_command_destroy (zlist_pop (entry->commands));
+    start_next_command (r, entry);
+}
+
+/* If state changes to running and the abort flag is set, send abort_signal.
+ * This closes a race where the 'entry' might continue running if the abort
+ * is called as a process is starting up.
+ */
+static void state_change_cb (flux_subprocess_t *p,
+                             flux_subprocess_state_t state)
+{
+    struct runat *r = flux_subprocess_aux_get (p, "runat");
+    struct runat_entry *entry = flux_subprocess_aux_get (p, "runat_entry");
+    flux_future_t *f = NULL;
+
+    switch (state) {
+        case FLUX_SUBPROCESS_INIT:
+        case FLUX_SUBPROCESS_EXITED:
+        case FLUX_SUBPROCESS_FAILED:
+            break;
+        case FLUX_SUBPROCESS_STOPPED:
+            /*
+             *  If stdin is a tty and the broker is in a foreground process
+             *  group, the subprocess may have stopped due to SIGTTIN/SIGTTOU.
+             *  Attempt to bring the subprocess into the foreground and
+             *  continue it. Set the foreground flag on the entry so that the
+             *  broker knows to bring its own process group back into the
+             *  foreground after this subprocess is complete.
+             */
+            if (isatty (STDIN_FILENO)
+                && tcgetpgrp (STDIN_FILENO) == getpgrp ()) {
+                entry->foreground = true;
+                if (tcsetpgrp (STDIN_FILENO, flux_subprocess_pid (p)) < 0
+                    || !(f = flux_subprocess_kill (p, SIGCONT))) {
+                    flux_log_error (r->h,
+                                    "error bringing %s into foreground",
+                                    entry->name);
+                }
+                flux_future_destroy (f);
+            }
+            break;
+        case FLUX_SUBPROCESS_RUNNING:
+            if (entry->aborted) {
+                if (!(f = flux_subprocess_kill (p, abort_signal))) {
+                    if (errno != ESRCH)
+                        flux_log_error (r->h, "kill %s", entry->name);
+                }
+                flux_future_destroy (f);
+            }
+            break;
+    }
+}
+
+static void stdio_cb (flux_subprocess_t *p, const char *stream)
+{
+    struct runat *r = flux_subprocess_aux_get (p, "runat");
+    struct runat_entry *entry = flux_subprocess_aux_get (p, "runat_entry");
+    int index = entry->count - zlist_size (entry->commands);
+    const char *line;
+    int len;
+
+    if ((len = flux_subprocess_getline (p, stream, &line)) > 0) {
+        if (streq (stream, "stderr"))
+            flux_log (r->h, LOG_ERR, "%s.%d: %s", entry->name, index, line);
+        else
+            flux_log (r->h, LOG_INFO, "%s.%d: %s", entry->name, index, line);
+    }
+}
+
+/* Start one command.
+ */
+static flux_subprocess_t *start_command (struct runat *r,
+                                         struct runat_entry *entry,
+                                         struct runat_command *cmd)
+{
+    flux_subprocess_t *p;
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_state_change = state_change_cb,
+        .on_channel_out = NULL,
+        .on_stdout = NULL,
+        .on_stderr = NULL,
+    };
+    if (!(cmd->flags & FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH)) {
+        ops.on_stdout = stdio_cb;
+        ops.on_stderr = stdio_cb;
+    }
+    if (!(p = flux_local_exec_ex (flux_get_reactor (r->h),
+                                  cmd->flags,
+                                  cmd->cmd,
+                                  &ops,
+                                  NULL,
+                                  flux_llog,
+                                  r->h)))
+        return NULL;
+    if (flux_subprocess_aux_set (p, "runat_entry", entry, NULL) < 0)
+        goto error;
+    if (flux_subprocess_aux_set (p, "runat", r, NULL) < 0)
+        goto error;
+    monotime (&cmd->t_start);
+    return p;
+error:
+    flux_subprocess_destroy (p);
+    return NULL;
+}
+
+/* Start the next command.
+ * If startup fails, try the next, and so on.
+ */
+static void start_next_command (struct runat *r, struct runat_entry *entry)
+{
+    struct runat_command *cmd;
+    bool started = false;
+
+    if (entry->aborted) {
+        while ((cmd = zlist_pop (entry->commands)))
+            runat_command_destroy (cmd);
+    }
+    else {
+        while (!started && (cmd = zlist_head (entry->commands))) {
+#if HAVE_LIBSYSTEMD
+            if (r->sd_notify) {
+                char *s = get_cmdline (cmd->cmd);
+                sd_notifyf (0, "STATUS=Running %s", s ? s : "unknown command");
+                free (s);
+            }
+#endif
+            if (!(cmd->p = start_command (r, entry, cmd))) {
+                log_command (r->h, entry, 1, 0, "error starting command");
+                if (entry->exit_code == 0)
+                    entry->exit_code = 1;
+                runat_command_destroy (zlist_pop (entry->commands));
+            }
+            else
+                started = true;
+        }
+    }
+    if (zlist_size (entry->commands) == 0) {
+        entry->completed = true;
+        if (entry->cb)
+            entry->cb (r, entry->name, entry->cb_arg);
+    }
+}
+
+static void runat_command_destroy (struct runat_command *cmd)
+{
+    if (cmd) {
+        int saved_errno = errno;
+        flux_cmd_destroy (cmd->cmd);
+        flux_subprocess_destroy (cmd->p);
+        free (cmd);
+        errno = saved_errno;
+    }
+}
+
+static struct runat_command *runat_command_create (char **env, int flags)
+{
+    struct runat_command *cmd;
+
+    if (!(cmd = calloc (1, sizeof (*cmd))))
+        return NULL;
+    if (!(flags & RUNAT_FLAG_LOG_STDIO))
+        cmd->flags |= FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH;
+    if (flags & RUNAT_FLAG_FORK_EXEC)
+        cmd->flags |= FLUX_SUBPROCESS_FLAGS_FORK_EXEC;
+    /*
+     * N.B. By default subprocesses call setpgrp() before exec(2).  So
+     * any processes spawned by command are also signaled by
+     * flux_subprocess_signal()
+     */
+    if (!(cmd->cmd = flux_cmd_create (0, NULL, env)))
+        goto error;
+    return cmd;
+error:
+    runat_command_destroy (cmd);
+    return NULL;
+}
+
+/* Unset blocklisted variables in command environment.
+ * Set FLUX_URI if local_uri is non-NULL.
+ */
+static int runat_command_modenv (struct runat_command *cmd,
+                                 const char **blocklist,
+                                 const char *local_uri)
+{
+    if (blocklist) {
+        int i;
+        for (i = 0; blocklist[i] != NULL; i++)
+            flux_cmd_unsetenv (cmd->cmd, blocklist[i]);
+    }
+    if (local_uri) {
+        if (flux_cmd_setenvf (cmd->cmd, 1, "FLUX_URI", "%s", local_uri) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static int runat_command_set_argz (struct runat_command *cmd,
+                                   const char *argz,
+                                   size_t argz_len)
+{
+    char *arg = argz_next (argz, argz_len, NULL);
+    while (arg) {
+        if (flux_cmd_argv_append (cmd->cmd, arg) < 0)
+            return -1;
+        arg = argz_next (argz, argz_len, arg);
+    }
+    return 0;
+}
+
+static int runat_command_set_cmdline (struct runat_command *cmd,
+                                      const char *shell,
+                                      const char *cmdline)
+{
+    if (shell == NULL)
+        shell = get_shell ();
+    if (flux_cmd_argv_append (cmd->cmd, shell) < 0)
+        return -1;
+    if (cmdline) {
+        if (flux_cmd_argv_append (cmd->cmd, "-c") < 0)
+            return -1;
+        if (flux_cmd_argv_append (cmd->cmd, cmdline) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static void runat_entry_destroy (struct runat_entry *entry)
+{
+    if (entry) {
+        int saved_errno = errno;
+        if (entry->commands) {
+            struct runat_command *cmd;
+            while ((cmd = zlist_pop (entry->commands)))
+                runat_command_destroy (cmd);
+            zlist_destroy (&entry->commands);
+        }
+        free (entry->name);
+        free (entry);
+        errno = saved_errno;
+    }
+}
+
+static struct runat_entry *runat_entry_create (const char *name)
+{
+    struct runat_entry *entry;
+
+    if (!(entry = calloc (1, sizeof (*entry))))
+        return NULL;
+    if (!(entry->name = strdup (name)))
+        goto error;
+    if (!(entry->commands = zlist_new ()))
+        goto error;
+    return entry;
+error:
+    runat_entry_destroy (entry);
+    return NULL;
+}
+
+/* zhashx_destructor_fn signature */
+static void runat_entry_destroy_wrapper (void **arg)
+{
+    if (arg) {
+        runat_entry_destroy (*arg);
+        *arg = NULL;
+    }
+}
+
+/* Push 'cmd' onto command list 'name', creating it if it doesn't exist.
+ */
+static int runat_push (struct runat *r,
+                       const char *name,
+                       struct runat_command *cmd,
+                       bool interactive)
+{
+    struct runat_entry *entry;
+
+    if (!(entry = zhashx_lookup (r->entries, name))) {
+        if (!(entry = runat_entry_create (name)))
+            return -1;
+        (void)zhashx_insert (r->entries, name, entry);
+    }
+    if (zlist_push (entry->commands, cmd) < 0) {
+        if (zlist_size (entry->commands) == 0)
+            zhashx_delete (r->entries, name);
+        errno = ENOMEM;
+        return -1;
+    }
+    entry->count++;
+    if (!entry->interactive)
+        entry->interactive = interactive;
+    return 0;
+}
+
+int runat_push_shell_command (struct runat *r,
+                              const char *name,
+                              const char *cmdline,
+                              int flags)
+{
+    struct runat_command *cmd;
+
+    if (!r || !name || !cmdline) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(cmd = runat_command_create (environ, flags)))
+        return -1;
+    if (runat_command_set_cmdline (cmd, NULL, cmdline) < 0)
+        goto error;
+    if (runat_command_modenv (cmd, env_blocklist, r->local_uri) < 0)
+        goto error;
+    if (runat_push (r, name, cmd, false) < 0)
+        goto error;
+    return 0;
+error:
+    runat_command_destroy (cmd);
+    return -1;
+}
+
+int runat_push_shell (struct runat *r,
+                      const char *name,
+                      const char *shell,
+                      int flags)
+{
+    struct runat_command *cmd;
+
+    if (!r || !name || (flags & RUNAT_FLAG_LOG_STDIO)) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(cmd = runat_command_create (environ, flags)))
+        return -1;
+    if (runat_command_set_cmdline (cmd, shell, NULL) < 0)
+        goto error;
+    if (runat_command_modenv (cmd, env_blocklist, r->local_uri) < 0)
+        goto error;
+    if (runat_push (r, name, cmd, true) < 0)
+        goto error;
+    return 0;
+error:
+    runat_command_destroy (cmd);
+    return -1;
+}
+
+int runat_push_command (struct runat *r,
+                        const char *name,
+                        const char *argz,
+                        size_t argz_len,
+                        int flags)
+{
+    struct runat_command *cmd;
+
+    if (!r || !name || !argz) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(cmd = runat_command_create (environ, flags)))
+        return -1;
+    if (runat_command_set_argz (cmd, argz, argz_len) < 0)
+        goto error;
+    if (runat_command_modenv (cmd, env_blocklist, r->local_uri) < 0)
+        goto error;
+    if (runat_push (r, name, cmd, false) < 0)
+        goto error;
+    return 0;
+error:
+    runat_command_destroy (cmd);
+    return -1;
+}
+
+int runat_get_exit_code (struct runat *r, const char *name, int *rc)
+{
+    struct runat_entry *entry;
+
+    if (!r || !name || !rc) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(entry = zhashx_lookup (r->entries, name))) {
+        errno = ENOENT;
+        return -1;
+    }
+    *rc = entry->exit_code;
+    return 0;
+}
+
+int runat_start (struct runat *r,
+                 const char *name,
+                 runat_completion_f cb,
+                 void *arg)
+{
+    struct runat_entry *entry;
+
+    if (!r || !name) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(entry = zhashx_lookup (r->entries, name))) {
+        errno = ENOENT;
+        return -1;
+    }
+    entry->cb = cb;
+    entry->cb_arg = arg;
+    start_next_command (r, entry);
+    return 0;
+}
+
+bool runat_is_defined (struct runat *r, const char *name)
+{
+    if (!r || !name || !zhashx_lookup (r->entries, name))
+        return false;
+    return true;
+}
+
+bool runat_is_completed (struct runat *r, const char *name)
+{
+    struct runat_entry *entry;
+
+    if (!r || !name || !(entry = zhashx_lookup (r->entries, name)))
+        return false;
+    return entry->completed;
+}
+
+bool runat_is_interactive (struct runat *r, const char *name)
+{
+    struct runat_entry *entry;
+
+    if (!r || !name || !(entry = zhashx_lookup (r->entries, name)))
+        return false;
+    return entry->interactive;
+}
+
+int runat_abort (struct runat *r, const char *name)
+{
+    struct runat_entry *entry;
+    struct runat_command *cmd;
+
+    if (!r || !name) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(entry = zhashx_lookup (r->entries, name))) {
+        errno = ENOENT;
+        return -1;
+    }
+    if ((cmd = zlist_head (entry->commands)) && cmd->p != NULL) {
+        flux_future_t *f;
+        if (!(f = flux_subprocess_kill (cmd->p, abort_signal))) {
+            if (errno != ESRCH)
+                flux_log_error (r->h, "kill %s", entry->name);
+        }
+        flux_future_destroy (f);
+    }
+    entry->aborted = true;
+    return 0;
+}
+
+static void runat_push_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct runat *r = arg;
+    const char *name;
+    json_t *commands;
+    const char *errstr = NULL;
+    size_t index;
+    json_t *el;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s:o}",
+                             "name",
+                             &name,
+                             "commands",
+                             &commands) < 0)
+        goto error;
+    if (json_array_size (commands) == 0) {
+        errno = EPROTO;
+        errstr = "commands array is empty";
+        goto error;
+    }
+    json_array_foreach (commands, index, el) {
+        const char *cmdline = json_string_value (el);
+        if (!cmdline || strlen (cmdline) == 0) {
+            errno = EPROTO;
+            errstr = "cannot push an empty command line";
+            goto error;
+        }
+        if (runat_push_shell_command (r,
+                                      name,
+                                      cmdline,
+                                      RUNAT_FLAG_LOG_STDIO) < 0)
+            goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log (h, LOG_ERR, "error responding to runat.push");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log (h, LOG_ERR, "error responding to runat.push");
+}
+
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,  "runat.push", runat_push_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+struct runat *runat_create (flux_t *h, const char *local_uri, bool sdnotify)
+{
+    struct runat *r;
+
+    if (!(r = calloc (1, sizeof (*r))))
+        return NULL;
+    if (!(r->entries = zhashx_new ()))
+        goto error;
+    if (flux_msg_handler_addvec (h, htab, r, &r->handlers) < 0)
+        goto error;
+    zhashx_set_destructor (r->entries, runat_entry_destroy_wrapper);
+    r->h = h;
+    r->local_uri = local_uri;
+    r->sd_notify = sdnotify;
+    if (isatty (STDIN_FILENO)
+        && tcgetattr (STDIN_FILENO, &r->saved_termios) < 0)
+        flux_log_error (r->h, "failed to save terminal attributes");
+    return r;
+error:
+    runat_destroy (r);
+    return NULL;
+}
+
+void runat_destroy (struct runat *r)
+{
+    if (r) {
+        int saved_errno = errno;
+        zhashx_destroy (&r->entries);
+        flux_msg_handler_delvec (r->handlers);
+        free (r);
+        errno = saved_errno;
+    }
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/broker/runat.h b/src/broker/runat.h
new file mode 100644
index 000000000000..9e759e0d357d
--- /dev/null
+++ b/src/broker/runat.h
@@ -0,0 +1,92 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Execute list of commands, sequentially, by name.
+ */
+
+#ifndef _BROKER_RUNAT_H
+#define _BROKER_RUNAT_H
+
+enum {
+    RUNAT_FLAG_LOG_STDIO = 1,   /* stdout/stderr go to flux_log (o/w
+                                 * combine w/ broker) */
+    RUNAT_FLAG_FORK_EXEC = 2,  /* require use of fork/exec, not
+                                * posix_spawn */
+};
+
+struct runat;
+
+typedef void (*runat_completion_f)(struct runat *r,
+                                   const char *name,
+                                   void *arg);
+
+struct runat *runat_create (flux_t *h, const char *local_uri, bool sdnotify);
+void runat_destroy (struct runat *r);
+
+/* Push command, to be run under shell -c, onto named list.
+ */
+int runat_push_shell_command (struct runat *r,
+                              const char *name,
+                              const char *cmdline,
+                              int flags);
+
+/* Push interactive shell onto named list.
+ * Note: RUNAT_FLAG_LOG_STDIO flag not allowed
+ */
+int runat_push_shell (struct runat *r,
+                      const char *name,
+                      const char *shell,
+                      int flags);
+
+/* Push command, to be run directly, onto named list.
+ * The command is specified by argz.
+ */
+int runat_push_command (struct runat *r,
+                        const char *name,
+                        const char *argz,
+                        size_t argz_len,
+                        int flags);
+
+/* Get exit code of completed command list.
+ * If multiple commands fail, the exit code is that of the first failure.
+ */
+int runat_get_exit_code (struct runat *r, const char *name, int *rc);
+
+/* Begin execution of named list.
+ * Completion callback is called once command finish execution.
+ * The completion callback may call runat_get_exit_code().
+ */
+int runat_start (struct runat *r,
+                 const char *name,
+                 runat_completion_f cb,
+                 void *arg);
+
+/* Abort execution of named list.
+ * If a command is running, it is signaled.
+ */
+int runat_abort (struct runat *r, const char *name);
+
+/* Test whether named list has been defined.
+ */
+bool runat_is_defined (struct runat *r, const char *name);
+
+/* Test whether named list has completed running.
+ */
+bool runat_is_completed (struct runat *r, const char *name);
+
+/* Test whether named list contains interactive commands.
+ */
+bool runat_is_interactive (struct runat *r, const char *name);
+
+#endif /* !_BROKER_RUNAT_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/broker/runlevel.c b/src/broker/runlevel.c
deleted file mode 100644
index 58bfbd0fcbc9..000000000000
--- a/src/broker/runlevel.c
+++ /dev/null
@@ -1,314 +0,0 @@
-/************************************************************\
- * Copyright 2016 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <signal.h>
-#include <unistd.h>
-#include <assert.h>
-#include <argz.h>
-#include <inttypes.h>
-#include <jansson.h>
-#include <flux/core.h>
-
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/monotime.h"
-
-#include "runlevel.h"
-
-struct level {
-    flux_subprocess_t *p;
-    flux_cmd_t *cmd;
-    struct timespec start;
-};
-
-struct runlevel {
-    int level;
-    flux_t *h;
-    struct level rc[4];
-    runlevel_cb_f cb;
-    void *cb_arg;
-    runlevel_io_cb_f io_cb;
-    void *io_cb_arg;
-};
-
-static int runlevel_attr_get (const char *name, const char **val, void *arg);
-
-struct runlevel *runlevel_create (flux_t *h, attr_t *attrs)
-{
-    struct runlevel *r;
-
-    if (!(r = calloc (1, sizeof (*r))))
-        return NULL;
-    r->h = h;
-    if (attr_add_active (attrs,
-                         "init.run-level",
-                         FLUX_ATTRFLAG_READONLY,
-                         runlevel_attr_get,
-                         NULL,
-                         r) < 0)
-        goto error;
-    return r;
-error:
-    runlevel_destroy (r);
-    return NULL;
-}
-
-void runlevel_destroy (struct runlevel *r)
-{
-    if (r) {
-        int saved_errno = errno;
-        int i;
-        for (i = 0; i < 4; i++) {
-            flux_subprocess_destroy (r->rc[i].p);
-            flux_cmd_destroy (r->rc[i].cmd);
-        }
-        free (r);
-        errno = saved_errno;
-    }
-}
-
-static int runlevel_attr_get (const char *name, const char **val, void *arg)
-{
-    struct runlevel *r = arg;
-
-    if (!strcmp (name, "init.run-level")) {
-        static char s[16];
-        snprintf (s, sizeof (s), "%d", runlevel_get_level (r));
-        if (val)
-            *val = s;
-    } else {
-        errno = EINVAL;
-        goto error;
-    }
-    return 0;
-error:
-    return -1;
-}
-
-void runlevel_set_callback (struct runlevel *r, runlevel_cb_f cb, void *arg)
-{
-    r->cb = cb;
-    r->cb_arg = arg;
-}
-
-void runlevel_set_io_callback (struct runlevel *r, runlevel_io_cb_f cb, void *arg)
-{
-    r->io_cb = cb;
-    r->io_cb_arg = arg;
-}
-
-/* See POSIX 2008 Volume 3 Shell and Utilities, Issue 7
- * Section 2.8.2 Exit status for shell commands (page 2315)
- */
-static void completion_cb (flux_subprocess_t *p)
-{
-    struct runlevel *r = flux_subprocess_aux_get (p, "runlevel");
-    const char *exit_string = NULL;
-    int rc;
-
-    if ((rc = flux_subprocess_exit_code (p)) < 0) {
-        /* bash standard, signals + 128 */
-        if ((rc = flux_subprocess_signaled (p)) >= 0) {
-            exit_string = strsignal (rc);
-            rc += 128;
-        }
-    }
-    else {
-        if (rc)
-            exit_string = "Exited with non-zero status";
-        else
-            exit_string = "Exited";
-    }
-
-    assert (r->rc[r->level].p == p);
-    r->rc[r->level].p = NULL;
-
-    if (r->cb) {
-        double elapsed = monotime_since (r->rc[r->level].start) / 1000;
-        r->cb (r, r->level, rc, elapsed, exit_string, r->cb_arg);
-    }
-    flux_subprocess_destroy (p);
-}
-
-static void io_cb (flux_subprocess_t *p, const char *stream)
-{
-    struct runlevel *r;
-    const char *ptr;
-    int lenp;
-
-    r = flux_subprocess_aux_get (p, "runlevel");
-
-    assert (r);
-    assert (r->level == 1 || r->level == 3);
-
-    if (!(ptr = flux_subprocess_getline (p, stream, &lenp))) {
-        flux_log_error (r->h, "%s: flux_subprocess_getline", __FUNCTION__);
-        return;
-    }
-
-    if (lenp && r->io_cb)
-        r->io_cb (r, stream, ptr, r->io_cb_arg);
-}
-
-static int runlevel_start_subprocess (struct runlevel *r, int level)
-{
-    flux_subprocess_t *p = NULL;
-
-    assert (r->h != NULL);
-
-    if (r->rc[level].cmd) {
-        flux_subprocess_ops_t ops = {
-            .on_completion = completion_cb,
-            .on_state_change = NULL,
-            .on_channel_out = NULL,
-            .on_stdout = NULL,
-            .on_stderr = NULL,
-        };
-        int flags = 0;
-
-        /* set alternate io callback for levels 1 and 3 */
-        if (level == 1 || level == 3) {
-            ops.on_stdout = io_cb;
-            ops.on_stderr = io_cb;
-        }
-        else
-            flags |= FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH;
-
-        if (!(p = flux_exec (r->h,
-                             flags,
-                             r->rc[level].cmd,
-                             &ops,
-                             NULL)))
-            goto error;
-
-        if (flux_subprocess_aux_set (p, "runlevel", r, NULL) < 0)
-            goto error;
-
-        r->rc[level].p = p;
-    }
-    else if (level == 1 || level == 3) {
-        if (r->cb)
-            r->cb (r, r->level, 0, 0., "Not configured", r->cb_arg);
-    }
-    monotime (&r->rc[level].start);
-    return 0;
-
-error:
-    flux_subprocess_destroy (p);
-    return -1;
-}
-
-int runlevel_set_level (struct runlevel *r, int level)
-{
-    if (level < 1 || level > 3 || level <= r->level) {
-        errno = EINVAL;
-        return -1;
-    }
-    r->level = level;
-    if (runlevel_start_subprocess (r, level) < 0)
-        return -1;
-    return 0;
-}
-
-int runlevel_get_level (struct runlevel *r)
-{
-    return r->level;
-}
-
-int runlevel_set_rc (struct runlevel *r, int level, const char *cmd_argz,
-                     size_t cmd_argz_len, const char *local_uri)
-{
-    flux_cmd_t *cmd = NULL;
-    const char *shell = getenv ("SHELL");
-    if (!shell)
-        shell = "/bin/bash";
-
-    if (level < 1 || level > 3 || r->rc[level].p != NULL) {
-        errno = EINVAL;
-        goto error;
-    }
-    if (!(cmd = flux_cmd_create (0, NULL, environ)))
-        goto error;
-
-    // Run interactive shell if there are no arguments
-    if (argz_count (cmd_argz, cmd_argz_len) == 0) {
-        if (flux_cmd_argv_append (cmd, shell) < 0)
-            goto error;
-    }
-    // Wrap in shell -c if there is only one argument
-    else if (argz_count (cmd_argz, cmd_argz_len) == 1) {
-        char *arg = argz_next (cmd_argz, cmd_argz_len, NULL);
-
-        if (flux_cmd_argv_append (cmd, shell) < 0)
-            goto error;
-        if (flux_cmd_argv_append (cmd, "-c") < 0)
-            goto error;
-        if (flux_cmd_argv_append (cmd, arg) < 0)
-            goto error;
-    }
-    else {
-        char *arg = argz_next (cmd_argz, cmd_argz_len, NULL);
-        while (arg) {
-            if (flux_cmd_argv_append (cmd, arg) < 0)
-                goto error;
-            arg = argz_next (cmd_argz, cmd_argz_len, arg);
-        }
-    }
-    flux_cmd_unsetenv (cmd, "PMI_FD");
-    flux_cmd_unsetenv (cmd, "PMI_RANK");
-    flux_cmd_unsetenv (cmd, "PMI_SIZE");
-    if (local_uri && flux_cmd_setenvf (cmd, 1, "FLUX_URI",
-                                       "%s", local_uri) < 0)
-        goto error;
-    r->rc[level].cmd = cmd;
-    return 0;
-error:
-    flux_cmd_destroy (cmd);
-    return -1;
-}
-
-/* Abort current runlevel.
- * If there is a subprocess, send it a SIGTERM and let subproc completion
- * callback call r->cb().  Otherwise, call r->cb() now.
- * N.B. the broker will use the exit code runlevel gives r->cb().
- * In the subprocess case the exit code is expected to be be 128 + 15 = 143.
- * Otherwise the exit code will be zero.
- */
-int runlevel_abort (struct runlevel *r)
-{
-
-    if (!r)
-        return -1;
-    if (r->rc[r->level].p) {
-        flux_future_t *f;
-        if (!(f = flux_subprocess_kill (r->rc[r->level].p, SIGTERM))) {
-            flux_log_error (r->h, "flux_subprocess_kill");
-            return -1;
-        }
-        flux_future_destroy (f); // ignore response
-    }
-    else {
-        if (r->cb) {
-            r->cb (r,
-                   r->level,
-                   0,
-                   monotime_since (r->rc[r->level].start) / 1000,
-                   "Not configured",
-                   r->cb_arg);
-        }
-    }
-    return 0;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/runlevel.h b/src/broker/runlevel.h
deleted file mode 100644
index 6c118e51b90c..000000000000
--- a/src/broker/runlevel.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/************************************************************\
- * Copyright 2016 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _BROKER_RUNLEVEL_H
-#define _BROKER_RUNLEVEL_H
-
-#include "attr.h"
-
-#include <stdint.h>
-#include <stddef.h> // for size_t
-
-struct runlevel;
-
-typedef void (*runlevel_cb_f)(struct runlevel *r,
-                              int level,
-                              int rc,
-                              double elapsed,
-                              const char *exit_string,
-                              void *arg);
-
-typedef void (*runlevel_io_cb_f)(struct runlevel *r,
-                                 const char *name,
-                                 const char *msg,
-                                 void *arg);
-
-/* Initialize, finalize runlevel class.
- */
-struct runlevel *runlevel_create (flux_t *h, attr_t *attr);
-void runlevel_destroy (struct runlevel *r);
-
-/* Handle run level subprocess completion.
- */
-void runlevel_set_callback (struct runlevel *r, runlevel_cb_f cb, void *arg);
-
-/* Handle stdout, stderr output lines from subprocesses.
- */
-void runlevel_set_io_callback (struct runlevel *r,
-                               runlevel_io_cb_f cb,
-                               void *arg);
-
-/* Associate 'command' with 'level'.
- * 'local_uri' is used to set FLUX_URI in the subprocess environment.
- */
-int runlevel_set_rc (struct runlevel *r,
-                     int level,
-                     const char *cmd_argz,
-                     size_t cmd_argz_len,
-                     const char *local_uri);
-
-/* Change the runlevel.  It is assumed that the previous run level (if any)
- * has completed and this is being called from the runlevel callback.
- * Transitions are completely driven by the broker.
- */
-int runlevel_set_level (struct runlevel *r, int level);
-
-/* Get the current runlevel.
- */
-int runlevel_get_level (struct runlevel *r);
-
-/* Terminate current runlevel.
- * Asynchronously results in runlevel callback, so broker can advance state.
- * If runlevel has no subprocess, callback is immediate with rc=0.
- * Return 0 on success, -1 on failure.
- */
-int runlevel_abort (struct runlevel *r);
-
-#endif /* !_BROKER_RUNLEVEL_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/rusage.c b/src/broker/rusage.c
deleted file mode 100644
index 3ec7834c4fde..000000000000
--- a/src/broker/rusage.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <time.h>
-#include <sys/resource.h>
-#include <flux/core.h>
-#include "rusage.h"
-
-struct rusage_context {
-    flux_msg_handler_t *mh;
-};
-
-static void rusage_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                               const flux_msg_t *msg, void *arg)
-{
-    struct rusage ru;
-
-    if (flux_request_decode (msg, NULL, NULL) < 0) {
-        flux_log_error (h, "%s: flux_request_decode", __FUNCTION__);
-        return;
-    }
-    if (getrusage (RUSAGE_THREAD, &ru) < 0)
-        goto error;
-    if (flux_respond_pack (h, msg,
-            "{s:f s:f s:i s:i s:i s:i s:i s:i s:i s:i s:i s:i s:i s:i s:i s:i}",
-            "utime", (double)ru.ru_utime.tv_sec + 1E-6 * ru.ru_utime.tv_usec,
-            "stime", (double)ru.ru_stime.tv_sec + 1E-6 * ru.ru_stime.tv_usec,
-            "maxrss", ru.ru_maxrss,
-            "ixrss", ru.ru_ixrss,
-            "idrss", ru.ru_idrss,
-            "isrss", ru.ru_isrss,
-            "minflt", ru.ru_minflt,
-            "majflt", ru.ru_majflt,
-            "nswap", ru.ru_nswap,
-            "inblock", ru.ru_inblock,
-            "oublock", ru.ru_oublock,
-            "msgsnd", ru.ru_msgsnd,
-            "msgrcv", ru.ru_msgrcv,
-            "nsignals", ru.ru_nsignals,
-            "nvcsw", ru.ru_nvcsw,
-            "nivcsw", ru.ru_nivcsw) < 0)
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-static void rusage_finalize (void *arg)
-{
-    struct rusage_context *r = arg;
-    flux_msg_handler_stop (r->mh);
-    flux_msg_handler_destroy (r->mh);
-    free (r);
-}
-
-int rusage_initialize (flux_t *h, const char *service)
-{
-    struct flux_match match = FLUX_MATCH_ANY;
-    struct rusage_context *r = calloc (1, sizeof (*r));
-    if (!r) {
-        errno = ENOMEM;
-        goto error;
-    }
-    match.typemask = FLUX_MSGTYPE_REQUEST;
-    if (flux_match_asprintf (&match, "%s.rusage", service) < 0) {
-        errno = ENOMEM;
-        goto error;
-    }
-    if (!(r->mh = flux_msg_handler_create (h, match, rusage_request_cb, r)))
-        goto error;
-    flux_msg_handler_start (r->mh);
-    flux_aux_set (h, "flux::rusage", r, rusage_finalize);
-    flux_match_free (match);
-    return 0;
-error:
-    if (r)
-        rusage_finalize (r);
-    flux_match_free (match);
-    return -1;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/rusage.h b/src/broker/rusage.h
deleted file mode 100644
index ed26a8e3d967..000000000000
--- a/src/broker/rusage.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef BROKER_RUSAGE_H
-#define BROKER_RUSAGE_H
-
-#include <flux/core.h>
-
-int rusage_initialize (flux_t *h, const char *service);
-
-#endif /* BROKER_RUSAGE_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/broker/service.c b/src/broker/service.c
index 83786500b588..7f5e3c6553f7 100644
--- a/src/broker/service.c
+++ b/src/broker/service.c
@@ -11,10 +11,11 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
+#include "ccan/str/str.h"
 
 #include "service.h"
 
@@ -97,7 +98,8 @@ json_t *service_list_byuuid (struct service_switch *sw, const char *uuid)
         return NULL;
     svc = zhash_first (sw->services);
     while (svc) {
-        if (uuid && svc->uuid && !strcmp (uuid, svc->uuid)) {
+        if (uuid && svc->uuid
+            && streq (uuid, svc->uuid)) {
             json_t *name = json_string  (zhash_cursor (sw->services));
             if (!name)
                 goto error;
@@ -124,7 +126,8 @@ void service_remove_byuuid (struct service_switch *sw, const char *uuid)
 
     svc = zhash_first (sw->services);
     while (svc != NULL) {
-        if (svc->uuid && !strcmp (svc->uuid, uuid)) {
+        if (svc->uuid
+            && streq (svc->uuid, uuid)) {
             if (!trash)
                 trash = zlist_new ();
             if (!trash)
@@ -141,8 +144,11 @@ void service_remove_byuuid (struct service_switch *sw, const char *uuid)
     }
 }
 
-int service_add (struct service_switch *sh, const char *name,
-                 const char *uuid, service_send_f cb, void *arg)
+int service_add (struct service_switch *sh,
+                 const char *name,
+                 const char *uuid,
+                 service_send_f cb,
+                 void *arg)
 {
     struct service *svc = NULL;
 
@@ -157,10 +163,7 @@ int service_add (struct service_switch *sh, const char *name,
     svc = service_create (uuid);
     svc->cb = cb;
     svc->cb_arg = arg;
-    if (zhash_insert (sh->services, name, svc) < 0) {
-        errno = ENOMEM;
-        goto error;
-    }
+    (void)zhash_insert (sh->services, name, svc);
     zhash_freefn (sh->services, name, (zhash_free_fn *)service_destroy);
     return 0;
 error:
@@ -172,7 +175,8 @@ int service_add (struct service_switch *sh, const char *name,
  * Avoid an extra malloc here if the substring is short.
  */
 static struct service *service_lookup_subtopic (struct service_switch *sw,
-                                                const char *topic, int length)
+                                                const char *topic,
+                                                int length)
 {
     char buf[16];
     char *cpy = NULL;
@@ -202,13 +206,13 @@ static struct service *service_lookup_subtopic (struct service_switch *sw,
  * If found, call the service's callback and return its return value.
  * If not found, return -1 with errno set (usually ENOSYS).
  */
-int service_send (struct service_switch *sw, const flux_msg_t *msg)
+int service_send_new (struct service_switch *sw, flux_msg_t **msg)
 {
     const char *topic, *p;
     int length;
     struct service *svc;
 
-    if (flux_msg_get_topic (msg, &topic) < 0)
+    if (flux_msg_get_topic (*msg, &topic) < 0)
         return -1;
     if ((p = strchr (topic, '.')))
         length = p - topic;
diff --git a/src/broker/service.h b/src/broker/service.h
index d4aa6103805a..6262d42cbaef 100644
--- a/src/broker/service.h
+++ b/src/broker/service.h
@@ -13,19 +13,22 @@
 
 #include <jansson.h>
 
-typedef int (*service_send_f)(const flux_msg_t *msg, void *arg);
+typedef int (*service_send_f)(flux_msg_t **msg, void *arg);
 
 struct service_switch *service_switch_create (void);
 void service_switch_destroy (struct service_switch *sw);
 
-int service_add (struct service_switch *sw, const char *name,
-                 const char *uuid, service_send_f cb, void *arg);
+int service_add (struct service_switch *sw,
+                 const char *name,
+                 const char *uuid,
+                 service_send_f cb,
+                 void *arg);
 
 void service_remove (struct service_switch *sw, const char *name);
 
 void service_remove_byuuid (struct service_switch *sw, const char *uuid);
 
-int service_send (struct service_switch *sw, const flux_msg_t *msg);
+int service_send_new (struct service_switch *sw, flux_msg_t **msg);
 
 /* Return the UUID currently registered for service `name` */
 const char *service_get_uuid (struct service_switch *sw, const char *name);
diff --git a/src/broker/shutdown.c b/src/broker/shutdown.c
index 1fdbf1741a67..762720e848c8 100644
--- a/src/broker/shutdown.c
+++ b/src/broker/shutdown.c
@@ -1,5 +1,5 @@
 /************************************************************\
- * Copyright 2020 Lawrence Livermore National Security, LLC
+ * Copyright 2022 Lawrence Livermore National Security, LLC
  * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
  *
  * This file is part of the Flux resource manager framework.
@@ -8,216 +8,283 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-/* shutdown.c - orderly distributed shutdown
+/* shutdown.c - manage instance shutdown on behalf of flux-shutdown(1)
  *
- * We must keep the overlay network up long enough to be sure every broker
- * is informed about the shutdown.  Here is the protocol:
- *
- * 1. Rank 0 broker publishes a shutdown event, to which all brokers subscribe
- * 2. Each broker (including 0) handles the event.
- * 3. Each broker starts a doomsday timer
- * 4. If a broker has descendants, it waits for them to disconnect
- * 5. Once all descendants disconnect, broker stops timer and calls callback.
- * 6. The callback should disconnect from parent (if any) and exit.
- * 7. Ultimately the rank 0 brokers exits with exit code of rc1/2/3.
- *
- * The rank 0 broker calls shutdown_instance().
- * All ranks register a shutdown callback which is called on state change, e.g.
- * - when shutdown begins: !shutdown_is_complete() and !shutdown_is_expired()
- * - when shutdown completes: shutdown_is_compete() and !shutdown_is_expired()
- * - when shutdown expires: shutdown_is_expired()
+ * This is only active on rank 0.
+ * On rank 0, this posts the "goodbye" event to the broker state machine.
+ * On other ranks it is generated internally in state_machine.c.
  */
 
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <errno.h>
-#include <assert.h>
 #include <flux/core.h>
 
-#include "src/common/libutil/kary.h"
+#include "src/common/libutil/stdlog.h"
 
-#include "overlay.h"
 #include "shutdown.h"
+#include "state_machine.h"
 
+#include "broker.h"
 
 struct shutdown {
-    flux_t *h;
-    overlay_t *overlay;
+    struct broker *ctx;
     flux_msg_handler_t **handlers;
-    flux_watcher_t *timer;
-    double grace;
-    bool expired;
-    bool complete;
 
-    shutdown_cb_f cb;
-    void *arg;
+    flux_future_t *f_monitor;
+    broker_state_t state;
+
+    flux_future_t *f_dmesg;
+
+    const flux_msg_t *request; // single flux-shutdown(1) client
 };
 
-static void shutdown_begin (struct shutdown *s)
+static void dmesg_cancel (flux_future_t *f);
+
+static void check_for_completion (struct shutdown *shutdown)
 {
-    assert (s->complete == false);
-    assert (s->expired == false);
-    if (s->cb)
-        s->cb (s, s->arg);
+    flux_t *h = shutdown->ctx->h;
+
+    if (shutdown->state != STATE_GOODBYE)
+        return;
+    if (shutdown->f_dmesg) {
+        dmesg_cancel (shutdown->f_dmesg);
+        return;
+    }
+    if (shutdown->request) {
+        if (flux_respond_error (h, shutdown->request, ENODATA, NULL) < 0)
+            flux_log_error (h, "error responding to shutdown.start");
+        return;
+    }
+    state_machine_post (shutdown->ctx->state_machine, "goodbye");
 }
 
-static void shutdown_complete (struct shutdown *s)
+static int forward_logbuf (flux_t *h,
+                           const flux_msg_t *request,
+                           const char *stdlog)
 {
-    s->complete = true;
-    flux_watcher_stop (s->timer);
-    overlay_set_monitor_cb (s->overlay, NULL, NULL);
-    if (s->cb)
-        s->cb (s, s->arg);
+    struct stdlog_header hdr;
+    const char *txt;
+    size_t txtlen;
+    char buf[FLUX_MAX_LOGBUF];
+    int loglevel;
+
+    if (flux_msg_unpack (request, "{s:i}", "loglevel", &loglevel) < 0)
+        loglevel = LOG_ERR;
+
+    if (stdlog_decode (stdlog,
+                       strlen (stdlog),
+                       &hdr,
+                       NULL,
+                       NULL,
+                       &txt,
+                       &txtlen) < 0
+        || STDLOG_SEVERITY (hdr.pri) > loglevel
+        || snprintf (buf,
+                     sizeof (buf),
+                     "%s.%s[%lu]: %.*s\n",
+                     hdr.appname,
+                     stdlog_severity_to_string (STDLOG_SEVERITY (hdr.pri)),
+                     strtoul (hdr.hostname, NULL, 10),
+                     (int)txtlen,
+                     txt) >= sizeof (buf))
+        return 0;
+    return flux_respond_pack (h, request, "{s:s}", "log", buf);
 }
 
-static void shutdown_timeout (struct shutdown *s)
+static void dmesg_continuation (flux_future_t *f, void *arg)
 {
-    s->expired = true;
-    overlay_set_monitor_cb (s->overlay, NULL, NULL);
-    if (s->cb)
-        s->cb (s, s->arg);
+    struct shutdown *shutdown = arg;
+    flux_t *h = flux_future_get_flux (f);
+    const char *buf;
+
+    if (flux_rpc_get (f, &buf) < 0) {
+        if (errno != ENODATA)
+            flux_log_error (h, "shutdown: log.dmesg");
+        flux_future_destroy (f);
+        shutdown->f_dmesg = NULL;
+        check_for_completion (shutdown);
+        return;
+    }
+    if (shutdown->request) {
+        if (forward_logbuf (h, shutdown->request, buf) < 0)
+            flux_log_error (h, "error responding to shutdown.start");
+    }
+    flux_future_reset (f);
 }
 
-static void grace_timeout_cb (flux_reactor_t *r,
-                              flux_watcher_t *w,
-                              int revents,
-                              void *arg)
+static void dmesg_cancel (flux_future_t *f)
 {
-    struct shutdown *s = arg;
-    shutdown_timeout (s);
+    flux_t *h = flux_future_get_flux (f);
+    uint32_t matchtag = flux_rpc_get_matchtag (f);
+    flux_future_t *f_cancel;
+
+    if (!(f_cancel = flux_rpc_pack (h,
+                                    "log.cancel",
+                                    FLUX_NODEID_ANY,
+                                    FLUX_RPC_NORESPONSE,
+                                    "{s:i}",
+                                    "matchtag", matchtag)))
+        flux_log_error (h, "shutdown: error sending dmesg.cancel RPC");
+    flux_future_destroy (f_cancel);
 }
 
-void monitor_cb (overlay_t *overlay, void *arg)
+static flux_future_t *dmesg_request (struct shutdown *shutdown)
 {
-    struct shutdown *s = arg;
-    if (overlay_get_child_peer_count (overlay) == 0)
-        shutdown_complete (s);
+    flux_future_t *f;
+
+    if (!(f = flux_rpc_pack (shutdown->ctx->h,
+                             "log.dmesg",
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_STREAMING,
+                             "{s:b s:b}",
+                             "follow", 1,
+                             "nobacklog", 1))
+        || flux_future_then (f,
+                             -1,
+                             dmesg_continuation,
+                             shutdown) < 0) {
+        flux_future_destroy (f);
+        return NULL;
+    }
+    return f;
 }
 
-void shutdown_event_cb (flux_t *h,
-                        flux_msg_handler_t *mh,
-                        const flux_msg_t *msg,
-                        void *arg)
+static void start_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
-    struct shutdown *s = arg;
+    struct shutdown *shutdown = arg;
+    flux_error_t error;
+    const char *errmsg = NULL;
 
-    shutdown_begin (s);
-    if (overlay_get_child_peer_count (s->overlay) == 0)
-        shutdown_complete (s);
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (shutdown->request) {
+        errno = EINVAL;
+        errmsg = "shutdown is already in progress";
+        goto error;
+    }
+    if (state_machine_shutdown (shutdown->ctx->state_machine, &error) < 0) {
+        errmsg = error.text;
+        goto error;
+    }
+    if (flux_msg_is_streaming (msg)) {
+        if (!(shutdown->f_dmesg)) {
+            if (!(shutdown->f_dmesg = dmesg_request (shutdown))) {
+                errmsg = "error requesting to follow log messages";
+                goto error;
+            }
+        }
+        shutdown->request = flux_msg_incref (msg);
+    }
     else {
-        overlay_set_monitor_cb (s->overlay, monitor_cb, s);
-        flux_watcher_start (s->timer);
+        if (flux_respond (h, msg, NULL) < 0)
+            flux_log_error (h, "error responding to shutdown.start");
     }
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to shutdown.start");
 }
 
-void publish_continuation (flux_future_t *f, void *arg)
+static void monitor_continuation (flux_future_t *f, void *arg)
 {
-    struct shutdown *s = arg;
+    flux_t *h = flux_future_get_flux (f);
+    struct shutdown *shutdown = arg;
 
-    if (flux_future_get (f, NULL) < 0)
-        flux_log_error (s->h, "publishing shutdown event");
-    flux_future_destroy (f);
+    if (flux_rpc_get_unpack (f, "{s:i}", "state", &shutdown->state) < 0) {
+        if (errno != ENODATA)
+            flux_log_error (h, "shutdown: state-machine.monitor");
+        flux_future_destroy (f);
+        shutdown->f_monitor = NULL;
+        check_for_completion (shutdown);
+        return;
+    }
+    flux_future_reset (f);
 }
 
-/* Called from rank 0 only */
-void shutdown_instance (struct shutdown *s)
+static flux_future_t *monitor_request (struct shutdown *shutdown)
 {
-    if (overlay_get_child_peer_count (s->overlay) == 0) {
-        shutdown_begin (s);
-        shutdown_complete (s);
-    }
-    else {
-        flux_future_t *f;
+    flux_future_t *f;
 
-        if (!(f = flux_event_publish (s->h, "shutdown", 0, NULL))) {
-            flux_log_error (s->h, "publishing shutdown event");
-            return;
-        }
-        if (flux_future_then (f, -1., publish_continuation, s) < 0) {
-            flux_log_error (s->h, "registering continuation for shutdown");
-            flux_future_destroy (f);
-            return;
-        }
+    if (!(f = flux_rpc_pack (shutdown->ctx->h,
+                             "state-machine.monitor",
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_STREAMING,
+                             "{s:i}",
+                             "final", STATE_GOODBYE))
+        || flux_future_then (f, -1, monitor_continuation, shutdown) < 0) {
+        flux_future_destroy (f);
+        return NULL;
     }
+    return f;
 }
 
-void shutdown_set_callback (struct shutdown *s, shutdown_cb_f cb, void *arg)
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
 {
-    s->cb = cb;
-    s->arg = arg;
-}
+    struct shutdown *shutdown = arg;
 
-bool shutdown_is_expired (struct shutdown *s)
-{
-    return s->expired;
+    if (shutdown->request && flux_disconnect_match (msg, shutdown->request)) {
+        flux_msg_decref (shutdown->request);
+        shutdown->request = NULL;
+        check_for_completion (shutdown);
+    }
 }
 
-bool shutdown_is_complete (struct shutdown *s)
-{
-    return s->complete;
-}
+static const struct flux_msg_handler_spec htab[] = {
+    {   FLUX_MSGTYPE_REQUEST,
+        "shutdown.disconnect",
+        disconnect_cb,
+        0
+    },
+    {    FLUX_MSGTYPE_REQUEST,
+        "shutdown.start",
+        start_cb,
+        0,
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
 
-void shutdown_destroy (struct shutdown *s)
+void shutdown_destroy (struct shutdown *shutdown)
 {
-    if (s) {
+    if (shutdown) {
         int saved_errno = errno;
-        flux_msg_handler_delvec (s->handlers);
-        if (s->h)
-            (void)flux_event_unsubscribe (s->h, "shutdown");
-        flux_watcher_destroy (s->timer);
-        free (s);
+        flux_msg_decref (shutdown->request);
+        flux_msg_handler_delvec (shutdown->handlers);
+        flux_future_destroy (shutdown->f_dmesg);
+        flux_future_destroy (shutdown->f_monitor);
+        free (shutdown);
         errno = saved_errno;
     }
 }
 
-static const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_EVENT,  "shutdown", shutdown_event_cb, 0 },
-    FLUX_MSGHANDLER_TABLE_END,
-};
-
-struct shutdown *shutdown_create (flux_t *h,
-                                  double grace,
-                                  uint32_t size,
-                                  int tbon_k,
-                                  overlay_t *overlay)
+struct shutdown *shutdown_create (struct broker *ctx)
 {
-    struct shutdown *s;
+    struct shutdown *shutdown;
 
-    if (!(s = calloc (1, sizeof (*s))))
+    if (!(shutdown = calloc (1, sizeof (*shutdown))))
         return NULL;
-    s->h = h;
-    s->overlay = overlay;
-
-    /* If grace is zero, select a default based on the maximum number of
-     * TBON levels in the instance since the latency at rank 0 will be based
-     * on the number of hops from the most distant leaf node.
-     * N.B. a size=1 instance has 1 level.
-     */
-    if (grace == 0) {
-        int levels = kary_levelof (tbon_k, size - 1) + 1;
-        s->grace = levels * 2; // e.g. 2s for size=1, 4s for size=3 k=2
-    }
-    else
-        s->grace = grace;
-
-    if (flux_msg_handler_addvec (h, htab, s, &s->handlers) < 0)
-        goto error;
-    if (!(s->timer = flux_timer_watcher_create (flux_get_reactor (h),
-                                                s->grace,
-                                                0.,
-                                                grace_timeout_cb,
-                                                s)))
+    shutdown->ctx = ctx;
+    if (flux_msg_handler_addvec (ctx->h,
+                                    htab,
+                                    shutdown,
+                                    &shutdown->handlers) < 0)
         goto error;
-    if (flux_event_subscribe (s->h, "shutdown") < 0)
-        goto error;
-    return s;
+    if (ctx->rank == 0) {
+        if (!(shutdown->f_monitor = monitor_request (shutdown)))
+            return NULL;
+    }
+    return shutdown;
 error:
-    shutdown_destroy (s);
+    shutdown_destroy (shutdown);
     return NULL;
 }
 
-
 /*
- * vi:tabstop=4 shiftwidth=4 expandtab
+ * vi:ts=4 sw=4 expandtab
  */
diff --git a/src/broker/shutdown.h b/src/broker/shutdown.h
index 45191ab0dee4..84ec87e77cfb 100644
--- a/src/broker/shutdown.h
+++ b/src/broker/shutdown.h
@@ -1,5 +1,5 @@
 /************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
+ * Copyright 2022 Lawrence Livermore National Security, LLC
  * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
  *
  * This file is part of the Flux resource manager framework.
@@ -11,26 +11,13 @@
 #ifndef _BROKER_SHUTDOWN_H
 #define _BROKER_SHUTDOWN_H
 
-struct shutdown;
+struct broker;
 
-typedef void (*shutdown_cb_f) (struct shutdown *s, void *arg);
-
-struct shutdown *shutdown_create (flux_t *h,
-                                  double grace,
-                                  uint32_t size,
-                                  int tbon_k,
-                                  overlay_t *overlay);
-void shutdown_destroy (struct shutdown *s);
-
-void shutdown_set_callback (struct shutdown *s, shutdown_cb_f cb, void *arg);
-
-bool shutdown_is_complete (struct shutdown *s);
-bool shutdown_is_expired (struct shutdown *s);
-
-void shutdown_instance (struct shutdown *s);
+struct shutdown *shutdown_create (struct broker *ctx);
+void shutdown_destroy (struct shutdown *shutdown);
 
 #endif /* !_BROKER_SHUTDOWN_H */
 
 /*
- * vi:tabstop=4 shiftwidth=4 expandtab
+ * vi:ts=4 sw=4 expandtab
  */
diff --git a/src/broker/state_machine.c b/src/broker/state_machine.c
new file mode 100644
index 000000000000..ce5a3fc9adfd
--- /dev/null
+++ b/src/broker/state_machine.c
@@ -0,0 +1,1345 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#include <signal.h>
+#endif
+#if HAVE_LIBSYSTEMD
+#include <systemd/sd-daemon.h>
+#endif
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/monotime.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libidset/idset.h"
+#include "src/common/libhostlist/hostlist.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libsubprocess/server.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+
+#include "state_machine.h"
+
+#include "broker.h"
+#include "runat.h"
+#include "overlay.h"
+#include "attr.h"
+#include "groups.h"
+#include "shutdown.h"
+
+struct quorum {
+    uint32_t size;
+    struct idset *all;
+    struct idset *online; // cumulative on rank 0, batch buffer on rank > 0
+    flux_future_t *f;
+    double timeout;
+    bool warned;
+    flux_watcher_t *timer;
+};
+
+struct cleanup {
+    bool expedite;
+    double timeout;
+    flux_watcher_t *timer;
+};
+
+struct monitor {
+    struct flux_msglist *requests;
+
+    flux_future_t *f;
+    broker_state_t parent_state;
+    unsigned int parent_valid:1;
+    unsigned int parent_error:1;
+};
+
+struct state_machine {
+    struct broker *ctx;
+    broker_state_t state;
+    struct timespec t_start;
+
+    zlist_t *events;
+    flux_watcher_t *prep;
+    flux_watcher_t *check;
+    flux_watcher_t *idle;
+
+    flux_msg_handler_t **handlers;
+
+    struct monitor monitor;
+    struct quorum quorum;
+    struct cleanup cleanup;
+
+    struct flux_msglist *wait_requests;
+
+    int exit_norestart;
+};
+
+typedef void (*action_f)(struct state_machine *s);
+
+struct state {
+    broker_state_t state;
+    const char *name;
+    action_f action;
+};
+
+struct state_next {
+    const char *event;
+    broker_state_t current;
+    broker_state_t next;
+};
+
+static void action_join (struct state_machine *s);
+static void action_quorum (struct state_machine *s);
+static void action_init (struct state_machine *s);
+static void action_run (struct state_machine *s);
+static void action_cleanup (struct state_machine *s);
+static void action_shutdown (struct state_machine *s);
+static void action_finalize (struct state_machine *s);
+static void action_goodbye (struct state_machine *s);
+static void action_exit (struct state_machine *s);
+
+static void runat_completion_cb (struct runat *r, const char *name, void *arg);
+static void monitor_update (flux_t *h,
+                            struct flux_msglist *requests,
+                            broker_state_t state);
+static void wait_update (flux_t *h,
+                         struct flux_msglist *requests,
+                         broker_state_t state);
+static void join_check_parent (struct state_machine *s);
+static void quorum_check_parent (struct state_machine *s);
+static void run_check_parent (struct state_machine *s);
+
+static struct state statetab[] = {
+    { STATE_NONE,       "none",             NULL },
+    { STATE_JOIN,       "join",             action_join },
+    { STATE_INIT,       "init",             action_init },
+    { STATE_QUORUM,     "quorum",           action_quorum },
+    { STATE_RUN,        "run",              action_run },
+    { STATE_CLEANUP,    "cleanup",          action_cleanup },
+    { STATE_SHUTDOWN,   "shutdown",         action_shutdown },
+    { STATE_FINALIZE,   "finalize",         action_finalize },
+    { STATE_GOODBYE,    "goodbye",          action_goodbye },
+    { STATE_EXIT,       "exit",             action_exit },
+};
+
+static struct state_next nexttab[] = {
+    { "start",              STATE_NONE,         STATE_JOIN },
+    { "parent-ready",       STATE_JOIN,         STATE_INIT },
+    { "parent-none",        STATE_JOIN,         STATE_INIT },
+    { "parent-fail",        STATE_JOIN,         STATE_SHUTDOWN },
+    { "rc1-success",        STATE_INIT,         STATE_QUORUM },
+    { "rc1-none",           STATE_INIT,         STATE_QUORUM },
+    { "rc1-ignorefail",     STATE_INIT,         STATE_QUORUM },
+    { "rc1-fail",           STATE_INIT,         STATE_SHUTDOWN},
+    { "quorum-full",        STATE_QUORUM,       STATE_RUN },
+    { "quorum-fail",        STATE_QUORUM,       STATE_SHUTDOWN},
+    { "rc2-success",        STATE_RUN,          STATE_CLEANUP },
+    { "rc2-fail",           STATE_RUN,          STATE_CLEANUP },
+    { "shutdown",           STATE_RUN,          STATE_CLEANUP },
+    { "rc2-none",           STATE_RUN,          STATE_RUN },
+    { "cleanup-success",    STATE_CLEANUP,      STATE_SHUTDOWN },
+    { "cleanup-none",       STATE_CLEANUP,      STATE_SHUTDOWN },
+    { "cleanup-fail",       STATE_CLEANUP,      STATE_SHUTDOWN },
+    { "children-complete",  STATE_SHUTDOWN,     STATE_FINALIZE },
+    { "children-none",      STATE_SHUTDOWN,     STATE_FINALIZE },
+    { "rc3-success",        STATE_FINALIZE,     STATE_GOODBYE },
+    { "rc3-none",           STATE_FINALIZE,     STATE_GOODBYE },
+    { "rc3-fail",           STATE_FINALIZE,     STATE_GOODBYE },
+    { "goodbye",            STATE_GOODBYE,      STATE_EXIT },
+};
+
+static const double default_quorum_timeout = 60; // log slow joiners
+static const double default_cleanup_timeout = -1;
+static const double goodbye_timeout = 60;
+
+static void state_action (struct state_machine *s, broker_state_t state)
+{
+    int i;
+    for (i = 0; i < ARRAY_SIZE (statetab); i++) {
+        if (statetab[i].state == state) {
+            if (statetab[i].action)
+                statetab[i].action (s);
+            break;
+        }
+    }
+}
+
+static const char *statestr (broker_state_t state)
+{
+    int i;
+    for (i = 0; i < ARRAY_SIZE (statetab); i++) {
+        if (statetab[i].state == state)
+            return statetab[i].name;
+    }
+    return "unknown";
+}
+
+static broker_state_t state_next (broker_state_t current, const char *event)
+{
+    int i;
+    for (i = 0; i < ARRAY_SIZE (nexttab); i++) {
+        if (nexttab[i].current == current
+            && streq (event, nexttab[i].event))
+            return nexttab[i].next;
+    }
+    return current;
+}
+
+static void action_init (struct state_machine *s)
+{
+    s->ctx->online = true;
+    if (runat_is_defined (s->ctx->runat, "rc1")) {
+        if (runat_start (s->ctx->runat, "rc1", runat_completion_cb, s) < 0) {
+            flux_log_error (s->ctx->h, "runat_start rc1");
+            state_machine_post (s, "rc1-fail");
+        }
+    }
+    else
+        state_machine_post (s, "rc1-none");
+}
+
+static void action_join (struct state_machine *s)
+{
+    if (s->ctx->rank == 0)
+        state_machine_post (s, "parent-none");
+    else {
+#if HAVE_LIBSYSTEMD
+        if (s->ctx->sd_notify) {
+            sd_notifyf (0,
+                        "STATUS=Joining Flux instance via %s",
+                        overlay_get_parent_uri (s->ctx->overlay));
+        }
+#endif
+        join_check_parent (s);
+    }
+#if HAVE_LIBSYSTEMD
+    sd_notify (0, "READY=1");
+#endif
+}
+
+static void quorum_timer_cb (flux_reactor_t *r,
+                             flux_watcher_t *w,
+                             int revents,
+                             void *arg)
+{
+    struct state_machine *s = arg;
+    flux_t *h = s->ctx->h;
+    struct idset *ids = NULL;
+    char *rankstr = NULL;
+    struct hostlist *hl = NULL;
+    char *hoststr = NULL;
+    unsigned int rank;
+
+    if (s->state != STATE_QUORUM)
+        return;
+
+    if (!(ids = idset_difference (s->quorum.all, s->quorum.online))
+        || !(rankstr = idset_encode (ids, IDSET_FLAG_RANGE))
+        || !(hl = hostlist_create ())) {
+        flux_log_error (h, "error computing slow brokers");
+        goto done;
+    }
+    rank = idset_first (ids);
+    while (rank != IDSET_INVALID_ID) {
+        if (hostlist_append (hl, flux_get_hostbyrank (s->ctx->h, rank)) < 0) {
+            flux_log_error (h, "error building slow brokers hostlist");
+            goto done;
+        }
+        rank = idset_next (ids, rank);
+    }
+    if (!(hoststr = hostlist_encode (hl))) {
+        flux_log_error (h, "error encoding slow brokers hostlist");
+        goto done;
+    }
+    flux_log (s->ctx->h,
+              LOG_ERR,
+              "quorum delayed: waiting for %s (rank %s)",
+              hoststr,
+              rankstr);
+    flux_timer_watcher_reset (w, s->quorum.timeout, 0.);
+    flux_watcher_start (w);
+    s->quorum.warned = true;
+done:
+    free (hoststr);
+    hostlist_destroy (hl);
+    free (rankstr);
+    idset_destroy (ids);
+}
+
+static void action_quorum_continuation (flux_future_t *f, void *arg)
+{
+    struct state_machine *s = arg;
+
+    if (flux_rpc_get (f, NULL) < 0)
+        state_machine_post (s, "quorum-fail");
+    flux_future_destroy (f);
+}
+
+static void action_quorum (struct state_machine *s)
+{
+    flux_future_t *f;
+
+#if HAVE_LIBSYSTEMD
+    if (s->ctx->sd_notify)
+        sd_notify (0, "STATUS=Waiting for instance quorum");
+#endif
+    if (!(f = flux_rpc_pack (s->ctx->h,
+                             "groups.join",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:s}",
+                             "name", "broker.online"))
+        || flux_future_then (f, -1, action_quorum_continuation, s) < 0) {
+        flux_future_destroy (f);
+        state_machine_post (s, "quorum-fail");
+        return;
+    }
+    if (s->ctx->rank > 0)
+        quorum_check_parent (s);
+    else if (s->quorum.timeout > 0.) {
+        flux_timer_watcher_reset (s->quorum.timer, s->quorum.timeout, 0.);
+        flux_watcher_start (s->quorum.timer);
+    }
+}
+
+static void action_run (struct state_machine *s)
+{
+    struct broker_attr *attrs = s->ctx->attrs;
+
+    if (runat_is_defined (s->ctx->runat, "rc2")) {
+        if (attr_get (attrs, "broker.recovery-mode", NULL, NULL) == 0
+            && runat_is_interactive (s->ctx->runat, "rc2")) {
+            const char *confdir = "unknown";
+            const char *rc1_path = "unknown";
+            const char *rc3_path = "unknown";
+            const char *statedir = "unknown";
+
+            (void)attr_get (attrs, "config.path", &confdir, NULL);
+            (void)attr_get (attrs, "broker.rc1_path", &rc1_path, NULL);
+            (void)attr_get (attrs, "broker.rc3_path", &rc3_path, NULL);
+            (void)attr_get (attrs, "statedir", &statedir, NULL);
+
+            printf ("+-----------------------------------------------------\n"
+                    "| Entering Flux recovery mode.\n"
+                    "| All resources will be offline during recovery.\n"
+                    "| Any rc1 failures noted above may result in\n"
+                    "|  reduced functionality until manually corrected.\n"
+                    "|\n"
+                    "| broker.rc1_path    %s\n"
+                    "| broker.rc3_path    %s\n"
+                    "| config.path        %s\n"
+                    "| statedir           %s\n"
+                    "|\n"
+                    "| Exit this shell when finished.\n"
+                    "+-----------------------------------------------------\n",
+                    rc1_path ? rc1_path : "-",
+                    rc3_path ? rc3_path : "-",
+                    confdir ? confdir : "-",
+                    statedir ? statedir
+                        : "changes will not be preserved");
+        }
+        if (runat_start (s->ctx->runat, "rc2", runat_completion_cb, s) < 0) {
+            flux_log_error (s->ctx->h, "runat_start rc2");
+            state_machine_post (s, "rc2-fail");
+        }
+    }
+    else if (s->ctx->rank > 0)
+        run_check_parent (s);
+    else
+        state_machine_post (s, "rc2-none");
+
+#if HAVE_LIBSYSTEMD
+    if (s->ctx->sd_notify) {
+        sd_notifyf (0,
+                    "STATUS=Running as %s of %d node Flux instance",
+                    s->ctx->rank == 0 ? "leader" : "member",
+                    (int)s->ctx->size);
+    }
+#endif
+}
+
+/* Abort the cleanup script if cleanup.timeout expires while it is running.
+ */
+static void cleanup_timer_cb (flux_reactor_t *r,
+                              flux_watcher_t *w,
+                              int revents,
+                              void *arg)
+{
+    struct state_machine *s = arg;
+
+    if (s->state == STATE_CLEANUP)
+        (void)runat_abort (s->ctx->runat, "cleanup");
+}
+
+static void action_cleanup (struct state_machine *s)
+{
+    /* Prevent new downstream clients from saying hello, but
+     * let existing ones continue to communicate so they can
+     * shut down and disconnect.
+     */
+    overlay_shutdown (s->ctx->overlay, false);
+
+    if (runat_is_defined (s->ctx->runat, "cleanup")) {
+        if (runat_start (s->ctx->runat, "cleanup", runat_completion_cb, s) < 0) {
+            flux_log_error (s->ctx->h, "runat_start cleanup");
+            state_machine_post (s, "cleanup-fail");
+        }
+        /* If the broker is shutting down on a terminating signal,
+         * impose a timeout on the cleanup script.
+         * See flux-framework/flux-core#6388.
+         */
+        if (s->cleanup.expedite && s->cleanup.timeout >= 0) {
+            flux_timer_watcher_reset (s->cleanup.timer, s->cleanup.timeout, 0.);
+            flux_watcher_start (s->cleanup.timer);
+        }
+    }
+    else
+        state_machine_post (s, "cleanup-none");
+}
+
+static void action_finalize (struct state_machine *s)
+{
+    /* Now that all clients have disconnected, finalize all
+     * downstream communication.
+     */
+    overlay_shutdown (s->ctx->overlay, true);
+
+    if (runat_is_defined (s->ctx->runat, "rc3")) {
+        if (runat_start (s->ctx->runat, "rc3", runat_completion_cb, s) < 0) {
+            flux_log_error (s->ctx->h, "runat_start rc3");
+            state_machine_post (s, "rc3-fail");
+        }
+    }
+    else
+        state_machine_post (s, "rc3-none");
+}
+
+static void action_shutdown (struct state_machine *s)
+{
+    if (overlay_get_child_peer_count (s->ctx->overlay) == 0)
+        state_machine_post (s, "children-none");
+#if HAVE_LIBSYSTEMD
+    if (s->ctx->sd_notify) {
+        sd_notifyf (0,
+                    "STATUS=Waiting for %d peers to shutdown",
+                    overlay_get_child_peer_count (s->ctx->overlay));
+    }
+#endif
+}
+
+static void goodbye_continuation (flux_future_t *f, void *arg)
+{
+    struct state_machine *s = arg;
+    if (flux_future_get (f, NULL) < 0) {
+        flux_log (s->ctx->h,
+                  LOG_ERR,
+                  "overlay.goodbye: %s",
+                  future_strerror (f, errno));
+    }
+    state_machine_post (s, "goodbye");
+    flux_future_destroy (f);
+}
+
+static void action_goodbye (struct state_machine *s)
+{
+    /* On rank 0, "goodbye" is posted by shutdown.c.
+     * On other ranks, send a goodbye message and wait for a response
+     * (with timeout) before continuing on.
+     */
+    if (s->ctx->rank > 0) {
+        flux_future_t *f;
+        if (!(f = overlay_goodbye_parent (s->ctx->overlay))
+            || flux_future_then (f,
+                                 goodbye_timeout,
+                                 goodbye_continuation,
+                                 s) < 0) {
+            flux_log_error (s->ctx->h, "error sending overlay.goodbye request");
+            flux_future_destroy (f);
+            state_machine_post (s, "goodbye");
+        }
+    }
+}
+
+static void rmmod_continuation (flux_future_t *f, void *arg)
+{
+    struct state_machine *s = arg;
+
+    if (flux_rpc_get (f, NULL) < 0)
+        flux_log_error (s->ctx->h, "module.remove connector-local");
+    flux_reactor_stop (flux_get_reactor (s->ctx->h));
+    flux_future_destroy (f);
+}
+
+static void subproc_continuation (flux_future_t *f, void *arg)
+{
+    struct state_machine *s = arg;
+    flux_t *h = s->ctx->h;
+
+    /* Log any subprocess shutdown timeout, then cause the subprocess server's
+     * destructor to be invoked by removing it from the flux_t aux container.
+     * Any remaining processes will get a SIGKILL.
+     */
+    if (flux_future_get (f, NULL) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "timed out waiting for subprocesses to exit on SIGTERM");
+    }
+    flux_future_destroy (f);
+    flux_aux_set (h, "flux::exec", NULL, NULL);
+
+    /* Next task is to remove the connector-local module.
+     */
+    if (!(f = flux_rpc_pack (h,
+                             "module.remove",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:s}",
+                             "name",
+                             "connector-local"))
+        || flux_future_then (f, -1, rmmod_continuation, s) < 0) {
+        flux_log_error (h, "error sending module.remove connector-local");
+        flux_reactor_stop (flux_get_reactor (h));
+        flux_future_destroy (f);
+    }
+}
+
+/* Stop all subprocesses, then unload the connector-local module,
+ * then stop the broker's reactor.
+ */
+static void action_exit (struct state_machine *s)
+{
+    flux_t *h = s->ctx->h;
+    subprocess_server_t *subserv = flux_aux_get (h, "flux::exec");
+    flux_future_t *f;
+
+    /* Send a SIGTERM to all procs.  The continuation is called after a 5s
+     * timeout or when all subprocesses are cleaned up.
+     */
+    if (!(f = subprocess_server_shutdown (subserv, SIGTERM))
+        || flux_future_then (f, 5., subproc_continuation, s) < 0) {
+        flux_log_error (h, "error initiating subprocess server shutdown");
+        flux_reactor_stop (flux_get_reactor (h));
+        flux_future_destroy (f);
+    }
+#if HAVE_LIBSYSTEMD
+    if (s->ctx->sd_notify)
+        sd_notify (0, "STATUS=Exiting");
+#endif
+}
+
+static void process_event (struct state_machine *s, const char *event)
+{
+    broker_state_t next_state;
+
+    next_state = state_next (s->state, event);
+
+    if (next_state != s->state) {
+        char fsd[64];
+        fsd_format_duration (fsd,
+                             sizeof (fsd),
+                             monotime_since (s->t_start) * 1E-3);
+        flux_log (s->ctx->h,
+                  LOG_INFO, "%s: %s->%s %s",
+                  event,
+                  statestr (s->state),
+                  statestr (next_state),
+                  fsd);
+        monotime (&s->t_start);
+        s->state = next_state;
+        state_action (s, s->state);
+        monitor_update (s->ctx->h, s->monitor.requests, s->state);
+        wait_update (s->ctx->h, s->wait_requests, s->state);
+    }
+    else {
+        flux_log (s->ctx->h,
+                  LOG_DEBUG,
+                  "%s: ignored in %s",
+                  event,
+                  statestr (s->state));
+    }
+}
+
+void state_machine_post (struct state_machine *s, const char *event)
+{
+    if (zlist_append (s->events, (char *)event) < 0)
+        flux_log (s->ctx->h, LOG_ERR, "state_machine_post %s failed", event);
+}
+
+void state_machine_kill (struct state_machine *s, int signum)
+{
+    flux_t *h = s->ctx->h;
+
+    s->cleanup.expedite = true;
+
+    switch (s->state) {
+        case STATE_INIT:
+            if (runat_abort (s->ctx->runat, "rc1") < 0)
+                flux_log_error (h, "runat_abort rc1 (signal %d)", signum);
+            break;
+        case STATE_JOIN:
+            state_machine_post (s, "parent-fail");
+            break;
+        case STATE_QUORUM:
+            state_machine_post (s, "quorum-fail");
+            break;
+        case STATE_RUN:
+            if (runat_is_defined (s->ctx->runat, "rc2")) {
+                if (runat_abort (s->ctx->runat, "rc2") < 0)
+                    flux_log_error (h, "runat_abort rc2 (signal %d)", signum);
+            }
+            else
+                state_machine_post (s, "shutdown");
+            break;
+        case STATE_CLEANUP:
+            if (!runat_is_defined (s->ctx->runat, "cleanup"))
+                break;
+            if (runat_abort (s->ctx->runat, "cleanup") < 0)
+                flux_log_error (h, "runat_abort cleanup (signal %d)", signum);
+            break;
+        case STATE_FINALIZE:
+            (void)runat_abort (s->ctx->runat, "rc3");
+            break;
+        case STATE_NONE:
+        case STATE_SHUTDOWN:
+        case STATE_GOODBYE:
+        case STATE_EXIT:
+            flux_log (h,
+                      LOG_INFO,
+                      "ignored signal %d in %s",
+                      signum,
+                      statestr (s->state));
+            break;
+    }
+}
+
+int state_machine_shutdown (struct state_machine *s, flux_error_t *error)
+{
+    if (s->state != STATE_RUN) {
+        errprintf (error,
+                   "shutdown cannot be initiated in state %s",
+                   statestr (s->state));
+        errno = EINVAL;
+        return -1;
+    }
+    if (s->ctx->rank != 0) {
+        errprintf (error, "shutdown may only be initiated on rank 0");
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (s->exit_norestart > 0)
+        s->ctx->exit_rc = s->exit_norestart;
+
+    if (runat_is_defined (s->ctx->runat, "rc2")) {
+        if (runat_abort (s->ctx->runat, "rc2") < 0)
+            flux_log_error (s->ctx->h, "runat_abort rc2 (shutdown)");
+    }
+    else
+        state_machine_post (s, "shutdown");
+    return 0;
+}
+
+static void runat_completion_cb (struct runat *r, const char *name, void *arg)
+{
+    struct state_machine *s = arg;
+    int rc = 1;
+
+    if (runat_get_exit_code (r, name, &rc) < 0)
+        log_err ("runat_get_exit_code %s", name);
+
+    if (streq (name, "rc1")) {
+        if (rc == 0)
+            state_machine_post (s, "rc1-success");
+        else if (attr_get (s->ctx->attrs,
+                           "broker.recovery-mode",
+                           NULL,
+                           NULL) == 0)
+            state_machine_post (s, "rc1-ignorefail");
+        /* If rc1 fails, it most likely will fail again on restart, so if
+         * running under systemd, exit with the broker.exit-norestart value.
+         */
+        else {
+            if (s->exit_norestart != 0)
+                s->ctx->exit_rc = s->exit_norestart;
+            else
+                s->ctx->exit_rc = rc;
+            state_machine_post (s, "rc1-fail");
+        }
+    }
+    else if (streq (name, "rc2")) {
+        if (s->ctx->exit_rc == 0 && rc != 0)
+            s->ctx->exit_rc = rc;
+        state_machine_post (s, rc == 0 ? "rc2-success" : "rc2-fail");
+    }
+    else if (streq (name, "cleanup")) {
+        if (s->ctx->exit_rc == 0 && rc != 0)
+            s->ctx->exit_rc = rc;
+        state_machine_post (s, rc == 0 ? "cleanup-success" : "cleanup-fail");
+    }
+    else if (streq (name, "rc3")) {
+        if (s->ctx->exit_rc == 0 && rc != 0)
+            s->ctx->exit_rc = rc;
+        state_machine_post (s, rc == 0 ? "rc3-success" : "rc3-fail");
+    }
+}
+
+/* If '-Sbroker.exit-norestart' was set on the command line, set
+ * s->exit_norestart to its value; otherwise leave it set it to 0.
+ */
+static void norestart_configure (struct state_machine *s)
+{
+    const char *val;
+
+    if (attr_get (s->ctx->attrs, "broker.exit-norestart", &val, NULL) == 0) {
+        errno = 0;
+        int rc = strtol (val, NULL, 10);
+        if (errno == 0 && rc >= 1)
+            s->exit_norestart = rc;
+    }
+}
+
+static void prep_cb (flux_reactor_t *r,
+                     flux_watcher_t *w,
+                     int revents,
+                     void *arg)
+{
+    struct state_machine *s = arg;
+
+    if (zlist_size (s->events) > 0)
+        flux_watcher_start (s->idle);
+}
+
+static void check_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct state_machine *s = arg;
+    char *event;
+
+    if ((event = zlist_pop (s->events))) {
+        process_event (s, event);
+        free (event);
+    }
+    flux_watcher_stop (s->idle);
+}
+
+static void run_check_parent (struct state_machine *s)
+{
+    if (s->monitor.parent_error)
+        state_machine_post (s, "parent-fail");
+    else if (s->monitor.parent_valid) {
+        switch (s->monitor.parent_state) {
+            case STATE_NONE:
+            case STATE_JOIN:
+            case STATE_INIT:
+            case STATE_QUORUM:
+            case STATE_RUN:
+            case STATE_CLEANUP:
+                break;
+            case STATE_SHUTDOWN:
+                state_machine_post (s, "shutdown");
+                break;
+            case STATE_FINALIZE:
+            case STATE_GOODBYE:
+            case STATE_EXIT:
+                state_machine_post (s, "parent-fail");
+                break;
+        }
+    }
+}
+
+/* Assumes local state is STATE_JOIN.
+ * If parent has left STATE_INIT, post parent-ready or parent-fail.
+ */
+static void join_check_parent (struct state_machine *s)
+{
+    if (s->monitor.parent_error)
+        state_machine_post (s, "parent-fail");
+    else if (s->monitor.parent_valid) {
+        switch (s->monitor.parent_state) {
+            case STATE_NONE:
+            case STATE_JOIN:
+            case STATE_INIT:
+                break;
+            case STATE_QUORUM:
+            case STATE_RUN:
+                state_machine_post (s, "parent-ready");
+                break;
+            case STATE_CLEANUP:
+            case STATE_SHUTDOWN:
+            case STATE_FINALIZE:
+            case STATE_GOODBYE:
+            case STATE_EXIT:
+                state_machine_post (s, "parent-fail");
+                break;
+        }
+    }
+}
+
+/* Assumes local state is STATE_QUORUM.
+ * If parent has left STATE_QUORUM, post quorum-full or quorum-fail.
+ */
+static void quorum_check_parent (struct state_machine *s)
+{
+    if (s->monitor.parent_error)
+        state_machine_post (s, "quorum-fail");
+    else if (s->monitor.parent_valid) {
+        switch (s->monitor.parent_state) {
+            case STATE_NONE:
+            case STATE_JOIN:
+            case STATE_QUORUM:
+                break;
+            case STATE_INIT:
+            case STATE_RUN:
+                state_machine_post (s, "quorum-full");
+                break;
+            case STATE_CLEANUP:
+            case STATE_SHUTDOWN:
+            case STATE_FINALIZE:
+            case STATE_GOODBYE:
+            case STATE_EXIT:
+                state_machine_post (s, "quorum-fail");
+                break;
+        }
+    }
+}
+
+/* For backwards compatibility, translate "0" and "0-<size-1>" to 1 and <size>,
+ * respectively, but print a warning on stderr.
+ */
+static bool quorum_configure_deprecated (struct state_machine *s,
+                                         const char *val)
+{
+    char all[64];
+    snprintf (all, sizeof (all), "0-%lu", (unsigned long)s->ctx->size - 1);
+    if (streq (val, all))
+        s->quorum.size = s->ctx->size;
+    else if (streq (val, "0"))
+        s->quorum.size = 1;
+    else
+        return false;
+    if (s->ctx->rank == 0) {
+        log_msg ("warning: broker.quorum is now a size - assuming %lu",
+                 (unsigned long)s->quorum.size);
+    }
+    return true;
+}
+
+/* Configure the count of broker ranks needed for quorum (default=<size>).
+ */
+static int quorum_configure (struct state_machine *s)
+{
+    const char *val;
+    if (attr_get (s->ctx->attrs, "broker.quorum", &val, NULL) == 0) {
+        if (!quorum_configure_deprecated (s, val)) {
+            errno = 0;
+            s->quorum.size = strtoul (val, NULL, 10);
+            if (errno != 0
+                || s->quorum.size < 1
+                || s->quorum.size > s->ctx->size) {
+                log_msg ("Error parsing broker.quorum attribute");
+                errno = EINVAL;
+                return -1;
+            }
+        }
+        if (attr_set_flags (s->ctx->attrs, "broker.quorum", ATTR_IMMUTABLE) < 0)
+            return -1;
+    }
+    else {
+        s->quorum.size = s->ctx->size;
+        char buf[16];
+        snprintf (buf, sizeof (buf), "%lu", (unsigned long)s->quorum.size);
+        if (attr_add (s->ctx->attrs, "broker.quorum", buf, ATTR_IMMUTABLE) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static int timeout_configure (struct state_machine *s,
+                              const char *name,
+                              double *value,
+                              double default_value)
+{
+    const char *val;
+    char fsd[32];
+
+    if (attr_get (s->ctx->attrs, name, &val, NULL) == 0) {
+        if (streq (val, "none"))
+            *value = -1;
+        else {
+            if (fsd_parse_duration (val, value) < 0) {
+                log_msg ("Error parsing %s attribute", name);
+                return -1;
+            }
+        }
+        if (attr_delete (s->ctx->attrs, name, true) < 0)
+            return -1;
+    }
+    else
+        *value = default_value;
+    if (*value == -1)
+        snprintf (fsd, sizeof (fsd), "none");
+    else {
+        if (fsd_format_duration (fsd, sizeof (fsd), *value) < 0)
+            return -1;
+    }
+    if (attr_add (s->ctx->attrs, name, fsd, ATTR_IMMUTABLE) < 0)
+        return -1;
+    return 0;
+}
+
+/* called on rank 0 only */
+static void broker_online_cb (flux_future_t *f, void *arg)
+{
+    struct state_machine *s = arg;
+    const char *members;
+    struct idset *ids;
+    struct idset *previous_online;
+    static double last_update = 0;
+    double now = flux_reactor_now (flux_get_reactor (s->ctx->h));
+    bool quorum_reached = false;
+
+    if (flux_rpc_get_unpack (f, "{s:s}", "members", &members) < 0
+        || !(ids = idset_decode (members))) {
+        flux_log_error (s->ctx->h, "groups.get failed");
+        state_machine_post (s, "quorum-fail");
+        return;
+    }
+
+    previous_online = s->quorum.online;
+    s->quorum.online = ids;
+    if (idset_count (s->quorum.online) >= s->quorum.size)
+        quorum_reached = true;
+
+    if (strlen (members) > 0
+        && s->state == STATE_QUORUM
+        && (quorum_reached || now - last_update > 5)) {
+        char *hosts = flux_hostmap_lookup (s->ctx->h, members, NULL);
+        flux_log (s->ctx->h, LOG_INFO, "online: %s (ranks %s)", hosts, members);
+        free (hosts);
+        last_update = now;
+    }
+
+    if (quorum_reached) {
+        if (s->state == STATE_QUORUM) {
+            state_machine_post (s, "quorum-full");
+            if (s->quorum.warned) {
+                flux_log (s->ctx->h, LOG_ERR, "quorum reached");
+                s->quorum.warned = false;
+            }
+        }
+    }
+    /* Log any nodes that leave broker.online during RUN and CLEANUP states.
+     */
+    if ((s->state == STATE_RUN || s->state == STATE_CLEANUP)) {
+        struct idset *loss = idset_difference (previous_online,
+                                               s->quorum.online);
+        if (idset_count (loss) > 0) {
+            char *ranks = idset_encode (loss, IDSET_FLAG_RANGE);
+            char *hosts = flux_hostmap_lookup (s->ctx->h, ranks, NULL);
+            flux_log (s->ctx->h,
+                      LOG_ERR,
+                      "dead to Flux: %s (rank %s)",
+                      hosts,
+                      ranks);
+            free (hosts);
+            free (ranks);
+        }
+        idset_destroy (loss);
+    }
+
+    idset_destroy (previous_online);
+    flux_future_reset (f);
+}
+
+static bool wait_respond (flux_t *h,
+                          const flux_msg_t *msg,
+                          broker_state_t state)
+{
+    int rc;
+
+    if (state < STATE_RUN)
+        return false;
+    if (state == STATE_RUN)
+        rc = flux_respond (h, msg, NULL);
+    else {
+        rc = flux_respond_error (h,
+                                 msg,
+                                 ENOENT,
+                                 "broker has surpassed RUN state");
+    }
+    if (rc < 0)
+        flux_log_error (h, "error responding to state-machine.wait request");
+    return true;
+}
+
+static void wait_update (flux_t *h,
+                         struct flux_msglist *requests,
+                         broker_state_t state)
+{
+    const flux_msg_t *msg;
+
+    msg = flux_msglist_first (requests);
+    while (msg) {
+        if (wait_respond (h, msg, state))
+            flux_msglist_delete (requests);
+        msg = flux_msglist_next (requests);
+    }
+}
+
+/* This request is answered once the local broker enters RUN state.
+ * An error response is generated if the local broker enters a state
+ * that cannot lead to the run state, e.g. CLEANUP, SHUTDOWN, FINALIZE, EXIT.
+ * This is handy when a running broker client tries to reconnect after a broker
+ * restart.  If it tries to send requests too early, it may receive "Upstream
+ * broker is offline" errors.  This request is specifically excluded from that
+ * error path.
+ */
+static void state_machine_wait_cb (flux_t *h,
+                                   flux_msg_handler_t *mh,
+                                   const flux_msg_t *msg,
+                                   void *arg)
+{
+    struct state_machine *s = arg;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (!wait_respond (h, msg, s->state)) {
+        if (flux_msglist_append (s->wait_requests, msg) < 0)
+            goto error;
+    }
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to state-machine.wait request");
+}
+
+static void log_monitor_respond_error (flux_t *h)
+{
+    if (errno != EHOSTUNREACH && errno != ENOSYS)
+        flux_log_error (h, "error responding to state-machine.monitor request");
+}
+
+/* Return true if request should continue to receive updates
+ */
+static bool monitor_update_one (flux_t *h,
+                                const flux_msg_t *msg,
+                                broker_state_t state)
+{
+    broker_state_t final;
+
+    if (flux_msg_unpack (msg, "{s:i}", "final", &final) < 0)
+        final = STATE_EXIT;
+    if (state > final)
+        goto nodata;
+    if (flux_respond_pack (h, msg, "{s:i}", "state", state) < 0)
+        log_monitor_respond_error (h);
+    if (!flux_msg_is_streaming (msg))
+        return false;
+    if (state == final)
+        goto nodata;
+    return true;
+nodata:
+    if (flux_respond_error (h, msg, ENODATA, NULL) < 0)
+        log_monitor_respond_error (h);
+    return false;
+}
+
+static void monitor_update (flux_t *h,
+                            struct flux_msglist *requests,
+                            broker_state_t state)
+{
+    const flux_msg_t *msg;
+
+    msg = flux_msglist_first (requests);
+    while (msg) {
+        if (!monitor_update_one (h, msg, state))
+            flux_msglist_delete (requests);
+        msg = flux_msglist_next (requests);
+    }
+}
+
+static void state_machine_monitor_cb (flux_t *h,
+                                      flux_msg_handler_t *mh,
+                                      const flux_msg_t *msg,
+                                      void *arg)
+{
+    struct state_machine *s = arg;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (monitor_update_one (h, msg, s->state)) {
+        if (flux_msglist_append (s->monitor.requests, msg) < 0)
+            goto error;
+    }
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        log_monitor_respond_error (h);
+}
+
+static void monitor_continuation (flux_future_t *f, void *arg)
+{
+    struct state_machine *s = arg;
+    flux_t *h = s->ctx->h;
+    int state;
+
+    if (flux_rpc_get_unpack (f, "{s:i}", "state", &state) < 0) {
+        if (errno != ENODATA) {
+            flux_log_error (h, "state-machine.monitor");
+            s->monitor.parent_error = 1;
+        }
+        return;
+    }
+    s->monitor.parent_state = state;
+    s->monitor.parent_valid = 1;
+    flux_future_reset (f);
+    if (s->state == STATE_JOIN)
+        join_check_parent (s);
+    else if (s->state == STATE_QUORUM)
+        quorum_check_parent (s);
+    else if (s->state == STATE_RUN)
+        run_check_parent (s);
+}
+
+/* Set up monitoring of parent state up to and including SHUTDOWN state.
+ * Skip monitoring states beyond that to avoid deadlock on disconnecting
+ * children on zeromq-4.1.4 (doesn't seem to be a problem on newer versions).
+ * State machine doesn't need to know about parent transition to these
+ * states anyway.
+ */
+static flux_future_t *monitor_parent (flux_t *h, void *arg)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_rpc_pack (h,
+                             "state-machine.monitor",
+                             FLUX_NODEID_UPSTREAM,
+                             FLUX_RPC_STREAMING,
+                             "{s:i}",
+                             "final", STATE_SHUTDOWN)))
+        return NULL;
+    if (flux_future_then (f, -1, monitor_continuation, arg) < 0) {
+        flux_future_destroy (f);
+        return NULL;
+    }
+    return f;
+}
+
+/* This callback is called when the overlay connection state has changed.
+ */
+static void overlay_monitor_cb (struct overlay *overlay,
+                                uint32_t rank,
+                                void *arg)
+{
+    struct state_machine *s = arg;
+
+    switch (s->state) {
+        /* IN JOIN state, post parent-fail if something goes wrong with the
+         * parent TBON connection.
+         */
+        case STATE_JOIN:
+            if (overlay_parent_error (overlay)) {
+                s->ctx->exit_rc = 1;
+                state_machine_post (s, "parent-fail");
+            }
+            break;
+        case STATE_RUN:
+            if (overlay_parent_error (overlay)) {
+                s->ctx->exit_rc = 1;
+                state_machine_post (s, "shutdown");
+            }
+            break;
+        /* In SHUTDOWN state, post exit event if children have disconnected.
+         * If there are no children on entry to SHUTDOWN state (e.g. leaf
+         * node) the exit event is posted immediately in action_shutdown().
+         */
+        case STATE_SHUTDOWN:
+            if (overlay_get_child_peer_count (overlay) == 0)
+                state_machine_post (s, "children-complete");
+            break;
+        default:
+            break;
+    }
+}
+
+static void state_machine_get_cb (flux_t *h,
+                                  flux_msg_handler_t *mh,
+                                  const flux_msg_t *msg,
+                                  void *arg)
+{
+    struct state_machine *s = arg;
+    double duration = monotime_since (s->t_start) * 1E-3;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:s s:f}",
+                           "state", statestr (s->state),
+                           "duration", duration) < 0)
+        flux_log_error (h,
+                        "error responding to state-machine.get request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0) {
+        flux_log_error (h,
+                        "error responding to state-machine.get request");
+    }
+}
+
+/* If a disconnect is received for streaming monitor request,
+ * drop the request.
+ */
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct state_machine *s = arg;
+
+    if (flux_msglist_disconnect (s->monitor.requests, msg) < 0
+        || flux_msglist_disconnect (s->wait_requests, msg) < 0)
+        flux_log_error (h, "error handling state-machine.disconnect");
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {    FLUX_MSGTYPE_REQUEST,
+        "state-machine.monitor",
+        state_machine_monitor_cb,
+        0
+    },
+    {   FLUX_MSGTYPE_REQUEST,
+        "state-machine.wait",
+        state_machine_wait_cb,
+        FLUX_ROLE_USER
+    },
+    {   FLUX_MSGTYPE_REQUEST,
+        "state-machine.disconnect",
+        disconnect_cb,
+        0
+    },
+    {    FLUX_MSGTYPE_REQUEST,
+        "state-machine.get",
+        state_machine_get_cb,
+        FLUX_ROLE_USER,
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void state_machine_destroy (struct state_machine *s)
+{
+    if (s) {
+        int saved_errno = errno;
+        zlist_destroy (&s->events);
+        flux_watcher_destroy (s->prep);
+        flux_watcher_destroy (s->check);
+        flux_watcher_destroy (s->idle);
+        flux_msg_handler_delvec (s->handlers);
+        flux_msglist_destroy (s->wait_requests);
+        flux_future_destroy (s->monitor.f);
+        flux_msglist_destroy (s->monitor.requests);
+        idset_destroy (s->quorum.all);
+        idset_destroy (s->quorum.online);
+        flux_watcher_destroy (s->quorum.timer);
+        flux_future_destroy (s->quorum.f);
+        flux_watcher_destroy (s->cleanup.timer);
+        free (s);
+        errno = saved_errno;
+    }
+}
+
+struct state_machine *state_machine_create (struct broker *ctx)
+{
+    struct state_machine *s;
+    flux_reactor_t *r = flux_get_reactor (ctx->h);
+
+    if (!(s = calloc (1, sizeof (*s))))
+        return NULL;
+    s->ctx = ctx;
+    s->state = STATE_NONE;
+    monotime (&s->t_start);
+    if (!(s->events = zlist_new ()))
+        goto nomem;
+    zlist_autofree (s->events);
+    if (flux_msg_handler_addvec (ctx->h, htab, s, &s->handlers) < 0)
+        goto error;
+    if (!(s->wait_requests = flux_msglist_create ()))
+        goto error;
+    if (!(s->prep = flux_prepare_watcher_create (r, prep_cb, s))
+        || !(s->check = flux_check_watcher_create (r, check_cb, s))
+        || !(s->idle = flux_idle_watcher_create (r, NULL, NULL))
+        || !(s->quorum.timer = flux_timer_watcher_create (r,
+                                                          0.,
+                                                          0.,
+                                                          quorum_timer_cb,
+                                                          s))
+        || !(s->cleanup.timer = flux_timer_watcher_create (r,
+                                                           0.,
+                                                           0.,
+                                                           cleanup_timer_cb,
+                                                           s)))
+        goto error;
+    flux_watcher_start (s->prep);
+    flux_watcher_start (s->check);
+    if (!(s->monitor.requests = flux_msglist_create ()))
+        goto error;
+    if (ctx->rank > 0) {
+        if (!(s->monitor.f = monitor_parent (ctx->h, s)))
+            goto error;
+    }
+    if (!(s->quorum.online = idset_create (ctx->size, 0)))
+        goto error;
+    if (!(s->quorum.all = idset_create (s->ctx->size, 0))
+        || idset_range_set (s->quorum.all, 0, s->ctx->size - 1) < 0)
+        goto error;
+
+    if (quorum_configure (s) < 0
+        || timeout_configure (s,
+                              "broker.quorum-timeout",
+                              &s->quorum.timeout,
+                              default_quorum_timeout) < 0) {
+        log_err ("error configuring quorum attributes");
+        goto error;
+    }
+    if (timeout_configure (s,
+                           "broker.cleanup-timeout",
+                           &s->cleanup.timeout,
+                           default_cleanup_timeout) < 0) {
+        log_err ("error configuring cleanup timeout attribute");
+        goto error;
+    }
+    norestart_configure (s);
+    overlay_set_monitor_cb (ctx->overlay, overlay_monitor_cb, s);
+    if (s->ctx->rank == 0) {
+        if (!(s->quorum.f = flux_rpc_pack (ctx->h,
+                                           "groups.get",
+                                           FLUX_NODEID_ANY,
+                                           FLUX_RPC_STREAMING,
+                                           "{s:s}",
+                                           "name", "broker.online"))
+            || flux_future_then (s->quorum.f, -1, broker_online_cb, s) < 0)
+            goto error;
+    }
+    return s;
+nomem:
+    errno = ENOMEM;
+error:
+    state_machine_destroy (s);
+    return NULL;
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/broker/state_machine.h b/src/broker/state_machine.h
new file mode 100644
index 000000000000..ebc06a9380a8
--- /dev/null
+++ b/src/broker/state_machine.h
@@ -0,0 +1,43 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _BROKER_STATE_MACHINE_H
+#define _BROKER_STATE_MACHINE_H
+
+struct broker;
+
+typedef enum {
+    STATE_NONE,
+    STATE_JOIN,
+    STATE_INIT,             // rc1
+    STATE_QUORUM,
+    STATE_RUN,              // initial program
+    STATE_CLEANUP,
+    STATE_SHUTDOWN,
+    STATE_FINALIZE,         // rc3
+    STATE_GOODBYE,
+    STATE_EXIT,
+} broker_state_t;
+
+
+struct state_machine *state_machine_create (struct broker *ctx);
+void state_machine_destroy (struct state_machine *s);
+
+void state_machine_post (struct state_machine *s, const char *event);
+
+void state_machine_kill (struct state_machine *s, int signum);
+
+int state_machine_shutdown (struct state_machine *s, flux_error_t *error);
+
+#endif /* !_BROKER_STATE_MACHINE_H */
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/broker/test/attr.c b/src/broker/test/attr.c
index b4571ec510cb..838285a79955 100644
--- a/src/broker/test/attr.c
+++ b/src/broker/test/attr.c
@@ -8,22 +8,22 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <sys/param.h>
 #include <stdio.h>
 
 #include "attr.h"
 
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
-int main (int argc, char **argv)
+void basic (void)
 {
     attr_t *attrs;
     const char *val;
     int flags;
-    int a, c;
-    uint32_t b;
-
-    plan (53);
 
     ok ((attrs = attr_create ()) != NULL,
         "attr_create works");
@@ -34,7 +34,7 @@ int main (int argc, char **argv)
     ok (attr_get (attrs, "foo", NULL, NULL) < 0 && errno == ENOENT,
         "attr_get on unknown attr fails with errno == ENOENT");
     errno = 0;
-    ok (attr_set (attrs, "foo", "bar", false) < 0 && errno == ENOENT,
+    ok (attr_set (attrs, "foo", "bar") < 0 && errno == ENOENT,
         "attr_set on unknown attr fails with errno == ENOENT");
 
     /* attr_add, attr_get works
@@ -48,7 +48,7 @@ int main (int argc, char **argv)
         "attr_get on new attr works with NULL args");
     val = NULL;
     flags = 0;
-    ok (attr_get (attrs, "foo", &val, &flags) == 0 && !strcmp (val, "bar")
+    ok (attr_get (attrs, "foo", &val, &flags) == 0 && streq (val, "bar")
         && flags == 0,
         "attr_get on new attr works returns correct val,flags");
 
@@ -60,45 +60,21 @@ int main (int argc, char **argv)
     ok (attr_get (attrs, "foo", NULL, NULL) < 0 && errno == ENOENT,
         "attr_get on deleted attr fails with errno == ENOENT");
 
-    /* FLUX_ATTRFLAG_READONLY protects against update/delete from users;
-     * update/delete can be forced on broker.
-     */
-    ok (attr_add (attrs, "foo", "baz", FLUX_ATTRFLAG_READONLY) == 0,
-        "attr_add FLUX_ATTRFLAG_READONLY works");
-    flags = 0;
-    val = NULL;
-    ok (attr_get (attrs, "foo", &val, &flags) == 0 && !strcmp (val, "baz")
-        && flags == FLUX_ATTRFLAG_READONLY,
-        "attr_get returns correct value and flags");
-    errno = 0;
-    ok (attr_set (attrs, "foo", "bar", false) < 0 && errno == EPERM,
-        "attr_set on readonly attr fails with EPERM");
-    ok (attr_set (attrs, "foo", "baz", true) == 0,
-        "attr_set (force) on readonly attr works");
-    errno = 0;
-    ok (attr_delete (attrs, "foo", false) < 0 && errno == EPERM,
-        "attr_delete on readonly attr fails with EPERM");
-    ok (attr_delete (attrs, "foo", true) == 0,
-        "attr_delete (force) works on readonly attr");
-    errno = 0;
-    ok (attr_get (attrs, "foo", NULL, NULL) < 0 && errno == ENOENT,
-        "attr_get on deleted attr fails with errno == ENOENT");
-
-    /* FLUX_ATTRFLAG_IMMUTABLE protects against update/delete from user;
+    /* ATTR_IMMUTABLE protects against update/delete from user;
      * update/delete can NOT be forced on broker.
      */
-    ok (attr_add (attrs, "foo", "baz", FLUX_ATTRFLAG_IMMUTABLE) == 0,
-        "attr_add FLUX_ATTRFLAG_IMMUTABLE works");
+    ok (attr_add (attrs, "foo", "baz", ATTR_IMMUTABLE) == 0,
+        "attr_add ATTR_IMMUTABLE works");
     flags = 0;
     val = NULL;
-    ok (attr_get (attrs, "foo", &val, &flags) == 0 && !strcmp (val, "baz")
-        && flags == FLUX_ATTRFLAG_IMMUTABLE,
+    ok (attr_get (attrs, "foo", &val, &flags) == 0 && streq (val, "baz")
+        && flags == ATTR_IMMUTABLE,
         "attr_get returns correct value and flags");
     errno = 0;
-    ok (attr_set (attrs, "foo", "bar", false) < 0 && errno == EPERM,
+    ok (attr_set (attrs, "foo", "bar") < 0 && errno == EPERM,
         "attr_set on immutable attr fails with EPERM");
     errno = 0;
-    ok (attr_set (attrs, "foo", "baz", true)  < 0 && errno == EPERM,
+    ok (attr_set (attrs, "foo", "baz")  < 0 && errno == EPERM,
         "attr_set (force) on immutable fails with EPERM");
     errno = 0;
     ok (attr_delete (attrs, "foo", false) < 0 && errno == EPERM,
@@ -111,7 +87,7 @@ int main (int argc, char **argv)
      * initial hash contents: foo=bar
      */
     val = attr_first (attrs);
-    ok (val && !strcmp (val, "foo"),
+    ok (val && streq (val, "foo"),
         "attr_first returned foo");
     ok (attr_next (attrs) == NULL,
         "attr_next returned NULL");
@@ -121,35 +97,48 @@ int main (int argc, char **argv)
         && attr_add (attrs, "foo4", "44", 0) == 0,
         "attr_add foo1, foo2, foo3, foo4 works");
     val = attr_first (attrs);
-    ok (val && !strncmp (val, "foo", 3),
+    ok (val && strstarts (val, "foo"),
         "attr_first returned foo-prefixed attr");
     val = attr_next (attrs);
-    ok (val && !strncmp (val, "foo", 3),
+    ok (val && strstarts (val, "foo"),
         "attr_next returned foo-prefixed attr");
     val = attr_next (attrs);
-    ok (val && !strncmp (val, "foo", 3),
+    ok (val && strstarts (val, "foo"),
         "attr_next returned foo-prefixed attr");
     val = attr_next (attrs);
-    ok (val && !strncmp (val, "foo", 3),
+    ok (val && strstarts (val, "foo"),
         "attr_next returned foo-prefixed attr");
     val = attr_next (attrs);
-    ok (val && !strncmp (val, "foo", 3),
+    ok (val && strstarts (val, "foo"),
         "attr_next returned foo-prefixed attr");
     ok (attr_next (attrs) == NULL,
         "attr_next returned NULL");
 
+    attr_destroy (attrs);
+}
+
+void active (void)
+{
+    attr_t *attrs;
+    const char *val;
+    int a, c;
+    uint32_t b;
+
+    if (!(attrs = attr_create ()))
+        BAIL_OUT ("attr_create failed");
+
     /* attr_add_active (int helper)
      */
     ok (attr_add_active_int (attrs, "a", &a, 0) == 0,
         "attr_add_active_int works");
     a = 0;
-    ok (attr_get (attrs, "a", &val, NULL) == 0 && val && !strcmp (val, "0"),
+    ok (attr_get (attrs, "a", &val, NULL) == 0 && val && streq (val, "0"),
         "attr_get on active int tracks val=0");
     a = 1;
-    ok (attr_get (attrs, "a", &val, NULL) == 0 && !strcmp (val, "1"),
+    ok (attr_get (attrs, "a", &val, NULL) == 0 && streq (val, "1"),
         "attr_get on active int tracks val=1");
     a = -1;
-    ok (attr_get (attrs, "a", &val, NULL) == 0 && !strcmp (val, "-1"),
+    ok (attr_get (attrs, "a", &val, NULL) == 0 && streq (val, "-1"),
         "attr_get on active int tracks val=-1");
     a = INT_MAX - 1;
     ok (attr_get (attrs, "a", &val, NULL) == 0
@@ -160,15 +149,12 @@ int main (int argc, char **argv)
         && strtol (val, NULL, 10) == INT_MIN + 1,
         "attr_get on active int tracks val=INT_MIN+1");
 
-    ok (attr_set (attrs, "a", "0", false) == 0 && a == 0,
+    ok (attr_set (attrs, "a", "0") == 0 && a == 0,
         "attr_set on active int sets val=0");
-    ok (attr_set (attrs, "a", "1", false) == 0 && a == 1,
+    ok (attr_set (attrs, "a", "1") == 0 && a == 1,
         "attr_set on active int sets val=1");
-    ok (attr_set (attrs, "a", "-1", false) == 0 && a == -1,
+    ok (attr_set (attrs, "a", "-1") == 0 && a == -1,
         "attr_set on active int sets val=-1");
-    errno = 0;
-    ok (attr_delete (attrs, "a", false) < 0 && errno == EPERM,
-        "attr_delete on active attr fails with EPERM");
     ok (attr_delete (attrs, "a", true) == 0,
         "attr_delete (force) works on active attr");
 
@@ -177,38 +163,46 @@ int main (int argc, char **argv)
     ok (attr_add_active_uint32 (attrs, "b", &b, 0) == 0,
         "attr_add_active_uint32 works");
     b = 0;
-    ok (attr_get (attrs, "b", &val, NULL) == 0 && val && !strcmp (val, "0"),
+    ok (attr_get (attrs, "b", &val, NULL) == 0 && val && streq (val, "0"),
         "attr_get on active uin32_t tracks val=0");
     b = 1;
-    ok (attr_get (attrs, "b", &val, NULL) == 0 && !strcmp (val, "1"),
+    ok (attr_get (attrs, "b", &val, NULL) == 0 && streq (val, "1"),
         "attr_get on active uint32_t tracks val=1");
     b = UINT_MAX - 1;
     ok (attr_get (attrs, "b", &val, NULL) == 0
         && strtoul (val, NULL, 10) == UINT_MAX - 1,
         "attr_get on active uint32_t tracks val=UINT_MAX-1");
 
-    ok (attr_set (attrs, "b", "0", false) == 0 && b == 0,
+    ok (attr_set (attrs, "b", "0") == 0 && b == 0,
         "attr_set on active uint32_t sets val=0");
-    ok (attr_set (attrs, "b", "1", false) == 0 && b == 1,
+    ok (attr_set (attrs, "b", "1") == 0 && b == 1,
         "attr_set on active uint32_t sets val=1");
     ok (attr_delete (attrs, "b", true) == 0,
         "attr_delete (force) works on active attr");
 
     /* immutable active int works as expected
      */
-    ok (attr_add_active_int (attrs, "c", &c, FLUX_ATTRFLAG_IMMUTABLE) == 0,
-        "attr_add_active_int FLUX_ATTRFLAG_IMMUTABLE works");
+    ok (attr_add_active_int (attrs, "c", &c, ATTR_IMMUTABLE) == 0,
+        "attr_add_active_int ATTR_IMMUTABLE works");
     c = 42;
-    ok (attr_get (attrs, "c", &val, NULL) == 0 && val && !strcmp (val, "42"),
+    ok (attr_get (attrs, "c", &val, NULL) == 0 && val && streq (val, "42"),
         "attr_get returns initial val=42");
     c = 43;
-    ok (attr_get (attrs, "c", &val, NULL) == 0 && val && !strcmp (val, "42"),
+    ok (attr_get (attrs, "c", &val, NULL) == 0 && val && streq (val, "42"),
         "attr_get ignores value changes");
     errno = 0;
     ok (attr_delete (attrs, "c", true) < 0 && errno == EPERM,
         "attr_delete (force) fails with EPERM");
 
     attr_destroy (attrs);
+}
+
+int main (int argc, char **argv)
+{
+    plan (NO_PLAN);
+
+    basic ();
+    active ();
 
     done_testing ();
     return 0;
diff --git a/src/broker/test/boot_config.c b/src/broker/test/boot_config.c
index a14da6f6237e..be8e7c8f7fa0 100644
--- a/src/broker/test/boot_config.c
+++ b/src/broker/test/boot_config.c
@@ -18,6 +18,7 @@
 
 #include "src/common/libtap/tap.h"
 #include "src/broker/boot_config.h"
+#include "ccan/str/str.h"
 
 
 static void
@@ -51,13 +52,12 @@ create_test_dir (char *dir, int dirlen)
         BAIL_OUT ("snprintf overflow");
     if (!mkdtemp (dir))
         BAIL_OUT ("mkdtemp %s: %s", dir, strerror (errno));
-    setenv ("FLUX_CONF_DIR", dir, 1);
 }
 
 void test_parse (const char *dir)
 {
     char path[PATH_MAX + 1];
-    flux_t *h;
+    flux_conf_t *cf;
     json_t *hosts = NULL;
     struct boot_conf conf;
     uint32_t rank;
@@ -68,17 +68,17 @@ void test_parse (const char *dir)
 "default_port = 42\n" \
 "default_bind = \"tcp://en0:%p\"\n" \
 "default_connect = \"tcp://x%h:%p\"\n" \
+"curve_cert = \"foo\"\n" \
 "hosts = [\n" \
 "  { host = \"foo0\" },\n" \
 "  { host = \"foo[1-62]\" },\n" \
 "  { host = \"foo63\" },\n" \
 "]\n";
 
-    if (!(h = flux_open ("loop://", 0)))
-        BAIL_OUT ("can't continue without loop handle");
-
     create_test_file (dir, "boot", path, sizeof (path), input);
-    rc = boot_config_parse (h, &conf, &hosts);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
+    rc = boot_config_parse (cf, &conf, &hosts);
     ok (rc == 0,
         "boot_conf_parse worked");
     ok (hosts != NULL && json_array_size (hosts) == 64,
@@ -88,9 +88,9 @@ void test_parse (const char *dir)
 
     ok (conf.default_port == 42,
         "set default_port correctly");
-    ok (!strcmp (conf.default_bind, "tcp://en0:42"),
+    ok (streq (conf.default_bind, "tcp://en0:42"),
         "and set default_bind correctly (with %%p substitution)");
-    ok (!strcmp (conf.default_connect, "tcp://x%h:42"),
+    ok (streq (conf.default_connect, "tcp://x%h:42"),
         "and set default_connect correctly (with %%p substitution)");
 
     ok (boot_config_getrankbyname (hosts, "foo0", &rank) == 0
@@ -106,25 +106,25 @@ void test_parse (const char *dir)
        "boot_config_getrankbyname fails on unknown entry");
 
     ok (boot_config_getbindbyrank (hosts, &conf, 0, uri, sizeof (uri)) == 0
-        && !strcmp (uri, "tcp://en0:42"),
+        && streq (uri, "tcp://en0:42"),
         "boot_config_getbindbyrank 0 works with expected value");
     ok (boot_config_getbindbyrank (hosts, &conf, 1, uri, sizeof (uri)) == 0
-        && !strcmp (uri, "tcp://en0:42"),
+        && streq (uri, "tcp://en0:42"),
         "boot_config_getbindbyrank 1 works with expected value");
     ok (boot_config_getbindbyrank (hosts, &conf, 63, uri, sizeof (uri)) == 0
-        && !strcmp (uri, "tcp://en0:42"),
+        && streq (uri, "tcp://en0:42"),
         "boot_config_getbindbyrank 63 works with expected value");
     ok (boot_config_getbindbyrank (hosts, &conf, 64, uri, sizeof (uri))  < 0,
         "boot_config_getbindbyrank 64 fails");
 
     ok (boot_config_geturibyrank (hosts, &conf, 0, uri, sizeof (uri)) == 0
-        && !strcmp (uri, "tcp://xfoo0:42"),
+        && streq (uri, "tcp://xfoo0:42"),
         "boot_config_geturibyrank 0 works with expected value");
     ok (boot_config_geturibyrank (hosts, &conf, 1, uri, sizeof (uri)) == 0
-        && !strcmp (uri, "tcp://xfoo1:42"),
+        && streq (uri, "tcp://xfoo1:42"),
         "boot_config_geturibyrank 1 works with expected value");
     ok (boot_config_geturibyrank (hosts, &conf, 63, uri, sizeof (uri)) == 0
-        && !strcmp (uri, "tcp://xfoo63:42"),
+        && streq (uri, "tcp://xfoo63:42"),
         "boot_config_geturibyrank 63 works with expected value");
     ok (boot_config_geturibyrank (hosts, &conf, 64, uri, sizeof (uri))  < 0,
         "boot_config_geturibyrank 64 fails");
@@ -133,48 +133,42 @@ void test_parse (const char *dir)
 
     if (unlink (path) < 0)
         BAIL_OUT ("could not cleanup test file %s", path);
-
-    flux_close (h);
+    flux_conf_decref (cf);
 }
 
 void test_overflow_bind (const char *dir)
 {
     char path[PATH_MAX + 1];
-    flux_t *h;
+    flux_conf_t *cf;
     struct boot_conf conf;
     char t[MAX_URI*2];
     json_t *hosts;
 
-    if (!(h = flux_open ("loop://", 0)))
-        BAIL_OUT ("can't continue without loop handle");
-
     if (snprintf (t,
                   sizeof (t),
                   "[bootstrap]\ndefault_bind=\"%*s\"\nhosts=[\"foo\"]\n",
                   MAX_URI+2, "foo") >= sizeof (t))
         BAIL_OUT ("snprintf overflow");
     create_test_file (dir, "boot", path, sizeof (path), t);
-
-    ok (boot_config_parse (h, &conf, &hosts) == -1,
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
+    ok (boot_config_parse (cf, &conf, &hosts) == -1,
         "boot_conf_parse caught default_bind overflow");
 
     if (unlink (path) < 0)
         BAIL_OUT ("could not cleanup test file %s", path);
 
-    flux_close (h);
+    flux_conf_decref (cf);
 }
 
 void test_overflow_connect (const char *dir)
 {
     char path[PATH_MAX + 1];
-    flux_t *h;
+    flux_conf_t *cf;
     struct boot_conf conf;
     char t[MAX_URI*2];
     json_t *hosts;
 
-    if (!(h = flux_open ("loop://", 0)))
-        BAIL_OUT ("can't continue without loop handle");
-
     if (snprintf (t,
                   sizeof (t),
                   "[bootstrap]\ndefault_connect=\"%*s\"\nhosts=[\"foo\"]\n",
@@ -182,19 +176,21 @@ void test_overflow_connect (const char *dir)
         BAIL_OUT ("snprintf overflow");
     create_test_file (dir, "boot", path, sizeof (path), t);
 
-    ok (boot_config_parse (h, &conf, &hosts) == -1,
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
+    ok (boot_config_parse (cf, &conf, &hosts) == -1,
         "boot_conf_parse caught default_connect overflow");
 
     if (unlink (path) < 0)
         BAIL_OUT ("could not cleanup test file %s", path);
 
-    flux_close (h);
+    flux_conf_decref (cf);
 }
 
 void test_bad_hosts_entry (const char *dir)
 {
     char path[PATH_MAX + 1];
-    flux_t *h;
+    flux_conf_t *cf;
     struct boot_conf conf;
     json_t *hosts;
     const char *input = \
@@ -203,24 +199,23 @@ void test_bad_hosts_entry (const char *dir)
 "  42,\n" \
 "]\n";
 
-    if (!(h = flux_open ("loop://", 0)))
-        BAIL_OUT ("can't continue without loop handle");
-
     create_test_file (dir, "boot", path, sizeof (path), input);
 
-    ok (boot_config_parse (h, &conf, &hosts) == -1,
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
+    ok (boot_config_parse (cf, &conf, &hosts) == -1,
         "boot_config_parse failed bad hosts entry");
 
     if (unlink (path) < 0)
         BAIL_OUT ("could not cleanup test file %s", path);
 
-    flux_close (h);
+    flux_conf_decref (cf);
 }
 
 void test_missing_info (const char *dir)
 {
     char path[PATH_MAX + 1];
-    flux_t *h;
+    flux_conf_t *cf;
     json_t *hosts;
     struct boot_conf conf;
     char uri[MAX_URI + 1];
@@ -231,12 +226,11 @@ void test_missing_info (const char *dir)
 "  { host = \"foo\" },\n" \
 "]\n";
 
-    if (!(h = flux_open ("loop://", 0)))
-        BAIL_OUT ("can't continue without loop handle");
-
     create_test_file (dir, "boot", path, sizeof (path), input);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
 
-    if (boot_config_parse (h, &conf, &hosts) < 0)
+    if (boot_config_parse (cf, &conf, &hosts) < 0)
         BAIL_OUT ("boot_config_parse unexpectedly failed");
     if (!hosts)
         BAIL_OUT ("cannot continue without hosts array");
@@ -253,67 +247,83 @@ void test_missing_info (const char *dir)
     if (unlink (path) < 0)
         BAIL_OUT ("could not cleanup test file %s", path);
 
-    flux_close (h);
+    flux_conf_decref (cf);
 }
 
-void test_bad_host_idset (const char *dir)
+void test_bad_host_hostlist (const char *dir)
 {
     char path[PATH_MAX + 1];
-    flux_t *h;
+    flux_conf_t *cf;
     struct boot_conf conf;
     json_t *hosts;
     const char *input = \
 "[bootstrap]\n" \
 "hosts = [\n" \
-"  { host=\"foo[1-]\" },\n" \
+"  { host=\"foo[1-\" },\n" \
 "]\n";
 
-    if (!(h = flux_open ("loop://", 0)))
-        BAIL_OUT ("can't continue without loop handle");
-
     create_test_file (dir, "boot", path, sizeof (path), input);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
 
-    ok (boot_config_parse (h, &conf, &hosts) == -1,
+    ok (boot_config_parse (cf, &conf, &hosts) == -1,
         "boot_config_parse failed on host entry containing bad idset");
 
     if (unlink (path) < 0)
         BAIL_OUT ("could not cleanup test file %s", path);
 
-    flux_close (h);
+    flux_conf_decref (cf);
 }
 
 void test_bad_host_bind (const char *dir)
 {
     char path[PATH_MAX + 1];
-    flux_t *h;
+    flux_conf_t *cf;
     struct boot_conf conf;
     json_t *hosts;
-    char uri[MAX_URI + 1];
     const char *input = \
 "[bootstrap]\n" \
 "hosts = [\n" \
 "  { host=\"foo\", bind=42 },\n" \
 "]\n";
 
-    if (!(h = flux_open ("loop://", 0)))
-        BAIL_OUT ("can't continue without loop handle");
-
     create_test_file (dir, "boot", path, sizeof (path), input);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
 
-    /* hosts will initially parse OK then fail in getbindbyrank */
-    if (boot_config_parse (h, &conf, &hosts) < 0)
-        BAIL_OUT ("boot_config_parse unexpectedly failed");
-    ok (boot_config_getbindbyrank (hosts, &conf, 0, uri, sizeof (uri)) < 0,
-        "boot_config_getbindbyrank failed on hoste entry wtih wrong bind type");
-
-    json_decref (hosts);
+    ok (boot_config_parse (cf, &conf, &hosts) < 0,
+        "boot_config_parse failed on host entry with wrong bind type");
 
     if (unlink (path) < 0)
         BAIL_OUT ("could not cleanup test file %s", path);
 
-    flux_close (h);
+    flux_conf_decref (cf);
 }
 
+void test_bad_host_key (const char *dir)
+{
+    char path[PATH_MAX + 1];
+    flux_conf_t *cf;
+    struct boot_conf conf;
+    json_t *hosts;
+    const char *input = \
+"[bootstrap]\n" \
+"hosts = [\n" \
+"  { host=\"foo\", wrongkey=42 },\n" \
+"]\n";
+
+    create_test_file (dir, "boot", path, sizeof (path), input);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
+
+    ok (boot_config_parse (cf, &conf, &hosts) < 0,
+        "boot_config_parse failed on host entry with unknown key");
+
+    if (unlink (path) < 0)
+        BAIL_OUT ("could not cleanup test file %s", path);
+
+    flux_conf_decref (cf);
+}
 
 /* Just double check that an array with mismatched types
  * fails early with the expected libtomlc99 error.
@@ -321,9 +331,8 @@ void test_bad_host_bind (const char *dir)
 void test_toml_mixed_array (const char *dir)
 {
     char path[PATH_MAX + 1];
-    flux_t *h;
-    const flux_conf_t *conf;
-    flux_conf_error_t error;
+    flux_conf_t *cf;
+    flux_error_t error;
     const char *input = \
 "[bootstrap]\n" \
 "hosts = [\n" \
@@ -331,72 +340,67 @@ void test_toml_mixed_array (const char *dir)
 "  { host = \"foo\" },\n" \
 "]\n";
 
-    if (!(h = flux_open ("loop://", 0)))
-        BAIL_OUT ("can't continue without loop handle");
-
     create_test_file (dir, "boot", path, sizeof (path), input);
 
-    conf = flux_get_conf (h, &error);
-    ok (conf == NULL && (strstr (error.errbuf, "array type mismatch")
-        || strstr (error.errbuf, "string array can only contain strings")),
+    cf = flux_conf_parse (dir, &error);
+    ok (cf == NULL && (strstr (error.text, "array type mismatch")
+        || strstr (error.text, "string array can only contain strings")),
         "Mixed type hosts array fails with reasonable error");
-    diag ("%s: line %d: %s", error.filename, error.lineno, error.errbuf);
+    diag ("%s", error.text);
 
     if (unlink (path) < 0)
         BAIL_OUT ("could not cleanup test file %s", path);
 
-    flux_close (h);
+    flux_conf_decref (cf);
 }
 
-void test_no_hosts (const char *dir)
+void test_empty (const char *dir)
 {
     char path[PATH_MAX + 1];
-    flux_t *h;
     json_t *hosts;
+    flux_conf_t *cf;
     struct boot_conf conf;
     const char *input = \
 "[bootstrap]\n";
 
-    if (!(h = flux_open ("loop://", 0)))
-        BAIL_OUT ("can't continue without loop handle");
-
     create_test_file (dir, "boot", path, sizeof (path), input);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
 
     hosts = (json_t *)(uintptr_t)1;
-    ok (boot_config_parse (h, &conf, &hosts) == 0 && hosts == NULL,
-        "boot_config_parse works with missing hosts array");
+    ok (boot_config_parse (cf, &conf, &hosts) == 0 && hosts == NULL,
+        "boot_config_parse works with empty bootstrap stanza");
 
     if (unlink (path) < 0)
         BAIL_OUT ("could not cleanup test file %s", path);
 
-    flux_close (h);
+    flux_conf_decref (cf);
 }
 
 void test_empty_hosts (const char *dir)
 {
     char path[PATH_MAX + 1];
-    flux_t *h;
     json_t *hosts;
+    flux_conf_t *cf;
     struct boot_conf conf;
     const char *input = \
 "[bootstrap]\n" \
 "hosts = [\n" \
-"]\n";
+"]\n" \
 ;
 
-    if (!(h = flux_open ("loop://", 0)))
-        BAIL_OUT ("can't continue without loop handle");
-
     create_test_file (dir, "boot", path, sizeof (path), input);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
 
     hosts = (json_t *)(uintptr_t)1;
-    ok (boot_config_parse (h, &conf, &hosts) == 0 && hosts == NULL,
+    ok (boot_config_parse (cf, &conf, &hosts) == 0 && hosts == NULL,
         "boot_config_parse works with empty hosts array");
 
     if (unlink (path) < 0)
         BAIL_OUT ("could not cleanup test file %s", path);
 
-    flux_close (h);
+    flux_conf_decref (cf);
 }
 
 void test_format (void)
@@ -404,49 +408,223 @@ void test_format (void)
     char buf[MAX_URI + 1];
 
     ok (boot_config_format_uri (buf, sizeof (buf), "abcd", NULL, 0) == 0
-        && !strcmp (buf, "abcd"),
+        && streq (buf, "abcd"),
         "format: plain string copy works");
     ok (boot_config_format_uri (buf, sizeof (buf), "abcd:%p", NULL, 42) == 0
-        && !strcmp (buf, "abcd:42"),
+        && streq (buf, "abcd:42"),
         "format: %%p substitution works end string");
     ok (boot_config_format_uri (buf, sizeof (buf), "a%pb", NULL, 42) == 0
-        && !strcmp (buf, "a42b"),
+        && streq (buf, "a42b"),
         "format: %%p substitution works mid string");
     ok (boot_config_format_uri (buf, sizeof (buf), "%p:abcd", NULL, 42) == 0
-        && !strcmp (buf, "42:abcd"),
+        && streq (buf, "42:abcd"),
         "format: %%p substitution works begin string");
     ok (boot_config_format_uri (buf, sizeof (buf), "%h", NULL, 0) == 0
-        && !strcmp (buf, "%h"),
+        && streq (buf, "%h"),
         "format: %%h passes through when host=NULL");
     ok (boot_config_format_uri (buf, sizeof (buf), "%h", "foo", 0) == 0
-        && !strcmp (buf, "foo"),
+        && streq (buf, "foo"),
         "format: %%h substitution works");
     ok (boot_config_format_uri (buf, sizeof (buf), "%%", NULL, 0) == 0
-        && !strcmp (buf, "%"),
+        && streq (buf, "%"),
         "format: %%%% literal works");
     ok (boot_config_format_uri (buf, sizeof (buf), "a%X", NULL, 0) == 0
-        && !strcmp (buf, "a%X"),
+        && streq (buf, "a%X"),
         "format: unknown token passes through");
 
     ok (boot_config_format_uri (buf, 5, "abcd", NULL, 0) == 0
-        && !strcmp (buf, "abcd"),
+        && streq (buf, "abcd"),
         "format: copy abcd to buf[5] works");
     ok (boot_config_format_uri (buf, 4, "abcd", NULL, 0) < 0,
         "format: copy abcd to buf[4] fails");
 
     ok (boot_config_format_uri (buf, 5, "a%p", NULL, 123) == 0
-        && !strcmp (buf, "a123"),
+        && streq (buf, "a123"),
         "format: %%p substitution into exact size buf works");
     ok (boot_config_format_uri (buf, 4, "a%p", NULL, 123) < 0,
         "format: %%p substitution overflow detected");
 
     ok (boot_config_format_uri (buf, 5, "a%h", "abc", 0) == 0
-        && !strcmp (buf, "aabc"),
+        && streq (buf, "aabc"),
         "format: %%h substitution into exact size buf works");
     ok (boot_config_format_uri (buf, 4, "a%h", "abc", 0) < 0,
         "format: %%h substitution overflow detected");
 }
 
+void test_attr (const char *dir)
+{
+    char path[PATH_MAX + 1];
+    flux_conf_t *cf;
+    json_t *hosts = NULL;
+    struct boot_conf conf;
+    attr_t *attrs;
+    int rc;
+    const char *input = \
+"[bootstrap]\n" \
+"curve_cert = \"foo\"\n" \
+"hosts = [\n" \
+"  { host = \"foo0\" },\n" \
+"  { host = \"foo4\" },\n" \
+"  { host = \"foo[1-5]\" },\n" \
+"  { host = \"foo14\" },\n" \
+"  { host = \"foo[6-9]\" },\n" \
+"]\n";
+    const char *val;
+    int flags;
+
+    create_test_file (dir, "boot", path, sizeof (path), input);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
+
+    ok ((attrs = attr_create ()) != NULL,
+        "attr_create_works");
+    if (attrs == NULL)
+        BAIL_OUT ("cannot continue without attrs");
+
+    rc = boot_config_attr (attrs, "localhost", NULL);
+    ok (rc == 0,
+        "boot_config_attr works NULL hosts");
+    ok (attr_get (attrs, "hostlist", NULL, NULL) == 0,
+        "attr_get finds hostlist after NULL hosts");
+    attr_destroy (attrs);
+
+    attrs = attr_create ();
+    if (!attrs)
+        BAIL_OUT ("attr_create failed");
+    hosts = json_array ();
+    if (hosts == NULL)
+        BAIL_OUT ("cannot continue without empty hosts array");
+    rc = boot_config_attr (attrs, "localhost", hosts);
+    ok (rc == 0,
+        "boot_config_attr works empty hosts");
+    ok (attr_get (attrs, "hostlist", NULL, NULL) == 0,
+        "attr_get finds hostlist after empty hosts");
+    json_decref (hosts);
+    hosts = NULL;
+    attr_destroy (attrs);
+
+    rc = boot_config_parse (cf, &conf, &hosts);
+    ok (rc == 0,
+        "boot_conf_parse worked");
+    if (hosts == NULL)
+        BAIL_OUT ("cannot continue without hosts array");
+
+    attrs = attr_create ();
+    if (!attrs)
+        BAIL_OUT ("attr_create failed");
+    rc = boot_config_attr (attrs, "foo0", hosts);
+    ok (rc == 0,
+        "boot_config_attr works on input hosts");
+    ok (attr_get (attrs, "hostlist", &val, &flags) == 0
+        && streq (val, "foo[0,4,1-3,5,14,6-9]")
+        && flags == ATTR_IMMUTABLE,
+        "attr_get returns correct value and flags");
+
+    json_decref (hosts);
+
+    if (unlink (path) < 0)
+        BAIL_OUT ("could not cleanup test file %s", path);
+    flux_conf_decref (cf);
+    attr_destroy (attrs);
+}
+
+void test_curve_cert (const char *dir)
+{
+    char path[PATH_MAX + 1];
+    json_t *hosts;
+    flux_conf_t *cf;
+    struct boot_conf conf;
+    const char *input = \
+"[bootstrap]\n" \
+"curve_cert = \"meep\"\n";
+
+    create_test_file (dir, "boot", path, sizeof (path), input);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
+
+    ok (boot_config_parse (cf, &conf, &hosts) == 0 && hosts == NULL,
+        "boot_config_parse works with curve_cert");
+    ok (conf.curve_cert != NULL && streq (conf.curve_cert, "meep"),
+        "and curve_cert has expected value");
+
+    if (unlink (path) < 0)
+        BAIL_OUT ("could not cleanup test file %s", path);
+
+    flux_conf_decref (cf);
+}
+
+void test_ipv6 (const char *dir)
+{
+    char path[PATH_MAX + 1];
+    json_t *hosts;
+    flux_conf_t *cf;
+    struct boot_conf conf;
+    const char *input = \
+"[bootstrap]\n" \
+"enable_ipv6 = true\n";
+
+    create_test_file (dir, "boot", path, sizeof (path), input);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
+
+    ok (boot_config_parse (cf, &conf, &hosts) == 0 && hosts == NULL,
+        "boot_config_parse works with enable_ipv6");
+    ok (conf.enable_ipv6 != 0,
+        "and enable_ipv6 has expected value");
+
+    if (unlink (path) < 0)
+        BAIL_OUT ("could not cleanup test file %s", path);
+
+    flux_conf_decref (cf);
+}
+
+void test_dup_hosts (const char *dir)
+{
+    char path[PATH_MAX + 1];
+    json_t *hosts = NULL;
+    const char *host;
+    flux_conf_t *cf;
+    struct boot_conf conf;
+    const char *input = \
+"[bootstrap]\n" \
+"curve_cert = \"foo\"\n" \
+"[[bootstrap.hosts]]\n" \
+"host = \"test[0-127]\"\n" \
+"[[bootstrap.hosts]]\n" \
+"host = \"test[1,64]\"\n" \
+"parent = \"test0\"\n" \
+"[[bootstrap.hosts]]\n" \
+"host = \"test[2-63]\"\n" \
+"parent = \"test1\"\n" \
+"[[bootstrap.hosts]]\n" \
+"host = \"test[65-127]\"\n" \
+"parent = \"test64\"\n";
+
+    create_test_file (dir, "boot", path, sizeof (path), input);
+    if (!(cf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failed");
+
+    ok (boot_config_parse (cf, &conf, &hosts) == 0,
+        "boot_config_parse handles duplicate hosts");
+    ok (hosts != NULL && json_array_size (hosts) == 128,
+        "and post-processed hosts array has expected size");
+    ok (json_unpack (json_array_get (hosts, 0), "{s:s}", "host", &host) == 0
+        && streq (host, "test0"),
+        "test0 has rank 0");
+    ok (json_unpack (json_array_get (hosts, 65), "{s:s}", "host", &host) == 0
+        && streq (host, "test65"),
+        "test65 has rank 65");
+    ok (json_unpack (json_array_get (hosts, 127), "{s:s}", "host", &host) == 0
+        && streq (host, "test127"),
+        "test127 has rank 127");
+
+    if (unlink (path) < 0)
+        BAIL_OUT ("could not cleanup test file %s", path);
+
+    json_decref (hosts);
+    flux_conf_decref (cf);
+}
+
 int main (int argc, char **argv)
 {
     char dir[PATH_MAX + 1];
@@ -461,12 +639,17 @@ int main (int argc, char **argv)
     test_overflow_bind (dir);
     test_overflow_connect (dir);
     test_bad_hosts_entry (dir);
-    test_bad_host_idset (dir);
+    test_bad_host_hostlist (dir);
     test_bad_host_bind (dir);
-    test_no_hosts (dir);
+    test_bad_host_key (dir);
+    test_empty (dir);
     test_empty_hosts (dir);
     test_missing_info (dir);
     test_toml_mixed_array (dir);
+    test_attr (dir);
+    test_curve_cert (dir);
+    test_ipv6 (dir);
+    test_dup_hosts (dir);
 
     if (rmdir (dir) < 0)
         BAIL_OUT ("could not cleanup test dir %s", dir);
diff --git a/src/broker/test/heartbeat.c b/src/broker/test/heartbeat.c
deleted file mode 100644
index f5f984ebea75..000000000000
--- a/src/broker/test/heartbeat.c
+++ /dev/null
@@ -1,109 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <flux/core.h>
-#include <czmq.h>
-#include <stdio.h>
-
-#include "heartbeat.h"
-
-#include "src/common/libtap/tap.h"
-
-void fatal_err (const char *message, void *arg)
-{
-    BAIL_OUT ("fatal error: %s", message);
-}
-
-void heartbeat_event_cb (flux_t *h, flux_msg_handler_t *w,
-                          const flux_msg_t *msg, void *arg)
-{
-    heartbeat_t *hb = arg;
-    int epoch = -1;
-    int rc = flux_heartbeat_decode (msg, &epoch);
-
-    ok (rc == 0,
-        "received heartbeat event epoch %d", epoch);
-    if (epoch == 2) {
-        flux_msg_handler_stop (w);
-        heartbeat_stop (hb);
-    }
-}
-
-void check_codec (void)
-{
-    flux_msg_t *msg;
-    int epoch;
-
-    ok ((msg = flux_heartbeat_encode (44000)) != NULL,
-        "flux_heartbeat_encode works");
-    ok (flux_heartbeat_decode (msg, &epoch) == 0 && epoch == 44000,
-        "flux_heartbeat_decode works and returns encoded epoch");
-    flux_msg_destroy (msg);
-}
-
-int main (int argc, char **argv)
-{
-    flux_t *h;
-    heartbeat_t *hb;
-    flux_msg_handler_t *w;
-
-    plan (16);
-
-    check_codec ();
-
-    ok ((h = flux_open ("loop://", 0)) != NULL,
-        "opened loop connector");
-    if (!h)
-        BAIL_OUT ("can't continue without loop handle");
-    flux_fatal_set (h, fatal_err, NULL);
-
-    ok ((hb = heartbeat_create ()) != NULL,
-        "heartbeat_create works");
-
-    heartbeat_set_flux (hb, h);
-
-    ok (heartbeat_get_rate (hb) == 2.,
-        "heartbeat_get_rate returns default of 2s");
-    errno = 0;
-    ok (heartbeat_set_rate (hb, -1) < 0 && errno == EINVAL,
-        "heartbeat_set_rate -1 fails with EINVAL");
-    errno = 0;
-    ok (heartbeat_set_rate (hb, 1000000) < 0 && errno == EINVAL,
-        "heartbeat_set_rate 1000000 fails with EINVAL");
-    ok (heartbeat_set_rate (hb, 0.1) == 0,
-        "heartbeat_set_rate 0.1 works");
-    ok (heartbeat_get_rate (hb) == 0.1,
-        "heartbeat_get_rate returns what was set");
-
-    ok (heartbeat_get_epoch (hb) == 0,
-        "heartbeat_get_epoch works, default is zero");
-
-    w = flux_msg_handler_create (h, FLUX_MATCH_EVENT, heartbeat_event_cb, hb);
-    ok (w != NULL,
-        "created event watcher");
-    flux_msg_handler_start (w);
-
-    ok (heartbeat_start (hb) == 0,
-        "heartbeat_start works");
-
-    ok (flux_reactor_run (flux_get_reactor (h), 0) == 0,
-        "flux reactor exited normally");
-
-    heartbeat_destroy (hb);
-    flux_msg_handler_destroy (w);
-    flux_close (h);
-
-    done_testing ();
-    return 0;
-}
-
-/*
- * vi:ts=4 sw=4 expandtab
- */
diff --git a/src/broker/test/hello.c b/src/broker/test/hello.c
deleted file mode 100644
index 31574142ce93..000000000000
--- a/src/broker/test/hello.c
+++ /dev/null
@@ -1,112 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <flux/core.h>
-#include <czmq.h>
-#include <stdio.h>
-
-#include "hello.h"
-
-#include "src/common/libtap/tap.h"
-
-static flux_t *h;
-
-void fatal_err (const char *message, void *arg)
-{
-    BAIL_OUT ("fatal error: %s", message);
-}
-
-void hello_cb (hello_t *hello, void *arg)
-{
-    int *ip = arg;
-    (*ip)++;
-}
-
-int main (int argc, char **argv)
-{
-    hello_t *hello;
-    uint32_t rank, size;
-    int cb_counter = 0;
-
-    plan (NO_PLAN);
-
-    ok ((h = flux_open ("loop://", 0)) != NULL,
-        "opened loop connector");
-    if (!h)
-        BAIL_OUT ("can't continue without loop handle");
-    flux_fatal_set (h, fatal_err, NULL);
-
-    /* Simulate a single rank session.
-     * Since broker attrs are not set, hwm defaults to 1 (self).
-     * It will immediately complete so no need to start reactor.
-     */
-    flux_attr_set_cacheonly (h, "size", "1");
-    flux_attr_set_cacheonly (h, "rank", "0");
-    ok (flux_get_size (h, &size) == 0 && size == 1,
-        "size == 1");
-    ok (flux_get_rank (h, &rank) == 0 && rank == 0,
-        "rank == 0");
-
-    ok ((hello = hello_create ()) != NULL,
-        "hello_create works");
-    hello_set_flux (hello, h);
-    hello_set_callback (hello, hello_cb, &cb_counter);
-    ok (hello_get_count (hello) == 0,
-        "hello_get_count returned 0");
-    ok (hello_complete (hello) == 0,
-        "hello_complete returned false");
-    ok (hello_start (hello) == 0,
-        "hello_start works");
-    ok (cb_counter == 1,
-        "callback was called");
-    ok (hello_get_count (hello) == 1,
-        "hello_get_count returned 1");
-    ok (hello_complete (hello) != 0,
-        "hello_complete returned true");
-    hello_destroy (hello);
-
-    /* Simulate a 2 node session.
-     * Same procedure as above except the session will not complete.
-     */
-    flux_attr_set_cacheonly (h, "size", "3");
-    flux_attr_set_cacheonly (h, "rank", "0");
-    ok (flux_get_size (h, &size) == 0 && size == 3,
-        "size == 1");
-    ok (flux_get_rank (h, &rank) == 0 && rank == 0,
-        "rank == 0");
-
-    cb_counter = 0;
-    ok ((hello = hello_create ()) != NULL,
-        "hello_create works");
-    hello_set_flux (hello, h);
-    hello_set_callback (hello, hello_cb, &cb_counter);
-    ok (hello_get_count (hello) == 0,
-        "hello_get_count returned 0");
-    ok (hello_complete (hello) == 0,
-        "hello_complete returned false");
-    ok (hello_start (hello) == 0,
-        "hello_start works");
-    ok (cb_counter == 1,
-        "callback was called once (for self)");
-    ok (hello_get_count (hello) == 1,
-        "hello_get_count returned 1");
-    ok (hello_complete (hello) == 0,
-        "hello_complete returned false");
-    hello_destroy (hello);
-
-    flux_close (h);
-
-    done_testing ();
-    return 0;
-}
-
-/*
- * vi:ts=4 sw=4 expandtab
- */
diff --git a/src/broker/test/liblist.c b/src/broker/test/liblist.c
deleted file mode 100644
index 3ad77e709677..000000000000
--- a/src/broker/test/liblist.c
+++ /dev/null
@@ -1,100 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/fcntl.h>
-#include <stdlib.h>
-#include <czmq.h>
-
-#include "src/common/libtap/tap.h"
-#include "src/common/libutil/iterators.h"
-
-#include "liblist.h"
-
-void diag_dump (zlist_t *libs)
-{
-    char *name;
-    int i = 0;
-    FOREACH_ZLIST (libs, name) {
-        diag ("%d: %s", i, name);
-        i++;
-    }
-}
-
-int main (int argc, char **argv)
-{
-    zlist_t *libs;
-    char *tmpdir = getenv ("TMPDIR");
-    char testdir[PATH_MAX + 1];
-    char path[PATH_MAX + 1];
-    int fd;
-    int n;
-
-    if (!tmpdir)
-        tmpdir = "/tmp";
-
-    plan (NO_PLAN);
-
-    /* First mode: library path contains slashes.
-     * List will just contain that path without checking if it exists..
-     */
-    libs = liblist_create ("/my/libfoo.so");
-    ok (libs != NULL,
-        "liblist_create libname=/my/libfoo.so works");
-    ok (zlist_size (libs) == 1,
-        "liblist contains one entry");
-    ok (!strcmp ("/my/libfoo.so", zlist_head (libs)),
-        "liblist contains /my/libfoo.so");
-    diag_dump (libs);
-    liblist_destroy (libs);
-
-    /* Second mode: library path contains no slashes.
-     * List will contain first any occurrences in LD_LIBRARY_PATH dirs,
-     * then any in ld.so.cache.
-     * Focus on LD_LIBRARY_PATH since we can control but try a common name
-     * to maybe pick up something from ld.so.cache.
-     */
-    n = snprintf (testdir, sizeof (testdir), "%s/test.XXXXXX", tmpdir);
-    if (n >= sizeof (testdir))
-        BAIL_OUT ("buffer overflow");
-    if (mkdtemp (testdir) == NULL)
-        BAIL_OUT ("mkdtemp failed");
-    n = snprintf (path, sizeof (path), "%s/libSegFault.so", testdir);
-    if (n >= sizeof (path))
-        BAIL_OUT ("buffer overflow");
-    if ((fd = open (path, O_CREAT | O_RDWR, 0666)) < 0)
-        BAIL_OUT ("could not create %s", path);
-    close (fd);
-
-    if (setenv ("LD_LIBRARY_PATH", testdir, 1) < 0)
-        BAIL_OUT ("setenv failed");
-    libs = liblist_create ("libSegFault.so");
-    ok (libs != NULL,
-        "liblist_create libname=libSegFault.so works");
-    ok (zlist_size (libs) >= 1,
-        "liblist contains at least one entry");
-    ok (!strcmp (path, zlist_head (libs)),
-        "liblist contains %s", path);
-    diag_dump (libs);
-    liblist_destroy (libs);
-
-    (void)rmdir (testdir);
-    (void)unlink (path);
-    done_testing ();
-    return 0;
-}
-
-/*
- * vi:ts=4 sw=4 expandtab
- */
diff --git a/src/broker/test/overlay.c b/src/broker/test/overlay.c
new file mode 100644
index 000000000000..29edb2a785bd
--- /dev/null
+++ b/src/broker/test/overlay.c
@@ -0,0 +1,757 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <string.h>
+#include <flux/core.h>
+#include <zmq.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libzmqutil/msg_zsock.h"
+#include "src/common/libzmqutil/sockopt.h"
+#include "src/common/libzmqutil/cert.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/stdlog.h"
+#include "src/common/libutil/unlink_recursive.h"
+#include "ccan/str/str.h"
+
+#include "src/broker/overlay.h"
+#include "src/broker/attr.h"
+#include "src/broker/topology.h"
+
+static zlist_t *logs;
+void *zctx;
+
+struct context {
+    struct overlay *ov;
+    flux_t *h;
+    attr_t *attrs;
+    char name[32];
+    int rank;
+    int size;
+    struct topology *topo;
+    const char *uuid;
+    const flux_msg_t *msg;
+};
+
+void clear_list (zlist_t *list)
+{
+    char *s;
+    while ((s = zlist_pop (list)))
+        free (s);
+}
+
+int match_list (zlist_t *list, const char *key)
+{
+    char *s;
+    int count = 0;
+
+    s = zlist_first (list);
+    while (s) {
+        if (strstr (s, key) != NULL)
+            count++;
+        s = zlist_next (list);
+    }
+    return count;
+}
+
+void check_attr (struct context *ctx, const char *k, const char *v)
+{
+    const char *val;
+
+    ok (attr_get (ctx->attrs, k, &val, NULL)  == 0
+        && ((v == NULL && val == NULL)
+            || (v != NULL && val != NULL && streq (v, val))),
+        "%s: %s=%s", ctx->name, k, v ? v : "NULL");
+}
+
+void ctx_destroy (struct context *ctx)
+{
+    attr_destroy (ctx->attrs);
+    overlay_destroy (ctx->ov);
+    flux_msg_decref (ctx->msg);
+    topology_decref (ctx->topo);
+    free (ctx);
+}
+
+struct context *ctx_create (flux_t *h,
+                            int size,
+                            int rank,
+                            const char *topo_uri,
+                            overlay_recv_f cb)
+{
+    struct context *ctx;
+    flux_error_t error;
+    const char *temp = getenv ("TMPDIR");
+    if (!temp)
+        temp = "/tmp";
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        BAIL_OUT ("calloc failed");
+    if (!(ctx->attrs = attr_create ()))
+        BAIL_OUT ("attr_create failed");
+    if (!(ctx->topo = topology_create (topo_uri, size, &error)))
+        BAIL_OUT ("cannot create '%s' topology: %s", topo_uri, error.text);
+    if (topology_set_rank (ctx->topo, rank) < 0)
+        BAIL_OUT ("cannot set topology rank");
+    ctx->h = h;
+    ctx->size = size;
+    ctx->rank = rank;
+    snprintf (ctx->name, sizeof (ctx->name), "test%d", rank);
+    if (!(ctx->ov = overlay_create (h, ctx->name, ctx->attrs, zctx, cb, ctx)))
+        BAIL_OUT ("overlay_create");
+    if (!(ctx->uuid = overlay_get_uuid (ctx->ov)))
+        BAIL_OUT ("overlay_get_uuid failed");
+    diag ("created %s: rank %d size %d uuid %s",
+          ctx->name, ctx->rank, ctx->size, ctx->uuid);
+
+    return ctx;
+}
+
+void single (flux_t *h)
+{
+    struct context *ctx = ctx_create (h, 1, 0, "kary:2", NULL);
+    flux_msg_t *msg;
+    char *s;
+    struct idset *critical_ranks;
+
+    ok (overlay_set_topology (ctx->ov, ctx->topo) == 0,
+        "%s: overlay_set_topology size=1 rank=0 works", ctx->name);
+
+    ok (overlay_get_size (ctx->ov) == 1,
+        "%s: overlay_get_size returns 1", ctx->name);
+    ok (overlay_get_rank (ctx->ov) == 0,
+        "%s: overlay_get_rank returns 0", ctx->name);
+
+    ok ((critical_ranks = overlay_get_default_critical_ranks (ctx->ov)) != NULL,
+        "%s: overlay_get_default_critical_ranks works");
+    if (!(s = idset_encode (critical_ranks, IDSET_FLAG_RANGE)))
+        BAIL_OUT ("idset_encode");
+    is (s, "0",
+        "%s: overlay_get_default_critical_ranks returned %s",
+        s);
+    free (s);
+    idset_destroy (critical_ranks);
+
+    ok (overlay_register_attrs (ctx->ov) == 0,
+        "%s: overlay_register_attrs works", ctx->name);
+    check_attr (ctx, "tbon.parent-endpoint", NULL);
+    check_attr (ctx, "rank", "0");
+    check_attr (ctx, "size", "1");
+    check_attr (ctx, "tbon.level", "0");
+    check_attr (ctx, "tbon.maxlevel", "0");
+    check_attr (ctx, "tbon.descendants", "0");
+
+    /* No parent uri.
+     * No bind uri because no children
+     */
+    ok (overlay_get_parent_uri (ctx->ov) == NULL,
+        "%s: overlay_get_parent_uri returned NULL", ctx->name);
+    ok (overlay_get_bind_uri (ctx->ov) == NULL,
+        "%s: overlay_get_bind_uri returned NULL", ctx->name);
+
+    /* Event
+     */
+    if (!(msg = flux_event_encode ("foo", NULL)))
+        BAIL_OUT ("flux_event_encode failed");
+    ok (overlay_sendmsg (ctx->ov, msg, OVERLAY_DOWNSTREAM) == 0,
+        "%s: overlay_sendmsg event where=DOWN succeeds",
+        ctx->name);
+    errno = 0;
+    ok (overlay_sendmsg (ctx->ov, msg, OVERLAY_UPSTREAM) < 0
+        && errno == EHOSTUNREACH,
+        "%s: overlay_sendmsg event where=UP fails with EHOSTUNREACH",
+        ctx->name);
+    flux_msg_decref (msg);
+
+    /* Response
+     */
+    if (!(msg = flux_response_encode ("foo", NULL)))
+        BAIL_OUT ("flux_response_encode failed");
+    errno = 0;
+    ok (overlay_sendmsg (ctx->ov, msg, OVERLAY_DOWNSTREAM) < 0
+        && errno == EHOSTUNREACH,
+        "%s: overlay_sendmsg response where=DOWN fails with EHOSTUNREACH",
+        ctx->name);
+    errno = 0;
+    ok (overlay_sendmsg (ctx->ov, msg, OVERLAY_ANY) < 0
+        && errno == EHOSTUNREACH,
+        "%s: overlay_sendmsg response where=ANY fails with EHOSTUNREACH",
+        ctx->name);
+    errno = 0;
+    ok (overlay_sendmsg (ctx->ov, msg, OVERLAY_UPSTREAM) < 0
+        && errno == EHOSTUNREACH,
+        "%s: overlay_sendmsg response where=UP fails with EHOSTUNREACH",
+        ctx->name);
+    flux_msg_decref (msg);
+
+    /* Request
+     */
+    if (!(msg = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+    errno = 0;
+    ok (overlay_sendmsg (ctx->ov, msg, OVERLAY_DOWNSTREAM) < 0
+        && errno == EHOSTUNREACH,
+        "%s: overlay_sendmsg request where=DOWN fails with EHOSTUNREACH",
+        ctx->name);
+    errno = 0;
+    ok (overlay_sendmsg (ctx->ov, msg, OVERLAY_ANY) < 0
+        && errno == EHOSTUNREACH,
+        "%s: overlay_sendmsg request where=ANY fails with EHOSTUNREACH",
+        ctx->name);
+    errno = 0;
+    ok (overlay_sendmsg (ctx->ov, msg, OVERLAY_UPSTREAM) < 0
+        && errno == EHOSTUNREACH,
+        "%s: overlay_sendmsg request where=UP fails with EHOSTUNREACH",
+        ctx->name);
+    flux_msg_decref (msg);
+
+    ok (overlay_get_child_peer_count (ctx->ov) == 0,
+        "%s: overlay_get_child_peer_count returns 0", ctx->name);
+
+    ctx_destroy (ctx);
+}
+
+int recv_cb (flux_msg_t **msg, overlay_where_t from, void *arg)
+{
+    struct context *ctx = arg;
+
+    diag ("%s message received",
+          from == OVERLAY_UPSTREAM ? "upstream" : "downstream");
+    ctx->msg = *msg;
+    *msg = NULL;
+    flux_reactor_stop (flux_get_reactor (ctx->h));
+    return 0;
+}
+
+void timeout_cb (flux_reactor_t *r, flux_watcher_t *w, int revents, void *arg)
+{
+    diag ("receive timeout");
+    errno = ETIMEDOUT;
+    flux_reactor_stop_error (r);
+}
+
+/* Receive a message with timeout.
+ * Returns 0 on success, or -1 with errno=ETIMEDOUT.
+ */
+const flux_msg_t *recvmsg_timeout (struct context *ctx, double timeout)
+{
+
+    flux_reactor_t *r = flux_get_reactor (ctx->h);
+    flux_watcher_t *w;
+    int rc;
+
+    flux_msg_decref (ctx->msg);
+    ctx->msg = NULL;
+
+    if (!(w = flux_timer_watcher_create (r, timeout, 0., timeout_cb, ctx)))
+        BAIL_OUT ("flux_timer_watcher_create failed");
+    flux_watcher_start (w);
+
+    rc = flux_reactor_run (r, 0);
+
+    flux_watcher_destroy (w);
+
+    return rc < 0 ? NULL : ctx->msg;
+}
+
+/* Rank 0,1 are properly configured.
+ * Rank 2 will try to get involved without proper credentials etc.
+ */
+void trio (flux_t *h)
+{
+    struct context *ctx[2];
+    int size = 3;
+    char parent_uri[64];
+    const char *server_pubkey;
+    const char *client_pubkey;
+    const char *tmp;
+    const flux_msg_t *rmsg;
+    flux_msg_t *msg;
+    const char *topic;
+    void *zsock_none;
+    void *zsock_curve;
+    struct cert *cert;
+    const char *sender;
+
+    ctx[0] = ctx_create (h, size, 0, "kary:2", recv_cb);
+
+    ok (overlay_set_topology (ctx[0]->ov, ctx[0]->topo) == 0,
+        "%s: overlay_set_topology works", ctx[0]->name);
+
+    ok ((server_pubkey = overlay_cert_pubkey (ctx[0]->ov)) != NULL,
+        "%s: overlay_cert_pubkey works", ctx[0]->name);
+
+    snprintf (parent_uri, sizeof (parent_uri), "ipc://@%s", ctx[0]->name);
+    ok (overlay_bind (ctx[0]->ov, parent_uri) == 0,
+        "%s: overlay_bind %s works", ctx[0]->name, parent_uri);
+
+    ctx[1] = ctx_create (h, size, 1, "kary:2", recv_cb);
+
+    ok (overlay_set_topology (ctx[1]->ov, ctx[1]->topo) == 0,
+        "%s: overlay_set_topology works", ctx[1]->name);
+
+    ok ((client_pubkey = overlay_cert_pubkey (ctx[1]->ov)) != NULL,
+        "%s: overlay_cert_pubkey works", ctx[1]->name);
+    ok (overlay_set_parent_uri (ctx[1]->ov, parent_uri) == 0,
+        "%s: overlay_set_parent_uri %s works", ctx[1]->name, parent_uri);
+    tmp = overlay_get_parent_uri (ctx[1]->ov);
+    ok (tmp != NULL && streq (tmp, parent_uri),
+        "%s: overlay_get_parent_uri returns same string", ctx[1]->name);
+    ok (overlay_set_parent_pubkey (ctx[1]->ov, server_pubkey) == 0,
+        "%s: overlay_set_parent_pubkey works", ctx[1]->name);
+
+    ok (overlay_authorize (ctx[0]->ov, ctx[0]->name, client_pubkey) == 0,
+        "%s: overlay_authorize %s works", ctx[0]->name, client_pubkey);
+    ok (overlay_connect (ctx[1]->ov) == 0,
+        "%s: overlay_connect works", ctx[1]->name);
+
+    errno = 0;
+    ok (overlay_authorize (ctx[0]->ov, "foo", "1234") < 0 && errno == EINVAL,
+        "overlay_authorize with short pubkey fails with EINVAL");
+
+    /* Send request 1->0
+     * Side effect: during recvmsg_timeout(), reactor allows hello request
+     * from 1->0 to be processed at 0.
+     */
+    if (!(msg = flux_request_encode ("meep", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+    ok (overlay_sendmsg (ctx[1]->ov, msg, OVERLAY_ANY) == 0,
+        "%s: overlay_sendmsg request where=ANY works", ctx[1]->name);
+    flux_msg_decref (msg);
+
+    rmsg = recvmsg_timeout (ctx[0], 5);
+    ok (rmsg != NULL,
+        "%s: request was received by overlay", ctx[0]->name);
+    ok (!flux_msg_is_local (rmsg),
+        "%s: flux_msg_is_local fails on parent from child",
+        ctx[1]->name);
+    ok (flux_msg_get_topic (rmsg, &topic) == 0 && streq (topic, "meep"),
+        "%s: received message has expected topic", ctx[0]->name);
+    ok ((sender = flux_msg_route_first (rmsg)) != NULL
+        && streq (sender, ctx[1]->uuid),
+        "%s: received message sender is rank 1", ctx[0]->name);
+
+    /* Send request 0->1
+     * Side effect: during recvmsg_timeout(), reactor allows hello response
+     * from 0->1 to be processed at 1.
+     */
+    if (!(msg = flux_request_encode ("errr", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+    if (flux_msg_set_nodeid (msg, 1) < 0)
+        BAIL_OUT ("flux_msg_set_nodeid failed");
+    ok (overlay_sendmsg (ctx[0]->ov, msg, OVERLAY_ANY) == 0,
+        "%s: overlay_sendmsg request where=ANY works", ctx[0]->name);
+    flux_msg_decref (msg);
+
+    rmsg = recvmsg_timeout (ctx[1], 5);
+    ok (rmsg != NULL,
+        "%s: request was received by overlay", ctx[1]->name);
+    ok (!flux_msg_is_local (rmsg),
+        "%s: flux_msg_is_local fails on child from parent",
+        ctx[1]->name);
+    ok (flux_msg_get_topic (rmsg, &topic) == 0 && streq (topic, "errr"),
+        "%s: request has expected topic", ctx[1]->name);
+    ok ((sender = flux_msg_route_first (rmsg)) != NULL
+        && streq (sender, ctx[0]->uuid),
+        "%s: request sender is rank 0", ctx[1]->name);
+
+    /* Response 1->0
+     */
+    if (!(msg = flux_response_encode ("m000", NULL)))
+        BAIL_OUT ("flux_response_encode failed");
+    if (flux_msg_route_push (msg, ctx[0]->uuid) < 0)
+        BAIL_OUT ("flux_msg_route_push failed");
+    ok (overlay_sendmsg (ctx[1]->ov, msg, OVERLAY_ANY) == 0,
+        "%s: overlay_sendmsg response where=ANY works", ctx[1]->name);
+    flux_msg_decref (msg);
+
+    rmsg = recvmsg_timeout (ctx[0], 5);
+    ok (rmsg != NULL,
+        "%s: response was received by overlay", ctx[0]->name);
+    ok (!flux_msg_is_local (rmsg),
+        "%s: flux_msg_is_local returns false for response from child",
+        ctx[0]->name);
+    ok (flux_msg_get_topic (rmsg, &topic) == 0 && streq (topic, "m000"),
+        "%s: received message has expected topic", ctx[0]->name);
+    ok (flux_msg_route_count (rmsg) == 0,
+        "%s: received message has no routes", ctx[0]->name);
+
+    /* Event 1->0
+     */
+    if (!(msg = flux_event_encode ("eeek", NULL)))
+        BAIL_OUT ("flux_event_encode failed");
+    ok (overlay_sendmsg (ctx[1]->ov, msg, OVERLAY_UPSTREAM) == 0,
+        "%s: overlay_sendmsg event where=UP works", ctx[1]->name);
+    flux_msg_decref (msg);
+
+    rmsg = recvmsg_timeout (ctx[0], 5);
+    ok (rmsg != NULL,
+        "%s: event was received by overlay", ctx[0]->name);
+    ok (flux_msg_get_topic (rmsg, &topic) == 0 && streq (topic, "eeek"),
+        "%s: received message has expected topic", ctx[0]->name);
+    ok (!flux_msg_is_local (rmsg),
+        "%s: flux_msg_is_local returns false for event from child",
+        ctx[0]->name);
+
+    /* Response 0->1
+     */
+    if (!(msg = flux_response_encode ("moop", NULL)))
+        BAIL_OUT ("flux_response_encode failed");
+    if (flux_msg_route_push (msg, ctx[1]->uuid) < 0)
+        BAIL_OUT ("flux_msg_route_push failed");
+    ok (overlay_sendmsg (ctx[0]->ov, msg, OVERLAY_ANY) == 0,
+        "%s: overlay_sendmsg response where=ANY works", ctx[0]->name);
+    flux_msg_decref (msg);
+
+    rmsg = recvmsg_timeout (ctx[1], 5);
+    ok (msg != NULL,
+        "%s: response was received by overlay", ctx[1]->name);
+    ok (flux_msg_get_topic (rmsg, &topic) == 0 && streq (topic, "moop"),
+        "%s: response has expected topic", ctx[1]->name);
+    ok (flux_msg_route_count (rmsg) == 0,
+        "%s: response has no routes", ctx[1]->name);
+
+    /* Event 0->1
+     */
+    if (!(msg = flux_event_encode ("eeeb", NULL)))
+        BAIL_OUT ("flux_event_encode failed");
+    ok (overlay_sendmsg (ctx[0]->ov, msg, OVERLAY_DOWNSTREAM) == 0,
+        "%s: overlay_sendmsg event where=DOWN works", ctx[0]->name);
+    flux_msg_decref (msg);
+
+    rmsg = recvmsg_timeout (ctx[1], 5);
+    ok (rmsg != NULL,
+        "%s: event was received by overlay", ctx[1]->name);
+    ok (flux_msg_get_topic (rmsg, &topic) == 0 && streq (topic, "eeeb"),
+        "%s: received message has expected topic", ctx[1]->name);
+
+    /* Cover some error code in overlay_bind() where the ZAP handler
+     * fails to initialize because its endpoint is already bound.
+     */
+    errno = 0;
+    ok (overlay_bind (ctx[1]->ov, "ipc://@foo") < 0 && errno == EADDRINUSE,
+        "%s: second overlay_bind in proc fails with EADDRINUSE", ctx[0]->name);
+
+    /* Various tests of rank 2 without proper authorization.
+     * First a baseline - resend 1->0 and make sure timed recv works.
+     * Test message will be reused below.
+     */
+    /* 0) Baseline
+     * 'msg' created here will be reused in each test.
+     */
+    if (!(msg = flux_request_encode ("erp", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+    ok (overlay_sendmsg (ctx[1]->ov, msg, OVERLAY_UPSTREAM) == 0,
+        "%s: overlay_sendmsg where=UPSTREAM works", ctx[1]->name);
+    rmsg = recvmsg_timeout (ctx[0], 5);
+    ok (rmsg != NULL,
+        "%s: message was received by overlay", ctx[0]->name);
+    errno = 0;
+    ok (recvmsg_timeout (ctx[0], 0.1) == NULL  && errno == ETIMEDOUT,
+        "%s: test reactor timed out as expected", ctx[0]->name);
+
+    /* 1) No security
+     */
+    if (!(zsock_none = zmq_socket (zctx, ZMQ_DEALER))
+        || zsetsockopt_int (zsock_none, ZMQ_LINGER, 5) < 0
+        || zsetsockopt_str (zsock_none, ZMQ_IDENTITY, "2") < 0)
+        BAIL_OUT ("zmq_socket failed");
+    ok (zmq_connect (zsock_none, parent_uri) == 0,
+        "none-2: zmq_connect %s (no security) works", parent_uri);
+    ok (zmqutil_msg_send (zsock_none, msg) == 0,
+        "none-2: zsock_msg_sendzsock works");
+
+    /* 2) Curve, and correct server public key, but client public key
+     * was not authorized
+     */
+    if (!(zsock_curve = zmq_socket (zctx, ZMQ_DEALER))
+        || zsetsockopt_int (zsock_curve, ZMQ_LINGER, 5) < 0
+        || zsetsockopt_str (zsock_curve, ZMQ_ZAP_DOMAIN, "flux") < 0
+        || zsetsockopt_str (zsock_curve, ZMQ_CURVE_SERVERKEY, server_pubkey) < 0
+        || zsetsockopt_str (zsock_curve, ZMQ_IDENTITY, "2") < 0)
+        BAIL_OUT ("zmq_socket failed");
+    if (!(cert = cert_create ()))
+        BAIL_OUT ("zcert_new failed");
+    cert_apply (cert, zsock_curve);
+    cert_destroy (cert);
+    ok (zmq_connect (zsock_curve, parent_uri) == 0,
+        "curve-2: zmq_connect %s works", parent_uri);
+    ok (zmqutil_msg_send (zsock_curve, msg) == 0,
+        "curve-2: zmqutil_msg_send works");
+
+    /* Neither of the above attempts should have gotten a message through.
+     */
+    errno = 0;
+    ok (recvmsg_timeout (ctx[0], 1.0) == NULL  && errno == ETIMEDOUT,
+        "%s: no messages received within 1.0s", ctx[0]->name);
+
+    flux_msg_decref (msg);
+    zmq_close (zsock_none);
+    zmq_close (zsock_curve);
+
+    ctx_destroy (ctx[1]);
+    ctx_destroy (ctx[0]);
+}
+
+void test_create (flux_t *h,
+                  int size,
+                  struct context *ctx[])
+{
+    char uri[64] = { 0 };
+    int rank;
+
+    for (rank = 0; rank < size; rank++) {
+        ctx[rank] = ctx_create (h, size, rank, NULL, recv_cb);
+        if (overlay_set_topology (ctx[rank]->ov, ctx[rank]->topo) < 0)
+            BAIL_OUT ("%s: overlay_set_topology failed", ctx[rank]->name);
+        if (rank == 0) {
+            snprintf (uri, sizeof (uri), "ipc://@%s", ctx[0]->name);
+            /* Call overlay_bind() before overlay_authorize() is called
+             * for the other ranks, since overlay_bind() creates the ZAP
+             * handler, and overlay_authorize() will fail if it doesn't
+             * exist.
+             */
+            if (overlay_bind (ctx[0]->ov, uri) < 0)
+                BAIL_OUT ("%s: overlay_bind failed", ctx[0]->name);
+        }
+        else {
+            if (overlay_authorize (ctx[0]->ov,
+                                   ctx[rank]->name,
+                                   overlay_cert_pubkey (ctx[rank]->ov)) < 0)
+                BAIL_OUT ("%s: overlay_authorize failed", ctx[rank]->name);
+            if (overlay_set_parent_pubkey (ctx[rank]->ov,
+                                   overlay_cert_pubkey (ctx[0]->ov)) < 0)
+                BAIL_OUT ("%s: overlay_set_parent_pubkey failed", ctx[1]->name);
+            if (overlay_set_parent_uri (ctx[rank]->ov, uri) < 0)
+                BAIL_OUT ("%s: overlay_set_parent_uri %s failed", ctx[1]->name);
+        }
+    }
+
+}
+
+void test_destroy (int size, struct context *ctx[])
+{
+    int rank;
+
+    for (rank = 0; rank < size; rank++)
+        ctx_destroy (ctx[rank]);
+}
+
+void monitor_diag_cb (struct overlay *ov, uint32_t rank, void *arg)
+{
+    struct context *ctx = arg;
+    diag ("%s: rank=%d status=%s children=%d parent_error=%s",
+          ctx->name,
+          (int)rank,
+          overlay_get_subtree_status (ov, rank),
+          overlay_get_child_peer_count (ov),
+          overlay_parent_error (ov) ? "true" : "false");
+}
+
+void monitor_cb (struct overlay *ov, uint32_t rank, void *arg)
+{
+    struct context *ctx = arg;
+    const char *status = overlay_get_subtree_status (ov, rank);
+    monitor_diag_cb (ov, rank, arg);
+    if (overlay_parent_error (ov)
+        || streq (status, "full")
+        || streq (status, "partial")
+        || streq (status, "lost")
+        || streq (status, "offline"))
+        flux_reactor_stop (flux_get_reactor (ctx->h));
+}
+
+
+void check_monitor (flux_t *h)
+{
+    const int size = 5;
+    struct context *ctx[size];
+
+    diag ("check_monitor BEGIN");
+
+    test_create (h, size, ctx);
+
+    diag ("check_monitor test_create returned");
+
+    /* If anything changes on rank 0, stop the reactor
+     */
+    overlay_set_monitor_cb (ctx[0]->ov, monitor_cb, ctx[0]);
+
+    /* connect (1->0) - rank 0 stops reactor on connect */
+    overlay_set_monitor_cb (ctx[1]->ov, monitor_diag_cb, ctx[1]);
+    if (overlay_connect (ctx[1]->ov) < 0)
+        BAIL_OUT ("%s: overlay_connect failed", ctx[1]->name);
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0,
+        "%s: reactor ran until child connected", ctx[0]->name);
+    ok (overlay_get_child_peer_count (ctx[0]->ov) == 1,
+        "%s: overlay_get_child_peer_count returns 1", ctx[0]->name);
+    overlay_set_monitor_cb (ctx[0]->ov, monitor_diag_cb, ctx[0]);
+
+    /* connect (2->0) - rank 2 stops reactor on connect */
+    overlay_set_monitor_cb (ctx[2]->ov, monitor_cb, ctx[2]);
+    if (overlay_connect (ctx[2]->ov) < 0)
+        BAIL_OUT ("%s: overlay_connect failed", ctx[2]->name);
+
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0,
+        "%s: reactor ran until child connected", ctx[0]->name);
+    ok (overlay_get_child_peer_count (ctx[0]->ov) == 2,
+        "%s: overlay_get_child_peer_count returns 2", ctx[0]->name);
+    ok (overlay_parent_error (ctx[2]->ov) == false,
+        "%s: overlay_parent_error returns false", ctx[2]->name);
+
+    /* rank 3 will try to connect with simulated wrong flux-core version
+     * Disable rank 0 stopping the reactor and enable rank 3 to do it.
+     */
+    overlay_set_monitor_cb (ctx[3]->ov, monitor_cb, ctx[3]);
+    overlay_set_version (ctx[3]->ov, 0xffffff);
+    if (overlay_connect (ctx[3]->ov) < 0)
+        BAIL_OUT ("%s: overlay_connect failed", ctx[3]->name);
+
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0,
+        "%s: reactor ran until bad version connection fails", ctx[0]->name);
+    ok (overlay_get_child_peer_count (ctx[0]->ov) == 2,
+        "%s: overlay_get_child_peer_count is still 2", ctx[0]->name);
+    ok (overlay_parent_error (ctx[3]->ov) == true,
+        "%s: overlay_parent_error returns true", ctx[3]->name);
+    overlay_set_monitor_cb (ctx[3]->ov, monitor_diag_cb, ctx[3]);
+
+    /* rank 4 will have its rank altered to '42' for overlay.hello
+     */
+    overlay_set_monitor_cb (ctx[4]->ov, monitor_cb, ctx[4]);
+    overlay_set_rank (ctx[4]->ov, 42);
+    if (overlay_connect (ctx[4]->ov) < 0)
+        BAIL_OUT ("%s: overlay_connect failed", ctx[4]->name);
+
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0,
+        "%s: reactor ran until bad rank connection fails", ctx[0]->name);
+    ok (overlay_get_child_peer_count (ctx[0]->ov) == 2,
+        "%s: overlay_get_child_peer_count is still 2", ctx[0]->name);
+    ok (overlay_parent_error (ctx[4]->ov) == true,
+        "%s: overlay_parent_error returns true", ctx[4]->name);
+
+    test_destroy (size, ctx);
+}
+
+/* Probe some possible failure cases
+ */
+void wrongness (flux_t *h)
+{
+    struct overlay *ov;
+    attr_t *attrs;
+
+    if (!(attrs = attr_create ()))
+        BAIL_OUT ("attr_create failed");
+    errno = 0;
+    ok (overlay_create (NULL, "test0", attrs, zctx, NULL, NULL) == NULL
+        && errno == EINVAL,
+        "overlay_create h=NULL fails with EINVAL");
+    errno = 0;
+    ok (overlay_create (h, "test0", NULL, zctx, NULL, NULL) == NULL
+        && errno == EINVAL,
+        "overlay_create attrs=NULL fails with EINVAL");
+    attr_destroy (attrs);
+
+    if (!(attrs = attr_create ()))
+        BAIL_OUT ("attr_create failed");
+    if (!(ov = overlay_create (h, "test0", attrs, zctx, NULL, NULL)))
+        BAIL_OUT ("overlay_create failed");
+
+    errno = 0;
+    ok (overlay_bind (ov, "ipc://@foobar") < 0 && errno == EINVAL,
+        "overlay_bind fails if called before rank is known");
+
+    ok (!flux_msg_is_local (NULL),
+        "flux_msg_is_local (NULL) returns false");
+
+    overlay_destroy (ov);
+    attr_destroy (attrs);
+}
+
+void diag_logger (const char *buf, int len, void *arg)
+{
+    struct stdlog_header hdr;
+    const char *msg;
+    size_t msglen;
+    int severity;
+    char *s;
+
+    if (stdlog_decode (buf, len, &hdr, NULL, NULL, &msg, &msglen) < 0)
+        BAIL_OUT ("stdlog_decode failed");
+    severity = STDLOG_SEVERITY (hdr.pri);
+    if (asprintf (&s,
+                  "%s: %.*s\n",
+                  stdlog_severity_to_string (severity),
+                  (int)msglen,
+                  msg) < 0)
+        BAIL_OUT ("asprintf failed");
+    diag (s);
+    if (zlist_append (logs, s) < 0)
+        BAIL_OUT ("zlist_append failed");
+}
+
+int main (int argc, char *argv[])
+{
+    flux_t *h;
+
+    plan (NO_PLAN);
+
+    if (!(zctx = zmq_ctx_new ()))
+        BAIL_OUT ("zmq_ctx_new failed");
+
+    if (!(logs = zlist_new ()))
+        BAIL_OUT ("zlist_new failed");
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop handle");
+    if (flux_attr_set_cacheonly (h, "rank", "0") < 0)
+        BAIL_OUT ("flux_attr_set_cacheonly rank failed");
+    if (flux_attr_set_cacheonly (h, "hostlist", "test[0-7]") < 0)
+        BAIL_OUT ("flux_attr_set_cacheonly hostlist failed");
+    flux_log_set_redirect (h, diag_logger, NULL);
+    flux_log (h, LOG_INFO, "test log message");
+
+    single (h);
+    clear_list (logs);
+
+    trio (h);
+    clear_list (logs);
+
+    /* trio() and check_monitor() tests will bind to the same address
+     * in their tests.  Test can be racy and fail with EADDRINUSE if
+     * prior tests did not complete cleanup.  To ensure there are no
+     * issues, destroy & recreate zctx.  See issue 6404.
+     */
+    zmq_ctx_term (zctx);
+    if (!(zctx = zmq_ctx_new ()))
+        BAIL_OUT ("failed to recreate zmq context");
+
+    check_monitor (h);
+    clear_list (logs);
+
+    wrongness (h);
+
+    flux_close (h);
+    zlist_destroy (&logs);
+
+    zmq_ctx_term (zctx);
+
+    done_testing ();
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
+
+
+
diff --git a/src/broker/test/pmiutil.c b/src/broker/test/pmiutil.c
deleted file mode 100644
index 82e8ffc3279b..000000000000
--- a/src/broker/test/pmiutil.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdlib.h>
-#include <string.h>
-
-#include "src/common/libtap/tap.h"
-#include "src/common/libpmi/pmi.h"
-
-#include "pmiutil.h"
-
-int main (int argc, char **argv)
-{
-    struct pmi_handle *pmi;
-    int result;
-    struct pmi_params params;
-    char val[64];
-
-    plan (NO_PLAN);
-
-    /* Enable some debug output on stderr.
-     */
-    (void)setenv ("PMI_DEBUG", "1", 1);
-
-    /* Force singleton (ours) by ensuring pmi_simple_client
-     * and dlopen will fail
-     */
-    (void)unsetenv ("PMI_FD");
-    (void)unsetenv ("PMI_RANK");
-    (void)unsetenv ("PMI_FD");
-    (void)setenv ("PMI_LIBRARY", "/nope.so", 1);
-
-    pmi = broker_pmi_create ();
-    ok (pmi != NULL,
-        "broker_pmi_create() works (singleton)");
-
-    result = broker_pmi_init (pmi);
-    ok (result == PMI_SUCCESS,
-        "broker_pmi_init() works");
-
-    memset (&params, 0, sizeof (params));
-    result = broker_pmi_get_params (pmi, &params);
-    ok (result == PMI_SUCCESS,
-        "broker_pmi_get_params() works");
-    ok (params.rank == 0 && params.size == 1,
-        "rank=0 size=1");
-    ok (strlen (params.kvsname) > 0,
-        "kvsname is not the empty string");
-    diag ("kvsname=%s", params.kvsname);
-
-    result = broker_pmi_kvs_put (pmi, params.kvsname, "foo", "bar");
-    ok (result == PMI_SUCCESS,
-        "broker_pmi_kvs_put %s foo=bar works", params.kvsname);
-
-    result = broker_pmi_barrier (pmi);
-    ok (result == PMI_SUCCESS,
-        "broker_pmi_barrier works");
-
-    result = broker_pmi_kvs_get (pmi, params.kvsname, "foo", val, sizeof (val));
-    ok (result != PMI_SUCCESS,
-        "broker_pmi_kvs_get fails since singleton doesn't implement kvs");
-    // at least while we can get away without it!
-
-    result = broker_pmi_finalize (pmi);
-    ok (result == PMI_SUCCESS,
-        "broker_pmi_finalize() works");
-
-    broker_pmi_destroy (pmi);
-    done_testing ();
-    return 0;
-}
-
-/*
- * vi:ts=4 sw=4 expandtab
- */
diff --git a/src/broker/test/reduce.c b/src/broker/test/reduce.c
deleted file mode 100644
index 072ffb7ef165..000000000000
--- a/src/broker/test/reduce.c
+++ /dev/null
@@ -1,381 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <errno.h>
-#include <czmq.h>
-#include <flux/core.h>
-
-#include "src/common/libutil/xzmalloc.h"
-#include "src/common/libutil/oom.h"
-#include "src/common/libtap/tap.h"
-#include "src/broker/reduce.h"
-
-int reduce_calls = 0;
-int reduce_items = 0;
-void reduce (flux_reduce_t *r, int batchnum, void *arg)
-{
-    void *item;
-    zlist_t *tmp = zlist_new ();
-
-    if (!tmp)
-        oom ();
-    reduce_calls++;
-    while ((item = flux_reduce_pop (r))) {
-        if (zlist_push (tmp, item) < 0)
-            oom ();
-        reduce_items++;
-    }
-    while ((item = zlist_pop (tmp))) {
-        if (flux_reduce_push (r, item) < 0)
-            oom ();
-    }
-    zlist_destroy (&tmp);
-}
-
-int sink_calls = 0;
-int sink_items = 0;
-void sink (flux_reduce_t *r, int batchnum, void *arg)
-{
-    void *item;
-
-    sink_calls++;
-    while ((item = flux_reduce_pop (r))) {
-        free (item);
-        sink_items++;
-    }
-}
-
-int forward_calls = 0;
-int forward_items = 0;
-void forward (flux_reduce_t *r, int batchnum, void *arg)
-{
-    void *item;
-
-    forward_calls++;
-    while ((item = flux_reduce_pop (r))) {
-        free (item);
-        forward_items++;
-    }
-}
-
-void clear_counts (void)
-{
-    sink_calls = sink_items = reduce_calls = reduce_items
-                            = forward_calls = forward_items = 0;
-}
-
-int itemweight (void *item)
-{
-    return 1;
-}
-
-static struct flux_reduce_ops reduce_ops =  {
-    .destroy = free,
-    .reduce = reduce,
-    .sink = sink,
-    .forward = forward,
-    .itemweight = itemweight,
-};
-
-void test_hwm (flux_t *h)
-{
-    flux_reduce_t *r;
-    int i, errors;
-    unsigned int hwm;
-
-    clear_counts ();
-
-    ok ((r = flux_reduce_create (h, reduce_ops, 0., NULL,
-        FLUX_REDUCE_HWMFLUSH)) != NULL,
-        "hwm: flux_reduce_create works");
-
-    ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
-        && hwm == 0,
-        "hwm: hwm is initially zero");
-
-    /* batch 0 is a training batch.
-     * It looks just like no policy.
-     */
-    errors = 0;
-    for (i = 0; i < 100; i++) {
-        if (flux_reduce_append (r, xstrdup ("hi"), 0) < 0)
-            errors++;
-    }
-    ok (errors == 0,
-        "hwm.0: flux_reduce_append added 100 items");
-    cmp_ok (reduce_calls, "==", 0,
-        "hwm.0: op.reduce not called (training)");
-    cmp_ok (sink_calls, "==", 100,
-        "hwm.0: op.sink called 100 times");
-    cmp_ok (sink_items, "==", 100,
-        "hwm.0: op.sink processed 100 items");
-
-    clear_counts ();
-
-    /* batch 1 has a hwm.  Put in one short of hwm items.
-     */
-    errors = 0;
-    for (i = 0; i < 99; i++) {
-        if (flux_reduce_append (r, xstrdup ("hi"), 1) < 0)
-            errors++;
-    }
-    ok (errors == 0,
-        "hwm.1: flux_reduce_append added 99 items");
-    ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
-        && hwm == 100,
-        "hwm.0: hwm is 100");
-    cmp_ok (reduce_calls, "==", 98,
-        "hwm.1: op.reduce called 98 times");
-    cmp_ok (sink_calls, "==", 0,
-        "hwm.1: op.sink not called yet");
-
-    /* Now finish batch 1 with one item.  Everything should go thru.
-     */
-    ok (flux_reduce_append (r, xstrdup ("hi"), 1) == 0,
-        "hwm.1: flux_reduce_append added 1 item");
-    cmp_ok (reduce_calls, "==", 99,
-        "hwm.1: op.reduce called");
-    cmp_ok (sink_calls, "==", 1,
-        "hwm.1: op.sink called 1 time");
-    cmp_ok (sink_items, "==", 100,
-        "hwm.1: op.sink handled 100 items");
-    ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
-        && hwm == 100,
-        "hwm.1: hwm is 100");
-
-    clear_counts ();
-
-    /* Straggler test
-     * Start batch 2, then append one item from batch 1.
-     * This should cause last hwm to be recomputed to be 101 instead of 100.
-     * Straggler should immediately be sinked.
-     */
-    ok (flux_reduce_append (r, xstrdup ("hi"), 2) == 0,
-        "hwm.2: flux_reduce_append added 1 item");
-    cmp_ok (reduce_calls, "==", 0,
-        "hwm.2: op.reduce not called");
-    cmp_ok (sink_calls, "==", 0,
-        "hwm.2: op.sink not called");
-    ok (flux_reduce_append (r, xstrdup ("hi"), 1) == 0,
-        "hwm.1: flux_reduce_append added 1 straggler");
-    cmp_ok (reduce_calls, "==", 0,
-        "hwm.1: op.reduce not called");
-    cmp_ok (sink_calls, "==", 1,
-        "hwm.1: op.sink called 1 time");
-    cmp_ok (sink_items, "==", 1,
-        "hwm.1: op.sink handled 1 item");
-    ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
-        && hwm == 101,
-        "hwm.1: hwm is 101");
-
-    sink_items = sink_calls = 0; // don't count batch 1 straggler below
-
-    /* At this point we have one batch 2 item in queue.
-     * Put in 99 more and we should be one short of 101 hwm.
-     */
-    errors = 0;
-    for (i = 0; i < 99; i++) {
-        if (flux_reduce_append (r, xstrdup ("hi"), 2) < 0)
-            errors++;
-    }
-    ok (errors == 0,
-        "hwm.2: flux_reduce_append added 99 items");
-    cmp_ok (reduce_calls, "==", 99,
-        "hwm.2: op.reduce called 99 times");
-    cmp_ok (sink_calls, "==", 0,
-        "hwm.2: op.sink not called yet");
-    ok (flux_reduce_append (r, xstrdup ("hi"), 2) == 0,
-        "hwm.2: flux_reduce_append added 1 item");
-    cmp_ok (sink_calls, "==", 1,
-        "hwm.2: op.sink called 1 time");
-    cmp_ok (sink_items, "==", 101,
-        "hwm.2: op.sink handled 101 items");
-    ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
-        && hwm == 101,
-        "hwm.2: hwm is 101");
-
-    clear_counts ();
-
-    /* Manually set the hwm to 10.
-     * Append 20 items to batch 3.
-     * Reduce is called on the first set of 10.
-     * The second set of 10 will be immediately flushed.
-     * Put in one batch 4 item and verify the HWM is still 10.
-     */
-    hwm = 10;
-    ok (flux_reduce_opt_set (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0,
-        "hwm.3: hwm set to 10");
-    errors = 0;
-    for (i = 0; i < 20; i++) {
-        if (flux_reduce_append (r, xstrdup ("hi"), 3) < 0)
-            errors++;
-    }
-    ok (errors == 0,
-        "hwm.3: flux_reduce_append added 20 items");
-    cmp_ok (reduce_calls, "==", 9,
-        "hwm.3: op.reduce called 9 times");
-    cmp_ok (sink_calls, "==", 11,
-        "hwm.3: op.sink called 11 times");
-    cmp_ok (sink_items, "==", 20,
-        "hwm.3: op.sink handled 20 items");
-    ok (flux_reduce_append (r, xstrdup ("hi"), 4) == 0,
-        "hwm.4: flux_reduce_append added one item");
-    hwm = 0;
-    ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
-        && hwm == 10,
-        "hwm.4: hwm is still 10");
-
-    flux_reduce_destroy (r);
-}
-
-void test_nopolicy (flux_t *h)
-{
-    flux_reduce_t *r;
-    int i, errors;
-
-    clear_counts ();
-
-    ok ((r = flux_reduce_create (h, reduce_ops, 0., NULL, 0)) != NULL,
-        "nopolicy: flux_reduce_create works");
-
-    errors = 0;
-    for (i = 0; i < 100; i++) {
-        if (flux_reduce_append (r, xstrdup ("hi"), 0) < 0)
-            errors++;
-    }
-    ok (errors == 0,
-        "nopolicy: flux_reduce_append added 100 items in batch 0");
-    cmp_ok (forward_calls, "==", 0,
-        "nopolicy: op.forward not called as we are rank 0");
-    cmp_ok (reduce_calls, "==", 0,
-        "nopolicy: op.reduce not called as we have no flush policy");
-    cmp_ok (sink_calls, "==", 100,
-        "nopolicy: op.sink called 100 times");
-    cmp_ok (sink_items, "==", 100,
-        "nopolicy: op.sink processed 100 items");
-
-    flux_reduce_destroy (r);
-}
-
-void test_timed (flux_t *h)
-{
-    flux_reduce_t *r;
-    int i, errors;
-    double timeout;
-
-    clear_counts ();
-
-    ok ((r = flux_reduce_create (h, reduce_ops, 0.1, NULL,
-                                 FLUX_REDUCE_TIMEDFLUSH)) != NULL,
-        "timed: flux_reduce_create works");
-    if (!r)
-        BAIL_OUT();
-    ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_TIMEOUT, &timeout,
-                             sizeof (timeout)) == 0 && timeout == 0.1,
-        "timed: flux_reduce_opt_get TIMEOUT returned timeout");
-
-    /* Append 100 items in batch 0 before starting reactor.
-     * Reduction occurs at each append.
-     * Nothing should be sinked.
-     */
-    errors = 0;
-    for (i = 0; i < 100; i++) {
-        if (flux_reduce_append (r, xstrdup ("hi"), 0) < 0)
-            errors++;
-    }
-    ok (errors == 0,
-        "timed.0: flux_reduce_append added 100 items");
-    cmp_ok (reduce_calls, "==", 99,
-        "timed.0: op.reduce called 99 times");
-    cmp_ok (sink_calls, "==", 0,
-        "timed.0: op.sink called 0 times");
-
-    /* Start reactor so timeout handler can run.
-     * It should fire once and sink all items in one sink call.
-     */
-    ok (flux_reactor_run (flux_get_reactor (h), 0) == 0,
-        "timed.0: reactor completed normally");
-    cmp_ok (sink_calls, "==", 1,
-        "timed.0: op.sink called 1 time");
-    cmp_ok (sink_items, "==", 100,
-        "timed.0: op.sink processed 100 items");
-
-    clear_counts ();
-
-    /* Now append one more item to batch 0.
-     * It should be immediately flushed.
-     */
-    ok (flux_reduce_append (r, xstrdup ("hi"), 0) == 0,
-        "timed.0: flux_reduce_append added 1 more item");
-    cmp_ok (reduce_calls, "==", 0,
-        "timed.0: op.reduce not called");
-    cmp_ok (sink_calls, "==", 1,
-        "timed.0: op.sink called 1 time");
-    cmp_ok (sink_items, "==", 1,
-        "timed.0: op.sink processed 1 items");
-
-    clear_counts ();
-
-    /* Append 100 items to batch 1.
-     * It should behave like the first batch.
-     */
-    errors = 0;
-    for (i = 0; i < 100; i++) {
-        if (flux_reduce_append (r, xstrdup ("hi"), 1) < 0)
-            errors++;
-    }
-    ok (errors == 0,
-        "timed.1: flux_reduce_append added 100 items");
-    cmp_ok (reduce_calls, "==", 99,
-        "timed.1: op.reduce called 99 times");
-    cmp_ok (sink_calls, "==", 0,
-        "timed.1: op.sink called 0 times");
-
-    /* Start reactor so timeout handler can run.
-     * It should fire once and sink all items in one sink call.
-     */
-    ok (flux_reactor_run (flux_get_reactor (h), 0) == 0,
-        "timed.1: reactor completed normally");
-    cmp_ok (sink_calls, "==", 1,
-        "timed.1: op.sink called 1 time");
-    cmp_ok (sink_items, "==", 100,
-        "timed.1: op.sink processed 100 items");
-
-    flux_reduce_destroy (r);
-}
-
-int main (int argc, char *argv[])
-{
-    flux_t *h;
-
-    plan (NO_PLAN);
-
-    ok ((h = flux_open ("loop://", 0)) != NULL,
-        "opened loop connector");
-    if (!h)
-        BAIL_OUT ("can't continue without loop handle");
-
-    flux_attr_set_cacheonly (h, "rank", "0");
-    flux_attr_set_cacheonly (h, "tbon.level", "0");
-    flux_attr_set_cacheonly (h, "tbon.maxlevel", "0");
-
-    test_nopolicy (h); // 6
-    test_hwm (h); // 37
-    test_timed(h); // 18
-
-    flux_close (h);
-    done_testing();
-    return (0);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/src/broker/test/runat.c b/src/broker/test/runat.c
new file mode 100644
index 000000000000..cf215d30a127
--- /dev/null
+++ b/src/broker/test/runat.c
@@ -0,0 +1,390 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <string.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/stdlog.h"
+
+#include "src/broker/runat.h"
+
+static zlist_t *logs;
+
+struct context {
+    flux_t *h;
+};
+
+void clear_list (zlist_t *list)
+{
+    char *s;
+    while ((s = zlist_pop (list)))
+        free (s);
+}
+
+int match_list (zlist_t *list, const char *key)
+{
+    char *s;
+    int count = 0;
+
+    s = zlist_first (list);
+    while (s) {
+        if (strstr (s, key) != NULL)
+            count++;
+        s = zlist_next (list);
+    }
+    return count;
+}
+
+/* Just stop the reactor so we can call flux_reactor_run() and process
+ * one completion.
+ */
+static int completion_called;
+void test_completion (struct runat *r, const char *name, void *arg)
+{
+    struct context *ctx = arg;
+    flux_reactor_stop (flux_get_reactor (ctx->h));
+    completion_called++;
+}
+
+void basic (flux_t *h)
+{
+    struct runat *r;
+    struct context ctx;
+    int flags, rc;
+
+    ctx.h = h;
+
+    r = runat_create (h, "local://notreally", false);
+    ok (r != NULL,
+        "runat_create works");
+
+    /* run true;true */
+    clear_list (logs);
+    ok (runat_is_defined (r, "test1") == false,
+        "runat_is_defined name=test1 returns false");
+    ok (runat_is_completed (r, "test1") == false,
+        "runat_is_completed name=test1 returns false");
+    ok (runat_push_shell_command (r, "test1", "true", 0) == 0
+        && runat_push_shell_command (r, "test1", "true", 0) == 0,
+        "pushed true;true");
+    ok (runat_is_defined (r, "test1") == true,
+        "runat_is_defined name=test1 returns true after creation");
+    ok (runat_is_completed (r, "test1") == false,
+        "runat_is_completed returns false");
+    ok (runat_start (r, "test1", test_completion, &ctx) == 0,
+        "runat_start works");
+    completion_called = 0;
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0
+        && completion_called == 1,
+        "completion called once");
+    rc = -1;
+    ok (runat_get_exit_code (r, "test1", &rc) == 0 && rc == 0,
+        "exit code is zero");
+    ok (match_list (logs, "Exited") == 2,
+        "Exited was logged twice");
+    ok (runat_is_completed (r, "test1") == true,
+        "runat_is_completed returns true");
+
+    /* run false;true */
+    clear_list (logs);
+    ok (runat_push_shell_command (r, "test2", "true", 0) == 0
+        && runat_push_shell_command (r, "test2", "false", 0) == 0,
+        "pushed true;true");
+    ok (runat_start (r, "test2", test_completion, &ctx) == 0,
+        "runat_start works");
+    completion_called = 0;
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0
+        && completion_called == 1,
+        "completion called once");
+    rc = -1;
+    ok (runat_get_exit_code (r, "test2", &rc) == 0 && rc == 1,
+        "exit code is 1");
+    ok (match_list (logs, "rc=1") == 1
+        && match_list (logs, "Exited") == 2,
+        "Both commands' exit status was logged");
+
+    /* run true;false */
+    clear_list (logs);
+    ok (runat_push_command (r, "test3", "false", 6, 0) == 0
+        && runat_push_command (r, "test3", "true", 5, 0) == 0,
+        "pushed true;true");
+    ok (runat_start (r, "test3", test_completion, &ctx) == 0,
+        "runat_start works");
+    completion_called = 0;
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0
+        && completion_called == 1,
+        "completion called once");
+    rc = -1;
+    ok (runat_get_exit_code (r, "test3", &rc) == 0 && rc == 1,
+        "exit code is 1");
+    ok (match_list (logs, "rc=1") == 1
+        && match_list (logs, "Exited") == 2,
+        "Both commands' exit status were logged");
+
+    /* generate output to stdout and stderr */
+    clear_list (logs);
+    ok (runat_push_shell_command (r,
+                                  "test4",
+                                  "echo test4-out",
+                                  RUNAT_FLAG_LOG_STDIO) == 0
+    && runat_push_shell_command (r,
+                                 "test4",
+                                 "echo test4-err>&2",
+                                 RUNAT_FLAG_LOG_STDIO) == 0,
+        "pushed echo;echo");
+    ok (runat_start (r, "test4", test_completion, &ctx) == 0,
+        "runat_start works");
+    completion_called = 0;
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0
+        && completion_called == 1,
+        "completion called once");
+    rc = -1;
+    ok (runat_get_exit_code (r, "test4", &rc) == 0 && rc == 0,
+        "exit code is 0");
+    ok (match_list (logs, "Exited") == 2,
+        "Both commands' exit status were logged");
+    ok (match_list (logs, "info: test4.1: test4-out") == 1,
+        "Stdout was logged");
+    ok (match_list (logs, "err: test4.0: test4-err") == 1,
+        "Stderr was logged");
+
+    /* run notfound;echo foo*/
+    clear_list (logs);
+    ok (runat_push_shell_command (r,
+                                  "test5",
+                                  "echo test5-out",
+                                  RUNAT_FLAG_LOG_STDIO) == 0
+        && runat_push_shell_command (r, "test5", "notfound", 0) == 0,
+        "pushed notfound;echo");
+    ok (runat_start (r, "test5", test_completion, &ctx) == 0,
+        "runat_start works");
+    completion_called = 0;
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0
+        && completion_called == 1,
+        "completion called once");
+    rc = -1;
+    ok (runat_get_exit_code (r, "test5", &rc) == 0 && rc != 1,
+        "exit code is nonzero");
+    errno = 0;
+    ok (match_list (logs, "notfound Exited") == 1
+        && match_list (logs, "echo test5-out Exited") == 1,
+        "Both commands' exit status were logged");
+
+    /* run printenv FLUX_URI */
+    clear_list (logs);
+    ok (runat_push_shell_command (r,
+                                  "test6",
+                                  "printenv FLUX_URI",
+                                  RUNAT_FLAG_LOG_STDIO) == 0,
+        "pushed printenv FLUX_URI");
+    ok (runat_start (r, "test6", test_completion, &ctx) == 0,
+        "runat_start works");
+    completion_called = 0;
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0
+        && completion_called == 1,
+        "completion called once");
+    rc = -1;
+    ok (runat_get_exit_code (r, "test6", &rc) == 0 && rc == 0,
+        "exit code zero");
+    ok (match_list (logs, "local://notreally") == 1,
+        "FLUX_URI was set for subprocess");
+
+    /* run sleep 3600, then abort */
+    /* N.B. if sleep has started, the abort function kills it.
+     * If it is not yet started, the subprocess state callback kills it
+     * when it transitions to running.  Either way we should see an
+     * exit code indicating terminated.
+     */
+    clear_list (logs);
+    flags = 0;
+    /* older versions of glibc, POSIX_SPAWN_SETPGROUP with
+     * posix_spawn(3) can be racy, and abort below can fail to kill
+     * the child.  The result would be a wait for the 3600 sleep to
+     * complete, giving the appearance of a hang.  If we're under an
+     * old version of glibc, force the use of fork(2)/exec(2) over
+     * posix_spawn(3).
+     */
+#if __GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ < 24)
+    flags |= RUNAT_FLAG_FORK_EXEC;
+#endif
+    ok (runat_push_shell_command (r, "test7", "true", flags) == 0
+            && runat_push_shell_command (r, "test7", "sleep 3600", flags) == 0,
+        "pushed true;sleep 3600");
+    ok (runat_start (r, "test7", test_completion, &ctx) == 0,
+        "runat_start works");
+    ok (runat_abort (r, "test7") == 0,
+        "runat_abort works");
+    completion_called = 0;
+    ok (flux_reactor_run (flux_get_reactor (h), 0) >= 0
+        && completion_called == 1,
+        "completion called once");
+    ok (runat_get_exit_code (r, "test7", &rc) == 0 && rc == 129,
+        "exit code 129 (= signal 15 + 1)");
+    ok (match_list (logs, "Hangup") == 1,
+        "process termination was logged");
+    rc = -1;
+
+    runat_destroy (r);
+}
+
+void diag_logger (const char *buf, int len, void *arg)
+{
+    struct stdlog_header hdr;
+    const char *msg;
+    size_t msglen;
+    int severity;
+    char *s;
+
+    if (stdlog_decode (buf, len, &hdr, NULL, NULL, &msg, &msglen) < 0)
+        BAIL_OUT ("stdlog_decode failed");
+    severity = STDLOG_SEVERITY (hdr.pri);
+    if (asprintf (&s,
+                  "%s: %.*s\n",
+                  stdlog_severity_to_string (severity),
+                  (int)msglen,
+                  msg) < 0)
+        BAIL_OUT ("asprintf failed");
+    diag (s);
+    if (zlist_append (logs, s) < 0)
+        BAIL_OUT ("zlist_append failed");
+}
+
+void badinput (flux_t *h)
+{
+    struct runat *r;
+    int rc;
+
+    if (!(r = runat_create (h, NULL, false)))
+        BAIL_OUT ("runat_create failed");
+
+    ok (runat_is_defined (NULL, "foo") == false,
+        "runat_is_defined r=NULL returns false");
+    ok (runat_is_defined (r, NULL) == false,
+        "runat_is_defined name=NULL returns false");
+    ok (runat_is_completed (NULL, "foo") == false,
+        "runat_is_completed r=NULL returns false");
+    ok (runat_is_completed (r, NULL) == false,
+        "runat_is_completed name=NULL returns false");
+
+    errno = 0;
+    ok (runat_start (NULL, "foo", NULL, NULL) < 0 && errno == EINVAL,
+        "runat_start r=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_start (r, NULL, NULL, NULL) < 0 && errno == EINVAL,
+        "runat_start name=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_start (r, "noexit", NULL, NULL) < 0 && errno == ENOENT,
+        "runat_start name=noexist fails with ENOENT");
+
+    errno = 0;
+    ok (runat_abort (NULL, "foo") < 0 && errno == EINVAL,
+        "runat_abort r=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_abort (r, NULL) < 0 && errno == EINVAL,
+        "runat_abort name=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_abort (r, "noexit") < 0 && errno == ENOENT,
+        "runat_abort name=noexist fails with ENOENT");
+
+    errno = 0;
+    ok (runat_get_exit_code (NULL, "foo", &rc) < 0 && errno == EINVAL,
+        "runat_get_exit_code r=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_get_exit_code (r, NULL, &rc) < 0 && errno == EINVAL,
+        "runat_get_exit_code name=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_get_exit_code (r, "foo", NULL) < 0 && errno == EINVAL,
+        "runat_get_exit_code rc=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_get_exit_code (r, "noexist", &rc) < 0 && errno == ENOENT,
+        "runat_get_exit_code rc=NULL fails with ENOENT");
+
+    errno = 0;
+    ok (runat_push_shell (NULL, "foo", NULL, 0) < 0 && errno == EINVAL,
+        "runat_push_shell r=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_push_shell (r, NULL, NULL, 0) < 0 && errno == EINVAL,
+        "runat_push_shell name=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_push_shell (r, "foo", NULL, RUNAT_FLAG_LOG_STDIO) < 0
+        && errno == EINVAL,
+        "runat_push_shell flags=RUNAT_FLAG_LOG_STDIO fails with EINVAL");
+
+    errno = 0;
+    ok (runat_push_shell_command (NULL, "a", "a", 0) < 0 && errno == EINVAL,
+        "runat_push_shell_command r=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_push_shell_command (r, NULL, "a", 0) < 0 && errno == EINVAL,
+        "runat_push_shell_command name=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_push_shell_command (r, "foo", NULL, 0) < 0 && errno == EINVAL,
+        "runat_push_shell_command cmdline=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (runat_push_command (NULL, "a", "a", 1, 0) < 0 && errno == EINVAL,
+        "runat_push_command r=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_push_command (r, NULL, "a", 1, 0) < 0 && errno == EINVAL,
+        "runat_push_command name=NULL fails with EINVAL");
+    errno = 0;
+    ok (runat_push_command (r, "foo", NULL, 1, 0) < 0 && errno == EINVAL,
+        "runat_push_command argz=NULL fails with EINVAL");
+
+    runat_destroy (r);
+}
+
+int main (int argc, char *argv[])
+{
+    flux_t *h;
+    flux_reactor_t *r;
+
+    plan (NO_PLAN);
+
+    /* these tests require bourne shell */
+    if (setenv ("SHELL", "/bin/sh", 1) < 0)
+        BAIL_OUT ("setenv set /bin/sh failed");
+
+    if (!(logs = zlist_new ()))
+        BAIL_OUT ("zlist_new failed");
+    if (!(r = flux_reactor_create (FLUX_REACTOR_SIGCHLD)))
+        BAIL_OUT ("flux_reactor_create failed");
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop handle");
+    if (flux_set_reactor (h, r) < 0)
+        BAIL_OUT ("flux_set_reactor failed");
+    if (flux_attr_set_cacheonly (h, "rank", "0") < 0)
+        BAIL_OUT ("flux_attr_set_cacheonly rank failed");
+    flux_log_set_redirect (h, diag_logger, NULL);
+    flux_log (h, LOG_INFO, "test log message");
+
+    basic (h);
+    badinput (h);
+
+    flux_reactor_destroy (r);
+    flux_close (h);
+
+    clear_list (logs);
+    zlist_destroy (&logs);
+    done_testing ();
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
+
+
+
diff --git a/src/broker/test/service.c b/src/broker/test/service.c
index 18df27724aa0..2ce6743a8eaa 100644
--- a/src/broker/test/service.c
+++ b/src/broker/test/service.c
@@ -8,8 +8,10 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <flux/core.h>
-#include <czmq.h>
 #include <stdio.h>
 
 #include <flux/core.h>
@@ -18,21 +20,26 @@
 
 #include "src/common/libtap/tap.h"
 
-const flux_msg_t *foo_cb_msg;
+flux_msg_t *foo_cb_msg;
 void *foo_cb_arg;
 int foo_cb_called;
 int foo_cb_rc;
 int foo_cb_errno;
 
-static int foo_cb (const flux_msg_t *msg, void *arg)
+static int foo_cb (flux_msg_t **msg, void *arg)
 {
-    foo_cb_msg = msg;
+    foo_cb_msg = *msg;
     foo_cb_arg = arg;
     foo_cb_called++;
 
     if (foo_cb_rc != 0)
         errno = foo_cb_errno;
 
+    if (foo_cb_rc == 0) {
+        flux_msg_decref (*msg);
+        *msg = NULL;
+    }
+
     return foo_cb_rc;
 }
 
@@ -41,6 +48,7 @@ int main (int argc, char **argv)
 {
     struct service_switch *sw;
     flux_msg_t *msg, *msg2, *msg3;
+    void *msg_ptr;
 
     plan (NO_PLAN);
 
@@ -48,12 +56,15 @@ int main (int argc, char **argv)
     ok (sw != NULL,
         "service_switch_create works");
 
-    msg = flux_request_encode ("foo", NULL);
+    msg_ptr = msg = flux_request_encode ("foo", NULL);
     if (!msg)
         BAIL_OUT ("flux_request_encode: %s", flux_strerror (errno));
     errno = 0;
-    ok (service_send (sw, msg) < 0 && errno == ENOSYS,
-        "service_send to 'foo' fails with ENOSYS");
+    ok (service_send_new (sw, &msg) < 0 && errno == ENOSYS,
+        "service_send_new to 'foo' fails with ENOSYS");
+    ok (msg != NULL,
+        "and message as not set to NULL");
+    msg = msg_ptr; // just in case that test failed
 
     ok (service_add (sw, "foo", NULL, foo_cb, NULL) == 0,
         "service_add foo works");
@@ -62,34 +73,43 @@ int main (int argc, char **argv)
     foo_cb_arg = (void *)(uintptr_t)1;
     foo_cb_called = 0;
     foo_cb_rc = 0;
-    ok (service_send (sw, msg) == 0,
-        "service_send to 'foo' works");
-    ok (foo_cb_called == 1 && foo_cb_arg == NULL && foo_cb_msg == msg,
+    ok (service_send_new (sw, &msg) == 0,
+        "service_send_new to 'foo' works");
+    ok (msg == NULL,
+        "and msg was set to NULL");
+    ok (foo_cb_called == 1 && foo_cb_arg == NULL && foo_cb_msg == msg_ptr,
         "and callback was called with expected arguments");
 
-    foo_cb_rc = 42;
+    // msg was destroyed above so recreate
+    msg_ptr = msg = flux_request_encode ("foo", NULL);
+    if (!msg)
+        BAIL_OUT ("flux_request_encode: %s", flux_strerror (errno));
+
+    foo_cb_rc = -1;
     foo_cb_errno = ENXIO;
     errno = 0;
-    ok (service_send (sw, msg) == 42 && errno == ENXIO,
-        "service_send returns callback's return code and preserves errno");
+    ok (service_send_new (sw, &msg) == -1,
+        "service_send_new returns callback's return code");
+    ok (errno == ENXIO,
+        "and callback's errno was set");
 
+    // msg was destroyed above so recreate
+    msg_ptr = msg = flux_request_encode ("foo", NULL);
+    if (!msg)
+        BAIL_OUT ("flux_request_encode: %s", flux_strerror (errno));
     service_remove (sw, "foo");
     errno = 0;
-    ok (service_send (sw, msg) < 0 && errno == ENOSYS,
+    ok (service_send_new (sw, &msg) < 0 && errno == ENOSYS,
         "service_remove works");
 
-    flux_msg_destroy (msg);
-
-
     msg = flux_request_encode ("bar.baz", NULL);
     if (!msg)
         BAIL_OUT ("flux_request_encode: %s", flux_strerror (errno));
     ok (service_add (sw, "bar", NULL, foo_cb, NULL) == 0,
         "service_add bar works");
     foo_cb_rc = 0;
-    ok (service_send (sw, msg) == 0,
+    ok (service_send_new (sw, &msg) == 0,
         "service_send to 'bar.baz' works");
-    flux_msg_destroy (msg);
 
  #define SVC_NAME "reallylongservicenamewowthisisimpressive"
  #define SVC_ALT1 "alt1"
@@ -113,25 +133,41 @@ int main (int argc, char **argv)
 
     foo_cb_rc = 0;
     foo_cb_called = 0;
-    ok (service_send (sw, msg) == 0 && foo_cb_called == 1,
-        "service_send matched long service name");
-    ok (service_send (sw, msg2) == 0 && foo_cb_called == 2,
-        "service_send matched first alternate name");
-    ok (service_send (sw, msg3) == 0 && foo_cb_called == 3,
-        "service_send matched second alternate name");
+    ok (service_send_new (sw, &msg) == 0 && foo_cb_called == 1,
+        "service_send_new matched long service name");
+    ok (service_send_new (sw, &msg2) == 0 && foo_cb_called == 2,
+        "service_send_new matched first alternate name");
+    ok (service_send_new (sw, &msg3) == 0 && foo_cb_called == 3,
+        "service_send_new matched second alternate name");
 
     service_remove_byuuid (sw, "fakeuuid");
 
+    // messages were destroyed above so recreate
+    msg = flux_request_encode (SVC_NAME ".baz", NULL);
+    if (!msg)
+        BAIL_OUT ("flux_request_encode: %s", flux_strerror (errno));
+    msg2 = flux_request_encode (SVC_ALT1 ".oooh", NULL);
+    if (!msg2)
+        BAIL_OUT ("flux_request_encode: %s", flux_strerror (errno));
+    msg3 = flux_request_encode (SVC_ALT2 ".vroom", NULL);
+    if (!msg3)
+        BAIL_OUT ("flux_request_encode: %s", flux_strerror (errno));
     foo_cb_rc = 0;
     foo_cb_called = 0;
     errno = 0;
-    ok (service_send (sw, msg) < 0 && errno == ENOSYS && foo_cb_called == 0,
-        "service_send to long service name fails after remove_byuuid");
+    ok (service_send_new (sw, &msg) < 0
+        && errno == ENOSYS
+        && foo_cb_called == 0,
+        "service_send_new to long service name fails after remove_byuuid");
     errno = 0;
-    ok (service_send (sw, msg2) < 0 && errno == ENOSYS && foo_cb_called == 0,
+    ok (service_send_new (sw, &msg2) < 0
+        && errno == ENOSYS
+        && foo_cb_called == 0,
         "service_send to first alternate name fails after remove_byuuid");
     errno = 0;
-    ok (service_send (sw, msg3) < 0  && errno == ENOSYS && foo_cb_called == 0,
+    ok (service_send_new (sw, &msg3) < 0
+        && errno == ENOSYS
+        && foo_cb_called == 0,
         "service_send to second alternate name fails after remove_byuuid");
 
     flux_msg_destroy (msg);
diff --git a/src/broker/test/topology.c b/src/broker/test/topology.c
new file mode 100644
index 000000000000..93b91cd9199f
--- /dev/null
+++ b/src/broker/test/topology.c
@@ -0,0 +1,657 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/ptrint/ptrint.h"
+
+#include "src/broker/topology.h"
+
+void check_subtree (json_t *o,
+                    const char *s,
+                    int exp_rank,
+                    int exp_size,
+                    size_t exp_count)
+{
+    int rank = -1;
+    int size = -1;
+    json_t *children = NULL;
+    int rc = -1;
+
+    if (o) {
+        rc = json_unpack (o,
+                          "{s:i s:i s:o}",
+                          "rank", &rank,
+                          "size", &size,
+                          "children", &children);
+    }
+
+    diag ("rank=%d size=%d children=%zu",
+          rank,
+          size,
+          children ? json_array_size (children) : -1);
+
+    ok (rc == 0
+        && rank == exp_rank
+        && size == exp_size
+        && children != NULL
+        && json_array_size (children) == exp_count,
+        "topology_get_json_subtree_at %s returns expected object", s);
+}
+
+void test_flat (void)
+{
+    struct topology *topo;
+    int child_ranks[15];
+    json_t *o;
+    bool pass;
+
+    topo = topology_create (NULL, 16, NULL);
+    ok (topo != NULL,
+        "topology_create size=16 works");
+    ok (topology_get_size (topo) == 16,
+        "topology_get_size returns 16");
+    ok (topology_get_rank (topo) == 0,
+        "topology_get_rank returns 0");
+    ok (topology_get_parent (topo) < 0,
+        "topology_get_parent fails");
+    ok (topology_get_child_ranks (topo, child_ranks, 15) == 15,
+        "topology_get_child_ranks returns 15");
+
+    pass = true;
+    for (int i = 0; i < 15; i++) {
+        if (child_ranks[i] != i + 1)
+            pass = false;
+    }
+
+    ok (pass == true,
+        "child_ranks array contains ranks 1-15");
+    ok (topology_get_level (topo) == 0,
+        "topology_get_level returns 0");
+    ok (topology_get_maxlevel (topo) == 1,
+        "topology_get_maxlevel returns 1");
+    ok (topology_get_descendant_count (topo) == 15,
+        "topology_get_descendant_count returns 15");
+    ok (topology_get_child_route (topo, 5) == 5,
+        "topology_get_child_route rank=5 returns 5");
+
+    o = topology_get_json_subtree_at (topo, 0);
+    check_subtree (o, "rank=0", 0, 16, 15);
+    json_decref (o);
+    o = topology_get_json_subtree_at (topo, 15);
+    check_subtree (o, "rank=15", 15, 1, 0);
+    json_decref (o);
+
+    ok (topology_incref (topo) == topo,
+        "topology_incref returns topo pointer");
+    topology_decref (topo);
+    topology_decref (topo);
+}
+
+void test_k1 (void)
+{
+    struct topology *topo;
+    int child_ranks[15];
+    json_t *o;
+
+    topo = topology_create ("kary:1", 16, NULL);
+    ok (topo != NULL,
+        "topology_create kary:1 size=16 works");
+    ok (topology_get_rank (topo) == 0,
+        "topology_get_rank returns 0");
+    ok (topology_get_size (topo) == 16,
+        "topology_get_size returns 16");
+    ok (topology_get_parent (topo) < 0,
+        "topology_get_parent fails");
+    ok (topology_get_child_ranks (topo, child_ranks, 15) == 1,
+        "topology_get_child_ranks returns 1");
+
+    ok (child_ranks[0] == 1,
+        "child_ranks array contains ranks 1");
+    ok (topology_get_level (topo) == 0,
+        "topology_get_level returns 0");
+    ok (topology_get_maxlevel (topo) == 15,
+        "topology_get_maxlevel returns 15");
+    ok (topology_get_descendant_count (topo) == 15,
+        "topology_get_descendant_count returns 15");
+    ok (topology_get_child_route (topo, 5) == 1,
+        "topology_get_child_route rank=5 returns 1");
+
+    o = topology_get_json_subtree_at (topo, 0);
+    check_subtree (o, "rank=0", 0, 16, 1);
+    json_decref (o);
+    o = topology_get_json_subtree_at (topo, 1);
+    check_subtree (o, "rank=1", 1, 15, 1);
+    json_decref (o);
+    o = topology_get_json_subtree_at (topo, 15);
+    check_subtree (o, "rank=15", 15, 1, 0);
+    json_decref (o);
+
+    topology_decref (topo);
+}
+
+void test_k2 (void)
+{
+    struct topology *topo;
+    int child_ranks[15];
+    json_t *o;
+
+    topo = topology_create ("kary:2", 16, NULL);
+    ok (topo != NULL,
+        "topology_create kary:2 size=16 works");
+    ok (topology_get_rank (topo) == 0,
+        "topology_get_rank returns 0");
+    ok (topology_get_size (topo) == 16,
+        "topology_get_size returns 16");
+    ok (topology_get_parent (topo) < 0,
+        "topology_get_parent fails");
+    ok (topology_get_child_ranks (topo, child_ranks, 15) == 2,
+        "topology_get_child_ranks returns 2");
+
+    ok (child_ranks[0] == 1 && child_ranks[1] == 2,
+        "child_ranks array contains ranks 1-2");
+    ok (topology_get_level (topo) == 0,
+        "topology_get_level returns 0");
+    ok (topology_get_maxlevel (topo) == 4,
+        "topology_get_maxlevel returns 4");
+    ok (topology_get_descendant_count (topo) == 15,
+        "topology_get_descendant_count returns 15");
+    ok (topology_get_child_route (topo, 5) == 2,
+        "topology_get_child_route rank=5 returns 2");
+
+    o = topology_get_json_subtree_at (topo, 0);
+    check_subtree (o, "rank=0", 0, 16, 2);
+    json_decref (o);
+    o = topology_get_json_subtree_at (topo, 1);
+    check_subtree (o, "rank=1", 1, 8, 2);
+    json_decref (o);
+    o = topology_get_json_subtree_at (topo, 2);
+    check_subtree (o, "rank=2", 2, 7, 2);
+    json_decref (o);
+    o = topology_get_json_subtree_at (topo, 3);
+    check_subtree (o, "rank=3", 3, 4, 2);
+    json_decref (o);
+    o = topology_get_json_subtree_at (topo, 4);
+    check_subtree (o, "rank=4", 4, 3, 2);
+    json_decref (o);
+    o = topology_get_json_subtree_at (topo, 15);
+    check_subtree (o, "rank=15", 15, 1, 0);
+    json_decref (o);
+
+    topology_decref (topo);
+}
+
+void test_k2_router (void)
+{
+    struct topology *topo;
+    int child_ranks[15];
+    json_t *o;
+
+    topo = topology_create ("kary:2", 16, NULL);
+    ok (topo != NULL,
+        "topology_create kary:2 size=16 works");
+    ok (topology_set_rank (topo, 1) == 0,
+        "topology_set_rank 1 works");
+    ok (topology_get_rank (topo) == 1,
+        "topology_get_rank returns 1");
+    ok (topology_get_size (topo) == 16,
+        "topology_get_size returns 16");
+    ok (topology_get_parent (topo) == 0,
+        "topology_get_parent returns 0");
+    ok (topology_get_child_ranks (topo, child_ranks, 15) == 2,
+        "topology_get_child_ranks returns 2");
+    ok (child_ranks[0] == 3 && child_ranks[1] == 4,
+        "child_ranks array contains ranks 3-4");
+    ok (topology_get_level (topo) == 1,
+        "topology_get_level returns 1");
+    ok (topology_get_maxlevel (topo) == 4,
+        "topology_get_maxlevel returns 4");
+    ok (topology_get_descendant_count (topo) == 7,
+        "topology_get_descendant_count returns 7");
+    ok (topology_get_child_route (topo, 10) == 4,
+        "topology_get_child_route rank=10 returns 4");
+
+    o = topology_get_json_subtree_at (topo, 1);
+    check_subtree (o, "rank=1", 1, 8, 2);
+    json_decref (o);
+
+    topology_decref (topo);
+}
+
+struct internal_ranks_test {
+    int size;
+    const char *uri;
+    const char *expected_ranks;
+};
+
+struct internal_ranks_test internal_ranks_tests[] = {
+    { 1,  "kary:2",  ""    },
+    { 2,  "kary:2",  "0"   },
+    { 4,  "kary:2",  "0-1" },
+    { 4,  "kary:0",  "0"   },
+    { 16, "kary:2",  "0-7" },
+    { 48, "kary:2",  "0-23"},
+    { 48, "kary:0",  "0"   },
+    { 48, "kary:16", "0-2" },
+    { 4,  "binomial","0,2" },
+    { 8,  "binomial","0,2,4,6" },
+    { 16, "binomial","0,2,4,6,8,10,12,14" },
+    { -1, NULL, NULL  }
+};
+
+void test_internal_ranks (void)
+{
+    struct topology *topo;
+    struct idset *result;
+    struct idset *expected;
+    char *s;
+
+    struct internal_ranks_test *t = internal_ranks_tests;
+    while (t && t->expected_ranks) {
+        if (!(topo = topology_create (t->uri, t->size, NULL)))
+            BAIL_OUT ("failed to create topology %s size=%d",
+                      t->uri,
+                      t->size);
+        if (!(expected = idset_decode (t->expected_ranks)))
+            BAIL_OUT ("failed to decode expected ranks=%d",
+                      t->expected_ranks);
+        result = topology_get_internal_ranks (topo);
+        ok (result != NULL,
+            "topology_get_internal_ranks(size=%d, kary=%d) works");
+        s = idset_encode (result, IDSET_FLAG_RANGE);
+
+        ok (idset_equal (result, expected),
+            "result was %s (expected %s)",
+            s,
+            t->expected_ranks);
+
+        topology_decref (topo);
+        idset_destroy (expected);
+        idset_destroy (result);
+        free (s);
+        t++;
+    }
+}
+
+struct pmap {
+    int parent;
+    const char *children;
+    struct idset *ids_children;
+};
+void pmap_clean (struct pmap *map, size_t count)
+{
+    for (int i = 0; i < count; i++) {
+        idset_destroy (map[i].ids_children);
+        map[i].ids_children = NULL;
+    }
+}
+int pmap_lookup (struct pmap *map, size_t count, int rank, int *parent_rank)
+{
+    for (int i = 0; i < count; i++) {
+        if (!map[i].ids_children) {
+            if (!(map[i].ids_children = idset_decode (map[i].children)))
+                BAIL_OUT ("idset_decode failed");
+        }
+        if (idset_test (map[i].ids_children, rank)) {
+            *parent_rank = map[i].parent;
+            return 0;
+        }
+    }
+    return -1;
+}
+json_t *pmap_hosts (struct pmap *map, size_t count, int size)
+{
+    json_t *hosts;
+    if (!(hosts = json_array ()))
+        return NULL;
+    for (int rank = 0; rank < size; rank++) {
+        char host[16];
+        char phost[16];
+        json_t *entry;
+        int parent_rank;
+
+        snprintf (host, sizeof (host), "test%d", rank);
+        if (pmap_lookup (map, count, rank, &parent_rank) < 0)
+            entry = json_pack ("{s:s}", "host", host);
+        else {
+            snprintf (phost, sizeof (phost), "test%d", parent_rank);
+            entry = json_pack ("{s:s s:s}", "host", host, "parent", phost);
+        }
+        if (!entry || json_array_append_new (hosts, entry) < 0)
+            BAIL_OUT ("failed to append hosts array entry");
+    }
+    return hosts;
+}
+
+/* Does topology have 'expected' (idset) internal ranks?
+ */
+bool check_internal (struct topology *topo, const char *expected)
+{
+    struct idset *exp;
+    struct idset *out;
+    bool result;
+
+    if (!(exp = idset_decode (expected)))
+        BAIL_OUT ("idset_decode failed");
+    if (!(out = topology_get_internal_ranks (topo)))
+        BAIL_OUT ("topology_get_internal_ranks failed");
+
+    result = idset_equal (out, exp);
+
+    char *s = idset_encode (out, IDSET_FLAG_RANGE);
+    diag ("%s %s %s", s, result ? "==" : "!=", expected);
+    free (s);
+
+    idset_destroy (out);
+    idset_destroy (exp);
+
+    return result;
+}
+
+struct pmap cust1[] = {
+    { .parent = 0, .children = "1,2,64,128,192" },
+    { .parent = 1, .children = "3-63" },
+    { .parent = 64, .children = "65-127" },
+    { .parent = 128, .children = "129-191" },
+    { .parent = 192, .children = "193-255" },
+};
+struct pmap bad1[] = {
+    { .parent = 1, .children = "0" }, // 0 can't have a parent
+};
+struct pmap bad2[] = {
+    { .parent = 1, .children = "2" },
+    { .parent = 2, .children = "3" },
+    { .parent = 3, .children = "1" }, // cycle
+};
+struct pmap bad3[] = {
+    { .parent = 1, .children = "1" }, // small cycle!
+};
+
+void test_custom (void)
+{
+    struct topology *topo;
+    flux_error_t error;
+    json_t *hosts;
+
+    topo = topology_create ("custom:zzz", 256, &error);
+    if (!topo)
+        diag ("%s", error.text);
+    ok (topo == NULL,
+        "topology_create custom: fails with URI path");
+
+    topology_hosts_set (NULL);
+    topo = topology_create ("custom:", 256, &error);
+    ok (topo != NULL,
+        "topology_create custom: works without hosts array");
+    topology_decref (topo);
+
+    hosts = pmap_hosts (bad1, ARRAY_SIZE (bad1), 2);
+    topology_hosts_set (hosts);
+    topo = topology_create ("custom:", 2, &error);
+    if (!topo)
+        diag ("%s", error.text);
+    ok (topo == NULL,
+        "topology_create custom failed with rank 0 parent");
+    topology_hosts_set (NULL);
+    json_decref (hosts);
+
+    hosts = pmap_hosts (bad2, ARRAY_SIZE (bad2), 16);
+    topology_hosts_set (hosts);
+    topo = topology_create ("custom:", 16, &error);
+    if (!topo)
+        diag ("%s", error.text);
+    ok (topo == NULL,
+        "topology_create custom failed with graph cycle");
+    topology_hosts_set (NULL);
+    json_decref (hosts);
+
+    hosts = pmap_hosts (bad3, ARRAY_SIZE (bad3), 16);
+    topology_hosts_set (hosts);
+    topo = topology_create ("custom:", 16, &error);
+    if (!topo)
+        diag ("%s", error.text);
+    ok (topo == NULL,
+        "topology_create custom failed with self as parent");
+    topology_hosts_set (NULL);
+    json_decref (hosts);
+
+    hosts = pmap_hosts (cust1, ARRAY_SIZE (cust1), 256);
+    topology_hosts_set (hosts);
+
+    topo = topology_create ("custom:", 2, &error);
+    if (!topo)
+        diag ("%s", error.text);
+    ok (topo == NULL,
+        "topology_create custom failed with mismatched topo and host size");
+
+    topo = topology_create ("custom:", 256, &error);
+    topology_hosts_set (NULL);
+    ok (topo != NULL,
+        "configured custom 256 node topo");
+    ok (topology_set_rank (topo, 1) == 0,
+        "set rank to 1");
+    ok (topology_get_parent (topo) == 0,
+        "parent is 0");
+    ok (topology_get_level (topo) == 1,
+        "level is 1");
+    ok (topology_get_descendant_count (topo) == 61,
+        "descendant_count is 61");
+    ok (check_internal (topo, "0-1,64,128,192"),
+        "topology has expected internal ranks");
+    topology_decref (topo);
+    json_decref (hosts);
+}
+
+void test_invalid (void)
+{
+    struct topology *topo;
+    int a[16];
+
+    if (!(topo = topology_create (NULL, 16, NULL)))
+        BAIL_OUT ("could not create topology");
+
+    errno = 0;
+    ok (topology_create (NULL, 0, NULL) == NULL && errno == EINVAL,
+        "topology_create size=0 fails with EINVAL");
+
+    lives_ok ({topology_decref (NULL);},
+             "topology_decref topo=NULL doesn't crash");
+
+    ok (topology_incref (NULL) == NULL,
+        "topology_incref topo=NULL returns NULL");
+
+    errno = 0;
+    ok (topology_set_rank (NULL, 0) < 0 && errno == EINVAL,
+        "topology_set_rank topo=NULL fails with EINVAL");
+    errno = 0;
+    ok (topology_set_rank (topo, -1) < 0 && errno == EINVAL,
+        "topology_set_rank rank=-1 fails with EINVAL");
+
+    ok (topology_get_rank (NULL) == -1,
+        "topology_get_rank topo=NULL returns -1");
+    ok (topology_get_size (NULL) == -1,
+        "topology_get_rank topo=NULL returns -1");
+    ok (topology_get_parent (NULL) == -1,
+        "topology_get_parent topo=NULL returns -1");
+    ok (topology_get_level (NULL) == 0,
+        "topology_get_level topo=NULL returns 0");
+    ok (topology_get_maxlevel (NULL) == 0,
+        "topology_get_maxlevel topo=NULL returns 0");
+
+    errno = 0;
+    ok (topology_get_child_ranks (NULL, NULL, 0) == -1 && errno == EINVAL,
+        "topology_get_child_ranks topo=NULL fails with EINVAL");
+    errno = 0;
+    ok (topology_get_child_ranks (topo, NULL, 2) == -1 && errno == EINVAL,
+        "topology_get_child_ranks buf=NULL size>0 fails with EINVAL");
+    errno = 0;
+    ok (topology_get_child_ranks (topo, a, 2) == -1 && errno == EOVERFLOW,
+        "topology_get_child_ranks size=too short fails with EOVERFLOW");
+
+    ok (topology_get_descendant_count (NULL) == 0,
+        "topology_get_descendant_count topo=NULL returns 0");
+
+    ok (topology_get_child_route (NULL, 1) == -1,
+        "topology_get_child_route topo=NULL returns -1");
+    ok (topology_get_child_route (topo, 0) == -1,
+        "topology_get_child_route rank=0 returns -1");
+    ok (topology_get_child_route (topo, 99) == -1,
+        "topology_get_child_route rank=99 returns -1");
+
+    errno = 0;
+    ok (topology_get_json_subtree_at (NULL, 0) == NULL && errno == EINVAL,
+        "topology_get_json_subtree_at topo=NULL fails with EINVAL");
+    errno = 0;
+    ok (topology_get_json_subtree_at (topo, -1) == NULL && errno == EINVAL,
+        "topology_get_json_subtree_at rank=-1 fails with EINVAL");
+
+    errno = 0;
+    ok (topology_rank_aux_get (NULL, 0, "foo") == NULL && errno == EINVAL,
+        "topology_rank_aux_get topo=NULL fails with EINVAL");
+    errno = 0;
+    ok (topology_rank_aux_get (topo, -1, "foo") == NULL && errno == EINVAL,
+        "topology_rank_aux_get rank=-1 fails with EINVAL");
+    errno = 0;
+    ok (topology_rank_aux_get (topo, 99, "foo") == NULL && errno == EINVAL,
+        "topology_rank_aux_get rank=99 fails with EINVAL");
+    errno = 0;
+    ok (topology_rank_aux_get (topo, 0, "foo") == NULL && errno == ENOENT,
+        "topology_rank_aux_get key=unknown fails with ENOENT");
+
+    errno = 0;
+    ok (topology_rank_aux_set (NULL, 0, "foo", "bar", NULL) < 0
+        && errno == EINVAL,
+        "topology_rank_aux_set topo=NULL fails with EINVAL");
+    errno = 0;
+    ok (topology_rank_aux_set (topo, -1, "foo", "bar", NULL) < 0
+        && errno == EINVAL,
+        "topology_rank_aux_set rank=-1 fails with EINVAL");
+    errno = 0;
+    ok (topology_rank_aux_set (topo, 99, "foo", "bar", NULL) < 0
+        && errno == EINVAL,
+        "topology_rank_aux_set rank=99 fails with EINVAL");
+
+    errno = 0;
+    ok (topology_get_internal_ranks (NULL) == NULL && errno == EINVAL,
+        "topolog_get_internal_ranks (NULL) returns EINVAL");
+
+    topology_decref (topo);
+}
+
+void test_rank_aux (void)
+{
+    struct topology *topo;
+    int errors;
+
+    if (!(topo = topology_create (NULL, 16, NULL)))
+        BAIL_OUT ("topology_create failed");
+
+    errors = 0;
+    for (int i = 0; i < 16; i++) {
+        if (topology_rank_aux_set (topo, i, "rank", int2ptr (i + 1), NULL) < 0)
+            errors++;
+    }
+    ok (errors == 0,
+        "topology_rank_aux_set works for all ranks");
+    errors = 0;
+    for (int i = 0; i < 16; i++) {
+        void *ptr;
+        if (!(ptr = (topology_rank_aux_get (topo, i, "rank")))
+            || ptr2int (ptr) != i + 1)
+            errors++;
+    }
+    ok (errors == 0,
+        "topology_rank_aux_get returns expected result for all ranks");
+
+    topology_decref (topo);
+}
+
+void test_binomial5 (void)
+{
+    struct topology *topo;
+    flux_error_t error;
+    int children[16];
+
+    topo = topology_create ("binomial:zz", 5, &error);
+    if (!topo)
+        diag ("%s", error.text);
+    ok (topo == NULL,
+        "binomial topology fails with unknown path");
+
+    topo = topology_create ("binomial", 5, &error);
+    ok (topo != NULL,
+        "binomal topology of size=5 (non power of 2) works");
+
+    ok (topology_set_rank (topo, 1) == 0,
+        "set rank to 1");
+    ok (topology_get_parent (topo) == 0,
+        "rank 1 parent is 0");
+    ok (topology_get_child_ranks (topo, NULL, 0) == 0,
+        "rank 1 has no children");
+    ok (topology_get_level (topo) == 1,
+        "rank 1 level is 1");
+
+    ok (topology_set_rank (topo, 2) == 0,
+        "set rank to 2");
+    ok (topology_get_parent (topo) == 0,
+        "rank 2 parent is 0");
+    ok (topology_get_child_ranks (topo, children, ARRAY_SIZE (children)) == 1,
+        "rank 2 has 1 child");
+    ok (children[0] == 3,
+        "child is rank 3");
+    ok (topology_get_level (topo) == 1,
+        "rank 2 level is 1");
+
+    ok (topology_set_rank (topo, 3) == 0,
+        "set rank to 3");
+    ok (topology_get_parent (topo) == 2,
+        "rank 3 parent is 2");
+    ok (topology_get_child_ranks (topo, NULL, 0) == 0,
+        "rank 3 has no children");
+    ok (topology_get_level (topo) == 2,
+        "rank 3 level is 2");
+
+    ok (topology_set_rank (topo, 4) == 0,
+        "set rank to 4");
+    ok (topology_get_parent (topo) == 0,
+        "rank 4 parent is 0");
+    ok (topology_get_child_ranks (topo, NULL, 0) == 0,
+        "rank 4 has no children");
+    ok (topology_get_level (topo) == 1,
+        "rank 4 level is 2");
+
+    topology_decref (topo);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_flat ();
+    test_k1 ();
+    test_k2 ();
+    test_k2_router ();
+    test_invalid ();
+    test_internal_ranks ();
+    test_custom ();
+    test_rank_aux ();
+    test_binomial5 ();
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/broker/topology.c b/src/broker/topology.c
new file mode 100644
index 000000000000..bc35c7152e61
--- /dev/null
+++ b/src/broker/topology.c
@@ -0,0 +1,546 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* topology.c - create arbitrary TBON topology and allow useful queries */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libutil/kary.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/aux.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+
+#include "topology.h"
+
+struct node {
+    int parent;
+    struct aux_item *aux;
+};
+
+struct topology {
+    int rank;
+    int size;
+    int refcount;
+    struct node *node;
+};
+
+static int kary_plugin_init (struct topology *topo,
+                             const char *path,
+                             flux_error_t *error);
+static int binomial_plugin_init (struct topology *topo,
+                                 const char *path,
+                                 flux_error_t *error);
+static int custom_plugin_init (struct topology *topo,
+                               const char *path,
+                               flux_error_t *error);
+
+static const struct topology_plugin builtin_plugins[] = {
+    { .name = "kary",   .init = kary_plugin_init },
+    { .name = "binomial", .init = binomial_plugin_init },
+    { .name = "custom", .init = custom_plugin_init },
+};
+
+static json_t *boot_hosts;
+
+void topology_hosts_set (json_t *hosts)
+{
+    boot_hosts = hosts;
+}
+
+static const struct topology_plugin *topology_plugin_lookup (const char *name)
+{
+    for (int i = 0; i < ARRAY_SIZE (builtin_plugins); i++)
+        if (streq (name, builtin_plugins[i].name))
+            return &builtin_plugins[i];
+    return NULL;
+}
+
+static int topology_plugin_call (struct topology *topo,
+                                 const char *uri,
+                                 flux_error_t *error)
+{
+    const struct topology_plugin *plugin;
+    char *name;
+    char *path;
+
+    if (!(name = strdup (uri))) {
+        errprintf (error, "out of memory");
+        goto error;
+    }
+    if ((path = strchr (name, ':')))
+        *path++ = '\0';
+    if (!(plugin = topology_plugin_lookup (name))) {
+        errprintf (error, "unknown topology scheme '%s'", name);
+        goto error;
+    }
+    if (plugin->init (topo, path, error) < 0)
+        goto error;
+    free (name);
+    return 0;
+error:
+    free (name);
+    errno = EINVAL;
+    return -1;
+}
+
+void topology_decref (struct topology *topo)
+{
+    if (topo && --topo->refcount == 0) {
+        int saved_errno = errno;
+        for (int i = 0; i < topo->size; i++)
+            aux_destroy (&topo->node[i].aux);
+        free (topo);
+        errno = saved_errno;
+    }
+}
+
+struct topology *topology_incref (struct topology *topo)
+{
+    if (topo)
+        topo->refcount++;
+    return topo;
+}
+
+struct topology *topology_create (const char *uri,
+                                  int size,
+                                  flux_error_t *error)
+{
+    struct topology *topo;
+
+    if (size < 1) {
+        errprintf (error, "invalid topology size %d", size);
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(topo = calloc (1, sizeof (*topo) + sizeof (topo->node[0]) * size))) {
+        errprintf (error, "out of memory");
+        goto error;
+    }
+    topo->refcount = 1;
+    topo->size = size;
+    topo->node = (struct node *)(topo + 1);
+    topo->node[0].parent = -1;
+    // topo->node is 0-initialized, so rank 0 is default parent of all nodes
+
+    if (uri) {
+        if (topology_plugin_call (topo, uri, error) < 0)
+            goto error;
+    }
+    return topo;
+error:
+    topology_decref (topo);
+    return NULL;
+}
+
+int topology_set_rank (struct topology *topo, int rank)
+{
+    if (!topo || rank < 0 || rank >= topo->size) {
+        errno = EINVAL;
+        return -1;
+    }
+    topo->rank = rank;
+    return 0;
+}
+
+void *topology_rank_aux_get (struct topology *topo, int rank, const char *name)
+{
+    if (!topo || rank < 0 || rank >= topo->size) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return aux_get (topo->node[rank].aux, name);
+}
+
+int topology_rank_aux_set (struct topology *topo,
+                           int rank,
+                           const char *name,
+                           void *val,
+                           flux_free_f destroy)
+{
+    if (!topo || rank < 0 || rank >= topo->size) {
+        errno = EINVAL;
+        return -1;
+    }
+    return aux_set (&topo->node[rank].aux, name, val, destroy);
+}
+
+int topology_get_rank (struct topology *topo)
+{
+    return topo ? topo->rank : -1;
+}
+
+int topology_get_size (struct topology *topo)
+{
+    return topo ? topo->size : -1;
+}
+
+// O(1)
+int topology_get_parent (struct topology *topo)
+{
+    return topo ? topo->node[topo->rank].parent : -1;
+}
+
+// O(size)
+static ssize_t topology_get_child_ranks_at (struct topology *topo,
+                                            int rank,
+                                            int *child_ranks,
+                                            size_t child_ranks_length)
+{
+    ssize_t count = 0;
+
+    if (!topo
+        || rank < 0
+        || rank >= topo->size
+        || (child_ranks_length > 0 && child_ranks == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+    for (int i = 0; i < topo->size; i++) {
+        if (topo->node[i].parent == rank) {
+            if (child_ranks) {
+                if (count >= child_ranks_length) {
+                    errno = EOVERFLOW;
+                    return -1;
+                }
+                child_ranks[count] = i;
+            }
+            count++;
+        }
+    }
+
+    return count;
+}
+
+ssize_t topology_get_child_ranks (struct topology *topo,
+                                  int *child_ranks,
+                                  size_t child_ranks_length)
+{
+    if (!topo) {
+        errno = EINVAL;
+        return -1;
+    }
+    return topology_get_child_ranks_at (topo,
+                                        topo->rank,
+                                        child_ranks,
+                                        child_ranks_length);
+}
+
+// O(level)
+int topology_get_level (struct topology *topo)
+{
+    int level = 0;
+
+    if (topo) {
+        int rank = topo->rank;
+        while (rank != 0) {
+            rank = topo->node[rank].parent;
+            level++;
+        }
+    }
+    return level;
+}
+
+// O(size*level)
+int topology_get_maxlevel (struct topology *topo)
+{
+    int maxlevel = 0;
+
+    if (topo) {
+        for (int i = 0; i < topo->size; i++) {
+            int rank = i;
+            int level = 0;
+            while (rank != 0) {
+                rank = topo->node[rank].parent;
+                level++;
+            }
+            if (maxlevel < level)
+                maxlevel = level;
+        }
+    }
+    return maxlevel;
+}
+
+// O(level)
+static bool is_descendant_of (struct topology *topo, int rank, int ancestor)
+{
+    if (rank < 0
+        || ancestor < 0
+        || !topo
+        || rank >= topo->size
+        || ancestor >= topo->size
+        || topo->node[rank].parent == -1)
+        return false;
+    if (topo->node[rank].parent == ancestor)
+        return true;
+    return is_descendant_of (topo, topo->node[rank].parent, ancestor);
+}
+
+// O(size*level)
+int topology_get_descendant_count_at (struct topology *topo, int rank)
+{
+    int count = 0;
+    if (topo) {
+        for (int i = 0; i < topo->size; i++) {
+            if (is_descendant_of (topo, i, rank))
+                count++;
+        }
+    }
+    return count;
+}
+
+int topology_get_descendant_count (struct topology *topo)
+{
+    return topology_get_descendant_count_at (topo, topo ? topo->rank : 0);
+}
+
+// O(level)
+int topology_get_child_route (struct topology *topo, int rank)
+{
+    if (!topo || rank <= 0 || rank >= topo->size)
+        return -1;
+    if (topo->node[rank].parent == topo->rank)
+        return rank;
+    return topology_get_child_route (topo, topo->node[rank].parent);
+}
+
+json_t *topology_get_json_subtree_at (struct topology *topo, int rank)
+{
+    int child_count;
+    int *child_ranks = NULL;
+    json_t *o;
+    json_t *children = NULL;
+    json_t *child;
+    int size;
+
+    if ((child_count = topology_get_child_ranks_at (topo, rank, NULL, 0)) < 0
+        || !(child_ranks = calloc (child_count, sizeof (child_ranks[0])))
+        || topology_get_child_ranks_at (topo,
+                                        rank,
+                                        child_ranks,
+                                        child_count) < 0)
+        goto error;
+    if (!(children = json_array()))
+        goto nomem;
+    for (int i = 0; i < child_count; i++) {
+        if (!(child = topology_get_json_subtree_at (topo, child_ranks[i])))
+            goto error;
+        if (json_array_append_new (children, child) < 0) {
+            json_decref (child);
+            goto nomem;
+        }
+    }
+
+    size = topology_get_descendant_count_at (topo, rank) + 1;
+    if (!(o = json_pack ("{s:i s:i s:O}",
+                         "rank", rank,
+                         "size", size,
+                         "children", children)))
+        goto nomem;
+    json_decref (children);
+    free (child_ranks);
+    return o;
+nomem:
+    errno = ENOMEM;
+error:
+    ERRNO_SAFE_WRAP (json_decref, children);
+    ERRNO_SAFE_WRAP (free, child_ranks);
+    return NULL;
+
+}
+
+struct idset *topology_get_internal_ranks (struct topology *topo)
+{
+    struct idset *ranks;
+
+    if (!topo) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(ranks = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        return NULL;
+    for (int i = 1; i < topo->size; i++) {
+        if (idset_set (ranks, topo->node[i].parent) < 0)
+            goto error;
+    }
+    return ranks;
+error:
+    idset_destroy (ranks);
+    return NULL;
+}
+
+/* kary plugin
+ */
+static int parse_k (const char *s, int *result, flux_error_t *error)
+{
+    char *endptr;
+    unsigned long val;
+
+    if (!s || strlen (s) == 0)
+        goto error;
+    errno = 0;
+    val = strtoul (s, &endptr, 10);
+    if (errno != 0 || *endptr != '\0')
+        goto error;
+    *result = val;
+    return 0;
+error:
+    errprintf (error, "kary k value must be an integer >= 0");
+    return -1;
+}
+
+static int kary_plugin_init (struct topology *topo,
+                             const char *path,
+                             flux_error_t *error)
+{
+    int k;
+
+    if (parse_k (path, &k, error) < 0)
+        return -1;
+    if (k > 0) {
+        for (int i = 0; i < topo->size; i++) {
+            topo->node[i].parent = kary_parentof (k, i);
+            if (topo->node[i].parent == KARY_NONE)
+                topo->node[i].parent = -1;
+        }
+    }
+    return 0;
+}
+
+/* binomial plugin
+ */
+static int binomial_smallest_k (int size)
+{
+    size_t max_k = sizeof (int) * 8 - 1;
+
+    for (int k = 0; k < max_k; k++) {
+        int tree_size = 1 << k;
+        if (size <= tree_size)
+            return k;
+    }
+    return -1;
+}
+
+static void binomial_generate (struct topology *topo, int root, int k)
+{
+    for (int j = 0; j < k; j++) {
+        int child = root + (1 << j);
+        if (child < topo->size) {
+            topo->node[child].parent = root;
+            binomial_generate (topo, child, j);
+        }
+    }
+}
+
+static int binomial_plugin_init (struct topology *topo,
+                                 const char *path,
+                                 flux_error_t *error)
+{
+    int k;
+
+    if (path && strlen (path) > 0) {
+        errprintf (error, "unknown binomial topology directive: '%s'", path);
+        return -1;
+    }
+    if ((k = binomial_smallest_k (topo->size)) < 0) {
+        errprintf (error, "binomial: internal overflow");
+        return -1;
+    }
+    binomial_generate (topo, 0, k);
+    return 0;
+}
+
+/* custom plugin
+ * Set rank-ordered hosts array with topology_hosts_set() before using.
+ * Each entry has the form { "host":s "parent"?s }.
+ */
+
+/* Helper to find rank of 'hostname'
+ */
+static int gethostrank (const char *hostname, json_t *hosts, int *rank)
+{
+    size_t index;
+    json_t *entry;
+    const char *host;
+
+    json_array_foreach (hosts, index, entry) {
+        if (json_unpack (entry, "{s:s}", "host", &host) == 0
+            && streq (host, hostname)) {
+            *rank = index;
+            return 0;
+        }
+    }
+    return -1;
+}
+
+static int custom_plugin_init (struct topology *topo,
+                               const char *path,
+                               flux_error_t *error)
+{
+    size_t rank;
+    json_t *entry;
+    const char *parent;
+    const char *host;
+    int parent_rank;
+
+    if (path && strlen (path) > 0) {
+        errprintf (error, "unknown custom topology directive: '%s'", path);
+        return -1;
+    }
+    if (!boot_hosts)
+        return 0;
+    json_array_foreach (boot_hosts, rank, entry) {
+        if (json_unpack (entry,
+                         "{s:s s:s}",
+                         "host", &host,
+                         "parent", &parent) < 0)
+            continue;
+        if (rank == 0) {
+            errprintf (error,
+                       "Config file [bootstrap] hosts:"
+                       " rank 0 (%s) may not have a parent in a tree topology",
+                       host);
+            return -1;
+        }
+        if (rank >= topo->size) {
+            errprintf (error, "topology size does not match host array size");
+            return -1;
+        }
+        if (gethostrank (parent, boot_hosts, &parent_rank) < 0
+            || parent_rank < 0 || parent_rank >= topo->size) {
+            errprintf (error,
+                       "Config file [bootstrap] hosts:"
+                       " invalid parent \"%s\" for %s (rank %zu)",
+                       parent,
+                       host,
+                       rank);
+            return -1;
+        }
+        if (parent_rank == rank
+            || is_descendant_of (topo, parent_rank, rank)) {
+            errprintf (error,
+                       "Config file [bootstrap] hosts: parent \"%s\""
+                       " for %s (rank %zu) violates rule against cycles",
+                       parent,
+                       host,
+                       rank);
+            return -1;
+        }
+        topo->node[rank].parent = parent_rank;
+    }
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/broker/topology.h b/src/broker/topology.h
new file mode 100644
index 000000000000..1948586012da
--- /dev/null
+++ b/src/broker/topology.h
@@ -0,0 +1,77 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _BROKER_TOPOLOGY_H
+#define _BROKER_TOPOLOGY_H
+
+#include <sys/types.h>
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/idset.h>
+
+/* Create/destroy tree topology of size.
+ * The default topology is "flat" (rank 0 is parent of all other ranks),
+ * and queries are from the perspective of rank 0.
+ * If uri is non-NULL, the scheme selects a topology type, and the path
+ * provides additional detail.  The following schemes are available:
+ *
+ * kary:K
+ * Set the topology to a complete k-ary tree with fanout K.
+ */
+struct topology *topology_create (const char *uri,
+                                  int size,
+                                  flux_error_t *error);
+void topology_decref (struct topology *topo);
+struct topology *topology_incref (struct topology *topo);
+
+/* Set "my rank", which provides the point of view for queries.
+ */
+int topology_set_rank (struct topology *topo, int rank);
+
+/* Associate aux data with rank for lookup in O(1*rank_aux_elements)
+ */
+void *topology_rank_aux_get (struct topology *topo, int rank, const char *name);
+int topology_rank_aux_set (struct topology *topo,
+                           int rank,
+                           const char *name,
+                           void *aux,
+                           flux_free_f destroy);
+
+/* Queries
+ */
+int topology_get_rank (struct topology *topo);
+int topology_get_size (struct topology *topo);
+int topology_get_parent (struct topology *topo);
+ssize_t topology_get_child_ranks (struct topology *topo,
+                                  int *child_ranks,
+                                  size_t child_ranks_length);
+int topology_get_level (struct topology *topo);
+int topology_get_maxlevel (struct topology *topo);
+int topology_get_descendant_count (struct topology *topo);
+int topology_get_descendant_count_at (struct topology *topo, int rank);
+int topology_get_child_route (struct topology *topo, int rank);
+json_t *topology_get_json_subtree_at (struct topology *topo, int rank);
+
+/*  Return internal ranks (ranks that have one or more children)
+ */
+struct idset *topology_get_internal_ranks (struct topology *topo);
+
+void topology_hosts_set (json_t *hosts);
+
+/* Plugins
+ */
+struct topology_plugin {
+    const char *name;
+    int (*init)(struct topology *topo, const char *path, flux_error_t *error);
+};
+
+#endif /* !_BROKER_TOPOLOGY_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/broker/trace.c b/src/broker/trace.c
new file mode 100644
index 000000000000..aabca09655b1
--- /dev/null
+++ b/src/broker/trace.c
@@ -0,0 +1,145 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+
+#include "overlay.h"
+#include "trace.h"
+
+static const char *fake_control_topic (char *buf,
+                                       size_t size,
+                                       const flux_msg_t *msg)
+{
+    int ctype;
+    int cstatus;
+
+    if (flux_control_decode (msg, &ctype, &cstatus) < 0)
+        return NULL;
+    snprintf (buf,
+              size,
+              "%s %d",
+              ctype == CONTROL_HEARTBEAT ? "heartbeat" :
+              ctype == CONTROL_STATUS ? "status" :
+              ctype == CONTROL_DISCONNECT ? "disconnect" : "unknown",
+              cstatus);
+    return buf;
+}
+
+static bool match_nodeid (uint32_t overlay_peer, int nodeid)
+{
+    if (nodeid == -1 || overlay_peer == FLUX_NODEID_ANY)
+        return true;
+    return nodeid == overlay_peer;
+}
+
+static void trace_msg (flux_t *h,
+                       const char *prefix,
+                       uint32_t overlay_peer,      // FLUX_NODEID_ANY if n/a
+                       const char *module_name,    // NULL if n/a
+                       struct flux_msglist *trace_requests,
+                       const flux_msg_t *msg)
+{
+    const flux_msg_t *req;
+    double now;
+    int type = 0;
+    char buf[64];
+    const char *topic = NULL;
+    size_t payload_size = 0;
+    json_t *payload_json = NULL;
+    int errnum = 0;
+    const char *errstr = NULL;
+
+    if (!h
+        || !prefix
+        || !msg
+        || !trace_requests
+        || flux_msglist_count (trace_requests) == 0)
+        return;
+
+    now = flux_reactor_now (flux_get_reactor (h));
+    (void)flux_msg_get_type (msg, &type);
+    switch (type) {
+        case FLUX_MSGTYPE_CONTROL:
+            topic = fake_control_topic (buf, sizeof (buf), msg);
+            break;
+        case FLUX_MSGTYPE_REQUEST:
+        case FLUX_MSGTYPE_EVENT:
+            (void)flux_msg_get_topic (msg, &topic);
+            (void)flux_msg_get_payload (msg, NULL, &payload_size);
+            break;
+        case FLUX_MSGTYPE_RESPONSE:
+            (void)flux_msg_get_topic (msg, &topic);
+            if (flux_msg_get_errnum (msg, &errnum) == 0 && errnum > 0)
+                (void)flux_msg_get_string (msg, &errstr);
+            else
+                flux_msg_get_payload (msg, NULL, &payload_size);
+            break;
+    }
+
+    req = flux_msglist_first (trace_requests);
+    while (req) {
+        struct flux_match match = FLUX_MATCH_ANY;
+        int nodeid = -1;
+        int full = 0;
+        if (flux_request_unpack (req,
+                                 NULL,
+                                 "{s:i s:s s?i s?b}",
+                                 "typemask", &match.typemask,
+                                 "topic_glob", &match.topic_glob,
+                                 "nodeid", &nodeid,
+                                 "full", &full) < 0
+            || !match_nodeid (overlay_peer, nodeid)
+            || !flux_msg_cmp (msg, match))
+            goto next;
+
+        if (full && errnum == 0 && !payload_json && payload_size > 0)
+            (void)flux_msg_unpack (msg, "o", &payload_json);
+
+        if (flux_respond_pack (h,
+                               req,
+                               "{s:f s:s s:i s:s? s:i s:s s:i s:O? s:i s:s}",
+                               "timestamp", now,
+                               "prefix", prefix,
+                               "rank", overlay_peer,
+                               "name", module_name,
+                               "type", type,
+                               "topic", topic ? topic : "",
+                               "payload_size", payload_size,
+                               "payload", full ? payload_json : NULL,
+                               "errnum", errnum,
+                               "errstr", full && errstr ? errstr : "") < 0)
+            flux_log_error (h, "error responding to overlay.trace");
+next:
+        req = flux_msglist_next (trace_requests);
+    }
+}
+
+void trace_overlay_msg (flux_t *h,
+                        const char *prefix,
+                        uint32_t overlay_peer,
+                        struct flux_msglist *trace_requests,
+                        const flux_msg_t *msg)
+{
+    trace_msg (h, prefix, overlay_peer, NULL, trace_requests, msg);
+}
+
+void trace_module_msg (flux_t *h,
+                       const char *prefix,
+                       const char *module_name,
+                       struct flux_msglist *trace_requests,
+                       const flux_msg_t *msg)
+{
+    trace_msg (h, prefix, FLUX_NODEID_ANY, module_name, trace_requests, msg);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/broker/trace.h b/src/broker/trace.h
new file mode 100644
index 000000000000..6c725eede12d
--- /dev/null
+++ b/src/broker/trace.h
@@ -0,0 +1,32 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef BROKER_TRACE_H
+#define BROKER_TRACE_H
+
+#include <flux/core.h>
+
+/* Send trace info for 'msg' to all tracers in 'trace_requests'.
+ */
+void trace_module_msg (flux_t *h,
+                       const char *prefix,
+                       const char *module_name,
+                       struct flux_msglist *trace_requests,
+                       const flux_msg_t *msg);
+
+void trace_overlay_msg (flux_t *h,
+                        const char *prefix,
+                        uint32_t overlay_peer,
+                        struct flux_msglist *trace_requests,
+                        const flux_msg_t *msg);
+
+#endif /* BROKER_TRACE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/cmd/.gitignore b/src/cmd/.gitignore
index 83338367e5d2..1c59f60c8d2b 100644
--- a/src/cmd/.gitignore
+++ b/src/cmd/.gitignore
@@ -1,18 +1,13 @@
 /flux
-/flux-comms
-/flux-comms-stats
-/flux-event
-/flux-jobspec-validate
 /flux-keygen
 /flux-kvs
 /flux-logger
 /flux-module
-/flux-mping
 /flux-ping
-/flux-snoop
 /flux-start
-/flux-start-screen
-/flux-start-single
-/flux-start-srun
-/flux-up
+/flux-exec
+/flux-event
+/flux-queue
+/flux-job
+/flux-terminus
 /builtin-cmds.c
diff --git a/src/cmd/Makefile.am b/src/cmd/Makefile.am
index 5573b6cdd13f..9b273a78696e 100644
--- a/src/cmd/Makefile.am
+++ b/src/cmd/Makefile.am
@@ -6,36 +6,53 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
 	-I$(top_builddir)/src/common/libflux \
-	$(FLUX_SECURITY_CFLAGS) \
+	-I$(top_srcdir)/src/common/libccan \
 	$(ZMQ_CFLAGS) \
-	$(LIBUUID_CFLAGS) \
-	$(LIBSODIUM_CFLAGS)
+	$(FLUX_SECURITY_CFLAGS) \
+	$(HWLOC_CFLAGS) \
+	$(JANSSON_CFLAGS) \
+	$(LIBSYSTEMD_CFLAGS) \
+	$(LIBARCHIVE_CFLAGS)
+
 
 fluxcmd_ldadd = \
 	$(top_builddir)/src/common/libkvs/libkvs.la \
-	$(top_builddir)/src/common/libaggregate/libaggregate.la \
+	$(top_builddir)/src/common/librlist/librlist.la \
 	$(top_builddir)/src/common/libflux-internal.la \
 	$(top_builddir)/src/common/libflux-core.la \
 	$(top_builddir)/src/common/libflux-optparse.la \
 	$(FLUX_SECURITY_LIBS) \
-	$(ZMQ_LIBS) \
-	$(LIBUUID_LIBS) \
 	$(LIBPTHREAD) \
-	$(LIBDL) \
-	$(HWLOC_LIBS) \
-	$(LIBSODIUM_LIBS)
+	$(JANSSON_LIBS)
 
 LDADD = $(fluxcmd_ldadd)
 
 EXTRA_DIST = \
 	builtin-cmds-list.sh
 CLEANFILES = \
-	builtin-cmds.c
+	builtin-cmds.c \
+	flux-pkill.py
+
 BUILT_SOURCES = \
-	builtin-cmds.c
+	builtin-cmds.c \
+	flux-pkill.py
+
+flux-pkill.py:
+	$(AM_V_GEN)$(LN_S) $(srcdir)/flux-pgrep.py flux-pkill.py
+
+install-exec-hook:
+	$(AM_V_at)echo Linking flux-pkill to flux-pgrep && \
+	 (cd $(DESTDIR)$(fluxcmddir) && \
+	  rm -f flux-pkill.py && \
+	  $(LN_S) flux-pgrep.py flux-pkill.py)
+
+uninstall-local:
+	$(AM_V_at)echo Removing $(fluxcmddir)/flux-pkill.py && \
+	  rm -f $(fluxcmddir)/flux-pkill.py
 
 bin_PROGRAMS = flux
 flux_SOURCES = \
@@ -44,19 +61,37 @@ flux_SOURCES = \
 	cmdhelp.c \
 	builtin.h \
 	builtin/attr.c \
+	builtin/config.c \
 	builtin/help.c \
 	builtin/dmesg.c \
 	builtin/env.c \
 	builtin/content.c \
 	builtin/version.c \
-	builtin/hwloc.c \
 	builtin/heaptrace.c \
 	builtin/proxy.c \
+	builtin/overlay.c \
 	builtin/relay.c \
 	builtin/python.c \
-	builtin/user.c
+	builtin/uptime.c \
+	builtin/startlog.c \
+	builtin/dump.c \
+	builtin/restore.c \
+	builtin/archive.c \
+	builtin/pmi.c \
+	builtin/shutdown.c \
+	builtin/lptest.c
 nodist_flux_SOURCES = \
 	builtin-cmds.c
+dist_bin_SCRIPTS = flux-python
+
+flux_LDADD = \
+	$(top_builddir)/src/common/libpmi/libupmi.la \
+	$(top_builddir)/src/common/libpmi/libpmi_client.la \
+	$(top_builddir)/src/common/libpmi/libpmi_common.la \
+	$(top_builddir)/src/common/libfilemap/libfilemap.la \
+	$(LIBARCHIVE_LIBS) \
+	$(LIBSYSTEMD_LIBS) \
+	$(fluxcmd_ldadd)
 
 #
 # Flux subcommands
@@ -64,28 +99,134 @@ nodist_flux_SOURCES = \
 
 dist_fluxcmd_SCRIPTS = \
 	flux-cron \
-	flux-jobspec.py \
-	flux-mini.py \
-	flux-jobs.py
+	flux-alloc.py \
+	flux-batch.py \
+	flux-run.py \
+	flux-submit.py \
+	flux-bulksubmit.py \
+	flux-jobs.py \
+	flux-fortune.py \
+	flux-resource.py \
+	flux-admin.py \
+	flux-jobtap.py \
+	flux-job-validator.py \
+	flux-job-frobnicator.py \
+	flux-job-exec-override.py \
+	flux-uri.py \
+	flux-pstree.py \
+	flux-pgrep.py \
+	flux-queue.py \
+	flux-cancel.py \
+	flux-watch.py \
+	flux-update.py \
+	flux-imp-exec-helper \
+	py-runner.py \
+	flux-hostlist.py \
+	flux-post-job-event.py \
+	flux-run-housekeeping \
+	flux-housekeeping.py \
+	flux-run-prolog \
+	flux-run-epilog
 
 fluxcmd_PROGRAMS = \
-	flux-aggregate \
+	flux-terminus \
 	flux-ping \
 	flux-keygen \
 	flux-logger \
 	flux-event \
 	flux-module \
-	flux-comms \
 	flux-kvs \
 	flux-start \
 	flux-job \
-	flux-queue \
-	flux-exec
+	flux-exec \
+	flux-R \
+	flux-top
+
+flux_job_SOURCES = \
+	job/main.c \
+	job/common.c \
+	job/common.h \
+	job/mpir.c \
+	job/mpir.h \
+	job/attach.c \
+	job/submit.c \
+	job/list.c \
+	job/status.c \
+	job/id.c \
+	job/namespace.c \
+	job/urgency.c \
+	job/cancel.c \
+	job/eventlog.c \
+	job/info.c \
+	job/stats.c \
+	job/wait.c \
+	job/memo.c \
+	job/purge.c \
+	job/taskmap.c \
+	job/timeleft.c \
+	job/last.c \
+	job/hostpids.c
 
 flux_start_LDADD = \
 	$(fluxcmd_ldadd) \
 	$(top_builddir)/src/common/libpmi/libpmi_server.la \
-	$(LIBUTIL)
+	$(top_builddir)/src/common/libpmi/libpmi_common.la \
+	$(top_builddir)/src/common/libczmqcontainers/libczmqcontainers.la
+
+flux_job_LDADD = \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(top_builddir)/src/shell/libmpir.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(top_builddir)/src/common/libutil/libutil.la \
+	$(top_builddir)/src/common/libczmqcontainers/libczmqcontainers.la \
+	$(top_builddir)/src/common/libdebugged/libdebugged.la \
+	$(top_builddir)/src/common/libterminus/libterminus.la \
+	$(fluxcmd_ldadd)
+
+flux_exec_LDADD = \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(fluxcmd_ldadd)
+
+flux_terminus_LDADD = \
+	$(top_builddir)/src/common/libterminus/libterminus.la \
+	$(top_builddir)/src/common/libidset/libidset.la \
+	$(top_builddir)/src/common/libutil/libutil.la \
+	$(fluxcmd_ldadd)
+
+flux_R_LDADD = \
+	$(fluxcmd_ldadd) \
+	$(top_builddir)/src/common/librlist/librlist-hwloc.la \
+	$(top_builddir)/src/common/librlist/librlist.la \
+	$(top_builddir)/src/common/libczmqcontainers/libczmqcontainers.la \
+	$(top_builddir)/src/common/libhostlist/libhostlist.la \
+	$(top_builddir)/src/common/libidset/libidset.la \
+	$(top_builddir)/src/common/libutil/libutil.la \
+	$(HWLOC_LIBS) \
+	$(JANSSON_LIBS)
+
+flux_keygen_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(ZMQ_CFLAGS)
+
+flux_keygen_LDADD = \
+	$(top_builddir)/src/common/libzmqutil/libzmqutil.la \
+	$(fluxcmd_ldadd) \
+	$(ZMQ_LIBS)
+
+flux_top_SOURCES = \
+	top/top.c \
+	top/top.h \
+	top/keys.c \
+	top/joblist_pane.c \
+	top/summary_pane.c \
+	top/ucache.c \
+	top/queues.c
+flux_top_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(CURSES_CFLAGS)
+flux_top_LDADD = \
+	$(fluxcmd_ldadd) \
+	$(CURSES_LIBS)
 
 #
 # Automatically build list of flux(1) builtins from
diff --git a/src/cmd/builtin-cmds-list.sh b/src/cmd/builtin-cmds-list.sh
index 84f3964aa52a..3e65533e94b5 100755
--- a/src/cmd/builtin-cmds-list.sh
+++ b/src/cmd/builtin-cmds-list.sh
@@ -34,4 +34,3 @@ cat <<EOF
     { NULL, NULL },
 };
 EOF
-
diff --git a/src/cmd/builtin/archive.c b/src/cmd/builtin/archive.c
new file mode 100644
index 000000000000..9822ba458ad7
--- /dev/null
+++ b/src/cmd/builtin/archive.c
@@ -0,0 +1,870 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "builtin.h"
+
+#include <unistd.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <libgen.h>
+#include <jansson.h>
+#include <archive.h>
+#include <fnmatch.h>
+#include <sys/mman.h>
+
+#include "ccan/base64/base64.h"
+#include "ccan/str/str.h"
+#include "src/common/libutil/dirwalk.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libkvs/treeobj.h"
+#include "src/common/libcontent/content.h"
+#include "src/common/libfilemap/filemap.h"
+#include "src/common/libfilemap/fileref.h"
+
+static void unlink_archive (flux_t *h,
+                            const char *namespace,
+                            const char *name,
+                            bool force);
+static void unmap_archive (flux_t *h, const char *name);
+
+static const char *default_chunksize = "1M";
+static const char *default_small_file_threshold = "1K";
+const char *default_archive_hashtype = "sha1";
+const char *default_name = "main";
+
+#define OLD_FILEMAP_COMMAND 1
+
+#if OLD_FILEMAP_COMMAND
+const char *filemap_warning = "⚠️This command is deprecated.⚠️  "
+                              " Use flux-archive(1) instead.";
+#endif
+
+/* Return true if RFC 37 fileref has blobref encoding
+ */
+static bool is_blobvec_encoding (json_t *fileref)
+{
+    const char *encoding;
+    if (json_unpack (fileref, "{s:s}", "encoding", &encoding) < 0
+        || !streq (encoding, "blobvec"))
+        return false;
+    return true;
+}
+
+struct create_ctx {
+    optparse_t *p;
+    flux_t *h;
+    const char *name;
+    const char *namespace;
+    int verbose;
+    bool use_mmap;
+    struct blobvec_param param;
+    json_t *archive;
+    flux_kvs_txn_t *txn;
+    int preserve_seq;
+};
+
+/* Request that the content module mmap(2) the file at 'path', providing
+ * the same 'chunksize' as was used to create the RFC 37 fileref below,
+ * so that all the same blobrefs are created and made available in the cache.
+ */
+static void mmap_fileref_data (struct create_ctx *ctx, const char *path)
+{
+    char *fullpath;
+    flux_future_t *f;
+
+    // relative path is preserved in the archive, but broker needs full path
+    if (!(fullpath = realpath (path, NULL)))
+        log_err_exit ("%s", path);
+
+    if (!(f = flux_rpc_pack (ctx->h,
+                             "content.mmap-add",
+                             0,
+                             0,
+                             "{s:s s:i s:s}",
+                             "path", fullpath,
+                             "chunksize", ctx->param.chunksize,
+                             "tag", ctx->name))
+        || flux_rpc_get (f, NULL))
+        log_msg_exit ("%s: %s", path, future_strerror (f, errno));
+    flux_future_destroy (f);
+    free (fullpath);
+}
+
+/* Store the blobs of an RFC 37 blobvec-encoded fileref to the content store.
+ * If the --preserve option was specified, create a KVS reference to each
+ * blob (added to the pending KVS transaction).
+ */
+static void store_fileref_data (struct create_ctx *ctx,
+                                const char *path,
+                                json_t *fileref,
+                                struct blobvec_mapinfo *mapinfo)
+{
+    json_t *data;
+
+    if (json_unpack (fileref, "{s:o}", "data", &data) == 0) {
+        size_t index;
+        json_t *entry;
+
+        // iterate over blobs in blobvec
+        json_array_foreach (data, index, entry) {
+            const char *blobref;
+            json_int_t size;
+            json_int_t offset;
+            flux_future_t *f;
+
+            if (json_unpack (entry, "[I,I,s]", &offset, &size, &blobref) < 0)
+                log_msg_exit ("%s: error decoding fileref object data", path);
+            if (offset + size > mapinfo->size)
+                log_msg_exit ("%s: fileref offset exceeds file size", path);
+
+            // store blob (synchronously)
+            if (!(f = content_store (ctx->h, mapinfo->base + offset, size, 0))
+                || flux_rpc_get (f, NULL) < 0)
+                log_msg_exit ("%s: error storing blob: %s",
+                              path,
+                              future_strerror (f, errno));
+            flux_future_destroy (f);
+
+            /* Optionally store a KVS key that references blob for --preserve.
+             * N.B. we don't attempt to combine blobrefs that belong to the
+             * same file to conserve metadata because dump/restore might not
+             * use the same chunksize, rendering archive blobrefs invalid.
+             */
+            if (optparse_hasopt (ctx->p, "preserve")) {
+                json_t *valref;
+                char *s;
+                char *key;
+
+                if (!(valref = treeobj_create_valref (blobref))
+                    || !(s = treeobj_encode (valref))
+                    || asprintf (&key,
+                                 "archive.%s_blobs.%d",
+                                 ctx->name,
+                                 ctx->preserve_seq++) < 0
+                    || flux_kvs_txn_put_treeobj (ctx->txn, 0, key, s) < 0)
+                    log_err_exit ("%s: error preserving blobrefs", path);
+                free (key);
+                free (s);
+                json_decref (valref);
+            }
+        }
+    }
+}
+
+/* Create an RFC 37 fileref object for 'path', and append it to ctx->archive.
+ * Then synchronously store any blobs to the content store if the file is not
+ * fully contained in the fileref.
+ */
+static void add_archive_file (struct create_ctx *ctx, const char *path)
+{
+    struct blobvec_mapinfo mapinfo;
+    flux_error_t error;
+    json_t *fileref;
+
+    if (!(fileref = fileref_create_ex (path,
+                                       &ctx->param,
+                                       &mapinfo,
+                                       &error)))
+        log_msg_exit ("%s", error.text); // error text includes path
+    if (json_array_append_new (ctx->archive, fileref) < 0)
+        log_msg_exit ("%s: out of memory", path);
+    if (is_blobvec_encoding (fileref)) {
+        if (ctx->use_mmap)
+            mmap_fileref_data (ctx, path);
+        else
+            store_fileref_data (ctx, path, fileref, &mapinfo);
+    }
+    (void)munmap (mapinfo.base, mapinfo.size);
+}
+
+// dirwalk visitor
+static int archive_visitor (dirwalk_t *d, void *arg)
+{
+    struct create_ctx *ctx = arg;
+    const char *path = dirwalk_path (d);
+
+    if (streq (path, "."))
+        return 0;
+    if (ctx->verbose > 0)
+        printf ("%s\n", path);
+    add_archive_file (ctx, path);
+    return 0;
+}
+
+static int subcmd_create (optparse_t *p, int ac, char *av[])
+{
+    struct create_ctx ctx;
+    int n = optparse_option_index (p);
+    const char *directory = optparse_get_str (p, "directory", NULL);
+    int flags = DIRWALK_FIND_DIR | DIRWALK_DEPTH;
+    const char *s;
+    char *hashtype;
+    char *key;
+
+    memset (&ctx, 0, sizeof (ctx));
+
+    if (n == ac) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    ctx.name = optparse_get_str (p, "name", NULL);
+#if OLD_FILEMAP_COMMAND
+    if (streq (av[0], "map")) {
+        fprintf (stderr, "%s\n", filemap_warning);
+        if (!optparse_hasopt (p, "disable-mmap"))
+            ctx.use_mmap = true;
+        ctx.name = optparse_get_str (p, "tags", ctx.name);
+    }
+    else if (optparse_hasopt (p, "tags")
+        || optparse_hasopt (p, "disable-mmap")) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+#endif
+    if (!ctx.name)
+        ctx.name = default_name;
+
+    ctx.p = p;
+    ctx.namespace = "primary";
+    if (optparse_hasopt (p, "no-force-primary"))
+        ctx.namespace = NULL;
+    ctx.verbose = optparse_get_int (p, "verbose", 0);
+    ctx.param.chunksize = optparse_get_size_int (p,
+                                                 "chunksize",
+                                                 default_chunksize);
+    ctx.param.small_file_threshold = optparse_get_size_int (p,
+                                                 "small-file-threshold",
+                                                 default_small_file_threshold);
+    if (!(ctx.h = builtin_get_flux_handle (p)))
+        log_err_exit ("flux_open");
+
+    /* --mmap lets large files be represented in the content cache without
+     * being copied.  It is efficient for broadcasting large files such as
+     * VM images that are not practical to copy into the KVS, but it has
+     * several caveats:
+     * - it is only supported on the rank 0 broker (via content.mmap-* RPCs)
+     * - the files must not change while they are mapped
+     * - when the files are unmapped, references (blobrefs) become invalid
+     */
+    if (optparse_hasopt (p, "mmap"))
+        ctx.use_mmap = true;
+    if (ctx.use_mmap) {
+        uint32_t rank;
+        if (flux_get_rank (ctx.h, &rank) < 0)
+            log_err_exit ("error fetching broker rank");
+        if (rank > 0)
+            log_msg_exit ("--mmap only works on the rank 0 broker");
+        if (optparse_hasopt (p, "preserve"))
+            log_msg_exit ("--mmap cannot work with --preserve");
+        if (optparse_hasopt (p, "no-force-primary"))
+            log_msg_exit ("--mmap cannot work with --no-force-primary");
+    }
+    if (optparse_hasopt (p, "overwrite") && optparse_hasopt (p, "append"))
+        log_msg_exit ("--overwrite and --append cannot be used together");
+
+    if (!(s = flux_attr_get (ctx.h, "content.hash")))
+        s = default_archive_hashtype;
+    if (!(hashtype = strdup (s))) // 's' will not remain valid for long so copy
+        log_msg_exit ("out of memory");
+    ctx.param.hashtype = hashtype;
+    if (asprintf (&key, "archive.%s", ctx.name) < 0)
+        log_msg_exit ("out of memory");
+
+    if (directory) {
+        if (chdir (directory) < 0)
+            log_err_exit ("chdir %s", directory);
+    }
+
+    /* Deal with pre-existing key.
+     */
+    if (optparse_hasopt (p, "overwrite")) {
+        unlink_archive (ctx.h, ctx.namespace, ctx.name, true);
+        unmap_archive (ctx.h, ctx.name);
+    }
+    else {
+        flux_future_t *f;
+        json_t *archive;
+
+        if ((f = flux_kvs_lookup (ctx.h, ctx.namespace, 0, key))
+            && flux_kvs_lookup_get_unpack (f, "o", &archive) == 0) {
+            if (optparse_hasopt (p, "append")) {
+                ctx.archive = json_incref (archive);
+            }
+            else
+                log_msg_exit ("%s: key exists", key);
+        }
+        flux_future_destroy (f);
+    }
+    if (!ctx.archive) {
+        if (!(ctx.archive = json_array ()))
+            log_msg_exit ("out of memory");
+    }
+
+    /* Prepare KVS transaction.
+     */
+    if (!(ctx.txn = flux_kvs_txn_create ()))
+        log_err_exit ("could not prepare KVS transaction");
+
+    /* Iterate over PATHs and (recursively) their contents, building the
+     * RFC 37 archive in 'ctx.archive'.
+     */
+    while (n < ac) {
+        const char *path = av[n++];
+        struct stat sb;
+
+        if (lstat (path, &sb) < 0)
+            log_err_exit ("%s", path);
+        if (S_ISDIR (sb.st_mode)) {
+            // archive_visitor() calls add_archive_file() for files under path
+            if (dirwalk (path, flags, archive_visitor, &ctx) < 0)
+                log_err_exit ("%s", path);
+        }
+        else {
+            if (ctx.verbose > 0)
+                printf ("%s\n", path);
+            add_archive_file (&ctx, path);
+        }
+    }
+
+    /* commit ctx.archive object to KVS
+     */
+    flux_future_t *f = NULL;
+    if (flux_kvs_txn_pack (ctx.txn, 0, key, "O", ctx.archive) < 0
+        || !(f = flux_kvs_commit (ctx.h, ctx.namespace, 0, ctx.txn))
+        || flux_rpc_get (f, NULL) < 0)
+        log_msg_exit ("kvs commit: %s", future_strerror (f, errno));
+    flux_future_destroy (f);
+
+    flux_close (ctx.h);
+
+    free (key);
+    flux_kvs_txn_destroy (ctx.txn);
+    free (hashtype);
+
+    return 0;
+}
+
+static bool key_exists (flux_t *h, const char *namespace, const char *key)
+{
+    flux_future_t *f;
+    bool exists = true;
+
+    if (!(f = flux_kvs_lookup (h, namespace, 0, key))
+        || flux_kvs_lookup_get (f, NULL) < 0)
+        exists = false;
+    flux_future_destroy (f);
+    return exists;
+}
+
+/* Unlink archive.name and archive.name_blobs from the KVS.
+ * If force is true, it is not an error if the keys do not exist.
+ */
+static void unlink_archive (flux_t *h,
+                            const char *namespace,
+                            const char *name,
+                            bool force)
+{
+    flux_kvs_txn_t *txn;
+    char *key_blobs = NULL;
+    char *key = NULL;
+    flux_future_t *f = NULL;
+
+    if (asprintf (&key, "archive.%s", name) < 0)
+        log_msg_exit ("out of memory");
+
+    if (!force && !key_exists (h, namespace, key))
+        log_msg_exit ("%s does not exist", key);
+
+    if (!(txn = flux_kvs_txn_create ())
+        || flux_kvs_txn_unlink (txn, 0, key) < 0
+        || asprintf (&key_blobs, "archive.%s_blobs", name) < 0
+        || flux_kvs_txn_unlink (txn, 0, key_blobs) < 0
+        || !(f = flux_kvs_commit (h, namespace, 0, txn))
+        || flux_rpc_get (f, NULL) < 0) {
+        log_msg ("unlink %s,%s: %s",
+                 key,
+                 key_blobs,
+                 future_strerror (f, errno));
+    }
+
+    flux_future_destroy (f);
+    free (key_blobs);
+    free (key);
+    flux_kvs_txn_destroy (txn);
+}
+
+/* Unmap files from the rank 0 content service.
+ * It is not an error if the tag does not match any files.
+ */
+static void unmap_archive (flux_t *h, const char *name)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_rpc_pack (h,
+                             "content.mmap-remove",
+                             0,
+                             0,
+                             "{s:s}",
+                             "tag", name))
+        || flux_rpc_get (f, NULL) < 0) {
+        log_msg ("unmap %s: %s", name, future_strerror (f, errno));
+    }
+    flux_future_destroy (f);
+}
+
+static int subcmd_remove (optparse_t *p, int ac, char *av[])
+{
+    const char *namespace = "primary";
+    const char *name;
+    int n = optparse_option_index (p);
+    flux_t *h;
+
+    if (n < ac) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    name = optparse_get_str (p, "name", NULL);
+#if OLD_FILEMAP_COMMAND
+    if (streq (av[0], "unmap")) {
+        fprintf (stderr, "%s\n", filemap_warning);
+        name = optparse_get_str (p, "tags", name);
+    }
+    else if (optparse_hasopt (p, "tags")) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+#endif
+    if (!name)
+        name = default_name;
+    if (optparse_hasopt (p, "no-force-primary"))
+        namespace = NULL;
+    if (!(h = builtin_get_flux_handle (p)))
+        log_err_exit ("flux_open");
+
+    unlink_archive (h, namespace, name, optparse_hasopt (p, "force"));
+    unmap_archive (h, name);
+
+    flux_close (h);
+    return 0;
+}
+
+/* Filter out archive entries that don't match 'pattern'.
+ * This presumes the RFC 37 archive was stored in array form.  If this
+ * is extended to support extracting files from jobspec, dictionary
+ * support must be added.
+ */
+static void apply_glob (json_t *archive, const char *pattern)
+{
+    size_t index = 0;
+    while (index < json_array_size (archive)) {
+        json_t *entry;
+        const char *path;
+
+        if (!(entry = json_array_get (archive, index))
+            || json_unpack (entry, "{s:s}", "path", &path) < 0
+            || fnmatch (pattern, path, 0) != 0) {
+            json_array_remove (archive, index);
+            continue;
+        }
+        index++;
+    }
+    if (json_array_size (archive) == 0)
+        log_msg ("No files matched pattern '%s'", pattern);
+}
+
+static void trace_fn (void *arg,
+                      json_t *fileref,
+                      const char *path,
+                      int mode,
+                      int64_t size,
+                      int64_t mtime,
+                      int64_t ctime,
+                      const char *encoding)
+{
+    int level = *(int *)arg;
+    if (level > 0)
+        fprintf (stderr, "%s\n", path);
+}
+
+static int subcmd_extract (optparse_t *p, int ac, char *av[])
+{
+    int n = optparse_option_index (p);
+    const char *directory = optparse_get_str (p, "directory", NULL);
+    const char *name;
+    const char *namespace = "primary";
+    const char *pattern = NULL;
+    int opts = 0;
+    char *key;
+    int kvs_flags = 0;
+    flux_t *h;
+    flux_future_t *f;
+    double timeout = -1;
+    json_t *archive;
+    flux_error_t error;
+
+    if (n < ac)
+        pattern = av[n++];
+    if (n < ac) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    name = optparse_get_str (p, "name", NULL);
+#if OLD_FILEMAP_COMMAND
+    if (streq (av[0], "get")) {
+        fprintf (stderr, "%s\n", filemap_warning);
+        name = optparse_get_str (p, "tags", name);
+    }
+    else if (optparse_hasopt (p, "tags")) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+#endif
+    if (!name)
+        name = default_name;
+    if (asprintf (&key, "archive.%s", name) < 0)
+        log_msg_exit ("out of memory");
+    if (optparse_hasopt (p, "no-force-primary"))
+        namespace = NULL;
+    if (!optparse_hasopt (p, "overwrite"))
+        opts |= ARCHIVE_EXTRACT_NO_OVERWRITE;
+    if (optparse_hasopt (p, "waitcreate")) {
+        kvs_flags |= FLUX_KVS_WAITCREATE;
+        const char *arg = optparse_get_str (p, "waitcreate", NULL);
+        if (arg && fsd_parse_duration (arg, &timeout) < 0)
+            log_err_exit ("could not parse --waitcreate timeout");
+    }
+    if (directory) {
+        if (chdir (directory) < 0)
+            log_err_exit ("chdir %s", directory);
+    }
+    if (!(h = builtin_get_flux_handle (p)))
+        log_err_exit ("flux_open");
+
+    /* Fetch the archive from KVS.
+     * If --waitcreate, block until the key appears, or timeout is reached.
+     */
+    if (!(f = flux_kvs_lookup (h, namespace, kvs_flags, key)))
+        log_err_exit ("error sending KVS lookup request");
+    if (flux_future_wait_for (f, timeout) < 0 && errno == ETIMEDOUT)
+        log_msg_exit ("%s: key was not created within timeout window", key);
+    if (flux_kvs_lookup_get_unpack (f, "o", &archive) < 0)
+        log_msg_exit ("KVS lookup %s: %s", key, future_strerror (f, errno));
+    if (pattern)
+        apply_glob (archive, pattern);
+
+    /* List files (no extraction).
+     */
+    if (optparse_hasopt (p, "list-only")) {
+        size_t index;
+        json_t *entry;
+        char buf[80];
+
+        json_array_foreach (archive, index, entry) {
+            fileref_pretty_print (entry,
+                                  NULL,
+                                  optparse_hasopt (p, "verbose"),
+                                  buf,
+                                  sizeof (buf));
+            printf ("%s\n", buf);
+        }
+    }
+    /* Extract files.
+     * filemap_extract() fetches any content blobs referenced by large files.
+     * This can fail if the instance was restarted and the archive was not
+     * created with --preserve.
+     */
+    else {
+        int level = optparse_get_int (p, "verbose", 0);
+        if (filemap_extract (h, archive, opts, &error, trace_fn, &level) < 0)
+            log_msg_exit ("%s", error.text);
+    }
+
+    free (key);
+    flux_future_destroy (f);
+    flux_close (h);
+    return 0;
+}
+
+static int subcmd_list (optparse_t *p, int ac, char *av[])
+{
+    int n = optparse_option_index (p);
+    const char *name;
+    const char *namespace = "primary";
+    const char *pattern = NULL;
+    char *key;
+    flux_t *h;
+    flux_future_t *f;
+    json_t *archive;
+    size_t index;
+    json_t *entry;
+
+    if (n < ac)
+        pattern = av[n++];
+    if (n < ac) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    name = optparse_get_str (p, "name", NULL);
+#if OLD_FILEMAP_COMMAND
+    if (optparse_hasopt (p, "tags")) {
+        fprintf (stderr, "%s\n", filemap_warning);
+        name = optparse_get_str (p, "tags", name);
+    }
+#endif
+    if (!name)
+        name = default_name;
+    if (asprintf (&key, "archive.%s", name) < 0)
+        log_msg_exit ("out of memory");
+    if (optparse_hasopt (p, "no-force-primary"))
+        namespace = NULL;
+
+    if (!(h = builtin_get_flux_handle (p)))
+        log_err_exit ("flux_open");
+    if (!(f = flux_kvs_lookup (h, namespace, 0, key)))
+        log_err_exit ("error sending KVS lookup request");
+    if (flux_kvs_lookup_get_unpack (f, "o", &archive) < 0)
+        log_msg_exit ("KVS lookup %s: %s", key, future_strerror (f, errno));
+    if (pattern)
+        apply_glob (archive, pattern);
+    json_array_foreach (archive, index, entry) {
+        if (optparse_hasopt (p, "raw")) {
+            if (json_dumpf (entry, stdout, JSON_COMPACT) < 0)
+                log_msg_exit ("error dumping RFC 37 file system object");
+        }
+        else {
+            char buf[120];
+            fileref_pretty_print (entry,
+                                  NULL,
+                                  optparse_hasopt (p, "long"),
+                                  buf,
+                                  sizeof (buf));
+            printf ("%s\n", buf);
+        }
+    }
+    free (key);
+    flux_future_destroy (f);
+    flux_close (h);
+    return 0;
+}
+
+int cmd_archive (optparse_t *p, int ac, char *av[])
+{
+    log_init ("flux-archive");
+
+    if (optparse_run_subcommand (p, ac, av) != OPTPARSE_SUCCESS)
+        exit (1);
+    return (0);
+}
+
+static struct optparse_option create_opts[] = {
+    { .name = "name", .key = 'n', .has_arg = 1, .arginfo = "NAME",
+      .usage = "Write to archive NAME (default main)", },
+    { .name = "no-force-primary", .has_arg = 0,
+      .usage = "Do not force archive to be in the primary KVS namespace", },
+    { .name = "directory", .key = 'C', .has_arg = 1, .arginfo = "DIR",
+      .usage = "Change to DIR before reading files", },
+    { .name = "verbose", .key = 'v', .has_arg = 2, .arginfo = "[LEVEL]",
+      .usage = "Increase output detail.", },
+    { .name = "overwrite", .has_arg = 0,
+      .usage = "Overwrite existing archive", },
+    { .name = "append", .has_arg = 0,
+      .usage = "Append to existing archive", },
+    { .name = "preserve", .has_arg = 0,
+      .usage = "Preserve data over Flux restart" },
+    { .name = "mmap", .has_arg = 0,
+      .usage = "Use mmap(2) to map file content" },
+    { .name = "chunksize", .has_arg = 1, .arginfo = "N[KMG]",
+      .usage = "Limit blob size to N bytes with 0=unlimited (default 1M)",
+      .flags = OPTPARSE_OPT_HIDDEN, },
+    { .name = "small-file-threshold", .has_arg = 1, .arginfo = "N[KMG]",
+      .usage = "Adjust the maximum size of a \"small file\" in bytes"
+               " (default 1K)",
+      .flags = OPTPARSE_OPT_HIDDEN, },
+#if OLD_FILEMAP_COMMAND
+    { .name = "tags", .key = 'T', .has_arg = 1, .arginfo = "TAG",
+      .usage = "alias for --name",
+      .flags = OPTPARSE_OPT_HIDDEN, },
+    { .name = "disable-mmap", .has_arg = 0,
+      .usage = "map subcommand no longer implies mmap",
+      .flags = OPTPARSE_OPT_HIDDEN, },
+#endif
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option remove_opts[] = {
+    { .name = "name", .key = 'n', .has_arg = 1, .arginfo = "NAME",
+      .usage = "Remove archive NAME (default main)", },
+    { .name = "no-force-primary", .has_arg = 0,
+      .usage = "Do not force archive to be in the primary KVS namespace", },
+    { .name = "force", .key = 'f', .has_arg = 0,
+      .usage = "Ignore a nonexistent archive", },
+#if OLD_FILEMAP_COMMAND
+    { .name = "tags", .key = 'T', .has_arg = 1, .arginfo = "TAG",
+      .usage = "alias for --name",
+      .flags = OPTPARSE_OPT_HIDDEN, },
+#endif
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option extract_opts[] = {
+    { .name = "name", .key = 'n', .has_arg = 1, .arginfo = "NAME",
+      .usage = "Read from archive NAME (default main)", },
+    { .name = "verbose", .key = 'v', .has_arg = 2, .arginfo = "[LEVEL]",
+      .usage = "Show filenames on stderr", },
+    { .name = "directory", .key = 'C', .has_arg = 1, .arginfo = "DIR",
+      .usage = "Change to DIR before extracting", },
+    { .name = "overwrite", .has_arg = 0,
+      .usage = "Overwrite existing files when extracting", },
+    { .name = "waitcreate", .has_arg = 2, .arginfo = "[FSD]",
+      .usage = "Wait for KVS archive key to appear (timeout optional)", },
+    { .name = "no-force-primary", .has_arg = 0,
+      .usage = "Do not force archive to be in the primary KVS namespace", },
+    { .name = "list-only", .key = 't', .has_arg = 0,
+      .usage = "List table of contents without extracting", },
+#if OLD_FILEMAP_COMMAND
+    { .name = "tags", .key = 'T', .has_arg = 1, .arginfo = "TAG",
+      .usage = "alias for --name",
+      .flags = OPTPARSE_OPT_HIDDEN, },
+#endif
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option list_opts[] = {
+    { .name = "name", .key = 'n', .has_arg = 1, .arginfo = "NAME",
+      .usage = "Read from archive NAME (default main)", },
+    { .name = "no-force-primary", .has_arg = 0,
+      .usage = "Do not force archive to be in the primary KVS namespace", },
+    { .name = "long", .key = 'l', .has_arg = 0,
+      .usage = "Show file type, mode, size", },
+    { .name = "raw", .has_arg = 0,
+      .usage = "Show raw RFC 37 file system object without decoding", },
+#if OLD_FILEMAP_COMMAND
+    { .name = "tags", .key = 'T', .has_arg = 1, .arginfo = "TAG",
+      .usage = "alias for --name",
+      .flags = OPTPARSE_OPT_HIDDEN, },
+#endif
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_subcommand archive_subcmds[] = {
+    { "create",
+      "[-n NAME] [-C DIR] [--preserve] PATH ...",
+      "Create a KVS file archive",
+      subcmd_create,
+      0,
+      create_opts,
+    },
+    { "remove",
+      "[-n NAME] [-f]",
+      "Remove a KVS file archive",
+      subcmd_remove,
+      0,
+      remove_opts,
+    },
+    { "extract",
+      "[-n NAME] [--overwrite] [-C DIR] [PATTERN]",
+      "Extract KVS file archive contents",
+      subcmd_extract,
+      0,
+      extract_opts,
+    },
+    { "list",
+      "[-n NAME] [PATTERN]",
+      "List KVS file archive contents",
+      subcmd_list,
+      0,
+      list_opts,
+    },
+    OPTPARSE_SUBCMD_END
+};
+
+#if OLD_FILEMAP_COMMAND
+int cmd_filemap (optparse_t *p, int ac, char *av[])
+{
+    if (optparse_run_subcommand (p, ac, av) != OPTPARSE_SUCCESS)
+        exit (1);
+    return 0;
+}
+
+static struct optparse_subcommand filemap_subcmds[] = {
+    { "map",
+      NULL,
+      NULL,
+      subcmd_create,
+      OPTPARSE_SUBCMD_HIDDEN,
+      create_opts,
+    },
+    { "unmap",
+      NULL,
+      NULL,
+      subcmd_remove,
+      OPTPARSE_SUBCMD_HIDDEN,
+      remove_opts,
+    },
+    { "get",
+      NULL,
+      NULL,
+      subcmd_extract,
+      OPTPARSE_SUBCMD_HIDDEN,
+      extract_opts,
+    },
+    { "list",
+      NULL,
+      NULL,
+      subcmd_list,
+      OPTPARSE_SUBCMD_HIDDEN,
+      list_opts,
+    },
+    OPTPARSE_SUBCMD_END
+};
+#endif
+
+int subcommand_archive_register (optparse_t *p)
+{
+    optparse_err_t e;
+
+    e = optparse_reg_subcommand (p,
+                                 "archive",
+                                 cmd_archive,
+                                 NULL,
+                                 "Flux KVS file archive utility",
+                                 0,
+                                 NULL);
+    if (e != OPTPARSE_SUCCESS)
+        return -1;
+    e = optparse_reg_subcommands (optparse_get_subcommand (p, "archive"),
+                                  archive_subcmds);
+    if (e != OPTPARSE_SUCCESS)
+        return -1;
+#if OLD_FILEMAP_COMMAND
+    e = optparse_reg_subcommand (p,
+                                 "filemap",
+                                 cmd_filemap,
+                                 NULL,
+                                 filemap_warning,
+                                 0,
+                                 NULL);
+    if (e != OPTPARSE_SUCCESS)
+        return -1;
+    e = optparse_reg_subcommands (optparse_get_subcommand (p, "filemap"),
+                                  filemap_subcmds);
+    if (e != OPTPARSE_SUCCESS)
+        return -1;
+#endif
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/builtin/attr.c b/src/cmd/builtin/attr.c
index a9c0cdc8bbb0..aadddbe697c4 100644
--- a/src/cmd/builtin/attr.c
+++ b/src/cmd/builtin/attr.c
@@ -8,16 +8,14 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include <czmq.h>
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <jansson.h>
-#include "builtin.h"
 
-static struct optparse_option setattr_opts[] = {
-    { .name = "expunge", .key = 'e', .has_arg = 0,
-      .usage = "Unset the specified attribute",
-    },
-    OPTPARSE_TABLE_END
-};
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "builtin.h"
 
 static int cmd_setattr (optparse_t *p, int ac, char *av[])
 {
@@ -28,25 +26,14 @@ static int cmd_setattr (optparse_t *p, int ac, char *av[])
 
     log_init ("flux-setattr");
 
-    if (optparse_hasopt (p, "expunge")) {
-        if (n != ac - 1) {
-            optparse_print_usage (p);
-            exit (1);
-        }
-        name = av[n];
-        if (flux_attr_set (h, name, NULL) < 0)
-            log_err_exit ("flux_attr_set");
-    }
-    else {
-        if (n != ac - 2) {
-            optparse_print_usage (p);
-            exit (1);
-        }
-        name = av[n];
-        val = av[n + 1];
-        if (flux_attr_set (h, name, val) < 0)
-            log_err_exit ("flux_attr_set");
+    if (n != ac - 2) {
+        optparse_print_usage (p);
+        exit (1);
     }
+    name = av[n];
+    val = av[n + 1];
+    if (flux_attr_set (h, name, val) < 0)
+        log_err_exit ("flux_attr_set");
 
     flux_close (h);
     return (0);
@@ -160,7 +147,7 @@ int subcommand_attr_register (optparse_t *p)
         "name value",
         "Set broker attribute",
         0,
-        setattr_opts);
+        NULL);
     if (e != OPTPARSE_SUCCESS)
         return (-1);
 
diff --git a/src/cmd/builtin/config.c b/src/cmd/builtin/config.c
new file mode 100644
index 000000000000..3ecc92b86fb8
--- /dev/null
+++ b/src/cmd/builtin/config.c
@@ -0,0 +1,370 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <jansson.h>
+
+#include "src/common/libutil/jpath.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/tomltk.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+
+#include "builtin.h"
+
+typedef enum {
+    FSD_NONE,
+    FSD_INTEGER,
+    FSD_REAL,
+    FSD_STRING,
+} fsd_subtype_t;
+
+struct map {
+    const char *s;
+    json_type type;
+    fsd_subtype_t fsd_subtype;
+};
+
+static struct map typemap[] = {
+    { "object", JSON_OBJECT, FSD_NONE },
+    { "array", JSON_ARRAY, FSD_NONE },
+    { "string", JSON_STRING, FSD_NONE },
+    { "integer", JSON_INTEGER, FSD_NONE },
+    { "real", JSON_REAL, FSD_NONE },
+    { "boolean", JSON_TRUE, FSD_NONE }, // special case
+    { "any", JSON_NULL, FSD_NONE },     // special case
+    { "fsd", JSON_STRING, FSD_STRING },
+    { "fsd-integer", JSON_STRING, FSD_INTEGER },
+    { "fsd-real", JSON_STRING, FSD_REAL },
+};
+
+static int parse_json_type (const char *s,
+                            json_type *type,
+                            fsd_subtype_t *fsd_subtype)
+{
+    for (int i = 0; i < ARRAY_SIZE (typemap); i++) {
+        if (!strcasecmp (s, typemap[i].s)) {
+            *type = typemap[i].type;
+            *fsd_subtype = typemap[i].fsd_subtype;
+            return 0;
+        }
+    }
+    return -1;
+}
+
+static void print_object (json_t *o)
+{
+    if (json_is_string (o))
+        printf ("%s\n", json_string_value (o));
+    else {
+        char *s;
+        if (!(s = json_dumps (o, JSON_COMPACT | JSON_ENCODE_ANY)))
+            log_msg_exit ("error encoding json object");
+        printf ("%s\n", s);
+        free (s);
+    }
+}
+
+static void print_config_item (json_t *o,
+                               const char *path,
+                               json_type want_type,
+                               fsd_subtype_t fsd_subtype,
+                               optparse_t *p)
+{
+    json_type type;
+    double t;
+
+    if (path) {
+        if (!(o = jpath_get (o, path))) {
+            if (errno == ENOENT) {
+                if (optparse_hasopt (p, "default")) {
+                    printf ("%s\n", optparse_get_str (p, "default", NULL));
+                    return;
+                }
+                if (optparse_hasopt (p, "quiet"))
+                    exit (1);
+                log_msg_exit ("%s is not set", path);
+            }
+            log_err_exit ("%s", path);
+        }
+    }
+    type = json_typeof (o);
+    if (want_type == JSON_NULL // any
+        || (want_type == JSON_TRUE && type == JSON_FALSE) // boolean
+        || (want_type == type && fsd_subtype == FSD_NONE)) {
+        print_object (o);
+    }
+    else if (want_type == type
+        && fsd_subtype != FSD_NONE
+        && fsd_parse_duration (json_string_value (o), &t) == 0) {
+        if (fsd_subtype == FSD_INTEGER)
+            printf ("%d\n", (int)t);
+        else if (fsd_subtype == FSD_REAL)
+            printf ("%f\n", t);
+        else
+            printf ("%s\n", json_string_value (o));
+    }
+    else
+        log_msg_exit ("%s does not have the requested type",
+                      path ? path : "value");
+}
+
+static int config_get (optparse_t *p, int ac, char *av[])
+{
+    int optindex = optparse_option_index (p);
+    flux_t *h = NULL;
+    flux_future_t *f = NULL;
+    const char *config_path = NULL;
+    flux_conf_t *conf = NULL;
+    json_t *o;
+    const char *path = NULL;
+    const char *typestr;
+    json_type type;
+    fsd_subtype_t fsd_subtype;
+
+    if (optindex < ac)
+        path = av[optindex++];
+    if (ac - optindex > 0) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if ((config_path = optparse_get_str (p, "config-path", NULL))) {
+        char buf[1024];
+        flux_error_t error;
+
+        if (streq (config_path, "system")
+            || streq (config_path, "security")
+            || streq (config_path, "imp")) {
+            snprintf (buf,
+                      sizeof (buf),
+                      "%s/%s/conf.d",
+                      FLUXCONFDIR,
+                      config_path);
+            config_path = buf;
+        }
+        if (!(conf = flux_conf_parse (config_path, &error))
+            || flux_conf_unpack (conf, &error, "o", &o) < 0)
+            log_msg_exit ("%s", error.text);
+    }
+    else {
+        if (!(h = builtin_get_flux_handle (p)))
+            log_err_exit ("flux_open");
+        if (!(f = flux_rpc (h, "config.get", NULL, FLUX_NODEID_ANY, 0))
+            || flux_rpc_get_unpack (f, "o", &o) < 0)
+            log_msg_exit ("Error fetching config object: %s",
+                          future_strerror (f, errno));
+    }
+    typestr = optparse_get_str (p, "type", "any");
+    if (parse_json_type (typestr, &type, &fsd_subtype) < 0)
+        log_msg_exit ("Unknown type: %s", typestr);
+
+    print_config_item (o,
+                       path,
+                       type,
+                       fsd_subtype,
+                       p);
+
+    flux_conf_decref (conf);
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+static int builtin_get (optparse_t *p, int ac, char *av[])
+{
+    int optindex = optparse_option_index (p);
+    const char *name;
+    const char *value;
+    int flags;
+
+    if (optindex != ac - 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (optparse_hasopt (p, "installed"))
+        flags = FLUX_CONF_INSTALLED;
+    else if (optparse_hasopt (p, "intree"))
+        flags = FLUX_CONF_INTREE;
+    else
+        flags = FLUX_CONF_AUTO;
+
+    name = av[optindex];
+    value = flux_conf_builtin_get (name, flags);
+    if (!value)
+        log_msg_exit ("%s is invalid", name);
+    printf ("%s\n", value);
+    return (0);
+}
+
+static int internal_config_reload (optparse_t *p, int ac, char *av[])
+{
+    flux_t *h;
+    flux_future_t *f = NULL;
+
+    if (optparse_option_index (p) != ac) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!(h = builtin_get_flux_handle (p)))
+        log_err_exit ("flux_open");
+    if (!(f = flux_rpc (h, "config.reload", NULL, FLUX_NODEID_ANY, 0)))
+        log_err_exit ("error constructing config.reload RPC");
+    if (flux_future_get (f, NULL) < 0)
+        log_msg_exit ("reload: %s", flux_future_error_string (f));
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+static int config_load (optparse_t *p, int ac, char *av[])
+{
+    int index = optparse_option_index (p);
+    const char *path = NULL;
+    json_t *obj;
+    flux_t *h;
+    flux_future_t *f;
+
+    if (index < ac)
+        path = av[index++];
+    if (index != ac) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (path) {
+        flux_conf_t *conf;
+        flux_error_t error;
+
+        if (!(conf = flux_conf_parse (path, &error))
+            || flux_conf_unpack (conf, &error, "O", &obj) < 0)
+            log_msg_exit ("Error parsing config: %s", error.text);
+        flux_conf_decref (conf);
+    }
+    else {
+        ssize_t n;
+        void *buf;
+
+        if ((n = read_all (STDIN_FILENO, &buf)) < 0)
+            log_err_exit ("error reading stdin");
+
+        if (!(obj = json_loads (buf, 0, NULL))) {
+            struct tomltk_error error;
+            toml_table_t *tab;
+
+            if (!(tab = tomltk_parse (buf, n, &error)))
+                log_msg_exit ("TOML parse error: %s", error.errbuf);
+            if (!(obj = tomltk_table_to_json (tab)))
+                log_err_exit ("error converting TOML to JSON");
+            toml_free (tab);
+        }
+        free (buf);
+    }
+    if (!(h = builtin_get_flux_handle (p)))
+        log_err_exit ("flux_open");
+    if (!(f = flux_rpc_pack (h, "config.load", FLUX_NODEID_ANY, 0, "O", obj)))
+        log_err_exit ("error constructing config.load RPC");
+    if (flux_future_get (f, NULL) < 0)
+        log_msg_exit ("load: %s", flux_future_error_string (f));
+    flux_future_destroy (f);
+    flux_close (h);
+
+    json_decref (obj);
+    return 0;
+}
+
+int cmd_config (optparse_t *p, int ac, char *av[])
+{
+    log_init ("flux-config");
+
+    if (optparse_run_subcommand (p, ac, av) != OPTPARSE_SUCCESS)
+        exit (1);
+    return (0);
+}
+
+static struct optparse_option get_opts[] = {
+    { .name = "config-path", .key = 'c', .has_arg = 1,
+      .arginfo = "PATH|system|security|imp",
+      .usage = "Get broker config from PATH (default: use live config)"
+    },
+    { .name = "type", .key = 't', .has_arg = 1, .arginfo = "TYPE",
+      .usage = "Set expected type (any, string, integer, real, boolean"
+          ", object, array, fsd, fsd-integer, fsd-real)",
+    },
+    { .name = "quiet", .key = 'q', .has_arg = 0,
+      .usage = "Suppress printing of \"[key] is not set\" errors."
+    },
+    { .name = "default", .key = 'd', .has_arg = 1, .arginfo = "VAL",
+      .usage = "Use this value if config key is unset"
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option builtin_opts[] = {
+    { .name = "intree", .has_arg = 0,
+      .usage = "Force in-tree paths to be used"
+    },
+    { .name = "installed", .has_arg = 0,
+      .usage = "Force installed paths to be used",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_subcommand config_subcmds[] = {
+    { "load",
+      "[PATH]",
+      "Load broker configuration from stdin or PATH",
+      config_load,
+      0,
+      NULL,
+    },
+    { "reload",
+      "[OPTIONS]",
+      "Reload broker configuration from files",
+      internal_config_reload,
+      0,
+      NULL,
+    },
+    { "get",
+      "[OPTIONS] [NAME]",
+      "Query broker configuration values",
+      config_get,
+      0,
+      get_opts,
+    },
+    { "builtin",
+      "NAME",
+      "Print compiled-in Flux configuration values",
+      builtin_get,
+      0,
+      builtin_opts,
+    },
+    OPTPARSE_SUBCMD_END
+};
+
+int subcommand_config_register (optparse_t *p)
+{
+    optparse_err_t e;
+
+    e = optparse_reg_subcommand (p,
+            "config", cmd_config, NULL, "Manage configuration", 0, NULL);
+    if (e != OPTPARSE_SUCCESS)
+        return (-1);
+
+    e = optparse_reg_subcommands (optparse_get_subcommand (p, "config"),
+                                  config_subcmds);
+    return (e == OPTPARSE_SUCCESS ? 0 : -1);
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/builtin/content.c b/src/cmd/builtin/content.c
index 8515672479ce..237e1e8f1fcc 100644
--- a/src/cmd/builtin/content.c
+++ b/src/cmd/builtin/content.c
@@ -8,54 +8,112 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include "builtin.h"
 
 #include <unistd.h>
 
 #include "src/common/libutil/blobref.h"
 #include "src/common/libutil/read_all.h"
+#include "src/common/libcontent/content.h"
+
+static void load_to_fd (flux_t *h, int fd, const char *blobref, int flags)
+{
+    flux_future_t *f;
+    const uint8_t *data;
+    size_t size;
+
+    if (!(f = content_load_byblobref (h, blobref, flags))
+        || content_load_get (f, (const void **)&data, &size) < 0)
+        log_msg_exit ("error loading blob: %s", future_strerror (f, errno));
+    if (write_all (fd, data, size) < 0)
+        log_err_exit ("write");
+    flux_future_destroy (f);
+}
+
+static void store_blob (flux_t *h,
+                        const char *hash_type,
+                        const void *data,
+                        size_t size,
+                        int flags)
+{
+    flux_future_t *f;
+    const char *blobref;
+
+    if (!(f = content_store (h, data, size, flags))
+        || content_store_get_blobref (f, hash_type, &blobref) < 0)
+        log_msg_exit ("error storing blob: %s", future_strerror (f, errno));
+    printf ("%s\n", blobref);
+    flux_future_destroy (f);
+}
+
+static void store_from_fd (flux_t *h,
+                           const char *hash_type,
+                           int fd,
+                           size_t chunksize,
+                           int flags)
+{
+    void *data;
+    ssize_t total_size;
+
+    if ((total_size = read_all (fd, &data)) < 0)
+        log_err_exit ("read from stdin");
+    if (chunksize == 0)
+        chunksize = total_size;
+    if (total_size == 0) // an empty blob is still valid
+        store_blob (h, hash_type, data, total_size, flags);
+    else {
+        for (off_t offset = 0; offset < total_size; offset += chunksize) {
+            if (chunksize > total_size - offset)
+                chunksize = total_size - offset;
+            store_blob (h, hash_type, data + offset, chunksize, flags);
+        }
+    }
+    free (data);
+}
 
 static int internal_content_load (optparse_t *p, int ac, char *av[])
 {
     int n;
-    const char *ref;
-    const uint8_t *data;
-    int size;
     flux_t *h;
-    flux_future_t *f;
     int flags = 0;
 
-    n = optparse_option_index (p);
-    if (n != ac - 1) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    ref = av[n];
     if (!(h = builtin_get_flux_handle (p)))
         log_err_exit ("flux_open");
     if (optparse_hasopt (p, "bypass-cache"))
         flags |= CONTENT_FLAG_CACHE_BYPASS;
-    if (!(f = flux_content_load (h, ref, flags)))
-        log_err_exit ("flux_content_load");
-    if (flux_content_load_get (f, (const void **)&data, &size) < 0)
-        log_err_exit ("flux_content_load_get");
-    if (write_all (STDOUT_FILENO, data, size) < 0)
-        log_err_exit ("write");
-    flux_future_destroy (f);
+
+    n = optparse_option_index (p);
+    if (n == ac) {
+        char blobref[BLOBREF_MAX_STRING_SIZE];
+        int count = 0;
+        while ((fgets (blobref, sizeof (blobref), stdin))) {
+            int len = strlen (blobref);
+            if (blobref[len - 1] == '\n')
+                blobref[len - 1] = '\0';
+            load_to_fd (h, STDOUT_FILENO, blobref, flags);
+            count++;
+        }
+        if (count == 0)
+            log_msg_exit ("no blobrefs were specified");
+    }
+    else while (n < ac) {
+        load_to_fd (h, STDOUT_FILENO, av[n++], flags);
+    }
     flux_close (h);
     return (0);
 }
 
 static int internal_content_store (optparse_t *p, int ac, char *av[])
 {
-    uint8_t *data;
-    int size;
     flux_t *h;
-    flux_future_t *f;
-    const char *blobref;
     int flags = 0;
+    int chunksize = optparse_get_int (p, "chunksize", 0);
+    const char *hash_type;
 
-    if (optparse_option_index (p)  != ac) {
+    if (optparse_option_index (p) != ac || chunksize < 0) {
         optparse_print_usage (p);
         exit (1);
     }
@@ -63,16 +121,10 @@ static int internal_content_store (optparse_t *p, int ac, char *av[])
         flags |= CONTENT_FLAG_CACHE_BYPASS;
     if (!(h = builtin_get_flux_handle (p)))
         log_err_exit ("flux_open");
-    if ((size = read_all (STDIN_FILENO, (void **)&data)) < 0)
-        log_err_exit ("read");
-    if (!(f = flux_content_store (h, data, size, flags)))
-        log_err_exit ("flux_content_store");
-    if (flux_content_store_get (f, &blobref) < 0)
-        log_err_exit ("flux_content_store_get");
-    printf ("%s\n", blobref);
-    flux_future_destroy (f);
+    if (!(hash_type = flux_attr_get (h, "content.hash")))
+        log_err_exit ("getattr content.hash");
+    store_from_fd (h, hash_type, STDIN_FILENO, chunksize, flags);
     flux_close (h);
-    free (data);
     return (0);
 }
 
@@ -116,61 +168,6 @@ static int internal_content_dropcache (optparse_t *p, int ac, char *av[])
     return (0);
 }
 
-static int spam_max_inflight;
-static int spam_cur_inflight;
-
-static void store_completion (flux_future_t *f, void *arg)
-{
-    flux_t *h = arg;
-    const char *blobref;
-
-    if (flux_content_store_get (f, &blobref) < 0)
-        log_err_exit ("store");
-    printf ("%s\n", blobref);
-    flux_future_destroy (f);
-    if (--spam_cur_inflight < spam_max_inflight/2)
-        flux_reactor_stop (flux_get_reactor (h));
-}
-
-static int internal_content_spam (optparse_t *p, int ac, char *av[])
-{
-    int i, count;
-    flux_future_t *f;
-    flux_t *h;
-    char data[256];
-    int size = 256;
-
-    if (ac != 2 && ac != 3) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    count = strtoul (av[1], NULL, 10);
-    if (ac == 3)
-        spam_max_inflight = strtoul (av[2], NULL, 10);
-    else
-        spam_max_inflight = 1;
-
-    if (!(h = builtin_get_flux_handle (p)))
-        log_err_exit ("flux_open");
-
-    spam_cur_inflight = 0;
-    i = 0;
-    while (i < count || spam_cur_inflight > 0) {
-        while (i < count && spam_cur_inflight < spam_max_inflight) {
-            snprintf (data, size, "spam-o-matic pid=%d seq=%d", getpid(), i);
-            if (!(f = flux_content_store (h, data, size, 0)))
-                log_err_exit ("flux_content_store(%d)", i);
-            if (flux_future_then (f, -1., store_completion, h) < 0)
-                log_err_exit ("flux_future_then(%d)", i);
-            spam_cur_inflight++;
-            i++;
-        }
-        if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
-            log_err ("flux_reactor_run");
-    }
-    return (0);
-}
-
 int cmd_content (optparse_t *p, int ac, char *av[])
 {
     log_init ("flux-content");
@@ -189,20 +186,22 @@ static struct optparse_option load_opts[] = {
 static struct optparse_option store_opts[] = {
     { .name = "bypass-cache",  .key = 'b',  .has_arg = 0,
       .usage = "Store directly to rank 0 content service", },
-      OPTPARSE_TABLE_END,
+    { .name = "chunksize", .has_arg = 1, .arginfo = "N",
+      .usage = "Limit blob size to N bytes with 0=unlimited (default 0)", },
+      OPTPARSE_TABLE_END
 };
 
 static struct optparse_subcommand content_subcmds[] = {
     { "load",
-      "[OPTIONS] BLOBREF",
-      "Load blob for digest BLOBREF to stdout",
+      "[OPTIONS] BLOBREF ...",
+      "Concatenate blobs stored under BLOBREF(s) to stdout",
       internal_content_load,
       0,
       load_opts,
     },
     { "store",
       "[OPTIONS]",
-      "Store blob from stdin, print BLOBREF on stdout",
+      "Store blob from stdin, print BLOBREF(s) on stdout",
       internal_content_store,
       0,
       store_opts,
@@ -221,13 +220,6 @@ static struct optparse_subcommand content_subcmds[] = {
       0,
       NULL,
     },
-    { "spam",
-      "N [M]",
-      "Store N random entries, keeping M requests in flight (default 1)",
-      internal_content_spam,
-      0,
-      NULL,
-    },
     OPTPARSE_SUBCMD_END
 };
 
diff --git a/src/cmd/builtin/dmesg.c b/src/cmd/builtin/dmesg.c
index 67f2b8e5bc97..deb5d2f8e896 100644
--- a/src/cmd/builtin/dmesg.c
+++ b/src/cmd/builtin/dmesg.c
@@ -8,7 +8,50 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include "builtin.h"
+#include <inttypes.h>
+#include <unistd.h>
+
+#include "src/common/libutil/stdlog.h"
+#include "src/common/libutil/timestamp.h"
+#include "src/common/libutil/ansi_color.h"
+#include "ccan/str/str.h"
+
+struct dmesg_ctx {
+    optparse_t *p;
+    unsigned int color:1;
+    unsigned int delta:1;
+    struct tm last_tm;
+    struct timeval last_tv;
+};
+
+enum {
+    DMESG_COLOR_NAME,
+    DMESG_COLOR_TIME,
+    DMESG_COLOR_TIMEBREAK,
+    DMESG_COLOR_ALERT,
+    DMESG_COLOR_EMERG,
+    DMESG_COLOR_CRIT,
+    DMESG_COLOR_ERR,
+    DMESG_COLOR_WARNING,
+    DMESG_COLOR_DEBUG,
+};
+
+static const char *dmesg_colors[] = {
+    [DMESG_COLOR_NAME]          = ANSI_COLOR_YELLOW,
+    [DMESG_COLOR_TIME]          = ANSI_COLOR_GREEN,
+    [DMESG_COLOR_TIMEBREAK]     = ANSI_COLOR_BOLD ANSI_COLOR_GREEN,
+    [DMESG_COLOR_ALERT]         = ANSI_COLOR_REVERSE ANSI_COLOR_RED,
+    [DMESG_COLOR_EMERG]         = ANSI_COLOR_REVERSE ANSI_COLOR_RED,
+    [DMESG_COLOR_CRIT]          = ANSI_COLOR_BOLD ANSI_COLOR_RED,
+    [DMESG_COLOR_ERR]           = ANSI_COLOR_RED,
+    [DMESG_COLOR_WARNING]       = ANSI_COLOR_BOLD,
+    [DMESG_COLOR_DEBUG]         = ANSI_COLOR_BLUE,
+};
+
 
 static struct optparse_option dmesg_opts[] = {
     { .name = "clear",  .key = 'C',  .has_arg = 0,
@@ -17,30 +60,255 @@ static struct optparse_option dmesg_opts[] = {
       .usage = "Clear the ring buffer contents after printing", },
     { .name = "follow",  .key = 'f',  .has_arg = 0,
       .usage = "Track new entries as are logged", },
+    { .name = "new",  .key = 'n',  .has_arg = 0,
+      .usage = "Show only new log messages", },
+    { .name = "human",  .key = 'H',  .has_arg = 0,
+      .usage = "Human readable output", },
+    { .name = "delta",  .key = 'd',  .has_arg = 0,
+      .usage = "With --human, show timestamp delta between messages", },
+    { .name = "color", .key = 'L', .has_arg = 2, .arginfo = "WHEN",
+      .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
+      .usage = "Colorize output when supported; WHEN can be 'always' "
+               "(default if omitted), 'never', or 'auto' (default)." },
     OPTPARSE_TABLE_END,
 };
 
+static const char *dmesg_color (struct dmesg_ctx *ctx, int type)
+{
+    if (ctx->color)
+        return dmesg_colors [type];
+    return "";
+}
+
+static const char *dmesg_color_reset (struct dmesg_ctx *ctx)
+{
+    if (ctx->color)
+        return ANSI_COLOR_RESET;
+    return "";
+}
+
+static double tv_to_double (struct timeval *tv)
+{
+    return (tv->tv_sec + (tv->tv_usec/1e6));
+}
+
+static const char *months[] = {
+    "Jan",
+    "Feb",
+    "Mar",
+    "Apr",
+    "May",
+    "Jun",
+    "Jul",
+    "Aug",
+    "Sep",
+    "Oct",
+    "Nov",
+    "Dec",
+    NULL
+};
+
+void print_iso_timestamp (struct dmesg_ctx *ctx, struct stdlog_header *hdr)
+{
+    struct tm tm;
+    struct timeval tv;
+    char buf[128];
+    char tz[16];
+    int len = sizeof (buf);
+
+    /* Fall back to using the hdr timestamp string if
+     * - the timestamp fails to parse
+     * - getting current timezone offset fails
+     * - timestamp_tzoffset() returns "Z" (hdr timestamp already in Zulu time)
+     */
+    if (timestamp_parse (hdr->timestamp, &tm, &tv) < 0
+        || strftime (buf, len, "%Y-%m-%dT%T", &tm) == 0
+        || timestamp_tzoffset (&tm, tz, sizeof (tz)) < 0
+        || streq (tz, "Z")) {
+        printf ("%s%s%s ",
+                dmesg_color (ctx, DMESG_COLOR_TIME),
+                hdr->timestamp,
+                dmesg_color_reset (ctx));
+        return;
+    }
+    printf ("%s%s.%.6lu%s%s ",
+            dmesg_color (ctx, DMESG_COLOR_TIME),
+            buf,
+            (unsigned long)tv.tv_usec,
+            tz,
+            dmesg_color_reset (ctx));
+}
+
+void print_human_timestamp (struct dmesg_ctx *ctx, struct stdlog_header *hdr)
+{
+    struct tm tm;
+    struct timeval tv;
+    if (timestamp_parse (hdr->timestamp, &tm, &tv) < 0) {
+        printf ("%s[%s]%s ",
+                dmesg_color (ctx, DMESG_COLOR_TIME),
+                hdr->timestamp,
+                dmesg_color_reset (ctx));
+    }
+    if (tm.tm_year == ctx->last_tm.tm_year
+        && tm.tm_mon == ctx->last_tm.tm_mon
+        && tm.tm_mday == ctx->last_tm.tm_mday
+        && tm.tm_hour == ctx->last_tm.tm_hour
+        && tm.tm_min == ctx->last_tm.tm_min) {
+        /*  Within same minute, print offset in sec */
+        double dt = tv_to_double (&tv) - tv_to_double (&ctx->last_tv);
+        printf ("%s[%+11.6f]%s ",
+                dmesg_color (ctx, DMESG_COLOR_TIME),
+                dt,
+                dmesg_color_reset (ctx));
+        if (ctx->delta)
+            ctx->last_tv = tv;
+    }
+    else {
+        /* New minute, print datetime */
+        printf ("%s[%s%02d %02d:%02d]%s ",
+                dmesg_color (ctx, DMESG_COLOR_TIMEBREAK),
+                months [tm.tm_mon],
+                tm.tm_mday,
+                tm.tm_hour,
+                tm.tm_min,
+                dmesg_color_reset (ctx));
+        ctx->last_tv = tv;
+        ctx->last_tm = tm;
+    }
+}
+
+static const char *severity_color (struct dmesg_ctx *ctx, int severity)
+{
+    switch (severity) {
+        case LOG_EMERG:
+            return dmesg_color (ctx, DMESG_COLOR_EMERG);
+        case LOG_ALERT:
+            return dmesg_color (ctx, DMESG_COLOR_ALERT);
+        case LOG_CRIT:
+            return dmesg_color (ctx, DMESG_COLOR_CRIT);
+        case LOG_ERR:
+            return dmesg_color (ctx, DMESG_COLOR_ERR);
+        case LOG_WARNING:
+            return dmesg_color (ctx, DMESG_COLOR_WARNING);
+        case LOG_NOTICE:
+        case LOG_INFO:
+            return "";
+        case LOG_DEBUG:
+            return dmesg_color (ctx, DMESG_COLOR_DEBUG);
+    }
+    return "";
+}
+
+typedef void (*timestamp_print_f) (struct dmesg_ctx *ctx,
+                                   struct stdlog_header *hdr);
+
+void dmesg_print (struct dmesg_ctx *ctx,
+                  const char *buf,
+                  int len,
+                  timestamp_print_f timestamp_print)
+{
+    struct stdlog_header hdr;
+    const char *msg;
+    size_t msglen;
+    int severity;
+    uint32_t nodeid;
+
+    if (stdlog_decode (buf, len, &hdr, NULL, NULL, &msg, &msglen) < 0)
+        printf ("%.*s\n", len, buf);
+    else {
+        nodeid = strtoul (hdr.hostname, NULL, 10);
+        severity = STDLOG_SEVERITY (hdr.pri);
+        (*timestamp_print) (ctx, &hdr);
+        printf ("%s%s.%s[%" PRIu32 "]%s: %s%.*s%s\n",
+                dmesg_color (ctx, DMESG_COLOR_NAME),
+                hdr.appname,
+                stdlog_severity_to_string (severity),
+                nodeid,
+                dmesg_color_reset (ctx),
+                severity_color (ctx, severity),
+                (int)msglen,
+                msg,
+                dmesg_color_reset (ctx));
+    }
+    fflush (stdout);
+}
+
+static void dmesg_colors_init (struct dmesg_ctx *ctx)
+{
+    const char *when;
+
+    if (!(when = optparse_get_str (ctx->p, "color", "auto")))
+        when = "always";
+    if (streq (when, "always"))
+        ctx->color = 1;
+    else if (streq (when, "never"))
+        ctx->color = 0;
+    else if (streq (when, "auto"))
+        ctx->color = isatty (STDOUT_FILENO) ? 1 : 0;
+    else
+        log_msg_exit ("Invalid argument to --color: '%s'", when);
+}
+
+static void dmesg_ctx_init (struct dmesg_ctx *ctx, optparse_t *p)
+{
+    memset (ctx, 0, sizeof (*ctx));
+    ctx->p = p;
+    dmesg_colors_init (ctx);
+    if (optparse_hasopt (p, "delta")) {
+        if (!optparse_hasopt (p, "human"))
+            log_msg_exit ("--delta can only be used with --human");
+        ctx->delta = 1;
+    }
+}
 
 static int cmd_dmesg (optparse_t *p, int ac, char *av[])
 {
     int n;
     flux_t *h;
-    int flags = 0;
-    flux_log_f print_cb = flux_log_fprint;
+    struct dmesg_ctx ctx;
+
+    tzset ();
 
+    log_init ("flux-dmesg");
     if ((n = optparse_option_index (p)) != ac)
         log_msg_exit ("flux-dmesg accepts no free arguments");
 
-    if (!(h = builtin_get_flux_handle (p)))
-        log_err_exit ("flux_open");
-    if (optparse_hasopt (p, "read-clear") || optparse_hasopt (p, "clear"))
-        flags |= FLUX_DMESG_CLEAR;
-    if (optparse_hasopt (p, "clear"))
-        print_cb = NULL;
-    if (optparse_hasopt (p, "follow"))
-        flags |= FLUX_DMESG_FOLLOW;
-    if (flux_dmesg (h, flags, print_cb, stdout) < 0)
-        log_err_exit ("flux_dmesg");
+    dmesg_ctx_init (&ctx, p);
+
+    h = builtin_get_flux_handle (p);
+
+    if (!optparse_hasopt (p, "clear")) {
+        flux_future_t *f;
+        const char *buf;
+
+        if (!(f = flux_rpc_pack (h,
+                                 "log.dmesg",
+                                 FLUX_NODEID_ANY,
+                                 FLUX_RPC_STREAMING,
+                                 "{s:b s:b}",
+                                 "follow", optparse_hasopt (p, "follow"),
+                                 "nobacklog", optparse_hasopt (p, "new"))))
+            log_err_exit ("error sending log.dmesg request");
+        while (flux_rpc_get (f, &buf) == 0) {
+            timestamp_print_f ts_print = print_iso_timestamp;
+            if (optparse_hasopt (p, "human"))
+                ts_print = print_human_timestamp;
+            dmesg_print (&ctx, buf, strlen (buf), ts_print);
+            flux_future_reset (f);
+        }
+        if (errno != ENODATA)
+            log_msg_exit ("log.dmesg: %s", future_strerror (f, errno));
+        flux_future_destroy (f);
+    }
+    if (optparse_hasopt (p, "read-clear") || optparse_hasopt (p, "clear")) {
+        flux_future_t *f;
+
+        if (!(f = flux_rpc (h, "log.clear", NULL, FLUX_NODEID_ANY, 0))
+            || flux_future_get (f, NULL) < 0)
+            log_msg_exit ("log.clear: %s", future_strerror (f, errno));
+        flux_future_destroy (f);
+    }
+
     flux_close (h);
     return (0);
 }
diff --git a/src/cmd/builtin/dump.c b/src/cmd/builtin/dump.c
new file mode 100644
index 000000000000..9266abe0b7f5
--- /dev/null
+++ b/src/cmd/builtin/dump.c
@@ -0,0 +1,494 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+#if HAVE_LIBSYSTEMD
+#include <systemd/sd-daemon.h>
+#endif
+#include <unistd.h>
+#include <stdarg.h>
+#include <jansson.h>
+#include <time.h>
+#include <stdarg.h>
+#include <archive.h>
+#include <archive_entry.h>
+
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libkvs/treeobj.h"
+#include "src/common/libkvs/kvs_checkpoint.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/blobref.h"
+#include "src/common/libcontent/content.h"
+#include "ccan/str/str.h"
+
+#include "builtin.h"
+
+static void dump_treeobj (struct archive *ar,
+                          flux_t *h,
+                          const char *path,
+                          json_t *treeobj);
+
+static bool sd_notify_flag;
+static bool verbose;
+static bool quiet;
+static bool ignore_failed_read;
+static int content_flags;
+static time_t dump_time;
+static gid_t dump_gid;
+static uid_t dump_uid;
+static int keycount;
+
+static void read_verror (const char *fmt, va_list ap)
+{
+    char buf[128];
+    vsnprintf (buf, sizeof (buf), fmt, ap);
+    fprintf (stderr, "%s\n", buf);
+    if (!ignore_failed_read)
+        exit (1);
+}
+
+static __attribute__ ((format (printf, 1, 2)))
+void read_error (const char *fmt, ...)
+{
+    va_list ap;
+    va_start (ap, fmt);
+    read_verror (fmt, ap);
+    va_end (ap);
+}
+
+static void progress (int delta_keys)
+{
+    keycount += delta_keys;
+
+    if (!verbose
+        && !quiet
+        && (keycount % 100 == 0 || keycount < 10))
+        fprintf (stderr, "\rflux-dump: archived %d keys", keycount);
+#if HAVE_LIBSYSTEMD
+    if (sd_notify_flag
+        && (keycount % 100 == 0 || keycount < 10)) {
+        sd_notifyf (0, "EXTEND_TIMEOUT_USEC=%d", 10000000); // 10s
+        sd_notifyf (0, "STATUS=flux-dump(1) has archived %d keys", keycount);
+    }
+#endif
+}
+
+static void progress_end (void)
+{
+    if (!quiet && !verbose)
+        fprintf (stderr, "\rflux-dump: archived %d keys\n", keycount);
+#if HAVE_LIBSYSTEMD
+    if (sd_notify_flag) {
+        sd_notifyf (0, "STATUS=flux-dump(1) has archived %d keys", keycount);
+    }
+#endif
+}
+
+static struct archive *dump_create (const char *outfile)
+{
+    struct archive *ar;
+
+    if (!(ar = archive_write_new ()))
+        log_msg_exit ("error creating libarchive write context");
+    if (streq (outfile, "-")) {
+        if (archive_write_set_format_pax_restricted (ar) != ARCHIVE_OK
+            || archive_write_open_FILE (ar, stdout) != ARCHIVE_OK)
+            log_msg_exit ("%s", archive_error_string (ar));
+    }
+    else {
+#if ARCHIVE_VERSION_NUMBER < 3002000
+        /* El7 has libarchive 3.1.2 (circa 2013), but
+         * archive_write_set_format_filter_by_ext() appeared in 3.2.0.
+         * Just force tar format / no compression on that platform.
+         */
+        if (archive_write_set_format_pax_restricted (ar) != ARCHIVE_OK
+#else
+        if (archive_write_set_format_filter_by_ext (ar, outfile) != ARCHIVE_OK
+#endif
+            || archive_write_open_filename (ar, outfile) != ARCHIVE_OK)
+            log_msg_exit ("%s", archive_error_string (ar));
+    }
+    return ar;
+}
+
+static void dump_destroy (struct archive *ar)
+{
+    if (archive_write_close (ar) != ARCHIVE_OK)
+        log_msg_exit ("%s", archive_error_string (ar));
+    archive_write_free (ar);
+}
+
+/* From archive_write_data(3):
+ *   Clients should treat any value less than zero as an error and consider
+ *   any non-negative value as success.
+ */
+static void dump_write_data (struct archive *ar, const void *data, int size)
+{
+    int n;
+
+    n = archive_write_data (ar, data, size);
+    if (n < 0)
+        log_msg_exit ("%s", archive_error_string (ar));
+    if (n != size)
+        log_msg ("short write to archive: %s",
+                 "assuming non-fatal libarchive write size reporting error");
+}
+
+static void dump_valref (struct archive *ar,
+                         flux_t *h,
+                         const char *path,
+                         json_t *treeobj)
+{
+    int count = treeobj_get_count (treeobj);
+    struct flux_msglist *l;
+    const flux_msg_t *msg;
+    int total_size = 0;
+    struct archive_entry *entry;
+    const void *data;
+    size_t len;
+
+    /* We need the total size before we start writing archive data,
+     * so make a first pass, saving the data for writing later.
+     */
+    /* N.B. first attempt was to save the futures in an array, but ran into:
+     *   flux: ev_epoll.c:134: epoll_modify: Assertion `("libev: I/O watcher
+     *    with invalid fd found in epoll_ctl", errno != EBADF && errno != ELOOP
+     *    && errno != EINVAL)' failed.
+     * while archiving a resource.eventlog with 781 entries.  Instead of
+     * retaining the futures for a second pass, just retain references to the
+     * content.load response messages.
+     */
+    if (!(l = flux_msglist_create ()))
+        log_err_exit ("could not create message list");
+    for (int i = 0; i < count; i++) {
+        flux_future_t *f;
+        if (!(f = content_load_byblobref (h,
+                                          treeobj_get_blobref (treeobj, i),
+                                          content_flags))
+            || flux_future_get (f, (const void **)&msg) < 0
+            || flux_response_decode_raw (msg, NULL, &data, &len) < 0) {
+            read_error ("%s: missing blobref %d: %s",
+                        path,
+                        i,
+                        future_strerror (f, errno));
+            flux_future_destroy (f);
+            flux_msglist_destroy (l);
+            return;
+        }
+        if (flux_msglist_append (l, msg) < 0)
+            log_err_exit ("could not stash load response message");
+        total_size += len;
+        flux_future_destroy (f);
+    }
+    if (!(entry = archive_entry_new ()))
+        log_msg_exit ("error creating archive entry");
+    archive_entry_set_pathname (entry, path);
+    archive_entry_set_size (entry, total_size);
+    archive_entry_set_perm (entry, 0644);
+    archive_entry_set_filetype (entry, AE_IFREG);
+    archive_entry_set_mtime (entry, dump_time, 0);
+    archive_entry_set_uid (entry, dump_uid);
+    archive_entry_set_gid (entry, dump_gid);
+
+    if (archive_write_header (ar, entry) != ARCHIVE_OK)
+        log_msg_exit ("%s", archive_error_string (ar));
+    while ((msg = flux_msglist_pop (l))) {
+        if (flux_response_decode_raw (msg, NULL, &data, &len) < 0)
+            log_err_exit ("error processing stashed valref responses");
+        if (len > 0)
+            dump_write_data (ar, data, len);
+        flux_msg_decref (msg);
+    }
+    archive_entry_free (entry);
+    progress (1);
+    flux_msglist_destroy (l);
+}
+
+static void dump_val (struct archive *ar,
+                      flux_t *h,
+                      const char *path,
+                      json_t *treeobj)
+{
+    struct archive_entry *entry;
+    void *data;
+    size_t len;
+
+    if (treeobj_decode_val (treeobj, &data, &len) < 0)
+        log_err_exit ("%s: invalid value object", path);
+    if (!(entry = archive_entry_new ()))
+        log_msg_exit ("error creating archive entry");
+    archive_entry_set_pathname (entry, path);
+    archive_entry_set_size (entry, len);
+    archive_entry_set_perm (entry, 0644);
+    archive_entry_set_filetype (entry, AE_IFREG);
+
+    if (archive_write_header (ar, entry) != ARCHIVE_OK)
+        log_msg_exit ("%s", archive_error_string (ar));
+    dump_write_data (ar, data, len);
+    progress (1);
+
+    archive_entry_free (entry);
+    free (data);
+}
+
+static void dump_symlink (struct archive *ar,
+                          flux_t *h,
+                          const char *path,
+                          json_t *treeobj)
+{
+    struct archive_entry *entry;
+    const char *ns;
+    const char *target;
+    char *target_with_ns = NULL;
+
+    if (treeobj_get_symlink (treeobj, &ns, &target) < 0)
+        log_err_exit ("%s: invalid symlink object", path);
+    if (ns) {
+        if (asprintf (&target_with_ns, "%s::%s", ns, target) < 0)
+            log_msg_exit ("out of memory");
+        target = target_with_ns;
+    }
+    if (!(entry = archive_entry_new ()))
+        log_msg_exit ("error creating archive entry");
+    archive_entry_set_pathname (entry, path);
+    archive_entry_set_perm (entry, 0644);
+    archive_entry_set_filetype (entry, AE_IFLNK);
+    archive_entry_set_symlink (entry, target);
+    if (archive_write_header (ar, entry) != ARCHIVE_OK)
+        log_msg_exit ("%s", archive_error_string (ar));
+    progress (1);
+
+    free (target_with_ns);
+    archive_entry_free (entry);
+}
+
+static void dump_dir (struct archive *ar,
+                      flux_t *h,
+                      const char *path,
+                      json_t *treeobj)
+{
+    json_t *dict = treeobj_get_data (treeobj);
+    const char *name;
+    json_t *entry;
+
+    json_object_foreach (dict, name, entry) {
+        char *newpath;
+        if (asprintf (&newpath, "%s/%s", path, name) < 0)
+            log_msg_exit ("out of memory");
+        dump_treeobj (ar, h, newpath, entry); // recurse
+        free (newpath);
+    }
+}
+
+static void dump_dirref (struct archive *ar,
+                         flux_t *h,
+                         const char *path,
+                         json_t *treeobj)
+{
+    flux_future_t *f;
+    const void *buf;
+    size_t buflen;
+    json_t *treeobj_deref = NULL;
+
+    if (treeobj_get_count (treeobj) != 1)
+        log_msg_exit ("%s: blobref count is not 1", path);
+    if (!(f = content_load_byblobref (h,
+                                      treeobj_get_blobref (treeobj, 0),
+                                      content_flags))
+        || content_load_get (f, &buf, &buflen) < 0) {
+        read_error ("%s: missing blobref: %s",
+                    path,
+                    future_strerror (f, errno));
+        flux_future_destroy (f);
+        return;
+    }
+    if (!(treeobj_deref = treeobj_decodeb (buf, buflen)))
+        log_err_exit ("%s: could not decode directory", path);
+    if (!treeobj_is_dir (treeobj_deref))
+        log_msg_exit ("%s: dirref references non-directory", path);
+    dump_dir (ar, h, path, treeobj_deref); // recurse
+    json_decref (treeobj_deref);
+    flux_future_destroy (f);
+}
+
+static void dump_treeobj (struct archive *ar,
+                          flux_t *h,
+                          const char *path,
+                          json_t *treeobj)
+{
+    if (treeobj_validate (treeobj) < 0)
+        log_msg_exit ("%s: invalid tree object", path);
+    if (treeobj_is_symlink (treeobj)) {
+        if (verbose)
+            fprintf (stderr, "%s\n", path);
+        dump_symlink (ar, h, path, treeobj);
+    }
+    else if (treeobj_is_val (treeobj)) {
+        if (verbose)
+            fprintf (stderr, "%s\n", path);
+        dump_val (ar, h, path, treeobj);
+    }
+    else if (treeobj_is_valref (treeobj)) {
+        if (verbose)
+            fprintf (stderr, "%s\n", path);
+        dump_valref (ar, h, path, treeobj);
+    }
+    else if (treeobj_is_dirref (treeobj)) {
+        dump_dirref (ar, h, path, treeobj); // recurse
+    }
+    else if (treeobj_is_dir (treeobj)) {
+        dump_dir (ar, h, path, treeobj); // recurse
+    }
+}
+
+static void dump_blobref (struct archive *ar,
+                          flux_t *h,
+                          const char *blobref)
+{
+    flux_future_t *f;
+    const void *buf;
+    size_t buflen;
+    json_t *treeobj;
+    json_t *dict;
+    const char *key;
+    json_t *entry;
+
+    if (!(f = content_load_byblobref (h, blobref, content_flags))
+        || content_load_get (f, &buf, &buflen) < 0) {
+        read_error ("cannot load root tree object: %s",
+                    future_strerror (f, errno));
+        flux_future_destroy (f);
+        return;
+    }
+    if (!(treeobj = treeobj_decodeb (buf, buflen)))
+        log_err_exit ("cannot decode root tree object");
+    if (treeobj_validate (treeobj) < 0)
+        log_msg_exit ("invalid root tree object");
+    if (!treeobj_is_dir (treeobj))
+        log_msg_exit ("root tree object is not a directory");
+
+    dict = treeobj_get_data (treeobj);
+    json_object_foreach (dict, key, entry) {
+        dump_treeobj (ar, h, key, entry);
+    }
+    json_decref (treeobj);
+    flux_future_destroy (f);
+}
+
+static int cmd_dump (optparse_t *p, int ac, char *av[])
+{
+    int optindex =  optparse_option_index (p);
+    flux_t *h;
+    struct archive *ar;
+    const char *outfile;
+    int kvs_checkpoint_flags = 0;
+
+    log_init ("flux-dump");
+
+    if (optindex != ac - 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    outfile = av[optindex++];
+    if (optparse_hasopt (p, "verbose"))
+        verbose = true;
+    if (optparse_hasopt (p, "quiet"))
+        quiet = true;
+    if (optparse_hasopt (p, "ignore-failed-read"))
+        ignore_failed_read = true;
+    if (optparse_hasopt (p, "no-cache")) {
+        content_flags |= CONTENT_FLAG_CACHE_BYPASS;
+        kvs_checkpoint_flags |= KVS_CHECKPOINT_FLAG_CACHE_BYPASS;
+    }
+
+    dump_time = time (NULL);
+    dump_uid = getuid ();
+    dump_gid = getgid ();
+
+    h = builtin_get_flux_handle (p);
+
+    /* If the broker is using sd_notify(3) to talk to systemd during
+     * start/stop, we can use it to ensure systemd doesn't kill us
+     * while dumping during shutdown.  See flux-framework/flux-core#5778.
+     */
+    const char *s;
+    if ((s = flux_attr_get (h, "broker.sd-notify")) && !streq (s, "0"))
+        sd_notify_flag = true;
+
+    ar = dump_create (outfile);
+    if (optparse_hasopt (p, "checkpoint")) {
+        flux_future_t *f;
+        const char *blobref;
+        double timestamp;
+
+        if (!(f = kvs_checkpoint_lookup (h, NULL, kvs_checkpoint_flags))
+            || kvs_checkpoint_lookup_get_rootref (f, &blobref) < 0
+            || kvs_checkpoint_lookup_get_timestamp (f, &timestamp) < 0)
+            log_msg_exit ("error fetching checkpoint: %s",
+                          future_strerror (f, errno));
+        dump_time = timestamp;
+        dump_blobref (ar, h, blobref);
+        flux_future_destroy (f);
+    }
+    else {
+        flux_future_t *f;
+        const char *blobref;
+
+        if (!(f = flux_kvs_getroot (h, NULL, 0))
+            || flux_kvs_getroot_get_blobref (f, &blobref) < 0)
+            log_msg_exit ("error fetching current KVS root: %s",
+                          future_strerror (f, errno));
+        dump_blobref (ar, h, blobref);
+        flux_future_destroy (f);
+    }
+
+    progress_end ();
+
+    dump_destroy (ar);
+    flux_close (h);
+
+    return 0;
+}
+
+static struct optparse_option dump_opts[] = {
+    { .name = "verbose", .key = 'v', .has_arg = 0,
+      .usage = "List keys on stderr as they are archived",
+    },
+    { .name = "quiet", .key = 'q', .has_arg = 0,
+      .usage = "Don't show periodic progress updates",
+    },
+    { .name = "checkpoint", .has_arg = 0,
+      .usage = "Dump from checkpoint",
+    },
+    { .name = "no-cache", .has_arg = 0,
+      .usage = "Bypass the broker content cache",
+    },
+    { .name = "ignore-failed-read", .has_arg = 0,
+      .usage = "Treat content load errors as non-fatal",
+    },
+    OPTPARSE_TABLE_END
+};
+
+int subcommand_dump_register (optparse_t *p)
+{
+    optparse_err_t e;
+    e = optparse_reg_subcommand (p,
+        "dump",
+        cmd_dump,
+        "[OPTIONS] OUTFILE",
+        "Dump KVS snapshot to a portable archive format",
+        0,
+        dump_opts);
+    return (e == OPTPARSE_SUCCESS ? 0 : -1);
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/builtin/env.c b/src/cmd/builtin/env.c
index 8cdce83f119c..fee2d7fe59c9 100644
--- a/src/cmd/builtin/env.c
+++ b/src/cmd/builtin/env.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <unistd.h>
 
 #include "builtin.h"
@@ -25,12 +28,12 @@ static int cmd_env (optparse_t *p, int ac, char *av[])
 {
     int n = optparse_option_index (p);
     if (av && av[n]) {
-        execvp (av[n], av+n); /* no return if sucessful */
+        execvp (av[n], av+n); /* no return if successful */
         log_err_exit ("execvp (%s)", av[n]);
     } else {
         struct environment *env = optparse_get_data (p, "env");
         if (env == NULL)
-            log_msg_exit ("flux-env: failed to get flux envirnoment!");
+            log_msg_exit ("flux-env: failed to get flux environment!");
         print_environment (env);
     }
     return (0);
diff --git a/src/cmd/builtin/heaptrace.c b/src/cmd/builtin/heaptrace.c
index 93c1ebdccdc2..2dc5f5844982 100644
--- a/src/cmd/builtin/heaptrace.c
+++ b/src/cmd/builtin/heaptrace.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include "builtin.h"
 
 static int internal_heaptrace_start (optparse_t *p, int ac, char *av[])
diff --git a/src/cmd/builtin/help.c b/src/cmd/builtin/help.c
index 15befc5a2e20..15d548203ebc 100644
--- a/src/cmd/builtin/help.c
+++ b/src/cmd/builtin/help.c
@@ -8,9 +8,13 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <stdlib.h>
 
 #include "src/common/libutil/setenvf.h"
+#include "ccan/str/str.h"
 #include "builtin.h"
 
 static int no_docs_set (optparse_t *p)
@@ -46,7 +50,7 @@ static int cmd_help (optparse_t *p, int ac, char *av[])
          *  O/w, assume user is asking for help on a flux command
          *   so prepend flux-
          */
-        if (strncmp (topic, "flux", 4) == 0)
+        if (strstarts (topic, "flux"))
             cmd = xasprintf ("man %s", topic);
         else
             cmd = xasprintf ("man flux-%s", topic);
diff --git a/src/cmd/builtin/hwloc.c b/src/cmd/builtin/hwloc.c
deleted file mode 100644
index db58c517f75d..000000000000
--- a/src/cmd/builtin/hwloc.c
+++ /dev/null
@@ -1,1029 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include "builtin.h"
-
-#include <sys/types.h> /* WIFEXTED */
-#include <sys/wait.h>
-#include <sys/param.h>
-#include <unistd.h>
-#include <signal.h>
-#include <assert.h>
-#include <argz.h>
-#include <inttypes.h>
-#include <jansson.h>
-
-#include <hwloc.h>
-
-#include "src/common/libidset/idset.h"
-#include "src/common/libaggregate/aggregate.h"
-#include "src/common/libutil/monotime.h"
-
-#define XML_BASEDIR "resource.hwloc.xml"
-
-extern char **environ;
-
-/*  idset helpers:
- */
-
-/*  Return an idset with all ranks set for the current Flux instance:
- */
-static struct idset *idset_all (uint32_t size)
-{
-    struct idset *idset = NULL;
-    if (!(idset = idset_create (size, 0))
-        || (idset_range_set (idset, 0, size-1) < 0)) {
-        idset_destroy (idset);
-        return NULL;
-    }
-    return (idset);
-}
-
-/*  Return an idset from the character string "ranks", returning all
- *   current ranks for "all"
- */
-static struct idset *ranks_to_idset (flux_t *h, const char *ranks)
-{
-    uint32_t size;
-    struct idset *idset;
-
-    if (flux_get_size (h, &size) < 0)
-        return NULL;
-
-    if (strcmp (ranks, "all") == 0)
-        idset = idset_all (size);
-    else {
-        idset = idset_decode (ranks);
-        if (idset_count (idset) > 0 && idset_last (idset) > size - 1) {
-            log_msg ("Invalid rank argument: '%s'", ranks);
-            idset_destroy (idset);
-            return (NULL);
-        }
-    }
-    return (idset);
-}
-
-/*  Topology kvs helpers:
- */
-
-static void lookup_continuation (flux_future_t *f, void *arg)
-{
-    char **valp = arg;
-    const char *xml;
-
-    if (flux_kvs_lookup_get_unpack (f, "s", &xml) < 0)
-        log_err_exit ("unable to unpack rank xml");
-
-    *valp = strdup (xml);
-
-    flux_future_destroy (f);
-}
-
-/*  Send lookup request for topology XML for all ranks in idset, returning
- *   copies of each XML in xmls array (The array must have space for
- *   idset_count (idset) members).
- *  There should be no other watchers registered on the main handle reactor
- *   here, so it is safe to drop into the handle reactor and return when
- *   all lookup handlers have completed.
- */
-static int lookup_all_topo_xml (flux_t *h, char **xmls, struct idset *idset)
-{
-    flux_future_t *f = NULL;
-    char key [1024];
-    int rank = idset_first (idset);
-    int i = 0;
-
-    while (rank != IDSET_INVALID_ID) {
-        snprintf (key, sizeof (key), "%s.%d", XML_BASEDIR, rank);
-        if (!(f = flux_kvs_lookup (h, NULL, 0, key))
-            || (flux_future_then (f, -1., lookup_continuation, &xmls[i]) < 0))
-            log_err_exit ("kvs lookup");
-
-        rank = idset_next (idset, rank);
-        i++;
-    }
-    return (flux_reactor_run (flux_get_reactor (h), 0));
-}
-
-/*  Lookup topo XML for a single rank using degenerate case of
- *  lookup_all_topo_xml()
- */
-static int lookup_one_topo_xml (flux_t *h, char **valp, uint32_t rank)
-{
-    int rc;
-    struct idset *idset = idset_create (0, IDSET_FLAG_AUTOGROW);
-    if (!idset || idset_set (idset, rank) < 0) {
-       log_err ("idset_create/set rank=%d", rank);
-        return (-1);
-    }
-    rc = lookup_all_topo_xml (h, valp, idset);
-    idset_destroy (idset);
-    return (rc);
-}
-
-/*  Given an array of topology XML strings with `n` entries, return
- *   a "global" topology object.
- */
-static hwloc_topology_t global_hwloc_create (char **xml, int n)
-{
-    hwloc_topology_t global;
-    int i;
-
-    if (hwloc_topology_init (&global) < 0
-        || hwloc_topology_set_custom (global) < 0)
-        log_err_exit ("gather: unable to init topology");
-
-    for (i = 0; i < n; i++) {
-        hwloc_topology_t topo;
-        if (hwloc_topology_init (&topo) < 0)
-            log_err_exit ("hwloc_topology_init");
-        if (hwloc_topology_set_xmlbuffer (topo, xml[i], strlen(xml[i])+1) < 0)
-            log_err_exit ("hwloc_topology_set_xmlbuffer");
-        if (hwloc_topology_load (topo) < 0)
-            log_err_exit ("hwloc_topology_load");
-        if (hwloc_custom_insert_topology (global,
-                                          hwloc_get_root_obj (global),
-                                          topo, NULL) < 0)
-            log_err_exit ("hwloc_custom_insert_topo");
-        hwloc_topology_destroy (topo);
-    }
-    hwloc_topology_load (global);
-
-    return (global);
-}
-
-static void string_array_destroy (char **arg, int n)
-{
-    int i;
-    for (i = 0; i < n; i++)
-        free (arg[i]);
-    free (arg);
-}
-
-char * flux_hwloc_global_xml (optparse_t *p)
-{
-    flux_t *h = NULL;
-    uint32_t size;
-    char *xml;
-    char *buf;
-    const char *arg;
-    int buflen;
-    struct idset *idset = NULL;
-    hwloc_topology_t global = NULL;
-    char **xmlv = NULL;
-
-    if (!(h = builtin_get_flux_handle (p)))
-        log_err_exit ("flux_open");
-
-    if (optparse_getopt (p, "rank", &arg) <= 0)
-        arg = "all";
-    if (!(idset = ranks_to_idset (h, arg)))
-        log_msg_exit ("failed to get target ranks");
-
-    if ((size = idset_count (idset)) == 0)
-        log_msg_exit ("Invalid rank set when fetching global XML");
-
-    if (!(xmlv = calloc (size, sizeof (char *))))
-        log_msg_exit ("failed to alloc array for %d ranks", size);
-
-    if (lookup_all_topo_xml (h, xmlv, idset) < 0)
-        log_err_exit ("gather: failed to get all topo xml");
-
-    if (!(global = global_hwloc_create (xmlv, size)))
-        log_err_exit ("gather: failed create global xml");
-
-    if (hwloc_topology_export_xmlbuffer (global, &buf, &buflen) < 0)
-        log_err_exit ("hwloc export XML");
-
-    /* XML buffer must be destroyed by hwloc_free_xmlbuffer, so copy it here */
-    xml = strdup (buf);
-
-    hwloc_free_xmlbuffer (global, buf);
-    string_array_destroy (xmlv, size);
-    hwloc_topology_destroy (global);
-    idset_destroy (idset);
-    flux_close (h);
-    return (xml);
-}
-
-/*  HWLOC topology helpers:
- */
-
-/*  Common hwloc_topology_init() and flags for Flux hwloc usage:
- */
-static void topo_init_common (hwloc_topology_t *tp)
-{
-    if (hwloc_topology_init (tp) < 0)
-        log_err_exit ("hwloc_topology_init");
-    if (hwloc_topology_set_flags (*tp, HWLOC_TOPOLOGY_FLAG_IO_DEVICES) < 0)
-        log_err_exit ("hwloc_topology_set_flags");
-    if (hwloc_topology_ignore_type (*tp, HWLOC_OBJ_CACHE) < 0)
-        log_err_exit ("hwloc_topology_ignore_type OBJ_CACHE failed");
-    if (hwloc_topology_ignore_type (*tp, HWLOC_OBJ_GROUP) < 0)
-        log_err_exit ("hwloc_topology_ignore_type OBJ_GROUP failed");
-}
-
-/*  Load the local topology in a manner most useful to Flux components,
- *   i.e. grab IO devices, ignore cache and group objects, and mask off
- *   objects not in our cpuset.
- */
-static hwloc_topology_t local_topo_load (void)
-{
-    hwloc_topology_t topo;
-    hwloc_bitmap_t rset = NULL;
-    uint32_t hwloc_version = hwloc_get_api_version ();
-
-    if ((hwloc_version >> 16) != (HWLOC_API_VERSION >> 16))
-        log_err_exit ("compiled for hwloc 0x%x but running against 0x%x\n",
-                      HWLOC_API_VERSION, hwloc_version);
-
-    topo_init_common (&topo);
-
-    if (hwloc_topology_load (topo) < 0)
-        log_err_exit ("hwloc_topology_load");
-    if (!(rset = hwloc_bitmap_alloc ())
-        || (hwloc_get_cpubind (topo, rset, HWLOC_CPUBIND_PROCESS) < 0))
-        log_err_exit ("hwloc_get_cpubind");
-    if (hwloc_topology_restrict (topo, rset, 0) < 0)
-        log_err_exit ("hwloc_topology_restrict");
-    hwloc_bitmap_free (rset);
-    return (topo);
-}
-
-static char *flux_hwloc_local_xml (void)
-{
-    char *buf;
-    int buflen;
-    char *copy;
-    hwloc_topology_t topo = local_topo_load ();
-    if (topo == NULL)
-        return (NULL);
-    if (hwloc_topology_export_xmlbuffer (topo, &buf, &buflen) < 0)
-        log_err_exit ("Failed to export hwloc to XML");
-    copy = strdup (buf);
-    hwloc_free_xmlbuffer (topo, buf);
-    return (copy);
-}
-
-/*
- *  Return hwloc XML as a malloc()'d string. Returns the topolog of this
- *   system if "--local" is set in the optparse object `p`, otherwise
- *   returns the global XML. Caller must free the result.
- */
-static char *flux_hwloc_xml (optparse_t *p)
-{
-    if (optparse_hasopt (p, "local"))
-        return flux_hwloc_local_xml ();
-    return flux_hwloc_global_xml (p);
-}
-
-static int argz_appendf (char **argzp, size_t *argz_len, const char *fmt, ...)
-{
-    char s [4096];
-    int rc = -1;
-    int n;
-    va_list ap;
-
-    va_start (ap, fmt);
-    if ((n = vsnprintf (s, sizeof (s), fmt, ap) < 0) || (n >= sizeof (s)))
-        goto out;
-    va_end (ap);
-
-    if ((errno = argz_add_sep (argzp, argz_len, s, ' ')) == 0)
-        rc = 0;
-out:
-    return (rc);
-}
-
-static void lstopo_argz_init (char *cmd, char **argzp, size_t *argz_lenp,
-                              char *extra_args[])
-{
-    char *extra;
-    size_t extra_len;
-    int e;
-    char *argv[] = { cmd, "-i", "-", "--if", "xml",
-                     "--of", "console", NULL };
-    if (  (e = argz_create (argv, argzp, argz_lenp)) != 0
-       || (e = argz_create (extra_args, &extra, &extra_len)) != 0)
-        log_msg_exit ("argz_create: %s", strerror (e));
-
-    /*  Append any extra args in av[] */
-    if ((e = argz_append (argzp, argz_lenp, extra, extra_len)) != 0)
-        log_msg_exit ("argz_append: %s", strerror (e));
-
-    free (extra);
-}
-
-int argz_execp (char *argz, size_t argz_len)
-{
-    char *argv [argz_count (argz, argz_len) + 1];
-    argz_extract (argz, argz_len, argv);
-    return execvp (argv[0], argv);
-}
-
-FILE * argz_popen (char *argz, size_t argz_len, pid_t *pidptr)
-{
-    int pfds[2];
-    pid_t pid;
-    if (pipe (pfds) < 0)
-        log_err_exit ("pipe");
-    switch ((pid = fork ())) {
-    case -1:
-        log_err_exit ("fork");
-    case  0:
-        close (pfds[1]);
-        dup2 (pfds[0], STDIN_FILENO);
-        argz_execp (argz, argz_len);
-        if (errno != ENOENT)
-            log_err ("exec");
-        exit (errno); /* So we can detect ENOENT.. Sorry */
-    default:
-        break;
-    }
-    close (pfds[0]);
-    *pidptr = pid;
-    return (fdopen (pfds[1], "w"));
-}
-
-/*
- *  Execute lstopo from user's PATH sending full topology XML over stdin.
- *   Pass any extra options along to lstopo(1).
- *
- *  If running lstopo fails with ENOENT, try lstopo-no-graphics.
- */
-static int exec_lstopo (optparse_t *p, int ac, char *av[], const char *topo)
-{
-    int status;
-    FILE *fp;
-    char *argz;
-    size_t argz_len;
-    pid_t pid;
-    const char *cmds[] = { "lstopo", "lstopo-no-graphics", NULL };
-    const char **cp = cmds;
-
-    /* Ignore SIGPIPE so we don't get killed when exec() fails */
-    signal (SIGPIPE, SIG_IGN);
-
-    /* Initialize argz with first command in cmds above: */
-    lstopo_argz_init ((char *) *cp, &argz, &argz_len, av+1);
-
-    while (true) {
-        const char *next = *(cp+1);
-        if (!(fp = argz_popen (argz, argz_len, &pid)))
-            log_err_exit ("popen (lstopo)");
-        fputs (topo, fp);
-        fclose (fp);
-        if (waitpid (pid, &status, 0) < 0)
-            log_err_exit ("waitpid");
-
-        /* Break out of loop if exec() was succcessful, failed with
-         *  an error other than "File not found", or we ran out programs
-         *  to try.
-         */
-        if (status == 0 || !next || WEXITSTATUS (status) != ENOENT)
-            break;
-
-        /* Replace previous cmd in argz with next command to try:
-         */
-        argz_replace (&argz, &argz_len, *(cp++), next, NULL);
-    }
-
-    return (status);
-}
-
-static int cmd_lstopo (optparse_t *p, int ac, char *av[])
-{
-    int status;
-    char *xml = flux_hwloc_global_xml (p);
-    if (xml == NULL)
-        log_msg_exit ("Failed to fetch global hwloc XML");
-
-    status = exec_lstopo (p, ac, av, xml);
-    if (status) {
-        if (WIFEXITED (status)) {
-            if (WEXITSTATUS (status) == ENOENT)
-                log_msg_exit ("Unable to find an lstopo variant to execute.");
-            else
-                log_msg_exit ("lstopo: Exited with %d", WEXITSTATUS (status));
-        }
-        if (WIFSIGNALED (status) && WTERMSIG (status) != SIGPIPE)
-            log_msg_exit ("lstopo: %s%s", strsignal (WTERMSIG (status)),
-                      WCOREDUMP (status) ? " (core dumped)" : "");
-    }
-
-    free (xml);
-
-    return (0);
-}
-
-/*  flux-hwloc topology:
- */
-
-static int cmd_topology (optparse_t *p, int ac, char *av[])
-{
-    char *xml = flux_hwloc_xml (p);
-    puts (xml);
-    free (xml);
-    return (0);
-}
-
-/*  flux-hwloc info:
- */
-
-/*  Initialize a hwloc toplogy from xml string `xml`, applying the common
- *   flags and options for Flux usage.
- */
-static int init_topo_from_xml (hwloc_topology_t *tp, const char *xml)
-{
-    topo_init_common (tp);
-    if ((hwloc_topology_set_xmlbuffer (*tp, xml, strlen (xml) + 1) < 0)
-        || (hwloc_topology_load (*tp) < 0)) {
-        hwloc_topology_destroy (*tp);
-        return (-1);
-    }
-    return (0);
-}
-
-static int cmd_info (optparse_t *p, int ac, char *av[])
-{
-    char *xml = flux_hwloc_xml (p);
-    hwloc_topology_t topo;
-
-    if (!xml || init_topo_from_xml (&topo, xml) < 0)
-        log_msg_exit ("info: Failed to initialize topology from XML");
-
-    int ncores = hwloc_get_nbobjs_by_type (topo, HWLOC_OBJ_CORE);
-    int npu    = hwloc_get_nbobjs_by_type (topo, HWLOC_OBJ_PU);
-    int nnodes = hwloc_get_nbobjs_by_type (topo, HWLOC_OBJ_MACHINE);
-
-    printf ("%d Machine%s, %d Cores, %d PUs\n",
-            nnodes, nnodes > 1 ? "s" : "", ncores, npu);
-
-    hwloc_topology_destroy (topo);
-    free (xml);
-    return (0);
-}
-
-/*  flux-hwloc reload:
- */
-
-/*  Add hwloc xml string `xml` to kvs for rank `rank` to a kvs txn
- */
-static int kvs_txn_put_xml (flux_kvs_txn_t *txn, uint32_t rank,
-                             const char *xml)
-{
-    char key [1024];
-    snprintf (key, sizeof (key), "%s.%ju", XML_BASEDIR, (uintmax_t) rank);
-    return (flux_kvs_txn_pack (txn, 0, key, "s", xml));
-}
-
-/*  Add hwloc xml from file at path <basedir>/<rank>.xml to kvs for
- *   rank. The topology is first loaded into a hwloc_topology_t object
- *   so that the common Flux hwloc flags may be applied,  and to check
- *   that the XML is valid before putting it in the kvs.
- */
-static flux_future_t *kvs_txn_put_xml_file (flux_kvs_txn_t *txn, int rank,
-                                            const char *basedir)
-{
-    char path [8192];
-    int n, len;
-    char *xml;
-    hwloc_topology_t topo = NULL;
-
-    if ((n = snprintf (path, sizeof (path), "%s/%d.xml", basedir, rank) < 0)
-        || (n >= sizeof (path)))
-        log_err_exit ("failed to create xml path");
-
-    topo_init_common (&topo);
-
-    if (hwloc_topology_set_xml (topo, path) < 0)
-        log_err_exit ("Unable to set XML to path=%s", path);
-
-    if (hwloc_topology_load (topo) < 0)
-        log_err_exit ("hwloc_topology_load (%s)", path);
-
-    if (hwloc_topology_export_xmlbuffer (topo, &xml, &len) < 0)
-        log_err_exit ("hwloc_topology_export_xmlbuffer");
-
-    if (kvs_txn_put_xml (txn, rank, xml) < 0)
-        log_err_exit ("kvs_txn_put_xml");
-
-    hwloc_free_xmlbuffer (topo, xml);
-    hwloc_topology_destroy (topo);
-    return (0);
-}
-
-/*  Load XML for all ranks in `idset` from files in `basedir`, one per
- *   rank: <basedir>/<rank>.xml. All KVS puts are performed under a
- *   single transaction.
- */
-flux_future_t * kvs_load_xml_idset (flux_t *h, const char *basedir,
-                                    struct idset *idset)
-{
-    flux_future_t *f = NULL;
-    flux_kvs_txn_t *txn = NULL;
-    unsigned int rank = idset_first (idset);
-
-    if (!(txn = flux_kvs_txn_create ()))
-        log_err_exit ("flux_kvs_txn_create");
-
-    while (rank != IDSET_INVALID_ID) {
-        kvs_txn_put_xml_file (txn, rank, basedir);
-        rank = idset_next (idset, rank);
-    }
-    if (!(f = flux_kvs_commit (h, NULL, 0, txn)))
-        log_err_exit ("flux_kvs_commit request");
-    flux_kvs_txn_destroy (txn);
-    return (f);
-}
-
-static double seconds_since (struct timespec t)
-{
-    return (monotime_since (t)/1000.);
-}
-
-/*  Execute flux-hwloc aggregate-load across all ranks, optionally
- *   reloading local hwloc XML on `reload_ranks`.
- */
-static int run_hwloc_aggregate (flux_t *h, const char *ranks, bool verbose,
-                                struct timespec t0)
-{
-    const char *base = "resource.hwloc";
-    char *argz = NULL;
-    size_t argz_len = 0;
-    uint32_t rank, size;
-    double timeout = 5.;
-    char *argv[] = {
-        "flux", "exec", "-n", "-r", "all", NULL
-    };
-
-    if (flux_get_rank (h, &rank) < 0)
-        log_err_exit ("flux_get_rank");
-    if (flux_get_size (h, &size) < 0)
-        log_err_exit ("flux_get_rank");
-
-    /* XXX: scale timeout by size just in case.. */
-    if (size > 512)
-        timeout = timeout + size/512.;
-
-    if ((errno = argz_create (argv, &argz, &argz_len)))
-        log_err_exit ("exec aggregate-load: argz_create");
-
-    /*  Append -v to flux-exec if verbose */
-    if (verbose && (argz_appendf (&argz, &argz_len, "-v")))
-        log_err_exit ("exec aggregate-load: argz_appendf");
-
-    /*  Build flux hwloc aggregate-load command */
-    if ((argz_appendf (&argz, &argz_len,
-                "flux hwloc aggregate-load "
-                "--timeout=%.3f --unpack=%s.by_rank --key=%s.reload:%u-%u",
-                timeout, base, base, rank, getpid()) < 0)
-        || (ranks && (argz_appendf (&argz, &argz_len, "--rank=%s", ranks) < 0))
-        || (verbose && argz_appendf (&argz, &argz_len, "--verbose")))
-        log_err_exit ("argz_appendf flux-hwloc aggregate-load command");
-
-    if (verbose) {
-        char copy [argz_len];
-        memcpy (copy, argz, argz_len);
-        argz_stringify (copy, argz_len, ' ');
-        log_msg ("%.3fs: Running %s", seconds_since (t0), copy);
-    }
-    argz_execp (argz, argz_len);
-
-    log_err_exit ("exec: flux-exec flux hwloc aggregate-load");
-}
-
-static int internal_hwloc_reload (optparse_t *p, int ac, char *av[])
-{
-    int n = optparse_option_index (p);
-    const char *default_nodeset = "all";
-    const char *nodeset = optparse_get_str (p, "rank", default_nodeset);
-    uint32_t size;
-    bool verbose;
-    struct timespec t0;
-    struct idset *idset = NULL;
-    char *dirpath = NULL;
-    char *reload_ranks;
-    flux_t *h;
-
-    if ((verbose = optparse_hasopt (p, "verbose")))
-        monotime (&t0);
-
-    if (!(h = builtin_get_flux_handle (p)))
-        log_err_exit ("flux_open");
-    if (flux_get_size (h, &size) < 0)
-        log_err_exit ("flux_get_size");
-    if (av[n] && !(dirpath = realpath (av[n], NULL)))
-        log_err_exit ("%s", av[n]);
-    if (!(idset = ranks_to_idset (h, nodeset)))
-        log_msg_exit ("--rank=%s: Invalid argument", nodeset);
-    if (idset_last (idset) > size - 1)
-        log_msg_exit ("--rank=%s: Invalid rank specified", nodeset);
-
-    if (verbose)
-        log_msg ("%.3fs: starting HWLOC reload on %ju ranks (%s)",
-                 seconds_since (t0), (uintmax_t) idset_count (idset), nodeset);
-
-    if (dirpath) {
-        if (verbose)
-            log_msg ("%.3fs: starting load of XML from %s",
-                     seconds_since (t0), dirpath);
-
-        flux_future_t *f = kvs_load_xml_idset (h, dirpath, idset);
-        if (!f || flux_future_get (f, NULL) < 0)
-            log_err_exit ("%s: failed to load all XML", dirpath);
-        flux_future_destroy (f);
-        reload_ranks = NULL;
-
-        if (verbose)
-            log_msg ("%.3fs: XML load complete", seconds_since (t0));
-    }
-    else
-        reload_ranks = strdup (nodeset);
-
-    if (verbose)
-        log_msg ("%.3fs: executing aggregate-load across all ranks",
-                seconds_since (t0));
-    run_hwloc_aggregate (h, reload_ranks, optparse_hasopt (p, "verbose"), t0);
-
-    // run_hwloc_aggregate doesn't return, but clean up anyway:
-    free (dirpath);
-    free (reload_ranks);
-    idset_destroy (idset);
-    flux_close (h);
-    return (0);
-}
-
-
-
-/*  flux-hwloc aggregate-load:
- */
-
-/*  Count supported GPU object in a hwloc topology object.
- *
- *  Currently only CUDA and OpenCL devices are counted.
- */
-static int hwloc_gpu_count (hwloc_topology_t topology)
-{
-    int nobjs = 0;
-    hwloc_obj_t obj = NULL;
-    while ((obj = hwloc_get_next_osdev (topology, obj))) {
-        /* Only count cudaX and openclX devices for now */
-        const char *s = hwloc_obj_get_info_by_name (obj, "Backend");
-        if (s && ((strcmp (s, "CUDA") == 0) || (strcmp (s, "OpenCL") == 0)))
-            nobjs++;
-    }
-    return (nobjs);
-}
-
-/*  Generate an allowed cpuset string for the current topology:
- */
-static char *hwloc_cpuset_idset_string (hwloc_topology_t topo)
-{
-    int i;
-    char *result = NULL;
-    struct idset *ids = NULL;
-    hwloc_const_cpuset_t cset = hwloc_topology_get_allowed_cpuset (topo);
-
-    if (!(ids = idset_create (0, IDSET_FLAG_AUTOGROW))
-        || !(cset = hwloc_topology_get_allowed_cpuset (topo)))
-        goto out;
-    i = hwloc_bitmap_first (cset);
-    while (i >= 0) {
-        idset_set (ids, i);
-        i = hwloc_bitmap_next (cset, i);
-    }
-    result = idset_encode (ids, IDSET_FLAG_RANGE);
-out:
-    idset_destroy (ids);
-    return result;
-}
-
-/*  Emit a json object containing summary statistics for the topology argument.
- */
-static json_t *topo_tojson (hwloc_topology_t topology)
-{
-    int nobj, i;
-    char *ids = NULL;
-    json_t *o = NULL;
-    json_t *v = NULL;
-    int depth = hwloc_topology_get_depth (topology);
-
-    if (!(o = json_object ()))
-        return NULL;
-    for (i = 0; i < depth; i++) {
-        hwloc_obj_type_t t = hwloc_get_depth_type (topology, i);
-        nobj = hwloc_get_nbobjs_by_depth (topology, i);
-        /* Skip "Machine" or "System" = 1 */
-        if ((t == HWLOC_OBJ_MACHINE || t == HWLOC_OBJ_SYSTEM) && nobj == 1)
-            continue;
-        if (!(v = json_integer (nobj)))
-            goto error;
-        if (json_object_set_new (o, hwloc_obj_type_string (t), v) < 0)
-            goto error;
-    }
-    if ((nobj = hwloc_gpu_count (topology))) {
-        if (!(v = json_integer (nobj))
-            || json_object_set_new (o, "GPU", v) < 0)
-            goto error;
-    }
-    if ((ids = hwloc_cpuset_idset_string (topology))) {
-        if (!(v = json_string (ids))
-            || json_object_set_new (o, "cpuset", v) < 0)
-            goto error;
-        free (ids);
-    }
-    return (o);
-error:
-    json_decref (o);
-    return (NULL);
-}
-
-static int get_fwd_count (flux_t *h)
-{
-    const char *s = flux_attr_get (h, "tbon.descendants");
-    long v = strtol (s, NULL, 10);
-    if (v >= 0)
-        return ((int) v + 1);
-    return (0);
-}
-
-/*  Create a JSON object summarizing object counts in local topology
- *   and initiate an aggregate across all ranks of that object.
- */
-static int aggregate_topo_summary (flux_t *h, const char *key, const char *xml)
-{
-    json_t *o = NULL;
-    flux_future_t *f = NULL;
-    hwloc_topology_t topo;
-    int fwd_count = get_fwd_count (h);
-
-    if (init_topo_from_xml (&topo, xml) < 0)
-        log_err_exit ("aggregate_topo_summary: failed to initialize topology");
-
-    if (!(o = topo_tojson (topo)))
-        log_err_exit ("Failed to convert topology to JSON");
-
-    if (!(f = aggregator_push_json (h, fwd_count, 1., key, o))
-        || (flux_future_get (f, NULL) < 0))
-        log_err_exit ("aggregator_push_json");
-
-    flux_future_destroy (f);
-    hwloc_topology_destroy (topo);
-    return (0);
-}
-
-static void aggregate_load_wait (optparse_t *p, flux_t *h, const char *key)
-{
-    const char *unpack_path = NULL;
-    double timeout;
-    flux_future_t *f = NULL;
-
-    timeout = optparse_get_duration (p, "timeout", 15.);
-
-    if (!(f = aggregate_wait (h, key))
-       || flux_future_wait_for (f, timeout) < 0)
-        log_err_exit ("aggregate_wait");
-
-    if (optparse_getopt (p, "unpack", &unpack_path)
-        && aggregate_unpack_to_kvs (f, unpack_path) < 0)
-        log_err_exit ("unable to unpack aggregate to kvs");
-
-    if (optparse_hasopt (p, "print-result")) {
-        const char *s;
-        if (aggregate_wait_get (f, &s) < 0)
-            log_err_exit ("aggregate_wait_get_unpack");
-        puts (s);
-    }
-    flux_future_destroy (f);
-}
-
-/*  Put xml string `xml` into hwloc xml entry in kvs for rank `rank`,
- *   and then perform synchronous kvs_fence for nprocs entries.
- */
-static int kvs_put_xml_fence (flux_t *h, int rank,
-                              const char *name, int nprocs,
-                              const char *xml)
-{
-    flux_future_t *f;
-    flux_kvs_txn_t *txn = NULL;
-
-    if (!(txn = flux_kvs_txn_create ())
-        || (kvs_txn_put_xml (txn, rank, xml) < 0))
-        log_err_exit ("kvs put xml (rank=%d)", rank);
-    if (!(f = flux_kvs_fence (h, NULL, 0, name, nprocs, txn))
-        || flux_future_get (f, NULL) < 0)
-        log_err_exit ("kvs_put_xml: commit");
-    flux_future_destroy (f);
-    flux_kvs_txn_destroy (txn);
-    return (0);
-}
-
-/*  Undocumented utility function that optionally loads local topology XML
- *   on selected ranks, then uses the aggregator module to create a
- *   summary of all rank HW topology object types, optionally storing the
- *   result in the KVS.
- */
-static int cmd_aggregate_load (optparse_t *p, int ac, char *av[])
-{
-    char *xml = NULL;
-    const char *key = NULL;
-    const char *ranks = NULL;
-    struct idset *idset = NULL;
-    uint32_t rank;
-    struct timespec t0;
-    bool verbose;
-    flux_t *h = NULL;
-
-    log_init ("flux-hwloc aggregate-load");
-
-    if ((verbose = optparse_hasopt (p, "verbose"))) {
-        setlinebuf (stderr);
-        monotime (&t0);
-    }
-
-    if (!optparse_getopt (p, "key", &key))
-        log_err_exit ("Missing required option --key");
-
-    if (!(h = builtin_get_flux_handle (p)))
-        log_err_exit ("flux_open");
-
-    if (flux_get_rank (h, &rank) < 0)
-        log_err_exit ("flux_get_rank");
-
-    if (verbose && rank != 0)
-        verbose = false;
-
-    /*  If rank not specified then default to an empty set */
-    if (optparse_getopt (p, "rank", &ranks) <= 0)
-        ranks = "";
-    if (!(idset = ranks_to_idset (h, ranks)))
-        log_err_exit ("Invalid argument: -rank='%s'", ranks);
-
-    if (verbose)
-        log_msg ("%.3fs: starting", seconds_since (t0));
-
-    /*  If this rank is in idset, then we need to reload local XML into
-     *   kvs before re-aggregation. Otherwise, fetch XML from kvs.
-     */
-    if (idset_test (idset, rank)) {
-        if (verbose)
-            log_msg ("%.3fs: pushing local xml", seconds_since (t0));
-        if (!(xml = flux_hwloc_local_xml ()))
-            log_err_exit ("Failed to get local XML");
-        if (verbose)
-            log_msg ("%.3fs: starting kvs fence", seconds_since (t0));
-        if (kvs_put_xml_fence (h, rank, key, idset_count (idset), xml) < 0)
-            log_err_exit ("Failed to store local XML in kvs");
-        if (verbose)
-            log_msg ("%.3fs: kvs fence complete", seconds_since (t0));
-    }
-    else if (lookup_one_topo_xml (h, &xml, rank) < 0)
-        log_err_exit ("lookup topo XML for this rank (%d)", rank);
-
-    if (verbose)
-        log_msg ("%.3fs: starting aggregate", seconds_since (t0));
-
-    /* Immediately push aggregate from all ranks
-     */
-    if (aggregate_topo_summary (h, key, xml) < 0)
-        log_err_exit ("Unable to aggregate topology summary");
-
-    if (verbose)
-        log_msg ("%.3fs: aggregate push complete", seconds_since (t0));
-
-    /*  Rank 0 waits for aggregate completion and optionally "unpacks"
-     *   aggregate to new KVS location.
-     */
-    if (rank == 0)
-        aggregate_load_wait (p, h, key);
-    if (verbose)
-        log_msg ("%.3fs: aggregate_wait complete", seconds_since (t0));
-
-    idset_destroy (idset);
-    free (xml);
-    flux_close (h);
-    if (verbose)
-        log_msg ("%.3fs: done.", monotime_since (t0)/1000.);
-    return (0);
-}
-
-
-/*  flux-hwloc:
- */
-
-int cmd_hwloc (optparse_t *p, int ac, char *av[])
-{
-    log_init ("flux-hwloc");
-    if (optparse_run_subcommand (p, ac, av) != OPTPARSE_SUCCESS)
-        exit (1);
-    return (0);
-}
-
-static struct optparse_option reload_opts[] = {
-    { .name = "verbose",  .key = 'v',  .has_arg = 0,
-      .usage = "Increase verbosity", },
-    { .name = "rank",  .key = 'r',  .has_arg = 1,
-      .usage = "Target specified nodeset, or \"all\" (default)", },
-    OPTPARSE_TABLE_END,
-};
-
-static struct optparse_option topology_opts[] = {
-    { .name = "local", .key = 'l', .has_arg = 0,
-      .usage = "Dump topology XML for the local host only",
-    },
-    { .name = "rank", .key = 'r', .has_arg = 1,
-      .usage = "Target specified nodeset, or \"all\" (default)",
-    },
-    OPTPARSE_TABLE_END,
-};
-
-static struct optparse_option aggregate_load_opts[] = {
-    { .name = "verbose", .key = 'v', .has_arg = 0,
-      .usage = "Increase verbosity (only affects rank 0)",
-    },
-    { .name = "timeout", .key = 't', .has_arg = 1,
-      .usage = "Duration to wait for aggregate completion (default 15.0s)",
-    },
-    { .name = "rank", .key = 'r', .has_arg = 1,
-      .usage = "ranks on which to perform a local topology reload",
-    },
-    { .name = "key", .key = 'k', .has_arg = 1,
-      .usage = "KVS key for aggregate",
-    },
-    { .name = "unpack", .key = 'u', .has_arg = 1,
-      .usage = "KVS key to which to optionally \"unpack\" aggregate",
-    },
-    { .name = "print-result", .key = 'p', .has_arg = 0,
-      .usage = "Print final aggregate on rank 0 upon completion",
-    },
-    OPTPARSE_TABLE_END,
-};
-
-static struct optparse_subcommand hwloc_subcmds[] = {
-    { "reload",
-      "[OPTIONS] [DIR]",
-      "Reload hwloc XML, optionally from DIR/<rank>.xml files",
-      internal_hwloc_reload,
-      0,
-      reload_opts,
-    },
-    { "lstopo",
-      "[lstopo-OPTIONS]",
-      "Show hwloc topology of the system",
-      cmd_lstopo,
-      OPTPARSE_SUBCMD_SKIP_OPTS,
-      NULL,
-    },
-    { "topology",
-      NULL,
-      "Dump system topology XML to stdout",
-      cmd_topology,
-      0,
-      topology_opts,
-    },
-    { "info",
-      NULL,
-      "Short-form dump of instance resources",
-      cmd_info,
-      0,
-      topology_opts,
-    },
-    { "aggregate-load",
-      "[OPTIONS]",
-      "aggregate hwloc summary with optional local topology reload",
-      cmd_aggregate_load,
-      OPTPARSE_SUBCMD_HIDDEN,
-      aggregate_load_opts,
-    },
-    OPTPARSE_SUBCMD_END,
-};
-
-int subcommand_hwloc_register (optparse_t *p)
-{
-    optparse_t *c;
-    optparse_err_t e;
-
-    e = optparse_reg_subcommand (p, "hwloc", cmd_hwloc, NULL,
-                                 "Control/query resource-hwloc service",
-                                 0, NULL);
-    if (e != OPTPARSE_SUCCESS)
-        return (-1);
-
-    c = optparse_get_subcommand (p, "hwloc");
-    if ((e = optparse_reg_subcommands (c, hwloc_subcmds)) != OPTPARSE_SUCCESS)
-        return (-1);
-    return (e == OPTPARSE_SUCCESS ? 0 : -1);
-}
-
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/cmd/builtin/lptest.c b/src/cmd/builtin/lptest.c
new file mode 100644
index 000000000000..0d986c78a55f
--- /dev/null
+++ b/src/cmd/builtin/lptest.c
@@ -0,0 +1,72 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* lptest.c - ripple test */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+
+#include "builtin.h"
+
+static void lptest (int length, int count)
+{
+    int i;
+    int j;
+
+    for (i = 0; i < count; i++) {
+        for (j = 0; j < length; j++)
+            putchar (0x21 + ((i + j) % 0x5e)); // charset: !(0x21) thru ~(0x7e)
+        putchar ('\n');
+    }
+}
+
+static int cmd_lptest (optparse_t *p, int ac, char **av)
+{
+    int n = optparse_option_index (p);
+    int length = 79;
+    int count = 200;
+    char *endptr;
+
+    log_init ("flux-lptest");
+
+    if (n < ac) {
+        errno = 0;
+        length = strtoul (av[n++], &endptr, 10);
+        if (errno != 0 || *endptr != '\0')
+            log_msg_exit ("error parsing length");
+    }
+    if (n < ac) {
+        errno = 0;
+        count = strtoul (av[n++], &endptr, 10);
+        if (errno != 0 || *endptr != '\0')
+            log_msg_exit ("error parsing count");
+    }
+    if (n < ac)
+        optparse_fatal_usage (p, 1, NULL);
+    lptest (length, count);
+    return 0;
+}
+
+int subcommand_lptest_register (optparse_t *p)
+{
+    optparse_err_t e;
+    e = optparse_reg_subcommand (p,
+        "lptest",
+        cmd_lptest,
+        "[length] [count]",
+        "Create lines of regular output for standard I/O testing",
+        0,
+        NULL);
+    return (e == OPTPARSE_SUCCESS ? 0 : -1);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/cmd/builtin/overlay.c b/src/cmd/builtin/overlay.c
new file mode 100644
index 000000000000..1f300d12055b
--- /dev/null
+++ b/src/cmd/builtin/overlay.c
@@ -0,0 +1,1488 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+#include <unistd.h>
+#include <time.h>
+#include <math.h>
+#include <jansson.h>
+#include <flux/core.h>
+#ifdef HAVE_STRERRORNAME_NP
+#include <string.h>
+#else
+#include "src/common/libmissing/strerrorname_np.h"
+#endif
+
+#include "src/common/libccan/ccan/str/str.h"
+#include "src/common/libccan/ccan/ptrint/ptrint.h"
+#include "src/common/libutil/monotime.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libhostlist/hostlist.h"
+#include "src/common/librlist/rlist.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/ansi_color.h"
+#include "src/common/libutil/parse_size.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+#include "builtin.h"
+
+static double default_timeout = 0.5;
+
+static struct optparse_option errors_opts[] = {
+    { .name = "timeout", .key = 't', .has_arg = 1, .arginfo = "FSD",
+      .usage = "Set RPC timeout, 0=disable (default 0.5s)",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option status_opts[] = {
+    { .name = "rank", .key = 'r', .has_arg = 1, .arginfo = "NODEID",
+      .usage = "Check health of subtree rooted at NODEID (default 0)",
+    },
+    { .name = "verbose", .key = 'v', .has_arg = 2, .arginfo = "[LEVEL]",
+      .usage = "Increase reporting detail:"
+               " 1=show time since current state was entered,"
+               " 2=show round-trip RPC times."
+    },
+    { .name = "timeout", .key = 't', .has_arg = 1, .arginfo = "FSD",
+      .usage = "Set RPC timeout, 0=disable (default 0.5s)",
+    },
+    { .name = "summary", .has_arg = 0,
+      .usage = "Show only the root subtree status."
+    },
+    { .name = "down", .has_arg = 0,
+      .usage = "Show only the partial/degraded subtrees."
+    },
+    { .name = "no-pretty", .has_arg = 0,
+      .usage = "Do not indent entries and use line drawing characters"
+               " to show overlay tree structure",
+    },
+    { .name = "no-ghost", .has_arg = 0,
+      .usage = "Do not fill in presumed state of nodes that are"
+               " inaccessible behind offline/lost overlay parents",
+    },
+    { .name = "color", .key = 'L', .has_arg = 2, .arginfo = "WHEN",
+      .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
+      .usage = "Colorize output when supported; WHEN can be 'always' "
+               "(default if omitted), 'never', or 'auto' (default)."
+    },
+    { .name = "highlight", .key = 'H', .has_arg = 1, .arginfo = "TARGET",
+      .usage = "Highlight one or more TARGETs and their ancestors."
+    },
+    { .name = "wait", .key = 'w', .has_arg = 1, .arginfo = "STATE",
+      .usage = "Wait until subtree enters STATE before reporting"
+               " (full, partial, offline, degraded, lost)",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option disconnect_opts[] = {
+    { .name = "parent", .key = 'r', .has_arg = 1, .arginfo = "NODEID",
+      .usage = "Set parent rank to NODEID (default: determine from topology)",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option trace_opts[] = {
+    { .name = "rank", .key = 'r', .has_arg = 1, .arginfo = "NODEID",
+      .usage = "Filter output by peer rank",
+    },
+    { .name = "full", .key = 'f', .has_arg = 0,
+      .usage = "Show JSON message payload, if any",
+    },
+    { .name = "type", .key = 't', .has_arg = 1,
+      .flags = OPTPARSE_OPT_AUTOSPLIT,
+      .arginfo = "TYPE,...",
+      .usage = "Filter output by message type "
+               "(request, response, event, control)",
+    },
+    { .name = "color", .key = 'L', .has_arg = 2, .arginfo = "WHEN",
+      .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
+      .usage = "Colorize output when supported; WHEN can be 'always' "
+               "(default if omitted), 'never', or 'auto' (default)." },
+    { .name = "human",  .key = 'H',  .has_arg = 0,
+      .usage = "Human readable output", },
+    { .name = "delta",  .key = 'd',  .has_arg = 0,
+      .usage = "With --human, show timestamp delta between messages", },
+    OPTPARSE_TABLE_END,
+};
+
+struct status {
+    flux_t *h;
+    int verbose;
+    int color;
+    double timeout;
+    optparse_t *opt;
+    struct timespec start;
+    const char *wait;
+    struct idset *highlight;
+    zlistx_t *stack;
+};
+
+enum connector {
+    PIPE = 1,
+    TEE = 2,
+    ELBOW = 3,
+    BLANK = 4,
+    NIL = 5,
+};
+
+struct status_node {
+    int rank;
+    struct idset *subtree_ranks;
+    const char *status;
+    double duration;
+    bool ghost;
+    enum connector connector;
+    flux_error_t error;
+};
+
+struct trace_ctx {
+    int color;
+    int nodeid;
+    struct flux_match match;
+    int delta;
+    double last_sec;
+    double last_timestamp;
+};
+
+static json_t *overlay_topology;
+static struct hostlist *overlay_hostmap;
+
+typedef bool (*map_f)(struct status *ctx,
+                      struct status_node *node,
+                      bool parent,
+                      int level);
+
+static int status_prefix_push (struct status *ctx, enum connector c)
+{
+    if (zlistx_add_end (ctx->stack, int2ptr (c)) == NULL)
+        return -1;
+    return 0;
+}
+
+static void status_prefix_pop (struct status *ctx)
+{
+    if (zlistx_last (ctx->stack))
+        zlistx_detach_cur (ctx->stack);
+}
+
+static json_t *get_topology (flux_t *h)
+{
+    if (!overlay_topology) {
+        flux_future_t *f;
+
+        if (!(f = flux_rpc_pack (h,
+                                 "overlay.topology",
+                                 0,
+                                 0,
+                                 "{s:i}",
+                                 "rank", 0))
+            || flux_rpc_get_unpack (f, "O", &overlay_topology) < 0)
+            log_err_exit ("error fetching overlay topology");
+
+        flux_future_destroy (f);
+    }
+    return overlay_topology;
+}
+
+static struct hostlist *get_hostmap (flux_t *h)
+{
+    if (!overlay_hostmap) {
+        const char *s;
+        struct hostlist *hl;
+
+        if (!(s = flux_attr_get (h, "hostlist"))
+            || !(hl = hostlist_decode (s)))
+            log_err_exit ("could not fetch/decode hostlist");
+        overlay_hostmap = hl;
+    }
+    return overlay_hostmap;
+}
+
+static const char *status_duration (struct status *ctx, double since)
+{
+    char dbuf[128];
+    static char buf[256];
+
+    if (ctx->verbose < 1
+        || since <= 0.
+        || fsd_format_duration (dbuf, sizeof (dbuf), since) < 0)
+        return "";
+    snprintf (buf, sizeof (buf), " for %s", dbuf);
+    return buf;
+}
+
+static const char *status_colorize (struct status *ctx,
+                                    const char *status,
+                                    bool ghost)
+{
+    static char buf[128];
+
+    if (ctx->color) {
+        if (streq (status, "lost") && !ghost) {
+            snprintf (buf, sizeof (buf), "%s%s%s",
+                      ANSI_COLOR_RED, status, ANSI_COLOR_DEFAULT);
+            status = buf;
+        }
+        else if (streq (status, "offline") && !ghost) {
+            snprintf (buf, sizeof (buf), "%s%s%s",
+                      ANSI_COLOR_YELLOW, status, ANSI_COLOR_DEFAULT);
+            status = buf;
+        }
+        else if (ghost) {
+            snprintf (buf, sizeof (buf), "%s%s%s",
+                      ANSI_COLOR_DARK_GRAY, status, ANSI_COLOR_DEFAULT);
+            status = buf;
+        }
+    }
+    return status;
+}
+
+static const char *connector_string (enum connector c)
+{
+    switch (c) {
+        case PIPE:
+            return "│  ";
+        case TEE:
+            return "├─ ";
+        case ELBOW:
+            return "└─ ";
+        case BLANK:
+            return "   ";
+        case NIL:
+            return "";
+    }
+    return "";
+}
+
+static const char *status_indent (struct status *ctx, int n)
+{
+    void *val;
+    static char buf[1024];
+    int nleft = sizeof (buf) - 1;
+    size_t size = zlistx_size (ctx->stack);
+    int i;
+
+    if (optparse_hasopt (ctx->opt, "no-pretty") || n == 0)
+        return "";
+
+    buf[0] = '\0';
+    i = 0;
+    val = zlistx_first (ctx->stack);
+    while (val) {
+        /*  Only print up to the penultimate connector on the stack.
+         *  The final connector is for the next level down.
+         */
+        if (i++ == size - 1)
+            break;
+        (void) strncat (buf, connector_string (ptr2int (val)), nleft);
+        nleft -= 4;
+        val = zlistx_next (ctx->stack);
+    }
+    return buf;
+}
+
+/* Return string containing hostname and rank.
+ */
+static const char *status_getname (struct status *ctx,
+                                   struct status_node *node)
+{
+    static char buf[128];
+    const char * highlight_start = "";
+    const char * highlight_end = "";
+
+    /*  Highlight name if colorized output is enabled and this rank's
+     *   subtree (when known) intersects requested highlighted ranks:
+     */
+    if (node->subtree_ranks
+        && idset_has_intersection (ctx->highlight, node->subtree_ranks)) {
+        highlight_start = ctx->color ? ANSI_COLOR_BOLD_BLUE : "<<";
+        highlight_end = ctx->color ? ANSI_COLOR_RESET : ">>";
+    }
+
+    snprintf (buf,
+              sizeof (buf),
+              "%s%d %s%s",
+              highlight_start,
+              node->rank,
+              flux_get_hostbyrank (ctx->h, node->rank),
+              highlight_end);
+    return buf;
+}
+
+/* If verbose >= 2, return string containing parenthesised elapsed
+ * time since last RPC was started, with leading space.
+ * Otherwise, return the empty string
+ */
+static const char *status_rpctime (struct status *ctx)
+{
+    static char buf[64];
+    if (ctx->verbose < 2)
+        return "";
+    snprintf (buf, sizeof (buf), " (%.3f ms)", monotime_since (ctx->start));
+    return buf;
+}
+
+static void status_print (struct status *ctx,
+                          struct status_node *node,
+                          bool parent,
+                          int level)
+{
+    enum connector connector = optparse_hasopt (ctx->opt, "no-pretty") ?
+                               0 :
+                               node->connector;
+    printf ("%s%s%s: %s%s%s%s%s\n",
+            status_indent (ctx, level),
+            connector_string (connector),
+            status_getname (ctx, node),
+            status_colorize (ctx, node->status, node->ghost),
+            status_duration (ctx, node->duration),
+            strlen (node->error.text) > 0 ? " " : "",
+            node->error.text,
+            parent ? status_rpctime (ctx) : "");
+}
+
+static void status_print_noname (struct status *ctx,
+                                 struct status_node *node,
+                                 bool parent,
+                                 int level)
+{
+    printf ("%s%s%s%s%s\n",
+            status_indent (ctx, level),
+            connector_string (node->connector),
+            status_colorize (ctx, node->status, node->ghost),
+            status_duration (ctx, node->duration),
+            parent ? status_rpctime (ctx) : "");
+}
+
+/* Recursive function to walk 'topology', adding all subtree ranks to 'ids'.
+ * Returns 0 on success, -1 on failure (errno is not set).
+ *
+ * Note: Lifted directly from src/broker/groups.c
+ */
+static int add_subtree_ids (struct idset *ids, json_t *topology)
+{
+    int rank;
+    json_t *a;
+    size_t index;
+    json_t *entry;
+
+    if (json_unpack (topology, "{s:i s:o}", "rank", &rank, "children", &a) < 0
+        || idset_set (ids, rank) < 0)
+        return -1;
+    json_array_foreach (a, index, entry) {
+        if (add_subtree_ids (ids, entry) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+/*  Return an idset of ranks included in subtree 'topology'
+ *   (including root rank).
+ */
+static struct idset *topology_subtree_ranks (json_t *topology)
+{
+    struct idset *ids;
+
+    if (!topology)
+        return NULL;
+
+    if (!(ids = idset_create (0, IDSET_FLAG_AUTOGROW))
+        || add_subtree_ids (ids, topology))
+        goto error;
+    return ids;
+error:
+    idset_destroy (ids);
+    return NULL;
+}
+
+/*  Return the subtree topology rooted at 'subtree_rank'.
+ */
+static json_t *get_subtree_topology (json_t *topo, int subtree_rank)
+{
+    int rank;
+    json_t *a;
+    json_t *result;
+    size_t index;
+    json_t *entry;
+
+    if (json_unpack (topo, "{s:i s:o}", "rank", &rank, "children", &a) < 0)
+        return NULL;
+    if (rank == subtree_rank)
+        return topo;
+    json_array_foreach (a, index, entry) {
+        if (json_unpack (entry, "{s:i}", "rank", &rank) < 0)
+            return NULL;
+        if (rank == subtree_rank)
+            return entry;
+        else if ((result = get_subtree_topology (entry, subtree_rank)))
+            return result;
+    }
+    return NULL;
+}
+
+/*  Return an idset of all ranks in the topology subtree rooted at 'rank'.
+ */
+static struct idset *subtree_ranks (flux_t *h, int rank)
+{
+    json_t *topo;
+    json_t *topology = get_topology (h);
+
+    if (!(topo = get_subtree_topology (topology, rank))) {
+        log_err ("get_subtree_topology");
+        return NULL;
+    }
+    return topology_subtree_ranks (topo);
+}
+
+/* Walk a "ghost" subtree from the fixed topology.  Each node is assumed to
+ * have the same 'status' as the offline/lost parent at the subtree root.
+ * This augments healthwalk() to fill in nodes that would otherwise be missing
+ * because they don't have a direct parent that is online for probing.
+ * N.B. the starting point, the rank at the root of topo, is assumed to
+ * have already been mapped/iterated over.
+ */
+static void status_ghostwalk (struct status *ctx,
+                              json_t *topo,
+                              int level,
+                              const char *status,
+                              enum connector connector,
+                              map_f fun)
+{
+    json_t *children;
+    size_t index;
+    size_t total;
+    json_t *entry;
+    struct status_node node = {
+        .status = status,
+        .duration = -1., // invalid - don't print
+        .ghost = true,
+        .connector = connector,
+        .subtree_ranks = NULL,
+    };
+
+    if (json_unpack (topo, "{s:o}", "children", &children) < 0)
+        return;
+    total = json_array_size (children);
+    json_array_foreach (children, index, entry) {
+        if (json_unpack (entry, "{s:i}", "rank", &node.rank) < 0)
+            return;
+        if (index == total - 1) {
+            status_prefix_push (ctx, BLANK);
+            node.connector = ELBOW;
+        }
+        else {
+            status_prefix_push (ctx, PIPE);
+            node.connector = TEE;
+        }
+        if (!(node.subtree_ranks = topology_subtree_ranks (entry)))
+            log_err_exit ("Unable to get subtree ranks for rank %d",
+                          node.rank);
+        if (fun (ctx, &node, false, level + 1))
+            status_ghostwalk (ctx,
+                              entry,
+                              level + 1,
+                              status,
+                              node.connector,
+                              fun);
+        status_prefix_pop (ctx);
+        idset_destroy (node.subtree_ranks);
+    }
+}
+
+static double time_now (flux_t *h)
+{
+    return flux_reactor_now (flux_get_reactor (h));
+}
+
+static flux_future_t *health_rpc (flux_t *h,
+                                  int rank,
+                                  const char *wait,
+                                  double timeout)
+{
+    flux_future_t *f;
+    double start = time_now (h);
+    const char *status;
+    int rpc_flags = 0;
+
+    if (wait)
+        rpc_flags |= FLUX_RPC_STREAMING;
+
+    if (!(f = flux_rpc (h,
+                        "overlay.health",
+                        NULL,
+                        rank,
+                        rpc_flags)))
+        return NULL;
+
+    do {
+        if (flux_future_wait_for (f, timeout - (time_now (h) - start)) < 0
+            || flux_rpc_get_unpack (f, "{s:s}", "status", &status) < 0) {
+            flux_future_destroy (f);
+            return NULL;
+        }
+        if (!wait || streq (wait, status))
+            break;
+        flux_future_reset (f);
+    } while (1);
+
+    return f;
+}
+
+/* Execute fun() for each online broker in subtree rooted at 'rank'.
+ * If fun() returns true, follow tree to the broker's children.
+ * If false, don't go down that path.
+ */
+static int status_healthwalk (struct status *ctx,
+                              int rank,
+                              int level,
+                              enum connector connector,
+                              map_f fun)
+{
+    struct status_node node = {
+        .ghost = false,
+        .connector = connector,
+        .rank = rank
+    };
+    flux_future_t *f;
+    json_t *children;
+    const char *errstr;
+    int rc = 0;
+
+    monotime (&ctx->start);
+    node.subtree_ranks = NULL;
+
+    if (!(f = health_rpc (ctx->h, rank, ctx->wait, ctx->timeout))
+        || flux_rpc_get_unpack (f,
+                                "{s:i s:s s:f s:o}",
+                                "rank", &node.rank,
+                                "status", &node.status,
+                                "duration", &node.duration,
+                                "children", &children) < 0) {
+        errstr = future_strerror (f, errno);
+        /* RPC failed.
+         * An error at level 0 should be fatal, e.g. unknown wait argument,
+         * bad rank, timeout.  An error at level > 0 should return -1 so
+         * ghostwalk() can be tried (parent hasn't noticed child crash?)
+         * and sibling subtrees can be probed.
+         */
+        if (level == 0)
+            log_msg_exit ("%s", errstr);
+        printf ("%s%s%s: %s%s\n",
+                status_indent (ctx, level),
+                connector_string (connector),
+                status_getname (ctx, &node),
+                errstr,
+                status_rpctime (ctx));
+        rc = -1;
+        goto done;
+    }
+    node.subtree_ranks = subtree_ranks (ctx->h, node.rank);
+    if (fun (ctx, &node, true, level)) {
+        if (children) {
+            size_t index;
+            size_t total;
+            json_t *entry;
+            struct status_node child = { .ghost = false };
+            json_t *topo;
+
+            total = json_array_size (children);
+            json_array_foreach (children, index, entry) {
+                const char *error = NULL;
+                if (json_unpack (entry,
+                                 "{s:i s:s s:f s?s}",
+                                 "rank", &child.rank,
+                                 "status", &child.status,
+                                 "duration", &child.duration,
+                                 "error", &error) < 0)
+                    log_msg_exit ("error parsing child array entry");
+                errprintf (&child.error, "%s", error ? error : "");
+                if (!(child.subtree_ranks = subtree_ranks (ctx->h, child.rank)))
+                      log_err_exit ("Unable to get subtree idset for rank %d",
+                                    child.rank);
+                if (index == total - 1) {
+                    status_prefix_push (ctx, BLANK);
+                    connector = child.connector = ELBOW;
+                }
+                else {
+                    status_prefix_push (ctx, PIPE);
+                    connector = child.connector = TEE;
+                }
+                if (fun (ctx, &child, false, level + 1)) {
+                    if (streq (child.status, "offline")
+                        || streq (child.status, "lost")
+                        || status_healthwalk (ctx, child.rank,
+                                              level + 1, connector, fun) < 0) {
+                        if (optparse_hasopt (ctx->opt, "no-ghost"))
+                            topo = NULL;
+                        else if ((topo = get_topology (ctx->h)))
+                            topo = get_subtree_topology (topo, child.rank);
+                        status_ghostwalk (ctx,
+                                          topo,
+                                          level + 1,
+                                          child.status,
+                                          connector,
+                                          fun);
+                    }
+                }
+                status_prefix_pop (ctx);
+                idset_destroy (child.subtree_ranks);
+            }
+        }
+    }
+done:
+    idset_destroy (node.subtree_ranks);
+    flux_future_destroy (f);
+    return rc;
+}
+
+/* map fun - print the first entry without adornment and stop the walk.
+ */
+static bool show_top (struct status *ctx,
+                      struct status_node *node,
+                      bool parent,
+                      int level)
+{
+    status_print_noname (ctx, node, parent, level);
+    return false;
+}
+
+/* map fun - only follow degraded/partial, but print all non-full nodes.
+ */
+static bool show_badtrees (struct status *ctx,
+                           struct status_node *node,
+                           bool parent,
+                           int level)
+{
+    int have_children = false;
+
+    if (streq (node->status, "full"))
+        return false;
+    if (idset_count (node->subtree_ranks) > 1)
+        have_children = true;
+    if (parent
+        || streq (node->status, "lost")
+        || streq (node->status, "offline")
+        || (!have_children && ctx->verbose < 2))
+        status_print (ctx, node, parent, level);
+    return have_children || ctx->verbose >= 2;
+}
+
+/* map fun - follow all live brokers and print everything
+ */
+static bool show_all (struct status *ctx,
+                      struct status_node *node,
+                      bool parent,
+                      int level)
+{
+    int have_children = false;
+
+    if (idset_count (node->subtree_ranks) > 1)
+        have_children = true;
+    if (parent
+        || streq (node->status, "lost")
+        || streq (node->status, "offline")
+        || (!have_children && ctx->verbose < 2))
+        status_print (ctx, node, parent, level);
+    return have_children || ctx->verbose >= 2;
+}
+
+static bool validate_wait (const char *wait)
+{
+    if (wait
+        && !streq (wait, "full")
+        && !streq (wait, "partial")
+        && !streq (wait, "degraded")
+        && !streq (wait, "lost")
+        && !streq (wait, "offline"))
+        return false;
+    return true;
+}
+
+// N.B. used by both status and trace subcommands
+static int status_use_color (optparse_t *p)
+{
+    const char *when;
+    int color;
+
+    if (!(when = optparse_get_str (p, "color", "auto")))
+        when = "always";
+    if (streq (when, "always"))
+        color = 1;
+    else if (streq (when, "never"))
+        color = 0;
+    else if (streq (when, "auto"))
+        color = isatty (STDOUT_FILENO) ? 1 : 0;
+    else
+        log_msg_exit ("Invalid argument to --color: '%s'", when);
+    return color;
+}
+
+static struct idset *highlight_ranks (struct status *ctx, optparse_t *p)
+{
+    const char *arg;
+    struct idset *ids;
+    struct idset *diff = NULL;
+    struct idset *allranks = NULL;
+    uint32_t size;
+
+    if (flux_get_size (ctx->h, &size) < 0)
+        log_err_exit ("flux_get_size");
+
+    if (!(allranks = idset_create (0, IDSET_FLAG_AUTOGROW))
+        || idset_range_set (allranks, 0, size - 1) < 0
+        || !(ids = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        log_err_exit ("Failed to create highlight idset");
+
+    optparse_getopt_iterator_reset (p, "highlight");
+    while ((arg = optparse_getopt_next (p, "highlight"))) {
+        flux_error_t error;
+        struct idset *idset;
+        char *result;
+
+        /*  First, attempt to decode as idset. If that fails, assume
+         *   a hostlist was provided and lookup using the hostmap.
+         */
+        if (!(idset = idset_decode (arg))) {
+            if (!(result = flux_hostmap_lookup (ctx->h, arg, &error)))
+                log_msg_exit ("Error decoding %s: %s", arg, error.text);
+            if (!(idset = idset_decode (result)))
+                log_err_exit ("Unable to decode %s", arg);
+            free (result);
+        }
+
+        /*  Accumulate ids in result idset
+         */
+        if (idset_add (ids, idset) < 0)
+            log_err_exit ("Failed to append %s to highlight idset", arg);
+        idset_destroy (idset);
+    }
+
+    /*  Fail with error if any ranks in returned idset will fall outside
+     *   the range 0-size.
+     */
+    if (!(diff = idset_difference (ids, allranks)))
+        log_err_exit ("Failed to determine validity of highlight idset");
+    if (idset_count (diff) > 0)
+        log_msg_exit ("--highlight: rank%s %s not in set %s",
+                      idset_count (diff) > 1 ? "s" : "",
+                      idset_encode (diff, IDSET_FLAG_RANGE),
+                      idset_encode (allranks, IDSET_FLAG_RANGE));
+
+    idset_destroy (diff);
+    idset_destroy (allranks);
+    return ids;
+}
+
+static int subcmd_status (optparse_t *p, int ac, char *av[])
+{
+    int rank = optparse_get_int (p, "rank", 0);
+    struct status ctx;
+    map_f fun;
+
+    ctx.h = builtin_get_flux_handle (p);
+    ctx.verbose = optparse_get_int (p, "verbose", 0);
+    ctx.color = status_use_color (p);
+    ctx.timeout = optparse_get_duration (p, "timeout", default_timeout);
+    if (ctx.timeout == 0)
+        ctx.timeout = -1.0; // disabled
+    ctx.opt = p;
+    ctx.wait = optparse_get_str (p, "wait", NULL);
+    if (!validate_wait (ctx.wait))
+        log_msg_exit ("invalid --wait state");
+    if (!(ctx.stack = zlistx_new ()))
+        log_msg_exit ("failed to create status zlistx");
+    if (!(ctx.highlight = highlight_ranks (&ctx, p)))
+        log_msg_exit ("failed to create highlight idset");
+
+    if (optparse_hasopt (p, "summary"))
+        fun = show_top;
+    else if (optparse_hasopt (p, "down"))
+        fun = show_badtrees;
+    else
+        fun = show_all;
+
+    status_healthwalk (&ctx, rank, 0, NIL, fun);
+
+    zlistx_destroy (&ctx.stack);
+    idset_destroy (ctx.highlight);
+    return 0;
+}
+
+// zhashx_destructor_fn footprint
+static void idset_destructor (void **item)
+{
+    if (*item) {
+        idset_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static struct idset *errhash_add_entry (zhashx_t *errhash,
+                                        const char *error)
+{
+    struct idset *entry;
+    if (!(entry = zhashx_lookup (errhash, error))) {
+        if (!(entry = idset_create (0, IDSET_FLAG_AUTOGROW)))
+            return NULL;
+        if (zhashx_insert (errhash, error, entry) < 0) {
+            idset_destroy (entry);
+            errno = ENOMEM;
+            return NULL;
+        }
+    }
+    return entry;
+}
+
+static int errhash_add_one (zhashx_t *errhash,
+                            int rank,
+                            const char *error)
+{
+    struct idset *entry;
+    if (!(entry = errhash_add_entry (errhash, error)))
+        return -1;
+    if (idset_set (entry, rank) < 0)
+        return -1;
+    return 0;
+}
+
+static int errhash_add_set (zhashx_t *errhash,
+                            const struct idset *ranks,
+                            const char *error)
+{
+    if (ranks && idset_count (ranks) > 0) {
+        struct idset *entry;
+        if (!(entry = errhash_add_entry (errhash, error))
+            || idset_add (entry, ranks) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+int errhash_add_children (zhashx_t *errhash, int rank, flux_t *h)
+{
+    struct idset *ranks = subtree_ranks (h, rank);
+    int rc;
+
+    idset_clear (ranks, rank);
+    rc = errhash_add_set (errhash, ranks, "lost parent");
+    idset_destroy (ranks);
+    return rc;
+}
+
+void gather_errors (flux_t *h,
+                    int rank,
+                    zhashx_t *errhash,
+                    double timeout)
+{
+    flux_future_t *f;
+    json_t *children;
+    size_t index;
+    json_t *value;
+
+    if (!(f = health_rpc (h, rank, NULL, timeout))
+        || flux_rpc_get_unpack (f,
+                                "{s:o}",
+                                "children", &children) < 0) {
+        const char *error = future_strerror (f, errno);
+        if (errhash_add_one (errhash, rank, error) < 0
+            || errhash_add_children (errhash, rank, h) < 0)
+            log_msg_exit ("error adding to error hash");
+        goto done;
+    }
+    json_array_foreach (children, index, value) {
+        int child_rank;
+        const char *status;
+        const char *error = NULL;
+        if (json_unpack (value,
+                         "{s:i s:s s?s}",
+                         "rank", &child_rank,
+                         "status", &status,
+                         "error", &error) < 0)
+            log_msg_exit ("error parsing topology");
+        if (streq (status, "lost")) {
+            if (!error)
+                error = "unknown error";
+            // record error for child_rank and its children
+            if (errhash_add_one (errhash, child_rank, error) < 0
+                || errhash_add_children (errhash, child_rank, h) < 0)
+                log_msg_exit ("error adding to error hash");
+        }
+        else if (streq (status, "offline")) {
+            // report offline only if there is error text
+            if (error && strlen (error) > 0) {
+                if (errhash_add_one (errhash, child_rank, error) < 0)
+                    log_msg_exit ("error adding to error hash");
+            }
+        }
+        else { // recurse
+            gather_errors (h, child_rank, errhash, timeout);
+        }
+    }
+done:
+    flux_future_destroy (f);
+}
+
+void print_errors (flux_t *h, zhashx_t *errhash)
+{
+    struct idset *r;
+
+    r = zhashx_first (errhash);
+    while (r) {
+        char *ranks;
+        char *hostlist;
+
+        if (!(ranks = idset_encode (r, IDSET_FLAG_RANGE))
+            || !(hostlist = flux_hostmap_lookup (h, ranks, NULL)))
+            log_err_exit ("converting ranks to hostnames");
+        printf ("%s: %s\n",
+                hostlist,
+                (const char *)zhashx_cursor (errhash));
+        free (hostlist);
+        free (ranks);
+
+        r = zhashx_next (errhash);
+    }
+}
+
+static int subcmd_errors (optparse_t *p, int ac, char *av[])
+{
+    flux_t *h = builtin_get_flux_handle (p);
+    zhashx_t *errhash; // error string => ranks idset
+    double timeout;
+
+    timeout = optparse_get_duration (p, "timeout", default_timeout);
+    if (timeout == 0)
+        timeout = -1.0; // disabled
+
+    if (!(errhash = zhashx_new ()))
+        log_msg_exit ("could not create error hash");
+    zhashx_set_destructor (errhash, idset_destructor);
+
+    gather_errors (h, 0, errhash, timeout);
+    print_errors (h, errhash);
+
+    zhashx_destroy (&errhash);
+    return 0;
+}
+
+static int subcmd_lookup (optparse_t *p, int ac, char *av[])
+{
+    int optindex = optparse_option_index (p);
+    flux_t *h = builtin_get_flux_handle (p);
+    flux_error_t error;
+    char *s;
+
+    if (optindex != ac - 1)
+        log_msg_exit ("single TARGET argument is required");
+
+    if (!(s = flux_hostmap_lookup (h, av[optindex], &error)))
+        log_msg_exit ("%s", error.text);
+
+    printf ("%s\n", s);
+
+    free (s);
+    return 0;
+}
+
+/* Recursively search 'topo' for the parent of 'rank'.
+ * Return the parent of rank, or -1 on error.
+ */
+static int parentof (json_t *topo, int rank)
+{
+    int parent, child;
+    json_t *children;
+    size_t index;
+    json_t *value;
+
+    if (json_unpack (topo,
+                     "{s:i s:o}",
+                     "rank", &parent,
+                     "children", &children) < 0)
+        log_msg_exit ("error parsing topology");
+
+    json_array_foreach (children, index, value) {
+        if (json_unpack (value, "{s:i}", "rank", &child) < 0)
+            log_msg_exit ("error parsing topology");
+        if (child == rank)
+            return parent;
+    }
+    json_array_foreach (children, index, value) {
+        if ((parent = parentof (value, rank)) >= 0)
+            return parent;
+    }
+    return -1;
+}
+
+/* Lookup instance topology from rank 0, then search for the parent of 'rank'.
+ * Return parent or -1 on error.
+ */
+static int lookup_parentof (flux_t *h, int rank)
+{
+    json_t *topo = get_topology (h);
+    int parent, size;
+
+    /* Validate 'rank'.
+     */
+    if (json_unpack (topo,
+                     "{s:i s:i}",
+                     "rank", &parent,
+                     "size", &size) < 0)
+        log_msg_exit ("error parsing topology");
+    if (rank < 0 || rank >= size)
+        log_msg_exit ("%d is not a valid rank in this instance", rank);
+    if (rank == 0)
+        log_msg_exit ("%d has no parent", rank);
+
+    return parentof (topo, rank);
+}
+
+static int subcmd_parentof (optparse_t *p, int ac, char *av[])
+{
+    int optindex = optparse_option_index (p);
+    flux_t *h = builtin_get_flux_handle (p);
+    int rank;
+
+    if (optindex != ac - 1)
+        log_msg_exit ("RANK is required");
+    rank = strtoul (av[optindex++], NULL, 10);
+
+    printf ("%d\n", lookup_parentof (h, rank));
+
+    return 0;
+}
+
+static int subcmd_disconnect (optparse_t *p, int ac, char *av[])
+{
+    int optindex = optparse_option_index (p);
+    flux_t *h = builtin_get_flux_handle (p);
+    int parent = optparse_get_int (p, "parent", -1);
+    int rank;
+    flux_future_t *f;
+    char *endptr;
+
+    if (optindex != ac - 1)
+        log_msg_exit ("TARGET is required");
+    errno = 0;
+    rank = strtol (av[optindex], &endptr, 10);
+    if (errno != 0 || *endptr != '\0' || rank < 0) {
+        struct hostlist *hostmap = get_hostmap (h);
+        if ((rank = hostlist_find (hostmap, av[optindex])) < 0)
+            log_msg_exit ("TARGET must be a valid rank or hostname");
+    }
+
+    if (parent == -1)
+        parent = lookup_parentof (h, rank); // might return -1 (unlikely)
+
+    char *host = strdup (flux_get_hostbyrank (h, rank));
+    if (!host)
+        log_msg_exit ("out of memory");
+    log_msg ("asking %s (rank %d) to disconnect child %s (rank %d)",
+             flux_get_hostbyrank (h, parent),
+             parent,
+             host,
+             rank);
+    free (host);
+
+    if (!(f = flux_rpc_pack (h,
+                             "overlay.disconnect-subtree",
+                             parent,
+                             0,
+                             "{s:i}",
+                             "rank", rank)))
+        log_err_exit ("overlay.disconnect-subtree");
+    if (flux_rpc_get (f, NULL) < 0) {
+        log_msg_exit ("overlay.disconnect-subtree: %s",
+                      future_strerror (f, errno));
+    }
+    flux_future_destroy (f);
+
+    return 0;
+}
+
+struct typemap {
+    const char *s;
+    int type;
+};
+
+static struct typemap typemap[] = {
+    { ">", FLUX_MSGTYPE_REQUEST },
+    { "<", FLUX_MSGTYPE_RESPONSE},
+    { "e", FLUX_MSGTYPE_EVENT},
+    { "c", FLUX_MSGTYPE_CONTROL},
+};
+
+static const char *type2str (int type)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE (typemap); i++)
+        if ((type & typemap[i].type))
+            return typemap[i].s;
+    return "?";
+}
+
+enum {
+    TRACE_COLOR_EVENT = FLUX_MSGTYPE_EVENT,
+    TRACE_COLOR_REQUEST = FLUX_MSGTYPE_REQUEST,
+    TRACE_COLOR_RESPONSE = FLUX_MSGTYPE_RESPONSE,
+    TRACE_COLOR_CONTROL = FLUX_MSGTYPE_CONTROL,
+    TRACE_COLOR_TIME = 0x10,
+    TRACE_COLOR_TIMEBREAK = 0x11,
+    TRACE_COLOR_RESPONSE_FAIL = 0x12,
+};
+
+static const char *trace_colors[] = {
+    [TRACE_COLOR_EVENT]         = ANSI_COLOR_YELLOW,
+    [TRACE_COLOR_REQUEST]       = ANSI_COLOR_BOLD_BLUE,
+    [TRACE_COLOR_RESPONSE]      = ANSI_COLOR_CYAN,
+    [TRACE_COLOR_CONTROL]       = ANSI_COLOR_BOLD,
+    [TRACE_COLOR_TIME]          = ANSI_COLOR_GREEN,
+    [TRACE_COLOR_TIMEBREAK]     = ANSI_COLOR_BOLD ANSI_COLOR_GREEN,
+    [TRACE_COLOR_RESPONSE_FAIL] = ANSI_COLOR_BOLD ANSI_COLOR_RED
+};
+
+static const char *months[] = {
+    "Jan",
+    "Feb",
+    "Mar",
+    "Apr",
+    "May",
+    "Jun",
+    "Jul",
+    "Aug",
+    "Sep",
+    "Oct",
+    "Nov",
+    "Dec",
+    NULL
+};
+
+static const char *trace_color (struct trace_ctx *ctx, int type)
+{
+    if (ctx->color)
+        return trace_colors[type];
+    return "";
+}
+
+static const char *trace_color_reset (struct trace_ctx *ctx)
+{
+    if (ctx->color)
+        return ANSI_COLOR_RESET;
+    return "";
+}
+
+
+static void trace_print_human_timestamp (struct trace_ctx *ctx,
+                                         double timestamp)
+{
+
+    if ((time_t)(timestamp/60) == (time_t)(ctx->last_timestamp/60)) {
+        /* Within same minute, print offset in sec */
+        printf ("%s[%+11.6f]%s ",
+                trace_color (ctx, TRACE_COLOR_TIME),
+                timestamp - ctx->last_timestamp
+                    + fmod (ctx->last_timestamp, 60.),
+                trace_color_reset (ctx));
+    }
+    else {
+        struct tm tm;
+        time_t t = timestamp;
+
+        localtime_r (&t, &tm);
+
+        /* New minute, print datetime */
+        printf ("%s[%s%02d %02d:%02d]%s ",
+                trace_color (ctx, TRACE_COLOR_TIMEBREAK),
+                months [tm.tm_mon],
+                tm.tm_mday,
+                tm.tm_hour,
+                tm.tm_min,
+                trace_color_reset (ctx));
+        ctx->last_timestamp  = timestamp;
+    }
+}
+
+static void trace_print_human (struct trace_ctx *ctx,
+                               double timestamp,
+                               int message_type,
+                               int errnum,
+                               const char *s,
+                               const char *payload_str)
+{
+    if (message_type == FLUX_MSGTYPE_RESPONSE && errnum > 0)
+        message_type = TRACE_COLOR_RESPONSE_FAIL;
+    trace_print_human_timestamp (ctx, timestamp);
+    printf (" %s%s%s%s%s\n",
+            trace_color (ctx, message_type),
+            s,
+            payload_str ? "\n" : "",
+            payload_str ? payload_str : "",
+            trace_color_reset (ctx));
+}
+
+static void trace_print_timestamp (struct trace_ctx *ctx, double timestamp)
+{
+    time_t t = timestamp;
+    struct tm tm;
+    char buf[64] = "";
+
+    localtime_r (&t, &tm);
+
+    strftime (buf, sizeof (buf), "%Y-%m-%dT%T", &tm);
+    printf ("%s%s.%.3d%s",
+            trace_color (ctx, TRACE_COLOR_TIME),
+            buf,
+            (int)(1e3 * (timestamp - t)),
+            trace_color_reset (ctx));
+}
+
+static void trace_print (struct trace_ctx *ctx,
+                         double timestamp,
+                         int message_type,
+                         int errnum,
+                         const char *s,
+                         const char *payload_str)
+{
+    if (message_type == FLUX_MSGTYPE_RESPONSE && errnum > 0)
+        message_type = TRACE_COLOR_RESPONSE_FAIL;
+    trace_print_timestamp (ctx, timestamp);
+    printf (" %s%s%s%s%s\n",
+            trace_color (ctx, message_type),
+            s,
+            payload_str ? "\n" : "",
+            payload_str ? payload_str : "",
+            trace_color_reset (ctx));
+}
+
+static void trace_ctx_init (struct trace_ctx *ctx,
+                            optparse_t *p,
+                            int ac,
+                            char *av[])
+{
+    int n = optparse_option_index (p);
+
+    memset (ctx, 0, sizeof (*ctx));
+
+    if (n < ac - 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (optparse_hasopt (p, "delta")) {
+        if (!optparse_hasopt (p, "human"))
+            log_msg_exit ("--delta can only be used with --human");
+        ctx->delta = 1;
+    }
+    ctx->nodeid = optparse_get_int (p, "rank", FLUX_NODEID_ANY);
+    ctx->match = FLUX_MATCH_ANY;
+    if (n < ac)
+        ctx->match.topic_glob = av[n++];
+    else
+        ctx->match.topic_glob = "*";
+    ctx->color = status_use_color (p); // borrowed from status subcommand
+    if (optparse_hasopt (p, "type")) {
+        const char *arg;
+
+        optparse_getopt_iterator_reset (p, "type");
+        ctx->match.typemask = 0;
+        while ((arg = optparse_getopt_next (p, "type"))) {
+            if (streq (arg, "request"))
+                ctx->match.typemask |= FLUX_MSGTYPE_REQUEST;
+            else if (streq (arg, "response"))
+                ctx->match.typemask |= FLUX_MSGTYPE_RESPONSE;
+            else if (streq (arg, "event"))
+                ctx->match.typemask |= FLUX_MSGTYPE_EVENT;
+            else if (streq (arg, "control"))
+                ctx->match.typemask |= FLUX_MSGTYPE_CONTROL;
+            else
+                log_msg_exit ("valid types: request, response, event, control");
+        }
+    }
+}
+
+static int subcmd_trace (optparse_t *p, int ac, char *av[])
+{
+    flux_t *h = builtin_get_flux_handle (p);
+    flux_future_t *f;
+    struct trace_ctx ctx;
+
+    trace_ctx_init (&ctx, p, ac, av);
+
+    if (!(f = flux_rpc_pack (h,
+                             "overlay.trace",
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_STREAMING,
+                             "{s:i s:s s:i s:b}",
+                             "typemask", ctx.match.typemask,
+                             "topic_glob", ctx.match.topic_glob,
+                             "nodeid", ctx.nodeid,
+                             "full", optparse_hasopt (p, "full") ? 1 : 0)))
+        log_err_exit ("error sending overlay.trace request");
+    do {
+        double timestamp;
+        const char *prefix;
+        int rank;
+        int type;
+        const char *topic;
+        int payload_size;
+        json_t *payload_json = json_null ();
+        const char *payload_str = NULL;
+        char *payload_tmp_str = NULL;
+        int errnum = 0;
+        const char *errstr = NULL;
+        char buf[160];
+
+        if (flux_rpc_get_unpack (f,
+                                 "{s:F s:s s:i s:i s:s s:i s?o s?i s?s}",
+                                 "timestamp", &timestamp,
+                                 "prefix", &prefix,
+                                 "rank", &rank,
+                                 "type", &type,
+                                 "topic", &topic,
+                                 "payload_size", &payload_size,
+                                 "payload", &payload_json,
+                                 "errnum", &errnum,
+                                 "errstr", &errstr) < 0)
+            log_msg_exit ("%s", future_strerror (f, errno));
+
+        if (errnum > 0) {
+            if (errstr && strlen (errstr) > 0)
+                payload_str = errstr;
+        }
+        else if (!json_is_null (payload_json)) {
+            payload_tmp_str = json_dumps (payload_json, JSON_INDENT(2));
+            payload_str = payload_tmp_str;
+        }
+
+        char rankstr[16];
+        if (rank < 0)
+            snprintf (rankstr, sizeof (rankstr), "*");
+        else
+            snprintf (rankstr, sizeof (rankstr), "%d", rank);
+
+        snprintf (buf,
+                  sizeof (buf),
+                  "%s %s %s %s [%s]",
+                  prefix,
+                  rankstr,
+                  type2str (type),
+                  topic,
+                  encode_size (payload_size));
+
+        if (type == FLUX_MSGTYPE_RESPONSE) {
+            const char *desc = strerrorname_np (errnum);
+            size_t len = strlen (buf);
+
+            if (errnum == 0)
+                snprintf (buf + len, sizeof (buf) - len, " success");
+            else if (desc && !isdigit (desc[0]))
+                snprintf (buf + len, sizeof (buf) - len, " %s", desc);
+            else
+                snprintf (buf + len, sizeof (buf) - len, " errno %d", errnum);
+        }
+
+        if (optparse_hasopt (p, "human"))
+            trace_print_human (&ctx, timestamp, type, errnum, buf, payload_str);
+        else
+            trace_print (&ctx, timestamp, type, errnum, buf, payload_str);
+        fflush (stdout);
+
+        free (payload_tmp_str);
+
+        flux_future_reset (f);
+    } while (1);
+}
+
+int cmd_overlay (optparse_t *p, int argc, char *argv[])
+{
+    log_init ("flux-overlay");
+
+    if (optparse_run_subcommand (p, argc, argv) != OPTPARSE_SUCCESS)
+        exit (1);
+
+    hostlist_destroy (overlay_hostmap);
+
+    return (0);
+}
+
+static struct optparse_subcommand overlay_subcmds[] = {
+    { "trace",
+      "[OPTIONS] [topic-glob]",
+      "Trace messages received on overlay network",
+      subcmd_trace,
+      0,
+      trace_opts,
+    },
+    { "errors",
+      "[OPTIONS]",
+      "Summarize overlay errors",
+      subcmd_errors,
+      0,
+      errors_opts,
+    },
+    { "status",
+      "[OPTIONS]",
+      "Display overlay subtree health status",
+      subcmd_status,
+      0,
+      status_opts,
+    },
+    { "lookup",
+      "[OPTIONS] TARGET",
+      "translate rank idset to hostlist or the reverse",
+      subcmd_lookup,
+      0,
+      NULL,
+    },
+    { "parentof",
+      "[OPTIONS] RANK",
+      "show the parent of RANK",
+      subcmd_parentof,
+      0,
+      NULL,
+    },
+    { "disconnect",
+      "[OPTIONS] TARGET",
+      "disconnect a subtree rooted at TARGET (hostname or rank)",
+      subcmd_disconnect,
+      0,
+      disconnect_opts,
+    },
+    OPTPARSE_SUBCMD_END
+};
+
+
+int subcommand_overlay_register (optparse_t *p)
+{
+    optparse_err_t e;
+
+    e = optparse_reg_subcommand (p,
+        "overlay",
+        cmd_overlay,
+        NULL,
+        "Manage overlay network",
+        0,
+        NULL);
+    if (e != OPTPARSE_SUCCESS)
+        return -1;
+
+    e = optparse_reg_subcommands (optparse_get_subcommand (p, "overlay"),
+                                  overlay_subcmds);
+    return (e == OPTPARSE_SUCCESS ? 0 : -1);
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/builtin/pmi.c b/src/cmd/builtin/pmi.c
new file mode 100644
index 000000000000..a2522cd4df35
--- /dev/null
+++ b/src/cmd/builtin/pmi.c
@@ -0,0 +1,302 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "builtin.h"
+
+#include <stdlib.h>
+#include <time.h>
+#include <flux/idset.h>
+
+#include "src/common/libpmi/upmi.h"
+#include "src/common/libutil/monotime.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+static struct upmi *upmi;
+
+static int internal_cmd_get (optparse_t *p, int argc, char *argv[])
+{
+    int n = optparse_option_index (p);
+    const char *arg = optparse_get_str (p, "ranks", "0");
+    struct idset *ranks = NULL;
+    flux_error_t error;
+    struct upmi_info info;
+
+    if (!streq (arg, "all")) {
+        if (!(ranks = idset_decode (arg)))
+            log_msg_exit ("could not decode --ranks argument");
+    }
+    if (upmi_initialize (upmi, &info, &error) < 0)
+        log_msg_exit ("%s", error.text);
+    if (!ranks || idset_test (ranks, info.rank)) {
+        while (n < argc) {
+            const char *key = argv[n++];
+            char *val;
+            if (upmi_get (upmi, key, -1, &val, &error) < 0)
+                log_msg_exit ("get %s: %s", key, error.text);
+            printf ("%s\n", val);
+            free (val);
+        }
+    }
+    if (upmi_finalize (upmi, &error) < 0)
+        log_msg_exit ("finalize: %s", error.text);
+    idset_destroy (ranks);
+
+    return 0;
+}
+
+static int internal_cmd_barrier (optparse_t *p, int argc, char *argv[])
+{
+    int n = optparse_option_index (p);
+    int count = optparse_get_int (p, "count", 1);
+    int abort = optparse_get_int (p, "abort", -1);
+    struct timespec t;
+    const char *label;
+    flux_error_t error;
+    struct upmi_info info;
+
+    if (n != argc) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!(label = getenv ("FLUX_JOB_CC")))
+        if (!(label = getenv ("FLUX_JOB_ID")))
+            label = "0";
+    if (upmi_initialize (upmi, &info, &error) < 0)
+        log_msg_exit ("%s", error.text);
+
+    // don't let task launch stragglers skew timing
+    if (upmi_barrier (upmi, &error) < 0)
+        log_msg_exit ("barrier: %s", error.text);
+
+    // abort one rank if --abort was specified
+    if (abort != -1) {
+        if (info.rank == abort) {
+            flux_error_t e;
+            errprintf (&e, "flux-pmi: rank %d is aborting", info.rank);
+            if (upmi_abort (upmi, e.text, &error) < 0) {
+                log_msg_exit ("abort: %s", error.text);
+            }
+        }
+    }
+
+    while (count-- > 0) {
+        monotime (&t);
+        if (upmi_barrier (upmi, &error) < 0)
+            log_msg_exit ("barrier: %s", error.text);
+        if (info.rank == 0) {
+            printf ("%s: completed pmi barrier on %d tasks in %0.3fs.\n",
+                    label,
+                    info.size,
+                    monotime_since (t) / 1000);
+            fflush (stdout);
+        }
+    }
+
+    if (upmi_finalize (upmi, &error) < 0)
+        log_msg_exit ("finalize: %s", error.text);
+
+    return 0;
+}
+
+static int internal_cmd_exchange (optparse_t *p, int argc, char *argv[])
+{
+    int n = optparse_option_index (p);
+    int count = optparse_get_int (p, "count", 1);
+    struct timespec t;
+    const char *label;
+    flux_error_t error;
+    struct upmi_info info;
+
+    if (n != argc) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!(label = getenv ("FLUX_JOB_CC")))
+        if (!(label = getenv ("FLUX_JOB_ID")))
+            label = "0";
+    if (upmi_initialize (upmi, &info, &error) < 0)
+        log_msg_exit ("%s", error.text);
+
+    // don't let task launch stragglers skew timing
+    if (upmi_barrier (upmi, &error) < 0)
+        log_msg_exit ("barrier: %s", error.text);
+
+    while (count-- > 0) {
+        char key[64];
+        char val[64];
+
+        monotime (&t);
+
+        /* Put data from this rank
+         */
+        snprintf (key, sizeof (key), "key.%d", info.rank);
+        snprintf (val,
+                  sizeof (val),
+                  "%s-%d-%d",
+                  info.name,
+                  info.rank,
+                  info.size);
+        if (upmi_put (upmi, key, val, &error) < 0)
+            log_msg_exit ("put %s: %s", key, error.text);
+
+        /* Synchronize
+         */
+        if (upmi_barrier (upmi, &error) < 0)
+            log_msg_exit ("barrier: %s", error.text);
+
+        /* Get data from all ranks (and verify).
+         */
+        for (int rank = 0; rank < info.size; rank++) {
+            char *cp;
+
+            snprintf (key, sizeof (key), "key.%d", rank);
+            snprintf (val,
+                      sizeof (val),
+                      "%s-%d-%d",
+                      info.name,
+                      rank,
+                      info.size);
+            if (upmi_get (upmi, key, rank, &cp, &error) < 0)
+                log_msg_exit ("get %s: %s", key, error.text);
+            if (!streq (val, cp))
+                log_msg_exit ("get %s: returned unexpected value", key);
+            free (cp);
+        }
+
+        // timing must reflect completion of gets by all ranks
+        if (upmi_barrier (upmi, &error) < 0)
+            log_msg_exit ("barrier: %s", error.text);
+
+        if (info.rank == 0) {
+            printf ("%s: completed pmi exchange on %d tasks in %0.3fs.\n",
+                    label,
+                    info.size,
+                    monotime_since (t) / 1000);
+            fflush (stdout);
+        }
+    }
+
+    if (upmi_finalize (upmi, &error) < 0)
+        log_msg_exit ("finalize: %s", error.text);
+
+    return 0;
+}
+
+
+static void trace (void *arg, const char *text)
+{
+    fprintf (stderr, "%s\n", text);
+}
+
+static int cmd_pmi (optparse_t *p, int argc, char *argv[])
+{
+    const char *method = optparse_get_str (p, "method", NULL);
+    int verbose = optparse_get_int (p, "verbose", 0);
+    flux_error_t error;
+    int flags = 0;
+
+    log_init ("flux-pmi");
+
+    if (verbose > 0)
+        flags |= UPMI_TRACE;
+    if (optparse_hasopt (p, "libpmi-noflux"))
+        flags |= UPMI_LIBPMI_NOFLUX;
+    if (optparse_hasopt (p, "libpmi2-cray"))
+        flags |= UPMI_LIBPMI2_CRAY;
+    if (!(upmi = upmi_create (method, flags, trace, NULL, &error)))
+        log_msg_exit ("%s", error.text);
+
+    if (optparse_run_subcommand (p, argc, argv) != OPTPARSE_SUCCESS)
+        exit (1);
+
+    upmi_destroy (upmi);
+
+    return 0;
+}
+
+static struct optparse_option barrier_opts[] = {
+    { .name = "count",      .has_arg = 1, .arginfo = "N",
+       .usage = "Execute N barrier operations (default 1)", },
+    { .name = "abort",      .has_arg = 1, .arginfo = "RANK",
+       .usage = "RANK calls abort instead of barrier", },
+    OPTPARSE_TABLE_END,
+};
+static struct optparse_option get_opts[] = {
+    { .name = "ranks",      .has_arg = 1, .arginfo = "{IDSET|all}",
+       .usage = "Print value on specified ranks (default: 0)", },
+    OPTPARSE_TABLE_END,
+};
+static struct optparse_option exchange_opts[] = {
+    { .name = "count",      .has_arg = 1, .arginfo = "N",
+       .usage = "Execute N exchange operations (default 1)", },
+    OPTPARSE_TABLE_END,
+};
+static struct optparse_option general_opts[] = {
+    { .name = "method",      .has_arg = 1, .arginfo = "URI",
+      .usage = "Specify PMI method to use", },
+    { .name = "libpmi-noflux", .has_arg = 0,
+      .usage = "Fail if libpmi method finds the Flux libpmi.so", },
+    { .name = "libpmi2-cray", .has_arg = 0,
+      .usage = "Force-enable libpmi2 cray workarounds for testing", },
+    { .name = "verbose",    .key = 'v', .has_arg = 2, .arginfo = "[LEVEL]",
+      .usage = "Trace PMI operations", },
+    OPTPARSE_TABLE_END,
+};
+
+static struct optparse_subcommand pmi_subcmds[] = {
+    { "barrier",
+      "[OPTIONS]",
+      "Execute PMI barrier",
+      internal_cmd_barrier,
+      0,
+      barrier_opts,
+    },
+    { "get",
+      "[OPTIONS]",
+      "Get PMI KVS key",
+      internal_cmd_get,
+      0,
+      get_opts,
+    },
+    { "exchange",
+      "[OPTIONS]",
+      "Perform an allgather style exchange",
+      internal_cmd_exchange,
+      0,
+      exchange_opts,
+    },
+    OPTPARSE_SUBCMD_END
+};
+
+int subcommand_pmi_register (optparse_t *p)
+{
+    optparse_err_t e;
+
+    e = optparse_reg_subcommand (p,
+                                 "pmi",
+                                 cmd_pmi,
+                                 NULL,
+                                 "Simple PMI test client",
+                                 0,
+                                 general_opts);
+    if (e != OPTPARSE_SUCCESS)
+        return (-1);
+
+    e = optparse_reg_subcommands (optparse_get_subcommand (p, "pmi"),
+                                  pmi_subcmds);
+    return (e == OPTPARSE_SUCCESS ? 0 : -1);
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/builtin/proxy.c b/src/cmd/builtin/proxy.c
index c011e6f84b6b..f788212fe4a6 100644
--- a/src/cmd/builtin/proxy.c
+++ b/src/cmd/builtin/proxy.c
@@ -18,21 +18,27 @@
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <stdio.h>
-#include <assert.h>
+#include <signal.h>
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
 #include <libgen.h>
 #include <stdbool.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
 #include <glob.h>
-#include <czmq.h>
 #include <inttypes.h>
+#include <termios.h>
 
 #include "src/common/libutil/cleanup.h"
+#include "src/common/libutil/uri.h"
 #include "src/common/librouter/usock.h"
 #include "src/common/librouter/router.h"
 
+extern char **environ;
 
 struct proxy_command {
     struct usock_server *server;
@@ -41,16 +47,45 @@ struct proxy_command {
     flux_subprocess_t *p;
     int exit_code;
     uid_t proxy_user;
+    char *remote_uri_authority;
 };
 
 static const char *route_auxkey = "flux::route";
 
+static struct termios orig_term;
+static bool term_needs_restore = false;
+
+static void save_terminal_state (void)
+{
+    if (isatty (STDIN_FILENO)) {
+        tcgetattr (STDIN_FILENO, &orig_term);
+        term_needs_restore = true;
+    }
+}
+
+static void restore_terminal_state (void)
+{
+    if (term_needs_restore) {
+        /*
+         *  Ignore SIGTTOU so we can write to controlling terminal
+         *   as background process
+         */
+        signal (SIGTTOU, SIG_IGN);
+        tcsetattr (STDIN_FILENO, TCSADRAIN, &orig_term);
+        /* https://en.wikipedia.org/wiki/ANSI_escape_code
+         * Best effort: attempt to ensure cursor is visible:
+         */
+        printf ("\033[?25h\r\n");
+        fflush (stdout);
+        term_needs_restore = false;
+    }
+}
+
+
 static void completion_cb (flux_subprocess_t *p)
 {
     struct proxy_command *ctx = flux_subprocess_aux_get (p, "ctx");
 
-    assert (ctx);
-
     if ((ctx->exit_code = flux_subprocess_exit_code (p)) < 0) {
        /* bash standard, signals + 128 */
        if ((ctx->exit_code = flux_subprocess_signaled (p)) >= 0)
@@ -77,6 +112,7 @@ static int child_create (struct proxy_command *ctx,
         .on_stderr = NULL,
     };
     flux_cmd_t *cmd = NULL;
+    int flags = 0;
     int i;
 
     if (!shell)
@@ -109,15 +145,22 @@ static int child_create (struct proxy_command *ctx,
 
     if (flux_cmd_setenvf (cmd, 1, "FLUX_URI", "local://%s", sockpath) < 0)
         goto error;
+    if (ctx->remote_uri_authority
+        && flux_cmd_setenvf (cmd, 1,
+                             "FLUX_PROXY_REMOTE",
+                             "%s",
+                             ctx->remote_uri_authority) < 0)
+        goto error;
 
     /* We want stdio fallthrough so subprocess can capture tty if
-     * necessary (i.e. an interactive shell)
+     * necessary (i.e. an interactive shell).
      */
+    flags |= FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH;
+    flags |= FLUX_SUBPROCESS_FLAGS_NO_SETPGRP;
     if (!(p = flux_local_exec (flux_get_reactor (ctx->h),
-                               FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH,
+                               flags,
                                cmd,
-                               &ops,
-                               NULL)))
+                               &ops)))
         goto error;
 
     if (flux_subprocess_aux_set (p, "ctx", ctx, NULL) < 0)
@@ -134,7 +177,7 @@ static int child_create (struct proxy_command *ctx,
     return -1;
 }
 
-/* Usock client encouters an error.
+/* Usock client encounters an error.
  */
 static void uconn_error (struct usock_conn *uconn, int errnum, void *arg)
 {
@@ -222,34 +265,140 @@ static void acceptor_cb (struct usock_conn *uconn, void *arg)
     usock_conn_destroy (uconn);
 }
 
+/* Compare proxy version with broker version.
+ * Require major and minor to match.  Ignore patch and any git suffix.
+ */
+static void version_check (flux_t *h, bool force)
+{
+    unsigned int n[3];
+    const char *version;
+
+    if (!(version = flux_attr_get (h, "version")))
+        log_err_exit ("flux_attr_get version");
+    if (sscanf (version, "%u.%u.%u", &n[0], &n[1], &n[2]) != 3
+        || n[0] != FLUX_CORE_VERSION_MAJOR
+        || n[1] != FLUX_CORE_VERSION_MINOR) {
+        if (force) {
+            log_msg ("warning: proxy version %s may not interoperate"
+                     " with broker version %s",
+                      FLUX_CORE_VERSION_STRING,
+                      version);
+        }
+        else {
+            log_msg_exit ("fatal: proxy version %s may not interoperate"
+                          " with broker version %s "
+                          "(--force to connect anyway)",
+                          FLUX_CORE_VERSION_STRING,
+                          version);
+        }
+    }
+}
+
+static void proxy_command_destroy_usock_and_router (struct proxy_command *ctx)
+{
+    usock_server_destroy (ctx->server); // destroy before router
+    ctx->server = NULL;
+    router_destroy (ctx->router);
+    ctx->router = NULL;
+}
+
+/* Attempt to reconnect to broker.  If successful, wait for for broker to
+ * reach RUN state to avoid "Upstream broker is offline" errors when connecting
+ * early in the broker's startup.  Returns 0 on success, -1 on failure.
+ * On failure, it is safe to call this function again to retry.  This function
+ * calls exit(1) on errors that are unexpected in the reconnect context.
+ *
+ * Notes
+ * - error callback is protected against reentry
+ * - send/recv within the callback can fail as though no callback is registered
+ * - state-machine.wait RPC fails if broker is shutting down, otherwise blocks
+ * - it is safe to call flux_reconnect(3) if already connected
+ * - router_renew() re-establishes client event subs and service regs
+ */
+static int try_reconnect (flux_t *h, struct router *router)
+{
+    flux_future_t *f;
+    int rc = -1;
+
+    if (flux_reconnect (h) < 0) {
+        if (errno == ENOSYS)
+            log_msg_exit ("reconnect not implemented by connector");
+        return -1;
+    }
+    if (!(f = flux_rpc (h, "state-machine.wait", NULL, FLUX_NODEID_ANY, 0))
+        || flux_rpc_get (f, NULL) < 0) {
+        log_msg ("state-machine.wait: %s", future_strerror (f, errno));
+        goto done;
+    }
+    if (router_renew (router) < 0) {
+        log_err ("failed to restore subscriptions/service registrations");
+        goto done;
+    }
+    rc = 0;
+done:
+    flux_future_destroy (f);
+    return rc;
+}
+
+static int comms_error (flux_t *h, void *arg)
+{
+    struct proxy_command *ctx = arg;
+
+    log_msg ("broker: %s", strerror (errno));
+    log_msg ("reconnecting");
+    while (try_reconnect (h, ctx->router) < 0)
+        sleep (2);
+    log_msg ("reconnected");
+    return 0;
+}
+
 static int cmd_proxy (optparse_t *p, int ac, char *av[])
 {
-    int n;
     struct proxy_command ctx;
     const char *tmpdir = getenv ("TMPDIR");
     char workpath[PATH_MAX + 1];
     char sockpath[PATH_MAX + 1];
-    const char *uri;
+    const char *target;
+    char *uri;
     int optindex;
     flux_reactor_t *r;
+    int flags = 0;
+
+    memset (&ctx, 0, sizeof (ctx));
 
     log_init ("flux-proxy");
 
     optindex = optparse_option_index (p);
     if (optindex == ac)
         optparse_fatal_usage (p, 1, "URI argument is required\n");
-    uri = av[optindex++];
 
-    memset (&ctx, 0, sizeof (ctx));
-    if (!(ctx.h = flux_open (uri, 0)))
+    target = av[optindex++];
+    if (!(uri = uri_resolve (target, NULL)))
+        log_msg_exit ("Unable to resolve %s to a URI", target);
+
+    if (optparse_hasopt (p, "reconnect"))
+        flags |= FLUX_O_RPCTRACK;
+
+    if (!(ctx.h = flux_open (uri, flags)))
         log_err_exit ("%s", uri);
+    ctx.remote_uri_authority = uri_remote_get_authority (uri);
+    free (uri);
     flux_log_set_appname (ctx.h, "proxy");
-    ctx.proxy_user = geteuid ();
-    if (!(r = flux_reactor_create (SIGCHLD)))
+    ctx.proxy_user = getuid ();
+    if (!(r = flux_reactor_create (FLUX_REACTOR_SIGCHLD)))
         log_err_exit ("flux_reactor_create");
     if (flux_set_reactor (ctx.h, r) < 0)
         log_err_exit ("flux_set_reactor");
 
+    /* Register handler for loss of broker connection if --reconnect
+     */
+    if (optparse_hasopt (p, "reconnect"))
+        flux_comms_error_set (ctx.h, comms_error, &ctx);
+
+    /* Check proxy version vs broker version
+     */
+    version_check (ctx.h, optparse_hasopt (p, "force"));
+
     /* Create router
      */
     if (!(ctx.router = router_create (ctx.h)))
@@ -257,15 +406,20 @@ static int cmd_proxy (optparse_t *p, int ac, char *av[])
 
     /* Create socket directory.
      */
-    n = snprintf (workpath, sizeof (workpath), "%s/flux-proxy-XXXXXX",
-                             tmpdir ? tmpdir : "/tmp");
-    assert (n < sizeof (workpath));
+    if (snprintf (workpath,
+                  sizeof (workpath),
+                  "%s/flux-proxy-XXXXXX",
+                  tmpdir ? tmpdir : "/tmp") >= sizeof (workpath))
+        log_msg_exit ("TMPDIR is too long for internal buffer");
     if (!mkdtemp (workpath))
         log_err_exit ("error creating proxy socket directory");
     cleanup_push_string(cleanup_directory, workpath);
 
-    n = snprintf (sockpath, sizeof (sockpath), "%s/local", workpath);
-    assert (n < sizeof (sockpath));
+    if (snprintf (sockpath,
+                  sizeof (sockpath),
+                  "%s/local",
+                  workpath) >= sizeof (sockpath))
+        log_msg_exit ("TMPDIR is too long for internal buffer");
 
     /* Create listen socket and watcher to handle new connections
      */
@@ -281,13 +435,29 @@ static int cmd_proxy (optparse_t *p, int ac, char *av[])
 
     /* Start reactor
      */
+    save_terminal_state ();
     if (flux_reactor_run (r, 0) < 0) {
-        log_err ("flux_reactor_run");
+        if (errno == ECONNRESET)
+            log_msg ("Lost connection to Flux");
+        else
+            log_err ("flux_reactor_run");
+        if (!optparse_hasopt (p, "nohup")) {
+            log_msg ("Sending SIGHUP to child processes");
+            flux_future_destroy (flux_subprocess_kill (ctx.p, SIGHUP));
+            flux_future_destroy (flux_subprocess_kill (ctx.p, SIGCONT));
+        }
+        proxy_command_destroy_usock_and_router (&ctx);
+
+        /*
+         * Wait for child to normally terminate
+         */
+        flux_reactor_run (r, 0);
+        restore_terminal_state ();
         goto done;
     }
 done:
-    usock_server_destroy (ctx.server); // destroy before router
-    router_destroy (ctx.router);
+    proxy_command_destroy_usock_and_router (&ctx);
+    free (ctx.remote_uri_authority);
 
     if (ctx.exit_code)
         exit (ctx.exit_code);
@@ -296,15 +466,26 @@ static int cmd_proxy (optparse_t *p, int ac, char *av[])
     return (0);
 }
 
+static struct optparse_option proxy_opts[] = {
+    { .name = "force",  .key = 'f',  .has_arg = 0,
+      .usage = "Skip checks when connecting to Flux broker", },
+    { .name = "nohup",  .key = 'n',  .has_arg = 0,
+      .usage = "Do not send SIGHUP to child processes when connection"
+               " to Flux is lost", },
+    { .name = "reconnect",  .has_arg = 0,
+      .usage = "If broker connection is lost, try to reconnect", },
+    OPTPARSE_TABLE_END,
+};
+
 int subcommand_proxy_register (optparse_t *p)
 {
     optparse_err_t e;
 
     e = optparse_reg_subcommand (p, "proxy", cmd_proxy,
-        "[OPTIONS] URI [COMMAND...]",
+        "[OPTIONS] JOBID|URI [COMMAND...]",
         "Route messages to/from Flux instance",
         0,
-        NULL);
+        proxy_opts);
     if (e != OPTPARSE_SUCCESS)
         return (-1);
 
diff --git a/src/cmd/builtin/python.c b/src/cmd/builtin/python.c
index 7f9b578beb4a..f576fc5e7c8d 100644
--- a/src/cmd/builtin/python.c
+++ b/src/cmd/builtin/python.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <unistd.h>
 
 #include "builtin.h"
@@ -16,7 +19,14 @@
 
 static int cmd_python (optparse_t *p, int ac, char *av[])
 {
-    execv (PYTHON_INTERPRETER, av); /* no return if sucessful */
+    /*
+     *  Ensure av[0] matches the full path of the interpreter we're executing.
+     *  Other than just being good practice, this ensures that sys.executable
+     *  is correct (since python uses sys.argv[0] for sys.executable) and so
+     *  that symlink'd binaries in virtualenvs are respected.
+     */
+    av[0] = PYTHON_INTERPRETER;
+    execv (PYTHON_INTERPRETER, av); /* no return if successful */
     log_err_exit ("execvp (%s)", PYTHON_INTERPRETER);
     return (0);
 }
@@ -37,4 +47,3 @@ int subcommand_python_register (optparse_t *p)
 /*
  * vi: ts=4 sw=4 expandtab
  */
-
diff --git a/src/cmd/builtin/relay.c b/src/cmd/builtin/relay.c
index ccb9dd880d9e..fd5e8fc99803 100644
--- a/src/cmd/builtin/relay.c
+++ b/src/cmd/builtin/relay.c
@@ -14,7 +14,7 @@
  * children that connect to a locally provided socket, it only handles
  * one client, pre-connected on stdin, stdout.
  *
- * The ssh connectors starts flux-relay(1) remotely with ssh.
+ * The ssh connector starts flux-relay(1) remotely with ssh.
  * flux-relay(1) connects to a flux broker on the remote system.
  * The ssh connector communicates with flux-relay through stdio.
  *
@@ -36,7 +36,7 @@
 #include "builtin.h"
 #include <unistd.h>
 #include <sys/types.h>
-#include <sys/param.h>
+#include <limits.h>
 #include <stdio.h>
 #include <glob.h>
 #include <ctype.h>
@@ -45,7 +45,7 @@
 #include "src/common/librouter/usock.h"
 #include "src/common/librouter/auth.h"
 
-/* Usock client encouters an error.
+/* Usock client encounters an error.
  */
 static void uconn_error (struct usock_conn *uconn, int errnum, void *arg)
 {
@@ -109,10 +109,10 @@ static void relay (int infd, int outfd, flux_t *h)
     usock_conn_set_error_cb (uconn, uconn_error, r);
     usock_conn_set_recv_cb (uconn, uconn_recv, entry);
 
-    /* Use effective uid of the relay process as the userid for the
+    /* Use uid of the relay process as the userid for the
      * single "client" on stdin.
      */
-    cred.userid = geteuid ();
+    cred.userid = getuid ();
     cred.rolemask = FLUX_ROLE_NONE; // delegate to "upstream"
     usock_conn_accept (uconn, &cred);
 
@@ -131,8 +131,18 @@ static int cmd_relay (optparse_t *p, int ac, char *av[])
     flux_t *h;
     int optindex;
     char *uri;
-
-    log_init ("flux-relay");
+    char hostname [_POSIX_HOST_NAME_MAX + 1];
+
+    /*  If possible, initialize logging prefix as local hostname. (In the
+     *  unlikely event gethostname(3) fails, use "unknown-host".)
+     *
+     *  This will be more helpful than a literal "flux-relay" logging prefix
+     *  for end users that may be unknowingly using flux-relay as part of
+     *  the ssh connector.
+     */
+    log_init ("unknown-host");
+    if (gethostname (hostname, sizeof (hostname)) == 0)
+        log_init (hostname);
 
     optindex = optparse_option_index (p);
     if (optindex == ac)
diff --git a/src/cmd/builtin/restore.c b/src/cmd/builtin/restore.c
new file mode 100644
index 000000000000..85cd3f9b830e
--- /dev/null
+++ b/src/cmd/builtin/restore.c
@@ -0,0 +1,522 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+#if HAVE_LIBSYSTEMD
+#include <systemd/sd-daemon.h>
+#endif
+#include <unistd.h>
+#include <stdarg.h>
+#include <jansson.h>
+#include <time.h>
+#include <archive.h>
+#include <archive_entry.h>
+#ifdef HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libkvs/treeobj.h"
+#include "src/common/libkvs/kvs_checkpoint.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/blobref.h"
+#include "src/common/libcontent/content.h"
+#include "ccan/str/str.h"
+
+#include "builtin.h"
+
+#define BLOCKSIZE 10240 // taken from libarchive example
+
+static bool sd_notify_flag;
+static bool verbose;
+static bool quiet;
+static int content_flags;
+static time_t restore_timestamp;
+static int blobcount;
+static int keycount;
+static int blob_size_limit;
+
+static void progress (int delta_blob, int delta_keys)
+{
+    blobcount += delta_blob;
+    keycount += delta_keys;
+
+    if (!quiet
+        && !verbose
+        && (keycount % 100 == 0 || keycount < 10)) {
+        fprintf (stderr,
+                 "\rflux-restore: restored %d keys (%d blobs)",
+                 keycount,
+                 blobcount);
+    }
+#if HAVE_LIBSYSTEMD
+    if (sd_notify_flag
+        && (keycount % 100 == 0 || keycount < 10)) {
+        sd_notifyf (0, "EXTEND_TIMEOUT_USEC=%d", 10000000); // 10s
+        sd_notifyf (0, "STATUS=flux-restore(1) has restored %d keys", keycount);
+    }
+#endif
+}
+static void progress_end (void)
+{
+    if (!quiet && !verbose) {
+        fprintf (stderr,
+                 "\rflux-restore: restored %d keys (%d blobs)\n",
+                 keycount,
+                 blobcount);
+    }
+#if HAVE_LIBSYSTEMD
+    if (sd_notify_flag) {
+        sd_notifyf (0, "STATUS=flux-restore(1) has restored %d keys", keycount);
+    }
+#endif
+}
+
+static struct archive *restore_create (const char *infile)
+{
+    struct archive *ar;
+
+    if (!(ar = archive_read_new ()))
+        log_msg_exit ("error creating libarchive read context");
+    if (archive_read_support_format_all (ar) != ARCHIVE_OK
+        || archive_read_support_filter_all (ar) != ARCHIVE_OK)
+        log_msg_exit ("%s", archive_error_string (ar));
+    if (streq (infile, "-")) {
+        if (archive_read_open_FILE (ar, stdin) != ARCHIVE_OK)
+            log_msg_exit ("%s", archive_error_string (ar));
+    }
+    else {
+        if (archive_read_open_filename (ar, infile, BLOCKSIZE) != ARCHIVE_OK)
+            log_msg_exit ("%s", archive_error_string (ar));
+    }
+    return ar;
+}
+
+static void restore_destroy (struct archive *ar)
+{
+    if (archive_read_close (ar) != ARCHIVE_OK)
+        log_msg_exit ("%s", archive_error_string (ar));
+    archive_read_free (ar);
+}
+
+static json_t *restore_dir (flux_t *h, const char *hash_type, json_t *dir)
+{
+    json_t *data = treeobj_get_data (dir);
+    const char *name;
+    json_t *entry;
+    json_t *ndir;
+
+    if (!(ndir = treeobj_create_dir ()))
+        log_msg_exit ("out of memory");
+    json_object_foreach (data, name, entry) {
+        json_t *nentry = NULL;
+        if (treeobj_is_dir (entry)) // recurse
+            nentry = restore_dir (h, hash_type, entry);
+        if (treeobj_insert_entry_novalidate (ndir,
+                                             name,
+                                             nentry ? nentry : entry) < 0)
+            log_msg_exit ("error inserting object");
+        json_decref (nentry);
+    }
+
+    char *s;
+    flux_future_t *f;
+    const char *blobref;
+    json_t *dirref;
+
+    if (!(s = treeobj_encode (ndir)))
+        log_msg_exit ("out of memory");
+    if (!(f = content_store (h, s, strlen (s), content_flags))
+        || content_store_get_blobref (f, hash_type, &blobref) < 0)
+        log_msg_exit ("error storing dirref blob: %s",
+                      future_strerror (f, errno));
+    progress (1, 0);
+    if (!(dirref = treeobj_create_dirref (blobref)))
+        log_msg_exit ("out of memory");
+    free (s);
+    flux_future_destroy (f);
+    json_decref (ndir);
+
+    return dirref;
+}
+
+static void restore_treeobj (json_t *root, const char *path, json_t *treeobj)
+{
+    char *argz = NULL;
+    size_t argz_len = 0;
+    int count;
+    char *name;
+    json_t *dir;
+    json_t *dp;
+
+    /* Walk 'path' to the penultimate component (creating any missing dirs)
+     * and leave 'dir' pointing to it.
+     */
+    if (argz_create_sep (path, '/', &argz, &argz_len) != 0)
+        log_msg_exit ("out of memory");
+    dir = root;
+    count = argz_count (argz, argz_len);
+    name = NULL;
+    for (int i = 0; i < count - 1; i++) {
+        name = argz_next (argz, argz_len, name);
+        if (!(dp = treeobj_get_entry (dir, name))) {
+            if (!(dp = treeobj_create_dir ())
+                || treeobj_insert_entry (dir, name, dp) < 0)
+                log_msg_exit ("out of memory");
+            json_decref (dp); // treeobj_insert_entry took a ref
+        }
+        else if (!treeobj_is_dir (dp))
+            log_msg_exit ("%s in %s is not a directory", name, path);
+        dir = dp;
+    }
+    /* Insert treeobj into 'dir' under final path component.
+     */
+    name = argz_next (argz, argz_len, name);
+    if (treeobj_insert_entry (dir, name, treeobj) < 0)
+        log_err_exit ("error inserting %s into root directory", path);
+
+    free (argz);
+}
+
+static void restore_symlink (flux_t *h,
+                             json_t *root,
+                             const char *path,
+                             const char *ns_target)
+{
+    char *cpy;
+    char *cp;
+    const char *ns;
+    const char *target;
+    json_t *treeobj;
+
+    if (!(cpy = strdup (ns_target)))
+        log_msg_exit ("out of memory");
+    if ((cp = strstr (cpy, "::"))) {
+        *cp = '\0';
+        ns = cpy;
+        target = cp + 2;
+    }
+    else {
+        ns = NULL;
+        target = cpy;
+    }
+    if (!(treeobj = treeobj_create_symlink (ns, target)))
+        log_err_exit ("cannot create symlink object for %s", path);
+    restore_treeobj (root, path, treeobj);
+    json_decref (treeobj);
+    free (cpy);
+    progress (0, 1);
+}
+
+static void restore_value (flux_t *h,
+                           const char *hash_type,
+                           json_t *root,
+                           const char *path,
+                           const void *buf,
+                           int size)
+{
+    json_t *treeobj;
+
+    if (size < BLOBREF_MAX_STRING_SIZE) {
+        if (!(treeobj = treeobj_create_val (buf, size)))
+            log_err_exit ("error creating val object for %s", path);
+    }
+    else {
+        flux_future_t *f;
+        const char *blobref;
+
+        if (!(f = content_store (h, buf, size, content_flags))
+            || content_store_get_blobref (f, hash_type, &blobref) < 0)
+            log_msg_exit ("error storing blob for %s: %s",
+                          path,
+                          future_strerror (f, errno));
+        progress (1, 0);
+        if (!(treeobj = treeobj_create_valref (blobref)))
+            log_err_exit ("error creating valref object for %s", path);
+        flux_future_destroy (f);
+    }
+    restore_treeobj (root, path, treeobj);
+    json_decref (treeobj);
+    progress (0, 1);
+}
+
+/* Restore archive and return a 'dirref' object pointing to it.
+ */
+static json_t *restore_snapshot (struct archive *ar,
+                                 flux_t *h,
+                                 const char *hash_type)
+{
+    void *buf = NULL;
+    int bufsize = 0;
+    json_t *root;
+    json_t *rootref;
+
+    if (!(root = treeobj_create_dir ()))
+        log_msg_exit ("out of memory");
+
+    for (;;) {
+        struct archive_entry *entry;
+        int res;
+        const char *path;
+        mode_t type;
+        time_t mtime;
+
+        res = archive_read_next_header (ar, &entry);
+        if (res == ARCHIVE_EOF)
+            break;
+        if (res != ARCHIVE_OK)
+            log_msg_exit ("%s", archive_error_string (ar));
+        path = archive_entry_pathname (entry);
+        type = archive_entry_filetype (entry);
+        mtime = archive_entry_mtime (entry);
+
+        if (restore_timestamp < mtime)
+            restore_timestamp = mtime;
+
+        if (type == AE_IFLNK) {
+            const char *target = archive_entry_symlink (entry);
+
+            restore_symlink (h, root, path, target);
+            if (verbose)
+                fprintf (stderr, "%s -> %s\n", path, target);
+        }
+        else if (type == AE_IFREG) {
+            int size = archive_entry_size (entry);
+
+            if (blob_size_limit > 0 && size > blob_size_limit) {
+                fprintf (stderr,
+                         "%s%s size %d exceeds %d limit, skipping\n",
+                         (!quiet && !verbose) ? "\r" : "",
+                         path,
+                         size,
+                         blob_size_limit);
+                // N.B.  archive_read_next_header() skips unconsumed data
+                //   automatically so it is safe to "continue" here.
+                continue;
+            }
+            if (size > bufsize) {
+                void *newbuf;
+                if (!(newbuf = realloc (buf, size)))
+                    log_msg_exit ("out of memory");
+                buf = newbuf;
+                bufsize = size;
+            }
+            res = archive_read_data (ar, buf, bufsize);
+            if (res != size) {
+                if (res < 0)
+                    log_msg_exit ("%s", archive_error_string (ar));
+                else
+                    log_msg_exit ("short read from archive");
+            }
+            restore_value (h, hash_type, root, path, buf, size);
+            if (verbose)
+                fprintf (stderr, "%s\n", path);
+        }
+    }
+    free (buf);
+    rootref = restore_dir (h, hash_type, root);
+    json_decref (root);
+
+    return rootref;
+}
+
+/* Return the number of characters of 'blobref' that a human might want to see.
+ */
+static int shortblobref_length (const char *blobref)
+{
+    int len = 8;
+    char *cp;
+
+    if ((cp = strchr (blobref, '-')))
+        len += (cp - blobref) + 1;
+    return len;
+}
+
+static bool kvs_is_running (flux_t *h)
+{
+    flux_future_t *f;
+    bool running = true;
+
+    if ((f = flux_kvs_getroot (h, NULL, 0)) != NULL
+        && flux_rpc_get (f, NULL) < 0
+        && errno == ENOSYS)
+        running = false;
+    flux_future_destroy (f);
+    return running;
+}
+
+static void flush_content (flux_t *h, uint32_t rank)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_rpc (h, "content.flush", NULL, rank, 0))
+        || flux_rpc_get (f, NULL) < 0)
+        log_msg ("error flushing content cache: %s", future_strerror (f, errno));
+    flux_future_destroy (f);
+}
+
+static int cmd_restore (optparse_t *p, int ac, char *av[])
+{
+    int optindex =  optparse_option_index (p);
+    flux_t *h;
+    struct archive *ar;
+    const char *infile;
+    int kvs_checkpoint_flags = 0;
+    char *hash_type = "sha1";
+
+    log_init ("flux-restore");
+
+    if (optindex != ac - 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    infile = av[optindex++];
+    if (optparse_hasopt (p, "verbose"))
+        verbose = true;
+    if (optparse_hasopt (p, "quiet"))
+        quiet = true;
+    if (optparse_hasopt (p, "no-cache")) {
+        content_flags |= CONTENT_FLAG_CACHE_BYPASS;
+        kvs_checkpoint_flags |= KVS_CHECKPOINT_FLAG_CACHE_BYPASS;
+    }
+    blob_size_limit = optparse_get_size_int (p, "size-limit", "0");
+
+    h = builtin_get_flux_handle (p);
+
+    const char *s;
+    if ((s = flux_attr_get (h, "broker.sd-notify")) && !streq (s, "0"))
+        sd_notify_flag = true;
+
+    ar = restore_create (infile);
+
+    if (optparse_hasopt (p, "checkpoint")) {
+        json_t *dirref;
+        const char *blobref;
+        flux_future_t *f;
+
+        if (kvs_is_running (h))
+            log_msg_exit ("please unload kvs module before using --checkpoint");
+
+        dirref = restore_snapshot (ar, h, hash_type);
+        blobref = treeobj_get_blobref (dirref, 0);
+        progress_end ();
+
+        if (!quiet) {
+            log_msg ("writing snapshot %.*s to checkpoint for next KVS start",
+                     shortblobref_length (blobref),
+                     blobref);
+        }
+        /* restoring, therefore we restart sequence number at 0 */
+        if (!(f = kvs_checkpoint_commit (h,
+                                         NULL,
+                                         blobref,
+                                         0,
+                                         restore_timestamp,
+                                         kvs_checkpoint_flags))
+            || flux_rpc_get (f, NULL) < 0) {
+            log_msg_exit ("error updating checkpoint: %s",
+                          future_strerror (f, errno));
+        }
+        flux_future_destroy (f);
+
+        json_decref (dirref);
+    }
+    else if (optparse_hasopt (p, "key")) {
+        const char *key = optparse_get_str (p, "key", NULL);
+        json_t *dirref;
+        char *s;
+        const char *blobref;
+        flux_kvs_txn_t *txn;
+        flux_future_t *f;
+
+        dirref = restore_snapshot (ar, h, hash_type);
+        blobref = treeobj_get_blobref (dirref, 0);
+        progress_end ();
+
+        if (!quiet) {
+            log_msg ("writing snapshot %.*s to KVS key '%s'",
+                     shortblobref_length (blobref),
+                     blobref,
+                     key);
+        }
+
+        if (!(s = json_dumps (dirref, JSON_COMPACT)))
+            log_msg_exit ("error encoding final dirref object");
+
+        f = NULL;
+        if (!(txn = flux_kvs_txn_create ())
+            || flux_kvs_txn_put_treeobj (txn, 0, key, s) < 0
+            || !(f = flux_kvs_commit (h, NULL, 0, txn))
+            || flux_rpc_get (f, NULL) < 0) {
+            log_msg_exit ("error updating %s: %s",
+                          key,
+                          future_strerror (f, errno));
+        }
+        flux_future_destroy (f);
+        flux_kvs_txn_destroy (txn);
+
+        free (s);
+        json_decref (dirref);
+    }
+    else {
+        log_msg_exit ("Please specify a restore target with"
+                      " --checkpoint or --key");
+    }
+
+    if (!optparse_hasopt (p, "no-cache"))
+        flush_content (h, 0);
+
+    restore_destroy (ar);
+    flux_close (h);
+
+    return 0;
+}
+
+static struct optparse_option restore_opts[] = {
+    { .name = "verbose", .key = 'v', .has_arg = 0,
+      .usage = "List keys on stderr as they are restored",
+    },
+    { .name = "quiet", .key = 'q', .has_arg = 0,
+      .usage = "Don't show periodic progress updates",
+    },
+    { .name = "checkpoint", .has_arg = 0,
+      .usage = "Restore to checkpoint",
+    },
+    { .name = "key", .has_arg = 1,
+      .arginfo = "KEY",
+      .usage = "Restore to KVS key",
+    },
+    { .name = "no-cache", .has_arg = 0,
+      .usage = "Bypass the broker content cache",
+    },
+    { .name = "size-limit", .has_arg = 1, .arginfo = "SIZE",
+      .usage = "Do not restore blobs greater than SIZE bytes",
+    },
+    OPTPARSE_TABLE_END
+};
+
+int subcommand_restore_register (optparse_t *p)
+{
+    optparse_err_t e;
+    e = optparse_reg_subcommand (p,
+        "restore",
+        cmd_restore,
+        "[OPTIONS] INFILE",
+        "Restore KVS snapshot from a portable archive format",
+        0,
+        restore_opts);
+    return (e == OPTPARSE_SUCCESS ? 0 : -1);
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/builtin/shutdown.c b/src/cmd/builtin/shutdown.c
new file mode 100644
index 000000000000..9cbc09d1c65e
--- /dev/null
+++ b/src/cmd/builtin/shutdown.c
@@ -0,0 +1,231 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+#include <unistd.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/broker/state_machine.h"
+#include "src/common/libkvs/kvs_checkpoint.h"
+#include "src/common/libutil/uri.h"
+
+#include "builtin.h"
+
+static void get_kvs_version (flux_t *h, int *version)
+{
+    (*version) = 0;
+    if (flux_kvs_get_version (h, NULL, version) < 0
+        && errno != ENOSYS)
+        log_err_exit ("Error fetching KVS version");
+}
+
+static void get_gc_threshold (flux_t *h, int *gc_threshold)
+{
+    flux_future_t *f;
+    json_t *o;
+    (*gc_threshold) = 0;
+    if (!(f = flux_rpc (h, "config.get", NULL, FLUX_NODEID_ANY, 0))
+        || flux_rpc_get_unpack (f, "o", &o) < 0)
+        log_msg_exit ("Error fetching flux config: %s",
+                      future_strerror (f, errno));
+    (void)json_unpack (o, "{s:{s:i}}", "kvs", "gc-threshold", gc_threshold);
+}
+
+static int askyn (char *prompt, bool default_value, bool *result)
+{
+    while (1) {
+        char buf[16];
+        printf ("%s [%s]? ", prompt, default_value ? "Y/n" : "y/N");
+        fflush (stdout);
+        if (fgets (buf, sizeof (buf), stdin) == NULL)
+            return -1;
+        if (buf[0] == '\n')
+            break;
+        if (buf[0] == 'y' || buf[0] == 'Y') {
+            (*result) = true;
+            return 0;
+        }
+        if (buf[0] == 'n' || buf[0] == 'N') {
+            (*result) = false;
+            return 0;
+        }
+        printf ("Please answer y or n\n");
+    };
+    (*result) = default_value;
+    return 0;
+}
+
+static bool gc_threshold_check (flux_t *h, optparse_t *p)
+{
+    int gc_threshold, version;
+    bool rc = false;
+
+    get_kvs_version (h, &version);
+    get_gc_threshold (h, &gc_threshold);
+
+    if (gc_threshold > 0 && version > gc_threshold) {
+        if (optparse_hasopt (p, "yes")
+            || optparse_hasopt (p, "no")
+            || optparse_hasopt (p, "skip-gc")) {
+            if (optparse_hasopt (p, "yes"))
+                rc = true;
+            else
+                rc = false;
+            return rc;
+        }
+
+        if (!isatty (STDIN_FILENO))
+            log_msg_exit ("gc threshold exceeded, specify -y or -n\n");
+
+        if (askyn ("gc threshold exceeded, "
+                   "do you want to perform garbage collection",
+                   true,
+                   &rc) < 0)
+            log_msg_exit ("error retrieving user input");
+    }
+    return rc;
+}
+
+static void process_updates (flux_future_t *f)
+{
+    const char *s;
+
+    while (flux_rpc_get_unpack (f, "{s:s}", "log", &s) == 0) {
+        fprintf (stderr, "%s", s);
+        flux_future_reset (f);
+    }
+    if (errno != ENODATA)
+        log_msg_exit ("%s", future_strerror (f, errno));
+}
+
+static int subcmd (optparse_t *p, int ac, char *av[])
+{
+    flux_t *h;
+    flux_future_t *f;
+    int flags = FLUX_RPC_STREAMING;
+    int optindex = optparse_option_index (p);
+    bool quiet = optparse_hasopt (p, "quiet");
+    int verbose = optparse_get_int (p, "verbose", 0);
+    int loglevel = quiet ? LOG_WARNING
+                 : verbose == 0 ? LOG_INFO : LOG_DEBUG;
+    const char *target = NULL;
+
+    log_init ("flux-shutdown");
+
+    if (optindex < ac)
+        target = av[optindex++];
+    if (optindex != ac) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+
+    if (target) {
+        char *uri = uri_resolve (target, NULL);
+        if (!uri)
+            log_msg_exit ("failed to resolve target %s to a Flux URI", target);
+        if (!(h = flux_open (uri, 0)))
+            log_err_exit ("error connecting to Flux");
+        free (uri);
+    }
+    else {
+        if (!(h = flux_open (NULL, 0)))
+            log_err_exit ("error connecting to Flux");
+    }
+
+    if (optparse_hasopt (p, "background"))
+        flags &= ~FLUX_RPC_STREAMING;
+
+    if (optparse_hasopt (p, "skip-gc")) {
+        if (flux_attr_set (h, "content.dump", "") < 0)
+            log_err_exit ("error clearing content.dump attribute");
+    }
+
+    if (optparse_hasopt (p, "gc")
+        || optparse_hasopt (p, "dump")
+        || gc_threshold_check (h, p)) {
+        const char *val = optparse_get_str (p, "dump", "auto");
+
+        if (flux_attr_set (h, "content.dump", val) < 0)
+            log_err_exit ("error setting content.dump attribute");
+
+        log_msg ("shutdown will dump KVS (this may take some time)");
+    }
+
+    /* N.B. set nodeid=FLUX_NODEID_ANY so we get immediate error from
+     * broker if run on rank > 0.
+     */
+    if (!(f = flux_rpc_pack (h,
+                             "shutdown.start",
+                             FLUX_NODEID_ANY,
+                             flags,
+                             "{s:i}",
+                             "loglevel", loglevel)))
+        log_err_exit ("could not send shutdown.start request");
+
+    if ((flags & FLUX_RPC_STREAMING))
+        process_updates (f);
+    else if (flux_rpc_get (f, NULL) < 0)
+        log_msg_exit ("%s", future_strerror (f, errno));
+
+    flux_future_destroy (f);
+    flux_close (h);
+    return 0;
+}
+
+static struct optparse_option opts[] = {
+    { .name = "skip-gc", .has_arg = 0,
+      .usage = "Skip KVS garbage collection this time, if already enabled",
+    },
+    { .name = "gc", .has_arg = 0,
+      .usage = "Garbage collect KVS (short for --dump=auto)",
+    },
+    { .name = "dump", .has_arg = 1, .arginfo = "PATH",
+      .usage = "Dump KVS content to specified archive file using flux-dump(1)."
+    },
+    { .name = "background", .has_arg = 0,
+      .usage = "Exit the command immediately after initiating shutdown",
+    },
+    { .name = "quiet", .has_arg = 0,
+      .usage = "Show only log messages <= LOG_WARNING level",
+    },
+    { .name = "verbose", .key = 'v', .has_arg = 2, .arginfo = "[LEVEL]",
+      .usage = "Increase log verbosity:"
+               " 0=show log messages <= LOG_INFO level (default),"
+               " 1=show all log messages",
+    },
+    { .name = "yes", .key = 'y', .has_arg = 0,
+      .usage = "Answer yes to any yes/no questions",
+    },
+    { .name = "no", .key = 'n', .has_arg = 0,
+      .usage = "Answer no to any yes/no questions",
+    },
+    OPTPARSE_TABLE_END
+};
+
+int subcommand_shutdown_register (optparse_t *p)
+{
+    optparse_err_t e;
+
+    e = optparse_reg_subcommand (p,
+        "shutdown",
+        subcmd,
+        "[OPTIONS] [TARGET]",
+        "Shut down the Flux instance",
+        0,
+        opts);
+    if (e != OPTPARSE_SUCCESS)
+        return -1;
+    return 0;
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/builtin/startlog.c b/src/cmd/builtin/startlog.c
new file mode 100644
index 000000000000..ba585967e550
--- /dev/null
+++ b/src/cmd/builtin/startlog.c
@@ -0,0 +1,280 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+#include <unistd.h>
+#include <stdarg.h>
+#include <jansson.h>
+#include <time.h>
+
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libkvs/treeobj.h"
+#include "src/common/libkvs/kvs_checkpoint.h"
+#include "src/common/libutil/fsd.h"
+#include "ccan/str/str.h"
+
+#include "builtin.h"
+
+#define DEFAULT_STARTLOG_KEY "admin.eventlog"
+#define DEFAULT_STARTLOG_VERSION 1
+
+static const char *startlog_key;
+static int startlog_version;
+
+enum post_flags {
+    POST_FLAG_FLUSH = 1,
+};
+
+static void post_startlog_event (flux_t *h,
+                                 enum post_flags flags,
+                                 const char *name,
+                                 const char *fmt, ...)
+{
+    va_list ap;
+    json_t *o;
+    char *event;
+    flux_kvs_txn_t *txn;
+    flux_future_t *f;
+    uint32_t rank;
+    int commit_flags = 0;
+
+    if (flux_get_rank (h, &rank) < 0)
+        log_err_exit ("Error fetching rank");
+    if (rank > 0)
+        log_msg_exit ("Startlog events may only be posted from rank 0");
+
+    va_start (ap, fmt);
+    o = eventlog_entry_vpack (0., name, fmt, ap);
+    va_end (ap);
+
+    if (!o
+        || !(event = eventlog_entry_encode (o))
+        || !(txn = flux_kvs_txn_create ())
+        || flux_kvs_txn_put (txn,
+                             FLUX_KVS_APPEND,
+                             startlog_key,
+                             event) < 0)
+        log_err_exit ("Error creating %s event", name);
+
+    if (flags == POST_FLAG_FLUSH)
+        commit_flags |= FLUX_KVS_SYNC;
+
+    if (!(f = flux_kvs_commit (h, NULL, commit_flags, txn))
+        || flux_rpc_get (f, NULL) < 0)
+        log_msg_exit ("Error committing %s event: %s",
+                      name,
+                      future_strerror (f, errno));
+
+    flux_future_destroy (f);
+    flux_kvs_txn_destroy (txn);
+    free (event);
+    json_decref (o);
+}
+
+static int startlog_parse_event (json_t *entry,
+                                 double *timestampp,
+                                 const char **namep,
+                                 json_t **contextp)
+{
+    double timestamp;
+    const char *name;
+    json_t *context;
+    int version;
+
+    if (eventlog_entry_parse (entry, &timestamp, &name, &context) < 0
+        || json_unpack (context, "{s:i}", "version", &version) < 0
+        || version < 0
+        || version > startlog_version)
+        return -1;
+    if (timestampp)
+        *timestampp = timestamp;
+    if (namep)
+        *namep = name;
+    if (contextp)
+        *contextp = context;
+    return 0;
+}
+
+static void startlog_post_start_event (flux_t *h, optparse_t *p)
+{
+    post_startlog_event (h,
+                         POST_FLAG_FLUSH,
+                         "start",
+                         "{s:i s:s}",
+                         "version", startlog_version,
+                         "core_version", flux_core_version_string ());
+}
+
+static void startlog_post_finish_event (flux_t *h, optparse_t *p)
+{
+    post_startlog_event (h,
+                         0,
+                         "finish",
+                         "{s:i}",
+                         "version", startlog_version);
+}
+
+static json_t *startlog_fetch (flux_t *h)
+{
+    flux_future_t *f;
+    const char *raw;
+    json_t *event_array;
+
+    if (!(f = flux_kvs_lookup (h, NULL, 0, startlog_key))
+        || flux_kvs_lookup_get (f, &raw) < 0)
+        log_msg_exit ("Error fetching eventlog: %s",
+                      future_strerror (f, errno));
+    if (!(event_array = eventlog_decode (raw)))
+        log_err_exit ("Error decoding eventlog");
+    flux_future_destroy (f);
+    return event_array;
+}
+
+static int format_timestamp (char *buf, size_t size, time_t t)
+{
+    struct tm tm;
+    if (t < 0 || !localtime_r (&t, &tm))
+        return -1;
+    if (strftime (buf, size, "%Y-%m-%d %R", &tm) == 0)
+        return -1;
+    return 0;
+}
+
+/* Interpret startlog and list instance restart durations.
+ */
+static void startlog_list (flux_t *h, optparse_t *p)
+{
+    json_t *event_array = startlog_fetch (h);
+    bool check = optparse_hasopt (p, "check");
+    bool quiet = optparse_hasopt (p, "quiet");
+    bool crashed = false;
+    size_t index;
+    json_t *entry;
+    double timestamp;
+    const char *name;
+    json_t *context;
+    char timebuf[64];
+    bool started = false;
+    double last_timestamp = 0;
+    char fsd[64];
+
+    json_array_foreach (event_array, index, entry) {
+        if (startlog_parse_event (entry, &timestamp, &name, &context) < 0)
+            continue; // ignore (but tolerate) non-conforming entries
+        format_timestamp (timebuf, sizeof (timebuf), timestamp);
+        if (streq (name, "start")) {
+            if (started) {
+                if (!quiet)
+                    printf ("crashed\n");
+                crashed = true;
+            }
+            if (!quiet) {
+                const char *s = "";
+                if (optparse_hasopt (p, "show-version")) {
+                    (void)json_unpack (context, "{s:s}", "core_version", &s);
+                    printf ("%25s  ", s);
+                }
+                printf ("%s - ", timebuf);
+            }
+            started = true;
+        }
+        else if (streq (name, "finish")) {
+            if (started) {
+                fsd_format_duration_ex (fsd,
+                                        sizeof (fsd),
+                                        timestamp - last_timestamp,
+                                        2);
+                if (!quiet)
+                    printf ("%s (%s)\n", timebuf, fsd);
+                crashed = false;
+            }
+            started = false;
+        }
+        last_timestamp = timestamp;
+    }
+    if (started) {
+        flux_reactor_t *r = flux_get_reactor (h);
+        fsd_format_duration_ex (fsd,
+                                sizeof (fsd),
+                                flux_reactor_now (r) - last_timestamp,
+                                2);
+        if (!quiet)
+            printf ("running (%s)\n", fsd);
+    }
+    if (check && crashed)
+        exit (1);
+}
+
+static int cmd_startlog (optparse_t *p, int ac, char *av[])
+{
+    flux_t *h = builtin_get_flux_handle (p);
+
+    startlog_key = optparse_get_str (p,
+                                     "test-startlog-key",
+                                     DEFAULT_STARTLOG_KEY);
+    startlog_version = optparse_get_int (p,
+                                         "test-startlog-version",
+                                         DEFAULT_STARTLOG_VERSION);
+    if (optparse_hasopt (p, "post-start-event"))
+        startlog_post_start_event (h, p);
+    else if (optparse_hasopt (p, "post-finish-event"))
+        startlog_post_finish_event (h, p);
+    else
+        startlog_list (h, p);
+    return 0;
+}
+
+static struct optparse_option startlog_opts[] = {
+    { .name = "check", .has_arg = 0,
+      .usage = "Check if instance was properly shut down",
+    },
+    { .name = "quiet", .has_arg = 0,
+      .usage = "Suppress listing, useful with --check",
+    },
+    { .name = "show-version", .key = 'v', .has_arg = 0,
+      .usage = "Show the flux-core version string in output",
+    },
+    { .name = "post-start-event", .has_arg = 0,
+      .usage = "Post start event to eventlog (for rc use only)",
+      .flags = OPTPARSE_OPT_HIDDEN,
+    },
+    { .name = "post-finish-event", .has_arg = 0,
+      .usage = "Post finish event to eventlog (for rc use only)",
+      .flags = OPTPARSE_OPT_HIDDEN,
+    },
+    { .name = "test-startlog-key", .has_arg = 1, .arginfo = "PATH",
+      .usage = "override startlog key (test only)",
+      .flags = OPTPARSE_OPT_HIDDEN,
+    },
+    { .name = "test-startlog-version", .has_arg = 1, .arginfo = "VERSION",
+      .usage = "override startlog version (test only)",
+      .flags = OPTPARSE_OPT_HIDDEN,
+    },
+    OPTPARSE_TABLE_END
+};
+
+int subcommand_startlog_register (optparse_t *p)
+{
+    optparse_err_t e;
+    e = optparse_reg_subcommand (p,
+        "startlog",
+        cmd_startlog,
+        "[OPTIONS]",
+        "List Flux instance startlog",
+        0,
+        startlog_opts);
+    return (e == OPTPARSE_SUCCESS ? 0 : -1);
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/builtin/uptime.c b/src/cmd/builtin/uptime.c
new file mode 100644
index 000000000000..bf6436d59dc1
--- /dev/null
+++ b/src/cmd/builtin/uptime.c
@@ -0,0 +1,343 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+#include <unistd.h>
+#include <sys/types.h>
+#include <pwd.h>
+#include <time.h>
+#include <jansson.h>
+
+#include "src/common/libidset/idset.h"
+#include "src/common/libutil/fsd.h"
+#include "ccan/str/str.h"
+
+#include "builtin.h"
+
+/* Fetch the named broker group idset membership, and return the
+ * number of members.
+ */
+static int groups_get_count (flux_t *h, const char *name)
+{
+    flux_future_t *f;
+    const char *members;
+    struct idset *ids;
+    int count;
+
+    if (!(f = flux_rpc_pack (h, "groups.get", 0, 0, "{s:s}", "name", name))
+        || flux_rpc_get_unpack (f, "{s:s}", "members", &members) < 0
+        || !(ids = idset_decode (members)))
+        log_err_exit ("Error fetching %s group", name);
+    count = idset_count (ids);
+    idset_destroy (ids);
+    flux_future_destroy (f);
+    return count;
+}
+
+static void get_queue_status (flux_t *h,
+                              const char *name,
+                              bool *enablep,
+                              bool *startp)
+{
+    flux_future_t *f;
+    const char *topic = "job-manager.queue-status";
+    int enable, start;
+
+    if (name)
+        f = flux_rpc_pack (h, topic, 0, 0, "{s:s}", "name", name);
+    else
+        f = flux_rpc_pack (h, topic, 0, 0, "{}");
+    if (!f || flux_rpc_get_unpack (f, "{s:b s:b}",
+                                   "enable", &enable,
+                                   "start", &start) < 0)
+        log_err_exit ("Error fetching queue status: %s",
+                      future_strerror (f, errno));
+    (*enablep) = enable ? true : false;
+    (*startp) = start ? true : false;
+    flux_future_destroy (f);
+}
+
+/* Return state of job submission and queue allocation
+ * If there are multiple queues, return true only if ALL queues are
+ * disabled / stopped.
+ */
+static void queue_status (flux_t *h, bool *disabledp, bool *stoppedp)
+{
+    flux_future_t *f;
+    bool disabled = true;
+    bool stopped = true;
+    json_t *queues;
+    size_t index;
+    json_t *value;
+    bool enable, start;
+
+    f = flux_rpc (h, "job-manager.queue-list", NULL, 0, 0);
+    if (!f || flux_rpc_get_unpack (f, "{s:o}", "queues", &queues))
+        log_msg_exit ("queue-list: %s", future_strerror (f, errno));
+    if (json_array_size (queues) > 0) {
+        json_array_foreach (queues, index, value) {
+            get_queue_status (h, json_string_value (value), &enable, &start);
+            if (enable)
+                disabled = false;
+            if (start)
+                stopped = false;
+            if (!disabled && !stopped)
+                break;
+        }
+    }
+    else {
+        get_queue_status (h, NULL, &enable, &start);
+        if (enable)
+            disabled = false;
+        if (start)
+            stopped = false;
+    }
+    flux_future_destroy (f);
+    (*disabledp) = disabled;
+    (*stoppedp) = stopped;
+}
+
+/* Each key in the drain object is an idset representing a group
+ * of drained nodes.  Sum the member count of all idsets and set 'countp'.
+ * Return 0 on success, -1 on failure.
+ */
+static int parse_drain_object (json_t *drain, int *countp)
+{
+    const char *key;
+    json_t *value;
+    struct idset *ids;
+    int count = 0;
+
+    json_object_foreach (drain, key, value) {
+        if (!(ids = idset_decode (key)))
+            return -1;
+        count += idset_count (ids);
+        idset_destroy (ids);
+    }
+    *countp = count;
+    return 0;
+}
+
+/* Get the number of drained nodes.
+ */
+static int resource_status_drained (flux_t *h)
+{
+    flux_future_t *f;
+    int count;
+    json_t *drain;
+
+    if (!(f = flux_rpc (h, "resource.status", NULL, 0, 0))
+        || flux_rpc_get_unpack (f, "{s:o}", "drain", &drain) < 0
+        || parse_drain_object (drain, &count) < 0)
+        log_err_exit ("Error fetching resource status");
+    flux_future_destroy (f);
+    return count;
+}
+
+/* Fetch an attribute and return its value as integer.
+ */
+static int attr_get_int (flux_t *h, const char *name)
+{
+    const char *s;
+    char *endptr;
+    int i;
+
+    if (!(s = flux_attr_get (h, name)))
+        log_err_exit ("Error fetching %s attribute", name);
+    errno = 0;
+    i = strtol (s, &endptr, 10);
+    if (errno != 0 || *endptr != '\0')
+        log_msg_exit ("Error parsing %s", name);
+    return i;
+}
+
+/* Fetch broker.starttime from rank 0 and return its value as double.
+ */
+static double attr_get_starttime (flux_t *h)
+{
+    const char *name = "broker.starttime";
+    flux_future_t *f;
+    const char *s;
+    char *endptr;
+    double d;
+
+    if (!(f = flux_rpc_pack (h, "attr.get", 0, 0, "{s:s}", "name", name))
+        || flux_rpc_get_unpack (f, "{s:s}", "value", &s) < 0)
+        log_err_exit ("Error fetching %s attribute", name);
+    errno = 0;
+    d = strtod (s, &endptr);
+    if (errno != 0 || *endptr != '\0')
+        log_msg_exit ("Error parsing %s", name);
+    flux_future_destroy (f);
+    return d;
+}
+
+/* Format seconds-since-epoch time in HH:MM:SS (24-hour) format.
+ */
+static int format_time (char *buf, size_t len, double t)
+{
+    struct tm tm;
+    time_t tt = t;
+
+    if (localtime_r (&tt, &tm) == NULL)
+        return -1;
+    if (strftime (buf, len, "%T", &tm) == 0)
+        return -1;
+    return 0;
+}
+
+/* Get the username for 'uid'.  If that fails, convert uid to string.
+ */
+static int format_user (char *buf, size_t len, int uid)
+{
+    char pwbuf[16384];
+    struct passwd pwd;
+    struct passwd *result;
+    int n;
+
+    if (getpwuid_r (uid, &pwd, pwbuf, sizeof (pwbuf), &result) < 0)
+        n = snprintf (buf, len, "%d", uid);
+    else
+        n = snprintf (buf, len, "%s", result->pw_name);
+    if (n >= len)
+        return -1;
+    return 0;
+}
+
+/* Append ",  name" to buf if condition is true.
+ * Ignore truncation errors.
+ */
+static void append_if (char *buf,
+                       size_t len,
+                       const char *name,
+                       bool condition)
+{
+    if (condition) {
+        int n = strlen (buf);
+        snprintf (buf + n,
+                  len - n,
+                  "%s%s",
+                  n > 0 ? ",  " : "",
+                  name);
+    }
+}
+
+/* Append ",  count name" to buf if condition is true
+ * Ignore truncation errors.
+ */
+static void append_count_if (char *buf,
+                             size_t len,
+                             const char *name,
+                             int count,
+                             bool condition)
+{
+    if (condition) {
+        int n = strlen (buf);
+        snprintf (buf + n,
+                  len - n,
+                  ",  %d %s",
+                  count,
+                  name);
+    }
+}
+
+/* Mimic uptime(1), sort of.
+ */
+static void default_summary (flux_t *h)
+{
+    double t_now = flux_reactor_now (flux_get_reactor (h));
+    int userid = attr_get_int (h, "security.owner");
+    int size = attr_get_int (h, "size");
+    int level = attr_get_int (h, "instance-level");
+    int drained = 0;
+    int offline = 0;
+    bool submit_is_disabled = false;
+    bool sched_is_stopped = false;
+    flux_future_t *f;
+    const char *broker_state;
+    double duration;
+    char fsd[32];
+    char now[32];
+    char owner[32];
+    char extra[512] = "";
+
+    /* Fetch the broker state.
+     * If it is "run", proceed to fetch info from high level services and
+     * the rank 0 broker and set duration to the instance runtime.
+     * Otherwise, an abbreviated set of info will be displayed and let the
+     * duration reflect the local broker's time in the current state.
+     */
+    if (!(f = flux_rpc (h, "state-machine.get", NULL, FLUX_NODEID_ANY, 0))
+        || flux_rpc_get_unpack (f,
+                                "{s:s s:f}",
+                                "state", &broker_state,
+                                "duration", &duration) < 0)
+        log_err_exit ("Error fetching broker state");
+    if (streq (broker_state, "run")) {
+        duration = t_now - attr_get_starttime (h);
+        drained = resource_status_drained (h);
+        offline = size - groups_get_count (h, "broker.online");
+        queue_status (h, &submit_is_disabled, &sched_is_stopped);
+    }
+    if (fsd_format_duration_ex (fsd, sizeof (fsd), duration, 2) < 0)
+        log_err_exit ("Error formatting uptime duration");
+    if (format_time (now, sizeof (now), t_now) < 0)
+        log_msg_exit ("Error formatting current time");
+    if (format_user (owner, sizeof (owner), userid) < 0)
+        log_msg_exit ("Error formatting instance owner");
+    append_count_if (extra, sizeof (extra), "drained", drained, drained > 0);
+    append_count_if (extra, sizeof (extra), "offline", offline, offline > 0);
+    printf (" %s %s %s,  owner %s,  depth %d,  size %d%s\n",
+            now,
+            broker_state,
+            fsd,
+            owner,
+            level,
+            size,
+            extra);
+
+    /* optional 2nd line for submit disabled/sched stopped
+     */
+    extra[0] = '\0';
+    append_if (extra, sizeof (extra), "submit disabled", submit_is_disabled);
+    append_if (extra, sizeof (extra), "scheduler stopped", sched_is_stopped);
+    if (strlen (extra) > 0)
+        printf ("  %s\n", extra);
+
+    flux_future_destroy (f);
+}
+
+static int cmd_uptime (optparse_t *p, int ac, char *av[])
+{
+    flux_t *h = builtin_get_flux_handle (p);
+
+    default_summary (h);
+
+    return (0);
+}
+
+int subcommand_uptime_register (optparse_t *p)
+{
+    optparse_err_t e;
+    e = optparse_reg_subcommand (p,
+        "uptime",
+        cmd_uptime,
+        NULL,
+        "Show how long this Flux instance has been running",
+        0,
+        NULL);
+    return (e == OPTPARSE_SUCCESS ? 0 : -1);
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/builtin/user.c b/src/cmd/builtin/user.c
deleted file mode 100644
index d47a5752ed3a..000000000000
--- a/src/cmd/builtin/user.c
+++ /dev/null
@@ -1,363 +0,0 @@
-/************************************************************\
- * Copyright 2017 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include "builtin.h"
-
-#include <unistd.h>
-#include <inttypes.h>
-#include <sys/types.h>
-#include <pwd.h>
-#include <argz.h>
-
-typedef struct {
-    const char *name;
-    uint32_t value;
-} role_t;
-
-static role_t roletab[] = {
-    { "owner",    FLUX_ROLE_OWNER },
-    { "user",     FLUX_ROLE_USER },
-    { NULL, 0 },
-};
-
-const char *unknown_role_msg = "Valid roles are owner, user.";
-
-static const char *rolestr (uint32_t rolemask, char *s, size_t len)
-{
-    const char *ret = s;
-    role_t *rp = &roletab[0];
-
-    if (rolemask == 0) {
-        snprintf (s, len, "none");
-        goto done;
-    }
-    while ((rp->name != NULL)) {
-        if ((rolemask & rp->value)) {
-            snprintf (s, len, "%s%s", ret < s ? "," : "", rp->name);
-            len -= strlen (s);
-            s += strlen (s);
-        }
-        rp++;
-    }
-done:
-    return ret;
-}
-
-static int strrole (const char *s, uint32_t *rolemask)
-{
-    role_t *rp = &roletab[0];
-
-    while ((rp->name != NULL)) {
-        if (!strcmp (rp->name, s)) {
-            *rolemask |= rp->value;
-            return 0;
-        }
-        rp++;
-    }
-    return -1;
-}
-
-static int parse_rolemask_string (const char *s, uint32_t *rolemask)
-{
-    char *argz = NULL;
-    size_t argz_len;
-    int e;
-    char *entry  = NULL;
-    uint32_t mask = FLUX_ROLE_NONE;
-    int rc = -1;
-
-    if ((e = argz_create_sep (s, ',', &argz, &argz_len)) != 0) {
-        errno = e;
-        goto done;
-    }
-    while ((entry = argz_next (argz, argz_len, entry))) {
-        if (strrole (entry, &mask) < 0)
-            goto done;
-    }
-    *rolemask = mask;
-    rc = 0;
-done:
-    free (argz);
-    return rc;
-}
-
-static void delrole (flux_t *h, uint32_t userid, uint32_t rolemask)
-{
-    flux_future_t *f;
-    uint32_t final;
-    char s[256];
-
-    f = flux_rpc_pack (h, "userdb.delrole", FLUX_NODEID_ANY, 0,
-                       "{s:i s:i}", "userid", userid,
-                                "rolemask", rolemask);
-    if (!f)
-        log_err_exit ("userdb.delrole");
-    if (flux_rpc_get_unpack (f, "{s:i s:i}", "userid", &userid,
-                             "rolemask", &final) < 0) {
-        if (errno == ENOSYS)
-            log_msg_exit ("userdb module is not loaded");
-        if (errno == ENOENT)
-            log_msg_exit ("No such user: %" PRIu32, userid);
-        log_err_exit ("userdb.delrole");
-    }
-    printf ("%" PRIu32 ":%s\n", userid, rolestr (final, s, sizeof (s)));
-    flux_future_destroy (f);
-}
-
-static void addrole (flux_t *h, uint32_t userid, uint32_t rolemask)
-{
-    flux_future_t *f;
-    uint32_t final;
-    char s[256];
-
-    f = flux_rpc_pack (h, "userdb.addrole", FLUX_NODEID_ANY, 0,
-                       "{s:i s:i}", "userid", userid,
-                       "rolemask", rolemask);
-    if (!f)
-        log_err_exit ("userdb.addrole");
-    if (flux_rpc_get_unpack (f, "{s:i s:i}",
-                                "userid", &userid,
-                                "rolemask", &final) < 0) {
-        if (errno == ENOSYS)
-            log_msg_exit ("userdb module is not loaded");
-        if (errno == ENOENT)
-            log_msg_exit ("No such user: %" PRIu32, userid);
-        log_err_exit ("userdb.addrole");
-    }
-    printf ("%" PRIu32 ":%s\n", userid, rolestr (final, s, sizeof (s)));
-    flux_future_destroy (f);
-}
-
-static uint32_t lookup_user (const char *name)
-{
-    struct passwd pwd, *result;
-    long bufsize = sysconf(_SC_GETPW_R_SIZE_MAX);
-    char *buf;
-    int e;
-    uint32_t userid = FLUX_USERID_UNKNOWN;
-
-    if (bufsize == -1)
-        bufsize = 16384;        /* Should be more than enough */
-    buf = xzmalloc (bufsize);
-    e = getpwnam_r (name, &pwd, buf, bufsize, &result);
-    if (result == NULL) {
-        if (e == 0)
-            log_msg_exit ("%s: unknown user", name);
-        else
-            log_errn_exit (e, "%s", name);
-    }
-    userid = result->pw_uid;
-    free (buf);
-    return userid;
-}
-
-static int internal_user_list (optparse_t *p, int ac, char *av[])
-{
-    int n;
-    flux_t *h;
-    flux_future_t *f;
-    uint32_t userid;
-    uint32_t rolemask;
-    char s[256];
-
-    n = optparse_option_index (p);
-    if (n != ac) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (!(h = builtin_get_flux_handle (p)))
-        log_err_exit ("flux_open");
-    for (;;) {
-        f = flux_rpc (h, "userdb.getnext", NULL, FLUX_NODEID_ANY, 0);
-        if (!f)
-            log_err_exit ("userdb.getnext");
-        if (flux_rpc_get_unpack (f, "{s:i s:i}", "userid", &userid,
-                                                 "rolemask", &rolemask) < 0) {
-            if (errno == ENOSYS)
-                log_msg_exit ("userdb module is not loaded");
-            if (errno != ENOENT)
-                log_err_exit ("userdb.getnext");
-            flux_future_destroy (f);
-            break;
-        }
-        printf ("%" PRIu32 ":%s\n",
-                userid, rolestr (rolemask, s, sizeof (s)));
-
-        flux_future_destroy (f);
-    }
-    flux_close (h);
-    return (0);
-}
-
-static int internal_user_lookup (optparse_t *p, int ac, char *av[])
-{
-    int n;
-    flux_t *h;
-    flux_future_t *f;
-    uint32_t userid;
-    uint32_t rolemask;
-    char *endptr;
-    char s[256];
-
-    n = optparse_option_index (p);
-    if (n != ac - 1) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    userid = strtoul (av[n], &endptr, 10);
-
-    if (*endptr != '\0')
-        userid = lookup_user (av[n]);
-    if (userid == FLUX_USERID_UNKNOWN)
-        log_msg_exit ("%s: invalid userid", av[n]);
-    if (!(h = builtin_get_flux_handle (p)))
-        log_err_exit ("flux_open");
-    f = flux_rpc_pack (h, "userdb.lookup", FLUX_NODEID_ANY, 0,
-                       "{s:i}", "userid", userid);
-    if (!f)
-        log_err_exit ("userdb.lookup");
-    if (flux_rpc_get_unpack (f, "{s:i s:i}", "userid", &userid,
-                                             "rolemask", &rolemask) < 0) {
-        if (errno == ENOSYS)
-            log_msg_exit ("userdb module is not loaded");
-        if (errno == ENOENT)
-            log_msg_exit ("No such user: %" PRIu32, userid);
-        log_err_exit ("userdb.lookup");
-    }
-    printf ("%" PRIu32 ":%s\n",
-            userid, rolestr (rolemask, s, sizeof (s)));
-    flux_future_destroy (f);
-    flux_close (h);
-    return (0);
-}
-
-static int internal_user_addrole (optparse_t *p, int ac, char *av[])
-{
-    int n;
-    flux_t *h;
-    uint32_t userid, rolemask;
-    char *endptr;
-
-    n = optparse_option_index (p);
-    if (n != ac - 2) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    userid = strtoul (av[n], &endptr, 10);
-    if (*endptr != '\0')
-        userid = lookup_user (av[n]);
-    if (userid == FLUX_USERID_UNKNOWN)
-        log_msg_exit ("%s: invalid userid", av[n]);
-    n++;
-    rolemask = strtoul (av[n], &endptr, 0);
-    if (*endptr != '\0') {
-        if (parse_rolemask_string (av[n], &rolemask) < 0)
-            log_err_exit ("%s: invalid rolemask", av[n]);
-    }
-    if (rolemask == FLUX_ROLE_NONE)
-        log_msg_exit ("%s: invalid rolemask", av[n]);
-    if (!(h = builtin_get_flux_handle (p)))
-        log_err_exit ("flux_open");
-    addrole (h, userid, rolemask);
-    flux_close (h);
-    return (0);
-}
-
-static int internal_user_delrole (optparse_t *p, int ac, char *av[])
-{
-    int n;
-    flux_t *h;
-    uint32_t userid, rolemask;
-    char *endptr;
-
-    n = optparse_option_index (p);
-    if (n != ac - 2) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    userid = strtoul (av[n], &endptr, 10);
-    if (*endptr != '\0')
-        userid = lookup_user (av[n]);
-    if (userid == FLUX_USERID_UNKNOWN)
-        log_msg_exit ("%s: invalid userid", av[n]);
-    n++;
-    rolemask = strtoul (av[n], &endptr, 0);
-    if (*endptr != '\0') {
-        if (parse_rolemask_string (av[n], &rolemask) < 0)
-            log_err_exit ("%s: invalid rolemask", av[n]);
-    }
-    if (rolemask == FLUX_ROLE_NONE)
-        log_msg_exit ("%s: invalid rolemask", av[n]);
-    if (!(h = builtin_get_flux_handle (p)))
-        log_err_exit ("flux_open");
-    delrole (h, userid, rolemask);
-    flux_close (h);
-    return (0);
-}
-
-
-int cmd_user (optparse_t *p, int ac, char *av[])
-{
-    log_init ("flux-user");
-
-    if (optparse_run_subcommand (p, ac, av) != OPTPARSE_SUCCESS)
-        exit (1);
-    return (0);
-}
-
-static struct optparse_subcommand user_subcmds[] = {
-    { "list",
-      "",
-      "List users and their assigned roles",
-      internal_user_list,
-      0,
-      NULL,
-    },
-    { "lookup",
-      "USERID",
-      "Lookup roles assigned to USERID",
-      internal_user_lookup,
-      0,
-      NULL,
-    },
-    { "addrole",
-      "USERID role[,role,...]",
-      "Add roles to USERID",
-      internal_user_addrole,
-      0,
-      NULL,
-    },
-    { "delrole",
-      "USERID role[,role,...]",
-      "Remove roles from USERID",
-      internal_user_delrole,
-      0,
-      NULL,
-    },
-    OPTPARSE_SUBCMD_END
-};
-
-int subcommand_user_register (optparse_t *p)
-{
-    optparse_err_t e;
-
-    e = optparse_reg_subcommand (p,
-            "user", cmd_user, NULL, "Access user database", 0, NULL);
-    if (e != OPTPARSE_SUCCESS)
-        return (-1);
-
-    e = optparse_reg_subcommands (optparse_get_subcommand (p, "user"),
-                                  user_subcmds);
-    return (e == OPTPARSE_SUCCESS ? 0 : -1);
-}
-
-/*
- * vi: ts=4 sw=4 expandtab
- */
diff --git a/src/cmd/builtin/version.c b/src/cmd/builtin/version.c
index 2f005d8396f3..248535f01fa5 100644
--- a/src/cmd/builtin/version.c
+++ b/src/cmd/builtin/version.c
@@ -8,7 +8,11 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include <config.h>
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+#include <hwloc.h>
+#include <zmq.h>
 
 #include "builtin.h"
 #if HAVE_FLUX_SECURITY_VERSION_H
@@ -48,15 +52,27 @@ static int cmd_version (optparse_t *p, int ac, char *av[])
 #endif
     print_broker_version (p);
     printf ("build-options:\t\t");
+#if ASSUME_BROKEN_LOCALE
+    printf("+ascii-only");
+#endif
 #if __SANITIZE_ADDRESS__
     printf ("+asan");
 #endif
 #if __SANITIZE_THREAD__
     printf ("+tsan");
 #endif
-#if HAVE_CALIPER
-    printf ("+caliper");
+#if HAVE_LIBSYSTEMD
+    printf ("+systemd");
 #endif
+    printf ("+hwloc==%d.%d.%d",
+            HWLOC_API_VERSION >> 16 & 0x000000ff,
+            HWLOC_API_VERSION >>  8 & 0x000000ff,
+            HWLOC_API_VERSION       & 0x000000ff
+            );
+    printf ("+zmq==%d.%d.%d",
+            ZMQ_VERSION_MAJOR,
+            ZMQ_VERSION_MINOR,
+            ZMQ_VERSION_PATCH);
     printf ("\n");
     return (0);
 }
diff --git a/src/cmd/cmdhelp.c b/src/cmd/cmdhelp.c
index 482c47bed837..a02d4539c531 100644
--- a/src/cmd/cmdhelp.c
+++ b/src/cmd/cmdhelp.c
@@ -13,54 +13,17 @@
 #endif
 #include <glob.h>
 #include <string.h>
-#include <czmq.h>
+#ifdef HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
 #include <jansson.h>
 
 #include "src/common/libutil/log.h"
-#include "src/common/libutil/xzmalloc.h"
-#include "src/common/libutil/sds.h"
 
 #include "cmdhelp.h"
 
-struct cmdhelp {
-    sds cmd;
-    sds description;
-};
-
-static struct cmdhelp *cmdhelp_create (const char *cmd, const char *desc)
-{
-    struct cmdhelp *ch = xzmalloc (sizeof (*ch));
-    ch->cmd = sdsnew (cmd);
-    ch->description = sdsnew (desc);
-    if (!ch->cmd || !ch->description) {
-        free (ch);
-        return (NULL);
-    }
-    return (ch);
-}
-
-static void cmdhelp_destroy (struct cmdhelp **p)
-{
-    if (*p) {
-        struct cmdhelp *ch = *p;
-        sdsfree (ch->cmd);
-        sdsfree (ch->description);
-        free (ch);
-        *p = NULL;
-    }
-}
-
-static void cmd_list_destroy (zlist_t *zl)
-{
-    struct cmdhelp *ch;
-    ch = zlist_first (zl);
-    while (ch) {
-        cmdhelp_destroy (&ch);
-        ch = zlist_next (zl);
-    }
-    zlist_destroy (&zl);
-}
-
 static json_t *command_list_file_read (const char *path)
 {
     json_t *o;
@@ -79,46 +42,43 @@ static json_t *command_list_file_read (const char *path)
     return (o);
 }
 
-static int command_list_read (zhash_t *h, const char *path)
+static int command_list_print (FILE *fp, const char *path)
 {
-    int i;
-    int rc = -1;
-    int n = 0;
+    int rc = 0;
+    size_t index;
     json_t *o = NULL;
+    json_t *entry;
 
     if (!(o = command_list_file_read (path)))
         goto out;
 
-    n = json_array_size (o);
-    for (i = 0; i < n; i++) {
-        const char *category;
-        const char *command;
+    json_array_foreach (o, index, entry) {
+        size_t i;
         const char *description;
-        json_t *entry;
-        zlist_t *zl;
-
-        if (!(entry = json_array_get (o, i))) {
-            log_msg ("%s: entry %d is not an object", path, i);
-            goto out;
-        }
-        if (json_unpack (entry, "{s:s s:s s:s}",
-                                "category", &category,
-                                "command", &command,
-                                "description", &description) < 0) {
-            log_msg ("%s: Missing element in JSON entry %d", path, i);
+        json_t *commands;
+        json_t *cmd;
+        json_error_t error;
+
+        if (json_unpack_ex (entry, &error, 0,
+                            "{s:s s:o}",
+                            "description", &description,
+                            "commands", &commands) < 0) {
+            log_msg ("%s:entry %zu: %s", path, index, error.text);
             goto out;
         }
-        if (!(zl = zhash_lookup (h, category))) {
-            char *s = strdup (category);
-            if (s == NULL)
+        fprintf (fp, "\n%s\n", description);
+        json_array_foreach (commands, i, cmd) {
+            const char *name = NULL;
+            const char *desc = NULL;
+            if (json_unpack_ex (cmd, &error, 0,
+                                "{s:s s:s}",
+                                "name", &name,
+                                "description", &desc) < 0) {
+                log_msg ("%s:entry %zu.%zu: %s", path, index, i, error.text);
                 goto out;
-            zl = zlist_new ();
-            //zlist_set_destructor (zl, (czmq_destructor *) cmdhelp_destroy);
-            zhash_insert (h, s, (void *) zl);
-            zhash_freefn (h, s, (zhash_free_fn *) cmd_list_destroy);
-            free (s);
+            }
+            fprintf (fp, "   %-18s %s\n", name, desc);
         }
-        zlist_append (zl, cmdhelp_create (command, description));
     }
     rc = 0;
 
@@ -128,111 +88,46 @@ static int command_list_read (zhash_t *h, const char *path)
 
 }
 
-static void emit_command_list_category (zhash_t *zh, const char *cat, FILE *fp)
-{
-    struct cmdhelp *c;
-    zlist_t *zl = zhash_lookup (zh, cat);
-
-    fprintf (fp, "Common commands from flux-%s:\n", cat);
-    c = zlist_first (zl);
-    while (c) {
-        fprintf (fp, "   %-18s %s\n", c->cmd, c->description);
-        c = zlist_next (zl);
-    }
-}
-
-/* strcmp() ensuring "core" is always sorted to front:
- */
-static int categorycmp (const char *s1, const char *s2)
-{
-    if (strcmp (s1, "core") == 0)
-        return (-1);
-    if (strcmp (s2, "core") == 0)
-        return (1);
-    return strcmp (s1, s2);
-}
-
-#if CZMQ_VERSION < CZMQ_MAKE_VERSION(3,0,1)
-static bool category_cmp (const char *s1, const char *s2)
-{
-    return (categorycmp (s1, s2) > 0);
-}
-#else
-static int category_cmp (const char *s1, const char *s2)
-{
-    return categorycmp (s1, s2);
-}
-#endif
-
-zhash_t *get_command_list_hash (const char *pattern)
+static void emit_command_help_from_pattern (FILE *fp, const char *pattern)
 {
     int rc;
     size_t i;
     glob_t gl;
-    zhash_t *zh = NULL;
 
     rc = glob (pattern, GLOB_ERR, NULL, &gl);
     switch (rc) {
         case 0:
             break; /* have results, fall-through. */
         case GLOB_ABORTED:
-            /* No help.d directory? */
-            goto out;
-            break;
         case GLOB_NOMATCH:
-            goto out;
-            break;
+            /* No help.d directory? */
+            return;
         default:
             fprintf (stderr, "glob: unknown error %d\n", rc);
             break;
     }
 
-    zh = zhash_new ();
-    //zhash_set_destructor (zh, (czmq_destructor *) zlist_destroy);
     for (i = 0; i < gl.gl_pathc; i++) {
         const char *file = gl.gl_pathv[i];
-        if (command_list_read (zh, file) < 0)
+        if (command_list_print (fp, file) < 0)
             log_err_exit ("%s: failed to read content\n", file);
     }
     globfree (&gl);
-
-out:
-    return (zh);
 }
 
-static void emit_command_help_from_pattern (const char *pattern, FILE *fp)
+void emit_command_help (const char *plist, FILE *fp)
 {
-    zhash_t *zh = NULL;
-    zlist_t *keys = NULL;
-    const char *cat;
+    char *argz = NULL;
+    size_t argz_len = 0;
+    char *entry = NULL;
 
-    zh = get_command_list_hash (pattern);
-    if (zh == NULL)
+    if (argz_create_sep (plist, ':', &argz, &argz_len) != 0)
         return;
 
-    keys = zhash_keys (zh);
-    zlist_sort (keys, (zlist_compare_fn *) category_cmp);
+    while ((entry = argz_next (argz, argz_len, entry)))
+        emit_command_help_from_pattern (fp, entry);
 
-    cat = zlist_first (keys);
-    while (cat) {
-        emit_command_list_category (zh, cat, fp);
-        if ((cat = zlist_next (keys)))
-            fprintf (fp, "\n");
-    }
-    zlist_destroy (&keys);
-    zhash_destroy (&zh);
-    return;
-}
-
-void emit_command_help (const char *plist, FILE *fp)
-{
-    int i, count;
-    sds *p;
-    if (!(p = sdssplitlen (plist, strlen (plist), ":", 1, &count)))
-        return;
-    for (i = 0; i < count; i++)
-        emit_command_help_from_pattern (p[i], fp);
-    sdsfreesplitres (p, count);
+    free (argz);
 }
 
 /*
diff --git a/src/cmd/flux-R.c b/src/cmd/flux-R.c
new file mode 100644
index 000000000000..24f21aa1e082
--- /dev/null
+++ b/src/cmd/flux-R.c
@@ -0,0 +1,909 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-R - encode/decode and operate on RFC 20 resource set objects
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/hostlist.h>
+#include <flux/optparse.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/read_all.h"
+#include "src/common/librlist/rlist.h"
+#include "src/common/librlist/rhwloc.h"
+#include "src/common/libtomlc99/toml.h"
+#include "ccan/str/str.h"
+
+#define RSET_DOC "\
+Read, generate, and process RFC 20 Resource Set objects.\n\
+Options:"
+
+int cmd_encode (optparse_t *p, int argc, char **argv);
+int cmd_append (optparse_t *p, int argc, char **argv);
+int cmd_diff (optparse_t *p, int argc, char **argv);
+int cmd_intersect (optparse_t *p, int argc, char **argv);
+int cmd_remap (optparse_t *p, int argc, char **argv);
+int cmd_rerank (optparse_t *p, int argc, char **argv);
+int cmd_decode (optparse_t *p, int argc, char **argv);
+int cmd_verify (optparse_t *p, int argc, char **argv);
+int cmd_set_property (optparse_t *p, int argc, char **argv);
+int cmd_parse_config (optparse_t *p, int argc, char **argv);
+
+static struct optparse_option global_opts[] =  {
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option encode_opts[] = {
+    { .name = "ranks", .key = 'r',
+      .has_arg = 1, .arginfo = "IDSET",
+      .usage = "Generate an R with ranks in IDSET. If not provided then "
+               "either match the number of nodes given  in --hosts option, "
+               "or emit a single rank: \"0\"",
+    },
+    { .name = "cores", .key = 'c',
+      .has_arg = 1, .arginfo = "IDS",
+      .usage = "Assign cores with IDS to each rank in R. Default is to "
+               "assign a single core \"0\" to each rank.",
+    },
+    { .name = "gpus", .key = 'g',
+      .has_arg = 1, .arginfo = "IDS",
+      .usage = "Assign gpu resources with IDS to each rank in R. Default "
+               "is to assign no gpu resources.",
+    },
+    { .name = "hosts", .key = 'H',
+      .has_arg = 1, .arginfo = "HOSTS",
+      .usage = "Generate R with nodelist set to HOSTS. By default, duplicate "
+               "the local hostname to match the number of ranks given in "
+               "--ranks.",
+    },
+    { .name = "property", .key = 'p',
+      .has_arg = 1, .arginfo = "NAME[:RANKS]",
+      .usage = "Assign property NAME to target ranks RANKS. If RANKS is not "
+               "specified then the property applies to all defined ranks. "
+               "This option may be specified multiple times for each property",
+    },
+    { .name = "local", .key = 'l',
+      .has_arg = 0,
+      .usage = "Generate child resources from local system",
+    },
+    { .name = "xml", .key = 'f',
+      .has_arg = 1,
+      .usage = "Generate child resources from hwloc XML",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option append_opts[] = {
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option diff_opts[] = {
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option intersect_opts[] = {
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option remap_opts[] = {
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option rerank_opts[] = {
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option verify_opts[] = {
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option set_property_opts[] = {
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option decode_opts[] = {
+    { .name = "short", .key = 's',
+      .usage = "Print short-form representation of R"
+    },
+    { .name = "nodelist", .key = 'n',
+      .usage = "Print nodelist in hostlist form from R, if any"
+    },
+    { .name = "ranks", .key = 'r',
+      .usage = "Print ranks in idset form from R, if any"
+    },
+    { .name = "count", .key = 'c',
+      .has_arg = 1, .arginfo = "TYPE",
+      .usage = "Print count of resource TYPE in R (ranks, core, gpu)",
+    },
+    { .name = "include", .key = 'i',
+      .has_arg = 1, .arginfo = "RANKS",
+      .usage = "Include only specified ranks.",
+    },
+    { .name = "exclude", .key = 'x',
+      .has_arg = 1, .arginfo = "RANKS",
+      .usage = "Exclude specified ranks.",
+    },
+    { .name = "properties", .key = 'p',
+      .has_arg = 1, .arginfo = "LIST",
+      .usage = "Filter on properties"
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option parse_config_opts[] = {
+    OPTPARSE_TABLE_END
+};
+
+
+static struct optparse_subcommand subcommands[] = {
+    { "encode",
+      "[OPTIONS]...",
+      "\nEncode RFC 20 R objects for testing.\n\nOptions:\n",
+      cmd_encode,
+      0,
+      encode_opts,
+    },
+    { "append",
+      "",
+      "Append multiple R objects on stdin. "
+      "Emits an error if resource sets are not disjoint.",
+      cmd_append,
+      0,
+      append_opts,
+    },
+    { "diff",
+      "",
+      "Return the set difference of multiple R objects on stdin. "
+      "(i.e. (R1 - R2) - R3 ...)",
+      cmd_diff,
+      0,
+      diff_opts,
+    },
+    { "intersect",
+      "",
+      "Return the intersection of all R objects on stdin",
+      cmd_intersect,
+      0,
+      intersect_opts,
+    },
+    { "remap",
+      "",
+      "Return the union of all R objects on stdin with ranks re-numbered "
+      "starting from index 0.",
+      cmd_remap,
+      0,
+      remap_opts,
+    },
+    { "rerank",
+      "HOSTLIST",
+      "Return the union of all R objects on stdin with ranks re-mapped "
+      " based on their index in HOSTLIST.",
+      cmd_rerank,
+      0,
+      rerank_opts,
+    },
+    { "decode",
+      "[OPTIONS]...",
+      "\nReturn the union of all R objects on stdin and print details or "
+      "summary of the result. By default an RFC 20 JSON object is emitted "
+      "on stdout, unless one or more options below are used\n"
+      "\nOptions:\n",
+      cmd_decode,
+      0,
+      decode_opts,
+    },
+    { "verify",
+      "",
+      "Takes 2 R objects on stdin and verifies that the resources in each "
+      "rank present in R2 has at least resources present for the same rank "
+      "in R1.",
+      cmd_verify,
+      0,
+      verify_opts,
+    },
+    { "set-property",
+      "PROPERTY:RANKS [PROPERTY:RANKS]...",
+      "Set properties on R object on stdin, emitting the result on stdout",
+      cmd_set_property,
+      0,
+      set_property_opts,
+    },
+    { "parse-config",
+      "PATH",
+      "Read config from resource.config array",
+      cmd_parse_config,
+      0,
+      parse_config_opts,
+    },
+    OPTPARSE_SUBCMD_END
+};
+
+int main (int argc, char *argv[])
+{
+    optparse_t *p;
+    int optindex;
+    int exitval;
+
+    log_init ("flux-R");
+
+    p = optparse_create ("flux-R");
+
+    if (optparse_add_option_table (p, global_opts) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_add_option_table() failed");
+
+    if (optparse_add_doc (p, RSET_DOC, 0) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_add_doc failed");
+
+    if (optparse_reg_subcommands (p, subcommands) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_reg_subcommands");
+
+    if ((optindex = optparse_parse_args (p, argc, argv)) < 0)
+        exit (1);
+
+    if ((argc - optindex == 0)
+        || !optparse_get_subcommand (p, argv[optindex]))
+        optparse_fatal_usage (p, 1, NULL);
+
+    if ((exitval = optparse_run_subcommand (p, argc, argv)) < 0)
+        exit (1);
+
+    optparse_destroy (p);
+    log_fini ();
+    return (exitval);
+}
+
+static struct idset * idset_from_option (optparse_t *p,
+                                         const char *name,
+                                         const char *dflt)
+{
+    const char *s = NULL;
+    struct idset *result = NULL;
+
+    if (optparse_getopt (p, name, &s) == 0)
+        s = dflt;
+    if (!(result = idset_decode (s)))
+        log_msg_exit ("Failed to decode %s='%s' as idset", name, s);
+    return result;
+}
+
+static struct hostlist *hostlist_from_option (optparse_t *p,
+                                              const char *name,
+                                              int expected_count)
+{
+    const char *s;
+    struct hostlist *hl;
+    if (optparse_getopt (p, name, &s) == 0) {
+        char host[1024];
+        if (gethostname (host, sizeof (host)) < 0)
+            log_err_exit ("gethostname");
+
+        hl = hostlist_create ();
+        for (int i = 0; i < expected_count; i++)
+            hostlist_append (hl, host);
+        return hl;
+    }
+    if (!(hl = hostlist_decode (s)))
+        log_msg_exit ("invalid hostlist '%s'", s);
+    if (expected_count && hostlist_count (hl) != expected_count)
+        log_msg_exit ("hostname count in '%s' does not match nranks (%d)",
+                      s, expected_count);
+    return hl;
+}
+
+static void rlist_puts (struct rlist *rl)
+{
+    char *R = rlist_encode (rl);
+    puts (R);
+    free (R);
+}
+
+static struct idset *idset_from_count (int n)
+{
+    struct idset *ids = idset_create (0, IDSET_FLAG_AUTOGROW);
+    if (ids) {
+        for (int i = 0; i < n; i++)
+            idset_set (ids, i);
+    }
+    return ids;
+}
+
+static void get_ranks_and_hostlist (optparse_t *p,
+                                    struct idset **ranksp,
+                                    struct hostlist **hlp)
+{
+    if (!optparse_hasopt (p, "ranks")) {
+        /*  --ranks not provided, either build from provided hostlist or
+         *   emit a single rank.
+         */
+        if (optparse_hasopt (p, "hosts"))
+            *hlp = hostlist_from_option (p, "hosts", 0);
+        else
+            *hlp = hostlist_from_option (p, "hosts", 1);
+        *ranksp = idset_from_count (hostlist_count (*hlp));
+    }
+    else {
+        *ranksp = idset_from_option (p, "ranks", "");
+        *hlp = hostlist_from_option (p, "hosts", idset_count (*ranksp));
+    }
+}
+
+static ssize_t fread_all (const char *path, char **bufp)
+{
+    int fd;
+    ssize_t size;
+    void *buf;
+
+    if (streq (path, "-"))
+        fd = STDIN_FILENO;
+    else {
+        if ((fd = open (path, O_RDONLY)) < 0)
+            log_err_exit ("%s", path);
+    }
+    if ((size = read_all (fd, &buf)) < 0)
+        log_err_exit ("%s", path);
+    if (fd != STDIN_FILENO)
+        (void)close (fd);
+    *bufp = buf;
+    return size;
+}
+
+static char *get_xml (optparse_t *p)
+{
+    const char *path = NULL;
+    char *s = NULL;
+
+    /*  If --xml option provided, then read XML file and return contents
+     *  O/w, if --local provided, then retern XML from local topology
+     *   (this allows the program to gather hwloc topology only once)
+     */
+    if (optparse_getopt (p, "xml", &path) > 0) {
+        if (fread_all (path, &s) < 0)
+            log_err_exit ("failed to read XML");
+    }
+    else if (optparse_hasopt (p, "local")) {
+        if (!(s = rhwloc_local_topology_xml (0)))
+            log_err_exit ("failed to gather local topology XML");
+    }
+
+    return s;
+}
+
+static char *rlist_ranks_string (struct rlist *rl)
+{
+    char *s = NULL;
+    struct idset *ranks = rlist_ranks (rl);
+    if (ranks)
+        s = idset_encode (ranks, IDSET_FLAG_RANGE);
+    idset_destroy (ranks);
+    return s;
+}
+
+static void set_one_property (json_t *o, char *allranks, const char *s)
+{
+    json_t *val;
+    char *ranks;
+    char *property;
+
+    if (!(property = strdup (s)))
+        log_err_exit ("get_properties: strdup");
+    if ((ranks = strchr (property, ':')))
+        *ranks++ = '\0';
+    else
+        ranks = allranks;
+    if (!(val = json_string (ranks))
+            || json_object_set_new (o, property, val) < 0)
+        log_err_exit ("failed to set property %s=%s", property, ranks);
+
+    free (property);
+}
+
+static void set_properties (optparse_t *p, struct rlist *rl)
+{
+    const char *s;
+    char *allranks;
+    json_t *o;
+    flux_error_t err;
+
+    if (!optparse_hasopt (p, "property"))
+        return;
+
+    if (!(o = json_object ()))
+        log_err_exit ("failed to create properties JSON object");
+
+    if (!(allranks = rlist_ranks_string (rl)))
+        log_err_exit ("failed to get rank idset string");
+
+    optparse_getopt_iterator_reset (p, "property");
+    while ((s = optparse_getopt_next (p, "property")))
+        set_one_property (o, allranks, s);
+
+    if (rlist_assign_properties (rl, o, &err) < 0)
+        log_msg_exit ("failed to assign properties: %s", err.text);
+
+    json_decref (o);
+    free (allranks);
+}
+
+int cmd_encode (optparse_t *p, int argc, char **argv)
+{
+    struct hostlist *hl;
+    struct idset *ranks;
+    struct rlist *rl;
+    char *xml = NULL;
+
+    unsigned int i;
+    const char *host;
+    const char *cores;
+    const char *gpus;
+
+    gpus = optparse_get_str (p, "gpus", "");
+    cores = optparse_get_str (p, "cores", "");
+
+    /* If neither cores nor gpus were set for these ranks, default
+     *  to a single coreid 0
+     */
+    if (strlen (gpus) == 0 && strlen (cores) == 0)
+        cores = "0";
+    else if (optparse_hasopt (p, "local") || optparse_hasopt (p, "xml"))
+        log_msg_exit ("do not specify --cores or --gpus with --local or --xml");
+
+    get_ranks_and_hostlist (p, &ranks, &hl);
+    xml = get_xml (p);
+
+    if (!(rl = rlist_create ()))
+        log_err_exit ("rlist_create failed");
+
+    i = idset_first (ranks);
+    host = hostlist_first (hl);
+
+    while (i != IDSET_INVALID_ID) {
+    struct rlist *rloc = NULL;
+        if (optparse_hasopt (p, "local") || xml != NULL) {
+            if (!(rloc = rlist_from_hwloc (i, xml)))
+                log_err_exit ("rlist_from_hwloc");
+            if (rlist_assign_hosts (rloc, host) < 0)
+                log_err_exit ("rlist_assign_hosts (%s)", host);
+            if (rlist_append (rl, rloc) < 0)
+                log_err_exit ("rlist_append");
+            rlist_destroy (rloc);
+        }
+        else if (rlist_append_rank_cores (rl, host, i, cores) < 0)
+            log_err_exit ("rlist_append rank=%u", i);
+        if (strlen (gpus))
+            rlist_rank_add_child (rl, i, "gpu", gpus);
+        i = idset_next (ranks, i);
+        host = hostlist_next (hl);
+    }
+
+    set_properties (p, rl);
+
+    rlist_puts (rl);
+
+    free (xml);
+    idset_destroy (ranks);
+    hostlist_destroy (hl);
+    rlist_destroy (rl);
+
+    return 0;
+}
+
+static void rlist_freefn (void **x)
+{
+    if (x) {
+        rlist_destroy ((struct rlist *) *x);
+        *x = NULL;
+    }
+}
+
+/*  Load a list of R objects from stream 'fp'.
+ */
+static zlistx_t * rlist_loadf (FILE *fp)
+{
+    json_t *o;
+    json_error_t err;
+    struct rlist *rl;
+    zlistx_t *l;
+
+    if (!(l = zlistx_new ()))
+        log_err_exit ("zlistx_new");
+    zlistx_set_destructor (l, rlist_freefn);
+
+    while ((o = json_loadf (stdin, JSON_DISABLE_EOF_CHECK, &err))) {
+        if (!(rl = rlist_from_json (o, &err)))
+            log_msg_exit ("Failed to decode R on stdin: %s", err.text);
+        json_decref (o);
+        zlistx_add_end (l, rl);
+    }
+    if (zlistx_size (l) == 0)
+        log_msg_exit ("Failed to read an R object: %s", err.text);
+    return l;
+}
+
+static struct rlist * rl_append_all (FILE *fp)
+{
+    struct rlist *result;
+    struct rlist *rl;
+    zlistx_t *l = rlist_loadf (fp);
+    if (l == NULL)
+        log_err_exit ("rlist_loadf()");
+
+    if (!(result = rlist_create ()))
+        log_err_exit ("rlist_create");
+
+    rl = zlistx_first (l);
+    while (rl) {
+        struct rlist *intersect = rlist_intersect (result, rl);
+        if (rlist_nnodes (intersect))
+            log_msg_exit ("R objects '%s' and '%s' overlap",
+                          rlist_dumps (result),
+                          rlist_dumps (rl));
+        if (rlist_append (result, rl) < 0)
+            log_err_exit ("rlist_append");
+        rl = zlistx_next (l);
+    }
+    zlistx_destroy (&l);
+    return result;
+}
+
+
+typedef struct rlist * (*rlist_transform_f) (const struct rlist *a,
+                                             const struct rlist *b);
+
+static struct rlist *rl_transform (const char *cmd,
+                                   FILE *fp,
+                                   int min_sets,
+                                   rlist_transform_f fn)
+{
+    struct rlist *rlprev;
+    struct rlist *rl;
+    struct rlist *result = NULL;
+
+    zlistx_t *l = rlist_loadf (fp);
+    if (l == NULL)
+        log_err_exit ("rlist_loadf()");
+
+    if (zlistx_size (l) < min_sets)
+        log_msg_exit ("%s requires at least %d resource sets", cmd, min_sets);
+
+    /*  Pop the first item off the list as our starting set:
+     */
+    rlprev = zlistx_first (l);
+    zlistx_detach_cur (l);
+
+    rl = zlistx_next (l);
+    while (rl) {
+        result = (*fn) (rlprev, rl);
+        if (result == NULL)
+            log_msg_exit ("%s (%s, %s) failed!",
+                          cmd,
+                          rlist_dumps (rlprev),
+                          rlist_dumps (rl));
+        rlist_destroy (rlprev);
+        rlprev = result;
+        rl = zlistx_next (l);
+    }
+    zlistx_destroy (&l);
+
+    return rlprev;
+}
+
+int cmd_append (optparse_t *p, int argc, char **argv)
+{
+    struct rlist *result = rl_append_all (stdin);
+    if (!result)
+        log_err_exit ("Failed to append all R objects on stdin");
+    rlist_puts (result);
+    rlist_destroy (result);
+    return 0;
+}
+
+int cmd_diff (optparse_t *p, int argc, char **argv)
+{
+    struct rlist *result = rl_transform ("diff", stdin, 2, rlist_diff);
+    if (!result)
+        log_err_exit ("Failed to transform R objects on stdin");
+    rlist_puts (result);
+    rlist_destroy (result);
+    return 0;
+}
+
+int cmd_intersect (optparse_t *p, int argc, char **argv)
+{
+    struct rlist *result = rl_transform ("intersect",
+                                         stdin,
+                                         2,
+                                         rlist_intersect);
+    if (!result)
+        log_err_exit ("Failed to transform R objects on stdin");
+    rlist_puts (result);
+    rlist_destroy (result);
+    return 0;
+}
+
+int cmd_remap (optparse_t *p, int argc, char **argv)
+{
+    struct rlist *rl = rl_transform ("union", stdin, 1, rlist_union);
+    if (!rl)
+        log_err_exit ("Failed to transform R objects on stdin");
+    if (rlist_remap (rl) < 0)
+        log_err_exit ("Failed to re-map R");
+    rlist_puts (rl);
+    rlist_destroy (rl);
+    return 0;
+}
+
+int cmd_rerank (optparse_t *p, int argc, char **argv)
+{
+    struct rlist *rl = rl_transform ("union", stdin, 1, rlist_union);
+    if (!rl)
+        log_err_exit ("Failed to transform R objects on stdin");
+    if (argc != 2)
+        log_err_exit ("Must provide a hostlist for re-ranking");
+    if (rlist_rerank (rl, argv[1], NULL) < 0) {
+        if (errno == ENOENT)
+            log_msg_exit ("failed to find one or more provided hosts in R");
+        else if (errno == EOVERFLOW)
+            log_msg_exit ("Too many hosts specified (expected %zu)",
+                          rlist_nnodes (rl));
+        else if (errno == ENOSPC)
+            log_msg_exit ("Too few hosts specified (expected %zu)",
+                          rlist_nnodes (rl));
+        else
+            log_err_exit ("rlist_rerank");
+    }
+    rlist_puts (rl);
+    rlist_destroy (rl);
+    return 0;
+}
+
+static json_t *property_constraint_create (const char *arg)
+{
+    char *tok;
+    char *p;
+    char *s;
+    char *cpy = strdup (arg);
+    json_t *result = NULL;
+    json_t *o = json_array ();
+
+
+    if (!cpy || !o)
+        goto out;
+
+    s = cpy;
+    while ((tok = strtok_r (s, ",", &p))) {
+        json_t *prop = json_string (tok);
+        if (!prop || json_array_append_new (o, prop) != 0)
+            log_msg_exit ("Failed to append %s to properties array",
+                          tok);
+        s = NULL;
+    }
+    result = json_pack ("{s:o}", "properties", o);
+out:
+    free (cpy);
+    return result;
+}
+
+int cmd_decode (optparse_t *p, int argc, char **argv)
+{
+    int lines = 0;
+    char *s;
+    const char *type;
+    const char *arg;
+    struct rlist *rl = rl_transform ("union", stdin, 1, rlist_union);
+    if (!rl)
+        log_msg_exit ("failed to read R on stdin");
+
+    if (optparse_getopt (p, "properties", &arg) > 0) {
+        struct rlist *tmp = rl;
+        flux_error_t error;
+        json_t *constraint = property_constraint_create (arg);
+
+        if (!(rl = rlist_copy_constraint (tmp, constraint, &error)))
+            log_err_exit ("Invalid property constraint: %s", error.text);
+
+        rlist_destroy (tmp);
+        json_decref (constraint);
+    }
+    if (optparse_getopt (p, "include", &arg) > 0) {
+        struct rlist *tmp;
+        struct idset *ranks = idset_decode (arg);
+        if (!ranks)
+            log_err_exit ("Invalid ranks option: '%s'", arg);
+        tmp = rl;
+        if (!(rl = rlist_copy_ranks (tmp, ranks)))
+            log_err_exit ("rlist_copy_ranks(%s) failed", arg);
+        rlist_destroy (tmp);
+        idset_destroy (ranks);
+    }
+    if (optparse_getopt (p, "exclude", &arg) > 0) {
+        struct idset *ranks = idset_decode (arg);
+        if (!ranks)
+            log_err_exit ("Invalid idset in --exclude option: %s", arg);
+        if (rlist_remove_ranks (rl, ranks) < 0)
+            log_err_exit ("error removing ranks %s from R", arg);
+        idset_destroy (ranks);
+    }
+    if (optparse_hasopt (p, "short")) {
+        if (!(s = rlist_dumps (rl)))
+            log_err_exit ("rlist_dumps");
+        printf ("%s\n", s);
+        free (s);
+        lines++;
+    }
+    if (optparse_hasopt (p, "nodelist")) {
+        struct hostlist *hl = rlist_nodelist (rl);
+        if (!hl)
+            log_err_exit ("rlist_nodelist");
+        if (!(s = hostlist_encode (hl)))
+            log_err_exit ("hostlist_encode");
+        printf ("%s\n", s);
+        free (s);
+        hostlist_destroy (hl);
+        lines++;
+    }
+    if (optparse_hasopt (p, "ranks")) {
+        struct idset *ids = rlist_ranks (rl);
+        if (!ids)
+            log_err_exit ("rlist_ranks");
+        if (!(s = idset_encode (ids, IDSET_FLAG_RANGE)))
+            log_err_exit ("idset_encode");
+        printf ("%s\n", s);
+        free (s);
+        idset_destroy (ids);
+        lines++;
+    }
+    if (optparse_getopt (p, "count", &type)) {
+        int count;
+        if (streq (type, "node"))
+            count = rlist_nnodes (rl);
+        else if (streq (type, "core"))
+            count = rl->avail;
+        else
+            count = rlist_count (rl, type);
+        printf ("%d\n", count);
+        lines++;
+    }
+    if (lines == 0)
+        rlist_puts (rl);
+    rlist_destroy (rl);
+    return 0;
+}
+
+
+int cmd_verify (optparse_t *p, int argc, char **argv)
+{
+    int rc;
+    flux_error_t error;
+    struct rlist *expected;
+    struct rlist *rl;
+    zlistx_t *l = rlist_loadf (stdin);
+    if (!l)
+        log_err_exit ("rlist_loadf");
+    if (zlistx_size (l) != 2)
+        log_msg_exit ("verify requires exactly 2 R objects on stdin");
+    if (!(expected = zlistx_first (l))
+        || !(rl = zlistx_next (l)))
+        log_err_exit ("Unexpected error getting first two rlists");
+
+    rc = rlist_verify (&error, expected, rl);
+    if (rc != 0)
+        log_msg ("%s", error.text);
+    zlistx_destroy (&l);
+    if (rc < 0)
+        exit (1);
+    return 0;
+}
+
+int cmd_set_property (optparse_t *p, int argc, char **argv)
+{
+    json_t *o;
+    char *allranks;
+    flux_error_t err;
+
+    struct rlist *rl = rl_transform ("union", stdin, 1, rlist_union);
+    if (!rl)
+        log_msg_exit ("failed to read R on stdin");
+
+    if (!(o = json_object ()))
+        log_err_exit ("failed to create properties JSON object");
+
+    if (!(allranks = rlist_ranks_string (rl)))
+        log_err_exit ("failed to get rank idset string");
+
+    for (int i = 1; i < argc; i++)
+        set_one_property (o, allranks, argv[i]);
+
+    if (rlist_assign_properties (rl, o, &err) < 0)
+        log_msg_exit ("failed to assign properties: %s", err.text);
+
+    rlist_puts (rl);
+
+    json_decref (o);
+    free (allranks);
+    rlist_destroy (rl);
+
+    return 0;
+}
+
+int cmd_parse_config (optparse_t *p, int argc, char **argv)
+{
+    flux_error_t error;
+    json_t *o = NULL;
+    const char *path = NULL;
+    const char *scheduling = NULL;
+    struct rlist *rl = NULL;
+    flux_conf_t *conf;
+
+    if (!(conf = flux_conf_parse (argv[1], &error)))
+        log_msg_exit ("flux_conf_parse: %s", error.text);
+
+    if (flux_conf_unpack (conf, &error,
+                          "{s:{s?o s?s s?s}}",
+                          "resource",
+                            "config", &o,
+                            "path", &path,
+                            "scheduling", &scheduling) < 0)
+        log_msg_exit ("Config file error: %s", error.text);
+
+    if (!o) {
+        json_error_t e;
+
+        if (!path) {
+            log_msg_exit ("Config file error:"
+                          " resource.config or resource.path must be defined");
+        }
+        if (!(o = json_load_file (path, 0, &e))
+            || !(rl = rlist_from_json (o, &e))) {
+            if (e.line == -1)
+                log_msg_exit ("%s: %s", path, e.text);
+            log_msg_exit ("%s: %s on line %d", path, e.text, e.line);
+        }
+        json_decref (o);
+    }
+    else {
+        if (!(rl = rlist_from_config (o, &error)))
+            log_msg_exit ("Config file error: %s", error.text);
+    }
+    if (scheduling) {
+        json_error_t e;
+        json_t *sched;
+
+        if (!(sched = json_load_file (scheduling, 0, &e))) {
+            log_msg_exit ("resource.scheduling: %s: %s on line %d",
+                          scheduling,
+                          e.text,
+                          e.line);
+        }
+        json_decref (rl->scheduling);
+        rl->scheduling = sched;
+    }
+
+    rlist_puts (rl);
+
+    flux_conf_decref (conf);
+    rlist_destroy (rl);
+
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/cmd/flux-admin.py b/src/cmd/flux-admin.py
new file mode 100755
index 000000000000..5f5dc409222d
--- /dev/null
+++ b/src/cmd/flux-admin.py
@@ -0,0 +1,66 @@
+##############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+import sys
+
+import flux
+from flux.rpc import RPC
+
+
+def cleanup_push(args):
+    """
+    Add a command to run after completion of the initial program, before rc3.
+    It is pushed to the front of the list of commands.
+
+    If command was not provided as args, read one command per line from
+    stdio.  Push these in reverse order to retain the order of the block of
+    commands.
+    """
+    if args.cmdline:
+        commands = [(" ".join(args.cmdline))]
+    else:
+        commands = [line.strip() for line in sys.stdin]
+
+    RPC(
+        flux.Flux(),
+        "runat.push",
+        {"name": "cleanup", "commands": commands[::-1]},
+    ).get()
+
+
+LOGGER = logging.getLogger("flux-admin")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    parser = argparse.ArgumentParser(prog="flux-admin")
+    subparsers = parser.add_subparsers(
+        title="subcommands", description="", dest="subcommand"
+    )
+    subparsers.required = True
+
+    cleanup_push_parser = subparsers.add_parser(
+        "cleanup-push", formatter_class=flux.util.help_formatter()
+    )
+    cleanup_push_parser.add_argument(
+        "cmdline", help="Command line", nargs=argparse.REMAINDER
+    )
+    cleanup_push_parser.set_defaults(func=cleanup_push)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-aggregate.c b/src/cmd/flux-aggregate.c
deleted file mode 100644
index edfcfaad1c85..000000000000
--- a/src/cmd/flux-aggregate.c
+++ /dev/null
@@ -1,272 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdarg.h>
-
-#include <jansson.h>
-#include <flux/core.h>
-#include <flux/optparse.h>
-
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/monotime.h"
-#include "src/common/libaggregate/aggregate.h"
-
-struct aggregate_args {
-    flux_t *h;
-    uint32_t size;
-    uint32_t rank;
-    const char *key;
-    json_t *o;
-    double timeout;
-    int fwd_count;
-    bool verbose;
-    struct timespec t0;
-};
-
-static const char usage[] = "[OPTIONS] KEY [JSON VALUE]";
-static const char doc[] = "\n\
-Front end test utility for creating \"aggregate\" JSON objects in the kvs. \
-Must be run across all ranks, i.e. as `flux exec flux aggregate ...`. \
-If JSON_VALUE is not supplied on the command line, reads value from stdin.\n\
-";
-
-static struct optparse_option opts[] = {
-    { .name = "timeout", .key = 't', .arginfo = "T", .has_arg = 1,
-      .usage = "Set reduction timeout to T seconds." },
-    { .name = "fwd-count", .key = 'c', .arginfo = "N", .has_arg = 1,
-      .usage = "Forward aggregate upstream after N" },
-    { .name = "verbose", .key = 'v', .has_arg = 0,
-      .usage = "Verbose operation" },
-    OPTPARSE_TABLE_END
-};
-
-static void verbose (struct aggregate_args *args, const char *fmt, ...)
-{
-    char buf [4096];
-    va_list ap;
-    if (!args->verbose)
-        return;
-    va_start (ap, fmt);
-    vsnprintf (buf, sizeof (buf), fmt, ap);
-    fprintf (stderr, "flux-aggregate: %.3fs: %s\n",
-                     monotime_since (args->t0)/1000., buf);
-    va_end (ap);
-}
-
-json_t *json_from_stdin (void)
-{
-    json_error_t e;
-    json_t *o = json_loadf (stdin, JSON_DECODE_ANY, &e);
-    if (o == NULL)
-        log_msg_exit ("Failed to decode JSON: %s", e.text);
-    return (o);
-}
-
-json_t *json_from_string (const char *s)
-{
-    json_error_t e;
-    json_t *o = json_loads (s, JSON_DECODE_ANY, &e);
-    if (o == NULL)
-        log_msg_exit ("Failed to decode JSON: %s", e.text);
-    return (o);
-}
-
-static void unlink_aggregate_key (struct aggregate_args *args)
-{
-    const char *key = args->key;
-    flux_t *h = args->h;
-    flux_future_t *f = NULL;
-    flux_kvs_txn_t *txn = NULL;
-
-    /*
-     * XXX: Special case: allow key of '.' to drop through for
-     *  testing purposes
-     */
-    if (key[0] == '.' && key[1] == '\0')
-        return;
-
-    verbose (args, "unlinking %s", key);
-
-    if (!(txn = flux_kvs_txn_create ()))
-        log_err_exit ("flux_kvs_txn_create");
-    if (flux_kvs_txn_unlink (txn, 0, key) < 0)
-        log_err_exit ("flux_kvs_txn_unlink");
-    if (!(f = flux_kvs_commit (h, NULL, 0, txn)))
-        log_err_exit ("flux_kvs_commit");
-    if (flux_future_get (f, NULL) < 0)
-        log_err_exit ("kvs commit rpc");
-
-    verbose (args, "unlink complete");
-
-    flux_future_destroy (f);
-    flux_kvs_txn_destroy (txn);
-}
-
-static void abort_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
-{
-    struct aggregate_args *args = arg;
-    fprintf (stderr, "flux-aggregate: %.3fs: aggregate aborted\n",
-                     monotime_since (args->t0)/1000.);
-    exit (1);
-}
-
-static flux_msg_handler_t *abort_msg_handler_create (struct aggregate_args *arg)
-{
-    int n;
-    char buf [1024];
-    struct flux_match match = FLUX_MATCH_EVENT;
-    flux_msg_handler_t *mh = NULL;
-
-    if ((n = snprintf (buf, sizeof (buf), "aggregator.abort.%s", arg->key) < 0)
-        || (n >= sizeof (buf)))
-        log_err_exit ("creating event name for key=%s", arg->key);
-
-    match.topic_glob = buf;
-    if (!(mh = flux_msg_handler_create (arg->h, match, abort_cb, (void *) arg)))
-        log_err_exit ("flux_msg_handler_create");
-    flux_msg_handler_start (mh);
-
-    if (flux_event_subscribe (arg->h, "aggregator.abort") < 0)
-        log_err_exit ("flux_event_subscribe");
-    verbose (arg, "subscribed to event %s", buf);
-    return (mh);
-}
-
-void print_entries (json_t *entries)
-{
-    const char *key;
-    json_t *value;
-    json_object_foreach (entries, key, value) {
-        char *s = json_dumps (value, JSON_ENCODE_ANY|JSON_COMPACT);
-        printf ("%s: %s\n", key, s);
-        free (s);
-    }
-}
-
-void print_result (flux_future_t *f, void *arg)
-{
-    json_t *entries;
-    if (aggregate_wait_get_unpack (f, "{s:o}", "entries", &entries) < 0)
-        log_err_exit ("aggregate_wait_unpack");
-    print_entries (entries);
-    flux_reactor_stop (flux_future_get_reactor (f));
-    flux_future_destroy (f);
-}
-
-static void aggregate_push_continue (flux_future_t *f, void *arg)
-{
-    flux_future_t *f2 = NULL;
-    struct aggregate_args *args = arg;
-
-    if (flux_future_get (f, NULL) < 0)
-        log_err_exit ("aggregate.push");
-    if (args->rank != 0) {
-        flux_future_destroy (f);
-        flux_reactor_stop (flux_get_reactor (args->h));
-        return;
-    }
-    verbose (args, "waiting for aggregate to complete");
-    if (!(f2 = aggregate_wait (args->h, args->key)))
-        log_err_exit ("aggregate_wait");
-    if (flux_future_then (f2, 5., print_result, args) < 0)
-        log_err_exit ("aggregate_wait");
-
-    flux_future_destroy (f);
-}
-
-static void barrier_continue (flux_future_t *f, void *arg)
-{
-    struct aggregate_args *args = arg;
-    flux_future_t *f2 = NULL;
-    verbose (args, "barrier complete, calling aggregate.push");
-    if (!(f2 = aggregator_push_json (args->h, args->fwd_count, args->timeout,
-                                     args->key, args->o))
-        || (flux_future_then (f2, -1., aggregate_push_continue, arg) < 0))
-        log_err_exit ("aggregator_push_json");
-    flux_future_destroy (f);
-}
-
-int main (int argc, char *argv[])
-{
-    struct aggregate_args args;
-    int optindex;
-    optparse_t *p = NULL;
-    flux_msg_handler_t *mh = NULL;
-    flux_future_t *f = NULL;
-
-    memset (&args, 0, sizeof (args));
-    monotime (&args.t0);
-
-    if (!(p = optparse_create ("flux-aggregate")))
-        log_msg_exit ("optparse_create");
-    if (optparse_set (p, OPTPARSE_USAGE, usage) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_set usage");
-    if (optparse_add_doc (p, doc, 0) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_add_doc");
-    if (optparse_add_option_table (p, opts) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_set usage");
-
-    if ((optindex = optparse_parse_args (p, argc, argv)) < 0)
-        exit (1);
-    if ((argc - optindex) == 0) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-
-    args.verbose = optparse_hasopt (p, "verbose");
-    args.fwd_count = optparse_get_int (p, "fwd-count", 0);
-    args.timeout = optparse_get_duration (p, "timeout", -1.);
-
-    if (!(args.h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (flux_get_rank (args.h, &args.rank) < 0)
-        log_err_exit ("flux_get_rank");
-    if (flux_get_size (args.h, &args.size) < 0)
-        log_err_exit ("flux_get_size");
-
-    /*  Only print messages from rank 0 on verbose operation */
-    args.verbose = (args.rank == 0 && args.verbose);
-
-    args.key = argv[optindex];
-    if ((argc - optindex) == 1) // read from stdin
-        args.o = json_from_stdin ();
-    else
-        args.o = json_from_string (argv [optindex+1]);
-
-    verbose (&args, "starting aggregate on %d ranks", args.size);
-
-    if (args.rank == 0) {
-        unlink_aggregate_key (&args);
-        mh = abort_msg_handler_create (&args);
-    }
-    if (!(f = flux_barrier (args.h, args.key, args.size))
-        || (flux_future_then (f, -1., barrier_continue, &args)))
-        log_err_exit ("flux_barrier");
-
-    verbose (&args, "starting reactor");
-
-    if (flux_reactor_run (flux_get_reactor (args.h), 0) < 0)
-        log_err_exit ("flux_reactor_run");
-
-    verbose (&args, "all done");
-
-    flux_msg_handler_destroy (mh);
-    flux_close (args.h);
-    optparse_destroy (p);
-    log_fini();
-    return (0);
-}
-
-/* vi: ts=4 sw=4 expandtab
- */
diff --git a/src/cmd/flux-alloc.py b/src/cmd/flux-alloc.py
new file mode 100755
index 000000000000..34bd3e158167
--- /dev/null
+++ b/src/cmd/flux-alloc.py
@@ -0,0 +1,49 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import logging
+import sys
+
+import flux
+import flux.cli.alloc as base
+import flux.job
+
+LOGGER = logging.getLogger("flux-alloc")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    description = """
+    Allocate resources and start a new Flux instance. Once the instance
+    has started, attach to it interactively.
+    """
+
+    alloc = base.AllocCmd(
+        "flux alloc",
+        usage="flux alloc [OPTIONS...] [COMMAND] [ARGS...]",
+        description=description,
+    )
+    parser = alloc.get_parser()
+    parser.set_defaults(func=alloc.main)
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-batch.py b/src/cmd/flux-batch.py
new file mode 100755
index 000000000000..0cd0433473c3
--- /dev/null
+++ b/src/cmd/flux-batch.py
@@ -0,0 +1,50 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import logging
+import sys
+
+import flux
+import flux.cli.batch as base
+import flux.job
+
+LOGGER = logging.getLogger("flux-batch")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    # batch
+    description = """
+    Submit a batch SCRIPT and ARGS to be run as the initial program of
+    a Flux instance.  If no batch script is provided, one will be read
+    from stdin.
+    """
+    batch = base.BatchCmd(
+        "flux batch",
+        usage="flux batch [OPTIONS...] [SCRIPT] [ARGS...]",
+        description=description,
+    )
+    parser = batch.get_parser()
+    parser.set_defaults(func=batch.main)
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-bulksubmit.py b/src/cmd/flux-bulksubmit.py
new file mode 100755
index 000000000000..41576042dc38
--- /dev/null
+++ b/src/cmd/flux-bulksubmit.py
@@ -0,0 +1,51 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import logging
+import sys
+
+import flux.cli.bulksubmit as base
+import flux.utils
+
+LOGGER = logging.getLogger("flux-bulksubmit")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    description = """
+    Submit a series of commands given on the command line or on stdin,
+    using an interface similar to GNU parallel or xargs. Allows jobs to be
+    submitted much faster than calling flux submit in a loop. Inputs on the
+    command line are separated from each other and the command with the
+    special delimiter ':::'.
+    """
+
+    # create the bulksubmit parser
+    bulksubmit = base.BulkSubmitCmd(
+        "flux bulksubmit",
+        description=description,
+    )
+    parser = bulksubmit.get_parser()
+    parser.set_defaults(func=bulksubmit.main)
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-cancel.py b/src/cmd/flux-cancel.py
new file mode 100755
index 000000000000..4bba535299a1
--- /dev/null
+++ b/src/cmd/flux-cancel.py
@@ -0,0 +1,187 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+import os
+import pwd
+import sys
+from operator import itemgetter
+
+import flux
+from flux.job import cancel_async
+
+LOGGER = logging.getLogger("flux-cancel")
+
+
+def parse_args():
+    description = """
+    Cancel pending or running jobs with an optional exception note.
+    """
+    parser = argparse.ArgumentParser(
+        prog="flux-cancel",
+        usage="flux cancel [OPTIONS] [JOBID...]",
+        description=description,
+        formatter_class=flux.util.help_formatter(),
+    )
+    parser.add_argument("--all", action="store_true", help="Target all jobs")
+    parser.add_argument(
+        "-n", "--dry-run", action="store_true", help="Do not cancel any jobs"
+    )
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        help="Suppress output when no jobs match",
+    )
+    parser.add_argument(
+        "-u",
+        "--user",
+        type=str,
+        metavar="USER",
+        help="Set target user or 'all' (instance owner only)",
+    )
+    parser.add_argument(
+        "-S",
+        "--states",
+        action="append",
+        help="List of job states to target (default=active)",
+    )
+    parser.add_argument(
+        "-m",
+        "--message",
+        type=str,
+        metavar="NOTE",
+        help="Set optional cancel exception note",
+    )
+    parser.add_argument(
+        "jobids",
+        metavar="JOBID...",
+        type=flux.job.JobID,
+        nargs="*",
+        help="jobids to cancel",
+    )
+    return parser.parse_args()
+
+
+def log(msg):
+    """Log to stderr without logging INFO prefix"""
+    print(f"flux-cancel: {msg}", file=sys.stderr)
+
+
+def cancel(args):
+    if not args.jobids:
+        raise ValueError("No target jobs were provided")
+    if args.dry_run:
+        count = len(args.jobids)
+        log(f"Would cancel {count} job{'s'[:count^1]}")
+        sys.exit(0)
+    h = flux.Flux()
+    futures = {x: cancel_async(h, x, args.message) for x in args.jobids}
+    errors = 0
+    for jobid, future in futures.items():
+        try:
+            future.get()
+        except OSError as exc:
+            errors += 1
+            LOGGER.error(f"{jobid}: {exc.strerror}")
+    if errors > 0:
+        sys.exit(1)
+
+
+def cancelall(args):
+
+    STATES = {
+        "depend": flux.constants.FLUX_JOB_STATE_DEPEND,
+        "priority": flux.constants.FLUX_JOB_STATE_PRIORITY,
+        "sched": flux.constants.FLUX_JOB_STATE_SCHED,
+        "run": flux.constants.FLUX_JOB_STATE_RUN,
+        "pending": flux.constants.FLUX_JOB_STATE_PENDING,
+        "running": flux.constants.FLUX_JOB_STATE_RUNNING,
+        "active": flux.constants.FLUX_JOB_STATE_ACTIVE,
+    }
+
+    if args.user is None:
+        userid = os.getuid()
+    elif args.user == "all":
+        userid = flux.constants.FLUX_USERID_UNKNOWN
+    else:
+        try:
+            userid = pwd.getpwnam(args.user).pw_uid
+        except KeyError:
+            try:
+                userid = int(args.user)
+            except ValueError:
+                raise ValueError(f"Invalid user {args.user} specified")
+
+    statemask = 0
+    if args.states:
+        for state in ",".join(args.states).split(","):
+            try:
+                statemask |= STATES[state.lower()]
+            except KeyError:
+                raise ValueError(f"Invalid state {state} specified")
+    else:
+        statemask = STATES["pending"] | STATES["running"]
+
+    payload = {
+        "dry_run": args.dry_run,
+        "userid": userid,
+        "states": statemask,
+        "severity": 0,
+        "type": "cancel",
+    }
+    if args.message is not None:
+        payload["note"] = args.message
+
+    count, errors = itemgetter("count", "errors")(
+        flux.Flux().rpc("job-manager.raiseall", payload).get()
+    )
+
+    if count > 0:
+        msg = f"{count} job{'s'[:count^1]}"
+        if args.dry_run:
+            log(f"Would cancel {msg}")
+        else:
+            log(f"Canceled {msg} ({errors} error{'s'[:errors^1]})")
+    elif not args.quiet:
+        log("Matched 0 jobs")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    args = parse_args()
+
+    if args.user or args.states:
+        args.all = True
+
+    if args.jobids and args.all:
+        LOGGER.error(
+            "Do not specify a list of jobids with --all,--user,--states options"
+        )
+        sys.exit(1)
+
+    if args.all:
+        cancelall(args)
+    else:
+        cancel(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-comms.c b/src/cmd/flux-comms.c
deleted file mode 100644
index 846822585856..000000000000
--- a/src/cmd/flux-comms.c
+++ /dev/null
@@ -1,141 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdio.h>
-#include <getopt.h>
-#include <stdbool.h>
-#include <argz.h>
-#include <flux/core.h>
-#include <inttypes.h>
-
-#include "src/common/libutil/log.h"
-
-
-#define OPTIONS "+hr:"
-static const struct option longopts[] = {
-    {"help",       no_argument,        0, 'h'},
-    {"rank",       required_argument,  0, 'r'},
-    { 0, 0, 0, 0 },
-};
-
-void usage (void)
-{
-    fprintf (stderr,
-"Usage: flux-comms [-r N] idle\n"
-"       flux-comms        info\n"
-"       flux-comms [-r N] panic [msg ...]\n"
-);
-    exit (1);
-}
-
-static char *flux_lspeer (flux_t *h, int rank)
-{
-    int saved_errno;
-    uint32_t nodeid = (rank == -1 ? FLUX_NODEID_ANY : rank);
-    flux_future_t *f;
-    const char *json_str;
-    char *ret = NULL;
-
-    if (!(f = flux_rpc (h, "cmb.lspeer", NULL, nodeid, 0)))
-        goto done;
-    if (flux_rpc_get (f, &json_str) < 0)
-        goto done;
-    if (!json_str) {
-        errno = EPROTO;
-        goto done;
-    }
-    ret = strdup (json_str);
-done:
-    saved_errno = errno;
-    flux_future_destroy (f);
-    errno = saved_errno;
-    return ret;
-}
-
-int main (int argc, char *argv[])
-{
-    flux_t *h;
-    int ch;
-    uint32_t rank = FLUX_NODEID_ANY; /* local */
-    char *cmd;
-    int e;
-
-    log_init ("flux-comms");
-
-    while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
-        switch (ch) {
-            case 'h': /* --help */
-                usage ();
-                break;
-            case 'r': /* --rank NODESET */
-                rank = strtoul (optarg, NULL, 10);
-                break;
-            default:
-                usage ();
-                break;
-        }
-    }
-    if (optind == argc)
-        usage ();
-    cmd = argv[optind++];
-    if (rank != FLUX_NODEID_ANY
-            && (!strcmp (cmd, "recover-all") || !strcmp (cmd, "info")))
-        usage ();
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (!strcmp (cmd, "idle")) {
-        if (optind != argc)
-            usage ();
-        char *peers;
-        if (!(peers = flux_lspeer (h, rank)))
-            log_err_exit ("flux_lspeer");
-        printf ("%s\n", peers);
-        free (peers);
-    } else if (!strcmp (cmd, "panic")) {
-        char *reason = NULL;
-        int flags = 0;
-        size_t len = 0;
-        if (optind < argc) {
-            if ((e = argz_create (argv + optind, &reason, &len)) != 0)
-                log_errn_exit (e, "argz_create");
-            argz_stringify (reason, len, ' ');
-        }
-        if (flux_panic (h, rank, flags,
-                        reason ? reason : "user request") < 0)
-            log_err_exit ("flux_panic");
-        free (reason);
-    } else if (!strcmp (cmd, "info")) {
-        int arity;
-        uint32_t rank, size;
-        const char *s;
-        if (flux_get_rank (h, &rank) < 0 || flux_get_size (h, &size) < 0)
-            log_err_exit ("flux_get_rank/size");
-        if (!(s = flux_attr_get (h, "tbon.arity")))
-            log_err_exit ("flux_attr_get tbon.arity");
-        arity = strtoul (s, NULL, 10);
-        printf ("rank=%"PRIu32"\n", rank);
-        printf ("size=%"PRIu32"\n", size);
-        printf ("arity=%d\n", arity);
-    } else
-        usage ();
-
-    flux_close (h);
-    log_fini ();
-    return 0;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/cmd/flux-cron b/src/cmd/flux-cron
index 4ee457e71ca3..c28cad5a7ce2 100755
--- a/src/cmd/flux-cron
+++ b/src/cmd/flux-cron
@@ -69,7 +69,7 @@ local function reladate (t)
     end
 
     if (diff < 90) then
-        return fmt ("%d seconds ago", diff)
+        return fmt ("%d seconds ago", math.floor(diff + 0.5))
     end
 
     -- Convert to minutes:
@@ -78,7 +78,7 @@ local function reladate (t)
         return fmt ("a minute ago")
     end
     if (diff < 90) then
-        return fmt ("%d minutes ago", diff)
+        return fmt ("%d minutes ago", math.floor (diff + 0.5))
     end
 
     -- Convert to hours:
@@ -96,10 +96,10 @@ local function reladate (t)
     -- weeks for past 10 weeks or so, reduce precision in output:
     local d = math.floor (diff)
     if (d < 70) then
-        return fmt ("%d weeks ago", (d + 3) / 7)
+        return fmt ("%d weeks ago", math.floor ((d + 3) / 7))
     end
     if (d < 365) then
-        return fmt ("%d months ago", (d + 15) / 30)
+        return fmt ("%d months ago", math.floor ((d + 15) / 30))
     end
 
     return fmt ("%.1f years ago", diff / 365)
@@ -353,7 +353,7 @@ end
 program:SubCommand {
  name = "dump",
  description = "Dump values from a cron entry",
- usage = "[IDs]",
+ usage = "IDs",
  options = {
     { name = "key", char = "k", arg = "KEY",
       usage = "Print only KEY from the entry",
@@ -415,7 +415,7 @@ program:SubCommand {
 program:SubCommand {
  name = "list",
  description = "List registered and stopped flux-cron jobs",
- usage = "[IDs]",
+ usage = "",
  options = {},
  handler = function (self, arg)
 
diff --git a/src/cmd/flux-event.c b/src/cmd/flux-event.c
index 14bf6f9d7d62..7fa924635716 100644
--- a/src/cmd/flux-event.c
+++ b/src/cmd/flux-event.c
@@ -13,12 +13,17 @@
 #endif
 #include <stdio.h>
 #include <libgen.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
 #include <ctype.h>
 #include <flux/core.h>
 #include <flux/optparse.h>
 
 #include "src/common/libutil/log.h"
+#include "ccan/str/str.h"
 
 
 static int event_pub (optparse_t *p, int argc, char **argv);
@@ -65,11 +70,13 @@ static bool match_payload (const char *s1, const char *s2)
         return true;
     if (!s2 || !s1)
         return false;
-    return !strcmp (s1, s2);
+    return streq (s1, s2);
 }
 
-static bool match_payload_raw (const void *p1, int p1sz,
-                               const char *p2, int p2sz)
+static bool match_payload_raw (const void *p1,
+                               int p1sz,
+                               const char *p2,
+                               int p2sz)
 {
     if (!p1 && !p2)
         return true;
@@ -78,8 +85,11 @@ static bool match_payload_raw (const void *p1, int p1sz,
     return !memcmp (p1, p2, p1sz);
 }
 
-static int publish_raw_sync (flux_t *h, const char *topic, int flags,
-                             char *payload, int payloadsz)
+static int publish_raw_sync (flux_t *h,
+                             const char *topic,
+                             int flags,
+                             char *payload,
+                             int payloadsz)
 {
     flux_future_t *f;
     int seq;
@@ -95,8 +105,11 @@ static int publish_raw_sync (flux_t *h, const char *topic, int flags,
     return rc;
 }
 
-static int publish_raw (flux_t *h, const char *topic, int flags,
-                        char *payload, int payloadsz)
+static int publish_raw (flux_t *h,
+                        const char *topic,
+                        int flags,
+                        char *payload,
+                        int payloadsz)
 {
     flux_msg_t *msg;
     int rc = -1;
@@ -113,7 +126,9 @@ static int publish_raw (flux_t *h, const char *topic, int flags,
     return rc;
 }
 
-static int publish_json_sync (flux_t *h, const char *topic, int flags,
+static int publish_json_sync (flux_t *h,
+                              const char *topic,
+                              int flags,
                               char *payload)
 {
     flux_future_t *f;
@@ -131,7 +146,9 @@ static int publish_json_sync (flux_t *h, const char *topic, int flags,
     return rc;
 }
 
-static int publish_json (flux_t *h, const char *topic, int flags,
+static int publish_json (flux_t *h,
+                         const char *topic,
+                         int flags,
                          char *payload)
 {
     flux_msg_t *msg;
@@ -167,8 +184,12 @@ static struct optparse_option pub_opts[] = {
 
 static void event_pub_register (optparse_t *parent)
 {
-    if (optparse_reg_subcommand (parent, "pub", event_pub,
-                                 "[OPTIONS] topic [payload]", NULL, 0,
+    if (optparse_reg_subcommand (parent,
+                                 "pub",
+                                 event_pub,
+                                 "[OPTIONS] topic [payload]",
+                                 NULL,
+                                 0,
                                  pub_opts) != OPTPARSE_SUCCESS)
         log_err_exit ("optparse_reg_subcommand");
 }
@@ -235,15 +256,15 @@ static int event_pub (optparse_t *p, int argc, char **argv)
                 log_err_exit ("flux_recv error");
             if (optparse_hasopt (p, "raw")) {
                 const void *data;
-                int len;
+                size_t len;
                 if ((flux_event_decode_raw (msg, NULL, &data, &len) == 0
-                        && match_payload_raw (payload, payloadsz, data, len)))
+                    && match_payload_raw (payload, payloadsz, data, len)))
                     received = true;
             }
             else {
                 const char *json_str;
                 if ((flux_event_decode (msg, NULL, &json_str) == 0
-                        && match_payload (payload, json_str)))
+                    && match_payload (payload, json_str)))
                     received = true;
             }
             flux_msg_destroy (msg);
@@ -310,26 +331,34 @@ static const struct optparse_option sub_opts [] = {
 
 void event_sub_register (optparse_t *parent)
 {
-    if (optparse_reg_subcommand (parent, "sub", event_sub,
-                                 "[OPTIONS] [topic...]", NULL, 0,
+    if (optparse_reg_subcommand (parent,
+                                 "sub",
+                                 event_sub,
+                                 "[OPTIONS] [topic...]",
+                                 NULL,
+                                 0,
                                  sub_opts) != OPTPARSE_SUCCESS)
         log_err_exit ("optparse_reg_subcommand");
 }
 
-static void event_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+static void event_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     optparse_t *p = arg;
     int max_count = optparse_get_int (p, "count", 0);
     static int recv_count = 0;
     const char *payload;
-    int payloadsz;
+    size_t payloadsz;
     const char *topic;
 
     if (flux_event_decode (msg, &topic, &payload) == 0)
         printf ("%s\t%s\n", topic, payload ? payload : "");
-    else if (flux_event_decode_raw (msg, &topic, (const void **)&payload,
-                                                          &payloadsz) == 0) {
+    else if (flux_event_decode_raw (msg,
+                                    &topic,
+                                    (const void **)&payload,
+                                    &payloadsz) == 0) {
         int maxlen = payloadsz; // no truncation
         char *s = make_printable (payload, payloadsz, maxlen);
         printf ("%s\t%s\n", topic, s);
diff --git a/src/cmd/flux-exec.c b/src/cmd/flux-exec.c
index fb112b9d6552..0507b1e1f5fb 100644
--- a/src/cmd/flux-exec.c
+++ b/src/cmd/flux-exec.c
@@ -11,6 +11,9 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 #include <stdio.h>
 #include <unistd.h>
 #include <string.h>
@@ -18,12 +21,22 @@
 #include <flux/core.h>
 #include <flux/optparse.h>
 #include <signal.h>
-#include <czmq.h>
+#ifndef HAVE_GET_CURRENT_DIR_NAME
+#include "src/common/libmissing/get_current_dir_name.h"
+#endif
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/monotime.h"
 #include "src/common/libidset/idset.h"
+#include "src/common/libeventlog/eventlog.h"
 #include "src/common/libutil/log.h"
+#include "src/common/libutil/basename.h"
+#include "src/common/libsubprocess/fbuf.h"
+#include "src/common/libsubprocess/fbuf_watcher.h"
+#include "ccan/str/str.h"
+
+#define NUMCMP(a,b) ((a)==(b)?0:((a)<(b)?-1:1))
 
 static struct optparse_option cmdopts[] = {
     { .name = "rank", .key = 'r', .has_arg = 1, .arginfo = "IDSET",
@@ -32,17 +45,34 @@ static struct optparse_option cmdopts[] = {
       .usage = "Exclude ranks from target." },
     { .name = "dir", .key = 'd', .has_arg = 1, .arginfo = "PATH",
       .usage = "Set the working directory to PATH" },
-    { .name = "labelio", .key = 'l', .has_arg = 0,
+    { .name = "label-io", .key = 'l', .has_arg = 0,
       .usage = "Label lines of output with the source RANK" },
     { .name = "noinput", .key = 'n', .has_arg = 0,
       .usage = "Redirect stdin from /dev/null" },
     { .name = "verbose", .key = 'v', .has_arg = 0,
       .usage = "Run with more verbosity." },
+    { .name = "quiet", .key = 'q', .has_arg = 0,
+      .usage = "Suppress extraneous output." },
+    { .name = "service", .has_arg = 1, .arginfo = "NAME",
+      .flags = OPTPARSE_OPT_HIDDEN,
+      .usage = "Override service name (default: rexec)." },
+    { .name = "setopt", .has_arg = 1, .arginfo = "NAME=VALUE",
+      .flags = OPTPARSE_OPT_HIDDEN,
+      .usage = "Set subprocess option NAME to VALUE (multiple use ok)" },
+    { .name = "stdin-flow", .has_arg = 1, .arginfo = "on|off",
+      .flags = OPTPARSE_OPT_HIDDEN,
+      .usage = "Forcibly enable or disable stdin flow control" },
+    { .name = "with-imp", .has_arg = 0,
+      .usage = "Run args under 'flux-imp run'" },
+    { .name = "jobid", .key = 'j', .has_arg = 1, .arginfo = "JOBID",
+      .usage = "Set target ranks to nodes assigned to JOBID and  "
+               "service name to job shell exec service" },
     OPTPARSE_TABLE_END
 };
 
 extern char **environ;
 
+flux_t *flux_handle = NULL;
 uint32_t rank_range;
 uint32_t rank_count;
 uint32_t started = 0;
@@ -51,12 +81,22 @@ int exit_code = 0;
 zhashx_t *exitsets;
 struct idset *hanging;
 
-zlist_t *subprocesses;
+zlistx_t *subprocesses;
+/* subprocess credits ordered low to high.  Exited and failed
+ * subprocesses are removed from the list.
+ */
+zlistx_t *subprocess_credits;
+
+struct subproc_credit {
+    void *handle;      /* handle to subprocess in credits list */
+    int credits;
+};
 
 optparse_t *opts = NULL;
 
 int stdin_flags;
 flux_watcher_t *stdin_w;
+bool stdin_enable_flow_control = true;
 
 /* time to wait in between SIGINTs */
 #define INTERRUPT_MILLISECS 1000.0
@@ -64,6 +104,9 @@ flux_watcher_t *stdin_w;
 struct timespec last;
 int sigint_count = 0;
 
+bool use_imp = false;
+const char *imp_path = NULL;
+
 void output_exitsets (const char *key, void *item)
 {
     struct idset *idset = item;
@@ -110,10 +153,8 @@ void completion_cb (flux_subprocess_t *p)
         if (!(idset = zhashx_lookup (exitsets, buf))) {
             if (!(idset = idset_create (rank_range, 0)))
                 log_err_exit ("idset_create");
-            if (zhashx_insert (exitsets, buf, idset) < 0)
-                log_err_exit ("zhashx_insert");
-            if (!zhashx_freefn (exitsets, buf, idset_destroy_wrapper))
-                log_err_exit ("zhashx_freefn");
+            (void)zhashx_insert (exitsets, buf, idset);
+            (void)zhashx_freefn (exitsets, buf, idset_destroy_wrapper);
         }
 
         if (idset_set (idset, rank) < 0)
@@ -124,6 +165,35 @@ void completion_cb (flux_subprocess_t *p)
         log_err_exit ("idset_clear");
 }
 
+int subprocess_min_credits (void)
+{
+    /* subprocess_credits ordered, min at head */
+    flux_subprocess_t *p = zlistx_head (subprocess_credits);
+    struct subproc_credit *spcred;
+    /* list possibly empty if all subprocesses failed, so return no
+     * credits so stdin watcher won't be started
+     */
+    if (!p)
+        return 0;
+    spcred = flux_subprocess_aux_get (p, "credits");
+    return spcred->credits;
+}
+
+void subprocess_update_credits (flux_subprocess_t *p, int bytes, bool reorder)
+{
+    struct subproc_credit *spcred = flux_subprocess_aux_get (p, "credits");
+    spcred->credits += bytes;
+    if (reorder)
+        zlistx_reorder (subprocess_credits, spcred->handle, false);
+}
+
+void subprocess_remove_credits (flux_subprocess_t *p)
+{
+    struct subproc_credit *spcred = flux_subprocess_aux_get (p, "credits");
+    if (zlistx_delete (subprocess_credits, spcred->handle) < 0)
+        log_err_exit ("zlistx_delete");
+}
+
 void state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
 {
     if (state == FLUX_SUBPROCESS_RUNNING) {
@@ -131,12 +201,9 @@ void state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
         /* see FLUX_SUBPROCESS_FAILED case below */
         (void)flux_subprocess_aux_set (p, "started", p, NULL);
     }
-    else if (state == FLUX_SUBPROCESS_EXITED)
-        exited++;
-    else if (state == FLUX_SUBPROCESS_EXEC_FAILED) {
-        /* EXEC_FAILED means RUNNING never reached, so must increment started */
-        started++;
+    else if (state == FLUX_SUBPROCESS_EXITED) {
         exited++;
+        subprocess_remove_credits (p);
     }
     else if (state == FLUX_SUBPROCESS_FAILED) {
         /* FLUX_SUBPROCESS_FAILED is a catch all error case, no way to
@@ -146,25 +213,39 @@ void state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
         if (flux_subprocess_aux_get (p, "started") == NULL)
             started++;
         exited++;
+        subprocess_remove_credits (p);
     }
 
     if (stdin_w) {
-        if (started == rank_count)
-            flux_watcher_start (stdin_w);
+        if (started == rank_count) {
+            /* don't start stdin_w unless all subprocesses have
+             * received credits to write to stdin */
+            if (stdin_enable_flow_control) {
+                int min_credits = subprocess_min_credits ();
+                if (min_credits)
+                    flux_watcher_start (stdin_w);
+            }
+            else
+                flux_watcher_start (stdin_w);
+        }
         if (exited == rank_count)
             flux_watcher_stop (stdin_w);
     }
 
-    if (state == FLUX_SUBPROCESS_EXEC_FAILED
-        || state == FLUX_SUBPROCESS_FAILED) {
+    if (state == FLUX_SUBPROCESS_FAILED) {
         flux_cmd_t *cmd = flux_subprocess_get_cmd (p);
         int errnum = flux_subprocess_fail_errno (p);
+        const char *errmsg = flux_subprocess_fail_error (p);
         int ec = 1;
 
+        /* N.B. if no error message available from
+         * flux_subprocess_fail_error(), errmsg is set to strerror of
+         * subprocess errno.
+         */
         log_msg ("Error: rank %d: %s: %s",
                  flux_subprocess_rank (p),
                  flux_cmd_arg (cmd, 0),
-                 strerror (errnum));
+                 errmsg);
 
         /* bash standard, 126 for permission/access denied, 127 for
          * command not found.  68 (EX_NOHOST) for No route to host.
@@ -183,57 +264,109 @@ void state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
 
 void output_cb (flux_subprocess_t *p, const char *stream)
 {
-    FILE *fstream = !strcmp (stream, "stderr") ? stderr : stdout;
-    const char *ptr;
-    int lenp;
+    FILE *fstream = streq (stream, "stderr") ? stderr : stdout;
+    const char *buf;
+    int len;
 
-    if (!(ptr = flux_subprocess_getline (p, stream, &lenp)))
-        log_err_exit ("flux_subprocess_getline");
+    if ((len = flux_subprocess_read (p, stream, &buf)) < 0)
+        log_err_exit ("flux_subprocess_read");
 
-    if (lenp) {
-        if (optparse_getopt (opts, "labelio", NULL) > 0)
+    if (len) {
+        if (optparse_getopt (opts, "label-io", NULL) > 0)
             fprintf (fstream, "%d: ", flux_subprocess_rank (p));
-        fwrite (ptr, lenp, 1, fstream);
+        fprintf (fstream, "%.*s", len, buf);
     }
 }
 
-static void stdin_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
+void credit_cb (flux_subprocess_t *p, const char *stream, int bytes)
 {
-    flux_buffer_t *fb = flux_buffer_read_watcher_get_buffer (w);
+    subprocess_update_credits (p, bytes, true);
+    if (started == rank_count) {
+        int min_credits = subprocess_min_credits ();
+        if (min_credits)
+            flux_watcher_start (stdin_w);
+    }
+}
+
+static void stdin_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct fbuf *fb = fbuf_read_watcher_get_buffer (w);
     flux_subprocess_t *p;
     const char *ptr;
-    int lenp;
+    int len, lenp;
+    int min_credits = -1;
+
+    if (stdin_enable_flow_control)
+        min_credits = subprocess_min_credits ();
 
-    if (!(ptr = flux_buffer_read (fb, -1, &lenp)))
-        log_err_exit ("flux_buffer_read");
+    if (!(ptr = fbuf_read (fb, min_credits, &lenp)))
+        log_err_exit ("fbuf_read");
 
     if (lenp) {
-        p = zlist_first (subprocesses);
+        p = zlistx_first (subprocesses);
         while (p) {
             if (flux_subprocess_state (p) == FLUX_SUBPROCESS_INIT
                 || flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING) {
-                if (flux_subprocess_write (p, "stdin", ptr, lenp) < 0)
+                if ((len = flux_subprocess_write (p, "stdin", ptr, lenp)) < 0)
                     log_err_exit ("flux_subprocess_write");
+                if (stdin_enable_flow_control) {
+                    /* N.B. since we are subtracting the same number
+                     * of credits from all subprocesses, the sorted
+                     * order in the credits list should not change
+                     */
+                    subprocess_update_credits (p, -1*len, false);
+                }
             }
-            p = zlist_next (subprocesses);
+            p = zlistx_next (subprocesses);
+        }
+        if (stdin_enable_flow_control) {
+            min_credits = subprocess_min_credits ();
+            if (min_credits == 0)
+                flux_watcher_stop (stdin_w);
         }
     }
     else {
-        p = zlist_first (subprocesses);
+        p = zlistx_first (subprocesses);
         while (p) {
             if (flux_subprocess_close (p, "stdin") < 0)
                 log_err_exit ("flux_subprocess_close");
-            p = zlist_next (subprocesses);
+            p = zlistx_next (subprocesses);
         }
         flux_watcher_stop (stdin_w);
     }
 }
 
-static void signal_cb (int signum)
+static void killall (zlistx_t *l, int signum)
 {
-    flux_subprocess_t *p = zlist_first (subprocesses);
+    flux_subprocess_t *p = zlistx_first (l);
+    while (p) {
+        if (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING) {
+            /* RFC 15 states that the IMP will treat SIGUSR1 as a surrogate
+             * for SIGKILL.
+             */
+            if (use_imp && signum == SIGKILL)
+                signum = SIGUSR1;
 
+            flux_future_t *f = flux_subprocess_kill (p, signum);
+            if (!f) {
+                if (optparse_getopt (opts, "verbose", NULL) > 0)
+                    fprintf (stderr,
+                             "failed to signal rank %d: %s\n",
+                            flux_subprocess_rank (p),
+                            strerror (errno));
+            }
+            /* don't care about response */
+            flux_future_destroy (f);
+        }
+        p = zlistx_next (l);
+    }
+}
+
+static void signal_cb (int signum)
+{
     if (signum == SIGINT) {
         if (sigint_count >= 2) {
             double since_last = monotime_since (last);
@@ -244,7 +377,8 @@ static void signal_cb (int signum)
                 if (!(idset_str = idset_encode (hanging, flags)))
                     log_err_exit ("idset_encode");
 
-                fprintf (stderr, "%s: command still running at exit\n",
+                fprintf (stderr,
+                         "%s: command still running at exit\n",
                          idset_str);
                 free (idset_str);
                 exit (1);
@@ -253,22 +387,12 @@ static void signal_cb (int signum)
     }
 
     if (optparse_getopt (opts, "verbose", NULL) > 0)
-        fprintf (stderr, "sending signal %d to %d running processes\n",
-                 signum, started - exited);
+        fprintf (stderr,
+                 "sending signal %d to %d running processes\n",
+                 signum,
+                 started - exited);
 
-    while (p) {
-        if (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING) {
-            flux_future_t *f = flux_subprocess_kill (p, signum);
-            if (!f) {
-                if (optparse_getopt (opts, "verbose", NULL) > 0)
-                    fprintf (stderr, "failed to signal rank %d: %s\n",
-                             flux_subprocess_rank (p), strerror (errno));
-            }
-            /* don't care about response */
-            flux_future_destroy (f);
-        }
-        p = zlist_next (subprocesses);
-    }
+    killall (subprocesses, signum);
 
     if (signum == SIGINT) {
         if (sigint_count)
@@ -282,12 +406,21 @@ static void signal_cb (int signum)
     }
 }
 
-void subprocess_destroy (void *arg)
+void subprocess_destroy (void **arg)
 {
-    flux_subprocess_t *p = arg;
+    flux_subprocess_t *p = *arg;
     flux_subprocess_destroy (p);
 }
 
+int subprocess_credits_compare (const void *item1, const void *item2)
+{
+    flux_subprocess_t *p1 = (flux_subprocess_t *) item1;
+    flux_subprocess_t *p2 = (flux_subprocess_t *) item2;
+    struct subproc_credit *spcred1 = flux_subprocess_aux_get (p1, "credits");
+    struct subproc_credit *spcred2 = flux_subprocess_aux_get (p2, "credits");
+    return NUMCMP (spcred1->credits, spcred2->credits);
+}
+
 /* atexit handler
  * This is a good faith attempt to restore stdin flags to what they were
  * before we set O_NONBLOCK per bug #1803.
@@ -297,13 +430,196 @@ void restore_stdin_flags (void)
     (void)fcntl (STDIN_FILENO, F_SETFL, stdin_flags);
 }
 
+char *split_opt (const char *s, char sep, const char **val)
+{
+    char *cpy = strdup (s);
+    if (!cpy)
+        return NULL;
+    char *cp = strchr (cpy, sep);
+    if (!cp) {
+        free (cpy);
+        errno = EINVAL;
+        return NULL;
+    }
+    *cp++ = '\0';
+    *val = cp;
+    return cpy;
+}
+
+static bool check_for_imp_run (int argc, char *argv[], const char **ppath)
+{
+    /* If argv0 basename is flux-imp, then we'll likely have to use
+     *  flux-imp kill to signal the resulting subprocesses
+     */
+    if (streq (basename_simple (argv[0]), "flux-imp")) {
+        *ppath = argv[0];
+        return true;
+    }
+    return false;
+}
+
+static const char *get_flux_imp_path (flux_t *h)
+{
+    const char *imp = NULL;
+    flux_future_t *f;
+
+    if (!(f = flux_rpc (h, "config.get", NULL, FLUX_NODEID_ANY, 0))
+        || flux_rpc_get_unpack (f,
+                                "{s?{s?s}}",
+                                "exec",
+                                "imp", &imp) < 0)
+        fprintf (stderr, "error fetching config object: %s",
+                 future_strerror (f, errno));
+    flux_aux_set (h, NULL, f, (flux_free_f) flux_future_destroy);
+    return imp;
+}
+
+/*  Return true if all ids in `idset` are valid indices into `ranks`.
+ */
+static bool check_valid_indices (struct idset *ranks,
+                                 struct idset *idset)
+{
+    /*  idset of NULL is valid since it will be treated as all ids
+     */
+    if (idset == NULL)
+        return true;
+    return (idset_last (idset) < idset_count (ranks));
+}
+
+static void filter_ranks (struct idset *ranks,
+                          const char *include,
+                          const char *exclude,
+                          bool relative)
+{
+    unsigned int i;
+    int n = 0;
+    struct idset *include_ids = NULL;
+    struct idset *exclude_ids = NULL;
+
+    if (!streq (include, "all")
+        && !(include_ids = idset_decode (include)))
+        log_err_exit ("failed to decode idset '%s'", include);
+
+    /*  include_ids is a set of indices into the `ranks` idset.
+     *  (This works because we always start with ranks [0, size-1])
+     *
+     *  Check that each index in include_ids is valid before proceeding.
+     */
+    if (!check_valid_indices (ranks, include_ids))
+        log_msg_exit ("One or more invalid --ranks specified: %s",
+                      include);
+
+    if (exclude && !(exclude_ids = idset_decode (exclude)))
+        log_err_exit ("error decoding --exclude idset");
+
+    /*  Note: it is not an error if exclude_ids falls outside of the
+     *  ranks idset, this is simply ignored.
+     */
+
+    i = idset_first (ranks);
+    while (i != IDSET_INVALID_ID) {
+        /*
+         * Remove this id from ranks if one of the following is true
+         *  - it is in exclude_ids if relative == false
+         *  - the index of this id is in exclude_ids if relative == true
+         *  - the index of this id is in include_ids if include_ids != NULL.
+         */
+        if (idset_test (exclude_ids, relative ? n : i)
+            || (include_ids && !idset_test (include_ids, n))) {
+            if (idset_clear (ranks, i) < 0)
+                log_err_exit ("idset_clear");
+        }
+        i = idset_next (ranks, i);
+        n++;
+    }
+    idset_destroy (include_ids);
+    idset_destroy (exclude_ids);
+}
+
+/*  Get job shell rexec service name and broker ranks for job.
+ */
+int get_jobid_rexec_info (flux_t *h,
+                          const char *jobid,
+                          char **servicep,
+                          struct idset **idsetp)
+{
+    flux_future_t *f;
+    flux_jobid_t id;
+    flux_job_state_t state;
+    const char *ranks;
+    struct idset *ids;
+    bool done = false;
+
+    if (flux_job_id_parse (jobid, &id) < 0)
+        log_msg_exit ("error parsing jobid: \"%s\"", jobid);
+
+    if (!(f = flux_rpc_pack (h,
+                            "job-list.list-id",
+                            FLUX_NODEID_ANY,
+                            0,
+                            "{s:I s:[ss]}",
+                            "id", id,
+                            "attrs", "ranks", "state"))
+        || flux_rpc_get_unpack (f,
+                                "{s:{s:i s:s}}",
+                                "job",
+                                 "state", &state,
+                                 "ranks", &ranks) < 0) {
+        if (errno == ENOENT)
+            log_msg_exit ("job %s not found", jobid);
+        log_err_exit ("unable to get info for job %s", jobid);
+    }
+
+    if (state != FLUX_JOB_STATE_RUN)
+        log_msg_exit ("job %s is not currently running", jobid);
+
+    if (!(ids = idset_decode (ranks)) || idset_empty (ids))
+        log_msg_exit ("failed to get assigned ranks for %s", jobid);
+    *idsetp = ids;
+
+    flux_future_destroy (f);
+
+    if (!(f = flux_job_event_watch (h,
+                                    id,
+                                    "guest.exec.eventlog",
+                                    FLUX_JOB_EVENT_WATCH_WAITCREATE)))
+        log_err_exit ("flux_job_event_watch");
+
+    while (!done) {
+        json_t *o;
+        json_t *context;
+        const char *event;
+        const char *name;
+
+        if (flux_job_event_watch_get (f, &event) < 0)
+            log_msg_exit ("failed to get shell.init event for %s", jobid);
+
+        if (!(o = eventlog_entry_decode (event))
+            || eventlog_entry_parse (o, NULL, &name, &context) < 0)
+            log_err_exit ("failed to decode exec eventlog event");
+
+        if (streq (name, "shell.init")) {
+            const char *service = NULL;
+            if (json_unpack (context, "{s:s}", "service", &service) < 0)
+                log_msg_exit ("failed to get service from shell.init event");
+            if (asprintf (servicep, "%s.rexec", service) < 0)
+                log_err_exit ("unable to create job rexec topic string");
+            done = true;
+        }
+        json_decref (o);
+        flux_future_reset (f);
+    }
+    flux_future_destroy (f);
+    return 0;
+}
+
+
 int main (int argc, char *argv[])
 {
     const char *optargp;
     int optindex;
-    flux_t *h;
     flux_reactor_t *r;
-    struct idset *ns;
+    struct idset *targets;
     uint32_t rank;
     flux_cmd_t *cmd;
     char *cwd = NULL;
@@ -313,8 +629,11 @@ int main (int argc, char *argv[])
         .on_channel_out = NULL,
         .on_stdout = output_cb,
         .on_stderr = output_cb,
+        .on_credit = credit_cb,
     };
     struct timespec t0;
+    const char *service_name;
+    char *job_service = NULL;
 
     log_init ("flux-exec");
 
@@ -332,6 +651,8 @@ int main (int argc, char *argv[])
     if (!(cmd = flux_cmd_create (argc - optindex, &argv[optindex], environ)))
         log_err_exit ("flux_cmd_create");
 
+    flux_cmd_unsetenv (cmd, "FLUX_PROXY_REMOTE");
+
     if (optparse_getopt (opts, "dir", &optargp) > 0) {
         if (!(cwd = strdup (optargp)))
             log_err_exit ("strdup");
@@ -341,74 +662,159 @@ int main (int argc, char *argv[])
             log_err_exit ("get_current_dir_name");
     }
 
-    if (flux_cmd_setcwd (cmd, cwd) < 0)
-        log_err_exit ("flux_cmd_setcwd");
+    if (!streq (cwd, "none")) {
+        if (flux_cmd_setcwd (cmd, cwd) < 0)
+            log_err_exit ("flux_cmd_setcwd");
+    }
+    if (optparse_hasopt (opts, "setopt")) {
+        const char *arg;
+        optparse_getopt_iterator_reset (opts, "setopt");
+        while ((arg = optparse_getopt_next (opts, "setopt"))) {
+            const char *value;
+            char *name = split_opt (arg, '=', &value);
+            if (!name || flux_cmd_setopt (cmd, name, value) < 0)
+                log_err_exit ("error handling '%s' option", arg);
+            free (name);
+        }
+    }
 
-    if (!(h = flux_open (NULL, 0)))
+    if (!(flux_handle = flux_open (NULL, 0)))
         log_err_exit ("flux_open");
 
+    /* Assign h to flux_handle for local usage in main()
+     */
+    flux_t *h = flux_handle;
+
     if (!(r = flux_get_reactor (h)))
         log_err_exit ("flux_get_reactor");
 
     if (flux_get_size (h, &rank_range) < 0)
         log_err_exit ("flux_get_size");
 
-    if (optparse_getopt (opts, "rank", &optargp) > 0
-        && strcmp (optargp, "all")) {
-        if (!(ns = idset_decode (optargp)))
-            log_err_exit ("idset_decode");
-        if (!(rank_count = idset_count (ns)))
-            log_err_exit ("idset_count");
+    if (optparse_hasopt (opts, "with-imp")) {
+        if (!(imp_path = get_flux_imp_path (h)))
+            log_err_exit ("--with-imp: exec.imp path not found in config");
+        use_imp = true;
+        if (flux_cmd_argv_insert (cmd, 0, "run") < 0
+            || flux_cmd_argv_insert (cmd, 0, imp_path) < 0)
+            log_err_exit ("failed to prepend 'flux-imp run' to command");
     }
     else {
-        if (!(ns = idset_create (0, IDSET_FLAG_AUTOGROW)))
-            log_err_exit ("idset_create");
-        if (idset_range_set (ns, 0, rank_range - 1) < 0)
-            log_err_exit ("idset_range_set");
-        rank_count = rank_range;
+        use_imp = check_for_imp_run (argc - optindex,
+                                     &argv[optindex],
+                                     &imp_path);
     }
 
-    if (optparse_hasopt (opts, "exclude")) {
-        struct idset *xset;
-
-        if (!(xset = idset_decode (optparse_get_str (opts, "exclude", NULL))))
-            log_err_exit ("error decoding --exclude idset");
-        if (idset_range_clear (ns, idset_first (xset), idset_last (xset)) < 0)
-            log_err_exit ("error apply --exclude idset");
-        idset_destroy (xset);
+    /* Allow systemd commands to work on flux systemd instance by
+     * setting DBUS_SESSION_BUS_ADDRESS if not already set.
+     * See flux-framework/flux-core#5901
+     */
+    const char *security_owner;
+    if (!(security_owner = flux_attr_get (h, "security.owner")))
+        log_err_exit ("failed to fetch security.owner attribute");
+    (void)flux_cmd_setenvf (cmd,
+                            0,
+                            "DBUS_SESSION_BUS_ADDRESS",
+                            "unix:path=/run/user/%s/bus",
+                            security_owner);
+
+    /* Get input ranks from --jobid if given:
+     */
+    if (optparse_getopt (opts, "jobid", &optargp) > 0) {
+        get_jobid_rexec_info (h, optargp, &job_service, &targets);
+    }
+    else {
+        if (!(targets = idset_create (0, IDSET_FLAG_AUTOGROW)))
+            log_err_exit ("idset_create");
+        if (idset_range_set (targets, 0, rank_range - 1) < 0)
+            log_err_exit ("idset_range_set");
     }
 
-    if (!(hanging = idset_copy (ns)))
+    /* Include and exclude ranks based on --rank and --exclude options
+     * Make rank exclusion relative to job ranks if --jobid was used.
+     */
+    filter_ranks (targets,
+                  optparse_get_str (opts, "rank", "all"),
+                  optparse_get_str (opts, "exclude", NULL),
+                  optparse_hasopt (opts, "jobid"));
+
+    rank_count = idset_count (targets);
+    if (rank_count == 0)
+        log_msg_exit ("No targets specified");
+    if (!(hanging = idset_copy (targets)))
         log_err_exit ("idset_copy");
 
     monotime (&t0);
     if (optparse_getopt (opts, "verbose", NULL) > 0) {
         const char *argv0 = flux_cmd_arg (cmd, 0);
-        char *nodeset = idset_encode (ns, IDSET_FLAG_RANGE
-                                        | IDSET_FLAG_BRACKETS);
+        char *nodeset = idset_encode (targets,
+                                      IDSET_FLAG_RANGE | IDSET_FLAG_BRACKETS);
         if (!nodeset)
             log_err_exit ("idset_encode");
-        fprintf (stderr, "%03fms: Starting %s on %s\n",
-                 monotime_since (t0), argv0, nodeset);
+        fprintf (stderr,
+                 "%03fms: Starting %s on %s\n",
+                 monotime_since (t0),
+                 argv0,
+                 nodeset);
         free (nodeset);
     }
 
-    if (!(subprocesses = zlist_new ()))
-        log_err_exit ("zlist_new");
+    if (!(subprocesses = zlistx_new ()))
+        log_err_exit ("zlistx_new");
+    zlistx_set_destructor (subprocesses, subprocess_destroy);
+
+    if (!(subprocess_credits = zlistx_new ()))
+        log_err_exit ("zlistx_new");
+    zlistx_set_comparator (subprocess_credits, subprocess_credits_compare);
 
     if (!(exitsets = zhashx_new ()))
         log_err_exit ("zhashx_new()");
 
-    rank = idset_first (ns);
+    service_name = optparse_get_str (opts,
+                                     "service",
+                                     job_service ? job_service : "rexec");
+
+    // sdexec stdin flow is disabled by default
+    if (streq (service_name, "sdexec"))
+        stdin_enable_flow_control = false;
+
+    const char *stdin_flow = optparse_get_str (opts, "stdin-flow", NULL);
+    if (stdin_flow) {
+        if (streq (stdin_flow, "off"))
+            stdin_enable_flow_control = false;
+        else if (streq (stdin_flow, "on"))
+            stdin_enable_flow_control = true;
+        else
+            log_msg_exit ("Set --stdin-flow to on or off");
+    }
+    if (!stdin_enable_flow_control)
+        ops.on_credit = NULL;
+
+    rank = idset_first (targets);
     while (rank != IDSET_INVALID_ID) {
         flux_subprocess_t *p;
-        if (!(p = flux_rexec (h, rank, 0, cmd, &ops)))
+        struct subproc_credit *spcred;
+        if (!(p = flux_rexec_ex (h,
+                                 service_name,
+                                 rank,
+                                 FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF,
+                                 cmd,
+                                 &ops,
+                                 NULL,
+                                 NULL)))
             log_err_exit ("flux_rexec");
-        if (zlist_append (subprocesses, p) < 0)
-            log_err_exit ("zlist_append");
-        if (!zlist_freefn (subprocesses, p, subprocess_destroy, true))
-            log_err_exit ("zlist_freefn");
-        rank = idset_next (ns, rank);
+        if (!(spcred = calloc (1, sizeof (*spcred))))
+            log_err_exit ("calloc");
+        if (!zlistx_add_end (subprocesses, p))
+            log_err_exit ("zlistx_add_end");
+        if (!(spcred->handle = zlistx_add_end (subprocess_credits, p)))
+            log_err_exit ("zlistx_add_end");
+        if (flux_subprocess_aux_set (p,
+                                     "credits",
+                                     spcred,
+                                     (flux_free_f) free) < 0)
+            log_err_exit ("flux_subprocess_aux_set");
+        rank = idset_next (targets, rank);
     }
 
     if (optparse_getopt (opts, "verbose", NULL) > 0)
@@ -418,11 +824,11 @@ int main (int argc, char *argv[])
      */
     if (optparse_getopt (opts, "noinput", NULL) > 0) {
         flux_subprocess_t *p;
-        p = zlist_first (subprocesses);
+        p = zlistx_first (subprocesses);
         while (p) {
             if (flux_subprocess_close (p, "stdin") < 0)
                 log_err_exit ("flux_subprocess_close");
-            p = zlist_next (subprocesses);
+            p = zlistx_next (subprocesses);
         }
     }
     /* configure stdin watcher
@@ -434,10 +840,13 @@ int main (int argc, char *argv[])
             log_err_exit ("atexit");
         if (fcntl (STDIN_FILENO, F_SETFL, stdin_flags | O_NONBLOCK) < 0)
             log_err_exit ("fcntl F_SETFL stdin");
-        if (!(stdin_w = flux_buffer_read_watcher_create (r, STDIN_FILENO,
-                                                         1 << 20, stdin_cb,
-                                                         0, NULL)))
-            log_err_exit ("flux_buffer_read_watcher_create");
+        if (!(stdin_w = fbuf_read_watcher_create (r,
+                                                  STDIN_FILENO,
+                                                  1 << 20,
+                                                  stdin_cb,
+                                                  0,
+                                                  NULL)))
+            log_err_exit ("fbuf_read_watcher_create");
     }
     if (signal (SIGINT, signal_cb) == SIG_ERR)
         log_err_exit ("signal");
@@ -449,11 +858,14 @@ int main (int argc, char *argv[])
         log_err_exit ("flux_reactor_run");
 
     if (optparse_getopt (opts, "verbose", NULL) > 0)
-        fprintf (stderr, "%03fms: %d tasks complete with code %d\n",
-                 monotime_since (t0), exited, exit_code);
+        fprintf (stderr,
+                 "%03fms: %d tasks complete with code %d\n",
+                 monotime_since (t0),
+                 exited,
+                 exit_code);
 
     /* output message on any tasks that exited non-zero */
-    if (zhashx_size (exitsets) > 0) {
+    if (!optparse_hasopt (opts, "quiet") && zhashx_size (exitsets) > 0) {
         struct id_set *idset = zhashx_first (exitsets);
         while (idset) {
             const char *key = zhashx_cursor (exitsets);
@@ -464,7 +876,8 @@ int main (int argc, char *argv[])
 
     /* Clean up.
      */
-    idset_destroy (ns);
+    idset_destroy (targets);
+    free (job_service);
     free (cwd);
     flux_cmd_destroy(cmd);
     flux_close (h);
@@ -472,7 +885,8 @@ int main (int argc, char *argv[])
     log_fini ();
 
     zhashx_destroy (&exitsets);
-    zlist_destroy (&subprocesses);
+    zlistx_destroy (&subprocesses);
+    zlistx_destroy (&subprocess_credits);
 
     return exit_code;
 }
diff --git a/src/cmd/flux-fortune.py b/src/cmd/flux-fortune.py
new file mode 100755
index 000000000000..185e7b9aaeba
--- /dev/null
+++ b/src/cmd/flux-fortune.py
@@ -0,0 +1,43 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import logging
+import sys
+
+import flux
+import flux.cli.fortune as base
+
+LOGGER = logging.getLogger("flux-fortune")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    # Prepare the fortune parser
+    fortune = base.FortuneCmd(
+        "flux fortune",
+        description="Eeenie meenie chilie beanie, the Flux fortune is about to speak!",
+    )
+    parser = fortune.get_parser()
+    parser.set_defaults(func=fortune.main)
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-hostlist.py b/src/cmd/flux-hostlist.py
new file mode 100755
index 000000000000..38dcce14d4a0
--- /dev/null
+++ b/src/cmd/flux-hostlist.py
@@ -0,0 +1,412 @@
+#############################################################
+# Copyright 2024 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+import os
+import select
+import sys
+
+import flux
+import flux.util
+from flux.hostlist import Hostlist
+from flux.idset import IDset
+from flux.job import JobID, job_list_id
+from flux.resource import resource_status
+
+LOGGER = logging.getLogger("flux-hostlist")
+
+sources_description = """
+SOURCES may include:
+instance     hosts from the broker 'hostlist' attribute.
+jobid        hosts assigned to a job
+local        hosts assigned to current job if FLUX_JOB_ID is set, otherwise
+             returns the 'instance' hostlist
+avail[able]  instance hostlist minus those nodes down or drained
+stdin, '-'   read a list of hosts on stdin
+hosts        literal list of hosts
+
+The default when no SOURCES are supplied is 'stdin', unless the -l, --local
+option is used, in which case the default is 'local'.
+"""
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        prog="flux-hostlist",
+        epilog=sources_description,
+        formatter_class=flux.util.help_formatter(raw_description=True),
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        "-e",
+        "--expand",
+        action="store_true",
+        help="Expand hostlist using defined output delimiter",
+    )
+    parser.add_argument(
+        "-d",
+        "--delimiter",
+        type=str,
+        metavar="S",
+        default=" ",
+        help="Set output delimiter for expanded hostlist (default=' ')",
+    )
+    group.add_argument(
+        "-c",
+        "--count",
+        action="store_true",
+        help="Print the total number of hosts",
+    )
+    parser.add_argument(
+        "-n",
+        "--nth",
+        type=str,
+        metavar="[-]IDS",
+        help="Output hosts at indices in idset IDS (-IDS to index from end)",
+    )
+    parser.add_argument(
+        "-L",
+        "--limit",
+        metavar="N",
+        type=int,
+        help="Output at most N hosts (-N for the last N hosts)",
+    )
+    parser.add_argument(
+        "-S",
+        "--sort",
+        action="store_true",
+        help="Return sorted result",
+    )
+    parser.add_argument(
+        "-x",
+        "--exclude",
+        metavar="IDS|HOSTS",
+        type=Hostlist,
+        help="Exclude all occurrences of HOSTS or indices from final result",
+    )
+    parser.add_argument(
+        "-u",
+        "--union",
+        "--unique",
+        action="store_true",
+        help="Return only unique hosts in the final hostlist. "
+        + "Without other options, this is the same as the union of all "
+        + "hostlist args (default mode is append).",
+    )
+    group2 = parser.add_mutually_exclusive_group()
+    group2.add_argument(
+        "-i",
+        "--intersect",
+        action="store_true",
+        help="Return the intersection of all hostlists",
+    )
+    group2.add_argument(
+        "-m",
+        "--minus",
+        action="store_true",
+        help="Subtract all hostlist args from first argument",
+    )
+    group2.add_argument(
+        "-X",
+        "--xor",
+        action="store_true",
+        help="Return the symmetric difference of all hostlists",
+    )
+    parser.add_argument(
+        "-f",
+        "--fallback",
+        action="store_true",
+        help="Fallback to treating jobids that are not found as hostnames"
+        + " (for hostnames that are also valid jobids e.g. f1, fuzz100, etc)",
+    )
+    parser.add_argument(
+        "-l",
+        "--local",
+        action="store_true",
+        help="Set the default source to 'local' instead of 'stdin'",
+    )
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        help="No output. Exit with nonzero exit status if hostlist is empty",
+    )
+    parser.add_argument(
+        "sources",
+        metavar="SOURCES",
+        nargs="*",
+        help="(optional) One or more hostlist sources",
+    )
+    return parser.parse_args()
+
+
+class FluxHandle:
+    """Singleton Flux handle"""
+
+    def __new__(cls):
+        if not hasattr(cls, "handle"):
+            cls.handle = flux.Flux()
+        return cls.handle
+
+
+class HostlistResult:
+    """class representing a simple hostlist result"""
+
+    def __init__(self, hosts, **kwargs):
+        self.result = Hostlist(hosts)
+
+
+class InstanceHostlistResult:
+    """class representing a hostlist from the instance hostlist attribute"""
+
+    def __init__(self, *args, **kwargs):
+        self.result = Hostlist(FluxHandle().attr_get("hostlist"))
+
+
+class JobHostlistResult:
+    """class representing a job hostlist obtained from job-list"""
+
+    def __init__(self, jobid, **kwargs):
+        self.arg = jobid
+        self.jobid = JobID(jobid)
+        self.fallback = kwargs.get("fallback", False)
+        self.future = job_list_id(FluxHandle(), self.jobid, attrs=["nodelist"])
+
+    @property
+    def result(self):
+        try:
+            job = self.future.get_jobinfo()
+        except OSError as exc:
+            if isinstance(exc, FileNotFoundError):
+                if self.fallback:
+                    # Fall back to treating potential jobid as hostname
+                    return Hostlist(self.arg)
+                else:
+                    raise ValueError(f"job {self.arg} not found") from None
+            else:
+                raise ValueError(f"job {self.arg}: {exc}") from None
+        return Hostlist(job.nodelist)
+
+
+class AvailableHostlistResult:
+    """class representing available hosts in enclosing instance"""
+
+    def __init__(self, *args, **kwargs):
+        """Get local hostlist and return only available hosts"""
+        # Store local hostlist in self.hl:
+        self.hl = LocalHostlistResult().result
+        # Send resource status RPCs to get available idset
+        self.rstatus = resource_status(FluxHandle())
+
+    @property
+    def result(self):
+        # Restrict returned hostlist to only those available:
+        avail = self.rstatus.get().avail
+        return self.hl[avail]
+
+
+class LocalHostlistResult:
+    """class representing 'local' hostlist from enclosing instance or job"""
+
+    def __init__(self, *args, **kwargs):
+        self.jobid_result = None
+        if "FLUX_JOB_ID" in os.environ:
+            # This process is running in the context of a job (not initial
+            # program) if "FLUX_JOB_ID" is found in current environment.
+            # Fetch hostlist via a query to job-list service:
+            self._base = JobHostlistResult(os.environ["FLUX_JOB_ID"])
+        else:
+            # O/w, this is either an initial program or the enclosing instance
+            # is the system instance. Fetch the hostlist attribuee either way
+            self._base = InstanceHostlistResult()
+
+    @property
+    def result(self):
+        return self._base.result
+
+
+class StdinHostlistResult:
+    """class representing a hostlist read on stdin"""
+
+    def __init__(self, *args, **kwargs):
+        hl = Hostlist()
+
+        # Note: Previous versions of this command defaulted to reading
+        # the current enclosing instance or job hostlist, not from stdin.
+        # To avoid potential hangs, only wait for stdin for 15s. This should
+        # be removed in a future version
+        timeout = float(os.environ.get("FLUX_HOSTLIST_STDIN_TIMEOUT", 15.0))
+        if not select.select([sys.stdin], [], [], timeout)[0]:
+            raise RuntimeError(f"timeout after {timeout}s waiting for stdin")
+
+        for line in sys.stdin.readlines():
+            hl.append(line.rstrip())
+        self.result = hl
+
+
+class HostlistResolver:
+    """
+    Resolve a set of hostlist references in 'sources' into a list of hostlists.
+    """
+
+    result_types = {
+        "instance": InstanceHostlistResult,
+        "local": LocalHostlistResult,
+        "stdin": StdinHostlistResult,
+        "-": StdinHostlistResult,
+        "avail": AvailableHostlistResult,
+        "available": AvailableHostlistResult,
+    }
+
+    def __init__(self, sources, fallback=False):
+        self._results = []
+        self.fallback = fallback
+        for arg in sources:
+            self.append(arg)
+
+    def append(self, arg):
+        if arg in self.result_types:
+            lookup = self.result_types[arg](arg, fallback=self.fallback)
+            self._results.append(lookup)
+        else:
+            try:
+                #  Try argument as a jobid:
+                result = JobHostlistResult(arg, fallback=self.fallback)
+                self._results.append(result)
+            except ValueError:
+                try:
+                    #  Try argument as a literal Hostlist
+                    self._results.append(HostlistResult(arg))
+                except (TypeError, OSError, ValueError):
+                    raise ValueError(f"Invalid jobid or hostlist {arg}")
+
+    def results(self):
+        return [entry.result for entry in self._results]
+
+
+def intersect(hl1, hl2):
+    """Set intersection of Hostlists hl1 and hl2"""
+    result = Hostlist()
+    for host in hl1:
+        if host in hl2:
+            result.append(host)
+    result.uniq()
+    return result
+
+
+def difference(hl1, hl2):
+    """Return hosts in hl1 not in hl2"""
+    result = Hostlist()
+    for host in hl1:
+        if host not in hl2:
+            result.append(host)
+    return result
+
+
+def xor(hl1, hl2):
+    """Return hosts in hl1 or hl2 but not both"""
+    result = difference(hl1, hl2)
+    result.append(difference(hl2, hl1))
+    result.uniq()
+    return result
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    args = parse_args()
+
+    if not args.sources:
+        if args.local:
+            args.sources = ["local"]
+        else:
+            args.sources = ["stdin"]
+
+    hostlists = HostlistResolver(args.sources, fallback=args.fallback).results()
+
+    hl = Hostlist()
+
+    if args.intersect:
+        hl = hostlists.pop(0)
+        for x in hostlists:
+            hl = intersect(hl, x)
+    elif args.xor:
+        hl = hostlists.pop(0)
+        for x in hostlists:
+            hl = xor(hl, x)
+    elif args.minus:
+        hl = hostlists.pop(0)
+        for x in hostlists:
+            hl.delete(x)
+    else:
+        for x in hostlists:
+            hl.append(x)
+
+    if args.exclude:
+        # Delete all occurrences of args.exclude
+        count = len(hl)
+        while hl.delete(args.exclude) > 0:
+            pass
+        if len(hl) == count:
+            # No hosts were deleted, try args.exclude as idset of indices:
+            try:
+                exclude = IDset(args.exclude)
+                hl = Hostlist([hl[i] for i in range(count) if i not in exclude])
+            except ValueError:
+                # not a valid idset, just pass unaltered hostlist along
+                pass
+
+    if args.sort:
+        hl.sort()
+
+    if args.union:
+        hl.uniq()
+
+    if args.limit:
+        if args.limit > 0:
+            hl = hl[: args.limit]
+        else:
+            hl = hl[args.limit :]
+
+    if args.quiet:
+        sys.stdout = open(os.devnull, "w")
+
+    if args.nth is not None:
+        if args.nth.startswith("-"):
+            # Iterate idset in reverse so that resultant hostlist is in
+            # the same order as the input hostlist instead of reversed:
+            hl = Hostlist([hl[-x] for x in reversed(list(IDset(args.nth[1:])))])
+        else:
+            hl = hl[IDset(args.nth)]
+
+    if args.count:
+        print(f"{hl.count()}")
+    elif args.expand:
+        # Convert '\n' specified on command line to actual newline char
+        if hl:
+            print(args.delimiter.replace("\\n", "\n").join(hl))
+    else:
+        if hl:
+            print(hl.encode())
+
+    if args.quiet and not hl:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-housekeeping.py b/src/cmd/flux-housekeeping.py
new file mode 100755
index 000000000000..18ebec894583
--- /dev/null
+++ b/src/cmd/flux-housekeeping.py
@@ -0,0 +1,275 @@
+#############################################################
+# Copyright 2024 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+import sys
+import time
+
+import flux
+import flux.util
+from flux.hostlist import Hostlist
+from flux.idset import IDset
+from flux.job import JobID
+from flux.util import Deduplicator, UtilConfig
+
+LOGGER = logging.getLogger("flux-housekeeping")
+
+
+class FluxHousekeepingConfig(UtilConfig):
+    """flux-housekeeping specific configuration"""
+
+    builtin_formats = {
+        "default": {
+            "description": "Default flux-housekeeping format string",
+            "format": (
+                "{id.f58:>12} {nnodes:>6} {pending.nnodes:>7} "
+                "{runtime!F:>8} {nodelist}"
+            ),
+        },
+        "pending": {
+            "description": "flux-housekeeping format string including active nodes only",
+            "format": (
+                "{id.f58:>12} {pending.nnodes:>7} {runtime!F:>8} {pending.nodelist}"
+            ),
+        },
+    }
+
+    def __init__(self):
+        initial_dict = {"formats": dict(self.builtin_formats)}
+        super().__init__(name="flux-housekeeping", initial_dict=initial_dict)
+
+
+class HKFormat(flux.util.OutputFormat):
+
+    headings = {
+        "id": "JOBID",
+        "id.dec": "JOBID",
+        "id.hex": "JOBID",
+        "id.f58": "JOBID",
+        "id.f58plain": "JOBID",
+        "id.emoji": "JOBID",
+        "id.kvs": "JOBID",
+        "id.words": "JOBID",
+        "id.dothex": "JOBID",
+        "t_start": "T_START",
+        "runtime": "RUNTIME",
+        "ranks": "RANKS",
+        "nodelist": "NODELIST",
+        "nnodes": "NNODES",
+        "allocated.ranks": "RANKS",
+        "allocated.nodelist": "NODELIST",
+        "allocated.nnodes": "NNODES",
+        "pending.ranks": "ACTIVE_RANKS",
+        "pending.nodelist": "ACTIVE_NODES",
+        "pending.nnodes": "#ACTIVE",
+    }
+
+
+class HousekeepingSet:
+    """Container for a set of ranks with ranks, nnodes, nodelist properties"""
+
+    def __init__(self, ranks, hostlist):
+        self.ranks = IDset(ranks)
+        self.hostlist = hostlist
+
+    @property
+    def nnodes(self):
+        return self.ranks.count()
+
+    @property
+    def nodelist(self):
+        return self.hostlist[self.ranks]
+
+    def update(self, other):
+        self.ranks += other.ranks
+
+
+class HousekeepingJob:
+    def __init__(self, jobid, stats_info, hostlist):
+        self.id = JobID(jobid)
+        self.t_start = stats_info["t_start"]
+        self.runtime = time.time() - self.t_start
+        self.pending = HousekeepingSet(stats_info["pending"], hostlist)
+        self.allocated = HousekeepingSet(stats_info["allocated"], hostlist)
+
+    @property
+    def ranks(self):
+        return self.allocated.ranks
+
+    @property
+    def nnodes(self):
+        return self.allocated.nnodes
+
+    @property
+    def nodelist(self):
+        return self.allocated.nodelist
+
+    def filter(self, include_ranks):
+        """Filter this HousekeepingJob's ranks to only include include_ranks"""
+        self.pending.ranks = self.pending.ranks.intersect(include_ranks)
+        self.allocated.ranks = self.allocated.ranks.intersect(include_ranks)
+
+    def combine(self, other):
+        self.pending.update(other.pending)
+        self.allocated.update(other.allocated)
+
+
+def housekeeping_list(args):
+    handle = flux.Flux()
+
+    hostlist = Hostlist(handle.attr_get("hostlist"))
+    stats = handle.rpc("job-manager.stats-get", {}).get()
+
+    include_ranks = None
+    if args.include:
+        try:
+            include_ranks = IDset(args.include)
+        except ValueError:
+            include_ranks = IDset(hostlist.index(args.include))
+
+    fmt = FluxHousekeepingConfig().load().get_format_string(args.format)
+    try:
+        formatter = HKFormat(fmt)
+    except ValueError as err:
+        raise ValueError(f"Error in user format: {err}")
+
+    jobs = Deduplicator(
+        formatter=formatter,
+        except_fields=[
+            "nodelist",
+            "ranks",
+            "nnodes",
+            "pending.nodelist",
+            "pending.ranks",
+            "pending.nnodes",
+        ],
+        combine=lambda job, other: job.combine(other),
+    )
+    for jobid, info in stats["housekeeping"]["running"].items():
+        job = HousekeepingJob(jobid, info, hostlist)
+        if include_ranks:
+            job.filter(include_ranks)
+        if job.nnodes > 0:
+            jobs.append(job)
+
+    formatter.print_items(jobs, no_header=args.no_header)
+
+
+def housekeeping_kill(args):
+    handle = flux.Flux()
+    payload = {"signum": args.signal}
+
+    # Require one selection option (do not default to --all)
+    if args.jobid is None and args.targets is None and not args.all:
+        raise ValueError("specify at least one of --targets, --jobid, or --all")
+    if args.all and args.jobid is not None:
+        raise ValueError("do not specify --jobid with --all")
+    if args.jobid:
+        payload["jobid"] = args.jobid
+    if args.targets:
+        try:
+            ranks = IDset(args.targets)
+        except ValueError:
+            try:
+                hosts = Hostlist(args.targets)
+            except ValueError:
+                raise ValueError("--targets must be a valid Idset or Hostlist")
+            hostlist = Hostlist(handle.attr_get("hostlist"))
+            ranks = IDset()
+            for host in hosts:
+                try:
+                    ranks.set(hostlist.find(host))
+                except OSError:
+                    raise ValueError(f"didn't find {host} in instance hostlist")
+        payload["ranks"] = str(ranks)
+    flux.Flux().rpc("job-manager.housekeeping-kill", payload).get()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(prog="flux-housekeeping")
+    subparsers = parser.add_subparsers(
+        title="subcommands", description="", dest="subcommand"
+    )
+    subparsers.required = True
+
+    list_parser = subparsers.add_parser(
+        "list", formatter_class=flux.util.help_formatter()
+    )
+    list_parser.add_argument(
+        "-i",
+        "--include",
+        metavar="HOSTS|RANKS",
+        type=str,
+        help="Limit output to housekeeping jobs on HOSTS|RANKS",
+    )
+    list_parser.add_argument(
+        "-n",
+        "--no-header",
+        action="store_true",
+        help="Suppress printing of header line",
+    )
+    list_parser.add_argument(
+        "-o",
+        "--format",
+        type=str,
+        default="default",
+        metavar="FORMAT",
+        help="Specify output format using Python's string format syntax "
+        + " or a defined format by name (use 'help' to get a list of names)",
+    )
+    list_parser.set_defaults(func=housekeeping_list)
+
+    kill_parser = subparsers.add_parser(
+        "kill", formatter_class=flux.util.help_formatter()
+    )
+    kill_parser.add_argument(
+        "-s",
+        "--signal",
+        metavar="SIGNUM",
+        type=int,
+        default=15,
+        help="Specify signal number to send to housekeeping task",
+    )
+    kill_parser.add_argument(
+        "-t",
+        "--targets",
+        metavar="RANKS|HOSTS",
+        type=str,
+        help="Only target specific ranks or hostnames",
+    )
+    kill_parser.add_argument(
+        "-j",
+        "--jobid",
+        type=JobID,
+        help='target housekeeping tasks for this jobid or "all" for all jobs',
+    )
+    kill_parser.add_argument(
+        "--all", action="store_true", help="kill all active housekeeping tasks"
+    )
+    kill_parser.set_defaults(func=housekeeping_kill)
+
+    return parser.parse_args()
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    args = parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/cmd/flux-imp-exec-helper b/src/cmd/flux-imp-exec-helper
new file mode 100755
index 000000000000..917514a152b9
--- /dev/null
+++ b/src/cmd/flux-imp-exec-helper
@@ -0,0 +1,20 @@
+#!/bin/sh
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+#
+# Helper for flux-imp exec functionality.
+# Emit input to IMP for jobid on stdout given jobid in $1
+#
+JOBID=${1:-$FLUX_JOB_ID}
+if test -z "$JOBID"; then
+    echo "flux-imp-exec-helper: Unable to determine jobid" >&2
+    exit 1
+fi
+printf '{"J": "%s"}' $(flux job info --orig $JOBID J)
diff --git a/src/cmd/flux-job-exec-override.py b/src/cmd/flux-job-exec-override.py
new file mode 100755
index 000000000000..66782b8980f1
--- /dev/null
+++ b/src/cmd/flux-job-exec-override.py
@@ -0,0 +1,77 @@
+#!/bin/false
+##############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+import sys
+
+import flux
+from flux.job import JobID
+
+LOGGER = logging.getLogger("flux-job-exec-override")
+
+
+def job_exec_start(args):
+    """Start testexec job under manual override"""
+    try:
+        flux.Flux().rpc(
+            "job-exec.override", {"event": "start", "jobid": args.jobid}
+        ).get()
+    except OSError as exc:
+        LOGGER.error("%s", exc.strerror)
+        sys.exit(1)
+
+
+def job_exec_finish(args):
+    """Finish testexec job under manual override"""
+    try:
+        flux.Flux().rpc(
+            "job-exec.override",
+            {"event": "finish", "jobid": args.jobid, "status": args.status},
+        ).get()
+    except OSError as exc:
+        LOGGER.error("%s", exc.strerror)
+        sys.exit(1)
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    parser = argparse.ArgumentParser(prog="flux-job-exec")
+    subparsers = parser.add_subparsers(
+        title="subcommands", description="", dest="subcommand"
+    )
+    subparsers.required = True
+
+    start_parser = subparsers.add_parser(
+        "start", formatter_class=flux.util.help_formatter()
+    )
+    start_parser.add_argument("jobid", metavar="JOBID", type=JobID, help="target JOBID")
+    start_parser.set_defaults(func=job_exec_start)
+
+    finish_parser = subparsers.add_parser(
+        "finish", formatter_class=flux.util.help_formatter()
+    )
+    finish_parser.add_argument(
+        "jobid", metavar="JOBID", type=JobID, help="target JOBID"
+    )
+    finish_parser.add_argument(
+        "status", metavar="STATUS", type=int, help="finish wait status", default=0
+    )
+    finish_parser.set_defaults(func=job_exec_finish)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-job-frobnicator.py b/src/cmd/flux-job-frobnicator.py
new file mode 100755
index 000000000000..b64c0ee4bd07
--- /dev/null
+++ b/src/cmd/flux-job-frobnicator.py
@@ -0,0 +1,130 @@
+###############################################################
+# Copyright 2022 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import argparse
+import json
+import logging
+import os
+import sys
+
+import flux
+from flux.job import Jobspec
+from flux.job.frobnicator import JobFrobnicator
+
+LOGGER = logging.getLogger("flux-job-frobnicator")
+
+
+class HelpAction(argparse.Action):
+    """
+    Copy of argparse._HelpAction, so that `--help` can be added
+    after initial parse_known_args() in JobFrobnicator constructor
+    """
+
+    def __init__(
+        self,
+        option_strings,
+        dest=argparse.SUPPRESS,
+        default=argparse.SUPPRESS,
+        xhelp=None,
+    ):
+        super(HelpAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=xhelp,
+        )
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.print_help()
+        parser.exit()
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog="flux-job-frobnicator",
+        formatter_class=flux.util.help_formatter(),
+        description="Modify Flux jobs from lines of JSON input on stdin",
+        add_help=False,
+    )
+    script_group = parser.add_argument_group("Program options")
+    script_group.add_argument(
+        "--jobspec-only",
+        action="store_true",
+        help="Expect only JSON encoded jobspec on stdin",
+    )
+    script_group.add_argument(
+        "--list-plugins",
+        action="store_true",
+        help="List available frobnicator plugins and exit",
+    )
+
+    frobnicator = JobFrobnicator(sys.argv[1:], parser=parser)
+
+    if frobnicator.args.list_plugins:
+        print("Available plugins:")
+        for name, plugin in frobnicator.plugins.items():
+            descr = ""
+            if plugin.__doc__:
+                descr = plugin.__doc__.partition("\n")[0]
+            print(f"{name:<20}  {descr}")
+        sys.exit(0)
+
+    parser.add_argument("-h", "--help", action=HelpAction)
+
+    try:
+        frobnicator.start()
+    except ValueError as exc:
+        LOGGER.critical(exc)
+        sys.exit(1)
+
+    exitcode = 0
+
+    # Ensure stdin is line buffered, with proper encoding
+    for line in os.fdopen(
+        sys.stdin.fileno(), "r", buffering=1, encoding="utf-8", errors="surrogateescape"
+    ):
+        if frobnicator.args.jobspec_only:
+            line = (
+                '{"jobspec":'
+                + line.rstrip()
+                + ',"userid":'
+                + str(os.getuid())
+                + ',"urgency":16,"flags":0}'
+            )
+        info = json.loads(line)
+
+        # Check for valid input
+        for key in ["jobspec", "userid", "urgency", "flags"]:
+            if key not in info:
+                LOGGER.critical("missing key %s in input", key)
+                sys.exit(1)
+        try:
+            jobspec = Jobspec(**info["jobspec"])
+        except Exception as exc:  # pylint: disable=broad-except
+            result = {"errnum": 1, "errstr": f"invalid jobspec: {exc}"}
+        else:
+            try:
+                frobnicator.frob(
+                    jobspec, info["userid"], info["urgency"], info["flags"]
+                )
+                result = {"errnum": 0, "data": jobspec.jobspec}
+            except Exception as exc:  # pylint: disable=broad-except
+                result = {"errnum": 1, "errstr": str(exc)}
+
+        print(json.dumps(result), flush=True)
+
+    sys.exit(exitcode)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/cmd/flux-job-validator.py b/src/cmd/flux-job-validator.py
new file mode 100755
index 000000000000..9e337f175053
--- /dev/null
+++ b/src/cmd/flux-job-validator.py
@@ -0,0 +1,119 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import argparse
+import json
+import logging
+import os
+import sys
+
+import flux
+from flux.job.validator import JobValidator
+
+LOGGER = logging.getLogger("flux-job-validator")
+
+
+class HelpAction(argparse.Action):
+    """
+    Copy of argparse._HelpAction, so that `--help` can be added
+    after initial parse_known_args() in JobValidator constructor
+    """
+
+    def __init__(
+        self,
+        option_strings,
+        dest=argparse.SUPPRESS,
+        default=argparse.SUPPRESS,
+        xhelp=None,
+    ):
+        super(HelpAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=xhelp,
+        )
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.print_help()
+        parser.exit()
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog="flux-job-validator",
+        formatter_class=flux.util.help_formatter(),
+        description="Validate Flux jobs from lines of JSON input on stdin",
+        add_help=False,
+    )
+    script_group = parser.add_argument_group("Program options")
+    script_group.add_argument(
+        "--jobspec-only",
+        action="store_true",
+        help="Expect only JSON encoded jobspec on stdin",
+    )
+    script_group.add_argument(
+        "--list-plugins",
+        action="store_true",
+        help="List available validator plugins and exit",
+    )
+
+    validator = JobValidator(sys.argv[1:], parser=parser)
+
+    if validator.args.list_plugins:
+        print("Available plugins:")
+        for name, plugin in validator.plugins.items():
+            descr = ""
+            if plugin.__doc__:
+                descr = plugin.__doc__.partition("\n")[0]
+            print(f"{name:<20}  {descr}")
+        sys.exit(0)
+
+    parser.add_argument("-h", "--help", action=HelpAction)
+
+    try:
+        validator.start()
+    except ValueError as exc:
+        LOGGER.critical(exc)
+        sys.exit(1)
+
+    exitcode = 0
+
+    # Ensure stdin is line buffered, with proper encoding
+    for line in os.fdopen(
+        sys.stdin.fileno(), "r", buffering=1, encoding="utf-8", errors="surrogateescape"
+    ):
+        if validator.args.jobspec_only:
+            jobspec = json.loads(line)
+            result = validator.validate(
+                {
+                    "jobspec": jobspec,
+                    "userid": os.getuid(),
+                    "flags": None,
+                    "urgency": 16,
+                }
+            )
+            #  In --jobspec-only mode, exit with nonzero exit code
+            #   if validation failed:
+            if result.errnum != 0:
+                exitcode = 1
+        else:
+            result = validator.validate(line)
+        print(result, flush=True)
+
+    validator.stop()
+
+    sys.exit(exitcode)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/cmd/flux-job.c b/src/cmd/flux-job.c
deleted file mode 100644
index f791d7a76ba7..000000000000
--- a/src/cmd/flux-job.c
+++ /dev/null
@@ -1,2642 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* "plumbing" commands (see git(1)) for Flux job management */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <unistd.h>
-#include <time.h>
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
-#include <assert.h>
-#include <jansson.h>
-#include <argz.h>
-#include <czmq.h>
-#include <flux/core.h>
-#include <flux/optparse.h>
-#if HAVE_FLUX_SECURITY
-#include <flux/security/sign.h>
-#endif
-#include "src/common/libutil/xzmalloc.h"
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/fluid.h"
-#include "src/common/libjob/job.h"
-#include "src/common/libutil/read_all.h"
-#include "src/common/libutil/monotime.h"
-#include "src/common/libidset/idset.h"
-#include "src/common/libeventlog/eventlog.h"
-#include "src/common/libioencode/ioencode.h"
-
-int cmd_list (optparse_t *p, int argc, char **argv);
-int cmd_list_inactive (optparse_t *p, int argc, char **argv);
-int cmd_status (optparse_t *p, int argc, char **argv);
-int cmd_list_ids (optparse_t *p, int argc, char **argv);
-int cmd_submit (optparse_t *p, int argc, char **argv);
-int cmd_attach (optparse_t *p, int argc, char **argv);
-int cmd_id (optparse_t *p, int argc, char **argv);
-int cmd_namespace (optparse_t *p, int argc, char **argv);
-int cmd_cancel (optparse_t *p, int argc, char **argv);
-int cmd_cancelall (optparse_t *p, int argc, char **argv);
-int cmd_raise (optparse_t *p, int argc, char **argv);
-int cmd_raiseall (optparse_t *p, int argc, char **argv);
-int cmd_kill (optparse_t *p, int argc, char **argv);
-int cmd_killall (optparse_t *p, int argc, char **argv);
-int cmd_priority (optparse_t *p, int argc, char **argv);
-int cmd_eventlog (optparse_t *p, int argc, char **argv);
-int cmd_wait_event (optparse_t *p, int argc, char **argv);
-int cmd_info (optparse_t *p, int argc, char **argv);
-int cmd_stats (optparse_t *p, int argc, char **argv);
-int cmd_wait (optparse_t *p, int argc, char **argv);
-
-int stdin_flags;
-
-static struct optparse_option global_opts[] =  {
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option list_opts[] =  {
-    { .name = "count", .key = 'c', .has_arg = 1, .arginfo = "N",
-      .usage = "Limit output to N jobs",
-    },
-    { .name = "states", .key = 's', .has_arg = 1, .arginfo = "STATES",
-      .flags = OPTPARSE_OPT_AUTOSPLIT,
-      .usage = "List jobs in specific states",
-    },
-    { .name = "user", .key = 'u', .has_arg = 1, .arginfo = "USER",
-      .usage = "Limit output to specific user. " \
-               "Specify \"all\" for all users.",
-    },
-    { .name = "all-user", .key = 'a', .has_arg = 0,
-      .usage = "List my jobs, regardless of state",
-    },
-    { .name = "all", .key = 'A', .has_arg = 0,
-      .usage = "List jobs for all users, regardless of state",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option list_inactive_opts[] =  {
-    { .name = "count", .key = 'c', .has_arg = 1, .arginfo = "N",
-      .usage = "Limit output to N jobs",
-    },
-    { .name = "since", .key = 's', .has_arg = 1, .arginfo = "T",
-      .usage = "Limit output to jobs that entered the inactive state since"
-               " timestamp T",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option cancelall_opts[] =  {
-    { .name = "user", .key = 'u', .has_arg = 1, .arginfo = "USER",
-      .usage = "Set target user or 'all' (instance owner only)",
-    },
-    { .name = "states", .key = 'S', .has_arg = 1, .arginfo = "STATES",
-      .flags = OPTPARSE_OPT_AUTOSPLIT,
-      .usage = "Set target job states (default=ACTIVE)",
-    },
-    { .name = "force", .key = 'f', .has_arg = 0,
-      .usage = "Confirm the command",
-    },
-    { .name = "quiet", .key = 'f', .has_arg = 0,
-      .usage = "Suppress output if no jobs match",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option raise_opts[] =  {
-    { .name = "severity", .key = 's', .has_arg = 1, .arginfo = "N",
-      .usage = "Set exception severity [0-7] (default=0)",
-    },
-    { .name = "type", .key = 't', .has_arg = 1, .arginfo = "TYPE",
-      .usage = "Set exception type (default=cancel)",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option raiseall_opts[] =  {
-    { .name = "severity", .key = 's', .has_arg = 1, .arginfo = "N",
-      .usage = "Set exception severity [0-7] (0 is fatal, default=7)",
-    },
-    { .name = "user", .key = 'u', .has_arg = 1, .arginfo = "USER",
-      .usage = "Set target user or 'all' (instance owner only)",
-    },
-    { .name = "states", .key = 'S', .has_arg = 1, .arginfo = "STATES",
-      .flags = OPTPARSE_OPT_AUTOSPLIT,
-      .usage = "Set target job states (default=ACTIVE)",
-    },
-    { .name = "force", .key = 'f', .has_arg = 0,
-      .usage = "Confirm the command",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option kill_opts[] = {
-    { .name = "signal", .key = 's', .has_arg = 1, .arginfo = "SIG",
-      .usage = "Send signal SIG (default SIGTERM)",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option killall_opts[] = {
-    { .name = "signal", .key = 's', .has_arg = 1, .arginfo = "SIG",
-      .usage = "Send signal SIG (default SIGTERM)",
-    },
-    { .name = "user", .key = 'u', .has_arg = 1, .arginfo = "USER",
-      .usage = "Set target user or 'all' (instance owner only)",
-    },
-    { .name = "force", .key = 'f', .has_arg = 0,
-      .usage = "Confirm the command",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option submit_opts[] =  {
-    { .name = "priority", .key = 'p', .has_arg = 1, .arginfo = "N",
-      .usage = "Set job priority (0-31, default=16)",
-    },
-    { .name = "flags", .key = 'f', .has_arg = 3,
-      .flags = OPTPARSE_OPT_AUTOSPLIT,
-      .usage = "Set submit comma-separated flags (e.g. debug, waitable)",
-    },
-#if HAVE_FLUX_SECURITY
-    { .name = "security-config", .key = 'c', .has_arg = 1, .arginfo = "pattern",
-      .usage = "Use non-default security config glob",
-    },
-    { .name = "sign-type", .key = 's', .has_arg = 1, .arginfo = "TYPE",
-      .usage = "Use non-default mechanism type to sign J",
-    },
-#endif
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option attach_opts[] =  {
-    { .name = "show-events", .key = 'E', .has_arg = 0,
-      .usage = "Show job events on stderr",
-    },
-    { .name = "show-exec", .key = 'X', .has_arg = 0,
-      .usage = "Show exec events on stderr",
-    },
-    { .name = "label-io", .key = 'l', .has_arg = 0,
-      .usage = "Label output by rank",
-    },
-    { .name = "verbose", .key = 'v', .has_arg = 0,
-      .usage = "Increase verbosity" },
-    { .name = "quiet", .key = 'q', .has_arg = 0,
-      .usage = "Suppress warnings written to stderr from flux-job",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option status_opts[] = {
-    { .name = "verbose", .key = 'v', .has_arg = 0,
-      .usage = "Increase verbosity"
-    },
-    { .name = "exception-exit-code", .key = 'e', .has_arg = 1,
-      .group = 1,
-      .arginfo = "N",
-      .usage = "Set the default exit code for any jobs that terminate"
-               " solely due to an exception (e.g. canceled jobs or"
-               " jobs rejected by the scheduler) to N [default=1]"
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option id_opts[] =  {
-    { .name = "from", .key = 'f', .has_arg = 1,
-      .arginfo = "dec|kvs|hex|words",
-      .usage = "Convert jobid from specified form",
-    },
-    { .name = "to", .key = 't', .has_arg = 1,
-      .arginfo = "dec|kvs|hex|words",
-      .usage = "Convert jobid to specified form",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option eventlog_opts[] =  {
-    { .name = "format", .key = 'f', .has_arg = 1, .arginfo = "FORMAT",
-      .usage = "Specify output format: text, json",
-    },
-    { .name = "time-format", .key = 'T', .has_arg = 1, .arginfo = "FORMAT",
-      .usage = "Specify time format: raw, iso, offset",
-    },
-    { .name = "path", .key = 'p', .has_arg = 1, .arginfo = "PATH",
-      .usage = "Specify alternate eventlog path suffix "
-               "(e.g. \"guest.exec.eventlog\")",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option wait_event_opts[] =  {
-    { .name = "format", .key = 'f', .has_arg = 1, .arginfo = "FORMAT",
-      .usage = "Specify output format: text, json",
-    },
-    { .name = "time-format", .key = 'T', .has_arg = 1, .arginfo = "FORMAT",
-      .usage = "Specify time format: raw, iso, offset",
-    },
-    { .name = "timeout", .key = 't', .has_arg = 1, .arginfo = "DURATION",
-      .usage = "timeout after DURATION",
-    },
-    { .name = "match-context", .key = 'm', .has_arg = 1, .arginfo = "KEY=VAL",
-      .usage = "match key=val in context",
-    },
-    { .name = "quiet", .key = 'q', .has_arg = 0,
-      .usage = "Do not output matched event",
-    },
-    { .name = "verbose", .key = 'v', .has_arg = 0,
-      .usage = "Output all events before matched event",
-    },
-    { .name = "path", .key = 'p', .has_arg = 1, .arginfo = "PATH",
-      .usage = "Specify alternate eventlog path suffix "
-               "(e.g. \"guest.exec.eventlog\")",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option wait_opts[] =  {
-    { .name = "all", .key = 'a', .has_arg = 0,
-      .usage = "Wait for all (waitable) jobs",
-    },
-    { .name = "verbose", .key = 'v', .has_arg = 0,
-      .usage = "Emit a line of output for all jobs, not just failing ones",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_subcommand subcommands[] = {
-    { "list",
-      "[OPTIONS]",
-      "List jobs",
-      cmd_list,
-      0,
-      list_opts
-    },
-    { "list-inactive",
-      "[OPTIONS]",
-      "List Inactive jobs",
-      cmd_list_inactive,
-      0,
-      list_inactive_opts
-    },
-    { "list-ids",
-      "[OPTIONS] ID [ID ...]",
-      "List job(s) by id",
-      cmd_list_ids,
-      0,
-      NULL,
-    },
-    { "priority",
-      "[OPTIONS] id priority",
-      "Set job priority",
-      cmd_priority,
-      0,
-      NULL,
-    },
-    { "cancel",
-      "[OPTIONS] id [message ...]",
-      "Cancel a job",
-      cmd_cancel,
-      0,
-      NULL,
-    },
-    { "cancelall",
-      "[OPTIONS] [message ...]",
-      "Cancel multiple jobs",
-      cmd_cancelall,
-      0,
-      cancelall_opts,
-    },
-    { "raise",
-      "[OPTIONS] id [message ...]",
-      "Raise exception for job",
-      cmd_raise,
-      0,
-      raise_opts,
-    },
-    { "raiseall",
-      "OPTIONS type [message ...]",
-      "Raise an exception on multiple jobs.",
-      cmd_raiseall,
-      0,
-      raiseall_opts,
-    },
-    { "kill",
-      "[OPTIONS] id",
-      "Send signal to running job",
-      cmd_kill,
-      0,
-      kill_opts,
-    },
-    { "killall",
-      "[OPTIONS]",
-      "Send signal to multiple running jobs",
-      cmd_killall,
-      0,
-      killall_opts,
-    },
-    { "attach",
-      "[OPTIONS] id",
-      "Interactively attach to job",
-      cmd_attach,
-      0,
-      attach_opts,
-    },
-    { "status",
-      "id [id...]",
-      "Wait for job(s) to complete and exit with largest exit code",
-      cmd_status,
-      0,
-      status_opts,
-    },
-    { "submit",
-      "[OPTIONS] [jobspec]",
-      "Run job",
-      cmd_submit,
-      0,
-      submit_opts
-    },
-    { "id",
-      "[OPTIONS] [id ...]",
-      "Convert jobid(s) to another form",
-      cmd_id,
-      0,
-      id_opts
-    },
-    { "eventlog",
-      "[-f text|json] [-T raw|iso|offset] [-p path] id",
-      "Display eventlog for a job",
-      cmd_eventlog,
-      0,
-      eventlog_opts
-    },
-    { "wait-event",
-      "[-f text|json] [-T raw|iso|offset] [-t seconds] [-m key=val] "
-      "[-p path] id event",
-      "Wait for an event ",
-      cmd_wait_event,
-      0,
-      wait_event_opts
-    },
-    { "info",
-      "id key ...",
-      "Display info for a job",
-      cmd_info,
-      0,
-      NULL
-    },
-    { "stats",
-      NULL,
-      "Get current job stats",
-      cmd_stats,
-      0,
-      NULL
-    },
-    { "namespace",
-      "[id ...]",
-      "Convert job ids to job guest kvs namespace names",
-      cmd_namespace,
-      0,
-      NULL
-    },
-    { "wait",
-      "[--all] [id]",
-      "Wait for job(s) to complete.",
-      cmd_wait,
-      0,
-      wait_opts,
-    },
-    OPTPARSE_SUBCMD_END
-};
-
-int usage (optparse_t *p, struct optparse_option *o, const char *optarg)
-{
-    struct optparse_subcommand *s;
-    optparse_print_usage (p);
-    fprintf (stderr, "\n");
-    fprintf (stderr, "Common commands from flux-job:\n");
-    s = subcommands;
-    while (s->name) {
-        fprintf (stderr, "   %-15s %s\n", s->name, s->doc);
-        s++;
-    }
-    exit (1);
-}
-
-int main (int argc, char *argv[])
-{
-    char *cmdusage = "[OPTIONS] COMMAND ARGS";
-    optparse_t *p;
-    int optindex;
-    int exitval;
-
-    log_init ("flux-job");
-
-    p = optparse_create ("flux-job");
-
-    if (optparse_add_option_table (p, global_opts) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_add_option_table() failed");
-
-    /* Override help option for our own */
-    if (optparse_set (p, OPTPARSE_USAGE, cmdusage) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_set (USAGE)");
-
-    /* Override --help callback in favor of our own above */
-    if (optparse_set (p, OPTPARSE_OPTION_CB, "help", usage) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_set() failed");
-
-    /* Don't print internal subcommands, we do it ourselves */
-    if (optparse_set (p, OPTPARSE_PRINT_SUBCMDS, 0) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_set (PRINT_SUBCMDS)");
-
-    if (optparse_reg_subcommands (p, subcommands) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_reg_subcommands");
-
-    if ((optindex = optparse_parse_args (p, argc, argv)) < 0)
-        exit (1);
-
-    if ((argc - optindex == 0)
-        || !optparse_get_subcommand (p, argv[optindex])) {
-        usage (p, NULL, NULL);
-        exit (1);
-    }
-
-    if ((exitval = optparse_run_subcommand (p, argc, argv)) < 0)
-        exit (1);
-
-    optparse_destroy (p);
-    log_fini ();
-    return (exitval);
-}
-
-/* Parse a free argument 's', expected to be a 64-bit unsigned.
- * On error, exit complaining about parsing 'name'.
- */
-static unsigned long long parse_arg_unsigned (const char *s, const char *name)
-{
-    unsigned long long i;
-    char *endptr;
-
-    errno = 0;
-    i = strtoull (s, &endptr, 10);
-    if (errno != 0 || *endptr != '\0')
-        log_msg_exit ("error parsing %s: \"%s\"", name, s);
-    return i;
-}
-
-/* Parse free arguments into a space-delimited message.
- * On error, exit complaning about parsing 'name'.
- * Caller must free the resulting string
- */
-static char *parse_arg_message (char **argv, const char *name)
-{
-    char *argz = NULL;
-    size_t argz_len = 0;
-    error_t e;
-
-    if ((e = argz_create (argv, &argz, &argz_len)) != 0)
-        log_errn_exit (e, "error parsing %s", name);
-    argz_stringify (argz, argz_len, ' ');
-    return argz;
-}
-
-/* Parse an OPTPARSE_OPT_AUTOSPLIT list of state names, returning a
- * mask of states.  Exit with error if unknown state encountered.
- */
-int parse_arg_states (optparse_t *p, const char *optname)
-{
-    int state_mask = 0;
-    const char *arg;
-
-    assert (optparse_hasopt (p, optname) == true);
-
-    optparse_getopt_iterator_reset (p, optname);
-    while ((arg = optparse_getopt_next (p, optname))) {
-        flux_job_state_t state;
-
-        if (flux_job_strtostate (arg, &state) == 0)
-            state_mask |= state;
-        else if (!strcasecmp (arg, "pending"))
-            state_mask |= FLUX_JOB_PENDING;
-        else if (!strcasecmp (arg, "running"))
-            state_mask |= FLUX_JOB_RUNNING;
-        else if (!strcasecmp (arg, "active"))
-            state_mask |= FLUX_JOB_ACTIVE;
-        else
-            log_msg_exit ("error parsing --%s: %s is unknown", optname, arg);
-    }
-    if (state_mask == 0)
-        log_msg_exit ("no states specified");
-    return state_mask;
-}
-
-/* Parse user argument, which may be a username, a user id, or "all".
- * Print an error and exit if there is a problem.
- * Return numeric userid (all -> FLUX_USERID_UNKNOWN).
- */
-uint32_t parse_arg_userid (optparse_t *p, const char *optname)
-{
-    uint32_t userid;
-    const char *s = optparse_get_str (p, optname, NULL);
-    struct passwd *pw;
-    char *endptr;
-
-    assert (s != NULL);
-    if (!strcmp (s, "all"))
-        return FLUX_USERID_UNKNOWN;
-    if ((pw = getpwnam (s)))
-        return pw->pw_uid;
-    errno = 0;
-    userid = strtoul (s, &endptr, 10);
-    if (errno != 0 || *endptr != '\0' || !isdigit (*s))
-        log_msg_exit ("unknown user %s", s);
-    return userid;
-}
-
-int cmd_priority (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-    flux_t *h;
-    flux_future_t *f;
-    int priority;
-    flux_jobid_t id;
-
-    if (optindex != argc - 2) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    id = parse_arg_unsigned (argv[optindex++], "jobid");
-    priority = parse_arg_unsigned (argv[optindex++], "priority");
-
-    if (!(f = flux_job_set_priority (h, id, priority)))
-        log_err_exit ("flux_job_set_priority");
-    if (flux_rpc_get (f, NULL) < 0)
-        log_msg_exit ("%llu: %s", (unsigned long long)id,
-                      future_strerror (f, errno));
-    flux_future_destroy (f);
-    flux_close (h);
-    return 0;
-}
-
-int cmd_raise (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-    int severity = optparse_get_int (p, "severity", 0);
-    const char *type = optparse_get_str (p, "type", "cancel");
-    flux_t *h;
-    flux_jobid_t id;
-    char *note = NULL;
-    flux_future_t *f;
-
-    if (argc - optindex < 1) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-
-    id = parse_arg_unsigned (argv[optindex++], "jobid");
-    if (optindex < argc)
-        note = parse_arg_message (argv + optindex, "message");
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (!(f = flux_job_raise (h, id, type, severity, note)))
-        log_err_exit ("flux_job_raise");
-    if (flux_rpc_get (f, NULL) < 0)
-        log_msg_exit ("%llu: %s", (unsigned long long)id,
-                      future_strerror (f, errno));
-    flux_future_destroy (f);
-    flux_close (h);
-    free (note);
-    return 0;
-}
-
-static int raiseall (flux_t *h,
-                     int dry_run,
-                     uint32_t userid,
-                     int state_mask,
-                     int severity,
-                     const char *type,
-                     const char *note,
-                     int *errorsp)
-{
-    flux_future_t *f;
-    int count;
-    int errors;
-
-    if (!(f = flux_rpc_pack (h,
-                             "job-manager.raiseall",
-                             FLUX_NODEID_ANY,
-                             0,
-                             "{s:b s:i s:i s:i s:s s:s}",
-                             "dry_run",
-                             dry_run,
-                             "userid",
-                             userid,
-                             "states",
-                             state_mask,
-                             "severity",
-                             severity,
-                             "type",
-                             type,
-                             "note",
-                             note ? note : "")))
-        log_err_exit ("error sending raiseall request");
-    if (flux_rpc_get_unpack (f,
-                             "{s:i s:i}",
-                             "count",
-                             &count,
-                             "errors",
-                             &errors) < 0)
-        log_msg_exit ("raiseall: %s", future_strerror (f, errno));
-    flux_future_destroy (f);
-    if (errorsp)
-        *errorsp = errors;
-    return count;
-}
-
-int cmd_raiseall (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-    int severity = optparse_get_int (p, "severity", 7);
-    const char *type;
-    uint32_t userid;
-    int state_mask;
-    flux_t *h;
-    char *note = NULL;
-    int count;
-    int errors;
-    int dry_run = 1;
-
-    if (optindex == argc) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    type = argv[optindex++];
-    if (optindex < argc)
-        note = parse_arg_message (argv + optindex, "message");
-    if (optparse_hasopt (p, "states")) {
-        state_mask = parse_arg_states (p, "states");
-        if ((state_mask & FLUX_JOB_INACTIVE))
-            log_msg_exit ("Exceptions cannot be raised on inactive jobs");
-    }
-    else
-        state_mask = FLUX_JOB_ACTIVE;
-    if (optparse_hasopt (p, "user"))
-        userid = parse_arg_userid (p, "user");
-    else
-        userid = geteuid ();
-    if (optparse_hasopt (p, "force"))
-        dry_run = 0;
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    count = raiseall (h,
-                      dry_run,
-                      userid,
-                      state_mask,
-                      severity,
-                      type,
-                      note,
-                      &errors);
-    if (count > 0 && dry_run)
-        log_msg ("Command matched %d jobs (-f to confirm)", count);
-    else if (count > 0 && !dry_run)
-        log_msg ("Raised exception on %d jobs (%d errors)", count, errors);
-    else
-        log_msg ("Command matched 0 jobs");
-
-    flux_close (h);
-    free (note);
-    return 0;
-}
-
-/*
- * List generated by:
- *
- * $ kill -l | sed 's/[0-9]*)//g' | xargs -n1 printf '    "%s",\n'
- *
- * (ignoring SIGRT*)
- */
-static const char *sigmap[] = {
-    NULL,      /* 1 origin */
-    "SIGHUP",
-    "SIGINT",
-    "SIGQUIT",
-    "SIGILL",
-    "SIGTRAP",
-    "SIGABRT",
-    "SIGBUS",
-    "SIGFPE",
-    "SIGKILL",
-    "SIGUSR1",
-    "SIGSEGV",
-    "SIGUSR2",
-    "SIGPIPE",
-    "SIGALRM",
-    "SIGTERM",
-    "SIGSTKFLT",
-    "SIGCHLD",
-    "SIGCONT",
-    "SIGSTOP",
-    "SIGTSTP",
-    "SIGTTIN",
-    "SIGTTOU",
-    "SIGURG",
-    "SIGXCPU",
-    "SIGXFSZ",
-    "SIGVTALRM",
-    "SIGPROF",
-    "SIGWINCH",
-    "SIGIO",
-    "SIGPWR",
-    "SIGSYS",
-    NULL,
-};
-
-static bool isnumber (const char *s, int *result)
-{
-    char *endptr;
-    long int l;
-
-    errno = 0;
-    l = strtol (s, &endptr, 10);
-    if (errno
-        || *endptr != '\0'
-        || l <= 0) {
-        return false;
-    }
-    *result = (int) l;
-    return true;
-}
-
-static int str2signum (const char *sigstr)
-{
-    int i;
-    if (isnumber (sigstr, &i)) {
-        if (i <= 0)
-            return -1;
-        return i;
-    }
-    i = 1;
-    while (sigmap[i] != NULL) {
-        if (strcmp (sigstr, sigmap[i]) == 0 ||
-            strcmp (sigstr, sigmap[i]+3) == 0)
-            return i;
-        i++;
-    }
-    return -1;
-}
-
-int cmd_kill (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    flux_future_t *f;
-    int optindex = optparse_option_index (p);
-    flux_jobid_t id;
-    const char *s;
-    int signum;
-
-    if (argc - optindex < 1) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-
-    id = parse_arg_unsigned (argv[optindex++], "jobid");
-
-    s = optparse_get_str (p, "signal", "SIGTERM");
-    if ((signum = str2signum (s))< 0)
-        log_msg_exit ("kill: Invalid signal %s", s);
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (!(f = flux_job_kill (h, id, signum)))
-        log_err_exit ("flux_job_kill");
-    if (flux_rpc_get (f, NULL) < 0)
-        log_msg_exit ("kill %ju: %s",
-                      (uintmax_t) id,
-                      future_strerror (f, errno));
-    flux_future_destroy (f);
-    flux_close (h);
-    return 0;
-}
-
-int cmd_killall (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    flux_future_t *f;
-    int optindex = optparse_option_index (p);
-    const char *s;
-    int signum;
-    uint32_t userid;
-    int count;
-    int dry_run = 1;
-    int errors;
-
-    if (argc - optindex > 0) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    s = optparse_get_str (p, "signal", "SIGTERM");
-    if ((signum = str2signum (s))< 0)
-        log_msg_exit ("killall: Invalid signal %s", s);
-    if (optparse_hasopt (p, "user"))
-        userid = parse_arg_userid (p, "user");
-    else
-        userid = geteuid ();
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (optparse_hasopt (p, "force"))
-        dry_run = 0;
-    if (!(f = flux_rpc_pack (h,
-                             "job-manager.killall",
-                             FLUX_NODEID_ANY,
-                             0,
-                             "{s:b s:i s:i}",
-                             "dry_run",
-                             dry_run,
-                             "userid",
-                             userid,
-                             "signum",
-                             signum)))
-        log_err_exit ("error sending killall request");
-    if (flux_rpc_get_unpack (f,
-                             "{s:i s:i}",
-                             "count",
-                             &count,
-                             "errors",
-                             &errors) < 0)
-        log_msg_exit ("killall: %s", future_strerror (f, errno));
-    flux_future_destroy (f);
-    if (count > 0 && dry_run)
-        log_msg ("Command matched %d jobs (-f to confirm)", count);
-    else if (count > 0 && !dry_run)
-        log_msg ("%s %d jobs (%d errors)", strsignal (signum), count, errors);
-    else
-        log_msg ("Command matched 0 jobs");
-    flux_close (h);
-    return 0;
-}
-
-int cmd_cancel (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-    flux_t *h;
-    flux_jobid_t id;
-    char *note = NULL;
-    flux_future_t *f;
-
-    if (argc - optindex < 1) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-
-    id = parse_arg_unsigned (argv[optindex++], "jobid");
-    if (optindex < argc)
-        note = parse_arg_message (argv + optindex, "message");
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (!(f = flux_job_cancel (h, id, note)))
-        log_err_exit ("flux_job_cancel");
-    if (flux_rpc_get (f, NULL) < 0)
-        log_msg_exit ("%llu: %s", (unsigned long long)id,
-                      future_strerror (f, errno));
-    flux_future_destroy (f);
-    flux_close (h);
-    free (note);
-    return 0;
-}
-
-int cmd_cancelall (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-    uint32_t userid;
-    int state_mask;
-    flux_t *h;
-    char *note = NULL;
-    int dry_run = 1;
-    int count;
-    int errors;
-
-    if (optindex < argc)
-        note = parse_arg_message (argv + optindex, "message");
-    if (optparse_hasopt (p, "states")) {
-        state_mask = parse_arg_states (p, "states");
-        if ((state_mask & FLUX_JOB_INACTIVE))
-            log_msg_exit ("Inactive jobs cannot be cancelled");
-    }
-    else
-        state_mask = FLUX_JOB_ACTIVE;
-    if (optparse_hasopt (p, "user"))
-        userid = parse_arg_userid (p, "user");
-    else
-        userid = geteuid ();
-    if (optparse_hasopt (p, "force"))
-        dry_run = 0;
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    count = raiseall (h,
-                      dry_run,
-                      userid,
-                      state_mask,
-                      0,
-                      "cancel",
-                      note,
-                      &errors);
-    if (count > 0 && dry_run)
-        log_msg ("Command matched %d jobs (-f to confirm)", count);
-    else if (count > 0 && !dry_run)
-        log_msg ("Canceled %d jobs (%d errors)", count, errors);
-    else if (!optparse_hasopt (p, "quiet"))
-        log_msg ("Command matched 0 jobs");
-    flux_close (h);
-    free (note);
-    return 0;
-}
-
-int cmd_list (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-    int max_entries = optparse_get_int (p, "count", 0);
-    char *attrs = "[\"userid\",\"priority\",\"t_submit\",\"state\"," \
-        "\"name\",\"ntasks\",\"nnodes\",\"ranks\",\"t_depend\",\"t_sched\"," \
-        "\"t_run\",\"t_cleanup\",\"t_inactive\"]";
-    flux_t *h;
-    flux_future_t *f;
-    json_t *jobs;
-    size_t index;
-    json_t *value;
-    uint32_t userid;
-    int states = 0;
-
-    if (optindex != argc) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (optparse_hasopt (p, "all-user") || optparse_hasopt (p, "all"))
-        states = FLUX_JOB_ACTIVE | FLUX_JOB_INACTIVE;
-    else if (optparse_hasopt (p, "states"))
-        states = parse_arg_states (p, "states");
-    else
-        states = FLUX_JOB_PENDING | FLUX_JOB_RUNNING;
-
-    if (optparse_hasopt (p, "all"))
-        userid = FLUX_USERID_UNKNOWN;
-    else if (optparse_hasopt (p, "user"))
-        userid = parse_arg_userid (p, "user");
-    else
-        userid = geteuid ();
-
-    if (!(f = flux_job_list (h, max_entries, attrs, userid, states)))
-        log_err_exit ("flux_job_list");
-    if (flux_rpc_get_unpack (f, "{s:o}", "jobs", &jobs) < 0)
-        log_err_exit ("flux_job_list");
-    json_array_foreach (jobs, index, value) {
-        char *str;
-        str = json_dumps (value, 0);
-        if (!str)
-            log_msg_exit ("error parsing list response");
-        printf ("%s\n", str);
-        free (str);
-    }
-    flux_future_destroy (f);
-    flux_close (h);
-
-   return (0);
-}
-
-int cmd_list_inactive (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-    int max_entries = optparse_get_int (p, "count", 0);
-    double since = optparse_get_double (p, "since", 0.);
-    char *attrs = "[\"userid\",\"priority\",\"t_submit\",\"state\"," \
-        "\"name\",\"ntasks\",\"nnodes\",\"ranks\",\"t_depend\",\"t_sched\"," \
-        "\"t_run\",\"t_cleanup\",\"t_inactive\"]";
-    flux_t *h;
-    flux_future_t *f;
-    json_t *jobs;
-    size_t index;
-    json_t *value;
-
-    if (optindex != argc) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (!(f = flux_job_list_inactive (h, max_entries, since, attrs)))
-        log_err_exit ("flux_job_list_inactive");
-    if (flux_rpc_get_unpack (f, "{s:o}", "jobs", &jobs) < 0)
-        log_err_exit ("flux_job_list_inactive");
-    json_array_foreach (jobs, index, value) {
-        char *str;
-        str = json_dumps (value, 0);
-        if (!str)
-            log_msg_exit ("error parsing list response");
-        printf ("%s\n", str);
-        free (str);
-    }
-    flux_future_destroy (f);
-    flux_close (h);
-
-    return (0);
-}
-
-void list_id_continuation (flux_future_t *f, void *arg)
-{
-    json_t *job;
-    char *str;
-    if (flux_rpc_get_unpack (f, "{s:o}", "job", &job) < 0)
-        log_err_exit ("flux_job_list_id");
-    str = json_dumps (job, 0);
-    if (!str)
-        log_msg_exit ("error parsing list-id response");
-    printf ("%s\n", str);
-    free (str);
-    flux_future_destroy (f);
-}
-
-int cmd_list_ids (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-    char *attrs = "[\"userid\",\"priority\",\"t_submit\",\"state\"," \
-        "\"name\",\"ntasks\",\"nnodes\",\"ranks\",\"t_depend\",\"t_sched\"," \
-        "\"t_run\",\"t_cleanup\",\"t_inactive\"]";
-    flux_t *h;
-    int i, ids_len;
-
-    if ((argc - optindex) < 1) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    ids_len = argc - optindex;
-    for (i = 0; i < ids_len; i++) {
-        flux_jobid_t id = parse_arg_unsigned (argv[optindex + i], "id");
-        flux_future_t *f;
-        if (!(f = flux_job_list_id (h, id, attrs)))
-            log_err_exit ("flux_job_list_id");
-        if (flux_future_then (f, -1, list_id_continuation, NULL) < 0)
-            log_err_exit ("flux_future_then");
-    }
-
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
-        log_err_exit ("flux_reactor_run");
-
-    flux_close (h);
-
-    return (0);
-}
-
-/* Read entire file 'name' ("-" for stdin).  Exit program on error.
- */
-size_t read_jobspec (const char *name, void **bufp)
-{
-    int fd;
-    ssize_t size;
-    void *buf;
-
-    if (!strcmp (name, "-"))
-        fd = STDIN_FILENO;
-    else {
-        if ((fd = open (name, O_RDONLY)) < 0)
-            log_err_exit ("%s", name);
-    }
-    if ((size = read_all (fd, &buf)) < 0)
-        log_err_exit ("%s", name);
-    if (fd != STDIN_FILENO)
-        (void)close (fd);
-    *bufp = buf;
-    return size;
-}
-
-int cmd_submit (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-#if HAVE_FLUX_SECURITY
-    flux_security_t *sec = NULL;
-    const char *sec_config;
-    const char *sign_type;
-#endif
-    int flags = 0;
-    void *jobspec;
-    int jobspecsz;
-    const char *J = NULL;
-    int priority;
-    int optindex = optparse_option_index (p);
-    flux_future_t *f;
-    flux_jobid_t id;
-    const char *input = "-";
-
-    if (optindex != argc - 1 && optindex != argc) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (optindex < argc)
-        input = argv[optindex++];
-    if (optparse_hasopt (p, "flags")) {
-        const char *name;
-        while ((name = optparse_getopt_next (p, "flags"))) {
-            if (!strcmp (name, "debug"))
-                flags |= FLUX_JOB_DEBUG;
-            else if (!strcmp (name, "waitable"))
-                flags |= FLUX_JOB_WAITABLE;
-            else
-                log_msg_exit ("unknown flag: %s", name);
-        }
-    }
-#if HAVE_FLUX_SECURITY
-    /* If any non-default security options are specified, create security
-     * context so jobspec can be pre-signed before submission.
-     */
-    if (optparse_hasopt (p, "security-config")
-                            || optparse_hasopt (p, "sign-type")) {
-        sec_config = optparse_get_str (p, "security-config", NULL);
-        if (!(sec = flux_security_create (0)))
-            log_err_exit ("security");
-        if (flux_security_configure (sec, sec_config) < 0)
-            log_err_exit ("security config %s", flux_security_last_error (sec));
-        sign_type = optparse_get_str (p, "sign-type", NULL);
-    }
-#endif
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    jobspecsz = read_jobspec (input, &jobspec);
-    assert (((char *)jobspec)[jobspecsz] == '\0');
-    if (jobspecsz == 0)
-        log_msg_exit ("required jobspec is empty");
-    priority = optparse_get_int (p, "priority", FLUX_JOB_PRIORITY_DEFAULT);
-
-#if HAVE_FLUX_SECURITY
-    if (sec) {
-        if (!(J = flux_sign_wrap (sec, jobspec, jobspecsz, sign_type, 0)))
-            log_err_exit ("flux_sign_wrap: %s", flux_security_last_error (sec));
-        flags |= FLUX_JOB_PRE_SIGNED;
-    }
-#endif
-    if (!(f = flux_job_submit (h, J ? J : jobspec, priority, flags)))
-        log_err_exit ("flux_job_submit");
-    if (flux_job_submit_get_id (f, &id) < 0) {
-        log_msg_exit ("%s", future_strerror (f, errno));
-    }
-    printf ("%llu\n", (unsigned long long)id);
-    flux_future_destroy (f);
-#if HAVE_FLUX_SECURITY
-    flux_security_destroy (sec); // invalidates J
-#endif
-    flux_close (h);
-    free (jobspec);
-    return 0;
-}
-
-struct attach_ctx {
-    flux_t *h;
-    int exit_code;
-    flux_jobid_t id;
-    flux_future_t *eventlog_f;
-    flux_future_t *exec_eventlog_f;
-    flux_future_t *output_f;
-    flux_watcher_t *sigint_w;
-    flux_watcher_t *sigtstp_w;
-    struct timespec t_sigint;
-    flux_watcher_t *stdin_w;
-    zlist_t *stdin_rpcs;
-    bool stdin_data_sent;
-    optparse_t *p;
-    bool output_header_parsed;
-    int leader_rank;
-    char *service;
-    double timestamp_zero;
-    int eventlog_watch_count;
-};
-
-void attach_completed_check (struct attach_ctx *ctx)
-{
-    /* stop all non-eventlog watchers and destroy all lingering
-     * futures so we can exit the reactor */
-    if (!ctx->eventlog_watch_count) {
-        if (ctx->stdin_rpcs) {
-            flux_future_t *f = zlist_pop (ctx->stdin_rpcs);
-            while (f) {
-                flux_future_destroy (f);
-                zlist_remove (ctx->stdin_rpcs, f);
-                f = zlist_pop (ctx->stdin_rpcs);
-            }
-        }
-        flux_watcher_stop (ctx->sigint_w);
-        flux_watcher_stop (ctx->sigtstp_w);
-        flux_watcher_stop (ctx->stdin_w);
-    }
-}
-
-/* Print eventlog entry to 'fp'.
- * Prefix and context may be NULL.
- */
-void print_eventlog_entry (FILE *fp,
-                           const char *prefix,
-                           double timestamp,
-                           const char *name,
-                           json_t *context)
-{
-    char *context_s = NULL;
-
-    if (context) {
-        if (!(context_s = json_dumps (context, JSON_COMPACT)))
-            log_err_exit ("%s: error re-encoding context", __func__);
-    }
-    fprintf (stderr, "%.3fs: %s%s%s%s%s\n",
-             timestamp,
-             prefix ? prefix : "",
-             prefix ? "." : "",
-             name,
-             context_s ? " " : "",
-             context_s ? context_s : "");
-    free (context_s);
-}
-
-static void handle_output_data (struct attach_ctx *ctx, json_t *context)
-{
-    FILE *fp;
-    const char *stream;
-    const char *rank;
-    char *data;
-    int len;
-    if (!ctx->output_header_parsed)
-        log_msg_exit ("stream data read before header");
-    if (iodecode (context, &stream, &rank, &data, &len, NULL) < 0)
-        log_msg_exit ("malformed event context");
-    if (!strcmp (stream, "stdout"))
-        fp = stdout;
-    else
-        fp = stderr;
-    if (len > 0) {
-        if (optparse_hasopt (ctx->p, "label-io"))
-            fprintf (fp, "%s: ", rank);
-        fwrite (data, len, 1, fp);
-    }
-    free (data);
-}
-
-static void handle_output_redirect (struct attach_ctx *ctx, json_t *context)
-{
-    const char *stream = NULL;
-    const char *rank = NULL;
-    const char *path = NULL;
-    if (!ctx->output_header_parsed)
-        log_msg_exit ("stream redirect read before header");
-    if (json_unpack (context, "{ s:s s:s s?:s }",
-                              "stream", &stream,
-                              "rank", &rank,
-                              "path", &path) < 0)
-        log_msg_exit ("malformed redirect context");
-    if (!optparse_hasopt (ctx->p, "quiet"))
-        fprintf (stderr, "%s: %s redirected%s%s\n",
-                         rank,
-                         stream,
-                         path ? " to " : "",
-                         path ? path : "");
-}
-
-/*  Level prefix strings. Nominally, output log event 'level' integers
- *   are Internet RFC 5424 severity levels. In the context of flux-shell,
- *   the first 3 levels are equivalently "fatal" errors.
- */
-static const char *levelstr[] = {
-    "FATAL", "FATAL", "FATAL", "ERROR", " WARN", NULL, "DEBUG", "TRACE"
-};
-
-static void handle_output_log (struct attach_ctx *ctx,
-                               double ts,
-                               json_t *context)
-{
-    const char *msg = NULL;
-    const char *file = NULL;
-    const char *component = NULL;
-    int rank = -1;
-    int line = -1;
-    int level = -1;
-    json_error_t err;
-
-    if (json_unpack_ex (context, &err, 0,
-                        "{ s?i s:i s:s s?:s s?:s s?:i }",
-                        "rank", &rank,
-                        "level", &level,
-                        "message", &msg,
-                        "component", &component,
-                        "file", &file,
-                        "line", &line) < 0) {
-        log_err ("invalid log event in guest.output: %s", err.text);
-        return;
-    }
-    if (!optparse_hasopt (ctx->p, "quiet")) {
-        const char *label = levelstr [level];
-        fprintf (stderr, "%.3fs: flux-shell", ts - ctx->timestamp_zero);
-        if (rank >= 0)
-            fprintf (stderr, "[%d]", rank);
-        if (label)
-            fprintf (stderr, ": %s", label);
-        if (component)
-            fprintf (stderr, ": %s", component);
-        if (optparse_hasopt (ctx->p, "verbose") && file) {
-            fprintf (stderr, ": %s", file);
-            if (line > 0)
-                fprintf (stderr, ":%d", line);
-        }
-        fprintf (stderr, ": %s\n", msg);
-    }
-}
-
-/* Handle an event in the guest.output eventlog.
- * This is a stream of responses, one response per event, terminated with
- * an ENODATA error response (or another error if something went wrong).
- * The first eventlog entry is a header; remaining entries are data,
- * redirect, or log messages.  Print each data entry to stdout/stderr,
- * with task/rank prefix if --label-io was specified.  For each redirect entry, print
- * information on paths to redirected locations if --quiet is not
- * speciifed.
- */
-void attach_output_continuation (flux_future_t *f, void *arg)
-{
-    struct attach_ctx *ctx = arg;
-    const char *entry;
-    json_t *o;
-    const char *name;
-    double ts;
-    json_t *context;
-
-    if (flux_job_event_watch_get (f, &entry) < 0) {
-        if (errno == ENODATA)
-            goto done;
-        if (errno == ENOENT) {
-            log_msg ("No job output found");
-            goto done;
-        }
-        log_msg_exit ("flux_job_event_watch_get: %s",
-                      future_strerror (f, errno));
-    }
-    if (!(o = eventlog_entry_decode (entry)))
-        log_err_exit ("eventlog_entry_decode");
-    if (eventlog_entry_parse (o, &ts, &name, &context) < 0)
-        log_err_exit ("eventlog_entry_parse");
-
-    if (!strcmp (name, "header")) {
-        /* Future: per-stream encoding */
-        ctx->output_header_parsed = true;
-    }
-    else if (!strcmp (name, "data")) {
-        handle_output_data (ctx, context);
-    }
-    else if (!strcmp (name, "redirect")) {
-        handle_output_redirect (ctx, context);
-    }
-    else if (!strcmp (name, "log")) {
-        handle_output_log (ctx, ts, context);
-    }
-
-    json_decref (o);
-    flux_future_reset (f);
-    return;
-done:
-    flux_future_destroy (f);
-    ctx->output_f = NULL;
-    ctx->eventlog_watch_count--;
-    attach_completed_check (ctx);
-}
-
-void attach_cancel_continuation (flux_future_t *f, void *arg)
-{
-    if (flux_future_get (f, NULL) < 0)
-        log_msg ("cancel: %s", future_strerror (f, errno));
-    flux_future_destroy (f);
-}
-
-/* Handle the user typing ctrl-C (SIGINT) and ctrl-Z (SIGTSTP).
- * If the user types ctrl-C twice within 2s, cancel the job.
- * If the user types ctrl-C then ctrl-Z within 2s, detach from the job.
- */
-void attach_signal_cb (flux_reactor_t *r, flux_watcher_t *w,
-                       int revents, void *arg)
-{
-    struct attach_ctx *ctx = arg;
-    flux_future_t *f;
-    int signum = flux_signal_watcher_get_signum (w);
-
-    if (signum == SIGINT) {
-        if (monotime_since (ctx->t_sigint) > 2000) {
-            monotime (&ctx->t_sigint);
-            flux_watcher_start (ctx->sigtstp_w);
-            log_msg ("one more ctrl-C within 2s to cancel or ctrl-Z to detach");
-        }
-        else {
-            if (!(f = flux_job_cancel (ctx->h, ctx->id,
-                                       "interrupted by ctrl-C")))
-                log_err_exit ("flux_job_cancel");
-            if (flux_future_then (f, -1, attach_cancel_continuation, NULL) < 0)
-                log_err_exit ("flux_future_then");
-        }
-    }
-    else if (signum == SIGTSTP) {
-        if (monotime_since (ctx->t_sigint) <= 2000) {
-            if (ctx->eventlog_f) {
-                if (flux_job_event_watch_cancel (ctx->eventlog_f) < 0)
-                    log_err_exit ("flux_job_event_watch_cancel");
-            }
-            if (ctx->exec_eventlog_f) {
-                if (flux_job_event_watch_cancel (ctx->exec_eventlog_f) < 0)
-                    log_err_exit ("flux_job_event_watch_cancel");
-            }
-            if (ctx->output_f) {
-                if (flux_job_event_watch_cancel (ctx->output_f) < 0)
-                    log_err_exit ("flux_job_event_watch_cancel");
-            }
-            log_msg ("detaching...");
-        }
-        else {
-            flux_watcher_stop (ctx->sigtstp_w);
-            log_msg ("one more ctrl-Z to suspend");
-        }
-    }
-}
-
-/* atexit handler
- * This is a good faith attempt to restore stdin flags to what they were
- * before we set O_NONBLOCK.
- */
-void restore_stdin_flags (void)
-{
-    (void)fcntl (STDIN_FILENO, F_SETFL, stdin_flags);
-}
-
-static void attach_send_shell_completion (flux_future_t *f, void *arg)
-{
-    struct attach_ctx *ctx = arg;
-
-    /* failng to write stdin to service is (generally speaking) a
-     * fatal error */
-    if (flux_future_get (f, NULL) < 0) {
-        /* stdin may not be accepted for multiple reasons
-         * - job has completed
-         * - user requested stdin via file
-         * - stdin stream already closed due to prior pipe in
-         */
-        if (errno == ENOSYS) {
-            /* Only generate an error if an attempt to send stdin failed.
-             */
-            if (ctx->stdin_data_sent)
-                log_msg_exit ("stdin not accepted by job");
-        }
-        else
-            log_err_exit ("attach_send_shell");
-    }
-    flux_future_destroy (f);
-    zlist_remove (ctx->stdin_rpcs, f);
-}
-
-static int attach_send_shell (struct attach_ctx *ctx,
-                              const void *buf,
-                              int len,
-                              bool eof)
-{
-    json_t *context = NULL;
-    char topic[1024];
-    flux_future_t *f = NULL;
-    int saved_errno;
-    int rc = -1;
-
-    snprintf (topic, sizeof (topic), "%s.stdin", ctx->service);
-    if (!(context = ioencode ("stdin", "all", buf, len, eof)))
-        goto error;
-    if (!(f = flux_rpc_pack (ctx->h, topic, ctx->leader_rank, 0, "O", context)))
-        goto error;
-    if (flux_future_then (f, -1, attach_send_shell_completion, ctx) < 0)
-        goto error;
-    if (zlist_append (ctx->stdin_rpcs, f) < 0)
-        goto error;
-    /* f memory now in hands of attach_send_shell_completion() or ctx->stdin_rpcs */
-    f = NULL;
-    rc = 0;
- error:
-    saved_errno = errno;
-    json_decref (context);
-    flux_future_destroy (f);
-    errno = saved_errno;
-    return rc;
-}
-
-/* Handle std input from user */
-void attach_stdin_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
-{
-    struct attach_ctx *ctx = arg;
-    flux_buffer_t *fb;
-    const char *ptr;
-    int len;
-
-    fb = flux_buffer_read_watcher_get_buffer (w);
-    assert (fb);
-
-    if (!(ptr = flux_buffer_read_line (fb, &len)))
-        log_err_exit ("flux_buffer_read_line on stdin");
-
-    if (len > 0) {
-        if (attach_send_shell (ctx, ptr, len, false) < 0)
-            log_err_exit ("attach_send_shell");
-        ctx->stdin_data_sent = true;
-    }
-    else {
-        /* possibly left over data before EOF */
-        if (!(ptr = flux_buffer_read (fb, -1, &len)))
-            log_err_exit ("flux_buffer_read on stdin");
-
-        if (len > 0) {
-            if (attach_send_shell (ctx, ptr, len, false) < 0)
-                log_err_exit ("attach_send_shell");
-            ctx->stdin_data_sent = true;
-        }
-        else {
-            if (attach_send_shell (ctx, NULL, 0, true) < 0)
-                log_err_exit ("attach_send_shell");
-            flux_watcher_stop (ctx->stdin_w);
-        }
-    }
-}
-
-/*  Start the guest.output eventlog watcher
- */
-void attach_output_start (struct attach_ctx *ctx)
-{
-    if (!(ctx->output_f = flux_job_event_watch (ctx->h,
-                                                ctx->id,
-                                                "guest.output",
-                                                0)))
-        log_err_exit ("flux_job_event_watch");
-
-    if (flux_future_then (ctx->output_f, -1.,
-                          attach_output_continuation,
-                          ctx) < 0)
-        log_err_exit ("flux_future_then");
-
-    ctx->eventlog_watch_count++;
-}
-
-/* Handle an event in the guest.exec eventlog.
- * This is a stream of responses, one response per event, terminated with
- * an ENODATA error response (or another error if something went wrong).
- * On the shell.init event, start watching the guest.output eventlog.
- * It is guaranteed to exist when guest.output is emitted.
- * If --show-exec was specified, print all events on stderr.
- */
-void attach_exec_event_continuation (flux_future_t *f, void *arg)
-{
-    struct attach_ctx *ctx = arg;
-    const char *entry;
-    json_t *o;
-    double timestamp;
-    const char *name;
-    json_t *context;
-    const char *service;
-
-    if (flux_job_event_watch_get (f, &entry) < 0) {
-        if (errno == ENODATA)
-            goto done;
-        log_msg_exit ("flux_job_event_watch_get: %s",
-                      future_strerror (f, errno));
-    }
-    if (!(o = eventlog_entry_decode (entry)))
-        log_err_exit ("eventlog_entry_decode");
-    if (eventlog_entry_parse (o, &timestamp, &name, &context) < 0)
-        log_err_exit ("eventlog_entry_parse");
-
-    if (!strcmp (name, "shell.init")) {
-        flux_watcher_t *w;
-
-        if (json_unpack (context,
-                         "{s:i s:s}",
-                         "leader-rank",
-                         &ctx->leader_rank,
-                         "service",
-                         &service) < 0)
-            log_err_exit ("error decoding shell.init context");
-        if (!(ctx->service = strdup (service)))
-            log_err_exit ("strdup service from shell.init");
-
-        /* flux_buffer_read_watcher_create() requires O_NONBLOCK on
-         * stdin */
-
-        if ((stdin_flags = fcntl (STDIN_FILENO, F_GETFL)) < 0)
-            log_err_exit ("fcntl F_GETFL stdin");
-        if (atexit (restore_stdin_flags) != 0)
-            log_err_exit ("atexit");
-        if (fcntl (STDIN_FILENO, F_SETFL, stdin_flags | O_NONBLOCK) < 0)
-            log_err_exit ("fcntl F_SETFL stdin");
-
-        w = flux_buffer_read_watcher_create (flux_get_reactor (ctx->h),
-                                             STDIN_FILENO,
-                                             1 << 20,
-                                             attach_stdin_cb,
-                                             FLUX_WATCHER_LINE_BUFFER,
-                                             ctx);
-        if (!w)
-            log_err_exit ("flux_buffer_read_watcher_create");
-
-        if (!(ctx->stdin_rpcs = zlist_new ()))
-            log_err_exit ("zlist_new");
-
-        ctx->stdin_w = w;
-        flux_watcher_start (ctx->stdin_w);
-
-        attach_output_start (ctx);
-    }
-
-    /*  If job is complete, and we haven't started watching
-     *   output eventlog, then start now in case shell.init event
-     *   was never emitted (failure in iniitialization)
-     */
-    if (!strcmp (name, "complete") && !ctx->output_f)
-        attach_output_start (ctx);
-
-    if (optparse_hasopt (ctx->p, "show-exec")) {
-        print_eventlog_entry (stderr,
-                              "exec",
-                              timestamp - ctx->timestamp_zero,
-                              name,
-                              context);
-    }
-
-    json_decref (o);
-    flux_future_reset (f);
-    return;
-done:
-    flux_future_destroy (f);
-    ctx->exec_eventlog_f = NULL;
-    ctx->eventlog_watch_count--;
-    attach_completed_check (ctx);
-}
-
-/* Handle an event in the main job eventlog.
- * This is a stream of responses, one response per event, terminated with
- * an ENODATA error response (or another error if something went wrong).
- * If a fatal exception event occurs, print it on stderr.
- * If --show-events was specified, print all events on stderr.
- * If submit event occurs, begin watching guest.exec.eventlog.
- * If finish event occurs, capture ctx->exit code.
- */
-void attach_event_continuation (flux_future_t *f, void *arg)
-{
-    struct attach_ctx *ctx = arg;
-    const char *entry;
-    json_t *o;
-    double timestamp;
-    const char *name;
-    json_t *context;
-    int status;
-
-    if (flux_job_event_watch_get (f, &entry) < 0) {
-        if (errno == ENODATA)
-            goto done;
-        if (errno == ENOENT)
-            log_msg_exit ("Failed to attach to %ju: No such job", ctx->id);
-        log_msg_exit ("flux_job_event_watch_get: %s",
-                      future_strerror (f, errno));
-    }
-    if (!(o = eventlog_entry_decode (entry)))
-        log_err_exit ("eventlog_entry_decode");
-    if (eventlog_entry_parse (o, &timestamp, &name, &context) < 0)
-        log_err_exit ("eventlog_entry_parse");
-
-    if (ctx->timestamp_zero == 0.)
-        ctx->timestamp_zero = timestamp;
-
-    if (!strcmp (name, "exception")) {
-        const char *type;
-        int severity;
-        const char *note = NULL;
-
-        if (json_unpack (context, "{s:s s:i s?:s}",
-                         "type", &type,
-                         "severity", &severity,
-                         "note", &note) < 0)
-            log_err_exit ("error decoding exception context");
-        fprintf (stderr, "%.3fs: job.exception type=%s severity=%d %s\n",
-                         timestamp - ctx->timestamp_zero,
-                         type,
-                         severity,
-                         note ? note : "");
-    }
-    else if (!strcmp (name, "submit")) {
-        if (!(ctx->exec_eventlog_f = flux_job_event_watch (ctx->h,
-                                                           ctx->id,
-                                                         "guest.exec.eventlog",
-                                                           0)))
-            log_err_exit ("flux_job_event_watch");
-        if (flux_future_then (ctx->exec_eventlog_f,
-                              -1,
-                              attach_exec_event_continuation,
-                              ctx) < 0)
-            log_err_exit ("flux_future_then");
-
-        ctx->eventlog_watch_count++;
-    }
-    else {
-        if (!strcmp (name, "finish")) {
-            if (json_unpack (context, "{s:i}", "status", &status) < 0)
-                log_err_exit ("error decoding finish context");
-            if (WIFSIGNALED (status)) {
-                ctx->exit_code = WTERMSIG (status) + 128;
-                log_msg ("task(s) %s", strsignal (WTERMSIG (status)));
-            }
-            else if (WIFEXITED (status)) {
-                ctx->exit_code = WEXITSTATUS (status);
-                if (ctx->exit_code != 0)
-                    log_msg ("task(s) exited with exit code %d",
-                             ctx->exit_code);
-            }
-        }
-    }
-
-    if (optparse_hasopt (ctx->p, "show-events")
-                                    && strcmp (name, "exception") != 0) {
-        print_eventlog_entry (stderr,
-                              "job",
-                              timestamp - ctx->timestamp_zero,
-                              name,
-                              context);
-    }
-
-    json_decref (o);
-    flux_future_reset (f);
-    return;
-done:
-    flux_future_destroy (f);
-    ctx->eventlog_f = NULL;
-    ctx->eventlog_watch_count--;
-    attach_completed_check (ctx);
-}
-
-int cmd_attach (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-    flux_reactor_t *r;
-    struct attach_ctx ctx;
-
-    memset (&ctx, 0, sizeof (ctx));
-    ctx.exit_code = 1;
-
-    if (argc - optindex != 1) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    ctx.id = parse_arg_unsigned (argv[optindex++], "jobid");
-    ctx.p = p;
-
-    if (!(ctx.h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (!(r = flux_get_reactor (ctx.h)))
-        log_err_exit ("flux_get_reactor");
-
-    if (!(ctx.eventlog_f = flux_job_event_watch (ctx.h,
-                                                 ctx.id,
-                                                 "eventlog",
-                                                 0)))
-        log_err_exit ("flux_job_event_watch");
-    if (flux_future_then (ctx.eventlog_f,
-                          -1,
-                          attach_event_continuation,
-                          &ctx) < 0)
-        log_err_exit ("flux_future_then");
-
-    ctx.eventlog_watch_count++;
-
-    /*  Ignore SIGTTIN, SIGTTOU.
-     *
-     *  SIGTTIN is ignored to avoid flux-job attach being stopped while
-     *   in the background. Normally, background flux-job attach doesn't
-     *   register activity on stdin, so this is not necessary. However,
-     *   in some cases (e.g. docker run -ti), activity on the terminal
-     *   does seem to wakeup epoll on background processes, and ignoring
-     *   SIGTTIN is a workaround in those cases.
-     *   (https://github.com/flux-framework/flux-core/issues/2599)
-     *
-     *  SIGTTOU is ignored so that flux-job attach can still write to
-     *   stderr/out even when in the background on a terminal with the
-     *   TOSTOP output mode set (also rare, but possible)
-     */
-    signal (SIGTTIN, SIG_IGN);
-    signal (SIGTTOU, SIG_IGN);
-
-    ctx.sigint_w = flux_signal_watcher_create (r,
-                                               SIGINT,
-                                               attach_signal_cb,
-                                               &ctx);
-    ctx.sigtstp_w = flux_signal_watcher_create (r,
-                                                SIGTSTP,
-                                                attach_signal_cb,
-                                                &ctx);
-    if (!ctx.sigint_w || !ctx.sigtstp_w)
-        log_err_exit ("flux_signal_watcher_create");
-    flux_watcher_start (ctx.sigint_w);
-
-    if (flux_reactor_run (r, 0) < 0)
-        log_err_exit ("flux_reactor_run");
-
-    zlist_destroy (&(ctx.stdin_rpcs));
-    flux_watcher_destroy (ctx.sigint_w);
-    flux_watcher_destroy (ctx.sigtstp_w);
-    flux_watcher_destroy (ctx.stdin_w);
-    flux_close (ctx.h);
-    free (ctx.service);
-    return ctx.exit_code;
-}
-
-#define EXCEPTION_TYPE_LENGTH 64
-struct job_status {
-    flux_jobid_t id;
-    int status;
-    int exit_code;
-    int exception_exit_code;
-    bool exception;
-    char ex_type[EXCEPTION_TYPE_LENGTH];
-};
-
-static void job_status_handle_exception (struct job_status *stat,
-                                         json_t *context)
-{
-    const char *type;
-    int severity;
-    const char *note = NULL;
-
-    if (json_unpack (context,
-                     "{s:s s:i s?:s}",
-                     "type", &type,
-                     "severity", &severity,
-                     "note", &note) < 0)
-        log_err_exit ("error decoding exception context");
-    if (severity == 0) {
-        /* Note: the exit_code and status will be overridden
-         *  by the finish event if this job is still running.
-         *  O/w, for a non-running job with a fatal exception
-         *  the default exit code is stat->exception_exit_code.
-         */
-        stat->exit_code = stat->exception_exit_code;
-        stat->status = stat->exit_code << 8;
-        stat->exception = true;
-        strncpy (stat->ex_type, type, sizeof(stat->ex_type) - 1);
-    }
-}
-
-static void status_eventlog_cb (flux_future_t *f, void *arg)
-{
-    struct job_status *stat = arg;
-    const char *entry = NULL;
-    const char *name = NULL;
-    json_t *context = NULL;
-    json_t *o = NULL;
-
-    if (flux_job_event_watch_get (f, &entry) < 0) {
-        if (errno == ENODATA)
-            goto done;
-        if (errno == ENOENT)
-            log_msg_exit ("%ju: No such job",
-                          (uintmax_t) stat->id);
-        log_msg_exit ("%ju: flux_job_event_watch_get: %s",
-                      (uintmax_t) stat->id,
-                      future_strerror (f, errno));
-    }
-    if (!(o = eventlog_entry_decode (entry)))
-        log_err_exit ("eventlog_entry_decode");
-    if (eventlog_entry_parse (o, NULL, &name, &context) < 0)
-        log_err_exit ("eventlog_entry_parse");
-
-    if (!strcmp (name, "finish")) {
-        if (json_unpack (context, "{s:i}", "status", &stat->status) < 0)
-            log_err_exit ("error decoding finish context");
-        if (flux_job_event_watch_cancel (f) < 0)
-            log_err_exit ("flux_job_event_watch_cancel");
-        if (WIFSIGNALED (stat->status))
-            stat->exit_code = WTERMSIG (stat->status) + 128;
-        else
-            stat->exit_code = WEXITSTATUS (stat->status);
-    }
-    else if (!strcmp (name, "exception"))
-        job_status_handle_exception (stat, context);
-
-    json_decref (o);
-    flux_future_reset (f);
-    return;
-done:
-    flux_future_destroy (f);
-}
-
-int cmd_status (optparse_t *p, int argc, char **argv)
-{
-    struct job_status *stats;
-    flux_t *h = NULL;
-    flux_future_t *f = NULL;
-    int exit_code;
-    int i;
-    int njobs;
-    int verbose = optparse_getopt (p, "verbose", NULL);
-    int optindex = optparse_option_index (p);
-    int exception_exit_code = optparse_get_int (p, "exception-exit-code", 1);
-
-    if ((njobs = (argc - optindex)) < 1) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (!(stats = calloc (njobs, sizeof (*stats))))
-        log_err_exit ("Failed to initialize stats array");
-
-    for (i = 0; i < njobs; i++) {
-        struct job_status *stat = &stats[i];
-        stat->id = parse_arg_unsigned (argv[optindex+i], "jobid");
-        stat->exception_exit_code = exception_exit_code;
-
-        if (!(f = flux_job_event_watch (h, stat->id, "eventlog", 0)))
-            log_err_exit ("flux_job_event_watch");
-        if (flux_future_then (f, -1, status_eventlog_cb, stat) < 0)
-            log_err_exit ("flux_future_then");
-    }
-
-    if (verbose && njobs > 1)
-        log_msg ("fetching status for %d jobs", njobs);
-
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
-        log_err ("flux_reactor_run");
-
-    if (verbose && njobs > 1)
-        log_msg ("all done.");
-
-    exit_code = 0;
-    for (i = 0; i < njobs; i++) {
-        struct job_status *stat = &stats[i];
-        if (stat->exit_code > exit_code)
-            exit_code = stat->exit_code;
-        if (optparse_hasopt (p, "verbose")) {
-            if (WIFSIGNALED (stat->status)) {
-                log_msg ("%ju: job shell died by signal %d",
-                         (uintmax_t) stat->id,
-                         WTERMSIG (stat->status));
-            }
-            else if (verbose > 1 || stat->exit_code != 0) {
-                if (!stat->exception)
-                    log_msg ("%ju: exited with exit code %d",
-                            (uintmax_t) stat->id,
-                            stat->exit_code);
-                else
-                    log_msg ("%ju: exception type=%s",
-                            (uintmax_t) stat->id,
-                            stat->ex_type);
-            }
-        }
-    }
-    flux_close (h);
-    free (stats);
-    return exit_code;
-}
-
-void id_convert (optparse_t *p, const char *src, char *dst, int dstsz)
-{
-    const char *from = optparse_get_str (p, "from", "dec");
-    const char *to = optparse_get_str (p, "to", "dec");
-    flux_jobid_t id;
-
-    /* src to id
-     */
-    if (!strcmp (from, "dec")) {
-        id = parse_arg_unsigned (src, "input");
-    }
-    else if (!strcmp (from, "hex")) {
-        if (fluid_decode (src, &id, FLUID_STRING_DOTHEX) < 0)
-            log_msg_exit ("%s: malformed input", src);
-    }
-    else if (!strcmp (from, "kvs")) {
-        if (strncmp (src, "job.", 4) != 0)
-            log_msg_exit ("%s: missing 'job.' prefix", src);
-        if (fluid_decode (src + 4, &id, FLUID_STRING_DOTHEX) < 0)
-            log_msg_exit ("%s: malformed input", src);
-    }
-    else if (!strcmp (from, "words")) {
-        if (fluid_decode (src, &id, FLUID_STRING_MNEMONIC) < 0)
-            log_msg_exit ("%s: malformed input", src);
-    }
-    else
-        log_msg_exit ("Unknown from=%s", from);
-
-    /* id to dst
-     */
-    if (!strcmp (to, "dec")) {
-        snprintf (dst, dstsz, "%llu", (unsigned long long)id);
-    }
-    else if (!strcmp (to, "kvs")) {
-        if (flux_job_kvs_key (dst, dstsz, id, NULL) < 0)
-            log_msg_exit ("error encoding id");
-    }
-    else if (!strcmp (to, "hex")) {
-        if (fluid_encode (dst, dstsz, id, FLUID_STRING_DOTHEX) < 0)
-            log_msg_exit ("error encoding id");
-    }
-    else if (!strcmp (to, "words")) {
-        if (fluid_encode (dst, dstsz, id, FLUID_STRING_MNEMONIC) < 0)
-            log_msg_exit ("error encoding id");
-    }
-    else
-        log_msg_exit ("Unknown to=%s", to);
-}
-
-char *trim_string (char *s)
-{
-    /* trailing */
-    int len = strlen (s);
-    while (len > 1 && isspace (s[len - 1])) {
-        s[len - 1] = '\0';
-        len--;
-    }
-    /* leading */
-    char *p = s;
-    while (*p && isspace (*p))
-        p++;
-    return p;
-}
-
-int cmd_id (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-    char dst[256];
-
-    if (optindex == argc) {
-        char src[256];
-        while ((fgets (src, sizeof (src), stdin))) {
-            id_convert (p, trim_string (src), dst, sizeof (dst));
-            printf ("%s\n", dst);
-        }
-    }
-    else {
-        while (optindex < argc) {
-            id_convert (p, argv[optindex++], dst, sizeof (dst));
-            printf ("%s\n", dst);
-        }
-    }
-    return 0;
-}
-
-static void print_job_namespace (const char *src)
-{
-    char ns[64];
-    flux_jobid_t id = parse_arg_unsigned (src, "jobid");
-    if (flux_job_kvs_namespace (ns, sizeof (ns), id) < 0)
-        log_msg_exit ("error getting kvs namespace for %ju", id);
-    printf ("%s\n", ns);
-}
-
-int cmd_namespace (optparse_t *p, int argc, char **argv)
-{
-    int optindex = optparse_option_index (p);
-
-    if (optindex == argc) {
-        char src[256];
-        while ((fgets (src, sizeof (src), stdin)))
-            print_job_namespace (trim_string (src));
-    }
-    else {
-        while (optindex < argc)
-            print_job_namespace (argv[optindex++]);
-    }
-    return 0;
-}
-
-struct entry_format {
-    const char *format;
-    const char *time_format;
-    double initial;
-};
-
-void entry_format_parse_options (optparse_t *p, struct entry_format *e)
-{
-    e->format = optparse_get_str (p, "format", "text");
-    if (strcasecmp (e->format, "text")
-        && strcasecmp (e->format, "json"))
-        log_msg_exit ("invalid format type");
-    e->time_format = optparse_get_str (p, "time-format", "raw");
-    if (strcasecmp (e->time_format, "raw")
-        && strcasecmp (e->time_format, "iso")
-        && strcasecmp (e->time_format, "offset"))
-        log_msg_exit ("invalid time-format type");
-}
-
-struct eventlog_ctx {
-    optparse_t *p;
-    flux_jobid_t id;
-    const char *path;
-    struct entry_format e;
-};
-
-/* convert floating point timestamp (UNIX epoch, UTC) to ISO 8601 string,
- * with microsecond precision
- */
-static int event_timestr (struct entry_format *e, double timestamp,
-                          char *buf, size_t size)
-{
-    if (!strcasecmp (e->time_format, "raw")) {
-        if (snprintf (buf, size, "%lf", timestamp) >= size)
-            return -1;
-    }
-    else if (!strcasecmp (e->time_format, "iso")) {
-        time_t sec = timestamp;
-        unsigned long usec = (timestamp - sec)*1E6;
-        struct tm tm;
-        if (!gmtime_r (&sec, &tm))
-            return -1;
-        if (strftime (buf, size, "%FT%T", &tm) == 0)
-            return -1;
-        size -= strlen (buf);
-        buf += strlen (buf);
-        if (snprintf (buf, size, ".%.6luZ", usec) >= size)
-            return -1;
-    }
-    else { /* !strcasecmp (e->time_format, "offset") */
-        if (e->initial == 0.)
-            e->initial = timestamp;
-        timestamp -= e->initial;
-        if (snprintf (buf, size, "%lf", timestamp) >= size)
-            return -1;
-    }
-    return 0;
-}
-
-void output_event_text (struct entry_format *e, json_t *event)
-{
-    double timestamp;
-    const char *name;
-    json_t *context = NULL;
-    char buf[128];
-
-    if (eventlog_entry_parse (event, &timestamp, &name, &context) < 0)
-        log_err_exit ("eventlog_entry_parse");
-
-    if (event_timestr (e, timestamp, buf, sizeof (buf)) < 0)
-        log_msg_exit ("error converting timestamp to ISO 8601");
-
-    printf ("%s %s", buf, name);
-
-    if (context) {
-        const char *key;
-        json_t *value;
-        json_object_foreach (context, key, value) {
-            char *sval;
-            sval = json_dumps (value, JSON_ENCODE_ANY|JSON_COMPACT);
-            printf (" %s=%s", key, sval);
-            free (sval);
-        }
-    }
-    printf ("\n");
-    fflush (stdout);
-}
-
-void output_event_json (json_t *event)
-{
-    char *e;
-
-    if (!(e = json_dumps (event, JSON_COMPACT)))
-        log_msg_exit ("json_dumps");
-    printf ("%s\n", e);
-    free (e);
-}
-
-void output_event (struct entry_format *e, json_t *event)
-{
-    if (!strcasecmp (e->format, "text"))
-        output_event_text (e, event);
-    else /* !strcasecmp (e->format, "json") */
-        output_event_json (event);
-}
-
-void eventlog_continuation (flux_future_t *f, void *arg)
-{
-    struct eventlog_ctx *ctx = arg;
-    const char *s;
-    json_t *a = NULL;
-    size_t index;
-    json_t *value;
-
-    if (flux_rpc_get_unpack (f, "{s:s}", ctx->path, &s) < 0) {
-        if (errno == ENOENT) {
-            flux_future_destroy (f);
-            if (!strcmp (ctx->path, "eventlog"))
-                log_msg_exit ("job %lu not found", ctx->id);
-            else
-                log_msg_exit ("eventlog path %s not found", ctx->path);
-        }
-        else
-            log_err_exit ("flux_job_eventlog_lookup_get");
-    }
-
-    if (!(a = eventlog_decode (s)))
-        log_err_exit ("eventlog_decode");
-
-    json_array_foreach (a, index, value) {
-        output_event (&ctx->e, value);
-    }
-
-    fflush (stdout);
-    json_decref (a);
-    flux_future_destroy (f);
-}
-
-int cmd_eventlog (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-    flux_future_t *f;
-    const char *topic = "job-info.lookup";
-    struct eventlog_ctx ctx = {0};
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (argc - optindex != 1) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-
-    ctx.id = parse_arg_unsigned (argv[optindex++], "jobid");
-    ctx.path = optparse_get_str (p, "path", "eventlog");
-    ctx.p = p;
-    entry_format_parse_options (p, &ctx.e);
-
-    if (!(f = flux_rpc_pack (h, topic, FLUX_NODEID_ANY, 0,
-                             "{s:I s:[s] s:i}",
-                             "id", ctx.id,
-                             "keys", ctx.path,
-                             "flags", 0)))
-        log_err_exit ("flux_rpc_pack");
-    if (flux_future_then (f, -1., eventlog_continuation, &ctx) < 0)
-        log_err_exit ("flux_future_then");
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
-        log_err_exit ("flux_reactor_run");
-
-    flux_close (h);
-    return (0);
-}
-
-struct wait_event_ctx {
-    optparse_t *p;
-    const char *wait_event;
-    double timeout;
-    flux_jobid_t id;
-    const char *path;
-    bool got_event;
-    struct entry_format e;
-    char *context_key;
-    char *context_value;
-};
-
-bool wait_event_test_context (struct wait_event_ctx *ctx, json_t *context)
-{
-    void *iter;
-    bool match = false;
-
-    iter = json_object_iter (context);
-    while (iter && !match) {
-        const char *key = json_object_iter_key (iter);
-        json_t *value = json_object_iter_value (iter);
-        if (!strcmp (key, ctx->context_key)) {
-            char *str = json_dumps (value, JSON_ENCODE_ANY|JSON_COMPACT);
-            if (!strcmp (str, ctx->context_value))
-                match = true;
-            free (str);
-        }
-        /* special case, json_dumps() will put quotes around string
-         * values.  Consider the case when user does not surround
-         * string value with quotes */
-        if (!match && json_is_string (value)) {
-            const char *str = json_string_value (value);
-            if (!strcmp (str, ctx->context_value))
-                match = true;
-        }
-        iter = json_object_iter_next (context, iter);
-    }
-    return match;
-}
-
-bool wait_event_test (struct wait_event_ctx *ctx, json_t *event)
-{
-    double timestamp;
-    const char *name;
-    json_t *context = NULL;
-    bool match = false;
-
-    if (eventlog_entry_parse (event, &timestamp, &name, &context) < 0)
-        log_err_exit ("eventlog_entry_parse");
-
-    if (ctx->e.initial == 0.)
-        ctx->e.initial = timestamp;
-
-    if (!strcmp (name, ctx->wait_event)) {
-        if (ctx->context_key) {
-            if (context)
-                match = wait_event_test_context (ctx, context);
-        }
-        else
-            match = true;
-    }
-
-    return match;
-}
-
-void wait_event_continuation (flux_future_t *f, void *arg)
-{
-    struct wait_event_ctx *ctx = arg;
-    json_t *o = NULL;
-    const char *event;
-
-    if (flux_rpc_get (f, NULL) < 0) {
-        if (errno == ENOENT) {
-            flux_future_destroy (f);
-            if (!strcmp (ctx->path, "eventlog"))
-                log_msg_exit ("job %lu not found", ctx->id);
-            else
-                log_msg_exit ("eventlog path %s not found", ctx->path);
-        }
-        else if (errno == ETIMEDOUT) {
-            flux_future_destroy (f);
-            log_msg_exit ("wait-event timeout on event '%s'\n",
-                          ctx->wait_event);
-        } else if (errno == ENODATA) {
-            flux_future_destroy (f);
-            if (!ctx->got_event)
-                log_msg_exit ("event '%s' never received",
-                              ctx->wait_event);
-            return;
-        }
-        /* else fallthrough and have `flux_job_event_watch_get'
-         * handle error */
-    }
-
-    if (flux_job_event_watch_get (f, &event) < 0)
-        log_err_exit ("flux_job_event_watch_get");
-
-    if (!(o = eventlog_entry_decode (event)))
-        log_err_exit ("eventlog_entry_decode");
-
-    if (wait_event_test (ctx, o)) {
-        ctx->got_event = true;
-        if (!optparse_hasopt (ctx->p, "quiet"))
-            output_event (&ctx->e, o);
-        if (flux_job_event_watch_cancel (f) < 0)
-            log_err_exit ("flux_job_event_watch_cancel");
-    } else if (optparse_hasopt (ctx->p, "verbose")) {
-        if (!ctx->got_event)
-            output_event (&ctx->e, o);
-    }
-
-    json_decref (o);
-
-    flux_future_reset (f);
-
-    /* re-call to set timeout */
-    if (flux_future_then (f, ctx->timeout, wait_event_continuation, arg) < 0)
-        log_err_exit ("flux_future_then");
-}
-
-int cmd_wait_event (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-    flux_future_t *f;
-    struct wait_event_ctx ctx = {0};
-    const char *str;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if ((argc - optindex) != 2) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    ctx.id = parse_arg_unsigned (argv[optindex++], "jobid");
-    ctx.p = p;
-    ctx.wait_event = argv[optindex++];
-    ctx.timeout = optparse_get_duration (p, "timeout", -1.0);
-    ctx.path = optparse_get_str (p, "path", "eventlog");
-    entry_format_parse_options (p, &ctx.e);
-    if ((str = optparse_get_str (p, "match-context", NULL))) {
-        ctx.context_key = xstrdup (str);
-        ctx.context_value = strchr (ctx.context_key, '=');
-        if (!ctx.context_value)
-            log_msg_exit ("must specify a context test as key=value");
-        *ctx.context_value++ = '\0';
-    }
-
-    if (!(f = flux_job_event_watch (h, ctx.id, ctx.path, 0)))
-        log_err_exit ("flux_job_event_watch");
-    if (flux_future_then (f, ctx.timeout, wait_event_continuation, &ctx) < 0)
-        log_err_exit ("flux_future_then");
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
-        log_err_exit ("flux_reactor_run");
-
-    free (ctx.context_key);
-    flux_close (h);
-    return (0);
-}
-
-struct info_ctx {
-    flux_jobid_t id;
-    json_t *keys;
-};
-
-void info_output (flux_future_t *f, const char *suffix, flux_jobid_t id)
-{
-    const char *s;
-
-    if (flux_rpc_get_unpack (f, "{s:s}", suffix, &s) < 0) {
-        if (errno == ENOENT) {
-            flux_future_destroy (f);
-            log_msg_exit ("job %lu id or key not found", id);
-        }
-        else
-            log_err_exit ("flux_rpc_get_unpack");
-    }
-
-    /* XXX - prettier output later */
-    printf ("%s\n", s);
-}
-
-void info_continuation (flux_future_t *f, void *arg)
-{
-    struct info_ctx *ctx = arg;
-    size_t index;
-    json_t *key;
-
-    json_array_foreach (ctx->keys, index, key) {
-        const char *s = json_string_value (key);
-        info_output (f, s, ctx->id);
-    }
-
-    flux_future_destroy (f);
-}
-
-int cmd_info (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-    flux_future_t *f;
-    const char *topic = "job-info.lookup";
-    struct info_ctx ctx = {0};
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if ((argc - optindex) < 2) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-
-    ctx.id = parse_arg_unsigned (argv[optindex++], "jobid");
-
-    if (!(ctx.keys = json_array ()))
-        log_msg_exit ("json_array");
-
-    while (optindex < argc) {
-        json_t *s;
-        if (!(s = json_string (argv[optindex])))
-            log_msg_exit ("json_string");
-        if (json_array_append_new (ctx.keys, s) < 0)
-            log_msg_exit ("json_array_append");
-        optindex++;
-    }
-
-    if (!(f = flux_rpc_pack (h, topic, FLUX_NODEID_ANY, 0,
-                             "{s:I s:O s:i}",
-                             "id", ctx.id,
-                             "keys", ctx.keys,
-                             "flags", 0)))
-        log_err_exit ("flux_rpc_pack");
-    if (flux_future_then (f, -1., info_continuation, &ctx) < 0)
-        log_err_exit ("flux_future_then");
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
-        log_err_exit ("flux_reactor_run");
-
-    json_decref (ctx.keys);
-    flux_close (h);
-    return (0);
-}
-
-int cmd_stats (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    flux_future_t *f;
-    const char *topic = "job-info.job-stats";
-    const char *s;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (!(f = flux_rpc (h, topic, NULL, FLUX_NODEID_ANY, 0)))
-        log_err_exit ("flux_rpc");
-    if (flux_rpc_get (f, &s) < 0)
-        log_msg_exit ("stats: %s", future_strerror (f, errno));
-
-    /* for time being, just output json object for result */
-    printf ("%s\n", s);
-    flux_close (h);
-    return (0);
-}
-
-int cmd_wait (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-    flux_future_t *f;
-    flux_jobid_t id = FLUX_JOBID_ANY;
-    bool success;
-    const char *errstr;
-    int rc = 0;
-
-    if ((argc - optindex) > 1) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (optindex < argc) {
-        id = parse_arg_unsigned (argv[optindex++], "jobid");
-        if (optparse_hasopt (p, "all"))
-            log_err_exit ("jobid not supported with --all");
-    }
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (optparse_hasopt (p, "all")) {
-        for (;;) {
-            if (!(f = flux_job_wait (h, FLUX_JOBID_ANY)))
-                log_err_exit ("flux_job_wait");
-            if (flux_job_wait_get_status (f, &success, &errstr) < 0) {
-                if (errno == ECHILD) { // no more waitable jobs
-                    flux_future_destroy (f);
-                    break;
-                }
-                log_msg_exit ("flux_job_wait_get_status: %s",
-                              future_strerror (f, errno));
-            }
-            if (flux_job_wait_get_id (f, &id) < 0)
-                log_msg_exit ("flux_job_wait_get_id: %s",
-                              future_strerror (f, errno));
-            if (!success) {
-                fprintf (stderr, "%ju: %s\n", (uintmax_t)id, errstr);
-                rc = 1;
-            }
-            else {
-                if (optparse_hasopt (p, "verbose"))
-                    fprintf (stderr,
-                             "%ju: job completed successfully\n",
-                             (uintmax_t)id);
-            }
-            flux_future_destroy (f);
-        }
-    }
-    else {
-        if (!(f = flux_job_wait (h, id)))
-            log_err_exit ("flux_job_wait");
-        if (flux_job_wait_get_status (f, &success, &errstr) < 0)
-            log_msg_exit ("%s", flux_future_error_string (f));
-        if (id == FLUX_JOBID_ANY) {
-            if (flux_job_wait_get_id (f, &id) < 0)
-                log_err_exit ("flux_job_wait_get_id");
-            printf ("%ju\n", (uintmax_t)id);
-        }
-        if (!success)
-            log_msg_exit ("%s", errstr);
-        flux_future_destroy (f);
-    }
-    flux_close (h);
-    return (rc);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/cmd/flux-jobs.py b/src/cmd/flux-jobs.py
index 292ae0745415..16a866f4a961 100755
--- a/src/cmd/flux-jobs.py
+++ b/src/cmd/flux-jobs.py
@@ -8,255 +8,193 @@
 # SPDX-License-Identifier: LGPL-3.0
 ##############################################################
 
-from __future__ import print_function
-
+import argparse
+import concurrent.futures
+import fileinput
+import json
+import logging
 import os
 import sys
-import logging
-import argparse
-import time
-import pwd
-from datetime import timedelta
 
-import flux.job
 import flux.constants
-import flux.util
-from flux.core.inner import raw
-from flux.memoized_property import memoized_property
-
-logger = logging.getLogger("flux-jobs")
-
-state_const_dict = {
-    "depend": flux.constants.FLUX_JOB_DEPEND,
-    "sched": flux.constants.FLUX_JOB_SCHED,
-    "run": flux.constants.FLUX_JOB_RUN,
-    "cleanup": flux.constants.FLUX_JOB_CLEANUP,
-    "inactive": flux.constants.FLUX_JOB_INACTIVE,
-    "pending": flux.constants.FLUX_JOB_PENDING,
-    "running": flux.constants.FLUX_JOB_RUNNING,
-    "active": flux.constants.FLUX_JOB_ACTIVE,
-}
-
-
-def fsd(t, hyphenifzero):
-    #  Round <1ms down to 0s for now
-    if t < 1.0e-3:
-        s = "0s"
-    elif t < 10.0:
-        s = "%.03fs" % t
-    elif t < 60.0:
-        s = "%.4gs" % t
-    elif t < (60.0 * 60.0):
-        s = "%.4gm" % (t / 60.0)
-    elif t < (60.0 * 60.0 * 24.0):
-        s = "%.4gh" % (t / (60.0 * 60.0))
-    else:
-        s = "%.4gd" % (t / (60.0 * 60.0 * 24.0))
-    if hyphenifzero and s == "0s":
-        return "-"
-    return s
-
-
-def statetostr(stateid, singlechar=False):
-    return raw.flux_job_statetostr(stateid, singlechar).decode("utf-8")
-
-
-def get_username(userid):
-    try:
-        return pwd.getpwuid(userid).pw_name
-    except KeyError:
-        return str(userid)
-
-
-class JobInfo:
-    """
-    JobInfo class: encapsulate job-info.list response in an object
-    that implements a getattr interface to job information with
-    memoization. Better for use with output formats since results
-    are only computed as-needed.
-    """
-
-    #  Default values for job properties.
-    defaults = dict(
-        t_depend=0.0,
-        t_sched=0.0,
-        t_run=0.0,
-        t_cleanup=0.0,
-        t_inactive=0.0,
-        nnodes="",
-        ranks="",
-    )
-
-    def __init__(self, info_resp):
-        #  Set defaults, then update with job-info.list response items:
-        d = self.defaults.copy()
-        d.update(info_resp)
-
-        #  Rename "state" to "state_id" until returned state is a string:
-        if "state" in d:
-            d["state_id"] = d.pop("state")
-
-        #  Set all keys as self._{key} to be found by getattr and
-        #   memoized_property decorator:
-        for key, value in d.items():
-            setattr(self, "_{0}".format(key), value)
-
-    #  getattr method to return all non-computed values in job-info.list
-    #   response by default. Avoids the need to wrap @property methods
-    #   that just return self._<attr>.
-    #
-    def __getattr__(self, attr):
-        if attr.startswith("_"):
-            raise AttributeError
-        try:
-            return getattr(self, "_{0}".format(attr))
-        except KeyError:
-            raise AttributeError("invalid JobInfo attribute '{}'".format(attr))
-
-    def get_runtime(self, roundup=False):
-        if self.t_cleanup > 0 and self.t_run > 0:
-            t = self.t_cleanup - self.t_run
-            if roundup:
-                t = round(t + 0.5)
-        elif self.t_run > 0:
-            t = time.time() - self.t_run
-            if roundup:
-                t = round(t + 0.5)
-        else:
-            t = 0.0
-        return t
-
-    @memoized_property
-    def state(self):
-        return statetostr(self.state_id)
-
-    @memoized_property
-    def state_single(self):
-        return statetostr(self.state_id, True)
-
-    @memoized_property
-    def username(self):
-        return get_username(self.userid)
-
-    @memoized_property
-    def nnodes_hyphen(self):
-        return self.nnodes or "-"
-
-    @memoized_property
-    def ranks_hyphen(self):
-        return self.ranks or "-"
-
-    @memoized_property
-    def runtime_fsd(self):
-        return fsd(self.runtime, False)
-
-    @memoized_property
-    def runtime_fsd_hyphen(self):
-        return fsd(self.runtime, True)
-
-    @memoized_property
-    def runtime_hms(self):
-        t = self.get_runtime(True)
-        return str(timedelta(seconds=t))
-
-    @memoized_property
-    def runtime(self):
-        return self.get_runtime()
-
-
-def fetch_jobs_stdin(args):
+from flux.hostlist import Hostlist
+from flux.idset import IDset
+from flux.job import JobID, JobInfo, JobInfoFormat, JobList, job_fields_to_attrs
+from flux.job.stats import JobStats
+from flux.util import (
+    FilterAction,
+    FilterActionSetUpdate,
+    FilterTrueAction,
+    UtilConfig,
+    help_formatter,
+)
+
+LOGGER = logging.getLogger("flux-jobs")
+
+
+class FluxJobsConfig(UtilConfig):
+    """flux-jobs specific user configuration class"""
+
+    builtin_formats = {
+        "default": {
+            "description": "Default flux-jobs format string",
+            "format": (
+                "{id.f58:>12} ?:{queue:<8.8} {username:<8.8} {name:<10.10+} "
+                "{status_abbrev:>2.2} {ntasks:>6} {nnodes:>6h} "
+                "{contextual_time!F:>8h} {contextual_info}"
+            ),
+        },
+        "cute": {
+            "description": "Cute flux-jobs format string (default with emojis)",
+            "format": (
+                "{id.f58:>12} ?:{queue:<8.8} {username:<8.8} {name:<10.10+} "
+                "{status_emoji:>5.5} {ntasks:>6} {nnodes:>6h} "
+                "{contextual_time!F:>8h} {contextual_info}"
+            ),
+        },
+        "long": {
+            "description": "Extended flux-jobs format string",
+            "format": (
+                "{id.f58:>12} ?:{queue:<8.8} {username:<8.8} {name:<10.10+} "
+                "{status:>9.9} {ntasks:>6} {nnodes:>6h} "
+                "{t_submit!d:%b%d %R::>12} {t_remaining!F:>12h} "
+                "{contextual_time!F:>8h} {contextual_info}"
+            ),
+        },
+        "deps": {
+            "description": "Show job urgency, priority, and dependencies",
+            "format": (
+                "{id.f58:>12} ?:{queue:<8.8} {name:<10.10+} {urgency:<3} "
+                "{priority:<12} {state:<8.8} {dependencies}"
+            ),
+        },
+        "endreason": {
+            "description": "Show why each job ended",
+            "format": (
+                "{id.f58:>12} ?:{queue:<8.8} {username:<8.8} {name:<10.10+} "
+                "{status_abbrev:>2.2} {t_inactive!d:%b%d %R::>12h} {inactive_reason}"
+            ),
+        },
+    }
+
+    def __init__(self):
+        initial_dict = {"formats": dict(self.builtin_formats)}
+        super().__init__(name="flux-jobs", initial_dict=initial_dict)
+
+    def validate(self, path, config):
+        """Validate a loaded flux-jobs config file as dictionary"""
+        for key, value in config.items():
+            if key == "formats":
+                self.validate_formats(path, value)
+            else:
+                raise ValueError(f"{path}: invalid key {key}")
+
+
+def fetch_jobs_stdin():
     """
     Return a list of jobs gathered from a series of JSON objects, one per
     line, presented on stdin. This function is used for testing of the
     flux-jobs utility, and thus, all filtering options are currently
     ignored.
     """
-    import fileinput
-    import json
-
     jobs = []
     for line in fileinput.input("-"):
         try:
-            job = json.loads(line)
+            job = JobInfo(json.loads(line))
         except ValueError as err:
-            logger.error(
-                "JSON input error: line {}: {}".format(fileinput.lineno(), err)
-            )
+            LOGGER.error("JSON input error: line %d: %s", fileinput.lineno(), str(err))
             sys.exit(1)
         jobs.append(job)
     return jobs
 
 
-def fetch_jobs_flux(args, fields):
-    h = flux.Flux()
-
-    # Note there is no attr for "id", its always returned
-    fields2attrs = dict(
-        id=(),
-        userid=("userid",),
-        username=("userid",),
-        priority=("priority",),
-        state=("state",),
-        state_single=("state",),
-        name=("name",),
-        ntasks=("ntasks",),
-        nnodes=("nnodes",),
-        nnodes_hyphen=("nnodes",),
-        ranks=("ranks",),
-        ranks_hyphen=("ranks",),
-        t_submit=("t_submit",),
-        t_depend=("t_depend",),
-        t_sched=("t_sched",),
-        t_run=("t_run",),
-        t_cleanup=("t_cleanup",),
-        t_inactive=("t_inactive",),
-        runtime=("t_run", "t_cleanup"),
-        runtime_fsd=("t_run", "t_cleanup"),
-        runtime_fsd_hyphen=("t_run", "t_cleanup"),
-        runtime_hms=("t_run", "t_cleanup"),
-    )
-
-    attrs = set()
+def need_instance_info(fields):
     for field in fields:
-        attrs.update(fields2attrs[field])
+        if field.startswith("instance."):
+            return True
+    return False
+
+
+# pylint: disable=too-many-branches
+def fetch_jobs_flux(args, fields, flux_handle=None):
+    if not flux_handle:
+        flux_handle = flux.Flux()
+
+    attrs = job_fields_to_attrs(fields)
+
+    if args.color == "always" or args.color == "auto":
+        attrs.update(job_fields_to_attrs(["result", "annotations"]))
+    if args.recursive:
+        attrs.update(job_fields_to_attrs(["annotations", "status", "userid"]))
+    if args.json:
+        args.no_header = True
+        attrs.add("all")
 
-    if args.a:
-        args.user = str(os.geteuid())
     if args.A:
         args.user = str(flux.constants.FLUX_USERID_UNKNOWN)
-    if args.a or args.A:
-        args.states = "pending,running,inactive"
 
-    if args.user == "all":
-        userid = flux.constants.FLUX_USERID_UNKNOWN
-    else:
+    since = 0.0
+    if args.since:
+        # Implies -a, *unless* another filter option is already in effect:
+        if not args.filter:
+            args.a = True
+
         try:
-            userid = pwd.getpwnam(args.user).pw_uid
-        except KeyError:
+            since = flux.util.parse_datetime(args.since).timestamp()
+        except ValueError:
+            since = float(args.since)
+
+        # Ensure args.since is in the past
+        if since > flux.util.parse_datetime("now").timestamp():
+            LOGGER.error("--since=%s appears to be in the future", args.since)
+            sys.exit(1)
+
+    if args.a:
+        if args.filter:
+            LOGGER.warning("Both -a and --filter specified, ignoring -a")
+        else:
+            args.filter.update(["pending", "running", "inactive"])
+
+    if not args.filter:
+        args.filter = {"pending", "running"}
+
+    constraint = None
+    if args.include:
+        try:
+            constraint = {"ranks": [IDset(args.include).encode()]}
+        except ValueError:
             try:
-                userid = int(args.user)
+                constraint = {"hostlist": [Hostlist(args.include).encode()]}
             except ValueError:
-                print("invalid user specified", file=sys.stderr)
-                sys.exit(1)
+                raise ValueError(f"-i/--include: invalid targets: {args.include}")
+
+    jobs_rpc = JobList(
+        flux_handle,
+        ids=args.jobids,
+        attrs=attrs,
+        filters=args.filter,
+        user=args.user,
+        max_entries=args.count,
+        since=since,
+        name=args.name,
+        queue=args.queue,
+        constraint=constraint,
+    )
 
-    states = 0
-    for state in args.states.split(","):
-        try:
-            states |= state_const_dict[state.lower()]
-        except KeyError:
-            print("Invalid state specified: {}".format(state), file=sys.stderr)
-            sys.exit(1)
+    jobs = jobs_rpc.jobs()
 
-    if states == 0:
-        states |= flux.constants.FLUX_JOB_PENDING
-        states |= flux.constants.FLUX_JOB_RUNNING
+    if need_instance_info(fields):
+        with concurrent.futures.ThreadPoolExecutor(args.threads) as executor:
+            concurrent.futures.wait(
+                [executor.submit(job.get_instance_info) for job in jobs]
+            )
 
-    rpc_handle = flux.job.job_list(h, args.count, list(attrs), userid, states)
+    #  Print all errors accumulated in JobList RPC:
     try:
-        jobs = rpc_handle.get_jobs()
-    except EnvironmentError as e:
-        print("{}: {}".format("rpc", e.strerror), file=sys.stderr)
-        sys.exit(1)
+        for err in jobs_rpc.errors:
+            print(err, file=sys.stderr)
+    except EnvironmentError:
+        pass
 
     return jobs
 
@@ -267,200 +205,345 @@ def fetch_jobs(args, fields):
     Returns a list of JobInfo objects
     """
     if args.from_stdin:
-        l = fetch_jobs_stdin(args)
+        lst = fetch_jobs_stdin()
     else:
-        l = fetch_jobs_flux(args, fields)
-    return [JobInfo(job) for job in l]
+        lst = fetch_jobs_flux(args, fields)
+    return lst
 
 
 def parse_args():
-    parser = argparse.ArgumentParser(
-        prog="flux-jobs", formatter_class=flux.util.help_formatter()
-    )
+    parser = argparse.ArgumentParser(prog="flux-jobs", formatter_class=help_formatter())
     # -a equivalent to -s "pending,running,inactive" and -u set to userid
+    parser.add_argument("-a", action=FilterTrueAction, help="List jobs in all states")
+    # -A equivalent to -s "pending,running,inactive" and -u set to "all"
     parser.add_argument(
-        "-a", action="store_true", help="List all jobs for current user"
+        "-A", action=FilterTrueAction, help="List all jobs for all users"
     )
-    # -A equivalent to -s "pending,running,inactive" and -u set to "all"
-    parser.add_argument("-A", action="store_true", help="List all jobs for all users")
     parser.add_argument(
         "-c",
         "--count",
+        action=FilterAction,
         type=int,
         metavar="N",
         default=1000,
         help="Limit output to N jobs(default 1000)",
     )
     parser.add_argument(
-        "-s",
-        "--states",
+        "-f",
+        "--filter",
+        action=FilterActionSetUpdate,
+        metavar="STATE|RESULT",
+        default=set(),
+        help="List jobs with specific job state or result",
+    )
+    parser.add_argument(
+        "--since",
+        action=FilterAction,
         type=str,
-        metavar="STATES",
-        default="pending,running",
-        help="List jobs in specific job states or virtual job states",
+        metavar="WHEN",
+        default=0.0,
+        help="Include jobs that have become inactive since WHEN. "
+        + "(implies -a if no other --filter option is specified)",
     )
     parser.add_argument(
         "-n",
-        "--suppress-header",
+        "--no-header",
         action="store_true",
         help="Suppress printing of header line",
     )
+    # --suppress-header, older legacy name
+    parser.add_argument(
+        "--suppress-header",
+        action="store_true",
+        dest="no_header",
+        help=argparse.SUPPRESS,
+    )
     parser.add_argument(
         "-u",
         "--user",
+        action=FilterAction,
         type=str,
         metavar="[USERNAME|UID]",
-        default=str(os.geteuid()),
-        help='Limit output to specific username or userid (Specify "all" for all users)',
+        default=str(os.getuid()),
+        help="Limit output to specific username or userid "
+        '(Specify "all" for all users)',
+    )
+    parser.add_argument(
+        "--name",
+        action=FilterAction,
+        type=str,
+        metavar="JOB-NAME",
+        help="Limit output to specific job name",
+    )
+    parser.add_argument(
+        "-q",
+        "--queue",
+        action=FilterActionSetUpdate,
+        default=set(),
+        metavar="QUEUE,...",
+        help="Limit output to specific queue or queues",
+    )
+    parser.add_argument(
+        "-i",
+        "--include",
+        type=str,
+        metavar="HOSTS|RANKS",
+        help="Limit output to jobs that were allocated to the specified "
+        + "HOSTS or RANKS provided as a hostlist or idset.",
     )
     parser.add_argument(
         "-o",
         "--format",
         type=str,
+        default="default",
         metavar="FORMAT",
-        help="Specify output format using Python's string format syntax",
+        help="Specify output format using Python's string format syntax "
+        + " or a defined format by name (use 'help' to get a list of names)",
+    )
+    parser.add_argument(
+        "--sort",
+        type=str,
+        default="",
+        metavar="KEY,...",
+        help="Specify sort order",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output jobs in JSON instead of formatted output",
+    )
+    parser.add_argument(
+        "--color",
+        type=str,
+        metavar="WHEN",
+        choices=["never", "always", "auto"],
+        nargs="?",
+        const="always",
+        default="auto",
+        help="Colorize output; WHEN can be 'never', 'always', or 'auto' (default)",
+    )
+    parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="List jobs recursively",
+    )
+    parser.add_argument(
+        "-L",
+        "--level",
+        type=int,
+        metavar="N",
+        default=9999,
+        help="With --recursive, only descend N levels",
+    )
+    parser.add_argument(
+        "--recurse-all",
+        action="store_true",
+        help="With --recursive, attempt to recurse all jobs, "
+        + "not just jobs of current user",
+    )
+    parser.add_argument(
+        "--threads",
+        type=int,
+        metavar="N",
+        help="Set max number of worker threads",
+    )
+    parser.add_argument(
+        "--stats", action="store_true", help="Print job statistics before header"
+    )
+    parser.add_argument(
+        "--stats-only",
+        action="store_true",
+        help="Print job statistics only and exit. Exits with non-zero status "
+        "if there are no active jobs. Allows usage like: "
+        "'while flux jobs --stats-only; do sleep 1; done'",
+    )
+    parser.add_argument(
+        "jobids",
+        metavar="JOBID",
+        type=JobID,
+        nargs="*",
+        help="Limit output to specific Job IDs",
     )
     # Hidden '--from-stdin' option for testing only.
     parser.add_argument("--from-stdin", action="store_true", help=argparse.SUPPRESS)
+    parser.set_defaults(filtered=False)
     return parser.parse_args()
 
 
-class OutputFormat:
-    """
-    Store a parsed version of the program's output format,
-    allowing the fields to iterated without modifiers, building
-    a new format suitable for headers display, etc...
-    """
-
-    #  List of legal format fields and their header names
-    headings = dict(
-        id="JOBID",
-        userid="UID",
-        username="USER",
-        priority="PRI",
-        state="STATE",
-        state_single="STATE",
-        name="NAME",
-        ntasks="NTASKS",
-        nnodes="NNODES",
-        nnodes_hyphen="NNODES",
-        ranks="RANKS",
-        ranks_hyphen="RANKS",
-        t_submit="T_SUBMIT",
-        t_depend="T_DEPEND",
-        t_sched="T_SCHED",
-        t_run="T_RUN",
-        t_cleanup="T_CLEANUP",
-        t_inactive="T_INACTIVE",
-        runtime="RUNTIME",
-        runtime_fsd="RUNTIME",
-        runtime_fsd_hyphen="RUNTIME",
-        runtime_hms="RUNTIME",
+def color_setup(args, job):
+    if args.color == "always" or (args.color == "auto" and sys.stdout.isatty()):
+        if job.result:
+            if job.result == "COMPLETED":
+                sys.stdout.write("\033[01;32m")
+            elif job.result == "FAILED":
+                sys.stdout.write("\033[01;31m")
+            elif job.result == "CANCELED":
+                sys.stdout.write("\033[37m")
+            elif job.result == "TIMEOUT":
+                sys.stdout.write("\033[01;31m")
+            return True
+        if job.uri:
+            sys.stdout.write("\033[01;34m")
+            return True
+    return False
+
+
+def color_reset(color_set):
+    if color_set:
+        sys.stdout.write("\033[0;0m")
+
+
+def is_user_instance(job, args):
+    """Return True if this job should be target of recursive job list"""
+    return (
+        job.uri
+        and job.status_abbrev == "R"
+        and (args.recurse_all or job.userid == os.getuid())
     )
 
-    def __init__(self, fmt):
-        """
-        Parse the input format fmt with string.Formatter.
-        Save off the fields and list of format tokens for later use,
-        (converting None to "" in the process)
-
-        Throws an exception if any format fields do not match the allowed
-        list of headings above.
-        """
-        from string import Formatter
-
-        self.fmt = fmt
-        #  Parse format into list of (string, field, spec, conv) tuples,
-        #   replacing any None values with empty string "" (this makes
-        #   substitution back into a format string in self.header() and
-        #   self.get_format() much simpler below)
-        l = Formatter().parse(fmt)
-        self.format_list = [[s or "" for s in t] for t in l]
-
-        #  Store list of requested fields in self.fields
-        self.fields = [field for (s, field, spec, conv) in self.format_list]
-
-        #  Throw an exception if any requested fields are invalid:
-        for field in self.fields:
-            if field and not field in self.headings:
-                raise ValueError("Unknown format field: " + field)
-
-    def _fmt_tuple(self, s, field, spec, conv):
-        #  If field is empty string or None, then the result of the
-        #   format (besides 's') doesn't make sense, just return 's'
-        if not field:
-            return s
-        #  The prefix of spec and conv are stripped by formatter.parse()
-        #   replace them here if the values are not empty:
-        spec = ":" + spec if spec else ""
-        conv = "!" + conv if conv else ""
-        return "{0}{{{1}{2}{3}}}".format(s, field, conv, spec)
-
-    def header(self):
-        """
-        Return the header row formatted by the user-provided format spec,
-        which will be made "safe" for use with string headings.
-        """
-        import re
-
-        l = []
-        for (s, field, spec, conv) in self.format_list:
-            #  Remove floating point formatting on any spec:
-            spec = re.sub(r"\.\d+[bcdoxXeEfFgGn%]$", "", spec)
-            l.append(self._fmt_tuple(s, field, spec, conv))
-        fmt = "".join(l)
-        return fmt.format(**self.headings)
-
-    def get_format(self):
-        """
-        Return the format string with prepended `0.` if necessary.
-        """
-        try:
-            return self.jobfmt
-        except AttributeError:
-            pass
-
-        l = []
-        for (s, field, spec, conv) in self.format_list:
-            # If field doesn't have `0.` then add it
-            if field and not field.startswith("0."):
-                field = "0." + field
-            l.append(self._fmt_tuple(s, field, spec, conv))
-        self.jobfmt = "".join(l)
-        return self.jobfmt
-
-    def format(self, job):
-        """
-        format JobInfo object with internal format
-        prepend `0.` if necessary to fields to invoke getattr method
-        """
-        return self.get_format().format(job)
-
-
-@flux.util.CLIMain(logger)
+
+def get_jobs_recursive(job, args, fields):
+    jobs = []
+    stats = None
+    try:
+        #  Don't generate an error if we fail to connect to this
+        #   job. This could be because job services aren't up yet,
+        #   (OSError with errno ENOSYS) or this user is not the owner
+        #   of the job. Either way, simply skip descending into the job
+        #
+        handle = flux.Flux(str(job.uri))
+        jobs = fetch_jobs_flux(args, fields, flux_handle=handle)
+        stats = None
+        if args.stats:
+            stats = JobStats(handle).update_sync()
+    except (OSError, FileNotFoundError):
+        pass
+    return (job, jobs, stats)
+
+
+def print_jobs(jobs, args, formatter, path="", level=0):
+    children = []
+    # Array of jobs as dict
+    result = []
+    # Convenience dict for looking up job in result by jobid:
+    job_dict = {}
+
+    def pre(job):
+        job.color_set = color_setup(args, job)
+
+    def post(job):
+        color_reset(job.color_set)
+        if args.recursive and is_user_instance(job, args):
+            children.append(job)
+
+    if args.json:
+        for job in jobs:
+            jdict = job.to_dict()
+            result.append(jdict)
+            job_dict[job.id] = jdict
+            #  Normally, print_items() handles collection of any children
+            #  in this job as a side effect of printing each line of output.
+            #  So, when generating JSON output instead, do that explicitly
+            #  here:
+            if args.recursive and is_user_instance(job, args):
+                children.append(job)
+    else:
+        formatter.print_items(jobs, no_header=True, pre=pre, post=post)
+
+    if not args.recursive or args.level == level:
+        return result
+
+    #  Reset args.jobids since it won't apply recursively:
+    args.jobids = None
+
+    futures = []
+    with concurrent.futures.ThreadPoolExecutor(args.threads) as executor:
+        for job in children:
+            futures.append(
+                executor.submit(get_jobs_recursive, job, args, formatter.fields)
+            )
+
+    if path:
+        path = f"{path}/"
+
+    for future in futures:
+        (job, jobs, stats) = future.result()
+
+        #  If generating JSON, just add this job's children to a job["jobs"]
+        #  array and continue:
+        if args.json:
+            job_dict[job.id]["jobs"] = [x.to_dict() for x in jobs]
+            continue
+
+        #  Otherwise generate a header for this jobs children and print them:
+        thispath = f"{path}{job.id.f58}"
+        print(f"\n{thispath}:")
+        if stats:
+            print(
+                f"{stats.running} running, {stats.successful} completed, "
+                f"{stats.failed} failed, {stats.pending} pending"
+            )
+        print_jobs(jobs, args, formatter, path=thispath, level=level + 1)
+
+    return result
+
+
+@flux.util.CLIMain(LOGGER)
 def main():
+
+    sys.stdout = open(sys.stdout.fileno(), "w", encoding="utf8")
+
     args = parse_args()
 
-    if args.format:
-        fmt = args.format
-    else:
-        fmt = (
-            "{id:>18} {username:<8.8} {name:<10.10} {state:<8.8} "
-            "{ntasks:>6} {nnodes_hyphen:>6} {runtime_fsd_hyphen:>8} "
-            "{ranks_hyphen}"
-        )
+    if args.json and (args.stats or args.stats_only):
+        LOGGER.error("--json incompatible with --stats or --stats-only")
+        sys.exit(1)
+
+    if args.jobids and args.filtered and not args.recursive:
+        LOGGER.warning("Filtering options ignored with jobid list")
+
+    if args.recurse_all:
+        args.recursive = True
+
+    fmt = FluxJobsConfig().load().get_format_string(args.format)
     try:
-        of = OutputFormat(fmt)
-    except ValueError as e:
-        raise ValueError("Error in user format: " + str(e))
+        formatter = JobInfoFormat(fmt)
+    except ValueError as err:
+        raise ValueError("Error in user format: " + str(err))
 
-    jobs = fetch_jobs(args, of.fields)
+    if args.stats or args.stats_only:
+        try:
+            stats = JobStats(flux.Flux(), queue=args.queue).update_sync()
+        except ValueError as err:
+            LOGGER.error("error retrieving job stats: %s", str(err))
+            sys.exit(1)
+
+        print(
+            f"{stats.running} running, {stats.successful} completed, "
+            f"{stats.failed} failed, {stats.pending} pending, "
+            f"{stats.inactive_purged} inactive purged"
+        )
+        if args.stats_only:
+            sys.exit(0 if stats.active else 1)
+
+    if args.sort:
+        formatter.set_sort_keys(args.sort)
 
-    if not args.suppress_header:
-        print(of.header())
+    jobs = fetch_jobs(args, formatter.fields)
+    sformatter = JobInfoFormat(formatter.filter(jobs))
 
-    for job in jobs:
-        print(of.format(job))
+    if not args.no_header:
+        print(sformatter.header())
+
+    result = print_jobs(jobs, args, sformatter)
+    if args.json:
+        # Only emit single JSON object if user asked for one specific job
+        if args.jobids and len(args.jobids) == 1:
+            if result:
+                print(json.dumps(result[0]))
+        else:
+            print(json.dumps({"jobs": result}))
 
 
 if __name__ == "__main__":
diff --git a/src/cmd/flux-jobspec.py b/src/cmd/flux-jobspec.py
deleted file mode 100755
index a9dc585485e2..000000000000
--- a/src/cmd/flux-jobspec.py
+++ /dev/null
@@ -1,266 +0,0 @@
-from __future__ import print_function
-
-import re
-import os
-import sys
-import math
-import logging
-import argparse
-import json
-import collections
-
-try:
-    collectionsAbc = collections.abc
-except AttributeError:
-    collectionsAbc = collections
-
-from flux import util
-import yaml
-
-
-def create_resource(res_type, count, with_child=[]):
-    assert isinstance(
-        with_child, collectionsAbc.Sequence
-    ), "child resource must be a sequence"
-    assert not isinstance(with_child, str), "child resource must not be a string"
-    assert count > 0, "resource count must be > 0"
-
-    res = {"type": res_type, "count": count}
-
-    if len(with_child) > 0:
-        res["with"] = with_child
-    return res
-
-
-def create_slot(label, count, with_child):
-    slot = create_resource("slot", count, with_child)
-    slot["label"] = label
-    return slot
-
-
-def get_environment(l=[{}]):
-    """
-    Return current environment as specified by argument `l`, a list of
-    dicts possibly provided by the --export command line option.
-    If "ALL" is set always export all environment variables, else if
-    "NONE" is set, export an empty environment.
-    """
-    environ = {}
-
-    # Argument from --export is a list of dicts. Combine this list into
-    #  a single "exports" dict:
-    exports = {k: v for e in l for (k, v) in e.items()}
-
-    # If --export option not used then return current environment:
-    if not exports:
-        return dict(os.environ)
-    # If --export=NONE then return empty environment:
-    elif "NONE" in exports:
-        return {}
-    # If --export=ALL,... then start with current environment, possibly
-    # modified by export --export arguments:
-    elif "ALL" in exports:
-        del exports["ALL"]
-        environ = dict(os.environ)
-
-    # Set each env var to provided value, e.g. --export=FOO=bar,
-    #  or current value if not provided, e.g. --export=FOO :
-    for k, v in exports.items():
-        try:
-            environ[k] = v or os.environ[k]
-        except KeyError:
-            logger.error("Variable {} not found in current env".format(k))
-            sys.exit(1)
-    return environ
-
-
-def create_slurm_style_jobspec(
-    command,
-    num_tasks,
-    cores_per_task,
-    num_nodes=0,
-    walltime=None,
-    environ=get_environment(),
-):
-    core = create_resource("core", cores_per_task)
-    if num_nodes > 0:
-        num_slots = int(math.ceil(num_tasks / float(num_nodes)))
-        if num_tasks % num_nodes != 0:
-            logging.warn(
-                "Number of tasks is not an integer multiple of the number of nodes. "
-                "More resources than required will be requested to satisfy your job."
-            )
-            task_count_dict = {"total": num_tasks}
-        else:
-            task_count_dict = {"per_slot": 1}
-        slot = create_slot("task", num_slots, [core])
-        resource_section = create_resource("node", num_nodes, [slot])
-    else:
-        task_count_dict = {"per_slot": 1}
-        slot = create_slot("task", num_tasks, [core])
-        resource_section = slot
-
-    jobspec = {
-        "version": 1,
-        "resources": [resource_section],
-        "tasks": [{"command": command, "slot": "task", "count": task_count_dict}],
-        "attributes": {"system": {"cwd": os.getcwd(), "environment": environ}},
-    }
-    if walltime:
-        jobspec["attributes"]["system"]["duration"] = walltime
-
-    return jobspec
-
-
-def validate_slurm_args(args):
-    if args.nodes > args.ntasks:
-        raise ValueError("Number of nodes greater than the number of tasks")
-
-    if (
-        args.time
-        and re.match(r"^(\d+-)?\d+:\d+:\d+$", args.time) is None
-        and re.match(r"^\d+(:\d+)?$", args.time) is None
-    ):
-        raise ValueError(
-            "invalid time limit string format. "
-            "Acceptable formats include minutes[:seconds], [days-]hours:minutes:seconds"
-        )
-
-    # TODO: is there any validation of the stdout redirection path that we can do?
-    # IDEA: print a warning if the file already exists or if the parent dir doesn't exist
-
-
-#  Convert slurm walltime string to floating point seconds
-def slurm_walltime_to_duration(time):
-    if not time:
-        return None
-    p1 = re.compile(
-        r"^((?P<days>\d+)-)?"
-        r"(?P<hours>\d+):"
-        r"(?P<minutes>\d+):"
-        r"(?P<seconds>\d+)$"
-    )
-    p2 = re.compile(r"^(?P<minutes>\d+)(:(?P<seconds>\d+))?$")
-
-    t = 0.0
-    m = p2.search(time) or p1.search(time)
-    if m is None:
-        return None
-    vals = {k: float(v) for k, v in m.groupdict().items() if v is not None}
-    if "days" in vals:
-        t = t + vals["days"] * 60 * 60 * 24
-    if "hours" in vals:
-        t = t + vals["hours"] * 60 * 60
-    if "minutes" in vals:
-        t = t + vals["minutes"] * 60
-    if "seconds" in vals:
-        t = t + vals["seconds"]
-    if t == 0.0:
-        return None
-    return t
-
-
-def slurm_jobspec(args):
-    if args.ntasks is None:
-        args.ntasks = max(args.nodes, 1)
-
-    try:
-        validate_slurm_args(args)
-    except ValueError as e:
-        logger.error(str(e))
-        sys.exit(1)
-    t = slurm_walltime_to_duration(args.time)
-    environ = get_environment(args.export)
-    return create_slurm_style_jobspec(
-        args.command, args.ntasks, args.cpus_per_task, args.nodes, t, environ
-    )
-
-
-def kv_list_split(s):
-    """
-    Split a key/value list with optional values, e.g. 'FOO,BAR=baz,...'
-    and return a dict.
-    """
-    return dict((a, b) for a, _, b in [x.partition("=") for x in s.split(",")])
-
-
-def get_slurm_common_parser():
-    """
-    Shared arguments amongst srun and sbatch.
-    Used src/srun/libsrun/opt.c and src/sbatch/opt.c of the
-    [SLURM repository](https://github.com/SchedMD/slurm.git) as reference
-    """
-    slurm_parser = argparse.ArgumentParser(add_help=False)
-    slurm_parser.add_argument(
-        "-N",
-        "--nodes",
-        help="Set the number of requested nodes to N",
-        type=int,
-        metavar="N",
-        default=0,
-    )
-    slurm_parser.add_argument(
-        "-n", "--ntasks", help="Set the number of tasks to N", type=int, metavar="N"
-    )
-    slurm_parser.add_argument(
-        "-c",
-        "--cpus-per-task",
-        help="Set number of cores per task to N",
-        type=int,
-        metavar="N",
-        default=1,
-    )
-    slurm_parser.add_argument(
-        "-t",
-        "--time",
-        help="time limit. Acceptable formats include minutes[:seconds], "
-        "[days-]hours:minutes:seconds",
-    )
-    slurm_parser.add_argument("-o", "--output", help="location of stdout redirection")
-    slurm_parser.add_argument(
-        "--export",
-        metavar="[ALL|NONE|VARS]",
-        action="append",
-        type=kv_list_split,
-        default=[{}],
-        help="List or specify environment variables to export",
-    )
-    slurm_parser.add_argument("command", nargs=argparse.REMAINDER)
-    return slurm_parser
-
-
-logger = logging.getLogger("flux-jobspec")
-
-
-@util.CLIMain(logger)
-def main():
-    parser = argparse.ArgumentParser(prog="flux-jobspec")
-    parser.add_argument("--format", choices=["json", "yaml"], default="json")
-
-    subparsers = parser.add_subparsers()
-    slurm_parser = get_slurm_common_parser()
-    srun_parser = subparsers.add_parser(
-        "srun",
-        parents=[slurm_parser],
-        help="subcommand for SLURM-style CLI arguments",
-        formatter_class=util.help_formatter(),
-    )
-    srun_parser.set_defaults(func=slurm_jobspec)
-
-    args = parser.parse_args()
-
-    if len(args.command) == 0:
-        parser.error("command is required")
-        sys.exit(1)
-
-    jobspec = args.func(args)
-
-    if args.format == "yaml":
-        out = yaml.dump(jobspec)
-    else:
-        out = json.dumps(jobspec)
-    print(out)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/src/cmd/flux-jobtap.py b/src/cmd/flux-jobtap.py
new file mode 100755
index 000000000000..e5541e97d7a4
--- /dev/null
+++ b/src/cmd/flux-jobtap.py
@@ -0,0 +1,127 @@
+#!/bin/false
+##############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+import sys
+
+import flux
+from flux.util import TreedictAction
+
+
+def jobtap_remove(args):
+    """Remove jobtap plugin matching name"""
+    try:
+        flux.Flux().rpc("job-manager.jobtap", {"remove": args.plugin}).get()
+    except FileNotFoundError:
+        LOGGER.error("%s not found", args.plugin)
+        sys.exit(1)
+
+
+def jobtap_load(args):
+    """Load a jobtap plugin into the job manager"""
+
+    req = {"load": args.plugin}
+    if args.conf:
+        req["conf"] = args.conf
+    if args.remove:
+        req["remove"] = args.remove
+
+    try:
+        flux.Flux().rpc("job-manager.jobtap", req).get()
+    except FileNotFoundError:
+        LOGGER.error(
+            "%s not found",
+            args.plugin,
+        )
+        sys.exit(1)
+
+
+def jobtap_list(args):
+    """List currently loaded jobtap plugins"""
+    resp = flux.Flux().rpc("job-manager.jobtap", {"query_only": True}).get()
+    for name in resp["plugins"]:
+        if not name.startswith(".") or args.all:
+            print(name)
+
+
+def jobtap_query(args):
+    """Query extended information about and from a specific plugin"""
+    service = "job-manager.jobtap-query"
+    print(flux.Flux().rpc(service, {"name": args.plugin}).get_str())
+
+
+LOGGER = logging.getLogger("flux-jobtap")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    parser = argparse.ArgumentParser(prog="flux-jobtap")
+    subparsers = parser.add_subparsers(
+        title="subcommands", description="", dest="subcommand"
+    )
+    subparsers.required = True
+
+    load_parser = subparsers.add_parser(
+        "load", formatter_class=flux.util.help_formatter()
+    )
+    load_parser.add_argument(
+        "-r",
+        "--remove",
+        metavar="NAME",
+        help="Remove plugin NAME before loading new plugin. "
+        + "NAME may optionally be a shell glob pattern which removes all "
+        + 'matching plugins. ("all" is a synonym for "*")',
+    )
+    load_parser.add_argument("plugin", help="Plugin path or builtin name")
+    load_parser.add_argument(
+        "conf",
+        help="List of key=value config keys to set as plugin configuration",
+        action=TreedictAction,
+        nargs=argparse.REMAINDER,
+    )
+    load_parser.set_defaults(func=jobtap_load)
+
+    remove_parser = subparsers.add_parser(
+        "remove", formatter_class=flux.util.help_formatter()
+    )
+    remove_parser.add_argument(
+        "plugin",
+        help="Plugin name or pattern to remove. "
+        + '"all" may be used to remove all loaded plugins',
+    )
+    remove_parser.set_defaults(func=jobtap_remove)
+
+    list_parser = subparsers.add_parser(
+        "list", formatter_class=flux.util.help_formatter()
+    )
+    list_parser.add_argument(
+        "-a",
+        "--all",
+        help="Do not ignore job-manager builtin plugins",
+        action="store_true",
+    )
+    list_parser.set_defaults(func=jobtap_list)
+
+    query_parser = subparsers.add_parser(
+        "query", formatter_class=flux.util.help_formatter()
+    )
+    query_parser.add_argument("plugin", help="Plugin name to query")
+    query_parser.set_defaults(func=jobtap_query)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-keygen.c b/src/cmd/flux-keygen.c
index dd0f413f1245..883911d5ba81 100644
--- a/src/cmd/flux-keygen.c
+++ b/src/cmd/flux-keygen.c
@@ -11,68 +11,142 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <getopt.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <time.h>
+#include <unistd.h>
 #include <flux/core.h>
+#include <flux/optparse.h>
+#include <zmq.h>
 
+#include "src/common/libzmqutil/cert.h"
 #include "src/common/libutil/log.h"
-#include "src/common/libutil/zsecurity.h"
 
-
-#define OPTIONS "hfpd:"
-static const struct option longopts[] = {
-    {"help",       no_argument,        0, 'h'},
-    {"force",      no_argument,        0, 'f'},
-    {"plain",      no_argument,        0, 'p'},
-    {"secdir",     required_argument,  0, 'd'},
-    { 0, 0, 0, 0 },
+static struct optparse_option opts[] = {
+    { .name = "name", .key = 'n', .has_arg = 1, .arginfo = "NAME",
+      .usage = "Set certificate name (default: hostname)", },
+    { .name = "meta", .has_arg = 1, .arginfo = "KEYVALS",
+      .flags = OPTPARSE_OPT_AUTOSPLIT,
+      .usage = "Add/update comma-separated key=value metadata", },
+    OPTPARSE_TABLE_END,
 };
 
-void usage (void)
+static char * ctime_iso8601_now (char *buf, size_t sz)
+{
+    struct tm tm;
+    time_t now = time (NULL);
+
+    memset (buf, 0, sz);
+
+    if (!localtime_r (&now, &tm))
+        return (NULL);
+    strftime (buf, sz, "%Y-%m-%dT%T", &tm);
+
+    return (buf);
+}
+
+/* Mimic zcert_set() behavior of doing nothing (silently) if metadata
+ * key already has a value.
+ */
+static void meta_set (struct cert *cert, const char *key, const char *val)
+{
+    if (!cert_meta_get (cert, key)
+        && cert_meta_set (cert, key, val) < 0)
+        log_err_exit ("error setting certificate metadata %s=%s", key, val);
+}
+
+static void meta_set_fmt (struct cert *cert,
+                          const char *key,
+                          const char *fmt,
+                          ...)
 {
-    fprintf (stderr,
-"Usage: flux-keygen [--secdir DIR] [--force] [--plain]\n"
-);
-    exit (1);
+    char buf[1024];
+    va_list ap;
+
+    va_start (ap, fmt);
+    vsnprintf (buf, sizeof (buf), fmt, ap);
+    va_end (ap);
+    meta_set (cert, key, buf);
 }
 
 int main (int argc, char *argv[])
 {
-    int ch;
-    zsecurity_t *sec;
-    int typemask = ZSECURITY_TYPE_CURVE | ZSECURITY_VERBOSE;
-    const char *secdir = getenv ("FLUX_SEC_DIRECTORY");
+    const char *usage_msg = "[OPTIONS] [PATH]";
+    optparse_t *p;
+    int optindex;
+    struct cert *cert;
+    const char *name = NULL;
+    char now[64];
+    char hostname[64];
+    char *path = NULL;
 
     log_init ("flux-keygen");
+    if (!(p = optparse_create ("flux-keygen"))
+        || optparse_add_option_table (p, opts) != OPTPARSE_SUCCESS
+        || optparse_set (p, OPTPARSE_USAGE, usage_msg) != OPTPARSE_SUCCESS)
+        log_err_exit ("error setting up option parsing");
+    if ((optindex = optparse_parse_args (p, argc, argv)) < 0)
+        exit (1);
+    if (optindex < argc)
+        path = argv[optindex++];
+    if (optindex < argc) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!path)
+        log_msg ("WARNING: add PATH argument to save generated certificate");
+
+    if (!(cert = cert_create ()))
+        log_err_exit ("error creating CURVE certificate");
+    if (gethostname (hostname, sizeof (hostname)) < 0)
+        log_err_exit ("gethostname");
+    if (ctime_iso8601_now (now, sizeof (now)) == NULL)
+        log_err_exit ("localtime");
 
-    while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
-        switch (ch) {
-            case 'h': /* --help */
-                usage ();
-                break;
-            case 'f': /* --force */
-                typemask |= ZSECURITY_KEYGEN_FORCE;
-                break;
-            case 'p': /* --plain */
-                typemask |= ZSECURITY_TYPE_PLAIN;
-                typemask &= ~ZSECURITY_TYPE_CURVE;
-                break;
-            case 'd': /* --secdir */
-                secdir = optarg;
-                break;
-            default:
-                usage ();
-                break;
+    if ((name = optparse_get_str (p, "name", NULL)))
+        meta_set (cert, "name", name);
+    if (optparse_hasopt (p, "meta")) {
+        const char *arg;
+
+        optparse_getopt_iterator_reset (p, "meta");
+        while ((arg = optparse_getopt_next (p, "meta"))) {
+            char *key;
+            char *val;
+            if (!(key = strdup (arg)))
+                log_msg_exit ("out of memory");
+            if ((val = strchr (key, '=')))
+                *val++ = '\0';
+            meta_set (cert, key, val ? val : "");
+            free (key);
         }
     }
-    if (optind < argc)
-        usage ();
+    meta_set (cert, "name", hostname); // used in overlay logging
+    meta_set (cert, "keygen.hostname", hostname);
+    meta_set (cert, "keygen.time", now);
+    meta_set_fmt (cert, "keygen.userid", "%d", getuid ());
+    meta_set (cert, "keygen.flux-core-version", FLUX_CORE_VERSION_STRING);
+    meta_set_fmt (cert,
+                  "keygen.zmq-version",
+                  "%d.%d.%d",
+                  ZMQ_VERSION_MAJOR,
+                  ZMQ_VERSION_MINOR,
+                  ZMQ_VERSION_PATCH);
 
-     if (!(sec = zsecurity_create (typemask, secdir)))
-        log_err_exit ("zsecurity_create");
-    if (zsecurity_keygen (sec) < 0)
-        log_msg_exit ("%s", zsecurity_errstr (sec));
-    zsecurity_destroy (sec);
+    if (path) {
+        int fd;
+        FILE *f;
+        if ((fd = open (path, O_WRONLY | O_CREAT | O_TRUNC, 0600)) < 0
+            || !(f = fdopen (fd, "w")))
+            log_err_exit ("open %s", path);
+        if (cert_write (cert, f) < 0)
+            log_err_exit ("write %s", path);
+        if (fclose (f) < 0)
+            log_err_exit ("close %s", path);
+    }
+    cert_destroy (cert);
 
+    optparse_destroy (p);
     log_fini ();
 
     return 0;
diff --git a/src/cmd/flux-kvs.c b/src/cmd/flux-kvs.c
index 64f4690e2146..1235ded9abab 100644
--- a/src/cmd/flux-kvs.c
+++ b/src/cmd/flux-kvs.c
@@ -16,14 +16,22 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <jansson.h>
-#include <czmq.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <ctype.h>
+#include <sys/ioctl.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/read_all.h"
 #include "src/common/libkvs/treeobj.h"
 #include "src/common/libeventlog/eventlog.h"
+#include "src/common/libeventlog/formatter.h"
+#include "ccan/str/str.h"
 
 int cmd_namespace (optparse_t *p, int argc, char **argv);
 int cmd_get (optparse_t *p, int argc, char **argv);
@@ -108,6 +116,9 @@ static struct optparse_option put_opts[] =  {
     { .name = "treeobj-root", .key = 'O', .has_arg = 0,
       .usage = "Output resulting RFC11 root containing puts",
     },
+    { .name = "blobref", .key = 'b', .has_arg = 0,
+      .usage = "Output blobref of root containing puts",
+    },
     { .name = "sequence", .key = 's', .has_arg = 0,
       .usage = "Output root sequence of root containing puts",
     },
@@ -123,6 +134,9 @@ static struct optparse_option put_opts[] =  {
     { .name = "append", .key = 'A', .has_arg = 0,
       .usage = "Append value(s) to key instead of overwriting",
     },
+    { .name = "sync", .key = 'S', .has_arg = 0,
+      .usage = "Flushes content and checkpoints after completing put",
+    },
     OPTPARSE_TABLE_END
 };
 
@@ -181,6 +195,9 @@ static struct optparse_option unlink_opts[] =  {
     { .name = "treeobj-root", .key = 'O', .has_arg = 0,
       .usage = "Output resulting RFC11 root containing unlinks",
     },
+    { .name = "blobref", .key = 'b', .has_arg = 0,
+      .usage = "Output blobref of root containing unlinks",
+    },
     { .name = "sequence", .key = 's', .has_arg = 0,
       .usage = "Output root sequence of root containing unlinks",
     },
@@ -203,6 +220,9 @@ static struct optparse_option getroot_opts[] =  {
     { .name = "owner", .key = 'o', .has_arg = 0,
       .usage = "Show owner",
     },
+    { .name = "blobref", .key = 'b', .has_arg = 0,
+      .usage = "Show blobref",
+    },
     OPTPARSE_TABLE_END
 };
 
@@ -226,6 +246,9 @@ static struct optparse_option link_opts[] =  {
     { .name = "treeobj-root", .key = 'O', .has_arg = 0,
       .usage = "Output resulting RFC11 root containing link",
     },
+    { .name = "blobref", .key = 'b', .has_arg = 0,
+      .usage = "Output blobref of root containing link",
+    },
     { .name = "sequence", .key = 's', .has_arg = 0,
       .usage = "Output root sequence of root containing link",
     },
@@ -239,6 +262,9 @@ static struct optparse_option mkdir_opts[] =  {
     { .name = "treeobj-root", .key = 'O', .has_arg = 0,
       .usage = "Output resulting RFC11 root containing new directory",
     },
+    { .name = "blobref", .key = 'b', .has_arg = 0,
+      .usage = "Output blobref of root containing new directory",
+    },
     { .name = "sequence", .key = 's', .has_arg = 0,
       .usage = "Output root sequence of root containing new directory",
     },
@@ -257,7 +283,7 @@ static struct optparse_subcommand subcommands[] = {
       NULL,
       "Perform KVS namespace operations",
       cmd_namespace,
-      0,
+      OPTPARSE_SUBCMD_SKIP_OPTS,
       NULL,
     },
     { "get",
@@ -269,7 +295,7 @@ static struct optparse_subcommand subcommands[] = {
       get_opts
     },
     { "put",
-      "[-N ns] [-O|-s] [-r|-t] [-n] [-A] key=value [key=value...]",
+      "[-N ns] [-O|-b|-s] [-r|-t] [-n] [-A] [-S] key=value [key=value...]",
       "Store value under key",
       cmd_put,
       0,
@@ -290,14 +316,14 @@ static struct optparse_subcommand subcommands[] = {
       ls_opts
     },
     { "unlink",
-      "[-N ns] [-O|-s] [-R] [-f] key [key...]",
+      "[-N ns] [-O|-b|-s] [-R] [-f] key [key...]",
       "Remove key",
       cmd_unlink,
       0,
       unlink_opts
     },
     { "link",
-      "[-N ns] [-T ns] [-O|-s] target linkname",
+      "[-N ns] [-T ns] [-O|-b|-s] target linkname",
       "Create a new name for target",
       cmd_link,
       0,
@@ -311,7 +337,7 @@ static struct optparse_subcommand subcommands[] = {
       readlink_opts
     },
     { "mkdir",
-      "[-N ns] [-O|-s] key [key...]",
+      "[-N ns] [-O|-b|-s] key [key...]",
       "Create a directory",
       cmd_mkdir,
       0,
@@ -340,7 +366,7 @@ static struct optparse_subcommand subcommands[] = {
     },
     { "version",
       "[-N ns]",
-      "Display curent KVS version",
+      "Display current KVS version",
       cmd_version,
       0,
       namespace_opt
@@ -353,7 +379,7 @@ static struct optparse_subcommand subcommands[] = {
       namespace_opt
     },
     { "getroot",
-      "[-N ns] [-s|-o]",
+      "[-N ns] [-s|-o|-b]",
       "Get KVS root treeobj",
       cmd_getroot,
       0,
@@ -363,7 +389,7 @@ static struct optparse_subcommand subcommands[] = {
       NULL,
       "Manipulate a KVS eventlog",
       cmd_eventlog,
-      0,
+      OPTPARSE_SUBCMD_SKIP_OPTS,
       NULL,
     },
     OPTPARSE_SUBCMD_END
@@ -386,7 +412,6 @@ int usage (optparse_t *p, struct optparse_option *o, const char *optarg)
 
 int main (int argc, char *argv[])
 {
-    flux_t *h;
     char *cmdusage = "[OPTIONS] COMMAND ARGS";
     optparse_t *p;
     int optindex;
@@ -420,15 +445,9 @@ int main (int argc, char *argv[])
         exit (1);
     }
 
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    optparse_set_data (p, "flux_handle", h);
-
     if ((exitval = optparse_run_subcommand (p, argc, argv)) < 0)
         exit (1);
 
-    flux_close (h);
     optparse_destroy (p);
     log_fini ();
     return (exitval);
@@ -436,11 +455,12 @@ int main (int argc, char *argv[])
 
 int cmd_namespace_create (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     flux_future_t *f;
     int optindex, i;
     uint32_t owner = FLUX_USERID_UNKNOWN;
     const char *str;
+    const char *rootref;
 
     optindex = optparse_option_index (p);
     if ((optindex - argc) == 0) {
@@ -455,20 +475,31 @@ int cmd_namespace_create (optparse_t *p, int argc, char **argv)
             log_msg_exit ("--owner requires an unsigned integer argument");
     }
 
+    rootref = optparse_get_str (p, "rootref", NULL);
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     for (i = optindex; i < argc; i++) {
         const char *name = argv[i];
         int flags = 0;
-        if (!(f = flux_kvs_namespace_create (h, name, owner, flags))
-            || flux_future_get (f, NULL) < 0)
+        if (rootref)
+            f = flux_kvs_namespace_create_with (h, name, rootref, owner, flags);
+        else
+            f = flux_kvs_namespace_create (h, name, owner, flags);
+        if (!f)
             log_err_exit ("%s", name);
+        if (flux_future_get (f, NULL) < 0)
+            log_msg_exit ("%s: %s", name, future_strerror (f, errno));
         flux_future_destroy (f);
     }
+    flux_close (h);
     return (0);
 }
 
 int cmd_namespace_remove (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     flux_future_t *f;
     int optindex, i;
 
@@ -477,6 +508,10 @@ int cmd_namespace_remove (optparse_t *p, int argc, char **argv)
         optparse_print_usage (p);
         exit (1);
     }
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     for (i = optindex; i < argc; i++) {
         const char *name = argv[i];
         if (!(f = flux_kvs_namespace_remove (h, name))
@@ -484,12 +519,13 @@ int cmd_namespace_remove (optparse_t *p, int argc, char **argv)
             log_err_exit ("%s", name);
         flux_future_destroy (f);
     }
+    flux_close (h);
     return (0);
 }
 
 int cmd_namespace_list (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     int optindex;
     flux_future_t *f;
     json_t *array;
@@ -501,6 +537,9 @@ int cmd_namespace_list (optparse_t *p, int argc, char **argv)
         exit (1);
     }
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     if (!(f = flux_rpc (h, "kvs.namespace-list", NULL, FLUX_NODEID_ANY, 0)))
         log_err_exit ("flux_rpc");
     if (flux_rpc_get_unpack (f, "{ s:o }", "namespaces", &array) < 0)
@@ -519,7 +558,8 @@ int cmd_namespace_list (optparse_t *p, int argc, char **argv)
         if (!(o = json_array_get (array, i)))
             log_err_exit ("json_array_get");
 
-        if (json_unpack (o, "{ s:s s:i s:i }",
+        if (json_unpack (o,
+                         "{ s:s s:i s:i }",
                          "namespace", &ns,
                          "owner", &owner,
                          "flags", &flags) < 0)
@@ -529,6 +569,7 @@ int cmd_namespace_list (optparse_t *p, int argc, char **argv)
     }
 
     flux_future_destroy (f);
+    flux_close (h);
     return (0);
 }
 
@@ -536,12 +577,15 @@ static struct optparse_option namespace_create_opts[] =  {
     { .name = "owner", .key = 'o', .has_arg = 1,
       .usage = "Specify alternate namespace owner via userid",
     },
+    { .name = "rootref", .key = 'r', .has_arg = 1,
+      .usage = "Initialize namespace with specific root reference",
+    },
     OPTPARSE_TABLE_END
 };
 
 static struct optparse_subcommand namespace_subcommands[] = {
     { "create",
-      "name [name...]",
+      "[-o owner] [-r rootref] name [name...]",
       "Create a KVS namespace",
       cmd_namespace_create,
       0,
@@ -581,7 +625,8 @@ int cmd_namespace (optparse_t *p, int argc, char **argv)
     return (0);
 }
 
-static void kv_printf (const char *key, int maxcol, const char *fmt, ...)
+static void __attribute__ ((format (printf, 3, 4)))
+kv_printf (const char *key, int maxcol, const char *fmt, ...)
 {
     va_list ap;
     int rc;
@@ -598,9 +643,11 @@ static void kv_printf (const char *key, int maxcol, const char *fmt, ...)
     if (rc < 0)
         log_err_exit ("%s", __FUNCTION__);
 
-    if (asprintf (&kv, "%s%s%s", key ? key : "",
-                                 key ? " = " : "",
-                                 val) < 0)
+    if (asprintf (&kv,
+                  "%s%s%s",
+                  key ? key : "",
+                  key ? " = " : "",
+                  val) < 0)
         log_err_exit ("%s", __FUNCTION__);
 
     /* There will be no truncation of output if maxcol = 0.
@@ -627,8 +674,9 @@ static void kv_printf (const char *key, int maxcol, const char *fmt, ...)
             }
         }
     }
-    printf ("%s%s\n", kv,
-                      overflow ? "..." : "");
+    printf ("%s%s\n",
+            kv,
+            overflow ? "..." : "");
 
     free (val);
     free (kv);
@@ -647,8 +695,9 @@ void lookup_continuation (flux_future_t *f, void *arg)
     struct lookup_ctx *ctx = arg;
     const char *key = flux_kvs_lookup_get_key (f);
 
-    if (optparse_hasopt (ctx->p, "watch") && flux_rpc_get (f, NULL) < 0
-                                          && errno == ENODATA) {
+    if (optparse_hasopt (ctx->p, "watch")
+        && flux_rpc_get (f, NULL) < 0
+        && errno == ENODATA) {
         flux_future_destroy (f);
         return; // EOF
     }
@@ -663,7 +712,7 @@ void lookup_continuation (flux_future_t *f, void *arg)
     }
     else if (optparse_hasopt (ctx->p, "raw")) {
         const void *data;
-        int len;
+        size_t len;
         if (flux_kvs_lookup_get_raw (f, &data, &len) < 0)
             log_err_exit ("%s", key);
         if (optparse_hasopt (ctx->p, "label"))
@@ -729,7 +778,7 @@ void cmd_get_one (flux_t *h, const char *key, struct lookup_ctx *ctx)
 
 int cmd_get (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     int optindex, i;
     struct lookup_ctx ctx;
 
@@ -743,6 +792,9 @@ int cmd_get (optparse_t *p, int argc, char **argv)
     ctx.maxcount = optparse_get_int (p, "count", 0);
     ctx.ns = optparse_get_str (p, "namespace", NULL);
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     for (i = optindex; i < argc; i++)
         cmd_get_one (h, argv[i], &ctx);
     /* Unless --watch is specified, cmd_get_one() starts the reactor and
@@ -755,6 +807,7 @@ int cmd_get (optparse_t *p, int argc, char **argv)
             log_err_exit ("flux_reactor_run");
     }
 
+    flux_close (h);
     return (0);
 }
 
@@ -766,6 +819,12 @@ void commit_finish (flux_future_t *f, optparse_t *p)
             log_err_exit ("flux_kvs_commit_get_treeobj");
         printf ("%s\n", treeobj);
     }
+    else if (optparse_hasopt (p, "blobref")) {
+        const char *blobref = NULL;
+        if (flux_kvs_commit_get_rootref (f, &blobref) < 0)
+            log_err_exit ("flux_kvs_commit_get_rootref");
+        printf ("%s\n", blobref);
+    }
     else if (optparse_hasopt (p, "sequence")) {
         int sequence;
         if (flux_kvs_commit_get_sequence (f, &sequence) < 0)
@@ -780,7 +839,7 @@ void commit_finish (flux_future_t *f, optparse_t *p)
 
 int cmd_put (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     const char *ns = NULL;
     int optindex, i;
     flux_future_t *f;
@@ -797,6 +856,13 @@ int cmd_put (optparse_t *p, int argc, char **argv)
 
     if (optparse_hasopt (p, "no-merge"))
         commit_flags |= FLUX_KVS_NO_MERGE;
+
+    if (optparse_hasopt (p, "sync"))
+        commit_flags |= FLUX_KVS_SYNC;
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     if (!(txn = flux_kvs_txn_create ()))
         log_err_exit ("flux_kvs_txn_create");
     for (i = optindex; i < argc; i++) {
@@ -810,7 +876,7 @@ int cmd_put (optparse_t *p, int argc, char **argv)
             int len;
             uint8_t *buf = NULL;
 
-            if (!strcmp (val, "-")) { // special handling for "--treeobj key=-"
+            if (streq (val, "-")) { // special handling for "--treeobj key=-"
                 if ((len = read_all (STDIN_FILENO, (void **)&buf)) < 0)
                     log_err_exit ("stdin");
                 val = (char *)buf;
@@ -826,7 +892,7 @@ int cmd_put (optparse_t *p, int argc, char **argv)
             if (optparse_hasopt (p, "append"))
                 put_flags |= FLUX_KVS_APPEND;
 
-            if (!strcmp (val, "-")) { // special handling for "--raw key=-"
+            if (streq (val, "-")) { // special handling for "--raw key=-"
                 if ((len = read_all (STDIN_FILENO, (void **)&buf)) < 0)
                     log_err_exit ("stdin");
                 val = (char *)buf;
@@ -850,6 +916,7 @@ int cmd_put (optparse_t *p, int argc, char **argv)
     commit_finish (f, p);
     flux_future_destroy (f);
     flux_kvs_txn_destroy (txn);
+    flux_close (h);
     return (0);
 }
 
@@ -920,7 +987,7 @@ static int unlink_safety_check (flux_t *h, const char *key, const char *ns,
 
 int cmd_unlink (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     const char *ns = NULL;
     int optindex, i;
     flux_future_t *f;
@@ -936,6 +1003,9 @@ int cmd_unlink (optparse_t *p, int argc, char **argv)
     Ropt = optparse_hasopt (p, "recursive");
     fopt = optparse_hasopt (p, "force");
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     if (!(txn = flux_kvs_txn_create ()))
         log_err_exit ("flux_kvs_txn_create");
     for (i = optindex; i < argc; i++) {
@@ -952,12 +1022,13 @@ int cmd_unlink (optparse_t *p, int argc, char **argv)
     commit_finish (f, p);
     flux_future_destroy (f);
     flux_kvs_txn_destroy (txn);
+    flux_close (h);
     return (0);
 }
 
 int cmd_link (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     const char *linkns = NULL;
     const char *targetns = NULL;
     const char *target, *linkname;
@@ -978,6 +1049,9 @@ int cmd_link (optparse_t *p, int argc, char **argv)
     target = argv[optindex];
     linkname = argv[optindex + 1];
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     if (!(txn = flux_kvs_txn_create ()))
         log_err_exit ("flux_kvs_txn_create");
     if (flux_kvs_txn_symlink (txn, 0, linkname, targetns, target) < 0)
@@ -987,12 +1061,13 @@ int cmd_link (optparse_t *p, int argc, char **argv)
     commit_finish (f, p);
     flux_future_destroy (f);
     flux_kvs_txn_destroy (txn);
+    flux_close (h);
     return (0);
 }
 
 int cmd_readlink (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     const char *ns = NULL;
     int optindex, i;
     flux_future_t *f;
@@ -1008,6 +1083,9 @@ int cmd_readlink (optparse_t *p, int argc, char **argv)
     ns_only = optparse_hasopt (p, "namespace-only");
     key_only = optparse_hasopt (p, "key-only");
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     for (i = optindex; i < argc; i++) {
         const char *linkns = NULL;
         const char *target = NULL;
@@ -1034,12 +1112,13 @@ int cmd_readlink (optparse_t *p, int argc, char **argv)
         }
         flux_future_destroy (f);
     }
+    flux_close (h);
     return (0);
 }
 
 int cmd_mkdir (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     const char *ns;
     int optindex, i;
     flux_kvs_txn_t *txn;
@@ -1052,6 +1131,9 @@ int cmd_mkdir (optparse_t *p, int argc, char **argv)
     }
     ns = optparse_get_str (p, "namespace", NULL);
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     if (!(txn = flux_kvs_txn_create ()))
         log_err_exit ("flux_kvs_txn_create");
     for (i = optindex; i < argc; i++) {
@@ -1063,6 +1145,7 @@ int cmd_mkdir (optparse_t *p, int argc, char **argv)
     commit_finish (f, p);
     flux_future_destroy (f);
     flux_kvs_txn_destroy (txn);
+    flux_close (h);
     return (0);
 }
 
@@ -1072,13 +1155,14 @@ int cmd_version (optparse_t *p, int argc, char **argv)
     const char *ns = NULL;
     int vers;
 
-    h = (flux_t *)optparse_get_data (p, "flux_handle");
-
     ns = optparse_get_str (p, "namespace", NULL);
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
     if (flux_kvs_get_version (h, ns, &vers) < 0)
         log_err_exit ("flux_kvs_get_version");
     printf ("%d\n", vers);
+    flux_close (h);
     return (0);
 }
 
@@ -1089,8 +1173,6 @@ int cmd_wait (optparse_t *p, int argc, char **argv)
     int vers;
     int optindex;
 
-    h = (flux_t *)optparse_get_data (p, "flux_handle");
-
     optindex = optparse_option_index (p);
 
     if ((optindex - argc) == 0) {
@@ -1103,8 +1185,12 @@ int cmd_wait (optparse_t *p, int argc, char **argv)
     ns = optparse_get_str (p, "namespace", NULL);
 
     vers = strtoul (argv[optindex], NULL, 10);
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
     if (flux_kvs_wait_version (h, ns, vers) < 0)
         log_err_exit ("flux_kvs_wait_version");
+    flux_close (h);
     return (0);
 }
 
@@ -1112,7 +1198,8 @@ int cmd_dropcache (optparse_t *p, int argc, char **argv)
 {
     flux_t *h;
 
-    h = (flux_t *)optparse_get_data (p, "flux_handle");
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
 
     if (optparse_hasopt (p, "all")) {
         flux_msg_t *msg = flux_event_encode ("kvs.dropcache", NULL);
@@ -1124,6 +1211,7 @@ int cmd_dropcache (optparse_t *p, int argc, char **argv)
         if (flux_kvs_dropcache (h) < 0)
             log_err_exit ("flux_kvs_dropcache");
     }
+    flux_close (h);
     return (0);
 }
 
@@ -1141,9 +1229,9 @@ static char *process_key (const char *key)
 static void dump_kvs_val (const char *key, int maxcol, const char *value)
 {
     if (!value)
-        kv_printf (key, maxcol, "");
+        kv_printf (key, maxcol, "%s", "");
     else
-        kv_printf (key, maxcol, value);
+        kv_printf (key, maxcol, "%s", value);
 }
 
 static void dump_kvs_dir (const flux_kvsdir_t *dir, int maxcol,
@@ -1203,7 +1291,7 @@ static void dump_kvs_dir (const flux_kvsdir_t *dir, int maxcol,
             if (!dopt) {
                 const char *value;
                 const void *buf;
-                int len;
+                size_t len;
                 if (rootref) {
                     if (!(f = flux_kvs_lookupat (h, 0, key, rootref)))
                         log_err_exit ("%s", key);
@@ -1215,7 +1303,7 @@ static void dump_kvs_dir (const flux_kvsdir_t *dir, int maxcol,
                 if (flux_kvs_lookup_get (f, &value) == 0) // null terminated
                     dump_kvs_val (key, maxcol, value);
                 else if (flux_kvs_lookup_get_raw  (f, &buf, &len) == 0)
-                    kv_printf (key, maxcol, "%.*s", len, buf);
+                    kv_printf (key, maxcol, "%.*s", (int)len, (char *)buf);
                 else
                     log_err_exit ("%s", key);
                 flux_future_destroy (f);
@@ -1230,7 +1318,7 @@ static void dump_kvs_dir (const flux_kvsdir_t *dir, int maxcol,
 
 int cmd_dir (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     int maxcol = get_window_width (p, STDOUT_FILENO);
     const char *ns = NULL;
     bool Ropt;
@@ -1251,6 +1339,9 @@ int cmd_dir (optparse_t *p, int argc, char **argv)
     else
         log_msg_exit ("dir: specify zero or one directory");
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     if (optparse_hasopt (p, "at")) {
         const char *reference = optparse_get_str (p, "at", NULL);
         if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READDIR, key, reference)))
@@ -1264,6 +1355,7 @@ int cmd_dir (optparse_t *p, int argc, char **argv)
         log_err_exit ("%s", key);
     dump_kvs_dir (dir, maxcol, ns, Ropt, dopt);
     flux_future_destroy (f);
+    flux_close (h);
     return (0);
 }
 
@@ -1335,7 +1427,8 @@ static bool need_newline (int col, int col_width, int win_width)
 /* List the content of 'dir', arranging output in columns that fit 'win_width',
  * and using a custom column width selected based on the longest entry name.
  */
-static void list_kvs_dir_single (const flux_kvsdir_t *dir, int win_width,
+static void list_kvs_dir_single (const flux_kvsdir_t *dir,
+                                 int win_width,
                                  optparse_t *p)
 {
     flux_kvsitr_t *itr;
@@ -1376,8 +1469,12 @@ static void list_kvs_dir_single (const flux_kvsdir_t *dir, int win_width,
 /* List contents of directory pointed to by 'key', descending into subdirs
  * if -R was specified.  First the directory is listed, then its subdirs.
  */
-static void list_kvs_dir (flux_t *h, const char *ns, const char *key,
-                          optparse_t *p, int win_width, bool print_label,
+static void list_kvs_dir (flux_t *h,
+                          const char *ns,
+                          const char *key,
+                          optparse_t *p,
+                          int win_width,
+                          bool print_label,
                           bool print_vspace)
 {
     flux_future_t *f;
@@ -1457,8 +1554,11 @@ static int sort_cmp (void *item1, void *item2)
 /* links are special.  If it points to a value, output the link name.
  * If it points to a dir, output contents of the dir.  If it points to
  * an illegal key, still output the link name. */
-static int categorize_link (flux_t *h, const char *ns, char *nkey,
-                            zlist_t *dirs, zlist_t *singles)
+static int categorize_link (flux_t *h,
+                            const char *ns,
+                            char *nkey,
+                            zlist_t *dirs,
+                            zlist_t *singles)
 {
     flux_future_t *f;
 
@@ -1488,10 +1588,13 @@ static int categorize_link (flux_t *h, const char *ns, char *nkey,
  * its contents are to be listed or not.  If -F is specified,
  * 'singles' key names are decorated based on their type.
  */
-static int categorize_key (optparse_t *p, const char *ns, const char *key,
-                           zlist_t *dirs, zlist_t *singles)
+static int categorize_key (flux_t *h,
+                           optparse_t *p,
+                           const char *ns,
+                           const char *key,
+                           zlist_t *dirs,
+                           zlist_t *singles)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
     flux_future_t *f;
     const char *json_str;
     char *nkey;
@@ -1572,7 +1675,7 @@ static int categorize_key (optparse_t *p, const char *ns, const char *key,
  */
 int cmd_ls (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     const char *ns = NULL;
     int optindex = optparse_option_index (p);
     int win_width = get_window_width (p, STDOUT_FILENO);
@@ -1590,12 +1693,15 @@ int cmd_ls (optparse_t *p, int argc, char **argv)
     if (!(dirs = zlist_new ()) || !(singles = zlist_new ()))
         log_err_exit ("zlist_new");
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     if (optindex == argc) {
-        if (categorize_key (p, ns, ".", dirs, singles) < 0)
+        if (categorize_key (h, p, ns, ".", dirs, singles) < 0)
             rc = -1;
     }
     while (optindex < argc) {
-        if (categorize_key (p, ns, argv[optindex++], dirs, singles) < 0)
+        if (categorize_key (h, p, ns, argv[optindex++], dirs, singles) < 0)
             rc = -1;
     }
     if (zlist_size (singles) > 0) {
@@ -1615,12 +1721,13 @@ int cmd_ls (optparse_t *p, int argc, char **argv)
     zlist_destroy (&dirs);
     zlist_destroy (&singles);
 
+    flux_close (h);
     return rc;
 }
 
 int cmd_copy (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
+    flux_t *h;
     int optindex;
     flux_future_t *f;
     const char *srckey, *dstkey;
@@ -1636,11 +1743,14 @@ int cmd_copy (optparse_t *p, int argc, char **argv)
     srckey = argv[optindex];
     dstkey = argv[optindex + 1];
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
     if (!(f = flux_kvs_copy (h, srcns, srckey, dstns, dstkey, 0))
             || flux_future_get (f, NULL) < 0)
         log_err_exit ("flux_kvs_copy");
     flux_future_destroy (f);
 
+    flux_close (h);
     return (0);
 }
 
@@ -1652,8 +1762,6 @@ int cmd_move (optparse_t *p, int argc, char **argv)
     const char *srckey, *dstkey;
     const char *srcns, *dstns;
 
-    h = (flux_t *)optparse_get_data (p, "flux_handle");
-
     optindex = optparse_option_index (p);
     if (optindex != (argc - 2))
         log_msg_exit ("move: specify srckey dstkey");
@@ -1664,11 +1772,14 @@ int cmd_move (optparse_t *p, int argc, char **argv)
     srckey = argv[optindex];
     dstkey = argv[optindex + 1];
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
     if (!(f = flux_kvs_move (h, srcns, srckey, dstns, dstkey, 0))
             || flux_future_get (f, NULL) < 0)
         log_err_exit ("flux_kvs_move");
     flux_future_destroy (f);
 
+    flux_close (h);
     return (0);
 }
 
@@ -1690,6 +1801,13 @@ void getroot_continuation (flux_future_t *f, void *arg)
             log_err_exit ("flux_kvs_getroot_get_sequence");
         printf ("%d\n", sequence);
     }
+    else if (optparse_hasopt (p, "blobref")) {
+        const char *blobref;
+
+        if (flux_kvs_getroot_get_blobref (f, &blobref) < 0)
+            log_err_exit ("flux_kvs_getroot_get_blobref");
+        printf ("%s\n", blobref);
+    }
     else {
         const char *treeobj;
 
@@ -1703,7 +1821,7 @@ void getroot_continuation (flux_future_t *f, void *arg)
 
 int cmd_getroot (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = optparse_get_data (p, "flux_handle");
+    flux_t *h;
     int optindex = optparse_option_index (p);
     const char *ns = NULL;
     flux_future_t *f;
@@ -1715,12 +1833,15 @@ int cmd_getroot (optparse_t *p, int argc, char **argv)
     }
     ns = optparse_get_str (p, "namespace", NULL);
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
     if (!(f = flux_kvs_getroot (h, ns, flags)))
         log_err_exit ("flux_kvs_getroot");
     if (flux_future_then (f, -1., getroot_continuation, p) < 0)
         log_err_exit ("flux_future_then");
     if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
         log_err_exit ("flux_reactor_run");
+    flux_close (h);
     return (0);
 }
 
@@ -1772,7 +1893,7 @@ static flux_future_t *eventlog_append_event (flux_t *h,
 
 int cmd_eventlog_append (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = optparse_get_data (p, "flux_handle");
+    flux_t *h;
     const char *ns = NULL;
     int optindex = optparse_option_index (p);
     const char *key;
@@ -1792,6 +1913,9 @@ int cmd_eventlog_append (optparse_t *p, int argc, char **argv)
     if (optindex < argc)
         context = eventlog_context_from_args (argv + optindex);
 
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
     f = eventlog_append_event (h, ns, key, timestamp, name, context);
     if (flux_future_get (f, NULL) < 0)
         log_err_exit ("flux_kvs_commit");
@@ -1799,6 +1923,7 @@ int cmd_eventlog_append (optparse_t *p, int argc, char **argv)
     flux_future_destroy (f);
     free (context);
 
+    flux_close (h);
     return (0);
 }
 
@@ -1806,43 +1931,48 @@ struct eventlog_get_ctx {
     optparse_t *p;
     int maxcount;
     int count;
+    struct eventlog_formatter *evf;
 };
 
-/* print event in raw form */
-void eventlog_unformatted_print (json_t *event)
-{
-    char *e;
-
-    if (!(e = json_dumps (event, JSON_COMPACT)))
-        log_msg_exit ("json_dumps");
-    printf ("%s\n", e);
-    free (e);
-}
+struct eventlog_wait_event_ctx {
+    optparse_t *p;
+    const char *key;
+    const char *wait_event;
+    bool got_event;
+    struct eventlog_formatter *evf;
+};
 
-/* print event with human-readable time
- */
-static void eventlog_prettyprint (json_t *event)
+static struct eventlog_formatter *formatter_create (optparse_t *p)
 {
-    double timestamp;
-    const char *name;
-    json_t *context = NULL;
-    char *context_str = NULL;
-
-    if (eventlog_entry_parse (event, &timestamp, &name, &context) < 0)
-        log_err_exit ("eventlog_entry_parse");
-
-    if (context) {
-        if (!(context_str = json_dumps (context, JSON_COMPACT)))
-            log_msg_exit ("json_dumps");
+    struct eventlog_formatter *evf;
+    const char *color = optparse_get_str (p, "color", "auto");
+    bool human = optparse_hasopt (p, "human");
+    bool unformatted = optparse_hasopt (p, "unformatted");
+
+    if (unformatted && (human || streq (color, "always"))) {
+        log_msg ("Do not specify --unformatted with --human or --color=always");
+        return NULL;
+    }
+    if (!(evf = eventlog_formatter_create ()))
+        return NULL;
+    if (unformatted) {
+        if (eventlog_formatter_set_format (evf, "json") < 0) {
+            log_err ("failed to set json output");
+            goto error;
+        }
     }
-
-    printf ("%lf %s%s%s\n", timestamp,
-                            name,
-                            context_str ? " " : "",
-                            context_str ? context_str : "");
-
-    free (context_str);
-    fflush (stdout);
+    if (human && eventlog_formatter_set_timestamp_format (evf, "human") < 0) {
+        log_err ("failed to set human timestamp format");
+        goto error;
+    }
+    if (eventlog_formatter_colors_init (evf, color ? color : "always") < 0) {
+        log_err ("invalid value: --color=%s", color);
+        goto error;
+    }
+    return evf;
+error:
+    eventlog_formatter_destroy (evf);
+    return NULL;
 }
 
 void eventlog_get_continuation (flux_future_t *f, void *arg)
@@ -1852,14 +1982,16 @@ void eventlog_get_continuation (flux_future_t *f, void *arg)
     json_t *a;
     size_t index;
     json_t *value;
+    flux_error_t error;
     bool limit_reached = false;
 
-    /* Handle cancelled lookup (FLUX_KVS_WATCH flag only).
+    /* Handle canceled lookup (FLUX_KVS_WATCH flag only).
      * Destroy the future and return (reactor will then terminate).
      * Errors other than ENODATA are handled by the flux_kvs_lookup_get().
      */
-    if (optparse_hasopt (ctx->p, "watch") && flux_rpc_get (f, NULL) < 0
-                                          && errno == ENODATA) {
+    if (optparse_hasopt (ctx->p, "watch")
+        && flux_rpc_get (f, NULL) < 0
+        && errno == ENODATA) {
         flux_future_destroy (f);
         return;
     }
@@ -1871,10 +2003,10 @@ void eventlog_get_continuation (flux_future_t *f, void *arg)
         log_err_exit ("eventlog_decode");
 
     json_array_foreach (a, index, value) {
-        if (optparse_hasopt (ctx->p, "unformatted"))
-            eventlog_unformatted_print (value);
-        else
-            eventlog_prettyprint (value);
+        if (ctx->maxcount == 0 || ctx->count < ctx->maxcount) {
+            if (eventlog_entry_dumpf (ctx->evf, stdout, &error, value) < 0)
+                log_msg ("failed to print eventlog entry: %s", error.text);
+        }
         if (ctx->maxcount > 0 && ++ctx->count == ctx->maxcount)
             limit_reached = true;
     }
@@ -1900,7 +2032,7 @@ void eventlog_get_continuation (flux_future_t *f, void *arg)
 
 int cmd_eventlog_get (optparse_t *p, int argc, char **argv)
 {
-    flux_t *h = optparse_get_data (p, "flux_handle");
+    flux_t *h;
     const char *ns = NULL;
     int optindex = optparse_option_index (p);
     const char *key;
@@ -1919,11 +2051,18 @@ int cmd_eventlog_get (optparse_t *p, int argc, char **argv)
         flags |= FLUX_KVS_WATCH;
         flags |= FLUX_KVS_WATCH_APPEND;
     }
+    if (optparse_hasopt (p, "waitcreate"))
+        flags |= FLUX_KVS_WAITCREATE;
 
     ctx.p = p;
     ctx.count = 0;
     ctx.maxcount = optparse_get_int (p, "count", 0);
 
+    if (!(ctx.evf = formatter_create (p)))
+        log_msg_exit ("failed to create eventlog formatter");
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
     if (!(f = flux_kvs_lookup (h, ns, flags, key)))
         log_err_exit ("flux_kvs_lookup");
     if (flux_future_then (f, -1., eventlog_get_continuation, &ctx) < 0)
@@ -1931,6 +2070,124 @@ int cmd_eventlog_get (optparse_t *p, int argc, char **argv)
     if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
         log_err_exit ("flux_reactor_run");
 
+    flux_close (h);
+    eventlog_formatter_destroy (ctx.evf);
+    return (0);
+}
+
+void eventlog_wait_event_continuation (flux_future_t *f, void *arg)
+{
+    struct eventlog_wait_event_ctx *ctx = arg;
+    const char *s;
+    json_t *a;
+    size_t index;
+    json_t *value;
+
+    if (flux_kvs_lookup_get (f, &s) < 0) {
+        if (errno == ENOENT) {
+            flux_future_destroy (f);
+            log_msg_exit ("eventlog path %s not found", ctx->key);
+        }
+        else if (errno == ETIMEDOUT) {
+            flux_future_destroy (f);
+            log_msg_exit ("wait-event timeout on event '%s'",
+                          ctx->wait_event);
+        }
+        else if (errno == ENODATA) {
+            flux_future_destroy (f);
+            if (!ctx->got_event)
+                log_msg_exit ("event '%s' never received",
+                              ctx->wait_event);
+            return;
+        }
+        else
+            log_err_exit ("flux_kvs_lookup_get");
+    }
+
+    if (!(a = eventlog_decode (s)))
+        log_err_exit ("eventlog_decode");
+
+    json_array_foreach (a, index, value) {
+        const char *name;
+        double timestamp;
+        flux_error_t error;
+
+        if (eventlog_entry_parse (value, &timestamp, &name, NULL) < 0)
+            log_err_exit ("eventlog_entry_parse");
+
+        eventlog_formatter_update_t0 (ctx->evf, timestamp);
+
+        if (streq (name, ctx->wait_event)) {
+            if (!optparse_hasopt (ctx->p, "quiet")) {
+                if (eventlog_entry_dumpf (ctx->evf, stdout, &error, value) < 0)
+                    log_msg ("failed to dump event %s: %s", name, error.text);
+            }
+            ctx->got_event = true;
+            break;
+        }
+        else if (optparse_hasopt (ctx->p, "verbose")) {
+            if (!ctx->got_event) {
+                if (eventlog_entry_dumpf (ctx->evf, stdout, &error, value) < 0)
+                    log_msg ("failed to dump event %s: %s", name, error.text);
+            }
+        }
+    }
+
+    fflush (stdout);
+
+    if (ctx->got_event) {
+        if (flux_kvs_lookup_cancel (f) < 0)
+            log_err_exit ("flux_kvs_lookup_cancel");
+    }
+    flux_future_reset (f);
+    json_decref (a);
+}
+
+int cmd_eventlog_wait_event (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    const char *ns = NULL;
+    int optindex = optparse_option_index (p);
+    flux_future_t *f;
+    double timeout;
+    int flags = 0;
+    struct eventlog_wait_event_ctx ctx;
+
+    if (argc - optindex != 2) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    ns = optparse_get_str (p, "namespace", NULL);
+
+    ctx.p = p;
+    ctx.key = argv[optindex++];
+    ctx.wait_event = argv[optindex++];
+    ctx.got_event = false;
+
+    if (!(ctx.evf = formatter_create (p)))
+        log_msg_exit ("failed to create eventlog formatter");
+
+    timeout = optparse_get_duration (p, "timeout", -1.0);
+
+    flags |= FLUX_KVS_WATCH;
+    flags |= FLUX_KVS_WATCH_APPEND;
+    if (optparse_hasopt (p, "waitcreate"))
+        flags |= FLUX_KVS_WAITCREATE;
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    if (!(f = flux_kvs_lookup (h, ns, flags, ctx.key)))
+        log_err_exit ("flux_kvs_lookup");
+    if (flux_future_then (f,
+                          timeout,
+                          eventlog_wait_event_continuation,
+                          &ctx) < 0)
+        log_err_exit ("flux_future_then");
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        log_err_exit ("flux_reactor_run");
+
+    flux_close (h);
+    eventlog_formatter_destroy (ctx.evf);
     return (0);
 }
 
@@ -1948,6 +2205,9 @@ static struct optparse_option eventlog_get_opts[] =  {
     { .name = "namespace", .key = 'N', .has_arg = 1,
       .usage = "Specify KVS namespace to use.",
     },
+    { .name = "waitcreate", .key = 'W', .has_arg = 0,
+      .usage = "Wait for creation to occur on watch",
+    },
     { .name = "watch", .key = 'w', .has_arg = 0,
       .usage = "Monitor eventlog",
     },
@@ -1957,6 +2217,44 @@ static struct optparse_option eventlog_get_opts[] =  {
     { .name = "unformatted", .key = 'u', .has_arg = 0,
       .usage = "Show event in RFC 18 form",
     },
+    { .name = "human", .key = 'H', .has_arg = 0,
+      .usage = "Display human-readable output",
+    },
+    { .name = "color", .key = 'L', .has_arg = 2, .arginfo = "WHEN",
+      .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
+      .usage = "Colorize output when supported; WHEN can be 'always' "
+               "(default if omitted), 'never', or 'auto' (default)."
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option eventlog_wait_event_opts[] =  {
+    { .name = "namespace", .key = 'N', .has_arg = 1,
+      .usage = "Specify KVS namespace to use.",
+    },
+    { .name = "waitcreate", .key = 'W', .has_arg = 0,
+      .usage = "Wait for creation to occur on eventlog",
+    },
+    { .name = "timeout", .key = 't', .has_arg = 1, .arginfo = "DURATION",
+      .usage = "timeout after DURATION",
+    },
+    { .name = "unformatted", .key = 'u', .has_arg = 0,
+      .usage = "Show event in RFC 18 form",
+    },
+    { .name = "quiet", .key = 'q', .has_arg = 0,
+      .usage = "Do not output matched event",
+    },
+    { .name = "verbose", .key = 'v', .has_arg = 0,
+      .usage = "Output all events before matched event",
+    },
+    { .name = "human", .key = 'H', .has_arg = 0,
+      .usage = "Display human-readable output",
+    },
+    { .name = "color", .key = 'L', .has_arg = 2, .arginfo = "WHEN",
+      .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
+      .usage = "Colorize output when supported; WHEN can be 'always' "
+               "(default if omitted), 'never', or 'auto' (default)."
+    },
     OPTPARSE_TABLE_END
 };
 
@@ -1969,12 +2267,20 @@ static struct optparse_subcommand eventlog_subcommands[] = {
       eventlog_append_opts,
     },
     { "get",
-      "[-N ns] [-u] [-w] [-c COUNT] key",
+      "[-N ns] [-u] [-W] [-w] [-c COUNT] [-H] [-L auto|always|never] key",
       "Get eventlog",
       cmd_eventlog_get,
       0,
       eventlog_get_opts,
     },
+    { "wait-event",
+      "[-N ns] [-t seconds] [-u] [-W] [-q] [-v] [-L auto|always|never] "
+      "key event",
+      "Wait for an event",
+      cmd_eventlog_wait_event,
+      0,
+      eventlog_wait_event_opts,
+    },
     OPTPARSE_SUBCMD_END
 };
 
diff --git a/src/cmd/flux-logger.c b/src/cmd/flux-logger.c
index d6c88e312a26..240ce08d8066 100644
--- a/src/cmd/flux-logger.c
+++ b/src/cmd/flux-logger.c
@@ -14,7 +14,11 @@
 #include <unistd.h>
 #include <stdio.h>
 #include <getopt.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
 #include <flux/core.h>
 
 #include "src/common/libutil/xzmalloc.h"
diff --git a/src/cmd/flux-mini.py b/src/cmd/flux-mini.py
deleted file mode 100755
index 53ad14980800..000000000000
--- a/src/cmd/flux-mini.py
+++ /dev/null
@@ -1,317 +0,0 @@
-##############################################################
-# Copyright 2019 Lawrence Livermore National Security, LLC
-# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
-#
-# This file is part of the Flux resource manager framework.
-# For details, see https://github.com/flux-framework.
-#
-# SPDX-License-Identifier: LGPL-3.0
-##############################################################
-
-from __future__ import print_function
-
-import os
-import sys
-import logging
-import argparse
-import json
-
-import flux
-from flux import job
-from flux.job import JobspecV1
-from flux import util
-from flux import constants
-
-
-class SubmitCmd:
-    """
-    SubmitCmd submits a job, displays the jobid on stdout, and returns.
-
-    Usage: flux mini submit [OPTIONS] cmd ...
-    """
-
-    def __init__(self):
-        self.parser = self.create_parser()
-
-    def create_parser(self):
-        """
-        Create parser with args for submit subcommand
-        """
-        parser = argparse.ArgumentParser(add_help=False)
-        parser.add_argument(
-            "-N", "--nodes", type=int, metavar="N", help="Number of nodes to allocate"
-        )
-        parser.add_argument(
-            "-n",
-            "--ntasks",
-            type=int,
-            metavar="N",
-            default=1,
-            help="Number of tasks to start",
-        )
-        parser.add_argument(
-            "-c",
-            "--cores-per-task",
-            type=int,
-            metavar="N",
-            default=1,
-            help="Number of cores to allocate per task",
-        )
-        parser.add_argument(
-            "-g",
-            "--gpus-per-task",
-            type=int,
-            metavar="N",
-            help="Number of GPUs to allocate per task",
-        )
-        parser.add_argument(
-            "-t",
-            "--time-limit",
-            type=str,
-            metavar="FSD",
-            help="Time limit in Flux standard duration, e.g. 2d, 1.5h",
-        )
-        parser.add_argument(
-            "--priority",
-            help="Set job priority (0-31, default=16)",
-            type=int,
-            metavar="N",
-            default=16,
-        )
-        parser.add_argument(
-            "--job-name",
-            type=str,
-            help="Set an optional name for job to NAME",
-            metavar="NAME",
-        )
-        parser.add_argument(
-            "-o",
-            "--setopt",
-            action="append",
-            help="Set shell option OPT. An optional value is supported with"
-            + " OPT=VAL (default VAL=1) (multiple use OK)",
-            metavar="OPT",
-        )
-        parser.add_argument(
-            "--setattr",
-            action="append",
-            help="Set job attribute ATTR to VAL (multiple use OK)",
-            metavar="ATTR=VAL",
-        )
-        parser.add_argument(
-            "--input",
-            type=str,
-            help="Redirect job stdin from FILENAME, bypassing KVS",
-            metavar="FILENAME",
-        )
-        parser.add_argument(
-            "--output",
-            type=str,
-            help="Redirect job stdout to FILENAME, bypassing KVS",
-            metavar="FILENAME",
-        )
-        parser.add_argument(
-            "--error",
-            type=str,
-            help="Redirect job stderr to FILENAME, bypassing KVS",
-            metavar="FILENAME",
-        )
-        parser.add_argument(
-            "--label-io",
-            action="store_true",
-            help="Add rank labels to stdout, stderr lines",
-        )
-        parser.add_argument(
-            "--flags",
-            action="append",
-            help="Set comma separated list of job submission flags. Possible "
-            + "flags:  debug, waitable",
-            metavar="FLAGS",
-        )
-        parser.add_argument(
-            "--dry-run",
-            action="store_true",
-            help="Don't actually submit job, just emit jobspec",
-        )
-        parser.add_argument(
-            "command", nargs=argparse.REMAINDER, help="Job command and arguments"
-        )
-        return parser
-
-    def submit(self, args):
-        """
-        Submit job, constructing jobspec from args.
-        Returns jobid.
-        """
-        if not args.command:
-            raise ValueError("job command and arguments are missing")
-
-        jobspec = JobspecV1.from_command(
-            args.command,
-            num_tasks=args.ntasks,
-            cores_per_task=args.cores_per_task,
-            gpus_per_task=args.gpus_per_task,
-            num_nodes=args.nodes,
-        )
-        jobspec.cwd = os.getcwd()
-        jobspec.environment = dict(os.environ)
-        if args.time_limit is not None:
-            jobspec.duration = args.time_limit
-
-        if args.job_name is not None:
-            jobspec.setattr("system.job.name", args.job_name)
-
-        if args.input is not None:
-            jobspec.setattr_shell_option("input.stdin.type", "file")
-            jobspec.setattr_shell_option("input.stdin.path", args.input)
-
-        if args.output is not None:
-            jobspec.setattr_shell_option("output.stdout.type", "file")
-            jobspec.setattr_shell_option("output.stdout.path", args.output)
-            if args.label_io:
-                jobspec.setattr_shell_option("output.stdout.label", True)
-
-        if args.error is not None:
-            jobspec.setattr_shell_option("output.stderr.type", "file")
-            jobspec.setattr_shell_option("output.stderr.path", args.error)
-            if args.label_io:
-                jobspec.setattr_shell_option("output.stderr.label", True)
-
-        if args.setopt is not None:
-            for kv in args.setopt:
-                # Split into key, val with a default for 1 if no val given:
-                key, val = (kv.split("=", 1) + [1])[:2]
-                try:
-                    val = json.loads(val)
-                except:
-                    pass
-                jobspec.setattr_shell_option(key, val)
-
-        if args.setattr is not None:
-            for kv in args.setattr:
-                tmp = kv.split("=", 1)
-                if len(tmp) != 2:
-                    raise ValueError("--setattr: Missing value for attr " + kv)
-                key = tmp[0]
-                try:
-                    val = json.loads(tmp[1])
-                except:
-                    val = tmp[1]
-                jobspec.setattr(key, val)
-
-        arg_debug = False
-        arg_waitable = False
-        if args.flags is not None:
-            for tmp in args.flags:
-                for flag in tmp.split(","):
-                    if flag == "debug":
-                        arg_debug = True
-                    elif flag == "waitable":
-                        arg_waitable = True
-                    else:
-                        raise ValueError("--flags: Unknown flag " + flag)
-
-        if args.dry_run:
-            print(jobspec.dumps(), file=sys.stdout)
-            sys.exit(0)
-
-        h = flux.Flux()
-        return job.submit(
-            h,
-            jobspec.dumps(),
-            priority=args.priority,
-            waitable=arg_waitable,
-            debug=arg_debug,
-        )
-
-    def main(self, args):
-        jobid = self.submit(args)
-        print(jobid, file=sys.stdout)
-
-    def get_parser(self):
-        return self.parser
-
-
-class RunCmd(SubmitCmd):
-    """
-    RunCmd is identical to SubmitCmd, except it attaches the the job
-    after submission.  Some additional options are added to modify the
-    attach behavior.
-
-    Usage: flux mini run [OPTIONS] cmd ...
-    """
-
-    def __init__(self):
-        self.parser = self.create_parser()
-        self.parser.add_argument(
-            "-v",
-            "--verbose",
-            action="count",
-            default=0,
-            help="Increase verbosity on stderr (multiple use OK)",
-        )
-
-    def main(self, args):
-        jobid = self.submit(args)
-
-        # Display job id on stderr if -v
-        # N.B. we must flush sys.stderr due to the fact that it is buffered
-        # when it points to a file, and os.execvp leaves it unflushed
-        if args.verbose > 0:
-            print("jobid:", jobid, file=sys.stderr)
-            sys.stderr.flush()
-
-        # Build args for flux job attach
-        attach_args = ["flux-job", "attach"]
-        if args.label_io:
-            attach_args.append("--label-io")
-        if args.verbose > 1:
-            attach_args.append("--show-events")
-        if args.verbose > 2:
-            attach_args.append("--show-exec")
-        attach_args.append(str(jobid))
-
-        # Exec flux-job attach, searching for it in FLUX_EXEC_PATH.
-        os.environ["PATH"] = os.environ["FLUX_EXEC_PATH"] + ":" + os.environ["PATH"]
-        os.execvp("flux-job", attach_args)
-
-
-logger = logging.getLogger("flux-mini")
-
-
-@util.CLIMain(logger)
-def main():
-    parser = argparse.ArgumentParser(prog="flux-mini")
-    subparsers = parser.add_subparsers(
-        title="supported subcommands", description="", dest="subcommand"
-    )
-    subparsers.required = True
-
-    # run
-    run = RunCmd()
-    mini_run_parser_sub = subparsers.add_parser(
-        "run",
-        parents=[run.get_parser()],
-        help="run a job interactively",
-        formatter_class=flux.util.help_formatter(),
-    )
-    mini_run_parser_sub.set_defaults(func=run.main)
-
-    # submit
-    submit = SubmitCmd()
-    mini_submit_parser_sub = subparsers.add_parser(
-        "submit",
-        parents=[submit.get_parser()],
-        help="enqueue a job",
-        formatter_class=flux.util.help_formatter(),
-    )
-    mini_submit_parser_sub.set_defaults(func=submit.main)
-
-    args = parser.parse_args()
-    args.func(args)
-
-
-if __name__ == "__main__":
-    main()
-
-# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-module.c b/src/cmd/flux-module.c
index 0ea77b899e2f..c6242edc7f95 100644
--- a/src/cmd/flux-module.c
+++ b/src/cmd/flux-module.c
@@ -11,21 +11,40 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <time.h>
+#include <math.h>
 #include <stdio.h>
+#include <signal.h>
 #include <getopt.h>
-#include <dlfcn.h>
 #include <flux/core.h>
 #include <flux/optparse.h>
 #include <jansson.h>
-#include <czmq.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <ctype.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#ifdef HAVE_STRERRORNAME_NP
+#include <string.h>
+#else
+#include "src/common/libmissing/strerrorname_np.h"
+#endif
+
 #include <assert.h>
 
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/oom.h"
-#include "src/common/libutil/read_all.h"
-#include "src/common/libutil/iterators.h"
+#include "src/common/libutil/jpath.h"
+#include "src/common/libutil/ansi_color.h"
+#include "src/common/libutil/parse_size.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
 
 const int max_idle = 99;
 
@@ -33,24 +52,37 @@ int cmd_list (optparse_t *p, int argc, char **argv);
 int cmd_remove (optparse_t *p, int argc, char **argv);
 int cmd_load (optparse_t *p, int argc, char **argv);
 int cmd_reload (optparse_t *p, int argc, char **argv);
-int cmd_info (optparse_t *p, int argc, char **argv);
 int cmd_stats (optparse_t *p, int argc, char **argv);
 int cmd_debug (optparse_t *p, int argc, char **argv);
+int cmd_trace (optparse_t *p, int argc, char **argv);
 
-static struct optparse_option legacy_opts[] =  {
-    { .name = "rank", .key = 'r', .has_arg = 1, .arginfo = "RANK",
-      .usage = "Send RPC to specified rank",
-    },
+static struct optparse_option list_opts[] = {
+    { .name = "long",  .key = 'l',  .has_arg = 0,
+      .usage = "Include full DSO path for each module", },
     OPTPARSE_TABLE_END,
 };
 
 static struct optparse_option remove_opts[] =  {
-    { .name = "rank", .key = 'r', .has_arg = 1, .arginfo = "RANK",
-      .usage = "Send RPC to specified rank",
+    { .name = "force", .key = 'f', .has_arg = 0,
+      .usage = "Ignore nonexistent modules",
     },
-    { .name = "force", .key = 'f',
+    OPTPARSE_TABLE_END,
+};
+
+static struct optparse_option reload_opts[] =  {
+    { .name = "force", .key = 'f', .has_arg = 0,
       .usage = "Ignore nonexistent modules",
     },
+    { .name = "name", .has_arg = 1, .arginfo = "NAME",
+      .usage = "Override default module name",
+    },
+    OPTPARSE_TABLE_END,
+};
+
+static struct optparse_option load_opts[] =  {
+    { .name = "name", .has_arg = 1, .arginfo = "NAME",
+      .usage = "Override default module name",
+    },
     OPTPARSE_TABLE_END,
 };
 
@@ -64,8 +96,10 @@ static struct optparse_option stats_opts[] =  {
     { .name = "type", .key = 't', .has_arg = 1, .arginfo = "int|double",
       .usage = "Convert JSON value to specified type",
     },
-    { .name = "rusage", .key = 'R', .has_arg = 0,
-      .usage = "Request rusage data instead of stats",
+    { .name = "rusage", .key = 'R', .has_arg = 2,
+      .arginfo = "[self|children|thread]",
+      .usage = "Request rusage data instead of stats (default: self)",
+      .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
     },
     { .name = "clear", .key = 'c', .has_arg = 0,
       .usage = "Clear stats on target rank",
@@ -87,13 +121,38 @@ static struct optparse_option debug_opts[] = {
     OPTPARSE_TABLE_END,
 };
 
+static struct optparse_option trace_opts[] = {
+    { .name = "full", .key = 'f', .has_arg = 0,
+      .usage = "Show JSON message payload, if any",
+    },
+    { .name = "topic", .key = 'T', .has_arg = 1,
+      .arginfo = "GLOB",
+      .usage = "Filter output by message topic glob",
+    },
+    { .name = "type", .key = 't', .has_arg = 1,
+      .flags = OPTPARSE_OPT_AUTOSPLIT,
+      .arginfo = "TYPE,...",
+      .usage = "Filter output by message type "
+               "(request, response, event, control)",
+    },
+    { .name = "color", .key = 'L', .has_arg = 2, .arginfo = "WHEN",
+      .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
+      .usage = "Colorize output when supported; WHEN can be 'always' "
+               "(default if omitted), 'never', or 'auto' (default)." },
+    { .name = "human",  .key = 'H',  .has_arg = 0,
+      .usage = "Human readable output", },
+    { .name = "delta",  .key = 'd',  .has_arg = 0,
+      .usage = "With --human, show timestamp delta between messages", },
+    OPTPARSE_TABLE_END,
+};
+
 static struct optparse_subcommand subcommands[] = {
     { "list",
-      "[OPTIONS] [module]",
+      "[OPTIONS]",
       "List loaded modules",
       cmd_list,
       0,
-      NULL,
+      list_opts,
     },
     { "remove",
       "[OPTIONS] module",
@@ -114,21 +173,14 @@ static struct optparse_subcommand subcommands[] = {
       "Load module",
       cmd_load,
       0,
-      legacy_opts,
+      load_opts,
     },
     { "reload",
       "[OPTIONS] module",
       "Reload module",
       cmd_reload,
       0,
-      remove_opts,
-    },
-    { "info",
-      "[OPTIONS] module",
-      "Display module info",
-      cmd_info,
-      0,
-      NULL
+      reload_opts,
     },
     { "stats",
       "[OPTIONS] module",
@@ -144,6 +196,13 @@ static struct optparse_subcommand subcommands[] = {
       0,
       debug_opts,
     },
+    { "trace",
+      "[OPTIONS] module [module...]",
+      "Trace module messages",
+      cmd_trace,
+      0,
+      trace_opts,
+    },
     OPTPARSE_SUBCMD_END
 };
 
@@ -196,188 +255,149 @@ int main (int argc, char *argv[])
     return (exitval);
 }
 
-char *sha1 (const char *path)
-{
-    zfile_t *zf = zfile_new (NULL, path);
-    char *digest = NULL;
-    if (zf)
-        digest = xstrdup (zfile_digest (zf));
-    zfile_destroy (&zf);
-    return digest;
-}
-
-int filesize (const char *path)
+static bool has_suffix (const char *s, const char *suffix)
 {
-    struct stat sb;
-    if (stat (path, &sb) < 0)
-        return 0;
-    return sb.st_size;
+    int n = strlen (s) - strlen (suffix);
+    if (n >= 0 && streq (&s[n], suffix))
+        return true;
+    return false;
 }
 
-void module_dlerror (const char *errmsg, void *arg)
-{
-    log_msg ("%s", errmsg);
-}
-
-void parse_modarg (const char *arg, char **name, char **path)
+/* If path looks like a dso filename, then canonicalize it so that
+ * the broker, possibly running in a different directory, can dlopen() it.
+ */
+static int canonicalize_if_path (const char *path, char **fullpathp)
 {
-    char *modpath = NULL;
-    char *modname = NULL;
-
-    if (strchr (arg, '/')) {
-        if (!(modpath = realpath (arg, NULL)))
-            log_err_exit ("%s", arg);
-        if (!(modname = flux_modname (modpath, module_dlerror, NULL)))
-            log_msg_exit ("%s", modpath);
-    } else {
-        char *searchpath = getenv ("FLUX_MODULE_PATH");
-        if (!searchpath)
-            log_msg_exit ("FLUX_MODULE_PATH is not set");
-        modname = xstrdup (arg);
-        if (!(modpath = flux_modfind (searchpath, modname,
-                                      module_dlerror, NULL)))
-            log_msg_exit ("%s: not found in module search path", modname);
+    char *fullpath = NULL;
+    if (strchr (path, '/') || has_suffix (path, ".so")) {
+        if (!(fullpath = realpath (path, NULL)))
+            return -1;
     }
-    *name = modname;
-    *path = modpath;
+    *fullpathp = fullpath;
+    return 0;
 }
 
-int cmd_info (optparse_t *p, int argc, char **argv)
+static json_t *args_create (int argc, char **argv)
 {
-    char *modpath = NULL;
-    char *modname = NULL;
-    char *digest = NULL;
-    int n;
-
-    if ((n = optparse_option_index (p)) != argc - 1) {
-        optparse_print_usage (p);
-        exit (1);
+    json_t *args;
+    if (!(args = json_array ()))
+        goto error;
+    for (int i = 0; i < argc; i++) {
+        json_t *s = json_string (argv[i]);
+        if (!s || json_array_append_new (args, s) < 0) {
+            json_decref (s);
+            goto error;
+        }
     }
-    parse_modarg (argv[n], &modname, &modpath);
-    digest = sha1 (modpath);
-    printf ("Module name:  %s\n", modname);
-    printf ("Module path:  %s\n", modpath);
-    printf ("SHA1 Digest:  %s\n", digest);
-    printf ("Size:         %d bytes\n", filesize (modpath));
-
-    free (modpath);
-    free (modname);
-    free (digest);
-    return (0);
+    return args;
+error:
+    json_decref (args);
+    return NULL;
 }
 
-/* Derive name of module loading service from module name.
- * Caller must free.
- */
-char *getservice (const char *modname)
+static int set_string (json_t *o, const char *key, const char *val)
 {
-    char *service = NULL;
-    if (strchr (modname, '.')) {
-        service = xstrdup (modname);
-        char *p = strrchr (service, '.');
-        *p = '\0';
-    } else
-        service = xstrdup ("cmb");
-    return service;
+    json_t *s = json_string (val);
+    if (!s || json_object_set_new (o, key, s) < 0) {
+        json_decref (s);
+        return -1;
+    }
+    return 0;
 }
 
-static void module_load (flux_t *h, optparse_t *p, int argc, char **argv)
+static void module_load (flux_t *h,
+                         optparse_t *p,
+                         const char *path,
+                         int argc,
+                         char **argv)
 {
-    char *modname;
-    char *modpath;
-    int n;
+    const char *name = optparse_get_str (p, "name", NULL);
+    char *fullpath = NULL;
     flux_future_t *f;
-
-    if ((n = optparse_option_index (p)) == argc) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    parse_modarg (argv[n++], &modname, &modpath);
-
-    char *service = getservice (modname);
-    char *topic = xasprintf ("%s.insmod", service);
-    json_t *args = json_array ();
-    if (!args)
-        log_msg_exit ("json_array() failed");
-    while (n < argc) {
-        json_t *str = json_string (argv[n]);
-        if (!str || json_array_append_new (args, str) < 0)
-            log_msg_exit ("json_string() or json_array_append_new() failed");
-        n++;
-    }
-
+    json_t *args;
+    json_t *payload;
+
+    if (canonicalize_if_path (path, &fullpath) < 0)
+        log_err_exit ("could not canonicalize module path '%s'", path);
+    if (!(args = args_create (argc, argv))
+        || !(payload = json_pack ("{s:s s:O}",
+                               "path", fullpath ? fullpath : path,
+                               "args", args))
+        || (name && set_string (payload, "name", name) < 0))
+        log_msg_exit ("failed to create module.load payload");
     if (!(f = flux_rpc_pack (h,
-                             topic,
-                             optparse_get_int (p, "rank", FLUX_NODEID_ANY),
+                             "module.load",
+                             FLUX_NODEID_ANY,
                              0,
-                             "{s:s s:O}",
-                             "path",
-                             modpath,
-                             "args",
-                             args)))
-        log_err_exit ("%s", topic);
-    if (flux_rpc_get (f, NULL) < 0) {
-        if (errno == EEXIST)
-            log_msg_exit ("%s: %s module/service is in use", topic, modname);
-        log_err_exit ("%s", topic);
+                             "O",
+                             payload))
+        || flux_rpc_get (f, NULL) < 0) {
+        log_msg_exit ("load %s: %s", path, future_strerror (f, errno));
     }
     flux_future_destroy (f);
-    free (topic);
-    free (service);
+    json_decref (payload);
     json_decref (args);
-    free (modpath);
-    free (modname);
+    free (fullpath);
 }
 
 int cmd_load (optparse_t *p, int argc, char **argv)
 {
+    int n = optparse_option_index (p);
+    const char *path;
     flux_t *h;
+
+    if (n == argc) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    path = argv[n++];
     if (!(h = flux_open (NULL, 0)))
         log_err_exit ("flux_open");
-    module_load (h, p, argc, argv);
+
+    module_load (h, p, path, argc - n, argv + n);
+
     flux_close (h);
     return 0;
 }
 
-static void module_remove (flux_t *h, const char *modname, optparse_t *p)
+static void module_remove (flux_t *h, optparse_t *p, const char *path)
 {
+    char *fullpath = NULL;
     flux_future_t *f;
-    char *service = getservice (modname);
-    char *topic = xasprintf ("%s.rmmod", service);
+
+    if (canonicalize_if_path (path, &fullpath) < 0)
+        log_err_exit ("could not canonicalize module path '%s'", path);
 
     if (!(f = flux_rpc_pack (h,
-                             topic,
-                             optparse_get_int (p, "rank", FLUX_NODEID_ANY),
+                             "module.remove",
+                             FLUX_NODEID_ANY,
                              0,
                              "{s:s}",
-                             "name",
-                             modname)))
-        log_err_exit ("%s %s", topic, modname);
-    if (flux_rpc_get (f, NULL) < 0) {
+                             "name", fullpath ? fullpath : path))
+        || flux_rpc_get (f, NULL) < 0) {
         if (!(optparse_hasopt (p, "force") && errno == ENOENT))
-            log_err_exit ("%s %s", topic, modname);
+            log_msg_exit ("remove %s: %s", path, future_strerror (f, errno));
     }
     flux_future_destroy (f);
-    free (topic);
-    free (service);
+    free (fullpath);
 }
 
 int cmd_remove (optparse_t *p, int argc, char **argv)
 {
-    char *modname;
+    int n = optparse_option_index (p);
+    char *path;
     flux_t *h;
-    int n;
 
-    if ((n = optparse_option_index (p)) != argc - 1) {
+    if (n != argc - 1) {
         optparse_print_usage (p);
         exit (1);
     }
-    modname = argv[n++];
+    path = argv[n];
 
     if (!(h = flux_open (NULL, 0)))
         log_err_exit ("flux_open");
 
-    module_remove (h, modname, p);
+    module_remove (h, p, path);
 
     flux_close (h);
     return (0);
@@ -385,22 +405,25 @@ int cmd_remove (optparse_t *p, int argc, char **argv)
 
 int cmd_reload (optparse_t *p, int argc, char **argv)
 {
-    char *modname;
-    char *modpath;
+    int n = optparse_option_index (p);
+    const char *name = optparse_get_str (p, "name", NULL);
+    const char *path;
     flux_t *h;
-    int n;
 
-    if ((n = optparse_option_index (p)) == argc) {
+    if (n == argc) {
         optparse_print_usage (p);
         exit (1);
     }
-    parse_modarg (argv[n], &modname, &modpath);
-
+    path = argv[n++];
     if (!(h = flux_open (NULL, 0)))
         log_err_exit ("flux_open");
 
-    module_remove (h, modname, p);
-    module_load (h, p, argc, argv);
+    /* If --name=NAME was specified, remove by that name rather than
+     * the path so the correct instantiation of the DSO is selected.
+     */
+    module_remove (h, p, name ? name : path);
+
+    module_load (h, p, path, argc - n, argv + n);
     return (0);
 }
 
@@ -410,7 +433,7 @@ int cmd_reload (optparse_t *p, int argc, char **argv)
  * the module name, implicitly registered as a service).
  * Caller must free result.
  */
-char *lsmod_services_string (json_t *services, const char *skip)
+char *lsmod_services_string (json_t *services, const char *skip, int maxcol)
 {
     size_t index;
     json_t *value;
@@ -419,22 +442,25 @@ char *lsmod_services_string (json_t *services, const char *skip)
 
     json_array_foreach (services, index, value) {
         const char *name = json_string_value (value);
-        if (name && (!skip || strcmp (name, skip) != 0))
+        if (name && (!skip || !streq (name, skip)))
             if (argz_add (&argz, &argz_len, name) != 0)
                 oom ();
     }
     if (argz)
         argz_stringify (argz, argz_len, ',');
+    if (argz && maxcol > 0 && strlen (argz) > maxcol) {
+        argz[maxcol - 1] = '+';
+        argz[maxcol] = '\0';
+    }
     return argz;
 }
 
-char *lsmod_idle_string (int idle, char *buf, int bufsz)
+void lsmod_idle_string (int idle, char *buf, int bufsz)
 {
     if (idle <= max_idle)
         snprintf (buf, bufsz, "%d", idle);
     else
         snprintf (buf, bufsz, "idle");
-    return buf;
 }
 
 char lsmod_state_char (int state)
@@ -442,8 +468,6 @@ char lsmod_state_char (int state)
     switch (state) {
     case FLUX_MODSTATE_INIT:
         return 'I';
-    case FLUX_MODSTATE_SLEEPING:
-        return 'S';
     case FLUX_MODSTATE_RUNNING:
         return 'R';
     case FLUX_MODSTATE_FINALIZING:
@@ -455,101 +479,119 @@ char lsmod_state_char (int state)
     }
 }
 
-void lsmod_print_header (FILE *f)
+void lsmod_print_header (FILE *f, bool longopt)
 {
-    fprintf (f, "%-20s %8s %-7s %4s  %c %s\n",
-            "Module", "Size", "Digest", "Idle", 'S', "Service");
+    if (longopt) {
+        fprintf (f,
+                 "%-24.24s %4s  %c %s %s %-8s %s\n",
+                 "Module", "Idle", 'S', "Sendq", "Recvq", "Service", "Path");
+    }
+    else {
+        fprintf (f,
+                 "%-24s %4s  %c %s %s %s\n",
+                 "Module", "Idle", 'S', "Sendq", "Recvq", "Service");
+    }
 }
 
 void lsmod_print_entry (FILE *f,
                        const char *name,
-                       int size,
-                       const char *digest,
+                       const char *path,
                        int idle,
                        int status,
-                       json_t *services)
+                       json_t *services,
+                       int sendqueue,
+                       int recvqueue,
+                       bool longopt)
 {
-    int digest_len = strlen (digest);
-    char *serv_s = lsmod_services_string (services, name);
     char idle_s[16];
-
-    fprintf (f, "%-20.20s %8d %7s %4s  %c %s\n",
-             name,
-             size,
-             digest_len > 7 ? digest + digest_len - 7 : digest,
-             lsmod_idle_string (idle, idle_s, sizeof (idle_s)),
-             lsmod_state_char (status),
-             serv_s ? serv_s : "");
-
-    free (serv_s);
+    char state = lsmod_state_char (status);
+
+    lsmod_idle_string (idle, idle_s, sizeof (idle_s));
+
+    if (longopt) {
+        char *s = lsmod_services_string (services, name, 8);
+        fprintf (f, "%-24.24s %4s  %c %5d %5d %-8s %s\n",
+                 name,
+                 idle_s,
+                 state,
+                 sendqueue,
+                 recvqueue,
+                 s ? s : "",
+                 path);
+        free (s);
+    }
+    else {
+        char *s = lsmod_services_string (services, name, 0);
+        fprintf (f, "%-24.24s %4s  %c %5d %5d %s\n",
+                 name,
+                 idle_s,
+                 state,
+                 sendqueue,
+                 recvqueue,
+                 s ? s : "");
+        free (s);
+    }
 }
 
-void lsmod_print_list (FILE *f, json_t *o)
+void lsmod_print_list (FILE *f, json_t *o, bool longopt)
 {
     size_t index;
     json_t *value;
     const char *name;
-    int size;
-    const char *digest;
+    const char *path;
     int idle;
     int status;
     json_t *services;
+    int recvqueue;
+    int sendqueue;
 
     json_array_foreach (o, index, value) {
-        if (json_unpack (value, "{s:s s:i s:s s:i s:i s:o}",
+        recvqueue = 0;
+        sendqueue = 0;
+        if (json_unpack (value, "{s:s s:s s:i s:i s:o s?i s?i}",
                          "name", &name,
-                         "size", &size,
-                         "digest", &digest,
+                         "path", &path,
                          "idle", &idle,
                          "status", &status,
-                         "services", &services) < 0)
-            log_msg_exit ("Erorr parsing lsmod response");
+                         "services", &services,
+                         "recvqueue", &recvqueue,
+                         "sendqueue", &sendqueue) < 0)
+            log_msg_exit ("Error parsing lsmod response");
         if (!json_is_array (services))
-            log_msg_exit ("Erorr parsing lsmod services array");
+            log_msg_exit ("Error parsing lsmod services array");
         lsmod_print_entry (f,
                            name,
-                           size,
-                           digest,
+                           path,
                            idle,
                            status,
-                           services);
+                           services,
+                           sendqueue,
+                           recvqueue,
+                           longopt);
     }
 }
 
-/* Fetch lsmod data from one or more ranks.  Each lsmod response returns
- * an array of module records.  The records are merged to produce a hash
- * by module SHA1 digest (computed over module file) + services list.
- * Each hash entry is then displayed as a line of output.
- */
 int cmd_list (optparse_t *p, int argc, char **argv)
 {
-    char *service = "cmb";
-    char *topic;
+    bool longopt = optparse_hasopt (p, "long");
     flux_future_t *f;
     flux_t *h;
     int n;
     json_t *o;
 
-    if ((n = optparse_option_index (p)) < argc - 1) {
+    if ((n = optparse_option_index (p)) < argc) {
         optparse_print_usage (p);
         exit (1);
     }
-    if (n < argc)
-        service = argv[n++];
     if (!(h = flux_open (NULL, 0)))
         log_err_exit ("flux_open");
 
-    topic = xasprintf ("%s.lsmod", service);
-    if (!(f = flux_rpc (h, topic, NULL, FLUX_NODEID_ANY, 0)))
-        log_err_exit ("%s", topic);
-    if (flux_rpc_get_unpack (f, "{s:o}", "mods", &o) < 0)
-        log_err_exit ("%s", topic);
-    if (!json_is_array (o))
-        log_msg_exit ("%s: module list is not an array", topic);
-    lsmod_print_header (stdout);
-    lsmod_print_list (stdout, o);
+    if (!(f = flux_rpc (h, "module.list", NULL, FLUX_NODEID_ANY, 0))
+        || flux_rpc_get_unpack (f, "{s:o}", "mods", &o) < 0)
+        log_err_exit ("list");
+    lsmod_print_header (stdout, longopt);
+    lsmod_print_list (stdout, o, longopt);
     flux_future_destroy (f);
-    free (topic);
     flux_close (h);
     return (0);
 }
@@ -566,27 +608,24 @@ static void parse_json (optparse_t *p, const char *json_str)
     /* If --parse OBJNAME was provided, walk to that
      * portion of the returned object.
      */
-    o = obj;
     if ((objname = optparse_get_str (p, "parse", NULL))) {
-        char *cpy = xstrdup (objname);
-        char *name, *saveptr = NULL, *a1 = cpy;
-        while ((name = strtok_r (a1, ".", &saveptr))) {
-            if (!(o = json_object_get (o, name)))
-                log_msg_exit ("`%s' not found in response", objname);
-            a1 = NULL;
-        }
-        free (cpy);
+        if (!(o = jpath_get (obj, objname)))
+            log_msg_exit ("`%s' not found in response", objname);
     }
+    else
+        o = obj;
 
     /* Display the resulting object/value, optionally forcing
-     * the type to int or dobule, and optionally scaling the result.
+     * the type to int or double, and optionally scaling the result.
      */
     scale = optparse_get_double (p, "scale", 1.0);
     typestr = optparse_get_str (p, "type", NULL);
-    if (json_typeof (o) == JSON_INTEGER || (typestr && !strcmp (typestr, "int"))) {
+    if (json_typeof (o) == JSON_INTEGER
+        || (typestr && streq (typestr, "int"))) {
         double d = json_number_value (o);
         printf ("%d\n", (int)(d * scale));
-    } else if (json_typeof (o) == JSON_REAL || (typestr && !strcmp (typestr, "double"))) {
+    } else if (json_typeof (o) == JSON_REAL
+        || (typestr && streq (typestr, "double"))) {
         double d = json_number_value (o);
         printf ("%lf\n", d * scale);
     } else {
@@ -613,20 +652,20 @@ int cmd_stats (optparse_t *p, int argc, char **argv)
         optparse_print_usage (p);
         exit (1);
     }
-    service = n < argc ? argv[n++] : "cmb";
+    service = n < argc ? argv[n++] : "broker";
     nodeid = FLUX_NODEID_ANY;
 
     if (!(h = flux_open (NULL, 0)))
         log_err_exit ("flux_open");
 
     if (optparse_hasopt (p, "clear")) {
-        topic = xasprintf ("%s.stats.clear", service);
+        topic = xasprintf ("%s.stats-clear", service);
         if (!(f = flux_rpc (h, topic, NULL, nodeid, 0)))
             log_err_exit ("%s", topic);
         if (flux_future_get (f, NULL) < 0)
             log_err_exit ("%s", topic);
     } else if (optparse_hasopt (p, "clear-all")) {
-        topic = xasprintf ("%s.stats.clear", service);
+        topic = xasprintf ("%s.stats-clear", service);
         flux_msg_t *msg = flux_event_encode (topic, NULL);
         if (!msg)
             log_err_exit ("creating event");
@@ -635,15 +674,27 @@ int cmd_stats (optparse_t *p, int argc, char **argv)
         flux_msg_destroy (msg);
     } else if (optparse_hasopt (p, "rusage")) {
         topic = xasprintf ("%s.rusage", service);
-        if (!(f = flux_rpc (h, topic, NULL, nodeid, 0)))
-            log_err_exit ("%s", topic);
+        const char *optarg = optparse_get_str (p, "rusage", NULL);
+        if (optarg) {
+            if (!(f = flux_rpc_pack (h,
+                                     topic,
+                                     nodeid,
+                                     0,
+                                     "{s:s}",
+                                     "who", optarg)))
+                log_err_exit ("%s", topic);
+        }
+        else {
+            if (!(f = flux_rpc (h, topic, NULL, nodeid, 0)))
+                log_err_exit ("%s", topic);
+        }
         if (flux_rpc_get (f, &json_str) < 0)
-            log_err_exit ("%s", topic);
+            log_msg_exit ("%s: %s", topic, future_strerror (f, errno));
         if (!json_str)
             log_errn_exit (EPROTO, "%s", topic);
         parse_json (p, json_str);
     } else {
-        topic = xasprintf ("%s.stats.get", service);
+        topic = xasprintf ("%s.stats-get", service);
         if (!(f = flux_rpc (h, topic, NULL, nodeid, 0)))
             log_err_exit ("%s", topic);
         if (flux_rpc_get (f, &json_str) < 0)
@@ -700,6 +751,364 @@ int cmd_debug (optparse_t *p, int argc, char **argv)
     return (0);
 }
 
+struct typemap {
+    const char *s;
+    int type;
+};
+
+static struct typemap typemap[] = {
+    { ">", FLUX_MSGTYPE_REQUEST },
+    { "<", FLUX_MSGTYPE_RESPONSE},
+    { "e", FLUX_MSGTYPE_EVENT},
+    { "c", FLUX_MSGTYPE_CONTROL},
+};
+
+static const char *type2str (int type)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE (typemap); i++)
+        if ((type & typemap[i].type))
+            return typemap[i].s;
+    return "?";
+}
+
+enum {
+    TRACE_COLOR_EVENT = FLUX_MSGTYPE_EVENT,
+    TRACE_COLOR_REQUEST = FLUX_MSGTYPE_REQUEST,
+    TRACE_COLOR_RESPONSE = FLUX_MSGTYPE_RESPONSE,
+    TRACE_COLOR_CONTROL = FLUX_MSGTYPE_CONTROL,
+    TRACE_COLOR_TIME = 0x10,
+    TRACE_COLOR_TIMEBREAK = 0x11,
+    TRACE_COLOR_RESPONSE_FAIL = 0x12,
+};
+
+static const char *trace_colors[] = {
+    [TRACE_COLOR_EVENT]         = ANSI_COLOR_YELLOW,
+    [TRACE_COLOR_REQUEST]       = ANSI_COLOR_BOLD_BLUE,
+    [TRACE_COLOR_RESPONSE]      = ANSI_COLOR_CYAN,
+    [TRACE_COLOR_CONTROL]       = ANSI_COLOR_BOLD,
+    [TRACE_COLOR_TIME]          = ANSI_COLOR_GREEN,
+    [TRACE_COLOR_TIMEBREAK]     = ANSI_COLOR_BOLD ANSI_COLOR_GREEN,
+    [TRACE_COLOR_RESPONSE_FAIL] = ANSI_COLOR_BOLD ANSI_COLOR_RED,
+};
+
+static const char *months[] = {
+    "Jan",
+    "Feb",
+    "Mar",
+    "Apr",
+    "May",
+    "Jun",
+    "Jul",
+    "Aug",
+    "Sep",
+    "Oct",
+    "Nov",
+    "Dec",
+    NULL
+};
+
+struct trace_ctx {
+    int color;
+    json_t *names;
+    int max_name_len;
+    struct flux_match match;
+    int delta;
+    double last_sec;
+    double last_timestamp;
+};
+
+static const char *trace_color (struct trace_ctx *ctx, int type)
+{
+    if (ctx->color)
+        return trace_colors[type];
+    return "";
+}
+
+static const char *trace_color_reset (struct trace_ctx *ctx)
+{
+    if (ctx->color)
+        return ANSI_COLOR_RESET;
+    return "";
+}
+
+static void trace_print_human_timestamp (struct trace_ctx *ctx,
+                                         double timestamp)
+{
+
+    if ((time_t)(timestamp/60) == (time_t)(ctx->last_timestamp/60)) {
+        /* Within same minute, print offset in sec */
+        printf ("%s[%+11.6f]%s ",
+                trace_color (ctx, TRACE_COLOR_TIME),
+                timestamp - ctx->last_timestamp
+                    + fmod (ctx->last_timestamp, 60.),
+                trace_color_reset (ctx));
+    }
+    else {
+        struct tm tm;
+        time_t t = timestamp;
+
+        localtime_r (&t, &tm);
+
+        /* New minute, print datetime */
+        printf ("%s[%s%02d %02d:%02d]%s ",
+                trace_color (ctx, TRACE_COLOR_TIMEBREAK),
+                months [tm.tm_mon],
+                tm.tm_mday,
+                tm.tm_hour,
+                tm.tm_min,
+                trace_color_reset (ctx));
+        ctx->last_timestamp  = timestamp;
+    }
+}
+
+static void trace_print_human (struct trace_ctx *ctx,
+                               double timestamp,
+                               int message_type,
+                               int errnum,
+                               const char *s,
+                               const char *payload_str)
+{
+    if (message_type == FLUX_MSGTYPE_RESPONSE && errnum > 0)
+        message_type = TRACE_COLOR_RESPONSE_FAIL;
+    trace_print_human_timestamp (ctx, timestamp);
+    printf (" %s%s%s%s%s\n",
+            trace_color (ctx, message_type),
+            s,
+            payload_str ? "\n" : "",
+            payload_str ? payload_str : "",
+            trace_color_reset (ctx));
+}
+
+static void trace_print_timestamp (struct trace_ctx *ctx, double timestamp)
+{
+    time_t t = timestamp;
+    struct tm tm;
+    char buf[64] = "";
+
+    localtime_r (&t, &tm);
+
+    strftime (buf, sizeof (buf), "%Y-%m-%dT%T", &tm);
+    printf ("%s%s.%.3d%s",
+            trace_color (ctx, TRACE_COLOR_TIME),
+            buf,
+            (int)(1e3 * (timestamp - t)),
+            trace_color_reset (ctx));
+}
+
+
+static void trace_print (struct trace_ctx *ctx,
+                         double timestamp,
+                         int message_type,
+                         int errnum,
+                         const char *s,
+                         const char *payload_str)
+{
+    if (message_type == FLUX_MSGTYPE_RESPONSE && errnum > 0)
+        message_type = TRACE_COLOR_RESPONSE_FAIL;
+    trace_print_timestamp (ctx, timestamp);
+    printf (" %s%s%s%s%s\n",
+            trace_color (ctx, message_type),
+            s,
+            payload_str ? "\n" : "",
+            payload_str ? payload_str : "",
+            trace_color_reset (ctx));
+}
+
+static int trace_use_color (optparse_t *p)
+{
+    const char *when;
+    int color;
+
+    if (!(when = optparse_get_str (p, "color", "auto")))
+        when = "always";
+    if (streq (when, "always"))
+        color = 1;
+    else if (streq (when, "never"))
+        color = 0;
+    else if (streq (when, "auto"))
+        color = isatty (STDOUT_FILENO) ? 1 : 0;
+    else
+        log_msg_exit ("Invalid argument to --color: '%s'", when);
+    return color;
+}
+
+static int parse_name_list (json_t **result, int ac, char **av)
+{
+    json_t *a;
+    int maxlen = 0;
+
+    if (!(a = json_array ()))
+        goto nomem;
+    for (int i = 0; i < ac; i++) {
+        json_t *o;
+        if (!(o = json_string (av[i]))
+            || json_array_append_new (a, o) < 0) {
+            json_decref (o);
+            goto nomem;
+        }
+        size_t len = strlen (av[i]);
+        if (maxlen < len)
+            maxlen = len;
+    }
+    *result = a;
+    return maxlen;
+nomem:
+    json_decref (a);
+    errno = ENOMEM;
+    return -1;
+}
+
+static void trace_ctx_init (struct trace_ctx *ctx,
+                            optparse_t *p,
+                            int ac,
+                            char *av[])
+{
+    int n = optparse_option_index (p);
+
+    memset (ctx, 0, sizeof (*ctx));
+
+    if (n == ac) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    ctx->max_name_len = parse_name_list (&ctx->names, ac - n, av + n);
+    if (ctx->max_name_len < 0)
+        log_err_exit ("could not parse module name list");
+    if (optparse_hasopt (p, "delta")) {
+        if (!optparse_hasopt (p, "human"))
+            log_msg_exit ("--delta can only be used with --human");
+        ctx->delta = 1;
+    }
+    ctx->match = FLUX_MATCH_ANY;
+    ctx->match.topic_glob = optparse_get_str (p, "topic", "*");
+    ctx->color = trace_use_color (p); // borrowed from status subcommand
+    if (optparse_hasopt (p, "type")) {
+        const char *arg;
+
+        optparse_getopt_iterator_reset (p, "type");
+        ctx->match.typemask = 0;
+        while ((arg = optparse_getopt_next (p, "type"))) {
+            if (streq (arg, "request"))
+                ctx->match.typemask |= FLUX_MSGTYPE_REQUEST;
+            else if (streq (arg, "response"))
+                ctx->match.typemask |= FLUX_MSGTYPE_RESPONSE;
+            else if (streq (arg, "event"))
+                ctx->match.typemask |= FLUX_MSGTYPE_EVENT;
+            else if (streq (arg, "control"))
+                ctx->match.typemask |= FLUX_MSGTYPE_CONTROL;
+            else
+                log_msg_exit ("valid types: request, response, event, control");
+        }
+    }
+}
+
+static void sighandler (int arg)
+{
+    exit (0);
+}
+
+int cmd_trace (optparse_t *p, int ac, char *av[])
+{
+    flux_t *h;
+    flux_future_t *f;
+    struct trace_ctx ctx;
+
+    trace_ctx_init (&ctx, p, ac, av);
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (!(f = flux_rpc_pack (h,
+                             "module.trace",
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_STREAMING,
+                             "{s:i s:s s:O s:b}",
+                             "typemask", ctx.match.typemask,
+                             "topic_glob", ctx.match.topic_glob,
+                             "names", ctx.names,
+                             "full", optparse_hasopt (p, "full") ? 1 : 0)))
+        log_err_exit ("error sending module.trace request");
+
+    signal (SIGINT, sighandler);
+    signal (SIGTERM, sighandler);
+
+    do {
+        double timestamp;
+        const char *prefix;
+        const char *name;
+        int type;
+        const char *topic;
+        int payload_size;
+        json_t *payload_json = json_null ();
+        char *payload_tmp_str = NULL;
+        const char *payload_str = NULL;
+        int errnum = 0;
+        const char *errstr = NULL;
+        char buf[160];
+
+        if (flux_rpc_get_unpack (f,
+                                 "{s:F s:s s:s s:i s:s s:i s?o s?i s?s}",
+                                 "timestamp", &timestamp,
+                                 "prefix", &prefix,
+                                 "name", &name,
+                                 "type", &type,
+                                 "topic", &topic,
+                                 "payload_size", &payload_size,
+                                 "payload", &payload_json,
+                                 "errnum", &errnum,
+                                 "errstr", &errstr) < 0)
+            log_msg_exit ("%s", future_strerror (f, errno));
+
+        if (errnum > 0) {
+            if (errstr && strlen (errstr) > 0)
+                payload_str = errstr;
+        }
+        else if (!json_is_null (payload_json)) {
+            payload_tmp_str = json_dumps (payload_json, JSON_INDENT(2));
+            payload_str = payload_tmp_str;
+        }
+
+        snprintf (buf,
+                  sizeof (buf),
+                  "%*s %s %s %s [%s]",
+                  ctx.max_name_len,
+                  name,
+                  prefix,
+                  type2str (type),
+                  topic,
+                  encode_size (payload_size));
+
+        if (type == FLUX_MSGTYPE_RESPONSE) {
+            const char *desc = strerrorname_np (errnum);
+            size_t len = strlen (buf);
+
+            if (errnum == 0)
+                snprintf (buf + len, sizeof (buf) - len, " success");
+            else if (desc && !isdigit (desc[0]))
+                snprintf (buf + len, sizeof (buf) - len, " %s", desc);
+            else
+                snprintf (buf + len, sizeof (buf) - len, " errno %d", errnum);
+        }
+
+        if (optparse_hasopt (p, "human"))
+            trace_print_human (&ctx, timestamp, type, errnum, buf, payload_str);
+        else
+            trace_print (&ctx, timestamp, type, errnum, buf, payload_str);
+        fflush (stdout);
+
+        free (payload_tmp_str);
+
+        flux_future_reset (f);
+    } while (1);
+
+    json_decref (ctx.names);
+    flux_future_destroy (f);
+    flux_close (h);
+
+}
+
 /*
  * vi: ts=4 sw=4 expandtab
  */
diff --git a/src/cmd/flux-pgrep.py b/src/cmd/flux-pgrep.py
new file mode 100755
index 000000000000..b2c4a52d6388
--- /dev/null
+++ b/src/cmd/flux-pgrep.py
@@ -0,0 +1,287 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+import os
+import re
+import sys
+from itertools import islice
+from pathlib import PurePath
+
+import flux
+from flux.job import JobID, JobInfoFormat, JobList
+from flux.util import FilterActionSetUpdate, UtilConfig
+
+PROGRAM = PurePath(sys.argv[0]).stem
+LOGGER = logging.getLogger(PROGRAM)
+
+
+class FluxPgrepConfig(UtilConfig):
+    """flux-pgrep configuration class"""
+
+    builtin_formats = {
+        "default": {
+            "description": "Default flux-pgrep format string",
+            "format": "{id.f58}",
+        },
+        "full": {
+            "description": "full flux-pgrep format string",
+            "format": (
+                "{id.f58:>12} ?:{queue:<8.8} {username:<8.8} {name:<10.10+} "
+                "{status_abbrev:>2.2} {ntasks:>6} {nnodes:>6h} "
+                "{contextual_time!F:>8h} {contextual_info}"
+            ),
+        },
+        "long": {
+            "description": "Extended flux-pgrep format string",
+            "format": (
+                "{id.f58:>12} ?:{queue:<8.8} {username:<8.8} {name:<10.10+} "
+                "{status:>9.9} {ntasks:>6} {nnodes:>6h} "
+                "{t_submit!d:%b%d %R::>12} {t_remaining!F:>12h} "
+                "{contextual_time!F:>8h} {contextual_info}"
+            ),
+        },
+        "deps": {
+            "description": "Show job urgency, priority, and dependencies",
+            "format": (
+                "{id.f58:>12} ?:{queue:<8.8} {name:<10.10+} {urgency:<3} "
+                "{priority:<12} {state:<8.8} {dependencies}"
+            ),
+        },
+    }
+
+    def __init__(self):
+        initial_dict = {"formats": dict(self.builtin_formats)}
+        super().__init__(
+            name="flux-jobs", toolname="flux-pgrep", initial_dict=initial_dict
+        )
+
+    def validate(self, path, config):
+        """Validate a loaded flux-pgrep config file as dictionary"""
+        for key, value in config.items():
+            if key == "formats":
+                self.validate_formats(path, value)
+            else:
+                raise ValueError(f"{path}: invalid key {key}")
+
+    def get_default(self):
+        return self.builtin_formats["default"]["format"]
+
+
+class JobPgrep:
+    def __init__(self, args):
+        self.regex = None
+        self.jobids = []
+        for arg in args:
+            if ":" not in arg and ".." in arg:
+                #  X..Y with no : is translated to a range of jobids
+                try:
+                    self.jobids = list(map(JobID, arg.split("..")))
+                    continue
+                except ValueError:
+                    # If terms cannot be translated to JobID, then fall
+                    # back to trying as a pattern
+                    pass
+            if self.regex:
+                raise ValueError("Only one pattern can be provided")
+            if arg.startswith("name:"):
+                arg = arg[5:]
+            self.regex = re.compile(arg)
+
+    def match(self, job):
+        if self.regex and not self.regex.search(job.name):
+            return False
+        if self.jobids:
+            if job.id > self.jobids[1] or job.id < self.jobids[0]:
+                return False
+        return True
+
+
+def fetch_jobs(args, flux_handle=None):
+    if not flux_handle:
+        flux_handle = flux.Flux()
+
+    if args.A:
+        args.user = "all"
+    if args.a:
+        args.filter.update(["pending", "running", "inactive"])
+    if not args.filter:
+        args.filter = {"pending", "running"}
+
+    jobs_rpc = JobList(
+        flux_handle,
+        filters=args.filter,
+        user=args.user,
+        max_entries=args.max_entries,
+        queue=args.queue,
+    )
+    jobs = jobs_rpc.jobs()
+
+    #  Print all errors accumulated in JobList RPC:
+    try:
+        for err in jobs_rpc.errors:
+            print(err, file=sys.stderr)
+    except EnvironmentError:
+        # Ignore errors printing to stderr
+        pass
+
+    return jobs
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        prog=PROGRAM, formatter_class=flux.util.help_formatter()
+    )
+    parser.add_argument(
+        "-a",
+        action="store_true",
+        help=(
+            "Target jobs in all states"
+            if PROGRAM == "flux-pgrep"
+            else argparse.SUPPRESS
+        ),
+    )
+    parser.add_argument("-A", action="store_true", help="Target all users")
+    parser.add_argument(
+        "-u",
+        "--user",
+        type=str,
+        metavar="[USERNAME|UID]",
+        default=str(os.getuid()),
+        help="Limit output to specific username or userid "
+        '(Specify "all" for all users)',
+    )
+    parser.add_argument(
+        "-f",
+        "--filter",
+        action=FilterActionSetUpdate,
+        metavar="STATE|RESULT",
+        default=set(),
+        help="List jobs with specific job state or result",
+    )
+    parser.add_argument(
+        "-q",
+        "--queue",
+        type=FilterActionSetUpdate,
+        metavar="QUEUE,...",
+        help="Limit output to specific queue or queues",
+    )
+    parser.add_argument(
+        "-c",
+        "--count",
+        type=int,
+        metavar="N",
+        default=99999,
+        help="Limit output to the first N matches",
+    )
+    parser.add_argument(
+        "--max-entries",
+        type=int,
+        metavar="N",
+        default=1000,
+        help="Limit number of searched jobs to N entries (default: 1000)",
+    )
+    if PROGRAM == "flux-pgrep":
+        parser.add_argument(
+            "-n",
+            "--no-header",
+            action="store_true",
+            help="Suppress printing of header line",
+        )
+        parser.add_argument(
+            "-o",
+            "--format",
+            type=str,
+            default="default",
+            metavar="FORMAT",
+            help="Specify output format using Python's string format syntax "
+            + " or a defined format by name (use 'help' to get a list of names)",
+        )
+    else:
+        parser.add_argument(
+            "--wait", action="store_true", help="Wait for jobs to finish after cancel"
+        )
+    parser.add_argument(
+        "expression",
+        metavar="EXPRESSION",
+        type=str,
+        nargs="+",
+        help="job name pattern or id range",
+    )
+    return parser.parse_args()
+
+
+def pkill(fh, args, jobs):
+    success = 0
+    exitcode = 0
+
+    def wait_cb(future, job):
+        future.get_dict()
+
+    def cancel_cb(future, args, job):
+        nonlocal success, exitcode
+        try:
+            future.get()
+            success = success + 1
+            if args.wait:
+                flux.job.result_async(fh, job.id).then(wait_cb, job)
+        except OSError as exc:
+            exitcode = 1
+            LOGGER.error(f"{job.id}: cancel: {exc}")
+
+    for job in jobs:
+        flux.job.cancel_async(fh, job.id).then(cancel_cb, args, job)
+    fh.reactor_run()
+    LOGGER.info("Canceled %d job%s", success, "s" if success != 1 else "")
+    sys.exit(exitcode)
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(sys.stdout.fileno(), "w", encoding="utf8")
+    args = parse_args()
+
+    if PROGRAM == "flux-pgrep":
+        #  Process format before fetching jobs in case user makes an
+        #  error, or 'help' is specified
+        fmt = FluxPgrepConfig().load().get_format_string(args.format)
+        try:
+            formatter = JobInfoFormat(fmt)
+        except ValueError as err:
+            raise ValueError("Error in user format: " + str(err))
+
+    fh = flux.Flux()
+    try:
+        pgrep = JobPgrep(args.expression)
+    except (ValueError, SyntaxError, TypeError, re.error) as exc:
+        LOGGER.error(f"expression error: {exc}")
+        sys.exit(2)
+
+    jobs = list(islice(filter(pgrep.match, fetch_jobs(args, fh)), args.count))
+
+    #  Exit with exitcode 1 when no jobs match
+    if not jobs:
+        sys.exit(1)
+
+    if PROGRAM == "flux-pkill":
+        pkill(fh, args, jobs)
+
+    sformatter = JobInfoFormat(formatter.filter(jobs))
+
+    # "default" can be overridden by environment variable, so check if
+    # it's different than the builtin default
+    if (
+        args.format == "default"
+        and FluxPgrepConfig().get_default() == sformatter.get_format(orig=True)
+    ):
+        args.no_header = True
+
+    sformatter.print_items(jobs, no_header=args.no_header)
diff --git a/src/cmd/flux-ping.c b/src/cmd/flux-ping.c
index 86eed2684580..5dc76f1a128c 100644
--- a/src/cmd/flux-ping.c
+++ b/src/cmd/flux-ping.c
@@ -22,6 +22,7 @@
 #include "src/common/libutil/monotime.h"
 #include "src/common/libutil/tstat.h"
 #include "src/common/libutil/log.h"
+#include "ccan/str/str.h"
 
 struct ping_ctx {
     double interval;    /* interval between sends, in seconds */
@@ -111,12 +112,12 @@ void ping_continuation (flux_future_t *f, void *arg)
     int seq;
     struct ping_data *pdata = flux_future_aux_get (f, "ping");
     tstat_t *tstat = pdata->tstat;
-    uint32_t rolemask, userid;
+    uint32_t rolemask, userid, rank;
     char ubuf[32];
     char rbuf[32];
 
     if (flux_rpc_get_unpack (f,
-                             "{ s:i s:I s:I s:s s:s s:i s:i !}",
+                             "{ s:i s:I s:I s:s s:s s:i s:i s:i !}",
                              "seq",
                              &seq,
                              "time.tv_sec",
@@ -130,12 +131,14 @@ void ping_continuation (flux_future_t *f, void *arg)
                              "userid",
                              &userid,
                              "rolemask",
-                             &rolemask) < 0)
+                             &rolemask,
+                             "rank",
+                             &rank) < 0)
         log_err_exit ("%s%s",
                       rank_bang_str (ctx->nodeid, rbuf, sizeof (rbuf)),
                       ctx->topic);
 
-    if (strcmp (ctx->pad, pad) != 0)
+    if (!streq (ctx->pad, pad))
         log_msg_exit ("%s%s: padding contents invalid",
                       rank_bang_str (ctx->nodeid, rbuf, sizeof (rbuf)),
                       ctx->topic);
@@ -152,7 +155,7 @@ void ping_continuation (flux_future_t *f, void *arg)
     pdata->rpc_count++;
 
     printf ("%s%s pad=%zu%s seq=%d time=%0.3f ms (%s)\n",
-            rank_bang_str (ctx->nodeid, rbuf, sizeof (rbuf)),
+            rank_bang_str (rank, rbuf, sizeof (rbuf)),
             ctx->topic,
             strlen (ctx->pad),
             ctx->userid_flag
@@ -214,89 +217,117 @@ void timer_cb (flux_reactor_t *r, flux_watcher_t *w, int revents, void *arg)
     }
 }
 
-int parse_nodeid (const char *s, uint32_t *np)
+int parse_nodeid (struct ping_ctx *ctx,
+                  const char *input,
+                  uint32_t *nodeid,
+                  char **nodeidstr,
+                  char **suffixstr)
 {
-    uint32_t n;
+    int rank;
     char *endptr;
 
+    if (streq (input, "any")) {
+        *nodeid = FLUX_NODEID_ANY;
+        (*nodeidstr) = xasprintf ("%s", "any");
+        return 0;
+    }
+    if (streq (input, "upstream")) {
+        *nodeid = FLUX_NODEID_UPSTREAM;
+        (*nodeidstr) = xasprintf ("%s", "upstream");
+        return 0;
+    }
+    if ((rank = flux_get_rankbyhost (ctx->h, input)) >= 0) {
+        *nodeid = rank;
+        (*nodeidstr) = xasprintf ("%s", input);
+        (*suffixstr) = xasprintf (" (rank %d)", rank);
+        return 0;
+    }
     errno = 0;
-    n = strtoul (s, &endptr, 10);
-    if (errno != 0 || *endptr != '\0')
-        return -1;
-    *np = n;
-    return 0;
+    rank = strtol (input, &endptr, 10);
+    if (errno == 0 && *endptr == '\0' && rank >= 0) {
+        *nodeid = rank;
+        (*nodeidstr) = xasprintf ("%d", rank);
+        return 0;
+    }
+    return -1;
 }
 
-/* Parse 's' to obtain the ping topic string and (optionally) nodeid.
- * If nodeid is non-NULL, then try to parse a nodeid from 's'
- * looking for nodeid!service expressions (including nodeid=any and upstream),
- * or just a number in which case the service is assumed to be "cmb".
- */
-void parse_service (const char *s, uint32_t *nodeid, char **topic)
+void parse_target (struct ping_ctx *ctx, const char *target, char **header)
 {
-
-    char *cpy = NULL;
-    char *service = NULL;
-
-    if (nodeid) {
-        if (!(cpy = strdup (s)))
-            log_err_exit ("strdup");
-        if ((service = strchr (cpy, '!')))
-            *service++ = '\0';
-        else
-            service = "cmb";
-        if (!strcmp (cpy, "any"))
-            *nodeid = FLUX_NODEID_ANY;
-        else if (!strcmp (cpy, "upstream"))
-            *nodeid = FLUX_NODEID_UPSTREAM;
-        else if (parse_nodeid (cpy, nodeid) < 0) {
-            *nodeid = FLUX_NODEID_ANY; // back where we started
-            service = NULL;
-        }
+    char *cpy;
+    char *service;
+    char *nodeidstr = NULL;
+    char *suffixstr = NULL;
+
+    if (!(cpy = strdup (target)))
+        log_err_exit ("out of memory");
+
+    /* TARGET specifies nodeid!service */
+    if ((service = strchr (cpy, '!'))) {
+        *service++ = '\0';
+        if (parse_nodeid (ctx, cpy, &ctx->nodeid, &nodeidstr, &suffixstr) < 0)
+            log_msg_exit ("invalid nodeid/host: '%s'", cpy);
+    }
+    /* TARGET only specifies service, assume nodeid is ANY */
+    else if (parse_nodeid (ctx, cpy, &ctx->nodeid, &nodeidstr, &suffixstr) < 0) {
+        service = cpy;
+        ctx->nodeid = FLUX_NODEID_ANY;
+        nodeidstr = xasprintf ("%s", "any");
     }
-    if (asprintf (topic, "%s.ping", service ? service : s) < 0)
-        log_err_exit ("asprintf");
+    /* TARGET only specifies nodeid, assume service is "broker" */
+    else
+        service = "broker";
+    ctx->topic = xasprintf ("%s.ping", service);
+    (*header) = xasprintf ("flux-ping %s!%s%s",
+                           nodeidstr,
+                           service,
+                           suffixstr ? suffixstr : "");
     free (cpy);
+    free (nodeidstr);
+    free (suffixstr);
 }
 
 int main (int argc, char *argv[])
 {
     int pad_bytes;
+    char *cmdusage = "[OPTIONS] TARGET";
     flux_watcher_t *tw = NULL;
     optparse_t *opts;
     struct ping_ctx ctx;
     int optindex;
+    char *target;
+    char *header = NULL;
 
     memset (&ctx, 0, sizeof (ctx));
 
     log_init ("flux-ping");
 
     opts = optparse_create ("flux-ping");
+    if (optparse_set (opts, OPTPARSE_USAGE, cmdusage) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_set (USAGE)");
     if (optparse_add_option_table (opts, cmdopts) != OPTPARSE_SUCCESS)
         log_msg_exit ("optparse_add_option_table");
     if ((optindex = optparse_parse_args (opts, argc, argv)) < 0)
         exit (1);
+    if (optindex != argc - 1) {
+        optparse_print_usage (opts);
+        exit (1);
+    }
+    if (!(target = strdup (argv[optindex])))
+        log_msg_exit ("out of memory");
 
-    pad_bytes = optparse_get_int (opts, "pad", 0);
-    if (pad_bytes < 0)
-        log_msg_exit ("pad must be >= 0");
+    pad_bytes = optparse_get_size_int (opts, "pad", "0");
 
     ctx.nodeid = FLUX_NODEID_ANY;
     if (optparse_hasopt (opts, "rank")) {
         const char *s = optparse_get_str (opts, "rank", NULL);
         if (!s)
             log_msg_exit ("error parsing --rank option");
-        if (!strcmp (s, "any"))
-            ctx.nodeid = FLUX_NODEID_ANY;
-        else if (!strcmp (s, "upstream"))
-            ctx.nodeid = FLUX_NODEID_UPSTREAM;
-        else {
-            char *endptr;
-            errno = 0;
-            ctx.nodeid = strtoul (s, &endptr, 10);
-            if (errno != 0 || *endptr != '\0')
-                log_msg_exit ("error parsing --rank option");
-        }
+        if (strchr (target, '!'))
+            log_msg_exit ("--rank and TARGET both try to specify a nodeid");
+        char *new_target = xasprintf ("%s!%s", s, target);
+        free (target);
+        target = new_target;
     }
 
     ctx.interval = optparse_get_duration (opts, "interval", 1.0);
@@ -313,26 +344,22 @@ int main (int argc, char *argv[])
     if (ctx.batch && ctx.count == 0)
         log_msg_exit ("--batch should only be used with --count");
 
-    if (optindex != argc - 1) {
-        optparse_print_usage (opts);
-        exit (1);
-    }
-    parse_service (argv[optindex++],
-                   optparse_hasopt (opts, "rank") ? NULL : &ctx.nodeid,
-                   &ctx.topic);
-
     /* Create null terminated pad string for reuse in each message.
      * By default it's the empty string.
      */
     ctx.pad = xzmalloc (pad_bytes + 1);
     memset (ctx.pad, 'p', pad_bytes);
 
-
     if (!(ctx.h = flux_open (NULL, 0)))
         log_err_exit ("flux_open");
     if (!(ctx.reactor = flux_get_reactor (ctx.h)))
         log_err_exit ("flux_get_reactor");
 
+    /* Set ctx.nodeid and ctx.topic from TARGET argument
+     */
+    parse_target (&ctx, target, &header);
+
+    printf ("%s\n", header);
     /* In batch mode, requests are sent before reactor is started
      * to process responses.  o/w requests are set in a timer watcher.
      */
@@ -358,6 +385,8 @@ int main (int argc, char *argv[])
     free (ctx.topic);
     free (ctx.pad);
 
+    free (target);
+    free (header);
     flux_close (ctx.h);
     optparse_destroy (opts);
     log_fini ();
diff --git a/src/cmd/flux-post-job-event.py b/src/cmd/flux-post-job-event.py
new file mode 100755
index 000000000000..33b5c8bc63fc
--- /dev/null
+++ b/src/cmd/flux-post-job-event.py
@@ -0,0 +1,55 @@
+#!/bin/false
+##############################################################
+# Copyright 2024 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+
+import flux
+from flux.job import JobID
+from flux.util import TreedictAction
+
+
+def post_event(args):
+    """Post an event to the job-manager.post-event.post service"""
+
+    req = {"id": JobID(args.id), "name": args.name}
+    if args.context:
+        req["context"] = args.context
+
+    try:
+        flux.Flux().rpc("job-manager.post-event.post", req).get()
+    except FileNotFoundError:
+        raise ValueError(f"No such job {args.id}")
+
+
+LOGGER = logging.getLogger("flux-job-post-event")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    parser = argparse.ArgumentParser(prog="flux-job-post-event")
+    parser.add_argument("id", help="jobid to which event shall be posted")
+    parser.add_argument("name", help="name of event to post")
+    parser.add_argument(
+        "context",
+        help="List of key=value pairs to set as event context",
+        action=TreedictAction,
+        nargs=argparse.REMAINDER,
+    )
+    parser.set_defaults(func=post_event)
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-pstree.py b/src/cmd/flux-pstree.py
new file mode 100755
index 000000000000..ffd557e84b6a
--- /dev/null
+++ b/src/cmd/flux-pstree.py
@@ -0,0 +1,449 @@
+##############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import concurrent.futures
+import logging
+import sys
+
+import flux
+import flux.uri
+from flux.job import JobID, JobInfo, JobInfoFormat, JobList
+from flux.util import FilterAction, FilterTrueAction, Tree, YesNoAction
+
+DETAILS_FORMAT = {
+    "default": (
+        "{id.f58:>12} {username:<8.8} {status_abbrev:>2.2} {ntasks:>6h}"
+        " {nnodes:>6h} {runtime!F:>8h}"
+    ),
+    "resources": (
+        "{id.f58:>12} "
+        "{instance.resources.all.nnodes:>5} "
+        "{instance.resources.allocated.nnodes:>5} "
+        "{instance.resources.all.ncores:>5} "
+        "{instance.resources.allocated.ncores:>5} "
+        "{instance.resources.all.ngpus:>5} "
+        "{instance.resources.allocated.ngpus:>5}"
+    ),
+    "progress": (
+        "{id.f58:>12} {ntasks:>6h} {instance.stats.total:>6} "
+        "{instance.progress!P:>5} {instance.utilization!P:>5} "
+        "{instance.gpu_utilization!P:>5} {runtime!H:>10}"
+    ),
+    "stats": "{id.f58:>12} {instance.stats:^25} {runtime!H:>10}",
+}
+
+LABEL_FORMAT = {
+    "default": "{name}",
+    "all": "{name}:{status_abbrev}",
+}
+PARENT_FORMAT = {
+    "default": "{name}",
+    "all": "{name}",
+}
+
+LOGGER = logging.getLogger("flux-pstree")
+
+
+class TreeFormatter:
+    """
+    Initialize formatters for Tree labels, parent labels, and optional
+    prefix
+    """
+
+    def __init__(self, args):
+
+        #  If -o, --label used, then also set parent label format:
+        if args.label and not args.parent_label:
+            args.parent_label = args.label
+
+        #  If -a, --all, then use "all" specific labels unless labels
+        #   were set explicitly:
+        if args.all:
+            if not args.label:
+                args.label = LABEL_FORMAT["all"]
+            if not args.parent_label:
+                args.parent_label = PARENT_FORMAT["all"]
+
+        #  O/w, set default labels:
+        if not args.label:
+            args.label = LABEL_FORMAT["default"]
+        if not args.parent_label:
+            args.parent_label = PARENT_FORMAT["default"]
+
+        #  For -p, --parent-ids, prepend jobid to parent labels:
+        if args.parent_ids:
+            args.parent_label = f"{{id.f58}} {args.parent_label}"
+
+        self.label = JobInfoFormat(args.label)
+        self.parent = JobInfoFormat(args.parent_label)
+
+        #  Set prefix format if -x, --details, or --prefix-format was used
+        self.prefix = None
+        if args.extended or args.details or args.prefix_format:
+            if not args.details:
+                args.details = "default"
+            if not args.prefix_format:
+                try:
+                    args.prefix_format = DETAILS_FORMAT[args.details]
+                except KeyError:
+                    LOGGER.error("Unknown --details format '%s'", args.details)
+                    sys.exit(1)
+            self.prefix = JobInfoFormat(args.prefix_format)
+
+    def format(self, job, parent):
+        """Format provided job label (as parent label if parent == True)"""
+        if parent:
+            return self.parent.format(job)
+        return self.label.format(job)
+
+    def format_prefix(self, job):
+        """Format Tree prefix if configured, otherwise return empty string"""
+        if self.prefix:
+            return self.prefix.format(job)
+        return ""
+
+
+def process_entry(entry, formatter, filters, level, max_level, combine):
+
+    job = JobInfo(entry).get_instance_info()
+
+    # pylint: disable=comparison-with-callable
+    parent = job.uri and job.state_single == "R"
+
+    label = formatter.format(job, parent)
+    prefix = formatter.format_prefix(job)
+
+    if not parent:
+        return Tree(label, prefix)
+    return load_tree(
+        label,
+        formatter,
+        prefix=prefix,
+        uri=str(job.uri),
+        filters=filters,
+        level=level + 1,
+        max_level=max_level,
+        combine_children=combine,
+    )
+
+
+# pylint: disable=too-many-locals
+def load_tree(
+    label,
+    formatter,
+    prefix="",
+    uri=None,
+    filters=None,
+    combine_children=True,
+    max_level=999,
+    level=0,
+    skip_root=True,
+    jobids=None,
+):
+
+    #  Only apply filters below root unless no_skip_root
+    orig_filters = filters
+    if filters is None or (level == 0 and skip_root):
+        filters = ["running"]
+
+    tree = Tree(label, prefix=prefix, combine_children=combine_children)
+    if level > max_level:
+        return tree
+
+    #  Attempt to load jobs from uri
+    #  This may fail if the instance hasn't loaded the job-list module
+    #   or if the current user is not owner
+    #
+    try:
+        jobs_rpc = JobList(
+            flux.Flux(uri), ids=jobids, filters=filters, attrs=["all"]
+        ).fetch_jobs()
+        jobs = jobs_rpc.get_jobs()
+    except (OSError, FileNotFoundError):
+        return tree
+
+    #  Print all errors accumulated in JobList RPC:
+    #  fetch_jobs() may not set errors list, must check first
+    if hasattr(jobs_rpc, "errors"):
+        try:
+            for err in jobs_rpc.errors:
+                print(err, file=sys.stderr)
+        except EnvironmentError:
+            pass
+
+    #  Since the executor cannot be used recursively, start one per
+    #   loop iteration. This is very wasteful but greatly speeds up
+    #   execution with even a moderate number of jobs using the ssh:
+    #   connector. At some point this should be replaced by a global
+    #   thread pool that can work with recursive execution.
+    #
+    executor = concurrent.futures.ThreadPoolExecutor()
+    futures = []
+    for entry in jobs:
+        futures.append(
+            executor.submit(
+                process_entry,
+                entry,
+                formatter,
+                orig_filters,
+                level,
+                max_level,
+                combine_children,
+            )
+        )
+
+    for future in futures:
+        tree.append_tree(future.result())
+
+    return tree
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        prog="flux-pstree", formatter_class=flux.util.help_formatter()
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action=FilterTrueAction,
+        help="Include all jobs for current user in all states",
+    )
+    parser.add_argument(
+        "-c",
+        "--count",
+        action=FilterAction,
+        type=int,
+        metavar="N",
+        default=1000,
+        help="Limit to N jobs per instance (default 1000)",
+    )
+    parser.add_argument(
+        "-f",
+        "--filter",
+        action=FilterAction,
+        type=str,
+        metavar="STATE|RESULT",
+        default="running",
+        help="list jobs with specific job state or result",
+    )
+    parser.add_argument(
+        "-x",
+        "--extended",
+        action="store_true",
+        help="print extended details before tree output",
+    )
+    parser.add_argument(
+        "-l",
+        "--long",
+        action="store_false",
+        dest="truncate",
+        help="do not truncate long lines to terminal width",
+    )
+    parser.add_argument(
+        "-L",
+        "--level",
+        type=int,
+        metavar="N",
+        default=999,
+        help="only descend N levels",
+    )
+    parser.add_argument(
+        "-p",
+        "--parent-ids",
+        action="store_true",
+        help="include jobids in parent labels",
+    )
+    parser.add_argument(
+        "-n",
+        "--no-header",
+        action="store_true",
+        help="Suppress header with -x, --extended",
+    )
+    parser.add_argument(
+        "-X",
+        "--no-combine",
+        action="store_false",
+        dest="combine_children",
+        help="disable combination of identical children",
+    )
+    parser.add_argument(
+        "-o",
+        "--label",
+        metavar="FORMAT",
+        help="change label format (default='{name}')",
+    )
+    parser.add_argument(
+        "--parent-label",
+        metavar="FORMAT",
+        help="change label format for parent only",
+    )
+    parser.add_argument(
+        "--detail-format",
+        metavar="FORMAT",
+        help="specify output format for extended details " + "(implies -x, --extended)",
+        dest="prefix_format",
+    )
+    parser.add_argument(
+        "-d",
+        "--details",
+        metavar="NAME",
+        help="Select a named extended details format ("
+        + ",".join(DETAILS_FORMAT.keys())
+        + ")",
+    )
+    parser.add_argument(
+        "-C",
+        "--compact",
+        action="store_true",
+        help="Use compact tree connectors",
+    )
+    parser.add_argument(
+        "--ascii",
+        action="store_true",
+        help="Use ASCII character tree connectors",
+    )
+    parser.add_argument(
+        "--skip-root",
+        action=YesNoAction,
+        help="suppress or include the enclosing instance in output. "
+        + "Default is 'no' unless the -x, --extended or -d, --details "
+        + "options are used.",
+    )
+    parser.add_argument(
+        "jobids",
+        metavar="JOBID",
+        type=JobID,
+        nargs="*",
+        help="Limit output to specific jobids",
+    )
+    parser.set_defaults(filtered=False)
+    return parser.parse_args()
+
+
+class RootJobID(JobID):
+    """Mock JobID class for pstree root (enclosing instance)
+
+    Replaces the encode method of the JobID class to always return ".",
+    so that normal `job.id.X` formatting always returns "." to indicate
+    root of our tree.
+    """
+
+    def __new__(cls):
+        return int.__new__(cls, 0)
+
+    # pylint: disable=no-self-use
+    # pylint: disable=unused-argument
+    def encode(self, encoding="dec"):
+        return "."
+
+
+def get_root_jobinfo():
+    """Fetch a mock JobInfo object for the current enclosing instance"""
+
+    handle = flux.Flux()
+    size = handle.attr_get("size")
+
+    try:
+        #  If the enclosing instance has a jobid and a parent-uri, then
+        #   fill in data from job-list in the parent:
+        #
+        jobid = JobID(handle.attr_get("jobid"))
+        parent = flux.Flux(handle.attr_get("parent-uri"))
+        info = JobList(parent, ids=[jobid]).fetch_jobs().get_jobs()[0]
+    except OSError:
+        #  Make a best-effort attempt to create a mock job info dictionary
+        uri = handle.attr_get("local-uri")
+        nodelist = handle.attr_get("hostlist")
+        userid = handle.attr_get("security.owner")
+        info = dict(
+            id=0,
+            userid=int(userid),
+            state=flux.constants.FLUX_JOB_STATE_RUN,
+            name=".",
+            ntasks=int(size),
+            nnodes=int(size),
+            nodelist=nodelist,
+            annotations={"user": {"uri": uri}},
+        )
+        try:
+            info["t_run"] = float(handle.attr_get("broker.starttime"))
+        except OSError:
+            pass
+
+    #  If 'ranks' idset came from parent, it could be confusing,
+    #   rewrite ranks to be relative to current instance, i.e.
+    #   0-(size-1)
+    #
+    info["ranks"] = "0-{}".format(int(size) - 1)
+
+    #  Fetch instance-specific information for the current instance:
+    job = JobInfo(info).get_instance_info()
+
+    #  If no jobid was discovered for the root instance, use RootJobID()
+    if job.id == 0:
+        job.id = RootJobID()
+
+    return job
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+
+    sys.stdout = open(sys.stdout.fileno(), "w", encoding="utf8")
+
+    args = parse_args()
+    if args.jobids and args.filtered:
+        LOGGER.warning("filtering options ignored with jobid list")
+    if args.ascii and args.compact:
+        LOGGER.fatal("Choose only one of --ascii, --compact")
+
+    if args.all:
+        args.filter = "pending,running,inactive"
+
+    formatter = TreeFormatter(args)
+
+    #  Default for skip_root is True if there is a "prefix" format or
+    #   specific jobids are targeted, possibly overridden by the value of
+    #   --skip-root provided by user
+    #
+    skip_root = formatter.prefix is not None or args.jobids
+    if args.skip_root is not None:
+        skip_root = args.skip_root
+
+    if skip_root:
+        label = "."
+        prefix = None
+    else:
+        root = get_root_jobinfo()
+        label = formatter.format(root, True)
+        prefix = formatter.format_prefix(root)
+
+    tree = load_tree(
+        label,
+        formatter,
+        prefix=prefix,
+        filters=[args.filter],
+        max_level=args.level,
+        skip_root=skip_root,
+        combine_children=args.combine_children,
+        jobids=args.jobids,
+    )
+
+    if args.compact:
+        style = "compact"
+    elif args.ascii:
+        style = "ascii"
+    else:
+        style = "box"
+
+    if formatter.prefix and not args.no_header:
+        print(formatter.prefix.header())
+    tree.render(skip_root=skip_root, style=style, truncate=args.truncate)
diff --git a/src/cmd/flux-python b/src/cmd/flux-python
new file mode 100755
index 000000000000..13f92762f4da
--- /dev/null
+++ b/src/cmd/flux-python
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# This minimal wrapper exists to help handle systems where env -S is
+# unavailable and scripts wish to use `flux python`. Using a shebang like this
+# for flux python scripts allows them to rely only on posix to start:
+# #!/usr/bin/env flux-python
+
+exec flux python "$@"
diff --git a/src/cmd/flux-queue.c b/src/cmd/flux-queue.c
deleted file mode 100644
index 481452495817..000000000000
--- a/src/cmd/flux-queue.c
+++ /dev/null
@@ -1,450 +0,0 @@
-/************************************************************\
- * Copyright 2020 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* flux-queue.c - control job manager queue
- */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <assert.h>
-#include <jansson.h>
-#include <argz.h>
-#include <flux/core.h>
-#include <flux/optparse.h>
-
-#include "src/common/libutil/log.h"
-
-int cmd_enable (optparse_t *p, int argc, char **argv);
-int cmd_disable (optparse_t *p, int argc, char **argv);
-int cmd_start (optparse_t *p, int argc, char **argv);
-int cmd_stop (optparse_t *p, int argc, char **argv);
-int cmd_status (optparse_t *p, int argc, char **argv);
-int cmd_drain (optparse_t *p, int argc, char **argv);
-int cmd_idle (optparse_t *p, int argc, char **argv);
-
-int stdin_flags;
-
-static struct optparse_option global_opts[] =  {
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option stop_opts[] = {
-    { .name = "verbose", .key = 'v',
-      .usage = "Display more detail about internal job manager state",
-    },
-    { .name = "quiet", .key = 'q',
-      .usage = "Display only errors",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option start_opts[] = {
-    { .name = "verbose", .key = 'v',
-      .usage = "Display more detail about internal job manager state",
-    },
-    { .name = "quiet", .key = 'q',
-      .usage = "Display only errors",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option status_opts[] = {
-    { .name = "verbose", .key = 'v',
-      .usage = "Display more detail about internal job manager state",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option drain_opts[] =  {
-    { .name = "timeout", .key = 't', .has_arg = 1, .arginfo = "DURATION",
-      .usage = "timeout after DURATION",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_option idle_opts[] =  {
-    { .name = "timeout", .key = 't', .has_arg = 1, .arginfo = "DURATION",
-      .usage = "timeout after DURATION",
-    },
-    { .name = "quiet", .key = 'q', .has_arg = 0,
-      .usage = "Only display pending job count if nonzero",
-    },
-    OPTPARSE_TABLE_END
-};
-
-static struct optparse_subcommand subcommands[] = {
-    { "enable",
-      "[OPTIONS]",
-      "Enable job submission",
-      cmd_enable,
-      0,
-      NULL,
-    },
-    { "disable",
-      "[OPTIONS] [message ...]",
-      "Disable job submission",
-      cmd_disable,
-      0,
-      NULL,
-    },
-    { "start",
-      "[OPTIONS]",
-      "Start scheduling",
-      cmd_start,
-      0,
-      start_opts,
-    },
-    { "stop",
-      "[OPTIONS] [message ...]",
-      "Stop scheduling",
-      cmd_stop,
-      0,
-      stop_opts,
-    },
-    { "status",
-      "[OPTIONS]",
-      "Get queue status",
-      cmd_status,
-      0,
-      status_opts,
-    },
-    { "drain",
-      "[OPTIONS]",
-      "Wait for queue to become empty.",
-      cmd_drain,
-      0,
-      drain_opts
-    },
-    { "idle",
-      "[OPTIONS]",
-      "Wait for queue to become idle.",
-      cmd_idle,
-      0,
-      idle_opts
-    },
-    OPTPARSE_SUBCMD_END
-};
-
-int usage (optparse_t *p, struct optparse_option *o, const char *optarg)
-{
-    struct optparse_subcommand *s;
-    optparse_print_usage (p);
-    fprintf (stderr, "\n");
-    fprintf (stderr, "Common commands from flux-queue:\n");
-    s = subcommands;
-    while (s->name) {
-        fprintf (stderr, "   %-15s %s\n", s->name, s->doc);
-        s++;
-    }
-    exit (1);
-}
-
-int main (int argc, char *argv[])
-{
-    char *cmdusage = "[OPTIONS] COMMAND ARGS";
-    optparse_t *p;
-    int optindex;
-    int exitval;
-
-    log_init ("flux-queue");
-
-    p = optparse_create ("flux-queue");
-
-    if (optparse_add_option_table (p, global_opts) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_add_option_table() failed");
-
-    /* Override help option for our own */
-    if (optparse_set (p, OPTPARSE_USAGE, cmdusage) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_set (USAGE)");
-
-    /* Override --help callback in favor of our own above */
-    if (optparse_set (p, OPTPARSE_OPTION_CB, "help", usage) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_set() failed");
-
-    /* Don't print internal subcommands, we do it ourselves */
-    if (optparse_set (p, OPTPARSE_PRINT_SUBCMDS, 0) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_set (PRINT_SUBCMDS)");
-
-    if (optparse_reg_subcommands (p, subcommands) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_reg_subcommands");
-
-    if ((optindex = optparse_parse_args (p, argc, argv)) < 0)
-        exit (1);
-
-    if ((argc - optindex == 0)
-        || !optparse_get_subcommand (p, argv[optindex])) {
-        usage (p, NULL, NULL);
-        exit (1);
-    }
-
-    if ((exitval = optparse_run_subcommand (p, argc, argv)) < 0)
-        exit (1);
-
-    optparse_destroy (p);
-    log_fini ();
-    return (exitval);
-}
-
-/* Parse free arguments into a space-delimited message.
- * On error, exit complaning about parsing 'name'.
- * Caller must free the resulting string
- */
-static char *parse_arg_message (char **argv, const char *name)
-{
-    char *argz = NULL;
-    size_t argz_len = 0;
-    error_t e;
-
-    if ((e = argz_create (argv, &argz, &argz_len)) != 0)
-        log_errn_exit (e, "error parsing %s", name);
-    argz_stringify (argz, argz_len, ' ');
-    return argz;
-}
-
-void submit_admin (flux_t *h,
-                   int query_only,
-                   int enable,
-                   const char *reason)
-{
-    flux_future_t *f;
-    if (!(f = flux_rpc_pack (h,
-                             "job-manager.submit-admin",
-                             FLUX_NODEID_ANY,
-                             0,
-                             "{s:b s:b s:s}",
-                             "query_only",
-                             query_only,
-                             "enable",
-                             enable,
-                             "reason",
-                             reason ? reason : "")))
-        log_err_exit ("error sending submit-admin request");
-    if (flux_rpc_get_unpack (f,
-                             "{s:b s:s}",
-                             "enable",
-                             &enable,
-                             "reason",
-                             &reason) < 0)
-        log_msg_exit ("submit-admin: %s", future_strerror (f, errno));
-    log_msg ("Job submission is %s%s%s",
-             enable ? "enabled" : "disabled",
-             enable ? "" : ": ",
-             enable ? "" : reason);
-    flux_future_destroy (f);
-}
-
-void alloc_admin (flux_t *h,
-                  bool verbose,
-                  bool quiet,
-                  int query_only,
-                  int enable,
-                  const char *reason)
-{
-    flux_future_t *f;
-    int free_pending;
-    int alloc_pending;
-    int queue_length;
-    int running;
-
-    if (!(f = flux_rpc_pack (h,
-                             "job-manager.alloc-admin",
-                             FLUX_NODEID_ANY,
-                             0,
-                             "{s:b s:b s:s}",
-                             "query_only",
-                             query_only,
-                             "enable",
-                             enable,
-                             "reason",
-                             reason ? reason : "")))
-        log_err_exit ("error sending alloc-admin request");
-    if (flux_rpc_get_unpack (f,
-                             "{s:b s:s s:i s:i s:i s:i}",
-                             "enable",
-                             &enable,
-                             "reason",
-                             &reason,
-                             "queue_length",
-                             &queue_length,
-                             "alloc_pending",
-                             &alloc_pending,
-                             "free_pending",
-                             &free_pending,
-                             "running",
-                             &running) < 0)
-        log_msg_exit ("alloc-admin: %s", future_strerror (f, errno));
-    if (!quiet) {
-        log_msg ("Scheduling is %s%s%s",
-                 enable ? "enabled" : "disabled",
-                 reason && strlen (reason) > 0 ? ": " : "",
-                 reason ? reason : "");
-    }
-    if (verbose) {
-        log_msg ("%d alloc requests queued", queue_length);
-        log_msg ("%d alloc requests pending to scheduler", alloc_pending);
-        log_msg ("%d free requests pending to scheduler", free_pending);
-        log_msg ("%d running jobs", running);
-    }
-    flux_future_destroy (f);
-}
-
-
-int cmd_enable (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-
-    if (argc - optindex > 0) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    submit_admin (h, 0, 1, NULL);
-    flux_close (h);
-    return (0);
-}
-
-int cmd_disable (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-    char *reason = NULL;
-
-    if (argc - optindex > 0)
-        reason = parse_arg_message (argv + optindex, "reason");
-    else
-        log_msg_exit ("submit: reason is required for disable");
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    submit_admin (h, 0, 0, reason);
-    flux_close (h);
-    free (reason);
-    return (0);
-}
-
-int cmd_start (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-
-    if (argc - optindex > 0) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    alloc_admin (h,
-                 optparse_hasopt (p, "verbose"),
-                 optparse_hasopt (p, "quiet"),
-                 0,
-                 1,
-                 NULL);
-    flux_close (h);
-    return (0);
-}
-
-int cmd_stop (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-    char *reason = NULL;
-
-    if (argc - optindex > 0)
-        reason = parse_arg_message (argv + optindex, "reason");
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    alloc_admin (h,
-                 optparse_hasopt (p, "verbose"),
-                 optparse_hasopt (p, "quiet"),
-                 0,
-                 0,
-                 reason);
-    flux_close (h);
-    free (reason);
-    return (0);
-}
-
-int cmd_status (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-
-    if (argc - optindex > 0) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    submit_admin (h, 1, 0, NULL);
-    alloc_admin (h,
-                 optparse_hasopt (p, "verbose"),
-                 false,
-                 1,
-                 0,
-                 NULL);
-    flux_close (h);
-    return (0);
-}
-
-int cmd_drain (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-    double timeout = optparse_get_duration (p, "timeout", -1.);
-    flux_future_t *f;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (argc - optindex != 0) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (!(f = flux_rpc (h, "job-manager.drain", NULL, FLUX_NODEID_ANY, 0)))
-        log_err_exit ("flux_rpc");
-    if (flux_future_wait_for (f, timeout) < 0 || flux_rpc_get (f, NULL) < 0)
-        log_msg_exit ("drain: %s", errno == ETIMEDOUT
-                                   ? "timeout" : future_strerror (f, errno));
-    flux_future_destroy (f);
-    flux_close (h);
-    return (0);
-}
-
-int cmd_idle (optparse_t *p, int argc, char **argv)
-{
-    flux_t *h;
-    int optindex = optparse_option_index (p);
-    double timeout = optparse_get_duration (p, "timeout", -1.);
-    flux_future_t *f;
-    int pending;
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-    if (argc - optindex != 0) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-    if (!(f = flux_rpc (h, "job-manager.idle", NULL, FLUX_NODEID_ANY, 0)))
-        log_err_exit ("flux_rpc");
-    if (flux_future_wait_for (f, timeout) < 0
-            || flux_rpc_get_unpack (f, "{s:i}", "pending", &pending) < 0)
-        log_msg_exit ("idle: %s", errno == ETIMEDOUT
-                                   ? "timeout" : future_strerror (f, errno));
-    if (!optparse_hasopt (p, "quiet") || pending > 0)
-        log_msg ("%d pending jobs", pending);
-    flux_future_destroy (f);
-    flux_close (h);
-    return (0);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/cmd/flux-queue.py b/src/cmd/flux-queue.py
new file mode 100755
index 000000000000..8ca9c9c08151
--- /dev/null
+++ b/src/cmd/flux-queue.py
@@ -0,0 +1,717 @@
+##############################################################
+# Copyright 2022 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import errno
+import json
+import logging
+import math
+import sys
+
+import flux
+from flux.util import AltField, FilterActionSetUpdate, UtilConfig, parse_fsd
+
+
+def print_enable_status(name, status):
+    if name:
+        print(f"{name}: ", end="")
+    if status["enable"]:
+        print("Job submission is enabled")
+    else:
+        disable_reason = status["disable_reason"]
+        print(f"Job submission is disabled: {disable_reason}")
+
+
+def print_start_status(name, status):
+    if name:
+        print(f"{name}: ", end="")
+    if status["start"]:
+        print("Scheduling is started")
+    else:
+        reason = ": " + status["stop_reason"] if "stop_reason" in status else None
+        print(f"""Scheduling is stopped{reason if reason else ""}""")
+
+
+def print_queue_status(name, status):
+    print_enable_status(name, status)
+    print_start_status(name, status)
+
+
+def queue_status_get_one(handle, name=None):
+    payload = {"name": name} if name else {}
+    future = handle.rpc("job-manager.queue-status", payload, nodeid=0)
+    try:
+        status = future.get()
+    except EnvironmentError:
+        LOGGER.error("status: {0}".format(future.error_string()))
+        sys.exit(1)
+    return status
+
+
+def queue_status_get(handle, queue, verbose):
+    statuses = []
+    future = handle.rpc("job-manager.queue-list")
+    try:
+        qlist = future.get()
+    except EnvironmentError:
+        LOGGER.error("list: {0}".format(future.error_string()))
+        sys.exit(1)
+    if not queue or verbose:
+        if len(qlist["queues"]) > 0:
+            for q in qlist["queues"]:
+                status = queue_status_get_one(handle, q)
+                statuses.append((q, status))
+        else:
+            status = queue_status_get_one(handle)
+            statuses.append((queue, status))
+    else:
+        status = queue_status_get_one(handle, queue)
+        statuses.append((queue, status))
+    return statuses
+
+
+def queue_status(handle, queue, verbose, out_cb):
+    statuses = queue_status_get(handle, queue, verbose)
+    for name, status in statuses:
+        out_cb(name, status)
+
+
+def alloc_query(handle):
+    future = handle.rpc("job-manager.alloc-query")
+    try:
+        query = future.get()
+    except EnvironmentError:
+        LOGGER.error("alloc-query: {0}".format(future.error_string()))
+        sys.exit(1)
+    print("{0} alloc requests queued".format(query["queue_length"]))
+    print("{0} alloc requests pending to scheduler".format(query["alloc_pending"]))
+    print("{0} running jobs".format(query["running"]))
+
+
+def status(args):
+    handle = flux.Flux()
+    queue_status(handle, args.queue, args.verbose, print_queue_status)
+    if args.verbose:
+        alloc_query(handle)
+
+
+class FluxQueueConfig(UtilConfig):
+    """flux-queue specific user configuration class"""
+
+    builtin_formats = {}
+    builtin_formats["list"] = {
+        "default": {
+            "description": "Default flux-queue list format string",
+            "format": (
+                "?:{queuem:<8.8} "
+                "{color_enabled}{enabled:>2}{color_off} "
+                "{color_started}{started:>2}{color_off} "
+                "{defaults.timelimit!F:>8} "
+                "{limits.timelimit!F:>8} "
+                "{limits.range.nnodes:>10} "
+                "{limits.range.ncores:>10} "
+                "{limits.range.ngpus:>10}"
+            ),
+        },
+    }
+
+    def __init__(self, subcommand):
+        initial_dict = {}
+        for key, value in self.builtin_formats.items():
+            initial_dict[key] = {"formats": value}
+        super().__init__(
+            name="flux-queue", subcommand=subcommand, initial_dict=initial_dict
+        )
+
+    def validate(self, path, conf):
+        """Validate a loaded flux-queue config file as dictionary"""
+
+        for key, value in conf.items():
+            if key in ["list"]:
+                for key2, val2 in value.items():
+                    if key2 == "formats":
+                        self.validate_formats(path, val2)
+                    else:
+                        raise ValueError(f"{path}: invalid key {key}.{key2}")
+            else:
+                raise ValueError(f"{path}: invalid key {key}")
+
+
+class QueueLimitsJobSizeInfo:
+    def __init__(self, name, config, minormax):
+        self.name = name
+        self.config = config
+        self.minormax = minormax
+
+    def get_limit(self, key):
+        try:
+            val = self.config["queues"][self.name]["policy"]["limits"]["job-size"][
+                self.minormax
+            ][key]
+        except KeyError:
+            try:
+                val = self.config["policy"]["limits"]["job-size"][self.minormax][key]
+            except KeyError:
+                val = math.inf if self.minormax == "max" else 0
+        if val < 0:
+            val = math.inf
+        return val
+
+    @property
+    def nnodes(self):
+        return self.get_limit("nnodes")
+
+    @property
+    def ncores(self):
+        return self.get_limit("ncores")
+
+    @property
+    def ngpus(self):
+        return self.get_limit("ngpus")
+
+
+class QueueLimitsRangeInfo:
+    def __init__(self, name, min, max):
+        self.name = name
+        self.min = min
+        self.max = max
+
+    @property
+    def nnodes(self):
+        return f"{self.min.nnodes}-{self.max.nnodes}"
+
+    @property
+    def ncores(self):
+        return f"{self.min.ncores}-{self.max.ncores}"
+
+    @property
+    def ngpus(self):
+        return f"{self.min.ngpus}-{self.max.ngpus}"
+
+
+class QueueLimitsInfo:
+    def __init__(self, name, config):
+        self.name = name
+        self.config = config
+        self.min = QueueLimitsJobSizeInfo(name, config, "min")
+        self.max = QueueLimitsJobSizeInfo(name, config, "max")
+        self.range = QueueLimitsRangeInfo(name, self.min, self.max)
+
+    @property
+    def timelimit(self):
+        try:
+            duration = self.config["queues"][self.name]["policy"]["limits"]["duration"]
+        except KeyError:
+            try:
+                duration = self.config["policy"]["limits"]["duration"]
+            except KeyError:
+                duration = "inf"
+        t = parse_fsd(duration)
+        return t
+
+
+class QueueDefaultsInfo:
+    def __init__(self, name, config):
+        self.name = name
+        self.config = config
+
+    @property
+    def timelimit(self):
+        try:
+            duration = self.config["queues"][self.name]["policy"]["jobspec"][
+                "defaults"
+            ]["system"]["duration"]
+        except KeyError:
+            try:
+                duration = self.config["policy"]["jobspec"]["defaults"]["system"][
+                    "duration"
+                ]
+            except KeyError:
+                duration = "inf"
+        t = parse_fsd(duration)
+        return t
+
+
+class QueueInfo:
+    def __init__(self, name, config, enabled, started):
+        self.name = name
+        self.config = config
+        self.limits = QueueLimitsInfo(name, config)
+        self.defaults = QueueDefaultsInfo(name, config)
+        self.scheduling = "started" if started else "stopped"
+        self.submission = "enabled" if enabled else "disabled"
+        self._enabled = enabled
+        self._started = started
+
+    def __getattr__(self, attr):
+        try:
+            return getattr(self, attr)
+        except (KeyError, AttributeError):
+            raise AttributeError("invalid QueueInfo attribute '{}'".format(attr))
+
+    @property
+    def queue(self):
+        return self.name if self.name else ""
+
+    @property
+    def queuem(self):
+        try:
+            defaultq = self.config["policy"]["jobspec"]["defaults"]["system"]["queue"]
+        except KeyError:
+            defaultq = ""
+        q = self.queue + ("*" if defaultq and self.queue == defaultq else "")
+        return q
+
+    @property
+    def color_enabled(self):
+        return "\033[01;32m" if self._enabled else "\033[01;31m"
+
+    @property
+    def color_off(self):
+        return "\033[0;0m"
+
+    @property
+    def enabled(self):
+        return AltField("✔", "y") if self._enabled else AltField("✗", "n")
+
+    @property
+    def color_started(self):
+        return "\033[01;32m" if self._started else "\033[01;31m"
+
+    @property
+    def started(self):
+        return AltField("✔", "y") if self._started else AltField("✗", "n")
+
+
+def fetch_all_queue_status(handle, queues=None):
+    if handle is None:
+        # Return fake payload if handle is not open (e.g. during testing)
+        return {"enable": True, "start": True}
+    topic = "job-manager.queue-status"
+    if queues is None:
+        return handle.rpc(topic, {}).get()
+    rpcs = {x: handle.rpc(topic, {"name": x}) for x in queues}
+    return {x: rpcs[x].get() for x in rpcs}
+
+
+def list(args):
+    headings = {
+        "queue": "QUEUE",
+        "queuem": "QUEUE",
+        "submission": "SUBMIT",
+        "scheduling": "SCHED",
+        "enabled": "EN",
+        "started": "ST",
+        "enabled.ascii": "EN",
+        "started.ascii": "ST",
+        "color_enabled": "",
+        "color_started": "",
+        "color_off": "",
+        "defaults.timelimit": "TDEFAULT",
+        "limits.timelimit": "TLIMIT",
+        "limits.range.nnodes": "NNODES",
+        "limits.range.ncores": "NCORES",
+        "limits.range.ngpus": "NGPUS",
+        "limits.min.nnodes": "MINNODES",
+        "limits.max.nnodes": "MAXNODES",
+        "limits.min.ncores": "MINCORES",
+        "limits.max.ncores": "MAXCORES",
+        "limits.min.ngpus": "MINGPUS",
+        "limits.max.ngpus": "MAXGPUS",
+    }
+    config = None
+    handle = None
+
+    if args.from_stdin:
+        config = json.loads(sys.stdin.read())
+    else:
+        handle = flux.Flux()
+        future = handle.rpc("config.get")
+        try:
+            config = future.get()
+        except Exception as e:
+            LOGGER.warning("Could not get flux config: " + str(e))
+
+    fmt = FluxQueueConfig("list").load().get_format_string(args.format)
+    formatter = flux.util.OutputFormat(fmt, headings=headings)
+
+    #  Build queue_config from args.queue, or config["queue"] if --queue
+    #  was unused:
+    queue_config = {}
+    if args.queue:
+        for queue in args.queue:
+            try:
+                queue_config[queue] = config["queues"][queue]
+            except KeyError:
+                raise ValueError(f"No such queue: {queue}")
+    elif config and "queues" in config:
+        queue_config = config["queues"]
+
+    queues = []
+    if config and "queues" in config:
+        status = fetch_all_queue_status(handle, queue_config.keys())
+        for key, value in queue_config.items():
+            queues.append(
+                QueueInfo(key, config, status[key]["enable"], status[key]["start"])
+            )
+    else:
+        # single anonymous queue
+        status = fetch_all_queue_status(handle)
+        queues.append(QueueInfo(None, config, status["enable"], status["start"]))
+
+    formatter.print_items(queues, no_header=args.no_header)
+
+
+def queue_enable(handle, enable, name, all, reason=None):
+    payload = {"enable": enable, "all": all}
+    if name:
+        payload["name"] = name
+    if reason:
+        payload["reason"] = reason
+    future = handle.rpc("job-manager.queue-enable", payload, nodeid=0)
+    try:
+        future.get()
+    except EnvironmentError:
+        LOGGER.error("enable: {0}".format(future.error_string()))
+        sys.exit(1)
+
+
+def enable(args):
+    handle = flux.Flux()
+    queue_enable(handle, True, args.queue, args.all)
+    if not args.quiet:
+        queue_status(handle, args.queue, args.verbose, print_enable_status)
+
+
+def disable(args):
+    reason = None
+    if args.message:
+        reason = " ".join(args.message)
+    handle = flux.Flux()
+    queue_enable(handle, False, args.queue, args.all, reason)
+    if not args.quiet:
+        queue_status(handle, args.queue, args.verbose, print_enable_status)
+
+
+def check_legacy_all(handle, name, all, quiet):
+    if not name and not all and not quiet:
+        future = handle.rpc("job-manager.queue-list")
+        try:
+            qlist = future.get()
+        except Exception:
+            return all
+        if len(qlist["queues"]) > 0:
+            print(
+                "warning: --queue/--all not specified, assuming --all", file=sys.stderr
+            )
+        return True
+    return all
+
+
+def queue_start(handle, start, name, all, nocheckpoint=False, reason=None):
+    payload = {"start": start, "all": all, "nocheckpoint": nocheckpoint}
+    if name:
+        payload["name"] = name
+    if reason:
+        payload["reason"] = reason
+    future = handle.rpc("job-manager.queue-start", payload, nodeid=0)
+    try:
+        future.get()
+    except EnvironmentError:
+        LOGGER.error("start: {0}".format(future.error_string()))
+        sys.exit(1)
+
+
+def start(args):
+    handle = flux.Flux()
+    args.all = check_legacy_all(handle, args.queue, args.all, args.quiet)
+    queue_start(handle, True, args.queue, args.all)
+    if not args.quiet:
+        queue_status(handle, args.queue, args.verbose, print_start_status)
+        if args.verbose:
+            alloc_query(handle)
+
+
+def stop(args):
+    reason = " ".join(args.message) if args.message else None
+    handle = flux.Flux()
+    args.all = check_legacy_all(handle, args.queue, args.all, args.quiet)
+    queue_start(handle, False, args.queue, args.all, args.nocheckpoint, reason)
+    if not args.quiet:
+        queue_status(handle, args.queue, args.verbose, print_start_status)
+        if args.verbose:
+            alloc_query(handle)
+
+
+def drain(args):
+    handle = flux.Flux()
+    future = handle.rpc("job-manager.drain")
+    try:
+        future.wait_for(args.timeout)
+        future.get()
+    except EnvironmentError as e:
+        if e.errno == errno.ETIMEDOUT:
+            LOGGER.error("drain: timeout")
+        else:
+            LOGGER.error("drain: {0}".format(future.error_string()))
+        sys.exit(1)
+
+
+def idle(args):
+    handle = flux.Flux()
+    future = handle.rpc("job-manager.idle")
+    try:
+        future.wait_for(args.timeout)
+        rsp = future.get()
+        pending = rsp["pending"]
+    except EnvironmentError as e:
+        if e.errno == errno.ETIMEDOUT:
+            LOGGER.error("idle: timeout")
+        else:
+            LOGGER.error("idle: {0}".format(future.error_string()))
+        sys.exit(1)
+    if not args.quiet or pending > 0:
+        print(f"{pending} jobs")
+
+
+class FSDAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        time = parse_fsd(values)
+        setattr(namespace, self.dest, time)
+
+
+LOGGER = logging.getLogger("flux-queue")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    parser = argparse.ArgumentParser(prog="flux-queue")
+    subparsers = parser.add_subparsers(
+        title="subcommands", description="", dest="subcommand"
+    )
+    subparsers.required = True
+
+    status_parser = subparsers.add_parser(
+        "status", formatter_class=flux.util.help_formatter()
+    )
+    status_parser.add_argument(
+        "-q",
+        "--queue",
+        type=str,
+        metavar="NAME",
+        help="Specify queue to show (default all)",
+    )
+    status_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Display more detail about internal job manager state",
+    )
+    status_parser.set_defaults(func=status)
+
+    list_parser = subparsers.add_parser(
+        "list", formatter_class=flux.util.help_formatter()
+    )
+    list_parser.add_argument(
+        "-q",
+        "--queue",
+        action=FilterActionSetUpdate,
+        default=set(),
+        metavar="QUEUE,...",
+        help="Include only specified queues in output",
+    )
+    list_parser.add_argument(
+        "-o",
+        "--format",
+        default="default",
+        help="Specify output format using Python's string format syntax "
+        + "or a defined format by name (use 'help' to get a list of names)",
+    )
+    list_parser.add_argument(
+        "-n", "--no-header", action="store_true", help="Suppress header output"
+    )
+    list_parser.add_argument(
+        "--from-stdin", action="store_true", help=argparse.SUPPRESS
+    )
+    list_parser.set_defaults(func=list)
+
+    enable_parser = subparsers.add_parser(
+        "enable", formatter_class=flux.util.help_formatter()
+    )
+    enable_parser.add_argument(
+        "-q",
+        "--queue",
+        type=str,
+        metavar="NAME",
+        help="Specify queue to enable",
+    )
+    enable_parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        help="Force command to apply to all queues if none specified",
+    )
+    enable_parser.add_argument(
+        "--quiet",
+        action="store_true",
+        help="Display only errors",
+    )
+    enable_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Display more detail about internal job manager state",
+    )
+    enable_parser.set_defaults(func=enable)
+
+    disable_parser = subparsers.add_parser(
+        "disable", formatter_class=flux.util.help_formatter()
+    )
+    disable_parser.add_argument(
+        "-q",
+        "--queue",
+        type=str,
+        metavar="NAME",
+        help="Specify queue to disable",
+    )
+    disable_parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        help="Force command to apply to all queues if none specified",
+    )
+    disable_parser.add_argument(
+        "--quiet",
+        action="store_true",
+        help="Display only errors",
+    )
+    disable_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Display details about all job manager queues",
+    )
+    disable_parser.add_argument(
+        "--nocheckpoint",
+        action="store_true",
+        help="Do not checkpoint that the queue has been disableped",
+    )
+    disable_parser.add_argument(
+        "message", help="disable reason", nargs=argparse.REMAINDER
+    )
+    disable_parser.set_defaults(func=disable)
+
+    start_parser = subparsers.add_parser(
+        "start", formatter_class=flux.util.help_formatter()
+    )
+    start_parser.add_argument(
+        "-q",
+        "--queue",
+        type=str,
+        metavar="NAME",
+        help="Specify queue to start",
+    )
+    start_parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        help="Force command to apply to all queues if none specified",
+    )
+    start_parser.add_argument(
+        "--quiet",
+        action="store_true",
+        help="Display only errors",
+    )
+    start_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Display more detail about internal job manager state",
+    )
+    start_parser.set_defaults(func=start)
+
+    stop_parser = subparsers.add_parser(
+        "stop", formatter_class=flux.util.help_formatter()
+    )
+    stop_parser.add_argument(
+        "-q",
+        "--queue",
+        type=str,
+        metavar="NAME",
+        help="Specify queue to stop",
+    )
+    stop_parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        help="Force command to apply to all queues if none specified",
+    )
+    stop_parser.add_argument(
+        "--quiet",
+        action="store_true",
+        help="Display only errors",
+    )
+    stop_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Display more detail about internal job manager state",
+    )
+    stop_parser.add_argument(
+        "--nocheckpoint",
+        action="store_true",
+        help="Do not checkpoint that the queue has been stopped",
+    )
+    stop_parser.add_argument("message", help="stop reason", nargs=argparse.REMAINDER)
+    stop_parser.set_defaults(func=stop)
+
+    drain_parser = subparsers.add_parser(
+        "drain", formatter_class=flux.util.help_formatter()
+    )
+    drain_parser.add_argument(
+        "-t",
+        "--timeout",
+        action=FSDAction,
+        metavar="DURATION",
+        default=-1.0,
+        help="timeout after DURATION",
+    )
+    drain_parser.set_defaults(func=drain)
+
+    idle_parser = subparsers.add_parser(
+        "idle", formatter_class=flux.util.help_formatter()
+    )
+    idle_parser.add_argument(
+        "-t",
+        "--timeout",
+        action=FSDAction,
+        metavar="DURATION",
+        default=-1.0,
+        help="timeout after DURATION",
+    )
+    idle_parser.add_argument(
+        "--quiet",
+        action="store_true",
+        help="Only display pending job count if nonzero",
+    )
+    idle_parser.set_defaults(func=idle)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-resource.py b/src/cmd/flux-resource.py
new file mode 100755
index 000000000000..984f30e96d24
--- /dev/null
+++ b/src/cmd/flux-resource.py
@@ -0,0 +1,1019 @@
+##############################################################
+# Copyright 2019 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import errno
+import json
+import logging
+import os.path
+import sys
+from itertools import combinations
+
+import flux
+from flux.hostlist import Hostlist
+from flux.idset import IDset
+from flux.resource import (
+    ResourceSet,
+    ResourceStatus,
+    SchedResourceList,
+    resource_list,
+    resource_status,
+)
+from flux.rpc import RPC
+from flux.util import AltField, Deduplicator, FilterActionSetUpdate, UtilConfig
+
+
+class FluxResourceConfig(UtilConfig):
+    """flux-resource specific user configuration class"""
+
+    builtin_formats = {}
+    builtin_formats["status"] = {
+        "default": {
+            "description": "Default flux-resource status format string",
+            "format": "{state:>12} {color_up}{up:>2}{color_off} {nnodes:>6} {nodelist}",
+        },
+        "long": {
+            "description": "Long flux-resource status format string",
+            "format": (
+                "{state:>12} {color_up}{up:>2}{color_off} "
+                "{nnodes:>6} +:{reason:<30.30+} {nodelist}"
+            ),
+        },
+    }
+    builtin_formats["drain"] = {
+        "long": {
+            "description": "Long flux-resource drain format string",
+            "format": (
+                "{timestamp!d:%b%d %R::<12} {state:<8.8} {ranks:<8.8+} "
+                "+:{reason:<30.30+} {nodelist}"
+            ),
+        },
+        "default": {
+            "description": "Default flux-resource drain format string",
+            "format": (
+                "{timestamp!d:%b%d %R::<12} {state:<8.8} {reason:<30.30+} {nodelist}"
+            ),
+        },
+    }
+    builtin_formats["list"] = {
+        "default": {
+            "description": "Default flux-resource list format string",
+            "format": (
+                "{state:>10} ?+:{queue:<5} ?:{propertiesx:<10.10+} {nnodes:>6} "
+                "+:{ncores:>6} ?+:{ngpus:>5} {nodelist}"
+            ),
+        },
+        "rlist": {
+            "description": "Format including resource list details",
+            "format": (
+                "{state:>10} ?+:{queue:<5} ?:{propertiesx:<10.10+} {nnodes:>6} "
+                "+:{ncores:>6} ?:+{ngpus:>5} {rlist}"
+            ),
+        },
+    }
+
+    def __init__(self, subcommand):
+        initial_dict = {}
+        for key, value in self.builtin_formats.items():
+            initial_dict[key] = {"formats": value}
+        super().__init__(
+            name="flux-resource", subcommand=subcommand, initial_dict=initial_dict
+        )
+
+    def validate(self, path, conf):
+        """Validate a loaded flux-resource config file as dictionary"""
+
+        for key, value in conf.items():
+            if key in ["status", "list", "drain"]:
+                for key2, val2 in value.items():
+                    if key2 == "formats":
+                        self.validate_formats(path, val2)
+                    else:
+                        raise ValueError(f"{path}: invalid key {key}.{key2}")
+            else:
+                raise ValueError(f"{path}: invalid key {key}")
+
+
+def reload(args):
+    """
+    Send a "reload" request to resource module
+    """
+    RPC(
+        flux.Flux(),
+        "resource.reload",
+        {"path": os.path.realpath(args.path), "xml": args.xml, "force": args.force},
+        nodeid=0,
+    ).get()
+
+
+def acquire_mute(args):
+    """
+    Send an "acquire-mute" request to resource module, to avoid the stream of
+    DOWN acquire responses to the scheduler during shutdown.
+    """
+    RPC(flux.Flux(), "resource.acquire-mute", {}, nodeid=0).get()
+
+
+def drain(args):
+    """
+    Send a drain request to resource module for args.targets, if args.targets
+    not specified, then list currently drained targets
+    """
+    if args.targets is None:
+        drain_list(args)
+        return
+    payload = {
+        "targets": args.targets,
+    }
+    if args.update and args.force:
+        LOGGER.error("Only one of --force and --update may be specified")
+        sys.exit(1)
+    if args.update:
+        payload["mode"] = "update"
+    elif args.force == 1:
+        payload["mode"] = "overwrite"
+    elif args.force == 2:
+        payload["mode"] = "force-overwrite"
+    if args.reason:
+        payload["reason"] = " ".join(args.reason)
+    try:
+        RPC(
+            flux.Flux(),
+            "resource.drain",
+            payload,
+            nodeid=0,
+        ).get()
+    except OSError as exc:
+        LOGGER.error(exc)
+        if exc.errno == errno.EEXIST:
+            LOGGER.info("Use --force to overwrite existing drain reason")
+        sys.exit(1)
+
+
+def undrain(args):
+    """
+    Send an "undrain" request to resource module for args.targets
+    """
+    payload = {"targets": args.targets}
+    if args.force:
+        payload["mode"] = "force"
+    RPC(flux.Flux(), "resource.undrain", payload, nodeid=0).get()
+
+
+class QueueResources:
+    """
+    Convenience class to map queues to resource sets
+    """
+
+    def __init__(self, resource_set, config):
+        self._queues = {}
+        if "queues" not in config:
+            return
+        for queue in config["queues"]:
+            if "requires" in config["queues"][queue]:
+                result = resource_set.copy_constraint(
+                    {"properties": config["queues"][queue]["requires"]}
+                )
+            else:
+                result = resource_set.copy()
+            self._queues[queue] = result
+
+    def queue(self, queue):
+        if queue not in self._queues:
+            raise ValueError(f"{queue}: no such queue")
+        return self._queues[queue]
+
+
+def ranks_by_queue(resource_set, config, queues):
+    """
+    Return all ranks associated with a list of queues
+    Args:
+        resource_set: The resource set to query
+        config: a Flux config object including queue configuration, if any.
+        queues: one or more queues specified as a comma-separated string
+    """
+    queue_resources = QueueResources(resource_set, config)
+    ranks = IDset()
+    for queue in queues:
+        ranks.add(queue_resources.queue(queue).ranks)
+    return ranks
+
+
+class ResourceStatusLine:
+    """Information specific to a given flux resource status line"""
+
+    def __init__(self, state, online, ranks, hosts, reason="", timestamp=""):
+        self._state = state
+        self._online = online
+        self.hostlist = Hostlist(hosts)
+        self._ranks = IDset(ranks)
+        self.reason = reason
+        self.timestamp = timestamp
+
+    def update(self, ranks, hosts):
+        self._ranks.add(ranks)
+        self.hostlist.append(hosts)
+        self.hostlist.sort()
+
+    @property
+    def statex(self):
+        return self._state
+
+    @property
+    def state(self):
+        state = self._state
+        if not self._online:
+            state += "*"
+        return state
+
+    @property
+    def offline(self):
+        return not self._online
+
+    @property
+    def status(self):
+        return "online" if self._online else "offline"
+
+    @property
+    def color_up(self):
+        return "\033[01;32m" if self._online else "\033[01;31m"
+
+    @property
+    def color_off(self):
+        return "\033[0;0m"
+
+    @property
+    def up(self):
+        return AltField("✔", "y") if self._online else AltField("✗", "n")
+
+    @property
+    def nodelist(self):
+        return str(self.hostlist)
+
+    @property
+    def ranks(self):
+        return str(self._ranks)
+
+    @property
+    def nnodes(self):
+        return len(self.hostlist)
+
+    def __repr__(self):
+        return f"{self.state} {self.hostlist}"
+
+
+def status_excluded(online, include_online, include_offline):
+    """
+    Return true if the current online status is excluded
+    """
+    included = (online and include_online) or (not online and include_offline)
+    return not included
+
+
+def statuslines(rstatus, states, formatter, include_online=True, include_offline=True):
+    """
+    Given a ResourceStatus object and OutputFormat formatter,
+    return a set of deduplicated ResourceStatusLine objects for
+    display.
+    """
+    result = Deduplicator(
+        formatter=formatter,
+        except_fields=["nodelist", "ranks", "nnodes"],
+        combine=lambda line, arg: line.update(arg.ranks, arg.hostlist),
+    )
+    states = set(states)
+    for state in ["avail", "exclude", "allocated", "torpid", "housekeeping"]:
+        if not states & {state, "all"}:
+            continue
+        for online in (True, False):
+            if status_excluded(online, include_online, include_offline):
+                continue
+            status_ranks = rstatus["online" if online else "offline"]
+            ranks = rstatus[state].intersect(status_ranks)
+            result.append(
+                ResourceStatusLine(state, online, ranks, rstatus.nodelist[ranks])
+            )
+    for state in ["draining", "drained"]:
+        if not states & {state, "all"}:
+            continue
+        ranks = rstatus[state]
+        if not ranks:
+            if not status_excluded(True, include_online, include_offline):
+                # Append one empty line for "draining" and "drained":
+                result.append(ResourceStatusLine(state, True, "", ""))
+        for rank in ranks:
+            online = rank in rstatus.online
+            if status_excluded(online, include_online, include_offline):
+                continue
+            info = rstatus.get_drain_info(rank)
+            result.append(
+                ResourceStatusLine(
+                    state,
+                    online,
+                    rank,
+                    rstatus.nodelist[rank],
+                    timestamp=info.timestamp,
+                    reason=info.reason,
+                )
+            )
+    return result
+
+
+def status_help(args, valid_states, headings):
+    if args.states == "help":
+        LOGGER.info("valid states: %s", ",".join(valid_states))
+    sys.exit(0)
+
+
+def status_get_state_list(args, valid_states, default_states):
+    #  Get list of states from command, or a default:
+    if args.states:
+        states = args.states
+    else:
+        states = default_states
+
+    #  Warn if listed state is not valid
+    states = states.split(",")
+    for state in states:
+        if state not in valid_states:
+            LOGGER.error("Invalid resource state %s specified", state)
+            LOGGER.info("valid states: %s", ",".join(valid_states))
+            sys.exit(1)
+
+    #  If only offline and/or online are specified, then append other
+    #  default states to states list, o/w status will return nothing
+    copy = list(filter(lambda x: x not in ("offline", "online"), states))
+    if not copy:
+        states.extend(default_states.split(","))
+
+    #  Expand special "drain" state to "draining", "drained"
+    if "drain" in states:
+        states.remove("drain")
+        states.extend(("draining", "drained"))
+    return states
+
+
+def status(args):
+    valid_states = [
+        "all",
+        "online",
+        "avail",
+        "offline",
+        "exclude",
+        "drain",
+        "draining",
+        "drained",
+        "torpid",
+        "allocated",
+        "housekeeping",
+    ]
+    default_states = "avail,exclude,draining,drained,torpid,housekeeping"
+
+    headings = {
+        "state": "STATE",
+        "statex": "STATE",
+        "nnodes": "NNODES",
+        "ranks": "RANKS",
+        "nodelist": "NODELIST",
+        "reason": "REASON",
+        "timestamp": "TIME",
+        "status": "STATUS",
+        "up": "UP",
+        "up.ascii": "UP",
+        "color_up": "",
+        "color_off": "",
+    }
+
+    #  Emit list of valid states if requested
+    if args.states == "help":
+        status_help(args, valid_states, headings)
+
+    #  Get state list from args or defaults:
+    states = status_get_state_list(args, valid_states, default_states)
+
+    fmt = FluxResourceConfig("status").load().get_format_string(args.format)
+
+    handle = None
+
+    #  Get payload from stdin or from resource.status RPC:
+    if args.from_stdin:
+        input_str = sys.stdin.read()
+        rstatus = ResourceStatus(json.loads(input_str) if input_str else None)
+    else:
+        handle = flux.Flux()
+        rstatus = resource_status(handle).get()
+
+    if args.queue:
+        if args.config_file:
+            with open(args.config_file) as fp:
+                config = json.load(fp)
+        else:
+            config = {}
+            if handle is not None:
+                config = handle.rpc("config.get").get()
+        try:
+            rstatus.filter(ranks_by_queue(rstatus.rset, config, args.queue))
+        except ValueError as exc:
+            raise ValueError(f"--queue: {exc}") from None
+
+    if args.include:
+        try:
+            rstatus.filter(include=args.include)
+        except (ValueError, TypeError) as exc:
+            raise ValueError(f"--include: {exc}") from None
+
+    formatter = flux.util.OutputFormat(fmt, headings=headings)
+
+    # Remove any `{color*}` fields if color is off
+    if args.color == "never" or (args.color == "auto" and not sys.stdout.isatty()):
+        formatter = formatter.copy(except_fields=["color_up", "color_off"])
+
+    #  Skip empty lines unless --states or --skip-empty
+    skip_empty = args.skip_empty or not args.states
+
+    #  Skip empty offline lines unless offline explicily requested
+    skip_empty_offline = "offline" not in states
+
+    #  Include both online and offline lines by default, except if
+    #  one of those statuses are explicitly requested, then it
+    #  becomes exclusive (unless both are present).
+    include_online = "online" in states or "offline" not in states
+    include_offline = "offline" in states or "online" not in states
+
+    #  Remove drained/draining ranks from torpid set if these
+    #  are in the list of displayed states
+    for drain_state in ("draining", "drained"):
+        if "all" in states or drain_state in states:
+            rstatus.torpid -= rstatus.get_idset(drain_state)
+
+    lines = []
+    for line in statuslines(
+        rstatus,
+        states,
+        formatter,
+        include_online=include_online,
+        include_offline=include_offline,
+    ):
+        if line.nnodes == 0:
+            if skip_empty or line.offline and skip_empty_offline:
+                continue
+        lines.append(line)
+    formatter.print_items(lines, no_header=args.no_header)
+
+
+def drain_list(args):
+    fmt = FluxResourceConfig("drain").load().get_format_string(args.format)
+    args.from_stdin = False
+    args.config_file = False
+    args.format = fmt
+    args.states = "drain"
+    args.skip_empty = True
+    status(args)
+
+
+class ResourceSetExtra(ResourceSet):
+    def __init__(self, arg=None, version=1, flux_config=None, queue=None):
+        self.flux_config = flux_config
+        self._queue = queue
+        if isinstance(arg, ResourceSet):
+            self._rset = arg
+            if arg.state:
+                self.state = arg.state
+        else:
+            self._rset = ResourceSet(arg, version)
+
+    def __getattr__(self, attr):
+        return getattr(self._rset, attr)
+
+    @property
+    def propertiesx(self):
+        properties = json.loads(self.get_properties())
+        queues = self.queue
+        if self.queue:
+            queues = queues.split(",")
+            for q in queues:
+                if q in properties:
+                    properties.pop(q)
+        return ",".join(properties.keys())
+
+    @property
+    def queue(self):
+        #  Note: queue may be set manually in self._queue for an empty
+        #  ResourceSet, which cannot otherwise have an associated queue.
+        if self._queue is not None:
+            return self._queue
+
+        #  If self._queue is not set, then build list of queues from
+        #  set properties and queue configuration:
+        queues = ""
+        if self.flux_config and "queues" in self.flux_config:
+            if not self.ranks:
+                return ""
+            properties = json.loads(self.get_properties())
+            for key, value in self.flux_config["queues"].items():
+                if "requires" not in value or set(value["requires"]).issubset(
+                    set(properties)
+                ):
+                    queues = queues + "," + key if queues else key
+        return queues
+
+
+def split_by_property_combinations(rset):
+    """
+    Split a resource set by all combinations of its properties.
+    This is done in hopes of splitting a resource into the minimum number
+    of subsets that may produce unique lines in the resource listing output.
+    """
+
+    def constraint_combinations(rset):
+        properties = set(json.loads(rset.get_properties()).keys())
+        sets = [
+            set(combination)
+            for i in range(1, len(properties) + 1)
+            for combination in combinations(properties, i)
+        ]
+        # Also include the empty set, i.e. resources with no properties
+        sets.append(set())
+
+        # generate RFC 31 constraint objects for each property combination
+        result = []
+        for cset in sets:
+            diff = properties - cset
+            cset.update(["^" + x for x in diff])
+            result.append({"properties": list(cset)})
+        return result
+
+    return [rset.copy_constraint(x) for x in constraint_combinations(rset)]
+
+
+def resources_uniq_lines(resources, states, formatter, config, queues=None):
+    """
+    Generate a set of resource sets that would produce unique lines given
+    the ResourceSet formatter argument. Include only the provided states
+    """
+    #  uniq_fields are the fields on which to combine like results
+    uniq_fields = ["state", "properties", "propertiesx", "queue"]
+
+    #
+    #  Create the uniq format by combining all provided uniq fields:
+    #   (but only if > 1 field was provided or "state" is in the list of
+    #   fields. This allows something like
+    #
+    #       flux resource -s all -no {properties}
+    #
+    #   to work as expected)
+    #
+    uniq_fmt = ""
+    if len(formatter.fields) > 1 or "state" in formatter.fields:
+        for field in formatter.fields:
+            if field in uniq_fields:
+                uniq_fmt += "{" + field + "}:"
+
+    fmt = flux.util.OutputFormat(uniq_fmt, headings=formatter.headings)
+
+    #  Get a list of configured queues if a specific list of queues
+    #  was not supplied by the caller. If no queues are configured then
+    #  one "anonymous" queue is simulated with [None]
+    if not queues:
+        if config and "queues" in config:
+            queues = config["queues"].keys()
+        else:
+            queues = [None]
+
+    #  Create a mapping of resources sets that generate uniq "lines":
+    lines = {}
+    for state in states:
+        if not resources[state].ranks:
+            #
+            #  If there are no resources in this state, generate an empty
+            #   resource set for output purposes. O/w the output for this
+            #   state would be suppressed.
+            #
+            for queue in queues:
+                rset = ResourceSetExtra(flux_config=config, queue=queue)
+                rset.state = state
+                key = fmt.format(rset)
+                if key not in lines:
+                    lines[key] = rset
+                else:
+                    lines[key].add(rset)
+            continue
+
+        for rset in split_by_property_combinations(resources[state]):
+            if not rset.ranks:
+                continue
+            rset.state = state
+            rset = ResourceSetExtra(rset, flux_config=config)
+            key = fmt.format(rset)
+
+            if key not in lines:
+                lines[key] = rset
+            else:
+                lines[key].add(rset)
+
+    return lines
+
+
+def get_resource_list(args):
+    """
+    Common function for list_handler() and emit_R()
+    """
+    valid_states = ["up", "down", "allocated", "free", "all"]
+    config = None
+
+    args.states = args.states.split(",")
+    for state in args.states:
+        if state not in valid_states:
+            LOGGER.error("Invalid resource state %s specified", state)
+            sys.exit(1)
+
+    if args.from_stdin:
+        resources = SchedResourceList(json.load(sys.stdin))
+        if args.config_file:
+            with open(args.config_file) as fp:
+                config = json.load(fp)
+    else:
+        handle = flux.Flux()
+        rpcs = [resource_list(handle), handle.rpc("config.get")]
+        resources = rpcs[0].get()
+        try:
+            config = rpcs[1].get()
+        except Exception as e:
+            LOGGER.warning("Could not get flux config: " + str(e))
+
+    if args.queue:
+        try:
+            resources.filter(ranks_by_queue(resources.all, config, args.queue))
+        except ValueError as exc:
+            raise ValueError(f"--queue: {exc}") from None
+
+    if args.include:
+        try:
+            resources.filter(include=args.include)
+        except (ValueError, TypeError) as exc:
+            raise ValueError(f"--include: {exc}") from None
+
+    return resources, config
+
+
+def sort_output(args, items):
+    """
+    Sort by args.states order, then first rank in resource set
+    """
+    statepos = {x[1]: x[0] for x in enumerate(args.states)}
+    return sorted(items, key=lambda x: (statepos[x.state], x.ranks.first()))
+
+
+def list_handler(args):
+    headings = {
+        "state": "STATE",
+        "queue": "QUEUE",
+        "properties": "PROPERTIES",
+        "propertiesx": "PROPERTIES",
+        "nnodes": "NNODES",
+        "ncores": "NCORES",
+        "ngpus": "NGPUS",
+        "ranks": "RANKS",
+        "nodelist": "NODELIST",
+        "rlist": "LIST",
+    }
+    resources, config = get_resource_list(args)
+
+    fmt = FluxResourceConfig("list").load().get_format_string(args.format)
+    formatter = flux.util.OutputFormat(fmt, headings=headings)
+
+    lines = resources_uniq_lines(
+        resources, args.states, formatter, config, queues=args.queue
+    )
+    items = sort_output(args, lines.values())
+    if args.skip_empty or (args.include and not args.no_skip_empty):
+        items = [x for x in items if x.ranks]
+    formatter.print_items(items, no_header=args.no_header)
+
+
+def info(args):
+    """Convenience function that wraps flux-resource list"""
+    if not args.states:
+        args.states = "all"
+    args.no_header = True
+    args.format = "{nnodes} Nodes, {ncores} Cores, {ngpus} GPUs"
+    list_handler(args)
+
+
+def emit_R(args):
+    """Emit R in JSON on stdout for requested set of resources"""
+    resources, config = get_resource_list(args)
+
+    rset = ResourceSet()
+    for state in args.states:
+        try:
+            rset.add(resources[state])
+        except AttributeError:
+            raise ValueError(f"unknown state {state}")
+    print(rset.encode())
+
+
+LOGGER = logging.getLogger("flux-resource")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    parser = argparse.ArgumentParser(prog="flux-resource")
+    subparsers = parser.add_subparsers(
+        title="subcommands", description="", dest="subcommand"
+    )
+    subparsers.required = True
+
+    acquire_mute_parser = subparsers.add_parser(
+        "acquire-mute", formatter_class=flux.util.help_formatter()
+    )
+    acquire_mute_parser.set_defaults(func=acquire_mute)
+
+    drain_parser = subparsers.add_parser(
+        "drain", formatter_class=flux.util.help_formatter()
+    )
+    drain_parser.add_argument(
+        "-f",
+        "--force",
+        action="count",
+        default=0,
+        help="Force overwrite of existing drain reason. Specify twice "
+        + "to also update drain timestamp.",
+    )
+    drain_parser.add_argument(
+        "-u",
+        "--update",
+        action="store_true",
+        help="Update only. Do not return an error if one or more targets "
+        + "are already drained. Do not overwrite any existing drain reason.",
+    )
+    drain_parser.add_argument(
+        "-i",
+        "--include",
+        metavar="TARGETS",
+        help="Include only specified targets in output set. TARGETS may be "
+        + "provided as an idset or hostlist.",
+    )
+    drain_parser.add_argument(
+        "-q",
+        "--queue",
+        action=FilterActionSetUpdate,
+        default=set(),
+        metavar="QUEUE,...",
+        help="Include only specified queues in output",
+    )
+    drain_parser.add_argument(
+        "-o",
+        "--format",
+        default="default",
+        help="Specify output format using Python's string format syntax "
+        + "or a defined format by name "
+        + "(only used when no drain targets specified)",
+    )
+    drain_parser.add_argument(
+        "-n", "--no-header", action="store_true", help="Suppress header output"
+    )
+    drain_parser.add_argument(
+        "-L",
+        "--color",
+        type=str,
+        metavar="WHEN",
+        choices=["never", "always", "auto"],
+        nargs="?",
+        const="always",
+        default="auto",
+        help="Use color; WHEN can be 'never', 'always', or 'auto' (default)",
+    )
+    drain_parser.add_argument(
+        "targets", nargs="?", help="List of targets to drain (IDSET or HOSTLIST)"
+    )
+    drain_parser.add_argument("reason", help="Reason", nargs=argparse.REMAINDER)
+    drain_parser.set_defaults(func=drain)
+
+    undrain_parser = subparsers.add_parser(
+        "undrain", formatter_class=flux.util.help_formatter()
+    )
+    undrain_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="Do not fail if any targets are not drained",
+    )
+    undrain_parser.add_argument(
+        "targets", help="List of targets to resume (IDSET or HOSTLIST)"
+    )
+    undrain_parser.set_defaults(func=undrain)
+
+    status_parser = subparsers.add_parser(
+        "status", formatter_class=flux.util.help_formatter()
+    )
+    status_parser.add_argument(
+        "-o",
+        "--format",
+        default="default",
+        help="Specify output format using Python's string format syntax "
+        + "or a defined format by name (use 'help' to get a list of names)",
+    )
+    status_parser.add_argument(
+        "-s",
+        "--states",
+        metavar="STATE,...",
+        help="Output resources in given states",
+    )
+    status_parser.add_argument(
+        "-i",
+        "--include",
+        metavar="TARGETS",
+        help="Include only specified targets in output set. TARGETS may be "
+        + "provided as an idset or hostlist.",
+    )
+    status_parser.add_argument(
+        "-q",
+        "--queue",
+        action=FilterActionSetUpdate,
+        default=set(),
+        metavar="QUEUE,...",
+        help="Include only specified queues in output",
+    )
+    status_parser.add_argument(
+        "-n", "--no-header", action="store_true", help="Suppress header output"
+    )
+    status_parser.add_argument(
+        "-L",
+        "--color",
+        type=str,
+        metavar="WHEN",
+        choices=["never", "always", "auto"],
+        nargs="?",
+        const="always",
+        default="auto",
+        help="Use color; WHEN can be 'never', 'always', or 'auto' (default)",
+    )
+    status_parser.add_argument(
+        "--from-stdin", action="store_true", help=argparse.SUPPRESS
+    )
+    status_parser.add_argument("--config-file", help=argparse.SUPPRESS)
+    status_parser.add_argument(
+        "--skip-empty",
+        action="store_true",
+        help="Skip empty lines of output even with --states",
+    )
+    status_parser.set_defaults(func=status)
+
+    list_parser = subparsers.add_parser(
+        "list", formatter_class=flux.util.help_formatter()
+    )
+    list_parser.add_argument(
+        "-o",
+        "--format",
+        default="default",
+        help="Specify output format using Python's string format syntax "
+        + "or a defined format by name (use 'help' to get a list of names)",
+    )
+    list_parser.add_argument(
+        "-s",
+        "--states",
+        metavar="STATE,...",
+        default="free,allocated,down",
+        help="Output resources in given states",
+    )
+    list_parser.add_argument(
+        "-i",
+        "--include",
+        metavar="TARGETS",
+        help="Include only specified targets in output set. TARGETS may be "
+        + "provided as an idset or hostlist.",
+    )
+    list_parser.add_argument(
+        "-q",
+        "--queue",
+        action=FilterActionSetUpdate,
+        default=set(),
+        metavar="QUEUE,...",
+        help="Include only specified queues in output",
+    )
+    list_parser.add_argument(
+        "--skip-empty",
+        action="store_true",
+        help="Skip empty lines. This is the default with -i, --include.",
+    )
+    list_parser.add_argument(
+        "--no-skip-empty",
+        action="store_true",
+        help="Do not skip empty lines, even with --include.",
+    )
+    list_parser.add_argument(
+        "-n", "--no-header", action="store_true", help="Suppress header output"
+    )
+    list_parser.add_argument(
+        "--from-stdin", action="store_true", help=argparse.SUPPRESS
+    )
+    list_parser.add_argument("--config-file", help=argparse.SUPPRESS)
+    list_parser.set_defaults(func=list_handler)
+
+    # flux-resource info:
+    info_parser = subparsers.add_parser(
+        "info", formatter_class=flux.util.help_formatter()
+    )
+    info_parser.add_argument(
+        "-s",
+        "--states",
+        metavar="STATE,...",
+        help="Show output only for resources in given states",
+    )
+    info_parser.add_argument(
+        "-i",
+        "--include",
+        metavar="TARGETS",
+        help="Include only specified targets in output set. TARGETS may be "
+        + "provided as an idset or hostlist.",
+    )
+    info_parser.add_argument(
+        "-q",
+        "--queue",
+        action=FilterActionSetUpdate,
+        default=set(),
+        metavar="QUEUE,...",
+        help="Include only specified queues in output",
+    )
+    info_parser.add_argument(
+        "--from-stdin", action="store_true", help=argparse.SUPPRESS
+    )
+    info_parser.add_argument("--config-file", help=argparse.SUPPRESS)
+    # Options required in `info` because they are also present in `list`:
+    info_parser.add_argument(
+        "--skip-empty",
+        action="store_true",
+        help=argparse.SUPPRESS,
+    )
+    info_parser.add_argument(
+        "--no-skip-empty", action="store_true", help=argparse.SUPPRESS
+    )
+    info_parser.set_defaults(func=info)
+
+    reload_parser = subparsers.add_parser(
+        "reload", formatter_class=flux.util.help_formatter()
+    )
+    reload_parser.set_defaults(func=reload)
+    reload_parser.add_argument("path", help="path to R or hwloc <rank>.xml dir")
+    reload_parser.add_argument(
+        "-x",
+        "--xml",
+        action="store_true",
+        default=False,
+        help="interpret path as XML dir",
+    )
+    reload_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="allow resources to contain invalid ranks",
+    )
+
+    R_parser = subparsers.add_parser("R", formatter_class=flux.util.help_formatter())
+    R_parser.set_defaults(func=emit_R)
+    R_parser.add_argument(
+        "-s",
+        "--states",
+        metavar="STATE,...",
+        default="all",
+        help="Emit R for resources in given states",
+    )
+    R_parser.add_argument(
+        "-i",
+        "--include",
+        metavar="TARGETS",
+        help="Include only specified targets in output set. TARGETS may be "
+        + "provided as an idset or hostlist.",
+    )
+    R_parser.add_argument(
+        "-q",
+        "--queue",
+        action=FilterActionSetUpdate,
+        default=set(),
+        metavar="QUEUE,...",
+        help="Include only specified queues in output",
+    )
+    R_parser.add_argument("--from-stdin", action="store_true", help=argparse.SUPPRESS)
+    R_parser.add_argument("--config-file", help=argparse.SUPPRESS)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-run-epilog.in b/src/cmd/flux-run-epilog.in
new file mode 100755
index 000000000000..f996b7f0a576
--- /dev/null
+++ b/src/cmd/flux-run-epilog.in
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+if test $FLUX_JOB_ID; then
+    FLUX_JOB_ID=$(flux job id --to=f58plain $FLUX_JOB_ID)
+fi
+unitname=flux-epilog@${FLUX_JOB_ID:-unknown}
+
+terminate() {
+    systemctl stop $unitname
+    exit 1
+}
+
+trap terminate INT TERM
+
+umask 022
+printenv >@X_RUNSTATEDIR@/${unitname}.env
+
+# Run systemctl start in background and `wait` for it so that the trap
+# will run immediately when signal is received:
+systemctl start $unitname --quiet &
+wait $!
diff --git a/src/cmd/flux-run-housekeeping.in b/src/cmd/flux-run-housekeeping.in
new file mode 100755
index 000000000000..6d4bea11cb0a
--- /dev/null
+++ b/src/cmd/flux-run-housekeeping.in
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+if test $FLUX_JOB_ID; then
+    FLUX_JOB_ID=$(flux job id --to=f58plain $FLUX_JOB_ID)
+fi
+unitname=flux-housekeeping@${FLUX_JOB_ID:-unknown}
+
+terminate() {
+    systemctl stop $unitname
+    exit 1
+}
+
+trap terminate INT TERM
+
+umask 022
+printenv >@X_RUNSTATEDIR@/${unitname}.env
+
+# Run systemctl start in background and `wait` for it so that the trap
+# will run immediately when signal is received:
+systemctl start $unitname --quiet &
+wait $!
diff --git a/src/cmd/flux-run-prolog.in b/src/cmd/flux-run-prolog.in
new file mode 100755
index 000000000000..944001659958
--- /dev/null
+++ b/src/cmd/flux-run-prolog.in
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+if test $FLUX_JOB_ID; then
+    FLUX_JOB_ID=$(flux job id --to=f58plain $FLUX_JOB_ID)
+fi
+unitname=flux-prolog@${FLUX_JOB_ID:-unknown}
+
+terminate() {
+    systemctl stop $unitname
+    exit 1
+}
+
+trap terminate INT TERM
+
+umask 022
+printenv >@X_RUNSTATEDIR@/${unitname}.env
+
+# Run systemctl start in background and `wait` for it so that the trap
+# will run immediately when signal is received:
+systemctl start $unitname --quiet &
+wait $!
diff --git a/src/cmd/flux-run.py b/src/cmd/flux-run.py
new file mode 100755
index 000000000000..05b5132c6dad
--- /dev/null
+++ b/src/cmd/flux-run.py
@@ -0,0 +1,43 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import logging
+import sys
+
+import flux
+import flux.cli.run as base
+
+LOGGER = logging.getLogger("flux-run")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    # Prepare the submit parser
+    run = base.RunCmd(
+        "flux run",
+        description="run a job interactively",
+    )
+    parser = run.get_parser()
+    parser.set_defaults(func=run.main)
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-start.c b/src/cmd/flux-start.c
index 01d0532ace55..4634bcb1c713 100644
--- a/src/cmd/flux-start.c
+++ b/src/cmd/flux-start.c
@@ -11,138 +11,153 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <limits.h>
 #include <termios.h>
 #include <unistd.h>
+#include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <sys/time.h>
 #include <sys/resource.h>
 #include <libgen.h>
+#include <signal.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <sys/ioctl.h>
+#include <jansson.h>
 #include <flux/core.h>
 #include <flux/optparse.h>
-#include <czmq.h>
+#include <flux/taskmap.h>
 
+#include "ccan/str/str.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/oom.h"
 #include "src/common/libutil/cleanup.h"
 #include "src/common/libutil/setenvf.h"
+#include "src/common/libutil/errno_safe.h"
 #include "src/common/libpmi/simple_server.h"
-#include "src/common/libpmi/dgetline.h"
+#include "src/common/libhostlist/hostlist.h"
+#include "src/common/librouter/usock_service.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+
+#define DEFAULT_EXIT_TIMEOUT 20.0
 
-#define DEFAULT_KILLER_TIMEOUT 2.0
+extern char **environ;
 
 static struct {
     struct termios saved_termios;
-    double killer_timeout;
+    double exit_timeout;
+    const char *exit_mode;
+    const char *start_mode;
     flux_reactor_t *reactor;
     flux_watcher_t *timer;
     zlist_t *clients;
     optparse_t *opts;
-    int size;
-    int count;
+    int verbose;
+    int test_size;
     int exit_rc;
     struct {
         zhash_t *kvs;
         struct pmi_simple_server *srv;
     } pmi;
+    flux_t *h;
+    flux_msg_handler_t **handlers;
 } ctx;
 
 struct client {
     int rank;
     flux_subprocess_t *p;
     flux_cmd_t *cmd;
+    int exit_rc;
+    const flux_msg_t *wait_request;
+    const flux_msg_t *run_request;
 };
 
-void killer (flux_reactor_t *r, flux_watcher_t *w, int revents, void *arg);
-int start_session (const char *cmd_argz, size_t cmd_argz_len,
+void exit_timeout (flux_reactor_t *r,
+                   flux_watcher_t *w,
+                   int revents,
+                   void *arg);
+int start_session (const char *cmd_argz,
+                   size_t cmd_argz_len,
                    const char *broker_path);
-int exec_broker (const char *cmd_argz, size_t cmd_argz_len,
+int exec_broker (const char *cmd_argz,
+                 size_t cmd_argz_len,
                  const char *broker_path);
-char *create_scratch_dir (void);
-struct client *client_create (const char *broker_path, const char *scratch_dir,
-                              int rank, const char *cmd_argz, size_t cmd_argz_len);
+char *create_rundir (void);
 void client_destroy (struct client *cli);
 char *find_broker (const char *searchpath);
-static void setup_profiling_env (void);
+static void client_wait_respond (struct client *cli);
+static void client_run_respond (struct client *cli, int errnum);
 
-#ifndef HAVE_CALIPER
-static int no_caliper_fatal_err (optparse_t *p, struct optparse_option *o,
-                                 const char *optarg)
-{
-    log_msg_exit ("Error: --caliper-profile used but no Caliper support found");
-}
-#endif /* !HAVE_CALIPER */
+const char *default_config_path = X_SYSCONFDIR "/flux/system/conf.d";
+const char *default_statedir = "/var/lib/flux";
 
 const char *usage_msg = "[OPTIONS] command ...";
 static struct optparse_option opts[] = {
-    { .name = "verbose",    .key = 'v', .has_arg = 0,
-      .usage = "Be annoyingly informative", },
+    { .name = "setattr",    .key = 'S', .has_arg = 1, .arginfo = "ATTR=VAL",
+      .flags = OPTPARSE_OPT_AUTOSPLIT,
+      .usage = "Set broker attribute", },
+    { .name = "config-path",.key = 'c', .has_arg = 1, .arginfo = "PATH",
+      .usage = "Set broker config from PATH (default: none)", },
+    { .name = "recovery",   .key = 'r', .has_arg = 2, .arginfo = "[TARGET]",
+      .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
+      .usage = "Start instance in recovery mode with dump file or statedir", },
+    { .name = "sysconfig",  .has_arg = 0,
+      .usage = "Load system configuration", },
+    { .name = "verbose",    .key = 'v', .has_arg = 2, .arginfo = "[LEVEL]",
+      .usage = "Be annoyingly informative by degrees", },
     { .name = "noexec",     .key = 'X', .has_arg = 0,
       .usage = "Don't execute (useful with -v, --verbose)", },
-    { .name = "bootstrap",  .key = 'b', .has_arg = 1, .arginfo = "METHOD",
-      .usage = "Set flux instance's network bootstrap method", },
-    { .name = "size",       .key = 's', .has_arg = 1, .arginfo = "N",
-      .usage = "Set number of ranks in new instance", },
     { .name = "broker-opts",.key = 'o', .has_arg = 1, .arginfo = "OPTS",
       .flags = OPTPARSE_OPT_AUTOSPLIT,
       .usage = "Add comma-separated broker options, e.g. \"-o,-v\"", },
-    { .name = "killer-timeout",.key = 'k', .has_arg = 1, .arginfo = "DURATION",
-      .usage = "After a broker exits, kill other brokers after DURATION", },
-    { .name = "trace-pmi-server", .has_arg = 0, .arginfo = NULL,
-      .usage = "Trace pmi simple server protocol exchange", },
-    { .name = "scratchdir", .key = 'D', .has_arg = 1, .arginfo = "DIR",
-      .usage = "Use DIR as scratch directory", },
-
-/* Option group 1, these options will be listed after those above */
-    { .group = 1,
-      .name = "caliper-profile", .has_arg = 1,
-      .arginfo = "PROFILE",
-      .usage = "Enable profiling in brokers using Caliper configuration "
-               "profile named `PROFILE'",
-#ifndef HAVE_CALIPER
-      .cb = no_caliper_fatal_err, /* Emit fatal err if not built w/ Caliper */
-#endif /* !HAVE_CALIPER */
-    },
+    /* Option group 1, these options will be listed after those above */
     { .group = 1,
       .name = "wrap", .has_arg = 1, .arginfo = "ARGS,...",
       .flags = OPTPARSE_OPT_AUTOSPLIT,
       .usage = "Wrap broker execution in comma-separated arguments"
     },
+    /* Option group 2 */
+    { .group = 2,
+      .usage = "\nOptions useful for testing:" },
+    { .group = 2,
+      .name = "test-size",       .key = 's', .has_arg = 1, .arginfo = "N",
+      .usage = "Start a test instance by launching N brokers locally", },
+    { .group = 2,
+      .name = "test-hosts", .has_arg = 1, .arginfo = "HOSTLIST",
+      .usage = "Set FLUX_FAKE_HOSTNAME in environment of each broker", },
+    { .group = 2,
+      .name = "test-exit-timeout", .has_arg = 1, .arginfo = "FSD",
+      .usage = "After a broker exits, kill other brokers after timeout", },
+    { .group = 2,
+      .name = "test-exit-mode", .has_arg = 1, .arginfo = "any|leader",
+      .usage = "Trigger exit timer on leader/any broker exit (default=any)", },
+    { .group = 2,
+      .name = "test-start-mode", .has_arg = 1, .arginfo = "all|leader",
+      .usage = "Start all brokers immediately or just leader (default=all)", },
+    { .group = 2,
+      .name = "test-rundir", .has_arg = 1, .arginfo = "DIR",
+      .usage = "Use DIR as broker run directory", },
+    { .group = 2,
+      .name = "test-rundir-cleanup", .has_arg = 0,
+      .usage = "Clean up --test-rundir DIR upon flux-start completion", },
+    { .group = 2,
+      .name = "test-pmi-clique",
+      .has_arg = 1,
+      .arginfo = "single|per-broker|none",
+      .usage = "Set PMI_process_mapping mode (default=single)", },
+    { .flags = OPTPARSE_OPT_HIDDEN,
+      .name = "killer-timeout", .has_arg = 1, .arginfo = "FSD",
+      .usage = "(deprecated)" },
     OPTPARSE_TABLE_END,
 };
 
-enum {
-    BOOTSTRAP_PMI,
-    BOOTSTRAP_SELFPMI
-};
-
-static struct {
-    char *string;
-    int num;
-} bootstrap_options[] = {
-    {"pmi", BOOTSTRAP_PMI},
-    {"selfpmi", BOOTSTRAP_SELFPMI},
-    {NULL, -1}
-};
-
-/* Turn the bootstrap option string into an integer value */
-static int parse_bootstrap_option (optparse_t *opts)
-{
-    const char *bootstrap;
-    int i;
-
-    bootstrap = optparse_get_str (opts, "bootstrap", "pmi");
-    for (i = 0; ; i++) {
-        if (bootstrap_options[i].string == NULL)
-            break;
-        if (!strcmp(bootstrap_options[i].string, bootstrap))
-            return bootstrap_options[i].num;
-    }
-    log_msg_exit("Unknown bootstrap method \"%s\"", bootstrap);
-}
-
 /* Various things will go wrong with module loading, process execution, etc.
  *  when current directory can't be found. Exit early with error to avoid
  *  chaotic stream of error messages later in startup.
@@ -162,23 +177,43 @@ int main (int argc, char *argv[])
     const char *searchpath;
     int optindex;
     char *broker_path;
-    int bootstrap;
 
     log_init ("flux-start");
 
     sanity_check_working_directory ();
 
-    ctx.opts = optparse_create ("flux-start");
-    if (optparse_add_option_table (ctx.opts, opts) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_add_option_table");
-    if (optparse_set (ctx.opts, OPTPARSE_USAGE, usage_msg) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_set usage");
+    if (!(ctx.opts = optparse_create ("flux-start"))
+        || optparse_add_option_table (ctx.opts, opts) != OPTPARSE_SUCCESS
+        || optparse_set (ctx.opts,
+                         OPTPARSE_OPTION_WIDTH,
+                         32) != OPTPARSE_SUCCESS
+        || optparse_set (ctx.opts,
+                         OPTPARSE_USAGE,
+                         usage_msg) != OPTPARSE_SUCCESS)
+        log_msg_exit ("error setting up option parsing");
     if ((optindex = optparse_parse_args (ctx.opts, argc, argv)) < 0)
         exit (1);
-    ctx.killer_timeout = optparse_get_duration (ctx.opts, "killer-timeout",
-                                                DEFAULT_KILLER_TIMEOUT);
-    if (ctx.killer_timeout < 0.)
-        log_msg_exit ("--killer-timeout argument must be >= 0");
+
+    ctx.exit_timeout = optparse_get_duration (ctx.opts,
+                                              "test-exit-timeout",
+                                              DEFAULT_EXIT_TIMEOUT);
+    if (!optparse_hasopt (ctx.opts, "test-exit-timeout"))
+        ctx.exit_timeout = optparse_get_duration (ctx.opts,
+                                                  "killer-timeout",
+                                                  ctx.exit_timeout);
+
+    ctx.exit_mode = optparse_get_str (ctx.opts, "test-exit-mode", "any");
+    if (!streq (ctx.exit_mode, "any")
+        && !streq (ctx.exit_mode, "leader"))
+        log_msg_exit ("unknown --test-exit-mode: %s", ctx.exit_mode);
+
+    ctx.start_mode = optparse_get_str (ctx.opts, "test-start-mode", "all");
+    if (!streq (ctx.start_mode, "all")
+        && !streq (ctx.start_mode, "leader"))
+        log_msg_exit ("unknown --test-start-mode: %s", ctx.start_mode);
+
+    ctx.verbose = optparse_get_int (ctx.opts, "verbose", 0);
+
     if (optindex < argc) {
         if ((e = argz_create (argv + optindex, &command, &len)) != 0)
             log_errn_exit (e, "argz_create");
@@ -189,35 +224,28 @@ int main (int argc, char *argv[])
     if (!(broker_path = find_broker (searchpath)))
         log_msg_exit ("Could not locate broker in %s", searchpath);
 
-    bootstrap = parse_bootstrap_option (ctx.opts);
-    if (optparse_hasopt (ctx.opts, "size")) {
-        if (bootstrap != BOOTSTRAP_SELFPMI) {
-            if (!optparse_hasopt (ctx.opts, "bootstrap")) {
-                bootstrap = BOOTSTRAP_SELFPMI;
-            } else {
-                log_errn_exit(EINVAL, "--size can only be used with --bootstrap=selfpmi");
-            }
+    if (optparse_hasopt (ctx.opts, "test-size")) {
+        ctx.test_size = optparse_get_int (ctx.opts, "test-size", -1);
+        if (ctx.test_size <= 0)
+            log_msg_exit ("--test-size argument must be > 0");
+    }
+
+    if (!optparse_hasopt (ctx.opts, "test-size")) {
+        int i;
+        for (i = 0; i < ARRAY_SIZE (opts); i++) {
+            if (opts[i].name
+                && strstarts (opts[i].name, "test-")
+                && optparse_hasopt (ctx.opts, opts[i].name))
+                log_msg_exit ("--%s only works with --test-size", opts[i].name);
         }
-        ctx.size = optparse_get_int (ctx.opts, "size", -1);
-        if (ctx.size <= 0)
-            log_msg_exit ("--size argument must be > 0");
-    }
-
-    setup_profiling_env ();
-
-    switch (bootstrap) {
-    case BOOTSTRAP_PMI:
-        if (optparse_hasopt (ctx.opts, "scratchdir"))
-            log_msg_exit ("--scratchdir only works with --bootstrap=selfpmi");
-        status = exec_broker (command, len, broker_path);
-        break;
-    case BOOTSTRAP_SELFPMI:
-        if (!optparse_hasopt (ctx.opts, "size"))
-            log_msg_exit ("--size must be specified for --bootstrap=selfpmi");
+    }
+
+    if (!optparse_hasopt (ctx.opts, "test-size")) {
+        if (exec_broker (command, len, broker_path) < 0)
+            log_err_exit ("error execing broker");
+    }
+    else {
         status = start_session (command, len, broker_path);
-        break;
-    default:
-        assert(0); /* should never happen */
     }
 
     optparse_destroy (ctx.opts);
@@ -231,34 +259,6 @@ int main (int argc, char *argv[])
     return status;
 }
 
-static void setup_profiling_env (void)
-{
-#if HAVE_CALIPER
-    const char *profile;
-    /*
-     *  If --profile was used, set or append libcaliper.so in LD_PRELOAD
-     *   to subprocess environment, swapping stub symbols for the actual
-     *   libcaliper symbols.
-     */
-    if (optparse_getopt (ctx.opts, "caliper-profile", &profile) == 1) {
-        const char *pl = getenv ("LD_PRELOAD");
-        int rc = setenvf ("LD_PRELOAD", 1, "%s%s%s",
-                          pl ? pl : "",
-                          pl ? " ": "",
-                          "libcaliper.so");
-        if (rc < 0)
-            log_err_exit ("Unable to set LD_PRELOAD in environment");
-
-        if ((profile != NULL) &&
-            (setenv ("CALI_CONFIG_PROFILE", profile, 1) < 0))
-                log_err_exit ("setenv (CALI_CONFIG_PROFILE)");
-        setenv ("CALI_LOG_VERBOSITY", "0", 0);
-    }
-#endif
-}
-
-
-
 char *find_broker (const char *searchpath)
 {
     char *cpy = xstrdup (searchpath);
@@ -275,42 +275,80 @@ char *find_broker (const char *searchpath)
     return dir ? xstrdup (path) : NULL;
 }
 
-void killer (flux_reactor_t *r, flux_watcher_t *w, int revents, void *arg)
+void exit_timeout (flux_reactor_t *r,
+                   flux_watcher_t *w,
+                   int revents, void *arg)
 {
     struct client *cli;
 
     cli = zlist_first (ctx.clients);
     while (cli) {
-        flux_future_t *f = flux_subprocess_kill (cli->p, SIGKILL);
-        if (f)
+        if (cli->p) {
+            flux_future_t *f = flux_subprocess_kill (cli->p, SIGKILL);
             flux_future_destroy (f);
+        }
         cli = zlist_next (ctx.clients);
     }
 }
 
+void update_timer (void)
+{
+    struct client *cli;
+    int count = 0;
+    bool leader_exit = false;
+    bool shutdown = false;
+
+    cli = zlist_first (ctx.clients);
+    while (cli) {
+        if (cli->p)
+            count++;
+        if (cli->rank == 0 && !cli->p)
+            leader_exit = true;
+        cli = zlist_next (ctx.clients);
+    }
+    if (streq (ctx.exit_mode, "any")) {
+        if (count > 0 && count < ctx.test_size)
+            shutdown = true;
+    }
+    else if (streq (ctx.exit_mode, "leader")) {
+        if (count > 0 && leader_exit)
+            shutdown = true;
+    }
+    if (shutdown)
+        flux_watcher_start (ctx.timer);
+    else
+        flux_watcher_stop (ctx.timer);
+}
+
 static void completion_cb (flux_subprocess_t *p)
 {
     struct client *cli = flux_subprocess_aux_get (p, "cli");
-    int rc;
 
     assert (cli);
 
-    if ((rc = flux_subprocess_exit_code (p)) < 0) {
+    if ((cli->exit_rc = flux_subprocess_exit_code (p)) < 0) {
         /* bash standard, signals + 128 */
-        if ((rc = flux_subprocess_signaled (p)) >= 0)
-            rc += 128;
+        if ((cli->exit_rc = flux_subprocess_signaled (p)) >= 0)
+            cli->exit_rc += 128;
     }
 
-    if (rc > ctx.exit_rc)
-        ctx.exit_rc = rc;
-
-    if (--ctx.count > 0)
-        flux_watcher_start (ctx.timer);
-    else
-        flux_watcher_stop (ctx.timer);
+    /* In 'any' mode, the highest of the broker exit codes is
+     * flux-start's exit code.  In 'leader' mode, the leader broker's
+     * exit code is flux-start's exit code.
+     */
+    if (streq (ctx.exit_mode, "any")) {
+        if (cli->exit_rc > ctx.exit_rc)
+            ctx.exit_rc = cli->exit_rc;
+    }
+    else if (streq (ctx.exit_mode, "leader")) {
+        if (cli->rank == 0)
+            ctx.exit_rc = cli->exit_rc;
+    }
 
-    zlist_remove (ctx.clients, cli);
-    client_destroy (cli);
+    flux_subprocess_destroy (cli->p);
+    cli->p = NULL;
+    client_wait_respond (cli);
+    update_timer ();
 }
 
 static void state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
@@ -319,43 +357,57 @@ static void state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
 
     assert (cli);
 
-    if (state == FLUX_SUBPROCESS_FAILED) {
-        log_errn (errno, "%d FAILED", cli->rank);
-        if (--ctx.count > 0)
-            flux_watcher_start (ctx.timer);
-        else
-            flux_watcher_stop (ctx.timer);
-        zlist_remove (ctx.clients, cli);
-        client_destroy (cli);
-    }
-    else if (state == FLUX_SUBPROCESS_EXITED) {
-        pid_t pid = flux_subprocess_pid (p);
-        int status;
-
-        if ((status = flux_subprocess_status (p)) >= 0) {
-            if (WIFSIGNALED (status))
-                log_msg ("%d (pid %d) %s", cli->rank, pid, strsignal (WTERMSIG (status)));
-            else if (WIFEXITED (status) && WEXITSTATUS (status) != 0)
-                log_msg ("%d (pid %d) exited with rc=%d", cli->rank, pid, WEXITSTATUS (status));
-        } else
-            log_msg ("%d (pid %d) exited, unknown status", cli->rank, pid);
+    switch (state) {
+        case FLUX_SUBPROCESS_INIT:
+            break;
+        case FLUX_SUBPROCESS_RUNNING:
+            client_run_respond (cli, 0);
+            break;
+        case FLUX_SUBPROCESS_FAILED: { // completion will not be called
+            log_errn_exit (flux_subprocess_fail_errno (p),
+                           "%d subprocess failed",
+                           cli->rank);
+            break;
+        }
+        case FLUX_SUBPROCESS_EXITED: {
+            pid_t pid = flux_subprocess_pid (p);
+            int status = flux_subprocess_status (p);
+
+            assert (status >= 0);
+            if (WIFSIGNALED (status)) {
+                log_msg ("%d (pid %d) %s",
+                         cli->rank,
+                         pid,
+                         strsignal (WTERMSIG (status)));
+            }
+            else if (WIFEXITED (status) && WEXITSTATUS (status) != 0) {
+                log_msg ("%d (pid %d) exited with rc=%d",
+                         cli->rank,
+                         pid,
+                         WEXITSTATUS (status));
+            }
+            break;
+        }
+        case FLUX_SUBPROCESS_STOPPED:
+            /* ignore */
+            break;
     }
 }
 
 void channel_cb (flux_subprocess_t *p, const char *stream)
 {
     struct client *cli = flux_subprocess_aux_get (p, "cli");
-    const char *ptr;
-    int rc, lenp;
+    const char *buf;
+    int rc, len;
 
     assert (cli);
-    assert (!strcmp (stream, "PMI_FD"));
+    assert (streq (stream, "PMI_FD"));
 
-    if (!(ptr = flux_subprocess_read_line (p, stream, &lenp)))
+    if ((len = flux_subprocess_read_line (p, stream, &buf)) < 0)
         log_err_exit ("%s: flux_subprocess_read_line", __FUNCTION__);
 
-    if (lenp) {
-        rc = pmi_simple_server_request (ctx.pmi.srv, ptr, cli, cli->rank);
+    if (len) {
+        rc = pmi_simple_server_request (ctx.pmi.srv, buf, cli, cli->rank);
         if (rc < 0)
             log_err_exit ("%s: pmi_simple_server_request", __FUNCTION__);
         if (rc == 1)
@@ -363,25 +415,40 @@ void channel_cb (flux_subprocess_t *p, const char *stream)
     }
 }
 
-void add_args_list (char **argz, size_t *argz_len, optparse_t *opt, const char *name)
+void add_argzf (char **argz, size_t *argz_len, const char *fmt, ...)
+{
+    va_list ap;
+    char *arg = NULL;
+
+    va_start (ap, fmt);
+    if (vasprintf (&arg, fmt, ap) < 0)
+        log_err_exit ("vasprintf");
+    va_end (ap);
+    if (argz_add (argz, argz_len, arg) != 0)
+        log_err_exit ("argz_add");
+    free (arg);
+}
+
+void add_args_list (char **argz,
+                    size_t *argz_len,
+                    optparse_t *opt,
+                    const char *name,
+                    const char *prepend)
 {
     const char *arg;
     optparse_getopt_iterator_reset (opt, name);
     while ((arg = optparse_getopt_next (opt, name)))
-        if (argz_add  (argz, argz_len, arg) != 0)
-            log_err_exit ("argz_add");
+        add_argzf (argz, argz_len, "%s%s", prepend, arg);
 }
 
-char *create_scratch_dir (void)
+char *create_rundir (void)
 {
     char *tmpdir = getenv ("TMPDIR");
-    char *scratchdir = xasprintf ("%s/flux-%d-XXXXXX",
-                                  tmpdir ? tmpdir : "/tmp", (int)getpid());
+    char *rundir = xasprintf ("%s/flux-XXXXXX", tmpdir ? tmpdir : "/tmp");
 
-    if (!mkdtemp (scratchdir))
-        log_err_exit ("mkdtemp %s", scratchdir);
-    cleanup_push_string (cleanup_directory_recursive, scratchdir);
-    return scratchdir;
+    if (!mkdtemp (rundir))
+        log_err_exit ("mkdtemp %s", rundir);
+    return rundir;
 }
 
 static int pmi_response_send (void *client, const char *buf)
@@ -396,15 +463,19 @@ static void pmi_debug_trace (void *client, const char *buf)
     fprintf (stderr, "%d: %s", cli->rank, buf);
 }
 
-int pmi_kvs_put (void *arg, const char *kvsname,
-                 const char *key, const char *val)
+int pmi_kvs_put (void *arg,
+                 const char *kvsname,
+                 const char *key,
+                 const char *val)
 {
     zhash_update (ctx.pmi.kvs, key, xstrdup (val));
     zhash_freefn (ctx.pmi.kvs, key, (zhash_free_fn *)free);
     return 0;
 }
 
-int pmi_kvs_get (void *arg, void *client, const char *kvsname,
+int pmi_kvs_get (void *arg,
+                 void *client,
+                 const char *kvsname,
                  const char *key)
 {
     char *v = zhash_lookup (ctx.pmi.kvs, key);
@@ -413,43 +484,150 @@ int pmi_kvs_get (void *arg, void *client, const char *kvsname,
     return 0;
 }
 
+void pmi_abort (void *arg,
+                void *client,
+                int exit_code,
+                const char *error_message)
+{
+    struct client *cli = client;
+
+    log_msg ("%d: PMI_Abort()%s%s",
+             cli->rank,
+             error_message ? ": " : "",
+             error_message ? error_message : "");
+
+    cli = zlist_first (ctx.clients);
+    while (cli) {
+        if (cli->p) {
+            flux_future_t *f = flux_subprocess_kill (cli->p, SIGKILL);
+            flux_future_destroy (f);
+        }
+        cli = zlist_next (ctx.clients);
+    }
+}
+
 int execvp_argz (char *argz, size_t argz_len)
 {
     char **av = malloc (sizeof (char *) * (argz_count (argz, argz_len) + 1));
-    if (!av) {
-        errno = ENOMEM;
+    if (!av)
         return -1;
-    }
     argz_extract (argz, argz_len, av);
     execvp (av[0], av);
     free (av);
     return -1;
 }
 
+bool system_instance_is_running (void)
+{
+    flux_t *h;
+    bool running = false;
+
+    unsetenv ("FLUX_URI");
+    if ((h = flux_open (NULL, 0))) {
+        running = true;
+        flux_close (h);
+    }
+    return running;
+}
+
+void process_recovery_option (char **argz,
+                              size_t *argz_len,
+                              bool *system_recovery)
+{
+    char path[1024];
+    const char *optarg;
+    struct stat sb;
+
+    add_argzf (argz, argz_len, "-Sbroker.recovery-mode=1");
+    add_argzf (argz, argz_len, "-Sbroker.quorum=1");
+    add_argzf (argz, argz_len, "-Slog-stderr-level=5");
+
+    // if --recovery has no optional argument, assume this is the system
+    // instance and make sure it is not running.
+    if (!(optarg = optparse_get_str (ctx.opts, "recovery", NULL))) {
+        optarg = default_statedir;
+        if (system_instance_is_running ())
+            log_msg_exit ("system instance is already running");
+        *system_recovery = true;
+    }
+
+    // if argument is a dir, assume statedir; if file, assume dump archive
+    if (stat (optarg, &sb) < 0)
+        log_err_exit ("%s", optarg);
+    if (S_ISDIR (sb.st_mode)) {
+        if (sb.st_uid != getuid ())
+            log_msg_exit ("%s: not owned by you", optarg);
+        if ((sb.st_mode & S_IRWXU) != S_IRWXU)
+            log_msg_exit ("%s: no access", optarg);
+        snprintf (path, sizeof (path), "%s/content.sqlite", optarg);
+        if (access (path, F_OK) < 0)
+            log_err_exit ("%s", path);
+        if (access (path, R_OK) < 0)
+            log_msg_exit ("%s: no read permission", path);
+        if (access (path, W_OK) < 0)
+            log_msg_exit ("%s: no write permission", path);
+        add_argzf (argz, argz_len, "-Sstatedir=%s", optarg);
+    }
+    else
+        add_argzf (argz, argz_len, "-Scontent.restore=%s", optarg);
+}
+
+int add_args_common (char **argz,
+                     size_t *argz_len,
+                     const char *broker_path)
+{
+    bool system_recovery = false;
+    const char *config_path = NULL;
+
+    add_args_list (argz, argz_len, ctx.opts, "wrap", "");
+    if (argz_add (argz, argz_len, broker_path) != 0) {
+        errno = ENOMEM;
+        return -1;
+    }
+    add_args_list (argz, argz_len, ctx.opts, "setattr", "-S");
+    add_args_list (argz, argz_len, ctx.opts, "broker-opts", "");
+
+    if (optparse_hasopt (ctx.opts, "recovery"))
+        process_recovery_option (argz, argz_len, &system_recovery);
+
+    if (system_recovery || optparse_hasopt (ctx.opts, "sysconfig"))
+        config_path = default_config_path;
+    if (optparse_hasopt (ctx.opts, "config-path")) {
+        const char *conf = optparse_get_str (ctx.opts, "config-path", NULL);
+        if (config_path)
+            log_msg ("Warning: overriding recovery/--sysconfig path with %s",
+                     conf);
+        config_path = conf;
+    }
+    if (config_path)
+        add_argzf (argz, argz_len, "-c%s", config_path);
+
+    return 0;
+}
+
 /* Directly exec() a single flux broker.  It is assumed that we
  * are running in an environment with an external PMI service, and the
  * broker will figure out how to bootstrap without any further aid from
  * flux-start.
  */
-int exec_broker (const char *cmd_argz, size_t cmd_argz_len,
+int exec_broker (const char *cmd_argz,
+                 size_t cmd_argz_len,
                  const char *broker_path)
 {
     char *argz = NULL;
     size_t argz_len = 0;
 
-    add_args_list (&argz, &argz_len, ctx.opts, "wrap");
-    if (argz_add (&argz, &argz_len, broker_path) != 0)
-        goto nomem;
+    if (add_args_common (&argz, &argz_len, broker_path) < 0)
+        goto error;
 
-    add_args_list (&argz, &argz_len, ctx.opts, "broker-opts");
     if (cmd_argz) {
         if (argz_append (&argz, &argz_len, cmd_argz, cmd_argz_len) != 0)
             goto nomem;
     }
-    if (optparse_hasopt (ctx.opts, "verbose")) {
+    if (ctx.verbose >= 1) {
         char *cpy = malloc (argz_len);
         if (!cpy)
-            goto nomem;
+            goto error;
         memcpy (cpy, argz, argz_len);
         argz_stringify (cpy, argz_len, ' ');
         log_msg ("%s", cpy);
@@ -464,28 +642,31 @@ int exec_broker (const char *cmd_argz, size_t cmd_argz_len,
 nomem:
     errno = ENOMEM;
 error:
-    free (argz);
+    ERRNO_SAFE_WRAP (free, argz);
     return -1;
 }
 
-struct client *client_create (const char *broker_path, const char *scratch_dir,
-                              int rank, const char *cmd_argz, size_t cmd_argz_len)
+struct client *client_create (const char *broker_path,
+                              const char *rundir,
+                              int rank,
+                              const char *cmd_argz,
+                              size_t cmd_argz_len,
+                              const char *hostname)
 {
     struct client *cli = xzmalloc (sizeof (*cli));
     char *arg;
-    char * argz = NULL;
+    char *argz = NULL;
     size_t argz_len = 0;
 
     cli->rank = rank;
-    add_args_list (&argz, &argz_len, ctx.opts, "wrap");
-    argz_add (&argz, &argz_len, broker_path);
-    char *dir_arg = xasprintf ("--setattr=rundir=%s", scratch_dir);
-    argz_add (&argz, &argz_len, dir_arg);
-    argz_add (&argz, &argz_len, "--setattr=tbon.endpoint=ipc://%B/req");
-    free (dir_arg);
-    add_args_list (&argz, &argz_len, ctx.opts, "broker-opts");
-    if (rank == 0 && cmd_argz)
-        argz_append (&argz, &argz_len, cmd_argz, cmd_argz_len); /* must be last arg */
+
+    if (add_args_common (&argz, &argz_len, broker_path) < 0)
+        goto fail;
+
+    add_argzf (&argz, &argz_len, "--setattr=rundir=%s", rundir);
+
+    if (rank == 0 && cmd_argz) /* must be last arg */
+        argz_append (&argz, &argz_len, cmd_argz, cmd_argz_len);
 
     if (!(cli->cmd = flux_cmd_create (0, NULL, environ)))
         goto fail;
@@ -499,14 +680,23 @@ struct client *client_create (const char *broker_path, const char *scratch_dir,
 
     if (flux_cmd_add_channel (cli->cmd, "PMI_FD") < 0)
         log_err_exit ("flux_cmd_add_channel");
-    if (flux_cmd_setenvf (cli->cmd, 1, "PMI_RANK", "%d", rank) < 0)
-        log_err_exit ("flux_cmd_setenvf");
-    if (flux_cmd_setenvf (cli->cmd, 1, "PMI_SIZE", "%d", ctx.size) < 0)
-        log_err_exit ("flux_cmd_setenvf");
+    if (flux_cmd_setenvf (cli->cmd, 1, "PMI_RANK", "%d", rank) < 0
+        || flux_cmd_setenvf (cli->cmd, 1, "PMI_SIZE", "%d", ctx.test_size) < 0
+        || flux_cmd_setenvf (cli->cmd,
+                             1,
+                             "FLUX_START_URI",
+                             "local://%s/start",
+                             rundir) < 0
+        || (hostname && flux_cmd_setenvf (cli->cmd,
+                                          1,
+                                          "FLUX_FAKE_HOSTNAME",
+                                          "%s",
+                                          hostname) < 0))
+            log_err_exit ("error setting up environment for rank %d", rank);
     return cli;
 fail:
-    free (argz);
-    client_destroy (cli);
+    ERRNO_SAFE_WRAP (free, argz);
+    ERRNO_SAFE_WRAP (client_destroy, cli);
     return NULL;
 }
 
@@ -538,7 +728,12 @@ void client_dumpargs (struct client *cli)
 
 void pmi_server_initialize (int flags)
 {
+    struct taskmap *map;
+    const char *mode = optparse_get_str (ctx.opts,
+                                         "test-pmi-clique",
+                                         "single");
     struct pmi_simple_ops ops = {
+        .abort = pmi_abort,
         .kvs_put = pmi_kvs_put,
         .kvs_get = pmi_kvs_get,
         .barrier_enter = NULL,
@@ -546,12 +741,39 @@ void pmi_server_initialize (int flags)
         .debug_trace = pmi_debug_trace,
     };
     int appnum = 0;
-    if (!(ctx.pmi.kvs = zhash_new()))
+
+
+    if (!(ctx.pmi.kvs = zhash_new ())
+        || !(map = taskmap_create ()))
         oom ();
-    ctx.pmi.srv = pmi_simple_server_create (ops, appnum, ctx.size,
-                                            ctx.size, "-", flags, NULL);
+
+    if (streq (mode, "single")) {
+        if (taskmap_append (map, 0, 1, ctx.test_size) < 0)
+            log_err_exit ("error encoding PMI_process_mapping");
+    }
+    else if (streq (mode, "per-broker")) {
+        if (taskmap_append (map, 0, ctx.test_size, 1) < 0)
+            log_err_exit ("error encoding PMI_process_mapping");
+    }
+    else if (!streq (mode, "none"))
+        log_msg_exit ("unsupported test-pmi-clique mode: %s", mode);
+
+    if (taskmap_nnodes (map) > 0) {
+        char *s;
+        if (!(s = taskmap_encode (map, TASKMAP_ENCODE_PMI)))
+            log_msg_exit ("error encoding PMI_process_mapping");
+        zhash_update (ctx.pmi.kvs, "PMI_process_mapping", s);
+    }
+    ctx.pmi.srv = pmi_simple_server_create (ops,
+                                            appnum,
+                                            ctx.test_size,
+                                            ctx.test_size,
+                                            "-",
+                                            flags,
+                                            NULL);
     if (!ctx.pmi.srv)
         log_err_exit ("pmi_simple_server_create");
+    taskmap_destroy (map);
 }
 
 void pmi_server_finalize (void)
@@ -569,14 +791,20 @@ int client_run (struct client *cli)
         .on_stdout = NULL,
         .on_stderr = NULL,
     };
+    int flags = 0;
+    if (cli->p) {
+        errno = EEXIST;
+        return -1;
+    }
     /* We want stdio fallthrough so subprocess can capture tty if
      * necessary (i.e. an interactive shell)
      */
+    flags |= FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH;
+    flags |= FLUX_SUBPROCESS_FLAGS_NO_SETPGRP;
     if (!(cli->p = flux_local_exec (ctx.reactor,
-                                    FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH,
+                                    flags,
                                     cli->cmd,
-                                    &ops,
-                                    NULL)))
+                                    &ops)))
         log_err_exit ("flux_exec");
     if (flux_subprocess_aux_set (cli->p, "cli", cli, NULL) < 0)
         log_err_exit ("flux_subprocess_aux_set");
@@ -589,20 +817,258 @@ void restore_termios (void)
         log_err ("tcsetattr");
 }
 
-/* Start an internal PMI server, and then launch "size" number of
- * broker processes that inherit a file desciptor to the internal PMI
+void status_cb (flux_t *h,
+                flux_msg_handler_t *mh,
+                const flux_msg_t *msg,
+                void *arg)
+{
+    struct client *cli;
+    json_t *procs = NULL;
+
+    if (!(procs = json_array()))
+        goto nomem;
+    cli = zlist_first (ctx.clients);
+    while (cli) {
+        json_t *entry;
+
+        if (!(entry = json_pack ("{s:i}",
+                                 "pid", flux_subprocess_pid (cli->p))))
+            goto nomem;
+        if (json_array_append_new (procs, entry) < 0) {
+            json_decref (entry);
+            goto nomem;
+        }
+        cli = zlist_next (ctx.clients);
+    }
+    if (flux_respond_pack (h, msg, "{s:O}", "procs", procs) < 0)
+        log_err ("error responding to status request");
+    json_decref (procs);
+    return;
+nomem:
+    errno = ENOMEM;
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        log_err ("error responding to status request");
+    json_decref (procs);
+}
+
+static struct client *client_lookup (int rank)
+{
+    struct client *cli;
+
+    cli = zlist_first (ctx.clients);
+    while (cli) {
+        if (cli->rank == rank)
+            return cli;
+        cli = zlist_next (ctx.clients);
+    }
+    errno = ESRCH;
+    return NULL;
+}
+
+/* Send 'signum' to 'cli'.  Since this is always a local operation,
+ * the future is immediately fulfilled, so just destroy it.
+ * If cli is not running, return success.
+ */
+static int client_kill (struct client *cli, int signum)
+{
+    flux_future_t *f;
+    if (!cli->p)
+        return 0;
+    if (!(f = flux_subprocess_kill (cli->p, signum)))
+        return -1;
+    flux_future_destroy (f);
+    return 0;
+}
+
+/* Respond with errum result to pending run request, if any.
+ */
+static void client_run_respond (struct client *cli, int errnum)
+{
+    if (cli->run_request) {
+        int rc;
+        if (errnum == 0)
+            rc = flux_respond (ctx.h, cli->run_request, NULL);
+        else
+            rc = flux_respond_error (ctx.h, cli->run_request, errnum, NULL);
+        if (rc < 0)
+            log_err ("error responding to start.run request");
+        flux_msg_decref (cli->run_request);
+        cli->run_request = NULL;
+    }
+}
+
+static void client_wait_respond (struct client *cli)
+{
+    if (cli->wait_request) {
+        if (flux_respond_pack (ctx.h,
+                               cli->wait_request,
+                               "{s:i}",
+                               "exit_rc", cli->exit_rc) < 0)
+            log_err ("error responding to wait request");
+        flux_msg_decref (cli->wait_request);
+        cli->wait_request = NULL;
+    }
+}
+
+/* Send signal to one broker by rank.
+ */
+void kill_cb (flux_t *h,
+              flux_msg_handler_t *mh,
+              const flux_msg_t *msg,
+              void *arg)
+{
+    int rank;
+    int signum;
+    struct client *cli;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s:i}",
+                             "rank", &rank,
+                             "signum", &signum) < 0)
+        goto error;
+    if (!(cli = client_lookup (rank)))
+        goto error;
+    if (client_kill (cli, signum) < 0)
+        goto error;
+    if (flux_respond (h, msg, NULL) < 0)
+        log_err ("error responding to kill request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        log_err ("error responding to kill request");
+}
+
+/* Wait for one broker to complete and return its exit_rc.
+ * If the child is not running, return cli->exit_rc immediately.  Otherwise,
+ * the request is parked on the 'struct child' (one request allowed per child),
+ * and response is sent by completion handler upon broker completion.
+ */
+void wait_cb (flux_t *h,
+              flux_msg_handler_t *mh,
+              const flux_msg_t *msg,
+              void *arg)
+{
+    int rank;
+    struct client *cli;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i}",
+                             "rank", &rank) < 0)
+        goto error;
+    if (!(cli = client_lookup (rank)))
+        goto error;
+    if (cli->wait_request) {
+        errno = EEXIST;
+        goto error;
+    }
+    cli->wait_request = flux_msg_incref (msg);
+    if (!cli->p)
+        client_wait_respond (cli);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        log_err ("error responding to start request");
+}
+
+/* Run one broker by rank.
+ */
+void run_cb (flux_t *h,
+             flux_msg_handler_t *mh,
+             const flux_msg_t *msg,
+             void *arg)
+{
+    int rank;
+    struct client *cli;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i}",
+                             "rank", &rank) < 0)
+        goto error;
+    if (!(cli = client_lookup (rank)))
+        goto error;
+    if (cli->run_request) {
+        errno = EEXIST;
+        goto error;
+    }
+    if (client_run (cli) < 0)
+        goto error;
+    cli->run_request = flux_msg_incref (msg);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        log_err ("error responding to start request");
+}
+
+void disconnect_cb (flux_t *h,
+                    flux_msg_handler_t *mh,
+                    const flux_msg_t *msg,
+                    void *arg)
+{
+    const char *uuid;
+
+    if (!(uuid = flux_msg_route_first (msg)))
+        return;
+    if (ctx.verbose >= 1)
+        log_msg ("disconnect from %.5s", uuid);
+}
+
+const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST, "start.status", status_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "start.kill", kill_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "start.wait", wait_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "start.run", run_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "disconnect", disconnect_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+/* Set up test-related RPC handlers on local://${rundir}/start
+ * Ensure that service-related reactor watchers do not contribute to the
+ * reactor usecount, since the reactor is expected to exit once the
+ * subprocesses are complete.
+ */
+void start_server_initialize (const char *rundir, bool verbose)
+{
+    char path[1024];
+    if (snprintf (path, sizeof (path), "%s/start", rundir) >= sizeof (path))
+        log_msg_exit ("internal buffer overflow");
+    if (!(ctx.h = usock_service_create (ctx.reactor, path, verbose)))
+        log_err_exit ("could not created embedded flux-start server");
+    if (flux_msg_handler_addvec (ctx.h, htab, NULL, &ctx.handlers) < 0)
+        log_err_exit ("could not register service methods");
+    /* Service related watchers:
+     * - usock server listen fd
+     * - flux_t handle watcher (adds 2 active prep/check watchers)
+     */
+    int ignore_watchers = 3;
+    while (ignore_watchers-- > 0)
+        flux_reactor_active_decref (ctx.reactor);
+}
+
+void start_server_finalize (void)
+{
+    flux_msg_handler_delvec (ctx.handlers);
+    flux_close (ctx.h);
+}
+
+/* Start an internal PMI server, and then launch the requested number of
+ * broker processes that inherit a file descriptor to the internal PMI
  * server.  They will use that to bootstrap.  Since the PMI server is
  * internal and the connections to it passed through inherited file
  * descriptors it implies that the brokers in this instance must all
  * be contained on one node.  This is mostly useful for testing purposes.
  */
-int start_session (const char *cmd_argz, size_t cmd_argz_len,
+int start_session (const char *cmd_argz,
+                   size_t cmd_argz_len,
                    const char *broker_path)
 {
     struct client *cli;
     int rank;
     int flags = 0;
-    char *scratch_dir;
+    char *rundir;
+    struct hostlist *hosts = NULL;
 
     if (isatty (STDIN_FILENO)) {
         if (tcgetattr (STDIN_FILENO, &ctx.saved_termios) < 0)
@@ -615,46 +1081,90 @@ int start_session (const char *cmd_argz, size_t cmd_argz_len,
     if (!(ctx.reactor = flux_reactor_create (FLUX_REACTOR_SIGCHLD)))
         log_err_exit ("flux_reactor_create");
     if (!(ctx.timer = flux_timer_watcher_create (ctx.reactor,
-                                                  ctx.killer_timeout, 0.,
-                                                  killer, NULL)))
+                                                 ctx.exit_timeout,
+                                                 0.,
+                                                 exit_timeout,
+                                                 NULL)))
         log_err_exit ("flux_timer_watcher_create");
     if (!(ctx.clients = zlist_new ()))
         log_err_exit ("zlist_new");
 
-    if (optparse_hasopt (ctx.opts, "scratchdir"))
-        scratch_dir = xstrdup (optparse_get_str (ctx.opts, "scratchdir", NULL));
+    if (optparse_hasopt (ctx.opts, "test-rundir")) {
+        struct stat sb;
+        rundir = xstrdup (optparse_get_str (ctx.opts, "test-rundir", NULL));
+        if (stat (rundir, &sb) < 0)
+            log_err_exit ("%s", rundir);
+        if (!S_ISDIR (sb.st_mode))
+            log_msg_exit ("%s: not a directory", rundir);
+    }
     else
-        scratch_dir = create_scratch_dir ();
+        rundir = create_rundir ();
+    /* Clean up rundir upon flux-start completion if we created it,
+     * or if cleanup was explicitly requested.
+     */
+    if (!optparse_hasopt (ctx.opts, "test-rundir")
+        || optparse_hasopt (ctx.opts, "test-rundir-cleanup"))
+        cleanup_push_string (cleanup_directory_recursive, rundir);
 
-    if (optparse_hasopt (ctx.opts, "trace-pmi-server"))
+    start_server_initialize (rundir, ctx.verbose >= 1 ? true : false);
+
+    if (ctx.verbose >= 2)
         flags |= PMI_SIMPLE_SERVER_TRACE;
 
     pmi_server_initialize (flags);
 
-    for (rank = 0; rank < ctx.size; rank++) {
-        if (!(cli = client_create (broker_path, scratch_dir, rank,
-                                   cmd_argz, cmd_argz_len)))
+    if (optparse_hasopt (ctx.opts, "test-hosts")) {
+        const char *s = optparse_get_str (ctx.opts, "test-hosts", NULL);
+        if (!(hosts = hostlist_decode (s)))
+            log_msg_exit ("could not decode --test-hosts hostlist");
+        if (hostlist_count (hosts) != ctx.test_size)
+            log_msg_exit ("--test-hosts hostlist has incorrect size");
+    }
+
+    for (rank = 0; rank < ctx.test_size; rank++) {
+        if (!(cli = client_create (broker_path,
+                                   rundir,
+                                   rank,
+                                   cmd_argz,
+                                   cmd_argz_len,
+                                   hosts ? hostlist_nth (hosts, rank) : NULL)))
             log_err_exit ("client_create");
-        if (optparse_hasopt (ctx.opts, "verbose"))
+        if (ctx.verbose >= 1)
             client_dumpargs (cli);
         if (optparse_hasopt (ctx.opts, "noexec")) {
             client_destroy (cli);
             continue;
         }
-        if (client_run (cli) < 0)
-            log_err_exit ("client_run");
         if (zlist_append (ctx.clients, cli) < 0)
             log_err_exit ("zlist_append");
-        ctx.count++;
+    }
+    if (streq (ctx.start_mode, "leader")) {
+        cli = zlist_first (ctx.clients);
+        if (client_run (cli) < 0)
+            log_err_exit ("client_run");
+    }
+    else if (streq (ctx.start_mode, "all")) {
+        cli = zlist_first (ctx.clients);
+        while (cli) {
+            if (client_run (cli) < 0)
+                log_err_exit ("client_run");
+            cli = zlist_next (ctx.clients);
+        }
     }
     if (flux_reactor_run (ctx.reactor, 0) < 0)
         log_err_exit ("flux_reactor_run");
 
     pmi_server_finalize ();
+    start_server_finalize ();
 
-    free (scratch_dir);
+    hostlist_destroy (hosts);
+    free (rundir);
 
-    zlist_destroy (&ctx.clients);
+    if (ctx.clients) {
+        while ((cli = zlist_pop (ctx.clients)))
+            client_destroy (cli);
+        zlist_destroy (&ctx.clients);
+    }
     flux_watcher_destroy (ctx.timer);
     flux_reactor_destroy (ctx.reactor);
 
diff --git a/src/cmd/flux-submit.py b/src/cmd/flux-submit.py
new file mode 100755
index 000000000000..be06c2d34fb4
--- /dev/null
+++ b/src/cmd/flux-submit.py
@@ -0,0 +1,43 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import logging
+import sys
+
+import flux
+import flux.cli.submit as base
+
+LOGGER = logging.getLogger("flux-submit")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    # Prepare the submit parser
+    submit = base.SubmitCmd(
+        "flux submit",
+        description="enqueue a job",
+    )
+    parser = submit.get_parser()
+    parser.set_defaults(func=submit.main)
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-terminus.c b/src/cmd/flux-terminus.c
new file mode 100644
index 000000000000..49c1ef632e88
--- /dev/null
+++ b/src/cmd/flux-terminus.c
@@ -0,0 +1,763 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-terminus.c - terminal session management service for Flux
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <string.h>
+#include <assert.h>
+#include <unistd.h>
+#include <time.h>
+
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/fdwalk.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/llog.h"
+
+#include "src/common/libterminus/terminus.h"
+#include "src/common/libterminus/pty.h"
+
+#define TERMINUS_DOC "\
+Simple terminal session manager and multiplexer for Flux.\n\
+Options:\n"
+
+int cmd_start (optparse_t *p, int argc, char **argv);
+int cmd_attach (optparse_t *p, int argc, char **argv);
+int cmd_list (optparse_t *p, int argc, char **argv);
+int cmd_kill (optparse_t *p, int argc, char **argv);
+int cmd_kill_server (optparse_t *p, int argc, char **argv);
+
+static struct optparse_option global_opts[] =  {
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option start_opts[] = {
+    { .name = "detach", .key = 'd',
+      .usage = "Start new session and immediately detach"
+    },
+    { .name = "wait", .key = 'w',
+      .usage = "Do not clear sessions from server on exit with --detach."
+               " Instead, hold session in an 'exited' state until at least"
+               " one client has attached."
+    },
+    { .name = "name", .key = 'n',
+      .has_arg = 1, .arginfo = "NAME",
+      .usage = "Set session name to NAME (default: arg0)",
+    },
+    { .name = "rank", .key = 'r',
+      .has_arg = 1, .arginfo = "RANK",
+      .usage = "Attach to session on rank RANK (default: local rank)",
+    },
+    { .name = "service", .key = 's',
+      .has_arg = 1, .arginfo = "NAME",
+      .usage = "Use service NAME (default USERID-terminus)."
+    },
+    { .name = "pipe", .key = 'p',
+      .usage = "Pipe stdin to the session and exit. Do not display output",
+      .flags = OPTPARSE_OPT_HIDDEN
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option attach_opts[] = {
+    { .name = "rank", .key = 'r',
+      .has_arg = 1, .arginfo = "RANK",
+      .usage = "Attach to session on rank RANK (default: local rank)",
+    },
+    { .name = "service", .key = 's',
+      .has_arg = 1, .arginfo = "NAME",
+      .usage = "Attach at service NAME (default USERID-terminus)."
+    },
+    { .name = "pipe", .key = 'p',
+      .usage = "Pipe stdin to the session and exit. Do not display output",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option list_opts[] = {
+    { .name = "rank", .key = 'r',
+      .has_arg = 1, .arginfo = "RANK",
+      .usage = "Attach to session on rank RANK (default: local rank)",
+    },
+    { .name = "service", .key = 's',
+      .has_arg = 1, .arginfo = "NAME",
+      .usage = "Use service NAME (default USERID-terminus)."
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option kill_opts[] = {
+    { .name = "rank", .key = 'r',
+      .has_arg = 1, .arginfo = "RANK",
+      .usage = "Kill session on rank RANK (default: local rank)",
+    },
+    { .name = "service", .key = 's',
+      .has_arg = 1, .arginfo = "NAME",
+      .usage = "Kill at service NAME (default USERID-terminus). "
+    },
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_option kill_server_opts[] = {
+    { .name = "rank", .key = 'r',
+      .has_arg = 1, .arginfo = "RANK",
+      .usage = "Kill server on rank RANK (default: local rank)",
+    },
+    { .name = "service", .key = 's',
+      .has_arg = 1, .arginfo = "NAME",
+      .usage = "Kill server at NAME (default USERID-terminus)."
+    },
+    OPTPARSE_TABLE_END
+};
+
+
+static struct optparse_subcommand subcommands[] = {
+    { "start",
+      "[OPTIONS] [COMMAND...]",
+      "Start a new session",
+      cmd_start,
+      0,
+      start_opts,
+    },
+    { "attach",
+      "[OPTIONS] ID",
+      "Attach to existing session",
+      cmd_attach,
+      0,
+      attach_opts,
+    },
+    { "list",
+      NULL,
+      "list active sessions",
+      cmd_list,
+      0,
+      list_opts,
+    },
+    { "kill",
+      "[OPTIONS] ID",
+      "kill active session ID",
+      cmd_kill,
+      0,
+      kill_opts,
+    },
+     { "kill-server",
+      NULL,
+      "tell terminus server to exit",
+      cmd_kill_server,
+      0,
+      kill_server_opts,
+    },
+    OPTPARSE_SUBCMD_END
+};
+
+int main (int argc, char *argv[])
+{
+    optparse_t *p;
+    int optindex;
+    int exitval;
+
+    log_init ("flux-terminus");
+
+    p = optparse_create ("flux-terminus");
+
+    if (optparse_add_option_table (p, global_opts) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_add_option_table() failed");
+
+    if (optparse_add_doc (p, TERMINUS_DOC, 0) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_add_doc failed");
+
+    if (optparse_reg_subcommands (p, subcommands) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_reg_subcommands");
+
+    if ((optindex = optparse_parse_args (p, argc, argv)) < 0)
+        exit (1);
+
+    if ((argc - optindex == 0)
+        || !optparse_get_subcommand (p, argv[optindex]))
+        optparse_fatal_usage (p, 1, NULL);
+
+    if ((exitval = optparse_run_subcommand (p, argc, argv)) < 0)
+        exit (1);
+
+    optparse_destroy (p);
+    log_fini ();
+    return (exitval);
+}
+
+char *service_name (optparse_t *p, const char *method,
+                          char *buf, int buflen)
+{
+    char default_service [64];
+    const char *service = NULL;
+
+    if (optparse_getopt (p, "service", &service) <= 0) {
+        /*  <userid>-terminus is guaranteed to fit in buffer:
+         */
+        (void) sprintf (default_service,
+                        "%u-terminus",
+                        (unsigned) getuid ());
+        service = default_service;
+    }
+    if (snprintf (buf,
+                  buflen,
+                  "%s%s%s",
+                  service,
+                  method ? "." : "",
+                  method ? method : "") >= buflen) {
+        log_msg ("service_name: service name too long");
+        return NULL;
+    }
+    return buf;
+}
+
+static void terminus_server_closefd (void *arg, int fd)
+{
+    int *savefd = arg;
+
+    if (fd != *savefd
+        && fd != STDIN_FILENO
+        && fd != STDOUT_FILENO
+        && fd != STDERR_FILENO)
+        (void) close (fd);
+}
+
+static void close_stdio ()
+{
+    int fd = open ("/dev/null", O_RDWR);
+    if (fd >= 0) {
+        dup2 (fd, STDIN_FILENO);
+        dup2 (fd, STDOUT_FILENO);
+        dup2 (fd, STDERR_FILENO);
+        close (fd);
+    }
+}
+
+static void f_logf (void *arg,
+                    const char *file,
+                    int line,
+                    const char *func,
+                    const char *subsys,
+                    int level,
+                    const char *fmt,
+                    va_list ap)
+{
+    flux_t *h = arg;
+    char buf [2048];
+    int buflen = sizeof (buf);
+    int n = vsnprintf (buf, buflen, fmt, ap);
+    if (n >= sizeof (buf)) {
+        buf[buflen - 1] = '\0';
+        buf[buflen - 1] = '+';
+    }
+    flux_log (h, level, "%s:%d: %s: %s", file, line, func, buf);
+}
+
+static void unregister_cb (flux_future_t *f, void *arg)
+{
+    if (flux_future_get (f, NULL) < 0)
+        log_err ("failed to unregister terminus service");
+    flux_reactor_stop ((flux_reactor_t *) arg);
+}
+
+static void empty_cb (struct flux_terminus_server *ts,
+                      void *arg)
+{
+    flux_future_t *f;
+    if (!(f = flux_terminus_server_unregister (ts))
+        || flux_future_then (f, -1., unregister_cb, arg) < 0)
+        log_err_exit ("failed to unregister terminus service");
+}
+
+static int run_service (const char *service, int fd)
+{
+    flux_t *h;
+    struct flux_terminus_server *ts;
+    flux_future_t *f;
+    int rc = -1;
+    flux_reactor_t *r;
+
+    if (fdwalk (terminus_server_closefd, &fd) < 0) {
+        log_err ("fdwalk");
+        goto err;
+    }
+
+    r = flux_reactor_create (FLUX_REACTOR_SIGCHLD);
+    if (!r) {
+        log_err ("flux_reactor_create");
+        goto err;
+    }
+    if (!(h = flux_open (NULL, 0))) {
+        log_err ("flux_open");
+        goto err;
+    }
+    flux_set_reactor (h, r);
+    if (!(f = flux_service_register (h, service))
+        || flux_future_get (f, NULL) < 0) {
+        log_err ("flux_service_register (%s)", service);
+        goto err;
+    }
+    flux_future_destroy (f);
+
+    if (!(ts = flux_terminus_server_create (h, service))) {
+        log_err ("flux_terminus_server_create");
+        goto err;
+    }
+
+    /* Notify grandparent that we're ready */
+    close (fd);
+    close_stdio ();
+
+    flux_terminus_server_set_log (ts, f_logf, h);
+
+    /*  Set up to exit when the last session exits
+     */
+    flux_terminus_server_notify_empty (ts, empty_cb, r);
+
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    flux_terminus_server_destroy (ts);
+    flux_reactor_destroy (r);
+    flux_close (h);
+    return rc;
+err:
+    if (write (fd, &errno, sizeof (int)) < 0)
+        log_err ("write");
+    return -1;
+}
+
+/*  Turn current process into daemon and start terminus server.
+ *   Close write end of pfds when ready.
+ */
+static int start_service_daemon (flux_t *orig_h, optparse_t *p)
+{
+    pid_t pid;
+    int pfds[2];
+    int result = 0;
+    char service [64];
+
+    if (!service_name (p, NULL, service, sizeof (service)))
+        log_msg_exit ("failed to get service name");
+
+    /*  Create pipe to allow server to signal readiness */
+    if (pipe (pfds) < 0)
+        log_msg_exit ("pipe");
+
+    if ((pid = fork ()) < 0)
+        log_err_exit ("fork");
+    else if (pid == 0) {
+        /* Child: cleanup, fork again and execute server */
+        flux_close (orig_h);
+        close (pfds[0]);
+        setsid ();
+        if ((pid = fork ()) < 0)
+            log_err_exit ("child: fork");
+        else if (pid == 0) {
+            /* Run server */
+            if (run_service (service, pfds[1]) < 0)
+                exit (1);
+            exit (0);
+        }
+        /* Parent: exit */
+        exit (0);
+    }
+
+    /*  Wait for child, wait for grandchild to close pipe */
+    waitpid (pid, NULL, 0);
+    close (pfds[1]);
+
+    /*  Read status from grandchild */
+    if (read (pfds[0], &result, sizeof (int)) < 0)
+        log_err_exit ("Failed to get status of server");
+    close (pfds[0]);
+
+    errno = result;
+    return result == 0 ? 0 : -1;
+}
+
+static json_t *build_cmd (optparse_t *p, int argc, char **argv)
+{
+    json_t *cmd = NULL;
+
+    if (!(cmd = json_array ()))
+        return NULL;
+
+    for (int i = 0; i < argc; i++) {
+        json_t *o = json_string (argv[i]);
+        if (o == NULL)
+            goto err;
+        if (json_array_append_new (cmd, o) < 0) {
+            json_decref (o);
+            goto err;
+        }
+    }
+    return cmd;
+err:
+    json_decref (cmd);
+    return NULL;
+}
+
+static int new_session (flux_t *h, optparse_t *p,
+                        const char *name,
+                        int argc,
+                        char **argv,
+                        char **pty_service)
+{
+    int rc = -1;
+    json_t *cmd = NULL;
+    flux_future_t *f = NULL;
+    char service [128];
+    const char *s;
+    int wait = 0;
+    int rank = optparse_get_int (p, "rank", FLUX_NODEID_ANY);
+
+    if (!(service_name (p, "new", service, sizeof (service))))
+        goto err;
+    cmd = build_cmd (p, argc, argv);
+
+    if (!optparse_hasopt (p, "detach") || optparse_hasopt (p, "wait"))
+        wait = 1;
+
+    if (!(f = flux_rpc_pack (h,
+                             service,
+                             rank,
+                             0,
+                             "{s:s s:o? s:i}",
+                             "name", name ? name : "",
+                             "cmd", cmd,
+                             "wait", wait)))
+        goto err;
+    if (flux_rpc_get_unpack (f,"{s:s}", "pty_service", &s) < 0) {
+        log_err ("new session: %s", future_strerror (f, errno));
+        goto err;
+    }
+    if (!(*pty_service = strdup (s)))
+        goto err;
+    rc = 0;
+err:
+    flux_future_destroy (f);
+    return rc;
+}
+
+flux_future_t * list_sessions (flux_t *h, optparse_t *p)
+{
+    char service [128];
+    int rank = optparse_get_int (p, "rank", FLUX_NODEID_ANY);
+
+    if (!(service_name (p, "list", service, sizeof (service))))
+        log_msg_exit ("Failed to build service name");
+
+    return flux_rpc (h, service, NULL, rank, 0);
+}
+
+static void exit_cb (struct flux_pty_client *c, void *arg)
+{
+    flux_t *h = arg;
+    flux_reactor_stop (flux_get_reactor (h));
+}
+
+static int attach_session (flux_t *h,
+                           optparse_t *p,
+                           const char *pty_service)
+{
+    int status = 0;
+    int flags = FLUX_PTY_CLIENT_CLEAR_SCREEN
+              | FLUX_PTY_CLIENT_NOTIFY_ON_DETACH
+              | FLUX_PTY_CLIENT_ATTACH_SYNC;
+    struct flux_pty_client *c = NULL;
+    int rank = optparse_get_int (p, "rank", FLUX_NODEID_ANY);
+
+    if (optparse_hasopt (p, "pipe")) {
+        flags = FLUX_PTY_CLIENT_STDIN_PIPE
+                | FLUX_PTY_CLIENT_ATTACH_SYNC
+                | FLUX_PTY_CLIENT_NORAW;
+    }
+
+    if (!(c = flux_pty_client_create ())
+        || flux_pty_client_set_flags (c, flags) < 0)
+        log_err_exit ("flux_pty_client_create");
+
+    if (flux_pty_client_attach (c, h, rank, pty_service) < 0) {
+        if (errno == ENOSYS)
+            log_msg_exit ("Invalid session or server at %s", pty_service);
+        else
+            log_err_exit ("flux_pty_client_attach");
+    }
+
+    if (flux_pty_client_notify_exit (c, exit_cb, h) < 0)
+        log_msg_exit ("flux_pty_client_notify_exit");
+
+    flux_reactor_run (flux_get_reactor (h), 0);
+
+    if (flux_pty_client_exit_status (c, &status) < 0)
+        log_err ("failed to get remote exit status");
+
+    /* Exit with semi "standard" exit code (as shell might have) */
+    if (status != 0) {
+        int code = 1;
+        if (WIFSIGNALED (status))
+            code = 128 + WTERMSIG (status);
+        else
+            code = WEXITSTATUS (status);
+        exit (code);
+    }
+    flux_pty_client_destroy (c);
+    return 0;
+}
+
+/*  from flux-job.c
+ */
+static bool isnumber (const char *s, int *result)
+{
+    char *endptr;
+    long int l;
+
+    errno = 0;
+    l = strtol (s, &endptr, 10);
+    if (errno
+        || *endptr != '\0'
+        || l < 0) {
+        return false;
+    }
+    *result = (int) l;
+    return true;
+}
+
+int cmd_attach (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    const char *idstr;
+    char service [128];
+    int id;
+    int optindex = optparse_option_index (p);
+
+    if (getenv ("FLUX_TERMINUS_SESSION"))
+        log_msg_exit ("Nesting flux-terminus sessions not supported");
+
+    if (argc - optindex != 1)
+        optparse_fatal_usage (p, 1, "session ID required\n");
+
+    idstr = argv[optindex];
+    if (!isnumber (idstr, &id))
+        optparse_fatal_usage (p, 1, "session ID must be an integer\n");
+    if (!service_name (p, idstr, service, sizeof (service)))
+        log_msg_exit ("service_name");
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (attach_session (h, p, service) < 0)
+        log_msg_exit ("Failed to attach to session at %s", service);
+
+    flux_close (h);
+    return 0;
+}
+
+int cmd_start (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    json_t *o = NULL;
+    const char *name = NULL;
+    char *pty_service = NULL;
+    flux_future_t *f = NULL;
+    int optindex = optparse_option_index (p);
+
+    if (getenv ("FLUX_TERMINUS_SESSION"))
+        log_msg_exit ("Nesting flux-terminus sessions not supported");
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    /*  If no server currently running at requested service endpoint,
+     *   then start one in background on this rank
+     */
+    if (!(f = list_sessions (h, p))
+        || flux_future_get (f, NULL) < 0) {
+        if (optparse_hasopt (p, "rank"))
+            log_msg_exit ("Unable to start a new server with --rank option");
+
+        flux_future_destroy (f);
+        f = NULL;
+
+        /*  Fork service daemon. Only parent returns */
+        if (start_service_daemon (h, p) < 0)
+            log_msg_exit ("Failed to start a new server");
+    }
+    json_decref (o);
+
+    argc -= optindex;
+    argv += optindex;
+
+    name = optparse_get_str (p, "name", NULL);
+    if (new_session (h, p, name, argc, argv, &pty_service) < 0)
+        log_msg_exit ("Failed to start new session");
+
+    if (!optparse_hasopt (p, "detach")
+        && attach_session (h, p, pty_service) < 0)
+        log_msg_exit ("Failed to attach to session at %s", pty_service);
+
+    free (pty_service);
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+static const char *timestr (double ts, char *buf, size_t size)
+{
+    struct tm tm;
+    time_t sec = ts;
+    if (!gmtime_r (&sec, &tm)
+        || strftime (buf, size, "%c", &tm) == 0)
+        return "Unknown";
+    return buf;
+}
+
+static int print_session (json_t *o)
+{
+    int id;
+    int clients;
+    int exited;
+    const char *name;
+    double ctime;
+    char timebuf [64];
+
+    if (json_unpack (o, "{s:i s:i s:s s:i s:f}",
+                     "id", &id,
+                     "clients", &clients,
+                     "name", &name,
+                     "exited", &exited,
+                     "ctime", &ctime) < 0)
+        return -1;
+    printf ("%d: [%s]%s %d clients (created %s)\n",
+            id, name, exited ? " (exited)" : "",
+            clients, timestr (ctime, timebuf, sizeof (timebuf)));
+    return 0;
+}
+
+int cmd_list (optparse_t *p, int argc, char **argv)
+{
+    flux_future_t *f = NULL;
+    json_t *l = NULL;
+    const char *service;
+    char datestr [64];
+    int rank;
+    size_t n;
+    double ctime;
+    flux_t *h;
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    if (!(f= list_sessions (h, p)))
+        log_err_exit ("list_sessions");
+    if (flux_rpc_get_unpack (f, "{s:o s:{s:s s:i s:f}}",
+                             "sessions", &l,
+                             "server",
+                               "service", &service,
+                               "rank", &rank,
+                               "ctime", &ctime) < 0) {
+        char name[64];
+        if (errno == ENOSYS)
+            log_msg_exit ("no server running at %s",
+                          service_name (p, NULL, name, sizeof (name)));
+        else
+            log_err_exit ("list sessions failed");
+    }
+    timestr (ctime, datestr, sizeof (datestr));
+    printf ("server at %s running on rank %d since %s\n",
+            service,
+            rank,
+            datestr);
+    if ((n = json_array_size (l)))
+        printf ("%zu current session%s:\n", n, n > 1 ? "s" : "");
+    else
+        printf ("no sessions\n");
+    for (int i = 0; i < json_array_size (l); i++)
+        print_session (json_array_get (l, i));
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+int cmd_kill (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    flux_future_t *f;
+    char service [128];
+    const char *idstr;
+    int id = -1;
+    int optindex = optparse_option_index (p);
+    int rank = optparse_get_int (p, "rank", FLUX_NODEID_ANY);
+
+    if (argc - optindex != 1)
+        optparse_fatal_usage (p, 1, "session ID required\n");
+
+    idstr = argv[optindex];
+    if (!isnumber (idstr, &id))
+        optparse_fatal_usage (p, 1, "session ID must be an integer\n");
+    if (!service_name (p, "kill", service, sizeof (service)))
+        log_msg_exit ("service_name");
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (!(f = flux_rpc_pack (h, service, rank, 0,
+                             "{s:i s:i s:i}",
+                             "id", id,
+                             "signal", SIGKILL,
+                             "wait", 1))
+        || flux_rpc_get (f, NULL) < 0) {
+        if (errno == ENOSYS)
+            log_msg_exit ("kill: no server running at %s",
+                          service_name (p, NULL, service, sizeof (service)));
+        log_err_exit ("kill failed");
+    }
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+int cmd_kill_server (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    flux_future_t *f;
+    char service [64];
+    int rank = optparse_get_int (p, "rank", FLUX_NODEID_ANY);
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (!service_name (p, "kill-server", service, sizeof (service)))
+        log_err_exit ("failed to build service name");
+
+    if (!(f = flux_rpc (h, service, NULL, rank, 0))
+        || flux_rpc_get (f, NULL) < 0) {
+        if (errno == ENOSYS)
+            log_msg_exit ("no server running at %s",
+                          service_name (p, NULL, service, sizeof (service)));
+        else
+            log_err_exit ("kill-server");
+    }
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/cmd/flux-update.py b/src/cmd/flux-update.py
new file mode 100755
index 000000000000..54080b0822ff
--- /dev/null
+++ b/src/cmd/flux-update.py
@@ -0,0 +1,215 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import json
+import logging
+import math
+import sys
+
+import flux
+import flux.job
+import flux.util
+
+LOGGER = logging.getLogger("flux-update")
+
+
+class JobspecUpdates:
+    """
+    Convenience class for building a jobspec-update payload from a
+    set of KEY=VALUE pairs on the command line, and a method to send
+    the update as a request to the job manager.
+    """
+
+    #  Mapping of short key names, i.e. as given on the command line,
+    #  to full dotted-path location in jobspec.
+    #
+    #  Note: If a key doesn't exist in this mapping, but also does not start
+    #  with 'attributes.', 'resources.' or 'tasks.', then 'attributes.system'
+    #  is assumed.
+    #
+    key_aliases = {"name": "attributes.system.job.name"}
+
+    def __init__(self, jobid, flux_handle=None):
+        self._flux_handle = flux_handle
+        self.jobid = jobid
+        self.updates = None
+        self.jobspec = None
+
+    @property
+    def flux_handle(self):
+        if self._flux_handle is None:
+            self._flux_handle = flux.Flux()
+        return self._flux_handle
+
+    def _fetch_jobspec(self, key):
+        """
+        Fetch dotted key 'key' in jobspec for this job.  Note that
+        job_kvs_lookup() will apply `jobspec-update` events from the
+        eventlog in the returned jobspec.
+        """
+        if self.jobspec is None:
+            lookup = flux.job.job_kvs_lookup(
+                self.flux_handle, jobid=self.jobid, keys=["jobspec"]
+            )
+            self.jobspec = flux.job.JobspecV1(**lookup["jobspec"])
+
+        return self.jobspec.getattr(key)
+
+    def update_attributes_system_duration(self, value):
+        """
+        Handle a duration update.
+
+        If update begins with "+" or "-", then get duration from jobspec and
+        increase or decrease by the amount of the remaining argument. O/w,
+        treat value as an explicit new duration.
+        """
+        result = None
+        if value.startswith(("-", "+")):
+            # relative update, fetch value first
+            duration = self._fetch_jobspec("attributes.system.duration")
+            if duration == 0:
+                raise ValueError(
+                    f"duration for {self.jobid} is unlimited, "
+                    f"can't update by {value}"
+                )
+
+            arg = flux.util.parse_fsd(value[1:])
+            if value.startswith("-"):
+                result = duration - arg
+                if result <= 0.0:
+                    duration = flux.util.fsd(duration)
+                    raise ValueError(
+                        f"current duration for {self.jobid} ({duration})"
+                        f" cannot be reduced by {value[1:]}"
+                    )
+            else:
+                result = duration + arg
+        else:
+            result = flux.util.parse_fsd(value)
+
+        # An unlimited duration is represented as 0. in jobspec, so
+        # check for infinity here and replace with 0.
+        #
+        if math.isinf(result):
+            result = 0.0
+        return result
+
+    def add_update(self, key, value):
+        """
+        Append an update to the current updates object.
+        """
+        if self.updates is None:
+            self.updates = {}
+
+        #  Handle any special keys aliases
+        if key in self.key_aliases:
+            key = self.key_aliases[key]
+
+        #  If key doesn't start with attributes, resources, or tasks,
+        #  assume 'attributes.system.' for convenience:
+        if not key.startswith(("attributes.", "resources.", "tasks.")):
+            key = f"attributes.system.{key}"
+        try:
+            #  Use any function update_attributes_system_blah() if
+            #  registered to process the value:
+            #
+            function_signature = "update_" + key.replace(".", "_")
+            value = getattr(self, function_signature)(value)
+        except AttributeError:
+            #  Otherwise, attempt to load value as JSON:
+            #
+            try:
+                value = json.loads(value)
+            except json.decoder.JSONDecodeError:
+                #  Otherwise, load value as string:
+                #
+                value = str(value)
+        self.updates[key] = value
+
+    def items(self):
+        """
+        Convenience wrapper to return a copy of the current update
+        dictionary key, value pairs
+        """
+        return self.updates.items()
+
+    def to_json(self):
+        return json.dumps(self.updates)
+
+    def send_rpc(self):
+        payload = {"id": self.jobid, "updates": self.updates}
+        return self.flux_handle.rpc("job-manager.update", payload)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        prog="flux-update", formatter_class=flux.util.help_formatter()
+    )
+    parser.add_argument(
+        "-n",
+        "--dry-run",
+        action="store_true",
+        help="Do not apply any updates, just emit update payload to stdout",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        default=0,
+        help="Be more verbose. Log updated items after success.",
+    )
+    parser.add_argument(
+        "jobid",
+        metavar="JOBID",
+        type=flux.job.JobID,
+        help="Target jobid",
+    )
+    parser.add_argument(
+        "updates",
+        metavar="KEY=VALUE",
+        type=str,
+        nargs="+",
+        help="Requested jobspec updates in KEY=VALUE form",
+    )
+    return parser.parse_args()
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    args = parse_args()
+
+    updates = JobspecUpdates(args.jobid)
+
+    for arg in args.updates:
+        key, _, value = arg.partition("=")
+        updates.add_update(key, value)
+
+    if args.dry_run:
+        print(updates.to_json())
+        sys.exit(0)
+
+    updates.send_rpc().get()
+    if args.verbose:
+        for key, value in updates.items():
+            LOGGER.info(f"updated {key} to {value}")
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-uri.py b/src/cmd/flux-uri.py
new file mode 100755
index 000000000000..8189208fff8a
--- /dev/null
+++ b/src/cmd/flux-uri.py
@@ -0,0 +1,76 @@
+##############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+import os
+import sys
+
+import flux
+from flux.uri import FluxURIResolver
+
+LOGGER = logging.getLogger("flux-uri")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    resolver = FluxURIResolver()
+    plugins = [f"  {x}\t\t{y}" for x, y in resolver.plugins().items()]
+    plugin_list = "Supported resolver schemes:\n" + "\n".join(plugins)
+
+    parser = argparse.ArgumentParser(
+        prog="flux-uri",
+        description="Resolve TARGET to a Flux URI",
+        epilog=plugin_list,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--remote",
+        action="store_true",
+        help="convert a local URI to remote",
+    )
+    parser.add_argument(
+        "--local",
+        action="store_true",
+        help="convert a remote URI to local",
+        default=os.environ.get("FLUX_URI_RESOLVE_LOCAL"),
+    )
+    parser.add_argument(
+        "--wait",
+        action="store_true",
+        help="wait for a URI to become available, if supported",
+    )
+    parser.add_argument(
+        "uri",
+        metavar="TARGET",
+        help="A Flux jobid or URI in scheme:argument form (e.g. jobid:f1234)",
+    )
+
+    args = parser.parse_args()
+    if args.wait:
+        args.uri = args.uri + "?wait"
+    uri = resolver.resolve(args.uri)
+    if args.remote:
+        print(uri.remote)
+    elif args.local:
+        print(uri.local)
+    else:
+        print(uri)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux-watch.py b/src/cmd/flux-watch.py
new file mode 100755
index 000000000000..2c8bee242b8b
--- /dev/null
+++ b/src/cmd/flux-watch.py
@@ -0,0 +1,208 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import logging
+import os
+import sys
+
+import flux
+from flux.job import JobID, JobList
+from flux.job.watcher import JobWatcher
+from flux.util import (
+    FilterAction,
+    FilterActionSetUpdate,
+    FilterTrueAction,
+    help_formatter,
+)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        prog="flux-watch", formatter_class=help_formatter()
+    )
+    parser.add_argument(
+        "-a",
+        "--active",
+        action=FilterTrueAction,
+        help="Watch all active jobs for current user",
+    )
+    parser.add_argument(
+        "-A",
+        "--all",
+        action=FilterTrueAction,
+        help="Watch all jobs for current user in all states including inactive",
+    )
+    parser.add_argument(
+        "-c",
+        "--count",
+        action=FilterAction,
+        type=int,
+        metavar="N",
+        default=0,
+        help="Limit to N jobs (default 1000)",
+    )
+    parser.add_argument(
+        "-f",
+        "--filter",
+        action=FilterActionSetUpdate,
+        metavar="STATE|RESULT",
+        default=set(),
+        help="Watch jobs with specific job state or result",
+    )
+    parser.add_argument(
+        "--since",
+        action=FilterAction,
+        type=str,
+        metavar="WHEN",
+        default=0.0,
+        help="Include only jobs that have been inactive since WHEN. "
+        + "(implies -a if no other --filter option is specified)",
+    )
+    parser.add_argument(
+        "--progress",
+        action="store_true",
+        default=False,
+        help="Show progress bar",
+    )
+    parser.add_argument(
+        "--jps",
+        action="store_true",
+        default=False,
+        help="Display jobs/s instead of elapsed time ",
+    )
+    parser.add_argument(
+        "-l",
+        "--label-io",
+        action="store_true",
+        default=False,
+        help="label lines of output with task id",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Increase verbosity",
+    )
+    parser.add_argument(
+        "jobids",
+        metavar="JOBID",
+        type=JobID,
+        nargs="*",
+        help="Watch specific jobids",
+    )
+    parser.set_defaults(filtered=False)
+    return parser.parse_args()
+
+
+LOGGER = logging.getLogger("flux-watch")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    starttime = None
+    since = 0.0
+
+    args = parse_args()
+
+    if args.jobids and args.filtered:
+        LOGGER.warning("Filtering options ignored with jobid list")
+
+    fh = flux.Flux()
+    instance_starttime = float(fh.attr_get("broker.starttime"))
+
+    if args.since:
+        #  Implies --all unless any other filter option is already in effect.
+        if not args.filter:
+            args.all = True
+        try:
+            since = flux.util.parse_datetime(args.since).timestamp()
+        except ValueError:
+            try:
+                since = float(args.since)
+            except ValueError:
+                LOGGER.error(f"--since: invalid value '{args.since}'")
+                sys.exit(2)
+
+        # Ensure args.since is in the past
+        if since > flux.util.parse_datetime("now").timestamp():
+            LOGGER.error("--since=%s appears to be in the future", args.since)
+            sys.exit(2)
+
+        #  With since, start elapsed timer at maximum of since time or
+        #  instance starttime
+        starttime = max(since, instance_starttime)
+
+    if args.active:
+        args.filter.update(("pending", "running"))
+
+    if args.all:
+        args.filter.update(("pending", "running", "inactive"))
+        if not starttime:
+            starttime = instance_starttime
+
+    if not args.filter and not args.jobids:
+        LOGGER.error("At least one job selection option is required")
+        print("Try 'flux watch --help' for more information.", file=sys.stderr)
+        sys.exit(1)
+
+    joblist = JobList(
+        fh,
+        ids=args.jobids,
+        attrs=["state", "result", "t_submit"],
+        filters=args.filter,
+        since=since,
+        user=str(os.getuid()),
+        max_entries=args.count,
+    )
+    jobs = list(joblist.jobs())
+    for errmsg in joblist.errors:
+        LOGGER.error(errmsg)
+
+    if not jobs:
+        LOGGER.info("No matching jobs")
+        sys.exit(0)
+
+    if args.verbose:
+        LOGGER.info(f"Watching {len(jobs)} jobs")
+
+    if args.progress and not sys.stdout.isatty():
+        LOGGER.warning("stdout is not a tty. Ignoring --progress option")
+        args.progress = False
+
+    watcher = JobWatcher(
+        fh,
+        jobs=jobs,
+        progress=args.progress,
+        jps=args.jps,
+        log_events=(args.verbose > 1),
+        log_status=(args.verbose > 0),
+        labelio=args.label_io,
+        starttime=starttime,
+    ).start()
+
+    fh.reactor_run()
+
+    sys.exit(watcher.exitcode)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/flux.c b/src/cmd/flux.c
index 20d8cab3c4ab..7334f9b05bca 100644
--- a/src/cmd/flux.c
+++ b/src/cmd/flux.c
@@ -22,6 +22,7 @@
 #include <flux/optparse.h>
 #include <pwd.h>
 
+#include "ccan/str/str.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/environment.h"
@@ -71,7 +72,12 @@ void usage (optparse_t *p)
 
     optparse_print_usage (p);
     fprintf (stderr, "\n");
+    fprintf (stderr, "For general Flux documentation, please visit\n");
+    fprintf (stderr, "    https://flux-framework.readthedocs.io\n");
     emit_command_help (help_pattern, stderr);
+    fprintf (stderr, "\n");
+    fprintf (stderr, "See 'flux help COMMAND' for more information about ");
+    fprintf (stderr, "a specific command.\n");
     free (help_pattern);
 }
 
@@ -183,7 +189,8 @@ int main (int argc, char *argv[])
     }
 
     environment_from_env (env, "FLUX_EXEC_PATH", "", ':');
-    environment_push (env, "FLUX_EXEC_PATH",
+    environment_push (env,
+                      "FLUX_EXEC_PATH",
                       flux_conf_builtin_get ("exec_path", flags));
     environment_push (env, "FLUX_EXEC_PATH", getenv ("FLUX_EXEC_PATH_PREPEND"));
 
@@ -191,30 +198,30 @@ int main (int argc, char *argv[])
     environment_push (env,
                       "FLUX_CONNECTOR_PATH",
                       flux_conf_builtin_get ("connector_path", flags));
-    environment_push (env, "FLUX_CONNECTOR_PATH",
+    environment_push (env,
+                      "FLUX_CONNECTOR_PATH",
                       getenv ("FLUX_CONNECTOR_PATH_PREPEND"));
 
     environment_from_env (env, "FLUX_MODULE_PATH", "", ':');
     environment_push (env,
                       "FLUX_MODULE_PATH",
                       flux_conf_builtin_get ("module_path", flags));
-    environment_push (env, "FLUX_MODULE_PATH",
+    environment_push (env,
+                      "FLUX_MODULE_PATH",
                       getenv ("FLUX_MODULE_PATH_PREPEND"));
 
-    /* Set FLUX_SEC_DIRECTORY, possibly to $HOME/.flux.
-     */
-    setup_keydir (env, flags);
-
     if (getenv ("FLUX_URI"))
         environment_from_env (env, "FLUX_URI", "", 0); /* pass-thru */
 
-    environment_from_env (env,
-                          "FLUX_PMI_LIBRARY_PATH",
-                          flux_conf_builtin_get ("pmi_library_path", flags), 0);
+    /*  Deduplicate any other FLUX_* PATH-type environment variables by
+     *   calling environment_from_env() on them
+     */
+    environment_from_env (env, "FLUX_RC_EXTRA", NULL, ':');
+    environment_from_env (env, "FLUX_SHELL_RC_PATH", NULL, ':');
 
     environment_apply (env);
 
-    /* If --parent, push parent environment for each occurence
+    /* If --parent, push parent environment for each occurrence
      */
     for (int n = optparse_getopt (p, "parent", NULL); n > 0; n--) {
         push_parent_environment (p, env);
@@ -245,10 +252,45 @@ bool flux_is_installed (void)
 {
     int rc = executable_is_intree ();
     if (rc < 0)
-        log_err_exit ("Failed to to determine if flux is installed");
+        log_err_exit ("Failed to determine if flux is installed");
     return (rc == 0);
 }
 
+void ensure_self_first_in_path (struct environment *e, const char *selfdir)
+{
+    const char *entry = NULL;
+    char realdir[PATH_MAX+1];
+    char path[PATH_MAX+1];
+
+    while ((entry = environment_var_next (e, "PATH", entry))) {
+        /*  Attempt to canonicalize path, skipping eny elements that
+         *  can't be resolved.
+         */
+        if (!(realpath (entry, realdir)))
+            continue;
+        /*  If this path matches "selfdir", then the current flux
+         *  executable already appears first in path. Return.
+         */
+        if (streq (realdir, selfdir))
+            return;
+        /*  Otherwise, check for a flux in this path element, if it
+         *  is present and executable, then the current flux is not
+         *  first in path, insert selfdir before this element.
+         */
+        if (snprintf (path,
+                      sizeof (path),
+                      "%s/flux",
+                      realpath (entry, realdir)) >= sizeof (path)
+            || access (path, R_OK|X_OK) == 0) {
+            if (environment_insert (e, "PATH", (char *) entry, selfdir) < 0)
+                break;
+            return;
+        }
+    }
+    /* No flux(1) found in current PATH, we can insert selfdir at back */
+    environment_push_back (e, "PATH", selfdir);
+}
+
 /*
  * If flux command was run with relative or absolute path, then
  *  prepend the directory for the flux executable to PATH. This
@@ -262,36 +304,16 @@ void setup_path (struct environment *env, const char *argv0)
     assert (argv0);
 
     /*  If argv[0] was explicitly "flux" then assume PATH is already set */
-    if (strcmp (argv0, "flux") == 0)
+    if (streq (argv0, "flux"))
         return;
     if ((selfdir = executable_selfdir ())) {
         environment_from_env (env, "PATH", "/bin:/usr/bin", ':');
-        environment_push (env, "PATH", executable_selfdir ());
+        ensure_self_first_in_path (env, selfdir);
     }
     else
         log_msg_exit ("Unable to determine flux executable dir");
 }
 
-void setup_keydir (struct environment *env, int flags)
-{
-    const char *dir = getenv ("FLUX_SEC_DIRECTORY");
-    char *new_dir = NULL;
-
-    if (!dir)
-        dir = flux_conf_builtin_get ("keydir", flags);
-    if (!dir) {
-        struct passwd *pw = getpwuid (getuid ());
-        if (!pw)
-            log_msg_exit ("Who are you!?!");
-        dir = new_dir = xasprintf ("%s/.flux", pw->pw_dir);
-    }
-    if (!dir)
-        log_msg_exit ("Could not determine keydir");
-    environment_set (env, "FLUX_SEC_DIRECTORY", dir, 0);
-    if (new_dir)
-        free (new_dir);
-}
-
 /* Check for a flux-<command>.py in dir and execute it under the configured
  * PYTHON_INTERPRETER if found.
  */
@@ -300,17 +322,24 @@ void exec_subcommand_py (bool vopt, const char *dir,
 	                     const char *prefix)
 {
     char *path = xasprintf ("%s%s%s%s.py",
-            dir ? dir : "",
-            dir ? "/" : "",
-            prefix ? prefix : "", argv[0]);
+                            dir ? dir : "",
+                            dir ? "/" : "",
+                            prefix ? prefix : "", argv[0]);
+
     if (access (path, R_OK|X_OK) == 0) {
-        char *av [argc+2];
+        const char *wrapper = flux_conf_builtin_get ("python_wrapper",
+                                                     FLUX_CONF_AUTO);
+        char *av [argc+3];
         av[0] = PYTHON_INTERPRETER;
-        av[1] = path;
-        for (int i = 2; i < argc+2; i++)
-            av[i] = argv[i-1];
+        av[1] = (char *) wrapper;
+        av[2] = path;
+        for (int i = 3; i < argc+3; i++)
+            av[i] = argv[i-2];
         if (vopt)
-            log_msg ("trying to exec %s %s", PYTHON_INTERPRETER, path);
+            log_msg ("trying to exec %s %s %s",
+                     PYTHON_INTERPRETER,
+                     wrapper,
+                     path);
         execvp (PYTHON_INTERPRETER, av);
     }
     free (path);
@@ -320,9 +349,9 @@ void exec_subcommand_dir (bool vopt, const char *dir, char *argv[],
         const char *prefix)
 {
     char *path = xasprintf ("%s%s%s%s",
-            dir ? dir : "",
-            dir ? "/" : "",
-            prefix ? prefix : "", argv[0]);
+                            dir ? dir : "",
+                            dir ? "/" : "",
+                            prefix ? prefix : "", argv[0]);
     if (vopt)
         log_msg ("trying to exec %s", path);
     execvp (path, argv); /* no return if successful */
@@ -347,7 +376,8 @@ void exec_subcommand (const char *searchpath, bool vopt, int argc, char *argv[])
             a1 = NULL;
         }
         free (cpy);
-        log_msg_exit ("`%s' is not a flux command.  See 'flux --help'", argv[0]);
+        log_msg_exit ("`%s' is not a flux command.  See 'flux --help'",
+                      argv[0]);
     }
 }
 
@@ -374,7 +404,6 @@ static void push_parent_environment (optparse_t *p, struct environment *env)
 {
     const char *uri;
     const char *ns;
-    const char *path;
     flux_t *h = flux_open_internal (p, NULL);
 
     if (h == NULL)
@@ -387,7 +416,7 @@ static void push_parent_environment (optparse_t *p, struct environment *env)
         return;
 
     environment_set (env, "FLUX_URI", uri, 0);
-    
+
     /*  Before closing current instance handle, set FLUX_KVS_NAMESPACE
      *   if parent-kvs-namespace attr is set.
      */
@@ -396,16 +425,11 @@ static void push_parent_environment (optparse_t *p, struct environment *env)
     else
         environment_unset (env, "FLUX_KVS_NAMESPACE");
 
-    /*  Now close current handle and connect to parent URI, pushing
-     *   parent exec and module paths onto env
+    /*  Now close current handle and connect to parent URI.
      */
     flux_close_internal (p);
     if (!(h = flux_open_internal (p, uri)))
         log_err_exit ("flux_open (parent)");
-    if ((path = flux_attr_get (h, "conf.exec_path")))
-        environment_push (env, "FLUX_EXEC_PATH", path);
-    if ((path = flux_attr_get (h, "conf.module_path")))
-        environment_push (env, "FLUX_MODULE_PATH", path);
 }
 
 static void print_environment (struct environment *env)
diff --git a/src/cmd/job/attach.c b/src/cmd/job/attach.c
new file mode 100644
index 000000000000..6123ba2b8bef
--- /dev/null
+++ b/src/cmd/job/attach.c
@@ -0,0 +1,1386 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job attach */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <stdio.h>
+#include <jansson.h>
+#include <sys/ioctl.h>
+#include <signal.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+#include <flux/idset.h>
+#include <flux/taskmap.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/xzmalloc.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libeventlog/formatter.h"
+#include "src/common/libioencode/ioencode.h"
+#include "src/common/libutil/fdutils.h"
+#include "src/common/libsubprocess/fbuf.h"
+#include "src/common/libsubprocess/fbuf_watcher.h"
+#include "src/common/libtaskmap/taskmap_private.h"
+#include "src/common/libjob/idf58.h"
+
+#include "src/common/libterminus/pty.h"
+#include "src/common/libdebugged/debugged.h"
+#include "src/common/libutil/monotime.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "job/common.h"
+#include "job/mpir.h"
+
+char *totalview_jobid = NULL;
+
+int stdin_flags;
+
+struct optparse_option attach_opts[] = {
+    { .name = "show-events", .key = 'E', .has_arg = 0,
+      .usage = "Show job events on stderr",
+    },
+    { .name = "show-exec", .key = 'X', .has_arg = 0,
+      .usage = "Show exec events on stderr",
+    },
+    {
+      .name = "show-status", .has_arg = 0,
+      .usage = "Show job status line while pending",
+    },
+    { .name = "wait-event", .key = 'w', .has_arg = 1, .arginfo = "NAME",
+      .usage = "Wait for event NAME before detaching from eventlog "
+               "(default=finish)"
+    },
+    { .name = "label-io", .key = 'l', .has_arg = 0,
+      .usage = "Label output by rank",
+    },
+    { .name = "verbose", .key = 'v', .has_arg = 0,
+      .usage = "Increase verbosity",
+    },
+    { .name = "quiet", .key = 'q', .has_arg = 0,
+      .usage = "Suppress warnings written to stderr from flux-job",
+    },
+    { .name = "read-only", .key = 'r', .has_arg = 0,
+      .usage = "Disable reading stdin and capturing signals",
+    },
+    { .name = "unbuffered", .key = 'u', .has_arg = 0,
+      .usage = "Disable buffering of stdin",
+    },
+    { .name = "stdin-ranks", .key = 'i', .has_arg = 1, .arginfo = "RANKS",
+      .usage = "Send standard input to only RANKS (default: all)"
+    },
+    { .name = "debug", .has_arg = 0,
+      .usage = "Enable parallel debugger to attach to a running job",
+    },
+    { .name = "debug-emulate", .has_arg = 0, .flags = OPTPARSE_OPT_HIDDEN,
+      .usage = "Set MPIR_being_debugged for testing",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct attach_ctx {
+    flux_t *h;
+    int exit_code;
+    flux_jobid_t id;
+    bool readonly;
+    bool unbuffered;
+    char *stdin_ranks;
+    const char *jobid;
+    const char *wait_event;
+    flux_future_t *eventlog_f;
+    flux_future_t *exec_eventlog_f;
+    flux_future_t *output_f;
+    flux_watcher_t *sigint_w;
+    flux_watcher_t *sigtstp_w;
+    flux_watcher_t *notify_timer;
+    struct flux_pty_client *pty_client;
+    int pty_capture;
+    struct timespec t_sigint;
+    flux_watcher_t *stdin_w;
+    zlist_t *stdin_rpcs;
+    bool stdin_data_sent;
+    optparse_t *p;
+    bool output_header_parsed;
+    int leader_rank;
+    char *service;
+    double timestamp_zero;
+    int eventlog_watch_count;
+    bool statusline;
+    char *status_msg;
+    int prolog_running;
+    bool fatal_exception;
+    int last_queue_update;
+    char *queue;
+    bool queue_stopped;
+};
+
+struct attach_event {
+    json_t *entry;
+    double timestamp;
+    const char *name;
+    json_t *context;
+};
+
+static void attach_event_destroy (struct attach_event *event)
+{
+    if (event) {
+        int saved_errno = errno;
+        json_decref (event->entry);
+        event->name = NULL;
+        event->context = NULL;
+        free (event);
+        errno = saved_errno;
+    }
+}
+static struct attach_event *attach_event_decode (const char *entry,
+                                                 flux_error_t *errp)
+{
+    struct attach_event *event = calloc (1, sizeof (*event));
+    if (!event)
+        return NULL;
+
+    if (!(event->entry = eventlog_entry_decode (entry))) {
+        errprintf (errp, "eventlog_entry_decode: %s", strerror (errno));
+        goto err;
+    }
+    if (eventlog_entry_parse (event->entry,
+                              &event->timestamp,
+                              &event->name,
+                              &event->context) < 0) {
+        errprintf (errp, "eventlog_entry_parse: %s", strerror (errno));
+        goto err;
+    }
+    return event;
+err:
+    attach_event_destroy (event);
+    return NULL;
+}
+
+void attach_completed_check (struct attach_ctx *ctx)
+{
+    /* stop all non-eventlog watchers and destroy all lingering
+     * futures so we can exit the reactor */
+    if (!ctx->eventlog_watch_count) {
+        if (ctx->stdin_rpcs) {
+            flux_future_t *f = zlist_pop (ctx->stdin_rpcs);
+            while (f) {
+                flux_future_destroy (f);
+                zlist_remove (ctx->stdin_rpcs, f);
+                f = zlist_pop (ctx->stdin_rpcs);
+            }
+        }
+        flux_watcher_stop (ctx->sigint_w);
+        flux_watcher_stop (ctx->sigtstp_w);
+        flux_watcher_stop (ctx->stdin_w);
+        flux_watcher_stop (ctx->notify_timer);
+    }
+}
+
+/* Print eventlog entry to 'fp'.
+ * Prefix and context may be NULL.
+ */
+void print_eventlog_entry (struct attach_ctx *ctx,
+                           FILE *fp,
+                           const char *prefix,
+                           struct attach_event *event)
+{
+    char *context_s = NULL;
+
+    if (event->context) {
+        if (!(context_s = json_dumps (event->context, JSON_COMPACT)))
+            log_err_exit ("%s: error re-encoding context", __func__);
+    }
+    fprintf (stderr, "%.3fs: %s%s%s%s%s\n",
+             event->timestamp - ctx->timestamp_zero,
+             prefix ? prefix : "",
+             prefix ? "." : "",
+             event->name,
+             context_s ? " " : "",
+             context_s ? context_s : "");
+    free (context_s);
+}
+
+static void handle_output_data (struct attach_ctx *ctx, json_t *context)
+{
+    FILE *fp;
+    const char *stream;
+    const char *rank;
+    char *data;
+    int len;
+    if (!ctx->output_header_parsed)
+        log_msg_exit ("stream data read before header");
+    if (iodecode (context, &stream, &rank, &data, &len, NULL) < 0)
+        log_msg_exit ("malformed event context");
+    /*
+     * If this process is attached to a pty (ctx->pty_client != NULL)
+     *  and output corresponds to rank 0 and the interactive pty is being
+     *  captured, then this data is a duplicate, so do nothing.
+     */
+    if (ctx->pty_client != NULL
+        && streq (rank, "0")
+        && ctx->pty_capture)
+        goto out;
+    if (streq (stream, "stdout"))
+        fp = stdout;
+    else
+        fp = stderr;
+    if (len > 0) {
+        if (optparse_hasopt (ctx->p, "label-io"))
+            fprintf (fp, "%s: ", rank);
+        fwrite (data, len, 1, fp);
+        /*  If attached to a pty, terminal is in raw mode so a carriage
+         *  return will be necessary to return cursor to the start of line.
+         */
+        if (ctx->pty_client)
+            fputc ('\r', fp);
+        fflush (fp);
+    }
+out:
+    free (data);
+}
+
+static void handle_output_redirect (struct attach_ctx *ctx, json_t *context)
+{
+    const char *stream = NULL;
+    const char *rank = NULL;
+    const char *path = NULL;
+    if (!ctx->output_header_parsed)
+        log_msg_exit ("stream redirect read before header");
+    if (json_unpack (context, "{ s:s s:s s?s }",
+                              "stream", &stream,
+                              "rank", &rank,
+                              "path", &path) < 0)
+        log_msg_exit ("malformed redirect context");
+    if (!optparse_hasopt (ctx->p, "quiet"))
+        fprintf (stderr, "%s: %s redirected%s%s\n",
+                         rank,
+                         stream,
+                         path ? " to " : "",
+                         path ? path : "");
+}
+
+/*  Level prefix strings. Nominally, output log event 'level' integers
+ *   are Internet RFC 5424 severity levels. In the context of flux-shell,
+ *   the first 3 levels are equivalently "fatal" errors.
+ */
+static const char *levelstr[] = {
+    "FATAL", "FATAL", "FATAL", "ERROR", " WARN", NULL, "DEBUG", "TRACE"
+};
+
+static void handle_output_log (struct attach_ctx *ctx,
+                               double ts,
+                               json_t *context)
+{
+    const char *msg = NULL;
+    const char *file = NULL;
+    const char *component = NULL;
+    int rank = -1;
+    int line = -1;
+    int level = -1;
+    json_error_t err;
+
+    if (json_unpack_ex (context, &err, 0,
+                        "{ s?i s:i s:s s?s s?s s?i }",
+                        "rank", &rank,
+                        "level", &level,
+                        "message", &msg,
+                        "component", &component,
+                        "file", &file,
+                        "line", &line) < 0) {
+        log_err ("invalid log event in guest.output: %s", err.text);
+        return;
+    }
+    if (!optparse_hasopt (ctx->p, "quiet")) {
+        const char *label = levelstr [level];
+        fprintf (stderr, "%.3fs: flux-shell", ts - ctx->timestamp_zero);
+        if (rank >= 0)
+            fprintf (stderr, "[%d]", rank);
+        if (label)
+            fprintf (stderr, ": %s", label);
+        if (component)
+            fprintf (stderr, ": %s", component);
+        if (optparse_hasopt (ctx->p, "verbose") && file) {
+            fprintf (stderr, ": %s", file);
+            if (line > 0)
+                fprintf (stderr, ":%d", line);
+        }
+        fprintf (stderr, ": %s\n", msg);
+        /*  If attached to a pty, terminal is in raw mode so a carriage
+         *  return will be necessary to return cursor to the start of line.
+         */
+        if (ctx->pty_client)
+            fprintf (stderr, "\r");
+    }
+}
+
+/* Handle an event in the guest.output eventlog.
+ * This is a stream of responses, one response per event, terminated with
+ * an ENODATA error response (or another error if something went wrong).
+ * The first eventlog entry is a header; remaining entries are data,
+ * redirect, or log messages.  Print each data entry to stdout/stderr,
+ * with task/rank prefix if --label-io was specified.  For each redirect entry, print
+ * information on paths to redirected locations if --quiet is not
+ * specified.
+ */
+void attach_output_continuation (flux_future_t *f, void *arg)
+{
+    struct attach_ctx *ctx = arg;
+    const char *entry;
+    json_t *o;
+    const char *name;
+    double ts;
+    json_t *context;
+
+    if (flux_job_event_watch_get (f, &entry) < 0) {
+        if (errno == ENODATA)
+            goto done;
+        if (errno == ENOENT) {
+            log_msg ("No job output found");
+            goto done;
+        }
+        log_msg_exit ("flux_job_event_watch_get: %s",
+                      future_strerror (f, errno));
+    }
+    if (!(o = eventlog_entry_decode (entry)))
+        log_err_exit ("eventlog_entry_decode");
+    if (eventlog_entry_parse (o, &ts, &name, &context) < 0)
+        log_err_exit ("eventlog_entry_parse");
+
+    if (streq (name, "header")) {
+        /* Future: per-stream encoding */
+        ctx->output_header_parsed = true;
+    }
+    else if (streq (name, "data")) {
+        handle_output_data (ctx, context);
+    }
+    else if (streq (name, "redirect")) {
+        handle_output_redirect (ctx, context);
+    }
+    else if (streq (name, "log")) {
+        handle_output_log (ctx, ts, context);
+    }
+
+    json_decref (o);
+    flux_future_reset (f);
+    return;
+done:
+    flux_future_destroy (f);
+    ctx->output_f = NULL;
+    ctx->eventlog_watch_count--;
+    attach_completed_check (ctx);
+}
+
+void attach_cancel_continuation (flux_future_t *f, void *arg)
+{
+    if (flux_future_get (f, NULL) < 0)
+        log_msg ("cancel: %s", future_strerror (f, errno));
+    flux_future_destroy (f);
+}
+
+/* Handle the user typing ctrl-C (SIGINT) and ctrl-Z (SIGTSTP).
+ * If the user types ctrl-C twice within 2s, cancel the job.
+ * If the user types ctrl-C then ctrl-Z within 2s, detach from the job.
+ */
+void attach_signal_cb (flux_reactor_t *r, flux_watcher_t *w,
+                       int revents, void *arg)
+{
+    struct attach_ctx *ctx = arg;
+    flux_future_t *f;
+    int signum = flux_signal_watcher_get_signum (w);
+
+    if (signum == SIGINT) {
+        if (monotime_since (ctx->t_sigint) > 2000) {
+            monotime (&ctx->t_sigint);
+            flux_watcher_start (ctx->sigtstp_w);
+            log_msg ("one more ctrl-C within 2s to cancel or ctrl-Z to detach");
+        }
+        else {
+            if (!(f = flux_job_cancel (ctx->h, ctx->id,
+                                       "interrupted by ctrl-C")))
+                log_err_exit ("flux_job_cancel");
+            if (flux_future_then (f, -1, attach_cancel_continuation, NULL) < 0)
+                log_err_exit ("flux_future_then");
+        }
+    }
+    else if (signum == SIGTSTP) {
+        if (monotime_since (ctx->t_sigint) <= 2000) {
+            if (ctx->eventlog_f) {
+                if (flux_job_event_watch_cancel (ctx->eventlog_f) < 0)
+                    log_err_exit ("flux_job_event_watch_cancel");
+            }
+            if (ctx->exec_eventlog_f) {
+                if (flux_job_event_watch_cancel (ctx->exec_eventlog_f) < 0)
+                    log_err_exit ("flux_job_event_watch_cancel");
+            }
+            if (ctx->output_f) {
+                if (flux_job_event_watch_cancel (ctx->output_f) < 0)
+                    log_err_exit ("flux_job_event_watch_cancel");
+            }
+            log_msg ("detaching...");
+        }
+        else {
+            flux_watcher_stop (ctx->sigtstp_w);
+            log_msg ("one more ctrl-Z to suspend");
+        }
+    }
+}
+
+/* atexit handler
+ * This is a good faith attempt to restore stdin flags to what they were
+ * before we set O_NONBLOCK.
+ */
+void restore_stdin_flags (void)
+{
+    (void)fd_set_flags (STDIN_FILENO, stdin_flags);
+}
+
+static void attach_send_shell_completion (flux_future_t *f, void *arg)
+{
+    struct attach_ctx *ctx = arg;
+
+    /* failing to write stdin to service is (generally speaking) a
+     * fatal error */
+    if (flux_future_get (f, NULL) < 0) {
+        /* stdin may not be accepted for multiple reasons
+         * - job has completed
+         * - user requested stdin via file
+         * - stdin stream already closed due to prior pipe in
+         */
+        if (errno == ENOSYS) {
+            /* Only generate an error if an attempt to send stdin failed.
+             */
+            if (ctx->stdin_data_sent)
+                log_msg_exit ("stdin not accepted by job");
+        }
+        else
+            log_err_exit ("attach_send_shell");
+    }
+    flux_future_destroy (f);
+    zlist_remove (ctx->stdin_rpcs, f);
+}
+
+static int attach_send_shell (struct attach_ctx *ctx,
+                              const char *ranks,
+                              const void *buf,
+                              int len,
+                              bool eof)
+{
+    json_t *context = NULL;
+    char topic[1024];
+    flux_future_t *f = NULL;
+    int saved_errno;
+    int rc = -1;
+
+    snprintf (topic, sizeof (topic), "%s.stdin", ctx->service);
+    if (!(context = ioencode ("stdin", ranks, buf, len, eof)))
+        goto error;
+    if (!(f = flux_rpc_pack (ctx->h, topic, ctx->leader_rank, 0, "O", context)))
+        goto error;
+    if (flux_future_then (f, -1, attach_send_shell_completion, ctx) < 0)
+        goto error;
+    if (zlist_append (ctx->stdin_rpcs, f) < 0)
+        goto error;
+    /* f memory now in hands of attach_send_shell_completion() or ctx->stdin_rpcs */
+    f = NULL;
+    rc = 0;
+ error:
+    saved_errno = errno;
+    json_decref (context);
+    flux_future_destroy (f);
+    errno = saved_errno;
+    return rc;
+}
+
+/* Handle std input from user */
+void attach_stdin_cb (flux_reactor_t *r, flux_watcher_t *w,
+                      int revents, void *arg)
+{
+    struct attach_ctx *ctx = arg;
+    const char *ptr;
+    int len;
+
+    if (!(ptr = fbuf_read_watcher_get_data (w, &len)))
+        log_err_exit ("fbuf_read_line on stdin");
+    if (len > 0) {
+        if (attach_send_shell (ctx, ctx->stdin_ranks, ptr, len, false) < 0)
+            log_err_exit ("attach_send_shell");
+        ctx->stdin_data_sent = true;
+    }
+    else {
+        /* EOF */
+        if (attach_send_shell (ctx, ctx->stdin_ranks, NULL, 0, true) < 0)
+            log_err_exit ("attach_send_shell");
+        flux_watcher_stop (ctx->stdin_w);
+    }
+}
+
+/*  Start the guest.output eventlog watcher
+ */
+void attach_output_start (struct attach_ctx *ctx)
+{
+    if (ctx->output_f)
+        return;
+
+    if (!(ctx->output_f = flux_job_event_watch (ctx->h,
+                                                ctx->id,
+                                                "guest.output",
+                                                0)))
+        log_err_exit ("flux_job_event_watch");
+
+    if (flux_future_then (ctx->output_f, -1.,
+                          attach_output_continuation,
+                          ctx) < 0)
+        log_err_exit ("flux_future_then");
+
+    ctx->eventlog_watch_count++;
+}
+
+static void valid_or_exit_for_debug (struct attach_ctx *ctx)
+{
+    flux_future_t *f = NULL;
+    char *attrs = "[\"state\"]";
+    flux_job_state_t state = FLUX_JOB_STATE_INACTIVE;
+
+    if (!(f = flux_job_list_id (ctx->h, ctx->id, attrs)))
+        log_err_exit ("flux_job_list_id");
+
+    if (flux_rpc_get_unpack (f, "{s:{s:i}}", "job", "state", &state) < 0)
+        log_err_exit ("Invalid job id (%s) for debugging", ctx->jobid);
+
+    flux_future_destroy (f);
+
+    if (state != FLUX_JOB_STATE_NEW
+        && state != FLUX_JOB_STATE_DEPEND
+        && state != FLUX_JOB_STATE_PRIORITY
+        && state != FLUX_JOB_STATE_SCHED
+        && state != FLUX_JOB_STATE_RUN) {
+        log_msg_exit ("cannot debug job that has finished running");
+    }
+
+    return;
+}
+
+static void attach_setup_stdin (struct attach_ctx *ctx)
+{
+    flux_watcher_t *w;
+    int flags = 0;
+
+    if (ctx->readonly)
+        return;
+
+    if (!ctx->unbuffered)
+        flags = FBUF_WATCHER_LINE_BUFFER;
+
+    /* fbuf_read_watcher_create() requires O_NONBLOCK on
+     * stdin */
+
+    if ((stdin_flags = fd_set_nonblocking (STDIN_FILENO)) < 0)
+        log_err_exit ("unable to set stdin nonblocking");
+    if (atexit (restore_stdin_flags) != 0)
+        log_err_exit ("atexit");
+
+    w = fbuf_read_watcher_create (flux_get_reactor (ctx->h),
+                                         STDIN_FILENO,
+                                         1 << 20,
+                                         attach_stdin_cb,
+                                         flags,
+                                         ctx);
+    if (!w) {
+        /* Users have reported rare occurrences of an EINVAL error
+         * from fbuf_read_watcher_create(), the cause of which
+         * is not understood (See issue #5175). In many cases, perhaps all,
+         * stdin is not used by the job, so aborting `flux job attach`
+         * is an unnecessary failure. Therefore, just ignore stdin when
+         * errno is EINVAL here:
+         */
+        if (errno == EINVAL) {
+            log_msg ("Warning: ignoring stdin: failed to create watcher");
+            return;
+        }
+        log_err_exit ("fbuf_read_watcher_create");
+    }
+
+    if (!(ctx->stdin_rpcs = zlist_new ()))
+        log_err_exit ("zlist_new");
+
+    ctx->stdin_w = w;
+
+    /*  Start stdin watcher only if --stdin-ranks=all (the default).
+     *  Otherwise, the watcher will be started in close_stdin_ranks()
+     *  after the idset of targeted ranks is adjusted based on the job
+     *  taskmap.
+     */
+    if (streq (ctx->stdin_ranks, "all"))
+        flux_watcher_start (ctx->stdin_w);
+}
+
+static void pty_client_exit_cb (struct flux_pty_client *c, void *arg)
+{
+    int status = 0;
+    struct attach_ctx *ctx = arg;
+
+    /*  If this client exited before the attach, then it must have been
+     *   due to an RPC error. In that case, perhaps the remote pty has
+     *   gone away, so fallback to attaching to KVS output eventlogs.
+     */
+    if (!flux_pty_client_attached (c)) {
+        attach_setup_stdin (ctx);
+        attach_output_start (ctx);
+        return;
+    }
+
+    if (flux_pty_client_exit_status (c, &status) < 0)
+        log_err ("Unable to get remote pty exit status");
+    flux_pty_client_restore_terminal ();
+
+    /*  Hm, should we force exit here?
+     *  Need to differentiate between pty detach and normal exit.
+     */
+    exit (status == 0 ? 0 : 1);
+}
+
+static void f_logf (void *arg,
+                    const char *file,
+                    int line,
+                    const char *func,
+                    const char *subsys,
+                    int level,
+                    const char *fmt,
+                    va_list ap)
+{
+    char buf [2048];
+    int buflen = sizeof (buf);
+    int n = vsnprintf (buf, buflen, fmt, ap);
+    if (n >= sizeof (buf)) {
+        buf[buflen - 1] = '\0';
+        buf[buflen - 1] = '+';
+    }
+    log_msg ("%s:%d: %s: %s", file, line, func, buf);
+}
+
+static void attach_pty (struct attach_ctx *ctx, const char *pty_service)
+{
+    int n;
+    char topic [128];
+    int flags = FLUX_PTY_CLIENT_NOTIFY_ON_DETACH;
+
+    if (!(ctx->pty_client = flux_pty_client_create ()))
+        log_err_exit ("flux_pty_client_create");
+
+    flux_pty_client_set_flags (ctx->pty_client, flags);
+    flux_pty_client_set_log (ctx->pty_client, f_logf, NULL);
+
+    n = snprintf (topic, sizeof (topic), "%s.%s", ctx->service, pty_service);
+    if (n >= sizeof (topic))
+        log_err_exit ("Failed to build pty service topic at %s.%s",
+                      ctx->service, pty_service);
+
+    /*  Attempt to attach to pty on rank 0 of this job.
+     *  The attempt may fail if this job is not currently running.
+     */
+    if (flux_pty_client_attach (ctx->pty_client,
+                                ctx->h,
+                                ctx->leader_rank,
+                                topic) < 0)
+        log_err_exit ("failed attempting to attach to pty");
+
+    if (flux_pty_client_notify_exit (ctx->pty_client,
+                                     pty_client_exit_cb,
+                                     ctx) < 0)
+        log_err_exit ("flux_pty_client_notify_exit");
+}
+
+void handle_exec_log_msg (struct attach_ctx *ctx, double ts, json_t *context)
+{
+    const char *rank = NULL;
+    const char *stream = NULL;
+    const char *component = NULL;
+    const char *data = NULL;
+    size_t len = 0;
+    json_error_t err;
+
+    if (json_unpack_ex (context, &err, 0,
+                        "{s:s s:s s:s s:s%}",
+                        "rank", &rank,
+                        "component", &component,
+                        "stream", &stream,
+                        "data", &data, &len) < 0) {
+        log_msg ("exec.log event malformed: %s", err.text);
+        return;
+    }
+    if (!optparse_hasopt (ctx->p, "quiet")) {
+        fprintf (stderr,
+                 "%.3fs: %s[%s]: %s: ",
+                 ts - ctx->timestamp_zero,
+                 component,
+                 rank,
+                 stream);
+    }
+    fwrite (data, len, 1, stderr);
+}
+
+static struct idset *all_taskids (const struct taskmap *map)
+{
+    struct idset *ids;
+    if (!(ids = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        return NULL;
+    if (idset_range_set (ids, 0, taskmap_total_ntasks (map) - 1) < 0) {
+        idset_destroy (ids);
+        return NULL;
+    }
+    return ids;
+}
+
+static void adjust_stdin_ranks (struct attach_ctx *ctx,
+                                struct idset *stdin_ranks,
+                                struct idset *all_ranks)
+{
+    struct idset *isect = idset_intersect (all_ranks, stdin_ranks);
+    if (!isect) {
+        log_err ("failed to get intersection of stdin ranks and all taskids");
+        return;
+    }
+    if (!idset_equal (stdin_ranks, isect)) {
+        char *new = idset_encode (isect, IDSET_FLAG_RANGE);
+        if (!new) {
+            log_err ("unable to adjust stdin-ranks to job");
+            goto out;
+        }
+        log_msg ("warning: adjusting --stdin-ranks from %s to %s",
+                 ctx->stdin_ranks,
+                 new);
+        free (ctx->stdin_ranks);
+        ctx->stdin_ranks = new;
+    }
+out:
+    idset_destroy (isect);
+}
+
+static void handle_stdin_ranks (struct attach_ctx *ctx, json_t *context)
+{
+    flux_error_t error;
+    json_t *omap;
+    struct taskmap *map = NULL;
+    struct idset *open = NULL;
+    struct idset *to_close = NULL;
+    struct idset *isect = NULL;
+    char *ranks = NULL;
+
+    if (streq (ctx->stdin_ranks, "all"))
+        return;
+    if (!(omap = json_object_get (context, "taskmap"))
+        || !(map = taskmap_decode_json (omap, &error))
+        || !(to_close = all_taskids (map))) {
+        log_msg ("failed to process taskmap in shell.start event");
+        goto out;
+    }
+    if (!(open = idset_decode (ctx->stdin_ranks))) {
+        log_err ("failed to decode stdin ranks (%s)", ctx->stdin_ranks);
+        goto out;
+    }
+    /* Ensure that stdin_ranks is a subset of all ranks
+     */
+    adjust_stdin_ranks (ctx, open, to_close);
+
+    if (idset_subtract (to_close, open) < 0
+        || !(ranks = idset_encode (to_close, IDSET_FLAG_RANGE))) {
+        log_err ("unable to close stdin on non-targeted ranks");
+        goto out;
+    }
+    if (attach_send_shell (ctx, ranks, NULL, 0, true) < 0)
+        log_err ("failed to close stdin for %s", ranks);
+
+    /*  Start watching stdin now that ctx->stdin_ranks has been
+     *  validated.
+     */
+    flux_watcher_start (ctx->stdin_w);
+out:
+    taskmap_destroy (map);
+    idset_destroy (open);
+    idset_destroy (to_close);
+    idset_destroy (isect);
+    free (ranks);
+}
+
+/* Handle an event in the guest.exec eventlog.
+ * This is a stream of responses, one response per event, terminated with
+ * an ENODATA error response (or another error if something went wrong).
+ * On the shell.init event, start watching the guest.output eventlog.
+ * It is guaranteed to exist when guest.output is emitted.
+ * If --show-exec was specified, print all events on stderr.
+ */
+void attach_exec_event_continuation (flux_future_t *f, void *arg)
+{
+    struct attach_ctx *ctx = arg;
+    const char *entry;
+    struct attach_event *event;
+    const char *service;
+    flux_error_t error;
+
+    if (flux_job_event_watch_get (f, &entry) < 0) {
+        if (errno == ENODATA)
+            goto done;
+        log_msg_exit ("flux_job_event_watch_get: %s",
+                      future_strerror (f, errno));
+    }
+    if (!(event = attach_event_decode (entry, &error)))
+        log_err_exit ("%s", error.text);
+
+    if (streq (event->name, "shell.init")) {
+        const char *pty_service = NULL;
+        if (json_unpack (event->context,
+                         "{s:i s:s s?s s?i}",
+                         "leader-rank",
+                         &ctx->leader_rank,
+                         "service",
+                         &service,
+                         "pty",
+                         &pty_service,
+                         "capture",
+                         &ctx->pty_capture) < 0)
+            log_err_exit ("error decoding shell.init context");
+        if (!(ctx->service = strdup (service)))
+            log_err_exit ("strdup service from shell.init");
+
+        /*  If there is a pty service for this job, try to attach to it.
+         *  The attach is asynchronous, and if it fails, we fall back to
+         *   to kvs stdio handlers in the pty "exit callback".
+         *
+         *  If there is not a pty service, or the pty attach fails, continue
+         *   to process normal stdio. (This may be because the job is
+         *   already complete).
+         */
+        attach_output_start (ctx);
+        if (pty_service) {
+            if (ctx->readonly)
+                log_msg_exit ("Cannot connect to pty in readonly mode");
+            attach_pty (ctx, pty_service);
+        }
+        else
+            attach_setup_stdin (ctx);
+    } else if (streq (event->name, "shell.start")) {
+        if (MPIR_being_debugged) {
+            int stop_tasks_in_exec = 0;
+            if (json_unpack (event->context,
+                             "{s?b}",
+                             "sync",
+                             &stop_tasks_in_exec) < 0)
+                log_err ("error decoding shell.start context");
+            mpir_setup_interface (ctx->h,
+                                  ctx->id,
+                                  optparse_hasopt (ctx->p, "debug-emulate"),
+                                  stop_tasks_in_exec,
+                                  ctx->leader_rank,
+                                  ctx->service);
+        }
+        handle_stdin_ranks (ctx, event->context);
+    }
+    else if (streq (event->name, "log")) {
+        handle_exec_log_msg (ctx, event->timestamp, event->context);
+    }
+
+    /*  If job is complete, and we haven't started watching
+     *   output eventlog, then start now in case shell.init event
+     *   was never emitted (failure in initialization)
+     */
+    if (streq (event->name, "complete") && !ctx->output_f)
+        attach_output_start (ctx);
+
+    if (optparse_hasopt (ctx->p, "show-exec")
+        && !streq (event->name, "log")) {
+        print_eventlog_entry (ctx, stderr, "exec", event);
+    }
+
+    attach_event_destroy (event);
+    flux_future_reset (f);
+    return;
+done:
+    flux_future_destroy (f);
+    ctx->exec_eventlog_f = NULL;
+    ctx->eventlog_watch_count--;
+    attach_completed_check (ctx);
+}
+
+static void queue_status_cb (flux_future_t *f, void *arg)
+{
+    struct attach_ctx *ctx = arg;
+    int start;
+    if (flux_rpc_get_unpack (f, "{s:b}", "start", &start) == 0)
+        ctx->queue_stopped = !start;
+    flux_future_destroy (f);
+}
+
+static void fetch_queue_status (struct attach_ctx *ctx)
+{
+    flux_future_t *f = NULL;
+
+    /*  We don't yet have the queue, do nothing
+     */
+    if (!ctx->queue)
+        return;
+
+    if (streq (ctx->queue, "default"))
+        f = flux_rpc (ctx->h, "job-manager.queue-status", "{}", 0, 0);
+    else
+        f = flux_rpc_pack (ctx->h,
+                           "job-manager.queue-status",
+                            0,
+                            0,
+                            "{s:s?}",
+                            "name", ctx->queue);
+    if (f && flux_future_then (f, -1., queue_status_cb, ctx) < 0)
+        flux_future_destroy (f);
+}
+
+static void job_queue_cb (flux_future_t *f, void *arg)
+{
+    struct attach_ctx *ctx = arg;
+    const char *queue = "default";
+    if (flux_rpc_get_unpack (f, "{s:{s?s}}", "job", "queue", &queue) == 0)
+        ctx->queue = strdup (queue);
+    flux_future_destroy (f);
+}
+
+static void fetch_job_queue (struct attach_ctx *ctx)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_rpc_pack (ctx->h,
+                             "job-list.list-id",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:I s:[s]}",
+                             "id", ctx->id,
+                             "attrs", "queue"))
+        || flux_future_then (f, -1., job_queue_cb, ctx) < 0)
+        flux_future_destroy (f);
+}
+
+static const char *attach_notify_msg (struct attach_ctx *ctx,
+                                      struct attach_event *event)
+{
+    const char *msg;
+    int severity;
+
+    if (!event) {
+        msg = ctx->status_msg;
+    }
+    else if (streq (event->name, "submit")) {
+        msg = "submitted";
+    }
+    else if (streq (event->name, "validate")) {
+        msg = "resolving dependencies";
+    }
+    else if (streq (event->name, "depend")) {
+        msg = "waiting for priority assignment";
+    }
+    else if (streq (event->name, "priority")) {
+        msg = "waiting for resources";
+    }
+    else if (streq (event->name, "alloc")) {
+        msg = "starting";
+    }
+    else if (streq (event->name, "prolog-start")) {
+        msg = "waiting for prolog";
+        ctx->prolog_running++;
+    }
+    else if (streq (event->name, "prolog-finish")) {
+        if (--ctx->prolog_running > 0)
+            msg = "waiting for prolog";
+        else
+            msg = "starting";
+    }
+    else if (streq (event->name, "exception")
+            && json_unpack (event->context, "{s:i}", "severity", &severity)
+            && severity == 0) {
+        msg = "canceling due to exception";
+    }
+    else if (streq (event->name, "start")) {
+        msg = "started";
+    }
+    else if (streq (event->name, "jobspec-update")) {
+        /* Keep existing status msg, but clear current queue name in case
+         * the queue was updated. This will force current queue and its
+         * status to be refreshed.
+         */
+        msg = ctx->status_msg;
+        free (ctx->queue);
+        ctx->queue = NULL;
+        ctx->queue_stopped = false;
+    }
+    else {
+        /* Keep existing status msg for any event not handled above
+         */
+        msg = ctx->status_msg;
+    }
+    return msg;
+}
+
+static void attach_notify (struct attach_ctx *ctx,
+                           struct attach_event *event,
+                           double ts)
+{
+    /* The following must be called for all events even if
+     * the statusline is not active for prolog-start/finish refcounting.
+     */
+    if (!ctx->fatal_exception) {
+        int dt = ts - ctx->timestamp_zero;
+        int width = 80;
+        struct winsize w;
+        char buf[64];
+        const char *msg;
+        char *msgcpy;
+
+        if (!(msg = attach_notify_msg (ctx, event)))
+            return;
+
+        if (strstarts (msg, "waiting for resources")) {
+            /*  Fetch job queue so queue status can be checked
+             */
+            if (!ctx->queue) {
+                fetch_job_queue (ctx);
+            }
+            else if (ctx->last_queue_update <= 0
+                     || (dt - ctx->last_queue_update >= 10)) {
+                ctx->last_queue_update = dt;
+                fetch_queue_status (ctx);
+            }
+            /*  Amend status if queue is stopped:
+             */
+            if (ctx->queue_stopped) {
+                if (snprintf (buf,
+                              sizeof (buf),
+                              "%s (%s queue stopped)",
+                              msg,
+                              ctx->queue) < sizeof (buf))
+                    msg = buf;
+            }
+        }
+
+        if (ctx->statusline) {
+            /* Adjust width of status so timer is right justified:
+             */
+            if (ioctl(0, TIOCGWINSZ, &w) == 0)
+                width = w.ws_col;
+            width -= 10 + strlen (ctx->jobid) + 10;
+
+            fprintf (stderr,
+                     "\rflux-job: %s %-*s %02d:%02d:%02d\r",
+                     ctx->jobid,
+                     width,
+                     msg,
+                     dt/3600,
+                     (dt/60) % 60,
+                     dt % 60);
+        }
+
+        /*  Save current statusline message for future callbacks:
+         */
+        if ((msgcpy = strdup (msg))) {
+            free (ctx->status_msg);
+            ctx->status_msg = msgcpy;
+        }
+    }
+
+    if (event) {
+        if (streq (event->name, "start")
+            || streq (event->name, "clean")) {
+            if (ctx->statusline) {
+                fprintf (stderr, "\n");
+                ctx->statusline = false;
+            }
+            flux_watcher_stop (ctx->notify_timer);
+        }
+    }
+}
+
+void attach_notify_cb (flux_reactor_t *r, flux_watcher_t *w,
+                       int revents, void *arg)
+{
+    struct attach_ctx *ctx = arg;
+    ctx->statusline = true;
+    attach_notify (ctx, NULL, flux_reactor_time ());
+}
+
+
+/* Handle an event in the main job eventlog.
+ * This is a stream of responses, one response per event, terminated with
+ * an ENODATA error response (or another error if something went wrong).
+ * If a fatal exception event occurs, print it on stderr.
+ * If --show-events was specified, print all events on stderr.
+ * If submit event occurs, begin watching guest.exec.eventlog.
+ * If finish event occurs, capture ctx->exit code.
+ */
+void attach_event_continuation (flux_future_t *f, void *arg)
+{
+    struct attach_ctx *ctx = arg;
+    const char *entry;
+    struct attach_event *event = NULL;
+    flux_error_t error;
+    int status;
+
+    if (flux_job_event_watch_get (f, &entry) < 0) {
+        if (errno == ENODATA)
+            goto done;
+        if (errno == ENOENT)
+            log_msg_exit ("Failed to attach to %s: No such job", ctx->jobid);
+        if (errno == EPERM)
+            log_msg_exit ("Failed to attach to %s: that is not your job",
+                          ctx->jobid);
+        log_msg_exit ("flux_job_event_watch_get: %s",
+                      future_strerror (f, errno));
+    }
+    if (!(event = attach_event_decode (entry, &error)))
+        log_err_exit ("%s", error.text);
+
+    if (ctx->timestamp_zero == 0.)
+        ctx->timestamp_zero = event->timestamp;
+
+    if (streq (event->name, "exception")) {
+        const char *type;
+        int severity;
+        const char *note;
+
+        if (json_unpack (event->context, "{s:s s:i s:s}",
+                         "type", &type,
+                         "severity", &severity,
+                         "note", &note) < 0)
+            log_err_exit ("error decoding exception context");
+
+        if (ctx->statusline)
+            fprintf (stderr, "\r\033[K");
+        fprintf (stderr,
+                 "%.3fs: job.exception %s type=%s severity=%d %s\n",
+                 event->timestamp - ctx->timestamp_zero,
+                 ctx->jobid,
+                 type,
+                 severity,
+                 note);
+
+        ctx->fatal_exception = (severity == 0);
+
+        /*  If this job has an interactive pty and the pty is not yet attached,
+         *   destroy the pty to avoid a potential hang attempting to connect
+         *   to job pty that will never exist.
+         */
+        if (severity == 0
+            && ctx->pty_client
+            && !flux_pty_client_attached (ctx->pty_client)) {
+            flux_pty_client_destroy (ctx->pty_client);
+            ctx->pty_client = NULL;
+        }
+    }
+    else if (streq (event->name, "submit")) {
+        if (!(ctx->exec_eventlog_f = flux_job_event_watch (ctx->h,
+                                                           ctx->id,
+                                                           "guest.exec.eventlog",
+                                                           0)))
+            log_err_exit ("flux_job_event_watch");
+        if (flux_future_then (ctx->exec_eventlog_f,
+                              -1,
+                              attach_exec_event_continuation,
+                              ctx) < 0)
+            log_err_exit ("flux_future_then");
+
+        ctx->eventlog_watch_count++;
+    }
+    else {
+        if (streq (event->name, "finish")) {
+            flux_error_t error;
+            if (json_unpack (event->context, "{s:i}", "status", &status) < 0)
+                log_err_exit ("error decoding finish context");
+            ctx->exit_code = flux_job_waitstatus_to_exitcode (status, &error);
+            if (ctx->exit_code != 0)
+                log_msg ("%s", error.text);
+        }
+    }
+
+    if (optparse_hasopt (ctx->p, "show-events")
+        && !streq (event->name, "exception")) {
+        print_eventlog_entry (ctx, stderr, "job", event);
+    }
+
+    attach_notify (ctx, event, event->timestamp);
+
+    if (streq (event->name, ctx->wait_event)) {
+        flux_job_event_watch_cancel (f);
+        goto done;
+    }
+
+    attach_event_destroy (event);
+    flux_future_reset (f);
+    return;
+done:
+    attach_event_destroy (event);
+    flux_future_destroy (f);
+    ctx->eventlog_f = NULL;
+    ctx->eventlog_watch_count--;
+    attach_completed_check (ctx);
+}
+
+static char *get_stdin_ranks (optparse_t *p)
+{
+    const char *value = optparse_get_str (p, "stdin-ranks", "all");
+    if (!streq (value, "all")) {
+        struct idset *ids;
+        if (!(ids = idset_decode (value)))
+            log_err_exit ("Invalid value '%s' for --stdin-ranks", value);
+        idset_destroy (ids);
+    }
+    return strdup (value);
+}
+
+static void initialize_attach_statusline (struct attach_ctx *ctx,
+                                          flux_reactor_t *r)
+{
+    /*  Never show status line if `FLUX_ATTACH_INTERACTIVE` is set
+     */
+    if (getenv ("FLUX_ATTACH_NONINTERACTIVE"))
+        return;
+
+    /*  Only enable the statusline if it is was explicitly requested via
+     *  --show status, or it is reasonably probable that flux-job attach
+     *  is being used interactively -- i.e. all of stdin, stdout, and stderr
+     *  are connected to a tty.
+     */
+    if ((ctx->statusline = optparse_hasopt (ctx->p, "show-status"))
+        || (!optparse_hasopt (ctx->p, "show-events")
+            && isatty (STDIN_FILENO)
+            && isatty (STDOUT_FILENO)
+            && isatty (STDERR_FILENO))) {
+        /*
+         * If flux-job is running interactively, and the job has not
+         * started within 2s, then display a status line notifying the
+         * user of the job's status. The timer repeats every second after
+         * the initial callback to update a clock displayed on the rhs of
+         * the status line.
+         *
+         * The timer is automatically stopped after the 'start' or 'clean'
+         * event.
+         */
+        ctx->notify_timer = flux_timer_watcher_create (r,
+                                                       ctx->statusline ? 0.:2.,
+                                                       1.,
+                                                       attach_notify_cb,
+                                                       ctx);
+        if (!ctx->notify_timer)
+            log_err ("Failed to start notification timer");
+        flux_watcher_start (ctx->notify_timer);
+    }
+}
+
+int cmd_attach (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    flux_reactor_t *r;
+    struct attach_ctx ctx;
+
+    memset (&ctx, 0, sizeof (ctx));
+    ctx.exit_code = 1;
+
+    if (argc - optindex != 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    ctx.jobid = argv[optindex++];
+    ctx.id = parse_jobid (ctx.jobid);
+    ctx.p = p;
+    ctx.readonly = optparse_hasopt (p, "read-only");
+    ctx.unbuffered = optparse_hasopt (p, "unbuffered");
+
+    if (optparse_hasopt (p, "stdin-ranks") && ctx.readonly)
+        log_msg_exit ("Do not use --stdin-ranks with --read-only");
+    ctx.stdin_ranks = get_stdin_ranks (p);
+
+    if (!(ctx.h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    if (!(r = flux_get_reactor (ctx.h)))
+        log_err_exit ("flux_get_reactor");
+
+    /*  Check for the event name that attach should wait for in the
+     *   main job eventlog. The default is the "clean" event.
+     *   If an event other than "clean" is specified, but never appears
+     *   in the eventlog, flux-job attach will still exit after the 'clean'
+     *   event, since the job-info module reponds with ENODATA after the
+     *   final event, which by definition is "clean".
+     */
+    ctx.wait_event = optparse_get_str (p, "wait-event", "clean");
+
+    if (optparse_hasopt (ctx.p, "debug")
+        || optparse_hasopt (ctx.p, "debug-emulate")) {
+        MPIR_being_debugged = 1;
+    }
+    if (MPIR_being_debugged) {
+        int verbose = optparse_getopt (p, "verbose", NULL);
+        valid_or_exit_for_debug (&ctx);
+        totalview_jobid = xasprintf ("%ju", (uintmax_t)ctx.id);
+        if (verbose > 1)
+            log_msg ("totalview_jobid=%s", totalview_jobid);
+    }
+
+    if (!(ctx.eventlog_f = flux_job_event_watch (ctx.h,
+                                                 ctx.id,
+                                                 "eventlog",
+                                                 0)))
+        log_err_exit ("flux_job_event_watch");
+    if (flux_future_then (ctx.eventlog_f,
+                          -1,
+                          attach_event_continuation,
+                          &ctx) < 0)
+        log_err_exit ("flux_future_then");
+
+    ctx.eventlog_watch_count++;
+
+    if (!ctx.readonly) {
+        ctx.sigint_w = flux_signal_watcher_create (r,
+                                                   SIGINT,
+                                                   attach_signal_cb,
+                                                   &ctx);
+        ctx.sigtstp_w = flux_signal_watcher_create (r,
+                                                    SIGTSTP,
+                                                    attach_signal_cb,
+                                                    &ctx);
+        if (!ctx.sigint_w || !ctx.sigtstp_w)
+            log_err_exit ("flux_signal_watcher_create");
+        flux_watcher_start (ctx.sigint_w);
+    }
+
+    initialize_attach_statusline (&ctx, r);
+
+    if (flux_reactor_run (r, 0) < 0)
+        log_err_exit ("flux_reactor_run");
+
+    zlist_destroy (&(ctx.stdin_rpcs));
+    flux_watcher_destroy (ctx.sigint_w);
+    flux_watcher_destroy (ctx.sigtstp_w);
+    flux_watcher_destroy (ctx.stdin_w);
+    flux_watcher_destroy (ctx.notify_timer);
+    flux_pty_client_destroy (ctx.pty_client);
+    flux_close (ctx.h);
+    free (ctx.service);
+    free (totalview_jobid);
+    free (ctx.queue);
+    free (ctx.stdin_ranks);
+    free (ctx.status_msg);
+
+    if (ctx.fatal_exception && ctx.exit_code == 0)
+        ctx.exit_code = 1;
+
+    return ctx.exit_code;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/cancel.c b/src/cmd/job/cancel.c
new file mode 100644
index 000000000000..2d2d220f453c
--- /dev/null
+++ b/src/cmd/job/cancel.c
@@ -0,0 +1,546 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job raise/cancel/kill */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/sigutil.h"
+#include "ccan/str/str.h"
+#include "common.h"
+
+struct optparse_option cancel_opts[] =  {
+    { .name = "message", .key = 'm', .has_arg = 1, .arginfo = "NOTE",
+      .usage = "Set cancel exception note",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct optparse_option cancelall_opts[] =  {
+    { .name = "user", .key = 'u', .has_arg = 1, .arginfo = "USER",
+      .usage = "Set target user or 'all' (instance owner only)",
+    },
+    { .name = "states", .key = 'S', .has_arg = 1, .arginfo = "STATES",
+      .flags = OPTPARSE_OPT_AUTOSPLIT,
+      .usage = "Set target job states (default=ACTIVE)",
+    },
+    { .name = "force", .key = 'f', .has_arg = 0,
+      .usage = "Confirm the command",
+    },
+    { .name = "quiet", .key = 'q', .has_arg = 0,
+      .usage = "Suppress output if no jobs match",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct optparse_option raise_opts[] =  {
+    { .name = "severity", .key = 's', .has_arg = 1, .arginfo = "N",
+      .usage = "Set exception severity [0-7] (default=0)",
+    },
+    { .name = "type", .key = 't', .has_arg = 1, .arginfo = "TYPE",
+      .usage = "Set exception type (default=cancel)",
+    },
+    { .name = "message", .key = 'm', .has_arg = 1, .arginfo = "NOTE",
+      .usage = "Set exception note",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct optparse_option raiseall_opts[] =  {
+    { .name = "severity", .key = 's', .has_arg = 1, .arginfo = "N",
+      .usage = "Set exception severity [0-7] (0 is fatal, default=7)",
+    },
+    { .name = "user", .key = 'u', .has_arg = 1, .arginfo = "USER",
+      .usage = "Set target user or 'all' (instance owner only)",
+    },
+    { .name = "states", .key = 'S', .has_arg = 1, .arginfo = "STATES",
+      .flags = OPTPARSE_OPT_AUTOSPLIT,
+      .usage = "Set target job states (default=ACTIVE)",
+    },
+    { .name = "force", .key = 'f', .has_arg = 0,
+      .usage = "Confirm the command",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct optparse_option kill_opts[] = {
+    { .name = "signal", .key = 's', .has_arg = 1, .arginfo = "SIG",
+      .usage = "Send signal SIG (default SIGTERM)",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct optparse_option killall_opts[] = {
+    { .name = "signal", .key = 's', .has_arg = 1, .arginfo = "SIG",
+      .usage = "Send signal SIG (default SIGTERM)",
+    },
+    { .name = "user", .key = 'u', .has_arg = 1, .arginfo = "USER",
+      .usage = "Set target user or 'all' (instance owner only)",
+    },
+    { .name = "force", .key = 'f', .has_arg = 0,
+      .usage = "Confirm the command",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct jobid_arg {
+    const char *arg;
+    flux_jobid_t id;
+};
+
+static void jobid_arg_free (void **item)
+{
+    if (item) {
+        struct jobid_arg *arg = *item;
+        free (arg);
+        arg = NULL;
+    }
+}
+
+static struct jobid_arg *jobid_arg_create (const char *s)
+{
+    flux_jobid_t id;
+    struct jobid_arg *arg;
+
+    if (flux_job_id_parse (s, &id) < 0
+        || !(arg = calloc (1, sizeof (*arg))))
+        return NULL;
+    arg->id = id;
+    arg->arg = s;
+
+    return arg;
+}
+
+/*  Parse a command line containing a list of jobids and optional
+ *   message (or note). Stops processing at first argument that
+ *   is not a jobid and, if note != NULL, consumes the rest of the
+ *   args and returns as a single string in `note`.
+ */
+static int parse_jobids_and_note (optparse_t *p,
+                                  char **argv,
+                                  zlistx_t **args,
+                                  char **note)
+{
+    if (!(*args = zlistx_new ()))
+        return -1;
+    zlistx_set_destructor (*args, jobid_arg_free);
+
+    /*  Convert each argument to a jobid, stopping at the first failure
+     */
+    while (*argv) {
+        struct jobid_arg *arg;
+        if (!(arg = jobid_arg_create (*argv))) {
+            if (errno == EINVAL)
+                break;
+            log_err_exit ("Error processing argument %s", *argv);
+        }
+        if (!zlistx_add_end (*args, arg))
+            log_msg_exit ("Unable to add jobid %s to list", *argv);
+        argv++;
+    }
+
+    if (*argv != NULL) {
+        /*  If a note was not expected, then this command takes
+         *  only jobids, and a non-jobid argument should raise a
+         *  fatal error.
+         */
+        if (note == NULL)
+            log_msg_exit ("invalid jobid: %s", *argv);
+
+        /*  Forward past `--`, which may have been used to force
+         *  separation of jobid and message on command line
+         */
+        if (streq (*argv, "--"))
+            argv++;
+        *note = parse_arg_message (argv, "message");
+    }
+    if (note) {
+        /*  If a note was expected, also see if --message was used to set
+         *  it on the cmdline. It is an error to specify both --message and
+         *  the note in free arguments.
+         */
+        const char *s;
+        if (optparse_getopt (p, "message", &s) && *note)
+            log_msg_exit ("Do not set note on command line and with --message");
+        else if (s && !(*note = strdup (s)))
+            log_err_exit ("failed to duplicate --message=%s", s);
+    }
+    return 0;
+}
+
+/*  Check the wait-all future in `f` and print any errors, one per line.
+ */
+static int wait_all_check (flux_future_t *f, const char *prefix)
+{
+    int rc;
+    if ((rc = flux_future_get (f, NULL)) < 0) {
+        const char *child = flux_future_first_child (f);
+        while (child) {
+            flux_future_t *cf = flux_future_get_child (f, child);
+            if (flux_future_get (cf, NULL) < 0)
+                log_msg ("%s %s: %s",
+                         prefix,
+                         child,
+                         future_strerror (cf, errno));
+            child = flux_future_next_child (f);
+        }
+    }
+    return rc;
+}
+
+int cmd_raise (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    int severity = optparse_get_int (p, "severity", 0);
+    const char *type = optparse_get_str (p, "type", "cancel");
+    flux_t *h;
+    char *note = NULL;
+    flux_future_t *f;
+    zlistx_t *args;
+    struct jobid_arg *arg;
+    int rc = 0;
+
+    if (argc - optindex < 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+
+    parse_jobids_and_note (p, argv + optindex, &args, &note);
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (!(f = flux_future_wait_all_create ()))
+        log_err_exit ("flux_future_wait_all_create");
+
+    /* Always need to set handle for wait_all future: */
+    flux_future_set_flux (f, h);
+
+    arg = zlistx_first (args);
+    while (arg) {
+        flux_future_t *rf = flux_job_raise (h, arg->id, type, severity, note);
+        if (!rf || flux_future_push (f, arg->arg, rf) < 0)
+            log_err_exit ("flux_job_raise");
+        arg = zlistx_next (args);
+    }
+    rc = wait_all_check (f, "raise");
+
+    zlistx_destroy (&args);
+    flux_future_destroy (f);
+    flux_close (h);
+    free (note);
+    return rc;
+}
+
+static int raiseall (flux_t *h,
+                     int dry_run,
+                     uint32_t userid,
+                     int state_mask,
+                     int severity,
+                     const char *type,
+                     const char *note,
+                     int *errorsp)
+{
+    flux_future_t *f;
+    int count;
+    int errors;
+
+    if (!(f = flux_rpc_pack (h,
+                             "job-manager.raiseall",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:b s:i s:i s:i s:s s:s}",
+                             "dry_run",
+                             dry_run,
+                             "userid",
+                             userid,
+                             "states",
+                             state_mask,
+                             "severity",
+                             severity,
+                             "type",
+                             type,
+                             "note",
+                             note ? note : "")))
+        log_err_exit ("error sending raiseall request");
+    if (flux_rpc_get_unpack (f,
+                             "{s:i s:i}",
+                             "count",
+                             &count,
+                             "errors",
+                             &errors) < 0)
+        log_msg_exit ("raiseall: %s", future_strerror (f, errno));
+    flux_future_destroy (f);
+    if (errorsp)
+        *errorsp = errors;
+    return count;
+}
+
+int cmd_raiseall (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    int severity = optparse_get_int (p, "severity", 7);
+    const char *type;
+    uint32_t userid;
+    int state_mask;
+    flux_t *h;
+    char *note = NULL;
+    int count;
+    int errors;
+    int dry_run = 1;
+
+    if (optindex == argc) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    type = argv[optindex++];
+    if (optindex < argc)
+        note = parse_arg_message (argv + optindex, "message");
+    if (optparse_hasopt (p, "states")) {
+        state_mask = parse_arg_states (p, "states");
+        if ((state_mask & FLUX_JOB_STATE_INACTIVE))
+            log_msg_exit ("Exceptions cannot be raised on inactive jobs");
+    }
+    else
+        state_mask = FLUX_JOB_STATE_ACTIVE;
+    if (optparse_hasopt (p, "user"))
+        userid = parse_arg_userid (p, "user");
+    else
+        userid = getuid ();
+    if (optparse_hasopt (p, "force"))
+        dry_run = 0;
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    count = raiseall (h,
+                      dry_run,
+                      userid,
+                      state_mask,
+                      severity,
+                      type,
+                      note,
+                      &errors);
+    if (count > 0 && dry_run)
+        log_msg ("Command matched %d jobs (-f to confirm)", count);
+    else if (count > 0 && !dry_run)
+        log_msg ("Raised exception on %d jobs (%d errors)", count, errors);
+    else
+        log_msg ("Command matched 0 jobs");
+
+    flux_close (h);
+    free (note);
+    return 0;
+}
+
+int cmd_kill (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    flux_future_t *f;
+    int optindex = optparse_option_index (p);
+    zlistx_t *args;
+    struct jobid_arg *arg;
+    const char *s;
+    int signum;
+    int rc = 0;
+
+    if (argc - optindex < 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+
+    parse_jobids_and_note (p, argv + optindex, &args, NULL);
+
+    s = optparse_get_str (p, "signal", "SIGTERM");
+    if ((signum = sigutil_signum (s)) < 0)
+        log_msg_exit ("kill: Invalid signal %s", s);
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (!(f = flux_future_wait_all_create ()))
+        log_err_exit ("flux_future_wait_all_create");
+    flux_future_set_flux (f, h);
+
+    arg = zlistx_first (args);
+    while (arg) {
+        flux_future_t *rf = flux_job_kill (h, arg->id, signum);
+        if (!rf || flux_future_push (f, arg->arg, rf) < 0)
+            log_err_exit ("flux_job_kill");
+        arg = zlistx_next (args);
+    }
+    rc = wait_all_check (f, "kill");
+
+    flux_future_destroy (f);
+    flux_close (h);
+    return rc;
+}
+
+int cmd_killall (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    flux_future_t *f;
+    int optindex = optparse_option_index (p);
+    const char *s;
+    int signum;
+    uint32_t userid;
+    int count;
+    int dry_run = 1;
+    int errors;
+
+    if (argc - optindex > 0) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    s = optparse_get_str (p, "signal", "SIGTERM");
+    if ((signum = sigutil_signum (s)) < 0)
+        log_msg_exit ("killall: Invalid signal %s", s);
+    if (optparse_hasopt (p, "user"))
+        userid = parse_arg_userid (p, "user");
+    else
+        userid = getuid ();
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    if (optparse_hasopt (p, "force"))
+        dry_run = 0;
+    if (!(f = flux_rpc_pack (h,
+                             "job-manager.killall",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:b s:i s:i}",
+                             "dry_run",
+                             dry_run,
+                             "userid",
+                             userid,
+                             "signum",
+                             signum)))
+        log_err_exit ("error sending killall request");
+    if (flux_rpc_get_unpack (f,
+                             "{s:i s:i}",
+                             "count",
+                             &count,
+                             "errors",
+                             &errors) < 0)
+        log_msg_exit ("killall: %s", future_strerror (f, errno));
+    flux_future_destroy (f);
+    if (count > 0 && dry_run)
+        log_msg ("Command matched %d jobs (-f to confirm)", count);
+    else if (count > 0 && !dry_run)
+        log_msg ("%s %d jobs (%d errors)", strsignal (signum), count, errors);
+    else
+        log_msg ("Command matched 0 jobs");
+    flux_close (h);
+    return 0;
+}
+
+int cmd_cancel (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    flux_t *h;
+    char *note = NULL;
+    zlistx_t *args;
+    struct jobid_arg *arg;
+    flux_future_t *f;
+    int rc = 0;
+
+    if (argc - optindex < 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+
+    parse_jobids_and_note (p, argv + optindex, &args, &note);
+
+    fprintf (stderr,
+             "WARNING: this command is deprecated.  Use flux-cancel(1).\n");
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (!(f = flux_future_wait_all_create ()))
+        log_err_exit ("flux_future_wait_all_create");
+    flux_future_set_flux (f, h);
+
+    arg = zlistx_first (args);
+    while (arg) {
+        flux_future_t *rf = flux_job_cancel (h, arg->id, note);
+        if (!rf || flux_future_push (f, arg->arg, rf) < 0)
+            log_err_exit ("flux_job_cancel");
+        arg = zlistx_next (args);
+    }
+    rc = wait_all_check (f, "cancel");
+
+    zlistx_destroy (&args);
+    flux_future_destroy (f);
+    flux_close (h);
+    free (note);
+    return rc;
+}
+
+int cmd_cancelall (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    uint32_t userid;
+    int state_mask;
+    flux_t *h;
+    char *note = NULL;
+    int dry_run = 1;
+    int count;
+    int errors;
+
+    if (optindex < argc)
+        note = parse_arg_message (argv + optindex, "message");
+    if (optparse_hasopt (p, "states")) {
+        state_mask = parse_arg_states (p, "states");
+        if ((state_mask & FLUX_JOB_STATE_INACTIVE))
+            log_msg_exit ("Inactive jobs cannot be canceled");
+    }
+    else
+        state_mask = FLUX_JOB_STATE_ACTIVE;
+    if (optparse_hasopt (p, "user"))
+        userid = parse_arg_userid (p, "user");
+    else
+        userid = getuid ();
+    if (optparse_hasopt (p, "force"))
+        dry_run = 0;
+    fprintf (stderr,
+             "WARNING: this command is deprecated.  Use flux-cancel(1).\n");
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    count = raiseall (h,
+                      dry_run,
+                      userid,
+                      state_mask,
+                      0,
+                      "cancel",
+                      note,
+                      &errors);
+    if (count > 0 && dry_run)
+        log_msg ("Command matched %d jobs (-f to confirm)", count);
+    else if (count > 0 && !dry_run)
+        log_msg ("Canceled %d jobs (%d errors)", count, errors);
+    else if (!optparse_hasopt (p, "quiet"))
+        log_msg ("Command matched 0 jobs");
+    flux_close (h);
+    free (note);
+    return 0;
+}
+
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/common.c b/src/cmd/job/common.c
new file mode 100644
index 000000000000..6fabea5c2c9b
--- /dev/null
+++ b/src/cmd/job/common.c
@@ -0,0 +1,144 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job common functions */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <assert.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <sys/types.h>
+#include <pwd.h>
+
+#ifdef HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+
+#include <flux/core.h>
+
+#include "src/common/libutil/log.h"
+#include "ccan/str/str.h"
+#include "common.h"
+
+flux_jobid_t parse_jobid (const char *s)
+{
+    flux_jobid_t id;
+    if (flux_job_id_parse (s, &id) < 0)
+        log_msg_exit ("error parsing jobid: \"%s\"", s);
+    return id;
+}
+
+/* Parse a free argument 's', expected to be a 64-bit unsigned.
+ * On error, exit complaining about parsing 'name'.
+ */
+unsigned long long parse_arg_unsigned (const char *s, const char *name)
+{
+    unsigned long long i;
+    char *endptr;
+
+    errno = 0;
+    i = strtoull (s, &endptr, 10);
+    if (errno != 0 || *endptr != '\0')
+        log_msg_exit ("error parsing %s: \"%s\"", name, s);
+    return i;
+}
+
+/* Parse free arguments into a space-delimited message.
+ * On error, exit complaning about parsing 'name'.
+ * Caller must free the resulting string
+ */
+char *parse_arg_message (char **argv, const char *name)
+{
+    char *argz = NULL;
+    size_t argz_len = 0;
+    error_t e;
+
+    if ((e = argz_create (argv, &argz, &argz_len)) != 0)
+        log_errn_exit (e, "error parsing %s", name);
+    argz_stringify (argz, argz_len, ' ');
+    return argz;
+}
+
+/* Parse an OPTPARSE_OPT_AUTOSPLIT list of state names, returning a
+ * mask of states.  Exit with error if unknown state encountered.
+ */
+int parse_arg_states (optparse_t *p, const char *optname)
+{
+    int state_mask = 0;
+    const char *arg;
+
+    assert (optparse_hasopt (p, optname) == true);
+
+    optparse_getopt_iterator_reset (p, optname);
+    while ((arg = optparse_getopt_next (p, optname))) {
+        flux_job_state_t state;
+
+        if (flux_job_strtostate (arg, &state) == 0)
+            state_mask |= state;
+        else if (!strcasecmp (arg, "pending"))
+            state_mask |= FLUX_JOB_STATE_PENDING;
+        else if (!strcasecmp (arg, "running"))
+            state_mask |= FLUX_JOB_STATE_RUNNING;
+        else if (!strcasecmp (arg, "active"))
+            state_mask |= FLUX_JOB_STATE_ACTIVE;
+        else
+            log_msg_exit ("error parsing --%s: %s is unknown", optname, arg);
+    }
+    if (state_mask == 0)
+        log_msg_exit ("no states specified");
+    return state_mask;
+}
+
+/* Parse user argument, which may be a username, a user id, or "all".
+ * Print an error and exit if there is a problem.
+ * Return numeric userid (all -> FLUX_USERID_UNKNOWN).
+ */
+uint32_t parse_arg_userid (optparse_t *p, const char *optname)
+{
+    uint32_t userid;
+    const char *s = optparse_get_str (p, optname, NULL);
+    struct passwd *pw;
+    char *endptr;
+
+    assert (s != NULL);
+    if (streq (s, "all"))
+        return FLUX_USERID_UNKNOWN;
+    if ((pw = getpwnam (s)))
+        return pw->pw_uid;
+    errno = 0;
+    userid = strtoul (s, &endptr, 10);
+    if (errno != 0 || *endptr != '\0' || !isdigit (*s))
+        log_msg_exit ("unknown user %s", s);
+    return userid;
+}
+
+char *trim_string (char *s)
+{
+    /* trailing */
+    int len = strlen (s);
+    while (len > 1 && isspace (s[len - 1])) {
+        s[len - 1] = '\0';
+        len--;
+    }
+    /* leading */
+    char *p = s;
+    while (*p && isspace (*p))
+        p++;
+    return p;
+}
+
+
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/common.h b/src/cmd/job/common.h
new file mode 100644
index 000000000000..7e26ac69fc0c
--- /dev/null
+++ b/src/cmd/job/common.h
@@ -0,0 +1,23 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef FLUX_JOB_COMMON_H
+#define FLUX_JOB_COMMON_H 1
+
+#include <flux/optparse.h>
+
+flux_jobid_t parse_jobid (const char *s);
+unsigned long long parse_arg_unsigned (const char *s, const char *name);
+char *parse_arg_message (char **argv, const char *name);
+int parse_arg_states (optparse_t *p, const char *optname);
+uint32_t parse_arg_userid (optparse_t *p, const char *optname);
+char *trim_string (char *s);
+
+#endif /* !FLUX_JOB_H */
diff --git a/src/cmd/job/eventlog.c b/src/cmd/job/eventlog.c
new file mode 100644
index 000000000000..8e9fa8becd86
--- /dev/null
+++ b/src/cmd/job/eventlog.c
@@ -0,0 +1,460 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job eventlog, wait-event */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/xzmalloc.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libeventlog/formatter.h"
+#include "ccan/str/str.h"
+#include "common.h"
+
+struct optparse_option eventlog_opts[] =  {
+    { .name = "format", .key = 'f', .has_arg = 1, .arginfo = "FORMAT",
+      .usage = "Specify output format: text, json",
+    },
+    { .name = "time-format", .key = 'T', .has_arg = 1, .arginfo = "FORMAT",
+      .usage = "Specify time format: raw, iso, offset",
+    },
+    { .name = "human", .key = 'H', .has_arg = 0,
+      .usage = "Display human-readable output. See also --color, --format, "
+               "and --time-format.",
+    },
+    { .name = "color", .key = 'L', .has_arg = 2, .arginfo = "WHEN",
+      .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
+      .usage = "Colorize output when supported; WHEN can be 'always' "
+               "(default if omitted), 'never', or 'auto' (default)."
+    },
+    { .name = "follow", .key = 'F', .has_arg = 0,
+      .usage = "Follow events until job is inactive.",
+    },
+    { .name = "path", .key = 'p', .has_arg = 1, .arginfo = "PATH",
+      .usage = "Specify alternate eventlog name or path suffix "
+               "(e.g. \"exec\", \"output\", or \"guest.exec.eventlog\")",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct optparse_option wait_event_opts[] =  {
+    { .name = "format", .key = 'f', .has_arg = 1, .arginfo = "FORMAT",
+      .usage = "Specify output format: text, json",
+    },
+    { .name = "time-format", .key = 'T', .has_arg = 1, .arginfo = "FORMAT",
+      .usage = "Specify time format: raw, iso, offset",
+    },
+    { .name = "human", .key = 'H', .has_arg = 0,
+      .usage = "Display human-readable output. See also --color, --format, "
+               "and --time-format.",
+    },
+    { .name = "timeout", .key = 't', .has_arg = 1, .arginfo = "DURATION",
+      .usage = "timeout after DURATION",
+    },
+    { .name = "match-context", .key = 'm', .has_arg = 1, .arginfo = "KEY=VAL",
+      .usage = "match key=val in context",
+    },
+    { .name = "count", .key = 'c', .has_arg = 1, .arginfo = "COUNT",
+      .usage = "required number of matches (default 1)",
+    },
+    { .name = "quiet", .key = 'q', .has_arg = 0,
+      .usage = "Do not output matched event",
+    },
+    { .name = "verbose", .key = 'v', .has_arg = 0,
+      .usage = "Output all events before matched event",
+    },
+    { .name = "color", .key = 'L', .has_arg = 2, .arginfo = "WHEN",
+      .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
+      .usage = "Colorize output when supported; WHEN can be 'always' "
+               "(default if omitted), 'never', or 'auto' (default)."
+    },
+    { .name = "path", .key = 'p', .has_arg = 1, .arginfo = "PATH",
+      .usage = "Specify alternate eventlog name or path suffix "
+               "(e.g. \"exec\", \"output\", or \"guest.exec.eventlog\")",
+    },
+    { .name = "waitcreate", .key = 'W', .has_arg = 0,
+      .usage = "If path does not exist, wait for its creation",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static int wait_event_run (optparse_t *p,
+                           const char *jobid,
+                           const char *wait_event,
+                           const char *path,
+                           const char *match_context,
+                           int count,
+                           double timeout,
+                           bool waitcreate,
+                           bool verbose,
+                           bool quiet);
+
+struct eventlog_ctx {
+    optparse_t *p;
+    const char *jobid;
+    flux_jobid_t id;
+    const char *path;
+    struct eventlog_formatter *evf;
+};
+
+struct path_shortname {
+    const char *name;
+    const char *path;
+};
+
+/*  Set of shorthand names for common job eventlog paths:
+ */
+struct path_shortname eventlog_paths[] = {
+    { "exec",   "guest.exec.eventlog" },
+    { "output", "guest.output"        },
+    { "input",  "guest.input"         },
+    { NULL,     NULL                  },
+};
+
+const char *path_lookup (const char *name)
+{
+    const struct path_shortname *path = eventlog_paths;
+    while (path->name) {
+        if (streq (name, path->name))
+            return path->path;
+        path++;
+    }
+    return name;
+}
+
+static void formatter_parse_options (optparse_t *p,
+                                     struct eventlog_formatter *evf)
+{
+    const char *format = optparse_get_str (p, "format", "text");
+    const char *time_format = optparse_get_str (p, "time-format", "raw");
+    const char *when = optparse_get_str (p, "color", "auto");
+
+    if (optparse_hasopt (p, "human")) {
+        format = "text",
+        time_format = "human";
+        when = "auto";
+    }
+
+    if (eventlog_formatter_set_format (evf, format) < 0)
+        log_msg_exit ("invalid format type '%s'", format);
+    if (eventlog_formatter_set_timestamp_format (evf, time_format) < 0)
+        log_msg_exit ("invalid time-format type '%s'", time_format);
+    if (eventlog_formatter_colors_init (evf, when ? when : "always") < 0)
+        log_msg_exit ("invalid value: --color=%s", when);
+}
+
+static void eventlog_continuation (flux_future_t *f, void *arg)
+{
+    struct eventlog_ctx *ctx = arg;
+    const char *s;
+    json_t *a = NULL;
+    size_t index;
+    json_t *value;
+
+    if (flux_rpc_get_unpack (f, "{s:s}", ctx->path, &s) < 0) {
+        if (errno == ENOENT) {
+            flux_future_destroy (f);
+            if (streq (ctx->path, "eventlog"))
+                log_msg_exit ("job %s not found", ctx->jobid);
+            else
+                log_msg_exit ("eventlog path %s not found", ctx->path);
+        }
+        else
+            log_err_exit ("flux_job_eventlog_lookup_get");
+    }
+
+    if (!(a = eventlog_decode (s)))
+        log_err_exit ("eventlog_decode");
+
+    json_array_foreach (a, index, value) {
+        flux_error_t error;
+        if (eventlog_entry_dumpf (ctx->evf, stdout, &error, value) < 0)
+            log_msg ("failed to print eventlog entry: %s", error.text);
+    }
+
+    fflush (stdout);
+    json_decref (a);
+    flux_future_destroy (f);
+}
+
+int cmd_eventlog (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    int optindex = optparse_option_index (p);
+    flux_future_t *f;
+    const char *topic = "job-info.lookup";
+    struct eventlog_ctx ctx = {0};
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (argc - optindex != 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+
+    ctx.jobid = argv[optindex++];
+
+    if (optparse_hasopt (p, "follow"))
+        return wait_event_run (p,
+                               ctx.jobid,
+                               "clean",
+                               optparse_get_str (p, "path", "eventlog"),
+                               NULL,
+                               1,
+                               -1.0,
+                               true,
+                               true,
+                               false);
+
+    ctx.id = parse_jobid (ctx.jobid);
+    ctx.path = path_lookup (optparse_get_str (p, "path", "eventlog"));
+    ctx.p = p;
+
+    if (!(ctx.evf = eventlog_formatter_create ()))
+        log_err_exit ("eventlog_formatter_create");
+    formatter_parse_options (p, ctx.evf);
+
+    if (!(f = flux_rpc_pack (h, topic, FLUX_NODEID_ANY, 0,
+                             "{s:I s:[s] s:i}",
+                             "id", ctx.id,
+                             "keys", ctx.path,
+                             "flags", 0)))
+        log_err_exit ("flux_rpc_pack");
+    if (flux_future_then (f, -1., eventlog_continuation, &ctx) < 0)
+        log_err_exit ("flux_future_then");
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        log_err_exit ("flux_reactor_run");
+
+    flux_close (h);
+    eventlog_formatter_destroy (ctx.evf);
+    return (0);
+}
+
+struct wait_event_ctx {
+    optparse_t *p;
+    const char *wait_event;
+    const char *jobid;
+    flux_jobid_t id;
+    const char *path;
+    bool got_event;
+    struct eventlog_formatter *evf;
+    char *context_key;
+    char *context_value;
+    int count;
+    int match_count;
+    bool verbose;
+    bool quiet;
+};
+
+static bool wait_event_test_context (struct wait_event_ctx *ctx,
+                                     json_t *context)
+{
+    const char *key;
+    json_t *value;
+
+    json_object_foreach (context, key, value) {
+        if (streq (key, ctx->context_key)) {
+            bool match;
+            char *str = json_dumps (value, JSON_ENCODE_ANY|JSON_COMPACT);
+            match = streq (str, ctx->context_value);
+            free (str);
+
+            /* Also try json_string_value() when value is a string:
+             */
+            if (!match
+                && json_is_string (value)
+                && streq (json_string_value (value), ctx->context_value))
+                match = true;
+
+            /*  Return immediately if a match was found:
+             */
+            if (match)
+                return true;
+        }
+    }
+    return false;
+}
+
+static bool wait_event_test (struct wait_event_ctx *ctx, json_t *event)
+{
+    double timestamp;
+    const char *name;
+    json_t *context = NULL;
+    bool match = false;
+
+    if (eventlog_entry_parse (event, &timestamp, &name, &context) < 0)
+        log_err_exit ("eventlog_entry_parse");
+
+    /*  Ensure that timestamp zero is captured in eventlog formatter
+     *  in case the entry is not processed in wait-event.
+     */
+    eventlog_formatter_update_t0 (ctx->evf, timestamp);
+
+    if (streq (name, ctx->wait_event)) {
+        if (ctx->context_key) {
+            if (context)
+                match = wait_event_test_context (ctx, context);
+        }
+        else
+            match = true;
+    }
+
+    if (match && (++ctx->match_count) == ctx->count)
+        return true;
+    return false;
+}
+
+static void wait_event_continuation (flux_future_t *f, void *arg)
+{
+    struct wait_event_ctx *ctx = arg;
+    json_t *o = NULL;
+    const char *event;
+    flux_error_t error;
+
+    if (flux_rpc_get (f, NULL) < 0) {
+        if (errno == ENOENT) {
+            flux_future_destroy (f);
+            if (streq (ctx->path, "eventlog"))
+                log_msg_exit ("job %s not found", ctx->jobid);
+            else
+                log_msg_exit ("eventlog path %s not found", ctx->path);
+        }
+        else if (errno == ETIMEDOUT) {
+            flux_future_destroy (f);
+            log_msg_exit ("wait-event timeout on event '%s'",
+                          ctx->wait_event);
+        } else if (errno == ENODATA) {
+            flux_future_destroy (f);
+            if (!ctx->got_event)
+                log_msg_exit ("event '%s' never received",
+                              ctx->wait_event);
+            return;
+        }
+        /* else fallthrough and have `flux_job_event_watch_get'
+         * handle error */
+    }
+
+    if (flux_job_event_watch_get (f, &event) < 0)
+        log_err_exit ("flux_job_event_watch_get");
+
+    if (!(o = eventlog_entry_decode (event)))
+        log_err_exit ("eventlog_entry_decode");
+
+    if (wait_event_test (ctx, o)) {
+        ctx->got_event = true;
+        if (!ctx->quiet) {
+            if (eventlog_entry_dumpf (ctx->evf, stdout, &error, o) < 0)
+                log_err ("failed to print eventlog entry: %s", error.text);
+            fflush (stdout);
+        }
+        if (flux_job_event_watch_cancel (f) < 0)
+            log_err_exit ("flux_job_event_watch_cancel");
+    } else if (ctx->verbose) {
+        if (!ctx->got_event) {
+            if (eventlog_entry_dumpf (ctx->evf, stdout, &error, o) < 0)
+                log_err ("failed to print eventlog entry: %s", error.text);
+            fflush (stdout);
+        }
+    }
+
+    json_decref (o);
+
+    flux_future_reset (f);
+}
+
+static int wait_event_run (optparse_t *p,
+                           const char *jobid,
+                           const char *wait_event,
+                           const char *path,
+                           const char *match_context,
+                           int count,
+                           double timeout,
+                           bool waitcreate,
+                           bool verbose,
+                           bool quiet)
+{
+    flux_t *h;
+    flux_future_t *f;
+    struct wait_event_ctx ctx = {0};
+    int flags = 0;
+
+    ctx.p = p;
+    ctx.jobid = jobid;
+    ctx.id = parse_jobid (ctx.jobid);
+    ctx.wait_event = wait_event;
+    ctx.path = path_lookup (path);
+    ctx.verbose = verbose;
+    ctx.quiet = quiet;
+    if ((ctx.count = count) <= 0)
+        log_msg_exit ("count must be > 0");
+    if (waitcreate)
+        flags |= FLUX_JOB_EVENT_WATCH_WAITCREATE;
+
+    if (!(ctx.evf = eventlog_formatter_create ()))
+        log_err_exit ("eventlog_formatter_create");
+    formatter_parse_options (p, ctx.evf);
+    if (match_context) {
+        ctx.context_key = xstrdup (match_context);
+        ctx.context_value = strchr (ctx.context_key, '=');
+        if (!ctx.context_value)
+            log_msg_exit ("must specify a context test as key=value");
+        *ctx.context_value++ = '\0';
+    }
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    if (!(f = flux_job_event_watch (h, ctx.id, ctx.path, flags)))
+        log_err_exit ("flux_job_event_watch");
+    if (flux_future_then (f, timeout, wait_event_continuation, &ctx) < 0)
+        log_err_exit ("flux_future_then");
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        log_err_exit ("flux_reactor_run");
+
+    free (ctx.context_key);
+    flux_close (h);
+    eventlog_formatter_destroy (ctx.evf);
+    return (0);
+}
+
+int cmd_wait_event (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    const char *jobid;
+    const char *wait_event;
+
+    if ((argc - optindex) != 2) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    jobid = argv[optindex++];
+    wait_event = argv[optindex++];
+
+    return wait_event_run (p,
+                           jobid,
+                           wait_event,
+                           optparse_get_str (p, "path", "eventlog"),
+                           optparse_get_str (p, "match-context", NULL),
+                           optparse_get_int (p, "count", 1),
+                           optparse_get_duration (p, "timeout", -1.0),
+                           optparse_hasopt (p, "waitcreate"),
+                           optparse_hasopt (p, "verbose"),
+                           optparse_hasopt (p, "quiet"));
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/hostpids.c b/src/cmd/job/hostpids.c
new file mode 100644
index 000000000000..65f5b3f82d5b
--- /dev/null
+++ b/src/cmd/job/hostpids.c
@@ -0,0 +1,219 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job hostpids */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <stdio.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+#include <flux/idset.h>
+
+#ifndef HAVE_STRLCPY
+#include "src/common/libmissing/strlcpy.h"
+#endif
+
+#include "src/common/libutil/log.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libdebugged/debugged.h"
+#include "ccan/str/str.h"
+
+#include "common.h"
+#include "mpir.h"
+
+typedef struct {
+    char *host_name;
+    char *executable_name;
+    int pid;
+} MPIR_PROCDESC;
+
+extern int MPIR_proctable_size;
+extern MPIR_PROCDESC *MPIR_proctable;
+
+struct optparse_option hostpids_opts[] =  {
+    { .name = "delimiter",
+      .key = 'd',
+      .has_arg = 1,
+      .arginfo = "STRING",
+      .usage = "Set output delimiter (default=\",\")",
+    },
+    { .name = "ranks",
+      .key = 'r',
+      .has_arg = 1,
+      .arginfo = "IDSET",
+      .usage = "Include only task ranks in IDSET",
+    },
+    { .name = "timeout",
+      .key = 't',
+      .has_arg = 1,
+      .arginfo = "DURATION",
+      .usage = "timeout after DURATION",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct hostpids_ctx {
+    flux_t *h;
+    flux_jobid_t id;
+    optparse_t *opts;
+    int leader_rank;
+    char shell_service[128];
+};
+
+static void print_hostpids (const char *delim, struct idset *ranks)
+{
+    const char *delimiter = "";
+    for (int i = 0; i < MPIR_proctable_size; i++) {
+        if (ranks && !idset_test (ranks, i))
+            continue;
+        printf ("%s%s:%d",
+                delimiter,
+                MPIR_proctable[i].host_name,
+                MPIR_proctable[i].pid);
+        delimiter = delim;
+    }
+    printf ("\n");
+}
+
+static void mpir_setup (struct hostpids_ctx *ctx)
+{
+    mpir_setup_interface (ctx->h,
+                          ctx->id,
+                          false,
+                          false,
+                          ctx->leader_rank,
+                          ctx->shell_service);
+}
+
+static void event_watch_cb (flux_future_t *f, void *arg)
+{
+    struct hostpids_ctx *ctx = arg;
+    const char *entry;
+    json_t *o = NULL;
+    double timestamp;
+    const char *name;
+    json_t *context;
+
+    if (flux_job_event_watch_get (f, &entry) < 0) {
+        if (errno == ENODATA)
+            goto done;
+        if (errno == ETIMEDOUT)
+            log_msg_exit ("hostpids: timeout waiting for shell.start event");
+        if (errno == EPERM)
+            log_msg_exit ("hostpids: Permission denied");
+        log_msg_exit ("flux_job_event_watch_get: %s",
+                      future_strerror (f, errno));
+    }
+    if (!(o = eventlog_entry_decode (entry)))
+        log_err_exit ("eventlog_entry_decode");
+    if (eventlog_entry_parse (o, &timestamp, &name, &context) < 0)
+        log_err_exit ("eventlog_entry_parse");
+    if (streq (name, "shell.init")) {
+        const char *service;
+        int len = sizeof (ctx->shell_service);
+        if (json_unpack (context,
+                         "{s:i s:s}",
+                         "leader-rank", &ctx->leader_rank,
+                         "service", &service) < 0)
+            log_err_exit ("error decoding shell.init event");
+        if (strlcpy (ctx->shell_service, service, len) >= len)
+            log_msg_exit ("error caching shell service name");
+    }
+    else if (streq (name, "shell.start")) {
+        /* Setup MPIR_procdesc and pop out of reactor to print result */
+        mpir_setup (ctx);
+        goto done;
+    }
+    json_decref (o);
+    flux_future_reset (f);
+    return;
+done:
+    json_decref (o);
+    flux_future_destroy (f);
+}
+
+static void check_valid_jobid (struct hostpids_ctx *ctx, const char *jobid)
+{
+    flux_future_t *f;
+    flux_job_state_t state;
+
+    /*  Arrange for early exit if job is in INACTIVE or CLEANUP
+     */
+    if (!(f = flux_job_list_id (ctx->h, ctx->id, "[\"state\"]")))
+        log_err_exit ("failed to issue job-list.list-id RPC");
+    if (flux_rpc_get_unpack (f, "{s:{s:i}}", "job", "state", &state) < 0) {
+        if (errno == ENOENT)
+            log_msg_exit ("%s: No such job", jobid);
+        log_err_exit ("job list failed for %s", jobid);
+    }
+    flux_future_destroy (f);
+    if (!(state & FLUX_JOB_STATE_ACTIVE))
+        log_msg_exit ("hostpids: job %s is inactive", jobid);
+}
+
+int cmd_hostpids (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    flux_future_t *f;
+    struct hostpids_ctx ctx = { 0 };
+    struct idset *task_ranks = NULL;
+    idset_error_t error;
+    const char *jobid;
+    const char *delim = optparse_get_str (p, "delimiter", ",");
+    const char *ranks = optparse_get_str (p, "ranks", "all");
+    double timeout = optparse_get_duration (p, "timeout", -1.);
+
+    if (streq (delim, "\\n"))
+        delim = "\n";
+
+    if (argc - optindex != 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+
+    if (!streq (ranks, "all")
+        && !(task_ranks = idset_decode_ex (ranks, -1, -1, 0, &error)))
+        log_err_exit ("--ranks=%s: %s", ranks, error.text);
+
+    jobid = argv[optindex];
+    ctx.opts = p;
+    ctx.id = parse_jobid (jobid);
+    if (!(ctx.h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    check_valid_jobid (&ctx, jobid);
+
+    if (!(f = flux_job_event_watch (ctx.h,
+                                    ctx.id,
+                                    "guest.exec.eventlog",
+                                    FLUX_JOB_EVENT_WATCH_WAITCREATE)))
+        log_err_exit ("flux_job_event_watch");
+    if (flux_future_then (f, timeout, event_watch_cb, &ctx) < 0)
+        log_err_exit ("flux_future_then");
+
+    if (flux_reactor_run (flux_get_reactor (ctx.h), 0) < 0)
+        log_err_exit ("flux_reactor_run");
+
+    if (MPIR_proctable_size <= 0)
+        log_msg_exit ("failed to get MPIR_proctable from job shell");
+
+    print_hostpids (delim, task_ranks);
+    idset_destroy (task_ranks);
+    flux_close (ctx.h);
+    return 0;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/id.c b/src/cmd/job/id.c
new file mode 100644
index 000000000000..5862504181a6
--- /dev/null
+++ b/src/cmd/job/id.c
@@ -0,0 +1,82 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job id */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "common.h"
+
+struct optparse_option id_opts[] = {
+    { .name = "to", .key = 't', .has_arg = 1,
+      .arginfo = "dec|kvs|hex|dothex|words|f58",
+      .usage = "Convert jobid to specified form",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static void id_convert (optparse_t *p, const char *src, char *dst, int dstsz)
+{
+    const char *to = optparse_get_str (p, "to", "dec");
+    flux_jobid_t id;
+
+    /*  Parse as any valid JOBID
+     */
+    if (flux_job_id_parse (src, &id) < 0)
+        log_msg_exit ("%s: malformed input", src);
+
+    /*  Now encode into requested representation:
+     */
+    if (flux_job_id_encode (id, to, dst, dstsz) < 0) {
+        if (errno == EPROTO)
+            log_msg_exit ("Unknown to=%s", to);
+        log_msg_exit ("Unable to encode id %s to %s", src, to);
+    }
+}
+
+int cmd_id (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    char dst[256];
+
+    /*  Require at least one id to be processed for success.
+     *  Thus, `flux job id` with no input or args will exit with
+     *   non-zero exit code.
+     */
+    int rc = -1;
+
+    if (optindex == argc) {
+        char src[256];
+        while ((fgets (src, sizeof (src), stdin))) {
+            id_convert (p, trim_string (src), dst, sizeof (dst));
+            printf ("%s\n", dst);
+            rc = 0;
+        }
+    }
+    else {
+        while (optindex < argc) {
+            id_convert (p, argv[optindex++], dst, sizeof (dst));
+            printf ("%s\n", dst);
+            rc = 0;
+        }
+    }
+    return rc;
+}
+
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/info.c b/src/cmd/job/info.c
new file mode 100644
index 000000000000..dd314be60cd2
--- /dev/null
+++ b/src/cmd/job/info.c
@@ -0,0 +1,139 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job info */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/jpath.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "ccan/str/str.h"
+#include "common.h"
+
+struct optparse_option info_opts[] =  {
+    { .name = "original", .key = 'o', .has_arg = 0,
+      .usage = "For key \"jobspec\", return the original submitted jobspec",
+    },
+    { .name = "base", .key = 'b', .has_arg = 0,
+      .usage = "For key \"jobspec\" or \"R\", "
+               "do not apply updates from eventlog",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static void info_usage (void)
+{
+    fprintf (stderr,
+             "Usage: flux job info id key\n"
+             "some useful keys are:\n"
+             "  J                    - signed jobspec\n"
+             "  R                    - allocated resources\n"
+             "  eventlog             - primary job eventlog\n"
+             "  jobspec              - job specification\n"
+             "  guest.exec.eventlog  - execution eventlog\n"
+             "  guest.input          - job input log\n"
+             "  guest.output         - job output log\n"
+             "Use flux job info -h to list available options\n");
+
+}
+
+int cmd_info (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    flux_error_t error;
+    int optindex = optparse_option_index (p);
+    flux_jobid_t id;
+    const char *id_str;
+    const char *key;
+    flux_future_t *f;
+    const char *val;
+    char *new_val = NULL;
+
+    // Usage: flux job info id key
+    if (argc - optindex != 2) {
+        info_usage ();
+        exit (1);
+    }
+    id_str = argv[optindex++];
+    id = parse_jobid (id_str);
+    key = argv[optindex++];
+
+    if (!(h = flux_open_ex (NULL, 0, &error)))
+        log_msg_exit ("flux_open: %s", error.text);
+
+    /* The --original (pre-frobnication) jobspec is obtained by fetching J.
+     * J is the original jobspec, signed, so we must unwrap it to get to the
+     * delicious jobspec inside.
+     */
+    if (optparse_hasopt (p, "original") && streq (key, "jobspec")) {
+        flux_error_t error;
+        if (!(f = flux_rpc_pack (h,
+                                 "job-info.lookup",
+                                 FLUX_NODEID_ANY,
+                                 0,
+                                 "{s:I s:[s] s:i}",
+                                 "id", id,
+                                 "keys", "J",
+                                 "flags", 0))
+            || flux_rpc_get_unpack (f, "{s:s}", "J", &val) < 0)
+            log_msg_exit ("%s", future_strerror (f, errno));
+        if (!(new_val = flux_unwrap_string (val, false, NULL, &error)))
+            log_msg_exit ("Failed to unwrap J to get jobspec: %s", error.text);
+        val = new_val;
+    }
+    /* The current (non --base) R and jobspec are obtained through the
+     * job-info.lookup RPC w/ the CURRENT flag.
+     */
+    else if (!optparse_hasopt (p, "base")
+             && (streq (key, "R") || streq (key, "jobspec"))) {
+        if (!(f = flux_rpc_pack (h,
+                                 "job-info.lookup",
+                                 FLUX_NODEID_ANY,
+                                 0,
+                                 "{s:I s:[s] s:i}",
+                                 "id", id,
+                                 "keys", key,
+                                 "flags", FLUX_JOB_LOOKUP_CURRENT))
+            || flux_rpc_get_unpack (f, "{s:s}", key, &val) < 0)
+            log_msg_exit ("%s", future_strerror (f, errno));
+    }
+    /* All other keys are obtained this way.
+     */
+    else {
+        if (!(f = flux_rpc_pack (h,
+                                 "job-info.lookup",
+                                 FLUX_NODEID_ANY,
+                                 0,
+                                 "{s:I s:[s] s:i}",
+                                 "id", id,
+                                 "keys", key,
+                                 "flags", 0))
+            || flux_rpc_get_unpack (f, "{s:s}", key, &val) < 0)
+            log_msg_exit ("%s", future_strerror (f, errno));
+    }
+
+    printf ("%s\n", val);
+
+    free (new_val);
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/last.c b/src/cmd/job/last.c
new file mode 100644
index 000000000000..3fd41548ff25
--- /dev/null
+++ b/src/cmd/job/last.c
@@ -0,0 +1,78 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job last */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libjob/idf58.h"
+#include "common.h"
+
+int cmd_last (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    flux_future_t *f;
+    flux_t *h;
+    json_t *jobs;
+    size_t index;
+    json_t *entry;
+    const char *slice = "[:1]";
+    char sbuf[16];
+
+    if (optindex < argc) {
+        slice = argv[optindex++];
+        // if slice doesn't contain '[', assume 'flux job last N' form
+        if (!strchr (slice, '[')) {
+            errno = 0;
+            char *endptr;
+            int n = strtol (slice, &endptr, 10);
+            if (errno != 0 || *endptr != '\0') {
+                optparse_print_usage (p);
+                exit (1);
+            }
+            snprintf (sbuf, sizeof (sbuf), "[:%d]", n);
+            slice = sbuf;
+        }
+    }
+    if (optindex < argc) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    if (!(f = flux_rpc_pack (h,
+                             "job-manager.history.get",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:s}",
+                             "slice", slice))
+        || flux_rpc_get_unpack (f, "{s:o}", "jobs", &jobs) < 0) {
+        log_msg_exit ("%s", future_strerror (f, errno));
+    }
+    if (json_array_size (jobs) == 0)
+        log_msg_exit ("job history is empty");
+    json_array_foreach (jobs, index, entry)
+        printf ("%s\n", idf58 (json_integer_value (entry)));
+    flux_future_destroy (f);
+    flux_close (h);
+    return 0;
+}
+
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/list.c b/src/cmd/job/list.c
new file mode 100644
index 000000000000..d67116b1ddfc
--- /dev/null
+++ b/src/cmd/job/list.c
@@ -0,0 +1,261 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job list, list-inactive, list-ids */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+
+#include "src/common/libutil/log.h"
+
+#include "common.h"
+
+struct optparse_option list_opts[] =  {
+    { .name = "count", .key = 'c', .has_arg = 1, .arginfo = "N",
+      .usage = "Limit output to N jobs",
+    },
+    { .name = "states", .key = 's', .has_arg = 1, .arginfo = "STATES",
+      .flags = OPTPARSE_OPT_AUTOSPLIT,
+      .usage = "List jobs in specific states",
+    },
+    { .name = "user", .key = 'u', .has_arg = 1, .arginfo = "USER",
+      .usage = "Limit output to specific user. " \
+               "Specify \"all\" for all users.",
+    },
+    { .name = "all-user", .key = 'a', .has_arg = 0,
+      .usage = "List my jobs, regardless of state",
+    },
+    { .name = "all", .key = 'A', .has_arg = 0,
+      .usage = "List jobs for all users, regardless of state",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct optparse_option list_inactive_opts[] =  {
+    { .name = "count", .key = 'c', .has_arg = 1, .arginfo = "N",
+      .usage = "Limit output to N jobs",
+    },
+    { .name = "since", .key = 's', .has_arg = 1, .arginfo = "T",
+      .usage = "Limit output to jobs that entered the inactive state since"
+               " timestamp T",
+    },
+    OPTPARSE_TABLE_END
+};
+
+struct optparse_option list_ids_opts[] =  {
+    { .name = "wait-state", .key = 'W', .has_arg = 1, .arginfo = "STATE",
+      .usage = "Return only after jobid has reached specified state",
+    },
+    OPTPARSE_TABLE_END
+};
+
+int cmd_list (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    int max_entries = optparse_get_int (p, "count", 0);
+    flux_t *h;
+    flux_future_t *f;
+    json_t *jobs;
+    size_t index;
+    json_t *value;
+    uint32_t userid;
+    int states = 0;
+    json_t *c;
+
+    if (isatty (STDOUT_FILENO)) {
+        fprintf (stderr,
+                 "This is not the command you are looking for. "
+                 "Try flux-jobs(1).\n");
+        exit (1);
+    }
+    if (optindex != argc) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (optparse_hasopt (p, "all-user") || optparse_hasopt (p, "all"))
+        states = FLUX_JOB_STATE_ACTIVE | FLUX_JOB_STATE_INACTIVE;
+    else if (optparse_hasopt (p, "states"))
+        states = parse_arg_states (p, "states");
+    else
+        states = FLUX_JOB_STATE_PENDING | FLUX_JOB_STATE_RUNNING;
+
+    if (optparse_hasopt (p, "all"))
+        userid = FLUX_USERID_UNKNOWN;
+    else if (optparse_hasopt (p, "user"))
+        userid = parse_arg_userid (p, "user");
+    else
+        userid = getuid ();
+
+    if (!(c = json_pack ("{ s:[ {s:[i]}, {s:[i]} ] }",
+                         "and",
+                         "userid", userid,
+                         "states", states)))
+        log_msg_exit ("failed to construct constraint object");
+
+    if (!(f = flux_rpc_pack (h,
+                             "job-list.list",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:i s:[s] s:o}",
+                             "max_entries", max_entries,
+                             "attrs", "all",
+                             "constraint", c)))
+        log_err_exit ("flux_rpc_pack");
+    if (flux_rpc_get_unpack (f, "{s:o}", "jobs", &jobs) < 0)
+        log_msg_exit ("flux job-list.list: %s", future_strerror (f, errno));
+    json_array_foreach (jobs, index, value) {
+        char *str;
+        str = json_dumps (value, 0);
+        if (!str)
+            log_msg_exit ("error parsing list response");
+        printf ("%s\n", str);
+        free (str);
+    }
+    flux_future_destroy (f);
+    flux_close (h);
+
+    return (0);
+}
+
+int cmd_list_inactive (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    int max_entries = optparse_get_int (p, "count", 0);
+    double since = optparse_get_double (p, "since", 0.);
+    flux_t *h;
+    flux_future_t *f;
+    json_t *jobs;
+    size_t index;
+    json_t *value;
+    json_t *c;
+
+    if (isatty (STDOUT_FILENO)) {
+        fprintf (stderr,
+                 "This is not the command you are looking for. "
+                 "Try flux-jobs(1).\n");
+        exit (1);
+    }
+    if (optindex != argc) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (!(c = json_pack ("{s:[i]}", "states", FLUX_JOB_STATE_INACTIVE)))
+        log_msg_exit ("failed to construct constraint object");
+
+    if (!(f = flux_rpc_pack (h,
+                             "job-list.list",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:i s:f s:[s] s:o}",
+                             "max_entries", max_entries,
+                             "since", since,
+                             "attrs", "all",
+                             "constraint", c)))
+        log_err_exit ("flux_rpc_pack");
+    if (flux_rpc_get_unpack (f, "{s:o}", "jobs", &jobs) < 0)
+        log_msg_exit ("flux job-list.list: %s", future_strerror (f, errno));
+    json_array_foreach (jobs, index, value) {
+        char *str;
+        str = json_dumps (value, 0);
+        if (!str)
+            log_msg_exit ("error parsing list response");
+        printf ("%s\n", str);
+        free (str);
+    }
+    flux_future_destroy (f);
+    flux_close (h);
+
+    return (0);
+}
+
+void list_id_continuation (flux_future_t *f, void *arg)
+{
+    json_t *job;
+    char *str;
+    if (flux_rpc_get_unpack (f, "{s:o}", "job", &job) < 0)
+        log_msg_exit ("flux job-list.list-d: %s", future_strerror (f, errno));
+    str = json_dumps (job, 0);
+    if (!str)
+        log_msg_exit ("error parsing list-id response");
+    printf ("%s\n", str);
+    free (str);
+    flux_future_destroy (f);
+}
+
+int cmd_list_ids (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    flux_t *h;
+    int i, ids_len;
+    flux_job_state_t state;
+    const char *state_str;
+
+    if (isatty (STDOUT_FILENO)) {
+        fprintf (stderr,
+                 "This is not the command you are looking for. "
+                 "Try flux-jobs(1).\n");
+        exit (1);
+    }
+    if ((argc - optindex) < 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    /* if no job state specified by user, pick first job state of
+     * depend, which means will return as soon as the job-list module
+     * is aware of the job
+     */
+    state_str = optparse_get_str (p, "wait-state", "depend");
+    if (flux_job_strtostate (state_str, &state) < 0)
+        log_msg_exit ("invalid job state specified");
+
+    ids_len = argc - optindex;
+    for (i = 0; i < ids_len; i++) {
+        flux_jobid_t id = parse_jobid (argv[optindex + i]);
+        flux_future_t *f;
+        if (!(f = flux_rpc_pack (h,
+                                 "job-list.list-id",
+                                 FLUX_NODEID_ANY,
+                                 0,
+                                 "{s:I s:[s] s:i}",
+                                 "id", id,
+                                 "attrs",
+                                   "all",
+                                 "state", state)))
+            log_err_exit ("flux_rpc_pack");
+        if (flux_future_then (f, -1, list_id_continuation, NULL) < 0)
+            log_err_exit ("flux_future_then");
+    }
+
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        log_err_exit ("flux_reactor_run");
+
+    flux_close (h);
+
+    return (0);
+}
+
+/*  vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/main.c b/src/cmd/job/main.c
new file mode 100644
index 000000000000..40960b06253f
--- /dev/null
+++ b/src/cmd/job/main.c
@@ -0,0 +1,356 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* "plumbing" commands (see git(1)) for Flux job management */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <locale.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+
+/* Externally built subcommands */
+extern int cmd_attach (optparse_t *p, int argc, char **argv);
+extern struct optparse_option attach_opts[];
+
+extern int cmd_submit (optparse_t *p, int argc, char **argv);
+extern struct optparse_option submit_opts[];
+
+extern int cmd_list (optparse_t *p, int argc, char **argv);
+extern struct optparse_option list_opts[];
+
+extern int cmd_list_inactive (optparse_t *p, int argc, char **argv);
+extern struct optparse_option list_inactive_opts[];
+
+extern int cmd_list_ids (optparse_t *p, int argc, char **argv);
+extern struct optparse_option list_ids_opts[];
+
+extern int cmd_status (optparse_t *p, int argc, char **argv);
+extern struct optparse_option status_opts[];
+
+extern int cmd_id (optparse_t *p, int argc, char **argv);
+extern struct optparse_option id_opts[];
+
+extern int cmd_namespace (optparse_t *p, int argc, char **argv);
+
+extern int cmd_urgency (optparse_t *p, int argc, char **argv);
+extern struct optparse_option urgency_opts[];
+
+extern int cmd_cancel (optparse_t *p, int argc, char **argv);
+extern struct optparse_option cancel_opts[];
+
+extern int cmd_cancelall (optparse_t *p, int argc, char **argv);
+extern struct optparse_option cancelall_opts[];
+
+extern int cmd_raise (optparse_t *p, int argc, char **argv);
+extern struct optparse_option raise_opts[];
+
+extern int cmd_raiseall (optparse_t *p, int argc, char **argv);
+extern struct optparse_option raiseall_opts[];
+
+extern int cmd_kill (optparse_t *p, int argc, char **argv);
+extern struct optparse_option kill_opts[];
+
+extern int cmd_killall (optparse_t *p, int argc, char **argv);
+extern struct optparse_option killall_opts[];
+
+extern int cmd_eventlog (optparse_t *p, int argc, char **argv);
+extern struct optparse_option eventlog_opts[];
+extern int cmd_wait_event (optparse_t *p, int argc, char **argv);
+extern struct optparse_option wait_event_opts[];
+
+extern int cmd_info (optparse_t *p, int argc, char **argv);
+extern struct optparse_option info_opts[];
+
+extern int cmd_stats (optparse_t *p, int argc, char **argv);
+
+extern int cmd_wait (optparse_t *p, int argc, char **argv);
+extern struct optparse_option wait_opts[];
+
+extern int cmd_memo (optparse_t *p, int argc, char **argv);
+extern struct optparse_option memo_opts[];
+
+extern int cmd_purge (optparse_t *p, int argc, char **argv);
+extern struct optparse_option purge_opts[];
+
+extern int cmd_taskmap (optparse_t *p, int argc, char **argv);
+extern struct optparse_option taskmap_opts[];
+
+extern int cmd_timeleft (optparse_t *p, int argc, char **argv);
+extern struct optparse_option timeleft_opts[];
+
+extern int cmd_last (optparse_t *p, int argc, char **argv);
+
+extern int cmd_hostpids (optparse_t *p, int argc, char **argv);
+extern struct optparse_option hostpids_opts[];
+
+
+static struct optparse_option global_opts[] =  {
+    OPTPARSE_TABLE_END
+};
+
+static struct optparse_subcommand subcommands[] = {
+    { "list",
+      "[OPTIONS]",
+      "List jobs",
+      cmd_list,
+      OPTPARSE_SUBCMD_HIDDEN,
+      list_opts
+    },
+    { "list-inactive",
+      "[OPTIONS]",
+      "List Inactive jobs",
+      cmd_list_inactive,
+      OPTPARSE_SUBCMD_HIDDEN,
+      list_inactive_opts
+    },
+    { "list-ids",
+      "[OPTIONS] ID [ID ...]",
+      "List job(s) by id",
+      cmd_list_ids,
+      OPTPARSE_SUBCMD_HIDDEN,
+      list_ids_opts,
+    },
+    { "urgency",
+      "[OPTIONS] id urgency",
+      "Set job urgency (0-31, HOLD, EXPEDITE, DEFAULT)",
+      cmd_urgency,
+      0,
+      urgency_opts,
+    },
+    { "cancel",
+      "[OPTIONS] ids... [--] [message ...]",
+      "Cancel one or more jobs",
+      cmd_cancel,
+      OPTPARSE_SUBCMD_HIDDEN,
+      cancel_opts,
+    },
+    { "cancelall",
+      "[OPTIONS] [message ...]",
+      "Cancel multiple jobs",
+      cmd_cancelall,
+      OPTPARSE_SUBCMD_HIDDEN,
+      cancelall_opts,
+    },
+    { "raise",
+      "[OPTIONS] ids... [--] [message ...]",
+      "Raise exception on one or more jobs",
+      cmd_raise,
+      0,
+      raise_opts,
+    },
+    { "raiseall",
+      "OPTIONS type [message ...]",
+      "Raise an exception on multiple jobs.",
+      cmd_raiseall,
+      0,
+      raiseall_opts,
+    },
+    { "kill",
+      "[OPTIONS] ids...",
+      "Send signal to one or more running jobs",
+      cmd_kill,
+      0,
+      kill_opts,
+    },
+    { "killall",
+      "[OPTIONS]",
+      "Send signal to multiple running jobs",
+      cmd_killall,
+      0,
+      killall_opts,
+    },
+    { "attach",
+      "[OPTIONS] id",
+      "Interactively attach to job",
+      cmd_attach,
+      0,
+      attach_opts,
+    },
+    { "status",
+      "id [id...]",
+      "Wait for job(s) to complete and exit with largest exit code",
+      cmd_status,
+      0,
+      status_opts,
+    },
+    { "submit",
+      "[OPTIONS] [jobspec]",
+      "Run job",
+      cmd_submit,
+      0,
+      submit_opts
+    },
+    { "id",
+      "[OPTIONS] [id ...]",
+      "Convert jobid(s) to another form",
+      cmd_id,
+      0,
+      id_opts
+    },
+    { "eventlog",
+      "[OPTIONS] id",
+      "Display eventlog for a job",
+      cmd_eventlog,
+      0,
+      eventlog_opts
+    },
+    { "wait-event",
+      "[OPTIONS] id event",
+      "Wait for an event ",
+      cmd_wait_event,
+      0,
+      wait_event_opts
+    },
+    { "info",
+      "id key",
+      "Display info for a job",
+      cmd_info,
+      0,
+      info_opts
+    },
+    { "stats",
+      NULL,
+      "Get current job stats",
+      cmd_stats,
+      0,
+      NULL
+    },
+    { "namespace",
+      "[id ...]",
+      "Convert job ids to job guest kvs namespace names",
+      cmd_namespace,
+      0,
+      NULL
+    },
+    { "wait",
+      "[--all] [id]",
+      "Wait for job(s) to complete.",
+      cmd_wait,
+      0,
+      wait_opts,
+    },
+    { "memo",
+      "[--volatile] id key=value [key=value, ...]",
+      "Post an RFC 21 memo to a job",
+      cmd_memo,
+      0,
+      memo_opts,
+    },
+    { "taskmap",
+      "[OPTION] JOBID|TASKMAP",
+      "Utility function for working with job task maps",
+      cmd_taskmap,
+      0,
+      taskmap_opts,
+    },
+    { "timeleft",
+      "[JOBID]",
+      "Find remaining runtime for job or enclosing instance",
+      cmd_timeleft,
+      0,
+      timeleft_opts,
+    },
+    { "hostpids",
+      "[OPTIONS] JOBID",
+      "Print host:pid pairs for tasks in JOBID",
+      cmd_hostpids,
+      0,
+      hostpids_opts,
+    },
+    { "last",
+      "SLICE",
+      "List my most recently submitted job id(s)",
+      cmd_last,
+      0,
+      NULL,
+    },
+    { "purge",
+      "[--age-limit=FSD] [--num-limit=N] [--batch=COUNT] [--force] [ID ...]",
+      "Purge the oldest inactive jobs",
+      cmd_purge,
+      0,
+      purge_opts,
+    },
+    OPTPARSE_SUBCMD_END
+};
+
+int usage (optparse_t *p, struct optparse_option *o, const char *optarg)
+{
+    struct optparse_subcommand *s;
+    optparse_print_usage (p);
+    fprintf (stderr, "\n");
+    fprintf (stderr, "Common commands from flux-job:\n");
+    s = subcommands;
+    while (s->name) {
+        if (!(s->flags & OPTPARSE_SUBCMD_HIDDEN))
+            fprintf (stderr, "   %-15s %s\n", s->name, s->doc);
+        s++;
+    }
+    exit (1);
+}
+
+int main (int argc, char *argv[])
+{
+    char *cmdusage = "[OPTIONS] COMMAND ARGS";
+    optparse_t *p;
+    int optindex;
+    int exitval;
+
+    /*  Initialize locale from environment. Allows unicode character
+     *   prefix in F58 encoded JOBIDs in wide-character capable locales.
+     */
+    setlocale (LC_ALL, "");
+
+    log_init ("flux-job");
+
+    p = optparse_create ("flux-job");
+
+    if (optparse_add_option_table (p, global_opts) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_add_option_table() failed");
+
+    /* Override help option for our own */
+    if (optparse_set (p, OPTPARSE_USAGE, cmdusage) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_set (USAGE)");
+
+    /* Override --help callback in favor of our own above */
+    if (optparse_set (p, OPTPARSE_OPTION_CB, "help", usage) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_set() failed");
+
+    /* Don't print internal subcommands, we do it ourselves */
+    if (optparse_set (p, OPTPARSE_PRINT_SUBCMDS, 0) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_set (PRINT_SUBCMDS)");
+
+    if (optparse_reg_subcommands (p, subcommands) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_reg_subcommands");
+
+    if ((optindex = optparse_parse_args (p, argc, argv)) < 0)
+        exit (1);
+
+    if ((argc - optindex == 0)
+        || !optparse_get_subcommand (p, argv[optindex])) {
+        usage (p, NULL, NULL);
+        exit (1);
+    }
+
+    if ((exitval = optparse_run_subcommand (p, argc, argv)) < 0)
+        exit (1);
+
+    optparse_destroy (p);
+    log_fini ();
+    return (exitval);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/cmd/job/memo.c b/src/cmd/job/memo.c
new file mode 100644
index 000000000000..3d443cc86e3a
--- /dev/null
+++ b/src/cmd/job/memo.c
@@ -0,0 +1,107 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job memo */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <stdio.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/jpath.h"
+#include "ccan/str/str.h"
+#include "common.h"
+
+struct optparse_option memo_opts[] = {
+    { .name = "volatile", .has_arg = 0,
+      .usage = "Memo will not appear in eventlog (will be lost on restart)",
+    },
+    OPTPARSE_TABLE_END
+};
+
+int cmd_memo (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    flux_t *h;
+    flux_jobid_t id;
+    json_t *memo = NULL;
+    flux_future_t *f;
+    int i;
+
+    if ((argc - optindex) < 2) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    id = parse_jobid (argv[optindex]);
+
+    /*  Build memo object from multiple values */
+    for (i = optindex + 1; i < argc; i++) {
+        void *valbuf = NULL;
+        char *key;
+        char *value;
+        json_t *val;
+
+        if (!(key = strdup (argv[i])))
+            log_msg_exit ("memo: out of memory duplicating key");
+        value = strchr (key, '=');
+        if (!value)
+            log_msg_exit ("memo: no value for key=%s", key);
+        *value++ = '\0';
+
+        if (streq (value, "-")) {
+            ssize_t size;
+            if ((size = read_all (STDIN_FILENO, &valbuf)) < 0)
+                log_err_exit ("read_all");
+            value = valbuf;
+        }
+
+        /* if value is not legal json, assume string */
+        if (!(val = json_loads (value, JSON_DECODE_ANY, NULL))) {
+            if (!(val = json_string (value)))
+                log_msg_exit ("json_string");
+        }
+
+        if (!(memo = jpath_set_new (memo, key, val)))
+            log_err_exit ("failed to set %s=%s in memo object\n", key, value);
+
+        free (valbuf);
+        free (key);
+    }
+
+    if (!(f = flux_rpc_pack (h,
+                             "job-manager.memo",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:I s:b s:o}",
+                             "id", id,
+                             "volatile", optparse_hasopt (p, "volatile"),
+                             "memo", memo)))
+        log_err_exit ("flux_rpc_pack");
+
+    if (flux_rpc_get (f, NULL) < 0)
+        log_msg_exit ("memo: %s", future_strerror (f, errno));
+
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/mpir.c b/src/cmd/job/mpir.c
new file mode 100644
index 000000000000..02f8ae88a534
--- /dev/null
+++ b/src/cmd/job/mpir.c
@@ -0,0 +1,309 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job MPIR support */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <errno.h>
+#include <signal.h>
+#include <libgen.h>
+#ifndef HAVE_GET_CURRENT_DIR_NAME
+#include "src/common/libmissing/get_current_dir_name.h"
+#endif
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/basename.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libdebugged/debugged.h"
+#include "src/shell/mpir/proctable.h"
+#include "ccan/str/str.h"
+
+extern char **environ;
+
+#ifndef VOLATILE
+# if defined(__STDC__) || defined(__cplusplus)
+# define VOLATILE volatile
+# else
+# define VOLATILE
+# endif
+#endif
+
+#define MPIR_NULL                  0
+#define MPIR_DEBUG_SPAWNED         1
+#define MPIR_DEBUG_ABORTING        2
+
+VOLATILE int MPIR_debug_state    = MPIR_NULL;
+struct proctable *proctable      = NULL;
+MPIR_PROCDESC *MPIR_proctable    = NULL;
+int MPIR_proctable_size          = 0;
+char *MPIR_debug_abort_string    = NULL;
+int MPIR_i_am_starter            = 1;
+int MPIR_acquired_pre_main       = 1;
+int MPIR_force_to_main           = 1;
+int MPIR_partial_attach_ok       = 1;
+
+char MPIR_executable_path[256];
+char MPIR_server_arguments[1024];
+
+static void setup_mpir_proctable (const char *s)
+{
+    if (!(proctable = proctable_from_json_string (s))) {
+        errno = EINVAL;
+        log_err_exit ("proctable_from_json_string");
+    }
+    MPIR_proctable = proctable_get_mpir_proctable (proctable,
+                                                   &MPIR_proctable_size);
+    if (!MPIR_proctable) {
+        errno = EINVAL;
+        log_err_exit ("proctable_get_mpir_proctable");
+    }
+}
+
+static void gen_attach_signal (flux_t *h, flux_jobid_t id)
+{
+    flux_future_t *f = NULL;
+    if (!(f = flux_job_kill (h, id, SIGCONT)))
+        log_err_exit ("flux_job_kill");
+    if (flux_rpc_get (f, NULL) < 0)
+        log_msg_exit ("kill %s: %s",
+                      idf58 (id),
+                      future_strerror (f, errno));
+    flux_future_destroy (f);
+}
+
+static int mpir_args_assign (const char *argv0,
+                             const char *args,
+                             int args_len,
+                             int *argc,
+                             const char ***argv)
+{
+    int i = 0;
+    int n = 0;
+
+    *argv = NULL;
+    *argc = 0;
+
+    /*  First get number of entries in args */
+    while (i < args_len) {
+        int len = strlen (args+i);
+        if (len == 0)
+            break;
+        n++;
+        i += len + 1;
+    }
+    n = n + 1;
+
+    if (!(*argv = calloc (n+1, sizeof (char *)))) {
+        log_err ("MPIR: failed to allocate argv for tool launch");
+        return -1;
+    }
+
+    (*argv)[0] = argv0;
+    *argc = n;
+
+    if (n == 1)
+        return 0;
+
+    /*  Now assign remainder of argv */
+    i = 0;
+    n = 1;
+    while (i < args_len) {
+        int len = strlen (args+i);
+        if (len == 0)
+            break;
+        (*argv)[n++] = args+i;
+        i += len + 1;
+    }
+    return 0;
+}
+
+static int mpir_create_argv (const char path[],
+                             const char args[],
+                             int args_len,
+                             int *argc,
+                             const char ***argv)
+{
+    return mpir_args_assign (path, args, args_len, argc, argv);
+}
+
+static void completion_cb (flux_subprocess_t *p)
+{
+    int rank = flux_subprocess_rank (p);
+    int exitcode = flux_subprocess_exit_code (p);
+    int signum = flux_subprocess_signaled (p);
+    const char *prog = basename_simple (MPIR_executable_path);
+
+    if (signum > 0)
+        log_msg ("MPIR: rank %d: %s: %s", rank, prog, strsignal (signum));
+    else if (exitcode != 0)
+        log_msg ("MPIR: rank %d: %s: Exit %d", rank, prog, exitcode);
+    flux_subprocess_destroy (p);
+}
+
+static flux_cmd_t *mpir_make_tool_cmd (const char *path,
+                                       const char *server_args,
+                                       int server_args_len)
+{
+    flux_cmd_t *cmd = NULL;
+    char *dir = NULL;
+    char **argv;
+    int argc;
+
+    if (mpir_create_argv (path,
+                          server_args,
+                          server_args_len,
+                          &argc,
+                          (const char ***) &argv) < 0)
+        return NULL;
+
+    if (argc == 0
+        || !(cmd = flux_cmd_create (argc, argv, environ))) {
+        log_err ("failed to create command from MPIR_executable_path");
+        return NULL;
+    }
+
+    flux_cmd_unsetenv (cmd, "FLUX_PROXY_REMOTE");
+    if (!(dir = get_current_dir_name ())
+        || flux_cmd_setcwd (cmd, dir) < 0)
+        log_err_exit ("failed to get or set current directory");
+    free (argv);
+    free (dir);
+    return cmd;
+}
+
+static void output_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *line;
+    int len = 0;
+    const char *prog = basename_simple (MPIR_executable_path);
+    int rank = flux_subprocess_rank (p);
+
+    len = flux_subprocess_read_trimmed_line (p, stream, &line);
+    if (len == 0)
+        len = flux_subprocess_read (p, stream, &line);
+    if (len > 0)
+        log_msg ("MPIR: rank %d: %s: %s: %s", rank, prog, stream, line);
+}
+
+static void state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
+{
+    if (state == FLUX_SUBPROCESS_FAILED) {
+        const char *prog = basename_simple (MPIR_executable_path);
+        int rank = flux_subprocess_rank (p);
+        const char *errmsg = flux_subprocess_fail_error (p);
+        log_msg ("MPIR: rank %d: %s: %s", rank, prog, errmsg);
+        flux_subprocess_destroy (p);
+    }
+}
+
+static void launch_tool_daemons (flux_t *h,
+                                 const char *exec_service,
+                                 const char *tool_path,
+                                 const char *tool_args,
+                                 int tool_args_len,
+                                 struct idset *ranks)
+{
+    unsigned int rank;
+    flux_cmd_t *cmd;
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_cb,
+        .on_stderr = output_cb,
+        .on_state_change = state_cb,
+    };
+
+    if (!(cmd = mpir_make_tool_cmd (tool_path, tool_args, tool_args_len)))
+        return;
+
+    rank = idset_first (ranks);
+    while (rank != IDSET_INVALID_ID) {
+        flux_subprocess_t *p;
+        if (!(p = flux_rexec_ex (h,
+                                 exec_service,
+                                 rank,
+                                 0,
+                                 cmd,
+                                 &ops,
+                                 NULL,
+                                 NULL)))
+            log_err ("MPIR: failed to launch %s", tool_path);
+        rank = idset_next (ranks, rank);
+    }
+    flux_cmd_destroy (cmd);
+    return;
+}
+
+void mpir_setup_interface (flux_t *h,
+                           flux_jobid_t id,
+                           bool debug_emulate,
+                           bool stop_tasks_in_exec,
+                           int leader_rank,
+                           const char *shell_service)
+{
+    int len;
+    char topic [1024];
+    const char *s = NULL;
+    flux_future_t *f = NULL;
+
+    len = snprintf (topic, sizeof (topic), "%s.proctable", shell_service);
+    if (len >= sizeof (topic))
+        log_msg_exit ("mpir: failed to create shell proctable topic string");
+
+    if (!(f = flux_rpc_pack (h, topic, leader_rank, 0, "{}")))
+        log_err_exit ("flux_rpc_pack");
+    if (flux_rpc_get (f, &s) < 0)
+        log_err_exit ("%s", topic);
+
+    setup_mpir_proctable (s);
+    flux_future_destroy (f);
+
+    if (strlen (MPIR_executable_path) > 0) {
+        struct idset *ranks = proctable_get_ranks (proctable, NULL);
+        len = snprintf (topic, sizeof (topic), "%s.rexec", shell_service);
+        if (len >= sizeof (topic))
+            log_msg_exit ("mpir: failed to create shell rexec topic string");
+        launch_tool_daemons (h,
+                             topic,
+                             MPIR_executable_path,
+                             MPIR_server_arguments,
+                             sizeof (MPIR_server_arguments),
+                             ranks);
+        idset_destroy (ranks);
+    }
+
+    MPIR_debug_state = MPIR_DEBUG_SPAWNED;
+
+    /* Signal the parallel debugger */
+    MPIR_Breakpoint ();
+
+    if (stop_tasks_in_exec || debug_emulate) {
+        /* To support MPIR_partial_attach_ok, we need to send SIGCONT to
+         * those MPI processes to which the debugger didn't attach.
+         * However, all of the debuggers that I know of do ignore
+         * additional SIGCONT being sent to the processes they attached to.
+         * Therefore, we send SIGCONT to *every* MPI process.
+         *
+         * We also send SIGCONT under the debug-emulate flag. This allows us
+         * to write a test for attach mode. The running job will exit
+         * on SIGCONT.
+         */
+        gen_attach_signal (h, id);
+    }
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/mpir.h b/src/cmd/job/mpir.h
new file mode 100644
index 000000000000..72e6a73a08d5
--- /dev/null
+++ b/src/cmd/job/mpir.h
@@ -0,0 +1,21 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef FLUX_JOB_MPIR_H
+#define FLUX_JOB_MPIR_H 1
+
+void mpir_setup_interface (flux_t *h,
+                           flux_jobid_t id,
+                           bool debug_emulate,
+                           bool stop_tasks_in_exec,
+                           int leader_rank,
+                           const char *shell_service);
+
+#endif /* !FLUX_JOB_MPIR_H */
diff --git a/src/cmd/job/namespace.c b/src/cmd/job/namespace.c
new file mode 100644
index 000000000000..bb33974e73e4
--- /dev/null
+++ b/src/cmd/job/namespace.c
@@ -0,0 +1,50 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job namespace */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "common.h"
+
+static void print_job_namespace (const char *src)
+{
+    char ns[64];
+    flux_jobid_t id = parse_jobid (src);
+    if (flux_job_kvs_namespace (ns, sizeof (ns), id) < 0)
+        log_msg_exit ("error getting kvs namespace for %s", src);
+    printf ("%s\n", ns);
+}
+
+int cmd_namespace (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+
+    if (optindex == argc) {
+        char src[256];
+        while ((fgets (src, sizeof (src), stdin)))
+            print_job_namespace (trim_string (src));
+    }
+    else {
+        while (optindex < argc)
+            print_job_namespace (argv[optindex++]);
+    }
+    return 0;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/purge.c b/src/cmd/job/purge.c
new file mode 100644
index 000000000000..d166532342ea
--- /dev/null
+++ b/src/cmd/job/purge.c
@@ -0,0 +1,152 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job purge */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "common.h"
+
+struct optparse_option purge_opts[] = {
+    { .name = "age-limit", .has_arg = 1, .arginfo = "FSD",
+      .usage = "Purge jobs that became inactive beyond age-limit.",
+    },
+    { .name = "num-limit", .has_arg = 1, .arginfo = "COUNT",
+      .usage = "Purge oldest inactive jobs until COUNT are left.",
+    },
+    { .name = "force", .key = 'f', .has_arg = 0,
+      .usage = "Perform the irreversible purge.",
+    },
+    { .name = "batch", .has_arg = 1, .arginfo = "COUNT",
+      .usage = "Limit number of jobs per request (default 50).",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static void purge_finish (flux_t *h, int force, int total)
+{
+    int inactives;
+    flux_future_t *f;
+
+    if (!(f = flux_rpc (h, "job-manager.stats-get", NULL, 0, 0))
+        || flux_rpc_get_unpack (f, "{s:i}", "inactive_jobs", &inactives) < 0)
+        log_err_exit ("purge: failed to fetch inactive job count");
+    flux_future_destroy (f);
+
+    if (force)
+        printf ("purged %d inactive jobs, %d remaining\n", total, inactives);
+    else {
+        printf ("use --force to purge %d of %d inactive jobs\n",
+                total,
+                inactives);
+    }
+}
+
+static int purge_range (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    double age_limit = optparse_get_duration (p, "age-limit", -1.);
+    int num_limit = optparse_get_int (p, "num-limit", -1);
+    int batch = optparse_get_int (p, "batch", 50);
+    int force = 0;
+    int count;
+    int total = 0;
+
+    if (optparse_hasopt (p, "force"))
+        force = 1;
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    do {
+        flux_future_t *f;
+        if (!(f = flux_rpc_pack (h,
+                                 "job-manager.purge",
+                                 0,
+                                 0,
+                                 "{s:f s:i s:i s:b}",
+                                 "age_limit", age_limit,
+                                 "num_limit", num_limit,
+                                 "batch", batch,
+                                 "force", force))
+            || flux_rpc_get_unpack (f, "{s:i}", "count", &count) < 0)
+            log_msg_exit ("purge: %s", future_strerror (f, errno));
+        total += count;
+        flux_future_destroy (f);
+    } while (force && count == batch);
+
+    purge_finish (h, force, total);
+    flux_close (h);
+    return 0;
+}
+
+static void purge_id_continuation (flux_future_t *f, void *arg)
+{
+    int *count = arg;
+    int tmp;
+    if (flux_rpc_get_unpack (f, "{s:i}", "count", &tmp) < 0)
+        log_msg_exit ("purge: %s", future_strerror (f, errno));
+    (*count) += tmp;
+    flux_future_destroy (f);
+}
+
+static int purge_ids (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    flux_t *h;
+    int force = 0;
+    int total = 0;
+    int i, ids_len;
+
+    if (optparse_hasopt (p, "force"))
+        force = 1;
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    ids_len = argc - optindex;
+    for (i = 0; i < ids_len; i++) {
+        flux_jobid_t id = parse_jobid (argv[optindex + i]);
+        flux_future_t *f;
+        if (!(f = flux_rpc_pack (h,
+                                 "job-manager.purge-id",
+                                 0,
+                                 0,
+                                 "{s:I s:i}",
+                                 "id", id,
+                                 "force", force)))
+            log_err_exit ("job-manager.purge-id");
+        if (flux_future_then (f, -1, purge_id_continuation, &total) < 0)
+            log_err_exit ("flux_future_then");
+    }
+
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        log_err_exit ("flux_reactor_run");
+
+    purge_finish (h, force, total);
+    flux_close (h);
+    return 0;
+}
+
+int cmd_purge (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    if ((argc - optindex) > 0)
+        return purge_ids (p, argc, argv);
+    return purge_range (p, argc, argv);
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/stats.c b/src/cmd/job/stats.c
new file mode 100644
index 000000000000..876a31b84d1e
--- /dev/null
+++ b/src/cmd/job/stats.c
@@ -0,0 +1,45 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job stats */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+
+int cmd_stats (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    flux_future_t *f;
+    const char *topic = "job-list.job-stats";
+    const char *s;
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (!(f = flux_rpc (h, topic, NULL, FLUX_NODEID_ANY, 0)))
+        log_err_exit ("flux_rpc");
+    if (flux_rpc_get (f, &s) < 0)
+        log_msg_exit ("stats: %s", future_strerror (f, errno));
+
+    /* for time being, just output json object for result */
+    printf ("%s\n", s);
+    flux_close (h);
+    return (0);
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/status.c b/src/cmd/job/status.c
new file mode 100644
index 000000000000..e084a0e7e1e1
--- /dev/null
+++ b/src/cmd/job/status.c
@@ -0,0 +1,149 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job status */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "common.h"
+
+struct optparse_option status_opts[] = {
+    { .name = "verbose", .key = 'v', .has_arg = 0,
+      .usage = "Increase verbosity"
+    },
+    { .name = "exception-exit-code", .key = 'e', .has_arg = 1,
+      .group = 1,
+      .arginfo = "N",
+      .usage = "Set the default exit code for any jobs that terminate"
+               " solely due to an exception (e.g. canceled jobs or"
+               " jobs rejected by the scheduler) to N [default=1]"
+    },
+    { .name = "json", .key = 'j', .has_arg = 0,
+      .usage = "Dump job result information gleaned from eventlog to stdout",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static void result_cb (flux_future_t *f, void *arg)
+{}
+
+/*  Translate status to an exit code as would be done by UNIX shell:
+ */
+static int status_to_exitcode (int status)
+{
+    if (status < 0)
+        return 0;
+    if (WIFSIGNALED (status))
+        return 128 + WTERMSIG (status);
+    return WEXITSTATUS (status);
+}
+
+int cmd_status (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h = NULL;
+    flux_future_t **futures = NULL;
+    int exit_code;
+    int i;
+    int njobs;
+    int verbose = optparse_getopt (p, "verbose", NULL);
+    int json = optparse_getopt (p, "json", NULL);
+    int optindex = optparse_option_index (p);
+    int exception_exit_code = optparse_get_int (p, "exception-exit-code", 1);
+
+    if ((njobs = (argc - optindex)) < 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (!(futures = calloc (njobs, sizeof (*futures))))
+        log_err_exit ("Failed to initialize futures array");
+
+    for (i = 0; i < njobs; i++) {
+        flux_jobid_t id = parse_jobid (argv[optindex+i]);
+        if (!(futures[i] = flux_job_result (h, id, 0)))
+            log_err_exit ("flux_job_wait_status");
+        if (flux_future_then (futures[i], -1., result_cb, NULL) < 0)
+            log_err_exit ("flux_future_then");
+    }
+
+    if (verbose && njobs > 1)
+        log_msg ("fetching status for %d jobs", njobs);
+
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        log_err ("flux_reactor_run");
+
+    if (verbose && njobs > 1)
+        log_msg ("all done.");
+
+    exit_code = 0;
+    for (i = 0; i < njobs; i++) {
+        const char *jobid = argv[optindex+i];
+        int status = -1;
+        int exitcode;
+        int exception = 0;
+        int exc_severity = 0;
+        const char *exc_type = NULL;
+
+        if (json) {
+            const char *s = NULL;
+            if (flux_job_result_get (futures[i], &s) < 0)
+                log_err_exit ("flux_job_result_get");
+            printf ("%s\n", s);
+        }
+
+        if (flux_job_result_get_unpack (futures[i],
+                                        "{s:b s?s s?i s?i}",
+                                        "exception_occurred", &exception,
+                                        "exception_type", &exc_type,
+                                        "exception_severity", &exc_severity,
+                                        "waitstatus", &status) < 0) {
+            if (errno == ENOENT)
+                log_msg_exit ("%s: No such job", jobid);
+            log_err_exit ("%s: flux_job_wait_status_get", jobid);
+        }
+        if ((exitcode = status_to_exitcode (status)) > exit_code)
+            exit_code = exitcode;
+        if (exception && exc_severity == 0 && exception_exit_code > exit_code)
+            exit_code = exception_exit_code;
+        if (optparse_hasopt (p, "verbose")) {
+            if (WIFSIGNALED (status)) {
+                log_msg ("%s: job shell died by signal %d",
+                         jobid,
+                         WTERMSIG (status));
+            }
+            else if (verbose > 1 || exitcode != 0) {
+                if (exception && exc_severity == 0)
+                    log_msg ("%s: exception type=%s",
+                             jobid,
+                             exc_type);
+                else
+                    log_msg ("%s: exited with exit code %d",
+                             jobid,
+                             exitcode);
+            }
+        }
+        flux_future_destroy (futures[i]);
+    }
+    flux_close (h);
+    free (futures);
+    return exit_code;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/submit.c b/src/cmd/job/submit.c
new file mode 100644
index 000000000000..360d25f876ea
--- /dev/null
+++ b/src/cmd/job/submit.c
@@ -0,0 +1,178 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job submit */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#if HAVE_FLUX_SECURITY
+#include <flux/security/sign.h>
+#endif
+
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/strstrip.h"
+#include "src/common/libjob/idf58.h"
+#include "ccan/str/str.h"
+
+struct optparse_option submit_opts[] =  {
+    { .name = "urgency", .key = 'u', .has_arg = 1, .arginfo = "N",
+      .usage = "Set job urgency (0-31), hold=0, default=16, expedite=31",
+    },
+    { .name = "flags", .key = 'f', .has_arg = 1,
+      .flags = OPTPARSE_OPT_AUTOSPLIT,
+      .usage = "Set submit comma-separated flags (e.g. debug, waitable)",
+    },
+#if HAVE_FLUX_SECURITY
+    { .name = "security-config", .key = 'c', .has_arg = 1, .arginfo = "pattern",
+      .usage = "Use non-default security config glob",
+    },
+    { .name = "sign-type", .key = 's', .has_arg = 1, .arginfo = "TYPE",
+      .usage = "Use non-default mechanism type to sign J",
+    },
+#endif
+    OPTPARSE_TABLE_END
+};
+
+/* Read entire file 'name' ("-" for stdin).  Exit program on error.
+ */
+static size_t read_jobspec (const char *name, void **bufp)
+{
+    int fd;
+    ssize_t size;
+    void *buf;
+
+    if (streq (name, "-"))
+        fd = STDIN_FILENO;
+    else {
+        if ((fd = open (name, O_RDONLY)) < 0)
+            log_err_exit ("%s", name);
+    }
+    if ((size = read_all (fd, &buf)) < 0)
+        log_err_exit ("%s", name);
+    if (fd != STDIN_FILENO)
+        (void)close (fd);
+    *bufp = buf;
+    return size;
+}
+
+
+static void print_jobid (flux_jobid_t id)
+{
+    printf ("%s\n", idf58 (id));
+}
+
+int cmd_submit (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+#if HAVE_FLUX_SECURITY
+    flux_security_t *sec = NULL;
+    const char *sec_config;
+    const char *sign_type = NULL;
+#endif
+    int flags = 0;
+    void *jobspec;
+    int jobspecsz;
+    const char *J = NULL;
+    int urgency;
+    int optindex = optparse_option_index (p);
+    flux_future_t *f;
+    flux_jobid_t id;
+    const char *input = "-";
+
+    if (optindex != argc - 1 && optindex != argc) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (optindex < argc)
+        input = argv[optindex++];
+    if (optparse_hasopt (p, "flags")) {
+        const char *name;
+        while ((name = optparse_getopt_next (p, "flags"))) {
+            if (streq (name, "debug"))
+                flags |= FLUX_JOB_DEBUG;
+            else if (streq (name, "waitable"))
+                flags |= FLUX_JOB_WAITABLE;
+            else if (streq (name, "signed"))
+                flags |= FLUX_JOB_PRE_SIGNED;
+            else if (streq (name, "novalidate"))
+                flags |= FLUX_JOB_NOVALIDATE;
+            else
+                log_msg_exit ("unknown flag: %s", name);
+        }
+    }
+#if HAVE_FLUX_SECURITY
+    /* If any non-default security options are specified, create security
+     * context so jobspec can be pre-signed before submission.
+     */
+    if (optparse_hasopt (p, "security-config")
+        || optparse_hasopt (p, "sign-type")) {
+        if (flags & FLUX_JOB_PRE_SIGNED)
+            log_msg ("Ignoring security config with --flags=pre-signed");
+        else {
+            sec_config = optparse_get_str (p, "security-config", NULL);
+            if (!(sec = flux_security_create (0)))
+                log_err_exit ("security");
+            if (flux_security_configure (sec, sec_config) < 0)
+                log_err_exit ("security config %s",
+                              flux_security_last_error (sec));
+            sign_type = optparse_get_str (p, "sign-type", NULL);
+        }
+    }
+#endif
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    if ((jobspecsz = read_jobspec (input, &jobspec)) == 0)
+        log_msg_exit ("required jobspec is empty");
+
+    /* If jobspec was pre-signed, then assign J to to jobspec after
+     * stripping surrounding whitespace.
+     */
+    if (flags & FLUX_JOB_PRE_SIGNED)
+        J = strstrip (jobspec);
+
+    urgency = optparse_get_int (p, "urgency", FLUX_JOB_URGENCY_DEFAULT);
+
+#if HAVE_FLUX_SECURITY
+    if (sec) {
+        if (!(J = flux_sign_wrap (sec, jobspec, jobspecsz, sign_type, 0)))
+            log_err_exit ("flux_sign_wrap: %s",
+                          flux_security_last_error (sec));
+        flags |= FLUX_JOB_PRE_SIGNED;
+    }
+#endif
+    if (!(f = flux_job_submit (h, J ? J : jobspec, urgency, flags)))
+        log_err_exit ("flux_job_submit");
+    if (flux_job_submit_get_id (f, &id) < 0) {
+        log_msg_exit ("%s", future_strerror (f, errno));
+    }
+    print_jobid (id);
+    flux_future_destroy (f);
+#if HAVE_FLUX_SECURITY
+    flux_security_destroy (sec); // invalidates J
+#endif
+    flux_close (h);
+    free (jobspec);
+    return 0;
+}
+
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/taskmap.c b/src/cmd/job/taskmap.c
new file mode 100644
index 000000000000..782f70ae4eee
--- /dev/null
+++ b/src/cmd/job/taskmap.c
@@ -0,0 +1,262 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job taskmap */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+#include <flux/taskmap.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libtaskmap/taskmap_private.h"
+#include "src/common/librlist/rlist.h"
+#include "ccan/str/str.h"
+#include "common.h"
+
+struct optparse_option taskmap_opts[] = {
+    { .name = "taskids", .has_arg = 1, .arginfo = "NODEID",
+      .usage = "Print idset of tasks on node NODEID",
+    },
+    { .name = "ntasks", .has_arg = 1, .arginfo = "NODEID",
+      .usage = "Print number of tasks on node NODEID",
+    },
+    { .name = "nodeid", .has_arg = 1, .arginfo = "TASKID",
+      .usage = "Print the shell rank/nodeid on which a taskid executed",
+    },
+    { .name = "hostname", .has_arg = 1, .arginfo = "TASKID",
+      .usage = "Print the hostname on which a taskid executed",
+    },
+    { .name = "to", .has_arg = 1, .arginfo="FORMAT",
+      .usage = "Convert an RFC 34 taskmap to another format "
+               "(FORMAT can be raw, pmi, hosts, or multiline)",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static flux_t *handle = NULL;
+
+static void global_flux_close (void)
+{
+    flux_close (handle);
+}
+
+static flux_t *global_flux_open (void)
+{
+    if (!handle) {
+        if (!(handle = flux_open (NULL, 0)))
+            log_err_exit ("flux_open");
+        atexit (global_flux_close);
+    }
+    return handle;
+}
+
+static struct taskmap *get_job_taskmap (flux_jobid_t id)
+{
+    struct taskmap *map;
+    flux_t *h;
+    flux_future_t *f;
+
+    h = global_flux_open ();
+
+    if (!(f = flux_job_event_watch (h, id, "guest.exec.eventlog", 0)))
+        log_err_exit ("flux_job_event_watch");
+    while (true) {
+        json_t *o;
+        json_t *context;
+        const char *event;
+        const char *name;
+        if (flux_job_event_watch_get (f, &event) < 0) {
+            if (errno == ENODATA)
+                log_msg_exit ("No taskmap found for job");
+            if (errno == ENOENT)
+                log_msg_exit ("Unable to get job taskmap: no such job");
+            log_msg_exit ("waiting for shell.start event: %s",
+                          future_strerror (f, errno));
+        }
+        if (!(o = eventlog_entry_decode (event)))
+            log_err_exit ("eventlog_entry_decode");
+        if (eventlog_entry_parse (o, NULL, &name, &context) < 0)
+            log_err_exit ("eventlog_entry_parse");
+        if (streq (name, "shell.start")) {
+            flux_error_t error;
+            json_t *omap;
+            if (!(omap = json_object_get (context, "taskmap"))
+                || !(map = taskmap_decode_json (omap, &error)))
+                log_msg_exit ("failed to get taskmap from shell.start event");
+            json_decref (o);
+            flux_job_event_watch_cancel (f);
+            break;
+        }
+        json_decref (o);
+        flux_future_reset (f);
+    }
+    flux_future_destroy (f);
+    return map;
+}
+
+static struct hostlist *job_hostlist (flux_jobid_t id)
+{
+    flux_t *h;
+    flux_future_t *f;
+    const char *R;
+    struct rlist *rl;
+    struct hostlist *hl;
+
+    h = global_flux_open ();
+
+    if (!(f = flux_rpc_pack (h,
+                             "job-info.lookup",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:I s:[s] s:i}",
+                             "id", id,
+                             "keys", "R",
+                             "flags", 0)))
+        log_err_exit ("flux_rpc_pack");
+    if (flux_rpc_get_unpack (f, "{s:s}", "R", &R) < 0
+        || !(rl = rlist_from_R (R))
+        || !(hl = rlist_nodelist (rl)))
+        log_err_exit ("failed to get hostlist for job");
+    rlist_destroy (rl);
+    flux_future_destroy (f);
+    return hl;
+}
+
+static char *job_nodeid_to_hostname (flux_jobid_t id, int nodeid)
+{
+    char *result;
+    const char *host;
+    struct hostlist *hl = job_hostlist (id);
+    if (!(host = hostlist_nth (hl, nodeid)))
+        log_err_exit ("failed to get hostname for node %d", nodeid);
+    result = strdup (host);
+    hostlist_destroy (hl);
+    return result;
+}
+
+static void output_hosts_to_taskids (struct taskmap *map, flux_jobid_t id)
+{
+    struct hostlist *hl = job_hostlist (id);
+
+    for (int i = 0; i < taskmap_nnodes (map); i++) {
+        char *ids;
+        const struct idset *idset;
+        const char *host;
+
+        if (!(idset = taskmap_taskids (map, i))
+                || !((ids = idset_encode (idset, IDSET_FLAG_RANGE))))
+            log_err_exit ("failed to get taskids for nodeid %d", i);
+        if (!(host = hostlist_nth (hl, i)))
+            log_err_exit ("failed to get hostname for nodeid %d", i);
+
+        printf ("%s: %s\n", host, ids);
+        free (ids);
+    }
+    hostlist_destroy (hl);
+}
+
+int cmd_taskmap (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    struct taskmap *map = NULL;
+    int val;
+    const char *to;
+    char *s;
+    flux_jobid_t id = FLUX_JOBID_ANY;
+
+    if (optindex == argc) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (flux_job_id_parse (argv[optindex], &id) < 0) {
+        flux_error_t error;
+        if (!(map = taskmap_decode (argv[optindex], &error)))
+            log_msg_exit ("error decoding taskmap: %s", error.text);
+    }
+    else
+        map = get_job_taskmap (id);
+
+    if ((val = optparse_get_int (p, "taskids", -1)) != -1) {
+        const struct idset *ids = taskmap_taskids (map, val);
+        if (!ids || !(s = idset_encode (ids, IDSET_FLAG_RANGE)))
+            log_err_exit ("No taskids for node %d", val);
+        printf ("%s\n", s);
+        free (s);
+        taskmap_destroy (map);
+        return 0;
+    }
+    if ((val = optparse_get_int (p, "ntasks", -1)) != -1) {
+        int result = taskmap_ntasks (map, val);
+        if (result < 0)
+            log_err_exit ("failed to get task count for node %d", val);
+        printf ("%d\n", result);
+        taskmap_destroy (map);
+        return 0;
+    }
+    if ((val = optparse_get_int (p, "nodeid", -1)) != -1
+        || (val = optparse_get_int (p, "hostname", -1)) != -1) {
+        int result = taskmap_nodeid (map, val);
+        if (result < 0)
+            log_err_exit ("failed to get nodeid for task %d", val);
+        if (optparse_hasopt (p, "hostname")) {
+            if (id == FLUX_JOBID_ANY)
+                log_msg_exit ("taskmap: can't use --hostname without a jobid");
+            char *host = job_nodeid_to_hostname (id, result);
+            printf ("%s\n", host);
+            free (host);
+        }
+        else
+            printf ("%d\n", result);
+        taskmap_destroy (map);
+        return 0;
+    }
+    if ((to = optparse_get_str (p, "to", NULL))) {
+        if (streq (to, "raw"))
+            s = taskmap_encode (map, TASKMAP_ENCODE_RAW);
+        else if (streq (to, "pmi"))
+            s = taskmap_encode (map, TASKMAP_ENCODE_PMI);
+        else if (streq (to, "multiline")) {
+            for (int i = 0; i < taskmap_total_ntasks (map); i++) {
+                printf ("%d: %d\n", i, taskmap_nodeid (map, i));
+            }
+            taskmap_destroy (map);
+            return 0;
+        }
+        else if (streq (to, "hosts")) {
+            if (id == FLUX_JOBID_ANY)
+                log_msg_exit ("taskmap: can't use --to=hosts without a jobid");
+            output_hosts_to_taskids (map, id);
+            return 0;
+        }
+        else
+            log_msg_exit ("invalid value --to=%s", to);
+        if (s == NULL)
+            log_err_exit ("failed to convert taskmap to %s", to);
+        printf ("%s\n", s);
+        free (s);
+        taskmap_destroy (map);
+        return 0;
+    }
+    if (!(s = taskmap_encode (map, 0)))
+        log_err_exit ("taskmap_encode");
+    printf ("%s\n", s);
+    free (s);
+    taskmap_destroy (map);
+    return 0;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/timeleft.c b/src/cmd/job/timeleft.c
new file mode 100644
index 000000000000..571e420239e8
--- /dev/null
+++ b/src/cmd/job/timeleft.c
@@ -0,0 +1,80 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job timeleft */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <limits.h>
+#include <math.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/fsd.h"
+#include "common.h"
+
+struct optparse_option timeleft_opts[] = {
+    { .name = "human", .key = 'H', .has_arg = 0,
+      .usage = "Output in Flux Standard Duration instead of seconds.",
+    },
+    OPTPARSE_TABLE_END
+};
+
+int cmd_timeleft (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    flux_t *h;
+    flux_error_t error;
+    double t;
+
+    if (optindex < argc - 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (optindex < argc) {
+        if (setenv ("FLUX_JOB_ID", argv[optindex++], 1) < 0)
+            log_err_exit ("setenv");
+    }
+    if (flux_job_timeleft (h, &error, &t) < 0)
+        log_msg_exit ("%s", error.text);
+    if (optparse_hasopt (p, "human")) {
+        char buf[64];
+        if (fsd_format_duration (buf, sizeof (buf), t) < 0)
+            log_err_exit ("fsd_format_duration");
+        printf ("%s\n", buf);
+    }
+    else {
+        unsigned long int sec;
+        /*  Report whole seconds remaining in job, unless value is
+         *  infinity, in which case we report UINT_MAX, or if value
+         *  is 0 < t < 1, in which case round up to 1 to avoid
+         *  printing "0" which would mean the job has expired.
+         */
+        if (isinf (t))
+            sec = UINT_MAX;
+        else if ((sec = floor (t)) == 0 && t > 0.)
+            sec = 1;
+        printf ("%lu\n", sec);
+    }
+    flux_close (h);
+    return 0;
+}
+
+
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/urgency.c b/src/cmd/job/urgency.c
new file mode 100644
index 000000000000..98404fc50978
--- /dev/null
+++ b/src/cmd/job/urgency.c
@@ -0,0 +1,78 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job urgency */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "common.h"
+
+struct optparse_option urgency_opts[] =  {
+    { .name = "verbose", .key = 'v', .has_arg = 0,
+      .usage = "Output old urgency value on success",
+    },
+    OPTPARSE_TABLE_END
+};
+
+int cmd_urgency (optparse_t *p, int argc, char **argv)
+{
+    int optindex = optparse_option_index (p);
+    flux_t *h;
+    flux_future_t *f;
+    int urgency, old_urgency;
+    flux_jobid_t id;
+    const char *jobid = NULL;
+    const char *urgencystr = NULL;
+
+    if (optindex != argc - 2) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    jobid = argv[optindex++];
+    id = parse_jobid (jobid);
+    urgencystr = argv[optindex++];
+    if (!strcasecmp (urgencystr, "hold"))
+        urgency = FLUX_JOB_URGENCY_HOLD;
+    else if (!strcasecmp (urgencystr, "expedite"))
+        urgency = FLUX_JOB_URGENCY_EXPEDITE;
+    else if (!strcasecmp (urgencystr, "default"))
+        urgency = FLUX_JOB_URGENCY_DEFAULT;
+    else
+        urgency = parse_arg_unsigned (urgencystr, "urgency");
+
+    if (!(f = flux_job_set_urgency (h, id, urgency)))
+        log_err_exit ("flux_job_set_urgency");
+    if (flux_rpc_get_unpack (f, "{s:i}", "old_urgency", &old_urgency) < 0)
+        log_msg_exit ("%s: %s", jobid, future_strerror (f, errno));
+    if (optparse_hasopt (p, "verbose")) {
+        if (old_urgency == FLUX_JOB_URGENCY_HOLD)
+            fprintf (stderr, "old urgency: job held\n");
+        else if (old_urgency == FLUX_JOB_URGENCY_EXPEDITE)
+            fprintf (stderr, "old urgency: job expedited\n");
+        else
+            fprintf (stderr, "old urgency: %d\n", old_urgency);
+    }
+    flux_future_destroy (f);
+    flux_close (h);
+    return 0;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/job/wait.c b/src/cmd/job/wait.c
new file mode 100644
index 000000000000..af186db81db6
--- /dev/null
+++ b/src/cmd/job/wait.c
@@ -0,0 +1,116 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* flux-job wait */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+
+#include <flux/core.h>
+#include <flux/optparse.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libjob/idf58.h"
+#include "common.h"
+
+struct optparse_option wait_opts[] =  {
+    { .name = "all", .key = 'a', .has_arg = 0,
+      .usage = "Wait for all (waitable) jobs",
+    },
+    { .name = "verbose", .key = 'v', .has_arg = 0,
+      .usage = "Emit a line of output for all jobs, not just failing ones",
+    },
+    OPTPARSE_TABLE_END
+};
+
+int cmd_wait (optparse_t *p, int argc, char **argv)
+{
+    flux_t *h;
+    int optindex = optparse_option_index (p);
+    flux_future_t *f;
+    flux_jobid_t id = FLUX_JOBID_ANY;
+    bool success;
+    const char *errstr;
+    int rc = 0;
+
+    if ((argc - optindex) > 1) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+    if (optindex < argc) {
+        id = parse_jobid (argv[optindex++]);
+        if (optparse_hasopt (p, "all"))
+            log_err_exit ("jobid not supported with --all");
+    }
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    if (optparse_hasopt (p, "all")) {
+        for (;;) {
+            if (!(f = flux_job_wait (h, FLUX_JOBID_ANY)))
+                log_err_exit ("flux_job_wait");
+            if (flux_job_wait_get_status (f, &success, &errstr) < 0) {
+                if (errno == ECHILD) { // no more waitable jobs
+                    flux_future_destroy (f);
+                    break;
+                }
+                log_msg_exit ("flux_job_wait_get_status: %s",
+                              future_strerror (f, errno));
+            }
+            if (flux_job_wait_get_id (f, &id) < 0)
+                log_msg_exit ("flux_job_wait_get_id: %s",
+                              future_strerror (f, errno));
+            if (!success) {
+                fprintf (stderr, "%s: %s\n", idf58 (id), errstr);
+                rc = 1;
+            }
+            else {
+                if (optparse_hasopt (p, "verbose"))
+                    fprintf (stderr,
+                             "%s: job completed successfully\n",
+                             idf58 (id));
+            }
+            flux_future_destroy (f);
+        }
+    }
+    else {
+        if (!(f = flux_job_wait (h, id)))
+            log_err_exit ("flux_job_wait");
+        if (flux_job_wait_get_status (f, &success, &errstr) < 0) {
+            /* ECHILD == no more waitable jobs or not waitable,
+             * exit code 2 instead of 1 */
+            if (errno == ECHILD)
+                rc = 2;
+            else
+                rc = 1;
+            log_msg ("%s", flux_future_error_string (f));
+            goto out;
+        }
+        if (id == FLUX_JOBID_ANY) {
+            if (flux_job_wait_get_id (f, &id) < 0)
+                log_err_exit ("flux_job_wait_get_id");
+            printf ("%s\n", idf58 (id));
+        }
+        if (!success)
+            log_msg_exit ("%s", errstr);
+        flux_future_destroy (f);
+    }
+out:
+    flux_close (h);
+    return (rc);
+}
+
+
+
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/cmd/py-runner.py b/src/cmd/py-runner.py
new file mode 100755
index 000000000000..5cf7cbb419c2
--- /dev/null
+++ b/src/cmd/py-runner.py
@@ -0,0 +1,97 @@
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import os
+import runpy
+import sys
+import sysconfig
+
+from _flux._core import ffi, lib
+
+
+def split_path(path):
+    return filter(None, path.split(":"))
+
+
+def builtin_python_path():
+    """
+    Get the path to this module and assume this is the builtin
+    Python path.
+    """
+    #  We can't use FLUX_CONF_AUTO since the executable in this context is
+    #  just python or python3, therefore determine which builtin conf flag to
+    #  use dynamically from the path to the current file.
+    #
+    #  Note: We need to avoid importing 'flux' here, since that might trigger
+    #  the problem this module is trying to solve (user modified PYTHONPATH
+    #  bringing incompatible modules first). So we copy the definitions of
+    #  FLUX_CONF_INTREE and FLUX_CONF_INSTALLED directly from `lib` here.
+    #
+    flag = lib.FLUX_CONF_INSTALLED
+    if "src/cmd" in __file__:
+        flag = lib.FLUX_CONF_INTREE
+
+    return split_path(
+        ffi.string(lib.flux_conf_builtin_get(b"python_path", flag)).decode("utf-8")
+    )
+
+
+def flux_python_path_prepend():
+    """
+    Get the value of FLUX_PYTHONPATH_PREPEND as a list. An empty list is
+    returned if the environment variable is not set.
+    """
+    return split_path(os.environ.get("FLUX_PYTHONPATH_PREPEND", ""))
+
+
+def prepend_standard_python_paths(patharray):
+    """
+    Prepend standard interpreter and Flux python module paths to the
+    input array `patharray`.
+
+    Paths that will be prepended include, in order:
+    - FLUX_PYTHONPATH_PREPEND
+    - The Flux builtin path to its own modules
+    - The Python interpreter configured values for
+      - stdlib
+      - platform specific stdlib
+      - platform modules
+      - pure python modules
+    """
+    #  Set of paths to append, in order, to sys.path after builtin path:
+    std_paths = ("stdlib", "platstdlib", "platlib", "purelib")
+
+    prepend_paths = [
+        *flux_python_path_prepend(),
+        *builtin_python_path(),
+        *[sysconfig.get_path(name) for name in std_paths],
+    ]
+
+    for path in prepend_paths:
+        try:
+            sys.path.remove(path)
+        except ValueError:
+            pass  # ignore missing path in sys.path
+
+    # Prepend list to sys.path:
+    sys.path[0:0] = prepend_paths
+
+
+if __name__ == "__main__":
+    #  Pop first argument which is this script, modify sys.path as noted
+    #  above, then invoke target script in this interpreter using
+    #  runpy.run_path():
+    #
+    sys.argv.pop(0)
+    prepend_standard_python_paths(sys.path)
+    runpy.run_path(sys.argv[0], run_name="__main__")
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/cmd/top/joblist_pane.c b/src/cmd/top/joblist_pane.c
new file mode 100644
index 000000000000..e7cf6e798edf
--- /dev/null
+++ b/src/cmd/top/joblist_pane.c
@@ -0,0 +1,481 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <unistd.h>
+#include <time.h>
+#include <math.h>
+#include <jansson.h>
+
+#include "src/common/libutil/fsd.h"
+#include "src/common/libjob/idf58.h"
+#include "ccan/str/str.h"
+
+#include "top.h"
+
+static const struct dimension win_dim = { 0, 6, 80, 60 };
+
+struct joblist_pane {
+    struct top *top;
+    int jobid_width;
+    WINDOW *win;
+    json_t *jobs_all;
+    json_t *jobs;
+    struct ucache *ucache;
+
+    bool show_queue;
+
+    /*  Currently selected jobid. Ironically FLUX_JOBID_ANY means
+     *   no current selection.
+     */
+    flux_jobid_t current;
+};
+
+static int lookup_jobid_index (json_t *jobs, flux_jobid_t id)
+{
+    int result = -1;
+    size_t index;
+    json_t *job;
+
+    if (jobs) {
+        json_array_foreach (jobs, index, job) {
+            flux_jobid_t jobid;
+
+            if (json_unpack (job, "{s:I}", "id", &jobid) < 0)
+                continue;
+            if (jobid == id)
+                return (int) index;
+        }
+    }
+    return result;
+}
+
+static json_t *get_current_job (struct joblist_pane *joblist)
+{
+    int index = lookup_jobid_index (joblist->jobs,
+                                    joblist->current);
+    return json_array_get (joblist->jobs, index);
+}
+
+
+void joblist_pane_draw (struct joblist_pane *joblist)
+{
+    double now = flux_reactor_now (flux_get_reactor (joblist->top->h));
+    size_t index;
+    json_t *job;
+    int queue_width = joblist->show_queue ? 8 : 0;
+    int name_width;
+    int job_output_count = 0;
+
+    werase (joblist->win);
+    wattron (joblist->win, A_REVERSE);
+
+    name_width = getmaxx (joblist->win)
+                 - (12 + 8 + queue_width + 2 + 6 + 6 + 7 + 6);
+    if (joblist->show_queue)
+        mvwprintw (joblist->win,
+                   0,
+                   0,
+                   "%*s %8s %8s %2s %6s %6s %7s %-*s",
+                   joblist->jobid_width,
+                   "JOBID", "QUEUE", "USER", "ST",
+                   "NTASKS", "NNODES", "RUNTIME",
+                   name_width, "NAME");
+    else
+        mvwprintw (joblist->win,
+                   0,
+                   0,
+                   "%*s %8s %2s %6s %6s %7s %-*s",
+                   joblist->jobid_width,
+                   "JOBID","USER", "ST", "NTASKS", "NNODES", "RUNTIME",
+                   name_width, "NAME");
+
+    wattroff (joblist->win, A_REVERSE);
+    if (joblist->jobs == NULL)
+        return;
+
+    if (json_array_size (joblist->jobs) == 0
+        && queues_configured (joblist->top->queues)) {
+        const char *filter_queue = NULL;
+        /* can return NULL filter_queue for "all" queues */
+        queues_get_queue_name (joblist->top->queues, &filter_queue);
+        if (filter_queue)
+            mvwprintw (joblist->win,
+                       5,
+                       25,
+                       "No jobs to display in queue %s",
+                       filter_queue);
+        return;
+    }
+
+    json_array_foreach (joblist->jobs, index, job) {
+        char *uri = NULL;
+        const char *idstr;
+        char run[16] = "";
+        flux_jobid_t id;
+        int userid;
+        const char *username;
+        const char *name;
+        const char *queue = "";
+        int state;
+        int ntasks;
+        int nnodes;
+        double t_run;
+
+        /* A running job is in the RUN or CLEANUP state.  Under racy circumstances
+         * a job could end up in the CLEANUP state without having entered the
+         * RUN state and some fields below would not be available.  If we fail
+         * to unpack this data, simply continue onto the next job.
+         */
+
+        if (json_unpack (job,
+                         "{s:I s:i s:i s:s s?s s:i s:i s:f s?{s?{s?s}}}",
+                         "id", &id,
+                         "userid", &userid,
+                         "state", &state,
+                         "name", &name,
+                         "queue", &queue,
+                         "nnodes", &nnodes,
+                         "ntasks", &ntasks,
+                         "t_run", &t_run,
+                         "annotations",
+                           "user",
+                             "uri", &uri) < 0)
+            continue;
+
+        idstr = idf58 (id);
+        (void)fsd_format_duration_ex (run, sizeof (run), fabs (now - t_run), 2);
+        if (!(username = ucache_lookup (joblist->ucache, userid)))
+            fatal (errno, "error looking up userid %d in ucache", (int)userid);
+
+        /*  Highlight current selection in blue, o/w color jobs that
+         *   are Flux instances blue (and bold for non-color terminals)
+         */
+        if (id == joblist->current)
+            wattron (joblist->win, A_REVERSE);
+        if (uri != NULL)
+            wattron (joblist->win, COLOR_PAIR(TOP_COLOR_BLUE) | A_BOLD);
+        if (joblist->show_queue) {
+            mvwprintw (joblist->win,
+                       1 + job_output_count,
+                       0,
+                        "%13.13s %8.8s %8.8s %2.2s %6d %6d %7.7s %-*.*s",
+                       idstr,
+                       queue,
+                       username,
+                       flux_job_statetostr (state, "S"),
+                       ntasks,
+                       nnodes,
+                       run,
+                       name_width,
+                       name_width,
+                       name);
+            if (joblist->top->testf)
+                fprintf (joblist->top->testf,
+                         "%s %s %s %s %d %d %s %s\n",
+                         idstr,
+                         queue,
+                         username,
+                         flux_job_statetostr (state, "S"),
+                         ntasks,
+                         nnodes,
+                         run,
+                         name);
+        }
+        else {
+            mvwprintw (joblist->win,
+                       1 + job_output_count,
+                       0,
+                        "%13.13s %8.8s %2.2s %6d %6d %7.7s %-*.*s",
+                       idstr,
+                       username,
+                       flux_job_statetostr (state, "S"),
+                       ntasks,
+                       nnodes,
+                       run,
+                       name_width,
+                       name_width,
+                       name);
+            if (joblist->top->testf)
+                fprintf (joblist->top->testf,
+                         "%s %s %s %d %d %s %s\n",
+                         idstr,
+                         username,
+                         flux_job_statetostr (state, "S"),
+                         ntasks,
+                         nnodes,
+                         run,
+                         name);
+        }
+        job_output_count++;
+        wattroff (joblist->win, A_REVERSE);
+        wattroff (joblist->win, COLOR_PAIR(TOP_COLOR_BLUE) | A_BOLD);
+    }
+}
+
+void joblist_filter_jobs (struct joblist_pane *joblist)
+{
+    json_decref (joblist->jobs);
+
+    if (queues_configured (joblist->top->queues)) {
+        const char *filter_queue;
+        /* can return NULL filter_queue for "all" queues */
+        queues_get_queue_name (joblist->top->queues, &filter_queue);
+        if (filter_queue) {
+            json_t *a;
+            size_t index;
+            json_t *job;
+            if (!(a = json_array ()))
+                fatal (ENOMEM, "error creating joblist array");
+            json_array_foreach (joblist->jobs_all, index, job) {
+                const char *queue;
+                /* skip jobs that don't have queue configured.  Potentially
+                 * rare case in racy scenarios.
+                 */
+                if (json_unpack (job, "{s:s}", "queue", &queue) < 0)
+                    continue;
+                if (!streq (filter_queue, queue))
+                    continue;
+                if (json_array_append (a, job) < 0)
+                    fatal (ENOMEM, "error appending job to joblist");
+            }
+            joblist->jobs = a;
+            return;
+        }
+    }
+    joblist->jobs = json_incref (joblist->jobs_all);
+}
+
+static void joblist_continuation (flux_future_t *f, void *arg)
+{
+    struct joblist_pane *joblist = arg;
+    json_t *jobs;
+
+    if (flux_rpc_get_unpack (f, "{s:o}", "jobs", &jobs) < 0) {
+        if (errno != ENOSYS)
+            fatal (errno, "error decoding job-list.list RPC response");
+        flux_future_destroy (f);
+        return;
+    }
+    json_decref (joblist->jobs_all);
+    joblist->jobs_all = json_incref (jobs);
+    joblist_filter_jobs (joblist);
+    joblist_pane_draw (joblist);
+    if (joblist->top->test_exit) {
+        /* Ensure joblist window is refreshed before exiting */
+        wrefresh (joblist->win);
+        test_exit_check (joblist->top);
+    }
+    flux_future_destroy (f);
+}
+
+
+/* Attempt to create a popup box over the joblist pane to
+ *  display one or more errors. The box will stay open until
+ *  the user presses a key.
+ */
+static void error_popup (struct joblist_pane *joblist,
+                         const char *msg)
+{
+    WINDOW *popup = newwin (6, 78, 15, 2);
+    WINDOW *errors = NULL;
+    if (!popup)
+        goto out;
+    box (popup, 0, 0);
+    touchwin (popup);
+    overwrite (popup, joblist->win);
+
+    if (!(errors = derwin (popup, 3, 75, 2, 2)))
+        goto out;
+
+    mvwprintw (errors, 0, 0, "%s", msg);
+
+    /*  Refresh windows
+     */
+    wrefresh (popup);
+    wrefresh (errors);
+
+    /*  Display error for up to 4s. Any key exits prematurely */
+    halfdelay (40);
+    getch ();
+
+    /* Leave halfdelay mode */
+    nocbreak ();
+    cbreak ();
+
+out:
+    if (popup)
+        delwin (popup);
+    if (errors)
+        delwin (errors);
+}
+
+void joblist_pane_enter (struct joblist_pane *joblist)
+{
+    struct top *top;
+    flux_jobid_t id;
+    char *uri = NULL;
+    char title [1024];
+    flux_error_t error;
+
+    json_t *job = get_current_job (joblist);
+    if (!job)
+        return;
+    if (json_unpack (job,
+                     "{s:I s:{s:{s:s}}}",
+                     "id", &id,
+                     "annotations",
+                       "user",
+                         "uri", &uri) < 0)
+        return;
+    if (uri == NULL)
+        return;
+    if (snprintf (title,
+                  sizeof(title),
+                  "%s/%s",
+                  joblist->top->title,
+                  idf58 (id)) > sizeof (title))
+        fatal (errno, "failed to build job title for job");
+
+    /*  Lazily attempt to run top on jobid, but for now simply return to the
+     *   original top window on failure.
+     */
+    if ((top = top_create (uri, title, NULL, &error)))
+        top_run (top, 0);
+    else
+        error_popup (joblist, error.text);
+    top_destroy (top);
+    return;
+}
+
+void joblist_pane_query (struct joblist_pane *joblist)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_rpc_pack (joblist->top->h,
+                             "job-list.list",
+                             0,
+                             0,
+                             "{s:i s:{s:[i]} s:[s,s,s,s,s,s,s,s]}",
+                             "max_entries", win_dim.y_length - 1,
+                             "constraint",
+                             "states", FLUX_JOB_STATE_RUNNING,
+                             "attrs",
+                               "annotations",
+                               "userid",
+                               "state",
+                               "name",
+                               "queue",
+                               "nnodes",
+                               "ntasks",
+                               "t_run"))
+        || flux_future_then (f, -1, joblist_continuation, joblist) < 0)
+        fatal (errno, "error sending job-list.list RPC request");
+}
+
+void joblist_pane_refresh (struct joblist_pane *joblist)
+{
+    wnoutrefresh (joblist->win);
+}
+
+void joblist_pane_set_current (struct joblist_pane *joblist, bool next)
+{
+    int index = -1;
+    json_t *job;
+    flux_jobid_t id = FLUX_JOBID_ANY;
+    int njobs;
+    int next_index;
+
+    if (joblist->jobs == NULL)
+        return;
+
+    if (joblist->current != FLUX_JOBID_ANY)
+        index = lookup_jobid_index (joblist->jobs, joblist->current);
+
+    /* find next valid index */
+    njobs = json_array_size (joblist->jobs);
+    next_index = next ? index + 1 : index - 1;
+
+    /* wrap around to top/bottom if index out of range */
+    if (next_index == njobs)
+        next_index = 0;
+    else if (next_index < 0)
+        next_index = njobs - 1;
+
+    if (!(job = json_array_get (joblist->jobs, next_index)))
+        return;
+
+    if (job && json_unpack (job, "{s:I}", "id", &id) < 0)
+        return;
+
+    if (id != joblist->current) {
+        joblist->current = id;
+        joblist_pane_draw (joblist);
+    }
+}
+
+/*
+ *  Workaround for mvwprintw(3) issues with multibyte jobid 'ƒ' character.
+ *
+ *  Empirically, the JOBID column must be formatted as %12s when the 'ƒ'
+ *  character appears in f58 encoded jobids, but %13s when ascii 'f' is used.
+ *  Guess at the current jobid encoding by determining if the f58 encoding
+ *  of jobid 0 has a length of 2 (ascii) or 3 (utf-8).
+ *
+ */
+static int estimate_jobid_width (void)
+{
+    const char *id = idf58 (0);
+    if (strlen (id) == 2)
+        return 13;
+    else
+        return 12;
+}
+
+struct joblist_pane *joblist_pane_create (struct top *top)
+{
+    struct joblist_pane *joblist;
+
+    if (!(joblist = calloc (1, sizeof (*joblist))))
+        fatal (errno, "could not allocate joblist context");
+    if (!(joblist->ucache = ucache_create ()))
+        fatal (errno, "could not create ucache");
+    joblist->top = top;
+    joblist->jobid_width = estimate_jobid_width ();
+    joblist->current = FLUX_JOBID_ANY;
+    joblist->show_queue = queues_configured (top->queues);
+    if (!(joblist->win = newwin (win_dim.y_length,
+                                 win_dim.x_length,
+                                 win_dim.y_begin,
+                                 win_dim.x_begin)))
+        fatal (0, "error creating joblist curses window");
+    joblist_pane_query (joblist);
+    joblist_pane_draw (joblist);
+    joblist_pane_refresh (joblist);
+    return joblist;
+}
+
+void joblist_pane_destroy (struct joblist_pane *joblist)
+{
+    if (joblist) {
+        int saved_errno = errno;
+        delwin (joblist->win);
+        ucache_destroy (joblist->ucache);
+        json_decref (joblist->jobs_all);
+        json_decref (joblist->jobs);
+        free (joblist);
+        errno = saved_errno;
+    }
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/cmd/top/keys.c b/src/cmd/top/keys.c
new file mode 100644
index 000000000000..c34558912b97
--- /dev/null
+++ b/src/cmd/top/keys.c
@@ -0,0 +1,109 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <unistd.h>
+#include <time.h>
+
+#include "top.h"
+
+struct keys {
+    struct top *top;
+    flux_watcher_t *w;
+};
+
+static void keys_cb (flux_reactor_t *r,
+                     flux_watcher_t *w,
+                     int revents,
+                     void *arg)
+{
+    struct keys *keys = arg;
+    int c;
+
+    switch ((c = getch())) {
+        case 'q':
+            flux_reactor_stop (r);
+            break;
+        case 'j':
+        case KEY_DOWN:
+            joblist_pane_set_current (keys->top->joblist_pane, true);
+            break;
+        case 'k':
+        case KEY_UP:
+            joblist_pane_set_current (keys->top->joblist_pane, false);
+            break;
+        case 'h':
+        case KEY_LEFT:
+            queues_prev (keys->top->queues);
+            summary_pane_query (keys->top->summary_pane);
+            summary_pane_draw (keys->top->summary_pane);
+            joblist_filter_jobs (keys->top->joblist_pane);
+            joblist_pane_draw (keys->top->joblist_pane);
+            break;
+        case 'l':
+        case KEY_RIGHT:
+            queues_next (keys->top->queues);
+            summary_pane_query (keys->top->summary_pane);
+            summary_pane_draw (keys->top->summary_pane);
+            joblist_filter_jobs (keys->top->joblist_pane);
+            joblist_pane_draw (keys->top->joblist_pane);
+            break;
+        case '\n':
+        case KEY_ENTER:
+            joblist_pane_enter (keys->top->joblist_pane);
+            break;
+        case 'd':
+            summary_pane_toggle_details (keys->top->summary_pane);
+            break;
+        case '':
+            clear ();
+            summary_pane_draw (keys->top->summary_pane);
+            joblist_pane_draw (keys->top->joblist_pane);
+            break;
+    }
+}
+
+struct keys *keys_create (struct top *top)
+{
+    struct keys *keys;
+
+    if (!(keys = calloc (1, sizeof (*keys))))
+        fatal (errno, "error creating context for key handling");
+    if (!(keys->w = flux_fd_watcher_create (flux_get_reactor (top->h),
+                                            STDIN_FILENO,
+                                            FLUX_POLLIN,
+                                            keys_cb,
+                                            keys)))
+        fatal (errno, "error creating fd watcher for stdin");
+    keys->top = top;
+
+    cbreak ();
+    noecho ();
+    intrflush (stdscr, FALSE);
+    keypad (stdscr, TRUE);
+
+    flux_watcher_start (keys->w);
+    return keys;
+}
+
+void keys_destroy (struct keys *keys)
+{
+    if (keys) {
+        int saved_errno = errno;
+        flux_watcher_destroy (keys->w);
+        free (keys);
+        errno = saved_errno;
+    }
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/cmd/top/queues.c b/src/cmd/top/queues.c
new file mode 100644
index 000000000000..aa41d03fc2ae
--- /dev/null
+++ b/src/cmd/top/queues.c
@@ -0,0 +1,212 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* queues.c - simple abstraction of queues in flux
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <jansson.h>
+
+#include "ccan/list/list.h"
+#include "src/common/libccan/ccan/str/str.h"
+
+#include "top.h"
+
+struct queue {
+    char *name;
+    json_t *constraint;
+    struct list_node list_node;
+};
+
+struct queues {
+    json_t *flux_config;
+    struct list_head queues_list;
+    struct queue *current;
+};
+
+static struct queue *queue_create (struct queues *queues, const char *name)
+{
+    json_t *requires = NULL;
+    struct queue *q = calloc (1, sizeof (*q));
+    if (!q)
+        fatal (0, "could not allocate queue entry");
+
+    /* name == NULL means "all" queues */
+    if (name) {
+        if (!(q->name = strdup (name)))
+            fatal (0, "could not duplicate queue name entry");
+
+        /* not required to be configured */
+        (void) json_unpack (queues->flux_config,
+                            "{s:{s:{s:o}}}",
+                            "queues",
+                            name,
+                            "requires",
+                            &requires);
+        if (requires) {
+            if (!(q->constraint = json_pack ("{s:O}",
+                                             "properties",
+                                             requires)))
+                fatal (0, "could not allocate queue constraint");
+        }
+    }
+
+    list_node_init (&q->list_node);
+    return q;
+}
+
+static void queue_destroy (void *data)
+{
+    if (data) {
+        int save_errno = errno;
+        struct queue *q = data;
+        free (q->name);
+        json_decref (q->constraint);
+        free (q);
+        errno = save_errno;
+    }
+}
+
+void queues_destroy (struct queues *queues)
+{
+    if (queues) {
+        int saved_errno = errno;
+        struct queue *q;
+        json_decref (queues->flux_config);
+        while ((q = list_pop (&queues->queues_list,
+                              struct queue,
+                              list_node)))
+            queue_destroy (q);
+        free (queues);
+        errno = saved_errno;
+    }
+}
+
+static void queues_list_setup (struct queues *queues)
+{
+    struct queue *q;
+    json_t *o;
+    const char *name;
+    json_t *value;
+
+    list_head_init (&queues->queues_list);
+
+    /* first, we add a queue for "all" queues with NULL queue name. */
+    q = queue_create (queues, NULL);
+    list_add_tail (&queues->queues_list, &q->list_node);
+
+    /* return if no queues configured */
+    if (json_unpack (queues->flux_config,
+                     "{s:o}",
+                     "queues", &o) < 0)
+        return;
+
+    json_object_foreach (o, name, value) {
+        q = queue_create (queues, name);
+        list_add_tail (&queues->queues_list, &q->list_node);
+    }
+}
+
+struct queues *queues_create (json_t *flux_config)
+{
+    struct queues *queues;
+
+    if (!(queues = calloc (1, sizeof (*queues))))
+        return NULL;
+    queues->flux_config = json_incref (flux_config);
+
+    queues_list_setup (queues);
+
+    /* must work, minimally the "all" queue is configured */
+    queues->current = list_top (&queues->queues_list, struct queue, list_node);
+    return queues;
+}
+
+static bool is_valid_queue (struct queues *queues, const char *name)
+{
+    json_t *tmp;
+
+    if (json_unpack (queues->flux_config,
+                     "{s:{s:o}}",
+                     "queues", name, &tmp) < 0)
+        return false;
+    return true;
+}
+
+bool queues_configured (struct queues *queues)
+{
+    json_t *tmp;
+
+    if (json_unpack (queues->flux_config,
+                     "{s:o}",
+                     "queues", &tmp) < 0)
+        return false;
+    return true;
+}
+
+void queues_set_queue (struct queues *queues, const char *name)
+{
+    struct queue *q = NULL;
+
+    assert (name);
+
+    /* first verify queue legit */
+    if (!is_valid_queue (queues, name))
+        fatal (0, "queue %s not configured", name);
+
+    list_for_each(&queues->queues_list, q, list_node) {
+        /* q->name == NULL is "all" queues, don't compare */
+        if (q->name && streq (q->name, name)) {
+            queues->current = q;
+            break;
+        }
+    }
+}
+
+void queues_next (struct queues *queues)
+{
+    queues->current = list_next (&queues->queues_list,
+                                 queues->current,
+                                 list_node);
+    if (!queues->current)
+        queues->current = list_top (&queues->queues_list,
+                                    struct queue,
+                                    list_node);
+}
+
+void queues_prev (struct queues *queues)
+{
+    queues->current = list_prev (&queues->queues_list,
+                                 queues->current,
+                                 list_node);
+    if (!queues->current)
+        queues->current = list_tail (&queues->queues_list,
+                                     struct queue,
+                                     list_node);
+}
+
+void queues_get_queue_name (struct queues *queues, const char **name)
+{
+    if (name)
+        (*name) = queues->current->name;
+}
+
+void queues_get_queue_constraint (struct queues *queues, json_t **constraint)
+{
+    if (constraint)
+        (*constraint) = queues->current->constraint;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/cmd/top/summary_pane.c b/src/cmd/top/summary_pane.c
new file mode 100644
index 000000000000..60f3a0091451
--- /dev/null
+++ b/src/cmd/top/summary_pane.c
@@ -0,0 +1,677 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <unistd.h>
+#include <time.h>
+#include <jansson.h>
+#include <math.h>
+
+#include "src/common/libutil/fsd.h"
+#include "src/common/librlist/rlist.h"
+#include "ccan/str/str.h"
+
+#include "top.h"
+
+#define DEFAULT_RESOURCE_LIST_RPC "resource.sched-status"
+
+static const struct dimension win_dim = { 0, 0, 80, 6 };
+static const struct dimension level_dim = { 0, 0, 2, 1 };
+static const struct dimension title_dim = { 6, 0, 73, 1 };
+static const struct dimension timeleft_dim = { 70, 0, 10, 1 };
+static const struct dimension resource_dim = { 4, 1, 36, 3 };
+static const struct dimension heart_dim = { 77, 3, 1, 1 };
+static const struct dimension stats_dim = { 60, 1, 15, 3 };
+static const struct dimension info_dim = { 1, 5, 78, 1 };
+
+static const double heartblink_duration = 0.5;
+
+struct resource_count {
+    int total;
+    int down;
+    int used;
+};
+
+struct stats {
+    int depend;
+    int priority;
+    int sched;
+    int run;
+    int cleanup;
+    int inactive;
+    int successful;
+    int failed;
+    int canceled;
+    int timeout;
+    int total;
+};
+
+struct summary_pane {
+    struct top *top;
+    WINDOW *win;
+    int instance_level;
+    int instance_size;
+    double starttime;
+    uid_t owner;
+    bool show_details;
+    const char *instance_version;
+    const char *resource_rpc;
+    double expiration;
+    struct stats stats;
+    struct resource_count node;
+    struct resource_count core;
+    struct resource_count gpu;
+    flux_watcher_t *heartblink;
+    bool heart_visible;
+    flux_jobid_t current;
+    json_t *jobs;
+    flux_future_t *f_resource;
+};
+
+static void draw_timeleft (struct summary_pane *sum)
+{
+    double now = flux_reactor_now (flux_get_reactor (sum->top->h));
+    double timeleft = sum->expiration - now;
+    char buf[32] = "";
+
+    if (timeleft > 0)
+        fsd_format_duration_ex (buf, sizeof (buf), timeleft, 2);
+
+    mvwprintw (sum->win,
+               timeleft_dim.y_begin,
+               timeleft_dim.x_begin,
+               "%*s%s",
+               timeleft_dim.x_length - 2,
+               buf,
+               timeleft > 0 ? "⌚" : "∞");
+}
+
+static void draw_f (struct summary_pane *sum)
+{
+    wattron (sum->win, COLOR_PAIR (TOP_COLOR_YELLOW));
+    mvwprintw (sum->win,
+               level_dim.y_begin,
+               level_dim.x_begin,
+               "%s",
+               sum->top->f_char);
+    wattroff (sum->win, COLOR_PAIR (TOP_COLOR_YELLOW));
+}
+
+static void draw_title (struct summary_pane *sum)
+{
+    int len = strlen (sum->top->title);
+    int begin = title_dim.x_begin + (title_dim.x_length - len)/2;
+    int start = 0;
+    char *dots = "";
+
+    if (len > title_dim.x_length) {
+        dots = "…";
+        begin = title_dim.x_begin;
+        start = (len - title_dim.x_length) + strlen (dots) - 1;
+    }
+    wattron (sum->win, COLOR_PAIR(TOP_COLOR_BLUE) | A_BOLD);
+    mvwprintw (sum->win,
+               title_dim.y_begin,
+                begin,
+               "%s%s",
+                dots,
+                sum->top->title + start);
+    wattroff (sum->win, COLOR_PAIR(TOP_COLOR_BLUE) | A_BOLD);
+}
+
+static void draw_stats (struct summary_pane *sum)
+{
+    mvwprintw (sum->win,
+               stats_dim.y_begin,
+               stats_dim.x_begin,
+               "%*d pending",
+               stats_dim.x_length - 10,
+               sum->stats.depend + sum->stats.priority + sum->stats.sched);
+    mvwprintw (sum->win,
+               stats_dim.y_begin + 1,
+               stats_dim.x_begin,
+               "%*d running",
+               stats_dim.x_length - 10,
+               sum->stats.run + sum->stats.cleanup);
+
+    if (sum->top->testf) {
+        fprintf (sum->top->testf,
+                 "%d pending\n",
+                 sum->stats.depend + sum->stats.priority + sum->stats.sched);
+        fprintf (sum->top->testf,
+                 "%d running\n",
+                 sum->stats.run + sum->stats.cleanup);
+    }
+
+    if (sum->show_details) {
+        /* flux-top reports the total number of unsuccessful jobs in
+         * the 'failed' display, not just the count of jobs that ran
+         * to completion with nonzero exit code
+         */
+        int failed = sum->stats.failed + sum->stats.timeout + sum->stats.canceled;
+        int complete = sum->stats.successful;
+
+        if (complete)
+            wattron (sum->win, COLOR_PAIR(TOP_COLOR_GREEN) | A_BOLD);
+        mvwprintw (sum->win,
+                   stats_dim.y_begin + 2,
+                   stats_dim.x_begin - 18,
+                   "%6d",
+                   complete);
+        if (complete)
+            wattroff (sum->win, COLOR_PAIR(TOP_COLOR_GREEN) | A_BOLD);
+        mvwprintw (sum->win,
+                   stats_dim.y_begin + 2,
+                   stats_dim.x_begin - 12,
+                   " complete, ");
+        if (failed)
+            wattron (sum->win, COLOR_PAIR(TOP_COLOR_RED) | A_BOLD);
+        mvwprintw (sum->win,
+                   stats_dim.y_begin + 2,
+                   stats_dim.x_begin - 1,
+                   "%6d",
+                   failed);
+        if (failed)
+            wattroff (sum->win, COLOR_PAIR(TOP_COLOR_RED) | A_BOLD);
+        mvwprintw (sum->win,
+                   stats_dim.y_begin + 2,
+                   stats_dim.x_begin + 5,
+                   " failed");
+
+        if (sum->top->testf) {
+            fprintf (sum->top->testf,
+                     "%d complete\n",
+                     complete);
+            fprintf (sum->top->testf,
+                     "%d failed\n",
+                     failed);
+        }
+    }
+    else {
+        mvwprintw (sum->win,
+                   stats_dim.y_begin + 2,
+                   stats_dim.x_begin,
+                   "%*d inactive",
+                   stats_dim.x_length - 10,
+                   sum->stats.inactive);
+        if (sum->top->testf) {
+            fprintf (sum->top->testf,
+                     "%d inactive\n",
+                     sum->stats.inactive);
+        }
+    }
+}
+
+/* Create a little graph like this that fits in bufsz:
+ *     name [||||||||||        |||32/128]
+ * "used" grows from the left in yellow; "down" grows from the right in red.
+ * Fraction is used/total.
+ */
+static void draw_bargraph (struct summary_pane *sum, int y, int x, int x_length,
+                           const char *name, struct resource_count res)
+{
+    char prefix[16];
+    char suffix[16];
+
+    if (x_length > 80)
+        x_length = 80;
+    if (res.used > res.total)
+        res.used = res.total;
+
+    snprintf (prefix, sizeof (prefix), "%5s [", name);
+    snprintf (suffix, sizeof (suffix), "%d/%d]", res.used, res.total);
+
+    int slots = x_length - strlen (prefix) - strlen (suffix) - 1;
+    mvwprintw (sum->win,
+               y,
+               x,
+               "%s%*s%s",
+               prefix,
+               slots, "",
+               suffix);
+    /* Graph used */
+    wattron (sum->win, COLOR_PAIR (TOP_COLOR_YELLOW));
+    for (int i = 0; i < ceil (((double)res.used / res.total) * slots); i++)
+        mvwaddch (sum->win, y, x + strlen (prefix) + i, '|');
+    wattroff (sum->win, COLOR_PAIR (TOP_COLOR_YELLOW));
+
+    /* Graph down */
+    wattron (sum->win, COLOR_PAIR (TOP_COLOR_RED));
+    for (int i = slots - 1;
+         i >= slots - ceil (((double)res.down / res.total) * slots); i--) {
+        mvwaddch (sum->win, y, x + strlen (prefix) + i, '|');
+    }
+    wattroff (sum->win, COLOR_PAIR (TOP_COLOR_RED));
+
+    if (sum->top->testf)
+        fprintf (sum->top->testf,
+                 "%s %d/%d\n",
+                 name,
+                 res.used,
+                 res.total);
+}
+
+static void draw_resource (struct summary_pane *sum)
+{
+    draw_bargraph (sum,
+                   resource_dim.y_begin,
+                   resource_dim.x_begin,
+                   resource_dim.x_length,
+                   "nodes",
+                   sum->node);
+    draw_bargraph (sum,
+                   resource_dim.y_begin + 1,
+                   resource_dim.x_begin,
+                   resource_dim.x_length,
+                   "cores",
+                   sum->core);
+    draw_bargraph (sum,
+                   resource_dim.y_begin + 2,
+                   resource_dim.x_begin,
+                   resource_dim.x_length,
+                   "gpus",
+                   sum->gpu);
+}
+
+static void draw_heartbeat (struct summary_pane *sum)
+{
+    mvwprintw (sum->win,
+               heart_dim.y_begin,
+               heart_dim.x_begin,
+               "%s",
+               sum->heart_visible ? "♡" : " ");
+}
+
+static void draw_info (struct summary_pane *sum)
+{
+    double now = flux_reactor_now (flux_get_reactor (sum->top->h));
+    char fsd[32] = "";
+
+    (void)fsd_format_duration_ex (fsd,
+                                  sizeof (fsd),
+                                  fabs (now - sum->starttime),
+                                  2);
+
+    wattron (sum->win, A_DIM);
+    mvwprintw (sum->win,
+               info_dim.y_begin,
+               info_dim.x_begin,
+               "size: %d",
+               sum->instance_size);
+    if (sum->instance_level)
+        mvwprintw (sum->win,
+                   info_dim.y_begin,
+                   info_dim.x_begin + 10,
+                   "depth: %d",
+                   sum->instance_level);
+    mvwprintw (sum->win,
+               info_dim.y_begin,
+               info_dim.x_begin + 30,
+               "uptime: %s",
+               fsd);
+    mvwprintw (sum->win,
+               info_dim.y_begin,
+               info_dim.x_begin +
+                   info_dim.x_length - strlen (sum->instance_version),
+               "%s",
+               sum->instance_version);
+    wattroff (sum->win, A_DIM);
+}
+
+/* Fetch expiration time (abs time relative to UNIX epoch) from resource.R.
+ * If unavailable (e.g. we are a guest in the system instance), return 0.
+ */
+static double get_expiration (flux_t *h)
+{
+    flux_future_t *f;
+    double val = 0;
+
+    if (!(f = flux_kvs_lookup (h, NULL, 0, "resource.R"))
+        || flux_kvs_lookup_get_unpack (f,
+                                       "{s:{s:f}}",
+                                       "execution",
+                                       "expiration", &val) < 0) {
+        if (errno == EPERM)
+            goto done;
+        fatal (errno, "error fetching or decoding resource.R");
+    }
+done:
+    flux_future_destroy (f);
+    return val;
+}
+
+static int get_instance_attr_int (flux_t *h, const char *attr)
+{
+    const char *s;
+    unsigned long level;
+
+    if (!(s = flux_attr_get (h, attr)))
+        fatal (errno, "error fetching %s broker attribute", attr);
+    errno = 0;
+    level = strtoul (s, NULL, 10);
+    if (errno != 0)
+        fatal (errno, "error parsing %s", attr);
+    return level;
+}
+
+
+static int resource_count (json_t *o,
+                           const char *name,
+                           int *nnodes,
+                           int *ncores,
+                           int *ngpus,
+                           json_t *queue_constraint)
+{
+    json_t *R;
+    struct rlist *rl_all = NULL;
+    struct rlist *rl_constraint = NULL;
+    struct rlist *rl;
+
+    if (!(R = json_object_get (o, name)))
+        return -1;
+    if (json_is_null (R)) { // N.B. fluxion sets objects to json null if empty
+        *nnodes = *ncores = *ngpus = 0;
+        return 0;
+    }
+    if (!(rl_all = rlist_from_json (R, NULL)))
+        return -1;
+    if (queue_constraint) {
+        flux_error_t error;
+        rl_constraint = rlist_copy_constraint (rl_all,
+                                               queue_constraint,
+                                               &error);
+        if (!rl_constraint)
+            fatal (errno, "failed to create constrained rlist: %s", error.text);
+        rl = rl_constraint;
+    }
+    else
+        rl = rl_all;
+    *nnodes = rlist_nnodes (rl);
+    *ncores = rlist_count (rl, "core");
+    *ngpus = rlist_count (rl, "gpu");
+    rlist_destroy (rl_all);
+    rlist_destroy (rl_constraint);
+    return 0;
+}
+
+static void resource_continuation (flux_future_t *f, void *arg)
+{
+    struct summary_pane *sum = arg;
+    json_t *o;
+
+    if (flux_rpc_get_unpack (f, "o", &o) < 0) {
+        if (errno != ENOSYS) /* Instance may not be up yet */
+            fatal (errno, "%s RPC failed", sum->resource_rpc);
+    }
+    else {
+        json_t *queue_constraint;
+        /* can return NULL constraint for "none" */
+        queues_get_queue_constraint (sum->top->queues, &queue_constraint);
+        if (resource_count (o,
+                            "all",
+                            &sum->node.total,
+                            &sum->core.total,
+                            &sum->gpu.total,
+                            queue_constraint) < 0
+            || resource_count (o,
+                               "allocated",
+                               &sum->node.used,
+                               &sum->core.used,
+                               &sum->gpu.used,
+                               queue_constraint) < 0
+            || resource_count (o,
+                               "down",
+                               &sum->node.down,
+                               &sum->core.down,
+                               &sum->gpu.down,
+                               queue_constraint) < 0)
+            fatal (0, "error decoding %s RPC response", sum->resource_rpc);
+    }
+    flux_future_destroy (f);
+    sum->f_resource = NULL;
+    draw_resource (sum);
+    if (sum->top->test_exit) {
+        /* Ensure resources are refreshed before exiting */
+        wnoutrefresh (sum->win);
+        test_exit_check (sum->top);
+    }
+}
+
+static int get_queue_stats (json_t *o, const char *queue_name, json_t **qstats)
+{
+    json_t *queues = NULL;
+    json_t *q = NULL;
+    size_t index;
+    json_t *value;
+    if (json_unpack (o, "{s:o}", "queues", &queues) < 0)
+        return -1;
+    if (json_is_array (queues) == 0)
+        return -1;
+    json_array_foreach (queues, index, value) {
+        const char *name;
+        if (json_unpack (value, "{s:s}", "name", &name) < 0)
+            return -1;
+        if (streq (queue_name, name)) {
+            q = value;
+            break;
+        }
+    }
+    (*qstats) = q;
+    return 0;
+}
+
+void summary_pane_jobstats (struct summary_pane *sum, flux_future_t *f)
+{
+    json_t *o = NULL;
+    const char *filter_queue;
+
+    if (flux_rpc_get_unpack (f, "o", &o) < 0) {
+        if (errno != ENOSYS)
+            fatal (errno, "error getting job-list.job-stats RPC response");
+    }
+
+    /* can return NULL filter_queue for "all" queues */
+    queues_get_queue_name (sum->top->queues, &filter_queue);
+    if (filter_queue) {
+        json_t *qstats = NULL;
+        if (get_queue_stats (o, filter_queue, &qstats) < 0)
+            fatal (EPROTO, "error parsing queue stats");
+        /* stats may not yet exist if no jobs submitted to the queue */
+        if (!qstats)
+            goto out;
+        o = qstats;
+    }
+
+    if (json_unpack (o,
+                     "{s:i s:i s:i s:i s:{s:i s:i s:i s:i s:i s:i s:i}}",
+                     "successful", &sum->stats.successful,
+                     "failed", &sum->stats.failed,
+                     "canceled", &sum->stats.canceled,
+                     "timeout", &sum->stats.timeout,
+                     "job_states",
+                       "depend", &sum->stats.depend,
+                       "priority", &sum->stats.priority,
+                       "sched", &sum->stats.sched,
+                       "run", &sum->stats.run,
+                       "cleanup", &sum->stats.cleanup,
+                       "inactive", &sum->stats.inactive,
+                       "total", &sum->stats.total) < 0)
+        fatal (0, "error decoding job-list.job-stats object");
+
+out:
+    draw_stats (sum);
+    if (sum->top->test_exit) {
+        /* Ensure stats is refreshed before exiting */
+        wnoutrefresh (sum->win);
+        test_exit_check (sum->top);
+    }
+}
+
+static void heartblink_cb (flux_reactor_t *r,
+                           flux_watcher_t *w,
+                           int revents,
+                           void *arg)
+{
+    struct summary_pane *sum = arg;
+
+    sum->heart_visible = false;
+    draw_heartbeat (sum);
+}
+
+void summary_pane_heartbeat (struct summary_pane *sum)
+{
+    sum->heart_visible = true;
+    flux_timer_watcher_reset (sum->heartblink, heartblink_duration, 0.);
+    flux_watcher_start (sum->heartblink);
+}
+
+static void resource_retry_cb (flux_future_t *f, void *arg)
+{
+    flux_future_t *result = arg;
+    flux_future_fulfill_with (result, f);
+    flux_future_destroy (f);
+}
+
+static void resource_enosys_check_cb (flux_future_t *f, void *arg)
+{
+    flux_t *h = flux_future_get_flux (f);
+    flux_future_t *fretry = NULL;
+    flux_future_t *result = arg;
+
+    if (flux_future_get (f, NULL) == 0 || errno != ENOSYS) {
+        flux_future_fulfill_with (result, f);
+        return;
+    }
+    /*  The RPC failed with ENOSYS. Retry with sched.resource-status:
+     */
+    if (!(fretry = flux_rpc (h, "sched.resource-status", NULL, 0, 0))
+        || flux_future_then (fretry, -1., resource_retry_cb, result) < 0) {
+        flux_future_fulfill_error (result, errno, NULL);
+        flux_future_destroy (fretry);
+    }
+}
+
+flux_future_t *resource_sched_status (struct summary_pane *sum)
+{
+    flux_future_t *result = NULL;
+    flux_future_t *f = NULL;
+
+    /* Create empty future to contain result from either the default resource
+     * list topic string or the sched.resource-status RPC:
+     */
+    if (!(result = flux_future_create (NULL, NULL))
+        || !(f = flux_rpc (sum->top->h, sum->resource_rpc, NULL, 0, 0))
+        || flux_future_then (f, -1., resource_enosys_check_cb, result) < 0)
+        goto error;
+
+    flux_future_set_flux (result, sum->top->h);
+    return result;
+error:
+    flux_future_destroy (result);
+    flux_future_destroy (f);
+    return NULL;
+}
+
+/* Send a query.
+ * If there's already one pending, do nothing.
+ */
+void summary_pane_query (struct summary_pane *sum)
+{
+    if (!sum->f_resource) {
+        if (!(sum->f_resource = resource_sched_status (sum))
+            || flux_future_then (sum->f_resource,
+                                 -1,
+                                 resource_continuation,
+                                 sum) < 0) {
+            flux_future_destroy (sum->f_resource);
+            sum->f_resource = NULL;
+        }
+    }
+}
+
+void summary_pane_toggle_details (struct summary_pane *sum)
+{
+    sum->show_details = !sum->show_details;
+    summary_pane_draw (sum);
+}
+
+void summary_pane_draw (struct summary_pane *sum)
+{
+    werase (sum->win);
+    draw_f (sum);
+    draw_title (sum);
+    draw_timeleft (sum);
+    draw_resource (sum);
+    draw_stats (sum);
+    draw_info (sum);
+    draw_heartbeat (sum);
+}
+
+void summary_pane_refresh (struct summary_pane *sum)
+{
+    wnoutrefresh (sum->win);
+}
+
+struct summary_pane *summary_pane_create (struct top *top)
+{
+    struct summary_pane *sum;
+    flux_reactor_t *r = flux_get_reactor (top->h);
+
+    if (!(sum = calloc (1, sizeof (*sum))))
+        fatal (errno, "error creating context for summary pane");
+    if (!(sum->heartblink = flux_timer_watcher_create (r,
+                                                       heartblink_duration,
+                                                       0.,
+                                                       heartblink_cb,
+                                                       sum)))
+        fatal (errno, "error creating timer for heartbeat blink");
+    if (!(sum->win = newwin (win_dim.y_length,
+                             win_dim.x_length,
+                             win_dim.y_begin,
+                             win_dim.x_begin)))
+        fatal (0, "error creating curses window for summary pane");
+    sum->top = top;
+
+    sum->expiration = get_expiration (top->h);
+    sum->instance_level = get_instance_attr_int (top->h, "instance-level");
+    sum->instance_size = get_instance_attr_int (top->h, "size");
+    sum->instance_version = flux_attr_get (top->h, "version");
+    if (flux_get_instance_starttime (top->h, &sum->starttime) < 0)
+        sum->starttime = flux_reactor_now (flux_get_reactor (top->h));
+
+    sum->owner = get_instance_attr_int (top->h, "security.owner");
+    if (sum->owner == getuid ())
+        sum->show_details = true;
+
+    if (!(sum->resource_rpc = getenv ("FLUX_RESOURCE_LIST_RPC")))
+        sum->resource_rpc = DEFAULT_RESOURCE_LIST_RPC;
+
+    summary_pane_query (sum);
+    summary_pane_draw (sum);
+    summary_pane_refresh (sum);
+    return sum;
+}
+
+void summary_pane_destroy (struct summary_pane *sum)
+{
+    if (sum) {
+        int saved_errno = errno;
+        flux_future_destroy (sum->f_resource);
+        flux_watcher_destroy (sum->heartblink);
+        delwin (sum->win);
+        free (sum);
+        errno = saved_errno;
+    }
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/cmd/top/top.c b/src/cmd/top/top.c
new file mode 100644
index 000000000000..3b065159f68f
--- /dev/null
+++ b/src/cmd/top/top.c
@@ -0,0 +1,417 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <unistd.h>
+#include <locale.h>
+
+#include "src/common/libutil/uri.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libjob/idf58.h"
+#include "ccan/str/str.h"
+#include "top.h"
+
+static const double job_activity_rate_limit = 2;
+
+__attribute__ ((noreturn)) void fatal (int errnum, const char *fmt, ...)
+{
+    va_list ap;
+    char buf[128];
+
+    va_start (ap, fmt);
+    vsnprintf (buf, sizeof (buf), fmt, ap);
+    va_end (ap);
+
+    endwin ();
+    fprintf (stderr,
+             "flux-top: %s%s%s\n",
+             buf,
+             errnum == 0 ? "" : ": ",
+             errnum == 0 ? "" : strerror (errnum));
+    exit (1);
+}
+
+/* When connection is lost to Flux, this function is called before
+ * errors propagate to higher level functions (like RPCs), so it
+ * is an opportunity to consolidate error handling for that case.
+ * N.B. ssh:// connections do not always propagate fatal errors as expected.
+ */
+static int comms_error (flux_t *h, void *arg)
+{
+    fatal (0, "lost connection to Flux: %s", strerror (errno));
+}
+
+static void heartbeat_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
+{
+    struct top *top = arg;
+    summary_pane_heartbeat (top->summary_pane);
+    summary_pane_draw (top->summary_pane);
+    joblist_pane_draw (top->joblist_pane);
+    summary_pane_query (top->summary_pane);
+}
+
+static void jobtimer_cb (flux_reactor_t *r,
+                         flux_watcher_t *w,
+                         int revents,
+                         void *arg)
+{
+    struct top *top = arg;
+
+    joblist_pane_query (top->joblist_pane);
+    top->jobtimer_running = false;
+}
+
+/* After some job stats activity, and after a rate-limited delay,
+ * trigger queries for info in the two panes.
+ */
+static void stats_continuation (flux_future_t *f, void *arg)
+{
+    struct top *top = arg;
+    if (!top->jobtimer_running) {
+        flux_timer_watcher_reset (top->jobtimer, job_activity_rate_limit, 0.);
+        flux_watcher_start (top->jobtimer);
+        top->jobtimer_running = true;
+    }
+    summary_pane_jobstats (top->summary_pane, f);
+    flux_future_reset (f);
+}
+
+void refresh_cb (flux_reactor_t *r,
+                 flux_watcher_t *w,
+                 int revents,
+                 void *arg)
+{
+    struct top *top = arg;
+
+    summary_pane_refresh (top->summary_pane);
+    joblist_pane_refresh (top->joblist_pane);
+    doupdate ();
+}
+
+/* Get handle to Flux instance to be monitored.
+ * If id = FLUX_JOBID_ANY, merely call flux_open().
+ * Otherwise, fetch remote-uri from job and open that.
+ */
+static flux_t *open_flux_instance (const char *target, flux_error_t *errp)
+{
+    flux_t *h;
+    flux_error_t error;
+    flux_future_t *f = NULL;
+    char *uri = NULL;
+
+    if (target && !(uri = uri_resolve (target, &error))) {
+        errprintf (errp,
+                   "%s\n%s",
+                   "failed to resolve target to a Flux URI",
+                   error.text);
+        return NULL;
+    }
+    if (!(h = flux_open_ex (uri, 0, &error)))
+        errprintf (errp,
+                   "error connecting to Flux: %s\n%s",
+                   strerror (errno),
+                   error.text);
+    free (uri);
+    flux_future_destroy (f);
+    return h;
+}
+
+/* Initialize 'stdscr' and register colors.
+ * N.B. this program does not use stdscr, but it does use getch(), which
+ * implicitly refreshes stdscr.  Therefore, make sure that stdscr is synced
+ * with its internal buffer by calling refresh() below to prevent unwanted
+ * screen updates on the first keypress.
+ */
+static void initialize_curses (int color)
+{
+    char *cap;
+    initscr ();
+    if (!(cap = tigetstr ("cup")) || cap == (char *) -1)
+        fatal (0, "terminal does not support required capabilities");
+
+    curs_set (0); // make cursor disappear
+
+    use_default_colors ();
+    start_color ();
+    if (color) {
+        init_pair (TOP_COLOR_YELLOW, COLOR_YELLOW, -1);
+        init_pair (TOP_COLOR_RED, COLOR_RED, -1);
+        init_pair (TOP_COLOR_GREEN, COLOR_GREEN, -1);
+        init_pair (TOP_COLOR_BLUE, COLOR_BLUE, -1);
+        init_pair (TOP_COLOR_BLUE_HIGHLIGHT, COLOR_BLACK, COLOR_BLUE);
+    }
+    else {
+        init_pair (TOP_COLOR_YELLOW, -1, -1);
+        init_pair (TOP_COLOR_RED, -1, -1);
+        init_pair (TOP_COLOR_GREEN, -1, -1);
+        init_pair (TOP_COLOR_BLUE, -1, -1);
+        init_pair (TOP_COLOR_BLUE_HIGHLIGHT, -1, -1);
+    }
+    clear ();
+    refresh ();
+}
+
+int top_run (struct top *top, int reactor_flags)
+{
+    /*  Force curses to redraw screen in case we're calling top recursively.
+     *  (unsure why refresh() doesn't have same effect as wrefresh (curscr))
+     */
+    wrefresh (curscr);
+    return flux_reactor_run (flux_get_reactor (top->h), reactor_flags);
+}
+
+void test_exit_check (struct top *top)
+{
+    /* 3 exit counts for
+     * - joblist output
+     * - summary stats output
+     * - summary resource output
+     */
+    if (top->test_exit && ++top->test_exit_count == 3)
+        flux_reactor_stop (flux_get_reactor (top->h));
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_EVENT, "heartbeat.pulse", heartbeat_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void top_destroy (struct top *top)
+{
+    if (top) {
+        flux_watcher_destroy (top->refresh);
+        flux_watcher_destroy (top->jobtimer);
+        flux_future_destroy (top->f_stats);
+        flux_msg_handler_delvec (top->handlers);
+        joblist_pane_destroy (top->joblist_pane);
+        summary_pane_destroy (top->summary_pane);
+        keys_destroy (top->keys);
+        queues_destroy (top->queues);
+        json_decref (top->flux_config);
+        if (top->testf)
+            fclose (top->testf);
+        flux_close (top->h);
+        free (top->title);
+        free (top);
+    }
+}
+
+static flux_jobid_t get_jobid (flux_t *h)
+{
+    const char *s;
+    flux_jobid_t jobid;
+
+    if (!(s = flux_attr_get (h, "jobid")))
+        return FLUX_JOBID_ANY;
+    if (flux_job_id_parse (s, &jobid) < 0)
+        fatal (errno, "error parsing value of jobid attribute: %s", s);
+    return jobid;
+}
+
+static char * build_title (struct top *top, const char *title)
+{
+    if (!title)
+        title = top->id == FLUX_JOBID_ANY ? "" : idf58 (top->id);
+    return strdup (title);
+}
+
+static void get_config (struct top *top)
+{
+    flux_future_t *f;
+    json_t *o;
+
+    if (!(f = flux_rpc (top->h, "config.get", NULL, FLUX_NODEID_ANY, 0))
+        || flux_rpc_get_unpack (f, "o", &o) < 0)
+        fatal (errno, "Error fetching flux config");
+
+    top->flux_config = json_incref (o);
+    flux_future_destroy (f);
+}
+
+struct top *top_create (const char *uri,
+                        const char *title,
+                        const char *queue,
+                        flux_error_t *errp)
+{
+    struct top *top = calloc (1, sizeof (*top));
+
+    if (!top || !(top->h = open_flux_instance (uri, errp)))
+        goto fail;
+
+    top->id = get_jobid (top->h);
+    if (!(top->title = build_title (top, title)))
+        goto fail;
+
+    get_config (top);
+
+    if (!(top->queues = queues_create (top->flux_config)))
+        goto fail;
+
+    /* setup / configure queue before calls to joblist_pane_create() and
+     * summary_pane_create() below */
+    if (queue)
+        queues_set_queue (top->queues, queue);
+
+    flux_comms_error_set (top->h, comms_error, &top);
+    top->refresh = flux_prepare_watcher_create (flux_get_reactor (top->h),
+                                                refresh_cb,
+                                                top);
+    top->jobtimer = flux_timer_watcher_create (flux_get_reactor (top->h),
+                                               0.,
+                                               0.,
+                                               jobtimer_cb,
+                                               top);
+    if (!top->refresh || !top->jobtimer)
+        goto fail;
+    flux_watcher_start (top->refresh);
+
+    if (flux_msg_handler_addvec (top->h, htab, top, &top->handlers) < 0)
+        goto fail;
+    if (flux_event_subscribe (top->h, "heartbeat.pulse") < 0)
+        fatal (errno, "error subscribing to events");
+     if (!(top->f_stats = flux_rpc (top->h,
+                                    "job-list.job-stats",
+                                    "{}",
+                                    0,
+                                    FLUX_RPC_STREAMING))
+            || flux_future_then (top->f_stats,
+                                 -1,
+                                 stats_continuation,
+                                 top) < 0)
+         fatal (errno, "error making streaming job-stats request");
+
+#if ASSUME_BROKEN_LOCALE
+    top->f_char = "f";
+#else
+    if (getenv ("FLUX_F58_FORCE_ASCII"))
+        top->f_char = "f";
+    else
+        top->f_char = "ƒ";
+#endif /* ASSUME_BROKEN_LOCALE */
+
+    top->keys = keys_create (top);
+    top->summary_pane = summary_pane_create (top);
+    top->joblist_pane = joblist_pane_create (top);
+    return top;
+fail:
+    top_destroy (top);
+    return NULL;
+}
+
+static int color_optparse (optparse_t *opts)
+{
+    const char *when;
+    int color = 0;
+
+    if (!(when = optparse_get_str (opts, "color", "auto")))
+        when = "always";
+    if (streq (when, "always"))
+        color = 1;
+    else if (streq (when, "never"))
+        color = 0;
+    else if (streq (when, "auto"))
+        color = isatty (STDOUT_FILENO) ? 1 : 0;
+    else
+        fatal (0, "Invalid argument to --color: '%s'", when);
+    return color;
+}
+
+static struct optparse_option cmdopts[] = {
+    { .name = "test-exit", .has_arg = 0, .flags = OPTPARSE_OPT_HIDDEN,
+      .usage = "Exit after screen initialization, for testing",
+    },
+    { .name = "test-exit-dump", .has_arg = 1, .arginfo = "FILE",
+      .flags = OPTPARSE_OPT_HIDDEN,
+      .usage = "Dump joblist/summary data to file for testing",
+    },
+    { .name = "color", .has_arg = 2, .arginfo = "WHEN",
+      .usage = "Colorize output when supported; WHEN can be 'always' "
+               "(default if omitted), 'never', or 'auto' (default)." },
+    { .name = "queue", .key = 'q', .has_arg = 1, .arginfo = "NAME",
+      .usage = "Limit to jobs belonging to a specific queue",
+    },
+    OPTPARSE_TABLE_END,
+};
+
+static const char *usage_msg = "[OPTIONS] [TARGET]";
+
+int main (int argc, char *argv[])
+{
+    int optindex;
+    struct top *top;
+    int reactor_flags = 0;
+    const char *target = NULL;
+    optparse_t *opts;
+    flux_error_t error;
+
+    setlocale (LC_ALL, "");
+
+    if (!(opts = optparse_create ("flux-top"))
+        || optparse_add_option_table (opts, cmdopts) != OPTPARSE_SUCCESS
+        || optparse_set (opts,
+                         OPTPARSE_USAGE,
+                         usage_msg) != OPTPARSE_SUCCESS)
+        fatal (0, "error setting up option parsing");
+
+    if ((optindex = optparse_parse_args (opts, argc, argv)) < 0)
+        exit (1);
+    if (optindex < argc)
+        target = argv[optindex++];
+    if (optindex != argc) {
+        optparse_print_usage (opts);
+        exit (1);
+    }
+    if (!isatty (STDIN_FILENO))
+        fatal (0, "stdin is not a terminal");
+    initialize_curses (color_optparse (opts));
+
+    if (!(top = top_create (target,
+                            NULL,
+                            optparse_get_str (opts, "queue", NULL),
+                            &error)))
+        fatal (0, "%s", error.text);
+    /* top_create() call above will call initial "draw" routines.  We
+     * do not want those draws to output anything during testing with
+     * the --test-exit-dump option.  Thus we handle --test-exit and
+     * --test-exit-dump setup after the top_create() call.
+     */
+    if (optparse_hasopt (opts, "test-exit")) {
+        const char *file;
+        top->test_exit = 1;
+        if ((file = optparse_get_str (opts, "test-exit-dump", NULL))) {
+            mode_t umask_orig = umask (022);
+            if (!(top->testf = fopen (file, "w+")))
+                fatal (errno, "failed to open test dump file");
+            umask (umask_orig);
+        }
+    }
+    if (top_run (top, reactor_flags) < 0)
+        fatal (errno, "reactor loop unexpectedly terminated");
+
+    if (top->test_exit) {
+        curs_set (1); // restore cursor
+        reset_shell_mode ();
+    }
+    else
+        endwin ();
+    top_destroy (top);
+    optparse_destroy (opts);
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/cmd/top/top.h b/src/cmd/top/top.h
new file mode 100644
index 000000000000..b8645a08195b
--- /dev/null
+++ b/src/cmd/top/top.h
@@ -0,0 +1,105 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#include <sys/types.h>
+#include <curses.h>
+#include <stdarg.h>
+#include <flux/core.h>
+#include <flux/optparse.h>
+#include <jansson.h>
+
+// set printf format attribute on this curses function for our convenience
+int mvwprintw(WINDOW *win, int y, int x, const char *fmt, ...)
+    __attribute__ ((format (printf, 4, 5)));
+
+enum {
+    TOP_COLOR_YELLOW = 1, // origin of 1 since 0 is an invalid color pair index
+    TOP_COLOR_RED,
+    TOP_COLOR_GREEN,
+    TOP_COLOR_BLUE,
+    TOP_COLOR_BLUE_HIGHLIGHT,
+};
+
+struct top {
+    flux_t *h;
+    char *title;
+    json_t *flux_config;
+    struct queues *queues;
+    flux_jobid_t id;
+
+    unsigned int test_exit:1;    /*  Exit after first output of all panes */
+    unsigned int test_exit_count;
+    FILE *testf;
+    const char *f_char;
+
+    uint32_t size;
+    struct summary_pane *summary_pane;
+    struct joblist_pane *joblist_pane;
+    struct keys *keys;
+    flux_watcher_t *refresh;
+    flux_watcher_t *jobtimer;
+    bool jobtimer_running;
+    flux_msg_handler_t **handlers;
+    flux_future_t *f_stats;
+};
+
+struct dimension {
+    int x_begin;
+    int y_begin;
+    int x_length;
+    int y_length;
+};
+
+struct top *top_create (const char *uri,
+                        const char *prefix,
+                        const char *queue,
+                        flux_error_t *errp);
+void top_destroy (struct top *top);
+int top_run (struct top *top, int reactor_flags);
+void test_exit_check (struct top *top);
+
+struct summary_pane *summary_pane_create (struct top *top);
+void summary_pane_destroy (struct summary_pane *sum);
+void summary_pane_draw (struct summary_pane *sum);
+void summary_pane_refresh (struct summary_pane *sum);
+void summary_pane_query (struct summary_pane *sum);
+void summary_pane_heartbeat (struct summary_pane *sum);
+void summary_pane_jobstats (struct summary_pane *sum, flux_future_t *f);
+void summary_pane_toggle_details (struct summary_pane *sum);
+
+struct joblist_pane *joblist_pane_create (struct top *top);
+void joblist_pane_destroy (struct joblist_pane *joblist);
+void joblist_pane_draw (struct joblist_pane *joblist);
+void joblist_pane_refresh (struct joblist_pane *joblist);
+void joblist_pane_query (struct joblist_pane *joblist);
+void joblist_pane_set_current (struct joblist_pane *joblist, bool next);
+void joblist_pane_enter (struct joblist_pane *joblist);
+void joblist_filter_jobs (struct joblist_pane *joblist);
+
+struct keys *keys_create (struct top *top);
+void keys_destroy (struct keys *keys);
+
+struct ucache *ucache_create (void);
+void ucache_destroy (struct ucache *ucache);
+const char *ucache_lookup (struct ucache *ucache, uid_t userid);
+
+void queues_destroy (struct queues *queues);
+struct queues *queues_create (json_t *flux_config);
+bool queues_configured (struct queues *queues);
+void queues_set_queue (struct queues *queues, const char *name);
+void queues_next (struct queues *queues);
+void queues_prev (struct queues *queues);
+void queues_get_queue_name (struct queues *queues, const char **name);
+void queues_get_queue_constraint (struct queues *queues, json_t **constraint);
+
+void fatal (int errnum, const char *fmt, ...)
+    __attribute__ ((format (printf, 2, 3)));
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/cmd/top/ucache.c b/src/cmd/top/ucache.c
new file mode 100644
index 000000000000..149275b3315a
--- /dev/null
+++ b/src/cmd/top/ucache.c
@@ -0,0 +1,91 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* ucache.c - simple username cache
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <stdlib.h>
+#include <pwd.h>
+#include <stdio.h>
+
+#include "top.h"
+
+struct ucache_entry {
+    uid_t id;
+    char name[9];
+};
+
+struct ucache {
+    struct ucache_entry *users;
+    size_t len;
+};
+
+/* Add a new entry to the end of the cache.
+ * If memory allocation fails, return NULL with errno set.
+ */
+static const char *ucache_add (struct ucache *ucache,
+                               uid_t userid,
+                               const char *name)
+{
+    struct ucache_entry *new_users;
+    struct ucache_entry *entry;
+
+    if (!(new_users = realloc (ucache->users,
+                               sizeof (ucache->users[0]) * (ucache->len + 1))))
+        return NULL;
+    ucache->users = new_users;
+    entry = &ucache->users[ucache->len++];
+
+    entry->id = userid;
+    snprintf (entry->name, sizeof (entry->name), "%s", name);
+    return entry->name;
+}
+
+/* Find userid in cache and return username.
+ * If not found, look up in password file, add to cache, and return username.
+ * If memory allocation or user lookup fails, return NULL with errno set.
+ */
+const char *ucache_lookup (struct ucache *ucache, uid_t userid)
+{
+    struct passwd *pwd;
+
+    for (int i = 0; i < ucache->len; i++) {
+        if (ucache->users[i].id == userid)
+            return ucache->users[i].name;
+    }
+    if (!(pwd = getpwuid (userid)))
+        return NULL;
+    return ucache_add (ucache, userid, pwd->pw_name);
+}
+
+void ucache_destroy (struct ucache *ucache)
+{
+    if (ucache) {
+        int saved_errno = errno;
+        free (ucache->users);
+        free (ucache);
+        errno = saved_errno;
+    }
+}
+
+struct ucache *ucache_create (void)
+{
+    struct ucache *ucache;
+
+    if (!(ucache = calloc (1, sizeof (*ucache))))
+        return NULL;
+    return ucache;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/Makefile.am b/src/common/Makefile.am
index acf9e20c1e65..3637c9b36e21 100644
--- a/src/common/Makefile.am
+++ b/src/common/Makefile.am
@@ -1,33 +1,48 @@
-SUBDIRS = libtap \
-	  libtestutil \
-	  libev \
-	  libyuarel \
-	  libpmi \
-	  liblsd \
-	  libutil \
-	  libflux \
-	  libkvs \
-	  libjob \
-	  liboptparse \
-	  libidset \
-	  libtomlc99 \
-          libsubprocess \
-          libaggregate \
-          libschedutil \
-	  libeventlog \
-	  libioencode \
-	  librouter
+SUBDIRS = \
+	libtap \
+	libtestutil \
+	libev \
+	libyuarel \
+	libpmi \
+	liblsd \
+	libutil \
+	libflux \
+	libfluxutil \
+	libkvs \
+	libjob \
+	liboptparse \
+	libidset \
+	libtomlc99 \
+	libsubprocess \
+	libschedutil \
+	libeventlog \
+	libioencode \
+	librouter \
+	libdebugged \
+	libterminus \
+	libcontent \
+	libhostlist \
+	librlist \
+	libczmqcontainers \
+	libccan \
+	libzmqutil \
+	libtaskmap \
+	libfilemap \
+	libsdexec \
+	libmissing
 
 AM_CFLAGS = $(WARNING_CFLAGS) $(CODE_COVERAGE_CFLAGS)
 AM_LDFLAGS = $(CODE_COVERAGE_LIBS)
 
-AM_CPPFLAGS =
+AM_CPPFLAGS = $(CODE_COVERAGE_CPPFLAGS)
 
 fluxinclude_HEADERS = core.h schedutil.h
 noinst_LTLIBRARIES = libflux-internal.la
 libflux_internal_la_SOURCES =
 libflux_internal_la_LIBADD = \
+	$(builddir)/libfluxutil/libfluxutil.la \
 	$(builddir)/liblsd/liblsd.la \
+	$(builddir)/libccan/libccan.la \
 	$(builddir)/libutil/libutil.la \
 	$(builddir)/libidset/libidset.la \
 	$(builddir)/libev/libev.la \
@@ -36,23 +51,44 @@ libflux_internal_la_LIBADD = \
 	$(builddir)/libeventlog/libeventlog.la \
 	$(builddir)/libioencode/libioencode.la \
 	$(builddir)/librouter/librouter.la \
+	$(builddir)/libhostlist/libhostlist.la \
+	$(builddir)/libczmqcontainers/libczmqcontainers.la \
+	$(builddir)/libcontent/libcontent.la \
+	$(builddir)/libtaskmap/libtaskmap.la \
+	$(builddir)/libmissing/libmissing.la \
 	$(JANSSON_LIBS) \
-	$(ZMQ_LIBS) \
 	$(LIBUUID_LIBS) \
 	$(LIBPTHREAD) \
-	$(LIBUTIL) \
 	$(LIBDL) \
 	$(LIBRT) \
-	$(FLUX_SECURITY_LIBS) \
-	$(LIBSODIUM_LIBS)
+	$(FLUX_SECURITY_LIBS)
 libflux_internal_la_LDFLAGS = $(san_ld_zdef_flag)
 
 lib_LTLIBRARIES = libflux-core.la \
 	libflux-optparse.la \
 	libflux-idset.la \
-	libflux-schedutil.la
+	libflux-schedutil.la \
+	libflux-hostlist.la \
+	libflux-taskmap.la
 
-fluxlib_LTLIBRARIES = libpmi.la libpmi2.la
+fluxlib_LTLIBRARIES = \
+	flux/libpmi.la \
+	flux/libpmi2.la
+
+#  With make -j, libpmi*.so will be installed in parallel with dependencies
+#  (libflux-taskmap.so and libflux-idset.so), so one or both .la files may
+#  not be created by the time they are needed for the libpmi*.so relink,
+#  resulting in an error like
+#
+#    /usr/bin/ld: cannot find -lflux-taskmap
+#
+#  This workaround forces the install of fluxlib_LTLIBRARIES after all
+#  lib_LTLIBRARIES.
+#
+#  ref: https://debbugs.gnu.org/cgi/bugreport.cgi?bug=7328
+#
+install_fluxlibLTLIBRARIES = install-fluxlibLTLIBRARIES
+$(install_fluxlibLTLIBRARIES): install-libLTLIBRARIES
 
 libflux_core_la_SOURCES =
 libflux_core_la_LIBADD = \
@@ -62,70 +98,110 @@ libflux_core_la_LIBADD = \
 	$(builddir)/libsubprocess/libsubprocess.la \
 	libflux-internal.la
 libflux_core_la_LDFLAGS = \
-        -Wl,--version-script=$(srcdir)/libflux-core.map \
+	-export-symbols-regex "^(flux_|__asan)" \
 	-version-info @LIBFLUX_CORE_VERSION_INFO@ \
-        -shared -export-dynamic --disable-static \
-        $(san_ld_zdef_flag)
+	-shared -export-dynamic --disable-static \
+	$(ld_gc_sections) \
+	$(san_ld_zdef_flag)
 
 libflux_optparse_la_SOURCES =
 libflux_optparse_la_LIBADD = \
 	$(builddir)/liboptparse/liboptparse.la \
+	$(builddir)/libczmqcontainers/libczmqcontainers.la \
 	$(builddir)/liblsd/liblsd.la \
 	$(builddir)/libutil/fsd.lo \
-	$(ZMQ_LIBS) $(LIBUUID_LIBS) $(LIBPTHREAD)
+	$(builddir)/libutil/parse_size.lo \
+	$(builddir)/libutil/fdutils.lo \
+	$(builddir)/libmissing/libmissing.la \
+	$(LIBPTHREAD)
 libflux_optparse_la_LDFLAGS = \
-        -Wl,--version-script=$(srcdir)/libflux-optparse.map \
+	-export-symbols-regex "^(optparse_|__asan)" \
 	-version-info @LIBFLUX_OPTPARSE_VERSION_INFO@ \
 	-shared -export-dynamic --disable-static \
+	$(ld_gc_sections) \
 	$(san_ld_zdef_flag)
 
 libflux_idset_la_SOURCES =
 libflux_idset_la_LIBADD = \
-	$(builddir)/libidset/libidset.la \
-	$(builddir)/libutil/veb.lo
+	$(builddir)/libidset/libidset.la
 libflux_idset_la_LDFLAGS = \
-        -Wl,--version-script=$(srcdir)/libflux-idset.map \
+	-export-symbols-regex "^(idset_|__asan)" \
 	-version-info @LIBFLUX_IDSET_VERSION_INFO@ \
 	-shared -export-dynamic --disable-static \
+	$(ld_gc_sections) \
 	$(san_ld_zdef_flag)
 
 libflux_schedutil_la_SOURCES =
 libflux_schedutil_la_LIBADD = \
 	$(builddir)/libschedutil/libschedutil.la \
+	$(builddir)/libczmqcontainers/libczmqcontainers.la \
 	libflux-core.la \
 	$(JANSSON_LIBS)
 libflux_schedutil_la_LDFLAGS = \
-        -Wl,--version-script=$(srcdir)/libflux-schedutil.map \
+	-export-symbols-regex "^(schedutil_|__asan)" \
 	-version-info @LIBFLUX_SCHEDUTIL_VERSION_INFO@ \
 	-shared -export-dynamic --disable-static \
+	$(ld_gc_sections) \
 	$(san_ld_zdef_flag)
 
-libpmi_la_SOURCES =
-libpmi_la_LIBADD = \
+libflux_hostlist_la_SOURCES =
+libflux_hostlist_la_LIBADD = \
+	$(builddir)/libhostlist/libhostlist.la
+libflux_hostlist_la_LDFLAGS = \
+	-export-symbols-regex "^(hostlist_|__asan)" \
+	-version-info @LIBFLUX_HOSTLIST_VERSION_INFO@ \
+	-shared -export-dynamic --disable-static \
+	$(ld_gc_sections) \
+	$(san_ld_zdef_flag)
+
+libflux_taskmap_la_SOURCES =
+libflux_taskmap_la_LIBADD = \
+	$(builddir)/libtaskmap/libtaskmap.la \
+	$(builddir)/libczmqcontainers/libczmqcontainers.la  \
+	$(builddir)/libutil/libutil.la  \
+	$(builddir)/libccan/libccan.la  \
+	$(builddir)/libtomlc99/libtomlc99.la  \
+	$(builddir)/libyuarel/libyuarel.la  \
+	$(builddir)/libmissing/libmissing.la  \
+	libflux-core.la \
+	libflux-idset.la \
+	$(JANSSON_LIBS)
+libflux_taskmap_la_LDFLAGS = \
+	-export-symbols-regex "^(taskmap_|__asan)" \
+	-version-info @LIBFLUX_TASKMAP_VERSION_INFO@ \
+	-shared -export-dynamic --disable-static \
+	$(ld_gc_sections) \
+	$(san_ld_zdef_flag)
+
+flux_libpmi_la_SOURCES =
+flux_libpmi_la_LIBADD = \
+	libflux-taskmap.la \
+	libflux-idset.la \
 	$(builddir)/libpmi/libpmi_client.la \
-	$(builddir)/libutil/aux.lo
-libpmi_la_LDFLAGS = \
-        -Wl,--version-script=$(srcdir)/libpmi.map \
+	$(builddir)/libpmi/libpmi_common.la \
+	$(builddir)/libutil/aux.lo \
+	$(builddir)/libutil/fdutils.lo \
+	$(builddir)/libmissing/libmissing.la
+
+flux_libpmi_la_LDFLAGS = \
+	-export-symbols-regex "^(PMI_|flux_pmi_library|__asan)" \
 	-version-info 0:0:0 \
-	-Wl,--defsym=flux_pmi_library=1 \
 	-shared -export-dynamic --disable-static \
+	$(ld_gc_sections) \
 	$(san_ld_zdef_flag)
 
-libpmi2_la_SOURCES =
-libpmi2_la_LIBADD = \
+flux_libpmi2_la_SOURCES =
+flux_libpmi2_la_LIBADD = \
+	libflux-taskmap.la \
+	libflux-idset.la \
 	$(builddir)/libpmi/libpmi_client.la \
-	$(builddir)/libutil/aux.lo
-libpmi2_la_LDFLAGS = \
-        -Wl,--version-script=$(srcdir)/libpmi2.map \
+	$(builddir)/libpmi/libpmi_common.la \
+	$(builddir)/libutil/aux.lo \
+	$(builddir)/libutil/fdutils.lo \
+	$(builddir)/libmissing/libmissing.la
+flux_libpmi2_la_LDFLAGS = \
+	-export-symbols-regex "^(PMI2_|flux_pmi_library|__asan)" \
 	-version-info 0:0:0 \
-	-Wl,--defsym=flux_pmi_library=1 \
 	-shared -export-dynamic --disable-static \
+	$(ld_gc_sections) \
 	$(san_ld_zdef_flag)
-
-EXTRA_DIST = \
-	libflux-core.map \
-	libflux-optparse.map \
-	libflux-idset.map \
-	libflux-schedutil.map \
-	libpmi.map \
-	libpmi2.map
diff --git a/src/common/core.h b/src/common/core.h
index cdd866372195..32470479bd98 100644
--- a/src/common/core.h
+++ b/src/common/core.h
@@ -18,6 +18,7 @@
 #include "core/flux.h"
 #include "core/kvs.h"
 #include "core/job.h"
+#include "core/jobspec1.h"
 #include "core/subprocess.h"
 
 #endif /* !_FLUX_CORE_H */
diff --git a/src/common/libaggregate/Makefile.am b/src/common/libaggregate/Makefile.am
deleted file mode 100644
index f319a3f15c23..000000000000
--- a/src/common/libaggregate/Makefile.am
+++ /dev/null
@@ -1,18 +0,0 @@
-AM_CFLAGS = \
-        $(WARNING_CFLAGS) \
-        $(CODE_COVERAGE_CFLAGS) \
-        $(ZMQ_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux
-
-noinst_LTLIBRARIES = libaggregate.la
-
-libaggregate_la_SOURCES = \
-	aggregate.h \
-	aggregate.c
diff --git a/src/common/libaggregate/aggregate.c b/src/common/libaggregate/aggregate.c
deleted file mode 100644
index 69c166dc18ff..000000000000
--- a/src/common/libaggregate/aggregate.c
+++ /dev/null
@@ -1,197 +0,0 @@
-/************************************************************\
- * Copyright 2016 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <jansson.h>
-
-#include <flux/core.h>
-
-static void aggregate_wait_set_errnum (flux_future_t *f, int errnum)
-{
-    if (flux_future_aux_set (f, "aggregate::errnum",
-                             (void *) (intptr_t) errnum, NULL) < 0) {
-        /* Not much we can do here but immediately fulfill future and
-         *  hope for the best.
-         */
-        flux_future_fulfill_error (f, errnum, NULL);
-    }
-}
-
-static intptr_t aggregate_wait_get_errnum (flux_future_t *f)
-{
-    void *v = flux_future_aux_get (f, "aggregate::errnum");
-    if (v == NULL)
-        return (0);
-    return ((intptr_t) v);
-}
-
-static void aggregate_fulfill_finalize (flux_future_t *f)
-{
-    intptr_t errnum = aggregate_wait_get_errnum (f);
-    if (errnum)
-        flux_future_fulfill_error (f, (int) errnum, NULL);
-    else
-        flux_future_fulfill (f, NULL, NULL);
-}
-
-static void aggregate_check (flux_future_t *f, void *arg)
-{
-    int count, total;
-    const char *result;
-    json_t *o = NULL;
-    flux_future_t *f_orig = arg;
-
-    if (flux_kvs_lookup_get (f, &result) < 0) {
-        if (errno == ENODATA) {
-            /* flux_kvs_lookup is now canceled, so it is now safe to destroy
-             *  the lookup future and finalize fulfillment of the
-             *  aggregate_wait future.
-             */
-            flux_future_destroy (f);
-            aggregate_fulfill_finalize (f_orig);
-            return;
-        }
-        flux_kvs_lookup_cancel (f);
-        aggregate_wait_set_errnum (f_orig, errno);
-    }
-    else if (!(o = json_loads (result, 0, NULL))
-            || (json_unpack (o, "{s:i,s:i}",
-                                "count", &count,
-                                "total", &total) < 0)) {
-        flux_kvs_lookup_cancel (f);
-        aggregate_wait_set_errnum (f_orig, errno);
-    }
-    else if (count == total) {
-        flux_future_aux_set (f_orig, "aggregate::json_t",
-                             json_incref (o), (flux_free_f) json_decref);
-        flux_future_aux_set (f_orig, "aggregate::json_str",
-                             strdup (result), free);
-        flux_kvs_lookup_cancel (f);
-        /*  f_orig will be fulfilled by aggregate_fulfill_finalize() once
-         *   flux_kvs_lookup_get() returns ENODATA. This ensures there
-         *   are no stray RPC responses and allows safe deletion of the
-         *   kvs lookup future
-         */
-    }
-    flux_future_reset (f);
-    json_decref (o);
-}
-
-static void initialize_cb (flux_future_t *f, void *arg)
-{
-    const char *key = arg;
-    flux_future_t *f2 = NULL;
-    flux_t *h = flux_future_get_flux (f);
-
-    f2 = flux_kvs_lookup (h, NULL, FLUX_KVS_WATCH|FLUX_KVS_WAITCREATE, key);
-    if ((f2 == NULL)
-        || (flux_future_then (f2, -1., aggregate_check, f) < 0)) {
-        flux_future_destroy (f2);
-        flux_future_fulfill_error (f, errno, NULL);
-    }
-}
-
-flux_future_t *aggregate_wait (flux_t *h, const char *key)
-{
-    char *s;
-    flux_future_t *f = flux_future_create (initialize_cb, (void *) key);
-    if ((f == NULL) || !(s = strdup (key))) {
-        flux_future_destroy (f);
-        return NULL;
-    }
-    flux_future_set_flux (f, h);
-    flux_future_aux_set (f, "aggregate::key", s, free);
-    return (f);
-}
-
-int aggregate_wait_get_unpack (flux_future_t *f, const char *fmt, ...)
-{
-    int rc, saved_errno = 0;
-    va_list ap;
-    json_t *o = NULL;
-    if (!(o = flux_future_aux_get (f, "aggregate::json_t")))
-        return -1;
-    va_start (ap, fmt);
-    json_error_t err;
-    rc = json_vunpack_ex (o, &err, 0, fmt, ap);
-    saved_errno = errno;
-    va_end (ap);
-    errno = saved_errno;
-    return rc;
-}
-
-int aggregate_wait_get (flux_future_t *f, const char **strp)
-{
-    if (!(*strp = flux_future_aux_get (f, "aggregate::json_str")))
-        return -1;
-    return (0);
-}
-
-const char *aggregate_wait_get_key (flux_future_t *f)
-{
-    return flux_future_aux_get (f, "aggregate::key");
-}
-
-int aggregate_unpack_to_kvs (flux_future_t *f, const char *path)
-{
-    int errnum = 0;
-    int rc = 0;
-    json_t *entries = NULL;
-    flux_kvs_txn_t *txn = NULL;
-    flux_future_t *fkvs = NULL;
-    flux_t *h = flux_future_get_flux (f);
-
-    if ((aggregate_wait_get_unpack (f, "{s:o}", "entries", &entries) < 0)
-        || !(txn = flux_kvs_txn_create ())
-        || (flux_kvs_txn_pack (txn, 0, path, "O", entries) < 0)
-        || (flux_kvs_txn_unlink (txn, 0, aggregate_wait_get_key (f)) < 0)
-        || !(fkvs = flux_kvs_commit (h, NULL, 0, txn))
-        || (flux_future_get (fkvs, NULL) < 0)) {
-        errnum = errno;
-        rc = -1;
-    }
-    flux_future_destroy (fkvs);
-    flux_kvs_txn_destroy (txn);
-    errno = errnum;
-    return (rc);
-}
-
-flux_future_t *aggregator_push_json (flux_t *h, int fwd_count, double timeout,
-                                     const char *key, json_t *o)
-{
-    uint32_t size;
-    uint32_t rank;
-    int n;
-    char rankstr [16]; /* aggregator expects ranks as string */
-
-    if ((flux_get_size (h, &size) < 0)
-        || (flux_get_rank (h, &rank) < 0)
-        || ((n = snprintf (rankstr, sizeof (rankstr), "%d", rank)) < 0)
-        || (n >= sizeof (rankstr)))
-        return NULL;
-
-    if (timeout >= 0.)
-        return flux_rpc_pack (h, "aggregator.push", FLUX_NODEID_ANY, 0,
-                             "{s:s,s:i,s:i,s:f,s:{s:o}}",
-                             "key", key,
-                             "total", size,
-                             "fwd_count", fwd_count,
-                             "timeout" , timeout,
-                             "entries", rankstr, o);
-    else
-        return flux_rpc_pack (h, "aggregator.push", FLUX_NODEID_ANY, 0,
-                             "{s:s,s:i,s:i,s:{s:o}}",
-                             "key", key,
-                             "total", size,
-                             "fwd_count", fwd_count,
-                             "entries", rankstr, o);
-}
-
-/* vi: ts=4 sw=4 expandtab
- */
diff --git a/src/common/libaggregate/aggregate.h b/src/common/libaggregate/aggregate.h
deleted file mode 100644
index 110deb72411f..000000000000
--- a/src/common/libaggregate/aggregate.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef HAVE_LIBAGGREGATE_AGGREGATE_H
-#define HAVE_LIBAGGREGATE_AGGREGATE_H 1
-
-/*
- *  Push single json object `o` to local aggregator module via RPC.
- *   Steals the reference to `o`. If fwd_count > 0, then set local
- *   forward count in aggregator message. If `t` is non-negative,
- *   then set local forward timeout to this value.
- */
-flux_future_t *aggregator_push_json (flux_t *h, int fwd_count, double t,
-		                     const char *key, json_t *o);
-
-/*  Fulfill future when aggregate at `key` is "complete", i.e.
- *   count == total. Use aggreate_wait_get_unpack () to unpack final
- *   aggregate kvs value after successful fulfillment.
- */
-flux_future_t *aggregate_wait (flux_t *h, const char *key);
-
-/*  Get final aggregate result as string
- */
-int aggregate_wait_get (flux_future_t *f, const char **s);
-
-/*  Get final aggregate JSON object using Jansson json_unpack() format:
- */
-int aggregate_wait_get_unpack (flux_future_t *f, const char *fmt, ...);
-
-/*  Unpack the aggregate fulfilled in `f` into the kvs at path.
- *   Just the aggregate `entries` object is pushed to the new location,
- *   dropping the aggregate context count, total, min, max, etc.
- *
- *  The original aggregate key is removed.
- */
-int aggregate_unpack_to_kvs (flux_future_t *f, const char *path);
-
-#endif /* !HAVE_LIBAGGREGATE_AGGREGATE_H */
diff --git a/src/common/libccan/Makefile.am b/src/common/libccan/Makefile.am
new file mode 100644
index 000000000000..202bf8fc0ef5
--- /dev/null
+++ b/src/common/libccan/Makefile.am
@@ -0,0 +1,65 @@
+AM_CFLAGS = \
+        $(WARNING_CFLAGS) \
+        $(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+        $(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS)
+
+EXTRA_DIST = \
+	licenses \
+	ccan/list/LICENSE \
+	ccan/str/LICENSE \
+	ccan/str/hex/LICENSE \
+	ccan/build_assert/LICENSE \
+	ccan/base64/LICENSE \
+	ccan/container_of/LICENSE \
+	ccan/bitmap/LICENSE \
+	ccan/compiler/LICENSE \
+	ccan/check_type/LICENSE \
+	ccan/array_size/LICENSE \
+	ccan/pushpull/LICENSE \
+	ccan/ptrint/LICENSE \
+	ccan/endian/LICENSE \
+	ccan/list/_info \
+	ccan/str/_info \
+	ccan/str/hex/_info \
+	ccan/build_assert/_info \
+	ccan/base64/_info \
+	ccan/container_of/_info \
+	ccan/bitmap/_info \
+	ccan/compiler/_info \
+	ccan/check_type/_info \
+	ccan/array_size/_info \
+	ccan/pushpull/_info \
+	ccan/ptrint/_info \
+	ccan/endian/_info
+
+noinst_LTLIBRARIES = libccan.la
+libccan_la_SOURCES = \
+	ccan/check_type/check_type.h \
+	ccan/pushpull/pull.c \
+	ccan/pushpull/pull.h \
+	ccan/pushpull/push.c \
+	ccan/pushpull/push.h \
+	ccan/pushpull/pushpull.h \
+	ccan/str/str.c \
+	ccan/str/debug.c \
+	ccan/str/str.h \
+	ccan/str/str_debug.h \
+	ccan/str/hex/hex.h \
+	ccan/str/hex/hex.c \
+	ccan/list/list.c \
+	ccan/list/list.h \
+	ccan/base64/base64.c \
+	ccan/base64/base64.h \
+	ccan/bitmap/bitmap.h \
+	ccan/bitmap/bitmap.c \
+	ccan/endian/endian.h \
+	ccan/compiler/compiler.h \
+	ccan/ptrint/ptrint.h \
+	ccan/build_assert/build_assert.h \
+	ccan/container_of/container_of.h \
+	ccan/array_size/array_size.h
diff --git a/src/common/libccan/ccan/array_size/LICENSE b/src/common/libccan/ccan/array_size/LICENSE
new file mode 120000
index 000000000000..b7951dabdc82
--- /dev/null
+++ b/src/common/libccan/ccan/array_size/LICENSE
@@ -0,0 +1 @@
+../../licenses/CC0
\ No newline at end of file
diff --git a/src/common/libccan/ccan/array_size/_info b/src/common/libccan/ccan/array_size/_info
new file mode 100644
index 000000000000..69570f34c8dd
--- /dev/null
+++ b/src/common/libccan/ccan/array_size/_info
@@ -0,0 +1,46 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * array_size - routine for safely deriving the size of a visible array.
+ *
+ * This provides a simple ARRAY_SIZE() macro, which (given a good compiler)
+ * will also break compile if you try to use it on a pointer.
+ *
+ * This can ensure your code is robust to changes, without needing a gratuitous
+ * macro or constant.
+ *
+ * Example:
+ *	// Outputs "Initialized 32 values\n"
+ *	#include <ccan/array_size/array_size.h>
+ *	#include <stdlib.h>
+ *	#include <stdio.h>
+ *
+ *	// We currently use 32 random values.
+ *	static unsigned int vals[32];
+ *
+ *	int main(void)
+ *	{
+ *		unsigned int i;
+ *		for (i = 0; i < ARRAY_SIZE(vals); i++)
+ *			vals[i] = random();
+ *		printf("Initialized %u values\n", i);
+ *		return 0;
+ *	}
+ *
+ * License: CC0 (Public domain)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/build_assert\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/array_size/array_size.h b/src/common/libccan/ccan/array_size/array_size.h
new file mode 100644
index 000000000000..0ca422a29168
--- /dev/null
+++ b/src/common/libccan/ccan/array_size/array_size.h
@@ -0,0 +1,26 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#ifndef CCAN_ARRAY_SIZE_H
+#define CCAN_ARRAY_SIZE_H
+#include "config.h"
+#include <ccan/build_assert/build_assert.h>
+
+/**
+ * ARRAY_SIZE - get the number of elements in a visible array
+ * @arr: the array whose size you want.
+ *
+ * This does not work on pointers, or arrays declared as [], or
+ * function parameters.  With correct compiler support, such usage
+ * will cause a build error (see build_assert).
+ */
+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + _array_size_chk(arr))
+
+#if HAVE_BUILTIN_TYPES_COMPATIBLE_P && HAVE_TYPEOF
+/* Two gcc extensions.
+ * &a[0] degrades to a pointer: a different type from an array */
+#define _array_size_chk(arr)						\
+	BUILD_ASSERT_OR_ZERO(!__builtin_types_compatible_p(typeof(arr),	\
+							typeof(&(arr)[0])))
+#else
+#define _array_size_chk(arr) 0
+#endif
+#endif /* CCAN_ALIGNOF_H */
diff --git a/src/common/libccan/ccan/array_size/test/compile_fail-function-param.c b/src/common/libccan/ccan/array_size/test/compile_fail-function-param.c
new file mode 100644
index 000000000000..164c826f5516
--- /dev/null
+++ b/src/common/libccan/ccan/array_size/test/compile_fail-function-param.c
@@ -0,0 +1,24 @@
+#include <ccan/array_size/array_size.h>
+#include <stdlib.h>
+
+struct foo {
+	unsigned int a, b;
+};
+
+int check_parameter(const struct foo *array);
+int check_parameter(const struct foo *array)
+{
+#ifdef FAIL
+	return (ARRAY_SIZE(array) == 4);
+#if !HAVE_TYPEOF || !HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if _array_size_chk is a noop."
+#endif
+#else
+	return sizeof(array) == 4 * sizeof(struct foo);
+#endif
+}
+
+int main(void)
+{
+	return check_parameter(NULL);
+}
diff --git a/src/common/libccan/ccan/array_size/test/compile_fail.c b/src/common/libccan/ccan/array_size/test/compile_fail.c
new file mode 100644
index 000000000000..9bd656826dc7
--- /dev/null
+++ b/src/common/libccan/ccan/array_size/test/compile_fail.c
@@ -0,0 +1,16 @@
+#include <ccan/array_size/array_size.h>
+
+int main(int argc, char *argv[8])
+{
+	(void)argc;
+	(void)argv;
+	char array[100];
+#ifdef FAIL
+	return ARRAY_SIZE(argv) + ARRAY_SIZE(array);
+#if !HAVE_TYPEOF || !HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#error "Unfortunately we don't fail if _array_size_chk is a noop."
+#endif
+#else
+	return ARRAY_SIZE(array);
+#endif
+}
diff --git a/src/common/libccan/ccan/array_size/test/run.c b/src/common/libccan/ccan/array_size/test/run.c
new file mode 100644
index 000000000000..3c4ae1893cca
--- /dev/null
+++ b/src/common/libccan/ccan/array_size/test/run.c
@@ -0,0 +1,33 @@
+#include <ccan/array_size/array_size.h>
+#include <ccan/tap/tap.h>
+
+static char array1[1];
+static int array2[2];
+static unsigned long array3[3][5];
+struct foo {
+	unsigned int a, b;
+	char string[100];
+};
+static struct foo array4[4];
+
+/* Make sure they can be used in initializers. */
+static int array1_size = ARRAY_SIZE(array1);
+static int array2_size = ARRAY_SIZE(array2);
+static int array3_size = ARRAY_SIZE(array3);
+static int array4_size = ARRAY_SIZE(array4);
+
+int main(void)
+{
+	plan_tests(8);
+	ok1(array1_size == 1);
+	ok1(array2_size == 2);
+	ok1(array3_size == 3);
+	ok1(array4_size == 4);
+
+	ok1(ARRAY_SIZE(array1) == 1);
+	ok1(ARRAY_SIZE(array2) == 2);
+	ok1(ARRAY_SIZE(array3) == 3);
+	ok1(ARRAY_SIZE(array4) == 4);
+
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/base64/LICENSE b/src/common/libccan/ccan/base64/LICENSE
new file mode 120000
index 000000000000..2354d12945d3
--- /dev/null
+++ b/src/common/libccan/ccan/base64/LICENSE
@@ -0,0 +1 @@
+../../licenses/BSD-MIT
\ No newline at end of file
diff --git a/src/common/libccan/ccan/base64/_info b/src/common/libccan/ccan/base64/_info
new file mode 100644
index 000000000000..229819902eab
--- /dev/null
+++ b/src/common/libccan/ccan/base64/_info
@@ -0,0 +1,41 @@
+#include "config.h"
+
+/**
+ * base64 - base64 encoding and decoding (rfc4648).
+ *
+ * base64 encoding is used to encode data in a 7-bit clean manner.
+ *  Commonly used for escaping data before encapsulation or transfer
+ *
+ * Example:
+ *      #include <stdio.h>
+ *      #include <string.h>
+ *      #include <ccan/base64/base64.h>
+ *
+ *      int main(int argc, char *argv[])
+ *      {
+ *      	char *base64_encoded_string;
+ *      	int i;
+ *
+ *      	// print the base64-encoded form of the program arguments
+ *      	for(i=1;i<argc;i++) {
+ *			size_t unencoded_length = strlen(argv[i]);
+ *      		size_t encoded_length = base64_encoded_length(unencoded_length);
+ *      		base64_encoded_string = malloc(encoded_length);
+ *      		base64_encode(base64_encoded_string, encoded_length,
+ *				 argv[i], unencoded_length);
+ *      		printf("%s\n", base64_encoded_string);
+ *      		free(base64_encoded_string);
+ *      	}
+ *
+ *      	return 0;
+ *      }
+ *
+ * License: BSD-MIT
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/base64/base64.c b/src/common/libccan/ccan/base64/base64.c
new file mode 100644
index 000000000000..bb60b331171a
--- /dev/null
+++ b/src/common/libccan/ccan/base64/base64.c
@@ -0,0 +1,253 @@
+/* Licensed under BSD-MIT - see LICENSE file for details */
+#include "base64.h"
+
+#include <errno.h>
+#include <string.h>
+#include <assert.h>
+#include <stdint.h>
+
+/**
+ * sixbit_to_b64 - maps a 6-bit value to the base64 alphabet
+ * @param map A base 64 map (see base64_init_map)
+ * @param sixbit Six-bit value to map
+ * @return a base 64 character
+ */
+static char sixbit_to_b64(const base64_maps_t *maps, const uint8_t sixbit)
+{
+	assert(sixbit <= 63);
+
+	return maps->encode_map[(unsigned char)sixbit];
+}
+
+/**
+ * sixbit_from_b64 - maps a base64-alphabet character to its 6-bit value
+ * @param maps A base 64 maps structure (see base64_init_maps)
+ * @param sixbit Six-bit value to map
+ * @return a six-bit value
+ */
+static int8_t sixbit_from_b64(const base64_maps_t *maps,
+			      const unsigned char b64letter)
+{
+	int8_t ret;
+
+	ret = maps->decode_map[(unsigned char)b64letter];
+	if (ret == (int8_t)0xff) {
+		errno = EDOM;
+		return -1;
+	}
+
+	return ret;
+}
+
+bool base64_char_in_alphabet(const base64_maps_t *maps, const char b64char)
+{
+	return (maps->decode_map[(const unsigned char)b64char] != (int8_t)0xff);
+}
+
+void base64_init_maps(base64_maps_t *dest, const char src[64])
+{
+	unsigned char i;
+
+	memcpy(dest->encode_map,src,64);
+	memset(dest->decode_map,0xff,256);
+	for (i=0; i<64; i++) {
+		dest->decode_map[(unsigned char)src[i]] = i;
+	}
+}
+
+size_t base64_encoded_length(size_t srclen)
+{
+	return ((srclen + 2) / 3) * 4;
+}
+
+void base64_encode_triplet_using_maps(const base64_maps_t *maps,
+				      char dest[4], const char src[3])
+{
+	char a = src[0];
+	char b = src[1];
+	char c = src[2];
+
+	dest[0] = sixbit_to_b64(maps, (a & 0xfc) >> 2);
+	dest[1] = sixbit_to_b64(maps, ((a & 0x3) << 4) | ((b & 0xf0) >> 4));
+	dest[2] = sixbit_to_b64(maps, ((c & 0xc0) >> 6) | ((b & 0xf) << 2));
+	dest[3] = sixbit_to_b64(maps, c & 0x3f);
+}
+
+void base64_encode_tail_using_maps(const base64_maps_t *maps, char dest[4],
+				   const char *src, const size_t srclen)
+{
+	char longsrc[3] = { 0 };
+
+	assert(srclen <= 3);
+
+	memcpy(longsrc, src, srclen);
+	base64_encode_triplet_using_maps(maps, dest, longsrc);
+	memset(dest+1+srclen, '=', 3-srclen);
+}
+
+ssize_t base64_encode_using_maps(const base64_maps_t *maps,
+				 char *dest, const size_t destlen,
+				 const char *src, const size_t srclen)
+{
+	size_t src_offset = 0;
+	size_t dest_offset = 0;
+
+	if (destlen < base64_encoded_length(srclen)) {
+		errno = EOVERFLOW;
+		return -1;
+	}
+
+	while (srclen - src_offset >= 3) {
+		base64_encode_triplet_using_maps(maps, &dest[dest_offset], &src[src_offset]);
+		src_offset += 3;
+		dest_offset += 4;
+	}
+
+	if (src_offset < srclen) {
+		base64_encode_tail_using_maps(maps, &dest[dest_offset], &src[src_offset], srclen-src_offset);
+		dest_offset += 4;
+	}
+
+	memset(&dest[dest_offset], '\0', destlen-dest_offset);
+
+	return dest_offset;
+}
+
+size_t base64_decoded_length(size_t srclen)
+{
+	return ((srclen+3)/4*3);
+}
+
+ssize_t base64_decode_quartet_using_maps(const base64_maps_t *maps, char dest[3],
+				     const char src[4])
+{
+	signed char a;
+	signed char b;
+	signed char c;
+	signed char d;
+
+	a = sixbit_from_b64(maps, src[0]);
+	b = sixbit_from_b64(maps, src[1]);
+	c = sixbit_from_b64(maps, src[2]);
+	d = sixbit_from_b64(maps, src[3]);
+
+	if ((a == -1) || (b == -1) || (c == -1) || (d == -1)) {
+		return -1;
+	}
+
+	dest[0] = (a << 2) | (b >> 4);
+	dest[1] = ((b & 0xf) << 4) | (c >> 2);
+	dest[2] = ((c & 0x3) << 6) | d;
+
+	return 0;
+}
+
+
+ssize_t base64_decode_tail_using_maps(const base64_maps_t *maps, char dest[3],
+				  const char * src, const size_t srclen)
+{
+	char longsrc[4];
+	int quartet_result;
+	size_t insize = srclen;
+
+	while (insize != 0 &&
+	       src[insize-1] == '=') { /* throw away padding symbols */
+		insize--;
+	}
+	if (insize == 0) {
+		return 0;
+	}
+	if (insize == 1) {
+		/* the input is malformed.... */
+		errno = EINVAL;
+		return -1;
+	}
+	memcpy(longsrc, src, insize);
+	memset(longsrc+insize, 'A', 4-insize);
+	quartet_result = base64_decode_quartet_using_maps(maps, dest, longsrc);
+	if (quartet_result == -1) {
+		return -1;
+	}
+
+	return insize - 1;
+}
+
+ssize_t base64_decode_using_maps(const base64_maps_t *maps,
+				 char *dest, const size_t destlen,
+				 const char *src, const size_t srclen)
+{
+	ssize_t dest_offset = 0;
+	ssize_t i;
+	ssize_t more;
+
+	if (destlen < base64_decoded_length(srclen)) {
+		errno = EOVERFLOW;
+		return -1;
+	}
+
+	for(i=0; srclen - i > 4; i+=4) {
+		if (base64_decode_quartet_using_maps(maps, &dest[dest_offset], &src[i]) == -1) {
+			return -1;
+		}
+		dest_offset += 3;
+	}
+
+	more = base64_decode_tail_using_maps(maps, &dest[dest_offset], &src[i], srclen - i);
+	if (more == -1) {
+		return -1;
+	}
+	dest_offset += more;
+
+	memset(&dest[dest_offset], '\0', destlen-dest_offset);
+
+	return dest_offset;
+}
+
+
+
+
+/**
+ * base64_maps_rfc4648 - pregenerated maps struct for rfc4648
+ */
+const base64_maps_t base64_maps_rfc4648 = {
+  "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",
+
+  "\xff\xff\xff\xff\xff" /* 0 */					\
+  "\xff\xff\xff\xff\xff" /* 5 */					\
+  "\xff\xff\xff\xff\xff" /* 10 */					\
+  "\xff\xff\xff\xff\xff" /* 15 */					\
+  "\xff\xff\xff\xff\xff" /* 20 */					\
+  "\xff\xff\xff\xff\xff" /* 25 */					\
+  "\xff\xff\xff\xff\xff" /* 30 */					\
+  "\xff\xff\xff\xff\xff" /* 35 */					\
+  "\xff\xff\xff\x3e\xff" /* 40 */					\
+  "\xff\xff\x3f\x34\x35" /* 45 */					\
+  "\x36\x37\x38\x39\x3a" /* 50 */					\
+  "\x3b\x3c\x3d\xff\xff" /* 55 */					\
+  "\xff\xff\xff\xff\xff" /* 60 */					\
+  "\x00\x01\x02\x03\x04" /* 65 A */					\
+  "\x05\x06\x07\x08\x09" /* 70 */					\
+  "\x0a\x0b\x0c\x0d\x0e" /* 75 */					\
+  "\x0f\x10\x11\x12\x13" /* 80 */					\
+  "\x14\x15\x16\x17\x18" /* 85 */					\
+  "\x19\xff\xff\xff\xff" /* 90 */					\
+  "\xff\xff\x1a\x1b\x1c" /* 95 */					\
+  "\x1d\x1e\x1f\x20\x21" /* 100 */					\
+  "\x22\x23\x24\x25\x26" /* 105 */					\
+  "\x27\x28\x29\x2a\x2b" /* 110 */					\
+  "\x2c\x2d\x2e\x2f\x30" /* 115 */					\
+  "\x31\x32\x33\xff\xff" /* 120 */					\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 125 */			\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 155 */			\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 185 */			\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 215 */			\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
+  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 245 */
+};
diff --git a/src/common/libccan/ccan/base64/base64.h b/src/common/libccan/ccan/base64/base64.h
new file mode 100644
index 000000000000..a899af4a357f
--- /dev/null
+++ b/src/common/libccan/ccan/base64/base64.h
@@ -0,0 +1,241 @@
+/* Licensed under BSD-MIT - see LICENSE file for details */
+#ifndef CCAN_BASE64_H
+#define CCAN_BASE64_H
+
+#include <stddef.h>
+#include <stdbool.h>
+#include <sys/types.h>
+
+/**
+ * base64_maps_t - structure to hold maps for encode/decode
+ */
+typedef struct {
+	char encode_map[64];
+	signed char decode_map[256];
+} base64_maps_t;
+
+/**
+ * base64_encoded_length - Calculate encode buffer length
+ * @param srclen the size of the data to be encoded
+ * @note add 1 to this to get null-termination
+ * @return Buffer length required for encode
+ */
+size_t base64_encoded_length(size_t srclen);
+
+/**
+ * base64_decoded_length - Calculate decode buffer length
+ * @param srclen Length of the data to be decoded
+ * @note This does not return the size of the decoded data!  see base64_decode
+ * @return Minimum buffer length for safe decode
+ */
+size_t base64_decoded_length(size_t srclen);
+
+/**
+ * base64_init_maps - populate a base64_maps_t based on a supplied alphabet
+ * @param dest A base64 maps object
+ * @param src Alphabet to populate the maps from (e.g. base64_alphabet_rfc4648)
+ */
+void base64_init_maps(base64_maps_t *dest, const char src[64]);
+
+
+/**
+ * base64_encode_triplet_using_maps - encode 3 bytes into base64 using a specific alphabet
+ * @param maps Maps to use for encoding (see base64_init_maps)
+ * @param dest Buffer containing 3 bytes
+ * @param src Buffer containing 4 characters
+ */
+void base64_encode_triplet_using_maps(const base64_maps_t *maps,
+				      char dest[4], const char src[3]);
+
+/**
+ * base64_encode_tail_using_maps - encode the final bytes of a source using a specific alphabet
+ * @param maps Maps to use for encoding (see base64_init_maps)
+ * @param dest Buffer containing 4 bytes
+ * @param src Buffer containing srclen bytes
+ * @param srclen Number of bytes (<= 3) to encode in src
+ */
+void base64_encode_tail_using_maps(const base64_maps_t *maps, char dest[4],
+				   const char *src, size_t srclen);
+
+/**
+ * base64_encode_using_maps - encode a buffer into base64 using a specific alphabet
+ * @param maps Maps to use for encoding (see base64_init_maps)
+ * @param dest Buffer to encode into
+ * @param destlen Length of dest
+ * @param src Buffer to encode
+ * @param srclen Length of the data to encode
+ * @return Number of encoded bytes set in dest. -1 on error (and errno set)
+ * @note dest will be nul-padded to destlen (past any required padding)
+ * @note sets errno = EOVERFLOW if destlen is too small
+ */
+ssize_t base64_encode_using_maps(const base64_maps_t *maps,
+				 char *dest, size_t destlen,
+				 const char *src, size_t srclen);
+
+/*
+ * base64_char_in_alphabet - returns true if character can be part of an encoded string
+ * @param maps A base64 maps object (see base64_init_maps)
+ * @param b64char Character to check
+ */
+bool base64_char_in_alphabet(const base64_maps_t *maps, char b64char);
+
+/**
+ * base64_decode_using_maps - decode a base64-encoded string using a specific alphabet
+ * @param maps A base64 maps object (see base64_init_maps)
+ * @param dest Buffer to decode into
+ * @param destlen length of dest
+ * @param src the buffer to decode
+ * @param srclen the length of the data to decode
+ * @return Number of decoded bytes set in dest. -1 on error (and errno set)
+ * @note dest will be nul-padded to destlen
+ * @note sets errno = EOVERFLOW if destlen is too small
+ * @note sets errno = EDOM if src contains invalid characters
+ */
+ssize_t base64_decode_using_maps(const base64_maps_t *maps,
+				 char *dest, size_t destlen,
+				 const char *src, size_t srclen);
+
+/**
+ * base64_decode_quartet_using_maps - decode 4 bytes from base64 using a specific alphabet
+ * @param maps A base64 maps object (see base64_init_maps)
+ * @param dest Buffer containing 3 bytes
+ * @param src Buffer containing 4 bytes
+ * @return Number of decoded bytes set in dest. -1 on error (and errno set)
+ * @note sets errno = EDOM if src contains invalid characters
+ */
+ssize_t base64_decode_quartet_using_maps(const base64_maps_t *maps,
+				         char dest[3], const char src[4]);
+
+/**
+ * base64_decode_tail_using_maps - decode the final bytes of a base64 string using a specific alphabet
+ * @param maps A base64 maps object (see base64_init_maps)
+ * @param dest Buffer containing 3 bytes
+ * @param src Buffer containing 4 bytes - padded with '=' as required
+ * @param srclen Number of bytes to decode in src
+ * @return Number of decoded bytes set in dest. -1 on error (and errno set)
+ * @note sets errno = EDOM if src contains invalid characters
+ * @note sets errno = EINVAL if src is an invalid base64 tail
+ */
+ssize_t base64_decode_tail_using_maps(const base64_maps_t *maps, char dest[3],
+				      const char *src, size_t srclen);
+
+
+/* the rfc4648 functions: */
+
+extern const base64_maps_t base64_maps_rfc4648;
+
+/**
+ * base64_encode - Encode a buffer into base64 according to rfc4648
+ * @param dest Buffer to encode into
+ * @param destlen Length of the destination buffer
+ * @param src Buffer to encode
+ * @param srclen Length of the data to encode
+ * @return Number of encoded bytes set in dest. -1 on error (and errno set)
+ * @note dest will be nul-padded to destlen (past any required padding)
+ * @note sets errno = EOVERFLOW if destlen is too small
+ *
+ * This function encodes src according to http://tools.ietf.org/html/rfc4648
+ *
+ * Example:
+ *	size_t encoded_length;
+ *	char dest[100];
+ *	const char *src = "This string gets encoded";
+ *	encoded_length = base64_encode(dest, sizeof(dest), src, strlen(src));
+ *	printf("Returned data of length %zd @%p\n", encoded_length, &dest);
+ */
+static inline
+ssize_t base64_encode(char *dest, size_t destlen,
+		      const char *src, size_t srclen)
+{
+	return base64_encode_using_maps(&base64_maps_rfc4648,
+					dest, destlen, src, srclen);
+}
+
+/**
+ * base64_encode_triplet - encode 3 bytes into base64 according to rfc4648
+ * @param dest Buffer containing 4 bytes
+ * @param src Buffer containing 3 bytes
+ */
+static inline
+void base64_encode_triplet(char dest[4], const char src[3])
+{
+	base64_encode_triplet_using_maps(&base64_maps_rfc4648, dest, src);
+}
+
+/**
+ * base64_encode_tail - encode the final bytes of a source according to rfc4648
+ * @param dest Buffer containing 4 bytes
+ * @param src Buffer containing srclen bytes
+ * @param srclen Number of bytes (<= 3) to encode in src
+ */
+static inline
+void base64_encode_tail(char dest[4], const char *src, size_t srclen)
+{
+	base64_encode_tail_using_maps(&base64_maps_rfc4648, dest, src, srclen);
+}
+
+
+/**
+ * base64_decode - decode An rfc4648 base64-encoded string
+ * @param dest Buffer to decode into
+ * @param destlen Length of the destination buffer
+ * @param src Buffer to decode
+ * @param srclen Length of the data to decode
+ * @return Number of decoded bytes set in dest. -1 on error (and errno set)
+ * @note dest will be nul-padded to destlen
+ * @note sets errno = EOVERFLOW if destlen is too small
+ * @note sets errno = EDOM if src contains invalid characters
+ *
+ * This function decodes the buffer according to
+ * http://tools.ietf.org/html/rfc4648
+ *
+ * Example:
+ *	size_t decoded_length;
+ *	char ret[100];
+ *	const char *src = "Zm9vYmFyYmF6";
+ *	decoded_length = base64_decode(ret, sizeof(ret), src, strlen(src));
+ *	printf("Returned data of length %zd @%p\n", decoded_length, &ret);
+ */
+static inline
+ssize_t base64_decode(char *dest, size_t destlen,
+		      const char *src, size_t srclen)
+{
+	return base64_decode_using_maps(&base64_maps_rfc4648,
+					dest, destlen, src, srclen);
+}
+
+/**
+ * base64_decode_quartet - decode the first 4 characters in src into dest
+ * @param dest Buffer containing 3 bytes
+ * @param src Buffer containing 4 characters
+ * @return Number of decoded bytes set in dest. -1 on error (and errno set)
+ * @note sets errno = EDOM if src contains invalid characters
+ */
+static inline
+ssize_t base64_decode_quartet(char dest[3], const char src[4])
+{
+	return base64_decode_quartet_using_maps(&base64_maps_rfc4648,
+						dest, src);
+}
+
+/**
+ * @brief decode the final bytes of a base64 string from src into dest
+ * @param dest Buffer containing 3 bytes
+ * @param src Buffer containing 4 bytes - padded with '=' as required
+ * @param srclen Number of bytes to decode in src
+ * @return Number of decoded bytes set in dest. -1 on error (and errno set)
+ * @note sets errno = EDOM if src contains invalid characters
+ * @note sets errno = EINVAL if src is an invalid base64 tail
+ */
+static inline
+ssize_t base64_decode_tail(char dest[3], const char *src, size_t srclen)
+{
+	return base64_decode_tail_using_maps(&base64_maps_rfc4648,
+					     dest, src, srclen);
+}
+
+/* end rfc4648 functions */
+
+
+
+#endif /* CCAN_BASE64_H */
diff --git a/src/common/libccan/ccan/base64/test/moretap.h b/src/common/libccan/ccan/base64/test/moretap.h
new file mode 100644
index 000000000000..114445c3c4b5
--- /dev/null
+++ b/src/common/libccan/ccan/base64/test/moretap.h
@@ -0,0 +1,96 @@
+#ifndef _BASE64_MORETAP_H
+#define _BASE64_MORETAP_H
+
+#include <ccan/str/str.h>
+
+/**
+ * is_str - OK if strings are equal
+ * @e1: expression for the variable string
+ * @e2: expression for the expected string
+ *
+ * If the strings are equal, the test passes.
+ *
+ * Example:
+ *     is_str(give_me_a_fred(),"fred");
+ */
+static void _is_str(char *got,const char *expected, const char *got_string, const char *expected_string, const char *func, const char *file, int line) {
+	if (streq(expected,got)) {
+		_gen_result(1, func, file, line,"%s eq %s",
+			    got_string,expected_string);
+	} else {
+		_gen_result(0, func, file, line,"%s eq %s",
+			    got_string,expected_string);
+		diag("Expected: %s",expected);
+		diag("     Got: %s",got);
+	}
+}
+# define is_str(got,expected) _is_str(got,expected,#got,#expected,__func__, __FILE__, __LINE__)
+
+
+/**
+ * is_int - OK if arguments are equal when cast to integers
+ * @e1: expression for the number
+ * @e2: expression for the expected number
+ *
+ * If the numbers are equal, the test passes.
+ *
+ * Example:
+ *     is_int(give_me_17(),17);
+ */
+# define is_int(e1,e2 ...)						\
+  (((int)e1)==((int)e2) ?						\
+   _gen_result(1, __func__, __FILE__, __LINE__,"%s == %s",#e1,#e2) :	\
+   (_gen_result(0, __func__, __FILE__, __LINE__,"%s == %s",#e1,#e2)) || (diag("Expected: %d",e2),diag("     Got: %d",e1),0)) /* diag is void; note commas. */
+
+
+
+/**
+ * is_mem - OK if arguments are identical up to length @e3
+ * @e1: expression for the buffer
+ * @e2: expression for the expected buffer
+ * @e2: length to compare in buffers
+ *
+ * If the buffers are equal up to @e2, the test passes.
+ *
+ * Example:
+ *     is_mem(give_me_foo(),"foo",3);
+ */
+static void _is_mem(const char *got, const char *expected, const size_t len,
+	      const char *got_string, const char *expected_string, const char *len_string,
+	      const char *func, const char *file, int line) {
+	size_t offset = 0;
+
+	for (offset=0; offset<len; offset++) {
+		if (got[offset] != expected[offset]) {
+			_gen_result(0, func, file, line,"%s eq %s",got_string,expected_string);
+			/* diag("Expected: %s",e2); */
+			/* diag("     Got: %s",e1); */
+			diag("Buffers differ at offset %zd (got=0x%02x expected=0x%02x)",
+			     offset,got[offset],expected[offset]);
+			return;
+		}
+	}
+
+	_gen_result(1, __func__, __FILE__, __LINE__,"%s eq %s",
+		    expected_string,got_string);
+}
+# define is_mem(got,expected,len) \
+	_is_mem(got,expected,len,#got,#expected,#len,__func__, __FILE__, __LINE__)
+
+/**
+ * is_size_t - OK if arguments are equal when cast to size_t
+ * @e1: expression for the number
+ * @e2: expression for the expected number
+ *
+ * If the numbers are equal, the test passes.
+ *
+ * Example:
+ *     is_size_t(give_me_17(),17);
+ */
+# define is_size_t(e1,e2 ...)						\
+  ((size_t)(e1)==((size_t)e2) ?						\
+   _gen_result(1, __func__, __FILE__, __LINE__,"%s == %s",#e1,#e2) :	\
+   (_gen_result(0, __func__, __FILE__, __LINE__,			\
+		"%s == %s",#e1,#e2)) || (diag("Expected: %zd",(size_t)e2),diag("     Got: %zd",(size_t)e1),0)) /* diag is void; note commas. */
+
+#endif
diff --git a/src/common/libccan/ccan/base64/test/run.c b/src/common/libccan/ccan/base64/test/run.c
new file mode 100644
index 000000000000..11045c8e704c
--- /dev/null
+++ b/src/common/libccan/ccan/base64/test/run.c
@@ -0,0 +1,359 @@
+/* Start of run.c test */
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <ccan/base64/base64.h>
+#include <ccan/tap/tap.h>
+
+#include <ccan/base64/base64.c>
+#include "moretap.h"
+
+static void * xmalloc(size_t size);
+
+/* not defined in terms of test_encode_using_maps so we cross
+   appropriate paths in library */
+#define test_encode(src,srclen,expected)			\
+	do {							\
+		size_t destlen;					\
+		char * dest;					\
+		destlen = base64_encoded_length(srclen);	\
+		destlen++; /* null termination */		\
+		dest = xmalloc(destlen);			\
+		ok1(base64_encode(dest,destlen,src,srclen) != -1);	\
+		is_str(dest,expected);					\
+		free(dest);						\
+	} while (0)
+
+#define test_encode_using_alphabet(alphastring,src,srclen,expected)	\
+	do {								\
+		size_t destlen;						\
+		char * dest;						\
+		base64_maps_t maps;				\
+		base64_init_maps(&maps,alphastring);		\
+		destlen = base64_encoded_length(srclen);		\
+		destlen++; /* null termination */		\
+		dest = xmalloc(destlen);				\
+		ok1(base64_encode_using_maps(&maps,dest,destlen,src,srclen) != -1); \
+		is_str(dest,expected);					\
+		free(dest);						\
+	} while (0)
+
+/* not defined in terms of test_decode_using_alphabet so we cross
+   appropriate paths in library */
+#define test_decode(src,srclen,expected,expectedlen)			\
+	do {								\
+		size_t destlen;						\
+		size_t bytes_used;					\
+		char * dest;						\
+		destlen = base64_decoded_length(srclen);		\
+		dest = xmalloc(destlen);				\
+		ok1((bytes_used = base64_decode(dest,destlen,src,srclen)) != -1); \
+		is_size_t(bytes_used,expectedlen);			\
+		is_mem(dest,expected,bytes_used);			\
+		free(dest);						\
+	} while (0)
+
+#define test_decode_using_alphabet(alphastring,src,srclen,expected,expectedlen) \
+	do {								\
+		size_t destlen;						\
+		size_t bytes_used;					\
+		char * dest;						\
+		base64_maps_t maps;				\
+									\
+		base64_init_maps(&maps,alphastring);		\
+		destlen = base64_decoded_length(srclen);		\
+		dest = xmalloc(destlen);				\
+		ok1((bytes_used = base64_decode_using_maps(&maps,dest,destlen,src,srclen)) != -1); \
+		is_size_t(bytes_used,expectedlen);			\
+		is_mem(dest,expected,bytes_used);			\
+		free(dest);						\
+	} while (0)
+
+#define check_bad_range_decode(stuff_to_test,stufflen)	\
+do {							\
+	char dest[10];							\
+	errno = 0;							\
+	is_size_t(base64_decode(dest,sizeof(dest),stuff_to_test,(size_t)stufflen), \
+		  (size_t)-1);						\
+	is_int(errno,EDOM);						\
+} while (0)
+
+int
+main(int argc, char *argv[])
+{
+	plan_tests(131);
+
+	is_size_t(base64_encoded_length(0),(size_t)0);
+	is_size_t(base64_encoded_length(1),(size_t)4);
+	is_size_t(base64_encoded_length(2),(size_t)4);
+	is_size_t(base64_encoded_length(3),(size_t)4);
+	is_size_t(base64_encoded_length(512),(size_t)684);
+
+	/* straight from page 11 of http://tools.ietf.org/html/rfc4648 */
+	test_encode("",0,"");
+	test_encode("f",1,"Zg==");
+	test_encode("fo",2,"Zm8=");
+
+	test_encode("foo",3,"Zm9v");
+	test_encode("foob",4,"Zm9vYg==");
+	test_encode("fooba",5,"Zm9vYmE=");
+	test_encode("foobar",6,"Zm9vYmFy");
+
+	/* a few more */
+	test_encode("foobarb",7,"Zm9vYmFyYg==");
+	test_encode("foobarba",8,"Zm9vYmFyYmE=");
+	test_encode("foobarbaz",9,"Zm9vYmFyYmF6");
+
+	test_encode("foobart",7,"Zm9vYmFydA==");
+
+	test_encode("abcdefghijklmnopqrstuvwxyz",26,"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=");
+	test_encode("\x05\x05\x01\x00\x07",5,"BQUBAAc=");
+
+	test_encode("FOO",3,"Rk9P");
+	test_encode("Z",1,"Wg==");
+
+	/* decode testing */
+
+	test_decode("",0,"",0);
+	test_decode("Zg==",4,"f",1);
+	test_decode("Zm8=",4,"fo",2);
+	test_decode("Zm9v",4,"foo",3);
+	test_decode("Zm9vYg==",8,"foob",4);
+	test_decode("Zm9vYmE=",8,"fooba",5);
+	test_decode("Zm9vYmFy",8,"foobar",6);
+	test_decode("Zm9vYmFyYg==",12,"foobarb",7);
+	test_decode("Zm9vYmFyYmE=",12,"foobarba",8);
+	test_decode("Zm9vYmFyYmF6",12,"foobarbaz",9);
+
+	test_decode("Rk9P",4,"FOO",3);
+
+	test_decode("Wg==",4,"Z",1);
+	test_decode("AA==",4,"\0",1);
+	test_decode("AAA=",4,"\0\0",2);
+
+	{
+		const char *binary = "\x01\x00\x03";
+		const size_t binarylen = 3;
+
+		char * decoded;
+		char * encoded;
+		size_t encoded_len;
+		size_t decoded_len;
+		size_t decoded_space_required;
+
+		size_t encoded_space_required = base64_encoded_length(binarylen);
+		encoded_space_required++; /* null termination */
+		encoded = xmalloc(encoded_space_required);
+		encoded_len = base64_encode(encoded,encoded_space_required,binary,binarylen);
+		is_mem(encoded,"AQAD",encoded_len);
+
+		decoded_space_required = base64_decoded_length(encoded_len);
+		decoded = xmalloc(decoded_space_required);
+		decoded_len = base64_decode(decoded,decoded_space_required,encoded,encoded_len);
+		is_size_t(decoded_len,binarylen);
+		is_mem(binary,decoded,decoded_len);
+	}
+
+	/* some expected encode failures: */
+	{
+		size_t destlen = 1;
+		char dest[destlen];
+		errno = 0;
+		is_size_t(base64_encode(dest,destlen,"A",1),(size_t)-1);
+		is_int(errno,EOVERFLOW);
+	}
+
+	/* some expected decode failures: */
+	{
+		base64_maps_t maps;
+		const char * src = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+		base64_init_maps(&maps,src);
+
+		is_int(sixbit_from_b64(&maps,'\xfe'),(signed char)-1);
+		is_int(errno,EDOM);
+	}
+	{
+		size_t destlen = 10;
+		char dest[destlen];
+		errno = 0;
+		is_size_t(base64_decode(dest,destlen,"A",1),(size_t)-1);
+		is_int(errno,EINVAL);
+	}
+	{
+		size_t destlen = 1;
+		char dest[destlen];
+		errno = 0;
+		is_size_t(base64_decode(dest,destlen,"A",1),(size_t)-1);
+		is_int(errno,EOVERFLOW);
+	}
+	{
+		/* (char)1 is not a valid base64 character: */
+		check_bad_range_decode("A\x01",2);
+		/* (char)255 is not a valid base64 character: (char is signed on most platforms, so this is actually < 0 */
+		check_bad_range_decode("\xff""A",2);
+		check_bad_range_decode("A\xff",2);
+		check_bad_range_decode("AA\xff",3);
+		check_bad_range_decode("A\xff""A",3);
+		check_bad_range_decode("\xff""AA",3);
+		check_bad_range_decode("AAA\xff",4);
+		check_bad_range_decode("\xff\x41\x41\x41\x41",5);
+		check_bad_range_decode("A\xff\x41\x41\x41\x41",6);
+		check_bad_range_decode("AA\xff\x41\x41\x41\x41",7);
+		check_bad_range_decode("AAA\xff\x41\x41\x41\x41",8);
+	}
+	/* trigger some failures in the sixbit-to-b64 encoder: */
+	/* this function now aborts rather than returning -1/setting errno */
+	/* { */
+	/* 	is_int(sixbit_to_b64(base64_maps_rfc4648,'\x70'),(char)-1); */
+	/* 	is_int(sixbit_to_b64(base64_maps_rfc4648,'\xff'),(char)-1); */
+	/* } */
+	/* following tests all of the mapping from b64 chars to 6-bit values: */
+	test_decode("//+FwHRSRIsFU2IhAEGD+AMPhOA=",28,"\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0",20);
+	test_encode("\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0",20,"//+FwHRSRIsFU2IhAEGD+AMPhOA=");
+
+
+	/* check the null-padding stuff */
+	{
+		size_t destlen = 8;
+		char dest[destlen];
+		memset(dest,'\1',sizeof(dest));
+		is_size_t(base64_encode(dest,destlen,"A",1),(size_t)4);
+		is_mem(&dest[4],"\0\0\0\0",4);
+	}
+	{
+		size_t destlen = 3;
+		char dest[destlen];
+		memset(dest,'\1',sizeof(dest));
+		is_size_t(base64_decode(dest,destlen,"Wg==",4), 1);
+		is_mem(&dest[1],"\0",2);
+	}
+
+	/* test encoding using different alphabets */
+	{
+		char alphabet_fs_safe[64];
+		memcpy(alphabet_fs_safe,base64_maps_rfc4648.encode_map,sizeof(alphabet_fs_safe));
+		alphabet_fs_safe[62] = '-';
+		alphabet_fs_safe[63] = '_';
+		test_encode_using_alphabet(alphabet_fs_safe,"\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0",20,"__-FwHRSRIsFU2IhAEGD-AMPhOA=");
+	}
+
+	/* test decoding using different alphabets */
+	{
+		char alphabet_fs_safe[64];
+		#define src "__-FwHRSRIsFU2IhAEGD-AMPhOA="
+		#define expected "\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0"
+
+		memcpy(alphabet_fs_safe,base64_maps_rfc4648.encode_map,sizeof(alphabet_fs_safe));
+		alphabet_fs_safe[62] = '-';
+		alphabet_fs_safe[63] = '_';
+
+		test_decode_using_alphabet(alphabet_fs_safe,src,strlen(src),expected,20);
+		#undef src
+		#undef expected
+	}
+
+	/* explicitly test the non-maps encode_triplet and
+	   encode_tail functions */
+	{
+		size_t destlen = 4;
+		char dest[destlen];
+		const char *src = "AB\04";
+		memset(dest,'\1',sizeof(dest));
+		base64_encode_triplet(dest,src);
+		is_mem(dest,"QUIE",sizeof(dest));
+	}
+	{
+		size_t destlen = 4;
+		char dest[destlen];
+		const char *src = "A";
+		memset(dest,'\1',sizeof(dest));
+		base64_encode_tail(dest,src,strlen(src));
+		is_mem(dest,"QQ==",sizeof(dest));
+	}
+
+	/* test the alphabet inversion */
+	{
+		base64_maps_t dest;
+		const char expected_inverse[] =
+			"\xff\xff\xff\xff\xff" /* 0 */
+			"\xff\xff\xff\xff\xff" /* 5 */
+			"\xff\xff\xff\xff\xff" /* 10 */
+			"\xff\xff\xff\xff\xff" /* 15 */
+			"\xff\xff\xff\xff\xff" /* 20 */
+			"\xff\xff\xff\xff\xff" /* 25 */
+			"\xff\xff\xff\xff\xff" /* 30 */
+			"\xff\xff\xff\xff\xff" /* 35 */
+			"\xff\xff\xff\x3e\xff" /* 40 */
+			"\xff\xff\x3f\x34\x35" /* 45 - */
+			"\x36\x37\x38\x39\x3a" /* 50 */
+			"\x3b\x3c\x3d\xff\xff" /* 55 */
+			"\xff\xff\xff\xff\xff" /* 60 */
+			"\x00\x01\x02\x03\x04" /* 65 A */
+			"\x05\x06\x07\x08\x09" /* 70 */
+			"\x0a\x0b\x0c\x0d\x0e" /* 75 */
+			"\x0f\x10\x11\x12\x13" /* 80 */
+			"\x14\x15\x16\x17\x18" /* 85 */
+			"\x19\xff\xff\xff\xff" /* 90 */
+			"\xff\xff\x1a\x1b\x1c" /* 95 _ */
+			"\x1d\x1e\x1f\x20\x21" /* 100 */
+			"\x22\x23\x24\x25\x26" /* 105 */
+			"\x27\x28\x29\x2a\x2b" /* 110 */
+			"\x2c\x2d\x2e\x2f\x30" /* 115 */
+			"\x31\x32\x33\xff\xff" /* 120 */
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 125 */
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 155 */
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 185 */
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 215 */
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 245 */
+			;
+		const char * src = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+		base64_init_maps(&dest, src);
+		is_mem((const char *)dest.decode_map, expected_inverse, 256);
+		ok1(base64_char_in_alphabet(&dest,'A'));
+		ok1(!base64_char_in_alphabet(&dest,'\n'));
+	}
+
+	/* explicitly test the non-alpha decode_tail and decode_quartet */
+	{
+		char dest[4];
+		const char *src = "QQ==";
+		const char * expected = "A";
+		memset(dest, '%', sizeof(dest));
+		base64_decode_tail(dest,src,4);
+		is_mem(dest, expected, 1);
+	}
+	{
+		char dest[4];
+		const char *src = "Zm9v";
+		const char * expected = "foo";
+		memset(dest, '%', sizeof(dest));
+		base64_decode_quartet(dest,src);
+		is_mem(dest, expected, 1);
+	}
+
+	exit(exit_status());
+}
+
+static void * xmalloc(size_t size)
+{
+	char * ret;
+	ret = malloc(size);
+	if (ret == NULL) {
+		perror("malloc");
+		abort();
+	}
+	return ret;
+}
+
+/* End of run.c test */
diff --git a/src/common/libccan/ccan/bitmap/LICENSE b/src/common/libccan/ccan/bitmap/LICENSE
new file mode 120000
index 000000000000..dc314ecac02a
--- /dev/null
+++ b/src/common/libccan/ccan/bitmap/LICENSE
@@ -0,0 +1 @@
+../../licenses/LGPL-2.1
\ No newline at end of file
diff --git a/src/common/libccan/ccan/bitmap/_info b/src/common/libccan/ccan/bitmap/_info
new file mode 100644
index 000000000000..b8d297d168d8
--- /dev/null
+++ b/src/common/libccan/ccan/bitmap/_info
@@ -0,0 +1,32 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * bitmap - bitmap handling
+ *
+ * This code handles manipulation of bitmaps, arbitrary length arrays
+ * of bits.
+ *
+ * License: LGPL (v2.1 or any later version)
+ * Author: David Gibson <david@gibson.dropbear.id.au>
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/endian\n");
+		return 0;
+	}
+
+        if (strcmp(argv[1], "testdepends") == 0) {
+                printf("ccan/array_size\n");
+		printf("ccan/foreach\n");
+                return 0;
+        }
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/bitmap/bitmap.c b/src/common/libccan/ccan/bitmap/bitmap.c
new file mode 100644
index 000000000000..d254b20e561e
--- /dev/null
+++ b/src/common/libccan/ccan/bitmap/bitmap.c
@@ -0,0 +1,125 @@
+/* Licensed under LGPLv2.1+ - see LICENSE file for details */
+
+#include "config.h"
+
+#include <ccan/bitmap/bitmap.h>
+
+#include <assert.h>
+
+#define BIT_ALIGN_DOWN(n)	((n) & ~(BITMAP_WORD_BITS - 1))
+#define BIT_ALIGN_UP(n)		BIT_ALIGN_DOWN((n) + BITMAP_WORD_BITS - 1)
+
+void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m)
+{
+	unsigned long an = BIT_ALIGN_UP(n);
+	unsigned long am = BIT_ALIGN_DOWN(m);
+	bitmap_word headmask = BITMAP_WORD_1 >> (n % BITMAP_WORD_BITS);
+	bitmap_word tailmask = ~(BITMAP_WORD_1 >> (m % BITMAP_WORD_BITS));
+
+	assert(m >= n);
+
+	if (am < an) {
+		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask & tailmask);
+		return;
+	}
+
+	if (an > n)
+		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask);
+
+	if (am > an)
+		memset(&BITMAP_WORD(bitmap, an), 0,
+		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
+
+	if (m > am)
+		BITMAP_WORD(bitmap, m) &= ~bitmap_bswap(tailmask);
+}
+
+void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m)
+{
+	unsigned long an = BIT_ALIGN_UP(n);
+	unsigned long am = BIT_ALIGN_DOWN(m);
+	bitmap_word headmask = BITMAP_WORD_1 >> (n % BITMAP_WORD_BITS);
+	bitmap_word tailmask = ~(BITMAP_WORD_1 >> (m % BITMAP_WORD_BITS));
+
+	assert(m >= n);
+
+	if (am < an) {
+		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask & tailmask);
+		return;
+	}
+
+	if (an > n)
+		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask);
+
+	if (am > an)
+		memset(&BITMAP_WORD(bitmap, an), 0xff,
+		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
+
+	if (m > am)
+		BITMAP_WORD(bitmap, m) |= bitmap_bswap(tailmask);
+}
+
+static int bitmap_clz(bitmap_word w)
+{
+#if HAVE_BUILTIN_CLZL
+	return __builtin_clzl(w);
+#else
+	int lz = 0;
+	bitmap_word mask = (bitmap_word)1 << (BITMAP_WORD_BITS - 1);
+
+	while (!(w & mask)) {
+		lz++;
+		mask >>= 1;
+	}
+
+	return lz;
+#endif
+}
+
+unsigned long bitmap_ffs(const bitmap *bitmap,
+			 unsigned long n, unsigned long m)
+{
+	unsigned long an = BIT_ALIGN_UP(n);
+	unsigned long am = BIT_ALIGN_DOWN(m);
+	bitmap_word headmask = BITMAP_WORD_1 >> (n % BITMAP_WORD_BITS);
+	bitmap_word tailmask = ~(BITMAP_WORD_1 >> (m % BITMAP_WORD_BITS));
+
+	assert(m >= n);
+
+	if (am < an) {
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
+
+		w &= (headmask & tailmask);
+
+		return w ? am + bitmap_clz(w) : m;
+	}
+
+	if (an > n) {
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
+
+		w &= headmask;
+
+		if (w)
+			return BIT_ALIGN_DOWN(n) + bitmap_clz(w);
+	}
+
+	while (an < am) {
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, an));
+
+		if (w)
+			return an + bitmap_clz(w);
+
+		an += BITMAP_WORD_BITS;
+	}
+
+	if (m > am) {
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, m));
+
+		w &= tailmask;
+
+		if (w)
+			return am + bitmap_clz(w);
+	}
+
+	return m;
+}
diff --git a/src/common/libccan/ccan/bitmap/bitmap.h b/src/common/libccan/ccan/bitmap/bitmap.h
new file mode 100644
index 000000000000..543828010188
--- /dev/null
+++ b/src/common/libccan/ccan/bitmap/bitmap.h
@@ -0,0 +1,246 @@
+/* Licensed under LGPLv2+ - see LICENSE file for details */
+#ifndef CCAN_BITMAP_H_
+#define CCAN_BITMAP_H_
+
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+#include <ccan/endian/endian.h>
+
+typedef unsigned long bitmap_word;
+
+#define BITMAP_WORD_BITS	(sizeof(bitmap_word) * CHAR_BIT)
+#define BITMAP_NWORDS(_n)	\
+	(((_n) + BITMAP_WORD_BITS - 1) / BITMAP_WORD_BITS)
+
+#define BITMAP_WORD_0		(0)
+#define BITMAP_WORD_1		((bitmap_word)-1UL)
+
+/*
+ * We wrap each word in a structure for type checking.
+ */
+typedef struct bitmap {
+	bitmap_word w;
+} bitmap;
+
+#define BITMAP_DECLARE(_name, _nbits) \
+	bitmap (_name)[BITMAP_NWORDS(_nbits)]
+
+static inline size_t bitmap_sizeof(unsigned long nbits)
+{
+	return BITMAP_NWORDS(nbits) * sizeof(bitmap_word);
+}
+
+static inline bitmap_word bitmap_bswap(bitmap_word w)
+{
+	if (BITMAP_WORD_BITS == 32)
+		return (ENDIAN_CAST bitmap_word)cpu_to_be32(w);
+	else if (BITMAP_WORD_BITS == 64)
+		return (ENDIAN_CAST bitmap_word)cpu_to_be64(w);
+}
+
+#define BITMAP_WORD(_bm, _n)	((_bm)[(_n) / BITMAP_WORD_BITS].w)
+#define BITMAP_WORDBIT(_n) 	\
+	(bitmap_bswap(1UL << (BITMAP_WORD_BITS - ((_n) % BITMAP_WORD_BITS) - 1)))
+
+#define BITMAP_HEADWORDS(_nbits) \
+	((_nbits) / BITMAP_WORD_BITS)
+#define BITMAP_HEADBYTES(_nbits) \
+	(BITMAP_HEADWORDS(_nbits) * sizeof(bitmap_word))
+
+#define BITMAP_TAILWORD(_bm, _nbits) \
+	((_bm)[BITMAP_HEADWORDS(_nbits)].w)
+#define BITMAP_HASTAIL(_nbits)	(((_nbits) % BITMAP_WORD_BITS) != 0)
+#define BITMAP_TAILBITS(_nbits)	\
+	(bitmap_bswap(~(-1UL >> ((_nbits) % BITMAP_WORD_BITS))))
+#define BITMAP_TAIL(_bm, _nbits) \
+	(BITMAP_TAILWORD(_bm, _nbits) & BITMAP_TAILBITS(_nbits))
+
+static inline void bitmap_set_bit(bitmap *bitmap, unsigned long n)
+{
+	BITMAP_WORD(bitmap, n) |= BITMAP_WORDBIT(n);
+}
+
+static inline void bitmap_clear_bit(bitmap *bitmap, unsigned long n)
+{
+	BITMAP_WORD(bitmap, n) &= ~BITMAP_WORDBIT(n);
+}
+
+static inline void bitmap_change_bit(bitmap *bitmap, unsigned long n)
+{
+	BITMAP_WORD(bitmap, n) ^= BITMAP_WORDBIT(n);
+}
+
+static inline bool bitmap_test_bit(const bitmap *bitmap, unsigned long n)
+{
+	return !!(BITMAP_WORD(bitmap, n) & BITMAP_WORDBIT(n));
+}
+
+void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m);
+void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m);
+
+static inline void bitmap_zero(bitmap *bitmap, unsigned long nbits)
+{
+	memset(bitmap, 0, bitmap_sizeof(nbits));
+}
+
+static inline void bitmap_fill(bitmap *bitmap, unsigned long nbits)
+{
+	memset(bitmap, 0xff, bitmap_sizeof(nbits));
+}
+
+static inline void bitmap_copy(bitmap *dst, const bitmap *src,
+			       unsigned long nbits)
+{
+	memcpy(dst, src, bitmap_sizeof(nbits));
+}
+
+#define BITMAP_DEF_BINOP(_name, _op) \
+	static inline void bitmap_##_name(bitmap *dst, bitmap *src1, bitmap *src2, \
+					  unsigned long nbits)		\
+	{ \
+		unsigned long i = 0; \
+		for (i = 0; i < BITMAP_NWORDS(nbits); i++) { \
+			dst[i].w = src1[i].w _op src2[i].w; \
+		} \
+	}
+
+BITMAP_DEF_BINOP(and, &)
+BITMAP_DEF_BINOP(or, |)
+BITMAP_DEF_BINOP(xor, ^)
+BITMAP_DEF_BINOP(andnot, & ~)
+
+#undef BITMAP_DEF_BINOP
+
+static inline void bitmap_complement(bitmap *dst, const bitmap *src,
+				     unsigned long nbits)
+{
+	unsigned long i;
+
+	for (i = 0; i < BITMAP_NWORDS(nbits); i++)
+		dst[i].w = ~src[i].w;
+}
+
+static inline bool bitmap_equal(const bitmap *src1, const bitmap *src2,
+				unsigned long nbits)
+{
+	return (memcmp(src1, src2, BITMAP_HEADBYTES(nbits)) == 0)
+		&& (!BITMAP_HASTAIL(nbits)
+		    || (BITMAP_TAIL(src1, nbits) == BITMAP_TAIL(src2, nbits)));
+}
+
+static inline bool bitmap_intersects(const bitmap *src1, const bitmap *src2,
+				     unsigned long nbits)
+{
+	unsigned long i;
+
+	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
+		if (src1[i].w & src2[i].w)
+			return true;
+	}
+	if (BITMAP_HASTAIL(nbits) &&
+	    (BITMAP_TAIL(src1, nbits) & BITMAP_TAIL(src2, nbits)))
+		return true;
+	return false;
+}
+
+static inline bool bitmap_subset(const bitmap *src1, const bitmap *src2,
+				 unsigned long nbits)
+{
+	unsigned long i;
+
+	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
+		if (src1[i].w  & ~src2[i].w)
+			return false;
+	}
+	if (BITMAP_HASTAIL(nbits) &&
+	    (BITMAP_TAIL(src1, nbits) & ~BITMAP_TAIL(src2, nbits)))
+		return false;
+	return true;
+}
+
+static inline bool bitmap_full(const bitmap *bitmap, unsigned long nbits)
+{
+	unsigned long i;
+
+	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
+		if (bitmap[i].w != -1UL)
+			return false;
+	}
+	if (BITMAP_HASTAIL(nbits) &&
+	    (BITMAP_TAIL(bitmap, nbits) != BITMAP_TAILBITS(nbits)))
+		return false;
+
+	return true;
+}
+
+static inline bool bitmap_empty(const bitmap *bitmap, unsigned long nbits)
+{
+	unsigned long i;
+
+	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
+		if (bitmap[i].w != 0)
+			return false;
+	}
+	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(bitmap, nbits) != 0))
+		return false;
+
+	return true;
+}
+
+unsigned long bitmap_ffs(const bitmap *bitmap,
+			 unsigned long n, unsigned long m);
+
+/*
+ * Allocation functions
+ */
+static inline bitmap *bitmap_alloc(unsigned long nbits)
+{
+	return malloc(bitmap_sizeof(nbits));
+}
+
+static inline bitmap *bitmap_alloc0(unsigned long nbits)
+{
+	bitmap *bitmap;
+
+	bitmap = bitmap_alloc(nbits);
+	if (bitmap)
+		bitmap_zero(bitmap, nbits);
+	return bitmap;
+}
+
+static inline bitmap *bitmap_alloc1(unsigned long nbits)
+{
+	bitmap *bitmap;
+
+	bitmap = bitmap_alloc(nbits);
+	if (bitmap)
+		bitmap_fill(bitmap, nbits);
+	return bitmap;
+}
+
+static inline bitmap *bitmap_realloc0(bitmap *bitmap,
+				      unsigned long obits, unsigned long nbits)
+{
+	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
+
+	if ((nbits > obits) && bitmap)
+		bitmap_zero_range(bitmap, obits, nbits);
+
+	return bitmap;
+}
+
+static inline bitmap *bitmap_realloc1(bitmap *bitmap,
+				      unsigned long obits, unsigned long nbits)
+{
+	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
+
+	if ((nbits > obits) && bitmap)
+		bitmap_fill_range(bitmap, obits, nbits);
+
+	return bitmap;
+}
+
+#endif /* CCAN_BITMAP_H_ */
diff --git a/src/common/libccan/ccan/bitmap/test/run-alloc.c b/src/common/libccan/ccan/bitmap/test/run-alloc.c
new file mode 100644
index 000000000000..fcdaf4daaab5
--- /dev/null
+++ b/src/common/libccan/ccan/bitmap/test/run-alloc.c
@@ -0,0 +1,101 @@
+#include <ccan/bitmap/bitmap.h>
+#include <ccan/tap/tap.h>
+#include <ccan/array_size/array_size.h>
+#include <ccan/foreach/foreach.h>
+
+#include <ccan/bitmap/bitmap.c>
+
+int bitmap_sizes[] = {
+	1, 2, 3, 4, 5, 6, 7, 8,
+	16, 17, 24, 32, 33,
+	64, 65, 127, 128, 129,
+	1023, 1024, 1025,
+};
+#define NSIZES ARRAY_SIZE(bitmap_sizes)
+#define NTESTS_BASE 4
+#define NTESTS_REALLOC 10
+
+static void test_basic_alloc(int nbits)
+{
+	bitmap *bitmap;
+
+	bitmap = bitmap_alloc0(nbits);
+	ok1(bitmap != NULL);
+	ok1(bitmap_empty(bitmap, nbits));
+
+	free(bitmap);
+
+	bitmap = bitmap_alloc1(nbits);
+	ok1(bitmap != NULL);
+	ok1(bitmap_full(bitmap, nbits));
+
+	free(bitmap);
+}
+
+static void test_realloc(int obits, int nbits)
+{
+	bitmap *bitmap;
+	int i;
+	bool wrong;
+
+	bitmap = bitmap_alloc0(obits);
+	ok1(bitmap != NULL);
+	ok1(bitmap_empty(bitmap, obits));
+
+	bitmap = bitmap_realloc1(bitmap, obits, nbits);
+	ok1(bitmap != NULL);
+	if (obits < nbits)
+		ok1(bitmap_empty(bitmap, obits));
+	else
+		ok1(bitmap_empty(bitmap, nbits));
+
+	wrong = false;
+	for (i = obits; i < nbits; i++)
+		wrong = wrong || !bitmap_test_bit(bitmap, i);
+	ok1(!wrong);
+
+	free(bitmap);
+
+	bitmap = bitmap_alloc1(obits);
+	ok1(bitmap != NULL);
+	ok1(bitmap_full(bitmap, obits));
+
+	bitmap = bitmap_realloc0(bitmap, obits, nbits);
+	ok1(bitmap != NULL);
+	if (obits < nbits)
+		ok1(bitmap_full(bitmap, obits));
+	else
+		ok1(bitmap_full(bitmap, nbits));
+
+	wrong = false;
+	for (i = obits; i < nbits; i++)
+		wrong = wrong || bitmap_test_bit(bitmap, i);
+	ok1(!wrong);
+
+	free(bitmap);
+}
+
+int main(void)
+{
+	int i, j;
+
+	/* This is how many tests you plan to run */
+	plan_tests(NSIZES * NTESTS_BASE + NSIZES * NSIZES * NTESTS_REALLOC);
+
+	for (i = 0; i < NSIZES; i++) {
+		diag("Testing %d-bit bitmap", bitmap_sizes[i]);
+		test_basic_alloc(bitmap_sizes[i]);
+	}
+
+	for (i = 0; i < NSIZES; i++) {
+		for (j = 0; j < NSIZES; j++) {
+			diag("Testing %d-bit => %d-bit bitmap",
+			     bitmap_sizes[i], bitmap_sizes[j]);
+
+			test_realloc(bitmap_sizes[i], bitmap_sizes[j]);
+		}
+	}
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/bitmap/test/run-ffs.c b/src/common/libccan/ccan/bitmap/test/run-ffs.c
new file mode 100644
index 000000000000..0f6cc4cf810c
--- /dev/null
+++ b/src/common/libccan/ccan/bitmap/test/run-ffs.c
@@ -0,0 +1,74 @@
+#include <ccan/bitmap/bitmap.h>
+#include <ccan/tap/tap.h>
+#include <ccan/array_size/array_size.h>
+#include <ccan/foreach/foreach.h>
+
+#include <ccan/bitmap/bitmap.c>
+
+int bitmap_sizes[] = {
+	1, 2, 3, 4, 5, 6, 7, 8,
+	16, 17, 24, 32, 33,
+	64, 65, 127, 128, 129,
+	1023, 1024, 1025,
+};
+#define NSIZES ARRAY_SIZE(bitmap_sizes)
+
+#define ok_eq(a, b) \
+	ok((a) == (b), "%s [%u] == %s [%u]", \
+	   #a, (unsigned)(a), #b, (unsigned)(b))
+
+static void test_size(int nbits)
+{
+	BITMAP_DECLARE(bitmap, nbits);
+	int i;
+
+	bitmap_zero(bitmap, nbits);
+	ok_eq(bitmap_ffs(bitmap, 0, nbits), nbits);
+
+	for (i = 0; i < nbits; i++) {
+		bitmap_zero(bitmap, nbits);
+		bitmap_set_bit(bitmap, i);
+
+		ok_eq(bitmap_ffs(bitmap, 0, nbits), i);
+		ok_eq(bitmap_ffs(bitmap, i, nbits), i);
+		ok_eq(bitmap_ffs(bitmap, i + 1, nbits), nbits);
+
+		bitmap_zero(bitmap, nbits);
+		bitmap_fill_range(bitmap, i, nbits);
+
+		ok_eq(bitmap_ffs(bitmap, 0, nbits), i);
+		ok_eq(bitmap_ffs(bitmap, i, nbits), i);
+		ok_eq(bitmap_ffs(bitmap, i + 1, nbits), (i + 1));
+		ok_eq(bitmap_ffs(bitmap, nbits - 1, nbits), (nbits - 1));
+
+		if (i > 0) {
+			ok_eq(bitmap_ffs(bitmap, 0, i), i);
+			ok_eq(bitmap_ffs(bitmap, 0, i - 1), (i - 1));
+		}
+
+		if (i > 0) {
+			bitmap_zero(bitmap, nbits);
+			bitmap_fill_range(bitmap, 0, i);
+
+			ok_eq(bitmap_ffs(bitmap, 0, nbits), 0);
+			ok_eq(bitmap_ffs(bitmap, i - 1, nbits), (i - 1));
+			ok_eq(bitmap_ffs(bitmap, i, nbits), nbits);
+		}
+	}
+}
+
+int main(void)
+{
+	int i;
+
+	/* Too complicated to work out the exact number */
+	plan_no_plan();
+
+	for (i = 0; i < NSIZES; i++) {
+		diag("Testing %d-bit bitmap", bitmap_sizes[i]);
+		test_size(bitmap_sizes[i]);
+	}
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/bitmap/test/run-ranges.c b/src/common/libccan/ccan/bitmap/test/run-ranges.c
new file mode 100644
index 000000000000..5ba383caf534
--- /dev/null
+++ b/src/common/libccan/ccan/bitmap/test/run-ranges.c
@@ -0,0 +1,77 @@
+#include <ccan/bitmap/bitmap.h>
+#include <ccan/tap/tap.h>
+#include <ccan/array_size/array_size.h>
+#include <ccan/foreach/foreach.h>
+
+#include <ccan/bitmap/bitmap.c>
+
+int bitmap_sizes[] = {
+	1, 2, 3, 4, 5, 6, 7, 8,
+	16, 24, 32, 64, 256,
+	/*
+	 * Don't put too big sizes in here, or it will take forever to
+	 * run under valgrind (the test is O(n^3)).
+	 */
+};
+#define NSIZES ARRAY_SIZE(bitmap_sizes)
+#define NTESTS 2
+
+static void test_size(int nbits)
+{
+	BITMAP_DECLARE(bitmap, nbits);
+	uint32_t marker = 0xdeadbeef;
+	int i, j, k;
+	bool wrong;
+
+	for (i = 0; i < nbits; i++) {
+		for (j = i; j <= nbits; j++) {
+			bitmap_zero(bitmap, nbits);
+			bitmap_fill_range(bitmap, i, j);
+
+			wrong = false;
+			for (k = 0; k < nbits; k++) {
+				bool inrange = (k >= i) && (k < j);
+				wrong = wrong || (bitmap_test_bit(bitmap, k) != inrange);
+			}
+			ok1(!wrong);
+		}
+	}
+
+	for (i = 0; i < nbits; i++) {
+		for (j = i; j <= nbits; j++) {
+			bitmap_fill(bitmap, nbits);
+			bitmap_zero_range(bitmap, i, j);
+
+			wrong = false;
+			for (k = 0; k < nbits; k++) {
+				bool inrange = (k >= i) && (k < j);
+				wrong = wrong || (bitmap_test_bit(bitmap, k) == inrange);
+			}
+			ok1(!wrong);
+		}
+	}
+
+	ok1(marker == 0xdeadbeef);
+}
+
+int main(void)
+{
+	int totaltests = 0;
+	int i;
+
+	for (i = 0; i < NSIZES; i++) {
+		int size = bitmap_sizes[i];
+
+		/* Summing the arithmetic series gives: */
+		totaltests += size*(size + 3) + 1;
+	}
+	plan_tests(totaltests);
+
+	for (i = 0; i < NSIZES; i++) {
+		diag("Testing %d-bit bitmap", bitmap_sizes[i]);
+		test_size(bitmap_sizes[i]);
+	}
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/bitmap/test/run.c b/src/common/libccan/ccan/bitmap/test/run.c
new file mode 100644
index 000000000000..efd245d7774e
--- /dev/null
+++ b/src/common/libccan/ccan/bitmap/test/run.c
@@ -0,0 +1,118 @@
+#include <ccan/bitmap/bitmap.h>
+#include <ccan/tap/tap.h>
+#include <ccan/array_size/array_size.h>
+#include <ccan/foreach/foreach.h>
+
+#include <ccan/bitmap/bitmap.c>
+
+int bitmap_sizes[] = {
+	1, 2, 3, 4, 5, 6, 7, 8,
+	16, 17, 24, 32, 33,
+	64, 65, 127, 128, 129,
+	1023, 1024, 1025,
+};
+#define NSIZES ARRAY_SIZE(bitmap_sizes)
+#define NTESTS 9
+
+static void test_sizes(int nbits, bool dynalloc)
+{
+	BITMAP_DECLARE(sbitmap, nbits);
+	uint32_t marker;
+	bitmap *bitmap;
+	int i, j;
+	bool wrong;
+
+	if (dynalloc) {
+		bitmap = bitmap_alloc(nbits);
+		ok1(bitmap != NULL);
+	} else {
+		bitmap = sbitmap;
+		marker = 0xdeadbeef;
+	}
+
+	bitmap_zero(bitmap, nbits);
+	wrong = false;
+	for (i = 0; i < nbits; i++) {
+		wrong = wrong || bitmap_test_bit(bitmap, i);
+	}
+	ok1(!wrong);
+
+	bitmap_fill(bitmap, nbits);
+	wrong = false;
+	for (i = 0; i < nbits; i++) {
+		wrong = wrong || !bitmap_test_bit(bitmap, i);
+	}
+	ok1(!wrong);
+
+	wrong = false;
+	for (i = 0; i < nbits; i++) {
+		bitmap_zero(bitmap, nbits);
+		bitmap_set_bit(bitmap, i);
+		for (j = 0; j < nbits; j++) {
+			bool val = (i == j);
+
+			wrong = wrong || (bitmap_test_bit(bitmap, j) != val);
+		}
+	}
+	ok1(!wrong);
+
+	wrong = false;
+	for (i = 0; i < nbits; i++) {
+		bitmap_fill(bitmap, nbits);
+		bitmap_clear_bit(bitmap, i);
+		for (j = 0; j < nbits; j++) {
+			bool val = !(i == j);
+
+			wrong = wrong || (bitmap_test_bit(bitmap, j) != val);
+		}
+	}
+	ok1(!wrong);
+
+	bitmap_zero(bitmap, nbits);
+	ok1(bitmap_empty(bitmap, nbits));
+
+	wrong = false;
+	for (i = 0; i < nbits; i++) {
+		bitmap_zero(bitmap, nbits);
+		bitmap_set_bit(bitmap, i);
+		wrong = wrong || bitmap_empty(bitmap, nbits);
+	}
+	ok1(!wrong);
+
+	bitmap_fill(bitmap, nbits);
+	ok1(bitmap_full(bitmap, nbits));
+
+	wrong = false;
+	for (i = 0; i < nbits; i++) {
+		bitmap_fill(bitmap, nbits);
+		bitmap_clear_bit(bitmap, i);
+		wrong = wrong || bitmap_full(bitmap, nbits);
+	}
+	ok1(!wrong);
+	
+	if (dynalloc) {
+		free(bitmap);
+	} else {
+		ok1(marker == 0xdeadbeef);
+	}
+}
+
+int main(void)
+{
+	int i;
+	bool dynalloc;
+
+	/* This is how many tests you plan to run */
+	plan_tests(NSIZES * NTESTS * 2);
+
+	for (i = 0; i < NSIZES; i++) {
+		foreach_int(dynalloc, false, true) {
+			diag("Testing %d-bit bitmap (%s allocation)",
+			     bitmap_sizes[i], dynalloc ? "dynamic" : "static");
+			test_sizes(bitmap_sizes[i], dynalloc);
+		}
+	}
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/build_assert/LICENSE b/src/common/libccan/ccan/build_assert/LICENSE
new file mode 120000
index 000000000000..b7951dabdc82
--- /dev/null
+++ b/src/common/libccan/ccan/build_assert/LICENSE
@@ -0,0 +1 @@
+../../licenses/CC0
\ No newline at end of file
diff --git a/src/common/libccan/ccan/build_assert/_info b/src/common/libccan/ccan/build_assert/_info
new file mode 100644
index 000000000000..97ebe6c96683
--- /dev/null
+++ b/src/common/libccan/ccan/build_assert/_info
@@ -0,0 +1,49 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * build_assert - routines for build-time assertions
+ *
+ * This code provides routines which will cause compilation to fail should some
+ * assertion be untrue: such failures are preferable to run-time assertions,
+ * but much more limited since they can only depends on compile-time constants.
+ *
+ * These assertions are most useful when two parts of the code must be kept in
+ * sync: it is better to avoid such cases if possible, but seconds best is to
+ * detect invalid changes at build time.
+ *
+ * For example, a tricky piece of code might rely on a certain element being at
+ * the start of the structure.  To ensure that future changes don't break it,
+ * you would catch such changes in your code like so:
+ *
+ * Example:
+ *	#include <stddef.h>
+ *	#include <ccan/build_assert/build_assert.h>
+ *
+ *	struct foo {
+ *		char string[5];
+ *		int x;
+ *	};
+ *
+ *	static char *foo_string(struct foo *foo)
+ *	{
+ *		// This trick requires that the string be first in the structure
+ *		BUILD_ASSERT(offsetof(struct foo, string) == 0);
+ *		return (char *)foo;
+ *	}
+ *
+ * License: CC0 (Public domain)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0)
+		/* Nothing. */
+		return 0;
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/build_assert/build_assert.h b/src/common/libccan/ccan/build_assert/build_assert.h
new file mode 100644
index 000000000000..b9ecd84028e3
--- /dev/null
+++ b/src/common/libccan/ccan/build_assert/build_assert.h
@@ -0,0 +1,40 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#ifndef CCAN_BUILD_ASSERT_H
+#define CCAN_BUILD_ASSERT_H
+
+/**
+ * BUILD_ASSERT - assert a build-time dependency.
+ * @cond: the compile-time condition which must be true.
+ *
+ * Your compile will fail if the condition isn't true, or can't be evaluated
+ * by the compiler.  This can only be used within a function.
+ *
+ * Example:
+ *	#include <stddef.h>
+ *	...
+ *	static char *foo_to_char(struct foo *foo)
+ *	{
+ *		// This code needs string to be at start of foo.
+ *		BUILD_ASSERT(offsetof(struct foo, string) == 0);
+ *		return (char *)foo;
+ *	}
+ */
+#define BUILD_ASSERT(cond) \
+	do { (void) sizeof(char [1 - 2*!(cond)]); } while(0)
+
+/**
+ * BUILD_ASSERT_OR_ZERO - assert a build-time dependency, as an expression.
+ * @cond: the compile-time condition which must be true.
+ *
+ * Your compile will fail if the condition isn't true, or can't be evaluated
+ * by the compiler.  This can be used in an expression: its value is "0".
+ *
+ * Example:
+ *	#define foo_to_char(foo)					\
+ *		 ((char *)(foo)						\
+ *		  + BUILD_ASSERT_OR_ZERO(offsetof(struct foo, string) == 0))
+ */
+#define BUILD_ASSERT_OR_ZERO(cond) \
+	(sizeof(char [1 - 2*!(cond)]) - 1)
+
+#endif /* CCAN_BUILD_ASSERT_H */
diff --git a/src/common/libccan/ccan/build_assert/test/compile_fail-expr.c b/src/common/libccan/ccan/build_assert/test/compile_fail-expr.c
new file mode 100644
index 000000000000..6322eb3523a2
--- /dev/null
+++ b/src/common/libccan/ccan/build_assert/test/compile_fail-expr.c
@@ -0,0 +1,10 @@
+#include <ccan/build_assert/build_assert.h>
+
+int main(void)
+{
+#ifdef FAIL
+	return BUILD_ASSERT_OR_ZERO(1 == 0);
+#else
+	return 0;
+#endif
+}
diff --git a/src/common/libccan/ccan/build_assert/test/compile_fail.c b/src/common/libccan/ccan/build_assert/test/compile_fail.c
new file mode 100644
index 000000000000..9fd827d6aef8
--- /dev/null
+++ b/src/common/libccan/ccan/build_assert/test/compile_fail.c
@@ -0,0 +1,9 @@
+#include <ccan/build_assert/build_assert.h>
+
+int main(void)
+{
+#ifdef FAIL
+	BUILD_ASSERT(1 == 0);
+#endif
+	return 0;
+}
diff --git a/src/common/libccan/ccan/build_assert/test/compile_ok.c b/src/common/libccan/ccan/build_assert/test/compile_ok.c
new file mode 100644
index 000000000000..b4de8b41402d
--- /dev/null
+++ b/src/common/libccan/ccan/build_assert/test/compile_ok.c
@@ -0,0 +1,7 @@
+#include <ccan/build_assert/build_assert.h>
+
+int main(void)
+{
+	BUILD_ASSERT(1 == 1);
+	return 0;
+}
diff --git a/src/common/libccan/ccan/build_assert/test/run-BUILD_ASSERT_OR_ZERO.c b/src/common/libccan/ccan/build_assert/test/run-BUILD_ASSERT_OR_ZERO.c
new file mode 100644
index 000000000000..72f9ad1ba478
--- /dev/null
+++ b/src/common/libccan/ccan/build_assert/test/run-BUILD_ASSERT_OR_ZERO.c
@@ -0,0 +1,9 @@
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	plan_tests(1);
+	ok1(BUILD_ASSERT_OR_ZERO(1 == 1) == 0);
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/check_type/LICENSE b/src/common/libccan/ccan/check_type/LICENSE
new file mode 120000
index 000000000000..b7951dabdc82
--- /dev/null
+++ b/src/common/libccan/ccan/check_type/LICENSE
@@ -0,0 +1 @@
+../../licenses/CC0
\ No newline at end of file
diff --git a/src/common/libccan/ccan/check_type/_info b/src/common/libccan/ccan/check_type/_info
new file mode 100644
index 000000000000..cc4267349281
--- /dev/null
+++ b/src/common/libccan/ccan/check_type/_info
@@ -0,0 +1,33 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * check_type - routines for compile time type checking
+ *
+ * C has fairly weak typing: ints get automatically converted to longs, signed
+ * to unsigned, etc.  There are some cases where this is best avoided, and
+ * these macros provide methods for evoking warnings (or build errors) when
+ * a precise type isn't used.
+ *
+ * On compilers which don't support typeof() these routines are less effective,
+ * since they have to use sizeof() which can only distiguish between types of
+ * different size.
+ *
+ * License: CC0 (Public domain)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+#if !HAVE_TYPEOF
+		printf("ccan/build_assert\n");
+#endif
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/check_type/check_type.h b/src/common/libccan/ccan/check_type/check_type.h
new file mode 100644
index 000000000000..837aef7b1a36
--- /dev/null
+++ b/src/common/libccan/ccan/check_type/check_type.h
@@ -0,0 +1,64 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#ifndef CCAN_CHECK_TYPE_H
+#define CCAN_CHECK_TYPE_H
+#include "config.h"
+
+/**
+ * check_type - issue a warning or build failure if type is not correct.
+ * @expr: the expression whose type we should check (not evaluated).
+ * @type: the exact type we expect the expression to be.
+ *
+ * This macro is usually used within other macros to try to ensure that a macro
+ * argument is of the expected type.  No type promotion of the expression is
+ * done: an unsigned int is not the same as an int!
+ *
+ * check_type() always evaluates to 0.
+ *
+ * If your compiler does not support typeof, then the best we can do is fail
+ * to compile if the sizes of the types are unequal (a less complete check).
+ *
+ * Example:
+ *	// They should always pass a 64-bit value to _set_some_value!
+ *	#define set_some_value(expr)			\
+ *		_set_some_value((check_type((expr), uint64_t), (expr)))
+ */
+
+/**
+ * check_types_match - issue a warning or build failure if types are not same.
+ * @expr1: the first expression (not evaluated).
+ * @expr2: the second expression (not evaluated).
+ *
+ * This macro is usually used within other macros to try to ensure that
+ * arguments are of identical types.  No type promotion of the expressions is
+ * done: an unsigned int is not the same as an int!
+ *
+ * check_types_match() always evaluates to 0.
+ *
+ * If your compiler does not support typeof, then the best we can do is fail
+ * to compile if the sizes of the types are unequal (a less complete check).
+ *
+ * Example:
+ *	// Do subtraction to get to enclosing type, but make sure that
+ *	// pointer is of correct type for that member.
+ *	#define container_of(mbr_ptr, encl_type, mbr)			\
+ *		(check_types_match((mbr_ptr), &((encl_type *)0)->mbr),	\
+ *		 ((encl_type *)						\
+ *		  ((char *)(mbr_ptr) - offsetof(encl_type, mbr))))
+ */
+#if HAVE_TYPEOF
+#define check_type(expr, type)			\
+	((typeof(expr) *)0 != (type *)0)
+
+#define check_types_match(expr1, expr2)		\
+	((typeof(expr1) *)0 != (typeof(expr2) *)0)
+#else
+#include <ccan/build_assert/build_assert.h>
+/* Without typeof, we can only test the sizes. */
+#define check_type(expr, type)					\
+	BUILD_ASSERT_OR_ZERO(sizeof(expr) == sizeof(type))
+
+#define check_types_match(expr1, expr2)				\
+	BUILD_ASSERT_OR_ZERO(sizeof(expr1) == sizeof(expr2))
+#endif /* HAVE_TYPEOF */
+
+#endif /* CCAN_CHECK_TYPE_H */
diff --git a/src/common/libccan/ccan/check_type/test/compile_fail-check_type.c b/src/common/libccan/ccan/check_type/test/compile_fail-check_type.c
new file mode 100644
index 000000000000..4c22c65a499c
--- /dev/null
+++ b/src/common/libccan/ccan/check_type/test/compile_fail-check_type.c
@@ -0,0 +1,11 @@
+#include <ccan/check_type/check_type.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+	(void)argv;
+#ifdef FAIL
+	check_type(argc, char);
+#endif
+	return 0;
+}
diff --git a/src/common/libccan/ccan/check_type/test/compile_fail-check_type_unsigned.c b/src/common/libccan/ccan/check_type/test/compile_fail-check_type_unsigned.c
new file mode 100644
index 000000000000..795e76067019
--- /dev/null
+++ b/src/common/libccan/ccan/check_type/test/compile_fail-check_type_unsigned.c
@@ -0,0 +1,16 @@
+#include <ccan/check_type/check_type.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+	(void)argv;
+#ifdef FAIL
+#if HAVE_TYPEOF
+	check_type(argc, unsigned int);
+#else
+	/* This doesn't work without typeof, so just fail */
+#error "Fail without typeof"
+#endif
+#endif
+	return 0;
+}
diff --git a/src/common/libccan/ccan/check_type/test/compile_fail-check_types_match.c b/src/common/libccan/ccan/check_type/test/compile_fail-check_types_match.c
new file mode 100644
index 000000000000..fb83738b90ab
--- /dev/null
+++ b/src/common/libccan/ccan/check_type/test/compile_fail-check_types_match.c
@@ -0,0 +1,11 @@
+#include <ccan/check_type/check_type.h>
+
+int main(int argc, char *argv[])
+{
+	unsigned char x = argc;
+	(void)argv;
+#ifdef FAIL
+	check_types_match(argc, x);
+#endif
+	return x;
+}
diff --git a/src/common/libccan/ccan/check_type/test/run.c b/src/common/libccan/ccan/check_type/test/run.c
new file mode 100644
index 000000000000..09fd3634376f
--- /dev/null
+++ b/src/common/libccan/ccan/check_type/test/run.c
@@ -0,0 +1,23 @@
+#include <ccan/check_type/check_type.h>
+#include <ccan/tap/tap.h>
+
+int main(int argc, char *argv[])
+{
+	int x = 0, y = 0;
+
+	(void)argv;
+	plan_tests(9);
+
+	ok1(check_type(argc, int) == 0);
+	ok1(check_type(&argc, int *) == 0);
+	ok1(check_types_match(argc, argc) == 0);
+	ok1(check_types_match(argc, x) == 0);
+	ok1(check_types_match(&argc, &x) == 0);
+
+	ok1(check_type(x++, int) == 0);
+	ok(x == 0, "check_type does not evaluate expression");
+	ok1(check_types_match(x++, y++) == 0);
+	ok(x == 0 && y == 0, "check_types_match does not evaluate expressions");
+
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/compiler/LICENSE b/src/common/libccan/ccan/compiler/LICENSE
new file mode 120000
index 000000000000..b7951dabdc82
--- /dev/null
+++ b/src/common/libccan/ccan/compiler/LICENSE
@@ -0,0 +1 @@
+../../licenses/CC0
\ No newline at end of file
diff --git a/src/common/libccan/ccan/compiler/_info b/src/common/libccan/ccan/compiler/_info
new file mode 100644
index 000000000000..12cb24757829
--- /dev/null
+++ b/src/common/libccan/ccan/compiler/_info
@@ -0,0 +1,64 @@
+#include "config.h"
+#include <string.h>
+#include <stdio.h>
+
+/**
+ * compiler - macros for common compiler extensions
+ *
+ * Abstracts away some compiler hints.  Currently these include:
+ * - COLD
+ *	For functions not called in fast paths (aka. cold functions)
+ * - PRINTF_FMT
+ *	For functions which take printf-style parameters.
+ * - CONST_FUNCTION
+ *	For functions which return the same value for same parameters.
+ * - NEEDED
+ *	For functions and variables which must be emitted even if unused.
+ * - UNNEEDED
+ *	For functions and variables which need not be emitted if unused.
+ * - UNUSED
+ *	For parameters which are not used.
+ * - IS_COMPILE_CONSTANT()
+ *	For using different tradeoffs for compiletime vs runtime evaluation.
+ *
+ * License: CC0 (Public domain)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ *
+ * Example:
+ *	#include <ccan/compiler/compiler.h>
+ *	#include <stdio.h>
+ *	#include <stdarg.h>
+ *
+ *	// Example of a (slow-path) logging function.
+ *	static int log_threshold = 2;
+ *	static void COLD PRINTF_FMT(2,3)
+ *		logger(int level, const char *fmt, ...)
+ *	{
+ *		va_list ap;
+ *		va_start(ap, fmt);
+ *		if (level >= log_threshold)
+ *			vfprintf(stderr, fmt, ap);
+ *		va_end(ap);
+ *	}
+ *
+ *	int main(int argc, char *argv[] UNNEEDED)
+ *	{
+ *		if (argc != 1) {
+ *			logger(3, "Don't want %i arguments!\n", argc-1);
+ *			return 1;
+ *		}
+ *		return 0;
+ *	}
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/compiler/compiler.h b/src/common/libccan/ccan/compiler/compiler.h
new file mode 100644
index 000000000000..562b29ec71cc
--- /dev/null
+++ b/src/common/libccan/ccan/compiler/compiler.h
@@ -0,0 +1,317 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#ifndef CCAN_COMPILER_H
+#define CCAN_COMPILER_H
+#include "config.h"
+
+#ifndef COLD
+#if HAVE_ATTRIBUTE_COLD
+/**
+ * COLD - a function is unlikely to be called.
+ *
+ * Used to mark an unlikely code path and optimize appropriately.
+ * It is usually used on logging or error routines.
+ *
+ * Example:
+ * static void COLD moan(const char *reason)
+ * {
+ *	fprintf(stderr, "Error: %s (%s)\n", reason, strerror(errno));
+ * }
+ */
+#define COLD __attribute__((__cold__))
+#else
+#define COLD
+#endif
+#endif
+
+#ifndef NORETURN
+#if HAVE_ATTRIBUTE_NORETURN
+/**
+ * NORETURN - a function does not return
+ *
+ * Used to mark a function which exits; useful for suppressing warnings.
+ *
+ * Example:
+ * static void NORETURN fail(const char *reason)
+ * {
+ *	fprintf(stderr, "Error: %s (%s)\n", reason, strerror(errno));
+ *	exit(1);
+ * }
+ */
+#define NORETURN __attribute__((__noreturn__))
+#else
+#define NORETURN
+#endif
+#endif
+
+#ifndef PRINTF_FMT
+#if HAVE_ATTRIBUTE_PRINTF
+/**
+ * PRINTF_FMT - a function takes printf-style arguments
+ * @nfmt: the 1-based number of the function's format argument.
+ * @narg: the 1-based number of the function's first variable argument.
+ *
+ * This allows the compiler to check your parameters as it does for printf().
+ *
+ * Example:
+ * void PRINTF_FMT(2,3) my_printf(const char *prefix, const char *fmt, ...);
+ */
+#define PRINTF_FMT(nfmt, narg) \
+	__attribute__((format(__printf__, nfmt, narg)))
+#else
+#define PRINTF_FMT(nfmt, narg)
+#endif
+#endif
+
+#ifndef CONST_FUNCTION
+#if HAVE_ATTRIBUTE_CONST
+/**
+ * CONST_FUNCTION - a function's return depends only on its argument
+ *
+ * This allows the compiler to assume that the function will return the exact
+ * same value for the exact same arguments.  This implies that the function
+ * must not use global variables, or dereference pointer arguments.
+ */
+#define CONST_FUNCTION __attribute__((__const__))
+#else
+#define CONST_FUNCTION
+#endif
+
+#ifndef PURE_FUNCTION
+#if HAVE_ATTRIBUTE_PURE
+/**
+ * PURE_FUNCTION - a function is pure
+ *
+ * A pure function is one that has no side effects other than it's return value
+ * and uses no inputs other than it's arguments and global variables.
+ */
+#define PURE_FUNCTION __attribute__((__pure__))
+#else
+#define PURE_FUNCTION
+#endif
+#endif
+#endif
+
+#if HAVE_ATTRIBUTE_UNUSED
+#ifndef UNNEEDED
+/**
+ * UNNEEDED - a variable/function may not be needed
+ *
+ * This suppresses warnings about unused variables or functions, but tells
+ * the compiler that if it is unused it need not emit it into the source code.
+ *
+ * Example:
+ * // With some preprocessor options, this is unnecessary.
+ * static UNNEEDED int counter;
+ *
+ * // With some preprocessor options, this is unnecessary.
+ * static UNNEEDED void add_to_counter(int add)
+ * {
+ *	counter += add;
+ * }
+ */
+#define UNNEEDED __attribute__((__unused__))
+#endif
+
+#ifndef NEEDED
+#if HAVE_ATTRIBUTE_USED
+/**
+ * NEEDED - a variable/function is needed
+ *
+ * This suppresses warnings about unused variables or functions, but tells
+ * the compiler that it must exist even if it (seems) unused.
+ *
+ * Example:
+ *	// Even if this is unused, these are vital for debugging.
+ *	static NEEDED int counter;
+ *	static NEEDED void dump_counter(void)
+ *	{
+ *		printf("Counter is %i\n", counter);
+ *	}
+ */
+#define NEEDED __attribute__((__used__))
+#else
+/* Before used, unused functions and vars were always emitted. */
+#define NEEDED __attribute__((__unused__))
+#endif
+#endif
+
+#ifndef UNUSED
+/**
+ * UNUSED - a parameter is unused
+ *
+ * Some compilers (eg. gcc with -W or -Wunused) warn about unused
+ * function parameters.  This suppresses such warnings and indicates
+ * to the reader that it's deliberate.
+ *
+ * Example:
+ *	// This is used as a callback, so needs to have this prototype.
+ *	static int some_callback(void *unused UNUSED)
+ *	{
+ *		return 0;
+ *	}
+ */
+#define UNUSED __attribute__((__unused__))
+#endif
+#else
+#ifndef UNNEEDED
+#define UNNEEDED
+#endif
+#ifndef NEEDED
+#define NEEDED
+#endif
+#ifndef UNUSED
+#define UNUSED
+#endif
+#endif
+
+#ifndef IS_COMPILE_CONSTANT
+#if HAVE_BUILTIN_CONSTANT_P
+/**
+ * IS_COMPILE_CONSTANT - does the compiler know the value of this expression?
+ * @expr: the expression to evaluate
+ *
+ * When an expression manipulation is complicated, it is usually better to
+ * implement it in a function.  However, if the expression being manipulated is
+ * known at compile time, it is better to have the compiler see the entire
+ * expression so it can simply substitute the result.
+ *
+ * This can be done using the IS_COMPILE_CONSTANT() macro.
+ *
+ * Example:
+ *	enum greek { ALPHA, BETA, GAMMA, DELTA, EPSILON };
+ *
+ *	// Out-of-line version.
+ *	const char *greek_name(enum greek greek);
+ *
+ *	// Inline version.
+ *	static inline const char *_greek_name(enum greek greek)
+ *	{
+ *		switch (greek) {
+ *		case ALPHA: return "alpha";
+ *		case BETA: return "beta";
+ *		case GAMMA: return "gamma";
+ *		case DELTA: return "delta";
+ *		case EPSILON: return "epsilon";
+ *		default: return "**INVALID**";
+ *		}
+ *	}
+ *
+ *	// Use inline if compiler knows answer.  Otherwise call function
+ *	// to avoid copies of the same code everywhere.
+ *	#define greek_name(g)						\
+ *		 (IS_COMPILE_CONSTANT(greek) ? _greek_name(g) : greek_name(g))
+ */
+#define IS_COMPILE_CONSTANT(expr) __builtin_constant_p(expr)
+#else
+/* If we don't know, assume it's not. */
+#define IS_COMPILE_CONSTANT(expr) 0
+#endif
+#endif
+
+#ifndef WARN_UNUSED_RESULT
+#if HAVE_WARN_UNUSED_RESULT
+/**
+ * WARN_UNUSED_RESULT - warn if a function return value is unused.
+ *
+ * Used to mark a function where it is extremely unlikely that the caller
+ * can ignore the result, eg realloc().
+ *
+ * Example:
+ * // buf param may be freed by this; need return value!
+ * static char *WARN_UNUSED_RESULT enlarge(char *buf, unsigned *size)
+ * {
+ *	return realloc(buf, (*size) *= 2);
+ * }
+ */
+#define WARN_UNUSED_RESULT __attribute__((__warn_unused_result__))
+#else
+#define WARN_UNUSED_RESULT
+#endif
+#endif
+
+
+#if HAVE_ATTRIBUTE_DEPRECATED
+/**
+ * WARN_DEPRECATED - warn that a function/type/variable is deprecated when used.
+ *
+ * Used to mark a function, type or variable should not be used.
+ *
+ * Example:
+ * WARN_DEPRECATED char *oldfunc(char *buf);
+ */
+#define WARN_DEPRECATED __attribute__((__deprecated__))
+#else
+#define WARN_DEPRECATED
+#endif
+
+
+#if HAVE_ATTRIBUTE_NONNULL
+/**
+ * NO_NULL_ARGS - specify that no arguments to this function can be NULL.
+ *
+ * The compiler will warn if any pointer args are NULL.
+ *
+ * Example:
+ * NO_NULL_ARGS char *my_copy(char *buf);
+ */
+#define NO_NULL_ARGS __attribute__((__nonnull__))
+
+/**
+ * NON_NULL_ARGS - specify that some arguments to this function can't be NULL.
+ * @...: 1-based argument numbers for which args can't be NULL.
+ *
+ * The compiler will warn if any of the specified pointer args are NULL.
+ *
+ * Example:
+ * char *my_copy2(char *buf, char *maybenull) NON_NULL_ARGS(1);
+ */
+#define NON_NULL_ARGS(...) __attribute__((__nonnull__(__VA_ARGS__)))
+#else
+#define NO_NULL_ARGS
+#define NON_NULL_ARGS(...)
+#endif
+
+#if HAVE_ATTRIBUTE_RETURNS_NONNULL
+/**
+ * RETURNS_NONNULL - specify that this function cannot return NULL.
+ *
+ * Mainly an optimization opportunity, but can also suppress warnings.
+ *
+ * Example:
+ * RETURNS_NONNULL char *my_copy(char *buf);
+ */
+#define RETURNS_NONNULL __attribute__((__returns_nonnull__))
+#else
+#define RETURNS_NONNULL
+#endif
+
+#if HAVE_ATTRIBUTE_SENTINEL
+/**
+ * LAST_ARG_NULL - specify the last argument of a variadic function must be NULL.
+ *
+ * The compiler will warn if the last argument isn't NULL.
+ *
+ * Example:
+ * char *join_string(char *buf, ...) LAST_ARG_NULL;
+ */
+#define LAST_ARG_NULL __attribute__((__sentinel__))
+#else
+#define LAST_ARG_NULL
+#endif
+
+#if HAVE_BUILTIN_CPU_SUPPORTS
+/**
+ * cpu_supports - test if current CPU supports the named feature.
+ *
+ * This takes a literal string, and currently only works on glibc platforms.
+ *
+ * Example:
+ * if (cpu_supports("mmx"))
+ *	printf("MMX support engaged!\n");
+ */
+#define cpu_supports(x) __builtin_cpu_supports(x)
+#else
+#define cpu_supports(x) 0
+#endif /* HAVE_BUILTIN_CPU_SUPPORTS */
+
+#endif /* CCAN_COMPILER_H */
diff --git a/src/common/libccan/ccan/compiler/test/compile_fail-printf.c b/src/common/libccan/ccan/compiler/test/compile_fail-printf.c
new file mode 100644
index 000000000000..7664f65d4803
--- /dev/null
+++ b/src/common/libccan/ccan/compiler/test/compile_fail-printf.c
@@ -0,0 +1,24 @@
+#include <ccan/compiler/compiler.h>
+
+static void PRINTF_FMT(2,3) my_printf(int x, const char *fmt, ...)
+{
+	(void)x;
+	(void)fmt;
+}
+
+int main(void)
+{
+	unsigned int i = 0;
+
+	my_printf(1, "Not a pointer "
+#ifdef FAIL
+		  "%p",
+#if !HAVE_ATTRIBUTE_PRINTF
+#error "Unfortunately we don't fail if !HAVE_ATTRIBUTE_PRINTF."
+#endif
+#else
+		  "%i",
+#endif
+		  i);
+	return 0;
+}
diff --git a/src/common/libccan/ccan/compiler/test/run-is_compile_constant.c b/src/common/libccan/ccan/compiler/test/run-is_compile_constant.c
new file mode 100644
index 000000000000..c914c6830086
--- /dev/null
+++ b/src/common/libccan/ccan/compiler/test/run-is_compile_constant.c
@@ -0,0 +1,17 @@
+#include <ccan/compiler/compiler.h>
+#include <ccan/tap/tap.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+	(void)argv;
+	plan_tests(2);
+
+	ok1(!IS_COMPILE_CONSTANT(argc));
+#if HAVE_BUILTIN_CONSTANT_P
+	ok1(IS_COMPILE_CONSTANT(7));
+#else
+	pass("If !HAVE_BUILTIN_CONSTANT_P, IS_COMPILE_CONSTANT always false");
+#endif
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/container_of/LICENSE b/src/common/libccan/ccan/container_of/LICENSE
new file mode 120000
index 000000000000..b7951dabdc82
--- /dev/null
+++ b/src/common/libccan/ccan/container_of/LICENSE
@@ -0,0 +1 @@
+../../licenses/CC0
\ No newline at end of file
diff --git a/src/common/libccan/ccan/container_of/_info b/src/common/libccan/ccan/container_of/_info
new file mode 100644
index 000000000000..b1160522edd4
--- /dev/null
+++ b/src/common/libccan/ccan/container_of/_info
@@ -0,0 +1,65 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * container_of - routine for upcasting
+ *
+ * It is often convenient to create code where the caller registers a pointer
+ * to a generic structure and a callback.  The callback might know that the
+ * pointer points to within a larger structure, and container_of gives a
+ * convenient and fairly type-safe way of returning to the enclosing structure.
+ *
+ * This idiom is an alternative to providing a void * pointer for every
+ * callback.
+ *
+ * Example:
+ *	#include <stdio.h>
+ *	#include <ccan/container_of/container_of.h>
+ *
+ *	struct timer {
+ *		void *members;
+ *	};
+ *
+ *	struct info {
+ *		int my_stuff;
+ *		struct timer timer;
+ *	};
+ *
+ *	static void my_timer_callback(struct timer *timer)
+ *	{
+ *		struct info *info = container_of(timer, struct info, timer);
+ *		printf("my_stuff is %u\n", info->my_stuff);
+ *	}
+ *
+ *	static void register_timer(struct timer *timer)
+ *	{
+ *		(void)timer;
+ *		(void)my_timer_callback;
+ *		//...
+ *	}
+ *
+ *	int main(void)
+ *	{
+ *		struct info info = { .my_stuff = 1 };
+ *
+ *		register_timer(&info.timer);
+ *		// ...
+ *		return 0;
+ *	}
+ *
+ * License: CC0 (Public domain)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/check_type\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/container_of/container_of.h b/src/common/libccan/ccan/container_of/container_of.h
new file mode 100644
index 000000000000..47a34d853b4c
--- /dev/null
+++ b/src/common/libccan/ccan/container_of/container_of.h
@@ -0,0 +1,145 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#ifndef CCAN_CONTAINER_OF_H
+#define CCAN_CONTAINER_OF_H
+#include <stddef.h>
+
+#include "config.h"
+#include <ccan/check_type/check_type.h>
+
+/**
+ * container_of - get pointer to enclosing structure
+ * @member_ptr: pointer to the structure member
+ * @containing_type: the type this member is within
+ * @member: the name of this member within the structure.
+ *
+ * Given a pointer to a member of a structure, this macro does pointer
+ * subtraction to return the pointer to the enclosing type.
+ *
+ * Example:
+ *	struct foo {
+ *		int fielda, fieldb;
+ *		// ...
+ *	};
+ *	struct info {
+ *		int some_other_field;
+ *		struct foo my_foo;
+ *	};
+ *
+ *	static struct info *foo_to_info(struct foo *foo)
+ *	{
+ *		return container_of(foo, struct info, my_foo);
+ *	}
+ */
+#define container_of(member_ptr, containing_type, member)		\
+	 ((containing_type *)						\
+	  ((char *)(member_ptr)						\
+	   - container_off(containing_type, member))			\
+	  + check_types_match(*(member_ptr), ((containing_type *)0)->member))
+
+
+/**
+ * container_of_or_null - get pointer to enclosing structure, or NULL
+ * @member_ptr: pointer to the structure member
+ * @containing_type: the type this member is within
+ * @member: the name of this member within the structure.
+ *
+ * Given a pointer to a member of a structure, this macro does pointer
+ * subtraction to return the pointer to the enclosing type, unless it
+ * is given NULL, in which case it also returns NULL.
+ *
+ * Example:
+ *	struct foo {
+ *		int fielda, fieldb;
+ *		// ...
+ *	};
+ *	struct info {
+ *		int some_other_field;
+ *		struct foo my_foo;
+ *	};
+ *
+ *	static struct info *foo_to_info_allowing_null(struct foo *foo)
+ *	{
+ *		return container_of_or_null(foo, struct info, my_foo);
+ *	}
+ */
+static inline char *container_of_or_null_(void *member_ptr, size_t offset)
+{
+	return member_ptr ? (char *)member_ptr - offset : NULL;
+}
+#define container_of_or_null(member_ptr, containing_type, member)	\
+	((containing_type *)						\
+	 container_of_or_null_(member_ptr,				\
+			       container_off(containing_type, member))	\
+	 + check_types_match(*(member_ptr), ((containing_type *)0)->member))
+
+/**
+ * container_off - get offset to enclosing structure
+ * @containing_type: the type this member is within
+ * @member: the name of this member within the structure.
+ *
+ * Given a pointer to a member of a structure, this macro does
+ * typechecking and figures out the offset to the enclosing type.
+ *
+ * Example:
+ *	struct foo {
+ *		int fielda, fieldb;
+ *		// ...
+ *	};
+ *	struct info {
+ *		int some_other_field;
+ *		struct foo my_foo;
+ *	};
+ *
+ *	static struct info *foo_to_info(struct foo *foo)
+ *	{
+ *		size_t off = container_off(struct info, my_foo);
+ *		return (void *)((char *)foo - off);
+ *	}
+ */
+#define container_off(containing_type, member)	\
+	offsetof(containing_type, member)
+
+/**
+ * container_of_var - get pointer to enclosing structure using a variable
+ * @member_ptr: pointer to the structure member
+ * @container_var: a pointer of same type as this member's container
+ * @member: the name of this member within the structure.
+ *
+ * Given a pointer to a member of a structure, this macro does pointer
+ * subtraction to return the pointer to the enclosing type.
+ *
+ * Example:
+ *	static struct info *foo_to_i(struct foo *foo)
+ *	{
+ *		struct info *i = container_of_var(foo, i, my_foo);
+ *		return i;
+ *	}
+ */
+#if HAVE_TYPEOF
+#define container_of_var(member_ptr, container_var, member) \
+	container_of(member_ptr, typeof(*container_var), member)
+#else
+#define container_of_var(member_ptr, container_var, member)	\
+	((void *)((char *)(member_ptr)	-			\
+		  container_off_var(container_var, member)))
+#endif
+
+/**
+ * container_off_var - get offset of a field in enclosing structure
+ * @container_var: a pointer to a container structure
+ * @member: the name of a member within the structure.
+ *
+ * Given (any) pointer to a structure and a its member name, this
+ * macro does pointer subtraction to return offset of member in a
+ * structure memory layout.
+ *
+ */
+#if HAVE_TYPEOF
+#define container_off_var(var, member)		\
+	container_off(typeof(*var), member)
+#else
+#define container_off_var(var, member)			\
+	((const char *)&(var)->member - (const char *)(var))
+#endif
+
+#endif /* CCAN_CONTAINER_OF_H */
diff --git a/src/common/libccan/ccan/container_of/test/compile_fail-bad-type.c b/src/common/libccan/ccan/container_of/test/compile_fail-bad-type.c
new file mode 100644
index 000000000000..55a911a1b006
--- /dev/null
+++ b/src/common/libccan/ccan/container_of/test/compile_fail-bad-type.c
@@ -0,0 +1,22 @@
+#include <ccan/container_of/container_of.h>
+#include <stdlib.h>
+
+struct foo {
+	int a;
+	char b;
+};
+
+int main(void)
+{
+	struct foo foo = { .a = 1, .b = 2 };
+	int *intp = &foo.a;
+	char *p;
+
+#ifdef FAIL
+	/* p is a char *, but this gives a struct foo * */
+	p = container_of(intp, struct foo, a);
+#else
+	p = (char *)intp;
+#endif
+	return p == NULL;
+}
diff --git a/src/common/libccan/ccan/container_of/test/compile_fail-types.c b/src/common/libccan/ccan/container_of/test/compile_fail-types.c
new file mode 100644
index 000000000000..fbb97a9ee4f9
--- /dev/null
+++ b/src/common/libccan/ccan/container_of/test/compile_fail-types.c
@@ -0,0 +1,22 @@
+#include <ccan/container_of/container_of.h>
+#include <stdlib.h>
+
+struct foo {
+	int a;
+	char b;
+};
+
+int main(void)
+{
+	struct foo foo = { .a = 1, .b = 2 }, *foop;
+	int *intp = &foo.a;
+
+#ifdef FAIL
+	/* b is a char, but intp is an int * */
+	foop = container_of(intp, struct foo, b);
+#else
+	foop = NULL;
+#endif
+	(void) foop; /* Suppress unused-but-set-variable warning. */
+	return intp == NULL;
+}
diff --git a/src/common/libccan/ccan/container_of/test/compile_fail-var-types.c b/src/common/libccan/ccan/container_of/test/compile_fail-var-types.c
new file mode 100644
index 000000000000..ecdd90916fa6
--- /dev/null
+++ b/src/common/libccan/ccan/container_of/test/compile_fail-var-types.c
@@ -0,0 +1,25 @@
+#include <ccan/container_of/container_of.h>
+#include <stdlib.h>
+
+struct foo {
+	int a;
+	char b;
+};
+
+int main(void)
+{
+	struct foo foo = { .a = 1, .b = 2 }, *foop;
+	int *intp = &foo.a;
+
+#ifdef FAIL
+	/* b is a char, but intp is an int * */
+	foop = container_of_var(intp, foop, b);
+#if !HAVE_TYPEOF
+#error "Unfortunately we don't fail if we don't have typeof."
+#endif
+#else
+	foop = NULL;
+#endif
+	(void) foop; /* Suppress unused-but-set-variable warning. */
+	return intp == NULL;
+}
diff --git a/src/common/libccan/ccan/container_of/test/run.c b/src/common/libccan/ccan/container_of/test/run.c
new file mode 100644
index 000000000000..32557295ddec
--- /dev/null
+++ b/src/common/libccan/ccan/container_of/test/run.c
@@ -0,0 +1,30 @@
+#include <ccan/container_of/container_of.h>
+#include <ccan/tap/tap.h>
+
+struct foo {
+	int a;
+	char b;
+};
+
+int main(void)
+{
+	struct foo foo = { .a = 1, .b = 2 };
+	int *intp = &foo.a;
+	char *charp = &foo.b;
+
+	plan_tests(12);
+	ok1(container_of(intp, struct foo, a) == &foo);
+	ok1(container_of(charp, struct foo, b) == &foo);
+	ok1(container_of_or_null(intp, struct foo, a) == &foo);
+	ok1(container_of_or_null(charp, struct foo, b) == &foo);
+	ok1(container_of_or_null((int *)NULL, struct foo, a) == NULL);
+	ok1(container_of_or_null((char *)NULL, struct foo, b) == NULL);
+	ok1(container_of_var(intp, &foo, a) == &foo);
+	ok1(container_of_var(charp, &foo, b) == &foo);
+
+	ok1(container_off(struct foo, a) == 0);
+	ok1(container_off(struct foo, b) == offsetof(struct foo, b));
+	ok1(container_off_var(&foo, a) == 0);
+	ok1(container_off_var(&foo, b) == offsetof(struct foo, b));
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/endian/LICENSE b/src/common/libccan/ccan/endian/LICENSE
new file mode 120000
index 000000000000..b7951dabdc82
--- /dev/null
+++ b/src/common/libccan/ccan/endian/LICENSE
@@ -0,0 +1 @@
+../../licenses/CC0
\ No newline at end of file
diff --git a/src/common/libccan/ccan/endian/_info b/src/common/libccan/ccan/endian/_info
new file mode 100644
index 000000000000..efe5a8bbde75
--- /dev/null
+++ b/src/common/libccan/ccan/endian/_info
@@ -0,0 +1,55 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * endian - endian conversion macros for simple types
+ *
+ * Portable protocols (such as on-disk formats, or network protocols)
+ * are often defined to be a particular endian: little-endian (least
+ * significant bytes first) or big-endian (most significant bytes
+ * first).
+ *
+ * Similarly, some CPUs lay out values in memory in little-endian
+ * order (most commonly, Intel's 8086 and derivatives), or big-endian
+ * order (almost everyone else).
+ *
+ * This module provides conversion routines, inspired by the linux kernel.
+ * It also provides leint32_t, beint32_t etc typedefs, which are annotated for
+ * the sparse checker.
+ *
+ * Example:
+ *	#include <stdio.h>
+ *	#include <err.h>
+ *	#include <ccan/endian/endian.h>
+ *
+ *	// 
+ *	int main(int argc, char *argv[])
+ *	{
+ *		uint32_t value;
+ *
+ *		if (argc != 2)
+ *			errx(1, "Usage: %s <value>", argv[0]);
+ *
+ *		value = atoi(argv[1]);
+ *		printf("native:        %08x\n", value);
+ *		printf("little-endian: %08x\n", cpu_to_le32(value));
+ *		printf("big-endian:    %08x\n", cpu_to_be32(value));
+ *		printf("byte-reversed: %08x\n", bswap_32(value));
+ *		exit(0);
+ *	}
+ *
+ * License: License: CC0 (Public domain)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0)
+		/* Nothing */
+		return 0;
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/endian/endian.h b/src/common/libccan/ccan/endian/endian.h
new file mode 100644
index 000000000000..3753f49003df
--- /dev/null
+++ b/src/common/libccan/ccan/endian/endian.h
@@ -0,0 +1,363 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#ifndef CCAN_ENDIAN_H
+#define CCAN_ENDIAN_H
+#include <stdint.h>
+#include "config.h"
+
+/**
+ * BSWAP_16 - reverse bytes in a constant uint16_t value.
+ * @val: constant value whose bytes to swap.
+ *
+ * Designed to be usable in constant-requiring initializers.
+ *
+ * Example:
+ *	struct mystruct {
+ *		char buf[BSWAP_16(0x1234)];
+ *	};
+ */
+#define BSWAP_16(val)				\
+	((((uint16_t)(val) & 0x00ff) << 8)	\
+	 | (((uint16_t)(val) & 0xff00) >> 8))
+
+/**
+ * BSWAP_32 - reverse bytes in a constant uint32_t value.
+ * @val: constant value whose bytes to swap.
+ *
+ * Designed to be usable in constant-requiring initializers.
+ *
+ * Example:
+ *	struct mystruct {
+ *		char buf[BSWAP_32(0xff000000)];
+ *	};
+ */
+#define BSWAP_32(val)					\
+	((((uint32_t)(val) & 0x000000ff) << 24)		\
+	 | (((uint32_t)(val) & 0x0000ff00) << 8)		\
+	 | (((uint32_t)(val) & 0x00ff0000) >> 8)		\
+	 | (((uint32_t)(val) & 0xff000000) >> 24))
+
+/**
+ * BSWAP_64 - reverse bytes in a constant uint64_t value.
+ * @val: constantvalue whose bytes to swap.
+ *
+ * Designed to be usable in constant-requiring initializers.
+ *
+ * Example:
+ *	struct mystruct {
+ *		char buf[BSWAP_64(0xff00000000000000ULL)];
+ *	};
+ */
+#define BSWAP_64(val)						\
+	((((uint64_t)(val) & 0x00000000000000ffULL) << 56)	\
+	 | (((uint64_t)(val) & 0x000000000000ff00ULL) << 40)	\
+	 | (((uint64_t)(val) & 0x0000000000ff0000ULL) << 24)	\
+	 | (((uint64_t)(val) & 0x00000000ff000000ULL) << 8)	\
+	 | (((uint64_t)(val) & 0x000000ff00000000ULL) >> 8)	\
+	 | (((uint64_t)(val) & 0x0000ff0000000000ULL) >> 24)	\
+	 | (((uint64_t)(val) & 0x00ff000000000000ULL) >> 40)	\
+	 | (((uint64_t)(val) & 0xff00000000000000ULL) >> 56))
+
+#if HAVE_BYTESWAP_H
+#include <byteswap.h>
+#else
+/**
+ * bswap_16 - reverse bytes in a uint16_t value.
+ * @val: value whose bytes to swap.
+ *
+ * Example:
+ *	// Output contains "1024 is 4 as two bytes reversed"
+ *	printf("1024 is %u as two bytes reversed\n", bswap_16(1024));
+ */
+static inline uint16_t bswap_16(uint16_t val)
+{
+	return BSWAP_16(val);
+}
+
+/**
+ * bswap_32 - reverse bytes in a uint32_t value.
+ * @val: value whose bytes to swap.
+ *
+ * Example:
+ *	// Output contains "1024 is 262144 as four bytes reversed"
+ *	printf("1024 is %u as four bytes reversed\n", bswap_32(1024));
+ */
+static inline uint32_t bswap_32(uint32_t val)
+{
+	return BSWAP_32(val);
+}
+#endif /* !HAVE_BYTESWAP_H */
+
+#if !HAVE_BSWAP_64
+/**
+ * bswap_64 - reverse bytes in a uint64_t value.
+ * @val: value whose bytes to swap.
+ *
+ * Example:
+ *	// Output contains "1024 is 1125899906842624 as eight bytes reversed"
+ *	printf("1024 is %llu as eight bytes reversed\n",
+ *		(unsigned long long)bswap_64(1024));
+ */
+static inline uint64_t bswap_64(uint64_t val)
+{
+	return BSWAP_64(val);
+}
+#endif
+
+/* Needed for Glibc like endiness check */
+#define	__LITTLE_ENDIAN	1234
+#define	__BIG_ENDIAN	4321
+
+/* Sanity check the defines.  We don't handle weird endianness. */
+#if !HAVE_LITTLE_ENDIAN && !HAVE_BIG_ENDIAN
+#error "Unknown endian"
+#elif HAVE_LITTLE_ENDIAN && HAVE_BIG_ENDIAN
+#error "Can't compile for both big and little endian."
+#elif HAVE_LITTLE_ENDIAN
+#ifndef __BYTE_ORDER
+#define __BYTE_ORDER	__LITTLE_ENDIAN
+#elif __BYTE_ORDER != __LITTLE_ENDIAN
+#error "__BYTE_ORDER already defined, but not equal to __LITTLE_ENDIAN"
+#endif
+#elif HAVE_BIG_ENDIAN
+#ifndef __BYTE_ORDER
+#define __BYTE_ORDER	__BIG_ENDIAN
+#elif __BYTE_ORDER != __BIG_ENDIAN
+#error "__BYTE_ORDER already defined, but not equal to __BIG_ENDIAN"
+#endif
+#endif
+
+
+#ifdef __CHECKER__
+/* sparse needs forcing to remove bitwise attribute from ccan/short_types */
+#define ENDIAN_CAST __attribute__((force))
+#define ENDIAN_TYPE __attribute__((bitwise))
+#else
+#define ENDIAN_CAST
+#define ENDIAN_TYPE
+#endif
+
+typedef uint64_t ENDIAN_TYPE leint64_t;
+typedef uint64_t ENDIAN_TYPE beint64_t;
+typedef uint32_t ENDIAN_TYPE leint32_t;
+typedef uint32_t ENDIAN_TYPE beint32_t;
+typedef uint16_t ENDIAN_TYPE leint16_t;
+typedef uint16_t ENDIAN_TYPE beint16_t;
+
+#if HAVE_LITTLE_ENDIAN
+/**
+ * CPU_TO_LE64 - convert a constant uint64_t value to little-endian
+ * @native: constant to convert
+ */
+#define CPU_TO_LE64(native) ((ENDIAN_CAST leint64_t)(native))
+
+/**
+ * CPU_TO_LE32 - convert a constant uint32_t value to little-endian
+ * @native: constant to convert
+ */
+#define CPU_TO_LE32(native) ((ENDIAN_CAST leint32_t)(native))
+
+/**
+ * CPU_TO_LE16 - convert a constant uint16_t value to little-endian
+ * @native: constant to convert
+ */
+#define CPU_TO_LE16(native) ((ENDIAN_CAST leint16_t)(native))
+
+/**
+ * LE64_TO_CPU - convert a little-endian uint64_t constant
+ * @le_val: little-endian constant to convert
+ */
+#define LE64_TO_CPU(le_val) ((ENDIAN_CAST uint64_t)(le_val))
+
+/**
+ * LE32_TO_CPU - convert a little-endian uint32_t constant
+ * @le_val: little-endian constant to convert
+ */
+#define LE32_TO_CPU(le_val) ((ENDIAN_CAST uint32_t)(le_val))
+
+/**
+ * LE16_TO_CPU - convert a little-endian uint16_t constant
+ * @le_val: little-endian constant to convert
+ */
+#define LE16_TO_CPU(le_val) ((ENDIAN_CAST uint16_t)(le_val))
+
+#else /* ... HAVE_BIG_ENDIAN */
+#define CPU_TO_LE64(native) ((ENDIAN_CAST leint64_t)BSWAP_64(native))
+#define CPU_TO_LE32(native) ((ENDIAN_CAST leint32_t)BSWAP_32(native))
+#define CPU_TO_LE16(native) ((ENDIAN_CAST leint16_t)BSWAP_16(native))
+#define LE64_TO_CPU(le_val) BSWAP_64((ENDIAN_CAST uint64_t)le_val)
+#define LE32_TO_CPU(le_val) BSWAP_32((ENDIAN_CAST uint32_t)le_val)
+#define LE16_TO_CPU(le_val) BSWAP_16((ENDIAN_CAST uint16_t)le_val)
+#endif /* HAVE_BIG_ENDIAN */
+
+#if HAVE_BIG_ENDIAN
+/**
+ * CPU_TO_BE64 - convert a constant uint64_t value to big-endian
+ * @native: constant to convert
+ */
+#define CPU_TO_BE64(native) ((ENDIAN_CAST beint64_t)(native))
+
+/**
+ * CPU_TO_BE32 - convert a constant uint32_t value to big-endian
+ * @native: constant to convert
+ */
+#define CPU_TO_BE32(native) ((ENDIAN_CAST beint32_t)(native))
+
+/**
+ * CPU_TO_BE16 - convert a constant uint16_t value to big-endian
+ * @native: constant to convert
+ */
+#define CPU_TO_BE16(native) ((ENDIAN_CAST beint16_t)(native))
+
+/**
+ * BE64_TO_CPU - convert a big-endian uint64_t constant
+ * @le_val: big-endian constant to convert
+ */
+#define BE64_TO_CPU(le_val) ((ENDIAN_CAST uint64_t)(le_val))
+
+/**
+ * BE32_TO_CPU - convert a big-endian uint32_t constant
+ * @le_val: big-endian constant to convert
+ */
+#define BE32_TO_CPU(le_val) ((ENDIAN_CAST uint32_t)(le_val))
+
+/**
+ * BE16_TO_CPU - convert a big-endian uint16_t constant
+ * @le_val: big-endian constant to convert
+ */
+#define BE16_TO_CPU(le_val) ((ENDIAN_CAST uint16_t)(le_val))
+
+#else /* ... HAVE_LITTLE_ENDIAN */
+#define CPU_TO_BE64(native) ((ENDIAN_CAST beint64_t)BSWAP_64(native))
+#define CPU_TO_BE32(native) ((ENDIAN_CAST beint32_t)BSWAP_32(native))
+#define CPU_TO_BE16(native) ((ENDIAN_CAST beint16_t)BSWAP_16(native))
+#define BE64_TO_CPU(le_val) BSWAP_64((ENDIAN_CAST uint64_t)le_val)
+#define BE32_TO_CPU(le_val) BSWAP_32((ENDIAN_CAST uint32_t)le_val)
+#define BE16_TO_CPU(le_val) BSWAP_16((ENDIAN_CAST uint16_t)le_val)
+#endif /* HAVE_LITTE_ENDIAN */
+
+
+/**
+ * cpu_to_le64 - convert a uint64_t value to little-endian
+ * @native: value to convert
+ */
+static inline leint64_t cpu_to_le64(uint64_t native)
+{
+	return CPU_TO_LE64(native);
+}
+
+/**
+ * cpu_to_le32 - convert a uint32_t value to little-endian
+ * @native: value to convert
+ */
+static inline leint32_t cpu_to_le32(uint32_t native)
+{
+	return CPU_TO_LE32(native);
+}
+
+/**
+ * cpu_to_le16 - convert a uint16_t value to little-endian
+ * @native: value to convert
+ */
+static inline leint16_t cpu_to_le16(uint16_t native)
+{
+	return CPU_TO_LE16(native);
+}
+
+/**
+ * le64_to_cpu - convert a little-endian uint64_t value
+ * @le_val: little-endian value to convert
+ */
+static inline uint64_t le64_to_cpu(leint64_t le_val)
+{
+	return LE64_TO_CPU(le_val);
+}
+
+/**
+ * le32_to_cpu - convert a little-endian uint32_t value
+ * @le_val: little-endian value to convert
+ */
+static inline uint32_t le32_to_cpu(leint32_t le_val)
+{
+	return LE32_TO_CPU(le_val);
+}
+
+/**
+ * le16_to_cpu - convert a little-endian uint16_t value
+ * @le_val: little-endian value to convert
+ */
+static inline uint16_t le16_to_cpu(leint16_t le_val)
+{
+	return LE16_TO_CPU(le_val);
+}
+
+/**
+ * cpu_to_be64 - convert a uint64_t value to big endian.
+ * @native: value to convert
+ */
+static inline beint64_t cpu_to_be64(uint64_t native)
+{
+	return CPU_TO_BE64(native);
+}
+
+/**
+ * cpu_to_be32 - convert a uint32_t value to big endian.
+ * @native: value to convert
+ */
+static inline beint32_t cpu_to_be32(uint32_t native)
+{
+	return CPU_TO_BE32(native);
+}
+
+/**
+ * cpu_to_be16 - convert a uint16_t value to big endian.
+ * @native: value to convert
+ */
+static inline beint16_t cpu_to_be16(uint16_t native)
+{
+	return CPU_TO_BE16(native);
+}
+
+/**
+ * be64_to_cpu - convert a big-endian uint64_t value
+ * @be_val: big-endian value to convert
+ */
+static inline uint64_t be64_to_cpu(beint64_t be_val)
+{
+	return BE64_TO_CPU(be_val);
+}
+
+/**
+ * be32_to_cpu - convert a big-endian uint32_t value
+ * @be_val: big-endian value to convert
+ */
+static inline uint32_t be32_to_cpu(beint32_t be_val)
+{
+	return BE32_TO_CPU(be_val);
+}
+
+/**
+ * be16_to_cpu - convert a big-endian uint16_t value
+ * @be_val: big-endian value to convert
+ */
+static inline uint16_t be16_to_cpu(beint16_t be_val)
+{
+	return BE16_TO_CPU(be_val);
+}
+
+/* Whichever they include first, they get these definitions. */
+#ifdef CCAN_SHORT_TYPES_H
+/**
+ * be64/be32/be16 - 64/32/16 bit big-endian representation.
+ */
+typedef beint64_t be64;
+typedef beint32_t be32;
+typedef beint16_t be16;
+
+/**
+ * le64/le32/le16 - 64/32/16 bit little-endian representation.
+ */
+typedef leint64_t le64;
+typedef leint32_t le32;
+typedef leint16_t le16;
+#endif
+#endif /* CCAN_ENDIAN_H */
diff --git a/src/common/libccan/ccan/endian/test/compile_ok-constant.c b/src/common/libccan/ccan/endian/test/compile_ok-constant.c
new file mode 100644
index 000000000000..1aef1dd1997c
--- /dev/null
+++ b/src/common/libccan/ccan/endian/test/compile_ok-constant.c
@@ -0,0 +1,12 @@
+#include <ccan/endian/endian.h>
+
+struct foo {
+	char one[BSWAP_16(0xFF00)];
+	char two[BSWAP_32(0xFF000000)];
+	char three[BSWAP_64(0xFF00000000000000ULL)];
+};
+
+int main(void)
+{
+	return 0;
+}
diff --git a/src/common/libccan/ccan/endian/test/run.c b/src/common/libccan/ccan/endian/test/run.c
new file mode 100644
index 000000000000..9bf47f13a6d6
--- /dev/null
+++ b/src/common/libccan/ccan/endian/test/run.c
@@ -0,0 +1,106 @@
+#include <ccan/endian/endian.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	union {
+		uint64_t u64;
+		unsigned char u64_bytes[8];
+	} u64;
+	union {
+		uint32_t u32;
+		unsigned char u32_bytes[4];
+	} u32;
+	union {
+		uint16_t u16;
+		unsigned char u16_bytes[2];
+	} u16;
+
+	plan_tests(48);
+
+	/* Straight swap tests. */
+	u64.u64_bytes[0] = 0x00;
+	u64.u64_bytes[1] = 0x11;
+	u64.u64_bytes[2] = 0x22;
+	u64.u64_bytes[3] = 0x33;
+	u64.u64_bytes[4] = 0x44;
+	u64.u64_bytes[5] = 0x55;
+	u64.u64_bytes[6] = 0x66;
+	u64.u64_bytes[7] = 0x77;
+	u64.u64 = bswap_64(u64.u64);
+	ok1(u64.u64_bytes[7] == 0x00);
+	ok1(u64.u64_bytes[6] == 0x11);
+	ok1(u64.u64_bytes[5] == 0x22);
+	ok1(u64.u64_bytes[4] == 0x33);
+	ok1(u64.u64_bytes[3] == 0x44);
+	ok1(u64.u64_bytes[2] == 0x55);
+	ok1(u64.u64_bytes[1] == 0x66);
+	ok1(u64.u64_bytes[0] == 0x77);
+
+	u32.u32_bytes[0] = 0x00;
+	u32.u32_bytes[1] = 0x11;
+	u32.u32_bytes[2] = 0x22;
+	u32.u32_bytes[3] = 0x33;
+	u32.u32 = bswap_32(u32.u32);
+	ok1(u32.u32_bytes[3] == 0x00);
+	ok1(u32.u32_bytes[2] == 0x11);
+	ok1(u32.u32_bytes[1] == 0x22);
+	ok1(u32.u32_bytes[0] == 0x33);
+
+	u16.u16_bytes[0] = 0x00;
+	u16.u16_bytes[1] = 0x11;
+	u16.u16 = bswap_16(u16.u16);
+	ok1(u16.u16_bytes[1] == 0x00);
+	ok1(u16.u16_bytes[0] == 0x11);
+
+	/* Endian tests. */
+	u64.u64 = cpu_to_le64(0x0011223344556677ULL);
+	ok1(u64.u64_bytes[0] == 0x77);
+	ok1(u64.u64_bytes[1] == 0x66);
+	ok1(u64.u64_bytes[2] == 0x55);
+	ok1(u64.u64_bytes[3] == 0x44);
+	ok1(u64.u64_bytes[4] == 0x33);
+	ok1(u64.u64_bytes[5] == 0x22);
+	ok1(u64.u64_bytes[6] == 0x11);
+	ok1(u64.u64_bytes[7] == 0x00);
+	ok1(le64_to_cpu(u64.u64) == 0x0011223344556677ULL);
+
+	u64.u64 = cpu_to_be64(0x0011223344556677ULL);
+	ok1(u64.u64_bytes[7] == 0x77);
+	ok1(u64.u64_bytes[6] == 0x66);
+	ok1(u64.u64_bytes[5] == 0x55);
+	ok1(u64.u64_bytes[4] == 0x44);
+	ok1(u64.u64_bytes[3] == 0x33);
+	ok1(u64.u64_bytes[2] == 0x22);
+	ok1(u64.u64_bytes[1] == 0x11);
+	ok1(u64.u64_bytes[0] == 0x00);
+	ok1(be64_to_cpu(u64.u64) == 0x0011223344556677ULL);
+
+	u32.u32 = cpu_to_le32(0x00112233);
+	ok1(u32.u32_bytes[0] == 0x33);
+	ok1(u32.u32_bytes[1] == 0x22);
+	ok1(u32.u32_bytes[2] == 0x11);
+	ok1(u32.u32_bytes[3] == 0x00);
+	ok1(le32_to_cpu(u32.u32) == 0x00112233);
+
+	u32.u32 = cpu_to_be32(0x00112233);
+	ok1(u32.u32_bytes[3] == 0x33);
+	ok1(u32.u32_bytes[2] == 0x22);
+	ok1(u32.u32_bytes[1] == 0x11);
+	ok1(u32.u32_bytes[0] == 0x00);
+	ok1(be32_to_cpu(u32.u32) == 0x00112233);
+
+	u16.u16 = cpu_to_le16(0x0011);
+	ok1(u16.u16_bytes[0] == 0x11);
+	ok1(u16.u16_bytes[1] == 0x00);
+	ok1(le16_to_cpu(u16.u16) == 0x0011);
+
+	u16.u16 = cpu_to_be16(0x0011);
+	ok1(u16.u16_bytes[1] == 0x11);
+	ok1(u16.u16_bytes[0] == 0x00);
+	ok1(be16_to_cpu(u16.u16) == 0x0011);
+
+	exit(exit_status());
+}
diff --git a/src/common/libccan/ccan/list/LICENSE b/src/common/libccan/ccan/list/LICENSE
new file mode 120000
index 000000000000..2354d12945d3
--- /dev/null
+++ b/src/common/libccan/ccan/list/LICENSE
@@ -0,0 +1 @@
+../../licenses/BSD-MIT
\ No newline at end of file
diff --git a/src/common/libccan/ccan/list/_info b/src/common/libccan/ccan/list/_info
new file mode 100644
index 000000000000..c4f3e2a0acf2
--- /dev/null
+++ b/src/common/libccan/ccan/list/_info
@@ -0,0 +1,72 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * list - double linked list routines
+ *
+ * The list header contains routines for manipulating double linked lists.
+ * It defines two types: struct list_head used for anchoring lists, and
+ * struct list_node which is usually embedded in the structure which is placed
+ * in the list.
+ *
+ * Example:
+ *	#include <err.h>
+ *	#include <stdio.h>
+ *	#include <stdlib.h>
+ *	#include <ccan/list/list.h>
+ *
+ *	struct parent {
+ *		const char *name;
+ *		struct list_head children;
+ *		unsigned int num_children;
+ *	};
+ *
+ *	struct child {
+ *		const char *name;
+ *		struct list_node list;
+ *	};
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		struct parent p;
+ *		struct child *c;
+ *		int i;
+ *
+ *		if (argc < 2)
+ *			errx(1, "Usage: %s parent children...", argv[0]);
+ *
+ *		p.name = argv[1];
+ *		list_head_init(&p.children);
+ *		p.num_children = 0;
+ *		for (i = 2; i < argc; i++) {
+ *			c = malloc(sizeof(*c));
+ *			c->name = argv[i];
+ *			list_add(&p.children, &c->list);
+ *			p.num_children++;
+ *		}
+ *
+ *		printf("%s has %u children:", p.name, p.num_children);
+ *		list_for_each(&p.children, c, list)
+ *			printf("%s ", c->name);
+ *		printf("\n");
+ *		return 0;
+ *	}
+ *
+ * License: BSD-MIT
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/str\n");
+		printf("ccan/container_of\n");
+		printf("ccan/check_type\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/list/list.c b/src/common/libccan/ccan/list/list.c
new file mode 100644
index 000000000000..2717fa3f17e5
--- /dev/null
+++ b/src/common/libccan/ccan/list/list.c
@@ -0,0 +1,43 @@
+/* Licensed under BSD-MIT - see LICENSE file for details */
+#include <stdio.h>
+#include <stdlib.h>
+#include "list.h"
+
+static void *corrupt(const char *abortstr,
+		     const struct list_node *head,
+		     const struct list_node *node,
+		     unsigned int count)
+{
+	if (abortstr) {
+		fprintf(stderr,
+			"%s: prev corrupt in node %p (%u) of %p\n",
+			abortstr, node, count, head);
+		abort();
+	}
+	return NULL;
+}
+
+struct list_node *list_check_node(const struct list_node *node,
+				  const char *abortstr)
+{
+	const struct list_node *p, *n;
+	int count = 0;
+
+	for (p = node, n = node->next; n != node; p = n, n = n->next) {
+		count++;
+		if (n->prev != p)
+			return corrupt(abortstr, node, n, count);
+	}
+	/* Check prev on head node. */
+	if (node->prev != p)
+		return corrupt(abortstr, node, node, 0);
+
+	return (struct list_node *)node;
+}
+
+struct list_head *list_check(const struct list_head *h, const char *abortstr)
+{
+	if (!list_check_node(&h->n, abortstr))
+		return NULL;
+	return (struct list_head *)h;
+}
diff --git a/src/common/libccan/ccan/list/list.h b/src/common/libccan/ccan/list/list.h
new file mode 100644
index 000000000000..a15321c59095
--- /dev/null
+++ b/src/common/libccan/ccan/list/list.h
@@ -0,0 +1,842 @@
+/* Licensed under BSD-MIT - see LICENSE file for details */
+#ifndef CCAN_LIST_H
+#define CCAN_LIST_H
+//#define CCAN_LIST_DEBUG 1
+#include <stdbool.h>
+#include <assert.h>
+#include <ccan/str/str.h>
+#include <ccan/container_of/container_of.h>
+#include <ccan/check_type/check_type.h>
+
+/**
+ * struct list_node - an entry in a doubly-linked list
+ * @next: next entry (self if empty)
+ * @prev: previous entry (self if empty)
+ *
+ * This is used as an entry in a linked list.
+ * Example:
+ *	struct child {
+ *		const char *name;
+ *		// Linked list of all us children.
+ *		struct list_node list;
+ *	};
+ */
+struct list_node
+{
+	struct list_node *next, *prev;
+};
+
+/**
+ * struct list_head - the head of a doubly-linked list
+ * @h: the list_head (containing next and prev pointers)
+ *
+ * This is used as the head of a linked list.
+ * Example:
+ *	struct parent {
+ *		const char *name;
+ *		struct list_head children;
+ *		unsigned int num_children;
+ *	};
+ */
+struct list_head
+{
+	struct list_node n;
+};
+
+/**
+ * list_check - check head of a list for consistency
+ * @h: the list_head
+ * @abortstr: the location to print on aborting, or NULL.
+ *
+ * Because list_nodes have redundant information, consistency checking between
+ * the back and forward links can be done.  This is useful as a debugging check.
+ * If @abortstr is non-NULL, that will be printed in a diagnostic if the list
+ * is inconsistent, and the function will abort.
+ *
+ * Returns the list head if the list is consistent, NULL if not (it
+ * can never return NULL if @abortstr is set).
+ *
+ * See also: list_check_node()
+ *
+ * Example:
+ *	static void dump_parent(struct parent *p)
+ *	{
+ *		struct child *c;
+ *
+ *		printf("%s (%u children):\n", p->name, p->num_children);
+ *		list_check(&p->children, "bad child list");
+ *		list_for_each(&p->children, c, list)
+ *			printf(" -> %s\n", c->name);
+ *	}
+ */
+struct list_head *list_check(const struct list_head *h, const char *abortstr);
+
+/**
+ * list_check_node - check node of a list for consistency
+ * @n: the list_node
+ * @abortstr: the location to print on aborting, or NULL.
+ *
+ * Check consistency of the list node is in (it must be in one).
+ *
+ * See also: list_check()
+ *
+ * Example:
+ *	static void dump_child(const struct child *c)
+ *	{
+ *		list_check_node(&c->list, "bad child list");
+ *		printf("%s\n", c->name);
+ *	}
+ */
+struct list_node *list_check_node(const struct list_node *n,
+				  const char *abortstr);
+
+#define LIST_LOC __FILE__  ":" stringify(__LINE__)
+#ifdef CCAN_LIST_DEBUG
+#define list_debug(h, loc) list_check((h), loc)
+#define list_debug_node(n, loc) list_check_node((n), loc)
+#else
+#define list_debug(h, loc) ((void)loc, h)
+#define list_debug_node(n, loc) ((void)loc, n)
+#endif
+
+/**
+ * LIST_HEAD_INIT - initializer for an empty list_head
+ * @name: the name of the list.
+ *
+ * Explicit initializer for an empty list.
+ *
+ * See also:
+ *	LIST_HEAD, list_head_init()
+ *
+ * Example:
+ *	static struct list_head my_list = LIST_HEAD_INIT(my_list);
+ */
+#define LIST_HEAD_INIT(name) { { &(name).n, &(name).n } }
+
+/**
+ * LIST_HEAD - define and initialize an empty list_head
+ * @name: the name of the list.
+ *
+ * The LIST_HEAD macro defines a list_head and initializes it to an empty
+ * list.  It can be prepended by "static" to define a static list_head.
+ *
+ * See also:
+ *	LIST_HEAD_INIT, list_head_init()
+ *
+ * Example:
+ *	static LIST_HEAD(my_global_list);
+ */
+#define LIST_HEAD(name) \
+	struct list_head name = LIST_HEAD_INIT(name)
+
+/**
+ * list_head_init - initialize a list_head
+ * @h: the list_head to set to the empty list
+ *
+ * Example:
+ *	...
+ *	struct parent *parent = malloc(sizeof(*parent));
+ *
+ *	list_head_init(&parent->children);
+ *	parent->num_children = 0;
+ */
+static inline void list_head_init(struct list_head *h)
+{
+	h->n.next = h->n.prev = &h->n;
+}
+
+/**
+ * list_node_init - initialize a list_node
+ * @n: the list_node to link to itself.
+ *
+ * You don't need to use this normally!  But it lets you list_del(@n)
+ * safely.
+ */
+static inline void list_node_init(struct list_node *n)
+{
+	n->next = n->prev = n;
+}
+
+/**
+ * list_add_after - add an entry after an existing node in a linked list
+ * @h: the list_head to add the node to (for debugging)
+ * @p: the existing list_node to add the node after
+ * @n: the new list_node to add to the list.
+ *
+ * The existing list_node must already be a member of the list.
+ * The new list_node does not need to be initialized; it will be overwritten.
+ *
+ * Example:
+ *	struct child c1, c2, c3;
+ *	LIST_HEAD(h);
+ *
+ *	list_add_tail(&h, &c1.list);
+ *	list_add_tail(&h, &c3.list);
+ *	list_add_after(&h, &c1.list, &c2.list);
+ */
+#define list_add_after(h, p, n) list_add_after_(h, p, n, LIST_LOC)
+static inline void list_add_after_(struct list_head *h,
+				   struct list_node *p,
+				   struct list_node *n,
+				   const char *abortstr)
+{
+	n->next = p->next;
+	n->prev = p;
+	p->next->prev = n;
+	p->next = n;
+	(void)list_debug(h, abortstr);
+}
+
+/**
+ * list_add - add an entry at the start of a linked list.
+ * @h: the list_head to add the node to
+ * @n: the list_node to add to the list.
+ *
+ * The list_node does not need to be initialized; it will be overwritten.
+ * Example:
+ *	struct child *child = malloc(sizeof(*child));
+ *
+ *	child->name = "marvin";
+ *	list_add(&parent->children, &child->list);
+ *	parent->num_children++;
+ */
+#define list_add(h, n) list_add_(h, n, LIST_LOC)
+static inline void list_add_(struct list_head *h,
+			     struct list_node *n,
+			     const char *abortstr)
+{
+	list_add_after_(h, &h->n, n, abortstr);
+}
+
+/**
+ * list_add_before - add an entry before an existing node in a linked list
+ * @h: the list_head to add the node to (for debugging)
+ * @p: the existing list_node to add the node before
+ * @n: the new list_node to add to the list.
+ *
+ * The existing list_node must already be a member of the list.
+ * The new list_node does not need to be initialized; it will be overwritten.
+ *
+ * Example:
+ *	list_head_init(&h);
+ *	list_add_tail(&h, &c1.list);
+ *	list_add_tail(&h, &c3.list);
+ *	list_add_before(&h, &c3.list, &c2.list);
+ */
+#define list_add_before(h, p, n) list_add_before_(h, p, n, LIST_LOC)
+static inline void list_add_before_(struct list_head *h,
+				    struct list_node *p,
+				    struct list_node *n,
+				    const char *abortstr)
+{
+	n->next = p;
+	n->prev = p->prev;
+	p->prev->next = n;
+	p->prev = n;
+	(void)list_debug(h, abortstr);
+}
+
+/**
+ * list_add_tail - add an entry at the end of a linked list.
+ * @h: the list_head to add the node to
+ * @n: the list_node to add to the list.
+ *
+ * The list_node does not need to be initialized; it will be overwritten.
+ * Example:
+ *	list_add_tail(&parent->children, &child->list);
+ *	parent->num_children++;
+ */
+#define list_add_tail(h, n) list_add_tail_(h, n, LIST_LOC)
+static inline void list_add_tail_(struct list_head *h,
+				  struct list_node *n,
+				  const char *abortstr)
+{
+	list_add_before_(h, &h->n, n, abortstr);
+}
+
+/**
+ * list_empty - is a list empty?
+ * @h: the list_head
+ *
+ * If the list is empty, returns true.
+ *
+ * Example:
+ *	assert(list_empty(&parent->children) == (parent->num_children == 0));
+ */
+#define list_empty(h) list_empty_(h, LIST_LOC)
+static inline bool list_empty_(const struct list_head *h, const char* abortstr)
+{
+	(void)list_debug(h, abortstr);
+	return h->n.next == &h->n;
+}
+
+/**
+ * list_empty_nodebug - is a list empty (and don't perform debug checks)?
+ * @h: the list_head
+ *
+ * If the list is empty, returns true.
+ * This differs from list_empty() in that if CCAN_LIST_DEBUG is set it
+ * will NOT perform debug checks. Only use this function if you REALLY
+ * know what you're doing.
+ *
+ * Example:
+ *	assert(list_empty_nodebug(&parent->children) == (parent->num_children == 0));
+ */
+#ifndef CCAN_LIST_DEBUG
+#define list_empty_nodebug(h) list_empty(h)
+#else
+static inline bool list_empty_nodebug(const struct list_head *h)
+{
+	return h->n.next == &h->n;
+}
+#endif
+
+/**
+ * list_empty_nocheck - is a list empty?
+ * @h: the list_head
+ *
+ * If the list is empty, returns true. This doesn't perform any
+ * debug check for list consistency, so it can be called without
+ * locks, racing with the list being modified. This is ok for
+ * checks where an incorrect result is not an issue (optimized
+ * bail out path for example).
+ */
+static inline bool list_empty_nocheck(const struct list_head *h)
+{
+	return h->n.next == &h->n;
+}
+
+/**
+ * list_del - delete an entry from an (unknown) linked list.
+ * @n: the list_node to delete from the list.
+ *
+ * Note that this leaves @n in an undefined state; it can be added to
+ * another list, but not deleted again.
+ *
+ * See also:
+ *	list_del_from(), list_del_init()
+ *
+ * Example:
+ *	list_del(&child->list);
+ *	parent->num_children--;
+ */
+#define list_del(n) list_del_(n, LIST_LOC)
+static inline void list_del_(struct list_node *n, const char* abortstr)
+{
+	(void)list_debug_node(n, abortstr);
+	n->next->prev = n->prev;
+	n->prev->next = n->next;
+#ifdef CCAN_LIST_DEBUG
+	/* Catch use-after-del. */
+	n->next = n->prev = NULL;
+#endif
+}
+
+/**
+ * list_del_init - delete a node, and reset it so it can be deleted again.
+ * @n: the list_node to be deleted.
+ *
+ * list_del(@n) or list_del_init() again after this will be safe,
+ * which can be useful in some cases.
+ *
+ * See also:
+ *	list_del_from(), list_del()
+ *
+ * Example:
+ *	list_del_init(&child->list);
+ *	parent->num_children--;
+ */
+#define list_del_init(n) list_del_init_(n, LIST_LOC)
+static inline void list_del_init_(struct list_node *n, const char *abortstr)
+{
+	list_del_(n, abortstr);
+	list_node_init(n);
+}
+
+/**
+ * list_del_from - delete an entry from a known linked list.
+ * @h: the list_head the node is in.
+ * @n: the list_node to delete from the list.
+ *
+ * This explicitly indicates which list a node is expected to be in,
+ * which is better documentation and can catch more bugs.
+ *
+ * See also: list_del()
+ *
+ * Example:
+ *	list_del_from(&parent->children, &child->list);
+ *	parent->num_children--;
+ */
+static inline void list_del_from(struct list_head *h, struct list_node *n)
+{
+#ifdef CCAN_LIST_DEBUG
+	{
+		/* Thorough check: make sure it was in list! */
+		struct list_node *i;
+		for (i = h->n.next; i != n; i = i->next)
+			assert(i != &h->n);
+	}
+#endif /* CCAN_LIST_DEBUG */
+
+	/* Quick test that catches a surprising number of bugs. */
+	assert(!list_empty(h));
+	list_del(n);
+}
+
+/**
+ * list_swap - swap out an entry from an (unknown) linked list for a new one.
+ * @o: the list_node to replace from the list.
+ * @n: the list_node to insert in place of the old one.
+ *
+ * Note that this leaves @o in an undefined state; it can be added to
+ * another list, but not deleted/swapped again.
+ *
+ * See also:
+ *	list_del()
+ *
+ * Example:
+ *	struct child x1, x2;
+ *	LIST_HEAD(xh);
+ *
+ *	list_add(&xh, &x1.list);
+ *	list_swap(&x1.list, &x2.list);
+ */
+#define list_swap(o, n) list_swap_(o, n, LIST_LOC)
+static inline void list_swap_(struct list_node *o,
+			      struct list_node *n,
+			      const char* abortstr)
+{
+	(void)list_debug_node(o, abortstr);
+	*n = *o;
+	n->next->prev = n;
+	n->prev->next = n;
+#ifdef CCAN_LIST_DEBUG
+	/* Catch use-after-del. */
+	o->next = o->prev = NULL;
+#endif
+}
+
+/**
+ * list_entry - convert a list_node back into the structure containing it.
+ * @n: the list_node
+ * @type: the type of the entry
+ * @member: the list_node member of the type
+ *
+ * Example:
+ *	// First list entry is children.next; convert back to child.
+ *	child = list_entry(parent->children.n.next, struct child, list);
+ *
+ * See Also:
+ *	list_top(), list_for_each()
+ */
+#define list_entry(n, type, member) container_of(n, type, member)
+
+/**
+ * list_top - get the first entry in a list
+ * @h: the list_head
+ * @type: the type of the entry
+ * @member: the list_node member of the type
+ *
+ * If the list is empty, returns NULL.
+ *
+ * Example:
+ *	struct child *first;
+ *	first = list_top(&parent->children, struct child, list);
+ *	if (!first)
+ *		printf("Empty list!\n");
+ */
+#define list_top(h, type, member)					\
+	((type *)list_top_((h), list_off_(type, member)))
+
+static inline const void *list_top_(const struct list_head *h, size_t off)
+{
+	if (list_empty(h))
+		return NULL;
+	return (const char *)h->n.next - off;
+}
+
+/**
+ * list_pop - remove the first entry in a list
+ * @h: the list_head
+ * @type: the type of the entry
+ * @member: the list_node member of the type
+ *
+ * If the list is empty, returns NULL.
+ *
+ * Example:
+ *	struct child *one;
+ *	one = list_pop(&parent->children, struct child, list);
+ *	if (!one)
+ *		printf("Empty list!\n");
+ */
+#define list_pop(h, type, member)					\
+	((type *)list_pop_((h), list_off_(type, member)))
+
+static inline const void *list_pop_(const struct list_head *h, size_t off)
+{
+	struct list_node *n;
+
+	if (list_empty(h))
+		return NULL;
+	n = h->n.next;
+	list_del(n);
+	return (const char *)n - off;
+}
+
+/**
+ * list_tail - get the last entry in a list
+ * @h: the list_head
+ * @type: the type of the entry
+ * @member: the list_node member of the type
+ *
+ * If the list is empty, returns NULL.
+ *
+ * Example:
+ *	struct child *last;
+ *	last = list_tail(&parent->children, struct child, list);
+ *	if (!last)
+ *		printf("Empty list!\n");
+ */
+#define list_tail(h, type, member) \
+	((type *)list_tail_((h), list_off_(type, member)))
+
+static inline const void *list_tail_(const struct list_head *h, size_t off)
+{
+	if (list_empty(h))
+		return NULL;
+	return (const char *)h->n.prev - off;
+}
+
+/**
+ * list_for_each - iterate through a list.
+ * @h: the list_head (warning: evaluated multiple times!)
+ * @i: the structure containing the list_node
+ * @member: the list_node member of the structure
+ *
+ * This is a convenient wrapper to iterate @i over the entire list.  It's
+ * a for loop, so you can break and continue as normal.
+ *
+ * Example:
+ *	list_for_each(&parent->children, child, list)
+ *		printf("Name: %s\n", child->name);
+ */
+#define list_for_each(h, i, member)					\
+	list_for_each_off(h, i, list_off_var_(i, member))
+
+/**
+ * list_for_each_rev - iterate through a list backwards.
+ * @h: the list_head
+ * @i: the structure containing the list_node
+ * @member: the list_node member of the structure
+ *
+ * This is a convenient wrapper to iterate @i over the entire list.  It's
+ * a for loop, so you can break and continue as normal.
+ *
+ * Example:
+ *	list_for_each_rev(&parent->children, child, list)
+ *		printf("Name: %s\n", child->name);
+ */
+#define list_for_each_rev(h, i, member)					\
+	list_for_each_rev_off(h, i, list_off_var_(i, member))
+
+/**
+ * list_for_each_rev_safe - iterate through a list backwards,
+ * maybe during deletion
+ * @h: the list_head
+ * @i: the structure containing the list_node
+ * @nxt: the structure containing the list_node
+ * @member: the list_node member of the structure
+ *
+ * This is a convenient wrapper to iterate @i over the entire list backwards.
+ * It's a for loop, so you can break and continue as normal.  The extra
+ * variable * @nxt is used to hold the next element, so you can delete @i
+ * from the list.
+ *
+ * Example:
+ *	struct child *next;
+ *	list_for_each_rev_safe(&parent->children, child, next, list) {
+ *		printf("Name: %s\n", child->name);
+ *	}
+ */
+#define list_for_each_rev_safe(h, i, nxt, member)			\
+	list_for_each_rev_safe_off(h, i, nxt, list_off_var_(i, member))
+
+/**
+ * list_for_each_safe - iterate through a list, maybe during deletion
+ * @h: the list_head
+ * @i: the structure containing the list_node
+ * @nxt: the structure containing the list_node
+ * @member: the list_node member of the structure
+ *
+ * This is a convenient wrapper to iterate @i over the entire list.  It's
+ * a for loop, so you can break and continue as normal.  The extra variable
+ * @nxt is used to hold the next element, so you can delete @i from the list.
+ *
+ * Example:
+ *	list_for_each_safe(&parent->children, child, next, list) {
+ *		list_del(&child->list);
+ *		parent->num_children--;
+ *	}
+ */
+#define list_for_each_safe(h, i, nxt, member)				\
+	list_for_each_safe_off(h, i, nxt, list_off_var_(i, member))
+
+/**
+ * list_next - get the next entry in a list
+ * @h: the list_head
+ * @i: a pointer to an entry in the list.
+ * @member: the list_node member of the structure
+ *
+ * If @i was the last entry in the list, returns NULL.
+ *
+ * Example:
+ *	struct child *second;
+ *	second = list_next(&parent->children, first, list);
+ *	if (!second)
+ *		printf("No second child!\n");
+ */
+#define list_next(h, i, member)						\
+	((list_typeof(i))list_entry_or_null(list_debug(h,		\
+					    __FILE__ ":" stringify(__LINE__)), \
+					    (i)->member.next,		\
+					    list_off_var_((i), member)))
+
+/**
+ * list_prev - get the previous entry in a list
+ * @h: the list_head
+ * @i: a pointer to an entry in the list.
+ * @member: the list_node member of the structure
+ *
+ * If @i was the first entry in the list, returns NULL.
+ *
+ * Example:
+ *	first = list_prev(&parent->children, second, list);
+ *	if (!first)
+ *		printf("Can't go back to first child?!\n");
+ */
+#define list_prev(h, i, member)						\
+	((list_typeof(i))list_entry_or_null(list_debug(h,		\
+					    __FILE__ ":" stringify(__LINE__)), \
+					    (i)->member.prev,		\
+					    list_off_var_((i), member)))
+
+/**
+ * list_append_list - empty one list onto the end of another.
+ * @to: the list to append into
+ * @from: the list to empty.
+ *
+ * This takes the entire contents of @from and moves it to the end of
+ * @to.  After this @from will be empty.
+ *
+ * Example:
+ *	struct list_head adopter;
+ *
+ *	list_append_list(&adopter, &parent->children);
+ *	assert(list_empty(&parent->children));
+ *	parent->num_children = 0;
+ */
+#define list_append_list(t, f) list_append_list_(t, f,			\
+				   __FILE__ ":" stringify(__LINE__))
+static inline void list_append_list_(struct list_head *to,
+				     struct list_head *from,
+				     const char *abortstr)
+{
+	struct list_node *from_tail = list_debug(from, abortstr)->n.prev;
+	struct list_node *to_tail = list_debug(to, abortstr)->n.prev;
+
+	/* Sew in head and entire list. */
+	to->n.prev = from_tail;
+	from_tail->next = &to->n;
+	to_tail->next = &from->n;
+	from->n.prev = to_tail;
+
+	/* Now remove head. */
+	list_del(&from->n);
+	list_head_init(from);
+}
+
+/**
+ * list_prepend_list - empty one list into the start of another.
+ * @to: the list to prepend into
+ * @from: the list to empty.
+ *
+ * This takes the entire contents of @from and moves it to the start
+ * of @to.  After this @from will be empty.
+ *
+ * Example:
+ *	list_prepend_list(&adopter, &parent->children);
+ *	assert(list_empty(&parent->children));
+ *	parent->num_children = 0;
+ */
+#define list_prepend_list(t, f) list_prepend_list_(t, f, LIST_LOC)
+static inline void list_prepend_list_(struct list_head *to,
+				      struct list_head *from,
+				      const char *abortstr)
+{
+	struct list_node *from_tail = list_debug(from, abortstr)->n.prev;
+	struct list_node *to_head = list_debug(to, abortstr)->n.next;
+
+	/* Sew in head and entire list. */
+	to->n.next = &from->n;
+	from->n.prev = &to->n;
+	to_head->prev = from_tail;
+	from_tail->next = to_head;
+
+	/* Now remove head. */
+	list_del(&from->n);
+	list_head_init(from);
+}
+
+/* internal macros, do not use directly */
+#define list_for_each_off_dir_(h, i, off, dir)				\
+	for (i = list_node_to_off_(list_debug(h, LIST_LOC)->n.dir,	\
+				   (off));				\
+	list_node_from_off_((void *)i, (off)) != &(h)->n;		\
+	i = list_node_to_off_(list_node_from_off_((void *)i, (off))->dir, \
+			      (off)))
+
+#define list_for_each_safe_off_dir_(h, i, nxt, off, dir)		\
+	for (i = list_node_to_off_(list_debug(h, LIST_LOC)->n.dir,	\
+				   (off)),				\
+	nxt = list_node_to_off_(list_node_from_off_(i, (off))->dir,	\
+				(off));					\
+	list_node_from_off_(i, (off)) != &(h)->n;			\
+	i = nxt,							\
+	nxt = list_node_to_off_(list_node_from_off_(i, (off))->dir,	\
+				(off)))
+
+/**
+ * list_for_each_off - iterate through a list of memory regions.
+ * @h: the list_head
+ * @i: the pointer to a memory region which contains list node data.
+ * @off: offset(relative to @i) at which list node data resides.
+ *
+ * This is a low-level wrapper to iterate @i over the entire list, used to
+ * implement all oher, more high-level, for-each constructs. It's a for loop,
+ * so you can break and continue as normal.
+ *
+ * WARNING! Being the low-level macro that it is, this wrapper doesn't know
+ * nor care about the type of @i. The only assumption made is that @i points
+ * to a chunk of memory that at some @offset, relative to @i, contains a
+ * properly filled `struct list_node' which in turn contains pointers to
+ * memory chunks and it's turtles all the way down. With all that in mind
+ * remember that given the wrong pointer/offset couple this macro will
+ * happily churn all you memory until SEGFAULT stops it, in other words
+ * caveat emptor.
+ *
+ * It is worth mentioning that one of legitimate use-cases for that wrapper
+ * is operation on opaque types with known offset for `struct list_node'
+ * member(preferably 0), because it allows you not to disclose the type of
+ * @i.
+ *
+ * Example:
+ *	list_for_each_off(&parent->children, child,
+ *				offsetof(struct child, list))
+ *		printf("Name: %s\n", child->name);
+ */
+#define list_for_each_off(h, i, off)                                    \
+	list_for_each_off_dir_((h),(i),(off),next)
+
+/**
+ * list_for_each_rev_off - iterate through a list of memory regions backwards
+ * @h: the list_head
+ * @i: the pointer to a memory region which contains list node data.
+ * @off: offset(relative to @i) at which list node data resides.
+ *
+ * See list_for_each_off for details
+ */
+#define list_for_each_rev_off(h, i, off)                                    \
+	list_for_each_off_dir_((h),(i),(off),prev)
+
+/**
+ * list_for_each_safe_off - iterate through a list of memory regions, maybe
+ * during deletion
+ * @h: the list_head
+ * @i: the pointer to a memory region which contains list node data.
+ * @nxt: the structure containing the list_node
+ * @off: offset(relative to @i) at which list node data resides.
+ *
+ * For details see `list_for_each_off' and `list_for_each_safe'
+ * descriptions.
+ *
+ * Example:
+ *	list_for_each_safe_off(&parent->children, child,
+ *		next, offsetof(struct child, list))
+ *		printf("Name: %s\n", child->name);
+ */
+#define list_for_each_safe_off(h, i, nxt, off)                          \
+	list_for_each_safe_off_dir_((h),(i),(nxt),(off),next)
+
+/**
+ * list_for_each_rev_safe_off - iterate backwards through a list of
+ * memory regions, maybe during deletion
+ * @h: the list_head
+ * @i: the pointer to a memory region which contains list node data.
+ * @nxt: the structure containing the list_node
+ * @off: offset(relative to @i) at which list node data resides.
+ *
+ * For details see `list_for_each_rev_off' and `list_for_each_rev_safe'
+ * descriptions.
+ *
+ * Example:
+ *	list_for_each_rev_safe_off(&parent->children, child,
+ *		next, offsetof(struct child, list))
+ *		printf("Name: %s\n", child->name);
+ */
+#define list_for_each_rev_safe_off(h, i, nxt, off)                      \
+	list_for_each_safe_off_dir_((h),(i),(nxt),(off),prev)
+
+/* Other -off variants. */
+#define list_entry_off(n, type, off)		\
+	((type *)list_node_from_off_((n), (off)))
+
+#define list_head_off(h, type, off)		\
+	((type *)list_head_off((h), (off)))
+
+#define list_tail_off(h, type, off)		\
+	((type *)list_tail_((h), (off)))
+
+#define list_add_off(h, n, off)                 \
+	list_add((h), list_node_from_off_((n), (off)))
+
+#define list_del_off(n, off)                    \
+	list_del(list_node_from_off_((n), (off)))
+
+#define list_del_from_off(h, n, off)			\
+	list_del_from(h, list_node_from_off_((n), (off)))
+
+/* Offset helper functions so we only single-evaluate. */
+static inline void *list_node_to_off_(struct list_node *node, size_t off)
+{
+	return (void *)((char *)node - off);
+}
+static inline struct list_node *list_node_from_off_(void *ptr, size_t off)
+{
+	return (struct list_node *)((char *)ptr + off);
+}
+
+/* Get the offset of the member, but make sure it's a list_node. */
+#define list_off_(type, member)					\
+	(container_off(type, member) +				\
+	 check_type(((type *)0)->member, struct list_node))
+
+#define list_off_var_(var, member)			\
+	(container_off_var(var, member) +		\
+	 check_type(var->member, struct list_node))
+
+#if HAVE_TYPEOF
+#define list_typeof(var) typeof(var)
+#else
+#define list_typeof(var) void *
+#endif
+
+/* Returns member, or NULL if at end of list. */
+static inline void *list_entry_or_null(const struct list_head *h,
+				       const struct list_node *n,
+				       size_t off)
+{
+	if (n == &h->n)
+		return NULL;
+	return (char *)n - off;
+}
+#endif /* CCAN_LIST_H */
diff --git a/src/common/libccan/ccan/list/test/compile_ok-constant.c b/src/common/libccan/ccan/list/test/compile_ok-constant.c
new file mode 100644
index 000000000000..c57cdadc3167
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/compile_ok-constant.c
@@ -0,0 +1,49 @@
+#include <ccan/list/list.h>
+#include <ccan/tap/tap.h>
+#include <ccan/list/list.c>
+#include <stdbool.h>
+#include <stdio.h>
+
+struct child {
+	const char *name;
+	struct list_node list;
+};
+
+static bool children(const struct list_head *list)
+{
+	return !list_empty(list);
+}
+
+static const struct child *first_child(const struct list_head *list)
+{
+	return list_top(list, struct child, list);
+}
+
+static const struct child *last_child(const struct list_head *list)
+{
+	return list_tail(list, struct child, list);
+}
+
+static void check_children(const struct list_head *list)
+{
+	list_check(list, "bad child list");
+}
+
+static void print_children(const struct list_head *list)
+{
+	const struct child *c;
+	list_for_each(list, c, list)
+		printf("%s\n", c->name);
+}
+
+int main(void)
+{
+	LIST_HEAD(h);
+
+	children(&h);
+	first_child(&h);
+	last_child(&h);
+	check_children(&h);
+	print_children(&h);
+	return 0;
+}
diff --git a/src/common/libccan/ccan/list/test/helper.c b/src/common/libccan/ccan/list/test/helper.c
new file mode 100644
index 000000000000..4fb1c5ac32d3
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/helper.c
@@ -0,0 +1,56 @@
+#include <stdlib.h>
+#include <stdbool.h>
+#include <time.h>
+
+#include <ccan/list/list.h>
+#include "helper.h"
+
+#define ANSWER_TO_THE_ULTIMATE_QUESTION_OF_LIFE_THE_UNIVERSE_AND_EVERYTHING \
+  (42)
+
+struct opaque {
+  struct list_node list;
+  size_t secret_offset;
+  char   secret_drawer[42];
+};
+
+static bool not_randomized = true;
+
+struct opaque *create_opaque_blob(void)
+{
+  struct opaque *blob = calloc(1, sizeof(struct opaque));
+
+  if (not_randomized) {
+    srandom((int)time(NULL));
+    not_randomized = false;
+  }
+
+  blob->secret_offset = random() % (sizeof(blob->secret_drawer));
+  blob->secret_drawer[blob->secret_offset] =
+    ANSWER_TO_THE_ULTIMATE_QUESTION_OF_LIFE_THE_UNIVERSE_AND_EVERYTHING;
+
+  return blob;
+}
+
+bool if_blobs_know_the_secret(struct opaque *blob)
+{
+  bool answer = true;
+  int i;
+  for (i = 0; i < sizeof(blob->secret_drawer) /
+               sizeof(blob->secret_drawer[0]); i++)
+          if (i != blob->secret_offset)
+                  answer = answer && (blob->secret_drawer[i] == 0);
+          else
+                  answer = answer &&
+                          (blob->secret_drawer[blob->secret_offset] ==
+           ANSWER_TO_THE_ULTIMATE_QUESTION_OF_LIFE_THE_UNIVERSE_AND_EVERYTHING);
+
+  return answer;
+}
+
+void destroy_opaque_blob(struct opaque *blob)
+{
+  free(blob);
+}
+
+
diff --git a/src/common/libccan/ccan/list/test/helper.h b/src/common/libccan/ccan/list/test/helper.h
new file mode 100644
index 000000000000..4b64a7d6aaf4
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/helper.h
@@ -0,0 +1,9 @@
+/* These are in a separate C file so we can test undefined structures. */
+struct opaque;
+typedef struct opaque opaque_t;
+
+opaque_t *create_opaque_blob(void);
+bool if_blobs_know_the_secret(opaque_t *blob);
+void destroy_opaque_blob(opaque_t *blob);
+
+
diff --git a/src/common/libccan/ccan/list/test/run-CCAN_LIST_DEBUG.c b/src/common/libccan/ccan/list/test/run-CCAN_LIST_DEBUG.c
new file mode 100644
index 000000000000..b8e5165a6441
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/run-CCAN_LIST_DEBUG.c
@@ -0,0 +1,60 @@
+/* Check that CCAN_LIST_DEBUG works */
+#include <setjmp.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <err.h>
+
+/* We don't actually want it to exit... */
+static jmp_buf aborted;
+#define abort() longjmp(aborted, 1)
+
+#define fprintf my_fprintf
+static char printf_buffer[1000];
+
+static int my_fprintf(FILE *stream, const char *format, ...)
+{
+	va_list ap;
+	int ret;
+	(void)stream;
+	va_start(ap, format);
+	ret = vsprintf(printf_buffer, format, ap);
+	va_end(ap);
+	return ret;
+}
+
+#define CCAN_LIST_DEBUG 1
+#include <ccan/list/list.h>
+#include <ccan/tap/tap.h>
+#include <ccan/list/list.c>
+
+int main(void)
+{
+	struct list_head list;
+	struct list_node n1;
+	char expect[100];
+
+	plan_tests(2);
+	/* Empty list. */
+	list.n.next = &list.n;
+	list.n.prev = &list.n;
+	ok1(list_check(&list, NULL) == &list);
+
+	/* Bad back ptr */
+	list.n.prev = &n1;
+
+	/* Aborting version. */
+	sprintf(expect, "run-CCAN_LIST_DEBUG.c:51: prev corrupt in node %p (0) of %p\n",
+		&list, &list);
+	if (setjmp(aborted) == 0) {
+		assert(list_empty(&list));
+		fail("list_empty on empty with bad back ptr didn't fail!");
+	} else {
+		/* __FILE__ might give full path. */
+		int prep = strlen(printf_buffer) - strlen(expect);
+		ok1(prep >= 0 && strcmp(printf_buffer + prep, expect) == 0);
+	}
+
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/list/test/run-check-corrupt.c b/src/common/libccan/ccan/list/test/run-check-corrupt.c
new file mode 100644
index 000000000000..94c2e67e7bed
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/run-check-corrupt.c
@@ -0,0 +1,90 @@
+#include <setjmp.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <err.h>
+
+/* We don't actually want it to exit... */
+static jmp_buf aborted;
+#define abort() longjmp(aborted, 1)
+
+#define fprintf my_fprintf
+static char printf_buffer[1000];
+
+static int my_fprintf(FILE *stream, const char *format, ...)
+{
+	va_list ap;
+	int ret;
+	(void)stream;
+	va_start(ap, format);
+	ret = vsprintf(printf_buffer, format, ap);
+	va_end(ap);
+	return ret;
+}
+
+#include <ccan/list/list.h>
+#include <ccan/tap/tap.h>
+#include <ccan/list/list.c>
+
+int main(void)
+{
+	struct list_head list;
+	struct list_node n1;
+	char expect[100];
+
+	plan_tests(9);
+	/* Empty list. */
+	list.n.next = &list.n;
+	list.n.prev = &list.n;
+	ok1(list_check(&list, NULL) == &list);
+
+	/* Bad back ptr */
+	list.n.prev = &n1;
+	/* Non-aborting version. */
+	ok1(list_check(&list, NULL) == NULL);
+
+	/* Aborting version. */
+	sprintf(expect, "test message: prev corrupt in node %p (0) of %p\n",
+		&list, &list);
+	if (setjmp(aborted) == 0) {
+		list_check(&list, "test message");
+		fail("list_check on empty with bad back ptr didn't fail!");
+	} else {
+		ok1(strcmp(printf_buffer, expect) == 0);
+	}
+
+	/* n1 in list. */
+	list.n.next = &n1;
+	list.n.prev = &n1;
+	n1.prev = &list.n;
+	n1.next = &list.n;
+	ok1(list_check(&list, NULL) == &list);
+	ok1(list_check_node(&n1, NULL) == &n1);
+
+	/* Bad back ptr */
+	n1.prev = &n1;
+	ok1(list_check(&list, NULL) == NULL);
+	ok1(list_check_node(&n1, NULL) == NULL);
+
+	/* Aborting version. */
+	sprintf(expect, "test message: prev corrupt in node %p (1) of %p\n",
+		&n1, &list);
+	if (setjmp(aborted) == 0) {
+		list_check(&list, "test message");
+		fail("list_check on n1 bad back ptr didn't fail!");
+	} else {
+		ok1(strcmp(printf_buffer, expect) == 0);
+	}
+
+	sprintf(expect, "test message: prev corrupt in node %p (0) of %p\n",
+		&n1, &n1);
+	if (setjmp(aborted) == 0) {
+		list_check_node(&n1, "test message");
+		fail("list_check_node on n1 bad back ptr didn't fail!");
+	} else {
+		ok1(strcmp(printf_buffer, expect) == 0);
+	}
+
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/list/test/run-check-nonconst.c b/src/common/libccan/ccan/list/test/run-check-nonconst.c
new file mode 100644
index 000000000000..d3cb652a34a5
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/run-check-nonconst.c
@@ -0,0 +1,27 @@
+#include <ccan/list/list.h>
+#include <ccan/tap/tap.h>
+#include <ccan/list/list.c>
+#include "helper.h"
+
+struct child {
+	const char *name;
+	struct list_node list;
+};
+
+int main(void)
+{
+	struct child c1, c2;
+	struct list_head list = LIST_HEAD_INIT(list);
+
+	plan_tests(1);
+
+	list_add(&list, &c1.list);
+	list_add_tail(list_check(&list, "Bad list!"), &c2.list);
+	list_del_from(list_check(&list, "Bad list!"),
+		      list_check_node(&c2.list, "Bad node!"));
+	list_del_from(list_check(&list, "Bad list!"),
+		      list_check_node(&c1.list, "Bad node!"));
+	ok1(list_empty(list_check(&list, "Bad emptied list")));
+
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/list/test/run-list_del_from-assert.c b/src/common/libccan/ccan/list/test/run-list_del_from-assert.c
new file mode 100644
index 000000000000..9404b7120ee9
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/run-list_del_from-assert.c
@@ -0,0 +1,36 @@
+#define CCAN_LIST_DEBUG 1
+#include <ccan/list/list.h>
+#include <ccan/tap/tap.h>
+#include <ccan/list/list.c>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <signal.h>
+
+int main(void)
+{
+	struct list_head list1, list2;
+	struct list_node n1, n2, n3;
+	pid_t child;
+	int status;
+
+	plan_tests(1);
+	list_head_init(&list1);
+	list_head_init(&list2);
+	list_add(&list1, &n1);
+	list_add(&list2, &n2);
+	list_add_tail(&list2, &n3);
+
+	child = fork();
+	if (child) {
+		wait(&status);
+	} else {
+		/* This should abort. */
+		list_del_from(&list1, &n3);
+		exit(0);
+	}
+
+	ok1(WIFSIGNALED(status) && WTERMSIG(status) == SIGABRT);
+	list_del_from(&list2, &n3);
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/list/test/run-list_prev-list_next.c b/src/common/libccan/ccan/list/test/run-list_prev-list_next.c
new file mode 100644
index 000000000000..cc61e03a01f3
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/run-list_prev-list_next.c
@@ -0,0 +1,65 @@
+#include <ccan/list/list.h>
+#include <ccan/tap/tap.h>
+#include <ccan/list/list.c>
+#include "helper.h"
+
+struct parent {
+	const char *name;
+	unsigned int num_children;
+	struct list_head children;
+};
+
+struct child {
+	const char *name;
+	struct list_node list;
+};
+
+int main(void)
+{
+	struct parent parent;
+	struct child c1, c2, c3;
+	const struct parent *p;
+	const struct child *c;
+
+	plan_tests(20);
+	parent.num_children = 0;
+	list_head_init(&parent.children);
+
+	c1.name = "c1";
+	list_add(&parent.children, &c1.list);
+
+	ok1(list_next(&parent.children, &c1, list) == NULL);
+	ok1(list_prev(&parent.children, &c1, list) == NULL);
+
+	c2.name = "c2";
+	list_add_tail(&parent.children, &c2.list);
+
+	ok1(list_next(&parent.children, &c1, list) == &c2);
+	ok1(list_prev(&parent.children, &c1, list) == NULL);
+	ok1(list_next(&parent.children, &c2, list) == NULL);
+	ok1(list_prev(&parent.children, &c2, list) == &c1);
+
+	c3.name = "c3";
+	list_add_tail(&parent.children, &c3.list);
+
+	ok1(list_next(&parent.children, &c1, list) == &c2);
+	ok1(list_prev(&parent.children, &c1, list) == NULL);
+	ok1(list_next(&parent.children, &c2, list) == &c3);
+	ok1(list_prev(&parent.children, &c2, list) == &c1);
+	ok1(list_next(&parent.children, &c3, list) == NULL);
+	ok1(list_prev(&parent.children, &c3, list) == &c2);
+
+	/* Const variants */
+	p = &parent;
+	c = &c2;
+	ok1(list_next(&p->children, &c1, list) == &c2);
+	ok1(list_prev(&p->children, &c1, list) == NULL);
+	ok1(list_next(&p->children, c, list) == &c3);
+	ok1(list_prev(&p->children, c, list) == &c1);
+	ok1(list_next(&parent.children, c, list) == &c3);
+	ok1(list_prev(&parent.children, c, list) == &c1);
+	ok1(list_next(&p->children, &c3, list) == NULL);
+	ok1(list_prev(&p->children, &c3, list) == &c2);
+
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/list/test/run-prepend_list.c b/src/common/libccan/ccan/list/test/run-prepend_list.c
new file mode 100644
index 000000000000..fecd4196fd70
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/run-prepend_list.c
@@ -0,0 +1,111 @@
+#include <ccan/list/list.h>
+#include <ccan/tap/tap.h>
+#include <ccan/list/list.c>
+#include <stdarg.h>
+
+static bool list_expect(struct list_head *h, ...)
+{
+	va_list ap;
+	struct list_node *n = &h->n, *expected;
+
+	va_start(ap, h);
+	while ((expected = va_arg(ap, struct list_node *)) != NULL) {
+		n = n->next;
+		if (n != expected)
+			return false;
+	}
+	return (n->next == &h->n);
+}
+
+int main(void)
+{
+	struct list_head h1, h2;
+	struct list_node n[4];
+
+	plan_tests(40);
+
+	list_head_init(&h1);
+	list_head_init(&h2);
+
+	/* Append an empty list to an empty list. */
+	list_append_list(&h1, &h2);
+	ok1(list_empty(&h1));
+	ok1(list_empty(&h2));
+	ok1(list_check(&h1, NULL));
+	ok1(list_check(&h2, NULL));
+
+	/* Prepend an empty list to an empty list. */
+	list_prepend_list(&h1, &h2);
+	ok1(list_empty(&h1));
+	ok1(list_empty(&h2));
+	ok1(list_check(&h1, NULL));
+	ok1(list_check(&h2, NULL));
+
+	/* Append an empty list to a non-empty list */
+	list_add(&h1, &n[0]);
+	list_append_list(&h1, &h2);
+	ok1(list_empty(&h2));
+	ok1(list_check(&h1, NULL));
+	ok1(list_check(&h2, NULL));
+	ok1(list_expect(&h1, &n[0], NULL));
+
+	/* Prepend an empty list to a non-empty list */
+	list_prepend_list(&h1, &h2);
+	ok1(list_empty(&h2));
+	ok1(list_check(&h1, NULL));
+	ok1(list_check(&h2, NULL));
+	ok1(list_expect(&h1, &n[0], NULL));
+
+	/* Append a non-empty list to an empty list. */
+	list_append_list(&h2, &h1);
+	ok1(list_empty(&h1));
+	ok1(list_check(&h1, NULL));
+	ok1(list_check(&h2, NULL));
+	ok1(list_expect(&h2, &n[0], NULL));
+
+	/* Prepend a non-empty list to an empty list. */
+	list_prepend_list(&h1, &h2);
+	ok1(list_empty(&h2));
+	ok1(list_check(&h1, NULL));
+	ok1(list_check(&h2, NULL));
+	ok1(list_expect(&h1, &n[0], NULL));
+
+	/* Prepend a non-empty list to non-empty list. */
+	list_add(&h2, &n[1]);
+	list_prepend_list(&h1, &h2);
+	ok1(list_empty(&h2));
+	ok1(list_check(&h1, NULL));
+	ok1(list_check(&h2, NULL));
+	ok1(list_expect(&h1, &n[1], &n[0], NULL));
+
+	/* Append a non-empty list to non-empty list. */
+	list_add(&h2, &n[2]);
+	list_append_list(&h1, &h2);
+	ok1(list_empty(&h2));
+	ok1(list_check(&h1, NULL));
+	ok1(list_check(&h2, NULL));
+	ok1(list_expect(&h1, &n[1], &n[0], &n[2], NULL));
+
+	/* Prepend a 2-entry list to a 2-entry list. */
+	list_del_from(&h1, &n[2]);
+	list_add(&h2, &n[2]);
+	list_add_tail(&h2, &n[3]);
+	list_prepend_list(&h1, &h2);
+	ok1(list_empty(&h2));
+	ok1(list_check(&h1, NULL));
+	ok1(list_check(&h2, NULL));
+	ok1(list_expect(&h1, &n[2], &n[3], &n[1], &n[0], NULL));
+
+	/* Append a 2-entry list to a 2-entry list. */
+	list_del_from(&h1, &n[2]);
+	list_del_from(&h1, &n[3]);
+	list_add(&h2, &n[2]);
+	list_add_tail(&h2, &n[3]);
+	list_append_list(&h1, &h2);
+	ok1(list_empty(&h2));
+	ok1(list_check(&h1, NULL));
+	ok1(list_check(&h2, NULL));
+	ok1(list_expect(&h1, &n[1], &n[0], &n[2], &n[3], NULL));
+
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/list/test/run-single-eval.c b/src/common/libccan/ccan/list/test/run-single-eval.c
new file mode 100644
index 000000000000..db0bffdda5aa
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/run-single-eval.c
@@ -0,0 +1,168 @@
+/* Make sure macros only evaluate their args once. */
+#include <ccan/list/list.h>
+#include <ccan/tap/tap.h>
+#include <ccan/list/list.c>
+
+struct parent {
+	const char *name;
+	struct list_head children;
+	unsigned int num_children;
+	int eval_count;
+};
+
+struct child {
+	const char *name;
+	struct list_node list;
+};
+
+static LIST_HEAD(static_list);
+
+#define ref(obj, counter) ((counter)++, (obj))
+
+int main(void)
+{
+	struct parent parent;
+	struct child c1, c2, c3, *c, *n;
+	unsigned int i;
+	unsigned int static_count = 0, parent_count = 0, list_count = 0,
+		node_count = 0;
+	struct list_head list = LIST_HEAD_INIT(list);
+
+	plan_tests(74);
+	/* Test LIST_HEAD, LIST_HEAD_INIT, list_empty and check_list */
+	ok1(list_empty(ref(&static_list, static_count)));
+	ok1(static_count == 1);
+	ok1(list_check(ref(&static_list, static_count), NULL));
+	ok1(static_count == 2);
+	ok1(list_empty(ref(&list, list_count)));
+	ok1(list_count == 1);
+	ok1(list_check(ref(&list, list_count), NULL));
+	ok1(list_count == 2);
+
+	parent.num_children = 0;
+	list_head_init(ref(&parent.children, parent_count));
+	ok1(parent_count == 1);
+	/* Test list_head_init */
+	ok1(list_empty(ref(&parent.children, parent_count)));
+	ok1(parent_count == 2);
+	ok1(list_check(ref(&parent.children, parent_count), NULL));
+	ok1(parent_count == 3);
+
+	c2.name = "c2";
+	list_add(ref(&parent.children, parent_count), &c2.list);
+	ok1(parent_count == 4);
+	/* Test list_add and !list_empty. */
+	ok1(!list_empty(ref(&parent.children, parent_count)));
+	ok1(parent_count == 5);
+	ok1(c2.list.next == &parent.children.n);
+	ok1(c2.list.prev == &parent.children.n);
+	ok1(parent.children.n.next == &c2.list);
+	ok1(parent.children.n.prev == &c2.list);
+	/* Test list_check */
+	ok1(list_check(ref(&parent.children, parent_count), NULL));
+	ok1(parent_count == 6);
+
+	c1.name = "c1";
+	list_add(ref(&parent.children, parent_count), &c1.list);
+	ok1(parent_count == 7);
+	/* Test list_add and !list_empty. */
+	ok1(!list_empty(ref(&parent.children, parent_count)));
+	ok1(parent_count == 8);
+	ok1(c2.list.next == &parent.children.n);
+	ok1(c2.list.prev == &c1.list);
+	ok1(parent.children.n.next == &c1.list);
+	ok1(parent.children.n.prev == &c2.list);
+	ok1(c1.list.next == &c2.list);
+	ok1(c1.list.prev == &parent.children.n);
+	/* Test list_check */
+	ok1(list_check(ref(&parent.children, parent_count), NULL));
+	ok1(parent_count == 9);
+
+	c3.name = "c3";
+	list_add_tail(ref(&parent.children, parent_count), &c3.list);
+	ok1(parent_count == 10);
+	/* Test list_add_tail and !list_empty. */
+	ok1(!list_empty(ref(&parent.children, parent_count)));
+	ok1(parent_count == 11);
+	ok1(parent.children.n.next == &c1.list);
+	ok1(parent.children.n.prev == &c3.list);
+	ok1(c1.list.next == &c2.list);
+	ok1(c1.list.prev == &parent.children.n);
+	ok1(c2.list.next == &c3.list);
+	ok1(c2.list.prev == &c1.list);
+	ok1(c3.list.next == &parent.children.n);
+	ok1(c3.list.prev == &c2.list);
+	/* Test list_check */
+	ok1(list_check(ref(&parent.children, parent_count), NULL));
+	ok1(parent_count == 12);
+
+	/* Test list_check_node */
+	ok1(list_check_node(&c1.list, NULL));
+	ok1(list_check_node(&c2.list, NULL));
+	ok1(list_check_node(&c3.list, NULL));
+
+	/* Test list_top */
+	ok1(list_top(ref(&parent.children, parent_count), struct child, list) == &c1);
+	ok1(parent_count == 13);
+
+	/* Test list_tail */
+	ok1(list_tail(ref(&parent.children, parent_count), struct child, list) == &c3);
+	ok1(parent_count == 14);
+
+	/* Test list_for_each. */
+	i = 0;
+	list_for_each(&parent.children, c, list) {
+		switch (i++) {
+		case 0:
+			ok1(c == &c1);
+			break;
+		case 1:
+			ok1(c == &c2);
+			break;
+		case 2:
+			ok1(c == &c3);
+			break;
+		}
+		if (i > 2)
+			break;
+	}
+	ok1(i == 3);
+
+	/* Test list_for_each_safe, list_del and list_del_from. */
+	i = 0;
+	list_for_each_safe(&parent.children, c, n, list) {
+		switch (i++) {
+		case 0:
+			ok1(c == &c1);
+			list_del(ref(&c->list, node_count));
+			ok1(node_count == 1);
+			break;
+		case 1:
+			ok1(c == &c2);
+			list_del_from(ref(&parent.children, parent_count),
+				      ref(&c->list, node_count));
+			ok1(node_count == 2);
+			break;
+		case 2:
+			ok1(c == &c3);
+			list_del_from(ref(&parent.children, parent_count),
+				      ref(&c->list, node_count));
+			ok1(node_count == 3);
+			break;
+		}
+		ok1(list_check(ref(&parent.children, parent_count), NULL));
+		if (i > 2)
+			break;
+	}
+	ok1(i == 3);
+	ok1(parent_count == 19);
+	ok1(list_empty(ref(&parent.children, parent_count)));
+	ok1(parent_count == 20);
+
+	/* Test list_top/list_tail on empty list. */
+	ok1(list_top(ref(&parent.children, parent_count), struct child, list) == NULL);
+	ok1(parent_count == 21);
+	ok1(list_tail(ref(&parent.children, parent_count), struct child, list) == NULL);
+	ok1(parent_count == 22);
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/list/test/run-with-debug.c b/src/common/libccan/ccan/list/test/run-with-debug.c
new file mode 100644
index 000000000000..d0902421f19c
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/run-with-debug.c
@@ -0,0 +1,3 @@
+/* Just like run.c, but with all debug checks enabled. */
+#define CCAN_LIST_DEBUG 1
+#include <ccan/list/test/run.c>
diff --git a/src/common/libccan/ccan/list/test/run.c b/src/common/libccan/ccan/list/test/run.c
new file mode 100644
index 000000000000..5787e4563e08
--- /dev/null
+++ b/src/common/libccan/ccan/list/test/run.c
@@ -0,0 +1,305 @@
+#include <ccan/list/list.h>
+#include <ccan/tap/tap.h>
+#include <ccan/list/list.c>
+#include "helper.h"
+
+struct parent {
+	const char *name;
+	struct list_head children;
+	unsigned int num_children;
+};
+
+struct child {
+	const char *name;
+	struct list_node list;
+};
+
+static LIST_HEAD(static_list);
+
+int main(void)
+{
+	struct parent parent;
+	struct child c1, c2, c3, x1, *c, *n;
+	unsigned int i;
+	struct list_head list = LIST_HEAD_INIT(list);
+	opaque_t *q, *nq;
+	struct list_head opaque_list = LIST_HEAD_INIT(opaque_list);
+	LIST_HEAD(rev);
+
+	plan_tests(92);
+	/* Test LIST_HEAD, LIST_HEAD_INIT, list_empty and check_list */
+	ok1(list_empty(&static_list));
+	ok1(list_check(&static_list, NULL));
+	ok1(list_empty(&list));
+	ok1(list_check(&list, NULL));
+
+	parent.num_children = 0;
+	list_head_init(&parent.children);
+	/* Test list_head_init */
+	ok1(list_empty(&parent.children));
+	ok1(list_check(&parent.children, NULL));
+
+	c2.name = "c2";
+	list_add(&parent.children, &c2.list);
+	/* Test list_add and !list_empty. */
+	ok1(!list_empty(&parent.children));
+	ok1(c2.list.next == &parent.children.n);
+	ok1(c2.list.prev == &parent.children.n);
+	ok1(parent.children.n.next == &c2.list);
+	ok1(parent.children.n.prev == &c2.list);
+	/* Test list_check */
+	ok1(list_check(&parent.children, NULL));
+
+	c1.name = "c1";
+	list_add(&parent.children, &c1.list);
+	/* Test list_add and !list_empty. */
+	ok1(!list_empty(&parent.children));
+	ok1(c2.list.next == &parent.children.n);
+	ok1(c2.list.prev == &c1.list);
+	ok1(parent.children.n.next == &c1.list);
+	ok1(parent.children.n.prev == &c2.list);
+	ok1(c1.list.next == &c2.list);
+	ok1(c1.list.prev == &parent.children.n);
+	/* Test list_check */
+	ok1(list_check(&parent.children, NULL));
+
+	c3.name = "c3";
+	list_add_tail(&parent.children, &c3.list);
+	/* Test list_add_tail and !list_empty. */
+	ok1(!list_empty(&parent.children));
+	ok1(parent.children.n.next == &c1.list);
+	ok1(parent.children.n.prev == &c3.list);
+	ok1(c1.list.next == &c2.list);
+	ok1(c1.list.prev == &parent.children.n);
+	ok1(c2.list.next == &c3.list);
+	ok1(c2.list.prev == &c1.list);
+	ok1(c3.list.next == &parent.children.n);
+	ok1(c3.list.prev == &c2.list);
+	/* Test list_check */
+	ok1(list_check(&parent.children, NULL));
+
+	/* Test list_check_node */
+	ok1(list_check_node(&c1.list, NULL));
+	ok1(list_check_node(&c2.list, NULL));
+	ok1(list_check_node(&c3.list, NULL));
+
+	/* Test list_top */
+	ok1(list_top(&parent.children, struct child, list) == &c1);
+
+	/* Test list_pop */
+	ok1(list_pop(&parent.children, struct child, list) == &c1);
+	ok1(list_top(&parent.children, struct child, list) == &c2);
+	list_add(&parent.children, &c1.list);
+
+	/* Test list_tail */
+	ok1(list_tail(&parent.children, struct child, list) == &c3);
+
+	/* Test list_for_each. */
+	i = 0;
+	list_for_each(&parent.children, c, list) {
+		switch (i++) {
+		case 0:
+			ok1(c == &c1);
+			break;
+		case 1:
+			ok1(c == &c2);
+			break;
+		case 2:
+			ok1(c == &c3);
+			break;
+		}
+		if (i > 2)
+			break;
+	}
+	ok1(i == 3);
+
+	/* Test list_for_each_rev. */
+	i = 0;
+	list_for_each_rev(&parent.children, c, list) {
+		switch (i++) {
+		case 0:
+			ok1(c == &c3);
+			break;
+		case 1:
+			ok1(c == &c2);
+			break;
+		case 2:
+			ok1(c == &c1);
+			break;
+		}
+		if (i > 2)
+			break;
+	}
+	ok1(i == 3);
+
+	/* Test list_for_each_safe, list_del and list_del_from. */
+	i = 0;
+	list_for_each_safe(&parent.children, c, n, list) {
+		switch (i++) {
+		case 0:
+			ok1(c == &c1);	
+			list_del(&c->list);
+			break;
+		case 1:
+			ok1(c == &c2);
+			list_del_from(&parent.children, &c->list);
+			break;
+		case 2:
+			ok1(c == &c3);
+			list_del_from(&parent.children, &c->list);
+			break;
+		}
+
+		/* prepare for list_for_each_rev_safe test */
+		list_add(&rev, &c->list);
+
+		ok1(list_check(&parent.children, NULL));
+		if (i > 2)
+			break;
+	}
+	ok1(i == 3);
+	ok1(list_empty(&parent.children));
+
+	/* Test list_for_each_rev_safe, list_del and list_del_from. */
+	i = 0;
+	list_for_each_rev_safe(&rev, c, n, list) {
+		switch (i++) {
+		case 0:
+			ok1(c == &c1);
+			list_del(&c->list);
+			break;
+		case 1:
+			ok1(c == &c2);
+			list_del_from(&rev, &c->list);
+			break;
+		case 2:
+			ok1(c == &c3);
+			list_del_from(&rev, &c->list);
+			break;
+		}
+		ok1(list_check(&rev, NULL));
+		if (i > 2)
+			break;
+	}
+	ok1(i == 3);
+	ok1(list_empty(&rev));
+
+	/* Test list_node_init: safe to list_del after this. */
+	list_node_init(&c->list);
+	list_del(&c->list);
+
+	/* Test list_del_init */
+	list_add(&parent.children, &c->list);
+	ok1(!list_empty(&parent.children));
+	list_del_init(&c->list);
+	ok1(list_empty(&parent.children));
+	/* We can call this as many times as we like. */
+	list_del_init(&c->list);
+	list_del_init(&c->list);
+
+	/* Test list_for_each_off. */
+	list_add_tail(&opaque_list,
+		      (struct list_node *)create_opaque_blob());
+	list_add_tail(&opaque_list,
+		      (struct list_node *)create_opaque_blob());
+	list_add_tail(&opaque_list,
+		      (struct list_node *)create_opaque_blob());
+
+	i = 0;
+
+	list_for_each_off(&opaque_list, q, 0) {
+	  i++;
+	  ok1(if_blobs_know_the_secret(q));
+	}
+	ok1(i == 3);
+
+	/* Test list_for_each_safe_off, list_del_off and list_del_from_off. */
+	i = 0;
+	list_for_each_safe_off(&opaque_list, q, nq, 0) {
+		switch (i++) {
+		case 0:
+			ok1(if_blobs_know_the_secret(q));
+			list_del_off(q, 0);
+			destroy_opaque_blob(q);
+			break;
+		case 1:
+			ok1(if_blobs_know_the_secret(q));
+			list_del_from_off(&opaque_list, q, 0);
+			destroy_opaque_blob(q);
+			break;
+		case 2:
+			ok1(c == &c3);
+			list_del_from_off(&opaque_list, q, 0);
+			destroy_opaque_blob(q);
+			break;
+		}
+		ok1(list_check(&opaque_list, NULL));
+		if (i > 2)
+			break;
+	}
+	ok1(i == 3);
+	ok1(list_empty(&opaque_list));
+
+	/* Test list_top/list_tail/list_pop on empty list. */
+	ok1(list_top(&parent.children, struct child, list) == NULL);
+	ok1(list_tail(&parent.children, struct child, list) == NULL);
+	ok1(list_pop(&parent.children, struct child, list) == NULL);
+
+	/* Test list_add_before and list_add_after */
+	list_add(&parent.children, &c1.list);
+	list_add_after(&parent.children, &c1.list, &c2.list);
+	ok1(list_check(&parent.children, "list_add_after"));
+
+	i = 0;
+	list_for_each(&parent.children, c, list) {
+		switch (i++) {
+		case 0:
+			ok1(c == &c1);
+			break;
+		case 1:
+			ok1(c == &c2);
+			break;
+		}
+	}
+	ok1(i == 2);
+
+	list_add_before(&parent.children, &c2.list, &c3.list);
+	ok1(list_check(&parent.children, "list_add_before"));
+
+	i = 0;
+	list_for_each(&parent.children, c, list) {
+		switch (i++) {
+		case 0:
+			ok1(c == &c1);
+			break;
+		case 1:
+			ok1(c == &c3);
+			break;
+		case 2:
+			ok1(c == &c2);
+			break;
+		}
+	}
+	ok1(i == 3);
+
+	/* test list_swap */
+	list_swap(&c3.list, &x1.list);
+	ok1(list_check(&parent.children, "list_swap"));
+	i = 0;
+	list_for_each(&parent.children, c, list) {
+		switch (i++) {
+		case 0:
+			ok1(c == &c1);
+			break;
+		case 1:
+			ok1(c == &x1);
+			break;
+		case 2:
+			ok1(c == &c2);
+			break;
+		}
+	}
+	ok1(i == 3);
+
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/ptrint/LICENSE b/src/common/libccan/ccan/ptrint/LICENSE
new file mode 120000
index 000000000000..b7951dabdc82
--- /dev/null
+++ b/src/common/libccan/ccan/ptrint/LICENSE
@@ -0,0 +1 @@
+../../licenses/CC0
\ No newline at end of file
diff --git a/src/common/libccan/ccan/ptrint/_info b/src/common/libccan/ccan/ptrint/_info
new file mode 100644
index 000000000000..30170da92bf2
--- /dev/null
+++ b/src/common/libccan/ccan/ptrint/_info
@@ -0,0 +1,60 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * ptrint - Encoding integers in pointer values
+ *
+ * Library (standard or ccan) functions which take user supplied
+ * callbacks usually have the callback supplied with a void * context
+ * pointer.  For simple cases, it's sometimes sufficient to pass a
+ * simple integer cast into a void *, rather than having to allocate a
+ * context structure.  This module provides some helper macros to do
+ * this relatively safely and portably.
+ *
+ * The key characteristics of these functions are:
+ *	ptr2int(int2ptr(val)) == val
+ * and
+ *      !int2ptr(val) == !val
+ * (i.e. the transformation preserves truth value).
+ *
+ * Example:
+ *	#include <ccan/ptrint/ptrint.h>
+ *
+ *	static void callback(void *opaque)
+ *	{
+ *		int val = ptr2int(opaque);
+ *		printf("Value is %d\n", val);
+ *	}
+ *
+ *	void (*cb)(void *opaque) = callback;
+ *
+ *	int main(void)
+ *	{
+ *		int val = 17;
+ *
+ *		(*cb)(int2ptr(val));
+ *		exit(0);
+ *	}
+ *
+ * License: CC0 (Public domain)
+ * Author: David Gibson <david@gibson.dropbear.id.au>
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/build_assert\n");
+		printf("ccan/compiler\n");
+		return 0;
+	}
+	if (strcmp(argv[1], "testdepends") == 0) {
+		printf("ccan/array_size\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/ptrint/ptrint.h b/src/common/libccan/ccan/ptrint/ptrint.h
new file mode 100644
index 000000000000..d8642e6fefc4
--- /dev/null
+++ b/src/common/libccan/ccan/ptrint/ptrint.h
@@ -0,0 +1,35 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#ifndef CCAN_PTRINT_H
+#define CCAN_PTRINT_H
+
+#include "config.h"
+
+#include <stddef.h>
+
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/compiler/compiler.h>
+
+/*
+ * This is a deliberately incomplete type, because it should never be
+ * dereferenced - instead it marks pointer values which are actually
+ * encoding integers
+ */
+typedef struct ptrint ptrint_t;
+
+CONST_FUNCTION static inline ptrdiff_t ptr2int(const ptrint_t *p)
+{
+	/*
+	 * ptrdiff_t is the right size by definition, but to avoid
+	 * surprises we want a warning if the user can't fit at least
+	 * a regular int in there
+	 */
+	BUILD_ASSERT(sizeof(int) <= sizeof(ptrdiff_t));
+	return (const char *)p - (const char *)NULL;
+}
+
+CONST_FUNCTION static inline ptrint_t *int2ptr(ptrdiff_t i)
+{
+	return (ptrint_t *)((char *)NULL + i);
+}
+
+#endif /* CCAN_PTRINT_H */
diff --git a/src/common/libccan/ccan/ptrint/test/run.c b/src/common/libccan/ccan/ptrint/test/run.c
new file mode 100644
index 000000000000..7d6f934b4236
--- /dev/null
+++ b/src/common/libccan/ccan/ptrint/test/run.c
@@ -0,0 +1,29 @@
+#include <limits.h>
+
+#include <ccan/array_size/array_size.h>
+
+#include <ccan/ptrint/ptrint.h>
+#include <ccan/tap/tap.h>
+
+static ptrdiff_t testvals[] = {
+	-INT_MAX, -1, 0, 1, 2, 17, INT_MAX,
+};
+
+int main(void)
+{
+	int i;
+
+	/* This is how many tests you plan to run */
+	plan_tests(2 * ARRAY_SIZE(testvals));
+
+	for (i = 0; i < ARRAY_SIZE(testvals); i++) {
+		ptrdiff_t val = testvals[i];
+		void *ptr = int2ptr(val);
+
+		ok1(ptr2int(ptr) == val);
+		ok1(!val == !ptr);
+	}
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/pushpull/LICENSE b/src/common/libccan/ccan/pushpull/LICENSE
new file mode 120000
index 000000000000..b7951dabdc82
--- /dev/null
+++ b/src/common/libccan/ccan/pushpull/LICENSE
@@ -0,0 +1 @@
+../../licenses/CC0
\ No newline at end of file
diff --git a/src/common/libccan/ccan/pushpull/_info b/src/common/libccan/ccan/pushpull/_info
new file mode 100644
index 000000000000..536744b00798
--- /dev/null
+++ b/src/common/libccan/ccan/pushpull/_info
@@ -0,0 +1,76 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * pushpull - simple marshalling/unmarshalling routines
+ *
+ * This code lets you clearly add simple types into a buffer (the push
+ * functions) and remove them (the pull functions).  The buffer stores
+ * the values as little-endian for machine portability.  The pull functions
+ * don't need to be checked on every call, but error state is kept so you
+ * can check if there was an error at the end.
+ *
+ * The normal way to use this is to create your own higher-level marshal
+ * and unmarshal functions in terms of these.
+ *
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ * License: CC0 (Public domain)
+ *
+ * Example:
+ *	#include <ccan/pushpull/push.h>
+ *	#include <ccan/pushpull/pull.h>
+ *	#include <ccan/err/err.h>
+ *	#include <string.h>
+ *	#include <stdio.h>
+ *	#include <unistd.h>
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		if (argv[1] && !strcmp(argv[1], "push")) {
+ *			int i;
+ *			char *buf = malloc(1);
+ *			size_t len = 0;
+ *
+ *			// We ignore allocation failure!
+ *			for (i = 2; i < argc; i++)
+ *				push_u32(&buf, &len, atol(argv[i]));
+ *
+ *			write(STDOUT_FILENO, buf, len);
+ *		} else if (argc == 2 && !strcmp(argv[1], "pull")) {
+ *			int r, max = 32;
+ *			size_t len = 0;
+ *			char *buf = malloc(max);
+ *			const char *p;
+ *			uint32_t val;
+ *
+ *			while ((r = read(STDIN_FILENO, buf+len, max-len)) > 0) {
+ *				len += r;
+ *				if (len == max) {
+ *					max *= 2;
+ *					// We crash on allocation failure
+ *					buf = realloc(buf, max);
+ *				}
+ *			}
+ *
+ *			p = buf;
+ *			while (pull_u32(&p, &len, &val))
+ *				printf("%u ", val);
+ *		} else
+ *			errx(1, "Usage: %s [push|pull] [<number>...]", argv[0]);
+ *		return 0;
+ *	}
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/endian\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/pushpull/pull.c b/src/common/libccan/ccan/pushpull/pull.c
new file mode 100644
index 000000000000..be26b41d2bea
--- /dev/null
+++ b/src/common/libccan/ccan/pushpull/pull.c
@@ -0,0 +1,105 @@
+/* CC0 license (public domain) - see LICENSE file for details */
+#include "pull.h"
+#include <ccan/endian/endian.h>
+#include <string.h>
+
+bool pull_bytes(const char **p, size_t *max_len, void *dst, size_t len)
+{
+	if (*max_len < len) {
+		*p = NULL;
+		*max_len = 0;
+		return false;
+	}
+	if (dst)
+		memcpy(dst, *p, len);
+	*max_len -= len;
+	*p += len;
+	return true;
+}
+
+bool pull_u64(const char **p, size_t *max_len, uint64_t *val)
+{
+	leint64_t v;
+
+	if (pull_bytes(p, max_len, &v, sizeof(v))) {
+		if (val)
+			*val = le64_to_cpu(v);
+		return true;
+	}
+	return false;
+}
+
+bool pull_u32(const char **p, size_t *max_len, uint32_t *val)
+{
+	leint32_t v;
+
+	if (pull_bytes(p, max_len, &v, sizeof(v))) {
+		if (val)
+			*val = le32_to_cpu(v);
+		return true;
+	}
+	return false;
+}
+
+bool pull_u16(const char **p, size_t *max_len, uint16_t *val)
+{
+	leint16_t v;
+
+	if (pull_bytes(p, max_len, &v, sizeof(v))) {
+		if (val)
+			*val = le16_to_cpu(v);
+		return true;
+	}
+	return false;
+}
+
+bool pull_u8(const char **p, size_t *max_len, uint8_t *val)
+{
+	return pull_bytes(p, max_len, val, sizeof(*val));
+}
+
+bool pull_s64(const char **p, size_t *max_len, int64_t *val)
+{
+	leint64_t v;
+
+	if (pull_bytes(p, max_len, &v, sizeof(v))) {
+		if (val)
+			*val = le64_to_cpu(v);
+		return true;
+	}
+	return false;
+}
+
+bool pull_s32(const char **p, size_t *max_len, int32_t *val)
+{
+	leint32_t v;
+
+	if (pull_bytes(p, max_len, &v, sizeof(v))) {
+		if (val)
+			*val = le32_to_cpu(v);
+		return true;
+	}
+	return false;
+}
+
+bool pull_s16(const char **p, size_t *max_len, int16_t *val)
+{
+	leint16_t v;
+
+	if (pull_bytes(p, max_len, &v, sizeof(v))) {
+		if (val)
+			*val = le16_to_cpu(v);
+		return true;
+	}
+	return false;
+}
+
+bool pull_s8(const char **p, size_t *max_len, int8_t *val)
+{
+	return pull_bytes(p, max_len, val, sizeof(*val));
+}
+
+bool pull_char(const char **p, size_t *max_len, char *val)
+{
+	return pull_bytes(p, max_len, val, sizeof(*val));
+}
diff --git a/src/common/libccan/ccan/pushpull/pull.h b/src/common/libccan/ccan/pushpull/pull.h
new file mode 100644
index 000000000000..e675a8dda037
--- /dev/null
+++ b/src/common/libccan/ccan/pushpull/pull.h
@@ -0,0 +1,140 @@
+/* CC0 license (public domain) - see LICENSE file for details */
+#ifndef CCAN_PUSHPULL_PULL_H
+#define CCAN_PUSHPULL_PULL_H
+#include "config.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+/**
+ * pull_bytes - unmarshall bytes from push_bytes.
+ * @p: pointer to bytes to unmarshal
+ * @max_len: pointer to number of bytes in unmarshal buffer
+ * @dst: destination to copy bytes (or NULL to discard)
+ * @len: length to copy into @dst.
+ *
+ * If @max_len isn't long enough, @p is set to NULL, @max_len is set to
+ * 0 (making chaining safe), and false is returned.  Otherwise, @len
+ * bytes are copied from *@p into @dst, *@p is incremented by @len,
+ * @max_len is decremented by @len, and true is returned.
+ */
+bool pull_bytes(const char **p, size_t *max_len, void *dst, size_t len);
+
+/**
+ * pull_u64 - unmarshall a little-endian 64-bit value.
+ * @p: pointer to bytes to unmarshal
+ * @max_len: pointer to number of bytes in unmarshal buffer
+ * @val: the value (or NULL to discard)
+ *
+ * This pulls 8 bytes and converts from little-endian (if necessary for
+ * this platform).  It returns false and sets @p to NULL on error (ie. not
+ * enough bytes).
+ *
+ * Example:
+ * struct foo {
+ *	uint64_t vu64;
+ *	uint32_t vu32;
+ *	uint16_t vu16;
+ *	uint8_t vu8;
+ * };
+ *
+ * static bool pull_foo(const char **p, size_t *len, struct foo *foo)
+ * {
+ * 	pull_u64(p, len, &foo->vu64);
+ * 	pull_u32(p, len, &foo->vu32);
+ * 	pull_u16(p, len, &foo->vu16);
+ * 	pull_u8(p, len, &foo->vu8);
+ *
+ *      // p is set to NULL on error (we could also use return codes)
+ *	return p != NULL;
+ * }
+ */
+bool pull_u64(const char **p, size_t *max_len, uint64_t *val);
+
+/**
+ * pull_u32 - unmarshall a little-endian 32-bit value.
+ * @p: pointer to bytes to unmarshal
+ * @max_len: pointer to number of bytes in unmarshal buffer
+ * @val: the value (or NULL to discard)
+ *
+ * This pulls 4 bytes and converts from little-endian (if necessary for
+ * this platform).
+ */
+bool pull_u32(const char **p, size_t *max_len, uint32_t *val);
+
+/**
+ * pull_u16 - unmarshall a little-endian 16-bit value.
+ * @p: pointer to bytes to unmarshal
+ * @max_len: pointer to number of bytes in unmarshal buffer
+ * @val: the value (or NULL to discard)
+ *
+ * This pulls 2 bytes and converts from little-endian (if necessary for
+ * this platform).
+ */
+bool pull_u16(const char **p, size_t *max_len, uint16_t *val);
+
+/**
+ * pull_u8 - unmarshall a single byte value.
+ * @p: pointer to bytes to unmarshal
+ * @max_len: pointer to number of bytes in unmarshal buffer
+ * @val: the value (or NULL to discard)
+ *
+ * This pulls one byte.
+ */
+bool pull_u8(const char **p, size_t *max_len, uint8_t *val);
+#define pull_uchar pull_u8
+
+/**
+ * pull_s64 - unmarshall a little-endian 64-bit signed value.
+ * @p: pointer to bytes to unmarshal
+ * @max_len: pointer to number of bytes in unmarshal buffer
+ * @val: the value (or NULL to discard)
+ *
+ * This pulls 8 bytes and converts from little-endian (if necessary for
+ * this platform).
+ */
+bool pull_s64(const char **p, size_t *max_len, int64_t *val);
+
+/**
+ * pull_s32 - unmarshall a little-endian 32-bit signed value.
+ * @p: pointer to bytes to unmarshal
+ * @max_len: pointer to number of bytes in unmarshal buffer
+ * @val: the value (or NULL to discard)
+ *
+ * This pulls 4 bytes and converts from little-endian (if necessary for
+ * this platform).
+ */
+bool pull_s32(const char **p, size_t *max_len, int32_t *val);
+
+/**
+ * pull_s16 - unmarshall a little-endian 16-bit signed value.
+ * @p: pointer to bytes to unmarshal
+ * @max_len: pointer to number of bytes in unmarshal buffer
+ * @val: the value (or NULL to discard)
+ *
+ * This pulls 2 bytes and converts from little-endian (if necessary for
+ * this platform).
+ */
+bool pull_s16(const char **p, size_t *max_len, int16_t *val);
+
+/**
+ * pull_s8 - unmarshall a single byte signed value.
+ * @p: pointer to bytes to unmarshal
+ * @max_len: pointer to number of bytes in unmarshal buffer
+ * @val: the value (or NULL to discard)
+ *
+ * This pulls one byte.
+ */
+bool pull_s8(const char **p, size_t *max_len, int8_t *val);
+
+/**
+ * pull_char - unmarshall a single char value.
+ * @p: pointer to bytes to unmarshal
+ * @max_len: pointer to number of bytes in unmarshal buffer
+ * @val: the value (or NULL to discard)
+ *
+ * This pulls one character.
+ */
+bool pull_char(const char **p, size_t *max_len, char *val);
+#endif /* CCAN_PUSHPULL_PULL_H */
diff --git a/src/common/libccan/ccan/pushpull/push.c b/src/common/libccan/ccan/pushpull/push.c
new file mode 100644
index 000000000000..c7c2250f3571
--- /dev/null
+++ b/src/common/libccan/ccan/pushpull/push.c
@@ -0,0 +1,73 @@
+/* CC0 license (public domain) - see LICENSE file for details */
+#include "push.h"
+#include <ccan/endian/endian.h>
+#include <string.h>
+
+static void *(*push_reallocfn)(void *ptr, size_t size) = realloc;
+
+bool push_bytes(char **p, size_t *len, const void *src, size_t srclen)
+{
+	char *n = push_reallocfn(*p, *len + srclen);
+	if (!n)
+		return false;
+	*p = n;
+	if (src)
+		memcpy(*p + *len, src, srclen);
+	else
+		memset(*p + *len, 0, srclen);
+	*len += srclen;
+	return true;
+}
+
+bool push_u64(char **p, size_t *len, uint64_t val)
+{
+	leint64_t v = cpu_to_le64(val);
+	return push_bytes(p, len, &v, sizeof(v));
+}
+
+bool push_u32(char **p, size_t *len, uint32_t val)
+{
+	leint32_t v = cpu_to_le32(val);
+	return push_bytes(p, len, &v, sizeof(v));
+}
+
+bool push_u16(char **p, size_t *len, uint16_t val)
+{
+	leint16_t v = cpu_to_le16(val);
+	return push_bytes(p, len, &v, sizeof(v));
+}
+
+bool push_u8(char **p, size_t *len, uint8_t val)
+{
+	return push_bytes(p, len, &val, sizeof(val));
+}
+
+bool push_s64(char **p, size_t *len, int64_t val)
+{
+	return push_u64(p, len, val);
+}
+
+bool push_s32(char **p, size_t *len, int32_t val)
+{
+	return push_u32(p, len, val);
+}
+
+bool push_s16(char **p, size_t *len, int16_t val)
+{
+	return push_u16(p, len, val);
+}
+
+bool push_s8(char **p, size_t *len, int8_t val)
+{
+	return push_u8(p, len, val);
+}
+
+bool push_char(char **p, size_t *len, char val)
+{
+	return push_u8(p, len, val);
+}
+
+void push_set_realloc(void *(*reallocfn)(void *ptr, size_t size))
+{
+	push_reallocfn = reallocfn;
+}
diff --git a/src/common/libccan/ccan/pushpull/push.h b/src/common/libccan/ccan/pushpull/push.h
new file mode 100644
index 000000000000..55274b97e414
--- /dev/null
+++ b/src/common/libccan/ccan/pushpull/push.h
@@ -0,0 +1,121 @@
+/* CC0 license (public domain) - see LICENSE file for details */
+#ifndef CCAN_PUSHPULL_PUSH_H
+#define CCAN_PUSHPULL_PUSH_H
+#include "config.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+/**
+ * push_bytes - marshall bytes into buffer
+ * @p: buffer of marshalled bytes
+ * @len: current length of @p buffer
+ * @src: source to copy bytes from (or NULL to copy zeroes)
+ * @srclen: length to copy from @src.
+ *
+ * If realloc() fails, false is returned.  Otherwise, *@p is increased
+ * by @srclen bytes, *@len is incremented by @srclen, and bytes appended
+ * to *@p (from @src if non-NULL).
+ */
+bool push_bytes(char **p, size_t *len, const void *src, size_t srclen);
+
+/**
+ * push_u64 - marshall a 64-bit value into buffer (as little-endian)
+ * @p: buffer of marshalled bytes
+ * @len: current length of @p buffer
+ * @val: the value to marshall.
+ *
+ * If realloc() fails, false is returned.
+ */
+bool push_u64(char **p, size_t *len, uint64_t val);
+
+/**
+ * push_u32 - marshall a 32-bit value into buffer (as little-endian)
+ * @p: buffer of marshalled bytes
+ * @len: current length of @p buffer
+ * @val: the value to marshall.
+ *
+ * If realloc() fails, false is returned.
+ */
+bool push_u32(char **p, size_t *len, uint32_t val);
+
+/**
+ * push_u16 - marshall a 16-bit value into buffer (as little-endian)
+ * @p: buffer of marshalled bytes
+ * @len: current length of @p buffer
+ * @val: the value to marshall.
+ *
+ * If realloc() fails, false is returned.
+ */
+bool push_u16(char **p, size_t *len, uint16_t val);
+
+/**
+ * push_u8 - marshall an 8-bit value into buffer
+ * @p: buffer of marshalled bytes
+ * @len: current length of @p buffer
+ * @val: the value to marshall.
+ *
+ * If realloc() fails, false is returned.
+ */
+bool push_u8(char **p, size_t *len, uint8_t val);
+#define push_uchar push_u8
+
+/**
+ * push_u64 - marshall a signed 64-bit value into buffer (as little-endian)
+ * @p: buffer of marshalled bytes
+ * @len: current length of @p buffer
+ * @val: the value to marshall.
+ *
+ * If realloc() fails, false is returned.
+ */
+bool push_s64(char **p, size_t *len, int64_t val);
+
+/**
+ * push_u32 - marshall a signed 32-bit value into buffer (as little-endian)
+ * @p: buffer of marshalled bytes
+ * @len: current length of @p buffer
+ * @val: the value to marshall.
+ *
+ * If realloc() fails, false is returned.
+ */
+bool push_s32(char **p, size_t *len, int32_t val);
+
+/**
+ * push_u16 - marshall a signed 16-bit value into buffer (as little-endian)
+ * @p: buffer of marshalled bytes
+ * @len: current length of @p buffer
+ * @val: the value to marshall.
+ *
+ * If realloc() fails, false is returned.
+ */
+bool push_s16(char **p, size_t *len, int16_t val);
+
+/**
+ * push_u8 - marshall a signed 8-bit value into buffer
+ * @p: buffer of marshalled bytes
+ * @len: current length of @p buffer
+ * @val: the value to marshall.
+ *
+ * If realloc() fails, false is returned.
+ */
+bool push_s8(char **p, size_t *len, int8_t val);
+
+/**
+ * push_char - marshall a character into buffer
+ * @p: buffer of marshalled bytes
+ * @len: current length of @p buffer
+ * @val: the value to marshall.
+ *
+ * If realloc() fails, false is returned.
+ */
+bool push_char(char **p, size_t *len, char val);
+
+/**
+ * push_set_realloc - set function to use (instead of realloc).
+ * @reallocfn: new reallocation function.
+ *
+ * This can be used, for example, to cache reallocations.
+ */
+void push_set_realloc(void *(reallocfn)(void *ptr, size_t size));
+#endif /* CCAN_PUSHPULL_PUSH_H */
diff --git a/src/common/libccan/ccan/pushpull/pushpull.h b/src/common/libccan/ccan/pushpull/pushpull.h
new file mode 100644
index 000000000000..e01bc09e1fca
--- /dev/null
+++ b/src/common/libccan/ccan/pushpull/pushpull.h
@@ -0,0 +1,7 @@
+/* CC0 license (public domain) - see LICENSE file for details */
+#ifndef CCAN_PUSHPULL_H
+#define CCAN_PUSHPULL_H
+/* You can also include these independently, if you don't need both. */
+#include <ccan/pushpull/push.h>
+#include <ccan/pushpull/pull.h>
+#endif /* CCAN_PUSHPULL_H */
diff --git a/src/common/libccan/ccan/pushpull/test/run.c b/src/common/libccan/ccan/pushpull/test/run.c
new file mode 100644
index 000000000000..f51ef2024015
--- /dev/null
+++ b/src/common/libccan/ccan/pushpull/test/run.c
@@ -0,0 +1,150 @@
+#include <ccan/pushpull/pushpull.h>
+/* Include the C files directly. */
+#include <ccan/pushpull/push.c>
+#include <ccan/pushpull/pull.c>
+#include <ccan/tap/tap.h>
+
+struct foo {
+	uint64_t vu64;
+	uint32_t vu32;
+	uint16_t vu16;
+	uint8_t vu8;
+	unsigned char vuchar;
+	int64_t vs64;
+	int32_t vs32;
+	int16_t vs16;
+	int8_t vs8;
+	char vchar;
+	char bytes[100];
+};
+
+static void *fail_reallocfn(void *ptr, size_t size)
+{
+	return NULL;
+}
+
+static bool push_foo(char **p, size_t *len, const struct foo *foo)
+{
+	return push_u64(p, len, foo->vu64) &&
+		push_u32(p, len, foo->vu32) &&
+		push_u16(p, len, foo->vu16) &&
+		push_u8(p, len, foo->vu8) &&
+		push_uchar(p, len, foo->vuchar) &&
+		push_s64(p, len, foo->vs64) &&
+		push_s32(p, len, foo->vs32) &&
+		push_s16(p, len, foo->vs16) &&
+		push_s8(p, len, foo->vs8) &&
+		push_char(p, len, foo->vchar) &&
+		push_bytes(p, len, foo->bytes, sizeof(foo->bytes));
+}
+
+static bool pull_foo(const char **p, size_t *len, struct foo *foo)
+{
+	int ret;
+
+	ret = pull_u64(p, len, &foo->vu64) +
+		pull_u32(p, len, &foo->vu32) +
+		pull_u16(p, len, &foo->vu16) +
+		pull_u8(p, len, &foo->vu8) +
+		pull_uchar(p, len, &foo->vuchar) +
+		pull_s64(p, len, &foo->vs64) +
+		pull_s32(p, len, &foo->vs32) +
+		pull_s16(p, len, &foo->vs16) +
+		pull_s8(p, len, &foo->vs8) +
+		pull_char(p, len, &foo->vchar) +
+		pull_bytes(p, len, foo->bytes, sizeof(foo->bytes));
+
+	if (ret != 11)
+		ok1(len == 0 && *p == NULL);
+	return ret == 11;
+}
+
+static bool foo_equal(const struct foo *f1, const struct foo *f2)
+{
+	return f1->vu64 == f2->vu64 &&
+		f1->vu32 == f2->vu32 &&
+		f1->vu16 == f2->vu16 &&
+		f1->vu8 == f2->vu8 &&
+		f1->vuchar == f2->vuchar &&
+		f1->vs64 == f2->vs64 &&
+		f1->vs32 == f2->vs32 &&
+		f1->vs16 == f2->vs16 &&
+		f1->vs8 == f2->vs8 &&
+		f1->vchar == f2->vchar &&
+		memcmp(f1->bytes, f2->bytes, sizeof(f1->bytes)) == 0;
+}
+
+int main(void)
+{
+	char *buffer;
+	const char *p;
+	size_t len, left;
+	struct foo *foo, *foo2;
+
+	/* This is how many tests you plan to run */
+	plan_tests(17);
+
+	/* Valgrind will make sure we don't read padding! */
+	foo = malloc(sizeof(*foo));
+	foo->vu64 = 0x01020304050607ULL;
+	foo->vu32 = 0x08090a0b;
+	foo->vu16 = 0x0c0d;
+	foo->vu8 = 0x0e;
+	foo->vuchar = 0x0f;
+	foo->vs64 = -0x1011121314151617LL;
+	foo->vs32 = -0x18191a1b;
+	foo->vs16 = -0x1c1d;
+	foo->vs8 = -0x1e;
+	foo->vchar = -0x1f;
+	memset(foo->bytes, 0x20, sizeof(foo->bytes));
+	strcpy(foo->bytes, "This is a test");
+
+	buffer = malloc(1);
+	len = 0;
+	ok1(push_foo(&buffer, &len, foo));
+	ok1(len <= sizeof(*foo));
+
+	/* Triggers valgrind's uninitialized value warning */
+	ok1(!memchr(buffer, 0x21, len));
+
+	p = buffer;
+	left = len;
+	foo2 = malloc(sizeof(*foo2));
+	ok1(pull_foo(&p, &left, foo2));
+	ok1(left == 0);
+	ok1(p == buffer + len);
+	ok1(foo_equal(foo, foo2));
+
+	/* Too-small for pull, it should fail and set ptr/len to 0 */
+	p = buffer;
+	left = 0;
+	ok1(!pull_u64(&p, &left, &foo2->vu64));
+	ok1(p == NULL && left == 0);
+	/* Shouldn't change field! */
+	ok1(foo_equal(foo, foo2));
+
+	left = 7;
+	ok1(!pull_u64(&p, &left, &foo2->vu64));
+	ok1(p == NULL && left == 0);
+	/* Shouldn't change field! */
+	ok1(foo_equal(foo, foo2));
+
+	/* Discard should work. */
+	left = len;
+	ok1(pull_bytes(&p, &left, NULL, sizeof(foo->bytes)));
+	ok1(left == len - sizeof(foo->bytes));
+
+	/* Push failures should be clean. */
+	push_set_realloc(fail_reallocfn);
+	p = buffer;
+	left = len;
+	ok1(!push_u64(&buffer, &left, foo->vu64));
+	ok1(p == buffer && left == len);
+
+	free(buffer);
+	free(foo);
+	free(foo2);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/str/LICENSE b/src/common/libccan/ccan/str/LICENSE
new file mode 120000
index 000000000000..b7951dabdc82
--- /dev/null
+++ b/src/common/libccan/ccan/str/LICENSE
@@ -0,0 +1 @@
+../../licenses/CC0
\ No newline at end of file
diff --git a/src/common/libccan/ccan/str/_info b/src/common/libccan/ccan/str/_info
new file mode 100644
index 000000000000..b579525faac1
--- /dev/null
+++ b/src/common/libccan/ccan/str/_info
@@ -0,0 +1,52 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * str - string helper routines
+ *
+ * This is a grab bag of functions for string operations, designed to enhance
+ * the standard string.h.
+ *
+ * Note that if you define CCAN_STR_DEBUG, you will get extra compile
+ * checks on common misuses of the following functions (they will now
+ * be out-of-line, so there is a runtime penalty!).
+ *
+ *	strstr, strchr, strrchr:
+ *		Return const char * if first argument is const (gcc only).
+ *
+ *	isalnum, isalpha, isascii, isblank, iscntrl, isdigit, isgraph,
+ *	    islower, isprint, ispunct, isspace, isupper, isxdigit:
+ *		Static and runtime check that input is EOF or an *unsigned*
+ *		char, as per C standard (really!).
+ *
+ * Example:
+ *	#include <stdio.h>
+ *	#include <ccan/str/str.h>
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		if (argc > 1 && streq(argv[1], "--verbose"))
+ *			printf("verbose set\n");
+ *		if (argc > 1 && strstarts(argv[1], "--"))
+ *			printf("Some option set\n");
+ *		if (argc > 1 && strends(argv[1], "cow-powers"))
+ *			printf("Magic option set\n");
+ *		return 0;
+ *	}
+ *
+ * License: CC0 (Public domain)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/build_assert\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/str/debug.c b/src/common/libccan/ccan/str/debug.c
new file mode 100644
index 000000000000..8c519442d792
--- /dev/null
+++ b/src/common/libccan/ccan/str/debug.c
@@ -0,0 +1,108 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#include "config.h"
+#include <ccan/str/str_debug.h>
+#include <assert.h>
+#include <ctype.h>
+#include <string.h>
+
+#ifdef CCAN_STR_DEBUG
+/* Because we mug the real ones with macros, we need our own wrappers. */
+int str_isalnum(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isalnum(i);
+}
+
+int str_isalpha(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isalpha(i);
+}
+
+int str_isascii(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isascii(i);
+}
+
+#if HAVE_ISBLANK
+int str_isblank(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isblank(i);
+}
+#endif
+
+int str_iscntrl(int i)
+{
+	assert(i >= -1 && i < 256);
+	return iscntrl(i);
+}
+
+int str_isdigit(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isdigit(i);
+}
+
+int str_isgraph(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isgraph(i);
+}
+
+int str_islower(int i)
+{
+	assert(i >= -1 && i < 256);
+	return islower(i);
+}
+
+int str_isprint(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isprint(i);
+}
+
+int str_ispunct(int i)
+{
+	assert(i >= -1 && i < 256);
+	return ispunct(i);
+}
+
+int str_isspace(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isspace(i);
+}
+
+int str_isupper(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isupper(i);
+}
+
+int str_isxdigit(int i)
+{
+	assert(i >= -1 && i < 256);
+	return isxdigit(i);
+}
+
+#undef strstr
+#undef strchr
+#undef strrchr
+
+char *str_strstr(const char *haystack, const char *needle)
+{
+	return strstr(haystack, needle);
+}
+
+char *str_strchr(const char *haystack, int c)
+{
+	return strchr(haystack, c);
+}
+
+char *str_strrchr(const char *haystack, int c)
+{
+	return strrchr(haystack, c);
+}
+#endif
diff --git a/src/common/libccan/ccan/str/hex/LICENSE b/src/common/libccan/ccan/str/hex/LICENSE
new file mode 120000
index 000000000000..08d5d486f53b
--- /dev/null
+++ b/src/common/libccan/ccan/str/hex/LICENSE
@@ -0,0 +1 @@
+../../../licenses/CC0
\ No newline at end of file
diff --git a/src/common/libccan/ccan/str/hex/_info b/src/common/libccan/ccan/str/hex/_info
new file mode 100644
index 000000000000..d70a1425e845
--- /dev/null
+++ b/src/common/libccan/ccan/str/hex/_info
@@ -0,0 +1,39 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * str/hex - hex-to-string conversions and vice-versa
+ *
+ * This code contains simple routines for hexadecimal strings.
+ *
+ * License: CC0 (Public domain)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ *
+ * Example:
+ *	int main(int argc, char *argv[])
+ *	{
+ *		int i;
+ *
+ *		for (i = 1; i < argc; i++) {
+ *			char str[hex_str_size(strlen(argv[i]))];
+ *
+ *			hex_encode(str, sizeof(str), argv[i], strlen(argv[i]));
+ *			printf("%s ", str);
+ *		}
+ *		printf("\n");
+ *		return 0;
+ *	}
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/src/common/libccan/ccan/str/hex/hex.c b/src/common/libccan/ccan/str/hex/hex.c
new file mode 100644
index 000000000000..6e031779fe87
--- /dev/null
+++ b/src/common/libccan/ccan/str/hex/hex.c
@@ -0,0 +1,66 @@
+/* CC0 license (public domain) - see LICENSE file for details */
+#include <ccan/str/hex/hex.h>
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static bool char_to_hex(unsigned char *val, char c)
+{
+	if (c >= '0' && c <= '9') {
+		*val = c - '0';
+		return true;
+	}
+ 	if (c >= 'a' && c <= 'f') {
+		*val = c - 'a' + 10;
+		return true;
+	}
+ 	if (c >= 'A' && c <= 'F') {
+		*val = c - 'A' + 10;
+		return true;
+	}
+	return false;
+}
+
+bool hex_decode(const char *str, size_t slen, void *buf, size_t bufsize)
+{
+	unsigned char v1, v2;
+	unsigned char *p = buf;
+
+	while (slen > 1) {
+		if (!char_to_hex(&v1, str[0]) || !char_to_hex(&v2, str[1]))
+			return false;
+		if (!bufsize)
+			return false;
+		*(p++) = (v1 << 4) | v2;
+		str += 2;
+		slen -= 2;
+		bufsize--;
+	}
+	return slen == 0 && bufsize == 0;
+}
+
+static char hexchar(unsigned int val)
+{
+	if (val < 10)
+		return '0' + val;
+	if (val < 16)
+		return 'a' + val - 10;
+	abort();
+}
+
+bool hex_encode(const void *buf, size_t bufsize, char *dest, size_t destsize)
+{
+	size_t i;
+
+	if (destsize < hex_str_size(bufsize))
+		return false;
+
+	for (i = 0; i < bufsize; i++) {
+		unsigned int c = ((const unsigned char *)buf)[i];
+		*(dest++) = hexchar(c >> 4);
+		*(dest++) = hexchar(c & 0xF);
+	}
+	*dest = '\0';
+
+	return true;
+}
diff --git a/src/common/libccan/ccan/str/hex/hex.h b/src/common/libccan/ccan/str/hex/hex.h
new file mode 100644
index 000000000000..2b81704e9856
--- /dev/null
+++ b/src/common/libccan/ccan/str/hex/hex.h
@@ -0,0 +1,73 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#ifndef CCAN_HEX_H
+#define CCAN_HEX_H
+#include "config.h"
+#include <stdbool.h>
+#include <stdlib.h>
+
+/**
+ * hex_decode - Unpack a hex string.
+ * @str: the hexadecimal string
+ * @slen: the length of @str
+ * @buf: the buffer to write the data into
+ * @bufsize: the length of @buf
+ *
+ * Returns false if there are any characters which aren't 0-9, a-f or A-F,
+ * of the string wasn't the right length for @bufsize.
+ *
+ * Example:
+ *	unsigned char data[20];
+ *
+ *	if (!hex_decode(argv[1], strlen(argv[1]), data, 20))
+ *		printf("String is malformed!\n");
+ */
+bool hex_decode(const char *str, size_t slen, void *buf, size_t bufsize);
+
+/**
+ * hex_encode - Create a nul-terminated hex string
+ * @buf: the buffer to read the data from
+ * @bufsize: the length of @buf
+ * @dest: the string to fill
+ * @destsize: the max size of the string
+ *
+ * Returns true if the string, including terminator, fit in @destsize;
+ *
+ * Example:
+ *	unsigned char buf[] = { 0x1F, 0x2F };
+ *	char str[5];
+ *
+ *	if (!hex_encode(buf, sizeof(buf), str, sizeof(str)))
+ *		abort();
+ */
+bool hex_encode(const void *buf, size_t bufsize, char *dest, size_t destsize);
+
+/**
+ * hex_str_size - Calculate how big a nul-terminated hex string is
+ * @bytes: bytes of data to represent
+ *
+ * Example:
+ *	unsigned char buf[] = { 0x1F, 0x2F };
+ *	char str[hex_str_size(sizeof(buf))];
+ *
+ *	hex_encode(buf, sizeof(buf), str, sizeof(str));
+ */
+static inline size_t hex_str_size(size_t bytes)
+{
+	return 2 * bytes + 1;
+}
+
+/**
+ * hex_data_size - Calculate how many bytes of data in a hex string
+ * @strlen: the length of the string (with or without NUL)
+ *
+ * Example:
+ *	const char str[] = "1F2F";
+ *	unsigned char buf[hex_data_size(sizeof(str))];
+ *
+ *	hex_decode(str, strlen(str), buf, sizeof(buf));
+ */
+static inline size_t hex_data_size(size_t strlen)
+{
+	return strlen / 2;
+}
+#endif /* CCAN_HEX_H */
diff --git a/src/common/libccan/ccan/str/hex/test/run.c b/src/common/libccan/ccan/str/hex/test/run.c
new file mode 100644
index 000000000000..e4c1a6c5aa3f
--- /dev/null
+++ b/src/common/libccan/ccan/str/hex/test/run.c
@@ -0,0 +1,42 @@
+#include <ccan/str/hex/hex.h>
+/* Include the C files directly. */
+#include <ccan/str/hex/hex.c>
+#include <ccan/tap/tap.h>
+#include <string.h>
+
+int main(void)
+{
+	const char teststr[] = "0123456789abcdefABCDEF";
+	const char bad_teststr[] = "0123456789abcdefABCDEF1O";
+	const unsigned char testdata[] = { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab,
+					   0xcd, 0xef, 0xAB, 0xCD, 0xEF };
+	unsigned char data[11];
+	char str[23];
+	size_t i;
+	
+	plan_tests(10 + sizeof(str));
+	
+	ok1(hex_str_size(sizeof(testdata)) == sizeof(teststr));
+	/* This gives right result with or without nul included */
+	ok1(hex_data_size(strlen(teststr)) == sizeof(testdata));
+	ok1(hex_data_size(sizeof(teststr)) == sizeof(testdata));
+
+	ok1(hex_decode(teststr, strlen(teststr), data, sizeof(data)));
+	ok1(memcmp(data, testdata, sizeof(testdata)) == 0);
+	ok1(hex_encode(testdata, sizeof(testdata), str, sizeof(str)));
+	ok1(strcmp(str, "0123456789abcdefabcdef") == 0);
+
+	/* Bad char */
+	ok1(!hex_decode(bad_teststr, strlen(bad_teststr), data, sizeof(data)));
+	/* Bad hex string len */
+	ok1(!hex_decode(teststr, strlen(teststr) - 1, data, sizeof(data)));
+	/* Bad buffer len */
+	ok1(!hex_decode(teststr, strlen(teststr), data, sizeof(data) - 1));
+
+	/* Bad deststring size. */
+	for (i = 1; i <= sizeof(str); i++)
+		ok1(!hex_encode(testdata, sizeof(testdata), str, sizeof(str)-i));
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}
diff --git a/src/common/libccan/ccan/str/str.c b/src/common/libccan/ccan/str/str.c
new file mode 100644
index 000000000000..a9245c1742ec
--- /dev/null
+++ b/src/common/libccan/ccan/str/str.c
@@ -0,0 +1,13 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#include <ccan/str/str.h>
+
+size_t strcount(const char *haystack, const char *needle)
+{
+	size_t i = 0, nlen = strlen(needle);
+
+	while ((haystack = strstr(haystack, needle)) != NULL) {
+		i++;
+		haystack += nlen;
+	}
+	return i;
+}
diff --git a/src/common/libccan/ccan/str/str.h b/src/common/libccan/ccan/str/str.h
new file mode 100644
index 000000000000..d919b84d49be
--- /dev/null
+++ b/src/common/libccan/ccan/str/str.h
@@ -0,0 +1,228 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#ifndef CCAN_STR_H
+#define CCAN_STR_H
+#include "config.h"
+#include <string.h>
+#include <stdbool.h>
+#include <limits.h>
+#include <ctype.h>
+
+/**
+ * streq - Are two strings equal?
+ * @a: first string
+ * @b: first string
+ *
+ * This macro is arguably more readable than "!strcmp(a, b)".
+ *
+ * Example:
+ *	if (streq(somestring, ""))
+ *		printf("String is empty!\n");
+ */
+#define streq(a,b) (strcmp((a),(b)) == 0)
+
+/**
+ * strstarts - Does this string start with this prefix?
+ * @str: string to test
+ * @prefix: prefix to look for at start of str
+ *
+ * Example:
+ *	if (strstarts(somestring, "foo"))
+ *		printf("String %s begins with 'foo'!\n", somestring);
+ */
+#define strstarts(str,prefix) (strncmp((str),(prefix),strlen(prefix)) == 0)
+
+/**
+ * strends - Does this string end with this postfix?
+ * @str: string to test
+ * @postfix: postfix to look for at end of str
+ *
+ * Example:
+ *	if (strends(somestring, "foo"))
+ *		printf("String %s end with 'foo'!\n", somestring);
+ */
+static inline bool strends(const char *str, const char *postfix)
+{
+	if (strlen(str) < strlen(postfix))
+		return false;
+
+	return streq(str + strlen(str) - strlen(postfix), postfix);
+}
+
+/**
+ * stringify - Turn expression into a string literal
+ * @expr: any C expression
+ *
+ * Example:
+ *	#define PRINT_COND_IF_FALSE(cond) \
+ *		((cond) || printf("%s is false!", stringify(cond)))
+ */
+#define stringify(expr)		stringify_1(expr)
+/* Double-indirection required to stringify expansions */
+#define stringify_1(expr)	#expr
+
+/**
+ * strcount - Count number of (non-overlapping) occurrences of a substring.
+ * @haystack: a C string
+ * @needle: a substring
+ *
+ * Example:
+ *      assert(strcount("aaa aaa", "a") == 6);
+ *      assert(strcount("aaa aaa", "ab") == 0);
+ *      assert(strcount("aaa aaa", "aa") == 2);
+ */
+size_t strcount(const char *haystack, const char *needle);
+
+/**
+ * STR_MAX_CHARS - Maximum possible size of numeric string for this type.
+ * @type_or_expr: a pointer or integer type or expression.
+ *
+ * This provides enough space for a nul-terminated string which represents the
+ * largest possible value for the type or expression.
+ *
+ * Note: The implementation adds extra space so hex values or negative
+ * values will fit (eg. sprintf(... "%p"). )
+ *
+ * Example:
+ *	char str[STR_MAX_CHARS(int)];
+ *
+ *	sprintf(str, "%i", 7);
+ */
+#define STR_MAX_CHARS(type_or_expr)				\
+	((sizeof(type_or_expr) * CHAR_BIT + 8) / 9 * 3 + 2	\
+	 + STR_MAX_CHARS_TCHECK_(type_or_expr))
+
+#if HAVE_TYPEOF
+/* Only a simple type can have 0 assigned, so test that. */
+#define STR_MAX_CHARS_TCHECK_(type_or_expr)		\
+	(sizeof(({ typeof(type_or_expr) x = 0; x; }))*0)
+#else
+#define STR_MAX_CHARS_TCHECK_(type_or_expr) 0
+#endif
+
+/**
+ * cisalnum - isalnum() which takes a char (and doesn't accept EOF)
+ * @c: a character
+ *
+ * Surprisingly, the standard ctype.h isalnum() takes an int, which
+ * must have the value of EOF (-1) or an unsigned char.  This variant
+ * takes a real char, and doesn't accept EOF.
+ */
+static inline bool cisalnum(char c)
+{
+	return isalnum((unsigned char)c);
+}
+static inline bool cisalpha(char c)
+{
+	return isalpha((unsigned char)c);
+}
+static inline bool cisascii(char c)
+{
+	return isascii((unsigned char)c);
+}
+#if HAVE_ISBLANK
+static inline bool cisblank(char c)
+{
+	return isblank((unsigned char)c);
+}
+#endif
+static inline bool ciscntrl(char c)
+{
+	return iscntrl((unsigned char)c);
+}
+static inline bool cisdigit(char c)
+{
+	return isdigit((unsigned char)c);
+}
+static inline bool cisgraph(char c)
+{
+	return isgraph((unsigned char)c);
+}
+static inline bool cislower(char c)
+{
+	return islower((unsigned char)c);
+}
+static inline bool cisprint(char c)
+{
+	return isprint((unsigned char)c);
+}
+static inline bool cispunct(char c)
+{
+	return ispunct((unsigned char)c);
+}
+static inline bool cisspace(char c)
+{
+	return isspace((unsigned char)c);
+}
+static inline bool cisupper(char c)
+{
+	return isupper((unsigned char)c);
+}
+static inline bool cisxdigit(char c)
+{
+	return isxdigit((unsigned char)c);
+}
+
+#include <ccan/str/str_debug.h>
+
+/* These checks force things out of line, hence they are under DEBUG. */
+#ifdef CCAN_STR_DEBUG
+#include <ccan/build_assert/build_assert.h>
+
+/* These are commonly misused: they take -1 or an *unsigned* char value. */
+#undef isalnum
+#undef isalpha
+#undef isascii
+#undef isblank
+#undef iscntrl
+#undef isdigit
+#undef isgraph
+#undef islower
+#undef isprint
+#undef ispunct
+#undef isspace
+#undef isupper
+#undef isxdigit
+
+/* You can use a char if char is unsigned. */
+#if HAVE_BUILTIN_TYPES_COMPATIBLE_P && HAVE_TYPEOF
+#define str_check_arg_(i)						\
+	((i) + BUILD_ASSERT_OR_ZERO(!__builtin_types_compatible_p(typeof(i), \
+								  char)	\
+				    || (char)255 > 0))
+#else
+#define str_check_arg_(i) (i)
+#endif
+
+#define isalnum(i) str_isalnum(str_check_arg_(i))
+#define isalpha(i) str_isalpha(str_check_arg_(i))
+#define isascii(i) str_isascii(str_check_arg_(i))
+#if HAVE_ISBLANK
+#define isblank(i) str_isblank(str_check_arg_(i))
+#endif
+#define iscntrl(i) str_iscntrl(str_check_arg_(i))
+#define isdigit(i) str_isdigit(str_check_arg_(i))
+#define isgraph(i) str_isgraph(str_check_arg_(i))
+#define islower(i) str_islower(str_check_arg_(i))
+#define isprint(i) str_isprint(str_check_arg_(i))
+#define ispunct(i) str_ispunct(str_check_arg_(i))
+#define isspace(i) str_isspace(str_check_arg_(i))
+#define isupper(i) str_isupper(str_check_arg_(i))
+#define isxdigit(i) str_isxdigit(str_check_arg_(i))
+
+#if HAVE_TYPEOF
+/* With GNU magic, we can make const-respecting standard string functions. */
+#undef strstr
+#undef strchr
+#undef strrchr
+
+/* + 0 is needed to decay array into pointer. */
+#define strstr(haystack, needle)					\
+	((typeof((haystack) + 0))str_strstr((haystack), (needle)))
+#define strchr(haystack, c)					\
+	((typeof((haystack) + 0))str_strchr((haystack), (c)))
+#define strrchr(haystack, c)					\
+	((typeof((haystack) + 0))str_strrchr((haystack), (c)))
+#endif
+#endif /* CCAN_STR_DEBUG */
+
+#endif /* CCAN_STR_H */
diff --git a/src/common/libccan/ccan/str/str_debug.h b/src/common/libccan/ccan/str/str_debug.h
new file mode 100644
index 000000000000..92c10c41cc61
--- /dev/null
+++ b/src/common/libccan/ccan/str/str_debug.h
@@ -0,0 +1,30 @@
+/* CC0 (Public domain) - see LICENSE file for details */
+#ifndef CCAN_STR_DEBUG_H
+#define CCAN_STR_DEBUG_H
+
+/* #define CCAN_STR_DEBUG 1 */
+
+#ifdef CCAN_STR_DEBUG
+/* Because we mug the real ones with macros, we need our own wrappers. */
+int str_isalnum(int i);
+int str_isalpha(int i);
+int str_isascii(int i);
+#if HAVE_ISBLANK
+int str_isblank(int i);
+#endif
+int str_iscntrl(int i);
+int str_isdigit(int i);
+int str_isgraph(int i);
+int str_islower(int i);
+int str_isprint(int i);
+int str_ispunct(int i);
+int str_isspace(int i);
+int str_isupper(int i);
+int str_isxdigit(int i);
+
+char *str_strstr(const char *haystack, const char *needle);
+char *str_strchr(const char *s, int c);
+char *str_strrchr(const char *s, int c);
+#endif /* CCAN_STR_DEBUG */
+
+#endif /* CCAN_STR_DEBUG_H */
diff --git a/src/common/libccan/ccan/str/test/compile_fail-STR_MAX_CHARS.c b/src/common/libccan/ccan/str/test/compile_fail-STR_MAX_CHARS.c
new file mode 100644
index 000000000000..8e0fd2e2f652
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-STR_MAX_CHARS.c
@@ -0,0 +1,23 @@
+#include <ccan/str/str.h>
+
+struct s {
+	int val;
+};
+
+int main(void)
+{
+	struct s
+#ifdef FAIL
+#if !HAVE_TYPEOF
+	#error We need typeof to check STR_MAX_CHARS.
+#endif
+#else
+	/* A pointer is OK. */
+		*
+#endif
+		val;
+	char str[STR_MAX_CHARS(val)];
+
+	str[0] = '\0';
+	return str[0] ? 0 : 1;
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-isalnum.c b/src/common/libccan/ccan/str/test/compile_fail-isalnum.c
new file mode 100644
index 000000000000..5d9895822ccd
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-isalnum.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isalnum.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isalnum(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-isalpha.c b/src/common/libccan/ccan/str/test/compile_fail-isalpha.c
new file mode 100644
index 000000000000..33d365538aff
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-isalpha.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isalpha.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isalpha(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-isascii.c b/src/common/libccan/ccan/str/test/compile_fail-isascii.c
new file mode 100644
index 000000000000..3946e0b3b88a
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-isascii.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isascii.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isascii(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-isblank.c b/src/common/libccan/ccan/str/test/compile_fail-isblank.c
new file mode 100644
index 000000000000..e14b0d7e66ac
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-isblank.c
@@ -0,0 +1,27 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF || !HAVE_ISBLANK
+#error We need typeof to check isblank.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+#if HAVE_ISBLANK
+	return isblank(c);
+#else
+	return c;
+#endif
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-iscntrl.c b/src/common/libccan/ccan/str/test/compile_fail-iscntrl.c
new file mode 100644
index 000000000000..f9abf1dc56e4
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-iscntrl.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check iscntrl.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return iscntrl(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-isdigit.c b/src/common/libccan/ccan/str/test/compile_fail-isdigit.c
new file mode 100644
index 000000000000..a3ee439c631e
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-isdigit.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isdigit.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isdigit(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-islower.c b/src/common/libccan/ccan/str/test/compile_fail-islower.c
new file mode 100644
index 000000000000..8f5c456197ff
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-islower.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check islower.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return islower(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-isprint.c b/src/common/libccan/ccan/str/test/compile_fail-isprint.c
new file mode 100644
index 000000000000..85ed028f60f1
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-isprint.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isprint.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isprint(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-ispunct.c b/src/common/libccan/ccan/str/test/compile_fail-ispunct.c
new file mode 100644
index 000000000000..09d4279a9344
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-ispunct.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check ispunct.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return ispunct(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-isspace.c b/src/common/libccan/ccan/str/test/compile_fail-isspace.c
new file mode 100644
index 000000000000..798cfcd470f8
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-isspace.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isspace.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isspace(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-isupper.c b/src/common/libccan/ccan/str/test/compile_fail-isupper.c
new file mode 100644
index 000000000000..56f5dee11b95
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-isupper.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isupper.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isupper(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-isxdigit.c b/src/common/libccan/ccan/str/test/compile_fail-isxdigit.c
new file mode 100644
index 000000000000..ea4d5269add2
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-isxdigit.c
@@ -0,0 +1,23 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+#ifdef FAIL
+#if !HAVE_BUILTIN_TYPES_COMPATIBLE_P || !HAVE_TYPEOF
+#error We need typeof to check isxdigit.
+#endif
+	char
+#else
+	unsigned char
+#endif
+		c = argv[0][0];
+
+#ifdef FAIL
+	/* Fake fail on unsigned char platforms. */
+	BUILD_ASSERT((char)255 < 0);
+#endif
+
+	return isxdigit(c);
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-strchr.c b/src/common/libccan/ccan/str/test/compile_fail-strchr.c
new file mode 100644
index 000000000000..bdaf034abd1f
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-strchr.c
@@ -0,0 +1,18 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(void)
+{
+#ifdef FAIL
+#if !HAVE_TYPEOF
+	#error We need typeof to check strstr.
+#endif
+#else
+	const
+#endif
+		char *ret;
+	const char *str = "hello";
+
+	ret = strchr(str, 'l');
+	return ret ? 0 : 1;
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-strrchr.c b/src/common/libccan/ccan/str/test/compile_fail-strrchr.c
new file mode 100644
index 000000000000..57fba0ed58d1
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-strrchr.c
@@ -0,0 +1,18 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(void)
+{
+#ifdef FAIL
+#if !HAVE_TYPEOF
+	#error We need typeof to check strstr.
+#endif
+#else
+	const
+#endif
+		char *ret;
+	const char *str = "hello";
+
+	ret = strrchr(str, 'l');
+	return ret ? 0 : 1;
+}
diff --git a/src/common/libccan/ccan/str/test/compile_fail-strstr.c b/src/common/libccan/ccan/str/test/compile_fail-strstr.c
new file mode 100644
index 000000000000..7bd8ac22f13b
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_fail-strstr.c
@@ -0,0 +1,18 @@
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/str.h>
+
+int main(void)
+{
+#ifdef FAIL
+#if !HAVE_TYPEOF
+	#error We need typeof to check strstr.
+#endif
+#else
+	const
+#endif
+		char *ret;
+	const char *str = "hello";
+
+	ret = strstr(str, "hell");
+	return ret ? 0 : 1;
+}
diff --git a/src/common/libccan/ccan/str/test/compile_ok-STR_MAX_CHARS-static.c b/src/common/libccan/ccan/str/test/compile_ok-STR_MAX_CHARS-static.c
new file mode 100644
index 000000000000..bc6aff7a3233
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/compile_ok-STR_MAX_CHARS-static.c
@@ -0,0 +1,8 @@
+#include <ccan/str/str.h>
+
+int main(void)
+{
+	static char str[STR_MAX_CHARS(int)];
+
+	return str[0] ? 0 : 1;
+}
diff --git a/src/common/libccan/ccan/str/test/debug.c b/src/common/libccan/ccan/str/test/debug.c
new file mode 100644
index 000000000000..4bd384f2c484
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/debug.c
@@ -0,0 +1,5 @@
+/* We can't use the normal "#include the .c file" trick, since this is
+   contaminated by str.h's macro overrides.  So we put it in all tests
+   like this. */
+#define CCAN_STR_DEBUG 1
+#include <ccan/str/debug.c>
diff --git a/src/common/libccan/ccan/str/test/run-STR_MAX_CHARS.c b/src/common/libccan/ccan/str/test/run-STR_MAX_CHARS.c
new file mode 100644
index 000000000000..fa45bad8a641
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/run-STR_MAX_CHARS.c
@@ -0,0 +1,59 @@
+#include <ccan/str/str.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <ccan/tap/tap.h>
+#include <stdint.h>
+
+int main(void)
+{
+	char str[1000];
+	struct {
+		uint8_t u1byte;
+		int8_t s1byte;
+		uint16_t u2byte;
+		int16_t s2byte;
+		uint32_t u4byte;
+		int32_t s4byte;
+		uint64_t u8byte;
+		int64_t s8byte;
+		void *ptr;
+	} types;
+
+	plan_tests(13);
+
+	memset(&types, 0xFF, sizeof(types));
+
+	/* Hex versions */
+	sprintf(str, "0x%llx", (unsigned long long)types.u1byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.u1byte));
+	sprintf(str, "0x%llx", (unsigned long long)types.u2byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.u2byte));
+	sprintf(str, "0x%llx", (unsigned long long)types.u4byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.u4byte));
+	sprintf(str, "0x%llx", (unsigned long long)types.u8byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.u8byte));
+
+	/* Decimal versions */
+	sprintf(str, "%u", types.u1byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.u1byte));
+	sprintf(str, "%d", types.s1byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.s1byte));
+	sprintf(str, "%u", types.u2byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.u2byte));
+	sprintf(str, "%d", types.s2byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.s2byte));
+	sprintf(str, "%u", types.u4byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.u4byte));
+	sprintf(str, "%d", types.s4byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.s4byte));
+	sprintf(str, "%llu", (unsigned long long)types.u8byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.u8byte));
+	sprintf(str, "%lld", (long long)types.s8byte);
+	ok1(strlen(str) < STR_MAX_CHARS(types.s8byte));
+
+	/* Pointer version. */
+	sprintf(str, "%p", types.ptr);
+	ok1(strlen(str) < STR_MAX_CHARS(types.ptr));
+
+	return exit_status();
+}				
diff --git a/src/common/libccan/ccan/str/test/run.c b/src/common/libccan/ccan/str/test/run.c
new file mode 100644
index 000000000000..9917fe7111c5
--- /dev/null
+++ b/src/common/libccan/ccan/str/test/run.c
@@ -0,0 +1,106 @@
+#include <ccan/str/str.h>
+#include <ccan/str/str.c>
+#include <stdlib.h>
+#include <stdio.h>
+#include <ccan/tap/tap.h>
+
+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))
+
+static const char *substrings[] = { "far", "bar", "baz", "b", "ba", "z", "ar",
+				    NULL };
+
+#define NUM_SUBSTRINGS (ARRAY_SIZE(substrings) - 1)
+
+static char *strdup_rev(const char *s)
+{
+	char *ret = strdup(s);
+	unsigned int i;
+
+	for (i = 0; i < strlen(s); i++)
+		ret[i] = s[strlen(s) - i - 1];
+	return ret;
+}
+
+int main(void)
+{
+	unsigned int i, j, n;
+	char *strings[NUM_SUBSTRINGS * NUM_SUBSTRINGS];
+	
+	n = 0;
+	for (i = 0; i < NUM_SUBSTRINGS; i++) {
+		for (j = 0; j < NUM_SUBSTRINGS; j++) {
+			strings[n] = malloc(strlen(substrings[i])
+					    + strlen(substrings[j]) + 1);
+			sprintf(strings[n++], "%s%s",
+				substrings[i], substrings[j]);
+		}
+	}
+
+	plan_tests(n * n * 5 + 16);
+	for (i = 0; i < n; i++) {
+		for (j = 0; j < n; j++) {
+			unsigned int k, identical = 0;
+			char *reva, *revb;
+
+			/* Find first difference. */
+			for (k = 0; strings[i][k]==strings[j][k]; k++) {
+				if (k == strlen(strings[i])) {
+					identical = 1;
+					break;
+				}
+			}
+
+			if (identical) 
+				ok1(streq(strings[i], strings[j]));
+			else
+				ok1(!streq(strings[i], strings[j]));
+
+			/* Postfix test should be equivalent to prefix
+			 * test on reversed string. */
+			reva = strdup_rev(strings[i]);
+			revb = strdup_rev(strings[j]);
+
+			if (!strings[i][k]) {
+				ok1(strstarts(strings[j], strings[i]));
+				ok1(strends(revb, reva));
+			} else {
+				ok1(!strstarts(strings[j], strings[i]));
+				ok1(!strends(revb, reva));
+			}
+			if (!strings[j][k]) {
+				ok1(strstarts(strings[i], strings[j]));
+				ok1(strends(reva, revb));
+			} else {
+				ok1(!strstarts(strings[i], strings[j]));
+				ok1(!strends(reva, revb));
+			}
+			free(reva);
+			free(revb);
+		}
+	}
+
+	for (i = 0; i < n; i++)
+		free(strings[i]);
+
+	ok1(streq(stringify(NUM_SUBSTRINGS),
+		  "((sizeof(substrings) / sizeof(substrings[0])) - 1)"));
+	ok1(streq(stringify(ARRAY_SIZE(substrings)),
+		  "(sizeof(substrings) / sizeof(substrings[0]))"));
+	ok1(streq(stringify(i == 0), "i == 0"));
+
+	ok1(strcount("aaaaaa", "b") == 0);
+	ok1(strcount("aaaaaa", "a") == 6);
+	ok1(strcount("aaaaaa", "aa") == 3);
+	ok1(strcount("aaaaaa", "aaa") == 2);
+	ok1(strcount("aaaaaa", "aaaa") == 1);
+	ok1(strcount("aaaaaa", "aaaaa") == 1);
+	ok1(strcount("aaaaaa", "aaaaaa") == 1);
+	ok1(strcount("aaa aaa", "b") == 0);
+	ok1(strcount("aaa aaa", "a") == 6);
+	ok1(strcount("aaa aaa", "aa") == 2);
+	ok1(strcount("aaa aaa", "aaa") == 2);
+	ok1(strcount("aaa aaa", "aaaa") == 0);
+	ok1(strcount("aaa aaa", "aaaaa") == 0);
+
+	return exit_status();
+}				
diff --git a/src/common/libccan/licenses/BSD-MIT b/src/common/libccan/licenses/BSD-MIT
new file mode 100644
index 000000000000..89de354795ec
--- /dev/null
+++ b/src/common/libccan/licenses/BSD-MIT
@@ -0,0 +1,17 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/src/common/libccan/licenses/CC0 b/src/common/libccan/licenses/CC0
new file mode 100644
index 000000000000..feb9b118e6b5
--- /dev/null
+++ b/src/common/libccan/licenses/CC0
@@ -0,0 +1,28 @@
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer exclusive Copyright and Related Rights (defined below) upon the creator and subsequent owner(s) (each and all, an "owner") of an original work of authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for the purpose of contributing to a commons of creative, cultural and scientific works ("Commons") that the public can reliably and without fear of later claims of infringement build upon, modify, incorporate in other works, reuse and redistribute as freely as possible in any form whatsoever and for any purposes, including without limitation commercial purposes. These owners may contribute to the Commons to promote the ideal of a free culture and the further production of creative, cultural and scientific works, or to gain reputation or greater distribution for their Work in part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any expectation of additional consideration or compensation, the person associating CC0 with a Work (the "Affirmer"), to the extent that he or she is an owner of Copyright and Related Rights in the Work, voluntarily elects to apply CC0 to the Work and publicly distribute the Work under its terms, with knowledge of his or her Copyright and Related Rights in the Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following:
+
+    the right to reproduce, adapt, distribute, perform, display, communicate, and translate a Work;
+    moral rights retained by the original author(s) and/or performer(s);
+    publicity and privacy rights pertaining to a person's image or likeness depicted in a Work;
+    rights protecting against unfair competition in regards to a Work, subject to the limitations in paragraph 4(a), below;
+    rights protecting the extraction, dissemination, use and reuse of data in a Work;
+    database rights (such as those arising under Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, and under any national implementation thereof, including any amended or successor version of such directive); and
+    other similar, equivalent or corresponding rights throughout the world based on applicable law or treaty, and any national implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+    No trademark or patent rights held by Affirmer are waived, abandoned, surrendered, licensed or otherwise affected by this document.
+    Affirmer offers the Work as-is and makes no representations or warranties of any kind concerning the Work, express, implied, statutory or otherwise, including without limitation warranties of title, merchantability, fitness for a particular purpose, non infringement, or the absence of latent or other defects, accuracy, or the present or absence of errors, whether or not discoverable, all to the greatest extent permissible under applicable law.
+    Affirmer disclaims responsibility for clearing rights of other persons that may apply to the Work or any use thereof, including without limitation any person's Copyright and Related Rights in the Work. Further, Affirmer disclaims responsibility for obtaining any necessary consents, permissions or other rights required for any use of the Work.
+    Affirmer understands and acknowledges that Creative Commons is not a party to this document and has no duty or obligation with respect to this CC0 or use of the Work.
diff --git a/src/common/libccan/licenses/LGPL-2.1 b/src/common/libccan/licenses/LGPL-2.1
new file mode 100644
index 000000000000..2d2d780e6014
--- /dev/null
+++ b/src/common/libccan/licenses/LGPL-2.1
@@ -0,0 +1,510 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
+
+
diff --git a/src/common/libcontent/Makefile.am b/src/common/libcontent/Makefile.am
new file mode 100644
index 000000000000..152895de3528
--- /dev/null
+++ b/src/common/libcontent/Makefile.am
@@ -0,0 +1,20 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_builddir)/src/common/libflux
+
+noinst_LTLIBRARIES = libcontent.la
+
+libcontent_la_SOURCES = \
+	content-util.h \
+	content-util.c \
+	content.h \
+	content.c
diff --git a/src/common/libcontent/content-util.c b/src/common/libcontent/content-util.c
new file mode 100644
index 000000000000..ac41842c62de
--- /dev/null
+++ b/src/common/libcontent/content-util.c
@@ -0,0 +1,77 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "content-util.h"
+
+int content_register_backing_store (flux_t *h, const char *name)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_rpc_pack (h,
+                             "content.register-backing",
+                             0,
+                             0,
+                             "{s:s}",
+                             "name",
+                             name))) {
+        flux_log_error (h, "register-backing");
+        return -1;
+    }
+    if (flux_future_get (f, NULL) < 0) {
+        flux_log_error (h, "register-backing: %s", future_strerror (f, errno));
+        flux_future_destroy (f);
+        return -1;
+    }
+    flux_future_destroy (f);
+    return 0;
+}
+
+int content_unregister_backing_store (flux_t *h)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_rpc (h, "content.unregister-backing", NULL, 0, 0))) {
+        flux_log_error (h, "unregister-backing");
+        return -1;
+    }
+    if (flux_future_get (f, NULL) < 0) {
+        flux_log_error (h, "unregister-backing: %s", future_strerror (f, errno));
+        flux_future_destroy (f);
+        return -1;
+    }
+    flux_future_destroy (f);
+    return 0;
+}
+
+int content_register_service (flux_t *h, const char *name)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_service_register (h, name))) {
+        flux_log_error (h, "register %s", name);
+        return -1;
+    }
+    if (flux_future_get (f, NULL) < 0) {
+        flux_log_error (h, "register %s: %s", name, future_strerror (f, errno));
+        flux_future_destroy (f);
+        return -1;
+    }
+    flux_future_destroy (f);
+    return 0;
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libcontent/content-util.h b/src/common/libcontent/content-util.h
new file mode 100644
index 000000000000..1c9a8c7133f0
--- /dev/null
+++ b/src/common/libcontent/content-util.h
@@ -0,0 +1,36 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* NOTE: these functions all log error messages to the broker.
+ */
+
+#ifndef _FLUX_CONTENT_UTIL_H
+#define _FLUX_CONTENT_UTIL_H
+
+/* Let the rank 0 content-cache service know the backing store is available.
+ * This function blocks while waiting for the RPC response.
+ */
+int content_register_backing_store (flux_t *h, const char *name);
+
+/* Let the rank 0 content-cache service know the backing store is not available.
+ * This function blocks while waiting for the RPC response.
+ */
+int content_unregister_backing_store (flux_t *h);
+
+/* Wrapper to synchronously register a flux service.
+ * This function blocks while waiting for the RPC response.
+ */
+int content_register_service (flux_t *h, const char *name);
+
+#endif /* !_FLUX_CONTENT_UTIL_H */
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libcontent/content.c b/src/common/libcontent/content.c
new file mode 100644
index 000000000000..196ac67f2c42
--- /dev/null
+++ b/src/common/libcontent/content.c
@@ -0,0 +1,125 @@
+/************************************************************\
+ * Copyright 2016 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <assert.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <flux/core.h>
+
+#include "content.h"
+
+#include "src/common/libutil/blobref.h"
+#include "src/common/libutil/errno_safe.h"
+
+flux_future_t *content_load_byhash (flux_t *h,
+                                    const void *hash,
+                                    size_t hash_size,
+                                    int flags)
+{
+    const char *topic = "content.load";
+    uint32_t rank = FLUX_NODEID_ANY;
+
+    if (!h || !hash) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if ((flags & CONTENT_FLAG_UPSTREAM))
+        rank = FLUX_NODEID_UPSTREAM;
+    if ((flags & CONTENT_FLAG_CACHE_BYPASS)) {
+        topic = "content-backing.load";
+        rank = 0;
+    }
+    return flux_rpc_raw (h, topic, hash, hash_size, rank, 0);
+}
+
+flux_future_t *content_load_byblobref (flux_t *h,
+                                       const char *blobref,
+                                       int flags)
+{
+    uint32_t hash[BLOBREF_MAX_DIGEST_SIZE];
+    ssize_t hash_size;
+
+    if ((hash_size = blobref_strtohash (blobref, hash, sizeof (hash))) < 0)
+        return NULL;
+    return content_load_byhash (h, hash, hash_size, flags);
+}
+
+int content_load_get (flux_future_t *f, const void **buf, size_t *len)
+{
+    return flux_rpc_get_raw (f, buf, len);
+}
+
+flux_future_t *content_store (flux_t *h, const void *buf, size_t len, int flags)
+{
+    const char *topic = "content.store";
+    uint32_t rank = FLUX_NODEID_ANY;
+
+    if ((flags & CONTENT_FLAG_UPSTREAM))
+        rank = FLUX_NODEID_UPSTREAM;
+    if ((flags & CONTENT_FLAG_CACHE_BYPASS)) {
+        topic = "content-backing.store";
+        rank = 0;
+    }
+    return flux_rpc_raw (h, topic, buf, len, rank, 0);
+}
+
+int content_store_get_hash (flux_future_t *f,
+                            const void **hash,
+                            size_t *hash_size)
+{
+    const void *buf;
+    size_t buf_size;
+
+    if (flux_rpc_get_raw (f, &buf, &buf_size) < 0)
+        return -1;
+    if (hash)
+        *hash = buf;
+    if (hash_size)
+        *hash_size = buf_size;
+    return 0;
+}
+
+int content_store_get_blobref (flux_future_t *f,
+                               const char *hash_name,
+                               const char **blobref)
+{
+    const char *auxkey = "flux::blobref";
+    const char *result;
+
+    if (!(result = flux_future_aux_get (f, auxkey))) {
+        const void *hash;
+        size_t hash_len;
+        char buf[BLOBREF_MAX_STRING_SIZE];
+        char *cpy = NULL;
+
+        if (content_store_get_hash (f, &hash, &hash_len) < 0
+            || blobref_hashtostr (hash_name,
+                                  hash,
+                                  hash_len,
+                                  buf,
+                                  sizeof (buf)) < 0
+            || !(cpy = strdup (buf))
+            || flux_future_aux_set (f, auxkey, cpy, (flux_free_f)free) < 0) {
+            ERRNO_SAFE_WRAP (free, cpy);
+            return -1;
+        }
+        result = cpy;
+    }
+    if (blobref)
+        *blobref = result;
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libcontent/content.h b/src/common/libcontent/content.h
new file mode 100644
index 000000000000..6d31ff04f98f
--- /dev/null
+++ b/src/common/libcontent/content.h
@@ -0,0 +1,59 @@
+/************************************************************\
+ * Copyright 2016 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CONTENT_H
+#define _FLUX_CONTENT_H
+
+/* flags */
+enum {
+    CONTENT_FLAG_CACHE_BYPASS = 1,/* request direct to backing store */
+    CONTENT_FLAG_UPSTREAM = 2,    /* make request of upstream TBON peer */
+};
+
+/* Send request to load blob by hash or blobref.
+ */
+flux_future_t *content_load_byhash (flux_t *h,
+                                    const void *hash,
+                                    size_t hash_len,
+                                    int flags);
+flux_future_t *content_load_byblobref (flux_t *h,
+                                       const char *blobref,
+                                       int flags);
+
+/* Get result of load request (blob).
+ * This blocks until response is received.
+ * Storage for 'buf' belongs to 'f' and is valid until 'f' is destroyed.
+ * Returns 0 on success, -1 on failure with errno set.
+ */
+int content_load_get (flux_future_t *f, const void **buf, size_t *len);
+
+/* Send request to store blob.
+ */
+flux_future_t *content_store (flux_t *h,
+                              const void *buf,
+                              size_t len,
+                              int flags);
+
+/* Get result of store request (hash or blobref).
+ * Storage belongs to 'f' and is valid until 'f' is destroyed.
+ * Returns 0 on success, -1 on failure with errno set.
+ */
+int content_store_get_hash (flux_future_t *f,
+                            const void **hash,
+                            size_t *hash_len);
+int content_store_get_blobref (flux_future_t *f,
+                               const char *hash_name,
+                               const char **blobref);
+
+#endif /* !_FLUX_CONTENT_H */
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libczmqcontainers/Makefile.am b/src/common/libczmqcontainers/Makefile.am
new file mode 100644
index 000000000000..2a157342b224
--- /dev/null
+++ b/src/common/libczmqcontainers/Makefile.am
@@ -0,0 +1,29 @@
+AM_CFLAGS = \
+        $(WARNING_CFLAGS) \
+        $(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+        $(CODE_COVERAGE_LDFLAGS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include
+
+noinst_LTLIBRARIES = \
+	libczmqcontainers.la
+
+libczmqcontainers_la_SOURCES = \
+	czmq_containers.h \
+	czmq_internal.h \
+	czmq_internal.c \
+	czmq_rename.h \
+	zhashx.h \
+	zhashx.c \
+	zhash_primes.inc \
+	zlistx.h \
+	zlistx.c \
+	zhash.h \
+	zhash.c \
+	zlist.h \
+	zlist.c
diff --git a/src/common/libczmqcontainers/czmq_containers.h b/src/common/libczmqcontainers/czmq_containers.h
new file mode 100644
index 000000000000..37d58e3de2b2
--- /dev/null
+++ b/src/common/libczmqcontainers/czmq_containers.h
@@ -0,0 +1,43 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _CZMQ_CONTAINERS_H
+#define _CZMQ_CONTAINERS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/types.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "czmq_rename.h"
+
+typedef struct _zhashx_t zhashx_t;
+typedef struct _zlistx_t zlistx_t;
+typedef struct _zhash_t zhash_t;
+typedef struct _zlist_t zlist_t;
+
+#ifndef CZMQ_EXPORT
+#define CZMQ_EXPORT
+#endif
+
+#include "zhashx.h"
+#include "zlistx.h"
+#include "zhash.h"
+#include "zlist.h"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/common/libczmqcontainers/czmq_internal.c b/src/common/libczmqcontainers/czmq_internal.c
new file mode 100644
index 000000000000..5183c86855fc
--- /dev/null
+++ b/src/common/libczmqcontainers/czmq_internal.c
@@ -0,0 +1,25 @@
+/*  =========================================================================
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+#include "czmq_containers.h"
+#include "czmq_internal.h"
+
+//  --------------------------------------------------------------------------
+//  Free a provided string, and nullify the parent pointer. Safe to call on
+//  a null pointer.
+
+void
+zstr_free (char **string_p)
+{
+    assert (string_p);
+    free (*string_p);
+    *string_p = NULL;
+}
diff --git a/src/common/libczmqcontainers/czmq_internal.h b/src/common/libczmqcontainers/czmq_internal.h
new file mode 100644
index 000000000000..a1a0279aa112
--- /dev/null
+++ b/src/common/libczmqcontainers/czmq_internal.h
@@ -0,0 +1,61 @@
+/*  =========================================================================
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+/* To avoid copying in an excess amount of code from czmq, the
+ * following have been manually cut and pasted in
+ */
+
+#ifndef __CZMQ_INTERNAL__
+#define __CZMQ_INTERNAL__
+
+#if HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef NDEBUG
+  #undef NDEBUG
+  #include <assert.h>
+  #define NDEBUG
+#else
+  #include <assert.h>
+#endif
+
+#define freen(x) do {free(x); x = NULL;} while(0)
+
+//  Replacement for malloc() which asserts if we run out of heap, and
+//  which zeroes the allocated block.
+static inline void *
+safe_malloc (size_t size, const char *file, unsigned line)
+{
+//     printf ("%s:%u %08d\n", file, line, (int) size);
+    void *mem = calloc (1, size);
+    if (mem == NULL) {
+        fprintf (stderr, "FATAL ERROR at %s:%u\n", file, line);
+        fprintf (stderr, "OUT OF MEMORY (malloc returned NULL)\n");
+        fflush (stderr);
+        abort ();
+    }
+    return mem;
+}
+
+#define zmalloc(size) safe_malloc((size), __FILE__, __LINE__)
+
+#define streq(s1,s2) (!strcmp((s1), (s2)))
+
+void zstr_free (char **string_p);
+
+
+#endif
+
diff --git a/src/common/libczmqcontainers/czmq_rename.h b/src/common/libczmqcontainers/czmq_rename.h
new file mode 100644
index 000000000000..c6321ff82f18
--- /dev/null
+++ b/src/common/libczmqcontainers/czmq_rename.h
@@ -0,0 +1,151 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _CZMQ_RENAME_H
+#define _CZMQ_RENAME_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define zhashx_t fzhashx_t
+#define zhashx_destructor_fn fzhashx_destructor_fn
+#define zhashx_duplicator_fn fzhashx_duplicator_fn
+#define zhashx_comparator_fn fzhashx_comparator_fn
+#define zhashx_free_fn fzhashx_free_fn
+#define zhashx_hash_fn fzhashx_hash_fn
+#define zhashx_serializer_fn fzhashx_serializer_fn
+#define zhashx_deserializer_fn fzhashx_deserializer_fn
+#define zhashx_new fzhashx_new
+#define zhashx_unpack fzhashx_unpack
+#define zhashx_destroy fzhashx_destroy
+#define zhashx_insert fzhashx_insert
+#define zhashx_update fzhashx_update
+#define zhashx_delete fzhashx_delete
+#define zhashx_purge fzhashx_purge
+#define zhashx_lookup fzhashx_lookup
+#define zhashx_rename fzhashx_rename
+#define zhashx_freefn fzhashx_freefn
+#define zhashx_size fzhashx_size
+#define zhashx_keys fzhashx_keys
+#define zhashx_values fzhashx_values
+#define zhashx_first fzhashx_first
+#define zhashx_next fzhashx_next
+#define zhashx_cursor fzhashx_cursor
+#define zhashx_comment fzhashx_comment
+#define zhashx_save fzhashx_save
+#define zhashx_load fzhashx_load
+#define zhashx_refresh fzhashx_refresh
+#define zhashx_pack fzhashx_pack
+#define zhashx_dup fzhashx_dup
+#define zhashx_set_destructor fzhashx_set_destructor
+#define zhashx_set_duplicator fzhashx_set_duplicator
+#define zhashx_set_key_destructor fzhashx_set_key_destructor
+#define zhashx_set_key_duplicator fzhashx_set_key_duplicator
+#define zhashx_set_key_comparator fzhashx_set_key_comparator
+#define zhashx_set_key_hasher fzhashx_set_key_hasher
+#define zhashx_dup_v2 fzhashx_dup_v2
+#define zhashx_test fzhashx_test
+#define zhashx_unpack_own fzhashx_unpack_own
+#define zhashx_pack_own fzhashx_pack_own
+
+#define zlistx_t fzlistx_t
+#define zlistx_destructor_fn fzlistx_destructor_fn
+#define zlistx_duplicator_fn fzlistx_duplicator_fn
+#define zlistx_comparator_fn fzlistx_comparator_fn
+#define zlistx_new fzlistx_new
+#define zlistx_destroy fzlistx_destroy
+#define zlistx_add_start fzlistx_add_start
+#define zlistx_add_end fzlistx_add_end
+#define zlistx_size fzlistx_size
+#define zlistx_head fzlistx_head
+#define zlistx_tail fzlistx_tail
+#define zlistx_first fzlistx_first
+#define zlistx_next fzlistx_next
+#define zlistx_prev fzlistx_prev
+#define zlistx_last fzlistx_last
+#define zlistx_item fzlistx_item
+#define zlistx_cursor fzlistx_cursor
+#define zlistx_handle_item fzlistx_handle_item
+#define zlistx_find fzlistx_find
+#define zlistx_detach fzlistx_detach
+#define zlistx_detach_cur fzlistx_detach_cur
+#define zlistx_delete fzlistx_delete
+#define zlistx_move_start fzlistx_move_start
+#define zlistx_move_end fzlistx_move_end
+#define zlistx_purge fzlistx_purge
+#define zlistx_sort fzlistx_sort
+#define zlistx_insert fzlistx_insert
+#define zlistx_reorder fzlistx_reorder
+#define zlistx_dup fzlistx_dup
+#define zlistx_set_destructor fzlistx_set_destructor
+#define zlistx_set_duplicator fzlistx_set_duplicator
+#define zlistx_set_comparator fzlistx_set_comparator
+#define zlistx_test fzlistx_test
+#define zlistx_unpack fzlistx_unpack
+#define zlistx_pack fzlistx_pack
+
+#define zhash_t fzhash_t
+#define zhash_free_fn fzhash_free_fn
+#define zhash_new fzhash_new
+#define zhash_unpack fzhash_unpack
+#define zhash_destroy fzhash_destroy
+#define zhash_insert fzhash_insert
+#define zhash_update fzhash_update
+#define zhash_delete fzhash_delete
+#define zhash_lookup fzhash_lookup
+#define zhash_rename fzhash_rename
+#define zhash_freefn fzhash_freefn
+#define zhash_size fzhash_size
+#define zhash_dup fzhash_dup
+#define zhash_keys fzhash_keys
+#define zhash_first fzhash_first
+#define zhash_next fzhash_next
+#define zhash_cursor fzhash_cursor
+#define zhash_comment fzhash_comment
+#define zhash_pack fzhash_pack
+#define zhash_save fzhash_save
+#define zhash_load fzhash_load
+#define zhash_refresh fzhash_refresh
+#define zhash_autofree fzhash_autofree
+#define zhash_test fzhash_test
+
+#define zlist_t fzlist_t
+#define zlist_compare_fn fzlist_compare_fn
+#define zlist_free_fn fzlist_free_fn
+#define zlist_new fzlist_new
+#define zlist_destroy fzlist_destroy
+#define zlist_first fzlist_first
+#define zlist_next fzlist_next
+#define zlist_last fzlist_last
+#define zlist_head fzlist_head
+#define zlist_tail fzlist_tail
+#define zlist_item fzlist_item
+#define zlist_append fzlist_append
+#define zlist_push fzlist_push
+#define zlist_pop fzlist_pop
+#define zlist_exists fzlist_exists
+#define zlist_remove fzlist_remove
+#define zlist_dup fzlist_dup
+#define zlist_purge fzlist_purge
+#define zlist_size fzlist_size
+#define zlist_sort fzlist_sort
+#define zlist_autofree fzlist_autofree
+#define zlist_comparefn fzlist_comparefn
+#define zlist_freefn fzlist_freefn
+#define zlist_test fzlist_test
+
+#define zstr_free fzstr_free
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/common/libczmqcontainers/zhash.c b/src/common/libczmqcontainers/zhash.c
new file mode 100644
index 000000000000..d73e55fec755
--- /dev/null
+++ b/src/common/libczmqcontainers/zhash.c
@@ -0,0 +1,965 @@
+/*  =========================================================================
+    zhash - simple generic hash container
+
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+/*
+@header
+    zhash is an expandable hash table container. This is a simple container.
+    For heavy-duty applications we recommend using zhashx.
+@discuss
+    Note that it's relatively slow (~50K insertions/deletes per second), so
+    don't do inserts/updates on the critical path for message I/O. It can
+    do ~2.5M lookups per second for 16-char keys. Timed on a 1.6GHz CPU.
+@end
+*/
+
+#include "czmq_containers.h"
+#include "czmq_internal.h"
+
+//  Hash table performance parameters
+
+#define INITIAL_SIZE    255     //  Initial size in items
+#define LOAD_FACTOR     75      //  Percent loading before splitting
+#define GROWTH_FACTOR   200     //  Increase in % after splitting
+
+
+//  Hash item, used internally only
+
+typedef struct _item_t {
+    void *value;                //  Opaque item value
+    struct _item_t *next;       //  Next item in the hash slot
+    size_t index;               //  Index of item in table
+    char *key;                  //  Item's original key
+    zhash_free_fn *free_fn;     //  Value free function if any
+} item_t;
+
+
+//  ---------------------------------------------------------------------
+//  Structure of our class
+
+struct _zhash_t {
+    size_t size;                //  Current size of hash table
+    size_t limit;               //  Current hash table limit
+    item_t **items;             //  Array of items
+    size_t cached_index;        //  Avoids duplicate hash calculations
+    bool autofree;              //  If true, free values in destructor
+    size_t cursor_index;        //  For first/next iteration
+    item_t *cursor_item;        //  For first/next iteration
+    const char *cursor_key;     //  After first/next call, points to key
+    zlist_t *comments;          //  File comments, if any
+    time_t modified;            //  Set during zhash_load
+    char *filename;             //  Set during zhash_load
+};
+
+//  Local helper functions
+static uint s_item_hash (const char *key, size_t limit);
+static item_t *s_item_lookup (zhash_t *self, const char *key);
+static item_t *s_item_insert (zhash_t *self, const char *key, void *value);
+static void s_item_destroy (zhash_t *self, item_t *item, bool hard);
+
+
+//  --------------------------------------------------------------------------
+//  Hash table constructor
+
+zhash_t *
+zhash_new (void)
+{
+    zhash_t *self = (zhash_t *) zmalloc (sizeof (zhash_t));
+    assert (self);
+    self->limit = INITIAL_SIZE;
+    self->items = (item_t **) zmalloc (sizeof (item_t *) * self->limit);
+    assert (self->items);
+    return self;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Hash table destructor
+
+void
+zhash_destroy (zhash_t **self_p)
+{
+    assert (self_p);
+    if (*self_p) {
+        zhash_t *self = *self_p;
+        uint index;
+        for (index = 0; index < self->limit; index++) {
+            //  Destroy all items in this hash bucket
+            item_t *cur_item = self->items [index];
+            while (cur_item) {
+                item_t *next_item = cur_item->next;
+                s_item_destroy (self, cur_item, true);
+                cur_item = next_item;
+            }
+        }
+        if (self->items)
+            freen (self->items);
+
+        zlist_destroy (&self->comments);
+        freen (self->filename);
+        freen (self);
+        *self_p = NULL;
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Local helper function
+//  Destroy item in hash table, item must exist in table
+
+static void
+s_item_destroy (zhash_t *self, item_t *item, bool hard)
+{
+    //  Find previous item since it's a singly-linked list
+    item_t *cur_item = self->items [item->index];
+    item_t **prev_item = &(self->items [item->index]);
+    while (cur_item) {
+        if (cur_item == item)
+            break;
+        prev_item = &(cur_item->next);
+        cur_item = cur_item->next;
+    }
+    assert (cur_item);
+    *prev_item = item->next;
+    self->size--;
+    if (hard) {
+        if (item->free_fn)
+            (item->free_fn) (item->value);
+        else
+        if (self->autofree)
+            freen (item->value);
+
+        freen (item->key);
+        self->cursor_item = NULL;
+        self->cursor_key = NULL;
+        freen (item);
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Insert item into hash table with specified key and item
+//  If key is already present returns -1 and leaves existing item unchanged
+//  Returns 0 on success.
+
+int
+zhash_insert (zhash_t *self, const char *key, void *value)
+{
+    assert (self);
+    assert (key);
+
+    //  If we're exceeding the load factor of the hash table,
+    //  resize it according to the growth factor
+    if (self->size >= self->limit * LOAD_FACTOR / 100) {
+        //  Create new hash table
+        size_t new_limit = self->limit * GROWTH_FACTOR / 100;
+        item_t **new_items = (item_t **) zmalloc (sizeof (item_t *) * new_limit);
+        if (!new_items)
+            return -1;
+
+        //  Move all items to the new hash table, rehashing to
+        //  take into account new hash table limit
+        uint index;
+        for (index = 0; index != self->limit; index++) {
+            item_t *cur_item = self->items [index];
+            while (cur_item) {
+                item_t *next_item = cur_item->next;
+                uint new_index = s_item_hash (cur_item->key, new_limit);
+                cur_item->index = new_index;
+                cur_item->next = new_items [new_index];
+                new_items [new_index] = cur_item;
+                cur_item = next_item;
+            }
+        }
+        //  Destroy old hash table
+        freen (self->items);
+        self->items = new_items;
+        self->limit = new_limit;
+    }
+    return s_item_insert (self, key, value)? 0: -1;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Local helper function
+//  Compute hash for key string
+
+static uint
+s_item_hash (const char *key, size_t limit)
+{
+    //  Modified Bernstein hashing function
+    uint key_hash = 0;
+    while (*key)
+        key_hash = 33 * key_hash ^ *key++;
+    key_hash %= limit;
+    return key_hash;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Local helper function
+//  Insert new item into hash table, returns item
+//  If item already existed, returns NULL
+
+static item_t *
+s_item_insert (zhash_t *self, const char *key, void *value)
+{
+    //  Check that item does not already exist in hash table
+    //  Leaves self->cached_index with calculated hash item
+    item_t *item = s_item_lookup (self, key);
+    if (item == NULL) {
+        item = (item_t *) zmalloc (sizeof (item_t));
+        assert (item);
+
+        //  If necessary, take duplicate of item (string) value
+        if (self->autofree) {
+            value = strdup ((char *) value);
+            assert (value);
+        }
+        item->value = value;
+        item->key = strdup (key);
+        item->index = self->cached_index;
+
+        //  Insert into start of bucket list
+        item->next = self->items [self->cached_index];
+        self->items [self->cached_index] = item;
+        self->size++;
+    }
+    else
+        item = NULL;            //  Signal duplicate insertion
+
+    return item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Local helper function
+//  Lookup item in hash table, returns item or NULL
+
+static item_t *
+s_item_lookup (zhash_t *self, const char *key)
+{
+    //  Look in bucket list for item by key
+    self->cached_index = s_item_hash (key, self->limit);
+    item_t *item = self->items [self->cached_index];
+    while (item) {
+        if (streq (item->key, key))
+            break;
+        item = item->next;
+    }
+    return item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Update item into hash table with specified key and item.
+//  If key is already present, destroys old item and inserts new one.
+//  Use free_fn method to ensure deallocator is properly called on item.
+
+void
+zhash_update (zhash_t *self, const char *key, void *value)
+{
+    assert (self);
+    assert (key);
+
+    item_t *item = s_item_lookup (self, key);
+    if (item) {
+        if (item->free_fn)
+            (item->free_fn) (item->value);
+        else
+        if (self->autofree)
+            freen (item->value);
+
+        //  If necessary, take duplicate of item (string) value
+        if (self->autofree) {
+            value = strdup ((char *) value);
+            assert (value);
+        }
+        item->value = value;
+    }
+    else
+        zhash_insert (self, key, value);
+}
+
+
+//  --------------------------------------------------------------------------
+//  Remove an item specified by key from the hash table. If there was no such
+//  item, this function does nothing.
+
+void
+zhash_delete (zhash_t *self, const char *key)
+{
+    assert (self);
+    assert (key);
+
+    item_t *item = s_item_lookup (self, key);
+    if (item)
+        s_item_destroy (self, item, true);
+}
+
+
+//  --------------------------------------------------------------------------
+//  Look for item in hash table and return its item, or NULL
+
+void *
+zhash_lookup (zhash_t *self, const char *key)
+{
+    assert (self);
+    assert (key);
+
+    item_t *item = s_item_lookup (self, key);
+    if (item)
+        return item->value;
+    else
+        return NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Reindexes an item from an old key to a new key. If there was no such
+//  item, does nothing. If the new key already exists, deletes old item.
+
+int
+zhash_rename (zhash_t *self, const char *old_key, const char *new_key)
+{
+    item_t *old_item = s_item_lookup (self, old_key);
+    item_t *new_item = s_item_lookup (self, new_key);
+    if (old_item && !new_item) {
+        s_item_destroy (self, old_item, false);
+        freen (old_item->key);
+        old_item->key = strdup (new_key);
+        assert (old_item->key);
+        old_item->index = self->cached_index;
+        old_item->next = self->items [self->cached_index];
+        self->items [self->cached_index] = old_item;
+        self->size++;
+        return 0;
+    }
+    else
+        return -1;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a free function for the specified hash table item. When the item is
+//  destroyed, the free function, if any, is called on that item.
+//  Use this when hash items are dynamically allocated, to ensure that
+//  you don't have memory leaks. You can pass 'free' or NULL as a free_fn.
+//  Returns the item, or NULL if there is no such item.
+
+void *
+zhash_freefn (zhash_t *self, const char *key, zhash_free_fn free_fn)
+{
+    assert (self);
+    assert (key);
+
+    item_t *item = s_item_lookup (self, key);
+    if (item) {
+        item->free_fn = free_fn;
+        return item->value;
+    }
+    else
+        return NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return size of hash table
+
+size_t
+zhash_size (zhash_t *self)
+{
+    assert (self);
+    return self->size;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Make copy of hash table
+//  Does not copy items themselves. Rebuilds new table so may be slow on
+//  very large tables. NOTE: only works with item values that are strings
+//  since there's no other way to know how to duplicate the item value.
+
+zhash_t *
+zhash_dup (zhash_t *self)
+{
+    if (!self)
+        return NULL;
+
+    zhash_t *copy = zhash_new ();
+    zhash_autofree (copy);
+    if (copy) {
+        uint index;
+        for (index = 0; index != self->limit; index++) {
+            item_t *item = self->items [index];
+            while (item) {
+                zhash_insert (copy, item->key, item->value);
+                item = item->next;
+            }
+        }
+    }
+    return copy;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return keys for items in table
+
+zlist_t *
+zhash_keys (zhash_t *self)
+{
+    assert (self);
+    zlist_t *keys = zlist_new ();
+    if (!keys)
+        return NULL;
+    zlist_autofree (keys);
+
+    uint index;
+    for (index = 0; index != self->limit; index++) {
+        item_t *item = self->items [index];
+        while (item) {
+            zlist_append (keys, item->key);
+            item = item->next;
+        }
+    }
+    return keys;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Simple iterator; returns first item in hash table, in no given order,
+//  or NULL if the table is empty. This method is simpler to use than the
+//  foreach() method, which is deprecated. NOTE: do NOT modify the table
+//  while iterating.
+
+void *
+zhash_first (zhash_t *self)
+{
+    assert (self);
+    //  Point to before or at first item
+    self->cursor_index = 0;
+    self->cursor_item = self->items [self->cursor_index];
+    //  Now scan forwards to find it, leave cursor after item
+    return zhash_next (self);
+}
+
+
+//  --------------------------------------------------------------------------
+//  Simple iterator; returns next item in hash table, in no given order,
+//  or NULL if the last item was already returned. Use this together with
+//  zhash_first() to process all items in a hash table. If you need the
+//  items in sorted order, use zhash_keys() and then zlist_sort(). NOTE:
+//  do NOT modify the table while iterating.
+
+void *
+zhash_next (zhash_t *self)
+{
+    assert (self);
+    //  Scan forward from cursor until we find an item
+    while (self->cursor_item == NULL) {
+        if (self->cursor_index < self->limit - 1)
+            self->cursor_index++;
+        else
+            return NULL;        //  At end of table
+
+        //  Get first item in next bucket
+        self->cursor_item = self->items [self->cursor_index];
+    }
+    //  We have an item, so return it, and bump past it
+    assert (self->cursor_item);
+    item_t *item = self->cursor_item;
+    self->cursor_key = item->key;
+    self->cursor_item = self->cursor_item->next;
+    return item->value;
+}
+
+
+//  --------------------------------------------------------------------------
+//  After a successful first/next method, returns the key for the item that
+//  was returned. This is a constant string that you may not modify or
+//  deallocate, and which lasts as long as the item in the hash. After an
+//  unsuccessful first/next, returns NULL.
+
+const char *
+zhash_cursor (zhash_t *self)
+{
+    assert (self);
+    return self->cursor_key;
+}
+
+
+#ifdef CZMQ_BUILD_EXTRA
+//  --------------------------------------------------------------------------
+//  Add a comment to hash table before saving to disk. You can add as many
+//  comment lines as you like. These comment lines are discarded when loading
+//  the file. If you use a null format, all comments are deleted.
+
+void
+zhash_comment (zhash_t *self, const char *format, ...)
+{
+    if (format) {
+        if (!self->comments) {
+            self->comments = zlist_new ();
+            if (!self->comments)
+                return;
+            zlist_autofree (self->comments);
+        }
+        va_list argptr;
+        va_start (argptr, format);
+        char *string = zsys_vprintf (format, argptr);
+        va_end (argptr);
+        if (string)
+            zlist_append (self->comments, string);
+        zstr_free (&string);
+    }
+    else
+        zlist_destroy (&self->comments);
+}
+
+
+//  --------------------------------------------------------------------------
+//  Save hash table to a text file in name=value format
+//  Hash values must be printable strings.
+//  Returns 0 if OK, else -1 if a file error occurred
+
+int
+zhash_save (zhash_t *self, const char *filename)
+{
+    assert (self);
+
+    FILE *handle = fopen (filename, "w");
+    if (!handle)
+        return -1;              //  Failed to create file
+
+    if (self->comments) {
+        char *comment = (char *) zlist_first (self->comments);
+        while (comment) {
+            fprintf (handle, "#   %s\n", comment);
+            comment = (char *) zlist_next (self->comments);
+        }
+        fprintf (handle, "\n");
+    }
+    uint index;
+    for (index = 0; index != self->limit; index++) {
+        item_t *item = self->items [index];
+        while (item) {
+            fprintf (handle, "%s=%s\n", item->key, (char *) item->value);
+            item = item->next;
+        }
+    }
+    fclose (handle);
+    return 0;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Load hash table from a text file in name=value format; hash table must
+//  already exist. Hash values must printable strings.
+//  Returns 0 if OK, else -1 if a file was not readable.
+
+int
+zhash_load (zhash_t *self, const char *filename)
+{
+    assert (self);
+    zhash_autofree (self);
+
+    //  Whether or not file exists, we'll track the filename and last
+    //  modification date (0 for unknown files), so that zhash_refresh ()
+    //  will always work after zhash_load (), to load a newly-created
+    //  file.
+
+    //  Take copy of filename in case self->filename is same string.
+    char *filename_copy = strdup (filename);
+    assert (filename_copy);
+    freen (self->filename);
+    self->filename = filename_copy;
+    self->modified = zsys_file_modified (self->filename);
+
+    FILE *handle = fopen (self->filename, "r");
+    if (handle) {
+        char *buffer = (char *) zmalloc (1024);
+        assert (buffer);
+        while (fgets (buffer, 1024, handle)) {
+            //  Skip lines starting with "#" or that do not look like
+            //  name=value data.
+            char *equals = strchr (buffer, '=');
+            if (buffer [0] == '#' || equals == buffer || !equals)
+                continue;
+
+            //  Buffer may end in newline, which we don't want
+            if (buffer [strlen (buffer) - 1] == '\n')
+                buffer [strlen (buffer) - 1] = 0;
+            *equals++ = 0;
+            zhash_update (self, buffer, equals);
+        }
+        freen (buffer);
+        fclose (handle);
+    }
+    else
+        return -1; // Failed to open file for reading
+
+    return 0;
+}
+
+
+//  --------------------------------------------------------------------------
+//  When a hash table was loaded from a file by zhash_load, this method will
+//  reload the file if it has been modified since, and is "stable", i.e. not
+//  still changing. Returns 0 if OK, -1 if there was an error reloading the
+//  file.
+
+int
+zhash_refresh (zhash_t *self)
+{
+    assert (self);
+
+    if (self->filename) {
+        if (zsys_file_modified (self->filename) > self->modified
+        &&  zsys_file_stable (self->filename)) {
+            //  Empty the hash table; code is copied from zhash_destroy
+            uint index;
+            for (index = 0; index < self->limit; index++) {
+                //  Destroy all items in this hash bucket
+                item_t *cur_item = self->items [index];
+                while (cur_item) {
+                    item_t *next_item = cur_item->next;
+                    s_item_destroy (self, cur_item, true);
+                    cur_item = next_item;
+                }
+            }
+            zhash_load (self, self->filename);
+        }
+    }
+    return 0;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Serialize hash table to a binary frame that can be sent in a message.
+//  The packed format is compatible with the 'dictionary' type defined in
+//  http://rfc.zeromq.org/spec:35/FILEMQ, and implemented by zproto:
+//
+//     ; A list of name/value pairs
+//     dictionary      = dict-count *( dict-name dict-value )
+//     dict-count      = number-4
+//     dict-value      = longstr
+//     dict-name       = string
+//
+//     ; Strings are always length + text contents
+//     longstr         = number-4 *VCHAR
+//     string          = number-1 *VCHAR
+//
+//     ; Numbers are unsigned integers in network byte order
+//     number-1        = 1OCTET
+//     number-4        = 4OCTET
+//
+//  Comments are not included in the packed data. Item values MUST be
+//  strings.
+
+zframe_t *
+zhash_pack (zhash_t *self)
+{
+    assert (self);
+
+    //  First, calculate packed data size
+    size_t frame_size = 4;      //  Dictionary size, number-4
+    uint index;
+    for (index = 0; index < self->limit; index++) {
+        item_t *item = self->items [index];
+        while (item) {
+            //  We store key as short string
+            frame_size += 1 + strlen ((char *) item->key);
+            //  We store value as long string
+            frame_size += 4 + strlen ((char *) item->value);
+            item = item->next;
+        }
+    }
+    //  Now serialize items into the frame
+    zframe_t *frame = zframe_new (NULL, frame_size);
+    if (!frame)
+        return NULL;
+
+    byte *needle = zframe_data (frame);
+
+    //  Store size as number-4
+    *(uint32_t *) needle = htonl ((uint32_t) self->size);
+    needle += 4;
+    for (index = 0; index < self->limit; index++) {
+        item_t *item = self->items [index];
+        while (item) {
+            //  Store key as string
+            size_t length = strlen ((char *) item->key);
+            *needle++ = (byte) length;
+            memcpy (needle, item->key, length);
+            needle += length;
+
+            //  Store value as longstr
+            length = strlen ((char *) item->value);
+            uint32_t serialize = htonl ((uint32_t) length);
+            memcpy (needle, &serialize, 4);
+            needle += 4;
+            memcpy (needle, (char *) item->value, length);
+            needle += length;
+            item = item->next;
+        }
+    }
+    return frame;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Unpack binary frame into a new hash table. Packed data must follow format
+//  defined by zhash_pack. Hash table is set to autofree. An empty frame
+//  unpacks to an empty hash table.
+
+zhash_t *
+zhash_unpack (zframe_t *frame)
+{
+    zhash_t *self = zhash_new ();
+    if (!self)
+        return NULL;
+    assert (frame);
+    if (zframe_size (frame) < 4)
+        return self;            //  Arguable...
+
+    byte *needle = zframe_data (frame);
+    byte *ceiling = needle + zframe_size (frame);
+    size_t nbr_items = ntohl (*(uint32_t *) needle);
+    needle += 4;
+    while (nbr_items && needle < ceiling) {
+        //  Get key as string
+        size_t key_size = *needle++;
+        if (needle + key_size <= ceiling) {
+            char key [256];
+            memcpy (key, needle, key_size);
+            key [key_size] = 0;
+            needle += key_size;
+
+            //  Get value as longstr
+            if (needle + 4 <= ceiling) {
+                size_t value_size = ntohl (*(uint32_t *) needle);
+                needle += 4;
+                //  Be wary of malformed frames
+                if (needle + value_size <= ceiling) {
+                    char *value = (char *) malloc (value_size + 1);
+                    memcpy (value, needle, value_size);
+                    value [value_size] = 0;
+                    needle += value_size;
+
+                    //  Hash takes ownership of value
+                    if (zhash_insert (self, key, value)) {
+                        zhash_destroy (&self);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    //  Hash will free values in destructor
+    if (self)
+        zhash_autofree (self);
+    return self;
+}
+#endif // CZMQ_BUILD_EXTRA
+
+
+//  --------------------------------------------------------------------------
+//  Set hash for automatic value destruction
+
+void
+zhash_autofree (zhash_t *self)
+{
+    assert (self);
+    self->autofree = true;
+}
+
+
+#ifdef CZMQ_BUILD_EXTRA
+//  --------------------------------------------------------------------------
+//  Runs selftest of class
+//
+
+void
+zhash_test (bool verbose)
+{
+    printf (" * zhash: ");
+
+    //  @selftest
+    zhash_t *hash = zhash_new ();
+    assert (hash);
+    assert (zhash_size (hash) == 0);
+    assert (zhash_first (hash) == NULL);
+    assert (zhash_cursor (hash) == NULL);
+
+    //  Insert some items
+    int rc;
+    rc = zhash_insert (hash, "DEADBEEF", "dead beef");
+    char *item = (char *) zhash_first (hash);
+    assert (streq (zhash_cursor (hash), "DEADBEEF"));
+    assert (streq (item, "dead beef"));
+    assert (rc == 0);
+    rc = zhash_insert (hash, "ABADCAFE", "a bad cafe");
+    assert (rc == 0);
+    rc = zhash_insert (hash, "C0DEDBAD", "coded bad");
+    assert (rc == 0);
+    rc = zhash_insert (hash, "DEADF00D", "dead food");
+    assert (rc == 0);
+    assert (zhash_size (hash) == 4);
+
+    //  Look for existing items
+    item = (char *) zhash_lookup (hash, "DEADBEEF");
+    assert (streq (item, "dead beef"));
+    item = (char *) zhash_lookup (hash, "ABADCAFE");
+    assert (streq (item, "a bad cafe"));
+    item = (char *) zhash_lookup (hash, "C0DEDBAD");
+    assert (streq (item, "coded bad"));
+    item = (char *) zhash_lookup (hash, "DEADF00D");
+    assert (streq (item, "dead food"));
+
+    //  Look for non-existent items
+    item = (char *) zhash_lookup (hash, "foo");
+    assert (item == NULL);
+
+    //  Try to insert duplicate items
+    rc = zhash_insert (hash, "DEADBEEF", "foo");
+    assert (rc == -1);
+    item = (char *) zhash_lookup (hash, "DEADBEEF");
+    assert (streq (item, "dead beef"));
+
+    //  Some rename tests
+
+    //  Valid rename, key is now LIVEBEEF
+    rc = zhash_rename (hash, "DEADBEEF", "LIVEBEEF");
+    assert (rc == 0);
+    item = (char *) zhash_lookup (hash, "LIVEBEEF");
+    assert (streq (item, "dead beef"));
+
+    //  Trying to rename an unknown item to a non-existent key
+    rc = zhash_rename (hash, "WHATBEEF", "NONESUCH");
+    assert (rc == -1);
+
+    //  Trying to rename an unknown item to an existing key
+    rc = zhash_rename (hash, "WHATBEEF", "LIVEBEEF");
+    assert (rc == -1);
+    item = (char *) zhash_lookup (hash, "LIVEBEEF");
+    assert (streq (item, "dead beef"));
+
+    //  Trying to rename an existing item to another existing item
+    rc = zhash_rename (hash, "LIVEBEEF", "ABADCAFE");
+    assert (rc == -1);
+    item = (char *) zhash_lookup (hash, "LIVEBEEF");
+    assert (streq (item, "dead beef"));
+    item = (char *) zhash_lookup (hash, "ABADCAFE");
+    assert (streq (item, "a bad cafe"));
+
+    //  Test keys method
+    zlist_t *keys = zhash_keys (hash);
+    assert (zlist_size (keys) == 4);
+    zlist_destroy (&keys);
+
+    //  Test dup method
+    zhash_t *copy = zhash_dup (hash);
+    assert (zhash_size (copy) == 4);
+    item = (char *) zhash_lookup (copy, "LIVEBEEF");
+    assert (item);
+    assert (streq (item, "dead beef"));
+    zhash_destroy (&copy);
+
+    //  Test pack/unpack methods
+    zframe_t *frame = zhash_pack (hash);
+    copy = zhash_unpack (frame);
+    zframe_destroy (&frame);
+    assert (zhash_size (copy) == 4);
+    item = (char *) zhash_lookup (copy, "LIVEBEEF");
+    assert (item);
+    assert (streq (item, "dead beef"));
+    zhash_destroy (&copy);
+
+    //  Test save and load
+    zhash_comment (hash, "This is a test file");
+    zhash_comment (hash, "Created by %s", "czmq_selftest");
+    zhash_save (hash, ".cache");
+    copy = zhash_new ();
+    assert (copy);
+    zhash_load (copy, ".cache");
+    item = (char *) zhash_lookup (copy, "LIVEBEEF");
+    assert (item);
+    assert (streq (item, "dead beef"));
+    zhash_destroy (&copy);
+    zsys_file_delete (".cache");
+
+    //  Delete a item
+    zhash_delete (hash, "LIVEBEEF");
+    item = (char *) zhash_lookup (hash, "LIVEBEEF");
+    assert (item == NULL);
+    assert (zhash_size (hash) == 3);
+
+    //  Check that the queue is robust against random usage
+    struct {
+        char name [100];
+        bool exists;
+    } testset [200];
+    memset (testset, 0, sizeof (testset));
+    int testmax = 200, testnbr, iteration;
+
+    srandom ((unsigned) time (NULL));
+    for (iteration = 0; iteration < 25000; iteration++) {
+        testnbr = randof (testmax);
+        assert (testnbr != testmax);
+        assert (testnbr < testmax);
+        if (testset [testnbr].exists) {
+            item = (char *) zhash_lookup (hash, testset [testnbr].name);
+            assert (item);
+            zhash_delete (hash, testset [testnbr].name);
+            testset [testnbr].exists = false;
+        }
+        else {
+            sprintf (testset [testnbr].name, "%x-%x", rand (), rand ());
+            if (zhash_insert (hash, testset [testnbr].name, "") == 0)
+                testset [testnbr].exists = true;
+        }
+    }
+    //  Test 10K lookups
+    for (iteration = 0; iteration < 10000; iteration++)
+        item = (char *) zhash_lookup (hash, "DEADBEEFABADCAFE");
+
+    //  Destructor should be safe to call twice
+    zhash_destroy (&hash);
+    zhash_destroy (&hash);
+    assert (hash == NULL);
+
+    // Test autofree; automatically copies and frees string values
+    hash = zhash_new ();
+    assert (hash);
+    zhash_autofree (hash);
+    char value [255];
+    strcpy (value, "This is a string");
+    rc = zhash_insert (hash, "key1", value);
+    assert (rc == 0);
+    strcpy (value, "Inserting with the same key will fail");
+    rc = zhash_insert (hash, "key1", value);
+    assert (rc == -1);
+    strcpy (value, "Ring a ding ding");
+    rc = zhash_insert (hash, "key2", value);
+    assert (rc == 0);
+    assert (streq ((char *) zhash_lookup (hash, "key1"), "This is a string"));
+    assert (streq ((char *) zhash_lookup (hash, "key2"), "Ring a ding ding"));
+    zhash_destroy (&hash);
+
+#if defined (__WINDOWS__)
+    zsys_shutdown();
+#endif
+    //  @end
+
+    printf ("OK\n");
+}
+#endif // CZMQ_BUILD_EXTRA
diff --git a/src/common/libczmqcontainers/zhash.h b/src/common/libczmqcontainers/zhash.h
new file mode 100644
index 000000000000..784833c6c8ed
--- /dev/null
+++ b/src/common/libczmqcontainers/zhash.h
@@ -0,0 +1,189 @@
+/*  =========================================================================
+    zhash - generic type-free hash container (simple)
+
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+#ifndef __FLUXZHASH_H_INCLUDED__
+#define __FLUXZHASH_H_INCLUDED__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//  @warning THE FOLLOWING @INTERFACE BLOCK IS AUTO-GENERATED BY ZPROJECT
+//  @warning Please edit the model at "api/zhash.api" to make changes.
+//  @interface
+//  This is a stable class, and may not change except for emergencies. It
+//  is provided in stable builds.
+// Callback function for zhash_freefn method
+typedef void (zhash_free_fn) (
+    void *data);
+
+//  Create a new, empty hash container
+CZMQ_EXPORT zhash_t *
+    zhash_new (void);
+
+#ifdef CZMQ_BUILD_EXTRA
+//  Unpack binary frame into a new hash table. Packed data must follow format
+//  defined by zhash_pack. Hash table is set to autofree. An empty frame
+//  unpacks to an empty hash table.
+CZMQ_EXPORT zhash_t *
+    zhash_unpack (zframe_t *frame);
+#endif // CZMQ_BUILD_EXTRA
+
+//  Destroy a hash container and all items in it
+CZMQ_EXPORT void
+    zhash_destroy (zhash_t **self_p);
+
+//  Insert item into hash table with specified key and item.
+//  If key is already present returns -1 and leaves existing item unchanged
+//  Returns 0 on success.
+CZMQ_EXPORT int
+    zhash_insert (zhash_t *self, const char *key, void *item);
+
+//  Update item into hash table with specified key and item.
+//  If key is already present, destroys old item and inserts new one.
+//  Use free_fn method to ensure deallocator is properly called on item.
+CZMQ_EXPORT void
+    zhash_update (zhash_t *self, const char *key, void *item);
+
+//  Remove an item specified by key from the hash table. If there was no such
+//  item, this function does nothing.
+CZMQ_EXPORT void
+    zhash_delete (zhash_t *self, const char *key);
+
+//  Return the item at the specified key, or null
+CZMQ_EXPORT void *
+    zhash_lookup (zhash_t *self, const char *key);
+
+//  Reindexes an item from an old key to a new key. If there was no such
+//  item, does nothing. Returns 0 if successful, else -1.
+CZMQ_EXPORT int
+    zhash_rename (zhash_t *self, const char *old_key, const char *new_key);
+
+//  Set a free function for the specified hash table item. When the item is
+//  destroyed, the free function, if any, is called on that item.
+//  Use this when hash items are dynamically allocated, to ensure that
+//  you don't have memory leaks. You can pass 'free' or NULL as a free_fn.
+//  Returns the item, or NULL if there is no such item.
+CZMQ_EXPORT void *
+    zhash_freefn (zhash_t *self, const char *key, zhash_free_fn free_fn);
+
+//  Return the number of keys/items in the hash table
+CZMQ_EXPORT size_t
+    zhash_size (zhash_t *self);
+
+//  Make copy of hash table; if supplied table is null, returns null.
+//  Does not copy items themselves. Rebuilds new table so may be slow on
+//  very large tables. NOTE: only works with item values that are strings
+//  since there's no other way to know how to duplicate the item value.
+//  Caller owns return value and must destroy it when done.
+CZMQ_EXPORT zhash_t *
+    zhash_dup (zhash_t *self);
+
+//  Return keys for items in table
+//  Caller owns return value and must destroy it when done.
+CZMQ_EXPORT zlist_t *
+    zhash_keys (zhash_t *self);
+
+//  Simple iterator; returns first item in hash table, in no given order,
+//  or NULL if the table is empty. This method is simpler to use than the
+//  foreach() method, which is deprecated. To access the key for this item
+//  use zhash_cursor(). NOTE: do NOT modify the table while iterating.
+CZMQ_EXPORT void *
+    zhash_first (zhash_t *self);
+
+//  Simple iterator; returns next item in hash table, in no given order,
+//  or NULL if the last item was already returned. Use this together with
+//  zhash_first() to process all items in a hash table. If you need the
+//  items in sorted order, use zhash_keys() and then zlist_sort(). To
+//  access the key for this item use zhash_cursor(). NOTE: do NOT modify
+//  the table while iterating.
+CZMQ_EXPORT void *
+    zhash_next (zhash_t *self);
+
+//  After a successful first/next method, returns the key for the item that
+//  was returned. This is a constant string that you may not modify or
+//  deallocate, and which lasts as long as the item in the hash. After an
+//  unsuccessful first/next, returns NULL.
+CZMQ_EXPORT const char *
+    zhash_cursor (zhash_t *self);
+
+#ifdef CZMQ_BUILD_EXTRA
+//  Add a comment to hash table before saving to disk. You can add as many
+//  comment lines as you like. These comment lines are discarded when loading
+//  the file. If you use a null format, all comments are deleted.
+CZMQ_EXPORT void
+    zhash_comment (zhash_t *self, const char *format, ...) CHECK_PRINTF (2);
+
+//  Serialize hash table to a binary frame that can be sent in a message.
+//  The packed format is compatible with the 'dictionary' type defined in
+//  http://rfc.zeromq.org/spec:35/FILEMQ, and implemented by zproto:
+//
+//     ; A list of name/value pairs
+//     dictionary      = dict-count *( dict-name dict-value )
+//     dict-count      = number-4
+//     dict-value      = longstr
+//     dict-name       = string
+//
+//     ; Strings are always length + text contents
+//     longstr         = number-4 *VCHAR
+//     string          = number-1 *VCHAR
+//
+//     ; Numbers are unsigned integers in network byte order
+//     number-1        = 1OCTET
+//     number-4        = 4OCTET
+//
+//  Comments are not included in the packed data. Item values MUST be
+//  strings.
+//  Caller owns return value and must destroy it when done.
+CZMQ_EXPORT zframe_t *
+    zhash_pack (zhash_t *self);
+
+//  Save hash table to a text file in name=value format. Hash values must be
+//  printable strings; keys may not contain '=' character. Returns 0 if OK,
+//  else -1 if a file error occurred.
+CZMQ_EXPORT int
+    zhash_save (zhash_t *self, const char *filename);
+
+//  Load hash table from a text file in name=value format; hash table must
+//  already exist. Hash values must printable strings; keys may not contain
+//  '=' character. Returns 0 if OK, else -1 if a file was not readable.
+CZMQ_EXPORT int
+    zhash_load (zhash_t *self, const char *filename);
+
+//  When a hash table was loaded from a file by zhash_load, this method will
+//  reload the file if it has been modified since, and is "stable", i.e. not
+//  still changing. Returns 0 if OK, -1 if there was an error reloading the
+//  file.
+CZMQ_EXPORT int
+    zhash_refresh (zhash_t *self);
+#endif // CZMQ_BUILD_EXTRA
+
+//  Set hash for automatic value destruction. Note that this assumes that
+//  values are NULL-terminated strings. Do not use with different types.
+CZMQ_EXPORT void
+    zhash_autofree (zhash_t *self);
+
+#ifdef CZMQ_BUILD_EXTRA
+//  Self test of this class.
+CZMQ_EXPORT void
+    zhash_test (bool verbose);
+#endif // CZMQ_BUILD_EXTRA
+
+//  @end
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/common/libczmqcontainers/zhash_primes.inc b/src/common/libczmqcontainers/zhash_primes.inc
new file mode 100644
index 000000000000..a572bd5d70bc
--- /dev/null
+++ b/src/common/libczmqcontainers/zhash_primes.inc
@@ -0,0 +1,339 @@
+/*  =========================================================================
+    zhash_primes.h - 5 largest primes less than 2^n for n = 4...63
+
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+/* This is a copy of the czmq zhashx library.
+ *
+ * Due to czmq issue #2173
+ * (https://github.com/zeromq/czmq/issues/2173) a localized copy was
+ * made instead of waiting for OS distros to pick up the bug fix.
+ *
+ * This file is a verbatim copy with only minor adjustments to header
+ * guards.
+ */
+
+#ifndef __FLUXZHASH_PRIMES_H_INCLUDED__
+#define __FLUXZHASH_PRIMES_H_INCLUDED__
+
+#ifdef _MSC_VER
+#	define PORTABLE_LLU(number) number##ULL
+#else
+#	define PORTABLE_LLU(number) number##LLU
+#endif
+
+static size_t primes [] = {
+    PORTABLE_LLU(3),
+    PORTABLE_LLU(5),
+    PORTABLE_LLU(7),
+    PORTABLE_LLU(11),
+    PORTABLE_LLU(13), // 2^4
+    PORTABLE_LLU(17),
+    PORTABLE_LLU(19),
+    PORTABLE_LLU(23),
+    PORTABLE_LLU(29),
+    PORTABLE_LLU(31), // 2^5
+    PORTABLE_LLU(43),
+    PORTABLE_LLU(47),
+    PORTABLE_LLU(53),
+    PORTABLE_LLU(59),
+    PORTABLE_LLU(61), // 2^6
+    PORTABLE_LLU(103),
+    PORTABLE_LLU(107),
+    PORTABLE_LLU(109),
+    PORTABLE_LLU(113),
+    PORTABLE_LLU(127), // 2^7
+    PORTABLE_LLU(229),
+    PORTABLE_LLU(233),
+    PORTABLE_LLU(239),
+    PORTABLE_LLU(241),
+    PORTABLE_LLU(251), // 2^8
+    PORTABLE_LLU(487),
+    PORTABLE_LLU(491),
+    PORTABLE_LLU(499),
+    PORTABLE_LLU(503),
+    PORTABLE_LLU(509), // 2^9
+    PORTABLE_LLU(997),
+    PORTABLE_LLU(1009),
+    PORTABLE_LLU(1013),
+    PORTABLE_LLU(1019),
+    PORTABLE_LLU(1021), // 2^10
+    PORTABLE_LLU(2011),
+    PORTABLE_LLU(2017),
+    PORTABLE_LLU(2027),
+    PORTABLE_LLU(2029),
+    PORTABLE_LLU(2039), // 2^11
+    PORTABLE_LLU(4057),
+    PORTABLE_LLU(4073),
+    PORTABLE_LLU(4079),
+    PORTABLE_LLU(4091),
+    PORTABLE_LLU(4093), // 2^12
+    PORTABLE_LLU(8161),
+    PORTABLE_LLU(8167),
+    PORTABLE_LLU(8171),
+    PORTABLE_LLU(8179),
+    PORTABLE_LLU(8191), // 2^13
+    PORTABLE_LLU(16349),
+    PORTABLE_LLU(16361),
+    PORTABLE_LLU(16363),
+    PORTABLE_LLU(16369),
+    PORTABLE_LLU(16381), // 2^14
+    PORTABLE_LLU(32707),
+    PORTABLE_LLU(32713),
+    PORTABLE_LLU(32717),
+    PORTABLE_LLU(32719),
+    PORTABLE_LLU(32749), // 2^15
+    PORTABLE_LLU(65449),
+    PORTABLE_LLU(65479),
+    PORTABLE_LLU(65497),
+    PORTABLE_LLU(65519),
+    PORTABLE_LLU(65521), // 2^16
+    PORTABLE_LLU(131023),
+    PORTABLE_LLU(131041),
+    PORTABLE_LLU(131059),
+    PORTABLE_LLU(131063),
+    PORTABLE_LLU(131071), // 2^17
+    PORTABLE_LLU(262111),
+    PORTABLE_LLU(262121),
+    PORTABLE_LLU(262127),
+    PORTABLE_LLU(262133),
+    PORTABLE_LLU(262139), // 2^18
+    PORTABLE_LLU(524243),
+    PORTABLE_LLU(524257),
+    PORTABLE_LLU(524261),
+    PORTABLE_LLU(524269),
+    PORTABLE_LLU(524287), // 2^19
+    PORTABLE_LLU(1048517),
+    PORTABLE_LLU(1048549),
+    PORTABLE_LLU(1048559),
+    PORTABLE_LLU(1048571),
+    PORTABLE_LLU(1048573), // 2^20
+    PORTABLE_LLU(2097091),
+    PORTABLE_LLU(2097097),
+    PORTABLE_LLU(2097131),
+    PORTABLE_LLU(2097133),
+    PORTABLE_LLU(2097143), // 2^21
+    PORTABLE_LLU(4194247),
+    PORTABLE_LLU(4194271),
+    PORTABLE_LLU(4194277),
+    PORTABLE_LLU(4194287),
+    PORTABLE_LLU(4194301), // 2^22
+    PORTABLE_LLU(8388547),
+    PORTABLE_LLU(8388571),
+    PORTABLE_LLU(8388581),
+    PORTABLE_LLU(8388587),
+    PORTABLE_LLU(8388593), // 2^23
+    PORTABLE_LLU(16777141),
+    PORTABLE_LLU(16777153),
+    PORTABLE_LLU(16777183),
+    PORTABLE_LLU(16777199),
+    PORTABLE_LLU(16777213), // 2^24
+    PORTABLE_LLU(33554341),
+    PORTABLE_LLU(33554347),
+    PORTABLE_LLU(33554371),
+    PORTABLE_LLU(33554383),
+    PORTABLE_LLU(33554393), // 2^25
+    PORTABLE_LLU(67108763),
+    PORTABLE_LLU(67108777),
+    PORTABLE_LLU(67108819),
+    PORTABLE_LLU(67108837),
+    PORTABLE_LLU(67108859), // 2^26
+    PORTABLE_LLU(134217593),
+    PORTABLE_LLU(134217613),
+    PORTABLE_LLU(134217617),
+    PORTABLE_LLU(134217649),
+    PORTABLE_LLU(134217689), // 2^27
+    PORTABLE_LLU(268435331),
+    PORTABLE_LLU(268435337),
+    PORTABLE_LLU(268435361),
+    PORTABLE_LLU(268435367),
+    PORTABLE_LLU(268435399), // 2^28
+    PORTABLE_LLU(536870839),
+    PORTABLE_LLU(536870849),
+    PORTABLE_LLU(536870869),
+    PORTABLE_LLU(536870879),
+    PORTABLE_LLU(536870909), // 2^29
+    PORTABLE_LLU(1073741719),
+    PORTABLE_LLU(1073741723),
+    PORTABLE_LLU(1073741741),
+    PORTABLE_LLU(1073741783),
+    PORTABLE_LLU(1073741789), // 2^30
+    PORTABLE_LLU(2147483563),
+    PORTABLE_LLU(2147483579),
+    PORTABLE_LLU(2147483587),
+    PORTABLE_LLU(2147483629),
+    PORTABLE_LLU(2147483647), // 2^31
+    PORTABLE_LLU(4294967197),
+    PORTABLE_LLU(4294967231),
+    PORTABLE_LLU(4294967279),
+    PORTABLE_LLU(4294967291),
+    PORTABLE_LLU(4294967295), // 2^32
+#if __WORDSIZE == 64
+    PORTABLE_LLU(8589934581),
+    PORTABLE_LLU(8589934585),
+    PORTABLE_LLU(8589934587),
+    PORTABLE_LLU(8589934589),
+    PORTABLE_LLU(8589934591), // 2^33
+    PORTABLE_LLU(17179869175),
+    PORTABLE_LLU(17179869177),
+    PORTABLE_LLU(17179869179),
+    PORTABLE_LLU(17179869181),
+    PORTABLE_LLU(17179869183), // 2^34
+    PORTABLE_LLU(34359738359),
+    PORTABLE_LLU(34359738361),
+    PORTABLE_LLU(34359738363),
+    PORTABLE_LLU(34359738365),
+    PORTABLE_LLU(34359738367), // 2^35
+    PORTABLE_LLU(68719476725),
+    PORTABLE_LLU(68719476727),
+    PORTABLE_LLU(68719476729),
+    PORTABLE_LLU(68719476733),
+    PORTABLE_LLU(68719476735), // 2^36
+    PORTABLE_LLU(137438953463),
+    PORTABLE_LLU(137438953465),
+    PORTABLE_LLU(137438953467),
+    PORTABLE_LLU(137438953469),
+    PORTABLE_LLU(137438953471), // 2^37
+    PORTABLE_LLU(274877906935),
+    PORTABLE_LLU(274877906937),
+    PORTABLE_LLU(274877906939),
+    PORTABLE_LLU(274877906941),
+    PORTABLE_LLU(274877906943), // 2^38
+    PORTABLE_LLU(549755813877),
+    PORTABLE_LLU(549755813879),
+    PORTABLE_LLU(549755813883),
+    PORTABLE_LLU(549755813885),
+    PORTABLE_LLU(549755813887), // 2^39
+    PORTABLE_LLU(1099511627767),
+    PORTABLE_LLU(1099511627769),
+    PORTABLE_LLU(1099511627771),
+    PORTABLE_LLU(1099511627773),
+    PORTABLE_LLU(1099511627775), // 2^40
+    PORTABLE_LLU(2199023255543),
+    PORTABLE_LLU(2199023255545),
+    PORTABLE_LLU(2199023255547),
+    PORTABLE_LLU(2199023255549),
+    PORTABLE_LLU(2199023255551), // 2^41
+    PORTABLE_LLU(4398046511095),
+    PORTABLE_LLU(4398046511097),
+    PORTABLE_LLU(4398046511099),
+    PORTABLE_LLU(4398046511101),
+    PORTABLE_LLU(4398046511103), // 2^42
+    PORTABLE_LLU(8796093022199),
+    PORTABLE_LLU(8796093022201),
+    PORTABLE_LLU(8796093022203),
+    PORTABLE_LLU(8796093022205),
+    PORTABLE_LLU(8796093022207), // 2^43
+    PORTABLE_LLU(17592186044407),
+    PORTABLE_LLU(17592186044409),
+    PORTABLE_LLU(17592186044411),
+    PORTABLE_LLU(17592186044413),
+    PORTABLE_LLU(17592186044415), // 2^44
+    PORTABLE_LLU(35184372088823),
+    PORTABLE_LLU(35184372088825),
+    PORTABLE_LLU(35184372088827),
+    PORTABLE_LLU(35184372088829),
+    PORTABLE_LLU(35184372088831), // 2^45
+    PORTABLE_LLU(70368744177655),
+    PORTABLE_LLU(70368744177657),
+    PORTABLE_LLU(70368744177659),
+    PORTABLE_LLU(70368744177661),
+    PORTABLE_LLU(70368744177663), // 2^46
+    PORTABLE_LLU(140737488355319),
+    PORTABLE_LLU(140737488355321),
+    PORTABLE_LLU(140737488355323),
+    PORTABLE_LLU(140737488355325),
+    PORTABLE_LLU(140737488355327), // 2^47
+    PORTABLE_LLU(281474976710647),
+    PORTABLE_LLU(281474976710649),
+    PORTABLE_LLU(281474976710651),
+    PORTABLE_LLU(281474976710653),
+    PORTABLE_LLU(281474976710655), // 2^48
+    PORTABLE_LLU(562949953421303),
+    PORTABLE_LLU(562949953421305),
+    PORTABLE_LLU(562949953421307),
+    PORTABLE_LLU(562949953421309),
+    PORTABLE_LLU(562949953421311), // 2^49
+    PORTABLE_LLU(1125899906842615),
+    PORTABLE_LLU(1125899906842617),
+    PORTABLE_LLU(1125899906842619),
+    PORTABLE_LLU(1125899906842621),
+    PORTABLE_LLU(1125899906842623), // 2^50
+    PORTABLE_LLU(2251799813685239),
+    PORTABLE_LLU(2251799813685241),
+    PORTABLE_LLU(2251799813685243),
+    PORTABLE_LLU(2251799813685245),
+    PORTABLE_LLU(2251799813685247), // 2^51
+    PORTABLE_LLU(4503599627370487),
+    PORTABLE_LLU(4503599627370489),
+    PORTABLE_LLU(4503599627370491),
+    PORTABLE_LLU(4503599627370493),
+    PORTABLE_LLU(4503599627370495), // 2^52
+    PORTABLE_LLU(9007199254740983),
+    PORTABLE_LLU(9007199254740985),
+    PORTABLE_LLU(9007199254740987),
+    PORTABLE_LLU(9007199254740989),
+    PORTABLE_LLU(9007199254740991), // 2^53
+    PORTABLE_LLU(18014398509481975),
+    PORTABLE_LLU(18014398509481977),
+    PORTABLE_LLU(18014398509481979),
+    PORTABLE_LLU(18014398509481981),
+    PORTABLE_LLU(18014398509481983), // 2^54
+    PORTABLE_LLU(36028797018963959),
+    PORTABLE_LLU(36028797018963961),
+    PORTABLE_LLU(36028797018963963),
+    PORTABLE_LLU(36028797018963965),
+    PORTABLE_LLU(36028797018963967), // 2^55
+    PORTABLE_LLU(72057594037927925),
+    PORTABLE_LLU(72057594037927927),
+    PORTABLE_LLU(72057594037927929),
+    PORTABLE_LLU(72057594037927933),
+    PORTABLE_LLU(72057594037927935), // 2^56
+    PORTABLE_LLU(144115188075855863),
+    PORTABLE_LLU(144115188075855865),
+    PORTABLE_LLU(144115188075855867),
+    PORTABLE_LLU(144115188075855869),
+    PORTABLE_LLU(144115188075855871), // 2^57
+    PORTABLE_LLU(288230376151711735),
+    PORTABLE_LLU(288230376151711737),
+    PORTABLE_LLU(288230376151711739),
+    PORTABLE_LLU(288230376151711741),
+    PORTABLE_LLU(288230376151711743), // 2^58
+    PORTABLE_LLU(576460752303423479),
+    PORTABLE_LLU(576460752303423481),
+    PORTABLE_LLU(576460752303423483),
+    PORTABLE_LLU(576460752303423485),
+    PORTABLE_LLU(576460752303423487), // 2^59
+    PORTABLE_LLU(1152921504606846967),
+    PORTABLE_LLU(1152921504606846969),
+    PORTABLE_LLU(1152921504606846971),
+    PORTABLE_LLU(1152921504606846973),
+    PORTABLE_LLU(1152921504606846975), // 2^60
+    PORTABLE_LLU(2305843009213693941),
+    PORTABLE_LLU(2305843009213693943),
+    PORTABLE_LLU(2305843009213693945),
+    PORTABLE_LLU(2305843009213693947),
+    PORTABLE_LLU(2305843009213693949), // 2^61
+    PORTABLE_LLU(4611686018427387895),
+    PORTABLE_LLU(4611686018427387897),
+    PORTABLE_LLU(4611686018427387899),
+    PORTABLE_LLU(4611686018427387901),
+    PORTABLE_LLU(4611686018427387903), // 2^62
+    PORTABLE_LLU(9223372036854775799),
+    PORTABLE_LLU(9223372036854775801),
+    PORTABLE_LLU(9223372036854775803),
+    PORTABLE_LLU(9223372036854775805),
+    PORTABLE_LLU(9223372036854775807) // 2^63
+#endif
+};
+
+#endif
+
diff --git a/src/common/libczmqcontainers/zhashx.c b/src/common/libczmqcontainers/zhashx.c
new file mode 100644
index 000000000000..f7d45b230211
--- /dev/null
+++ b/src/common/libczmqcontainers/zhashx.c
@@ -0,0 +1,1392 @@
+/*  =========================================================================
+    zhashx - extended generic hash container
+
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+/* This is a copy of the czmq zhashx library.
+ *
+ * Due to czmq issue #2173
+ * (https://github.com/zeromq/czmq/issues/2173) a localized copy was
+ * made instead of waiting for OS distros to pick up the bug fix.
+ *
+ * This file is a verbatim copy with only minor adjustments included
+ * headers.
+ */
+
+/*
+@header
+    zhashx is an extended hash table container with more functionality than
+    zhash, its simpler cousin.
+@discuss
+    The hash table always has a size that is prime and roughly doubles its
+    size when 75% full. In case of hash collisions items are chained in a
+    linked list. The hash table size is increased slightly (up to 5 times
+    before roughly doubling the size) when an overly long chain (between 1
+    and 63 items depending on table size) is detected.
+@end
+*/
+
+#include "czmq_containers.h"
+#include "czmq_internal.h"
+
+//  Hash table performance parameters
+
+#define INITIAL_PRIME    0    //  Initial size in items (index into primes)
+#define GROWTH_FACTOR    5    //  Increase after splitting (index into primes)
+#define LOAD_FACTOR     75    //  Percent loading before splitting
+#define INITIAL_CHAIN    1    //  Initial chaining limit
+#define CHAIN_GROWS      1    //  Increase after splitting (chaining limit)
+
+#include "zhash_primes.inc"
+
+
+//  Hash item, used internally only
+
+typedef struct _item_t {
+    void *value;                //  Opaque item value
+    struct _item_t *next;       //  Next item in the hash slot
+    size_t index;               //  Index of item in table
+    const void *key;            //  Item's original key
+    zhashx_free_fn *free_fn;     //  Value free function if any
+} item_t;
+
+
+//  ---------------------------------------------------------------------
+//  Structure of our class
+
+struct _zhashx_t {
+    size_t size;                //  Current size of hash table
+    uint prime_index;           //  Current prime number used as limit
+    uint chain_limit;           //  Current limit on chain length
+    item_t **items;             //  Array of items
+    size_t cached_index;        //  Avoids duplicate hash calculations
+    size_t cursor_index;        //  For first/next iteration
+    item_t *cursor_item;        //  For first/next iteration
+    const void *cursor_key;     //  After first/next call, points to key
+    zlistx_t *comments;         //  File comments, if any
+    time_t modified;            //  Set during zhashx_load
+    char *filename;             //  Set during zhashx_load
+    //  Function callbacks for duplicating and destroying items, if any
+    zhashx_duplicator_fn *duplicator;
+    zhashx_destructor_fn *destructor;
+    //  Function callbacks for duplicating and destroying keys, if any
+    zhashx_duplicator_fn *key_duplicator;
+    zhashx_destructor_fn *key_destructor;
+    zhashx_comparator_fn *key_comparator;
+    //  Custom hash function
+    zhashx_hash_fn *hasher;
+};
+
+//  Local helper functions
+static item_t *s_item_lookup (zhashx_t *self, const void *key);
+static item_t *s_item_insert (zhashx_t *self, const void *key, void *value);
+static void s_item_destroy (zhashx_t *self, item_t *item, bool hard);
+
+
+//  --------------------------------------------------------------------------
+//  Modified Bernstein hashing function
+
+static size_t
+s_bernstein_hash (const void *key)
+{
+    const char *pointer = (const char *) key;
+    size_t key_hash = 0;
+    while (*pointer)
+        key_hash = 33 * key_hash ^ *pointer++;
+    return key_hash;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Hash table constructor
+
+zhashx_t *
+zhashx_new (void)
+{
+    zhashx_t *self = (zhashx_t *) zmalloc (sizeof (zhashx_t));
+    assert (self);
+    self->prime_index = INITIAL_PRIME;
+    self->chain_limit = INITIAL_CHAIN;
+    size_t limit = primes [self->prime_index];
+    self->items = (item_t **) zmalloc (sizeof (item_t *) * limit);
+    assert (self->items);
+    self->hasher = s_bernstein_hash;
+    self->key_destructor = (zhashx_destructor_fn *) zstr_free;
+    self->key_duplicator = (zhashx_duplicator_fn *) strdup;
+    self->key_comparator = (zhashx_comparator_fn *) strcmp;
+
+    return self;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Purge all items from a hash table
+
+static void
+s_purge (zhashx_t *self)
+{
+    uint index;
+    size_t limit = primes [self->prime_index];
+
+    for (index = 0; index < limit; index++) {
+        //  Destroy all items in this hash bucket
+        item_t *cur_item = self->items [index];
+        while (cur_item) {
+            item_t *next_item = cur_item->next;
+            s_item_destroy (self, cur_item, true);
+            cur_item = next_item;
+        }
+        self->items [index] = NULL;
+    }
+}
+
+//  --------------------------------------------------------------------------
+//  Hash table destructor
+
+void
+zhashx_destroy (zhashx_t **self_p)
+{
+    assert (self_p);
+    if (*self_p) {
+        zhashx_t *self = *self_p;
+        if (self->items) {
+            s_purge (self);
+            freen (self->items);
+        }
+        zlistx_destroy (&self->comments);
+        freen (self->filename);
+        freen (self);
+        *self_p = NULL;
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Local helper function
+//  Destroy item in hash table, item must exist in table
+
+static void
+s_item_destroy (zhashx_t *self, item_t *item, bool hard)
+{
+    //  Find previous item since it's a singly-linked list
+    item_t *cur_item = self->items [item->index];
+    item_t **prev_item = &(self->items [item->index]);
+    while (cur_item) {
+        if (cur_item == item)
+            break;
+        prev_item = &(cur_item->next);
+        cur_item = cur_item->next;
+    }
+    assert (cur_item);
+    *prev_item = item->next;
+    self->size--;
+    if (hard) {
+        if (self->destructor)
+            (self->destructor)(&item->value);
+        else
+        if (item->free_fn)
+            (item->free_fn)(item->value);
+
+        self->cursor_item = NULL;
+        self->cursor_key = NULL;
+
+        if (self->key_destructor)
+            (self->key_destructor)((void **) &item->key);
+        freen (item);
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Rehash hash table with specified new prime index
+//  Returns 0 on success, or fails the assertions (e.g. insufficient memory)
+//  Note: Older code used to return -1 in case of errors - this is no longer so
+
+static int
+s_zhashx_rehash (zhashx_t *self, uint new_prime_index)
+{
+    assert (self);
+    assert (new_prime_index < sizeof (primes));
+
+    size_t limit = primes [self->prime_index];
+    size_t new_limit = primes [new_prime_index];
+    item_t **new_items = (item_t **) zmalloc (sizeof (item_t *) * new_limit);
+    assert (new_items);
+
+    //  Move all items to the new hash table, rehashing to
+    //  take into account new hash table limit
+    size_t index;
+    for (index = 0; index < limit; index++) {
+        item_t *cur_item = self->items [index];
+        while (cur_item) {
+            item_t *next_item = cur_item->next;
+            size_t new_index = self->hasher (cur_item->key);
+            new_index %= new_limit;
+            cur_item->index = new_index;
+            cur_item->next = new_items [new_index];
+            new_items [new_index] = cur_item;
+            cur_item = next_item;
+        }
+    }
+    //  Destroy old hash table
+    freen (self->items);
+    self->items = new_items;
+    self->prime_index = new_prime_index;
+    return 0;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Insert item into hash table with specified key and item. Returns 0 on
+//  success. If the key is already present, returns -1 and leaves existing
+//  item unchanged. Sets the hash cursor to the item, if found. Dies with
+//  assertion if the process heap memory ran out. (Note: older code returned
+//  -1 in such cases; this is no longer so).
+
+int
+zhashx_insert (zhashx_t *self, const void *key, void *value)
+{
+    assert (self);
+    assert (key);
+
+    //  If we're exceeding the load factor of the hash table,
+    //  resize it according to the growth factor
+    size_t limit = primes [self->prime_index];
+    if (self->size >= limit * LOAD_FACTOR / 100) {
+        //  Create new hash table
+        uint new_prime_index = self->prime_index + GROWTH_FACTOR;
+        assert (s_zhashx_rehash (self, new_prime_index) == 0);
+        self->chain_limit += CHAIN_GROWS;
+    }
+    return s_item_insert (self, key, value)? 0: -1;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Local helper function
+//  Insert new item into hash table, returns item
+//  If item already existed, returns NULL
+//  Sets the hash cursor to the item, if found.
+
+static item_t *
+s_item_insert (zhashx_t *self, const void *key, void *value)
+{
+    //  Check that item does not already exist in hash table
+    //  Leaves self->cached_index with calculated hash item
+    item_t *item = s_item_lookup (self, key);
+    if (item == NULL) {
+        item = (item_t *) zmalloc (sizeof (item_t));
+        assert (item);
+
+        //  If necessary, take duplicate of item key
+        if (self->key_duplicator)
+            item->key = (self->key_duplicator)((void *) key);
+        else
+            item->key = key;
+
+        //  If necessary, take duplicate of item value
+        if (self->duplicator)
+            item->value = (self->duplicator)(value);
+        else
+            item->value = value;
+
+        item->index = self->cached_index;
+
+        //  Insert into start of bucket list
+        item->next = self->items [self->cached_index];
+        self->items [self->cached_index] = item;
+        self->size++;
+        self->cursor_item = item;
+        self->cursor_key = item->key;
+    }
+    else
+        item = NULL;            //  Signal duplicate insertion
+
+    return item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Local helper function
+//  Lookup item in hash table, returns item or NULL
+//  Dies with assertion if the process heap memory ran out (Note: older code
+//  returned NULL in such cases; this is no longer so).
+
+static item_t *
+s_item_lookup (zhashx_t *self, const void *key)
+{
+    //  Look in bucket list for item by key
+    size_t limit = primes [self->prime_index];
+    self->cached_index = self->hasher (key) % limit;
+    item_t *item = self->items [self->cached_index];
+    uint len = 0;
+    while (item) {
+        if ((self->key_comparator)(item->key, key) == 0)
+            break;
+        item = item->next;
+        ++len;
+    }
+    if (len > self->chain_limit) {
+        //  Create new hash table
+        uint new_prime_index = self->prime_index + GROWTH_FACTOR;
+        assert (s_zhashx_rehash (self, new_prime_index) == 0);
+        limit = primes [self->prime_index];
+        self->cached_index = self->hasher (key) % limit;
+        self->chain_limit += CHAIN_GROWS;
+    }
+    return item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Update or insert item into hash table with specified key and item. If the
+//  key is already present, destroys old item and inserts new one. If you set
+//  a container item destructor, this is called on the old value. If the key
+//  was not already present, inserts a new item. Sets the hash cursor to the
+//  new item.
+
+void
+zhashx_update (zhashx_t *self, const void *key, void *value)
+{
+    assert (self);
+    assert (key);
+
+    item_t *item = s_item_lookup (self, key);
+    if (item) {
+        if (self->destructor)
+            (self->destructor)(&item->value);
+        else
+        if (item->free_fn)
+            (item->free_fn)(item->value);
+
+        //  If necessary, take duplicate of item value
+        if (self->duplicator)
+            item->value = (self->duplicator)(value);
+        else
+            item->value = value;
+    }
+    else
+        zhashx_insert (self, key, value);
+}
+
+
+//  --------------------------------------------------------------------------
+//  Remove an item specified by key from the hash table. If there was no such
+//  item, this function does nothing.
+
+void
+zhashx_delete (zhashx_t *self, const void *key)
+{
+    assert (self);
+    assert (key);
+
+    item_t *item = s_item_lookup (self, key);
+    if (item)
+        s_item_destroy (self, item, true);
+}
+
+
+//  --------------------------------------------------------------------------
+//  Delete all items from the hash table. If the key destructor is
+//  set, calls it on every key. If the item destructor is set, calls
+//  it on every item.
+void
+zhashx_purge (zhashx_t *self)
+{
+    assert (self);
+    s_purge (self);
+
+    if (self->prime_index > INITIAL_PRIME) {
+        // Try to shrink hash table
+        size_t limit = primes [INITIAL_PRIME];
+        item_t **items = (item_t **) zmalloc (sizeof (item_t *) * limit);
+        assert (items);
+        freen (self->items);
+        self->prime_index = INITIAL_PRIME;
+        self->chain_limit = INITIAL_CHAIN;
+        self->items = items;
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Look for item in hash table and return its item, or NULL. Sets the hash
+//  cursor to the item, if found.
+
+void *
+zhashx_lookup (zhashx_t *self, const void *key)
+{
+    assert (self);
+    assert (key);
+
+    item_t *item = s_item_lookup (self, key);
+    if (item) {
+        self->cursor_item = item;
+        self->cursor_key = item->key;
+        return item->value;
+    }
+    else
+        return NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Reindexes an item from an old key to a new key. If there was no such
+//  item, does nothing. If the new key already exists, deletes old item.
+//  Sets the item cursor to the renamed item.
+
+int
+zhashx_rename (zhashx_t *self, const void *old_key, const void *new_key)
+{
+    item_t *old_item = s_item_lookup (self, old_key);
+    item_t *new_item = s_item_lookup (self, new_key);
+    if (old_item && !new_item) {
+        s_item_destroy (self, old_item, false);
+        if (self->key_destructor)
+            (self->key_destructor)((void **) &old_item->key);
+
+        if (self->key_duplicator)
+            old_item->key = (self->key_duplicator)(new_key);
+        else
+            old_item->key = new_key;
+
+        old_item->index = self->cached_index;
+        old_item->next = self->items [self->cached_index];
+        self->items [self->cached_index] = old_item;
+        self->size++;
+        self->cursor_item = old_item;
+        self->cursor_key = old_item->key;
+        return 0;
+    }
+    else
+        return -1;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a free function for the specified hash table item. When the item is
+//  destroyed, the free function, if any, is called on that item.
+//  Use this when hash items are dynamically allocated, to ensure that
+//  you don't have memory leaks. You can pass 'free' or NULL as a free_fn.
+//  Returns the item, or NULL if there is no such item.
+
+void *
+zhashx_freefn (zhashx_t *self, const void *key, zhashx_free_fn free_fn)
+{
+    assert (self);
+    assert (key);
+
+    item_t *item = s_item_lookup (self, key);
+    if (item) {
+        item->free_fn = free_fn;
+        return item->value;
+    }
+    else
+        return NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return size of hash table
+
+size_t
+zhashx_size (zhashx_t *self)
+{
+    assert (self);
+    return self->size;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return a zlistx_t containing the keys for the items in the
+//  table. Uses the key_duplicator to duplicate all keys and sets the
+//  key_destructor as destructor for the list.
+
+zlistx_t *
+zhashx_keys (zhashx_t *self)
+{
+    assert (self);
+    zlistx_t *keys = zlistx_new ();
+    if (!keys)
+        return NULL;
+    zlistx_set_destructor (keys, self->key_destructor);
+    zlistx_set_duplicator (keys, self->key_duplicator);
+
+    uint index;
+    size_t limit = primes [self->prime_index];
+    for (index = 0; index < limit; index++) {
+        item_t *item = self->items [index];
+        while (item) {
+            if (zlistx_add_end (keys, (void *) item->key) == NULL) {
+                zlistx_destroy (&keys);
+                return NULL;
+            }
+            item = item->next;
+        }
+    }
+    return keys;
+}
+
+//  Return a zlistx_t containing the items in the table. If there exists
+//  a duplicator, then it is used to duplicate all items, and if there
+//  is a destructor then it set as the destructor for the list.
+
+zlistx_t *
+zhashx_values (zhashx_t *self)
+{
+    assert (self);
+
+    zlistx_t *values = zlistx_new ();
+    if (!values)
+        return NULL;
+
+    zlistx_set_destructor (values, self->destructor);
+    zlistx_set_duplicator (values, self->duplicator);
+
+    uint index;
+    size_t limit = primes [self->prime_index];
+    for (index = 0; index < limit; index++) {
+        item_t *item = self->items [index];
+        while (item) {
+            if (zlistx_add_end (values, (void *) item->value) == NULL) {
+                zlistx_destroy (&values);
+                return NULL;
+            }
+            item = item->next;
+        }
+     }
+    return values;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Simple iterator; returns first item in hash table, in no given order,
+//  or NULL if the table is empty. This method is simpler to use than the
+//  foreach() method, which is deprecated. NOTE: do NOT modify the table
+//  while iterating.
+
+void *
+zhashx_first (zhashx_t *self)
+{
+    assert (self);
+    //  Point to before or at first item
+    self->cursor_index = 0;
+    self->cursor_item = self->items [self->cursor_index];
+    //  Now scan forwards to find it, leave cursor after item
+    return zhashx_next (self);
+}
+
+
+//  --------------------------------------------------------------------------
+//  Simple iterator; returns next item in hash table, in no given order,
+//  or NULL if the last item was already returned. Use this together with
+//  zhashx_first() to process all items in a hash table. If you need the
+//  items in sorted order, use zhashx_keys() and then zlistx_sort(). NOTE:
+//  do NOT modify the table while iterating.
+
+void *
+zhashx_next (zhashx_t *self)
+{
+    assert (self);
+    //  Scan forward from cursor until we find an item
+    size_t limit = primes [self->prime_index];
+    while (self->cursor_item == NULL) {
+        if (self->cursor_index < limit - 1)
+            self->cursor_index++;
+        else
+            return NULL;        //  At end of table
+
+        //  Get first item in next bucket
+        self->cursor_item = self->items [self->cursor_index];
+    }
+    //  We have an item, so return it, and bump past it
+    assert (self->cursor_item);
+    item_t *item = self->cursor_item;
+    self->cursor_key = item->key;
+    self->cursor_item = self->cursor_item->next;
+    return item->value;
+}
+
+
+//  --------------------------------------------------------------------------
+//  After a successful insert, update, or first/next method, returns the key
+//  for the item that was returned. You may not modify or deallocate
+//  the key, and it lasts as long as the item in the hash.
+//  After an unsuccessful first/next, returns NULL.
+
+const void *
+zhashx_cursor (zhashx_t *self)
+{
+    assert (self);
+    return self->cursor_key;
+}
+
+
+#ifdef CZMQ_BUILD_EXTRA
+//  --------------------------------------------------------------------------
+//  Add a comment to hash table before saving to disk. You can add as many
+//  comment lines as you like. These comment lines are discarded when loading
+//  the file. If you use a null format, all comments are deleted.
+//  FIXME: return 0 on success, -1 on error
+
+void
+zhashx_comment (zhashx_t *self, const char *format, ...)
+{
+    if (format) {
+        if (!self->comments) {
+            self->comments = zlistx_new ();
+            if (!self->comments)
+                return;
+            zlistx_set_destructor (self->comments, (zhashx_destructor_fn *) zstr_free);
+        }
+        va_list argptr;
+        va_start (argptr, format);
+        char *string = zsys_vprintf (format, argptr);
+        va_end (argptr);
+        if (string)
+            zlistx_add_end (self->comments, string);
+    }
+    else
+        zlistx_destroy (&self->comments);
+}
+
+
+//  --------------------------------------------------------------------------
+//  Save hash table to a text file in name=value format
+//  Hash values must be printable strings.
+//  Returns 0 if OK, else -1 if a file error occurred
+
+int
+zhashx_save (zhashx_t *self, const char *filename)
+{
+    assert (self);
+
+    FILE *handle = fopen (filename, "w");
+    if (!handle)
+        return -1;              //  Failed to create file
+
+    if (self->comments) {
+        char *comment = (char *) zlistx_first (self->comments);
+        while (comment) {
+            fprintf (handle, "#   %s\n", comment);
+            comment = (char *) zlistx_next (self->comments);
+        }
+        fprintf (handle, "\n");
+    }
+    uint index;
+    size_t limit = primes [self->prime_index];
+    for (index = 0; index < limit; index++) {
+        item_t *item = self->items [index];
+        while (item) {
+            fprintf (handle, "%s=%s\n", (char *) item->key, (char *) item->value);
+            item = item->next;
+        }
+    }
+    fclose (handle);
+    return 0;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Load hash table from a text file in name=value format; hash table must
+//  already exist. Hash values must printable strings.
+//  Returns 0 if OK, else -1 if a file was not readable.
+
+int
+zhashx_load (zhashx_t *self, const char *filename)
+{
+    assert (self);
+    zhashx_set_destructor (self, (zhashx_destructor_fn *) zstr_free);
+    zhashx_set_duplicator (self, (zhashx_duplicator_fn *) strdup);
+
+    //  Whether or not file exists, we'll track the filename and last
+    //  modification date (0 for unknown files), so that zhashx_refresh ()
+    //  will always work after zhashx_load (), to load a newly-created
+    //  file.
+
+    //  Take copy of filename in case self->filename is same string.
+    char *filename_copy = strdup (filename);
+    assert (filename_copy);
+    freen (self->filename);
+    self->filename = filename_copy;
+    self->modified = zsys_file_modified (self->filename);
+    FILE *handle = fopen (self->filename, "r");
+    if (handle) {
+        char *buffer = (char *) zmalloc (1024);
+        assert (buffer);
+        while (fgets (buffer, 1024, handle)) {
+            //  Skip lines starting with "#" or that do not look like
+            //  name=value data.
+            char *equals = strchr (buffer, '=');
+            if (buffer [0] == '#' || equals == buffer || !equals)
+                continue;
+
+            //  Buffer may end in newline, which we don't want
+            if (buffer [strlen (buffer) - 1] == '\n')
+                buffer [strlen (buffer) - 1] = 0;
+            *equals++ = 0;
+            zhashx_update (self, buffer, equals);
+        }
+        freen (buffer);
+        fclose (handle);
+    }
+    else
+        return -1; //  Failed to open file for reading
+
+    return 0;
+}
+
+
+//  --------------------------------------------------------------------------
+//  When a hash table was loaded from a file by zhashx_load, this method will
+//  reload the file if it has been modified since, and is "stable", i.e. not
+//  still changing. Returns 0 if OK, -1 if there was an error reloading the
+//  file.
+
+int
+zhashx_refresh (zhashx_t *self)
+{
+    assert (self);
+
+    if (self->filename) {
+        if (zsys_file_modified (self->filename) > self->modified
+        &&  zsys_file_stable (self->filename)) {
+            //  Empty the hash table; code is copied from zhashx_destroy
+            uint index;
+            size_t limit = primes [self->prime_index];
+            for (index = 0; index < limit; index++) {
+                //  Destroy all items in this hash bucket
+                item_t *cur_item = self->items [index];
+                while (cur_item) {
+                    item_t *next_item = cur_item->next;
+                    s_item_destroy (self, cur_item, true);
+                    cur_item = next_item;
+                }
+            }
+            zhashx_load (self, self->filename);
+        }
+    }
+    return 0;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Same as pack but uses a user-defined serializer function to convert items
+//  into longstr.
+//  Caller owns return value and must destroy it when done.
+
+zframe_t *
+zhashx_pack_own (zhashx_t *self, zhashx_serializer_fn serializer)
+{
+    assert (self);
+
+    //  First, calculate packed data size
+    size_t frame_size = 4;      //  Dictionary size, number-4
+    uint index;
+    uint vindex = 0;
+    size_t limit = primes [self->prime_index];
+    char **values = (char **) zmalloc (self->size * sizeof (char*));
+    for (index = 0; index < limit; index++) {
+        item_t *item = self->items [index];
+        while (item) {
+            //  We store key as short string
+            frame_size += 1 + strlen ((char *) item->key);
+            //  We store value as long string
+            if (serializer != NULL)
+                values [vindex] = serializer (item->value);
+            else
+                values [vindex] = (char *) item->value;
+
+            frame_size += 4 + strlen ((char *) values [vindex]);
+            item = item->next;
+            vindex++;
+        }
+    }
+    //  Now serialize items into the frame
+    zframe_t *frame = zframe_new (NULL, frame_size);
+    if (!frame) {
+        freen (values);
+        return NULL;
+    }
+
+    byte *needle = zframe_data (frame);
+    //  Store size as number-4
+    *(uint32_t *) needle = htonl ((u_long) self->size);
+    needle += 4;
+    vindex = 0;
+    for (index = 0; index < limit; index++) {
+        item_t *item = self->items [index];
+        while (item) {
+            //  Store key as string
+            size_t length = strlen ((char *) item->key);
+            *needle++ = (byte) length;
+            memcpy (needle, item->key, length);
+            needle += length;
+
+            //  Store value as longstr
+            length = strlen (values [vindex]);
+            uint32_t serialize = htonl ((u_long) length);
+            memcpy (needle, &serialize, 4);
+            needle += 4;
+            memcpy (needle, values [vindex], length);
+            needle += length;
+            item = item->next;
+
+            //  Destroy serialized value
+            if (serializer != NULL)
+                zstr_free (&values [vindex]);
+
+            vindex++;
+        }
+    }
+    freen (values);
+    return frame;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Serialize hash table to a binary frame that can be sent in a message.
+//  The packed format is compatible with the 'dictionary' type defined in
+//  http://rfc.zeromq.org/spec:35/FILEMQ, and implemented by zproto:
+//
+//     ; A list of name/value pairs
+//     dictionary      = dict-count *( dict-name dict-value )
+//     dict-count      = number-4
+//     dict-value      = longstr
+//     dict-name       = string
+//
+//     ; Strings are always length + text contents
+//     longstr         = number-4 *VCHAR
+//     string          = number-1 *VCHAR
+//
+//     ; Numbers are unsigned integers in network byte order
+//     number-1        = 1OCTET
+//     number-4        = 4OCTET
+//
+//  Comments are not included in the packed data. Item values MUST be
+//  strings.
+
+zframe_t *
+zhashx_pack (zhashx_t *self)
+{
+    return zhashx_pack_own (self, NULL);
+}
+
+
+//  --------------------------------------------------------------------------
+//  Same as unpack but uses a user-defined deserializer function to convert
+//  a longstr back into item format.
+
+zhashx_t *
+zhashx_unpack_own (zframe_t *frame, zhashx_deserializer_fn deserializer)
+{
+    zhashx_t *self = zhashx_new ();
+    if (!self)
+        return NULL;
+
+    //  Hash will free values in destructor
+    zhashx_set_destructor (self, (zhashx_destructor_fn *) zstr_free);
+
+    assert (frame);
+    if (zframe_size (frame) < 4)
+        return self;            //  Arguable...
+
+    byte *needle = zframe_data (frame);
+    byte *ceiling = needle + zframe_size (frame);
+    size_t nbr_items = ntohl (*(uint32_t *) needle);
+    needle += 4;
+    while (nbr_items && needle < ceiling) {
+        //  Get key as string
+        size_t key_size = *needle++;
+        if (needle + key_size <= ceiling) {
+            char key [256];
+            memcpy (key, needle, key_size);
+            key [key_size] = 0;
+            needle += key_size;
+
+            //  Get value as longstr
+            if (needle + 4 <= ceiling) {
+                size_t value_size = ntohl (*(uint32_t *) needle);
+                needle += 4;
+                //  Be wary of malformed frames
+                if (needle + value_size <= ceiling) {
+                    char *value = (char *) zmalloc (value_size + 1);
+                    assert (value);
+                    memcpy (value, needle, value_size);
+                    value [value_size] = 0;
+                    needle += value_size;
+
+                    //  Convert string to real value
+                    void *real_value;
+                    if (deserializer != NULL) {
+                        real_value = deserializer (value);
+                        zstr_free (&value);
+                    }
+                    else
+                        real_value = value;
+
+                    //  Hash takes ownership of real_value
+                    if (zhashx_insert (self, key, real_value)) {
+                        zhashx_destroy (&self);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    if (self)
+        zhashx_set_duplicator (self, (zhashx_duplicator_fn *) strdup);
+
+    return self;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Unpack binary frame into a new hash table. Packed data must follow format
+//  defined by zhashx_pack. Hash table is set to autofree. An empty frame
+//  unpacks to an empty hash table.
+
+zhashx_t *
+zhashx_unpack (zframe_t *frame)
+{
+    return zhashx_unpack_own (frame, NULL);
+}
+#endif // CZMQ_BUILD_EXTRA
+
+
+//  --------------------------------------------------------------------------
+//  Make a copy of the list; items are duplicated if you set a duplicator
+//  for the list, otherwise not. Copying a null reference returns a null
+//  reference. Note that this method's behavior changed slightly for CZMQ
+//  v3.x, as it does not set nor respect autofree. It does however let you
+//  duplicate any hash table safely. The old behavior is in zhashx_dup_v2.
+
+zhashx_t *
+zhashx_dup (zhashx_t *self)
+{
+    if (!self)
+        return NULL;
+
+    zhashx_t *copy = zhashx_new ();
+    if (copy) {
+        copy->destructor = self->destructor;
+        copy->duplicator = self->duplicator;
+        copy->key_duplicator = self->key_duplicator;
+        copy->key_destructor = self->key_destructor;
+        copy->key_comparator = self->key_comparator;
+        copy->hasher = self->hasher;
+        uint index;
+        size_t limit = primes [self->prime_index];
+        for (index = 0; index < limit; index++) {
+            item_t *item = self->items [index];
+            while (item) {
+                if (zhashx_insert (copy, item->key, item->value)) {
+                    zhashx_destroy (&copy);
+                    break;
+                }
+                item = item->next;
+            }
+        }
+    }
+    return copy;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a user-defined deallocator for hash items; by default items are not
+//  freed when the hash is destroyed.
+
+void
+zhashx_set_destructor (zhashx_t *self, zhashx_destructor_fn destructor)
+{
+    assert (self);
+    self->destructor = destructor;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a user-defined duplicator for hash items; by default items are not
+//  copied when the hash is duplicated.
+
+void
+zhashx_set_duplicator (zhashx_t *self, zhashx_duplicator_fn duplicator)
+{
+    assert (self);
+    self->duplicator = duplicator;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a user-defined deallocator for keys; by default keys are
+//  freed when the hash is destroyed by calling free().
+
+void
+zhashx_set_key_destructor (zhashx_t *self, zhashx_destructor_fn destructor)
+{
+    assert (self);
+    self->key_destructor = destructor;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a user-defined duplicator for keys; by default keys are
+//  duplicated by calling strdup().
+
+void
+zhashx_set_key_duplicator (zhashx_t *self, zhashx_duplicator_fn duplicator)
+{
+    assert (self);
+    self->key_duplicator = duplicator;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a user-defined comparator for keys; by default keys are
+//  compared using streq.
+
+void
+zhashx_set_key_comparator (zhashx_t *self, zhashx_comparator_fn comparator)
+{
+    assert (self);
+    assert (comparator != NULL);
+    self->key_comparator = comparator;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a user-defined hash function for keys; by default keys are
+//  hashed by a modified Bernstein hashing function.
+
+void
+zhashx_set_key_hasher (zhashx_t *self, zhashx_hash_fn hasher)
+{
+    assert (self);
+    self->hasher = hasher;
+}
+
+
+//  --------------------------------------------------------------------------
+//  DEPRECATED by zhashx_dup
+//  Make copy of hash table; if supplied table is null, returns null.
+//  Does not copy items themselves. Rebuilds new table so may be slow on
+//  very large tables. NOTE: only works with item values that are strings
+//  since there's no other way to know how to duplicate the item value.
+
+zhashx_t *
+zhashx_dup_v2 (zhashx_t *self)
+{
+    if (!self)
+        return NULL;
+
+    zhashx_t *copy = zhashx_new ();
+    if (copy) {
+        zhashx_set_destructor (copy, (zhashx_destructor_fn *) zstr_free);
+        zhashx_set_duplicator (copy, (zhashx_duplicator_fn *) strdup);
+        uint index;
+        size_t limit = primes [self->prime_index];
+        for (index = 0; index < limit; index++) {
+            item_t *item = self->items [index];
+            while (item) {
+                if (zhashx_insert (copy, item->key, item->value)) {
+                    zhashx_destroy (&copy);
+                    break;
+                }
+                item = item->next;
+            }
+        }
+    }
+    return copy;
+}
+
+
+#ifdef CZMQ_BUILD_EXTRA
+//  --------------------------------------------------------------------------
+//  Runs selftest of class
+//
+
+#ifdef CZMQ_BUILD_DRAFT_API
+static char *
+s_test_serialize_int (const void *item)
+{
+    int *int_item = (int *) item;
+    char *str_item = (char *) zmalloc (sizeof (char) * 10);
+    sprintf (str_item, "%d", *int_item);
+    return str_item;
+}
+
+static void *
+s_test_deserialze_int (const char *str_item)
+{
+    int *int_item = (int *) zmalloc (sizeof (int));
+    sscanf (str_item, "%d", int_item);
+    return int_item;
+}
+
+static void
+s_test_destroy_int (void **item)
+{
+    int *int_item = (int *) *item;
+    freen (int_item);
+}
+#endif // CZMQ_BUILD_DRAFT_API
+
+void
+zhashx_test (bool verbose)
+{
+    printf (" * zhashx: ");
+
+    //  @selftest
+    zhashx_t *hash = zhashx_new ();
+    assert (hash);
+    assert (zhashx_size (hash) == 0);
+    assert (zhashx_first (hash) == NULL);
+    assert (zhashx_cursor (hash) == NULL);
+
+    //  Insert some items
+    int rc;
+    rc = zhashx_insert (hash, "DEADBEEF", "dead beef");
+    char *item = (char *) zhashx_first (hash);
+    assert (streq ((char *) zhashx_cursor (hash), "DEADBEEF"));
+    assert (streq (item, "dead beef"));
+    assert (rc == 0);
+    rc = zhashx_insert (hash, "ABADCAFE", "a bad cafe");
+    assert (rc == 0);
+    rc = zhashx_insert (hash, "C0DEDBAD", "coded bad");
+    assert (rc == 0);
+    rc = zhashx_insert (hash, "DEADF00D", "dead food");
+    assert (rc == 0);
+    assert (zhashx_size (hash) == 4);
+
+    //  Look for existing items
+    item = (char *) zhashx_lookup (hash, "DEADBEEF");
+    assert (streq (item, "dead beef"));
+    item = (char *) zhashx_lookup (hash, "ABADCAFE");
+    assert (streq (item, "a bad cafe"));
+    item = (char *) zhashx_lookup (hash, "C0DEDBAD");
+    assert (streq (item, "coded bad"));
+    item = (char *) zhashx_lookup (hash, "DEADF00D");
+    assert (streq (item, "dead food"));
+
+    //  Look for non-existent items
+    item = (char *) zhashx_lookup (hash, "foo");
+    assert (item == NULL);
+
+    //  Try to insert duplicate items
+    rc = zhashx_insert (hash, "DEADBEEF", "foo");
+    assert (rc == -1);
+    item = (char *) zhashx_lookup (hash, "DEADBEEF");
+    assert (streq (item, "dead beef"));
+
+    //  Some rename tests
+
+    //  Valid rename, key is now LIVEBEEF
+    rc = zhashx_rename (hash, "DEADBEEF", "LIVEBEEF");
+    assert (rc == 0);
+    item = (char *) zhashx_lookup (hash, "LIVEBEEF");
+    assert (streq (item, "dead beef"));
+
+    //  Trying to rename an unknown item to a non-existent key
+    rc = zhashx_rename (hash, "WHATBEEF", "NONESUCH");
+    assert (rc == -1);
+
+    //  Trying to rename an unknown item to an existing key
+    rc = zhashx_rename (hash, "WHATBEEF", "LIVEBEEF");
+    assert (rc == -1);
+    item = (char *) zhashx_lookup (hash, "LIVEBEEF");
+    assert (streq (item, "dead beef"));
+
+    //  Trying to rename an existing item to another existing item
+    rc = zhashx_rename (hash, "LIVEBEEF", "ABADCAFE");
+    assert (rc == -1);
+    item = (char *) zhashx_lookup (hash, "LIVEBEEF");
+    assert (streq (item, "dead beef"));
+    item = (char *) zhashx_lookup (hash, "ABADCAFE");
+    assert (streq (item, "a bad cafe"));
+
+    //  Test keys method
+    zlistx_t *keys = zhashx_keys (hash);
+    assert (zlistx_size (keys) == 4);
+    zlistx_destroy (&keys);
+
+    zlistx_t *values = zhashx_values (hash);
+    assert (zlistx_size (values) == 4);
+    zlistx_destroy (&values);
+
+    //  Test dup method
+    zhashx_t *copy = zhashx_dup (hash);
+    assert (zhashx_size (copy) == 4);
+    item = (char *) zhashx_lookup (copy, "LIVEBEEF");
+    assert (item);
+    assert (streq (item, "dead beef"));
+    zhashx_destroy (&copy);
+
+    //  Test pack/unpack methods
+    zframe_t *frame = zhashx_pack (hash);
+    copy = zhashx_unpack (frame);
+    zframe_destroy (&frame);
+    assert (zhashx_size (copy) == 4);
+    item = (char *) zhashx_lookup (copy, "LIVEBEEF");
+    assert (item);
+    assert (streq (item, "dead beef"));
+    zhashx_destroy (&copy);
+
+#ifdef CZMQ_BUILD_DRAFT_API
+    //  Test own pack/unpack methods
+    zhashx_t *own_hash = zhashx_new ();
+    zhashx_set_destructor (own_hash, s_test_destroy_int);
+    assert (own_hash);
+    int *val1 = (int *) zmalloc (sizeof (int));
+    int *val2 = (int *) zmalloc (sizeof (int));
+    *val1 = 25;
+    *val2 = 100;
+    zhashx_insert (own_hash, "val1", val1);
+    zhashx_insert (own_hash, "val2", val2);
+    frame = zhashx_pack_own (own_hash, s_test_serialize_int);
+    copy = zhashx_unpack_own (frame, s_test_deserialze_int);
+    zhashx_set_destructor (copy, s_test_destroy_int);
+    zframe_destroy (&frame);
+    assert (zhashx_size (copy) == 2);
+    assert (*((int *) zhashx_lookup (copy, "val1")) == 25);
+    assert (*((int *) zhashx_lookup (copy, "val2")) == 100);
+    zhashx_destroy (&copy);
+    zhashx_destroy (&own_hash);
+#endif // CZMQ_BUILD_DRAFT_API
+
+    //  Test save and load
+    zhashx_comment (hash, "This is a test file");
+    zhashx_comment (hash, "Created by %s", "czmq_selftest");
+    zhashx_save (hash, ".cache");
+    copy = zhashx_new ();
+    assert (copy);
+    zhashx_load (copy, ".cache");
+    item = (char *) zhashx_lookup (copy, "LIVEBEEF");
+    assert (item);
+    assert (streq (item, "dead beef"));
+    zhashx_destroy (&copy);
+    zsys_file_delete (".cache");
+
+    //  Delete a item
+    zhashx_delete (hash, "LIVEBEEF");
+    item = (char *) zhashx_lookup (hash, "LIVEBEEF");
+    assert (item == NULL);
+    assert (zhashx_size (hash) == 3);
+
+    //  Check that the queue is robust against random usage
+    struct {
+        char name [100];
+        bool exists;
+    } testset [200];
+    memset (testset, 0, sizeof (testset));
+    int testmax = 200, testnbr, iteration;
+
+    srandom ((unsigned) time (NULL));
+    for (iteration = 0; iteration < 25000; iteration++) {
+        testnbr = randof (testmax);
+        assert (testnbr != testmax);
+        assert (testnbr < testmax);
+        if (testset [testnbr].exists) {
+            item = (char *) zhashx_lookup (hash, testset [testnbr].name);
+            assert (item);
+            zhashx_delete (hash, testset [testnbr].name);
+            testset [testnbr].exists = false;
+        }
+        else {
+            sprintf (testset [testnbr].name, "%x-%x", rand (), rand ());
+            if (zhashx_insert (hash, testset [testnbr].name, "") == 0)
+                testset [testnbr].exists = true;
+        }
+    }
+    //  Test 10K lookups
+    for (iteration = 0; iteration < 10000; iteration++)
+        item = (char *) zhashx_lookup (hash, "DEADBEEFABADCAFE");
+
+    //  Destructor should be safe to call twice
+    zhashx_destroy (&hash);
+    zhashx_destroy (&hash);
+    assert (hash == NULL);
+
+    //  Test randof() limits - should be within (0..testmax)
+    //  and randomness distribution - should not have (many) zero-counts
+    //  If there are - maybe the ZSYS_RANDOF_MAX is too big for this platform
+    //  Note: This test can take a while on systems with weak floating point HW
+    testmax = 999;
+    size_t rndcnt[999];
+    assert ((sizeof (rndcnt)/sizeof(rndcnt[0])) == testmax);
+    memset (rndcnt, 0, sizeof (rndcnt));
+    for (iteration = 0; iteration < 10000000; iteration++) {
+        testnbr = randof (testmax);
+        assert (testnbr != testmax);
+        assert (testnbr < testmax);
+        assert (testnbr >= 0);
+        rndcnt[testnbr]++;
+    }
+    int rndmisses = 0;
+    for (iteration = 0; iteration < testmax; iteration++) {
+        if (rndcnt[iteration] == 0) {
+            zsys_warning("zhashx_test() : random distribution fault : got 0 hits for %d/%d",
+                iteration, testmax);
+            rndmisses++;
+        }
+    }
+    //  Too many misses are suspicious... we can lose half the entries
+    //  for each bit not used in the assumed ZSYS_RANDOF_MAX...
+    assert ( (rndmisses < (testmax / 3 )) );
+
+    //  Test destructor; automatically copies and frees string values
+    hash = zhashx_new ();
+    assert (hash);
+    zhashx_set_destructor (hash, (zhashx_destructor_fn *) zstr_free);
+    zhashx_set_duplicator (hash, (zhashx_duplicator_fn *) strdup);
+    char value [255];
+    strcpy (value, "This is a string");
+    rc = zhashx_insert (hash, "key1", value);
+    assert (rc == 0);
+    strcpy (value, "Ring a ding ding");
+    rc = zhashx_insert (hash, "key2", value);
+    assert (rc == 0);
+    assert (streq ((char *) zhashx_lookup (hash, "key1"), "This is a string"));
+    assert (streq ((char *) zhashx_lookup (hash, "key2"), "Ring a ding ding"));
+    zhashx_destroy (&hash);
+
+    //  Test purger and shrinker: no data should end up unreferenced in valgrind
+    hash = zhashx_new ();
+    assert (hash);
+    zhashx_set_destructor (hash, (zhashx_destructor_fn *) zstr_free);
+    zhashx_set_duplicator (hash, (zhashx_duplicator_fn *) strdup);
+    char valuep [255];
+    strcpy (valuep, "This is a string");
+    rc = zhashx_insert (hash, "key1", valuep);
+    assert (rc == 0);
+    strcpy (valuep, "Ring a ding ding");
+    rc = zhashx_insert (hash, "key2", valuep);
+    assert (rc == 0);
+    strcpy (valuep, "Cartahena delenda est");
+    rc = zhashx_insert (hash, "key3", valuep);
+    assert (rc == 0);
+    strcpy (valuep, "So say we all!");
+    rc = zhashx_insert (hash, "key4", valuep);
+    assert (rc == 0);
+    assert (streq ((char *) zhashx_lookup (hash, "key1"), "This is a string"));
+    assert (streq ((char *) zhashx_lookup (hash, "key2"), "Ring a ding ding"));
+    assert (streq ((char *) zhashx_lookup (hash, "key3"), "Cartahena delenda est"));
+    assert (streq ((char *) zhashx_lookup (hash, "key4"), "So say we all!"));
+    zhashx_purge (hash);
+    zhashx_destroy (&hash);
+
+#if defined (__WINDOWS__)
+    zsys_shutdown();
+#endif
+    //  @end
+
+    printf ("OK\n");
+}
+#endif // CZMQ_BUILD_EXTRA
diff --git a/src/common/libczmqcontainers/zhashx.h b/src/common/libczmqcontainers/zhashx.h
new file mode 100644
index 000000000000..d9cbe171296b
--- /dev/null
+++ b/src/common/libczmqcontainers/zhashx.h
@@ -0,0 +1,293 @@
+/*  =========================================================================
+    zhashx - extended generic type-free hash container
+
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+/* This is a copy of the czmq zhashx library.
+ *
+ * Due to czmq issue #2173
+ * (https://github.com/zeromq/czmq/issues/2173) a localized copy was
+ * made instead of waiting for OS distros to pick up the bug fix.
+ *
+ * This file is a verbatim copy with only minor adjustments to header
+ * guards and included headers.
+ */
+
+#ifndef __FLUXZHASHX_H_INCLUDED__
+#define __FLUXZHASHX_H_INCLUDED__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//  @warning THE FOLLOWING @INTERFACE BLOCK IS AUTO-GENERATED BY ZPROJECT
+//  @warning Please edit the model at "api/zhashx.api" to make changes.
+//  @interface
+//  This is a stable class, and may not change except for emergencies. It
+//  is provided in stable builds.
+//  This class has draft methods, which may change over time. They are not
+//  in stable releases, by default. Use --enable-drafts to enable.
+// Destroy an item
+typedef void (zhashx_destructor_fn) (
+    void **item);
+
+// Duplicate an item
+typedef void * (zhashx_duplicator_fn) (
+    const void *item);
+
+// Compare two items, for sorting
+typedef int (zhashx_comparator_fn) (
+    const void *item1, const void *item2);
+
+// Destroy an item.
+typedef void (zhashx_free_fn) (
+    void *data);
+
+// Hash function for keys.
+typedef size_t (zhashx_hash_fn) (
+    const void *key);
+
+// Serializes an item to a longstr.
+// The caller takes ownership of the newly created object.
+typedef char * (zhashx_serializer_fn) (
+    const void *item);
+
+// Deserializes a longstr into an item.
+// The caller takes ownership of the newly created object.
+typedef void * (zhashx_deserializer_fn) (
+    const char *item_str);
+
+//  Create a new, empty hash container
+CZMQ_EXPORT zhashx_t *
+    zhashx_new (void);
+
+#ifdef CZMQ_BUILD_EXTRA
+//  Unpack binary frame into a new hash table. Packed data must follow format
+//  defined by zhashx_pack. Hash table is set to autofree. An empty frame
+//  unpacks to an empty hash table.
+CZMQ_EXPORT zhashx_t *
+    zhashx_unpack (zframe_t *frame);
+#endif // CZMQ_BUILD_EXTRA
+
+//  Destroy a hash container and all items in it
+CZMQ_EXPORT void
+    zhashx_destroy (zhashx_t **self_p);
+
+//  Insert item into hash table with specified key and item.
+//  If key is already present returns -1 and leaves existing item unchanged
+//  Returns 0 on success.
+CZMQ_EXPORT int
+    zhashx_insert (zhashx_t *self, const void *key, void *item);
+
+//  Update or insert item into hash table with specified key and item. If the
+//  key is already present, destroys old item and inserts new one. If you set
+//  a container item destructor, this is called on the old value. If the key
+//  was not already present, inserts a new item. Sets the hash cursor to the
+//  new item.
+CZMQ_EXPORT void
+    zhashx_update (zhashx_t *self, const void *key, void *item);
+
+//  Remove an item specified by key from the hash table. If there was no such
+//  item, this function does nothing.
+CZMQ_EXPORT void
+    zhashx_delete (zhashx_t *self, const void *key);
+
+//  Delete all items from the hash table. If the key destructor is
+//  set, calls it on every key. If the item destructor is set, calls
+//  it on every item.
+CZMQ_EXPORT void
+    zhashx_purge (zhashx_t *self);
+
+//  Return the item at the specified key, or null
+CZMQ_EXPORT void *
+    zhashx_lookup (zhashx_t *self, const void *key);
+
+//  Reindexes an item from an old key to a new key. If there was no such
+//  item, does nothing. Returns 0 if successful, else -1.
+CZMQ_EXPORT int
+    zhashx_rename (zhashx_t *self, const void *old_key, const void *new_key);
+
+//  Set a free function for the specified hash table item. When the item is
+//  destroyed, the free function, if any, is called on that item.
+//  Use this when hash items are dynamically allocated, to ensure that
+//  you don't have memory leaks. You can pass 'free' or NULL as a free_fn.
+//  Returns the item, or NULL if there is no such item.
+CZMQ_EXPORT void *
+    zhashx_freefn (zhashx_t *self, const void *key, zhashx_free_fn free_fn);
+
+//  Return the number of keys/items in the hash table
+CZMQ_EXPORT size_t
+    zhashx_size (zhashx_t *self);
+
+//  Return a zlistx_t containing the keys for the items in the
+//  table. Uses the key_duplicator to duplicate all keys and sets the
+//  key_destructor as destructor for the list.
+//  Caller owns return value and must destroy it when done.
+CZMQ_EXPORT zlistx_t *
+    zhashx_keys (zhashx_t *self);
+
+//  Return a zlistx_t containing the values for the items in the
+//  table. Uses the duplicator to duplicate all items and sets the
+//  destructor as destructor for the list.
+//  Caller owns return value and must destroy it when done.
+CZMQ_EXPORT zlistx_t *
+    zhashx_values (zhashx_t *self);
+
+//  Simple iterator; returns first item in hash table, in no given order,
+//  or NULL if the table is empty. This method is simpler to use than the
+//  foreach() method, which is deprecated. To access the key for this item
+//  use zhashx_cursor(). NOTE: do NOT modify the table while iterating.
+CZMQ_EXPORT void *
+    zhashx_first (zhashx_t *self);
+
+//  Simple iterator; returns next item in hash table, in no given order,
+//  or NULL if the last item was already returned. Use this together with
+//  zhashx_first() to process all items in a hash table. If you need the
+//  items in sorted order, use zhashx_keys() and then zlistx_sort(). To
+//  access the key for this item use zhashx_cursor(). NOTE: do NOT modify
+//  the table while iterating.
+CZMQ_EXPORT void *
+    zhashx_next (zhashx_t *self);
+
+//  After a successful first/next method, returns the key for the item that
+//  was returned. This is a constant string that you may not modify or
+//  deallocate, and which lasts as long as the item in the hash. After an
+//  unsuccessful first/next, returns NULL.
+CZMQ_EXPORT const void *
+    zhashx_cursor (zhashx_t *self);
+
+#ifdef CZMQ_BUILD_EXTRA
+//  Add a comment to hash table before saving to disk. You can add as many
+//  comment lines as you like. These comment lines are discarded when loading
+//  the file. If you use a null format, all comments are deleted.
+CZMQ_EXPORT void
+    zhashx_comment (zhashx_t *self, const char *format, ...) CHECK_PRINTF (2);
+
+//  Save hash table to a text file in name=value format. Hash values must be
+//  printable strings; keys may not contain '=' character. Returns 0 if OK,
+//  else -1 if a file error occurred.
+CZMQ_EXPORT int
+    zhashx_save (zhashx_t *self, const char *filename);
+
+//  Load hash table from a text file in name=value format; hash table must
+//  already exist. Hash values must printable strings; keys may not contain
+//  '=' character. Returns 0 if OK, else -1 if a file was not readable.
+CZMQ_EXPORT int
+    zhashx_load (zhashx_t *self, const char *filename);
+
+//  When a hash table was loaded from a file by zhashx_load, this method will
+//  reload the file if it has been modified since, and is "stable", i.e. not
+//  still changing. Returns 0 if OK, -1 if there was an error reloading the
+//  file.
+CZMQ_EXPORT int
+    zhashx_refresh (zhashx_t *self);
+
+//  Serialize hash table to a binary frame that can be sent in a message.
+//  The packed format is compatible with the 'dictionary' type defined in
+//  http://rfc.zeromq.org/spec:35/FILEMQ, and implemented by zproto:
+//
+//     ; A list of name/value pairs
+//     dictionary      = dict-count *( dict-name dict-value )
+//     dict-count      = number-4
+//     dict-value      = longstr
+//     dict-name       = string
+//
+//     ; Strings are always length + text contents
+//     longstr         = number-4 *VCHAR
+//     string          = number-1 *VCHAR
+//
+//     ; Numbers are unsigned integers in network byte order
+//     number-1        = 1OCTET
+//     number-4        = 4OCTET
+//
+//  Comments are not included in the packed data. Item values MUST be
+//  strings.
+//  Caller owns return value and must destroy it when done.
+CZMQ_EXPORT zframe_t *
+    zhashx_pack (zhashx_t *self);
+#endif // CZMQ_BUILD_EXTRA
+
+//  Make a copy of the list; items are duplicated if you set a duplicator
+//  for the list, otherwise not. Copying a null reference returns a null
+//  reference. Note that this method's behavior changed slightly for CZMQ
+//  v3.x, as it does not set nor respect autofree. It does however let you
+//  duplicate any hash table safely. The old behavior is in zhashx_dup_v2.
+//  Caller owns return value and must destroy it when done.
+CZMQ_EXPORT zhashx_t *
+    zhashx_dup (zhashx_t *self);
+
+//  Set a user-defined deallocator for hash items; by default items are not
+//  freed when the hash is destroyed.
+CZMQ_EXPORT void
+    zhashx_set_destructor (zhashx_t *self, zhashx_destructor_fn destructor);
+
+//  Set a user-defined duplicator for hash items; by default items are not
+//  copied when the hash is duplicated.
+CZMQ_EXPORT void
+    zhashx_set_duplicator (zhashx_t *self, zhashx_duplicator_fn duplicator);
+
+//  Set a user-defined deallocator for keys; by default keys are freed
+//  when the hash is destroyed using free().
+CZMQ_EXPORT void
+    zhashx_set_key_destructor (zhashx_t *self, zhashx_destructor_fn destructor);
+
+//  Set a user-defined duplicator for keys; by default keys are duplicated
+//  using strdup.
+CZMQ_EXPORT void
+    zhashx_set_key_duplicator (zhashx_t *self, zhashx_duplicator_fn duplicator);
+
+//  Set a user-defined comparator for keys; by default keys are
+//  compared using strcmp.
+//  The callback function should return zero (0) on matching
+//  items.
+CZMQ_EXPORT void
+    zhashx_set_key_comparator (zhashx_t *self, zhashx_comparator_fn comparator);
+
+//  Set a user-defined hash function for keys; by default keys are
+//  hashed by a modified Bernstein hashing function.
+CZMQ_EXPORT void
+    zhashx_set_key_hasher (zhashx_t *self, zhashx_hash_fn hasher);
+
+//  Make copy of hash table; if supplied table is null, returns null.
+//  Does not copy items themselves. Rebuilds new table so may be slow on
+//  very large tables. NOTE: only works with item values that are strings
+//  since there's no other way to know how to duplicate the item value.
+CZMQ_EXPORT zhashx_t *
+    zhashx_dup_v2 (zhashx_t *self);
+
+#ifdef CZMQ_BUILD_EXTRA
+//  Self test of this class.
+CZMQ_EXPORT void
+    zhashx_test (bool verbose);
+
+#ifdef CZMQ_BUILD_DRAFT_API
+//  *** Draft method, for development use, may change without warning ***
+//  Same as unpack but uses a user-defined deserializer function to convert
+//  a longstr back into item format.
+CZMQ_EXPORT zhashx_t *
+    zhashx_unpack_own (zframe_t *frame, zhashx_deserializer_fn deserializer);
+
+//  *** Draft method, for development use, may change without warning ***
+//  Same as pack but uses a user-defined serializer function to convert items
+//  into longstr.
+//  Caller owns return value and must destroy it when done.
+CZMQ_EXPORT zframe_t *
+    zhashx_pack_own (zhashx_t *self, zhashx_serializer_fn serializer);
+
+#endif // CZMQ_BUILD_DRAFT_API
+#endif // CZMQ_BUILD_EXTRA
+//  @end
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/common/libczmqcontainers/zlist.c b/src/common/libczmqcontainers/zlist.c
new file mode 100644
index 000000000000..9552c223a0d0
--- /dev/null
+++ b/src/common/libczmqcontainers/zlist.c
@@ -0,0 +1,637 @@
+/*  =========================================================================
+    zlist - simple generic list container
+
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+/*
+@header
+    Provides a generic container implementing a fast singly-linked list. You
+    can use this to construct multi-dimensional lists, and other structures
+    together with other generic containers like zhash. This is a simple
+    class. For demanding applications we recommend using zlistx.
+@discuss
+    To iterate through a list, use zlist_first to get the first item, then
+    loop while not null, and do zlist_next at the end of each iteration.
+@end
+*/
+
+#include "czmq_containers.h"
+#include "czmq_internal.h"
+
+//  List node, used internally only
+
+typedef struct _node_t {
+    struct _node_t *next;
+    void *item;
+    zlist_free_fn *free_fn;
+} node_t;
+
+
+//  ---------------------------------------------------------------------
+//  Structure of our class
+
+struct _zlist_t {
+    node_t *head;                 //  First item in list, if any
+    node_t *tail;                 //  Last item in list, if any
+    node_t *cursor;               //  Current cursors for iteration
+    size_t size;                  //  Number of items in list
+    bool autofree;                //  If true, free items in destructor
+    zlist_compare_fn *compare_fn; //  Function to compare two list item for
+                                  //  less than, equals or greater than
+};
+
+
+//  --------------------------------------------------------------------------
+//  List constructor
+
+zlist_t *
+zlist_new (void)
+{
+    zlist_t *self = (zlist_t *) zmalloc (sizeof (zlist_t));
+    assert (self);
+    return self;
+}
+
+
+//  --------------------------------------------------------------------------
+//  List destructor
+
+void
+zlist_destroy (zlist_t **self_p)
+{
+    assert (self_p);
+    if (*self_p) {
+        zlist_t *self = *self_p;
+        zlist_purge (self);
+        freen (self);
+        *self_p = NULL;
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the item at the head of list. If the list is empty, returns NULL.
+//  Leaves cursor pointing at the head item, or NULL if the list is empty.
+
+void *
+zlist_first (zlist_t *self)
+{
+    assert (self);
+    self->cursor = self->head;
+    if (self->cursor)
+        return self->cursor->item;
+    else
+        return NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the next item. If the list is empty, returns NULL. To move to
+//  the start of the list call zlist_first (). Advances the cursor.
+
+void *
+zlist_next (zlist_t *self)
+{
+    assert (self);
+    if (self->cursor)
+        self->cursor = self->cursor->next;
+    else
+        self->cursor = self->head;
+    if (self->cursor)
+        return self->cursor->item;
+    else
+        return NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the item at the tail of list. If the list is empty, returns NULL.
+//  Leaves cursor pointing at the tail item, or NULL if the list is empty.
+
+void *
+zlist_last (zlist_t *self)
+{
+    assert (self);
+    self->cursor = self->tail;
+    if (self->cursor)
+        return self->cursor->item;
+    else
+        return NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the item at the head of list. If the list is empty, returns NULL.
+//  Leaves cursor as-is.
+
+void *
+zlist_head (zlist_t *self)
+{
+    assert (self);
+    return self->head? self->head->item: NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the item at the tail of list. If the list is empty, returns NULL.
+//  Leaves cursor as-is.
+
+void *
+zlist_tail (zlist_t *self)
+{
+    assert (self);
+    return self->tail? self->tail->item: NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return current item in the list. If the list is empty, or the cursor
+//  passed the end of the list, returns NULL. Does not change the cursor.
+
+void *
+zlist_item (zlist_t *self)
+{
+    assert (self);
+    if (self->cursor)
+        return self->cursor->item;
+    else
+        return NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Append an item to the end of the list, return 0 if OK or -1 if this
+//  failed for some reason.
+
+int
+zlist_append (zlist_t *self, void *item)
+{
+    if (!item)
+        return -1;
+
+    node_t *node = (node_t *) zmalloc (sizeof (node_t));
+    assert (node);
+
+    //  If necessary, take duplicate of (string) item
+    if (self->autofree) {
+        item = strdup ((char *) item);
+        assert (item);
+    }
+    node->item = item;
+    if (self->tail)
+        self->tail->next = node;
+    else
+        self->head = node;
+
+    self->tail = node;
+    node->next = NULL;
+
+    self->size++;
+    self->cursor = NULL;
+    return 0;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Push an item to the start of the list, return 0 if OK or -1 if this
+//  failed for some reason.
+
+int
+zlist_push (zlist_t *self, void *item)
+{
+    if (!item)
+        return -1;
+
+    node_t *node = (node_t *) zmalloc (sizeof (node_t));
+    assert (node);
+
+    //  If necessary, take duplicate of (string) item
+    if (self->autofree) {
+        item = strdup ((char *) item);
+        assert (item);
+    }
+    node->item = item;
+    node->next = self->head;
+    self->head = node;
+    if (self->tail == NULL)
+        self->tail = node;
+
+    self->size++;
+    self->cursor = NULL;
+    return 0;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Remove item from the beginning of the list, returns NULL if none
+
+void *
+zlist_pop (zlist_t *self)
+{
+    node_t *node = self->head;
+    void *item = NULL;
+    if (node) {
+        item = node->item;
+        self->head = node->next;
+        if (self->tail == node)
+            self->tail = NULL;
+        freen (node);
+        self->size--;
+    }
+    self->cursor = NULL;
+    return item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Checks if an item already is present. Uses compare method to determine if
+//  items are equal. If the compare method is NULL the check will only compare
+//  pointers. Returns true if item is present else false.
+
+bool
+zlist_exists (zlist_t *self, void *item)
+{
+    assert (self);
+    assert (item);
+    node_t *node = self->head;
+
+    while (node) {
+        if (self->compare_fn) {
+            if ((*self->compare_fn)(node->item, item) == 0)
+                return true;
+        }
+        else
+        if (node->item == item)
+            return true;
+
+        node = node->next;
+    }
+    return false;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Remove the item from the list, if present. Safe to call on items that
+//  are not in the list.
+
+void
+zlist_remove (zlist_t *self, void *item)
+{
+    node_t *node, *prev = NULL;
+
+    //  First off, we need to find the list node
+    for (node = self->head; node != NULL; node = node->next) {
+        if (self->compare_fn) {
+            if ((*self->compare_fn)(node->item, item) == 0)
+               break;
+        }
+        else
+        if (node->item == item)
+            break;
+
+        prev = node;
+    }
+    if (node) {
+        if (prev)
+            prev->next = node->next;
+        else
+            self->head = node->next;
+
+        if (node->next == NULL)
+            self->tail = prev;
+        if (self->cursor == node)
+            self->cursor = prev;
+
+        if (self->autofree)
+            freen (node->item);
+        else
+        if (node->free_fn)
+            (node->free_fn)(node->item);
+
+        freen (node);
+        self->size--;
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Make a copy of list. If the list has autofree set, the copied list will
+//  duplicate all items, which must be strings. Otherwise, the list will hold
+//  pointers back to the items in the original list. If list is null, returns
+//  NULL.
+
+zlist_t *
+zlist_dup (zlist_t *self)
+{
+    if (!self)
+        return NULL;
+
+    zlist_t *copy = zlist_new ();
+    assert (copy);
+
+    if (self->autofree)
+        zlist_autofree(copy);
+
+    copy->compare_fn = self->compare_fn;
+
+    node_t *node;
+    for (node = self->head; node; node = node->next) {
+        if (zlist_append (copy, node->item) == -1) {
+            zlist_destroy (&copy);
+            break;
+        }
+    }
+    return copy;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Purge all items from list
+
+void
+zlist_purge (zlist_t *self)
+{
+    assert (self);
+    node_t *node = self->head;
+    while (node) {
+        node_t *next = node->next;
+        if (self->autofree)
+            freen (node->item);
+        else
+        if (node->free_fn)
+            (node->free_fn)(node->item);
+
+        freen (node);
+        node = next;
+    }
+    self->head = NULL;
+    self->tail = NULL;
+    self->cursor = NULL;
+    self->size = 0;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the number of items in the list
+
+size_t
+zlist_size (zlist_t *self)
+{
+    return self->size;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Sort the list. If the compare function is null, sorts the list by
+//  ascending key value using a straight ASCII comparison. If you specify
+//  a compare function, this decides how items are sorted. The sort is not
+//  stable, so may reorder items with the same keys. The algorithm used is
+//  combsort, a compromise between performance and simplicity.
+
+void
+zlist_sort (zlist_t *self, zlist_compare_fn compare_fn)
+{
+    zlist_compare_fn *compare = compare_fn;
+    if (!compare) {
+        compare = self->compare_fn;
+        if (!compare)
+            compare = (zlist_compare_fn *) strcmp;
+    }
+    //  Uses a comb sort, which is simple and reasonably fast.
+    //  See http://en.wikipedia.org/wiki/Comb_sort
+    size_t gap = self->size;
+    bool swapped = false;
+    while (gap > 1 || swapped) {
+        if (gap > 1)
+            gap = (size_t) ((double) gap / 1.3);
+        node_t *base = self->head;
+        node_t *test = self->head;
+        size_t jump = gap;
+        while (jump--)
+            test = test->next;
+
+        swapped = false;
+        while (base && test) {
+            if ((*compare) (base->item, test->item) > 0) {
+                //  It's trivial to swap items in a generic container
+                void *item = base->item;
+                base->item = test->item;
+                test->item = item;
+                swapped = true;
+            }
+            base = base->next;
+            test = test->next;
+        }
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Sets a compare function for this list. The function compares two items.
+//  It returns an integer less than, equal to, or greater than zero if the
+//  first item is found, respectively, to be less than, to match, or be
+//  greater than the second item.
+//  This function is used for sorting, removal and exists checking.
+
+void
+zlist_comparefn (zlist_t *self,  zlist_compare_fn fn)
+{
+    assert (self);
+    self->compare_fn = fn;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a free function for the specified list item. When the item is
+//  destroyed, the free function, if any, is called on that item.
+//  Use this when list items are dynamically allocated, to ensure that
+//  you don't have memory leaks. You can pass 'free' or NULL as a free_fn.
+//  Returns the item, or NULL if there is no such item.
+
+void *
+zlist_freefn (zlist_t *self, void *item, zlist_free_fn fn, bool at_tail)
+{
+    node_t *node = self->head;
+    if (at_tail)
+        node = self->tail;
+
+    while (node) {
+        if (node->item == item) {
+            node->free_fn = fn;
+            return item;
+        }
+        node = node->next;
+    }
+    return NULL;
+}
+
+//  --------------------------------------------------------------------------
+//  Set list for automatic item destruction; item values MUST be strings.
+//  By default a list item refers to a value held elsewhere. When you set
+//  this, each time you append or push a list item, zlist will take a copy
+//  of the string value. Then, when you destroy the list, it will free all
+//  item values automatically. If you use any other technique to allocate
+//  list values, you must free them explicitly before destroying the list.
+//  The usual technique is to pop list items and destroy them, until the
+//  list is empty.
+
+void
+zlist_autofree (zlist_t *self)
+{
+    assert (self);
+    self->autofree = true;
+}
+
+
+#ifdef CZMQ_BUILD_EXTRA
+static void
+s_zlist_free (void *data)
+{
+    zlist_t *self = (zlist_t *) data;
+    zlist_destroy (&self);
+}
+
+//  --------------------------------------------------------------------------
+//  Runs selftest of class
+
+void
+zlist_test (bool verbose)
+{
+    printf (" * zlist: ");
+
+    //  @selftest
+    zlist_t *list = zlist_new ();
+    assert (list);
+    assert (zlist_size (list) == 0);
+
+    //  Three items we'll use as test data
+    //  List items are void *, not particularly strings
+    char *cheese = "boursin";
+    char *bread = "baguette";
+    char *wine = "bordeaux";
+
+    zlist_append (list, cheese);
+    assert (zlist_size (list) == 1);
+    assert ( zlist_exists (list, cheese));
+    assert (!zlist_exists (list, bread));
+    assert (!zlist_exists (list, wine));
+    zlist_append (list, bread);
+    assert (zlist_size (list) == 2);
+    assert ( zlist_exists (list, cheese));
+    assert ( zlist_exists (list, bread));
+    assert (!zlist_exists (list, wine));
+    zlist_append (list, wine);
+    assert (zlist_size (list) == 3);
+    assert ( zlist_exists (list, cheese));
+    assert ( zlist_exists (list, bread));
+    assert ( zlist_exists (list, wine));
+
+    assert (zlist_head (list) == cheese);
+    assert (zlist_next (list) == cheese);
+
+    assert (zlist_first (list) == cheese);
+    assert (zlist_tail (list) == wine);
+    assert (zlist_next (list) == bread);
+
+    assert (zlist_first (list) == cheese);
+    assert (zlist_next (list) == bread);
+    assert (zlist_next (list) == wine);
+    assert (zlist_next (list) == NULL);
+    //  After we reach end of list, next wraps around
+    assert (zlist_next (list) == cheese);
+    assert (zlist_size (list) == 3);
+
+    zlist_remove (list, wine);
+    assert (zlist_size (list) == 2);
+
+    assert (zlist_first (list) == cheese);
+    zlist_remove (list, cheese);
+    assert (zlist_size (list) == 1);
+    assert (zlist_first (list) == bread);
+
+    zlist_remove (list, bread);
+    assert (zlist_size (list) == 0);
+
+    zlist_append (list, cheese);
+    zlist_append (list, bread);
+    assert (zlist_last (list) == bread);
+    zlist_remove (list, bread);
+    assert (zlist_last (list) == cheese);
+    zlist_remove (list, cheese);
+    assert (zlist_last (list) == NULL);
+
+    zlist_push (list, cheese);
+    assert (zlist_size (list) == 1);
+    assert (zlist_first (list) == cheese);
+
+    zlist_push (list, bread);
+    assert (zlist_size (list) == 2);
+    assert (zlist_first (list) == bread);
+    assert (zlist_item (list) == bread);
+
+    zlist_append (list, wine);
+    assert (zlist_size (list) == 3);
+    assert (zlist_first (list) == bread);
+
+    zlist_t *sub_list = zlist_dup (list);
+    assert (sub_list);
+    assert (zlist_size (sub_list) == 3);
+
+    zlist_sort (list, NULL);
+    char *item;
+    item = (char *) zlist_pop (list);
+    assert (item == bread);
+    item = (char *) zlist_pop (list);
+    assert (item == wine);
+    item = (char *) zlist_pop (list);
+    assert (item == cheese);
+    assert (zlist_size (list) == 0);
+
+    assert (zlist_size (sub_list) == 3);
+    zlist_push (list, sub_list);
+    zlist_t *sub_list_2 = zlist_dup (sub_list);
+    zlist_append (list, sub_list_2);
+    assert (zlist_freefn (list, sub_list, &s_zlist_free, false) == sub_list);
+    assert (zlist_freefn (list, sub_list_2, &s_zlist_free, true) == sub_list_2);
+    zlist_destroy (&list);
+
+    //  Test autofree functionality
+    list = zlist_new ();
+    assert (list);
+    zlist_autofree (list);
+    //  Set equals function otherwise equals will not work as autofree copies strings
+    zlist_comparefn (list, (zlist_compare_fn *) strcmp);
+    zlist_push (list, bread);
+    zlist_append (list, cheese);
+    assert (zlist_size (list) == 2);
+    zlist_append (list, wine);
+    assert (zlist_exists (list, wine));
+    zlist_remove (list, wine);
+    assert (!zlist_exists (list, wine));
+    assert (streq ((const char *) zlist_first (list), bread));
+    item = (char *) zlist_pop (list);
+    assert (streq (item, bread));
+    freen (item);
+    item = (char *) zlist_pop (list);
+    assert (streq (item, cheese));
+    freen (item);
+
+    zlist_destroy (&list);
+    assert (list == NULL);
+
+#if defined (__WINDOWS__)
+    zsys_shutdown();
+#endif
+    //  @end
+
+    printf ("OK\n");
+}
+#endif // CZMQ_BUILD_EXTRA
diff --git a/src/common/libczmqcontainers/zlist.h b/src/common/libczmqcontainers/zlist.h
new file mode 100644
index 000000000000..fd47229be010
--- /dev/null
+++ b/src/common/libczmqcontainers/zlist.h
@@ -0,0 +1,160 @@
+/*  =========================================================================
+    zlist - simple generic list container
+
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+#ifndef __FLUX_ZLIST_H_INCLUDED__
+#define __FLUX_ZLIST_H_INCLUDED__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//  @warning THE FOLLOWING @INTERFACE BLOCK IS AUTO-GENERATED BY ZPROJECT
+//  @warning Please edit the model at "api/zlist.api" to make changes.
+//  @interface
+//  This is a stable class, and may not change except for emergencies. It
+//  is provided in stable builds.
+// Comparison function e.g. for sorting and removing.
+typedef int (zlist_compare_fn) (
+    void *item1, void *item2);
+
+// Callback function for zlist_freefn method
+typedef void (zlist_free_fn) (
+    void *data);
+
+//  Create a new list container
+CZMQ_EXPORT zlist_t *
+    zlist_new (void);
+
+//  Destroy a list container
+CZMQ_EXPORT void
+    zlist_destroy (zlist_t **self_p);
+
+//  Return the item at the head of list. If the list is empty, returns NULL.
+//  Leaves cursor pointing at the head item, or NULL if the list is empty.
+CZMQ_EXPORT void *
+    zlist_first (zlist_t *self);
+
+//  Return the next item. If the list is empty, returns NULL. To move to
+//  the start of the list call zlist_first (). Advances the cursor.
+CZMQ_EXPORT void *
+    zlist_next (zlist_t *self);
+
+//  Return the item at the tail of list. If the list is empty, returns NULL.
+//  Leaves cursor pointing at the tail item, or NULL if the list is empty.
+CZMQ_EXPORT void *
+    zlist_last (zlist_t *self);
+
+//  Return first item in the list, or null, leaves the cursor
+CZMQ_EXPORT void *
+    zlist_head (zlist_t *self);
+
+//  Return last item in the list, or null, leaves the cursor
+CZMQ_EXPORT void *
+    zlist_tail (zlist_t *self);
+
+//  Return the current item of list. If the list is empty, returns NULL.
+//  Leaves cursor pointing at the current item, or NULL if the list is empty.
+CZMQ_EXPORT void *
+    zlist_item (zlist_t *self);
+
+//  Append an item to the end of the list, return 0 if OK or -1 if this
+//  failed for some reason (out of memory). Note that if a duplicator has
+//  been set, this method will also duplicate the item.
+CZMQ_EXPORT int
+    zlist_append (zlist_t *self, void *item);
+
+//  Push an item to the start of the list, return 0 if OK or -1 if this
+//  failed for some reason (out of memory). Note that if a duplicator has
+//  been set, this method will also duplicate the item.
+CZMQ_EXPORT int
+    zlist_push (zlist_t *self, void *item);
+
+//  Pop the item off the start of the list, if any
+CZMQ_EXPORT void *
+    zlist_pop (zlist_t *self);
+
+//  Checks if an item already is present. Uses compare method to determine if
+//  items are equal. If the compare method is NULL the check will only compare
+//  pointers. Returns true if item is present else false.
+CZMQ_EXPORT bool
+    zlist_exists (zlist_t *self, void *item);
+
+//  Remove the specified item from the list if present
+CZMQ_EXPORT void
+    zlist_remove (zlist_t *self, void *item);
+
+//  Make a copy of list. If the list has autofree set, the copied list will
+//  duplicate all items, which must be strings. Otherwise, the list will hold
+//  pointers back to the items in the original list. If list is null, returns
+//  NULL.
+//  Caller owns return value and must destroy it when done.
+CZMQ_EXPORT zlist_t *
+    zlist_dup (zlist_t *self);
+
+//  Purge all items from list
+CZMQ_EXPORT void
+    zlist_purge (zlist_t *self);
+
+//  Return number of items in the list
+CZMQ_EXPORT size_t
+    zlist_size (zlist_t *self);
+
+//  Sort the list. If the compare function is null, sorts the list by
+//  ascending key value using a straight ASCII comparison. If you specify
+//  a compare function, this decides how items are sorted. The sort is not
+//  stable, so may reorder items with the same keys. The algorithm used is
+//  combsort, a compromise between performance and simplicity.
+CZMQ_EXPORT void
+    zlist_sort (zlist_t *self, zlist_compare_fn compare);
+
+//  Set list for automatic item destruction; item values MUST be strings.
+//  By default a list item refers to a value held elsewhere. When you set
+//  this, each time you append or push a list item, zlist will take a copy
+//  of the string value. Then, when you destroy the list, it will free all
+//  item values automatically. If you use any other technique to allocate
+//  list values, you must free them explicitly before destroying the list.
+//  The usual technique is to pop list items and destroy them, until the
+//  list is empty.
+CZMQ_EXPORT void
+    zlist_autofree (zlist_t *self);
+
+//  Sets a compare function for this list. The function compares two items.
+//  It returns an integer less than, equal to, or greater than zero if the
+//  first item is found, respectively, to be less than, to match, or be
+//  greater than the second item.
+//  This function is used for sorting, removal and exists checking.
+CZMQ_EXPORT void
+    zlist_comparefn (zlist_t *self, zlist_compare_fn fn);
+
+//  Set a free function for the specified list item. When the item is
+//  destroyed, the free function, if any, is called on that item.
+//  Use this when list items are dynamically allocated, to ensure that
+//  you don't have memory leaks. You can pass 'free' or NULL as a free_fn.
+//  Returns the item, or NULL if there is no such item.
+CZMQ_EXPORT void *
+    zlist_freefn (zlist_t *self, void *item, zlist_free_fn fn, bool at_tail);
+
+#ifdef CZMQ_BUILD_EXTRA
+//  Self test of this class.
+CZMQ_EXPORT void
+    zlist_test (bool verbose);
+#endif // CZMQ_BUILD_EXTRA
+
+//  @end
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/common/libczmqcontainers/zlistx.c b/src/common/libczmqcontainers/zlistx.c
new file mode 100644
index 000000000000..f24277c00193
--- /dev/null
+++ b/src/common/libczmqcontainers/zlistx.c
@@ -0,0 +1,882 @@
+/*  =========================================================================
+    zlistx - extended generic list container
+
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+/*
+@header
+    Provides a generic doubly-linked list container. This container provides
+    hooks for duplicator, comparator, and destructor functions. These tie
+    into CZMQ and standard C semantics, so e.g. for string items you can
+    use strdup, strcmp, and zstr_free. To store custom objects, define your
+    own duplicator and comparator, and use the standard object destructor.
+@discuss
+    This is a reworking of the simpler zlist container. It is faster to
+    insert and delete items anywhere in the list, and to keep ordered lists.
+@end
+*/
+
+#include "czmq_containers.h"
+#include "czmq_internal.h"
+
+
+#define NODE_TAG            0xcafe0006
+
+//  List node, used internally only
+
+typedef struct _node_t {
+    uint32_t tag;                   //  Object tag for validity checking
+    struct _node_t *next;
+    struct _node_t *prev;
+    void *item;
+} node_t;
+
+
+//  ---------------------------------------------------------------------
+//  Structure of our class
+
+struct _zlistx_t {
+    node_t *head;                   //  First item in list, if any
+    node_t *cursor;                 //  Current cursors for iteration
+    size_t size;                    //  Number of items in list
+    //  Function callbacks for duplicating and destroying items, if any
+    zlistx_duplicator_fn *duplicator;
+    zlistx_destructor_fn *destructor;
+    zlistx_comparator_fn *comparator;
+};
+
+
+//  Initialize a list node and attach to the prev and next nodes, or itself
+//  if these are specified as null. Returns new node.
+
+static node_t *
+s_node_new (void *item)
+{
+    node_t *self = (node_t *) zmalloc (sizeof (node_t));
+    assert (self);
+    self->tag = NODE_TAG;
+    self->prev = self;
+    self->next = self;
+    self->item = item;
+    return self;
+}
+
+
+//  Removing and inserting a node are actually the same operation:
+//      swap (node->next, prev->next)
+//      swap (node->prev, next->prev)
+//  Which require only that the node be initialized to point to itself.
+//  When inserting, node goes in between prev and next.
+
+static void
+s_node_relink (node_t *node, node_t *prev, node_t *next)
+{
+    node_t *temp = node->next;
+    node->next = prev->next;
+    prev->next = temp;
+    temp = node->prev;
+    node->prev = next->prev;
+    next->prev = temp;
+}
+
+//  Default comparator
+
+static int
+s_comparator (const void *item1, const void *item2)
+{
+    if (item1 == item2)
+        return 0;
+    else
+    if (item1 < item2)
+        return -1;
+    else
+        return 1;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Create a new, empty list.
+
+zlistx_t *
+zlistx_new (void)
+{
+    zlistx_t *self = (zlistx_t *) zmalloc (sizeof (zlistx_t));
+    assert (self);
+    self->head = s_node_new (NULL);
+    assert (self->head);
+    self->cursor = self->head;
+    self->comparator = s_comparator;
+    return self;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Destroy a list. If an item destructor was specified, all items in the
+//  list are automatically destroyed as well.
+
+void
+zlistx_destroy (zlistx_t **self_p)
+{
+    assert (self_p);
+    if (*self_p) {
+        zlistx_t *self = *self_p;
+        zlistx_purge (self);
+        freen (self->head);
+        freen (self);
+        *self_p = NULL;
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Add an item to the head of the list. Calls the item duplicator, if any,
+//  on the item. Resets cursor to list head. Returns an item handle on
+//  success.
+
+void *
+zlistx_add_start (zlistx_t *self, void *item)
+{
+    assert (self);
+    assert (item);
+
+    if (self->duplicator) {
+        item = (self->duplicator) (item);
+        assert (item);
+    }
+    node_t *node = s_node_new (item);
+    assert (node);
+
+    //  Insert after head
+    s_node_relink (node, self->head, self->head->next);
+    self->cursor = self->head;
+    self->size++;
+    return node;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Add an item to the tail of the list. Calls the item duplicator, if any,
+//  on the item. Resets cursor to list head. Returns an item handle on
+//  success.
+
+void *
+zlistx_add_end (zlistx_t *self, void *item)
+{
+    assert (self);
+    assert (item);
+
+    if (self->duplicator) {
+        item = (self->duplicator) (item);
+        assert (item);
+    }
+    node_t *node = s_node_new (item);
+    assert (node);
+
+    //  Insert before head
+    s_node_relink (node, self->head->prev, self->head);
+    self->cursor = self->head;
+    self->size++;
+    return node;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the number of items in the list
+
+size_t
+zlistx_size (zlistx_t *self)
+{
+    assert (self);
+    return self->size;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the item at the head of list. If the list is empty, returns NULL.
+//  Leaves cursor as-is.
+
+void *
+zlistx_head (zlistx_t *self)
+{
+    assert (self);
+    return self->head? self->head->next->item: NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the item at the tail of list. If the list is empty, returns NULL.
+
+//  Leaves cursor as-is.
+
+void *
+zlistx_tail (zlistx_t *self)
+{
+    assert (self);
+    return self->head? self->head->prev->item: NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the item at the head of list. If the list is empty, returns NULL.
+//  Leaves cursor pointing at the head item, or NULL if the list is empty.
+
+void *
+zlistx_first (zlistx_t *self)
+{
+    assert (self);
+    self->cursor = self->head->next;
+    return self->cursor == self->head? NULL: self->cursor->item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the next item. At the end of the list (or in an empty list),
+//  returns NULL. Use repeated zlistx_next () calls to work through the list
+//  from zlistx_first (). First time, acts as zlistx_first().
+
+void *
+zlistx_next (zlistx_t *self)
+{
+    assert (self);
+    self->cursor = self->cursor->next;
+    return self->cursor == self->head? NULL: self->cursor->item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the previous item. At the start of the list (or in an empty list),
+//  returns NULL. Use repeated zlistx_prev () calls to work through the list
+//  backwards from zlistx_last (). First time, acts as zlistx_last().
+
+void *
+zlistx_prev (zlistx_t *self)
+{
+    assert (self);
+    self->cursor = self->cursor->prev;
+    return self->cursor == self->head? NULL: self->cursor->item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Return the item at the tail of list. If the list is empty, returns NULL.
+//  Leaves cursor pointing at the tail item, or NULL if the list is empty.
+
+void *
+zlistx_last (zlistx_t *self)
+{
+    assert (self);
+    self->cursor = self->head->prev;
+    return self->cursor == self->head? NULL: self->cursor->item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Returns the value of the item at the cursor, or NULL if the cursor is
+//  not pointing to an item.
+
+void *
+zlistx_item (zlistx_t *self)
+{
+    assert (self);
+    return self->cursor == self->head? NULL: self->cursor->item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Returns the handle of the item at the cursor, or NULL if the cursor is
+//  not pointing to an item.
+
+void *
+zlistx_cursor (zlistx_t *self)
+{
+    assert (self);
+    return self->cursor == self->head? NULL: self->cursor;
+}
+
+//  --------------------------------------------------------------------------
+//  Returns the item associated with the given list handle, or NULL if passed
+//  handle is NULL. asserts that the passed in ptr points to a list node.
+
+void *
+zlistx_handle_item (void *handle)
+{
+    if (!handle)
+        return NULL;
+
+    node_t *node = (node_t *) handle;
+    assert (node->tag == NODE_TAG);
+    return node->item;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Find an item in the list, searching from the start. Uses the item
+//  comparator, if any, else compares item values directly. Returns the
+//  item handle found, or NULL. Sets the cursor to the found item, if any.
+
+void *
+zlistx_find (zlistx_t *self, void *item)
+{
+    assert (self);
+    assert (item);
+
+    //  Scan list for item, this is a O(N) operation
+    node_t *node = self->head->next;
+    while (node != self->head) {
+        if (self->comparator (node->item, item) == 0) {
+            self->cursor = node;
+            return node;
+        }
+        node = node->next;
+    }
+    return NULL;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Detach an item from the list, using its handle. The item is not modified,
+//  and the caller is responsible for destroying it if necessary. If handle is
+//  null, detaches the first item on the list. Returns item that was detached,
+//  or null if none was. If cursor was at item, moves cursor to previous item,
+//  so you can detach items while iterating forwards through a list.
+
+void *
+zlistx_detach (zlistx_t *self, void *handle)
+{
+    assert (self);
+    node_t *node = (node_t *) handle;
+    if (!node)
+        node = self->head->next == self->head? NULL: self->head->next;
+
+    //  Now detach node from list, without destroying item
+    if (node) {
+        //  Reposition cursor so that delete/detach works during iteration
+        if (self->cursor == node)
+            self->cursor = self->cursor->prev;
+
+        //  Remove node from list
+        assert (node->tag == NODE_TAG);
+        s_node_relink (node, node->prev, node->next);
+        node->tag = 0xDeadBeef;
+        void *item = node->item;
+        freen (node);
+        self->size--;
+        return item;
+    }
+    else {
+        assert (self->size == 0);
+        return NULL;
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Detach item at the cursor, if any, from the list. The item is not modified,
+//  and the caller is responsible for destroying it as necessary. Returns item
+//  that was detached, or null if none was. Moves cursor to previous item, so
+//  you can detach items while iterating forwards through a list.
+
+void *
+zlistx_detach_cur (zlistx_t *self)
+{
+    return zlistx_detach (self, zlistx_cursor (self));
+}
+
+
+//  --------------------------------------------------------------------------
+//  Delete an item, using its handle. Calls the item destructor if any is
+//  set. If handle is null, deletes the first item on the list. Returns 0
+//  if an item was deleted, -1 if not. If cursor was at item, moves cursor
+//  to previous item, so you can delete items while iterating forwards
+//  through a list.
+
+int
+zlistx_delete (zlistx_t *self, void *handle)
+{
+    assert (self);
+    void *item = zlistx_detach (self, handle);
+    if (item) {
+        if (self->destructor)
+            self->destructor (&item);
+        return 0;
+    }
+    else
+        return -1;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Move an item to the start of the list, via its handle.
+
+void
+zlistx_move_start (zlistx_t *self, void *handle)
+{
+    assert (self);
+    assert (handle);
+    node_t *node = (node_t *) handle;
+    assert (node->tag == NODE_TAG);
+
+    node_t *next = self->head->next;
+    if (node != next) {
+        //  Remove node from list, insert before next node
+        s_node_relink (node, node->prev, node->next);
+        s_node_relink (node, next->prev, next);
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Move an item to the end of the list, via its handle.
+
+void
+zlistx_move_end (zlistx_t *self, void *handle)
+{
+    assert (self);
+    assert (handle);
+    node_t *node = (node_t *) handle;
+    assert (node->tag == NODE_TAG);
+
+    node_t *prev = self->head->prev;
+    if (node != prev) {
+        //  Remove node from list, insert after prev node
+        s_node_relink (node, node->prev, node->next);
+        s_node_relink (node, prev, prev->next);
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Remove all items from the list, and destroy them if the item destructor
+//  is set.
+
+void
+zlistx_purge (zlistx_t *self)
+{
+    assert (self);
+    while (zlistx_size (self) > 0)
+        zlistx_delete (self, NULL);
+}
+
+
+//  --------------------------------------------------------------------------
+//  Sort the list. If an item comparator was set, calls that to compare
+//  items, otherwise compares on item value. The sort is not stable, so may
+//  reorder equal items.
+
+void
+zlistx_sort (zlistx_t *self)
+{
+    //  Uses a comb sort, which is simple and reasonably fast
+    //  See http://en.wikipedia.org/wiki/Comb_sort
+    assert (self);
+    size_t gap = self->size;
+    bool swapped = false;
+    while (gap > 1 || swapped) {
+        gap = (size_t) ((double) gap / 1.3);
+        node_t *base = self->head->next;
+        node_t *test = self->head->next;
+        size_t jump = gap;
+        while (jump--)
+            test = test->next;
+
+        swapped = false;
+        while (base != self->head && test != self->head) {
+            if (self->comparator (base->item, test->item) > 0) {
+                //  We don't actually swap nodes, just the items in the nodes.
+                //  This is ridiculously simple and confuses the heck out of
+                //  me every time I re-read the code, as I expect to see the
+                //  nodes being swapped. Hence this comment. -- PH 2014/09/06
+                void *item = base->item;
+                base->item = test->item;
+                test->item = item;
+                swapped = true;
+            }
+            base = base->next;
+            test = test->next;
+        }
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Create a new node and insert it into a sorted list. Calls the item
+//  duplicator, if any, on the item. If low_value is true, starts searching
+//  from the start of the list, otherwise searches from the end. Use the item
+//  comparator, if any, to find where to place the new node. Returns a handle
+//  to the new node, or NULL if memory was exhausted. Resets the cursor to the
+//  list head.
+
+void *
+zlistx_insert (zlistx_t *self, void *item, bool low_value)
+{
+    assert (self);
+    if (self->duplicator) {
+        item = (self->duplicator) (item);
+        assert (item);
+    }
+    node_t *node = s_node_new (item);
+    assert (node);
+    zlistx_reorder (self, node, low_value);
+    self->cursor = self->head;
+    self->size++;
+    return node;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Move an item, specified by handle, into position in a sorted list. Uses
+//  the item comparator, if any, to determine the new location. If low_value
+//  is true, starts searching from the start of the list, otherwise searches
+//  from the end.
+
+void
+zlistx_reorder (zlistx_t *self, void *handle, bool low_value)
+{
+    assert (self);
+    assert (handle);
+    node_t *node = (node_t *) handle;
+    assert (node->tag == NODE_TAG);
+
+    //  Remove node from list, if it's attached
+    s_node_relink (node, node->prev, node->next);
+
+    if (low_value) {
+        node_t *next = self->head->next;
+        while (next != self->head) {
+            if (self->comparator (node->item, next->item) <= 0)
+                break;
+            next = next->next;
+        }
+        //  Relink node before next node
+        s_node_relink (node, next->prev, next);
+    }
+    else {
+        node_t *prev = self->head->prev;
+        while (prev != self->head) {
+            if (self->comparator (prev->item, node->item) <= 0)
+                break;
+            prev = prev->prev;
+        }
+        //  Relink node after prev node
+        s_node_relink (node, prev, prev->next);
+    }
+}
+
+
+//  --------------------------------------------------------------------------
+//  Make a copy of the list; items are duplicated if you set a duplicator
+//  for the list, otherwise not. Copying a null reference returns a null
+//  reference.
+
+zlistx_t *
+zlistx_dup (zlistx_t *self)
+{
+    if (!self)
+        return NULL;
+
+    zlistx_t *copy = zlistx_new ();
+    if (copy) {
+        //  Copy item handlers
+        copy->destructor = self->destructor;
+        copy->duplicator = self->duplicator;
+        copy->comparator = self->comparator;
+
+        //  Copy nodes
+        node_t *node;
+        for (node = self->head->next; node != self->head; node = node->next)
+            zlistx_add_end (copy, node->item);
+    }
+    return copy;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a user-defined deallocator for list items; by default items are not
+//  freed when the list is destroyed.
+
+void
+zlistx_set_destructor (zlistx_t *self, zlistx_destructor_fn destructor)
+{
+    assert (self);
+    self->destructor = destructor;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a user-defined duplicator for list items; by default items are not
+//  copied when the list is duplicated.
+
+void
+zlistx_set_duplicator (zlistx_t *self, zlistx_duplicator_fn duplicator)
+{
+    assert (self);
+    self->duplicator = duplicator;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Set a user-defined comparator for zlistx_find and zlistx_sort; the method
+//  must return -1, 0, or 1 depending on whether item1 is less than, equal to,
+//  or greater than, item2.
+
+void
+zlistx_set_comparator (zlistx_t *self, zlistx_comparator_fn comparator)
+{
+    assert (self);
+    self->comparator = comparator;
+}
+
+#ifdef CZMQ_BUILD_EXTRA
+//  --------------------------------------------------------------------------
+//  Serialize list to a binary frame that can be sent in a message.
+//  The packed format is compatible with the 'strings' type implemented by zproto:
+//
+//     ; A list of strings
+//     list            = list-count *longstr
+//     list-count      = number-4
+//
+//     ; Strings are always length + text contents
+//     longstr         = number-4 *VCHAR
+//
+//     ; Numbers are unsigned integers in network byte order
+//     number-4        = 4OCTET
+//  Caller owns return value and must destroy it when done.
+zframe_t *
+zlistx_pack (zlistx_t *self) {
+    assert (self);
+
+    //  First, calculate the packed data size
+    size_t frame_size = 4;      // List size, number-4
+    char* item = (char *) zlistx_first (self);
+
+    while (item) {
+        frame_size += 4 + strlen (item);
+        item = (char *) zlistx_next (self);
+    }
+
+    //  Now serialize items into the frame
+    zframe_t *frame = zframe_new (NULL, frame_size);
+    if (!frame)
+        return NULL;
+
+    byte *needle = zframe_data (frame);
+    *(uint32_t *) needle = htonl ((u_long) self->size);
+    needle += 4;
+
+    item = (char *) zlistx_first (self);
+    while (item) {
+        size_t length = strlen (item);
+        uint32_t serialize = htonl ((u_long) length);
+        memcpy (needle, &serialize, 4);
+        needle += 4;
+        memcpy (needle, item, length);
+        needle += length;
+
+        item = (char *) zlistx_next (self);
+    }
+
+    return frame;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Unpack binary frame into a new list. Packed data must follow format
+//  defined by zlistx_pack. List is set to autofree. An empty frame
+//  unpacks to an empty list.
+zlistx_t *
+zlistx_unpack (zframe_t *frame) {
+    zlistx_t *self = zlistx_new ();
+    if (!self)
+        return NULL;
+
+    //  List will free values in destructor
+    zlistx_set_destructor (self, (zlistx_destructor_fn *) zstr_free);
+
+    assert (frame);
+    if (zframe_size (frame) < 4)
+        return self;
+
+    byte *needle = zframe_data (frame);
+    byte *ceiling = needle + zframe_size (frame);
+    size_t nbr_items = ntohl (*(uint32_t *) needle);
+    needle +=4;
+    while (nbr_items && needle < ceiling) {
+        if (needle + 4 <= ceiling) {
+            size_t length = ntohl (*(uint32_t *)needle);
+            needle += 4;
+            // Be wary of malformed frames
+            if (needle + length <= ceiling) {
+                char * item = (char *) zmalloc (length + 1);
+                assert (item);
+                memcpy (item, needle, length);
+                item[length] = 0;
+                needle += length;
+
+                if (!zlistx_add_end (self, item)) {
+                    zlistx_destroy (&self);
+                    break;
+                }
+            } else {
+
+                zlistx_destroy (&self);
+                break;
+            }
+        } else {
+            zlistx_destroy (&self);
+            break;
+        }
+    }
+
+    if (self)
+    	zlistx_set_duplicator (self, (zlistx_duplicator_fn *) strdup);
+
+    return self;
+}
+
+
+//  --------------------------------------------------------------------------
+//  Runs selftest of class
+
+void
+zlistx_test (bool verbose)
+{
+    printf (" * zlistx: ");
+
+    //  @selftest
+    zlistx_t *list = zlistx_new ();
+    assert (list);
+    assert (zlistx_size (list) == 0);
+
+    //  Test operations on an empty list
+    assert (zlistx_head (list) == NULL);
+    assert (zlistx_first (list) == NULL);
+    assert (zlistx_last (list) == NULL);
+    assert (zlistx_next (list) == NULL);
+    assert (zlistx_prev (list) == NULL);
+    assert (zlistx_find (list, "hello") == NULL);
+    assert (zlistx_delete (list, NULL) == -1);
+    assert (zlistx_detach (list, NULL) == NULL);
+    assert (zlistx_delete (list, NULL) == -1);
+    assert (zlistx_detach (list, NULL) == NULL);
+    zlistx_purge (list);
+    zlistx_sort (list);
+
+    //  Use item handlers
+    zlistx_set_destructor (list, (zlistx_destructor_fn *) zstr_free);
+    zlistx_set_duplicator (list, (zlistx_duplicator_fn *) strdup);
+    zlistx_set_comparator (list, (zlistx_comparator_fn *) strcmp);
+
+    //  Try simple insert/sort/delete/next
+    assert (zlistx_next (list) == NULL);
+    zlistx_add_end (list, "world");
+    assert (streq ((char *) zlistx_next (list), "world"));
+    assert (streq ((char *) zlistx_head (list), "world"));
+    zlistx_add_end (list, "hello");
+    assert (streq ((char *) zlistx_prev (list), "hello"));
+    zlistx_sort (list);
+    assert (zlistx_size (list) == 2);
+    void *handle = zlistx_find (list, "hello");
+    char *item1 = (char *) zlistx_item (list);
+    char *item2 = (char *) zlistx_handle_item (handle);
+    assert (item1 == item2);
+    assert (streq (item1, "hello"));
+    zlistx_delete (list, handle);
+    assert (zlistx_size (list) == 1);
+    char *string = (char *) zlistx_detach (list, NULL);
+    assert (streq (string, "world"));
+    freen (string);
+    assert (zlistx_size (list) == 0);
+
+    //  Check next/back work
+    //  Now populate the list with items
+    zlistx_add_start (list, "five");
+    zlistx_add_end   (list, "six");
+    zlistx_add_start (list, "four");
+    zlistx_add_end   (list, "seven");
+    zlistx_add_start (list, "three");
+    zlistx_add_end   (list, "eight");
+    zlistx_add_start (list, "two");
+    zlistx_add_end   (list, "nine");
+    zlistx_add_start (list, "one");
+    zlistx_add_end   (list, "ten");
+
+    //  Test our navigation skills
+    assert (zlistx_size (list) == 10);
+    assert (streq ((char *) zlistx_last (list), "ten"));
+    assert (streq ((char *) zlistx_prev (list), "nine"));
+    assert (streq ((char *) zlistx_prev (list), "eight"));
+    assert (streq ((char *) zlistx_prev (list), "seven"));
+    assert (streq ((char *) zlistx_prev (list), "six"));
+    assert (streq ((char *) zlistx_prev (list), "five"));
+    assert (streq ((char *) zlistx_first (list), "one"));
+    assert (streq ((char *) zlistx_next (list), "two"));
+    assert (streq ((char *) zlistx_next (list), "three"));
+    assert (streq ((char *) zlistx_next (list), "four"));
+
+    //  Sort by alphabetical order
+    zlistx_sort (list);
+    assert (streq ((char *) zlistx_first (list), "eight"));
+    assert (streq ((char *) zlistx_last (list), "two"));
+
+    //  Moving items around
+    handle = zlistx_find (list, "six");
+    zlistx_move_start (list, handle);
+    assert (streq ((char *) zlistx_first (list), "six"));
+    zlistx_move_end (list, handle);
+    assert (streq ((char *) zlistx_last (list), "six"));
+    zlistx_sort (list);
+    assert (streq ((char *) zlistx_last (list), "two"));
+
+    //  Copying a list
+    zlistx_t *copy = zlistx_dup (list);
+    assert (copy);
+    assert (zlistx_size (copy) == 10);
+    assert (streq ((char *) zlistx_first (copy), "eight"));
+    assert (streq ((char *) zlistx_last (copy), "two"));
+    zlistx_destroy (&copy);
+
+    //  Delete items while iterating
+    string = (char *) zlistx_first (list);
+    assert (streq (string, "eight"));
+    string = (char *) zlistx_next (list);
+    assert (streq (string, "five"));
+    zlistx_delete (list, zlistx_cursor (list));
+    string = (char *) zlistx_next (list);
+    assert (streq (string, "four"));
+
+    //  Test pack/unpack methods
+    zframe_t *frame = zlistx_pack (list);
+    copy = zlistx_unpack (frame);
+    assert (copy);
+    zframe_destroy (&frame);
+    assert (zlistx_size (copy) == zlistx_size (list));
+
+    char *item_orig = (char *) zlistx_first (list);
+    char *item_copy = (char *) zlistx_first (copy);
+    while (item_orig) {
+        assert (strcmp(item_orig, item_copy) == 0);
+        item_orig = (char *) zlistx_next (list);
+        item_copy = (char *) zlistx_next (copy);
+    }
+
+    zlistx_destroy (&copy);
+
+    zlistx_purge (list);
+    zlistx_destroy (&list);
+
+#if defined (__WINDOWS__)
+    zsys_shutdown();
+#endif
+    //  @end
+
+    printf ("OK\n");
+}
+#endif // CZMQ_BUILD_EXTRA
diff --git a/src/common/libczmqcontainers/zlistx.h b/src/common/libczmqcontainers/zlistx.h
new file mode 100644
index 000000000000..1bb55eb6c7b3
--- /dev/null
+++ b/src/common/libczmqcontainers/zlistx.h
@@ -0,0 +1,235 @@
+/*  =========================================================================
+    zlistx - extended generic list container
+
+    Copyright (c) the Contributors as noted in the AUTHORS file.
+    This file is part of CZMQ, the high-level C binding for 0MQ:
+    http://czmq.zeromq.org.
+
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    =========================================================================
+*/
+
+#ifndef __FLUXZLISTX_H_INCLUDED__
+#define __FLUXZLISTX_H_INCLUDED__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//  @warning THE FOLLOWING @INTERFACE BLOCK IS AUTO-GENERATED BY ZPROJECT
+//  @warning Please edit the model at "api/zlistx.api" to make changes.
+//  @interface
+//  This is a stable class, and may not change except for emergencies. It
+//  is provided in stable builds.
+//  This class has draft methods, which may change over time. They are not
+//  in stable releases, by default. Use --enable-drafts to enable.
+// Destroy an item
+typedef void (zlistx_destructor_fn) (
+    void **item);
+
+// Duplicate an item
+typedef void * (zlistx_duplicator_fn) (
+    const void *item);
+
+// Compare two items, for sorting
+typedef int (zlistx_comparator_fn) (
+    const void *item1, const void *item2);
+
+//  Create a new, empty list.
+CZMQ_EXPORT zlistx_t *
+    zlistx_new (void);
+
+//  Destroy a list. If an item destructor was specified, all items in the
+//  list are automatically destroyed as well.
+CZMQ_EXPORT void
+    zlistx_destroy (zlistx_t **self_p);
+
+//  Add an item to the head of the list. Calls the item duplicator, if any,
+//  on the item. Resets cursor to list head. Returns an item handle on
+//  success, NULL if memory was exhausted.
+CZMQ_EXPORT void *
+    zlistx_add_start (zlistx_t *self, void *item);
+
+//  Add an item to the tail of the list. Calls the item duplicator, if any,
+//  on the item. Resets cursor to list head. Returns an item handle on
+//  success, NULL if memory was exhausted.
+CZMQ_EXPORT void *
+    zlistx_add_end (zlistx_t *self, void *item);
+
+//  Return the number of items in the list
+CZMQ_EXPORT size_t
+    zlistx_size (zlistx_t *self);
+
+//  Return first item in the list, or null, leaves the cursor
+CZMQ_EXPORT void *
+    zlistx_head (zlistx_t *self);
+
+//  Return last item in the list, or null, leaves the cursor
+CZMQ_EXPORT void *
+    zlistx_tail (zlistx_t *self);
+
+//  Return the item at the head of list. If the list is empty, returns NULL.
+//  Leaves cursor pointing at the head item, or NULL if the list is empty.
+CZMQ_EXPORT void *
+    zlistx_first (zlistx_t *self);
+
+//  Return the next item. At the end of the list (or in an empty list),
+//  returns NULL. Use repeated zlistx_next () calls to work through the list
+//  from zlistx_first (). First time, acts as zlistx_first().
+CZMQ_EXPORT void *
+    zlistx_next (zlistx_t *self);
+
+//  Return the previous item. At the start of the list (or in an empty list),
+//  returns NULL. Use repeated zlistx_prev () calls to work through the list
+//  backwards from zlistx_last (). First time, acts as zlistx_last().
+CZMQ_EXPORT void *
+    zlistx_prev (zlistx_t *self);
+
+//  Return the item at the tail of list. If the list is empty, returns NULL.
+//  Leaves cursor pointing at the tail item, or NULL if the list is empty.
+CZMQ_EXPORT void *
+    zlistx_last (zlistx_t *self);
+
+//  Returns the value of the item at the cursor, or NULL if the cursor is
+//  not pointing to an item.
+CZMQ_EXPORT void *
+    zlistx_item (zlistx_t *self);
+
+//  Returns the handle of the item at the cursor, or NULL if the cursor is
+//  not pointing to an item.
+CZMQ_EXPORT void *
+    zlistx_cursor (zlistx_t *self);
+
+//  Returns the item associated with the given list handle, or NULL if passed
+//  in handle is NULL. Asserts that the passed in handle points to a list element.
+CZMQ_EXPORT void *
+    zlistx_handle_item (void *handle);
+
+//  Find an item in the list, searching from the start. Uses the item
+//  comparator, if any, else compares item values directly. Returns the
+//  item handle found, or NULL. Sets the cursor to the found item, if any.
+CZMQ_EXPORT void *
+    zlistx_find (zlistx_t *self, void *item);
+
+//  Detach an item from the list, using its handle. The item is not modified,
+//  and the caller is responsible for destroying it if necessary. If handle is
+//  null, detaches the first item on the list. Returns item that was detached,
+//  or null if none was. If cursor was at item, moves cursor to previous item,
+//  so you can detach items while iterating forwards through a list.
+CZMQ_EXPORT void *
+    zlistx_detach (zlistx_t *self, void *handle);
+
+//  Detach item at the cursor, if any, from the list. The item is not modified,
+//  and the caller is responsible for destroying it as necessary. Returns item
+//  that was detached, or null if none was. Moves cursor to previous item, so
+//  you can detach items while iterating forwards through a list.
+CZMQ_EXPORT void *
+    zlistx_detach_cur (zlistx_t *self);
+
+//  Delete an item, using its handle. Calls the item destructor if any is
+//  set. If handle is null, deletes the first item on the list. Returns 0
+//  if an item was deleted, -1 if not. If cursor was at item, moves cursor
+//  to previous item, so you can delete items while iterating forwards
+//  through a list.
+CZMQ_EXPORT int
+    zlistx_delete (zlistx_t *self, void *handle);
+
+//  Move an item to the start of the list, via its handle.
+CZMQ_EXPORT void
+    zlistx_move_start (zlistx_t *self, void *handle);
+
+//  Move an item to the end of the list, via its handle.
+CZMQ_EXPORT void
+    zlistx_move_end (zlistx_t *self, void *handle);
+
+//  Remove all items from the list, and destroy them if the item destructor
+//  is set.
+CZMQ_EXPORT void
+    zlistx_purge (zlistx_t *self);
+
+//  Sort the list. If an item comparator was set, calls that to compare
+//  items, otherwise compares on item value. The sort is not stable, so may
+//  reorder equal items.
+CZMQ_EXPORT void
+    zlistx_sort (zlistx_t *self);
+
+//  Create a new node and insert it into a sorted list. Calls the item
+//  duplicator, if any, on the item. If low_value is true, starts searching
+//  from the start of the list, otherwise searches from the end. Use the item
+//  comparator, if any, to find where to place the new node. Returns a handle
+//  to the new node, or NULL if memory was exhausted. Resets the cursor to the
+//  list head.
+CZMQ_EXPORT void *
+    zlistx_insert (zlistx_t *self, void *item, bool low_value);
+
+//  Move an item, specified by handle, into position in a sorted list. Uses
+//  the item comparator, if any, to determine the new location. If low_value
+//  is true, starts searching from the start of the list, otherwise searches
+//  from the end.
+CZMQ_EXPORT void
+    zlistx_reorder (zlistx_t *self, void *handle, bool low_value);
+
+//  Make a copy of the list; items are duplicated if you set a duplicator
+//  for the list, otherwise not. Copying a null reference returns a null
+//  reference.
+CZMQ_EXPORT zlistx_t *
+    zlistx_dup (zlistx_t *self);
+
+//  Set a user-defined deallocator for list items; by default items are not
+//  freed when the list is destroyed.
+CZMQ_EXPORT void
+    zlistx_set_destructor (zlistx_t *self, zlistx_destructor_fn destructor);
+
+//  Set a user-defined duplicator for list items; by default items are not
+//  copied when the list is duplicated.
+CZMQ_EXPORT void
+    zlistx_set_duplicator (zlistx_t *self, zlistx_duplicator_fn duplicator);
+
+//  Set a user-defined comparator for zlistx_find and zlistx_sort; the method
+//  must return -1, 0, or 1 depending on whether item1 is less than, equal to,
+//  or greater than, item2.
+CZMQ_EXPORT void
+    zlistx_set_comparator (zlistx_t *self, zlistx_comparator_fn comparator);
+
+#ifdef CZMQ_BUILD_EXTRA
+//  Self test of this class.
+CZMQ_EXPORT void
+    zlistx_test (bool verbose);
+
+#ifdef CZMQ_BUILD_DRAFT_API
+//  *** Draft method, for development use, may change without warning ***
+//  Unpack binary frame into a new list. Packed data must follow format
+//  defined by zlistx_pack. List is set to autofree. An empty frame
+//  unpacks to an empty list.
+CZMQ_EXPORT zlistx_t *
+    zlistx_unpack (zframe_t *frame);
+
+//  *** Draft method, for development use, may change without warning ***
+//  Serialize list to a binary frame that can be sent in a message.
+//  The packed format is compatible with the 'strings' type implemented by zproto:
+//
+//     ; A list of strings
+//     list            = list-count *longstr
+//     list-count      = number-4
+//
+//     ; Strings are always length + text contents
+//     longstr         = number-4 *VCHAR
+//
+//     ; Numbers are unsigned integers in network byte order
+//     number-4        = 4OCTET
+//  Caller owns return value and must destroy it when done.
+CZMQ_EXPORT zframe_t *
+    zlistx_pack (zlistx_t *self);
+
+#endif // CZMQ_BUILD_DRAFT_API
+#endif // CZMQ_BUILD_EXTRA
+//  @end
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/common/libdebugged/Makefile.am b/src/common/libdebugged/Makefile.am
new file mode 100644
index 000000000000..31f0ec96588f
--- /dev/null
+++ b/src/common/libdebugged/Makefile.am
@@ -0,0 +1,27 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+        -I$(top_srcdir)
+
+noinst_LTLIBRARIES = libdebugged.la
+libdebugged_la_SOURCES = debugged.c debugged.h
+
+TESTS = test_debugged.t
+
+check_PROGRAMS = $(TESTS)
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+       $(top_srcdir)/config/tap-driver.sh
+
+test_debugged_t_SOURCES = test/test_debugged.c
+test_debugged_t_CPPFLAGS = $(AM_CPPFLAGS)
+test_debugged_t_LDADD = \
+        $(top_builddir)/src/common/libtap/libtap.la \
+        $(top_builddir)/src/common/libdebugged/libdebugged.la
diff --git a/src/common/libdebugged/debugged.c b/src/common/libdebugged/debugged.c
new file mode 100644
index 000000000000..55d273fba55e
--- /dev/null
+++ b/src/common/libdebugged/debugged.c
@@ -0,0 +1,36 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "debugged.h"
+
+int MPIR_being_debugged = 0;
+
+void MPIR_Breakpoint ()
+{
+
+}
+
+int get_mpir_being_debugged ()
+{
+    return MPIR_being_debugged;
+}
+
+void set_mpir_being_debugged (int v)
+{
+    MPIR_being_debugged = v;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libdebugged/debugged.h b/src/common/libdebugged/debugged.h
new file mode 100644
index 000000000000..6192b78ea18f
--- /dev/null
+++ b/src/common/libdebugged/debugged.h
@@ -0,0 +1,31 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_DEBUGGED_H
+#define _FLUX_CORE_DEBUGGED_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int MPIR_being_debugged;
+extern void MPIR_Breakpoint (void);
+extern int get_mpir_being_debugged (void);
+extern void set_mpir_being_debugged (int v);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_FLUX_CORE_DEBUGGED_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libdebugged/test/test_debugged.c b/src/common/libdebugged/test/test_debugged.c
new file mode 100644
index 000000000000..0b7d8c0bbbc4
--- /dev/null
+++ b/src/common/libdebugged/test/test_debugged.c
@@ -0,0 +1,44 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libdebugged/debugged.h"
+
+void test_initial_state (void)
+{
+    int debugged = get_mpir_being_debugged ();
+    ok (debugged == 0,
+        "mpir_being_debugged is initially 0");
+    set_mpir_being_debugged (1);
+    debugged = get_mpir_being_debugged ();
+    ok (debugged == 1,
+        "mpir_being_debugged is set to 1 (e.g., under debugger control)");
+    set_mpir_being_debugged (0);
+    debugged = get_mpir_being_debugged ();
+    ok (debugged == 0,
+        "mpir_being_debugged is unset to 0 (e.g., debugger detached)");
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_initial_state ();
+
+    done_testing ();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libev/LICENSE b/src/common/libev/LICENSE
new file mode 100644
index 000000000000..2fdabd48afad
--- /dev/null
+++ b/src/common/libev/LICENSE
@@ -0,0 +1,37 @@
+All files in libev are
+Copyright (c)2007,2008,2009,2010,2011,2012,2013 Marc Alexander Lehmann.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Alternatively, the contents of this package may be used under the terms
+of the GNU General Public License ("GPL") version 2 or any later version,
+in which case the provisions of the GPL are applicable instead of the
+above. If you wish to allow the use of your version of this package only
+under the terms of the GPL and not to allow others to use your version of
+this file under the BSD license, indicate your decision by deleting the
+provisions above and replace them with the notice and other provisions
+required by the GPL in this and the other files of this package. If you do
+not delete the provisions above, a recipient may use your version of this
+file under either the BSD or the GPL.
diff --git a/src/common/libev/Makefile.am b/src/common/libev/Makefile.am
index 7ab8c63a6738..421fc36afe5a 100644
--- a/src/common/libev/Makefile.am
+++ b/src/common/libev/Makefile.am
@@ -1,4 +1,4 @@
-AM_CPPFLAGS = -w
+AM_CPPFLAGS = -w $(CODE_COVERAGE_CPPFLAGS)
 
 noinst_LTLIBRARIES = libev.la
 
@@ -13,4 +13,8 @@ EXTRA_DIST = \
 	ev_poll.c \
 	ev_epoll.c \
 	ev_linuxaio.c \
-	libev.m4
+	ev_port.c \
+	ev_kqueue.c \
+	ev_iouring.c \
+	libev.m4 \
+	LICENSE
diff --git a/src/common/libev/ev.c b/src/common/libev/ev.c
index ffa4091b2b2a..cc9e57455e55 100644
--- a/src/common/libev/ev.c
+++ b/src/common/libev/ev.c
@@ -119,13 +119,22 @@
    
 # if HAVE_LINUX_AIO_ABI_H
 #  ifndef EV_USE_LINUXAIO
-#   define EV_USE_LINUXAIO EV_FEATURE_BACKENDS
+#   define EV_USE_LINUXAIO 0 /* was: EV_FEATURE_BACKENDS, always off by default */
 #  endif
 # else
 #  undef EV_USE_LINUXAIO
 #  define EV_USE_LINUXAIO 0
 # endif
    
+# if HAVE_LINUX_FS_H && HAVE_SYS_TIMERFD_H && HAVE_KERNEL_RWF_T
+#  ifndef EV_USE_IOURING
+#   define EV_USE_IOURING EV_FEATURE_BACKENDS
+#  endif
+# else
+#  undef EV_USE_IOURING
+#  define EV_USE_IOURING 0
+# endif
+ 
 # if HAVE_KQUEUE && HAVE_SYS_EVENT_H
 #  ifndef EV_USE_KQUEUE
 #   define EV_USE_KQUEUE EV_FEATURE_BACKENDS
@@ -170,7 +179,16 @@
 #  undef EV_USE_EVENTFD
 #  define EV_USE_EVENTFD 0
 # endif
- 
+
+# if HAVE_SYS_TIMERFD_H
+#  ifndef EV_USE_TIMERFD
+#   define EV_USE_TIMERFD EV_FEATURE_OS
+#  endif
+# else
+#  undef EV_USE_TIMERFD
+#  define EV_USE_TIMERFD 0
+# endif
+
 #endif
 
 /* OS X, in its infinite idiocy, actually HARDCODES
@@ -328,12 +346,20 @@
 
 #ifndef EV_USE_LINUXAIO
 # if __linux /* libev currently assumes linux/aio_abi.h is always available on linux */
-#  define EV_USE_LINUXAIO 1
+#  define EV_USE_LINUXAIO 0 /* was: 1, always off by default */
 # else
 #  define EV_USE_LINUXAIO 0
 # endif
 #endif
 
+#ifndef EV_USE_IOURING
+# if __linux /* later checks might disable again */
+#  define EV_USE_IOURING 1
+# else
+#  define EV_USE_IOURING 0
+# endif
+#endif
+
 #ifndef EV_USE_INOTIFY
 # if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 4))
 #  define EV_USE_INOTIFY EV_FEATURE_OS
@@ -366,6 +392,14 @@
 # endif
 #endif
 
+#ifndef EV_USE_TIMERFD
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 8))
+#  define EV_USE_TIMERFD EV_FEATURE_OS
+# else
+#  define EV_USE_TIMERFD 0
+# endif
+#endif
+
 #if 0 /* debugging */
 # define EV_VERIFY 3
 # define EV_USE_4HEAP 1
@@ -408,6 +442,7 @@
 #  define clock_gettime(id, ts) syscall (SYS_clock_gettime, (id), (ts))
 #  undef EV_USE_MONOTONIC
 #  define EV_USE_MONOTONIC 1
+#  define EV_NEED_SYSCALL 1
 # else
 #  undef EV_USE_CLOCK_SYSCALL
 #  define EV_USE_CLOCK_SYSCALL 0
@@ -431,6 +466,14 @@
 # define EV_USE_INOTIFY 0
 #endif
 
+#if __linux && EV_USE_IOURING
+# include <linux/version.h>
+# if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+#  undef EV_USE_IOURING
+#  define EV_USE_IOURING 0
+# endif
+#endif
+
 #if !EV_USE_NANOSLEEP
 /* hp-ux has it in sys/time.h, which we unconditionally include above */
 # if !defined _WIN32 && !defined __hpux
@@ -440,12 +483,29 @@
 
 #if EV_USE_LINUXAIO
 # include <sys/syscall.h>
-# if !SYS_io_getevents || !EV_USE_EPOLL /* ev_linxaio uses ev_poll.c:ev_epoll_create */
+# if SYS_io_getevents && EV_USE_EPOLL /* linuxaio backend requires epoll backend */
+#  define EV_NEED_SYSCALL 1
+# else
 #  undef EV_USE_LINUXAIO
 #  define EV_USE_LINUXAIO 0
 # endif
 #endif
 
+#if EV_USE_IOURING
+# include <sys/syscall.h>
+# if !SYS_io_uring_setup && __linux && !__alpha
+#  define SYS_io_uring_setup     425
+#  define SYS_io_uring_enter     426
+#  define SYS_io_uring_wregister 427
+# endif
+# if SYS_io_uring_setup && EV_USE_EPOLL /* iouring backend requires epoll backend */
+#  define EV_NEED_SYSCALL 1
+# else
+#  undef EV_USE_IOURING
+#  define EV_USE_IOURING 0
+# endif
+#endif
+
 #if EV_USE_INOTIFY
 # include <sys/statfs.h>
 # include <sys/inotify.h>
@@ -457,7 +517,7 @@
 #endif
 
 #if EV_USE_EVENTFD
-/* our minimum requirement is glibc 2.7 which has the stub, but not the header */
+/* our minimum requirement is glibc 2.7 which has the stub, but not the full header */
 # include <stdint.h>
 # ifndef EFD_NONBLOCK
 #  define EFD_NONBLOCK O_NONBLOCK
@@ -473,7 +533,7 @@ EV_CPP(extern "C") int (eventfd) (unsigned int initval, int flags);
 #endif
 
 #if EV_USE_SIGNALFD
-/* our minimum requirement is glibc 2.7 which has the stub, but not the header */
+/* our minimum requirement is glibc 2.7 which has the stub, but not the full header */
 # include <stdint.h>
 # ifndef SFD_NONBLOCK
 #  define SFD_NONBLOCK O_NONBLOCK
@@ -485,7 +545,7 @@ EV_CPP(extern "C") int (eventfd) (unsigned int initval, int flags);
 #   define SFD_CLOEXEC 02000000
 #  endif
 # endif
-EV_CPP (extern "C") int signalfd (int fd, const sigset_t *mask, int flags);
+EV_CPP (extern "C") int (signalfd) (int fd, const sigset_t *mask, int flags);
 
 struct signalfd_siginfo
 {
@@ -494,7 +554,23 @@ struct signalfd_siginfo
 };
 #endif
 
-/**/
+/* for timerfd, libev core requires TFD_TIMER_CANCEL_ON_SET &c */
+#if EV_USE_TIMERFD
+# include <sys/timerfd.h>
+/* timerfd is only used for periodics */
+# if !(defined (TFD_TIMER_CANCEL_ON_SET) && defined (TFD_CLOEXEC) && defined (TFD_NONBLOCK)) || !EV_PERIODIC_ENABLE
+#  undef EV_USE_TIMERFD
+#  define EV_USE_TIMERFD 0
+# endif
+#endif
+
+/* disable timerfd on macos */
+#ifdef __APPLE__
+#  undef EV_USE_TIMERFD
+#  define EV_USE_TIMERFD 0
+#endif
+
+/*****************************************************************************/
 
 #if EV_VERIFY >= 3
 # define EV_FREQUENT_CHECK ev_verify (EV_A)
@@ -509,18 +585,34 @@ struct signalfd_siginfo
 #define MIN_INTERVAL  0.0001220703125 /* 1/2**13, good till 4000 */
 /*#define MIN_INTERVAL  0.00000095367431640625 /* 1/2**20, good till 2200 */
 
-#define MIN_TIMEJUMP  1. /* minimum timejump that gets detected (if monotonic clock available) */
-#define MAX_BLOCKTIME 59.743 /* never wait longer than this time (to detect time jumps) */
+#define MIN_TIMEJUMP   1. /* minimum timejump that gets detected (if monotonic clock available) */
+#define MAX_BLOCKTIME  59.743 /* never wait longer than this time (to detect time jumps) */
+#define MAX_BLOCKTIME2 1500001.07 /* same, but when timerfd is used to detect jumps, also safe delay to not overflow */
+
+/* find a portable timestamp that is "always" in the future but fits into time_t.
+ * this is quite hard, and we are mostly guessing - we handle 32 bit signed/unsigned time_t,
+ * and sizes larger than 32 bit, and maybe the unlikely floating point time_t */
+#define EV_TSTAMP_HUGE \
+  (sizeof (time_t) >= 8     ? 10000000000000.  \
+   : 0 < (time_t)4294967295 ?     4294967295.  \
+   :                              2147483647.) \
 
-#define EV_TV_SET(tv,t) do { tv.tv_sec = (long)t; tv.tv_usec = (long)((t - tv.tv_sec) * 1e6); } while (0)
-#define EV_TS_SET(ts,t) do { ts.tv_sec = (long)t; ts.tv_nsec = (long)((t - ts.tv_sec) * 1e9); } while (0)
+#ifndef EV_TS_CONST
+# define EV_TS_CONST(nv) nv
+# define EV_TS_TO_MSEC(a) a * 1e3 + 0.9999
+# define EV_TS_FROM_USEC(us) us * 1e-6
+# define EV_TV_SET(tv,t) do { tv.tv_sec = (long)t; tv.tv_usec = (long)((t - tv.tv_sec) * 1e6); } while (0)
+# define EV_TS_SET(ts,t) do { ts.tv_sec = (long)t; ts.tv_nsec = (long)((t - ts.tv_sec) * 1e9); } while (0)
+# define EV_TV_GET(tv) ((tv).tv_sec + (tv).tv_usec * 1e-6)
+# define EV_TS_GET(ts) ((ts).tv_sec + (ts).tv_nsec * 1e-9)
+#endif
 
 /* the following is ecb.h embedded into libev - use update_ev_c to update from an external copy */
 /* ECB.H BEGIN */
 /*
  * libecb - http://software.schmorp.de/pkg/libecb
  *
- * Copyright (©) 2009-2015 Marc Alexander Lehmann <libecb@schmorp.de>
+ * Copyright (©) 2009-2015,2018-2020 Marc Alexander Lehmann <libecb@schmorp.de>
  * Copyright (©) 2011 Emanuele Giaquinta
  * All rights reserved.
  *
@@ -561,15 +653,23 @@ struct signalfd_siginfo
 #define ECB_H
 
 /* 16 bits major, 16 bits minor */
-#define ECB_VERSION 0x00010006
+#define ECB_VERSION 0x00010008
 
-#ifdef _WIN32
+#include <string.h> /* for memcpy */
+
+#if defined (_WIN32) && !defined (__MINGW32__)
   typedef   signed char   int8_t;
   typedef unsigned char  uint8_t;
+  typedef   signed char   int_fast8_t;
+  typedef unsigned char  uint_fast8_t;
   typedef   signed short  int16_t;
   typedef unsigned short uint16_t;
+  typedef   signed int    int_fast16_t;
+  typedef unsigned int   uint_fast16_t;
   typedef   signed int    int32_t;
   typedef unsigned int   uint32_t;
+  typedef   signed int    int_fast32_t;
+  typedef unsigned int   uint_fast32_t;
   #if __GNUC__
     typedef   signed long long int64_t;
     typedef unsigned long long uint64_t;
@@ -577,6 +677,8 @@ struct signalfd_siginfo
     typedef   signed __int64   int64_t;
     typedef unsigned __int64   uint64_t;
   #endif
+  typedef  int64_t  int_fast64_t;
+  typedef uint64_t uint_fast64_t;
   #ifdef _WIN64
     #define ECB_PTRSIZE 8
     typedef uint64_t uintptr_t;
@@ -598,6 +700,14 @@ struct signalfd_siginfo
 #define ECB_GCC_AMD64 (__amd64 || __amd64__ || __x86_64 || __x86_64__)
 #define ECB_MSVC_AMD64 (_M_AMD64 || _M_X64)
 
+#ifndef ECB_OPTIMIZE_SIZE
+  #if __OPTIMIZE_SIZE__
+    #define ECB_OPTIMIZE_SIZE 1
+  #else
+    #define ECB_OPTIMIZE_SIZE 0
+  #endif
+#endif
+
 /* work around x32 idiocy by defining proper macros */
 #if ECB_GCC_AMD64 || ECB_MSVC_AMD64
   #if _ILP32
@@ -687,7 +797,7 @@ struct signalfd_siginfo
   #if ECB_GCC_VERSION(2,5) || defined __INTEL_COMPILER || (__llvm__ && __GNUC__) || __SUNPRO_C >= 0x5110 || __SUNPRO_CC >= 0x5110
     #define ECB_MEMORY_FENCE_RELAXED __asm__ __volatile__ ("" : : : "memory")
     #if __i386 || __i386__
-      #define ECB_MEMORY_FENCE         __asm__ __volatile__ ("lock; orb $0, -1(%%esp)" : : : "memory")
+      #define ECB_MEMORY_FENCE         __asm__ __volatile__ ("lock; orb $0, 0(%%esp)"  : : : "memory")
       #define ECB_MEMORY_FENCE_ACQUIRE __asm__ __volatile__ (""                        : : : "memory")
       #define ECB_MEMORY_FENCE_RELEASE __asm__ __volatile__ (""                        : : : "memory")
     #elif ECB_GCC_AMD64
@@ -1113,6 +1223,44 @@ ecb_inline ecb_const uint32_t ecb_rotr32 (uint32_t x, unsigned int count) { retu
 ecb_inline ecb_const uint64_t ecb_rotl64 (uint64_t x, unsigned int count) { return (x >> (64 - count)) | (x << count); }
 ecb_inline ecb_const uint64_t ecb_rotr64 (uint64_t x, unsigned int count) { return (x << (64 - count)) | (x >> count); }
 
+#if ECB_CPP
+
+inline uint8_t  ecb_ctz (uint8_t  v) { return ecb_ctz32 (v); }
+inline uint16_t ecb_ctz (uint16_t v) { return ecb_ctz32 (v); }
+inline uint32_t ecb_ctz (uint32_t v) { return ecb_ctz32 (v); }
+inline uint64_t ecb_ctz (uint64_t v) { return ecb_ctz64 (v); }
+
+inline bool ecb_is_pot (uint8_t  v) { return ecb_is_pot32 (v); }
+inline bool ecb_is_pot (uint16_t v) { return ecb_is_pot32 (v); }
+inline bool ecb_is_pot (uint32_t v) { return ecb_is_pot32 (v); }
+inline bool ecb_is_pot (uint64_t v) { return ecb_is_pot64 (v); }
+
+inline int ecb_ld (uint8_t  v) { return ecb_ld32 (v); }
+inline int ecb_ld (uint16_t v) { return ecb_ld32 (v); }
+inline int ecb_ld (uint32_t v) { return ecb_ld32 (v); }
+inline int ecb_ld (uint64_t v) { return ecb_ld64 (v); }
+
+inline int ecb_popcount (uint8_t  v) { return ecb_popcount32 (v); }
+inline int ecb_popcount (uint16_t v) { return ecb_popcount32 (v); }
+inline int ecb_popcount (uint32_t v) { return ecb_popcount32 (v); }
+inline int ecb_popcount (uint64_t v) { return ecb_popcount64 (v); }
+
+inline uint8_t  ecb_bitrev (uint8_t  v) { return ecb_bitrev8  (v); }
+inline uint16_t ecb_bitrev (uint16_t v) { return ecb_bitrev16 (v); }
+inline uint32_t ecb_bitrev (uint32_t v) { return ecb_bitrev32 (v); }
+
+inline uint8_t  ecb_rotl (uint8_t  v, unsigned int count) { return ecb_rotl8  (v, count); }
+inline uint16_t ecb_rotl (uint16_t v, unsigned int count) { return ecb_rotl16 (v, count); }
+inline uint32_t ecb_rotl (uint32_t v, unsigned int count) { return ecb_rotl32 (v, count); }
+inline uint64_t ecb_rotl (uint64_t v, unsigned int count) { return ecb_rotl64 (v, count); }
+
+inline uint8_t  ecb_rotr (uint8_t  v, unsigned int count) { return ecb_rotr8  (v, count); }
+inline uint16_t ecb_rotr (uint16_t v, unsigned int count) { return ecb_rotr16 (v, count); }
+inline uint32_t ecb_rotr (uint32_t v, unsigned int count) { return ecb_rotr32 (v, count); }
+inline uint64_t ecb_rotr (uint64_t v, unsigned int count) { return ecb_rotr64 (v, count); }
+
+#endif
+
 #if ECB_GCC_VERSION(4,3) || (ECB_CLANG_BUILTIN(__builtin_bswap32) && ECB_CLANG_BUILTIN(__builtin_bswap64))
   #if ECB_GCC_VERSION(4,8) || ECB_CLANG_BUILTIN(__builtin_bswap16)
   #define ecb_bswap16(x)  __builtin_bswap16 (x)
@@ -1193,6 +1341,78 @@ ecb_inline ecb_const ecb_bool ecb_big_endian    (void) { return ecb_byteorder_he
 ecb_inline ecb_const ecb_bool ecb_little_endian (void);
 ecb_inline ecb_const ecb_bool ecb_little_endian (void) { return ecb_byteorder_helper () == 0x44332211; }
 
+/*****************************************************************************/
+/* unaligned load/store */
+
+ecb_inline uint_fast16_t ecb_be_u16_to_host (uint_fast16_t v) { return ecb_little_endian () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_be_u32_to_host (uint_fast32_t v) { return ecb_little_endian () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_be_u64_to_host (uint_fast64_t v) { return ecb_little_endian () ? ecb_bswap64 (v) : v; }
+
+ecb_inline uint_fast16_t ecb_le_u16_to_host (uint_fast16_t v) { return ecb_big_endian    () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_le_u32_to_host (uint_fast32_t v) { return ecb_big_endian    () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_le_u64_to_host (uint_fast64_t v) { return ecb_big_endian    () ? ecb_bswap64 (v) : v; }
+
+ecb_inline uint_fast16_t ecb_peek_u16_u (const void *ptr) { uint16_t v; memcpy (&v, ptr, sizeof (v)); return v; }
+ecb_inline uint_fast32_t ecb_peek_u32_u (const void *ptr) { uint32_t v; memcpy (&v, ptr, sizeof (v)); return v; }
+ecb_inline uint_fast64_t ecb_peek_u64_u (const void *ptr) { uint64_t v; memcpy (&v, ptr, sizeof (v)); return v; }
+
+ecb_inline uint_fast16_t ecb_peek_be_u16_u (const void *ptr) { return ecb_be_u16_to_host (ecb_peek_u16_u (ptr)); }
+ecb_inline uint_fast32_t ecb_peek_be_u32_u (const void *ptr) { return ecb_be_u32_to_host (ecb_peek_u32_u (ptr)); }
+ecb_inline uint_fast64_t ecb_peek_be_u64_u (const void *ptr) { return ecb_be_u64_to_host (ecb_peek_u64_u (ptr)); }
+
+ecb_inline uint_fast16_t ecb_peek_le_u16_u (const void *ptr) { return ecb_le_u16_to_host (ecb_peek_u16_u (ptr)); }
+ecb_inline uint_fast32_t ecb_peek_le_u32_u (const void *ptr) { return ecb_le_u32_to_host (ecb_peek_u32_u (ptr)); }
+ecb_inline uint_fast64_t ecb_peek_le_u64_u (const void *ptr) { return ecb_le_u64_to_host (ecb_peek_u64_u (ptr)); }
+
+ecb_inline uint_fast16_t ecb_host_to_be_u16 (uint_fast16_t v) { return ecb_little_endian () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_host_to_be_u32 (uint_fast32_t v) { return ecb_little_endian () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_host_to_be_u64 (uint_fast64_t v) { return ecb_little_endian () ? ecb_bswap64 (v) : v; }
+
+ecb_inline uint_fast16_t ecb_host_to_le_u16 (uint_fast16_t v) { return ecb_big_endian    () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_host_to_le_u32 (uint_fast32_t v) { return ecb_big_endian    () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_host_to_le_u64 (uint_fast64_t v) { return ecb_big_endian    () ? ecb_bswap64 (v) : v; }
+
+ecb_inline void ecb_poke_u16_u (void *ptr, uint16_t v) { memcpy (ptr, &v, sizeof (v)); }
+ecb_inline void ecb_poke_u32_u (void *ptr, uint32_t v) { memcpy (ptr, &v, sizeof (v)); }
+ecb_inline void ecb_poke_u64_u (void *ptr, uint64_t v) { memcpy (ptr, &v, sizeof (v)); }
+
+ecb_inline void ecb_poke_be_u16_u (void *ptr, uint_fast16_t v) { ecb_poke_u16_u (ptr, ecb_host_to_be_u16 (v)); }
+ecb_inline void ecb_poke_be_u32_u (void *ptr, uint_fast32_t v) { ecb_poke_u32_u (ptr, ecb_host_to_be_u32 (v)); }
+ecb_inline void ecb_poke_be_u64_u (void *ptr, uint_fast64_t v) { ecb_poke_u64_u (ptr, ecb_host_to_be_u64 (v)); }
+                                                                                                
+ecb_inline void ecb_poke_le_u16_u (void *ptr, uint_fast16_t v) { ecb_poke_u16_u (ptr, ecb_host_to_le_u16 (v)); }
+ecb_inline void ecb_poke_le_u32_u (void *ptr, uint_fast32_t v) { ecb_poke_u32_u (ptr, ecb_host_to_le_u32 (v)); }
+ecb_inline void ecb_poke_le_u64_u (void *ptr, uint_fast64_t v) { ecb_poke_u64_u (ptr, ecb_host_to_le_u64 (v)); }
+
+#if ECB_CPP
+
+inline uint8_t  ecb_bswap (uint8_t  v) { return v; }
+inline uint16_t ecb_bswap (uint16_t v) { return ecb_bswap16 (v); }
+inline uint32_t ecb_bswap (uint32_t v) { return ecb_bswap32 (v); }
+inline uint64_t ecb_bswap (uint64_t v) { return ecb_bswap64 (v); }
+
+template<typename T> inline T ecb_be_to_host (T v) { return ecb_little_endian () ? ecb_bswap (v) : v; }
+template<typename T> inline T ecb_le_to_host (T v) { return ecb_big_endian    () ? ecb_bswap (v) : v; }
+template<typename T> inline T ecb_peek       (const void *ptr) { return *(const T *)ptr; }
+template<typename T> inline T ecb_peek_be    (const void *ptr) { return ecb_be_to_host (ecb_peek  <T> (ptr)); }
+template<typename T> inline T ecb_peek_le    (const void *ptr) { return ecb_le_to_host (ecb_peek  <T> (ptr)); }
+template<typename T> inline T ecb_peek_u     (const void *ptr) { T v; memcpy (&v, ptr, sizeof (v)); return v; }
+template<typename T> inline T ecb_peek_be_u  (const void *ptr) { return ecb_be_to_host (ecb_peek_u<T> (ptr)); }
+template<typename T> inline T ecb_peek_le_u  (const void *ptr) { return ecb_le_to_host (ecb_peek_u<T> (ptr)); }
+
+template<typename T> inline T ecb_host_to_be (T v) { return ecb_little_endian () ? ecb_bswap (v) : v; }
+template<typename T> inline T ecb_host_to_le (T v) { return ecb_big_endian    () ? ecb_bswap (v) : v; }
+template<typename T> inline void ecb_poke      (void *ptr, T v) { *(T *)ptr = v; }
+template<typename T> inline void ecb_poke_be   (void *ptr, T v) { return ecb_poke  <T> (ptr, ecb_host_to_be (v)); }
+template<typename T> inline void ecb_poke_le   (void *ptr, T v) { return ecb_poke  <T> (ptr, ecb_host_to_le (v)); }
+template<typename T> inline void ecb_poke_u    (void *ptr, T v) { memcpy (ptr, &v, sizeof (v)); }
+template<typename T> inline void ecb_poke_be_u (void *ptr, T v) { return ecb_poke_u<T> (ptr, ecb_host_to_be (v)); }
+template<typename T> inline void ecb_poke_le_u (void *ptr, T v) { return ecb_poke_u<T> (ptr, ecb_host_to_le (v)); }
+
+#endif
+
+/*****************************************************************************/
+
 #if ECB_GCC_VERSION(3,0) || ECB_C99
   #define ecb_mod(m,n) ((m) % (n) + ((m) % (n) < 0 ? (n) : 0))
 #else
@@ -1226,6 +1446,8 @@ ecb_inline ecb_const ecb_bool ecb_little_endian (void) { return ecb_byteorder_he
   #define ecb_array_length(name) (sizeof (name) / sizeof (name [0]))
 #endif
 
+/*****************************************************************************/
+
 ecb_function_ ecb_const uint32_t ecb_binary16_to_binary32 (uint32_t x);
 ecb_function_ ecb_const uint32_t
 ecb_binary16_to_binary32 (uint32_t x)
@@ -1343,7 +1565,6 @@ ecb_binary32_to_binary16 (uint32_t x)
     || (defined __arm__ && (defined __ARM_EABI__ || defined __EABI__ || defined __VFP_FP__ || defined _WIN32_WCE || defined __ANDROID__)) \
     || defined __aarch64__
   #define ECB_STDFP 1
-  #include <string.h> /* for memcpy */
 #else
   #define ECB_STDFP 0
 #endif
@@ -1538,7 +1759,7 @@ ecb_binary32_to_binary16 (uint32_t x)
 #if ECB_MEMORY_FENCE_NEEDS_PTHREADS
 /* if your architecture doesn't need memory fences, e.g. because it is
  * single-cpu/core, or if you use libev in a project that doesn't use libev
- * from multiple threads, then you can define ECB_AVOID_PTHREADS when compiling
+ * from multiple threads, then you can define ECB_NO_THREADS when compiling
  * libev, in which cases the memory fences become nops.
  * alternatively, you can remove this #error and link against libpthread,
  * which will then provide the memory fences.
@@ -1552,18 +1773,80 @@ ecb_binary32_to_binary16 (uint32_t x)
 # define ECB_MEMORY_FENCE_RELEASE ECB_MEMORY_FENCE
 #endif
 
-#define expect_false(cond) ecb_expect_false (cond)
-#define expect_true(cond)  ecb_expect_true  (cond)
-#define noinline           ecb_noinline
-
 #define inline_size        ecb_inline
 
 #if EV_FEATURE_CODE
 # define inline_speed      ecb_inline
 #else
-# define inline_speed      noinline static
+# define inline_speed      ecb_noinline static
+#endif
+
+/*****************************************************************************/
+/* raw syscall wrappers */
+
+#if EV_NEED_SYSCALL
+
+#include <sys/syscall.h>
+
+/*
+ * define some syscall wrappers for common architectures
+ * this is mostly for nice looks during debugging, not performance.
+ * our syscalls return < 0, not == -1, on error. which is good
+ * enough for linux aio.
+ * TODO: arm is also common nowadays, maybe even mips and x86
+ * TODO: after implementing this, it suddenly looks like overkill, but its hard to remove...
+ */
+#if __GNUC__ && __linux && ECB_AMD64 && !EV_FEATURE_CODE
+  /* the costly errno access probably kills this for size optimisation */
+
+  #define ev_syscall(nr,narg,arg1,arg2,arg3,arg4,arg5,arg6)            \
+    ({                                                                 \
+        long res;                                                      \
+        register unsigned long r6 __asm__ ("r9" );                     \
+        register unsigned long r5 __asm__ ("r8" );                     \
+        register unsigned long r4 __asm__ ("r10");                     \
+        register unsigned long r3 __asm__ ("rdx");                     \
+        register unsigned long r2 __asm__ ("rsi");                     \
+        register unsigned long r1 __asm__ ("rdi");                     \
+        if (narg >= 6) r6 = (unsigned long)(arg6);                     \
+        if (narg >= 5) r5 = (unsigned long)(arg5);                     \
+        if (narg >= 4) r4 = (unsigned long)(arg4);                     \
+        if (narg >= 3) r3 = (unsigned long)(arg3);                     \
+        if (narg >= 2) r2 = (unsigned long)(arg2);                     \
+        if (narg >= 1) r1 = (unsigned long)(arg1);                     \
+        __asm__ __volatile__ (                                         \
+          "syscall\n\t"                                                \
+          : "=a" (res)                                                 \
+          : "0" (nr), "r" (r1), "r" (r2), "r" (r3), "r" (r4), "r" (r5) \
+          : "cc", "r11", "cx", "memory");                              \
+        errno = -res;                                                  \
+        res;                                                           \
+    })
+
+#endif
+
+#ifdef ev_syscall
+  #define ev_syscall0(nr)                               ev_syscall (nr, 0,    0,    0,    0,    0,    0,   0)
+  #define ev_syscall1(nr,arg1)                          ev_syscall (nr, 1, arg1,    0,    0,    0,    0,   0)
+  #define ev_syscall2(nr,arg1,arg2)                     ev_syscall (nr, 2, arg1, arg2,    0,    0,    0,   0)
+  #define ev_syscall3(nr,arg1,arg2,arg3)                ev_syscall (nr, 3, arg1, arg2, arg3,    0,    0,   0)
+  #define ev_syscall4(nr,arg1,arg2,arg3,arg4)           ev_syscall (nr, 3, arg1, arg2, arg3, arg4,    0,   0)
+  #define ev_syscall5(nr,arg1,arg2,arg3,arg4,arg5)      ev_syscall (nr, 5, arg1, arg2, arg3, arg4, arg5,   0)
+  #define ev_syscall6(nr,arg1,arg2,arg3,arg4,arg5,arg6) ev_syscall (nr, 6, arg1, arg2, arg3, arg4, arg5,arg6)
+#else
+  #define ev_syscall0(nr)                               syscall (nr)
+  #define ev_syscall1(nr,arg1)                          syscall (nr, arg1)
+  #define ev_syscall2(nr,arg1,arg2)                     syscall (nr, arg1, arg2)
+  #define ev_syscall3(nr,arg1,arg2,arg3)                syscall (nr, arg1, arg2, arg3)
+  #define ev_syscall4(nr,arg1,arg2,arg3,arg4)           syscall (nr, arg1, arg2, arg3, arg4)
+  #define ev_syscall5(nr,arg1,arg2,arg3,arg4,arg5)      syscall (nr, arg1, arg2, arg3, arg4, arg5)
+  #define ev_syscall6(nr,arg1,arg2,arg3,arg4,arg5,arg6) syscall (nr, arg1, arg2, arg3, arg4, arg5,arg6)
+#endif
+
 #endif
 
+/*****************************************************************************/
+
 #define NUMPRI (EV_MAXPRI - EV_MINPRI + 1)
 
 #if EV_MINPRI == EV_MAXPRI
@@ -1621,7 +1904,7 @@ static EV_ATOMIC_T have_monotonic; /* did clock_gettime (CLOCK_MONOTONIC) work?
 #include <float.h>
 
 /* a floor() replacement function, should be independent of ev_tstamp type */
-noinline
+ecb_noinline
 static ev_tstamp
 ev_floor (ev_tstamp v)
 {
@@ -1632,26 +1915,26 @@ ev_floor (ev_tstamp v)
   const ev_tstamp shift = sizeof (unsigned long) >= 8 ? 18446744073709551616. : 4294967296.;
 #endif
 
-  /* argument too large for an unsigned long? */
-  if (expect_false (v >= shift))
+  /* special treatment for negative arguments */
+  if (ecb_expect_false (v < 0.))
+    {
+      ev_tstamp f = -ev_floor (-v);
+
+      return f - (f == v ? 0 : 1);
+    }
+
+  /* argument too large for an unsigned long? then reduce it */
+  if (ecb_expect_false (v >= shift))
     {
       ev_tstamp f;
 
       if (v == v - 1.)
-        return v; /* very large number */
+        return v; /* very large numbers are assumed to be integer */
 
       f = shift * ev_floor (v * (1. / shift));
       return f + ev_floor (v - f);
     }
 
-  /* special treatment for negative args? */
-  if (expect_false (v < 0.))
-    {
-      ev_tstamp f = -ev_floor (-v);
-
-      return f - (f == v ? 0 : 1);
-    }
-
   /* fits into an unsigned long */
   return (unsigned long)v;
 }
@@ -1664,7 +1947,7 @@ ev_floor (ev_tstamp v)
 # include <sys/utsname.h>
 #endif
 
-noinline ecb_cold
+ecb_noinline ecb_cold
 static unsigned int
 ev_linux_version (void)
 {
@@ -1704,7 +1987,7 @@ ev_linux_version (void)
 /*****************************************************************************/
 
 #if EV_AVOID_STDIO
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 ev_printerr (const char *msg)
 {
@@ -1721,7 +2004,7 @@ ev_set_syserr_cb (void (*cb)(const char *msg) EV_NOEXCEPT) EV_NOEXCEPT
   syserr_cb = cb;
 }
 
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 ev_syserr (const char *msg)
 {
@@ -1803,7 +2086,7 @@ typedef struct
   unsigned char events; /* the events watched for */
   unsigned char reify;  /* flag set when this ANFD needs reification (EV_ANFD_REIFY, EV__IOFDSET) */
   unsigned char emask;  /* some backends store the actual kernel mask in here */
-  unsigned char unused;
+  unsigned char eflags; /* flags field for use by backends */
 #if EV_USE_EPOLL
   unsigned int egen;    /* generation counter to counter epoll bugs */
 #endif
@@ -1867,7 +2150,7 @@ typedef struct
 
 #else
 
-  EV_API_DECL ev_tstamp ev_rt_now = 0; /* needs to be initialised to make it a definition despite extern */
+  EV_API_DECL ev_tstamp ev_rt_now = EV_TS_CONST (0.); /* needs to be initialised to make it a definition despite extern */
   #define VAR(name,decl) static decl;
     #include "ev_vars.h"
   #undef VAR
@@ -1877,8 +2160,8 @@ typedef struct
 #endif
 
 #if EV_FEATURE_API
-# define EV_RELEASE_CB if (expect_false (release_cb)) release_cb (EV_A)
-# define EV_ACQUIRE_CB if (expect_false (acquire_cb)) acquire_cb (EV_A)
+# define EV_RELEASE_CB if (ecb_expect_false (release_cb)) release_cb (EV_A)
+# define EV_ACQUIRE_CB if (ecb_expect_false (acquire_cb)) acquire_cb (EV_A)
 # define EV_INVOKE_PENDING invoke_cb (EV_A)
 #else
 # define EV_RELEASE_CB (void)0
@@ -1895,17 +2178,19 @@ ev_tstamp
 ev_time (void) EV_NOEXCEPT
 {
 #if EV_USE_REALTIME
-  if (expect_true (have_realtime))
+  if (ecb_expect_true (have_realtime))
     {
       struct timespec ts;
       clock_gettime (CLOCK_REALTIME, &ts);
-      return ts.tv_sec + ts.tv_nsec * 1e-9;
+      return EV_TS_GET (ts);
     }
 #endif
 
-  struct timeval tv;
-  gettimeofday (&tv, 0);
-  return tv.tv_sec + tv.tv_usec * 1e-6;
+  {
+    struct timeval tv;
+    gettimeofday (&tv, 0);
+    return EV_TV_GET (tv);
+  }
 }
 #endif
 
@@ -1913,11 +2198,11 @@ inline_size ev_tstamp
 get_clock (void)
 {
 #if EV_USE_MONOTONIC
-  if (expect_true (have_monotonic))
+  if (ecb_expect_true (have_monotonic))
     {
       struct timespec ts;
       clock_gettime (CLOCK_MONOTONIC, &ts);
-      return ts.tv_sec + ts.tv_nsec * 1e-9;
+      return EV_TS_GET (ts);
     }
 #endif
 
@@ -1935,7 +2220,7 @@ ev_now (EV_P) EV_NOEXCEPT
 void
 ev_sleep (ev_tstamp delay) EV_NOEXCEPT
 {
-  if (delay > 0.)
+  if (delay > EV_TS_CONST (0.))
     {
 #if EV_USE_NANOSLEEP
       struct timespec ts;
@@ -1945,7 +2230,7 @@ ev_sleep (ev_tstamp delay) EV_NOEXCEPT
 #elif defined _WIN32
       /* maybe this should round up, as ms is very low resolution */
       /* compared to select (µs) or nanosleep (ns) */
-      Sleep ((unsigned long)(delay * 1e3));
+      Sleep ((unsigned long)(EV_TS_TO_MSEC (delay)));
 #else
       struct timeval tv;
 
@@ -1985,7 +2270,7 @@ array_nextsize (int elem, int cur, int cnt)
   return ncur;
 }
 
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void *
 array_realloc (int elem, void *base, int *cur, int cnt)
 {
@@ -1999,7 +2284,7 @@ array_realloc (int elem, void *base, int *cur, int cnt)
   memset ((void *)(base + offset), 0, sizeof (*(base)) * (count))
 
 #define array_needsize(type,base,cur,cnt,init)			\
-  if (expect_false ((cnt) > (cur)))				\
+  if (ecb_expect_false ((cnt) > (cur)))				\
     {								\
       ecb_unused int ocur_ = (cur);				\
       (base) = (type *)array_realloc				\
@@ -2023,20 +2308,20 @@ array_realloc (int elem, void *base, int *cur, int cnt)
 /*****************************************************************************/
 
 /* dummy callback for pending events */
-noinline
+ecb_noinline
 static void
 pendingcb (EV_P_ ev_prepare *w, int revents)
 {
 }
 
-noinline
+ecb_noinline
 void
 ev_feed_event (EV_P_ void *w, int revents) EV_NOEXCEPT
 {
   W w_ = (W)w;
   int pri = ABSPRI (w_);
 
-  if (expect_false (w_->pending))
+  if (ecb_expect_false (w_->pending))
     pendings [pri][w_->pending - 1].events |= revents;
   else
     {
@@ -2097,7 +2382,7 @@ fd_event (EV_P_ int fd, int revents)
 {
   ANFD *anfd = anfds + fd;
 
-  if (expect_true (!anfd->reify))
+  if (ecb_expect_true (!anfd->reify))
     fd_event_nocheck (EV_A_ fd, revents);
 }
 
@@ -2115,8 +2400,20 @@ fd_reify (EV_P)
 {
   int i;
 
+  /* most backends do not modify the fdchanges list in backend_modfiy.
+   * except io_uring, which has fixed-size buffers which might force us
+   * to handle events in backend_modify, causing fdchanges to be amended,
+   * which could result in an endless loop.
+   * to avoid this, we do not dynamically handle fds that were added
+   * during fd_reify. that means that for those backends, fdchangecnt
+   * might be non-zero during poll, which must cause them to not block.
+   * to not put too much of a burden on other backends, this detail
+   * needs to be handled in the backend.
+   */
+  int changecnt = fdchangecnt;
+
 #if EV_SELECT_IS_WINSOCKET || EV_USE_IOCP
-  for (i = 0; i < fdchangecnt; ++i)
+  for (i = 0; i < changecnt; ++i)
     {
       int fd = fdchanges [i];
       ANFD *anfd = anfds + fd;
@@ -2140,7 +2437,7 @@ fd_reify (EV_P)
     }
 #endif
 
-  for (i = 0; i < fdchangecnt; ++i)
+  for (i = 0; i < changecnt; ++i)
     {
       int fd = fdchanges [i];
       ANFD *anfd = anfds + fd;
@@ -2151,7 +2448,7 @@ fd_reify (EV_P)
 
       anfd->reify = 0;
 
-      /*if (expect_true (o_reify & EV_ANFD_REIFY)) probably a deoptimisation */
+      /*if (ecb_expect_true (o_reify & EV_ANFD_REIFY)) probably a deoptimisation */
         {
           anfd->events = 0;
 
@@ -2166,7 +2463,14 @@ fd_reify (EV_P)
         backend_modify (EV_A_ fd, o_events, anfd->events);
     }
 
-  fdchangecnt = 0;
+  /* normally, fdchangecnt hasn't changed. if it has, then new fds have been added.
+   * this is a rare case (see beginning comment in this function), so we copy them to the
+   * front and hope the backend handles this case.
+   */
+  if (ecb_expect_false (fdchangecnt != changecnt))
+    memmove (fdchanges, fdchanges + changecnt, (fdchangecnt - changecnt) * sizeof (*fdchanges));
+
+  fdchangecnt -= changecnt;
 }
 
 /* something about the given fd changed */
@@ -2175,9 +2479,9 @@ void
 fd_change (EV_P_ int fd, int flags)
 {
   unsigned char reify = anfds [fd].reify;
-  anfds [fd].reify |= flags;
+  anfds [fd].reify = reify | flags;
 
-  if (expect_true (!reify))
+  if (ecb_expect_true (!reify))
     {
       ++fdchangecnt;
       array_needsize (int, fdchanges, fdchangemax, fdchangecnt, array_needsize_noinit);
@@ -2210,7 +2514,7 @@ fd_valid (int fd)
 }
 
 /* called on EBADF to verify fds */
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 fd_ebadf (EV_P)
 {
@@ -2223,7 +2527,7 @@ fd_ebadf (EV_P)
 }
 
 /* called on ENOMEM in select/poll to kill some fds and retry */
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 fd_enomem (EV_P)
 {
@@ -2238,7 +2542,7 @@ fd_enomem (EV_P)
 }
 
 /* usually called after fork if backend needs to re-arm all fds from scratch */
-noinline
+ecb_noinline
 static void
 fd_rearm_all (EV_P)
 {
@@ -2302,19 +2606,19 @@ downheap (ANHE *heap, int N, int k)
       ANHE *pos = heap + DHEAP * (k - HEAP0) + HEAP0 + 1;
 
       /* find minimum child */
-      if (expect_true (pos + DHEAP - 1 < E))
+      if (ecb_expect_true (pos + DHEAP - 1 < E))
         {
           /* fast path */                               (minpos = pos + 0), (minat = ANHE_at (*minpos));
-          if (               ANHE_at (pos [1]) < minat) (minpos = pos + 1), (minat = ANHE_at (*minpos));
-          if (               ANHE_at (pos [2]) < minat) (minpos = pos + 2), (minat = ANHE_at (*minpos));
-          if (               ANHE_at (pos [3]) < minat) (minpos = pos + 3), (minat = ANHE_at (*minpos));
+          if (               minat > ANHE_at (pos [1])) (minpos = pos + 1), (minat = ANHE_at (*minpos));
+          if (               minat > ANHE_at (pos [2])) (minpos = pos + 2), (minat = ANHE_at (*minpos));
+          if (               minat > ANHE_at (pos [3])) (minpos = pos + 3), (minat = ANHE_at (*minpos));
         }
       else if (pos < E)
         {
           /* slow path */                               (minpos = pos + 0), (minat = ANHE_at (*minpos));
-          if (pos + 1 < E && ANHE_at (pos [1]) < minat) (minpos = pos + 1), (minat = ANHE_at (*minpos));
-          if (pos + 2 < E && ANHE_at (pos [2]) < minat) (minpos = pos + 2), (minat = ANHE_at (*minpos));
-          if (pos + 3 < E && ANHE_at (pos [3]) < minat) (minpos = pos + 3), (minat = ANHE_at (*minpos));
+          if (pos + 1 < E && minat > ANHE_at (pos [1])) (minpos = pos + 1), (minat = ANHE_at (*minpos));
+          if (pos + 2 < E && minat > ANHE_at (pos [2])) (minpos = pos + 2), (minat = ANHE_at (*minpos));
+          if (pos + 3 < E && minat > ANHE_at (pos [3])) (minpos = pos + 3), (minat = ANHE_at (*minpos));
         }
       else
         break;
@@ -2332,7 +2636,7 @@ downheap (ANHE *heap, int N, int k)
   ev_active (ANHE_w (he)) = k;
 }
 
-#else /* 4HEAP */
+#else /* not 4HEAP */
 
 #define HEAP0 1
 #define HPARENT(k) ((k) >> 1)
@@ -2414,7 +2718,7 @@ reheap (ANHE *heap, int N)
 
 /*****************************************************************************/
 
-/* associate signal watchers to a signal signal */
+/* associate signal watchers to a signal */
 typedef struct
 {
   EV_ATOMIC_T pending;
@@ -2430,7 +2734,7 @@ static ANSIG signals [EV_NSIG - 1];
 
 #if EV_SIGNAL_ENABLE || EV_ASYNC_ENABLE
 
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 evpipe_init (EV_P)
 {
@@ -2481,7 +2785,7 @@ evpipe_write (EV_P_ EV_ATOMIC_T *flag)
 {
   ECB_MEMORY_FENCE; /* push out the write before this function was called, acquire flag */
 
-  if (expect_true (*flag))
+  if (ecb_expect_true (*flag))
     return;
 
   *flag = 1;
@@ -2568,7 +2872,7 @@ pipecb (EV_P_ ev_io *iow, int revents)
       ECB_MEMORY_FENCE;
 
       for (i = EV_NSIG - 1; i--; )
-        if (expect_false (signals [i].pending))
+        if (ecb_expect_false (signals [i].pending))
           ev_feed_signal_event (EV_A_ i + 1);
     }
 #endif
@@ -2619,13 +2923,13 @@ ev_sighandler (int signum)
   ev_feed_signal (signum);
 }
 
-noinline
+ecb_noinline
 void
 ev_feed_signal_event (EV_P_ int signum) EV_NOEXCEPT
 {
   WL w;
 
-  if (expect_false (signum <= 0 || signum >= EV_NSIG))
+  if (ecb_expect_false (signum <= 0 || signum >= EV_NSIG))
     return;
 
   --signum;
@@ -2634,7 +2938,7 @@ ev_feed_signal_event (EV_P_ int signum) EV_NOEXCEPT
   /* it is permissible to try to feed a signal to the wrong loop */
   /* or, likely more useful, feeding a signal nobody is waiting for */
 
-  if (expect_false (signals [signum].loop != EV_A))
+  if (ecb_expect_false (signals [signum].loop != EV_A))
     return;
 #endif
 
@@ -2728,6 +3032,57 @@ childcb (EV_P_ ev_signal *sw, int revents)
 
 /*****************************************************************************/
 
+#if EV_USE_TIMERFD
+
+static void periodics_reschedule (EV_P);
+
+static void
+timerfdcb (EV_P_ ev_io *iow, int revents)
+{
+  struct itimerspec its = { 0 };
+
+  its.it_value.tv_sec = ev_rt_now + (int)MAX_BLOCKTIME2;
+  timerfd_settime (timerfd, TFD_TIMER_ABSTIME | TFD_TIMER_CANCEL_ON_SET, &its, 0);
+
+  ev_rt_now = ev_time ();
+  /* periodics_reschedule only needs ev_rt_now */
+  /* but maybe in the future we want the full treatment. */
+  /*
+  now_floor = EV_TS_CONST (0.);
+  time_update (EV_A_ EV_TSTAMP_HUGE);
+  */
+#if EV_PERIODIC_ENABLE
+  periodics_reschedule (EV_A);
+#endif
+}
+
+ecb_noinline ecb_cold
+static void
+evtimerfd_init (EV_P)
+{
+  if (!ev_is_active (&timerfd_w))
+    {
+      timerfd = timerfd_create (CLOCK_REALTIME, TFD_NONBLOCK | TFD_CLOEXEC);
+
+      if (timerfd >= 0)
+        {
+          fd_intern (timerfd); /* just to be sure */
+
+          ev_io_init (&timerfd_w, timerfdcb, timerfd, EV_READ);
+          ev_set_priority (&timerfd_w, EV_MINPRI);
+          ev_io_start (EV_A_ &timerfd_w);
+          ev_unref (EV_A); /* watcher should not keep loop alive */
+
+          /* (re-) arm timer */
+          timerfdcb (EV_A_ 0, 0);
+        }
+    }
+}
+
+#endif
+
+/*****************************************************************************/
+
 #if EV_USE_IOCP
 # include "ev_iocp.c"
 #endif
@@ -2743,6 +3098,9 @@ childcb (EV_P_ ev_signal *sw, int revents)
 #if EV_USE_LINUXAIO
 # include "ev_linuxaio.c"
 #endif
+#if EV_USE_IOURING
+# include "ev_iouring.c"
+#endif
 #if EV_USE_POLL
 # include "ev_poll.c"
 #endif
@@ -2780,13 +3138,14 @@ ev_supported_backends (void) EV_NOEXCEPT
 {
   unsigned int flags = 0;
 
-  if (EV_USE_PORT    ) flags |= EVBACKEND_PORT;
-  if (EV_USE_KQUEUE  ) flags |= EVBACKEND_KQUEUE;
-  if (EV_USE_EPOLL   ) flags |= EVBACKEND_EPOLL;
-  if (EV_USE_LINUXAIO) flags |= EVBACKEND_LINUXAIO;
-  if (EV_USE_POLL    ) flags |= EVBACKEND_POLL;
-  if (EV_USE_SELECT  ) flags |= EVBACKEND_SELECT;
-  
+  if (EV_USE_PORT                                      ) flags |= EVBACKEND_PORT;
+  if (EV_USE_KQUEUE                                    ) flags |= EVBACKEND_KQUEUE;
+  if (EV_USE_EPOLL                                     ) flags |= EVBACKEND_EPOLL;
+  if (EV_USE_LINUXAIO                                  ) flags |= EVBACKEND_LINUXAIO;
+  if (EV_USE_IOURING && ev_linux_version () >= 0x050601) flags |= EVBACKEND_IOURING; /* 5.6.1+ */
+  if (EV_USE_POLL                                      ) flags |= EVBACKEND_POLL;
+  if (EV_USE_SELECT                                    ) flags |= EVBACKEND_SELECT;
+
   return flags;
 }
 
@@ -2814,6 +3173,10 @@ ev_recommended_backends (void) EV_NOEXCEPT
 #if !EV_RECOMMEND_LINUXAIO
   flags &= ~EVBACKEND_LINUXAIO;
 #endif
+  /* TODO: linuxaio is super experimental */
+#if !EV_RECOMMEND_IOURING
+  flags &= ~EVBACKEND_IOURING;
+#endif
 
   return flags;
 }
@@ -2822,12 +3185,14 @@ ecb_cold
 unsigned int
 ev_embeddable_backends (void) EV_NOEXCEPT
 {
-  int flags = EVBACKEND_EPOLL | EVBACKEND_KQUEUE | EVBACKEND_PORT;
+  int flags = EVBACKEND_EPOLL | EVBACKEND_KQUEUE | EVBACKEND_PORT | EVBACKEND_IOURING;
 
   /* epoll embeddability broken on all linux versions up to at least 2.6.23 */
   if (ev_linux_version () < 0x020620) /* disable it on linux < 2.6.32 */
     flags &= ~EVBACKEND_EPOLL;
 
+  /* EVBACKEND_LINUXAIO is theoretically embeddable, but suffers from a performance overhead */
+
   return flags;
 }
 
@@ -2889,7 +3254,7 @@ ev_set_loop_release_cb (EV_P_ void (*release)(EV_P) EV_NOEXCEPT, void (*acquire)
 #endif
 
 /* initialise a loop structure, must be zero-initialised */
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 loop_init (EV_P_ unsigned int flags) EV_NOEXCEPT
 {
@@ -2954,6 +3319,9 @@ loop_init (EV_P_ unsigned int flags) EV_NOEXCEPT
 #if EV_USE_SIGNALFD
       sigfd              = flags & EVFLAG_SIGNALFD  ? -2 : -1;
 #endif
+#if EV_USE_TIMERFD
+      timerfd            = flags & EVFLAG_NOTIMERFD ? -1 : -2;
+#endif
 
       if (!(flags & EVBACKEND_MASK))
         flags |= ev_recommended_backends ();
@@ -2967,6 +3335,9 @@ loop_init (EV_P_ unsigned int flags) EV_NOEXCEPT
 #if EV_USE_KQUEUE
       if (!backend && (flags & EVBACKEND_KQUEUE  )) backend = kqueue_init    (EV_A_ flags);
 #endif
+#if EV_USE_IOURING
+      if (!backend && (flags & EVBACKEND_IOURING )) backend = iouring_init   (EV_A_ flags);
+#endif
 #if EV_USE_LINUXAIO
       if (!backend && (flags & EVBACKEND_LINUXAIO)) backend = linuxaio_init  (EV_A_ flags);
 #endif
@@ -3004,7 +3375,7 @@ ev_loop_destroy (EV_P)
 
 #if EV_CLEANUP_ENABLE
   /* queue cleanup watchers (and execute them) */
-  if (expect_false (cleanupcnt))
+  if (ecb_expect_false (cleanupcnt))
     {
       queue_events (EV_A_ (W *)cleanups, cleanupcnt, EV_CLEANUP);
       EV_INVOKE_PENDING;
@@ -3033,6 +3404,11 @@ ev_loop_destroy (EV_P)
     close (sigfd);
 #endif
 
+#if EV_USE_TIMERFD
+  if (ev_is_active (&timerfd_w))
+    close (timerfd);
+#endif
+
 #if EV_USE_INOTIFY
   if (fs_fd >= 0)
     close (fs_fd);
@@ -3050,6 +3426,9 @@ ev_loop_destroy (EV_P)
 #if EV_USE_KQUEUE
   if (backend == EVBACKEND_KQUEUE  ) kqueue_destroy   (EV_A);
 #endif
+#if EV_USE_IOURING
+  if (backend == EVBACKEND_IOURING ) iouring_destroy  (EV_A);
+#endif
 #if EV_USE_LINUXAIO
   if (backend == EVBACKEND_LINUXAIO) linuxaio_destroy (EV_A);
 #endif
@@ -3117,6 +3496,9 @@ loop_fork (EV_P)
 #if EV_USE_KQUEUE
   if (backend == EVBACKEND_KQUEUE  ) kqueue_fork   (EV_A);
 #endif
+#if EV_USE_IOURING
+  if (backend == EVBACKEND_IOURING ) iouring_fork  (EV_A);
+#endif
 #if EV_USE_LINUXAIO
   if (backend == EVBACKEND_LINUXAIO) linuxaio_fork (EV_A);
 #endif
@@ -3127,22 +3509,44 @@ loop_fork (EV_P)
   infy_fork (EV_A);
 #endif
 
-#if EV_SIGNAL_ENABLE || EV_ASYNC_ENABLE
-  if (ev_is_active (&pipe_w) && postfork != 2)
+  if (postfork != 2)
     {
-      /* pipe_write_wanted must be false now, so modifying fd vars should be safe */
-
-      ev_ref (EV_A);
-      ev_io_stop (EV_A_ &pipe_w);
-
-      if (evpipe [0] >= 0)
-        EV_WIN32_CLOSE_FD (evpipe [0]);
+      #if EV_USE_SIGNALFD
+        /* surprisingly, nothing needs to be done for signalfd, accoridng to docs, it does the right thing on fork */
+      #endif
+      
+      #if EV_USE_TIMERFD
+        if (ev_is_active (&timerfd_w))
+          {
+            ev_ref (EV_A);
+            ev_io_stop (EV_A_ &timerfd_w);
 
-      evpipe_init (EV_A);
-      /* iterate over everything, in case we missed something before */
-      ev_feed_event (EV_A_ &pipe_w, EV_CUSTOM);
+            close (timerfd);
+            timerfd = -2;
+      
+            evtimerfd_init (EV_A);
+            /* reschedule periodics, in case we missed something */
+            ev_feed_event (EV_A_ &timerfd_w, EV_CUSTOM);
+          }
+      #endif
+      
+      #if EV_SIGNAL_ENABLE || EV_ASYNC_ENABLE
+        if (ev_is_active (&pipe_w))
+          {
+            /* pipe_write_wanted must be false now, so modifying fd vars should be safe */
+      
+            ev_ref (EV_A);
+            ev_io_stop (EV_A_ &pipe_w);
+      
+            if (evpipe [0] >= 0)
+              EV_WIN32_CLOSE_FD (evpipe [0]);
+      
+            evpipe_init (EV_A);
+            /* iterate over everything, in case we missed something before */
+            ev_feed_event (EV_A_ &pipe_w, EV_CUSTOM);
+          }
+      #endif
     }
-#endif
 
   postfork = 0;
 }
@@ -3168,7 +3572,7 @@ ev_loop_new (unsigned int flags) EV_NOEXCEPT
 #endif /* multiplicity */
 
 #if EV_VERIFY
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 verify_watcher (EV_P_ W w)
 {
@@ -3178,7 +3582,7 @@ verify_watcher (EV_P_ W w)
     assert (("libev: pending watcher not on pending queue", pendings [ABSPRI (w)][w->pending - 1].w == w));
 }
 
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 verify_heap (EV_P_ ANHE *heap, int N)
 {
@@ -3194,7 +3598,7 @@ verify_heap (EV_P_ ANHE *heap, int N)
     }
 }
 
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 array_verify (EV_P_ W *ws, int cnt)
 {
@@ -3353,7 +3757,7 @@ ev_pending_count (EV_P) EV_NOEXCEPT
   return count;
 }
 
-noinline
+ecb_noinline
 void
 ev_invoke_pending (EV_P)
 {
@@ -3382,7 +3786,7 @@ ev_invoke_pending (EV_P)
 inline_size void
 idle_reify (EV_P)
 {
-  if (expect_false (idleall))
+  if (ecb_expect_false (idleall))
     {
       int pri;
 
@@ -3422,7 +3826,7 @@ timers_reify (EV_P)
               if (ev_at (w) < mn_now)
                 ev_at (w) = mn_now;
 
-              assert (("libev: negative ev_timer repeat value found while processing timers", w->repeat > 0.));
+              assert (("libev: negative ev_timer repeat value found while processing timers", w->repeat > EV_TS_CONST (0.)));
 
               ANHE_at_cache (timers [HEAP0]);
               downheap (timers, timercnt, HEAP0);
@@ -3441,7 +3845,7 @@ timers_reify (EV_P)
 
 #if EV_PERIODIC_ENABLE
 
-noinline
+ecb_noinline
 static void
 periodic_recalc (EV_P_ ev_periodic *w)
 {
@@ -3454,7 +3858,7 @@ periodic_recalc (EV_P_ ev_periodic *w)
       ev_tstamp nat = at + w->interval;
 
       /* when resolution fails us, we use ev_rt_now */
-      if (expect_false (nat == at))
+      if (ecb_expect_false (nat == at))
         {
           at = ev_rt_now;
           break;
@@ -3510,7 +3914,7 @@ periodics_reify (EV_P)
 
 /* simply recalculate all periodics */
 /* TODO: maybe ensure that at least one event happens when jumping forward? */
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 periodics_reschedule (EV_P)
 {
@@ -3534,7 +3938,7 @@ periodics_reschedule (EV_P)
 #endif
 
 /* adjust all timers by a given offset */
-noinline ecb_cold
+ecb_noinline ecb_cold
 static void
 timers_reschedule (EV_P_ ev_tstamp adjust)
 {
@@ -3554,7 +3958,7 @@ inline_speed void
 time_update (EV_P_ ev_tstamp max_block)
 {
 #if EV_USE_MONOTONIC
-  if (expect_true (have_monotonic))
+  if (ecb_expect_true (have_monotonic))
     {
       int i;
       ev_tstamp odiff = rtmn_diff;
@@ -3563,7 +3967,7 @@ time_update (EV_P_ ev_tstamp max_block)
 
       /* only fetch the realtime clock every 0.5*MIN_TIMEJUMP seconds */
       /* interpolate in the meantime */
-      if (expect_true (mn_now - now_floor < MIN_TIMEJUMP * .5))
+      if (ecb_expect_true (mn_now - now_floor < EV_TS_CONST (MIN_TIMEJUMP * .5)))
         {
           ev_rt_now = rtmn_diff + mn_now;
           return;
@@ -3587,7 +3991,7 @@ time_update (EV_P_ ev_tstamp max_block)
 
           diff = odiff - rtmn_diff;
 
-          if (expect_true ((diff < 0. ? -diff : diff) < MIN_TIMEJUMP))
+          if (ecb_expect_true ((diff < EV_TS_CONST (0.) ? -diff : diff) < EV_TS_CONST (MIN_TIMEJUMP)))
             return; /* all is well */
 
           ev_rt_now = ev_time ();
@@ -3606,7 +4010,7 @@ time_update (EV_P_ ev_tstamp max_block)
     {
       ev_rt_now = ev_time ();
 
-      if (expect_false (mn_now > ev_rt_now || ev_rt_now > mn_now + max_block + MIN_TIMEJUMP))
+      if (ecb_expect_false (mn_now > ev_rt_now || ev_rt_now > mn_now + max_block + EV_TS_CONST (MIN_TIMEJUMP)))
         {
           /* adjust timers. this is easy, as the offset is the same for all of them */
           timers_reschedule (EV_A_ ev_rt_now - mn_now);
@@ -3639,8 +4043,8 @@ ev_run (EV_P_ int flags)
 #endif
 
 #ifndef _WIN32
-      if (expect_false (curpid)) /* penalise the forking check even more */
-        if (expect_false (getpid () != curpid))
+      if (ecb_expect_false (curpid)) /* penalise the forking check even more */
+        if (ecb_expect_false (getpid () != curpid))
           {
             curpid = getpid ();
             postfork = 1;
@@ -3649,7 +4053,7 @@ ev_run (EV_P_ int flags)
 
 #if EV_FORK_ENABLE
       /* we might have forked, so queue fork handlers */
-      if (expect_false (postfork))
+      if (ecb_expect_false (postfork))
         if (forkcnt)
           {
             queue_events (EV_A_ (W *)forks, forkcnt, EV_FORK);
@@ -3659,18 +4063,18 @@ ev_run (EV_P_ int flags)
 
 #if EV_PREPARE_ENABLE
       /* queue prepare watchers (and execute them) */
-      if (expect_false (preparecnt))
+      if (ecb_expect_false (preparecnt))
         {
           queue_events (EV_A_ (W *)prepares, preparecnt, EV_PREPARE);
           EV_INVOKE_PENDING;
         }
 #endif
 
-      if (expect_false (loop_done))
+      if (ecb_expect_false (loop_done))
         break;
 
       /* we might have forked, so reify kernel state if necessary */
-      if (expect_false (postfork))
+      if (ecb_expect_false (postfork))
         loop_fork (EV_A);
 
       /* update fd-related kernel structures */
@@ -3685,16 +4089,28 @@ ev_run (EV_P_ int flags)
         ev_tstamp prev_mn_now = mn_now;
 
         /* update time to cancel out callback processing overhead */
-        time_update (EV_A_ 1e100);
+        time_update (EV_A_ EV_TS_CONST (EV_TSTAMP_HUGE));
 
         /* from now on, we want a pipe-wake-up */
         pipe_write_wanted = 1;
 
         ECB_MEMORY_FENCE; /* make sure pipe_write_wanted is visible before we check for potential skips */
 
-        if (expect_true (!(flags & EVRUN_NOWAIT || idleall || !activecnt || pipe_write_skipped)))
+        if (ecb_expect_true (!(flags & EVRUN_NOWAIT || idleall || !activecnt || pipe_write_skipped)))
           {
-            waittime = MAX_BLOCKTIME;
+            waittime = EV_TS_CONST (MAX_BLOCKTIME);
+
+#if EV_USE_TIMERFD
+            /* sleep a lot longer when we can reliably detect timejumps */
+            if (ecb_expect_true (timerfd >= 0))
+              waittime = EV_TS_CONST (MAX_BLOCKTIME2);
+#endif
+#if !EV_PERIODIC_ENABLE
+            /* without periodics but with monotonic clock there is no need */
+            /* for any time jump detection, so sleep longer */
+            if (ecb_expect_true (have_monotonic))
+              waittime = EV_TS_CONST (MAX_BLOCKTIME2);
+#endif
 
             if (timercnt)
               {
@@ -3711,23 +4127,28 @@ ev_run (EV_P_ int flags)
 #endif
 
             /* don't let timeouts decrease the waittime below timeout_blocktime */
-            if (expect_false (waittime < timeout_blocktime))
+            if (ecb_expect_false (waittime < timeout_blocktime))
               waittime = timeout_blocktime;
 
-            /* at this point, we NEED to wait, so we have to ensure */
-            /* to pass a minimum nonzero value to the backend */
-            if (expect_false (waittime < backend_mintime))
-              waittime = backend_mintime;
+            /* now there are two more special cases left, either we have
+             * already-expired timers, so we should not sleep, or we have timers
+             * that expire very soon, in which case we need to wait for a minimum
+             * amount of time for some event loop backends.
+             */
+            if (ecb_expect_false (waittime < backend_mintime))
+              waittime = waittime <= EV_TS_CONST (0.)
+                 ? EV_TS_CONST (0.)
+                 : backend_mintime;
 
             /* extra check because io_blocktime is commonly 0 */
-            if (expect_false (io_blocktime))
+            if (ecb_expect_false (io_blocktime))
               {
                 sleeptime = io_blocktime - (mn_now - prev_mn_now);
 
                 if (sleeptime > waittime - backend_mintime)
                   sleeptime = waittime - backend_mintime;
 
-                if (expect_true (sleeptime > 0.))
+                if (ecb_expect_true (sleeptime > EV_TS_CONST (0.)))
                   {
                     ev_sleep (sleeptime);
                     waittime -= sleeptime;
@@ -3768,13 +4189,13 @@ ev_run (EV_P_ int flags)
 
 #if EV_CHECK_ENABLE
       /* queue check watchers, to be executed first */
-      if (expect_false (checkcnt))
+      if (ecb_expect_false (checkcnt))
         queue_events (EV_A_ (W *)checks, checkcnt, EV_CHECK);
 #endif
 
       EV_INVOKE_PENDING;
     }
-  while (expect_true (
+  while (ecb_expect_true (
     activecnt
     && !loop_done
     && !(flags & (EVRUN_ONCE | EVRUN_NOWAIT))
@@ -3811,7 +4232,7 @@ ev_unref (EV_P) EV_NOEXCEPT
 void
 ev_now_update (EV_P) EV_NOEXCEPT
 {
-  time_update (EV_A_ 1e100);
+  time_update (EV_A_ EV_TSTAMP_HUGE);
 }
 
 void
@@ -3848,7 +4269,7 @@ wlist_del (WL *head, WL elem)
 {
   while (*head)
     {
-      if (expect_true (*head == elem))
+      if (ecb_expect_true (*head == elem))
         {
           *head = elem->next;
           break;
@@ -3875,7 +4296,7 @@ ev_clear_pending (EV_P_ void *w) EV_NOEXCEPT
   W w_ = (W)w;
   int pending = w_->pending;
 
-  if (expect_true (pending))
+  if (ecb_expect_true (pending))
     {
       ANPENDING *p = pendings [ABSPRI (w_)] + pending - 1;
       p->w = (W)&pending_w;
@@ -3912,13 +4333,13 @@ ev_stop (EV_P_ W w)
 
 /*****************************************************************************/
 
-noinline
+ecb_noinline
 void
 ev_io_start (EV_P_ ev_io *w) EV_NOEXCEPT
 {
   int fd = w->fd;
 
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   assert (("libev: ev_io_start called with negative fd", fd >= 0));
@@ -3942,12 +4363,12 @@ ev_io_start (EV_P_ ev_io *w) EV_NOEXCEPT
   EV_FREQUENT_CHECK;
 }
 
-noinline
+ecb_noinline
 void
 ev_io_stop (EV_P_ ev_io *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   assert (("libev: ev_io_stop called with illegal fd (must stay constant after start!)", w->fd >= 0 && w->fd < anfdmax));
@@ -3965,11 +4386,11 @@ ev_io_stop (EV_P_ ev_io *w) EV_NOEXCEPT
   EV_FREQUENT_CHECK;
 }
 
-noinline
+ecb_noinline
 void
 ev_timer_start (EV_P_ ev_timer *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   ev_at (w) += mn_now;
@@ -3990,12 +4411,12 @@ ev_timer_start (EV_P_ ev_timer *w) EV_NOEXCEPT
   /*assert (("libev: internal timer heap corruption", timers [ev_active (w)] == (WT)w));*/
 }
 
-noinline
+ecb_noinline
 void
 ev_timer_stop (EV_P_ ev_timer *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4007,7 +4428,7 @@ ev_timer_stop (EV_P_ ev_timer *w) EV_NOEXCEPT
 
     --timercnt;
 
-    if (expect_true (active < timercnt + HEAP0))
+    if (ecb_expect_true (active < timercnt + HEAP0))
       {
         timers [active] = timers [timercnt + HEAP0];
         adjustheap (timers, timercnt, active);
@@ -4021,7 +4442,7 @@ ev_timer_stop (EV_P_ ev_timer *w) EV_NOEXCEPT
   EV_FREQUENT_CHECK;
 }
 
-noinline
+ecb_noinline
 void
 ev_timer_again (EV_P_ ev_timer *w) EV_NOEXCEPT
 {
@@ -4052,17 +4473,22 @@ ev_timer_again (EV_P_ ev_timer *w) EV_NOEXCEPT
 ev_tstamp
 ev_timer_remaining (EV_P_ ev_timer *w) EV_NOEXCEPT
 {
-  return ev_at (w) - (ev_is_active (w) ? mn_now : 0.);
+  return ev_at (w) - (ev_is_active (w) ? mn_now : EV_TS_CONST (0.));
 }
 
 #if EV_PERIODIC_ENABLE
-noinline
+ecb_noinline
 void
 ev_periodic_start (EV_P_ ev_periodic *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
+#if EV_USE_TIMERFD
+  if (timerfd == -2)
+    evtimerfd_init (EV_A);
+#endif
+
   if (w->reschedule_cb)
     ev_at (w) = w->reschedule_cb (w, ev_rt_now);
   else if (w->interval)
@@ -4087,12 +4513,12 @@ ev_periodic_start (EV_P_ ev_periodic *w) EV_NOEXCEPT
   /*assert (("libev: internal periodic heap corruption", ANHE_w (periodics [ev_active (w)]) == (WT)w));*/
 }
 
-noinline
+ecb_noinline
 void
 ev_periodic_stop (EV_P_ ev_periodic *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4104,7 +4530,7 @@ ev_periodic_stop (EV_P_ ev_periodic *w) EV_NOEXCEPT
 
     --periodiccnt;
 
-    if (expect_true (active < periodiccnt + HEAP0))
+    if (ecb_expect_true (active < periodiccnt + HEAP0))
       {
         periodics [active] = periodics [periodiccnt + HEAP0];
         adjustheap (periodics, periodiccnt, active);
@@ -4116,7 +4542,7 @@ ev_periodic_stop (EV_P_ ev_periodic *w) EV_NOEXCEPT
   EV_FREQUENT_CHECK;
 }
 
-noinline
+ecb_noinline
 void
 ev_periodic_again (EV_P_ ev_periodic *w) EV_NOEXCEPT
 {
@@ -4132,11 +4558,11 @@ ev_periodic_again (EV_P_ ev_periodic *w) EV_NOEXCEPT
 
 #if EV_SIGNAL_ENABLE
 
-noinline
+ecb_noinline
 void
 ev_signal_start (EV_P_ ev_signal *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   assert (("libev: ev_signal_start called with illegal signal number", w->signum > 0 && w->signum < EV_NSIG));
@@ -4215,12 +4641,12 @@ ev_signal_start (EV_P_ ev_signal *w) EV_NOEXCEPT
   EV_FREQUENT_CHECK;
 }
 
-noinline
+ecb_noinline
 void
 ev_signal_stop (EV_P_ ev_signal *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4263,7 +4689,7 @@ ev_child_start (EV_P_ ev_child *w) EV_NOEXCEPT
 #if EV_MULTIPLICITY
   assert (("libev: child watchers are only supported in the default loop", loop == ev_default_loop_ptr));
 #endif
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4278,7 +4704,7 @@ void
 ev_child_stop (EV_P_ ev_child *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4302,14 +4728,14 @@ ev_child_stop (EV_P_ ev_child *w) EV_NOEXCEPT
 #define NFS_STAT_INTERVAL 30.1074891 /* for filesystems potentially failing inotify */
 #define MIN_STAT_INTERVAL  0.1074891
 
-noinline static void stat_timer_cb (EV_P_ ev_timer *w_, int revents);
+ecb_noinline static void stat_timer_cb (EV_P_ ev_timer *w_, int revents);
 
 #if EV_USE_INOTIFY
 
 /* the * 2 is to allow for alignment padding, which for some reason is >> 8 */
 # define EV_INOTIFY_BUFSIZE (sizeof (struct inotify_event) * 2 + NAME_MAX)
 
-noinline
+ecb_noinline
 static void
 infy_add (EV_P_ ev_stat *w)
 {
@@ -4384,7 +4810,7 @@ infy_add (EV_P_ ev_stat *w)
   if (ev_is_active (&w->timer)) ev_unref (EV_A);
 }
 
-noinline
+ecb_noinline
 static void
 infy_del (EV_P_ ev_stat *w)
 {
@@ -4402,7 +4828,7 @@ infy_del (EV_P_ ev_stat *w)
   inotify_rm_watch (fs_fd, wd);
 }
 
-noinline
+ecb_noinline
 static void
 infy_wd (EV_P_ int slot, int wd, struct inotify_event *ev)
 {
@@ -4558,7 +4984,7 @@ ev_stat_stat (EV_P_ ev_stat *w) EV_NOEXCEPT
     w->attr.st_nlink = 1;
 }
 
-noinline
+ecb_noinline
 static void
 stat_timer_cb (EV_P_ ev_timer *w_, int revents)
 {
@@ -4602,7 +5028,7 @@ stat_timer_cb (EV_P_ ev_timer *w_, int revents)
 void
 ev_stat_start (EV_P_ ev_stat *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   ev_stat_stat (EV_A_ w);
@@ -4634,7 +5060,7 @@ void
 ev_stat_stop (EV_P_ ev_stat *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4659,7 +5085,7 @@ ev_stat_stop (EV_P_ ev_stat *w) EV_NOEXCEPT
 void
 ev_idle_start (EV_P_ ev_idle *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   pri_adjust (EV_A_ (W)w);
@@ -4683,7 +5109,7 @@ void
 ev_idle_stop (EV_P_ ev_idle *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4706,7 +5132,7 @@ ev_idle_stop (EV_P_ ev_idle *w) EV_NOEXCEPT
 void
 ev_prepare_start (EV_P_ ev_prepare *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4722,7 +5148,7 @@ void
 ev_prepare_stop (EV_P_ ev_prepare *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4744,7 +5170,7 @@ ev_prepare_stop (EV_P_ ev_prepare *w) EV_NOEXCEPT
 void
 ev_check_start (EV_P_ ev_check *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4760,7 +5186,7 @@ void
 ev_check_stop (EV_P_ ev_check *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4779,7 +5205,7 @@ ev_check_stop (EV_P_ ev_check *w) EV_NOEXCEPT
 #endif
 
 #if EV_EMBED_ENABLE
-noinline
+ecb_noinline
 void
 ev_embed_sweep (EV_P_ ev_embed *w) EV_NOEXCEPT
 {
@@ -4813,6 +5239,7 @@ embed_prepare_cb (EV_P_ ev_prepare *prepare, int revents)
   }
 }
 
+#if EV_FORK_ENABLE
 static void
 embed_fork_cb (EV_P_ ev_fork *fork_w, int revents)
 {
@@ -4829,6 +5256,7 @@ embed_fork_cb (EV_P_ ev_fork *fork_w, int revents)
 
   ev_embed_start (EV_A_ w);
 }
+#endif
 
 #if 0
 static void
@@ -4841,7 +5269,7 @@ embed_idle_cb (EV_P_ ev_idle *idle, int revents)
 void
 ev_embed_start (EV_P_ ev_embed *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   {
@@ -4859,8 +5287,10 @@ ev_embed_start (EV_P_ ev_embed *w) EV_NOEXCEPT
   ev_set_priority (&w->prepare, EV_MINPRI);
   ev_prepare_start (EV_A_ &w->prepare);
 
+#if EV_FORK_ENABLE
   ev_fork_init (&w->fork, embed_fork_cb);
   ev_fork_start (EV_A_ &w->fork);
+#endif
 
   /*ev_idle_init (&w->idle, e,bed_idle_cb);*/
 
@@ -4873,14 +5303,16 @@ void
 ev_embed_stop (EV_P_ ev_embed *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
 
   ev_io_stop      (EV_A_ &w->io);
   ev_prepare_stop (EV_A_ &w->prepare);
+#if EV_FORK_ENABLE
   ev_fork_stop    (EV_A_ &w->fork);
+#endif
 
   ev_stop (EV_A_ (W)w);
 
@@ -4892,7 +5324,7 @@ ev_embed_stop (EV_P_ ev_embed *w) EV_NOEXCEPT
 void
 ev_fork_start (EV_P_ ev_fork *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4908,7 +5340,7 @@ void
 ev_fork_stop (EV_P_ ev_fork *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4930,7 +5362,7 @@ ev_fork_stop (EV_P_ ev_fork *w) EV_NOEXCEPT
 void
 ev_cleanup_start (EV_P_ ev_cleanup *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4948,7 +5380,7 @@ void
 ev_cleanup_stop (EV_P_ ev_cleanup *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
@@ -4971,7 +5403,7 @@ ev_cleanup_stop (EV_P_ ev_cleanup *w) EV_NOEXCEPT
 void
 ev_async_start (EV_P_ ev_async *w) EV_NOEXCEPT
 {
-  if (expect_false (ev_is_active (w)))
+  if (ecb_expect_false (ev_is_active (w)))
     return;
 
   w->sent = 0;
@@ -4991,7 +5423,7 @@ void
 ev_async_stop (EV_P_ ev_async *w) EV_NOEXCEPT
 {
   clear_pending (EV_A_ (W)w);
-  if (expect_false (!ev_is_active (w)))
+  if (ecb_expect_false (!ev_is_active (w)))
     return;
 
   EV_FREQUENT_CHECK;
diff --git a/src/common/libev/ev.h b/src/common/libev/ev.h
index f5bac00c2dd8..4669c39b6145 100644
--- a/src/common/libev/ev.h
+++ b/src/common/libev/ev.h
@@ -1,7 +1,7 @@
 /*
  * libev native API header
  *
- * Copyright (c) 2007-2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * Copyright (c) 2007-2020 Marc Alexander Lehmann <libev@schmorp.de>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modifica-
@@ -151,7 +151,10 @@ EV_CPP(extern "C" {)
 
 /*****************************************************************************/
 
-typedef double ev_tstamp;
+#ifndef EV_TSTAMP_T
+# define EV_TSTAMP_T double
+#endif
+typedef EV_TSTAMP_T ev_tstamp;
 
 #include <string.h> /* for memmove */
 
@@ -212,7 +215,7 @@ struct ev_loop;
 /*****************************************************************************/
 
 #define EV_VERSION_MAJOR 4
-#define EV_VERSION_MINOR 27
+#define EV_VERSION_MINOR 33
 
 /* eventmask, revents, events... */
 enum {
@@ -389,14 +392,12 @@ typedef struct ev_stat
 } ev_stat;
 #endif
 
-#if EV_IDLE_ENABLE
 /* invoked when the nothing else needs to be done, keeps the process from blocking */
 /* revent EV_IDLE */
 typedef struct ev_idle
 {
   EV_WATCHER (ev_idle)
 } ev_idle;
-#endif
 
 /* invoked for each run of the mainloop, just before the blocking call */
 /* you can still change events in any way you like */
@@ -413,23 +414,19 @@ typedef struct ev_check
   EV_WATCHER (ev_check)
 } ev_check;
 
-#if EV_FORK_ENABLE
 /* the callback gets invoked before check in the child process when a fork was detected */
 /* revent EV_FORK */
 typedef struct ev_fork
 {
   EV_WATCHER (ev_fork)
 } ev_fork;
-#endif
 
-#if EV_CLEANUP_ENABLE
 /* is invoked just before the loop gets destroyed */
 /* revent EV_CLEANUP */
 typedef struct ev_cleanup
 {
   EV_WATCHER (ev_cleanup)
 } ev_cleanup;
-#endif
 
 #if EV_EMBED_ENABLE
 /* used to embed an event loop inside another */
@@ -439,16 +436,18 @@ typedef struct ev_embed
   EV_WATCHER (ev_embed)
 
   struct ev_loop *other; /* ro */
+#undef EV_IO_ENABLE
+#define EV_IO_ENABLE 1
   ev_io io;              /* private */
+#undef EV_PREPARE_ENABLE
+#define EV_PREPARE_ENABLE 1
   ev_prepare prepare;    /* private */
   ev_check check;        /* unused */
   ev_timer timer;        /* unused */
   ev_periodic periodic;  /* unused */
   ev_idle idle;          /* unused */
   ev_fork fork;          /* private */
-#if EV_CLEANUP_ENABLE
   ev_cleanup cleanup;    /* unused */
-#endif
 } ev_embed;
 #endif
 
@@ -501,17 +500,18 @@ union ev_any_watcher
 /* flag bits for ev_default_loop and ev_loop_new */
 enum {
   /* the default */
-  EVFLAG_AUTO      = 0x00000000U, /* not quite a mask */
+  EVFLAG_AUTO       = 0x00000000U, /* not quite a mask */
   /* flag bits */
-  EVFLAG_NOENV     = 0x01000000U, /* do NOT consult environment */
-  EVFLAG_FORKCHECK = 0x02000000U, /* check for a fork in each iteration */
+  EVFLAG_NOENV      = 0x01000000U, /* do NOT consult environment */
+  EVFLAG_FORKCHECK  = 0x02000000U, /* check for a fork in each iteration */
   /* debugging/feature disable */
-  EVFLAG_NOINOTIFY = 0x00100000U, /* do not attempt to use inotify */
+  EVFLAG_NOINOTIFY  = 0x00100000U, /* do not attempt to use inotify */
 #if EV_COMPAT3
-  EVFLAG_NOSIGFD   = 0, /* compatibility to pre-3.9 */
+  EVFLAG_NOSIGFD    = 0, /* compatibility to pre-3.9 */
 #endif
-  EVFLAG_SIGNALFD  = 0x00200000U, /* attempt to use signalfd */
-  EVFLAG_NOSIGMASK = 0x00400000U  /* avoid modifying the signal mask */
+  EVFLAG_SIGNALFD   = 0x00200000U, /* attempt to use signalfd */
+  EVFLAG_NOSIGMASK  = 0x00400000U, /* avoid modifying the signal mask */
+  EVFLAG_NOTIMERFD  = 0x00800000U  /* avoid creating a timerfd */
 };
 
 /* method bits to be ored together */
@@ -522,8 +522,9 @@ enum {
   EVBACKEND_KQUEUE   = 0x00000008U, /* bsd, broken on osx */
   EVBACKEND_DEVPOLL  = 0x00000010U, /* solaris 8 */ /* NYI */
   EVBACKEND_PORT     = 0x00000020U, /* solaris 10 */
-  EVBACKEND_LINUXAIO = 0x00000040U, /* Linuix AIO */
-  EVBACKEND_ALL      = 0x0000007FU, /* all known backends */
+  EVBACKEND_LINUXAIO = 0x00000040U, /* linux AIO, 4.19+ */
+  EVBACKEND_IOURING  = 0x00000080U, /* linux io_uring, 5.1+ */
+  EVBACKEND_ALL      = 0x000000FFU, /* all known backends */
   EVBACKEND_MASK     = 0x0000FFFFU  /* all future backends */
 };
 
@@ -655,6 +656,8 @@ EV_API_DECL void ev_unref (EV_P) EV_NOEXCEPT;
  */
 EV_API_DECL void ev_once (EV_P_ int fd, int events, ev_tstamp timeout, void (*cb)(int revents, void *arg), void *arg) EV_NOEXCEPT;
 
+EV_API_DECL void ev_invoke_pending (EV_P); /* invoke all pending watchers */
+
 # if EV_FEATURE_API
 EV_API_DECL unsigned int ev_iteration (EV_P) EV_NOEXCEPT; /* number of loop iterations */
 EV_API_DECL unsigned int ev_depth     (EV_P) EV_NOEXCEPT; /* #ev_loop enters - #ev_loop leaves */
@@ -672,7 +675,6 @@ EV_API_DECL void ev_set_invoke_pending_cb (EV_P_ ev_loop_callback invoke_pending
 EV_API_DECL void ev_set_loop_release_cb (EV_P_ void (*release)(EV_P) EV_NOEXCEPT, void (*acquire)(EV_P) EV_NOEXCEPT) EV_NOEXCEPT;
 
 EV_API_DECL unsigned int ev_pending_count (EV_P) EV_NOEXCEPT; /* number of pending events, if any */
-EV_API_DECL void ev_invoke_pending (EV_P); /* invoke all pending watchers */
 
 /*
  * stop/start the timer handling.
@@ -692,6 +694,7 @@ EV_API_DECL void ev_resume  (EV_P) EV_NOEXCEPT;
   ev_set_cb ((ev), cb_);			\
 } while (0)
 
+#define ev_io_modify(ev,events_)             do { (ev)->events = (ev)->events & EV__IOFDSET | (events_); } while (0)
 #define ev_io_set(ev,fd_,events_)            do { (ev)->fd = (fd_); (ev)->events = (events_) | EV__IOFDSET; } while (0)
 #define ev_timer_set(ev,after_,repeat_)      do { ((ev_watcher_time *)(ev))->at = (after_); (ev)->repeat = (repeat_); } while (0)
 #define ev_periodic_set(ev,ofs_,ival_,rcb_)  do { (ev)->offset = (ofs_); (ev)->interval = (ival_); (ev)->reschedule_cb = (rcb_); } while (0)
@@ -737,6 +740,7 @@ EV_API_DECL void ev_resume  (EV_P) EV_NOEXCEPT;
 #define ev_periodic_at(ev)                   (+((ev_watcher_time *)(ev))->at)
 
 #ifndef ev_set_cb
+/* memmove is used here to avoid strict aliasing violations, and hopefully is optimized out by any reasonable compiler */
 # define ev_set_cb(ev,cb_)                   (ev_cb_ (ev) = (cb_), memmove (&((ev_watcher *)(ev))->cb, &ev_cb_ (ev), sizeof (ev_cb_ (ev))))
 #endif
 
diff --git a/src/common/libev/ev_epoll.c b/src/common/libev/ev_epoll.c
index 440e46b0225b..58cfa684d513 100644
--- a/src/common/libev/ev_epoll.c
+++ b/src/common/libev/ev_epoll.c
@@ -93,10 +93,10 @@ epoll_modify (EV_P_ int fd, int oev, int nev)
   ev.events   = (nev & EV_READ  ? EPOLLIN  : 0)
               | (nev & EV_WRITE ? EPOLLOUT : 0);
 
-  if (expect_true (!epoll_ctl (backend_fd, oev && oldmask != nev ? EPOLL_CTL_MOD : EPOLL_CTL_ADD, fd, &ev)))
+  if (ecb_expect_true (!epoll_ctl (backend_fd, oev && oldmask != nev ? EPOLL_CTL_MOD : EPOLL_CTL_ADD, fd, &ev)))
     return;
 
-  if (expect_true (errno == ENOENT))
+  if (ecb_expect_true (errno == ENOENT))
     {
       /* if ENOENT then the fd went away, so try to do the right thing */
       if (!nev)
@@ -105,7 +105,7 @@ epoll_modify (EV_P_ int fd, int oev, int nev)
       if (!epoll_ctl (backend_fd, EPOLL_CTL_ADD, fd, &ev))
         return;
     }
-  else if (expect_true (errno == EEXIST))
+  else if (ecb_expect_true (errno == EEXIST))
     {
       /* EEXIST means we ignored a previous DEL, but the fd is still active */
       /* if the kernel mask is the same as the new mask, we assume it hasn't changed */
@@ -115,7 +115,7 @@ epoll_modify (EV_P_ int fd, int oev, int nev)
       if (!epoll_ctl (backend_fd, EPOLL_CTL_MOD, fd, &ev))
         return;
     }
-  else if (expect_true (errno == EPERM))
+  else if (ecb_expect_true (errno == EPERM))
     {
       /* EPERM means the fd is always ready, but epoll is too snobbish */
       /* to handle it, unlike select or poll. */
@@ -146,16 +146,16 @@ epoll_poll (EV_P_ ev_tstamp timeout)
   int i;
   int eventcnt;
 
-  if (expect_false (epoll_epermcnt))
-    timeout = 0.;
+  if (ecb_expect_false (epoll_epermcnt))
+    timeout = EV_TS_CONST (0.);
 
   /* epoll wait times cannot be larger than (LONG_MAX - 999UL) / HZ msecs, which is below */
   /* the default libev max wait time, however. */
   EV_RELEASE_CB;
-  eventcnt = epoll_wait (backend_fd, epoll_events, epoll_eventmax, timeout * 1e3);
+  eventcnt = epoll_wait (backend_fd, epoll_events, epoll_eventmax, EV_TS_TO_MSEC (timeout));
   EV_ACQUIRE_CB;
 
-  if (expect_false (eventcnt < 0))
+  if (ecb_expect_false (eventcnt < 0))
     {
       if (errno != EINTR)
         ev_syserr ("(libev) epoll_wait");
@@ -178,14 +178,14 @@ epoll_poll (EV_P_ ev_tstamp timeout)
        * other spurious notifications will be found by epoll_ctl, below
        * we assume that fd is always in range, as we never shrink the anfds array
        */
-      if (expect_false ((uint32_t)anfds [fd].egen != (uint32_t)(ev->data.u64 >> 32)))
+      if (ecb_expect_false ((uint32_t)anfds [fd].egen != (uint32_t)(ev->data.u64 >> 32)))
         {
           /* recreate kernel state */
           postfork |= 2;
           continue;
         }
 
-      if (expect_false (got & ~want))
+      if (ecb_expect_false (got & ~want))
         {
           anfds [fd].emask = want;
 
@@ -197,6 +197,8 @@ epoll_poll (EV_P_ ev_tstamp timeout)
            * above with the gencounter check (== our fd is not the event fd), and
            * partially here, when epoll_ctl returns an error (== a child has the fd
            * but we closed it).
+           * note: for events such as POLLHUP, where we can't know whether it refers
+           * to EV_READ or EV_WRITE, we might issue redundant EPOLL_CTL_MOD calls.
            */
           ev->events = (want & EV_READ  ? EPOLLIN  : 0)
                      | (want & EV_WRITE ? EPOLLOUT : 0);
@@ -214,7 +216,7 @@ epoll_poll (EV_P_ ev_tstamp timeout)
     }
 
   /* if the receive array was full, increase its size */
-  if (expect_false (eventcnt == epoll_eventmax))
+  if (ecb_expect_false (eventcnt == epoll_eventmax))
     {
       ev_free (epoll_events);
       epoll_eventmax = array_nextsize (sizeof (struct epoll_event), epoll_eventmax, epoll_eventmax + 1);
@@ -264,7 +266,7 @@ epoll_init (EV_P_ int flags)
   if ((backend_fd = epoll_epoll_create ()) < 0)
     return 0;
 
-  backend_mintime = 1e-3; /* epoll does sometimes return early, this is just to avoid the worst */
+  backend_mintime = EV_TS_CONST (1e-3); /* epoll does sometimes return early, this is just to avoid the worst */
   backend_modify  = epoll_modify;
   backend_poll    = epoll_poll;
 
@@ -282,8 +284,8 @@ epoll_destroy (EV_P)
   array_free (epoll_eperm, EMPTY);
 }
 
-inline_size
-void
+ecb_cold
+static void
 epoll_fork (EV_P)
 {
   close (backend_fd);
diff --git a/src/common/libev/ev_iouring.c b/src/common/libev/ev_iouring.c
new file mode 100644
index 000000000000..bfd3de65f14e
--- /dev/null
+++ b/src/common/libev/ev_iouring.c
@@ -0,0 +1,694 @@
+/*
+ * libev linux io_uring fd activity backend
+ *
+ * Copyright (c) 2019-2020 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ *   1.  Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *   2.  Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/*
+ * general notes about linux io_uring:
+ *
+ * a) it's the best interface I have seen so far. on linux.
+ * b) best is not necessarily very good.
+ * c) it's better than the aio mess, doesn't suffer from the fork problems
+ *    of linux aio or epoll and so on and so on. and you could do event stuff
+ *    without any syscalls. what's not to like?
+ * d) ok, it's vastly more complex, but that's ok, really.
+ * e) why two mmaps instead of one? one would be more space-efficient,
+ *    and I can't see what benefit two would have (other than being
+ *    somehow resizable/relocatable, but that's apparently not possible).
+ * f) hmm, it's practically undebuggable (gdb can't access the memory, and
+ *    the bizarre way structure offsets are communicated makes it hard to
+ *    just print the ring buffer heads, even *iff* the memory were visible
+ *    in gdb. but then, that's also ok, really.
+ * g) well, you cannot specify a timeout when waiting for events. no,
+ *    seriously, the interface doesn't support a timeout. never seen _that_
+ *    before. sure, you can use a timerfd, but that's another syscall
+ *    you could have avoided. overall, this bizarre omission smells
+ *    like a µ-optimisation by the io_uring author for his personal
+ *    applications, to the detriment of everybody else who just wants
+ *    an event loop. but, umm, ok, if that's all, it could be worse.
+ *    (from what I gather from the author Jens Axboe, it simply didn't
+ *    occur to him, and he made good on it by adding an unlimited nuber
+ *    of timeouts later :).
+ * h) initially there was a hardcoded limit of 4096 outstanding events.
+ *    later versions not only bump this to 32k, but also can handle
+ *    an unlimited amount of events, so this only affects the batch size.
+ * i) unlike linux aio, you *can* register more then the limit
+ *    of fd events. while early verisons of io_uring signalled an overflow
+ *    and you ended up getting wet. 5.5+ does not do this anymore.
+ * j) but, oh my! it had exactly the same bugs as the linux aio backend,
+ *    where some undocumented poll combinations just fail. fortunately,
+ *    after finally reaching the author, he was more than willing to fix
+ *    this probably in 5.6+.
+ * k) overall, the *API* itself is, I dare to say, not a total trainwreck.
+ *    once the bugs ae fixed (probably in 5.6+), it will be without
+ *    competition.
+ */
+
+/* TODO: use internal TIMEOUT */
+/* TODO: take advantage of single mmap, NODROP etc. */
+/* TODO: resize cq/sq size independently */
+
+#include <sys/timerfd.h>
+#include <sys/mman.h>
+#include <poll.h>
+#include <stdint.h>
+
+#define IOURING_INIT_ENTRIES 32
+
+/*****************************************************************************/
+/* syscall wrapdadoop - this section has the raw api/abi definitions */
+
+#include <linux/fs.h>
+#include <linux/types.h>
+
+/* mostly directly taken from the kernel or documentation */
+
+struct io_uring_sqe
+{
+  __u8 opcode;
+  __u8 flags;
+  __u16 ioprio;
+  __s32 fd;
+  union {
+    __u64 off;
+    __u64 addr2;
+  };
+  __u64 addr;
+  __u32 len;
+  union {
+    __kernel_rwf_t rw_flags;
+    __u32 fsync_flags;
+    __u16 poll_events;
+    __u32 sync_range_flags;
+    __u32 msg_flags;
+    __u32 timeout_flags;
+    __u32 accept_flags;
+    __u32 cancel_flags;
+    __u32 open_flags;
+    __u32 statx_flags;
+  };
+  __u64 user_data;
+  union {
+    __u16 buf_index;
+    __u64 __pad2[3];
+  };
+};
+
+struct io_uring_cqe
+{
+  __u64 user_data;
+  __s32 res;
+  __u32 flags;
+};
+
+struct io_sqring_offsets
+{
+  __u32 head;
+  __u32 tail;
+  __u32 ring_mask;
+  __u32 ring_entries;
+  __u32 flags;
+  __u32 dropped;
+  __u32 array;
+  __u32 resv1;
+  __u64 resv2;
+};
+
+struct io_cqring_offsets
+{
+  __u32 head;
+  __u32 tail;
+  __u32 ring_mask;
+  __u32 ring_entries;
+  __u32 overflow;
+  __u32 cqes;
+  __u64 resv[2];
+};
+
+struct io_uring_params
+{
+  __u32 sq_entries;
+  __u32 cq_entries;
+  __u32 flags;
+  __u32 sq_thread_cpu;
+  __u32 sq_thread_idle;
+  __u32 features;
+  __u32 resv[4];
+  struct io_sqring_offsets sq_off;
+  struct io_cqring_offsets cq_off;
+};
+
+#define IORING_SETUP_CQSIZE 0x00000008
+
+#define IORING_OP_POLL_ADD        6
+#define IORING_OP_POLL_REMOVE     7
+#define IORING_OP_TIMEOUT        11
+#define IORING_OP_TIMEOUT_REMOVE 12
+
+/* relative or absolute, reference clock is CLOCK_MONOTONIC */
+struct iouring_kernel_timespec
+{
+  int64_t tv_sec;
+  long long tv_nsec;
+};
+
+#define IORING_TIMEOUT_ABS 0x00000001
+
+#define IORING_ENTER_GETEVENTS 0x01
+
+#define IORING_OFF_SQ_RING 0x00000000ULL
+#define IORING_OFF_CQ_RING 0x08000000ULL
+#define IORING_OFF_SQES	   0x10000000ULL
+
+#define IORING_FEAT_SINGLE_MMAP   0x00000001
+#define IORING_FEAT_NODROP        0x00000002
+#define IORING_FEAT_SUBMIT_STABLE 0x00000004
+
+inline_size
+int
+evsys_io_uring_setup (unsigned entries, struct io_uring_params *params)
+{
+  return ev_syscall2 (SYS_io_uring_setup, entries, params);
+}
+
+inline_size
+int
+evsys_io_uring_enter (int fd, unsigned to_submit, unsigned min_complete, unsigned flags, const sigset_t *sig, size_t sigsz)
+{
+  return ev_syscall6 (SYS_io_uring_enter, fd, to_submit, min_complete, flags, sig, sigsz);
+}
+
+/*****************************************************************************/
+/* actual backed implementation */
+
+/* we hope that volatile will make the compiler access this variables only once */
+#define EV_SQ_VAR(name) *(volatile unsigned *)((char *)iouring_sq_ring + iouring_sq_ ## name)
+#define EV_CQ_VAR(name) *(volatile unsigned *)((char *)iouring_cq_ring + iouring_cq_ ## name)
+
+/* the index array */
+#define EV_SQ_ARRAY     ((unsigned *)((char *)iouring_sq_ring + iouring_sq_array))
+
+/* the submit/completion queue entries */
+#define EV_SQES         ((struct io_uring_sqe *)         iouring_sqes)
+#define EV_CQES         ((struct io_uring_cqe *)((char *)iouring_cq_ring + iouring_cq_cqes))
+
+inline_speed
+int
+iouring_enter (EV_P_ ev_tstamp timeout)
+{
+  int res;
+
+  EV_RELEASE_CB;
+
+  res = evsys_io_uring_enter (iouring_fd, iouring_to_submit, 1,
+                              timeout > EV_TS_CONST (0.) ? IORING_ENTER_GETEVENTS : 0, 0, 0);
+
+  assert (("libev: io_uring_enter did not consume all sqes", (res < 0 || res == iouring_to_submit)));
+
+  iouring_to_submit = 0;
+
+  EV_ACQUIRE_CB;
+
+  return res;
+}
+
+/* TODO: can we move things around so we don't need this forward-reference? */
+static void
+iouring_poll (EV_P_ ev_tstamp timeout);
+
+static
+struct io_uring_sqe *
+iouring_sqe_get (EV_P)
+{
+  unsigned tail;
+  
+  for (;;)
+    {
+      tail = EV_SQ_VAR (tail);
+
+      if (ecb_expect_true (tail + 1 - EV_SQ_VAR (head) <= EV_SQ_VAR (ring_entries)))
+        break; /* whats the problem, we have free sqes */
+
+      /* queue full, need to flush and possibly handle some events */
+
+#if EV_FEATURE_CODE
+      /* first we ask the kernel nicely, most often this frees up some sqes */
+      int res = iouring_enter (EV_A_ EV_TS_CONST (0.));
+
+      ECB_MEMORY_FENCE_ACQUIRE; /* better safe than sorry */
+
+      if (res >= 0)
+        continue; /* yes, it worked, try again */
+#endif
+
+      /* some problem, possibly EBUSY - do the full poll and let it handle any issues */
+
+      iouring_poll (EV_A_ EV_TS_CONST (0.));
+      /* iouring_poll should have done ECB_MEMORY_FENCE_ACQUIRE for us */
+    }
+
+  /*assert (("libev: io_uring queue full after flush", tail + 1 - EV_SQ_VAR (head) <= EV_SQ_VAR (ring_entries)));*/
+
+  return EV_SQES + (tail & EV_SQ_VAR (ring_mask));
+}
+
+inline_size
+struct io_uring_sqe *
+iouring_sqe_submit (EV_P_ struct io_uring_sqe *sqe)
+{
+  unsigned idx = sqe - EV_SQES;
+
+  EV_SQ_ARRAY [idx] = idx;
+  ECB_MEMORY_FENCE_RELEASE;
+  ++EV_SQ_VAR (tail);
+  /*ECB_MEMORY_FENCE_RELEASE; /* for the time being we assume this is not needed */
+  ++iouring_to_submit;
+}
+
+/*****************************************************************************/
+
+/* when the timerfd expires we simply note the fact,
+ * as the purpose of the timerfd is to wake us up, nothing else.
+ * the next iteration should re-set it.
+ */
+static void
+iouring_tfd_cb (EV_P_ struct ev_io *w, int revents)
+{
+  iouring_tfd_to = EV_TSTAMP_HUGE;
+}
+
+/* called for full and partial cleanup */
+ecb_cold
+static int
+iouring_internal_destroy (EV_P)
+{
+  close (iouring_tfd);
+  close (iouring_fd);
+
+  if (iouring_sq_ring != MAP_FAILED) munmap (iouring_sq_ring, iouring_sq_ring_size);
+  if (iouring_cq_ring != MAP_FAILED) munmap (iouring_cq_ring, iouring_cq_ring_size);
+  if (iouring_sqes    != MAP_FAILED) munmap (iouring_sqes   , iouring_sqes_size   );
+
+  if (ev_is_active (&iouring_tfd_w))
+    {
+      ev_ref (EV_A);
+      ev_io_stop (EV_A_ &iouring_tfd_w);
+    }
+}
+
+ecb_cold
+static int
+iouring_internal_init (EV_P)
+{
+  struct io_uring_params params = { 0 };
+
+  iouring_to_submit = 0;
+
+  iouring_tfd     = -1;
+  iouring_sq_ring = MAP_FAILED;
+  iouring_cq_ring = MAP_FAILED;
+  iouring_sqes    = MAP_FAILED;
+
+  if (!have_monotonic) /* cannot really happen, but what if11 */
+    return -1;
+
+  for (;;)
+    {
+      iouring_fd = evsys_io_uring_setup (iouring_entries, &params);
+
+      if (iouring_fd >= 0)
+        break; /* yippie */
+
+      if (errno != EINVAL)
+        return -1; /* we failed */
+
+#if TODO
+      if ((~params.features) & (IORING_FEAT_NODROP | IORING_FEATURE_SINGLE_MMAP | IORING_FEAT_SUBMIT_STABLE))
+        return -1; /* we require the above features */
+#endif
+
+      /* EINVAL: lots of possible reasons, but maybe
+       * it is because we hit the unqueryable hardcoded size limit
+       */
+
+      /* we hit the limit already, give up */
+      if (iouring_max_entries)
+        return -1;
+
+      /* first time we hit EINVAL? assume we hit the limit, so go back and retry */
+      iouring_entries >>= 1;
+      iouring_max_entries = iouring_entries;
+    }
+
+  iouring_sq_ring_size = params.sq_off.array + params.sq_entries * sizeof (unsigned);
+  iouring_cq_ring_size = params.cq_off.cqes  + params.cq_entries * sizeof (struct io_uring_cqe);
+  iouring_sqes_size    =                       params.sq_entries * sizeof (struct io_uring_sqe);
+
+  iouring_sq_ring = mmap (0, iouring_sq_ring_size, PROT_READ | PROT_WRITE,
+                          MAP_SHARED | MAP_POPULATE, iouring_fd, IORING_OFF_SQ_RING);
+  iouring_cq_ring = mmap (0, iouring_cq_ring_size, PROT_READ | PROT_WRITE,
+                          MAP_SHARED | MAP_POPULATE, iouring_fd, IORING_OFF_CQ_RING);
+  iouring_sqes    = mmap (0, iouring_sqes_size, PROT_READ | PROT_WRITE,
+                          MAP_SHARED | MAP_POPULATE, iouring_fd, IORING_OFF_SQES);
+
+  if (iouring_sq_ring == MAP_FAILED || iouring_cq_ring == MAP_FAILED || iouring_sqes == MAP_FAILED)
+    return -1;
+
+  iouring_sq_head         = params.sq_off.head;
+  iouring_sq_tail         = params.sq_off.tail;
+  iouring_sq_ring_mask    = params.sq_off.ring_mask;
+  iouring_sq_ring_entries = params.sq_off.ring_entries;
+  iouring_sq_flags        = params.sq_off.flags;
+  iouring_sq_dropped      = params.sq_off.dropped;
+  iouring_sq_array        = params.sq_off.array;
+
+  iouring_cq_head         = params.cq_off.head;
+  iouring_cq_tail         = params.cq_off.tail;
+  iouring_cq_ring_mask    = params.cq_off.ring_mask;
+  iouring_cq_ring_entries = params.cq_off.ring_entries;
+  iouring_cq_overflow     = params.cq_off.overflow;
+  iouring_cq_cqes         = params.cq_off.cqes;
+
+  iouring_tfd = timerfd_create (CLOCK_MONOTONIC, TFD_CLOEXEC);
+
+  if (iouring_tfd < 0)
+    return iouring_tfd;
+
+  iouring_tfd_to = EV_TSTAMP_HUGE;
+
+  return 0;
+}
+
+ecb_cold
+static void
+iouring_fork (EV_P)
+{
+  iouring_internal_destroy (EV_A);
+
+  while (iouring_internal_init (EV_A) < 0)
+    ev_syserr ("(libev) io_uring_setup");
+
+  fd_rearm_all (EV_A);
+
+  ev_io_stop  (EV_A_ &iouring_tfd_w);
+  ev_io_set   (EV_A_ &iouring_tfd_w, iouring_tfd, EV_READ);
+  ev_io_start (EV_A_ &iouring_tfd_w);
+}
+
+/*****************************************************************************/
+
+static void
+iouring_modify (EV_P_ int fd, int oev, int nev)
+{
+  if (oev)
+    {
+      /* we assume the sqe's are all "properly" initialised */
+      struct io_uring_sqe *sqe = iouring_sqe_get (EV_A);
+      sqe->opcode    = IORING_OP_POLL_REMOVE;
+      sqe->fd        = fd;
+      /* Jens Axboe notified me that user_data is not what is documented, but is
+       * some kind of unique ID that has to match, otherwise the request cannot
+       * be removed. Since we don't *really* have that, we pass in the old
+       * generation counter - if that fails, too bad, it will hopefully be removed
+       * at close time and then be ignored. */
+      sqe->addr      = (uint32_t)fd | ((__u64)(uint32_t)anfds [fd].egen << 32);
+      sqe->user_data = (uint64_t)-1;
+      iouring_sqe_submit (EV_A_ sqe);
+
+      /* increment generation counter to avoid handling old events */
+      ++anfds [fd].egen;
+    }
+
+  if (nev)
+    {
+      struct io_uring_sqe *sqe = iouring_sqe_get (EV_A);
+      sqe->opcode      = IORING_OP_POLL_ADD;
+      sqe->fd          = fd;
+      sqe->addr        = 0;
+      sqe->user_data   = (uint32_t)fd | ((__u64)(uint32_t)anfds [fd].egen << 32);
+      sqe->poll_events =
+        (nev & EV_READ ? POLLIN : 0)
+        | (nev & EV_WRITE ? POLLOUT : 0);
+      iouring_sqe_submit (EV_A_ sqe);
+    }
+}
+
+inline_size
+void
+iouring_tfd_update (EV_P_ ev_tstamp timeout)
+{
+  ev_tstamp tfd_to = mn_now + timeout;
+
+  /* we assume there will be many iterations per timer change, so
+   * we only re-set the timerfd when we have to because its expiry
+   * is too late.
+   */
+  if (ecb_expect_false (tfd_to < iouring_tfd_to))
+    {
+       struct itimerspec its;
+
+       iouring_tfd_to = tfd_to;
+       EV_TS_SET (its.it_interval, 0.);
+       EV_TS_SET (its.it_value, tfd_to);
+
+       if (timerfd_settime (iouring_tfd, TFD_TIMER_ABSTIME, &its, 0) < 0)
+         assert (("libev: iouring timerfd_settime failed", 0));
+    }
+}
+
+inline_size
+void
+iouring_process_cqe (EV_P_ struct io_uring_cqe *cqe)
+{
+  int      fd  = cqe->user_data & 0xffffffffU;
+  uint32_t gen = cqe->user_data >> 32;
+  int      res = cqe->res;
+
+  /* user_data -1 is a remove that we are not atm. interested in */
+  if (cqe->user_data == (uint64_t)-1)
+    return;
+
+  assert (("libev: io_uring fd must be in-bounds", fd >= 0 && fd < anfdmax));
+
+  /* documentation lies, of course. the result value is NOT like
+   * normal syscalls, but like linux raw syscalls, i.e. negative
+   * error numbers. fortunate, as otherwise there would be no way
+   * to get error codes at all. still, why not document this?
+   */
+
+  /* ignore event if generation doesn't match */
+  /* other than skipping removal events, */
+  /* this should actually be very rare */
+  if (ecb_expect_false (gen != (uint32_t)anfds [fd].egen))
+    return;
+
+  if (ecb_expect_false (res < 0))
+    {
+      /*TODO: EINVAL handling (was something failed with this fd)*/
+
+      if (res == -EBADF)
+        {
+          assert (("libev: event loop rejected bad fd", res != -EBADF));
+          fd_kill (EV_A_ fd);
+        }
+      else
+        {
+          errno = -res;
+          ev_syserr ("(libev) IORING_OP_POLL_ADD");
+        }
+
+      return;
+    }
+
+  /* feed events, we do not expect or handle POLLNVAL */
+  fd_event (
+    EV_A_
+    fd,
+    (res & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+    | (res & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+  );
+
+  /* io_uring is oneshot, so we need to re-arm the fd next iteration */
+  /* this also means we usually have to do at least one syscall per iteration */
+  anfds [fd].events = 0;
+  fd_change (EV_A_ fd, EV_ANFD_REIFY);
+}
+
+/* called when the event queue overflows */
+ecb_cold
+static void
+iouring_overflow (EV_P)
+{
+  /* we have two options, resize the queue (by tearing down
+   * everything and recreating it, or living with it
+   * and polling.
+   * we implement this by resizing the queue, and, if that fails,
+   * we just recreate the state on every failure, which
+   * kind of is a very inefficient poll.
+   * one danger is, due to the bios toward lower fds,
+   * we will only really get events for those, so
+   * maybe we need a poll() fallback, after all.
+   */
+  /*EV_CQ_VAR (overflow) = 0;*/ /* need to do this if we keep the state and poll manually */
+
+  fd_rearm_all (EV_A);
+
+  /* we double the size until we hit the hard-to-probe maximum */
+  if (!iouring_max_entries)
+    {
+      iouring_entries <<= 1;
+      iouring_fork (EV_A);
+    }
+  else
+    {
+      /* we hit the kernel limit, we should fall back to something else.
+       * we can either poll() a few times and hope for the best,
+       * poll always, or switch to epoll.
+       * TODO: is this necessary with newer kernels?
+       */
+
+      iouring_internal_destroy (EV_A);
+
+      /* this should make it so that on return, we don't call any uring functions */
+      iouring_to_submit = 0;
+
+      for (;;)
+        {
+          backend = epoll_init (EV_A_ 0);
+
+          if (backend)
+            break;
+
+          ev_syserr ("(libev) iouring switch to epoll");
+        }
+    }
+}
+
+/* handle any events in the completion queue, return true if there were any */
+static int
+iouring_handle_cq (EV_P)
+{
+  unsigned head, tail, mask;
+  
+  head = EV_CQ_VAR (head);
+  ECB_MEMORY_FENCE_ACQUIRE;
+  tail = EV_CQ_VAR (tail);
+
+  if (head == tail)
+    return 0;
+
+  /* it can only overflow if we have events, yes, yes? */
+  if (ecb_expect_false (EV_CQ_VAR (overflow)))
+    {
+      iouring_overflow (EV_A);
+      return 1;
+    }
+
+  mask = EV_CQ_VAR (ring_mask);
+
+  do
+    iouring_process_cqe (EV_A_ &EV_CQES [head++ & mask]);
+  while (head != tail);
+
+  EV_CQ_VAR (head) = head;
+  ECB_MEMORY_FENCE_RELEASE;
+
+  return 1;
+}
+
+static void
+iouring_poll (EV_P_ ev_tstamp timeout)
+{
+  /* if we have events, no need for extra syscalls, but we might have to queue events */
+  /* we also clar the timeout if there are outstanding fdchanges */
+  /* the latter should only happen if both the sq and cq are full, most likely */
+  /* because we have a lot of event sources that immediately complete */
+  /* TODO: fdchacngecnt is always 0 because fd_reify does not have two buffers yet */
+  if (iouring_handle_cq (EV_A) || fdchangecnt)
+    timeout = EV_TS_CONST (0.);
+  else
+    /* no events, so maybe wait for some */
+    iouring_tfd_update (EV_A_ timeout);
+
+  /* only enter the kernel if we have something to submit, or we need to wait */
+  if (timeout || iouring_to_submit)
+    {
+      int res = iouring_enter (EV_A_ timeout);
+
+      if (ecb_expect_false (res < 0))
+        if (errno == EINTR)
+          /* ignore */;
+        else if (errno == EBUSY)
+          /* cq full, cannot submit - should be rare because we flush the cq first, so simply ignore */;
+        else
+          ev_syserr ("(libev) iouring setup");
+      else
+        iouring_handle_cq (EV_A);
+    }
+}
+
+inline_size
+int
+iouring_init (EV_P_ int flags)
+{
+  iouring_entries     = IOURING_INIT_ENTRIES;
+  iouring_max_entries = 0;
+
+  if (iouring_internal_init (EV_A) < 0)
+    {
+      iouring_internal_destroy (EV_A);
+      return 0;
+    }
+
+  ev_io_init  (&iouring_tfd_w, iouring_tfd_cb, iouring_tfd, EV_READ);
+  ev_set_priority (&iouring_tfd_w, EV_MINPRI);
+  ev_io_start (EV_A_ &iouring_tfd_w);
+  ev_unref (EV_A); /* watcher should not keep loop alive */
+
+  backend_modify = iouring_modify;
+  backend_poll   = iouring_poll;
+
+  return EVBACKEND_IOURING;
+}
+
+inline_size
+void
+iouring_destroy (EV_P)
+{
+  iouring_internal_destroy (EV_A);
+}
+
diff --git a/src/common/libev/ev_kqueue.c b/src/common/libev/ev_kqueue.c
new file mode 100644
index 000000000000..69c5147f1388
--- /dev/null
+++ b/src/common/libev/ev_kqueue.c
@@ -0,0 +1,224 @@
+/*
+ * libev kqueue backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2012,2013,2016,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ *   1.  Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *   2.  Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/event.h>
+#include <string.h>
+#include <errno.h>
+
+inline_speed
+void
+kqueue_change (EV_P_ int fd, int filter, int flags, int fflags)
+{
+  ++kqueue_changecnt;
+  array_needsize (struct kevent, kqueue_changes, kqueue_changemax, kqueue_changecnt, array_needsize_noinit);
+
+  EV_SET (&kqueue_changes [kqueue_changecnt - 1], fd, filter, flags, fflags, 0, 0);
+}
+
+/* OS X at least needs this */
+#ifndef EV_ENABLE
+# define EV_ENABLE 0
+#endif
+#ifndef NOTE_EOF
+# define NOTE_EOF 0
+#endif
+
+static void
+kqueue_modify (EV_P_ int fd, int oev, int nev)
+{
+  if (oev != nev)
+    {
+      if (oev & EV_READ)
+        kqueue_change (EV_A_ fd, EVFILT_READ , EV_DELETE, 0);
+
+      if (oev & EV_WRITE)
+        kqueue_change (EV_A_ fd, EVFILT_WRITE, EV_DELETE, 0);
+    }
+
+  /* to detect close/reopen reliably, we have to re-add */
+  /* event requests even when oev == nev */
+
+  if (nev & EV_READ)
+    kqueue_change (EV_A_ fd, EVFILT_READ , EV_ADD | EV_ENABLE, NOTE_EOF);
+
+  if (nev & EV_WRITE)
+    kqueue_change (EV_A_ fd, EVFILT_WRITE, EV_ADD | EV_ENABLE, NOTE_EOF);
+}
+
+static void
+kqueue_poll (EV_P_ ev_tstamp timeout)
+{
+  int res, i;
+  struct timespec ts;
+
+  /* need to resize so there is enough space for errors */
+  if (kqueue_changecnt > kqueue_eventmax)
+    {
+      ev_free (kqueue_events);
+      kqueue_eventmax = array_nextsize (sizeof (struct kevent), kqueue_eventmax, kqueue_changecnt);
+      kqueue_events = (struct kevent *)ev_malloc (sizeof (struct kevent) * kqueue_eventmax);
+    }
+
+  EV_RELEASE_CB;
+  EV_TS_SET (ts, timeout);
+  res = kevent (backend_fd, kqueue_changes, kqueue_changecnt, kqueue_events, kqueue_eventmax, &ts);
+  EV_ACQUIRE_CB;
+  kqueue_changecnt = 0;
+
+  if (ecb_expect_false (res < 0))
+    {
+      if (errno != EINTR)
+        ev_syserr ("(libev) kqueue kevent");
+
+      return;
+    }
+
+  for (i = 0; i < res; ++i)
+    {
+      int fd = kqueue_events [i].ident;
+
+      if (ecb_expect_false (kqueue_events [i].flags & EV_ERROR))
+        {
+          int err = kqueue_events [i].data;
+
+          /* we are only interested in errors for fds that we are interested in :) */
+          if (anfds [fd].events)
+            {
+              if (err == ENOENT) /* resubmit changes on ENOENT */
+                kqueue_modify (EV_A_ fd, 0, anfds [fd].events);
+              else if (err == EBADF) /* on EBADF, we re-check the fd */
+                {
+                  if (fd_valid (fd))
+                    kqueue_modify (EV_A_ fd, 0, anfds [fd].events);
+                  else
+                    {
+                      assert (("libev: kqueue found invalid fd", 0));
+                      fd_kill (EV_A_ fd);
+                    }
+                }
+              else /* on all other errors, we error out on the fd */
+                {
+                  assert (("libev: kqueue found invalid fd", 0));
+                  fd_kill (EV_A_ fd);
+                }
+            }
+        }
+      else
+        fd_event (
+          EV_A_
+          fd,
+          kqueue_events [i].filter == EVFILT_READ ? EV_READ
+          : kqueue_events [i].filter == EVFILT_WRITE ? EV_WRITE
+          : 0
+        );
+    }
+
+  if (ecb_expect_false (res == kqueue_eventmax))
+    {
+      ev_free (kqueue_events);
+      kqueue_eventmax = array_nextsize (sizeof (struct kevent), kqueue_eventmax, kqueue_eventmax + 1);
+      kqueue_events = (struct kevent *)ev_malloc (sizeof (struct kevent) * kqueue_eventmax);
+    }
+}
+
+inline_size
+int
+kqueue_init (EV_P_ int flags)
+{
+  /* initialize the kernel queue */
+  kqueue_fd_pid = getpid ();
+  if ((backend_fd = kqueue ()) < 0)
+    return 0;
+
+  fcntl (backend_fd, F_SETFD, FD_CLOEXEC); /* not sure if necessary, hopefully doesn't hurt */
+
+  backend_mintime = EV_TS_CONST (1e-9); /* apparently, they did the right thing in freebsd */
+  backend_modify  = kqueue_modify;
+  backend_poll    = kqueue_poll;
+
+  kqueue_eventmax = 64; /* initial number of events receivable per poll */
+  kqueue_events = (struct kevent *)ev_malloc (sizeof (struct kevent) * kqueue_eventmax);
+
+  kqueue_changes   = 0;
+  kqueue_changemax = 0;
+  kqueue_changecnt = 0;
+
+  return EVBACKEND_KQUEUE;
+}
+
+inline_size
+void
+kqueue_destroy (EV_P)
+{
+  ev_free (kqueue_events);
+  ev_free (kqueue_changes);
+}
+
+inline_size
+void
+kqueue_fork (EV_P)
+{
+  /* some BSD kernels don't just destroy the kqueue itself,
+   * but also close the fd, which isn't documented, and
+   * impossible to support properly.
+   * we remember the pid of the kqueue call and only close
+   * the fd if the pid is still the same.
+   * this leaks fds on sane kernels, but BSD interfaces are
+   * notoriously buggy and rarely get fixed.
+   */
+  pid_t newpid = getpid ();
+
+  if (newpid == kqueue_fd_pid)
+    close (backend_fd);
+
+  kqueue_fd_pid = newpid;
+  while ((backend_fd = kqueue ()) < 0)
+    ev_syserr ("(libev) kqueue");
+
+  fcntl (backend_fd, F_SETFD, FD_CLOEXEC);
+
+  /* re-register interest in fds */
+  fd_rearm_all (EV_A);
+}
+
+/* sys/event.h defines EV_ERROR */
+#undef EV_ERROR
+
diff --git a/src/common/libev/ev_linuxaio.c b/src/common/libev/ev_linuxaio.c
index ceba8dd29227..4687a703e8d5 100644
--- a/src/common/libev/ev_linuxaio.c
+++ b/src/common/libev/ev_linuxaio.c
@@ -118,57 +118,6 @@ struct aio_ring
   struct io_event io_events[0];
 };
 
-/*
- * define some syscall wrappers for common architectures
- * this is mostly for nice looks during debugging, not performance.
- * our syscalls return < 0, not == -1, on error. which is good
- * enough for linux aio.
- * TODO: arm is also common nowadays, maybe even mips and x86
- * TODO: after implementing this, it suddenly looks like overkill, but its hard to remove...
- */
-#if __GNUC__ && __linux && ECB_AMD64 && !defined __OPTIMIZE_SIZE__
-  /* the costly errno access probably kills this for size optimisation */
-
-  #define ev_syscall(nr,narg,arg1,arg2,arg3,arg4,arg5)                 \
-    ({                                                                 \
-        long res;                                                      \
-        register unsigned long r5 __asm__ ("r8" );                     \
-        register unsigned long r4 __asm__ ("r10");                     \
-        register unsigned long r3 __asm__ ("rdx");                     \
-        register unsigned long r2 __asm__ ("rsi");                     \
-        register unsigned long r1 __asm__ ("rdi");                     \
-        if (narg >= 5) r5 = (unsigned long)(arg5);                     \
-        if (narg >= 4) r4 = (unsigned long)(arg4);                     \
-        if (narg >= 3) r3 = (unsigned long)(arg3);                     \
-        if (narg >= 2) r2 = (unsigned long)(arg2);                     \
-        if (narg >= 1) r1 = (unsigned long)(arg1);                     \
-        __asm__ __volatile__ (                                         \
-          "syscall\n\t"                                                \
-          : "=a" (res)                                                 \
-          : "0" (nr), "r" (r1), "r" (r2), "r" (r3), "r" (r4), "r" (r5) \
-          : "cc", "r11", "cx", "memory");                              \
-        errno = -res;                                                  \
-        res;                                                           \
-    })
-
-#endif
-
-#ifdef ev_syscall
-  #define ev_syscall0(nr)                          ev_syscall (nr, 0,    0,    0,    0,    0,    0
-  #define ev_syscall1(nr,arg1)                     ev_syscall (nr, 1, arg1,    0,    0,    0,    0)
-  #define ev_syscall2(nr,arg1,arg2)                ev_syscall (nr, 2, arg1, arg2,    0,    0,    0)
-  #define ev_syscall3(nr,arg1,arg2,arg3)           ev_syscall (nr, 3, arg1, arg2, arg3,    0,    0)
-  #define ev_syscall4(nr,arg1,arg2,arg3,arg4)      ev_syscall (nr, 3, arg1, arg2, arg3, arg4,    0)
-  #define ev_syscall5(nr,arg1,arg2,arg3,arg4,arg5) ev_syscall (nr, 5, arg1, arg2, arg3, arg4, arg5)
-#else
-  #define ev_syscall0(nr)                          syscall (nr)
-  #define ev_syscall1(nr,arg1)                     syscall (nr, arg1)
-  #define ev_syscall2(nr,arg1,arg2)                syscall (nr, arg1, arg2)
-  #define ev_syscall3(nr,arg1,arg2,arg3)           syscall (nr, arg1, arg2, arg3)
-  #define ev_syscall4(nr,arg1,arg2,arg3,arg4)      syscall (nr, arg1, arg2, arg3, arg4)
-  #define ev_syscall5(nr,arg1,arg2,arg3,arg4,arg5) syscall (nr, arg1, arg2, arg3, arg4, arg5)
-#endif
-
 inline_size
 int
 evsys_io_setup (unsigned nr_events, aio_context_t *ctx_idp)
@@ -265,7 +214,6 @@ linuxaio_array_needsize_iocbp (ANIOCBP *base, int offset, int count)
       memset (iocb, 0, sizeof (*iocb));
 
       iocb->io.aio_lio_opcode = IOCB_CMD_POLL;
-      iocb->io.aio_data       = offset;
       iocb->io.aio_fildes     = offset;
 
       base [offset++] = iocb;
@@ -287,28 +235,47 @@ linuxaio_modify (EV_P_ int fd, int oev, int nev)
 {
   array_needsize (ANIOCBP, linuxaio_iocbps, linuxaio_iocbpmax, fd + 1, linuxaio_array_needsize_iocbp);
   ANIOCBP iocb = linuxaio_iocbps [fd];
+  ANFD *anfd = &anfds [fd];
 
-  if (iocb->io.aio_reqprio < 0)
+  if (ecb_expect_false (iocb->io.aio_reqprio < 0))
     {
       /* we handed this fd over to epoll, so undo this first */
       /* we do it manually because the optimisations on epoll_modify won't do us any good */
       epoll_ctl (backend_fd, EPOLL_CTL_DEL, fd, 0);
-      anfds [fd].emask = 0;
+      anfd->emask = 0;
       iocb->io.aio_reqprio = 0;
     }
-
-  if (iocb->io.aio_buf)
+  else if (ecb_expect_false (iocb->io.aio_buf))
     {
-      evsys_io_cancel (linuxaio_ctx, &iocb->io, (struct io_event *)0);
-      /* on relevant kernels, io_cancel fails with EINPROGRES if everything is fine */
-      assert (("libev: linuxaio unexpected io_cancel failed", errno == EINPROGRESS));
+      /* iocb active, so cancel it first before resubmit */
+      /* this assumes we only ever get one call per fd per loop iteration */
+      for (;;)
+        {
+          /* on all relevant kernels, io_cancel fails with EINPROGRESS on "success" */
+          if (ecb_expect_false (evsys_io_cancel (linuxaio_ctx, &iocb->io, (struct io_event *)0) == 0))
+            break;
+
+          if (ecb_expect_true (errno == EINPROGRESS))
+            break;
+
+          /* the EINPROGRESS test is for nicer error message. clumsy. */
+          if (errno != EINTR)
+            {
+              assert (("libev: linuxaio unexpected io_cancel failed", errno != EINTR && errno != EINPROGRESS));
+              break;
+            }
+       }
+
+      /* increment generation counter to avoid handling old events */
+      ++anfd->egen;
     }
 
+  iocb->io.aio_buf = (nev & EV_READ  ? POLLIN  : 0)
+                   | (nev & EV_WRITE ? POLLOUT : 0);
+
   if (nev)
     {
-      iocb->io.aio_buf =
-          (nev & EV_READ ? POLLIN : 0)
-          | (nev & EV_WRITE ? POLLOUT : 0);
+      iocb->io.aio_data = (uint32_t)fd | ((__u64)(uint32_t)anfd->egen << 32);
 
       /* queue iocb up for io_submit */
       /* this assumes we only ever get one call per fd per loop iteration */
@@ -338,21 +305,26 @@ linuxaio_parse_events (EV_P_ struct io_event *ev, int nr)
 {
   while (nr)
     {
-      int fd  = ev->data;
-      int res = ev->res;
+      int fd       = ev->data & 0xffffffff;
+      uint32_t gen = ev->data >> 32;
+      int res      = ev->res;
 
       assert (("libev: iocb fd must be in-bounds", fd >= 0 && fd < anfdmax));
 
-      /* feed events, we do not expect or handle POLLNVAL */
-      fd_event (
-        EV_A_
-        fd,
-        (res & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
-        | (res & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
-      );
-
-      /* linux aio is oneshot: rearm fd. TODO: this does more work than strictly needed */
-      linuxaio_fd_rearm (EV_A_ fd);
+      /* only accept events if generation counter matches */
+      if (ecb_expect_true (gen == (uint32_t)anfds [fd].egen))
+        {
+          /* feed events, we do not expect or handle POLLNVAL */
+          fd_event (
+            EV_A_
+            fd,
+            (res & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+            | (res & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+          );
+
+          /* linux aio is oneshot: rearm fd. TODO: this does more work than strictly needed */
+          linuxaio_fd_rearm (EV_A_ fd);
+        }
 
       --nr;
       ++ev;
@@ -364,21 +336,20 @@ static int
 linuxaio_get_events_from_ring (EV_P)
 {
   struct aio_ring *ring = (struct aio_ring *)linuxaio_ctx;
+  unsigned head, tail;
 
   /* the kernel reads and writes both of these variables, */
   /* as a C extension, we assume that volatile use here */
   /* both makes reads atomic and once-only */
-  unsigned head = *(volatile unsigned *)&ring->head;
-  unsigned tail = *(volatile unsigned *)&ring->tail;
+  head = *(volatile unsigned *)&ring->head;
+  ECB_MEMORY_FENCE_ACQUIRE;
+  tail = *(volatile unsigned *)&ring->tail;
 
   if (head == tail)
     return 0;
 
-  /* make sure the events up to tail are visible */
-  ECB_MEMORY_FENCE_ACQUIRE;
-
   /* parse all available events, but only once, to avoid starvation */
-  if (tail > head) /* normal case around */
+  if (ecb_expect_true (tail > head)) /* normal case around */
     linuxaio_parse_events (EV_A_ ring->io_events + head, tail - head);
   else /* wrapped around */
     {
@@ -399,7 +370,7 @@ linuxaio_ringbuf_valid (EV_P)
 {
   struct aio_ring *ring = (struct aio_ring *)linuxaio_ctx;
 
-  return expect_true (ring->magic == AIO_RING_MAGIC)
+  return ecb_expect_true (ring->magic == AIO_RING_MAGIC)
                       && ring->incompat_features == EV_AIO_RING_INCOMPAT_FEATURES
                       && ring->header_length == sizeof (struct aio_ring); /* TODO: or use it to find io_event[0]? */
 }
@@ -414,7 +385,7 @@ linuxaio_get_events (EV_P_ ev_tstamp timeout)
   int want = 1; /* how many events to request */
   int ringbuf_valid = linuxaio_ringbuf_valid (EV_A);
 
-  if (expect_true (ringbuf_valid))
+  if (ecb_expect_true (ringbuf_valid))
     {
       /* if the ring buffer has any events, we don't wait or call the kernel at all */
       if (linuxaio_get_events_from_ring (EV_A))
@@ -437,9 +408,7 @@ linuxaio_get_events (EV_P_ ev_tstamp timeout)
 
       EV_RELEASE_CB;
 
-      ts.tv_sec  = (long)timeout;
-      ts.tv_nsec = (long)((timeout - ts.tv_sec) * 1e9);
-
+      EV_TS_SET (ts, timeout);
       res = evsys_io_getevents (linuxaio_ctx, 1, want, ioev, &ts);
 
       EV_ACQUIRE_CB;
@@ -454,7 +423,7 @@ linuxaio_get_events (EV_P_ ev_tstamp timeout)
           /* at least one event available, handle them */
           linuxaio_parse_events (EV_A_ ioev, res);
 
-          if (expect_true (ringbuf_valid))
+          if (ecb_expect_true (ringbuf_valid))
             {
               /* if we have a ring buffer, handle any remaining events in it */
               linuxaio_get_events_from_ring (EV_A);
@@ -469,7 +438,7 @@ linuxaio_get_events (EV_P_ ev_tstamp timeout)
       else
         break; /* no events from the kernel, we are done */
 
-      timeout = 0; /* only wait in the first iteration */
+      timeout = EV_TS_CONST (0.); /* only wait in the first iteration */
     }
 }
 
@@ -495,7 +464,7 @@ linuxaio_poll (EV_P_ ev_tstamp timeout)
     {
       int res = evsys_io_submit (linuxaio_ctx, linuxaio_submitcnt - submitted, linuxaio_submits + submitted);
 
-      if (expect_false (res < 0))
+      if (ecb_expect_false (res < 0))
         if (errno == EINVAL)
           {
             /* This happens for unsupported fds, officially, but in my testing,
@@ -535,16 +504,21 @@ linuxaio_poll (EV_P_ ev_tstamp timeout)
             ++linuxaio_iteration;
             if (linuxaio_io_setup (EV_A) < 0)
               {
+                /* TODO: rearm all and recreate epoll backend from scratch */
+                /* TODO: might be more prudent? */
+
                 /* to bad, we can't get a new aio context, go 100% epoll */
                 linuxaio_free_iocbp (EV_A);
                 ev_io_stop (EV_A_ &linuxaio_epoll_w);
                 ev_ref (EV_A);
                 linuxaio_ctx = 0;
+
+                backend        = EVBACKEND_EPOLL;
                 backend_modify = epoll_modify;
                 backend_poll   = epoll_poll;
               }
 
-            timeout = 0;
+            timeout = EV_TS_CONST (0.);
             /* it's easiest to handle this mess in another iteration */
             return;
           }
@@ -555,8 +529,13 @@ linuxaio_poll (EV_P_ ev_tstamp timeout)
 
             res = 1; /* skip this iocb */
           }
+        else if (errno == EINTR) /* not seen in reality, not documented */
+          res = 0; /* silently ignore and retry */
         else
-          ev_syserr ("(libev) linuxaio io_submit");
+          {
+            ev_syserr ("(libev) linuxaio io_submit");
+            res = 0;
+          }
 
       submitted += res;
     }
@@ -589,13 +568,13 @@ linuxaio_init (EV_P_ int flags)
       return 0;
     }
 
-  ev_io_init  (EV_A_ &linuxaio_epoll_w, linuxaio_epoll_cb, backend_fd, EV_READ);
+  ev_io_init  (&linuxaio_epoll_w, linuxaio_epoll_cb, backend_fd, EV_READ);
   ev_set_priority (&linuxaio_epoll_w, EV_MAXPRI);
   ev_io_start (EV_A_ &linuxaio_epoll_w);
   ev_unref (EV_A); /* watcher should not keep loop alive */
 
-  backend_modify  = linuxaio_modify;
-  backend_poll    = linuxaio_poll;
+  backend_modify = linuxaio_modify;
+  backend_poll   = linuxaio_poll;
 
   linuxaio_iocbpmax = 0;
   linuxaio_iocbps = 0;
@@ -616,13 +595,13 @@ linuxaio_destroy (EV_P)
   evsys_io_destroy (linuxaio_ctx); /* fails in child, aio context is destroyed */
 }
 
-inline_size
-void
+ecb_cold
+static void
 linuxaio_fork (EV_P)
 {
-  /* this frees all iocbs, which is very heavy-handed */
-  linuxaio_destroy (EV_A);
   linuxaio_submitcnt = 0; /* all pointers were invalidated */
+  linuxaio_free_iocbp (EV_A); /* this frees all iocbs, which is very heavy-handed */
+  evsys_io_destroy (linuxaio_ctx); /* fails in child, aio context is destroyed */
 
   linuxaio_iteration = 0; /* we start over in the child */
 
@@ -631,12 +610,11 @@ linuxaio_fork (EV_P)
 
   /* forking epoll should also effectively unregister all fds from the backend */
   epoll_fork (EV_A);
+  /* epoll_fork already did this. hopefully */
+  /*fd_rearm_all (EV_A);*/
 
   ev_io_stop  (EV_A_ &linuxaio_epoll_w);
   ev_io_set   (EV_A_ &linuxaio_epoll_w, backend_fd, EV_READ);
   ev_io_start (EV_A_ &linuxaio_epoll_w);
-
-  /* epoll_fork already did this. hopefully */
-  /*fd_rearm_all (EV_A);*/
 }
 
diff --git a/src/common/libev/ev_poll.c b/src/common/libev/ev_poll.c
index a3ef464ac814..e5508ddb0c25 100644
--- a/src/common/libev/ev_poll.c
+++ b/src/common/libev/ev_poll.c
@@ -80,7 +80,7 @@ poll_modify (EV_P_ int fd, int oev, int nev)
     {
       pollidxs [fd] = -1;
 
-      if (expect_true (idx < --pollcnt))
+      if (ecb_expect_true (idx < --pollcnt))
         {
           polls [idx] = polls [pollcnt];
           pollidxs [polls [idx].fd] = idx;
@@ -95,10 +95,10 @@ poll_poll (EV_P_ ev_tstamp timeout)
   int res;
   
   EV_RELEASE_CB;
-  res = poll (polls, pollcnt, timeout * 1e3);
+  res = poll (polls, pollcnt, EV_TS_TO_MSEC (timeout));
   EV_ACQUIRE_CB;
 
-  if (expect_false (res < 0))
+  if (ecb_expect_false (res < 0))
     {
       if (errno == EBADF)
         fd_ebadf (EV_A);
@@ -112,11 +112,11 @@ poll_poll (EV_P_ ev_tstamp timeout)
       {
         assert (("libev: poll returned illegal result, broken BSD kernel?", p < polls + pollcnt));
 
-        if (expect_false (p->revents)) /* this expect is debatable */
+        if (ecb_expect_false (p->revents)) /* this expect is debatable */
           {
             --res;
 
-            if (expect_false (p->revents & POLLNVAL))
+            if (ecb_expect_false (p->revents & POLLNVAL))
               {
                 assert (("libev: poll found invalid fd in poll set", 0));
                 fd_kill (EV_A_ p->fd);
@@ -136,7 +136,7 @@ inline_size
 int
 poll_init (EV_P_ int flags)
 {
-  backend_mintime = 1e-3;
+  backend_mintime = EV_TS_CONST (1e-3);
   backend_modify  = poll_modify;
   backend_poll    = poll_poll;
 
diff --git a/src/common/libev/ev_port.c b/src/common/libev/ev_port.c
new file mode 100644
index 000000000000..f4cd9d99cd39
--- /dev/null
+++ b/src/common/libev/ev_port.c
@@ -0,0 +1,192 @@
+/*
+ * libev solaris event port backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ *   1.  Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *   2.  Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/* useful reading:
+ *
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6268715 (random results)
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6455223 (just totally broken)
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6873782 (manpage ETIME)
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6874410 (implementation ETIME)
+ * http://www.mail-archive.com/networking-discuss@opensolaris.org/msg11898.html ETIME vs. nget
+ * http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/lib/libc/port/gen/event_port.c (libc)
+ * http://cvs.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/uts/common/fs/portfs/port.c#1325 (kernel)
+ */
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <poll.h>
+#include <port.h>
+#include <string.h>
+#include <errno.h>
+
+inline_speed
+void
+port_associate_and_check (EV_P_ int fd, int ev)
+{
+  if (0 >
+      port_associate (
+         backend_fd, PORT_SOURCE_FD, fd,
+         (ev & EV_READ ? POLLIN : 0)
+         | (ev & EV_WRITE ? POLLOUT : 0),
+         0
+      )
+  )
+    {
+      if (errno == EBADFD)
+        {
+          assert (("libev: port_associate found invalid fd", errno != EBADFD));
+          fd_kill (EV_A_ fd);
+        }
+      else
+        ev_syserr ("(libev) port_associate");
+    }
+}
+
+static void
+port_modify (EV_P_ int fd, int oev, int nev)
+{
+  /* we need to reassociate no matter what, as closes are
+   * once more silently being discarded.
+   */
+  if (!nev)
+    {
+      if (oev)
+        port_dissociate (backend_fd, PORT_SOURCE_FD, fd);
+    }
+  else
+    port_associate_and_check (EV_A_ fd, nev);
+}
+
+static void
+port_poll (EV_P_ ev_tstamp timeout)
+{
+  int res, i;
+  struct timespec ts;
+  uint_t nget = 1;
+
+  /* we initialise this to something we will skip in the loop, as */
+  /* port_getn can return with nget unchanged, but no indication */
+  /* whether it was the original value or has been updated :/ */
+  port_events [0].portev_source = 0;
+
+  EV_RELEASE_CB;
+  EV_TS_SET (ts, timeout);
+  res = port_getn (backend_fd, port_events, port_eventmax, &nget, &ts);
+  EV_ACQUIRE_CB;
+
+  /* port_getn may or may not set nget on error */
+  /* so we rely on port_events [0].portev_source not being updated */
+  if (res == -1 && errno != ETIME && errno != EINTR)
+    ev_syserr ("(libev) port_getn (see http://bugs.opensolaris.org/view_bug.do?bug_id=6268715, try LIBEV_FLAGS=3 env variable)");
+
+  for (i = 0; i < nget; ++i)
+    {
+      if (port_events [i].portev_source == PORT_SOURCE_FD)
+        {
+          int fd = port_events [i].portev_object;
+
+          fd_event (
+            EV_A_
+            fd,
+            (port_events [i].portev_events & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+            | (port_events [i].portev_events & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+          );
+
+          fd_change (EV_A_ fd, EV__IOFDSET);
+        }
+    }
+
+  if (ecb_expect_false (nget == port_eventmax))
+    {
+      ev_free (port_events);
+      port_eventmax = array_nextsize (sizeof (port_event_t), port_eventmax, port_eventmax + 1);
+      port_events = (port_event_t *)ev_malloc (sizeof (port_event_t) * port_eventmax);
+    }
+}
+
+inline_size
+int
+port_init (EV_P_ int flags)
+{
+  /* Initialize the kernel queue */
+  if ((backend_fd = port_create ()) < 0)
+    return 0;
+
+  assert (("libev: PORT_SOURCE_FD must not be zero", PORT_SOURCE_FD));
+
+  fcntl (backend_fd, F_SETFD, FD_CLOEXEC); /* not sure if necessary, hopefully doesn't hurt */
+
+  /* if my reading of the opensolaris kernel sources are correct, then
+   * opensolaris does something very stupid: it checks if the time has already
+   * elapsed and doesn't round up if that is the case, otherwise it DOES round
+   * up. Since we can't know what the case is, we need to guess by using a
+   * "large enough" timeout. Normally, 1e-9 would be correct.
+   */
+  backend_mintime = EV_TS_CONST (1e-3); /* needed to compensate for port_getn returning early */
+  backend_modify  = port_modify;
+  backend_poll    = port_poll;
+
+  port_eventmax = 64; /* initial number of events receivable per poll */
+  port_events = (port_event_t *)ev_malloc (sizeof (port_event_t) * port_eventmax);
+
+  return EVBACKEND_PORT;
+}
+
+inline_size
+void
+port_destroy (EV_P)
+{
+  ev_free (port_events);
+}
+
+inline_size
+void
+port_fork (EV_P)
+{
+  close (backend_fd);
+
+  while ((backend_fd = port_create ()) < 0)
+    ev_syserr ("(libev) port");
+
+  fcntl (backend_fd, F_SETFD, FD_CLOEXEC);
+
+  /* re-register interest in fds */
+  fd_rearm_all (EV_A);
+}
+
diff --git a/src/common/libev/ev_select.c b/src/common/libev/ev_select.c
index ed1fc7ad9298..b862c8113783 100644
--- a/src/common/libev/ev_select.c
+++ b/src/common/libev/ev_select.c
@@ -108,7 +108,7 @@ select_modify (EV_P_ int fd, int oev, int nev)
     int     word = fd / NFDBITS;
     fd_mask mask = 1UL << (fd % NFDBITS);
 
-    if (expect_false (vec_max <= word))
+    if (ecb_expect_false (vec_max <= word))
       {
         int new_max = word + 1;
 
@@ -171,7 +171,7 @@ select_poll (EV_P_ ev_tstamp timeout)
 #endif
   EV_ACQUIRE_CB;
 
-  if (expect_false (res < 0))
+  if (ecb_expect_false (res < 0))
     {
       #if EV_SELECT_IS_WINSOCKET
       errno = WSAGetLastError ();
@@ -197,7 +197,7 @@ select_poll (EV_P_ ev_tstamp timeout)
         {
           if (timeout)
             {
-              unsigned long ms = timeout * 1e3;
+              unsigned long ms = EV_TS_TO_MSEC (timeout);
               Sleep (ms ? ms : 1);
             }
 
@@ -236,7 +236,7 @@ select_poll (EV_P_ ev_tstamp timeout)
           if (FD_ISSET (handle, (fd_set *)vec_eo)) events |= EV_WRITE;
           #endif
 
-          if (expect_true (events))
+          if (ecb_expect_true (events))
             fd_event (EV_A_ fd, events);
         }
   }
@@ -262,7 +262,7 @@ select_poll (EV_P_ ev_tstamp timeout)
               events |= word_r & mask ? EV_READ  : 0;
               events |= word_w & mask ? EV_WRITE : 0;
 
-              if (expect_true (events))
+              if (ecb_expect_true (events))
                 fd_event (EV_A_ word * NFDBITS + bit, events);
             }
       }
@@ -275,7 +275,7 @@ inline_size
 int
 select_init (EV_P_ int flags)
 {
-  backend_mintime = 1e-6;
+  backend_mintime = EV_TS_CONST (1e-6);
   backend_modify  = select_modify;
   backend_poll    = select_poll;
 
diff --git a/src/common/libev/ev_vars.h b/src/common/libev/ev_vars.h
index 20082409edc0..fb0c58316245 100644
--- a/src/common/libev/ev_vars.h
+++ b/src/common/libev/ev_vars.h
@@ -118,6 +118,35 @@ VARx(int, linuxaio_submitmax)
 VARx(ev_io, linuxaio_epoll_w)
 #endif
 
+#if EV_USE_IOURING || EV_GENWRAP
+VARx(int, iouring_fd)
+VARx(unsigned, iouring_to_submit);
+VARx(int, iouring_entries)
+VARx(int, iouring_max_entries)
+VARx(void *, iouring_sq_ring)
+VARx(void *, iouring_cq_ring)
+VARx(void *, iouring_sqes)
+VARx(uint32_t, iouring_sq_ring_size)
+VARx(uint32_t, iouring_cq_ring_size)
+VARx(uint32_t, iouring_sqes_size)
+VARx(uint32_t, iouring_sq_head)
+VARx(uint32_t, iouring_sq_tail)
+VARx(uint32_t, iouring_sq_ring_mask)
+VARx(uint32_t, iouring_sq_ring_entries)
+VARx(uint32_t, iouring_sq_flags)
+VARx(uint32_t, iouring_sq_dropped)
+VARx(uint32_t, iouring_sq_array)
+VARx(uint32_t, iouring_cq_head)
+VARx(uint32_t, iouring_cq_tail)
+VARx(uint32_t, iouring_cq_ring_mask)
+VARx(uint32_t, iouring_cq_ring_entries)
+VARx(uint32_t, iouring_cq_overflow)
+VARx(uint32_t, iouring_cq_cqes)
+VARx(ev_tstamp, iouring_tfd_to)
+VARx(int, iouring_tfd)
+VARx(ev_io, iouring_tfd_w)
+#endif
+
 #if EV_USE_KQUEUE || EV_GENWRAP
 VARx(pid_t, kqueue_fd_pid)
 VARx(struct kevent *, kqueue_changes)
@@ -198,6 +227,11 @@ VARx(ev_io, sigfd_w)
 VARx(sigset_t, sigfd_set)
 #endif
 
+#if EV_USE_TIMERFD || EV_GENWRAP
+VARx(int, timerfd) /* timerfd for time jump detection */
+VARx(ev_io, timerfd_w)
+#endif
+
 VARx(unsigned int, origflags) /* original loop flags */
 
 #if EV_FEATURE_API || EV_GENWRAP
diff --git a/src/common/libev/ev_wrap.h b/src/common/libev/ev_wrap.h
index 90bb0a182d91..45d793cedb8e 100644
--- a/src/common/libev/ev_wrap.h
+++ b/src/common/libev/ev_wrap.h
@@ -44,6 +44,32 @@
 #define invoke_cb ((loop)->invoke_cb)
 #define io_blocktime ((loop)->io_blocktime)
 #define iocp ((loop)->iocp)
+#define iouring_cq_cqes ((loop)->iouring_cq_cqes)
+#define iouring_cq_head ((loop)->iouring_cq_head)
+#define iouring_cq_overflow ((loop)->iouring_cq_overflow)
+#define iouring_cq_ring ((loop)->iouring_cq_ring)
+#define iouring_cq_ring_entries ((loop)->iouring_cq_ring_entries)
+#define iouring_cq_ring_mask ((loop)->iouring_cq_ring_mask)
+#define iouring_cq_ring_size ((loop)->iouring_cq_ring_size)
+#define iouring_cq_tail ((loop)->iouring_cq_tail)
+#define iouring_entries ((loop)->iouring_entries)
+#define iouring_fd ((loop)->iouring_fd)
+#define iouring_max_entries ((loop)->iouring_max_entries)
+#define iouring_sq_array ((loop)->iouring_sq_array)
+#define iouring_sq_dropped ((loop)->iouring_sq_dropped)
+#define iouring_sq_flags ((loop)->iouring_sq_flags)
+#define iouring_sq_head ((loop)->iouring_sq_head)
+#define iouring_sq_ring ((loop)->iouring_sq_ring)
+#define iouring_sq_ring_entries ((loop)->iouring_sq_ring_entries)
+#define iouring_sq_ring_mask ((loop)->iouring_sq_ring_mask)
+#define iouring_sq_ring_size ((loop)->iouring_sq_ring_size)
+#define iouring_sq_tail ((loop)->iouring_sq_tail)
+#define iouring_sqes ((loop)->iouring_sqes)
+#define iouring_sqes_size ((loop)->iouring_sqes_size)
+#define iouring_tfd ((loop)->iouring_tfd)
+#define iouring_tfd_to ((loop)->iouring_tfd_to)
+#define iouring_tfd_w ((loop)->iouring_tfd_w)
+#define iouring_to_submit ((loop)->iouring_to_submit)
 #define kqueue_changecnt ((loop)->kqueue_changecnt)
 #define kqueue_changemax ((loop)->kqueue_changemax)
 #define kqueue_changes ((loop)->kqueue_changes)
@@ -97,6 +123,8 @@
 #define sigfd_w ((loop)->sigfd_w)
 #define timeout_blocktime ((loop)->timeout_blocktime)
 #define timercnt ((loop)->timercnt)
+#define timerfd ((loop)->timerfd)
+#define timerfd_w ((loop)->timerfd_w)
 #define timermax ((loop)->timermax)
 #define timers ((loop)->timers)
 #define userdata ((loop)->userdata)
@@ -151,6 +179,32 @@
 #undef invoke_cb
 #undef io_blocktime
 #undef iocp
+#undef iouring_cq_cqes
+#undef iouring_cq_head
+#undef iouring_cq_overflow
+#undef iouring_cq_ring
+#undef iouring_cq_ring_entries
+#undef iouring_cq_ring_mask
+#undef iouring_cq_ring_size
+#undef iouring_cq_tail
+#undef iouring_entries
+#undef iouring_fd
+#undef iouring_max_entries
+#undef iouring_sq_array
+#undef iouring_sq_dropped
+#undef iouring_sq_flags
+#undef iouring_sq_head
+#undef iouring_sq_ring
+#undef iouring_sq_ring_entries
+#undef iouring_sq_ring_mask
+#undef iouring_sq_ring_size
+#undef iouring_sq_tail
+#undef iouring_sqes
+#undef iouring_sqes_size
+#undef iouring_tfd
+#undef iouring_tfd_to
+#undef iouring_tfd_w
+#undef iouring_to_submit
 #undef kqueue_changecnt
 #undef kqueue_changemax
 #undef kqueue_changes
@@ -204,6 +258,8 @@
 #undef sigfd_w
 #undef timeout_blocktime
 #undef timercnt
+#undef timerfd
+#undef timerfd_w
 #undef timermax
 #undef timers
 #undef userdata
diff --git a/src/common/libev/libev.m4 b/src/common/libev/libev.m4
index 642e7b175182..f859eff27c19 100644
--- a/src/common/libev/libev.m4
+++ b/src/common/libev/libev.m4
@@ -2,8 +2,8 @@ dnl this file is part of libev, do not make local modifications
 dnl http://software.schmorp.de/pkg/libev
 
 dnl libev support
-AC_CHECK_HEADERS(sys/inotify.h sys/epoll.h sys/event.h port.h poll.h)
-AC_CHECK_HEADERS(sys/select.h sys/eventfd.h sys/signalfd.h linux/aio_abi.h)
+AC_CHECK_HEADERS(sys/inotify.h sys/epoll.h sys/event.h port.h poll.h sys/timerfd.h)
+AC_CHECK_HEADERS(sys/select.h sys/eventfd.h sys/signalfd.h linux/aio_abi.h linux/fs.h)
  
 AC_CHECK_FUNCS(inotify_init epoll_ctl kqueue port_create poll select eventfd signalfd)
  
@@ -36,6 +36,10 @@ AC_CHECK_FUNCS(nanosleep, [], [
    fi
 ])
 
+AC_CHECK_TYPE(__kernel_rwf_t, [
+   AC_DEFINE(HAVE_KERNEL_RWF_T, 1, Define to 1 if linux/fs.h defined kernel_rwf_t)
+], [], [#include <linux/fs.h>])
+
 if test -z "$LIBEV_M4_AVOID_LIBM"; then
    LIBM=m
 fi
diff --git a/src/common/libeventlog/Makefile.am b/src/common/libeventlog/Makefile.am
index c68c3572cd97..f28236c277f7 100644
--- a/src/common/libeventlog/Makefile.am
+++ b/src/common/libeventlog/Makefile.am
@@ -6,23 +6,25 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
+	$(JANSSON_CFLAGS)
 
 noinst_LTLIBRARIES = libeventlog.la
-fluxinclude_HEADERS = eventlog.h
-libeventlog_la_SOURCES = eventlog.c
-
-libeventlog_la_CPPFLAGS = \
-	$(AM_CPPFLAGS)
-libeventlog_la_LDFLAGS = \
-	-avoid-version -module -shared -export-dynamic \
-	$(AM_LDFLAGS)
-
-
-TESTS = test_eventlog.t
+libeventlog_la_SOURCES = \
+	eventlog.h \
+	eventlog.c \
+	eventlogger.h \
+	eventlogger.c \
+	formatter.h \
+	formatter.c
+
+TESTS = \
+	test_eventlog.t \
+	test_formatter.t
 
 check_PROGRAMS = $(TESTS)
 
@@ -36,4 +38,15 @@ test_eventlog_t_LDADD = \
 	$(top_builddir)/src/common/libtap/libtap.la \
 	$(top_builddir)/src/common/libeventlog/libeventlog.la \
 	$(top_builddir)/src/common/libutil/libutil.la \
-	$(JANSSON_LIBS)
+	$(JANSSON_LIBS) \
+	$(LIBRT)
+
+test_formatter_t_SOURCES = test/formatter.c
+test_formatter_t_CPPFLAGS = $(AM_CPPFLAGS)
+test_formatter_t_LDADD = \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libeventlog/libeventlog.la \
+	$(top_builddir)/src/common/libutil/libutil.la \
+	$(top_builddir)/src/common/libmissing/libmissing.la \
+	$(JANSSON_LIBS) \
+	$(LIBRT)
diff --git a/src/common/libeventlog/eventlog.c b/src/common/libeventlog/eventlog.c
index 9d68d39f2f6b..d94688c2fb5a 100644
--- a/src/common/libeventlog/eventlog.c
+++ b/src/common/libeventlog/eventlog.c
@@ -13,8 +13,10 @@
 #endif
 #include <flux/core.h>
 #include <jansson.h>
-#include <czmq.h>
-#include <argz.h>
+#include <time.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
 
 #include "eventlog.h"
 
@@ -28,29 +30,18 @@ int eventlog_entry_parse (json_t *entry,
 {
     double t;
     const char *n;
-    json_t *c;
-
-    if (!entry) {
-        errno = EINVAL;
-        return -1;
-    }
+    json_t *c = NULL;
 
-    if (json_unpack (entry, "{ s:F s:s }",
-                     "timestamp", &t,
-                     "name", &n) < 0) {
+    if (!entry
+        || json_unpack (entry,
+                        "{s:F s:s s?o}",
+                        "timestamp", &t,
+                        "name", &n,
+                        "context", &c) < 0
+        || (c && !json_is_object (c))) {
         errno = EINVAL;
         return -1;
     }
-
-    if (!json_unpack (entry, "{ s:o }", "context", &c)) {
-        if (!json_is_object (c)) {
-            errno = EINVAL;
-            return -1;
-        }
-    }
-    else
-        c = NULL;
-
     if (timestamp)
         (*timestamp) = t;
     if (name)
@@ -172,7 +163,7 @@ static json_t *eventlog_entry_decode_common (const char *entry,
             goto einval;
     }
 
-    if (!(o = json_loads (entry, 0, NULL)))
+    if (!(o = json_loads (entry, JSON_ALLOW_NUL, NULL)))
         goto einval;
 
     if (!eventlog_entry_validate (o)) {
@@ -339,7 +330,7 @@ char *eventlog_entry_encode (json_t *entry)
     return buf;
 }
 
-char *eventlog_encode (json_t *array)
+char *eventlog_encode (json_t *a)
 {
     json_t *value;
     size_t index;
@@ -347,11 +338,11 @@ char *eventlog_encode (json_t *array)
     int bufsz = 0;
     int used = 0;
 
-    if (!array || !json_is_array (array)) {
+    if (!a || !json_is_array (a)) {
         errno = EINVAL;
         return NULL;
     }
-    json_array_foreach (array, index, value) {
+    json_array_foreach (a, index, value) {
         char *s = json_dumps (value, JSON_COMPACT);
 
         if (!s || (used = append_string_nl (&buf, &bufsz, used, s)) < 0) {
@@ -367,6 +358,39 @@ char *eventlog_encode (json_t *array)
     return buf;
 }
 
+int eventlog_contains_event (const char *s, const char *name)
+{
+    json_t *a = NULL;
+    size_t index;
+    json_t *value;
+    int rv = -1;
+
+    if (!s || !name) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!(a = eventlog_decode (s)))
+        return -1;
+
+    json_array_foreach (a, index, value) {
+        double t;
+        const char *n;
+        json_t *c;
+        if (eventlog_entry_parse (value, &t, &n, &c) < 0)
+            goto out;
+        if (streq (name, n)) {
+            rv = 1;
+            goto out;
+        }
+    }
+
+    rv = 0;
+out:
+    json_decref (a);
+    return rv;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libeventlog/eventlog.h b/src/common/libeventlog/eventlog.h
index 2f67e78c01a1..7bb854bb0df4 100644
--- a/src/common/libeventlog/eventlog.h
+++ b/src/common/libeventlog/eventlog.h
@@ -13,6 +13,10 @@
 
 #include <jansson.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* convenience function to extract timestamp, name, and optional
  * context from an event entry */
 int eventlog_entry_parse (json_t *entry,
@@ -24,7 +28,7 @@ int eventlog_entry_parse (json_t *entry,
 json_t *eventlog_decode (const char *s);
 
 /* encode json array of event objects into an eventlog */
-char *eventlog_encode (json_t *o);
+char *eventlog_encode (json_t *a);
 
 /* decode a single eventlog entry into a json object */
 json_t *eventlog_entry_decode (const char *entry);
@@ -48,6 +52,15 @@ json_t *eventlog_entry_vpack (double timestamp,
 
 char *eventlog_entry_encode (json_t *entry);
 
+/* Convenience function to search eventlog for event with name.
+ * Returns 1 if found, 0 if not, -1 on error.
+ */
+int eventlog_contains_event (const char *s, const char *name);
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* !_EVENTLOG_H */
 
 /*
diff --git a/src/common/libeventlog/eventlogger.c b/src/common/libeventlog/eventlogger.c
new file mode 100644
index 000000000000..e4b63e0b726a
--- /dev/null
+++ b/src/common/libeventlog/eventlogger.c
@@ -0,0 +1,400 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <assert.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "eventlog.h"
+#include "eventlogger.h"
+
+struct eventlog_batch {
+    zlist_t *entries;
+    flux_kvs_txn_t *txn;
+    flux_watcher_t *timer;
+    struct eventlogger *ev;
+};
+
+struct eventlogger {
+    int refcount;
+    flux_t *h;
+    char *ns;
+    double batch_timeout;
+    double commit_timeout;
+    zlist_t *pending;
+    struct eventlog_batch *current;
+    struct eventlogger_ops ops;
+    void *arg;
+};
+
+static void eventlogger_decref (struct eventlogger *ev)
+{
+    if (ev && --ev->refcount == 0) {
+        free (ev->ns);
+        if (ev->pending) {
+            assert (zlist_size (ev->pending) == 0);
+            zlist_destroy (&ev->pending);
+        }
+        free (ev);
+    }
+}
+
+static void eventlogger_incref (struct eventlogger *ev)
+{
+    ev->refcount++;
+}
+
+int eventlogger_setns (struct eventlogger *ev, const char *ns)
+{
+    char *s = NULL;
+    if (!ev || !ns) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(s = strdup (ns)))
+        return -1;
+    free (ev->ns);
+    ev->ns = s;
+    return 0;
+}
+
+int eventlogger_set_commit_timeout (struct eventlogger *ev, double timeout)
+{
+    if (!ev || (timeout < 0. && timeout != -1.)) {
+        errno = EINVAL;
+        return -1;
+    }
+    ev->commit_timeout = timeout;
+    return 0;
+}
+
+static void eventlog_batch_destroy (struct eventlog_batch *batch)
+{
+    if (batch) {
+        if (batch->entries)
+            zlist_destroy (&batch->entries);
+        flux_watcher_destroy (batch->timer);
+        flux_kvs_txn_destroy (batch->txn);
+        free (batch);
+    }
+}
+
+static void eventlogger_batch_complete (struct eventlog_batch *batch)
+{
+    struct eventlogger *ev = batch->ev;
+
+    /*  zlist_remove destroys batch */
+    zlist_remove (ev->pending, batch);
+
+    /*  If no more batches on list, notify idle */
+    if (zlist_size (ev->pending) == 0) {
+        if (ev->ops.idle)
+            (*ev->ops.idle) (ev, ev->arg);
+        eventlogger_decref (ev);
+    }
+}
+
+static int eventlogger_batch_start (struct eventlogger *ev,
+                                    struct eventlog_batch *batch)
+{
+    if (zlist_append (ev->pending, batch) < 0)
+        return -1;
+    zlist_freefn (ev->pending,
+                  batch,
+                  (zlist_free_fn *) eventlog_batch_destroy,
+                  true);
+
+    /*  If we were idle before, now notify that eventlogger is busy */
+    if (zlist_size (ev->pending) == 1) {
+        if (ev->ops.busy)
+            (*ev->ops.busy) (ev, ev->arg);
+        eventlogger_incref (ev);
+    }
+    return 0;
+}
+
+static void eventlog_batch_error (struct eventlog_batch *batch, int errnum)
+{
+    struct eventlogger *ev = batch->ev;
+    json_t *entry;
+    if (!ev->ops.err)
+        return;
+    entry = zlist_first (batch->entries);
+    while (entry) {
+        (*ev->ops.err) (ev, ev->arg, errnum, entry);
+        entry = zlist_next (batch->entries);
+    }
+}
+
+static void commit_cb (flux_future_t *f, void *arg)
+{
+    struct eventlog_batch *batch = arg;
+    eventlogger_batch_complete (batch);
+    flux_future_destroy (f);
+}
+
+static flux_future_t *eventlogger_commit_batch (struct eventlogger *ev,
+                                                struct eventlog_batch *batch)
+{
+    flux_future_t *f = NULL;
+    flux_future_t *fc = NULL;
+    int flags = FLUX_KVS_TXN_COMPACT;
+
+    if (!batch) {
+        /*  If batch is NULL, return a fulfilled future immediately.
+         *
+         *  Note: There isn't much we can do if flux_future_create()
+         *   fails, until the eventlogger gets a logging interface.
+         *   Just return the NULL future to indicate some kind of
+         *   failure occurred. Likely other parts of the system are
+         *   in big trouble anyway...
+         */
+        if ((f = flux_future_create (NULL, NULL)))
+            flux_future_fulfill (f, NULL, NULL);
+    }
+    else {
+        /*  Otherwise, stop any pending timer watcher and start a
+         *   kvs commit operation. Call eventlogger_batch_complete()
+         *   when the commit is done, and return a future to the caller
+         *   that will be fulfilled on return from that function.
+         */
+        flux_watcher_stop (batch->timer);
+        if (!(fc = flux_kvs_commit (ev->h, ev->ns, flags, batch->txn)))
+            return NULL;
+        if (!(f = flux_future_and_then (fc, commit_cb, batch)))
+            flux_future_destroy (fc);
+    }
+    return f;
+}
+
+flux_future_t *eventlogger_commit (struct eventlogger *ev)
+{
+    struct eventlog_batch *batch = ev->current;
+
+    ev->current = NULL;
+    return eventlogger_commit_batch (ev, batch);
+}
+
+static void timer_commit_cb (flux_future_t *f, void *arg)
+{
+    struct eventlog_batch *batch = arg;
+    if (flux_future_get (f, NULL) < 0)
+        eventlog_batch_error (batch, errno);
+    flux_future_destroy (f);
+}
+
+static void
+timer_cb (flux_reactor_t *r, flux_watcher_t *w, int revents, void *arg)
+{
+    struct eventlog_batch *batch = arg;
+    double timeout = batch->ev->commit_timeout;
+    flux_future_t *f = NULL;
+
+    batch->ev->current = NULL;
+    if (!(f = eventlogger_commit_batch (batch->ev, batch))
+        || flux_future_then (f, timeout, timer_commit_cb, batch) < 0) {
+        eventlog_batch_error (batch, errno);
+        flux_future_destroy (f);
+    }
+}
+
+static struct eventlog_batch * eventlog_batch_create (struct eventlogger *ev)
+{
+    struct eventlog_batch *batch = calloc (1, sizeof (*batch));
+    if (!batch)
+        return NULL;
+    flux_reactor_t *r = flux_get_reactor (ev->h);
+    batch->ev = ev;
+    batch->entries = zlist_new ();
+    batch->txn = flux_kvs_txn_create ();
+    batch->timer = flux_timer_watcher_create (r,
+                                              ev->batch_timeout, 0.,
+                                              timer_cb,
+                                              batch);
+    if (!batch->entries || !batch->txn || !batch->timer) {
+        eventlog_batch_destroy (batch);
+        return NULL;
+    }
+    flux_watcher_start (batch->timer);
+    return batch;
+}
+
+void eventlogger_destroy (struct eventlogger *ev)
+{
+    eventlogger_decref (ev);
+}
+
+struct eventlogger *eventlogger_create (flux_t *h,
+                                        double timeout,
+                                        struct eventlogger_ops *ops,
+                                        void *arg)
+{
+    struct eventlogger *ev = calloc (1, sizeof (*ev));
+    if (ev) {
+        ev->pending = zlist_new ();
+        if (!ev->pending) {
+            eventlogger_destroy (ev);
+            return NULL;
+        }
+        ev->h = h;
+        ev->batch_timeout = timeout;
+        ev->commit_timeout = -1.;
+        ev->current = NULL;
+        ev->ops = *ops;
+        ev->arg = arg;
+        ev->refcount = 1;
+    }
+    return ev;
+}
+
+static struct eventlog_batch * eventlog_batch_get (struct eventlogger *ev)
+{
+    struct eventlog_batch *batch = ev->current;
+    if (!batch) {
+        if (!(ev->current = eventlog_batch_create (ev)))
+            return NULL;
+        batch = ev->current;
+        eventlogger_batch_start (ev, batch);
+    }
+    return batch;
+}
+
+static int append_wait (struct eventlogger *ev,
+                        const char *path,
+                        const char *entrystr)
+{
+    /*  append_wait also appends all pending transactions synchronously  */
+    struct eventlog_batch *batch = eventlog_batch_get (ev);
+    if (!batch)
+        return -1;
+
+    if (flux_kvs_txn_put (ev->current->txn,
+                          FLUX_KVS_APPEND,
+                          path, entrystr) < 0)
+        return -1;
+
+    return eventlogger_flush (ev);
+}
+
+static int append_async (struct eventlogger *ev,
+                         const char *path,
+                         json_t *entry,
+                         const char *entrystr)
+{
+    struct eventlog_batch *batch = eventlog_batch_get (ev);
+
+    if (!batch)
+        return -1;
+    if (flux_kvs_txn_put (ev->current->txn,
+                          FLUX_KVS_APPEND,
+                          path,
+                          entrystr) < 0)
+            return -1;
+
+    if (zlist_append (batch->entries, entry) < 0)
+        return -1;
+    json_incref (entry);
+    zlist_freefn (batch->entries,
+                  entry,
+                  (zlist_free_fn *) json_decref,
+                  true);
+    return 0;
+}
+
+int eventlogger_append_entry (struct eventlogger *ev,
+                              int flags,
+                              const char *path,
+                              json_t *entry)
+{
+    char *entrystr = NULL;
+    int rc = -1;
+
+    if (!(entrystr = eventlog_entry_encode (entry)))
+        return -1;
+
+    if (flags & EVENTLOGGER_FLAG_WAIT)
+        rc = append_wait (ev, path, entrystr);
+    else
+        rc = append_async (ev, path, entry, entrystr);
+    free (entrystr);
+    return rc;
+}
+
+int eventlogger_append (struct eventlogger *ev,
+                        int flags,
+                        const char *path,
+                        const char *name,
+                        const char *context)
+{
+    int rc = -1;
+    json_t *entry = NULL;
+
+    if (!(entry = eventlog_entry_create (0., name, context)))
+        goto out;
+    rc = eventlogger_append_entry (ev, flags, path, entry);
+out:
+    json_decref (entry);
+    return rc;
+}
+
+int eventlogger_append_vpack (struct eventlogger *ev,
+                              int flags,
+                              const char *path,
+                              const char *name,
+                              const char *fmt,
+                              va_list ap)
+{
+    int rc;
+    json_t *entry = NULL;
+
+    if (!(entry = eventlog_entry_vpack (0., name, fmt, ap)))
+        return -1;
+    rc = eventlogger_append_entry (ev, flags, path, entry);
+    json_decref (entry);
+    return rc;
+}
+
+int eventlogger_append_pack (struct eventlogger *ev,
+                             int flags,
+                             const char *path,
+                             const char *name,
+                             const char *fmt, ...)
+{
+    int rc = -1;
+    va_list ap;
+    va_start (ap, fmt);
+    rc = eventlogger_append_vpack (ev, flags, path, name, fmt, ap);
+    va_end (ap);
+    return rc;
+}
+
+int eventlogger_flush (struct eventlogger *ev)
+{
+    int rc = -1;
+    flux_future_t *f;
+
+    if (!(f = eventlogger_commit (ev))
+        || flux_future_wait_for (f, ev->commit_timeout) < 0)
+        goto out;
+    if ((rc = flux_future_get (f, NULL)) < 0)
+        eventlog_batch_error (eventlog_batch_get (ev), errno);
+out:
+    flux_future_destroy (f);
+    return rc;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libeventlog/eventlogger.h b/src/common/libeventlog/eventlogger.h
new file mode 100644
index 000000000000..23a90c374b4c
--- /dev/null
+++ b/src/common/libeventlog/eventlogger.h
@@ -0,0 +1,90 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_FLUX_EVENTLOGGER_H
+#define HAVE_FLUX_EVENTLOGGER_H
+
+#include <stdarg.h>
+#include <flux/core.h>
+#include <jansson.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct eventlogger;
+
+typedef void (*eventlogger_state_f) (struct eventlogger *ev, void *arg);
+typedef void (*eventlogger_err_f) (struct eventlogger *ev,
+                                   void *arg,
+                                   int err,
+                                   json_t *entry);
+
+struct eventlogger_ops {
+    eventlogger_state_f busy; /* Called after idle when starting a batch    */
+    eventlogger_state_f idle; /* Called when no more batches pending        */
+    eventlogger_err_f  err;   /* Called on error, once per failed entry     */
+};
+
+enum {
+    EVENTLOGGER_FLAG_ASYNC = 0,  /* Append entry to eventlog asynchronously */
+    EVENTLOGGER_FLAG_WAIT =  1,  /* Append entry to eventlog synchronously  */
+};
+
+/*  Create an eventlogger with batched eventlog appends at interval
+ *   `timeout`. Eventlogger will process user callbacks in `ops` as
+ *   defined above.
+ */
+struct eventlogger *eventlogger_create (flux_t *h,
+                                        double timeout,
+                                        struct eventlogger_ops *ops,
+                                        void *arg);
+
+/*  Set eventlogger namespace
+ */
+int eventlogger_setns (struct eventlogger *ev, const char *ns);
+
+void eventlogger_destroy (struct eventlogger *ev);
+
+int eventlogger_append (struct eventlogger *ev,
+                        int flags,
+                        const char *path,
+                        const char *name,
+                        const char *context);
+
+int eventlogger_append_entry (struct eventlogger *ev,
+                              int flags,
+                              const char *path,
+                              json_t *entry);
+
+int eventlogger_append_vpack (struct eventlogger *ev,
+                              int flags,
+                              const char *path,
+                              const char *name,
+                              const char *fmt,
+                              va_list ap);
+
+int eventlogger_append_pack (struct eventlogger *ev,
+                             int flags,
+                             const char *path,
+                             const char *name,
+                             const char *fmt, ...);
+
+int eventlogger_set_commit_timeout (struct eventlogger *ev, double timeout);
+
+int eventlogger_flush (struct eventlogger *ev);
+
+flux_future_t *eventlogger_commit (struct eventlogger *ev);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !HAVE_FLUX_EVENTLOGGER_H */
diff --git a/src/common/libeventlog/formatter.c b/src/common/libeventlog/formatter.c
new file mode 100644
index 000000000000..9e1887c3c140
--- /dev/null
+++ b/src/common/libeventlog/formatter.c
@@ -0,0 +1,477 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <unistd.h>
+#include <string.h>
+#include <time.h>
+#include <errno.h>
+
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "ccan/str/str.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/timestamp.h"
+#include "src/common/libutil/ansi_color.h"
+
+#include "eventlog.h"
+#include "formatter.h"
+
+enum {
+    EVENTLOG_COLOR_NAME,
+    EVENTLOG_COLOR_TIME,
+    EVENTLOG_COLOR_TIMEBREAK,
+    EVENTLOG_COLOR_KEY,
+    EVENTLOG_COLOR_VALUE,
+    EVENTLOG_COLOR_VALUE_NUM,
+    EVENTLOG_COLOR_EXCEPTION,
+};
+
+static const char *eventlog_colors[] = {
+    [EVENTLOG_COLOR_NAME]      = ANSI_COLOR_YELLOW,
+    [EVENTLOG_COLOR_TIME]      = ANSI_COLOR_GREEN,
+    [EVENTLOG_COLOR_TIMEBREAK] = ANSI_COLOR_BOLD ANSI_COLOR_GREEN,
+    [EVENTLOG_COLOR_KEY]       = ANSI_COLOR_BLUE,
+    [EVENTLOG_COLOR_VALUE]     = ANSI_COLOR_MAGENTA,
+    [EVENTLOG_COLOR_VALUE_NUM] = ANSI_COLOR_GRAY,
+    [EVENTLOG_COLOR_EXCEPTION] = ANSI_COLOR_BOLD ANSI_COLOR_RED,
+};
+
+enum entry_format {
+    EVENTLOG_ENTRY_TEXT,
+    EVENTLOG_ENTRY_JSON
+};
+
+enum timestamp_format {
+    EVENTLOG_TIMESTAMP_RAW,
+    EVENTLOG_TIMESTAMP_ISO,
+    EVENTLOG_TIMESTAMP_OFFSET,
+    EVENTLOG_TIMESTAMP_HUMAN,
+};
+
+
+struct eventlog_formatter {
+    /*  End of line separator for entries */
+    const char *endl;
+
+    /*  Emit unformatted output (RFC 18 JSON) */
+    unsigned int unformatted:1;
+
+    /*  Enable color: */
+    unsigned int color:1;
+
+    /*  Also color event context key/values: */
+    unsigned int context_color:1;
+
+    /*  Timestamp and entry formats for non RFC 18 operation: */
+    enum timestamp_format ts_format;
+    enum entry_format entry_format;
+
+    /*  Initial timestamp */
+    double t0;
+
+    /*  For relative timestamp output */
+    struct tm last_tm;
+    double last_ts;
+};
+
+void eventlog_formatter_destroy (struct eventlog_formatter *evf)
+{
+    if (evf) {
+        int saved_errno = errno;
+        free (evf);
+        errno = saved_errno;
+    }
+}
+
+struct eventlog_formatter *eventlog_formatter_create ()
+{
+    struct eventlog_formatter *evf;
+
+    if (!(evf = calloc (1, sizeof (*evf))))
+        return NULL;
+    evf->endl = "\n";
+    return evf;
+}
+
+int eventlog_formatter_colors_init (struct eventlog_formatter *evf,
+                                    const char *when)
+{
+    if (!evf || !when) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (streq (when, "always"))
+        evf->color = 1;
+    else if (streq (when, "never"))
+        evf->color = 0;
+    else if (streq (when, "auto"))
+        evf->color = isatty (STDOUT_FILENO) ? 1 : 0;
+    else {
+        errno = EINVAL;
+        return -1;
+    }
+    /* For now, always enable context colorization if evf->color is set:
+     * (This is a separate variable to allow for future possible disablement)
+     */
+    evf->context_color = evf->color;
+    return 0;
+}
+
+void eventlog_formatter_set_no_newline (struct eventlog_formatter *evf)
+{
+    if (evf) {
+        evf->endl = "";
+    }
+}
+
+void eventlog_formatter_update_t0 (struct eventlog_formatter *evf, double ts)
+{
+    if (evf) {
+        if (evf->t0 == 0.)
+            evf->t0 = ts;
+    }
+}
+
+int eventlog_formatter_set_timestamp_format (struct eventlog_formatter *evf,
+                                             const char *format)
+{
+    if (!evf || !format) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (strcasecmp (format, "raw") == 0)
+        evf->ts_format = EVENTLOG_TIMESTAMP_RAW;
+    else if (strcasecmp (format, "human") == 0
+             || strcasecmp (format, "reltime") == 0)
+        evf->ts_format = EVENTLOG_TIMESTAMP_HUMAN;
+    else if (strcasecmp (format, "iso") == 0)
+        evf->ts_format = EVENTLOG_TIMESTAMP_ISO;
+    else if (strcasecmp (format, "offset") == 0)
+        evf->ts_format = EVENTLOG_TIMESTAMP_OFFSET;
+    else {
+        errno = EINVAL;
+        return -1;
+    }
+    return 0;
+}
+
+int eventlog_formatter_set_format (struct eventlog_formatter *evf,
+                                   const char *format)
+{
+    if (!evf || !format) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (strcasecmp (format, "text") == 0)
+        evf->unformatted = false;
+    else if (strcasecmp (format, "json") == 0)
+        evf->unformatted = true;
+    else {
+        errno = EINVAL;
+        return -1;
+    }
+    return 0;
+}
+
+void eventlog_formatter_reset (struct eventlog_formatter *evf)
+{
+    if (evf) {
+        evf->t0 = 0.;
+        evf->last_ts = 0.;
+        memset (&evf->last_tm, 0, sizeof (struct tm));
+    }
+}
+
+static const char *eventlog_color (struct eventlog_formatter *evf, int type)
+{
+    if (evf->color)
+        return eventlog_colors [type];
+    return "";
+}
+
+/*  Color the name of specific events
+ */
+static const char *eventlog_color_event_name (struct eventlog_formatter *evf,
+                                              const char *name)
+{
+    if (evf->color) {
+        if (streq (name, "exception"))
+            return eventlog_colors [EVENTLOG_COLOR_EXCEPTION];
+        else
+            return eventlog_colors [EVENTLOG_COLOR_NAME];
+    }
+    return "";
+}
+
+static const char *eventlog_context_color (struct eventlog_formatter *evf,
+                                           int type)
+{
+    if (evf->context_color)
+        return eventlog_color (evf, type);
+    return "";
+}
+
+static const char *eventlog_color_reset (struct eventlog_formatter *evf)
+{
+    if (evf->color)
+        return ANSI_COLOR_RESET;
+    return "";
+}
+
+static const char *
+eventlog_context_color_reset (struct eventlog_formatter *evf)
+{
+    if (evf->context_color)
+        return eventlog_color_reset (evf);
+    return "";
+}
+
+static const char *months[] = {
+    "Jan",
+    "Feb",
+    "Mar",
+    "Apr",
+    "May",
+    "Jun",
+    "Jul",
+    "Aug",
+    "Sep",
+    "Oct",
+    "Nov",
+    "Dec",
+    NULL
+};
+
+static int event_timestamp_human (struct eventlog_formatter *evf,
+                                  double timestamp,
+                                  char *buf,
+                                  size_t size)
+{
+    struct tm tm;
+
+    if (timestamp_from_double (timestamp, &tm, NULL) < 0)
+        return snprintf (buf,
+                         size,
+                         "%s[%+11.6f]%s",
+                         eventlog_color (evf, EVENTLOG_COLOR_TIME),
+                         timestamp,
+                         eventlog_color_reset (evf));
+
+    if (tm.tm_year == evf->last_tm.tm_year
+        && tm.tm_mon == evf->last_tm.tm_mon
+        && tm.tm_mday == evf->last_tm.tm_mday
+        && tm.tm_hour == evf->last_tm.tm_hour
+        && tm.tm_min == evf->last_tm.tm_min) {
+        /*  Within same minute, print offset in sec */
+        return snprintf (buf,
+                         size,
+                         "%s[%+11.6f]%s",
+                         eventlog_color (evf, EVENTLOG_COLOR_TIME),
+                         timestamp - evf->last_ts,
+                         eventlog_color_reset (evf));
+    }
+    /* New minute. Save last timestamp,  print datetime */
+    evf->last_ts = timestamp;
+    evf->last_tm = tm;
+
+    return snprintf (buf,
+                     size,
+                     "%s[%s%02d %02d:%02d]%s",
+                     eventlog_color (evf, EVENTLOG_COLOR_TIMEBREAK),
+                     months [tm.tm_mon],
+                     tm.tm_mday,
+                     tm.tm_hour,
+                     tm.tm_min,
+                     eventlog_color_reset (evf));
+}
+
+
+static int event_timestamp (struct eventlog_formatter *evf,
+                            flux_error_t *errp,
+                            double timestamp,
+                            char *buf,
+                            size_t size)
+{
+    if (evf->ts_format == EVENTLOG_TIMESTAMP_RAW) {
+        if (snprintf (buf,
+                      size,
+                      "%s%lf%s",
+                      eventlog_color (evf, EVENTLOG_COLOR_TIME),
+                      timestamp,
+                      eventlog_color_reset (evf)) >= size)
+            return errprintf (errp, "buffer truncated writing timestamp");
+    }
+    else if (evf->ts_format == EVENTLOG_TIMESTAMP_HUMAN) {
+        if (event_timestamp_human (evf, timestamp, buf, size) >= size)
+            return errprintf (errp,
+                              "buffer truncated writing relative timestamp");
+    }
+    else if (evf->ts_format == EVENTLOG_TIMESTAMP_ISO) {
+        time_t sec = timestamp;
+        unsigned long usec = (timestamp - sec)*1E6;
+        struct tm tm;
+        char tz[16];
+
+        if (!localtime_r (&sec, &tm))
+            return errprintf (errp,
+                              "localtime(%lf): %s",
+                              timestamp,
+                              strerror (errno));
+        if (snprintf (buf,
+                      size,
+                      "%s",
+                      eventlog_color (evf, EVENTLOG_COLOR_TIME)) >= size)
+            return errprintf (errp,
+                              "failed to write timestamp color to buffer");
+        size -= strlen (buf);
+        buf += strlen (buf);
+        if (strftime (buf, size, "%Y-%m-%dT%T", &tm) == 0
+            || timestamp_tzoffset (&tm, tz, sizeof (tz)) < 0)
+            return errprintf (errp,
+                              "strftime(%lf): %s",
+                              timestamp,
+                              strerror (errno));
+        size -= strlen (buf);
+        buf += strlen (buf);
+        if (snprintf (buf,
+                      size,
+                      ".%.6lu%s%s",
+                      usec,
+                      tz,
+                      eventlog_color_reset (evf)) >= size)
+            return errprintf (errp,
+                              "buffer truncated writing ISO 8601 timestamp");
+    }
+    else { /* EVENTLOG_TIMESTAMP_OFFSET */
+        eventlog_formatter_update_t0 (evf, timestamp);
+        timestamp -= evf->t0;
+        if (snprintf (buf,
+                      size,
+                      "%s%lf%s",
+                      eventlog_color (evf, EVENTLOG_COLOR_TIME),
+                      timestamp,
+                      eventlog_color_reset (evf)) >= size)
+            return errprintf (errp,
+                              "buffer truncated writing timestamp offset");
+    }
+    return 0;
+}
+
+static int entry_format_text (struct eventlog_formatter *evf,
+                              flux_error_t *errp,
+                              json_t *event,
+                              double timestamp,
+                              const char *name,
+                              json_t *context,
+                              FILE *fp)
+{
+    char ts[128];
+
+    if (event_timestamp (evf, errp, timestamp, ts, sizeof (ts)) < 0)
+        return -1;
+
+    if (fprintf (fp,
+                 "%s %s%s%s",
+                 ts,
+                 eventlog_color_event_name (evf, name),
+                 name,
+                 eventlog_color_reset (evf)) < 0)
+        return errprintf (errp, "fprintf: %s", strerror (errno));
+
+    if (context) {
+        const char *key;
+        json_t *value;
+        json_object_foreach (context, key, value) {
+            char *sval;
+            int rc;
+            int color = EVENTLOG_COLOR_VALUE;
+            if (json_is_number (value))
+                color = EVENTLOG_COLOR_VALUE_NUM;
+            sval = json_dumps (value, JSON_ENCODE_ANY|JSON_COMPACT);
+            rc = fprintf (fp,
+                          " %s%s%s=%s%s%s",
+                          eventlog_context_color (evf, EVENTLOG_COLOR_KEY),
+                          key,
+                          eventlog_context_color_reset (evf),
+                          eventlog_context_color (evf, color),
+                          sval,
+                          eventlog_context_color_reset (evf));
+            free (sval);
+            if (rc < 0)
+                return errprintf (errp, "fprintf: %s", strerror (errno));
+        }
+    }
+
+    fputs (evf->endl, fp);
+    return 0;
+}
+
+int eventlog_entry_dumpf (struct eventlog_formatter *evf,
+                          FILE *fp,
+                          flux_error_t *errp,
+                          json_t *event)
+{
+    double timestamp;
+    json_t *context;
+    const char *name;
+
+    if (!evf || !event || !fp) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    /* Validity check:
+     */
+    if (eventlog_entry_parse (event, &timestamp, &name, &context) < 0)
+        return errprintf (errp, "eventlog_entry_parse: %s", strerror (errno));
+
+    if (evf->unformatted) {
+        if (json_dumpf (event, fp, JSON_COMPACT) < 0)
+            return errprintf (errp, "json_dumpf failed");
+        fputs (evf->endl, fp);
+        return 0;
+    }
+
+    return entry_format_text (evf, errp, event, timestamp, name, context, fp);
+}
+
+char *eventlog_entry_dumps (struct eventlog_formatter *evf,
+                            flux_error_t *errp,
+                            json_t *event)
+{
+    size_t size;
+    char *result = NULL;
+    FILE *fp;
+
+    if (!evf || !event) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(fp = open_memstream (&result, &size))) {
+        errprintf (errp, "open_memstream: %s", strerror (errno));
+        return NULL;
+    }
+    if (eventlog_entry_dumpf (evf, fp, errp, event) < 0) {
+        fclose (fp);
+        free (result);
+        return NULL;
+    }
+    fclose (fp);
+    return result;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libeventlog/formatter.h b/src/common/libeventlog/formatter.h
new file mode 100644
index 000000000000..fc46c1221c09
--- /dev/null
+++ b/src/common/libeventlog/formatter.h
@@ -0,0 +1,72 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _EVENTLOG_FORMATTER_H
+#define _EVENTLOG_FORMATTER_H
+
+#include <stdio.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+
+struct eventlog_formatter *eventlog_formatter_create (void);
+void eventlog_formatter_destroy (struct eventlog_formatter *evf);
+
+/*  Set color: when can be "always", "never", or "auto":
+ */
+int eventlog_formatter_colors_init (struct eventlog_formatter *evf,
+                                    const char *when);
+
+/*  Reset an eventlog formatter. (Clear t0 timestamp).
+ *  Formatting options remain unchanged.
+ */
+void eventlog_formatter_reset (struct eventlog_formatter *evf);
+
+/*  Update formatter initial timestamp if not currently set.
+ */
+void eventlog_formatter_update_t0 (struct eventlog_formatter *evf, double ts);
+
+/*  Set the timestamp format for output. String 'format' may be one of
+ *  "raw"    - raw double precision timestamp (default)
+ *  "iso"    - ISO 8601
+ *  "offset" - offset from initial timestamp in eventlog.
+ */
+int eventlog_formatter_set_timestamp_format (struct eventlog_formatter *evf,
+                                             const char *format);
+
+/*  Set eventlog entry format by name for formatter 'evf':
+ *  "text" - formatted text (default)
+ *  "json" - unformatted raw JSON output (if set, timestamp format is ignored)
+ */
+int eventlog_formatter_set_format (struct eventlog_formatter *evf,
+                                   const char *format);
+
+/*  Disable newline character after eventlog_entry_dumps/f().
+ */
+void eventlog_formatter_set_no_newline (struct eventlog_formatter *evf);
+
+/*  Dump eventlog entry encoded by formatter 'evf' to a string.
+ *  Caller must free result.
+ *  Returns 0 on success, -1 with errno set and error text in errp->text.
+ */
+char *eventlog_entry_dumps (struct eventlog_formatter *evf,
+                            flux_error_t *errp,
+                            json_t *event);
+
+/*  Dump the eventlog entry encoded by 'evf' to stream 'fp'.
+ *  Returns 0 on success, -1 with errno set and error text in errp->text.
+ */
+int eventlog_entry_dumpf (struct eventlog_formatter *evf,
+                          FILE *fp,
+                          flux_error_t *errp,
+                          json_t *event);
+
+
+#endif /* !_EVENTLOG_FORMATTER_H */
diff --git a/src/common/libeventlog/test/eventlog.c b/src/common/libeventlog/test/eventlog.c
index 68ddbdfa6e32..eec8dac75ca6 100644
--- a/src/common/libeventlog/test/eventlog.c
+++ b/src/common/libeventlog/test/eventlog.c
@@ -8,12 +8,16 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
 #include <string.h>
 #include <stdarg.h>
 #include <stdbool.h>
 
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 #include "src/common/libeventlog/eventlog.h"
 
 void eventlog_entry_parsing (void)
@@ -62,7 +66,7 @@ void eventlog_entry_parsing (void)
 
     ok (eventlog_entry_parse (event, &timestamp, &name, &context) == 0
         && timestamp == 42.0
-        && !strcmp (name, "foo")
+        && streq (name, "foo")
         && !context,
         "eventlog_entry_parse on event w/o context works");
     json_decref (event);
@@ -77,12 +81,12 @@ void eventlog_entry_parsing (void)
 
     ok (eventlog_entry_parse (event, &timestamp, &name, &context) == 0
         && timestamp == 52.0
-        && !strcmp (name, "bar")
+        && streq (name, "bar")
         && context
         && json_is_object (context)
         && (jstr = json_object_get (context, "foo"))
         && (str = json_string_value (jstr))
-        && !strcmp (str, "bar"),
+        && streq (str, "bar"),
         "eventlog_entry_parse on event w/ context works");
     json_decref (event);
 }
@@ -172,7 +176,7 @@ void eventlog_decoding (void)
             "eventlog_decode input=\"%s\" success",
             printable (buf, goodevent[i]));
         s = eventlog_encode (o);
-        ok (s != NULL && !strcmp (s, goodevent[i]),
+        ok (s != NULL && streq (s, goodevent[i]),
             "eventlog_encode reversed it");
         json_decref (o);
         free (s);
@@ -184,7 +188,7 @@ void eventlog_decoding (void)
             "eventlog_decode input=\"%s\" success",
             printable (buf, goodlog[i]));
         s = eventlog_encode (o);
-        ok (s != NULL && !strcmp (s, goodlog[i]),
+        ok (s != NULL && streq (s, goodlog[i]),
             "eventlog_encode reversed it");
         json_decref (o);
         free (s);
@@ -289,9 +293,9 @@ void eventlog_entry_check (json_t *entry, double xtimestamp, const char *xname,
         context_str = json_dumps (context, JSON_COMPACT);
 
     ok ((xtimestamp == 0. || timestamp == xtimestamp)
-        && (!xname || !strcmp (name, xname))
+        && (!xname || streq (name, xname))
         && ((!xcontext && !context)
-            || (context_str && !strcmp (context_str, xcontext))),
+            || (context_str && streq (context_str, xcontext))),
         "eventlog_entry_parse time=%llu name=%s context=%s",
         (unsigned long long)xtimestamp, xname, xcontext);
 
@@ -479,6 +483,34 @@ void eventlog_entry_encoding_errors (void)
         "eventlog_entry_vpack context=\"[\"foo\"]\" fails with EINVAL");
 }
 
+void eventlog_contains_event_test (void)
+{
+    char *goodlog =
+        "{\"timestamp\":42.0,\"name\":\"foo\"}\n"
+        "{\"timestamp\":43.0,\"name\":\"bar\",\"context\":{\"bar\":16}}\n";
+    char *badlog = "fdsafdsafsdafd";
+
+    errno = 0;
+    ok (eventlog_contains_event (NULL, NULL) < 0
+        && errno == EINVAL,
+        "eventlog_contains_event returns EINVAL on bad input");
+
+    errno = 0;
+    ok (eventlog_contains_event (badlog, "foo") < 0
+        && errno == EINVAL,
+        "eventlog_contains_event returns EINVAL on bad log");
+
+    ok (eventlog_contains_event ("", "foo") == 0,
+        "eventlog_contains_event returns 0, no events in eventlog");
+
+    ok (eventlog_contains_event (goodlog, "foo") == 1,
+        "eventlog_contains_event returns 1, found foo event in eventlog");
+    ok (eventlog_contains_event (goodlog, "bar") == 1,
+        "eventlog_contains_event returns 1, found bar event in eventlog");
+    ok (eventlog_contains_event (goodlog, "foobar") == 0,
+        "eventlog_contains_event returns 0, no foobar event in eventlog");
+}
+
 int main (int argc, char *argv[])
 {
     plan (NO_PLAN);
@@ -490,6 +522,7 @@ int main (int argc, char *argv[])
     eventlog_entry_decoding_errors ();
     eventlog_entry_encoding ();
     /* eventlog_entry_encoding_errors (); */
+    eventlog_contains_event_test ();
 
     done_testing ();
 }
diff --git a/src/common/libeventlog/test/formatter.c b/src/common/libeventlog/test/formatter.c
new file mode 100644
index 000000000000..be6129d3b228
--- /dev/null
+++ b/src/common/libeventlog/test/formatter.c
@@ -0,0 +1,355 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <string.h>
+#include <time.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libeventlog/formatter.h"
+
+struct test_entry {
+    const char *input;
+    const char *raw;
+    const char *iso;
+    const char *offset;
+    const char *human;
+};
+
+/*  This input set was constructed from an real eventlog.
+ *  Events must be kept in sequence so that offsets are calculated
+ *  correctly
+ */
+struct test_entry tests[] = {
+    { "{\"timestamp\":1699995759.5377746,\"name\":\"submit\",\"context\":{\"userid\":1001,\"urgency\":16,\"flags\":0,\"version\":1}}",
+      "1699995759.537775 submit userid=1001 urgency=16 flags=0 version=1",
+      "2023-11-14T21:02:39.537774Z submit userid=1001 urgency=16 flags=0 version=1",
+      "0.000000 submit userid=1001 urgency=16 flags=0 version=1",
+      "\x1b[1m\x1b[32m[Nov14 21:02]\x1b[0m \x1b[33msubmit\x1b[0m \x1b[34muserid\x1b[0m=\x1b[37m1001\x1b[0m \x1b[34murgency\x1b[0m=\x1b[37m16\x1b[0m \x1b[34mflags\x1b[0m=\x1b[37m0\x1b[0m \x1b[34mversion\x1b[0m=\x1b[37m1\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1699995759.5597851,\"name\":\"validate\"}",
+      "1699995759.559785 validate",
+      "2023-11-14T21:02:39.559785Z validate",
+      "0.022011 validate",
+      "\x1b[32m[  +0.022011]\x1b[0m \x1b[33mvalidate\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1699995759.5738351,\"name\":\"depend\"}",
+      "1699995759.573835 depend",
+      "2023-11-14T21:02:39.573835Z depend",
+      "0.036061 depend",
+      "\x1b[32m[  +0.036061]\x1b[0m \x1b[33mdepend\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1699995759.5739679,\"name\":\"priority\",\"context\":{\"priority\":66963}}",
+      "1699995759.573968 priority priority=66963",
+      "2023-11-14T21:02:39.573967Z priority priority=66963",
+      "0.036193 priority priority=66963",
+      "\x1b[32m[  +0.036193]\x1b[0m \x1b[33mpriority\x1b[0m \x1b[34mpriority\x1b[0m=\x1b[37m66963\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1699995759.6047542,\"name\":\"alloc\"}",
+      "1699995759.604754 alloc",
+      "2023-11-14T21:02:39.604754Z alloc",
+      "0.066980 alloc",
+      "\x1b[32m[  +0.066980]\x1b[0m \x1b[33malloc\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1699995759.6055193,\"name\":\"prolog-start\",\"context\":{\"description\":\"job-manager.prolog\"}}",
+      "1699995759.605519 prolog-start description=\"job-manager.prolog\"",
+      "2023-11-14T21:02:39.605519Z prolog-start description=\"job-manager.prolog\"",
+      "0.067745 prolog-start description=\"job-manager.prolog\"",
+      "\x1b[32m[  +0.067745]\x1b[0m \x1b[33mprolog-start\x1b[0m \x1b[34mdescription\x1b[0m=\x1b[35m\"job-manager.prolog\"\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1699995759.6055939,\"name\":\"prolog-start\",\"context\":{\"description\":\"cray-pals-port-distributor\"}}",
+      "1699995759.605594 prolog-start description=\"cray-pals-port-distributor\"",
+      "2023-11-14T21:02:39.605593Z prolog-start description=\"cray-pals-port-distributor\"",
+      "0.067819 prolog-start description=\"cray-pals-port-distributor\"",
+      "\x1b[32m[  +0.067819]\x1b[0m \x1b[33mprolog-start\x1b[0m \x1b[34mdescription\x1b[0m=\x1b[35m\"cray-pals-port-distributor\"\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1699995759.7634473,\"name\":\"prolog-finish\",\"context\":{\"description\":\"cray-pals-port-distributor\",\"status\":0}}",
+      "1699995759.763447 prolog-finish description=\"cray-pals-port-distributor\" status=0",
+      "2023-11-14T21:02:39.763447Z prolog-finish description=\"cray-pals-port-distributor\" status=0",
+      "0.225673 prolog-finish description=\"cray-pals-port-distributor\" status=0",
+      "\x1b[32m[  +0.225673]\x1b[0m \x1b[33mprolog-finish\x1b[0m \x1b[34mdescription\x1b[0m=\x1b[35m\"cray-pals-port-distributor\"\x1b[0m \x1b[34mstatus\x1b[0m=\x1b[37m0\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1699995760.3795953,\"name\":\"prolog-finish\",\"context\":{\"description\":\"job-manager.prolog\",\"status\":0}}",
+      "1699995760.379595 prolog-finish description=\"job-manager.prolog\" status=0",
+      "2023-11-14T21:02:40.379595Z prolog-finish description=\"job-manager.prolog\" status=0",
+      "0.841821 prolog-finish description=\"job-manager.prolog\" status=0",
+      "\x1b[32m[  +0.841821]\x1b[0m \x1b[33mprolog-finish\x1b[0m \x1b[34mdescription\x1b[0m=\x1b[35m\"job-manager.prolog\"\x1b[0m \x1b[34mstatus\x1b[0m=\x1b[37m0\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1699995760.3859105,\"name\":\"start\"}",
+      "1699995760.385911 start",
+      "2023-11-14T21:02:40.385910Z start",
+      "0.848136 start",
+      "\x1b[32m[  +0.848136]\x1b[0m \x1b[33mstart\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1699995760.7054179,\"name\":\"memo\",\"context\":{\"uri\":\"ssh://host/var/tmp/user/flux-0QZyMU/local-0\"}}",
+      "1699995760.705418 memo uri=\"ssh://host/var/tmp/user/flux-0QZyMU/local-0\"",
+      "2023-11-14T21:02:40.705417Z memo uri=\"ssh://host/var/tmp/user/flux-0QZyMU/local-0\"",
+      "1.167643 memo uri=\"ssh://host/var/tmp/user/flux-0QZyMU/local-0\"",
+      "\x1b[32m[  +1.167643]\x1b[0m \x1b[33mmemo\x1b[0m \x1b[34muri\x1b[0m=\x1b[35m\"ssh://host/var/tmp/user/flux-0QZyMU/local-0\"\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1700074161.0240808,\"name\":\"finish\",\"context\":{\"status\":0}}",
+      "1700074161.024081 finish status=0",
+      "2023-11-15T18:49:21.024080Z finish status=0",
+      "78401.486306 finish status=0",
+      "\x1b[1m\x1b[32m[Nov15 18:49]\x1b[0m \x1b[33mfinish\x1b[0m \x1b[34mstatus\x1b[0m=\x1b[37m0\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1700074161.0250554,\"name\":\"epilog-start\",\"context\":{\"description\":\"job-manager.epilog\"}}",
+      "1700074161.025055 epilog-start description=\"job-manager.epilog\"",
+      "2023-11-15T18:49:21.025055Z epilog-start description=\"job-manager.epilog\"",
+      "78401.487281 epilog-start description=\"job-manager.epilog\"",
+      "\x1b[32m[  +0.000975]\x1b[0m \x1b[33mepilog-start\x1b[0m \x1b[34mdescription\x1b[0m=\x1b[35m\"job-manager.epilog\"\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1700074161.1864166,\"name\":\"release\",\"context\":{\"ranks\":\"all\",\"final\":true}}",
+      "1700074161.186417 release ranks=\"all\" final=true",
+      "2023-11-15T18:49:21.186416Z release ranks=\"all\" final=true",
+      "78401.648642 release ranks=\"all\" final=true",
+      "\x1b[32m[  +0.162336]\x1b[0m \x1b[33mrelease\x1b[0m \x1b[34mranks\x1b[0m=\x1b[35m\"all\"\x1b[0m \x1b[34mfinal\x1b[0m=\x1b[35mtrue\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1700074445.1199436,\"name\":\"epilog-finish\",\"context\":{\"description\":\"job-manager.epilog\",\"status\":0}}",
+      "1700074445.119944 epilog-finish description=\"job-manager.epilog\" status=0",
+      "2023-11-15T18:54:05.119943Z epilog-finish description=\"job-manager.epilog\" status=0",
+      "78685.582169 epilog-finish description=\"job-manager.epilog\" status=0",
+      "\x1b[1m\x1b[32m[Nov15 18:54]\x1b[0m \x1b[33mepilog-finish\x1b[0m \x1b[34mdescription\x1b[0m=\x1b[35m\"job-manager.epilog\"\x1b[0m \x1b[34mstatus\x1b[0m=\x1b[37m0\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1700074445.1203697,\"name\":\"free\"}",
+      "1700074445.120370 free",
+      "2023-11-15T18:54:05.120369Z free",
+      "78685.582595 free",
+      "\x1b[32m[  +0.000426]\x1b[0m \x1b[33mfree\x1b[0m",
+    },
+    {
+      "{\"timestamp\":1700074445.120451,\"name\":\"clean\"}",
+      "1700074445.120451 clean",
+      "2023-11-15T18:54:05.120450Z clean",
+      "78685.582676 clean",
+      "\x1b[32m[  +0.000507]\x1b[0m \x1b[33mclean\x1b[0m",
+    },
+    {0},
+};
+
+static void test_basic ()
+{
+    struct test_entry *test;
+    flux_error_t error;
+    struct eventlog_formatter *evf;
+    if (!(evf = eventlog_formatter_create ()))
+        BAIL_OUT ("failed to create eventlog formatter");
+
+    eventlog_formatter_set_no_newline (evf);
+
+    test = tests;
+    while (test->input) {
+        char *result;
+        json_t *entry = json_loads (test->input, 0, NULL);
+        if (!entry)
+            BAIL_OUT ("failed to load JSON input '%s'", test->input);
+
+        /* Disable color for tests */
+        ok (eventlog_formatter_colors_init (evf, "never") == 0,
+            "eventlog_formatter_colors_init (never)");
+
+        /* 1. Test unformatted, should equal input
+         */
+        ok (eventlog_formatter_set_format (evf, "json") == 0,
+            "eventlog_formatter_set_format(json)");
+        ok ((result = eventlog_entry_dumps (evf, &error, entry)) != NULL,
+            "eventlog_entry_dumps");
+        diag (result);
+        is (result, test->input,
+            "json output is expected");
+        free (result);
+        /*  Need to reset unformatted flag for other tests:
+         */
+        ok (eventlog_formatter_set_format (evf, "text") == 0,
+            "eventlog_formatter_set_format (text)");
+
+        /*  2. Test raw timestamp
+         */
+        ok (eventlog_formatter_set_timestamp_format (evf, "raw") == 0,
+            "eventlog_formatter_set_timestamp_format raw works");
+        ok ((result = eventlog_entry_dumps (evf, &error, entry)) != NULL,
+            "eventlog_entry_dumps");
+        diag (result);
+        is (result, test->raw,
+            "raw timestamp output is expected");
+        free (result);
+
+        /* 3. Test ISO timestamp
+         */
+        ok (eventlog_formatter_set_timestamp_format (evf, "iso") == 0,
+            "eventlog_formatter_set_timestamp_format iso works");
+        ok ((result = eventlog_entry_dumps (evf, &error, entry)) != NULL,
+            "eventlog_entry_dumps");
+        diag (result);
+        is (result, test->iso,
+            "iso timestamp output is expected");
+        free (result);
+
+        /* 4. Test offset timestamp
+         */
+        ok (eventlog_formatter_set_timestamp_format (evf, "offset") == 0,
+            "eventlog_formatter_set_timestamp_format offset works");
+        ok ((result = eventlog_entry_dumps (evf, &error, entry)) != NULL,
+            "eventlog_entry_dumps");
+        diag (result);
+        is (result, test->offset,
+            "offset timestamp output is expected");
+        free (result);
+
+        /* 5. Test "reltime"/"human" timestamp with color
+         */
+        ok (eventlog_formatter_colors_init (evf, "always") == 0,
+            "eventlog_formatter_colors_init (always)");
+        ok (eventlog_formatter_set_timestamp_format (evf, "human") == 0,
+            "eventlog_formatter_set_timestamp_format human works");
+        ok ((result = eventlog_entry_dumps (evf, &error, entry)) != NULL,
+            "eventlog_entry_dumps");
+        diag (result);
+        is (result, test->human,
+            "human timestamp output is expected");
+        free (result);
+
+        json_decref (entry);
+        test++;
+    }
+
+    eventlog_formatter_destroy (evf);
+}
+
+static void test_invalid ()
+{
+    struct eventlog_formatter *evf;
+    json_t *good, *bad;
+    flux_error_t error;
+
+    if (!(evf = eventlog_formatter_create ()))
+        BAIL_OUT ("failed to create eventlog formatter");
+
+    lives_ok ({eventlog_formatter_destroy (NULL);});
+    lives_ok ({eventlog_formatter_reset (NULL);});
+    lives_ok ({eventlog_formatter_set_no_newline (NULL);});
+    lives_ok ({eventlog_formatter_update_t0 (NULL, 0.);});
+
+    ok (eventlog_formatter_set_timestamp_format (NULL, "") < 0
+        && errno == EINVAL,
+        "eventlog_formatter_set_timestamp_format (NULL) returns EINVAL");
+    ok (eventlog_formatter_set_timestamp_format (evf, NULL) < 0
+        && errno == EINVAL,
+        "eventlog_formatter_set_timestamp_format (evf, NULL) returns EINVAL");
+    ok (eventlog_formatter_set_timestamp_format (evf, "") < 0
+        && errno == EINVAL,
+        "eventlog_formatter_set_timestamp_format (evf, \"\") returns EINVAL");
+
+    ok (eventlog_formatter_set_format (NULL, "text") < 0 && errno == EINVAL,
+        "eventlog_formatter_set_format (NULL, \"text\") returns EINVAL");
+    ok (eventlog_formatter_set_format (evf, "foo") < 0 && errno == EINVAL,
+        "eventlog_formatter_set_format (evf, \"foo\") returns EINVAL");
+
+    ok (eventlog_formatter_colors_init (NULL, "auto") < 0 && errno == EINVAL,
+        "eventlog_formatter_colors_init (NULL, \"auto\") returns EINVAL");
+    ok (eventlog_formatter_colors_init (evf, NULL) < 0 && errno == EINVAL,
+        "eventlog_formatter_colors_init (evf, NULL) returns EINVAL");
+    ok (eventlog_formatter_colors_init (evf, "foo") < 0 && errno == EINVAL,
+        "eventlog_formatter_colors_init (evf, \"foo\") returns EINVAL");
+
+    if (!(good = json_pack ("{s:f s:s s:{s:s}}",
+                            "timestamp", 1699995759.0,
+                            "name", "good",
+                            "context",
+                             "foo", "bar")))
+        BAIL_OUT ("Failed to create good eventlog event");
+
+    if (!(bad = json_pack ("{s:f s:s s:[s]}",
+                            "timestamp", 1699995759.0,
+                            "name", "bad",
+                            "context",
+                             "foo")))
+        BAIL_OUT ("Failed to create bad eventlog event");
+
+    /*  Check all results with default evf, then json formatted evf
+     */
+    for (int i = 0; i < 2; i++) {
+        ok (eventlog_entry_dumpf (NULL, NULL, NULL, NULL) < 0
+            && errno == EINVAL,
+            "eventlog_entry_dumpf (NULL, ...) returns EINVAL");
+        ok (eventlog_entry_dumpf (evf, NULL, NULL, NULL) < 0
+            && errno == EINVAL,
+            "eventlog_entry_dumpf (evf, NULL, ...) returns EINVAL");
+        ok (eventlog_entry_dumpf (evf, stderr, NULL, NULL) < 0
+            && errno == EINVAL,
+            "eventlog_entry_dumpf (evf, stdout, NULL, ...) returns EINVAL");
+        ok (eventlog_entry_dumpf (evf, NULL, &error, good) < 0
+            && errno == EINVAL,
+            "eventlog_entry_dumpf (evf, NULL, &error, event) returns EINVAL");
+        ok (eventlog_entry_dumpf (evf, stderr, &error, bad) < 0
+            && errno == EINVAL,
+            "eventlog_entry_dumpf bad event returns EINVAL");
+        is (error.text, "eventlog_entry_parse: Invalid argument");
+
+        memset (&error, 0, sizeof (error));
+
+        ok ((eventlog_entry_dumps (NULL, NULL, NULL) == NULL)
+            && errno == EINVAL,
+            "eventlog_entry_dumps (NULL, ...) returns EINVAL");
+        ok ((eventlog_entry_dumps (evf, NULL, NULL) == NULL)
+            && errno == EINVAL,
+            "eventlog_entry_dumps (evf, NULL, ...) returns EINVAL");
+        ok ((eventlog_entry_dumps (NULL, &error, good) == NULL)
+            && errno == EINVAL,
+            "eventlog_entry_dumps (NULL, &error, event) returns EINVAL");
+        ok ((eventlog_entry_dumps (evf, &error, bad) == NULL)
+            && errno == EINVAL,
+            "eventlog_entry_dumps (NULL, &error, event) returns EINVAL");
+        is (error.text, "eventlog_entry_parse: Invalid argument");
+
+        ok (eventlog_formatter_set_format (evf, "json") == 0,
+            "eventlog_formatter_set_format json");
+    }
+
+    json_decref (good);
+    json_decref (bad);
+    eventlog_formatter_destroy (evf);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    /*  Set TZ=UTC for predictable results in human-readable timestamps */
+    setenv ("TZ", "", 1);
+    tzset ();
+
+    test_invalid ();
+    test_basic ();
+
+    done_testing ();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libfilemap/Makefile.am b/src/common/libfilemap/Makefile.am
new file mode 100644
index 000000000000..7209cd8d5e23
--- /dev/null
+++ b/src/common/libfilemap/Makefile.am
@@ -0,0 +1,49 @@
+AM_CFLAGS = \
+        $(WARNING_CFLAGS) \
+        $(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+        $(CODE_COVERAGE_LDFLAGS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+        -I$(top_srcdir) \
+        -I$(top_srcdir)/src/include \
+        -I$(top_srcdir)/src/common/libccan \
+        -I$(top_builddir)/src/common/libflux \
+        $(JANSSON_CFLAGS) \
+        $(LIBARCHIVE_CFLAGS)
+
+noinst_LTLIBRARIES = \
+        libfilemap.la
+
+libfilemap_la_SOURCES = \
+        filemap.h \
+        filemap.c \
+        fileref.h \
+        fileref.c
+
+TESTS = \
+	test_fileref.t
+
+test_ldadd = \
+        $(top_builddir)/src/common/libfilemap/libfilemap.la \
+        $(top_builddir)/src/common/libutil/libutil.la \
+        $(top_builddir)/src/common/libtap/libtap.la \
+        $(top_builddir)/src/common/libccan/libccan.la \
+        $(top_builddir)/src/common/libczmqcontainers/libczmqcontainers.la \
+        $(JANSSON_LIBS)
+
+test_cppflags = \
+        -I$(top_srcdir)/src/common/libtap \
+        $(AM_CPPFLAGS)
+
+check_PROGRAMS = $(TESTS)
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+	$(top_srcdir)/config/tap-driver.sh
+
+test_fileref_t_SOURCES = test/fileref.c
+test_fileref_t_CPPFLAGS = $(test_cppflags)
+test_fileref_t_LDADD = $(test_ldadd)
diff --git a/src/common/libfilemap/filemap.c b/src/common/libfilemap/filemap.c
new file mode 100644
index 000000000000..9fab416538fa
--- /dev/null
+++ b/src/common/libfilemap/filemap.c
@@ -0,0 +1,323 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <unistd.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <libgen.h>
+#include <jansson.h>
+#include <archive.h>
+#include <archive_entry.h>
+
+#include "filemap.h"
+
+#include "ccan/base64/base64.h"
+#include "ccan/str/str.h"
+#include "src/common/libutil/dirwalk.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libcontent/content.h"
+
+#include "fileref.h"
+
+
+/* Decode the raw data field a fileref object, setting the result in 'data'
+ * and 'data_size'.  Caller must free.
+ */
+static int decode_data (const char *s, void **data, size_t *data_size)
+{
+    if (s) {
+        size_t len = strlen (s);
+        size_t bufsize = base64_decoded_length (len);
+        void *buf;
+        ssize_t n;
+
+        if (!(buf = malloc (bufsize)))
+            return -1;
+        if ((n = base64_decode (buf, bufsize, s, len)) < 0) {
+            free (buf);
+            errno = EINVAL;
+            return -1;
+        }
+        *data = buf;
+        *data_size = n;
+    }
+    return 0;
+}
+
+/* When ARCHIVE_EXTRACT_NO_OVERWRITE is set, the overwrite error from
+ * libarchive-3.6 is "Attempt to write to an empty file". This is
+ * going to be confusing when the file is not empty, such as the common
+ * situation where source and destination of a copy operation are the
+ * same file.  Rewrite that message.
+ */
+static const char *fixup_archive_error_string (struct archive *archive)
+{
+    const char *errstr = archive_error_string (archive);
+    if (strstarts (errstr, "Attempt to write to an empty file"))
+        errstr = "Attempt to overwrite existing file";
+    return errstr;
+}
+
+static int extract_blob (flux_t *h,
+                         struct archive *archive,
+                         const char *path,
+                         json_t *o,
+                         flux_error_t *errp)
+{
+    struct {
+        json_int_t offset;
+        json_int_t size;
+        const char *blobref;
+    } entry;
+    flux_future_t *f;
+    const void *buf;
+    size_t size;
+
+    if (json_unpack (o,
+                     "[I,I,s]",
+                     &entry.offset,
+                     &entry.size,
+                     &entry.blobref) < 0)
+        return errprintf (errp, "%s: error decoding blobvec entry", path);
+    if (!(f = content_load_byblobref (h, entry.blobref, 0))
+        || content_load_get (f, &buf, &size) < 0) {
+        return errprintf (errp,
+                          "%s: error loading offset=%ju size=%ju from %s: %s",
+                          path,
+                          (uintmax_t)entry.offset,
+                          (uintmax_t)entry.size,
+                          entry.blobref,
+                          future_strerror (f, errno));
+    }
+    if (size != entry.size) {
+        return errprintf (errp,
+                          "%s: error loading offset=%ju size=%ju from %s:"
+                          " unexpected size %ju",
+                          path,
+                          (uintmax_t)entry.offset,
+                          (uintmax_t)entry.size,
+                          entry.blobref,
+                          (uintmax_t)size);
+    }
+    if (archive_write_data_block (archive,
+                                  buf,
+                                  size,
+                                  entry.offset) != ARCHIVE_OK) {
+        return errprintf (errp,
+                          "%s: write: %s",
+                          path,
+                          fixup_archive_error_string (archive));
+    }
+    flux_future_destroy (f);
+    return 0;
+}
+
+/*  Extract a single file from a 'fileref' object using an existing
+ *  libarchive object 'archive' and using 'path' as the default path
+ *  if no path is encoded in 'fileref'.
+ */
+static int extract_file (flux_t *h,
+                         struct archive *archive,
+                         const char *path,
+                         json_t *fileref,
+                         flux_error_t *errp,
+                         filemap_trace_f trace_cb,
+                         void *arg)
+{
+    int mode;
+    json_int_t size = -1;
+    json_int_t ctime = -1;
+    json_int_t mtime = -1;
+    const char *encoding = NULL;
+    json_t *data = NULL;
+    size_t index;
+    json_t *o;
+    struct archive_entry *entry;
+    json_error_t error;
+
+    if (json_unpack_ex (fileref,
+                        &error,
+                        0,
+                        "{s?s s:i s?I s?I s?I s?s s?o}",
+                        "path", &path,
+                        "mode", &mode,
+                        "size", &size,
+                        "mtime", &mtime,
+                        "ctime", &ctime,
+                        "encoding", &encoding,
+                        "data", &data) < 0)
+        return errprintf (errp,
+                          "error decoding fileref object: %s",
+                          error.text);
+
+    if (trace_cb)
+        (*trace_cb) (arg, fileref, path, mode, size, mtime, ctime, encoding);
+
+    /* metadata
+     */
+    if (!(entry = archive_entry_new ()))
+        return errprintf (errp, "%s: error creating libarchive entry", path);
+    archive_entry_set_pathname (entry, path);
+    archive_entry_set_mode (entry, mode);
+    if (mtime != -1)
+        archive_entry_set_mtime (entry, mtime, 0);
+    if (ctime != -1)
+        archive_entry_set_ctime (entry, ctime, 0);
+    if (S_ISREG (mode)) {
+        if (size != -1)
+            archive_entry_set_size (entry, size);
+    }
+    else if (S_ISLNK (mode)) {
+        const char *target;
+        if (!data || !(target = json_string_value (data)))
+            return errprintf (errp, "%s: missing symlink data", path);
+        archive_entry_set_symlink (entry, target);
+    }
+    else if (!S_ISDIR (mode)) // nothing to do for directory
+        return errprintf (errp,
+                          "%s: unknown file type (mode=0%o)",
+                          path,
+                          mode);
+    if (archive_write_header (archive, entry) != ARCHIVE_OK)
+        return errprintf (errp,
+                          "%s: %s",
+                          path,
+                          archive_error_string (archive));
+
+    /* data
+     */
+    if (S_ISREG (mode) && data != NULL) {
+        if (!encoding) {
+          char *str;
+            if (!(str = json_dumps (data, JSON_ENCODE_ANY | JSON_COMPACT)))
+                return errprintf (errp,
+                                  "%s: could not encode JSON file data",
+                                  path);
+            if (archive_write_data_block (archive,
+                                          str,
+                                          strlen (str),
+                                          0) != ARCHIVE_OK) {
+                return errprintf (errp,
+                                  "%s: write: %s",
+                                  path,
+                                  fixup_archive_error_string (archive));
+            }
+            free (str);
+        }
+        else if (streq (encoding, "base64")) {
+            const char *str = json_string_value (data);
+            void *buf;
+            size_t buf_size;
+
+            if (!str || decode_data (str, &buf, &buf_size) < 0)
+                return errprintf (errp,
+                                  "%s: could not decode base64 file data",
+                                  path);
+            if (archive_write_data_block (archive,
+                                          buf,
+                                          buf_size,
+                                          0) != ARCHIVE_OK) {
+                return errprintf (errp,
+                                  "%s: write: %s",
+                                  path,
+                                  fixup_archive_error_string (archive));
+            }
+            free (buf);
+        }
+        else if (streq (encoding, "blobvec")) {
+            json_array_foreach (data, index, o) {
+                if (extract_blob (h, archive, path, o, errp) < 0)
+                    return -1;
+            }
+        }
+        else if (streq (encoding, "utf-8")) {
+            const char *str = json_string_value (data);
+            if (!str)
+                return errprintf (errp,
+                                  "%s: unexpected data type for utf8",
+                                  path);
+            if (archive_write_data_block (archive,
+                                          str,
+                                          strlen (str),
+                                          0) != ARCHIVE_OK) {
+                return errprintf (errp,
+                                  "%s: write: %s",
+                                  path,
+                                  fixup_archive_error_string (archive));
+            }
+        }
+        else {
+            return errprintf (errp,
+                              "%s: unknown RFC 37 encoding %s",
+                              path,
+                              encoding);
+        }
+    }
+
+    archive_entry_free (entry);
+    return 0;
+}
+
+int filemap_extract (flux_t *h,
+                     json_t *files,
+                     int libarchive_flags,
+                     flux_error_t *errp,
+                     filemap_trace_f trace_cb,
+                     void *arg)
+{
+    const char *key;
+    size_t index;
+    json_t *entry;
+    struct archive *archive;
+    int rc = -1;
+
+    if (!(archive = archive_write_disk_new ())
+        || archive_write_disk_set_options (archive,
+                                           libarchive_flags) != ARCHIVE_OK) {
+        errprintf (errp, "error creating libarchive context");
+        goto out;
+    }
+
+    if (json_is_array (files)) {
+        json_array_foreach (files, index, entry) {
+            if (extract_file (h,
+                              archive,
+                              NULL,
+                              entry,
+                              errp,
+                              trace_cb,
+                              arg) < 0)
+            goto out;
+        }
+    } else {
+        json_object_foreach (files, key, entry) {
+            if (extract_file (h,
+                              archive,
+                              key,
+                              entry,
+                              errp,
+                              trace_cb,
+                              arg) < 0)
+            goto out;
+        }
+    }
+    rc = 0;
+out:
+    if (archive)
+        archive_write_free (archive);
+    return rc;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libfilemap/filemap.h b/src/common/libfilemap/filemap.h
new file mode 100644
index 000000000000..84af42a431de
--- /dev/null
+++ b/src/common/libfilemap/filemap.h
@@ -0,0 +1,46 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_FLUX_FILEMAP_H
+#define HAVE_FLUX_FILEMAP_H
+
+#include <stdarg.h>
+
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/errprintf.h"
+
+/*
+ *  Tracing callback for filemap_extract()
+ */
+typedef void (*filemap_trace_f) (void *arg,
+                                 json_t *fileref,
+                                 const char *path,
+                                 int mode,
+                                 int64_t size,
+                                 int64_t mtime,
+                                 int64_t ctime,
+                                 const char *encoding);
+
+/*  Extract an RFC 37 File Archive in either array or dictionary form.
+ *  If 'trace_cb' is set, then it will be called for each extracted file.
+ *
+ *  Returns 0 on success, or -1 with error set in errp when non-NULL.
+ *
+ */
+int filemap_extract (flux_t *h,
+                     json_t *files,
+		     int libarchive_flags,
+                     flux_error_t *errp,
+                     filemap_trace_f trace_cb,
+                     void *arg);
+
+#endif /* !HAVE_FLUX_FILEMAP_H */
diff --git a/src/common/libfilemap/fileref.c b/src/common/libfilemap/fileref.c
new file mode 100644
index 000000000000..b5997068c3ce
--- /dev/null
+++ b/src/common/libfilemap/fileref.c
@@ -0,0 +1,497 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* fileref.c - helpers for RFC 37 file system objects
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <jansson.h>
+#include <assert.h>
+
+#include "ccan/base64/base64.h"
+
+#include "src/common/libutil/blobref.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/fdutils.h"
+#include "fileref.h"
+
+static int blobvec_append (json_t *blobvec,
+                           const void *mapbuf,
+                           off_t offset,
+                           size_t blobsize,
+                           const char *hashtype)
+{
+    char blobref[BLOBREF_MAX_STRING_SIZE];
+    json_t *o;
+    json_int_t offsetj = offset;
+    json_int_t blobsizej = blobsize;
+
+    if (blobref_hash (hashtype,
+                      mapbuf + offset,
+                      blobsize,
+                      blobref,
+                      sizeof (blobref)) < 0)
+        return -1;
+    if (!(o = json_pack ("[I,I,s]", offsetj, blobsizej, blobref))
+        || json_array_append_new (blobvec, o) < 0) {
+        json_decref (o);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+static bool file_has_no_data (int fd)
+{
+#ifdef SEEK_DATA
+    if (lseek (fd, 0, SEEK_DATA) == (off_t)-1 && errno == ENXIO)
+        return true;
+#endif
+    return false;
+}
+
+/* Walk the regular file represented by 'fd', appending blobvec array entries
+ * to 'blobvec' array for each 'chunksize' region.  Use SEEK_DATA and SEEK_HOLE
+ * to skip holes in sparse files - see lseek(2).
+ */
+static json_t *blobvec_create (int fd,
+                               const void *mapbuf,
+                               size_t size,
+                               const char *hashtype,
+                               size_t chunksize)
+{
+    json_t *blobvec;
+    off_t offset = 0;
+
+    assert (fd >= 0);
+    assert (size > 0);
+
+    if (!(blobvec = json_array ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    while (offset < size) {
+#ifdef SEEK_DATA
+        // N.B. fails with ENXIO if there is no more data
+        if ((offset = lseek (fd, offset, SEEK_DATA)) == (off_t)-1) {
+            if (errno == ENXIO)
+                break;
+            goto error;
+        }
+#endif
+        if (offset < size) {
+            off_t notdata;
+            size_t blobsize;
+
+#ifdef SEEK_HOLE
+            // N.B. returns size if there are no more holes
+            if ((notdata = lseek (fd, offset, SEEK_HOLE)) == (off_t)-1)
+                goto error;
+#else
+            notdata = size;
+#endif /* SEEK_HOLE */
+
+            blobsize = notdata - offset;
+            if (blobsize > chunksize)
+                blobsize = chunksize;
+            if (blobvec_append (blobvec,
+                                mapbuf,
+                                offset,
+                                blobsize,
+                                hashtype) < 0)
+                goto error;
+            offset += blobsize;
+        }
+    }
+    return blobvec;
+error:
+    ERRNO_SAFE_WRAP (json_decref, blobvec);
+    return NULL;
+}
+
+static json_t *fileref_create_nonempty (const char *path,
+                                        const char *encoding,
+                                        json_t *data,
+                                        struct stat *sb,
+                                        flux_error_t *error)
+{
+    json_t *o;
+
+    if (!(o = json_pack ("{s:s s:s s:O s:I s:I s:I s:i}",
+                         "path", path,
+                         "encoding", encoding,
+                         "data", data,
+                         "size", (json_int_t)sb->st_size,
+                         "mtime", (json_int_t)sb->st_mtime,
+                         "ctime", (json_int_t)sb->st_ctime,
+                         "mode", sb->st_mode))) {
+        errprintf (error, "%s: error packing %s file object", path, encoding);
+        errno = ENOMEM;
+        return NULL;
+    }
+    return o;
+}
+
+static json_t *fileref_create_blobvec (const char *path,
+                                       int fd,
+                                       void *mapbuf,
+                                       struct stat *sb,
+                                       const char *hashtype,
+                                       size_t chunksize,
+                                       flux_error_t *error)
+{
+    json_t *blobvec;
+    json_t *o;
+
+    blobvec = blobvec_create (fd, mapbuf, sb->st_size, hashtype, chunksize);
+    if (!blobvec) {
+        errprintf (error,
+                   "%s: error creating blobvec array: %s",
+                   path,
+                   strerror (errno));
+        goto error;
+    }
+    if (!(o = fileref_create_nonempty (path, "blobvec", blobvec, sb, error)))
+        goto error;
+    json_decref (blobvec);
+    return o;
+error:
+    ERRNO_SAFE_WRAP (json_decref, blobvec);
+    return NULL;
+}
+
+static void *read_whole_file (const char *path,
+                              int fd,
+                              size_t size,
+                              flux_error_t *error)
+{
+    void *data;
+    ssize_t n;
+
+    if ((n = read_all (fd, &data)) < 0) {
+        errprintf (error, "%s: %s", path, strerror (errno));
+        return NULL;
+    }
+    if (n < size) {
+        errprintf (error, "%s: short read", path);
+        free (data);
+        errno = EINVAL;
+    }
+    return data;
+}
+
+static json_t *fileref_create_base64 (const char *path,
+                                      void *data,
+                                      struct stat *sb,
+                                      flux_error_t *error)
+{
+    json_t *o;
+    char *buf = NULL;
+    size_t bufsize;
+    json_t *obuf = NULL;
+
+    bufsize = base64_encoded_length (sb->st_size) + 1; // +1 NULL
+    if (!(buf = malloc (bufsize))) {
+        errprintf (error, "%s: out of memory while encoding", path);
+        return NULL;
+    }
+    if (base64_encode (buf, bufsize, data, sb->st_size) < 0) {
+        errprintf (error, "%s: base64_encode error", path);
+        goto inval;
+    }
+    if (!(obuf = json_string (buf))) {
+        errprintf (error, "%s: error creating base64 json string", path);
+        errno = EINVAL;
+        goto error;
+    }
+    if (!(o = fileref_create_nonempty (path, "base64", obuf, sb, error)))
+        goto error;
+    json_decref (obuf);
+    free (buf);
+    return o;
+inval:
+    errno = EINVAL;
+error:
+    ERRNO_SAFE_WRAP (json_decref , obuf);
+    ERRNO_SAFE_WRAP (free, buf);
+    return NULL;
+}
+
+static json_t *fileref_create_utf8 (const char *path,
+                                    void *data,
+                                    struct stat *sb,
+                                    flux_error_t *error)
+{
+    json_t *o;
+    json_t *obuf = NULL;
+
+    if (!(obuf = json_stringn (data, sb->st_size))) {
+        errprintf (error, "%s: error creating utf-8 json string", path);
+        errno = EINVAL;
+        goto error;
+    }
+    if (!(o = fileref_create_nonempty (path, "utf-8", obuf, sb, error)))
+        goto error;
+    json_decref (obuf);
+    return o;
+error:
+    ERRNO_SAFE_WRAP (json_decref , obuf);
+    return NULL;
+}
+
+static json_t *fileref_create_empty (const char *path,
+                                     struct stat *sb,
+                                     flux_error_t *error)
+{
+    json_t *o;
+
+    if (!(o = json_pack ("{s:s s:I s:I s:I s:i}",
+                         "path", path,
+                         "size", (json_int_t)sb->st_size,
+                         "mtime", (json_int_t)sb->st_mtime,
+                         "ctime", (json_int_t)sb->st_ctime,
+                         "mode", sb->st_mode))) {
+        errprintf (error, "%s: error packing empty file object", path);
+        errno = ENOMEM;
+        return NULL;
+    }
+    return o;
+}
+
+static json_t *fileref_create_directory (const char *path,
+                                         struct stat *sb,
+                                         flux_error_t *error)
+{
+    json_t *o;
+
+    if (!(o = json_pack ("{s:s s:I s:I s:i}",
+                         "path", path,
+                         "mtime", (json_int_t)sb->st_mtime,
+                         "ctime", (json_int_t)sb->st_ctime,
+                         "mode", sb->st_mode))) {
+        errprintf (error, "%s: error packing directory file object", path);
+        errno = ENOMEM;
+        return NULL;
+    }
+    return o;
+}
+
+static json_t *fileref_create_symlink (const char *path,
+                                       const char *fullpath,
+                                       struct stat *sb,
+                                       flux_error_t *error)
+{
+    json_t *o;
+    char *target;
+
+    if (!(target = calloc (1, sb->st_size + 1))
+        || readlink (fullpath, target, sb->st_size) < 0) {
+        errprintf (error, "readlink %s: %s", fullpath, strerror (errno));
+        goto error;
+    }
+    if (!(o = json_pack ("{s:s s:s s:I s:I s:i}",
+                         "path", path,
+                         "data", target,
+                         "mtime", (json_int_t)sb->st_mtime,
+                         "ctime", (json_int_t)sb->st_ctime,
+                         "mode", sb->st_mode))) {
+        errprintf (error, "%s: error packing symlink file object", path);
+        errno = ENOMEM;
+        goto error;
+    }
+    free (target);
+    return o;
+error:
+    ERRNO_SAFE_WRAP (free, target);
+    return NULL;
+}
+
+json_t *fileref_create_ex (const char *path,
+                           struct blobvec_param *param,
+                           struct blobvec_mapinfo *mapinfop,
+                           flux_error_t *error)
+{
+    const char *relative_path;
+    json_t *o;
+    int fd = -1;
+    struct stat sb;
+    struct blobvec_mapinfo mapinfo = { .base = MAP_FAILED, .size = 0 };
+    int saved_errno;
+
+    if (param) {
+        if (param->hashtype == NULL) {
+            errprintf (error, "invalid blobvec encoding parameters");
+            goto inval;
+        }
+    }
+    /* Store a relative path in the object so that extraction can specify a
+     * destination directory, like tar(1) default behavior.
+     */
+    relative_path = path;
+    while (*relative_path == '/')
+        relative_path++;
+    if (strlen (relative_path) == 0)
+        relative_path = ".";
+    /* Avoid TOCTOU in S_ISREG case by opening before checking its type.
+     * If open fails due to O_NOFOLLOW (ELOOP), get link info with lstat(2).
+     * Avoid open(2) blocking on a FIFO with O_NONBLOCK, but restore blocking
+     * behavior after open(2) succeeds.
+     */
+    if ((fd = open (path, O_RDONLY | O_NOFOLLOW | O_NONBLOCK)) < 0) {
+        if (errno != ELOOP || lstat (path, &sb) < 0) {
+            errprintf (error, "%s: %s", path, strerror (errno));
+            goto error;
+        }
+    }
+    else {
+        if (fstat (fd, &sb) < 0 || fd_set_blocking (fd) < 0) {
+            errprintf (error, "%s: %s", path, strerror (errno));
+            goto error;
+        }
+    }
+
+    /* Empty reg file, possibly sparse with size > 0.
+     */
+    if (S_ISREG (sb.st_mode) && file_has_no_data (fd)) {
+        if (!(o = fileref_create_empty (relative_path, &sb, error)))
+            goto error;
+    }
+    /* Large reg file will be encoded with blobvec.
+     */
+    else if (S_ISREG (sb.st_mode)
+        && param != NULL
+        && sb.st_size > param->small_file_threshold) {
+        size_t chunksize;
+
+        mapinfo.size = sb.st_size;
+        mapinfo.base = mmap (NULL, mapinfo.size, PROT_READ, MAP_PRIVATE, fd, 0);
+        if (mapinfo.base == MAP_FAILED) {
+            errprintf (error, "mmap: %s", strerror (errno));
+            goto error;
+        }
+        chunksize = param->chunksize;
+        if (chunksize == 0)
+            chunksize = sb.st_size;
+        if (!(o = fileref_create_blobvec (relative_path,
+                                          fd,
+                                          mapinfo.base,
+                                          &sb,
+                                          param->hashtype,
+                                          chunksize,
+                                          error)))
+            goto error;
+    }
+    /* Other reg file will be encoded with base64.
+     */
+    else if (S_ISREG (sb.st_mode)) {
+        void *data;
+        if (!(data = read_whole_file (path, fd, sb.st_size, error)))
+            goto error;
+        if (!(o = fileref_create_utf8 (relative_path, data, &sb, error))
+            && !(o = fileref_create_base64 (relative_path, data, &sb, error))) {
+            ERRNO_SAFE_WRAP (free, data);
+            goto error;
+        }
+        free (data);
+    }
+    /* symlink
+     */
+    else if (S_ISLNK (sb.st_mode)) {
+        if (!(o = fileref_create_symlink (relative_path, path, &sb, error)))
+            goto error;
+    }
+    /* directory
+     */
+    else if (S_ISDIR (sb.st_mode)) {
+        if (!(o = fileref_create_directory (relative_path, &sb, error)))
+            goto error;
+    }
+    else {
+        errprintf (error, "%s: unsupported file type", path);
+        goto inval;
+    }
+
+    if (mapinfop)
+        *mapinfop = mapinfo;
+    else
+        (void)munmap (mapinfo.base, mapinfo.size);
+    if (fd >= 0)
+        close (fd);
+    return o;
+inval:
+    errno = EINVAL;
+error:
+    saved_errno = errno;
+    if (mapinfo.base != MAP_FAILED)
+        (void)munmap (mapinfo.base, mapinfo.size);
+    if (fd >= 0)
+        close (fd);
+    errno = saved_errno;
+    return NULL;
+}
+
+json_t *fileref_create (const char *path, flux_error_t *error)
+{
+    return fileref_create_ex (path, NULL, NULL, error);
+}
+
+void fileref_pretty_print (json_t *fileref,
+                           const char *path,
+                           bool long_form,
+                           char *buf,
+                           size_t bufsize)
+{
+    json_int_t size = 0;
+    int mode;
+    int n;
+
+    if (!buf)
+        return;
+    /* RFC 37 says path is optional in the file object (to support dict archive
+     * containers) so let it be passed in as 'path' arg and override if present
+     * in the object.  It's an error if it's not set by one of those.
+     */
+    if (!fileref
+        || json_unpack (fileref,
+                        "{s?s s:i s?I}",
+                        "path", &path,
+                        "mode", &mode,
+                        "size", &size) < 0
+        || path == NULL) {
+        n = snprintf (buf, bufsize, "invalid fileref");
+    }
+    else if (long_form) {
+        n = snprintf (buf,
+                      bufsize,
+                      "%s 0%o %8ju %s",
+                      S_ISREG (mode) ? "f" :
+                      S_ISLNK (mode) ? "l" :
+                      S_ISDIR (mode) ? "d" : "?",
+                      mode & 0777,
+                      (uintmax_t)size,
+                      path);
+    }
+    else
+        n = snprintf (buf, bufsize, "%s", path);
+    if (n >= bufsize && bufsize > 1)
+        buf[bufsize - 2] = '+';
+}
+
+// vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/common/libfilemap/fileref.h b/src/common/libfilemap/fileref.h
new file mode 100644
index 000000000000..43a5512c0e95
--- /dev/null
+++ b/src/common/libfilemap/fileref.h
@@ -0,0 +1,62 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_FILEREF_H
+#define _UTIL_FILEREF_H
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <jansson.h>
+
+#include "src/common/libflux/types.h"
+
+struct blobvec_param {
+    const char *hashtype;
+    size_t chunksize;              // maximum size of each blob
+    size_t small_file_threshold;   // no blobvec encoding for regular files of
+};                                 //  size <= thresh (0=always blobvec)
+
+struct blobvec_mapinfo {
+    void *base;
+    size_t size;
+};
+
+
+/* Variant of fileref_create() with extra parameters to allow for 'blobvec'
+ * encoding.
+ * - If 'param' is non-NULL, blobvec encoding is enabled with the specified
+ *   params.
+ * - If 'mapinfo' is non-NULL, and the file meets conditions for blobvec
+ *   encoding, the file remains mapped in memory and its address is returned.
+ */
+json_t *fileref_create_ex (const char *path,
+                           struct blobvec_param *param,
+                           struct blobvec_mapinfo *mapinfo,
+                           flux_error_t *error);
+
+/* Create a fileref object for the file system object at 'path'.
+ * The blobvec encoding is never used thus the object is self-contained.
+ */
+json_t *fileref_create (const char *path, flux_error_t *error);
+
+/* Build a "directory listing" of a fileref and set it in 'buf'.
+ * Set 'path' if provided from archive container (fileref->path overrides).
+ * If the fileref is invalid, set "invalid fileref".
+ * If output is truncated, '+' is substituted for the last character.
+ */
+void fileref_pretty_print (json_t *fileref,
+                           const char *path,
+                           bool long_form,
+                           char *buf,
+                           size_t bufsize);
+
+#endif /* !_UTIL_FILEREF_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libfilemap/test/fileref.c b/src/common/libfilemap/test/fileref.c
new file mode 100644
index 000000000000..468fa8119765
--- /dev/null
+++ b/src/common/libfilemap/test/fileref.c
@@ -0,0 +1,730 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/unlink_recursive.h"
+#include "src/common/libutil/blobref.h"
+#include "ccan/array_size/array_size.h"
+#include "src/common/libccan/ccan/str/str.h"
+#include "ccan/base64/base64.h"
+
+#include "fileref.h"
+
+static char testdir[1024];
+
+static bool have_sparse;
+
+static json_t *xfileref_create_vec (const char *path,
+                                    const char *hashtype,
+                                    int chunksize)
+{
+    json_t *o;
+    flux_error_t error;
+    struct blobvec_param blobvec_param = {
+        .hashtype = hashtype,
+        .chunksize = chunksize,
+        .small_file_threshold = 4096,
+    };
+
+    o = fileref_create_ex (path, &blobvec_param, NULL, &error);
+    if (!o)
+        diag ("%s", error.text);
+    return o;
+}
+
+static json_t *xfileref_create (const char *path)
+{
+    json_t *o;
+    flux_error_t error;
+
+    o = fileref_create (path, &error);
+    if (!o)
+        diag ("%s", error.text);
+    return o;
+}
+
+const char *mkpath (const char *name)
+{
+    static char tmppath[2048];
+    snprintf (tmppath, sizeof (tmppath), "%s/%s", testdir, name);
+    return tmppath;
+}
+const char *mkpath_relative (const char *name)
+{
+    const char *cp = mkpath (name);
+    while (*cp == '/')
+        cp++;
+    return cp;
+}
+
+void rmfile (const char *name)
+{
+    if (unlink (mkpath (name)) < 0)
+        BAIL_OUT ("error unlinking %s", mkpath (name));
+}
+
+/* SEEK_DATA support was added to the linux NFS client in kernel 3.18.
+ * In el7 based distros, it is defined but doesn't work on NFS.  So ensure
+ * SEEK_DATA returns ENXIO on file that is 100% empty.
+ */
+bool test_sparse (void)
+{
+    bool result = false;
+#ifdef SEEK_DATA
+    int fd;
+    struct stat sb;
+
+    fd = open (mkpath ("testhole"), O_WRONLY | O_CREAT | O_TRUNC, 0600);
+    if (fd < 0)
+        BAIL_OUT ("error creating test file: %s", strerror (errno));
+    if (ftruncate (fd, 8192) < 0)
+        BAIL_OUT ("error truncating test file: %s", strerror (errno));
+    if (fstat (fd, &sb) < 0)
+        BAIL_OUT ("error stating test file: %s", strerror (errno));
+    if (sb.st_blocks == 0
+        && lseek (fd, 0, SEEK_DATA) == (off_t)-1
+        && errno == ENXIO)
+        result = true;
+    close (fd);
+    rmfile ("testhole");
+#endif /* SEEK_DATA */
+    return result;
+}
+
+bool is_sparse (const char *name)
+{
+    bool result = false;
+#ifdef SEEK_HOLE
+    int fd;
+    struct stat sb;
+    off_t offset;
+
+    fd = open (mkpath (name), O_RDONLY);
+    if (fd < 0)
+        return false;
+    if (fstat (fd, &sb) < 0 || !S_ISREG (sb.st_mode))
+        goto error;
+    offset = lseek (fd, 0, SEEK_HOLE);
+    if (offset < sb.st_size)
+        result = true;
+error:
+    close (fd);
+#endif /* SEEK_HOLE */
+    return result;
+}
+
+/* Create test file 'name' under test directory.
+ * Each character in  'spec' represents one block filled with that character,
+ * except for "-" which tries to create a hole (if supported by file system).
+ */
+void mkfile (const char *name, int blocksize, const char *spec)
+{
+    char *buf;
+    int fd;
+    const char *cp;
+
+    if (!(buf = malloc (blocksize)))
+        BAIL_OUT ("malloc failed");
+    if ((fd = open (mkpath (name), O_WRONLY | O_CREAT | O_TRUNC, 0644)) < 0)
+        BAIL_OUT ("could not create %s: %s", name, strerror (errno));
+    for (cp = &spec[0]; *cp != '\0'; cp++) {
+        if (*cp == '-') {
+            if (lseek (fd, blocksize, SEEK_CUR) == (off_t)-1)
+                BAIL_OUT ("error lseeking in %s: %s", name, strerror (errno));
+        }
+        else {
+            memset (buf, *cp, blocksize);
+            size_t n = write (fd, buf, blocksize);
+            if (n < 0)
+                BAIL_OUT ("error writing to %s: %s", name, strerror (errno));
+            if (n < blocksize)
+                BAIL_OUT ("short write to %s", name);
+        }
+    }
+    if (close (fd) < 0)
+        BAIL_OUT ("error closing %s: %s", name, strerror (errno));
+    free (buf);
+}
+
+void mkfile_string (const char *name, const char *s)
+{
+    int fd;
+    if ((fd = open (mkpath (name), O_WRONLY | O_CREAT | O_TRUNC, 0600)) < 0
+        || write (fd, s, strlen (s)) != strlen (s)
+        || close (fd) < 0)
+        BAIL_OUT ("could not create %s: %s", strerror (errno));
+    close (fd);
+}
+
+void mkfile_empty (const char *name, size_t size)
+{
+    int fd;
+    if ((fd = open (mkpath (name), O_WRONLY | O_CREAT | O_TRUNC, 0600)) < 0
+        || ftruncate (fd, size) < 0
+        || close (fd) < 0)
+        BAIL_OUT ("could not create %s: %s", strerror (errno));
+    close (fd);
+}
+
+/* Check that blobref 'bref' hash matches specified file region.
+ * If bref is NULL, check that the region contains all zeroes.
+ */
+bool check_blob (int fd, off_t offset, size_t size, const char *bref)
+{
+    char *buf;
+    ssize_t n;
+    char blobref[BLOBREF_MAX_STRING_SIZE];
+
+    if (!(buf = malloc (size)))
+        BAIL_OUT ("out of memory");
+    if (lseek (fd, offset, SEEK_SET) == (off_t)-1) {
+        diag ("lseek: %s", strerror (errno));
+        goto error;
+    }
+    if ((n = read (fd, buf, size)) < 0) {
+        diag ("read: %s", strerror (errno));
+        goto error;
+    }
+    if (n < size) {
+        errno = EINVAL;
+        diag ("short read");
+        goto error;
+    }
+    if (bref) {
+        char hashtype[64], *cp;
+        snprintf (hashtype, sizeof (hashtype), "%s", bref);
+        if ((cp = strchr (hashtype, '-')))
+            *cp = '\0';
+        if (blobref_hash (hashtype, buf, size, blobref, sizeof (blobref)) < 0)
+            goto error;
+        if (!streq (blobref, bref)) {
+            diag ("blobref mismatch");
+            goto error;
+        }
+    }
+    else { // check hole
+        for (int i = 0; i < size; i++) {
+            if (buf[i] != 0) {
+                diag ("hole mismatch");
+                goto error;
+            }
+        }
+    }
+    free (buf);
+    return true;
+error:
+    free (buf);
+    return false;
+}
+
+/* Check that 'fileref' matches the metadata and content of test file 'name'
+ * and has the expected blobvec length.
+ */
+bool check_fileref (json_t *fileref, const char *name, int blobcount)
+{
+    const char *path;
+    int mode;
+    json_int_t size = -1;
+    json_int_t mtime = -1;
+    json_int_t ctime = -1;
+    const char *encoding = NULL;
+    json_t *data = NULL;
+    struct stat sb;
+    int myblobcount = 0;
+
+    if (!fileref) {
+        diag ("fileref is NULL");
+        return false;
+    }
+    if (json_unpack (fileref,
+                     "{s:s s:i s?I s?I s?I s?s s?o}",
+                     "path", &path,
+                     "mode", &mode,
+                     "size", &size,
+                     "mtime", &mtime,
+                     "ctime", &ctime,
+                     "encoding", &encoding,
+                     "data", &data) < 0) {
+
+        diag ("error decoding fileref object");
+        return false;
+    }
+    if (!streq (path, mkpath_relative (name))) {
+        diag ("fileref.path != expected path");
+        return false;
+    }
+    if (lstat (mkpath (name), &sb) < 0)
+        BAIL_OUT ("could not stat %s", path);
+    if (size != -1 && size != sb.st_size) {
+        diag ("fileref.size is %ju not %zu", (uintmax_t)size, sb.st_size);
+        goto error;
+    }
+    if (mtime != -1 && mtime != sb.st_mtime) {
+        diag ("fileref.mtime is wrong");
+        goto error;
+    }
+    if (ctime != -1 && ctime != sb.st_ctime) {
+        diag ("fileref.ctime is wrong");
+        goto error;
+    }
+    if (mode != sb.st_mode) {
+        diag ("fileref.mode is wrong");
+        goto error;
+    }
+    if (!S_ISREG (mode) && !S_ISDIR (mode) && !S_ISLNK (mode)) {
+        diag ("unknown file type");
+        goto error;
+    }
+    if (S_ISLNK (mode)) {
+        char buf[1024];
+        if (!data || !json_is_string (data)) {
+            diag ("symlink data is missing");
+            goto error;
+        }
+        if (encoding || size != -1) {
+            diag ("symlink encoding/size unexpectedly set");
+            goto error;
+        }
+        if (readlink (mkpath (name), buf, sizeof (buf)) < 0
+            || !streq (buf, json_string_value (data))) {
+            diag ("symlink target is wrong");
+            goto error;
+        }
+    }
+    else if (S_ISREG (mode)) {
+        if (!encoding) { // json encoding
+        }
+        else if (streq (encoding, "utf-8")) {
+            if (!json_is_string (data)) {
+                diag ("regfile utf-8 data is not a string");
+                goto error;
+            }
+        }
+        else if (streq (encoding, "base64")) {
+            if (!json_is_string (data)) {
+                diag ("regfile base64 data is not a string");
+                goto error;
+            }
+        }
+        else if (streq (encoding, "blobvec")) {
+            if (!json_is_array (data)) {
+                diag ("regfile blobvec data is not an array");
+                goto error;
+            }
+            myblobcount = json_array_size (data);
+        }
+        else {
+            diag ("unknown encoding %s", encoding);
+            goto error;
+        }
+    }
+    else if (S_ISDIR (mode)) {
+        if (data != NULL) {
+            diag ("directory has data");
+            goto error;
+        }
+    }
+    if (myblobcount != blobcount) {
+        diag ("fileref.blobvec has incorrect length (expected %d got %zu)",
+              blobcount, myblobcount);
+        goto error;
+    }
+    if (blobcount > 0) {
+        int fd;
+        size_t index;
+        json_t *o;
+        off_t cursor;
+
+        if ((fd = open (mkpath (name), O_RDONLY)) < 0) {
+            diag ("open %s: %s", path, strerror (errno));
+            return false;
+        }
+        cursor = 0;
+        json_array_foreach (data, index, o) {
+            struct {
+                json_int_t offset;
+                json_int_t size;
+                const char *blobref;
+            } entry;
+
+            if (json_unpack (o, "[I,I,s]",
+                             &entry.offset,
+                             &entry.size,
+                             &entry.blobref) < 0) {
+                diag ("failed to unpack blovec entry");
+                close (fd);
+                goto error;
+            }
+            // if offset > cursor, we've hit a zero region, check that first
+            if (entry.offset > cursor) {
+                if (!check_blob (fd, cursor, entry.offset - cursor, NULL)) {
+                    diag ("zero region error");
+                    close (fd);
+                    goto error;
+                }
+            }
+            if (!check_blob (fd, entry.offset, entry.size, entry.blobref)) {
+                diag ("content error");
+                close (fd);
+                goto error;
+            }
+            cursor = entry.offset + entry.size;
+        }
+        if (cursor < size) {
+            if (!check_blob (fd, cursor, size - cursor, NULL)) {
+                diag ("zero region error");
+                close (fd);
+                goto error;
+            }
+        }
+        close (fd);
+    }
+    else if (S_ISREG (mode) && data != NULL && encoding
+        && streq (encoding, "base64")) {
+        const char *str = json_string_value (data);
+        int fd;
+        char buf[8192];
+        char buf2[8192];
+        ssize_t n;
+
+        if (!str
+            || (n = base64_decode (buf, sizeof (buf), str, strlen (str))) < 0) {
+            diag ("base64_decode failed");
+            goto error;
+        }
+        if ((fd = open (mkpath (name), O_RDONLY)) < 0) {
+            diag ("open %s: %s", path, strerror (errno));
+            goto error;
+        }
+        if (read (fd, buf2, sizeof (buf2)) != n) {
+            diag ("read %s: returned wrong size", path);
+            close (fd);
+            goto error;
+        }
+        if (memcmp (buf, buf2, n) != 0) {
+            diag ("%s: data is wrong", path);
+            close (fd);
+            goto error;
+        }
+        close (fd);
+    }
+    else if (S_ISREG (mode) && data != NULL
+        && encoding && streq (encoding, "utf-8")) {
+        const char *str = json_string_value (data);
+        ssize_t n = strlen (str);
+        int fd;
+        char buf[8192];
+
+        if ((fd = open (mkpath (name), O_RDONLY)) < 0) {
+            diag ("open %s: %s", path, strerror (errno));
+            goto error;
+        }
+        if (read (fd, buf, sizeof (buf)) != n) {
+            diag ("read %s: returned wrong size", path);
+            close (fd);
+            goto error;
+        }
+        if (memcmp (str, buf, n) != 0) {
+            diag ("%s: data is wrong", path);
+            close (fd);
+            goto error;
+        }
+        close (fd);
+    }
+    return true;
+error:
+    return false;
+}
+
+int blobcount (json_t *fileref)
+{
+    json_t *o;
+    if (json_unpack (fileref, "{s:o}", "blobvec", &o) < 0)
+        return -1;
+    return json_array_size (o);
+}
+
+void diagjson (json_t *o)
+{
+    char *s = NULL;
+    if (o)
+        s = json_dumps (o, JSON_INDENT(2));
+    diag ("%s", s ? s : "(NULL)");
+    free (s);
+}
+
+struct testfile {
+    const char *spec;
+    int chunksize;
+    const char *hashtype;
+    int exp_blobs;
+};
+
+struct testfile testvec[] = {
+    { "aaaa", 0, "sha1", 1 },
+    { "-aaa", 0, "sha1", 1 },
+    { "a-aa", 0, "sha1", 2 },
+    { "aaa-", 0, "sha1", 1 },
+    { "----", 0, "sha1", 0 },
+    { "ac-e--f-", 0, "sha1", 3 },
+    { "aaaa", 5500, "sha1", 3 },
+    { "aaaa", 8192, "sha1", 2 },
+    { "aaaa", 16384, "sha1", 1 },
+    { "a--a", 4096, "sha1", 2 },
+    { "a--a", 5000, "sha1", 2 },
+    { "a--a", 3000, "sha256", 4 },
+    { "", 0, "sha1", 0 },
+};
+
+void test_vec (void)
+{
+    for (int i = 0; i < ARRAY_SIZE (testvec); i++) {
+        json_t *o;
+        bool rc;
+
+        mkfile ("testfile", 4096, testvec[i].spec);
+        skip ((strchr (testvec[i].spec, '-')
+              && (!have_sparse || !is_sparse ("testfile"))),
+              1, "sparse %s test file could not be created", testvec[i].spec);
+
+        o = xfileref_create_vec (mkpath ("testfile"),
+                             testvec[i].hashtype,
+                             testvec[i].chunksize);
+        rc = check_fileref (o, "testfile", testvec[i].exp_blobs);
+        ok (rc == true,
+            "fileref_create chunksize=%d '%s' works (%d %s blobrefs)",
+            testvec[i].chunksize,
+            testvec[i].spec,
+            testvec[i].exp_blobs,
+            testvec[i].hashtype);
+        json_decref (o);
+        end_skip;
+        rmfile ("testfile");
+    }
+}
+
+void test_dir (void)
+{
+    json_t *o;
+    bool rc;
+
+    if (mkdir (mkpath ("testdir"), 0510) < 0)
+        BAIL_OUT ("could not create test directory");
+    o = xfileref_create (mkpath ("testdir"));
+    diagjson (o);
+    rc = check_fileref (o, "testdir", 0);
+    ok (rc == true,
+        "fileref_create directory works");
+    json_decref (o);
+    rmdir (mkpath ("testdir"));
+}
+
+void test_link (void)
+{
+    json_t *o;
+    const char *target = "/a/b/c/d/e/f/g";
+    bool rc;
+
+    if (symlink (target, mkpath ("testlink")) < 0)
+        BAIL_OUT ("could not create test symlink");
+    o = xfileref_create (mkpath ("testlink"));
+    rc = check_fileref (o, "testlink", 0);
+    ok (rc == true,
+        "fileref_create symlink works");
+    json_decref (o);
+    rmfile ("testlink");
+}
+
+void test_small (void)
+{
+    json_t *o;
+    bool rc;
+    const char *encoding;
+
+    mkfile ("testsmall", 512, "a");
+    o = xfileref_create_vec (mkpath ("testsmall"), "sha1", 0);
+    rc = check_fileref (o, "testsmall", 0);
+    ok (rc == true,
+        "fileref_create small file works");
+    diagjson (o);
+    json_decref (o);
+    rmfile ("testsmall");
+
+    mkfile_string ("testsmall2", "\xc3\x28");
+    o = xfileref_create (mkpath ("testsmall2"));
+    ok (o != NULL && json_unpack (o, "{s:s}", "encoding", &encoding) == 0
+        && streq (encoding, "base64"),
+        "small file with invalid utf-8 encodes as base64");
+    diagjson (o);
+    json_decref (o);
+    rmfile ("testsmall2");
+
+    mkfile_string ("testsmall3", "abcd");
+    o = xfileref_create (mkpath ("testsmall3"));
+    ok (o != NULL && json_unpack (o, "{s:s}", "encoding", &encoding) == 0
+        && streq (encoding, "utf-8"),
+        "small file with valid utf-8 encodes as utf-8");
+    diagjson (o);
+    json_decref (o);
+    rmfile ("testsmall3");
+}
+
+void test_empty (void)
+{
+    json_t *o;
+    json_int_t size;
+
+    mkfile_empty ("testempty", 0);
+    o = xfileref_create (mkpath ("testempty"));
+    ok (o != NULL && json_object_get (o, "data") == NULL,
+        "empty file has no data member");
+    ok (o != NULL && json_object_get (o, "encoding") == NULL,
+        "empty file has no encoding member");
+    ok (o != NULL && json_unpack (o, "{s:I}", "size", &size) == 0
+            && size == 0,
+        "empty file has size member set to zero");
+    diagjson (o);
+    json_decref (o);
+    rmfile ("testempty");
+
+    if (!have_sparse) {
+        tap_skip (3, "test directory does not support sparse files");
+        return;
+    }
+    mkfile_empty ("testempty2", 1024);
+    o = xfileref_create (mkpath ("testempty2"));
+    ok (o != NULL && json_object_get (o, "data") == NULL,
+        "sparse,empty file has no data member");
+    ok (o != NULL && json_object_get (o, "encoding") == NULL,
+        "sparse,empty file has no encoding member");
+    ok (o != NULL && json_unpack (o, "{s:I}", "size", &size) == 0
+            && size == 1024,
+        "sparse,empty file has size member set to expected size");
+    diagjson (o);
+    json_decref (o);
+    rmfile ("testempty2");
+}
+
+void test_expfail (void)
+{
+    json_t *o;
+    flux_error_t error;
+    struct blobvec_param param;
+
+    mkfile ("test", 4096, "zz");
+
+    errno = 0;
+    o = fileref_create ("/noexist", &error);
+    if (!o)
+        diag ("%s", error.text);
+    ok (o == NULL && errno == ENOENT,
+        "fileref_create path=/noexist fails with ENOENT");
+
+    errno = 0;
+    o = fileref_create ("/dev/null", &error);
+    if (!o)
+        diag ("%s", error.text);
+    ok (o == NULL && errno == EINVAL,
+        "fileref_create path=/dev/null fails with EINVAL");
+
+    errno = 0;
+    param.chunksize = 1024;
+    param.hashtype = "smurfette";
+    param.small_file_threshold = 0;
+    o = fileref_create_ex (mkpath ("test"), &param, NULL, &error);
+    if (!o)
+        diag ("%s", error.text);
+    ok (o == NULL && errno == EINVAL,
+        "fileref_create_ex param.hashtype=smurfette fails with EINVAL");
+
+    rmfile ("test");
+}
+
+void test_pretty_print (void)
+{
+    char buf[1024];
+    json_t *o;
+
+    mkfile ("testfile", 4096, "a");
+    if (!(o = xfileref_create_vec (mkpath ("testfile"), "sha1", 0)))
+        BAIL_OUT ("failed to create test object");
+
+    buf[0] = '\0';
+    fileref_pretty_print (NULL, NULL, false, buf, sizeof (buf));
+    ok (streq (buf, "invalid fileref"),
+        "fileref_pretty_print obj=NULL printed an error");
+
+    buf[0] = '\0';
+    fileref_pretty_print (NULL, NULL, false, buf, 5);
+    ok (streq (buf, "inv+"),
+        "fileref_pretty_print obj=NULL bufsize=5 includes trunc character +");
+
+    buf[0] = '\0';
+    fileref_pretty_print (o, NULL, false, buf, sizeof (buf));
+    ok (strlen (buf) > 0,
+        "fileref_pretty_print long_form=false works");
+    diag (buf);
+
+    buf[0] = '\0';
+    fileref_pretty_print (o, NULL, true, buf, sizeof (buf));
+    ok (strlen (buf) > 0,
+        "fileref_pretty_print long_form=true works");
+    diag (buf);
+
+    lives_ok ({fileref_pretty_print (o, NULL, true, NULL, sizeof (buf));},
+             "fileref_pretty_print buf=NULL doesn't crash");
+
+    json_decref (o);
+    rmfile ("testfile");
+}
+
+int main (int argc, char *argv[])
+{
+    char *tmpdir = getenv ("TMPDIR");
+
+    plan (NO_PLAN);
+
+    /* create testdir to contain test files
+     */
+    snprintf (testdir,
+              sizeof (testdir),
+              "%s/fileref-XXXXXX",
+              tmpdir ? tmpdir : "/tmp");
+    if (!mkdtemp (testdir))
+        BAIL_OUT ("could not create test directory");
+
+    have_sparse = test_sparse ();
+
+    test_vec ();
+    test_dir ();
+    test_link ();
+    test_small ();
+    test_empty ();
+    test_expfail ();
+    test_pretty_print ();
+
+    unlink_recursive (testdir);
+
+    done_testing ();
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libflux-core.map b/src/common/libflux-core.map
deleted file mode 100644
index 01c7e91de803..000000000000
--- a/src/common/libflux-core.map
+++ /dev/null
@@ -1,7 +0,0 @@
-{ global:
-    flux_*;
-    JSC_*;
-    __asan*;
-    local: *;
-};
-
diff --git a/src/common/libflux-idset.map b/src/common/libflux-idset.map
deleted file mode 100644
index 2937af2fab7f..000000000000
--- a/src/common/libflux-idset.map
+++ /dev/null
@@ -1,6 +0,0 @@
-{ global:
-    idset_*;
-    __asan*;
-    local: *;
-};
-
diff --git a/src/common/libflux-optparse.map b/src/common/libflux-optparse.map
deleted file mode 100644
index 29c3d2c09df4..000000000000
--- a/src/common/libflux-optparse.map
+++ /dev/null
@@ -1,6 +0,0 @@
-{ global:
-    optparse_*;
-    __asan*;
-    local: *;
-};
-
diff --git a/src/common/libflux-schedutil.map b/src/common/libflux-schedutil.map
deleted file mode 100644
index 9794a2b4d12f..000000000000
--- a/src/common/libflux-schedutil.map
+++ /dev/null
@@ -1,6 +0,0 @@
-{ global:
-    schedutil_*;
-    __asan*;
-    local: *;
-};
-
diff --git a/src/common/libflux/Makefile.am b/src/common/libflux/Makefile.am
index b25181c1c636..8d173f521382 100644
--- a/src/common/libflux/Makefile.am
+++ b/src/common/libflux/Makefile.am
@@ -6,61 +6,18 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir) \
 	-I$(top_builddir)/src/common/libflux \
+	-DABS_TOP_BUILDDIR=\"${abs_top_builddir}\" \
+	-DABS_TOP_SRCDIR=\"${abs_top_srcdir}\" \
+	-DLUADIR=\"$(luadir)\" \
+	-DLUAEXECDIR=\"$(luaexecdir)\" \
 	$(JANSSON_CFLAGS) \
-	$(ZMQ_CFLAGS) \
-	$(LIBUUID_CFLAGS) \
-	$(LIBSODIUM_CFLAGS)
-
-installed_conf_cppflags = \
-	-DINSTALLED_MODULE_PATH=\"$(fluxmoddir)\" \
-	-DINSTALLED_CONNECTOR_PATH=\"$(fluxconnectordir)\" \
-	-DINSTALLED_EXEC_PATH=\"$(fluxcmddir)\" \
-	-DINSTALLED_LUA_PATH_ADD=\"$(luadir)/?.lua\" \
-	-DINSTALLED_LUA_CPATH_ADD=\"$(luaexecdir)/?.so\" \
-	-DINSTALLED_PYTHON_PATH=\"$(fluxpylinkdir)\" \
-	-DINSTALLED_MAN_PATH=\"$(mandir)\" \
-	-DINSTALLED_PROGRAM_LIBRARY_PATH=\"$(fluxlibdir)\" \
-	-DINSTALLED_PMI_LIBRARY_PATH=\"$(fluxlibdir)/libpmi.so\" \
-	-DINSTALLED_RC1_PATH=\"$(fluxrcdir)/rc1\" \
-	-DINSTALLED_RC3_PATH=\"$(fluxrcdir)/rc3\" \
-	-DINSTALLED_CF_PATH=\"$(fluxcfdir)\" \
-	-DINSTALLED_SHELL_PATH=\"$(fluxlibexecdir)/flux-shell\" \
-	-DINSTALLED_SHELL_PLUGINPATH=\"$(fluxlibdir)/shell/plugins\" \
-	-DINSTALLED_SHELL_INITRC=\"$(fluxrcdir)/shell/initrc.lua\" \
-	-DINSTALLED_CMDHELP_PATTERN=\"${datadir}/flux/help.d/*.json\" \
-	-DINSTALLED_NO_DOCS_PATH=\"${datadir}/flux/.nodocs\" \
-	-DINSTALLED_RUNDIR=\"${runstatedir}/flux\" \
-	-DINSTALLED_BINDIR=\"$(fluxcmddir)\" \
-	-DINSTALLED_JOBSPEC_VALIDATE_PATH=\"${fluxlibexecdir}/validate-jobspec.py\" \
-	-DINSTALLED_JOBSPEC_VALIDATOR_ARGS=\"\"
-
-intree_conf_cppflags = \
-	-DINTREE_MODULE_PATH=\"$(abs_top_builddir)/src/modules\" \
-	-DINTREE_CONNECTOR_PATH=\"$(abs_top_builddir)/src/connectors\" \
-	-DINTREE_EXEC_PATH=\"$(abs_top_builddir)/src/cmd:$(abs_top_srcdir)/src/cmd:$(abs_top_builddir)/src/broker\" \
-	-DINTREE_LUA_PATH_ADD=\"$(abs_top_builddir)/t/?.lua\;$(abs_top_srcdir)/src/bindings/lua/?.lua\" \
-	-DINTREE_LUA_CPATH_ADD=\"$(abs_top_builddir)/src/bindings/lua/?.so\" \
-	-DINTREE_PYTHON_PATH=\"$(abs_top_srcdir)/src/bindings/python:$(abs_top_builddir)/src/bindings/python\" \
-	-DINTREE_MAN_PATH=\"$(abs_top_builddir)/doc\" \
-	-DINTREE_PROGRAM_LIBRARY_PATH=\"$(abs_top_builddir)/src/common/.libs\" \
-	-DINTREE_PMI_LIBRARY_PATH=\"$(abs_top_builddir)/src/common/.libs/libpmi.so\" \
-	-DINTREE_RC1_PATH=\"$(abs_top_srcdir)/etc/rc1\" \
-	-DINTREE_RC3_PATH=\"$(abs_top_srcdir)/etc/rc3\" \
-	-DINTREE_CF_PATH=\"$(abs_top_srcdir)/etc/conf.d\" \
-	-DINTREE_SHELL_PATH=\"$(abs_top_builddir)/src/shell/flux-shell\" \
-	-DINTREE_SHELL_PLUGINPATH=\"$(abs_top_builddir)/src/shell/plugins\" \
-	-DINTREE_SHELL_INITRC=\"$(abs_top_srcdir)/src/shell/initrc.lua\" \
-	-DINTREE_CMDHELP_PATTERN=\"${abs_top_builddir}/etc/flux/help.d/*.json\" \
-	-DINTREE_KEYDIR=\"${abs_top_builddir}/etc/flux\" \
-	-DINTREE_NO_DOCS_PATH=\"${abs_top_builddir}/etc/flux/.nodocs\" \
-	-DINTREE_BINDIR=\"${abs_top_builddir}/src/cmd\" \
-	-DINTREE_JOBSPEC_VALIDATE_PATH=\"${abs_top_srcdir}/src/modules/job-ingest/validators/validate-jobspec.py\" \
-	-DINTREE_JOBSPEC_VALIDATOR_ARGS=\"\"
-
+	$(LIBUUID_CFLAGS)
 
 fluxcoreinclude_HEADERS = \
 	flux.h \
@@ -70,23 +27,23 @@ fluxcoreinclude_HEADERS = \
 	reactor.h \
 	msg_handler.h \
 	message.h \
+	msglist.h \
 	request.h \
-	keepalive.h \
+	control.h \
 	response.h \
 	rpc.h \
-	panic.h \
 	event.h \
 	module.h \
 	attr.h \
 	flog.h \
 	conf.h \
-	heartbeat.h \
-	content.h \
 	future.h \
 	barrier.h \
-	buffer.h \
 	service.h \
-	plugin.h
+	plugin.h \
+	sync.h \
+	disconnect.h \
+	stats.h
 
 nodist_fluxcoreinclude_HEADERS = \
 	version.h
@@ -98,52 +55,58 @@ libflux_la_SOURCES = \
 	flog.c \
 	attr.c \
 	handle.c \
+	msg_deque.c \
+	msg_deque.h \
+	connector_loop.c \
+	connector_interthread.c \
+	connector_local.c \
 	reactor.c \
+	reactor_private.h \
 	msg_handler.c \
 	message.c \
+	message_private.h \
+	message_iovec.h \
+	message_iovec.c \
+	message_route.h \
+	message_route.c \
+	message_proto.h \
+	message_proto.c \
+	msglist.c \
 	request.c \
 	response.c \
 	rpc.c \
-	panic.c \
 	event.c \
 	module.c \
 	conf_private.h \
 	conf.c \
-	tagpool.h \
-	tagpool.c \
 	ev_flux.h \
 	ev_flux.c \
-	ev_buffer_read.h \
-	ev_buffer_read.c \
-	ev_buffer_write.h \
-	ev_buffer_write.c \
-	heartbeat.c \
-	keepalive.c \
-	content.c \
+	control.c \
 	future.c \
 	composite_future.c \
 	barrier.c \
-	buffer_private.h \
-	buffer.c \
 	service.c \
 	version.c \
-	plugin.c
+	plugin.c \
+	plugin_private.h \
+	sync.c \
+	disconnect.c \
+	stats.c \
+	fripp.h \
+	fripp.c
 
-libflux_la_CPPFLAGS = \
-	$(installed_conf_cppflags) \
-	$(intree_conf_cppflags) \
-	$(AM_CPPFLAGS)
-libflux_la_LDFLAGS = -avoid-version -module -shared -export-dynamic
+libflux_la_LDFLAGS = \
+	$(AM_LDFLAGS)
 
 TESTS = test_message.t \
+	test_msglist.t \
+	test_interthread.t \
 	test_request.t \
 	test_response.t \
 	test_event.t \
-	test_tagpool.t \
 	test_future.t \
 	test_composite_future.t \
 	test_reactor.t \
-	test_buffer.t \
 	test_conf.t \
 	test_rpc.t \
 	test_rpc_chained.t \
@@ -155,34 +118,32 @@ TESTS = test_message.t \
 	test_log.t \
 	test_reactor_loop.t \
 	test_rpc_security.t \
-	test_panic.t \
 	test_attr.t \
 	test_module.t \
-	test_plugin.t
+	test_plugin.t \
+	test_sync.t \
+	test_disconnect.t \
+	test_msg_deque.t \
+	test_rpcscale.t
 
 test_ldadd = \
 	$(top_builddir)/src/common/libtestutil/libtestutil.la \
 	$(top_builddir)/src/common/libflux/libflux.la \
-	$(top_builddir)/src/common/libutil/libutil.la \
-	$(top_builddir)/src/common/libidset/libidset.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libkvs/libkvs.la \
 	$(top_builddir)/src/common/libtap/libtap.la \
-	$(top_builddir)/src/common/liblsd/liblsd.la \
-	$(top_builddir)/src/common/libtomlc99/libtomlc99.la \
-	$(top_builddir)/src/common/libev/libev.la \
-	$(ZMQ_LIBS) \
 	$(LIBUUID_LIBS) \
 	$(JANSSON_LIBS) \
 	$(LIBPTHREAD) \
-	$(LIBSODIUM_LIBS)
+	$(LIBDL)
 
 test_cppflags = \
         -I$(top_srcdir)/src/common/libtap \
         $(AM_CPPFLAGS)
 
 check_LTLIBRARIES = \
-	test/module_fake1.la \
-	test/module_fake2.la \
-	test/plugin_foo.la
+	test/plugin_foo.la \
+	test/plugin_bar.la
 
 
 check_PROGRAMS = $(TESTS)
@@ -193,107 +154,116 @@ T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
 
 test_message_t_SOURCES = test/message.c
 test_message_t_CPPFLAGS = $(test_cppflags)
-test_message_t_LDADD = $(test_ldadd) $(LIBDL)
+test_message_t_LDADD = $(test_ldadd)
+
+test_msglist_t_SOURCES = test/msglist.c
+test_msglist_t_CPPFLAGS = $(test_cppflags)
+test_msglist_t_LDADD = $(test_ldadd)
 
 test_event_t_SOURCES = test/event.c
 test_event_t_CPPFLAGS = $(test_cppflags)
-test_event_t_LDADD = $(test_ldadd) $(LIBDL)
-
-test_tagpool_t_SOURCES = test/tagpool.c
-test_tagpool_t_CPPFLAGS = $(test_cppflags)
-test_tagpool_t_LDADD = $(test_ldadd) $(LIBDL)
+test_event_t_LDADD = $(test_ldadd)
 
 test_request_t_SOURCES = test/request.c
 test_request_t_CPPFLAGS = $(test_cppflags)
-test_request_t_LDADD = $(test_ldadd) $(LIBDL)
+test_request_t_LDADD = $(test_ldadd)
 
 test_response_t_SOURCES = test/response.c
 test_response_t_CPPFLAGS = $(test_cppflags)
-test_response_t_LDADD = $(test_ldadd) $(LIBDL)
+test_response_t_LDADD = $(test_ldadd)
 
 test_reactor_t_SOURCES = test/reactor.c
 test_reactor_t_CPPFLAGS = $(test_cppflags)
-test_reactor_t_LDADD = $(test_ldadd) $(LIBDL)
+test_reactor_t_LDADD = $(test_ldadd)
 
 test_future_t_SOURCES = test/future.c
 test_future_t_CPPFLAGS = $(test_cppflags)
-test_future_t_LDADD = $(test_ldadd) $(LIBDL)
+test_future_t_LDADD = $(test_ldadd)
 
 test_composite_future_t_SOURCES = test/composite_future.c
 test_composite_future_t_CPPFLAGS = $(test_cppflags)
-test_composite_future_t_LDADD = $(test_ldadd) $(LIBDL)
+test_composite_future_t_LDADD = $(test_ldadd)
 
 test_conf_t_SOURCES = test/conf.c
 test_conf_t_CPPFLAGS = $(test_cppflags)
-test_conf_t_LDADD = $(test_ldadd) $(LIBDL)
-
-test_buffer_t_SOURCES = test/buffer.c
-test_buffer_t_CPPFLAGS = $(test_cppflags)
-test_buffer_t_LDADD = $(test_ldadd) $(LIBDL)
+test_conf_t_LDADD = $(test_ldadd)
 
 test_handle_t_SOURCES = test/handle.c
 test_handle_t_CPPFLAGS = $(test_cppflags)
-test_handle_t_LDADD = $(test_ldadd) $(LIBDL)
+test_handle_t_LDADD = $(test_ldadd)
 
 test_msg_handler_t_SOURCES = test/msg_handler.c
 test_msg_handler_t_CPPFLAGS = $(test_cppflags)
-test_msg_handler_t_LDADD = $(test_ldadd) $(LIBDL)
+test_msg_handler_t_LDADD = $(test_ldadd)
 
 test_version_t_SOURCES = test/version.c
 test_version_t_CPPFLAGS = $(test_cppflags)
-test_version_t_LDADD = $(test_ldadd) $(LIBDL)
+test_version_t_LDADD = $(test_ldadd)
 
 test_rpc_t_SOURCES = test/rpc.c
 test_rpc_t_CPPFLAGS = $(test_cppflags)
-test_rpc_t_LDADD = $(test_ldadd) $(LIBDL)
+test_rpc_t_LDADD = $(test_ldadd)
+
+test_rpcscale_t_SOURCES = test/rpcscale.c
+test_rpcscale_t_CPPFLAGS = $(test_cppflags)
+test_rpcscale_t_LDADD = \
+	$(top_builddir)/src/common/liboptparse/liboptparse.la \
+	$(test_ldadd)
 
 test_rpc_chained_t_SOURCES = test/rpc_chained.c
 test_rpc_chained_t_CPPFLAGS = $(test_cppflags)
-test_rpc_chained_t_LDADD = $(test_ldadd) $(LIBDL)
+test_rpc_chained_t_LDADD = $(test_ldadd)
 
 test_dispatch_t_SOURCES = test/dispatch.c
 test_dispatch_t_CPPFLAGS = $(test_cppflags)
-test_dispatch_t_LDADD = $(test_ldadd) $(LIBDL)
+test_dispatch_t_LDADD = $(test_ldadd)
 
 test_log_t_SOURCES = test/log.c
 test_log_t_CPPFLAGS = $(test_cppflags)
-test_log_t_LDADD = $(test_ldadd) $(LIBDL)
+test_log_t_LDADD = $(test_ldadd)
 
 test_reactor_loop_t_SOURCES = test/reactor_loop.c
 test_reactor_loop_t_CPPFLAGS = $(test_cppflags)
-test_reactor_loop_t_LDADD = $(test_ldadd) $(LIBDL)
+test_reactor_loop_t_LDADD = $(test_ldadd)
 
 test_rpc_security_t_SOURCES = test/rpc_security.c
 test_rpc_security_t_CPPFLAGS = $(test_cppflags)
-test_rpc_security_t_LDADD = $(test_ldadd) $(LIBDL)
-
-test_panic_t_SOURCES = test/panic.c
-test_panic_t_CPPFLAGS = $(test_cppflags)
-test_panic_t_LDADD = $(test_ldadd) $(LIBDL)
+test_rpc_security_t_LDADD = $(test_ldadd)
 
 test_attr_t_SOURCES = test/attr.c
 test_attr_t_CPPFLAGS = $(test_cppflags)
-test_attr_t_LDADD = $(test_ldadd) $(LIBDL)
+test_attr_t_LDADD = $(test_ldadd)
 
-test_module_t_SOURCES = test/module.c
-test_module_t_CPPFLAGS = $(test_cppflags) \
-	-DFAKE1=\"$(abs_builddir)/test/.libs/module_fake1.so\" \
-	-DFAKE2=\"$(abs_builddir)/test/.libs/module_fake2.so\"
-test_module_t_LDADD = $(test_ldadd) $(LIBDL)
+test_sync_t_SOURCES = test/sync.c
+test_sync_t_CPPFLAGS = $(test_cppflags)
+test_sync_t_LDADD = $(test_ldadd)
+
+test_disconnect_t_SOURCES = test/disconnect.c
+test_disconnect_t_CPPFLAGS = $(test_cppflags)
+test_disconnect_t_LDADD = $(test_ldadd)
 
-test_module_fake1_la_SOURCES = test/module_fake1.c
-test_module_fake1_la_CPPFLAGS = $(test_cppflags)
-test_module_fake1_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowhere
+test_interthread_t_SOURCES = test/interthread.c
+test_interthread_t_CPPFLAGS = $(test_cppflags)
+test_interthread_t_LDADD = $(test_ldadd)
 
-test_module_fake2_la_SOURCES = test/module_fake2.c
-test_module_fake2_la_CPPFLAGS = $(test_cppflags)
-test_module_fake2_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowhere
+test_msg_deque_t_SOURCES = test/msg_deque.c
+test_msg_deque_t_CPPFLAGS = $(test_cppflags)
+test_msg_deque_t_LDADD = $(test_ldadd)
+
+test_module_t_SOURCES = test/module.c
+test_module_t_CPPFLAGS = $(test_cppflags)
+test_module_t_LDADD = $(test_ldadd)
 
 test_plugin_t_SOURCES = test/plugin.c
 test_plugin_t_CPPFLAGS = $(test_cppflags)
-test_plugin_t_LDADD = $(test_ldadd) $(LIBDL)
+test_plugin_t_LDADD = $(test_ldadd)
 
 test_plugin_foo_la_SOURCES = test/plugin_foo.c
 test_plugin_foo_la_CPPFLAGS = $(test_cppflags)
 test_plugin_foo_la_LDFLAGS = -module -rpath /nowhere
-test_plugin_foo_la_LIBADD = $(test_ldadd) $(LIBDL)
+test_plugin_foo_la_LIBADD = $(test_ldadd)
+
+test_plugin_bar_la_SOURCES = test/plugin_bar.c
+test_plugin_bar_la_CPPFLAGS = $(test_cppflags)
+test_plugin_bar_la_LDFLAGS = -module -rpath /nowhere
+test_plugin_bar_la_LIBADD = $(test_ldadd)
diff --git a/src/common/libflux/attr.c b/src/common/libflux/attr.c
index 820f14010a63..5d3ef7686d6b 100644
--- a/src/common/libflux/attr.c
+++ b/src/common/libflux/attr.c
@@ -13,11 +13,14 @@
 #endif
 #include <errno.h>
 #include <stdbool.h>
-#include <czmq.h>
 #include <jansson.h>
+#include <flux/core.h>
 
-#include "attr.h"
-#include "rpc.h"
+#include "ccan/str/str.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libhostlist/hostlist.h"
+#include "src/common/libidset/idset.h"
+#include "src/common/libutil/errprintf.h"
 
 enum {
     FLUX_ATTRFLAG_IMMUTABLE = 1,
@@ -26,14 +29,19 @@ enum {
 struct attr_cache {
     zhashx_t *cache;        // immutable values
     zhashx_t *temp;         // values that stay valid until next lookup
+
+    zlistx_t *cache_iter;
+    struct hostlist *hostlist;
 };
 
 static void attr_cache_destroy (struct attr_cache *c)
 {
     if (c) {
         int saved_errno = errno;
+        zlistx_destroy (&c->cache_iter);
         zhashx_destroy (&c->cache);
         zhashx_destroy (&c->temp);
+        hostlist_destroy (c->hostlist);
         free (c);
         errno = saved_errno;
     }
@@ -85,8 +93,10 @@ const char *flux_attr_get (flux_t *h, const char *name)
     struct attr_cache *c;
     const char *val;
     int flags;
-    flux_future_t *f;
+    flux_future_t *f = NULL;
     char *cpy = NULL;
+    const char *orig_name = name;
+    char *proxy_remote = NULL;
 
     if (!h || !name) {
         errno = EINVAL;
@@ -94,15 +104,58 @@ const char *flux_attr_get (flux_t *h, const char *name)
     }
     if (!(c = get_attr_cache (h)))
         return NULL;
+
+    /*  The attribute parent-uri is treated specifically here, since
+     *  a process connected to this instance via flux-proxy(1) should
+     *  have the parent-uri returned as a usable remote URI.
+     *
+     *  Therefore, if FLUX_PROXY_REMOTE is set in the current environment,
+     *  and the attribute is parent-uri, instead actually return the
+     *  handle-specific parent-remote-uri attribute. If this is not yet
+     *  available, fetch parent-uri and construct the remote uri by
+     *  substituting FLUX_PROXY_REMOTE in an ssh:// uri.
+     */
+    if (streq (name, "parent-uri")
+        && (proxy_remote = getenv ("FLUX_PROXY_REMOTE")))
+        name = "parent-remote-uri";
     if ((val = zhashx_lookup (c->cache, name)))
         return val;
-    if (!(f = flux_rpc_pack (h, "attr.get", FLUX_NODEID_ANY, 0, "{s:s}",
-                                                                "name", name)))
-        return NULL;
-    if (flux_rpc_get_unpack (f, "{s:s s:i}", "value", &val,
-                                             "flags", &flags) < 0)
-        goto done;
-    if (!(cpy = strdup (val)))
+
+    /*  If FLUX_PROXY_REMOTE was set, try a lookup of 'orig_name' in
+     *  the cache before attempting an RPC. If successful, then set
+     *  the immutable flag for the parent-remote-uri attr (it should not
+     *  change), since 'flags' will not be set by the RPC, which is being
+     *  skipped.
+     */
+    if (proxy_remote && (val = zhashx_lookup (c->cache, orig_name)))
+        flags = FLUX_ATTRFLAG_IMMUTABLE;
+
+    /*  If we still don't have a value for this attribute, try an RPC:
+     */
+    if (!val) {
+        if (!(f = flux_rpc_pack (h,
+                                 "attr.get",
+                                 FLUX_NODEID_ANY,
+                                 0,
+                                 "{s:s}",
+                                 "name", orig_name)))
+            return NULL;
+        if (flux_rpc_get_unpack (f,
+                                 "{s:s s:i}",
+                                 "value", &val,
+                                 "flags", &flags) < 0)
+            goto done;
+    }
+
+    /*  If proxy_remote is non-NULL then parent-uri has been aliased to
+     *  parent-remote-uri. Swap a local URI to a remote:
+     */
+    if (proxy_remote
+        && strstarts (val, "local://")) {
+        if (asprintf (&cpy, "ssh://%s%s", proxy_remote, val+8) < 0)
+            goto done;
+    }
+    else if (!(cpy = strdup (val)))
         goto done;
     if ((flags & FLUX_ATTRFLAG_IMMUTABLE))
         zhashx_update (c->cache, name, cpy);
@@ -117,17 +170,17 @@ int flux_attr_set (flux_t *h, const char *name, const char *val)
 {
     flux_future_t *f;
 
-    if (!h || !name) {
+    if (!h || !name || !val) {
         errno = EINVAL;
         return -1;
     }
-    if (val)
-        f = flux_rpc_pack (h, "attr.set", FLUX_NODEID_ANY, 0, "{s:s s:s}",
-                                                              "name", name,
-                                                              "value", val);
-    else
-        f = flux_rpc_pack (h, "attr.rm", FLUX_NODEID_ANY, 0, "{s:s}",
-                                                             "name", name);
+    f = flux_rpc_pack (h,
+                       "attr.set",
+                       FLUX_NODEID_ANY,
+                       0,
+                       "{s:s s:s}",
+                       "name", name,
+                       "value", val);
     if (!f)
         return -1;
     if (flux_future_get (f, NULL) < 0) {
@@ -163,6 +216,29 @@ int flux_attr_set_cacheonly (flux_t *h, const char *name, const char *val)
     return 0;
 }
 
+const char *flux_attr_cache_first (flux_t *h)
+{
+    struct attr_cache *c;
+
+    if (!h || !(c = get_attr_cache (h)))
+        return NULL;
+    zlistx_destroy (&c->cache_iter);
+    if (!(c->cache_iter = zhashx_keys (c->cache)))
+        return NULL;
+    return zlistx_first (c->cache_iter);
+}
+
+const char *flux_attr_cache_next (flux_t *h)
+{
+    struct attr_cache *c;
+
+    if (!h || !(c = get_attr_cache (h)))
+        return NULL;
+    if (!c->cache_iter)
+        return NULL;
+    return zlistx_next (c->cache_iter);
+}
+
 int flux_get_size (flux_t *h, uint32_t *size)
 {
     const char *val;
@@ -183,6 +259,154 @@ int flux_get_rank (flux_t *h, uint32_t *rank)
     return 0;
 }
 
+static struct hostlist * get_hostlist (flux_t *h)
+{
+    struct attr_cache *c;
+
+    if (!(c = get_attr_cache (h)))
+        return NULL;
+    if (!c->hostlist) {
+        const char *val;
+        if (!(val = flux_attr_get (h, "hostlist"))
+            || !(c->hostlist = hostlist_decode (val)))
+            return NULL;
+    }
+    return c->hostlist;
+}
+
+const char *flux_get_hostbyrank (flux_t *h, uint32_t rank)
+{
+    struct hostlist *hl;
+    const char *result;
+
+    if (rank == FLUX_NODEID_ANY)
+        return "any";
+    if (rank == FLUX_NODEID_UPSTREAM)
+        return "upstream";
+    if (!(hl = get_hostlist (h)))
+        goto error;
+    if (!(result = hostlist_nth (hl, rank)))
+        goto error;
+    return result;
+error:
+    return "(null)";
+}
+
+int flux_get_rankbyhost (flux_t *h, const char *host)
+{
+    struct hostlist *hl;
+
+    if (!(hl = get_hostlist (h)))
+        return -1;
+    return hostlist_find (hl, host);
+}
+
+char *flux_hostmap_lookup (flux_t *h,
+                           const char *targets,
+                           flux_error_t *errp)
+{
+    struct hostlist *hostmap;
+    struct hostlist *hosts = NULL;
+    struct idset *ranks = NULL;
+    char *s = NULL;
+
+    if (!(hostmap = get_hostlist (h))) {
+        errprintf (errp, "%s", strerror (errno));
+        return NULL;
+    }
+
+    if ((ranks = idset_decode (targets))) {
+        unsigned int rank;
+        if (!(hosts = hostlist_create ())) {
+            errprintf (errp, "Out of memory");
+            goto err;
+        }
+        rank = idset_first (ranks);
+        while (rank != IDSET_INVALID_ID) {
+            const char *host;
+            if (!(host = hostlist_nth (hostmap, rank))) {
+                errprintf (errp, "rank %u is not in host map", rank);
+                goto err;
+            }
+            if (hostlist_append (hosts, host) < 0) {
+                errprintf (errp,
+                           "failed appending host %s: %s",
+                           host,
+                           strerror (errno));
+                goto err;
+            }
+            rank = idset_next (ranks, rank);
+        }
+        if (!(s = hostlist_encode (hosts))) {
+            errprintf (errp,
+                       "hostlist_encode: %s",
+                       strerror (errno));
+            goto err;
+        }
+    }
+    else if ((hosts = hostlist_decode (targets))) {
+        const char *host;
+        int rank = 0;
+        if (!(ranks = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+            errprintf (errp, "Out of memory");
+            goto err;
+        }
+        host = hostlist_first (hosts);
+        while (host) {
+            if ((rank = hostlist_find (hostmap, host)) < 0) {
+                errprintf (errp, "host %s not found in host map", host);
+                goto err;
+            }
+            if (idset_set (ranks, rank) < 0) {
+                errprintf (errp,
+                           "idset_set (rank=%d): %s",
+                           rank,
+                           strerror (errno));
+                goto err;
+            }
+            host = hostlist_next (hosts);
+        }
+        if (!(s = idset_encode (ranks, IDSET_FLAG_RANGE))) {
+            errprintf (errp,
+                       "error encoding idset: %s",
+                       strerror (errno));
+            goto err;
+        }
+    }
+    else {
+        errprintf (errp, "target must be a valid idset or hostlist");
+        goto err;
+    }
+err:
+    hostlist_destroy (hosts);
+    idset_destroy (ranks);
+    return s;
+}
+
+int flux_get_instance_starttime (flux_t *h, double *starttimep)
+{
+    flux_future_t *f;
+    const char *attr = "broker.starttime";
+    const char *s;
+    double starttime;
+
+    if (!(f = flux_rpc_pack (h, "attr.get", 0, 0, "{s:s}", "name", attr)))
+        return -1;
+    if (flux_rpc_get_unpack (f, "{s:s}", "value", &s) < 0)
+        goto error;
+    errno = 0;
+    starttime = strtod (s, NULL);
+    if (errno != 0)
+        goto error;
+    flux_future_destroy (f);
+    if (starttimep)
+        *starttimep = starttime;
+    return 0;
+error:
+    flux_future_destroy (f);
+    return -1;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libflux/attr.h b/src/common/libflux/attr.h
index 59fbe9b6164c..5e01e8b03545 100644
--- a/src/common/libflux/attr.h
+++ b/src/common/libflux/attr.h
@@ -21,12 +21,10 @@
  *   flux lsattr [-v]
  *   flux setattr name value
  *   flux getattr name
- * In additon, the following functions may be used to get/set broker
+ * In addition, the following functions may be used to get/set broker
  * attributes programmatically.
  */
 
-#include "handle.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -46,10 +44,14 @@ const char *flux_attr_get (flux_t *h, const char *name);
  */
 int flux_attr_set (flux_t *h, const char *name, const char *val);
 
-
 /* hotwire flux_attr_get()'s cache for testing */
 int flux_attr_set_cacheonly (flux_t *h, const char *name, const char *val);
 
+/* Iterate over the attribute names that are stored in the local
+ * attribute cache.
+ */
+const char *flux_attr_cache_first (flux_t *h);
+const char *flux_attr_cache_next (flux_t *h);
 
 /* Get "rank" attribute, and convert to an unsigned integer.
  * Returns 0 on success, or -1 on failure with errno set.
@@ -61,6 +63,41 @@ int flux_get_rank (flux_t *h, uint32_t *rank);
  */
 int flux_get_size (flux_t *h, uint32_t *size);
 
+/* Look up hostname of broker rank, by consulting "hostlist" attribute.
+ * This function always returns a printable string, though it may be "(null)".
+ */
+const char *flux_get_hostbyrank (flux_t *h, uint32_t rank);
+
+/* Find the lowest numbered broker rank running on 'host', by consulting
+ * the "hostlist" attribute.
+ * Returns rank on success, -1 on failure with errno set.
+ */
+int flux_get_rankbyhost (flux_t *h, const char *host);
+
+/* Return a list/set of hosts/ranks in Hostlist/Idset form given 'targets'
+ * in Idset/Hostlist form. Caller must free returned value.
+ *
+ * Returns NULL on failure with error message in errp->text (if errp != NULL).
+ *
+ * NOTES:
+ *  - The source of the mapping is the rank-ordered broker 'hostlist' attribute.
+ *  - An Idset (RFC 22) is a set (unordered, no duplicates)
+ *  - A Hostlist (RFC 29) is a list (ordered, may be duplicates)
+ *  - If there are multiple ranks per host, this function can only map
+ *    hostnames to the first rank found on the host.
+ *
+ */
+char *flux_hostmap_lookup (flux_t *h,
+                           const char *targets,
+                           flux_error_t *errp);
+
+/* Look up the broker.starttime attribute on rank 0.
+ * The instance uptime is flux_reactor_now() - starttime.
+ * N.B. if the instance has been restarted, this value is the most
+ * recent restart time.
+ */
+int flux_get_instance_starttime (flux_t *h, double *starttime);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/common/libflux/barrier.c b/src/common/libflux/barrier.c
index 34a6cfafb5fe..a93a5ce53bf0 100644
--- a/src/common/libflux/barrier.c
+++ b/src/common/libflux/barrier.c
@@ -12,9 +12,6 @@
 #include "config.h"
 #endif
 #include <flux/core.h>
-#include <stdbool.h>
-
-#include "src/common/libutil/xzmalloc.h"
 
 typedef struct {
     const char *id;
@@ -43,15 +40,11 @@ static libbarrier_ctx_t *getctx (flux_t *h)
             errno = EINVAL;
             goto error;
         }
-        if (!(ctx = calloc (1, sizeof (*ctx)))) {
-            errno = ENOMEM;
+        if (!(ctx = calloc (1, sizeof (*ctx))))
             goto error;
-        }
         ctx->name_len = strlen (id) + 16;
-        if (!(ctx->name = calloc (1, ctx->name_len))) {
-            errno = ENOMEM;
+        if (!(ctx->name = calloc (1, ctx->name_len)))
             goto error;
-        }
         ctx->id = id;
         if (flux_aux_set (h, "flux::barrier_client", ctx, freectx) < 0)
             goto error;
@@ -80,13 +73,21 @@ flux_future_t *flux_barrier (flux_t *h, const char *name, int nprocs)
     if (!name && !(name = generate_unique_name (h)))
         return NULL;
 
-    return flux_rpc_pack (h,
-                          "barrier.enter",
-                          FLUX_NODEID_ANY,
-                          0,
-                          "{s:s s:i}",
-                           "name", name,
-                           "nprocs", nprocs);
+    if (nprocs == 1) {
+        flux_future_t *f = flux_future_create (NULL, NULL);
+        if (f)
+            flux_future_fulfill (f, NULL, NULL);
+        return f;
+    }
+    else {
+        return flux_rpc_pack (h,
+                              "barrier.enter",
+                              FLUX_NODEID_ANY,
+                              0,
+                              "{s:s s:i}",
+                               "name", name,
+                               "nprocs", nprocs);
+    }
 }
 
 /*
diff --git a/src/common/libflux/barrier.h b/src/common/libflux/barrier.h
index 4a1c368a6524..ef18cba5f56f 100644
--- a/src/common/libflux/barrier.h
+++ b/src/common/libflux/barrier.h
@@ -11,14 +11,12 @@
 #ifndef _FLUX_CORE_BARRIER_H
 #define _FLUX_CORE_BARRIER_H
 
-#include "handle.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /* Execute a barrier across 'nprocs' processes.
- * The 'name' must be unique across the comms session, or
+ * The 'name' must be unique across the Flux instance, or
  * if running in a Flux/slurm job, may be NULL.
  */
 flux_future_t *flux_barrier (flux_t *h, const char *name, int nprocs);
diff --git a/src/common/libflux/buffer.c b/src/common/libflux/buffer.c
deleted file mode 100644
index a2de42b59157..000000000000
--- a/src/common/libflux/buffer.c
+++ /dev/null
@@ -1,584 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <assert.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdbool.h>
-#include <stdarg.h>
-
-#include "buffer.h"
-#include "buffer_private.h"
-
-#include "src/common/liblsd/cbuf.h"
-
-#define FLUX_BUFFER_MIN   4096
-#define FLUX_BUFFER_MAGIC 0xeb4feb4f
-
-enum {
-    FLUX_BUFFER_CB_TYPE_NONE,
-    FLUX_BUFFER_CB_TYPE_READ,
-    FLUX_BUFFER_CB_TYPE_READ_LINE,
-    FLUX_BUFFER_CB_TYPE_WRITE,
-};
-
-struct flux_buffer {
-    int magic;
-    int size;
-    bool readonly;
-    cbuf_t cbuf;
-    char *buf;                  /* internal buffer for user reads */
-    int buflen;
-    int cb_type;
-    flux_buffer_cb cb;
-    int cb_len;
-    void *cb_arg;
-};
-
-flux_buffer_t *flux_buffer_create (int size)
-{
-    flux_buffer_t *fb = NULL;
-    int minsize = FLUX_BUFFER_MIN;
-
-    if (size <= 0) {
-        errno = EINVAL;
-        goto cleanup;
-    }
-
-    if (!(fb = calloc (1, sizeof (*fb)))) {
-        errno = ENOMEM;
-        goto cleanup;
-    }
-
-    fb->magic = FLUX_BUFFER_MAGIC;
-    fb->size = size;
-    if (size < FLUX_BUFFER_MIN)
-        minsize = size;
-    else
-        minsize = FLUX_BUFFER_MIN;
-    fb->readonly = false;
-
-    /* buffer can grow to size specified by user */
-    if (!(fb->cbuf = cbuf_create (minsize, fb->size)))
-        goto cleanup;
-
-    if (cbuf_opt_set (fb->cbuf, CBUF_OPT_OVERWRITE, CBUF_NO_DROP) < 0)
-        goto cleanup;
-
-    /* +1 for possible NUL on line reads */
-    fb->buflen = minsize + 1;
-
-    if (!(fb->buf = malloc (fb->buflen))) {
-        errno = ENOMEM;
-        goto cleanup;
-    }
-
-    fb->cb_type = FLUX_BUFFER_CB_TYPE_NONE;
-
-    return fb;
-
-cleanup:
-    flux_buffer_destroy (fb);
-    return NULL;
-}
-
-void flux_buffer_destroy (void *data)
-{
-    flux_buffer_t *fb = data;
-    if (fb && fb->magic == FLUX_BUFFER_MAGIC) {
-        fb->magic = ~FLUX_BUFFER_MAGIC;
-        cbuf_destroy (fb->cbuf);
-        free (fb->buf);
-        free (fb);
-    }
-}
-
-int flux_buffer_size (flux_buffer_t *fb)
-{
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    return fb->size;
-}
-
-int flux_buffer_bytes (flux_buffer_t *fb)
-{
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    return cbuf_used (fb->cbuf);
-}
-
-int flux_buffer_space (flux_buffer_t *fb)
-{
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    return cbuf_free (fb->cbuf);
-}
-
-int flux_buffer_readonly (flux_buffer_t *fb)
-{
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    fb->readonly = true;
-    return 0;
-}
-
-bool flux_buffer_is_readonly (flux_buffer_t *fb)
-{
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return false;
-    }
-    return fb->readonly;
-}
-
-static int set_cb (flux_buffer_t *fb,
-                   int cb_type,
-                   flux_buffer_cb cb,
-                   int cb_len,
-                   void *cb_arg)
-{
-    if (fb->cb_type == FLUX_BUFFER_CB_TYPE_NONE) {
-        if (!cb)
-            return 0;
-
-        if (cb_len < 0) {
-            errno = EINVAL;
-            return -1;
-        }
-
-        fb->cb_type = cb_type;
-        fb->cb = cb;
-        fb->cb_len = cb_len;
-        fb->cb_arg = cb_arg;
-    }
-    else if (fb->cb_type == cb_type) {
-        if (!cb) {
-            fb->cb_type = FLUX_BUFFER_CB_TYPE_NONE;
-            fb->cb = NULL;
-            fb->cb_len = 0;
-            fb->cb_arg = NULL;
-        }
-        else {
-            if (cb_len < 0) {
-                errno = EINVAL;
-                return -1;
-            }
-
-            fb->cb_type = cb_type;
-            fb->cb = cb;
-            fb->cb_len = cb_len;
-            fb->cb_arg = cb_arg;
-        }
-    }
-    else {
-        errno = EEXIST;
-        return -1;
-    }
-
-    return 0;
-}
-
-int flux_buffer_set_low_read_cb (flux_buffer_t *fb,
-                                 flux_buffer_cb cb,
-                                 int low,
-                                 void *arg)
-{
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    return set_cb (fb, FLUX_BUFFER_CB_TYPE_READ, cb, low, arg);
-}
-
-int flux_buffer_set_read_line_cb (flux_buffer_t *fb,
-                                  flux_buffer_cb cb,
-                                  void *arg)
-{
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    return set_cb (fb, FLUX_BUFFER_CB_TYPE_READ_LINE, cb, 0, arg);
-}
-
-int flux_buffer_set_high_write_cb (flux_buffer_t *fb,
-                                   flux_buffer_cb cb,
-                                   int high,
-                                   void *arg)
-{
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    return set_cb (fb, FLUX_BUFFER_CB_TYPE_WRITE, cb, high, arg);
-}
-
-void check_write_cb (flux_buffer_t *fb)
-{
-    if (fb->cb_type == FLUX_BUFFER_CB_TYPE_WRITE
-        && flux_buffer_bytes (fb) < fb->cb_len) {
-        fb->cb (fb, fb->cb_arg);
-    }
-}
-
-void check_read_cb (flux_buffer_t *fb)
-{
-    if (fb->cb_type == FLUX_BUFFER_CB_TYPE_READ
-        && flux_buffer_bytes (fb) > fb->cb_len)
-            fb->cb (fb, fb->cb_arg);
-    else if (fb->cb_type == FLUX_BUFFER_CB_TYPE_READ_LINE
-             && flux_buffer_has_line (fb))
-            fb->cb (fb, fb->cb_arg);
-}
-
-int flux_buffer_drop (flux_buffer_t *fb, int len)
-{
-    int ret;
-
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if ((ret = cbuf_drop (fb->cbuf, len)) < 0)
-        return -1;
-
-    check_write_cb (fb);
-
-    return ret;
-}
-
-/* check if internal buffer can hold data from user */
-static int return_buffer_check (flux_buffer_t *fb)
-{
-    int used = cbuf_used (fb->cbuf);
-
-    if (used < 0)
-        return -1;
-
-    assert (used <= fb->size);
-
-    /* +1 for potential NUL char */
-    if (fb->buflen < (used + 1)) {
-        size_t newsize = fb->buflen;
-        char *newbuf;
-
-        while ((newsize < (used + 1))) {
-            newsize = (newsize - 1) * 2 + 1;
-            if (newsize > (fb->size + 1))
-                newsize = fb->size + 1;
-        }
-
-        if (!(newbuf = realloc (fb->buf, newsize)))
-            return -1;
-        fb->buf = newbuf;
-        fb->buflen = newsize;
-    }
-
-    return 0;
-}
-
-const void *flux_buffer_peek (flux_buffer_t *fb, int len, int *lenp)
-{
-    int ret;
-
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    if (return_buffer_check (fb) < 0)
-        return NULL;
-
-    if (len < 0)
-        len = cbuf_used (fb->cbuf);
-
-    if (len > fb->buflen)
-        len = fb->buflen;
-
-    if ((ret = cbuf_peek (fb->cbuf, fb->buf, len)) < 0)
-        return NULL;
-    fb->buf[ret] = '\0';
-
-    if (lenp)
-        (*lenp) = ret;
-
-    return fb->buf;
-}
-
-const void *flux_buffer_read (flux_buffer_t *fb, int len, int *lenp)
-{
-    int ret;
-
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    if (return_buffer_check (fb) < 0)
-        return NULL;
-
-    if (len < 0)
-        len = cbuf_used (fb->cbuf);
-
-    if (len > fb->buflen)
-        len = fb->buflen;
-
-    if ((ret = cbuf_read (fb->cbuf, fb->buf, len)) < 0)
-        return NULL;
-    fb->buf[ret] = '\0';
-
-    if (lenp)
-        (*lenp) = ret;
-
-    check_write_cb (fb);
-
-    return fb->buf;
-}
-
-int flux_buffer_write (flux_buffer_t *fb, const void *data, int len)
-{
-    int ret;
-
-    if (!fb
-        || fb->magic != FLUX_BUFFER_MAGIC
-        || !data
-        || len < 0) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if (fb->readonly) {
-        errno = EROFS;
-        return -1;
-    }
-
-    if ((ret = cbuf_write (fb->cbuf, (void *)data, len, NULL)) < 0)
-        return -1;
-
-    check_read_cb (fb);
-
-    return ret;
-}
-
-int flux_buffer_lines (flux_buffer_t *fb)
-{
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    return cbuf_lines_used (fb->cbuf);
-}
-
-bool flux_buffer_has_line (flux_buffer_t *fb)
-{
-    char buf[1];
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return false;
-    }
-    return (cbuf_peek_line (fb->cbuf, buf, 0, 1) > 0);
-}
-
-int flux_buffer_drop_line (flux_buffer_t *fb)
-{
-    int ret;
-
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if ((ret = cbuf_drop_line (fb->cbuf, fb->buflen, 1)) < 0)
-        return -1;
-
-    check_write_cb (fb);
-
-    return ret;
-}
-
-const void *flux_buffer_peek_line (flux_buffer_t *fb, int *lenp)
-{
-    int ret;
-
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    if (return_buffer_check (fb) < 0)
-        return NULL;
-
-    if ((ret = cbuf_peek_line (fb->cbuf, fb->buf, fb->buflen, 1)) < 0)
-        return NULL;
-
-    if (lenp)
-        (*lenp) = ret;
-
-    return fb->buf;
-}
-
-const void *flux_buffer_peek_trimmed_line (flux_buffer_t *fb, int *lenp)
-{
-    int tmp_lenp = 0;
-
-    if (!flux_buffer_peek_line (fb, &tmp_lenp))
-        return NULL;
-
-    if (tmp_lenp) {
-        if (fb->buf[tmp_lenp - 1] == '\n') {
-            fb->buf[tmp_lenp - 1] = '\0';
-            tmp_lenp--;
-        }
-    }
-    if (lenp)
-        (*lenp) = tmp_lenp;
-
-    return fb->buf;
-}
-
-const void *flux_buffer_read_line (flux_buffer_t *fb, int *lenp)
-{
-    int ret;
-
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    if (return_buffer_check (fb) < 0)
-        return NULL;
-
-    if ((ret = cbuf_read_line (fb->cbuf, fb->buf, fb->buflen, 1)) < 0)
-        return NULL;
-
-    if (lenp)
-        (*lenp) = ret;
-
-    check_write_cb (fb);
-
-    return fb->buf;
-}
-
-const void *flux_buffer_read_trimmed_line (flux_buffer_t *fb, int *lenp)
-{
-    int tmp_lenp = 0;
-
-    if (!flux_buffer_read_line (fb, &tmp_lenp))
-        return NULL;
-
-    if (tmp_lenp) {
-        if (fb->buf[tmp_lenp - 1] == '\n') {
-            fb->buf[tmp_lenp - 1] = '\0';
-            tmp_lenp--;
-        }
-    }
-    if (lenp)
-        (*lenp) = tmp_lenp;
-
-    return fb->buf;
-}
-
-int flux_buffer_write_line (flux_buffer_t *fb, const char *data)
-{
-    int ret;
-
-    if (!fb
-        || fb->magic != FLUX_BUFFER_MAGIC
-        || !data) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if (fb->readonly) {
-        errno = EROFS;
-        return -1;
-    }
-
-    if ((ret = cbuf_write_line (fb->cbuf, (char *)data, NULL)) < 0)
-        return -1;
-
-    check_read_cb (fb);
-
-    return ret;
-}
-
-int flux_buffer_peek_to_fd (flux_buffer_t *fb, int fd, int len)
-{
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    return cbuf_peek_to_fd (fb->cbuf, fd, len);
-}
-
-int flux_buffer_read_to_fd (flux_buffer_t *fb, int fd, int len)
-{
-    int ret;
-
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if ((ret = cbuf_read_to_fd (fb->cbuf, fd, len)) < 0)
-        return -1;
-
-    check_write_cb (fb);
-
-    return ret;
-}
-
-int flux_buffer_write_from_fd (flux_buffer_t *fb, int fd, int len)
-{
-    int ret;
-
-    if (!fb || fb->magic != FLUX_BUFFER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if (fb->readonly) {
-        errno = EROFS;
-        return -1;
-    }
-
-    if ((ret = cbuf_write_from_fd (fb->cbuf, fd, len, NULL)) < 0)
-        return -1;
-
-    check_read_cb (fb);
-
-    return ret;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/buffer.h b/src/common/libflux/buffer.h
deleted file mode 100644
index 02d5c07e15d8..000000000000
--- a/src/common/libflux/buffer.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef FLUX_BUFFER_H
-#define FLUX_BUFFER_H
-
-#include <stdbool.h>
-
-typedef struct flux_buffer flux_buffer_t;
-
-/* Create buffer.
- */
-flux_buffer_t *flux_buffer_create (int size);
-
-void flux_buffer_destroy (void *fb);
-
-/* Returns the buffer size, set when flux_buffer_create () was called */
-int flux_buffer_size (flux_buffer_t *fb);
-
-/* Returns the number of bytes current stored in flux_buffer */
-int flux_buffer_bytes (flux_buffer_t *fb);
-
-/* Returns the number of bytes of space available in flux_buffer */
-int flux_buffer_space (flux_buffer_t *fb);
-
-/* Manage "readonly" status
- * - flux_buffer_readonly() makes it so writes are no longer allowed
- *   to the buffer.  Reads are allowed until the buffer is empty.
- *   Changing a buffer to "readonly" can only be called once and
- *   cannot be disabled.  This is a convenience status can be used to
- *   indicate to users that the buffer is no longer usable.
- * - flux_buffer_is_readonly() returns true if a buffer is readonly,
- *    and false if not.
- */
-int flux_buffer_readonly (flux_buffer_t *fb);
-bool  flux_buffer_is_readonly (flux_buffer_t *fb);
-
-/* Drop up to [len] bytes of data in the buffer. Set [len] to -1
- * to drop all data.  Returns number of bytes dropped on success.
- */
-int flux_buffer_drop (flux_buffer_t *fb, int len);
-
-/* Read up to [len] bytes of data in the buffer without consuming it.
- * Pointer to buffer is returned to user and optionally length read
- * can be returned to user in [lenp].  The buffer will always be NUL
- * terminated, so the user may treat returned ptr as a string.  User
- * shall not free returned pointer.  If no data is available, returns
- * pointer and length of 0.  Set [len] to -1 to read all data.
- */
-const void *flux_buffer_peek (flux_buffer_t *fb, int len, int *lenp);
-
-/* Read up to [len] bytes of data in the buffer and mark data as
- * consumed.  Pointer to buffer is returned to user and optionally
- * length read can be returned to user in [lenp].  The buffer will
- * always be NUL terminated, so the user may treat returned ptr as a
- * string.  User shall not free returned pointer.  If no data is
- * available, returns pointer and length of 0.  Set [len] to -1 to
- * read all data.
- */
-const void *flux_buffer_read (flux_buffer_t *fb, int len, int *lenp);
-
-/* Write [len] bytes of data into the buffer.  Returns number of bytes
- * written on success.
- */
-int flux_buffer_write (flux_buffer_t *fb, const void *data, int len);
-
-/* Determines lines available for peeking/reading.  Returns -1
- * on error, >= 0 for lines available */
-int flux_buffer_lines (flux_buffer_t *fb);
-
-/* Return true if buffer has at least one unread line */
-bool flux_buffer_has_line (flux_buffer_t *fb);
-
-/* Drop a line in the buffer.  Returns number of bytes dropped on
- * success. */
-int flux_buffer_drop_line (flux_buffer_t *fb);
-
-/* Read a line in the buffer without consuming it.  Return buffer will
- * include newline.  Optionally return length of data returned in
- * [lenp].  If no line is available, returns pointer and length of
- * 0. Return NULL on error.
- */
-const void *flux_buffer_peek_line (flux_buffer_t *fb, int *lenp);
-
-/* Identical to flux_buffer_peek_line(), but does not return trailing
- * newline */
-const void *flux_buffer_peek_trimmed_line (flux_buffer_t *fb, int *lenp);
-
-/* Read a line in the buffer and mark data as consumed.  Return buffer
- * will include newline.  Optionally return length of data returned in
- * [lenp].  If no line is available, returns pointer and length of 0.
- * Return NULL on error.
- */
-const void *flux_buffer_read_line (flux_buffer_t *fb, int *lenp);
-
-/* Identical to flux_buffer_read_line(), but does not return trailing
- * newline */
-const void *flux_buffer_read_trimmed_line (flux_buffer_t *fb, int *lenp);
-
-/* Write NUL terminated string data into the buffer and appends a
- * newline.  Returns number of bytes written on success.
- */
-int flux_buffer_write_line (flux_buffer_t *fb, const char *data);
-
-/* Read up to [len] bytes from buffer to file descriptor [fd] without
- * consuming data.  Set [len] to -1 to read all data.  Returns number
- * of bytes read or -1 on error. */
-int flux_buffer_peek_to_fd (flux_buffer_t *fb, int fd, int len);
-
-/* Read up to [len] bytes from buffer to file descriptor [fd] and mark
- * data as consumed.  Set [len] to -1 to read all data.  Returns
- * number of bytes read or -1 on error. */
-int flux_buffer_read_to_fd (flux_buffer_t *fb, int fd, int len);
-
-/* Write up to [len] bytes to buffer from file descriptor [fd].  Set
- * [len] to -1 to read an appropriate chunk size.  Returns number of
- * bytes written on success.
- */
-int flux_buffer_write_from_fd (flux_buffer_t *fb, int fd, int len);
-
-/* FUTURE: append, prepend, printf, add_flux_buffer, etc. */
-
-#endif /* !_FLUX_BUFFER_H */
diff --git a/src/common/libflux/buffer_private.h b/src/common/libflux/buffer_private.h
deleted file mode 100644
index f86328c2d8fa..000000000000
--- a/src/common/libflux/buffer_private.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef FLUX_BUFFER_PRIVATE_H
-#define FLUX_BUFFER_PRIVATE_H
-
-#include "buffer.h"
-
-typedef void (*flux_buffer_cb) (flux_buffer_t *fb, void *arg);
-
-/* Call [cb] when the number of bytes stored is greater than [low]
- * bytes.
- *
- * The callback is typically called after an flux_buffer_write() or other
- * write action has occurred on the buffer.  Often, users set [low] to
- * 0, so that the callback is called anytime data has been added to
- * the buffer.
- *
- * At most one callback handler can be set per flux_buffer.  If another
- * callback type has already been set, EEXIST is the errno returned.
- * Setting [cb] to NULL disables the callback.
- */
-int flux_buffer_set_low_read_cb (flux_buffer_t *fb,
-                                 flux_buffer_cb cb,
-                                 int low,
-                                 void *arg);
-
-/* Call [cb] when a line has been stored.  This callback is typically
- * called after an flux_buffer_write() or other write action has occurred on
- * the buffer.
- *
- * At most one callback handler can be set per flux_buffer.  If another
- * callback type has already been set, EEXIST is the errno returned.
- * Setting [cb] to NULL disables the callback.
- */
-int flux_buffer_set_read_line_cb (flux_buffer_t *fb,
-                                  flux_buffer_cb cb,
-                                  void *arg);
-
-/* Call [cb] when the number of bytes stored falls less than
- * [high] bytes.  Setting [cb] to NULL disables the callback.
- *
- * This callback is generally called after a flux_buffer_drop(), flux_buffer_read()
- * or other consumption action has occurred on the buffer.  Often,
- * users set [high] to [maxsize], so that the callback is called when
- * the buffer has space for writing.
- *
- * At most one callback handler can be set per flux_buffer.  If another
- * callback type has already been set, EEXIST is the errno returned.
- * Setting [cb] to NULL disables the callback.
- */
-int flux_buffer_set_high_write_cb (flux_buffer_t *fb,
-                                   flux_buffer_cb cb,
-                                   int high,
-                                   void *arg);
-
-#endif /* !_FLUX_BUFFER_PRIVATE_H */
diff --git a/src/common/libflux/composite_future.c b/src/common/libflux/composite_future.c
index b869f41396df..bb2e0044bce2 100644
--- a/src/common/libflux/composite_future.c
+++ b/src/common/libflux/composite_future.c
@@ -11,9 +11,9 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
+#include <flux/core.h>
 
-#include "future.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 /*  Type-specific data for a composite future:
  */
@@ -164,8 +164,12 @@ static flux_future_t *future_create_composite (int wait_any)
 {
     struct composite_future *cf = composite_future_create ();
     flux_future_t *f = flux_future_create (composite_future_init, (void *) cf);
-    if (!f || !cf || flux_future_aux_set (f, "flux::composite", cf,
-                         (flux_free_f) composite_future_destroy) < 0) {
+    if (!f
+        || !cf
+        || flux_future_aux_set (f,
+                                "flux::composite",
+                                cf,
+                                (flux_free_f) composite_future_destroy) < 0) {
         composite_future_destroy (cf);
         flux_future_destroy (f);
         return (NULL);
@@ -203,8 +207,10 @@ int flux_future_push (flux_future_t *f, const char *name, flux_future_t *child)
             return -1;
         name = anon;
     }
-    if (zhash_insert (cf->children, name, child) < 0)
+    if (zhash_insert (cf->children, name, child) < 0) {
+        errno = EEXIST;
         goto done;
+    }
     zhash_freefn (cf->children, name, (flux_free_f) flux_future_destroy);
     if (flux_future_aux_set (child, "flux::parent", f, NULL) < 0) {
         zhash_delete (cf->children, name);
@@ -262,11 +268,11 @@ const char *flux_future_next_child (flux_future_t *f)
  *   If the user calls both and_then() and or_then(), the same cf->next
  *   future is returned, since only one of these calls will be used.
  *
- *  The underlying then() callback for `f` is subsequently set to use 
+ *  The underlying then() callback for `f` is subsequently set to use
  *   chained_continuation() below, which will call `and_then()` on successful
  *   fulfillment of f (aka `prev`) or or_then() on failure. These continuations
  *   are passed `f, arg` as if a normal continuation was used with
- *   flux_future_then(3). These callbacks may use one of 
+ *   flux_future_then(3). These callbacks may use one of
  *   flux_future_continue(3) or flux_future_continue_error(3) to schedule
  *   fulfillment of the internal `cf->next` future based on a new
  *   intermediate future created during the continuation (e.g. when a
@@ -297,6 +303,7 @@ struct continuation_info {
 };
 
 struct chained_future {
+    int refcount;
     bool continued;
     flux_future_t *next;
     flux_future_t *prev;
@@ -353,12 +360,19 @@ static void chained_continuation (flux_future_t *prev, void *arg)
         ran_callback = true;
     }
 
+    /*  Check if prev future was reset. If so return and allow this
+     *   continuation to run again.
+     */
+    if (!flux_future_is_ready (prev))
+        return;
+
     /*  If future prev was not continued with flux_future_continue()
      *   or flux_future_continue_error(), then fallback to continue
      *   cf->next using prev directly.
      */
     if (!cf->continued && flux_future_fulfill_with (cf->next, prev) < 0) {
-        flux_future_fatal_error (cf->next, errno,
+        flux_future_fatal_error (cf->next,
+                                 errno,
                                  "chained_continuation: fulfill_with failed");
     }
 
@@ -376,6 +390,15 @@ static void chained_continuation (flux_future_t *prev, void *arg)
         flux_future_destroy (prev);
 }
 
+static void chained_future_decref (struct chained_future *cf)
+{
+    if (--cf->refcount == 0) {
+        int saved_errno = errno;
+        free (cf);
+        errno = saved_errno;
+    }
+}
+
 /*  Initialization for a chained future. Get current reactor for this
  *   context and install it in "previous" future, _then_ set the "then"
  *   callback for that future to `chained_continuation()` which will
@@ -384,7 +407,9 @@ static void chained_continuation (flux_future_t *prev, void *arg)
 static void chained_future_init (flux_future_t *f, void *arg)
 {
     struct chained_future *cf = arg;
-    if (cf == NULL || cf->prev == NULL || cf->next == NULL
+    if (cf == NULL
+        || cf->prev == NULL
+        || cf->next == NULL
         || !(flux_future_get_reactor (f))) {
         errno = EINVAL;
         goto error;
@@ -416,6 +441,18 @@ static void chained_future_init (flux_future_t *f, void *arg)
     fulfill_next (f, cf->next);
 }
 
+static void cf_next_destroy (struct chained_future *cf)
+{
+    /*  cf->next is being destroyed. If cf->prev is still active, destroy
+     *  it now since no longer makes sense to trigger the cf->prev callback.
+     */
+    flux_future_destroy (cf->prev);
+    cf->prev = NULL;
+
+    /*  Release reference on cf taken by cf->next
+     */
+    chained_future_decref (cf);
+}
 
 /*  Allocate a chained future structure */
 static struct chained_future *chained_future_alloc (void)
@@ -427,12 +464,40 @@ static struct chained_future *chained_future_alloc (void)
         free (cf);
         return (NULL);
     }
+
+    /*  If cf->next is destroyed then later cf->prev is fulfilled, this
+     *  can cause use-after-free of cf->next in chained_continuation() and/or
+     *  flux_future_continue(3). Additionally, a reasonable expectation
+     *  is that destruction of cf->next will stop any previous future in
+     *  the chain. Therefore, arrange to have this object notified when
+     *  cf->next is destroyed so that cf->prev can also be destroyed
+     *  (if it has not yet been destroyed already. That case is handled
+     *   by nullifying cf->prev when prev is destroyed)
+     */
+    if (flux_future_aux_set (cf->next,
+                             NULL,
+                             cf,
+                             (flux_free_f) cf_next_destroy) < 0) {
+        flux_future_destroy (cf->next);
+        free (cf);
+        return NULL;
+    }
+
+    /*  Take one reference on cf for cf->next (released in cf_next_destroy())
+     */
+    cf->refcount = 1;
     return (cf);
 }
 
-static void chained_future_destroy (struct chained_future *cf)
+static void nullify_prev (struct chained_future *cf)
 {
-    free (cf);
+    /*  cf->prev has been destroyed. Ensure we don't reference it again
+     *  by nullifying the reference in cf.
+     */
+    cf->prev = NULL;
+
+    /* Remove reference added by this callback */
+    chained_future_decref (cf);
 }
 
 /*  Create a chained future on `f` by embedding a chained future
@@ -444,16 +509,48 @@ static void chained_future_destroy (struct chained_future *cf)
  */
 static struct chained_future *chained_future_create (flux_future_t *f)
 {
-    struct chained_future *cf = flux_future_aux_get (f, "flux::chained");
-    if (cf == NULL && (cf = chained_future_alloc ())) {
-        if (flux_future_aux_set (f, "flux::chained",
-                                 (void *) cf,
-                                 (flux_free_f) chained_future_destroy) < 0) {
-            chained_future_destroy (cf);
-            return (NULL);
-        }
+    struct chained_future *cf;
+
+    /*  If future `f` is already chained, then just return the existing
+     *  stored chained_future object:
+     */
+    if ((cf = flux_future_aux_get (f, "flux::chained")))
+        return cf;
+
+    /*  Otherwise, create one and store it:
+     */
+    if (!(cf = chained_future_alloc ())
+        || flux_future_aux_set (f,
+                                "flux::chained",
+                               (void *) cf,
+                               (flux_free_f) chained_future_decref) < 0) {
+        chained_future_decref (cf);
+        return NULL;
     }
+
+    /* Increment refcount for cf->prev
+     */
+    cf->refcount++;
     cf->prev = f;
+
+    /* Nullify cf->prev on destruction of f to notify cf_next_destroy() that
+     *  prev was already destroyed. (See note in chained_future_alloc()).
+     */
+    if (flux_future_aux_set (f,
+                             NULL,
+                             cf,
+                             (flux_free_f) nullify_prev) < 0) {
+        chained_future_decref (cf);
+        return NULL;
+    }
+
+    /* Increment refcount again for nullify_prev aux item. This is necessary
+     *  since the order of aux_item destructors can't be guaranteed.
+     *  This increment is done separately to avoid leaking cf in case of
+     *  flux_future_aux_set() failure above.
+     */
+    cf->refcount++;
+
     /*
      * Ensure the empty "next" future we have just created inherits
      *  the same reactor (if any) and handle (if any) from the previous
@@ -512,7 +609,8 @@ int flux_future_continue (flux_future_t *prev, flux_future_t *f)
 
 /* "Continue" the chained "next" future embedded in `prev` with an error
  */
-void flux_future_continue_error (flux_future_t *prev, int errnum,
+void flux_future_continue_error (flux_future_t *prev,
+                                 int errnum,
                                  const char *errstr)
 {
     struct chained_future *cf = chained_future_get (prev);
@@ -522,8 +620,23 @@ void flux_future_continue_error (flux_future_t *prev, int errnum,
     }
 }
 
+int flux_future_fulfill_next (flux_future_t *f,
+                              void *result,
+                              flux_free_f free_fn)
+{
+    struct chained_future *cf = chained_future_get (f);
+    if (cf == NULL || !cf->next) {
+        errno = EINVAL;
+        return -1;
+    }
+    cf->continued = true;
+    flux_future_fulfill (cf->next, result, free_fn);
+    return 0;
+}
+
 flux_future_t *flux_future_and_then (flux_future_t *prev,
-                                     flux_continuation_f next_cb, void *arg)
+                                     flux_continuation_f next_cb,
+                                     void *arg)
 {
     struct chained_future *cf = chained_future_create (prev);
     if (!cf)
@@ -534,7 +647,8 @@ flux_future_t *flux_future_and_then (flux_future_t *prev,
 }
 
 flux_future_t *flux_future_or_then (flux_future_t *prev,
-                                    flux_continuation_f or_cb, void *arg)
+                                    flux_continuation_f or_cb,
+                                    void *arg)
 {
     struct chained_future *cf = chained_future_create (prev);
     if (!cf)
diff --git a/src/common/libflux/conf.c b/src/common/libflux/conf.c
index a254819088fb..106cb0523652 100644
--- a/src/common/libflux/conf.c
+++ b/src/common/libflux/conf.c
@@ -11,6 +11,7 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <unistd.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
@@ -18,13 +19,18 @@
 #include <stdbool.h>
 #include <glob.h>
 #include <jansson.h>
+#include <flux/core.h>
+#if !HAVE_JSON_OBJECT_UPDATE_RECURSIVE
+#include "src/common/libmissing/json_object_update_recursive.h"
+#endif
 
 #include "src/common/libutil/intree.h"
 #include "src/common/libutil/errno_safe.h"
 #include "src/common/libtomlc99/toml.h"
 #include "src/common/libutil/tomltk.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
 
-#include "conf.h"
 #include "conf_private.h"
 
 struct builtin {
@@ -35,36 +41,116 @@ struct builtin {
 
 struct flux_conf {
     json_t *obj;
+    int refcount;
 };
 
 static const char *conf_auxkey = "flux::conf_object";
 
 static struct builtin builtin_tab[] = {
-    { "lua_cpath_add",  INSTALLED_LUA_CPATH_ADD,    INTREE_LUA_CPATH_ADD },
-    { "lua_path_add",   INSTALLED_LUA_PATH_ADD,     INTREE_LUA_PATH_ADD },
-    { "python_path",    INSTALLED_PYTHON_PATH,      INTREE_PYTHON_PATH },
-    { "man_path",       INSTALLED_MAN_PATH,         INTREE_MAN_PATH },
-    { "exec_path",      INSTALLED_EXEC_PATH,        INTREE_EXEC_PATH },
-    { "connector_path", INSTALLED_CONNECTOR_PATH,   INTREE_CONNECTOR_PATH },
-    { "module_path",    INSTALLED_MODULE_PATH,      INTREE_MODULE_PATH },
-    { "rc1_path",       INSTALLED_RC1_PATH,         INTREE_RC1_PATH },
-    { "rc3_path",       INSTALLED_RC3_PATH,         INTREE_RC3_PATH },
-    { "cf_path",        INSTALLED_CF_PATH,          INTREE_CF_PATH },
-    { "cmdhelp_pattern",INSTALLED_CMDHELP_PATTERN,  INTREE_CMDHELP_PATTERN },
-    { "pmi_library_path",
-                        INSTALLED_PMI_LIBRARY_PATH, INTREE_PMI_LIBRARY_PATH },
-    { "shell_path",     INSTALLED_SHELL_PATH,       INTREE_SHELL_PATH },
-    { "shell_pluginpath",
-                        INSTALLED_SHELL_PLUGINPATH, INTREE_SHELL_PLUGINPATH },
-    { "shell_initrc",   INSTALLED_SHELL_INITRC,     INTREE_SHELL_INITRC },
-    { "keydir",         NULL,                       INTREE_KEYDIR },
-    { "no_docs_path",   INSTALLED_NO_DOCS_PATH,     INTREE_NO_DOCS_PATH },
-    { "rundir",         INSTALLED_RUNDIR,           NULL },
-    { "bindir",         INSTALLED_BINDIR,           INTREE_BINDIR },
-    { "jobspec_validate_path", INSTALLED_JOBSPEC_VALIDATE_PATH,
-                                            INTREE_JOBSPEC_VALIDATE_PATH },
-    { "jobspec_validator_args", INSTALLED_JOBSPEC_VALIDATOR_ARGS,
-                                            INTREE_JOBSPEC_VALIDATOR_ARGS },
+    {
+        .key = "confdir",
+        .val_installed = FLUXCONFDIR,
+        .val_intree = ABS_TOP_SRCDIR "/etc",
+    },
+    {
+        .key = "lua_cpath_add",
+        .val_installed = LUAEXECDIR "/?.so",
+        .val_intree = ABS_TOP_BUILDDIR "/src/bindings/lua/?.so",
+    },
+    {
+        .key = "lua_path_add",
+        .val_installed = LUADIR "/?.lua",
+        .val_intree = ABS_TOP_BUILDDIR "/t/?.lua;" \
+                      ABS_TOP_SRCDIR "/src/bindings/lua/?.lua",
+    },
+    {
+        .key = "python_path",
+        .val_installed = FLUXPYLINKDIR,
+        .val_intree = ABS_TOP_BUILDDIR "/src/bindings/python:" \
+                      ABS_TOP_SRCDIR "/src/bindings/python",
+    },
+    {
+        .key = "python_wrapper",
+        .val_installed = FLUXCMDDIR "/py-runner.py",
+        .val_intree = ABS_TOP_SRCDIR "/src/cmd/py-runner.py",
+    },
+    {
+        .key = "man_path",
+        .val_installed = X_MANDIR,
+        .val_intree =  ABS_TOP_BUILDDIR "/doc",
+    },
+    {
+        .key = "exec_path",
+        .val_installed = FLUXCMDDIR,
+        .val_intree = ABS_TOP_BUILDDIR "/src/cmd:" \
+                      ABS_TOP_SRCDIR "/src/cmd:" \
+                      ABS_TOP_BUILDDIR "/src/broker",
+    },
+    {
+        .key = "connector_path",
+        .val_installed = FLUXCONNECTORDIR,
+        .val_intree = ABS_TOP_BUILDDIR "/src/connectors/.libs",
+    },
+    {
+        .key = "module_path",
+        .val_installed = FLUXMODDIR,
+        .val_intree = ABS_TOP_BUILDDIR "/src/modules/.libs",
+    },
+    {
+        .key = "rc1_path",
+        .val_installed = FLUXCONFDIR "/rc1",
+        .val_intree = ABS_TOP_SRCDIR "/etc/rc1",
+    },
+    {
+        .key = "rc3_path",
+        .val_installed = FLUXCONFDIR "/rc3",
+        .val_intree = ABS_TOP_SRCDIR "/etc/rc3",
+    },
+    {
+        .key = "cmdhelp_pattern",
+        .val_installed = X_DATADIR "/flux/help.d/*.json",
+        .val_intree = ABS_TOP_BUILDDIR "/etc/flux/help.d/*.json",
+    },
+    {
+        .key = "pmi_library_path",
+        .val_installed = FLUXLIBDIR "/libpmi.so",
+        .val_intree = ABS_TOP_BUILDDIR "/src/common/flux/.libs/libpmi.so",
+    },
+    {
+        .key = "shell_path",
+        .val_installed = FLUXLIBEXECDIR "/flux-shell",
+        .val_intree = ABS_TOP_BUILDDIR "/src/shell/flux-shell",
+    },
+    {
+        .key = "shell_pluginpath",
+        .val_installed = FLUXLIBDIR "/shell/plugins",
+        .val_intree = ABS_TOP_BUILDDIR "/src/shell/plugins",
+    },
+    {
+        .key = "shell_initrc",
+        .val_installed = FLUXCONFDIR "/shell/initrc.lua",
+        .val_intree = ABS_TOP_SRCDIR "/src/shell/initrc.lua",
+    },
+    {
+        .key = "jobtap_pluginpath",
+        .val_installed = JOBTAP_PLUGINDIR,
+        .val_intree = ABS_TOP_BUILDDIR "/src/modules/job-manager/plugins/.libs",
+    },
+    {
+        .key = "upmi_pluginpath",
+        .val_installed = FLUXLIBDIR "/upmi/plugins",
+        .val_intree = ABS_TOP_BUILDDIR "/src/common/libpmi/plugins/.libs",
+    },
+    {
+        .key = "no_docs_path",
+        .val_installed = X_DATADIR "/flux/.nodocs",
+        .val_intree = ABS_TOP_BUILDDIR "/etc/flux/.nodocs",
+    },
+    {
+        .key = "rundir",
+        .val_installed = X_RUNSTATEDIR "/flux",
+        .val_intree = NULL
+    },
     { NULL, NULL, NULL },
 };
 
@@ -90,7 +176,7 @@ const char *flux_conf_builtin_get (const char *name,
             break;
     }
     for (entry = &builtin_tab[0]; entry->key != NULL; entry++) {
-        if (name && !strcmp (entry->key, name)) {
+        if (name && streq (entry->key, name)) {
             val = intree ? entry->val_intree : entry->val_installed;
             break;
         }
@@ -103,80 +189,101 @@ const char *flux_conf_builtin_get (const char *name,
     return NULL;
 }
 
-static void __attribute__ ((format (printf, 4, 5)))
-errprintf (flux_conf_error_t *error,
-           const char *filename,
-           int lineno,
-           const char *fmt,
-           ...)
+void flux_conf_decref (const flux_conf_t *conf_const)
 {
-    va_list ap;
-    int saved_errno = errno;
-
-    if (error) {
-        memset (error, 0, sizeof (*error));
-        va_start (ap, fmt);
-        (void)vsnprintf (error->errbuf, sizeof (error->errbuf), fmt, ap);
-        va_end (ap);
-        if (filename)
-            strncpy (error->filename, filename, sizeof (error->filename) - 1);
-        error->lineno = lineno;
+    flux_conf_t *conf = (flux_conf_t *)conf_const;
+    if (conf && --conf->refcount == 0) {
+        ERRNO_SAFE_WRAP (json_decref, conf->obj);
+        ERRNO_SAFE_WRAP (free, conf);
     }
-    errno = saved_errno;
 }
 
-void conf_destroy (flux_conf_t *conf)
+const flux_conf_t *flux_conf_incref (const flux_conf_t *conf_const)
 {
-    if (conf) {
-        ERRNO_SAFE_WRAP (json_decref, conf->obj);
-        ERRNO_SAFE_WRAP (free, conf);
-    }
+    flux_conf_t *conf = (flux_conf_t *)conf_const;
+
+    if (conf)
+        conf->refcount++;
+    return conf;
 }
 
-static flux_conf_t *conf_create (void)
+flux_conf_t *flux_conf_create (void)
 {
     flux_conf_t *conf;
 
     if (!(conf = calloc (1, sizeof (*conf))))
         return NULL;
+    conf->refcount = 1;
     if (!(conf->obj = json_object ())) {
-        conf_destroy (conf);
+        flux_conf_decref (conf);
         errno = ENOMEM;
         return NULL;
     }
     return conf;
 }
 
-static int conf_update (flux_conf_t *conf,
-                        const char *filename,
-                        flux_conf_error_t *error)
+flux_conf_t *flux_conf_copy (const flux_conf_t *conf)
+{
+    flux_conf_t *cpy;
+
+    if (!(cpy = flux_conf_create ()))
+        return NULL;
+    json_decref (cpy->obj);
+    if (!(cpy->obj = json_deep_copy (conf->obj))) {
+        flux_conf_decref (cpy);
+        return NULL;
+    }
+    return cpy;
+}
+
+static int conf_update_obj (flux_conf_t *conf,
+                            const char *filename,
+                            json_t *obj,
+                            flux_error_t *error)
+{
+    if (json_object_update_recursive (conf->obj, obj) < 0) {
+        errprintf (error,
+                   "%s: updating JSON object: out of memory",
+                   filename);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+static int conf_update_toml (flux_conf_t *conf,
+                             const char *filename,
+                             flux_error_t *error)
 {
     struct tomltk_error toml_error;
     toml_table_t *tab;
     json_t *obj = NULL;
 
     if (!(tab = tomltk_parse_file (filename, &toml_error))) {
-        errprintf (error,
-                   toml_error.filename,
-                   toml_error.lineno,
-                   "%s",
-                   toml_error.errbuf);
+        if (toml_error.lineno == -1) {
+            errprintf (error,
+                       "%s: %s",
+                       toml_error.filename,
+                       toml_error.errbuf);
+        }
+        else {
+            errprintf (error,
+                       "%s:%d: %s",
+                       toml_error.filename,
+                       toml_error.lineno,
+                       toml_error.errbuf);
+        }
         goto error;
     }
     if (!(obj = tomltk_table_to_json (tab))) {
-        errprintf (error, filename, -1, "converting TOML to JSON: %s",
-                   strerror (errno));
-        goto error;
-    }
-    if (json_object_update (conf->obj, obj) < 0) {
-        errno = ENOMEM;
         errprintf (error,
+                   "%s: converting TOML to JSON: %s",
                    filename,
-                   -1,
-                   "updating JSON object: %s",
                    strerror (errno));
         goto error;
     }
+    if (conf_update_obj (conf, filename, obj, error) < 0)
+        goto error;
     json_decref (obj);
     toml_free (tab);
     return 0;
@@ -186,6 +293,44 @@ static int conf_update (flux_conf_t *conf,
     return -1;
 }
 
+static int conf_update_json (flux_conf_t *conf,
+                             const char *filename,
+                             flux_error_t *error)
+{
+    json_error_t err;
+    json_t *obj = NULL;
+
+    if (!(obj = json_load_file (filename, 0, &err))) {
+        errprintf (error, "%s:%d: %s", filename, err.line, err.text);
+        goto error;
+    }
+    if (conf_update_obj (conf, filename, obj, error) < 0)
+        goto error;
+    json_decref (obj);
+    return 0;
+error:
+    ERRNO_SAFE_WRAP (json_decref, obj);
+    return -1;
+}
+
+static const char *file_extension (const char *path)
+{
+    const char *p;
+    if (path && (p = strrchr (path,  '.')))
+        return p + 1;
+    return "";
+}
+
+static int conf_update (flux_conf_t *conf,
+                        const char *filename,
+                        flux_error_t *error)
+{
+    const char *ext = file_extension (filename);
+    if (streq (ext, "json"))
+        return conf_update_json (conf, filename, error);
+    return conf_update_toml (conf, filename, error);
+}
+
 struct globerr {
     int rc;
     const char *msg;
@@ -201,96 +346,108 @@ struct globerr globerr_tab[] = {
 
 /* Set errno and optionally 'error' based on glob error return code.
  */
-void conf_globerr (flux_conf_error_t *error, const char *pattern, int rc)
+void conf_globerr (flux_error_t *error, const char *pattern, int rc)
 {
     struct globerr *entry;
     for (entry = &globerr_tab[0]; entry->rc != 0; entry++) {
         if (rc == entry->rc)
             break;
     }
-    errprintf (error, pattern, -1, "%s", entry->msg);
+    errprintf (error, "%s: %s", pattern, entry->msg);
     errno = entry->errnum;
 }
 
-flux_conf_t *conf_parse (const char *pattern, flux_conf_error_t *error)
+static flux_conf_t *conf_parse_dir (const char *path, flux_error_t *error)
 {
     flux_conf_t *conf;
     glob_t gl;
     int rc;
     size_t i;
+    char pattern[4096];
 
-    if (!pattern) {
-        errno = EINVAL;
-        errprintf (error, pattern, -1, "%s", strerror (errno));
+    if (access (path, R_OK | X_OK) < 0) {
+        errprintf (error, "%s: %s", path, strerror (errno));
+        return NULL;
+    }
+    rc = snprintf (pattern, sizeof (pattern), "%s/*.toml", path);
+    if (rc >= sizeof (pattern)) {
+        errno = EOVERFLOW;
+        errprintf (error, "%s: %s", path, strerror (errno));
         return NULL;
     }
-    if (!(conf = conf_create ())) {
-        errprintf (error, pattern, -1, "%s", strerror (errno));
+    if (!(conf = flux_conf_create ())) {
+        errprintf (error, "%s: %s", path, strerror (errno));
         return NULL;
     }
-    if ((rc = glob (pattern, GLOB_ERR, NULL, &gl)) != 0) {
+    rc = glob (pattern, GLOB_ERR, NULL, &gl);
+    if (rc == 0) {
+        for (i = 0; i < gl.gl_pathc; i++) {
+            if (conf_update_toml (conf, gl.gl_pathv[i], error) < 0)
+                goto error_glob;
+        }
+        globfree (&gl);
+    }
+    else if (rc != GLOB_NOMATCH) {
         conf_globerr (error, pattern, rc);
         goto error;
     }
-    for (i = 0; i < gl.gl_pathc; i++) {
-        if (conf_update (conf, gl.gl_pathv[i], error) < 0)
-            goto error_glob;
-    }
-    globfree (&gl);
     return conf;
 
 error_glob:
     ERRNO_SAFE_WRAP (globfree, &gl);
 error:
-    conf_destroy (conf);
+    flux_conf_decref (conf);
     return NULL;
 }
 
-int conf_get_default_pattern (char *buf, int bufsz)
+static flux_conf_t *conf_parse_file (const char *path, flux_error_t *error)
 {
-    const char *cf_path = getenv ("FLUX_CONF_DIR");
+    flux_conf_t *conf;
 
-    if (cf_path && !strcmp (cf_path, "installed"))
-        cf_path = flux_conf_builtin_get ("cf_path", FLUX_CONF_INSTALLED);
-    else if (!cf_path)
-        cf_path = flux_conf_builtin_get ("cf_path", FLUX_CONF_AUTO);
-    if (snprintf (buf, bufsz, "%s/*.toml", cf_path) >= bufsz) {
-        errno = EOVERFLOW;
-        return -1;
+    if (!(conf = flux_conf_create ())) {
+        errprintf (error, "%s: %s", path, strerror (errno));
+        return NULL;
     }
-    return 0;
+    if (conf_update (conf, path, error) < 0) {
+        flux_conf_decref (conf);
+        return NULL;
+    }
+    return conf;
 }
 
-int handle_set_conf (flux_t *h, flux_conf_t *conf)
+flux_conf_t *flux_conf_parse (const char *path, flux_error_t *error)
+{
+    struct stat st;
+
+    if (!path) {
+        errno = EINVAL;
+        errprintf (error, "%s", strerror (errno));
+        return NULL;
+    }
+    if (stat (path, &st) < 0) {
+        errprintf (error, "stat: %s: %s", path, strerror (errno));
+        return NULL;
+    }
+    if (S_ISDIR(st.st_mode))
+        return conf_parse_dir (path, error);
+    return conf_parse_file (path, error);
+}
+
+int flux_set_conf (flux_t *h, const flux_conf_t *conf)
 {
     return flux_aux_set (h,
                          conf_auxkey,
-                         conf,
-                         conf ? (flux_free_f)conf_destroy : NULL);
+                         (flux_conf_t *)conf,
+                         conf ? (flux_free_f)flux_conf_decref : NULL);
 }
 
-const flux_conf_t *flux_get_conf (flux_t *h, flux_conf_error_t *error)
+const flux_conf_t *flux_get_conf (flux_t *h)
 {
-    flux_conf_t *conf;
-
-    if (!(conf = flux_aux_get (h, conf_auxkey))) {
-        char pattern[PATH_MAX + 1];
-        if (conf_get_default_pattern (pattern, sizeof (pattern)) < 0) {
-            errprintf (error, NULL, -1, "%s", strerror (errno));
-            return NULL;
-        }
-        if (!(conf = conf_parse (pattern, error)))
-            return NULL;
-        if (handle_set_conf (h, conf) < 0) {
-            conf_destroy (conf);
-            return NULL;
-        }
-    }
-    return conf;
+    return flux_aux_get (h, conf_auxkey);
 }
 
 int flux_conf_vunpack (const flux_conf_t *conf,
-                       flux_conf_error_t *error,
+                       flux_error_t *error,
                        const char *fmt,
                        va_list ap)
 {
@@ -298,14 +455,14 @@ int flux_conf_vunpack (const flux_conf_t *conf,
 
     if (!conf || !conf->obj || !fmt || *fmt == '\0') {
         errno = EINVAL;
-        errprintf (error, NULL, -1, "%s", strerror (errno));
+        errprintf (error, "%s", strerror (errno));
         return -1;
     }
     if (json_vunpack_ex (conf->obj, &j_error, 0, fmt, ap) < 0) {
         /* N.B. Cannot translate JSON error back to TOML file/line,
          * so only the text[] portion of the JSON error is used here.
          */
-        errprintf (error, NULL, -1, "%s", j_error.text);
+        errprintf (error, "%s", j_error.text);
         errno = EINVAL;
         return -1;
     }
@@ -313,7 +470,7 @@ int flux_conf_vunpack (const flux_conf_t *conf,
 }
 
 int flux_conf_unpack (const flux_conf_t *conf,
-                      flux_conf_error_t *error,
+                      flux_error_t *error,
                       const char *fmt, ...)
 {
     va_list ap;
@@ -326,6 +483,32 @@ int flux_conf_unpack (const flux_conf_t *conf,
     return rc;
 }
 
+int flux_conf_reload_decode (const flux_msg_t *msg, const flux_conf_t **confp)
+{
+    const char *auxkey = "flux::conf";
+    json_t *o;
+    flux_conf_t *conf;
+
+    if (!(conf = flux_msg_aux_get (msg, auxkey))) {
+        if (flux_request_unpack (msg, NULL, "o", &o) < 0)
+            return -1;
+        if (!(conf = flux_conf_create ()))
+            return -1;
+        json_decref (conf->obj);
+        conf->obj = json_incref (o);
+        if (flux_msg_aux_set (msg,
+                              auxkey,
+                              (flux_conf_t *)conf,
+                              (flux_free_f)flux_conf_decref) < 0) {
+            flux_conf_decref (conf);
+            return -1;
+        }
+    }
+    if (confp)
+        *confp = conf;
+    return 0;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libflux/conf.h b/src/common/libflux/conf.h
index 2637364bb89b..048857db4d26 100644
--- a/src/common/libflux/conf.h
+++ b/src/common/libflux/conf.h
@@ -12,7 +12,6 @@
 #define _FLUX_CORE_CONF_H
 
 #include <stdarg.h>
-#include "handle.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -36,38 +35,41 @@ const char *flux_conf_builtin_get (const char *name,
 
 typedef struct flux_conf flux_conf_t;
 
-typedef struct {
-    char filename[80];  // if unused, will be set to empty string
-    int lineno;         // if unused, will be set to -1
-    char errbuf[160];
-} flux_conf_error_t;
+/* Create/copy/incref/decref config object
+ */
+flux_conf_t *flux_conf_create (void);
+flux_conf_t *flux_conf_copy (const flux_conf_t *conf);
+const flux_conf_t *flux_conf_incref (const flux_conf_t *conf);
+void flux_conf_decref (const flux_conf_t *conf);
 
-/* Parse Flux's config files (*.toml in cf_path).
- *
- * The builtin value for cf_path (intree|installed selected automatically) is
- * used unless overridden with the FLUX_CONF_DIR environment variable.
- * FLUX_CONF_DIR may be set to a directory path or may be set to the
- * special value "installed" to force use of the installed cf_path.
- *
- * On success, return the object, saving the result for reuse on future calls.
- * On failure, return NULL, set errno, and (if non-NULL) fill 'error' with
- * details about the failure.
- *
- * The config object is destroyed when the handle is destroyed.
+/* Decode config-reload request, setting 'conf' to a config object
+ * owned by 'msg'.
+ */
+int flux_conf_reload_decode (const flux_msg_t *msg, const flux_conf_t **conf);
+
+/* Parse TOML config in 'path' and return a new flux_conf_t on success.
+ * If path is a directory, then parse all files matching *.toml in path.
+ */
+flux_conf_t *flux_conf_parse (const char *path, flux_error_t *error);
+
+/* Get/set config object cached in flux_t handle, with destructor.
+ * Re-setting the object decrefs the old one.
  */
-const flux_conf_t *flux_get_conf (flux_t *h, flux_conf_error_t *error);
+const flux_conf_t *flux_get_conf (flux_t *h);
+int flux_set_conf (flux_t *h, const flux_conf_t *conf);
 
 /* Access config object.
  * If error is non-NULL, it is filled with error details on failure.
  */
 int flux_conf_vunpack (const flux_conf_t *conf,
-                       flux_conf_error_t *error,
+                       flux_error_t *error,
                        const char *fmt,
                        va_list ap);
 
 int flux_conf_unpack (const flux_conf_t *conf,
-                      flux_conf_error_t *error,
-                      const char *fmt, ...);
+                      flux_error_t *error,
+                      const char *fmt,
+                      ...);
 
 #ifdef __cplusplus
 }
diff --git a/src/common/libflux/conf_private.h b/src/common/libflux/conf_private.h
index f5bebe63ceb9..013598aadf44 100644
--- a/src/common/libflux/conf_private.h
+++ b/src/common/libflux/conf_private.h
@@ -11,37 +11,11 @@
 #ifndef _FLUX_CORE_CONF_PRIVATE_H
 #define _FLUX_CORE_CONF_PRIVATE_H
 
-#include <stdarg.h>
-#include "handle.h"
 #include "conf.h"
 
-/* Cache 'conf' in the flux handle for subsequent use by flux_get_conf().
- * Call with conf=NULL to invalidate cache.
- * The config object is destroyed when the handle is destroyed.
- */
-int handle_set_conf (flux_t *h, flux_conf_t *conf);
-
-/* Destroy a flux_conf_t
- */
-void conf_destroy (flux_conf_t *conf);
-
-/* Parse 'pattern' (a glob) of toml files.
- * If error is non-NULL, it is filled with error details on failure.
- * Caller must destroy returned object with conf_destroy().
- */
-flux_conf_t *conf_parse (const char *pattern, flux_conf_error_t *error);
-
-/* Fill 'buf' with glob pattern for Flux config files:
- * If FLUX_CONF_DIR=<directory>, use *.toml in <directory>.
- * If FLUX_CONF_DIR="installed", use *.toml in the installed cf_path.
- * If FLUX_CONF_DIR is unset, use *.toml auto-selected installed/intree cf_path.
- * Returns 0 on success, or -1 with errno=EOVERFLOW if buf is too small.
- */
-int conf_get_default_pattern (char *buf, int bufsz);
-
 /* Set errno and optionally 'error' based on glob error return code.
  */
-void conf_globerr (flux_conf_error_t *error, const char *pattern, int rc);
+void conf_globerr (flux_error_t *error, const char *pattern, int rc);
 
 #endif /* !_FLUX_CORE_CONF_PRIVATE_H */
 
diff --git a/src/common/libflux/connector.h b/src/common/libflux/connector.h
index b72b5d43b295..d128a609b83c 100644
--- a/src/common/libflux/connector.h
+++ b/src/common/libflux/connector.h
@@ -19,25 +19,37 @@ extern "C" {
  ** Only handle implementation stuff below.
  ** Flux_t handle users should not use these interfaces.
  */
-typedef flux_t *(connector_init_f)(const char *uri, int flags);
+typedef flux_t *(connector_init_f)(const char *uri,
+                                   int flags,
+                                   flux_error_t *errp);
 
 struct flux_handle_ops {
-    int         (*setopt)(void *impl, const char *option,
-                          const void *val, size_t len);
-    int         (*getopt)(void *impl, const char *option,
-                          void *val, size_t len);
+    int         (*setopt)(void *impl,
+                          const char *option,
+                          const void *val,
+                          size_t len);
+    int         (*getopt)(void *impl,
+                          const char *option,
+                          void *val,
+                          size_t len);
     int         (*pollfd)(void *impl);
     int         (*pollevents)(void *impl);
     int         (*send)(void *impl, const flux_msg_t *msg, int flags);
     flux_msg_t* (*recv)(void *impl, int flags);
-
-    int         (*event_subscribe)(void *impl, const char *topic);
-    int         (*event_unsubscribe)(void *impl, const char *topic);
+    int         (*reconnect)(void *impl);
 
     void        (*impl_destroy)(void *impl);
+
+    // added in v0.56.0
+    int         (*send_new)(void *impl, flux_msg_t **msg, int flags);
+
+    // added in v0.56.0
+    void        *_pad[4]; // reserved for future use
 };
 
-flux_t *flux_handle_create (void *impl, const struct flux_handle_ops *ops, int flags);
+flux_t *flux_handle_create (void *impl,
+                            const struct flux_handle_ops *ops,
+                            int flags);
 void flux_handle_destroy (flux_t *hp);
 
 #ifdef __cplusplus
diff --git a/src/common/libflux/connector_interthread.c b/src/common/libflux/connector_interthread.c
new file mode 100644
index 000000000000..f30cb3066ba2
--- /dev/null
+++ b/src/common/libflux/connector_interthread.c
@@ -0,0 +1,368 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* connector_interthread.c - bidirectional, inter-thread message channel */
+
+/* Notes:
+ * - Each channel has a unique name and 1-2 attached flux_t handles.
+ * - Channels can be safely opened and closed from multiple threads.
+ * - The channel is created on first open and destroyed on last close.
+ * - There are no active/passive roles.
+ * - Writing is always non-blocking.
+ * - Reading can be either blocking or non-blocking.
+ * - Neither reading nor writing are affected if the other end disconnects.
+ * - Reconnect is allowed (by happenstance, not for any particular use case)
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <poll.h>
+#include <pthread.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/aux.h"
+#include "ccan/list/list.h"
+#include "ccan/str/str.h"
+
+#include "message_private.h" // for access to msg->aux
+#include "msg_deque.h"
+
+struct channel {
+    char *name;
+    struct msg_deque *pair[2];
+    int refcount; // max of 2
+    struct list_node list;
+};
+
+struct interthread_ctx {
+    flux_t *h;
+    struct flux_msg_cred cred;
+    char *router;
+    struct channel *chan;
+    struct msg_deque *send; // refers to ctx->chan->pair[x]
+    struct msg_deque *recv; // refers to ctx->chan->pair[y]
+};
+
+/* Global state.
+ * Threads accessing a channel must share this global in order to "connect".
+ */
+static struct list_head channels = LIST_HEAD_INIT (channels);
+static pthread_mutex_t channels_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static const struct flux_handle_ops handle_ops;
+
+static void channel_destroy (struct channel *chan)
+{
+    if (chan) {
+        int saved_errno = errno;
+        msg_deque_destroy (chan->pair[0]);
+        msg_deque_destroy (chan->pair[1]);
+        free (chan->name);
+        free (chan);
+        errno = saved_errno;
+    }
+}
+
+static struct channel *channel_create (const char *name)
+{
+    struct channel *chan;
+
+    if (!(chan = calloc (1, sizeof (*chan)))
+        || !(chan->name = strdup (name))
+        || !(chan->pair[0] = msg_deque_create (0))
+        || !(chan->pair[1] = msg_deque_create (0)))
+        goto error;
+    list_node_init (&chan->list);
+    return chan;
+error:
+    channel_destroy (chan);
+    return NULL;
+}
+
+/* Add new channel to global list under lock and take a reference.
+ */
+static void channel_add_safe (struct channel *chan)
+{
+    pthread_mutex_lock (&channels_lock);
+    list_add (&channels, &chan->list);
+    chan->refcount++;
+    pthread_mutex_unlock (&channels_lock);
+}
+
+/* Drop a reference on channel under lock.
+ * If the refcount becomes zero, remove it from the global list and return true.
+ */
+static bool channel_remove_safe (struct channel *chan)
+{
+    bool result = false;
+    if (chan) {
+        pthread_mutex_lock (&channels_lock);
+        if (--chan->refcount == 0) {
+            list_del (&chan->list);
+            result = true;
+        }
+        pthread_mutex_unlock (&channels_lock);
+    }
+    return result;
+}
+
+/* Look up a channel by name under lock, for pairing.
+ * If found but already paired (refcount == 2), fail with EADDRINUSE.
+ * If not found, fail with ENOENT.
+ * Otherwise take a reference under lock and return the channel.
+ */
+static struct channel *channel_pair_safe (const char *name)
+{
+    struct channel *result = NULL;
+    struct channel *chan = NULL;
+
+    pthread_mutex_lock (&channels_lock);
+    list_for_each (&channels, chan, list) {
+        if (streq (name, chan->name)) {
+            if (chan->refcount > 1) {
+                pthread_mutex_unlock (&channels_lock);
+                errno = EADDRINUSE;
+                return NULL;
+            }
+            chan->refcount++;
+            result = chan;
+            break;
+        }
+    }
+    pthread_mutex_unlock (&channels_lock);
+    if (!result)
+        errno = ENOENT;
+    return result;
+}
+
+/**
+ ** Connector interfaces follow
+ **/
+
+static int op_pollevents (void *impl)
+{
+    struct interthread_ctx *ctx = impl;
+    int e, revents = 0;
+
+    e = msg_deque_pollevents (ctx->recv);
+    if (e & POLLIN)
+        revents |= FLUX_POLLIN;
+    if (e & POLLOUT)
+        revents |= FLUX_POLLOUT;
+    if (e & POLLERR)
+        revents |= FLUX_POLLERR;
+    return revents;
+}
+
+static int op_pollfd (void *impl)
+{
+    struct interthread_ctx *ctx = impl;
+    return msg_deque_pollfd (ctx->recv);
+}
+
+static int router_process (flux_msg_t *msg, const char *name)
+{
+    int type;
+    if (flux_msg_get_type (msg, &type) < 0)
+        return -1;
+    switch (type) {
+        case FLUX_MSGTYPE_RESPONSE:
+            if (flux_msg_route_delete_last (msg) < 0)
+                return -1;
+            break;
+        case FLUX_MSGTYPE_REQUEST:
+        case FLUX_MSGTYPE_EVENT:
+            flux_msg_route_enable (msg);
+            if (flux_msg_route_push (msg, name) < 0)
+                return -1;
+            break;
+        default:
+            break;
+    }
+    return 0;
+}
+
+static int op_send_new (void *impl, flux_msg_t **msg, int flags)
+{
+    struct interthread_ctx *ctx = impl;
+    struct flux_msg_cred cred;
+
+    if (flux_msg_get_cred (*msg, &cred) < 0)
+        return -1;
+    if (cred.userid == FLUX_USERID_UNKNOWN
+        && cred.rolemask == FLUX_ROLE_NONE) {
+        if (flux_msg_set_cred (*msg, ctx->cred) < 0)
+            return -1;
+    }
+    if (ctx->router) {
+        if (router_process (*msg, ctx->router) < 0)
+            return -1;
+    }
+    /* The aux container doesn't survive transit of a TCP channel
+     * so it shouldn't survive transit of this kind either.
+     */
+    aux_destroy (&(*msg)->aux);
+    if (msg_deque_push_back (ctx->send, *msg) < 0)
+        return -1;
+    *msg = NULL;
+    return 0;
+}
+
+static int op_send (void *impl, const flux_msg_t *msg, int flags)
+{
+    flux_msg_t *cpy;
+
+    if (!(cpy = flux_msg_copy (msg, true))
+        || op_send_new (impl, &cpy, flags)) {
+        flux_msg_destroy (cpy);
+        return -1;
+    }
+    return 0;
+}
+
+static flux_msg_t *op_recv (void *impl, int flags)
+{
+    struct interthread_ctx *ctx = impl;
+    flux_msg_t *msg;
+
+    do {
+        msg = msg_deque_pop_front (ctx->recv);
+        if (!msg) {
+            if ((flags & FLUX_O_NONBLOCK)) {
+                errno = EWOULDBLOCK;
+                return NULL;
+            }
+            struct pollfd pfd = {
+                .fd = msg_deque_pollfd (ctx->recv),
+                .events = POLLIN,
+                .revents = 0,
+            };
+            if (poll (&pfd, 1, -1) < 0)
+                return NULL;
+        }
+    } while (!msg);
+    if (ctx->router) {
+        if (router_process (msg, ctx->chan->name) < 0) {
+            flux_msg_destroy (msg);
+            return NULL;
+        }
+    }
+    return msg;
+}
+
+static int op_getopt (void *impl, const char *option, void *val, size_t size)
+{
+    struct interthread_ctx *ctx = impl;
+
+    if (streq (option, FLUX_OPT_RECV_QUEUE_COUNT)) {
+        size_t count = msg_deque_count (ctx->recv);
+        if (size != sizeof (count) || !val)
+            goto error;
+        memcpy (val, &count, size);
+    }
+    else if (streq (option, FLUX_OPT_SEND_QUEUE_COUNT)) {
+        size_t count = msg_deque_count (ctx->send);
+        if (size != sizeof (count) || !val)
+            goto error;
+        memcpy (val, &count, size);
+    }
+    else
+        goto error;
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+static int op_setopt (void *impl,
+                      const char *option,
+                      const void *val,
+                      size_t size)
+{
+    struct interthread_ctx *ctx = impl;
+
+    if (streq (option, FLUX_OPT_ROUTER_NAME)) {
+        free (ctx->router);
+        if (!(ctx->router = strndup (val, size)))
+            return -1;
+    }
+    else
+        goto error;
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+static void op_fini (void *impl)
+{
+    struct interthread_ctx *ctx = impl;
+    if (ctx) {
+        int saved_errno = errno;
+        if (channel_remove_safe (ctx->chan))
+            channel_destroy (ctx->chan);
+        free (ctx->router);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+flux_t *connector_interthread_init (const char *path,
+                                    int flags,
+                                    flux_error_t *errp)
+{
+    struct interthread_ctx *ctx;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        goto error_seterrp;
+    ctx->cred.userid = getuid ();
+    ctx->cred.rolemask = FLUX_ROLE_OWNER | FLUX_ROLE_LOCAL;
+    if (!(ctx->chan = channel_pair_safe (path))) {
+        if (errno == EADDRINUSE) {
+            errprintf (errp, "interthread channel %s is already paired", path);
+            goto error;
+        }
+        else if (errno != ENOENT)
+            goto error_seterrp;
+        if (!(ctx->chan = channel_create (path)))
+            goto error_seterrp;
+        channel_add_safe (ctx->chan);
+        ctx->send = ctx->chan->pair[0];
+        ctx->recv = ctx->chan->pair[1];
+    }
+    else {
+        ctx->send = ctx->chan->pair[1];
+        ctx->recv = ctx->chan->pair[0];
+    }
+    if (!(ctx->h = flux_handle_create (ctx, &handle_ops, flags)))
+        goto error_seterrp;
+    return ctx->h;
+error_seterrp:
+    errprintf (errp, "%s", strerror (errno));
+error:
+    op_fini (ctx);
+    return NULL;
+}
+
+static const struct flux_handle_ops handle_ops = {
+    .pollfd = op_pollfd,
+    .pollevents = op_pollevents,
+    .send = op_send,
+    .send_new = op_send_new,
+    .recv = op_recv,
+    .setopt = op_setopt,
+    .getopt = op_getopt,
+    .impl_destroy = op_fini,
+};
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libflux/connector_local.c b/src/common/libflux/connector_local.c
new file mode 100644
index 000000000000..56dbdcdfc8bb
--- /dev/null
+++ b/src/common/libflux/connector_local.c
@@ -0,0 +1,234 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <flux/core.h>
+
+#include "src/common/librouter/usock.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+struct local_connector {
+    struct usock_client *uclient;
+    uint32_t testing_userid;
+    uint32_t testing_rolemask;
+    flux_t *h;
+    int fd;
+    char *path;
+};
+
+static const struct flux_handle_ops handle_ops;
+
+static void local_disconnect (struct local_connector *ctx);
+static int local_connect (struct local_connector *ctx);
+
+static int op_pollevents (void *impl)
+{
+    struct local_connector *ctx = impl;
+
+    return ctx->uclient ? usock_client_pollevents (ctx->uclient) : 0;
+}
+
+static int op_pollfd (void *impl)
+{
+    struct local_connector *ctx = impl;
+
+    return ctx->uclient ? usock_client_pollfd (ctx->uclient) : -1;
+}
+
+/* Special send function for testing that sets the userid/rolemask to
+ * values set with flux_opt_set().  The connector-local module
+ * overwrites these credentials for guests, but allows pass-through for
+ * instance owner.  This is useful for service access control testing.
+ */
+static int send_testing (struct local_connector *ctx,
+                         const flux_msg_t *msg,
+                         int flags)
+{
+    flux_msg_t *cpy;
+
+    if (!(cpy = flux_msg_copy (msg, true)))
+        return -1;
+    if (flux_msg_set_userid (cpy, ctx->testing_userid) < 0)
+        goto error;
+    if (flux_msg_set_rolemask (cpy, ctx->testing_rolemask) < 0)
+        goto error;
+    if (usock_client_send (ctx->uclient, cpy, flags) < 0)
+        goto error;
+    flux_msg_destroy (cpy);
+    return 0;
+error:
+    flux_msg_destroy (cpy);
+    return -1;
+}
+
+static int op_send (void *impl, const flux_msg_t *msg, int flags)
+{
+    struct local_connector *ctx = impl;
+
+    if (ctx->testing_userid != FLUX_USERID_UNKNOWN
+        || ctx->testing_rolemask != FLUX_ROLE_NONE)
+        return send_testing (ctx, msg, flags);
+
+    return usock_client_send (ctx->uclient, msg, flags);
+}
+
+static flux_msg_t *op_recv (void *impl, int flags)
+{
+    struct local_connector *ctx = impl;
+
+    return usock_client_recv (ctx->uclient, flags);
+}
+
+static int op_setopt (void *impl,
+                      const char *option,
+                      const void *val,
+                      size_t size)
+{
+    struct local_connector *ctx = impl;
+    size_t val_size;
+    int rc = -1;
+
+    if (streq (option, FLUX_OPT_TESTING_USERID)) {
+        val_size = sizeof (ctx->testing_userid);
+        if (size != val_size || !val) {
+            errno = EINVAL;
+            goto done;
+        }
+        memcpy (&ctx->testing_userid, val, val_size);
+    }
+    else if (streq (option, FLUX_OPT_TESTING_ROLEMASK)) {
+        val_size = sizeof (ctx->testing_rolemask);
+        if (size != val_size || !val) {
+            errno = EINVAL;
+            goto done;
+        }
+        memcpy (&ctx->testing_rolemask, val, val_size);
+    }
+    else {
+        errno = EINVAL;
+        goto done;
+    }
+    rc = 0;
+done:
+    return rc;
+}
+
+static void op_fini (void *impl)
+{
+    struct local_connector *ctx = impl;
+
+    if (ctx) {
+        int saved_errno = errno;
+        local_disconnect (ctx);
+        free (ctx->path);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static int override_retry_count (struct usock_retry_params *retry)
+{
+    const char *s;
+
+    if ((s = getenv ("FLUX_LOCAL_CONNECTOR_RETRY_COUNT"))) {
+        char *endptr;
+        int n;
+
+        errno = 0;
+        n = strtol (s, &endptr, 10);
+        if (errno != 0 || *endptr != '\0') {
+            errno = EINVAL;
+            return  -1;
+        }
+        retry->max_retry = n;
+    }
+    return 0;
+}
+
+flux_t *connector_local_init (const char *path, int flags, flux_error_t *errp)
+{
+    struct local_connector *ctx;
+
+    if (!path) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+
+    ctx->testing_userid = FLUX_USERID_UNKNOWN;
+    ctx->testing_rolemask = FLUX_ROLE_NONE;
+    ctx->fd = -1;
+
+    if (!(ctx->path = strdup (path)))
+        goto error;
+    if (local_connect (ctx) < 0) {
+        if (errno == ENOENT)
+            errprintf (errp,
+                       "broker socket %s was not found",
+                       path);
+        goto error;
+    }
+    if (!(ctx->h = flux_handle_create (ctx, &handle_ops, flags)))
+        goto error;
+    return ctx->h;
+error:
+    op_fini (ctx);
+    return NULL;
+}
+
+static void local_disconnect (struct local_connector *ctx)
+{
+    if (ctx->uclient) {
+        usock_client_destroy (ctx->uclient);
+        ctx->uclient = NULL;
+    }
+    if (ctx->fd >= 0) {
+        close (ctx->fd);
+        ctx->fd = -1;
+    }
+}
+
+static int local_connect (struct local_connector *ctx)
+{
+    struct usock_retry_params retry = USOCK_RETRY_DEFAULT;
+
+    if (override_retry_count (&retry) < 0
+        || (ctx->fd = usock_client_connect (ctx->path, retry)) < 0
+        || !(ctx->uclient = usock_client_create (ctx->fd)))
+        return -1;
+    return 0;
+}
+
+static int op_reconnect (void *impl)
+{
+    struct local_connector *ctx = impl;
+
+    local_disconnect (ctx);
+    return local_connect (ctx);
+}
+
+static const struct flux_handle_ops handle_ops = {
+    .pollfd = op_pollfd,
+    .pollevents = op_pollevents,
+    .send = op_send,
+    .recv = op_recv,
+    .reconnect = op_reconnect,
+    .setopt = op_setopt,
+    .impl_destroy = op_fini,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/connector_loop.c b/src/common/libflux/connector_loop.c
new file mode 100644
index 000000000000..35e24b369fa7
--- /dev/null
+++ b/src/common/libflux/connector_loop.c
@@ -0,0 +1,182 @@
+/************************************************************\
+ * Copyright 2015 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* loop connector - mainly for testing */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <poll.h>
+#include <flux/core.h>
+
+#include "ccan/str/str.h"
+#include "msg_deque.h"
+
+struct loop_ctx {
+    flux_t *h;
+    struct flux_msg_cred cred;
+    struct msg_deque *queue;
+};
+
+static const struct flux_handle_ops handle_ops;
+
+static int op_pollevents (void *impl)
+{
+    struct loop_ctx *ctx = impl;
+    int e, revents = 0;
+
+    if ((e = msg_deque_pollevents (ctx->queue)) < 0)
+        return e;
+    if (e & POLLIN)
+        revents |= FLUX_POLLIN;
+    if (e & POLLOUT)
+        revents |= FLUX_POLLOUT;
+    if (e & POLLERR)
+        revents |= FLUX_POLLERR;
+    return revents;
+}
+
+static int op_pollfd (void *impl)
+{
+    struct loop_ctx *ctx = impl;
+    return msg_deque_pollfd (ctx->queue);
+}
+
+static int op_send (void *impl, const flux_msg_t *msg, int flags)
+{
+    struct loop_ctx *ctx = impl;
+    flux_msg_t *cpy;
+    struct flux_msg_cred cred;
+
+    if (!(cpy = flux_msg_copy (msg, true)))
+        goto error;
+    if (flux_msg_get_cred (cpy, &cred) < 0)
+        goto error;
+    if (cred.userid == FLUX_USERID_UNKNOWN)
+        cred.userid = ctx->cred.userid;
+    if (cred.rolemask == FLUX_ROLE_NONE)
+        cred.rolemask = ctx->cred.rolemask;
+    if (flux_msg_set_cred (cpy, cred) < 0)
+        goto error;
+    if (msg_deque_push_back (ctx->queue, cpy) < 0)
+        goto error;
+    return 0;
+error:
+    flux_msg_destroy (cpy);
+    return -1;
+}
+
+static flux_msg_t *op_recv (void *impl, int flags)
+{
+    struct loop_ctx *ctx = impl;
+    flux_msg_t *msg = msg_deque_pop_front (ctx->queue);
+    if (!msg)
+        errno = EWOULDBLOCK;
+    return msg;
+}
+
+static int op_getopt (void *impl, const char *option, void *val, size_t size)
+{
+    struct loop_ctx *ctx = impl;
+
+    if (streq (option, FLUX_OPT_TESTING_USERID)) {
+        if (size != sizeof (ctx->cred.userid) || !val)
+            goto error;
+        memcpy (val, &ctx->cred.userid, size);
+    }
+    else if (streq (option, FLUX_OPT_TESTING_ROLEMASK)) {
+        if (size != sizeof (ctx->cred.rolemask) || !val)
+            goto error;
+        memcpy (val, &ctx->cred.rolemask, size);
+    }
+    else
+        goto error;
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+static int op_setopt (void *impl,
+                      const char *option,
+                      const void *val,
+                      size_t size)
+{
+    struct loop_ctx *ctx = impl;
+    size_t val_size;
+
+    if (streq (option, FLUX_OPT_TESTING_USERID)) {
+        val_size = sizeof (ctx->cred.userid);
+        if (size != val_size || !val)
+            goto error;
+        memcpy (&ctx->cred.userid, val, val_size);
+    }
+    else if (streq (option, FLUX_OPT_TESTING_ROLEMASK)) {
+        val_size = sizeof (ctx->cred.rolemask);
+        if (size != val_size || !val)
+            goto error;
+        memcpy (&ctx->cred.rolemask, val, val_size);
+    }
+    else
+        goto error;
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+static void op_fini (void *impl)
+{
+    struct loop_ctx *ctx = impl;
+
+    if (ctx) {
+        int saved_errno = errno;
+        msg_deque_destroy (ctx->queue);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+flux_t *connector_loop_init (const char *path, int flags, flux_error_t *errp)
+{
+    struct loop_ctx *ctx;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+    if (!(ctx->queue = msg_deque_create (MSG_DEQUE_SINGLE_THREAD)))
+        goto error;
+    ctx->cred.userid = getuid ();
+    ctx->cred.rolemask = FLUX_ROLE_OWNER;
+    if (!(ctx->h = flux_handle_create (ctx, &handle_ops, flags)))
+        goto error;
+    /* Fake out size, rank attributes for testing.
+     */
+    (void)flux_attr_set_cacheonly(ctx->h, "rank", "0");
+    (void)flux_attr_set_cacheonly (ctx->h, "size", "1");
+    return ctx->h;
+error:
+    op_fini (ctx);
+    return NULL;
+}
+
+static const struct flux_handle_ops handle_ops = {
+    .pollfd = op_pollfd,
+    .pollevents = op_pollevents,
+    .send = op_send,
+    .recv = op_recv,
+    .getopt = op_getopt,
+    .setopt = op_setopt,
+    .impl_destroy = op_fini,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/content.c b/src/common/libflux/content.c
deleted file mode 100644
index 3c90d59df3e2..000000000000
--- a/src/common/libflux/content.c
+++ /dev/null
@@ -1,78 +0,0 @@
-/************************************************************\
- * Copyright 2016 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <assert.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <flux/core.h>
-
-#include "content.h"
-
-#include "src/common/libutil/blobref.h"
-
-flux_future_t *flux_content_load (flux_t *h, const char *blobref, int flags)
-{
-    const char *topic = "content.load";
-    uint32_t rank = FLUX_NODEID_ANY;
-
-    if (!h || !blobref || blobref_validate (blobref) < 0) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if ((flags & CONTENT_FLAG_UPSTREAM))
-        rank = FLUX_NODEID_UPSTREAM;
-    if ((flags & CONTENT_FLAG_CACHE_BYPASS)) {
-        topic = "content-backing.load";
-        rank = 0;
-    }
-    return flux_rpc_raw (h, topic, blobref, strlen (blobref) + 1, rank, 0);
-}
-
-int flux_content_load_get (flux_future_t *f, const void **buf, int *len)
-{
-    return flux_rpc_get_raw (f, buf, len);
-}
-
-flux_future_t *flux_content_store (flux_t *h, const void *buf, int len, int flags)
-{
-    const char *topic = "content.store";
-    uint32_t rank = FLUX_NODEID_ANY;
-
-    if ((flags & CONTENT_FLAG_UPSTREAM))
-        rank = FLUX_NODEID_UPSTREAM;
-    if ((flags & CONTENT_FLAG_CACHE_BYPASS)) {
-        topic = "content-backing.store";
-        rank = 0;
-    }
-    return flux_rpc_raw (h, topic, buf, len, rank, 0);
-}
-
-int flux_content_store_get (flux_future_t *f, const char **blobref)
-{
-    int ref_size;
-    const char *ref;
-
-    if (flux_rpc_get_raw (f, (const void **)&ref, &ref_size) < 0)
-        return -1;
-    if (!ref || ref[ref_size - 1] != '\0' || blobref_validate (ref) < 0) {
-        errno = EPROTO;
-        return -1;
-    }
-    if (blobref)
-        *blobref = ref;
-    return 0;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/content.h b/src/common/libflux/content.h
deleted file mode 100644
index 65b45df7fed3..000000000000
--- a/src/common/libflux/content.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/************************************************************\
- * Copyright 2016 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_CORE_CONTENT_H
-#define _FLUX_CORE_CONTENT_H
-
-#include <flux/core.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* flags */
-enum {
-    CONTENT_FLAG_CACHE_BYPASS = 1,/* request direct to backing store */
-    CONTENT_FLAG_UPSTREAM = 2,    /* make request of upstream TBON peer */
-};
-
-/* Send request to load blob by blobref.
- */
-flux_future_t *flux_content_load (flux_t *h,
-                                  const char *blobref, int flags);
-
-/* Get result of load request (blob).
- * This blocks until response is received.
- * Storage for 'buf' belongs to 'f' and is valid until 'f' is destroyed.
- * Returns 0 on success, -1 on failure with errno set.
- */
-int flux_content_load_get (flux_future_t *f, const void **buf, int *len);
-
-/* Send request to store blob.
- */
-flux_future_t *flux_content_store (flux_t *h,
-                                   const void *buf, int len, int flags);
-
-/* Get result of store request (blobref).
- * Storage for 'blobref' belongs to 'f' and is valid until 'f' is destroyed.
- * Returns 0 on success, -1 on failure with errno set.
- */
-int flux_content_store_get (flux_future_t *f, const char **blobref);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* !_FLUX_CORE_CONTENT_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/control.c b/src/common/libflux/control.c
new file mode 100644
index 000000000000..ef42f69ea413
--- /dev/null
+++ b/src/common/libflux/control.c
@@ -0,0 +1,47 @@
+/************************************************************\
+ * Copyright 2016 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "control.h"
+
+flux_msg_t *flux_control_encode (int type, int status)
+{
+    flux_msg_t *msg;
+
+    if (!(msg = flux_msg_create (FLUX_MSGTYPE_CONTROL)))
+        goto error;
+    if (flux_msg_set_control (msg, type, status) < 0)
+        goto error;
+    return msg;
+error:
+    flux_msg_destroy (msg);
+    return NULL;
+}
+
+int flux_control_decode (const flux_msg_t *msg, int *typep, int *statusp)
+{
+    int type, status;
+
+    if (flux_msg_get_control (msg, &type, &status) < 0)
+        return -1;
+    if (typep)
+        *typep = type;
+    if (statusp)
+        *statusp = status;
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/control.h b/src/common/libflux/control.h
new file mode 100644
index 000000000000..277d5567f478
--- /dev/null
+++ b/src/common/libflux/control.h
@@ -0,0 +1,30 @@
+/************************************************************\
+ * Copyright 2016 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_CONTROL_H
+#define _FLUX_CORE_CONTROL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+flux_msg_t *flux_control_encode (int type, int status);
+
+int flux_control_decode (const flux_msg_t *msg, int *type, int *status);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_FLUX_CORE_CONTROL_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/disconnect.c b/src/common/libflux/disconnect.c
new file mode 100644
index 000000000000..8e6ecf8ea96f
--- /dev/null
+++ b/src/common/libflux/disconnect.c
@@ -0,0 +1,99 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* disconnect.c - helpers for managing RFC 6 disconnect and cancel requests
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+bool flux_disconnect_match (const flux_msg_t *msg1, const flux_msg_t *msg2)
+{
+    struct flux_msg_cred cred;
+    uint32_t userid;
+
+    if (!flux_msg_route_match_first (msg1, msg2))
+        return false;
+
+    if (flux_msg_get_cred (msg1, &cred) < 0)
+        return false;
+
+    if (flux_msg_get_userid (msg2, &userid) < 0)
+        return false;
+
+    if (flux_msg_cred_authorize (cred, userid) < 0)
+        return false;
+
+    return true;
+}
+
+int flux_msglist_disconnect (struct flux_msglist *l, const flux_msg_t *msg)
+{
+    const flux_msg_t *item;
+    int count = 0;
+
+    item = flux_msglist_first (l);
+    while (item) {
+        if (flux_disconnect_match (msg, item)) {
+            flux_msglist_delete (l);
+            count++;
+        }
+        item = flux_msglist_next (l);
+    }
+    return count;
+}
+
+bool flux_cancel_match (const flux_msg_t *msg1, const flux_msg_t *msg2)
+{
+    uint32_t matchtag;
+    uint32_t tag;
+
+    if (!flux_disconnect_match (msg1, msg2))
+        return false;
+
+    if (flux_request_unpack (msg1, NULL, "{s:i}", "matchtag", &matchtag) < 0)
+        return false;
+
+    if (flux_msg_get_matchtag (msg2, &tag) < 0)
+        return false;
+
+    if (tag != matchtag)
+        return false;
+
+    return true;
+}
+
+int flux_msglist_cancel (flux_t *h,
+                         struct flux_msglist *l,
+                         const flux_msg_t *msg)
+{
+    const flux_msg_t *item;
+    int count = 0;
+
+    item = flux_msglist_first (l);
+    while (item) {
+        if (flux_cancel_match (msg, item)) {
+            if (flux_respond_error (h, item, ENODATA, NULL) < 0)
+                return -1;
+            flux_msglist_delete (l);
+            count++;
+            break;
+        }
+        item = flux_msglist_next (l);
+    }
+    return count;
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
+
diff --git a/src/common/libflux/disconnect.h b/src/common/libflux/disconnect.h
new file mode 100644
index 000000000000..e753f60e7e77
--- /dev/null
+++ b/src/common/libflux/disconnect.h
@@ -0,0 +1,49 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_DISCONNECT_H
+#define _FLUX_CORE_DISCONNECT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Return true if disconnect request msg1 came from same sender as
+ * msg2 and has appropriate authorization
+ */
+bool flux_disconnect_match (const flux_msg_t *msg1, const flux_msg_t *msg2);
+
+/* Remove all messages in 'l' with the same sender as 'msg'.
+ * Return 0 or the number of messages removed.
+ */
+int flux_msglist_disconnect (struct flux_msglist *l, const flux_msg_t *msg);
+
+/* Return true if cancel request msg1 came from same sender as msg2,
+ * has appropriate authorization, and references the matchtag in
+ * msg2 */
+bool flux_cancel_match (const flux_msg_t *msg1, const flux_msg_t *msg2);
+
+/* Respond to and remove the first message in 'l' that matches 'msg'.
+ * The sender must match 'msg', and the matchtag must match the one in
+ * the cancel request payload.
+ */
+int flux_msglist_cancel (flux_t *h,
+                         struct flux_msglist *l,
+                         const flux_msg_t *msg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_FLUX_CORE_DISCONNECT_H */
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libflux/ev_buffer_read.c b/src/common/libflux/ev_buffer_read.c
deleted file mode 100644
index fc5988f79449..000000000000
--- a/src/common/libflux/ev_buffer_read.c
+++ /dev/null
@@ -1,198 +0,0 @@
-/************************************************************\
- * Copyright 2015 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <stddef.h>
-#include <stdbool.h>
-
-#include "src/common/libev/ev.h"
-
-#include "ev_buffer_read.h"
-
-#include "buffer_private.h"
-
-static bool data_to_read (struct ev_buffer_read *ebr, bool *is_eof)
-{
-    if (ebr->line) {
-        if (flux_buffer_has_line (ebr->fb))
-            return true;
-        /* if eof read, no lines, but left over data non-line data,
-         * this data should be flushed to the user */
-        else if (ebr->eof_read
-                 && flux_buffer_bytes (ebr->fb))
-            return true;
-    }
-    else {
-        if (flux_buffer_bytes (ebr->fb) > 0)
-            return true;
-    }
-
-    if (ebr->eof_read
-        && !ebr->eof_sent
-        && !flux_buffer_bytes (ebr->fb)) {
-        if (is_eof)
-            (*is_eof) = true;
-        return true;
-    }
-
-    return false;
-}
-
-static void buffer_space_available_cb (flux_buffer_t *fb, void *arg)
-{
-    struct ev_buffer_read *ebr = arg;
-
-    /* space is available, start ev io watcher again, assuming watcher
-     * is not stopped by user */
-    if (ebr->start)
-        ev_io_start (ebr->loop, &(ebr->io_w));
-
-    /* clear this callback */
-    if (flux_buffer_set_high_write_cb (ebr->fb,
-                                       NULL,
-                                       0,
-                                       NULL) < 0)
-        return;
-}
-
-static void prepare_cb (struct ev_loop *loop, ev_prepare *w, int revents)
-{
-    struct ev_buffer_read *ebr = (struct ev_buffer_read *)((char *)w
-                            - offsetof (struct ev_buffer_read, prepare_w));
-
-    if (data_to_read (ebr, NULL) == true)
-        ev_idle_start (loop, &ebr->idle_w);
-}
-
-static void buffer_read_cb (struct ev_loop *loop, ev_io *iow, int revents)
-{
-    struct ev_buffer_read *ebr = iow->data;
-
-    if (revents & EV_READ) {
-        int ret, space;
-
-        if ((space = flux_buffer_space (ebr->fb)) < 0)
-            return;
-
-        if ((ret = flux_buffer_write_from_fd (ebr->fb, ebr->fd, space)) < 0)
-            return;
-
-        if (!ret) {
-            ebr->eof_read = true;
-            (void)flux_buffer_readonly (ebr->fb);
-            ev_io_stop (ebr->loop, iow);
-        }
-        else if (ret == space) {
-            /* buffer full, buffer_space_available_cb will be called
-             * to re-enable io reactor when space is available */
-            if (flux_buffer_set_high_write_cb (ebr->fb,
-                                               buffer_space_available_cb,
-                                               flux_buffer_size (ebr->fb),
-                                               ebr) < 0)
-                return;
-
-            ev_io_stop (ebr->loop, iow);
-        }
-    }
-    else {
-        if (ebr->cb)
-            ebr->cb (loop, ebr, revents);
-    }
-}
-
-static void check_cb (struct ev_loop *loop, ev_check *w, int revents)
-{
-    struct ev_buffer_read *ebr = (struct ev_buffer_read *)((char *)w
-                            - offsetof (struct ev_buffer_read, check_w));
-    bool is_eof = false;
-
-    ev_idle_stop (loop, &ebr->idle_w);
-
-    if (data_to_read (ebr, &is_eof) == true) {
-        if (ebr->cb)
-            ebr->cb (loop, ebr, EV_READ);
-
-        if (is_eof)
-            ebr->eof_sent = true;
-    }
-}
-
-int ev_buffer_read_init (struct ev_buffer_read *ebr,
-                         int fd,
-                         int size,
-                         ev_buffer_read_f cb,
-                         struct ev_loop *loop)
-{
-    ebr->cb = cb;
-    ebr->fd = fd;
-    ebr->loop = loop;
-    ebr->start = false;
-    ebr->eof_read = false;
-    ebr->eof_sent = false;
-
-    if (!(ebr->fb = flux_buffer_create (size)))
-        goto cleanup;
-
-    ev_prepare_init (&ebr->prepare_w, prepare_cb);
-    ev_check_init (&ebr->check_w, check_cb);
-    ev_idle_init (&ebr->idle_w, NULL);
-    ev_io_init (&ebr->io_w, buffer_read_cb, ebr->fd, EV_READ);
-    ebr->io_w.data = ebr;
-
-    return 0;
-
-cleanup:
-    ev_buffer_read_cleanup (ebr);
-    return -1;
-}
-
-void ev_buffer_read_cleanup (struct ev_buffer_read *ebr)
-{
-    if (ebr) {
-        flux_buffer_destroy (ebr->fb);
-        ebr->fb = NULL;
-    }
-}
-
-void ev_buffer_read_start (struct ev_loop *loop, struct ev_buffer_read *ebr)
-{
-    if (!ebr->start) {
-        ebr->start = true;
-        ev_prepare_start (loop, &ebr->prepare_w);
-        ev_check_start (loop, &ebr->check_w);
-
-        if (flux_buffer_space (ebr->fb) > 0)
-            ev_io_start (ebr->loop, &(ebr->io_w));
-        else {
-            /* buffer full, buffer_space_available_cb will be called
-             * to re-enable io reactor when space is available */
-            if (flux_buffer_set_high_write_cb (ebr->fb,
-                                               buffer_space_available_cb,
-                                               flux_buffer_size (ebr->fb),
-                                               ebr) < 0)
-                return;
-        }
-    }
-}
-
-void ev_buffer_read_stop (struct ev_loop *loop, struct ev_buffer_read *ebr)
-{
-    if (ebr->start) {
-        ev_prepare_stop (loop, &ebr->prepare_w);
-        ev_check_stop (loop, &ebr->check_w);
-        ev_io_stop (loop, &ebr->io_w);
-        ev_idle_stop (loop, &ebr->idle_w);
-        ebr->start = false;
-    }
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/src/common/libflux/ev_buffer_read.h b/src/common/libflux/ev_buffer_read.h
deleted file mode 100644
index 7ebc2d63a13b..000000000000
--- a/src/common/libflux/ev_buffer_read.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _EV_BUFFER_READ_H
-#define _EV_BUFFER_READ_H
-
-#include "src/common/libev/ev.h"
-#include "src/common/libflux/buffer.h"
-
-struct ev_buffer_read;
-
-typedef void (*ev_buffer_read_f)(struct ev_loop *loop,
-                                 struct ev_buffer_read *ebr,
-                                 int revents);
-
-struct ev_buffer_read {
-    ev_io            io_w;
-    ev_prepare       prepare_w;
-    ev_idle          idle_w;
-    ev_check         check_w;
-    int              fd;
-    ev_buffer_read_f cb;
-    flux_buffer_t    *fb;
-    struct ev_loop   *loop;
-    bool             start;     /* flag, if user started reactor */
-    bool             eof_read;  /* flag, if EOF on stream seen */
-    bool             eof_sent;  /* flag, if EOF to user sent */
-    bool             line;      /* flag, if line buffered */
-    void             *data;
-};
-
-int ev_buffer_read_init (struct ev_buffer_read *ebr,
-                         int fd,
-                         int size,
-                         ev_buffer_read_f cb,
-                         struct ev_loop *loop);
-void ev_buffer_read_cleanup (struct ev_buffer_read *ebr);
-void ev_buffer_read_start (struct ev_loop *loop, struct ev_buffer_read *ebr);
-void ev_buffer_read_stop (struct ev_loop *loop, struct ev_buffer_read *ebr);
-
-#endif /* !_EV_BUFFER_READ_H */
diff --git a/src/common/libflux/ev_buffer_write.c b/src/common/libflux/ev_buffer_write.c
deleted file mode 100644
index 052d2d9e3ccf..000000000000
--- a/src/common/libflux/ev_buffer_write.c
+++ /dev/null
@@ -1,127 +0,0 @@
-/************************************************************\
- * Copyright 2015 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <stddef.h>
-#include <stdbool.h>
-#include <unistd.h>
-#include <errno.h>
-
-#include "src/common/libev/ev.h"
-
-#include "ev_buffer_write.h"
-
-#include "buffer_private.h"
-
-static void buffer_write_cb (struct ev_loop *loop, ev_io *iow, int revents)
-{
-    struct ev_buffer_write *ebw = iow->data;
-
-    if (revents & EV_WRITE) {
-
-        if (flux_buffer_read_to_fd (ebw->fb, ebw->fd, -1) < 0)
-            return;
-
-        if (!flux_buffer_bytes (ebw->fb) && ebw->eof) {
-            if (close (ebw->fd) < 0)
-                ebw->close_errno = errno;
-            ebw->fd = -1;
-            ebw->closed = true;
-            ebw->eof = false;
-            if (ebw->cb)
-                ebw->cb (loop, ebw, revents);
-        }
-
-        if (!flux_buffer_bytes (ebw->fb) && !ebw->eof)
-            ev_io_stop (ebw->loop, &(ebw->io_w));
-    }
-    else {
-        if (ebw->cb)
-            ebw->cb (loop, ebw, revents);
-    }
-}
-
-/* data is available, start ev io watcher assuming user has
- * started the watcher.
- */
-void ev_buffer_write_wakeup (struct ev_buffer_write *ebw)
-{
-    if (ebw->start)
-        ev_io_start (ebw->loop, &(ebw->io_w));
-}
-
-static void buffer_data_available_cb (flux_buffer_t *fb, void *arg)
-{
-    struct ev_buffer_write *ebw = arg;
-    ev_buffer_write_wakeup (ebw);
-}
-
-int ev_buffer_write_init (struct ev_buffer_write *ebw,
-                          int fd,
-                          int size,
-                          ev_buffer_write_f cb,
-                          struct ev_loop *loop)
-{
-    ebw->cb = cb;
-    ebw->fd = fd;
-    ebw->loop = loop;
-    ebw->start = false;
-
-    if (!(ebw->fb = flux_buffer_create (size)))
-        goto cleanup;
-
-    /* When any data becomes available, call buffer_data_available_cb,
-     * which will start io reactor
-     */
-    if (flux_buffer_set_low_read_cb (ebw->fb,
-                                     buffer_data_available_cb,
-                                     0,
-                                     ebw) < 0)
-        goto cleanup;
-
-    ev_io_init (&ebw->io_w, buffer_write_cb, ebw->fd, EV_WRITE);
-    ebw->io_w.data = ebw;
-
-    return 0;
-
-cleanup:
-    ev_buffer_write_cleanup (ebw);
-    return -1;
-}
-
-void ev_buffer_write_cleanup (struct ev_buffer_write *ebw)
-{
-    if (ebw) {
-        flux_buffer_destroy (ebw->fb);
-        ebw->fb = NULL;
-    }
-}
-
-void ev_buffer_write_start (struct ev_loop *loop, struct ev_buffer_write *ebw)
-{
-    if (!ebw->start) {
-        ebw->start = true;
-        /* do not start watcher unless there is data or EOF to be written out */
-        if (flux_buffer_bytes (ebw->fb) || ebw->eof)
-            ev_io_start (ebw->loop, &(ebw->io_w));
-    }
-}
-
-void ev_buffer_write_stop (struct ev_loop *loop, struct ev_buffer_write *ebw)
-{
-    if (ebw->start) {
-        ev_io_stop (loop, &ebw->io_w);
-        ebw->start = false;
-    }
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/src/common/libflux/ev_buffer_write.h b/src/common/libflux/ev_buffer_write.h
deleted file mode 100644
index e104a07632c8..000000000000
--- a/src/common/libflux/ev_buffer_write.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/************************************************************\
- * Copyright 2015 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _EV_BUFFER_WRITE_H
-#define _EV_BUFFER_WRITE_H
-
-#include "src/common/libev/ev.h"
-#include "src/common/libflux/buffer.h"
-
-struct ev_buffer_write;
-
-typedef void (*ev_buffer_write_f)(struct ev_loop *loop,
-                                  struct ev_buffer_write *ebw,
-                                  int revents);
-
-struct ev_buffer_write {
-    ev_io             io_w;
-    int               fd;
-    ev_buffer_write_f cb;
-    flux_buffer_t     *fb;
-    struct ev_loop    *loop;
-    bool              start;    /* flag, if user started reactor */
-    bool              eof;      /* flag, eof written             */
-    bool              closed;   /* flag, fd has been closed      */
-    int               close_errno;  /* errno from close          */
-    void              *data;
-};
-
-int ev_buffer_write_init (struct ev_buffer_write *ebw,
-                          int fd,
-                          int size,
-                          ev_buffer_write_f cb,
-                          struct ev_loop *loop);
-void ev_buffer_write_cleanup (struct ev_buffer_write *ebw);
-void ev_buffer_write_start (struct ev_loop *loop, struct ev_buffer_write *ebw);
-void ev_buffer_write_stop (struct ev_loop *loop, struct ev_buffer_write *ebw);
-void ev_buffer_write_wakeup (struct ev_buffer_write *ebw);
-#endif /* !_EV_BUFFER_WRITE_H */
diff --git a/src/common/libflux/ev_flux.c b/src/common/libflux/ev_flux.c
index 48bc5cd3968a..4c939d7aa3e2 100644
--- a/src/common/libflux/ev_flux.c
+++ b/src/common/libflux/ev_flux.c
@@ -8,10 +8,12 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include <zmq.h>
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stddef.h>
 #include <stdbool.h>
-
-#include "handle.h"
+#include <flux/core.h>
 
 #include "src/common/libev/ev.h"
 
@@ -94,6 +96,11 @@ void ev_flux_stop (struct ev_loop *loop, struct ev_flux *w)
     ev_idle_stop (loop, &w->idle_w);
 }
 
+bool ev_flux_is_active (struct ev_flux *w)
+{
+    return ev_is_active (&w->prepare_w);
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libflux/ev_flux.h b/src/common/libflux/ev_flux.h
index d037ce691590..2fb7a269fcd8 100644
--- a/src/common/libflux/ev_flux.h
+++ b/src/common/libflux/ev_flux.h
@@ -32,5 +32,6 @@ struct ev_flux {
 int ev_flux_init (struct ev_flux *w, ev_flux_f cb, flux_t *h, int events);
 void ev_flux_start (struct ev_loop *loop, struct ev_flux *w);
 void ev_flux_stop (struct ev_loop *loop, struct ev_flux *w);
+bool ev_flux_is_active (struct ev_flux *w);
 
 #endif /* !_EV_FLUX_H */
diff --git a/src/common/libflux/event.c b/src/common/libflux/event.c
index 011a3d6e9868..f56d187da1ed 100644
--- a/src/common/libflux/event.c
+++ b/src/common/libflux/event.c
@@ -15,11 +15,71 @@
 #include <stdbool.h>
 #include <stdarg.h>
 #include <jansson.h>
-#include <sodium.h>
+#include <flux/core.h>
 
-#include "event.h"
-#include "rpc.h"
-#include "message.h"
+#include "ccan/base64/base64.h"
+
+flux_future_t *flux_event_subscribe_ex (flux_t *h,
+                                        const char *topic,
+                                        int flags)
+{
+    if (!topic) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return flux_rpc_pack (h,
+                          "event.subscribe",
+                          FLUX_NODEID_ANY,
+                          flags,
+                          "{s:s}",
+                          "topic", topic);
+}
+
+int flux_event_subscribe (flux_t *h, const char *topic)
+{
+    flux_future_t *f;
+
+    if (h && (flux_flags_get (h) & FLUX_O_TEST_NOSUB))
+        return 0;
+    if (!(f = flux_event_subscribe_ex (h, topic, 0))
+        || flux_future_get (f, NULL) < 0) {
+        flux_future_destroy (f);
+        return -1;
+    }
+    flux_future_destroy (f);
+    return 0;
+}
+
+flux_future_t *flux_event_unsubscribe_ex (flux_t *h,
+                                          const char *topic,
+                                          int flags)
+{
+    if (!topic) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return flux_rpc_pack (h,
+                          "event.unsubscribe",
+                          FLUX_NODEID_ANY,
+                          flags,
+                          "{s:s}",
+                          "topic", topic);
+}
+
+int flux_event_unsubscribe (flux_t *h, const char *topic)
+{
+    flux_future_t *f;
+
+    if (h && (flux_flags_get (h) & FLUX_O_TEST_NOSUB))
+        return 0;
+    if (!(f = flux_event_unsubscribe_ex (h, topic, 0))
+        || flux_future_get (f, NULL) < 0) {
+        flux_future_destroy (f);
+        return -1;
+    }
+    flux_future_destroy (f);
+    return 0;
+}
 
 static int event_decode (const flux_msg_t *msg, const char **topic)
 {
@@ -47,7 +107,8 @@ static int event_decode (const flux_msg_t *msg, const char **topic)
 }
 
 
-int flux_event_decode (const flux_msg_t *msg, const char **topicp,
+int flux_event_decode (const flux_msg_t *msg,
+                       const char **topicp,
                        const char **sp)
 {
     const char *topic, *s;
@@ -66,12 +127,14 @@ int flux_event_decode (const flux_msg_t *msg, const char **topicp,
     return rc;
 }
 
-int flux_event_decode_raw (const flux_msg_t *msg, const char **topicp,
-                           const void **datap, int *lenp)
+int flux_event_decode_raw (const flux_msg_t *msg,
+                           const char **topicp,
+                           const void **datap,
+                           size_t *lenp)
 {
     const char *topic;
     const void *data = NULL;
-    int len = 0;
+    size_t len = 0;
     int rc = -1;
 
     if (!datap || !lenp) {
@@ -94,8 +157,10 @@ int flux_event_decode_raw (const flux_msg_t *msg, const char **topicp,
     return rc;
 }
 
-static int flux_event_vunpack (const flux_msg_t *msg, const char **topic,
-                               const char *fmt, va_list ap)
+static int flux_event_vunpack (const flux_msg_t *msg,
+                               const char **topic,
+                               const char *fmt,
+                               va_list ap)
 {
     const char *ts;
     int rc = -1;
@@ -111,8 +176,10 @@ static int flux_event_vunpack (const flux_msg_t *msg, const char **topic,
     return rc;
 }
 
-int flux_event_unpack (const flux_msg_t *msg, const char **topic,
-                       const char *fmt, ...)
+int flux_event_unpack (const flux_msg_t *msg,
+                       const char **topic,
+                       const char *fmt,
+                       ...)
 {
     va_list ap;
     int rc;
@@ -135,8 +202,7 @@ static flux_msg_t *flux_event_create (const char *topic)
         goto error;
     if (flux_msg_set_topic (msg, topic) < 0)
         goto error;
-    if (flux_msg_enable_route (msg) < 0)
-        goto error;
+    flux_msg_route_enable (msg);
     return msg;
 error:
     flux_msg_destroy (msg);
@@ -157,7 +223,8 @@ flux_msg_t *flux_event_encode (const char *topic, const char *s)
 }
 
 flux_msg_t *flux_event_encode_raw (const char *topic,
-                                   const void *data, int len)
+                                   const void *data,
+                                   size_t len)
 {
     flux_msg_t *msg = flux_event_create (topic);
     if (!msg)
@@ -171,7 +238,8 @@ flux_msg_t *flux_event_encode_raw (const char *topic,
 }
 
 static flux_msg_t *flux_event_vpack (const char *topic,
-                                     const char *fmt, va_list ap)
+                                     const char *fmt,
+                                     va_list ap)
 {
     flux_msg_t *msg = flux_event_create (topic);
     if (!msg)
@@ -196,23 +264,31 @@ flux_msg_t *flux_event_pack (const char *topic, const char *fmt, ...)
 }
 
 static flux_future_t *wrap_event_rpc (flux_t *h,
-                                      const char *topic, int flags,
-                                      const void *src, int srclen)
+                                      const char *topic,
+                                      int flags,
+                                      const void *src,
+                                      size_t srclen)
 {
     flux_future_t *f;
 
     if (src) {
-        size_t dstlen = sodium_base64_encoded_len (srclen,
-                                            sodium_base64_VARIANT_ORIGINAL);
-        void *dst;
-        if (!(dst = malloc (dstlen)))
+        size_t dstbuflen = base64_encoded_length (srclen) + 1; /* +1 for NUL */
+        char *dst;
+        if (!(dst = malloc (dstbuflen)))
             return NULL;
-        sodium_bin2base64 (dst, dstlen, (unsigned char *)src, srclen,
-                           sodium_base64_VARIANT_ORIGINAL);
-        if (!(f = flux_rpc_pack (h, "event.pub", FLUX_NODEID_ANY, 0,
-                                 "{s:s s:i s:s}", "topic", topic,
-                                                  "flags", flags,
-                                                  "payload", dst))) {
+        if (base64_encode (dst, dstbuflen, (const char *)src, srclen) < 0) {
+            free (dst);
+            errno = EPROTO;
+            return NULL;
+        }
+        if (!(f = flux_rpc_pack (h,
+                                 "event.publish",
+                                 0,
+                                 0,
+                                 "{s:s s:i s:s}",
+                                 "topic", topic,
+                                 "flags", flags,
+                                 "payload", dst))) {
             int saved_errno = errno;
             free (dst);
             errno = saved_errno;
@@ -221,9 +297,13 @@ static flux_future_t *wrap_event_rpc (flux_t *h,
         free (dst);
     }
     else {
-        if (!(f = flux_rpc_pack (h, "event.pub", FLUX_NODEID_ANY, 0,
-                                    "{s:s s:i}", "topic", topic,
-                                                 "flags", flags))) {
+        if (!(f = flux_rpc_pack (h,
+                                 "event.publish",
+                                 0,
+                                 0,
+                                 "{s:s s:i}",
+                                 "topic", topic,
+                                 "flags", flags))) {
             return NULL;
         }
     }
@@ -231,7 +311,8 @@ static flux_future_t *wrap_event_rpc (flux_t *h,
 }
 
 flux_future_t *flux_event_publish (flux_t *h,
-                                   const char *topic, int flags,
+                                   const char *topic,
+                                   int flags,
                                    const char *json_str)
 {
     int len = 0;
@@ -245,8 +326,10 @@ flux_future_t *flux_event_publish (flux_t *h,
 }
 
 flux_future_t *flux_event_publish_pack (flux_t *h,
-                                        const char *topic, int flags,
-                                        const char *fmt, ...)
+                                        const char *topic,
+                                        int flags,
+                                        const char *fmt,
+                                        ...)
 {
     va_list ap;
     json_t *o;
@@ -270,8 +353,11 @@ flux_future_t *flux_event_publish_pack (flux_t *h,
         return NULL;
     }
     json_decref (o);
-    if (!(f = wrap_event_rpc (h,  topic, flags,
-                              json_str, strlen (json_str) + 1))) {
+    if (!(f = wrap_event_rpc (h,
+                              topic,
+                              flags,
+                              json_str,
+                              strlen (json_str) + 1))) {
         int saved_errno = errno;
         free (json_str);
         errno = saved_errno;
@@ -282,8 +368,10 @@ flux_future_t *flux_event_publish_pack (flux_t *h,
 }
 
 flux_future_t *flux_event_publish_raw (flux_t *h,
-                                       const char *topic, int flags,
-                                       const void *data, int len)
+                                       const char *topic,
+                                       int flags,
+                                       const void *data,
+                                       size_t len)
 {
     if (!h || !topic || (flags & ~(FLUX_MSGFLAG_PRIVATE)) != 0) {
         errno = EINVAL;
diff --git a/src/common/libflux/event.h b/src/common/libflux/event.h
index b9034a4daf6f..1848a811de19 100644
--- a/src/common/libflux/event.h
+++ b/src/common/libflux/event.h
@@ -11,9 +11,6 @@
 #ifndef _FLUX_CORE_EVENT_H
 #define _FLUX_CORE_EVENT_H
 
-#include "message.h"
-#include "future.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -22,22 +19,35 @@ enum event_flags {
     FLUX_EVENT_PRIVATE = 1,
 };
 
+/* Event subscribe/unsubscribe.
+ */
+int flux_event_subscribe (flux_t *h, const char *topic);
+int flux_event_unsubscribe (flux_t *h, const char *topic);
+flux_future_t *flux_event_subscribe_ex (flux_t *h,
+                                        const char *topic,
+                                        int flags);
+flux_future_t *flux_event_unsubscribe_ex (flux_t *h,
+                                          const char *topic,
+                                          int flags);
+
 /* Decode an event message with optional string payload.
  * If topic is non-NULL, assign the event topic string.
  * If s is non-NULL, assign string payload or set to NULL if none
  * exists.  Returns 0 on success, or -1 on failure with errno set.
  */
-int flux_event_decode (const flux_msg_t *msg, const char **topic,
+int flux_event_decode (const flux_msg_t *msg,
+                       const char **topic,
                        const char **s);
 
 /* Decode an event message with required JSON payload.  These functions use
  * jansson unpack style variable arguments for decoding the JSON object
  * payload directly.  Returns 0 on success, or -1 on failure with errno set.
  */
-int flux_event_unpack (const flux_msg_t *msg, const char **topic,
+int flux_event_unpack (const flux_msg_t *msg,
+                       const char **topic,
                        const char *fmt, ...);
 
-/* Encode an event message with optinal string payload.
+/* Encode an event message with optional string payload.
  * If s is non-NULL, it is copied to the message payload.
  * Returns message or NULL on failure with errno set.
  */
@@ -52,7 +62,8 @@ flux_msg_t *flux_event_pack (const char *topic, const char *fmt, ...);
 /* Encode an event message with raw payload.
  */
 flux_msg_t *flux_event_encode_raw (const char *topic,
-                                   const void *data, int len);
+                                   const void *data,
+                                   size_t len);
 
 /* Decode an event message, with optional raw payload.
  * If topic is non-NULL, assign the event topic string.
@@ -60,28 +71,35 @@ flux_msg_t *flux_event_encode_raw (const char *topic,
  * If there is no payload, they will be assigned NULL and zero.
  * Returns 0 on success, or -1 on failure with errno set.
  */
-int flux_event_decode_raw (const flux_msg_t *msg, const char **topic,
-                           const void **data, int *len);
+int flux_event_decode_raw (const flux_msg_t *msg,
+                           const char **topic,
+                           const void **data,
+                           size_t *len);
 
 /* Publish an event with optional string payload.
  * The future is fulfilled once the event has been assigned a sequence number,
  * and does not indicate that the event has yet reached all subscribers.
  */
 flux_future_t *flux_event_publish (flux_t *h,
-                                   const char *topic, int flags,
+                                   const char *topic,
+                                   int flags,
                                    const char *s);
 
 /* Publish an event with JSON payload.
  */
 flux_future_t *flux_event_publish_pack (flux_t *h,
-                                        const char *topic, int flags,
-                                        const char *fmt, ...);
+                                        const char *topic,
+                                        int flags,
+                                        const char *fmt,
+                                        ...);
 
-/* Publish an event with optinal raw paylaod.
+/* Publish an event with optional raw payload.
  */
 flux_future_t *flux_event_publish_raw (flux_t *h,
-                                       const char *topic, int flags,
-                                       const void *data, int len);
+                                       const char *topic,
+                                       int flags,
+                                       const void *data,
+                                       size_t len);
 
 /* Obtain the event sequence number from the fulfilled
  * flux_event_publish() future.
diff --git a/src/common/libflux/flog.c b/src/common/libflux/flog.c
index e81dfc543984..95a6f91bbbdf 100644
--- a/src/common/libflux/flog.c
+++ b/src/common/libflux/flog.c
@@ -20,13 +20,7 @@
 #include <unistd.h>
 #include <assert.h>
 #include <inttypes.h>
-#include <zmq.h>
-
-#include "flog.h"
-#include "attr.h"
-#include "message.h"
-#include "request.h"
-#include "rpc.h"
+#include <flux/core.h>
 
 #include "src/common/libutil/wallclock.h"
 #include "src/common/libutil/stdlog.h"
@@ -34,7 +28,7 @@
 typedef struct {
     char appname[STDLOG_MAX_APPNAME + 1];
     char procid[STDLOG_MAX_PROCID + 1];
-    char buf[FLUX_MAX_LOGBUF + 1];
+    char buf[FLUX_MAX_LOGBUF];
     flux_log_f cb;
     void *cb_arg;
 } logctx_t;
@@ -99,27 +93,22 @@ void flux_log_set_redirect (flux_t *h, flux_log_f fun, void *arg)
     }
 }
 
+
 const char *flux_strerror (int errnum)
 {
-    /* zeromq-4.2.1 reports EHOSTUNREACH as "Host unreachable",
-     * but "No route to host" is canonical on Linux and we have some
-     * tests that depend on it, so remap here.
-     */
-    if (errnum == EHOSTUNREACH)
-        return "No route to host";
-    return zmq_strerror (errnum);
+    return strerror (errnum);
 }
 
-static int log_rpc (flux_t *h, const char *buf, int len)
+static int log_rpc (flux_t *h, const char *buf, size_t len)
 {
     flux_msg_t *msg;
-    int rc;
 
-    if (!(msg = flux_request_encode_raw ("log.append", buf, len)))
+    if (!(msg = flux_request_encode_raw ("log.append", buf, len))
+        || flux_send_new (h, &msg, 0) < 0) {
+        flux_msg_destroy (msg);
         return -1;
-    rc = flux_send (h, msg, 0);
-    flux_msg_destroy (msg);
-    return rc;
+    }
+    return 0;
 }
 
 int flux_vlog (flux_t *h, int level, const char *fmt, va_list ap)
@@ -127,14 +116,15 @@ int flux_vlog (flux_t *h, int level, const char *fmt, va_list ap)
     logctx_t *ctx;
     int saved_errno = errno;
     uint32_t rank;
-    int len;
+    int n;
+    size_t len;
     char timestamp[WALLCLOCK_MAXLEN];
     char hostname[STDLOG_MAX_HOSTNAME + 1];
     struct stdlog_header hdr;
     char *xtra = NULL;
 
     if (!h) {
-        char buf[FLUX_MAX_LOGBUF + 1];
+        char buf[FLUX_MAX_LOGBUF];
         const char *lstr = stdlog_severity_to_string (LOG_PRI (level));
 
         (void)vsnprintf (buf, sizeof (buf), fmt, ap);
@@ -159,10 +149,13 @@ int flux_vlog (flux_t *h, int level, const char *fmt, va_list ap)
     hdr.appname = ctx->appname;
     hdr.procid = ctx->procid;
 
-    len = stdlog_vencodef (ctx->buf, sizeof (ctx->buf), &hdr,
-                           STDLOG_NILVALUE, fmt, ap);
-    if (len >= sizeof (ctx->buf))
-        len = sizeof (ctx->buf);
+    n = stdlog_vencodef (ctx->buf,
+                         sizeof (ctx->buf),
+                         &hdr,
+                         STDLOG_NILVALUE,
+                         fmt,
+                         ap);
+    len = n >= sizeof (ctx->buf) ? sizeof (ctx->buf) : n;
     /* If log message contains multiple lines, log the first
      * line and save the rest.
      */
@@ -182,7 +175,6 @@ int flux_vlog (flux_t *h, int level, const char *fmt, va_list ap)
     return 0;
 fatal:
     free (xtra);
-    FLUX_FATAL (h);
     return -1;
 }
 
@@ -200,7 +192,7 @@ int flux_log (flux_t *h, int lev, const char *fmt, ...)
 void flux_log_verror (flux_t *h, const char *fmt, va_list ap)
 {
     int saved_errno = errno;
-    char buf[FLUX_MAX_LOGBUF + 1];
+    char buf[FLUX_MAX_LOGBUF];
 
     (void)vsnprintf (buf, sizeof (buf), fmt, ap);
     flux_log (h, LOG_ERR, "%s: %s", buf, flux_strerror (saved_errno));
@@ -216,98 +208,20 @@ void flux_log_error (flux_t *h, const char *fmt, ...)
     va_end (ap);
 }
 
-static int dmesg_clear (flux_t *h, int seq)
+void flux_llog (void *arg,
+                const char *file,
+                int line,
+                const char *func,
+                const char *subsys,
+                int level,
+                const char *fmt,
+                va_list ap)
 {
-    flux_future_t *f;
-    int rc = -1;
-
-    if (!(f = flux_rpc_pack (h, "log.clear", FLUX_NODEID_ANY, 0,
-                             "{s:i}", "seq", seq)))
-        goto done;
-    if (flux_future_get (f, NULL) < 0)
-        goto done;
-    rc = 0;
-done:
-    flux_future_destroy (f);
-    return rc;
-}
-
-static flux_future_t *dmesg_rpc (flux_t *h, int seq, bool follow)
-{
-    return flux_rpc_pack (h, "log.dmesg", FLUX_NODEID_ANY, 0,
-                          "{s:i s:b}", "seq", seq, "follow", follow);
+    flux_t *h = arg;
+    // ignoring subsys, file, line
+    flux_vlog (h, level, fmt, ap);
 }
 
-static int dmesg_rpc_get (flux_future_t *f, int *seq, flux_log_f fun, void *arg)
-{
-    const char *buf;
-    int rc = -1;
-
-    if (flux_rpc_get_unpack (f, "{s:i s:s}", "seq", seq, "buf", &buf) < 0)
-        goto done;
-    fun (buf, strlen (buf), arg);
-    rc = 0;
-done:
-    return rc;
-}
-
-int flux_dmesg (flux_t *h, int flags, flux_log_f fun, void *arg)
-{
-    int rc = -1;
-    int seq = -1;
-    bool eof = false;
-    bool follow = false;
-
-    if (flags & FLUX_DMESG_FOLLOW)
-        follow = true;
-    if (fun) {
-        while (!eof) {
-            flux_future_t *f;
-            if (!(f = dmesg_rpc (h, seq, follow)))
-                goto done;
-            if (dmesg_rpc_get (f, &seq, fun, arg) < 0) {
-                if (errno != ENOENT) {
-                    flux_future_destroy (f);
-                    goto done;
-                }
-                eof = true;
-            }
-            flux_future_destroy (f);
-        }
-    }
-    if ((flags & FLUX_DMESG_CLEAR)) {
-        if (dmesg_clear (h, seq) < 0)
-            goto done;
-    }
-    rc = 0;
-done:
-    return rc;
-}
-
-void flux_log_fprint (const char *buf, int len, void *arg)
-{
-    FILE *f = arg;
-    struct stdlog_header hdr;
-    const char *msg;
-    int msglen, severity;
-    uint32_t nodeid;
-
-    if (f) {
-        if (stdlog_decode (buf, len, &hdr, NULL, NULL, &msg, &msglen) < 0)
-            fprintf (f, "%.*s\n", len, buf);
-        else {
-            nodeid = strtoul (hdr.hostname, NULL, 10);
-            severity = STDLOG_SEVERITY (hdr.pri);
-            fprintf (f, "%s %s.%s[%" PRIu32 "]: %.*s\n",
-                     hdr.timestamp,
-                     hdr.appname,
-                     stdlog_severity_to_string (severity),
-                     nodeid,
-                     msglen, msg);
-        }
-        fflush (f);
-    }
-}
 
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/common/libflux/flog.h b/src/common/libflux/flog.h
index 93881465eabb..4399fb8ba2df 100644
--- a/src/common/libflux/flog.h
+++ b/src/common/libflux/flog.h
@@ -16,8 +16,6 @@
 #include <stdlib.h>
 #include <errno.h>
 
-#include "handle.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -46,8 +44,7 @@ int flux_log (flux_t *h, int level, const char *fmt, ...)
               __attribute__ ((format (printf, 3, 4)));
 
 /* Log a message at LOG_ERR level, appending a colon, space, and error string.
- * The system 'errno' is assumed to be valid and contain an error code
- * that can be decoded with zmq_strerror(3).
+ * The system 'errno' is assumed to be valid.
  *
  * Flux handle is optional, if set to NULL output to stderr.
  */
@@ -62,26 +59,22 @@ void flux_log_error (flux_t *h, const char *fmt, ...)
  */
 void flux_log_set_redirect (flux_t *h, flux_log_f fun, void *arg);
 
-
-/* Manipulate the broker's ring buffer.
- */
-enum {
-    FLUX_DMESG_CLEAR = 1,
-    FLUX_DMESG_FOLLOW = 2,
-};
-
-int flux_dmesg (flux_t *h, int flags, flux_log_f fun, void *arg);
-
-/* flux_log_f callback that prints a log message to a FILE stream
- * passed in as 'arg'.
- */
-void flux_log_fprint (const char *buf, int len, void *arg);
-
 /* Convert errno to string.
  * Flux errno space includes POSIX errno + zeromq errors.
  */
 const char *flux_strerror (int errnum);
 
+/* Flux log function compatible with libutil llog interface
+ */
+void flux_llog (void *arg,
+                const char *file,
+                int line,
+                const char *func,
+                const char *subsys,
+                int level,
+                const char *fmt,
+                va_list ap);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/common/libflux/flux.h b/src/common/libflux/flux.h
index 067ea9aa94a7..eb5ab5d9e8fd 100644
--- a/src/common/libflux/flux.h
+++ b/src/common/libflux/flux.h
@@ -12,29 +12,29 @@
 #define _FLUX_CORE_FLUX_H
 
 #include "types.h"
+#include "message.h"
 #include "handle.h"
 #include "reactor.h"
 #include "msg_handler.h"
 #include "connector.h"
-#include "message.h"
+#include "msglist.h"
 #include "request.h"
 #include "response.h"
-#include "keepalive.h"
+#include "control.h"
 #include "rpc.h"
-#include "panic.h"
 #include "event.h"
 #include "module.h"
 #include "attr.h"
 #include "flog.h"
 #include "conf.h"
-#include "heartbeat.h"
-#include "content.h"
 #include "future.h"
 #include "barrier.h"
-#include "buffer.h"
 #include "service.h"
 #include "version.h"
 #include "plugin.h"
+#include "sync.h"
+#include "disconnect.h"
+#include "stats.h"
 
 #endif /* !_FLUX_CORE_FLUX_H */
 
diff --git a/src/common/libflux/fripp.c b/src/common/libflux/fripp.c
new file mode 100644
index 000000000000..d0ad414e1643
--- /dev/null
+++ b/src/common/libflux/fripp.c
@@ -0,0 +1,507 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <stdbool.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <syslog.h>
+#include <time.h>
+#include <unistd.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/iterators.h"
+
+#define FRIPP_MAX_PACKET_LEN 1440
+#define INTERNAL_BUFFSIZE 128
+#define DEFAULT_AGG_PERIOD 1.0
+
+typedef enum {
+    BRUBECK_COUNTER,
+    BRUBECK_GAUGE,
+    BRUBECK_TIMER
+} metric_type;
+
+union val {
+    ssize_t l;
+    double d;
+};
+
+struct metric {
+    union val cur;
+    union val prev;
+    bool inc;
+    metric_type type;
+};
+
+struct fripp_ctx {
+    struct addrinfo *addrinfo;
+    int sock;
+    int buf_len;
+    int tail;
+    char *buf;
+    char prefix[INTERNAL_BUFFSIZE];
+
+    zhashx_t *metrics;
+    zlist_t *done;
+    flux_watcher_t *w;
+    double period;
+
+    bool enabled;
+};
+
+bool fripp_enabled (struct fripp_ctx *ctx)
+{
+    if (ctx && ctx->enabled)
+        return true;
+    return false;
+}
+
+void fripp_set_prefix (struct fripp_ctx *ctx, const char *prefix)
+{
+    strncpy (ctx->prefix, prefix, INTERNAL_BUFFSIZE - 1);
+}
+
+static int split_packet (char *packet, int len)
+{
+    for (int i = len - 1; i >= 0; --i) {
+        if (packet[i] == '\n' || packet[i] == '\0') {
+            packet[i] = '\0';
+            return i;
+        }
+    }
+
+    return -1;
+}
+
+static int fripp_send_metrics (struct fripp_ctx *ctx)
+{
+    int len;
+    bool split;
+    char *packet = ctx->buf;
+
+    do {
+        split = false;
+        len = strlen (packet);
+
+        if (len > FRIPP_MAX_PACKET_LEN)
+            split = true;
+        if ((len = split_packet (packet, len > FRIPP_MAX_PACKET_LEN ?
+                FRIPP_MAX_PACKET_LEN : len)) == -1)
+            return -1;
+        (void)sendto (ctx->sock,
+                      packet,
+                      len,
+                      0,
+                      ctx->addrinfo->ai_addr,
+                      ctx->addrinfo->ai_addrlen);
+    } while (split && (packet = &packet[len + 1]));
+
+    return 0;
+}
+
+int fripp_packet_appendf (struct fripp_ctx *ctx, const char *fmt, ...)
+{
+    va_list ap, cpy;
+    char buf[INTERNAL_BUFFSIZE];
+    int len, rc = 0;
+
+    if (!fripp_enabled (ctx))
+        return 0;
+
+    va_start (ap, fmt);
+    va_copy (cpy, ap);
+
+    if ((len = vsnprintf (buf, INTERNAL_BUFFSIZE, fmt, ap))
+            >= INTERNAL_BUFFSIZE) {
+        rc = -1;
+        goto done;
+    }
+
+    ctx->tail += len;
+
+    if (ctx->tail >= ctx->buf_len) {
+        char *tmp;
+        if (!(tmp = realloc(ctx->buf, (ctx->tail + 1) * sizeof (char)))) {
+            rc = -1;
+            goto done;
+        }
+
+        ctx->buf_len = ctx->tail + 1;
+        ctx->buf = tmp;
+    }
+
+    strcat (ctx->buf, buf);
+
+done:
+    va_end (ap);
+    va_end (cpy);
+
+    return rc;
+}
+
+int fripp_sendf (struct fripp_ctx *ctx, const char *fmt, ...)
+{
+    va_list ap, cpy;
+
+    if (!fripp_enabled (ctx))
+        return 0;
+
+    va_start (ap, fmt);
+    va_copy (cpy, ap);
+
+    int len, rc = 0;
+
+    if ((len = vsnprintf (ctx->buf, ctx->buf_len, fmt, ap))
+         >= ctx->buf_len) {
+
+        char *tmp;
+        if (!(tmp = realloc (ctx->buf, (len + 1) * sizeof (char)))) {
+            rc = -1;
+            goto done;
+        }
+
+        ctx->buf = tmp;
+        ctx->buf_len = len + 1;
+        (void) vsnprintf (ctx->buf, ctx->buf_len, fmt, cpy);
+    }
+
+    rc = fripp_send_metrics (ctx);
+
+done:
+    va_end (ap);
+    va_end (cpy);
+
+    return rc;
+}
+
+int fripp_count (struct fripp_ctx *ctx,
+                 const char *name,
+                 ssize_t count)
+{
+    if (!fripp_enabled (ctx))
+        return 0;
+
+    if (ctx->period == 0)
+        return fripp_sendf (ctx, "%s.%s:%zd|C\n", ctx->prefix, name, count);
+
+    struct metric *m;
+
+    if (!(m = zhashx_lookup (ctx->metrics, name))) {
+        if (!(m = malloc (sizeof (*m))))
+            return -1;
+
+        zhashx_insert (ctx->metrics, name, (void *) m);
+        m->prev.l = -1;
+    }
+
+    m->type = BRUBECK_COUNTER;
+    m->inc = false;
+    m->cur.l = count;
+
+    flux_watcher_start (ctx->w);
+
+    return 0;
+}
+
+int fripp_gauge (struct fripp_ctx *ctx,
+                 const char *name,
+                 ssize_t value,
+                 bool inc)
+{
+    if (!fripp_enabled (ctx))
+        return 0;
+
+    if (ctx->period == 0)
+        return fripp_sendf (ctx,
+                            "%s.%s:%s%zd|g\n",
+                            ctx->prefix,
+                            name,
+                            inc && value > 0 ? "+" : "",
+                            value);
+
+    struct metric *m;
+
+    if (!(m = zhashx_lookup (ctx->metrics, name))) {
+        if (!(m = calloc (1, sizeof (*m))))
+            return -1;
+
+        zhashx_insert (ctx->metrics, name, (void *) m);
+        m->prev.l = -1;
+    }
+
+    m->type = BRUBECK_GAUGE;
+    m->inc = inc;
+    m->cur.l = inc ? m->cur.l + value : value;
+
+    flux_watcher_start (ctx->w);
+
+    return 0;
+}
+
+int fripp_timing (struct fripp_ctx *ctx,
+                  const char *name,
+                  double ms)
+{
+    if (!fripp_enabled (ctx))
+        return 0;
+
+    if (ctx->period == 0)
+        return fripp_sendf (ctx, "%s.%s:%lf|ms\n", ctx->prefix, name, ms);
+
+    struct metric *m;
+
+    if (!(m = zhashx_lookup (ctx->metrics, name))) {
+        if (!(m = malloc (sizeof (*m))))
+            return -1;
+
+        zhashx_insert (ctx->metrics, name, (void *) m);
+        m->prev.d = -1.0;
+    }
+
+    m->type = BRUBECK_TIMER;
+    m->inc = false;
+    m->cur.d = ms;
+
+    flux_watcher_start (ctx->w);
+
+    return 0;
+}
+
+static void metric_destroy (void **item)
+{
+    if (item) {
+        free (*item);
+        *item = NULL;
+    }
+}
+
+static void timer_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct fripp_ctx *ctx = arg;
+    const char *name;
+    struct metric *m;
+
+    if (!fripp_enabled (ctx)) {
+        flux_watcher_stop (w);
+        return;
+    }
+
+    int rc = 0;
+
+    if (!ctx->buf[0] && zhashx_size (ctx->metrics) == 0) {
+        flux_watcher_stop (ctx->w);
+        return;
+    }
+
+    FOREACH_ZHASHX (ctx->metrics, name, m) {
+        if ((m->type == BRUBECK_COUNTER || m->type == BRUBECK_GAUGE)
+            && m->cur.l == m->prev.l) {
+            zlist_append (ctx->done, (void *) name);
+            continue;
+        }
+        if (m->type == BRUBECK_TIMER && m->cur.d == m->prev.d) {
+            zlist_append (ctx->done, (void *) name);
+            continue;
+        }
+
+        m->prev.l = m->cur.l;
+
+        switch (m->type) {
+            case BRUBECK_COUNTER:
+                rc = fripp_packet_appendf (ctx,
+                                           "%s.%s:%zd|C\n",
+                                           ctx->prefix,
+                                           name,
+                                           m->cur.l);
+                break;
+            case BRUBECK_GAUGE:
+                rc = fripp_packet_appendf (ctx,
+                                           "%s.%s:%zd|g\n",
+                                           ctx->prefix,
+                                           name,
+                                           m->cur.l);
+                break;
+            case BRUBECK_TIMER:
+                rc = fripp_packet_appendf (ctx,
+                                           "%s.%s:%lf|ms\n",
+                                           ctx->prefix,
+                                           name,
+                                           m->cur.d);
+                break;
+            default:
+                break;
+        }
+
+        if (rc)
+            continue;
+    }
+
+    fripp_send_metrics (ctx);
+
+    char *s;
+    while ((s = zlist_pop (ctx->done)))
+        zhashx_delete (ctx->metrics, s);
+
+    memset (ctx->buf, 0, ctx->buf_len);
+
+    ctx->tail = 0;
+}
+
+void fripp_set_agg_period (struct fripp_ctx *ctx, double period)
+{
+    if (!fripp_enabled (ctx))
+        return;
+
+    if (period <= 0) {
+        flux_watcher_stop (ctx->w);
+        ctx->period = 0;
+        return;
+    }
+
+    ctx->period = period;
+
+    double after = flux_watcher_next_wakeup (ctx->w) - (double) time (NULL);
+
+    flux_timer_watcher_reset (ctx->w, after, ctx->period);
+}
+
+void fripp_ctx_destroy (struct fripp_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        flux_watcher_destroy (ctx->w);
+        zhashx_destroy (&ctx->metrics);
+        zlist_destroy (&ctx->done);
+        if (ctx->sock != -1)
+            close (ctx->sock);
+        free (ctx->buf);
+        if (ctx->addrinfo)
+            freeaddrinfo (ctx->addrinfo);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static int parse_address (struct addrinfo **aip,
+                          const char *s,
+                          char *errbuf,
+                          int errbufsz)
+{
+    char *node;
+    char *service;
+    struct addrinfo hints, *res = NULL;
+    int e;
+
+    if (!(node = strdup (s))) {
+        snprintf (errbuf, errbufsz, "out of memory");
+        return -1;
+    }
+    if (!(service = strchr (node, ':'))) {
+        snprintf (errbuf, errbufsz, "missing colon delimiter");
+        goto error;
+    }
+    *service++ = '\0';
+
+    memset (&hints, 0, sizeof (hints));
+    hints.ai_family = AF_INET;
+    hints.ai_socktype = SOCK_DGRAM;
+    hints.ai_protocol = IPPROTO_UDP;
+
+    if ((e = getaddrinfo (node, service, &hints, &res)) != 0) {
+        snprintf (errbuf, errbufsz, "%s", gai_strerror (e));
+        goto error;
+    }
+    free (node);
+    *aip = res;
+    return 0;
+error:
+    if (res)
+        freeaddrinfo (res);
+    free (node);
+    errno = EINVAL;
+    return -1;
+}
+
+struct fripp_ctx *fripp_ctx_create (flux_t *h)
+{
+    struct fripp_ctx *ctx;
+    char *addr;
+    char errbuf[128];
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        goto error;
+    ctx->sock = -1;
+    /* If environment variable is unset, or it is set wrong, let the context
+     * be created with disabled status so we don't repeatedly try to create
+     * it "on demand" in the flux_stats_*() functions.
+     */
+    if (!(addr = getenv ("FLUX_FRIPP_STATSD")))
+        return ctx;
+    if (parse_address (&ctx->addrinfo, addr, errbuf, sizeof (errbuf)) < 0) {
+        flux_log (h, LOG_ERR, "FLUX_FRIPP_STATSD parse error: %s", errbuf);
+        return ctx;
+    }
+    if ((ctx->sock = socket (ctx->addrinfo->ai_family,
+                             ctx->addrinfo->ai_socktype,
+                             ctx->addrinfo->ai_protocol)) == -1)
+        goto error;
+    if (!(ctx->buf = calloc (1, INTERNAL_BUFFSIZE)))
+        goto error;
+
+    ctx->buf_len = INTERNAL_BUFFSIZE;
+    ctx->tail = 0;
+
+    uint32_t rank;
+    char buf[16];
+    flux_get_rank (h, &rank);
+    sprintf (buf, "flux.%d", rank);
+    fripp_set_prefix (ctx, buf);
+
+    if (!(ctx->metrics = zhashx_new ()))
+        goto nomem;
+    zhashx_set_destructor (ctx->metrics, metric_destroy);
+
+    if (!(ctx->done = zlist_new ()))
+        goto nomem;
+    if (!(ctx->w = flux_timer_watcher_create (
+              flux_get_reactor(h),
+              DEFAULT_AGG_PERIOD,
+              DEFAULT_AGG_PERIOD,
+              timer_cb,
+              ctx)))
+        goto error;
+
+    ctx->period = DEFAULT_AGG_PERIOD;
+    flux_watcher_start (ctx->w);
+
+    ctx->enabled = true;
+
+    return ctx;
+nomem:
+    errno = ENOMEM;
+error:
+    fripp_ctx_destroy (ctx);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/fripp.h b/src/common/libflux/fripp.h
new file mode 100644
index 000000000000..29afa999bd3e
--- /dev/null
+++ b/src/common/libflux/fripp.h
@@ -0,0 +1,71 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_FRIPP_H
+#define _FLUX_CORE_FRIPP_H
+
+#include <stdarg.h>
+
+#include "handle.h"
+
+struct fripp_ctx;
+
+struct fripp_ctx *fripp_ctx_create (flux_t *h);
+void fripp_ctx_destroy (struct fripp_ctx *ctx);
+
+/* Format and append a packet to the internal queue to be sent on the
+ * next flush.
+ */
+int fripp_packet_appendf (struct fripp_ctx *ctx, const char *fmt, ...)
+        __attribute__ ((format (printf, 2, 3)));
+
+/* Format and send a single packet immediately.
+ */
+int fripp_sendf (struct fripp_ctx *ctx, const char *fmt, ...)
+        __attribute__ ((format (printf, 2, 3)));
+
+/* Update (or create) and store 'count' for 'name' to be sent on the
+ * next flush.
+ */
+int fripp_count (struct fripp_ctx *ctx, const char *name, ssize_t count);
+
+/* Update (or create) and store 'value' for 'name' to be sent on the
+ * next flush. The'inc' indicates whether or not 'value' is some delta on
+ * the previous value. If 'inc' is set and 'name' was not previously stored,
+ * then the value is stored directly.
+ */
+int fripp_gauge (struct fripp_ctx *ctx, const char *name, ssize_t value, bool inc);
+
+/* Update (or create) and store 'ms' for 'name' to be sent on the
+ * next flush.
+ */
+int fripp_timing (struct fripp_ctx *ctx, const char *name, double ms);
+
+/* Update the internal aggregation period over which metrics accumulate
+ * before being set. A 'period' of '0' indicates the metrics should be
+ * sent immediately.
+ */
+void fripp_set_agg_period (struct fripp_ctx *ctx, double period);
+
+/* Check whether fripp collection is enabled.
+ */
+bool fripp_enabled (struct fripp_ctx *ctx);
+
+/* Set the prefix to be prepended to all metrics sent from the handle.
+ * The prefix has a max limit of 127 characters. The default prefix is
+ * flux.{{rank}}.
+ */
+void fripp_set_prefix (struct fripp_ctx *ctx, const char *prefix);
+
+#endif
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/future.c b/src/common/libflux/future.c
index 025d597e8cbd..effff38120e0 100644
--- a/src/common/libflux/future.c
+++ b/src/common/libflux/future.c
@@ -15,12 +15,15 @@
 #include <stdbool.h>
 #include <stdlib.h>
 #include <errno.h>
-#include <czmq.h>
+#include <assert.h>
+#include <flux/core.h>
 
-#include "src/common/libutil/aux.h"
+#ifndef EDEADLOCK
+#define EDEADLOCK EDEADLK
+#endif
 
-#include "future.h"
-#include "flog.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/aux.h"
 
 struct now_context {
     flux_t *h;              // (optional) cloned flux_t handle
@@ -33,6 +36,7 @@ struct now_context {
 struct then_context {
     flux_reactor_t *r;      // external reactor for then
     flux_watcher_t *timer;  // timer watcher (if timeout set)
+    double timeout;
     flux_watcher_t *check;
     flux_watcher_t *idle;
     bool init_called;
@@ -66,12 +70,18 @@ struct flux_future {
     int refcount;
 };
 
-static void check_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg);
-static void now_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                          int revents, void *arg);
-static void then_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                           int revents, void *arg);
+static void check_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg);
+static void now_timer_cb (flux_reactor_t *r,
+                          flux_watcher_t *w,
+                          int revents,
+                          void *arg);
+static void then_timer_cb (flux_reactor_t *r,
+                           flux_watcher_t *w,
+                           int revents,
+                           void *arg);
 
 /* "now" reactor context - used for flux_future_wait_on()
  * This is set up lazily; wait until the user calls flux_future_wait_on().
@@ -90,11 +100,10 @@ static void now_context_destroy (struct now_context *now)
 
 static struct now_context *now_context_create (void)
 {
-    struct now_context *now = calloc (1, sizeof (*now));
-    if (!now) {
-        errno = ENOMEM;
-        goto error;
-    }
+    struct now_context *now;
+
+    if (!(now = calloc (1, sizeof (*now))))
+        return NULL;
     if (!(now->r = flux_reactor_create (0)))
         goto error;
     return now;
@@ -104,15 +113,19 @@ static struct now_context *now_context_create (void)
 }
 
 static int now_context_set_timeout (struct now_context *now,
-                                    double timeout, void *arg)
+                                    double timeout,
+                                    void *arg)
 {
     if (now) {
         if (timeout < 0.)       // disable
             flux_watcher_stop (now->timer);
         else {
             if (!now->timer) {  // set
-                now->timer = flux_timer_watcher_create (now->r, timeout, 0.,
-                                                        now_timer_cb, arg);
+                now->timer = flux_timer_watcher_create (now->r,
+                                                        timeout,
+                                                        0.,
+                                                        now_timer_cb,
+                                                        arg);
                 if (!now->timer)
                     return -1;
             }
@@ -148,11 +161,10 @@ static void then_context_destroy (struct then_context *then)
 
 static struct then_context *then_context_create (flux_reactor_t *r, void *arg)
 {
-    struct then_context *then = calloc (1, sizeof (*then));
-    if (!then) {
-        errno = ENOMEM;
-        goto error;
-    }
+    struct then_context *then;
+
+    if (!(then = calloc (1, sizeof (*then))))
+        return NULL;
     then->r = r;
     if (!(then->check = flux_check_watcher_create (r, check_cb, arg)))
         goto error;
@@ -177,33 +189,29 @@ static void then_context_stop (struct then_context *then)
 }
 
 static int then_context_set_timeout (struct then_context *then,
-                                     double timeout, void *arg)
+                                     double timeout,
+                                     void *arg)
 {
     if (then) {
-        if (timeout < 0.)       // disable
+        then->timeout = timeout;
+        if (timeout < 0.)
             flux_watcher_stop (then->timer);
         else {
-            if (!then->timer) { // set
-                then->timer = flux_timer_watcher_create (then->r, timeout, 0.,
-                                                         then_timer_cb, arg);
+            if (!then->timer) {
+                then->timer = flux_timer_watcher_create (then->r,
+                                                         0.,
+                                                         timeout,
+                                                         then_timer_cb,
+                                                         arg);
                 if (!then->timer)
                     return -1;
             }
-            else {              // reset
-                flux_timer_watcher_reset (then->timer, timeout, 0.);
-            }
-            flux_watcher_start (then->timer);
+            flux_timer_watcher_again (then->timer);
         }
     }
     return 0;
 }
 
-static void then_context_clear_timer (struct then_context *then)
-{
-    if (then)
-        flux_watcher_stop (then->timer);
-}
-
 static void init_result (struct future_result *fs)
 {
     fs->is_error = false;
@@ -222,7 +230,8 @@ static void clear_result (struct future_result *fs)
     init_result (fs);
 }
 
-static void set_result_value (struct future_result *fs, void *value,
+static void set_result_value (struct future_result *fs,
+                              void *value,
                               flux_free_f value_free)
 {
     assert (fs->is_error == false);
@@ -230,7 +239,8 @@ static void set_result_value (struct future_result *fs, void *value,
     fs->value_free = value_free;
 }
 
-static int set_result_errnum (struct future_result *fs, int errnum,
+static int set_result_errnum (struct future_result *fs,
+                              int errnum,
                               const char *errstr)
 {
     assert (!fs->value && !fs->value_free);
@@ -313,20 +323,16 @@ void flux_future_destroy (flux_future_t *f)
  */
 flux_future_t *flux_future_create (flux_future_init_f cb, void *arg)
 {
-    flux_future_t *f = calloc (1, sizeof (*f));
-    if (!f) {
-        errno = ENOMEM;
-        goto error;
-    }
+    flux_future_t *f;
+
+    if (!(f = calloc (1, sizeof (*f))))
+        return NULL;
     f->init = cb;
     f->init_arg = arg;
     f->queue = NULL;
     f->embed = NULL;
     f->refcount = 1;
     return f;
-error:
-    flux_future_destroy (f);
-    return NULL;
 }
 
 void flux_future_incref (flux_future_t *f)
@@ -343,7 +349,6 @@ void flux_future_decref (flux_future_t *f)
 static void post_fulfill (flux_future_t *f)
 {
     now_context_clear_timer (f->now);
-    then_context_clear_timer (f->then);
     if (f->now)
         flux_reactor_stop (f->now->r);
     if (f->then)
@@ -357,8 +362,10 @@ void flux_future_reset (flux_future_t *f)
     if (f) {
         clear_result (&f->result);
         f->result_valid = false;
-        if (f->then)
+        if (f->then) {
             then_context_stop (f->then);
+            (void)then_context_set_timeout (f->then, f->then->timeout, f);
+        }
         if (f->queue && zlist_size (f->queue) > 0) {
             struct future_result *fs = zlist_pop (f->queue);
             move_result (&f->result, fs);
@@ -369,7 +376,6 @@ void flux_future_reset (flux_future_t *f)
     }
 }
 
-
 /* Set the flux reactor to be used for 'then' context.
  * In 'now' context, reactor will be a temporary one.
  */
@@ -454,7 +460,7 @@ static bool future_is_ready (flux_future_t *f)
 
 /* Block until future is fulfilled or timeout expires.
  * This function can be called multiple times, with different timeouts.
- * If timeout <= 0., there is no timeout.
+ * If timeout < 0., there is no timeout.
  * If timeout == 0., time out immediately if future has not been fulfilled.
  * If timeout > 0., lazily set up the "now" reactor context (first call)
  * and run that reactor until fulfilled or error.  If timer expires,
@@ -492,8 +498,20 @@ int flux_future_wait_for (flux_future_t *f, double timeout)
             flux_dispatch_requeue (f->now->h);
         f->now->running = false;
     }
-    if (!future_is_ready (f))
+    if (!future_is_ready (f)) {
+        /* This block of code is reached b/c flux_reactor_run() on the
+         * "now" reactor returned >= 0 AND the future was
+         * not fulfilled, e.g.
+         * - no init callback was registered when future was created
+         * - the init callback didn't register any watchers
+         * - the registered watchers stopped themselves or called
+         *   flux_reactor_stop() without fulfilling the future.
+         * Return EDEADLOCK indicating that the future cannot be
+         * fulfilled.
+         */
+        errno = EDEADLOCK;
         return -1;
+    }
     return 0;
 }
 
@@ -534,10 +552,11 @@ int flux_future_get (flux_future_t *f, const void **result)
 /* Set up continuation to run once future is fulfilled.
  * Lazily set up the "then" reactor context.
  * If timer expires, fulfill the future with ETIMEDOUT error.
- * This function can only be called once.
  */
-int flux_future_then (flux_future_t *f, double timeout,
-                      flux_continuation_f cb, void *arg)
+int flux_future_then (flux_future_t *f,
+                      double timeout,
+                      flux_continuation_f cb,
+                      void *arg)
 {
     if (!f || !f->r || !cb) {
         errno = EINVAL;
@@ -576,8 +595,10 @@ void *flux_future_aux_get (flux_future_t *f, const char *name)
 
 /* Store 'aux' object by name.
  */
-int flux_future_aux_set (flux_future_t *f, const char *name,
-                         void *aux, flux_free_f destroy)
+int flux_future_aux_set (flux_future_t *f,
+                         const char *name,
+                         void *aux,
+                         flux_free_f destroy)
 {
     if (!f) {
         errno = EINVAL;
@@ -634,7 +655,8 @@ void flux_future_fulfill (flux_future_t *f, void *result, flux_free_f free_fn)
     }
 }
 
-void flux_future_fulfill_error (flux_future_t *f, int errnum,
+void flux_future_fulfill_error (flux_future_t *f,
+                                int errnum,
                                 const char *errstr)
 {
     if (f) {
@@ -691,8 +713,9 @@ int flux_future_fulfill_with (flux_future_t *f, flux_future_t *p)
     if (p->fatal_errnum_valid)
         flux_future_fatal_error (f, p->fatal_errnum, p->fatal_errnum_string);
     else if (p->result.is_error)
-        flux_future_fulfill_error (f, p->result.errnum,
-                                      p->result.errnum_string);
+        flux_future_fulfill_error (f,
+                                   p->result.errnum,
+                                   p->result.errnum_string);
     else {
         /*  Nornal result, if result has a free_fn registered, then we have
          *   to steal the reference for the result. We do this by copying
@@ -766,8 +789,10 @@ const char *flux_future_error_string (flux_future_t *f)
 /* timer - for flux_future_then() timeout
  * fulfill the future with an error
  */
-static void then_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                           int revents, void *arg)
+static void then_timer_cb (flux_reactor_t *r,
+                           flux_watcher_t *w,
+                           int revents,
+                           void *arg)
 {
     flux_future_t *f = arg;
     flux_future_fulfill_error (f, ETIMEDOUT, NULL);
@@ -776,8 +801,10 @@ static void then_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
 /* timer - for flux_future_wait_for() timeout
  * stop the reactor but don't fulfill the future
  */
-static void now_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                          int revents, void *arg)
+static void now_timer_cb (flux_reactor_t *r,
+                          flux_watcher_t *w,
+                          int revents,
+                          void *arg)
 {
     errno = ETIMEDOUT;
     flux_reactor_stop_error (r);
@@ -785,8 +812,10 @@ static void now_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
 
 /* check - if results are ready, call the continuation
  */
-static void check_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
+static void check_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
 {
     flux_future_t *f = arg;
 
diff --git a/src/common/libflux/future.h b/src/common/libflux/future.h
index 4bb76794f76f..83b5413fc6a2 100644
--- a/src/common/libflux/future.h
+++ b/src/common/libflux/future.h
@@ -11,12 +11,6 @@
 #ifndef _FLUX_CORE_FUTURE_H
 #define _FLUX_CORE_FUTURE_H
 
-#include "reactor.h"
-#include "types.h"
-#include "handle.h"
-#include "msg_handler.h"
-#include "flog.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -25,12 +19,12 @@ extern "C" {
  * See flux_future_then(3).
  */
 
-typedef struct flux_future flux_future_t;
-
 typedef void (*flux_continuation_f)(flux_future_t *f, void *arg);
 
-int flux_future_then (flux_future_t *f, double timeout,
-                      flux_continuation_f cb, void *arg);
+int flux_future_then (flux_future_t *f,
+                      double timeout,
+                      flux_continuation_f cb,
+                      void *arg);
 
 int flux_future_wait_for (flux_future_t *f, double timeout);
 
@@ -41,8 +35,10 @@ void flux_future_reset (flux_future_t *f);
 void flux_future_destroy (flux_future_t *f);
 
 void *flux_future_aux_get (flux_future_t *f, const char *name);
-int flux_future_aux_set (flux_future_t *f, const char *name,
-                         void *aux, flux_free_f destroy);
+int flux_future_aux_set (flux_future_t *f,
+                         const char *name,
+                         void *aux,
+                         flux_free_f destroy);
 
 /* Functions primarily used by implementors of classes that return futures.
  * See flux_future_create(3).
@@ -55,7 +51,9 @@ flux_future_t *flux_future_create (flux_future_init_f cb, void *arg);
 int flux_future_get (flux_future_t *f, const void **result);
 
 void flux_future_fulfill (flux_future_t *f, void *result, flux_free_f free_fn);
-void flux_future_fulfill_error (flux_future_t *f, int errnum, const char *errstr);
+void flux_future_fulfill_error (flux_future_t *f,
+                                int errnum,
+                                const char *errstr);
 
 int flux_future_fulfill_with (flux_future_t *f, flux_future_t *p);
 
@@ -104,13 +102,15 @@ flux_future_t *flux_future_get_child (flux_future_t *cf, const char *name);
  *  has been used.
  */
 flux_future_t *flux_future_and_then (flux_future_t *f,
-                                     flux_continuation_f cb, void *arg);
+                                     flux_continuation_f cb,
+                                     void *arg);
 
 /* Like flux_future_and_then(3), but run the continuation `cb` when
  *  future `f` is fulfilled with an error.
  */
 flux_future_t *flux_future_or_then (flux_future_t *f,
-                                    flux_continuation_f cb, void *arg);
+                                    flux_continuation_f cb,
+                                    void *arg);
 
 /* Set the next future for the chained future `prev` to `f`.
  *  This function steals a reference to `f` and thus flux_future_destroy()
@@ -122,9 +122,16 @@ int flux_future_continue (flux_future_t *prev, flux_future_t *f);
 /*  Set the next future for the chained future `prev` to be fulfilled
  *   with an error `errnum` and an optional error string.
  */
-void flux_future_continue_error (flux_future_t *prev, int errnum,
+void flux_future_continue_error (flux_future_t *prev,
+                                 int errnum,
                                  const char *errstr);
 
+/*  Fulfill the next future in the chain immediately with a result.
+ */
+int flux_future_fulfill_next (flux_future_t *prev,
+                              void *result,
+                              flux_free_f free_fn);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/common/libflux/handle.c b/src/common/libflux/handle.c
index 089780b574f5..2d1e4175efd0 100644
--- a/src/common/libflux/handle.c
+++ b/src/common/libflux/handle.c
@@ -11,6 +11,8 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <limits.h>
+#include <unistd.h>
 #include <assert.h>
 #include <string.h>
 #include <errno.h>
@@ -18,46 +20,25 @@
 #include <dlfcn.h>
 #include <sys/epoll.h>
 #include <poll.h>
-#include <czmq.h>
-#if HAVE_CALIPER
-#include <caliper/cali.h>
-#include <sys/syscall.h>
-#endif
-
-#include "handle.h"
-#include "reactor.h"
-#include "connector.h"
-#include "message.h"
-#include "tagpool.h"
-#include "msg_handler.h" // for flux_sleep_on ()
-#include "flog.h"
-#include "conf.h"
-#include "conf_private.h"
+#include <flux/core.h>
 
+#include "src/common/libflux/plugin_private.h"
+#include "src/common/librouter/rpc_track.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
-#include "src/common/libutil/msglist.h"
 #include "src/common/libutil/dirwalk.h"
 #include "src/common/libutil/aux.h"
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/monotime.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libidset/idset.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
 
-#if HAVE_CALIPER
-struct profiling_context {
-    int initialized;
-    cali_id_t msg_type;
-    cali_id_t msg_seq;
-    cali_id_t msg_topic;
-    cali_id_t msg_sender;
-    cali_id_t msg_rpc;
-    cali_id_t msg_rpc_nodeid;
-    cali_id_t msg_rpc_resp_expected;
-    cali_id_t msg_action;
-    cali_id_t msg_match_type;
-    cali_id_t msg_match_tag;
-    cali_id_t msg_match_glob;
-};
-#endif
+#include "msg_deque.h"
+#include "message_private.h" // to check msg refcount in flux_send_new ()
 
-struct flux_handle_struct {
+struct flux_handle {
     flux_t          *parent; // if FLUX_O_CLONE, my parent
     struct aux_item *aux;
     int             usecount;
@@ -67,142 +48,82 @@ struct flux_handle_struct {
     const struct flux_handle_ops *ops;
     void            *impl;
     void            *dso;
-    msglist_t       *queue;
+    struct msg_deque *queue;
     int             pollfd;
 
-    struct tagpool  *tagpool;
+    struct idset    *tagpool;
     flux_msgcounters_t msgcounters;
-    flux_fatal_f    fatal;
-    void            *fatal_arg;
-    bool            fatality;
+    flux_comms_error_f comms_error_cb;
+    void            *comms_error_arg;
+    bool            comms_error_in_progress;
     bool            destroy_in_progress;
-#if HAVE_CALIPER
-    struct profiling_context prof;
-#endif
+    struct rpc_track *tracker;
 };
 
-static flux_t *lookup_clone_ancestor (flux_t *h)
-{
-    while ((h->flags & FLUX_O_CLONE))
-        h = h->parent;
-    return h;
-}
+struct builtin_connector {
+    const char *scheme;
+    connector_init_f *init;
+};
 
-void tagpool_grow_notify (void *arg, uint32_t old, uint32_t new);
-
-#if HAVE_CALIPER
-void profiling_context_init (struct profiling_context* prof)
-{
-    prof->msg_type = cali_create_attribute ("flux.message.type",
-                                            CALI_TYPE_STRING,
-                                            CALI_ATTR_DEFAULT | CALI_ATTR_ASVALUE);
-    prof->msg_seq = cali_create_attribute ("flux.message.seq",
-                                           CALI_TYPE_INT,
-                                           CALI_ATTR_SKIP_EVENTS);
-    prof->msg_topic = cali_create_attribute ("flux.message.topic",
-                                             CALI_TYPE_STRING,
-                                             CALI_ATTR_DEFAULT | CALI_ATTR_ASVALUE);
-    prof->msg_sender = cali_create_attribute ("flux.message.sender",
-                                              CALI_TYPE_STRING,
-                                              CALI_ATTR_SKIP_EVENTS);
-    // if flux.message.rpc is set, we're inside an RPC, it will be set to a
-    // type, single or multi
-    prof->msg_rpc = cali_create_attribute ("flux.message.rpc",
-                                           CALI_TYPE_STRING,
-                                           CALI_ATTR_SKIP_EVENTS);
-    prof->msg_rpc_nodeid = cali_create_attribute ("flux.message.rpc.nodeid",
-                                                  CALI_TYPE_INT,
-                                                  CALI_ATTR_SKIP_EVENTS);
-    prof->msg_rpc_resp_expected =
-        cali_create_attribute ("flux.message.response_expected",
-                               CALI_TYPE_INT,
-                               CALI_ATTR_SKIP_EVENTS);
-    prof->msg_action = cali_create_attribute ("flux.message.action",
-                                              CALI_TYPE_STRING,
-                                              CALI_ATTR_DEFAULT | CALI_ATTR_ASVALUE);
-    prof->msg_match_type = cali_create_attribute ("flux.message.match.type",
-                                                  CALI_TYPE_INT,
-                                                  CALI_ATTR_SKIP_EVENTS);
-    prof->msg_match_tag = cali_create_attribute ("flux.message.match.tag",
-                                                 CALI_TYPE_INT,
-                                                 CALI_ATTR_SKIP_EVENTS);
-    prof->msg_match_glob = cali_create_attribute ("flux.message.match.glob",
-                                                  CALI_TYPE_STRING,
-                                                  CALI_ATTR_SKIP_EVENTS);
-    prof->initialized=1;
-}
-
-static void profiling_msg_snapshot (flux_t *h,
-                          const flux_msg_t *msg,
-                          int flags,
-                          const char *msg_action)
-{
-    h = lookup_clone_ancestor (h);
-    cali_id_t attributes[3];
-    const void * data[3];
-    size_t size[3];
+flux_t *connector_loop_init (const char *uri, int flags, flux_error_t *errp);
+flux_t *connector_interthread_init (const char *uri,
+                                    int flags,
+                                    flux_error_t *errp);
+flux_t *connector_local_init (const char *uri, int flags, flux_error_t *errp);
 
-    // This can get called before the handle is really ready
-    if(! h->prof.initialized) return;
+static struct builtin_connector builtin_connectors[] = {
+    { "loop", &connector_loop_init },
+    { "interthread", &connector_interthread_init },
+    { "local", &connector_local_init },
+};
 
-    int len = 0;
+static void handle_trace (flux_t *h, const char *fmt, ...)
+    __attribute__ ((format (printf, 2, 3)));
 
-    if (msg_action) {
-        attributes[len] = h->prof.msg_action;
-        data[len] = msg_action;
-        size[len] = strlen(msg_action);
-        ++len;
-    }
 
-    int type;
-    flux_msg_get_type (msg, &type);
-    const char *msg_type = flux_msg_typestr (type);
-    if (msg_type) {
-        attributes[len] = h->prof.msg_type;
-        data[len] = msg_type;
-        size[len] = strlen(msg_type);
-        ++len;
-    }
-
-    const char *msg_topic;
-    if (type != FLUX_MSGTYPE_KEEPALIVE)
-        flux_msg_get_topic (msg, &msg_topic);
-    else
-        msg_topic = "NONE";
-    /* attributes[len] = h->prof.msg_topic; */
-    /* data[len] = msg_topic; */
-    /* size[len] = strlen(msg_topic); */
-    /* ++len; */
-
-    if (type == FLUX_MSGTYPE_EVENT) {
-        uint32_t seq;
-        flux_msg_get_seq (msg, &seq);
-        cali_begin_int (h->prof.msg_seq, seq);
+static int validate_flags (int flags, int allowed)
+{
+    if ((flags & allowed) != flags) {
+        errno = EINVAL;
+        return -1;
     }
-    cali_push_snapshot (CALI_SCOPE_PROCESS | CALI_SCOPE_THREAD,
-                        len /* n_entries */,
-                        attributes /* event_attributes */,
-                        data /* event_data */,
-                        size /* event_size */);
-    if (type == FLUX_MSGTYPE_EVENT)
-        cali_end (h->prof.msg_seq);
+    return 0;
 }
 
+static flux_t *lookup_clone_ancestor (flux_t *h)
+{
+    while ((h->flags & FLUX_O_CLONE))
+        h = h->parent;
+    return h;
+}
 
-#endif
+static connector_init_f *find_connector_builtin (const char *scheme)
+{
+    for (int i = 0; i < ARRAY_SIZE (builtin_connectors); i++)
+        if (streq (scheme, builtin_connectors[i].scheme))
+            return builtin_connectors[i].init;
+    return NULL;
+}
 
 static char *find_file (const char *name, const char *searchpath)
 {
     char *path;
     zlist_t *l;
-    if (!(l = dirwalk_find (searchpath, DIRWALK_REALPATH, name, 1, NULL, NULL)))
+    if (!(l = dirwalk_find (searchpath,
+                            DIRWALK_REALPATH | DIRWALK_NORECURSE,
+                            name,
+                            1,
+                            NULL,
+                            NULL)))
         return NULL;
     path = zlist_pop (l);
     zlist_destroy (&l);
     return path;
 }
 
-static connector_init_f *find_connector (const char *scheme, void **dsop)
+static connector_init_f *find_connector_dso (const char *scheme,
+                                             void **dsop,
+                                             flux_error_t *errp)
 {
     char name[PATH_MAX];
     const char *searchpath = getenv ("FLUX_CONNECTOR_PATH");
@@ -213,18 +134,22 @@ static connector_init_f *find_connector (const char *scheme, void **dsop)
     if (!searchpath)
         searchpath = flux_conf_builtin_get ("connector_path", FLUX_CONF_AUTO);
     if (snprintf (name, sizeof (name), "%s.so", scheme) >= sizeof (name)) {
+        errprintf (errp, "Connector path too long");
         errno = ENAMETOOLONG;
         return NULL;
     }
     if (!(path = find_file (name, searchpath))) {
+        errprintf (errp, "Unable to find connector name '%s'", scheme);
         errno = ENOENT;
         goto error;
     }
-    if (!(dso = dlopen (path, RTLD_LAZY | RTLD_LOCAL | FLUX_DEEPBIND))) {
+    if (!(dso = dlopen (path, RTLD_LAZY | RTLD_LOCAL | plugin_deepbind ()))) {
+        errprintf (errp, "dlopen: %s: %s", path, dlerror ());
         errno = EINVAL;
         goto error;
     }
     if (!(connector_init = dlsym (dso, "connector_init"))) {
+        errprintf (errp, "%s: missing connector_init symbol", path);
         errno = EINVAL;
         goto error_dlopen;
     }
@@ -246,25 +171,7 @@ static char *strtrim (char *s, const char *trim)
     return *s ? s : NULL;
 }
 
-/* Helper for flux_open() - parse config (best effort) and return it.
- * Return NULL if config could not be parsed.
- * If "flux_uri" is configured, set 'uri' to its value, else set it to NULL.
- */
-static flux_conf_t *parse_conf_flux_uri (const char **uri)
-{
-    flux_conf_t *conf;
-    char pat[PATH_MAX + 1];
-
-    if (conf_get_default_pattern (pat, sizeof (pat)) < 0)
-        return NULL;
-    if (!(conf = conf_parse (pat, NULL)))
-        return NULL;
-    if (flux_conf_unpack (conf, NULL, "{s:s}", "flux_uri", uri) < 0)
-        *uri = NULL;
-    return conf;
-}
-
-flux_t *flux_open (const char *uri, int flags)
+flux_t *flux_open_ex (const char *uri, int flags, flux_error_t *errp)
 {
     char *default_uri = NULL;
     char *path = NULL;
@@ -273,15 +180,26 @@ flux_t *flux_open (const char *uri, int flags)
     connector_init_f *connector_init = NULL;
     const char *s;
     flux_t *h = NULL;
-    flux_conf_t *conf = NULL;
+    const int valid_flags = FLUX_O_TRACE
+                          | FLUX_O_CLONE
+                          | FLUX_O_NONBLOCK
+                          | FLUX_O_MATCHDEBUG
+                          | FLUX_O_TEST_NOSUB
+                          | FLUX_O_RPCTRACK
+                          | FLUX_O_NOREQUEUE;
+
+    err_init (errp);
+
+    if (validate_flags (flags, valid_flags) < 0) {
+        errprintf (errp, "invalid flags specified");
+        return NULL;
+    }
 
     /* Try to get URI from (in descending precedence):
-     *   argument > environment > config file > builtin
+     *   argument > environment > builtin
      */
     if (!uri)
         uri = getenv ("FLUX_URI");
-    if (!uri)
-        conf = parse_conf_flux_uri (&uri);
     if (!uri) {
         if (asprintf (&default_uri, "local://%s/local",
                       flux_conf_builtin_get ("rundir",
@@ -296,34 +214,35 @@ flux_t *flux_open (const char *uri, int flags)
         *path = '\0';
         path = strtrim (path + 3, " \t");
     }
-    if (!(connector_init = find_connector (scheme, &dso)))
+    if (!(connector_init = find_connector_builtin (scheme))
+        && !(connector_init = find_connector_dso (scheme, &dso, errp)))
         goto error;
     if (getenv ("FLUX_HANDLE_TRACE"))
         flags |= FLUX_O_TRACE;
     if (getenv ("FLUX_HANDLE_MATCHDEBUG"))
         flags |= FLUX_O_MATCHDEBUG;
-    if (!(h = connector_init (path, flags))) {
-        ERRNO_SAFE_WRAP (dlclose, dso);
+    if (!(h = connector_init (path, flags, errp))) {
+        if (dso)
+            ERRNO_SAFE_WRAP (dlclose, dso);
         goto error;
     }
     h->dso = dso;
-#if HAVE_CALIPER
-    profiling_context_init(&h->prof);
-#endif
     if ((s = getenv ("FLUX_HANDLE_USERID"))) {
         uint32_t userid = strtoul (s, NULL, 10);
-        if (flux_opt_set (h, FLUX_OPT_TESTING_USERID, &userid,
-                                                      sizeof (userid)) < 0)
+        if (flux_opt_set (h,
+                          FLUX_OPT_TESTING_USERID,
+                          &userid,
+                          sizeof (userid)) < 0)
             goto error_handle;
     }
     if ((s = getenv ("FLUX_HANDLE_ROLEMASK"))) {
         uint32_t rolemask = strtoul (s, NULL, 0);
-        if (flux_opt_set (h, FLUX_OPT_TESTING_ROLEMASK, &rolemask,
-                                                    sizeof (rolemask)) < 0)
+        if (flux_opt_set (h,
+                          FLUX_OPT_TESTING_ROLEMASK,
+                          &rolemask,
+                          sizeof (rolemask)) < 0)
             goto error_handle;
     }
-    if (conf && handle_set_conf (h, conf) < 0)
-        goto error_handle;
     free (scheme);
     free (default_uri);
     return h;
@@ -332,64 +251,200 @@ flux_t *flux_open (const char *uri, int flags)
 error:
     ERRNO_SAFE_WRAP (free, scheme);
     ERRNO_SAFE_WRAP (free, default_uri);
-    conf_destroy (conf);
+    /*
+     *  Fill errp->text with strerror only when a more specific error has not
+     *   already been set.
+     */
+    if (errp && !strlen (errp->text))
+        errprintf (errp, "%s", strerror (errno));
     return NULL;
 }
 
+flux_t *flux_open (const char *uri, int flags)
+{
+    return flux_open_ex (uri, flags, NULL);
+}
+
 void flux_close (flux_t *h)
 {
     flux_handle_destroy (h);
 }
 
-flux_t *flux_handle_create (void *impl, const struct flux_handle_ops *ops, int flags)
+int flux_set_reactor (flux_t *h, flux_reactor_t *r)
 {
-    flux_t *h = malloc (sizeof (*h));
-    if (!h)
-        goto nomem;
-    memset (h, 0, sizeof (*h));
+    if (flux_aux_get (h, "flux::reactor")) {
+        errno = EEXIST;
+        return -1;
+    }
+    if (flux_aux_set (h, "flux::reactor", r, NULL) < 0)
+        return -1;
+    return 0;
+}
+
+flux_reactor_t *flux_get_reactor (flux_t *h)
+{
+    flux_reactor_t *r = flux_aux_get (h, "flux::reactor");
+    if (!r) {
+        if ((r = flux_reactor_create (0))) {
+            if (flux_aux_set (h,
+                              "flux::reactor",
+                              r,
+                              (flux_free_f)flux_reactor_destroy) < 0) {
+                flux_reactor_destroy (r);
+                r = NULL;
+            }
+        }
+    }
+    return r;
+}
+
+/* Create an idset that is configured as an allocator per idset_alloc(3) to
+ * be used as a matchtag allocator.  Remove FLUX_MATCHTAG_NONE from the pool.
+ */
+static struct idset *tagpool_create (void)
+{
+    struct idset *idset;
+    int flags = IDSET_FLAG_AUTOGROW
+              | IDSET_FLAG_INITFULL
+              | IDSET_FLAG_COUNT_LAZY;
+
+    if (!(idset = idset_create (0, flags))
+        || idset_clear (idset, FLUX_MATCHTAG_NONE) < 0) {
+        idset_destroy (idset);
+        return NULL;
+    }
+    return idset;
+}
+
+/* Destroy matchtag allocator.  If 'debug' is true, then print something
+ * if any tags were still allocated.
+ */
+static void tagpool_destroy (struct idset *idset, bool debug)
+{
+    if (idset) {
+        int saved_errno = errno;
+        if (debug) {
+            idset_set (idset, FLUX_MATCHTAG_NONE);
+            uint32_t count = idset_universe_size (idset) - idset_count (idset);
+            if (count > 0) {
+                fprintf (stderr,
+                         "MATCHDEBUG: pool destroy with %d allocated\n",
+                         count);
+            }
+        }
+        idset_destroy (idset);
+        errno = saved_errno;
+    }
+}
+
+flux_t *flux_handle_create (void *impl,
+                            const struct flux_handle_ops *ops,
+                            int flags)
+{
+    flux_t *h;
+
+    if (!(h = calloc (1, sizeof (*h))))
+        return NULL;
     h->usecount = 1;
     h->flags = flags;
     h->ops = ops;
     h->impl = impl;
     if (!(h->tagpool = tagpool_create ()))
-        goto nomem;
-    tagpool_set_grow_cb (h->tagpool, tagpool_grow_notify, h);
-    if (!(h->queue = msglist_create ((msglist_free_f)flux_msg_destroy)))
-        goto nomem;
+        goto error;
+    if (!(flags & FLUX_O_NOREQUEUE)) {
+        if (!(h->queue = msg_deque_create (MSG_DEQUE_SINGLE_THREAD)))
+            goto error;
+    }
+    if ((flags & FLUX_O_RPCTRACK)) {
+        /* N.B. rpc_track functions are safe to call with tracker == NULL,
+         * so skipping creation here disables tracking without further ado.
+         */
+        if (!(h->tracker = rpc_track_create (MSG_HASH_TYPE_UUID_MATCHTAG)))
+            goto error;
+    }
     h->pollfd = -1;
     return h;
-nomem:
+error:
     flux_handle_destroy (h);
-    errno = ENOMEM;
     return NULL;
 }
 
 flux_t *flux_clone (flux_t *orig)
 {
+    flux_t *h;
     if (!orig) {
         errno = EINVAL;
         return NULL;
     }
-    flux_t *h = calloc (1, sizeof (*h));
-    if (!h)
-        goto nomem;
+    if (!(h = calloc (1, sizeof (*h))))
+        return NULL;
     h->parent = flux_incref (orig);
     h->usecount = 1;
     h->flags = orig->flags | FLUX_O_CLONE;
     return h;
-nomem:
-    free (h);
-    errno = ENOMEM;
-    return NULL;
 }
 
-static void report_leaked_matchtags (struct tagpool *tp)
+static void fail_tracked_request (const flux_msg_t *msg, void *arg)
 {
-    uint32_t count = tagpool_getattr (tp, TAGPOOL_ATTR_SIZE) -
-                     tagpool_getattr (tp, TAGPOOL_ATTR_AVAIL);
-    if (count > 0)
-        fprintf (stderr,
-                 "MATCHDEBUG: pool destroy with %d allocated\n", count);
+    flux_t *h = arg;
+    flux_msg_t *rep;
+    const char *topic = "NULL";
+    struct flux_msg_cred lies = { .userid = 0, .rolemask = FLUX_ROLE_OWNER };
+
+    flux_msg_get_topic (msg, &topic);
+    if (!(rep = flux_response_derive (msg, ECONNRESET))
+        || flux_msg_set_string (rep, "RPC aborted due to broker reconnect") < 0
+        || flux_msg_set_cred (rep, lies) < 0
+        || flux_requeue (h, rep, FLUX_RQ_TAIL) < 0) {
+        handle_trace (h,
+                      "error responding to tracked rpc topic=%s: %s",
+                      topic,
+                      strerror (errno));
+    }
+    else
+        handle_trace (h, "responded to tracked rpc topic=%s", topic);
+    flux_msg_destroy (rep);
+}
+
+/* Call connector's reconnect method, if available.
+ */
+int flux_reconnect (flux_t *h)
+{
+    if (!h) {
+        errno = EINVAL;
+        return -1;
+    }
+    h = lookup_clone_ancestor (h);
+    if (!h->ops->reconnect) {
+        errno = ENOSYS;
+        return -1;
+    }
+    /* Assume that ops->pollfd() returns -1 if it is not connected.
+     * If connected, remove the soon to be disconnected fd from h->pollfd.
+     */
+    if (h->ops->pollfd) {
+        int fd = h->ops->pollfd (h->impl);
+        if (fd >= 0)
+            (void)epoll_ctl (h->pollfd, EPOLL_CTL_DEL, fd, NULL);
+    }
+    handle_trace (h, "trying to reconnect");
+    if (h->ops->reconnect (h->impl) < 0) {
+        handle_trace (h, "reconnect failed");
+        return -1;
+    }
+    /* Reconnect was successful, add new ops->pollfd() to h->pollfd.
+     */
+    if (h->ops->pollfd) {
+        struct epoll_event ev = {
+            .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLERR | EPOLLHUP,
+        };
+        if ((ev.data.fd = h->ops->pollfd (h->impl)) < 0
+            || epoll_ctl (h->pollfd, EPOLL_CTL_ADD, ev.data.fd, &ev) < 0)
+            return -1;
+    }
+    handle_trace (h, "reconnected");
+    rpc_track_purge (h->tracker, fail_tracked_request, h);
+    return 0;
 }
 
 void flux_handle_destroy (flux_t *h)
@@ -399,6 +454,7 @@ void flux_handle_destroy (flux_t *h)
         if (h->destroy_in_progress)
             return;
         h->destroy_in_progress = true;
+        rpc_track_destroy (h->tracker);
         aux_destroy (&h->aux);
         if ((h->flags & FLUX_O_CLONE)) {
             flux_handle_destroy (h->parent); // decr usecount
@@ -406,16 +462,13 @@ void flux_handle_destroy (flux_t *h)
         else {
             if (h->ops->impl_destroy)
                 h->ops->impl_destroy (h->impl);
-            if ((h->flags & FLUX_O_MATCHDEBUG))
-                report_leaked_matchtags (h->tagpool);
-            tagpool_destroy (h->tagpool);
+            tagpool_destroy (h->tagpool,
+                             (h->flags & FLUX_O_MATCHDEBUG) ? true : false);
 #ifndef __SANITIZE_ADDRESS__
             if (h->dso)
                 dlclose (h->dso);
 #endif
-            msglist_destroy (h->queue);
-            if (h->pollfd >= 0)
-                (void)close (h->pollfd);
+            msg_deque_destroy (h->queue);
         }
         free (h);
         errno = saved_errno;
@@ -436,7 +489,13 @@ void flux_decref (flux_t *h)
 
 void flux_flags_set (flux_t *h, int flags)
 {
-    h->flags |= flags;
+    /* N.B. FLUX_O_RPCTRACK not permitted here, only in flux_open() */
+    const int valid_flags = FLUX_O_TRACE
+                          | FLUX_O_CLONE
+                          | FLUX_O_NONBLOCK
+                          | FLUX_O_MATCHDEBUG
+                          | FLUX_O_TEST_NOSUB;
+    h->flags |= (flags & valid_flags);
 }
 
 void flux_flags_unset (flux_t *h, int flags)
@@ -451,6 +510,10 @@ int flux_flags_get (flux_t *h)
 
 int flux_opt_get (flux_t *h, const char *option, void *val, size_t len)
 {
+    if (!h || !option) {
+        errno = EINVAL;
+        return -1;
+    }
     h = lookup_clone_ancestor (h);
     if (!h->ops->getopt) {
         errno = EINVAL;
@@ -461,6 +524,10 @@ int flux_opt_get (flux_t *h, const char *option, void *val, size_t len)
 
 int flux_opt_set (flux_t *h, const char *option, const void *val, size_t len)
 {
+    if (!h || !option) {
+        errno = EINVAL;
+        return -1;
+    }
     h = lookup_clone_ancestor (h);
     if (!h->ops->setopt) {
         errno = EINVAL;
@@ -487,31 +554,25 @@ int flux_aux_set (flux_t *h, const char *name, void *aux, flux_free_f destroy)
     return aux_set (&h->aux, name, aux, destroy);
 }
 
-void flux_fatal_set (flux_t *h, flux_fatal_f fun, void *arg)
+void flux_comms_error_set (flux_t *h, flux_comms_error_f fun, void *arg)
 {
     h = lookup_clone_ancestor (h);
-    h->fatal = fun;
-    h->fatal_arg = arg;
-    h->fatality = false;
+    h->comms_error_cb = fun;
+    h->comms_error_arg = arg;
 }
 
-void flux_fatal_error (flux_t *h, const char *fun, const char *msg)
+static int comms_error (flux_t *h, int errnum)
 {
-    h = lookup_clone_ancestor (h);
-    if (!h->fatality) {
-        h->fatality = true;
-        if (h->fatal) {
-            char buf[256];
-            snprintf (buf, sizeof (buf), "%s: %s", fun, msg);
-            h->fatal (buf, h->fatal_arg);
-        }
-    }
-}
+    int rc = -1;
 
-bool flux_fatality (flux_t *h)
-{
     h = lookup_clone_ancestor (h);
-    return h->fatality;
+    if (h->comms_error_cb && !h->comms_error_in_progress) {
+        h->comms_error_in_progress = true;
+        errno = errnum;
+        rc = h->comms_error_cb (h, h->comms_error_arg);
+        h->comms_error_in_progress = false;
+    }
+    return rc;
 }
 
 void flux_get_msgcounters (flux_t *h, flux_msgcounters_t *mcs)
@@ -526,50 +587,63 @@ void flux_clr_msgcounters (flux_t *h)
     memset (&h->msgcounters, 0, sizeof (h->msgcounters));
 }
 
-void tagpool_grow_notify (void *arg, uint32_t old, uint32_t new)
-{
-    flux_t *h = arg;
-    flux_log (h, LOG_INFO, "tagpool expanded from %u to %u entries", old, new);
-}
-
 uint32_t flux_matchtag_alloc (flux_t *h)
 {
+    if (!h)
+        goto inval;
     h = lookup_clone_ancestor (h);
-    uint32_t tag;
-
-    tag = tagpool_alloc (h->tagpool);
-    if (tag == FLUX_MATCHTAG_NONE) {
+    if (!h->tagpool)
+        goto inval;
+    unsigned int id;
+    size_t oldsize = idset_universe_size (h->tagpool);
+    if (idset_alloc (h->tagpool, &id) < 0) {
         flux_log (h, LOG_ERR, "tagpool temporarily out of tags");
-        errno = EBUSY; /* appropriate error? */
+        return FLUX_MATCHTAG_NONE;
+    }
+    size_t newsize = idset_universe_size (h->tagpool);
+    if (newsize > oldsize) {
+        flux_log (h,
+                  LOG_INFO,
+                  "tagpool expanded from %zu to %zu entries",
+                  oldsize,
+                  newsize);
     }
-    return tag;
+    return id;
+inval:
+    errno = EINVAL;
+    return FLUX_MATCHTAG_NONE;
 }
 
-/* Free matchtag, first deleting any queued matching responses.
- */
 void flux_matchtag_free (flux_t *h, uint32_t matchtag)
 {
+    if (!h)
+        return;
     h = lookup_clone_ancestor (h);
-    struct flux_match match = {
-        .typemask = FLUX_MSGTYPE_RESPONSE,
-        .topic_glob = NULL,
-        .matchtag = matchtag,
-    };
-    flux_msg_t *msg = msglist_first (h->queue);
-    while (msg) {
-        if (flux_msg_cmp (msg, match)) {
-            msglist_remove (h->queue, msg);
-            flux_msg_destroy (msg);
-        }
-        msg = msglist_next (h->queue);
+    if (!h->tagpool)
+        return;
+    /* fast path */
+    if (!(h->flags & FLUX_O_MATCHDEBUG)) {
+        if (matchtag != FLUX_MATCHTAG_NONE)
+            idset_free (h->tagpool, matchtag);
+        return;
+    }
+    /* MATCHDEBUG path */
+    if (matchtag == FLUX_MATCHTAG_NONE)
+        fprintf (stderr, "MATCHDEBUG: invalid tag=%d", (int)matchtag);
+    else if (idset_free_check (h->tagpool, matchtag) < 0) {
+        if (errno == EEXIST)
+            fprintf (stderr, "MATCHDEBUG: double free tag=%d", (int)matchtag);
+        else
+            fprintf (stderr, "MATCHDEBUG: invalid tag=%d", (int)matchtag);
     }
-    tagpool_free (h->tagpool, matchtag);
 }
 
 uint32_t flux_matchtag_avail (flux_t *h)
 {
+    if (!h)
+        return 0;
     h = lookup_clone_ancestor (h);
-    return tagpool_getattr (h->tagpool, TAGPOOL_ATTR_AVAIL);
+    return h->tagpool ? idset_count (h->tagpool) : 0;
 }
 
 static void update_tx_stats (flux_t *h, const flux_msg_t *msg)
@@ -586,8 +660,8 @@ static void update_tx_stats (flux_t *h, const flux_msg_t *msg)
             case FLUX_MSGTYPE_EVENT:
                 h->msgcounters.event_tx++;
                 break;
-            case FLUX_MSGTYPE_KEEPALIVE:
-                h->msgcounters.keepalive_tx++;
+            case FLUX_MSGTYPE_CONTROL:
+                h->msgcounters.control_tx++;
                 break;
         }
     } else
@@ -608,34 +682,112 @@ static void update_rx_stats (flux_t *h, const flux_msg_t *msg)
             case FLUX_MSGTYPE_EVENT:
                 h->msgcounters.event_rx++;
                 break;
-        case FLUX_MSGTYPE_KEEPALIVE:
-            h->msgcounters.keepalive_rx++;
+        case FLUX_MSGTYPE_CONTROL:
+            h->msgcounters.control_rx++;
             break;
         }
     } else
         errno = 0;
 }
 
+static double handle_trace_timestamp (flux_t *h)
+{
+    struct timespec *ts;
+    const char *auxkey = "flux::trace_start";
+
+    if (!(ts = flux_aux_get (h, auxkey))) {
+        if ((ts = calloc (1, sizeof (*ts)))) {
+            monotime (ts);
+            if (flux_aux_set (h, auxkey, ts, free) < 0) {
+                free (ts);
+                ts = NULL;
+            }
+        }
+    }
+    return ts ? monotime_since (*ts)/1000 : -1;
+}
+
+static void handle_trace_message (flux_t *h, const flux_msg_t *msg)
+{
+    if ((h->flags & FLUX_O_TRACE)) {
+        flux_msg_fprint_ts (stderr, msg, handle_trace_timestamp (h));
+    }
+}
+
+static void handle_trace (flux_t *h, const char *fmt, ...)
+{
+    if ((h->flags & FLUX_O_TRACE)) {
+        va_list ap;
+        char buf[256];
+        int saved_errno = errno;
+        double timestamp = handle_trace_timestamp (h);
+
+        va_start (ap, fmt);
+        vsnprintf (buf, sizeof (buf), fmt, ap);
+        va_end (ap);
+
+        fprintf (stderr, "--------------------------------------\n");
+        if (timestamp >= 0.)
+            fprintf (stderr, "c %.5f\n", timestamp);
+        fprintf (stderr, "c %s\n", buf);
+
+        errno = saved_errno;
+    }
+}
+
 int flux_send (flux_t *h, const flux_msg_t *msg, int flags)
 {
+    if (!h || !msg || validate_flags (flags, FLUX_O_NONBLOCK) < 0) {
+        errno = EINVAL;
+        return -1;
+    }
     h = lookup_clone_ancestor (h);
     if (!h->ops->send || h->destroy_in_progress) {
         errno = ENOSYS;
-        goto fatal;
+        return -1;
     }
     flags |= h->flags;
     update_tx_stats (h, msg);
-    if (flags & FLUX_O_TRACE)
-        flux_msg_fprint (stderr, msg);
-    if (h->ops->send (h->impl, msg, flags) < 0)
-        goto fatal;
-#if HAVE_CALIPER
-    profiling_msg_snapshot(h, msg, flags, "send");
-#endif
+    handle_trace_message (h, msg);
+    while (h->ops->send (h->impl, msg, flags) < 0) {
+        if (comms_error (h, errno) < 0)
+            return -1;
+        /* retry if comms_error() returns success */
+    }
+    rpc_track_update (h->tracker, msg);
+    return 0;
+}
+
+int flux_send_new (flux_t *h, flux_msg_t **msg, int flags)
+{
+    if (!h || !msg || !*msg || (*msg)->refcount > 1) {
+        errno = EINVAL;
+        return -1;
+    }
+    h = lookup_clone_ancestor (h);
+    /* Fall back to flux_send() if there are complications to using
+     * op->send_new(), e.g. when message is still needed after send.
+     */
+    if (!h->ops->send_new || h->tracker != NULL) {
+        if (flux_send (h, *msg, flags) < 0)
+            return -1;
+        flux_msg_destroy (*msg);
+        *msg = NULL;
+        return 0;
+    }
+    if (h->destroy_in_progress) {
+        errno = ENOSYS;
+        return -1;
+    }
+    flags |= h->flags;
+    update_tx_stats (h, *msg);
+    handle_trace_message (h, *msg);
+    while (h->ops->send_new (h->impl, msg, flags) < 0) {
+        if (comms_error (h, errno) < 0)
+            return -1;
+        /* retry if comms_error() returns success */
+    }
     return 0;
-fatal:
-    FLUX_FATAL (h);
-    return -1;
 }
 
 static int defer_enqueue (zlist_t **l, flux_msg_t *msg)
@@ -663,23 +815,34 @@ static int defer_requeue (zlist_t **l, flux_t *h)
 
 static void defer_destroy (zlist_t **l)
 {
-    flux_msg_t *msg;
     if (*l) {
+        flux_msg_t *msg;
+        int saved_errno = errno;;
         while ((msg = zlist_pop (*l)))
             flux_msg_destroy (msg);
         zlist_destroy (l);
+        errno = saved_errno;
     }
 }
 
 static flux_msg_t *flux_recv_any (flux_t *h, int flags)
 {
-    flux_msg_t *msg = NULL;
-    if (msglist_count (h->queue) > 0)
-        msg = msglist_pop (h->queue);
-    else if (h->ops->recv)
-        msg = h->ops->recv (h->impl, flags);
-    else
+    flux_msg_t *msg;
+
+    if (!(h->flags & FLUX_O_NOREQUEUE)) {
+        if (!msg_deque_empty (h->queue))
+            return msg_deque_pop_front (h->queue);
+    }
+    if (!h->ops->recv) {
         errno = ENOSYS;
+        return NULL;
+    }
+    while (!(msg = h->ops->recv (h->impl, flags))) {
+        if (errno == EAGAIN
+            || errno == EWOULDBLOCK
+            || comms_error (h, errno) < 0)
+            return NULL;
+    }
     return msg;
 }
 
@@ -690,169 +853,110 @@ static flux_msg_t *flux_recv_any (flux_t *h, int flags)
  */
 flux_msg_t *flux_recv (flux_t *h, struct flux_match match, int flags)
 {
+    if (!h || validate_flags (flags, FLUX_O_NONBLOCK) < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
     h = lookup_clone_ancestor (h);
     zlist_t *l = NULL;
     flux_msg_t *msg = NULL;
-    int saved_errno;
 
     flags |= h->flags;
     do {
         if (!(msg = flux_recv_any (h, flags))) {
             if (errno != EAGAIN && errno != EWOULDBLOCK)
-                goto fatal;
+                goto error;
             if (defer_requeue (&l, h) < 0)
-                goto fatal;
+                goto error;
             defer_destroy (&l);
             errno = EWOULDBLOCK;
             return NULL;
         }
         if (!flux_msg_cmp (msg, match)) {
             if (defer_enqueue (&l, msg) < 0)
-                goto fatal;
+                goto error;
             msg = NULL;
         }
     } while (!msg);
     update_rx_stats (h, msg);
-    if ((flags & FLUX_O_TRACE))
-        flux_msg_fprint (stderr, msg);
+    handle_trace_message (h, msg);
     if (defer_requeue (&l, h) < 0)
-        goto fatal;
+        goto error;
     defer_destroy (&l);
-#if HAVE_CALIPER
-    cali_begin_int (h->prof.msg_match_type, match.typemask);
-    cali_begin_int (h->prof.msg_match_tag, match.matchtag);
-    cali_begin_string (h->prof.msg_match_glob,
-                       match.topic_glob ? match.topic_glob : "NONE");
-    char *sender = NULL;
-    flux_msg_get_route_first (msg, &sender);
-    if (sender)
-        cali_begin_string (h->prof.msg_sender, sender);
-    profiling_msg_snapshot (h, msg, flags, "recv");
-    if (sender)
-        cali_end (h->prof.msg_sender);
-    cali_end (h->prof.msg_match_type);
-    cali_end (h->prof.msg_match_tag);
-    cali_end (h->prof.msg_match_glob);
-
-    free (sender);
-#endif
+    rpc_track_update (h->tracker, msg);
     return msg;
-fatal:
-    saved_errno = errno;
-    FLUX_FATAL (h);
-    if (msg)
-        flux_msg_destroy (msg);
+error:
+    flux_msg_destroy (msg);
     defer_destroy (&l);
-    errno = saved_errno;
     return NULL;
 }
 
 /* FIXME: FLUX_O_TRACE will show these messages being received again
  * So will message counters.
  */
-static int requeue (flux_t *h, flux_msg_t *msg, int flags)
+int flux_requeue (flux_t *h, const flux_msg_t *msg, int flags)
 {
+    if (!h || !msg) {
+        errno = EINVAL;
+        return -1;
+    }
     h = lookup_clone_ancestor (h);
     int rc;
 
-    if (flags != FLUX_RQ_TAIL && flags != FLUX_RQ_HEAD) {
+    if ((h->flags & FLUX_O_NOREQUEUE)
+        || (flags != FLUX_RQ_TAIL && flags != FLUX_RQ_HEAD)) {
         errno = EINVAL;
-        goto fatal;
+        return -1;
     }
+    flux_msg_incref (msg);
     if (flags == FLUX_RQ_TAIL)
-        rc = msglist_append (h->queue, msg);
+        rc = msg_deque_push_back (h->queue, (flux_msg_t *)msg);
     else
-        rc = msglist_push (h->queue, msg);
+        rc = msg_deque_push_front (h->queue, (flux_msg_t *)msg);
     if (rc < 0)
-        goto fatal;
-    return 0;
-fatal:
-    FLUX_FATAL (h);
-    return -1;
-}
-
-int flux_requeue (flux_t *h, const flux_msg_t *msg, int flags)
-{
-    flux_msg_t *cpy = NULL;
-
-    if (!(cpy = flux_msg_copy (msg, true))) {
-        FLUX_FATAL (h);
-        return -1;
-    }
-
-    if (requeue (h, cpy, flags) < 0) {
-        flux_msg_destroy (cpy);
-        return -1;
-    }
-
-    return 0;
+        flux_msg_decref (msg);
+    return rc;
 }
 
-int flux_requeue_nocopy (flux_t *h, flux_msg_t *msg, int flags)
-{
-    if (requeue (h, msg, flags) < 0)
-        return -1;
-
-    return 0;
-}
-
-int flux_event_subscribe (flux_t *h, const char *topic)
+int flux_pollfd (flux_t *h)
 {
     h = lookup_clone_ancestor (h);
-    if (h->ops->event_subscribe) {
-        if (h->ops->event_subscribe (h->impl, topic) < 0)
-            goto fatal;
-    }
-    return 0;
-fatal:
-    FLUX_FATAL (h);
-    return -1;
-}
 
-int flux_event_unsubscribe (flux_t *h, const char *topic)
-{
-    h = lookup_clone_ancestor (h);
-    if (h->ops->event_unsubscribe) {
-        if (h->ops->event_unsubscribe (h->impl, topic) < 0)
-            goto fatal;
+    if ((h->flags & FLUX_O_NOREQUEUE)) {
+        if (!h->ops->pollfd) {
+            errno = ENOTSUP;
+            return -1;
+        }
+        return h->ops->pollfd (h->impl);
     }
-    return 0;
-fatal:
-    FLUX_FATAL (h);
-    return -1;
-}
 
-int flux_pollfd (flux_t *h)
-{
-    h = lookup_clone_ancestor (h);
     if (h->pollfd < 0) {
         struct epoll_event ev = {
             .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLERR | EPOLLHUP,
         };
         if ((h->pollfd = epoll_create1 (EPOLL_CLOEXEC)) < 0)
-            goto fatal;
+            goto error;
         /* add queue pollfd */
-        ev.data.fd = msglist_pollfd (h->queue);
+        ev.data.fd = msg_deque_pollfd (h->queue);
         if (ev.data.fd < 0)
-            goto fatal;
+            goto error;
         if (epoll_ctl (h->pollfd, EPOLL_CTL_ADD, ev.data.fd, &ev) < 0)
-            goto fatal;
+            goto error;
         /* add connector pollfd (if defined) */
         if (h->ops->pollfd) {
             ev.data.fd = h->ops->pollfd (h->impl);
             if (ev.data.fd < 0)
-                goto fatal;
+                goto error;
             if (epoll_ctl (h->pollfd, EPOLL_CTL_ADD, ev.data.fd, &ev) < 0)
-                goto fatal;
+                goto error;
         }
     }
     return h->pollfd;
-fatal:
+error:
     if (h->pollfd >= 0) {
         (void)close (h->pollfd);
         h->pollfd = -1;
     }
-    FLUX_FATAL (h);
     return -1;
 }
 
@@ -861,29 +965,34 @@ int flux_pollevents (flux_t *h)
     h = lookup_clone_ancestor (h);
     int e, events = 0;
 
-    /* wait for handle event */
-    if (h->pollfd >= 0) {
-        struct epoll_event ev;
-        (void)epoll_wait (h->pollfd, &ev, 1, 0);
+    if (!(h->flags & FLUX_O_NOREQUEUE)) {
+        /* wait for handle event */
+        if (h->pollfd >= 0) {
+            struct epoll_event ev;
+            (void)epoll_wait (h->pollfd, &ev, 1, 0);
+        }
     }
     /* get connector events (if applicable) */
     if (h->ops->pollevents) {
         if ((events = h->ops->pollevents (h->impl)) < 0)
-            goto fatal;
-    }
-    /* get queue events */
-    if ((e = msglist_pollevents (h->queue)) < 0)
-        goto fatal;
-    if ((e & POLLIN))
-        events |= FLUX_POLLIN;
-    if ((e & POLLOUT))
-        events |= FLUX_POLLOUT;
-    if ((e & POLLERR))
-        events |= FLUX_POLLERR;
+            return -1;
+        if ((events & FLUX_POLLERR)) {
+            if (comms_error (h, ECONNRESET) == 0)
+                events &= ~FLUX_POLLERR;
+        }
+    }
+    if (!(h->flags & FLUX_O_NOREQUEUE)) {
+        /* get queue events */
+        if ((e = msg_deque_pollevents (h->queue)) < 0)
+            return -1;
+        if ((e & POLLIN))
+            events |= FLUX_POLLIN;
+        if ((e & POLLOUT))
+            events |= FLUX_POLLOUT;
+        if ((e & POLLERR))
+            events |= FLUX_POLLERR;
+    }
     return events;
-fatal:
-    FLUX_FATAL (h);
-    return -1;
 }
 
 /*
diff --git a/src/common/libflux/handle.h b/src/common/libflux/handle.h
index 6fc925051d3c..9318d4ba78d4 100644
--- a/src/common/libflux/handle.h
+++ b/src/common/libflux/handle.h
@@ -15,15 +15,10 @@
 #include <stdbool.h>
 #include <string.h>
 
-#include "types.h"
-#include "message.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct flux_handle_struct flux_t;
-
 typedef struct {
     int request_tx;
     int request_rx;
@@ -31,19 +26,22 @@ typedef struct {
     int response_rx;
     int event_tx;
     int event_rx;
-    int keepalive_tx;
-    int keepalive_rx;
+    int control_tx;
+    int control_rx;
 } flux_msgcounters_t;
 
-typedef void (*flux_fatal_f)(const char *msg, void *arg);
+typedef int (*flux_comms_error_f)(flux_t *h, void *arg);
 
 /* Flags for handle creation and flux_flags_set()/flux_flags_unset.
  */
 enum {
-    FLUX_O_TRACE = 1,   /* send message trace to stderr */
-    FLUX_O_CLONE = 2,   /* handle was created with flux_clone() */
-    FLUX_O_NONBLOCK = 4,/* handle should not block on send/recv */
-    FLUX_O_MATCHDEBUG = 8,/* enable matchtag debugging */
+    FLUX_O_TRACE = 1,       /* send message trace to stderr */
+    FLUX_O_CLONE = 2,       /* handle was created with flux_clone() */
+    FLUX_O_NONBLOCK = 4,    /* handle should not block on send/recv */
+    FLUX_O_MATCHDEBUG = 8,  /* enable matchtag debugging */
+    FLUX_O_TEST_NOSUB = 16, /* for testing: make (un)subscribe a no-op */
+    FLUX_O_RPCTRACK = 32,   /* track RPCs for recovery after reconnect */
+    FLUX_O_NOREQUEUE = 64,  /* disable flux_requeue() to save file descr. */
 };
 
 /* Flags for flux_requeue().
@@ -61,11 +59,14 @@ enum {
     FLUX_POLLERR = 4,
 };
 
-/* Options for flux_setopt().
+/* Options for flux_opt_set().
  * (Connectors may define custom option names)
  */
 #define FLUX_OPT_TESTING_USERID     "flux::testing_userid"
 #define FLUX_OPT_TESTING_ROLEMASK   "flux::testing_rolemask"
+#define FLUX_OPT_ROUTER_NAME        "flux::router_name"
+#define FLUX_OPT_SEND_QUEUE_COUNT   "flux::send_queue_count"
+#define FLUX_OPT_RECV_QUEUE_COUNT   "flux::recv_queue_count"
 
 /* Create/destroy a broker handle.
  * The 'uri' scheme name selects a connector to dynamically load.
@@ -74,6 +75,13 @@ enum {
  * in the environment variable FLUX_URI.
  */
 flux_t *flux_open (const char *uri, int flags);
+
+/* Like flux_open(), but if optional flux_error_t parameter is non-NULL,
+ * then any errors normally emitted to stderr will instead be returned
+ * in error->text.
+ */
+flux_t *flux_open_ex (const char *uri, int flags, flux_error_t *error);
+
 void flux_close (flux_t *h);
 
 /* Increment internal reference count on 'h'.
@@ -87,34 +95,35 @@ void flux_decref(flux_t *h);
  */
 flux_t *flux_clone (flux_t *orig);
 
-/* Get/set handle options.  Options are interpreted by connectors.
- * Returns 0 on success, or -1 on failure with errno set (e.g. EINVAL).
+/* Get/set the reactor associated with the handle.
+ * A reactor is created with flags=0 if 'get' is called before one is set.
+ * 'set' fails if there is already a reactor.
  */
-int flux_opt_set (flux_t *h, const char *option, const void *val, size_t len);
-int flux_opt_get (flux_t *h, const char *option, void *val, size_t len);
+flux_reactor_t *flux_get_reactor (flux_t *h);
+int flux_set_reactor (flux_t *h, flux_reactor_t *r);
 
-/* Register a handler for fatal handle errors.
- * A fatal error is ENOMEM or a handle send/recv error after which
- * it is inadvisable to continue using the handle.
+/* Drop connection to broker and re-establish, if supported by connector.
  */
-void flux_fatal_set (flux_t *h, flux_fatal_f fun, void *arg);
+int flux_reconnect (flux_t *h);
 
-/* Set the fatality bit and call the user's fatal error handler, if any.
- * The fatal error handler will only be called once.
+/* Get/set handle options.  Options are interpreted by connectors.
+ * Returns 0 on success, or -1 on failure with errno set (e.g. EINVAL).
  */
-void flux_fatal_error (flux_t *h, const char *fun, const char *msg);
-#define FLUX_FATAL(h) do { \
-    flux_fatal_error((h),__FUNCTION__,(strerror (errno))); \
-} while (0)
+int flux_opt_set (flux_t *h, const char *option, const void *val, size_t len);
+int flux_opt_get (flux_t *h, const char *option, void *val, size_t len);
 
-/* Return true if the handle 'h' has encountered a fatal error.
+/* Register a callback to flux_send() / flux_recv() errors.
+ * The callback return value becomes the send/receive function return value.
  */
-bool flux_fatality (flux_t *h);
+void flux_comms_error_set (flux_t *h, flux_comms_error_f fun, void *arg);
 
 /* A mechanism is provide for users to attach auxiliary state to the flux_t
  * handle by name.  The destructor, if non-NULL, will be called
  * to destroy this state when the handle is destroyed.
  * Key names used internally by flux-core are prefixed with "flux::".
+ *
+ * N.B. flux_aux_get does not scale to a large number of items, and
+ * broker module handles may persist for a long time.
  */
 void *flux_aux_get (flux_t *h, const char *name);
 int flux_aux_set (flux_t *h, const char *name, void *aux, flux_free_f destroy);
@@ -138,6 +147,11 @@ uint32_t flux_matchtag_avail (flux_t *h);
  */
 int flux_send (flux_t *h, const flux_msg_t *msg, int flags);
 
+/* Send a message - same as above but '*msg' ownership is transferred to 'h'.
+ * N.B. this fails with EINVAL if '*msg' reference count is greater than 1.
+ */
+int flux_send_new (flux_t *h, flux_msg_t **msg, int flags);
+
 /* Receive a message
  * flags may be 0 or FLUX_O_TRACE or FLUX_O_NONBLOCK (FLUX_O_COPROC is ignored)
  * flux_recv reads messages from the handle until 'match' is matched,
@@ -154,11 +168,6 @@ flux_msg_t *flux_recv (flux_t *h, struct flux_match match, int flags);
  */
 int flux_requeue (flux_t *h, const flux_msg_t *msg, int flags);
 
-/* Requeue a message without copying
- * Identical to flux_requeue(), but takes ownership of 'msg'.
- */
-int flux_requeue_nocopy (flux_t *h, flux_msg_t *msg, int flags);
-
 /* Obtain a bitmask of FLUX_POLL* bits for the flux handle.
  * Returns bitmask on success, -1 on error with errno set.
  * See flux_pollfd() comment below.
@@ -172,15 +181,10 @@ int flux_pollevents (flux_t *h);
  * should then call flux_pollevents().  See src/common/libflux/ev_flux.[ch]
  * for an example of a libev "composite watcher" based on these interfaces,
  * that is used internally by the flux reactor.
- * Returns fd on sucess, -1 on failure with errno set.
+ * Returns fd on success, -1 on failure with errno set.
  */
 int flux_pollfd (flux_t *h);
 
-/* Event subscribe/unsubscribe.
- */
-int flux_event_subscribe (flux_t *h, const char *topic);
-int flux_event_unsubscribe (flux_t *h, const char *topic);
-
 /* Get/clear handle message counters.
  */
 void flux_get_msgcounters (flux_t *h, flux_msgcounters_t *mcs);
diff --git a/src/common/libflux/heartbeat.c b/src/common/libflux/heartbeat.c
deleted file mode 100644
index f88353ef212b..000000000000
--- a/src/common/libflux/heartbeat.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <string.h>
-#include <errno.h>
-
-#include "event.h"
-#include "message.h"
-
-
-flux_msg_t *flux_heartbeat_encode (int epoch)
-{
-    flux_msg_t *msg = NULL;
-
-    if (!(msg = flux_event_pack ("hb", "{ s:i }", "epoch", epoch)))
-        return NULL;
-    return msg;
-}
-
-int flux_heartbeat_decode (const flux_msg_t *msg, int *epoch)
-{
-    const char *topic_str;
-    int rc = -1;
-
-    if (flux_event_unpack (msg, &topic_str, "{ s:i }", "epoch", epoch) < 0)
-        goto done;
-    if (strcmp (topic_str, "hb") != 0) {
-        errno = EPROTO;
-        goto done;
-    }
-    rc = 0;
-done:
-    return rc;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/heartbeat.h b/src/common/libflux/heartbeat.h
deleted file mode 100644
index 67f28f603ef1..000000000000
--- a/src/common/libflux/heartbeat.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_CORE_HEARTBEAT
-#define _FLUX_CORE_HEARTBEAT
-
-#include "message.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-flux_msg_t *flux_heartbeat_encode (int epoch);
-int flux_heartbeat_decode (const flux_msg_t *msg, int *epoch);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* !_FLUX_CORE_HEARTBEAT */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/src/common/libflux/keepalive.c b/src/common/libflux/keepalive.c
deleted file mode 100644
index 477afbe68d90..000000000000
--- a/src/common/libflux/keepalive.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/************************************************************\
- * Copyright 2016 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <flux/core.h>
-
-#include "keepalive.h"
-
-flux_msg_t *flux_keepalive_encode (int errnum, int status)
-{
-    flux_msg_t *msg;
-
-    if (!(msg = flux_msg_create (FLUX_MSGTYPE_KEEPALIVE)))
-        goto error;
-    if (flux_msg_set_errnum (msg, errnum) < 0)
-        goto error;
-    if (flux_msg_set_status (msg, status) < 0)
-        goto error;
-    return msg;
-error:
-    flux_msg_destroy (msg);
-    return NULL;
-}
-
-int flux_keepalive_decode (const flux_msg_t *msg, int *ep, int *sp)
-{
-    int rc = -1;
-    int errnum, status;
-
-    if (flux_msg_get_errnum (msg, &errnum) < 0)
-        goto done;
-    if (flux_msg_get_status (msg, &status) < 0)
-        goto done;
-    if (ep)
-        *ep = errnum;
-    if (sp)
-        *sp = status;
-    rc = 0;
-done:
-    return rc;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/keepalive.h b/src/common/libflux/keepalive.h
deleted file mode 100644
index 67e5e3a80a58..000000000000
--- a/src/common/libflux/keepalive.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/************************************************************\
- * Copyright 2016 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_CORE_KEEPALIVE_H
-#define _FLUX_CORE_KEEPALIVE_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-flux_msg_t *flux_keepalive_encode (int errnum, int status);
-
-int flux_keepalive_decode (const flux_msg_t *msg, int *errnum, int *status);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* !_FLUX_CORE_KEEPALIVE_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/message.c b/src/common/libflux/message.c
index 5bc99b5b8e2f..e34870527d4f 100644
--- a/src/common/libflux/message.c
+++ b/src/common/libflux/message.c
@@ -8,7 +8,8 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-/* A flux messages consist of a list of zeromq frames:
+/* A flux message contains route, topic, payload protocol information.
+ * When sent it is formed into the following zeromq frames.
  *
  * [route]
  * [route]
@@ -33,194 +34,90 @@
 #include <assert.h>
 #include <fnmatch.h>
 #include <inttypes.h>
-#include <czmq.h>
 #include <jansson.h>
+#include <flux/core.h>
 
 #include "src/common/libutil/aux.h"
 #include "src/common/libutil/errno_safe.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
 
-#include "message.h"
+#include "message_private.h"
+#include "message_iovec.h"
+#include "message_route.h"
+#include "message_proto.h"
 
-struct flux_msg {
-    zmsg_t *zmsg;
-    json_t *json;
-    struct aux_item *aux;
-    int refcount;
-};
-
-/* Begin manual codec
- * PROTO consists of 4 byte prelude followed by a fixed length
- * array of u32's in network byte order.
- */
-#define PROTO_MAGIC         0x8e
-#define PROTO_VERSION       1
-
-#define PROTO_OFF_MAGIC     0 /* 1 byte */
-#define PROTO_OFF_VERSION   1 /* 1 byte */
-#define PROTO_OFF_TYPE      2 /* 1 byte */
-#define PROTO_OFF_FLAGS     3 /* 1 byte */
-#define PROTO_OFF_U32_ARRAY 4
-
-#define PROTO_IND_USERID    0
-#define PROTO_IND_ROLEMASK  1
-#define PROTO_IND_AUX1      2
-#define PROTO_IND_AUX2      3
-
-#define PROTO_U32_COUNT     4
-#define PROTO_SIZE          4 + (PROTO_U32_COUNT * 4)
-
-/* Helpful aliases */
-#define PROTO_IND_NODEID    PROTO_IND_AUX1 // request
-#define PROTO_IND_MATCHTAG  PROTO_IND_AUX2 // request, response
-#define PROTO_IND_SEQUENCE  PROTO_IND_AUX1 // event
-#define PROTO_IND_ERRNUM    PROTO_IND_AUX1 // response, keepalive
-#define PROTO_IND_STATUS    PROTO_IND_AUX2 // keepalive
-
-static int proto_set_u32 (uint8_t *data, int len, int index, uint32_t val);
-
-static int proto_set_type (uint8_t *data, int len, int type)
+static int msg_validate (const flux_msg_t *msg)
 {
-    if (len < PROTO_SIZE || data[PROTO_OFF_MAGIC] != PROTO_MAGIC
-                         || data[PROTO_OFF_VERSION] != PROTO_VERSION)
+    if (!msg || msg->refcount <= 0) {
+        errno = EINVAL;
         return -1;
-    switch (type) {
+    }
+    return 0;
+}
+
+static void msg_setup_type (flux_msg_t *msg)
+{
+    switch (msg_typeof (msg)) {
         case FLUX_MSGTYPE_REQUEST:
-            if (proto_set_u32 (data, len, PROTO_IND_NODEID,
-                               FLUX_NODEID_ANY) < 0)
-                return -1;
-            if (proto_set_u32 (data, len, PROTO_IND_MATCHTAG,
-                               FLUX_MATCHTAG_NONE) < 0)
-                return -1;
+            msg->proto.nodeid = FLUX_NODEID_ANY;
+            msg->proto.matchtag = FLUX_MATCHTAG_NONE;
             break;
         case FLUX_MSGTYPE_RESPONSE:
             /* N.B. don't clobber matchtag from request on set_type */
-            if (proto_set_u32 (data, len, PROTO_IND_ERRNUM, 0) < 0)
-                return -1;
+            msg->proto.errnum = 0;
             break;
         case FLUX_MSGTYPE_EVENT:
-            if (proto_set_u32 (data, len, PROTO_IND_SEQUENCE, 0) < 0)
-                return -1;
-            if (proto_set_u32 (data, len, PROTO_IND_AUX2, 0) < 0)
-                return -1;
+            msg->proto.sequence = 0;
+            msg->proto.aux2 = 0;
             break;
-        case FLUX_MSGTYPE_KEEPALIVE:
-            if (proto_set_u32 (data, len, PROTO_IND_STATUS, 0) < 0)
-                return -1;
-            if (proto_set_u32 (data, len, PROTO_IND_ERRNUM, 0) < 0)
-                return -1;
+        case FLUX_MSGTYPE_CONTROL:
+            msg->proto.control_type = 0;
+            msg->proto.control_status = 0;
             break;
-        default:
-            return -1;
     }
-    data[PROTO_OFF_TYPE] = type;
-    return 0;
 }
-static int proto_get_type (uint8_t *data, int len, int *type)
-{
-    if (len < PROTO_SIZE || data[PROTO_OFF_MAGIC] != PROTO_MAGIC
-                         || data[PROTO_OFF_VERSION] != PROTO_VERSION)
-        return -1;
-    *type = data[PROTO_OFF_TYPE];
-    return 0;
-}
-static int proto_set_flags (uint8_t *data, int len, uint8_t flags)
-{
-    if (len < PROTO_SIZE || data[PROTO_OFF_MAGIC] != PROTO_MAGIC
-                         || data[PROTO_OFF_VERSION] != PROTO_VERSION)
-        return -1;
-    data[PROTO_OFF_FLAGS] = flags;
-    return 0;
-}
-static int proto_get_flags (uint8_t *data, int len, uint8_t *val)
-{
-    if (len < PROTO_SIZE || data[PROTO_OFF_MAGIC] != PROTO_MAGIC
-                         || data[PROTO_OFF_VERSION] != PROTO_VERSION)
-        return -1;
-    *val = data[PROTO_OFF_FLAGS];
-    return 0;
-}
-static int proto_set_u32 (uint8_t *data, int len, int index, uint32_t val)
-{
-    uint32_t x = htonl (val);
-    int offset = PROTO_OFF_U32_ARRAY + index * 4;
 
-    if (len < PROTO_SIZE || data[PROTO_OFF_MAGIC] != PROTO_MAGIC
-                         || data[PROTO_OFF_VERSION] != PROTO_VERSION
-                         || index < 0 || index >= PROTO_U32_COUNT)
-        return -1;
-    memcpy (&data[offset], &x, sizeof (x));
-    return 0;
-}
-static int proto_get_u32 (uint8_t *data, int len, int index, uint32_t *val)
-{
-    uint32_t x;
-    int offset = PROTO_OFF_U32_ARRAY + index * 4;
-
-    if (len < PROTO_SIZE || data[PROTO_OFF_MAGIC] != PROTO_MAGIC
-                         || data[PROTO_OFF_VERSION] != PROTO_VERSION
-                         || index < 0 || index >= PROTO_U32_COUNT)
-        return -1;
-    memcpy (&x, &data[offset], sizeof (x));
-    *val = ntohl (x);
-    return 0;
-}
-static void proto_init (uint8_t *data, int len, uint8_t flags)
-{
-    int n;
-    assert (len >= PROTO_SIZE);
-    memset (data, 0, len);
-    data[PROTO_OFF_MAGIC] = PROTO_MAGIC;
-    data[PROTO_OFF_VERSION] = PROTO_VERSION;
-    data[PROTO_OFF_FLAGS] = flags;
-    n = proto_set_u32 (data, len, PROTO_IND_USERID, FLUX_USERID_UNKNOWN);
-    assert (n == 0);
-    n = proto_set_u32 (data, len, PROTO_IND_ROLEMASK, FLUX_ROLE_NONE);
-    assert (n == 0);
-}
-/* End manual codec
- */
-
-static flux_msg_t *flux_msg_create_common (void)
+flux_msg_t *msg_create (void)
 {
     flux_msg_t *msg;
 
     if (!(msg = calloc (1, sizeof (*msg))))
         return NULL;
+    list_head_init (&msg->routes);
+    list_node_init (&msg->list);
+    msg->proto.userid = FLUX_USERID_UNKNOWN;
+    msg->proto.rolemask = FLUX_ROLE_NONE;
     msg->refcount = 1;
     return msg;
 }
 
 flux_msg_t *flux_msg_create (int type)
 {
-    uint8_t proto[PROTO_SIZE];
     flux_msg_t *msg;
 
-    if (!(msg = flux_msg_create_common ()))
-        return NULL;
-    proto_init (proto, PROTO_SIZE, 0);
-    if (proto_set_type (proto, PROTO_SIZE, type) < 0) {
+    if (!msgtype_is_valid (type)) {
         errno = EINVAL;
-        goto error;
-    }
-    if (!(msg->zmsg = zmsg_new ())) {
-        errno = ENOMEM;
-        goto error;
+        return NULL;
     }
-    if (zmsg_addmem (msg->zmsg, proto, PROTO_SIZE) < 0)
-        goto error;
+    if (!(msg = msg_create ()))
+        return NULL;
+    msg->proto.type = type;
+    msg_setup_type (msg);
     return msg;
-error:
-    flux_msg_destroy (msg);
-    return NULL;
 }
 
 void flux_msg_destroy (flux_msg_t *msg)
 {
-    if (msg && --msg->refcount == 0) {
+    if (msg && msg->refcount > 0 && --msg->refcount == 0) {
         int saved_errno = errno;
+        if (msg_has_route (msg))
+            msg_route_clear (msg);
+        free (msg->topic);
+        free (msg->payload);
         json_decref (msg->json);
-        zmsg_destroy (&msg->zmsg);
         aux_destroy (&msg->aux);
+        free (msg->lasterr);
         free (msg);
         errno = saved_errno;
     }
@@ -241,10 +138,8 @@ const flux_msg_t *flux_msg_incref (const flux_msg_t *const_msg)
 {
     flux_msg_t *msg = (flux_msg_t *)const_msg;
 
-    if (!msg) {
-        errno = EINVAL;
+    if (msg_validate (msg) < 0)
         return NULL;
-    }
     msg->refcount++;
     return msg;
 }
@@ -253,84 +148,148 @@ const flux_msg_t *flux_msg_incref (const flux_msg_t *const_msg)
  * allow msg to be "annotated" for convenience.
  * The message content is otherwise unchanged.
  */
-int flux_msg_aux_set (const flux_msg_t *const_msg, const char *name,
-                      void *aux, flux_free_f destroy)
+int flux_msg_aux_set (const flux_msg_t *const_msg,
+                      const char *name,
+                      void *aux,
+                      flux_free_f destroy)
 {
     flux_msg_t *msg = (flux_msg_t *)const_msg;
-    if (!msg) {
-        errno = EINVAL;
+    if (msg_validate (msg) < 0)
         return -1;
-    }
     return aux_set (&msg->aux, name, aux, destroy);
 }
 
 void *flux_msg_aux_get (const flux_msg_t *msg, const char *name)
 {
-    if (!msg) {
-        errno = EINVAL;
+    if (msg_validate (msg) < 0)
         return NULL;
-    }
     return aux_get (msg->aux, name);
 }
 
-size_t flux_msg_encode_size (const flux_msg_t *msg)
+static void encode_count (ssize_t *size, size_t len)
 {
-    zframe_t *zf;
-    size_t size = 0;
+    if (len < 255)
+        (*size) += 1;
+    else
+        (*size) += 1 + 4;
+    (*size) += len;
+}
 
-    zf = zmsg_first (msg->zmsg);
-    while (zf) {
-        size_t n = zframe_size (zf);
-        if (n < 255)
-            size += 1;
-        else
-            size += 1 + 4;
-        size += n;
-        zf = zmsg_next (msg->zmsg);
+ssize_t flux_msg_encode_size (const flux_msg_t *msg)
+{
+    ssize_t size = 0;
+
+    if (msg_validate (msg) < 0)
+        return -1;
+
+    encode_count (&size, PROTO_SIZE);
+    if (msg_has_payload (msg))
+        encode_count (&size, msg->payload_size);
+    if (msg_has_topic (msg))
+        encode_count (&size, strlen (msg->topic));
+    if (msg_has_route (msg)) {
+        struct route_id *r = NULL;
+        /* route delimiter */
+        encode_count (&size, 0);
+        list_for_each (&msg->routes, r, route_id_node)
+            encode_count (&size, strlen (r->id));
     }
     return size;
 }
 
+static ssize_t encode_frame (uint8_t *buf,
+                             size_t buf_len,
+                             void *frame,
+                             size_t frame_size)
+{
+    ssize_t n = 0;
+    if (frame_size < 0xff) {
+        if (buf_len < (frame_size + 1)) {
+            errno = EINVAL;
+            return -1;
+        }
+        *buf++ = (uint8_t)frame_size;
+        n += 1;
+    } else {
+        if (buf_len < (frame_size + 1 + 4)) {
+            errno = EINVAL;
+            return -1;
+        }
+        *buf++ = 0xff;
+        *(uint32_t *)buf = htonl (frame_size);
+        buf += 4;
+        n += 1 + 4;
+    }
+    if (frame && frame_size)
+        memcpy (buf, frame, frame_size);
+    return (frame_size + n);
+}
 
 int flux_msg_encode (const flux_msg_t *msg, void *buf, size_t size)
 {
-    uint8_t *p = buf;
-    zframe_t *zf;
+    uint8_t proto[PROTO_SIZE];
+    ssize_t total = 0;
+    ssize_t n;
 
-    zf = zmsg_first (msg->zmsg);
-    while (zf) {
-        size_t n = zframe_size (zf);
-        if (n < 0xff) {
-            if (size - (p - (uint8_t *)buf) < n + 1)
-                goto nospace;
-            *p++ = (uint8_t)n;
-        } else {
-            if (size - (p - (uint8_t *)buf) < n + 1 + 4)
-                goto nospace;
-            *p++ = 0xff;
-            *(uint32_t *)p = htonl (n);
-            p += 4;
+    if (msg_validate (msg) < 0)
+        return -1;
+    /* msg never completed initial setup */
+    if (!msg_type_is_valid (msg)) {
+        errno = EPROTO;
+        return -1;
+    }
+    if (msg_has_route (msg)) {
+        struct route_id *r = NULL;
+        list_for_each (&msg->routes, r, route_id_node) {
+            if ((n = encode_frame (buf + total,
+                                   size - total,
+                                   r->id,
+                                   strlen (r->id))) < 0)
+                return -1;
+            total += n;
         }
-        memcpy (p, zframe_data (zf), n);
-        p += n;
-        zf = zmsg_next (msg->zmsg);
+        /* route delimiter */
+        if ((n = encode_frame (buf + total,
+                               size - total,
+                               NULL,
+                               0)) < 0)
+            return -1;
+        total += n;
     }
+    if (msg_has_topic (msg)) {
+        if ((n = encode_frame (buf + total,
+                               size - total,
+                               msg->topic,
+                               strlen (msg->topic))) < 0)
+            return -1;
+        total += n;
+    }
+    if (msg_has_payload (msg)) {
+        if ((n = encode_frame (buf + total,
+                               size - total,
+                               msg->payload,
+                               msg->payload_size)) < 0)
+            return -1;
+        total += n;
+    }
+    if (proto_encode (&msg->proto, proto, PROTO_SIZE) < 0
+        || (n = encode_frame (buf + total,
+                              size - total,
+                              proto,
+                              PROTO_SIZE)) < 0)
+        return -1;
+    total += n;
     return 0;
-nospace:
-    errno = EINVAL;
-    return -1;
 }
 
 flux_msg_t *flux_msg_decode (const void *buf, size_t size)
 {
     flux_msg_t *msg;
-    uint8_t const *p = buf;
-    zframe_t *zf;
+    const uint8_t *p = buf;
+    struct msg_iovec *iov = NULL;
+    int iovlen = 0;
+    int iovcnt = 0;
 
-    if (!(msg = flux_msg_create_common ()))
-        return NULL;
-    if (!(msg->zmsg = zmsg_new ()))
-        goto nomem;
     while (p - (uint8_t *)buf < size) {
         size_t n = *p++;
         if (n == 0xff) {
@@ -345,187 +304,166 @@ flux_msg_t *flux_msg_decode (const void *buf, size_t size)
             errno = EINVAL;
             goto error;
         }
-        if (!(zf = zframe_new (p, n)))
-            goto nomem;
-        if (zmsg_append (msg->zmsg, &zf) < 0)
-            goto nomem;
+        if (iovlen <= iovcnt) {
+            struct msg_iovec *tmp;
+            iovlen += IOVECINCR;
+            if (!(tmp = realloc (iov, sizeof (*iov) * iovlen)))
+                goto error;
+            iov = tmp;
+        }
+        iov[iovcnt].data = p;
+        iov[iovcnt].size = n;
+        iovcnt++;
         p += n;
     }
+    if (!(msg = iovec_to_msg (iov, iovcnt)))
+        goto error;
+    free (iov);
     return msg;
-nomem:
-    errno = ENOMEM;
 error:
-    flux_msg_destroy (msg);
+    ERRNO_SAFE_WRAP (free, iov);
     return NULL;
 }
 
 int flux_msg_set_type (flux_msg_t *msg, int type)
 {
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    if (!zf || proto_set_type (zframe_data (zf), zframe_size (zf), type) < 0) {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!msgtype_is_valid (type)) {
         errno = EINVAL;
         return -1;
     }
+    msg->proto.type = type;
+    msg_setup_type (msg);
     return 0;
 }
 
 int flux_msg_get_type (const flux_msg_t *msg, int *type)
 {
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    if (!zf || proto_get_type (zframe_data (zf), zframe_size (zf), type) < 0) {
-        errno = EPROTO;
+    if (msg_validate (msg) < 0)
         return -1;
-    }
+    if (type)
+        *type = msg_typeof (msg);
     return 0;
 }
 
-int flux_msg_set_flags (flux_msg_t *msg, uint8_t fl)
+bool flux_msg_has_flag (const flux_msg_t *msg, int flag)
 {
-    const uint8_t valid_flags = FLUX_MSGFLAG_TOPIC | FLUX_MSGFLAG_PAYLOAD
-                              | FLUX_MSGFLAG_ROUTE | FLUX_MSGFLAG_UPSTREAM
-                              | FLUX_MSGFLAG_PRIVATE | FLUX_MSGFLAG_STREAMING;
+    if (msg_validate (msg) < 0)
+        return false;
+    return msg_has_flag (msg, flag) ? true : false;
+}
 
-    if (!msg || (fl & ~valid_flags) != 0) {
-        errno = EINVAL;
+int flux_msg_set_flag (flux_msg_t *msg, int flag)
+{
+    if (msg_validate (msg) < 0)
         return -1;
-    }
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    if (!zf || proto_set_flags (zframe_data (zf), zframe_size (zf), fl) < 0) {
+    if (!msgflags_is_valid (flag)) {
         errno = EINVAL;
         return -1;
     }
+    msg_set_flag (msg, flag);
     return 0;
 }
 
-int flux_msg_get_flags (const flux_msg_t *msg, uint8_t *fl)
+int flux_msg_clear_flag (flux_msg_t *msg, int flag)
 {
-    if (!msg || !fl) {
-        errno = EINVAL;
+    if (msg_validate (msg) < 0)
         return -1;
-    }
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    if (!zf || proto_get_flags (zframe_data (zf), zframe_size (zf), fl) < 0) {
-        errno = EPROTO;
+    if (!msgflags_is_valid (flag)) {
+        errno = EINVAL;
         return -1;
     }
+    msg_clear_flag (msg, flag);
     return 0;
 }
 
 int flux_msg_set_private (flux_msg_t *msg)
 {
-    uint8_t flags;
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        return -1;
-    if (flux_msg_set_flags (msg, flags | FLUX_MSGFLAG_PRIVATE) < 0)
+    if (msg_validate (msg) < 0)
         return -1;
+    msg_set_flag (msg, FLUX_MSGFLAG_PRIVATE);
     return 0;
 }
 
 bool flux_msg_is_private (const flux_msg_t *msg)
 {
-    uint8_t flags;
-    if (flux_msg_get_flags (msg, &flags) < 0)
+    if (msg_validate (msg) < 0)
         return true;
-    return (flags & FLUX_MSGFLAG_PRIVATE) ? true : false;
+    return msg_has_private (msg) ? true : false;
 }
 
 int flux_msg_set_streaming (flux_msg_t *msg)
 {
-    uint8_t flags;
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        return -1;
-    if (flux_msg_set_flags (msg, flags | FLUX_MSGFLAG_STREAMING) < 0)
+    if (msg_validate (msg) < 0)
         return -1;
+    msg_clear_flag (msg, FLUX_MSGFLAG_NORESPONSE);
+    msg_set_flag (msg, FLUX_MSGFLAG_STREAMING);
     return 0;
 }
 
 bool flux_msg_is_streaming (const flux_msg_t *msg)
 {
-    uint8_t flags;
-    if (flux_msg_get_flags (msg, &flags) < 0)
+    if (msg_validate (msg) < 0)
         return true;
-    return (flags & FLUX_MSGFLAG_STREAMING) ? true : false;
+    return msg_has_streaming (msg) ? true : false;
 }
 
-int flux_msg_set_userid (flux_msg_t *msg, uint32_t userid)
+int flux_msg_set_noresponse (flux_msg_t *msg)
 {
-    zframe_t *zf;
-
-    if (!msg) {
-        errno = EINVAL;
-        return -1;
-    }
-    zf = zmsg_last (msg->zmsg);
-    if (!zf || proto_set_u32 (zframe_data (zf),
-                              zframe_size (zf),
-                              PROTO_IND_USERID,
-                              userid) < 0) {
-        errno = EINVAL;
+    if (msg_validate (msg) < 0)
         return -1;
-    }
+    msg_clear_flag (msg, FLUX_MSGFLAG_STREAMING);
+    msg_set_flag (msg, FLUX_MSGFLAG_NORESPONSE);
     return 0;
 }
 
-int flux_msg_get_userid (const flux_msg_t *msg, uint32_t *userid)
+bool flux_msg_is_noresponse (const flux_msg_t *msg)
 {
-    zframe_t *zf;
+    if (msg_validate (msg) < 0)
+        return true;
+    return msg_has_noresponse (msg) ? true : false;
+}
 
-    if (!msg || !userid) {
-        errno = EINVAL;
+int flux_msg_set_userid (flux_msg_t *msg, uint32_t userid)
+{
+    if (msg_validate (msg) < 0)
         return -1;
-    }
-    zf = zmsg_last (msg->zmsg);
-    if (!zf || proto_get_u32 (zframe_data (zf),
-                              zframe_size (zf),
-                              PROTO_IND_USERID,
-                              userid) < 0) {
-        errno = EPROTO;
+    msg->proto.userid = userid;
+    return 0;
+}
+
+int flux_msg_get_userid (const flux_msg_t *msg, uint32_t *userid)
+{
+    if (msg_validate (msg) < 0)
         return -1;
-    }
+    if (userid)
+        *userid = msg->proto.userid;
     return 0;
 }
 
 int flux_msg_set_rolemask (flux_msg_t *msg, uint32_t rolemask)
 {
-    zframe_t *zf;
-
-    if (!msg) {
-        errno = EINVAL;
+    if (msg_validate (msg) < 0)
         return -1;
-    }
-    zf = zmsg_last (msg->zmsg);
-    if (!zf || proto_set_u32 (zframe_data (zf),
-                              zframe_size (zf),
-                              PROTO_IND_ROLEMASK,
-                              rolemask) < 0) {
-        errno = EINVAL;
-        return -1;
-    }
+    msg->proto.rolemask = rolemask;
     return 0;
 }
 
 int flux_msg_get_rolemask (const flux_msg_t *msg, uint32_t *rolemask)
 {
-    zframe_t *zf;
-
-    if (!msg || !rolemask) {
-        errno = EINVAL;
+    if (msg_validate (msg) < 0)
         return -1;
-    }
-    zf = zmsg_last (msg->zmsg);
-    if (!zf || proto_get_u32 (zframe_data (zf),
-                              zframe_size (zf),
-                              PROTO_IND_ROLEMASK,
-                              rolemask) < 0) {
-        errno = EPROTO;
-        return -1;
-    }
+    if (rolemask)
+        *rolemask = msg->proto.rolemask;
     return 0;
 }
 
 int flux_msg_get_cred (const flux_msg_t *msg, struct flux_msg_cred *cred)
 {
-    if (!msg || !cred) {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!cred) {
         errno = EINVAL;
         return -1;
     }
@@ -538,10 +476,8 @@ int flux_msg_get_cred (const flux_msg_t *msg, struct flux_msg_cred *cred)
 
 int flux_msg_set_cred (flux_msg_t *msg, struct flux_msg_cred cred)
 {
-    if (!msg) {
-        errno = EINVAL;
+    if (msg_validate (msg) < 0)
         return -1;
-    }
     if (flux_msg_set_rolemask (msg, cred.rolemask) < 0)
         return -1;
     if (flux_msg_set_userid (msg, cred.userid) < 0)
@@ -553,8 +489,9 @@ int flux_msg_cred_authorize (struct flux_msg_cred cred, uint32_t userid)
 {
     if ((cred.rolemask & FLUX_ROLE_OWNER))
         return 0;
-    if ((cred.rolemask & FLUX_ROLE_USER) && cred.userid != FLUX_USERID_UNKNOWN
-                                         && cred.userid == userid)
+    if ((cred.rolemask & FLUX_ROLE_USER)
+        && cred.userid != FLUX_USERID_UNKNOWN
+        && cred.userid == userid)
         return 0;
     errno = EPERM;
     return -1;
@@ -571,177 +508,143 @@ int flux_msg_authorize (const flux_msg_t *msg, uint32_t userid)
     return 0;
 }
 
-int flux_msg_set_nodeid (flux_msg_t *msg, uint32_t nodeid)
+bool flux_msg_is_local (const flux_msg_t *msg)
 {
-    zframe_t *zf;
-    int type;
+    uint32_t rolemask;
+    if (flux_msg_get_rolemask (msg, &rolemask) == 0
+        && (rolemask & FLUX_ROLE_LOCAL))
+        return true;
+    return false;
+}
 
-    if (!msg)
-        goto error;
+int flux_msg_set_nodeid (flux_msg_t *msg, uint32_t nodeid)
+{
+    if (msg_validate (msg) < 0)
+        return -1;
     if (nodeid == FLUX_NODEID_UPSTREAM) /* should have been resolved earlier */
         goto error;
-    if (!(zf = zmsg_last (msg->zmsg)))
-        goto error;
-    if (proto_get_type (zframe_data (zf), zframe_size (zf), &type) < 0)
-        goto error;
-    if (type != FLUX_MSGTYPE_REQUEST)
-        goto error;
-    if (proto_set_u32 (zframe_data (zf), zframe_size (zf),
-                       PROTO_IND_NODEID, nodeid) < 0)
+    if (!msg_is_request (msg))
         goto error;
+    msg->proto.nodeid = nodeid;
     return 0;
 error:
     errno = EINVAL;
     return -1;
 }
 
-int flux_msg_get_nodeid (const flux_msg_t *msg, uint32_t *nodeidp)
+int flux_msg_get_nodeid (const flux_msg_t *msg, uint32_t *nodeid)
 {
-    zframe_t *zf;
-    int type;
-    uint32_t nodeid;
-
-    if (!msg || !nodeidp) {
-        errno = EINVAL;
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!msg_is_request (msg)) {
+        errno = EPROTO;
         return -1;
     }
-    if (!(zf = zmsg_last (msg->zmsg)))
-        goto error;
-    if (proto_get_type (zframe_data (zf), zframe_size (zf), &type) < 0)
-        goto error;
-    if (type != FLUX_MSGTYPE_REQUEST)
-        goto error;
-    if (proto_get_u32 (zframe_data (zf), zframe_size (zf),
-                       PROTO_IND_NODEID, &nodeid) < 0)
-        goto error;
-    *nodeidp = nodeid;
+    if (nodeid)
+        *nodeid = msg->proto.nodeid;
     return 0;
-error:
-    return EPROTO;
-    return -1;
 }
 
-int flux_msg_set_errnum (flux_msg_t *msg, int e)
+int flux_msg_set_errnum (flux_msg_t *msg, int errnum)
 {
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    int type;
-
-    if (!zf || proto_get_type (zframe_data (zf), zframe_size (zf), &type) < 0
-            || (type != FLUX_MSGTYPE_RESPONSE && type != FLUX_MSGTYPE_KEEPALIVE)
-            || proto_set_u32 (zframe_data (zf), zframe_size (zf),
-                              PROTO_IND_ERRNUM, e) < 0) {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!msg_is_response (msg)) {
         errno = EINVAL;
         return -1;
     }
+    msg->proto.errnum = errnum;
     return 0;
 }
 
-int flux_msg_get_errnum (const flux_msg_t *msg, int *e)
+int flux_msg_get_errnum (const flux_msg_t *msg, int *errnum)
 {
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    int type;
-    uint32_t xe;
-
-    if (!zf || proto_get_type (zframe_data (zf), zframe_size (zf), &type) < 0
-            || (type != FLUX_MSGTYPE_RESPONSE && type != FLUX_MSGTYPE_KEEPALIVE)
-            || proto_get_u32 (zframe_data (zf), zframe_size (zf),
-                              PROTO_IND_ERRNUM, &xe) < 0) {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!msg_is_response (msg)) {
         errno = EPROTO;
         return -1;
     }
-    *e = xe;
+    if (errnum)
+        *errnum = msg->proto.errnum;
     return 0;
 }
 
 int flux_msg_set_seq (flux_msg_t *msg, uint32_t seq)
 {
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    int type;
-
-    if (!zf || proto_get_type (zframe_data (zf), zframe_size (zf), &type) < 0
-            || type != FLUX_MSGTYPE_EVENT
-            || proto_set_u32 (zframe_data (zf), zframe_size (zf),
-                              PROTO_IND_SEQUENCE, seq) < 0) {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!msg_is_event (msg)) {
         errno = EINVAL;
         return -1;
     }
+    msg->proto.sequence = seq;
     return 0;
 }
 
 int flux_msg_get_seq (const flux_msg_t *msg, uint32_t *seq)
 {
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    int type;
-
-    if (!zf || proto_get_type (zframe_data (zf), zframe_size (zf), &type) < 0
-            || type != FLUX_MSGTYPE_EVENT
-            || proto_get_u32 (zframe_data (zf), zframe_size (zf),
-                              PROTO_IND_SEQUENCE, seq) < 0) {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!(msg_is_event (msg))) {
         errno = EPROTO;
         return -1;
     }
+    if (seq)
+        *seq = msg->proto.sequence;
     return 0;
 }
 
-int flux_msg_set_matchtag (flux_msg_t *msg, uint32_t t)
+int flux_msg_set_matchtag (flux_msg_t *msg, uint32_t matchtag)
 {
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    int type;
-
-    if (!zf || proto_get_type (zframe_data (zf), zframe_size (zf), &type) < 0
-            || (type != FLUX_MSGTYPE_REQUEST && type != FLUX_MSGTYPE_RESPONSE)
-            || proto_set_u32 (zframe_data (zf), zframe_size (zf),
-                              PROTO_IND_MATCHTAG, t) < 0) {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!msg_is_request (msg) && !msg_is_response (msg)) {
         errno = EINVAL;
         return -1;
     }
+    msg->proto.matchtag = matchtag;
     return 0;
 }
 
-int flux_msg_get_matchtag (const flux_msg_t *msg, uint32_t *t)
+int flux_msg_get_matchtag (const flux_msg_t *msg, uint32_t *matchtag)
 {
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    int type;
-
-    if (!zf || proto_get_type (zframe_data (zf), zframe_size (zf), &type) < 0
-            || (type != FLUX_MSGTYPE_REQUEST && type != FLUX_MSGTYPE_RESPONSE)
-            || proto_get_u32 (zframe_data (zf), zframe_size (zf),
-                              PROTO_IND_MATCHTAG, t) < 0) {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!msg_is_request (msg) && !msg_is_response (msg)) {
         errno = EPROTO;
         return -1;
     }
+    if (matchtag)
+        *matchtag = msg->proto.matchtag;
     return 0;
 }
 
-int flux_msg_set_status (flux_msg_t *msg, int s)
+int flux_msg_set_control (flux_msg_t *msg, int type, int status)
 {
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    int type;
-
-    if (!zf || proto_get_type (zframe_data (zf), zframe_size (zf), &type) < 0
-            || type != FLUX_MSGTYPE_KEEPALIVE
-            || proto_set_u32 (zframe_data (zf), zframe_size (zf),
-                              PROTO_IND_STATUS, s) < 0) {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!msg_is_control (msg)) {
         errno = EINVAL;
         return -1;
     }
+    msg->proto.control_type = type;
+    msg->proto.control_status = status;
     return 0;
 }
 
-int flux_msg_get_status (const flux_msg_t *msg, int *s)
+int flux_msg_get_control (const flux_msg_t *msg, int *type, int *status)
 {
-    zframe_t *zf = zmsg_last (msg->zmsg);
-    int type;
-    uint32_t u;
-
-    if (!zf || proto_get_type (zframe_data (zf), zframe_size (zf), &type) < 0
-            || type != FLUX_MSGTYPE_KEEPALIVE
-            || proto_get_u32 (zframe_data (zf), zframe_size (zf),
-                              PROTO_IND_STATUS, &u) < 0) {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!msg_is_control (msg)) {
         errno = EPROTO;
         return -1;
     }
-    *s = u;
+    if (type)
+        *type = msg->proto.control_type;
+    if (status)
+        *status = msg->proto.control_status;
     return 0;
 }
 
@@ -749,7 +652,7 @@ bool flux_msg_cmp_matchtag (const flux_msg_t *msg, uint32_t matchtag)
 {
     uint32_t tag;
 
-    if (flux_msg_get_route_count (msg) > 0)
+    if (flux_msg_route_count (msg) > 0)
         return false; /* don't match in foreign matchtag domain */
     if (flux_msg_get_matchtag (msg, &tag) < 0)
         return false;
@@ -762,7 +665,7 @@ static bool isa_matchany (const char *s)
 {
     if (!s || strlen(s) == 0)
         return true;
-    if (!strcmp (s, "*"))
+    if (streq (s, "*"))
         return true;
     return false;
 }
@@ -777,7 +680,7 @@ static bool isa_glob (const char *s)
 bool flux_msg_cmp (const flux_msg_t *msg, struct flux_match match)
 {
     if (match.typemask != 0) {
-        int type;
+        int type = 0;
         if (flux_msg_get_type (msg, &type) < 0)
             return false;
         if ((type & match.typemask) == 0)
@@ -795,335 +698,252 @@ bool flux_msg_cmp (const flux_msg_t *msg, struct flux_match match)
             if (fnmatch (match.topic_glob, topic, 0) != 0)
                 return false;
         } else {
-            if (strcmp (match.topic_glob, topic) != 0)
+            if (!streq (match.topic_glob, topic))
                 return false;
         }
     }
     return true;
 }
 
-int flux_msg_enable_route (flux_msg_t *msg)
+void flux_msg_route_enable (flux_msg_t *msg)
 {
-    uint8_t flags;
-
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        return -1;
-    if ((flags & FLUX_MSGFLAG_ROUTE))
-        return 0;
-    if (zmsg_pushmem (msg->zmsg, NULL, 0) < 0) {
-        errno = ENOMEM;
-        return -1;
-    }
-    flags |= FLUX_MSGFLAG_ROUTE;
-    return flux_msg_set_flags (msg, flags);
+    if (msg_validate (msg) < 0 || msg_has_route (msg))
+        return;
+    msg_set_flag (msg, FLUX_MSGFLAG_ROUTE);
 }
 
-int flux_msg_clear_route (flux_msg_t *msg)
+void flux_msg_route_disable (flux_msg_t *msg)
 {
-    uint8_t flags;
-    zframe_t *zf;
-    int size;
-
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        return -1;
-    if (!(flags & FLUX_MSGFLAG_ROUTE))
-        return 0;
-    while ((zf = zmsg_pop (msg->zmsg))) {
-        size = zframe_size (zf);
-        zframe_destroy (&zf);
-        if (size == 0)
-            break;
-    }
-    flags &= ~(uint8_t)FLUX_MSGFLAG_ROUTE;
-    return flux_msg_set_flags (msg, flags);
+    if (msg_validate (msg) < 0 || !msg_has_route (msg))
+        return;
+    msg_route_clear (msg);
+    msg_clear_flag (msg, FLUX_MSGFLAG_ROUTE);
 }
 
-int flux_msg_push_route (flux_msg_t *msg, const char *id)
+void flux_msg_route_clear (flux_msg_t *msg)
 {
-    uint8_t flags;
+    if (msg_validate (msg) < 0 || !msg_has_route (msg))
+        return;
+    msg_route_clear (msg);
+}
 
-    if (flux_msg_get_flags (msg, &flags) < 0)
+int flux_msg_route_push (flux_msg_t *msg, const char *id)
+{
+    if (msg_validate (msg) < 0)
         return -1;
-    if (!(flags & FLUX_MSGFLAG_ROUTE)) {
-        errno = EPROTO;
+    if (!id) {
+        errno = EINVAL;
         return -1;
     }
-    if (zmsg_pushstr (msg->zmsg, id) < 0) {
-        errno = ENOMEM;
+    if (!msg_has_route (msg)) {
+        errno = EPROTO;
         return -1;
     }
-    return 0;
+    return msg_route_push (msg, id, strlen (id));
 }
 
-int flux_msg_pop_route (flux_msg_t *msg, char **id)
+int flux_msg_route_delete_last (flux_msg_t *msg)
 {
-    uint8_t flags;
-    zframe_t *zf;
-
-    if (flux_msg_get_flags (msg, &flags) < 0)
+    if (msg_validate (msg) < 0)
         return -1;
-    if (!(flags & FLUX_MSGFLAG_ROUTE) || !(zf = zmsg_first (msg->zmsg))) {
+    if (!msg_has_route (msg)) {
         errno = EPROTO;
         return -1;
     }
-    if (zframe_size (zf) > 0 && (zf = zmsg_pop (msg->zmsg))) {
-        if (id) {
-            char *s = zframe_strdup (zf);
-            if (!s) {
-                zframe_destroy (&zf);
-                errno = ENOMEM;
-                return -1;
-            }
-            *id = s;
-        }
-        zframe_destroy (&zf);
-    } else {
-        if (id)
-            *id = NULL;
-    }
-    return 0;
+    return msg_route_delete_last (msg);
 }
 
 /* replaces flux_msg_nexthop */
-int flux_msg_get_route_last (const flux_msg_t *msg, char **id)
+const char *flux_msg_route_last (const flux_msg_t *msg)
 {
-    uint8_t flags;
-    zframe_t *zf;
-    char *s = NULL;
+    struct route_id *r;
 
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        return -1;
-    if (!(flags & FLUX_MSGFLAG_ROUTE) || !(zf = zmsg_first (msg->zmsg))) {
-        errno = EPROTO;
-        return -1;
-    }
-    if (zframe_size (zf) > 0 && !(s = zframe_strdup (zf))) {
-        errno = ENOMEM;
-        return -1;
-    }
-    *id = s;
-    return 0;
+    if (msg_validate (msg) < 0 || !msg_has_route (msg))
+        return NULL;
+    if ((r = list_top (&msg->routes, struct route_id, route_id_node)))
+        return r->id;
+    return NULL;
 }
 
 /* replaces flux_msg_sender */
-int flux_msg_get_route_first (const flux_msg_t *msg, char **id)
+const char *flux_msg_route_first (const flux_msg_t *msg)
 {
-    uint8_t flags;
-    zframe_t *zf, *zf_next;
-    char *s = NULL;
+    struct route_id *r;
 
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        return -1;
-    if (!(flags & FLUX_MSGFLAG_ROUTE)) {
-        errno = EPROTO;
-        return -1;
-    }
-    zf = zmsg_first (msg->zmsg);
-    while (zf && zframe_size (zf) > 0) {
-        zf_next = zmsg_next (msg->zmsg);
-        if (zf_next && zframe_size (zf_next) == 0)
-            break;
-        zf = zf_next;
-    }
-    if (!zf) {
-        errno = EPROTO;
-        return -1;
-    }
-    if (zframe_size (zf) > 0 && !(s = zframe_strdup (zf))) {
-        errno = ENOMEM;
-        return -1;
-    }
-    *id = s;
-    return 0;
+    if (msg_validate (msg) < 0 || !msg_has_route (msg))
+        return NULL;
+    if ((r = list_tail (&msg->routes, struct route_id, route_id_node)))
+        return r->id;
+    return NULL;
 }
 
-int flux_msg_get_route_count (const flux_msg_t *msg)
+int flux_msg_route_count (const flux_msg_t *msg)
 {
-    uint8_t flags;
-    zframe_t *zf;
-    int count = 0;
-
-    if (flux_msg_get_flags (msg, &flags) < 0)
+    if (msg_validate (msg) < 0)
         return -1;
-    if (!(flags & FLUX_MSGFLAG_ROUTE)) {
+    if (!msg_has_route (msg)) {
         errno = EPROTO;
         return -1;
     }
-    zf = zmsg_first (msg->zmsg);
-    while (zf && zframe_size (zf) > 0) {
-        zf = zmsg_next (msg->zmsg);
-        count++;
-    }
-    return count;
+    return msg->routes_len;
 }
 
 /* Get sum of size in bytes of route frames
  */
-static int flux_msg_get_route_size (const flux_msg_t *msg)
+static int flux_msg_route_size (const flux_msg_t *msg)
 {
-    uint8_t flags;
-    zframe_t *zf;
+    struct route_id *r = NULL;
     int size = 0;
 
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        return -1;
-    if (!(flags & FLUX_MSGFLAG_ROUTE)) {
+    assert (msg);
+    if (!msg_has_route (msg)) {
         errno = EPROTO;
         return -1;
     }
-    zf = zmsg_first (msg->zmsg);
-    while (zf && zframe_size (zf) > 0) {
-        size += zframe_size (zf);
-        zf = zmsg_next (msg->zmsg);
-    }
+    list_for_each (&msg->routes, r, route_id_node)
+        size += strlen (r->id);
     return size;
 }
 
-static zframe_t *flux_msg_get_route_nth (const flux_msg_t *msg, int n)
-{
-    uint8_t flags;
-    zframe_t *zf;
-    int count = 0;
-
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        return NULL;
-    if (!(flags & FLUX_MSGFLAG_ROUTE)) {
-        errno = EPROTO;
-        return NULL;
-    }
-    zf = zmsg_first (msg->zmsg);
-    while (zf && zframe_size (zf) > 0) {
-        if (count == n)
-            return zf;
-        zf = zmsg_next (msg->zmsg);
-        count++;
-    }
-    errno = ENOENT;
-    return NULL;
-}
-
-char *flux_msg_get_route_string (const flux_msg_t *msg)
+char *flux_msg_route_string (const flux_msg_t *msg)
 {
+    struct route_id *r = NULL;
     int hops, len;
-    int n;
-    zframe_t *zf;
     char *buf, *cp;
 
-    if (msg == NULL) {
-        errno = EINVAL;
+    if (msg_validate (msg) < 0)
         return NULL;
-    }
-    if ((hops = flux_msg_get_route_count (msg)) < 0
-                    || (len = flux_msg_get_route_size (msg)) < 0) {
+    if (!msg_has_route (msg)) {
+        errno = EPROTO;
         return NULL;
     }
+    if ((hops = flux_msg_route_count (msg)) < 0
+        || (len = flux_msg_route_size (msg)) < 0)
+        return NULL;
     if (!(cp = buf = malloc (len + hops + 1)))
         return NULL;
-    for (n = hops - 1; n >= 0; n--) {
+    list_for_each_rev (&msg->routes, r, route_id_node) {
         if (cp > buf)
             *cp++ = '!';
-        if (!(zf = flux_msg_get_route_nth (msg, n))) {
-            ERRNO_SAFE_WRAP (free, buf);
-            return NULL;
-        }
-        int cpylen = zframe_size (zf);
-        if (cpylen == 36) /* abbreviate long UUID */
+        int cpylen = strlen (r->id);
+        if (cpylen > 8) /* abbreviate long UUID */
             cpylen = 8;
         assert (cp - buf + cpylen < len + hops);
-        memcpy (cp, zframe_data (zf), cpylen);
+        memcpy (cp, r->id, cpylen);
         cp += cpylen;
     }
     *cp = '\0';
     return buf;
 }
 
-static bool payload_overlap (const void *b, zframe_t *zf)
+static bool payload_overlap (flux_msg_t *msg, const void *b)
 {
-    return ((char *)b >= (char *)zframe_data (zf)
-         && (char *)b <  (char *)zframe_data (zf) + zframe_size (zf));
+    return ((char *)b >= (char *)msg->payload
+         && (char *)b <  (char *)msg->payload + msg->payload_size);
 }
 
-int flux_msg_set_payload (flux_msg_t *msg, const void *buf, int size)
+int flux_msg_set_payload (flux_msg_t *msg, const void *buf, size_t size)
 {
-    zframe_t *zf;
-    uint8_t flags;
-    int rc = -1;
-
-    if (!msg) {
-        errno = EINVAL;
-        goto done;
-    }
+    if (msg_validate (msg) < 0)
+        return -1;
     json_decref (msg->json);            /* invalidate cached json object */
     msg->json = NULL;
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        goto done;
-    if (!(flags & FLUX_MSGFLAG_PAYLOAD) && (buf == NULL || size == 0)) {
-        rc = 0;
-        goto done;
-    }
-    zf = zmsg_first (msg->zmsg);
-    if ((flags & FLUX_MSGFLAG_ROUTE)) {
-        while (zf && zframe_size (zf) > 0)
-            zf = zmsg_next (msg->zmsg);      /* skip route frame */
-        if (zf)
-            zf = zmsg_next (msg->zmsg);      /* skip route delim */
-    }
-    if ((flags & FLUX_MSGFLAG_TOPIC)) {
-        if (zf)
-            zf = zmsg_next (msg->zmsg);      /* skip topic frame */
-    }
-    if (!zf) {                          /* must at least have proto frame */
-        errno = EPROTO;
-        goto done;
-    }
+    if (!msg_has_payload (msg) && (buf == NULL || size == 0))
+        return 0;
     /* Case #1: replace existing payload.
      */
-    if ((flags & FLUX_MSGFLAG_PAYLOAD) && (buf != NULL && size > 0)) {
-        if (zframe_data (zf) != buf || zframe_size (zf) != size) {
-            if (payload_overlap (buf, zf)) {
+    if (msg_has_payload (msg) && (buf != NULL && size > 0)) {
+        assert (msg->payload);
+        if (msg->payload != buf || msg->payload_size != size) {
+            if (payload_overlap (msg, buf)) {
                 errno = EINVAL;
-                goto done;
+                return -1;
+            }
+        }
+        if (size > msg->payload_size) {
+            void *ptr;
+            if (!(ptr = realloc (msg->payload, size))) {
+                errno = ENOMEM;
+                return -1;
             }
-            zframe_reset (zf, buf, size);
+            msg->payload = ptr;
+            msg->payload_size = size;
         }
+        memcpy (msg->payload, buf, size);
     /* Case #2: add payload.
      */
-    } else if (!(flags & FLUX_MSGFLAG_PAYLOAD) && (buf != NULL && size > 0)) {
-        zmsg_remove (msg->zmsg, zf);
-        if (zmsg_addmem (msg->zmsg, buf, size) < 0
-                                        || zmsg_append (msg->zmsg, &zf) < 0) {
-            errno = ENOMEM;
-            goto done;
-        }
-        flags |= FLUX_MSGFLAG_PAYLOAD;
+    } else if (!msg_has_payload (msg) && (buf != NULL && size > 0)) {
+        assert (!msg->payload);
+        if (!(msg->payload = malloc (size)))
+            return -1;
+        msg->payload_size = size;
+        memcpy (msg->payload, buf, size);
+        msg_set_flag (msg, FLUX_MSGFLAG_PAYLOAD);
     /* Case #3: remove payload.
      */
-    } else if ((flags & FLUX_MSGFLAG_PAYLOAD) && (buf == NULL || size == 0)) {
-        zmsg_remove (msg->zmsg, zf);
-        zframe_destroy (&zf);
-        flags &= ~(uint8_t)(FLUX_MSGFLAG_PAYLOAD);
+    } else if (msg_has_payload (msg) && (buf == NULL || size == 0)) {
+        assert (msg->payload);
+        free (msg->payload);
+        msg->payload = NULL;
+        msg->payload_size = 0;
+        msg_clear_flag (msg, FLUX_MSGFLAG_PAYLOAD);
     }
-    if (flux_msg_set_flags (msg, flags) < 0)
-        goto done;
-    rc = 0;
-done:
-    return rc;
+    return 0;
+}
+
+static inline void msg_lasterr_reset (flux_msg_t *msg)
+{
+    if (msg_validate (msg) == 0) {
+        free (msg->lasterr);
+        msg->lasterr = NULL;
+    }
+}
+
+static inline void msg_lasterr_set (flux_msg_t *msg,
+                                    const char *fmt,
+                                    ...)
+{
+    va_list ap;
+    int saved_errno = errno;
+
+    va_start (ap, fmt);
+    if (vasprintf (&msg->lasterr, fmt, ap) < 0)
+        msg->lasterr = NULL;
+    va_end (ap);
+
+    errno = saved_errno;
 }
 
 int flux_msg_vpack (flux_msg_t *msg, const char *fmt, va_list ap)
 {
     char *json_str = NULL;
-    json_t *json;
+    json_t *json = NULL;
+    json_error_t err;
     int saved_errno;
 
-    if (!(json = json_vpack_ex (NULL, 0, fmt, ap)))
+    msg_lasterr_reset (msg);
+
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!fmt || *fmt == '\0')
         goto error_inval;
-    if (!json_is_object (json))
+
+    if (!(json = json_vpack_ex (&err, 0, fmt, ap))) {
+        msg_lasterr_set (msg, "%s", err.text);
         goto error_inval;
-    if (!(json_str = json_dumps (json, JSON_COMPACT)))
+    }
+    if (!json_is_object (json)) {
+        msg_lasterr_set (msg, "payload is not a JSON object");
         goto error_inval;
-    if (flux_msg_set_string (msg, json_str) < 0)
+    }
+    if (!(json_str = json_dumps (json, JSON_COMPACT))) {
+        msg_lasterr_set (msg, "json_dumps failed on pack result");
+        goto error_inval;
+    }
+    if (flux_msg_set_string (msg, json_str) < 0) {
+        msg_lasterr_set (msg, "flux_msg_set_string: %s", strerror (errno));
         goto error;
+    }
     free (json_str);
     json_decref (json);
     return 0;
@@ -1148,47 +968,26 @@ int flux_msg_pack (flux_msg_t *msg, const char *fmt, ...)
     return rc;
 }
 
-int flux_msg_get_payload (const flux_msg_t *msg, const void **buf, int *size)
+int flux_msg_get_payload (const flux_msg_t *msg, const void **buf, size_t *size)
 {
-    zframe_t *zf;
-    uint8_t flags;
-
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        return -1;
-    if (!(flags & FLUX_MSGFLAG_PAYLOAD)) {
-        errno = EPROTO;
+    if (msg_validate (msg) < 0)
         return -1;
-    }
-    zf = zmsg_first (msg->zmsg);
-    if ((flags & FLUX_MSGFLAG_ROUTE)) {
-        while (zf && zframe_size (zf) > 0)
-            zf = zmsg_next (msg->zmsg);
-        if (zf)
-            zf = zmsg_next (msg->zmsg);
-    }
-    if ((flags & FLUX_MSGFLAG_TOPIC)) {
-        if (zf)
-            zf = zmsg_next (msg->zmsg);
-    }
-    if (!zf) {
+    if (!msg_has_payload (msg)) {
         errno = EPROTO;
         return -1;
     }
     if (buf)
-        *buf = zframe_data (zf);
+        *buf = msg->payload;
     if (size)
-        *size = zframe_size (zf);
+        *size = msg->payload_size;
     return 0;
 }
 
 bool flux_msg_has_payload (const flux_msg_t *msg)
 {
-    uint8_t flags;
-    if (flux_msg_get_flags (msg, &flags) < 0) {
-        errno = 0;
+    if (msg_validate (msg) < 0)
         return false;
-    }
-    return ((flags & FLUX_MSGFLAG_PAYLOAD));
+    return msg_has_payload (msg) ? true : false;
 }
 
 int flux_msg_set_string (flux_msg_t *msg, const char *s)
@@ -1203,26 +1002,24 @@ int flux_msg_set_string (flux_msg_t *msg, const char *s)
 int flux_msg_get_string (const flux_msg_t *msg, const char **s)
 {
     const char *buf;
-    int size;
-    int rc = -1;
+    const char *result;
+    size_t size;
 
-    if (!s) {
-        errno = EINVAL;
-        goto done;
-    }
+    if (msg_validate (msg) < 0)
+        return -1;
     if (flux_msg_get_payload (msg, (const void **)&buf, &size) < 0) {
         errno = 0;
-        *s = NULL;
+        result = NULL;
     } else {
         if (!buf || size == 0 || buf[size - 1] != '\0') {
             errno = EPROTO;
-            goto done;
+            return -1;
         }
-        *s = buf;
+        result = buf;
     }
-    rc = 0;
-done:
-    return rc;
+    if (s)
+        *s = result;
+    return 0;
 }
 
 /* N.B. const attribute of msg argument is defeated internally to
@@ -1231,31 +1028,45 @@ int flux_msg_get_string (const flux_msg_t *msg, const char **s)
  */
 int flux_msg_vunpack (const flux_msg_t *cmsg, const char *fmt, va_list ap)
 {
-    int rc = -1;
     const char *json_str;
-    json_error_t error;
+    json_error_t err;
     flux_msg_t *msg = (flux_msg_t *)cmsg;
 
-    if (!msg || !fmt || *fmt == '\0') {
+    msg_lasterr_reset (msg);
+
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!fmt || *fmt == '\0') {
         errno = EINVAL;
-        goto done;
+        return -1;
     }
     if (!msg->json) {
-        if (flux_msg_get_string (msg, &json_str) < 0)
-            goto done;
-        if (!json_str || !(msg->json = json_loads (json_str, 0, &error))
-                      || !json_is_object (msg->json)) {
+        if (flux_msg_get_string (msg, &json_str) < 0) {
+            msg_lasterr_set (msg, "flux_msg_get_string: %s", strerror (errno));
+            return -1;
+        }
+        if (!json_str) {
+            msg_lasterr_set (msg, "message does not have a string payload");
+            errno = EPROTO;
+            return -1;
+        }
+        if (!(msg->json = json_loads (json_str, JSON_ALLOW_NUL, &err))) {
+            msg_lasterr_set (msg, "%s", err.text);
             errno = EPROTO;
-            goto done;
+            return -1;
+        }
+        if (!json_is_object (msg->json)) {
+            msg_lasterr_set (msg, "payload is not a JSON object");
+            errno = EPROTO;
+            return -1;
         }
     }
-    if (json_vunpack_ex (msg->json, &error, 0, fmt, ap) < 0) {
+    if (json_vunpack_ex (msg->json, &err, 0, fmt, ap) < 0) {
+        msg_lasterr_set (msg, "%s", err.text);
         errno = EPROTO;
-        goto done;
+        return -1;
     }
-    rc = 0;
-done:
-    return rc;
+    return 0;
 }
 
 int flux_msg_unpack (const flux_msg_t *msg, const char *fmt, ...)
@@ -1269,138 +1080,90 @@ int flux_msg_unpack (const flux_msg_t *msg, const char *fmt, ...)
     return rc;
 }
 
-int flux_msg_set_topic (flux_msg_t *msg, const char *topic)
+const char *flux_msg_last_error (const flux_msg_t *msg)
 {
-    zframe_t *zf, *zf2 = NULL;
-    uint8_t flags;
-    int rc = -1;
-
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        goto done;
-    zf = zmsg_first (msg->zmsg);
-    if ((flags & FLUX_MSGFLAG_ROUTE)) {   /* skip over routing frames, if any */
-        while (zf && zframe_size (zf) > 0)
-            zf = zmsg_next (msg->zmsg);
-        if (zf)
-            zf = zmsg_next (msg->zmsg);
-    }
-    if (!zf) {                          /* must at least have proto frame */
-        errno = EPROTO;
-        goto done;
-    }
-    if ((flags & FLUX_MSGFLAG_TOPIC) && topic) {        /* case 1: repl topic */
-        zframe_reset (zf, topic, strlen (topic) + 1);
-    } else if (!(flags & FLUX_MSGFLAG_TOPIC) && topic) {/* case 2: add topic */
-        zmsg_remove (msg->zmsg, zf);
-        if ((flags & FLUX_MSGFLAG_PAYLOAD) && (zf2 = zmsg_next (msg->zmsg)))
-            zmsg_remove (msg->zmsg, zf2);
-        if (zmsg_addmem (msg->zmsg, topic, strlen (topic) + 1) < 0
-                                    || zmsg_append (msg->zmsg, &zf) < 0
-                                    || (zf2 && zmsg_append (msg->zmsg, &zf2) < 0)) {
-            errno = ENOMEM;
-            goto done;
-        }
-        flags |= FLUX_MSGFLAG_TOPIC;
-        if (flux_msg_set_flags (msg, flags) < 0)
-            goto done;
-    } else if ((flags & FLUX_MSGFLAG_TOPIC) && !topic) { /* case 3: del topic */
-        zmsg_remove (msg->zmsg, zf);
-        zframe_destroy (&zf);
-        flags &= ~(uint8_t)FLUX_MSGFLAG_TOPIC;
-        if (flux_msg_set_flags (msg, flags) < 0)
-            goto done;
-    }
-    rc = 0;
-done:
-    return rc;
+    if (!msg)
+        return "msg object is NULL";
+    if (msg->refcount <= 0)
+        return "msg object is unreferenced";
+    if (msg->lasterr == NULL)
+        return "";
+    return msg->lasterr;
 }
 
-static int zf_topic (const flux_msg_t *msg, zframe_t **zfp)
+int flux_msg_set_topic (flux_msg_t *msg, const char *topic)
 {
-    uint8_t flags;
-    zframe_t *zf = NULL;
-    int rc = -1;
-
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        goto done;
-    if (!(flags & FLUX_MSGFLAG_TOPIC)) {
-        errno = EPROTO;
-        goto done;
-    }
-    zf = zmsg_first (msg->zmsg);
-    if ((flags & FLUX_MSGFLAG_ROUTE)) {
-        while (zf && zframe_size (zf) > 0)
-            zf = zmsg_next (msg->zmsg);
-        if (zf)
-            zf = zmsg_next (msg->zmsg);
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (msg_is_control (msg)) {
+        errno = EINVAL;
+        return -1;
     }
-    if (!zf) {
-        errno = EPROTO;
-        goto done;
+    if (msg_has_topic (msg) && topic) {         /* case 1: replace topic */
+        free (msg->topic);
+        if (!(msg->topic = strdup (topic)))
+            return -1;
+    } else if (!msg_has_topic (msg) && topic) { /* case 2: add topic */
+        if (!(msg->topic = strdup (topic)))
+            return -1;
+        msg_set_flag (msg, FLUX_MSGFLAG_TOPIC);
+    } else if (msg_has_topic (msg) && !topic) { /* case 3: delete topic */
+        free (msg->topic);
+        msg->topic = NULL;
+        msg_clear_flag (msg, FLUX_MSGFLAG_TOPIC);
     }
-    *zfp = zf;
-    rc = 0;
-done:
-    return rc;
+    return 0;
 }
 
 int flux_msg_get_topic (const flux_msg_t *msg, const char **topic)
 {
-    zframe_t *zf;
-    const char *s;
-    int rc = -1;
-
-    if (zf_topic (msg, &zf) < 0)
-        goto done;
-    s = (const char *)zframe_data (zf);
-    if (s[zframe_size (zf) - 1] != '\0') {
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (!topic) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!msg_has_topic (msg)) {
         errno = EPROTO;
-        goto done;
+        return -1;
     }
-    *topic = s;
-    rc = 0;
-done:
-    return rc;
+    *topic = msg->topic;
+    return 0;
 }
 
 flux_msg_t *flux_msg_copy (const flux_msg_t *msg, bool payload)
 {
     flux_msg_t *cpy = NULL;
-    zframe_t *zf;
-    int count;
-    uint8_t flags;
-    bool skip_payload = false;
 
-    /* Set skip_payload = true if caller set 'payload' flag false
-     * AND message contains a payload frame.
-     */
-    if (flux_msg_get_flags (msg, &flags) < 0)
+    if (msg_validate (msg) < 0)
         return NULL;
-    if (!payload && (flags & FLUX_MSGFLAG_PAYLOAD)) {
-        flags &= ~(FLUX_MSGFLAG_PAYLOAD);
-        skip_payload = true;
-    }
-    if (!(cpy = flux_msg_create_common ()))
+
+    if (!(cpy = msg_create ()))
         return NULL;
-    if (!(cpy->zmsg = zmsg_new ()))
-        goto nomem;
 
-    /* Copy frames from 'msg' to 'cpy'.
-     * 'count' indexes frames from 0 to zmsg_size (msg) - 1.
-     * The payload frame (if it exists) will be in the second to last position.
-     */
-    count = 0;
-    zf = zmsg_first (msg->zmsg);
-    while (zf) {
-        if (!skip_payload || count != zmsg_size (msg->zmsg) - 2) {
-            if (zmsg_addmem (cpy->zmsg, zframe_data (zf), zframe_size (zf)) < 0)
-                goto nomem;
+    cpy->proto = msg->proto;
+
+    if (flux_msg_route_count (msg) > 0) {
+        struct route_id *r = NULL;
+        list_for_each_rev (&msg->routes, r, route_id_node) {
+            if (flux_msg_route_push (cpy, r->id) < 0)
+                goto error;
         }
-        zf = zmsg_next (msg->zmsg);
-        count++;
     }
-    if (flux_msg_set_flags (cpy, flags) < 0)
-        goto error;
+    if (msg->topic) {
+        if (!(cpy->topic = strdup (msg->topic)))
+            goto nomem;
+    }
+    if (msg->payload) {
+        if (payload) {
+            cpy->payload_size = msg->payload_size;
+            if (!(cpy->payload = malloc (cpy->payload_size)))
+                goto error;
+            memcpy (cpy->payload, msg->payload, msg->payload_size);
+        }
+        else
+            msg_clear_flag (cpy, FLUX_MSGFLAG_PAYLOAD);
+    }
     return cpy;
 nomem:
     errno = ENOMEM;
@@ -1419,131 +1182,243 @@ static struct typemap typemap[] = {
     { "request", ">", FLUX_MSGTYPE_REQUEST },
     { "response", "<", FLUX_MSGTYPE_RESPONSE},
     { "event", "e", FLUX_MSGTYPE_EVENT},
-    { "keepalive", "k", FLUX_MSGTYPE_KEEPALIVE},
+    { "control", "c", FLUX_MSGTYPE_CONTROL},
 };
-static const int typemap_len = sizeof (typemap) / sizeof (typemap[0]);
 
 const char *flux_msg_typestr (int type)
 {
     int i;
 
-    for (i = 0; i < typemap_len; i++)
+    for (i = 0; i < ARRAY_SIZE (typemap); i++)
         if ((type & typemap[i].type))
             return typemap[i].name;
     return "unknown";
 }
 
-static const char *msgtype_shortstr (int type)
+static const char *type2prefix (int type)
 {
     int i;
 
-    for (i = 0; i < typemap_len; i++)
+    for (i = 0; i < ARRAY_SIZE (typemap); i++)
         if ((type & typemap[i].type))
             return typemap[i].sname;
     return "?";
 }
 
-void flux_msg_fprint (FILE *f, const flux_msg_t *msg)
+struct flagmap {
+    const char *name;
+    int flag;
+};
+
+static struct flagmap flagmap[] = {
+    { "topic", FLUX_MSGFLAG_TOPIC},
+    { "payload", FLUX_MSGFLAG_PAYLOAD},
+    { "noresponse", FLUX_MSGFLAG_NORESPONSE},
+    { "route", FLUX_MSGFLAG_ROUTE},
+    { "upstream", FLUX_MSGFLAG_UPSTREAM},
+    { "private", FLUX_MSGFLAG_PRIVATE},
+    { "streaming", FLUX_MSGFLAG_STREAMING},
+};
+
+static void flags2str (int flags, char *buf, int buflen)
+{
+    int i, len = 0;
+    buf[0] = '\0';
+    for (i = 0; i < ARRAY_SIZE (flagmap); i++) {
+        if ((flags & flagmap[i].flag)) {
+            if (len) {
+                assert (len < (buflen - 1));
+                strcat (buf, ",");
+                len++;
+            }
+            assert ((len + strlen (flagmap[i].name)) < (buflen - 1));
+            strcat (buf, flagmap[i].name);
+            len += strlen (flagmap[i].name);
+        }
+    }
+}
+
+static void userid2str (uint32_t userid, char *buf, int buflen)
+{
+    if (userid == FLUX_USERID_UNKNOWN)
+        (void)snprintf (buf, buflen, "unknown");
+    else
+        (void)snprintf (buf, buflen, "%u", userid);
+}
+
+static int roletostr (uint32_t role, const char *sep, char *buf, int buflen)
+{
+    int n;
+    if (role == FLUX_ROLE_OWNER)
+        n = snprintf (buf, buflen, "%sowner", sep);
+    else if (role == FLUX_ROLE_USER)
+        n = snprintf (buf, buflen, "%suser", sep);
+    else if (role == FLUX_ROLE_LOCAL)
+        n = snprintf (buf, buflen, "%slocal", sep);
+    else
+        n = snprintf (buf, buflen, "%s0x%x", sep, role);
+    if (n >= buflen)
+        n = buflen;
+    return n;
+}
+
+static void rolemask2str (uint32_t rolemask, char *buf, int buflen)
+{
+    if (rolemask == FLUX_ROLE_NONE)
+        snprintf (buf, buflen, "none");
+    else if (rolemask == FLUX_ROLE_ALL)
+        snprintf (buf, buflen, "all");
+    else {
+        int offset = 0;
+        for (int i = 0; i < sizeof (rolemask)*8; i++) {
+            if (rolemask & 1<<i) {
+                offset += roletostr (rolemask & 1<<i,
+                                     offset > 0 ? "," : "",
+                                     buf + offset,
+                                     buflen - offset);
+            }
+        }
+    }
+}
+
+static void nodeid2str (uint32_t nodeid, char *buf, int buflen)
+{
+    if (nodeid == FLUX_NODEID_ANY)
+        (void)snprintf (buf, buflen, "any");
+    else if (nodeid == FLUX_NODEID_UPSTREAM)
+        (void)snprintf (buf, buflen, "upstream");
+    else
+        (void)snprintf (buf, buflen, "%u", nodeid);
+}
+
+void flux_msg_fprint_ts (FILE *f, const flux_msg_t *msg, double timestamp)
 {
     int hops;
-    int type = 0;
-    zframe_t *proto;
-    const char *prefix, *topic = NULL;
+    const char *prefix;
+    char flagsstr[128];
+    char useridstr[32];
+    char rolemaskstr[32];
+    char nodeidstr[32];
 
     fprintf (f, "--------------------------------------\n");
     if (!msg) {
         fprintf (f, "NULL");
         return;
     }
-    if (flux_msg_get_type (msg, &type) < 0
-            || (!(proto = zmsg_last (msg->zmsg)))) {
-        fprintf (f, "malformed message");
+    if (msg->refcount <= 0) {
+        fprintf (f, "unref");
         return;
     }
-    prefix = msgtype_shortstr (type);
-    (void)flux_msg_get_topic (msg, &topic);
+    prefix = type2prefix (msg_typeof (msg));
+    /* Timestamp
+     */
+    if (timestamp >= 0.)
+        fprintf (f, "%s %.5f\n", prefix, timestamp);
+    /* Topic (control has none)
+     */
+    if (msg->topic)
+        fprintf (f, "%s %s\n", prefix, msg->topic);
+    /* Proto info
+     */
+    flags2str (msg->proto.flags, flagsstr, sizeof (flagsstr));
+    userid2str (msg->proto.userid, useridstr, sizeof (useridstr));
+    rolemask2str (msg->proto.rolemask, rolemaskstr, sizeof (rolemaskstr));
+    fprintf (f, "%s flags=%s userid=%s rolemask=%s ",
+             prefix, flagsstr, useridstr, rolemaskstr);
+    switch (msg_typeof (msg)) {
+        case FLUX_MSGTYPE_REQUEST:
+            nodeid2str (msg->proto.nodeid, nodeidstr, sizeof (nodeidstr));
+            fprintf (f, "nodeid=%s matchtag=%u\n",
+                     nodeidstr,
+                     msg->proto.matchtag);
+            break;
+        case FLUX_MSGTYPE_RESPONSE:
+            fprintf (f, "errnum=%u matchtag=%u\n",
+                     msg->proto.errnum,
+                     msg->proto.matchtag);
+            break;
+        case FLUX_MSGTYPE_EVENT:
+            fprintf (f, "sequence=%u\n",
+                     msg->proto.sequence);
+            break;
+        case FLUX_MSGTYPE_CONTROL:
+            fprintf (f, "type=%u status=%u\n",
+                     msg->proto.control_type,
+                     msg->proto.control_status);
+            break;
+        default:
+            fprintf (f, "aux1=0x%X aux2=0x%X\n",
+                     msg->proto.aux1,
+                     msg->proto.aux2);
+            break;
+    }
     /* Route stack
      */
-    hops = flux_msg_get_route_count (msg); /* -1 if no route stack */
-    if (hops >= 0) {
-        int len = flux_msg_get_route_size (msg);
-        char *rte = flux_msg_get_route_string (msg);
+    hops = flux_msg_route_count (msg); /* -1 if no route stack */
+    if (hops > 0) {
+        char *rte = flux_msg_route_string (msg);
         assert (rte != NULL);
-        fprintf (f, "%s[%3.3d] |%s|\n", prefix, len, rte);
+        fprintf (f, "%s |%s|\n", prefix, rte);
         free (rte);
     };
-    /* Topic (keepalive has none)
-     */
-    if (topic)
-        fprintf (f, "%s[%3.3zu] %s\n", prefix, strlen (topic), topic);
     /* Payload
      */
     if (flux_msg_has_payload (msg)) {
         const char *s;
         const void *buf;
-        int size;
+        size_t size;
         if (flux_msg_get_string (msg, &s) == 0)
-            fprintf (f, "%s[%3.3zu] %s\n", prefix, strlen (s), s);
-        else if (flux_msg_get_payload (msg, &buf, &size) == 0)
-            fprintf (f, "%s[%3.3d] ...\n", prefix, size);
+            fprintf (f, "%s %s\n", prefix, s);
+        else if (flux_msg_get_payload (msg, &buf, &size) == 0) {
+            /* output at max 80 cols worth of info.  We subtract 2 and
+             * set 'max' to 78 b/c of the prefix taking 2 bytes.
+             */
+            int i, iter, max = 78;
+            bool ellipses = false;
+            fprintf (f, "%s ", prefix);
+            if ((size * 2) > max) {
+                /* -3 for ellipses, divide by 2 b/c 2 chars of output
+                 * per byte */
+                iter = (max - 3) / 2;
+                ellipses = true;
+            }
+            else
+                iter = size;
+            for (i = 0; i < iter; i++)
+                fprintf (f, "%02X", ((uint8_t *)buf)[i]);
+            if (ellipses)
+                fprintf (f, "...");
+            fprintf (f, "\n");
+        }
         else
             fprintf (f, "malformed payload\n");
     }
-    /* Proto block
-     */
-    zframe_fprint (proto, prefix, f);
 }
 
-int flux_msg_sendzsock (void *sock, const flux_msg_t *msg)
+void flux_msg_fprint (FILE *f, const flux_msg_t *msg)
 {
-    int rc = -1;
-
-    if (!sock || !msg || !zmsg_is (msg->zmsg)) {
-        errno = EINVAL;
-        goto done;
-    }
-
-    void *handle = zsock_resolve (sock);
-    int flags = ZFRAME_REUSE | ZFRAME_MORE;
-    zframe_t *zf = zmsg_first (msg->zmsg);
-    size_t count = 0;
-
-    while (zf) {
-        if (++count == zmsg_size (msg->zmsg))
-            flags &= ~ZFRAME_MORE;
-        if (zframe_send (&zf, handle, flags) < 0)
-            goto done;
-        zf = zmsg_next (msg->zmsg);
-    }
-    rc = 0;
-done:
-    return rc;
+    flux_msg_fprint_ts (f, msg, -1);
 }
 
-flux_msg_t *flux_msg_recvzsock (void *sock)
+int msg_frames (const flux_msg_t *msg)
 {
-    zmsg_t *zmsg;
-    flux_msg_t *msg;
-
-    if (!(zmsg = zmsg_recv (sock)))
-        return NULL;
-    if (!(msg = flux_msg_create_common ())) {
-        zmsg_destroy (&zmsg);
-        errno = ENOMEM;
-        return NULL;
+    int n = 1; /* 1 for proto frame */
+    if (msg_validate (msg) < 0)
+        return -1;
+    if (msg_has_payload (msg))
+        n++;
+    if (msg_has_topic (msg))
+        n++;
+    if (msg_has_route (msg)) {
+        /* +1 for routes delimiter frame */
+        n += msg->routes_len + 1;
     }
-    msg->zmsg = zmsg;
-    return msg;
-}
-
-int flux_msg_frames (const flux_msg_t *msg)
-{
-    return zmsg_size (msg->zmsg);
+    return n;
 }
 
 struct flux_match flux_match_init (int typemask,
-                                     uint32_t matchtag,
-                                     const char *topic_glob)
+                                   uint32_t matchtag,
+                                   const char *topic_glob)
 {
     struct flux_match m = {typemask, matchtag, topic_glob};
     return m;
@@ -1554,19 +1429,35 @@ void flux_match_free (struct flux_match m)
     ERRNO_SAFE_WRAP (free, (char *)m.topic_glob);
 }
 
-int flux_match_asprintf (struct flux_match *m, const char *topic_glob_fmt, ...)
+int flux_match_asprintf (struct flux_match *m, const char *fmt, ...)
 {
-    va_list args;
-    va_start (args, topic_glob_fmt);
+    va_list ap;
     char *topic = NULL;
-    int res = vasprintf (&topic, topic_glob_fmt, args);
-    va_end (args);
+    int res;
+
+    va_start (ap, fmt);
+    res = vasprintf (&topic, fmt, ap);
+    va_end (ap);
+    if (res < 0)
+        return -1;
+
     m->topic_glob = topic;
-    return res;
+    return 0;
 }
 
+bool flux_msg_route_match_first (const flux_msg_t *msg1,
+                                 const flux_msg_t *msg2)
+{
+    const char *id1 = flux_msg_route_first (msg1);
+    const char *id2 = flux_msg_route_first (msg2);
+
+    if (!id1 && !id2)
+        return true;
+    if (id1 && id2 && streq (id1, id2))
+        return true;
+    return false;
+}
 
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
-
diff --git a/src/common/libflux/message.h b/src/common/libflux/message.h
index 6ec2f6415bc7..94fab74a6bb9 100644
--- a/src/common/libflux/message.h
+++ b/src/common/libflux/message.h
@@ -16,30 +16,30 @@
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include "types.h"
+#include <sys/types.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct flux_msg flux_msg_t;
-
 enum {
     FLUX_MSGTYPE_REQUEST    = 0x01,
     FLUX_MSGTYPE_RESPONSE   = 0x02,
     FLUX_MSGTYPE_EVENT      = 0x04,
-    FLUX_MSGTYPE_KEEPALIVE  = 0x08,
+    FLUX_MSGTYPE_CONTROL    = 0x08,
     FLUX_MSGTYPE_ANY        = 0x0f,
     FLUX_MSGTYPE_MASK       = 0x0f,
 };
 
 enum {
-    FLUX_MSGFLAG_TOPIC      = 0x01,	/* message has topic string */
-    FLUX_MSGFLAG_PAYLOAD    = 0x02,	/* message has payload */
-    FLUX_MSGFLAG_ROUTE      = 0x08,	/* message is routable */
+    FLUX_MSGFLAG_TOPIC      = 0x01, /* message has topic string */
+    FLUX_MSGFLAG_PAYLOAD    = 0x02, /* message has payload */
+    FLUX_MSGFLAG_NORESPONSE = 0x04, /* request needs no response */
+    FLUX_MSGFLAG_ROUTE      = 0x08, /* message is routable */
     FLUX_MSGFLAG_UPSTREAM   = 0x10, /* request nodeid is sender (route away) */
     FLUX_MSGFLAG_PRIVATE    = 0x20, /* private to instance owner and sender */
     FLUX_MSGFLAG_STREAMING  = 0x40, /* request/response is streaming RPC */
+    FLUX_MSGFLAG_USER1      = 0x80, /* user-defined message flag */
 };
 
 /* N.B. FLUX_NODEID_UPSTREAM should be used in the RPC interface only.
@@ -54,16 +54,18 @@ enum {
 struct flux_match {
     int typemask;           /* bitmask of matching message types (or 0) */
     uint32_t matchtag;      /* matchtag (or FLUX_MATCHTAG_NONE) */
-    const char *topic_glob;       /* glob matching topic string (or NULL) */
+    const char *topic_glob; /* glob matching topic string (or NULL) */
 };
 
 struct flux_match flux_match_init (int typemask,
-                                                   uint32_t matchtag,
-                                                   const char *topic_glob);
+                                   uint32_t matchtag,
+                                   const char *topic_glob);
 
 void flux_match_free (struct flux_match m);
 
-int flux_match_asprintf (struct flux_match *m, const char *topic_glob_fmt, ...);
+int flux_match_asprintf (struct flux_match *m,
+                         const char *topic_glob_fmt,
+                         ...);
 
 #define FLUX_MATCH_ANY flux_match_init( \
     FLUX_MSGTYPE_ANY, \
@@ -86,7 +88,9 @@ int flux_match_asprintf (struct flux_match *m, const char *topic_glob_fmt, ...);
     NULL \
 )
 
-/* Create a new Flux message.
+/* Create a new Flux message.  If the type of the message is not yet
+ * known at creation time, FLUX_MSGTYPE_ANY can be used.
+ *
  * Returns new message or null on failure, with errno set (e.g. ENOMEM, EINVAL)
  * Caller must destroy message with flux_msg_destroy() or equivalent.
  */
@@ -96,8 +100,10 @@ void flux_msg_destroy (flux_msg_t *msg);
 /* Access auxiliary data members in Flux message.
  * These are for convenience only - they are not sent over the wire.
  */
-int flux_msg_aux_set (const flux_msg_t *msg, const char *name,
-                      void *aux, flux_free_f destroy);
+int flux_msg_aux_set (const flux_msg_t *msg,
+                      const char *name,
+                      void *aux,
+                      flux_free_f destroy);
 void *flux_msg_aux_get (const flux_msg_t *msg, const char *name);
 
 /* Duplicate msg, omitting payload if 'payload' is false.
@@ -112,29 +118,15 @@ void flux_msg_decref (const flux_msg_t *msg);
 /* Encode a flux_msg_t to buffer (pre-sized by calling flux_msg_encode_size()).
  * Returns 0 on success, -1 on failure with errno set.
  */
-size_t flux_msg_encode_size (const flux_msg_t *msg);
+ssize_t flux_msg_encode_size (const flux_msg_t *msg);
 int flux_msg_encode (const flux_msg_t *msg, void *buf, size_t size);
 
-/* Get the number of message frames in 'msg'.
- */
-int flux_msg_frames (const flux_msg_t *msg);
-
 /* Decode a flux_msg_t from buffer.
  * Returns message on success, NULL on failure with errno set.
  * Caller must destroy message with flux_msg_destroy().
  */
 flux_msg_t *flux_msg_decode (const void *buf, size_t size);
 
-/* Send message to zeromq socket.
- * Returns 0 on success, -1 on failure with errno set.
- */
-int flux_msg_sendzsock (void *dest, const flux_msg_t *msg);
-
-/* Receive a message from zeromq socket.
- * Returns message on success, NULL on failure with errno set.
- */
-flux_msg_t *flux_msg_recvzsock (void *dest);
-
 /* Get/set message type
  * For FLUX_MSGTYPE_REQUEST: set_type initializes nodeid to FLUX_NODEID_ANY
  * For FLUX_MSGTYPE_RESPONSE: set_type initializes errnum to 0
@@ -156,6 +148,12 @@ bool flux_msg_is_private (const flux_msg_t *msg);
 int flux_msg_set_streaming (flux_msg_t *msg);
 bool flux_msg_is_streaming (const flux_msg_t *msg);
 
+/* Get/set noresponse flag.
+ * Request is advisory and should not receive a response.
+ */
+int flux_msg_set_noresponse (flux_msg_t *msg);
+bool flux_msg_is_noresponse (const flux_msg_t *msg);
+
 /* Get/set/compare message topic string.
  * set adds/deletes/replaces topic frame as needed.
  */
@@ -168,22 +166,22 @@ int flux_msg_get_topic (const flux_msg_t *msg, const char **topic);
  * Any old payload is deleted.
  * flux_msg_get_payload returns pointer to msg-owned buf.
  */
-int flux_msg_get_payload (const flux_msg_t *msg, const void **buf, int *size);
-int flux_msg_set_payload (flux_msg_t *msg, const void *buf, int size);
+int flux_msg_get_payload (const flux_msg_t *msg,
+                          const void **buf,
+                          size_t *size);
+int flux_msg_set_payload (flux_msg_t *msg, const void *buf, size_t size);
 bool flux_msg_has_payload (const flux_msg_t *msg);
 
-/* Get/set flags
- * Users should avoid using flux_msg_set_flags(), and instead use the
- * higher level functions that manipulate message flags.  It is exposed
- * mainly for testing.
+/* Test/set/clear message flags
  */
-int flux_msg_get_flags (const flux_msg_t *msg, uint8_t *flags);
-int flux_msg_set_flags (flux_msg_t *msg, uint8_t flags);
+bool flux_msg_has_flag (const flux_msg_t *msg, int flag);
+int flux_msg_set_flag (flux_msg_t *msg, int flag);
+int flux_msg_clear_flag (flux_msg_t *msg, int flag);
 
 /* Get/set string payload.
  * flux_msg_set_string() accepts a NULL 's' (no payload).
  * flux_msg_get_string() will set 's' to NULL if there is no payload
- * N.B. the raw paylaod includes C string \0 terminator.
+ * N.B. the raw payload includes C string \0 terminator.
  */
 int flux_msg_set_string (flux_msg_t *msg, const char *);
 int flux_msg_get_string (const flux_msg_t *msg, const char **s);
@@ -198,6 +196,15 @@ int flux_msg_vpack (flux_msg_t *msg, const char *fmt, va_list ap);
 int flux_msg_unpack (const flux_msg_t *msg, const char *fmt, ...);
 int flux_msg_vunpack (const flux_msg_t *msg, const char *fmt, va_list ap);
 
+/* Return a string representation of the last error encountered for `msg`.
+ *
+ * If no last error is available, an empty string will be returned.
+ *
+ * Currently, only flux_msg_pack/unpack() (and related) functions will set
+ * the last error for `msg`. (Useful to get more detail from EPROTO errors)
+ */
+const char *flux_msg_last_error (const flux_msg_t *msg);
+
 /* Get/set nodeid (request only)
  */
 int flux_msg_set_nodeid (flux_msg_t *msg, uint32_t nodeid);
@@ -217,6 +224,7 @@ enum {
     FLUX_ROLE_NONE = 0,
     FLUX_ROLE_OWNER = 1,
     FLUX_ROLE_USER = 2,
+    FLUX_ROLE_LOCAL = 4,
     FLUX_ROLE_ALL = 0xFFFFFFFF,
 };
 int flux_msg_set_rolemask (flux_msg_t *msg, uint32_t rolemask);
@@ -239,12 +247,17 @@ int flux_msg_set_cred (flux_msg_t *msg, struct flux_msg_cred cred);
  */
 int flux_msg_cred_authorize (struct flux_msg_cred cred, uint32_t userid);
 
-/* Convenience functions that calls
+/* Convenience function that calls
  * flux_msg_get_cred() + flux_msg_cred_authorize().
  */
 int flux_msg_authorize (const flux_msg_t *msg, uint32_t userid);
 
-/* Get/set errnum (response/keepalive only)
+/* Return true if 'msg' credential carries FLUX_ROLE_LOCAL, indicating
+ * that the message has not traversed brokers.
+ */
+bool flux_msg_is_local (const flux_msg_t *msg);
+
+/* Get/set errnum (response only)
  */
 int flux_msg_set_errnum (flux_msg_t *msg, int errnum);
 int flux_msg_get_errnum (const flux_msg_t *msg, int *errnum);
@@ -254,10 +267,10 @@ int flux_msg_get_errnum (const flux_msg_t *msg, int *errnum);
 int flux_msg_set_seq (flux_msg_t *msg, uint32_t seq);
 int flux_msg_get_seq (const flux_msg_t *msg, uint32_t *seq);
 
-/* Get/set status (keepalive only)
+/* Get/set type, status (control only)
  */
-int flux_msg_set_status (flux_msg_t *msg, int status);
-int flux_msg_get_status (const flux_msg_t *msg, int *status);
+int flux_msg_set_control (flux_msg_t *msg, int type, int status);
+int flux_msg_get_control (const flux_msg_t *msg, int *type, int *status);
 
 /* Get/set/compare match tag (request/response only)
  */
@@ -275,6 +288,7 @@ bool flux_msg_cmp (const flux_msg_t *msg, struct flux_match match);
 /* Print a Flux message on specified output stream.
  */
 void flux_msg_fprint (FILE *f, const flux_msg_t *msg);
+void flux_msg_fprint_ts (FILE *f, const flux_msg_t *msg, double timestamp);
 
 /* Convert a numeric FLUX_MSGTYPE value to string,
  * or "unknown" if unrecognized.
@@ -287,57 +301,67 @@ const char *flux_msg_typestr (int type);
  * routing.
  */
 
-/* Prepare a message for routing, which consists of pushing a nil delimiter
- * frame and setting FLUX_MSGFLAG_ROUTE.  This function is a no-op if the
- * flag is already set.
- * Returns 0 on success, -1 with errno set on failure.
+/* Enable routes on a message by setting FLUX_MSGFLAG_ROUTE.  This
+ * function is a no-op if the flag is already set.
+ */
+void flux_msg_route_enable (flux_msg_t *msg);
+
+/* Disable routes on a message by clearing the FLUX_MSGFLAG_ROUTE
+ * flag.  In addition, clear all presently stored routes.  This
+ * function is a no-op if the flag is already set.
  */
-int flux_msg_enable_route (flux_msg_t *msg);
+void flux_msg_route_disable (flux_msg_t *msg);
 
-/* Strip route frames, nil delimiter, and clear FLUX_MSGFLAG_ROUTE flag.
- * This function is a no-op if the flag is already clear.
- * Returns 0 on success, -1 with errno set on failure.
+/* Clear all routes stored in a message.  This function is a no-op if
+ * routes are not enabled.
  */
-int flux_msg_clear_route (flux_msg_t *msg);
+void flux_msg_route_clear (flux_msg_t *msg);
 
 /* Push a route frame onto the message (mimic what dealer socket does).
  * 'id' is copied internally.
  * Returns 0 on success, -1 with errno set (e.g. EINVAL) on failure.
  */
-int flux_msg_push_route (flux_msg_t *msg, const char *id);
+int flux_msg_route_push (flux_msg_t *msg, const char *id);
 
-/* Pop a route frame off the message and return identity (or NULL) in 'id'.
- * Caller must free 'id'.
+/* Delete last route frame off the message.  Effectively performs the
+ * "opposite" of flux_msg_route_push().
+ *
  * Returns 0 on success, -1 with errno set (e.g. EPROTO) on failure.
  */
-int flux_msg_pop_route (flux_msg_t *msg, char **id);
+int flux_msg_route_delete_last (flux_msg_t *msg);
 
-/* Copy the first routing frame (closest to delimiter) contents (or NULL)
- * to 'id'.  Caller must free 'id'.
+/* Return the first route (e.g. first pushed route) or NULL if there
+ * are no routes.
  * For requests, this is the sender; for responses, this is the recipient.
- * Returns 0 on success, -1 with errno set (e.g. EPROTO) on failure.
+ * Returns route id on success, NULL for no route or error.
  */
-int flux_msg_get_route_first (const flux_msg_t *msg, char **id); /* closest to delim */
+const char *flux_msg_route_first (const flux_msg_t *msg);
 
-/* Copy the last routing frame (farthest from delimiter) contents (or NULL)
- * to 'id'.  Caller must free 'id'.
+/* Return the last route (e.g. most recent pushed route) or NULL if there
+ * are no routes.
  * For requests, this is the last hop; for responses: this is the next hop.
- * Returns 0 on success, -1 with errno set (e.g. EPROTO) on failure.
+ * Returns route id on success, NULL for no route or error.
  */
-int flux_msg_get_route_last (const flux_msg_t *msg, char **id); /* farthest from delim */
+const char *flux_msg_route_last (const flux_msg_t *msg);
 
 /* Return the number of route frames in the message.
  * It is an EPROTO error if there is no route stack.
  * Returns 0 on success, -1 with errno set (e.g. EPROTO) on failure.
  */
-int flux_msg_get_route_count (const flux_msg_t *msg);
+int flux_msg_route_count (const flux_msg_t *msg);
 
 /* Return a string representing the route stack in message.
- * Return NULL if there is no route delimiter; empty string if
+ * Return NULL if routes are not enabled; empty string if
  * the route stack contains no route frames).
  * Caller must free the returned string.
  */
-char *flux_msg_get_route_string (const flux_msg_t *msg);
+char *flux_msg_route_string (const flux_msg_t *msg);
+
+/* Return true if messages have the same first routing frame.
+ * (For requests, the sender)
+ */
+bool flux_msg_route_match_first (const flux_msg_t *msg1,
+                                 const flux_msg_t *msg2);
 
 #ifdef __cplusplus
 }
diff --git a/src/common/libflux/message_iovec.c b/src/common/libflux/message_iovec.c
new file mode 100644
index 000000000000..bc73d6780357
--- /dev/null
+++ b/src/common/libflux/message_iovec.c
@@ -0,0 +1,165 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <stdbool.h>
+#include <string.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <inttypes.h>
+#include <flux/core.h>
+
+#include "message_private.h"
+#include "message_route.h"
+#include "message_proto.h"
+#include "message_iovec.h"
+
+flux_msg_t *iovec_to_msg (struct msg_iovec *iov, int iovcnt)
+{
+    unsigned int index = 0;
+    flux_msg_t *msg;
+
+    assert (iov);
+
+    if (iovcnt < 1) {
+        errno = EPROTO;
+        return NULL;
+    }
+    if (!(msg = msg_create ()))
+        return NULL;
+
+    /* proto frame is last frame */
+    if (proto_decode (&msg->proto,
+                      iov[iovcnt - 1].data,
+                      iov[iovcnt - 1].size) < 0)
+        goto error;
+    if (!msg_type_is_valid (msg)) {
+        errno = EPROTO;
+        goto error;
+    }
+    if (msg_has_route (msg)) {
+        /* On first access index == 0 && iovcnt > 0 guaranteed
+         * Re-add check if code changes. */
+        /* if (index == iovcnt) { */
+        /*     errno = EPROTO; */
+        /*     return -1; */
+        /* } */
+        while ((index < iovcnt) && iov[index].size > 0) {
+            if (msg_route_append (msg,
+                                  (char *)iov[index].data,
+                                  iov[index].size) < 0)
+                goto error;
+            index++;
+        }
+        if (index < iovcnt)
+            index++;
+    }
+    if (msg_has_topic (msg)) {
+        if (index == iovcnt) {
+            errno = EPROTO;
+            goto error;
+        }
+        if (!(msg->topic = strndup ((char *)iov[index].data,
+                                    iov[index].size)))
+            goto error;
+        if (index < iovcnt)
+            index++;
+    }
+    if (msg_has_payload (msg)) {
+        if (index == iovcnt) {
+            errno = EPROTO;
+            goto error;
+        }
+        msg->payload_size = iov[index].size;
+        if (!(msg->payload = malloc (msg->payload_size)))
+            goto error;
+        memcpy (msg->payload, iov[index].data, msg->payload_size);
+        if (index < iovcnt)
+            index++;
+    }
+    /* proto frame required */
+    if (index == iovcnt) {
+        errno = EPROTO;
+        goto error;
+    }
+    return msg;
+error:
+    flux_msg_decref (msg);
+    return NULL;
+}
+
+int msg_to_iovec (const flux_msg_t *msg,
+                  uint8_t *proto,
+                  int proto_len,
+                  struct msg_iovec **iovp,
+                  int *iovcntp)
+{
+    struct msg_iovec *iov = NULL;
+    int index;
+    int frame_count;
+
+    /* msg never completed initial setup */
+    if (!msg_type_is_valid (msg)) {
+        errno = EPROTO;
+        return -1;
+    }
+    if (proto_encode (&msg->proto, proto, proto_len) < 0)
+        return -1;
+
+    if ((frame_count = msg_frames (msg)) < 0)
+        return -1;
+
+    assert (frame_count);
+
+    if (!(iov = malloc (frame_count * sizeof (*iov))))
+        return -1;
+
+    index = frame_count - 1;
+
+    iov[index].data = proto;
+    iov[index].size = PROTO_SIZE;
+    if (msg_has_payload (msg)) {
+        index--;
+        assert (index >= 0);
+        iov[index].data = msg->payload;
+        iov[index].size = msg->payload_size;
+    }
+    if (msg_has_topic (msg)) {
+        index--;
+        assert (index >= 0);
+        iov[index].data = msg->topic;
+        iov[index].size = strlen (msg->topic);
+    }
+    if (msg_has_route (msg)) {
+        struct route_id *r = NULL;
+        /* delimiter */
+        index--;
+        assert (index >= 0);
+        iov[index].data = NULL;
+        iov[index].size = 0;
+        list_for_each_rev (&msg->routes, r, route_id_node) {
+            index--;
+            assert (index >= 0);
+            iov[index].data = r->id;
+            iov[index].size = strlen (r->id);
+        }
+    }
+    (*iovp) = iov;
+    (*iovcntp) = frame_count;
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libflux/message_iovec.h b/src/common/libflux/message_iovec.h
new file mode 100644
index 000000000000..213380500fa1
--- /dev/null
+++ b/src/common/libflux/message_iovec.h
@@ -0,0 +1,39 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_MESSAGE_IOVEC_H
+#define _FLUX_CORE_MESSAGE_IOVEC_H
+
+#define IOVECINCR           4
+
+/* 'transport_data' is for any auxiliary transport data user may wish
+ * to associate with iovec, user is responsible to free/destroy the
+ * field
+ */
+struct msg_iovec {
+    const void *data;
+    size_t size;
+    void *transport_data;
+};
+
+flux_msg_t *iovec_to_msg (struct msg_iovec *iov, int iovcnt);
+
+int msg_to_iovec (const flux_msg_t *msg,
+                  uint8_t *proto,
+                  int proto_len,
+                  struct msg_iovec **iovp,
+                  int *iovcntp);
+
+#endif /* !_FLUX_CORE_MESSAGE_IOVEC_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libflux/message_private.h b/src/common/libflux/message_private.h
new file mode 100644
index 000000000000..fddcf3b6a7aa
--- /dev/null
+++ b/src/common/libflux/message_private.h
@@ -0,0 +1,83 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_MESSAGE_PRIVATE_H
+#define _FLUX_CORE_MESSAGE_PRIVATE_H
+
+#include <stdint.h>
+#include <jansson.h>
+
+#include "ccan/list/list.h"
+
+#include "message_proto.h"
+
+struct flux_msg {
+    // optional route list, if FLUX_MSGFLAG_ROUTE
+    struct list_head routes;
+    int routes_len;     /* to avoid looping */
+
+    // optional topic frame, if FLUX_MSGFLAG_TOPIC
+    char *topic;
+
+    // optional payload frame, if FLUX_MSGFLAG_PAYLOAD
+    void *payload;
+    size_t payload_size;
+
+    // required proto frame data
+    struct proto proto;
+
+    json_t *json;
+    char *lasterr;
+    struct aux_item *aux;
+    int refcount;
+    struct list_node list; // for use by msg_deque container only
+};
+
+flux_msg_t *msg_create (void);
+
+int msg_frames (const flux_msg_t *msg);
+
+#define msgtype_is_valid(tp) \
+    ((tp) == FLUX_MSGTYPE_REQUEST || (tp) == FLUX_MSGTYPE_RESPONSE \
+     || (tp) == FLUX_MSGTYPE_EVENT || (tp) == FLUX_MSGTYPE_CONTROL)
+
+#define msg_typeof(msg)         ((msg)->proto.type)
+#define msg_type_is_valid(msg)  (msgtype_is_valid (msg_typeof (msg)))
+#define msg_is_request(msg)     (msg_typeof(msg) == FLUX_MSGTYPE_REQUEST)
+#define msg_is_response(msg)    (msg_typeof(msg) == FLUX_MSGTYPE_RESPONSE)
+#define msg_is_event(msg)       (msg_typeof(msg) == FLUX_MSGTYPE_EVENT)
+#define msg_is_control(msg)     (msg_typeof(msg) == FLUX_MSGTYPE_CONTROL)
+
+#define msg_has_flag(msg,flag)  ((msg)->proto.flags & (flag))
+#define msg_set_flag(msg,flag)  ((msg)->proto.flags |= (flag))
+#define msg_clear_flag(msg,flag) \
+                                ((msg)->proto.flags &= ~(flag))
+#define msg_has_topic(msg)      (msg_has_flag(msg, FLUX_MSGFLAG_TOPIC))
+#define msg_has_payload(msg)    (msg_has_flag(msg, FLUX_MSGFLAG_PAYLOAD))
+#define msg_has_noresponse(msg) (msg_has_flag(msg, FLUX_MSGFLAG_NORESPONSE))
+#define msg_has_route(msg)      (msg_has_flag(msg, FLUX_MSGFLAG_ROUTE))
+#define msg_has_upstream(msg)   (msg_has_flag(msg, FLUX_MSGFLAG_UPSTREAM))
+#define msg_has_private(msg)    (msg_has_flag(msg, FLUX_MSGFLAG_PRIVATE))
+#define msg_has_streaming(msg)  (msg_has_flag(msg, FLUX_MSGFLAG_STREAMING))
+#define msg_has_user1(msg)      (msg_has_flag(msg, FLUX_MSGFLAG_USER1))
+
+#define msgflags_is_valid(fl) \
+    (((fl) & ~(FLUX_MSGFLAG_TOPIC | FLUX_MSGFLAG_PAYLOAD | FLUX_MSGFLAG_ROUTE \
+     | FLUX_MSGFLAG_UPSTREAM | FLUX_MSGFLAG_PRIVATE | FLUX_MSGFLAG_STREAMING \
+     | FLUX_MSGFLAG_NORESPONSE | FLUX_MSGFLAG_USER1)) == 0 \
+     && !(((fl) & FLUX_MSGFLAG_NORESPONSE) && ((fl) & FLUX_MSGFLAG_STREAMING)))
+#define msg_flags_is_valid(msg) (msgflags_is_valid ((msg)->proto.flags))
+
+#endif /* !_FLUX_CORE_MESSAGE_PRIVATE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libflux/message_proto.c b/src/common/libflux/message_proto.c
new file mode 100644
index 000000000000..a78f251a9777
--- /dev/null
+++ b/src/common/libflux/message_proto.c
@@ -0,0 +1,102 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* message_proto.c - marshal RFC 3 PROTO frame */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <flux/core.h>
+
+#include "ccan/pushpull/pushpull.h"
+#include "ccan/endian/endian.h"
+
+#include "message_proto.h"
+
+/* push/pull functions that use network byte order (big endian)
+ */
+
+static bool be_pull_u32 (const char **p, size_t *max_len, uint32_t *val)
+{
+    beint32_t v;
+
+    if (pull_bytes (p, max_len, &v, sizeof (v))) {
+        if (val)
+            *val = be32_to_cpu (v);
+        return true;
+    }
+    return false;
+}
+
+static bool be_push_u32(char **p, size_t *len, uint32_t val)
+{
+    beint32_t v = cpu_to_be32 (val);
+    return push_bytes (p, len, &v, sizeof (v));
+}
+
+/* Realloc(3) replacement for push() that simply returns the pointer, unless
+ * the requested length exceeds the fixed size of an encoded proto frame.
+ * It assumes proto_encode() has received a static buffer >= PROTO_SIZE.
+ */
+static void *proto_realloc (void *ptr, size_t len)
+{
+    if (len > PROTO_SIZE)
+        return NULL;
+    return ptr;
+}
+
+int proto_encode (const struct proto *proto, void *buf, size_t size)
+{
+    char *cp = buf;
+    size_t len = 0;
+
+    push_set_realloc (proto_realloc);
+
+    if (size < PROTO_SIZE
+        || !push_u8 (&cp, &len, PROTO_MAGIC)
+        || !push_u8 (&cp, &len, PROTO_VERSION)
+        || !push_u8 (&cp, &len, proto->type)
+        || !push_u8 (&cp, &len, proto->flags)
+        || !be_push_u32 (&cp, &len, proto->userid)
+        || !be_push_u32 (&cp, &len, proto->rolemask)
+        || !be_push_u32 (&cp, &len, proto->aux1)
+        || !be_push_u32 (&cp, &len, proto->aux2)
+        || len < size) {
+        errno = EINVAL;
+        return -1;
+    }
+    return 0;
+}
+
+int proto_decode (struct proto *proto, const void *buf, size_t size)
+{
+    const char *cp = buf;
+    size_t len = size;
+    uint8_t magic, version;
+
+    if (!pull_u8 (&cp, &len, &magic)
+        || magic != PROTO_MAGIC
+        || !pull_u8 (&cp, &len, &version)
+        || version != PROTO_VERSION
+        || !pull_u8 (&cp, &len, &proto->type)
+        || !pull_u8 (&cp, &len, &proto->flags)
+        || !be_pull_u32 (&cp, &len, &proto->userid)
+        || !be_pull_u32 (&cp, &len, &proto->rolemask)
+        || !be_pull_u32 (&cp, &len, &proto->aux1)
+        || !be_pull_u32 (&cp, &len, &proto->aux2)
+        || len > 0) {
+        errno = EPROTO;
+        return -1;
+    }
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libflux/message_proto.h b/src/common/libflux/message_proto.h
new file mode 100644
index 000000000000..2298cb5ed4d1
--- /dev/null
+++ b/src/common/libflux/message_proto.h
@@ -0,0 +1,42 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_MESSAGE_PROTO_H
+#define _FLUX_CORE_MESSAGE_PROTO_H
+
+#define PROTO_MAGIC         0x8e
+#define PROTO_VERSION       1
+#define PROTO_SIZE          20
+
+struct proto {
+    uint8_t type;
+    uint8_t flags;
+    uint32_t userid;
+    uint32_t rolemask;
+    union {
+        uint32_t nodeid;  // request
+        uint32_t sequence; // event
+        uint32_t errnum; // response
+        uint32_t control_type; // control
+        uint32_t aux1; // common accessor
+    };
+    union {
+        uint32_t matchtag; // request, response
+        uint32_t control_status; // control
+        uint32_t aux2; // common accessor
+    };
+};
+
+int proto_encode (const struct proto *proto, void *buf, size_t size);
+int proto_decode (struct proto *proto, const void *buf, size_t size);
+
+#endif /* !_FLUX_CORE_MESSAGE_PROTO_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libflux/message_route.c b/src/common/libflux/message_route.c
new file mode 100644
index 000000000000..225d9edb5784
--- /dev/null
+++ b/src/common/libflux/message_route.c
@@ -0,0 +1,99 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <stdbool.h>
+#include <string.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <inttypes.h>
+#include <flux/core.h>
+
+#include "message.h"
+#include "message_private.h"
+#include "message_route.h"
+
+static void route_id_destroy (void *data)
+{
+    if (data) {
+        struct route_id *r = data;
+        free (r);
+    }
+}
+
+static struct route_id *route_id_create (const char *id, unsigned int id_len)
+{
+    struct route_id *r;
+    if (!(r = calloc (1, sizeof (*r) + id_len + 1)))
+        return NULL;
+    if (id && id_len) {
+        memcpy (r->id, id, id_len);
+        list_node_init (&(r->route_id_node));
+    }
+    return r;
+}
+
+int msg_route_push (flux_msg_t *msg,
+                    const char *id,
+                    unsigned int id_len)
+{
+    struct route_id *r;
+    if (!(r = route_id_create (id, strlen (id))))
+        return -1;
+    list_add (&msg->routes, &r->route_id_node);
+    msg->routes_len++;
+    return 0;
+}
+
+int msg_route_append (flux_msg_t *msg,
+                      const char *id,
+                      unsigned int id_len)
+{
+    struct route_id *r;
+    assert (msg);
+    assert (msg_has_route (msg));
+    assert (id);
+    if (!(r = route_id_create (id, id_len)))
+        return -1;
+    list_add_tail (&msg->routes, &r->route_id_node);
+    msg->routes_len++;
+    return 0;
+}
+
+void msg_route_clear (flux_msg_t *msg)
+{
+    struct route_id *r;
+    assert (msg);
+    assert (msg_has_route (msg));
+    while ((r = list_pop (&msg->routes, struct route_id, route_id_node)))
+        route_id_destroy (r);
+    list_head_init (&msg->routes);
+    msg->routes_len = 0;
+}
+
+int msg_route_delete_last (flux_msg_t *msg)
+{
+    struct route_id *r;
+    assert (msg);
+    assert (msg_has_route (msg));
+    if ((r = list_pop (&msg->routes, struct route_id, route_id_node))) {
+        route_id_destroy (r);
+        msg->routes_len--;
+    }
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libflux/message_route.h b/src/common/libflux/message_route.h
new file mode 100644
index 000000000000..2844a2b6bc24
--- /dev/null
+++ b/src/common/libflux/message_route.h
@@ -0,0 +1,36 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_MESSAGE_ROUTE_H
+#define _FLUX_CORE_MESSAGE_ROUTE_H
+
+struct route_id {
+    struct list_node route_id_node;
+    char id[0];                 /* variable length id stored at end of struct */
+};
+
+int msg_route_push (flux_msg_t *msg,
+                    const char *id,
+                    unsigned int id_len);
+
+int msg_route_append (flux_msg_t *msg,
+                      const char *id,
+                      unsigned int id_len);
+
+void msg_route_clear (flux_msg_t *msg);
+
+int msg_route_delete_last (flux_msg_t *msg);
+
+#endif /* !_FLUX_CORE_MESSAGE_ROUTE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libflux/module.c b/src/common/libflux/module.c
index cd4643ff377a..f0268d8aa463 100644
--- a/src/common/libflux/module.c
+++ b/src/common/libflux/module.c
@@ -12,108 +12,14 @@
 #include "config.h"
 #endif
 #include <dlfcn.h>
-#include <argz.h>
 #include <dirent.h>
 #include <sys/stat.h>
 #include <jansson.h>
-
-#include "module.h"
-#include "message.h"
-#include "rpc.h"
+#include <flux/core.h>
 
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/dirwalk.h"
 
-struct modfind_ctx {
-    const char *modname;
-    flux_moderr_f *cb;
-    void *arg;
-};
-
-char *flux_modname (const char *path, flux_moderr_f *cb, void *arg)
-{
-    void *dso;
-    const char **np;
-    char *cpy = NULL;
-    int saved_errno;
-
-    if (!path) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if (!(dso = dlopen (path, RTLD_LAZY | RTLD_LOCAL | FLUX_DEEPBIND))) {
-        if (cb)
-            cb (dlerror (), arg);
-        errno = ENOENT;
-        return NULL;
-    }
-    dlerror ();
-    if (!(np = dlsym (dso, "mod_name"))) {
-        char *errmsg;
-        if (cb && (errmsg = dlerror ()))
-            cb (errmsg, arg);
-        errno = EINVAL;
-        goto error;
-    }
-    if (!*np) {
-        errno = EINVAL;
-        goto error;
-    }
-    if (!(cpy = strdup (*np)))
-        goto error;
-    dlclose (dso);
-    return cpy;
-error:
-    saved_errno = errno;
-    dlclose (dso);
-    errno = saved_errno;
-    return NULL;
-}
-
-/* dirwalk_filter_f callback for dirwalk_find()
- * This function should return 1 on match, 0 on no match.
- * dirwalk_find() will stop on first match since its count parameter is 1.
- */
-static int mod_find_f (dirwalk_t *d, void *arg)
-{
-    struct modfind_ctx *ctx = arg;
-    const char *path = dirwalk_path (d);
-    char *name;
-    int rc = 0;
-
-    if ((name = flux_modname (path, ctx->cb, ctx->arg))) {
-        if (!strcmp (name, ctx->modname))
-            rc = 1;
-        free (name);
-    }
-    return rc;
-}
-
-char *flux_modfind (const char *searchpath, const char *modname,
-                    flux_moderr_f *cb, void *arg)
-{
-    char *result = NULL;
-    zlist_t *l;
-    struct modfind_ctx ctx;
-
-    if (!searchpath || !modname) {
-        errno = EINVAL;
-        return NULL;
-    }
-    ctx.modname = modname;
-    ctx.cb = cb;
-    ctx.arg = arg;
-
-    l = dirwalk_find (searchpath, 0, "*.so", 1, mod_find_f, &ctx);
-    if (l) {
-        result = zlist_pop (l);
-        zlist_destroy (&l);
-    }
-    if (!result)
-        errno = ENOENT;
-    return result;
-}
-
 bool flux_module_debug_test (flux_t *h, int flag, bool clear)
 {
     int *flagsp = flux_aux_get (h, "flux::debug_flags");
@@ -125,6 +31,21 @@ bool flux_module_debug_test (flux_t *h, int flag, bool clear)
     return true;
 }
 
+int flux_module_set_running (flux_t *h)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_rpc_pack (h,
+                             "module.status",
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_NORESPONSE,
+                             "{s:i}",
+                             "status", FLUX_MODSTATE_RUNNING)))
+        return -1;
+    flux_future_destroy (f);
+    return 0;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libflux/module.h b/src/common/libflux/module.h
index 682755bb2725..8aff4170778c 100644
--- a/src/common/libflux/module.h
+++ b/src/common/libflux/module.h
@@ -11,15 +11,9 @@
 #ifndef _FLUX_CORE_MODULE_H
 #define _FLUX_CORE_MODULE_H
 
-/* Module management messages are constructed according to Flux RFC 5.
- * https://github.com/flux-framework/rfc/blob/master/spec_5.adoc
- */
-
 #include <stdint.h>
 #include <stdbool.h>
 
-#include "handle.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -28,10 +22,9 @@ extern "C" {
  */
 enum {
     FLUX_MODSTATE_INIT           = 0,
-    FLUX_MODSTATE_SLEEPING       = 1,
-    FLUX_MODSTATE_RUNNING        = 2,
-    FLUX_MODSTATE_FINALIZING     = 3,
-    FLUX_MODSTATE_EXITED         = 4,
+    FLUX_MODSTATE_RUNNING        = 1,
+    FLUX_MODSTATE_FINALIZING     = 2,
+    FLUX_MODSTATE_EXITED         = 3,
 };
 
 /* Mandatory symbols for modules
@@ -39,22 +32,6 @@ enum {
 #define MOD_NAME(x) const char *mod_name = x
 typedef int (mod_main_f)(flux_t *h, int argc, char *argv[]);
 
-typedef void (flux_moderr_f)(const char *errmsg, void *arg);
-
-/* Read the value of 'mod_name' from the specified module filename.
- * Caller must free the returned name.  Returns NULL on failure.
- * If 'cb' is non-NULL, any dlopen/dlsym errors are reported via callback.
- */
-char *flux_modname (const char *filename, flux_moderr_f *cb, void *arg);
-
-/* Search a colon-separated list of directories (recursively) for a .so file
- * with the requested module name and return its path, or NULL on failure.
- * Caller must free the returned path.
- * If 'cb' is non-NULL, any dlopen/dlsym errors are reported via callback.
- */
-char *flux_modfind (const char *searchpath, const char *modname,
-                    flux_moderr_f *cb, void *arg);
-
 /* Test and optionally clear module debug bit from within a module, as
  * described in RFC 5.  Return true if 'flag' bit is set.  If clear=true,
  * clear the bit after testing.  The flux-module(1) debug subcommand
@@ -62,6 +39,13 @@ char *flux_modfind (const char *searchpath, const char *modname,
  */
 bool flux_module_debug_test (flux_t *h, int flag, bool clear);
 
+/* Set module state to RUNNING.  This transition occurs automatically when the
+ * reactor is entered, but this function can set the state to RUNNING early,
+ * e.g. if flux module load must complete before the module enters the reactor.
+ * Returns 0 on success, -1 on error with errno set.
+ */
+int flux_module_set_running (flux_t *h);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/common/libflux/msg_deque.c b/src/common/libflux/msg_deque.c
new file mode 100644
index 000000000000..f658c7475fd5
--- /dev/null
+++ b/src/common/libflux/msg_deque.c
@@ -0,0 +1,266 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* msg_dequeue.c - reactive, thread-safe, output-restricted message deque */
+
+/* The pollfd/pollevents pattern was borrowed from zeromq's ZMQ_EVENTS/ZMQ_FD,
+ * described in zmq_getsockopt(3).  It is an edge-triggered notification system
+ * in which the pollfd, a special file descriptor created with eventfd(2),
+ * can be watched reactively for a POLLIN event, then the actual event on the
+ * queue is determined by sampling pollevents.  The valid pollevents bits are:
+ *
+ * POLLIN   messages are available to pop
+ * POLLOUT  messages may be pushed (always asserted currently)
+ *
+ * The pollevents should not be confused with pollfd events.  In pollfd, only
+ * POLLIN is expected, signaling that one of the bits is newly set in
+ * pollevents, and used to wake up a reactor loop to service those bits.
+ *
+ * "edge-triggered" means that pollfd does not reassert if the reactor handler
+ * returns with the condition that caused the event still true.  In the case
+ * of msg_deque POLLIN events, a handler must pop all messages before
+ * returning, or if fairness is a concern (one message queue starving out other
+ * reactor handlers), a specialized watcher in the pattern of ev_flux.c or
+ * ev_zmq.c is needed.  ev_zmq.c contains further explanation about that
+ * technique. When msg_deque is used within a connector, the reactive signaling
+ * is encapsulated in the flux_t handle, so flux_handle_watcher_create(3),
+ * based on ev_flux.c, already implements a fair handler.
+ *
+ * In the current implementation, msg_deque size is unlimited, so POLLOUT is
+ * always asserted in pollevents.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <poll.h>
+#include <sys/eventfd.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <errno.h>
+#include <flux/core.h>
+
+#include "ccan/list/list.h"
+
+#include "message_private.h" // for access to msg->list
+#include "msg_deque.h"
+
+struct msg_deque {
+    struct list_head messages;
+    int pollevents;
+    int pollfd;
+    uint64_t event;
+    pthread_mutex_t lock;
+    int flags;
+};
+
+void msg_deque_destroy (struct msg_deque *q)
+{
+    if (q) {
+        int saved_errno = errno;
+        flux_msg_t *msg;
+        while ((msg = msg_deque_pop_front (q)))
+            flux_msg_destroy (msg);
+        if (q->pollfd >= 0)
+            (void)close (q->pollfd);
+        if (!(q->flags & MSG_DEQUE_SINGLE_THREAD))
+            pthread_mutex_destroy (&q->lock);
+        free (q);
+        errno = saved_errno;
+    };
+}
+
+struct msg_deque *msg_deque_create (int flags)
+{
+    struct msg_deque *q;
+
+    if (flags != 0 && flags != MSG_DEQUE_SINGLE_THREAD) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(q = calloc (1, sizeof (*q))))
+        return NULL;
+    q->flags = flags;
+    q->pollfd = -1;
+    q->pollevents = POLLOUT;
+    if (!(flags & MSG_DEQUE_SINGLE_THREAD))
+        pthread_mutex_init (&q->lock, NULL);
+    list_head_init (&q->messages);
+    return q;
+}
+
+static inline void msg_deque_lock (struct msg_deque *q)
+{
+    if (!(q->flags & MSG_DEQUE_SINGLE_THREAD))
+        pthread_mutex_lock (&q->lock);
+}
+
+static inline void msg_deque_unlock (struct msg_deque *q)
+{
+    if (!(q->flags & MSG_DEQUE_SINGLE_THREAD))
+        pthread_mutex_unlock (&q->lock);
+}
+
+// See eventfd(2) for an explanation of how signaling on q->pollfd works
+static int msg_deque_raise_event (struct msg_deque *q)
+{
+    if (q->pollfd >= 0 && q->event == 0) {
+        q->event = 1;
+        if (write (q->pollfd, &q->event, sizeof (q->event)) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static int msg_deque_clear_event (struct msg_deque *q)
+{
+    if (q->pollfd >= 0 && q->event == 1) {
+        if (read (q->pollfd, &q->event, sizeof (q->event)) < 0) {
+            if (errno != EAGAIN  && errno != EWOULDBLOCK)
+                return -1;
+            errno = 0;
+        }
+        q->event = 0;
+    }
+    return 0;
+}
+
+bool check_push_args (struct msg_deque *q, flux_msg_t *msg)
+{
+    if (!q || !msg)
+        return false;
+    /* When deque is used as a transport between threads, retaining a
+     * reference on a message after pushing it might result in both threads
+     * modifying the message simultaneously.  Therefore reject the operation
+     * if references other than the one being transferred are held.
+     */
+    if (!(q->flags & MSG_DEQUE_SINGLE_THREAD) && msg->refcount > 1)
+        return false;
+    /* A message can only be in one msg_deque at a time.  Reject the operation
+     * if the list node is not in the state left by list_node_init() and
+     * list_del_init(), which is n.next == n.prev == n.
+     */
+    if (msg->list.next != &msg->list || msg->list.prev != &msg->list)
+        return false;
+
+    return true;
+}
+
+int msg_deque_push_back (struct msg_deque *q, flux_msg_t *msg)
+{
+    if (!check_push_args (q, msg)) {
+        errno = EINVAL;
+        return -1;
+    }
+    msg_deque_lock (q);
+    if (!(q->pollevents & POLLIN)) {
+        q->pollevents |= POLLIN;
+        if (msg_deque_raise_event (q) < 0) {
+            msg_deque_unlock (q);
+            return -1;
+        }
+    }
+    list_add (&q->messages, &msg->list);
+    msg_deque_unlock (q);
+    return 0;
+}
+
+int msg_deque_push_front (struct msg_deque *q, flux_msg_t *msg)
+{
+    if (!check_push_args (q, msg)) {
+        errno = EINVAL;
+        return -1;
+    }
+    msg_deque_lock (q);
+    if (!(q->pollevents & POLLIN)) {
+        q->pollevents |= POLLIN;
+        if (msg_deque_raise_event (q) < 0) {
+            msg_deque_unlock (q);
+            return -1;
+        }
+    }
+    list_add_tail (&q->messages, &msg->list);
+    msg_deque_unlock (q);
+    return 0;
+}
+
+flux_msg_t *msg_deque_pop_front (struct msg_deque *q)
+{
+    if (!q)
+        return NULL;
+    msg_deque_lock (q);
+    flux_msg_t *msg = list_tail (&q->messages, struct flux_msg, list);
+    if (msg) {
+        list_del_init (&msg->list);
+        if ((q->pollevents & POLLIN) && list_empty (&q->messages))
+            q->pollevents &= ~POLLIN;
+    }
+    msg_deque_unlock (q);
+    return msg;
+}
+
+bool msg_deque_empty (struct msg_deque *q)
+{
+    if (!q)
+        return true;
+    msg_deque_lock (q);
+    bool res = list_empty (&q->messages);
+    msg_deque_unlock (q);
+    return res;
+}
+
+size_t msg_deque_count (struct msg_deque *q)
+{
+    if (!q)
+        return 0;
+    msg_deque_lock (q);
+    size_t count = 0;
+    flux_msg_t *msg = NULL;
+    list_for_each (&q->messages, msg, list)
+        count++;
+    msg_deque_unlock (q);
+    return count;
+}
+
+int msg_deque_pollfd (struct msg_deque *q)
+{
+    if (!q) {
+        errno = EINVAL;
+        return -1;
+    }
+    int rc;
+    msg_deque_lock (q);
+    if (q->pollfd < 0) {
+        q->event = q->pollevents ? 1 : 0;
+        q->pollfd = eventfd (q->pollevents, EFD_NONBLOCK);
+    }
+    rc = q->pollfd;
+    msg_deque_unlock (q);
+    return rc;
+}
+
+int msg_deque_pollevents (struct msg_deque *q)
+{
+    if (!q) {
+        errno = EINVAL;
+        return -1;
+    }
+    int rc = -1;
+    msg_deque_lock (q);
+    if (msg_deque_clear_event (q) < 0)
+        goto done;
+    rc = q->pollevents;
+done:
+    msg_deque_unlock (q);
+    return rc;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libflux/msg_deque.h b/src/common/libflux/msg_deque.h
new file mode 100644
index 000000000000..ba2badf10ab8
--- /dev/null
+++ b/src/common/libflux/msg_deque.h
@@ -0,0 +1,43 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_MSG_DEQUE_H
+#define _FLUX_CORE_MSG_DEQUE_H
+
+#include <sys/types.h>
+
+/* If flags contains MSG_DEQUE_SINGLE_THREAD, pthread locking is eliminated
+ * and messages are permitted to be pushed with a reference count > 1.
+ */
+enum {
+    MSG_DEQUE_SINGLE_THREAD = 1,
+};
+
+struct msg_deque *msg_deque_create (int flags);
+void msg_deque_destroy (struct msg_deque *q);
+
+/* msg_deque_push_back() and msg_deque_push_front() steal a reference on
+ * 'msg' on success.  If MSG_DEQUE_SINGLE_THREAD was not specified, then
+ * that is expected to be the *only* reference and further access to the
+ * message by the caller is not permitted.
+ */
+int msg_deque_push_back (struct msg_deque *q, flux_msg_t *msg);
+int msg_deque_push_front (struct msg_deque *q, flux_msg_t *msg);
+flux_msg_t *msg_deque_pop_front (struct msg_deque *q);
+
+int msg_deque_pollfd (struct msg_deque *q);
+int msg_deque_pollevents (struct msg_deque *q);
+
+bool msg_deque_empty (struct msg_deque *q);
+size_t msg_deque_count (struct msg_deque *q);
+
+#endif // !_FLUX_CORE_MSG_DEQUE_H
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libflux/msg_handler.c b/src/common/libflux/msg_handler.c
index b4c3641ba530..fb948b772a3f 100644
--- a/src/common/libflux/msg_handler.c
+++ b/src/common/libflux/msg_handler.c
@@ -11,20 +11,18 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
-#if HAVE_CALIPER
-#include <caliper/cali.h>
-#include <sys/syscall.h>
-#endif
-
-#include "message.h"
-#include "reactor.h"
-#include "msg_handler.h"
-#include "response.h"
-#include "flog.h"
+#include <assert.h>
+#include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/iterators.h"
+#include "src/common/libutil/errno_safe.h"
+
+struct handler_stack {
+    flux_msg_handler_t *mh;  // current message handler in stack
+    zlistx_t *stack;         // stack of message handlers if >1
+};
 
 struct dispatch {
     flux_t *h;
@@ -36,11 +34,6 @@ struct dispatch {
     int running_count;
     int usecount;
     zlist_t *unmatched;
-#if HAVE_CALIPER
-    cali_id_t prof_msg_type;
-    cali_id_t prof_msg_topic;
-    cali_id_t prof_msg_dispatch;
-#endif
 };
 
 #define HANDLER_MAGIC 0x44433322
@@ -54,13 +47,126 @@ struct flux_msg_handler {
     uint8_t running:1;
 };
 
-static void handle_cb (flux_reactor_t *r, flux_watcher_t *w,
-                       int revents, void *arg);
+static void handle_cb (flux_reactor_t *r,
+                       flux_watcher_t *w,
+                       int revents,
+                       void *arg);
 static void free_msg_handler (flux_msg_handler_t *mh);
 
 static size_t matchtag_hasher (const void *key);
 static int matchtag_cmp (const void *key1, const void *key2);
 
+
+static void handler_stack_destroy (struct handler_stack *hs)
+{
+    if (hs->stack)
+        zlistx_destroy (&hs->stack);
+    free (hs);
+}
+
+static void handler_stack_destructor (void **item)
+{
+    if (item && *item) {
+        handler_stack_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static struct handler_stack *handler_stack_create (void)
+{
+    struct handler_stack *hs = calloc (1, sizeof (*hs));
+    return hs;
+}
+
+static int handler_stack_push (struct handler_stack *hs,
+                               flux_msg_handler_t *mh)
+{
+    if (!hs->mh) {
+        hs->mh = mh;
+        return 0;
+    }
+    if (!hs->stack) {
+        /*  Create stack, push current entry */
+        if (!(hs->stack = zlistx_new ())
+            || !zlistx_add_start (hs->stack, hs->mh))
+            return -1;
+    }
+    if (!zlistx_add_start (hs->stack, mh))
+        return -1;
+    hs->mh = mh;
+    return 0;
+}
+
+static int handler_stack_remove (struct handler_stack *hs,
+                                 flux_msg_handler_t *mh)
+{
+    void *handle;
+    if (!hs->stack) {
+        hs->mh = NULL;
+        return 0;
+    }
+    if (!(handle = zlistx_find (hs->stack, mh))
+        || !zlistx_detach (hs->stack, handle)) {
+        errno = ENOENT;
+        return -1;
+    }
+    if (hs->mh == mh)
+        hs->mh = zlistx_first (hs->stack);
+    return 0;
+}
+
+static int handler_stack_empty (struct handler_stack *hs)
+{
+    return (hs->mh == NULL);
+}
+
+static zhashx_t *method_hash_create (void)
+{
+    zhashx_t *hash = zhashx_new ();
+    if (!hash) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    zhashx_set_destructor (hash, handler_stack_destructor);
+    return hash;
+}
+
+
+static flux_msg_handler_t *method_hash_lookup (zhashx_t *hash,
+                                               const char *topic)
+{
+    struct handler_stack *hs = zhashx_lookup (hash, topic);
+    return hs ? hs->mh : NULL;
+}
+
+static int method_hash_add (zhashx_t *hash, flux_msg_handler_t *mh)
+{
+    struct handler_stack *hs = zhashx_lookup (hash, mh->match.topic_glob);
+    if (!hs) {
+        if (!(hs = handler_stack_create ()))
+            return -1;
+        if (zhashx_insert (hash, mh->match.topic_glob, hs) < 0) {
+            errno = EEXIST;
+            return -1;
+        }
+    }
+    return handler_stack_push (hs, mh);
+}
+
+static void method_hash_remove (zhashx_t *hash, flux_msg_handler_t *mh)
+{
+    struct handler_stack *hs = zhashx_lookup (hash, mh->match.topic_glob);
+    /*
+     *  Remove requested handler from this stack. If stack is empty
+     *   after remove, remove the hash entry entirely.
+     */
+    if (hs
+        && handler_stack_remove (hs, mh) == 0
+        && handler_stack_empty (hs)) {
+        zhashx_delete (hash, mh->match.topic_glob);
+    }
+}
+
 /* Return true if topic string 's' could match multiple request topics,
  * e.g. contains a glob character, or is NULL or "" which match anything.
  */
@@ -144,24 +250,9 @@ static struct dispatch *dispatch_get (flux_t *h)
         zhashx_set_key_comparator (d->handlers_rpc, matchtag_cmp);
         zhashx_set_key_destructor (d->handlers_rpc, NULL);
         zhashx_set_key_duplicator (d->handlers_rpc, NULL);
-        /* N.B. d->handlers_method key points to mh->match.topic_glob in entry,
-         * so disable the key duplicator and destructor to avoid extra malloc.
-         */
-        if (!(d->handlers_method = zhashx_new ()))
+
+        if (!(d->handlers_method = method_hash_create ()))
             goto nomem;
-        zhashx_set_key_destructor (d->handlers_method, NULL);
-        zhashx_set_key_duplicator (d->handlers_method, NULL);
-#if HAVE_CALIPER
-        d->prof_msg_type = cali_create_attribute ("flux.message.type",
-                                                  CALI_TYPE_STRING,
-                                                  CALI_ATTR_SKIP_EVENTS);
-        d->prof_msg_topic = cali_create_attribute ("flux.message.topic",
-                                                   CALI_TYPE_STRING,
-                                                   CALI_ATTR_SKIP_EVENTS);
-        d->prof_msg_dispatch = cali_create_attribute ("flux.message.dispatch",
-                                                      CALI_TYPE_BOOL,
-                                                      CALI_ATTR_DEFAULT);
-#endif
         if (flux_aux_set (h, "flux::dispatch", d, dispatch_destroy) < 0)
             goto error;
     }
@@ -211,17 +302,16 @@ static int copy_match (struct flux_match *dst,
 
 static void call_handler (flux_msg_handler_t *mh, const flux_msg_t *msg)
 {
-    uint32_t rolemask, matchtag;
+    uint32_t rolemask;
 
     if (flux_msg_get_rolemask (msg, &rolemask) < 0)
         return;
     if (!(rolemask & mh->rolemask)) {
         if (flux_msg_cmp (msg, FLUX_MATCH_REQUEST)
-                        && flux_msg_get_matchtag (msg, &matchtag) == 0
-                        && matchtag != FLUX_MATCHTAG_NONE) {
+            && !flux_msg_has_flag (msg, FLUX_MSGFLAG_NORESPONSE)) {
             const char *errmsg;
             if (mh->rolemask == 0 || mh->rolemask == FLUX_ROLE_OWNER)
-                errmsg = "Request requires owner credentals";
+                errmsg = "Request requires owner credentials";
             else
                 errmsg = "Request rejected due to insufficient privilege";
             (void)flux_respond_error (mh->d->h, msg, EPERM, errmsg);
@@ -248,12 +338,12 @@ static bool dispatch_message (struct dispatch *d,
     /* rpc response w/matchtag */
     if (type == FLUX_MSGTYPE_RESPONSE) {
         uint32_t matchtag;
-        if (flux_msg_get_route_count (msg) == 0
-                && flux_msg_get_matchtag (msg, &matchtag) == 0
-                && matchtag != FLUX_MATCHTAG_NONE
-                && (mh = zhashx_lookup (d->handlers_rpc, &matchtag))
-                && mh->running
-                && flux_msg_cmp (msg, mh->match)) {
+        if (flux_msg_route_count (msg) == 0
+            && flux_msg_get_matchtag (msg, &matchtag) == 0
+            && matchtag != FLUX_MATCHTAG_NONE
+            && (mh = zhashx_lookup (d->handlers_rpc, &matchtag))
+            && mh->running
+            && flux_msg_cmp (msg, mh->match)) {
             call_handler (mh, msg);
             match = true;
         }
@@ -262,8 +352,8 @@ static bool dispatch_message (struct dispatch *d,
     else if (type == FLUX_MSGTYPE_REQUEST) {
         const char *topic;
         if (flux_msg_get_topic (msg, &topic) == 0
-                && (mh = zhashx_lookup (d->handlers_method, topic))
-                && mh->running) {
+            && (mh = method_hash_lookup (d->handlers_method, topic))
+            && mh->running) {
             call_handler (mh, msg);
             match = true;
         }
@@ -296,7 +386,7 @@ static void handle_late_response (struct dispatch *d, const flux_msg_t *msg)
 
     if (flux_msg_get_matchtag (msg, &matchtag) < 0)
         return;
-    if (flux_msg_get_route_count (msg) != 0)
+    if (flux_msg_route_count (msg) != 0)
         return; // foreign matchtag domain (or error getting count)
     if (matchtag == FLUX_MATCHTAG_NONE)
         return; // no matchtag was allocated
@@ -335,6 +425,7 @@ static void handle_cb (flux_reactor_t *r,
     int rc = -1;
     int type;
     bool match;
+    const char *topic;
 
     if (revents & FLUX_POLLERR)
         goto done;
@@ -347,33 +438,17 @@ static void handle_cb (flux_reactor_t *r,
         rc = 0; /* ignore mangled message */
         goto done;
     }
+    if (flux_msg_get_topic (msg, &topic) < 0)
+        topic = "unknown"; /* used for logging */
 
-    const char *topic;
-    flux_msg_get_topic (msg, &topic);
     /* Add any new handlers here, making handler creation
      * safe to call during handlers list traversal below.
      */
     if (transfer_items_zlist (d->handlers_new, d->handlers) < 0)
         goto done;
 
-#if defined(HAVE_CALIPER)
-    cali_begin_string (d->prof_msg_type, flux_msg_typestr (type));
-    cali_begin_string (d->prof_msg_topic, topic);
-    cali_begin (d->prof_msg_dispatch);
-    cali_end (d->prof_msg_topic);
-    cali_end (d->prof_msg_type);
-#endif
-
     match = dispatch_message (d, msg, type);
 
-#if defined(HAVE_CALIPER)
-    cali_begin_string (d->prof_msg_type, flux_msg_typestr (type));
-    cali_begin_string (d->prof_msg_topic, topic);
-    cali_end (d->prof_msg_dispatch);
-    cali_end (d->prof_msg_topic);
-    cali_end (d->prof_msg_type);
-#endif
-
     /* Message was not "consumed".
      * If in a cloned handle, queue message for later.
      * Otherwise, respond with ENOSYS if it was a request,
@@ -393,13 +468,16 @@ static void handle_cb (flux_reactor_t *r,
         }
         else {
             switch (type) {
-                case FLUX_MSGTYPE_REQUEST:
-                    if (flux_respond_error (d->h,
-                                            msg,
-                                            ENOSYS,
-                                            "Unknown service method"))
+                case FLUX_MSGTYPE_REQUEST: {
+                    char errmsg[256];
+                    (void)snprintf (errmsg,
+                                    sizeof (errmsg),
+                                    "Unknown service method '%s'",
+                                    topic);
+                    if (flux_respond_error (d->h, msg, ENOSYS, errmsg))
                         goto done;
                     break;
+                }
                 case FLUX_MSGTYPE_EVENT:
                     break;
                 case FLUX_MSGTYPE_RESPONSE:
@@ -407,12 +485,10 @@ static void handle_cb (flux_reactor_t *r,
                     break;
                 default:
                     if (flux_flags_get (d->h) & FLUX_O_TRACE) {
-                        const char *topic = NULL;
-                        (void)flux_msg_get_topic (msg, &topic);
                         fprintf (stderr,
                                  "nomatch: %s '%s'\n",
                                  flux_msg_typestr (type),
-                                 topic ? topic : "");
+                                 topic);
                     }
                     break;
             }
@@ -420,10 +496,8 @@ static void handle_cb (flux_reactor_t *r,
     }
     rc = 0;
 done:
-    if (rc < 0) {
+    if (rc < 0)
         flux_reactor_stop_error (r);
-        FLUX_FATAL (d->h);
-    }
     flux_msg_destroy (msg);
 }
 
@@ -486,12 +560,12 @@ void flux_msg_handler_destroy (flux_msg_handler_t *mh)
         int saved_errno = errno;
         assert (mh->magic == HANDLER_MAGIC);
         if (mh->match.typemask == FLUX_MSGTYPE_RESPONSE
-                            && mh->match.matchtag != FLUX_MATCHTAG_NONE) {
+            && mh->match.matchtag != FLUX_MATCHTAG_NONE) {
             zhashx_delete (mh->d->handlers_rpc, &mh->match.matchtag);
         }
         else if (mh->match.typemask == FLUX_MSGTYPE_REQUEST
-                            && !isa_multmatch (mh->match.topic_glob)) {
-            zhashx_delete (mh->d->handlers_method, mh->match.topic_glob);
+                 && !isa_multmatch (mh->match.topic_glob)) {
+            method_hash_remove (mh->d->handlers_method, mh);
         }
         else {
             zlist_remove (mh->d->handlers_new, mh);
@@ -506,7 +580,8 @@ void flux_msg_handler_destroy (flux_msg_handler_t *mh)
 
 flux_msg_handler_t *flux_msg_handler_create (flux_t *h,
                                              const struct flux_match match,
-                                             flux_msg_handler_f cb, void *arg)
+                                             flux_msg_handler_f cb,
+                                             void *arg)
 {
     struct dispatch *d;
     flux_msg_handler_t *mh;
@@ -531,19 +606,20 @@ flux_msg_handler_t *flux_msg_handler_create (flux_t *h,
      * indicates a matchtag reuse problem!
      */
     if (mh->match.typemask == FLUX_MSGTYPE_RESPONSE
-                            && mh->match.matchtag != FLUX_MATCHTAG_NONE) {
+        && mh->match.matchtag != FLUX_MATCHTAG_NONE) {
         if (zhashx_insert (d->handlers_rpc, &mh->match.matchtag, mh) < 0) {
             errno = EEXIST;
             goto error;
         }
     }
     /* Request (non-glob):
-     * Replace existing entry in the handlers_method hash, if any.
+     * Push entry onto top of the handlers_method stack, if any.
      * This allows builtin module methods to be overridden.
      */
     else if (mh->match.typemask == FLUX_MSGTYPE_REQUEST
-                            && !isa_multmatch (mh->match.topic_glob)) {
-        zhashx_update (d->handlers_method, mh->match.topic_glob, mh);
+             && !isa_multmatch (mh->match.topic_glob)) {
+        if (method_hash_add (d->handlers_method, mh) < 0)
+            goto error;
     }
     /* Request (glob), response (FLUX_MATCHTAG_NONE), events:
      * Message handler is pushed to the front of the handlers list,
@@ -577,10 +653,11 @@ static bool at_end (struct flux_msg_handler_spec spec)
             && spec.rolemask == end.rolemask);
 }
 
-int flux_msg_handler_addvec (flux_t *h,
-                             const struct flux_msg_handler_spec tab[],
-                             void *arg,
-                             flux_msg_handler_t **hp[])
+int flux_msg_handler_addvec_ex (flux_t *h,
+                                const char *service_name,
+                                const struct flux_msg_handler_spec tab[],
+                                void *arg,
+                                flux_msg_handler_t **hp[])
 {
     int i;
     struct flux_match match = FLUX_MATCH_ANY;
@@ -596,12 +673,16 @@ int flux_msg_handler_addvec (flux_t *h,
     if (!(handlers = calloc (count + 1, sizeof (flux_msg_handler_t *))))
         return -1;
     for (i = 0; i < count; i++) {
+        char *topic = NULL;
         match.typemask = tab[i].typemask;
-        /* flux_msg_handler_create() will make a copy of the topic_glob
-         * so it is safe to temporarily remove "const" from
-         * tab[i].topic_glob with a cast. */
-        match.topic_glob = (char *)tab[i].topic_glob;
-        if (!(handlers[i] = flux_msg_handler_create (h, match, tab[i].cb, arg)))
+        if (service_name) {
+            if (asprintf (&topic, "%s.%s", service_name, tab[i].topic_glob) < 0)
+                goto error;
+        }
+        match.topic_glob = topic ? topic : (char *)tab[i].topic_glob;
+        handlers[i] = flux_msg_handler_create (h, match, tab[i].cb, arg);
+        ERRNO_SAFE_WRAP (free, topic);
+        if (!handlers[i])
             goto error;
         flux_msg_handler_allow_rolemask (handlers[i], tab[i].rolemask);
         flux_msg_handler_start (handlers[i]);
@@ -613,6 +694,14 @@ int flux_msg_handler_addvec (flux_t *h,
     return -1;
 }
 
+int flux_msg_handler_addvec (flux_t *h,
+                             const struct flux_msg_handler_spec tab[],
+                             void *arg,
+                             flux_msg_handler_t **hp[])
+{
+    return flux_msg_handler_addvec_ex (h, NULL, tab, arg, hp);
+}
+
 void flux_msg_handler_delvec (flux_msg_handler_t *handlers[])
 {
     if (handlers) {
diff --git a/src/common/libflux/msg_handler.h b/src/common/libflux/msg_handler.h
index 8d79a6cf7a0a..da71f6f9b0ae 100644
--- a/src/common/libflux/msg_handler.h
+++ b/src/common/libflux/msg_handler.h
@@ -11,9 +11,6 @@
 #ifndef _FLUX_CORE_MSG_HANDLER_H
 #define _FLUX_CORE_MSG_HANDLER_H
 
-#include "message.h"
-#include "handle.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -49,6 +46,12 @@ struct flux_msg_handler_spec {
 };
 #define FLUX_MSGHANDLER_TABLE_END { 0, NULL, NULL, 0 }
 
+int flux_msg_handler_addvec_ex (flux_t *h,
+                                const char *service_name,
+                                const struct flux_msg_handler_spec tab[],
+                                void *arg,
+                                flux_msg_handler_t **hp[]);
+
 int flux_msg_handler_addvec (flux_t *h,
                              const struct flux_msg_handler_spec tab[],
                              void *arg,
diff --git a/src/common/libflux/msglist.c b/src/common/libflux/msglist.c
new file mode 100644
index 000000000000..2b708b69b757
--- /dev/null
+++ b/src/common/libflux/msglist.c
@@ -0,0 +1,188 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <poll.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/eventfd.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "msglist.h"
+
+struct flux_msglist {
+    zlistx_t *zl;
+    int pollevents;
+    int pollfd;
+    uint64_t event;
+};
+
+static void *msg_duplicator (const void *item)
+{
+    return (flux_msg_t *)flux_msg_incref (item);
+}
+
+static void msg_destructor (void **item)
+{
+    if (item) {
+        flux_msg_decref (*item);
+        *item = NULL;
+    }
+}
+
+struct flux_msglist *flux_msglist_create (void)
+{
+    struct flux_msglist *l;
+
+    if (!(l = calloc (1, sizeof (*l))))
+        return NULL;
+    l->pollfd = -1;
+    l->pollevents = POLLOUT;
+    if (!(l->zl = zlistx_new ())) {
+        free (l);
+        errno = ENOMEM;
+        return NULL;
+    }
+    zlistx_set_destructor (l->zl, msg_destructor);
+    zlistx_set_duplicator (l->zl, msg_duplicator);
+    return l;
+}
+
+void flux_msglist_destroy (struct flux_msglist *l)
+{
+    if (l) {
+        int saved_errno = errno;
+        zlistx_destroy (&l->zl);
+        if (l->pollfd >= 0)
+            close (l->pollfd);
+        free (l);
+        errno = saved_errno;
+    }
+}
+
+static int msglist_raise_event (struct flux_msglist *l)
+{
+    if (l->pollfd >= 0 && l->event == 0) {
+        l->event = 1;
+    if (write (l->pollfd, &l->event, sizeof (l->event)) < 0)
+        return -1;
+    }
+    return 0;
+}
+
+static int msglist_clear_event (struct flux_msglist *l)
+{
+    if (l->pollfd >= 0 && l->event == 1) {
+        if (read (l->pollfd, &l->event, sizeof (l->event)) < 0) {
+            if (errno != EAGAIN  && errno != EWOULDBLOCK)
+                return -1;
+            errno = 0;
+        }
+        l->event = 0;
+    }
+    return 0;
+}
+
+int flux_msglist_append (struct flux_msglist *l, const flux_msg_t *msg)
+{
+    if (!(l->pollevents & POLLIN)) {
+        l->pollevents |= POLLIN;
+        if (msglist_raise_event (l) < 0)
+            return -1;
+    }
+    if (!zlistx_add_end (l->zl, (flux_msg_t *)msg)) {
+        l->pollevents |= POLLERR;
+        msglist_raise_event (l);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+int flux_msglist_push (struct flux_msglist *l, const flux_msg_t *msg)
+{
+    if (!(l->pollevents & POLLIN)) {
+        l->pollevents |= POLLIN;
+        if (msglist_raise_event (l) < 0)
+            return -1;
+    }
+    if (!zlistx_add_start (l->zl, (flux_msg_t *)msg)) {
+        l->pollevents |= POLLERR;
+        msglist_raise_event (l);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+const flux_msg_t *flux_msglist_first (struct flux_msglist *l)
+{
+    return zlistx_first (l->zl);
+}
+
+const flux_msg_t *flux_msglist_next (struct flux_msglist *l)
+{
+    return zlistx_next (l->zl);
+}
+
+const flux_msg_t *flux_msglist_last (struct flux_msglist *l)
+{
+    return zlistx_last (l->zl);
+}
+
+void flux_msglist_delete (struct flux_msglist *l)
+{
+    void *handle = zlistx_cursor (l->zl);
+    if (handle) {
+        zlistx_delete (l->zl, handle);
+        if ((l->pollevents & POLLIN) && zlistx_size (l->zl) == 0)
+            l->pollevents &= ~POLLIN;
+    }
+}
+
+const flux_msg_t *flux_msglist_pop (struct flux_msglist *l)
+{
+    void *item = zlistx_detach_cur (l->zl);
+    if (item) {
+        if ((l->pollevents & POLLIN) && zlistx_size (l->zl) == 0)
+            l->pollevents &= ~POLLIN;
+    }
+    return item;
+}
+
+int flux_msglist_count (struct flux_msglist *l)
+{
+    return l ? zlistx_size (l->zl) : 0;
+}
+
+int flux_msglist_pollfd (struct flux_msglist *l)
+{
+    if (l->pollfd < 0) {
+        l->event = l->pollevents ? 1 : 0;
+        l->pollfd = eventfd (l->pollevents, EFD_NONBLOCK);
+    }
+    return l->pollfd;
+}
+
+int flux_msglist_pollevents (struct flux_msglist *l)
+{
+    if (msglist_clear_event (l) < 0)
+        return -1;
+    return l->pollevents;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libflux/msglist.h b/src/common/libflux/msglist.h
new file mode 100644
index 000000000000..a3f21b153517
--- /dev/null
+++ b/src/common/libflux/msglist.h
@@ -0,0 +1,55 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_MSGLIST_H
+#define _FLUX_CORE_MSGLIST_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* message lists
+ * - takes a ref on 'msg' upon append
+ * - drops ref on 'msg' on delete / list destroy
+ */
+struct flux_msglist *flux_msglist_create (void);
+void flux_msglist_destroy (struct flux_msglist *l);
+
+int flux_msglist_push (struct flux_msglist *l, const flux_msg_t *msg);
+int flux_msglist_append (struct flux_msglist *l, const flux_msg_t *msg);
+void flux_msglist_delete (struct flux_msglist *l); // (at cursor) iteration safe
+const flux_msg_t *flux_msglist_pop (struct flux_msglist *l);
+
+const flux_msg_t *flux_msglist_first (struct flux_msglist *l);
+const flux_msg_t *flux_msglist_next (struct flux_msglist *l);
+const flux_msg_t *flux_msglist_last (struct flux_msglist *l);
+
+int flux_msglist_count (struct flux_msglist *l);
+
+/* These functions are for integration of flux_msglist into an event loop.
+ * The pollfd file descriptor becomes readable when a poll event has been
+ * raised (edge triggered).  This indicates that the pollevents mask has been
+ * updated.  The mask cnosists of POLLIN | POLLOUT | POLLERR.  N.B. POLLOUT
+ * is always ready in the current implementation.
+ * Both functions return -1 on error with errno set.
+ */
+int flux_msglist_pollevents (struct flux_msglist *l);
+int flux_msglist_pollfd (struct flux_msglist *l);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_FLUX_CORE_MSGLIST_H */
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
+
diff --git a/src/common/libflux/panic.c b/src/common/libflux/panic.c
deleted file mode 100644
index e81062bc9101..000000000000
--- a/src/common/libflux/panic.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <syslog.h>
-#include <unistd.h>
-#include <signal.h>
-#include <flux/core.h>
-
-#include "panic.h"
-
-
-int flux_panic (flux_t *h, uint32_t nodeid, int flags, const char *reason)
-{
-    flux_future_t *f;
-
-    if (!h || !reason || flags != 0) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (!(f = flux_rpc_pack (h, "cmb.panic", nodeid, FLUX_RPC_NORESPONSE,
-                             "{s:s s:i}",
-                             "reason", reason,
-                             "flags", flags)))
-        return -1;
-    flux_future_destroy (f);
-    return 0;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/panic.h b/src/common/libflux/panic.h
deleted file mode 100644
index fa15294aa8ed..000000000000
--- a/src/common/libflux/panic.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_CORE_PANIC_H
-#define _FLUX_CORE_PANIC_H
-
-#include <stdint.h>
-#include "handle.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Tell broker on 'nodeid' to call _exit() after displaying 'reason'
- * on stderr.  'nodeid' may be FLUX_NODEID_ANY to select the local
- * broker.  Currently flags should be set to zero.
- */
-int flux_panic (flux_t *h, uint32_t nodeid, int flags, const char *reason);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* !_FLUX_CORE_PANIC_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/paths.h b/src/common/libflux/paths.h
new file mode 100644
index 000000000000..992412450cb0
--- /dev/null
+++ b/src/common/libflux/paths.h
@@ -0,0 +1 @@
+#define INSTALLED_MODULE_PATH "${exec_prefix}/lib/flux/modules"
diff --git a/src/common/libflux/plugin.c b/src/common/libflux/plugin.c
index fa8e5a8577fd..26053948f9d3 100644
--- a/src/common/libflux/plugin.c
+++ b/src/common/libflux/plugin.c
@@ -12,27 +12,60 @@
 #include "config.h"
 #endif
 
+#include <fcntl.h>
+#include <unistd.h>
 #include <errno.h>
 #include <string.h>
 #include <fnmatch.h>
 #include <dlfcn.h>
 #include <stdarg.h>
+#include <uuid.h>
 #include <jansson.h>
-#include <czmq.h>
+#include <assert.h>
+#include <flux/core.h>
+#include <pthread.h>
 #include <flux/core.h>
 
+#include "src/common/libflux/plugin_private.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/aux.h"
+#include "ccan/str/str.h"
+
+#ifndef UUID_STR_LEN
+#define UUID_STR_LEN 37     // defined in later libuuid headers
+#endif
+
+static int use_deepbind = 1;
+static pthread_once_t deepbind_once = PTHREAD_ONCE_INIT;
 
-#include "plugin.h"
+static void init_use_deepbind (void)
+{
+    const char *var = getenv ("FLUX_LOAD_WITH_DEEPBIND");
+    if (!var) {
+        // default is to allow the flag
+        return;
+    }
+    use_deepbind = strtol (var, NULL, 10);
+}
+
+int plugin_deepbind (void)
+{
+    pthread_once (&deepbind_once, init_use_deepbind);
+    return use_deepbind != 0 ? FLUX_DEEPBIND : 0;
+}
 
 struct flux_plugin {
     char *path;
     char *name;
     json_t *conf;
+    char *conf_str;
     struct aux_item *aux;
     void *dso;
     zlistx_t *handlers;
+    int flags;
     char last_error [128];
+    uuid_t uuid;
+    char uuid_str[UUID_STR_LEN];
 };
 
 struct flux_plugin_arg {
@@ -47,7 +80,7 @@ typedef const struct flux_plugin_handler *
 static void flux_plugin_handler_destroy (struct flux_plugin_handler *h)
 {
     if (h) {
-        free (h->topic);
+        free ((char *) h->topic);
         free (h);
     }
 }
@@ -66,7 +99,7 @@ static const struct flux_plugin_handler * find_handler (flux_plugin_t *p,
 {
     struct flux_plugin_handler *h = zlistx_first (p->handlers);
     while (h) {
-        if (strcmp (h->topic, string) == 0)
+        if (streq (h->topic, string))
             return h;
         h = zlistx_next (p->handlers);
     }
@@ -105,6 +138,7 @@ void flux_plugin_destroy (flux_plugin_t *p)
         int saved_errno = errno;
         json_decref (p->conf);
         zlistx_destroy (&p->handlers);
+        free (p->conf_str);
         free (p->path);
         free (p->name);
         aux_destroy (&p->aux);
@@ -147,10 +181,41 @@ flux_plugin_t *flux_plugin_create (void)
         flux_plugin_destroy (p);
         return NULL;
     }
+    p->flags = FLUX_PLUGIN_RTLD_LAZY;
+    uuid_generate (p->uuid);
+    uuid_unparse (p->uuid, p->uuid_str);
     zlistx_set_destructor (p->handlers, handler_free);
     return p;
 }
 
+static int flags_invalid (int flags)
+{
+    const int valid_flags =
+        FLUX_PLUGIN_RTLD_LAZY
+        | FLUX_PLUGIN_RTLD_NOW
+        | FLUX_PLUGIN_RTLD_GLOBAL
+        | FLUX_PLUGIN_RTLD_DEEPBIND;
+    return (flags & ~valid_flags);
+}
+
+int flux_plugin_set_flags (flux_plugin_t *p, int flags)
+{
+    if (!p || flags_invalid (flags)) {
+        errno = EINVAL;
+        return -1;
+    }
+    p->flags = flags;
+    return 0;
+}
+
+int flux_plugin_get_flags (flux_plugin_t *p)
+{
+    if (p) {
+        return p->flags;
+    }
+    return 0;
+}
+
 int flux_plugin_set_name (flux_plugin_t *p, const char *name)
 {
     char *new = NULL;
@@ -174,8 +239,27 @@ const char * flux_plugin_get_name (flux_plugin_t *p)
     return p->name;
 }
 
-int flux_plugin_aux_set (flux_plugin_t *p, const char *key,
-                         void *val, aux_free_f free_fn)
+const char * flux_plugin_get_uuid (flux_plugin_t *p)
+{
+    plugin_error_clear (p);
+    if (!p) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return p->uuid_str;
+}
+
+const char * flux_plugin_get_path (flux_plugin_t *p)
+{
+    if (p)
+        return p->path;
+    return NULL;
+}
+
+int flux_plugin_aux_set (flux_plugin_t *p,
+                         const char *key,
+                         void *val,
+                         aux_free_f free_fn)
 {
     return aux_set (&p->aux, key, val, free_fn);
 }
@@ -185,11 +269,32 @@ void *flux_plugin_aux_get (flux_plugin_t *p, const char *key)
     return aux_get (p->aux, key);
 }
 
+void flux_plugin_aux_delete (flux_plugin_t *p, const void *val)
+{
+    return aux_delete (&p->aux, val);
+}
+
 const char *flux_plugin_strerror (flux_plugin_t *p)
 {
     return p->last_error;
 }
 
+static int open_flags (flux_plugin_t *p)
+{
+    int flags = 0;
+    if ((p->flags & FLUX_PLUGIN_RTLD_LAZY))
+        flags |= RTLD_LAZY;
+    if ((p->flags & FLUX_PLUGIN_RTLD_NOW))
+        flags |= RTLD_NOW;
+    if ((p->flags & FLUX_PLUGIN_RTLD_GLOBAL))
+        flags |= RTLD_GLOBAL;
+    else
+        flags |= RTLD_LOCAL;
+    if ((p->flags & FLUX_PLUGIN_RTLD_DEEPBIND))
+        flags |= plugin_deepbind ();
+    return flags;
+}
+
 int flux_plugin_load_dso (flux_plugin_t *p, const char *path)
 {
     flux_plugin_init_f init;
@@ -199,7 +304,7 @@ int flux_plugin_load_dso (flux_plugin_t *p, const char *path)
     if (access (path, R_OK) < 0)
         return plugin_seterror (p, errno, "%s: %s", path, strerror (errno));
     dlerror ();
-    if (!(p->dso = dlopen (path, RTLD_LAZY|RTLD_LOCAL|FLUX_DEEPBIND)))
+    if (!(p->dso = dlopen (path, open_flags (p))))
         return plugin_seterror (p, errno, "dlopen: %s", dlerror ());
 
     free (p->path);
@@ -207,8 +312,9 @@ int flux_plugin_load_dso (flux_plugin_t *p, const char *path)
     if (!(p->path = strdup (path)) || !(p->name = strdup (path)))
         return plugin_seterror (p, ENOMEM, NULL);
 
-    if ((init = dlsym (p->dso, "flux_plugin_init")))
-        return (*init) (p);
+    if ((init = dlsym (p->dso, "flux_plugin_init"))
+        && (*init) (p) < 0)
+        return plugin_seterror (p, 0, "%s: flux_plugin_init failed", path);
     return 0;
 }
 
@@ -219,13 +325,42 @@ int flux_plugin_set_conf (flux_plugin_t *p, const char *json_str)
     if (!p || !json_str)
         return plugin_seterror (p, EINVAL, NULL);
     if (!(p->conf = json_loads (json_str, 0, &err))) {
-        return plugin_seterror (p, errno,
+        return plugin_seterror (p,
+                                errno,
                                 "parse error: col %d: %s",
-                                 err.column, err.text);
+                                err.column,
+                                err.text);
+    }
+    if (p->conf_str) {
+        free (p->conf_str);
+        p->conf_str = NULL;
     }
     return 0;
 }
 
+const char *flux_plugin_get_conf (flux_plugin_t *p)
+{
+    if (!p) {
+        plugin_seterror (p, EINVAL, NULL);
+        return NULL;
+    }
+    if (!p->conf_str) {
+        if (!p->conf) {
+            plugin_seterror (p, ENOENT, "No plugin conf set");
+            return NULL;
+        }
+        p->conf_str = json_dumps (p->conf, JSON_ENCODE_ANY|JSON_COMPACT);
+        if (!p->conf_str) {
+            plugin_seterror (p,
+                             errno,
+                             "json_dumps failed: %s",
+                             strerror (errno));
+            return NULL;
+        }
+    }
+    return p->conf_str;
+}
+
 int flux_plugin_conf_unpack (flux_plugin_t *p, const char *fmt, ...)
 {
     json_error_t err;
@@ -324,18 +459,22 @@ int flux_plugin_register (flux_plugin_t *p,
     return 0;
 }
 
-static int arg_seterror (flux_plugin_arg_t *arg, int errnum,
-                         const char *fmt, ...)
+static int arg_seterror (flux_plugin_arg_t *arg,
+                         int errnum,
+                         const char *fmt,
+                         ...)
 {
     if (fmt) {
         va_list ap;
         va_start (ap, fmt);
         vsnprintf (arg->error.text, sizeof (arg->error.text), fmt, ap);
         va_end (ap);
-    } else if (arg) {
+    }
+    else if (arg) {
         snprintf (arg->error.text,
                   sizeof (arg->error.text),
-                  "%s", strerror (errno));
+                  "%s",
+                  strerror (errno));
     }
     errno = errnum;
     return -1;
@@ -378,7 +517,7 @@ static int arg_set (flux_plugin_arg_t *args, int flags, json_t *o)
 {
     json_t **dstp;
     dstp = arg_get (args, flags);
-    if (flags & FLUX_PLUGIN_ARG_UPDATE && *dstp != NULL) {
+    if (!(flags & FLUX_PLUGIN_ARG_REPLACE) && *dstp != NULL) {
         /*  On update, the object 'o' is spiritually inherited by
          *   args, so decref this object after attempting the update.
          */
@@ -416,8 +555,10 @@ int flux_plugin_arg_get (flux_plugin_arg_t *args, int flags, char **json_str)
     return 0;
 }
 
-int flux_plugin_arg_vpack (flux_plugin_arg_t *args, int flags,
-                           const char *fmt, va_list ap)
+int flux_plugin_arg_vpack (flux_plugin_arg_t *args,
+                           int flags,
+                           const char *fmt,
+                           va_list ap)
 {
     json_t *o;
     arg_clear_error (args);
@@ -428,8 +569,10 @@ int flux_plugin_arg_vpack (flux_plugin_arg_t *args, int flags,
     return arg_set (args, flags, o);
 }
 
-int flux_plugin_arg_pack (flux_plugin_arg_t *args, int flags,
-                          const char *fmt, ...)
+int flux_plugin_arg_pack (flux_plugin_arg_t *args,
+                          int flags,
+                          const char *fmt,
+                          ...)
 {
     int rc;
     va_list ap;
@@ -439,8 +582,10 @@ int flux_plugin_arg_pack (flux_plugin_arg_t *args, int flags,
     return rc;
 }
 
-int flux_plugin_arg_vunpack (flux_plugin_arg_t *args, int flags,
-                             const char *fmt, va_list ap)
+int flux_plugin_arg_vunpack (flux_plugin_arg_t *args,
+                             int flags,
+                             const char *fmt,
+                             va_list ap)
 {
     json_t **op;
     arg_clear_error (args);
@@ -450,8 +595,10 @@ int flux_plugin_arg_vunpack (flux_plugin_arg_t *args, int flags,
     return json_vunpack_ex (*op, &args->error, 0, fmt, ap);
 }
 
-int flux_plugin_arg_unpack (flux_plugin_arg_t *args, int flags,
-                            const char *fmt, ...)
+int flux_plugin_arg_unpack (flux_plugin_arg_t *args,
+                            int flags,
+                            const char *fmt,
+                            ...)
 {
     int rc;
     va_list ap;
@@ -461,18 +608,20 @@ int flux_plugin_arg_unpack (flux_plugin_arg_t *args, int flags,
     return rc;
 }
 
-int flux_plugin_call (flux_plugin_t *p, const char *string,
+int flux_plugin_call (flux_plugin_t *p,
+                      const char *string,
                       flux_plugin_arg_t *args)
 {
     const struct flux_plugin_handler *h = NULL;
     plugin_error_clear (p);
     if (!p || !string)
         return plugin_seterror (p, EINVAL, NULL);
-    h = match_handler (p, string);
-    if (!h)
+    if (!(h = match_handler (p, string)))
         return 0;
     assert (h->cb);
-    return (*h->cb) (p, string, args, h->data);
+    if ((*h->cb) (p, string, args, h->data) < 0)
+        return -1;
+    return 1;
 }
 
 
diff --git a/src/common/libflux/plugin.h b/src/common/libflux/plugin.h
index 49903349821d..c8465df8906c 100644
--- a/src/common/libflux/plugin.h
+++ b/src/common/libflux/plugin.h
@@ -11,12 +11,17 @@
 #ifndef FLUX_CORE_PLUGIN_H
 #define FLUX_CORE_PLUGIN_H
 
-#include <flux/core.h>
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+enum {
+    FLUX_PLUGIN_RTLD_LAZY =      1,  /* Lazy function binding                */
+    FLUX_PLUGIN_RTLD_NOW =       2,  /* Immediate function binding           */
+    FLUX_PLUGIN_RTLD_GLOBAL =    4,  /* Load with RTLD_GLOBAL                */
+    FLUX_PLUGIN_RTLD_DEEPBIND =  8,  /* Load with RTLD_DEEPBIND if available */
+};
+
 typedef struct flux_plugin flux_plugin_t;
 typedef struct flux_plugin_arg flux_plugin_arg_t;
 
@@ -28,7 +33,7 @@ typedef int (*flux_plugin_f) (flux_plugin_t *p,
 typedef int (*flux_plugin_init_f) (flux_plugin_t *p);
 
 struct flux_plugin_handler {
-    char *topic;
+    const char *topic;
     flux_plugin_f cb;
     void *data;
 };
@@ -38,6 +43,11 @@ struct flux_plugin_handler {
 flux_plugin_t * flux_plugin_create (void);
 void flux_plugin_destroy (flux_plugin_t *p);
 
+/*  Get and set plugin flags. Flags currently only apply to load_dso()
+ */
+int flux_plugin_get_flags (flux_plugin_t *p);
+int flux_plugin_set_flags (flux_plugin_t *p, int flags);
+
 /*  Returns the last error from a plugin. Only valid if
  *   the last call returned an error.
  */
@@ -49,6 +59,10 @@ int flux_plugin_set_name (flux_plugin_t *p, const char *name);
 
 const char * flux_plugin_get_name (flux_plugin_t *p);
 
+const char * flux_plugin_get_uuid (flux_plugin_t *p);
+
+const char * flux_plugin_get_path (flux_plugin_t *p);
+
 /*  Add a handler for topic 'topic' for the plugin 'p'.
  *  The topic string may be a glob to cause 'cb' to be invoked for
  *  a set of topic strings called by the host.
@@ -79,7 +93,7 @@ int flux_plugin_register (flux_plugin_t *p,
                           const char *name,
                           const struct flux_plugin_handler t[]);
 
-/*  Associate auxillary data with the plugin handle 'p'. If free_fn is
+/*  Associate auxiliary data with the plugin handle 'p'. If free_fn is
  *   set then this function will be called on the data at plugin
  *   destruction.
  *
@@ -93,15 +107,23 @@ int flux_plugin_aux_set (flux_plugin_t *p,
                          void *val,
                          flux_free_f free_fn);
 
-/*  Get current auxillary data under `key`.
+/*  Get current auxiliary data under `key`.
  */
 void * flux_plugin_aux_get (flux_plugin_t *p, const char *key);
 
+/*  Delete auxiliary data by value.
+ */
+void flux_plugin_aux_delete (flux_plugin_t *p, const void *val);
 
 /*  Set optional JSON string as load-time config for plugin 'p'.
  */
 int flux_plugin_set_conf (flux_plugin_t *p, const char *json_str);
 
+/*  Get the current JSON string value of config for plugin 'p'.
+ *  Returns NULL on failure.
+ */
+const char *flux_plugin_get_conf (flux_plugin_t *p);
+
 /*  Read configuration for plugin 'p' using jansson style unpack args */
 int flux_plugin_conf_unpack (flux_plugin_t *p, const char *fmt, ...);
 
@@ -118,7 +140,7 @@ const char *flux_plugin_arg_strerror (flux_plugin_arg_t *args);
 enum {
     FLUX_PLUGIN_ARG_IN =  0,    /* Operate on input args    */
     FLUX_PLUGIN_ARG_OUT = 1,    /* Operate on output args   */
-    FLUX_PLUGIN_ARG_UPDATE = 2  /* Update args for set/pack */
+    FLUX_PLUGIN_ARG_REPLACE = 2 /* Replace args for set/pack */
 };
 
 /*  Get/set arguments in plugin arg object using JSON encoded strings
@@ -132,20 +154,32 @@ int flux_plugin_arg_get (flux_plugin_arg_t *args,
 
 /*  Pack/unpack arguments into plugin arg object using jansson pack style args
  */
-int flux_plugin_arg_pack (flux_plugin_arg_t *args, int flags,
+int flux_plugin_arg_pack (flux_plugin_arg_t *args,
+                          int flags,
                           const char *fmt, ...);
-int flux_plugin_arg_vpack (flux_plugin_arg_t *args, int flags,
-                           const char *fmt, va_list ap);
-
-int flux_plugin_arg_unpack (flux_plugin_arg_t *args, int flags,
-                            const char *fmt, ...);
-int flux_plugin_arg_vunpack (flux_plugin_arg_t *args, int flags,
-                             const char *fmt, va_list ap);
+int flux_plugin_arg_vpack (flux_plugin_arg_t *args,
+                           int flags,
+                           const char *fmt,
+                           va_list ap);
+
+int flux_plugin_arg_unpack (flux_plugin_arg_t *args,
+                            int flags,
+                            const char *fmt,
+                            ...);
+int flux_plugin_arg_vunpack (flux_plugin_arg_t *args,
+                             int flags,
+                             const char *fmt,
+                             va_list ap);
 
 /*  Call first plugin callback matching 'name', passing optional plugin
  *   arguments in 'args'.
+ *
+ *  Returns 0 if no callback was found for `name`, -1 if callback was
+ *   called with return value < 0, and 1 if callback was called with
+ *   return value >= 0.
  */
-int flux_plugin_call (flux_plugin_t *p, const char *name,
+int flux_plugin_call (flux_plugin_t *p,
+                      const char *name,
                       flux_plugin_arg_t *args);
 
 /*  Load a plugin from a shared object found in 'path'
diff --git a/src/common/libflux/plugin_private.h b/src/common/libflux/plugin_private.h
new file mode 100644
index 000000000000..39ceaa3aa48e
--- /dev/null
+++ b/src/common/libflux/plugin_private.h
@@ -0,0 +1,16 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef FLUX_CORE_PLUGIN_PRIVATE_H
+#define FLUX_CORE_PLUGIN_PRIVATE_H
+
+int plugin_deepbind (void);
+
+#endif /* FLUX_CORE_PLUGIN_PRIVATE_H */
diff --git a/src/common/libflux/reactor.c b/src/common/libflux/reactor.c
index 7cf31c2679cf..0c67b2110bab 100644
--- a/src/common/libflux/reactor.c
+++ b/src/common/libflux/reactor.c
@@ -15,36 +15,26 @@
 #include <string.h>
 #include <errno.h>
 #include <stdbool.h>
-#include <czmq.h>
-
-#include "handle.h"
-#include "reactor.h"
-#include "ev_flux.h"
-#include "ev_buffer_read.h"
-#include "ev_buffer_write.h"
-#include "buffer.h"
-#include "buffer_private.h"
+#include <fcntl.h>
+#include <flux/core.h>
 
 #include "src/common/libev/ev.h"
-#include "src/common/libutil/ev_zmq.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/fdutils.h"
 
-struct flux_reactor {
-    struct ev_loop *loop;
-    int usecount;
-    unsigned int errflag:1;
-};
+#include "ev_flux.h"
+#include "reactor_private.h"
 
-struct flux_watcher {
-    flux_reactor_t *r;
-    flux_watcher_f fn;
-    void *arg;
-    struct flux_watcher_ops *ops;
-    void *data;
-};
+static int valid_flags (int flags, int valid)
+{
+    if ((flags & ~valid)) {
+        errno = EINVAL;
+        return -1;
+    }
+    return 0;
+}
 
-static void reactor_usecount_decr (flux_reactor_t *r)
+void flux_reactor_decref (flux_reactor_t *r)
 {
     if (r && --r->usecount == 0) {
         int saved_errno = errno;
@@ -59,20 +49,24 @@ static void reactor_usecount_decr (flux_reactor_t *r)
     }
 }
 
-static void reactor_usecount_incr (flux_reactor_t *r)
+void flux_reactor_incref (flux_reactor_t *r)
 {
-    r->usecount++;
+    if (r)
+        r->usecount++;
 }
 
 void flux_reactor_destroy (flux_reactor_t *r)
 {
-    reactor_usecount_decr (r);
+    flux_reactor_decref (r);
 }
 
 flux_reactor_t *flux_reactor_create (int flags)
 {
-    flux_reactor_t *r = calloc (1, sizeof (*r));
-    if (!r)
+    flux_reactor_t *r;
+
+    if (valid_flags (flags, FLUX_REACTOR_SIGCHLD) < 0)
+        return NULL;
+    if (!(r = calloc (1, sizeof (*r))))
         return NULL;
     if ((flags & FLUX_REACTOR_SIGCHLD))
         r->loop = ev_default_loop (EVFLAG_SIGNALFD);
@@ -88,36 +82,13 @@ flux_reactor_t *flux_reactor_create (int flags)
     return r;
 }
 
-int flux_set_reactor (flux_t *h, flux_reactor_t *r)
-{
-    if (flux_aux_get (h, "flux::reactor")) {
-        errno = EEXIST;
-        return -1;
-    }
-    if (flux_aux_set (h, "flux::reactor", r, NULL) < 0)
-        return -1;
-    return 0;
-}
-
-flux_reactor_t *flux_get_reactor (flux_t *h)
-{
-    flux_reactor_t *r = flux_aux_get (h, "flux::reactor");
-    if (!r) {
-        if ((r = flux_reactor_create (0))) {
-            if (flux_aux_set (h, "flux::reactor", r,
-                              (flux_free_f)flux_reactor_destroy) < 0) {
-                flux_reactor_destroy (r);
-                r = NULL;
-            }
-        }
-    }
-    return r;
-}
-
 int flux_reactor_run (flux_reactor_t *r, int flags)
 {
     int ev_flags = 0;
     int count;
+
+    if (valid_flags (flags, FLUX_REACTOR_NOWAIT | FLUX_REACTOR_ONCE) < 0)
+        return -1;
     if (flags & FLUX_REACTOR_NOWAIT)
         ev_flags |= EVRUN_NOWAIT;
     if (flags & FLUX_REACTOR_ONCE)
@@ -166,63 +137,16 @@ void flux_reactor_active_decref (flux_reactor_t *r)
         ev_unref (r->loop);
 }
 
-static int events_to_libev (int events)
-{
-    int e = 0;
-    if (events & FLUX_POLLIN)
-        e |= EV_READ;
-    if (events & FLUX_POLLOUT)
-        e |= EV_WRITE;
-    if (events & FLUX_POLLERR)
-        e |= EV_ERROR;
-    return e;
-}
-
-static int libev_to_events (int events)
-{
-    int e = 0;
-    if (events & EV_READ)
-        e |= FLUX_POLLIN;
-    if (events & EV_WRITE)
-        e |= FLUX_POLLOUT;
-    if (events & EV_ERROR)
-        e |= FLUX_POLLERR;
-    return e;
-}
-
 /**
  ** Watchers
  **/
 
-flux_watcher_t *flux_watcher_create (flux_reactor_t *r,
-                                     size_t data_size,
-                                     struct flux_watcher_ops *ops,
-                                     flux_watcher_f fun, void *arg)
+void flux_watcher_set_priority (flux_watcher_t *w, int priority)
 {
-    struct flux_watcher *w = calloc (1, sizeof (*w) + data_size);
-    if (!w)
-        return NULL;
-    w->r = r;
-    w->ops = ops;
-    w->data = w + 1;
-    w->fn = fun;
-    w->arg = arg;
-    reactor_usecount_incr (r);
-    return w;
-}
-
-void * flux_watcher_get_data (flux_watcher_t *w)
-{
-    if (w)
-        return w->data;
-    return NULL;
-}
-
-struct flux_watcher_ops * flux_watcher_get_ops (flux_watcher_t *w)
-{
-    if (w)
-        return w->ops;
-    return NULL;
+    if (w) {
+        if (w->ops->set_priority)
+            w->ops->set_priority (w, priority);
+    }
 }
 
 void flux_watcher_start (flux_watcher_t *w)
@@ -248,8 +172,7 @@ void flux_watcher_destroy (flux_watcher_t *w)
             w->ops->stop (w);
         if (w->ops->destroy)
             w->ops->destroy (w);
-        if (w->r)
-            reactor_usecount_decr (w->r);
+        flux_reactor_decref (w->r);
         free (w);
     }
 }
@@ -277,7 +200,13 @@ static void watcher_stop_safe (flux_watcher_t *w)
     }
 }
 
-
+/* This is_active() callback works for "native" libev watchers, where
+ * w->data points to a struct ev_TYPE.
+ */
+static bool wrap_ev_active (flux_watcher_t *w)
+{
+    return ev_is_active (w->data);
+}
 
 /* flux_t handle
  */
@@ -292,6 +221,11 @@ static void handle_stop (flux_watcher_t *w)
     ev_flux_stop (w->r->loop, (struct ev_flux *)w->data);
 }
 
+static bool handle_is_active (flux_watcher_t *w)
+{
+    return ev_flux_is_active (w->data);
+}
+
 static void handle_cb (struct ev_loop *loop, struct ev_flux *fw, int revents)
 {
     struct flux_watcher *w = fw->data;
@@ -302,18 +236,21 @@ static void handle_cb (struct ev_loop *loop, struct ev_flux *fw, int revents)
 static struct flux_watcher_ops handle_watcher = {
     .start = handle_start,
     .stop = handle_stop,
+    .is_active = handle_is_active,
     .destroy = NULL,
 };
 
 flux_watcher_t *flux_handle_watcher_create (flux_reactor_t *r,
-                                            flux_t *h, int events,
-                                            flux_watcher_f cb, void *arg)
+                                            flux_t *h,
+                                            int events,
+                                            flux_watcher_f cb,
+                                            void *arg)
 {
     struct ev_flux *fw;
     flux_watcher_t *w;
-    if (!(w = flux_watcher_create (r, sizeof (*fw), &handle_watcher, cb, arg)))
+    if (!(w = watcher_create (r, sizeof (*fw), &handle_watcher, cb, arg)))
         return NULL;
-    fw = flux_watcher_get_data (w);
+    fw = watcher_get_data (w);
     ev_flux_init (fw, handle_cb, h, events_to_libev (events) & ~EV_ERROR);
     fw->data = w;
 
@@ -322,7 +259,7 @@ flux_watcher_t *flux_handle_watcher_create (flux_reactor_t *r,
 
 flux_t *flux_handle_watcher_get_flux (flux_watcher_t *w)
 {
-    assert (flux_watcher_get_ops (w) == &handle_watcher);
+    assert (watcher_get_ops (w) == &handle_watcher);
     struct ev_flux *fw = w->data;
     return fw->h;
 }
@@ -350,18 +287,22 @@ static void fd_cb (struct ev_loop *loop, ev_io *iow, int revents)
 static struct flux_watcher_ops fd_watcher = {
     .start = fd_start,
     .stop = fd_stop,
-    .destroy = NULL
+    .destroy = NULL,
+    .is_active = wrap_ev_active,
 };
 
-flux_watcher_t *flux_fd_watcher_create (flux_reactor_t *r, int fd, int events,
-                                        flux_watcher_f cb, void *arg)
+flux_watcher_t *flux_fd_watcher_create (flux_reactor_t *r,
+                                        int fd,
+                                        int events,
+                                        flux_watcher_f cb,
+                                        void *arg)
 {
     ev_io *iow;
     flux_watcher_t *w;
 
-    if (!(w = flux_watcher_create (r, sizeof (*iow), &fd_watcher, cb, arg)))
+    if (!(w = watcher_create (r, sizeof (*iow), &fd_watcher, cb, arg)))
         return NULL;
-    iow = flux_watcher_get_data (w);
+    iow = watcher_get_data (w);
     ev_io_init (iow, fd_cb, fd, events_to_libev (events) & ~EV_ERROR);
     iow->data = w;
 
@@ -370,271 +311,11 @@ flux_watcher_t *flux_fd_watcher_create (flux_reactor_t *r, int fd, int events,
 
 int flux_fd_watcher_get_fd (flux_watcher_t *w)
 {
-    assert (flux_watcher_get_ops (w) == &fd_watcher);
+    assert (watcher_get_ops (w) == &fd_watcher);
     ev_io *iow = w->data;
     return iow->fd;
 }
 
-/* buffer
- */
-
-static void buffer_read_start (flux_watcher_t *w)
-{
-    struct ev_buffer_read *ebr = (struct ev_buffer_read *)w->data;
-    ev_buffer_read_start (w->r->loop, ebr);
-}
-
-static void buffer_read_stop (flux_watcher_t *w)
-{
-    struct ev_buffer_read *ebr = (struct ev_buffer_read *)w->data;
-    ev_buffer_read_stop (w->r->loop, ebr);
-}
-
-static void buffer_read_destroy (flux_watcher_t *w)
-{
-    struct ev_buffer_read *ebr = (struct ev_buffer_read *)w->data;
-    ev_buffer_read_cleanup (ebr);
-}
-
-static void buffer_read_cb (struct ev_loop *loop,
-                            struct ev_buffer_read *ebr,
-                            int revents)
-{
-    struct flux_watcher *w = ebr->data;
-    if (w->fn)
-        w->fn (ev_userdata (loop), w, libev_to_events (revents), w->arg);
-}
-
-static struct flux_watcher_ops buffer_read_watcher = {
-    .start = buffer_read_start,
-    .stop = buffer_read_stop,
-    .destroy = buffer_read_destroy,
-};
-
-flux_watcher_t *flux_buffer_read_watcher_create (flux_reactor_t *r, int fd,
-                                                 int size, flux_watcher_f cb,
-                                                 int flags, void *arg)
-{
-    struct ev_buffer_read *ebr;
-    flux_watcher_t *w = NULL;
-    int fd_flags;
-
-    if (fd < 0) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if ((fd_flags = fd_get_flags (fd)) < 0)
-        return NULL;
-    if (!(fd_flags & O_NONBLOCK)) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    if (!(w = flux_watcher_create (r,
-                                   sizeof (*ebr),
-                                   &buffer_read_watcher,
-                                   cb,
-                                   arg)))
-        goto cleanup;
-
-    ebr = flux_watcher_get_data (w);
-
-    if (ev_buffer_read_init (ebr,
-                             fd,
-                             size,
-                             buffer_read_cb,
-                             r->loop) < 0)
-        goto cleanup;
-
-    if (flags & FLUX_WATCHER_LINE_BUFFER)
-        ebr->line = true;
-
-    ebr->data = w;
-
-    return w;
-
-cleanup:
-    flux_watcher_destroy (w);
-    return NULL;
-}
-
-flux_buffer_t *flux_buffer_read_watcher_get_buffer (flux_watcher_t *w)
-{
-    if (w)
-        return ((struct ev_buffer_read *)(w->data))->fb;
-    return NULL;
-}
-
-static void buffer_write_start (flux_watcher_t *w)
-{
-    struct ev_buffer_write *ebw = (struct ev_buffer_write *)w->data;
-    ev_buffer_write_start (w->r->loop, ebw);
-}
-
-static void buffer_write_stop (flux_watcher_t *w)
-{
-    struct ev_buffer_write *ebw = (struct ev_buffer_write *)w->data;
-    ev_buffer_write_stop (w->r->loop, ebw);
-}
-
-static void buffer_write_destroy (flux_watcher_t *w)
-{
-    struct ev_buffer_write *ebw = (struct ev_buffer_write *)w->data;
-    ev_buffer_write_cleanup (ebw);
-}
-
-static void buffer_write_cb (struct ev_loop *loop,
-                             struct ev_buffer_write *ebw,
-                             int revents)
-{
-    struct flux_watcher *w = ebw->data;
-    if (w->fn)
-        w->fn (ev_userdata (loop), w, libev_to_events (revents), w->arg);
-}
-
-static struct flux_watcher_ops buffer_write_watcher = {
-    .start = buffer_write_start,
-    .stop = buffer_write_stop,
-    .destroy = buffer_write_destroy,
-};
-
-flux_watcher_t *flux_buffer_write_watcher_create (flux_reactor_t *r, int fd,
-                                                  int size, flux_watcher_f cb,
-                                                  int flags, void *arg)
-{
-    struct ev_buffer_write *ebw;
-    flux_watcher_t *w = NULL;
-    int fd_flags;
-
-    if (fd < 0) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    if ((fd_flags = fd_get_flags (fd)) < 0)
-        return NULL;
-    if (!(fd_flags & O_NONBLOCK)) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    if (!(w = flux_watcher_create (r,
-                                   sizeof (*ebw),
-                                   &buffer_write_watcher,
-                                   cb,
-                                   arg)))
-        goto cleanup;
-
-    ebw = flux_watcher_get_data (w);
-
-    if (ev_buffer_write_init (ebw,
-                              fd,
-                              size,
-                              buffer_write_cb,
-                              r->loop) < 0)
-        goto cleanup;
-
-    ebw->data = w;
-
-    return w;
-
-cleanup:
-    flux_watcher_destroy (w);
-    return NULL;
-}
-
-flux_buffer_t *flux_buffer_write_watcher_get_buffer (flux_watcher_t *w)
-{
-    if (w)
-        return ((struct ev_buffer_write *)(w->data))->fb;
-    return NULL;
-}
-
-int flux_buffer_write_watcher_close (flux_watcher_t *w)
-{
-    struct ev_buffer_write *evw;
-    if (!w) {
-        errno = EINVAL;
-        return (-1);
-    }
-    evw = w->data;
-    if (evw->eof) {
-        errno = EINPROGRESS;
-        return (-1);
-    }
-    if (evw->closed) {
-        errno = EINVAL;
-        return (-1);
-    }
-    evw->eof = true;
-    flux_buffer_readonly (evw->fb);
-    ev_buffer_write_wakeup (evw);
-    return (0);
-}
-
-int flux_buffer_write_watcher_is_closed (flux_watcher_t *w, int *errp)
-{
-    if (w) {
-        struct ev_buffer_write *evw = w->data;
-        if (evw->closed && errp != NULL)
-            *errp = evw->close_errno;
-        return (evw->closed);
-    }
-    return (0);
-}
-
-/* 0MQ sockets
- */
-
-static void zmq_start (flux_watcher_t *w)
-{
-    ev_zmq_start (w->r->loop, (ev_zmq *)w->data);
-}
-
-static void zmq_stop (flux_watcher_t *w)
-{
-    ev_zmq_stop (w->r->loop, (ev_zmq *)w->data);
-}
-
-static void zmq_cb (struct ev_loop *loop, ev_zmq *pw, int revents)
-{
-    struct flux_watcher *w = pw->data;
-    if (w->fn)
-        w->fn (ev_userdata (loop), w, libev_to_events (revents), w->arg);
-}
-
-static struct flux_watcher_ops zmq_watcher  = {
-    .start = zmq_start,
-    .stop = zmq_stop,
-    .destroy = NULL,
-};
-
-flux_watcher_t *flux_zmq_watcher_create (flux_reactor_t *r,
-                                         void *zsock, int events,
-                                         flux_watcher_f cb, void *arg)
-{
-    ev_zmq *zw;
-    flux_watcher_t *w;
-
-    if (!(w = flux_watcher_create (r, sizeof (*zw), &zmq_watcher, cb, arg)))
-        return NULL;
-    zw = flux_watcher_get_data (w);
-    ev_zmq_init (zw, zmq_cb, zsock, events_to_libev (events) & ~EV_ERROR);
-    zw->data = w;
-
-    return w;
-}
-
-void *flux_zmq_watcher_get_zsock (flux_watcher_t *w)
-{
-    if (flux_watcher_get_ops (w) != &zmq_watcher) {
-        errno = EINVAL;
-        return NULL;
-    }
-    ev_zmq *zw = w->data;
-    return zw->zsock;
-}
-
 /* Timer
  */
 
@@ -659,11 +340,14 @@ static struct flux_watcher_ops timer_watcher = {
     .start = timer_start,
     .stop = timer_stop,
     .destroy = NULL,
+    .is_active = wrap_ev_active,
 };
 
 flux_watcher_t *flux_timer_watcher_create (flux_reactor_t *r,
-                                           double after, double repeat,
-                                           flux_watcher_f cb, void *arg)
+                                           double after,
+                                           double repeat,
+                                           flux_watcher_f cb,
+                                           void *arg)
 {
     ev_timer *tw;
     flux_watcher_t *w;
@@ -671,9 +355,9 @@ flux_watcher_t *flux_timer_watcher_create (flux_reactor_t *r,
         errno = EINVAL;
         return NULL;
     }
-    if (!(w = flux_watcher_create (r, sizeof (*tw), &timer_watcher, cb, arg)))
+    if (!(w = watcher_create (r, sizeof (*tw), &timer_watcher, cb, arg)))
         return NULL;
-    tw = flux_watcher_get_data (w);
+    tw = watcher_get_data (w);
     ev_timer_init (tw, timer_cb, after, repeat);
     tw->data = w;
 
@@ -682,11 +366,19 @@ flux_watcher_t *flux_timer_watcher_create (flux_reactor_t *r,
 
 void flux_timer_watcher_reset (flux_watcher_t *w, double after, double repeat)
 {
-    assert (flux_watcher_get_ops (w) == &timer_watcher);
+    assert (watcher_get_ops (w) == &timer_watcher);
     ev_timer *tw = w->data;
     ev_timer_set (tw, after, repeat);
 }
 
+void flux_timer_watcher_again (flux_watcher_t *w)
+{
+    assert (watcher_get_ops (w) == &timer_watcher);
+    ev_timer *tw = w->data;
+    struct ev_loop *loop = w->r->loop;
+    ev_timer_again (loop, tw);
+}
+
 /* Periodic
  */
 struct f_periodic {
@@ -707,6 +399,12 @@ static void periodic_stop (flux_watcher_t *w)
     ev_periodic_stop (w->r->loop, &fp->evp);
 }
 
+static bool periodic_is_active (flux_watcher_t *w)
+{
+    struct f_periodic *fp = w->data;
+    return ev_is_active (&fp->evp);
+}
+
 static void periodic_cb (struct ev_loop *loop, ev_periodic *pw, int revents)
 {
     struct f_periodic *fp = pw->data;
@@ -738,12 +436,15 @@ static struct flux_watcher_ops periodic_watcher = {
     .start = periodic_start,
     .stop = periodic_stop,
     .destroy = NULL,
+    .is_active = periodic_is_active,
 };
 
 flux_watcher_t *flux_periodic_watcher_create (flux_reactor_t *r,
-                                              double offset, double interval,
+                                              double offset,
+                                              double interval,
                                               flux_reschedule_f reschedule_cb,
-                                              flux_watcher_f cb, void *arg)
+                                              flux_watcher_f cb,
+                                              void *arg)
 {
     flux_watcher_t *w;
     struct f_periodic *fp;
@@ -752,39 +453,45 @@ flux_watcher_t *flux_periodic_watcher_create (flux_reactor_t *r,
         errno = EINVAL;
         return NULL;
     }
-    if (!(w = flux_watcher_create (r, size, &periodic_watcher, cb, arg)))
+    if (!(w = watcher_create (r, size, &periodic_watcher, cb, arg)))
         return NULL;
-    fp = flux_watcher_get_data (w);
+    fp = watcher_get_data (w);
     fp->evp.data = fp;
     fp->w = w;
     fp->reschedule_cb = reschedule_cb;
 
-    ev_periodic_init (&fp->evp, periodic_cb, offset, interval,
+    ev_periodic_init (&fp->evp,
+                      periodic_cb,
+                      offset,
+                      interval,
                       reschedule_cb ? periodic_reschedule_cb : NULL);
 
     return w;
 }
 
 void flux_periodic_watcher_reset (flux_watcher_t *w,
-                                  double next, double interval,
+                                  double next,
+                                  double interval,
                                   flux_reschedule_f reschedule_cb)
 {
     struct f_periodic *fp = w->data;
     struct ev_loop *loop = w->r->loop;
-    assert (flux_watcher_get_ops (w) == &periodic_watcher);
+    assert (watcher_get_ops (w) == &periodic_watcher);
     fp->reschedule_cb = reschedule_cb;
-    ev_periodic_set (&fp->evp, next, interval,
+    ev_periodic_set (&fp->evp,
+                     next,
+                     interval,
                      reschedule_cb ? periodic_reschedule_cb : NULL);
     ev_periodic_again (loop, &fp->evp);
 }
 
 double flux_watcher_next_wakeup (flux_watcher_t *w)
 {
-    if (flux_watcher_get_ops (w) == &periodic_watcher) {
+    if (watcher_get_ops (w) == &periodic_watcher) {
         struct f_periodic *fp = w->data;
         return ((double) ev_periodic_at (&fp->evp));
     }
-    else if (flux_watcher_get_ops (w) == &timer_watcher) {
+    else if (watcher_get_ops (w) == &timer_watcher) {
         ev_timer *tw = w->data;
         struct ev_loop *loop = w->r->loop;
         return ((double) (ev_now (loop) +  ev_timer_remaining (loop, tw)));
@@ -816,17 +523,19 @@ static struct flux_watcher_ops prepare_watcher = {
     .start = prepare_start,
     .stop = prepare_stop,
     .destroy = NULL,
+    .is_active = wrap_ev_active,
 };
 
 flux_watcher_t *flux_prepare_watcher_create (flux_reactor_t *r,
-                                             flux_watcher_f cb, void *arg)
+                                             flux_watcher_f cb,
+                                             void *arg)
 {
     ev_prepare *pw;
     flux_watcher_t *w;
 
-    if (!(w = flux_watcher_create (r, sizeof (*pw), &prepare_watcher, cb, arg)))
+    if (!(w = watcher_create (r, sizeof (*pw), &prepare_watcher, cb, arg)))
         return NULL;
-    pw = flux_watcher_get_data (w);
+    pw = watcher_get_data (w);
     ev_prepare_init (pw, prepare_cb);
     pw->data = w;
 
@@ -836,6 +545,11 @@ flux_watcher_t *flux_prepare_watcher_create (flux_reactor_t *r,
 /* Check
  */
 
+static void check_set_priority (flux_watcher_t *w, int priority)
+{
+    ev_set_priority ((ev_check *)w->data, priority);
+}
+
 static void check_start (flux_watcher_t *w)
 {
     ev_check_start (w->r->loop, (ev_check *)w->data);
@@ -854,20 +568,23 @@ static void check_cb (struct ev_loop *loop, ev_check *cw, int revents)
 }
 
 static struct flux_watcher_ops check_watcher = {
+    .set_priority = check_set_priority,
     .start = check_start,
     .stop = check_stop,
     .destroy = NULL,
+    .is_active = wrap_ev_active,
 };
 
 flux_watcher_t *flux_check_watcher_create (flux_reactor_t *r,
-                                           flux_watcher_f cb, void *arg)
+                                           flux_watcher_f cb,
+                                           void *arg)
 {
     ev_check *cw;
     flux_watcher_t *w;
 
-    if (!(w = flux_watcher_create (r, sizeof (*cw), &check_watcher, cb, arg)))
+    if (!(w = watcher_create (r, sizeof (*cw), &check_watcher, cb, arg)))
         return NULL;
-    cw = flux_watcher_get_data (w);
+    cw = watcher_get_data (w);
     ev_check_init (cw, check_cb);
     cw->data = w;
 
@@ -898,17 +615,19 @@ static struct flux_watcher_ops idle_watcher = {
     .start = idle_start,
     .stop = idle_stop,
     .destroy = NULL,
+    .is_active = wrap_ev_active,
 };
 
 flux_watcher_t *flux_idle_watcher_create (flux_reactor_t *r,
-                                          flux_watcher_f cb, void *arg)
+                                          flux_watcher_f cb,
+                                          void *arg)
 {
     ev_idle *iw;
     flux_watcher_t *w;
 
-    if (!(w = flux_watcher_create (r, sizeof (*iw), &idle_watcher, cb, arg)))
+    if (!(w = watcher_create (r, sizeof (*iw), &idle_watcher, cb, arg)))
         return NULL;
-    iw = flux_watcher_get_data (w);
+    iw = watcher_get_data (w);
     ev_idle_init (iw, idle_cb);
     iw->data = w;
 
@@ -939,12 +658,15 @@ static struct flux_watcher_ops child_watcher = {
     .start = child_start,
     .stop = child_stop,
     .destroy = NULL,
+    .is_active = wrap_ev_active,
 };
 
 
 flux_watcher_t *flux_child_watcher_create (flux_reactor_t *r,
-                                           int pid, bool trace,
-                                           flux_watcher_f cb, void *arg)
+                                           int pid,
+                                           bool trace,
+                                           flux_watcher_f cb,
+                                           void *arg)
 {
     flux_watcher_t *w;
     ev_child *cw;
@@ -953,9 +675,9 @@ flux_watcher_t *flux_child_watcher_create (flux_reactor_t *r,
         errno = EINVAL;
         return NULL;
     }
-    if (!(w = flux_watcher_create (r, sizeof (*cw), &child_watcher, cb, arg)))
+    if (!(w = watcher_create (r, sizeof (*cw), &child_watcher, cb, arg)))
         return NULL;
-    cw = flux_watcher_get_data (w);
+    cw = watcher_get_data (w);
     ev_child_init (cw, child_cb, pid, trace ? 1 : 0);
     cw->data = w;
 
@@ -964,7 +686,7 @@ flux_watcher_t *flux_child_watcher_create (flux_reactor_t *r,
 
 int flux_child_watcher_get_rpid (flux_watcher_t *w)
 {
-    if (flux_watcher_get_ops (w) != &child_watcher) {
+    if (watcher_get_ops (w) != &child_watcher) {
         errno = EINVAL;
         return -1;
     }
@@ -974,7 +696,7 @@ int flux_child_watcher_get_rpid (flux_watcher_t *w)
 
 int flux_child_watcher_get_rstatus (flux_watcher_t *w)
 {
-    if (flux_watcher_get_ops (w) != &child_watcher) {
+    if (watcher_get_ops (w) != &child_watcher) {
         errno = EINVAL;
         return -1;
     }
@@ -1006,17 +728,20 @@ static struct flux_watcher_ops signal_watcher = {
     .start = signal_start,
     .stop = signal_stop,
     .destroy = NULL,
+    .is_active = wrap_ev_active,
 };
 
-flux_watcher_t *flux_signal_watcher_create (flux_reactor_t *r, int signum,
-                                            flux_watcher_f cb, void *arg)
+flux_watcher_t *flux_signal_watcher_create (flux_reactor_t *r,
+                                            int signum,
+                                            flux_watcher_f cb,
+                                            void *arg)
 {
     flux_watcher_t *w;
     ev_signal *sw;
 
-    if (!(w = flux_watcher_create (r, sizeof (*sw), &signal_watcher, cb, arg)))
+    if (!(w = watcher_create (r, sizeof (*sw), &signal_watcher, cb, arg)))
         return NULL;
-    sw = flux_watcher_get_data (w);
+    sw = watcher_get_data (w);
     ev_signal_init (sw, signal_cb, signum);
     sw->data = w;
 
@@ -1025,7 +750,7 @@ flux_watcher_t *flux_signal_watcher_create (flux_reactor_t *r, int signum,
 
 int flux_signal_watcher_get_signum (flux_watcher_t *w)
 {
-    if (flux_watcher_get_ops (w) != &signal_watcher) {
+    if (watcher_get_ops (w) != &signal_watcher) {
         errno = EINVAL;
         return (-1);
     }
@@ -1057,18 +782,21 @@ static struct flux_watcher_ops stat_watcher = {
     .start = stat_start,
     .stop = stat_stop,
     .destroy = NULL,
+    .is_active = wrap_ev_active,
 };
 
 flux_watcher_t *flux_stat_watcher_create (flux_reactor_t *r,
-                                          const char *path, double interval,
-                                          flux_watcher_f cb, void *arg)
+                                          const char *path,
+                                          double interval,
+                                          flux_watcher_f cb,
+                                          void *arg)
 {
     flux_watcher_t *w;
     ev_stat *sw;
 
-    if (!(w = flux_watcher_create (r, sizeof (*sw), &stat_watcher, cb, arg)))
+    if (!(w = watcher_create (r, sizeof (*sw), &stat_watcher, cb, arg)))
         return NULL;
-    sw = flux_watcher_get_data (w);
+    sw = watcher_get_data (w);
     ev_stat_init (sw, stat_cb, path, interval);
     sw->data = w;
 
@@ -1076,16 +804,25 @@ flux_watcher_t *flux_stat_watcher_create (flux_reactor_t *r,
 }
 
 void flux_stat_watcher_get_rstat (flux_watcher_t *w,
-                                  struct stat *stat, struct stat *prev)
+                                  struct stat *stat,
+                                  struct stat *prev)
 {
     ev_stat *sw = w->data;
-    assert (flux_watcher_get_ops (w) == &stat_watcher);
+    assert (watcher_get_ops (w) == &stat_watcher);
     if (stat)
         *stat = sw->attr;
     if (prev)
         *prev = sw->prev;
 }
 
+bool flux_watcher_is_active (flux_watcher_t *w)
+{
+    if (w) {
+        if (w->ops->is_active)
+            return w->ops->is_active (w);
+    }
+    return false;
+}
 
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/common/libflux/reactor.h b/src/common/libflux/reactor.h
index fbb0b921b65b..3c8105cbfde7 100644
--- a/src/common/libflux/reactor.h
+++ b/src/common/libflux/reactor.h
@@ -15,9 +15,6 @@
 #include <sys/stat.h>
 #include <stdbool.h>
 
-#include "handle.h"
-#include "buffer.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -25,8 +22,6 @@ extern "C" {
 /* Reactor
  */
 
-typedef struct flux_reactor flux_reactor_t;
-
 /* Flags for flux_reactor_run()
  */
 enum {
@@ -42,16 +37,10 @@ enum {
                                /*    only one thread can do this per program */
 };
 
-/* Flags for buffer watchers */
-enum {
-    FLUX_WATCHER_LINE_BUFFER = 1, /* line buffer data before invoking callback */
-};
-
 flux_reactor_t *flux_reactor_create (int flags);
 void flux_reactor_destroy (flux_reactor_t *r);
-
-flux_reactor_t *flux_get_reactor (flux_t *h);
-int flux_set_reactor (flux_t *h, flux_reactor_t *r);
+void flux_reactor_incref (flux_reactor_t *r);
+void flux_reactor_decref (flux_reactor_t *r);
 
 int flux_reactor_run (flux_reactor_t *r, int flags);
 
@@ -73,92 +62,73 @@ void flux_reactor_active_decref (flux_reactor_t *r);
 /* Watchers
  */
 
-typedef struct flux_watcher flux_watcher_t;
+typedef void (*flux_watcher_f)(flux_reactor_t *r,
+                               flux_watcher_t *w,
+                               int revents,
+                               void *arg);
 
-typedef void (*flux_watcher_f)(flux_reactor_t *r, flux_watcher_t *w,
-                               int revents, void *arg);
+/* Set the watcher priority.  The range is [-2:2] (default 0).
+ * Higher priority watchers run first.
+ * This is a no-op if the underlying watcher doesn't support it.
+ * If the priority is out of range, the max or min value is set.
+ * The priority should only be set when the watcher is stopped.
+ * Currently only the check watcher supports it.
+ */
+void flux_watcher_set_priority (flux_watcher_t *w, int priority);
 
 void flux_watcher_start (flux_watcher_t *w);
 void flux_watcher_stop (flux_watcher_t *w);
 void flux_watcher_destroy (flux_watcher_t *w);
 double flux_watcher_next_wakeup (flux_watcher_t *w);
+bool flux_watcher_is_active (flux_watcher_t *w);
 
 /* flux_t handle
  */
 
 flux_watcher_t *flux_handle_watcher_create (flux_reactor_t *r,
-                                            flux_t *h, int events,
-                                            flux_watcher_f cb, void *arg);
+                                            flux_t *h,
+                                            int events,
+                                            flux_watcher_f cb,
+                                            void *arg);
 flux_t *flux_handle_watcher_get_flux (flux_watcher_t *w);
 
 /* file descriptor
  */
 
-flux_watcher_t *flux_fd_watcher_create (flux_reactor_t *r, int fd, int events,
-                                        flux_watcher_f cb, void *arg);
+flux_watcher_t *flux_fd_watcher_create (flux_reactor_t *r,
+                                        int fd, int events,
+                                        flux_watcher_f cb,
+                                        void *arg);
 int flux_fd_watcher_get_fd (flux_watcher_t *w);
 
-/* buffer
- */
-
-/* on eof, callback will be called with an empty buffer */
-/* if line buffered, second to last callback may not contain a full line */
-flux_watcher_t *flux_buffer_read_watcher_create (flux_reactor_t *r, int fd,
-                                                 int size, flux_watcher_f cb,
-                                                 int flags, void *arg);
-
-flux_buffer_t *flux_buffer_read_watcher_get_buffer (flux_watcher_t *w);
-
-/* 'cb' only called after fd closed (FLUX_POLLOUT) or error (FLUX_POLLERR) */
-flux_watcher_t *flux_buffer_write_watcher_create (flux_reactor_t *r, int fd,
-                                                  int size, flux_watcher_f cb,
-                                                  int flags, void *arg);
-
-flux_buffer_t *flux_buffer_write_watcher_get_buffer (flux_watcher_t *w);
-
-/* "write" EOF to buffer write watcher 'w'. The underlying fd will be closed
- *  once the buffer is emptied. The underlying flux_buffer_t will be marked
- *  readonly and subsequent flux_buffer_write* calls will return EROFS.
- *
- *  Once close(2) completes, the watcher callback is called with FLUX_POLLOUT.
- *  Use flux_buffer_write_watcher_is_closed() to check for errors.
- *
- * Returns 0 on success, -1 on error with errno set.
- */
-int flux_buffer_write_watcher_close (flux_watcher_t *w);
-
-/* Returns 1 if write watcher is closed, errnum from close in close_err */
-int flux_buffer_write_watcher_is_closed (flux_watcher_t *w, int *close_err);
-
-/* zmq socket
- */
-
-flux_watcher_t *flux_zmq_watcher_create (flux_reactor_t *r,
-                                         void *zsock, int events,
-                                         flux_watcher_f cb, void *arg);
-void *flux_zmq_watcher_get_zsock (flux_watcher_t *w);
-
 /* timer
  */
 
 flux_watcher_t *flux_timer_watcher_create (flux_reactor_t *r,
-                                           double after, double repeat,
-                                           flux_watcher_f cb, void *arg);
+                                           double after,
+                                           double repeat,
+                                           flux_watcher_f cb,
+                                           void *arg);
 
 void flux_timer_watcher_reset (flux_watcher_t *w, double after, double repeat);
 
+void flux_timer_watcher_again (flux_watcher_t *w);
+
 /* periodic
  */
 
 typedef double (*flux_reschedule_f) (flux_watcher_t *w, double now, void *arg);
 
 flux_watcher_t *flux_periodic_watcher_create (flux_reactor_t *r,
-                                             double offset, double interval,
+                                             double offset,
+                                             double interval,
                                              flux_reschedule_f reschedule_cb,
-                                             flux_watcher_f cb, void *arg);
+                                             flux_watcher_f cb,
+                                             void *arg);
 
 void flux_periodic_watcher_reset (flux_watcher_t *w,
-                                  double next_wakeup, double interval,
+                                  double next_wakeup,
+                                  double interval,
                                   flux_reschedule_f reschedule_cb);
 
 
@@ -166,28 +136,35 @@ void flux_periodic_watcher_reset (flux_watcher_t *w,
  */
 
 flux_watcher_t *flux_prepare_watcher_create (flux_reactor_t *r,
-                                             flux_watcher_f cb, void *arg);
+                                             flux_watcher_f cb,
+                                             void *arg);
 
 flux_watcher_t *flux_check_watcher_create (flux_reactor_t *r,
-                                          flux_watcher_f cb, void *arg);
+                                           flux_watcher_f cb,
+                                           void *arg);
 
 flux_watcher_t *flux_idle_watcher_create (flux_reactor_t *r,
-                                          flux_watcher_f cb, void *arg);
+                                          flux_watcher_f cb,
+                                          void *arg);
 
 /* child
  */
 
 flux_watcher_t *flux_child_watcher_create (flux_reactor_t *r,
-                                           int pid, bool trace,
-                                           flux_watcher_f cb, void *arg);
+                                           int pid,
+                                           bool trace,
+                                           flux_watcher_f cb,
+                                           void *arg);
 int flux_child_watcher_get_rpid (flux_watcher_t *w);
 int flux_child_watcher_get_rstatus (flux_watcher_t *w);
 
 /* signal
  */
 
-flux_watcher_t *flux_signal_watcher_create (flux_reactor_t *r, int signum,
-                                            flux_watcher_f cb, void *arg);
+flux_watcher_t *flux_signal_watcher_create (flux_reactor_t *r,
+                                            int signum,
+                                            flux_watcher_f cb,
+                                            void *arg);
 
 int flux_signal_watcher_get_signum (flux_watcher_t *w);
 
@@ -195,38 +172,13 @@ int flux_signal_watcher_get_signum (flux_watcher_t *w);
  */
 
 flux_watcher_t *flux_stat_watcher_create (flux_reactor_t *r,
-                                          const char *path, double interval,
-                                          flux_watcher_f cb, void *arg);
+                                          const char *path,
+                                          double interval,
+                                          flux_watcher_f cb,
+                                          void *arg);
 void flux_stat_watcher_get_rstat (flux_watcher_t *w,
-                                  struct stat *stat, struct stat *prev);
-
-/* Custom watcher construction functions:
- */
-
-struct flux_watcher_ops {
-    void (*start) (flux_watcher_t *w);
-    void (*stop) (flux_watcher_t *w);
-    void (*destroy) (flux_watcher_t *w);
-};
-
-/*  Create a custom watcher on reactor 'r' with 'data_size' bytes reserved
- *   for the implementor, implementation operations in 'ops' and user
- *   watcher callback and data 'fn' and 'arg'.
- *
- *  Caller retrieves pointer to allocated implementation data with
- *   flux_watcher_data (w).
- */
-flux_watcher_t * flux_watcher_create (flux_reactor_t *r, size_t data_size,
-                                      struct flux_watcher_ops *ops,
-                                      flux_watcher_f fn, void *arg);
-
-/*  Return pointer to implementation data reserved by watcher object 'w'.
- */
-void * flux_watcher_get_data (flux_watcher_t *w);
-
-/*  Return pointer to flux_watcher_ops structure for this watcher.
- */
-struct flux_watcher_ops * flux_watcher_get_ops (flux_watcher_t *w);
+                                  struct stat *stat,
+                                  struct stat *prev);
 
 #ifdef __cplusplus
 }
diff --git a/src/common/libflux/reactor_private.h b/src/common/libflux/reactor_private.h
new file mode 100644
index 000000000000..a4f8f40fa4de
--- /dev/null
+++ b/src/common/libflux/reactor_private.h
@@ -0,0 +1,110 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_REACTOR_PRIVATE_H
+#define _FLUX_CORE_REACTOR_PRIVATE_H
+
+#include "src/common/libev/ev.h"
+#include "reactor.h"
+
+struct flux_watcher_ops {
+    void (*set_priority) (flux_watcher_t *w, int priority);
+    void (*start) (flux_watcher_t *w);
+    void (*stop) (flux_watcher_t *w);
+    void (*destroy) (flux_watcher_t *w);
+    bool (*is_active) (flux_watcher_t *w);
+};
+
+struct flux_reactor {
+    struct ev_loop *loop;
+    int usecount;
+    unsigned int errflag:1;
+};
+
+struct flux_watcher {
+    flux_reactor_t *r;
+    flux_watcher_f fn;
+    void *arg;
+    struct flux_watcher_ops *ops;
+    void *data;
+};
+
+static inline int events_to_libev (int events)
+{
+    int e = 0;
+    if (events & FLUX_POLLIN)
+        e |= EV_READ;
+    if (events & FLUX_POLLOUT)
+        e |= EV_WRITE;
+    if (events & FLUX_POLLERR)
+        e |= EV_ERROR;
+    return e;
+}
+
+static inline int libev_to_events (int events)
+{
+    int e = 0;
+    if (events & EV_READ)
+        e |= FLUX_POLLIN;
+    if (events & EV_WRITE)
+        e |= FLUX_POLLOUT;
+    if (events & EV_ERROR)
+        e |= FLUX_POLLERR;
+    return e;
+}
+
+/*  Create a custom watcher on reactor 'r' with 'data_size' bytes reserved
+ *   for the implementor, implementation operations in 'ops' and user
+ *   watcher callback and data 'fn' and 'arg'.
+ *
+ *  Caller retrieves pointer to allocated implementation data with
+ *   flux_watcher_data (w).
+ */
+static inline flux_watcher_t *watcher_create (flux_reactor_t *r,
+                                              size_t data_size,
+                                              struct flux_watcher_ops *ops,
+                                              flux_watcher_f fn,
+                                              void *arg)
+{
+    struct flux_watcher *w = calloc (1, sizeof (*w) + data_size);
+    if (!w)
+        return NULL;
+    w->r = r;
+    w->ops = ops;
+    w->data = w + 1;
+    w->fn = fn;
+    w->arg = arg;
+    flux_reactor_incref (r);
+    return w;
+}
+
+/*  Return pointer to implementation data reserved by watcher object 'w'.
+ */
+static inline void *watcher_get_data (flux_watcher_t *w)
+{
+    if (w)
+        return w->data;
+    return NULL;
+}
+
+/*  Return pointer to flux_watcher_ops structure for this watcher.
+ */
+static inline struct flux_watcher_ops *watcher_get_ops (flux_watcher_t *w)
+{
+    if (w)
+        return w->ops;
+    return NULL;
+}
+
+#endif /* !_FLUX_CORE_REACTOR_PRIVATE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/request.c b/src/common/libflux/request.c
index 70b3ad052e82..d42f09d8e562 100644
--- a/src/common/libflux/request.c
+++ b/src/common/libflux/request.c
@@ -13,8 +13,7 @@
 #endif
 #include <errno.h>
 #include <stdarg.h>
-#include "request.h"
-#include "message.h"
+#include <flux/core.h>
 
 static int request_decode (const flux_msg_t *msg, const char **topic)
 {
@@ -41,7 +40,8 @@ static int request_decode (const flux_msg_t *msg, const char **topic)
     return rc;
 }
 
-int flux_request_decode (const flux_msg_t *msg, const char **topicp,
+int flux_request_decode (const flux_msg_t *msg,
+                         const char **topicp,
                          const char **sp)
 {
     const char *topic, *s;
@@ -60,12 +60,14 @@ int flux_request_decode (const flux_msg_t *msg, const char **topicp,
     return rc;
 }
 
-int flux_request_decode_raw (const flux_msg_t *msg, const char **topic,
-                             const void **data, int *len)
+int flux_request_decode_raw (const flux_msg_t *msg,
+                             const char **topic,
+                             const void **data,
+                             size_t *len)
 {
     const char *ts;
     const void *d = NULL;
-    int l = 0;
+    size_t l = 0;
     int rc = -1;
 
     if (!data || !len) {
@@ -88,8 +90,10 @@ int flux_request_decode_raw (const flux_msg_t *msg, const char **topic,
     return rc;
 }
 
-static int flux_request_vunpack (const flux_msg_t *msg, const char **topic,
-                                 const char *fmt, va_list ap)
+static int flux_request_vunpack (const flux_msg_t *msg,
+                                 const char **topic,
+                                 const char *fmt,
+                                 va_list ap)
 {
     const char *ts;
     int rc = -1;
@@ -109,8 +113,10 @@ static int flux_request_vunpack (const flux_msg_t *msg, const char **topic,
     return rc;
 }
 
-int flux_request_unpack (const flux_msg_t *msg, const char **topic,
-                         const char *fmt, ...)
+int flux_request_unpack (const flux_msg_t *msg,
+                         const char **topic,
+                         const char *fmt,
+                         ...)
 {
     va_list ap;
     int rc;
@@ -133,8 +139,7 @@ static flux_msg_t *request_encode (const char *topic)
         goto error;
     if (flux_msg_set_topic (msg, topic) < 0)
         goto error;
-    if (flux_msg_enable_route (msg) < 0)
-        goto error;
+    flux_msg_route_enable (msg);
     return msg;
 error:
     flux_msg_destroy (msg);
@@ -156,7 +161,8 @@ flux_msg_t *flux_request_encode (const char *topic, const char *s)
 }
 
 flux_msg_t *flux_request_encode_raw (const char *topic,
-                                     const void *data, int len)
+                                     const void *data,
+                                     size_t len)
 {
     flux_msg_t *msg = request_encode (topic);
 
diff --git a/src/common/libflux/request.h b/src/common/libflux/request.h
index 80d8e45f70a9..b460fdee9d17 100644
--- a/src/common/libflux/request.h
+++ b/src/common/libflux/request.h
@@ -11,8 +11,6 @@
 #ifndef _FLUX_CORE_REQUEST_H
 #define _FLUX_CORE_REQUEST_H
 
-#include "message.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -22,15 +20,18 @@ extern "C" {
  * If s is non-NULL, assign the string payload or set to NULL if none
  * exists.  Returns 0 on success, or -1 on failure with errno set.
  */
-int flux_request_decode (const flux_msg_t *msg, const char **topic,
+int flux_request_decode (const flux_msg_t *msg,
+                         const char **topic,
                          const char **s);
 
 /* Decode a request message with required json payload.  These functions use
  * jansson unpack style variable arguments for decoding the JSON object
  * payload directly.  Returns 0 on success, or -1 on failure with errno set.
  */
-int flux_request_unpack (const flux_msg_t *msg, const char **topic,
-                         const char *fmt, ...);
+int flux_request_unpack (const flux_msg_t *msg,
+                         const char **topic,
+                         const char *fmt,
+                         ...);
 
 /* Decode a request message with optional raw payload.
  * If topic is non-NULL, assign the request topic string.
@@ -38,8 +39,10 @@ int flux_request_unpack (const flux_msg_t *msg, const char **topic,
  * If there is no payload, they will be assigned NULL and zero.
  * Returns 0 on success, or -1 on failure with errno set.
  */
-int flux_request_decode_raw (const flux_msg_t *msg, const char **topic,
-                             const void **data, int *len);
+int flux_request_decode_raw (const flux_msg_t *msg,
+                             const char **topic,
+                             const void **data,
+                             size_t *len);
 
 /* Encode a request message with optional string payload.
  * If s is non-NULL, assign the string payload.
@@ -51,7 +54,8 @@ flux_msg_t *flux_request_encode (const char *topic, const char *s);
  * Otherwise there will be no payload.
  */
 flux_msg_t *flux_request_encode_raw (const char *topic,
-                                     const void *data, int len);
+                                     const void *data,
+                                     size_t len);
 
 #ifdef __cplusplus
 }
diff --git a/src/common/libflux/response.c b/src/common/libflux/response.c
index 2cd5320ca4ae..cef0718eadf1 100644
--- a/src/common/libflux/response.c
+++ b/src/common/libflux/response.c
@@ -47,7 +47,8 @@ static int response_decode (const flux_msg_t *msg, const char **topic)
     return rc;
 }
 
-int flux_response_decode (const flux_msg_t *msg, const char **topicp,
+int flux_response_decode (const flux_msg_t *msg,
+                          const char **topicp,
                           const char **sp)
 {
     const char *topic, *s;
@@ -67,12 +68,14 @@ int flux_response_decode (const flux_msg_t *msg, const char **topicp,
     return rc;
 }
 
-int flux_response_decode_raw (const flux_msg_t *msg, const char **topic,
-                              const void **data, int *len)
+int flux_response_decode_raw (const flux_msg_t *msg,
+                              const char **topic,
+                              const void **data,
+                              size_t *len)
 {
     const char *ts;
     const void *d = NULL;
-    int l = 0;
+    size_t l = 0;
     int rc = -1;
 
     if (!data || !len) {
@@ -139,8 +142,7 @@ static flux_msg_t *response_encode (const char *topic, int errnum)
         goto error;
     if (flux_msg_set_topic (msg, topic) < 0)
         goto error;
-    if (flux_msg_enable_route (msg) < 0)
-        goto error;
+    flux_msg_route_enable (msg);
     if (flux_msg_set_errnum (msg, errnum) < 0)
         goto error;
     return msg;
@@ -164,7 +166,8 @@ flux_msg_t *flux_response_encode (const char *topic, const char *s)
 }
 
 flux_msg_t *flux_response_encode_raw (const char *topic,
-                                      const void *data, int len)
+                                      const void *data,
+                                      size_t len)
 {
     flux_msg_t *msg;
 
@@ -178,7 +181,8 @@ flux_msg_t *flux_response_encode_raw (const char *topic,
     return NULL;
 }
 
-flux_msg_t *flux_response_encode_error (const char *topic, int errnum,
+flux_msg_t *flux_response_encode_error (const char *topic,
+                                        int errnum,
                                         const char *errstr)
 {
     flux_msg_t *msg;
@@ -200,15 +204,19 @@ flux_msg_t *flux_response_encode_error (const char *topic, int errnum,
 flux_msg_t *flux_response_derive (const flux_msg_t *request, int errnum)
 {
     flux_msg_t *msg;
+    uint32_t matchtag;
 
-    if (!request) {
+    if (!request || flux_msg_is_noresponse (request)) {
         errno = EINVAL;
         return NULL;
     }
-    if (!(msg = flux_msg_copy (request, false)))
+    if (flux_msg_get_matchtag (request, &matchtag) < 0
+        || !(msg = flux_msg_copy (request, false)))
         return NULL;
     if (flux_msg_set_type (msg, FLUX_MSGTYPE_RESPONSE) < 0)
         goto error;
+    if (flux_msg_set_matchtag (msg, matchtag) < 0)
+        goto error;
     if (flux_msg_set_userid (msg, FLUX_USERID_UNKNOWN) < 0)
         goto error;
     if (flux_msg_set_rolemask (msg, FLUX_ROLE_NONE) < 0)
@@ -223,51 +231,49 @@ flux_msg_t *flux_response_derive (const flux_msg_t *request, int errnum)
 
 int flux_respond (flux_t *h, const flux_msg_t *request, const char *s)
 {
-    flux_msg_t *msg = NULL;
+    flux_msg_t *msg;
 
-    if (!h || !request)
-        goto inval;
-    msg = flux_response_derive (request, 0);
-    if (!msg)
-        goto error;
-    if (s && flux_msg_set_string (msg, s) < 0)
-        goto error;
-    if (flux_send (h, msg, 0) < 0)
-        goto error;
-    flux_msg_destroy (msg);
+    if (!h || !request) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_msg_is_noresponse (request))
+        return 0;
+    if (!(msg = flux_response_derive (request, 0))
+        || (s && flux_msg_set_string (msg, s) < 0)
+        || flux_send_new (h, &msg, 0) < 0) {
+        flux_msg_destroy (msg);
+        return -1;
+    }
     return 0;
-inval:
-    errno = EINVAL;
-error:
-    flux_msg_destroy (msg);
-    return -1;
 }
 
-static int flux_respond_vpack (flux_t *h, const flux_msg_t *request,
-                               const char *fmt, va_list ap)
+static int flux_respond_vpack (flux_t *h,
+                               const flux_msg_t *request,
+                               const char *fmt,
+                               va_list ap)
 {
-    flux_msg_t *msg = NULL;
+    flux_msg_t *msg;
 
-    if (!h || !request || !fmt)
-        goto inval;
-    msg = flux_response_derive (request, 0);
-    if (!msg)
-        goto error;
-    if (flux_msg_vpack (msg, fmt, ap) < 0)
-        goto error;
-    if (flux_send (h, msg, 0) < 0)
-        goto error;
-    flux_msg_destroy (msg);
+    if (!h || !request || !fmt) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_msg_is_noresponse (request))
+        return 0;
+    if (!(msg = flux_response_derive (request, 0))
+        || flux_msg_vpack (msg, fmt, ap) < 0
+        || flux_send_new (h, &msg, 0) < 0) {
+        flux_msg_destroy (msg);
+        return -1;
+    }
     return 0;
-inval:
-    errno = EINVAL;
-error:
-    flux_msg_destroy (msg);
-    return -1;
 }
 
-int flux_respond_pack (flux_t *h, const flux_msg_t *request,
-                       const char *fmt, ...)
+int flux_respond_pack (flux_t *h,
+                       const flux_msg_t *request,
+                       const char *fmt,
+                       ...)
 {
     int rc;
     va_list ap;
@@ -282,52 +288,50 @@ int flux_respond_pack (flux_t *h, const flux_msg_t *request,
     return rc;
 }
 
-int flux_respond_raw (flux_t *h, const flux_msg_t *request,
-                      const void *data, int len)
+int flux_respond_raw (flux_t *h,
+                      const flux_msg_t *request,
+                      const void *data,
+                      int len)
 {
-    flux_msg_t *msg = NULL;
+    flux_msg_t *msg;
 
-    if (!h || !request)
-        goto inval;
-    msg  = flux_response_derive (request, 0);
-    if (!msg)
-        goto error;
-    if (data && flux_msg_set_payload (msg, data, len) < 0)
-        goto error;
-    if (flux_send (h, msg, 0) < 0)
-        goto error;
-    flux_msg_destroy (msg);
+    if (!h || !request) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_msg_is_noresponse (request))
+        return 0;
+    if (!(msg  = flux_response_derive (request, 0))
+        || (data && flux_msg_set_payload (msg, data, len) < 0)
+        || flux_send_new (h, &msg, 0) < 0) {
+        flux_msg_destroy (msg);
+        return -1;
+    }
     return 0;
-inval:
-    errno = EINVAL;
-error:
-    flux_msg_destroy (msg);
-    return -1;
 }
 
-int flux_respond_error (flux_t *h, const flux_msg_t *request,
-                        int errnum, const char *errstr)
+int flux_respond_error (flux_t *h,
+                        const flux_msg_t *request,
+                        int errnum,
+                        const char *errstr)
 {
-    flux_msg_t *msg = NULL;
+    flux_msg_t *msg;
 
-    if (!h || !request || errnum == 0)
-        goto inval;
-    msg = flux_response_derive (request, errnum);
-    if (!msg)
-        goto error;
-    if (errstr) {
-        if (flux_msg_set_string (msg, errstr) < 0)
-            goto error;
+    if (!h || !request) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (errnum == 0)
+        errnum = EINVAL;
+    if (flux_msg_is_noresponse (request))
+        return 0;
+    if (!(msg = flux_response_derive (request, errnum))
+        || (errstr && flux_msg_set_string (msg, errstr) < 0)
+        || flux_send_new (h, &msg, 0) < 0) {
+        flux_msg_destroy (msg);
+        return -1;
     }
-    if (flux_send (h, msg, 0) < 0)
-        goto error;
-    flux_msg_destroy (msg);
     return 0;
-inval:
-    errno = EINVAL;
-error:
-    flux_msg_destroy (msg);
-    return -1;
 }
 
 /*
diff --git a/src/common/libflux/response.h b/src/common/libflux/response.h
index 781a4e2edfb2..d06839d8d8a9 100644
--- a/src/common/libflux/response.h
+++ b/src/common/libflux/response.h
@@ -11,9 +11,6 @@
 #ifndef _FLUX_CORE_RESPONSE_H
 #define _FLUX_CORE_RESPONSE_H
 
-#include "message.h"
-#include "handle.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -26,7 +23,8 @@ extern "C" {
  * to topic or s.  Returns 0 on success, or -1 on failure with
  * errno set.
  */
-int flux_response_decode (const flux_msg_t *msg, const char **topic,
+int flux_response_decode (const flux_msg_t *msg,
+                          const char **topic,
                           const char **s);
 
 /* Decode a response message, with optional raw payload.
@@ -37,8 +35,10 @@ int flux_response_decode (const flux_msg_t *msg, const char **topic,
  * and -1 is returned with no assignments to topic, data, or len.
  * Returns 0 on success, or -1 on failure with errno set.
  */
-int flux_response_decode_raw (const flux_msg_t *msg, const char **topic,
-                              const void **data, int *len);
+int flux_response_decode_raw (const flux_msg_t *msg,
+                              const char **topic,
+                              const void **data,
+                              size_t *len);
 
 /* If failed response includes an error string payload, assign to 'errstr',
  * otherwise fail.
@@ -54,12 +54,14 @@ flux_msg_t *flux_response_encode (const char *topic, const char *s);
 /* Encode a response message with optional raw payload.
  */
 flux_msg_t *flux_response_encode_raw (const char *topic,
-                                      const void *data, int len);
+                                      const void *data,
+                                      size_t len);
 
 /* Encode an error response with 'errnum' (must be nonzero) and
  * if non-NULL, an error string payload.
  */
-flux_msg_t *flux_response_encode_error (const char *topic, int errnum,
+flux_msg_t *flux_response_encode_error (const char *topic,
+                                        int errnum,
                                         const char *errstr);
 
 /* Derive a response message from a request message, setting 'errnum' to
@@ -76,20 +78,27 @@ int flux_respond (flux_t *h, const flux_msg_t *request, const char *s);
  * jansson pack style variable arguments for encoding the JSON object
  * payload directly.
  */
-int flux_respond_pack (flux_t *h, const flux_msg_t *request,
-                       const char *fmt, ...);
+int flux_respond_pack (flux_t *h,
+                       const flux_msg_t *request,
+                       const char *fmt,
+                       ...);
 
 
 /* Create a response to the provided request message with optional raw payload.
  */
-int flux_respond_raw (flux_t *h, const flux_msg_t *request,
-                      const void *data, int len);
+int flux_respond_raw (flux_t *h,
+                      const flux_msg_t *request,
+                      const void *data,
+                      int len);
 
 /* Create an error response to the provided request message with optional
- * error string payload (if errstr is non-NULL).
+ * error string payload (if errstr is non-NULL).  If errnum is zero, EINVAL
+ * is substituted.
  */
-int flux_respond_error (flux_t *h, const flux_msg_t *request,
-                        int errnum, const char *errstr);
+int flux_respond_error (flux_t *h,
+                        const flux_msg_t *request,
+                        int errnum,
+                        const char *errstr);
 
 
 #ifdef __cplusplus
diff --git a/src/common/libflux/rpc.c b/src/common/libflux/rpc.c
index 48d85dbe8627..74917c371723 100644
--- a/src/common/libflux/rpc.c
+++ b/src/common/libflux/rpc.c
@@ -16,24 +16,14 @@
 #include <stdlib.h>
 #include <stdbool.h>
 #include <stdarg.h>
-#if HAVE_CALIPER
-#include <caliper/cali.h>
-#include <sys/syscall.h>
-#endif
 #include <jansson.h>
-#include <czmq.h>
+#include <flux/core.h>
 
-#include "request.h"
-#include "response.h"
-#include "message.h"
-#include "attr.h"
-#include "rpc.h"
-#include "reactor.h"
-#include "msg_handler.h"
-#include "flog.h"
+#include "src/common/libutil/errno_safe.h"
 
 struct flux_rpc {
     uint32_t matchtag;
+    uint32_t nodeid;
     int flags;
     flux_future_t *f;
     bool sent;
@@ -80,6 +70,7 @@ static int rpc_finalize (struct flux_rpc *rpc)
 static void rpc_destroy (struct flux_rpc *rpc)
 {
     if (rpc) {
+        int saved_errno = errno;
         if (rpc_finalize (rpc) < 0)
             log_matchtag_leak (flux_future_get_flux (rpc->f),
                                (rpc->flags & FLUX_RPC_STREAMING)
@@ -87,18 +78,21 @@ static void rpc_destroy (struct flux_rpc *rpc)
                                     : "unfulfilled RPC",
                                rpc->matchtag);
         free (rpc);
+        errno = saved_errno;
     }
 }
 
-static struct flux_rpc *rpc_create (flux_t *h, flux_future_t *f, int flags)
+static struct flux_rpc *rpc_create (flux_t *h,
+                                    flux_future_t *f,
+                                    uint32_t nodeid,
+                                    int flags)
 {
     struct flux_rpc *rpc;
 
-    if (!(rpc = calloc (1, sizeof (*rpc)))) {
-        errno = ENOMEM;
-        goto error;
-    }
+    if (!(rpc = calloc (1, sizeof (*rpc))))
+        return NULL;
     rpc->f = f;
+    rpc->nodeid = nodeid;
     rpc->flags = flags;
     if ((flags & FLUX_RPC_NORESPONSE)) {
         rpc->matchtag = FLUX_MATCHTAG_NONE;
@@ -111,7 +105,7 @@ static struct flux_rpc *rpc_create (flux_t *h, flux_future_t *f, int flags)
     flux_future_set_flux (f, h);
     return rpc;
 error:
-    free (rpc);
+    ERRNO_SAFE_WRAP (free, rpc);
     return NULL;
 }
 
@@ -129,7 +123,7 @@ int flux_rpc_get (flux_future_t *f, const char **s)
     return rc;
 }
 
-int flux_rpc_get_raw (flux_future_t *f, const void **data, int *len)
+int flux_rpc_get_raw (flux_future_t *f, const void **data, size_t *len)
 {
     const flux_msg_t *msg;
     int rc = -1;
@@ -174,25 +168,25 @@ int flux_rpc_get_unpack (flux_future_t *f, const char *fmt, ...)
  * instead of flux_rpc_get() to test result of RPC with no response payload.
  * Fulfill future.
 */
-static void response_cb (flux_t *h, flux_msg_handler_t *mh,
-                         const flux_msg_t *msg, void *arg)
+static void response_cb (flux_t *h,
+                         flux_msg_handler_t *mh,
+                         const flux_msg_t *msg,
+                         void *arg)
 {
     flux_future_t *f = arg;
+    struct flux_rpc *rpc = flux_future_aux_get (f, "flux::rpc");
     flux_msg_t *cpy;
     int saved_errno;
     const char *errstr;
 
-#if HAVE_CALIPER
-    cali_begin_string_byname ("flux.message.rpc", "single");
-#endif
-#if HAVE_CALIPER
-    cali_end_byname ("flux.message.rpc");
-#endif
     if (flux_response_decode (msg, NULL, NULL) < 0)
         goto error;
     if (!(cpy = flux_msg_copy (msg, true)))
         goto error;
     flux_future_fulfill (f, cpy, (flux_free_f)flux_msg_destroy);
+
+    if (!(rpc->flags & FLUX_RPC_STREAMING))
+        flux_msg_handler_stop (mh);
     return;
 error:
     saved_errno = errno;
@@ -203,6 +197,7 @@ static void response_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_future_fulfill_error (f, saved_errno, errstr);
     else
         flux_future_fulfill_error (f, saved_errno, NULL);
+    flux_msg_handler_stop (mh);
 }
 
 /* Callback to initialize future in main or alternate reactor contexts.
@@ -230,51 +225,42 @@ static void initialize_cb (flux_future_t *f, void *arg)
     flux_future_fulfill_error (f, errno, NULL);
 }
 
-static flux_future_t *flux_rpc_message_nocopy (flux_t *h,
-                                               flux_msg_t *msg,
-                                               uint32_t nodeid,
-                                               int flags)
+static flux_future_t *flux_rpc_message_send_new (flux_t *h,
+                                                 flux_msg_t **msg,
+                                                 uint32_t nodeid,
+                                                 int flags)
 {
     struct flux_rpc *rpc = NULL;
     flux_future_t *f;
-    uint8_t msgflags;
 
     if (!(f = flux_future_create (initialize_cb, NULL)))
         goto error;
-    if (!(rpc = rpc_create (h, f, flags)))
+    if (!(rpc = rpc_create (h, f, nodeid, flags)))
         goto error;
     if (flux_future_aux_set (f, "flux::rpc", rpc,
                              (flux_free_f)rpc_destroy) < 0) {
         rpc_destroy (rpc);
         goto error;
     }
-    if (flux_msg_set_matchtag (msg, rpc->matchtag) < 0)
-        goto error;
-    if (flux_msg_get_flags (msg, &msgflags) < 0)
+    if (flux_msg_set_matchtag (*msg, rpc->matchtag) < 0)
         goto error;
     if (nodeid == FLUX_NODEID_UPSTREAM) {
-        msgflags |= FLUX_MSGFLAG_UPSTREAM;
+        if (flux_msg_set_flag (*msg, FLUX_MSGFLAG_UPSTREAM) < 0)
+            goto error;
         if (flux_get_rank (h, &nodeid) < 0)
             goto error;
     }
-    if ((flags & FLUX_RPC_STREAMING))
-        msgflags |= FLUX_MSGFLAG_STREAMING;
-    if (flux_msg_set_flags (msg, msgflags) < 0)
-        goto error;
-    if (flux_msg_set_nodeid (msg, nodeid) < 0)
+    if ((flags & FLUX_RPC_STREAMING)) {
+        if (flux_msg_set_flag (*msg, FLUX_MSGFLAG_STREAMING) < 0)
+            goto error;
+    }
+    if ((flags & FLUX_RPC_NORESPONSE)) {
+        if (flux_msg_set_flag (*msg, FLUX_MSGFLAG_NORESPONSE) < 0)
+            goto error;
+    }
+    if (flux_msg_set_nodeid (*msg, nodeid) < 0)
         goto error;
-#if HAVE_CALIPER
-    cali_begin_string_byname ("flux.message.rpc", "single");
-    cali_begin_int_byname ("flux.message.rpc.nodeid", nodeid);
-    cali_begin_int_byname ("flux.message.response_expected",
-                           !(flags & FLUX_RPC_NORESPONSE));
-#endif
-    int rc = flux_send (h, msg, 0);
-#if HAVE_CALIPER
-    cali_end_byname ("flux.message.response_expected");
-    cali_end_byname ("flux.message.rpc.nodeid");
-    cali_end_byname ("flux.message.rpc");
-#endif
+    int rc = flux_send_new (h, msg, 0);
     if (rc < 0)
         goto error;
     rpc->sent = true;
@@ -305,20 +291,19 @@ flux_future_t *flux_rpc_message (flux_t *h,
     flux_msg_t *cpy;
     flux_future_t *f;
 
-    if (!h || !msg || validate_flags (flags, FLUX_RPC_NORESPONSE
-                                           | FLUX_RPC_STREAMING)) {
+    if (!h
+        || !msg
+        || validate_flags (flags,
+                           FLUX_RPC_NORESPONSE | FLUX_RPC_STREAMING) < 0) {
         errno = EINVAL;
         return NULL;
     }
-    if (!(cpy = flux_msg_copy (msg, true)))
+    if (!(cpy = flux_msg_copy (msg, true))
+        || !(f = flux_rpc_message_send_new (h, &cpy, nodeid, flags))) {
+        flux_msg_destroy (cpy);
         return NULL;
-    if (!(f = flux_rpc_message_nocopy (h, cpy, nodeid, flags)))
-        goto error;
-    flux_msg_destroy (cpy);
+    }
     return f;
-error:
-    flux_msg_destroy (cpy);
-    return NULL;
 }
 
 flux_future_t *flux_rpc (flux_t *h,
@@ -327,44 +312,42 @@ flux_future_t *flux_rpc (flux_t *h,
                          uint32_t nodeid,
                          int flags)
 {
-    flux_msg_t *msg = NULL;
-    flux_future_t *f = NULL;
+    flux_msg_t *msg;
+    flux_future_t *f;
 
-    if (!h || validate_flags (flags, FLUX_RPC_NORESPONSE
-                                   | FLUX_RPC_STREAMING)) {
+    if (validate_flags (flags, FLUX_RPC_NORESPONSE | FLUX_RPC_STREAMING) < 0
+        || !h) {
         errno = EINVAL;
         return NULL;
     }
-    if (!(msg = flux_request_encode (topic, s)))
-        goto done;
-    if (!(f = flux_rpc_message_nocopy (h, msg, nodeid, flags)))
-        goto done;
-done:
-    flux_msg_destroy (msg);
+    if (!(msg = flux_request_encode (topic, s))
+        || !(f = flux_rpc_message_send_new (h, &msg, nodeid, flags))) {
+        flux_msg_destroy (msg);
+        return NULL;
+    }
     return f;
 }
 
 flux_future_t *flux_rpc_raw (flux_t *h,
                              const char *topic,
                              const void *data,
-                             int len,
+                             size_t len,
                              uint32_t nodeid,
                              int flags)
 {
     flux_msg_t *msg;
-    flux_future_t *f = NULL;
+    flux_future_t *f;
 
-    if (!h || validate_flags (flags, FLUX_RPC_NORESPONSE
-                                   | FLUX_RPC_STREAMING)) {
+    if (validate_flags (flags, FLUX_RPC_NORESPONSE | FLUX_RPC_STREAMING) < 0
+        || !h) {
         errno = EINVAL;
         return NULL;
     }
-    if (!(msg = flux_request_encode_raw (topic, data, len)))
-        goto done;
-    if (!(f = flux_rpc_message_nocopy (h, msg, nodeid, flags)))
-        goto done;
-done:
-    flux_msg_destroy (msg);
+    if (!(msg = flux_request_encode_raw (topic, data, len))
+        || !(f = flux_rpc_message_send_new (h, &msg, nodeid, flags))) {
+        flux_msg_destroy (msg);
+        return NULL;
+    }
     return f;
 }
 
@@ -372,28 +355,32 @@ flux_future_t *flux_rpc_vpack (flux_t *h,
                                const char *topic,
                                uint32_t nodeid,
                                int flags,
-                               const char *fmt, va_list ap)
+                               const char *fmt,
+                               va_list ap)
 {
     flux_msg_t *msg;
-    flux_future_t *f = NULL;
+    flux_future_t *f;
 
-    if (!h || validate_flags (flags, FLUX_RPC_NORESPONSE
-                                   | FLUX_RPC_STREAMING)) {
+    if (validate_flags (flags, FLUX_RPC_NORESPONSE | FLUX_RPC_STREAMING) < 0
+        || !h) {
         errno = EINVAL;
         return NULL;
     }
-    if (!(msg = flux_request_encode (topic, NULL)))
-        goto done;
-    if (flux_msg_vpack (msg, fmt, ap) < 0)
-        goto done;
-    f = flux_rpc_message_nocopy (h, msg, nodeid, flags);
-done:
-    flux_msg_destroy (msg);
+    if (!(msg = flux_request_encode (topic, NULL))
+        || flux_msg_vpack (msg, fmt, ap) < 0
+        || !(f = flux_rpc_message_send_new (h, &msg, nodeid, flags))) {
+        flux_msg_destroy (msg);
+        return NULL;
+    }
     return f;
 }
 
-flux_future_t *flux_rpc_pack (flux_t *h, const char *topic, uint32_t nodeid,
-                              int flags, const char *fmt, ...)
+flux_future_t *flux_rpc_pack (flux_t *h,
+                              const char *topic,
+                              uint32_t nodeid,
+                              int flags,
+                              const char *fmt,
+                              ...)
 {
     va_list ap;
     flux_future_t *f;
@@ -410,6 +397,12 @@ uint32_t flux_rpc_get_matchtag (flux_future_t *f)
     return rpc ? rpc->matchtag : FLUX_MATCHTAG_NONE;
 }
 
+uint32_t flux_rpc_get_nodeid (flux_future_t *f)
+{
+    struct flux_rpc *rpc = flux_future_aux_get (f, "flux::rpc");
+    return rpc ? rpc->nodeid : FLUX_NODEID_ANY;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libflux/rpc.h b/src/common/libflux/rpc.h
index f89dd5990320..667f56d13972 100644
--- a/src/common/libflux/rpc.h
+++ b/src/common/libflux/rpc.h
@@ -11,9 +11,6 @@
 #ifndef _FLUX_CORE_RPC_H
 #define _FLUX_CORE_RPC_H
 
-#include "handle.h"
-#include "future.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -23,35 +20,52 @@ enum {
     FLUX_RPC_STREAMING = 2,
 };
 
-flux_future_t *flux_rpc (flux_t *h, const char *topic, const char *s,
-                         uint32_t nodeid, int flags);
+flux_future_t *flux_rpc (flux_t *h,
+                         const char *topic,
+                         const char *s,
+                         uint32_t nodeid,
+                         int flags);
 
-flux_future_t *flux_rpc_pack (flux_t *h, const char *topic, uint32_t nodeid,
-                              int flags, const char *fmt, ...);
+flux_future_t *flux_rpc_pack (flux_t *h,
+                              const char *topic,
+                              uint32_t nodeid,
+                              int flags,
+                              const char *fmt,
+                              ...);
 
 flux_future_t *flux_rpc_vpack (flux_t *h,
                                const char *topic,
                                uint32_t nodeid,
                                int flags,
-                               const char *fmt, va_list ap);
+                               const char *fmt,
+                               va_list ap);
 
-flux_future_t *flux_rpc_raw (flux_t *h, const char *topic,
-                             const void *data, int len,
-                             uint32_t nodeid, int flags);
+flux_future_t *flux_rpc_raw (flux_t *h,
+                             const char *topic,
+                             const void *data,
+                             size_t len,
+                             uint32_t nodeid,
+                             int flags);
 
-flux_future_t *flux_rpc_message (flux_t *h, const flux_msg_t *msg,
-                                 uint32_t nodeid, int flags);
+flux_future_t *flux_rpc_message (flux_t *h,
+                                 const flux_msg_t *msg,
+                                 uint32_t nodeid,
+                                 int flags);
 
 int flux_rpc_get (flux_future_t *f, const char **s);
 
 int flux_rpc_get_unpack (flux_future_t *f, const char *fmt, ...);
 
-int flux_rpc_get_raw (flux_future_t *f, const void **data, int *len);
+int flux_rpc_get_raw (flux_future_t *f, const void **data, size_t *len);
 
 /* Accessor for RPC matchtag (see RFC 6).
  */
 uint32_t flux_rpc_get_matchtag (flux_future_t *f);
 
+/* Accessor for original nodeid target of flux_rpc()
+ */
+uint32_t flux_rpc_get_nodeid (flux_future_t *f);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/common/libflux/service.c b/src/common/libflux/service.c
index 86c1e2b5253b..ab2c0dd86339 100644
--- a/src/common/libflux/service.c
+++ b/src/common/libflux/service.c
@@ -19,8 +19,12 @@ flux_future_t *flux_service_register (flux_t *h, const char *name)
         errno = EINVAL;
         return NULL;
     }
-    return flux_rpc_pack (h, "service.add", FLUX_NODEID_ANY, 0,
-                          "{s:s}", "service", name);
+    return flux_rpc_pack (h,
+                          "service.add",
+                          FLUX_NODEID_ANY,
+                          0,
+                          "{s:s}",
+                          "service", name);
 }
 
 flux_future_t *flux_service_unregister (flux_t *h, const char *name)
@@ -29,8 +33,12 @@ flux_future_t *flux_service_unregister (flux_t *h, const char *name)
         errno = EINVAL;
         return NULL;
     }
-    return flux_rpc_pack (h, "service.remove", FLUX_NODEID_ANY, 0,
-                          "{s:s}", "service", name);
+    return flux_rpc_pack (h,
+                          "service.remove",
+                          FLUX_NODEID_ANY,
+                          0,
+                          "{s:s}",
+                          "service", name);
 }
 
 /*
diff --git a/src/common/libflux/service.h b/src/common/libflux/service.h
index fb5e7b455c96..bbcd0dc09886 100644
--- a/src/common/libflux/service.h
+++ b/src/common/libflux/service.h
@@ -11,17 +11,15 @@
 #ifndef _FLUX_CORE_SERVICE_H
 #define _FLUX_CORE_SERVICE_H
 
-#include "handle.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/* 
+/*
  *  Register service `name` with the broker for this handle. On success
- *   request messages sent to "name.*" will be routed to this handle 
- *   until `flux_service_remove()` is called for `name`, or upon
- *   disconnect. 
+ *   request messages sent to "name.*" will be routed to this handle
+ *   until `flux_service_unregister()` is called for `name`, or upon
+ *   disconnect.
  *
  *  On success, the returned future will be fulfilled with no error, o/w
  *   the future is fulfilled with errnum set appropriately:
@@ -29,7 +27,7 @@ extern "C" {
  *   EINVAL - Invalid service name
  *   EEXIST - Service already registered under this name
  *   ENOENT - Unable to lookup route to requesting sender
- * 
+ *
  */
 flux_future_t *flux_service_register (flux_t *h, const char *name);
 
@@ -41,8 +39,8 @@ flux_future_t *flux_service_register (flux_t *h, const char *name);
  *
  *  ENOENT - No such service registered as `name`
  *  EINVAL - Sender does not match current owner of service
- * 
- */  
+ *
+ */
 flux_future_t *flux_service_unregister (flux_t *h, const char *name);
 
 #ifdef __cplusplus
diff --git a/src/common/libflux/stats.c b/src/common/libflux/stats.c
new file mode 100644
index 000000000000..c2270c5db694
--- /dev/null
+++ b/src/common/libflux/stats.c
@@ -0,0 +1,110 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <flux/core.h>
+
+#include "fripp.h"
+
+#define FRIPP_AUX_TAG "flux::fripp"
+
+static struct fripp_ctx *get_fripp_ctx (flux_t *h)
+{
+    struct fripp_ctx *ctx;
+
+    if (!(ctx = flux_aux_get (h, FRIPP_AUX_TAG))) {
+        if (!(ctx = fripp_ctx_create (h)))
+            return NULL;
+        if (flux_aux_set (h, FRIPP_AUX_TAG, ctx,
+                          (flux_free_f)fripp_ctx_destroy) == -1) {
+            fripp_ctx_destroy (ctx);
+            return NULL;
+        }
+    }
+    return fripp_enabled (ctx) ? ctx : NULL;
+}
+
+void flux_stats_count (flux_t *h, const char *name, ssize_t count)
+{
+    struct fripp_ctx *ctx;
+    if (!(ctx = get_fripp_ctx (h)))
+        return;
+
+    fripp_count (ctx, name, count);
+}
+
+void flux_stats_gauge_set (flux_t *h, const char *name, ssize_t value)
+{
+    struct fripp_ctx *ctx;
+    if (!(ctx = get_fripp_ctx (h)))
+        return;
+
+    fripp_gauge (ctx, name, value, false);
+}
+
+void flux_stats_gauge_inc (flux_t *h, const char *name, ssize_t inc)
+{
+    struct fripp_ctx *ctx;
+    if (!(ctx = get_fripp_ctx (h)))
+        return;
+
+    fripp_gauge (ctx, name, inc, true);
+}
+
+void flux_stats_timing (flux_t *h, const char *name, double ms)
+{
+    struct fripp_ctx *ctx;
+    if (!(ctx = get_fripp_ctx (h)))
+        return;
+
+    fripp_timing (ctx, name, ms);
+}
+
+
+void flux_stats_set_period (flux_t *h, double period)
+{
+    struct fripp_ctx *ctx;
+    if (!(ctx = get_fripp_ctx (h)))
+        return;
+
+    fripp_set_agg_period (ctx, period);
+}
+
+void flux_stats_set_prefix (flux_t *h, const char *fmt, ...)
+{
+    struct fripp_ctx *ctx;
+    if (!(ctx = get_fripp_ctx (h)))
+        return;
+
+    va_list ap;
+    va_start (ap, fmt);
+
+    char prefix[128];
+    if (vsnprintf (prefix, sizeof (prefix), fmt, ap) >= 128)
+        goto done;
+
+    fripp_set_prefix (ctx, prefix);
+
+done:
+    va_end (ap);
+}
+
+bool flux_stats_enabled (flux_t *h, const char *metric)
+{
+    return fripp_enabled (get_fripp_ctx (h));
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/stats.h b/src/common/libflux/stats.h
new file mode 100644
index 000000000000..c92c003875cd
--- /dev/null
+++ b/src/common/libflux/stats.h
@@ -0,0 +1,80 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_STATS_H
+#define _FLUX_CORE_STATS_H
+
+/* Metric types:
+ * Counter - An integer value that will, on the backend (brubeck) send the
+ *           count and reset to 0 at each flush. It calculates the change
+ *           from the value sent at the previous flush.
+ *           An example of where to use a counter is the builtin msgcounters
+ *           that are part of each flux_t handle. The counts will continually
+ *           increase and brubeck will handle calculating the count sent in
+ *           the interval.
+ * Gauge   - An integer that takes an arbitrary value and maintains its
+ *           value until it is set again. Gauges can also take incremental
+ *           values in the form of a + or - in front of the value which
+ *           increments the previously stored value.
+ *           An example of where to use a gauge is to track the current
+ *           size of the broker's content-cache. At each point, the cache's
+ *           sizes are independent of each other.
+ * Timing  - A double value which represents the time taken for a given
+ *           task in ms.
+ *           An example of where to use a timer is timing the length of
+ *           asynchronous loads in the broker's content-cache. The cache
+ *           entry can keep track of when the load was started and then
+ *           calculate and send the time taken once the entry is loaded.
+ */
+
+/* Update (or create) and store 'count' for 'name' to be sent on the
+ * next flush.
+ */
+void flux_stats_count (flux_t *h, const char *name, ssize_t count);
+
+/* Update (or create) and store 'value' for 'name' to be sent on the
+ * next flush.
+ */
+void flux_stats_gauge_set (flux_t *h, const char *name, ssize_t value);
+
+/* Update (or create) and increment 'value' for 'name' to be sent on the
+ * next flush. If'name' was not previously stored, then the value is stored
+ * directly (i.e. assumed 0 previous value).
+ */
+void flux_stats_gauge_inc (flux_t *h, const char *name, ssize_t inc);
+
+
+/* Update (or create) and store 'ms' for 'name' to be sent on the
+ * next flush.
+ */
+void flux_stats_timing (flux_t *h, const char *name, double ms);
+
+/* Update the internal aggregation period over which metrics accumulate
+ * before being set. A 'period' of '0' indicates the metrics should be
+ * sent immediately. The default aggregation period is 1s.
+ */
+void flux_stats_set_period (flux_t *h, double period);
+
+/* Set the prefix to be prepended to all metrics sent from the handle.
+ * The prefix has a max limit of 127 characters. The default prefix is
+ * flux.{{rank}}.
+ */
+void flux_stats_set_prefix (flux_t *h, const char *fmt, ...);
+
+/* Check whether stats collection is enabled on the flux handle.
+ * If 'metric' is non-NULL check if it is currently being tracked.
+ */
+bool flux_stats_enabled (flux_t *h, const char *metric);
+
+#endif
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/sync.c b/src/common/libflux/sync.c
new file mode 100644
index 000000000000..e2ea283ab77d
--- /dev/null
+++ b/src/common/libflux/sync.c
@@ -0,0 +1,152 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* sync.c - future synchronized with heartbeat
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+struct flux_sync {
+    flux_t *h;
+    uint32_t seq;
+    int count;
+    double last;
+    double minimum;
+};
+
+static const char *auxkey = "flux::sync";
+
+static void sync_destroy (struct flux_sync *sync)
+{
+    if (sync) {
+        int saved_errno = errno;
+        flux_future_t *f;
+        f = flux_event_unsubscribe_ex (sync->h,
+                                       "heartbeat.pulse",
+                                       FLUX_RPC_NORESPONSE);
+        flux_future_destroy (f);
+        free (sync);
+        errno = saved_errno;
+    }
+}
+
+static struct flux_sync *sync_create (flux_t *h, double minimum)
+{
+    struct flux_sync *sync;
+    flux_future_t *f;
+
+    if (!(sync = calloc (1, sizeof (*sync))))
+        return NULL;
+    sync->h = h;
+    sync->minimum = minimum;
+    if (!(f = flux_event_subscribe_ex (sync->h,
+                                       "heartbeat.pulse",
+                                       FLUX_RPC_NORESPONSE)))
+        goto error;
+    flux_future_destroy (f);
+    return sync;
+error:
+    sync_destroy (sync);
+    return NULL;
+}
+
+/* Special note about the non-reactive case:  events are delivered to all
+ * matching message handlers, not first match like requests.  Therefore, the
+ * future implementation must requeue events even if they were matched in the
+ * temporary reactor, in case another matching handler exists in the main
+ * reactor.  Thus, calling flux_future_get() in a loop on the sync object,
+ * where the main reactor's dispatcher doesn't retire the event, would fulfill
+ * the future using the same message over and over unless we take steps to
+ * prevent that by watching the event sequence number.
+ */
+static void heartbeat_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
+{
+    double now = flux_reactor_now (flux_get_reactor (h));
+    flux_future_t *f = arg;
+    struct flux_sync *sync = flux_future_aux_get (f, auxkey);
+    uint32_t seq;
+
+    if (flux_msg_authorize (msg, FLUX_USERID_UNKNOWN) < 0
+        || flux_event_decode (msg, NULL, NULL) < 0
+        || flux_msg_get_seq (msg, &seq) < 0)
+        return;
+    if (sync->count > 0) {
+        if (seq <= sync->seq)
+            return;
+        if (sync->minimum > 0. && now - sync->last < sync->minimum)
+            return;
+    }
+    sync->seq = seq;
+    sync->count++;
+    sync->last = now;
+    flux_future_fulfill (f, NULL, NULL);
+}
+
+/* Initialize callback is called at most once per reactor context.
+ * A message handler must be installed in the reactor context that is being
+ * initialized.  Store it in the aux hash so it is destroyed with the future.
+ * Use a NULL key since there may be two of them and we don't want one to
+ * clobber the other.
+ */
+static void initialize_cb (flux_future_t *f, void *arg)
+{
+    flux_t *h = flux_future_get_flux (f);
+    flux_msg_handler_t *mh;
+    struct flux_match match = FLUX_MATCH_EVENT;
+
+    match.topic_glob = "heartbeat.pulse";
+    if (!(mh = flux_msg_handler_create (h, match, heartbeat_cb, f)))
+        goto error;
+    if (flux_future_aux_set (f,
+                             NULL,
+                             mh,
+                             (flux_free_f)flux_msg_handler_destroy) < 0) {
+        flux_msg_handler_destroy (mh);
+        goto error;
+    }
+    flux_msg_handler_start (mh);
+    return;
+error:
+    flux_future_fulfill_error (f, errno, NULL);
+}
+
+flux_future_t *flux_sync_create (flux_t *h, double minimum)
+{
+    flux_future_t *f;
+    struct flux_sync *sync;
+
+    if (h == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(f = flux_future_create (initialize_cb, NULL)))
+        return NULL;
+    flux_future_set_flux (f, h);
+    if (!(sync = sync_create (h, minimum)))
+        goto error;
+    if (flux_future_aux_set (f, auxkey, sync, (flux_free_f)sync_destroy) < 0) {
+        sync_destroy (sync);
+        goto error;
+    }
+    return f;
+error:
+    flux_future_destroy (f);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/sync.h b/src/common/libflux/sync.h
new file mode 100644
index 000000000000..c452e2cd75ef
--- /dev/null
+++ b/src/common/libflux/sync.h
@@ -0,0 +1,32 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_SYNC_H
+#define _FLUX_CORE_SYNC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Synchronize future to the system heartbeat.
+ * Set minimum > 0. to establish a minimum time between fulfillments.
+ * Use a continuation timeout to establish a maximum time between fulfillments.
+ */
+flux_future_t *flux_sync_create (flux_t *h, double minimum);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_FLUX_CORE_SYNC_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/tagpool.c b/src/common/libflux/tagpool.c
deleted file mode 100644
index bd2ad78071b7..000000000000
--- a/src/common/libflux/tagpool.c
+++ /dev/null
@@ -1,158 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* tagpool.c - allocator for 32-bit matchtags */
-
-/* Matchtags are used to match requests and responses in RPC's.
- *
- * Requests that receive no response use FLUX_MATCHTAG_NONE (0).
- */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <assert.h>
-#include <string.h>
-#include <errno.h>
-#include <stdbool.h>
-#include <stdlib.h>
-
-#include "tagpool.h"
-#include "message.h"
-
-#include "src/common/libutil/veb.h"
-#include "src/common/libutil/log.h"
-
-#define TAGPOOL_COUNT (1UL<<20)
-#define TAGPOOL_START (1UL<<10)
-
-#define TAGPOOL_MAGIC   0x34447ff2
-struct tagpool {
-    int             magic;
-    Veb             veb;
-    int             avail;
-    tagpool_grow_f  grow_cb;
-    void            *grow_arg;
-    int             grow_depth;
-};
-
-static void pool_set (Veb veb, uint32_t from, uint32_t to, uint8_t value)
-{
-    while (from < to) {
-        if (value)
-            vebput (veb, from);
-        else
-            vebdel (veb, from);
-        from++;
-    }
-}
-
-struct tagpool *tagpool_create (void)
-{
-    struct tagpool *t = calloc (1, sizeof (*t));
-    if (!t)
-        goto nomem;
-    t->magic = TAGPOOL_MAGIC;
-    t->veb = vebnew (TAGPOOL_START, 1);
-    if (!t->veb.D)
-        goto nomem;
-    vebdel (t->veb, FLUX_MATCHTAG_NONE); /* allocate reserved value */
-    t->avail = TAGPOOL_COUNT - 1;
-    return t;
-nomem:
-    tagpool_destroy (t);
-    errno = ENOMEM;
-    return NULL;
-}
-
-void tagpool_destroy (struct tagpool *t)
-{
-    if (t) {
-        assert (t->magic == TAGPOOL_MAGIC);
-        free (t->veb.D);
-        t->magic = ~TAGPOOL_MAGIC;
-        free (t);
-    }
-}
-
-void tagpool_set_grow_cb (struct tagpool *t, tagpool_grow_f cb, void *arg)
-{
-    t->grow_cb = cb;
-    t->grow_arg = arg;
-}
-
-static uint32_t alloc_with_resize (struct tagpool *t)
-{
-    uint32_t max = TAGPOOL_COUNT;
-    uint32_t oldsize, newsize, tag;
-
-    oldsize = t->veb.M;
-    newsize = oldsize << 1;
-    tag = vebsucc (t->veb, 0);
-
-    if (tag == t->veb.M && newsize <= max) {
-        if (t->grow_cb && t->grow_depth == 0) {
-            t->grow_depth++;
-            t->grow_cb (t->grow_arg, oldsize, newsize);
-            t->grow_depth--;
-        }
-        Veb new = vebnew (newsize, 0);
-        if (new.D) {
-            pool_set (new, oldsize, newsize, 1);
-            free (t->veb.D);
-            t->veb = new;
-            tag = vebsucc (t->veb, oldsize);
-            assert (tag == oldsize);
-        }
-    }
-    if (tag < t->veb.M)
-        vebdel (t->veb, tag);
-    return tag;
-}
-
-uint32_t tagpool_alloc (struct tagpool *t)
-{
-    assert (t->magic == TAGPOOL_MAGIC);
-    uint32_t tag;
-
-    tag = alloc_with_resize (t);
-    if (tag < t->veb.M) {
-        t->avail--;
-        return tag;
-    }
-    return FLUX_MATCHTAG_NONE;
-}
-
-void tagpool_free (struct tagpool *t, uint32_t tag)
-{
-    assert (t->magic == TAGPOOL_MAGIC);
-    if (tag != FLUX_MATCHTAG_NONE) {
-        if (tag < t->veb.M) {
-            vebput (t->veb, tag);
-            t->avail++;
-        }
-    }
-}
-
-uint32_t tagpool_getattr (struct tagpool *t, int attr)
-{
-    assert (t->magic == TAGPOOL_MAGIC);
-    switch (attr) {
-        case TAGPOOL_ATTR_SIZE:
-            return TAGPOOL_COUNT - 1;
-        case TAGPOOL_ATTR_AVAIL:
-            return t->avail;
-    }
-    return 0;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/tagpool.h b/src/common/libflux/tagpool.h
deleted file mode 100644
index 1412ba258d35..000000000000
--- a/src/common/libflux/tagpool.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_CORE_TAGPOOL_H
-#define _FLUX_CORE_TAGPOOL_H
-
-#include <stdint.h>
-#include <stdbool.h>
-
-struct tagpool *tagpool_create (void);
-void tagpool_destroy (struct tagpool *t);
-uint32_t tagpool_alloc (struct tagpool *t);
-void tagpool_free (struct tagpool *t, uint32_t matchtag);
-
-typedef void (*tagpool_grow_f)(void *arg, uint32_t oldsize, uint32_t newsize);
-void tagpool_set_grow_cb (struct tagpool *t, tagpool_grow_f cb, void *arg);
-
-enum {
-    TAGPOOL_ATTR_SIZE,
-    TAGPOOL_ATTR_AVAIL,
-};
-uint32_t tagpool_getattr (struct tagpool *t, int attr);
-
-
-#endif /* _FLUX_CORE_TAGPOOL_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/test/attr.c b/src/common/libflux/test/attr.c
index e75561d0ab02..d8622354f352 100644
--- a/src/common/libflux/test/attr.c
+++ b/src/common/libflux/test/attr.c
@@ -8,13 +8,17 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
 #include <string.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libtestutil/util.h"
+#include "ccan/str/str.h"
 
 struct entry {
     char *key;
@@ -22,12 +26,18 @@ struct entry {
     int flags;
 };
 
+// FLUX_ATTRFLAG_IMMUTABLE = 1
 static struct entry hardwired[] = {
     { .key = "cow",     .val = "moo",   .flags = 1 },
     { .key = "duck",    .val = "quack", .flags = 1 },
     { .key = "chick",   .val = "peep",  .flags = 1  },
     { .key = "fox",     .val = "-",     .flags = 1  },
     { .key = "bear",    .val = "roar",  .flags = 1  },
+    { .key = "hostlist", .val = "foo[0-2]", .flags = 1 },
+    { .key = "broker.starttime", .val = "3.14", .flags = 1 },
+    { .key = "parent-uri",
+      .val = "local:///tmp/foo/bar",
+      .flags = 1 },
     { .key = NULL,      .val = NULL,    .flags = 1  },
 };
 
@@ -35,7 +45,7 @@ static bool lookup_hardwired (const char *key, const char **val, int *flags)
 {
     int i;
     for (i = 0; hardwired[i].key != NULL; i++) {
-        if (!strcmp (hardwired[i].key, key)) {
+        if (streq (hardwired[i].key, key)) {
             if (val)
                 *val = hardwired[i].val;
             if (flags)
@@ -46,11 +56,6 @@ static bool lookup_hardwired (const char *key, const char **val, int *flags)
     return false;
 }
 
-/* Two hardwired value:
- *   cow=moo (flags = 1)
- *   chicken=cluck (flags = 1)
- * all other values come from the hash and are returned with (flags = 0)
- */
 static volatile int get_count = 0;
 void get_cb (flux_t *h, flux_msg_handler_t *mh,
              const flux_msg_t *msg, void *arg)
@@ -164,6 +169,17 @@ int test_server (flux_t *h, void *arg)
     return 0;
 }
 
+int cache_size (flux_t *h)
+{
+    int count = 0;
+    const char *name = flux_attr_cache_first (h);
+    while (name) {
+        count++;
+        name = flux_attr_cache_next (h);
+    }
+    return count;
+}
+
 int main (int argc, char *argv[])
 {
     flux_t *h;
@@ -172,9 +188,7 @@ int main (int argc, char *argv[])
 
     plan (NO_PLAN);
 
-    test_server_environment_init ("attr-test");
-
-    if (!(h = test_server_create (test_server, NULL)))
+    if (!(h = test_server_create (0, test_server, NULL)))
         BAIL_OUT ("test_server_create failed");
 
     /* get ENOENT */
@@ -185,74 +199,92 @@ int main (int argc, char *argv[])
         && get_count == 1,
         "flux_attr_get name=notakey fails with ENOENT (with rpc)");
 
+    /* cache iterator */
+    ok (flux_attr_cache_first (NULL) == NULL,
+        "flux_attr_cache_first h=NULL returns NULL");
+    ok (flux_attr_cache_next (NULL) == NULL,
+        "flux_attr_cache_next h=NULL returns NULL");
+    ok (cache_size (h) == 0,
+        "cache is empty");
+
     /* set, get */
 
     ok (flux_attr_set (h, "foo", "bar") == 0,
         "flux_attr_set foo=bar works");
     ok (flux_attr_set (h, "baz", "meep") == 0,
         "flux_attr_set baz=meep works");
+    ok (cache_size (h) == 0,
+        "cache is empty");
 
     get_count = 0;
     value = flux_attr_get (h, "foo");
-    ok (value && !strcmp (value, "bar") && get_count == 1,
+    ok (value && streq (value, "bar") && get_count == 1,
         "flux_attr_get foo=bar (with rpc)");
     value = flux_attr_get (h, "foo");
-    ok (value && !strcmp (value, "bar") && get_count == 2,
+    ok (value && streq (value, "bar") && get_count == 2,
         "flux_attr_get foo=bar (with 2nd rpc)");
+    ok (cache_size (h) == 0,
+        "cache is empty");
 
     get_count = 0;
     value2 = flux_attr_get (h, "baz");
-    ok (value2 && !strcmp (value2, "meep") && get_count == 1,
+    ok (value2 && streq (value2, "meep") && get_count == 1,
         "flux_attr_get baz=meep (with rpc)");
     value2 = flux_attr_get (h, "baz");
-    ok (value2 && !strcmp (value2, "meep") && get_count == 2,
+    ok (value2 && streq (value2, "meep") && get_count == 2,
         "flux_attr_get baz=meep (with 2nd rpc)");
 
-    ok (value && !strcmp (value, "bar"),
+    ok (value && streq (value, "bar"),
         "const return value of flux_attr_get foo=bar still valid");
+    ok (cache_size (h) == 0,
+        "cache is empty");
 
     /* get (cached) */
 
     get_count = 0;
     value = flux_attr_get (h, "cow");
-    ok (value && !strcmp (value, "moo") && get_count == 1,
+    ok (value && streq (value, "moo") && get_count == 1,
         "flux_attr_get cow=moo (with rpc)");
+    ok (cache_size (h) == 1,
+        "cache contains 1 item");
+    const char *key = flux_attr_cache_first (h);
+    ok (key != NULL && streq (key, "cow"),
+        "flux_attr_cache_first returned key name");
+
     get_count = 0;
     value = flux_attr_get (h, "chick");
-    ok (value && !strcmp (value, "peep") && get_count == 1,
+    ok (value && streq (value, "peep") && get_count == 1,
         "flux_attr_get chick=peep (with rpc)");
+    ok (cache_size (h) == 2,
+        "cache contains 2 items");
     get_count = 0;
     value = flux_attr_get (h, "cow");
-    ok (value && !strcmp (value, "moo") && get_count == 0,
+    ok (value && streq (value, "moo") && get_count == 0,
         "flux_attr_get cow=moo (cached)");
+    ok (cache_size (h) == 2,
+        "cache contains 2 items");
     get_count = 0;
     value = flux_attr_get (h, "chick");
-    ok (value && !strcmp (value, "peep") && get_count == 0,
+    ok (value && streq (value, "peep") && get_count == 0,
         "flux_attr_get chick=peep (cached)");
-
-    /* rm (set val=NULL) */
-
-    errno = 0;
-    ok (flux_attr_set (h, "notakey", NULL) < 0 && errno == ENOENT,
-        "flux_attr_set notakey=NULL fails with ENOENT");
-
-    ok (flux_attr_set (h, "foo", NULL) == 0,
-        "flux_attr_set foo=NULL works");
-    errno = 0;
-    ok (flux_attr_get (h, "foo") == NULL && errno == ENOENT,
-        "flux_attr_get foo fails with ENOENT");
+    ok (cache_size (h) == 2,
+        "cache contains 2 items");
 
     /* cacheonly */
 
     ok (flux_attr_set_cacheonly (h, "fake", "42") == 0,
         "flux_attr_set_cacheonly fake=42");
+    ok (cache_size (h) == 3,
+        "cache contains 3 items");
     get_count = 0;
     value = flux_attr_get (h, "fake");
-    ok (value && !strcmp (value, "42") && get_count == 0,
+    ok (value && streq (value, "42") && get_count == 0,
         "flux_attr_get fake=42 (no rpc)");
 
     ok (flux_attr_set_cacheonly (h, "fake", NULL) == 0,
         "flux_attr_set_cacheonly fake=NULL");
+    ok (cache_size (h) == 2,
+        "local cache contains 2 items");
     get_count = 0;
     errno = 0;
     ok (flux_attr_get (h, "fake") == NULL && errno == ENOENT && get_count == 1,
@@ -282,6 +314,92 @@ int main (int argc, char *argv[])
     ok (flux_attr_set_cacheonly (h, NULL, "bar") < 0 && errno == EINVAL,
         "flux_attr_set_cacheonly name=NULL fails with EINVAL");
 
+    /* test flux_get_hostbyrank () */
+    ok ((value = flux_get_hostbyrank (NULL, 42)) != NULL
+        && streq (value, "(null)"),
+        "flux_get_hostbyrank h=NULL returns (null)");
+    ok ((value = flux_get_hostbyrank (h, FLUX_NODEID_ANY)) != NULL
+        && streq (value, "any"),
+        "flux_get_hostbyrank FLUX_NODEID_ANY returns any");
+    ok ((value = flux_get_hostbyrank (h, FLUX_NODEID_UPSTREAM)) != NULL
+        && streq (value, "upstream"),
+        "flux_get_hostbyrank FLUX_NODEID_UPSTREAMreturns upstream");
+    ok ((value = flux_get_hostbyrank (h, 2)) != NULL
+        && streq (value, "foo2"),
+        "flux_get_hostbyrank 2 returns foo2");
+    ok ((value = flux_get_hostbyrank (h, 3)) != NULL
+        && streq (value, "(null)"),
+        "flux_get_hostbyrank 3 returns (null)");
+
+    /* test flux_get_rankbyhost () */
+    errno = 0;
+    ok (flux_get_rankbyhost (NULL, "foo2") < 0 && errno == EINVAL,
+        "flux_get_rankbyhost h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_get_rankbyhost (h, NULL) < 0 && errno == EINVAL,
+        "flux_get_rankbyhost host=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_get_rankbyhost (h, "foo3") < 0 && errno == ENOENT,
+        "flux_get_rankbyhost host=foo3 fails with ENOENT");
+    ok (flux_get_rankbyhost (h, "foo2") == 2,
+        "flux_get_rankbyhost host=foo2 returns 2");
+
+    /* test flux_hostmap_lookup () */
+    flux_error_t error;
+    errno = 0;
+    ok (flux_hostmap_lookup (NULL, "foo", NULL) == NULL && errno == EINVAL,
+        "flux_hostmap_lookup h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_hostmap_lookup (h, NULL, &error) == NULL && errno == EINVAL,
+        "flux_hostmap_lookup targets=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_hostmap_lookup (h, "foo3", &error) == NULL && errno == ENOENT,
+        "flux_hostmap_lookup targets=foo3 fails with ENOENT");
+    is (error.text, "host foo3 not found in host map",
+        "error.text is as expected");
+
+    ok (flux_hostmap_lookup (h, "3", &error) == NULL && errno == ENOENT,
+        "flux_hostmap_lookup targets=3 fails with ENOENT");
+    is (error.text, "rank 3 is not in host map",
+        "error.text is as expected");
+
+    ok (flux_hostmap_lookup (h, "foo[", &error) == NULL && errno == EINVAL,
+        "flux_hostmap_lookup targets=foo[ fails with EINVAL");
+    is (error.text, "target must be a valid idset or hostlist",
+        "error.text is as expected");
+
+    char *s = flux_hostmap_lookup (h, "foo2", &error);
+    ok (s != NULL,
+        "flux_hostmap_lookup targets=foo2 returns %s", s);
+    is (s, "2",
+        "value is expected");
+    free (s);
+
+    s = flux_hostmap_lookup (h, "1-2", &error);
+    ok (s != NULL,
+        "flux_hostmap_lookup targets=1-2 returns %s", s);
+    is (s, "foo[1-2]",
+        "value is expected");
+    free (s);
+
+    /* test flux_get_instance_starttime */
+    double d;
+    ok (flux_get_instance_starttime (h, &d) == 0 && d == 3.14,
+        "flux_get_instance_starttime works");
+    errno = 0;
+    ok (flux_get_instance_starttime (NULL, &d) < 0 && errno == EINVAL,
+        "flux_get_instance_starttime h=NULL fails with EINVAL");
+
+    /* test special handling of parent-uri */
+    unsetenv ("FLUX_PROXY_REMOTE");
+    value = flux_attr_get (h, "parent-uri");
+    is (value, "local:///tmp/foo/bar",
+        "flux_attr_get 'parent-uri' is local uri without FLUX_PROXY_REMOTE");
+    setenv ("FLUX_PROXY_REMOTE", "host123", 1);
+    value = flux_attr_get (h, "parent-uri");
+    is (value, "ssh://host123/tmp/foo/bar",
+        "flux_attr_get 'parent-uri' with FLUX_PROXY_REMOTE returns remote");
+
     test_server_stop (h);
     flux_close (h);
 
diff --git a/src/common/libflux/test/buffer.c b/src/common/libflux/test/buffer.c
deleted file mode 100644
index 904fb07b8faa..000000000000
--- a/src/common/libflux/test/buffer.c
+++ /dev/null
@@ -1,1233 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <errno.h>
-
-#include "src/common/libflux/buffer.h"
-#include "src/common/libflux/buffer_private.h"
-#include "src/common/libtap/tap.h"
-
-#define FLUX_BUFFER_TEST_MAXSIZE 1048576
-
-void empty_cb (flux_buffer_t *fb, void *arg)
-{
-    /* do nothing */
-}
-
-void basic (void)
-{
-    flux_buffer_t *fb;
-    int pipefds[2];
-    char buf[1024];
-    const char *ptr;
-    int len;
-
-    ok (pipe (pipefds) == 0,
-        "pipe succeeded");
-
-    ok ((fb = flux_buffer_create (FLUX_BUFFER_TEST_MAXSIZE)) != NULL,
-        "flux_buffer_create works");
-
-    ok (flux_buffer_size (fb) == FLUX_BUFFER_TEST_MAXSIZE,
-        "flux_buffer_size returns correct size");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes initially returns 0");
-
-    ok (flux_buffer_space (fb) == FLUX_BUFFER_TEST_MAXSIZE,
-        "flux_buffer_space initially returns FLUX_BUFFER_TEST_MAXSIZE");
-
-    /* write & peek tests */
-
-    ok (flux_buffer_write (fb, "foo", 3) == 3,
-        "flux_buffer_write works");
-
-    ok (flux_buffer_bytes (fb) == 3,
-        "flux_buffer_bytes returns length of bytes written");
-
-    ok (flux_buffer_space (fb) == (FLUX_BUFFER_TEST_MAXSIZE - 3),
-        "flux_buffer_space returns length of space left");
-
-    ok ((ptr = flux_buffer_peek (fb, 2, &len)) != NULL
-        && len == 2,
-        "flux_buffer_peek with specific length works");
-
-    ok (!memcmp (ptr, "fo", 2),
-        "flux_buffer_peek returns expected data");
-
-    ok ((ptr = flux_buffer_peek (fb, -1, &len)) != NULL
-        && len == 3,
-        "flux_buffer_peek with length -1 works");
-
-    ok (!memcmp (ptr, "foo", 3),
-        "flux_buffer_peek returns expected data");
-
-    ok (flux_buffer_bytes (fb) == 3,
-        "flux_buffer_bytes returns unchanged length after peek");
-
-    ok (flux_buffer_drop (fb, 2) == 2,
-        "flux_buffer_drop works");
-
-    ok (flux_buffer_bytes (fb) == 1,
-        "flux_buffer_bytes returns length of remaining bytes written");
-
-    ok (flux_buffer_space (fb) == (FLUX_BUFFER_TEST_MAXSIZE - 1),
-        "flux_buffer_space returns length of space left");
-
-    ok (flux_buffer_drop (fb, -1) == 1,
-        "flux_buffer_drop drops remaining bytes");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 with all bytes dropped");
-
-    ok (flux_buffer_space (fb) == FLUX_BUFFER_TEST_MAXSIZE,
-        "flux_buffer_space initially returns FLUX_BUFFER_TEST_MAXSIZE");
-
-    /* write and read tests */
-
-    ok (flux_buffer_write (fb, "foo", 3) == 3,
-        "flux_buffer_write works");
-
-    ok (flux_buffer_bytes (fb) == 3,
-        "flux_buffer_bytes returns length of bytes written");
-
-    ok (flux_buffer_space (fb) == (FLUX_BUFFER_TEST_MAXSIZE - 3),
-        "flux_buffer_space returns length of space left");
-
-    ok ((ptr = flux_buffer_read (fb, 2, &len)) != NULL
-        && len == 2,
-        "flux_buffer_read with specific length works");
-
-    ok (!memcmp (ptr, "fo", 2),
-        "flux_buffer_read returns expected data");
-
-    ok (flux_buffer_bytes (fb) == 1,
-        "flux_buffer_bytes returns new length after read");
-
-    ok ((ptr = flux_buffer_read (fb, -1, &len)) != NULL
-        && len == 1,
-        "flux_buffer_peek with length -1 works");
-
-    ok (!memcmp (ptr, "o", 1),
-        "flux_buffer_peek returns expected data");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 with all bytes read");
-
-    ok (flux_buffer_space (fb) == FLUX_BUFFER_TEST_MAXSIZE,
-        "flux_buffer_space initially returns FLUX_BUFFER_TEST_MAXSIZE");
-
-    /* write_line & peek_line tests */
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 on no line");
-
-    ok (flux_buffer_write_line (fb, "foo") == 4,
-        "flux_buffer_write_line works");
-
-    ok (flux_buffer_bytes (fb) == 4,
-        "flux_buffer_bytes returns length of bytes written");
-
-    ok (flux_buffer_space (fb) == (FLUX_BUFFER_TEST_MAXSIZE - 4),
-        "flux_buffer_space returns length of space left");
-
-    ok (flux_buffer_lines (fb) == 1,
-        "flux_buffer_lines returns 1 on line written");
-    ok (flux_buffer_has_line (fb) == 1,
-        "flux_buffer_has_line returns true on line written");
-
-    ok ((ptr = flux_buffer_peek_line (fb, &len)) != NULL
-        && len == 4,
-        "flux_buffer_peek_line works");
-
-    ok (!memcmp (ptr, "foo\n", 4),
-        "flux_buffer_peek_line returns expected data");
-
-    ok (flux_buffer_bytes (fb) == 4,
-        "flux_buffer_bytes returns unchanged length after peek_line");
-
-    ok (flux_buffer_drop_line (fb) == 4,
-        "flux_buffer_drop_line works");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 after drop_line");
-
-    ok (flux_buffer_space (fb) == FLUX_BUFFER_TEST_MAXSIZE,
-        "flux_buffer_space initially returns FLUX_BUFFER_TEST_MAXSIZE");
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 after drop_line");
-
-    /* write_line & peek_trimmed_line tests */
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 on no line");
-
-    ok (flux_buffer_write_line (fb, "foo") == 4,
-        "flux_buffer_write_line works");
-
-    ok (flux_buffer_bytes (fb) == 4,
-        "flux_buffer_bytes returns length of bytes written");
-
-    ok (flux_buffer_space (fb) == (FLUX_BUFFER_TEST_MAXSIZE - 4),
-        "flux_buffer_space returns length of space left");
-
-    ok (flux_buffer_lines (fb) == 1,
-        "flux_buffer_lines returns 1 on line written");
-
-    ok ((ptr = flux_buffer_peek_trimmed_line (fb, &len)) != NULL
-        && len == 3,
-        "flux_buffer_peek_trimmed_line works");
-
-    ok (!memcmp (ptr, "foo", 3),
-        "flux_buffer_peek_trimmed_line returns expected data");
-
-    ok (flux_buffer_bytes (fb) == 4,
-        "flux_buffer_bytes returns unchanged length after peek_trimmed_line");
-
-    ok (flux_buffer_drop_line (fb) == 4,
-        "flux_buffer_drop_line works");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 after drop_line");
-
-    ok (flux_buffer_space (fb) == FLUX_BUFFER_TEST_MAXSIZE,
-        "flux_buffer_space initially returns FLUX_BUFFER_TEST_MAXSIZE");
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 after drop_line");
-
-    /* write_line & read_line tests */
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 on no line");
-
-    ok (flux_buffer_write_line (fb, "foo") == 4,
-        "flux_buffer_write_line works");
-
-    ok (flux_buffer_bytes (fb) == 4,
-        "flux_buffer_bytes returns length of bytes written");
-
-    ok (flux_buffer_space (fb) == (FLUX_BUFFER_TEST_MAXSIZE - 4),
-        "flux_buffer_space returns length of space left");
-
-    ok (flux_buffer_lines (fb) == 1,
-        "flux_buffer_lines returns 1 on line written");
-
-    ok ((ptr = flux_buffer_read_line (fb, &len)) != NULL
-        && len == 4,
-        "flux_buffer_read_line works");
-
-    ok (!memcmp (ptr, "foo\n", 4),
-        "flux_buffer_read_line returns expected data");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 after read_line");
-
-    ok (flux_buffer_space (fb) == FLUX_BUFFER_TEST_MAXSIZE,
-        "flux_buffer_space initially returns FLUX_BUFFER_TEST_MAXSIZE");
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 after read_line");
-
-    /* write_line & read_trimmed_line tests */
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 on no line");
-
-    ok (flux_buffer_write_line (fb, "foo") == 4,
-        "flux_buffer_write_line works");
-
-    ok (flux_buffer_bytes (fb) == 4,
-        "flux_buffer_bytes returns length of bytes written");
-
-    ok (flux_buffer_space (fb) == (FLUX_BUFFER_TEST_MAXSIZE - 4),
-        "flux_buffer_space returns length of space left");
-
-    ok (flux_buffer_lines (fb) == 1,
-        "flux_buffer_lines returns 1 on line written");
-
-    ok ((ptr = flux_buffer_read_trimmed_line (fb, &len)) != NULL
-        && len == 3,
-        "flux_buffer_read_trimmed_line works");
-
-    ok (!memcmp (ptr, "foo", 3),
-        "flux_buffer_read_trimmed_line returns expected data");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 after read_trimmed_line");
-
-    ok (flux_buffer_space (fb) == FLUX_BUFFER_TEST_MAXSIZE,
-        "flux_buffer_space initially returns FLUX_BUFFER_TEST_MAXSIZE");
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 after read_trimmed_line");
-
-    /* peek_to_fd tests */
-
-    ok (flux_buffer_write (fb, "foo", 3) == 3,
-        "flux_buffer_write works");
-
-    ok (flux_buffer_peek_to_fd (fb, pipefds[1], 2) == 2,
-        "flux_buffer_peek_to_fd specific length works");
-
-    memset (buf, '\0', 1024);
-    ok (read (pipefds[0], buf, 1024) == 2,
-        "read correct number of bytes");
-
-    ok (memcmp (buf, "fo", 2) == 0,
-        "read returned correct data");
-
-    ok (flux_buffer_bytes (fb) == 3,
-        "flux_buffer_bytes returns correct length after peek");
-
-    ok (flux_buffer_peek_to_fd (fb, pipefds[1], -1) == 3,
-        "flux_buffer_peek_to_fd length -1 works");
-
-    memset (buf, '\0', 1024);
-    ok (read (pipefds[0], buf, 1024) == 3,
-        "read correct number of bytes");
-
-    ok (memcmp (buf, "foo", 3) == 0,
-        "read returned correct data");
-
-    ok (flux_buffer_bytes (fb) == 3,
-        "flux_buffer_bytes returns correct length after peek");
-
-    ok (flux_buffer_drop (fb, -1) == 3,
-        "flux_buffer_drop drops remaining bytes");
-
-    /* read_to_fd tests */
-
-    ok (flux_buffer_write (fb, "foo", 3) == 3,
-        "flux_buffer_write works");
-
-    ok (flux_buffer_read_to_fd (fb, pipefds[1], 2) == 2,
-        "flux_buffer_read_to_fd specific length works");
-
-    memset (buf, '\0', 1024);
-    ok (read (pipefds[0], buf, 1024) == 2,
-        "read correct number of bytes");
-
-    ok (memcmp (buf, "fo", 2) == 0,
-        "read returned correct data");
-
-    ok (flux_buffer_bytes (fb) == 1,
-        "flux_buffer_bytes returns correct length after read");
-
-    ok (flux_buffer_read_to_fd (fb, pipefds[1], -1) == 1,
-        "flux_buffer_read_to_fd length -1 works");
-
-    memset (buf, '\0', 1024);
-    ok (read (pipefds[0], buf, 1024) == 1,
-        "read correct number of bytes");
-
-    ok (memcmp (buf, "o", 1) == 0,
-        "read returned correct data");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns correct length after read");
-
-    /* write_from_fd and read tests */
-
-    ok (write (pipefds[1], "foo", 3) == 3,
-        "write to pipe works");
-
-    ok (flux_buffer_write_from_fd (fb, pipefds[0], -1) == 3,
-        "flux_buffer_write_from_fd works");
-
-    ok ((ptr = flux_buffer_read (fb, 2, &len)) != NULL
-        && len == 2,
-        "flux_buffer_read with specific length works");
-
-    ok (!memcmp (ptr, "fo", 2),
-        "flux_buffer_read returns expected data");
-
-    ok (flux_buffer_bytes (fb) == 1,
-        "flux_buffer_bytes returns new length after read");
-
-    ok ((ptr = flux_buffer_read (fb, -1, &len)) != NULL
-        && len == 1,
-        "flux_buffer_peek with length -1 works");
-
-    ok (!memcmp (ptr, "o", 1),
-        "flux_buffer_peek returns expected data");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 with all bytes read");
-
-    flux_buffer_destroy (fb);
-    close (pipefds[0]);
-    close (pipefds[1]);
-}
-
-void read_cb (flux_buffer_t *fb, void *arg)
-{
-    int *count = arg;
-    const char *ptr;
-    int len;
-
-    (*count)++;
-
-    ok ((ptr = flux_buffer_read (fb, -1, &len)) != NULL
-        && len == 6,
-        "flux_buffer_read in callback works");
-
-    ok (!memcmp (ptr, "foobar", 6),
-        "read in callback returns expected data");
-}
-
-void read_line_cb (flux_buffer_t *fb, void *arg)
-{
-    int *count = arg;
-    const char *ptr;
-    int len;
-
-    (*count)++;
-
-    ok ((ptr = flux_buffer_read_line (fb, &len)) != NULL
-        && len == 7,
-        "flux_buffer_read_line in callback works");
-
-    ok (!memcmp (ptr, "foobar\n", 7),
-        "read_line in callback returns expected data");
-}
-
-void write_cb (flux_buffer_t *fb, void *arg)
-{
-    int *count = arg;
-
-    (*count)++;
-
-    ok (flux_buffer_write (fb, "a", 1) == 1,
-        "flux_buffer_write in callback works");
-}
-
-void basic_callback (void)
-{
-    flux_buffer_t *fb;
-    const char *ptr;
-    int len;
-    int pipefds[2];
-    int count;
-    char buf[1024];
-
-    ok (pipe (pipefds) == 0,
-        "pipe succeeded");
-
-    ok ((fb = flux_buffer_create (FLUX_BUFFER_TEST_MAXSIZE)) != NULL,
-        "flux_buffer_create works");
-
-    /* low read callback w/ write */
-
-    count = 0;
-    ok (flux_buffer_set_low_read_cb (fb, read_cb, 3, &count) == 0,
-        "flux_buffer_set_low_read_cb success");
-
-    ok (flux_buffer_write (fb, "foobar", 6) == 6,
-        "flux_buffer_write success");
-
-    ok (count == 1,
-        "read_cb called");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 because callback read all data");
-
-    ok (flux_buffer_write (fb, "foo", 3) == 3,
-        "flux_buffer_write success");
-
-    ok (count == 1,
-        "read_cb not called again, because not above low mark");
-
-    count = 0;
-    ok (flux_buffer_set_low_read_cb (fb, NULL, 0, &count) == 0,
-        "flux_buffer_set_low_read_cb clear callback success");
-
-    ok (flux_buffer_write (fb, "foo", 3) == 3,
-        "flux_buffer_write success");
-
-    ok (count == 0,
-        "read_cb cleared successfully");
-
-    ok (flux_buffer_drop (fb, -1) == 6,
-        "flux_buffer_drop cleared all data");
-
-    /* read line callback w/ write_line */
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 on no line");
-
-    count = 0;
-    ok (flux_buffer_set_read_line_cb (fb, read_line_cb, &count) == 0,
-        "flux_buffer_set_read_line_cb success");
-
-    ok (flux_buffer_write (fb, "foo", 3) == 3,
-        "flux_buffer_write success");
-
-    ok (count == 0,
-        "read_line_cb not called, no line written yet");
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 on no line");
-
-    ok (flux_buffer_write (fb, "bar\n", 4) == 4,
-        "flux_buffer_write success");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 because callback read all data");
-
-    ok (count == 1,
-        "read_line_cb called");
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 on no line, callback read all data");
-
-    count = 0;
-    ok (flux_buffer_set_read_line_cb (fb, NULL, &count) == 0,
-        "flux_buffer_set_read_line_cb clear callback success");
-
-    ok (flux_buffer_write_line (fb, "foo") == 4,
-        "flux_buffer_write_line success");
-
-    ok (count == 0,
-        "read_line_cb cleared successfully");
-
-    ok (flux_buffer_lines (fb) == 1,
-        "flux_buffer_lines returns 1, callback did not read line");
-
-    ok (flux_buffer_drop (fb, -1) == 4,
-        "flux_buffer_drop cleared all data");
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 after drop line");
-
-    /* low read callback w/ write_from_fd */
-
-    count = 0;
-    ok (flux_buffer_set_low_read_cb (fb, read_cb, 3, &count) == 0,
-        "flux_buffer_set_low_read_cb success");
-
-    ok (write (pipefds[1], "foobar", 6) == 6,
-        "write to pipe works");
-
-    ok (flux_buffer_write_from_fd (fb, pipefds[0], 6) == 6,
-        "flux_buffer_write_from_fd success");
-
-    ok (count == 1,
-        "read_cb called");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 because callback read all data");
-
-    ok (write (pipefds[1], "foo", 3) == 3,
-        "write to pipe works");
-
-    ok (flux_buffer_write_from_fd (fb, pipefds[0], 3) == 3,
-        "flux_buffer_write_from_fd success");
-
-    ok (count == 1,
-        "read_cb not called again, because not above low mark");
-
-    count = 0;
-    ok (flux_buffer_set_low_read_cb (fb, NULL, 0, &count) == 0,
-        "flux_buffer_set_low_read_cb clear callback success");
-
-    ok (write (pipefds[1], "foo", 3) == 3,
-        "write to pipe works");
-
-    ok (flux_buffer_write_from_fd (fb, pipefds[0], 3) == 3,
-        "flux_buffer_write_from_fd success");
-
-    ok (count == 0,
-        "read_cb cleared successfully");
-
-    ok (flux_buffer_drop (fb, -1) == 6,
-        "flux_buffer_drop cleared all data");
-
-    /* high write callback w/ read */
-
-    ok (flux_buffer_write (fb, "foobar", 6) == 6,
-        "flux_buffer_write success");
-
-    count = 0;
-    ok (flux_buffer_set_high_write_cb (fb, write_cb, 3, &count) == 0,
-        "flux_buffer_set_high_write_cb success");
-
-    ok ((ptr = flux_buffer_read (fb, 3, &len)) != NULL
-        && len == 3,
-        "flux_buffer_read success");
-
-    ok (!memcmp (ptr, "foo", 3),
-        "flux_buffer_read returns expected data");
-
-    ok (count == 0,
-        "write_cb not called, not less than high");
-
-    ok ((ptr = flux_buffer_read (fb, 3, &len)) != NULL
-        && len == 3,
-        "flux_buffer_read success");
-
-    ok (!memcmp (ptr, "bar", 3),
-        "flux_buffer_read returns expected data");
-
-    ok (count == 1,
-        "write_cb called");
-
-    ok (flux_buffer_bytes (fb) == 1,
-        "flux_buffer_bytes returns 1 because callback wrote a byte");
-
-    count = 0;
-    ok (flux_buffer_set_high_write_cb (fb, NULL, 0, &count) == 0,
-        "flux_buffer_set_high_write_cb clear callback success");
-
-    ok ((ptr = flux_buffer_read (fb, -1, &len)) != NULL
-        && len == 1,
-        "flux_buffer_read success");
-
-    ok (!memcmp (ptr, "a", 1),
-        "flux_buffer_read returns expected data");
-
-    ok (count == 0,
-        "write_cb cleared successfully");
-
-    /* high write callback w/ drop */
-
-    ok (flux_buffer_write (fb, "foobar", 6) == 6,
-        "flux_buffer_write success");
-
-    count = 0;
-    ok (flux_buffer_set_high_write_cb (fb, write_cb, 3, &count) == 0,
-        "flux_buffer_set_high_write_cb success");
-
-    ok (flux_buffer_drop (fb, 3) == 3,
-        "flux_buffer_drop success");
-
-    ok (count == 0,
-        "write_cb not called, not less than high");
-
-    ok (flux_buffer_drop (fb, 1) == 1,
-        "flux_buffer_drop success");
-
-    ok (count == 1,
-        "write_cb called");
-
-    ok (flux_buffer_bytes (fb) == 3,
-        "flux_buffer_bytes return correct bytes after drop and write cb");
-
-    count = 0;
-    ok (flux_buffer_set_high_write_cb (fb, NULL, 0, &count) == 0,
-        "flux_buffer_set_high_write_cb clear callback success");
-
-    ok (flux_buffer_drop (fb, 1) == 1,
-        "flux_buffer_drop success");
-
-    ok (count == 0,
-        "write_cb cleared successfully");
-
-    ok (flux_buffer_drop (fb, -1) == 2,
-        "flux_buffer_drop success");
-
-    /* high write callback w/ read_to_fd */
-
-    ok (flux_buffer_write (fb, "foobar", 6) == 6,
-        "flux_buffer_write success");
-
-    count = 0;
-    ok (flux_buffer_set_high_write_cb (fb, write_cb, 3, &count) == 0,
-        "flux_buffer_set_high_write_cb success");
-
-    ok (flux_buffer_read_to_fd (fb, pipefds[1], 3) == 3,
-        "flux_buffer_read_to_fd success");
-
-    ok (count == 0,
-        "write_cb not called, not less than high");
-
-    ok (flux_buffer_read_to_fd (fb, pipefds[1], 1) == 1,
-        "flux_buffer_read_to_fd success");
-
-    ok (count == 1,
-        "write_cb called");
-
-    ok (flux_buffer_bytes (fb) == 3,
-        "flux_buffer_bytes return correct bytes after read_to_fd and write cb");
-
-    count = 0;
-    ok (flux_buffer_set_high_write_cb (fb, NULL, 0, &count) == 0,
-        "flux_buffer_set_high_write_cb clear callback success");
-
-    ok (flux_buffer_read_to_fd (fb, pipefds[1], 1) == 1,
-        "flux_buffer_read_to_fd success");
-
-    ok (count == 0,
-        "write_cb cleared successfully");
-
-    ok (flux_buffer_drop (fb, -1) == 2,
-        "flux_buffer_drop success");
-
-    /* drain pipe, place in if statement to avoid uncheck return warnings */
-    if (read (pipefds[0], buf, 1024) < 0)
-        fprintf (stderr, "read error: %s\n", strerror (errno));
-
-    flux_buffer_destroy (fb);
-    close (pipefds[0]);
-    close (pipefds[1]);
-}
-
-void read_loop_cb (flux_buffer_t *fb, void *arg)
-{
-    int *count = arg;
-    const char *ptr;
-    int len;
-
-    while ((ptr = flux_buffer_read (fb, 6, &len)) && len > 0) {
-        (*count)++;
-        ok (ptr && len == 6,
-           "flux_buffer_read in loop works");
-        is (ptr, "foobar",
-           "flux_buffer_read in loop returns expected data");
-    }
-    ok (len == 0,
-        "flux_buffer_read() returns len == 0 when buffer is empty");
-}
-
-void read_line_loop_cb (flux_buffer_t *fb, void *arg)
-{
-    int *count = arg;
-    const char *ptr;
-    int len;
-
-    while ((ptr = flux_buffer_read_line (fb, &len)) && len > 0) {
-        (*count)++;
-        ok (ptr && len == 7,
-           "flux_buffer_read in loop works");
-        is (ptr, "foobar\n",
-           "flux_buffer_read in loop returns expected data");
-    }
-    ok (len == 0,
-        "flux_buffer_read_line() returns len == 0 when buffer is empty");
-
-}
-
-void callback_loops (void)
-{
-
-    flux_buffer_t *fb;
-    int count;
-
-    ok ((fb = flux_buffer_create (FLUX_BUFFER_TEST_MAXSIZE)) != NULL,
-        "flux_buffer_create works");
-
-    /* low read callback w/ write */
-
-    count = 0;
-    ok (flux_buffer_set_low_read_cb (fb, read_loop_cb, 3, &count) == 0,
-        "flux_buffer_set_low_read_cb success");
-
-    ok (flux_buffer_write (fb, "foobarfoobar", 12) == 12,
-        "flux_buffer_write success");
-
-    ok (count == 2,
-        "read_loop_cb called, loop ran twice");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 because callback read all data");
-
-    ok (flux_buffer_set_low_read_cb (fb, NULL, 0, &count) == 0,
-        "flux_buffer clear read_cb");
-
-    /* read line callback w/ write_line */
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 on no line");
-    count = 0;
-    ok (flux_buffer_set_read_line_cb (fb, read_line_loop_cb, &count) == 0,
-        "flux_buffer_set_read_line_cb success");
-
-    ok (flux_buffer_write (fb, "foobar\nfoobar\n", 14) == 14,
-        "flux_buffer_write two lines success");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes returns 0 because callback read all data");
-
-    ok (count == 2,
-        "read_line_loop_cb called, two loops");
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 on no line, callback read all data");
-    flux_buffer_destroy (fb);
-}
-
-void disable_read_cb (flux_buffer_t *fb, void *arg)
-{
-    int *count = arg;
-    const char *ptr;
-    int len;
-
-    (*count)++;
-
-    ok ((ptr = flux_buffer_read (fb, 3, &len)) != NULL
-        && len == 3,
-        "flux_buffer_read in callback works");
-
-    ok (!flux_buffer_set_low_read_cb (fb,
-                                      NULL,
-                                      0,
-                                      NULL),
-        "read cb successfully disabled");
-}
-
-void disable_read_line_cb (flux_buffer_t *fb, void *arg)
-{
-    int *count = arg;
-    const char *ptr;
-    int len;
-
-    (*count)++;
-
-    ok ((ptr = flux_buffer_read_line (fb, &len)) != NULL
-        && len == 4,
-        "flux_buffer_read_line in callback works");
-
-    ok (!flux_buffer_set_read_line_cb (fb,
-                                       NULL,
-                                       NULL),
-        "read line cb successfully disabled");
-}
-
-void disable_write_cb (flux_buffer_t *fb, void *arg)
-{
-    int *count = arg;
-
-    (*count)++;
-
-    ok (!flux_buffer_set_high_write_cb (fb,
-                                        NULL,
-                                        0,
-                                        NULL),
-        "write cb successfully disabled");
-}
-
-void disable_callback (void)
-{
-    flux_buffer_t *fb;
-    const char *ptr;
-    int len;
-    int count;
-
-    ok ((fb = flux_buffer_create (FLUX_BUFFER_TEST_MAXSIZE)) != NULL,
-        "flux_buffer_create works");
-
-    /* low read callback w/ write */
-
-    count = 0;
-    ok (flux_buffer_set_low_read_cb (fb, disable_read_cb, 0, &count) == 0,
-        "flux_buffer_set_low_read_cb success");
-
-    ok (flux_buffer_write (fb, "foobar", 6) == 6,
-        "flux_buffer_write success");
-
-    ok (count == 1,
-        "disable_read_cb called only once, disabling callback in callback worked");
-
-    ok (flux_buffer_write (fb, "foo", 3) == 3,
-        "flux_buffer_write success");
-
-    ok (count == 1,
-        "disable_read_cb not called again, callback is disabled");
-
-    ok (flux_buffer_drop (fb, -1) == 6,
-        "flux_buffer_drop cleared all data");
-
-    /* read line callback w/ write_line */
-
-    ok (flux_buffer_lines (fb) == 0,
-        "flux_buffer_lines returns 0 on no line");
-    ok (!flux_buffer_has_line (fb),
-        "flux_buffer_has_line returns false on no line");
-
-    count = 0;
-    ok (flux_buffer_set_read_line_cb (fb, disable_read_line_cb, &count) == 0,
-        "flux_buffer_set_read_line_cb success");
-
-    ok (flux_buffer_write (fb, "foo\nfoo\n", 8) == 8,
-        "flux_buffer_write success");
-
-    ok (count == 1,
-        "disable_read_line_cb called only once, disabling callback in callback worked");
-
-    ok (flux_buffer_write (fb, "foo\n", 4) == 4,
-        "flux_buffer_write success");
-
-    ok (count == 1,
-        "disable_read_line_cb not called again, callback is disabled");
-
-    ok (flux_buffer_drop (fb, -1) == 8,
-        "flux_buffer_drop cleared all data");
-
-    /* high write callback w/ read */
-
-    ok (flux_buffer_write (fb, "foofoo", 6) == 6,
-        "flux_buffer_write success");
-
-    count = 0;
-    ok (flux_buffer_set_high_write_cb (fb, disable_write_cb, 6, &count) == 0,
-        "flux_buffer_set_high_write_cb success");
-
-    ok ((ptr = flux_buffer_read (fb, 3, &len)) != NULL
-        && len == 3,
-        "flux_buffer_read success");
-
-    ok (count == 1,
-        "disable_write_cb called correct number of times");
-
-    ok ((ptr = flux_buffer_read (fb, 3, &len)) != NULL
-        && len == 3,
-        "flux_buffer_read success");
-
-    ok (count == 1,
-        "disable_write_cb not called again, successfully disabled");
-
-    flux_buffer_destroy (fb);
-}
-
-void corner_case (void)
-{
-    flux_buffer_t *fb;
-    const char *ptr;
-    int len;
-
-    ok (flux_buffer_create (-1) == NULL
-        && errno == EINVAL,
-        "flux_buffer_create fails on bad input -1");
-
-    ok (flux_buffer_create (0) == NULL
-        && errno == EINVAL,
-        "flux_buffer_create fails on bad input 0");
-
-    /* all functions fail on NULL fb pointer */
-    ok (flux_buffer_size (NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_size fails on NULL pointer");
-    ok (flux_buffer_bytes (NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_bytes fails on NULL pointer");
-    ok (flux_buffer_space (NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_space fails on NULL pointer");
-    ok (flux_buffer_readonly (NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_readonly fails on NULL pointer");
-    errno = 0;
-    ok (!flux_buffer_is_readonly (NULL) && errno == EINVAL,
-        "flux_buffer_is_readonly returns false on NULL pointer");
-    ok (flux_buffer_set_low_read_cb (NULL, empty_cb, 0, NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_set_low_read_cb fails on NULL pointer");
-    ok (flux_buffer_set_read_line_cb (NULL, empty_cb, NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_set_read_line_cb fails on NULL pointer");
-    ok (flux_buffer_set_high_write_cb (NULL, empty_cb, 0, NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_set_high_write_cb fails on NULL pointer");
-    ok (flux_buffer_drop (NULL, -1) < 0
-        && errno == EINVAL,
-        "flux_buffer_drop fails on NULL pointer");
-    ok (flux_buffer_peek (NULL, -1, NULL) == NULL
-        && errno == EINVAL,
-        "flux_buffer_peek fails on NULL pointer");
-    ok (flux_buffer_read (NULL, -1, NULL) == NULL
-        && errno == EINVAL,
-        "flux_buffer_read fails on NULL pointer");
-    ok (flux_buffer_write (NULL, NULL, 0) < 0
-        && errno == EINVAL,
-        "flux_buffer_write fails on NULL pointer");
-    ok (flux_buffer_lines (NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_lines fails on NULL pointer");
-    errno = 0;
-    ok (!flux_buffer_has_line (NULL)
-        && errno == EINVAL,
-        "flux_buffer_has_line returns false with errno set on NULL pointer");
-    ok (flux_buffer_drop_line (NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_drop_line fails on NULL pointer");
-    ok (flux_buffer_peek_line (NULL, NULL) == NULL
-        && errno == EINVAL,
-        "flux_buffer_peek_line fails on NULL pointer");
-    ok (flux_buffer_peek_trimmed_line (NULL, NULL) == NULL
-        && errno == EINVAL,
-        "flux_buffer_peek_trimmed_line fails on NULL pointer");
-    ok (flux_buffer_read_line (NULL, NULL) == NULL
-        && errno == EINVAL,
-        "flux_buffer_read_line fails on NULL pointer");
-    ok (flux_buffer_read_trimmed_line (NULL, NULL) == NULL
-        && errno == EINVAL,
-        "flux_buffer_read_trimmed_line fails on NULL pointer");
-    ok (flux_buffer_write_line (NULL, "foo") < 0
-        && errno == EINVAL,
-        "flux_buffer_write_line fails on NULL pointer");
-    ok (flux_buffer_peek_to_fd (NULL, 0, 0) < 0
-        && errno == EINVAL,
-        "flux_buffer_peek_to_fd fails on NULL pointer");
-    ok (flux_buffer_read_to_fd (NULL, 0, 0) < 0
-        && errno == EINVAL,
-        "flux_buffer_read_to_fd fails on NULL pointer");
-    ok (flux_buffer_write_from_fd (NULL, 0, 0) < 0
-        && errno == EINVAL,
-        "flux_buffer_write_from_fd fails on NULL pointer");
-
-    ok ((fb = flux_buffer_create (FLUX_BUFFER_TEST_MAXSIZE)) != NULL,
-        "flux_buffer_create works");
-
-    ok ((ptr = flux_buffer_peek (fb, -1, &len)) != NULL,
-        "flux_buffer_peek works when no data available");
-    ok (len == 0,
-        "flux_buffer_peek returns length 0 when no data available");
-    ok ((ptr = flux_buffer_read (fb, -1, &len)) != NULL,
-        "flux_buffer_read works when no data available");
-    ok (len == 0,
-        "flux_buffer_read returns length 0 when no data available");
-
-    ok ((ptr = flux_buffer_peek_line (fb, &len)) != NULL,
-        "flux_buffer_peek_line works when no data available");
-    ok (len == 0,
-        "flux_buffer_peek_line returns length 0 when no data available");
-    ok ((ptr = flux_buffer_peek_trimmed_line (fb, &len)) != NULL,
-        "flux_buffer_peek_trimmed_line works when no data available");
-    ok (len == 0,
-        "flux_buffer_peek_trimmed_line returns length 0 when no data available");
-    ok ((ptr = flux_buffer_read_line (fb, &len)) != NULL,
-        "flux_buffer_read_line works when no data available");
-    ok (len == 0,
-        "flux_buffer_read_line returns length 0 when no data available");
-    ok ((ptr = flux_buffer_read_trimmed_line (fb, &len)) != NULL,
-        "flux_buffer_read_trimmed_line works when no data available");
-    ok (len == 0,
-        "flux_buffer_read_trimmed_line returns length 0 when no data available");
-
-    /* callback corner case tests */
-
-    ok (flux_buffer_set_low_read_cb (fb, empty_cb, -1, NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_set_low_read_cb fails on bad input");
-    ok (flux_buffer_set_low_read_cb (fb, empty_cb, 0, NULL) == 0,
-        "flux_buffer_set_low_read_cb success");
-    ok (flux_buffer_set_low_read_cb (fb, empty_cb, -1, NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_set_low_read_cb fails on bad input overwrite callback");
-    ok (flux_buffer_set_read_line_cb (fb, empty_cb, NULL) < 0
-        && errno == EEXIST,
-        "flux_buffer_set_read_line_cb fails if callback already set");
-    ok (flux_buffer_set_high_write_cb (fb, empty_cb, 0, NULL) < 0
-        && errno == EEXIST,
-        "flux_buffer_set_high_write_cb fails if callback already set");
-    ok (flux_buffer_set_low_read_cb (fb, NULL, 0, NULL) == 0,
-        "flux_buffer_set_low_read_cb success clear callback");
-
-    ok (flux_buffer_set_read_line_cb (fb, empty_cb, NULL) == 0,
-        "flux_buffer_set_read_line_cb success");
-    ok (flux_buffer_set_low_read_cb (fb, empty_cb, 0, NULL) < 0
-        && errno == EEXIST,
-        "flux_buffer_set_low_read_cb fails if callback already set");
-    ok (flux_buffer_set_high_write_cb (fb, empty_cb, 0, NULL) < 0
-        && errno == EEXIST,
-        "flux_buffer_set_high_write_cb fails if callback already set");
-    ok (flux_buffer_set_read_line_cb (fb, NULL, NULL) == 0,
-        "flux_buffer_set_read_line_cb success clear callback");
-
-    ok (flux_buffer_set_high_write_cb (fb, empty_cb, -1, NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_set_high_write_cb fails on bad input");
-    ok (flux_buffer_set_high_write_cb (fb, empty_cb, 0, NULL) == 0,
-        "flux_buffer_set_high_write_cb success");
-    ok (flux_buffer_set_high_write_cb (fb, empty_cb, -1, NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_set_high_write_cb fails on bad input overwrite callback");
-    ok (flux_buffer_set_low_read_cb (fb, empty_cb, 0, NULL) < 0
-        && errno == EEXIST,
-        "flux_buffer_set_low_read_cb fails if callback already set");
-    ok (flux_buffer_set_read_line_cb (fb, empty_cb, NULL) < 0
-        && errno == EEXIST,
-        "flux_buffer_set_read_line_cb fails if callback already set");
-    ok (flux_buffer_set_high_write_cb (fb, NULL, 0, NULL) == 0,
-        "flux_buffer_set_high_write_cb success clear callback");
-
-    /* write corner case tests */
-
-    ok (flux_buffer_write (fb, NULL, 0) < 0
-        && errno == EINVAL,
-        "flux_buffer_write fails on bad input");
-    ok (flux_buffer_write (fb, "foo", -1) < 0
-        && errno == EINVAL,
-        "flux_buffer_write fails on bad input");
-    ok (flux_buffer_write_line (fb, NULL) < 0
-        && errno == EINVAL,
-        "flux_buffer_write_line fails on bad input");
-    ok (flux_buffer_write_from_fd (fb, -1, 0) < 0
-        && errno == EINVAL,
-        "flux_buffer_write_from_fd fails on bad input");
-
-    /* flux_buffer_destroy works with NULL */
-    flux_buffer_destroy (NULL);
-
-    flux_buffer_destroy (fb);
-}
-
-void full_buffer (void)
-{
-    flux_buffer_t *fb;
-
-    ok ((fb = flux_buffer_create (4)) != NULL,
-        "flux_buffer_create works");
-
-    ok (flux_buffer_write (fb, "1234", 4) == 4,
-        "flux_buffer_write success");
-
-    ok (flux_buffer_bytes (fb) == 4,
-        "flux_buffer_bytes returns length of bytes written");
-
-    ok (flux_buffer_space (fb) == 0,
-        "flux_buffer_space returns length of space left");
-
-    ok (flux_buffer_write (fb, "5", 1) < 0
-        && errno == ENOSPC,
-        "flux_buffer_write fails with ENOSPC if exceeding buffer size");
-
-    ok (flux_buffer_drop (fb, -1) == 4,
-        "flux_buffer_drop works");
-
-    ok (flux_buffer_write_line (fb, "1234") < 0
-        && errno == ENOSPC,
-        "flux_buffer_write_line fails with ENOSPC if exceeding buffer size");
-
-    flux_buffer_destroy (fb);
-}
-
-void readonly_buffer (void)
-{
-    flux_buffer_t *fb;
-    int pipefds[2];
-
-    ok (pipe (pipefds) == 0,
-        "pipe succeeded");
-
-    ok ((fb = flux_buffer_create (FLUX_BUFFER_TEST_MAXSIZE)) != NULL,
-        "flux_buffer_create works");
-
-    ok (!flux_buffer_is_readonly (fb),
-        "flux buffer is not readonly on creation");
-
-    ok (flux_buffer_readonly (fb) == 0,
-        "flux buffer readonly set");
-
-    ok (flux_buffer_is_readonly (fb),
-        "flux buffer is readonly after setting");
-
-    flux_buffer_destroy (fb);
-
-    ok ((fb = flux_buffer_create (FLUX_BUFFER_TEST_MAXSIZE)) != NULL,
-        "flux_buffer_create works");
-
-    ok (flux_buffer_write (fb, "foobar", 6) == 6,
-        "flux_buffer_write success");
-
-    ok (flux_buffer_readonly (fb) == 0,
-        "flux buffer readonly set");
-
-    ok (flux_buffer_write (fb, "foobar", 6) < 0
-        && errno == EROFS,
-        "flux_buffer_write fails b/c readonly is set");
-
-    ok (flux_buffer_write_line (fb, "foobar") < 0
-        && errno == EROFS,
-        "flux_buffer_write_line fails b/c readonly is set");
-
-    ok (write (pipefds[1], "foo", 3) == 3,
-        "write to pipe works");
-
-    ok (flux_buffer_write_from_fd (fb, pipefds[0], -1) < 0
-        && errno == EROFS,
-        "flux_buffer_write_from_fd fails b/c readonly is set");
-
-    flux_buffer_destroy (fb);
-    close (pipefds[0]);
-    close (pipefds[1]);
-}
-
-/* tests to ensure internal buffers grow appropriately.  Current
- * buffer default min is 4096, so we need to test > 4096 bytes of
- * data.
- */
-void large_data (void)
-{
-    flux_buffer_t *fb;
-    const char *data = "0123456789ABCDEF0123456789ABCDEF";
-    const char *ptr;
-    int len;
-    int i;
-
-    ok ((fb = flux_buffer_create (FLUX_BUFFER_TEST_MAXSIZE)) != NULL,
-        "flux_buffer_create works");
-
-    ok (flux_buffer_size (fb) == FLUX_BUFFER_TEST_MAXSIZE,
-        "flux_buffer_size returns correct size");
-
-    ok (flux_buffer_bytes (fb) == 0,
-        "flux_buffer_bytes initially returns 0");
-
-    ok (flux_buffer_space (fb) == FLUX_BUFFER_TEST_MAXSIZE,
-        "flux_buffer_space initially returns FLUX_BUFFER_TEST_MAXSIZE");
-
-    for (i = 0; i < 256; i++) {
-        if (flux_buffer_write (fb, data, 32) != 32)
-            ok (false, "flux_buffer_write fail: %s", strerror (errno));
-    }
-
-    ok (flux_buffer_space (fb) == (FLUX_BUFFER_TEST_MAXSIZE - 8192),
-        "flux_buffer_space returns length of space left");
-
-    ok ((ptr = flux_buffer_read (fb, -1, &len)) != NULL
-        && len == 8192,
-        "flux_buffer_read with length -1 works");
-
-    for (i = 0; i < 256; i++) {
-        if (memcmp (ptr + (i * 32), data, 32) != 0)
-            ok (false, "flux_buffer_read returned bad data");
-    }
-
-    flux_buffer_destroy (fb);
-}
-
-int main (int argc, char *argv[])
-{
-    plan (NO_PLAN);
-
-    basic ();
-    basic_callback ();
-    callback_loops ();
-    disable_callback ();
-    corner_case ();
-    full_buffer ();
-    readonly_buffer ();
-    large_data ();
-
-    done_testing();
-
-    return (0);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/src/common/libflux/test/composite_future.c b/src/common/libflux/test/composite_future.c
index e6bab9d380b1..2a7d32fac6b0 100644
--- a/src/common/libflux/test/composite_future.c
+++ b/src/common/libflux/test/composite_future.c
@@ -8,13 +8,14 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#define _GNU_SOURCE
-#include <czmq.h>
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <stdio.h>
+#include <flux/core.h>
 
-#include "src/common/libflux/reactor.h"
-#include "src/common/libflux/future.h"
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 static bool init_and_fulfill_called = false;
 static bool init_no_fulfill_called = false;
@@ -72,18 +73,18 @@ static void test_composite_basic_any (flux_reactor_t *r, bool with_error)
         "flux_future_get_child (any, 'f1') == f1");
 
     s = flux_future_first_child (any);
-    ok ((s != NULL) && !strcmp (s, "f1"),
+    ok ((s != NULL) && streq (s, "f1"),
         "flux_future_first_child() == 'f1'");
 
     ok (flux_future_push (any, "f2", f2) == 0,
         "flux_future_push (any, 'f2', f2)");
 
     s = flux_future_first_child (any);
-    ok (s != NULL && (!strcmp (s, "f1") || !strcmp (s, "f2")),
+    ok (s != NULL && (streq (s, "f1") || streq (s, "f2")),
         "flux_future_first_child (any) returns one of two children");
     p = flux_future_next_child (any);
-    ok ((p != NULL) && (!strcmp (p, "f1") || !strcmp (p, "f2"))
-        && (strcmp (p, s) != 0),
+    ok ((p != NULL) && (streq (p, "f1") || streq (p, "f2"))
+        && (!streq (p, s)),
         "flux_future_next_child (any) returns different child (%s)", s);
     ok (flux_future_next_child (any) == NULL,
         "flux_future_next_child (any) == NULL signifies end of list");
@@ -134,7 +135,7 @@ static void test_composite_basic_all (flux_reactor_t *r, bool with_error)
         "flux_future_get_child (all, 'f1') == f1");
 
     s = flux_future_first_child (all);
-    ok ((s != NULL) && !strcmp (s, "f1"),
+    ok ((s != NULL) && streq (s, "f1"),
         "flux_future_first_child() == 'f1'");
 
     ok (flux_future_push (all, "f2", f2) == 0,
@@ -370,7 +371,7 @@ void timeout_init (flux_future_t *f, void *arg)
     flux_future_fulfill_error (f, errno, NULL);
 }
 
-static void flux_future_timeout_clear (flux_future_t *f)
+static void future_timeout_clear (flux_future_t *f)
 {
     flux_watcher_t *w = flux_future_aux_get (f, "watcher");
     ok (w != NULL, "timeout stop: got timer watcher");
@@ -378,7 +379,7 @@ static void flux_future_timeout_clear (flux_future_t *f)
         flux_watcher_stop (w);
 }
 
-static flux_future_t *flux_future_timeout (double s)
+static flux_future_t *future_timeout (double s)
 {
     double *dptr = calloc (1, sizeof (*dptr));
     if (dptr == NULL)
@@ -427,8 +428,8 @@ void test_composite_all_async (bool with_error)
     ok (flux_future_push (fc, "f1", f) == 0,
         "flux_future_push success");
 
-    if (!(f = flux_future_timeout (0.1)))
-        BAIL_OUT ("flux_future_timeout failed");
+    if (!(f = future_timeout (0.1)))
+        BAIL_OUT ("future_timeout failed");
 
     ok (flux_future_push (fc, "timeout", f) == 0,
         "flux_future_push timeout success");
@@ -468,7 +469,7 @@ void async_any_check (flux_future_t *fc, void *arg)
         "async: retrieved handle to timeout future");
     ok (flux_future_is_ready (f) == false,
         "async: timeout future not yet fulfilled");
-    flux_future_timeout_clear (f);
+    future_timeout_clear (f);
     async_any_check_rc = 0;
     /* Required so we pop out of reactor since we will still have
      *  active watchers */
@@ -491,8 +492,8 @@ void test_composite_any_async (bool with_error)
     ok (flux_future_push (fc, "f1", f) == 0,
         "flux_future_push success");
 
-    if (!(f = flux_future_timeout (1.0)))
-        BAIL_OUT ("flux_future_timeout failed");
+    if (!(f = future_timeout (1.0)))
+        BAIL_OUT ("future_timeout failed");
 
     ok (flux_future_push (fc, "timeout", f) == 0,
         "flux_future_push timeout success");
@@ -724,6 +725,80 @@ void test_empty_composite (flux_reactor_t *r)
     flux_future_destroy (all);
 }
 
+static void fnext_cb (flux_future_t *f, void *arg)
+{
+    // Dummy continuation callback for testing below
+}
+
+void test_future_fulfill_next (flux_reactor_t *r)
+{
+    char *result = NULL;
+    const char *s = NULL;
+    flux_future_t *f = flux_future_create (NULL, NULL);
+    flux_future_t *f2 = flux_future_and_then (f, fnext_cb, NULL);
+    if (!f || !f2)
+        BAIL_OUT ("Error creating test futures");
+
+    ok (flux_future_fulfill_next (f2, NULL, NULL) < 0 && errno == EINVAL,
+        "flux_future_fulfill_next() returns EINVAL if future is not chained");
+
+    if (!(result = strdup ("ok")))
+        BAIL_OUT ("Failed to create result");
+
+    ok (flux_future_fulfill_next (f, result, free) == 0,
+        "flux_future_fulfill_next() works on chained future");
+
+    flux_future_destroy (f);
+
+    ok (flux_future_wait_for (f2, 0.1) == 0,
+        "next future was fulfilled by flux_future_fulfill_next()");
+    ok (flux_future_get (f2, (const void **) &s) == 0,
+        "flux_future_get on fulfilled future works");
+    ok (s == result,
+        "flux_future_get() returns expected result", s, s);
+
+    flux_future_destroy (f2);
+}
+
+static bool issue5923_and_then_called = false;
+static bool issue5923_then_called = false;
+
+static void issue5923_and_then_cb (flux_future_t *f, void *arg)
+{
+    issue5923_and_then_called = false;
+    flux_future_destroy (f);
+}
+
+static void issue5923_then_cb (flux_future_t *f, void *arg)
+{
+    issue5923_then_called = true;
+    ok (flux_future_get (f, NULL) < 0 && errno == ETIMEDOUT,
+        "issue5923: then cb timed out");
+    flux_future_destroy (f);
+}
+
+static void test_issue_5923 (flux_reactor_t *r)
+{
+    flux_future_t *prev, *next;
+
+    if (!(prev = future_timeout (0.1)))
+        BAIL_OUT ("future_timeout failed");
+    flux_future_set_reactor (prev, r);
+
+    if (!(next = flux_future_and_then (prev, issue5923_and_then_cb, NULL)))
+        BAIL_OUT ("flux_future_and_then failed");
+
+    if (flux_future_then (next, 0.001, issue5923_then_cb, NULL) < 0)
+        BAIL_OUT ("flux_future_then failed");
+
+    ok (flux_reactor_run (r, 0) == 0,
+        "flux_reactor_run returns 0");
+    ok (issue5923_then_called,
+        "issue5923: then_cb was called");
+    ok (!issue5923_and_then_called,
+        "issue5923: and_then_cb was not called");
+}
+
 int main (int argc, char *argv[])
 {
     flux_reactor_t *reactor;
@@ -751,6 +826,10 @@ int main (int argc, char *argv[])
     test_composite_anon_child (reactor, false);
     test_empty_composite (reactor);
 
+    test_future_fulfill_next (reactor);
+
+    test_issue_5923 (reactor);
+
     flux_reactor_destroy (reactor);
 
     done_testing();
diff --git a/src/common/libflux/test/conf.c b/src/common/libflux/test/conf.c
index 3d1fb539fef1..2c926e917488 100644
--- a/src/common/libflux/test/conf.c
+++ b/src/common/libflux/test/conf.c
@@ -25,7 +25,7 @@
 #include "src/common/libflux/conf_private.h"
 
 #include "src/common/libtap/tap.h"
-#include "src/common/libtestutil/util.h"
+#include "ccan/str/str.h"
 
 const char *t1 = \
 "i = 1\n" \
@@ -45,13 +45,29 @@ const char *tab3 = \
 "[tab3]\n" \
 "id = 3\n";
 
+const char *tab3_json = \
+"{\"tab3\": {\"id\": 4}}";
+
+const char *tab4 = \
+"[tab]\n" \
+"added = \"bar\"";
+
 
 static void
-create_test_file (const char *dir, char *prefix, char *path, size_t pathlen,
+create_test_file (const char *dir,
+                  char *prefix,
+                  char *ext,
+                  char *path,
+                  size_t pathlen,
                   const char *contents)
 {
     int fd;
-    snprintf (path, pathlen, "%s/%s.XXXXXX.toml", dir ? dir : "/tmp", prefix);
+    snprintf (path,
+              pathlen,
+              "%s/%s.XXXXXX.%s",
+              dir ? dir : "/tmp",
+              prefix,
+              ext);
     fd = mkstemps (path, 5);
     if (fd < 0)
         BAIL_OUT ("mkstemp %s: %s", path, strerror (errno));
@@ -59,6 +75,7 @@ create_test_file (const char *dir, char *prefix, char *path, size_t pathlen,
         BAIL_OUT ("write %s: %s", path, strerror (errno));
     if (close (fd) < 0)
         BAIL_OUT ("close %s: %s", path, strerror (errno));
+    diag ("created %s", path);
 }
 
 void test_builtin (void)
@@ -74,7 +91,7 @@ void test_builtin (void)
     s3 = flux_conf_builtin_get ("shell_path", FLUX_CONF_AUTO);
     ok (s3 != NULL,
         "flux_conf_builtin_get shell_path AUTO works");
-    ok (s2 && s3 && !strcmp (s2, s3),
+    ok (s2 && s3 && streq (s2, s3),
         "AUTO returned INTREE value for test executable");
 
     errno = 0;
@@ -91,15 +108,15 @@ void test_builtin (void)
 void test_basic (void)
 {
     int rc;
-    int len;
     const char *tmpdir = getenv ("TMPDIR");
     char dir[PATH_MAX + 1];
     char path1[PATH_MAX + 1];
     char path2[PATH_MAX + 1];
     char path3[PATH_MAX + 1];
+    char path4[PATH_MAX + 1];
+    char pathj[PATH_MAX + 1];
     char invalid[PATH_MAX + 1];
-    char p[PATH_MAX + 1];
-    flux_conf_error_t error;
+    flux_error_t error;
     flux_conf_t *conf;
     int i, j, k;
     double d;
@@ -112,19 +129,62 @@ void test_basic (void)
     if (!mkdtemp (dir))
         BAIL_OUT ("mkdtemp %s: %s", dir, strerror (errno));
 
-    create_test_file (dir, "01", path1, sizeof (path1), t1);
-    create_test_file (dir, "02", path2, sizeof (path2), tab2);
-    create_test_file (dir, "03", path3, sizeof (path3), tab3);
+    /* Empty directory is allowed
+     */
+    conf = flux_conf_parse (dir, &error);
+    ok (conf != NULL,
+        "flux_conf_parse successfully parsed empty directory");
+    flux_conf_decref (conf);
 
-    len = snprintf (p, sizeof (p), "%s/*.toml", dir);
-    if ((len < 0) || (len >= sizeof (p)))
-        BAIL_OUT ("snprintf failed in creating toml file path");
+    /* Add files
+     */
+    create_test_file (dir, "01", "toml", path1, sizeof (path1), t1);
+    create_test_file (dir, "02", "toml", path2, sizeof (path2), tab2);
+    create_test_file (dir, "03", "toml", path3, sizeof (path3), tab3);
+    create_test_file (dir, "04", "toml", path4, sizeof (path4), tab4);
+    create_test_file (NULL, "03", "json", pathj, sizeof (pathj), tab3_json);
+
+    /* Parse of one file works
+     */
+    conf = flux_conf_parse (path3, &error);
+    ok (conf != NULL,
+        "flux_conf_parse successfully parsed a single file");
+    if (!conf)
+        BAIL_OUT ("cannot continue without config object");
+
+    /* Check table from path3 toml file
+     */
+    i = 0;
+    rc = flux_conf_unpack (conf,
+                           &error,
+                           "{s:{s:i}}",
+                           "tab3",
+                           "id", &i);
+    ok (rc == 0 && i == 3,
+        "unpacked integer from [tab3] and got expected value");
+
+    flux_conf_decref (conf);
+
+    /* Parse one file JSON edition
+     */
+    conf = flux_conf_parse (pathj, &error);
+    ok (conf != NULL,
+        "flux_conf_parse works for just one file (JSON)");
+    i = 0;
+    rc = flux_conf_unpack (conf,
+                           &error,
+                           "{s:{s:i}}",
+                           "tab3",
+                           "id", &i);
+    ok (rc == 0 && i == 4,
+        "unpacked integer from [tab3] and got expected value");
+    flux_conf_decref (conf);
 
     /* Parse it
      */
-    conf = conf_parse (p, &error);
+    conf = flux_conf_parse (dir, &error);
     ok (conf != NULL,
-        "conf_parse successfully parsed 3 files");
+        "flux_conf_parse successfully parsed 3 files");
     if (!conf)
         BAIL_OUT ("cannot continue without config object");
 
@@ -149,7 +209,7 @@ void test_basic (void)
         "unpacked double value");
     ok (b == true,
         "unpacked boolean value");
-    ok (s != NULL && !strcmp (s, "foo"),
+    ok (s != NULL && streq (s, "foo"),
         "unpacked string value");
 
     /* Check array contents
@@ -188,6 +248,18 @@ void test_basic (void)
     ok (rc == 0 && i == 42,
         "unpacked integer from [tab] and got expected value");
 
+    /* Check that tab was updated with added value from tab4
+     */
+    rc = flux_conf_unpack (conf,
+                           &error,
+                           "{s:{s:s}}",
+                           "tab",
+                           "added", &s);
+    diag ("added = %s", s);
+    ok (rc == 0 && streq (s, "bar"),
+        "unpacked added string from [tab] and got expected value");
+
+
     /* Check table from second toml file
      */
     i = 0;
@@ -216,9 +288,9 @@ void test_basic (void)
     ok (flux_conf_unpack (conf, &error, "{s:s}", "noexist", &s) < 0
         && errno == EINVAL,
         "flux_conf_unpack key=noexist failed with EINVAL");
-    ok (strstr (error.errbuf, "noexist") != NULL,
-        "and errbuf mentions noexist");
-    diag ("%s: %d: %s", error.filename, error.lineno, error.errbuf);
+    ok (strstr (error.text, "noexist") != NULL,
+        "and error.text mentions noexist");
+    diag ("%s", error.text);
 
     /* Bad args fail with EINVAL
      */
@@ -227,48 +299,58 @@ void test_basic (void)
         && errno == EINVAL,
         "flux_conf_unpack conf=NULL fails with EINVAL");
 
-    conf_destroy (conf);
+    flux_conf_decref (conf);
 
     /* Now make an invalid file and ensure cf_update_glob() aborts
      * all updates after any one failure
      */
-    create_test_file (dir, "99", invalid, sizeof (invalid), "key = \n");
+    create_test_file (dir, "99", "toml", invalid, sizeof (invalid), "key = \n");
 
-    conf = conf_parse (p, &error);
+    conf = flux_conf_parse (invalid, &error);
     ok (conf == NULL,
-        "conf_parse choked on glob referencing some good and one bad file");
+        "flux_conf_parse failed on bad individual file");
+    like (error.text, "99.*\\.toml",
+          "Failed file contained in error.text");
 
-    diag ("%s: %d: %s", error.filename, error.lineno, error.errbuf);
-    like (error.filename, "99.*\\.toml",
-          "Failed file contained in error.filename");
+    conf = flux_conf_parse (dir, &error);
+    ok (conf == NULL,
+        "flux_conf_parse choked on glob referencing some good and one bad file");
 
-    /* Invalid pattern arg
+    diag ("%s", error.text);
+    like (error.text, "99.*\\.toml",
+          "Failed file contained in error.text");
+
+    /* Parse invalid JSON file
      */
-    errno = 0;
-    ok (conf_parse (NULL, &error) == NULL && errno == EINVAL,
-        "conf_parse pattern=NULL fails with EINVAL");
-    diag ("%s: %d: %s", error.filename, error.lineno, error.errbuf);
+    unlink (invalid);
+    create_test_file (dir, "foo", "json", invalid, sizeof (invalid), "{");
+    conf = flux_conf_parse (invalid, &error);
+    ok (conf == NULL,
+        "flux_conf_parse choked on bad file");
 
-    /* Directory not found causes a GLOB_ABORTED error (returned as EINVAL).
+    diag ("%s", error.text);
+    like (error.text, "foo.*\\.json",
+          "Failed file contained in error.text");
+
+    /* Invalid pattern arg
      */
     errno = 0;
-    ok (conf_parse ("/noexist/*.toml", &error) == NULL && errno == EINVAL,
-        "conf_parse pattern=/noexist/*.toml fails with EINVAL");
-    diag ("%s: %d: %s", error.filename, error.lineno, error.errbuf);
+    ok (flux_conf_parse (NULL, &error) == NULL && errno == EINVAL,
+        "flux_conf_parse path=NULL fails with EINVAL");
+    diag ("%s", error.text);
 
-    /* No glob match causes GLOB_NOMATCH error (returned as ENOENT)
+    /* Directory not found triggers ENOENT error
      */
-    if (snprintf (p, sizeof (p), "%s/*.noexist", dir) >= sizeof (p))
-        BAIL_OUT ("snprintf failed in creating toml file pattern");
     errno = 0;
-    ok (conf_parse (p, &error) == NULL && errno == ENOENT,
-        "conf_parse pattern=*.noexist fails with ENOENT");
-    diag ("%s: %d: %s", error.filename, error.lineno, error.errbuf);
-
+    ok (flux_conf_parse ("/noexist", &error) == NULL && errno == ENOENT,
+        "flux_conf_parse pattern=/noexist fails with ENOENT");
+    diag ("%s", error.text);
 
     if (   (unlink (path1) < 0)
         || (unlink (path2) < 0)
         || (unlink (path3) < 0)
+        || (unlink (path4) < 0)
+        || (unlink (pathj) < 0)
         || (unlink (invalid) < 0) )
         BAIL_OUT ("unlink: %s", strerror (errno));
     if (rmdir (dir) < 0)
@@ -276,76 +358,31 @@ void test_basic (void)
 
 }
 
-void test_default_pattern (void)
-{
-    char toosmall[1];
-    char buf[PATH_MAX+1];
-    char exp[PATH_MAX+1];
-    const char *cf_path;
-
-    /* default
-     */
-    cf_path = flux_conf_builtin_get ("cf_path", FLUX_CONF_AUTO);
-    (void)snprintf (exp, sizeof (exp), "%s/*.toml", cf_path);
-    ok (conf_get_default_pattern (buf, sizeof (buf)) == 0
-        && !strcmp (buf, exp),
-        "conf_get_default_pattern works");
-
-    /* FLUX_CONF_DIR="/a/b"
-     */
-    setenv ("FLUX_CONF_DIR", "/a/b", 1);
-    (void)snprintf (exp, sizeof (exp), "/a/b/*.toml");
-    ok (conf_get_default_pattern (buf, sizeof (buf)) == 0
-        && !strcmp (buf, exp),
-        "conf_get_default_pattern FLUX_CONF_DIR=/a/b works");
-    unsetenv ("FLUX_CONF_DIR");
-
-    /* FLUX_CONF_DIR="installed"
-     */
-    setenv ("FLUX_CONF_DIR", "installed", 1);
-    cf_path = flux_conf_builtin_get ("cf_path", FLUX_CONF_INSTALLED);
-    (void)snprintf (exp, sizeof (exp), "%s/*.toml", cf_path);
-    ok (conf_get_default_pattern (buf, sizeof (buf)) == 0
-        && !strcmp (buf, exp),
-        "conf_get_default_pattern FLUX_CONF_DIR=installed works");
-    unsetenv ("FLUX_CONF_DIR");
-
-    /* Tiny buffer fails
-     */
-    errno = 0;
-    ok (conf_get_default_pattern (toosmall, sizeof (toosmall)) < 0
-        && errno == EOVERFLOW,
-        "conf_get_default_pattern bufsz=1 failed with EOVERFLOW");
-
-}
-
 void test_in_handle (void)
 {
     const char *tmpdir = getenv ("TMPDIR");
     char dir[PATH_MAX + 1];
     char path[PATH_MAX + 1];
-    char invalid[PATH_MAX + 1];
-    const flux_conf_t *conf;
-    flux_conf_error_t error;
+    flux_conf_t *conf;
     flux_t *h;
     int i;
 
     /* create test handle
      */
-    if (!(h = loopback_create (0)))
-        BAIL_OUT ("loopback_create failed");
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("failed to create loop handle");
 
     /* create test config
      */
     snprintf (dir, sizeof (dir), "%s/cf.XXXXXXX", tmpdir ? tmpdir : "/tmp");
     if (!mkdtemp (dir))
         BAIL_OUT ("mkdtemp %s: %s", dir, strerror (errno));
-    create_test_file (dir, "foo", path, sizeof (path), t1);
-
-    setenv ("FLUX_CONF_DIR", dir, 1);
-    conf = flux_get_conf (h, NULL);
-    unsetenv ("FLUX_CONF_DIR");
-    ok (conf != NULL,
+    create_test_file (dir, "foo", "toml", path, sizeof (path), t1);
+    if (!(conf = flux_conf_parse (dir, NULL)))
+        BAIL_OUT ("flux_conf_parse failure: %s", strerror (errno));
+    ok (flux_set_conf (h, conf) == 0,
+        "flux_set_conf works");
+    ok (flux_get_conf (h) == conf,
         "flux_get_conf works");
 
     /* quick spot check content
@@ -354,19 +391,12 @@ void test_in_handle (void)
     ok (flux_conf_unpack (conf, NULL, "{s:i}", "i", &i) == 0 && i == 1,
         "and config content is as expected");
 
-    /* add invalid toml, reset handle config, and get again (should fail)
-     */
-    create_test_file (dir, "99", invalid, sizeof (invalid), "key = \n");
-    ok (handle_set_conf (h, NULL) == 0,
-        "clearing flux_t handle conf cache works");
-    setenv ("FLUX_CONF_DIR", dir, 1);
-    conf = flux_get_conf (h, &error);
-    unsetenv ("FLUX_CONF_DIR");
-    ok (conf == NULL,
-        "flux_get_conf fails on bad TOML");
-    diag ("%s: %d: %s", error.filename, error.lineno, error.errbuf);
+    ok (flux_set_conf (h, NULL) == 0,
+        "flux_set_conf conf=NULL works");
+    ok (flux_get_conf (h) == NULL,
+        "flux_get_conf now returns NULL");
 
-    if (unlink (path) < 0 || unlink (invalid) < 0)
+    if (unlink (path) < 0)
         BAIL_OUT ("unlink: %s", strerror (errno));
     if (rmdir (dir) < 0)
         BAIL_OUT ("rmdir: %s: %s", dir, strerror (errno));
@@ -376,42 +406,34 @@ void test_in_handle (void)
 
 void test_globerr (void)
 {
-    flux_conf_error_t error;
+    flux_error_t error;
 
     errno = 0;
     memset (&error, 0, sizeof (error));
     conf_globerr (&error, "meep", GLOB_NOMATCH);
     ok (errno == ENOENT
-        && !strcmp (error.filename, "meep")
-        && !strcmp (error.errbuf, "No match")
-        && error.lineno == -1,
+        && streq (error.text, "meep: No match"),
         "conf_globerr pat=meep rc=NOMATCH sets errno and error as expected");
 
     errno = 0;
     memset (&error, 0, sizeof (error));
     conf_globerr (&error, "moo", GLOB_NOSPACE);
     ok (errno == ENOMEM
-        && !strcmp (error.filename, "moo")
-        && !strcmp (error.errbuf, "Out of memory")
-        && error.lineno == -1,
+        && streq (error.text, "moo: Out of memory"),
         "conf_globerr pat=moo rc=NOSPACE sets errno and error as expected");
 
     errno = 0;
     memset (&error, 0, sizeof (error));
     conf_globerr (&error, "foo", GLOB_ABORTED);
     ok (errno == EINVAL
-        && !strcmp (error.filename, "foo")
-        && !strcmp (error.errbuf, "Read error")
-        && error.lineno == -1,
+        && streq (error.text, "foo: Read error"),
         "conf_globerr pat=moo rc=ABORTED sets errno and error as expected");
 
     errno = 0;
     memset (&error, 0, sizeof (error));
     conf_globerr (&error, "oops", 666);
     ok (errno == EINVAL
-        && !strcmp (error.filename, "oops")
-        && !strcmp (error.errbuf, "Unknown glob error")
-        && error.lineno == -1,
+        && streq (error.text, "oops: Unknown glob error"),
         "conf_globerr pat=oops rc=666 sets errno and error as expected");
 }
 
@@ -422,8 +444,7 @@ int main (int argc, char *argv[])
     unsetenv ("FLUX_CONF_DIR");
 
     test_builtin ();
-    test_basic (); // conf_parse(), conf_destroy(), flux_conf_unpack()
-    test_default_pattern ();
+    test_basic (); // flux_conf_parse(), flux_conf_decref(), flux_conf_unpack()
     test_in_handle ();
     test_globerr ();
 
diff --git a/src/common/libflux/test/disconnect.c b/src/common/libflux/test/disconnect.c
new file mode 100644
index 000000000000..e14c0b4105b4
--- /dev/null
+++ b/src/common/libflux/test/disconnect.c
@@ -0,0 +1,242 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <string.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+
+flux_msg_t *create_request (int sender,
+                            uint32_t rolemask,
+                            uint32_t userid,
+                            uint32_t matchtag)
+{
+    char id[32];
+    flux_msg_t *msg;
+
+    if (!(msg = flux_request_encode ("foo", NULL)))
+        return NULL;
+    snprintf (id, sizeof (id), "%d", sender);
+    if (flux_msg_route_push (msg, id) < 0)
+        return NULL;
+    if (flux_msg_set_rolemask (msg, rolemask) < 0)
+        return NULL;
+    if (flux_msg_set_userid (msg, userid) < 0)
+        return NULL;
+    if (flux_msg_set_matchtag (msg, matchtag) < 0)
+        return NULL;
+    return msg;
+}
+
+flux_msg_t *create_cancel (int sender,
+                           uint32_t rolemask,
+                           uint32_t userid,
+                           uint32_t matchtag)
+{
+    char id[32];
+    flux_msg_t *msg;
+
+    if (!(msg = flux_request_encode ("foo", NULL)))
+        return NULL;
+    snprintf (id, sizeof (id), "%d", sender);
+    if (flux_msg_route_push (msg, id) < 0)
+        return NULL;
+    if (flux_msg_set_rolemask (msg, rolemask) < 0)
+        return NULL;
+    if (flux_msg_set_userid (msg, userid) < 0)
+        return NULL;
+    if (flux_msg_pack (msg, "{s:i}", "matchtag", matchtag) < 0)
+        return NULL;
+    return msg;
+}
+
+void check_disconnect (void)
+{
+    struct flux_msglist *l;
+    int i;
+    flux_msg_t *msg;
+    int count;
+
+    /* populate list of requests with unique senders
+     */
+    if (!(l = flux_msglist_create ()))
+        BAIL_OUT ("flux_msglist_create failed");
+    for (i = 0; i < 8; i++) {
+        if (!(msg = create_request (i, 0, i, 0)))
+            BAIL_OUT ("could not create test message");
+        if (flux_msglist_append (l, msg) < 0)
+            BAIL_OUT ("flux_msglist_append failed");
+        flux_msg_decref (msg);
+    }
+    ok (flux_msglist_count (l) == 8,
+        "msglist contains 8 messages");
+
+    /* disconnect the first four requests
+     * The disconnect request will be sent by the same sender and user.
+     */
+    for (i = 0; i < 4; i++) {
+        if (!(msg = create_request (i, FLUX_ROLE_USER, i, 0)))
+            BAIL_OUT ("could not create disconnect message");
+        count = flux_msglist_disconnect (l, msg);
+        ok (count == 1,
+            "flux_msglist_disconnect removed message");
+        flux_msg_decref (msg);
+    }
+    ok (flux_msglist_count (l) == 4,
+        "msglist contains 4 messages");
+
+    /* sender doesn't match
+     */
+    if (!(msg = create_request (42, FLUX_ROLE_USER, 4, 0)))
+        BAIL_OUT ("could not create disconnect message");
+    count = flux_msglist_disconnect (l, msg);
+    ok (count == 0,
+        "flux_msglist_disconnect with unknown sender has no effect");
+    flux_msg_decref (msg);
+    ok (flux_msglist_count (l) == 4,
+        "msglist contains 4 messages");
+
+    /* FLUX_ROLE_USER and non-matching userid
+     */
+    if (!(msg = create_request (4, FLUX_ROLE_USER, 5, 0)))
+        BAIL_OUT ("could not create disconnect message");
+    count = flux_msglist_disconnect (l, msg);
+    ok (count == 0,
+        "flux_msglist_disconnect (user) with wrong userid has no effect");
+    flux_msg_decref (msg);
+    ok (flux_msglist_count (l) == 4,
+        "msglist contains 4 messages");
+
+    /* FLUX_ROLE_OWNER and non-matching userid
+     */
+    if (!(msg = create_request (4, FLUX_ROLE_OWNER, 5, 0)))
+        BAIL_OUT ("could not create disconnect message");
+    count = flux_msglist_disconnect (l, msg);
+    ok (count == 1,
+        "flux_msglist_disconnect (owner) with wrong userid removed message");
+    flux_msg_decref (msg);
+    ok (flux_msglist_count (l) == 3,
+        "msglist contains 3 messages");
+
+    flux_msglist_destroy (l);
+}
+
+void check_cancel (void)
+{
+    flux_t *h;
+    struct flux_msglist *l;
+    int i;
+    flux_msg_t *msg;
+    int count;
+    uint32_t matchtag;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("failed to create loop handle");
+
+    /* populate list of requests with unique senders
+     */
+    if (!(l = flux_msglist_create ()))
+        BAIL_OUT ("flux_msglist_create failed");
+    for (i = 1; i < 8; i++) {
+        if (!(msg = create_request (i, 0, i, i)))
+            BAIL_OUT ("could not create test message");
+        if (flux_msglist_append (l, msg) < 0)
+            BAIL_OUT ("flux_msglist_append failed");
+        flux_msg_decref (msg);
+    }
+    ok (flux_msglist_count (l) == 7,
+        "msglist contains 7 messages");
+
+    /* cancel the first three requests
+     */
+    int failures = 0;
+    for (i = 1; i < 4; i++) {
+        if (!(msg = create_cancel (i, FLUX_ROLE_USER, i, i)))
+            BAIL_OUT ("could not create cancel message");
+        count = flux_msglist_cancel (h, l, msg);
+        flux_msg_decref (msg);
+        if (count != 1) {
+            failures++;
+            continue;
+        }
+        if (!(msg = flux_recv (h, FLUX_MATCH_ANY, FLUX_O_NONBLOCK))) {
+            failures++;
+            continue;
+        }
+        if (flux_msg_get_matchtag (msg, &matchtag) < 0 || i != matchtag)
+            failures++;
+        flux_msg_decref (msg);
+    }
+    ok (failures == 0,
+        "flux_msglist_cancel canceled 3 messages");
+    ok (flux_msglist_count (l) == 4,
+        "msglist contains 4 messages");
+
+    /* sender doesn't match
+     */
+    if (!(msg = create_cancel (42, FLUX_ROLE_USER, 4, 4)))
+        BAIL_OUT ("could not create cancel message");
+    count = flux_msglist_cancel (h, l, msg);
+    ok (count == 0,
+        "flux_msglist_cancel with unknown sender has no effect");
+    flux_msg_decref (msg);
+    ok (flux_msglist_count (l) == 4,
+        "msglist contains 4 messages");
+
+    /* FLUX_ROLE_USER and non-matching userid
+     */
+    if (!(msg = create_cancel (4, FLUX_ROLE_USER, 5, 4)))
+        BAIL_OUT ("could not create cancel message");
+    count = flux_msglist_cancel (h, l, msg);
+    ok (count == 0,
+        "flux_msglist_cancel (user) with wrong userid has no effect");
+    flux_msg_decref (msg);
+    ok (flux_msglist_count (l) == 4,
+        "msglist contains 4 messages");
+
+    /* FLUX_ROLE_OWNER and non-matching userid
+     */
+    if (!(msg = create_cancel (6, FLUX_ROLE_OWNER, 5, 6)))
+        BAIL_OUT ("could not create cancel message");
+    count = flux_msglist_cancel (h, l, msg);
+    ok (count == 1,
+        "flux_msglist_cancel (owner) with wrong userid removed message");
+    flux_msg_decref (msg);
+    ok (flux_msglist_count (l) == 3,
+        "msglist contains 3 messages");
+    ok ((msg = flux_recv (h, FLUX_MATCH_ANY, FLUX_O_NONBLOCK)) != NULL
+       && flux_msg_get_matchtag (msg, &matchtag) == 0
+       && matchtag == 6,
+       "flux_msglist_cancel responded to message");
+    flux_msg_destroy (msg);
+
+    flux_msglist_destroy (l);
+    flux_close (h);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    check_disconnect ();
+    check_cancel ();
+
+    done_testing();
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libflux/test/dispatch.c b/src/common/libflux/test/dispatch.c
index b4258dc93587..73fd2084487a 100644
--- a/src/common/libflux/test/dispatch.c
+++ b/src/common/libflux/test/dispatch.c
@@ -8,13 +8,14 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libtap/tap.h"
-#include "src/common/libtestutil/util.h"
 
 int cb2_called;
 void cb2 (flux_t *h, flux_msg_handler_t *mh, const flux_msg_t *msg, void *arg)
@@ -112,8 +113,8 @@ void test_fastpath (flux_t *h)
     ok (cb_called == 1,
         "message handler was called after being started");
 
-    ok (flux_msg_enable_route (msg) == 0
-        && flux_msg_push_route (msg, "myuuid") == 0,
+    flux_msg_route_enable (msg);
+    ok (flux_msg_route_push (msg, "myuuid") == 0,
         "added route to message");
     ok (flux_send (h, msg, 0) == 0,
         "sent response message on loop connector");
@@ -133,7 +134,7 @@ void test_fastpath (flux_t *h)
 }
 
 /* Verify that a non-glob request handler overrides earlier-registered one.
- * "Built-in" methods like "ping" should be overrideable in comms modules.
+ * "Built-in" methods like "ping" should be overridable.
  */
 void test_method_override (flux_t *h)
 {
@@ -172,12 +173,61 @@ void test_method_override (flux_t *h)
     diag ("%d %d", cb_called, cb2_called);
     ok (cb_called == 0 && cb2_called == 1,
         "first handler not called, second handler called");
-
     flux_future_destroy (f);
+
+    /* now remove mh2 and ensure old mh is reinstated */
     flux_msg_handler_destroy (mh2);
+
+    cb_called = 0;
+    cb2_called = 0;
+
+    /* send message - who got it?
+     * N.B. The test doesn't generate a response so just destroy f
+     * after message is sent.
+     */
+    f = flux_rpc (h, "foo.bar", NULL, FLUX_NODEID_ANY, 0);
+    ok (f != NULL,
+        "sent foo.bar RPC");
+    rc = flux_reactor_run (flux_get_reactor (h), FLUX_REACTOR_NOWAIT);
+    ok (rc >= 0,
+        "flux_reactor_run NOWAIT ran");
+    diag ("%d %d", cb_called, cb2_called);
+    ok (cb_called == 1 && cb2_called == 0,
+        "first handler called, second handler not called");
+    flux_future_destroy (f);
+
+    /* override foo.bar request handler again */
+    match.topic_glob = "foo.bar";
+    mh2 = flux_msg_handler_create (h, match, cb2, NULL);
+    flux_msg_handler_start (mh2);
+    ok (mh2 != NULL,
+        "foo.bar second request handler created and started");
+
+    /* now remove original message handler *after* override added */
     flux_msg_handler_destroy (mh);
+    diag ("removed first message handler");
+
+    cb_called = 0;
+    cb2_called = 0;
+
+    /* send message - who got it?
+     * N.B. The test doesn't generate a response so just destroy f
+     * after message is sent.
+     */
+    f = flux_rpc (h, "foo.bar", NULL, FLUX_NODEID_ANY, 0);
+    ok (f != NULL,
+        "sent foo.bar RPC");
+    rc = flux_reactor_run (flux_get_reactor (h), FLUX_REACTOR_NOWAIT);
+    ok (rc >= 0,
+        "flux_reactor_run NOWAIT ran");
+    diag ("%d %d", cb_called, cb2_called);
+    ok (cb_called == 0 && cb2_called == 1,
+        "first handler not called, second handler called");
+    flux_future_destroy (f);
+    flux_msg_handler_destroy (mh2);
 }
 
+
 /* Verify that a request handler for a specific method is matched before
  * one for a glob.  A "router" should be able to register a catch-all
  * request handler that doesn't override its own service methods.
@@ -237,11 +287,13 @@ void test_response_catchall (flux_t *h)
     /* craft response message with valid matchtag */
     if ((mtag = flux_matchtag_alloc (h)) == FLUX_MATCHTAG_NONE)
         BAIL_OUT ("flux_matchtag_alloc failed");
-    msg = flux_response_encode ("foo.bar", NULL);
+    msg = flux_response_encode ("baz.fop", NULL);
     ok (msg != NULL,
-        "foo.bar RPC response created");
+        "baz.fop RPC response created");
     if (flux_msg_set_matchtag (msg, mtag) < 0)
         BAIL_OUT ("flux_msg_set_matchtag failed");
+    ok (flux_msg_cmp (msg, match),
+        "RPC response matches match object");
 
     /* register RPC response handler */
     match.matchtag = mtag;
@@ -249,14 +301,14 @@ void test_response_catchall (flux_t *h)
         BAIL_OUT ("flux_msg_handler_create");
     flux_msg_handler_start (mh);
     ok (mh != NULL,
-        "foo.bar RPC response handler created and started");
+        "baz.fop RPC response handler created and started mh=%p", mh);
 
     /* register catchall response handler */
     match.matchtag = FLUX_MATCHTAG_NONE;
     mh2 = flux_msg_handler_create (h, match, cb2, NULL);
     flux_msg_handler_start (mh2);
     ok (mh2 != NULL,
-        "catchall response handler created and started");
+        "catchall response handler created and started mh=%p", mh2);
 
     cb_called = 0;
     cb2_called = 0;
@@ -264,12 +316,13 @@ void test_response_catchall (flux_t *h)
     /* send message to method - who got it?
      */
     ok (flux_send (h, msg, 0) == 0,
-        "sent foo.bar response message on loop connector");
+        "sent baz.fop response message on loop connector");
     rc = flux_reactor_run (flux_get_reactor (h), FLUX_REACTOR_NOWAIT);
     ok (rc >= 0,
         "flux_reactor_run NOWAIT ran");
     ok (cb_called == 1 && cb2_called == 0,
-        "RPC response handler called, catchall not called");
+        "RPC response handler called, catchall not called (%d, %d)",
+        cb_called, cb2_called);
 
     flux_matchtag_free (h, mtag);
     flux_msg_destroy (msg);
@@ -297,8 +350,9 @@ void test_response_with_routes (flux_t *h)
         "foo.bar RPC response created");
     if (flux_msg_set_matchtag (msg, mtag) < 0)
         BAIL_OUT ("flux_msg_set_matchtag failed");
-    if (flux_msg_enable_route (msg) < 0 || flux_msg_push_route (msg, "9") < 0)
-        BAIL_OUT ("flux_msg_enable/push_route failed");
+    flux_msg_route_enable (msg);
+    if (flux_msg_route_push (msg, "9") < 0)
+        BAIL_OUT ("flux_msg_route_push failed");
 
     /* register RPC response handler */
     match.matchtag = mtag;
@@ -469,8 +523,8 @@ int main (int argc, char *argv[])
 
     plan (NO_PLAN);
 
-    if (!(h = loopback_create (0)))
-        BAIL_OUT ("can't continue without loopback handle");
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("can't continue without loop handle");
     ok ((r = flux_get_reactor (h)) != NULL,
         "handle created reactor on demand");
 
diff --git a/src/common/libflux/test/event.c b/src/common/libflux/test/event.c
index 673218a47720..6aa6fea5d541 100644
--- a/src/common/libflux/test/event.c
+++ b/src/common/libflux/test/event.c
@@ -8,12 +8,16 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include <czmq.h>
-#include "message.h"
-#include "event.h"
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
 #include "src/common/libtap/tap.h"
+#include "src/common/libtestutil/util.h"
+#include "ccan/str/str.h"
 
-int main (int argc, char *argv[])
+void test_codec (void)
 {
     flux_msg_t *msg;
     const char *topic, *s;
@@ -21,10 +25,8 @@ int main (int argc, char *argv[])
     const char data[] = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
     int len = strlen (data);
     const void *d;
-    int l;
-    int i;
-
-    plan (NO_PLAN);
+    size_t l;
+    size_t i;
 
     /* no topic is an error */
     errno = 0;
@@ -41,7 +43,7 @@ int main (int argc, char *argv[])
 
     topic = NULL;
     ok (flux_event_decode (msg, &topic, NULL) == 0
-        && topic != NULL && !strcmp (topic, "foo.bar"),
+        && topic != NULL && streq (topic, "foo.bar"),
         "flux_event_decode returns encoded topic");
     ok (flux_event_decode (msg, NULL, NULL) == 0,
         "flux_event_decode topic is optional");
@@ -56,7 +58,7 @@ int main (int argc, char *argv[])
 
     s = NULL;
     ok (flux_event_decode (msg, NULL, &s) == 0
-        && s != NULL && !strcmp (s, json_str),
+        && s != NULL && streq (s, json_str),
         "flux_event_decode returns encoded payload");
     errno = 0;
     ok (flux_event_decode (msg, NULL, NULL) == 0,
@@ -69,7 +71,7 @@ int main (int argc, char *argv[])
     i = 0;
     ok (flux_event_unpack (msg, &topic, "{s:i}", "foo", &i) == 0,
         "flux_event_unpack unpacked payload object");
-    ok (i == 42 && topic != NULL && !strcmp (topic, "foo.bar"),
+    ok (i == 42 && topic != NULL && streq (topic, "foo.bar"),
         "unpacked payload matched packed");
     flux_msg_destroy (msg);
 
@@ -80,7 +82,7 @@ int main (int argc, char *argv[])
     l = 0;
     topic = NULL;
     ok (flux_event_decode_raw (msg, &topic, &d, &l) == 0
-        && topic != NULL && strcmp (topic, "foo.bar") == 0
+        && topic != NULL && streq (topic, "foo.bar")
         && d != NULL && len == len && memcmp (d, data, len) == 0,
         "flux_event_decode_raw returns encoded topic and payload");
     ok (flux_event_decode_raw (msg, NULL, &d, &l) == 0
@@ -94,6 +96,211 @@ int main (int argc, char *argv[])
     ok (flux_event_decode_raw (msg, NULL, &d, NULL) < 0 && errno == EINVAL,
         "flux_event_decode_raw len=NULL fails with EINVAL");
     flux_msg_destroy (msg);
+}
+
+void test_subscribe_badparams (void)
+{
+    flux_t *h;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop handle");
+
+    errno = 0;
+    ok (flux_event_subscribe_ex (NULL, "foo", 0) == NULL && errno == EINVAL,
+        "flux_event_subscribe_ex h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_event_subscribe_ex (h, NULL, 0) == NULL && errno == EINVAL,
+        "flux_event_subscribe_ex topic=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_event_subscribe_ex (h, "foo", -1) == NULL && errno == EINVAL,
+        "flux_event_subscribe_ex flags=-1 fails with EINVAL");
+
+    errno = 0;
+    ok (flux_event_unsubscribe_ex (NULL, "foo", 0) == NULL && errno == EINVAL,
+        "flux_event_unsubscribe_ex h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_event_unsubscribe_ex (h, NULL, 0) == NULL && errno == EINVAL,
+        "flux_event_unsubscribe_ex topic=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_event_unsubscribe_ex (h, "foo", -1) == NULL && errno == EINVAL,
+        "flux_event_unsubscribe_ex flags=-1 fails with EINVAL");
+
+    errno = 0;
+    ok (flux_event_subscribe (NULL, "foo") < 0 && errno == EINVAL,
+        "flux_event_subscribe h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_event_subscribe (h, NULL) < 0 && errno == EINVAL,
+        "flux_event_subscribe topic=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (flux_event_unsubscribe (NULL, "foo") < 0 && errno == EINVAL,
+        "flux_event_unsubscribe h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_event_unsubscribe (h, NULL) < 0 && errno == EINVAL,
+        "flux_event_unsubscribe topic=NULL fails with EINVAL");
+
+    flux_close (h);
+}
+
+bool fake_failure;
+
+void subscribe_cb (flux_t *h, flux_msg_handler_t *mh,
+                   const flux_msg_t *msg, void *arg)
+{
+    const char *topic = NULL;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "topic", &topic) < 0)
+        goto error;
+    diag ("subscribe %s", topic);
+    if (fake_failure) {
+        errno = EIO;
+        fake_failure = false;
+        goto error;
+    }
+    if (!flux_msg_is_noresponse (msg)
+        && flux_respond (h, msg, NULL) < 0)
+        diag ("error responding to subscribe request");
+    return;
+error:
+    if (!flux_msg_is_noresponse (msg)
+        && flux_respond_error (h, msg, errno, NULL) < 0)
+        diag ("error responding to subscribe request: %s", strerror (errno));
+}
+
+void unsubscribe_cb (flux_t *h, flux_msg_handler_t *mh,
+                     const flux_msg_t *msg, void *arg)
+{
+    const char *topic = NULL;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "topic", &topic) < 0)
+        goto error;
+    diag ("unsubscribe %s", topic);
+    if (fake_failure) {
+        errno = EIO;
+        fake_failure = false;
+        goto error;
+    }
+    if (!flux_msg_is_noresponse (msg)
+        && flux_respond (h, msg, NULL) < 0)
+        diag ("error responding to unsubscribe request");
+    return;
+error:
+    if (!flux_msg_is_noresponse (msg)
+        && flux_respond_error (h, msg, errno, NULL) < 0)
+        diag ("error responding to unsubscribe request: %s", strerror (errno));
+}
+
+const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,   "event.subscribe",    subscribe_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST,   "event.unsubscribe",  unsubscribe_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+int test_server (flux_t *h, void *arg)
+{
+    flux_msg_handler_t **handlers = NULL;
+    int rc = -1;
+
+    if (flux_msg_handler_addvec (h, htab, NULL, &handlers) < 0) {
+        diag ("flux_msg_handler_addvec failed");
+        return -1;
+    }
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
+        diag ("flux_reactor_run failed");
+        goto done;
+    }
+    rc = 0;
+done:
+    flux_msg_handler_delvec (handlers);
+    return rc;
+}
+
+void test_subscribe_rpc (void)
+{
+    flux_t *h;
+    flux_future_t *f;
+
+    if (!(h = test_server_create (0, test_server, NULL)))
+        BAIL_OUT ("test_server_create: %s", strerror (errno));
+
+    ok (flux_event_subscribe (h, "fubar") == 0,
+        "flux_event_subscribe topic=FUBAR works");
+
+    ok (flux_event_unsubscribe (h, "fubar") == 0,
+        "flux_event_unsubscribe topic=FUBAR works");
+
+    fake_failure = true;
+    errno = 0;
+    ok (flux_event_subscribe (h, "fubar") < 0 && errno == EIO,
+        "flux_event_subscribe failure works");
+
+    fake_failure = true;
+    errno = 0;
+    ok (flux_event_unsubscribe (h, "fubar") < 0 && errno == EIO,
+        "flux_event_unsubscribe failure works");
+
+    f = flux_event_subscribe_ex (h, "fubar", FLUX_RPC_NORESPONSE);
+    ok (f != NULL,
+        "flux_event_subscribe_ex flags=FLUX_RPC_NORESPONSE works");
+    flux_future_destroy (f);
+
+    f = flux_event_unsubscribe_ex (h, "fubar", FLUX_RPC_NORESPONSE);
+    ok (f != NULL,
+        "flux_event_unsubscribe_ex flags=FLUX_RPC_NORESPONSE works");
+    flux_future_destroy (f);
+
+    f = flux_event_subscribe_ex (h, "fubar", 0);
+    ok (f && flux_future_get (f, NULL) == 0,
+        "flux_event_subscribe_ex works");
+    flux_future_destroy (f);
+
+    f = flux_event_unsubscribe_ex (h, "fubar", 0);
+    ok (f && flux_future_get (f, NULL) == 0,
+        "flux_event_unsubscribe_ex works");
+    flux_future_destroy (f);
+
+    fake_failure = true;
+    errno = 0;
+    f = flux_event_subscribe_ex (h, "fubar", 0);
+    ok (f && flux_future_get (f, NULL) < 0 && errno == EIO,
+        "flux_event_subscribe_ex failure works");
+    flux_future_destroy (f);
+
+    fake_failure = true;
+    errno = 0;
+    f = flux_event_unsubscribe_ex (h, "fubar", 0);
+    ok (f && flux_future_get (f, NULL) < 0 && errno == EIO,
+        "flux_event_unsubscribe_ex failure works");
+    flux_future_destroy (f);
+
+    if (test_server_stop (h) < 0)
+        BAIL_OUT ("error stopping test server: %s", strerror (errno));
+    flux_close (h);
+}
+
+void test_subscribe_nosub (void)
+{
+    flux_t *h;
+
+    if (!(h = flux_open ("loop://", FLUX_O_TEST_NOSUB)))
+        BAIL_OUT ("could not create loop handle");
+
+    ok (flux_event_subscribe (h, "foo") == 0,
+        "flux_event_subscribe succeeds in loopback with TEST_NOSUB flag");
+    ok (flux_event_unsubscribe (h, "foo") == 0,
+        "flux_event_unsubscribe succeeds in loopback with TEST_NOSUB flag");
+
+    flux_close (h);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_codec ();
+    test_subscribe_badparams ();
+    test_subscribe_rpc ();
+    test_subscribe_nosub ();
 
     done_testing();
     return (0);
diff --git a/src/common/libflux/test/future.c b/src/common/libflux/test/future.c
index 4f67536499b3..3d01b624e2f8 100644
--- a/src/common/libflux/test/future.c
+++ b/src/common/libflux/test/future.c
@@ -8,17 +8,22 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
+#include <assert.h>
 #include <string.h>
-#include <czmq.h>
+#include <flux/core.h>
 
-#include "future.h"
-#include "reactor.h"
-#include "response.h"
+#ifndef EDEADLOCK
+#define EDEADLOCK EDEADLK
+#endif
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/xzmalloc.h"
-#include "src/common/libtestutil/util.h"
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 int aux_destroy_called;
 void *aux_destroy_arg;
@@ -101,7 +106,7 @@ void test_simple (void)
         && errno == ENOENT,
         "flux_future_aux_get of wrong value returns ENOENT");
     p = flux_future_aux_get (f, "foo");
-    ok (p != NULL && !strcmp (p, "bar"),
+    ok (p != NULL && streq (p, "bar"),
         "flux_future_aux_get of known returns it");
     // same value as "foo" key to not muck up destructor arg test
     ok (flux_future_aux_set (f, NULL, "bar", aux_destroy_fun) == 0,
@@ -117,16 +122,21 @@ void test_simple (void)
     ok (!flux_future_is_ready (f),
         "flux_future_is_ready returns false");
     errno = 0;
+    ok (flux_future_wait_for (f, -1.0) < 0 && errno == EDEADLOCK,
+        "flux_future_wait_for fails with EDEADLOCK with timeout < 0");
+    ok (!flux_future_is_ready (f),
+        "flux_future_is_ready returns false");
+    errno = 0;
     const void *result = NULL;
     result_destroy_called = 0;
     result_destroy_arg = NULL;
     flux_future_fulfill (f, "Hello", result_destroy);
     ok (flux_future_wait_for (f, 0.) == 0,
-        "flux_future_wait_for succedes after result is set");
+        "flux_future_wait_for succeeds after result is set");
     ok (flux_future_is_ready (f),
         "flux_future_is_ready returns true after result is set");
     ok (flux_future_get (f, &result) == 0
-        && result != NULL && !strcmp (result, "Hello"),
+        && result != NULL && streq (result, "Hello"),
         "flux_future_get returns correct result");
     ok (flux_future_get (f, NULL) == 0,
         "flux_future_get with NULL results argument also works");
@@ -145,10 +155,10 @@ void test_simple (void)
         "flux_future_then registered continuation");
     ok (flux_reactor_run (r, 0) == 0,
         "reactor ran successfully");
-    ok (contin_called && contin_arg != NULL && !strcmp (contin_arg, "nerp"),
+    ok (contin_called && contin_arg != NULL && streq (contin_arg, "nerp"),
         "continuation was called with correct argument");
     ok (contin_get_rc == 0 && contin_get_result != NULL
-        && !strcmp (contin_get_result, "Hello"),
+        && streq (contin_get_result, "Hello"),
         "continuation obtained correct result with flux_future_get");
     ok (contin_reactor == r,
         "flux_future_get_reactor from continuation returned set reactor");
@@ -156,10 +166,10 @@ void test_simple (void)
     /* destructors */
     flux_future_destroy (f);
     ok (aux_destroy_called == 2 && aux_destroy_arg != NULL
-        && !strcmp (aux_destroy_arg, "bar"),
+        && streq (aux_destroy_arg, "bar"),
         "flux_future_destroy called aux destructor correctly");
     ok (result_destroy_called && result_destroy_arg != NULL
-        && !strcmp (result_destroy_arg, "Hello"),
+        && streq (result_destroy_arg, "Hello"),
         "flux_future_destroy called result destructor correctly");
 
     flux_reactor_destroy (r);
@@ -221,6 +231,56 @@ void test_timeout_then (void)
     diag ("%s: timeout works in reactor context", __FUNCTION__);
 }
 
+void timeout_reset_contin (flux_future_t *f, void *arg)
+{
+    int *count = arg;
+
+    diag ("%s: count=%d\n", __FUNCTION__, *count);
+
+    /* Expecting ETIMEDOUT every time
+     */
+    if (flux_future_get (f, NULL) == 0)
+        errno = EPROTO; // fall through
+    if (errno != ETIMEDOUT)
+        goto error;
+
+    if (--(*count) > 0)
+        flux_future_reset (f);
+    else
+        flux_reactor_stop (flux_future_get_reactor (f));
+    return;
+error:
+    flux_reactor_stop_error (flux_future_get_reactor (f));
+}
+
+void test_timeout_then_reset (void)
+{
+    flux_future_t *f;
+    flux_reactor_t *r;
+    int count = 10;
+
+    r = flux_reactor_create (0);
+    if (!r)
+        BAIL_OUT ("flux_reactor_create failed");
+
+    f = flux_future_create (NULL, NULL);
+    if (!f)
+        BAIL_OUT ("could not create future");
+    flux_future_set_reactor (f, r);
+
+    ok (flux_future_then (f, 0.01, timeout_reset_contin, &count) == 0,
+        "flux_future_then registered continuation with timeout");
+    ok (flux_reactor_run (r, 0) == 0,
+        "reactor ran successfully");
+    ok (count == 0,
+        "future timed out the expected number of times");
+
+    flux_future_destroy (f);
+    flux_reactor_destroy (r);
+
+    diag ("%s: timeout with reset works in reactor context", __FUNCTION__);
+}
+
 void simple_init_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
                               int revents, void *arg)
 {
@@ -270,10 +330,10 @@ void test_init_now (void)
     result = NULL;
     ok (flux_future_get (f, (const void **)&result) == 0,
         "flux_future_get worked");
-    ok (result != NULL && !strcmp (result, "Result!"),
+    ok (result != NULL && streq (result, "Result!"),
         "and correct result was returned");
     ok (simple_init_called == 1 && simple_init_arg != NULL
-        && !strcmp (simple_init_arg, "testarg"),
+        && streq (simple_init_arg, "testarg"),
         "init was called once with correct arg");
     ok (simple_init_r != NULL,
         "flux_future_get_reactor returned tmp reactor in init");
@@ -316,7 +376,7 @@ void test_init_then (void)
     ok (flux_future_then (f, -1., simple_contin, NULL) == 0,
         "flux_future_then registered continuation");
     ok (simple_init_called == 1 && simple_init_arg != NULL
-        && !strcmp (simple_init_arg, "testarg"),
+        && streq (simple_init_arg, "testarg"),
         "init was called once with correct arg");
     ok (simple_init_r == r,
         "flux_future_get_reactor return set reactor in init");
@@ -327,7 +387,7 @@ void test_init_then (void)
     ok (simple_contin_rc == 0,
         "continuation get succeeded");
     ok (simple_contin_result != NULL
-        && !strcmp (simple_contin_result, "Result!"),
+        && streq (simple_contin_result, "Result!"),
         "continuation get returned correct result");
 
     flux_future_destroy (f);
@@ -381,7 +441,8 @@ void test_fclass_synchronous (char *tag, flux_future_t *f, const char *expected)
 
     ok (flux_future_wait_for (f, -1.) == 0,
         "%s: flux_future_wait_for returned successfully", tag);
-    ok (flux_future_get (f, (const void **)&s) == 0 && s != NULL && !strcmp (s, expected),
+    ok (flux_future_get (f, (const void **)&s) == 0
+        && s != NULL && streq (s, expected),
         "%s: flux_future_get worked", tag);
 }
 
@@ -404,7 +465,7 @@ void test_fclass_asynchronous (char *tag,
         "%s: flux_reactor_run returned", tag);
     ok (fclass_contin_rc == 0,
         "%s: continuation called flux_future_get with success", tag);
-    ok (s != NULL && !strcmp (s, expected),
+    ok (s != NULL && streq (s, expected),
         "%s: continuation fetched expected value", tag);
 
     flux_reactor_destroy (r);
@@ -521,7 +582,7 @@ void test_mumble_inception (void)
 }
 
 /* walk - multiple mumbles wrapped in a future, executed serially
- * The next future is created in the current future's contination.
+ * The next future is created in the current future's continuation.
  */
 struct walk {
     zlist_t *f; // stack of futures
@@ -896,7 +957,7 @@ void test_error_string (void)
     ok (flux_future_get (f, NULL) < 0
         && errno == ENOENT
         && (str = flux_future_error_string (f)) != NULL
-        && !strcmp (str, "No such file or directory"),
+        && streq (str, "No such file or directory"),
         "flux_future_error_string returns ENOENT strerror string");
 
     flux_future_destroy (f);
@@ -909,7 +970,7 @@ void test_error_string (void)
     ok (flux_future_get (f, NULL) < 0
         && errno == ENOENT
         && (str = flux_future_error_string (f)) != NULL
-        && !strcmp (str, "foobar"),
+        && streq (str, "foobar"),
         "flux_future_error_string returns correct string when error string set");
 
     flux_future_destroy (f);
@@ -922,7 +983,7 @@ void test_error_string (void)
     ok (flux_future_get (f, NULL) < 0
         && errno == ENOENT
         && (str = flux_future_error_string (f)) != NULL
-        && !strcmp (str, "No such file or directory"),
+        && streq (str, "No such file or directory"),
         "flux_future_error_string returns ENOENT strerror string");
 
     flux_future_destroy (f);
@@ -935,7 +996,7 @@ void test_error_string (void)
     ok (flux_future_get (f, NULL) < 0
         && errno == ENOENT
         && (str = flux_future_error_string (f)) != NULL
-        && !strcmp (str, "boobaz"),
+        && streq (str, "boobaz"),
         "flux_future_error_string returns correct fatal error string "
         "when error string set");
 
@@ -964,7 +1025,7 @@ void test_multiple_fulfill (void)
     result = NULL;
     ok (flux_future_get (f, (const void **)&result) == 0
         && result
-        && !strcmp (result, "foo"),
+        && streq (result, "foo"),
         "flux_future_get gets fulfillment");
     flux_future_reset (f);
 
@@ -976,7 +1037,7 @@ void test_multiple_fulfill (void)
     result = NULL;
     ok (flux_future_get (f, (const void **)&result) == 0
         && result
-        && !strcmp (result, "bar"),
+        && streq (result, "bar"),
         "flux_future_get gets queued fulfillment");
     flux_future_reset (f);
 
@@ -988,7 +1049,7 @@ void test_multiple_fulfill (void)
     result = NULL;
     ok (flux_future_get (f, (const void **)&result) == 0
         && result
-        && !strcmp (result, "baz"),
+        && streq (result, "baz"),
         "flux_future_get gets queued fulfillment");
     flux_future_reset (f);
 
@@ -1025,7 +1086,7 @@ void test_multiple_fulfill_asynchronous (void)
     /* Call continuation once to get first value and reset future */
     multiple_fulfill_continuation (f, &result);
 
-    ok (strcmp (result, "foo") == 0,
+    ok (streq (result, "foo"),
         "calling multiple_fulfill_continuation synchronously worked");
 
     rc = flux_future_then (f, -1., multiple_fulfill_continuation, &result);
@@ -1033,7 +1094,7 @@ void test_multiple_fulfill_asynchronous (void)
         "flux_future_then() works for multiple fulfilled future");
     if (flux_reactor_run (r, FLUX_REACTOR_NOWAIT) < 0)
         BAIL_OUT ("flux_reactor_run NOWAIT failed");
-    ok (strcmp (result, "bar") == 0,
+    ok (streq (result, "bar"),
         "continuation was called for multiple-fulfilled future");
 
     flux_future_destroy (f);
@@ -1082,7 +1143,7 @@ void test_fulfill_with (void)
         "flux_future_t f is now ready");
     ok (flux_future_get (f, (const void **)&result) == 0,
         "flux_future_get (f) works");
-    ok (result == p_result && strcmp (result, "result") == 0,
+    ok (result == p_result && streq (result, "result"),
         "flux_future_get (f) returns result from p");
     ok (flux_future_aux_get (f, "test") == (void *) 0x42,
         "flux_future_aux_get (f, ...) retrieves aux item from p");
@@ -1102,7 +1163,7 @@ void test_fulfill_with (void)
     ok (flux_future_get (f, NULL) < 0 && errno == EFAULT,
         "flux_future_get returns expected error and errno");
     ok (flux_future_error_string (f) &&
-        strcmp (flux_future_error_string (f), "test error string") == 0,
+        streq (flux_future_error_string (f), "test error string"),
         "flux_future_error_string() has expected error string");
 
     /* Test fulfill_with multiple fulfillment:
@@ -1149,7 +1210,7 @@ void test_fulfill_with (void)
     ok (flux_future_get (f, NULL) < 0 && errno == EFAULT,
         "flux_future_get returns expected error and errno");
     ok (flux_future_error_string (f) &&
-        strcmp (flux_future_error_string (f), "fatal error string") == 0,
+        streq (flux_future_error_string (f), "fatal error string"),
         "flux_future_error_string() has expected error string");
 
     flux_future_destroy (f);
@@ -1304,8 +1365,8 @@ void test_rpc_like_thing_async (bool reverse_destroy_order)
     struct thing *t;
     flux_reactor_t *r;
 
-    if (!(h = loopback_create (0)))
-        BAIL_OUT ("could not create loopback handle");
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop handle");
     if (!(r = flux_get_reactor (h)))
         BAIL_OUT ("flux_get_reactor failed");
 
@@ -1350,8 +1411,8 @@ void test_rpc_like_thing (bool reverse_destroy_order)
     flux_msg_t *msg;
     struct thing *t;
 
-    if (!(h = loopback_create (0)))
-        BAIL_OUT ("could not create loopback handle");
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop handle");
 
     t = thing_create (h);
 
@@ -1383,7 +1444,7 @@ void test_rpc_like_thing_unfulfilled (bool reverse_destroy_order)
     flux_t *h;
     struct thing *t;
 
-    if (!(h = loopback_create (0)))
+    if (!(h = flux_open ("loop://", 0)))
         BAIL_OUT ("could not create loopback handle");
 
     t = thing_create (h);
@@ -1405,6 +1466,7 @@ int main (int argc, char *argv[])
     test_simple ();
     test_timeout_now ();
     test_timeout_then ();
+    test_timeout_then_reset ();
 
     test_init_now ();
     test_init_then ();
diff --git a/src/common/libflux/test/handle.c b/src/common/libflux/test/handle.c
index 31d0158363c1..886898eeafb3 100644
--- a/src/common/libflux/test/handle.c
+++ b/src/common/libflux/test/handle.c
@@ -8,24 +8,18 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libtap/tap.h"
-#include "src/common/libtestutil/util.h"
-
-/* Fake handle flags for testing flux_flags_get/set/unset
- */
-enum {
-    FAKE_FLAG1 = 0x10000000,
-    FAKE_FLAG2 = 0x20000000,
-    FAKE_FLAG3 = 0x30000000,
-};
+#include "ccan/str/str.h"
 
 /* Destructor for malloc'ed string.
- * Set flag so we no this was called when aux was destroyed.
+ * Set flag so we know this was called when aux was destroyed.
  */
 static bool aux_destroyed = false;
 static void aux_free (void *arg)
@@ -34,27 +28,121 @@ static void aux_free (void *arg)
     aux_destroyed = true;
 }
 
-/* First time this is called, don't BAIL_OUT, just set fatal_tested so
- * we can verify that flux_fatal_set() sets the hook.
- * After that, abort the test if the handle suffers a fatality.
- */
-static bool fatal_tested = false;
-static void fatal_err (const char *message, void *arg)
+static int comms_err (flux_t *h, void *arg)
 {
-    if (fatal_tested)
-        BAIL_OUT ("fatal error: %s", message);
-    else
-        fatal_tested = true;
+    BAIL_OUT ("fatal comms error: %s", strerror (errno));
+    return -1;
 }
 
-void test_handle_invalid_args (void)
+void test_handle_invalid_args (flux_t *h)
 {
+    flux_msg_t *msg;
+
     errno = 0;
     ok (flux_aux_set (NULL, "foo", "bar", NULL) < 0 && errno == EINVAL,
         "flux_aux_set h=NULL fails with EINVAL");
     errno = 0;
     ok (flux_aux_get (NULL, "foo") == NULL && errno == EINVAL,
         "flux_aux_get h=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (flux_open (NULL, 0x100000) == NULL && errno == EINVAL,
+        "flux_open flags=BOGUS fails with EINVAL");
+
+    if (!(msg = flux_msg_create (FLUX_MSGTYPE_EVENT)))
+        BAIL_OUT ("failed to create message");
+    errno = 0;
+    ok (flux_send_new (NULL, &msg, 0) < 0 && errno == EINVAL,
+       "flux_send_new h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_send_new (h, NULL, 0) < 0 && errno == EINVAL,
+       "flux_send_new msg=NULL fails with EINVAL");
+    errno = 0;
+    flux_msg_t *nullmsg = NULL;
+    ok (flux_send_new (h, &nullmsg, 0) < 0 && errno == EINVAL,
+       "flux_send_new *msg=NULL fails with EINVAL");
+    ok (flux_send_new (h, &msg, 0x100000) < 0 && errno == EINVAL,
+       "flux_send_new flags=BOGUS fails with EINVAL");
+
+    errno = 0;
+    ok (flux_send (NULL, msg, 0) < 0 && errno == EINVAL,
+       "flux_send h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_send (h, NULL, 0) < 0 && errno == EINVAL,
+       "flux_send msg=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_send (h, msg, 0x100000) < 0 && errno == EINVAL,
+       "flux_send flags=BOGUS fails with EINVAL");
+    errno = 0;
+    ok (flux_recv (NULL, FLUX_MATCH_ANY, 0) == NULL && errno == EINVAL,
+       "flux_recv h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_recv (h, FLUX_MATCH_ANY, 0x1000000) == NULL && errno == EINVAL,
+       "flux_recv flags=BOGUS fails with EINVAL");
+    flux_msg_destroy (msg);
+}
+
+static void test_flux_open_ex ()
+{
+    flux_error_t error;
+
+    ok (flux_open_ex ("foo://foo", 0, &error) == NULL,
+        "flux_open_ex with invalid connector name fails");
+    is ("Unable to find connector name 'foo'", error.text,
+        "flux_open_ex returns expected error in error.text");
+
+    ok (flux_open_ex (NULL, 0x1000000, &error) == NULL,
+        "flux_open_ex with invalid flags fails");
+    is ("invalid flags specified", error.text,
+        "flux_open_ex returns expected error in error.text");
+
+    lives_ok ({flux_open_ex ("foo://foo", 0, NULL);},
+        "flux_open_ex doesn't crash if error parameter is NULL");
+}
+
+static void test_send_new (void)
+{
+    flux_msg_t *msg;
+    flux_msg_t *msg2;
+    void *msgptr;
+    flux_t *h1;
+    flux_t *h2;
+    int type;
+
+    if (!(h1 = flux_open ("interthread://xyz", 0)))
+        BAIL_OUT ("can't continue without interthread pair");
+    if (!(h2 = flux_open ("interthread://xyz", 0)))
+        BAIL_OUT ("can't continue without interthread pair");
+
+    if (!(msg = flux_msg_create (FLUX_MSGTYPE_EVENT)))
+        BAIL_OUT ("could not create message");
+    if (flux_msg_aux_set (msg, "foo", "bar", NULL) < 0)
+        BAIL_OUT ("could not set message aux item");
+    msgptr = msg;
+
+    flux_msg_incref (msg);
+    errno = 0;
+    ok (flux_send_new (h1, &msg, 0) < 0 && errno == EINVAL,
+        "flux_send_new fails if message refcount > 1");
+    flux_msg_decref (msg);
+
+    ok (flux_send_new (h1, &msg, 0) == 0,
+        "flux_send_new works");
+    ok (msg == NULL,
+        "msg was set to NULL after send");
+
+    ok ((msg2 = flux_recv (h2, FLUX_MATCH_ANY, 0)) != NULL,
+        "flux_recv got the message");
+    ok (msg2 == msgptr,
+        "received message was not copied");
+    ok (flux_msg_get_type (msg2, &type) == 0 && type == FLUX_MSGTYPE_EVENT,
+        "and message is the correct type");
+    ok (flux_msg_aux_get (msg2, "foo") == NULL,
+        "aux item in sent message is cleared in received message");
+    flux_msg_destroy (msg2);
+
+    flux_close (h1);
+    flux_close (h2);
 }
 
 int main (int argc, char *argv[])
@@ -62,28 +150,59 @@ int main (int argc, char *argv[])
     flux_t *h;
     char *s;
     flux_msg_t *msg;
+    flux_msg_t *msg2;
     const char *topic;
     uint32_t matchtag;
 
     plan (NO_PLAN);
 
-    if (!(h = loopback_create (0)))
-        BAIL_OUT ("can't continue without loopback handle");
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("can't continue without loop handle");
 
-    test_handle_invalid_args ();
+    test_handle_invalid_args (h);
 
-    /* Test flux_fatal_set, flux_fatal_err
-     */
-    flux_fatal_set (h, fatal_err, NULL);
-    flux_fatal_error (h, __FUNCTION__, "Foo");
-    ok (fatal_tested == true,
-        "flux_fatal function is called on fatal error");
+    flux_comms_error_set (h, comms_err, NULL);
 
     /* Test flux_opt_set, flux_opt_get.
      */
     errno = 0;
+    uint32_t uid = 999;
+    ok (flux_opt_set (NULL, FLUX_OPT_TESTING_USERID, &uid, sizeof (uid)) < 0
+        && errno == EINVAL,
+        "flux_opt_set h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_opt_set (h, NULL, &uid, sizeof (uid)) < 0
+        && errno == EINVAL,
+        "flux_opt_set option=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_opt_set (h, FLUX_OPT_TESTING_USERID, NULL, sizeof (uid)) < 0
+        && errno == EINVAL,
+        "flux_opt_set option=testing_userid val=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_opt_set (h, FLUX_OPT_TESTING_USERID, &uid, sizeof (uid)+1) < 0
+        && errno == EINVAL,
+        "flux_opt_set option=testing_userid size=wrong fails with EINVAL");
+    errno = 0;
     ok (flux_opt_set (h, "nonexistent", NULL, 0) < 0 && errno == EINVAL,
         "flux_opt_set fails with EINVAL on unknown option");
+
+    errno = 0;
+    ok (flux_opt_get (NULL, FLUX_OPT_TESTING_USERID, &uid, sizeof (uid)) < 0
+        && errno == EINVAL,
+        "flux_opt_get h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_opt_get (h, NULL, &uid, sizeof (uid)) < 0
+        && errno == EINVAL,
+        "flux_opt_get option=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_opt_get (h, FLUX_OPT_TESTING_USERID, NULL, sizeof (uid)) < 0
+        && errno == EINVAL,
+        "flux_opt_get option=testing_userid val=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_opt_get (h, FLUX_OPT_TESTING_USERID, &uid, sizeof (uid)+1) < 0
+        && errno == EINVAL,
+        "flux_opt_get option=testing_userid size=wrong fails with EINVAL");
+
     errno = 0;
     ok (flux_opt_get (h, "nonexistent", NULL, 0) < 0 && errno == EINVAL,
         "flux_opt_get fails with EINVAL on unknown option");
@@ -95,7 +214,7 @@ int main (int argc, char *argv[])
         "flux_aux_get returns NULL on unknown key");
     flux_aux_set (h, "handletest::thing1", xstrdup ("hello"), aux_free);
     s = flux_aux_get (h, "handletest::thing1");
-    ok (s != NULL && !strcmp (s, "hello"),
+    ok (s != NULL && streq (s, "hello"),
         "flux_aux_get returns what was set");
     flux_aux_set (h, "handletest::thing1", NULL, NULL);
     ok (aux_destroyed,
@@ -108,24 +227,27 @@ int main (int argc, char *argv[])
      */
     ok (flux_flags_get (h) == 0,
         "flux_flags_get returns flags handle was opened with");
-    flux_flags_set (h, (FAKE_FLAG1 | FAKE_FLAG2));
-    ok (flux_flags_get (h) == (FAKE_FLAG1 | FAKE_FLAG2),
+    flux_flags_set (h, (FLUX_O_TRACE | FLUX_O_MATCHDEBUG));
+    ok (flux_flags_get (h) == (FLUX_O_TRACE | FLUX_O_MATCHDEBUG),
         "flux_flags_set sets specified flags");
-    flux_flags_unset (h, FAKE_FLAG1);
-    ok (flux_flags_get (h) == FAKE_FLAG2,
+    flux_flags_unset (h, FLUX_O_MATCHDEBUG);
+    ok (flux_flags_get (h) == FLUX_O_TRACE,
         "flux_flags_unset clears specified flag without clearing others");
-    flux_flags_set (h, FAKE_FLAG1);
-    ok (flux_flags_get (h) == (FAKE_FLAG1 | FAKE_FLAG2),
+    flux_flags_set (h, FLUX_O_MATCHDEBUG);
+    ok (flux_flags_get (h) == (FLUX_O_TRACE | FLUX_O_MATCHDEBUG),
         "flux_flags_set sets specified flag without clearing others");
     flux_flags_set (h, 0);
-    ok (flux_flags_get (h) == (FAKE_FLAG1 | FAKE_FLAG2),
+    ok (flux_flags_get (h) == (FLUX_O_TRACE | FLUX_O_MATCHDEBUG),
         "flux_flags_set (0) has no effect");
     flux_flags_unset (h, 0);
-    ok (flux_flags_get (h) == (FAKE_FLAG1 | FAKE_FLAG2),
+    ok (flux_flags_get (h) == (FLUX_O_TRACE | FLUX_O_MATCHDEBUG),
         "flux_flags_unset (0) has no effect");
     flux_flags_unset (h, ~0);
     ok (flux_flags_get (h) == 0,
         "flux_flags_unset (~0) clears all flags");
+    flux_flags_set (h, FLUX_O_RPCTRACK);
+    ok (flux_flags_get (h) == 0,
+    "flux_flags_set flags=FLUX_O_RPCTRACK has no effect");
 
     /* Test flux_send, flux_recv, flux_requeue
      * Check flux_pollevents along the way.
@@ -141,17 +263,38 @@ int main (int argc, char *argv[])
        "flux_pollevents shows FLUX_POLLIN set on non-empty queue");
     ok ((msg = flux_recv (h, FLUX_MATCH_ANY, 0)) != NULL
         && flux_request_decode (msg, &topic, NULL) == 0
-        && !strcmp (topic, "foo"),
+        && streq (topic, "foo"),
         "flux_recv works and sent message was received");
     ok ((flux_pollevents (h) & FLUX_POLLIN) == 0,
        "flux_pollevents shows FLUX_POLLIN clear after queue is emptied");
 
-    /* flux_requeue bad flags */
+    /* flux_requeue bad args */
+    errno = 0;
+    ok (flux_requeue (NULL, msg, FLUX_RQ_HEAD) < 0 && errno == EINVAL,
+        "flux_requeue h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_requeue (h, NULL, FLUX_RQ_HEAD) < 0 && errno == EINVAL,
+        "flux_requeue msg=NULL fails with EINVAL");
     errno = 0;
     ok (flux_requeue (h, msg, 0) < 0 && errno == EINVAL,
         "flux_requeue fails with EINVAL if HEAD|TAIL unspecified");
     flux_msg_destroy (msg);
 
+    /* requeue preserves aux container (kvs needs this) */
+    if (!(msg = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("couldn't encode request");
+    if (flux_msg_aux_set (msg, "fubar", "xyz", NULL) < 0)
+        BAIL_OUT ("couldn't attach something to message aux container");
+    ok (flux_requeue (h, msg, FLUX_RQ_HEAD) == 0,
+        "flux_requeue works");
+    msg2 = flux_recv (h, FLUX_MATCH_ANY, 0);
+    ok (msg2 == msg,
+        "flux_recv returned requeued message and it has the same address");
+    ok (flux_msg_aux_get (msg2, "fubar") != NULL,
+        "and aux was preserved");
+    flux_msg_decref (msg);
+    flux_msg_decref (msg2);
+
     /* flux_requeue: add foo, bar to HEAD; then receive bar, foo */
     if (!(msg = flux_request_encode ("foo", NULL)))
         BAIL_OUT ("couldn't encode request");
@@ -167,12 +310,12 @@ int main (int argc, char *argv[])
        "flux_pollevents shows FLUX_POLLIN set after requeue");
     ok ((msg = flux_recv (h, FLUX_MATCH_ANY, 0)) != NULL
         && flux_request_decode (msg, &topic, NULL) == 0
-        && !strcmp (topic, "bar"),
+        && streq (topic, "bar"),
         "flux_recv got bar");
     flux_msg_destroy (msg);
     ok ((msg = flux_recv (h, FLUX_MATCH_ANY, 0)) != NULL
         && flux_request_decode (msg, &topic, NULL) == 0
-        && !strcmp (topic, "foo"),
+        && streq (topic, "foo"),
         "flux_recv got foo");
     flux_msg_destroy (msg);
     ok ((flux_pollevents (h) & FLUX_POLLIN) == 0,
@@ -193,68 +336,12 @@ int main (int argc, char *argv[])
        "flux_pollevents shows FLUX_POLLIN set after requeue");
     ok ((msg = flux_recv (h, FLUX_MATCH_ANY, 0)) != NULL
         && flux_request_decode (msg, &topic, NULL) == 0
-        && !strcmp (topic, "foo"),
+        && streq (topic, "foo"),
         "flux_recv got foo");
     flux_msg_destroy (msg);
     ok ((msg = flux_recv (h, FLUX_MATCH_ANY, 0)) != NULL
         && flux_request_decode (msg, &topic, NULL) == 0
-        && !strcmp (topic, "bar"),
-        "flux_recv got bar");
-    flux_msg_destroy (msg);
-    ok ((flux_pollevents (h) & FLUX_POLLIN) == 0,
-       "flux_pollevents shows FLUX_POLLIN clear after queue is emptied");
-
-    /* flux_requeue_nocopy bad flags */
-    if (!(msg = flux_request_encode ("foo", NULL)))
-        BAIL_OUT ("couldn't encode request");
-    errno = 0;
-    ok (flux_requeue_nocopy (h, msg, 0) < 0 && errno == EINVAL,
-        "flux_requeue_nocopy fails with EINVAL if HEAD|TAIL unspecified");
-    flux_msg_destroy (msg);
-
-    /* flux_requeue_nocopy: add foo, bar to HEAD; then receive bar, foo */
-    if (!(msg = flux_request_encode ("foo", NULL)))
-        BAIL_OUT ("couldn't encode request");
-    ok (flux_requeue_nocopy (h, msg, FLUX_RQ_HEAD) == 0,
-        "flux_requeue_nocopy foo HEAD works");
-    if (!(msg = flux_request_encode ("bar", NULL)))
-        BAIL_OUT ("couldn't encode request");
-    ok (flux_requeue_nocopy (h, msg, FLUX_RQ_HEAD) == 0,
-        "flux_requeue_nocopy bar HEAD works");
-    ok ((flux_pollevents (h) & FLUX_POLLIN) != 0,
-       "flux_pollevents shows FLUX_POLLIN set after requeue");
-    ok ((msg = flux_recv (h, FLUX_MATCH_ANY, 0)) != NULL
-        && flux_request_decode (msg, &topic, NULL) == 0
-        && !strcmp (topic, "bar"),
-        "flux_recv got bar");
-    flux_msg_destroy (msg);
-    ok ((msg = flux_recv (h, FLUX_MATCH_ANY, 0)) != NULL
-        && flux_request_decode (msg, &topic, NULL) == 0
-        && !strcmp (topic, "foo"),
-        "flux_recv got foo");
-    flux_msg_destroy (msg);
-    ok ((flux_pollevents (h) & FLUX_POLLIN) == 0,
-       "flux_pollevents shows FLUX_POLLIN clear after queue is emptied");
-
-    /* flux_requeue_nocopy: add foo, bar to TAIL; then receive foo, bar */
-    if (!(msg = flux_request_encode ("foo", NULL)))
-        BAIL_OUT ("couldn't encode request");
-    ok (flux_requeue_nocopy (h, msg, FLUX_RQ_TAIL) == 0,
-        "flux_requeue_nocopy foo TAIL works");
-    if (!(msg = flux_request_encode ("bar", NULL)))
-        BAIL_OUT ("couldn't encode request");
-    ok (flux_requeue_nocopy (h, msg, FLUX_RQ_TAIL) == 0,
-        "flux_requeue_nocopy bar TAIL works");
-    ok ((flux_pollevents (h) & FLUX_POLLIN) != 0,
-       "flux_pollevents shows FLUX_POLLIN set after requeue");
-    ok ((msg = flux_recv (h, FLUX_MATCH_ANY, 0)) != NULL
-        && flux_request_decode (msg, &topic, NULL) == 0
-        && !strcmp (topic, "foo"),
-        "flux_recv got foo");
-    flux_msg_destroy (msg);
-    ok ((msg = flux_recv (h, FLUX_MATCH_ANY, 0)) != NULL
-        && flux_request_decode (msg, &topic, NULL) == 0
-        && !strcmp (topic, "bar"),
+        && streq (topic, "bar"),
         "flux_recv got bar");
     flux_msg_destroy (msg);
     ok ((flux_pollevents (h) & FLUX_POLLIN) == 0,
@@ -266,7 +353,21 @@ int main (int argc, char *argv[])
         "flux_matchtag_alloc works");
     flux_matchtag_free (h, matchtag);
 
+    /* reconnect */
+    errno = 0;
+    ok (flux_reconnect (NULL) < 0 && errno == EINVAL,
+        "flux_reconnect h=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_reconnect (h) < 0 && errno == ENOSYS,
+        "flux_reconnect with null reconnect method fails with ENOSYS");
+
     flux_close (h);
+
+    /* flux_open_ex() */
+    test_flux_open_ex ();
+
+    test_send_new ();
+
     done_testing();
     return (0);
 }
diff --git a/src/common/libflux/test/interthread.c b/src/common/libflux/test/interthread.c
new file mode 100644
index 000000000000..6fd12715f7d3
--- /dev/null
+++ b/src/common/libflux/test/interthread.c
@@ -0,0 +1,438 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <poll.h>
+#include <pthread.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+void recv_count_is (flux_t *h, size_t expected, const char *msg)
+{
+    size_t n;
+    ok (flux_opt_get (h, FLUX_OPT_RECV_QUEUE_COUNT, &n, sizeof (n)) == 0
+        && n == expected,
+        "%s", msg);
+}
+void send_count_is (flux_t *h, size_t expected, const char *msg)
+{
+    size_t n;
+    ok (flux_opt_get (h, FLUX_OPT_SEND_QUEUE_COUNT, &n, sizeof (n)) == 0
+        && n == expected,
+        "%s", msg);
+}
+
+void test_basic (void)
+{
+    const char *uri = "interthread://test1";
+    const char *uri2 = "interthread://test2";
+    flux_t *h;
+    flux_t *h2;
+    flux_t *h3;
+    flux_t *h4;
+    flux_error_t error;
+    flux_msg_t *msg;
+    flux_msg_t *req;
+    flux_msg_t *rep;
+    struct flux_msg_cred cred;
+    const char *topic;
+    const char *payload;
+
+    /* create pair to use in test */
+    h = flux_open (uri, 0);
+    ok (h != NULL,
+        "basic: flux_open %s (1) works", uri);
+    h2 = flux_open (uri, 0);
+    ok (h2 != NULL,
+        "basic: flux_open %s (2) works", uri);
+
+    send_count_is (h, 0, "SEND_QUEUE_COUNT h = 0");
+    recv_count_is (h, 0, "RECV_QUEUE_COUNT h = 0");
+    send_count_is (h2, 0, "SEND_QUEUE_COUNT h2 = 0");
+    recv_count_is (h2, 0, "RECV_QUEUE_COUNT h2 = 0");
+
+    /* cover connecting to a paired channel */
+    errno = 0;
+    ok (flux_open (uri, 0) == NULL && errno == EADDRINUSE,
+        "basic: flux_open %s (3) fails with EADDRINUSE", uri);
+    errno = 0;
+    error.text[0] = '\0';
+    ok (flux_open_ex (uri, 0, &error) == NULL && errno == EADDRINUSE,
+        "basic: flux_open_ex %s also fails with EADDRINUSE", uri);
+    diag ("%s", error.text);
+    like (error.text, "already paired",
+        "basic: and error string contains something useful", uri);
+
+    /* create another pair to exercise channel allocation */
+    h3 = flux_open (uri2, 0);
+    ok (h3 != NULL,
+        "basic: flux_open %s (1) works", uri2);
+    h4 = flux_open (uri2, 0);
+    ok (h4 != NULL,
+        "basic: flux_open %s (2) works", uri2);
+    flux_close (h4);
+    flux_close (h3);
+
+    /* send request h -> h2 */
+    if (!(req = flux_request_encode ("foo.bar", "baz")))
+        BAIL_OUT ("basic: could not create request");
+    ok (flux_send (h, req, 0) == 0,
+       "basic: flux_send on first handle works");
+    send_count_is (h, 1, "SEND_QUEUE_COUNT h = 1");
+    recv_count_is (h2, 1, "RECV_QUEUE_COUNT h2 = 1");
+    msg = flux_recv (h2, FLUX_MATCH_ANY, 0);
+    ok (msg != NULL,
+        "basic: flux_recv on second handle works");
+    send_count_is (h, 0, "SEND_QUEUE_COUNT h = 0");
+    recv_count_is (h2, 0, "RECV_QUEUE_COUNT h = 0");
+    ok (flux_msg_route_count (msg) == 0,
+        "basic: request has no route stack");
+    ok (flux_request_decode (msg, &topic, &payload) == 0
+        && streq (topic, "foo.bar")
+        && streq (payload, "baz"),
+        "basic: request has expected topic and payload");
+    if (!(rep = flux_response_derive (msg, 0)))
+        BAIL_OUT ("basic: could not create response");
+    ok (flux_msg_get_cred (msg, &cred) == 0
+        && cred.userid == getuid ()
+        && cred.rolemask == (FLUX_ROLE_OWNER | FLUX_ROLE_LOCAL),
+        "basic: message cred has expected values");
+    flux_msg_destroy (msg);
+
+    /* send response h2 -> h */
+    ok (flux_send (h2, rep, 0) == 0,
+       "basic: flux_send on second handle works");
+    recv_count_is (h, 1, "RECV_QUEUE_COUNT h = 1");
+    send_count_is (h2, 1, "SEND_QUEUE_COUNT h2 = 1");
+    msg = flux_recv (h, FLUX_MATCH_ANY, 0);
+    ok (msg != NULL,
+        "basic: flux_recv on first handle works");
+    recv_count_is (h, 0, "RECV_QUEUE_COUNT h = 0");
+    send_count_is (h2, 0, "SEND_QUEUE_COUNT h2 = 0");
+    ok (flux_msg_route_count (msg) == 0,
+        "basic: response has no route stack");
+    ok (flux_response_decode (msg, &topic, &payload) == 0
+        && streq (topic, "foo.bar")
+        && payload == NULL,
+        "basic: response has expected topic and payload");
+    flux_msg_destroy (msg);
+    flux_msg_destroy (req);
+    flux_msg_destroy (rep);
+
+    flux_close (h2);
+    flux_close (h);
+}
+
+void test_router (void)
+{
+    const char *uri = "interthread://test1";
+    flux_t *h;
+    flux_t *h2;
+    flux_msg_t *msg;
+    flux_msg_t *req;
+    flux_msg_t *rep;
+
+    /* create pair to use in test */
+    h = flux_open (uri, 0);
+    ok (h != NULL,
+        "router: flux_open %s (1) works", uri);
+    ok (flux_opt_set (h, FLUX_OPT_ROUTER_NAME, "testrouter", 11) == 0,
+        "router: flux_opt_set FLUX_OPT_ROUTER_NAME=testrouter works");
+    h2 = flux_open (uri, 0);
+    ok (h2 != NULL,
+        "router: flux_open %s (2) works", uri);
+
+    /* send request h -> h2 */
+    if (!(req = flux_request_encode ("foo.bar", "baz")))
+        BAIL_OUT ("router: could not create request");
+    ok (flux_send (h, req, 0) == 0,
+       "router: flux_send on first handle works");
+    msg = flux_recv (h2, FLUX_MATCH_ANY, 0);
+    ok (msg != NULL,
+        "router: flux_recv on second handle works");
+    ok (flux_msg_route_count (msg) == 1
+        && streq (flux_msg_route_last (msg), "testrouter"),
+        "router: request is from testrouter");
+    if (!(rep = flux_response_derive (msg, 0)))
+        BAIL_OUT ("router: could not create response");
+    flux_msg_destroy (msg);
+
+    /* send response h2 -> h */
+    ok (flux_send (h2, rep, 0) == 0,
+       "router: flux_send on second handle works");
+    msg = flux_recv (h, FLUX_MATCH_ANY, 0);
+    ok (msg != NULL,
+        "router: flux_recv on first handle works");
+    ok (flux_msg_route_count (msg) == 0,
+        "router: response has no route stack");
+    flux_msg_destroy (msg);
+    flux_msg_destroy (rep);
+
+    /* send request h2 -> h */
+    ok (flux_send (h2, req, 0) == 0,
+       "router: flux_send on second handle works");
+    msg = flux_recv (h, FLUX_MATCH_ANY, 0);
+    ok (msg != NULL,
+        "router: flux_recv on first handle works");
+    ok (flux_msg_route_count (msg) == 1
+        && streq (flux_msg_route_last (msg), "test1"),
+        "router: request is from test1");
+    if (!(rep = flux_response_derive (msg, 0)))
+        BAIL_OUT ("router: could not create response");
+    flux_msg_destroy (msg);
+
+    /* send response h -> h2 */
+    ok (flux_send (h, rep, 0) == 0,
+       "router: flux_send on first handle works");
+    msg = flux_recv (h2, FLUX_MATCH_ANY, 0);
+    ok (msg != NULL,
+        "router: flux_recv on second handle works");
+    ok (flux_msg_route_count (msg) == 0,
+        "router: response has no route stack");
+    flux_msg_destroy (msg);
+    flux_msg_destroy (rep);
+
+    flux_msg_destroy (req);
+
+    flux_close (h2);
+    flux_close (h);
+}
+
+struct test_thread {
+    pthread_t t;
+    flux_t *h;
+    flux_watcher_t *w;
+    char uri[64];
+    int total;
+    int count;
+};
+
+static flux_watcher_t *timer;
+static int num_active_threads;
+
+void *test_thread (void *arg)
+{
+    struct test_thread *test = arg;
+    flux_t *h;
+    flux_msg_t *msg;
+
+    if (!(h = flux_open (test->uri, 0)))
+        BAIL_OUT ("%s: flux_open: %s", test->uri, strerror (errno));
+    if (!(msg = flux_request_encode ("foo.bar", NULL)))
+        BAIL_OUT ("%s: flux_request_encode: %s", test->uri, strerror (errno));
+    for (int i = 0; i < test->total; i++) {
+        if (flux_send (h, msg, 0) < 0)
+            BAIL_OUT ("%s: flux_send: %s", test->uri, strerror (errno));
+    }
+    flux_msg_destroy (msg);
+    flux_close (h);
+    return NULL;
+}
+
+void timeout (flux_reactor_t *r, flux_watcher_t *w, int revents, void *arg)
+{
+    diag ("test timed out");
+    flux_reactor_stop_error (r);
+}
+
+void watcher (flux_reactor_t *r, flux_watcher_t *w, int revents, void *arg)
+{
+    struct test_thread *test = arg;
+    flux_msg_t *msg;
+
+    if (!(msg = flux_recv (test->h, FLUX_MATCH_ANY, 0))) {
+        diag ("%s: flux_recv: %s", test->uri, strerror (errno));
+        flux_reactor_stop_error (r);
+        return;
+    }
+    if (flux_request_decode (msg, NULL, NULL) < 0) {
+        diag ("%s: flux_request_decode: %s", test->uri, strerror (errno));
+        flux_reactor_stop_error (r);
+    }
+    flux_msg_destroy (msg);
+    if (++test->count == test->total) {
+        flux_watcher_stop (w);
+        if (--num_active_threads == 0)
+            flux_watcher_stop (timer);
+    }
+}
+
+int test_threads_init (struct test_thread *test,
+                       size_t count,
+                       flux_reactor_t *r,
+                       int num_messages)
+{
+    for (int i = 0; i < count; i++) {
+        int e;
+        snprintf (test[i].uri, sizeof (test[i].uri), "interthread://%d", i);
+        test[i].total = num_messages;
+        if (!(test[i].h = flux_open (test[i].uri, 0))) {
+            diag ("flux_open %s failed: %s", test[i].uri, strerror (errno));
+            return -1;
+        }
+        if (!(test[i].w = flux_handle_watcher_create (r,
+                                                      test[i].h,
+                                                      FLUX_POLLIN,
+                                                      watcher,
+                                                      &test[i]))) {
+            diag ("watcher create %s failed: %s", test[i].uri, strerror (errno));
+            return -1;
+        }
+        flux_watcher_start (test[i].w);
+        if ((e = pthread_create (&test[i].t, NULL, test_thread, &test[i]))) {
+            diag ("pthread_create %s failed: %s", test[i].uri, strerror (e));
+            return -1;
+        }
+    }
+    return 0;
+}
+
+int test_threads_join (struct test_thread *test, size_t count)
+{
+    for (int i = 0; i < count; i++) {
+        void *result;
+        int e;
+        if ((e = pthread_join (test[i].t, &result))) {
+            diag ("pthread_join %s failed: %s", test[i].uri, strerror (e));
+            return -1;
+        }
+        flux_watcher_destroy (test[i].w);
+        flux_close (test[i].h);
+    }
+    return 0;
+}
+
+void test_threads (void)
+{
+    const double timeout_s = 30.;
+    const int num_messages = 32;
+    const int num_threads = 16;
+    struct test_thread test[num_threads];
+    flux_reactor_t *r;
+
+    if (!(r = flux_reactor_create (0)))
+        BAIL_OUT ("could not create reactor");
+
+    num_active_threads = num_threads;
+    if (!(timer = flux_timer_watcher_create (r, timeout_s, 0, timeout, NULL)))
+        BAIL_OUT ("could not create timer watcher");
+    flux_watcher_start (timer);
+
+    memset (test, 0, sizeof (test));
+
+    ok (test_threads_init (test, num_threads, r, num_messages) == 0,
+        "started %d threads that will each send %d messages",
+        num_threads,
+        num_messages);
+    ok (flux_reactor_run (r, 0) == 0,
+        "all messages received with no errors");
+    ok (test_threads_join (test, num_threads) == 0,
+        "finalized test threads");
+
+    flux_watcher_destroy (timer);
+    flux_reactor_destroy (r);
+}
+
+void test_poll (void)
+{
+    const char *uri = "interthread://polltest";
+    flux_t *h1;
+    flux_t *h2;
+    flux_msg_t *msg;
+    flux_msg_t *msg2;
+    struct pollfd pfd;
+
+    // with NOREQUEUE, pollfd/pollevents come directly from connector
+    if (!(h1 = flux_open (uri, FLUX_O_NOREQUEUE)))
+        BAIL_OUT ("%s: flux_open: %s", uri, strerror (errno));
+    if (!(h2 = flux_open (uri, FLUX_O_NOREQUEUE)))
+        BAIL_OUT ("%s: flux_open: %s", uri, strerror (errno));
+    diag ("poll: opened h1 and h2");
+
+    if (!(msg = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+
+    // enqueue 2 messages
+    ok (flux_pollevents (h2) == FLUX_POLLOUT,
+        "flux_pollevents h2 returns POLLOUT");
+    ok (flux_send (h1, msg, 0) == 0,
+        "flux_send h1 works");
+    ok (flux_send (h1, msg, 0) == 0,
+        "flux_send h1 works");
+    ok (flux_pollevents (h2) == (FLUX_POLLOUT | FLUX_POLLIN),
+        "flux_pollevents h2 returns POLLOUT|POLLIN");
+
+    // read 1 message
+    ok ((msg2 = flux_recv (h2, FLUX_MATCH_ANY, 0)) != NULL,
+        "flux_recv h2 works");
+    ok (flux_pollevents (h2) == (FLUX_POLLOUT | FLUX_POLLIN),
+        "flux_pollevents h2 returns POLLOUT|POLLIN");
+    flux_msg_decref (msg2);
+
+    // read 2nd message
+    ok ((msg2 = flux_recv (h2, FLUX_MATCH_ANY, 0)) != NULL,
+        "flux_recv h2 works");
+    ok (flux_pollevents (h2) == FLUX_POLLOUT,
+        "flux_pollevents h2 returns POLLOUT");
+    flux_msg_destroy (msg2);
+
+    // get pollfd set up with no messages pending
+    ok ((pfd.fd = flux_pollfd (h2)) >= 0,
+        "flux_pollfd works");
+    pfd.events = POLLIN; // poll fd becomes "readable" when pollevents should
+    pfd.revents = 0;     //   be checked
+    ok (poll (&pfd, 1, 0) == 1 && pfd.revents == POLLIN,
+        "flux_pollfd suggests we check pollevents");
+    ok (flux_pollevents (h2) == FLUX_POLLOUT,
+        "flux_pollevents returns POLLOUT only");
+    pfd.events = POLLIN;
+    pfd.revents = 0;
+    ok (poll (&pfd, 1, 0) == 0, // because edge triggered
+        "flux_pollfd says not ready, now that we've checked pollevents");
+
+    // enqueue 2 messages
+    ok (flux_send (h1, msg, 0) == 0,
+        "flux_send h1 works");
+    ok (flux_send (h1, msg, 0) == 0,
+        "flux_send h1 works");
+    pfd.events = POLLIN,
+    pfd.revents = 0,
+    ok (poll (&pfd, 1, 0) == 1 && pfd.revents == POLLIN,
+        "pollfd suggests we read pollevents");
+    ok (flux_pollevents (h2) == (FLUX_POLLOUT | FLUX_POLLIN),
+        "flux_pollevents returns POLLOUT|POLLIN");
+    ok (poll (&pfd, 1, 0) == 0,
+        "flux_pollfd says not ready, now that we've checked pollevents");
+
+    // N.B. we don't own the pollfd so no close here
+    flux_msg_destroy (msg);
+    flux_close (h1);
+    flux_close (h2);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_basic ();
+    test_router ();
+    test_threads ();
+    test_poll ();
+
+    done_testing ();
+    return 0;
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libflux/test/log.c b/src/common/libflux/test/log.c
index 50b4c9abb817..345cecc2ab2b 100644
--- a/src/common/libflux/test/log.c
+++ b/src/common/libflux/test/log.c
@@ -8,8 +8,10 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
@@ -21,9 +23,7 @@ int main (int argc, char *argv[])
 
     plan (NO_PLAN);
 
-    test_server_environment_init ("log-test");
-
-    if (!(h = test_server_create (NULL, NULL)))
+    if (!(h = test_server_create (0, NULL, NULL)))
         BAIL_OUT ("could not create test server");
     if (flux_attr_set_cacheonly (h, "rank", "0") < 0)
         BAIL_OUT ("flux_attr_set_cacheonly failed");
diff --git a/src/common/libflux/test/message.c b/src/common/libflux/test/message.c
index 9f54554aa44b..0ec8d18bc975 100644
--- a/src/common/libflux/test/message.c
+++ b/src/common/libflux/test/message.c
@@ -11,94 +11,418 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
+#include <stdbool.h>
 #include <errno.h>
 #include <stdio.h>
 #include <jansson.h>
+#include <string.h>
+#include <assert.h>
+#include <flux/core.h>
 
-#include "src/common/libflux/message.h"
 #include "src/common/libtap/tap.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
 
-/* flux_msg_get_route_first, flux_msg_get_route_last, _get_route_count
- *   on message with variable number of routing frames
+#include "message_private.h"
+
+static bool verbose = false;
+
+void check_cornercase (void)
+{
+    flux_msg_t *msg;
+    flux_msg_t *req, *rsp, *evt;
+    struct flux_msg_cred cred;
+    uint32_t seq, nodeid;
+    uint8_t encodebuf[64];
+    size_t encodesize = 64;
+    int type, errnum, status;
+    uint32_t tag;
+    const char *topic;
+    const void *payload;
+    size_t payload_size;
+
+    errno = 0;
+    ok (flux_msg_create (0xFFFF) == NULL && errno == EINVAL,
+        "flux_msg_create fails with EINVAL on invalid type");
+
+    if (!(msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
+        BAIL_OUT ("flux_msg_create failed");
+    if (!(req = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
+        BAIL_OUT ("flux_msg_create failed");
+    if (!(rsp = flux_msg_create (FLUX_MSGTYPE_RESPONSE)))
+        BAIL_OUT ("flux_msg_create failed");
+    if (!(evt = flux_msg_create (FLUX_MSGTYPE_EVENT)))
+        BAIL_OUT ("flux_msg_create failed");
+
+    lives_ok ({flux_msg_destroy (NULL);},
+        "flux_msg_destroy msg=NULL doesnt crash");
+
+    errno = 0;
+    ok (flux_msg_aux_set (NULL, "foo", "bar", NULL) < 0 && errno == EINVAL,
+        "flux_msg_aux_set msg=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_aux_get (NULL, "foo") == NULL && errno == EINVAL,
+        "flux_msg_aux_get msg=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (flux_msg_copy (NULL, true) == NULL && errno == EINVAL,
+        "flux_msg_copy msg=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (flux_msg_incref (NULL) == NULL && errno == EINVAL,
+        "flux_msg_incref msg=NULL fails with EINVAL");
+    lives_ok ({flux_msg_decref (NULL);},
+        "flux_msg_decref msg=NULL doesnt crash");
+
+    errno = 0;
+    ok (flux_msg_encode_size (NULL) < 0 && errno == EINVAL,
+        "flux_msg_encode_size fails with EINVAL on msg = NULL");
+    errno = 0;
+    ok (flux_msg_encode (NULL, encodebuf, encodesize) < 0 && errno == EINVAL,
+        "flux_msg_encode fails on EINVAL with msg=NULL");
+    errno = 0;
+    ok (msg_frames (NULL) < 0 && errno == EINVAL,
+        "msg_frames returns -1 errno EINVAL on msg = NULL");
+
+    errno = 0;
+    ok (flux_msg_set_type (NULL, 0) < 0 && errno == EINVAL,
+        "flux_msg_set_type fails with EINVAL on msg = NULL");
+    errno = 0;
+    ok (flux_msg_get_type (NULL, &type) < 0 && errno == EINVAL,
+        "flux_msg_get_type fails with EINVAL on msg = NULL");
+    lives_ok ({flux_msg_get_type (msg, NULL);},
+        "flux_msg_get_type doesn't segfault with NULL type arg");
+
+    errno = 0;
+    ok (flux_msg_set_private (NULL) < 0 && errno == EINVAL,
+        "flux_msg_set_private msg=NULL fails with EINVAL");
+    ok (flux_msg_is_private (NULL) == true,
+        "flux_msg_is_private msg=NULL returns true");
+    errno = 0;
+    ok (flux_msg_set_streaming (NULL) < 0 && errno == EINVAL,
+        "flux_msg_set_streaming msg=NULL fails with EINVAL");
+    ok (flux_msg_is_streaming (NULL) == true,
+        "flux_msg_is_streaming msg=NULL returns true");
+    errno = 0;
+    ok (flux_msg_set_noresponse (NULL) < 0 && errno == EINVAL,
+        "flux_msg_set_noresponse msg=NULL fails with EINVAL");
+    ok (flux_msg_is_noresponse (NULL) == true,
+        "flux_msg_is_noresponse msg=NULL returns true");
+
+    errno = 0;
+    ok (flux_msg_set_topic (NULL, "foobar") < 0 && errno == EINVAL,
+       "flux_msg_set_topic fails with EINVAL on msg = NULL");
+    errno = 0;
+    ok (flux_msg_get_topic (msg, NULL) < 0 && errno == EINVAL,
+       "flux_msg_get_topic fails with EINVAL on in-and-out param = NULL");
+    errno = 0;
+    ok (flux_msg_get_topic (msg, &topic) < 0 && errno == EPROTO,
+       "flux_msg_get_topic fails with EPROTO on msg w/o topic");
+
+    errno = 0;
+    ok (flux_msg_set_payload (NULL, NULL, 0) < 0 && errno == EINVAL,
+        "flux_msg_set_payload msg=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_get_payload (NULL, NULL, NULL) < 0 && errno == EINVAL,
+        "flux_msg_get_payload msg=NULL fails with EINVAL");
+    lives_ok ({flux_msg_get_payload (msg, NULL, NULL);},
+       "flux_msg_get_payload does not segfault on in-and-out params = NULL");
+    errno = 0;
+    ok (flux_msg_get_payload (msg, &payload, &payload_size) < 0
+        && errno == EPROTO,
+       "flux_msg_get_payload fails with EPROTO on msg w/o payload");
+    ok (flux_msg_has_payload (NULL) == false,
+        "flux_msg_has_payload returns false on msg = NULL");
+
+    errno = 0;
+    ok (flux_msg_set_flag (NULL, FLUX_MSGFLAG_STREAMING) < 0
+        && errno == EINVAL,
+        "flux_msg_set_flag msg=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_clear_flag (NULL, FLUX_MSGFLAG_STREAMING) < 0
+        && errno == EINVAL,
+        "flux_msg_clear_flag msg=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_set_flag (msg, 0x80000000) < 0 && errno == EINVAL,
+        "flux_msg_set_flag flag=0x80000000 fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_clear_flag (msg, 0x80000000) < 0 && errno == EINVAL,
+        "flux_msg_clear_flag flag=0x80000000 fails with EINVAL");
+    lives_ok ({flux_msg_has_flag (NULL, FLUX_MSGFLAG_STREAMING);},
+       "flux_msg_has_flag msg=NULL does not segfault");
+
+    errno = 0;
+    ok (flux_msg_set_flag (msg, FLUX_MSGFLAG_STREAMING
+                              | FLUX_MSGFLAG_NORESPONSE) < 0
+        && errno == EINVAL,
+        "flux_msg_set_flag streaming|noresponse fails with EINVAL");
+
+    errno = 0;
+    ok (flux_msg_pack (NULL, "{s:i}", "foo", 42) < 0 && errno == EINVAL,
+       "flux_msg_pack msg=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_pack (msg, NULL) < 0 && errno == EINVAL,
+        "flux_msg_pack fails with EINVAL with NULL format");
+    errno = 0;
+    ok (flux_msg_unpack (NULL, "{s:i}", "type", &type) < 0 && errno == EINVAL,
+       "flux_msg_unpack msg=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_unpack (msg, NULL) < 0 && errno == EINVAL,
+        "flux_msg_unpack fails with EINVAL with NULL format");
+
+    errno = 0;
+    ok (flux_msg_set_nodeid (NULL, 0) < 0 && errno == EINVAL,
+                "flux_msg_set_nodeid fails with EINVAL on msg = NULL");
+    errno = 0;
+    ok (flux_msg_get_nodeid (NULL, &nodeid) < 0 && errno == EINVAL,
+        "flux_msg_get_nodeid fails with EINVAL on msg = NULL");
+    errno = 0;
+    ok (flux_msg_get_nodeid (rsp, &nodeid) < 0 && errno == EPROTO,
+        "flux_msg_get_nodeid fails with PROTO on msg != request type");
+    errno = 0;
+    ok (flux_msg_get_userid (NULL, &cred.userid) < 0 && errno == EINVAL,
+        "flux_msg_get_userid msg=NULL fails with EINVAL");
+    lives_ok ({flux_msg_get_userid (msg, NULL);},
+        "flux_msg_get_userid userid=NULL does not segfault");
+    errno = 0;
+    ok (flux_msg_set_userid (NULL, cred.userid) < 0 && errno == EINVAL,
+        "flux_msg_set_userid msg=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_get_rolemask (NULL, &cred.rolemask) < 0 && errno == EINVAL,
+        "flux_msg_get_rolemask msg=NULL fails with EINVAL");
+    lives_ok ({flux_msg_get_rolemask (msg, NULL);},
+        "flux_msg_get_rolemask rolemask=NULL does not segfault");
+    errno = 0;
+    ok (flux_msg_set_rolemask (NULL, cred.rolemask) < 0 && errno == EINVAL,
+        "flux_msg_set_rolemask msg=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (flux_msg_get_cred (NULL, &cred) < 0 && errno == EINVAL,
+        "flux_msg_get_cred msg=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_get_cred (msg, NULL) < 0 && errno == EINVAL,
+        "flux_msg_get_cred cred=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_set_cred (NULL, cred) < 0 && errno == EINVAL,
+        "flux_msg_set_cred msg=NULL fails with EINVAL");
+    errno = 0;
+    ok (flux_msg_authorize (NULL, 42) < 0 && errno == EINVAL,
+        "flux_msg_authorize msg=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (flux_msg_set_errnum (NULL, 42) < 0 && errno == EINVAL,
+        "flux_msg_set_errnum on fails with errno == EINVAL on msg = NULL");
+    errno = 0;
+    ok (flux_msg_get_errnum (NULL, &errnum) < 0 && errno == EINVAL,
+        "flux_msg_get_errnum fails with EINVAL on msg = NULL");
+    lives_ok ({flux_msg_get_errnum (msg, NULL);},
+        "flux_msg_get_errnum errnum = NULL does not segfault");
+    errno = 0;
+    ok (flux_msg_get_errnum (req, &errnum) < 0 && errno == EPROTO,
+        "flux_msg_get_errnum fails with EPROTO on msg != response type");
+    errno = 0;
+    ok (flux_msg_set_seq (NULL, 0) < 0 && errno == EINVAL,
+        "flux_msg_set_seq fails with EINVAL on msg = NULL");
+    errno = 0;
+    ok (flux_msg_get_seq (NULL, &seq) < 0 && errno == EINVAL,
+        "flux_msg_get_seq fails with EINVAL on msg = NULL");
+    lives_ok ({flux_msg_get_seq (msg, NULL);},
+        "flux_msg_get_seq seq = NULL does not segfault");
+    errno = 0;
+    ok (flux_msg_get_seq (req, &seq) < 0 && errno == EPROTO,
+        "flux_msg_get_seq fails with EPROTO on msg != event type");
+    errno = 0;
+    ok (flux_msg_set_control (NULL, 0, 0) < 0 && errno == EINVAL,
+        "flux_msg_set_status fails with EINVAL on msg = NULL");
+    errno = 0;
+    ok (flux_msg_get_control (NULL, &type, &status) < 0 && errno == EINVAL,
+        "flux_msg_get_status fails with EINVAL on msg = NULL");
+    lives_ok ({flux_msg_get_control (msg, &type, NULL);},
+        "flux_msg_get_status status = NULL does not segfault");
+    errno = 0;
+    ok (flux_msg_get_control (req, &type, &status) < 0 && errno == EPROTO,
+        "flux_msg_get_status fails with EPROTO on msg != control type");
+    errno = 0;
+    ok (flux_msg_set_matchtag (NULL, 42) < 0 && errno == EINVAL,
+        "flux_msg_set_matchtag fails with EINVAL on msg = NULL");
+    errno = 0;
+    ok (flux_msg_get_matchtag (NULL, &tag) < 0 && errno == EINVAL,
+        "flux_msg_get_matchtag fails with EINVAL on msg = NULL");
+    lives_ok ({flux_msg_get_matchtag (msg, NULL);},
+        "flux_msg_get_matchtag matchtag = NULL does not segfault");
+    errno = 0;
+    ok (flux_msg_get_matchtag (evt, &tag) < 0 && errno == EPROTO,
+        "flux_msg_get_matchtag fails with EPROTO on msg != req/rsp type");
+
+    lives_ok ({flux_msg_route_enable (NULL);},
+        "flux_msg_route_enable msg=NULL doesnt crash");
+    lives_ok ({flux_msg_route_disable (NULL);},
+        "flux_msg_route_disable msg=NULL doesnt crash");
+    lives_ok ({flux_msg_route_clear (NULL);},
+        "flux_msg_route_clear msg=NULL doesnt crash");
+
+    errno = 0;
+    ok (flux_msg_route_push (NULL, "foo") == -1 && errno == EINVAL,
+        "flux_msg_route_push returns -1 errno EINVAL on msg = NULL");
+    errno = 0;
+    ok (flux_msg_route_push (msg, NULL) == -1 && errno == EINVAL,
+        "flux_msg_route_push returns -1 errno EINVAL on id = NULL");
+    errno = 0;
+    ok (flux_msg_route_push (msg, "foo") == -1 && errno == EPROTO,
+        "flux_msg_route_push returns -1 errno EPROTO on msg w/o routes enabled");
+    errno = 0;
+    ok (flux_msg_route_delete_last (NULL) == -1 && errno == EINVAL,
+        "flux_msg_route_delete_last returns -1 errno EINVAL on id = NULL");
+    errno = 0;
+    ok (flux_msg_route_delete_last (msg) == -1 && errno == EPROTO,
+        "flux_msg_route_delete_last returns -1 errno EPROTO on msg "
+        "w/o routes enabled");
+    ok (flux_msg_route_first (NULL) == NULL,
+        "flux_msg_route_first returns NULL on msg = NULL");
+    ok (flux_msg_route_first (msg) == NULL,
+        "flux_msg_route_first returns NULL on msg w/o routes enabled");
+    ok (flux_msg_route_last (NULL) == NULL,
+        "flux_msg_route_last returns NULL on msg = NULL");
+    ok (flux_msg_route_last (msg) == NULL,
+        "flux_msg_route_last returns NULL on msg w/o routes enabled");
+    errno = 0;
+    ok ((flux_msg_route_count (NULL) == -1 && errno == EINVAL),
+        "flux_msg_route_count returns -1 errno EINVAL on msg = NULL");
+    errno = 0;
+    ok ((flux_msg_route_count (msg) == -1 && errno == EPROTO),
+        "flux_msg_route_count returns -1 errno EPROTO on msg "
+        "w/o routes enabled");
+    errno = 0;
+    ok ((flux_msg_route_string (NULL) == NULL && errno == EINVAL),
+        "flux_msg_route_string returns NULL errno EINVAL on msg = NULL");
+    errno = 0;
+    ok ((flux_msg_route_string (msg) == NULL && errno == EPROTO),
+        "flux_msg_route_string returns NULL errno EPROTO on msg "
+        "w/o routes enabled");
+
+    flux_msg_destroy (msg);
+}
+
+/* flux_msg_route_first, flux_msg_route_last,
+ *   flux_msg_route_count on message with variable number of
+ *   routing frames
  */
 void check_routes (void)
 {
     flux_msg_t *msg;
+    const char *route;
     char *s;
 
     ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL
-        && flux_msg_frames (msg) == 1,
+        && msg_frames (msg) == 1,
         "flux_msg_create works and creates msg with 1 frame");
-    errno = 0;
-    ok (flux_msg_get_route_count (msg) < 0 && errno == EPROTO,
-        "flux_msg_get_route_count returns -1 errno EPROTO on msg w/o delim");
-    errno = 0;
-    ok ((flux_msg_get_route_first (msg, &s) == -1 && errno == EPROTO),
-        "flux_msg_get_route_first returns -1 errno EPROTO on msg w/o delim");
-    errno = 0;
-    ok ((flux_msg_get_route_last (msg, &s) == -1 && errno == EPROTO),
-        "flux_msg_get_route_last returns -1 errno EPROTO on msg w/o delim");
-    ok ((flux_msg_pop_route (msg, &s) == -1 && errno == EPROTO),
-        "flux_msg_pop_route returns -1 errno EPROTO on msg w/o delim");
-
-    ok (flux_msg_clear_route (msg) == 0 && flux_msg_frames (msg) == 1,
-        "flux_msg_clear_route works, is no-op on msg w/o delim");
-    ok (flux_msg_enable_route (msg) == 0 && flux_msg_frames (msg) == 2,
-        "flux_msg_enable_route works, adds one frame on msg w/o delim");
-    ok ((flux_msg_get_route_count (msg) == 0),
-        "flux_msg_get_route_count returns 0 on msg w/delim");
-    ok (flux_msg_pop_route (msg, &s) == 0 && s == NULL,
-        "flux_msg_pop_route works and sets id to NULL on msg w/o routes");
-
-    ok (flux_msg_get_route_first (msg, &s) == 0 && s == NULL,
-        "flux_msg_get_route_first returns 0, id=NULL on msg w/delim");
-    ok (flux_msg_get_route_last (msg, &s) == 0 && s == NULL,
-        "flux_msg_get_route_last returns 0, id=NULL on msg w/delim");
-    ok (flux_msg_push_route (msg, "sender") == 0 && flux_msg_frames (msg) == 3,
-        "flux_msg_push_route works and adds a frame");
-    ok ((flux_msg_get_route_count (msg) == 1),
-        "flux_msg_get_route_count returns 1 on msg w/delim+id");
-
-    ok (flux_msg_get_route_first (msg, &s) == 0 && s != NULL,
-        "flux_msg_get_route_first works");
+
+    flux_msg_route_clear (msg);
+    ok (msg_frames (msg) == 1,
+        "flux_msg_route_clear works, is no-op on msg w/o routes enabled");
+    flux_msg_route_disable (msg);
+    ok (msg_frames (msg) == 1,
+        "flux_msg_route_disable works, is no-op on msg w/o routes enabled");
+    flux_msg_route_enable (msg);
+    ok (msg_frames (msg) == 2,
+        "flux_msg_route_enable works, adds one frame on msg w/ routes enabled");
+    ok ((flux_msg_route_count (msg) == 0),
+        "flux_msg_route_count returns 0 on msg w/o routes");
+
+    ok ((route = flux_msg_route_first (msg)) == NULL,
+        "flux_msg_route_first returns NULL on msg w/o routes");
+    ok ((route = flux_msg_route_last (msg)) == NULL,
+        "flux_msg_route_last returns NULL on msg w/o routes");
+    ok (flux_msg_route_push (msg, "sender") == 0 && msg_frames (msg) == 3,
+        "flux_msg_route_push works and adds a frame");
+    ok ((flux_msg_route_count (msg) == 1),
+        "flux_msg_route_count returns 1 on msg w/ id1");
+
+    ok ((route = flux_msg_route_first (msg)) != NULL,
+        "flux_msg_route_first works");
+    like (route, "sender",
+        "flux_msg_route_first returns id on msg w/ id1");
+
+    ok ((route = flux_msg_route_last (msg)) != NULL,
+        "flux_msg_route_last works");
+    like (route, "sender",
+        "flux_msg_route_last returns id on msg w/ id1");
+
+    ok ((s = flux_msg_route_string (msg)) != NULL,
+        "flux_msg_route_string works");
     like (s, "sender",
-        "flux_msg_get_route_first returns id on msg w/delim+id");
+        "flux_msg_route_string returns correct string on msg w/ id1");
     free (s);
 
-    ok (flux_msg_get_route_last (msg, &s) == 0 && s != NULL,
-        "flux_msg_get_route_last works");
-    like (s, "sender",
-        "flux_msg_get_route_last returns id on msg w/delim+id");
+    ok (flux_msg_route_push (msg, "router") == 0 && msg_frames (msg) == 4,
+        "flux_msg_route_push works and adds a frame");
+    ok ((flux_msg_route_count (msg) == 2),
+        "flux_msg_route_count returns 2 on msg w/ id1+id2");
+
+    ok ((route = flux_msg_route_first (msg)) != NULL,
+        "flux_msg_route_first works");
+    like (route, "sender",
+        "flux_msg_route_first returns id1 on msg w/ id1+id2");
+
+    ok ((route = flux_msg_route_last (msg)) != NULL,
+        "flux_msg_route_last works");
+    like (route, "router",
+        "flux_msg_route_last returns id2 on message with id1+id2");
+
+    ok ((s = flux_msg_route_string (msg)) != NULL,
+        "flux_msg_route_string works");
+    like (s, "sender!router",
+        "flux_msg_route_string returns correct string on msg w/ id1+id2");
     free (s);
 
-    ok (flux_msg_push_route (msg, "router") == 0 && flux_msg_frames (msg) == 4,
-        "flux_msg_push_route works and adds a frame");
-    ok ((flux_msg_get_route_count (msg) == 2),
-        "flux_msg_get_route_count returns 2 on msg w/delim+id1+id2");
+    ok (flux_msg_route_delete_last (msg) == 0 && msg_frames (msg) == 3,
+        "flux_msg_route_delete_last works and removed a frame");
+    ok (flux_msg_route_count (msg) == 1,
+        "flux_msg_route_count returns 1 on message w/ id1");
 
-    ok (flux_msg_get_route_first (msg, &s) == 0 && s != NULL,
-        "flux_msg_get_route_first works");
-    like (s, "sender",
-        "flux_msg_get_route_first returns id1 on msg w/delim+id1+id2");
-    free (s);
+    flux_msg_route_clear (msg);
+    ok (flux_msg_route_count (msg) == 0,
+        "flux_msg_route_clear clear routing frames");
+    ok (msg_frames (msg) == 2,
+        "flux_msg_route_clear did not disable routing frames");
 
-    ok (flux_msg_get_route_last (msg, &s) == 0 && s != NULL,
-        "flux_msg_get_route_last works");
-    like (s, "router",
-        "flux_msg_get_route_last returns id2 on message with delim+id1+id2");
-    free (s);
+    ok (flux_msg_route_push (msg, "foobar") == 0 && msg_frames (msg) == 3,
+        "flux_msg_route_push works and adds a frame after flux_msg_route_clear()");
+    ok ((flux_msg_route_count (msg) == 1),
+        "flux_msg_route_count returns 1 on msg w/ id1");
 
-    s = NULL;
-    ok (flux_msg_pop_route (msg, &s) == 0 && s != NULL,
-        "flux_msg_pop_route works on msg w/routes");
-    like (s, "router",
-        "flux_msg_pop_routet returns id2 on message with delim+id1+id2");
-    free (s);
+    flux_msg_route_disable (msg);
+    ok (msg_frames (msg) == 1,
+        "flux_msg_route_disable clear routing frames");
+
+    ok (flux_msg_route_push (msg, "boobar") < 0 && errno == EPROTO,
+        "flux_msg_route_push fails with EPROTO after flux_msg_route_disable()");
 
-    ok (flux_msg_clear_route (msg) == 0 && flux_msg_frames (msg) == 1,
-        "flux_msg_clear_route strips routing frames and delim");
     flux_msg_destroy (msg);
+
+    msg = flux_msg_create (FLUX_MSGTYPE_REQUEST);
+    flux_msg_t *msg2 = flux_msg_create (FLUX_MSGTYPE_REQUEST);
+    if (!msg || !msg2)
+        BAIL_OUT ("flux_msg_create failed");
+    flux_msg_route_enable (msg);
+    flux_msg_route_enable (msg2);
+    ok (flux_msg_route_match_first (msg, msg2) == true,
+        "flux_msg_route_match_first returns true on messages with no routes");
+    if (flux_msg_route_push (msg, "foobar") < 0)
+        BAIL_OUT ("flux_msg_route_push failed");
+    ok (flux_msg_route_match_first (msg, msg2) == false,
+        "flux_msg_route_match_first returns false on route and no route");
+    if (flux_msg_route_push (msg2, "foobar") < 0)
+        BAIL_OUT ("flux_msg_route_push failed");
+    ok (flux_msg_route_match_first (msg, msg2) == true,
+        "flux_msg_route_match_first returns true if routes match");
+    if (flux_msg_route_push (msg2, "bar") < 0)
+        BAIL_OUT ("flux_msg_route_push failed");
+    ok (flux_msg_route_match_first (msg, msg2) == true,
+        "flux_msg_route_match_first still returns true with more routes pushed");
+
+    flux_msg_destroy (msg);
+    flux_msg_destroy (msg2);
 }
 
 /* flux_msg_get_topic, flux_msg_set_topic on message with and without routes
@@ -110,9 +434,6 @@ void check_topic (void)
 
     ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL,
        "flux_msg_create works");
-    errno = 0;
-    ok (flux_msg_get_topic (msg, &s) < 0 && errno == EPROTO,
-       "flux_msg_get_topic fails with EPROTO on msg w/o topic");
     ok (flux_msg_set_topic (msg, "blorg") == 0,
        "flux_msg_set_topic works");
     ok (flux_msg_get_topic (msg, &s) == 0,
@@ -120,10 +441,9 @@ void check_topic (void)
     like (s, "blorg",
        "and we got back the topic string we set");
 
-    ok (flux_msg_enable_route (msg) == 0,
-        "flux_msg_enable_route works");
-    ok (flux_msg_push_route (msg, "id1") == 0,
-        "flux_msg_push_route works");
+    flux_msg_route_enable (msg);
+    ok (flux_msg_route_push (msg, "id1") == 0,
+        "flux_msg_route_push works");
     ok (flux_msg_get_topic (msg, &s) == 0,
        "flux_msg_get_topic still works, with routes");
     like (s, "blorg",
@@ -145,15 +465,38 @@ void check_payload_json (void)
     ok (flux_msg_get_string (msg, &s) == 0 && s == NULL,
        "flux_msg_get_string returns success with no payload");
 
+    ok (strlen(flux_msg_last_error (msg)) == 0,
+        "flux_msg_last_error() returns empty string before pack/unpack");
+
+    is (flux_msg_last_error (NULL), "msg object is NULL",
+        "flux_msg_last_error() returns 'msg object is NULL' on NULL arg");
+
+    /* Unpack on a message with invalid string payload should be an error
+     */
+    errno = 0;
+    ok (flux_msg_set_payload (msg, "fluffy", 6) == 0,
+        "set invalid string payload on msg");
+    errno = 0;
+    ok (flux_msg_unpack (msg, "{s:i}", "foo", &i) < 0 && errno == EPROTO,
+        "flux_msg_unpack() on message with invalid payload returns EPROTO");
+    is (flux_msg_last_error (msg),
+        "flux_msg_get_string: Protocol error",
+        "flux_msg_last_error reports '%s'",
+        flux_msg_last_error(msg));
+
     /* RFC 3 - json payload must be an object
      * Encoding should return EINVAL.
      */
     errno = 0;
     ok (flux_msg_pack (msg, "[1,2,3]") < 0 && errno == EINVAL,
        "flux_msg_pack array fails with EINVAL");
+    ok (strlen(flux_msg_last_error (msg)) > 0,
+        "flux_msg_last_error: %s", flux_msg_last_error (msg));
     errno = 0;
     ok (flux_msg_pack (msg, "3.14") < 0 && errno == EINVAL,
        "flux_msg_pack scalar fails with EINVAL");
+    ok (strlen(flux_msg_last_error (msg)) > 0,
+        "flux_msg_last_error: %s", flux_msg_last_error (msg));
 
     /* Sneak in a malformed JSON payloads and test decoding.
      * 1) array
@@ -163,6 +506,8 @@ void check_payload_json (void)
     errno = 0;
     ok (flux_msg_unpack (msg, "o", &o) < 0 && errno == EPROTO,
         "flux_msg_unpack array fails with EPROTO");
+    ok (strlen(flux_msg_last_error (msg)) > 0,
+        "flux_msg_last_error: %s", flux_msg_last_error (msg));
     /* 2) bare value
      */
     if (flux_msg_set_string (msg, "3.14") < 0)
@@ -170,6 +515,8 @@ void check_payload_json (void)
     errno = 0;
     ok (flux_msg_unpack (msg, "o", &o) < 0 && errno == EPROTO,
         "flux_msg_unpack scalar fails with EPROTO");
+    ok (strlen(flux_msg_last_error (msg)) > 0,
+        "flux_msg_last_error: %s", flux_msg_last_error (msg));
     /* 3) malformed object (no trailing })
      */
     if (flux_msg_set_string (msg, "{\"a\":42") < 0)
@@ -177,12 +524,18 @@ void check_payload_json (void)
     errno = 0;
     ok (flux_msg_unpack (msg, "o", &o) < 0 && errno == EPROTO,
         "flux_msg_unpack malformed object fails with EPROTO");
+    ok (strlen(flux_msg_last_error (msg)) > 0,
+        "flux_msg_last_error: %s", flux_msg_last_error (msg));
 
     ok (flux_msg_pack (msg, "{s:i}", "foo", 42) == 0,
        "flux_msg_pack works");
+    ok (strlen(flux_msg_last_error (msg)) == 0,
+        "flux_msg_last_error returns empty string after ok pack");
     i = 0;
     ok (flux_msg_unpack (msg, "{s:i}", "foo", &i) == 0 && i == 42,
        "flux_msg_unpack returns payload intact");
+    ok (strlen(flux_msg_last_error (msg)) == 0,
+        "flux_msg_last_error returns empty string after ok unpack");
 
     flux_msg_destroy (msg);
 }
@@ -198,20 +551,30 @@ void check_payload_json_formatted (void)
     errno = 0;
     ok (flux_msg_unpack (msg, "{}") < 0 && errno == EPROTO,
         "flux_msg_unpack fails with EPROTO with no payload");
+    ok (strlen (flux_msg_last_error (msg)) > 0,
+        "flux_msg_last_error: %s", flux_msg_last_error (msg));
 
     errno = 0;
     ok (flux_msg_pack (msg, "[i,i,i]", 1,2,3) < 0 && errno == EINVAL,
         "flux_msg_pack array fails with EINVAL");
+    is (flux_msg_last_error (msg), "payload is not a JSON object",
+        "flux_msg_last_error: %s", flux_msg_last_error (msg));
     errno = 0;
     ok (flux_msg_pack (msg, "i", 3.14) < 0 && errno == EINVAL,
        "flux_msg_pack scalar fails with EINVAL");
+    ok (strlen (flux_msg_last_error (msg)) > 0,
+        "flux_msg_last_error: %s", flux_msg_last_error (msg));
     ok (flux_msg_pack (msg, "{s:i, s:s}", "foo", 42, "bar", "baz") == 0,
        "flux_msg_pack object works");
+    ok (strlen (flux_msg_last_error (msg)) == 0,
+        "flux_msg_last_error is empty string after ok pack");
     i = 0;
     s = NULL;
     ok (flux_msg_unpack (msg, "{s:i, s:s}", "foo", &i, "bar", &s) == 0,
        "flux_msg_unpack object works");
-    ok (i == 42 && s != NULL && !strcmp (s, "baz"),
+    ok (strlen (flux_msg_last_error (msg)) == 0,
+        "flux_msg_last_error is empty string after ok unpack");
+    ok (i == 42 && s != NULL && streq (s, "baz"),
         "decoded content matches encoded content");
 
     /* reset payload */
@@ -221,27 +584,44 @@ void check_payload_json_formatted (void)
     s = NULL;
     ok (flux_msg_unpack (msg, "{s:i, s:s}", "foo", &i, "bar", &s) == 0,
        "flux_msg_unpack object works");
-    ok (i == 43 && s != NULL && !strcmp (s, "smurf"),
+    ok (i == 43 && s != NULL && streq (s, "smurf"),
         "decoded content matches new encoded content");
 
     i = 0;
     s = NULL;
     ok (flux_msg_unpack (msg, "{s:s, s:i}", "bar", &s, "foo", &i) == 0,
        "flux_msg_unpack object works out of order");
-    ok (i == 43 && s != NULL && !strcmp (s, "smurf"),
+    ok (i == 43 && s != NULL && streq (s, "smurf"),
         "decoded content matches new encoded content");
 
-    errno = 0;
-    ok (flux_msg_unpack (msg, NULL) < 0 && errno == EINVAL,
-        "flux_msg_unpack fails with EINVAL with NULL format");
+    ok (strlen (flux_msg_last_error (msg)) == 0,
+        "flux_msg_last_error is empty string on EINVAL");
 
     errno = 0;
     ok (flux_msg_unpack (msg, "") < 0 && errno == EINVAL,
         "flux_msg_unpack fails with EINVAL with \"\" format");
+    ok (strlen (flux_msg_last_error (msg)) == 0,
+        "flux_msg_last_error is empty string on EINVAL");
 
     errno = 0;
     ok (flux_msg_unpack (msg, "{s:s}", "nope", &s) < 0 && errno == EPROTO,
         "flux_msg_unpack fails with EPROTO with nonexistent key");
+    ok (strlen (flux_msg_last_error (msg)) > 0,
+        "flux_msg_last_error is %s", flux_msg_last_error (msg));
+
+    /* flux_msg_pack/unpack doesn't reject packed NUL chars */
+    char buf[4] = "foo";
+    char *result = NULL;
+    size_t len = -1;
+
+    ok (flux_msg_pack (msg, "{ss#}", "result", buf, 4) == 0,
+        "flux_msg_pack with NUL char works");
+    ok (flux_msg_unpack (msg, "{ss%}", "result", &result, &len) == 0,
+        "flux_msg_unpack with NUL char works");
+    ok (len == 4,
+        "flux_msg_unpack returned correct length");
+    ok (memcmp (buf, result, 4) == 0,
+        "original buffer and result match");
 
     flux_msg_destroy (msg);
 }
@@ -254,14 +634,12 @@ void check_payload (void)
     flux_msg_t *msg;
     const void *buf;
     void *pay[1024];
-    int plen = sizeof (pay), len;
+    size_t plen = sizeof (pay);
+    size_t len;
 
     ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL,
        "flux_msg_create works");
     errno = 0;
-    ok (flux_msg_get_payload (msg, &buf, &len) < 0 && errno == EPROTO,
-       "flux_msg_get_payload fails with EPROTO on msg w/o payload");
-    errno = 0;
     ok (flux_msg_set_payload (msg, NULL, 0) == 0 && errno == 0,
         "flux_msg_set_payload NULL works with no payload");
     errno = 0;
@@ -271,7 +649,7 @@ void check_payload (void)
     errno = 0;
     memset (pay, 42, plen);
     ok (flux_msg_set_payload (msg, pay, plen) == 0
-        && flux_msg_frames (msg) == 2,
+        && msg_frames (msg) == 2,
        "flux_msg_set_payload works");
 
     len = 0; buf = NULL; errno = 0;
@@ -281,7 +659,7 @@ void check_payload (void)
     cmp_mem (buf, pay, len,
        "and we got back the payload we set");
 
-    ok (flux_msg_set_topic (msg, "blorg") == 0 && flux_msg_frames (msg) == 3,
+    ok (flux_msg_set_topic (msg, "blorg") == 0 && msg_frames (msg) == 3,
        "flux_msg_set_topic works");
     len = 0; buf = NULL; errno = 0;
     ok (flux_msg_get_payload (msg, &buf, &len) == 0
@@ -289,13 +667,14 @@ void check_payload (void)
        "flux_msg_get_payload works with topic");
     cmp_mem (buf, pay, len,
        "and we got back the payload we set");
-    ok (flux_msg_set_topic (msg, NULL) == 0 && flux_msg_frames (msg) == 2,
+    ok (flux_msg_set_topic (msg, NULL) == 0 && msg_frames (msg) == 2,
        "flux_msg_set_topic NULL works");
 
-    ok (flux_msg_enable_route (msg) == 0 && flux_msg_frames (msg) == 3,
-        "flux_msg_enable_route works");
-    ok (flux_msg_push_route (msg, "id1") == 0 && flux_msg_frames (msg) == 4,
-        "flux_msg_push_route works");
+    flux_msg_route_enable (msg);
+    ok (msg_frames (msg) == 3,
+        "flux_msg_route_enable works");
+    ok (flux_msg_route_push (msg, "id1") == 0 && msg_frames (msg) == 4,
+        "flux_msg_route_push works");
 
     len = 0; buf = NULL; errno = 0;
     ok (flux_msg_get_payload (msg, &buf, &len) == 0
@@ -304,7 +683,7 @@ void check_payload (void)
     cmp_mem (buf, pay, len,
        "and we got back the payload we set");
 
-    ok (flux_msg_set_topic (msg, "blorg") == 0 && flux_msg_frames (msg) == 5,
+    ok (flux_msg_set_topic (msg, "blorg") == 0 && msg_frames (msg) == 5,
        "flux_msg_set_topic works");
     len = 0; buf = NULL; errno = 0;
     ok (flux_msg_get_payload (msg, &buf, &len) == 0
@@ -338,6 +717,7 @@ void check_payload (void)
 /* flux_msg_set_type, flux_msg_get_type
  * flux_msg_set_nodeid, flux_msg_get_nodeid
  * flux_msg_set_errnum, flux_msg_get_errnum
+ *
  */
 void check_proto (void)
 {
@@ -505,39 +885,6 @@ void check_security (void)
         && errno == EPERM,
         "flux_msg_authorize denies guest (EPERM) when role=NONE");
 
-    /* Elicit EINVAL from bad args.
-     */
-    errno = 0;
-    ok (flux_msg_get_cred (msg, NULL) < 0 && errno == EINVAL,
-        "flux_msg_get_cred cred=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_get_cred (NULL, &cred) < 0 && errno == EINVAL,
-        "flux_msg_get_cred msg=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_set_cred (NULL, cred) < 0 && errno == EINVAL,
-        "flux_msg_set_cred msg=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_get_userid (msg, NULL) < 0 && errno == EINVAL,
-        "flux_msg_get_userid userid=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_get_userid (NULL, &cred.userid) < 0 && errno == EINVAL,
-        "flux_msg_get_userid msg=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_get_rolemask (msg, NULL) < 0 && errno == EINVAL,
-        "flux_msg_get_rolemask rolemask=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_get_rolemask (NULL, &cred.rolemask) < 0 && errno == EINVAL,
-        "flux_msg_get_rolemask msg=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_set_rolemask (NULL, cred.rolemask) < 0 && errno == EINVAL,
-        "flux_msg_set_rolemask msg=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_set_userid (NULL, cred.userid) < 0 && errno == EINVAL,
-        "flux_msg_set_userid msg=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_authorize (NULL, 42) < 0 && errno == EINVAL,
-        "flux_msg_authorize msg=NULL fails with EINVAL");
-
     flux_msg_destroy (msg);
 }
 
@@ -586,6 +933,7 @@ void check_cmp (void)
 void check_encode (void)
 {
     flux_msg_t *msg, *msg2;
+    uint8_t smallbuf[1];
     void *buf;
     size_t size;
     const char *topic;
@@ -595,6 +943,9 @@ void check_encode (void)
         "flux_msg_create works");
     ok (flux_msg_set_topic (msg, "foo.bar") == 0,
         "flux_msg_set_topic works");
+    errno = 0;
+    ok (flux_msg_encode (msg, smallbuf, 1) < 0 && errno == EINVAL,
+        "flux_msg_encode fails on EINVAL with buffer too small");
     size = flux_msg_encode_size (msg);
     ok (size > 0,
         "flux_msg_encode_size works");
@@ -607,7 +958,7 @@ void check_encode (void)
     free (buf);
     ok (flux_msg_get_type (msg2, &type) == 0 && type == FLUX_MSGTYPE_REQUEST,
         "decoded expected message type");
-    ok (flux_msg_get_topic (msg2, &topic) == 0 && !strcmp (topic, "foo.bar"),
+    ok (flux_msg_get_topic (msg2, &topic) == 0 && streq (topic, "foo.bar"),
         "decoded expected topic string");
     ok (flux_msg_has_payload (msg2) == false,
         "decoded expected (lack of) payload");
@@ -616,66 +967,6 @@ void check_encode (void)
     flux_msg_destroy (msg2);
 }
 
-void check_sendzsock (void)
-{
-    zsock_t *zsock[2] = { NULL, NULL };
-    flux_msg_t *msg, *msg2;
-    const char *topic;
-    int type;
-    const char *uri = "inproc://test";
-
-    /* zsys boiler plate:
-     * appears to be needed to avoid atexit assertions when lives_ok()
-     * macro (which calls fork()) is used.
-     */
-    zsys_init ();
-    zsys_set_logstream (stderr);
-    zsys_set_logident ("test_message.t");
-    zsys_handler_set (NULL);
-    zsys_set_linger (5); // msec
-
-    ok ((zsock[0] = zsock_new_pair (NULL)) != NULL
-                    && zsock_bind (zsock[0], "%s", uri) == 0
-                    && (zsock[1] = zsock_new_pair (uri)) != NULL,
-        "got inproc socket pair");
-
-    ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL
-            && flux_msg_set_topic (msg, "foo.bar") == 0,
-        "created test message");
-
-    ok (flux_msg_sendzsock (zsock[1], msg) == 0,
-        "flux_msg_sendzsock works");
-    ok ((msg2 = flux_msg_recvzsock (zsock[0])) != NULL,
-        "flux_msg_recvzsock works");
-    ok (flux_msg_get_type (msg2, &type) == 0 && type == FLUX_MSGTYPE_REQUEST
-            && flux_msg_get_topic (msg2, &topic) == 0
-            && !strcmp (topic, "foo.bar")
-            && flux_msg_has_payload (msg2) == false,
-        "decoded message looks like what was sent");
-    flux_msg_destroy (msg2);
-
-    /* Send it again.
-     */
-    ok (flux_msg_sendzsock (zsock[1], msg) == 0,
-        "try2: flux_msg_sendzsock works");
-    ok ((msg2 = flux_msg_recvzsock (zsock[0])) != NULL,
-        "try2: flux_msg_recvzsock works");
-    ok (flux_msg_get_type (msg2, &type) == 0 && type == FLUX_MSGTYPE_REQUEST
-            && flux_msg_get_topic (msg2, &topic) == 0
-            && !strcmp (topic, "foo.bar")
-            && flux_msg_has_payload (msg2) == false,
-        "try2: decoded message looks like what was sent");
-    flux_msg_destroy (msg2);
-    flux_msg_destroy (msg);
-
-    zsock_destroy (&zsock[0]);
-    zsock_destroy (&zsock[1]);
-
-    /* zsys boiler plate - see note above
-     */
-    zsys_shutdown();
-}
-
 void *myfree_arg = NULL;
 void myfree (void *arg)
 {
@@ -689,18 +980,12 @@ void check_aux (void)
 
     ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL,
         "created test message");
-    errno = 0;
-    ok (flux_msg_aux_set (NULL, "foo", "bar", NULL) < 0 && errno == EINVAL,
-        "flux_msg_aux_set msg=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_aux_get (NULL, "foo") == NULL && errno == EINVAL,
-        "flux_msg_aux_get msg=NULL fails with EINVAL");
     ok (flux_msg_aux_set (msg, "test", test_data, myfree) == 0,
         "hang aux data member on message with destructor");
     ok (flux_msg_aux_get (msg, "incorrect") == NULL,
         "flux_msg_aux_get for unknown key returns NULL");
     ok (flux_msg_aux_get (msg, "test") == test_data,
-        "flux_msg_aux_get aux data memeber key returns orig pointer");
+        "flux_msg_aux_get aux data member key returns orig pointer");
     flux_msg_destroy (msg);
     ok (myfree_arg == test_data,
         "destroyed message and aux destructor was called");
@@ -711,27 +996,31 @@ void check_copy (void)
     flux_msg_t *msg, *cpy;
     int type;
     const char *topic;
-    int cpylen;
+    size_t cpylen;
     const char buf[] = "xxxxxxxxxxxxxxxxxx";
     const void *cpybuf;
+    const char *s;
 
-    ok ((msg = flux_msg_create (FLUX_MSGTYPE_KEEPALIVE)) != NULL,
-        "created no-payload keepalive");
+    ok ((msg = flux_msg_create (FLUX_MSGTYPE_CONTROL)) != NULL,
+        "created no-payload control");
     ok ((cpy = flux_msg_copy (msg, true)) != NULL,
         "flux_msg_copy works");
     flux_msg_destroy (msg);
     type = -1;
-    ok (flux_msg_get_type (cpy, &type) == 0 && type == FLUX_MSGTYPE_KEEPALIVE
+    ok (flux_msg_get_type (cpy, &type) == 0&& type == FLUX_MSGTYPE_CONTROL
              && !flux_msg_has_payload (cpy)
-             && flux_msg_get_route_count (cpy) < 0
+             && flux_msg_route_count (cpy) < 0
              && flux_msg_get_topic (cpy, &topic) < 0,
         "copy is keepalive: no routes, topic, or payload");
     flux_msg_destroy (cpy);
 
     ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL,
         "created request");
-    ok (flux_msg_enable_route (msg) == 0,
-        "added route delim");
+    flux_msg_route_enable (msg);
+    ok (flux_msg_route_push (msg, "first") == 0,
+        "added route 1");
+    ok (flux_msg_route_push (msg, "second") == 0,
+        "added route 2");
     ok (flux_msg_set_topic (msg, "foo") == 0,
         "set topic string");
     ok (flux_msg_set_payload (msg, buf, sizeof (buf)) == 0,
@@ -743,9 +1032,24 @@ void check_copy (void)
              && flux_msg_has_payload (cpy)
              && flux_msg_get_payload (cpy, &cpybuf, &cpylen) == 0
              && cpylen == sizeof (buf) && memcmp (cpybuf, buf, cpylen) == 0
-             && flux_msg_get_route_count (cpy) == 0
-             && flux_msg_get_topic (cpy, &topic) == 0 && !strcmp (topic,"foo"),
-        "copy is request: w/route delim, topic, and payload");
+             && flux_msg_route_count (cpy) == 2
+             && flux_msg_get_topic (cpy, &topic) == 0 && streq (topic,"foo"),
+        "copy is request: w/routes, topic, and payload");
+
+    ok ((s = flux_msg_route_last (cpy)) != NULL,
+        "flux_msg_route_last gets route from copy");
+    like (s, "second",
+          "flux_msg_route_last returns correct second route");
+    ok (flux_msg_route_delete_last (cpy) == 0,
+        "flux_msg_route_delete_last removes second route");
+
+    ok ((s = flux_msg_route_last (cpy)) != NULL,
+        "flux_msg_route_last pops route from copy");
+    like (s, "first",
+          "flux_msg_route_last returns correct first route");
+    ok (flux_msg_route_delete_last (cpy) == 0,
+        "flux_msg_route_delete_last removes first route");
+
     flux_msg_destroy (cpy);
 
     ok ((cpy = flux_msg_copy (msg, false)) != NULL,
@@ -753,9 +1057,9 @@ void check_copy (void)
     type = -1;
     ok (flux_msg_get_type (cpy, &type) == 0 && type == FLUX_MSGTYPE_REQUEST
              && !flux_msg_has_payload (cpy)
-             && flux_msg_get_route_count (cpy) == 0
-             && flux_msg_get_topic (cpy, &topic) == 0 && !strcmp (topic,"foo"),
-        "copy is request: w/route delim, topic, and no payload");
+             && flux_msg_route_count (cpy) == 2
+             && flux_msg_get_topic (cpy, &topic) == 0 && streq (topic,"foo"),
+        "copy is request: w/routes, topic, and no payload");
     flux_msg_destroy (cpy);
     flux_msg_destroy (msg);
 }
@@ -763,15 +1067,17 @@ void check_copy (void)
 void check_print (void)
 {
     flux_msg_t *msg;
+    char *strpayload = "a.special.payload";
     char buf[] = "xxxxxxxx";
-    FILE *f = fopen ("/dev/null", "w");
+    char buf_long[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+    FILE *f = verbose ? stderr : fopen ("/dev/null", "w");
     if (!f)
         BAIL_OUT ("cannot open /dev/null for writing");
 
-    ok ((msg = flux_msg_create (FLUX_MSGTYPE_KEEPALIVE)) != NULL,
+    ok ((msg = flux_msg_create (FLUX_MSGTYPE_CONTROL)) != NULL,
         "created test message");
-    lives_ok ({flux_msg_fprint (f, msg);},
-        "flux_msg_fprint doesn't segfault on keepalive");
+    lives_ok ({flux_msg_fprint_ts (f, msg, 0.);},
+        "flux_msg_fprint_ts doesn't segfault on control");
     flux_msg_destroy (msg);
 
     ok ((msg = flux_msg_create (FLUX_MSGTYPE_EVENT)) != NULL,
@@ -786,9 +1092,8 @@ void check_print (void)
         "created test message");
     ok (flux_msg_set_topic (msg, "foo.bar") == 0,
         "set topic string");
-    ok (flux_msg_enable_route (msg) == 0,
-        "enabled routing");
-    ok (flux_msg_push_route (msg, "id1") == 0,
+    flux_msg_route_enable (msg);
+    ok (flux_msg_route_push (msg, "id1") == 0,
         "added one route");
     ok (flux_msg_set_payload (msg, buf, strlen (buf)) == 0,
         "added payload");
@@ -796,10 +1101,53 @@ void check_print (void)
         "flux_msg_fprint doesn't segfault on fully loaded request");
     flux_msg_destroy (msg);
 
+    ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL,
+        "created test message");
+    ok (flux_msg_set_userid (msg, 42) == 0,
+        "set userid");
+    ok (flux_msg_set_rolemask (msg, FLUX_ROLE_OWNER) == 0,
+        "set rolemask");
+    ok (flux_msg_set_nodeid (msg, 42) == 0,
+        "set nodeid");
+    ok (flux_msg_set_string (msg, strpayload) == 0,
+        "added payload");
+    lives_ok ({flux_msg_fprint (f, msg);},
+        "flux_msg_fprint doesn't segfault on request settings #1");
+    flux_msg_destroy (msg);
+
+    ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL,
+        "created test message");
+    ok (flux_msg_set_rolemask (msg, FLUX_ROLE_USER) == 0,
+        "set rolemask");
+    ok (flux_msg_set_flag (msg, FLUX_MSGFLAG_NORESPONSE) == 0
+        && flux_msg_set_flag (msg, FLUX_MSGFLAG_UPSTREAM) == 0,
+        "set new flags");
+    lives_ok ({flux_msg_fprint (f, msg);},
+        "flux_msg_fprint doesn't segfault on request settings #2");
+    flux_msg_destroy (msg);
+
+    ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL,
+        "created test message");
+    ok (flux_msg_set_rolemask (msg, FLUX_ROLE_ALL) == 0,
+        "set rolemask");
+    ok (flux_msg_set_flag (msg, FLUX_MSGFLAG_PRIVATE) == 0
+        && flux_msg_set_flag (msg, FLUX_MSGFLAG_STREAMING) == 0,
+        "set new flags");
+    lives_ok ({flux_msg_fprint (f, msg);},
+        "flux_msg_fprint doesn't segfault on request settings #3");
+    flux_msg_destroy (msg);
+
+    ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL,
+        "created test message");
+    ok (flux_msg_set_payload (msg, buf_long, strlen (buf_long)) == 0,
+        "added long payload");
+    lives_ok ({flux_msg_fprint (f, msg);},
+        "flux_msg_fprint doesn't segfault on long payload");
+    flux_msg_destroy (msg);
+
     ok ((msg = flux_msg_create (FLUX_MSGTYPE_RESPONSE)) != NULL,
         "created test message");
-    ok (flux_msg_enable_route (msg) == 0,
-        "enabled routing");
+    flux_msg_route_enable (msg);
     lives_ok ({flux_msg_fprint (f, msg);},
         "flux_msg_fprint doesn't segfault on response with empty route stack");
     flux_msg_destroy (msg);
@@ -807,31 +1155,46 @@ void check_print (void)
     fclose (f);
 }
 
-void check_params (void)
+void check_print_rolemask (void)
 {
     flux_msg_t *msg;
-
-    if (!(msg = flux_msg_create (FLUX_MSGTYPE_EVENT)))
-        BAIL_OUT ("flux_msg_create failed");
-    errno = 0;
-    ok (flux_msg_set_payload (NULL, NULL, 0) < 0 && errno == EINVAL,
-        "flux_msg_set_payload msg=NULL fails with EINVAL");
-
+    FILE *fp;
+    uint32_t rolemask;
+    char *buf = NULL;
+    size_t size = 0;
+
+    rolemask = FLUX_ROLE_LOCAL | FLUX_ROLE_USER | 0x10;
+    if (!(msg = flux_msg_create (FLUX_MSGTYPE_REQUEST))
+        || flux_msg_set_rolemask (msg, rolemask) < 0)
+        BAIL_OUT ("failed to create test request");
+    if (!(fp = open_memstream (&buf, &size)))
+        BAIL_OUT ("open_memstream failed");
+    flux_msg_fprint (fp, msg);
+    fclose (fp); // close flushes content
+    diag ("%s", buf);
+    ok (buf && strstr (buf, "rolemask=user,local,0x10") != NULL,
+        "flux_msg_fprint() rolemask string is correct");
+    free (buf);
     flux_msg_destroy (msg);
 }
 
 void check_flags (void)
 {
     flux_msg_t *msg;
-    uint8_t flags;
 
     if (!(msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
         BAIL_OUT ("flux_msg_create failed");
-    ok (flux_msg_get_flags (msg, &flags) == 0,
-        "flux_msg_get_flags works");
-    ok (flags == 0,
+    ok (msg->proto.flags == 0,
         "flags are initially zero");
 
+    /* FLUX_MSGFLAG_USER1 */
+    ok (flux_msg_has_flag (msg, FLUX_MSGFLAG_USER1) == false,
+        "flux_msg_has_flag FLUX_MSGFLAG_USER1 = false");
+    ok (flux_msg_set_flag (msg, FLUX_MSGFLAG_USER1) == 0,
+        "flux_msg_set_flag FLUX_MSGFLAG_USER1 works");
+    ok (flux_msg_has_flag (msg, FLUX_MSGFLAG_USER1) == true,
+        "flux_msg_has_flag FLUX_MSGFLAG_USER1 = true");
+
     /* FLUX_MSGFLAG_PRIVATE */
     ok (flux_msg_is_private (msg) == false,
         "flux_msg_is_private = false");
@@ -848,50 +1211,39 @@ void check_flags (void)
     ok (flux_msg_is_streaming (msg) == true,
         "flux_msg_is_streaming = true");
 
+    /* FLUX_MSGFLAG_NORESPONSE */
+    ok (flux_msg_is_noresponse (msg) == false,
+        "flux_msg_is_noresponse = false");
+    ok (flux_msg_set_noresponse (msg) == 0,
+        "flux_msg_set_noresponse_works");
+    ok (flux_msg_is_noresponse (msg) == true,
+        "flux_msg_is_noresponse = true");
+
+    /* noresponse and streaming are mutually exclusive */
+    ok (flux_msg_set_streaming (msg) == 0
+        && flux_msg_set_noresponse (msg) == 0
+        && flux_msg_is_streaming (msg) == false
+        && flux_msg_is_noresponse (msg) == true,
+        "flux_msg_set_noresponse clears streaming flag");
+    ok (flux_msg_set_noresponse (msg) == 0
+        && flux_msg_set_streaming (msg) == 0
+        && flux_msg_is_noresponse (msg) == false
+        && flux_msg_is_streaming (msg) == true,
+        "flux_msg_set_streaming clears noresponse flag");
+
     ok (flux_msg_set_topic (msg, "foo") == 0
-        && flux_msg_get_flags (msg, &flags) == 0
-        && (flags & FLUX_MSGFLAG_TOPIC),
+        && flux_msg_has_flag (msg, FLUX_MSGFLAG_TOPIC),
         "flux_msg_set_topic sets FLUX_MSGFLAG_TOPIC");
 
     ok (flux_msg_set_payload (msg, "foo", 3) == 0
-        && flux_msg_get_flags (msg, &flags) == 0
-        && (flags & FLUX_MSGFLAG_PAYLOAD),
+        && flux_msg_has_flag (msg, FLUX_MSGFLAG_PAYLOAD),
         "flux_msg_set_payload sets FLUX_MSGFLAG_PAYLOAD");
 
-    ok (flux_msg_enable_route (msg) == 0
-        && flux_msg_get_flags (msg, &flags) == 0
-        && (flags & FLUX_MSGFLAG_ROUTE),
-        "flux_msg_enable_route sets FLUX_MSGFLAG_ROUTE");
+    flux_msg_route_enable (msg);
+    ok (flux_msg_has_flag (msg, FLUX_MSGFLAG_ROUTE),
+        "flux_msg_route_enable sets FLUX_MSGFLAG_ROUTE");
 
     flux_msg_destroy (msg);
-
-    /* invalid params checks */
-
-    errno = 0;
-    ok (flux_msg_get_flags (NULL, &flags) < 0 && errno == EINVAL,
-        "flux_msg_get_flags msg=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_get_flags (msg, NULL) < 0 && errno == EINVAL,
-        "flux_msg_get_flags flags=NULL fails with EINVAL");
-
-    errno = 0;
-    ok (flux_msg_set_flags (NULL, 0) < 0 && errno == EINVAL,
-        "flux_msg_set_flags msg=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_msg_set_flags (msg, 0xff) < 0 && errno == EINVAL,
-        "flux_msg_set_flags flags=(invalid) fails with EINVAL");
-
-    errno = 0;
-    ok (flux_msg_set_private (NULL) < 0 && errno == EINVAL,
-        "flux_msg_set_private msg=NULL fails with EINVAL");
-    ok (flux_msg_is_private (NULL) == true,
-        "flux_msg_is_private msg=NULL returns true");
-
-    errno = 0;
-    ok (flux_msg_set_streaming (NULL) < 0 && errno == EINVAL,
-        "flux_msg_set_streaming msg=NULL fails with EINVAL");
-    ok (flux_msg_is_streaming (NULL) == true,
-        "flux_msg_is_streaming msg=NULL returns true");
 }
 
 void check_refcount (void)
@@ -900,31 +1252,149 @@ void check_refcount (void)
     const flux_msg_t *p;
     int type;
 
-    if (!(msg = flux_msg_create (FLUX_MSGTYPE_KEEPALIVE)))
+    if (!(msg = flux_msg_create (FLUX_MSGTYPE_CONTROL)))
         BAIL_OUT ("failed to create test message");
     p = flux_msg_incref (msg);
     ok (p == msg,
         "flux_msg_incref returns pointer to original");
     flux_msg_destroy (msg);
-    ok (flux_msg_get_type (p, &type) == 0 && type == FLUX_MSGTYPE_KEEPALIVE,
+    ok (flux_msg_get_type (p, &type) == 0 && type == FLUX_MSGTYPE_CONTROL,
         "reference remains valid after destroy");
     flux_msg_decref (p);
+}
 
-    errno = 0;
-    p = flux_msg_incref (NULL);
-    ok (p == NULL && errno == EINVAL,
-        "flux_msg_incref msg=NULL fails with EINVAL");
+struct pvec {
+    const char *desc;
+    struct proto p;
+    uint8_t buf[PROTO_SIZE];
+};
+
+// N.B. RFC 3 describes this encoding
+// 4-byte integers are encoded in network order (big endian = MSB first)
+static struct pvec testvec[] = {
+    { "fake test message",
+      { .type = 0xab, .flags = 0xcd,
+        .userid = 0x00010203, .rolemask = 0x04050607,
+        .aux1 = 0x08090a0b, .aux2 = 0x0c0d0e0f },
+      { PROTO_MAGIC, PROTO_VERSION, 0xab, 0xcd,
+        0x00, 0x01, 0x02, 0x03,
+        0x04, 0x05, 0x06, 0x07,
+        0x08, 0x09, 0x0a, 0x0b,
+        0x0c, 0x0d, 0x0e, 0x0f }
+    },
+    { "overlay control disconnect",
+      { .type = FLUX_MSGTYPE_CONTROL, .flags = 0,
+        .userid = 100, .rolemask = FLUX_ROLE_OWNER,
+        .control_type = 2, .control_status = 0 },
+      { PROTO_MAGIC, PROTO_VERSION, 0x08, 0,
+        0, 0, 0, 100,
+        0, 0, 0, 1,
+        0, 0, 0, 2,
+        0, 0, 0, 0 }
+    },
+    { "hello request",
+      { .type = FLUX_MSGTYPE_REQUEST,
+        .flags = FLUX_MSGFLAG_TOPIC | FLUX_MSGFLAG_PAYLOAD | FLUX_MSGFLAG_ROUTE,
+        .userid = 100, .rolemask = FLUX_ROLE_OWNER,
+        .nodeid = FLUX_NODEID_ANY, .matchtag = 0 },
+      { PROTO_MAGIC, PROTO_VERSION, 0x01, 0x0b,
+        0, 0, 0, 100,
+        0, 0, 0, 1,
+        0xff, 0xff, 0xff, 0xff,
+        0, 0, 0, 0 }
+    },
+    { "hello response",
+      { .type = FLUX_MSGTYPE_RESPONSE,
+        .flags = FLUX_MSGFLAG_TOPIC | FLUX_MSGFLAG_PAYLOAD | FLUX_MSGFLAG_ROUTE,
+        .userid = 100, .rolemask = FLUX_ROLE_OWNER,
+        .nodeid = FLUX_NODEID_ANY, .matchtag = 0 },
+      { PROTO_MAGIC, PROTO_VERSION, 0x02, 0x0b,
+        0, 0, 0, 100,
+        0, 0, 0, 1,
+        0xff, 0xff, 0xff, 0xff,
+        0, 0, 0, 0 }
+    },
+};
+
+int check_proto_encode (const struct pvec *pvec)
+{
+    uint8_t buf[PROTO_SIZE];
+
+    if (proto_encode (&pvec->p, buf, sizeof (buf)) < 0) {
+        diag ("proto_encode returned -1");
+        return -1;
+    }
+    int errors = 0;
+    for (int i = 0; i < sizeof (buf); i++) {
+        if (buf[i] != pvec->buf[i]) {
+            diag ("buf[%d]=0x%x != 0x%x", i, buf[i], pvec->buf[i]);
+            errors++;
+        }
+    }
+    if (errors > 0)
+        return -1;
+    return 0;
+}
+int check_proto_decode (const struct pvec *pvec)
+{
+    struct proto p;
+    memset (&p, 0, sizeof (p));
+    if (proto_decode (&p, pvec->buf, PROTO_SIZE) < 0) {
+        diag ("proto_decode returned -1");
+        return -1;
+    }
+    int errors = 0;
+    if (p.type != pvec->p.type) {
+        diag ("proto->type=0x%x != 0x%x", p.type, pvec->p.type);
+        errors++;
+    }
+    if (p.flags != pvec->p.flags) {
+        diag ("proto->flags=0x%x != 0x%x", p.flags, pvec->p.flags);
+        errors++;
+    }
+    if (p.userid != pvec->p.userid) {
+        diag ("proto->userid=0x%x != 0x%x", p.userid, pvec->p.userid);
+        errors++;
+    }
+    if (p.rolemask != pvec->p.rolemask) {
+        diag ("proto->rolemask=0x%x != 0x%x", p.rolemask, pvec->p.rolemask);
+        errors++;
+    }
+    if (p.aux1 != pvec->p.aux1) {
+        diag ("proto->aux1=0x%x != 0x%x", p.aux1, pvec->p.aux1);
+        errors++;
+    }
+    if (p.aux2 != pvec->p.aux2) {
+        diag ("proto->aux2=0x%x != 0x%x", p.aux2, pvec->p.aux2);
+        errors++;
+    }
+    if (errors > 0)
+        return -1;
+    return 0;
+}
 
-    lives_ok ({flux_msg_decref (NULL);},
-        "flux_msg_decref msg=NULL doesnt crash crash");
-    lives_ok ({flux_msg_destroy (NULL);},
-        "flux_msg_destroy msg=NULL doesnt crash crash");
+void check_proto_internal (void)
+{
+    for (int i = 0; i < ARRAY_SIZE (testvec); i++) {
+        ok (check_proto_encode (&testvec[i]) == 0,
+           "proto encode worked on %s", testvec[i].desc);
+        ok (check_proto_decode (&testvec[i]) == 0,
+           "proto decode worked on %s", testvec[i].desc);
+    }
 }
 
 int main (int argc, char *argv[])
 {
+    int opt;
+
+    while ((opt = getopt (argc, argv, "v")) != -1) {
+        if (opt == 'v')
+            verbose = true;
+    }
+
     plan (NO_PLAN);
 
+    check_cornercase ();
     check_proto ();
     check_routes ();
     check_topic ();
@@ -940,13 +1410,13 @@ int main (int argc, char *argv[])
     check_cmp ();
 
     check_encode ();
-    check_sendzsock ();
-
-    check_params ();
 
     check_refcount();
 
-    //check_print ();
+    check_print ();
+    check_print_rolemask ();
+
+    check_proto_internal ();
 
     done_testing();
     return (0);
diff --git a/src/common/libflux/test/module.c b/src/common/libflux/test/module.c
index f63ac3f1fd23..a7bdad6969f7 100644
--- a/src/common/libflux/test/module.c
+++ b/src/common/libflux/test/module.c
@@ -11,143 +11,10 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <sys/param.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-
 #include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
-
-/* N.B. FAKE1 and FAKE2 are defined with -D on the CC command line.
- * They are set to the full path of two test modules, module_fake1.so
- * and module_fake2.so.  module_fake1.so simply defines mod_name to "fake1".
- * module_fake2.so omits the mod_name symbol to cause an error.
- */
-
-int errmsg_count;
-void errmsg_cb (const char *msg, void *arg)
-{
-    diag ("%s", msg);
-    errmsg_count++;
-}
-
-void test_modname (void)
-{
-    char *name;
-
-    name = flux_modname (FAKE1, NULL, NULL);
-    ok (name != NULL && !strcmp (name, "fake1"),
-        "flux_modname path=module_fake1 works");
-
-    errno = 0;
-    errmsg_count = 0;
-    name = flux_modname (FAKE2, errmsg_cb, NULL);
-    ok (name == NULL && errno == EINVAL && errmsg_count == 1,
-        "flux_modname path=module_fake2 fails with EINVAL and extended error");
-
-    errmsg_count = 0;
-    name = flux_modname (FAKE2, NULL, NULL);
-    ok (name == NULL && errmsg_count == 0,
-        "flux_modname moderr callback can be NULL");
-
-    errno = 0;
-    errmsg_count = 0;
-    name = flux_modname ("/noexist", errmsg_cb, NULL);
-    ok (name == NULL && errno == ENOENT && errmsg_count == 1,
-        "flux_modname path=/noexist fails with ENOENT and extended error");
-
-    errno = 0;
-    errmsg_count = 0;
-    name = flux_modname (NULL, errmsg_cb, NULL);
-    ok (name == NULL && errno == EINVAL && errmsg_count == 0,
-        "flux_modname path=NULL fails with EINVAL and no extended error");
-}
-
-/* modfind test:
- * Create 3 directory 'searchpath' containing symlinks to test modules.
- * module fake1.so is named 'fake1'.
- * module fake2.so does not define the mod_name symbol.
- */
-char dir[3][PATH_MAX + 1];
-char searchpath[3*PATH_MAX + 4];
-char link1[PATH_MAX + 1];
-char link2[PATH_MAX + 1];
-
-void test_modfind_init (void)
-{
-    const char *tmpdir = getenv ("TMPDIR");
-    int i, n;
-
-    if (!tmpdir)
-        tmpdir = "/tmp";
-    for (i = 0; i < 3; i++) {
-        n = snprintf (dir[i], sizeof (dir[i]), "%s/modfind.XXXXXX", tmpdir);
-        if (n >= sizeof (dir[i]))
-            BAIL_OUT ("snprintf buffer overflow");
-        if (!mkdtemp (dir[i]))
-            BAIL_OUT ("mkdtemp: %s", strerror (errno));
-        if (strlen (searchpath) > 0)
-            strcat (searchpath, ":");
-        strcat (searchpath, dir[i]);
-    }
-    /* Symlink test modules into dirs 1 and 2 */
-    n = snprintf (link1, sizeof (link1), "%s/fake1.so", dir[1]);
-    if (n >= sizeof (dir[i]))
-        BAIL_OUT ("snprintf buffer overflow");
-    n = snprintf (link2, sizeof (link2), "%s/fake2.so", dir[2]);
-    if (n >= sizeof (dir[i]))
-        BAIL_OUT ("snprintf buffer overflow");
-    if (symlink (FAKE1, link1) < 0)
-        BAIL_OUT ("symlink %s: %s", link1, strerror (errno));
-    if (symlink (FAKE2, link2) < 0)
-        BAIL_OUT ("symlink %s: %s", link2, strerror (errno));
-}
-
-void test_modfind_fini (void)
-{
-    int i;
-
-    if (unlink (link1) < 0)
-        BAIL_OUT ("unlink %s: %s", link1, strerror (errno));
-    if (unlink (link2) < 0)
-        BAIL_OUT ("unlink %s: %s", link2, strerror (errno));
-    for (i = 0; i < 3; i++) {
-        if (rmdir (dir[i]) < 0)
-            BAIL_OUT ("unlink %s: %s", dir[i], strerror (errno));
-    }
-}
-
-void test_modfind (void)
-{
-    const char *path;
-
-    test_modfind_init ();
-
-    path = flux_modfind (searchpath, "fake1", errmsg_cb, NULL);
-    ok (path != NULL && !strcmp (path, link1),
-        "flux_modfind modname=fake1 returns correct path");
-
-    errno = 0;
-    errmsg_count = 0;
-    path = flux_modfind (searchpath, "fake2", errmsg_cb, NULL);
-    ok (path == NULL && errno == ENOENT && errmsg_count == 1,
-        "flux_modfind modname=fake2 fails with ENOENT and extended error");
-
-    errno = 0;
-    path = flux_modfind (searchpath, NULL, NULL, NULL);
-    ok (path == NULL && errno == EINVAL,
-        "flux_modfind modname=NULL fails with EINVAL");
-
-    errno = 0;
-    path = flux_modfind (NULL, "fake1", NULL, NULL);
-    ok (path == NULL && errno == EINVAL,
-        "flux_modfind searchpath=NULL fails with EINVAL");
-
-    test_modfind_fini ();
-}
+#include "src/common/libtestutil/util.h"
 
 void test_debug (void)
 {
@@ -182,14 +49,29 @@ void test_debug (void)
     flux_handle_destroy (h);
 }
 
+void test_set_running (void)
+{
+    flux_t *h;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop handle");
+
+    ok (flux_module_set_running (h) == 0,
+        "flux_module_set_running returns success");
+    errno = 0;
+    ok (flux_module_set_running (NULL) < 0 && errno == EINVAL,
+        "flux_module_set_running h=NULL fails with EINVAL");
+
+    flux_close (h);
+}
+
 int main (int argc, char *argv[])
 {
 
     plan (NO_PLAN);
 
-    test_modname ();
-    test_modfind ();
     test_debug ();
+    test_set_running ();
 
     done_testing();
     return (0);
diff --git a/src/common/libflux/test/module_fake1.c b/src/common/libflux/test/module_fake1.c
deleted file mode 100644
index 496fbd4dac2a..000000000000
--- a/src/common/libflux/test/module_fake1.c
+++ /dev/null
@@ -1,17 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include "src/common/libflux/module.h"
-
-int mod_main (int argc, char **argv)
-{
-     return 0;
-}
-MOD_NAME ("fake1");
diff --git a/src/common/libflux/test/module_fake2.c b/src/common/libflux/test/module_fake2.c
deleted file mode 100644
index 2c158ec0961d..000000000000
--- a/src/common/libflux/test/module_fake2.c
+++ /dev/null
@@ -1,18 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include "src/common/libflux/module.h"
-
-int mod_main (int argc, char **argv)
-{
-     return 0;
-}
-// leave undefined for unit test
-//MOD_NAME ("fake2");
diff --git a/src/common/libflux/test/msg_deque.c b/src/common/libflux/test/msg_deque.c
new file mode 100644
index 000000000000..9db1aebd2563
--- /dev/null
+++ b/src/common/libflux/test/msg_deque.c
@@ -0,0 +1,287 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <poll.h>
+#include <stdbool.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+
+#include "msg_deque.h"
+
+
+void check_queue (void)
+{
+    struct msg_deque *q;
+    flux_msg_t *msg1;
+    flux_msg_t *msg2;
+    flux_msg_t *msg;
+
+    if (!(msg1 = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
+        BAIL_OUT ("flux_msg_create failed");
+    if (!(msg2 = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
+        BAIL_OUT ("flux_msg_create failed");
+
+    q = msg_deque_create (0);
+    ok (q != NULL,
+        "msg_deque_create works");
+    ok (msg_deque_empty (q) == true,
+        "msg_deque_empty is true");
+    ok (msg_deque_count (q) == 0,
+        "msg_deque_count = 0");
+    ok (msg_deque_push_back (q, msg1) == 0,
+        "msg_deque_push_back msg1 works");
+    ok (msg_deque_empty (q) == false,
+        "msg_deque_empty is false");
+    ok (msg_deque_count (q) == 1,
+        "msg_deque_count = 1");
+    ok (msg_deque_push_back (q, msg2) == 0,
+        "msg_deque_push_back msg2 works");
+    ok (msg_deque_empty (q) == false,
+        "msg_deque_empty is false");
+    ok (msg_deque_count (q) == 2,
+        "msg_deque_count = 2");
+    ok ((msg = msg_deque_pop_front (q)) == msg1,
+        "msg_deque_pop_front popped msg1");
+    flux_msg_destroy (msg);
+    ok (msg_deque_count (q) == 1,
+        "msg_deque_count = 1");
+    ok (msg_deque_empty (q) == false,
+        "msg_deque_empty is false");
+    ok ((msg = msg_deque_pop_front (q)) == msg2,
+        "msg_deque_pop_front popped msg2");
+    flux_msg_destroy (msg);
+    ok (msg_deque_empty (q) == true,
+        "msg_deque_empty is true");
+    ok (msg_deque_count (q) == 0,
+        "msg_deque_count = 0");
+    ok (msg_deque_pop_front (q) == NULL,
+        "msg_deque_pop_front returned NULL");
+
+    /* Now use push_front and verify messages are popped in opposite order */
+    if (!(msg1 = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
+        BAIL_OUT ("flux_msg_create failed");
+    if (!(msg2 = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
+        BAIL_OUT ("flux_msg_create failed");
+    ok (msg_deque_empty (q) == true,
+        "msg_deque_empty is true");
+    ok (msg_deque_push_front (q, msg1) == 0,
+        "msg_deque_push_front msg1 works");
+    ok (msg_deque_empty (q) == false,
+        "msg_deque_empty is false");
+    ok (msg_deque_push_front (q, msg2) == 0,
+        "msg_deque_push_front msg2 works");
+    ok (msg_deque_empty (q) == false,
+        "msg_deque_empty is false");
+    ok ((msg = msg_deque_pop_front (q)) == msg2,
+        "msg_deque_pop_front popped msg2");
+    flux_msg_destroy (msg);
+    ok (msg_deque_empty (q) == false,
+        "msg_deque_empty is false");
+    ok ((msg = msg_deque_pop_front (q)) == msg1,
+        "msg_deque_pop_front popped msg1");
+    flux_msg_destroy (msg);
+    ok (msg_deque_empty (q) == true,
+        "msg_deque_empty is true");
+
+    msg_deque_destroy (q);
+}
+
+void check_poll (void)
+{
+    struct msg_deque *q;
+    flux_msg_t *msg1;
+    flux_msg_t *msg2;
+    flux_msg_t *msg;
+    struct pollfd pfd;
+
+    if (!(msg1 = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+    if (!(msg2 = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+
+    ok ((q = msg_deque_create (0)) != NULL,
+        "msg_deque_create works");
+    ok (msg_deque_pollevents (q) == POLLOUT,
+        "msg_deque_pollevents on empty queue returns POLLOUT");
+    ok (msg_deque_push_back (q, msg1) == 0,
+        "msg_deque_push_back msg1 works");
+    ok (msg_deque_pollevents (q) == (POLLOUT | POLLIN),
+        "msg_deque_pollevents on non-empty queue returns POLLOUT|POLLIN");
+    ok (msg_deque_push_back (q, msg2) == 0,
+        "msg_deque_push_back msg2 works");
+    ok (msg_deque_pollevents (q) == (POLLOUT | POLLIN),
+        "msg_deque_pollevents still returns POLLOUT|POLLIN");
+    ok ((msg = msg_deque_pop_front (q)) != NULL,
+        "msg_deque_pop_front returns a message");
+    flux_msg_decref (msg);
+    ok (msg_deque_pollevents (q) == (POLLOUT | POLLIN),
+        "msg_deque_pollevents still returns POLLOUT|POLLIN");
+
+    ok ((msg = msg_deque_pop_front (q)) != NULL,
+        "msg_deque_pop_front returns a message");
+    flux_msg_decref (msg);
+    ok (msg_deque_pollevents (q) == POLLOUT,
+        "msg_deque_pollevents on empty queue returns POLLOUT");
+
+    /* now test pollfd */
+    if (!(msg1 = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+
+    ok ((pfd.fd = msg_deque_pollfd (q)) >= 0,
+        "msg_deque_pollfd works");
+    pfd.events = POLLIN,
+    pfd.revents = 0,
+    ok (poll (&pfd, 1, 0) == 1 && pfd.revents == POLLIN,
+        "msg_deque_pollfd suggests we read pollevents");
+    ok (msg_deque_pollevents (q) == POLLOUT,
+        "msg_deque_pollevents on empty queue returns POLLOUT");
+    pfd.events = POLLIN,
+    pfd.revents = 0,
+    ok (poll (&pfd, 1, 0) == 0,
+        "pollfd is no longer ready");
+    ok (msg_deque_push_back (q, msg1) == 0,
+        "msg_deque_push_back works");
+    pfd.events = POLLIN,
+    pfd.revents = 0,
+    ok (poll (&pfd, 1, 0) == 1 && pfd.revents == POLLIN,
+        "pollfd suggests we read pollevents");
+    ok (msg_deque_pollevents (q) == (POLLOUT | POLLIN),
+        "msg_deque_pollevents on non-empty queue returns POLLOUT|POLLIN");
+    pfd.events = POLLIN,
+    pfd.revents = 0,
+    ok (poll (&pfd, 1, 0) == 0,
+        "pollfd is no longer ready");
+    ok (msg_deque_pollevents (q) == (POLLOUT | POLLIN),
+        "msg_deque_pollevents still returns POLLOUT|POLLIN");
+
+    msg_deque_destroy (q);
+}
+
+void check_single_thread (void)
+{
+    struct msg_deque *q;
+    flux_msg_t *msg1;
+    flux_msg_t *msg;
+
+    if (!(msg1 = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
+        BAIL_OUT ("flux_msg_create failed");
+
+    q = msg_deque_create (MSG_DEQUE_SINGLE_THREAD);
+    ok (q != NULL,
+        "msg_deque_create flags=SINGLE_THREAD works");
+    flux_msg_incref (msg1);
+    ok (msg_deque_push_back (q, msg1) == 0,
+        "msg_deque_push_back msg1 works with refcount==2");
+    flux_msg_decref (msg1);
+    ok ((msg = msg_deque_pop_front (q)) == msg1,
+        "msg_deque_pop_front popped msg1");
+    flux_msg_destroy (msg);
+
+    msg_deque_destroy (q);
+}
+
+void check_inval (void)
+{
+    struct msg_deque *q;
+    flux_msg_t *msg1;
+    flux_msg_t *msg;
+
+    if (!(q = msg_deque_create (0)))
+        BAIL_OUT ("could not create msg_deque");
+    if (!(msg1 = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+
+    errno = 0;
+    ok (msg_deque_create (0x1000) == NULL && errno == EINVAL,
+        "msg_deque_create flags=0x1000 fails with EINVAL");
+
+    ok (msg_deque_empty (NULL) == true,
+        "msg_deque_empty q=NULL is true");
+    errno = 42;
+    lives_ok ({msg_deque_destroy (NULL);},
+        "msg_deque_destroy q=NULL doesn't crash");
+    ok (errno == 42,
+        "msg_deque_destroy doesn't clobber errno");
+
+    ok (msg_deque_count (NULL) == 0,
+        "msg_deque_count q=NULL is 0");
+
+    // msg_deque_push_back
+    errno = 0;
+    ok (msg_deque_push_back (NULL, msg1) < 0 && errno == EINVAL,
+        "msg_deque_push_back q=NULL fails with EINVAL");
+    errno = 0;
+    ok (msg_deque_push_back (q, NULL) < 0 && errno == EINVAL,
+        "msg_deque_push_back msg=NULL fails with EINVAL");
+    flux_msg_incref (msg1);
+    errno = 0;
+    ok (msg_deque_push_back (q, msg1) < 0 && errno == EINVAL,
+        "msg_deque_push_back msg with ref=2 fails with EINVAL");
+    flux_msg_decref (msg1);
+
+    // msg_deque_push_front
+    errno = 0;
+    ok (msg_deque_push_front (NULL, msg1) < 0 && errno == EINVAL,
+        "msg_deque_push_front q=NULL fails with EINVAL");
+    errno = 0;
+    ok (msg_deque_push_front (q, NULL) < 0 && errno == EINVAL,
+        "msg_deque_push_front msg=NULL fails with EINVAL");
+    errno = 0;
+    msg = NULL;
+    ok (msg_deque_push_front (q, msg) < 0 && errno == EINVAL,
+        "msg_deque_push_front *msg=NULL fails with EINVAL");
+    flux_msg_incref (msg1);
+    errno = 0;
+    ok (msg_deque_push_front (q, msg1) < 0 && errno == EINVAL,
+        "msg_deque_push_front msg with ref=2 fails with EINVAL");
+    flux_msg_decref (msg1);
+
+    ok (msg_deque_pop_front (NULL) == NULL,
+        "msg_deque_pop_front q=NULL returns NULL");
+    ok (msg_deque_empty (NULL) == true,
+        "msg_deque_empty q=NULL returns true");
+    errno = 0;
+    ok (msg_deque_pollfd (NULL) < 0 && errno == EINVAL,
+        "msg_deque_pollfd q=NULL fails with EINVAL");
+    errno = 0;
+    ok (msg_deque_pollevents (NULL) < 0 && errno == EINVAL,
+        "msg_deque_pollevents q=NULL fails with EINVAL");
+
+    msg = msg1;
+    ok (msg_deque_push_back (q, msg1) == 0,
+        "msg_deque_push_back msg1 works");
+    errno = 0;
+    ok (msg_deque_push_back (q, msg) < 0 && errno == EINVAL,
+        "msg_deque_push_back msg1 again fails with EINVAL");
+    // this test ends with msg1 owned by q
+
+    msg_deque_destroy (q);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    check_queue ();
+    check_poll ();
+    check_inval ();
+    check_single_thread ();
+
+    done_testing ();
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/test/msg_handler.c b/src/common/libflux/test/msg_handler.c
index 5c2aadc11126..c5d44d0f99f9 100644
--- a/src/common/libflux/test/msg_handler.c
+++ b/src/common/libflux/test/msg_handler.c
@@ -8,9 +8,11 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <flux/core.h>
 
-#include "src/common/libflux/msg_handler.h"
 #include "src/common/libtap/tap.h"
 
 /* Create a flux handle with no implementation operation callbacks
@@ -42,7 +44,7 @@ void test_msg_handler_create (flux_t *h)
      */
     mh = flux_msg_handler_create (h, FLUX_MATCH_ANY, dummy_msg_handler, NULL);
     ok (mh != NULL,
-        "able to creat fake message handler");
+        "able to create fake message handler");
     flux_msg_handler_destroy (mh);
 
     /* invalid arguments
diff --git a/src/common/libflux/test/msglist.c b/src/common/libflux/test/msglist.c
new file mode 100644
index 000000000000..851e2d71084e
--- /dev/null
+++ b/src/common/libflux/test/msglist.c
@@ -0,0 +1,159 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* msglist.c - test pollevents/pollfd aspect of flux_msglist
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <poll.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+
+
+void check_msglist (void)
+{
+    struct flux_msglist *l;
+    flux_msg_t *msg1;
+    flux_msg_t *msg2;
+
+    if (!(msg1 = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
+        BAIL_OUT ("flux_msg_create failed");
+    if (!(msg2 = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
+        BAIL_OUT ("flux_msg_create failed");
+
+    ok (flux_msglist_count (NULL) == 0,
+        "flux_msglist_count l=NULL is 0");
+    l = flux_msglist_create ();
+    ok (l != NULL,
+        "flux_msglist_create works");
+    ok (flux_msglist_count (l) == 0,
+        "flux_msglist_count is 0");
+
+    ok (flux_msglist_append (l, msg1) == 0,
+        "flux_msglist_append msg1 works");
+    ok (flux_msglist_count (l) == 1,
+        "flux_msglist_count is 1");
+    ok (flux_msglist_first (l) == msg1,
+        "flux_msglist_first is msg1");
+    ok (flux_msglist_last (l) == msg1,
+        "flux_msglist_last is msg1");
+    ok (flux_msglist_next (l) == NULL,
+        "flux_msglist_next is NULL");
+
+    ok (flux_msglist_append (l, msg2) == 0,
+        "flux_msglist_append msg2 works");
+    ok (flux_msglist_count (l) == 2,
+        "flux_msglist_count is 2");
+    ok (flux_msglist_first (l) == msg1,
+        "flux_msglist_first is msg1");
+    ok (flux_msglist_next (l) == msg2,
+        "flux_msglist_next is msg2");
+    ok (flux_msglist_last (l) == msg2,
+        "flux_msglist_last is msg2");
+
+    ok (flux_msglist_first (l) == msg1,
+        "flux_msglist_last is msg1 (assigning curosr to msg1)");
+    flux_msglist_delete (l);
+    ok (flux_msglist_count (l) == 1,
+        "flux_msglist_count is 1 after delete");
+    ok (flux_msglist_first (l) == msg2,
+        "flux_msglist_first is now msg2");
+
+    flux_msg_decref (msg1);
+    flux_msg_decref (msg2);
+
+    flux_msglist_destroy (l);
+}
+
+
+void check_poll (void)
+{
+    struct flux_msglist *ml;
+    int e;
+    flux_msg_t *msg;
+    const flux_msg_t *tmp;
+    struct pollfd pfd;
+
+    if (!(msg = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+
+    ok ((ml = flux_msglist_create ()) != NULL,
+        "flux_msglist_create works");
+    ok ((e = flux_msglist_pollevents (ml)) >= 0 && e == POLLOUT,
+        "flux_msglist_pollevents on empty msglist returns POLLOUT");
+    ok (flux_msglist_push (ml, msg) == 0,
+        "flux_msglist_push works");
+    ok ((e = flux_msglist_pollevents (ml)) >= 0 && e == (POLLOUT | POLLIN),
+        "flux_msglist_pollevents on non-empty msglist returns POLLOUT|POLLIN");
+    ok (flux_msglist_push (ml, msg) == 0,
+        "flux_msglist_push works");
+    ok ((e = flux_msglist_pollevents (ml)) >= 0 && e == (POLLOUT | POLLIN),
+        "flux_msglist_pollevents still returns POLLOUT|POLLIN");
+    ok ((tmp = flux_msglist_pop (ml)) != NULL,
+        "flux_msglist_pop returns a message");
+    ok ((e = flux_msglist_pollevents (ml)) >= 0 && e == (POLLOUT | POLLIN),
+        "flux_msglist_pollevents still returns POLLOUT|POLLIN");
+    flux_msg_decref (tmp);
+
+    ok ((tmp = flux_msglist_pop (ml)) != NULL,
+        "flux_msglist_pop returns a message");
+    ok ((e = flux_msglist_pollevents (ml)) >= 0 && e == POLLOUT,
+        "flux_msglist_pollevents on empty msglist returns POLLOUT");
+    flux_msg_decref (tmp);
+
+    ok ((pfd.fd = flux_msglist_pollfd (ml)) >= 0,
+        "flux_msglist_pollfd works");
+    pfd.events = POLLIN,
+    pfd.revents = 0,
+    ok (poll (&pfd, 1, 0) == 1 && pfd.revents == POLLIN,
+        "flux_msglist_pollfd suggests we read pollevents");
+    ok ((e = flux_msglist_pollevents (ml)) >= 0 && e == POLLOUT,
+        "flux_msglist_pollevents on empty msglist returns POLLOUT");
+    pfd.events = POLLIN,
+    pfd.revents = 0,
+    ok (poll (&pfd, 1, 0) == 0,
+        "pollfd is no longer ready");
+    ok (flux_msglist_push (ml, msg) == 0,
+        "flux_msglist_push works");
+    pfd.events = POLLIN,
+    pfd.revents = 0,
+    ok (poll (&pfd, 1, 0) == 1 && pfd.revents == POLLIN,
+        "pollfd suggests we read pollevents");
+    ok ((e = flux_msglist_pollevents (ml)) >= 0 && e == (POLLOUT | POLLIN),
+        "flux_msglist_pollevents on non-empty msglist returns POLLOUT|POLLIN");
+    pfd.events = POLLIN,
+    pfd.revents = 0,
+    ok (poll (&pfd, 1, 0) == 0,
+        "pollfd is no longer ready");
+    ok ((e = flux_msglist_pollevents (ml)) >= 0 && e == (POLLOUT | POLLIN),
+        "msglist_pollevents still returns POLLOUT|POLLIN");
+
+    flux_msg_decref (msg);
+    flux_msglist_destroy (ml);
+
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    check_msglist ();
+    check_poll ();
+
+    done_testing ();
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libflux/test/panic.c b/src/common/libflux/test/panic.c
deleted file mode 100644
index 372ee6c5123f..000000000000
--- a/src/common/libflux/test/panic.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <errno.h>
-#include <string.h>
-#include <flux/core.h>
-
-#include "src/common/libtap/tap.h"
-#include "src/common/libtestutil/util.h"
-
-int main (int argc, char *argv[])
-{
-    flux_t *h;
-    flux_msg_t *msg;
-    const char *topic;
-    const char *reason;
-    int flags;
-
-    plan (NO_PLAN);
-
-    if (!(h = loopback_create (0)))
-        BAIL_OUT ("loopback_create failed");
-
-    /* Send request
-     */
-    ok (flux_panic (h, 0, 0, "fubar") == 0,
-        "flux_panic works");
-
-    /* Receive request on the loopback
-     */
-    msg = flux_recv (h, FLUX_MATCH_ANY, 0);
-    ok (msg != NULL,
-        "flux_recv received message on loop");
-    ok (flux_request_unpack (msg, &topic, "{s:s s:i}",
-                             "reason", &reason, "flags", &flags) == 0,
-        "flux_request_unpack worked on panic request");
-    ok (topic != NULL && !strcmp (topic, "cmb.panic"),
-        "topic string is correct");
-    ok (!strcmp (reason, "fubar"),
-        "reason is correct");
-    ok (flags == 0,
-        "flags is correct");
-    flux_msg_destroy (msg);
-
-    /* invalid arguments
-     */
-    errno = 0;
-    ok (flux_panic (NULL, 0, 0, "foo") < 0 && errno == EINVAL,
-        "flux_panic h=NULL fails with EINVAL");
-    errno = 0;
-    ok (flux_panic (h, 0, 1, "foo") < 0 && errno == EINVAL,
-        "flux_panic flags=1 fails with EINVAL");
-    errno = 0;
-    ok (flux_panic (h, 0, 0, NULL) < 0 && errno == EINVAL,
-        "flux_panic reason=NULL fails with EINVAL");
-
-    flux_close (h);
-
-    done_testing();
-    return (0);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/src/common/libflux/test/plugin.c b/src/common/libflux/test/plugin.c
index 63a55f17f7df..a014506e8d05 100644
--- a/src/common/libflux/test/plugin.c
+++ b/src/common/libflux/test/plugin.c
@@ -8,11 +8,15 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 #include <errno.h>
+#include <flux/core.h>
 
-#include "src/common/libflux/plugin.h"
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 
 /* function prototype for invalid args testing below */
@@ -69,6 +73,20 @@ void test_invalid_args ()
     ok (flux_plugin_get_name (NULL) == NULL && errno == EINVAL,
         "flux_plugin_get_name (NULL) returns EINVAL");
 
+    errno = 0;
+    ok (flux_plugin_get_uuid (NULL) == NULL && errno == EINVAL,
+        "flux_plugin_get_uuid (NULL) returns EINVAL");
+
+    ok (flux_plugin_get_path (NULL) == NULL,
+        "flux_plugin_get_path (NULL) returns NULL");
+
+    ok (flux_plugin_get_flags (NULL) == 0,
+        "flux_plugin_get_flags (NULL) returns 0");
+    ok (flux_plugin_set_flags (NULL, 0) < 0 && errno == EINVAL,
+        "flux_plugin_set_flags (NULL, 0) returns EINVAL");
+    ok (flux_plugin_set_flags (p, 1024) < 0 && errno == EINVAL,
+        "flux_plugin_set_flags with invalid flags returns EINVAL");
+
     ok (flux_plugin_set_conf (NULL, NULL) < 0 && errno == EINVAL,
         "flux_plugin_set_conf (NULL, NULL) returns EINVAL");
     ok (flux_plugin_set_conf (p, NULL) < 0 && errno == EINVAL,
@@ -78,6 +96,11 @@ void test_invalid_args ()
     like (flux_plugin_strerror (p), "^parse error: col 1:.*",
         "flux_plugin_last_error returns error text");
 
+    ok (flux_plugin_get_conf (NULL) == NULL && errno == EINVAL,
+        "flux_plugin_get_conf () with NULL arg returns EINVAL");
+    ok (flux_plugin_get_conf (p) == NULL && errno == ENOENT,
+        "flux_plugin_get_conf () with no conf returns ENOENT");
+
     ok (flux_plugin_conf_unpack (p, "{s:i}", "bar", &i) < 0 && errno == ENOENT,
         "flux_plugin_conf_unpack () with no conf returns ENOENT");
 
@@ -99,6 +122,8 @@ void test_invalid_args ()
         "flux_plugin_aux_get (p, NULL) returns EINVAL");
     ok (flux_plugin_aux_get (p, "foo") == NULL && errno == ENOENT,
         "flux_plugin_aux_get (p, 'foo') returns ENOENT");
+    lives_ok ({flux_plugin_aux_delete (p, NULL);},
+        "flux_plugin_aux_delete (p, NULL) doesn't crash");
 
     ok (flux_plugin_add_handler (NULL, "foo.*", foo, NULL) < 0 && errno == EINVAL,
         "flux_plugin_add_handler (NULL, ...) returns EINVAL");
@@ -179,9 +204,9 @@ void test_plugin_args ()
         "flux_plugin_arg_unpack returned valid value for arg");
 
     ok (flux_plugin_arg_set (args,
-                             FLUX_PLUGIN_ARG_IN | FLUX_PLUGIN_ARG_UPDATE,
+                             FLUX_PLUGIN_ARG_IN,
                              "{\"b\":7}") == 0,
-        "flux_plugin_arg_set with FLUX_PLUGIN_ARG_UPDATE works");
+        "flux_plugin_arg_set can update existing args");
     ok (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
                                 "{s:i}", "b", &arg) == 0,
         "flux_plugin_arg_unpack worked");
@@ -194,14 +219,14 @@ void test_plugin_args ()
         "flux_plugin_arg_unpack returned valid value for old arg");
 
 
-    /* Test update with unset args */
+    /* Test replace with unset args */
     flux_plugin_arg_t *new = flux_plugin_arg_create ();
     if (!new)
         BAIL_OUT ("flux_plugin_arg_create failed");
     ok (flux_plugin_arg_set (args,
-                             FLUX_PLUGIN_ARG_IN | FLUX_PLUGIN_ARG_UPDATE,
+                             FLUX_PLUGIN_ARG_IN|FLUX_PLUGIN_ARG_REPLACE,
                             "{\"count\": 29}") == 0,
-        "flux_plugin_arg_set with ARG_UPDATE works for empty args");
+        "flux_plugin_arg_set with ARG_REPLACE works for empty args");
     flux_plugin_arg_destroy (new);
 
     ok (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
@@ -260,9 +285,9 @@ int op1 (flux_plugin_t *p, const char *topic,
     int a, b;
     if (flux_plugin_arg_unpack (args, 0, "{s:i s:i}", "a", &a, "b", &b) < 0)
         return -1;
-    if (strcmp (topic, "op.add") == 0)
+    if (streq (topic, "op.add"))
         a += b;
-    else if (strcmp (topic, "op.multiply") == 0)
+    else if (streq (topic, "op.multiply"))
         a *= b;
     else {
         errno = ENOTSUP;
@@ -286,6 +311,9 @@ void test_basic ()
     is (flux_plugin_get_name (p), "op",
         "flux_plugin_get_name() works");
 
+    ok (flux_plugin_get_path (p) == NULL,
+        "flux_plugin_get_path() returns NULL when no loaded plugin path");
+
     ok (flux_plugin_add_handler (p, "foo.*", NULL, NULL) == 0,
         "flux_plugin_add_handler (p, 'foo.*', NULL) works");
     ok (flux_plugin_get_handler (p, "foo.*") == NULL,
@@ -302,7 +330,7 @@ void test_basic ()
     b = 4;
     ok (flux_plugin_arg_pack (args, 0, "{s:i s:i}", "a", a, "b", b) == 0,
         "flux_plugin_arg_pack works");
-    ok (flux_plugin_call (p, "op.add", args) == 0,
+    ok (flux_plugin_call (p, "op.add", args) >= 0,
         "flux_plugin_call op.add works");
 
     ok (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_OUT,
@@ -314,7 +342,7 @@ void test_basic ()
     a = 2;
     ok (flux_plugin_arg_pack (args, 0, "{s:i s:i}", "a", a, "b", b) == 0,
         "flux_plugin_arg_pack works");
-    ok (flux_plugin_call (p, "op.multiply", args) == 0,
+    ok (flux_plugin_call (p, "op.multiply", args) >= 0,
         "callback with topic op.multiply worked");
     ok (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_OUT,
                                 "{s:i}", "a", &a) == 0,
@@ -349,7 +377,7 @@ void test_register ()
 
     ok (flux_plugin_register (p, "test_register", tab) == 0,
         "flux_plugin_register 2 handlers works");
-    ok (flux_plugin_call (p, "foo.test", args) == 0,
+    ok (flux_plugin_call (p, "foo.test", args) >= 0,
         "flux_plugin_call foo.test worked");
     ok (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_OUT,
                                 "{s:s s:s}",
@@ -361,7 +389,7 @@ void test_register ()
     is (data, foodata,
         "flux_plugin_call passed correct void *data to foo()");
 
-    ok (flux_plugin_call (p, "fallthru", args) == 0,
+    ok (flux_plugin_call (p, "fallthru", args) >= 0,
         "flux_plugin_call fallthru worked");
     ok (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_OUT,
                                 "{s:s s:s}",
@@ -378,6 +406,7 @@ void test_register ()
 
 void test_load ()
 {
+    int rc;
     char *out;
     const char *result;
     flux_plugin_t *p = flux_plugin_create ();
@@ -395,15 +424,28 @@ void test_load ()
 
     ok (flux_plugin_set_conf (p, "{\"foo\":\"bar\"}") == 0,
         "flux_plugin_set_conf (): %s", flux_plugin_strerror (p));
-    ok (flux_plugin_load_dso (p, "test/.libs/plugin_foo.so") == 0,
+    ok ((result = flux_plugin_get_conf (p)) != NULL,
+        "flux_plugin_get_conf () works");
+    ok (result != NULL,
+        "conf = %s", result);
+
+    ok ((rc = flux_plugin_load_dso (p, "test/.libs/plugin_foo.so")) == 0,
         "flux_plugin_load worked");
+    if (rc < 0)
+        BAIL_OUT ("Failed to load test plugin: %s",
+                  flux_plugin_strerror (p));
     is (flux_plugin_get_name (p), "plugin-test",
         "loaded dso registered its own name");
 
+    result = flux_plugin_get_path (p);
+    diag (result);
+    like (result, ".*test/.libs/plugin_foo.so",
+        "flux_plugin_get_path() on loaded dso works");
+
     flux_plugin_arg_t *args = flux_plugin_arg_create ();
     if (!args)
         BAIL_OUT ("flux_plugin_arg_create failed");
-    ok (flux_plugin_call (p, "test.foo", args) == 0,
+    ok (flux_plugin_call (p, "test.foo", args) >= 0,
         "flux_plugin_call (test.foo) success");
     ok (flux_plugin_arg_unpack (args,
                                 FLUX_PLUGIN_ARG_OUT,
@@ -418,7 +460,7 @@ void test_load ()
         "flux_plugin_arg_out works");
     diag ("out = %s", out);
     free (out);
-    ok (flux_plugin_call (p, "test.bar", args) == 0,
+    ok (flux_plugin_call (p, "test.bar", args) >= 0,
         "flux_plugin_call (test.bar) success");
     ok (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_OUT,
                                 "{s:s}", "result", &result) == 0,
@@ -430,6 +472,60 @@ void test_load ()
     flux_plugin_destroy (p);
 }
 
+void test_load_rtld_now ()
+{
+    flux_plugin_t *p = flux_plugin_create ();
+    if (!p)
+        BAIL_OUT ("flux_plugin_create");
+
+    ok (flux_plugin_set_flags (p, FLUX_PLUGIN_RTLD_NOW) == 0,
+        "flux_plugin_set_flags (p, RTLD_NOW) == 0");
+    ok (flux_plugin_load_dso (p, "test/.libs/plugin_bar.so") < 0,
+        "load of plugin with invalid symbol fails immediately");
+    like (flux_plugin_strerror (p), "^dlopen: .*: (undefined symbol|symbol not found)",
+        "got expected error message: %s", flux_plugin_strerror (p));
+
+    flux_plugin_destroy (p);
+}
+
+void test_uuid (void)
+{
+    flux_plugin_t *p;
+    const char *uuid;
+    char *ouuid;
+
+    if (!(p = flux_plugin_create ()))
+        BAIL_OUT ("flux_plugin_create failed");
+    uuid = flux_plugin_get_uuid (p);
+    ok (uuid != NULL,
+        "flux_plugin_get_uuid works");
+    if (!(ouuid = strdup (uuid)))
+        BAIL_OUT ("strdup failed");
+    flux_plugin_destroy (p);
+
+    if (!(p = flux_plugin_create ()))
+        BAIL_OUT ("flux_plugin_create failed");
+    uuid = flux_plugin_get_uuid (p);
+
+    ok (uuid != NULL && !streq (ouuid, uuid),
+        "second plugin instance has different uuid");
+    flux_plugin_destroy (p);
+    free (ouuid);
+}
+
+void test_plugin_init_failure (void)
+{
+    flux_plugin_t *p = flux_plugin_create ();
+    if (!p || flux_plugin_set_conf (p, "{\"fail\": 1}") < 0)
+        BAIL_OUT ("flux_plugin_create/set_conf");
+    ok (flux_plugin_load_dso (p, "test/.libs/plugin_foo.so") < 0,
+        "flux_plugin_load fails if plugin init callback fails");
+    diag("%s", flux_plugin_strerror (p));
+    like (flux_plugin_strerror (p), "flux_plugin_init failed",
+        "flux_plugin_strerror() notes that plugin init failed");
+    flux_plugin_destroy (p);
+}
+
 int main (int argc, char *argv[])
 {
     plan (NO_PLAN);
@@ -438,6 +534,9 @@ int main (int argc, char *argv[])
     test_basic ();
     test_register ();
     test_load ();
+    test_load_rtld_now ();
+    test_uuid ();
+    test_plugin_init_failure ();
     done_testing();
     return (0);
 }
diff --git a/src/common/libflux/test/plugin_bar.c b/src/common/libflux/test/plugin_bar.c
new file mode 100644
index 000000000000..c6c409dae47c
--- /dev/null
+++ b/src/common/libflux/test/plugin_bar.c
@@ -0,0 +1,23 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#include <flux/core.h>
+
+/*  Invalid global symbol to force dlopen failure
+ */
+extern int my_invalid_sym (void);
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    return my_invalid_sym ();
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libflux/test/plugin_foo.c b/src/common/libflux/test/plugin_foo.c
index 191b0ccc8c8e..acff16ab6b7e 100644
--- a/src/common/libflux/test/plugin_foo.c
+++ b/src/common/libflux/test/plugin_foo.c
@@ -36,9 +36,11 @@ static const struct flux_plugin_handler tab []= {
 
 int flux_plugin_init (flux_plugin_t *p)
 {
+    int fail = 0;
     if (flux_plugin_register (p, "plugin-test", tab) < 0)
         return -1;
-    return 0;
+    (void) flux_plugin_conf_unpack (p, "{s?i}", "fail", &fail);
+    return fail ? -1 : 0;
 }
 
 /* vi: ts=4 sw=4 expandtab
diff --git a/src/common/libflux/test/reactor.c b/src/common/libflux/test/reactor.c
index e71e62344b82..d500464cef38 100644
--- a/src/common/libflux/test/reactor.c
+++ b/src/common/libflux/test/reactor.c
@@ -8,114 +8,29 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
-#include <czmq.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <stdlib.h>
+#include <time.h>
+#include <signal.h>
+#include <math.h>
+#include <flux/core.h>
 
-#include "src/common/libflux/reactor.h"
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/fdutils.h"
 #include "src/common/libtap/tap.h"
-
-static const size_t zmqwriter_msgcount = 1024;
-
-static void zmqwriter (flux_reactor_t *r, flux_watcher_t *w,
-                       int revents, void *arg)
-{
-    void *sock = flux_zmq_watcher_get_zsock (w);
-    static int count = 0;
-    if (revents & FLUX_POLLERR) {
-        fprintf (stderr, "%s: FLUX_POLLERR is set\n", __FUNCTION__);
-        goto error;
-    }
-    if (revents & FLUX_POLLOUT) {
-        uint8_t blob[64];
-        zmsg_t *zmsg = zmsg_new ();
-        if (!zmsg || zmsg_addmem (zmsg, blob, sizeof (blob)) < 0) {
-            fprintf (stderr, "%s: failed to create message: %s\n",
-                     __FUNCTION__, strerror (errno));
-            goto error;
-        }
-        if (zmsg_send (&zmsg, sock) < 0) {
-            fprintf (stderr, "%s: zmsg_send: %s\n",
-                     __FUNCTION__, strerror (errno));
-            goto error;
-        }
-        count++;
-        if (count == zmqwriter_msgcount)
-            flux_watcher_stop (w);
-    }
-    return;
-error:
-    flux_reactor_stop_error (r);
-}
-
-static void zmqreader (flux_reactor_t *r, flux_watcher_t *w,
-                       int revents, void *arg)
-{
-    void *sock = flux_zmq_watcher_get_zsock (w);
-    static int count = 0;
-    if (revents & FLUX_POLLERR) {
-        fprintf (stderr, "%s: FLUX_POLLERR is set\n", __FUNCTION__);
-        goto error;
-    }
-    if (revents & FLUX_POLLIN) {
-        zmsg_t *zmsg = zmsg_recv (sock);
-        if (!zmsg) {
-            fprintf (stderr, "%s: zmsg_recv: %s\n",
-                     __FUNCTION__, strerror (errno));
-            goto error;
-        }
-        zmsg_destroy (&zmsg);
-        count++;
-        if (count == zmqwriter_msgcount)
-            flux_watcher_stop (w);
-    }
-    return;
-error:
-    flux_reactor_stop_error (r);
-}
-
-static void test_zmq (flux_reactor_t *reactor)
-{
-    zsock_t *zs[2];
-    flux_watcher_t *r, *w;
-    const char *uri = "inproc://test_zmq";
-
-    zsys_set_logstream (stderr);
-    zsys_handler_set (NULL);
-
-    zs[0] = zsock_new_pair (NULL);
-    zs[1] = zsock_new_pair (NULL);
-    ok (zs[0] && zs[1]
-        && zsock_bind (zs[0], "%s", uri) == 0
-        && zsock_connect (zs[1], "%s", uri) == 0,
-        "zmq: connected ZMQ_PAIR sockets over inproc");
-    r = flux_zmq_watcher_create (reactor, zs[0], FLUX_POLLIN, zmqreader, NULL);
-    w = flux_zmq_watcher_create (reactor, zs[1], FLUX_POLLOUT, zmqwriter, NULL);
-    ok (r != NULL && w != NULL,
-        "zmq: nonblocking reader and writer created");
-    flux_watcher_start (r);
-    flux_watcher_start (w);
-    ok (flux_reactor_run  (reactor, 0) == 0,
-        "zmq: reactor ran to completion after %d messages", zmqwriter_msgcount);
-    flux_watcher_stop (r);
-    flux_watcher_stop (w);
-    flux_watcher_destroy (r);
-    flux_watcher_destroy (w);
-
-    zsock_destroy (&zs[0]);
-    zsock_destroy (&zs[1]);
-
-    zsys_shutdown ();
-}
+#include "ccan/array_size/array_size.h"
 
 static const size_t fdwriter_bufsize = 10*1024*1024;
 
-static void fdwriter (flux_reactor_t *r, flux_watcher_t *w,
-                       int revents, void *arg)
+static void fdwriter (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
 {
     int fd = flux_fd_watcher_get_fd (w);
     static char *buf = NULL;
@@ -130,9 +45,10 @@ static void fdwriter (flux_reactor_t *r, flux_watcher_t *w,
     }
     if (revents & FLUX_POLLOUT) {
         if ((n = write (fd, buf + count, fdwriter_bufsize - count)) < 0
-                                && errno != EWOULDBLOCK && errno != EAGAIN) {
+            && errno != EWOULDBLOCK && errno != EAGAIN) {
             fprintf (stderr, "%s: write failed: %s\n",
-                     __FUNCTION__, strerror (errno));
+                     __FUNCTION__,
+                     strerror (errno));
             goto error;
         }
         if (n > 0) {
@@ -147,8 +63,10 @@ static void fdwriter (flux_reactor_t *r, flux_watcher_t *w,
 error:
     flux_reactor_stop_error (r);
 }
-static void fdreader (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
+static void fdreader (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
 {
     int fd = flux_fd_watcher_get_fd (w);
     static char *buf = NULL;
@@ -163,9 +81,10 @@ static void fdreader (flux_reactor_t *r, flux_watcher_t *w,
     }
     if (revents & FLUX_POLLIN) {
         if ((n = read (fd, buf + count, fdwriter_bufsize - count)) < 0
-                            && errno != EWOULDBLOCK && errno != EAGAIN) {
+            && errno != EWOULDBLOCK && errno != EAGAIN) {
             fprintf (stderr, "%s: read failed: %s\n",
-                     __FUNCTION__, strerror (errno));
+                     __FUNCTION__,
+                     strerror (errno));
             goto error;
         }
         if (n > 0) {
@@ -186,13 +105,23 @@ static void test_fd (flux_reactor_t *reactor)
     int fd[2];
     flux_watcher_t *r, *w;
 
+#ifdef SOCK_NONBLOCK
     ok (socketpair (PF_LOCAL, SOCK_STREAM|SOCK_NONBLOCK, 0, fd) == 0,
+#else
+    ok (socketpair (PF_LOCAL, SOCK_STREAM, 0, fd) == 0
+	&& fd_set_nonblocking (fd[0]) >= 0
+	&& fd_set_nonblocking (fd[1]) >= 0,
+#endif
         "fd: successfully created non-blocking socketpair");
     r = flux_fd_watcher_create (reactor, fd[0], FLUX_POLLIN, fdreader, NULL);
     w = flux_fd_watcher_create (reactor, fd[1], FLUX_POLLOUT, fdwriter, NULL);
     ok (r != NULL && w != NULL,
         "fd: reader and writer created");
+    ok (!flux_watcher_is_active (r),
+        "flux_watcher_is_active() returns false");
     flux_watcher_start (r);
+    ok (flux_watcher_is_active (r),
+        "flux_watcher_is_active() returns true after flux_watcher_start()");
     flux_watcher_start (w);
     ok (flux_reactor_run (reactor, 0) == 0,
         "fd: reactor ran to completion after %lu bytes", fdwriter_bufsize);
@@ -204,740 +133,11 @@ static void test_fd (flux_reactor_t *reactor)
     close (fd[1]);
 }
 
-static void buffer_read (flux_reactor_t *r, flux_watcher_t *w,
-                        int revents, void *arg)
-{
-    int *count = arg;
-
-    if (revents & FLUX_POLLERR) {
-        ok (false,
-            "buffer: read callback incorrectly called with FLUX_POLLERR");
-    }
-    else if (revents & FLUX_POLLIN) {
-        flux_buffer_t *fb = flux_buffer_read_watcher_get_buffer (w);
-        const void *ptr;
-        int len;
-
-        ok ((ptr = flux_buffer_read (fb, -1, &len)) != NULL,
-            "buffer: read from buffer success");
-
-        ok (len == 6,
-            "buffer: read returned correct length");
-
-        ok (!memcmp (ptr, "foobar", 6),
-            "buffer: read returned correct data");
-    }
-    else {
-        ok (false,
-            "buffer: read callback failed to return FLUX_POLLIN: %d", revents);
-    }
-
-    (*count)++;
-    flux_watcher_stop (w);
-    return;
-}
-
-static void buffer_read_line (flux_reactor_t *r, flux_watcher_t *w,
-                              int revents, void *arg)
-{
-    int *count = arg;
-
-    if (revents & FLUX_POLLERR) {
-        ok (false,
-            "buffer: read line callback incorrectly called with FLUX_POLLERR");
-    }
-    else if (revents & FLUX_POLLIN) {
-        flux_buffer_t *fb = flux_buffer_read_watcher_get_buffer (w);
-        const void *ptr;
-        int len;
-
-        ok ((ptr = flux_buffer_read_line (fb, &len)) != NULL,
-            "buffer: read line from buffer success");
-
-        ok (len == 4,
-            "buffer: read line returned correct length");
-
-        if ((*count) == 0) {
-            ok (!memcmp (ptr, "foo\n", 4),
-                "buffer: read line returned correct data");
-        }
-        else {
-            ok (!memcmp (ptr, "bar\n", 4),
-                "buffer: read line returned correct data");
-        }
-    }
-    else {
-        ok (false,
-            "buffer: read line callback failed to return FLUX_POLLIN: %d", revents);
-    }
-
-    (*count)++;
-    if ((*count) == 2)
-        flux_watcher_stop (w);
-    return;
-}
-
-static void buffer_write (flux_reactor_t *r, flux_watcher_t *w,
-                          int revents, void *arg)
-{
-    int *count = arg;
-
-    if (revents & FLUX_POLLERR) {
-        ok (false,
-            "buffer: write callback called with FLUX_POLLERR");
-    }
-    else {
-        ok (flux_buffer_write_watcher_is_closed (w, NULL),
-            "buffer: write callback called after close");
-    }
-
-    (*count)++;
-    flux_watcher_stop (w);
-    return;
-}
-
-static void buffer_read_fill (flux_reactor_t *r, flux_watcher_t *w,
-                              int revents, void *arg)
-{
-    int *count = arg;
-
-    if (revents & FLUX_POLLERR) {
-        ok (false,
-            "buffer: read callback incorrectly called with FLUX_POLLERR");
-    }
-    else if (revents & FLUX_POLLIN) {
-        flux_buffer_t *fb = flux_buffer_read_watcher_get_buffer (w);
-        const void *ptr;
-        int len;
-
-        ok ((ptr = flux_buffer_read (fb, 6, &len)) != NULL,
-            "buffer: read from buffer success");
-
-        ok (len == 6,
-            "buffer: read returned correct length");
-
-        ok (!memcmp (ptr, "foobar", 6),
-            "buffer: read returned correct data");
-    }
-    else {
-        ok (false,
-            "buffer: read callback failed to return FLUX_POLLIN: %d", revents);
-    }
-
-    (*count)++;
-    if ((*count) == 3)
-        flux_watcher_stop (w);
-    return;
-}
-
-static void buffer_read_overflow (flux_reactor_t *r, flux_watcher_t *w,
-                                  int revents, void *arg)
-{
-    int *count = arg;
-
-    if (revents & FLUX_POLLERR) {
-        ok (false,
-            "buffer overflow test: read callback incorrectly called with FLUX_POLLERR");
-    }
-    else if (revents & FLUX_POLLIN) {
-        flux_buffer_t *fb = flux_buffer_read_watcher_get_buffer (w);
-        const void *ptr;
-        int len;
-
-        ok ((ptr = flux_buffer_read (fb, 6, &len)) != NULL,
-            "buffer overflow test: read from buffer success");
-
-        ok (len == 6,
-            "buffer overflow test: read returned correct length");
-
-        ok (!memcmp (ptr, "foobar", 6),
-            "buffer overflow test: read returned correct data");
-    }
-    else {
-        ok (false,
-            "buffer overflow test: read callback failed to return FLUX_POLLIN: %d", revents);
-    }
-
-    (*count)++;
-    if ((*count) == 3)
-        flux_watcher_stop (w);
-    return;
-}
-
-static void test_buffer (flux_reactor_t *reactor)
-{
-    int errnum = 0;
-    int fd[2];
-    int pfds[2];
-    flux_watcher_t *w;
-    flux_buffer_t *fb;
-    int count;
-    char buf[1024];
-
-    ok (socketpair (PF_LOCAL, SOCK_STREAM|SOCK_NONBLOCK, 0, fd) == 0,
-        "buffer: successfully created socketpair");
-
-    /* read buffer test */
-
-    count = 0;
-    w = flux_buffer_read_watcher_create (reactor,
-                                         fd[0],
-                                         1024,
-                                         buffer_read,
-                                         0,
-                                         &count);
-    ok (w != NULL,
-        "buffer: read created");
-
-    fb = flux_buffer_read_watcher_get_buffer (w);
-
-    ok (fb != NULL,
-        "buffer: buffer retrieved");
-
-    ok (write (fd[1], "foobar", 6) == 6,
-        "buffer: write to socketpair success");
-
-    flux_watcher_start (w);
-
-    ok (flux_reactor_run (reactor, 0) == 0,
-        "buffer: reactor ran to completion");
-
-    ok (count == 1,
-        "buffer: read callback successfully called");
-
-    flux_watcher_stop (w);
-    flux_watcher_destroy (w);
-
-    /* read line buffer test */
-
-    count = 0;
-    w = flux_buffer_read_watcher_create (reactor,
-                                         fd[0],
-                                         1024,
-                                         buffer_read_line,
-                                         FLUX_WATCHER_LINE_BUFFER,
-                                         &count);
-    ok (w != NULL,
-        "buffer: read line created");
-
-    fb = flux_buffer_read_watcher_get_buffer (w);
-
-    ok (fb != NULL,
-        "buffer: buffer retrieved");
-
-    ok (write (fd[1], "foo\nbar\n", 8) == 8,
-        "buffer: write to socketpair success");
-
-    flux_watcher_start (w);
-
-    ok (flux_reactor_run (reactor, 0) == 0,
-        "buffer: reactor ran to completion");
-
-    ok (count == 2,
-        "buffer: read line callback successfully called twice");
-
-    flux_watcher_stop (w);
-    flux_watcher_destroy (w);
-
-    /* write buffer test */
-
-    count = 0;
-    w = flux_buffer_write_watcher_create (reactor,
-                                          fd[0],
-                                          1024,
-                                          buffer_write,
-                                          0,
-                                          &count);
-    ok (w != NULL,
-        "buffer: write created");
-
-    fb = flux_buffer_write_watcher_get_buffer (w);
-
-    ok (fb != NULL,
-        "buffer: buffer retrieved");
-
-    flux_watcher_start (w);
-
-    ok (flux_buffer_write (fb, "bazbar", 6) == 6,
-        "buffer: write to buffer success");
-
-    ok (flux_reactor_run (reactor, 0) == 0,
-        "buffer: reactor ran to completion");
-
-    ok (count == 0,
-        "buffer: write callback never called");
-
-    ok (read (fd[1], buf, 1024) == 6,
-        "buffer: read from socketpair success");
-
-    ok (!memcmp (buf, "bazbar", 6),
-        "buffer: read from socketpair returned correct data");
-
-
-    flux_watcher_stop (w);
-    flux_watcher_destroy (w);
-
-    /* write buffer test, write to buffer before start */
-
-    count = 0;
-    w = flux_buffer_write_watcher_create (reactor,
-                                          fd[0],
-                                          1024,
-                                          buffer_write,
-                                          0,
-                                          &count);
-    ok (w != NULL,
-        "buffer: write created");
-
-    fb = flux_buffer_write_watcher_get_buffer (w);
-
-    ok (fb != NULL,
-        "buffer: buffer retrieved");
-
-    ok (flux_buffer_write (fb, "foobaz", 6) == 6,
-        "buffer: write to buffer success");
-
-    flux_watcher_start (w);
-
-    ok (flux_reactor_run (reactor, 0) == 0,
-        "buffer: reactor ran to completion");
-
-    ok (count == 0,
-        "buffer: write callback never called");
-
-    ok (read (fd[1], buf, 1024) == 6,
-        "buffer: read from socketpair success");
-
-    ok (!memcmp (buf, "foobaz", 6),
-        "buffer: read from socketpair returned correct data");
-
-    flux_watcher_stop (w);
-    flux_watcher_destroy (w);
-
-    /* read buffer test, fill buffer before start */
-
-    count = 0;
-    w = flux_buffer_read_watcher_create (reactor,
-                                         fd[0],
-                                         12, /* 12 bytes = 2 "foobars"s */
-                                         buffer_read_fill,
-                                         0,
-                                         &count);
-    ok (w != NULL,
-        "buffer: read created");
-
-    fb = flux_buffer_read_watcher_get_buffer (w);
-
-    ok (fb != NULL,
-        "buffer: buffer retrieved");
-
-    ok (flux_buffer_write (fb, "foobarfoobar", 12) == 12,
-        "buffer: write to buffer success");
-
-    ok (write (fd[1], "foobar", 6) == 6,
-        "buffer: write to socketpair success");
-
-    flux_watcher_start (w);
-
-    ok (flux_reactor_run (reactor, 0) == 0,
-        "buffer: reactor ran to completion");
-
-    ok (count == 3,
-        "buffer: read callback successfully called 3 times");
-
-    flux_watcher_stop (w);
-    flux_watcher_destroy (w);
-
-    /* read line buffer corner case test - fill buffer to max still works */
-
-    count = 0;
-    w = flux_buffer_read_watcher_create (reactor,
-                                         fd[0],
-                                         12, /* 12 bytes = 2 "foobar"s */
-                                         buffer_read_overflow,
-                                         0,
-                                         &count);
-    ok (w != NULL,
-        "buffer overflow test: read line created");
-
-    fb = flux_buffer_read_watcher_get_buffer (w);
-
-    ok (fb != NULL,
-        "buffer overflow test: buffer retrieved");
-
-    ok (write (fd[1], "foobarfoobarfoobar", 18) == 18,
-        "buffer overflow test: write to socketpair success");
-
-    flux_watcher_start (w);
-
-    ok (flux_reactor_run (reactor, 0) == 0,
-        "buffer overflow test: reactor ran to completion");
-
-    ok (count == 3,
-        "buffer overflow test: read line callback successfully called three times");
-
-    flux_watcher_stop (w);
-    flux_watcher_destroy (w);
-
-    /* write buffer watcher close() testcase */
-
-    ok (flux_buffer_write_watcher_close (NULL) == -1 && errno == EINVAL,
-        "buffer: flux_buffer_write_watcher_close handles NULL argument");
-
-    count = 0;
-    ok (pipe (pfds) == 0,
-        "buffer: hey I can has a pipe!");
-
-    w = flux_buffer_write_watcher_create (reactor,
-                                          pfds[1],
-                                          1024,
-                                          buffer_write,
-                                          0,
-                                          &count);
-    ok (w == NULL && errno == EINVAL,
-        "buffer: write_watcher_create fails with EINVAL if fd !nonblocking");
-
-    ok (fd_set_nonblocking (pfds[1]) >= 0,
-        "buffer: fd_set_nonblocking");
-
-    w = flux_buffer_write_watcher_create (reactor,
-                                          pfds[1],
-                                          1024,
-                                          buffer_write,
-                                          0,
-                                          &count);
-    ok (w != NULL,
-        "buffer: write watcher close: watcher created");
-    fb = flux_buffer_write_watcher_get_buffer (w);
-    ok (fb != NULL,
-        "buffer: write watcher close: buffer retrieved");
-
-    ok (flux_buffer_write (fb, "foobaz", 6) == 6,
-        "buffer: write to buffer success");
-
-    ok (flux_buffer_write_watcher_is_closed (w, NULL) == 0,
-        "buffer: flux_buffer_write_watcher_is_closed returns false");
-    ok (flux_buffer_write_watcher_close (w) == 0,
-        "buffer: flux_buffer_write_watcher_close: Success");
-    ok (flux_buffer_write_watcher_is_closed (w, NULL) == 0,
-        "buffer: watcher still not closed (close(2) not called yet)");
-    ok (flux_buffer_write_watcher_close (w) == -1 && errno == EINPROGRESS,
-        "buffer: flux_buffer_write_watcher_close: In progress");
-
-    ok (flux_buffer_write (fb, "shouldfail", 10) == -1 && errno == EROFS,
-        "buffer: flux_buffer_write after close fails with EROFS");
-
-    flux_watcher_start (w);
-
-    ok (flux_reactor_run (reactor, 0) == 0,
-        "buffer: reactor ran to completion");
-
-    ok (count == 1,
-        "buffer: write callback called once");
-    ok (flux_buffer_write_watcher_is_closed (w, &errnum) == 1 && errnum == 0,
-        "buffer: flux_buffer_write_watcher_is_closed returns true");
-    ok (flux_buffer_write_watcher_close (w) == -1 && errno == EINVAL,
-        "buffer: flux_buffer_write_watcher_close after close returns EINVAL");
-
-    ok (read (pfds[0], buf, 1024) == 6,
-        "buffer: read from pipe success");
-
-    ok (!memcmp (buf, "foobaz", 6),
-        "buffer: read from pipe returned correct data");
-
-    ok (read (pfds[0], buf, 1024) == 0,
-        "buffer: read from pipe got EOF");
-
-    flux_watcher_stop (w);
-    flux_watcher_destroy (w);
-
-    close (pfds[0]);
-    close (fd[0]);
-    close (fd[1]);
-}
-
-struct buffer_fd_close
-{
-    int count;
-    int fd;
-};
-
-static void buffer_read_fd_close (flux_reactor_t *r, flux_watcher_t *w,
-                                  int revents, void *arg)
-{
-    struct buffer_fd_close *bfc = arg;
-
-    if (revents & FLUX_POLLERR) {
-        ok (false,
-            "buffer corner case: read callback incorrectly called with FLUX_POLLERR");
-    }
-    else if (revents & FLUX_POLLIN) {
-        flux_buffer_t *fb = flux_buffer_read_watcher_get_buffer (w);
-        const void *ptr;
-        int len;
-
-        if (!bfc->count) {
-            ok ((ptr = flux_buffer_read (fb, -1, &len)) != NULL,
-                "buffer corner case: read from buffer success");
-
-            ok (len == 6,
-                "buffer corner case: read returned correct length");
-
-            ok (!memcmp (ptr, "foobar", 6),
-                "buffer corner case: read returned correct data");
-
-            close (bfc->fd);
-        }
-        else {
-            ok ((ptr = flux_buffer_read (fb, -1, &len)) != NULL,
-                "buffer corner case: read from buffer success");
-
-            ok (len == 0,
-                "buffer corner case: read returned 0, socketpair is closed");
-        }
-    }
-    else {
-        ok (false,
-            "buffer corner case: read callback failed to return FLUX_POLLIN: %d", revents);
-    }
-
-    bfc->count++;
-    if (bfc->count == 2)
-        flux_watcher_stop (w);
-    return;
-}
-
-static void buffer_read_line_fd_close (flux_reactor_t *r, flux_watcher_t *w,
-                                       int revents, void *arg)
-{
-    struct buffer_fd_close *bfc = arg;
-
-    if (revents & FLUX_POLLERR) {
-        ok (false,
-            "buffer corner case: read line callback incorrectly called with FLUX_POLLERR");
-    }
-    else if (revents & FLUX_POLLIN) {
-        flux_buffer_t *fb = flux_buffer_read_watcher_get_buffer (w);
-        const void *ptr;
-        int len;
-
-        if (!bfc->count) {
-            ok ((ptr = flux_buffer_read_line (fb, &len)) != NULL,
-                "buffer corner case: read line from buffer success");
-
-            ok (len == 7,
-                "buffer corner case: read line returned correct length");
-
-            ok (!memcmp (ptr, "foobar\n", 7),
-                "buffer corner case: read line returned correct data");
-
-            close (bfc->fd);
-        }
-        else {
-            ok ((ptr = flux_buffer_read_line (fb, &len)) != NULL,
-                "buffer corner case: read line from buffer success");
-
-            ok (len == 0,
-                "buffer corner case: read line returned 0, socketpair is closed");
-        }
-    }
-    else {
-        ok (false,
-            "buffer corner case: read line callback failed to return FLUX_POLLIN: %d", revents);
-    }
-
-    bfc->count++;
-    if (bfc->count == 2)
-        flux_watcher_stop (w);
-    return;
-}
-
-static void buffer_read_line_fd_close_and_left_over_data (flux_reactor_t *r,
-                                                          flux_watcher_t *w,
-                                                          int revents,
-                                                          void *arg)
-{
-    struct buffer_fd_close *bfc = arg;
-
-    if (revents & FLUX_POLLERR) {
-        ok (false,
-            "buffer corner case: read line callback incorrectly called with FLUX_POLLERR");
-    }
-    else if (revents & FLUX_POLLIN) {
-        flux_buffer_t *fb = flux_buffer_read_watcher_get_buffer (w);
-        const void *ptr;
-        int len;
-
-        if (!bfc->count) {
-            ok ((ptr = flux_buffer_read_line (fb, &len)) != NULL,
-                "buffer corner case: read line from buffer success");
-
-            ok (len == 7,
-                "buffer corner case: read line returned correct length");
-
-            ok (!memcmp (ptr, "foobar\n", 7),
-                "buffer corner case: read line returned correct data");
-
-            close (bfc->fd);
-        }
-        else if (bfc->count == 1) {
-            ok ((ptr = flux_buffer_read_line (fb, &len)) != NULL,
-                "buffer corner case: read line from buffer success");
-
-            ok (len == 0,
-                "buffer corner case: read line says no lines available");
-
-            ok ((ptr = flux_buffer_read (fb, -1, &len)) != NULL,
-                "buffer corner case: read from buffer success");
-
-            ok (len == 3,
-                "buffer corner case: read line returned correct length");
-
-            ok (!memcmp (ptr, "foo", 3),
-                "buffer corner case: read line returned correct data");
-        }
-        else {
-            ok ((ptr = flux_buffer_read_line (fb, &len)) != NULL,
-                "buffer corner case: read line from buffer success");
-
-            ok (len == 0,
-                "buffer corner case: read line returned 0, socketpair is closed");
-        }
-    }
-    else {
-        ok (false,
-            "buffer corner case: read line callback failed to return FLUX_POLLIN: %d", revents);
-    }
-
-    bfc->count++;
-    if (bfc->count == 3)
-        flux_watcher_stop (w);
-    return;
-}
-
-static void test_buffer_corner_case (flux_reactor_t *reactor)
-{
-    int fd[2];
-    flux_watcher_t *w;
-    flux_buffer_t *fb;
-    struct buffer_fd_close bfc;
-
-    /* read buffer corner case test - other end closes stream */
-
-    ok (socketpair (PF_LOCAL, SOCK_STREAM|SOCK_NONBLOCK, 0, fd) == 0,
-        "buffer corner case: successfully created socketpair");
-
-    bfc.count = 0;
-    bfc.fd = fd[1];
-    w = flux_buffer_read_watcher_create (reactor,
-                                         fd[0],
-                                         1024,
-                                         buffer_read_fd_close,
-                                         0,
-                                         &bfc);
-    ok (w != NULL,
-        "buffer corner case: read created");
-
-    fb = flux_buffer_read_watcher_get_buffer (w);
-
-    ok (fb != NULL,
-        "buffer corner case: buffer retrieved");
-
-    ok (write (fd[1], "foobar", 6) == 6,
-        "buffer corner case: write to socketpair success");
-
-    flux_watcher_start (w);
-
-    ok (flux_reactor_run (reactor, 0) == 0,
-        "buffer corner case: reactor ran to completion");
-
-    ok (bfc.count == 2,
-        "buffer corner case: read callback successfully called twice");
-
-    flux_watcher_stop (w);
-    flux_watcher_destroy (w);
-
-    close (fd[0]);
-
-    /* read line buffer corner case test - other end closes stream */
-
-    ok (socketpair (PF_LOCAL, SOCK_STREAM|SOCK_NONBLOCK, 0, fd) == 0,
-        "buffer corner case: successfully created socketpair");
-
-    bfc.count = 0;
-    bfc.fd = fd[1];
-    w = flux_buffer_read_watcher_create (reactor,
-                                         fd[0],
-                                         1024,
-                                         buffer_read_line_fd_close,
-                                         FLUX_WATCHER_LINE_BUFFER,
-                                         &bfc);
-    ok (w != NULL,
-        "buffer corner case: read line created");
-
-    fb = flux_buffer_read_watcher_get_buffer (w);
-
-    ok (fb != NULL,
-        "buffer corner case: buffer retrieved");
-
-    ok (write (fd[1], "foobar\n", 7) == 7,
-        "buffer corner case: write to socketpair success");
-
-    flux_watcher_start (w);
-
-    ok (flux_reactor_run (reactor, 0) == 0,
-        "buffer corner case: reactor ran to completion");
-
-    ok (bfc.count == 2,
-        "buffer corner case: read line callback successfully called twice");
-
-    flux_watcher_stop (w);
-    flux_watcher_destroy (w);
-
-    close (fd[0]);
-
-    /* read line buffer corner case test - left over data not a line */
-
-    ok (socketpair (PF_LOCAL, SOCK_STREAM|SOCK_NONBLOCK, 0, fd) == 0,
-        "buffer corner case: successfully created socketpair");
-
-    bfc.count = 0;
-    bfc.fd = fd[1];
-    w = flux_buffer_read_watcher_create (reactor,
-                                         fd[0],
-                                         1024,
-                                         buffer_read_line_fd_close_and_left_over_data,
-                                         FLUX_WATCHER_LINE_BUFFER,
-                                         &bfc);
-    ok (w != NULL,
-        "buffer corner case: read line created");
-
-    fb = flux_buffer_read_watcher_get_buffer (w);
-
-    ok (fb != NULL,
-        "buffer corner case: buffer retrieved");
-
-    ok (write (fd[1], "foobar\nfoo", 10) == 10,
-        "buffer corner case: write to socketpair success");
-
-    flux_watcher_start (w);
-
-    ok (flux_reactor_run (reactor, 0) == 0,
-        "buffer corner case: reactor ran to completion");
-
-    ok (bfc.count == 3,
-        "buffer corner case: read line callback successfully called three times");
-
-    flux_watcher_stop (w);
-    flux_watcher_destroy (w);
-
-    close (fd[0]);
-    close (fd[1]);
-}
-
 static int repeat_countdown = 10;
-static void repeat (flux_reactor_t *r, flux_watcher_t *w,
-                    int revents, void *arg)
+static void repeat (flux_reactor_t *r,
+                    flux_watcher_t *w,
+                    int revents,
+                    void *arg)
 {
     repeat_countdown--;
     if (repeat_countdown == 0)
@@ -946,8 +146,10 @@ static void repeat (flux_reactor_t *r, flux_watcher_t *w,
 
 static int oneshot_runs = 0;
 static int oneshot_errno = 0;
-static void oneshot (flux_reactor_t *r, flux_watcher_t *w,
-                     int revents, void *arg)
+static void oneshot (flux_reactor_t *r,
+                     flux_watcher_t *w,
+                     int revents,
+                     void *arg)
 {
     oneshot_runs++;
     if (oneshot_errno != 0) {
@@ -975,7 +177,11 @@ static void test_timer (flux_reactor_t *reactor)
         "timer: creating negative repeat fails with EINVAL");
     ok ((w = flux_timer_watcher_create (reactor, 0, 0, oneshot, NULL)) != NULL,
         "timer: creating zero timeout oneshot works");
+    ok (!flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns false");
     flux_watcher_start (w);
+    ok (flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns true after flux_watcher_start()");
     oneshot_runs = 0;
     t0 = flux_reactor_now (reactor);
     ok (flux_reactor_run (reactor, 0) == 0,
@@ -1016,7 +222,7 @@ static void test_timer (flux_reactor_t *reactor)
     oneshot_errno = 0;
     ok ((w = flux_timer_watcher_create (reactor, 0, 0, oneshot, NULL)) != NULL,
         "timer: creating timer watcher works");
-    for (i = 0; i < sizeof (t) / sizeof (t[0]); i++) {
+    for (i = 0; i < ARRAY_SIZE (t); i++) {
         flux_timer_watcher_reset (w, t[i], 0);
         flux_watcher_start (w);
         t0 = flux_reactor_now (reactor);
@@ -1032,8 +238,10 @@ static void test_timer (flux_reactor_t *reactor)
 
 /* A reactor callback that immediately stops reactor without error */
 static bool do_stop_callback_ran = false;
-static void do_stop_reactor (flux_reactor_t *r, flux_watcher_t *w,
-                     int revents, void *arg)
+static void do_stop_reactor (flux_reactor_t *r,
+                             flux_watcher_t *w,
+                             int revents,
+                             void *arg)
 {
     do_stop_callback_ran = true;
     flux_reactor_stop (r);
@@ -1084,7 +292,11 @@ static void test_periodic (flux_reactor_t *reactor)
     ok ((w = flux_periodic_watcher_create (reactor, 0, 0, NULL, oneshot, NULL))
         != NULL,
         "periodic: creating zero offset/interval works");
+    ok (!flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns false");
     flux_watcher_start (w);
+    ok (flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns true after flux_watcher_start()");
     oneshot_runs = 0;
     ok (flux_reactor_run (reactor, 0) == 0,
         "periodic: reactor ran to completion");
@@ -1095,8 +307,12 @@ static void test_periodic (flux_reactor_t *reactor)
     flux_watcher_destroy (w);
 
     repeat_countdown = 5;
-    ok ((w = flux_periodic_watcher_create (reactor, 0.01, 0.01,
-                                           NULL, repeat, NULL)) != NULL,
+    ok ((w = flux_periodic_watcher_create (reactor,
+                                           0.01,
+                                           0.01,
+                                           NULL,
+                                           repeat,
+                                           NULL)) != NULL,
         "periodic: creating 10ms interval works");
     flux_watcher_start (w);
     ok (flux_reactor_run (reactor, 0) == 0,
@@ -1114,8 +330,12 @@ static void test_periodic (flux_reactor_t *reactor)
     flux_watcher_stop (w);
     flux_watcher_destroy (w);
 
-    ok ((w = flux_periodic_watcher_create (reactor, 0, 0, resched_cb,
-                                           do_stop_reactor, reactor)) != NULL,
+    ok ((w = flux_periodic_watcher_create (reactor,
+                                           0,
+                                           0,
+                                           resched_cb,
+                                           do_stop_reactor,
+                                           reactor)) != NULL,
         "periodic: creating with resched callback works");
     flux_watcher_start (w);
     ok (flux_reactor_run (reactor, 0) >= 0,
@@ -1127,8 +347,12 @@ static void test_periodic (flux_reactor_t *reactor)
     flux_watcher_destroy (w);
 
     do_stop_callback_ran = false;
-    ok ((w = flux_periodic_watcher_create (reactor, 0, 0, resched_cb_negative,
-                                           do_stop_reactor, reactor)) != NULL,
+    ok ((w = flux_periodic_watcher_create (reactor,
+                                           0,
+                                           0,
+                                           resched_cb_negative,
+                                           do_stop_reactor,
+                                           reactor)) != NULL,
         "periodic: create watcher with misconfigured resched callback");
     flux_watcher_start (w);
     ok (flux_reactor_run (reactor, 0) == 0,
@@ -1140,8 +364,10 @@ static void test_periodic (flux_reactor_t *reactor)
 }
 
 static int idle_count = 0;
-static void idle_cb (flux_reactor_t *r, flux_watcher_t *w,
-                     int revents, void *arg)
+static void idle_cb (flux_reactor_t *r,
+                     flux_watcher_t *w,
+                     int revents,
+                     void *arg)
 {
     if (++idle_count == 42)
         flux_watcher_stop (w);
@@ -1154,7 +380,11 @@ static void test_idle (flux_reactor_t *reactor)
     w = flux_idle_watcher_create (reactor, idle_cb, NULL);
     ok (w != NULL,
         "created idle watcher");
+    ok (!flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns false");
     flux_watcher_start (w);
+    ok (flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns true after flux_watcher_start()");
 
     ok (flux_reactor_run (reactor, 0) == 0,
         "reactor ran successfully");
@@ -1164,22 +394,28 @@ static void test_idle (flux_reactor_t *reactor)
 }
 
 static int prepare_count = 0;
-static void prepare_cb (flux_reactor_t *r, flux_watcher_t *w,
-                        int revents, void *arg)
+static void prepare_cb (flux_reactor_t *r,
+                        flux_watcher_t *w,
+                        int revents,
+                        void *arg)
 {
     prepare_count++;
 }
 
 static int check_count = 0;
-static void check_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
+static void check_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
 {
     check_count++;
 }
 
 static int prepchecktimer_count = 0;
-static void prepchecktimer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                               int revents, void *arg)
+static void prepchecktimer_cb (flux_reactor_t *r,
+                               flux_watcher_t *w,
+                               int revents,
+                               void *arg)
 {
     if (++prepchecktimer_count == 8)
         flux_reactor_stop (r);
@@ -1191,11 +427,18 @@ static void test_prepcheck (flux_reactor_t *reactor)
     flux_watcher_t *prep;
     flux_watcher_t *chk;
 
-    w = flux_timer_watcher_create (reactor, 0.01, 0.01,
-                                   prepchecktimer_cb, NULL);
+    w = flux_timer_watcher_create (reactor,
+                                   0.01,
+                                   0.01,
+                                   prepchecktimer_cb,
+                                   NULL);
     ok (w != NULL,
         "created timer watcher that fires every 0.01s");
+    ok (!flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns false");
     flux_watcher_start (w);
+    ok (flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns true after flux_watcher_start()");
 
     prep = flux_prepare_watcher_create (reactor, prepare_cb, NULL);
     ok (w != NULL,
@@ -1211,8 +454,10 @@ static void test_prepcheck (flux_reactor_t *reactor)
         "reactor ran successfully");
     ok (prepchecktimer_count == 8,
         "timer fired 8 times, then reactor was stopped");
-    diag ("prep %d check %d timer %d", prepare_count, check_count,
-                                       prepchecktimer_count);
+    diag ("prep %d check %d timer %d",
+          prepare_count,
+          check_count,
+          prepchecktimer_count);
     ok (prepare_count >= 8,
         "prepare watcher ran at least once per timer");
     ok (check_count >= 8,
@@ -1224,15 +469,19 @@ static void test_prepcheck (flux_reactor_t *reactor)
 }
 
 static int sigusr1_count = 0;
-static void sigusr1_cb (flux_reactor_t *r, flux_watcher_t *w,
-                        int revents, void *arg)
+static void sigusr1_cb (flux_reactor_t *r,
+                        flux_watcher_t *w,
+                        int revents,
+                        void *arg)
 {
     if (++sigusr1_count == 8)
         flux_reactor_stop (r);
 }
 
-static void sigidle_cb (flux_reactor_t *r, flux_watcher_t *w,
-                        int revents, void *arg)
+static void sigidle_cb (flux_reactor_t *r,
+                        flux_watcher_t *w,
+                        int revents,
+                        void *arg)
 {
     if (kill (getpid (), SIGUSR1) < 0)
         flux_reactor_stop_error (r);
@@ -1246,7 +495,11 @@ static void test_signal (flux_reactor_t *reactor)
     w = flux_signal_watcher_create (reactor, SIGUSR1, sigusr1_cb, NULL);
     ok (w != NULL,
         "created signal watcher");
+    ok (!flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns false");
     flux_watcher_start (w);
+    ok (flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns true after flux_watcher_start()");
 
     idle = flux_idle_watcher_create (reactor, sigidle_cb, NULL);
     ok (idle != NULL,
@@ -1263,8 +516,10 @@ static void test_signal (flux_reactor_t *reactor)
 }
 
 static pid_t child_pid = -1;
-static void child_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
+static void child_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
 {
     int pid = flux_child_watcher_get_rpid (w);
     int rstatus = flux_child_watcher_get_rstatus (w);
@@ -1297,7 +552,11 @@ static void test_child  (flux_reactor_t *reactor)
 
     ok (kill (child_pid, SIGHUP) == 0,
         "sent child SIGHUP");
+    ok (!flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns false");
     flux_watcher_start (w);
+    ok (flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns true after flux_watcher_start()");
 
     ok (flux_reactor_run (r, 0) == 0,
         "reactor ran successfully");
@@ -1312,8 +571,10 @@ struct stat_ctx {
     int stat_nlink;
     enum { STAT_APPEND, STAT_WAIT, STAT_UNLINK } state;
 };
-static void stat_cb (flux_reactor_t *r, flux_watcher_t *w,
-                     int revents, void *arg)
+static void stat_cb (flux_reactor_t *r,
+                     flux_watcher_t *w,
+                     int revents,
+                     void *arg)
 {
     struct stat_ctx *ctx = arg;
     struct stat new, old;
@@ -1333,8 +594,10 @@ static void stat_cb (flux_reactor_t *r, flux_watcher_t *w,
     }
 }
 
-static void stattimer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                          int revents, void *arg)
+static void stattimer_cb (flux_reactor_t *r,
+                          flux_watcher_t *w,
+                          int revents,
+                          void *arg)
 {
     struct stat_ctx *ctx = arg;
     if (ctx->state == STAT_APPEND) {
@@ -1363,10 +626,17 @@ static void test_stat (flux_reactor_t *reactor)
     w = flux_stat_watcher_create (reactor, ctx.path, 0., stat_cb, &ctx);
     ok (w != NULL,
         "created stat watcher");
+    ok (!flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns false");
     flux_watcher_start (w);
-
-    tw = flux_timer_watcher_create (reactor, 0.01, 0.01,
-                                    stattimer_cb, &ctx);
+    ok (flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns true after flux_watcher_start()");
+
+    tw = flux_timer_watcher_create (reactor,
+                                    0.01,
+                                    0.01,
+                                    stattimer_cb,
+                                    &ctx);
     ok (tw != NULL,
         "created timer watcher");
     flux_watcher_start (tw);
@@ -1407,7 +677,12 @@ static void test_active_ref (flux_reactor_t *r)
 
     if (!(w = flux_idle_watcher_create (r, active_idle_cb, &count)))
         BAIL_OUT ("flux_idle_watcher_create failed");
+
+    ok (!flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns false");
     flux_watcher_start (w);
+    ok (flux_watcher_is_active (w),
+        "flux_watcher_is_active() returns true after flux_watcher_start()");
 
     count = 0;
     ok (flux_reactor_run (r, 0) < 0 && count == 16,
@@ -1442,6 +717,90 @@ static void reactor_destroy_early (void)
     flux_watcher_destroy (w);
 }
 
+static void test_reactor_flags (flux_reactor_t *r)
+{
+    errno = 0;
+    ok (flux_reactor_run (r, 0xffff) < 0 && errno == EINVAL,
+        "flux_reactor_run flags=0xffff fails with EINVAL");
+
+    errno = 0;
+    ok (flux_reactor_create (0xffff) == NULL && errno == EINVAL,
+        "flux_reactor_create flags=0xffff fails with EINVAL");
+}
+
+static char cblist[6] = {0};
+static int cblist_index = 0;
+static flux_watcher_t *priority_prep = NULL;
+static flux_watcher_t *priority_idle = NULL;
+
+static void priority_prep_cb (flux_reactor_t *r,
+                              flux_watcher_t *w,
+                              int revents,
+                              void *arg)
+{
+    flux_watcher_start (priority_idle);
+}
+
+static void priority_check_cb (flux_reactor_t *r,
+                               flux_watcher_t *w,
+                               int revents,
+                               void *arg)
+{
+    char *s = arg;
+    /* stick the char name of this watcher into the array, we'll
+     * compare later
+     */
+    cblist[cblist_index++] = s[0];
+    if (cblist_index >= 5) {
+        flux_watcher_stop (priority_prep);
+        flux_watcher_stop (priority_idle);
+    }
+    flux_watcher_stop (w);
+}
+
+static void test_priority (flux_reactor_t *r)
+{
+    flux_watcher_t *a, *b, *c, *d, *e;
+    priority_prep = flux_prepare_watcher_create (r, priority_prep_cb, NULL);
+    ok (priority_prep != NULL,
+        "prep watcher create worked");
+    priority_idle = flux_idle_watcher_create (r, NULL, NULL);
+    ok (priority_idle != NULL,
+        "idle watcher create worked");
+    a = flux_check_watcher_create (r, priority_check_cb, "A");
+    b = flux_check_watcher_create (r, priority_check_cb, "B");
+    c = flux_check_watcher_create (r, priority_check_cb, "C");
+    d = flux_check_watcher_create (r, priority_check_cb, "D");
+    e = flux_check_watcher_create (r, priority_check_cb, "E");
+    ok (a != NULL && b != NULL && c != NULL && d != NULL && e != NULL,
+        "check watcher create worked");
+    // Don't set priority of 'a', it'll be default
+    flux_watcher_set_priority (b, -2);
+    flux_watcher_set_priority (c, 1);
+    flux_watcher_set_priority (d, 2);
+    flux_watcher_set_priority (e, -1);
+    flux_watcher_start (a);
+    flux_watcher_start (b);
+    flux_watcher_start (c);
+    flux_watcher_start (d);
+    flux_watcher_start (e);
+    flux_watcher_start (priority_prep);
+    ok (flux_reactor_run (r, 0) == 0,
+        "reactor ran to completion");
+    /* given priorities, callbacks should be called in the following order
+     * DCAEB
+     */
+    ok (memcmp (cblist, "DCAEB", 5) == 0,
+        "callbacks called in the correct order");
+    flux_watcher_destroy (a);
+    flux_watcher_destroy (b);
+    flux_watcher_destroy (c);
+    flux_watcher_destroy (d);
+    flux_watcher_destroy (e);
+    flux_watcher_destroy (priority_prep);
+    flux_watcher_destroy (priority_idle);
+}
+
 int main (int argc, char *argv[])
 {
     flux_reactor_t *reactor;
@@ -1456,18 +815,20 @@ int main (int argc, char *argv[])
     ok (flux_reactor_run (reactor, 0) == 0,
         "reactor ran to completion (no watchers)");
 
+    ok (!flux_watcher_is_active (NULL),
+        "flux_watcher_is_active (NULL) returns false");
+
     test_timer (reactor);
     test_periodic (reactor);
     test_fd (reactor);
-    test_buffer (reactor);
-    test_buffer_corner_case (reactor);
-    test_zmq (reactor);
     test_idle (reactor);
     test_prepcheck (reactor);
     test_signal (reactor);
     test_child (reactor);
     test_stat (reactor);
     test_active_ref (reactor);
+    test_reactor_flags (reactor);
+    test_priority (reactor);
 
     flux_reactor_destroy (reactor);
 
diff --git a/src/common/libflux/test/reactor_loop.c b/src/common/libflux/test/reactor_loop.c
index 297c950c58cb..d4859f339a4e 100644
--- a/src/common/libflux/test/reactor_loop.c
+++ b/src/common/libflux/test/reactor_loop.c
@@ -8,8 +8,10 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
-#include <czmq.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <stdlib.h>
@@ -18,6 +20,7 @@
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libtap/tap.h"
 #include "src/common/libtestutil/util.h"
+#include "ccan/str/str.h"
 
 static int send_request (flux_t *h, const char *topic)
 {
@@ -39,7 +42,7 @@ static void multmatch1 (flux_t *h, flux_msg_handler_t *mh,
                         const flux_msg_t *msg, void *arg)
 {
     const char *topic;
-    if (flux_msg_get_topic (msg, &topic) < 0 || strcmp (topic, "foo.baz"))
+    if (flux_msg_get_topic (msg, &topic) < 0 || !streq (topic, "foo.baz"))
         flux_reactor_stop_error (flux_get_reactor (h));
     flux_msg_handler_stop (mh);
     multmatch_count++;
@@ -49,7 +52,7 @@ static void multmatch2 (flux_t *h, flux_msg_handler_t *mh,
                         const flux_msg_t *msg, void *arg)
 {
     const char *topic;
-    if (flux_msg_get_topic (msg, &topic) < 0 || strcmp (topic, "foo.bar"))
+    if (flux_msg_get_topic (msg, &topic) < 0 || !streq (topic, "foo.bar"))
         flux_reactor_stop_error (flux_get_reactor (h));
     flux_msg_handler_stop (mh);
     multmatch_count++;
@@ -124,7 +127,7 @@ static void leak_msg_handler (void)
     flux_t *h;
     flux_msg_handler_t *mh;
 
-    if (!(h = loopback_create (0)))
+    if (!(h = flux_open ("loop://", 0)))
         exit (1);
     if (!(mh = flux_msg_handler_create (h, FLUX_MATCH_ANY, dummy, NULL)))
         exit (1);
@@ -132,9 +135,10 @@ static void leak_msg_handler (void)
     flux_close (h);
 }
 
-static void fatal_err (const char *message, void *arg)
+static int comms_err (flux_t *h, void *arg)
 {
-    BAIL_OUT ("fatal error: %s", message);
+    BAIL_OUT ("fatal comms error: %s", strerror (errno));
+    return -1;
 }
 
 int main (int argc, char *argv[])
@@ -144,9 +148,9 @@ int main (int argc, char *argv[])
 
     plan (NO_PLAN);
 
-    if (!(h = loopback_create (0)))
+    if (!(h = flux_open ("loop://", 0)))
         BAIL_OUT ("can't continue without loop handle");
-    flux_fatal_set (h, fatal_err, NULL);
+    flux_comms_error_set (h, comms_err, NULL);
     ok ((reactor = flux_get_reactor (h)) != NULL,
         "obtained reactor");
     if (!reactor)
diff --git a/src/common/libflux/test/request.c b/src/common/libflux/test/request.c
index b842d7fe0995..35452f81c9c7 100644
--- a/src/common/libflux/test/request.c
+++ b/src/common/libflux/test/request.c
@@ -8,13 +8,15 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
 #include <string.h>
-
-#include "message.h"
-#include "request.h"
+#include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 int main (int argc, char *argv[])
 {
@@ -23,7 +25,9 @@ int main (int argc, char *argv[])
     const char *json_str = "{\"a\":42}";
     const void *d;
     const char data[] = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-    int i, l, len = strlen (data);
+    size_t i;
+    size_t l;
+    size_t len = strlen (data);
 
     plan (NO_PLAN);
 
@@ -41,7 +45,7 @@ int main (int argc, char *argv[])
         "flux_request_encode works with NULL payload");
     topic = NULL;
     ok (flux_request_decode (msg, &topic, NULL) == 0
-        && topic != NULL && !strcmp (topic, "foo.bar"),
+        && topic != NULL && streq (topic, "foo.bar"),
         "flux_request_decode returns encoded topic");
     ok (flux_request_decode (msg, NULL, NULL) == 0,
         "flux_request_decode topic is optional");
@@ -56,12 +60,12 @@ int main (int argc, char *argv[])
 
     s = NULL;
     ok (flux_request_decode (msg, NULL, &s) == 0
-        && s != NULL && !strcmp (s, json_str),
+        && s != NULL && streq (s, json_str),
         "flux_request_decode returns encoded payload");
     topic = NULL;
     i = 0;
     ok (flux_request_unpack (msg, &topic, "{s:i}", "a", &i) == 0
-        && i == 42 && topic != NULL && !strcmp (topic, "foo.bar"),
+        && i == 42 && topic != NULL && streq (topic, "foo.bar"),
         "flux_request_unpack returns encoded payload");
 
     errno = 0;
@@ -74,7 +78,7 @@ int main (int argc, char *argv[])
         "flux_request_encode_raw works with NULL payload");
     topic = NULL;
     ok (flux_request_decode_raw (msg, &topic, &d, &l) == 0
-        && topic != NULL && !strcmp (topic, "foo.bar"),
+        && topic != NULL && streq (topic, "foo.bar"),
         "flux_request_decode_raw returns encoded topic");
     ok (flux_request_decode_raw (msg, NULL, &d, &l) == 0,
         "flux_request_decode_raw topic is optional");
diff --git a/src/common/libflux/test/response.c b/src/common/libflux/test/response.c
index f8820ddf1732..6128c270cfff 100644
--- a/src/common/libflux/test/response.c
+++ b/src/common/libflux/test/response.c
@@ -8,22 +8,28 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 #include <errno.h>
 #include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
-#include "src/common/libtestutil/util.h"
+#include "ccan/str/str.h"
 
 int main (int argc, char *argv[])
 {
     flux_t *h;
     flux_msg_t *msg;
+    flux_msg_t *msg2;
     const char *topic, *s;
     const char *json_str = "{\"a\":42}";
     const void *d;
     const char data[] = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
-    int l, len = strlen (data);
+    size_t l;
+    size_t len = strlen (data);
+    int errnum;
 
     plan (NO_PLAN);
 
@@ -43,7 +49,7 @@ int main (int argc, char *argv[])
 
     topic = NULL;
     ok (flux_response_decode (msg, &topic, NULL) == 0
-        && topic != NULL && !strcmp (topic, "foo.bar"),
+        && topic != NULL && streq (topic, "foo.bar"),
         "flux_response_decode returns encoded topic");
     ok (flux_response_decode (msg, NULL, NULL) == 0,
         "flux_response_decode topic is optional");
@@ -60,7 +66,7 @@ int main (int argc, char *argv[])
 
     topic = NULL;
     ok (flux_response_decode_raw (msg, &topic, &d, &l) == 0
-        && topic != NULL && !strcmp (topic, "foo.bar"),
+        && topic != NULL && streq (topic, "foo.bar"),
         "flux_response_decode_raw returns encoded topic");
     ok (flux_response_decode_raw (msg, NULL, &d, &l) == 0,
         "flux_response_decode_raw topic is optional");
@@ -79,7 +85,7 @@ int main (int argc, char *argv[])
 
     s = NULL;
     ok (flux_response_decode (msg, NULL, &s) == 0
-        && s != NULL && !strcmp (s, json_str),
+        && s != NULL && streq (s, json_str),
         "flux_response_decode returns encoded payload");
     ok (flux_response_decode (msg, NULL, NULL) == 0,
         "flux_response_decode works with payload but don't want the payload");
@@ -123,7 +129,7 @@ int main (int argc, char *argv[])
     ok (flux_response_decode (msg, NULL, NULL) < 0
         && errno == 42,
         "flux_response_decode fails with encoded errnum");
-    ok (flux_response_decode_error (msg, &s) == 0 && !strcmp (s, "My Error"),
+    ok (flux_response_decode_error (msg, &s) == 0 && streq (s, "My Error"),
         "flux_response_decode_error includes error message");
     flux_msg_destroy (msg);
 
@@ -151,9 +157,9 @@ int main (int argc, char *argv[])
     flux_msg_destroy (msg);
 
     /* respond with request=NULL */
-    h = loopback_create (0);
+    h = flux_open ("loop://", 0);
     if (!h)
-        BAIL_OUT ("loopback_create");
+        BAIL_OUT ("could not create loop handle");
     errno = 0;
     ok (flux_respond (h, NULL, NULL) < 0 && errno == EINVAL,
         "flux_respond msg=NULL fails with EINVAL");
@@ -168,6 +174,22 @@ int main (int argc, char *argv[])
         "flux_respond_error msg=NULL fails with EINVAL");
     flux_close (h);
 
+    /* errnum=0 */
+    h = flux_open ("loop://", 0);
+    if (!h)
+        BAIL_OUT ("could not create loop handle");
+    msg = flux_request_encode ("foo", NULL);
+    if (!msg)
+        BAIL_OUT ("flux_request_encode failed");
+    ok (flux_respond_error (h, msg, 0, NULL) == 0,
+        "flux_respond_error errno=0 works");
+    msg2 = flux_recv (h, FLUX_MATCH_ANY, 0);
+    ok (flux_msg_get_errnum (msg2, &errnum) == 0 && errnum == EINVAL,
+        "and send a response message with errnum=EINVAL");
+    flux_msg_destroy (msg2);
+    flux_msg_destroy (msg);
+    flux_close (h);
+
     done_testing();
     return (0);
 }
diff --git a/src/common/libflux/test/rpc.c b/src/common/libflux/test/rpc.c
index 2faac97286d4..4b6c783cb404 100644
--- a/src/common/libflux/test/rpc.c
+++ b/src/common/libflux/test/rpc.c
@@ -8,17 +8,26 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include <czmq.h>
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <flux/core.h>
 #include <jansson.h>
+#include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
 #include "src/common/libtestutil/util.h"
 #include "src/common/libtestutil/util_rpc.h"
+#include "ccan/str/str.h"
+
+#include "message_private.h"
+
 
 /* increment integer and send it back */
-void rpctest_incr_cb (flux_t *h, flux_msg_handler_t *mh,
-                     const flux_msg_t *msg, void *arg)
+void rpctest_incr_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     int i;
 
@@ -33,21 +42,22 @@ void rpctest_incr_cb (flux_t *h, flux_msg_handler_t *mh,
 }
 
 /* request nodeid and flags returned in response */
-void rpctest_nodeid_cb (flux_t *h, flux_msg_handler_t *mh,
-                        const flux_msg_t *msg, void *arg)
+void rpctest_nodeid_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
 {
     uint32_t nodeid;
-    uint8_t flags;
 
     if (flux_request_decode (msg, NULL, NULL) < 0)
         goto error;
     if (flux_msg_get_nodeid (msg, &nodeid) < 0)
         goto error;
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        goto error;
-    if (flux_respond_pack (h, msg, "s:i s:i",
-                                   "nodeid", (int)nodeid,
-                                   "flags", flags) < 0)
+    if (flux_respond_pack (h,
+                           msg,
+                           "s:i s:i",
+                           "nodeid", (int)nodeid,
+                           "flags", msg->proto.flags) < 0)
         BAIL_OUT ("flux_respond_pack: %s", flux_strerror (errno));
     return;
 error:
@@ -56,8 +66,10 @@ void rpctest_nodeid_cb (flux_t *h, flux_msg_handler_t *mh,
 }
 
 /* request payload echoed in response */
-void rpctest_echo_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+void rpctest_echo_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     const char *s;
 
@@ -76,14 +88,19 @@ void rpctest_echo_cb (flux_t *h, flux_msg_handler_t *mh,
 }
 
 /* request payload sets error response content */
-void rpctest_echo_error_cb (flux_t *h, flux_msg_handler_t *mh,
-                            const flux_msg_t *msg, void *arg)
+void rpctest_echo_error_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
 {
     int errnum;
     const char *errstr = NULL;
 
-    if (flux_request_unpack (msg, NULL, "{s:i s?:s}",
-                             "errnum", &errnum, "errstr", &errstr) < 0)
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s?s}",
+                             "errnum", &errnum,
+                             "errstr", &errstr) < 0)
         goto error;
     if (errstr) {
         if (flux_respond_error (h, msg, errnum, errstr) < 0)
@@ -101,11 +118,13 @@ void rpctest_echo_error_cb (flux_t *h, flux_msg_handler_t *mh,
 
 
 /* raw request payload echoed in response */
-void rpctest_rawecho_cb (flux_t *h, flux_msg_handler_t *mh,
-                         const flux_msg_t *msg, void *arg)
+void rpctest_rawecho_cb (flux_t *h,
+                         flux_msg_handler_t *mh,
+                         const flux_msg_t *msg,
+                         void *arg)
 {
     const void *d = NULL;
-    int l = 0;
+    size_t l = 0;
 
     if (flux_request_decode_raw (msg, NULL, &d, &l) < 0)
         goto error;
@@ -118,8 +137,10 @@ void rpctest_rawecho_cb (flux_t *h, flux_msg_handler_t *mh,
 }
 
 /* no-payload response */
-void rpctest_hello_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
+void rpctest_hello_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
 {
     const char *s;
 
@@ -137,8 +158,10 @@ void rpctest_hello_cb (flux_t *h, flux_msg_handler_t *mh,
         BAIL_OUT ("flux_respond_error: %s", flux_strerror (errno));
 }
 
-void rpcftest_hello_cb (flux_t *h, flux_msg_handler_t *mh,
-                        const flux_msg_t *msg, void *arg)
+void rpcftest_hello_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
 {
     if (flux_request_unpack (msg, NULL, "{ ! }") < 0)
         goto error;
@@ -152,25 +175,30 @@ void rpcftest_hello_cb (flux_t *h, flux_msg_handler_t *mh,
 
 /* Send back the requested number of responses followed an ENODATA error.
  */
-void rpctest_multi_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
+void rpctest_multi_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
 {
     int i, count;
-    uint8_t flags;
     int noterm;
 
-    if (flux_request_unpack (msg, NULL, "{s:i s:i}", "count", &count,
-                                                     "noterm", &noterm) < 0)
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s:i}",
+                             "count", &count,
+                             "noterm", &noterm) < 0)
         goto error;
     if (!flux_msg_is_streaming (msg)) {
         errno = EPROTO;
         goto error;
     }
-    if (flux_msg_get_flags (msg, &flags) < 0)
-        goto error;
     for (i = 0; i < count; i++) {
-        if (flux_respond_pack (h, msg, "{s:i s:i}", "seq", i,
-                                                    "flags", flags) < 0)
+        if (flux_respond_pack (h,
+                               msg,
+                               "{s:i s:i}",
+                               "seq", i,
+                               "flags", msg->proto.flags) < 0)
             BAIL_OUT ("flux_respond_pack: %s", flux_strerror (errno));
     }
     if (noterm)
@@ -274,6 +302,8 @@ void test_corner_case (flux_t *h)
     ok (flux_rpc_pack (h, "topic", 0, 0xFF, "{ s:s }", "foo", "bar") == NULL
         && errno == EINVAL,
         "flux_rpc_pack fails with EINVAL on invalid flags");
+
+    flux_msg_destroy (msg);
 }
 
 void test_service (flux_t *h)
@@ -295,24 +325,26 @@ void test_service (flux_t *h)
     r = flux_rpc (h, "rpctest.hello", NULL, FLUX_NODEID_ANY, 0);
     ok (r != NULL,
         "flux_rpc sent request to rpctest.hello service");
+    ok (flux_rpc_get_nodeid (r) == FLUX_NODEID_ANY,
+        "flux_rpc_get_nodeid works");
     ok (flux_matchtag_avail (h) == count - 1,
         "flux_rpc allocated one matchtag");
     msg = flux_recv (h, FLUX_MATCH_RESPONSE, 0);
     ok (msg != NULL,
         "flux_recv matched response");
     rc = flux_msg_get_topic (msg, &topic);
-    ok (rc == 0 && !strcmp (topic, "rpctest.hello"),
+    ok (rc == 0 && streq (topic, "rpctest.hello"),
         "response has expected topic %s", topic);
     rc = flux_msg_get_matchtag (msg, &matchtag);
     ok (rc == 0 && matchtag == 1,
         "response has first matchtag", matchtag);
     rc = flux_msg_get_cred (msg, &cred);
     ok (rc == 0
-        && cred.userid == geteuid ()
+        && cred.userid == getuid ()
         && (cred.rolemask & FLUX_ROLE_OWNER),
-        "response has cred.userid=EUID, cred.rolemask including OWNER");
+        "response has cred.userid=UID, cred.rolemask including OWNER");
     errno = 0;
-    rc = flux_msg_get_route_count (msg);
+    rc = flux_msg_route_count (msg);
     ok ((rc == -1 && errno == EINVAL) || (rc == 0),
         "response has no residual route stack");
     flux_future_destroy (r);
@@ -358,7 +390,10 @@ void test_error (flux_t *h)
 
     /* Error response with error message payload.
      */
-    f = flux_rpc_pack (h, "rpctest.echoerr", FLUX_NODEID_ANY, 0,
+    f = flux_rpc_pack (h,
+                       "rpctest.echoerr",
+                       FLUX_NODEID_ANY,
+                       0,
                        "{s:i s:s}",
                        "errnum", 69,
                        "errstr", "Hello world");
@@ -371,13 +406,16 @@ void test_error (flux_t *h)
     ok (flux_rpc_get (f, NULL) < 0 && errno == 69,
         "flux_rpc_get failed with expected errno");
     errstr = flux_future_error_string (f);
-    ok (errstr != NULL && !strcmp (errstr, "Hello world"),
+    ok (errstr != NULL && streq (errstr, "Hello world"),
         "flux_rpc_get_error returned expected error string");
     flux_future_destroy (f);
 
     /* Error response with no error message payload.
      */
-    f = flux_rpc_pack (h, "rpctest.echoerr", FLUX_NODEID_ANY, 0,
+    f = flux_rpc_pack (h,
+                       "rpctest.echoerr",
+                       FLUX_NODEID_ANY,
+                       0,
                        "{s:i}",
                        "errnum", ENOTDIR);
     ok (f != NULL,
@@ -389,7 +427,7 @@ void test_error (flux_t *h)
     ok (flux_rpc_get (f, NULL) < 0 && errno == ENOTDIR,
         "flux_rpc_get failed with expected errno");
     errstr = flux_future_error_string (f);
-    ok (errstr != NULL && !strcmp (errstr, "Not a directory"),
+    ok (errstr != NULL && streq (errstr, "Not a directory"),
         "flux_future_error_string returned ENOTDIR strerror string");
     flux_future_destroy (f);
 }
@@ -449,16 +487,20 @@ void test_encoding (flux_t *h)
         "flux_rpc with payload when payload is expected works");
     json_str = NULL;
     ok (flux_rpc_get (r, &json_str) == 0
-        && json_str && !strcmp (json_str, "{}"),
+        && json_str && streq (json_str, "{}"),
         "flux_rpc_get works and returned expected payload");
     flux_future_destroy (r);
 
     /* working with-payload RPC (raw) */
     const void *d;
     const char data[] = "aaaaaaaaaaaaaaaaaaaa";
-    int l, len = strlen (data);
-    ok ((r = flux_rpc_raw (h, "rpctest.rawecho", data, len,
-                          FLUX_NODEID_ANY, 0)) != NULL,
+    size_t l, len = strlen (data);
+    ok ((r = flux_rpc_raw (h,
+                           "rpctest.rawecho",
+                           data,
+                           len,
+                           FLUX_NODEID_ANY,
+                           0)) != NULL,
         "flux_rpc_raw with payload when payload is expected works");
     d = NULL;
     l = -1;
@@ -470,8 +512,12 @@ void test_encoding (flux_t *h)
 
     /* use newish pack/unpack payload interfaces */
     int i = 0;
-    ok ((r = flux_rpc_pack (h, "rpctest.incr", FLUX_NODEID_ANY, 0,
-                            "{s:i}", "n", 107)) != NULL,
+    ok ((r = flux_rpc_pack (h,
+                            "rpctest.incr",
+                            FLUX_NODEID_ANY,
+                            0,
+                            "{s:i}",
+                            "n", 107)) != NULL,
         "flux_rpc_pack works");
     ok (flux_rpc_get_unpack (r, NULL) < 0
         && errno == EINVAL,
@@ -483,8 +529,12 @@ void test_encoding (flux_t *h)
     flux_future_destroy (r);
 
     /* cause remote EPROTO (unexpected payload) - will be picked up in _getf() */
-    ok ((r = flux_rpc_pack (h, "rpcftest.hello", FLUX_NODEID_ANY, 0,
-                            "{ s:i }", "foo", 42)) != NULL,
+    ok ((r = flux_rpc_pack (h,
+                            "rpcftest.hello",
+                            FLUX_NODEID_ANY,
+                            0,
+                            "{ s:i }",
+                            "foo", 42)) != NULL,
         "flux_rpc_pack with payload when none is expected works, at first");
     errno = 0;
     ok (flux_rpc_get_unpack (r, "{}") < 0
@@ -493,7 +543,11 @@ void test_encoding (flux_t *h)
     flux_future_destroy (r);
 
     /* cause local EPROTO (user incorrectly expects payload) */
-    ok ((r = flux_rpc_pack (h, "rpcftest.hello", FLUX_NODEID_ANY, 0, "{}")) != NULL,
+    ok ((r = flux_rpc_pack (h,
+                            "rpcftest.hello",
+                            FLUX_NODEID_ANY,
+                            0,
+                            "{}")) != NULL,
         "flux_rpc_pack with empty payload works");
     errno = 0;
     ok (flux_rpc_get_unpack (r, "{ s:i }", "foo", &i) < 0
@@ -503,7 +557,12 @@ void test_encoding (flux_t *h)
 
     /* cause local EPROTO (user incorrectly expects empty payload) */
     errno = 0;
-    ok ((r = flux_rpc_pack (h, "rpctest.echo", FLUX_NODEID_ANY, 0, "{ s:i }", "foo", 42)) != NULL,
+    ok ((r = flux_rpc_pack (h,
+                            "rpctest.echo",
+                            FLUX_NODEID_ANY,
+                            0,
+                            "{ s:i }",
+                            "foo", 42)) != NULL,
         "flux_rpc_pack with payload works");
     errno = 0;
     ok (flux_rpc_get_unpack (r, "{ ! }") < 0
@@ -523,7 +582,7 @@ static void then_cb (flux_future_t *r, void *arg)
         "flux_future_wait_for works (ready) in continuation");
     json_str = NULL;
     ok (flux_rpc_get (r, &json_str) == 0
-        && json_str && !strcmp (json_str, "{}"),
+        && json_str && streq (json_str, "{}"),
         "flux_rpc_get works and returned expected payload in continuation");
     flux_reactor_stop (flux_get_reactor (h));
 }
@@ -541,13 +600,13 @@ void test_then (flux_t *h)
         "reactor completed normally");
     flux_future_destroy (r);
 
-    /* ensure contination is called if "get" called before "then"
+    /* ensure continuation is called if "get" called before "then"
      */
     ok ((r = flux_rpc (h, "rpctest.echo", "{}", FLUX_NODEID_ANY, 0)) != NULL,
         "flux_rpc with payload when payload is expected works");
     json_str = NULL;
     ok (flux_rpc_get (r, &json_str) == 0
-        && json_str && !strcmp (json_str, "{}"),
+        && json_str && streq (json_str, "{}"),
         "flux_rpc_get works synchronously and returned expected payload");
     ok (flux_future_then (r, -1., then_cb, h) == 0,
         "flux_future_then works");
@@ -563,20 +622,27 @@ void test_multi_response (flux_t *h)
     flux_future_t *f;
     int seq = -1;
     int inflags = 0;
-    uint8_t outflags = 0;
+    int outflags = 0;
     int count = 0;
     int t1, t2;
     const flux_msg_t *response;
 
-    f = flux_rpc_pack (h, "rpctest.multi", FLUX_NODEID_ANY, FLUX_RPC_STREAMING,
-                          "{s:i s:i}", "count", 3, "noterm", 0);
+    f = flux_rpc_pack (h,
+                       "rpctest.multi",
+                       FLUX_NODEID_ANY,
+                       FLUX_RPC_STREAMING,
+                       "{s:i s:i}",
+                       "count", 3,
+                       "noterm", 0);
     if (!f)
         BAIL_OUT ("flux_rpc_pack failed");
     errno = 0;
-    while (flux_rpc_get_unpack (f, "{s:i s:i}", "seq", &seq,
-                                                "flags", &inflags) == 0) {
+    while (flux_rpc_get_unpack (f,
+                                "{s:i s:i}",
+                                "seq", &seq,
+                                "flags", &inflags) == 0) {
         if (flux_future_get (f, (const void **)&response) == 0)
-            (void)flux_msg_get_flags (response, &outflags);
+            outflags = response->proto.flags;
         count++;
         flux_future_reset (f);
     }
@@ -605,8 +671,13 @@ void test_multi_response_noterm (flux_t *h)
     int t1, t2;
 
     // service will send two responses: seq=0, ENODATA
-    f = flux_rpc_pack (h, "rpctest.multi", FLUX_NODEID_ANY, FLUX_RPC_STREAMING,
-                          "{s:i s:i}", "count", 1, "noterm", 0);
+    f = flux_rpc_pack (h,
+                       "rpctest.multi",
+                       FLUX_NODEID_ANY,
+                       FLUX_RPC_STREAMING,
+                       "{s:i s:i}",
+                       "count", 1,
+                       "noterm", 0);
     if (!f)
         BAIL_OUT ("flux_rpc_pack failed");
     // consume seq=0 response
@@ -630,8 +701,13 @@ void test_multi_response_server_noterm (flux_t *h)
     flux_future_t *f;
 
     // service will send only seq=0 (not ENODATA).
-    f = flux_rpc_pack (h, "rpctest.multi", FLUX_NODEID_ANY, FLUX_RPC_STREAMING,
-                          "{s:i s:i}", "count", 1, "noterm", 1);
+    f = flux_rpc_pack (h,
+                       "rpctest.multi",
+                       FLUX_NODEID_ANY,
+                       FLUX_RPC_STREAMING,
+                       "{s:i s:i}",
+                       "count", 1,
+                       "noterm", 1);
     if (!f)
         BAIL_OUT ("flux_rpc_pack failed");
     flux_future_destroy (f);
@@ -666,8 +742,13 @@ void test_multi_response_then (flux_t *h)
 {
     flux_future_t *f;
 
-    f = flux_rpc_pack (h, "rpctest.multi", FLUX_NODEID_ANY, FLUX_RPC_STREAMING,
-                          "{s:i s:i}", "count", 3, "noterm", 0);
+    f = flux_rpc_pack (h,
+                       "rpctest.multi",
+                       FLUX_NODEID_ANY,
+                       FLUX_RPC_STREAMING,
+                       "{s:i s:i}",
+                       "count", 3,
+                       "noterm", 0);
     if (!f)
         BAIL_OUT ("flux_rpc_pack failed");
     ok (flux_future_then (f, -1., multi_then_cb, NULL) == 0,
@@ -718,8 +799,13 @@ void test_multi_response_then_chain (flux_t *h)
 {
     flux_future_t *f;
 
-    f = flux_rpc_pack (h, "rpctest.multi", FLUX_NODEID_ANY, FLUX_RPC_STREAMING,
-                          "{s:i s:i}", "count", 3, "noterm", 0);
+    f = flux_rpc_pack (h,
+                       "rpctest.multi",
+                       FLUX_NODEID_ANY,
+                       FLUX_RPC_STREAMING,
+                       "{s:i s:i}",
+                       "count", 3,
+                       "noterm", 0);
     if (!f)
         BAIL_OUT ("flux_rpc_pack failed");
     ok (flux_future_then (f, -1., multi_then_first_cb, NULL) == 0,
@@ -728,6 +814,104 @@ void test_multi_response_then_chain (flux_t *h)
         BAIL_OUT ("flux_reactor_run failed");
 }
 
+static void then_nodestroy_cb (flux_future_t *f, void *arg)
+{
+    int *pdone = arg;
+    *pdone = 1;
+}
+
+void test_rpc_active_count (flux_t *h)
+{
+    flux_future_t *f;
+    int rc;
+    int done = 0;
+    int loopcount = 0;
+
+    f = flux_rpc (h, "rpctest.echo", NULL, FLUX_NODEID_ANY, 0);
+    if (!f)
+        BAIL_OUT ("flux_rpc_pack failed");
+    ok (flux_future_then (f, -1., then_nodestroy_cb, &done) == 0,
+        "rpc_active_count: flux_future_then works");
+     while ((rc = flux_reactor_run (flux_get_reactor (h),
+                                   FLUX_REACTOR_NOWAIT))) {
+        if (done == 1 && loopcount++ > 1) {
+            diag ("rpc complete but reactor active count = %d", rc);
+            break;
+        }
+    }
+    ok (rc == 0,
+        "rpc_active_count: rpc active count is 0 after future fulfilled");
+    flux_future_destroy (f);
+
+    /* active count is also decremented on error */
+    done = 0;
+    loopcount = 0;
+    f = flux_rpc (h, "foo", NULL, FLUX_NODEID_ANY, 0);
+    if (!f)
+        BAIL_OUT ("flux_rpc_pack failed");
+    ok (flux_future_then (f, -1., then_nodestroy_cb, &done) == 0,
+        "rpc_active_count: flux_future_then works");
+    while ((rc = flux_reactor_run (flux_get_reactor (h),
+                                   FLUX_REACTOR_NOWAIT))) {
+        if (done == 1 && loopcount++ > 1) {
+            diag ("rpc complete but reactor active count = %d", rc);
+            break;
+        }
+    }
+    ok (rc == 0,
+        "rpc_active_count: rpc active count is 0 after future error");
+    flux_future_destroy (f);
+}
+
+static void multi_then_nodestroy_cb (flux_future_t *f, void *arg)
+{
+    int *pdone = arg;
+    int seq = 0;
+    static int count = 0;
+
+    errno = 0;
+    if (flux_rpc_get_unpack (f, "{s:i}", "seq", &seq) == 0) {
+        flux_future_reset (f);
+        count++;
+        return;
+    }
+    ok (errno == ENODATA,
+        "multi-then: got ENODATA as EOF in continuation");
+    ok (count == 3,
+        "multi-then: received 3 valid responses");
+    *pdone = 1;
+}
+
+void test_multi_rpc_active_count (flux_t *h)
+{
+    flux_future_t *f;
+    int rc;
+    int done = 0;
+    int loopcount = 0;
+
+    f = flux_rpc_pack (h,
+                       "rpctest.multi",
+                        FLUX_NODEID_ANY,
+                        FLUX_RPC_STREAMING,
+                        "{s:i s:i}",
+                        "count", 3,
+                        "noterm", 0);
+    if (!f)
+        BAIL_OUT ("flux_rpc_pack failed");
+    ok (flux_future_then (f, -1., multi_then_nodestroy_cb, &done) == 0,
+        "multi-rpc-active-count: flux_future_then works");
+    while ((rc = flux_reactor_run (flux_get_reactor (h),
+                                   FLUX_REACTOR_NOWAIT))) {
+        if (done == 1 && loopcount++ > 1) {
+            diag ("multi-rpc complete but reactor active count = %d", rc);
+            break;
+        }
+    }
+    ok (rc == 0,
+        "mult-rpc: flux_reactor_run() returns 0 after streaming rpc complete");
+    flux_future_destroy (f);
+}
+
 /* Try flux_rpc_message() with various bad arguments
  */
 void test_rpc_message_inval (flux_t *h)
@@ -791,14 +975,16 @@ void test_rpc_message (flux_t *h)
  */
 static int fake_server (flux_t *h, void *arg)
 {
-    flux_msg_t *msg;
+    flux_msg_t *msg = NULL;
 
     while ((msg = flux_recv (h, FLUX_MATCH_ANY, 0)) != NULL) {
         const char *topic = "unknown";
         (void)flux_msg_get_topic (msg, &topic);
-        if (!strcmp (topic, "shutdown"))
+        if (streq (topic, "shutdown"))
             break;
+        flux_msg_destroy (msg);
     }
+    flux_msg_destroy (msg);
     return 0;
 }
 
@@ -811,14 +997,14 @@ void test_fake_server (void)
 {
     flux_t *h;
 
-    ok ((h = test_server_create (fake_server, NULL)) != NULL,
+    ok ((h = test_server_create (0, fake_server, NULL)) != NULL,
         "test_server_create (recv loop)");
     ok (test_server_stop (h) == 0,
         "test_server_stop worked");
     flux_close (h);
     diag ("completed test with server recv loop");
 
-    ok ((h = test_server_create (fake_server_reactor, NULL)) != NULL,
+    ok ((h = test_server_create (0, fake_server_reactor, NULL)) != NULL,
         "test_server_create (reactor)");
     ok ((test_server_stop (h)) == 0,
         "test_server_stop worked");
@@ -826,9 +1012,28 @@ void test_fake_server (void)
     flux_close (h);
 }
 
-static void fatal_err (const char *message, void *arg)
+static void test_rpc_get_nodeid (flux_t *h)
 {
-    BAIL_OUT ("fatal error: %s", message);
+    flux_future_t *f;
+
+    errno = 0;
+    f = flux_rpc (h, "rpctest.hello", NULL, 0, 0);
+    ok (f != NULL,
+        "flux_rpc sent request to rpctest.hello service");
+    ok (flux_rpc_get_nodeid (f) == 0,
+        "flux_rpc_get_nodeid works");
+    ok (flux_future_get (f, NULL) == 0,
+        "flux_future_get works");
+    ok (flux_rpc_get_nodeid (f) == 0,
+        "flux_rpc_get_nodeid still works after future_get()");
+
+    flux_future_destroy (f);
+}
+
+static int comms_err (flux_t *h, void *arg)
+{
+    BAIL_OUT ("fatal coms error: %s", strerror (errno));
+    return -1;
 }
 
 int main (int argc, char *argv[])
@@ -837,16 +1042,14 @@ int main (int argc, char *argv[])
 
     plan (NO_PLAN);
 
-    test_server_environment_init ("rpc-test");
-
     test_fake_server ();
 
-    h = test_server_create (test_server, NULL);
+    h = test_server_create (0, test_server, NULL);
     ok (h != NULL,
         "created test server thread");
     if (!h)
         BAIL_OUT ("can't continue without test server");
-    flux_fatal_set (h, fatal_err, NULL);
+    flux_comms_error_set (h, comms_err, NULL);
     flux_flags_set (h, FLUX_O_MATCHDEBUG);
 
     test_corner_case (h);
@@ -863,6 +1066,11 @@ int main (int argc, char *argv[])
     test_rpc_message_inval (h);
     test_rpc_message (h);
 
+    test_rpc_active_count (h);
+    test_multi_rpc_active_count (h);
+
+    test_rpc_get_nodeid (h);
+
     ok (test_server_stop (h) == 0,
         "stopped test server thread");
     flux_close (h); // destroys test server
diff --git a/src/common/libflux/test/rpc_chained.c b/src/common/libflux/test/rpc_chained.c
index a4bc89e7d338..3015e659d2f9 100644
--- a/src/common/libflux/test/rpc_chained.c
+++ b/src/common/libflux/test/rpc_chained.c
@@ -8,10 +8,14 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include <czmq.h>
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <flux/core.h>
+
 #include "src/common/libtap/tap.h"
 #include "src/common/libtestutil/util.h"
+#include "ccan/str/str.h"
 
 void rpctest_incr_cb (flux_t *h, flux_msg_handler_t *mh,
                       const flux_msg_t *msg, void *arg)
@@ -50,9 +54,10 @@ int test_server (flux_t *h, void *arg)
     return 0;
 }
 
-void fatal_err (const char *message, void *arg)
+int comms_err (flux_t *h, void *arg)
 {
-    BAIL_OUT ("fatal error: %s", message);
+    BAIL_OUT ("fatal comms error: %s", strerror (errno));
+    return -1;
 }
 
 flux_future_t *incr (flux_t *h, int n)
@@ -297,7 +302,7 @@ void test_or_then (flux_t *h)
     cmp_ok (rc, "<", 0, "or-then: flux_future_get on composite returns < 0");
     cmp_ok (errno, "==", EPROTO, "or-then: errno is expected");
     errmsg = flux_future_error_string (f2);
-    ok (!strcmp (errmsg, "Protocol error"),
+    ok (streq (errmsg, "Protocol error"),
         "or-then: error string reported correctly");
     flux_future_destroy (f2);
 }
@@ -333,7 +338,7 @@ void test_or_then_error_string (flux_t *h)
     cmp_ok (rc, "<", 0, "or-then: flux_future_get on composite returns < 0");
     cmp_ok (errno, "==", EPROTO, "or-then: errno is expected");
     errmsg = flux_future_error_string (f2);
-    ok (!strcmp (errmsg, "my errstr"),
+    ok (streq (errmsg, "my errstr"),
         "or-then: error string reported correctly");
     flux_future_destroy (f2);
 }
@@ -344,14 +349,12 @@ int main (int argc, char *argv[])
 
     plan (NO_PLAN);
 
-    test_server_environment_init ("rpc-chained-test");
-
-    h = test_server_create (test_server, NULL);
+    h = test_server_create (0, test_server, NULL);
     ok (h != NULL,
         "created test server thread");
     if (!h)
         BAIL_OUT ("can't continue without test server");
-    flux_fatal_set (h, fatal_err, NULL);
+    flux_comms_error_set (h, comms_err, NULL);
 
     test_sanity_now (h);
     test_sanity_then (h);
diff --git a/src/common/libflux/test/rpc_security.c b/src/common/libflux/test/rpc_security.c
index 2ed2a27eed86..0b58c1fe7b8e 100644
--- a/src/common/libflux/test/rpc_security.c
+++ b/src/common/libflux/test/rpc_security.c
@@ -8,13 +8,15 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
 #include <flux/core.h>
 #include <inttypes.h>
 
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libtap/tap.h"
-#include "src/common/libtestutil/util.h"
 
 static int cred_get (flux_t *h, struct flux_msg_cred *cr)
 {
@@ -55,9 +57,9 @@ static void check_rpc_oneway (flux_t *h)
     ok (msg != NULL,
         "received looped back request");
     ok (flux_msg_get_cred (msg, &cr) == 0
-        && cr.userid == geteuid ()
+        && cr.userid == getuid ()
         && cr.rolemask == FLUX_ROLE_OWNER,
-        "request contains userid=EUID, rolemask=OWNER");
+        "request contains userid=UID, rolemask=OWNER");
     flux_msg_destroy (msg);
 }
 
@@ -68,7 +70,7 @@ static void check_rpc_oneway_faked (flux_t *h)
     struct flux_msg_cred saved, new, cr;
 
     ok (cred_get (h, &saved) == 0
-        && saved.userid == geteuid() && saved.rolemask == FLUX_ROLE_OWNER,
+        && saved.userid == getuid() && saved.rolemask == FLUX_ROLE_OWNER,
         "saved connector creds, with expected values");
 
     new.userid = 9999;
@@ -155,7 +157,7 @@ static void check_rpc_default_policy (flux_t *h)
     /* Attempt with non-owner creds
      */
     ok (cred_get (h, &saved) == 0
-        && saved.userid == geteuid() && saved.rolemask == FLUX_ROLE_OWNER,
+        && saved.userid == getuid() && saved.rolemask == FLUX_ROLE_OWNER,
         "saved connector creds, with expected values");
     new.userid = 9999;
     new.rolemask = 0x80000000;
@@ -211,7 +213,7 @@ static void check_rpc_open_policy (flux_t *h)
     /* Attempt with non-owner creds
      */
     ok (cred_get (h, &saved) == 0
-        && saved.userid == geteuid() && saved.rolemask == FLUX_ROLE_OWNER,
+        && saved.userid == getuid() && saved.rolemask == FLUX_ROLE_OWNER,
         "saved connector creds, with expected values");
     new.userid = 9999;
     new.rolemask = 0x80000000;
@@ -235,7 +237,7 @@ static void check_rpc_open_policy (flux_t *h)
     flux_msg_handler_destroy (mh);
 }
 
-static void check_rpc_targetted_policy (flux_t *h)
+static void check_rpc_targeted_policy (flux_t *h)
 {
     flux_future_t *f;
     flux_msg_handler_t *mh;
@@ -244,14 +246,14 @@ static void check_rpc_targetted_policy (flux_t *h)
     int rc;
 
     ok ((mh = testrpc1_handler_create (h)) != NULL,
-        "created message handler with targetted policy");
+        "created message handler with targeted policy");
     if (mh == NULL)
         BAIL_OUT ("flux_msg_handler_create: %s", flux_strerror (errno));
     flux_msg_handler_deny_rolemask (mh, FLUX_ROLE_ALL);
     flux_msg_handler_allow_rolemask (mh, allow);
 
     ok (cred_get (h, &saved) == 0
-        && saved.userid == geteuid() && saved.rolemask == FLUX_ROLE_OWNER,
+        && saved.userid == getuid() && saved.rolemask == FLUX_ROLE_OWNER,
         "saved connector creds, with expected values");
 
     /* Attempt with default creds.
@@ -313,9 +315,10 @@ static void check_rpc_targetted_policy (flux_t *h)
     flux_msg_handler_destroy (mh);
 }
 
-static void fatal_err (const char *message, void *arg)
+static int comms_err (flux_t *h, void *arg)
 {
-    BAIL_OUT ("fatal error: %s", message);
+    BAIL_OUT ("fatal comms error: %s", strerror (errno));
+    return -1;
 }
 
 int main (int argc, char *argv[])
@@ -324,15 +327,15 @@ int main (int argc, char *argv[])
 
     plan (NO_PLAN);
 
-    if (!(h = loopback_create (0)))
-        BAIL_OUT ("cannot continue without loopback handle");
-    flux_fatal_set (h, fatal_err, NULL);
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("cannot continue without loop handle");
+    flux_comms_error_set (h, comms_err, NULL);
 
     check_rpc_oneway (h);
     check_rpc_oneway_faked (h);
     check_rpc_default_policy (h);
     check_rpc_open_policy (h);
-    check_rpc_targetted_policy (h);
+    check_rpc_targeted_policy (h);
 
     flux_close (h);
     done_testing();
diff --git a/src/common/libflux/test/rpcscale.c b/src/common/libflux/test/rpcscale.c
new file mode 100644
index 000000000000..7048dc4e5ac3
--- /dev/null
+++ b/src/common/libflux/test/rpcscale.c
@@ -0,0 +1,250 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* rpcscale.c - send a batch of requests / handle batch of responses */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <flux/optparse.h>
+#include <sys/resource.h>
+
+#include "src/common/libutil/monotime.h"
+#include "src/common/libutil/parse_size.h"
+#include "src/common/libtestutil/util.h"
+
+#include "tap.h"
+
+void ping_cb (flux_t *h,
+              flux_msg_handler_t *mh,
+              const flux_msg_t *msg,
+              void *arg)
+{
+    const void *payload;
+    size_t payload_len;
+
+    if (flux_request_decode_raw (msg, NULL, &payload, &payload_len) < 0)
+        goto error;
+    if (flux_msg_is_noresponse (msg))
+        return;
+    if (flux_respond_raw (h, msg, payload, payload_len) < 0)
+        diag ("error responding to ping: %s", strerror (errno));
+    // if --streaming, terminate the RPC with ENODATA response
+    if (flux_msg_is_streaming (msg)) {
+        errno = ENODATA;
+        goto error;
+    }
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        diag ("error responding to ping: %s", strerror (errno));
+}
+
+const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,   "ping",       ping_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+int test_server (flux_t *h, void *arg)
+{
+    flux_msg_handler_t **handlers = NULL;
+    if (flux_msg_handler_addvec (h, htab, NULL, &handlers) < 0) {
+        diag ("flux_msg_handler_addvec failed");
+        return -1;
+    }
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
+        diag ("flux_reactor_run failed");
+        return -1;
+    }
+    flux_msg_handler_delvec (handlers);
+    return 0;
+}
+
+static int response_count;
+
+void ping_continuation (flux_future_t *f, void *arg)
+{
+    optparse_t *p = arg;
+    bool streaming = optparse_hasopt (p, "streaming");
+
+    response_count++;
+
+    if (flux_rpc_get (f, NULL) < 0) {
+        if (errno != ENODATA)
+            diag ("ping error: %s", strerror (errno));
+        flux_future_destroy (f);
+        return;
+    }
+    if (streaming)
+        flux_future_reset (f);
+    else
+        flux_future_destroy (f);
+}
+
+void logger (const char *buf, int len, void *arg)
+{
+    diag ("%.*s", len, buf);
+}
+
+static struct optparse_option opts[] = {
+    { .name = "count", .key = 'c', .has_arg = 1, .arginfo = "N",
+      .usage = "Set message count per iteration (default 10000)",
+    },
+    { .name = "iter", .key = 'I', .has_arg = 1, .arginfo = "N",
+      .usage = "Set number of iterations (default 2)",
+    },
+    { .name = "rpctrack", .key = 'r', .has_arg = 0,
+      .usage = "Enable FLUX_O_RPCTRACK",
+    },
+    { .name = "matchdebug", .key = 'd', .has_arg = 0,
+      .usage = "Enable FLUX_O_MATCHDEBUG",
+    },
+    { .name = "streaming", .key = 's', .has_arg = 0,
+      .usage = "Enable FLUX_RPC_STREAMING",
+    },
+    { .name = "noresponse", .key = 'n', .has_arg = 0,
+      .usage = "Enable FLUX_RPC_NORESPONSE",
+    },
+    { .name = "pad", .key = 'p', .has_arg = 1, .arginfo = "N[kKMGPE]",
+      .usage = "pad message with payload",
+    },
+    OPTPARSE_TABLE_END
+};
+
+int main (int argc, char *argv[])
+{
+    optparse_t *p;
+    flux_t *h;
+    int test_size;
+    int test_iterations;
+    struct rusage ru;
+    int open_flags = 0;
+    int rpc_flags = 0;
+    uint64_t payload_size = 0;
+    void *payload = NULL;
+
+    plan (NO_PLAN);
+
+    if (!(p = optparse_create ("rpcscale")))
+        BAIL_OUT ("optparse_create");
+    if (optparse_add_option_table (p, opts) != OPTPARSE_SUCCESS)
+        BAIL_OUT ("optparse_add_option_table() failed");
+    if (optparse_parse_args (p, argc, argv) < argc)
+        BAIL_OUT ("Type rpcscale -h for options.");
+    test_size = optparse_get_int (p, "count", 10000);
+    test_iterations = optparse_get_int (p, "iter", 2);
+    if (optparse_hasopt (p, "rpctrack"))
+        open_flags |= FLUX_O_RPCTRACK;
+    if (optparse_hasopt (p, "matchdebug"))
+        open_flags |= FLUX_O_MATCHDEBUG;
+    if (optparse_hasopt (p, "streaming"))
+        rpc_flags |= FLUX_RPC_STREAMING;
+    if (optparse_hasopt (p, "noresponse"))
+        rpc_flags |= FLUX_RPC_NORESPONSE;
+    if (optparse_hasopt (p, "pad")) {
+        const char *s = optparse_get_str (p, "pad", "0");
+        if (parse_size (s, &payload_size) < 0)
+            BAIL_OUT ("could not parse pad size");
+        if (!(payload = calloc (1, payload_size)))
+            BAIL_OUT ("out of memory");
+    }
+    h = test_server_create (open_flags, test_server, NULL);
+    ok (h != NULL,
+        "created test server thread");
+    if (!h)
+        BAIL_OUT ("can't continue without test server");
+    flux_log_set_redirect (h, logger, NULL);
+
+    for (int iter = 1; iter <= test_iterations; iter++) {
+        int errors;
+        int rc;
+        struct timespec t0;
+        double t;
+        flux_future_t *f;
+
+        diag ("Iteration %d of %d", iter, test_iterations);
+
+        /* Send
+         */
+        monotime (&t0);
+        errors = 0;
+        for (int i = 0; i < test_size; i++) {
+            if (!(f = flux_rpc_raw (h,
+                                    "ping",
+                                    payload,
+                                    payload_size,
+                                    FLUX_NODEID_ANY,
+                                    rpc_flags))) {
+                diag ("error sending rpc #%d", i);
+                errors++;
+            }
+            else if ((rpc_flags & FLUX_RPC_NORESPONSE)) {
+                flux_future_destroy (f);
+            }
+            else {
+                if (flux_future_then (f, -1, ping_continuation, p) < 0) {
+                    diag ("error registeirng continuation for rpc #%d", i);
+                    flux_future_destroy (f);
+                }
+            }
+        }
+        t = monotime_since (t0) / 1000;
+        if (getrusage (RUSAGE_SELF, &ru) < 0)
+            BAIL_OUT ("gerusage failed");
+        diag ("send %d req in %.2fs (%.1f Kmsg/s) rss %.1fMB",
+              test_size,
+              t,
+              1E-3 * test_size / t,
+              1E-3 * ru.ru_maxrss);
+        ok (errors == 0,
+            "sent batch of requests with no errors");
+
+        /* Recv
+         */
+        response_count = 0;
+        monotime (&t0);
+        rc = flux_reactor_run (flux_get_reactor (h), 0);
+        t = monotime_since (t0) / 1000;
+        if (getrusage (RUSAGE_SELF, &ru) < 0)
+            BAIL_OUT ("gerusage failed");
+        diag ("recv %d rep in %.2fs (%.1f Kmsg/s) rss %.1fMB",
+              response_count,
+              t,
+              1E-3 * response_count / t,
+              1E-3 * ru.ru_maxrss);
+        ok (rc == 0,
+            "processed responses with no errors");
+        if (rc != 0)
+            diag ("reactor returned %d", rc);
+
+        /* Wait for server to clear requests if --noresponse
+         */
+        if ((rpc_flags & FLUX_RPC_NORESPONSE)) {
+            flux_future_t *f2;
+            if (!(f2 = flux_rpc (h, "ping", NULL, FLUX_NODEID_ANY, 0))
+                || flux_rpc_get (f2, NULL) < 0)
+                BAIL_OUT ("synchronous ping failed");
+            flux_future_destroy (f2);
+        }
+    }
+
+    ok (test_server_stop (h) == 0,
+        "stopped test server thread");
+    flux_close (h); // destroys test server
+
+    free (payload);
+    optparse_destroy (p);
+
+    done_testing();
+    return (0);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libflux/test/sync.c b/src/common/libflux/test/sync.c
new file mode 100644
index 000000000000..dd2f3c141f04
--- /dev/null
+++ b/src/common/libflux/test/sync.c
@@ -0,0 +1,163 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <string.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+
+void send_fake_heartbeat (flux_t *h, int seq)
+{
+    flux_msg_t *msg;
+
+    if (!(msg = flux_event_encode ("heartbeat.pulse", NULL))
+            || flux_msg_set_seq (msg, seq) < 0
+            || flux_send (h, msg, 0) < 0)
+        BAIL_OUT ("failed to send fake heartbeat");
+    flux_msg_destroy (msg);
+    diag ("sent heartbeat");
+}
+
+/* N.B. It's not advisable to use flux_sync_create() in this manner as
+ * the old event messages accumulate in the handle queue.  (See "special note"
+ * in src/common/libflux/sync.c).  However it should _work_ to be consistent
+ * with expected future semantics.
+ */
+void test_non_reactive_loop (void)
+{
+    flux_t *h;
+    flux_future_t *f;
+    int rc;
+    int i;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop handle");
+
+    f = flux_sync_create (h, 0.);
+    ok (f != NULL,
+        "flux_sync_create works");
+
+    errno = 0;
+    ok (flux_future_wait_for (f, 0.1) < 0,
+        "flux_future_wait_for timed out waiting for (not sent) heartbeat");
+    flux_future_reset (f); // not needed on timeout but if test fails...
+
+    for (i = 0; i < 4; i++) {
+        send_fake_heartbeat (h, i);
+        rc = flux_future_wait_for (f, 10.);
+        ok (rc == 0,
+            "flux_future_wait_for (%d) success", i);
+        if (rc < 0)
+            diag ("flux_future_wait_for (%d): %s", i, strerror (errno));
+        ok (flux_future_get (f, NULL) == 0,
+            "flux_future_get (%d) success", i);
+        flux_future_reset (f);
+    }
+
+    errno = 0;
+    rc = flux_future_wait_for (f, 0.1);
+    ok (rc < 0 && errno == ETIMEDOUT,
+        "flux_future_wait_for timed out waiting for (not sent) heartbeat");
+    if (rc == 0 || errno != ETIMEDOUT)
+        diag ("flux_future_wait_for: %s", rc==0 ? "success" : strerror (errno));
+
+    flux_future_destroy (f);
+    flux_close (h);
+}
+
+struct heartbeat_ctx {
+    int seq;
+    flux_t *h;
+};
+
+void heartbeat_timer (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct heartbeat_ctx *ctx = arg;
+
+    send_fake_heartbeat (ctx->h, ctx->seq++);
+}
+
+void continuation (flux_future_t *f, void *arg)
+{
+    int *count = arg;
+
+    diag ("continuation %d", *count);
+    if (--(*count) == 0)
+        flux_reactor_stop (flux_future_get_reactor (f));
+    else
+        flux_future_reset (f);
+}
+
+void test_sync_reactive (double heartrate, double min, double max)
+{
+    flux_t *h = flux_open ("loop://", 0);
+    struct heartbeat_ctx ctx = { .h = h, .seq = 0 };
+    flux_reactor_t *r;
+    flux_watcher_t *timer;
+    flux_future_t *f;
+    int count;
+
+    if (!h)
+        BAIL_OUT ("could not create loop handle");
+    if (!(r = flux_get_reactor (h)))
+        BAIL_OUT ("flux_get_reactor failed on loopback handle");
+
+    if (!(timer = flux_timer_watcher_create (r,
+                                             0.,
+                                             heartrate,
+                                             heartbeat_timer,
+                                             &ctx)))
+        BAIL_OUT ("could not create timer watcher");
+    flux_watcher_start (timer);
+
+    f = flux_sync_create (h, min);
+    ok (f != NULL,
+        "flux_sync_create works");
+    ok (flux_future_then (f, max, continuation, &count) == 0,
+        "flux_future_then heartrate=%.2f min=%.2f max=max", heartrate, min,max);
+    count = 4;
+    ok (flux_reactor_run (r, 0) >= 0,
+        "flux_reactor_run returned success");
+    ok (count == 0,
+        "sync continuation ran the expected number of times");
+
+    flux_future_destroy (f);
+    flux_watcher_destroy (timer);
+    flux_close (h);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    errno = 0;
+    ok (flux_sync_create (NULL, 0.) == NULL && errno == EINVAL,
+        "flux_sync_create h=NULL fails with EINVAL");
+
+    test_non_reactive_loop ();
+    test_sync_reactive (0.01, 0.,   5.);    // driven by heartbeat
+    test_sync_reactive (0.01, 0.1,  5.);    //   same, but skip some heartbeats
+    test_sync_reactive (5.,   0,    0.01);  // driven by 'max' timeout
+
+    done_testing();
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libflux/test/tagpool.c b/src/common/libflux/test/tagpool.c
deleted file mode 100644
index ff31f07976e7..000000000000
--- a/src/common/libflux/test/tagpool.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include "src/common/libflux/message.h"
-#include "src/common/libflux/tagpool.h"
-#include "src/common/libtap/tap.h"
-
-int main (int argc, char *argv[])
-{
-    struct tagpool *t;
-    uint32_t tags[256];
-    uint32_t avail;
-    uint32_t size;
-    int i, j, k, count, duplicates;
-
-    plan (NO_PLAN);
-
-    t = tagpool_create ();
-    ok (t != NULL,
-        "tagpool_create works");
-
-    tags[0] = tagpool_alloc (t);
-    ok (tags[0] == 1,
-        "regular: allocated first tag");
-    tags[1] = tagpool_alloc (t);
-    ok (tags[1] == 2,
-        "regular: allocated second tag");
-    tagpool_free (t, tags[0]);
-    tags[2] = tagpool_alloc (t);
-    ok (tags[2] == 1,
-        "regular: got first tag again after it was freed");
-    tagpool_free (t, tags[1]);
-    tags[3] = tagpool_alloc (t);
-    ok (tags[3] == 2,
-        "regular: got second tag again after it was freed");
-    tagpool_free (t, tags[2]);
-    tagpool_free (t, tags[3]);
-
-    size = tagpool_getattr (t, TAGPOOL_ATTR_SIZE);
-    avail = tagpool_getattr (t, TAGPOOL_ATTR_AVAIL);
-    ok (avail == size,
-        "regular: all tags available");
-
-    ok (avail >= 256,
-        "regular: at least 256 tags available");
-    for (i = 0; i < 256; i++) {
-        tags[i] = tagpool_alloc (t);
-        if (tags[i] == FLUX_MATCHTAG_NONE)
-            break;
-    }
-    ok (i == 256,
-        "regular: tagpool_alloc worked 256 times");
-    avail = tagpool_getattr (t, TAGPOOL_ATTR_AVAIL);
-    if (avail != size - 256)
-        diag ("wrong number avail: %u of %u", avail, size);
-    ok (avail == size - 256,
-        "regular: pool depleted by 256");
-
-    duplicates = 0;
-    for (j = 0; j < i; j++) {
-        for (k = j + 1; k < i; k++)
-            if (tags[j] == tags[k])
-                duplicates++;
-    }
-    ok (duplicates == 0,
-        "regular: allocated tags contain no duplicates");
-
-    while (--i >= 0)
-        tagpool_free (t, tags[i]);
-    avail = tagpool_getattr (t, TAGPOOL_ATTR_AVAIL);
-    ok (avail == size,
-        "regular: tagpool_free restored all to pool");
-
-    count = 0;
-    while (tagpool_alloc (t) != FLUX_MATCHTAG_NONE)
-        count++;
-    ok (count == size,
-        "regular: entire pool allocated by tagpool_alloc loop");
-    avail = tagpool_getattr (t, TAGPOOL_ATTR_AVAIL);
-    ok (avail == 0,
-        "regular: pool is exhausted");
-
-    tagpool_destroy (t);
-
-    done_testing ();
-    return (0);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libflux/test/version.c b/src/common/libflux/test/version.c
index e12ecad82e5d..762bb4e5e578 100644
--- a/src/common/libflux/test/version.c
+++ b/src/common/libflux/test/version.c
@@ -8,10 +8,14 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include "src/common/libflux/version.h"
-#include "src/common/libtap/tap.h"
-
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 int main (int argc, char *argv[])
 {
@@ -30,7 +34,7 @@ int main (int argc, char *argv[])
 
     snprintf (vs, sizeof (vs), "%d.%d.%d", a,b,c);
     s = flux_core_version_string ();
-    ok (s != NULL && !strncmp (s, vs, strlen (vs)),
+    ok (s != NULL && strstarts (s, vs),
         "flux_core_version_string returned expected string");
     diag (s);
 
diff --git a/src/common/libflux/types.h b/src/common/libflux/types.h
index b524de3117c0..9f053534b488 100644
--- a/src/common/libflux/types.h
+++ b/src/common/libflux/types.h
@@ -17,6 +17,27 @@ extern "C" {
 
 typedef void (*flux_free_f)(void *arg);
 
+/*  Generic container for holding textual errors from selected libflux
+ *   functions:
+ */
+typedef struct {
+    char text[160];
+} flux_error_t;
+
+/* libflux convenience typedefs
+ */
+typedef struct flux_msg flux_msg_t;
+typedef struct flux_handle flux_t;
+typedef struct flux_watcher flux_watcher_t;
+typedef struct flux_reactor flux_reactor_t;
+typedef struct flux_future flux_future_t;
+
+/* FLUX_DEPRECATED may be altered during pre-processing, check for
+ * definition */
+#ifndef FLUX_DEPRECATED
+#define FLUX_DEPRECATED(...) __VA_ARGS__ __attribute__((deprecated))
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/common/libflux/version.c b/src/common/libflux/version.c
index acb204f5a526..c6821cf5c2e5 100644
--- a/src/common/libflux/version.c
+++ b/src/common/libflux/version.c
@@ -11,8 +11,7 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-
-#include "version.h"
+#include <flux/core.h>
 
 const char *flux_core_version_string (void)
 {
diff --git a/src/common/libfluxutil/Makefile.am b/src/common/libfluxutil/Makefile.am
new file mode 100644
index 000000000000..c3a8397bfde7
--- /dev/null
+++ b/src/common/libfluxutil/Makefile.am
@@ -0,0 +1,21 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux \
+	$(JANSSON_CFLAGS)
+
+noinst_LTLIBRARIES = libfluxutil.la
+libfluxutil_la_SOURCES = \
+	method.h \
+	method.c \
+	policy.h \
+	policy.c
diff --git a/src/common/libfluxutil/method.c b/src/common/libfluxutil/method.c
new file mode 100644
index 000000000000..da50c51cf2c7
--- /dev/null
+++ b/src/common/libfluxutil/method.c
@@ -0,0 +1,234 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <time.h>
+#include <sys/resource.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "method.h"
+
+static char *make_json_response_payload (flux_t *h,
+                                         const char *request_payload,
+                                         const char *route,
+                                         struct flux_msg_cred cred)
+{
+    uint32_t rank;
+    json_t *o = NULL;
+    json_t *add = NULL;
+    char *result = NULL;
+
+    if (!request_payload || !(o = json_loads (request_payload, 0, NULL))) {
+        errno = EPROTO;
+        goto done;
+    }
+    if (flux_get_rank (h, &rank) < 0)
+        goto done;
+    if (!(add = json_pack ("{s:s s:i s:i s:i}",
+                           "route", route,
+                           "userid", cred.userid,
+                           "rolemask", cred.rolemask,
+                           "rank", rank))) {
+        errno = ENOMEM;
+        goto done;
+    }
+    if (json_object_update (o, add) < 0) {
+        errno = ENOMEM;
+        goto done;
+    }
+    if (!(result = json_dumps (o, 0))) {
+        errno = ENOMEM;
+        goto done;
+    }
+done:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    ERRNO_SAFE_WRAP (json_decref, add);
+    return result;
+}
+
+/* The uuid is tacked onto the route string constructed for
+ * ping responses.  Truncate the uuid to 8 chars to match policy
+ * of flux_msg_route_string().
+ */
+static const char *get_uuid (flux_t *h)
+{
+    static char uuid[9] = { 0 };
+
+    if (strlen (uuid) == 0) {
+        const char *s = flux_aux_get (h, "flux::uuid");
+        if (!s)
+            return NULL;
+        (void)snprintf (uuid, sizeof (uuid), "%s", s);
+    }
+    return uuid;
+}
+
+void method_ping_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    const char *uuid;
+    const char *json_str;
+    char *route_str = NULL;
+    char *new_str;
+    size_t new_size;
+    char *resp_str = NULL;
+    struct flux_msg_cred cred;
+
+    if (flux_request_decode (msg, NULL, &json_str) < 0
+        || flux_msg_get_cred (msg, &cred) < 0
+        || !(uuid = get_uuid (h)))
+        goto error;
+
+    /* The route string as obtained from the message includes all
+     * hops but the last one, e.g. the identity of the destination.
+     */
+    if (!(route_str = flux_msg_route_string (msg)))
+        goto error;
+    new_size = strlen (route_str) + strlen (uuid) + 2;
+    if (!(new_str = realloc (route_str, new_size)))
+        goto error;
+    route_str = new_str;
+    strcat (route_str, "!");
+    strcat (route_str, uuid);
+
+    if (!(resp_str = make_json_response_payload (h, json_str, route_str, cred)))
+        goto error;
+    if (flux_respond (h, msg, resp_str) < 0)
+        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
+    free (route_str);
+    free (resp_str);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    free (route_str);
+    free (resp_str);
+}
+
+void method_rusage_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    struct rusage ru;
+    const char *s = NULL;
+    int who;
+    const char *errmsg = NULL;
+    flux_error_t error;
+
+    if (flux_request_unpack (msg, NULL, "{s?s}", "who", &s) < 0
+        && flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (!s || streq (s, "self"))
+        who = RUSAGE_SELF;
+    else if (streq (s, "children"))
+        who = RUSAGE_CHILDREN;
+#ifdef RUSAGE_THREAD
+    else if (streq (s, "thread"))
+        who = RUSAGE_THREAD;
+#endif
+    else {
+        errprintf (&error, "%s is unsupported", s);
+        errmsg = error.text;
+        errno = EINVAL;
+        goto error;
+    }
+    if (getrusage (who, &ru) < 0)
+        goto error;
+    if (flux_respond_pack (h, msg,
+            "{s:f s:f s:i s:i s:i s:i s:i s:i s:i s:i s:i s:i s:i s:i s:i s:i}",
+            "utime", (double)ru.ru_utime.tv_sec + 1E-6 * ru.ru_utime.tv_usec,
+            "stime", (double)ru.ru_stime.tv_sec + 1E-6 * ru.ru_stime.tv_usec,
+            "maxrss", ru.ru_maxrss,
+            "ixrss", ru.ru_ixrss,
+            "idrss", ru.ru_idrss,
+            "isrss", ru.ru_isrss,
+            "minflt", ru.ru_minflt,
+            "majflt", ru.ru_majflt,
+            "nswap", ru.ru_nswap,
+            "inblock", ru.ru_inblock,
+            "oublock", ru.ru_oublock,
+            "msgsnd", ru.ru_msgsnd,
+            "msgrcv", ru.ru_msgrcv,
+            "nsignals", ru.ru_nsignals,
+            "nvcsw", ru.ru_nvcsw,
+            "nivcsw", ru.ru_nivcsw) < 0)
+        flux_log_error (h, "error responding to rusage request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to rusage request");
+}
+
+void method_stats_get_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
+{
+    flux_msgcounters_t mcs;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    flux_get_msgcounters (h, &mcs);
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:{s:i s:i s:i s:i} s:{s:i s:i s:i s:i}}",
+                           "tx",
+                             "request", mcs.request_tx,
+                             "response", mcs.response_tx,
+                             "event", mcs.event_tx,
+                             "control", mcs.control_tx,
+                           "rx",
+                             "request", mcs.request_rx,
+                             "response", mcs.response_rx,
+                             "event", mcs.event_rx,
+                             "control", mcs.control_rx) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+}
+
+void method_stats_clear_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
+{
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    flux_clr_msgcounters (h);
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to stats-clear request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to stats-clear request");
+}
+
+void method_stats_clear_event_cb (flux_t *h,
+                                  flux_msg_handler_t *mh,
+                                  const flux_msg_t *msg,
+                                  void *arg)
+{
+    if (flux_event_decode (msg, NULL, NULL) == 0)
+        flux_clr_msgcounters (h);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libfluxutil/method.h b/src/common/libfluxutil/method.h
new file mode 100644
index 000000000000..b0f3aa8f5d21
--- /dev/null
+++ b/src/common/libfluxutil/method.h
@@ -0,0 +1,47 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_METHOD_H
+#define _FLUX_CORE_METHOD_H
+
+#include <flux/core.h>
+
+/* The ping method requires the server uuid to be stored as a string in
+ * the flux_t aux container under the "flux::uuid" key.
+ */
+void method_ping_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg);
+
+void method_rusage_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg);
+
+void method_stats_get_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg);
+
+void method_stats_clear_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg);
+
+void method_stats_clear_event_cb (flux_t *h,
+                                  flux_msg_handler_t *mh,
+                                  const flux_msg_t *msg,
+                                  void *arg);
+
+
+#endif /* !_FLUX_CORE_METHOD_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libfluxutil/policy.c b/src/common/libfluxutil/policy.c
new file mode 100644
index 000000000000..276e64d5eff7
--- /dev/null
+++ b/src/common/libfluxutil/policy.c
@@ -0,0 +1,408 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* policy.c - parse and validate RFC 33 queue/policy config tables
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/fsd.h"
+
+#include "policy.h"
+
+static int validate_policy_jobspec (json_t *o,
+                                    const char *key,
+                                    const char **default_queue,
+                                    flux_error_t *error)
+{
+    json_error_t jerror;
+    json_t *duration = NULL;
+    json_t *queue = NULL;
+
+    if (json_unpack_ex (o,
+                        &jerror,
+                        0,
+                        "{s?{s?{s?o s?o !} !} !}",
+                        "defaults",
+                          "system",
+                            "duration", &duration,
+                            "queue", &queue) < 0) {
+        errprintf (error,
+                   "error parsing [%s] config table: %s", key, jerror.text);
+        goto inval;
+    }
+    if (duration) {
+        double d;
+
+        if (!json_is_string (duration)
+            || fsd_parse_duration (json_string_value (duration), &d) < 0) {
+            errprintf (error,
+                       "error parsing [%s] config table:"
+                       " 'defaults.system.duration' is not a valid FSD",
+                       key);
+            goto inval;
+        }
+    }
+    if (queue) {
+        if (!json_is_string (queue)) {
+            errprintf (error,
+                       "error parsing [%s] config table:"
+                       " 'defaults.system.queue' is not a string",
+                       key);
+            goto inval;
+        }
+    }
+    if (default_queue)
+        *default_queue = queue ? json_string_value (queue) : NULL;
+    return 0;
+inval:
+    errno = EINVAL;
+    return -1;
+}
+
+static int validate_policy_limits_job_size (json_t *o,
+                                            const char *key,
+                                            const char *key2,
+                                            flux_error_t *error)
+{
+    json_error_t jerror;
+    int nnodes = -1;
+    int ncores = -1;
+    int ngpus = -1;
+
+    if (json_unpack_ex (o,
+                        &jerror,
+                        0,
+                        "{s?i s?i s?i !}",
+                        "nnodes", &nnodes,
+                        "ncores", &ncores,
+                        "ngpus", &ngpus) < 0) {
+        errprintf (error,
+                   "error parsing [%s.%s] config table: %s",
+                   key,
+                   key2,
+                   jerror.text);
+        goto inval;
+    }
+    if (nnodes < -1 || ncores < -1 || ngpus < -1) {
+        errprintf (error,
+                   "error parsing [%s.%s] config table:"
+                   " values must be >= -1",
+                   key,
+                   key2);
+        goto inval;
+    }
+    return 0;
+inval:
+    errno = EINVAL;
+    return -1;
+}
+
+static int validate_policy_limits (json_t *o,
+                                   const char *key,
+                                   flux_error_t *error)
+{
+    json_error_t jerror;
+    json_t *job_size = NULL;
+    json_t *duration = NULL;
+
+    if (json_unpack_ex (o,
+                        &jerror,
+                        0,
+                        "{s?o s?o !}",
+                        "job-size", &job_size,
+                        "duration", &duration) < 0) {
+        errprintf (error,
+                   "error parsing [%s] config table: %s",
+                   key,
+                   jerror.text);
+        goto inval;
+    }
+    if (duration) {
+        double d;
+
+        if (!json_is_string (duration)
+            || fsd_parse_duration (json_string_value (duration), &d) < 0) {
+            errprintf (error,
+                       "error parsing [%s] config table:"
+                       " 'duration' is not a valid FSD",
+                       key);
+            goto inval;
+        }
+    }
+    if (job_size) {
+        json_t *min = NULL;
+        json_t *max = NULL;
+
+        if (json_unpack_ex (job_size,
+                            &jerror,
+                            0,
+                            "{s?o s?o !}",
+                            "min", &min,
+                            "max", &max) < 0) {
+            errprintf (error,
+                       "error parsing [%s.job-size] config table: %s",
+                       key,
+                       jerror.text);
+            goto inval;
+        }
+        if (min) {
+            if (validate_policy_limits_job_size (min, key, "min", error) < 0)
+                goto inval;
+        }
+        if (max) {
+            if (validate_policy_limits_job_size (max, key, "max", error) < 0)
+                goto inval;
+        }
+    }
+    return  0;
+inval:
+    errno = EINVAL;
+    return -1;
+}
+
+static bool is_string_array (json_t *o, const char *banned)
+{
+    size_t index;
+    json_t *val;
+
+    if (!json_is_array (o))
+        return false;
+    json_array_foreach (o, index, val) {
+        if (!json_is_string (val))
+            return false;
+        if (banned) {
+            for (int i = 0; banned[i] != '\0'; i++) {
+                if (strchr (json_string_value (val), banned[i]))
+                    return false;
+            }
+        }
+    }
+    return true;
+}
+
+static int validate_policy_access (json_t *o,
+                                   const char *key,
+                                   flux_error_t *error)
+{
+    json_error_t jerror;
+    json_t *allow_user = NULL;
+    json_t *allow_group = NULL;
+
+    if (json_unpack_ex (o,
+                        &jerror,
+                        0,
+                        "{s?o s?o !}",
+                        "allow-user", &allow_user,
+                        "allow-group", &allow_group) < 0) {
+        errprintf (error,
+                   "error parsing [%s] config table: %s",
+                   key,
+                   jerror.text);
+        goto inval;
+    }
+    if (allow_user) {
+        if (!is_string_array (allow_user, NULL)) {
+            errprintf (error,
+                       "error parsing [%s] config table:"
+                       " 'allow-user' must be a string array",
+                       key);
+            goto inval;
+        }
+    }
+    if (allow_group) {
+        if (!is_string_array (allow_group, NULL)) {
+            errprintf (error,
+                       "error parsing [%s] config table:"
+                       " 'allow-group' must be a string array",
+                       key);
+            goto inval;
+        }
+    }
+    return  0;
+inval:
+    errno = EINVAL;
+    return -1;
+}
+
+/* Validate the policy table as defined by RFC 33.  The table can appear at
+ * the top level of the config or within a queues entry.
+ */
+static int validate_policy_json (json_t *policy,
+                                 const char *key,
+                                 const char **default_queue,
+                                 flux_error_t *error)
+{
+    json_error_t jerror;
+    json_t *jobspec = NULL;
+    json_t *limits = NULL;
+    json_t *access = NULL;
+    json_t *scheduler = NULL;
+    const char *defqueue = NULL;
+    char key2[1024];
+
+    if (json_unpack_ex (policy,
+                        &jerror,
+                        0,
+                        "{s?o s?o s?o s?o !}",
+                        "jobspec", &jobspec,
+                        "limits", &limits,
+                        "access", &access,
+                        "scheduler", &scheduler) < 0) {
+        errprintf (error,
+                   "error parsing [%s] config table: %s",
+                   key,
+                   jerror.text);
+        errno = EINVAL;
+        return -1;
+    }
+    if (jobspec) {
+        snprintf (key2, sizeof (key2), "%s.jobspec", key);
+        if (validate_policy_jobspec (jobspec, key2, &defqueue, error) < 0)
+            return -1;
+    }
+    if (limits) {
+        snprintf (key2, sizeof (key2), "%s.limits", key);
+        if (validate_policy_limits (limits, key2, error) < 0)
+            return -1;
+    }
+    if (access) {
+        snprintf (key2, sizeof (key2), "%s.access", key);
+        if (validate_policy_access (access, key2, error) < 0)
+            return -1;
+    }
+    if (default_queue)
+        *default_queue = defqueue;
+    return 0;
+}
+
+static int validate_policy_config (const flux_conf_t *conf,
+                                   const char **default_queue,
+                                   flux_error_t *error)
+{
+    json_t *policy = NULL;
+    const char *defqueue = NULL;
+    flux_error_t e;
+
+    if (flux_conf_unpack (conf,
+                          &e,
+                          "{s?o}",
+                          "policy", &policy) < 0) {
+        errprintf (error, "error parsing [policy] config table: %s", e.text);
+        return -1;
+    }
+    if (policy) {
+        if (validate_policy_json (policy, "policy", &defqueue, error) < 0)
+            return -1;
+    }
+    if (default_queue)
+        *default_queue = defqueue;
+    return 0;
+}
+
+static int validate_queues_config (const flux_conf_t *conf,
+                                   const char *default_queue,
+                                   flux_error_t *error)
+{
+    json_t *queues = NULL;
+    flux_error_t e;
+
+    if (flux_conf_unpack (conf,
+                          &e,
+                          "{s?o}",
+                          "queues", &queues) < 0) {
+        errprintf (error, "error parsing [queues] config table: %s", e.text);
+        return -1;
+    }
+    if (queues) {
+        const char *name;
+        json_t *entry;
+
+        if (!json_is_object (queues)) {
+            errprintf (error,
+                       "error parsing [queues] config table:"
+                       " not a table");
+            goto inval;
+        }
+        json_object_foreach (queues, name, entry) {
+            json_error_t jerror;
+            json_t *policy = NULL;
+            json_t *requires = NULL;
+
+            if (json_unpack_ex (entry,
+                                &jerror,
+                                0,
+                                "{s?o s?o !}",
+                                "policy", &policy,
+                                "requires", &requires) < 0) {
+                errprintf (error,
+                           "error parsing [queues.%s] config table: %s",
+                           name,
+                           jerror.text);
+                goto inval;
+            }
+            if (policy) {
+                char key[1024];
+                const char *defqueue;
+                snprintf (key, sizeof (key), "queues.%s.policy", name);
+                if (validate_policy_json (policy, key, &defqueue, error) < 0)
+                    return -1;
+                if (defqueue) {
+                    errprintf (error,
+                               "error parsing [queues.%s] config table:"
+                               " 'policy' must not define a default queue!",
+                               name);
+                    goto inval;
+                }
+            }
+            if (requires) {
+                const char *banned_property_chars = " \t!&'\"`'|()";
+                if (!is_string_array (requires, banned_property_chars)) {
+                    errprintf (error,
+                               "error parsing [queues.%s] config table:"
+                               " 'requires' must be an array of property"
+                               " strings (RFC 20)",
+                               name);
+                    goto inval;
+                }
+            }
+        }
+    }
+    if (default_queue) {
+        if (!queues || !json_object_get (queues, default_queue)) {
+            errprintf (error,
+                       "the [policy] config table defines a default queue %s"
+                       " that is not in [queues] table",
+                       default_queue);
+            goto inval;
+        }
+    }
+    return 0;
+inval:
+    errno = EINVAL;
+    return -1;
+}
+
+int policy_validate (const flux_conf_t *conf, flux_error_t *error)
+{
+    const char *defqueue;
+
+    if (validate_policy_config (conf, &defqueue, error) < 0
+        || validate_queues_config (conf, defqueue, error) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libfluxutil/policy.h b/src/common/libfluxutil/policy.h
new file mode 100644
index 000000000000..a60e72fcccfa
--- /dev/null
+++ b/src/common/libfluxutil/policy.h
@@ -0,0 +1,23 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBFLUXUTIL_POLICY_H
+#define _LIBFLUXUTIL_POLICY_H
+
+#include <flux/core.h>
+
+/* Validate [policy] and [queue] configuration tables defined in RFC 33.
+ * Return 0 on success, -1 on failure with errno set and 'error' filled in.
+ */
+int policy_validate (const flux_conf_t *conf, flux_error_t *error);
+
+// vi:ts=4 sw=4 expandtab
+
+#endif // !_LIBFLUXUTIL_POLICY_H
diff --git a/src/common/libhostlist/Makefile.am b/src/common/libhostlist/Makefile.am
new file mode 100644
index 000000000000..963daa1167df
--- /dev/null
+++ b/src/common/libhostlist/Makefile.am
@@ -0,0 +1,71 @@
+AM_CFLAGS = \
+    -I$(top_srcdir) \
+    $(WARNING_CFLAGS) \
+    $(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+    $(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+    $(CODE_COVERAGE_CPPFLAGS)
+
+noinst_LTLIBRARIES = \
+    libhostlist.la
+
+fluxinclude_HEADERS = \
+    hostlist.h
+
+libhostlist_la_SOURCES = \
+    util.h \
+    util.c \
+    hostname.h \
+    hostname.c \
+    hostrange.h \
+    hostrange.c \
+    hostlist.h \
+    hostlist.c
+
+TESTS = \
+    test_util.t \
+    test_hostname.t \
+    test_hostrange.t \
+    test_hostlist.t
+
+check_PROGRAMS = \
+    $(TESTS)
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+       $(top_srcdir)/config/tap-driver.sh
+
+test_hostname_t_SOURCES = \
+    test/hostname.c
+test_hostname_t_CPPFLAGS = \
+    $(AM_CPPFLAGS)
+test_hostname_t_LDADD = \
+    $(top_builddir)/src/common/libtap/libtap.la \
+    $(top_builddir)/src/common/libhostlist/libhostlist.la
+
+test_util_t_SOURCES = \
+    test/util.c
+test_util_t_CPPFLAGS = \
+    $(AM_CPPFLAGS)
+test_util_t_LDADD = \
+    $(top_builddir)/src/common/libtap/libtap.la \
+    $(top_builddir)/src/common/libhostlist/libhostlist.la
+
+test_hostrange_t_SOURCES = \
+    test/hostrange.c
+test_hostrange_t_CPPFLAGS = \
+    $(AM_CPPFLAGS)
+test_hostrange_t_LDADD = \
+    $(top_builddir)/src/common/libtap/libtap.la \
+    $(top_builddir)/src/common/libhostlist/libhostlist.la
+
+test_hostlist_t_SOURCES = \
+    test/hostlist.c
+test_hostlist_t_CPPFLAGS = \
+    $(AM_CPPFLAGS)
+test_hostlist_t_LDADD = \
+    $(top_builddir)/src/common/libtap/libtap.la \
+    $(top_builddir)/src/common/libhostlist/libhostlist.la
diff --git a/src/common/libhostlist/hostlist.c b/src/common/libhostlist/hostlist.c
new file mode 100644
index 000000000000..d2acba4f3517
--- /dev/null
+++ b/src/common/libhostlist/hostlist.c
@@ -0,0 +1,1188 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif /* !HAVE_CONFIG_H */
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <assert.h>
+#include <errno.h>
+#include <ctype.h>
+#include <sys/param.h>
+#include <unistd.h>
+
+#include "src/common/libutil/errno_safe.h"
+
+#include "hostlist.h"
+#include "hostrange.h"
+#include "hostname.h"
+#include "util.h"
+
+/* number of elements to allocate when extending the hostlist array */
+#define HOSTLIST_CHUNK    16
+
+/* max host range: anything larger will be assumed to be an error */
+#define MAX_RANGE    1<<20    /* 1M Hosts */
+
+/* max host suffix value */
+#define MAX_HOST_SUFFIX 1<<25
+
+/* max number of ranges that will be processed between brackets */
+#define MAX_RANGES    10240    /* 10K Ranges */
+
+/* max size of internal hostrange buffer */
+#define MAXHOSTRANGELEN 1024
+
+/* Helper structure for hostlist iteration
+ */
+struct current {
+    char *host;
+    int index;
+    int depth;
+};
+
+/* Hostlist - a dynamic array of hostrange objects
+ */
+struct hostlist {
+    int size;               /* current number of elements available in array */
+    int nranges;            /* current number of ranges stored in array */
+    int nhosts;             /* current number of hosts stored in hostlist */
+    struct hostrange **hr;  /* pointer to hostrange array */
+
+    struct current current; /* iterator cursor */
+};
+
+/* _range struct helper for parsing hostlist strings
+ */
+struct _range {
+    unsigned long lo, hi;
+    int width;
+};
+
+
+/*
+ * Helper function for host list string parsing routines
+ * Returns a pointer to the next token; additionally advance *str
+ * to the next separator.
+ *
+ * next_tok was taken directly from pdsh courtesy of Jim Garlick.
+ * (with modifications to support bracketed hostlists, i.e.:
+ *  xxx[xx,xx,xx] is a single token)
+ *
+ * next_tok now handles multiple brackets within the same token,
+ * e.g.  node[01-30]-[1-2,6].
+ */
+static char * next_tok (char *sep, char **str)
+{
+    char *tok;
+    int level = 0;
+
+    /* push str past any leading separators */
+    while (**str != '\0' && strchr (sep, **str) != NULL)
+        (*str)++;
+
+    if (**str == '\0')
+        return NULL;
+
+    /* assign token ptr */
+    tok = *str;
+
+    while ( **str != '\0' &&
+	        (level != 0 || strchr (sep, **str) == NULL) ) {
+        if (**str == '[')
+            level++;
+        else if (**str == ']')
+            level--;
+        (*str)++;
+    }
+
+    /* nullify consecutive separators and push str beyond them */
+    while (**str != '\0' && strchr (sep, **str) != NULL)
+        *(*str)++ = '\0';
+
+    return tok;
+}
+
+
+struct hostlist * hostlist_create (void)
+{
+    struct hostlist * new = calloc (1, sizeof (*new));
+    if (!new)
+        return NULL;
+
+    new->hr = calloc (HOSTLIST_CHUNK, sizeof (struct hostrange *));
+    if (!new->hr)
+        goto fail;
+
+    new->size = HOSTLIST_CHUNK;
+    return new;
+
+fail:
+    free (new);
+    return NULL;
+}
+
+
+/* Resize the internal array used to store the list of hostrange objects.
+ *
+ * returns 1 for a successful resize,
+ *         0 if call to _realloc fails
+ *
+ * It is assumed that the caller has the hostlist hl locked
+ */
+static int hostlist_resize (struct hostlist *hl, size_t newsize)
+{
+    int i;
+    size_t oldsize;
+    assert (hl != NULL);
+    oldsize = hl->size;
+    hl->size = newsize;
+    hl->hr = realloc ((void *) hl->hr, hl->size*sizeof (struct hostrange *));
+    if (!(hl->hr))
+        return 0;
+
+    for (i = oldsize; i < newsize; i++)
+        hl->hr[i] = NULL;
+
+    return 1;
+}
+
+/* Resize hostlist by one HOSTLIST_CHUNK
+ * Assumes that hostlist hl is locked by caller
+ */
+static int hostlist_expand (struct hostlist *hl)
+{
+    if (!hostlist_resize (hl, hl->size + HOSTLIST_CHUNK))
+        return 0;
+    else
+        return 1;
+}
+
+/* Push a hostrange object onto hostlist hl
+ * Returns the number of hosts successfully pushed onto hl
+ * or -1 if there was an error allocating memory
+ */
+static int hostlist_append_range (struct hostlist *hl, struct hostrange * hr)
+{
+    struct hostrange * tail;
+    int retval;
+
+    assert (hr != NULL);
+
+    tail = (hl->nranges > 0) ? hl->hr[hl->nranges-1] : hl->hr[0];
+
+    if (hl->size == hl->nranges && !hostlist_expand (hl))
+        goto error;
+
+    if (hl->nranges > 0
+        && hostrange_prefix_cmp (tail, hr) == 0
+        && tail->hi == hr->lo - 1
+        && hostrange_width_combine (tail, hr)) {
+        tail->hi = hr->hi;
+    } else {
+        if (!(hl->hr[hl->nranges++] = hostrange_copy (hr)))
+            goto error;
+    }
+
+    retval = hostrange_count (hr);
+    hl->nhosts += retval;
+
+    return retval;
+  error:
+    return -1;
+}
+
+
+/* Same as hostlist_append_range() above, but prefix, lo, hi, and width
+ * are passed as args
+ */
+static int hostlist_append_hr (struct hostlist *hl,
+                               char *prefix,
+                               unsigned long lo,
+                               unsigned long hi,
+                               int width)
+{
+    int retval;
+    struct hostrange * hr = hostrange_create (prefix, lo, hi, width);
+    if (!hr)
+        return -1;
+    retval = hostlist_append_range (hl, hr);
+    hostrange_destroy (hr);
+    return retval;
+}
+
+/* Insert a range object hr into position n of the hostlist hl
+ */
+static int hostlist_insert_range (struct hostlist *hl,
+                                  struct hostrange * hr,
+                                  int n)
+{
+    int i;
+    struct hostrange * tmp;
+
+    assert (hl != NULL);
+    assert (hr != NULL);
+
+    if (n > hl->nranges)
+        return 0;
+
+    if (hl->size == hl->nranges && !hostlist_expand (hl))
+        return 0;
+
+    /* copy new hostrange into slot "n" in array */
+    tmp = hl->hr[n];
+    hl->hr[n] = hostrange_copy (hr);
+
+    /* push remaining hostrange entries up */
+    for (i = n + 1; i < hl->nranges + 1; i++) {
+        struct hostrange * last = hl->hr[i];
+        hl->hr[i] = tmp;
+        tmp = last;
+    }
+    hl->nranges++;
+
+    /* adjust current if necessary
+     */
+    if (hl->current.index >= n)
+        hl->current.index++;
+
+    return 1;
+}
+
+
+static void hostlist_shift_current (struct hostlist *hl,
+                                    int index,
+                                    int depth,
+                                    int n)
+{
+    struct current *c = &hl->current;
+
+    /*  Case 1: a single host was deleted in hl->hr[index] at depth.
+     *   To leave c->depth pointing to the next host, shift c->depth
+     *   back only if the deletion depth < c->depth.
+     */
+    if (n == 0 && c->index == index && depth < c->depth) {
+        c->depth--;
+        return;
+    }
+    /*  Case 2: current points to the hostrange which was deleted.
+     *  Simply reset depth to -1 so that hostlist_next() starts at the
+     *   next hostrange in the array.
+     */
+    if (index == c->index) {
+        c->depth = -1;
+        return;
+    }
+    /*  Case 3: hostrange elements were deleted at a position < c->index:
+     *  In this case, adjust current index back by the number of deleted
+     *   elements. (ensure index is at least -1 however.)
+     */
+    if (index < c->index) {
+        /*  Move index up by the number of deleted ranges */
+        if ((c->index -= n) < 0)
+            c->index = -1;
+    }
+    /* O/w current iterator should not be affected.
+     */
+}
+
+
+
+/* Delete the range at position n in the range array
+ * Assumes the hostlist lock is already held.
+ */
+static void hostlist_delete_range (struct hostlist *hl, int n)
+{
+    int i;
+    struct hostrange * old;
+
+    assert (hl != NULL);
+    assert (n < hl->nranges && n >= 0);
+
+    old = hl->hr[n];
+    for (i = n; i < hl->nranges - 1; i++)
+        hl->hr[i] = hl->hr[i + 1];
+    hl->nranges--;
+    hl->hr[hl->nranges] = NULL;
+
+    hostlist_shift_current (hl, n, 0, 1);
+
+    /* XXX caller responsible for adjusting nhosts */
+    /*hl->nhosts -= hostrange_count(old) */
+
+    hostrange_destroy (old);
+}
+
+
+/* Grab a single range from str
+ * returns 0 if str contained a valid number or range,
+ *        -1 if conversion of str to a range failed.
+ */
+static int parse_next_range (const char *str, struct _range *range)
+{
+    int saved_errno;
+    char *p, *q;
+    char *orig = strdup (str);
+    if (!orig)
+        return -1;
+
+    if ((p = strchr (str, '-'))) {
+        *p++ = '\0';
+        if (*p == '-') {  /* don't allow negative numbers */
+            errno = EINVAL;
+            goto error;
+        }
+    }
+    range->lo = strtoul (str, &q, 10);
+    if (q == str)  {
+        errno = EINVAL;
+        goto error;
+    }
+
+    range->hi = (p && *p) ? strtoul (p, &q, 10) : range->lo;
+
+    errno = EINVAL;
+    if (q == p || *q != '\0')
+        goto error;
+
+    if (range->lo > range->hi)
+        goto error;
+
+    if (range->hi - range->lo + 1 > MAX_RANGE ) {
+        errno = ERANGE;
+        goto error;
+    }
+
+    free (orig);
+    range->width = strlen (str);
+    return 0;
+
+  error:
+    saved_errno = errno;
+    free (orig);
+    errno = saved_errno;
+    return -1;
+}
+
+static int hostlist_append_host (struct hostlist *hl, const char *str)
+{
+    int rc = -1;
+    struct hostrange * hr;
+    struct hostlist_hostname * hn;
+
+    assert (hl != NULL);
+
+    if (str == NULL || *str == '\0')
+        return 0;
+
+    if (!(hn = hostname_create (str)))
+        return -1;
+
+    if (hostname_suffix_is_valid (hn)) {
+        hr = hostrange_create (hn->prefix,
+                               hn->num,
+                               hn->num,
+                               hostname_suffix_width (hn));
+    } else
+        hr = hostrange_create_single (str);
+
+    if (hr && hostlist_append_range (hl, hr) >= 0)
+        rc = 0;
+
+    hostrange_destroy (hr);
+    hostname_destroy (hn);
+
+    return rc;
+}
+
+/*
+ * Convert 'str' containing comma separated digits and ranges into an array
+ *  of struct _range types (max 'len' elements).
+ *
+ * Return number of ranges created, or -1 on error.
+ */
+static int parse_range_list (char *str, struct _range *ranges, int len)
+{
+    char *p;
+    int count = 0;
+
+    while (str) {
+        if (count == len)
+            return -1;
+        if ((p = strchr (str, ',')))
+            *p++ = '\0';
+        if (parse_next_range (str, &ranges[count++]) < 0)
+            return -1;
+        str = p;
+    }
+    return count;
+}
+
+static int append_range_list (struct hostlist *hl,
+                              char *pfx,
+                              struct _range *rng,
+                              int n)
+{
+    int i;
+    for (i = 0; i < n; i++) {
+        if (hostlist_append_hr (hl, pfx, rng->lo, rng->hi, rng->width) < 0)
+            return -1;
+        rng++;
+    }
+    return 0;
+}
+
+static int append_range_list_with_suffix (struct hostlist *hl,
+                                          char *pfx,
+                                          char *sfx,
+                                          struct _range *rng,
+                                          int n)
+{
+    int i;
+    unsigned long j;
+
+    /* compute max buffer size for this set of hosts */
+    int size = strlen (pfx) + strlen (sfx) + 20 + rng->width;
+
+
+    for (i = 0; i < n; i++) {
+        for (j = rng->lo; j <= rng->hi; j++) {
+            char host[size];
+            struct hostrange * hr;
+            snprintf (host,
+                      sizeof (host),
+                      "%s%0*lu%s",
+                      pfx,
+                      rng->width,
+                      j,
+                      sfx);
+            if (!(hr = hostrange_create_single (host)))
+                return -1;
+            hostlist_append_range (hl, hr);
+            /*
+             * hr is copied in hostlist_append_range. Need to free here.
+             */
+            hostrange_destroy (hr);
+        }
+        rng++;
+    }
+    return 0;
+}
+
+/*
+ * Create a hostlist from a string with brackets '[' ']'
+ */
+static struct hostlist * hostlist_create_bracketed (const char *hostlist,
+                                                    char *sep,
+                                                    char *r_op)
+{
+    struct hostlist * new = hostlist_create ();
+    struct {
+        struct _range ranges[MAX_RANGES];
+        char cur_tok[1024];
+    } *ctx;
+    int nr;
+    int rc;
+    char *p, *tok, *str, *orig;
+
+    if (!new)
+        return NULL;
+
+    if (hostlist == NULL)
+        return new;
+
+    if (!(orig = str = strdup (hostlist))
+        || !(ctx = calloc (1, sizeof (*ctx)))) {
+        hostlist_destroy (new);
+        ERRNO_SAFE_WRAP (free, orig);
+        return NULL;
+    }
+
+    while ((tok = next_tok (sep, &str)) != NULL) {
+        strncpy (ctx->cur_tok, tok, 1024 - 1);
+
+        if ((p = strchr (tok, '[')) != NULL) {
+            char *q, *prefix = tok;
+            *p++ = '\0';
+
+            if ((q = strchr (p, ']'))) {
+                *q = '\0';
+                nr = parse_range_list (p, ctx->ranges, MAX_RANGES);
+                if (nr < 0)
+                    goto error;
+
+                if (*(++q) != '\0')
+                    rc = append_range_list_with_suffix (new,
+                                                        prefix,
+                                                        q,
+                                                        ctx->ranges,
+                                                        nr);
+                else
+                    rc = append_range_list (new,
+                                            prefix,
+                                            ctx->ranges,
+                                            nr);
+                if (rc < 0)
+                    goto error;
+
+            } else                    /* Error: brackets must be balanced */
+                goto error_unmatched;
+
+        } else if (strchr (tok, ']')) /* Error: brackets must be balanced */
+            goto error_unmatched;
+        else                          /* Ok: No brackets found, single host */
+            hostlist_append_host (new, ctx->cur_tok);
+    }
+
+    free (orig);
+    free (ctx);
+    return new;
+
+  error_unmatched:
+    errno = EINVAL;
+  error:
+    hostlist_destroy (new);
+    ERRNO_SAFE_WRAP (free, orig);
+    ERRNO_SAFE_WRAP (free, ctx);
+    return NULL;
+}
+
+struct hostlist * hostlist_decode (const char *str)
+{
+    if (!str) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return hostlist_create_bracketed (str, "\t, ", "-");
+}
+
+struct hostlist * hostlist_copy (const struct hostlist *hl)
+{
+    int i;
+    struct hostlist * new;
+
+    if (hl == NULL)
+        return NULL;
+
+    if (!(new = hostlist_create ()))
+        goto done;
+
+    new->nranges = hl->nranges;
+    new->nhosts = hl->nhosts;
+    if (new->nranges > new->size)
+        hostlist_resize (new, new->nranges);
+
+    for (i = 0; i < hl->nranges; i++)
+        new->hr[i] = hostrange_copy (hl->hr[i]);
+
+  done:
+    return new;
+}
+
+
+void hostlist_destroy (struct hostlist *hl)
+{
+    if (hl) {
+        int saved_errno = errno;
+        for (int i = 0; i < hl->nranges; i++)
+            hostrange_destroy (hl->hr[i]);
+        free (hl->hr);
+        free (hl->current.host);
+        free (hl);
+        errno = saved_errno;
+    }
+}
+
+int hostlist_append (struct hostlist *hl, const char *hosts)
+{
+    struct hostlist * new;
+    int retval;
+    if (hl == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (hosts == NULL)
+        return 0;
+    new = hostlist_decode (hosts);
+    if (!new)
+        return -1;
+    retval = new->nhosts;
+    hostlist_append_list (hl, new);
+    hostlist_destroy (new);
+    return retval;
+}
+
+int hostlist_append_list (struct hostlist * h1, struct hostlist * h2)
+{
+    int i, n = 0;
+
+    if (h1 == NULL || h2 == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (i = 0; i < h2->nranges; i++)
+        n += hostlist_append_range (h1, h2->hr[i]);
+
+    return n;
+}
+
+static inline int nth_args_valid (struct hostlist *hl, int n)
+{
+    if (hl == NULL || n < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (n >= hl->nhosts) {
+        errno = ENOENT;
+        return -1;
+    }
+    return 0;
+}
+
+static void set_current (struct current *cur, int index, int depth)
+{
+    if (cur) {
+        free (cur->host);
+        cur->host = NULL;
+        cur->index = index;
+        cur->depth = depth;
+    }
+}
+
+const char * hostlist_nth (struct hostlist *hl, int n)
+{
+    int   i, count;
+
+    if (nth_args_valid (hl, n) < 0)
+        return NULL;
+
+    count = 0;
+    for (i = 0; i < hl->nranges; i++) {
+        int num_in_range = hostrange_count (hl->hr[i]);
+
+        if (n <= (num_in_range - 1 + count)) {
+            int d = n - count;
+            set_current (&hl->current, i, d);
+            return hostlist_current (hl);
+        } else
+            count += num_in_range;
+    }
+    return NULL;
+}
+
+int hostlist_count (struct hostlist *hl)
+{
+    return hl ? hl->nhosts : 0;
+}
+
+static int hostlist_find_host (struct hostlist *hl,
+                               struct stack_hostname *hn,
+                               struct current *cur)
+{
+    int i, count, ret = -1;
+    for (i = 0, count = 0; i < hl->nranges; i++) {
+        int offset = hostrange_hn_within (hl->hr[i], hn);
+        if (offset >= 0) {
+            ret = count + offset;
+            set_current (cur, i, offset);
+            break;
+        }
+        else
+            count += hostrange_count (hl->hr[i]);
+    }
+
+    if (ret < 0)
+        errno = ENOENT;
+    return ret;
+
+}
+
+int hostlist_find (struct hostlist *hl, const char *hostname)
+{
+    if (!hl || !hostname) {
+        errno = EINVAL;
+        return -1;
+    }
+    struct stack_hostname hn_storage;
+
+    struct stack_hostname *hn = hostname_stack_create (&hn_storage, hostname);
+    if (!hn)
+        return -1;
+    return hostlist_find_host (hl, hn, &hl->current);
+}
+
+int hostlist_find_hostname (struct hostlist *hl, struct hostlist_hostname *hn)
+{
+    struct stack_hostname hn_storage;
+    struct stack_hostname *shn;
+
+    if (!hl || !hn) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    shn = hostname_stack_create_from_hostname (&hn_storage, hn);
+    if (!hn)
+        return -1;
+    return hostlist_find_host (hl, shn, &hl->current);
+}
+
+struct hostlist_hostname *hostlist_hostname_create (const char *hn)
+{
+    return hostname_create (hn);
+}
+
+void hostlist_hostname_destroy (struct hostlist_hostname *hn)
+{
+    hostname_destroy (hn);
+}
+
+/*  Remove host at cursor 'cur'. If the current real cursor hl->current
+ *   is affected, adjust it accordingly.
+ */
+static int hostlist_remove_at (struct hostlist *hl, struct current *cur)
+{
+    struct hostrange *hr;
+    struct hostrange *new;
+
+    if (cur->index > hl->nhosts - 1)
+        return 0;
+
+    hr = hl->hr[cur->index];
+
+    /*  If we're removing the current host, invalidate cursor hostname
+     */
+    if (hl->current.index == cur->index
+        && hl->current.depth == cur->depth) {
+        free (hl->current.host);
+        hl->current.host = NULL;
+    }
+
+    new = hostrange_delete_host (hr, hr->lo + cur->depth);
+    if (new) {
+        hostlist_insert_range (hl, new, cur->index + 1);
+        hostrange_destroy (new);
+
+        /*  If the split hostrange affects the cursor, adjust it now
+         */
+        if (hl->current.index == cur->index
+            && hl->current.depth >= cur->depth) {
+            /*
+             *  Current cursor was at or ahead of split. Advance to new
+             *   hostrange at the correct "depth".
+             */
+            hl->current.index++;
+            hl->current.depth = hl->current.depth - cur->depth - 1;
+        }
+    }
+    else if (hostrange_empty (hr)) {
+        /*  Hostrange is now empty, remove it. Cursor will be adjusted
+         *   accordingly in hostlist_delete_range ()
+         */
+        hostlist_delete_range (hl, cur->index);
+    }
+    else if (hl->current.index == cur->index
+            && hl->current.depth >= cur->depth) {
+        /*
+         *  Current range affected, but range was not split. Adjust
+         *   current cursor appropriately
+         */
+        hl->current.depth = hl->current.depth - cur->depth - 1;
+    }
+
+    hl->nhosts--;
+    return 1;
+}
+
+/* implementation needs improvement
+ */
+static int hostlist_delete_host (struct hostlist *hl, const char *hostname)
+{
+    struct current cur = { 0 };
+    struct stack_hostname hn_storage;
+
+    struct stack_hostname *hn = hostname_stack_create (&hn_storage, hostname);
+    if (!hn)
+        return -1;
+    int n = hostlist_find_host (hl, hn, &cur);
+    if (n < 0)
+        return errno == ENOENT ? 0 : -1;
+    return hostlist_remove_at (hl, &cur);
+}
+
+/* XXX: Note: efficiency improvements needed */
+int hostlist_delete (struct hostlist *hl, const char *hosts)
+{
+    int n = 0;
+    const char *host = NULL;
+    struct hostlist *hltmp;
+
+    if (!hl || !hosts) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!(hltmp = hostlist_decode (hosts)))
+        return -1;
+
+    host = hostlist_first (hltmp);
+    while (host) {
+        n += hostlist_delete_host (hl, host);
+        host = hostlist_next (hltmp);
+    }
+    hostlist_destroy (hltmp);
+
+    return n;
+}
+
+/* search through hostlist for ranges that can be collapsed
+ * does =not= delete any hosts
+ */
+static void hostlist_collapse (struct hostlist *hl)
+{
+    int i;
+
+    for (i = hl->nranges - 1; i > 0; i--) {
+        struct hostrange * hprev = hl->hr[i - 1];
+        struct hostrange * hnext = hl->hr[i];
+
+        if (hostrange_prefix_cmp (hprev, hnext) == 0 &&
+            hprev->hi == hnext->lo - 1 &&
+            hostrange_width_combine (hprev, hnext)) {
+            hprev->hi = hnext->hi;
+            hostlist_delete_range (hl, i);
+        }
+    }
+}
+
+/* search through hostlist (hl) for intersecting ranges
+ * split up duplicates and coalesce ranges where possible
+ */
+static void hostlist_coalesce (struct hostlist *hl)
+{
+    int i, j;
+    struct hostrange * new;
+
+    /*  Scan backwards through hostranges, and determine if the
+     *   current and previous ranges intersect. If so, then
+     *   collect and split contiguous ranges into new ranges.
+     */
+    for (i = hl->nranges - 1; i > 0; i--) {
+        struct hostrange * hprev = hl->hr[i - 1];
+        struct hostrange * hnext = hl->hr[i];
+
+        /* If ranges intersect, then the common (duplicated) hosts
+         *  are returned in 'new'.
+         */
+        new = hostrange_intersect (hprev, hnext);
+        if (new) {
+            j = i;
+
+            /*  Upper bound of duplicated range is below hprev -
+             *   hnext will now hold duplicates, set next->hi to prev->hi
+             */
+            if (new->hi < hprev->hi)
+                hnext->hi = hprev->hi;
+
+            /*
+             *  The duplicated range will inserted piecemeal below,
+             *   e.g. [5-7,6-8] -> [5-6,6-7,7-8]
+             *  Therefore adjust the end of hprev to new->lo (the
+             *   first duplicated host), and the start of hnext to
+             *   new->hi (the last duplicated host). The rest of the
+             *   duplicates will be inserted below.
+             */
+            hprev->hi = new->lo;
+            hnext->lo = new->hi;
+
+            /*  N.B.: It should not be possible that hprev is now
+             *   empty, so the below should be deadcode
+             *
+             *if (hostrange_empty (hprev))
+             *   hostlist_delete_range (hl, i);
+             */
+
+            /*  Now insert each duplicated number between hprev and
+             *   hnext:
+             */
+            while (new->lo <= new->hi) {
+                struct hostrange * hr = hostrange_create (new->prefix,
+                                                          new->lo,
+                                                          new->lo,
+                                                          new->width);
+
+                if (new->lo > hprev->hi)
+                    hostlist_insert_range (hl, hr, j++);
+
+                if (new->lo < hnext->lo)
+                    hostlist_insert_range (hl, hr, j++);
+
+                hostrange_destroy (hr);
+
+                new->lo++;
+            }
+            i = hl->nranges;
+            hostrange_destroy (new);
+        }
+    }
+
+    hostlist_collapse (hl);
+}
+
+
+/* hostrange compare with void * arguments to allow use with
+ * libc qsort()
+ */
+int _cmp (const void *h1, const void *h2)
+{
+    return hostrange_cmp ( *(struct hostrange **) h1,
+                           *(struct hostrange **) h2);
+}
+
+
+void hostlist_sort (struct hostlist *hl)
+{
+    if (hl == NULL)
+        return;
+    if (hl->nranges <= 1)
+        return;
+    qsort (hl->hr, hl->nranges, sizeof (struct hostrange *), _cmp);
+    hostlist_coalesce (hl);
+}
+
+/* attempt to join ranges at loc and loc-1 in a hostlist  */
+/* delete duplicates, return the number of hosts deleted  */
+/* assumes that the hostlist hl has been locked by caller */
+/* returns -1 if no range join occurred */
+static int attempt_range_join (struct hostlist *hl, int loc)
+{
+    int ndup;
+    assert (hl != NULL);
+    assert (loc > 0);
+    assert (loc < hl->nranges);
+    ndup = hostrange_join (hl->hr[loc - 1], hl->hr[loc]);
+    if (ndup >= 0) {
+        hostlist_delete_range (hl, loc);
+        hl->nhosts -= ndup;
+    }
+    return ndup;
+}
+
+void hostlist_uniq (struct hostlist *hl)
+{
+    int i = 1;
+    if (hl == NULL)
+        return;
+
+    if (hl->nranges <= 1)
+        return;
+
+    qsort (hl->hr, hl->nranges, sizeof (struct hostrange *), &_cmp);
+
+    while (i < hl->nranges) {
+        if (attempt_range_join (hl, i) < 0) /* No range join occurred */
+            i++;
+    }
+}
+
+/* return true if a bracket is needed for the range at i in hostlist hl */
+static int is_bracket_needed (struct hostlist *hl, int i)
+{
+    struct hostrange * h1 = hl->hr[i];
+    struct hostrange * h2 = i < hl->nranges - 1 ? hl->hr[i + 1] : NULL;
+    return hostrange_count (h1) > 1 || hostrange_within_range (h1, h2);
+}
+
+/* write the next bracketed hostlist, i.e. prefix[n-m,k,...]
+ * into buf, writing at most n chars including the terminating '\0'
+ *
+ * leaves start pointing to one past last range object in bracketed list,
+ * and returns the number of bytes written into buf.
+ *
+ * Assumes hostlist is locked.
+ */
+static int get_bracketed_list (struct hostlist *hl,
+                              int *start,
+                              FILE *fp)
+{
+    struct hostrange * *hr = hl->hr;
+    int i = *start;
+    int bracket_needed = is_bracket_needed (hl, i);
+
+    if (fprintf (fp, "%s", hr[i]->prefix) < 0
+        || (bracket_needed && fputc ('[', fp) == EOF))
+        return -1;
+
+    while (1) {
+        if (!hr[i]->singlehost) {
+            if (fprintf (fp, "%0*lu", hr[i]->width, hr[i]->lo) < 0)
+                return -1;
+            if (hr[i]->lo < hr[i]->hi)
+                if (fprintf (fp, "-%0*lu", hr[i]->width, hr[i]->hi) < 0)
+                    return -1;
+        }
+        if (++i >= hl->nranges
+            || !hostrange_within_range (hr[i], hr[i-1]))
+            break;
+        /* Only need comma inside brackets */
+        if (bracket_needed && fputc (',', fp) == EOF)
+            return -1;
+    }
+
+    if (bracket_needed && fputc (']', fp) == EOF)
+        return -1;
+
+    *start = i;
+    return 0;
+}
+
+static int hostlist_ranged_string (struct hostlist *hl, FILE *fp)
+{
+    int i = 0;
+    while (i < hl->nranges) {
+        if (get_bracketed_list (hl, &i, fp) < 0)
+            return -1;
+        if (i < hl->nranges && fputc (',', fp) == EOF)
+            return -1;
+    }
+    return 0;
+}
+
+char * hostlist_encode (struct hostlist *hl)
+{
+    int saved_errno;
+    char *result = NULL;
+    size_t len;
+    FILE *fp;
+
+    if (hl == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(fp = open_memstream (&result, &len))
+        || hostlist_ranged_string (hl, fp) < 0)
+        goto fail;
+    if (fclose (fp) < 0) {
+        fp = NULL;
+        goto fail;
+    }
+    return result;
+fail:
+    saved_errno = errno;
+    if (fp)
+        fclose (fp);
+    free (result);
+    errno = saved_errno;
+    return NULL;
+}
+
+static struct hostrange * hr_current (struct hostlist *hl)
+{
+    assert (hl != NULL);
+    assert (hl->current.index <= hl->nhosts);
+    return hl->hr[hl->current.index];
+}
+
+const char *hostlist_current (struct hostlist *hl)
+{
+    int depth;
+
+    if (hl == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    depth = hl->current.depth;
+    if (depth < 0 || hl->current.index > hl->nhosts - 1)
+        return NULL;
+    if (!hl->current.host)
+        hl->current.host = hostrange_host_tostring (hr_current (hl), depth);
+    return hl->current.host;
+}
+
+const char *hostlist_first (struct hostlist *hl)
+{
+    if (hl == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    errno = 0;
+    if (hl->nranges == 0)
+        return NULL;
+
+    set_current (&hl->current, 0, 0);
+
+    return hostlist_current (hl);
+}
+
+const char *hostlist_last (struct hostlist *hl)
+{
+    struct hostrange *last;
+
+    if (hl == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    errno = 0;
+    if (hl->nranges == 0)
+        return NULL;
+
+    last = hl->hr[hl->nranges - 1];
+    set_current (&hl->current, hl->nranges - 1, hostrange_count (last) - 1);
+
+    return hostlist_current (hl);
+}
+
+const char *hostlist_next (struct hostlist *hl)
+{
+    int n;
+
+    if (hl == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    errno = 0;
+    free (hl->current.host);
+    hl->current.host = NULL;
+
+    /*  Already at end of list */
+    if (hl->current.index > hl->nranges - 1)
+        return NULL;
+
+    n = hostrange_count (hr_current (hl));
+
+    /* Advance current within hostrange object,
+     * Move to next object if necessary
+     */
+    if (++(hl->current.depth) > n - 1) {
+        if (++(hl->current.index) >= hl->nranges)
+            return NULL;
+        hl->current.depth = 0;
+    }
+    return hostlist_current (hl);
+}
+
+int hostlist_remove_current (struct hostlist *hl)
+{
+    if (hl == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    return hostlist_remove_at (hl, &hl->current);
+}
+
+/*
+ * vi: tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libhostlist/hostlist.h b/src/common/libhostlist/hostlist.h
new file mode 100644
index 000000000000..3ed1c9d595df
--- /dev/null
+++ b/src/common/libhostlist/hostlist.h
@@ -0,0 +1,173 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Functions for encoding/decoding and manipulating RFC 29 Hostlists */
+
+#ifndef FLUX_HOSTLIST_H
+#define FLUX_HOSTLIST_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct hostlist *hostlist_create (void);
+
+void hostlist_destroy (struct hostlist *hl);
+
+/*  Decode string 's' in RFC 29 Hostlist Format.
+ *
+ *  Returns hostlist on success or NULL on failure with errno set.
+ */
+struct hostlist *hostlist_decode (const char *s);
+
+/*
+ *  Encode a hostlist to a string in RFC 29 Hostlist Format
+ */
+char *hostlist_encode (struct hostlist *hl);
+
+/*
+ *  Copy a hostlist
+ */
+struct hostlist *hostlist_copy (const struct hostlist *hl);
+
+/*  Add a string representation of a hostlist onto the tail of the
+ *   hostlist 'hl'.
+ *  Returns the total number of hosts appended to 'hl', or -1 on failure.
+ */
+int hostlist_append (struct hostlist *hl, const char *hosts);
+
+/*
+ *  Append hostlist 'hl2' onto 'hl1'.
+ *
+ *  Returns number of hosts appended for success, -1 for failure.
+ *
+ */
+int hostlist_append_list (struct hostlist *hl1, struct hostlist *hl2);
+
+/*  Return the nth host in hostlist 'hl' or NULL on failure.
+ *
+ *  Moves the hostlist cursor to the returned host on success.
+ */
+const char * hostlist_nth (struct hostlist * hl, int n);
+
+/*
+ *  Search hostlist hl for the first host matching hostname
+ *   and return position in list if found.
+ *
+ *  Leaves cursor pointing to the matching host.
+ *
+ *  Returns -1 if host is not found.
+ */
+int hostlist_find (struct hostlist * hl, const char *hostname);
+
+/*
+ *  Create an opaque hostname object which can be used with the
+ *   more efficient hostlist_find_hostname() below. This interface
+ *   should be used when matching the same hostname to many hostlists
+ *   since it caches processing of the hostname instead of duplicating
+ *   the work on each call to hostlist_find().
+ *
+ *  Caller must call hostlist_hostname_destroy() to free memory associated
+ *   with the returns hostlist_hostname object.
+ *
+ *  Returns NULL on error.
+ */
+struct hostlist_hostname *hostlist_hostname_create (const char *hostname);
+
+/*
+ *  Free resources associated with hostname hn.
+ */
+void hostlist_hostname_destroy (struct hostlist_hostname *hn);
+
+/*
+ *  Search hostlist hl for the first host matching hostlist_hostname `hn`
+ *   and return position in list if found.
+ *
+ *  Leaves cursor pointing to the matching host.
+ *
+ *  Returns -1 if host is not found.
+ */
+int hostlist_find_hostname (struct hostlist *hl, struct hostlist_hostname *hn);
+
+/*
+ *  Delete all hosts in the list represented by `hosts'
+ *
+ *  Returns the number of hosts successfully deleted, -1 on failure.
+ */
+int hostlist_delete (struct hostlist * hl, const char *hosts);
+
+/*
+ *  Return the number of hosts in hostlist hl.
+ */
+int hostlist_count (struct hostlist * hl);
+
+/* hostlist_is_empty(): return true if hostlist is empty. */
+#define hostlist_is_empty(__hl) ( hostlist_count(__hl) == 0 )
+
+/*
+ *  Sort the hostlist hl.
+ */
+void hostlist_sort (struct hostlist * hl);
+
+/*
+ *  Sort the hostlist hl and remove duplicate entries.
+ */
+void hostlist_uniq (struct hostlist *hl);
+
+/*
+ *  Return the host at the head of hostlist 'hl', or NULL if list is empty.
+ *  Leaves internal cursor pointing at the head item.
+ *
+ *  The returned value is only valid until the next call that modifies
+ *   the cursor, i.e. hostlist_next(), hostlist_first(), hostlist_last(),
+ *    hostlist_find(), and hostlist_nth().
+ */
+const char * hostlist_first (struct hostlist *hl);
+
+/*
+ *  Return the host at the tail of hostlist 'hl', or NULL if list is empty.
+ *  Leaves internal cursor pointing at the last item.
+ *
+ *  The returned value is only valid until the next call that modifies
+ *   the cursor, i.e. hostlist_next(), hostlist_first(), hostlist_last(),
+ *    hostlist_find(), and hostlist_nth().
+ */
+const char * hostlist_last (struct hostlist *hl);
+
+/*
+ *  Advance the internal cursor and return the next host in 'hl' or NULL
+ *   if list is empty or the end of the list has been reached.
+ *
+ *  The returned value is only valid until the next call that modifies
+ *   the cursor, i.e. hostlist_next(), hostlist_first(), hostlist_last(),
+ *    hostlist_find(), and hostlist_nth().
+ */
+const char * hostlist_next (struct hostlist *hl);
+
+/*  Return the current host to which the cursor is pointing, or NULL if
+ *   cursor is pointing to end of list or list is empty.
+ *
+ *  The returned value is only valid until the next call that modifies
+ *   the cursor, i.e. hostlist_next(), hostlist_first(), hostlist_last(),
+ *    hostlist_find(), and hostlist_nth().
+ */
+const char * hostlist_current (struct hostlist *hl);
+
+/* Remove the host at which the current cursor is pointing.
+ * Returns 1 on Success, 0 if the cursor doesn't point to a host
+ *  or -1 on error.
+ */
+int hostlist_remove_current (struct hostlist *hl);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_HOSTLIST_H */
diff --git a/src/common/libhostlist/hostname.c b/src/common/libhostlist/hostname.c
new file mode 100644
index 000000000000..bf2b6c0aee43
--- /dev/null
+++ b/src/common/libhostlist/hostname.c
@@ -0,0 +1,259 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif  /* !HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "hostname.h"
+
+#define INVALID_CHARS ",[]\t "
+
+/*
+ * return the location of the last char in the hostname prefix
+ */
+static int host_prefix_end (const char *hostname, int len)
+{
+    if (len < 0)
+        return -1;
+    int idx = len - 1;
+
+    /* Rewind to the last non-digit in hostname
+     */
+    while (idx >= 0 && isdigit ((char) hostname[idx]))
+        idx--;
+
+    return idx;
+}
+
+/*  Return < 0 if hostname is not valid (e.g. contains invalid characters),
+ *  O/w, return the length of the string.
+ */
+static int hostname_len (const char *hostname)
+{
+    int len = strlen (hostname);
+    if (len != strcspn (hostname, INVALID_CHARS))
+        return -1;
+    return len;
+}
+
+struct stack_hostname *hostname_stack_create_from_hostname (
+    struct stack_hostname *hn,
+    struct hostlist_hostname *src)
+{
+    if (!hn || !src) {
+        errno = EINVAL;
+        return NULL;
+    }
+    hn->hostname = src->hostname;
+    hn->suffix = src->suffix;
+    hn->len = src->len;
+    hn->len_prefix = src->len_prefix;
+    hn->width = src->width;
+    hn->num = src->num;
+    return hn;
+}
+
+struct stack_hostname *hostname_stack_create_with_suffix (
+    struct stack_hostname *hn,
+    const char *hostname,
+    int len,
+    int idx)
+{
+    if (!hostname || !hn || len < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    hn->hostname = hostname;
+    hn->len = len;
+    hn->len_prefix = idx + 1;
+    hn->num = 0;
+    hn->suffix = NULL;
+    if (idx == hn->len - 1) {
+        return hn;
+    }
+    hn->suffix = hn->hostname + hn->len_prefix;
+
+    char *p = NULL;
+    hn->num = strtoul (hn->suffix, &p, 10);
+    if (p == hn->suffix && hn->num == 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    hn->width = hn->len - hn->len_prefix;
+    return hn;
+}
+
+struct stack_hostname *hostname_stack_copy_one_less_digit (
+    struct stack_hostname *dst,
+    struct stack_hostname *src)
+{
+
+    if (!dst || !src) {
+        errno = EINVAL;
+        return NULL;
+    }
+    *dst = *src;
+    dst->len_prefix = src->len_prefix + 1;
+    if (src->len_prefix == dst->len - 1) {
+        return dst;
+    }
+    dst->suffix = dst->hostname + dst->len_prefix;
+
+    dst->width = dst->len - dst->len_prefix;
+    if (dst->width < 0 || dst->width > 10) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    // remove the most significant decimal digit without reparsing
+    // lookup table because pow is slow
+    static const int pow10[10] = {
+        1,
+        10,
+        100,
+        1000,
+        10000,
+        100000,
+        1000000,
+        10000000,
+        100000000,
+        1000000000};
+    dst->num = src->num % pow10[dst->width];
+    return dst;
+}
+
+struct hostlist_hostname *hostname_create_with_suffix (const char *hostname,
+                                                       int idx)
+{
+    struct hostlist_hostname *hn = NULL;
+    int len;
+    char *p = NULL;
+
+    if (!hostname || (len = hostname_len (hostname)) < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (!(hn = calloc (1, sizeof (*hn))))
+        return NULL;
+
+    if (!(hn->hostname = strdup (hostname))) {
+        hostname_destroy (hn);
+        return NULL;
+    }
+
+    hn->num = 0;
+    hn->len = len;
+    hn->len_prefix = idx + 1;
+    hn->width = hn->len - hn->len_prefix;
+    hn->prefix = NULL;
+    hn->suffix = NULL;
+
+    if (idx == len - 1) {
+        if ((hn->prefix = strdup (hostname)) == NULL) {
+            hostname_destroy (hn);
+            return NULL;
+        }
+        return hn;
+    }
+
+    hn->suffix = hn->hostname + idx + 1;
+    errno = 0;
+    hn->num = strtoul (hn->suffix, &p, 10);
+    if (p == hn->suffix || errno != 0) {
+        hostname_destroy (hn);
+        errno = EINVAL;
+        return NULL;
+    }
+    if (*p == '\0') {
+        if (!(hn->prefix = malloc ((idx + 2) * sizeof (char)))) {
+            hostname_destroy (hn);
+            return NULL;
+        }
+        memcpy (hn->prefix, hostname, idx + 1);
+        hn->prefix[idx + 1] = '\0';
+    }
+    return hn;
+}
+
+struct stack_hostname *hostname_stack_create (struct stack_hostname *hn,
+                                              const char *hostname)
+{
+    int len = 0;
+    if (!hostname || (len = hostname_len (hostname)) < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return hostname_stack_create_with_suffix (hn,
+                                              hostname,
+                                              len,
+                                              host_prefix_end (hostname, len));
+}
+
+/*
+ * create a struct hostlist_hostname * object from a string hostname
+ */
+struct hostlist_hostname *hostname_create (const char *hostname)
+{
+    int end;
+    if (!hostname) {
+        errno = EINVAL;
+        return NULL;
+    }
+    end = host_prefix_end (hostname, hostname_len (hostname));
+    return hostname_create_with_suffix (hostname, end);
+}
+
+/* free a hostname object
+ */
+void hostname_destroy (struct hostlist_hostname * hn)
+{
+    int saved_errno = errno;
+    if (hn) {
+        hn->suffix = NULL;
+        free (hn->hostname);
+        free (hn->prefix);
+        free (hn);
+    }
+    errno = saved_errno;
+}
+
+/* return true if the hostname has a valid numeric suffix
+ */
+int hostname_suffix_is_valid (struct hostlist_hostname * hn)
+{
+    return (hn && hn->suffix != NULL);
+}
+
+/* return the width (in characters) of the numeric part of the hostname
+ */
+int hostname_suffix_width (struct hostlist_hostname * hn)
+{
+    if (!hn) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!hn->suffix)
+        return 0;
+    return strlen (hn->suffix);
+}
+
+/*
+ * vi: tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libhostlist/hostname.h b/src/common/libhostlist/hostname.h
new file mode 100644
index 000000000000..cb2856adcf97
--- /dev/null
+++ b/src/common/libhostlist/hostname.h
@@ -0,0 +1,63 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_FLUX_HOSTLIST_HOSTNAME_H
+#define HAVE_FLUX_HOSTLIST_HOSTNAME_H
+
+/*
+ * Convenience structure representing a hostname as a prefix,
+ *  optional numeric part, and suffix.
+ */
+struct hostlist_hostname {
+    char *hostname;         /* cache of initialized hostname        */
+    char *prefix;           /* hostname prefix                      */
+    int len; /* length minus invalid characters */
+    int len_prefix; /* length of the prefix */
+    int width;   /* length of the suffix */
+    unsigned long num;      /* numeric suffix                       */
+
+    /* string representation of numeric suffix
+     * points into `hostname'                                       */
+    char *suffix;
+};
+
+struct stack_hostname {
+    const char *hostname;         /* cache of initialized hostname        */
+    int len; /* length minus invalid characters */
+    int len_prefix; /* length of the prefix */
+    int width;   /* length of the suffix */
+    unsigned long num;      /* numeric suffix                       */
+
+    /* string representation of numeric suffix
+     * points into `hostname'                                       */
+    const char *suffix;
+};
+
+struct hostlist_hostname * hostname_create (const char *s);
+struct hostlist_hostname * hostname_create_with_suffix (const char *s, int i);
+struct stack_hostname *hostname_stack_create (struct stack_hostname *hn,
+                                              const char *hostname);
+struct stack_hostname *hostname_stack_create_from_hostname (
+    struct stack_hostname *hn,
+    struct hostlist_hostname *hn_src);
+struct stack_hostname *hostname_stack_create_with_suffix (
+    struct stack_hostname *hn,
+    const char *hostname,
+    int len,
+    int idx);
+struct stack_hostname *hostname_stack_copy_one_less_digit (
+    struct stack_hostname *dst,
+    struct stack_hostname *src);
+void hostname_destroy (struct hostlist_hostname *hn);
+
+int hostname_suffix_is_valid (struct hostlist_hostname *hn);
+int hostname_suffix_width (struct hostlist_hostname *hn);
+
+#endif /* !HAVE_FLUX_HOSTLIST_HOSTNAME_H */
diff --git a/src/common/libhostlist/hostrange.c b/src/common/libhostlist/hostrange.c
new file mode 100644
index 000000000000..e362f24cf63e
--- /dev/null
+++ b/src/common/libhostlist/hostrange.c
@@ -0,0 +1,479 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif /* !HAVE_CONFIG_H */
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <assert.h>
+#include <errno.h>
+#include <ctype.h>
+#include <limits.h>
+#include <unistd.h>
+#include <stdbool.h>
+
+#include "util.h"
+#include "hostrange.h"
+
+/* max host range: anything larger will be assumed to be an error */
+#define MAX_RANGE    16384    /* 16K Hosts */
+
+/* max number of ranges that will be processed between brackets */
+#define MAX_RANGES    10240    /* 10K Ranges */
+
+/* max size of internal hostrange buffer */
+#define MAXHOSTRANGELEN 1024
+
+
+/* allocate a new hostrange object
+ */
+static struct hostrange * hostrange_new (const char *prefix)
+{
+    struct hostrange *new;
+
+    if (prefix == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(new = calloc (1, sizeof (*new)))
+        || (!(new->prefix = strdup (prefix)))) {
+        hostrange_destroy (new);
+        return NULL;
+    }
+    new->len_prefix = strlen (prefix);
+    return new;
+}
+
+/* Create a struct hostrange * containing a single host without a valid suffix
+ * hr->prefix will represent the entire hostname.
+ */
+struct hostrange * hostrange_create_single (const char *prefix)
+{
+    struct hostrange * new = hostrange_new (prefix);
+    if (!new)
+        return NULL;
+    new->singlehost = 1;
+    return new;
+}
+
+/* Create a hostrange object with a prefix, hi, lo, and format width
+ */
+struct hostrange * hostrange_create (const char *prefix,
+                                     unsigned long lo,
+                                     unsigned long hi,
+                                     int width)
+{
+    struct hostrange * new;
+
+    if (lo > hi || width < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (!(new = hostrange_new (prefix)))
+        return NULL;
+
+    new->lo = lo;
+    new->hi = hi;
+    new->width = width;
+    return new;
+}
+
+
+/* Return the number of hosts stored in the hostrange object
+ */
+unsigned long hostrange_count (struct hostrange * hr)
+{
+    if (!hr) {
+        errno = EINVAL;
+        return 0;
+    }
+    errno = 0;
+    if (hr->singlehost)
+        return 1;
+    else
+        return hr->hi - hr->lo + 1;
+}
+
+/* Copy a hostrange object
+ */
+struct hostrange * hostrange_copy (struct hostrange * hr)
+{
+    if (hr == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (hr->singlehost)
+        return hostrange_create_single (hr->prefix);
+    else
+        return hostrange_create (hr->prefix, hr->lo, hr->hi, hr->width);
+}
+
+
+/* free memory allocated by the hostrange object
+ */
+void hostrange_destroy (struct hostrange * hr)
+{
+    if (hr) {
+        int saved_errno = errno;
+        if (hr->prefix)
+            free (hr->prefix);
+        free (hr);
+        errno = saved_errno;
+    }
+}
+
+/* hostrange_delete_host() deletes a specific host from the range.
+ * If the range is split into two, the greater range is returned,
+ * and `hi' of the lesser range is adjusted accordingly. If the
+ * highest or lowest host is deleted from a range, NULL is returned
+ * and the hostrange hr is adjusted properly.
+ */
+struct hostrange * hostrange_delete_host (struct hostrange * hr,
+                                          unsigned long n)
+{
+    struct hostrange * new = NULL;
+
+    if (!hr) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    assert (hr != NULL);
+    assert (n >= hr->lo && n <= hr->hi);
+
+    if (n == hr->lo)
+        hr->lo++;
+    else if (n == hr->hi)
+        hr->hi--;
+    else {
+        if (!(new = hostrange_copy (hr)))
+            return NULL;
+        hr->hi = n - 1;
+        new->lo = n + 1;
+    }
+
+    return new;
+}
+
+/* compare the prefixes of two hostrange objects.
+ * returns:
+ *    < 0   if h1 prefix is less than h2 OR h1 == NULL.
+ *
+ *      0   if h1's prefix and h2's prefix match,
+ *          UNLESS, either h1 or h2 (NOT both) do not have a valid suffix.
+ *
+ *    > 0   if h1's prefix is greater than h2's OR h2 == NULL.
+ */
+int hostrange_prefix_cmp (struct hostrange * h1, struct hostrange * h2)
+{
+    int retval;
+    if (h1 == NULL)
+        return -1;
+    if (h2 == NULL)
+        return 1;
+
+    int min_len = h1->len_prefix < h2->len_prefix ? h1->len_prefix : h2->len_prefix;
+    retval = memcmp (h1->prefix, h2->prefix, min_len);
+
+    if (retval == 0) {
+        if (h1->len_prefix < h2->len_prefix)
+            return -1;
+        if (h1->len_prefix > h2->len_prefix)
+            return 1;
+        retval = h2->singlehost - h1->singlehost;
+    }
+
+    return retval;
+}
+
+
+/* hostrange_cmp() is used to sort hostrange objects. It will
+ * sort based on the following (in order):
+ *  o result of strcmp on prefixes
+ *  o if widths are compatible, then:
+ *       sort based on lowest suffix in range
+ *    else
+ *       sort based on width                     */
+int hostrange_cmp (struct hostrange * h1, struct hostrange * h2)
+{
+    int retval;
+
+    assert (h1 != NULL);
+    assert (h2 != NULL);
+
+    if ((retval = hostrange_prefix_cmp (h1, h2)) == 0)
+        retval = hostrange_width_combine (h1, h2) ?
+            h1->lo - h2->lo : h1->width - h2->width;
+
+    /*  Only return -1, 0, or 1 */
+    if (retval != 0)
+        retval = retval < 0 ? -1 : 1;
+
+    return retval;
+}
+
+/* returns true if h1 and h2 would be included in the same bracketed hostlist.
+ * h1 and h2 will be in the same bracketed list iff:
+ *
+ *  1. h1 and h2 have same prefix
+ *  2. neither h1 nor h2 are singlet hosts (i.e. invalid suffix)
+ *
+ *  (XXX: Should incompatible widths be placed in the same bracketed list?
+ *        There's no good reason not to, except maybe aesthetics)
+ */
+int hostrange_within_range (struct hostrange * h1, struct hostrange * h2)
+{
+    if (hostrange_prefix_cmp (h1, h2) == 0)
+        return h1->singlehost || h2->singlehost ? 0 : 1;
+    else
+        return 0;
+}
+
+/* compare two hostrange objects to determine if they are width
+ * compatible,  returns:
+ *  1 if widths can safely be combined
+ *  0 if widths cannot be safely combined
+ */
+int hostrange_width_combine (struct hostrange * h0, struct hostrange * h1)
+{
+    assert (h0 != NULL);
+    assert (h1 != NULL);
+
+    return width_equiv (h0->lo, &h0->width, h1->lo, &h1->width);
+}
+
+
+/* Return true if hostrange hr contains no hosts, i.e. hi < lo
+ */
+int hostrange_empty (struct hostrange * hr)
+{
+    assert (hr != NULL);
+    return ((hr->hi < hr->lo) || (hr->hi == (unsigned long) -1));
+}
+
+/* join two hostrange objects.
+ *
+ * returns:
+ *
+ * -1 if ranges do not overlap (including incompatible zero padding)
+ *  0 if ranges join perfectly
+ * >0 number of hosts that were duplicated in  h1 and h2
+ *
+ * h2 will be coalesced into h1 if rc >= 0
+ *
+ * it is assumed that h1->lo <= h2->lo, i.e. hr1 <= hr2
+ *
+ */
+int hostrange_join (struct hostrange * h1, struct hostrange * h2)
+{
+    int duplicated = -1;
+
+    assert (h1 != NULL);
+    assert (h2 != NULL);
+    assert (hostrange_cmp (h1, h2) <= 0);
+
+    if (hostrange_prefix_cmp (h1, h2) == 0 &&
+        hostrange_width_combine (h1, h2)) {
+
+        if (h1->singlehost && h2->singlehost) {    /* matching singlets  */
+            duplicated = 1;
+        } else if (h1->hi == h2->lo - 1) {    /* perfect join       */
+            h1->hi = h2->hi;
+            duplicated = 0;
+        } else if (h1->hi >= h2->lo) {    /* some duplication   */
+            if (h1->hi < h2->hi) {
+                duplicated = h1->hi - h2->lo + 1;
+                h1->hi = h2->hi;
+            } else
+                duplicated = hostrange_count (h2);
+        }
+    }
+
+    return duplicated;
+}
+
+/* hostrange intersect returns the intersection (common hosts)
+ * of hostrange objects h1 and h2. If there is no intersection,
+ * NULL is returned.
+ *
+ * It is assumed that h1 <= h2 (i.e. h1->lo <= h2->lo)
+ */
+struct hostrange * hostrange_intersect (struct hostrange * h1,
+                                        struct hostrange * h2)
+{
+    struct hostrange * new = NULL;
+
+    assert (h1 != NULL);
+    assert (h2 != NULL);
+
+    if (h1->singlehost || h2->singlehost)
+        return NULL;
+
+    assert (hostrange_cmp (h1, h2) <= 0);
+
+    if ((hostrange_prefix_cmp (h1, h2) == 0)
+        && (h1->hi > h2->lo)
+        && (hostrange_width_combine (h1, h2))) {
+
+        if (!(new = hostrange_copy (h1)))
+            return NULL;
+        new->lo = h2->lo;
+        new->hi = h2->hi < h1->hi ? h2->hi : h1->hi;
+    }
+
+    return new;
+}
+
+/* return offset of hn if it is in the hostlist or
+ *        -1 if not.
+ */
+int hostrange_hn_within (struct hostrange * hr, struct stack_hostname * hn)
+{
+    if (hr->singlehost) {
+        /*
+         *  If the current hostrange [hr] is a `singlehost' (no valid
+         *   numeric suffix (lo and hi)), then the hostrange [hr]
+         *   stores just one host with name == hr->prefix.
+         *
+         *  Thus the full hostname in [hn] must match hr->prefix, in
+         *   which case we return true. Otherwise, there is no
+         *   possibility that [hn] matches [hr].
+         */
+        if (hr->len_prefix != hn->len)
+            return -1;
+        if (memcmp (hn->hostname, hr->prefix, hr->len_prefix) == 0)
+            return 0;
+        else
+            return -1;
+    }
+
+    /*
+     *  Now we know [hr] is not a "singlehost", so hostname
+     *   better have a valid numeric suffix, or there is no
+     *   way we can match
+     */
+    if (!hn || !hn->suffix)
+        return -1;
+
+    /*
+     *  If hostrange and hostname prefixes don't match to at least
+     *   the length of the hostname object (which will have the min
+     *   possible prefix length), then there is no way the hostname
+     *   falls within the range [hr].
+     */
+    if (hr->len_prefix < hn->len_prefix)
+        return -1;
+    if (memcmp (hr->prefix, hn->hostname, hn->len_prefix) != 0)
+        return -1;
+
+    /*
+     *  Now we know hostrange and hostname prefixes match up to the
+     *   length of the hostname prefix.  If the hostrange and hostname
+     *   prefix lengths do not match (specifically if the hostname prefix
+     *   length is less than the hostrange prefix length) and the
+     *   hostrange prefix contains trailing digits, then it might be
+     *   the case that the hostrange was created by forcing the prefix
+     *   to contain digits a la f00[1-2]. So we try adjusting the
+     *   hostname with the longer prefix and calling this function
+     *   again with the new hostname. (Yes, this is ugly, sorry)
+     */
+    if ((hn->len_prefix < hr->len_prefix)
+         && (hn->width > 1)
+         && (isdigit (hr->prefix [hr->len_prefix - 1]))
+         && (hr->prefix [hn->len_prefix] == hn->suffix[0]) ) {
+        int rc;
+        /*
+         *  Create new hostname object with its prefix offset by one
+         */
+        struct stack_hostname shn;
+        struct stack_hostname * h = hostname_stack_copy_one_less_digit (&shn, hn);
+        /*
+         *  Recursive call :-o
+         */
+        rc = hostrange_hn_within (hr, h);
+        return rc;
+    }
+
+
+    /*
+     *  Finally, check whether [hn], with a valid numeric suffix,
+     *   falls within the range of [hr] if [hn] and [hr] prefix are
+     *   identical.
+     */
+    if ((hr->len_prefix == hn->len_prefix)
+        && (hn->num <= hr->hi)
+        && (hn->num >= hr->lo)) {
+        int width = hn->width;
+        if (!width_equiv (hr->lo, &hr->width, hn->num, &width))
+            return -1;
+        return (hn->num - hr->lo);
+    }
+
+    return -1;
+}
+
+/* Place the string representation of the numeric part of hostrange into buf
+ * writing at most n chars including NUL termination.
+ */
+size_t hostrange_numstr (struct hostrange * hr, size_t n, char *buf)
+{
+    int len = 0;
+
+    assert (hr != NULL && buf != NULL);
+
+    if (hr->singlehost || n == 0)
+        return 0;
+
+    len = snprintf (buf, n, "%0*lu", hr->width, hr->lo);
+
+    if ((len >= 0) && (len < n) && (hr->lo < hr->hi)) {
+        int len2 = snprintf (buf+len, n-len, "-%0*lu", hr->width, hr->hi);
+        if (len2 < 0)
+            len = -1;
+        else
+            len += len2;
+    }
+
+    return len;
+}
+
+char * hostrange_host_tostring (struct hostrange * hr, int depth)
+{
+    char buf[_POSIX_HOST_NAME_MAX + 16];
+    int len;
+
+    if (!hr || depth < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    len = snprintf (buf, sizeof (buf), "%s", hr->prefix);
+    if (len < 0 || len >= sizeof (buf))
+        return NULL;
+
+    if (!hr->singlehost) {
+        unsigned long n = hr->lo + depth;
+        if (n > hr->hi) {
+            errno = ERANGE;
+            return NULL;
+        }
+        snprintf (buf+len, sizeof (buf) - len, "%0*lu",
+                 hr->width, hr->lo + depth);
+    }
+    return strdup (buf);
+}
+
+/*
+ * vi: tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libhostlist/hostrange.h b/src/common/libhostlist/hostrange.h
new file mode 100644
index 000000000000..1938946409e6
--- /dev/null
+++ b/src/common/libhostlist/hostrange.h
@@ -0,0 +1,68 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_FLUX_HOSTRANGE_H
+#define HAVE_FLUX_HOSTRANGE_H
+
+#include "hostname.h"
+
+/* A single prefix with `hi' and `lo' numeric suffix values
+ */
+struct hostrange {
+    char *prefix;        /* alphanumeric prefix: */
+    unsigned long len_prefix; /* length of the prefix */
+
+    /* beginning (lo) and end (hi) of suffix range */
+    unsigned long lo, hi;
+
+    /* width of numeric output format
+     * (pad with zeros up to this width) */
+    int width;
+
+    /* If singlehost is 1, `lo' and `hi' are invalid */
+    unsigned singlehost:1;
+};
+
+struct hostrange * hostrange_create_single (const char *);
+
+struct hostrange * hostrange_create (const char *s,
+                                     unsigned long lo,
+                                     unsigned long hi,
+                                     int width);
+
+unsigned long hostrange_count (struct hostrange *);
+
+struct hostrange * hostrange_copy (struct hostrange *);
+
+void hostrange_destroy (struct hostrange *);
+
+struct hostrange * hostrange_delete_host (struct hostrange *, unsigned long);
+
+int hostrange_cmp (struct hostrange *, struct hostrange *);
+
+int hostrange_prefix_cmp (struct hostrange *, struct hostrange *);
+int hostrange_within_range (struct hostrange *, struct hostrange *);
+int hostrange_width_combine (struct hostrange *, struct hostrange *);
+int hostrange_empty (struct hostrange *);
+
+int hostrange_join (struct hostrange *, struct hostrange *);
+
+struct hostrange * hostrange_intersect (struct hostrange *,
+                                        struct hostrange *);
+
+int hostrange_hn_within (struct hostrange *, struct stack_hostname *);
+
+size_t hostrange_numstr(struct hostrange *, size_t, char *);
+
+/*  Return the string representation of the nth string in 'hr'.
+ */
+char * hostrange_host_tostring (struct hostrange *hr, int n);
+
+#endif /* !HAVE_FLUX_HOSTRANGE_H */
diff --git a/src/common/libhostlist/test/hostlist.c b/src/common/libhostlist/test/hostlist.c
new file mode 100644
index 000000000000..fba8f49c4ee0
--- /dev/null
+++ b/src/common/libhostlist/test/hostlist.c
@@ -0,0 +1,660 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <errno.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libhostlist/hostlist.h"
+
+void test_basic ()
+{
+    struct hostlist *hl = hostlist_create ();
+    if (!hl)
+        BAIL_OUT ("hostlist_create failed");
+    ok (hostlist_count (hl) == 0,
+        "hostlist_create creates empty hostlist");
+    hostlist_destroy (hl);
+
+    hl = hostlist_decode (NULL);
+    ok (hostlist_count (hl) == 0,
+        "hostlist_decode (NULL) returns empty hostlist");
+    hostlist_destroy (hl);
+
+    ok (hostlist_decode ("foo[0-1048576]") == NULL && errno == ERANGE,
+        "hostlist_decode () fails with ERANGE for too large host range");
+
+    ok (hostlist_copy (NULL) == NULL,
+        "hostlist_copy (NULL) returns NULL");
+    ok (hostlist_append (NULL, "foo") < 0 && errno == EINVAL,
+        "hostlist_append (NULL, 'foo') returns EINVAL");
+
+    hl = hostlist_create ();
+    ok (hostlist_append (hl, NULL) == 0,
+        "hostlist_append (hl, NULL) returns 0");
+    ok (hostlist_append (hl, "") == 0,
+        "hostlist_append (hl, '') returns 0");
+
+    ok (hostlist_append_list (NULL, NULL) < 0 && errno == EINVAL,
+        "hostlist_append_list (NULL, NULL) returns EINVAL");
+
+    ok (hostlist_nth (NULL, 0) == NULL && errno == EINVAL,
+        "hostlist_nth (NULL, 0) returns NULL");
+    ok (hostlist_nth (hl, -1) == NULL && errno == EINVAL,
+        "hostlist_nth (hl, -1) returns EINVAL");
+
+    ok (hostlist_find (NULL, NULL) < 0 && errno == EINVAL,
+        "hostlist_find (NULL, NULL) returns EINVAL");
+
+    ok (hostlist_delete (NULL, NULL) < 0 && errno == EINVAL,
+        "hostlist_delete (NULL, NULL) returns EINVAL");
+
+    ok (hostlist_count (NULL) == 0,
+        "hostlist_count(NULL) returns 0");
+
+    lives_ok ({hostlist_sort (NULL);},
+              "hostlist_sort (NULL) doesn't crash");
+
+    lives_ok ({hostlist_uniq (NULL);},
+              "hostlist_uniq (NULL) doesn't crash");
+
+    hostlist_destroy (hl);
+}
+
+void test_encode_decode_basic ()
+{
+    struct hostlist * hl;
+    char *s;
+
+    ok (hostlist_encode (NULL) == NULL && errno == EINVAL,
+        "hostlist_encode (NULL) returns EINVAL");
+    ok (hostlist_decode (NULL) == NULL && errno == EINVAL,
+        "hostlist_decode (NULL) returns EINVAL");
+
+    hl = hostlist_decode ("");
+    if (!hl)
+        BAIL_OUT ("hostlist_encode failed");
+
+    ok (hl && hostlist_count (hl) == 0,
+        "hostlist_encode ('') creates hostlist with zero size");
+
+    s = hostlist_encode (hl);
+    if (!s)
+        BAIL_OUT ("hostlist_decode failed");
+    is (s, "",
+        "hostlist_decode of empty list returns empty string");
+    free (s);
+    hostlist_destroy (hl);
+
+}
+
+void test_iteration_basic ()
+{
+    struct hostlist *hl = hostlist_create ();
+    if (hl == NULL)
+        BAIL_OUT ("hostlist_create failed!");
+
+    ok (hostlist_first (NULL) == NULL && errno == EINVAL,
+        "hostlist_first (NULL) returns EINVAL");
+    ok (hostlist_last (NULL) == NULL && errno == EINVAL,
+        "hostlist_last (NULL) returns EINVAL");
+    ok (hostlist_next (NULL) == NULL && errno == EINVAL,
+        "hostlist_next (NULL) returns EINVAL");
+    ok (hostlist_current (NULL) == NULL && errno == EINVAL,
+        "hostlist_current (NULL) returns EINVAL");
+    ok (hostlist_remove_current (NULL) < 0 && errno == EINVAL,
+        "hostlist_remove_current (NULL) returns EINVAL");
+
+    ok (hostlist_first (hl) == NULL,
+        "hostlist_first on empty hostlist returns NULL");
+    ok (hostlist_last (hl) == NULL,
+        "hostlist_last on empty hostlist returns NULL");
+    ok (hostlist_current (hl) == NULL,
+        "hostlist_current on empty hostlist returns NULL");
+    ok (hostlist_next (hl) == NULL,
+        "hostlist_next on empty hostlist returns NULL");
+    ok (hostlist_remove_current (hl) == 0,
+        "hostlist_remove_current on empty list returns 0");
+
+    hostlist_destroy (hl);
+}
+
+void test_invalid_decode ()
+{
+    const char *input[] = {
+        "[]",
+        "foo[]",
+        "foo[",
+        "foo[1,3",
+        "foo[[1,3]",
+        "foo]",
+        "foo[x-y]",
+        "foo[0-1,2--5]",
+        NULL,
+    };
+    for (const char **entry = input; *entry; entry++) {
+        struct hostlist *hl = hostlist_decode (*entry);
+        ok (hl == NULL,
+            "hostlist_decode (%s) returns NULL", *entry);
+        if (hl) {
+            char *s = hostlist_encode (hl);
+            diag ("%s", s);
+            free (s);
+        }
+    }
+}
+
+
+struct codec_test {
+    char *input;
+    char *output;
+    int count;
+};
+
+struct codec_test codec_tests[] = {
+    { "foo-1a-2,foo-1a-3",                      "foo-1a-[2-3]",        2 },
+    { "foo1,foo2,foo3,fooi",                    "foo[1-3],fooi",       4 },
+    { "foo1,fooi,foo2,foo3",                    "foo1,fooi,foo[2-3]",  4 },
+    { "fooi,foo1,foo2,foo3",                    "fooi,foo[1-3]",       4 },
+    { "fooi,foo1,foo2,foo3,foo5,foo7,foo8",     "fooi,foo[1-3,5,7-8]", 7 },
+    { "1,2,3,4,5,9",                            "[1-5,9]",             6 },
+    { ",1,2,3,4,5,9",                           "[1-5,9]",             6 },
+    { ",1,2,3,4,5,9,",                          "[1-5,9]",             6 },
+    { "[1-5]",                                  "[1-5]",               5 },
+    { "foo[1,3]-bar",                           "foo1-bar,foo3-bar",   2 },
+    { "[00-03]p",                               "00p,01p,02p,03p",     4 },
+    { "p[00-3]p",                               "p00p,p01p,p02p,p03p", 4 },
+    { "14636",                                  "14636",               1 },
+    { "mcr[336-359,488-550,553,556,559,561,567,569-571,573-575,578,581,584,587-589,592,594,597,600-602,605,608,610,618-622,627,634,636-670,687-696,699-733,735-742,744-760,762-773]",
+       "mcr[336-359,488-550,553,556,559,561,567,569-571,573-575,578,581,584,587-589,592,594,597,600-602,605,608,610,618-622,627,634,636-670,687-696,699-733,735-742,744-760,762-773]", 237 },
+    { 0 },
+};
+
+void test_encode_decode ()
+{
+    struct codec_test *t = codec_tests;
+    while (t && t->input) {
+        char *result;
+        struct hostlist *hl = hostlist_decode (t->input);
+        struct hostlist *copy = hostlist_copy (hl);
+        if (!hl)
+            BAIL_OUT ("hostlist_decode failed");
+        ok (hostlist_count (hl) == t->count,
+            "hostlist_decode returned count=%d", hostlist_count (hl));
+        result = hostlist_encode (hl);
+        if (!result)
+            BAIL_OUT ("hostlist_encode failed");
+        is (result, t->output,
+            "hostlist_decode: %s -> %s", t->input, result);
+        free (result);
+
+        /* Ensure copy works */
+        result = hostlist_encode (copy);
+        is (result, t->output,
+            "hostlist_copy worked");
+        free (result);
+
+        hostlist_destroy (hl);
+        hostlist_destroy (copy);
+        t++;
+    }
+}
+
+void test_append ()
+{
+    struct hostlist *hl = hostlist_create ();
+    struct hostlist *hl2;
+    int n;
+    char *s;
+
+    ok (hostlist_append (hl, "") == 0,
+        "hostlist_append ("") returns 0");
+    ok (hostlist_count (hl) == 0,
+        "hostlist_count returns 0");
+    ok (hostlist_append (hl, "foo12") == 1,
+        "hostlist_append ('foo12') returns 1");
+    ok (hostlist_append (hl, "foo[4,1-2]") == 3,
+        "hostlist_append ('foo[4,1-2]') == 3");
+    ok (hostlist_count (hl) == 4,
+        "hostlist_count is now 4");
+    if (!(s = hostlist_encode (hl)))
+        BAIL_OUT ("hostlist_encode failed!");
+    is (s, "foo[12,4,1-2]",
+        "hostlist is encoded to %s", s);
+    free (s);
+
+    hl2 = hostlist_decode ("bar[26-30]");
+    if (!hl2)
+        BAIL_OUT ("hostlist_create failed");
+
+    n = hostlist_append_list (hl, hl2);
+    ok (n == 5,
+        "hostlist_append_list returned %d", n);
+
+    ok (hostlist_count (hl) == 9,
+        "hostlist_count is now 9");
+
+    if (!(s = hostlist_encode (hl)))
+        BAIL_OUT ("hostlist_encode failed");
+    is (s, "foo[12,4,1-2],bar[26-30]",
+        "hostlist is now %s", s);
+    free (s);
+
+    hostlist_destroy (hl);
+    hostlist_destroy (hl2);
+}
+
+void test_nth ()
+{
+    int count;
+    const char *host;
+    struct hostlist *hl = hostlist_create ();
+    if (!hl)
+        BAIL_OUT ("hostlist_create failed");
+
+    ok (hostlist_nth (hl, 0) == NULL && errno == ENOENT,
+        "hostlist_nth (hl, 0) on empty list returns ENOENT");
+
+    count = hostlist_append (hl, "foo[1-2,4,5],bar");
+    ok (count == 5,
+        "Added 5 hosts to hostlist");
+
+    ok (hostlist_nth (hl, count) == NULL && errno == ENOENT,
+        "hostlist_nth (hl, hostlist_count (hl)) returns ENOENT");
+
+    if (!(host = hostlist_nth (hl, 0)))
+        BAIL_OUT ("hostlist_nth (hl, 0) failed!");
+    is (host, "foo1",
+        "hostlist_nth (hl, 0) returns %s", host);
+    is (hostlist_current (hl), "foo1",
+        "hostlist_nth (hl, 0) leaves cursor at %s", hostlist_current (hl));
+
+    if (!(host = hostlist_nth (hl, 4)))
+        BAIL_OUT ("hostlist_nth (hl, 4) failed");
+    is (host, "bar",
+        "hostlist_nth (hl, 4) returns %s", host);
+    is (hostlist_current (hl), "bar",
+        "hostlist_nth (hl, 4) leaves cursor at %s", hostlist_current (hl));
+
+    if (!(host = hostlist_nth (hl, 2)))
+        BAIL_OUT ("hostlist_nth (hl, 2) failed");
+    is (host, "foo4",
+        "hostlist_nth (hl, 2) returns %s", host);
+    hostlist_destroy (hl);
+}
+
+struct find_test {
+    char *input;
+    char *arg;
+    int rc;
+};
+
+static struct find_test find_tests[] = {
+    { "tst0",          "tst",        -1  },
+    { "tst0,tst",      "tst",         1  },
+    { "tst,tst0",      "tst",         0  },
+    { "tst",           "tst0",       -1  },
+    { "foo[1-5]-eth0", "foo3-eth0",   2  },
+    { "foo[1-5]",      "foo3-eth0",  -1  },
+    { "[0-5]",         "3",           3  },
+    { "[0-5]i",        "0",          -1  },
+    { "i[0-5]",        "i00",        -1  },
+    { "i[00-05]",      "i00",         0  },
+    { "i[00-05]",      "i04",         4  },
+    { "f00[7-8]",      "f007",        0  },
+    { "f00[7-8,10]",   "f0010",       2  },
+    { "f0010001[07-08]","f001000108", 1  },
+    { "cornp2",        "corn",       -1  },
+    { "cornp2",        "corn2",      -1  },
+    { "corn-p2",       "corn2",      -1  },
+    { "corn1-p2",      "corn2",      -1  },
+    { 0 },
+};
+
+void test_find ()
+{
+    struct find_test *t = find_tests;
+
+    while (t && t->input) {
+        int rc;
+        struct hostlist *hl = hostlist_decode (t->input);
+        if (!hl)
+            BAIL_OUT ("hostlist_decode (%s) failed!", t->input);
+        rc = hostlist_find (hl, t->arg);
+        ok (rc == t->rc,
+            "hostlist_find ('%s', '%s') returned %d",
+            t->input, t->arg, rc);
+        if (t->rc >= 0)
+            is (hostlist_current (hl), t->arg,
+                "hostlist_find leaves cursor pointing to found host");
+        hostlist_destroy (hl);
+        t++;
+    }
+}
+
+void test_find_hostname ()
+{
+    struct find_test *t = find_tests;
+
+    ok (hostlist_find_hostname (NULL, NULL) == -1 && errno == EINVAL,
+        "hostlist_find_hostname (NULL, NULL) returns EINVAL");
+
+    while (t && t->input) {
+        int rc;
+        struct hostlist_hostname *hn;
+        struct hostlist *hl = hostlist_decode (t->input);
+        if (!hl)
+            BAIL_OUT ("hostlist_decode (%s) failed!", t->input);
+        if (!(hn = hostlist_hostname_create (t->arg)))
+            BAIL_OUT ("hostlist_hostname_create (%s) failed!", t->arg);
+        rc = hostlist_find_hostname (hl, hn);
+        ok (rc == t->rc,
+            "hostlist_find_hostname ('%s', '%s') returned %d",
+            t->input, t->arg, rc);
+        if (t->rc >= 0)
+            is (hostlist_current (hl), t->arg,
+                "hostlist_find leaves cursor pointing to found host");
+        hostlist_hostname_destroy (hn);
+        hostlist_destroy (hl);
+        t++;
+    }
+}
+
+
+struct delete_test {
+    char *input;
+    char *delete;
+    int rc;
+    char *result;
+};
+
+static struct delete_test delete_tests[] = {
+    { "foo[2-5]",      "foo6",         0, "foo[2-5]"            },
+    { "foo[2-5]",      "foo3",         1, "foo[2,4-5]"          },
+    { "foo[2-5],fooi", "fooi",         1, "foo[2-5]"            },
+    { "foo[2-5],fooi", "foo3",         1, "foo[2,4-5],fooi"     },
+    { "foo[2-5],fooi", "foo[1-2]",     1, "foo[3-5],fooi"       },
+    { "foo[0-7]",      "foo[1,0,2-7]", 8, ""                    },
+    { "foo[2-4]-eth2", "foo3-eth2",    1, "foo2-eth2,foo4-eth2" },
+    { 0 },
+};
+
+void test_delete ()
+{
+    struct delete_test *t = delete_tests;
+
+    while (t && t->input) {
+        char *s;
+        int rc;
+        struct hostlist *hl = hostlist_decode (t->input);
+        if (!hl)
+            BAIL_OUT ("hostlist_decode (%s) failed!", t->input);
+        rc = hostlist_delete (hl, t->delete);
+        ok (rc == t->rc,
+            "del ('%s', '%s') returned %d",
+            t->input, t->delete, rc);
+        s = hostlist_encode (hl);
+        is (s, t->result,
+            "result = '%s'", s);
+        free (s);
+
+        hostlist_destroy (hl);
+        t++;
+    }
+}
+
+struct sortuniq_test {
+    char *input;
+    char *sorted;
+    char *uniq;
+};
+
+struct sortuniq_test sortuniq_tests[] = {
+    { "foo,f,bar,baz",         "bar,baz,f,foo",         "bar,baz,f,foo"     },
+    { "[5-6],[3-4],[1-2,0]",   "[0-6]",                 "[0-6]"             },
+    { "[0-20],12,15",          "[0-12,12-15,15-20]",    "[0-20]"            },
+    { "0,1,2,3,4,5,1,5",       "[0-1,1-5,5]",           "[0-5]"             },
+    { "[0-20],45,12,15",       "[0-12,12-15,15-20,45]", "[0-20,45]"         },
+    { "[0-20],45,12,015",      "[0-12,12-20,45,015]",   "[0-20,45,015]"     },
+    { "bar1,bar2,foo1,foo,foo","bar[1-2],foo,foo,foo1", "bar[1-2],foo,foo1" },
+    { "foo[5-6],foo3,foo4",    "foo[3-6]",              "foo[3-6]"          },
+    { "foo[5-6],foo[4-7]",     "foo[4-5,5-6,6-7]",      "foo[4-7]"          },
+    { "foo[0-3],foo[0-3]",     "foo[0,0-1,1-2,2-3,3]",  "foo[0-3]"          },
+    { "foo[0-2],foo[0-2],foo[0-2]",
+      "foo[0,0,0-1,1,1-2,2,2]",
+      "foo[0-2]"
+    },
+    { 0 }
+};
+
+void test_sortuniq ()
+{
+    struct sortuniq_test *t = sortuniq_tests;
+
+    while (t && t->input) {
+        char *sorted, *uniq;
+        struct hostlist *hl = hostlist_decode (t->input);
+        struct hostlist *hl2 = hostlist_decode (t->input);
+        if (!hl || !hl2)
+            BAIL_OUT ("hostlist_decode (%s) failed!", t->input);
+
+        hostlist_sort (hl);
+        if (!(sorted = hostlist_encode (hl)))
+            BAIL_OUT ("hostlist_encode failed!");
+        is (sorted, t->sorted,
+            "hostlist_sort(%s) = '%s'", t->input, sorted);
+
+        hostlist_uniq (hl2);
+        if (!(uniq = hostlist_encode (hl2)))
+            BAIL_OUT ("hostlist_encode failed!");
+        is (uniq, t->uniq,
+            "hostlist_uniq(%s) = '%s'", t->input, uniq);
+
+        free (sorted);
+        free (uniq);
+        hostlist_destroy (hl);
+        hostlist_destroy (hl2);
+        t++;
+    }
+
+}
+
+const char *iterator_inputs[] = {
+    "",
+    "mcr[336-359,488-550,553,556,559,561,567,569-571,573-575,578,581,584,587-589,592,594,597,600-602,605,608,610,618-622,627,634,636-670,687-696,699-733,735-742,744-760,762-773]",
+    "mcr[774-796,799-814,986,1096-1114,1147-1151]",
+    "really-long-hostname-prefix[10101,55,35,2]",
+    "really-really-really-super-duper-long-hostname-prefix[10101,55,35,2]",
+    "[336-359,488-550,553,556,559,561,567,569-571,573-575,578,581,584,587-589,592,594,597,600-602,605,608,610,618-622,627,634,636-670,687-696,699-733,735-742,744-760,762-773]",
+    "one,two,three,four,five",
+    NULL
+};
+
+void test_iteration ()
+{
+    struct hostlist *hl;
+    struct hostlist *nl;
+    const char *host;
+    char *result;
+
+    for (const char **input = iterator_inputs; *input != NULL; input++) {
+        char *first = NULL;
+        char *last = NULL;
+
+        if (!(hl = hostlist_decode (*input)))
+            BAIL_OUT ("hostlist_decode failed!");
+        if (!(nl = hostlist_create ()))
+            BAIL_OUT ("hostlist_create failed!");
+
+        if ((host = hostlist_last (hl)))
+            last = strdup (host);
+        if ((host = hostlist_first (hl)))
+            first = strdup (host);
+
+        while (host) {
+            hostlist_append (nl, hostlist_current (hl));
+            host = hostlist_next (hl);
+        }
+
+        if (!(result = hostlist_encode (nl)))
+            BAIL_OUT ("hostlist_encode failed!");
+        is (result, (char *) *input,
+            "hostlist_next iterated %d hosts in order",
+            hostlist_count (hl));
+
+        if (first) {
+            is (hostlist_first (hl), first,
+                "hostlist_first resets to first host");
+            is (hostlist_current (hl), first,
+                "hostlist_current() works");
+
+            if (hostlist_next (hl) && hostlist_next (hl)) {
+                ok (hostlist_remove_current (hl) == 1,
+                    "hostlist_remove_current works");
+            }
+
+            is (hostlist_last (hl), last,
+                "hostlist_last resets to last host");
+            is (hostlist_current (hl), last,
+                "hostlist_current() works");
+            ok (hostlist_remove_current (hl) == 1,
+                "hostlist_remove_current() works at last host");
+        }
+
+        free (result);
+        free (first);
+        free (last);
+        hostlist_destroy (hl);
+        hostlist_destroy (nl);
+    }
+}
+
+
+struct test_next_delete {
+    const char *descr;
+    const char *input;
+    int n;
+    const char *delete;
+    const char *next;
+};
+
+struct test_next_delete next_delete_tests[] = {
+    { "delete host at cursor in hr",
+      "foo[0-7]", 1, "foo1", "foo2"
+    },
+    { "delete host before cursor in hr",
+      "foo[0-7]", 4, "foo1", "foo5"
+    },
+    { "delete host which removes hr at cursor",
+      "foo[0,2,4-5]", 1, "foo2", "foo4",
+    },
+    { "delete host which removes hr before cursor",
+      "foo[0,2,4-5]", 2, "foo2", "foo5",
+    },
+    { "delete current at beginning of list",
+      "foo[0-15]", 0, NULL, "foo1",
+    },
+    { "delete current in middle of list",
+      "foo[0-15]", 7, NULL, "foo8",
+    },
+    { "delete current in middle of list with multiple hostranges",
+      "foo[0-1,3,15]", 2, NULL, "foo15",
+    },
+    { "single hostrange, delete host near beginning",
+      "foo[0-100]", 50, "foo1", "foo51"
+    },
+    { "single hostrange, delete host at beginning",
+      "foo[0-100]", 50, "foo0", "foo51"
+    },
+    { 0 },
+};
+
+void test_iteration_with_delete ()
+{
+    struct test_next_delete *t = next_delete_tests;
+
+    /* Other tests */
+    while (t && t->descr) {
+        struct hostlist *hl = hostlist_decode (t->input);
+        const char *host;
+        if (!hl)
+            BAIL_OUT ("hostlist_decode failed!");
+        if (!hostlist_first (hl))
+            BAIL_OUT ("hostlist_first failed!");
+        for (int i = 0; i < t->n; i++)
+            if (!hostlist_next (hl))
+                BAIL_OUT ("hostlist_next i=%d failed!", i);
+
+        if (t->delete != NULL)
+            ok (hostlist_delete (hl, t->delete) == 1,
+                "hostlist_delete %s from %s works",
+                t->delete, t->input);
+        else
+            ok (hostlist_remove_current (hl) == 1,
+                "hostlist_remove_current works");
+        host = hostlist_next (hl);
+        is (host, t->next,
+            "hostlist_next returns %s", host);
+
+        hostlist_destroy (hl);
+        t++;
+    }
+}
+
+/*  Ensure a very large host list can be encoded without error
+ */
+static void test_encode_large ()
+{
+    struct hostlist *hl = hostlist_create ();
+    if (hl == NULL)
+        BAIL_OUT ("hostlist_create");
+    for (int i = 0; i < 8192; i++)
+        hostlist_append (hl, "host");
+    ok (hostlist_count (hl) == 8192,
+        "created hostlist with 8K hosts");
+    char *s = hostlist_encode (hl);
+    ok (s != NULL,
+        "hostlist_encode works");
+    ok (strlen (s) > 0,
+        "string length of result is %ld",
+        strlen (s));
+    free (s);
+    hostlist_destroy (hl);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_basic ();
+    test_encode_decode_basic ();
+    test_iteration_basic ();
+    test_encode_decode ();
+    test_invalid_decode ();
+    test_append ();
+    test_nth ();
+    test_find ();
+    test_find_hostname ();
+    test_delete ();
+    test_sortuniq ();
+    test_iteration ();
+    test_iteration_with_delete ();
+    test_encode_large ();
+
+    done_testing ();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libhostlist/test/hostname.c b/src/common/libhostlist/test/hostname.c
new file mode 100644
index 000000000000..b65a1bc01221
--- /dev/null
+++ b/src/common/libhostlist/test/hostname.c
@@ -0,0 +1,95 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libhostlist/hostname.h"
+
+struct hostname_test {
+    char *input;
+    char *prefix;
+    unsigned long num;
+    char *suffix;
+    int suffix_valid;
+    int suffix_width;
+};
+
+struct hostname_test hostname_tests[] = {
+    { "foo",      "foo",     0,   NULL,    0, 0 },
+    { "foo0",     "foo",     0,   "0",     1, 1 },
+    { "foo001",   "foo",     1,   "001",   1, 3 },
+    { "foo01bar", "foo01bar",0,   NULL,    0, 0 },
+    { " ",        NULL,      0,   NULL,    0, 0 },
+    { "bar[1-5]", NULL,      0,   NULL,    0, 0 },
+    { "bar,",     NULL,      0,   NULL,    0, 0 },
+    { NULL,       NULL,     -1,   NULL,    0, 0 },
+};
+
+int main (int argc, char *argv[])
+{
+    struct hostname_test *t;
+
+    plan (NO_PLAN);
+
+    ok (hostname_create_with_suffix ("testname", 4) == NULL,
+        "hostname_create_with_suffix() with invalid index returns EINVAL");
+
+    t = hostname_tests;
+    while (t && t->input != NULL) {
+        struct hostlist_hostname *hn = hostname_create (t->input);
+        if (t->prefix == NULL) {
+            /* Check expected failure */
+            ok (hn == NULL && errno == EINVAL,
+                "hostname_create (%s) fails with EINVAL", t->input);
+            t++;
+            continue;
+        }
+        if (!hn)
+            BAIL_OUT ("hostname_create (%s) failed!", t->input);
+        is (hn->prefix, t->prefix,
+            "input=%s: prefix=%s", t->input, hn->prefix);
+        ok (hn->num == t->num,
+            "input=%s: hn->num = %lu", t->input, hn->num);
+        if (t->suffix_valid) {
+            ok (hn->suffix != NULL,
+                "input=%s: hostname got valid suffix: %s",
+                t->input, hn->suffix);
+            ok (hostname_suffix_is_valid (hn),
+                "input=%s: hostname_suffix_is_valid returns true",
+                t->input);
+            ok (hostname_suffix_width (hn) == t->suffix_width,
+                "input=%s: hostname_suffix_width = %d",
+                t->input, hostname_suffix_width (hn));
+            is (hn->suffix, t->suffix,
+                "input=%s: suffixes match", t->input);
+        }
+        else {
+            ok (hostname_suffix_is_valid (hn) == 0,
+                "input=%s: hostname_suffix_is_valid returns false",
+                t->input);
+            ok (hostname_suffix_width (hn) == 0,
+                "input=%s: hostname_suffix_width returns 0",
+                t->input);
+        }
+        hostname_destroy (hn);
+        t++;
+    }
+
+    done_testing ();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libhostlist/test/hostrange.c b/src/common/libhostlist/test/hostrange.c
new file mode 100644
index 000000000000..a203eaad4488
--- /dev/null
+++ b/src/common/libhostlist/test/hostrange.c
@@ -0,0 +1,503 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#include "src/common/libhostlist/hostname.h"
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <errno.h>
+#include <stdbool.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libhostlist/hostrange.h"
+
+void test_create_single ()
+{
+    struct hostrange *hr;
+
+    ok (hostrange_create_single (NULL) == NULL && errno == EINVAL,
+        "hostrange_create_single (NULL) returns EINVAL");
+    hr = hostrange_create_single ("");
+    ok (hr != NULL,
+        "hostrange_create_single() empty string");
+    is (hr->prefix, "",
+        "hostrange_create_single() got expected prefix");
+    ok (hr->singlehost,
+        "hr->singlehost is true");
+    ok (hr->lo == hr->hi && hr->lo == 0 && hr->width == 0,
+        "hr->lo,hi,width have expected values");
+    hostrange_destroy (hr);
+
+    hr = hostrange_create_single ("hostname");
+    ok (hr != NULL,
+        "hostrange_create_single() works");
+    is (hr->prefix, "hostname",
+        "hostrange_create_single() got expected prefix");
+    ok (hr->singlehost,
+        "hr->singlehost is true");
+    ok (hr->lo == hr->hi && hr->lo == 0 && hr->width == 0,
+        "hr->lo,hi,width have expected values");
+    hostrange_destroy (hr);
+}
+
+void test_create ()
+{
+    struct hostrange *hr;
+
+    ok (hostrange_create (NULL, 0, 0, 0) == NULL && errno == EINVAL,
+        "hostrange_create (NULL, 0, 0, 0) returns EINVAL");
+    ok (hostrange_create ("foo", 1, 0, 0) == NULL && errno == EINVAL,
+        "hostrange_create ('foo', 1, 0, 0) returns EINVAL");
+    ok (hostrange_create ("foo", 0, 1, -1) == NULL && errno == EINVAL,
+        "hostrange_create ('foo', 0, 1, -1) returns EINVAL");
+
+    hr = hostrange_create ("foo", 0, 0, 0);
+    ok (hr != NULL,
+        "hostrange_create ('foo', 0, 0, 0) works");
+    is (hr->prefix, "foo",
+        "hostrange prefix is expected");
+    ok (hr->lo == 0 && hr->hi == 0 && hr->width == 0,
+        "hostrange components are expected values");
+    hostrange_destroy (hr);
+
+    hr = hostrange_create ("foo", 10, 20, 3);
+    ok (hr != NULL,
+        "hostrange_create ('foo', 10, 20, 3) works");
+    is (hr->prefix, "foo",
+        "hostrange prefix is expected");
+    ok (hr->lo == 10 && hr->hi == 20 && hr->width == 3,
+        "hostrange components are expected values");
+    hostrange_destroy (hr);
+
+}
+
+void test_copy ()
+{
+    struct hostrange *hr = hostrange_create ("foo", 0, 10, 3);
+    if (!hr)
+        BAIL_OUT ("hostrange_create failed");
+
+    struct hostrange *hr2 = hostrange_copy (hr);
+    if (!hr2)
+        BAIL_OUT ("hostrange_copy failed");
+
+    is (hr2->prefix, hr->prefix,
+        "hostrange_copy copies prefix");
+    ok (hr->hi == hr2->hi && hr->lo == hr2->lo && hr->width == hr2->width,
+        "hostrange_copy worked");
+
+    hostrange_destroy (hr);
+    hostrange_destroy (hr2);
+}
+
+void test_count ()
+{
+    struct hostrange *hr = hostrange_create ("foo", 0, 10, 3);
+    if (!hr)
+        BAIL_OUT ("hostrange_create failed");
+    ok (hostrange_count (hr) == 11,
+        "hostrange_count works with hostrange");
+    hostrange_destroy (hr);
+
+    hr = hostrange_create ("foo", 12, 12, 0);
+    if (!hr)
+        BAIL_OUT ("hostrange_create failed");
+    ok (hostrange_count (hr) == 1,
+        "hostrange_count works with foo12");
+    hostrange_destroy (hr);
+
+    hr = hostrange_create_single ("bar");
+    if (!hr)
+        BAIL_OUT ("hostrange_create_single failed");
+    ok (hostrange_count (hr) == 1,
+        "hostrange_count == 1 for singlehost");
+    hostrange_destroy (hr);
+
+    errno = 0;
+    ok (hostrange_count (NULL) == 0 && errno == EINVAL,
+        "hostrange_count (NULL) returns 0 with errno set");
+}
+
+void test_delete ()
+{
+    struct hostrange *hr;
+    struct hostrange *result;
+
+    hr = hostrange_create ("foo", 0, 10, 0);
+    if (!hr)
+        BAIL_OUT ("hostrange_create failed");
+
+    result = hostrange_delete_host (hr, 0);
+    ok (result == NULL && hr->lo == 1,
+        "hostrange_delete first host works");
+    result = hostrange_delete_host (hr, 10);
+    ok (result == NULL && hr->hi == 9,
+        "hostrange_delete first host works");
+    result = hostrange_delete_host (hr, 5);
+    ok (result != NULL,
+        "hostrange_delete host in middle of range returns result");
+    ok (result->lo == 6 && result->hi == 9,
+        "hostrange_delete returns corrects range in result");
+    ok (hr->lo == 1 && hr->hi == 4,
+        "hostrange_delete adjusts range of original hostrange");
+
+    hostrange_destroy (hr);
+    hostrange_destroy (result);
+}
+
+struct cmp_test {
+    struct hostrange h1;
+    struct hostrange h2;
+    int result;
+};
+
+struct cmp_test cmp_tests[] = {
+    { { "foo", 3, 0, 15, 0, 0 },
+      { "foo", 3, 1, 15, 0, 0 },
+      -1, },
+    { { "foo", 3, 0, 15, 0, 0 },
+      { "foo", 3, 0, 15, 0, 0 },
+      0, },
+    { { "foo", 3, 0, 15, 0, 0 },
+      { "foo", 3, 0,  0, 0, 1 },
+      1, },
+    { { "bar", 3, 0,  0, 0, 1 },
+      { "foo", 3, 0,  0, 0, 1 },
+      -1, },
+    { { "", 0,   0,  5, 0, 0 },
+      { "",  0,  5,  5, 0, 0 },
+      -1,
+    },
+    { { "", 0,   0,  5, 0, 0 },
+      { "", 0,   0,  5, 2, 0 },
+      -1,
+    },
+    { { "", 0,   12,  12, 0, 0 },
+      { "", 0,   15,  15, 0, 0 },
+      -1,
+    },
+    { { "", 0,   15,  15, 0, 0 },
+      { "", 0,   12,  12, 0, 0 },
+      1,
+    },
+    { { 0 }, { 0 }, 0 },
+};
+
+char *hrstr (struct hostrange *hr)
+{
+    int n = 0;
+    char buf [128];
+
+    n = snprintf (buf, sizeof (buf), "%s", hr->prefix);
+    if (!hr->singlehost) {
+        buf[n++] = '[';
+        n += hostrange_numstr (hr, sizeof (buf) - n, buf + n);
+        buf[n++] = ']';
+        buf[n] = '\0';
+    }
+    return strdup (buf);
+}
+
+void test_cmp ()
+{
+    struct cmp_test *t = cmp_tests;
+    while (t && t->h1.prefix) {
+        char *s1 = hrstr (&t->h1);
+        char *s2 = hrstr (&t->h2);
+        int result = hostrange_cmp (&t->h1, &t->h2);
+        ok (t->result == result,
+            "hostrange_cmp (%s, %s) = %d, expected %d",
+             s1, s2, result, t->result);
+        t++;
+        free (s1);
+        free (s2);
+    }
+}
+
+void test_join ()
+{
+    struct hostrange * hr1 = hostrange_create ("foo", 0, 10, 0);
+    struct hostrange * hr2 = hostrange_create ("foo", 5, 15, 0);
+    struct hostrange * hr3 = hostrange_create ("foo", 5, 15, 3);
+    struct hostrange * hr4 = hostrange_create_single ("foo");
+    struct hostrange * hr5 = hostrange_create_single ("bar");
+    struct hostrange * hr6 = hostrange_create ("foo", 16, 20, 0);
+
+    if (!hr1 || !hr2 || !hr3 || !hr4 || !hr5 || !hr6)
+        BAIL_OUT ("hostrange_create failed");
+
+    int rc = hostrange_join (hr2, hr3);
+    ok (rc < 0,
+        "hostrange_join fails when widths do not match (got rc==%d)", rc);
+    ok (hostrange_join (hr5, hr4) < 0,
+        "hostrange_join fails when prefixes do not match");
+    ok (hostrange_join (hr1, hr6) < 0,
+        "hostrange_join fails when ranges do not match");
+
+    ok (hostrange_join (hr4, hr4) == 1,
+        "hostrange_join identical hosts returns 1");
+
+    ok (hostrange_join (hr1, hr2) == 6,
+        "hostrange_join (foo[0-10], foo[5-15]) == 6");
+    ok (hr1->lo == 0 && hr1->hi == 15,
+        "hostrange joined hosts in first argument");
+    diag ("hr1=%s[%lu-%lu], hr2=%s[%lu-%lu]",
+          hr1->prefix, hr1->lo, hr1->hi,
+          hr2->prefix, hr2->lo, hr2->hi);
+
+    ok (hostrange_join (hr2, hr6) == 0,
+        "hostrange_join returns zero for perfect overlap");
+    ok (hr2->lo == 5 && hr2->hi == 20,
+        "hostrange joined hosts in first argument");
+    diag ("hr2=%s[%lu-%lu], hr6=%s[%lu-%lu]",
+          hr2->prefix, hr2->lo, hr2->hi,
+          hr6->prefix, hr6->lo, hr6->hi);
+
+    hostrange_destroy (hr1);
+    hostrange_destroy (hr2);
+    hostrange_destroy (hr3);
+    hostrange_destroy (hr4);
+    hostrange_destroy (hr5);
+    hostrange_destroy (hr6);
+}
+
+void test_intersect ()
+{
+    struct hostrange *hr1 = hostrange_create ("foo", 5, 10, 0);
+    struct hostrange *hr2 = hostrange_create ("foo", 9, 15, 0);
+    struct hostrange *hr3 = hostrange_create ("foo", 11, 15, 0);
+    struct hostrange *hr4 = hostrange_create ("foo", 8, 9, 0);
+    struct hostrange *hr5 = hostrange_create_single ("foo");
+    struct hostrange *result;
+
+    result = hostrange_intersect (hr1, hr2);
+    ok (result != NULL,
+        "hostrange_intersect works");
+    is (result->prefix, "foo",
+        "hostrange_intersect returned range with prefix");
+    ok (result->lo == 9 && result->hi == 10,
+        "hostrange_intersect got expected result");
+    hostrange_destroy (result);
+
+    ok (hostrange_intersect (hr1, hr3) == NULL,
+        "hostrange_intersect returns NULL for nonintersecting sets");
+
+    result = hostrange_intersect (hr1, hr4);
+    ok (result != NULL,
+        "hostrange_intersect works");
+    ok (hostrange_cmp (result, hr4) == 0,
+        "hostrange_intersect got expected result");
+    hostrange_destroy (result);
+
+    ok (hostrange_intersect (hr5, hr1) == NULL,
+        "hostrange_intersect returns NULL if one of the hosts is a singlehost");
+
+    hostrange_destroy (hr1);
+    hostrange_destroy (hr2);
+    hostrange_destroy (hr3);
+    hostrange_destroy (hr4);
+    hostrange_destroy (hr5);
+}
+
+struct within_test {
+    char *hostname;
+    char *prefix;
+    bool singlehost;
+    unsigned long lo;
+    unsigned long hi;
+    int width;
+    int result;
+};
+
+struct within_test within_tests[] = {
+    {
+        .hostname = "foo",
+        .prefix = "foo",
+        .singlehost = true,
+        .result = 0,
+    },
+    {
+        .hostname = "bar",
+        .prefix = "foo",
+        .singlehost = true,
+        .result = -1,
+    },
+    {
+        .hostname = "foo0",
+        .prefix = "foo",
+        .lo = 0,
+        .hi = 10,
+        .width = 1,
+        .result = 0,
+    },
+    {
+        .hostname = "foo5",
+        .prefix = "foo",
+        .lo = 0,
+        .hi = 10,
+        .width = 1,
+        .result = 5,
+    },
+    {
+        .hostname = "foo10",
+        .prefix = "foo",
+        .lo = 0,
+        .hi = 10,
+        .width = 1,
+        .result = 10 ,
+    },
+    {
+        .hostname = "foo01",
+        .prefix = "foo",
+        .lo = 0,
+        .hi = 10,
+        .width = 1,
+        .result = -1,
+    },
+    {
+        .hostname = "foo03",
+        .prefix = "foo0",
+        .lo = 0,
+        .hi = 5,
+        .width = 1,
+        .result = 3,
+    },
+    {
+        .hostname = "foo11",
+        .prefix = "foo",
+        .lo = 0,
+        .hi = 10,
+        .width = 1,
+        .result = -1,
+    },
+    {
+        .hostname = "bar5",
+        .prefix = "foo",
+        .lo = 0,
+        .hi = 10,
+        .width = 1,
+        .result = -1,
+    },
+    {
+        .hostname = "foo",
+        .prefix = "foo",
+        .lo = 0,
+        .hi = 10,
+        .width = 1,
+        .result = -1,
+    },
+    { .hostname = NULL },
+};
+
+void test_within ()
+{
+    struct within_test *t;
+
+    t = within_tests;
+    while (t && t->hostname) {
+        int result;
+        struct hostrange *hr;
+        struct stack_hostname hn_storage = {};
+        struct stack_hostname *hn = hostname_stack_create (&hn_storage, t->hostname);
+
+        if (hn == NULL)
+            BAIL_OUT ("hostname_create failed!");
+
+        if (t->singlehost)
+            hr = hostrange_create_single (t->prefix);
+        else
+            hr = hostrange_create (t->prefix, t->lo, t->hi, t->width);
+
+        result = hostrange_hn_within (hr, hn);
+        ok (result == t->result,
+            "hostrange_hn_within (%s[%lu-%lu], %s) returned %d",
+            t->prefix, t->lo, t->hi, t->hostname, result);
+
+        hostrange_destroy (hr);
+        t++;
+    }
+}
+
+struct tostring_test {
+    char *prefix;
+    bool singlehost;
+    unsigned long lo;
+    unsigned long hi;
+    int width;
+
+    ssize_t rc;
+    const char *tostring;
+
+    size_t numstr_rc;
+    const char *numstr;
+};
+
+void test_host_tostring ()
+{
+    char *host;
+    struct hostrange *hr = hostrange_create ("foo", 1, 10, 0);
+    if (!hr)
+        BAIL_OUT ("hostrange_create failed!");
+
+    ok (hostrange_host_tostring (NULL, 0) == NULL && errno == EINVAL,
+        "hostrange_host_tostring (NULL, 0) returns EINVAL");
+    ok (hostrange_host_tostring (hr, -1) == NULL && errno == EINVAL,
+        "hostrange_host_tostring (hr, -1) returns EINVAL");
+    ok (hostrange_host_tostring (hr, 42) == NULL && errno == ERANGE,
+        "hostrange_host_tostring (hr, 42) return ERANGE");
+
+    host = hostrange_host_tostring (hr, 0);
+    is (host, "foo1",
+        "hostrange_host_tostring (hr, 0) returns first host");
+    free (host);
+
+    host = hostrange_host_tostring (hr, hostrange_count (hr) - 1);
+    is (host, "foo10",
+        "hostrange_host_tostring (hr, count - 1) returns last host");
+    free (host);
+
+    host = hostrange_host_tostring (hr, 4);
+    is (host, "foo5",
+        "hostrange_host_tostring (hr, 4) returns expected host");
+    free (host);
+
+
+    hr->width = 3;
+    host = hostrange_host_tostring (hr, 4);
+    is (host, "foo005",
+        "hostrange_host_tostring (hr, 4) preserves width");
+    free (host);
+
+    hostrange_destroy (hr);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_create_single ();
+    test_create ();
+    test_copy ();
+    test_count ();
+    test_delete ();
+    test_cmp ();
+    test_join ();
+    test_intersect ();
+    test_within ();
+    test_host_tostring ();
+
+    done_testing ();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libhostlist/test/util.c b/src/common/libhostlist/test/util.c
new file mode 100644
index 000000000000..bc9d2caaf9a4
--- /dev/null
+++ b/src/common/libhostlist/test/util.c
@@ -0,0 +1,65 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libhostlist/util.h"
+
+struct width_test {
+    const char *descr;
+
+    unsigned long n;
+    int wn;
+
+    unsigned long m;
+    int wm;
+
+    int result;
+};
+
+struct width_test width_tests[] = {
+    { "single digit widths are equivalent",
+        0,     1,     1,     1,     1 },
+    { "single digit basic test",
+        0,     1,     100,   3,     1 },
+    { "multi digit basic test",
+        0,     3,     100,   1,     1 },
+    { "003 and 10 are not equivalent",
+        0,     3,     10,    1,     0 },
+    { 0 },
+};
+
+int main (int argc, char *argv[])
+{
+    struct width_test *t;
+
+    plan (NO_PLAN);
+
+    t = width_tests;
+    while (t && t->descr != NULL) {
+        int result = width_equiv (t->n, &t->wn, t->m, &t->wm);
+        ok (result == t->result,
+            "%s", t->descr);
+        if (result == 1)
+            ok (t->wn == t->wm,
+                "%s: wn = wm",
+                t->descr);
+        t++;
+    }
+
+    done_testing ();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libhostlist/util.c b/src/common/libhostlist/util.c
new file mode 100644
index 000000000000..b713761f2398
--- /dev/null
+++ b/src/common/libhostlist/util.c
@@ -0,0 +1,59 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif /* !HAVE_CONFIG_H */
+
+#include "util.h"
+
+static int zero_padded (unsigned long num, int width)
+{
+    int n = 1;
+    while (num /= 10L)
+        n++;
+    return width > n ? width - n : 0;
+}
+
+int width_equiv(unsigned long n, int *wn, unsigned long m, int *wm)
+{
+    int npad, nmpad, mpad, mnpad;
+
+    if (wn == wm)
+        return 1;
+
+    npad = zero_padded(n, *wn);
+    nmpad = zero_padded(n, *wm);
+    mpad = zero_padded(m, *wm);
+    mnpad = zero_padded(m, *wn);
+
+    if (npad != nmpad && mpad != mnpad)
+        return 0;
+
+    if (npad != nmpad) {
+        if (mpad == mnpad) {
+            *wm = *wn;
+            return 1;
+        } else
+            return 0;
+    } else {        /* mpad != mnpad */
+        if (npad == nmpad) {
+            *wn = *wm;
+            return 1;
+        } else
+            return 0;
+    }
+
+    /* not reached */
+}
+
+/*
+ * vi: tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libhostlist/util.h b/src/common/libhostlist/util.h
new file mode 100644
index 000000000000..cd42e50b0102
--- /dev/null
+++ b/src/common/libhostlist/util.h
@@ -0,0 +1,30 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_FLUX_HOSTLIST_UTIL_H
+#define HAVE_FLUX_HOSTLIST_UTIL_H
+
+/* Test whether two format `width' parameters are "equivalent"
+ * The width arguments "wn" and "wm" for integers "n" and "m"
+ * are equivalent if:
+ *
+ *  o  wn == wm  OR
+ *
+ *  o  applying the same format width (either wn or wm) to both of
+ *     'n' and 'm' will not change the zero padding of *either* 'm' nor 'n'.
+ *
+ *  If this function returns 1 (or true), the appropriate width value
+ *  (either 'wm' or 'wn') will have been adjusted such that both format
+ *  widths are equivalent.
+ */
+int width_equiv (unsigned long n, int *wn,
+                 unsigned long m, int *wm);
+
+#endif /* !HAVE_FLUX_HOSTLIST_UTIL_H */
diff --git a/src/common/libidset/Makefile.am b/src/common/libidset/Makefile.am
index 111e9e2b04bf..06cae1761590 100644
--- a/src/common/libidset/Makefile.am
+++ b/src/common/libidset/Makefile.am
@@ -6,10 +6,11 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux
 
 noinst_LTLIBRARIES = libidset.la
 fluxinclude_HEADERS = idset.h
@@ -17,27 +18,39 @@ libidset_la_SOURCES = idset.c \
 		      idset_private.h \
 		      idset_decode.c \
 		      idset_encode.c \
-		      idset_format.c
+		      idset_format.c \
+		      veb.c \
+		      veb.h
 
-libidset_la_CPPFLAGS = \
-	$(AM_CPPFLAGS)
-libidset_la_LDFLAGS = \
-	-avoid-version -module -shared -export-dynamic \
-	$(AM_LDFLAGS)
+EXTRA_DIST = veb_mach.c
 
+TESTS = test_veb.t \
+	test_idset.t
 
-TESTS = test_idset.t
-
-check_PROGRAMS = $(TESTS)
+check_PROGRAMS = \
+	$(TESTS) \
+	test_idsetutil
 
 TEST_EXTENSIONS = .t
 T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
        $(top_srcdir)/config/tap-driver.sh
 
+test_veb_t_SOURCES = test/veb.c
+test_veb_t_CPPFLAGS = $(AM_CPPFLAGS)
+test_veb_t_LDADD = \
+	$(top_builddir)/src/common/libidset/libidset.la \
+	$(top_builddir)/src/common/libutil/libutil.la \
+	$(top_builddir)/src/common/libtap/libtap.la
+
 test_idset_t_SOURCES = test/idset.c
 test_idset_t_CPPFLAGS = $(AM_CPPFLAGS)
 test_idset_t_LDADD = \
-	$(top_builddir)/src/common/libtap/libtap.la \
 	$(top_builddir)/src/common/libidset/libidset.la \
 	$(top_builddir)/src/common/libutil/libutil.la \
-	$(ZMQ_LIBS)
+	$(top_builddir)/src/common/libtap/libtap.la
+
+test_idsetutil_SOURCES = test/idsetutil.c
+test_idsetutil_CPPFLAGS = $(AM_CPPFLAGS)
+test_idsetutil_LDADD = \
+	$(top_builddir)/src/common/libidset/libidset.la \
+	$(top_builddir)/src/common/libutil/libutil.la
diff --git a/src/common/libidset/idset.c b/src/common/libidset/idset.c
index 070dc3b22b46..a7cb43231702 100644
--- a/src/common/libidset/idset.c
+++ b/src/common/libidset/idset.c
@@ -34,21 +34,30 @@ int validate_idset_flags (int flags, int allowed)
 struct idset *idset_create (size_t size, int flags)
 {
     struct idset *idset;
+    int valid_flags = IDSET_FLAG_AUTOGROW
+                    | IDSET_FLAG_INITFULL
+                    | IDSET_FLAG_COUNT_LAZY;
 
-    if (validate_idset_flags (flags, IDSET_FLAG_AUTOGROW) < 0)
+    if (validate_idset_flags (flags, valid_flags) < 0)
         return NULL;
     if (size == 0)
         size = IDSET_DEFAULT_SIZE;
     if (!(idset = malloc (sizeof (*idset))))
         return NULL;
-    idset->T = vebnew (size, 0);
+    if ((flags & IDSET_FLAG_INITFULL))
+        idset->T = vebnew (size, 1);
+    else
+        idset->T = vebnew (size, 0);
     if (!idset->T.D) {
         free (idset);
         errno = ENOMEM;
         return NULL;
     }
     idset->flags = flags;
-    idset->count = 0;
+    if ((flags & IDSET_FLAG_INITFULL))
+        idset->count = size;
+    else
+        idset->count = 0;
     return idset;
 }
 
@@ -62,6 +71,11 @@ void idset_destroy (struct idset *idset)
     }
 }
 
+size_t idset_universe_size (const struct idset *idset)
+{
+    return idset ? idset->T.M : 0;
+}
+
 static Veb vebdup (Veb T)
 {
     size_t size = vebsize (T.M);
@@ -74,17 +88,13 @@ static Veb vebdup (Veb T)
     return cpy;
 }
 
-struct idset *idset_copy (const struct idset *idset)
+static struct idset *idset_copy_flags (const struct idset *idset, int flags)
 {
     struct idset *cpy;
 
-    if (!idset) {
-        errno = EINVAL;
-        return NULL;
-    }
     if (!(cpy = malloc (sizeof (*idset))))
         return NULL;
-    cpy->flags = idset->flags;
+    cpy->flags = flags;
     cpy->T = vebdup (idset->T);
     if (!cpy->T.D) {
         idset_destroy (cpy);
@@ -94,6 +104,15 @@ struct idset *idset_copy (const struct idset *idset)
     return cpy;
 }
 
+struct idset *idset_copy (const struct idset *idset)
+{
+    if (!idset) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return idset_copy_flags (idset, idset->flags);
+}
+
 static bool valid_id (unsigned int id)
 {
     if (id == UINT_MAX || id == IDSET_INVALID_ID)
@@ -101,7 +120,7 @@ static bool valid_id (unsigned int id)
     return true;
 }
 
-/* Double idset size until it has at least 'size' slots.
+/* Double the idset universe size until it is at least 'size'.
  * Return 0 on success, -1 on failure with errno == ENOMEM.
  */
 static int idset_grow (struct idset *idset, size_t size)
@@ -127,27 +146,71 @@ static int idset_grow (struct idset *idset, size_t size)
             vebput (T, id);
             id = vebsucc (idset->T, id + 1);
         }
+        if ((idset->flags & IDSET_FLAG_INITFULL)) {
+            for (id = idset->T.M; id < newsize; id++)
+                vebput (T, id);
+            idset->count += (newsize - idset->T.M);
+        }
         free (idset->T.D);
         idset->T = T;
     }
     return 0;
 }
 
-/* Wrapper for vebput() which increments idset count if needed
+/* Helper to avoid costly idset_test() operation in idset_put()/idset_del()
+ * in some cases that may commonly arise in idset_encode(), for example.
+ * This function runs in constant time.  Return true if id is definitely not
+ * in set.  A false result is indeterminate.
+ */
+static bool nonmember_fast (struct idset *idset, unsigned int id)
+{
+    unsigned int last = idset_last (idset);
+    if (last == IDSET_INVALID_ID || id > last)
+        return true;
+    unsigned int first = idset_first (idset);
+    if (first == IDSET_INVALID_ID || id < first)
+        return true;
+    return false;
+}
+
+/* Wrapper for vebput() which increments idset count.
+ * The operation is skipped if id is already in the set.
  */
 static void idset_put (struct idset *idset, unsigned int id)
 {
-    if (!idset_test (idset, id))
+    if ((idset->flags & IDSET_FLAG_COUNT_LAZY)
+        || nonmember_fast (idset, id)
+        || !idset_test (idset, id)) {
         idset->count++;
+        vebput (idset->T, id);
+    }
+}
+
+/* Call this variant if id is known to NOT be in the set
+ */
+static void idset_put_nocheck (struct idset *idset, unsigned int id)
+{
+    idset->count++;
     vebput (idset->T, id);
 }
 
-/* Wrapper for vebdel() which decrements idset count if needed
+/* Wrapper for vebdel() which decrements idset count.
+ * The operation is skipped if id is not in the set.
  */
 static void idset_del (struct idset *idset, unsigned int id)
 {
-    if (idset_test (idset, id))
+    if ((idset->flags & IDSET_FLAG_COUNT_LAZY)
+        || (!nonmember_fast (idset, id) && idset_test (idset, id))) {
         idset->count--;
+        vebdel (idset->T, id);
+    }
+}
+
+/* Call this variant if id is known to be IN the set
+ */
+static void idset_del_nocheck (struct idset *idset, unsigned int id)
+{
+    idset->count--;
     vebdel (idset->T, id);
 }
 
@@ -157,9 +220,19 @@ int idset_set (struct idset *idset, unsigned int id)
         errno = EINVAL;
         return -1;
     }
-    if (idset_grow (idset, id + 1) < 0)
-        return -1;
-    idset_put (idset, id);
+    if (id >= idset_universe_size (idset)) {
+        /* N.B. we do not try to grow the idset to accommodate out of range ids
+         * when the operation is 'set' and IDSET_FLAG_INITFULL is set.
+         * Treat it as a successful no-op.
+         */
+        if ((idset->flags & IDSET_FLAG_INITFULL))
+            return 0;
+        if (idset_grow (idset, id + 1) < 0)
+            return -1;
+        idset_put_nocheck (idset, id);
+    }
+    else
+        idset_put (idset, id);
     return 0;
 }
 
@@ -181,10 +254,22 @@ int idset_range_set (struct idset *idset, unsigned int lo, unsigned int hi)
         return -1;
     }
     normalize_range (&lo, &hi);
-    if (idset_grow (idset, hi + 1) < 0)
-        return -1;
-    for (id = lo; id <= hi; id++)
-        idset_put (idset, id);
+
+    // see IDSET_FLAG_INITFULL note in idset_set()
+    size_t oldsize = idset_universe_size (idset);
+    if (!(idset->flags & IDSET_FLAG_INITFULL)) {
+        if (idset_grow (idset, hi + 1) < 0)
+            return -1;
+    }
+    for (id = lo; id <= hi; id++) {
+        if (id >= oldsize) {
+            if ((idset->flags & IDSET_FLAG_INITFULL))
+                return 0;
+            idset_put_nocheck (idset, id);
+        }
+        else
+            idset_put (idset, id);
+    }
     return 0;
 }
 
@@ -194,7 +279,19 @@ int idset_clear (struct idset *idset, unsigned int id)
         errno = EINVAL;
         return -1;
     }
-    idset_del (idset, id);
+    if (id >= idset_universe_size (idset)) {
+        /* N.B. we do not try to grow the idset to accommodate out of range ids
+         * when the operation is 'clear' and IDSET_FLAG_INITFULL is NOT set.
+         * Treat this as a successful no-op.
+         */
+        if (!(idset->flags & IDSET_FLAG_INITFULL))
+            return 0;
+        if (idset_grow (idset, id + 1) < 0)
+            return -1;
+        idset_del_nocheck (idset, id);
+    }
+    else
+        idset_del (idset, id);
     return 0;
 }
 
@@ -207,8 +304,21 @@ int idset_range_clear (struct idset *idset, unsigned int lo, unsigned int hi)
         return -1;
     }
     normalize_range (&lo, &hi);
-    for (id = lo; id <= hi && id < idset->T.M; id++)
-        idset_del (idset, id);
+    // see IDSET_FLAG_INITFULL note in idset_clear()
+    size_t oldsize = idset_universe_size (idset);
+    if ((idset->flags & IDSET_FLAG_INITFULL)) {
+        if (idset_grow (idset, hi + 1) < 0)
+            return -1;
+    }
+    for (id = lo; id <= hi; id++) {
+        if (id >= oldsize) {
+            if (!(idset->flags & IDSET_FLAG_INITFULL))
+                return 0;
+            idset_del_nocheck (idset, id);
+        }
+        else
+            idset_del (idset, id);
+    }
     return 0;
 }
 
@@ -232,12 +342,12 @@ unsigned int idset_first (const struct idset *idset)
 }
 
 
-unsigned int idset_next (const struct idset *idset, unsigned int prev)
+unsigned int idset_next (const struct idset *idset, unsigned int id)
 {
     unsigned int next = IDSET_INVALID_ID;
 
     if (idset) {
-        next = vebsucc (idset->T, prev + 1);
+        next = vebsucc (idset->T, id + 1);
         if (next == idset->T.M)
             next = IDSET_INVALID_ID;
     }
@@ -256,22 +366,64 @@ unsigned int idset_last (const struct idset *idset)
     return last;
 }
 
+unsigned int idset_prev (const struct idset *idset, unsigned int id)
+{
+    unsigned int next = IDSET_INVALID_ID;
+
+    if (idset) {
+        next = vebpred (idset->T, id - 1);
+        if (next == idset->T.M)
+            next = IDSET_INVALID_ID;
+    }
+    return next;
+}
+
 size_t idset_count (const struct idset *idset)
 {
     if (!idset)
         return 0;
-    return idset->count;
+    if (!(idset->flags & IDSET_FLAG_COUNT_LAZY))
+        return idset->count;
+
+    /* IDSET_FLAG_COUNT_LAZY was set, causing set/clear operations to ignore
+     * safeguards that kept idset->count accurate.  Pay now by iterating.
+     */
+    unsigned int id;
+    size_t count = 0;
+
+    id = idset_first (idset);
+    while (id != IDSET_INVALID_ID) {
+        count++;
+        id = idset_next (idset, id);
+    }
+    return count;
+}
+
+bool idset_empty (const struct idset *idset)
+{
+    if (!idset || vebsucc (idset->T, 0) == idset->T.M)
+        return true;
+    return false;
 }
 
 bool idset_equal (const struct idset *idset1,
                   const struct idset *idset2)
 {
     unsigned int id;
+    bool count_checked = false;
 
     if (!idset1 || !idset2)
         return false;
-    if (idset_count (idset1) != idset_count (idset2))
-        return false;
+
+    /* As an optimization, declare the sets unequal if counts differ.
+     * If lazy counts are used, this is potentially slow, so skip.
+     */
+    if (!(idset1->flags & IDSET_FLAG_COUNT_LAZY)
+        && !(idset2->flags & IDSET_FLAG_COUNT_LAZY)) {
+        if (idset_count (idset1) != idset_count (idset2))
+            return false;
+        count_checked = true;
+    }
 
     id = vebsucc (idset1->T, 0);
     while (id < idset1->T.M) {
@@ -279,6 +431,13 @@ bool idset_equal (const struct idset *idset1,
             return false; // id in idset1 not set in idset2
         id = vebsucc (idset1->T, id + 1);
     }
+
+    /* No need to iterate idset2 if counts were equal and all ids in idset1
+     * were found in idset2.
+     */
+    if (count_checked)
+        return true;
+
     id = vebsucc (idset2->T, 0);
     while (id < idset2->T.M) {
         if (vebsucc (idset1->T, id) != id)
@@ -288,6 +447,191 @@ bool idset_equal (const struct idset *idset1,
     return true;
 }
 
+bool idset_has_intersection (const struct idset *a, const struct idset *b)
+{
+    if (a && b) {
+        unsigned int id;
+
+        /*  If there isn't a penalty for idset_count(3), then ensure
+         *  we're going to iterate the smaller of the provided idsets
+         *  for efficiency:
+         */
+        if (!(a->flags & IDSET_FLAG_COUNT_LAZY)
+            && !(b->flags & IDSET_FLAG_COUNT_LAZY)
+            && idset_count (a) < idset_count (b)) {
+            const struct idset *tmp = a;
+            a = b;
+            b = tmp;
+        }
+
+        id = idset_first (b);
+        while (id != IDSET_INVALID_ID) {
+            if (idset_test (a, id))
+                return true;
+            id = idset_next (b, id);
+        }
+    }
+    return false;
+}
+
+int idset_add (struct idset *a, const struct idset *b)
+{
+    if (!a) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (b) {
+        unsigned int id;
+        id = idset_first (b);
+        while (id != IDSET_INVALID_ID) {
+            if (idset_set (a, id) < 0)
+                return -1;
+            id = idset_next (b, id);
+        }
+    }
+    return 0;
+}
+
+struct idset *idset_union (const struct idset *a, const struct idset *b)
+{
+    struct idset *result;
+
+    if (!a) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(result = idset_copy_flags (a, IDSET_FLAG_AUTOGROW)))
+        return NULL;
+    if (idset_add (result, b) < 0) {
+        idset_destroy (result);
+        return NULL;
+    }
+    return result;
+}
+
+int idset_subtract (struct idset *a, const struct idset *b)
+{
+    if (!a) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (b) {
+        unsigned int id;
+
+        id = idset_first (b);
+        while (id != IDSET_INVALID_ID) {
+            if (idset_clear (a, id) < 0)
+                return -1;
+            id = idset_next (b, id);
+        }
+    }
+    return 0;
+}
+
+struct idset *idset_difference (const struct idset *a, const struct idset *b)
+{
+    struct idset *result;
+
+    if (!a) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(result = idset_copy (a)))
+        return NULL;
+    if (idset_subtract (result, b) < 0) {
+        idset_destroy (result);
+        return NULL;
+    }
+    return result;
+}
+
+struct idset *idset_intersect (const struct idset *a, const struct idset *b)
+{
+    struct idset *result;
+    unsigned int id;
+
+    if (!a || !b) {
+        errno = EINVAL;
+        return NULL;
+    }
+    /*  If there isn't a penalty for idset_count(3), then ensure
+     *  we start with the smaller of the two idsets for efficiency:
+     */
+    if (!(a->flags & IDSET_FLAG_COUNT_LAZY)
+        && !(b->flags & IDSET_FLAG_COUNT_LAZY)
+        && idset_count (b) < idset_count (a)) {
+        const struct idset *tmp = a;
+        a = b;
+        b = tmp;
+    }
+
+    if (!(result = idset_copy (a)))
+        return NULL;
+    id = idset_first (a);
+    while (id != IDSET_INVALID_ID) {
+        if (!idset_test (b, id) && idset_clear (result, id) < 0) {
+            idset_destroy (result);
+            return NULL;
+        }
+        id = idset_next (a, id);
+    }
+    return result;
+}
+
+/* Find the next available id.  If there isn't one, try to grow the set.
+ * The grow attempt will fail if IDSET_FLAG_AUTOGROW is not set.
+ * Finally call vebdel() to take the id out of the set and return it.
+ */
+int idset_alloc (struct idset *idset, unsigned int *val)
+{
+    unsigned int id;
+
+    if (!idset || !(idset->flags & IDSET_FLAG_INITFULL) || !val) {
+        errno = EINVAL;
+        return -1;
+    }
+    id = idset_first (idset);
+    if (id == IDSET_INVALID_ID) {
+        id = idset_universe_size (idset);
+        if (idset_grow (idset, id + 1) < 0)
+            return -1;
+    }
+    // code above ensures that id is a member of idset
+    idset_del_nocheck (idset, id);
+    *val = id;
+    return 0;
+}
+
+/* Return an id to the set, ignoring invalid or out of range ones.
+ * This does not catch double-frees.
+ */
+void idset_free (struct idset *idset, unsigned int val)
+{
+    if (!idset || !(idset->flags & IDSET_FLAG_INITFULL))
+        return;
+    idset_put (idset, val);
+}
+
+/* Same as above but fail if the id is already in the set.
+ */
+int idset_free_check (struct idset *idset, unsigned int val)
+{
+    if (!idset
+        || !(idset->flags & IDSET_FLAG_INITFULL)
+        || !valid_id (val)
+        || val >= idset_universe_size (idset)) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (idset_test (idset, val)) {
+        errno = EEXIST;
+        return -1;
+    }
+    // code above ensures that id is NOT a member of idset
+    idset_put_nocheck (idset, val);
+    return 0;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libidset/idset.h b/src/common/libidset/idset.h
index ffb9a67878ad..599f67dce715 100644
--- a/src/common/libidset/idset.h
+++ b/src/common/libidset/idset.h
@@ -19,16 +19,28 @@
 #include <sys/param.h>
 #include <stdbool.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 enum idset_flags {
-    IDSET_FLAG_AUTOGROW = 1, // allow idset size to automatically grow
+    IDSET_FLAG_AUTOGROW = 1, // allow idset universe size to automatically grow
     IDSET_FLAG_BRACKETS = 2, // encode non-singleton idset with brackets
     IDSET_FLAG_RANGE = 4,    // encode with ranges ("2,3,4,8" -> "2-4,8")
+    IDSET_FLAG_INITFULL = 8, // initilize/grow idset with all ids set
+    IDSET_FLAG_COUNT_LAZY = 16, // disable running count, which speeds up
+                             //  idset_set/clear, but slows down idset_count()
 };
 
+typedef struct {
+    char text[160];
+} idset_error_t;
+
 #define IDSET_INVALID_ID    (UINT_MAX - 1)
 
 /* Create/destroy an idset.
- * Set the initial size to 'size' (0 means implementation uses a default size).
+ * Set the initial universe size to 'size' (0 means implementation uses a
+ * default size).
  * If 'flags' includes IDSET_FLAG_AUTOGROW, the idset is resized to fit if
  * an id >= size is set.
  * Returns idset on success, or NULL on failure with errno set.
@@ -36,6 +48,11 @@ enum idset_flags {
 struct idset *idset_create (size_t size, int flags);
 void idset_destroy (struct idset *idset);
 
+/* Get the current universe size of the idset.
+ * A set with a universe size of N holds ids from 0 to N - 1.
+ */
+size_t idset_universe_size (const struct idset *idset);
+
 /* Make an exact duplicate of idset.
  * Returns copy on success, NULL on failure with errno set.
  */
@@ -52,10 +69,68 @@ char *idset_encode (const struct idset *idset, int flags);
  */
 struct idset *idset_decode (const char *s);
 
+#ifndef FLUX_DEPRECATED
+#define FLUX_DEPRECATED(...) __VA_ARGS__ __attribute__((deprecated))
+#endif
+
 /* Decode 'len' chars of string 's' to an idset.
  * Returns idset on success, or NULL on failure with errno set.
  */
-struct idset *idset_ndecode (const char *s, size_t len);
+FLUX_DEPRECATED(
+struct idset *idset_ndecode (const char *s, size_t len)
+);
+
+/* Parse 'len' chars of string 's' to an idset created with 'size' and 'flags'.
+ * If len < 0, strlen (s) is used.
+ * If size = 0, the implementation's default size is used.
+ * If size < 0, the idset size is made to fit exactly (fail on empty set*).
+ * On failure, a human readable error is placed in 'error'.
+ * [*] Exception: if IDSET_FLAG_AUTOGROW, empty set is created with size of 1.
+ */
+struct idset *idset_decode_ex (const char *s,
+                               ssize_t len,
+                               ssize_t size,
+                               int flags,
+                               idset_error_t *error);
+
+/* Parse 'len' chars of string 's' to determine if it is the empty set.
+ * Return false on parse error.  This may be useful when representing the
+ * empty set as a NULL.
+ */
+bool idset_decode_empty (const char *s, ssize_t len);
+
+/* Parse 'len' chars of string 's' to determine the id count and maximum id
+ * without creating an idset.  Either of the return parameters may be NULL.
+ * If len < 0, strlen (s) is used.
+ * If the set is empty, maxid is IDSET_INVALID_ID.
+ * On failure, a human readable error is placed in 'error'.
+ */
+int idset_decode_info (const char *s,
+                       ssize_t len,
+                       size_t *count,
+                       unsigned int *maxid,
+                       idset_error_t *error);
+
+/* Parse 'len' chars of string 's' and add it to 'idset',
+ * without creating an intermediate idset.
+ * If len < 0, strlen (s) is used.
+ * On failure, a human readable error is placed in 'error'.
+ */
+int idset_decode_add (struct idset *idset,
+                      const char *s,
+                      ssize_t len,
+                      idset_error_t *error);
+
+/* Parse 'len' chars of string 's' and subtract it from 'idset',
+ * without creating an intermediate idset.
+ * If len < 0, strlen (s) is used.
+ * On failure, a human readable error is placed in 'error'.
+ */
+int idset_decode_subtract (struct idset *idset,
+                           const char *s,
+                           ssize_t len,
+                           idset_error_t *error);
+
 
 /* Add id (or range [lo-hi]) to idset.
  * Return 0 on success, -1 on failure with errno set.
@@ -80,44 +155,81 @@ bool idset_test (const struct idset *idset, unsigned int id);
  */
 unsigned int idset_first (const struct idset *idset);
 
-/* Return the next id after 'prev' in the idset.
- * Returns IDSET_INVALID_ID if prev is the last id.
+/* Return the next id after 'id' in the idset.
+ * Returns IDSET_INVALID_ID if 'id' is the last id.
  */
-unsigned int idset_next (const struct idset *idset, unsigned int prev);
+unsigned int idset_next (const struct idset *idset, unsigned int id);
 
 /* Returns the last id in the idset.
  * Returns IDSET_INVALID_ID if the idset is empty.
  */
 unsigned int idset_last (const struct idset *idset);
 
+/* Return the previous id before 'id' in the idset.
+ * Returns IDSET_INVALID_ID if 'id' is the first id.
+ */
+unsigned int idset_prev (const struct idset *idset, unsigned int id);
+
 /* Return the number of id's in idset.
  * If idset is invalid, return 0.
  */
 size_t idset_count (const struct idset *idset);
 
+/* Return true if idset is empty.
+ * If idset is invalid, return true.
+ */
+bool idset_empty (const struct idset *idset);
+
 /* Return true if the two idsets set1, set2 are equal, i.e. they both
  * have the same integers set.
  */
-bool idset_equal (const struct idset *set1, const struct idset *set2);
+bool idset_equal (const struct idset *a, const struct idset *);
 
-/* Expand bracketed idset string(s) in 's', calling 'fun()' for each
- * expanded string.  'fun()' should return 0 on success, or -1 on failure
- * with errno set.  A fun() failure causes idset_format_map () to immediately
- * return -1.  'fun()' may may halt iteration without triggering an error
- * by setting *stop = true.
- *
- * idset_format_map () returns the number of times the map function was called
- * (including the stopping one, if any), or -1 on failure with errno set.
- *
- * This function recursively expands multiple bracketed idset strings from
- * left to right, so for example, "r[0-1]n[0-1]" expands to "r0n0", "r0n1",
- * "r1n0", "r1n1".
- *
+/* Return a new set = a union b on success,
+ * NULL on failure with errno set.
+ */
+struct idset *idset_union (const struct idset *a, const struct idset *b);
+
+/* Perform a += b.
+ * Return 0 on success, -1 on failure with errno set.
+ */
+int idset_add (struct idset *a, const struct idset *b);
+
+/* Return a new set = a - b on success,
+ * NULL on failure with errno set.
+ */
+struct idset *idset_difference (const struct idset *a, const struct idset *b);
+
+/* Perform a -= b.
+ * Return 0 on success, -1 on failure with errno set.
  */
-typedef int (*idset_format_map_f)(const char *s, bool *stop, void *arg);
+int idset_subtract (struct idset *a, const struct idset *b);
 
-int idset_format_map (const char *s, idset_format_map_f fun, void *arg);
+/* Remove all elements from set x.
+ */
+#define idset_clear_all(x)  idset_subtract ((x), (x))
+
+/* Return a new set = a intersect b on success,
+ * NULL on failure with errno set.
+ */
+struct idset *idset_intersect (const struct idset *a, const struct idset *b);
+
+/* Return true if set a intersects set b.
+ */
+bool idset_has_intersection (const struct idset *a, const struct idset *b);
+
+/* Convenience interfaces for using idset as an "integer allocator".
+ * The set must be created with IDSET_FLAG_INITFULL.
+ * idset_free() doesn't check for double frees, but idset_free_check() does.
+ * It returns -1 and sets errno to EEXIST when that happens.
+ */
+int idset_alloc (struct idset *idset, unsigned int *val);
+void idset_free (struct idset *idset, unsigned int val);
+int idset_free_check (struct idset *idset, unsigned int val);
 
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* !FLUX_IDSET_H */
 
diff --git a/src/common/libidset/idset_decode.c b/src/common/libidset/idset_decode.c
index bc6412ebc7d1..5c19a3ffbf8b 100644
--- a/src/common/libidset/idset_decode.c
+++ b/src/common/libidset/idset_decode.c
@@ -18,95 +18,326 @@
 #include <stdio.h>
 #include <stdarg.h>
 #include <stdbool.h>
+#include <ctype.h>
 
 #include "idset.h"
 #include "idset_private.h"
 
+static int verrprintf (idset_error_t *errp, const char *fmt, va_list ap)
+{
+    if (errp) {
+        int saved_errno = errno;
+        memset (errp->text, 0, sizeof (errp->text));
+        if (fmt) {
+            int n;
+            n = vsnprintf (errp->text, sizeof (errp->text), fmt, ap);
+            if (n > sizeof (errp->text))
+                errp->text[sizeof (errp->text) - 2] = '+';
+        }
+        errno = saved_errno;
+    }
+    return -1;
+}
+
+static int errprintf (idset_error_t *errp, const char *fmt, ...)
+{
+    va_list ap;
+    va_start (ap, fmt);
+    verrprintf (errp, fmt, ap);
+    va_end (ap);
+    return -1;
+}
+
+/* strtoul() with result parameter, assumed base=10.
+ * Fail if no digits, leading non-digits, or leading zero.
+ * Returns 0 on success, -1 on failure.
+ */
+static int strtoul_check (const char *s, char **endptr, unsigned long *result)
+{
+    unsigned long n;
+    char *ep;
+
+    errno = 0;
+    n = strtoul (s, &ep, 10);
+    if (errno != 0)
+        return -1;
+    if (ep == s) // no digits
+        return -1;
+    if (!isdigit (*s))
+        return -1;
+    if (*s == '0' && ep - s > 1) // leading zero (RFC 22)
+        return -1;
+    *result = n;
+    if (endptr)
+        *endptr = ep;
+    return 0;
+}
+
 static int parse_range (const char *s, unsigned int *hi, unsigned int *lo)
 {
     char *endptr;
     unsigned int h, l;
     unsigned long n;
 
-    n = strtoul (s, &endptr, 10);
-    if (n >= UINT_MAX || endptr == s || (*endptr != '\0' && *endptr != '-'))
+    if (strtoul_check (s, &endptr, &n) < 0)
+        return -1;
+    if (*endptr != '\0' && *endptr != '-')
         return -1;
     h = l = n;
     if (*endptr == '-') {
         s = endptr + 1;
-        n = strtoul (s, &endptr, 10);
-        if (n >= UINT_MAX || endptr == s || *endptr != '\0')
+        if (strtoul_check (s, &endptr, &n) < 0)
+            return -1;
+        if (*endptr != '\0')
+            return -1;
+        if (n <= l)
             return -1;
         h = n;
     }
-    if (h >= l) {
-        *hi = h;
-        *lo = l;
+    *hi = h;
+    *lo = l;
+    return 0;
+}
+
+/* Append one element (single digit or range) to 'idset'.
+ * Each element must ascend from the previous ones.
+ * On the first call set *maxid = IDSET_INVALID_ID and *count = 0.
+ * Each call builds the max id value and member count in those values.
+ * On success return 0.  On failure, return -1 with errno and error set.
+ */
+static int append_element (struct idset *idset,
+                           const char *s,
+                           size_t *count,
+                           unsigned int *maxid,
+                           idset_error_t *error)
+{
+    unsigned int hi, lo;
+
+    if (parse_range (s, &hi, &lo) < 0) {
+        errprintf (error, "error parsing range '%s'", s);
+        goto inval;
     }
-    else {
-        *hi = l;
-        *lo = h;
+    if (*maxid != IDSET_INVALID_ID && lo <= *maxid) {
+        errprintf (error, "range '%s' is out of order", s);
+        goto inval;
     }
+    if (idset && idset_range_set (idset, lo, hi) < 0) {
+        errprintf (error, "error appending '%s': %s", s, strerror (errno));
+        goto error;
+    }
+    *count += hi - lo + 1;
+    *maxid = hi;
     return 0;
+inval:
+    errno = EINVAL;
+error:
+    return -1;
 }
 
-static char *trim_brackets (char *s)
+/* like append_element() except remove one element from 'idset' and
+ * don't maintain the count.  We still need maxid to enforce range order.
+ */
+static int remove_element (struct idset *idset,
+                           const char *s,
+                           unsigned int *maxid,
+                           idset_error_t *error)
 {
-    char *p = s;
-    if (*p == '[')
-        p++;
-    size_t len = strlen (p);
-    if (len > 0 && p[len - 1] == ']')
-        p[len - 1] = '\0';
-    return p;
+    unsigned int hi, lo;
+
+    if (parse_range (s, &hi, &lo) < 0) {
+        errprintf (error, "error parsing range '%s'", s);
+        goto inval;
+    }
+    if (*maxid != IDSET_INVALID_ID && lo <= *maxid) {
+        errprintf (error, "range '%s' is out of order", s);
+        goto inval;
+    }
+    if (idset && idset_range_clear (idset, lo, hi) < 0) {
+        errprintf (error, "error clearing '%s': %s", s, strerror (errno));
+        goto error;
+    }
+    *maxid = hi;
+    return 0;
+inval:
+    errno = EINVAL;
+error:
+    return -1;
 }
 
-struct idset *idset_ndecode (const char *str, size_t size)
+/* Trim brackets by dropping a \0 on the tail of 's' and returning a starting
+ * pointer within 's'.  On failure return NULL with errno and error set.
+ */
+static char *trim_brackets (char *s, idset_error_t *error)
 {
-    struct idset *idset;
-    char *cpy = NULL;
-    char *tok, *saveptr, *a1;
-    int saved_errno;
+    int len = strlen (s);
 
+    if (len >= 2 && s[0] == '[' && s[len - 1] == ']') {
+        s[len - 1] = '\0';
+        s++;
+    }
+    if (strchr (s, '[') || strchr (s, ']')) {
+        errprintf (error, "mismatched or nested brackets");
+        errno = EINVAL;
+        return NULL;
+    }
+    return s;
+}
+
+static char *dup_input (const char *str, ssize_t len, idset_error_t *error)
+{
+    char *cpy;
     if (!str) {
+        errprintf (error, "input is NULL");
         errno = EINVAL;
         return NULL;
     }
-    if (!(idset = idset_create (0, IDSET_FLAG_AUTOGROW)))
+    if (len < 0)
+        len = strlen (str);
+    if (!(cpy = strndup (str, len))) {
+        errprintf (error, "out of memory");
         return NULL;
-    if (!(cpy = strndup (str, size)))
+    }
+    return cpy;
+}
+
+/* Decode 'str' (up to 'len') and add it to 'idset'.
+ * If 'idset' is NULL, this is a parsing pass to determine count/maxid only.
+ * If 'len' < 0, use strlen (str).
+ * If non-NULL, return member count in *countp, max id in *maxidp.
+ * On success return 0.  On failure, return -1 with errno and error set.
+ */
+static int decode_and_set_with_info (struct idset *idset,
+                                     const char *str,
+                                     ssize_t len,
+                                     size_t *countp,
+                                     unsigned int *maxidp,
+                                     idset_error_t *error)
+{
+    char *cpy;
+    char *tok, *saveptr, *a1;
+    int saved_errno;
+    unsigned int maxid = IDSET_INVALID_ID;
+    size_t count = 0;
+
+    if (!(cpy = dup_input (str, len, error)))
+        return -1;
+    if (!(a1 = trim_brackets (cpy, error)))
         goto error;
-    a1 = trim_brackets (cpy);
     saveptr = NULL;
     while ((tok = strtok_r (a1, ",", &saveptr))) {
-        unsigned int hi, lo, i;
-        if (parse_range (tok, &hi, &lo) < 0)
-            goto inval;
-        /* Count backwards so that idset_set() can grow the
-         * idset to the maximum size on the first access,
-         * rather than possibly doing it multiple times.
-         */
-        for (i = hi; i >= lo && i != UINT_MAX; i--) {
-            if (idset_set (idset, i) < 0)
-                goto error;
-        }
+        if (append_element (idset, tok, &count, &maxid, error) < 0)
+            goto error;
         a1 = NULL;
     }
     free (cpy);
-    return idset;
-inval:
-    errno = EINVAL;
+    if (countp)
+        *countp = count;
+    if (maxidp)
+        *maxidp = maxid;
+    return 0;
 error:
     saved_errno = errno;
-    idset_destroy (idset);
     free (cpy);
     errno = saved_errno;
-    return NULL;
+    return -1;
+}
+
+struct idset *idset_decode_ex (const char *str,
+                               ssize_t len,
+                               ssize_t size,
+                               int flags,
+                               idset_error_t *error)
+{
+    struct idset *idset;
+
+    if (size < 0) {
+        unsigned int maxid;
+        if (decode_and_set_with_info (NULL, str, len, NULL, &maxid, error) < 0)
+            return NULL;
+        if (maxid != IDSET_INVALID_ID)
+            size = maxid + 1;
+        else if ((flags & IDSET_FLAG_AUTOGROW))
+            size = 1;
+        else {
+            errprintf (error, "cannot create an empty idset");
+            errno = EINVAL;
+            return NULL;
+        }
+    }
+    if (!(idset = idset_create (size, flags))) {
+        errprintf (error, "error creating idset object: %s", strerror (errno));
+        return NULL;
+    }
+    if (decode_and_set_with_info (idset, str, len, NULL, NULL, error) < 0) {
+        idset_destroy (idset);
+        return NULL;
+    }
+    return idset;
+}
+
+struct idset *idset_ndecode (const char *str, size_t len)
+{
+    return idset_decode_ex (str, len, 0, IDSET_FLAG_AUTOGROW, NULL);
 }
 
 struct idset *idset_decode (const char *str)
 {
-    return idset_ndecode (str, str ? strlen (str) : 0);
+    return idset_decode_ex (str, -1, 0, IDSET_FLAG_AUTOGROW, NULL);
+}
+
+bool idset_decode_empty (const char *str, ssize_t len)
+{
+    size_t count;
+    if (decode_and_set_with_info (NULL, str, len, &count, NULL, NULL) < 0
+        || count > 0)
+        return false;
+    return true;
+}
+
+int idset_decode_info (const char *str,
+                       ssize_t len,
+                       size_t *count,
+                       unsigned int *maxid,
+                       idset_error_t *error)
+{
+    return decode_and_set_with_info (NULL, str, len, count, maxid, error);
+}
+
+int idset_decode_add (struct idset *idset,
+                      const char *str,
+                      ssize_t len,
+                      idset_error_t *error)
+{
+    return decode_and_set_with_info (idset, str, len, NULL, NULL, error);
+}
+
+int idset_decode_subtract (struct idset *idset,
+                           const char *str,
+                           ssize_t len,
+                           idset_error_t *error)
+{
+    char *cpy;
+    char *tok, *saveptr, *a1;
+    int saved_errno;
+    unsigned int maxid = IDSET_INVALID_ID;
+
+    if (!(cpy = dup_input (str, len, error)))
+        return -1;
+    if (!(a1 = trim_brackets (cpy, error)))
+        goto error;
+    saveptr = NULL;
+    while ((tok = strtok_r (a1, ",", &saveptr))) {
+        if (remove_element (idset, tok, &maxid, error) < 0)
+            goto error;
+        a1 = NULL;
+    }
+    free (cpy);
+    return 0;
+error:
+    saved_errno = errno;
+    free (cpy);
+    errno = saved_errno;
+    return -1;
 }
 
 /*
diff --git a/src/common/libidset/idset_format.c b/src/common/libidset/idset_format.c
index 30e6bea90d4d..ee8ecd628a38 100644
--- a/src/common/libidset/idset_format.c
+++ b/src/common/libidset/idset_format.c
@@ -57,61 +57,6 @@ int format_first (char *buf,
     return 0;
 }
 
-static int idset_format_map_ex (const char *s,
-                                size_t maxsize,
-                                idset_format_map_f fun,
-                                void *arg,
-                                bool *stop)
-
-{
-    const char *start, *end;
-    struct idset *idset = NULL;
-    char *buf = NULL;
-    int count = 0;
-    unsigned int id;
-    int n;
-
-    if (find_brackets (s, &start, &end) == 0) {
-        if (!(idset = idset_ndecode (start, end - start + 1)))
-            goto error;
-        if (!(buf = malloc (maxsize)))
-            goto error;
-        id = idset_first (idset);
-        while (id != IDSET_INVALID_ID) {
-            if (format_first (buf, maxsize, s, id) < 0)
-                goto error;
-            if ((n = idset_format_map_ex (buf, maxsize, fun, arg, stop)) < 0)
-                goto error;
-            count += n;
-            if (*stop)
-                break;
-            id = idset_next (idset, id);
-        }
-        free (buf);
-        idset_destroy (idset);
-    }
-    else {
-        if (fun && fun (s, stop, arg) < 0)
-            goto error;
-        count = 1;
-    }
-    return count;
-error:
-    ERRNO_SAFE_WRAP (free, buf);
-    idset_destroy (idset);
-    return -1;
-}
-
-int idset_format_map (const char *s, idset_format_map_f fun, void *arg)
-{
-    bool stop = false;
-    if (!s) {
-        errno = EINVAL;
-        return -1;
-    }
-    return idset_format_map_ex (s, 4096, fun, arg, &stop);
-}
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libidset/idset_private.h b/src/common/libidset/idset_private.h
index 16edbbca0c43..1f6e6829e53a 100644
--- a/src/common/libidset/idset_private.h
+++ b/src/common/libidset/idset_private.h
@@ -16,7 +16,7 @@
  * All ops are O(log m), for key bitsize m: 2^m == T.M.
  */
 
-#include "src/common/libutil/veb.h"
+#include "veb.h"
 #include "idset.h"
 
 struct idset {
diff --git a/src/common/libidset/test/idset.c b/src/common/libidset/test/idset.c
index 96b80f85ed59..9040f560c1b8 100644
--- a/src/common/libidset/test/idset.c
+++ b/src/common/libidset/test/idset.c
@@ -14,11 +14,13 @@
 #include <errno.h>
 #include <string.h>
 #include <stdbool.h>
-#include <czmq.h>
 
 #include "src/common/libtap/tap.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libidset/idset.h"
 #include "src/common/libidset/idset_private.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
 
 struct inout {
     const char *in;
@@ -29,41 +31,59 @@ struct inout {
 struct inout test_inputs[] = {
     { "2",              0,          "2" },
     { "7-9",            0,          "7,8,9" },
-    { "9-7",            0,          "7,8,9" },
     { "1,7-9",          0,          "1,7,8,9" },
     { "1,7-9,16",       0,          "1,7,8,9,16" },
     { "1,7-9,14,16",    0,          "1,7,8,9,14,16" },
     { "1-3,7-9,14,16",  0,          "1,2,3,7,8,9,14,16" },
-    { "3,2,4,5",        0,          "2,3,4,5" },
+    { "2,3,4,5",        0,          "2,3,4,5" },
     { "",               0,          ""},
     { "1048576",        0,          "1048576"},
 
     { "[2]",            0,          "2" },
     { "[7-9]",          0,          "7,8,9" },
-    { "[9-7]",          0,          "7,8,9" },
-    { "[3,2,4,5]",      0,          "2,3,4,5" },
+    { "[2,3,4,5]",      0,          "2,3,4,5" },
+    { "[0]",            0,          "0" },
     { "[]",             0,          ""},
 
     { "2",              IDSET_FLAG_RANGE,  "2" },
     { "7-9",            IDSET_FLAG_RANGE,  "7-9" },
-    { "9-7",            IDSET_FLAG_RANGE,  "7-9" },
     { "1,7-9",          IDSET_FLAG_RANGE,  "1,7-9" },
     { "1,7-9,16",       IDSET_FLAG_RANGE,  "1,7-9,16" },
     { "1,7-9,14,16",    IDSET_FLAG_RANGE,  "1,7-9,14,16" },
     { "1-3,7-9,14,16",  IDSET_FLAG_RANGE,  "1-3,7-9,14,16" },
-    { "3,2,4,5",        IDSET_FLAG_RANGE,  "2-5" },
+    { "2,3,4,5",        IDSET_FLAG_RANGE,  "2-5" },
     { "",               IDSET_FLAG_RANGE,  ""},
 
     { "2",             IDSET_FLAG_RANGE|IDSET_FLAG_BRACKETS, "2" },
     { "7-9",           IDSET_FLAG_RANGE|IDSET_FLAG_BRACKETS, "[7-9]" },
-    { "9-7",           IDSET_FLAG_RANGE|IDSET_FLAG_BRACKETS, "[7-9]" },
     { "1,7-9",         IDSET_FLAG_RANGE|IDSET_FLAG_BRACKETS, "[1,7-9]" },
     { "1,7-9,16",      IDSET_FLAG_RANGE|IDSET_FLAG_BRACKETS, "[1,7-9,16]" },
     { "1,7-9,14,16",   IDSET_FLAG_RANGE|IDSET_FLAG_BRACKETS, "[1,7-9,14,16]" },
     { "1-3,7-9,14,16", IDSET_FLAG_RANGE|IDSET_FLAG_BRACKETS, "[1-3,7-9,14,16]"},
-    { "3,2,4,5",       IDSET_FLAG_RANGE|IDSET_FLAG_BRACKETS, "[2-5]" },
+    { "2,3,4,5",       IDSET_FLAG_RANGE|IDSET_FLAG_BRACKETS, "[2-5]" },
     { "",              IDSET_FLAG_RANGE|IDSET_FLAG_BRACKETS, ""},
 
+    /* expected failures */
+    { "4.2",            0,          NULL },
+    { "x",              0,          NULL },
+    { "1-2x",           0,          NULL },
+    { "01,2",           0,          NULL },
+    { "00",             0,          NULL },
+    { "3,2",            0,          NULL },
+    { "3-0",            0,          NULL },
+    { "2,2,2,2",        0,          NULL },
+    { "[0",             0,          NULL },
+    { "0]",             0,          NULL },
+    { "[[0]]",          0,          NULL },
+    { "[[0,2]",         0,          NULL },
+    { "[0,2]]",         0,          NULL },
+    { "0,[2",           0,          NULL },
+    { "0]2",            0,          NULL },
+    { "0-",             0,          NULL },
+    { "[0-]",           0,          NULL },
+    { "-5",             0,          NULL },
+    { "[-5]",           0,          NULL },
+
     { NULL, 0, NULL },
 };
 
@@ -74,6 +94,10 @@ void test_basic (void)
     idset = idset_create (0, 0);
     ok (idset != NULL,
         "idset_create size=0 works");
+    ok (idset_count (idset) == 0,
+        "idset_count returns 0");
+    ok (idset_empty (idset) == true,
+        "idset_empty returns true");
 
     idset_destroy (idset);
 }
@@ -84,20 +108,28 @@ void test_codec (void)
 
     for (ip = &test_inputs[0]; ip->in != NULL; ip++) {
         struct idset *idset;
-        char *s;
 
+        errno = 0;
         idset = idset_decode (ip->in);
-        ok (idset != NULL,
-            "idset_decode '%s' works", ip->in);
-        s = idset_encode (idset, ip->flags);
-        bool match = (s == NULL && ip->out == NULL)
-                  || (s && ip->out && !strcmp (s, ip->out));
-        ok (match == true,
-            "idset_encode flags=0x%x '%s' works",
-            ip->flags, ip->out ? ip->out : "NULL");
-        if (!match)
-            diag ("%s", s ? s : "NULL");
-        free (s);
+        if (ip->out == NULL) { // expected fail
+            ok (idset == NULL && errno == EINVAL,
+                "idset_encode flags=0x%x '%s' fails with EINVAL",
+                    ip->flags, ip->in);
+        }
+        else {
+            ok (idset != NULL,
+                "idset_decode '%s' works", ip->in);
+            if (idset != NULL) {
+                char *s = idset_encode (idset, ip->flags);
+                bool match = (s && streq (s, ip->out));
+                ok (match == true,
+                    "idset_encode flags=0x%x '%s'->'%s' works",
+                    ip->flags, ip->in, ip->out);
+                if (!match)
+                    diag ("%s", s ? s : "NULL");
+                free (s);
+            }
+        }
         idset_destroy (idset);
     }
 }
@@ -156,57 +188,63 @@ void test_badparam (void)
 
     errno = 0;
     ok (idset_set (NULL, 1) < 0 && errno == EINVAL,
-        "iset_set(idset=NULL) fails with EINVAL");
+        "idset_set(idset=NULL) fails with EINVAL");
     errno = 0;
     ok (idset_set (idset, IDSET_INVALID_ID) < 0 && errno == EINVAL,
-        "iset_set(id=INVALID) fails with EINVAL");
+        "idset_set(id=INVALID) fails with EINVAL");
     errno = 0;
     ok (idset_set (idset, 101) < 0 && errno == EINVAL,
-        "iset_set(id=out of range) fails with EINVAL");
+        "idset_set(id=out of range) fails with EINVAL");
     errno = 0;
     ok (idset_range_set (NULL, 1, 2) < 0 && errno == EINVAL,
-        "iset_range_set(idset=NULL) fails with EINVAL");
+        "idset_range_set(idset=NULL) fails with EINVAL");
     errno = 0;
     ok (idset_range_set (idset, 1, IDSET_INVALID_ID) < 0 && errno == EINVAL,
-        "iset_range_set(hi=INVALID) fails with EINVAL");
+        "idset_range_set(hi=INVALID) fails with EINVAL");
     errno = 0;
     ok (idset_range_set (idset, IDSET_INVALID_ID, 1) < 0 && errno == EINVAL,
-        "iset_range_set(lo=INVALID) fails with EINVAL");
+        "idset_range_set(lo=INVALID) fails with EINVAL");
     errno = 0;
     ok (idset_range_set (idset, 101, 1) < 0 && errno == EINVAL,
-        "iset_range_set(lo=out of range) fails with EINVAL");
+        "idset_range_set(lo=out of range) fails with EINVAL");
     errno = 0;
     ok (idset_range_set (idset, 1, 101) < 0 && errno == EINVAL,
-        "iset_range_set(hi=out of range) fails with EINVAL");
+        "idset_range_set(hi=out of range) fails with EINVAL");
 
     errno = 0;
     ok (idset_clear (NULL, 1) < 0 && errno == EINVAL,
-        "iset_clear(idset=NULL) fails with EINVAL");
+        "idset_clear(idset=NULL) fails with EINVAL");
     errno = 0;
     ok (idset_clear (idset, IDSET_INVALID_ID) < 0 && errno == EINVAL,
-        "iset_clear(id=INVALID) fails with EINVAL");
+        "idset_clear(id=INVALID) fails with EINVAL");
     errno = 0;
     ok (idset_clear (idset, 101) == 0,
-        "iset_clear(id=out of range) works");
+        "idset_clear(id=out of range) works");
     errno = 0;
     ok (idset_range_clear (NULL, 1, 2) < 0 && errno == EINVAL,
-        "iset_range_clear(idset=NULL) fails with EINVAL");
+        "idset_range_clear(idset=NULL) fails with EINVAL");
     errno = 0;
     ok (idset_range_clear (idset, 1, IDSET_INVALID_ID) < 0 && errno == EINVAL,
-        "iset_range_clear(hi=INVALID) fails with EINVAL");
+        "idset_range_clear(hi=INVALID) fails with EINVAL");
     errno = 0;
     ok (idset_range_clear (idset, IDSET_INVALID_ID, 1) < 0 && errno == EINVAL,
-        "iset_range_clear(lo=INVALID) fails with EINVAL");
+        "idset_range_clear(lo=INVALID) fails with EINVAL");
 
     ok (idset_test (NULL, 1) == false,
-        "iset_test(idset=NULL) returns false");
+        "idset_test(idset=NULL) returns false");
 
     ok (idset_count (NULL) == 0,
-        "iset_count(idset=NULL) returns 0");
+        "idset_count(idset=NULL) returns 0");
+
+    ok (idset_universe_size (NULL) == 0,
+        "idset_universe_size(idset=NULL) returns 0");
+
+    ok (idset_empty (NULL) == true,
+        "idset_empty(idset=NULL) returns true");
 
     errno = 0;
     ok (idset_copy (NULL) == NULL && errno == EINVAL,
-        "iset_copy(idset=NULL) fails with EINVAL");
+        "idset_copy(idset=NULL) fails with EINVAL");
 
     ok (idset_first (NULL) == IDSET_INVALID_ID,
         "idset_first (idset=NULL) returns IDSET_INVALID_ID");
@@ -216,6 +254,12 @@ void test_badparam (void)
         "idset_next (prev=INVALID) returns IDSET_INVALID_ID");
     ok (idset_next (idset, 101) == IDSET_INVALID_ID,
         "idset_next (prev=out of range) returns IDSET_INVALID_ID");
+    ok (idset_prev (NULL, 0) == IDSET_INVALID_ID,
+        "idset_prev (idset=NULL) returns IDSET_INVALID_ID");
+    ok (idset_prev (idset, IDSET_INVALID_ID) == IDSET_INVALID_ID,
+        "idset_prev (id=INVALID) returns IDSET_INVALID_ID");
+    ok (idset_prev (idset, 101) == IDSET_INVALID_ID,
+        "idset_next (id=out of range) returns IDSET_INVALID_ID");
     ok (idset_last (NULL) == IDSET_INVALID_ID,
         "idset_last (idset=NULL) returns IDSET_INVALID_ID");
 
@@ -225,12 +269,16 @@ void test_badparam (void)
 void test_iter (void)
 {
     struct idset *idset;
-    struct idset *idset_empty;
+    struct idset *idset_nil;
 
     if (!(idset = idset_decode ("7-9")))
         BAIL_OUT ("idset_decode 7-9 failed");
-    if (!(idset_empty = idset_create (0, 0)))
+    if (!(idset_nil = idset_create (0, 0)))
         BAIL_OUT ("idset_create (0, 0) failed");
+    ok (idset_empty (idset) == false,
+        "idset_empty (idset=[7-9]) returns false");
+    ok (idset_empty (idset_nil) == true,
+        "idset_empty (idset=[]) returns true");
 
     ok (idset_first (idset) == 7,
         "idset_first idset=[7-9] returned 7");
@@ -246,18 +294,27 @@ void test_iter (void)
         "idset_next idset=[7-9] prev=4096 returned INVALID");
     ok (idset_next (idset, IDSET_INVALID_ID) == IDSET_INVALID_ID,
         "idset_next idset=[7-9] prev=INVALID returned INVALID");
+
     ok (idset_last (idset) == 9,
         "idset_last idset=[7-9] returned 9");
-
-    ok (idset_first (idset_empty) == IDSET_INVALID_ID,
+    ok (idset_prev (idset, 9) == 8,
+        "idset_prev idset=[7-9] id=9 returned 8");
+    ok (idset_prev (idset, 8) == 7,
+        "idset_prev idset=[7-9] id=8 returned 7");
+    ok (idset_prev (idset, 7) == IDSET_INVALID_ID,
+        "idset_prev idset=[7-9] id=7 returned INVALID");
+    ok (idset_prev (idset, IDSET_INVALID_ID) == IDSET_INVALID_ID,
+        "idset_prev idset=[7-9] id=INVALID returned INVALID");
+
+    ok (idset_first (idset_nil) == IDSET_INVALID_ID,
         "idset_first idset=[] returned IDSET_INVALID_ID");
-    ok (idset_last (idset_empty) == IDSET_INVALID_ID,
+    ok (idset_last (idset_nil) == IDSET_INVALID_ID,
         "idset_last idset=[] returned IDSET_INVALID_ID");
-    ok (idset_next (idset_empty, 0) == IDSET_INVALID_ID,
+    ok (idset_next (idset_nil, 0) == IDSET_INVALID_ID,
         "idset_next idset=[] prev=0 returned IDSET_INVALID_ID");
 
     idset_destroy (idset);
-    idset_destroy (idset_empty);
+    idset_destroy (idset_nil);
 }
 
 void test_set (void)
@@ -465,6 +522,137 @@ void test_equal (void)
     idset_destroy (set2);
 }
 
+typedef enum { OP_UNION, OP_DIFF, OP_INTER, OP_ADD, OP_SUB} op_t;
+struct testop {
+    const char *a;
+    op_t op;
+    const char *b;
+    const char *result; // a if OP_ADD or OP_SUB, else return value
+    int xrc;            // ignored unless OP_ADD or OP_SUB
+    int errnum;
+};
+
+static struct testop optab[] = {
+    { NULL,     OP_UNION,   "[0]",      NULL,       0,  EINVAL },
+    { "[0]",    OP_UNION,   NULL,       "[0]",      0,  0 },
+    { "[0]",    OP_UNION,   "[0]",      "[0]",      0,  0 },
+    { "[0]",    OP_UNION,   "[1]",      "[0-1]",    0,  0 },
+    { NULL,     OP_DIFF,    "[0]",      NULL,       0,  EINVAL },
+    { "[0]",    OP_DIFF,    NULL,       "[0]",      0,  0 },
+    { "[0]",    OP_DIFF,    "[0]",      "[]",       0,  0 },
+    { "[0-1]",  OP_DIFF,    "[0]",      "[1]",      0,  0 },
+    { NULL,     OP_INTER,   "[0]",      NULL,       0,  EINVAL },
+    { "[0]",    OP_INTER,   NULL,       NULL,       0,  EINVAL },
+    { "[0-1]",  OP_INTER,   "[2-3]",    "[]",       0,  0 },
+    { "[0-1]",  OP_INTER,   "[1-2]",    "[1]",      0,  0 },
+    { "[0-1]",  OP_INTER,   "[0-1]",    "[0-1]",    0,  0 },
+    { NULL,     OP_ADD,     "[0]",      NULL,       -1, EINVAL },
+    { "[0]",    OP_ADD,     NULL,       "[0]",      0,  0 },
+    { "[0]",    OP_ADD,     "[0]",      "[0]",      0,  0 },
+    { "[0]",    OP_ADD,     "[1]",      "[0,1]",    0,  0 },
+    { NULL,     OP_SUB,     "[0]",      NULL,       -1, EINVAL },
+    { "[0]",    OP_SUB,     NULL,       "[0]",      0,  0 },
+    { "[0,1]",  OP_SUB,     "[1]",      "[0]",      0,  0 },
+    { "[0,1]",  OP_SUB,     "[2]",      "[0,1]",    0,  0 },
+};
+
+static void tryop (const char *s1,
+                   op_t op,
+                   const char *s2,
+                   const char *s3,
+                   int xrc,
+                   int errnum)
+{
+    struct idset *a = NULL;
+    struct idset *b = NULL;
+    struct idset *expect = NULL;
+    struct idset *result = NULL;
+    int rc = -1;
+
+    if (s1) {
+        if (!(a = idset_decode (s1)))
+            BAIL_OUT ("tryop failed to decode %s", s1);
+    }
+    if (s2) {
+        if (!(b = idset_decode (s2)))
+            BAIL_OUT ("tryop failed to decode %s", s2);
+    }
+    if (s3) {
+        if (!(expect = idset_decode (s3)))
+            BAIL_OUT ("tryop failed to decode %s", s3);
+    }
+    errno = 0;
+    switch (op) {
+        case OP_UNION:
+            result = idset_union (a, b);
+            break;
+        case OP_DIFF:
+            result = idset_difference (a, b);
+            break;
+        case OP_INTER:
+            result = idset_intersect (a, b);
+            break;
+        case OP_ADD:
+            rc = idset_add (a, b);
+            break;
+        case OP_SUB:
+            rc = idset_subtract (a, b);
+            break;
+    }
+    /* a in add and subtract is in/out arg, and returns -1 or 0.
+     */
+    if (op == OP_ADD || op == OP_SUB) {
+        ok (((xrc < 0 && xrc == rc && errno == errnum)
+            || idset_equal (expect, a)),
+            "idset_%s %s %s leaves arg1=%s%s%s",
+            op == OP_ADD ? "add" : "subtract",
+            s1 ? s1 : "NULL",
+            s2 ? s2 : "NULL",
+            s3 ? s3 : "NULL",
+            xrc < 0 ? ", fails with " : "",
+            xrc < 0 ? (errnum == EINVAL ? "EINVAL" : "expected errno") : "");
+    }
+    /* Other funcs return a new idset.
+     */
+    else {
+        ok (((expect == result && errno == errnum)
+            || idset_equal (expect, result)),
+            "idset_%s %s %s %s%s%s",
+            op == OP_UNION ? "union" : op == OP_DIFF
+                           ? "difference" : "intersect",
+            s1 ? s1 : "NULL",
+            s2 ? s2 : "NULL",
+            s3 ? "= " : "",
+            s3 ? s3 : "fails with ",
+            errnum > 0 ? (errnum == EINVAL ? "EINVAL" : "expected errno") : "");
+    }
+
+    idset_destroy (a);
+    idset_destroy (b);
+    idset_destroy (expect);
+    idset_destroy (result);
+}
+
+void test_ops (void)
+{
+    for (int i = 0; i < ARRAY_SIZE (optab); i++) {
+        tryop (optab[i].a,
+               optab[i].op,
+               optab[i].b,
+               optab[i].result,
+               optab[i].xrc,
+               optab[i].errnum);
+    }
+    struct idset *a;
+
+    if (!(a = idset_decode ("1-10")))
+        BAIL_OUT ("idset_decode [1-10] failed");
+    idset_clear_all (a);
+    ok (idset_count (a) == 0,
+        "idset_clear_all results in empty set");
+    idset_destroy (a);
+}
+
 void test_copy (void)
 {
     struct idset *idset;
@@ -499,26 +687,31 @@ void test_autogrow (void)
     idset = idset_create (1, 0);
     ok (idset != NULL,
         "idset_create size=1 flags=0 works");
-    ok (idset->T.M == 1,
-        "idset internal size is 1");
+    ok (idset_universe_size (idset) == 1,
+        "idset_universe_size is 1");
     ok (idset_set (idset, 0) == 0,
         "idset_set 0 works");
     errno = 0;
     ok (idset_set (idset, 1) < 0 && errno == EINVAL,
         "idset_set 1 fails with EINVAL");
+    ok (idset_clear (idset, 1) == 0,
+        "idset_clear 1 is a no-op");
     idset_destroy (idset);
 
     idset = idset_create (1, IDSET_FLAG_AUTOGROW);
     ok (idset != NULL,
         "idset_create size=1 flags=AUTOGROW works");
-    ok (idset->T.M == 1,
-        "idset internal size is 1");
+    ok (idset_universe_size (idset) == 1,
+        "idset_universe_size is 1");
     ok (idset_set (idset, 0) == 0,
         "idset_set 0 works");
+    ok (idset_clear (idset, 2) == 0
+        && idset_universe_size (idset) == 1,
+        "idset_clear 2 is a no-op");
     ok (idset_set (idset, 2) == 0,
         "idset_set 2 works");
-    ok (idset->T.M > 1,
-        "idset internal size grew");
+    ok (idset_universe_size (idset) > 1,
+        "idset_universe_size returned a larger size");
     ok (   idset_test (idset, 0)
         && !idset_test (idset, 1)
         && idset_test (idset, 2)
@@ -533,13 +726,13 @@ void test_format_first (void)
     char buf[64];
 
     ok (format_first (buf, sizeof (buf), "[]xyz", 42) == 0
-        && !strcmp (buf, "42xyz"),
+        && streq (buf, "42xyz"),
         "format_first works with leading idset");
     ok (format_first (buf, sizeof (buf), "abc[]xyz", 42) == 0
-        && !strcmp (buf, "abc42xyz"),
+        && streq (buf, "abc42xyz"),
         "format_first works with mid idset");
     ok (format_first (buf, sizeof (buf), "abc[]", 42) == 0
-        && !strcmp (buf, "abc42"),
+        && streq (buf, "abc42"),
         "format_first works with end idset");
 
     errno = 0;
@@ -568,180 +761,431 @@ void test_format_first (void)
         "format_first fails with EOVERFLOW when buffer exhausted");
 }
 
-bool verify_map (zlist_t *list, const char **expected, int count)
+void issue_1974(void)
 {
-    char *s;
-    int i;
+    struct idset *idset;
 
-    s = zlist_first (list);
-    for (i = 0; i < count; i++) {
-        if (strcmp (s, expected[i]) != 0) {
-            diag ("map called with %s, expected %s", s, expected[i]);
-            return false;
-        }
-        s = zlist_next (list);
-    }
-    return true;
+    idset = idset_create (1024, 0);
+    ok (idset != NULL,
+        "1974: idset_create size=1024 worked");
+    ok (idset_test (idset, 1024) == false,
+        "1974: idset_test id=1024 returned false");
+    idset_destroy (idset);
 }
 
-void empty_list (zlist_t *list)
+/* At size 32, veb_pred() returns T.M when checking T.M-1.
+ * We added a workaround, and a TODO test in libutil/test/veb.c for now.
+ * This checks size 31, 32, 33.
+ */
+void issue_2336 (void)
 {
-    char *s;
-    while ((s = zlist_pop (list)))
-        free (s);
+    struct idset *idset;
+    unsigned int t, u, M;
+    int failure = 0;
+
+    for (M = 31; M <= 33; M++) {
+        if (!(idset = idset_create (M, 0)))
+            BAIL_OUT ("idset_create size=32 failed");
+        for (t = 0; t < M; t++) {
+            if (idset_set (idset, t) < 0)
+                BAIL_OUT ("idset_set %u failed", t);
+            u = idset_last (idset);
+            if (u != t) {
+                diag ("idset_last %u returned %u", t, u);
+                failure++;
+            }
+        }
+        ok (failure == 0,
+            "2336: idset_last works for all bits in size=%u idset", M);
+        idset_destroy (idset);
+    }
 }
 
-int mapfun (const char *s, bool *stop, void *arg)
+void test_initfull (void)
 {
-    zlist_t *list = arg;
-    char *cpy;
+    struct idset *idset;
+    size_t size = 128;
 
-    if (!(cpy = strdup (s)))
-        BAIL_OUT ("strdup failed");
-    if (zlist_append (list, cpy) < 0)
-        BAIL_OUT ("zlist_append failed");
+    idset = idset_create (size, IDSET_FLAG_INITFULL);
+    ok (idset != NULL,
+        "idset_create size=%zu flags=INITFULL works", size);
+    ok (idset_count (idset) == size,
+        "idset_count returns correct size");
+    ok (idset_test (idset, 0) == true,
+        "idset_test 0 is true");
+    ok (idset_test (idset, size - 1) == true,
+        "idset_test %zu is true", size - 1);
+    ok (idset_first (idset) == 0,
+        "idset_first returns 0");
+    ok (idset_next (idset, 0) == 1,
+        "idset_next prev=0 returns 1");
+    ok (idset_test (idset, size) == false,
+        "idset_test %zu is false", size);
+    idset_destroy (idset);
 
-    return 0;
+    // grow by clearing a bit
+    idset = idset_create (0, IDSET_FLAG_INITFULL | IDSET_FLAG_AUTOGROW);
+    ok (idset != NULL,
+        "idset_create size=0 flags=INITFULL|AUTOGROW works");
+    ok (idset_clear (idset, IDSET_DEFAULT_SIZE) == 0,
+        "idset_clear id=size works");
+    ok (idset_count (idset) == IDSET_DEFAULT_SIZE*2 - 1,
+        "idset_count returns 2*default size - 1");
+    ok (idset_test (idset, IDSET_DEFAULT_SIZE) == false,
+        "idset_test id=size is false");
+    ok (idset_test (idset, IDSET_DEFAULT_SIZE + 1) == true,
+        "idset_test id=size+1 is true");
+    ok (idset_test (idset, IDSET_DEFAULT_SIZE*2 - 1) == true,
+        "idset_test id=2*size-1 is true");
+    idset_destroy (idset);
+
+    // setting a bit should not cause growth
+    idset = idset_create (4, IDSET_FLAG_INITFULL | IDSET_FLAG_AUTOGROW);
+    ok (idset != NULL,
+        "idset_create size=4 flags=INITFULL|AUTOGROW works");
+    ok (idset_count (idset) == 4,
+        "idset_count returns 4");
+    ok (idset_set (idset, 4) == 0,
+        "idset_set 4 (out of range) works");
+    ok (idset_count (idset) == 4,
+        "idset_count still returns 4");
+    idset_destroy (idset);
 }
 
-int mapfun_err3 (const char *s, bool *stop, void *arg)
+void test_alloc (int flags)
 {
-    zlist_t *list = arg;
-    char *cpy;
+    struct idset *idset;
+    unsigned int ids[64];
+    unsigned int id;
+    int errors;
+    size_t size_before;
+    size_t size_after;
+
+    flags |= IDSET_FLAG_AUTOGROW | IDSET_FLAG_INITFULL;
+    idset = idset_create (16, flags);
+    if (!idset)
+        BAIL_OUT ("could not create idset");
+
+    ok (idset_count (idset) == 16,
+        "idset_count returns 16");
+
+    size_before = idset_universe_size (idset);
+    errors = 0;
+    for (int i = 0; i < ARRAY_SIZE (ids); i++) {
+        if (idset_alloc (idset, &ids[i]) < 0)
+            errors++;
+    }
+    size_after = idset_universe_size (idset);
+    ok (errors == 0,
+        "idset_alloc allocated multiple ids with no errors");
+    errors = 0;
+    for (int i = 0; i < ARRAY_SIZE (ids); i++)
+        if (ids[i] != i) {
+            diag ("allocation %d is %lu", i, ids[i]);
+            errors++;
+        }
+    ok (errors == 0,
+        "ids were allocated monotonically");
+    ok (size_before < size_after,
+        "idset size grew automatically");
+    diag ("before=%zu after=%zu", size_before, size_after);
+    ok (idset_count (idset) == 0,
+        "idset_count returns 0");
 
-    if (zlist_size (list) == 3) {
-        errno = EPERM; // arbitrary
-        return -1;
+    errors = 0;
+    for (int i = 0; i < ARRAY_SIZE (ids); i += 2) {
+        if (idset_free_check (idset, ids[i]) < 0) {
+            diag ("idset_free_check %lu: %s", ids[i], strerror (errno));
+            errors++;
+        }
     }
-    if (!(cpy = strdup (s)))
-        BAIL_OUT ("strdup failed");
-    if (zlist_append (list, cpy) < 0)
-        BAIL_OUT ("zlist_append failed");
-    return 0;
-}
+    ok (errors == 0,
+        "idset_free_check freed multiple ids with no errors");
+    ok (idset_count (idset) == 32,
+        "idset_count returns 32");
+
+    errors = 0;
+    for (int i = 0; i < ARRAY_SIZE (ids); i += 2) {
+        if (idset_alloc (idset, &ids[i]) < 0)
+            errors++;
+    }
+    ok (errors == 0,
+        "idset_alloc re-allocated multiple ids with no errors");
+
+    errors = 0;
+    for (int i = 0; i < ARRAY_SIZE (ids); i++)
+        if (ids[i] != i) {
+            diag ("allocation %d is %lu", i, ids[i]);
+            errors++;
+        }
+    ok (errors == 0,
+        "ids were allocated monotonically");
+    ok (idset_count (idset) == 0,
+        "idset_count returns 0");
 
-int mapfun_stop3 (const char *s, bool *stop, void *arg)
-{
-    zlist_t *list = arg;
-    char *cpy;
-
-    if (!(cpy = strdup (s)))
-        BAIL_OUT ("strdup failed");
-    if (zlist_append (list, cpy) < 0)
-        BAIL_OUT ("zlist_append failed");
-    if (zlist_size (list) == 3)
-        *stop = true;
-    return 0;
-}
-struct maptest {
-    const char *input;
-    const char *expected[16];
-    int count;
-};
+    for (int i = 0; i < ARRAY_SIZE (ids); i++)
+        idset_free (idset, ids[i]);
+    ok (idset_count (idset) == idset_universe_size (idset),
+        "idset_free freed all ids");
 
-struct maptest maptests[] = {
-    { "n[0-3]", { "n0", "n1", "n2", "n3" }, 4 },
-    { "r[0-1]n[0-1]", { "r0n0", "r0n1", "r1n0", "r1n1" }, 4 },
-    { "[0-1][0-1][0-2]", { "000", "001", "002", "010", "011", "012",
-                           "100", "101", "102", "110", "111", "112"}, 12 },
-    { "n[0,99-100]x", { "n0x", "n99x", "n100x" }, 3 },
-    { "foo", { "foo" }, 1 },
-    { "foo[", { "foo[" }, 1 },
-    { "foo]", { "foo]" }, 1 },
-    { "foo][", { "foo][" }, 1 },
-    { "foo[]", { }, 0 },
-    { "", { "" }, 1 },
-    { NULL, {}, 0 },
-};
+    idset_destroy (idset);
+    idset = idset_create (16, IDSET_FLAG_INITFULL);
+    if (!idset)
+        BAIL_OUT ("could not create idset");
+    for (int i = 0; i < 16; i++)
+        if (idset_alloc (idset, &id) < 0)
+            BAIL_OUT ("could not allocate ids in existing universe");
+    errno = 0;
+    ok (idset_alloc (idset, &id) < 0 && errno == EINVAL,
+        "idset_alloc fails with EINVAL when universe is full and no autofree");
+    idset_destroy (idset);
+}
 
-void test_format_map (void)
+void test_alloc_badparam (void)
 {
-    zlist_t *list;
-    int i;
-    int rc;
+    unsigned int id;
+    struct idset *idset;
+    struct idset *idset2;
 
-    if (!(list = zlist_new ()))
-        BAIL_OUT ("zlist_new failed");
+    idset = idset_create (16, IDSET_FLAG_INITFULL | IDSET_FLAG_AUTOGROW);
+    if (!idset)
+        BAIL_OUT ("could not create idset");
+    idset2 = idset_create (16, 0);
+    if (!idset2)
+        BAIL_OUT ("could not create idset");
 
-    /* bad params */
     errno = 0;
-    ok (idset_format_map (NULL, mapfun, NULL) < 0
-        && errno == EINVAL,
-        "idset_format_map input=NULL fails with EINVAL");
-
-    /* bad idset, but correctly embedded */
+    ok (idset_alloc (NULL, &id) < 0 && errno == EINVAL,
+        "idset_alloc idset=NULL fails with EINVAL");
     errno = 0;
-    ok (idset_format_map ("[foo]", mapfun, NULL) < 0
-        && errno == EINVAL,
-        "idset_format_map input=[foo] fails with EINVAL");
-
-    /* check for expected expansion */
-    for (i = 0; maptests[i].input != NULL; i++) {
-        rc = idset_format_map (maptests[i].input, mapfun, list);
-        ok (rc == maptests[i].count
-            && verify_map (list, maptests[i].expected, maptests[i].count),
-            "idset_format_map input='%s' works",
-            maptests[i].input);
+    ok (idset_alloc (idset2, &id) < 0 && errno == EINVAL,
+        "idset_alloc fails with EINVAL without IDSET_FLAG_INITFULL");
+    errno = 0;
+    ok (idset_alloc (idset, NULL) < 0 && errno == EINVAL,
+        "idset_alloc id=NULL fails with EINVAL");
 
-        empty_list (list);
-    }
+    lives_ok ({idset_free (NULL, 42);},
+        "idset_free idset=NULL doesn't crash");
+    idset_free (idset2, 2);
+    diag ("idset_free without IDSET_FLAG_INITFULL is a no-op");
 
-    /* map() returns -1 with errno == EPERM on 4th call */
     errno = 0;
-    ok (idset_format_map ("h[0-15]", mapfun_err3, list) < 0
-        && errno == EPERM
-        && zlist_size (list) == 3,
-        "idset_format_map input handles map() failure OK");
-    empty_list (list);
-
-    /* map() pokes *stop on 4th call */
+    ok (idset_free_check (NULL, 2) < 0 && errno == EINVAL,
+        "idset_free_check idset=NULL fails with EINVAL");
     errno = 0;
-    ok (idset_format_map ("h[0-15]", mapfun_stop3, list) == 3
-        && zlist_size (list) == 3,
-        "idset_format_map input handles *stop = true OK");
-    empty_list (list);
+    ok (idset_free_check (idset2, 2) < 0 && errno == EINVAL,
+        "idset_free_check without IDSET_FLAG_INITFULL fails with EINVAL");
+    errno = 0;
+    ok (idset_free_check (idset, 16) < 0 && errno == EINVAL,
+        "idset_free_check without out of range fails with EINVAL");
+    errno = 0;
+    ok (idset_free_check (idset, 2) < 0 && errno == EEXIST,
+        "idset_free_check on non-free id fails with EEXIST");
 
-    zlist_destroy (&list);
+    idset_destroy (idset);
+    idset_destroy (idset2);
 }
 
-void issue_1974(void)
+void test_decode_ex (void)
 {
     struct idset *idset;
+    idset_error_t error;
 
-    idset = idset_create (1024, 0);
+    /* First, for fun, just generate some run of the mill parsing errors that
+     * are already tested above and show the textual errors on the diag output.
+     */
+    errno = 0;
+    error.text[0] = '\0';
+    idset = idset_decode_ex ("0", 1, 0, 0xffff, &error);
+    ok (idset == NULL && errno == EINVAL && strlen (error.text) > 0,
+        "idset_decode_ex flags=0xffff fails with EINVAL");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    idset = idset_decode_ex ("[0-1", 4, 0, 0, &error);
+    ok (idset == NULL && errno == EINVAL && strlen (error.text) > 0,
+        "idset_decode_ex s=[0-1 fails with EINVAL");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    idset = idset_decode_ex ("2,1,0", 4, 0, 0, &error);
+    ok (idset == NULL && errno == EINVAL && strlen (error.text) > 0,
+        "idset_decode_ex s=2,1,0 fails with EINVAL");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    idset = idset_decode_ex ("0-255xxx", 8, 0, 0, &error);
+    ok (idset == NULL && errno == EINVAL && strlen (error.text) > 0,
+        "idset_decode_ex s=0-255xxx fails with EINVAL");
+    diag ("%s", error.text);
+
+    /* Decode to a fixed size (256) set
+     * This works because len=5 turns the input into 0-255
+     */
+    idset = idset_decode_ex ("0-255,256", 5, 256, 0, &error);
     ok (idset != NULL,
-        "1974: idset_create size=1024 worked");
-    ok (idset_test (idset, 1024) == false,
-        "1974: idset_test id=1024 returned false");
+        "idset_decode_ex s=0-255,256 len=5 size=256 works");
+    idset_destroy (idset);
+
+    /* Overflow a fixed size (256) set
+     */
+    errno = 0;
+    error.text[0] = '\0';
+    idset = idset_decode_ex ("0-255,256", 9, 256, 0, &error);
+    ok (idset == NULL && errno == EINVAL && strlen (error.text) > 0,
+        "idset_decode_ex s=0-255,256 len=9 size=256 fails with EINVAL");
+    diag ("%s", error.text);
+
+    /* Show that overflow is handled with AUTOGROW.
+     */
+    idset = idset_decode_ex ("255,256", 7, 256, IDSET_FLAG_AUTOGROW, &error);
+    ok (idset != NULL,
+        "idset_decode_ex s=255,256 size=256 works with AUTOGROW");
+    idset_destroy (idset);
+
+    /* An empty set with size=-1 is not allowed without AUTOGROW.
+     */
+    errno = 0;
+    error.text[0] = '\0';
+    idset = idset_decode_ex ("", -1, -1, 0, &error);
+    ok (idset == NULL && errno == EINVAL && strlen (error.text) > 0,
+        "idset_decode_ex s=\"\" size=-1 fails with EINVAL");
+    diag ("%s", error.text);
+
+    /* But an empty set with size=-1 and AUTOGROW is allowed.
+     */
+    idset = idset_decode_ex ("", -1, -1, IDSET_FLAG_AUTOGROW, &error);
+    ok (idset != NULL,
+        "idset_decode_ex s=\"\" size=-1 works with AUTOGROW");
+    idset_destroy (idset);
+
+    /* A set with size-1 exactly fits the max id
+     */
+    idset = idset_decode_ex ("1,3,5", -1, -1, 0, &error);
+    ok (idset != NULL,
+        "idset_decode_ex s=1,3,5 size=-1 works");
+    ok (idset_universe_size (idset) == 6,
+        "idset_universe_size returns 6");
     idset_destroy (idset);
 }
 
-/* At size 32, veb_pred() returns T.M when checking T.M-1.
- * We added a workaround, and a TODO test in libutil/test/veb.c for now.
- * This checks size 31, 32, 33.
- */
-void issue_2336 (void)
+void test_decode_empty (void)
 {
-    struct idset *idset;
-    unsigned int t, u, M;
-    int failure = 0;
+    ok (idset_decode_empty ("[]", -1) == true,
+        "idset_decode_empty [] returns true");
+    ok (idset_decode_empty ("", -1) == true,
+        "idset_decode_empty \"\" returns true");
+    ok (idset_decode_empty ("1-4", -1) == false,
+        "idset_decode_empty 1-4 returns false");
+    ok (idset_decode_empty ("[", -1) == false,
+        "idset_decode_empty [ returns false");
+    ok (idset_decode_empty (NULL, -1) == false,
+        "idset_decode_empty NULL returns false");
+}
 
-    for (M = 31; M <= 33; M++) {
-        if (!(idset = idset_create (M, 0)))
-            BAIL_OUT ("idset_create size=32 failed");
-        for (t = 0; t < M; t++) {
-            if (idset_set (idset, t) < 0)
-                BAIL_OUT ("idset_set %u failed", t);
-            u = idset_last (idset);
-            if (u != t) {
-                diag ("idset_last %u returned %u", t, u);
-                failure++;
-            }
+struct infovec {
+    const char *input;
+    int errnum;
+    size_t count;
+    unsigned int maxid;
+};
+
+struct infovec infovec[] = {
+    { "[]", 0, 0, IDSET_INVALID_ID },
+    { "", 0, 0, IDSET_INVALID_ID },
+    { "[", EINVAL, 0, 0 },
+    { NULL, EINVAL, 0, 0 },
+    { "0", 0, 1, 0 },
+    { "1,2", 0, 2, 2 },
+    { "1,2-1024", 0, 1024, 1024 },
+    { "0-3", 0, 4, 3 },
+};
+
+
+void test_decode_info (void)
+{
+    for (int i = 0; i < ARRAY_SIZE (infovec); i++) {
+        idset_error_t error;
+        size_t count = 0;
+        unsigned int maxid = 0;
+        int rc;
+
+        errno = 0;
+        error.text[0] = '\0';
+
+        rc = idset_decode_info (infovec[i].input,
+                                -1,
+                                &count,
+                                &maxid,
+                                &error);
+        if (infovec[i].errnum != 0) {
+            ok (rc < 0
+                && errno == infovec[i].errnum
+                && strlen (error.text) > 0,
+                "idset_decode_info %s failed with expected error",
+                infovec[i].input ? infovec[i].input : "NULL");
+            diag ("%s", error.text);
+        }
+        else {
+            ok (rc == 0
+                && count == infovec[i].count
+                && maxid == infovec[i].maxid,
+                "idset_decode_info %s  works",
+                infovec[i].input);
         }
-        ok (failure == 0,
-            "2336: idset_last works for all bits in size=%u idset", M);
-        idset_destroy (idset);
     }
+
+    ok (idset_decode_info ("1", -1, NULL, NULL, NULL) == 0,
+        "idset_decode_info accepts NULL maxid/count");
+}
+
+void test_decode_addsub (void)
+{
+    struct idset *idset;
+    idset_error_t error;
+
+    if (!(idset = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        BAIL_OUT ("idset_create failed");
+
+    ok (idset_decode_add (idset, "1-4", -1, &error) == 0
+        && idset_count (idset) == 4,
+        "idset_decode_add 1-4 works");
+    ok (idset_decode_add (idset, "5-8", -1, &error) == 0
+        && idset_count (idset) == 8,
+        "idset_decode_add 5-8 works");
+    ok (idset_decode_add (idset, "1,5", -1, &error) == 0
+        && idset_count (idset) == 8,
+        "idset_decode_add 1,5 works");
+    ok (idset_decode_subtract (idset, "1,5", -1, &error) == 0
+        && idset_count (idset) == 6,
+        "idset_decode_subtract 1,5 works");
+    ok (idset_decode_subtract (idset, "", -1, &error) == 0
+        && idset_count (idset) == 6,
+        "idset_decode_subtract \"\"  works");
+    ok (idset_decode_subtract (idset, "0-100", -1, &error) == 0
+        && idset_count (idset) == 0,
+        "idset_decode_subtract 0-100 works");
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok  (idset_decode_add (idset, "[", -1, &error) < 0
+         && errno == EINVAL
+         && strlen (error.text) > 0,
+         "idset_decode_add [ fails with errno and error");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok  (idset_decode_subtract (idset, "]", -1, &error) < 0
+         && errno == EINVAL
+         && strlen (error.text) > 0,
+         "idset_decode_subtract ] fails with errno and error");
+    diag ("%s", error.text);
+
+    idset_destroy (idset);
 }
 
 int main (int argc, char *argv[])
@@ -761,9 +1205,17 @@ int main (int argc, char *argv[])
     test_copy ();
     test_autogrow ();
     test_format_first ();
-    test_format_map ();
     issue_1974 ();
     issue_2336 ();
+    test_ops ();
+    test_initfull();
+    test_alloc (0);
+    test_alloc (IDSET_FLAG_COUNT_LAZY);
+    test_alloc_badparam();
+    test_decode_ex ();
+    test_decode_empty ();
+    test_decode_info ();
+    test_decode_addsub ();
 
     done_testing ();
 }
diff --git a/src/common/libidset/test/idsetutil.c b/src/common/libidset/test/idsetutil.c
new file mode 100644
index 000000000000..ef97da10cc09
--- /dev/null
+++ b/src/common/libidset/test/idsetutil.c
@@ -0,0 +1,90 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* idsetutil.c - command line idset calculator for testing */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include "src/common/libidset/idset.h"
+#include "src/common/libutil/read_all.h"
+#include "ccan/str/str.h"
+
+
+int expand (int argc, char **argv)
+{
+    struct idset *ids = NULL;
+    char *input;
+    char *buf = NULL;
+    unsigned int id;
+
+    if (argc != 0 && argc != 1) {
+        fprintf (stderr, "Usage: idsetutil expand [IDSET]\n");
+        return -1;
+    }
+    if (argc == 0) {
+        if (read_all (STDIN_FILENO, (void **)&buf) < 0) {
+            fprintf (stderr, "error reading stdin: %s\n", strerror (errno));
+            goto error;
+        }
+        if (buf[strlen (buf) - 1] == '\n') // drop trailing newline, if any
+            buf[strlen (buf) - 1] = '\0';
+        input = buf;
+    }
+    else
+        input = argv[0];
+    if (!(ids = idset_decode (input))) {
+        fprintf (stderr, "error decoding idset: %s\n", strerror (errno));
+        goto error;
+    }
+    id = idset_first (ids);
+    while (id != IDSET_INVALID_ID) {
+        printf ("%u\n", id);
+        id = idset_next (ids, id);
+    }
+    idset_destroy (ids);
+    free (buf);
+    return 0;
+error:
+    idset_destroy (ids);
+    free (buf);
+    return -1;
+}
+
+void usage (void)
+{
+    fprintf (stderr,
+             "Usage: idsetutil CMD ARGS\n"
+             "where CMD is one of:\n"
+             "expand [IDSET]\n"
+             );
+}
+
+int main (int argc, char *argv[])
+{
+    if (argc < 2 || !streq (argv[1], "expand")) {
+        fprintf (stderr, "Usage: isdetutil expand IDSET\n");
+        return 1;
+    }
+    if (expand (argc - 2, argv + 2) < 0)
+        return 1;
+
+    return 0;
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libutil/test/veb.c b/src/common/libidset/test/veb.c
similarity index 99%
rename from src/common/libutil/test/veb.c
rename to src/common/libidset/test/veb.c
index 80fb2eb1f485..1c5d5d021fdc 100644
--- a/src/common/libutil/test/veb.c
+++ b/src/common/libidset/test/veb.c
@@ -8,10 +8,14 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 #include <stdlib.h>
+
 #include "src/common/libtap/tap.h"
-#include "src/common/libutil/veb.h"
+#include "veb.h"
 
 void empty_pred_test1 (void)
 {
diff --git a/src/common/libutil/veb.c b/src/common/libidset/veb.c
similarity index 100%
rename from src/common/libutil/veb.c
rename to src/common/libidset/veb.c
diff --git a/src/common/libutil/veb.h b/src/common/libidset/veb.h
similarity index 100%
rename from src/common/libutil/veb.h
rename to src/common/libidset/veb.h
diff --git a/src/common/libutil/veb_mach.c b/src/common/libidset/veb_mach.c
similarity index 100%
rename from src/common/libutil/veb_mach.c
rename to src/common/libidset/veb_mach.c
diff --git a/src/common/libioencode/Makefile.am b/src/common/libioencode/Makefile.am
index 421f54aac388..4c2560fc6242 100644
--- a/src/common/libioencode/Makefile.am
+++ b/src/common/libioencode/Makefile.am
@@ -6,10 +6,12 @@ AM_LDFLAGS = \
         $(CODE_COVERAGE_LDFLAGS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
+	$(JANSSON_CFLAGS)
 
 noinst_LTLIBRARIES = \
 	libioencode.la
@@ -30,10 +32,13 @@ T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
 
 test_ldadd = \
         $(top_builddir)/src/common/libioencode/libioencode.la \
-        $(top_builddir)/src/common/libflux-internal.la \
         $(top_builddir)/src/common/libflux-core.la \
+        $(top_builddir)/src/common/libflux-internal.la \
         $(top_builddir)/src/common/libtap/libtap.la
 
+test_ldflags = \
+	-no-install
+
 test_cppflags = \
         $(AM_CPPFLAGS) \
 	-I$(top_srcdir)/src/common/libtap
@@ -41,3 +46,4 @@ test_cppflags = \
 test_ioencode_t_SOURCES = test/ioencode.c
 test_ioencode_t_CPPFLAGS = $(test_cppflags)
 test_ioencode_t_LDADD = $(test_ldadd)
+test_ioencode_t_LDFLAGS = $(test_ldflags)
diff --git a/src/common/libioencode/ioencode.c b/src/common/libioencode/ioencode.c
index 6aab89c51d9f..e23a3ae8bf3d 100644
--- a/src/common/libioencode/ioencode.c
+++ b/src/common/libioencode/ioencode.c
@@ -17,50 +17,36 @@
 #include <stdbool.h>
 #include <errno.h>
 
-#include <sodium.h>
 #include <jansson.h>
 
-#include "src/common/libutil/macros.h"
+#include "src/common/libccan/ccan/base64/base64.h"
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
 
 #include "ioencode.h"
 
-static char *bin2base64 (const char *bin_data, int bin_len)
-{
-    char *base64_data;
-    int base64_len;
-
-    base64_len = sodium_base64_encoded_len (bin_len, sodium_base64_VARIANT_ORIGINAL);
-
-    if (!(base64_data = calloc (1, base64_len)))
-        return NULL;
-
-    sodium_bin2base64 (base64_data, base64_len,
-                       (unsigned char *)bin_data, bin_len,
-                       sodium_base64_VARIANT_ORIGINAL);
-
-    return base64_data;
-}
 
-static char *base642bin (const char *base64_data, size_t *bin_len)
+static json_t *data_encode_base64 (const char *stream,
+                                   const char *rank,
+                                   const char *data,
+                                   int len)
 {
-    size_t base64_len;
-    char *bin_data = NULL;
-
-    base64_len = strlen (base64_data);
-    (*bin_len) = BASE64_DECODE_SIZE (base64_len);
-
-    if (!(bin_data = calloc (1, (*bin_len))))
-        return NULL;
+    ssize_t n;
+    json_t *o = NULL;
+    char *dest = NULL;
+    size_t destlen = base64_encoded_length (len) + 1; /* +1 for NUL */
 
-    if (sodium_base642bin ((unsigned char *)bin_data, (*bin_len),
-                           base64_data, base64_len,
-                           NULL, bin_len, NULL,
-                           sodium_base64_VARIANT_ORIGINAL) < 0) {
-        free (bin_data);
-        return NULL;
+    if ((dest = malloc (destlen))
+        && (n = base64_encode (dest, destlen, data, len)) >= 0) {
+        if (!(o = json_pack ("{s:s s:s s:s s:s#}",
+                             "stream", stream,
+                             "rank", rank,
+                             "encoding", "base64",
+                             "data", dest, n)))
+            errno = ENOMEM;
     }
-
-    return bin_data;
+    ERRNO_SAFE_WRAP (free, dest);
+    return o;
 }
 
 json_t *ioencode (const char *stream,
@@ -69,9 +55,7 @@ json_t *ioencode (const char *stream,
                   int len,
                   bool eof)
 {
-    char *base64_data = NULL;
     json_t *o = NULL;
-    json_t *rv = NULL;
 
     /* data can be NULL and len == 0 if eof true */
     if (!stream
@@ -84,30 +68,60 @@ json_t *ioencode (const char *stream,
     }
 
     if (data && len) {
-        if (!(base64_data = bin2base64 (data, len)))
-            goto error;
-        if (!(o = json_pack ("{s:s s:s s:s}",
+        if (!(o = json_pack ("{s:s s:s s:s#}",
                              "stream", stream,
                              "rank", rank,
-                             "data", base64_data)))
-            goto error;
+                             "data", data, len))) {
+            /*
+             *  json_pack() may fail if there are bytes that cannot
+             *   be encoded as UTF-8 - try again, this time
+             *   encoding the data in base64:
+             */
+            if (!(o = data_encode_base64 (stream, rank, data, len)))
+                return NULL;
+        }
     }
     else {
         if (!(o = json_pack ("{s:s s:s}",
                              "stream", stream,
-                             "rank", rank)))
-            goto error;
+                             "rank", rank))) {
+            errno = ENOMEM;
+            return NULL;
+        }
     }
     if (eof) {
-        if (json_object_set_new (o, "eof", json_true ()) < 0)
-            goto error;
+        if (json_object_set_new (o, "eof", json_true ()) < 0) {
+            json_decref (o);
+            errno = ENOMEM;
+            return NULL;
+        }
     }
-    rv = o;
-error:
-    free (base64_data);
-    return rv;
+    return o;
 }
 
+static int decode_data_base64 (char *src,
+                               size_t srclen,
+                               char **datap,
+                               size_t *lenp)
+{
+    ssize_t rc;
+    size_t size = base64_decoded_length (srclen);
+    if (datap) {
+        if (!(*datap = malloc (size)))
+            return -1;
+        if ((rc = base64_decode (*datap, size, src, srclen)) < 0) {
+            ERRNO_SAFE_WRAP (free, (*datap));
+            return -1;
+        }
+        if (lenp)
+            *lenp = rc;
+    }
+    else if (lenp)
+        *lenp = size;
+    return 0;
+}
+
+
 int iodecode (json_t *o,
               const char **streamp,
               const char **rankp,
@@ -117,53 +131,64 @@ int iodecode (json_t *o,
 {
     const char *stream;
     const char *rank;
-    const char *base64_data;
-    char *bin_data = NULL;
+    const char *encoding = NULL;
     size_t bin_len = 0;
+    char *bufp = NULL;
+    char *data = NULL;
+    size_t len = 0;
     int eof = 0;
     bool has_data = false;
     bool has_eof = false;
-    int rv = -1;
 
     if (!o) {
         errno = EINVAL;
         return -1;
     }
 
-    if (json_unpack (o, "{s:s s:s}",
+    if (json_unpack (o, "{s:s s:s s?s}",
                      "stream", &stream,
-                     "rank", &rank) < 0)
-        goto cleanup;
-    if (json_unpack (o, "{s:s}", "data", &base64_data) == 0) {
-        has_data = true;
-        if (!(bin_data = base642bin (base64_data, &bin_len)))
-            goto cleanup;
+                     "rank", &rank,
+                     "encoding", &encoding) < 0) {
+        errno = EPROTO;
+        return -1;
     }
+    if (json_unpack (o, "{s:s%}", "data", &data, &len) == 0)
+        has_data = true;
     if (json_unpack (o, "{s:b}", "eof", &eof) == 0)
         has_eof = true;
 
     if (!has_data && !has_eof) {
         errno = EPROTO;
-        goto cleanup;
+        return -1;
     }
 
     if (streamp)
         (*streamp) = stream;
     if (rankp)
         (*rankp) = rank;
-    if (datap) {
-        (*datap) = bin_data;
-        bin_data = NULL;
+    if (datap || lenp) {
+        if (data) {
+            if (encoding && streq (encoding, "base64")) {
+                if (decode_data_base64 (data, len, &bufp, &bin_len) < 0)
+                    return -1;
+            }
+            else {
+                bin_len = len;
+                if (datap) {
+                    if (!(bufp = malloc (bin_len)))
+                        return -1;
+                    memcpy (bufp, data, bin_len);
+                }
+            }
+        }
     }
+    if (datap)
+        (*datap) = bufp;
     if (lenp)
         (*lenp) = bin_len;
     if (eofp)
         (*eofp) = eof;
-
-    rv = 0;
-cleanup:
-    free (bin_data);
-    return rv;
+    return 0;
 }
 
 /*
diff --git a/src/common/libioencode/ioencode.h b/src/common/libioencode/ioencode.h
index 4fc1fe15e42f..6c4a804accf1 100644
--- a/src/common/libioencode/ioencode.h
+++ b/src/common/libioencode/ioencode.h
@@ -17,6 +17,7 @@
 /* encode io data and/or EOF into RFC24 data event object
  * - to set only EOF, set data to NULL and data_len to 0
  * - it is an error to provide no data and EOF = false
+ * - returns RFC24 object on success, NULL on error with errno set
  * - returned object should be json_decref()'d after use
  */
 json_t *ioencode (const char *stream,
@@ -29,6 +30,8 @@ json_t *ioencode (const char *stream,
  * - both data and EOF can be available
  * - if no data available, data set to NULL and len to 0
  * - data must be freed after return
+ * - data can be NULL and len non-NULL to retrieve data length
+ * - returns 0 on success, -1 on error with errno set
  */
 int iodecode (json_t *o,
               const char **stream,
diff --git a/src/common/libioencode/test/ioencode.c b/src/common/libioencode/test/ioencode.c
index d8fbc8780433..4a2017c99540 100644
--- a/src/common/libioencode/test/ioencode.c
+++ b/src/common/libioencode/test/ioencode.c
@@ -8,11 +8,15 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <stdio.h>
 #include <string.h>
 #include <jansson.h>
 #include <errno.h>
 
+#include "ccan/str/str.h"
 #include "src/common/libtap/tap.h"
 #include "src/common/libioencode/ioencode.h"
 
@@ -42,8 +46,8 @@ void basic (void)
         "ioencode success (data, eof = false)");
     ok (!iodecode (o, &stream, &rank, &data, &len, &eof),
         "iodecode success");
-    ok (!strcmp (stream, "stdout")
-        && !strcmp (rank, "1")
+    ok (streq (stream, "stdout")
+        && streq (rank, "1")
         && len == 3
         && !strncmp (data, "foo", len)
         && eof == false,
@@ -55,26 +59,80 @@ void basic (void)
         "ioencode success (data, eof = true)");
     ok (!iodecode (o, &stream, &rank, &data, &len, &eof),
         "iodecode success");
-    ok (!strcmp (stream, "stdout")
-        && !strcmp (rank, "[0-8]")
+    ok (streq (stream, "stdout")
+        && streq (rank, "[0-8]")
         && len == 3
         && !strncmp (data, "bar", len)
         && eof == true,
         "iodecode returned correct info");
     free (data);
+
+    ok (iodecode (o, &stream, &rank, NULL, &len, &eof) == 0,
+        "iodecode can be passed NULL data to query len");
+    ok (streq (stream, "stdout")
+        && streq (rank, "[0-8]")
+        && len == 3
+        && eof == true,
+        "iodecode returned correct info");
     json_decref (o);
 
     ok ((o = ioencode ("stderr", "[4,5]", NULL, 0, true)) != NULL,
         "ioencode success (no data, eof = true)");
     ok (!iodecode (o, &stream, &rank, &data, &len, &eof),
         "iodecode success");
-    ok (!strcmp (stream, "stderr")
-        && !strcmp (rank, "[4,5]")
+    ok (streq (stream, "stderr")
+        && streq (rank, "[4,5]")
         && data == NULL
         && len == 0
         && eof == true,
         "iodecode returned correct info");
     free (data);
+
+    ok (iodecode (o, &stream, &rank, NULL, &len, &eof) == 0,
+        "iodecode can be passed NULL data to query len");
+    ok (streq (stream, "stderr")
+        && streq (rank, "[4,5]")
+        && len == 0
+        && eof == true,
+        "iodecode returned correct info");
+    json_decref (o);
+}
+
+static void binary_data (void)
+{
+    json_t *o = NULL;
+    const char *stream;
+    const char *rank;
+    char *data;
+    char *encoding = NULL;
+    int len;
+    bool eof;
+
+    /*  \xed\xbf\xbf is not a valid utf-8 codepoint */
+    const char buffer[15] = "\xed\xbf\xbf\x4\x5\x6\x7\x8\x9\xa\xb\xc\xd\xe\xf";
+
+    ok ((o = ioencode ("stdout", "1", buffer, sizeof (buffer), false)) != NULL,
+        "ioencode of binary data works");
+    ok (json_unpack (o, "{s:s}", "encoding", &encoding) == 0,
+        "ioencode used alternate encoding");
+    is (encoding, "base64",
+        "ioencode encoded data as base64");
+    ok (iodecode (o, &stream, &rank, &data, &len, &eof) == 0,
+        "iodecode success");
+    is (rank, "1",
+        "rank is correct");
+    ok (len == sizeof (buffer),
+        "len is correct");
+    ok (eof == false,
+        "eof is correct");
+    ok (memcmp (data, buffer, len) == 0,
+        "data matches");
+    free (data);
+    ok (iodecode (o, &stream, &rank, NULL, &len, &eof) == 0,
+        "iodecode can be passed NULL data to query len");
+    ok (len == sizeof (buffer),
+        "len is correct");
+    json_decref (o);
 }
 
 int main (int argc, char *argv[])
@@ -83,6 +141,7 @@ int main (int argc, char *argv[])
 
     basic_corner_case ();
     basic ();
+    binary_data ();
 
     done_testing ();
 
diff --git a/src/common/libjob/Makefile.am b/src/common/libjob/Makefile.am
index 8fbe2597cf6a..9d729223b178 100644
--- a/src/common/libjob/Makefile.am
+++ b/src/common/libjob/Makefile.am
@@ -6,58 +6,82 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) \
-	$(LIBUUID_CFLAGS) \
+	-I$(top_srcdir)/src/common/libccan \
 	$(JANSSON_CFLAGS) \
-	$(FLUX_SECURITY_CFLAGS) \
-	$(LIBSODIUM_CFLAGS)
+	$(FLUX_SECURITY_CFLAGS)
 
 noinst_LTLIBRARIES = libjob.la
-fluxcoreinclude_HEADERS = job.h
+fluxcoreinclude_HEADERS = \
+	job.h \
+	jobspec1.h
 
 libjob_la_SOURCES = \
 	job.c \
+	submit.c \
+	wait.c \
+	list.c \
+	result.c \
+	kvs.c \
+	info.c \
+	id.c \
+	state.c \
 	sign_none.c \
 	sign_none.h \
 	job_hash.c \
-	job_hash.h
+	job_hash.h \
+	jobspec1.c \
+	jobspec1_private.h \
+	strtab.c \
+	strtab.h \
+	jj.c \
+	jj.h \
+	unwrap.c \
+	unwrap.h \
+	idf58.h
 
 TESTS = \
 	test_job.t \
-	test_sign_none.t
+	test_sign_none.t \
+	test_unwrap.t \
+	test_jobspec1.t
 
 check_PROGRAMS = \
-        $(TESTS)
+	$(TESTS)
 
 TEST_EXTENSIONS = .t
 T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
-        $(top_srcdir)/config/tap-driver.sh
+	$(top_srcdir)/config/tap-driver.sh
 
 test_ldadd = \
-        $(top_builddir)/src/common/libjob/libjob.la \
-        $(top_builddir)/src/common/libflux/libflux.la \
-        $(top_builddir)/src/common/libflux-internal.la \
-        $(top_builddir)/src/common/libtap/libtap.la \
-        $(ZMQ_LIBS) \
-	$(LIBUUID_LIBS) \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(top_builddir)/src/common/libflux/libflux.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libtap/libtap.la \
 	$(JANSSON_LIBS) \
 	$(LIBPTHREAD) \
-	$(LIBRT) \
-        $(FLUX_SECURITY_LIBS) \
-	$(LIBDL) \
-	$(LIBSODIUM_LIBS)
+	$(FLUX_SECURITY_LIBS)
 
 test_cppflags = \
-        $(AM_CPPFLAGS) \
-        -I$(top_srcdir)/src/common/libtap
+	$(AM_CPPFLAGS) \
+	-I$(top_srcdir)/src/common/libtap
 
 test_job_t_SOURCES = test/job.c
+test_job_t_CFLAGS = -Wno-deprecated-declarations
 test_job_t_CPPFLAGS = $(test_cppflags)
-test_job_t_LDADD = $(test_ldadd) $(LIBDL)
+test_job_t_LDADD = $(test_ldadd)
 
 test_sign_none_t_SOURCES = test/sign_none.c
 test_sign_none_t_CPPFLAGS = $(test_cppflags)
-test_sign_none_t_LDADD = $(test_ldadd) $(LIBDL)
+test_sign_none_t_LDADD = $(test_ldadd)
+
+test_unwrap_t_SOURCES = test/unwrap.c
+test_unwrap_t_CPPFLAGS = $(test_cppflags)
+test_unwrap_t_LDADD = $(test_ldadd)
+
+test_jobspec1_t_SOURCES = test/jobspec1.c
+test_jobspec1_t_CPPFLAGS = $(test_cppflags)
+test_jobspec1_t_LDADD = $(test_ldadd)
diff --git a/src/common/libjob/id.c b/src/common/libjob/id.c
new file mode 100644
index 000000000000..ba35264b398e
--- /dev/null
+++ b/src/common/libjob/id.c
@@ -0,0 +1,107 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <ctype.h>
+#include <flux/core.h>
+
+#include "job.h"
+
+#include "ccan/str/str.h"
+#include "src/common/libutil/fluid.h"
+
+int flux_job_id_parse (const char *s, flux_jobid_t *idp)
+{
+    int len;
+    const char *p = s;
+    if (s == NULL
+        || idp == NULL
+        || (len = strlen (s)) == 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    /*  Remove leading whitespace
+     */
+    while (isspace(*p))
+        p++;
+    /*  Ignore any `job.` prefix. This allows a "kvs" encoding
+     *   created by flux_job_id_encode(3) to properly decode.
+     */
+    if (strstarts (p, "job."))
+        p += 4;
+    return fluid_parse (p, idp);
+}
+
+int flux_job_id_encode (flux_jobid_t id,
+                        const char *type,
+                        char *buf,
+                        size_t bufsz)
+{
+    fluid_string_type_t t;
+    if (buf == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (type == NULL || strcasecmp (type, "dec") == 0) {
+        int len = snprintf (buf, bufsz, "%ju", (uintmax_t) id);
+        if (len >= bufsz) {
+            errno = EOVERFLOW;
+            return -1;
+        }
+        return 0;
+    }
+    if (strcasecmp (type, "hex") == 0) {
+        int len = snprintf (buf, bufsz, "0x%jx", (uintmax_t) id);
+        if (len >= bufsz) {
+            errno = EOVERFLOW;
+            return -1;
+        }
+        return 0;
+    }
+
+    /* The following encodings all use fluid_encode(3).
+     */
+    if (strcasecmp (type, "kvs") == 0) {
+        /* kvs: prepend "job." to "dothex" encoding.
+         */
+        int len = snprintf (buf, bufsz, "job.");
+        if (len >= bufsz) {
+            errno = EOVERFLOW;
+            return -1;
+        }
+        buf += len;
+        bufsz -= len;
+        type = "dothex";
+    }
+    if (strcasecmp (type, "dothex") == 0)
+        t = FLUID_STRING_DOTHEX;
+    else if (strcasecmp (type, "words") == 0)
+        t = FLUID_STRING_MNEMONIC;
+    else if (strcasecmp (type, "f58") == 0)
+        t = FLUID_STRING_F58;
+    else if (strcasecmp (type, "f58plain") == 0)
+        t = FLUID_STRING_F58_PLAIN;
+    else if (strcasecmp (type, "emoji") == 0)
+        t = FLUID_STRING_EMOJI;
+    else {
+        /*  Return EPROTO for invalid type to differentiate from
+         *   other invalid arguments.
+         */
+        errno = EPROTO;
+        return -1;
+    }
+    return fluid_encode (buf, bufsz, id, t);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/idf58.h b/src/common/libjob/idf58.h
new file mode 100644
index 000000000000..842c8c2f24ea
--- /dev/null
+++ b/src/common/libjob/idf58.h
@@ -0,0 +1,35 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _IDF58_H
+#define _IDF58_H
+
+#include <flux/core.h>
+
+/*  Convenience function to convert a flux_jobid_t to F58 encoding
+ *  If the encode fails (unlikely), then the decimal encoding is returned.
+ */
+static inline const char *idf58 (flux_jobid_t id)
+{
+    static __thread char buf[21];
+    if (flux_job_id_encode (id, "f58", buf, sizeof (buf)) < 0) {
+        /* 64bit integer is guaranteed to fit in 21 bytes
+         * floor(log(2^64-1)/log(1)) + 1 = 20
+         */
+        (void) sprintf (buf, "%ju", (uintmax_t) id);
+    }
+    return buf;
+}
+
+#endif /* !_IDF58_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/info.c b/src/common/libjob/info.c
new file mode 100644
index 000000000000..f4972065c219
--- /dev/null
+++ b/src/common/libjob/info.c
@@ -0,0 +1,72 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "job.h"
+
+flux_future_t *flux_job_event_watch (flux_t *h, flux_jobid_t id,
+                                     const char *path, int flags)
+{
+    flux_future_t *f;
+    const char *topic = "job-info.eventlog-watch";
+    int rpc_flags = FLUX_RPC_STREAMING;
+    int valid_flags = FLUX_JOB_EVENT_WATCH_WAITCREATE;
+
+    if (!h || !path || (flags & ~valid_flags)) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(f = flux_rpc_pack (h, topic, FLUX_NODEID_ANY, rpc_flags,
+                             "{s:I s:s s:i}",
+                             "id", id,
+                             "path", path,
+                             "flags", flags)))
+        return NULL;
+    return f;
+}
+
+int flux_job_event_watch_get (flux_future_t *f, const char **event)
+{
+    const char *s;
+
+    if (flux_rpc_get_unpack (f, "{s:s}", "event", &s) < 0)
+        return -1;
+    if (event)
+        *event = s;
+    return 0;
+}
+
+int flux_job_event_watch_cancel (flux_future_t *f)
+{
+    flux_future_t *f2;
+    const char *topic = "job-info.eventlog-watch-cancel";
+
+    if (!f) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(f2 = flux_rpc_pack (flux_future_get_flux (f),
+                              topic,
+                              FLUX_NODEID_ANY,
+                              FLUX_RPC_NORESPONSE,
+                              "{s:i}",
+                              "matchtag", (int)flux_rpc_get_matchtag (f))))
+        return -1;
+    flux_future_destroy (f2);
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/jj.c b/src/common/libjob/jj.c
new file mode 100644
index 000000000000..0ebab881f778
--- /dev/null
+++ b/src/common/libjob/jj.c
@@ -0,0 +1,165 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <string.h>
+#include <jansson.h>
+
+#include "ccan/str/str.h"
+
+#include "jj.h"
+
+static int jj_read_level (json_t *o, int level, struct jj_counts *jj);
+
+static int jj_read_vertex (json_t *o, int level, struct jj_counts *jj)
+{
+    int count;
+    const char *type = NULL;
+    json_t *with = NULL;
+    json_error_t error;
+    int exclusive = 0;
+
+    if (json_unpack_ex (o, &error, 0, "{ s:s s:i s?b s?o }",
+                       "type", &type,
+                       "count", &count,
+                       "exclusive", &exclusive,
+                       "with", &with) < 0) {
+        snprintf (jj->error, sizeof (jj->error) - 1,
+                  "level %d: %s", level, error.text);
+        errno = EINVAL;
+        return -1;
+    }
+    if (count <= 0) {
+        sprintf (jj->error, "Invalid count %d for type '%s'",
+                            count, type);
+        errno = EINVAL;
+        return -1;
+    }
+    if (streq (type, "node")) {
+        jj->nnodes = count;
+        if (exclusive)
+            jj->exclusive = true;
+    }
+    else if (streq (type, "slot"))
+        jj->nslots = count;
+    else if (streq (type, "core"))
+        jj->slot_size = count;
+    else if (streq (type, "gpu"))
+        jj->slot_gpus = count;
+    else {
+        sprintf (jj->error, "Unsupported resource type '%s'", type);
+        errno = EINVAL;
+        return -1;
+    }
+    if (with)
+        return jj_read_level (with, level+1, jj);
+    return 0;
+
+}
+
+static int jj_read_level (json_t *o, int level, struct jj_counts *jj)
+{
+    int i;
+    json_t *v = NULL;
+
+    if (!json_is_array (o)) {
+        snprintf (jj->error, sizeof (jj->error) - 1,
+                  "level %d: must be an array", level);
+        errno = EINVAL;
+        return -1;
+    }
+    json_array_foreach (o, i, v) {
+        if (jj_read_vertex (v, level, jj) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+int jj_get_counts (const char *spec, struct jj_counts *jj)
+{
+    json_t *o = NULL;
+    json_error_t error;
+    int rc = -1;
+
+    if ((o = json_loads (spec, 0, &error)) == NULL) {
+        snprintf (jj->error, sizeof (jj->error) - 1,
+                  "JSON load: %s", error.text);
+        errno = EINVAL;
+        return -1;
+    }
+
+    rc = jj_get_counts_json (o, jj);
+    json_decref (o);
+    return rc;
+}
+
+int jj_get_counts_json (json_t *jobspec, struct jj_counts *jj)
+{
+    int version;
+    json_t *resources = NULL;
+    json_error_t error;
+
+    if (!jj) {
+        errno = EINVAL;
+        return -1;
+    }
+    memset (jj, 0, sizeof (*jj));
+
+    if (json_unpack_ex (jobspec, &error, 0, "{s:i s:o}",
+                        "version", &version,
+                        "resources", &resources) < 0) {
+        snprintf (jj->error, sizeof (jj->error) - 1,
+                  "at top level: %s", error.text);
+        errno = EINVAL;
+        return -1;
+    }
+    if (version != 1) {
+        snprintf (jj->error, sizeof (jj->error) - 1,
+                 "Invalid version: expected 1, got %d", version);
+        errno = EINVAL;
+        return -1;
+    }
+    /* N.B. attributes.system is generally optional, but
+     * attributes.system.duration is required in jobspec version 1 */
+    if (json_unpack_ex (jobspec, &error, 0, "{s:{s:{s:F}}}",
+                        "attributes",
+                          "system",
+                            "duration", &jj->duration) < 0) {
+        snprintf (jj->error, sizeof (jj->error) - 1,
+                  "at top level: getting duration: %s", error.text);
+        errno = EINVAL;
+        return -1;
+    }
+    if (jj_read_level (resources, 0, jj) < 0)
+        return -1;
+
+    if (jj->nslots <= 0) {
+        snprintf (jj->error, sizeof (jj->error) - 1,
+                 "Unable to determine slot count");
+        errno = EINVAL;
+        return -1;
+    }
+    if (jj->slot_size <= 0) {
+        snprintf (jj->error, sizeof (jj->error) - 1,
+                 "Unable to determine slot size");
+        errno = EINVAL;
+        return -1;
+    }
+    if (jj->nnodes)
+        jj->nslots *= jj->nnodes;
+    return 0;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libjob/jj.h b/src/common/libjob/jj.h
new file mode 100644
index 000000000000..0f4ca6bf2d9e
--- /dev/null
+++ b/src/common/libjob/jj.h
@@ -0,0 +1,47 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_JJ_H
+#define HAVE_JJ_H 1
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <jansson.h>
+#include <stdbool.h>
+
+#define JJ_ERROR_TEXT_LENGTH 256
+
+struct jj_counts {
+    int nnodes;    /* total number of nodes requested */
+    int nslots;    /* total number of slots requested */
+    int slot_size; /* number of cores per slot        */
+    int slot_gpus; /* number of gpus per slot        */
+
+    bool exclusive;  /* enable node exclusive allocation if available */
+
+    double duration; /* attributes.system.duration if set */
+
+    char error[JJ_ERROR_TEXT_LENGTH]; /* On error, contains error description */
+};
+
+/*  Parse jobspec from json string `spec`, return resource request summary
+ *   in `counts` on success.
+ *  Returns 0 on success and -1 on failure with errno set and jj->error[]
+ *   with an error message string.
+ */
+
+int jj_get_counts (const char *spec, struct jj_counts *counts);
+
+/*  Identical to jj_get_counts, but take json_t  */
+int jj_get_counts_json (json_t *jobspec, struct jj_counts *counts);
+
+#endif /* !HAVE_JJ_H */
diff --git a/src/common/libjob/job.c b/src/common/libjob/job.c
index 2fc1e9da6aeb..9b11bd05af84 100644
--- a/src/common/libjob/job.c
+++ b/src/common/libjob/job.c
@@ -11,266 +11,13 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <unistd.h>
-#include <sys/types.h>
 #include <flux/core.h>
-#if HAVE_FLUX_SECURITY
-#include <flux/security/sign.h>
-#endif
 #include <jansson.h>
+#include <math.h>
 
-#include "job.h"
-#include "sign_none.h"
-#include "src/common/libutil/fluid.h"
-
-#if HAVE_FLUX_SECURITY
-/* If a textual error message is available in flux-security,
- * enclose it in a future and return.  Otherwise return NULL
- * with errno set to the original error.
- */
-static flux_future_t *get_security_error (flux_security_t *sec)
-{
-    int errnum = flux_security_last_errnum (sec);
-    const char *errmsg = flux_security_last_error (sec);
-    flux_future_t *f = NULL;
-
-    if (errmsg && (f = flux_future_create (NULL, NULL)))
-        flux_future_fulfill_error (f, errnum, errmsg);
-    errno = errnum;
-    return f;
-}
-
-/* Cache flux-security context in the handle on first use.
- * On failure, return NULL and set the value of 'f_error' to NULL,
- * or if a textual error message is available such as from config
- * file parsing, set the value of 'f_error' to a future containing
- * the textual error.
- */
-static flux_security_t *get_security_ctx (flux_t *h, flux_future_t **f_error)
-{
-    const char *auxkey = "flux::job_security_ctx";
-    flux_security_t *sec = flux_aux_get (h, auxkey);
-
-    if (!sec) {
-        if (!(sec = flux_security_create (0)))
-            goto error;
-        if (flux_security_configure (sec, NULL) < 0)
-            goto error;
-        if (flux_aux_set (h, auxkey, sec,
-                          (flux_free_f)flux_security_destroy) < 0)
-            goto error;
-    }
-    return sec;
-error:
-    *f_error = sec ? get_security_error (sec) : NULL;
-    flux_security_destroy (sec);
-    return NULL;
-}
-#endif
-
-flux_future_t *flux_job_submit (flux_t *h, const char *jobspec, int priority,
-                                int flags)
-{
-    flux_future_t *f = NULL;
-    const char *J;
-    char *s = NULL;
-    int saved_errno;
-
-    if (!h || !jobspec) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if (!(flags & FLUX_JOB_PRE_SIGNED)) {
-#if HAVE_FLUX_SECURITY
-        flux_security_t *sec;
-        if (!(sec = get_security_ctx (h, &f)))
-            return f;
-        if (!(J = flux_sign_wrap (sec, jobspec, strlen (jobspec), NULL, 0)))
-            return get_security_error (sec);
-#else
-        if (!(s = sign_none_wrap (jobspec, strlen (jobspec), geteuid ())))
-            goto error;
-        J = s;
-#endif
-    }
-    else {
-        J = jobspec;
-        flags &= ~FLUX_JOB_PRE_SIGNED; // client only flag
-    }
-    if (!(f = flux_rpc_pack (h, "job-ingest.submit", FLUX_NODEID_ANY, 0,
-                             "{s:s s:i s:i}",
-                             "J", J,
-                             "priority", priority,
-                             "flags", flags)))
-        goto error;
-    return f;
-error:
-    saved_errno = errno;
-    free (s);
-    errno = saved_errno;
-    return NULL;
-}
-
-int flux_job_submit_get_id (flux_future_t *f, flux_jobid_t *jobid)
-{
-    flux_jobid_t id;
-
-    if (!f) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (flux_rpc_get_unpack (f, "{s:I}",
-                                "id", &id) < 0)
-        return -1;
-    if (jobid)
-        *jobid = id;
-    return 0;
-}
-
-flux_future_t *flux_job_wait (flux_t *h, flux_jobid_t id)
-{
-    if (!h) {
-        errno = EINVAL;
-        return NULL;
-    }
-    return flux_rpc_pack (h,
-                          "job-manager.wait",
-                          FLUX_NODEID_ANY,
-                          0,
-                          "{s:I}",
-                          "id",
-                          id);
-}
-
-int flux_job_wait_get_status (flux_future_t *f,
-                              bool *successp,
-                              const char **errstrp)
-{
-    int success;
-    const char *errstr;
-
-    if (!f) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (flux_rpc_get_unpack (f,
-                             "{s:b s:s}",
-                             "success",
-                             &success,
-                             "errstr",
-                             &errstr) < 0)
-        return -1;
-    if (successp)
-        *successp = success ? true : false;
-    if (errstrp)
-        *errstrp = errstr;
-    return 0;
-}
-
-int flux_job_wait_get_id (flux_future_t *f, flux_jobid_t *jobid)
-{
-    flux_jobid_t id;
-
-    if (!f) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (flux_rpc_get_unpack (f, "{s:I}",
-                                "id", &id) < 0)
-        return -1;
-    if (jobid)
-        *jobid = id;
-    return 0;
-}
-
-flux_future_t *flux_job_list (flux_t *h,
-                              int max_entries,
-                              const char *json_str,
-                              uint32_t userid,
-                              int states)
-{
-    flux_future_t *f;
-    json_t *o = NULL;
-    int valid_states = (FLUX_JOB_PENDING
-                        | FLUX_JOB_RUNNING
-                        | FLUX_JOB_INACTIVE);
-    int saved_errno;
-
-    if (!h || max_entries < 0 || !json_str
-           || !(o = json_loads (json_str, 0, NULL))
-           || states & ~valid_states) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if (!(f = flux_rpc_pack (h, "job-info.list", FLUX_NODEID_ANY, 0,
-                             "{s:i s:o s:i s:i}",
-                             "max_entries", max_entries,
-                             "attrs", o,
-                             "userid", userid,
-                             "states", states))) {
-        saved_errno = errno;
-        json_decref (o);
-        errno = saved_errno;
-        return NULL;
-    }
-    return f;
-}
-
-flux_future_t *flux_job_list_inactive (flux_t *h,
-                                       int max_entries,
-                                       double since,
-                                       const char *json_str)
-{
-    flux_future_t *f;
-    json_t *o = NULL;
-    int saved_errno;
+#include "src/common/libutil/errprintf.h"
 
-    if (!h || max_entries < 0 || since < 0. || !json_str
-           || !(o = json_loads (json_str, 0, NULL))) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if (!(f = flux_rpc_pack (h, "job-info.list-inactive", FLUX_NODEID_ANY, 0,
-                             "{s:i s:f s:o}",
-                             "max_entries", max_entries,
-                             "since", since,
-                             "attrs", o))) {
-        saved_errno = errno;
-        json_decref (o);
-        errno = saved_errno;
-        return NULL;
-    }
-    return f;
-}
-
-flux_future_t *flux_job_list_id (flux_t *h,
-                                 flux_jobid_t id,
-                                 const char *json_str)
-{
-    flux_future_t *f;
-    json_t *o = NULL;
-    int saved_errno;
-
-    if (!h || !json_str
-           || !(o = json_loads (json_str, 0, NULL))) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if (!(f = flux_rpc_pack (h, "job-info.list-id", FLUX_NODEID_ANY, 0,
-                             "{s:I s:O}",
-                             "id", id,
-                             "attrs", o)))
-        goto error;
-
-    json_decref (o);
-    return f;
-
-error:
-    saved_errno = errno;
-    json_decref (o);
-    errno = saved_errno;
-    return NULL;
-}
+#include "job.h"
 
 flux_future_t *flux_job_raise (flux_t *h, flux_jobid_t id,
                                const char *type, int severity, const char *note)
@@ -325,7 +72,7 @@ flux_future_t *flux_job_kill (flux_t *h, flux_jobid_t id, int signum)
                              "signum", signum);
 }
 
-flux_future_t *flux_job_set_priority (flux_t *h, flux_jobid_t id, int priority)
+flux_future_t *flux_job_set_urgency (flux_t *h, flux_jobid_t id, int urgency)
 {
     flux_future_t *f;
 
@@ -333,195 +80,154 @@ flux_future_t *flux_job_set_priority (flux_t *h, flux_jobid_t id, int priority)
         errno = EINVAL;
         return NULL;
     }
-    if (!(f = flux_rpc_pack (h, "job-manager.priority", FLUX_NODEID_ANY, 0,
+    if (!(f = flux_rpc_pack (h, "job-manager.urgency", FLUX_NODEID_ANY, 0,
                              "{s:I s:i}",
                              "id", id,
-                             "priority", priority)))
+                             "urgency", urgency)))
         return NULL;
     return f;
 }
 
-static int buffer_arg_check (char *buf, int bufsz)
-{
-    if (!buf || bufsz <= 0) {
-        errno = EINVAL;
-        return -1;
-    }
-    return 0;
-}
-
-int flux_job_kvs_key (char *buf, int bufsz, flux_jobid_t id, const char *key)
-{
-    char idstr[32];
-    int len;
-
-    if (buffer_arg_check (buf, bufsz) < 0)
-        return -1;
-
-    if (fluid_encode (idstr, sizeof (idstr), id, FLUID_STRING_DOTHEX) < 0)
-        return -1;
-    len = snprintf (buf, bufsz, "job.%s%s%s",
-                    idstr,
-                    key ? "." : "",
-                    key ? key : "");
-    if (len >= bufsz) {
-        errno = EOVERFLOW;
-        return -1;
-    }
-    return len;
-}
-
-int flux_job_kvs_guest_key (char *buf,
-                            int bufsz,
-                            flux_jobid_t id,
-                            const char *key)
+int flux_job_timeleft (flux_t *h, flux_error_t *errp, double *timeleft)
 {
-    char idstr[32];
-    int len;
+    flux_jobid_t id;
+    flux_job_state_t state;
+    const char *s;
+    double expiration = 0.;
+    flux_future_t *f = NULL;
+    flux_t *parent_h = NULL;
+    int rc = -1;
 
-    if (buffer_arg_check (buf, bufsz) < 0)
-        return -1;
-    if (getenv ("FLUX_KVS_NAMESPACE"))
-        len = snprintf (buf, bufsz, "%s", key ? key : ".");
-    else {
-        if (fluid_encode (idstr, sizeof (idstr), id, FLUID_STRING_DOTHEX) < 0)
+    if (!h || !timeleft) {
+        errno = EINVAL;
+        return errprintf (errp, "Invalid argument");
+    }
+
+    /*  Check for FLUX_JOB_ID environment variable. If set, this process
+     *  is part of a job in the current instance. If not, then check to
+     *  see if this process is part of an "initial program".
+     */
+    if (!(s = getenv ("FLUX_JOB_ID"))) {
+        const char *uri;
+
+        /*  Check if we're in "initial program" context.
+         *  If not, then this instance may be the system instance, or
+         *  may be a job in a foreign RM. Either way we cannot provide
+         *  a remaining time, so return an error.
+         */
+        if (!(s = flux_attr_get (h, "jobid"))) {
+            errprintf (errp,
+                       "unable to associate this process with a Flux jobid");
             return -1;
-        len = snprintf (buf, bufsz, "job.%s.guest%s%s",
-                        idstr,
-                        key ? "." : "",
-                        key ? key : "");
-    }
-    if (len >= bufsz) {
-        errno = EOVERFLOW;
-        return -1;
-    }
-    return len;
-}
+        }
 
-int flux_job_kvs_namespace (char *buf, int bufsz, flux_jobid_t id)
-{
-    int len;
-    if (buffer_arg_check (buf, bufsz) < 0)
-        return -1;
-    if ((len = snprintf (buf, bufsz, "job-%ju", id)) >= bufsz) {
-        errno = EOVERFLOW;
-        return -1;
+        /* This is an initial program. Switch the handle to the parent */
+        if (!(uri = flux_attr_get (h, "parent-uri")))
+            return errprintf (errp,
+                              "failed to get parent-uri attribute: %s",
+                              strerror (errno));
+        if (!(parent_h = flux_open (uri, 0)))
+            return errprintf (errp,
+                              "failed to connect to parent instance: %s",
+                              strerror (errno));
+        h = parent_h;
     }
-    return len;
-}
 
-flux_future_t *flux_job_event_watch (flux_t *h, flux_jobid_t id,
-                                     const char *path, int flags)
-{
-    flux_future_t *f;
-    const char *topic = "job-info.eventlog-watch";
-    int rpc_flags = FLUX_RPC_STREAMING;
-    bool guest = false;
+    /*  Parse jobid and lookup expiration
+     */
+    if (flux_job_id_parse (s, &id) < 0) {
+        errprintf (errp, "failed to parse jobid %s", s);
+        goto out;
+    }
 
-    /* No flags supported yet */
-    if (!h || !path || flags) {
-        errno = EINVAL;
-        return NULL;
+    /*  Fetch job expiration from this or parent's job-list service
+     */
+    if (!(f = flux_job_list_id (h, id, "[\"expiration\", \"state\"]"))) {
+        errprintf (errp, "flux_job_list_id: %s: %s", s, strerror (errno));
+        goto out;
     }
-    if (path && !strncmp (path, "guest.", 6)) {
-        topic = "job-info.guest-eventlog-watch";
-        path += 6;
-        guest = true;
+    if (flux_rpc_get_unpack (f,
+                             "{s:{s?f s:i}}",
+                             "job",
+                             "expiration", &expiration,
+                             "state", &state) < 0) {
+        if (errno == ENOENT)
+            errprintf (errp, "%s: no such jobid", s);
+        else
+            errprintf (errp,
+                       "flux_job_list_id: %s: %s",
+                       s,
+                       future_strerror (f, errno));
+        goto out;
+    }
+    if (state & FLUX_JOB_STATE_PENDING) {
+        /*  The remaining time for a pending job is undefined, so report
+         *  an error instead.
+         */
+        errprintf (errp, "job %s has not started", s);
+        goto out;
+    }
+    else if (state != FLUX_JOB_STATE_RUN) {
+        /*  Only jobs in RUN state have any time left.
+         *  Return 0 for jobs in any other state besides RUN.
+         */
+        *timeleft = 0.;
+    }
+    else if (expiration == 0.) {
+        /*  If expiration is 0. then job time left is unlimited.
+         *  Return INFINITY.
+         */
+        *timeleft = INFINITY;
     }
-    if (!(f = flux_rpc_pack (h, topic, FLUX_NODEID_ANY, rpc_flags,
-                             "{s:I s:s s:i}",
-                             "id", id,
-                             "path", path,
-                             "flags", flags)))
-        return NULL;
-    if (guest) {
-        /* value not relevant, set to anything */
-        if (flux_future_aux_set (f, "guest", "", NULL) < 0) {
-            flux_future_destroy (f);
-            return NULL;
-        }
+    else {
+        *timeleft = expiration - flux_reactor_now (flux_get_reactor (h));
+        /*  Avoid returning negative number. If expiration has elapsed,
+         *  then the time remaining is 0.
+         */
+        if (*timeleft < 0.)
+            *timeleft = 0.;
+    }
+    rc = 0;
+out:
+    flux_future_destroy (f);
+    flux_close (parent_h);
+    return rc;
+}
+
+int flux_job_waitstatus_to_exitcode (int waitstatus, flux_error_t *errp)
+{
+    int code;
+
+    /*  If waitstatus indicates WIFSIGNALED, then this means the job shell
+     *  was signaled, not the tasks. Report accordingly:
+     */
+    if (WIFSIGNALED (waitstatus)) {
+        /*  Whether the job shell or one or more tasks is terminated by a
+         *  signal, set the exit code to signal + 128
+         */
+        code = WTERMSIG (waitstatus) + 128;
+        errprintf (errp, "job shell %s", strsignal (WTERMSIG (waitstatus)));
+    }
+    else if (WIFEXITED (waitstatus)) {
+        code = WEXITSTATUS (waitstatus);
+        /*  If exit code > 128, then tasks were likely terminated by a
+         *  signal. (job shell returns 128+signo in this case)
+         */
+        if (code > 128)
+            errprintf (errp, "task(s) %s", strsignal (code - 128));
+        else if (code > 0)
+            errprintf (errp, "task(s) exited with exit code %d", code);
+        else /* Ensure errp->text is cleared */
+            err_init (errp);
     }
-    return f;
-}
-
-int flux_job_event_watch_get (flux_future_t *f, const char **event)
-{
-    const char *s;
-
-    if (flux_rpc_get_unpack (f, "{s:s}", "event", &s) < 0)
-        return -1;
-    if (event)
-        *event = s;
-    return 0;
-}
-
-int flux_job_event_watch_cancel (flux_future_t *f)
-{
-    flux_future_t *f2;
-    const char *topic = "job-info.eventlog-watch-cancel";
-
-    if (!f) {
+    else {
+        errprintf (errp, "unexpected wait(2) status %d", waitstatus);
+        code = -1;
         errno = EINVAL;
-        return -1;
-    }
-    if (flux_future_aux_get (f, "guest") != NULL)
-        topic = "job-info.guest-eventlog-watch-cancel";
-    if (!(f2 = flux_rpc_pack (flux_future_get_flux (f),
-                              topic,
-                              FLUX_NODEID_ANY,
-                              FLUX_RPC_NORESPONSE,
-                              "{s:i}",
-                              "matchtag", (int)flux_rpc_get_matchtag (f))))
-        return -1;
-    flux_future_destroy (f2);
-    return 0;
-}
-
-const char *flux_job_statetostr (flux_job_state_t state, bool single_char)
-{
-    switch (state) {
-        case FLUX_JOB_NEW:
-            return single_char ? "N" : "NEW";
-        case FLUX_JOB_DEPEND:
-            return single_char ? "D" : "DEPEND";
-        case FLUX_JOB_SCHED:
-            return single_char ? "S" : "SCHED";
-        case FLUX_JOB_RUN:
-            return single_char ? "R" : "RUN";
-        case FLUX_JOB_CLEANUP:
-            return single_char ? "C" : "CLEANUP";
-        case FLUX_JOB_INACTIVE:
-            return single_char ? "I" : "INACTIVE";
     }
-    return single_char ? "?" : "(unknown)";
+    return code;
 }
 
-int flux_job_strtostate (const char *s, flux_job_state_t *state)
-{
-    if (!s || !state)
-        goto inval;
-    if (!strcasecmp (s, "N") || !strcasecmp (s, "NEW"))
-        *state = FLUX_JOB_NEW;
-    else if (!strcasecmp (s, "D") || !strcasecmp (s, "DEPEND"))
-        *state = FLUX_JOB_DEPEND;
-    else if (!strcasecmp (s, "S") || !strcasecmp (s, "SCHED"))
-        *state = FLUX_JOB_SCHED;
-    else if (!strcasecmp (s, "R") || !strcasecmp (s, "RUN"))
-        *state = FLUX_JOB_RUN;
-    else if (!strcasecmp (s, "C") || !strcasecmp (s, "CLEANUP"))
-        *state = FLUX_JOB_CLEANUP;
-    else if (!strcasecmp (s, "I") || !strcasecmp (s, "INACTIVE"))
-        *state = FLUX_JOB_INACTIVE;
-    else
-        goto inval;
-
-    return 0;
-inval:
-    errno = EINVAL;
-    return -1;
-}
-
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libjob/job.h b/src/common/libjob/job.h
index 159bf631c178..19cb4aacb5aa 100644
--- a/src/common/libjob/job.h
+++ b/src/common/libjob/job.h
@@ -23,49 +23,118 @@ enum job_submit_flags {
     FLUX_JOB_PRE_SIGNED = 1,    // 'jobspec' is already signed
     FLUX_JOB_DEBUG = 2,
     FLUX_JOB_WAITABLE = 4,      // flux_job_wait() will be used on this job
+    FLUX_JOB_NOVALIDATE = 8,    // don't validate jobspec (instance owner only)
 };
 
-enum job_priority {
-    FLUX_JOB_PRIORITY_MIN = 0,
-    FLUX_JOB_PRIORITY_DEFAULT = 16,
-    FLUX_JOB_PRIORITY_MAX = 31,
+enum job_event_watch_flags {
+    FLUX_JOB_EVENT_WATCH_WAITCREATE = 1, // wait for path to exist
 };
 
-enum {
-    FLUX_JOBID_ANY = 0xFFFFFFFFFFFFFFFF, // ~(uint64_t)0
+enum job_lookup_flags {
+    /* return special fields as decoded JSON objects instead of strings
+     * - currently works for jobspec and R
+     */
+    FLUX_JOB_LOOKUP_JSON_DECODE = 1,
+    /* get current value of special fields by applying eventlog
+     * updates for those fields
+     * - currently works for jobspec and R
+     */
+    FLUX_JOB_LOOKUP_CURRENT = 2,
+};
+
+enum job_urgency {
+    FLUX_JOB_URGENCY_MIN = 0,
+    FLUX_JOB_URGENCY_HOLD = FLUX_JOB_URGENCY_MIN,
+    FLUX_JOB_URGENCY_DEFAULT = 16,
+    FLUX_JOB_URGENCY_MAX = 31,
+    FLUX_JOB_URGENCY_EXPEDITE = FLUX_JOB_URGENCY_MAX,
+};
+
+enum job_queue_priority {
+    FLUX_JOB_PRIORITY_MIN = 0,
+    FLUX_JOB_PRIORITY_MAX = 4294967295,
 };
 
+// N.B. value is duplicated in python bindings
+#define FLUX_JOBID_ANY 0xFFFFFFFFFFFFFFFF // ~(uint64_t)0
+
 typedef enum {
-    FLUX_JOB_NEW                    = 1,
-    FLUX_JOB_DEPEND                 = 2,
-    FLUX_JOB_SCHED                  = 4,
-    FLUX_JOB_RUN                    = 8,
-    FLUX_JOB_CLEANUP                = 16,
-    FLUX_JOB_INACTIVE               = 32,   // captive end state
+    FLUX_JOB_STATE_NEW                    = 1,
+    FLUX_JOB_STATE_DEPEND                 = 2,
+    FLUX_JOB_STATE_PRIORITY               = 4,
+    FLUX_JOB_STATE_SCHED                  = 8,
+    FLUX_JOB_STATE_RUN                    = 16,
+    FLUX_JOB_STATE_CLEANUP                = 32,
+    FLUX_JOB_STATE_INACTIVE               = 64,   // captive end state
 } flux_job_state_t;
 
+#define FLUX_JOB_NR_STATES 7
+
 /* Virtual states, for convenience.
  */
 enum {
-    FLUX_JOB_PENDING    = 6,    // (FLUX_JOB_DEPEND | FLUX_JOB_SCHED)
-    FLUX_JOB_RUNNING    = 24,   // (FLUX_JOB_RUN | FLUX_JOB_CLEANUP)
-    FLUX_JOB_ACTIVE     = 30,   // (FLUX_JOB_PENDING | FLUX_JOB_RUNNING)
+    /* FLUX_JOB_STATE_DEPEND | FLUX_JOB_STATE_PRIORITY | FLUX_JOB_STATE_SCHED */
+    FLUX_JOB_STATE_PENDING    = 14,
+    /* FLUX_JOB_STATE_RUN | FLUX_JOB_STATE_CLEANUP */
+    FLUX_JOB_STATE_RUNNING    = 48,
+    /* FLUX_JOB_STATE_PENDING | FLUX_JOB_STATE_RUNNING */
+    FLUX_JOB_STATE_ACTIVE     = 62,
 };
 
+/* Result of a job
+ */
+typedef enum {
+    FLUX_JOB_RESULT_COMPLETED = 1,
+    FLUX_JOB_RESULT_FAILED = 2,
+    FLUX_JOB_RESULT_CANCELED = 4,
+    FLUX_JOB_RESULT_TIMEOUT = 8,
+} flux_job_result_t;
+
 typedef uint64_t flux_jobid_t;
 
-const char *flux_job_statetostr (flux_job_state_t state, bool single_char);
+/*  Parse a jobid from NULL-terminated string 's' in any supported encoding.
+ *  Returns 0 on success, -1 on failure.
+ */
+int flux_job_id_parse (const char *s, flux_jobid_t *id);
+
+/*  Encode a jobid into encoding "type", writing the result to buffer
+ *   buf of size bufsz.
+ *  Supported encoding types include:
+ *   "dec", "hex", "kvs", "dothex", "words", "f58", or "f58plain"
+ *  Returns 0 on success, -1 on failure with errno set:
+ *   EPROTO: Invalid encoding type
+ *   EINVAL: Invalid other argument
+ */
+int flux_job_id_encode (flux_jobid_t id, const char *type,
+                        char *buf, size_t bufsz);
 
+/* Convert state to string.  'fmt' may be:
+ * "s" - lower case short name
+ * "S" - upper case short name
+ * "l" - lower case long name
+ * "L" - upper case long name
+ * This function always returns a valid string, though it may
+ * be something like "(unknown)".
+ */
+const char *flux_job_statetostr (flux_job_state_t state, const char *fmt);
+
+/* Convert state string in any of the forms produced by flux_job_statetostr()
+ * back to state.  Returns 0 on success, -1 on failure with errno set.
+ */
 int flux_job_strtostate (const char *s, flux_job_state_t *state);
 
+const char *flux_job_resulttostr (flux_job_result_t result, const char *fmt);
+
+int flux_job_strtoresult (const char *s, flux_job_result_t *result);
+
 /* Submit a job to the system.
  * 'jobspec' should be RFC 14 jobspec.
- * 'priority' should be a value from 0 to 31 (16 if not instance owner).
+ * 'urgency' should be a value from 0 to 31 (16 if not instance owner).
  * 'flags' should be 0 for now.
  * The system assigns a jobid and returns it in the response.
  */
 flux_future_t *flux_job_submit (flux_t *h, const char *jobspec,
-                                int priority, int flags);
+                                int urgency, int flags);
 
 /* Parse jobid from response to flux_job_submit() request.
  * Returns 0 on success, -1 on failure with errno set - and an extended
@@ -83,42 +152,22 @@ int flux_job_wait_get_status (flux_future_t *f,
                               const char **errstr);
 int flux_job_wait_get_id (flux_future_t *f, flux_jobid_t *id);
 
-/* Request a list of jobs.
- * If 'max_entries' > 0, fetch at most that many jobs.
- * 'json_str' is an encoded JSON array of attribute strings, e.g.
- * ["id","userid",...] that will be returned in response.
- *
- * Process the response payload with flux_rpc_get() or flux_rpc_get_unpack().
- * It is a JSON object containing an array of job objects, e.g.
- * { "jobs":[
- *   {"id":m, "userid":n},
- *   {"id":m, "userid":n},
- *   ...
- * ])
- *
- * states can be set to an OR of any job state or any virtual job
- * states to retrieve jobs of only those states.  Specify 0 for all
- * states.
- */
-flux_future_t *flux_job_list (flux_t *h,
-                              int max_entries,
-                              const char *json_str,
-                              uint32_t userid,
-                              int states);
+FLUX_DEPRECATED(flux_future_t *flux_job_list (flux_t *,
+                                              int,
+                                              const char *,
+                                              uint32_t,
+                                              int));
 
-/* Similar to flux_job_list(), but retrieve inactive jobs newer
- * than a timestamp.
- */
-flux_future_t *flux_job_list_inactive (flux_t *h,
-                                       int max_entries,
-                                       double since,
-                                       const char *json_str);
+FLUX_DEPRECATED(flux_future_t *flux_job_list_inactive (flux_t *,
+                                                       int,
+                                                       double,
+                                                       const char *));
 
 /* Similar to flux_job_list(), but retrieve job info for a single
  * job id */
 flux_future_t *flux_job_list_id (flux_t *h,
                                  flux_jobid_t id,
-                                 const char *json_str);
+                                 const char *attrs_json_str);
 
 /* Raise an exception for job.
  * Severity is 0-7, with severity=0 causing the job to abort.
@@ -137,9 +186,9 @@ flux_future_t *flux_job_cancel (flux_t *h, flux_jobid_t id, const char *reason);
  */
 flux_future_t *flux_job_kill (flux_t *h, flux_jobid_t id, int signum);
 
-/* Change job priority.
+/* Change job urgency.
  */
-flux_future_t *flux_job_set_priority (flux_t *h, flux_jobid_t id, int priority);
+flux_future_t *flux_job_set_urgency (flux_t *h, flux_jobid_t id, int urgency);
 
 /* Write KVS path to 'key' relative to job directory for job 'id'.
  * If key=NULL, write the job directory.
@@ -171,6 +220,75 @@ flux_future_t *flux_job_event_watch (flux_t *h, flux_jobid_t id,
 int flux_job_event_watch_get (flux_future_t *f, const char **event);
 int flux_job_event_watch_cancel (flux_future_t *f);
 
+/*  Wait for job to reach its terminal state and fetch the job result
+ *   along with other ancillary information from the job eventlog.
+ */
+flux_future_t *flux_job_result (flux_t *h, flux_jobid_t id, int flags);
+
+/*  Get the job result "payload" as a JSON string
+ */
+int flux_job_result_get (flux_future_t *f,
+                         const char **json_str);
+
+/*  Decode and unpack the result payload from future `f`.
+ *  The result object contains the following information:
+ *
+ *   {
+ *     "id":i,                 jobid
+ *     "result:i,              flux_job_result_t
+ *     "t_submit":f,           timestamp of job submit event
+ *     "t_run":f,              timestamp of job alloc event
+ *     "t_cleanup":f,          timestamp of job finish event
+ *     "waitstatus"?i,         wait status (if job started)
+ *     "exception_occurred":b, true if job exception occurred
+ *     "exception_severity"?i, exception severity (if exception)
+ *     "exception_type"?s,     exception type (if exception)
+ *     "exception_note"?s      exception note (if exception)
+ *   }
+ *
+ */
+int flux_job_result_get_unpack (flux_future_t *f, const char *fmt, ...);
+
+
+/*  Get remaining time in floating point seconds for the current job or
+ *  enclosing instancce (i.e., if the current process is associated with
+ *  a flux instance, but is not part of a parallel job).
+ *
+ *  Returns 0 on success with timeleft assigned to the remaining time.
+ *  If there is no expiration in the current context (e.g. the job has
+ *  no timelimit), then timeleft is set to infinity. If the job is not
+ *  in RUN state, or the job has expired, then timeleft is set to 0.
+ *
+ *  Returns -1 with error string assigned to 'errp' on failure.
+ */
+int flux_job_timeleft (flux_t *h, flux_error_t *errp, double *timeleft);
+
+/*  Unwrap signed data to a NUL terminated string, e.g. J -> jobspec.
+ *
+ *  If verify is true, then fail if signing mechanism is invalid or
+ *   signing user does not match current uid. On failure, error.text
+ *   is filled in with an error message. (errno not necessarily
+ *   guaranteed to be valid).
+ *
+ *  Works when flux-core is built with or without flux-security
+ *
+ *  Caller must free returned value if non-NULL.
+ *
+ *  Predominantly used by flux-core internally and associated
+ *  libraries/tools.
+ */
+char *flux_unwrap_string (const char *in,
+                          bool verify,
+                          uint32_t *userid,
+                          flux_error_t *error);
+
+
+/* Convert the waitstatus from a job `finish` event to an exit code.
+ * If the job exited with nonzero status, then place an appropriate error
+ * message in errp->text.
+ */
+int flux_job_waitstatus_to_exitcode (int waitstatus, flux_error_t *errp);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/common/libjob/job_hash.c b/src/common/libjob/job_hash.c
index 64aba7237b9c..a17081331667 100644
--- a/src/common/libjob/job_hash.c
+++ b/src/common/libjob/job_hash.c
@@ -12,7 +12,8 @@
 #include "config.h"
 #endif
 #include <flux/core.h>
-#include <czmq.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "job_hash.h"
 
diff --git a/src/common/libjob/job_hash.h b/src/common/libjob/job_hash.h
index e348eb0b7500..c14d26533d6a 100644
--- a/src/common/libjob/job_hash.h
+++ b/src/common/libjob/job_hash.h
@@ -12,7 +12,8 @@
 #define _JOB_HASH_H
 
 #include <flux/core.h>
-#include <czmq.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 /* Create a zhashx_t with hasher and comparator set to use flux_jobid_t
  * as the hash key.  The default key duplicator and destructor are disabled
diff --git a/src/common/libjob/jobspec1.c b/src/common/libjob/jobspec1.c
new file mode 100644
index 000000000000..5fc57d1a4f4e
--- /dev/null
+++ b/src/common/libjob/jobspec1.c
@@ -0,0 +1,845 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <jansson.h>
+#include <string.h>
+#include <stdbool.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/jpath.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "jobspec1.h"
+#include "jobspec1_private.h"
+
+struct flux_jobspec1 {
+    json_t *obj;
+};
+
+/* Return a new reference to a json array of strings. */
+static json_t *argv_to_json (int argc, char **argv)
+{
+    int i;
+    json_t *a, *o;
+
+    if (!(a = json_array ()))
+        goto nomem;
+    for (i = 0; i < argc; i++) {
+        if (!(o = json_string (argv[i])) || json_array_append_new (a, o) < 0) {
+            json_decref (o);
+            goto nomem;
+        }
+    }
+    return a;
+nomem:
+    json_decref (a);
+    errno = ENOMEM;
+    return NULL;
+}
+
+static json_t *jobspec1_attr_get (flux_jobspec1_t *jobspec, const char *name)
+{
+    char *path;
+    json_t *val;
+
+    if (!jobspec || !name) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (asprintf (&path, "attributes.%s", name) < 0)
+        return NULL;
+    val = jpath_get (jobspec->obj, path);
+    ERRNO_SAFE_WRAP (free, path);
+    return val;
+}
+
+static int jobspec1_attr_set (flux_jobspec1_t *jobspec,
+                              const char *name,
+                              json_t *val)
+{
+    char *path;
+    int rc;
+
+    if (!jobspec || !name || !val) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (asprintf (&path, "attributes.%s", name) < 0)
+        return -1;
+    rc = jpath_set (jobspec->obj, path, val);
+    ERRNO_SAFE_WRAP (free, path);
+    return rc;
+}
+
+int flux_jobspec1_attr_del (flux_jobspec1_t *jobspec, const char *name)
+{
+    char *path;
+    int rc;
+
+    if (!jobspec || !name) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (asprintf (&path, "attributes.%s", name) < 0)
+        return -1;
+    rc = jpath_del (jobspec->obj, path);
+    ERRNO_SAFE_WRAP (free, path);
+    return rc;
+}
+
+int flux_jobspec1_attr_unpack (flux_jobspec1_t *jobspec,
+                               const char *path,
+                               const char *fmt,
+                               ...)
+{
+    va_list ap;
+    json_t *val;
+    int rc;
+
+    if (!fmt) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(val = jobspec1_attr_get (jobspec, path)))
+        return -1;
+    va_start (ap, fmt);
+    rc = json_vunpack_ex (val, NULL, 0, fmt, ap);
+    va_end (ap);
+    return rc;
+}
+
+int flux_jobspec1_attr_pack (flux_jobspec1_t *jobspec,
+                             const char *path,
+                             const char *fmt,
+                             ...)
+{
+    va_list ap;
+    json_t *val;
+
+    if (!jobspec || !path || !fmt) {
+        errno = EINVAL;
+        return -1;
+    }
+    va_start (ap, fmt);
+    val = json_vpack_ex (NULL, 0, fmt, ap);
+    va_end (ap);
+    if (!val) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (jobspec1_attr_set (jobspec, path, val) < 0) {
+        ERRNO_SAFE_WRAP (json_decref, val);
+        return -1;
+    }
+    json_decref (val);
+    return 0;
+}
+
+static int tasks_check (json_t *tasks, flux_jobspec1_error_t *error)
+{
+    json_error_t json_error;
+    json_t *argv_json;
+    const char *slot;
+    json_t *count;
+    size_t index;
+    json_t *value;
+    int n;
+
+    if (json_unpack_ex (tasks,
+                        &json_error,
+                        0,
+                        "[{s:o s:s s:o !}]",
+                        "command", &argv_json,
+                        "slot", &slot,
+                        "count", &count) < 0) {
+        errprintf (error, "tasks section: %s", json_error.text);
+        goto error;
+    }
+    if (!json_is_array (argv_json)) {
+        errprintf (error, "tasks command must be an array");
+        goto error;
+    }
+    if (json_array_size (argv_json) < 1) {
+        errprintf (error, "tasks command array length must be >= 1");
+        goto error;
+    }
+    json_array_foreach (argv_json, index, value) {
+        if (!json_is_string (value)) {
+            errprintf (error, "tasks command array entry must be a string");
+            goto error;
+        }
+    }
+    if (json_unpack (count, "{s:i}", "per_slot", &n) == 0) {
+        if (n < 1) {
+            errprintf (error, "tasks per_slot count must be >= 1");
+            goto error;
+        }
+    }
+    else if (json_unpack (count, "{s:i}", "total", &n) == 0) {
+        if (n < 1) {
+            errprintf (error, "tasks total count must be >= 1");
+            goto error;
+        }
+    }
+    else {
+        errprintf (error, "tasks count object is malformed");
+        goto error;
+    }
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+static int slot_vertex_check (json_t *slot, flux_jobspec1_error_t *error)
+{
+    json_error_t json_error;
+    const char *type;
+    int count;
+    json_t *with;
+    const char *label;
+    int exclusive = 0;
+    size_t index;
+    json_t *value;
+
+    if (json_unpack_ex (slot,
+                        &json_error,
+                        0,
+                        "{s:s s:i s:o s:s s?b !}",
+                        "type", &type,
+                        "count", &count,
+                        "with", &with,
+                        "label", &label,
+                        "exclusive", &exclusive) < 0) {
+        errprintf (error, "slot vertex: %s", json_error.text);
+        goto error;
+    }
+    if (count < 1) {
+        errprintf (error, "slot count must be >= 1");
+        goto error;
+    }
+    if (!json_is_array (with)) {
+        errprintf (error, "slot with must be an array");
+        goto error;
+    }
+    if (json_array_size (with) != 1 && json_array_size (with) != 2) {
+        errprintf (error, "slot with array must have 1-2 elements");
+        goto error;
+    }
+    json_array_foreach (with, index, value) {
+        int min_count = 0;
+        if (json_unpack_ex (value,
+                            &json_error,
+                            0,
+                            "{s:s s:i !}",
+                            "type", &type,
+                            "count", &count) < 0) {
+            errprintf (error, "slot with: %s", json_error.text);
+            goto error;
+        }
+        if (!streq (type, "core") && !streq (type, "gpu")) {
+            errprintf (error, "slot with type must be core or gpu");
+            goto error;
+        }
+        if (streq (type, "core"))
+            min_count = 1;
+        if (count < min_count) {
+            errprintf (error, "slot %s count must be >= %d", type, min_count);
+            goto error;
+        }
+    }
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+static int node_vertex_check (json_t *node, flux_jobspec1_error_t *error)
+{
+    json_error_t json_error;
+    const char *type;
+    int count;
+    json_t *with;
+    const char *unit = NULL;
+    size_t index;
+    json_t *value;
+
+    if (json_unpack_ex (node,
+                        &json_error,
+                        0,
+                        "{s:s s:i s:o s?s !}",
+                        "type", &type,
+                        "count", &count,
+                        "with", &with,
+                        "unit", &unit) < 0) {
+        errprintf (error, "node vertex: %s", json_error.text);
+        goto error;
+    }
+    if (count < 1) {
+        errprintf (error, "node count must be >= 1");
+        goto error;
+    }
+    if (!json_is_array (with)) {
+        errprintf (error, "node with must be an array");
+        goto error;
+    }
+    if (json_array_size (with) != 1) {
+        errprintf (error, "node with array must have 1 elements");
+        goto error;
+    }
+    json_array_foreach (with, index, value) {
+        if (slot_vertex_check (value, error) < 0)
+            goto error;
+    }
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+static int resources_check (json_t *res, flux_jobspec1_error_t *error)
+{
+    json_error_t json_error;
+    json_t *vertex;
+    const char *type;
+
+    if (json_unpack_ex (res, &json_error, 0, "[o]", &vertex) < 0) {
+        errprintf (error, "resources section: %s", json_error.text);
+        goto error;
+    }
+    if (json_unpack_ex (vertex, &json_error, 0, "{s:s}", "type", &type) < 0) {
+        errprintf (error, "resource vertex: %s", json_error.text);
+        goto error;
+    }
+    if (streq (type, "node")) {
+        if (node_vertex_check (vertex, error) < 0)
+            goto error;
+    }
+    else if (streq (type, "slot")) {
+        if (slot_vertex_check (vertex, error) < 0)
+            goto error;
+    }
+    else {
+        errprintf (error, "unknown resource vertex type");
+        goto error;
+    }
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+static int attr_system_check (json_t *o, flux_jobspec1_error_t *error)
+{
+    const char *key;
+    json_t *value;
+    bool has_duration = false;
+
+    json_object_foreach (o, key, value) {
+        if (streq (key, "duration")) {
+            if (!json_is_number (value)) {
+                errprintf (error,
+                           "attributes.system.duration must be a number");
+                return -1;
+            }
+            has_duration = true;
+        }
+        else if (streq (key, "environment")) {
+            if (!(json_is_object (value))) {
+                errprintf (error,
+                         "attributes.system.environment must be a dictionary");
+                return -1;
+            }
+        }
+        else if (streq (key, "constraints")) {
+            if (!(json_is_object (value))) {
+                errprintf (error,
+                         "attributes.system.constraints must be a dictionary");
+                return -1;
+            }
+        }
+        else if (streq (key, "dependencies")) {
+            size_t index;
+            json_t *el;
+            const char *scheme;
+            const char *val;
+
+            if (!json_is_array (value)) {
+                errprintf (error,
+                           "attributes.system.dependencies must be an array");
+                return -1;
+            }
+            json_array_foreach (value, index, el) {
+                if (!json_is_object (el)) {
+                    errprintf (error,
+                               "attributes.system.dependencies elements"
+                               " must be an object");
+                    return -1;
+                }
+                if (json_unpack (el,
+                                 "{s:s s:s}",
+                                 "scheme", &scheme,
+                                 "value", &val) < 0) {
+                    errprintf (error,
+                               "attributes.system.dependencies elements"
+                               " must contain scheme and value strings");
+                    return -1;
+                }
+            }
+        }
+        else if (streq (key, "shell")) {
+            json_t *opt;
+            if ((opt = json_object_get (value, "options"))
+                && !json_is_object (opt)) {
+                errprintf (error,
+                           "attributes.shell.options must be a dictionary");
+                return -1;
+            }
+        }
+    }
+    if (!has_duration) {
+        errprintf (error, "attributes.system.duration is required");
+        return -1;
+    }
+    return 0;
+}
+
+int flux_jobspec1_attr_check (flux_jobspec1_t *jobspec,
+                              flux_jobspec1_error_t *error)
+{
+    json_t *o;
+    const char *key;
+    json_t *value;
+    bool has_system = false;
+
+    if (!jobspec) {
+        errprintf (error, "jobspec must not be NULL");
+        goto error;
+    }
+    if (!(o = json_object_get (jobspec->obj, "attributes"))) {
+        errprintf (error, "attributes must exist");
+        goto error;
+    }
+    if (!json_is_object (o)) {
+        errprintf (error, "attributes must be an object");
+        goto error;
+    }
+    json_object_foreach (o, key, value) {
+        if (streq (key, "user")) {
+            if (json_object_size (value) == 0) {
+                errprintf (error,
+                           "if present, attributes.user must contain values");
+                goto error;
+            }
+        }
+        else if (streq (key, "system")) {
+            if (json_object_size (value) == 0) {
+                errprintf (error,
+                           "if present, attributes.system must contain values");
+                goto error;
+            }
+            if (attr_system_check (value, error) < 0)
+                goto error;
+            has_system = true;
+        }
+        else {
+            errprintf (error, "unknown attributes section %s", key);
+            goto error;
+        }
+    }
+    if (!has_system) {
+        errprintf (error, "attributes.system is required");
+        goto error;
+    }
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+int flux_jobspec1_check (flux_jobspec1_t *jobspec, flux_jobspec1_error_t *error)
+{
+    json_error_t json_error;
+    json_t *resources;
+    json_t *tasks;
+    json_t *attributes;
+    int version;
+
+    if (!jobspec) {
+        errprintf (error, "jobspec cannot be NULL");
+        goto error;
+    }
+    if (json_unpack_ex (jobspec->obj,
+                        &json_error,
+                        0,
+                        "{s:o s:o s:o s:i !}",
+                        "resources", &resources,
+                        "tasks", &tasks,
+                        "attributes", &attributes,
+                        "version", &version) < 0) {
+        errprintf (error, "jobspec object: %s", json_error.text);
+        goto error;
+    }
+    if (version != 1) {
+        errprintf (error, "only version 1 jobspec is supported");
+        goto error;
+    }
+    if (resources_check (resources, error) < 0)
+        goto error;
+    if (tasks_check (tasks, error) < 0)
+        goto error;
+    if (flux_jobspec1_attr_check (jobspec, error) < 0)
+        goto error;
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+int flux_jobspec1_unsetenv (flux_jobspec1_t *jobspec, const char *name)
+{
+    char *path;
+
+    if (!jobspec || !name) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (asprintf (&path, "system.environment.%s", name) < 0)
+        return -1;
+    if (flux_jobspec1_attr_del (jobspec, path) < 0) {
+        ERRNO_SAFE_WRAP (free, path);
+        return -1;
+    }
+    return 0;
+}
+
+int flux_jobspec1_setenv (flux_jobspec1_t *jobspec,
+                           const char *name,
+                           const char *value,
+                           int overwrite)
+{
+    char *path;
+    const char *s;
+
+    if (!jobspec || !name || !value) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (asprintf (&path, "system.environment.%s", name) < 0)
+        return -1;
+    if (!overwrite) {
+        if (flux_jobspec1_attr_unpack (jobspec, path, "s", &s) == 0) {
+            free (path);
+            return 0;
+        }
+    }
+    if (flux_jobspec1_attr_pack (jobspec, path, "s", value) < 0) {
+        ERRNO_SAFE_WRAP (free, path);
+        return -1;
+    }
+    free (path);
+    return 0;
+}
+
+static int jobspec1_putenv (flux_jobspec1_t *jobspec, const char *entry)
+{
+    char *name;
+    char *value;
+    int rc = -1;
+
+    if (!(name = strdup (entry)))
+        return -1;
+    if ((value = strchr (name, '=')))
+        *value++ = '\0';
+    if (!value || name[0] == '\0') {
+        errno = EINVAL;
+        goto done;
+    }
+    rc = flux_jobspec1_setenv (jobspec, name, value, 1);
+done:
+    ERRNO_SAFE_WRAP (free, name);
+    return rc;
+}
+
+/* 'stdio_name' should be one of: 'output.stdout', 'output.stderr',
+ * or 'input.stdin'.
+ */
+static int flux_jobspec1_set_stdio (flux_jobspec1_t *jobspec,
+                                    const char *stdio_name,
+                                    const char *path)
+{
+    char key[256];
+
+    if (!jobspec || !path || (!streq (stdio_name, "input.stdin")
+        && !streq (stdio_name, "output.stdout")
+        && !streq (stdio_name, "output.stderr"))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (snprintf (key, sizeof (key), "system.shell.options.%s", stdio_name)
+        >= sizeof (key)) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    return flux_jobspec1_attr_pack (jobspec,
+                                    key,
+                                    "{s:s s:s}",
+                                    "type",
+                                    "file",
+                                    "path",
+                                    path);
+}
+
+int flux_jobspec1_set_stdin (flux_jobspec1_t *jobspec, const char *path)
+{
+    return flux_jobspec1_set_stdio (jobspec, "input.stdin", path);
+}
+
+int flux_jobspec1_set_stdout (flux_jobspec1_t *jobspec, const char *path)
+{
+    return flux_jobspec1_set_stdio (jobspec, "output.stdout", path);
+}
+
+int flux_jobspec1_set_stderr (flux_jobspec1_t *jobspec, const char *path)
+{
+    return flux_jobspec1_set_stdio (jobspec, "output.stderr", path);
+}
+
+int flux_jobspec1_set_cwd (flux_jobspec1_t *jobspec, const char *cwd)
+{
+    if (!cwd) {  // 'jobspec' checked by 'attr_pack'
+        errno = EINVAL;
+        return -1;
+    }
+    return flux_jobspec1_attr_pack (jobspec, "system.cwd", "s", cwd);
+}
+
+char *flux_jobspec1_encode (flux_jobspec1_t *jobspec, size_t flags)
+{
+    char *returnval;
+
+    if (!jobspec) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(returnval = json_dumps (jobspec->obj, flags))) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    return returnval;
+}
+
+flux_jobspec1_t *jobspec1_from_json (json_t *obj)
+{
+    flux_jobspec1_t *jobspec;
+
+    if (!obj) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(jobspec = calloc (1, sizeof (*jobspec))))
+        return NULL;
+    jobspec->obj = json_incref (obj);
+    return jobspec;
+}
+
+flux_jobspec1_t *flux_jobspec1_decode (const char *s,
+                                       flux_jobspec1_error_t *error)
+{
+    flux_jobspec1_t *jobspec;
+    json_t *obj;
+    json_error_t json_error;
+
+    if (!s) {
+        errno = EINVAL;
+        errprintf (error, "%s", strerror (errno));
+        return NULL;
+    }
+    if (!(obj = json_loads (s, 0, &json_error))) {
+        errno = EINVAL;
+        errprintf (error, "%s", json_error.text);
+        return NULL;
+    }
+    if (!(jobspec = jobspec1_from_json (obj))) {
+        ERRNO_SAFE_WRAP (json_decref, obj);
+        errprintf (error, "%s", strerror (errno));
+        return NULL;
+    }
+    json_decref (obj);
+    return jobspec;
+}
+
+static json_t *tasks_create (int argc, char **argv)
+{
+    json_t *tasks;
+    json_t *argv_json;
+
+    if (!(argv_json = argv_to_json (argc, argv))) {
+        return NULL;
+    }
+    if (!(tasks = json_pack ("[{s:o s:s s:{s:i}}]",
+                             "command",
+                             argv_json,
+                             "slot",
+                             "task",
+                             "count",
+                             "per_slot",
+                             1))) {
+        json_decref (argv_json);
+        errno = ENOMEM;
+        return NULL;
+    }
+    return tasks;
+}
+
+/* Create and return the 'resources' section of a jobspec.
+ * Return a new reference on success, NULL with errno set on error.
+ * Negative values of 'ntasks' and 'cores_per_task' are interpreted as 1.
+ * Negative values for 'gpus_per_task' and 'nnodes' are ignored.
+ */
+static json_t *resources_create (int ntasks,
+                                 int cores_per_task,
+                                 int gpus_per_task,
+                                 int nnodes)
+{
+    json_t *slot;
+
+    if (cores_per_task < 1)
+        cores_per_task = 1;
+    if (ntasks < 1)
+        ntasks = 1;
+    if (nnodes > ntasks) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (gpus_per_task > 0) {
+        if (!(slot = json_pack ("[{s:s s:i s:[{s:s s:i} {s:s s:i}] s:s}]",
+                                "type", "slot",
+                                "count", ntasks,
+                                "with",
+                                "type", "core",
+                                "count", cores_per_task,
+                                "type", "gpu",
+                                "count", gpus_per_task,
+                                "label", "task")))
+            goto nomem;
+    }
+    else {
+        if (!(slot = json_pack ("[{s:s s:i s:[{s:s s:i}] s:s}]",
+                                "type", "slot",
+                                "count", ntasks,
+                                "with",
+                                "type", "core",
+                                "count", cores_per_task,
+                                "label", "task")))
+            goto nomem;
+    }
+    if (nnodes > 0) {
+        json_t *node;
+        if (!(node = json_pack ("[{s:s s:i s:o}]",
+                                "type", "node",
+                                "count", nnodes,
+                                "with",
+                                slot))) {
+            json_decref (slot);
+            goto nomem;
+        }
+        return node;
+    }
+    return slot;
+nomem:
+    errno = ENOMEM;
+    return NULL;
+}
+
+flux_jobspec1_t *flux_jobspec1_from_command (int argc,
+                                             char **argv,
+                                             char **env,
+                                             int ntasks,
+                                             int cores_per_task,
+                                             int gpus_per_task,
+                                             int nnodes,
+                                             double duration)
+{
+    json_t *resources;
+    json_t *tasks;
+    json_t *obj;
+    flux_jobspec1_t *jobspec;
+
+    // resource arguments are checked by 'resources_create'
+    if (argc < 0 || !argv || duration < 0.0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(tasks = tasks_create (argc, argv))
+        || !(resources = resources_create (ntasks,
+                                           cores_per_task,
+                                           gpus_per_task,
+                                           nnodes))) {
+        ERRNO_SAFE_WRAP (json_decref, tasks);
+        return NULL;
+    }
+    if (!(obj = json_pack ("{s:o, s:o, s:{s:{s:f, s:{}}}, s:i}",
+                           "resources", resources,
+                           "tasks", tasks,
+                           "attributes",
+                             "system",
+                               "duration", duration,
+                             "environment",
+                           "version", 1))) {
+        json_decref (tasks);
+        json_decref (resources);
+        errno = ENOMEM;
+        return NULL;
+    }
+    if (!(jobspec = jobspec1_from_json (obj))) {
+        ERRNO_SAFE_WRAP (json_decref, obj);
+        return NULL;
+    }
+    json_decref (obj);
+
+    if (env) {
+        int i;
+        for (i = 0; env[i] != NULL; i++) {
+            if (jobspec1_putenv (jobspec, env[i]) < 0)
+                goto error;
+        }
+    }
+    return jobspec;
+
+error:
+    flux_jobspec1_destroy (jobspec);
+    return NULL;
+}
+
+void flux_jobspec1_destroy (flux_jobspec1_t *jobspec)
+{
+    int saved_errno = errno;
+
+    if (jobspec) {
+        json_decref (jobspec->obj);
+        free (jobspec);
+        errno = saved_errno;
+    }
+}
+
+json_t *jobspec1_get_json (flux_jobspec1_t *jobspec)
+{
+    return jobspec->obj;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/jobspec1.h b/src/common/libjob/jobspec1.h
new file mode 100644
index 000000000000..d5493727ec31
--- /dev/null
+++ b/src/common/libjob/jobspec1.h
@@ -0,0 +1,143 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_CORE_JOBSPEC1_H
+#define _FLUX_CORE_JOBSPEC1_H
+
+#include <flux/core.h>
+
+typedef struct flux_jobspec1 flux_jobspec1_t;
+typedef flux_error_t flux_jobspec1_error_t;
+
+/* Remove the value in the jobspec's attributes section at the given path,
+ * where "." is treated as a path separator.
+ * It is not an error if 'path' does not exist.
+ * Return 0 on success, -1 on error with errno set.
+ */
+int flux_jobspec1_attr_del (flux_jobspec1_t *jobspec, const char *path);
+
+/* Set the value in the jobspec's attributes section at the given path,
+ * where "." is treated as a path separator.  'fmt' should be a jansson
+ * pack-style string corresponding to the remaining arguments.
+ * Return 0 on success, -1 on error with errno set.
+ */
+int flux_jobspec1_attr_pack (flux_jobspec1_t *jobspec,
+                             const char *path,
+                             const char *fmt,
+                             ...);
+
+/* Unpack the values in the jobspec's attributes section at the given path,
+ * where "." is treated as a path separator.  'fmt' should be a jansson
+ * unpack-style string corresponding to the remaining args.
+ * Return 0 on success, -1 on error with errno set.
+ */
+int flux_jobspec1_attr_unpack (flux_jobspec1_t *jobspec,
+                               const char *path,
+                               const char *fmt,
+                               ...);
+
+/* Check the validity of only the attributes section of a jobspec, sufficient
+ * if the jobspec object was created by flux_jobspec1_from_command(), then
+ * modified with the other jobspec1 functions.
+ * Return 0 on success, -1 on error with errno set.
+ * On error, write an error message written to 'error', if non-NULL.
+ */
+int flux_jobspec1_attr_check (flux_jobspec1_t *jobspec,
+                              flux_jobspec1_error_t *error);
+
+/* Check the validity of the full jobspec, which may be necessary if the
+ * jobspec object was created by flux_jobspec1_decode().
+ * Return 0 on success, -1 on error with errno set.
+ * On error, write an error message written to 'error', if non-NULL.
+ */
+int flux_jobspec1_check (flux_jobspec1_t *jobspec,
+                         flux_jobspec1_error_t *error);
+
+/* Remove the variable 'name' from the environment.
+ * This function succeeds if 'name' does not exist.
+ * Return 0 on success, -1 on error with errno set.
+ */
+int flux_jobspec1_unsetenv (flux_jobspec1_t *jobspec, const char *name);
+
+/* Add the variable 'name' to the environment with the value 'value'.
+ * If 'name' exists in the environment and 'overwrite' is nonzero, change
+ * value to 'value'.  If 'name' exists and 'overwrite' is zero, do not
+ * change the value (and return success).
+ * Return 0 on success, -1 on error with errno set.
+ */
+int flux_jobspec1_setenv (flux_jobspec1_t *jobspec,
+                           const char *name,
+                           const char *value,
+                           int overwrite);
+
+/* Redirect job stdin from the KVS to a file system path given by 'path'.
+ * Return 0 on success, -1 on error with errno set.
+ */
+int flux_jobspec1_set_stdin (flux_jobspec1_t *jobspec, const char *path);
+
+/* Redirect job stdout from the KVS to a file system path given by 'path'.
+ * Return 0 on success, -1 on error with errno set.
+ */
+int flux_jobspec1_set_stdout (flux_jobspec1_t *jobspec, const char *path);
+
+/* Redirect job stderr from the KVS to a file system path given by 'path'.
+ * Return 0 on success, -1 on error with errno set.
+ */
+int flux_jobspec1_set_stderr (flux_jobspec1_t *jobspec, const char *path);
+
+/* Set the working directory of a jobspec.
+ * Return 0 on success, -1 on error with errno set.
+ */
+int flux_jobspec1_set_cwd (flux_jobspec1_t *jobspec, const char *cwd);
+
+/* Encode a jobspec to a string, e.g. for usage with flux_job_submit().
+ * 'flags' should be 0.  The returned string must be released with free().
+ * Return NULL on error with errno set.
+ */
+char *flux_jobspec1_encode (flux_jobspec1_t *jobspec, size_t flags);
+
+/* Decode a string to jobspec.  No validation is performed on the content.
+ * The returned jobspec must be released with flux_jobspec1_destroy().
+ * Return NULL on error with errno set.
+ * On error, write an error message to 'error', if non-NULL.
+ */
+flux_jobspec1_t *flux_jobspec1_decode (const char *s,
+                                       flux_jobspec1_error_t *error);
+
+
+/* Create and return a minimum viable V1 Jobspec.
+ * The jobspec must be released with flux_jobspec1_destroy()'
+ * The jobspec will have stdin, stdout, stderr all directed to the KVS.
+ * 'argc' and 'argv' define the command and its arguments.
+ * 'env' should be an environ(7)-style array, or NULL for empty.
+ * 'tasks' should be the number of tasks to launch
+ * 'cores_per_task' should be the number of cores per task to allocate
+ * 'gpus_per_task' should be the number of gpus per task to allocate
+ * 'nodes' should be the number of nodes to spread the allocated cores
+ * and gpus across. If 0, the scheduler will determine the node allocation.
+ * Return NULL on error with errno set.
+ */
+flux_jobspec1_t *flux_jobspec1_from_command (int argc,
+                                             char **argv,
+                                             char **env,
+                                             int ntasks,
+                                             int cores_per_task,
+                                             int gpus_per_task,
+                                             int nnodes,
+                                             double duration);
+
+/* Free a jobspec. */
+void flux_jobspec1_destroy (flux_jobspec1_t *jobspec);
+
+#endif /* _FLUX_CORE_JOBSPEC1_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/jobspec1_private.h b/src/common/libjob/jobspec1_private.h
new file mode 100644
index 000000000000..861da3a78624
--- /dev/null
+++ b/src/common/libjob/jobspec1_private.h
@@ -0,0 +1,19 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _JOBSPEC1_PRIVATE_H
+#define _JOBSPEC1_PRIVATE_H
+
+json_t *jobspec1_get_json (flux_jobspec1_t *jobspec);
+flux_jobspec1_t *jobspec1_from_json (json_t *obj);
+
+#endif // !_JOBSPEC1_PRIVATE_H
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libjob/kvs.c b/src/common/libjob/kvs.c
new file mode 100644
index 000000000000..b34abed47d12
--- /dev/null
+++ b/src/common/libjob/kvs.c
@@ -0,0 +1,91 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "src/common/libutil/fluid.h"
+
+#include "job.h"
+
+static int buffer_arg_check (char *buf, int bufsz)
+{
+    if (!buf || bufsz <= 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    return 0;
+}
+
+int flux_job_kvs_key (char *buf, int bufsz, flux_jobid_t id, const char *key)
+{
+    char idstr[32];
+    int len;
+
+    if (buffer_arg_check (buf, bufsz) < 0)
+        return -1;
+
+    if (fluid_encode (idstr, sizeof (idstr), id, FLUID_STRING_DOTHEX) < 0)
+        return -1;
+    len = snprintf (buf, bufsz, "job.%s%s%s",
+                    idstr,
+                    key ? "." : "",
+                    key ? key : "");
+    if (len >= bufsz) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    return len;
+}
+
+int flux_job_kvs_guest_key (char *buf,
+                            int bufsz,
+                            flux_jobid_t id,
+                            const char *key)
+{
+    char idstr[32];
+    int len;
+
+    if (buffer_arg_check (buf, bufsz) < 0)
+        return -1;
+    if (getenv ("FLUX_KVS_NAMESPACE"))
+        len = snprintf (buf, bufsz, "%s", key ? key : ".");
+    else {
+        if (fluid_encode (idstr, sizeof (idstr), id, FLUID_STRING_DOTHEX) < 0)
+            return -1;
+        len = snprintf (buf, bufsz, "job.%s.guest%s%s",
+                        idstr,
+                        key ? "." : "",
+                        key ? key : "");
+    }
+    if (len >= bufsz) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    return len;
+}
+
+int flux_job_kvs_namespace (char *buf, int bufsz, flux_jobid_t id)
+{
+    int len;
+    if (buffer_arg_check (buf, bufsz) < 0)
+        return -1;
+    if ((len = snprintf (buf, bufsz, "job-%ju", (uintmax_t)id)) >= bufsz) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    return len;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/list.c b/src/common/libjob/list.c
new file mode 100644
index 000000000000..b33b8fb8b67b
--- /dev/null
+++ b/src/common/libjob/list.c
@@ -0,0 +1,129 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <ctype.h>
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "job.h"
+
+flux_future_t *flux_job_list (flux_t *h,
+                              int max_entries,
+                              const char *attrs_json_str,
+                              uint32_t userid,
+                              int states)
+{
+    flux_future_t *f;
+    json_t *o = NULL;
+    json_t *c = NULL;
+    int valid_states = (FLUX_JOB_STATE_PENDING
+                        | FLUX_JOB_STATE_RUNNING
+                        | FLUX_JOB_STATE_INACTIVE);
+    int saved_errno;
+
+    if (!h || max_entries < 0 || !attrs_json_str
+           || !(o = json_loads (attrs_json_str, 0, NULL))
+           || states & ~valid_states) {
+        json_decref (o);
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(c = json_pack ("{ s:[ {s:[i]}, {s:[i]} ] }",
+                         "and",
+                         "userid", userid,
+                         "states", states ? states : valid_states))) {
+        json_decref (o);
+        errno = ENOMEM;
+        return NULL;
+    }
+    if (!(f = flux_rpc_pack (h, "job-list.list", FLUX_NODEID_ANY, 0,
+                             "{s:i s:o s:o}",
+                             "max_entries", max_entries,
+                             "attrs", o,
+                             "constraint", c))) {
+        saved_errno = errno;
+        json_decref (o);
+        json_decref (c);
+        errno = saved_errno;
+        return NULL;
+    }
+    return f;
+}
+
+flux_future_t *flux_job_list_inactive (flux_t *h,
+                                       int max_entries,
+                                       double since,
+                                       const char *attrs_json_str)
+{
+    flux_future_t *f;
+    json_t *o = NULL;
+    json_t *c = NULL;
+    int saved_errno;
+
+    if (!h || max_entries < 0 || since < 0. || !attrs_json_str
+           || !(o = json_loads (attrs_json_str, 0, NULL))) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(c = json_pack ("{s:[i]}", "states", FLUX_JOB_STATE_INACTIVE))) {
+        json_decref (o);
+        errno = ENOMEM;
+        return NULL;
+    }
+    if (!(f = flux_rpc_pack (h, "job-list.list", FLUX_NODEID_ANY, 0,
+                             "{s:i s:f s:o s:o}",
+                             "max_entries", max_entries,
+                             "since", since,
+                             "attrs", o,
+                             "constraint", c))) {
+        saved_errno = errno;
+        json_decref (o);
+        json_decref (c);
+        errno = saved_errno;
+        return NULL;
+    }
+    return f;
+}
+
+flux_future_t *flux_job_list_id (flux_t *h,
+                                 flux_jobid_t id,
+                                 const char *attrs_json_str)
+{
+    flux_future_t *f;
+    json_t *o = NULL;
+    int saved_errno;
+
+    if (!h || !attrs_json_str
+           || !(o = json_loads (attrs_json_str, 0, NULL))) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(f = flux_rpc_pack (h, "job-list.list-id", FLUX_NODEID_ANY, 0,
+                             "{s:I s:O}",
+                             "id", id,
+                             "attrs", o)))
+        goto error;
+
+    json_decref (o);
+    return f;
+
+error:
+    saved_errno = errno;
+    json_decref (o);
+    errno = saved_errno;
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/result.c b/src/common/libjob/result.c
new file mode 100644
index 000000000000..47a988e1bcdc
--- /dev/null
+++ b/src/common/libjob/result.c
@@ -0,0 +1,277 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <assert.h>
+#include <flux/core.h>
+
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+
+#include "job.h"
+#include "strtab.h"
+
+#include "src/common/libeventlog/eventlog.h"
+
+static struct strtab results[] = {
+    { FLUX_JOB_RESULT_COMPLETED, "COMPLETED", "completed", "CD", "cd" },
+    { FLUX_JOB_RESULT_FAILED,    "FAILED",    "failed",    "F",  "f" },
+    { FLUX_JOB_RESULT_CANCELED,  "CANCELED",  "canceled",  "CA", "ca" },
+    { FLUX_JOB_RESULT_TIMEOUT,   "TIMEOUT",   "timeout",   "TO", "to" },
+};
+
+
+const char *flux_job_resulttostr (flux_job_result_t result, const char *fmt)
+{
+    return strtab_numtostr (result, fmt, results, ARRAY_SIZE (results));
+}
+
+int flux_job_strtoresult (const char *s, flux_job_result_t *result)
+{
+    int num;
+
+    if ((num = strtab_strtonum (s, results, ARRAY_SIZE (results))) < 0)
+        return -1;
+    if (result)
+        *result = num;
+    return 0;
+}
+
+static flux_job_result_t job_result_calc (json_t *res)
+{
+    double t_run = -1.;
+    int status = -1;
+    bool exception_occurred = false;
+    const char *exception_type = NULL;
+    json_error_t error;
+
+    if (json_unpack_ex (res, &error, 0,
+                        "{s?f s:b s?i s?s}",
+                        "t_run", &t_run,
+                        "exception_occurred", &exception_occurred,
+                        "waitstatus", &status,
+                        "exception_type", &exception_type) < 0)
+        return FLUX_JOB_RESULT_FAILED;
+
+    if (t_run > 0. && status == 0)
+        return FLUX_JOB_RESULT_COMPLETED;
+    if (exception_occurred) {
+        if (exception_type != NULL) {
+            if (streq (exception_type, "cancel"))
+                return FLUX_JOB_RESULT_CANCELED;
+            if (streq (exception_type, "timeout"))
+                return FLUX_JOB_RESULT_TIMEOUT;
+        }
+    }
+    return FLUX_JOB_RESULT_FAILED;
+}
+
+static void result_eventlog_error_cb (flux_future_t *f, void *arg)
+{
+    json_t *res = arg;
+    json_t *o = NULL;
+    char *s = NULL;
+
+    if (flux_future_get (f, NULL) < 0 && errno != ENODATA) {
+        flux_future_continue_error (f, errno, NULL);
+        goto out;
+    }
+    flux_job_result_t result = job_result_calc (res);
+    if (!(o = json_integer (result))
+        || json_object_set_new (res, "result", o) < 0
+        || !(s = json_dumps (res, JSON_COMPACT))) {
+        flux_future_continue_error (f, ENOMEM, NULL);
+        goto out;
+    }
+    flux_future_fulfill_next (f, s, free);
+out:
+    flux_future_destroy (f);
+}
+
+
+static int result_exception_severity (json_t *res)
+{
+    json_t *sev = json_object_get (res, "exception_severity");
+    if (sev) {
+        return json_integer_value (sev);
+    }
+    return -1;
+}
+
+static int job_result_handle_exception (json_t *res,
+                                        json_t *context)
+{
+    json_t *type;
+    json_t *severity;
+    json_t *note;
+
+    if (json_unpack (context,
+                     "{s:o s:o s:o}",
+                     "type", &type,
+                     "severity", &severity,
+                     "note", &note) < 0) {
+        errno = EPROTO;
+        return -1;
+    }
+
+    if (json_is_true (json_object_get (res, "exception_occurred"))) {
+        /* Only overwrite previous exception if the latest
+         *  is of greater severity.
+         */
+        int sev = json_integer_value (severity);
+        int prev_sev = result_exception_severity (res);
+        if (prev_sev > 0 && prev_sev < sev)
+            return 0;
+    }
+    if (json_object_set (res, "exception_occurred", json_true ()) < 0
+        || json_object_set (res, "exception_type", type) < 0
+        || json_object_set (res, "exception_note", note) < 0
+        || json_object_set (res, "exception_severity", severity) < 0) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+static void result_eventlog_cb (flux_future_t *f, void *arg)
+{
+    json_t *res = arg;
+    const char *entry = NULL;
+    const char *name = NULL;
+    json_t *o = NULL;
+    json_t *context = NULL;
+    json_t *timestamp = NULL;
+
+    if (flux_job_event_watch_get (f, &entry) < 0) {
+        /* This should never happen, since this is an "and_then" callback
+         */
+        goto error;
+    }
+    if (!(o = eventlog_entry_decode (entry)))
+        goto error;
+    if (!(timestamp = json_object_get (o, "timestamp"))) {
+        errno = EPROTO;
+        goto error;
+    }
+    if (eventlog_entry_parse (o, NULL, &name, &context) < 0)
+        goto error;
+
+    if (streq (name, "submit")) {
+        if (json_object_set (res, "t_submit", timestamp) < 0)
+            goto enomem;
+    }
+    else if (streq (name, "alloc")) {
+        if (json_object_set (res, "t_run", timestamp) < 0)
+            goto enomem;
+    }
+    else if (streq (name, "finish")) {
+        json_t *wstatus = NULL;
+        if (json_object_set (res, "t_cleanup", timestamp) < 0)
+            goto enomem;
+        if (!(wstatus = json_object_get (context, "status"))) {
+            errno = EPROTO;
+            goto error;
+        }
+        if (json_object_set (res, "waitstatus", wstatus) < 0)
+            goto enomem;
+    }
+    else if (streq (name, "exception")) {
+        if (job_result_handle_exception (res, context) < 0)
+            goto error;
+    }
+
+    json_decref (o);
+
+    /*  Ensure "next" future is not auto-continued by chained future
+     *   implementation. This is non-obvious, but if this call is not
+     *   made then our next future would be prematurely fulfilled.
+     */
+    flux_future_continue (f, NULL);
+    flux_future_reset (f);
+    return;
+enomem:
+    errno = ENOMEM;
+error:
+    flux_future_continue_error (f, errno, NULL);
+    flux_future_destroy (f);
+}
+
+int flux_job_result_get_unpack (flux_future_t *f, const char *fmt, ...)
+{
+    json_t *res;
+    json_error_t error;
+    int rc;
+    va_list ap;
+
+    if (f == NULL
+        || !(res = flux_future_aux_get (f, "flux::result"))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_future_get (f, NULL) < 0)
+        return -1;
+
+    va_start (ap, fmt);
+    rc = json_vunpack_ex (res, &error, 0, fmt, ap);
+    va_end (ap);
+    if (rc < 0)
+        errno = EINVAL;
+    return rc;
+}
+
+int flux_job_result_get (flux_future_t *f,
+                         const char **json_str)
+{
+    return flux_future_get (f, (const void **) json_str);
+}
+
+json_t *job_result_alloc (flux_jobid_t id)
+{
+    return json_pack ("{s:I s:b}",
+                      "id", id,
+                      "exception_occurred", false);
+}
+
+flux_future_t *flux_job_result (flux_t *h, flux_jobid_t id, int flags)
+{
+    json_t *res = NULL;
+    flux_future_t *f = NULL;
+    flux_future_t *event_f = NULL;
+    int saved_errno;
+
+    if (!(res = job_result_alloc (id))
+        || !(event_f = flux_job_event_watch (h, id, "eventlog", 0))
+        || !(f = flux_future_and_then (event_f, result_eventlog_cb, res))
+        || !(f = flux_future_or_then (event_f,
+                                      result_eventlog_error_cb,
+                                      res))
+        || flux_future_aux_set (f,
+                                "flux::result",
+                                res,
+                                (flux_free_f) json_decref) < 0)
+        goto error;
+
+    return f;
+error:
+    saved_errno = errno;
+    json_decref (res);
+    if (event_f) {
+        flux_job_event_watch_cancel (event_f);
+        flux_future_destroy (f);
+    }
+    errno = saved_errno;
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/sign_none.c b/src/common/libjob/sign_none.c
index bdb2e35e48cf..cc77eb76b4f5 100644
--- a/src/common/libjob/sign_none.c
+++ b/src/common/libjob/sign_none.c
@@ -16,16 +16,21 @@
 #include <string.h>
 #include <assert.h>
 #include <errno.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
-#include <sodium.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
 
-#include "src/common/libutil/macros.h"
+#include "ccan/base64/base64.h"
+#include "ccan/str/str.h"
 
 #include "sign_none.h"
 
 int header_decode (const char *src, int srclen, uint32_t *useridp)
 {
-    size_t dstlen = BASE64_DECODE_SIZE (srclen);
+    size_t dstbuflen = base64_decoded_length (srclen) + 1; /* +1 for NUL */
+    ssize_t dstlen;
     char *dst;
     char *entry = NULL;
     char *key;
@@ -36,28 +41,24 @@ int header_decode (const char *src, int srclen, uint32_t *useridp)
     uint32_t userid;
     char *endptr;
 
-    if (!(dst = calloc (1, dstlen)))
+    if (!(dst = malloc (dstbuflen)))
         return -1;
-    if (sodium_base642bin ((unsigned char *)dst, dstlen, src, srclen,
-                           NULL, &dstlen, NULL,
-                           sodium_base64_VARIANT_ORIGINAL) < 0)
-        goto error_inval;
-    if (dst[dstlen - 1] != '\0')
+    if ((dstlen = base64_decode (dst, dstbuflen, src, srclen)) < 0)
         goto error_inval;
     while ((key = entry = argz_next (dst, dstlen, entry))) {
         if (!(val = entry = argz_next (dst, dstlen, entry)))
             goto error_inval;
-        if (!strcmp (key, "version"))
+        if (streq (key, "version"))
             val_version = val;
-        else if (!strcmp (key, "userid"))
+        else if (streq (key, "userid"))
             val_userid = val;
-        else if (!strcmp (key, "mech"))
+        else if (streq (key, "mechanism"))
             val_mech = val;
         else
             goto error_inval;
     }
-    if (!val_version || !val_mech || strcmp (val_version, "i1") != 0
-                                  || strcmp (val_mech, "snone") != 0)
+    if (!val_version || !val_mech || !streq (val_version, "i1")
+                                  || !streq (val_mech, "snone"))
         goto error_inval;
     if (!val_userid || *val_userid != 'i')
         goto error_inval;
@@ -81,47 +82,49 @@ static char *header_encode (uint32_t userid)
     char src[128];
     int srclen;
     char *dst;
-    int dstlen;
+    size_t dstbuflen;
     int i;
 
-    srclen = snprintf (src, sizeof (src), "version:i1:userid:i%lu:mech:snone:",
+    srclen = snprintf (src, sizeof (src), "version:i1:userid:i%lu:mechanism:snone:",
                        (unsigned long)userid);
     assert (srclen < sizeof (src));
     for (i = 0; i < srclen; i++) {
         if (src[i] == ':')
             src[i] = '\0';
     }
-    dstlen = sodium_base64_encoded_len (srclen, sodium_base64_VARIANT_ORIGINAL);
-    if (!(dst = calloc (1, dstlen)))
+    dstbuflen = base64_encoded_length (srclen) + 1; /* +1 for NUL */
+    if (!(dst = malloc (dstbuflen)))
         return NULL;
-
-    return sodium_bin2base64 (dst, dstlen, (unsigned char *)src, srclen,
-                              sodium_base64_VARIANT_ORIGINAL);
+    if (base64_encode (dst, dstbuflen, src, srclen) < 0)
+        return NULL;
+    return dst;
 }
 
-static char *payload_encode (const void *src, int srclen)
+static char *payload_encode (const char *src, int srclen)
 {
     char *dst;
-    int dstlen;
+    size_t dstbuflen;
 
-    dstlen = sodium_base64_encoded_len (srclen, sodium_base64_VARIANT_ORIGINAL);
-    if (!(dst = calloc (1, dstlen)))
+    dstbuflen = base64_encoded_length (srclen) + 1; /* +1 for NUL */
+    if (!(dst = malloc (dstbuflen)))
         return NULL;
-    return sodium_bin2base64 (dst, dstlen, (unsigned char *)src, srclen,
-                              sodium_base64_VARIANT_ORIGINAL);
+    if (base64_encode (dst, dstbuflen, src, srclen) < 0) {
+        free (dst);
+        return NULL;
+    }
+    return dst;
 }
 
 static int payload_decode (const void *src, int srclen,
                            void **payload, int *payloadsz)
 {
-    size_t dstlen = BASE64_DECODE_SIZE (srclen);
-    void *dst;
+    size_t dstbuflen = base64_decoded_length (srclen) + 1; /* +1 for NUL */
+    ssize_t dstlen;
+    char *dst;
 
-    if (!(dst = calloc (1, dstlen)))
+    if (!(dst = malloc (dstbuflen)))
         return -1;
-    if (sodium_base642bin ((unsigned char *)dst, dstlen, src, srclen,
-                           NULL, &dstlen, NULL,
-                           sodium_base64_VARIANT_ORIGINAL) < 0)
+    if ((dstlen = base64_decode (dst, dstbuflen, src, srclen)) < 0)
         goto error_inval;
     *payload = dst;
     *payloadsz = dstlen;
@@ -174,7 +177,7 @@ int sign_none_unwrap (const char *input,
     input = p + 1;
     if (!(p = strchr (input, '.')))
         goto error_inval;
-    if (strcmp (p, ".none") != 0)
+    if (!streq (p, ".none"))
         goto error_inval;
     if (payload_decode (input, p - input, payload, payloadsz) < 0)
         goto error;
diff --git a/src/common/libjob/state.c b/src/common/libjob/state.c
new file mode 100644
index 000000000000..c27858a2243f
--- /dev/null
+++ b/src/common/libjob/state.c
@@ -0,0 +1,53 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "ccan/array_size/array_size.h"
+
+#include "job.h"
+#include "strtab.h"
+
+static struct strtab states[] = {
+    { FLUX_JOB_STATE_NEW,       "NEW",      "new",      "N", "n" },
+    { FLUX_JOB_STATE_DEPEND,    "DEPEND",   "depend",   "D", "d" },
+    { FLUX_JOB_STATE_PRIORITY,  "PRIORITY", "priority", "P", "p" },
+    { FLUX_JOB_STATE_SCHED,     "SCHED",    "sched",    "S", "s" },
+    { FLUX_JOB_STATE_RUN,       "RUN",      "run",      "R", "r" },
+    { FLUX_JOB_STATE_CLEANUP,   "CLEANUP",  "cleanup",  "C", "c" },
+    { FLUX_JOB_STATE_INACTIVE,  "INACTIVE", "inactive", "I", "i" },
+    { FLUX_JOB_STATE_PENDING,   "PENDING",  "pending",  "PD", "pd" },
+    { FLUX_JOB_STATE_RUNNING,   "RUNNING",  "running",  "RU", "ru" },
+    { FLUX_JOB_STATE_ACTIVE,    "ACTIVE",   "active",   "A",  "a" },
+};
+
+
+const char *flux_job_statetostr (flux_job_state_t state, const char *fmt)
+{
+    return strtab_numtostr (state, fmt, states, ARRAY_SIZE (states));
+}
+
+int flux_job_strtostate (const char *s, flux_job_state_t *state)
+{
+    int num;
+
+    if ((num = strtab_strtonum (s, states, ARRAY_SIZE (states))) < 0)
+        return -1;
+    if (state)
+        *state = num;
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/strtab.c b/src/common/libjob/strtab.c
new file mode 100644
index 000000000000..b9aff6439368
--- /dev/null
+++ b/src/common/libjob/strtab.c
@@ -0,0 +1,65 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "ccan/str/str.h"
+
+#include "strtab.h"
+
+static const char *format_entry (struct strtab *entry, const char *fmt)
+{
+    switch (fmt ? *fmt : 0) {
+        case 's':
+            return entry->short_lower;
+        case 'S':
+            return entry->short_upper;
+        case 'l':
+            return entry->long_lower;
+        case 'L':
+        default:
+            return entry->long_upper;
+    }
+}
+
+const char *strtab_numtostr (int num,
+                             const char *fmt,
+                             struct strtab *strtab,
+                             size_t count)
+{
+    struct strtab unknown = { 0, "(unknown)", "(unknown)", "?", "?" };
+
+    for (int i = 0; i < count; i++)
+        if (strtab[i].num == num)
+            return format_entry (&strtab[i], fmt);
+    return format_entry (&unknown, fmt);
+}
+
+int strtab_strtonum (const char *s, struct strtab *strtab, size_t count)
+{
+    if (s) {
+        for (int i = 0; i < count; i++) {
+            if (streq (strtab[i].short_lower, s)
+                || streq (strtab[i].short_upper, s)
+                || streq (strtab[i].long_lower, s)
+                || streq (strtab[i].long_upper, s))
+                return strtab[i].num;
+        }
+    }
+    errno = EINVAL;
+    return -1;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/strtab.h b/src/common/libjob/strtab.h
new file mode 100644
index 000000000000..6096328f2f87
--- /dev/null
+++ b/src/common/libjob/strtab.h
@@ -0,0 +1,39 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/types.h>
+
+#ifndef _LIBJOB_STRTAB_H
+#define _LIBJOB_STRTAB_H
+
+struct strtab {
+    int num;
+    const char *long_upper;
+    const char *long_lower;
+    const char *short_upper;
+    const char *short_lower;
+};
+
+const char *strtab_numtostr (int num,
+                             const char *fmt,
+                             struct strtab *strtab,
+                             size_t count);
+
+int strtab_strtonum (const char *s, struct strtab *strtab, size_t count);
+
+#endif // !_LIBJOB_STRTAB_H
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/submit.c b/src/common/libjob/submit.c
new file mode 100644
index 000000000000..062280caa1de
--- /dev/null
+++ b/src/common/libjob/submit.c
@@ -0,0 +1,158 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <sys/types.h>
+//#include <ctype.h>
+#include <flux/core.h>
+#if HAVE_FLUX_SECURITY
+#include <flux/security/sign.h>
+#endif
+//#include <jansson.h>
+
+#include "job.h"
+#include "sign_none.h"
+//#include "src/common/libutil/fluid.h"
+//#include "src/common/libeventlog/eventlog.h"
+
+#if HAVE_FLUX_SECURITY
+/* If a textual error message is available in flux-security,
+ * enclose it in a future and return.  Otherwise return NULL
+ * with errno set to the original error.
+ */
+static flux_future_t *get_security_error (flux_security_t *sec)
+{
+    int errnum = flux_security_last_errnum (sec);
+    const char *errmsg = flux_security_last_error (sec);
+    flux_future_t *f = NULL;
+
+    if (errmsg && (f = flux_future_create (NULL, NULL)))
+        flux_future_fulfill_error (f, errnum, errmsg);
+    errno = errnum;
+    return f;
+}
+
+/* Cache flux-security context in the handle on first use.
+ * On failure, return NULL and set the value of 'f_error' to NULL,
+ * or if a textual error message is available such as from config
+ * file parsing, set the value of 'f_error' to a future containing
+ * the textual error.
+ */
+static flux_security_t *get_security_ctx (flux_t *h, flux_future_t **f_error)
+{
+    const char *auxkey = "flux::job_security_ctx";
+    flux_security_t *sec = flux_aux_get (h, auxkey);
+
+    if (!sec) {
+        if (!(sec = flux_security_create (0)))
+            goto error;
+        if (flux_security_configure (sec, NULL) < 0)
+            goto error;
+        if (flux_aux_set (h, auxkey, sec,
+                          (flux_free_f)flux_security_destroy) < 0)
+            goto error;
+    }
+    return sec;
+error:
+    *f_error = sec ? get_security_error (sec) : NULL;
+    flux_security_destroy (sec);
+    return NULL;
+}
+#endif
+
+flux_future_t *flux_job_submit (flux_t *h, const char *jobspec, int urgency,
+                                int flags)
+{
+    flux_future_t *f = NULL;
+    const char *J;
+    char *s = NULL;
+    int saved_errno;
+
+    if (!h || !jobspec) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(flags & FLUX_JOB_PRE_SIGNED)) {
+#if HAVE_FLUX_SECURITY
+        flux_security_t *sec;
+        const char *mech = NULL;
+        const char *owner;
+
+        /* Security note:
+         * Instance owner jobs do not need a cryptographic signature since
+         * they do not require the IMP to be executed.  Force the signing
+         * mechanism to 'none' if the broker security.owner matches getuid ().
+         * This side-steps the requirement that the munge daemon is running
+         * for single user instances compiled --with-flux-security, as
+         * described in flux-framework/flux-core#3305.
+         *
+         * This method also works with flux-proxy(1) as described in
+         * flux-framework/flux-core#5530.
+         *
+         * N.B. Guest submissions signed with mech=none are summarily rejected
+         * by job-ingest so the impact of getting this code wrong is job
+         * submission failure, not any weakening of security.
+         */
+        if ((owner = flux_attr_get (h, "security.owner"))) {
+            errno = 0;
+            unsigned int userid = strtoul (owner, NULL, 10);
+            if (errno == 0 && userid == getuid ())
+                mech = "none";
+        }
+        if (!(sec = get_security_ctx (h, &f)))
+            return f;
+        if (!(J = flux_sign_wrap (sec, jobspec, strlen (jobspec), mech, 0)))
+            return get_security_error (sec);
+#else
+        if (!(s = sign_none_wrap (jobspec, strlen (jobspec), getuid ())))
+            goto error;
+        J = s;
+#endif
+    }
+    else {
+        J = jobspec;
+        flags &= ~FLUX_JOB_PRE_SIGNED; // client only flag
+    }
+    if (!(f = flux_rpc_pack (h, "job-ingest.submit", FLUX_NODEID_ANY, 0,
+                             "{s:s s:i s:i}",
+                             "J", J,
+                             "urgency", urgency,
+                             "flags", flags)))
+        goto error;
+    return f;
+error:
+    saved_errno = errno;
+    free (s);
+    errno = saved_errno;
+    return NULL;
+}
+
+int flux_job_submit_get_id (flux_future_t *f, flux_jobid_t *jobid)
+{
+    flux_jobid_t id;
+
+    if (!f) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_rpc_get_unpack (f, "{s:I}",
+                                "id", &id) < 0)
+        return -1;
+    if (jobid)
+        *jobid = id;
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/test/job.c b/src/common/libjob/test/job.c
index b9e2968a9456..35f36a449536 100644
--- a/src/common/libjob/test/job.c
+++ b/src/common/libjob/test/job.c
@@ -12,9 +12,12 @@
 #include "config.h"
 #endif
 
+#include <locale.h>
+
 #include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 struct jobkey_input {
     bool guest;
@@ -73,7 +76,7 @@ void check_one_jobkey (struct jobkey_input *try)
 
     if (try->expected) {
         if (len >= 0 && len == strlen (try->expected)
-                     && !strcmp (path, try->expected))
+                     && streq (path, try->expected))
             valid = true;
     }
     else { // expected failure
@@ -81,8 +84,8 @@ void check_one_jobkey (struct jobkey_input *try)
             valid = true;
     }
     ok (valid == true,
-        "util_jobkey id=%llu key=%s %s",
-        (unsigned long long)try->id,
+        "util_jobkey id=%ju key=%s %s",
+        (uintmax_t)try->id,
         try->key ? try->key : "NULL",
         try->expected ? "works" : "fails");
 
@@ -178,11 +181,11 @@ void check_corner_case (void)
     ok (flux_job_raise (h, 0, NULL, 0, NULL) == NULL && errno == EINVAL,
         "flux_job_raise type=NULL fails with EINVAL");
 
-    /* flux_job_set_priority */
+    /* flux_job_set_urgency */
 
     errno = 0;
-    ok (flux_job_set_priority (NULL, 0, 0) == NULL && errno == EINVAL,
-        "flux_job_set_priority h=NULL fails with EINVAL");
+    ok (flux_job_set_urgency (NULL, 0, 0) == NULL && errno == EINVAL,
+        "flux_job_set_urgency h=NULL fails with EINVAL");
 
     /* flux_job_kvs_key */
 
@@ -226,16 +229,22 @@ struct ss {
     flux_job_state_t state;
     const char *s;
     const char *s_long;
+    const char *s_lower;
+    const char *s_long_lower;
 };
 
 struct ss sstab[] = {
-    { FLUX_JOB_NEW,     "N", "NEW" },
-    { FLUX_JOB_DEPEND,  "D", "DEPEND" },
-    { FLUX_JOB_SCHED,   "S", "SCHED" },
-    { FLUX_JOB_RUN,     "R", "RUN" },
-    { FLUX_JOB_CLEANUP, "C", "CLEANUP" },
-    { FLUX_JOB_INACTIVE,"I", "INACTIVE" },
-    { -1, NULL, NULL },
+    { FLUX_JOB_STATE_NEW,      "N", "NEW", "n", "new" },
+    { FLUX_JOB_STATE_DEPEND,   "D", "DEPEND", "d", "depend" },
+    { FLUX_JOB_STATE_PRIORITY, "P", "PRIORITY", "p", "priority" },
+    { FLUX_JOB_STATE_SCHED,    "S", "SCHED", "s", "sched" },
+    { FLUX_JOB_STATE_RUN,      "R", "RUN", "r", "run" },
+    { FLUX_JOB_STATE_CLEANUP,  "C", "CLEANUP", "c", "cleanup" },
+    { FLUX_JOB_STATE_INACTIVE, "I", "INACTIVE", "i", "inactive" },
+    { FLUX_JOB_STATE_PENDING,  "PD", "PENDING", "pd", "pending" },
+    { FLUX_JOB_STATE_RUNNING,  "RU", "RUNNING", "ru", "running" },
+    { FLUX_JOB_STATE_ACTIVE,   "A",  "ACTIVE",  "a",  "active" },
+    { -1, NULL, NULL, NULL, NULL },
 };
 
 void check_statestr(void)
@@ -243,24 +252,97 @@ void check_statestr(void)
     struct ss *ss;
 
     for (ss = &sstab[0]; ss->s != NULL; ss++) {
-        const char *s = flux_job_statetostr (ss->state, true);
-        const char *s_long = flux_job_statetostr (ss->state, false);
-        ok (s && !strcmp (s, ss->s),
-            "flux_job_statetostr (%d, true) = %s", ss->state, ss->s);
-        ok (s_long && !strcmp (s_long, ss->s_long),
-            "flux_job_statetostr (%d, false) = %s", ss->state, ss->s_long);
+        const char *s = flux_job_statetostr (ss->state, "S");
+        const char *s_long = flux_job_statetostr (ss->state, "L");
+        const char *s_lower = flux_job_statetostr (ss->state, "s");
+        const char *s_long_lower = flux_job_statetostr (ss->state, "l");
+        ok (s && streq (s, ss->s),
+            "flux_job_statetostr (%d, S) = %s", ss->state, ss->s);
+        ok (s_long && streq (s_long, ss->s_long),
+            "flux_job_statetostr (%d, L) = %s", ss->state, ss->s_long);
+        ok (s_lower && streq (s_lower, ss->s_lower),
+            "flux_job_statetostr (%d, s) = %s", ss->state, ss->s_lower);
+        ok (s_long_lower && streq (s_long_lower, ss->s_long_lower),
+            "flux_job_statetostr (%d, l) = %s", ss->state, ss->s_long_lower);
     }
     for (ss = &sstab[0]; ss->s != NULL; ss++) {
         flux_job_state_t state;
-        ok (flux_job_strtostate (ss->s, &state) == 0 && state == ss->state,
+        ok (flux_job_strtostate (ss->s, &state) == 0
+            && state == ss->state,
             "flux_job_strtostate (%s) = %d", ss->s, ss->state);
-        ok (flux_job_strtostate (ss->s_long, &state) == 0 && state == ss->state,
+        ok (flux_job_strtostate (ss->s_long, &state) == 0
+            && state == ss->state,
             "flux_job_strtostate (%s) = %d", ss->s_long, ss->state);
+        ok (flux_job_strtostate (ss->s_lower, &state) == 0
+            && state == ss->state,
+            "flux_job_strtostate (%s) = %d", ss->s_lower, ss->state);
+        ok (flux_job_strtostate (ss->s_long_lower, &state) == 0
+            && state == ss->state,
+            "flux_job_strtostate (%s) = %d", ss->s_long_lower, ss->state);
+    }
+    ok (flux_job_statetostr (0, "S") != NULL,
+        "flux_job_statetostr (0, S) returned non-NULL");
+    ok (flux_job_statetostr (0, "s") != NULL,
+        "flux_job_statetostr (0, s) returned non-NULL");
+    ok (flux_job_statetostr (0, "L") != NULL,
+        "flux_job_statetostr (0, L) returned non-NULL");
+    ok (flux_job_statetostr (0, "l") != NULL,
+        "flux_job_statetostr (0, l) returned non-NULL");
+    ok (flux_job_statetostr (0, "") != NULL,
+        "flux_job_statetostr (0, <empty string>) returned non-NULL");
+    ok (flux_job_statetostr (0, NULL) != NULL,
+        "flux_job_statetostr (0, NULL) returned non-NULL");
+}
+
+struct rr {
+    flux_job_result_t result;
+    const char *r;
+    const char *r_long;
+    const char *r_lower;
+    const char *r_long_lower;
+};
+
+struct rr rrtab[] = {
+    { FLUX_JOB_RESULT_COMPLETED, "CD", "COMPLETED", "cd", "completed" },
+    { FLUX_JOB_RESULT_FAILED,    "F",  "FAILED", "f", "failed" },
+    { FLUX_JOB_RESULT_CANCELED,  "CA", "CANCELED", "ca", "canceled" },
+    { FLUX_JOB_RESULT_TIMEOUT,   "TO", "TIMEOUT", "to", "timeout" },
+    { -1, NULL, NULL, NULL, NULL },
+};
+
+void check_resultstr(void)
+{
+    struct rr *rr;
+
+    for (rr = &rrtab[0]; rr->r != NULL; rr++) {
+        const char *r = flux_job_resulttostr (rr->result, "S");
+        const char *r_long = flux_job_resulttostr (rr->result, "L");
+        const char *r_lower = flux_job_resulttostr (rr->result, "s");
+        const char *r_long_lower = flux_job_resulttostr (rr->result, "l");
+        ok (r && streq (r, rr->r),
+            "flux_job_resulttostr (%d, S) = %s", rr->result, rr->r);
+        ok (r_long && streq (r_long, rr->r_long),
+            "flux_job_resulttostr (%d, L) = %s", rr->result, rr->r_long);
+        ok (r_lower && streq (r_lower, rr->r_lower),
+            "flux_job_resulttostr (%d, s) = %s", rr->result, rr->r_lower);
+        ok (r_long_lower && streq (r_long_lower, rr->r_long_lower),
+            "flux_job_resulttostr (%d, l) = %s", rr->result, rr->r_long_lower);
+    }
+    for (rr = &rrtab[0]; rr->r != NULL; rr++) {
+        flux_job_result_t result;
+        ok (flux_job_strtoresult (rr->r, &result) == 0 && result == rr->result,
+            "flux_job_strtoresult (%s) = %d", rr->r, rr->result);
+        ok (flux_job_strtoresult (rr->r_long, &result) == 0 && result == rr->result,
+            "flux_job_strtoresult (%s) = %d", rr->r_long, rr->result);
     }
-    ok (flux_job_statetostr (0, true) != NULL,
-        "flux_job_statetostr (0, true) returned non-NULL");
-    ok (flux_job_statetostr (0, false) != NULL,
-        "flux_job_statetostr (0, false) returned non-NULL");
+    ok (flux_job_resulttostr (0, "S") != NULL,
+        "flux_job_resulttostr (0, S) returned non-NULL");
+    ok (flux_job_resulttostr (0, "L") != NULL,
+        "flux_job_resulttostr (0, L) returned non-NULL");
+    ok (flux_job_resulttostr (0, "") != NULL,
+        "flux_job_resulttostr (0, <empty string>) returned non-NULL");
+    ok (flux_job_resulttostr (0, NULL) != NULL,
+        "flux_job_resulttostr (0, NULL) returned non-NULL");
 }
 
 void check_kvs_namespace (void)
@@ -278,19 +360,172 @@ void check_kvs_namespace (void)
         "flux_job_kvs_namespace returns EINVAL on invalid buffer");
 }
 
+struct jobid_parse_test {
+    const char *type;
+    flux_jobid_t id;
+    const char *string;
+};
+
+struct jobid_parse_test jobid_parse_tests[] = {
+    { "dec",    0,     "0" },
+    { "hex",    0,     "0x0" },
+    { "dothex", 0,     "0000.0000.0000.0000" },
+    { "kvs",    0,     "job.0000.0000.0000.0000" },
+    { "words",  0,     "academy-academy-academy--academy-academy-academy" },
+    { "emoji",  0,     "😃" },
+#if ASSUME_BROKEN_LOCALE
+    { "f58",    0,     "f1" },
+#else
+    { "f58",    0,     "ƒ1" },
+#endif
+
+    { "dec",    1,     "1" },
+    { "hex",    1,     "0x1" },
+    { "dothex", 1,     "0000.0000.0000.0001" },
+    { "kvs",    1,     "job.0000.0000.0000.0001" },
+    { "words",  1,     "acrobat-academy-academy--academy-academy-academy" },
+    { "emoji",  1,     "😄" },
+#if ASSUME_BROKEN_LOCALE
+    { "f58",    1,     "f2" },
+#else
+    { "f58",    1,     "ƒ2" },
+#endif
+
+    { "dec",    65535, "65535" },
+    { "hex",    65535, "0xffff" },
+    { "dothex", 65535, "0000.0000.0000.ffff" },
+    { "kvs",    65535, "job.0000.0000.0000.ffff" },
+    { "words",  65535, "nevada-archive-academy--academy-academy-academy" },
+    { "emoji",  65535, "💁📚" },
+#if ASSUME_BROKEN_LOCALE
+    { "f58",    65535, "fLUv" },
+#else
+    { "f58",    65535, "ƒLUv" },
+#endif
+
+    { "dec",    6787342413402046, "6787342413402046" },
+    { "hex",    6787342413402046, "0x181d0d4d850fbe" },
+    { "dothex", 6787342413402046, "0018.1d0d.4d85.0fbe" },
+    { "kvs",    6787342413402046, "job.0018.1d0d.4d85.0fbe" },
+    { "words",  6787342413402046, "cake-plume-nepal--neuron-pencil-academy" },
+    { "emoji",  6787342413402046, "👴😱🔚🎮🕙🚩" },
+#if ASSUME_BROKEN_LOCALE
+    { "f58",    6787342413402046, "fuzzybunny" },
+#else
+    { "f58",    6787342413402046, "ƒuzzybunny" },
+#endif
+
+    { NULL, 0, NULL }
+};
+
+void check_jobid_parse_encode (void)
+{
+    char buf[1024];
+    flux_jobid_t jobid;
+    struct jobid_parse_test *tp = jobid_parse_tests;
+    while (tp->type != NULL) {
+        memset (buf, 0, sizeof (buf));
+        if (MB_CUR_MAX == 1 && streq (tp->type, "f58")) {
+            tap_skip (4, "Skipping F58 encode/decode due to current locale");
+            tp++;
+            continue;
+        }
+        ok (flux_job_id_encode (tp->id, tp->type, buf, sizeof (buf)) == 0,
+            "flux_job_id_encode (%ju, %s) == 0", (uintmax_t) tp->id, tp->type);
+        is (buf, tp->string,
+            "flux_job_id_encode() got %s", buf);
+        ok (flux_job_id_parse (buf, &jobid) == 0,
+            "flux_job_id_parse() of result works: %s", strerror (errno));
+        ok (jobid == tp->id,
+            "flux_job_id_parse() returned correct id");
+        tp++;
+    }
+
+    ok (flux_job_id_encode (1234, NULL, buf, sizeof (buf)) == 0,
+        "flux_job_id_encode() with NULL type works");
+    is (buf, "1234",
+        "flux_job_id_encode() encodes to decimal by default");
+
+    ok (flux_job_id_parse ("  1234  ", &jobid) == 0,
+        "flux_job_id_parse works with leading whitespace");
+    ok (jobid == 1234,
+        "flux_job_id_parse got expected result");
+
+    ok (flux_job_id_encode (1234, NULL, NULL, 33) < 0 && errno == EINVAL,
+        "flux_job_id_encode with NULL buffer returns EINVAL");
+    ok (flux_job_id_encode (1234, "dec", buf, 4) < 0 && errno == EOVERFLOW,
+        "flux_job_id_encode with too small buffer returns EOVERFLOW");
+    ok (flux_job_id_encode (1234, "dothex", buf, 19) < 0 && errno == EOVERFLOW,
+        "flux_job_id_encode with too small buffer returns EOVERFLOW");
+    ok (flux_job_id_encode (1234, "foo", buf, 1024) < 0 && errno == EPROTO,
+        "flux_job_id_encode with unknown encode type returns EPROTO");
+}
+
+static void check_job_timeleft (void)
+{
+    flux_t *h = (flux_t *)(uintptr_t)42; // fake but non-NULL
+    flux_error_t error;
+    double timeleft;
+
+    ok (flux_job_timeleft (NULL, &error, &timeleft) < 0 && errno == EINVAL,
+        "flux_job_timeleft (NULL, ...) returns EINVAL");
+    ok (flux_job_timeleft (h, &error, NULL) < 0 && errno == EINVAL,
+        "flux_job_timeleft (h, error, NULL) returns EINVAL");
+}
+
+static void check_waitstatus_to_exitcode (void)
+{
+    flux_error_t error;
+    ok (flux_job_waitstatus_to_exitcode (-1, &error) < 0 && errno == EINVAL,
+        "flux_job_waitstatus_to_exitcode (-1) returns EINVAL");
+    is (error.text, "unexpected wait(2) status -1",
+        "error.text is %s", error.text);
+    ok (flux_job_waitstatus_to_exitcode (0, &error) == 0,
+        "flux_job_waitstatus_to_exitcode (0) returns 0");
+    is (error.text, "",
+        "error.text is cleared");
+    ok (flux_job_waitstatus_to_exitcode (9, &error) == 128+9,
+        "flux_job_waitstatus_to_exitcode (9) == %d", 128+9);
+    is (error.text, "job shell Killed",
+        "error.text is %s", error.text);
+    ok (flux_job_waitstatus_to_exitcode (1<<8, &error) == 1,
+        "flux_job_waitstatus_to_exitcode (1<<8) = 1");
+    is (error.text, "task(s) exited with exit code 1",
+        "error.text is %s", error.text);
+    ok (flux_job_waitstatus_to_exitcode ((128+15)<< 8, &error) == 128+15,
+        "flux_job_waitstatus_to_exitcode ((128+15)<<8) = 128+15");
+    is (error.text, "task(s) Terminated",
+        "error.text is %s", error.text);
+    ok (flux_job_waitstatus_to_exitcode ((128+11)<<8, &error) == 128+11,
+        "flux_job_waitstatus_to_exitcode ((128+11)<<8) = 128+11");
+    is (error.text, "task(s) Segmentation fault",
+        "error.text is %s", error.text);
+}
 
 int main (int argc, char *argv[])
 {
     plan (NO_PLAN);
 
+    /* fluid F58 tests require unicode locale initialization */
+    setlocale (LC_ALL, "en_US.UTF-8");
+    unsetenv ("FLUX_F58_FORCE_ASCII");
+
     check_jobkey ();
 
     check_corner_case ();
 
     check_statestr ();
 
+    check_resultstr ();
+
     check_kvs_namespace ();
 
+    check_jobid_parse_encode ();
+
+    check_job_timeleft ();
+
+    check_waitstatus_to_exitcode ();
+
     done_testing ();
     return 0;
 }
diff --git a/src/common/libjob/test/jobspec1.c b/src/common/libjob/test/jobspec1.c
new file mode 100644
index 000000000000..b23378186e0f
--- /dev/null
+++ b/src/common/libjob/test/jobspec1.c
@@ -0,0 +1,435 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <jansson.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "src/common/libjob/jobspec1.h"
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+extern char **environ;
+
+void check_stdio_cwd (void)
+{
+    char *argv[] = {"this", "is", "a", "test"};
+    int argc = ARRAY_SIZE (argv);
+    flux_jobspec1_t *jobspec;
+    char *path;
+
+    if (!(jobspec = flux_jobspec1_from_command (argc, argv, NULL, 1, 1, 1, 0, 0.0))) {
+        BAIL_OUT ("flux_jobspec1_from_command failed");
+    }
+    ok (flux_jobspec1_set_cwd (NULL, "/foo/bar/baz") < 0 && errno == EINVAL,
+        "flux_jobspec1_set_cwd catches NULL jobspec");
+    errno = 0;
+    ok (flux_jobspec1_set_cwd (jobspec, NULL) < 0 && errno == EINVAL,
+        "flux_jobspec1_set_cwd catches NULL cwd");
+    errno = 0;
+    ok (flux_jobspec1_set_cwd (jobspec, "/foo/bar/baz") == 0
+            && (flux_jobspec1_attr_unpack (jobspec, "system.cwd", "s", &path) == 0)
+
+            && streq ("/foo/bar/baz", path),
+        "flux_jobspec1_set_cwd works");
+    ok (flux_jobspec1_set_stdin (NULL, "/foo/bar/baz") < 0 && errno == EINVAL,
+        "flux_jobspec1_set_stdin catches NULL jobspec");
+    errno = 0;
+    ok (flux_jobspec1_set_stdin (jobspec, NULL) < 0 && errno == EINVAL,
+        "flux_jobspec1_set_stdin catches NULL path");
+    errno = 0;
+    ok (flux_jobspec1_set_stdin (jobspec, "/foo/bar/stdin.txt") == 0
+            && (flux_jobspec1_attr_unpack (jobspec,
+                                           "system.shell.options.input.stdin.path",
+                                           "s",
+                                           &path)
+                == 0)
+
+            && streq ("/foo/bar/stdin.txt", path),
+        "flux_jobspec1_set_stdin sets right path");
+    ok ((flux_jobspec1_attr_unpack (jobspec,
+                                    "system.shell.options.input.stdin.type",
+                                    "s",
+                                    &path)
+         == 0)
+            && streq ("file", path),
+        "flux_jobspec1_set_stdin sets right type");
+    ok (flux_jobspec1_set_stdout (NULL, "/foo/bar/baz") < 0 && errno == EINVAL,
+        "flux_jobspec1_set_stdout catches NULL jobspec");
+    errno = 0;
+    ok (flux_jobspec1_set_stdout (jobspec, NULL) < 0 && errno == EINVAL,
+        "flux_jobspec1_set_stdout catches NULL path");
+    errno = 0;
+    ok (flux_jobspec1_set_stdout (jobspec, "/foo/bar/stdout.txt") == 0
+            && (flux_jobspec1_attr_unpack (jobspec,
+                                           "system.shell.options.output.stdout.path",
+                                           "s",
+                                           &path)
+                == 0)
+
+            && streq ("/foo/bar/stdout.txt", path),
+        "flux_jobspec1_set_stdout sets right path");
+    ok ((flux_jobspec1_attr_unpack (jobspec,
+                                    "system.shell.options.output.stdout.type",
+                                    "s",
+                                    &path)
+         == 0)
+            && streq ("file", path),
+        "flux_jobspec1_set_stdout sets right type");
+    ok (flux_jobspec1_set_stderr (NULL, "/foo/bar/baz") < 0 && errno == EINVAL,
+        "flux_jobspec1_set_stderr catches NULL jobspec");
+    errno = 0;
+    ok (flux_jobspec1_set_stderr (jobspec, NULL) < 0 && errno == EINVAL,
+        "flux_jobspec1_set_stderr catches NULL path");
+    errno = 0;
+    ok (flux_jobspec1_set_stderr (jobspec, "/foo/bar/stderr.txt") == 0
+            && (flux_jobspec1_attr_unpack (jobspec,
+                                           "system.shell.options.output.stderr.path",
+                                           "s",
+                                           &path)
+                == 0)
+
+            && streq ("/foo/bar/stderr.txt", path),
+        "flux_jobspec1_set_stderr sets right path");
+    ok ((flux_jobspec1_attr_unpack (jobspec,
+                                    "system.shell.options.output.stderr.type",
+                                    "s",
+                                    &path)
+         == 0)
+            && streq ("file", path),
+        "flux_jobspec1_set_stderr sets right type");
+    flux_jobspec1_destroy (jobspec);
+}
+
+void check_env (void)
+{
+    char *argv[] = {"this", "is", "a", "test"};
+    int argc = ARRAY_SIZE (argv);
+    flux_jobspec1_t *jobspec;
+    char *val;
+
+    if (!(jobspec =
+              flux_jobspec1_from_command (argc, argv, environ, 1, 1, 1, 0, 0.0))) {
+        BAIL_OUT ("flux_jobspec1_from_command failed with environ");
+    }
+    ok (flux_jobspec1_setenv (NULL, "FOO", "BAR", 1) < 0 && errno == EINVAL,
+        "flux_jobspec1_setenv catches NULL jobspec");
+    errno = 0;
+    ok (flux_jobspec1_setenv (jobspec, NULL, "BAR", 1) < 0 && errno == EINVAL,
+        "flux_jobspec1_setenv catches NULL variable name");
+    errno = 0;
+    ok (flux_jobspec1_setenv (jobspec, "FOO", NULL, 1) < 0 && errno == EINVAL,
+        "flux_jobspec1_setenv catches NULL variable value");
+    errno = 0;
+    ok (flux_jobspec1_unsetenv (NULL, "FOO") < 0 && errno == EINVAL,
+        "flux_jobspec1_unsetenv catches NULL jobspec");
+    errno = 0;
+    ok (flux_jobspec1_unsetenv (jobspec, NULL) < 0 && errno == EINVAL,
+        "flux_jobspec1_unsetenv catches NULL variable");
+    errno = 0;
+    ok (flux_jobspec1_setenv (jobspec, "FOO1", "BAR1", 1) == 0
+            && (flux_jobspec1_attr_unpack (jobspec,
+                                           "system.environment.FOO1",
+                                           "s",
+                                           &val)
+                == 0)
+            && streq ("BAR1", val),
+        "jobspec_setenv FOO1=BAR1 works");
+    ok (flux_jobspec1_setenv (jobspec, "FOO1", "BAZ1", 1) == 0
+            && (flux_jobspec1_attr_unpack (jobspec,
+                                           "system.environment.FOO1",
+                                           "s",
+                                           &val)
+                == 0)
+            && streq ("BAZ1", val),
+        "jobspec_setenv FOO1=BAZ1 works (overwrite=1)");
+    ok (flux_jobspec1_setenv (jobspec, "FOO1", "BAZ2", 0) == 0
+            && (flux_jobspec1_attr_unpack (jobspec,
+                                           "system.environment.FOO1",
+                                           "s",
+                                           &val)
+                == 0)
+            && streq ("BAZ1", val),
+        "jobspec_setenv FOO1=BAZ2 works (overwrite=0)");
+    ok (flux_jobspec1_unsetenv (jobspec, "FOO1") == 0
+            && (flux_jobspec1_attr_unpack (jobspec,
+                                           "system.environment.FOO1",
+                                           "s",
+                                           &val)
+                < 0),
+        "unset FOO1 works");
+    ok (flux_jobspec1_setenv (jobspec, "FOO2", "BAR2", 1) == 0
+            && (flux_jobspec1_attr_unpack (jobspec,
+                                           "system.environment.FOO2",
+                                           "s",
+                                           &val)
+                == 0)
+            && streq ("BAR2", val),
+        "jobspec_setenv FOO2=BAR2 works");
+
+    // ensure empty ("") value works
+    ok (flux_jobspec1_setenv (jobspec, "empty", "", 1) == 0,
+        "flux_jobspec1_setenv accepts empty string value");
+    ok (flux_jobspec1_attr_unpack (jobspec,
+                                  "system.environment.empty",
+                                  "s",
+                                  &val) == 0
+        && val != NULL
+        && streq (val, ""),
+        "empty string value was correctly represented in object");
+    // test functions when environment object is deleted
+    ok (flux_jobspec1_attr_del (jobspec, "system.environment") == 0,
+        "deleting environment works");
+    ok (flux_jobspec1_setenv (jobspec, "FOO1", "BAR1", 1) == 0,
+        "flux_jobspec1_setenv works after deleting environment object");
+    ok (flux_jobspec1_unsetenv (jobspec, "FOO") == 0,
+        "flux_jobspec1_unsetenv works after deleting environment");
+    flux_jobspec1_destroy (jobspec);
+}
+
+void check_attr (void)
+{
+    char *argv[] = {"this", "is", "a", "test"};
+    int argc = ARRAY_SIZE (argv);
+    flux_jobspec1_t *jobspec;
+    json_t *json_ptr;
+    int int_val;
+    char *char_ptr;
+
+    if (!(jobspec = flux_jobspec1_from_command (argc, argv, NULL, 1, 1, 1, 0, 0.0))) {
+        BAIL_OUT ("flux_jobspec1_from_command failed");
+    }
+    ok (flux_jobspec1_attr_pack (NULL, "foo.bar", "i", 5) < 0 && errno == EINVAL,
+        "flux_jobspec1_attr_pack catches NULL jobspec");
+    errno = 0;
+    ok (flux_jobspec1_attr_pack (jobspec, NULL, "i", 5) < 0 && errno == EINVAL,
+        "flux_jobspec1_attr_pack catches NULL path");
+    errno = 0;
+    ok (flux_jobspec1_attr_pack (jobspec, "foo.bar", NULL, 5) < 0 && errno == EINVAL,
+        "flux_jobspec1_attr_pack catches NULL format string");
+    errno = 0;
+    ok (flux_jobspec1_attr_unpack (NULL, "foo.bar", "i", 5) < 0 && errno == EINVAL,
+        "flux_jobspec1_attr_unpack catches NULL jobspec");
+    errno = 0;
+    ok (flux_jobspec1_attr_unpack (jobspec, NULL, "i", 5) < 0 && errno == EINVAL,
+        "flux_jobspec1_attr_unpack catches NULL path");
+    errno = 0;
+    ok (flux_jobspec1_attr_unpack (jobspec, "foo.bar", NULL, 5) < 0 && errno == EINVAL,
+        "flux_jobspec1_attr_unpack catches NULL format string");
+    errno = 0;
+    ok (flux_jobspec1_attr_del (jobspec, NULL) < 0 && errno == EINVAL,
+        "flux_jobspec1_attr_del catches NULL path");
+    errno = 0;
+    ok (flux_jobspec1_attr_del (NULL, "foo.bar") < 0 && errno == EINVAL,
+        "flux_jobspec1_attr_del catches NULL jobspec");
+    errno = 0;
+
+    ok (flux_jobspec1_attr_pack (jobspec, "foo.bar", "s", "baz") == 0
+            && (flux_jobspec1_attr_unpack (jobspec, "foo.bar", "s", &char_ptr) == 0)
+            && streq ("baz", char_ptr),
+        "flux_jobspec1_attr_pack works on strings");
+    ok (flux_jobspec1_attr_pack (jobspec, "foo.bar", "i", 19) == 0
+            && (flux_jobspec1_attr_unpack (jobspec, "foo.bar", "i", &int_val) == 0)
+            && 19 == int_val,
+        "flux_jobspec1_attr_pack works on integers");
+    ok (flux_jobspec1_attr_pack (jobspec, "foo", "{s:s}", "bar", "baz") == 0
+            && (flux_jobspec1_attr_unpack (jobspec, "foo", "{s:s}", "bar", &char_ptr)
+                == 0)
+            && streq ("baz", char_ptr),
+        "flux_jobspec1_attr_pack works on objects");
+    ok (flux_jobspec1_attr_del (jobspec, "foo.bar.baz") == 0
+            && (flux_jobspec1_attr_unpack (jobspec, "foo.bar.baz", "o", &json_ptr) < 0),
+        "flux_jobspec1_attr_del works");
+    ok (flux_jobspec1_attr_del (jobspec, "foo") == 0
+            && (flux_jobspec1_attr_unpack (jobspec, "foo", "o", &json_ptr) < 0),
+        "flux_jobspec1_attr_del works");
+    flux_jobspec1_destroy (jobspec);
+}
+
+void check_jobspec (void)
+{
+    char *argv[] = {"this", "is", "a", "test"};
+    int argc = ARRAY_SIZE (argv);
+    flux_jobspec1_t *jobspec;
+    flux_jobspec1_error_t error;
+    char *str;
+    json_t *val;
+    double passed_duration = 5.0;
+    double duration;
+
+    if (!(jobspec = flux_jobspec1_from_command (argc,
+                                                argv,
+                                                NULL,
+                                                1,
+                                                1,
+                                                1,
+                                                0,
+                                                passed_duration))) {
+        BAIL_OUT ("flux_jobspec1_from_command failed");
+    }
+    errno = 0;
+    ok (flux_jobspec1_attr_check (NULL, &error) < 0 && errno == EINVAL,
+        "flux_jobspec1_attr_check catches NULL jobspec");
+    ok (flux_jobspec1_attr_check (jobspec, NULL) == 0,
+        "flux_jobspec1_attr_check works with NULL error struct");
+
+    ok (flux_jobspec1_attr_check (jobspec, &error) == 0,
+        "flux_jobspec1_attr_check passed");
+    ok (flux_jobspec1_attr_unpack (jobspec, "system", "o", &val) == 0,
+        "jobspec has system attribute");
+    ok (flux_jobspec1_attr_unpack (jobspec, "system.duration", "f", &duration) == 0
+            && (duration - passed_duration < 0.001),
+        "jobspec has system.duration attribute set to correct value");
+    ok (flux_jobspec1_attr_unpack (jobspec, "system.environment", "o", &val) == 0
+            && json_is_object (val) && json_object_size (val) == 0,
+        "jobspec has system.environment object of size 0");
+    ok (flux_jobspec1_attr_unpack (jobspec, "foo.bar", "s", &str) < 0,
+        "jobspec has no foo.bar attribute");
+    flux_jobspec1_destroy (jobspec);
+    passed_duration = 0.0;
+    if (!(jobspec = flux_jobspec1_from_command (argc,
+                                                argv,
+                                                NULL,
+                                                1,
+                                                1,
+                                                1,
+                                                0,
+                                                passed_duration))) {
+        BAIL_OUT ("flux_jobspec1_from_command failed");
+    }
+    ok ((flux_jobspec1_attr_unpack (jobspec, "system.duration", "f", &duration) == 0)
+            && (duration == passed_duration),
+        "jobspec has system.duration attribute set to correct value");
+    flux_jobspec1_destroy (jobspec);
+    ok (flux_jobspec1_from_command (argc, argv, NULL, 1, 1, 1, 5, 0) == NULL,
+        "flux_jobspec1_from_command failed when nnodes > ntasks");
+    if (!(jobspec = flux_jobspec1_from_command (argc, argv, NULL, 5, 1, 1, 3, 0.0))) {
+        BAIL_OUT ("flux_jobspec1_from_command failed when nnodes < ntasks");
+    }
+    ok (flux_jobspec1_attr_check (jobspec, &error) == 0,
+        "flux_jobspec1_attr_check passed when nnodes < ntasks");
+    ok (flux_jobspec1_attr_pack (jobspec, "system.duration", "s", "not a number") == 0
+            && flux_jobspec1_attr_unpack (jobspec, "system.duration", "f", &duration)
+                   < 0,
+        "deleting system.duration works");
+    ok (flux_jobspec1_attr_check (jobspec, &error) < 0,
+        "flux_jobspec1_attr_check failed after changing system.duration to a string");
+    flux_jobspec1_destroy (jobspec);
+    if (!(jobspec = flux_jobspec1_from_command (argc, argv, NULL, 5, 1, 1, 5, 0.0))) {
+        BAIL_OUT ("flux_jobspec1_from_command failed when nnodes == ntasks");
+    }
+    ok (flux_jobspec1_attr_check (jobspec, &error) == 0,
+        "flux_jobspec1_attr_check passed when nnodes == ntasks");
+    ok (flux_jobspec1_attr_pack (jobspec, "foo", "f", 19.5) == 0
+            && flux_jobspec1_attr_check (jobspec, &error) < 0,
+        "attr_check failed after adding spurious attribute");
+    flux_jobspec1_destroy (jobspec);
+}
+
+void check_encoding (void)
+{
+    char *argv[] = {"this", "is", "a", "test"};
+    int argc = ARRAY_SIZE (argv);
+    flux_jobspec1_t *jobspec;
+    char *encoded;
+    flux_jobspec1_t *dup;
+    flux_jobspec1_error_t error;
+
+    if (!(jobspec = flux_jobspec1_from_command (argc, argv,
+                                                NULL, 5, 3, 2, 0, 0.0)))
+        BAIL_OUT ("flux_jobspec1_from_command failed");
+
+    ok (flux_jobspec1_check (jobspec, &error) == 0,
+        "flux_jobspec1_check returns success on valid jobspec");
+
+    ok ((encoded = flux_jobspec1_encode (jobspec, 0)) != NULL,
+        "flux_jobspec1_encode works");
+    ok ((dup = flux_jobspec1_decode (encoded, &error)) != NULL,
+        "flux_jobspec1_decode works");
+    free (encoded);
+    flux_jobspec1_destroy (dup);
+
+    errno = EINVAL;
+    flux_jobspec1_destroy (jobspec);
+    ok (errno == EINVAL,
+        "flux_jobspec1_destroy preserves errno");
+
+    errno = 0;
+    ok (flux_jobspec1_encode (NULL, 0) == NULL && errno == EINVAL,
+        "flux_jobspec1_encode catches NULL jobspec");
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok (flux_jobspec1_decode ("{", &error) == NULL
+        && errno == EINVAL
+        && error.text[0] != '\0',
+        "flux_jobspec1_decode on bad JSON fails with EINVAL and error buf set");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok (flux_jobspec1_decode (NULL, &error) == NULL
+        && errno == EINVAL
+        && error.text[0] != '\0',
+        "flux_jobspec1_decode NULL fails with EINVAL and error buf set");
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok (flux_jobspec1_check (NULL, &error) < 0
+        && errno == EINVAL
+        && error.text[0] != '\0',
+        "flux_jobspec1_check NULL fails with EINVAL and error buf set");
+}
+
+void check_bad_args (void)
+{
+    char *argv[] = {"this", "is", "a", "test"};
+    int argc = ARRAY_SIZE (argv);
+
+    ok (flux_jobspec1_from_command (-1, argv, NULL, 1, 1, 1, 0, 5.0) == NULL
+            && errno == EINVAL,
+        "flux_jobspec1_from_command catches bad argc");
+    errno = 0;
+    ok (flux_jobspec1_from_command (argc, NULL, NULL, 1, 1, 1, 0, 5.0) == NULL
+            && errno == EINVAL,
+        "flux_jobspec1_from_command catches bad argv");
+    errno = 0;
+    ok (flux_jobspec1_from_command (argc, argv, NULL, 1, 1, 1, 0, -1.5) == NULL
+            && errno == EINVAL,
+        "flux_jobspec1_from_command catches bad duration");
+    errno = 0;
+
+    flux_jobspec1_destroy (NULL);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    check_stdio_cwd ();
+    check_env ();
+    check_jobspec ();
+    check_attr ();
+    check_encoding ();
+    check_bad_args ();
+
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/test/sign_none.c b/src/common/libjob/test/sign_none.c
index 28dfa7d8579f..6431a7ed7ed3 100644
--- a/src/common/libjob/test/sign_none.c
+++ b/src/common/libjob/test/sign_none.c
@@ -13,10 +13,14 @@
 #endif
 
 #include <flux/core.h>
-#include <sodium.h>
+#if HAVE_FLUX_SECURITY
+#include <flux/security/sign.h>
+#endif
 
 #include "src/common/libtap/tap.h"
 #include "src/common/libjob/sign_none.h"
+#include "ccan/base64/base64.h"
+#include "ccan/str/str.h"
 
 void simple (void)
 {
@@ -40,22 +44,23 @@ void simple (void)
     ok (rc == 0 && userid == 1000
         && payloadsz == 4 && memcmp (payload, "foo", 4) == 0,
         "sign_none_unwrap works");
+    free (payload);
 
     free (s);
 }
 
-char *encode_base64 (const void *src, int srclen)
+char *encode_base64 (const char *src, int srclen)
 {
-    int dstlen = sodium_base64_encoded_len (srclen,
-                                            sodium_base64_VARIANT_ORIGINAL);
-    char *dst = calloc (1, dstlen);
+    size_t dstbuflen = base64_encoded_length (srclen) + 1; /* +1 for NUL */
+    char *dst = malloc (dstbuflen);
     if (!dst)
         BAIL_OUT ("calloc failed");
-    return sodium_bin2base64 (dst, dstlen, (unsigned char *)src, srclen,
-                              sodium_base64_VARIANT_ORIGINAL);
+    if (base64_encode (dst, dstbuflen, src, srclen) < 0)
+        BAIL_OUT ("base64_encode");
+    return dst;
 }
 
-char *wrap (const char *header, int headerlen, void *payload, int payloadlen)
+char *wrap (const char *header, int headerlen, char *payload, int payloadlen)
 {
     char *h = encode_base64 (header, headerlen);
     char *p = encode_base64 (payload, payloadlen);
@@ -72,11 +77,11 @@ char *wrap (const char *header, int headerlen, void *payload, int payloadlen)
  */
 void decode_good (void)
 {
-    const char good1_header[] = "version\0i1\0userid\0i1000\0mech\0snone";
+    const char good1_header[] = "version\0i1\0userid\0i1000\0mechanism\0snone";
     char *good1 = wrap (good1_header, sizeof (good1_header), "foo", 4);
-    const char good2_header[] = "userid\0i1000\0mech\0snone\0version\0i1";
+    const char good2_header[] = "userid\0i1000\0mechanism\0snone\0version\0i1";
     char *good2 = wrap (good2_header, sizeof (good2_header), NULL, 0);
-    const char good3_header[] = "mech\0snone\0version\0i1\0userid\0i0";
+    const char good3_header[] = "mechanism\0snone\0version\0i1\0userid\0i0";
     char *good3 = wrap (good3_header, sizeof (good3_header), "", 1);
 
     uint32_t userid;
@@ -92,6 +97,7 @@ void decode_good (void)
     ok (rc == 0 && userid == 1000
         && payloadsz == 4 && memcmp (payload, "foo", 4) == 0,
         "dummy encode 1 decodes as expected");
+    free (payload);
 
     userid = 0;
     payload = NULL;
@@ -100,6 +106,7 @@ void decode_good (void)
     rc = sign_none_unwrap (good2, &payload, &payloadsz, &userid);
     ok (rc == 0 && userid == 1000 && payloadsz == 0,
         "dummy encode 2 decodes as expected");
+    free (payload);
 
     userid = 1;
     payload = NULL;
@@ -108,6 +115,7 @@ void decode_good (void)
     rc = sign_none_unwrap (good3, &payload, &payloadsz, &userid);
     ok (rc == 0 && userid == 0 && payloadsz == 1 && *(char *)payload == '\0',
         "dummy encode 3 decodes as expected");
+    free (payload);
 
     free (good1);
     free (good2);
@@ -117,37 +125,37 @@ void decode_good (void)
 void decode_bad_header (void)
 {
     /* version 2 */
-    const char bad1_header[] = "version\0i2\0userid\0i1000\0mech\0snone";
+    const char bad1_header[] = "version\0i2\0userid\0i1000\0mechanism\0snone";
     char *bad1 = wrap (bad1_header, sizeof (bad1_header), NULL, 0);
     /* string version */
-    const char bad2_header[] = "version\0s1\0userid\0i1000\0mech\0snone";
+    const char bad2_header[] = "version\0s1\0userid\0i1000\0mechanism\0snone";
     char *bad2 = wrap (bad2_header, sizeof (bad2_header), NULL, 0);
     /* missing version */
-    const char bad3_header[] = "userid\0i1000\0mech\0snone";
+    const char bad3_header[] = "userid\0i1000\0mechanism\0snone";
     char *bad3 = wrap (bad3_header, sizeof (bad3_header), NULL, 0);
     /* extra foo field */
-    const char bad4_header[] = "foo\0i0\0version\0i1\0userid\0i1000\0mech\0snone";
+    const char bad4_header[] = "foo\0i0\0version\0i1\0userid\0i1000\0mechanism\0snone";
     char *bad4 = wrap (bad4_header, sizeof (bad4_header), NULL, 0);
     /* negative userid */
-    const char bad5_header[] = "version\0i1\0userid\0i-1\0mech\0snone";
+    const char bad5_header[] = "version\0i1\0userid\0i-1\0mechanism\0snone";
     char *bad5 = wrap (bad5_header, sizeof (bad5_header), NULL, 0);
     /* wrong type userid */
-    const char bad6_header[] = "version\0i1\0userid\0s42\0mech\0snone";
+    const char bad6_header[] = "version\0i1\0userid\0s42\0mechanism\0snone";
     char *bad6 = wrap (bad6_header, sizeof (bad6_header), NULL, 0);
     /* missing userid */
-    const char bad7_header[] = "version\0i1\0mech\0snone";
+    const char bad7_header[] = "version\0i1\0mechanism\0snone";
     char *bad7 = wrap (bad7_header, sizeof (bad7_header), NULL, 0);
     /* wrong mech */
-    const char bad8_header[] = "version\0i1\0userid\0i1000\0mech\0smunge";
+    const char bad8_header[] = "version\0i1\0userid\0i1000\0mechanism\0smunge";
     char *bad8 = wrap (bad8_header, sizeof (bad8_header), NULL, 0);
     /* wrong type mech */
-    const char bad9_header[] = "version\0i1\0userid\0i1000\0mech\0inone";
+    const char bad9_header[] = "version\0i1\0userid\0i1000\0mechanism\0inone";
     char *bad9 = wrap (bad9_header, sizeof (bad9_header), NULL, 0);
     /* missing mech */
     const char bad10_header[] = "version\0i1\0userid\0i1000";
     char *bad10 = wrap (bad10_header, sizeof (bad10_header), NULL, 0);
     /* extra separator */
-    const char bad11_header[] = "\0version\0i1\0userid\0i1000\0mech\0snone";
+    const char bad11_header[] = "\0version\0i1\0userid\0i1000\0mechanism\0snone";
     char *bad11 = wrap (bad11_header, sizeof (bad11_header), NULL, 0);
 
     uint32_t userid;
@@ -208,7 +216,7 @@ void decode_bad_header (void)
     errno = 0;
     rc = sign_none_unwrap (bad11, &payload, &payloadsz, &userid);
     ok (rc < 0 && errno == EINVAL,
-        "sign_none_unwrap extra seprator fails with EINVAL");
+        "sign_none_unwrap extra separator fails with EINVAL");
 
     free (bad1);
     free (bad2);
@@ -225,17 +233,18 @@ void decode_bad_header (void)
 
 void decode_bad_other (void)
 {
-    const char *good = "dmVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaABzbm9uZQA=.Zm9vAA==.none";
+    const char *good = "dmVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaGFuaXNtAHNub25lAA==.Zm9vAA==.none";
     /* wrong suffix */
-    const char *bad1  = "dmVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaABzbm9uZQA=.Zm9vAA==.wrong";
+    const char *bad1  = "dmVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaGFuaXNtAHNub25lAA==.Zm9vAA==.wrong";
+
     /* missing field */
-    const char *bad2  = "dmVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaABzbm9uZQA=.none";
+    const char *bad2  = "dmVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaGFuaXNtAHNub25lAA==.none";
     /* two missing fields */
     const char *bad3  = "none";
     /* invalid base64 payload (% character) */
-    const char *bad4  = "dmVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaABzbm9uZQA=.%m9vAA==.none";
+    const char *bad4  = "dmVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaGFuaXNtAHNub25lAA==.%m9vAA==.none";
     /* invalid base64 header (% character) */
-    const char *bad5  = "%mVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaABzbm9uZQA=.Zm9vAA==.none";
+    const char *bad5 = "%mVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaGFuaXNtAHNub25lAA==.Zm9vAA==.none";
 
     uint32_t userid;
     void *payload;
@@ -278,7 +287,7 @@ void decode_bad_other (void)
     errno = 0;
     rc = sign_none_unwrap ("", &payload, &payloadsz, &userid);
     ok (rc < 0 && errno == EINVAL,
-        "sign_none_unwrap emtpy input fails with EINVAL");
+        "sign_none_unwrap empty input fails with EINVAL");
 }
 
 
@@ -292,7 +301,7 @@ void badarg (void)
 
     /* unwrap */
 
-    const char good1_header[] = "version\0i1\0userid\0i1000\0mech\0snone";
+    const char good1_header[] = "version\0i1\0userid\0i1000\0mechanism\0snone";
     char *good1 = wrap (good1_header, sizeof (good1_header), "foo", 4);
 
     errno = 0;
@@ -331,6 +340,82 @@ void badarg (void)
 
 }
 
+#if HAVE_FLUX_SECURITY
+void interop_sign_core (void)
+{
+    flux_security_t *sec;
+    const char *mech_type;
+    int64_t userid;
+    const void *payload;
+    int payloadsz;
+    char *s;
+    int rc;
+
+    if (!(sec = flux_security_create (0)))
+        BAIL_OUT ("error creating flux-security context");
+    if (flux_security_configure (sec, NULL) < 0)
+        BAIL_OUT ("error configuring flux-security");
+
+    s = sign_none_wrap ("foo", 4, 1000);
+    if (!s)
+        BAIL_OUT ("sign_none_wrap returned NULL");
+
+    userid = 0;
+    payload = NULL;
+    payloadsz = 0;
+    rc = flux_sign_unwrap_anymech (sec,
+                                   s,
+                                   &payload,
+                                   &payloadsz,
+                                   &mech_type,
+                                   &userid,
+                                   FLUX_SIGN_NOVERIFY);
+    if (rc < 0)
+        diag ("unwrap: %s", flux_security_last_error (sec));
+    ok (rc == 0
+        && userid == 1000
+        && streq (mech_type, "none")
+        && payloadsz == 4
+        && memcmp (payload, "foo", 4) == 0,
+        "flux-security can unwrap envelope from flux-core internal signer");
+
+    free (s);
+    flux_security_destroy (sec);
+}
+
+void interop_sign_security (void)
+{
+    flux_security_t *sec;
+    uint32_t userid;
+    void *payload;
+    int payloadsz;
+    const char *s;
+    int rc;
+
+    if (!(sec = flux_security_create (0)))
+        BAIL_OUT ("error creating flux-security context");
+    if (flux_security_configure (sec, NULL) < 0)
+        BAIL_OUT ("error configuring flux-security");
+
+    s = flux_sign_wrap_as (sec, 1000, "foo", 4, "none", 0);
+    if (!s) {
+        BAIL_OUT ("flux_sign_wrap_as returned NULL: %s",
+                  flux_security_last_error (sec));
+    }
+
+    userid = 0;
+    payload = NULL;
+    payloadsz = 0;
+    rc = sign_none_unwrap (s, &payload, &payloadsz, &userid);
+    ok (rc == 0 && userid == 1000
+        && payloadsz == 4 && memcmp (payload, "foo", 4) == 0,
+        "flux-core can unwrap envelope from flux-security signer");
+    free (payload);
+
+    flux_security_destroy (sec);
+}
+#endif
+
 int main (int argc, char *argv[])
 {
     plan (NO_PLAN);
@@ -340,6 +425,10 @@ int main (int argc, char *argv[])
     decode_bad_header ();
     decode_bad_other ();
     badarg ();
+#if HAVE_FLUX_SECURITY
+    interop_sign_core ();
+    interop_sign_security ();
+#endif
 
     done_testing ();
     return 0;
diff --git a/src/common/libjob/test/unwrap.c b/src/common/libjob/test/unwrap.c
new file mode 100644
index 000000000000..791c9e9b47dd
--- /dev/null
+++ b/src/common/libjob/test/unwrap.c
@@ -0,0 +1,244 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h>
+#if HAVE_FLUX_SECURITY
+#include <flux/security/sign.h>
+#endif
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libjob/sign_none.h"
+#include "src/common/libjob/unwrap.h"
+
+typedef char * (*unwrap_f) (const char *s,
+                            bool verify,
+                            uint32_t *uidp,
+                            flux_error_t *error);
+
+static void test_api (unwrap_f unwrap)
+{
+    flux_error_t error;
+    uint32_t userid;
+    char *result;
+    char *s;
+
+    if (!(s = sign_none_wrap ("bar", 4, getuid ())))
+        BAIL_OUT ("sign_none_wrap failed");
+
+    userid = 0;
+    memset (error.text, 0, sizeof (error.text));
+    result = (*unwrap) (NULL, false, &userid, &error);
+    ok (result == NULL,
+        "unwrapp_string() fails with NULL argument");
+    ok (userid == 0,
+        "userid argument unmodified");
+    ok (strlen (error.text),
+        "error.text says: %s",
+        error.text);
+
+    userid = 0;
+    result = (*unwrap) (s, false, NULL, &error);
+    ok (result != NULL && userid == 0,
+        "unwrap_string() works with NULL userid");
+    if (result == NULL)
+        diag ("got error: %s", error.text);
+    is (result, "bar",
+        "got expected result");
+    free (result);
+
+    userid = 0;
+    result = (*unwrap) (s, false, NULL, NULL);
+    ok (result != NULL && userid == 0,
+        "unwrap_string() works with NULL userid and error parameters");
+    is (result, "bar",
+        "got expected result");
+    free (result);
+
+    userid = 0;
+    result = (*unwrap) (s, true, NULL, NULL);
+    ok (result != NULL && userid == 0,
+        "unwrap_string() works with verify and NULL userid and error parameters");
+    is (result, "bar",
+        "got expected result");
+    free (result);
+    free (s);
+
+    if (!(s = sign_none_wrap ("bar", 4, getuid () - 1)))
+        BAIL_OUT ("sign_none_wrap failed");
+
+    userid = 0;
+    result = (*unwrap) (s, true, &userid, NULL);
+    ok (result == NULL,
+        "unwrap_string() fails with verify == true and errp == NULL");
+    free (result);
+
+    free (s);
+}
+
+/*  Test a good and bad payload (cribbed from test/sign_none.c)
+ */
+static void decode_bad_other (unwrap_f unwrap)
+{
+    const char *good = "dmVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaGFuaXNtAHNub25lAA==.Zm9vAA==.none";
+    /* invalid base64 payload (% character) */
+    const char *bad  = "dmVyc2lvbgBpMQB1c2VyaWQAaTEwMDAAbWVjaGFuaXNtAHNub25lAA==.%m9vAA==.none";
+
+    flux_error_t error;
+    uint32_t userid;
+    char *result;
+
+    /* Double check good input, the basis for bad input.
+     *  (do not verify since uid will not match)
+     */
+    result = (*unwrap) (good, false, &userid, &error);
+    ok (result != NULL,
+        "unwrap_string() works for good sign-none payload");
+    if (!result)
+        diag ("%s", error.text);
+    is (result, "foo",
+        "result is %s", result);
+    if (result)
+        free (result);
+
+    result = (*unwrap) (bad, false, &userid, &error);
+    ok (result == NULL,
+        "unwrap_string() fails on bad payload");
+    diag ("%s", error.text);
+    if (result)
+        free (result);
+}
+
+static void unwrap_sign_none (unwrap_f unwrap)
+{
+    flux_error_t error;
+    uint32_t userid;
+    char *s;
+    char *result;
+
+    if (!(s = sign_none_wrap ("bar", 4, 1000)))
+        BAIL_OUT ("sign_none_wrap failed");
+
+    userid = 0;
+    result = (*unwrap) (s, false, &userid, &error);
+    ok (result != NULL && userid == 1000,
+        "unwrap_string() after sign_none_wrap() works");
+    is (result, "bar",
+        "got expected result");
+    free (result);
+    free (s);
+
+
+    if (!(s = sign_none_wrap ("bar", 3, 1000)))
+        BAIL_OUT ("sign_none_wrap failed");
+    userid = 0;
+    result = (*unwrap) (s, false, &userid, &error);
+    ok (result != NULL && userid == 1000,
+        "unwrap_string() after sign_none_wrap() excluding NUL works");
+    is (result, "bar",
+        "got expected result");
+    free (result);
+    free (s);
+
+}
+
+#if HAVE_FLUX_SECURITY
+static void sign_security (void)
+{
+    flux_security_t *sec;
+    flux_error_t error;
+    uint32_t userid;
+    char *result;
+    const char *s;
+
+    if (!(sec = flux_security_create (0)))
+        BAIL_OUT ("error creating flux-security context");
+    if (flux_security_configure (sec, NULL) < 0)
+        BAIL_OUT ("error configuring flux-security");
+
+    s = flux_sign_wrap_as (sec, 1000, "foo", 4, "none", 0);
+    if (!s) {
+        BAIL_OUT ("flux_sign_wrap_as returned NULL: %s",
+                  flux_security_last_error (sec));
+    }
+
+    userid = 0;
+    result = flux_unwrap_string (s, false, &userid, &error);
+    ok (result && userid == 1000,
+        "flux_unwrap_string() from flux-security signer");
+    free (result);
+
+    /* valid userid */
+    if (!(s = flux_sign_wrap_as (sec, getuid (), "foo", 4, "none", 0)))
+        BAIL_OUT ("flux_sign_wrap_as returned NULL: %s",
+                  flux_security_last_error (sec));
+    userid = 0;
+    result = flux_unwrap_string (s, true, &userid, &error);
+    ok (result && userid == getuid (),
+        "flux_unwrap_string() with verify = true works");
+    free (result);
+
+    /* Invalid userid */
+    if (!(s = flux_sign_wrap_as (sec, getuid() - 1, "foo", 4, "none", 0)))
+        BAIL_OUT ("flux_sign_wrap_as returned NULL: %s",
+                  flux_security_last_error (sec));
+    userid = 0;
+    memset (error.text, 0, sizeof (error.text));
+    result = flux_unwrap_string (s, true, &userid, &error);
+    ok (result == NULL,
+        "flux_unwrap_string() with verify = true and incorrect userid fails");
+    ok (strlen (error.text),
+        "flux_unwrap_string() expected error: %s",
+        error.text);
+    free (result);
+
+    /* Invalid userid (noverify) */
+    userid = 0;
+    memset (error.text, 0, sizeof (error.text));
+    result = flux_unwrap_string (s, false, &userid, &error);
+    ok (result != NULL,
+        "flux_unwrap_string() with verify = false and incorrect userid succeeds");
+    ok (userid == getuid() - 1,
+        "flux_unwrap_string() returned userid used for signing");
+    ok (strlen (error.text) == 0,
+        "flux_unwrap_string() error.text still empty");
+    free (result);
+
+
+    flux_security_destroy (sec);
+}
+#endif
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_api (flux_unwrap_string);
+    test_api (unwrap_string_sign_none);
+
+    decode_bad_other (flux_unwrap_string);
+    decode_bad_other (unwrap_string_sign_none);
+
+    unwrap_sign_none (flux_unwrap_string);
+    unwrap_sign_none (unwrap_string_sign_none);
+#if HAVE_FLUX_SECURITY
+    sign_security ();
+#endif
+
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/unwrap.c b/src/common/libjob/unwrap.c
new file mode 100644
index 000000000000..6fc4f39d8371
--- /dev/null
+++ b/src/common/libjob/unwrap.c
@@ -0,0 +1,121 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <string.h>
+#include <stdint.h>
+#include <unistd.h>
+
+#if HAVE_FLUX_SECURITY
+#include <flux/security/context.h>
+#include <flux/security/sign.h>
+#endif
+
+#include "src/common/libutil/errprintf.h"
+
+#include "unwrap.h"
+#include "sign_none.h"
+
+char *unwrap_string_sign_none (const char *s,
+                               bool verify,
+                               uint32_t *userid,
+                               flux_error_t *errp)
+{
+    char *result = NULL;
+    int len;
+    uint32_t uid;
+    void *data = NULL;
+    if (sign_none_unwrap (s, (void **) &data, &len, &uid) < 0) {
+        errprintf (errp, "sign-none-unwrap failed: %s", strerror (errno));
+        return NULL;
+    }
+    if (verify && uid != getuid ()) {
+        errprintf (errp,
+                   "sign-none-unwrap: signing userid %lu != current %lu",
+                   (unsigned long) uid,
+                   (unsigned long) getuid ());
+        free (data);
+        return NULL;
+    }
+    /*  Add one extra byte to ensure NUL termination
+     */
+    if (!(result = calloc (1, len+1))) {
+        errprintf (errp, "Out of memory");
+        goto out;
+    }
+    memcpy (result, data, len);
+    if (userid)
+        *userid = uid;
+out:
+    free (data);
+    return result;
+}
+
+char *flux_unwrap_string (const char *s,
+                          bool verify,
+                          uint32_t *userid,
+                          flux_error_t *errp)
+{
+#if HAVE_FLUX_SECURITY
+    flux_security_t *sec;
+    const void *data = NULL;
+    char *result = NULL;
+    int len;
+    const char *mech;
+    int64_t userid64;
+    int flags = verify ? 0 : FLUX_SIGN_NOVERIFY;
+
+    if (!(sec = flux_security_create (0))) {
+        errprintf (errp,
+                   "failed to create security context: %s",
+                   strerror (errno));
+        return NULL;
+    }
+    if (flux_security_configure (sec, NULL) < 0) {
+        errprintf (errp,
+                   "failed to configure security context: %s",
+                   flux_security_last_error (sec));
+        goto done;
+    }
+    if (flux_sign_unwrap_anymech (sec,
+                                  s,
+                                  &data,
+                                  &len,
+                                  &mech,
+                                  &userid64,
+                                  flags) < 0) {
+        errprintf (errp, "%s", flux_security_last_error (sec));
+        goto done;
+    }
+    /*  Add one extra byte to ensure NUL termination in case NUL byte
+     *   not included in len:
+     */
+    if (!(result = calloc (1, len+1))) {
+        errprintf (errp, "Out of memory");
+        goto done;
+    }
+    memcpy (result, data, len);
+    if (userid)
+        *userid = (uint32_t) userid64;
+done:
+    flux_security_destroy (sec);
+    return result;
+#else
+    return unwrap_string_sign_none (s, verify, userid, errp);
+#endif /* !HAVE_FLUX_SECURITY */
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/unwrap.h b/src/common/libjob/unwrap.h
new file mode 100644
index 000000000000..e436f00cda5f
--- /dev/null
+++ b/src/common/libjob/unwrap.h
@@ -0,0 +1,28 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UNWRAP_STRING
+#define _UNWRAP_STRING
+
+#include <flux/core.h>
+
+/*  Like flux_unwrap_string(), but use version without flux-security
+ *  (for testing only)
+ */
+char *unwrap_string_sign_none (const char *s,
+                               bool verify,
+                               uint32_t *userid,
+                               flux_error_t *errp);
+
+#endif /* !_UNWRAP_STRING */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libjob/wait.c b/src/common/libjob/wait.c
new file mode 100644
index 000000000000..82aaa246b513
--- /dev/null
+++ b/src/common/libjob/wait.c
@@ -0,0 +1,76 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "job.h"
+
+flux_future_t *flux_job_wait (flux_t *h, flux_jobid_t id)
+{
+    if (!h) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return flux_rpc_pack (h,
+                          "job-manager.wait",
+                          FLUX_NODEID_ANY,
+                          0,
+                          "{s:I}",
+                          "id",
+                          id);
+}
+
+int flux_job_wait_get_status (flux_future_t *f,
+                              bool *successp,
+                              const char **errstrp)
+{
+    int success;
+    const char *errstr;
+
+    if (!f) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_rpc_get_unpack (f,
+                             "{s:b s:s}",
+                             "success",
+                             &success,
+                             "errstr",
+                             &errstr) < 0)
+        return -1;
+    if (successp)
+        *successp = success ? true : false;
+    if (errstrp)
+        *errstrp = errstr;
+    return 0;
+}
+
+int flux_job_wait_get_id (flux_future_t *f, flux_jobid_t *jobid)
+{
+    flux_jobid_t id;
+
+    if (!f) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_rpc_get_unpack (f, "{s:I}",
+                                "id", &id) < 0)
+        return -1;
+    if (jobid)
+        *jobid = id;
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libkvs/Makefile.am b/src/common/libkvs/Makefile.am
index c5b6347104d6..b8ab8cb56cda 100644
--- a/src/common/libkvs/Makefile.am
+++ b/src/common/libkvs/Makefile.am
@@ -6,11 +6,12 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) \
-	$(LIBUUID_CFLAGS)
+	$(JANSSON_CFLAGS)
 
 noinst_LTLIBRARIES = libkvs.la
 
@@ -18,6 +19,8 @@ libkvs_la_SOURCES = \
 	kvs.c \
 	kvs_lookup.c \
 	kvs_getroot.c \
+	kvs_checkpoint.c \
+	kvs_checkpoint.h \
 	kvs_dir.c \
 	kvs_dir_private.h \
 	kvs_commit.c \
@@ -48,6 +51,7 @@ TESTS = \
 	test_kvs_commit.t \
 	test_kvs_getroot.t \
 	test_treeobj.t \
+	test_kvs_checkpoint.t \
 	test_kvs_copy.t \
 	test_kvs_util.t
 
@@ -63,12 +67,8 @@ test_ldadd = \
 	$(top_builddir)/src/common/libflux/libflux.la \
 	$(top_builddir)/src/common/libflux-internal.la \
 	$(top_builddir)/src/common/libtap/libtap.la \
-	$(ZMQ_LIBS) \
-	$(LIBUUID_LIBS) \
 	$(JANSSON_LIBS) \
-	$(LIBPTHREAD) \
-	$(LIBRT) \
-	$(LIBDL)
+	$(LIBPTHREAD)
 
 test_cppflags = \
 	$(AM_CPPFLAGS) \
@@ -76,43 +76,46 @@ test_cppflags = \
 
 test_kvs_t_SOURCES = test/kvs.c
 test_kvs_t_CPPFLAGS = $(test_cppflags)
-test_kvs_t_LDADD = $(test_ldadd) $(LIBDL)
+test_kvs_t_LDADD = $(test_ldadd)
 
 test_kvs_txn_t_SOURCES = test/kvs_txn.c
 test_kvs_txn_t_CPPFLAGS = $(test_cppflags)
-test_kvs_txn_t_LDADD = $(test_ldadd) $(LIBDL)
+test_kvs_txn_t_LDADD = $(test_ldadd)
 
 test_kvs_txn_compact_t_SOURCES = test/kvs_txn_compact.c
 test_kvs_txn_compact_t_CPPFLAGS = $(test_cppflags)
-test_kvs_txn_compact_t_LDADD = $(test_ldadd) $(LIBDL)
+test_kvs_txn_compact_t_LDADD = $(test_ldadd)
 
 test_kvs_lookup_t_SOURCES = test/kvs_lookup.c
 test_kvs_lookup_t_CPPFLAGS = $(test_cppflags)
 test_kvs_lookup_t_LDADD = \
 	$(top_builddir)/src/common/libkvs/kvs_txn.o \
-	$(test_ldadd) \
-	$(LIBDL)
+	$(test_ldadd)
 
 test_kvs_dir_t_SOURCES = test/kvs_dir.c
 test_kvs_dir_t_CPPFLAGS = $(test_cppflags)
-test_kvs_dir_t_LDADD = $(test_ldadd) $(LIBDL)
+test_kvs_dir_t_LDADD = $(test_ldadd)
 
 test_kvs_commit_t_SOURCES = test/kvs_commit.c
 test_kvs_commit_t_CPPFLAGS = $(test_cppflags)
-test_kvs_commit_t_LDADD = $(test_ldadd) $(LIBDL)
+test_kvs_commit_t_LDADD = $(test_ldadd)
 
 test_kvs_getroot_t_SOURCES = test/kvs_getroot.c
 test_kvs_getroot_t_CPPFLAGS = $(test_cppflags)
-test_kvs_getroot_t_LDADD = $(test_ldadd) $(LIBDL)
+test_kvs_getroot_t_LDADD = $(test_ldadd)
 
 test_treeobj_t_SOURCES = test/treeobj.c
 test_treeobj_t_CPPFLAGS = $(test_cppflags)
-test_treeobj_t_LDADD = $(test_ldadd) $(LIBDL)
+test_treeobj_t_LDADD = $(test_ldadd)
+
+test_kvs_checkpoint_t_SOURCES = test/kvs_checkpoint.c
+test_kvs_checkpoint_t_CPPFLAGS = $(test_cppflags)
+test_kvs_checkpoint_t_LDADD = $(test_ldadd)
 
 test_kvs_copy_t_SOURCES = test/kvs_copy.c
 test_kvs_copy_t_CPPFLAGS = $(test_cppflags)
-test_kvs_copy_t_LDADD = $(test_ldadd) $(LIBDL)
+test_kvs_copy_t_LDADD = $(test_ldadd)
 
 test_kvs_util_t_SOURCES = test/kvs_util.c
 test_kvs_util_t_CPPFLAGS = $(test_cppflags)
-test_kvs_util_t_LDADD = $(test_ldadd) $(LIBDL)
+test_kvs_util_t_LDADD = $(test_ldadd)
diff --git a/src/common/libkvs/kvs.c b/src/common/libkvs/kvs.c
index d6d535ce7cfa..53564240b9f2 100644
--- a/src/common/libkvs/kvs.c
+++ b/src/common/libkvs/kvs.c
@@ -16,20 +16,79 @@
 #include <jansson.h>
 #include <flux/core.h>
 
+#include "src/common/libutil/blobref.h"
+
+#include "treeobj.h"
 #include "kvs_util_private.h"
 
-flux_future_t *flux_kvs_namespace_create (flux_t *h, const char *ns,
-                                          uint32_t owner, int flags)
+flux_future_t *flux_kvs_namespace_create (flux_t *h,
+                                          const char *ns,
+                                          uint32_t owner,
+                                          int flags)
 {
+    flux_future_t *rv = NULL;
+    const char *hash_name;
+    json_t *rootdir = NULL;
+    char rootref[BLOBREF_MAX_STRING_SIZE];
+    void *data = NULL;
+    int len;
+
     if (!ns || flags) {
         errno = EINVAL;
         return NULL;
     }
 
+    if (!(hash_name = flux_attr_get (h, "content.hash")))
+        goto cleanup;
+
+    if (!(rootdir = treeobj_create_dir ()))
+        goto cleanup;
+
+    if (!(data = treeobj_encode (rootdir)))
+        goto cleanup;
+    len = strlen (data);
+
+    /* N.B. blobref of empty treeobj dir guaranteed to be in content store
+     * b/c that is how the primary KVS is initialized.
+     */
+    if (blobref_hash (hash_name, data, len, rootref, sizeof (rootref)) < 0)
+        goto cleanup;
+
+    /* N.B. owner cast to int */
+    rv = flux_rpc_pack (h,
+                        "kvs.namespace-create",
+                        0,
+                        0,
+                        "{ s:s s:s s:i s:i }",
+                        "namespace", ns,
+                        "rootref", rootref,
+                        "owner", owner,
+                        "flags", flags);
+cleanup:
+    json_decref (rootdir);
+    free (data);
+    return rv;
+}
+
+flux_future_t *flux_kvs_namespace_create_with (flux_t *h,
+                                               const char *ns,
+                                               const char *rootref,
+                                               uint32_t owner,
+                                               int flags)
+{
+    if (!ns || !rootref || flags) {
+        errno = EINVAL;
+        return NULL;
+    }
+
     /* N.B. owner cast to int */
-    return flux_rpc_pack (h, "kvs.namespace-create", 0, 0,
-                          "{ s:s s:i s:i }",
+    return flux_rpc_pack (h,
+                          "kvs.namespace-create",
+                          0,
+                          0,
+                          "{ s:s s:s s:i s:i }",
                           "namespace", ns,
+                          "rootref", rootref,
                           "owner", owner,
                           "flags", flags);
 }
@@ -41,7 +100,10 @@ flux_future_t *flux_kvs_namespace_remove (flux_t *h, const char *ns)
         return NULL;
     }
 
-    return flux_rpc_pack (h, "kvs.namespace-remove", 0, 0,
+    return flux_rpc_pack (h,
+                          "kvs.namespace-remove",
+                          0,
+                          0,
                           "{ s:s }",
                           "namespace", ns);
 }
@@ -56,7 +118,11 @@ int flux_kvs_get_version (flux_t *h, const char *ns, int *versionp)
         if (!(ns = kvs_get_namespace ()))
             return -1;
     }
-    if (!(f = flux_rpc_pack (h, "kvs.getroot", FLUX_NODEID_ANY, 0, "{ s:s }",
+    if (!(f = flux_rpc_pack (h,
+                             "kvs.getroot",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{ s:s }",
                              "namespace", ns)))
         goto done;
     if (flux_rpc_get_unpack (f, "{ s:i }", "rootseq", &version) < 0)
@@ -78,7 +144,11 @@ int flux_kvs_wait_version (flux_t *h, const char *ns, int version)
         if (!(ns = kvs_get_namespace ()))
             return -1;
     }
-    if (!(f = flux_rpc_pack (h, "kvs.sync", FLUX_NODEID_ANY, 0, "{ s:i s:s }",
+    if (!(f = flux_rpc_pack (h,
+                             "kvs.wait-version",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{ s:i s:s }",
                              "rootseq", version,
                              "namespace", ns)))
         goto done;
diff --git a/src/common/libkvs/kvs.h b/src/common/libkvs/kvs.h
index 423d1a7dc53b..c8333aa7ef24 100644
--- a/src/common/libkvs/kvs.h
+++ b/src/common/libkvs/kvs.h
@@ -47,8 +47,15 @@ enum kvs_op {
  *   namespace will official be removed.  The removal is "eventually
  *   consistent".
  */
-flux_future_t *flux_kvs_namespace_create (flux_t *h, const char *ns,
-                                          uint32_t owner, int flags);
+flux_future_t *flux_kvs_namespace_create (flux_t *h,
+                                          const char *ns,
+                                          uint32_t owner,
+                                          int flags);
+flux_future_t *flux_kvs_namespace_create_with (flux_t *h,
+                                               const char *ns,
+                                               const char *rootref,
+                                               uint32_t owner,
+                                               int flags);
 flux_future_t *flux_kvs_namespace_remove (flux_t *h, const char *ns);
 
 /* Synchronization:
@@ -56,11 +63,10 @@ flux_future_t *flux_kvs_namespace_remove (flux_t *h, const char *ns);
  * Process B waits for the store version to be >= V, then reads data.
  */
 int flux_kvs_get_version (flux_t *h, const char *ns, int *versionp);
-int flux_kvs_wait_version (flux_t *h,const char *ns, int version);
+int flux_kvs_wait_version (flux_t *h, const char *ns, int version);
 
-/* Garbage collect the cache.  On the root node, drop all data that
- * doesn't have a reference in the namespace.  On other nodes, the entire
- * cache is dropped and will be reloaded on demand.
+/* Garbage collect the cache.  Drop all data that doesn't have a
+ * reference in the namespace.
  * Returns -1 on error (errno set), 0 on success.
  */
 int flux_kvs_dropcache (flux_t *h);
diff --git a/src/common/libkvs/kvs_checkpoint.c b/src/common/libkvs/kvs_checkpoint.c
new file mode 100644
index 000000000000..cc791b3ea4b1
--- /dev/null
+++ b/src/common/libkvs/kvs_checkpoint.c
@@ -0,0 +1,159 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <assert.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <jansson.h>
+#include <flux/core.h>
+#include <time.h>
+
+#include "kvs_checkpoint.h"
+
+flux_future_t *kvs_checkpoint_commit (flux_t *h,
+                                      const char *key,
+                                      const char *rootref,
+                                      int sequence,
+                                      double timestamp,
+                                      int flags)
+{
+    flux_future_t *f = NULL;
+    const char *topic = "content.checkpoint-put";
+    int valid_flags = KVS_CHECKPOINT_FLAG_CACHE_BYPASS;
+
+    if (!h || !rootref || (flags & ~valid_flags)) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!key)
+        key = KVS_DEFAULT_CHECKPOINT;
+    if (timestamp == 0)
+        timestamp = flux_reactor_now (flux_get_reactor (h));
+    if (flags & KVS_CHECKPOINT_FLAG_CACHE_BYPASS)
+        topic = "content-backing.checkpoint-put";
+
+    if (!(f = flux_rpc_pack (h,
+                             topic,
+                             0,
+                             0,
+                             "{s:s s:{s:i s:s s:i s:f}}",
+                             "key",
+                             key,
+                             "value",
+                             "version", 1,
+                             "rootref", rootref,
+                             "sequence", sequence,
+                             "timestamp", timestamp)))
+        return NULL;
+
+    return f;
+}
+
+flux_future_t *kvs_checkpoint_lookup (flux_t *h, const char *key, int flags)
+{
+    const char *topic = "content.checkpoint-get";
+    int valid_flags = KVS_CHECKPOINT_FLAG_CACHE_BYPASS;
+
+    if (!h || (flags & ~valid_flags)) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!key)
+        key = KVS_DEFAULT_CHECKPOINT;
+    if (flags & KVS_CHECKPOINT_FLAG_CACHE_BYPASS)
+        topic = "content-backing.checkpoint-get";
+
+    return flux_rpc_pack (h,
+                          topic,
+                          0,
+                          0,
+                          "{s:s}",
+                          "key",
+                          key);
+}
+
+int kvs_checkpoint_lookup_get_rootref (flux_future_t *f, const char **rootref)
+{
+    const char *tmp_rootref;
+    int version;
+
+    if (!f || !rootref) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (flux_rpc_get_unpack (f,
+                             "{s:{s:i s:s}}",
+                             "value",
+                               "version", &version,
+                               "rootref", &tmp_rootref) < 0)
+        return -1;
+
+    if (version != 0 && version != 1) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    (*rootref) = tmp_rootref;
+    return 0;
+}
+
+int kvs_checkpoint_lookup_get_timestamp (flux_future_t *f, double *timestamp)
+{
+    int version;
+    double ts = 0.;
+
+    if (!f || !timestamp) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_rpc_get_unpack (f,
+                             "{s:{s:i s?f}}",
+                             "value",
+                               "version", &version,
+                               "timestamp", &ts) < 0)
+        return -1;
+    if (version != 0 && version != 1) {
+        errno = EINVAL;
+        return -1;
+    }
+    *timestamp = ts;
+    return 0;
+}
+
+int kvs_checkpoint_lookup_get_sequence (flux_future_t *f, int *sequence)
+{
+    int version;
+    int seq = 0;
+
+    if (!f || !sequence) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_rpc_get_unpack (f,
+                             "{s:{s:i s?i}}",
+                             "value",
+                               "version", &version,
+                               "sequence", &seq) < 0)
+        return -1;
+    if (version != 0 && version != 1) {
+        errno = EINVAL;
+        return -1;
+    }
+    *sequence = seq;
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libkvs/kvs_checkpoint.h b/src/common/libkvs/kvs_checkpoint.h
new file mode 100644
index 000000000000..6c43e137de45
--- /dev/null
+++ b/src/common/libkvs/kvs_checkpoint.h
@@ -0,0 +1,54 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _KVS_CHECKPOINT_H
+#define _KVS_CHECKPOINT_H
+
+#include <flux/core.h>
+
+/* flags */
+enum {
+    KVS_CHECKPOINT_FLAG_CACHE_BYPASS = 1,/* request direct to backing store */
+};
+
+#define KVS_DEFAULT_CHECKPOINT "kvs-primary"
+
+/* Calls to kvs_checkpoint_commit() can be racy when the KVS module is
+ * loaded, as callers can use the FLUX_KVS_SYNC flag with
+ * flux_kvs_commit().
+ *
+ * In most cases, use the FLUX_KVS_SYNC flag with flux_kvs_commit() to
+ * ensure committed data survives a Flux restart.
+ */
+
+flux_future_t *kvs_checkpoint_commit (flux_t *h,
+                                      const char *key,
+                                      const char *rootref,
+                                      int sequence,
+                                      double timestamp,
+                                      int flags);
+
+flux_future_t *kvs_checkpoint_lookup (flux_t *h, const char *key, int flags);
+
+int kvs_checkpoint_lookup_get_rootref (flux_future_t *f, const char **rootref);
+
+/* sets timestamp to 0 if unavailable
+ */
+int kvs_checkpoint_lookup_get_timestamp (flux_future_t *f, double *timestamp);
+
+/* sets sequence to 0 if unavailable
+ */
+int kvs_checkpoint_lookup_get_sequence (flux_future_t *f, int *sequence);
+
+#endif /* !_KVS_CHECKPOINT_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libkvs/kvs_commit.c b/src/common/libkvs/kvs_commit.c
index c55d0b82a46b..fb2e1a11d6e5 100644
--- a/src/common/libkvs/kvs_commit.c
+++ b/src/common/libkvs/kvs_commit.c
@@ -12,7 +12,6 @@
 #include "config.h"
 #endif
 #include <jansson.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "treeobj.h"
@@ -44,8 +43,11 @@ static struct commit_ctx *alloc_ctx (void)
     return ctx;
 }
 
-flux_future_t *flux_kvs_fence (flux_t *h, const char *ns, int flags,
-                               const char *name, int nprocs,
+flux_future_t *flux_kvs_fence (flux_t *h,
+                               const char *ns,
+                               int flags,
+                               const char *name,
+                               int nprocs,
                                flux_kvs_txn_t *txn)
 {
     flux_future_t *f;
@@ -76,7 +78,10 @@ flux_future_t *flux_kvs_fence (flux_t *h, const char *ns, int flags,
     if (!(ctx = alloc_ctx ()))
         return NULL;
 
-    if (!(f = flux_rpc_pack (h, "kvs.fence", FLUX_NODEID_ANY, 0,
+    if (!(f = flux_rpc_pack (h,
+                             "kvs.fence",
+                             FLUX_NODEID_ANY,
+                             0,
                              "{s:s s:i s:s s:i s:O}",
                              "name", name,
                              "nprocs", nprocs,
@@ -97,7 +102,9 @@ flux_future_t *flux_kvs_fence (flux_t *h, const char *ns, int flags,
     return NULL;
 }
 
-flux_future_t *flux_kvs_commit (flux_t *h, const char *ns, int flags,
+flux_future_t *flux_kvs_commit (flux_t *h,
+                                const char *ns,
+                                int flags,
                                 flux_kvs_txn_t *txn)
 {
     flux_future_t *f;
@@ -128,7 +135,10 @@ flux_future_t *flux_kvs_commit (flux_t *h, const char *ns, int flags,
     if (!(ctx = alloc_ctx ()))
         return NULL;
 
-    if (!(f = flux_rpc_pack (h, "kvs.commit", FLUX_NODEID_ANY, 0,
+    if (!(f = flux_rpc_pack (h,
+                             "kvs.commit",
+                             FLUX_NODEID_ANY,
+                             0,
                              "{s:s s:i s:O}",
                              "namespace", ns,
                              "flags", flags,
@@ -147,13 +157,15 @@ flux_future_t *flux_kvs_commit (flux_t *h, const char *ns, int flags,
     return NULL;
 }
 
-static int decode_response (flux_future_t *f, const char **rootrefp,
+static int decode_response (flux_future_t *f,
+                            const char **rootrefp,
                             int *rootseqp)
 {
     const char *rootref;
     int rootseq;
 
-    if (flux_rpc_get_unpack (f, "{s:s s:i}",
+    if (flux_rpc_get_unpack (f,
+                             "{s:s s:i}",
                              "rootref", &rootref,
                              "rootseq", &rootseq) < 0)
         return -1;
@@ -164,6 +176,15 @@ static int decode_response (flux_future_t *f, const char **rootrefp,
     return 0;
 }
 
+int flux_kvs_commit_get_rootref (flux_future_t *f, const char **rootref)
+{
+    if (!f || !rootref) {
+        errno = EINVAL;
+        return -1;
+    }
+    return decode_response (f, rootref, NULL);
+}
+
 int flux_kvs_commit_get_sequence (flux_future_t *f, int *rootseq)
 {
     if (!f || !rootseq) {
diff --git a/src/common/libkvs/kvs_commit.h b/src/common/libkvs/kvs_commit.h
index 00c406e0fe66..137c0d82a37a 100644
--- a/src/common/libkvs/kvs_commit.h
+++ b/src/common/libkvs/kvs_commit.h
@@ -24,21 +24,40 @@ extern "C" {
  * operations will be illegal to compact and result in an error.  Most
  * notably, if a key has data appended to it, then is overwritten in
  * the same transaction, a compaction of appends is not possible.
+ *
+ * FLUX_KVS_SYNC will ensure all data flushed to the backing store and
+ * the root reference is checkpointed.  It effectively performs a:
+ *
+ * content.flush on rank 0
+ * checkpoint on the new root reference from the commit
+ *
+ * FLUX_KVS_SYNC only works against the primary KVS namespace.  If any
+ * part of the content.flush or checkpoint fails an error will be
+ * returned and the entire commit will fail.  For example, if a
+ * content backing store is not loaded, ENOSYS will returned from this
+ * commit.
  */
 enum kvs_commit_flags {
     FLUX_KVS_NO_MERGE = 1, /* disallow commits to be mergeable with others */
     FLUX_KVS_TXN_COMPACT = 2, /* try to combine ops on same key within txn */
+    FLUX_KVS_SYNC = 4, /* flush and checkpoint after commit is done */
 };
 
-flux_future_t *flux_kvs_commit (flux_t *h, const char *ns, int flags,
+flux_future_t *flux_kvs_commit (flux_t *h,
+                                const char *ns,
+                                int flags,
                                 flux_kvs_txn_t *txn);
 
-flux_future_t *flux_kvs_fence (flux_t *h, const char *ns, int flags,
-                               const char *name, int nprocs,
+flux_future_t *flux_kvs_fence (flux_t *h,
+                               const char *ns,
+                               int flags,
+                               const char *name,
+                               int nprocs,
                                flux_kvs_txn_t *txn);
 
 /* accessors can be used for commit or fence futures */
 int flux_kvs_commit_get_treeobj (flux_future_t *f, const char **treeobj);
+int flux_kvs_commit_get_rootref (flux_future_t *f, const char **rootref);
 int flux_kvs_commit_get_sequence (flux_future_t *f, int *rootseq);
 
 #ifdef __cplusplus
diff --git a/src/common/libkvs/kvs_copy.c b/src/common/libkvs/kvs_copy.c
index 6669e7145454..c573768f06c2 100644
--- a/src/common/libkvs/kvs_copy.c
+++ b/src/common/libkvs/kvs_copy.c
@@ -174,7 +174,10 @@ flux_future_t *flux_kvs_copy (flux_t *h,
                                      dstkey,
                                      commit_flags)))
         goto error;
-    if (flux_aux_set (h, NULL, ctx, (flux_free_f)copy_context_destroy) < 0) {
+    if (flux_future_aux_set (f1,
+                             NULL,
+                             ctx,
+                             (flux_free_f)copy_context_destroy) < 0) {
         copy_context_destroy (ctx);
         goto error;
     }
@@ -209,7 +212,10 @@ flux_future_t *flux_kvs_move (flux_t *h,
                                      dstkey,
                                      commit_flags)))
         goto error;
-    if (flux_aux_set (h, NULL, ctx, (flux_free_f)copy_context_destroy) < 0) {
+    if (flux_future_aux_set (f1,
+                             NULL,
+                             ctx,
+                             (flux_free_f)copy_context_destroy) < 0) {
         copy_context_destroy (ctx);
         goto error;
     }
diff --git a/src/common/libkvs/kvs_dir.c b/src/common/libkvs/kvs_dir.c
index 5cbffbda663a..a2750fc31d99 100644
--- a/src/common/libkvs/kvs_dir.c
+++ b/src/common/libkvs/kvs_dir.c
@@ -15,9 +15,11 @@
 #include <errno.h>
 #include <stdlib.h>
 #include <jansson.h>
-#include <czmq.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
 #include "kvs_dir_private.h"
 #include "treeobj.h"
 
@@ -53,16 +55,20 @@ void flux_kvsdir_destroy (flux_kvsdir_t *dir)
 
 flux_kvsdir_t *flux_kvsdir_copy (const flux_kvsdir_t *dir)
 {
-    return kvsdir_create_fromobj (dir->handle, dir->rootref,
-                                  dir->key, dir->dirobj);
+    return kvsdir_create_fromobj (dir->handle,
+                                  dir->rootref,
+                                  dir->key,
+                                  dir->dirobj);
 }
 
 /* If rootref is non-NULL, the kvsdir records the root reference
  * so that subsequent flux_kvsdir_get_* accesses can be relative to that
  * snapshot.  Otherwise, they are relative to the current root.
  */
-flux_kvsdir_t *flux_kvsdir_create (flux_t *h, const char *rootref,
-                                   const char *key, const char *json_str)
+flux_kvsdir_t *flux_kvsdir_create (flux_t *h,
+                                   const char *rootref,
+                                   const char *key,
+                                   const char *json_str)
 {
     flux_kvsdir_t *dir = NULL;
     json_t *dirobj = NULL;
@@ -202,7 +208,7 @@ char *flux_kvsdir_key_at (const flux_kvsdir_t *dir, const char *name)
     char *s;
     int len;
 
-    if (!strcmp (dir->key, ".")) {
+    if (streq (dir->key, ".")) {
         if (!(s = strdup (name)))
             goto nomem;
     }
@@ -225,13 +231,17 @@ char *flux_kvsdir_key_at (const flux_kvsdir_t *dir, const char *name)
 
 /* kvs_txn_private.h */
 
-flux_kvsdir_t *kvsdir_create_fromobj (flux_t *handle, const char *rootref,
-                                      const char *key, json_t *treeobj)
+flux_kvsdir_t *kvsdir_create_fromobj (flux_t *handle,
+                                      const char *rootref,
+                                      const char *key,
+                                      json_t *treeobj)
 {
     flux_kvsdir_t *dir = NULL;
 
-    if (!key || !treeobj || treeobj_validate (treeobj) < 0
-                         || !treeobj_is_dir (treeobj)) {
+    if (!key
+        || !treeobj
+        || treeobj_validate (treeobj) < 0
+        || !treeobj_is_dir (treeobj)) {
         errno = EINVAL;
         goto error;
     }
diff --git a/src/common/libkvs/kvs_dir.h b/src/common/libkvs/kvs_dir.h
index 1629aa388ce2..ed82e175685b 100644
--- a/src/common/libkvs/kvs_dir.h
+++ b/src/common/libkvs/kvs_dir.h
@@ -35,8 +35,10 @@ typedef struct flux_kvsitr flux_kvsitr_t;
  * are not freed until the reference count reaches zero.  The object is
  * initially created with a reference count of one.
  */
-flux_kvsdir_t *flux_kvsdir_create (flux_t *handle, const char *rootref,
-                                   const char *key, const char *json_str);
+flux_kvsdir_t *flux_kvsdir_create (flux_t *handle,
+                                   const char *rootref,
+                                   const char *key,
+                                   const char *json_str);
 void flux_kvsdir_destroy (flux_kvsdir_t *dir);
 
 flux_kvsdir_t *flux_kvsdir_copy (const flux_kvsdir_t *dir);
diff --git a/src/common/libkvs/kvs_dir_private.h b/src/common/libkvs/kvs_dir_private.h
index 403860178ed1..91fd76395e49 100644
--- a/src/common/libkvs/kvs_dir_private.h
+++ b/src/common/libkvs/kvs_dir_private.h
@@ -11,8 +11,10 @@
 #ifndef _KVS_DIR_PRIVATE_H
 #define _KVS_DIR_PRIVATE_H
 
-flux_kvsdir_t *kvsdir_create_fromobj (flux_t *handle, const char *rootref,
-                                      const char *key, json_t *treeobj);
+flux_kvsdir_t *kvsdir_create_fromobj (flux_t *handle,
+                                      const char *rootref,
+                                      const char *key,
+                                      json_t *treeobj);
 
 json_t *kvsdir_get_obj (flux_kvsdir_t *dir);
 
diff --git a/src/common/libkvs/kvs_getroot.c b/src/common/libkvs/kvs_getroot.c
index c8611951993d..7798712cc9f8 100644
--- a/src/common/libkvs/kvs_getroot.c
+++ b/src/common/libkvs/kvs_getroot.c
@@ -15,7 +15,6 @@
 #include <errno.h>
 #include <stdlib.h>
 #include <jansson.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "kvs_getroot.h"
@@ -59,7 +58,10 @@ flux_future_t *flux_kvs_getroot (flux_t *h, const char *ns, int flags)
         return NULL;
     if (!ns && !(ns = kvs_get_namespace ()))
         goto error;
-    if (!(f = flux_rpc_pack (h, "kvs.getroot", FLUX_NODEID_ANY, 0,
+    if (!(f = flux_rpc_pack (h,
+                             "kvs.getroot",
+                             FLUX_NODEID_ANY,
+                             0,
                              "{s:s s:i}",
                              "namespace", ns,
                              "flags", flags)))
@@ -75,15 +77,18 @@ flux_future_t *flux_kvs_getroot (flux_t *h, const char *ns, int flags)
     return NULL;
 }
 
-static int decode_response (flux_future_t *f, const char **rootrefp,
-                            int *rootseqp, uint32_t *ownerp)
+static int decode_response (flux_future_t *f,
+                            const char **rootrefp,
+                            int *rootseqp,
+                            uint32_t *ownerp)
 {
     const char *rootref;
     int rootseq;
     int owner;
     int flags;
 
-    if (flux_rpc_get_unpack (f, "{s:s s:i s:i s:i}",
+    if (flux_rpc_get_unpack (f,
+                             "{s:s s:i s:i s:i}",
                              "rootref", &rootref,
                              "rootseq", &rootseq,
                              "owner", &owner,
diff --git a/src/common/libkvs/kvs_getroot.h b/src/common/libkvs/kvs_getroot.h
index 5cdfe995e88c..94c370ba40db 100644
--- a/src/common/libkvs/kvs_getroot.h
+++ b/src/common/libkvs/kvs_getroot.h
@@ -21,7 +21,7 @@ flux_future_t *flux_kvs_getroot (flux_t *h, const char *ns, int flags);
 /* Decode KVS root hash response.
  *
  * treeobj - get the hash as an RFC 11 "dirref" object.
- * blobref - get the raw hash as a n RFC 10 "blobref".
+ * blobref - get the raw hash as an RFC 10 "blobref".
  * sequence - get the commit sequence number
  * owner - get the userid of the namespace owner
  */
diff --git a/src/common/libkvs/kvs_lookup.c b/src/common/libkvs/kvs_lookup.c
index 986409ef3d77..716fb6f7dc4c 100644
--- a/src/common/libkvs/kvs_lookup.c
+++ b/src/common/libkvs/kvs_lookup.c
@@ -15,7 +15,6 @@
 #include <errno.h>
 #include <stdlib.h>
 #include <jansson.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "kvs_dir_private.h"
@@ -32,7 +31,7 @@ struct lookup_ctx {
     json_t *treeobj;
     char *treeobj_str; // json_dumps of tree object returned from lookup
     void *val_data;    // result of base64 decode of val object data
-    int val_len;
+    size_t val_len;
     bool val_valid;
     json_t *val_obj;
     flux_kvsdir_t *dir;
@@ -116,7 +115,9 @@ flux_future_t *flux_kvs_lookup (flux_t *h,
     const char *topic = "kvs.lookup";
     int rpc_flags = 0;
 
-    if (!h || !key || strlen (key) == 0
+    if (!h
+        || !key
+        || strlen (key) == 0
         || validate_lookup_flags (flags, true) < 0) {
         errno = EINVAL;
         return NULL;
@@ -132,7 +133,10 @@ flux_future_t *flux_kvs_lookup (flux_t *h,
         topic = "kvs-watch.lookup"; // redirect to kvs-watch module
     if ((flags & FLUX_KVS_WATCH))
         rpc_flags |= FLUX_RPC_STREAMING;
-    if (!(f = flux_rpc_pack (h, topic, FLUX_NODEID_ANY, rpc_flags,
+    if (!(f = flux_rpc_pack (h,
+                             topic,
+                             FLUX_NODEID_ANY,
+                             rpc_flags,
                              "{s:s s:s s:i}",
                              "key", key,
                              "namespace", ns,
@@ -148,7 +152,9 @@ flux_future_t *flux_kvs_lookup (flux_t *h,
     return f;
 }
 
-flux_future_t *flux_kvs_lookupat (flux_t *h, int flags, const char *key,
+flux_future_t *flux_kvs_lookupat (flux_t *h,
+                                  int flags,
+                                  const char *key,
                                   const char *treeobj)
 {
     flux_future_t *f;
@@ -157,7 +163,10 @@ flux_future_t *flux_kvs_lookupat (flux_t *h, int flags, const char *key,
 
     /* N.B. FLUX_KVS_WATCH is not valid for lookupat (r/o snapshot).
      */
-    if (!h || !key || strlen (key) == 0 || !treeobj
+    if (!h
+        || !key
+        || strlen (key) == 0
+        || !treeobj
         || validate_lookup_flags (flags, false) < 0) {
         errno = EINVAL;
         return NULL;
@@ -170,7 +179,10 @@ flux_future_t *flux_kvs_lookupat (flux_t *h, int flags, const char *key,
         errno = EINVAL;
         return NULL;
     }
-    if (!(f = flux_rpc_pack (h, "kvs.lookup", FLUX_NODEID_ANY, 0,
+    if (!(f = flux_rpc_pack (h,
+                             "kvs.lookup",
+                             FLUX_NODEID_ANY,
+                             0,
                              "{s:s s:i s:O}",
                              "key", key,
                              "flags", flags,
@@ -258,8 +270,9 @@ int flux_kvs_lookup_get (flux_future_t *f, const char **value)
     if (parse_response (f, ctx) < 0)
         return -1;
     if (!ctx->val_valid) {
-        if (treeobj_decode_val (ctx->treeobj, &ctx->val_data,
-                                              &ctx->val_len) < 0)
+        if (treeobj_decode_val (ctx->treeobj,
+                                &ctx->val_data,
+                                &ctx->val_len) < 0)
             return -1;
         ctx->val_valid = true;
         // N.B. val_data includes xtra 0 byte term not reflected in val_len
@@ -297,14 +310,17 @@ int flux_kvs_lookup_get_unpack (flux_future_t *f, const char *fmt, ...)
     if (parse_response (f, ctx) < 0)
         return -1;
     if (!ctx->val_valid) {
-        if (treeobj_decode_val (ctx->treeobj, &ctx->val_data,
-                                              &ctx->val_len) < 0)
+        if (treeobj_decode_val (ctx->treeobj,
+                                &ctx->val_data,
+                                &ctx->val_len) < 0)
             return -1;
         ctx->val_valid = true;
     }
     if (!ctx->val_obj) {
-        if (!(ctx->val_obj = json_loadb (ctx->val_data, ctx->val_len,
-                                         JSON_DECODE_ANY, NULL))) {
+        if (!(ctx->val_obj = json_loadb (ctx->val_data,
+                                         ctx->val_len,
+                                         JSON_DECODE_ANY,
+                                         NULL))) {
             errno = EINVAL;
             return -1;
         }
@@ -317,7 +333,7 @@ int flux_kvs_lookup_get_unpack (flux_future_t *f, const char *fmt, ...)
     return rc;
 }
 
-int flux_kvs_lookup_get_raw (flux_future_t *f, const void **data, int *len)
+int flux_kvs_lookup_get_raw (flux_future_t *f, const void **data, size_t *len)
 {
     struct lookup_ctx *ctx;
 
@@ -326,8 +342,9 @@ int flux_kvs_lookup_get_raw (flux_future_t *f, const void **data, int *len)
     if (parse_response (f, ctx) < 0)
         return -1;
     if (!ctx->val_valid) {
-        if (treeobj_decode_val (ctx->treeobj, &ctx->val_data,
-                                              &ctx->val_len) < 0)
+        if (treeobj_decode_val (ctx->treeobj,
+                                &ctx->val_data,
+                                &ctx->val_len) < 0)
             return -1;
         ctx->val_valid = true;
     }
@@ -347,8 +364,10 @@ int flux_kvs_lookup_get_dir (flux_future_t *f, const flux_kvsdir_t **dirp)
     if (parse_response (f, ctx) < 0)
         return -1;
     if (!ctx->dir) {
-        if (!(ctx->dir = kvsdir_create_fromobj (ctx->h, ctx->atref,
-                                                ctx->key, ctx->treeobj)))
+        if (!(ctx->dir = kvsdir_create_fromobj (ctx->h,
+                                                ctx->atref,
+                                                ctx->key,
+                                                ctx->treeobj)))
             return -1;
     }
     if (dirp)
diff --git a/src/common/libkvs/kvs_lookup.h b/src/common/libkvs/kvs_lookup.h
index 2614789caef8..d3a7717ec89a 100644
--- a/src/common/libkvs/kvs_lookup.h
+++ b/src/common/libkvs/kvs_lookup.h
@@ -15,14 +15,18 @@
 extern "C" {
 #endif
 
-flux_future_t *flux_kvs_lookup (flux_t *h, const char *ns, int flags,
+flux_future_t *flux_kvs_lookup (flux_t *h,
+                                const char *ns,
+                                int flags,
                                 const char *key);
-flux_future_t *flux_kvs_lookupat (flux_t *h, int flags, const char *key,
+flux_future_t *flux_kvs_lookupat (flux_t *h,
+                                  int flags,
+                                  const char *key,
                                   const char *treeobj);
 
 int flux_kvs_lookup_get (flux_future_t *f, const char **value);
 int flux_kvs_lookup_get_unpack (flux_future_t *f, const char *fmt, ...);
-int flux_kvs_lookup_get_raw (flux_future_t *f, const void **data, int *len);
+int flux_kvs_lookup_get_raw (flux_future_t *f, const void **data, size_t *len);
 int flux_kvs_lookup_get_treeobj (flux_future_t *f, const char **treeobj);
 int flux_kvs_lookup_get_dir (flux_future_t *f, const flux_kvsdir_t **dir);
 int flux_kvs_lookup_get_symlink (flux_future_t *f,
diff --git a/src/common/libkvs/kvs_txn.c b/src/common/libkvs/kvs_txn.c
index ff595cda87c1..ca4e08dfced3 100644
--- a/src/common/libkvs/kvs_txn.c
+++ b/src/common/libkvs/kvs_txn.c
@@ -13,7 +13,6 @@
 #endif
 #include <jansson.h>
 #include <flux/core.h>
-#include <czmq.h>
 #include <string.h>
 
 #include "kvs_txn_private.h"
@@ -76,8 +75,10 @@ static int validate_flags (int flags, int allowed)
 /* Add an operation on dirent to the transaction.
  * Takes a reference on dirent so caller retains ownership.
  */
-static int append_op_to_txn (flux_kvs_txn_t *txn, int flags,
-                             const char *key, json_t *dirent)
+static int append_op_to_txn (flux_kvs_txn_t *txn,
+                             int flags,
+                             const char *key,
+                             json_t *dirent)
 {
     json_t *op = NULL;
     int saved_errno;
@@ -96,8 +97,11 @@ static int append_op_to_txn (flux_kvs_txn_t *txn, int flags,
     return -1;
 }
 
-int flux_kvs_txn_put_raw (flux_kvs_txn_t *txn, int flags,
-                          const char *key, const void *data, int len)
+int flux_kvs_txn_put_raw (flux_kvs_txn_t *txn,
+                          int flags,
+                          const char *key,
+                          const void *data,
+                          size_t len)
 {
     json_t *dirent = NULL;
     int saved_errno;
@@ -122,8 +126,10 @@ int flux_kvs_txn_put_raw (flux_kvs_txn_t *txn, int flags,
     return -1;
 }
 
-int flux_kvs_txn_put_treeobj (flux_kvs_txn_t *txn, int flags,
-                              const char *key, const char *treeobj)
+int flux_kvs_txn_put_treeobj (flux_kvs_txn_t *txn,
+                              int flags,
+                              const char *key,
+                              const char *treeobj)
 {
     json_t *dirent = NULL;
     int saved_errno;
@@ -148,8 +154,10 @@ int flux_kvs_txn_put_treeobj (flux_kvs_txn_t *txn, int flags,
     return -1;
 }
 
-int flux_kvs_txn_put (flux_kvs_txn_t *txn, int flags,
-                      const char *key, const char *value)
+int flux_kvs_txn_put (flux_kvs_txn_t *txn,
+                      int flags,
+                      const char *key,
+                      const char *value)
 {
     json_t *dirent = NULL;
     int saved_errno;
@@ -174,8 +182,11 @@ int flux_kvs_txn_put (flux_kvs_txn_t *txn, int flags,
     return -1;
 }
 
-int flux_kvs_txn_vpack (flux_kvs_txn_t *txn, int flags,
-                        const char *key, const char *fmt, va_list ap)
+int flux_kvs_txn_vpack (flux_kvs_txn_t *txn,
+                        int flags,
+                        const char *key,
+                        const char *fmt,
+                        va_list ap)
 {
     json_t *val, *dirent = NULL;
     int saved_errno;
@@ -192,7 +203,7 @@ int flux_kvs_txn_vpack (flux_kvs_txn_t *txn, int flags,
         errno = EINVAL;
         goto error;
     }
-    if (!(s = json_dumps (val, JSON_ENCODE_ANY))) {
+    if (!(s = json_dumps (val, JSON_ENCODE_ANY | JSON_COMPACT))) {
         errno = ENOMEM;
         json_decref (val);
         goto error;
@@ -215,8 +226,11 @@ int flux_kvs_txn_vpack (flux_kvs_txn_t *txn, int flags,
     return -1;
 }
 
-int flux_kvs_txn_pack (flux_kvs_txn_t *txn, int flags,
-                       const char *key, const char *fmt, ...)
+int flux_kvs_txn_pack (flux_kvs_txn_t *txn,
+                       int flags,
+                       const char *key,
+                       const char *fmt,
+                       ...)
 {
     va_list ap;
     int rc;
@@ -231,7 +245,8 @@ int flux_kvs_txn_pack (flux_kvs_txn_t *txn, int flags,
     return rc;
 }
 
-int flux_kvs_txn_mkdir (flux_kvs_txn_t *txn, int flags,
+int flux_kvs_txn_mkdir (flux_kvs_txn_t *txn,
+                        int flags,
                         const char *key)
 {
     json_t *dirent = NULL;
@@ -257,7 +272,8 @@ int flux_kvs_txn_mkdir (flux_kvs_txn_t *txn, int flags,
     return -1;
 }
 
-int flux_kvs_txn_unlink (flux_kvs_txn_t *txn, int flags,
+int flux_kvs_txn_unlink (flux_kvs_txn_t *txn,
+                         int flags,
                          const char *key)
 {
     json_t *dirent = NULL;
@@ -282,8 +298,10 @@ int flux_kvs_txn_unlink (flux_kvs_txn_t *txn, int flags,
     return -1;
 }
 
-int flux_kvs_txn_symlink (flux_kvs_txn_t *txn, int flags,
-                          const char *key, const char *ns,
+int flux_kvs_txn_symlink (flux_kvs_txn_t *txn,
+                          int flags,
+                          const char *key,
+                          const char *ns,
                           const char *target)
 {
     json_t *dirent = NULL;
@@ -308,6 +326,27 @@ int flux_kvs_txn_symlink (flux_kvs_txn_t *txn, int flags,
     return -1;
 }
 
+int flux_kvs_txn_clear (flux_kvs_txn_t *txn)
+{
+    if (!txn) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (json_array_clear (txn->ops) < 0) {
+        /* right errno? no obvious best choice */
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+bool flux_kvs_txn_is_empty (flux_kvs_txn_t *txn)
+{
+    if (!txn)
+        return true;
+    return json_array_size (txn->ops) ? false : true;
+}
+
 /* kvs_txn_private.h */
 
 int txn_get_op_count (flux_kvs_txn_t *txn)
@@ -338,10 +377,11 @@ int txn_decode_op (json_t *op, const char **keyp, int *flagsp, json_t **direntp)
     int flags;
     json_t *dirent;
 
-    if (json_unpack (op, "{s:s s:i s:o !}",
-                         "key", &key,
-                         "flags", &flags,
-                         "dirent", &dirent) < 0) {
+    if (json_unpack (op,
+                     "{s:s s:i s:o !}",
+                     "key", &key,
+                     "flags", &flags,
+                     "dirent", &dirent) < 0) {
         errno = EPROTO;
         return -1;
     }
@@ -358,8 +398,10 @@ int txn_encode_op (const char *key, int flags, json_t *dirent, json_t **opp)
 {
     json_t *op;
 
-    if (!key || strlen (key) == 0 || !dirent
-             || (!json_is_null (dirent) && treeobj_validate (dirent) < 0)) {
+    if (!key
+        || strlen (key) == 0
+        || !dirent
+        || (!json_is_null (dirent) && treeobj_validate (dirent) < 0)) {
         errno = EINVAL;
         return -1;
     }
diff --git a/src/common/libkvs/kvs_txn.h b/src/common/libkvs/kvs_txn.h
index 2dfd51ae7f7f..d884afb16754 100644
--- a/src/common/libkvs/kvs_txn.h
+++ b/src/common/libkvs/kvs_txn.h
@@ -11,42 +11,63 @@
 #ifndef _FLUX_CORE_KVS_TXN_H
 #define _FLUX_CORE_KVS_TXN_H
 
+#include <stdarg.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include <stdarg.h>
-
 typedef struct flux_kvs_txn flux_kvs_txn_t;
 
 flux_kvs_txn_t *flux_kvs_txn_create (void);
 void flux_kvs_txn_destroy (flux_kvs_txn_t *txn);
 
-int flux_kvs_txn_put (flux_kvs_txn_t *txn, int flags,
-                      const char *key, const char *value);
-
-int flux_kvs_txn_vpack (flux_kvs_txn_t *txn, int flags, const char *key,
-                        const char *fmt, va_list ap);
-
-int flux_kvs_txn_pack (flux_kvs_txn_t *txn, int flags, const char *key,
-                       const char *fmt, ...);
-
-int flux_kvs_txn_put_raw (flux_kvs_txn_t *txn, int flags,
-                          const char *key, const void *data, int len);
-
-int flux_kvs_txn_put_treeobj (flux_kvs_txn_t *txn, int flags,
-                              const char *key, const char *treeobj);
-
-int flux_kvs_txn_mkdir (flux_kvs_txn_t *txn, int flags,
+int flux_kvs_txn_put (flux_kvs_txn_t *txn,
+                      int flags,
+                      const char *key,
+                      const char *value);
+
+int flux_kvs_txn_vpack (flux_kvs_txn_t *txn,
+                        int flags,
+                        const char *key,
+                        const char *fmt,
+                        va_list ap);
+
+int flux_kvs_txn_pack (flux_kvs_txn_t *txn,
+                       int flags,
+                       const char *key,
+                       const char *fmt,
+                       ...);
+
+int flux_kvs_txn_put_raw (flux_kvs_txn_t *txn,
+                          int flags,
+                          const char *key,
+                          const void *data,
+                          size_t len);
+
+int flux_kvs_txn_put_treeobj (flux_kvs_txn_t *txn,
+                              int flags,
+                              const char *key,
+                              const char *treeobj);
+
+int flux_kvs_txn_mkdir (flux_kvs_txn_t *txn,
+                        int flags,
                         const char *key);
 
-int flux_kvs_txn_unlink (flux_kvs_txn_t *txn, int flags,
+int flux_kvs_txn_unlink (flux_kvs_txn_t *txn,
+                         int flags,
                          const char *key);
 
-int flux_kvs_txn_symlink (flux_kvs_txn_t *txn, int flags,
-                          const char *key, const char *ns,
+int flux_kvs_txn_symlink (flux_kvs_txn_t *txn,
+                          int flags,
+                          const char *key,
+                          const char *ns,
                           const char *target);
 
+int flux_kvs_txn_clear (flux_kvs_txn_t *txn);
+
+bool flux_kvs_txn_is_empty (flux_kvs_txn_t *txn);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/common/libkvs/kvs_txn_compact.c b/src/common/libkvs/kvs_txn_compact.c
index e51732e1c7b4..827e67a88bf4 100644
--- a/src/common/libkvs/kvs_txn_compact.c
+++ b/src/common/libkvs/kvs_txn_compact.c
@@ -13,9 +13,10 @@
 #endif
 #include <jansson.h>
 #include <flux/core.h>
-#include <czmq.h>
 #include <string.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
 #include "kvs_txn_private.h"
 #include "treeobj.h"
 
@@ -25,7 +26,7 @@
  * performance issues.
  *
  * One way to improve performance is to "compact" these appends in KVS
- * transactions before they are commited to the KVS.  If multiple
+ * transactions before they are committed to the KVS.  If multiple
  * appends to the same key exist in a KVS transaction, combine them
  * into a single append.  For example, an append of "A" to the key
  * "foo", followed by an append of "B" to the key "foo", could be
@@ -50,7 +51,7 @@
 
 struct append_data {
     void *data;
-    int len;
+    size_t len;
 };
 
 struct compact_key {
@@ -209,7 +210,7 @@ int txn_compact (flux_kvs_txn_t *txn)
     }
 
     len = json_array_size (txn->ops);
-    if (!len)
+    if (len < 2)
         return 0;
 
     if (!(ops_new = json_array ())) {
@@ -274,7 +275,7 @@ int txn_compact (flux_kvs_txn_t *txn)
                     goto error;
                 }
                 if (zhash_insert (append_keys, key, ck) < 0) {
-                    errno = ENOMEM;
+                    errno = EEXIST;
                     goto error;
                 }
                 zhash_freefn (append_keys, key, compact_key_destroy);
diff --git a/src/common/libkvs/test/kvs.c b/src/common/libkvs/test/kvs.c
index a89d75722caa..96f719fdc30e 100644
--- a/src/common/libkvs/test/kvs.c
+++ b/src/common/libkvs/test/kvs.c
@@ -14,9 +14,8 @@
 
 #include <string.h>
 #include <errno.h>
+#include <flux/core.h>
 
-#include "src/common/libflux/flux.h"
-#include "kvs.h"
 #include "src/common/libtap/tap.h"
 
 void errors (void)
@@ -27,6 +26,11 @@ void errors (void)
     ok (flux_kvs_namespace_create (NULL, NULL, 0, 5) == NULL && errno == EINVAL,
         "flux_kvs_namespace_create fails on bad input");
 
+    errno = 0;
+    ok (flux_kvs_namespace_create_with (NULL, NULL, NULL, 0, 5) == NULL
+        && errno == EINVAL,
+        "flux_kvs_namespace_create_with fails on bad input");
+
     errno = 0;
     ok (flux_kvs_namespace_remove (NULL, NULL) == NULL && errno == EINVAL,
         "flux_kvs_namespace_remove fails on bad input");
diff --git a/src/common/libkvs/test/kvs_checkpoint.c b/src/common/libkvs/test/kvs_checkpoint.c
new file mode 100644
index 000000000000..d4a975d522f7
--- /dev/null
+++ b/src/common/libkvs/test/kvs_checkpoint.c
@@ -0,0 +1,94 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <errno.h>
+#include <flux/core.h>
+
+#ifndef EDEADLOCK
+#define EDEADLOCK EDEADLK
+#endif
+
+#include "src/common/libtap/tap.h"
+
+#include "kvs_checkpoint.h"
+
+void errors (void)
+{
+    flux_future_t *f;
+    const char *rootref;
+
+    errno = 0;
+    ok (kvs_checkpoint_commit (NULL, NULL, NULL, 0, 0, -1) == NULL
+        && errno == EINVAL,
+        "kvs_checkpoint_commit fails on bad input");
+
+    errno = 0;
+    ok (kvs_checkpoint_lookup (NULL, NULL, -1) == NULL
+        && errno == EINVAL,
+        "kvs_checkpoint_lookup fails on bad input");
+
+    errno = 0;
+    ok (kvs_checkpoint_lookup_get_rootref (NULL, NULL) < 0
+        && errno == EINVAL,
+        "kvs_checkpoint_lookup_get_rootref fails on bad input");
+
+    errno = 0;
+    ok (kvs_checkpoint_lookup_get_timestamp (NULL, NULL) < 0
+        && errno == EINVAL,
+        "kvs_checkpoint_lookup_get_timestamp fails on bad input");
+
+    errno = 0;
+    ok (kvs_checkpoint_lookup_get_sequence (NULL, NULL) < 0
+        && errno == EINVAL,
+        "kvs_checkpoint_lookup_get_sequence fails on bad input");
+
+    if (!(f = flux_future_create (NULL, NULL)))
+        BAIL_OUT ("flux_future_create failed");
+
+    errno = 0;
+    ok (kvs_checkpoint_lookup_get_rootref (f, &rootref) < 0
+        && errno == EDEADLOCK,
+        "kvs_checkpoint_lookup_get_rootref fails on unfulfilled future");
+
+    errno = 0;
+    double timestamp;
+    ok (kvs_checkpoint_lookup_get_timestamp (f, &timestamp) < 0
+        && errno == EDEADLOCK,
+        "kvs_checkpoint_lookup_get_timestamp fails on unfulfilled future");
+
+    errno = 0;
+    int sequence;
+    ok (kvs_checkpoint_lookup_get_sequence (f, &sequence) < 0
+        && errno == EDEADLOCK,
+        "kvs_checkpoint_lookup_get_sequence fails on unfulfilled future");
+
+    flux_future_destroy (f);
+}
+
+int main (int argc, char *argv[])
+{
+
+    plan (NO_PLAN);
+
+    errors ();
+
+    done_testing();
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libkvs/test/kvs_commit.c b/src/common/libkvs/test/kvs_commit.c
index 5afcaab64fcd..66808601d2b7 100644
--- a/src/common/libkvs/test/kvs_commit.c
+++ b/src/common/libkvs/test/kvs_commit.c
@@ -14,10 +14,8 @@
 
 #include <string.h>
 #include <errno.h>
+#include <flux/core.h>
 
-#include "src/common/libflux/flux.h"
-#include "kvs_commit.h"
-#include "kvs_txn.h"
 #include "src/common/libtap/tap.h"
 
 void errors (void)
@@ -53,6 +51,11 @@ void errors (void)
         && errno == EINVAL,
         "flux_kvs_commit_get_treeobj fails on bad input");
 
+    errno = 0;
+    ok (flux_kvs_commit_get_rootref (NULL, NULL) < 0
+        && errno == EINVAL,
+        "flux_kvs_commit_get_rootref fails on bad input");
+
     errno = 0;
     ok (flux_kvs_commit_get_sequence (NULL, NULL) < 0
         && errno == EINVAL,
diff --git a/src/common/libkvs/test/kvs_dir.c b/src/common/libkvs/test/kvs_dir.c
index 23f4e507e8f3..ad9b8b183666 100644
--- a/src/common/libkvs/test/kvs_dir.c
+++ b/src/common/libkvs/test/kvs_dir.c
@@ -15,15 +15,15 @@
 #include <jansson.h>
 #include <string.h>
 #include <errno.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
-#include "kvs.h"
 #include "kvs_txn_private.h"
 #include "kvs_dir_private.h"
 #include "treeobj.h"
 
-#include "src/common/libflux/flux.h"
-#include "kvs_dir.h"
-#include "src/common/libtap/tap.h"
 
 void jdiag (json_t *o)
 {
@@ -39,9 +39,10 @@ void test_empty (void)
     json_t *o;
     char *s;
     const char *key;
+    char *keyat;
 
     lives_ok ({flux_kvsdir_destroy (NULL);},
-        "flux_kvsdir_destroy with NULL paramter doesn't crash");
+        "flux_kvsdir_destroy with NULL parameter doesn't crash");
 
     errno = 0;
     dir = flux_kvsdir_create (NULL, NULL, NULL, NULL);
@@ -76,6 +77,7 @@ void test_empty (void)
         BAIL_OUT ("json_dumps failed on new treeobj");
     dir = flux_kvsdir_create (NULL, NULL, "foo", s);
     free (s);
+    json_decref (o);
 
     ok (dir != NULL,
         "flux_kvsdir_create with empty directory works");
@@ -89,11 +91,12 @@ void test_empty (void)
         "flux_kvsdir_issymlink on nonexistent key returns false");
 
     key = flux_kvsdir_key (dir);
-    ok (key != NULL && !strcmp (key, "foo"),
+    ok (key != NULL && streq (key, "foo"),
         "flux_kvsdir_key returns the key we put in");
-    key = flux_kvsdir_key_at (dir, "a.b.c");
-    ok (key != NULL && !strcmp (key, "foo.a.b.c"),
+    keyat = flux_kvsdir_key_at (dir, "a.b.c");
+    ok (keyat != NULL && streq (keyat, "foo.a.b.c"),
         "flux_kvsdir_key_at a.b.c returns foo.a.b.c");
+    free (keyat);
     ok (flux_kvsdir_handle (dir) == NULL,
         "flux_kvsdir_handle returns NULL since that's what we put in");
     ok (flux_kvsdir_rootref (dir) == NULL,
@@ -239,6 +242,8 @@ void test_full (void)
         "flux_kvsdir_get_size on copy still returns 4 after orig freed");
 
     flux_kvsdir_destroy (cpy);
+
+    json_decref (o);
 }
 
 int main (int argc, char *argv[])
diff --git a/src/common/libkvs/test/kvs_getroot.c b/src/common/libkvs/test/kvs_getroot.c
index b78c65212ee9..60cb08ba5184 100644
--- a/src/common/libkvs/test/kvs_getroot.c
+++ b/src/common/libkvs/test/kvs_getroot.c
@@ -16,8 +16,6 @@
 #include <errno.h>
 #include <flux/core.h>
 
-#include "src/common/libflux/flux.h"
-#include "kvs_getroot.h"
 #include "src/common/libtap/tap.h"
 
 void errors (void)
diff --git a/src/common/libkvs/test/kvs_lookup.c b/src/common/libkvs/test/kvs_lookup.c
index f88af93a8f24..adda81452ed8 100644
--- a/src/common/libkvs/test/kvs_lookup.c
+++ b/src/common/libkvs/test/kvs_lookup.c
@@ -16,8 +16,6 @@
 #include <errno.h>
 #include <flux/core.h>
 
-#include "src/common/libflux/flux.h"
-#include "kvs_lookup.h"
 #include "src/common/libtap/tap.h"
 
 void errors (void)
diff --git a/src/common/libkvs/test/kvs_txn.c b/src/common/libkvs/test/kvs_txn.c
index c302a55cc226..d582ce1f0335 100644
--- a/src/common/libkvs/test/kvs_txn.c
+++ b/src/common/libkvs/test/kvs_txn.c
@@ -15,12 +15,14 @@
 #include <jansson.h>
 #include <string.h>
 #include <errno.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
-#include "kvs.h"
 #include "kvs_txn_private.h"
 #include "treeobj.h"
 
-#include "src/common/libtap/tap.h"
 
 void jdiag (json_t *o)
 {
@@ -35,7 +37,7 @@ void jdiag (json_t *o)
 int check_null_value (json_t *dirent)
 {
     char *data = "";
-    int len = -1;
+    size_t len = 42;
 
     if (treeobj_decode_val (dirent, (void **)&data, &len) < 0) {
         diag ("%s: initial base64 decode failed", __FUNCTION__);
@@ -54,7 +56,8 @@ int check_null_value (json_t *dirent)
 int check_int_value (json_t *dirent, int expected)
 {
     char *data;
-    int rc, len, i;
+    int rc, i;
+    size_t len;
     json_t *val;
 
     if (treeobj_decode_val (dirent, (void **)&data, &len) < 0) {
@@ -87,7 +90,8 @@ int check_int_value (json_t *dirent, int expected)
 int check_string_value (json_t *dirent, const char *expected)
 {
     char *data;
-    int rc, len;
+    int rc;
+    size_t len;
     const char *s;
     json_t *val;
 
@@ -107,7 +111,7 @@ int check_string_value (json_t *dirent, const char *expected)
         json_decref (val);
         return -1;
     }
-    if (strcmp (expected, s) != 0) {
+    if (!streq (expected, s)) {
         diag ("%s: expected %s received %s", __FUNCTION__, expected, s);
         json_decref (val);
         return -1;
@@ -119,7 +123,7 @@ int check_string_value (json_t *dirent, const char *expected)
 int check_raw_value (json_t *dirent, const char *expected, int expected_len)
 {
     char *data;
-    int len;
+    size_t len;
 
     if (treeobj_decode_val (dirent, (void **)&data, &len) < 0) {
         diag ("%s: initial base64 decode failed", __FUNCTION__);
@@ -144,6 +148,8 @@ void basic (void)
     txn = flux_kvs_txn_create ();
     ok (txn != NULL,
         "flux_kvs_txn_create works");
+    ok (flux_kvs_txn_is_empty (txn) == true,
+        "flux_kvs_txn_is_empty returns true immediately after create");
     rc = flux_kvs_txn_pack (txn, FLUX_KVS_APPEND, "foo.bar.baz",  "i", 42);
     ok (rc == 0,
         "1: flux_kvs_txn_pack(i) flags=FLUX_KVS_APPEND works");
@@ -171,21 +177,11 @@ void basic (void)
     rc = flux_kvs_txn_symlink (txn, 0, "f.f.f", "g.g.g", "h.h.h");
     ok (rc == 0,
         "9: flux_kvs_txn_symlink works (namespace)");
-    errno = 0;
-    rc = flux_kvs_txn_pack (txn, 0xFFFF, "foo.bar.blorp",  "s", "foo");
-    ok (rc < 0 && errno == EINVAL,
-        "error: flux_kvs_txn_pack(bad flags) fails with EINVAL");
-    errno = 0;
-    rc = flux_kvs_txn_mkdir (txn, 0xFFFF, "b.b.b");
-    ok (rc < 0 && errno == EINVAL,
-        "error: flux_kvs_txn_mkdir works");
-    errno = 0;
-    rc = flux_kvs_txn_put (txn, 0xFFFF, "f", "42");
-    ok (rc < 0 && errno == EINVAL,
-        "error: flux_kvs_txn_put(bad flags) fails with EINVAL");
 
     /* Verify transaction contents
      */
+    ok (flux_kvs_txn_is_empty (txn) == false,
+        "flux_kvs_txn_is_empty returns false after transactions added");
     ok (txn_get_op_count (txn) == 9,
         "txn contains 9 ops");
     ok (txn_get_op (txn, 0, &entry) == 0
@@ -193,7 +189,7 @@ void basic (void)
         "1: retrieved");
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "1: txn_decode_op works");
-    ok (!strcmp (key, "foo.bar.baz")
+    ok (streq (key, "foo.bar.baz")
         && flags == FLUX_KVS_APPEND
         && check_int_value (dirent, 42) == 0,
         "1: put foo.bar.baz = 42");
@@ -203,7 +199,7 @@ void basic (void)
     jdiag (entry);
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "2: txn_decode_op works");
-    ok (!strcmp (key, "foo.bar.bleep")
+    ok (streq (key, "foo.bar.bleep")
         && flags == 0
         && check_string_value (dirent, "foo") == 0,
         "2: put foo.bar.baz = \"foo\"");
@@ -213,7 +209,7 @@ void basic (void)
     jdiag (entry);
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "3: txn_decode_op works");
-    ok (!strcmp (key, "a")
+    ok (streq (key, "a")
         && flags == 0
         && json_is_null (dirent),
         "3: unlink a");
@@ -223,7 +219,7 @@ void basic (void)
     jdiag (entry);
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "4: txn_decode_op works");
-    ok (!strcmp (key, "b.b.b")
+    ok (streq (key, "b.b.b")
         && flags == 0
         && treeobj_is_dir (dirent) && treeobj_get_count (dirent) == 0,
         "4: mkdir b.b.b");
@@ -233,12 +229,12 @@ void basic (void)
     jdiag (entry);
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "5: txn_decode_op works");
-    ok (!strcmp (key, "c.c.c")
+    ok (streq (key, "c.c.c")
         && flags == 0
         && treeobj_is_symlink (dirent)
         && !json_object_get (treeobj_get_data (dirent),
                              "namespace")
-        && !strcmp (json_string_value (json_object_get (treeobj_get_data (dirent),
+        && streq (json_string_value (json_object_get (treeobj_get_data (dirent),
                                                         "target")),
                     "b.b.b"),
         "5: symlink c.c.c b.b.b (no namespace)");
@@ -248,7 +244,7 @@ void basic (void)
     jdiag (entry);
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "6: txn_decode_op works");
-    ok (!strcmp (key, "d.d.d")
+    ok (streq (key, "d.d.d")
         && flags == 0
         && check_int_value (dirent, 43) == 0,
         "6: put foo.bar.baz = 43");
@@ -258,7 +254,7 @@ void basic (void)
     jdiag (entry);
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "7: txn_decode_op works");
-    ok (!strcmp (key, "e")
+    ok (streq (key, "e")
         && flags == 0
         && json_is_null (dirent),
         "7: unlink e");
@@ -268,7 +264,7 @@ void basic (void)
     jdiag (entry);
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "8: txn_decode_op works");
-    ok (!strcmp (key, "nerrrrb")
+    ok (streq (key, "nerrrrb")
         && flags == 0
         && check_null_value (dirent) == 0,
         "8: put nerrrrb = NULL");
@@ -278,20 +274,29 @@ void basic (void)
     jdiag (entry);
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "9: txn_decode_op works");
-    ok (!strcmp (key, "f.f.f")
+    ok (streq (key, "f.f.f")
         && flags == 0
-        && !strcmp (json_string_value (json_object_get (treeobj_get_data (dirent),
-                                                        "namespace")),
-                    "g.g.g")
-        && !strcmp (json_string_value (json_object_get (treeobj_get_data (dirent),
-                                                        "target")),
-                    "h.h.h"),
+        && streq (json_string_value (json_object_get (treeobj_get_data (dirent),
+                                                      "namespace")),
+                  "g.g.g")
+        && streq (json_string_value (json_object_get (treeobj_get_data (dirent),
+                                                      "target")),
+                  "h.h.h"),
         "9: symlink f.f.f g.g.g h.h.h (namespace)");
 
     errno = 0;
     ok (txn_get_op (txn, 9, &entry) < 0 && errno == EINVAL,
         "10: invalid (end of transaction)");
 
+    ok (flux_kvs_txn_clear (txn) == 0,
+        "flux_kvs_txn_clear success");
+
+    ok (txn_get_op_count (txn) == 0,
+        "txn contains 0 ops");
+
+    ok (flux_kvs_txn_is_empty (txn) == true,
+        "flux_kvs_txn_is_empty returns true after clear");
+
     flux_kvs_txn_destroy (txn);
 }
 
@@ -301,7 +306,8 @@ void test_raw_values (void)
     char buf[13], *nbuf;
     json_t *entry, *dirent;
     const char *key;
-    int flags, nlen;
+    int flags;
+    size_t nlen;
 
     memset (buf, 'c', sizeof (buf));
 
@@ -309,18 +315,6 @@ void test_raw_values (void)
     ok (txn != NULL,
         "flux_kvs_txn_create works");
 
-    /* Try some bad params
-     */
-    errno = 0;
-    ok (flux_kvs_txn_put_raw (txn, FLUX_KVS_TREEOBJ, "a.b.c",
-                              buf, sizeof (buf)) < 0
-        && errno == EINVAL,
-        "flux_kvs_txn_put_raw fails with EINVAL when fed TREEOBJ flag");
-    errno = 0;
-    ok (flux_kvs_txn_put_raw (txn, 0, NULL, buf, sizeof (buf)) < 0
-        && errno == EINVAL,
-        "flux_kvs_txn_put_raw fails with EINVAL when fed NULL key");
-
     /* Put an empty buffer.
      */
     ok (flux_kvs_txn_put_raw (txn, 0, "a.a.a", NULL, 0) == 0,
@@ -334,7 +328,7 @@ void test_raw_values (void)
     ok (txn_get_op_count (txn) == 2,
         "txn contains two ops");
     ok (txn_get_op (txn, 0, &entry) == 0 && entry != NULL,
-        "retreived 1st op from txn");
+        "retrieved 1st op from txn");
     jdiag (entry);
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "txn_decode_op works");
@@ -350,7 +344,7 @@ void test_raw_values (void)
     /* Get 2nd
      */
     ok (txn_get_op (txn, 1, &entry) == 0 && entry != NULL,
-        "retreived 2nd op from txn");
+        "retrieved 2nd op from txn");
     jdiag (entry);
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "txn_decode_op works");
@@ -366,8 +360,11 @@ void test_raw_values (void)
 
 void test_corner_cases (void)
 {
+    flux_kvs_txn_t *txn;
     json_t *val;
     json_t *op;
+    char *treeobj;
+    int rc;
 
     ok (txn_encode_op (NULL, 0, NULL, NULL) < 0 && errno == EINVAL,
         "txn_encode_op fails on bad input");
@@ -377,7 +374,99 @@ void test_corner_cases (void)
     ok (txn_encode_op ("key", 0x44, val, &op) < 0 && errno == EINVAL,
         "txn_encode_op fails on bad flags");
 
+    if (!(txn = flux_kvs_txn_create()))
+        BAIL_OUT ("flux_kvs_txn_create failed");
+
+    errno = 0;
+    rc = flux_kvs_txn_put (NULL, 0, NULL, NULL);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_put fails w/ EINVAL on bad inputs");
+
+    errno = 0;
+    rc = flux_kvs_txn_vpack (NULL, 0, NULL, NULL, (va_list){0});
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_vpack fails w/ EINVAL on bad inputs");
+
+    errno = 0;
+    rc = flux_kvs_txn_pack (NULL, 0, NULL, NULL);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_pack fails w/ EINVAL on bad inputs");
+
+    errno = 0;
+    rc = flux_kvs_txn_put_raw (NULL, 0, NULL, NULL, 0);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_put_raw fails w/ EINVAL on bad inputs");
+
+    errno = 0;
+    rc = flux_kvs_txn_put_treeobj (NULL, 0, NULL, NULL);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_put_treeobj fails w/ EINVAL on bad inputs");
+
+    errno = 0;
+    rc = flux_kvs_txn_mkdir (NULL, 0, NULL);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_mkdir fails w/ EINVAL on bad inputs");
+
+    errno = 0;
+    rc = flux_kvs_txn_unlink (NULL, 0, NULL);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_unlink fails w/ EINVAL on bad inputs");
+
+    errno = 0;
+    rc = flux_kvs_txn_symlink (NULL, 0, NULL, NULL, NULL);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_symlink fails w/ EINVAL on bad inputs");
+
+    errno = 0;
+    rc = flux_kvs_txn_clear (NULL);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_clear fails w/ EINVAL on bad input");
+
+    ok (flux_kvs_txn_is_empty (NULL) == true,
+        "flux_kvs_txn_is_empty returns true on NULL input");
+
+    errno = 0;
+    rc = flux_kvs_txn_put (txn, 0xFFFF, "a", "42");
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_put fails with EINVAL on bad flags");
+
+    errno = 0;
+    rc = flux_kvs_txn_pack (txn, 0xFFFF, "b",  "s", "foo");
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_pack fails with EINVAL on bad flags");
+
+    errno = 0;
+    rc = flux_kvs_txn_put_raw (txn, 0xFFFF, "c", "bar", 3);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_put_raw fails with EINVAL on bad flags");
+
+    treeobj = json_dumps (val, 0);
+    if (!treeobj)
+        BAIL_OUT ("failed to created treeobj string");
+
+    errno = 0;
+    rc = flux_kvs_txn_put_treeobj (txn, 0xFFFF, "d", treeobj);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_put_treeobj fails with EINVAL on bad flags");
+
+    errno = 0;
+    rc = flux_kvs_txn_mkdir (txn, 0xFFFF, "e");
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_mkdir fails with EINVAL on bad flags");
+
+    errno = 0;
+    rc = flux_kvs_txn_unlink (txn, 0xFFFF, "f");
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_unlink fails with EINVAL on bad flags");
+
+    errno = 0;
+    rc = flux_kvs_txn_symlink (txn, 0xFFFF, "g", "ns", "h");
+    ok (rc < 0 && errno == EINVAL,
+        "flux_kvs_txn_symlink fails with EINVAL on bad flags");
+
     json_decref (val);
+    free (treeobj);
+    flux_kvs_txn_destroy (txn);
 }
 
 int main (int argc, char *argv[])
diff --git a/src/common/libkvs/test/kvs_txn_compact.c b/src/common/libkvs/test/kvs_txn_compact.c
index 7c20d11ddc97..377bcd3e1aee 100644
--- a/src/common/libkvs/test/kvs_txn_compact.c
+++ b/src/common/libkvs/test/kvs_txn_compact.c
@@ -15,19 +15,21 @@
 #include <jansson.h>
 #include <string.h>
 #include <errno.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
-#include "kvs.h"
 #include "kvs_txn_private.h"
 #include "treeobj.h"
 
-#include "src/common/libtap/tap.h"
 
 /* Decode a 'val' containing base64 encoded emptiness.
  */
 int check_null_value (json_t *dirent)
 {
     char *data = "";
-    int len = -1;
+    size_t len = 42;
 
     if (treeobj_decode_val (dirent, (void **)&data, &len) < 0) {
         diag ("%s: initial base64 decode failed", __FUNCTION__);
@@ -42,8 +44,9 @@ int check_null_value (json_t *dirent)
 
 int check_raw_value (json_t *dirent, const char *expected, int expected_len)
 {
-    char *data;
-    int len;
+    char *data = NULL;
+    size_t len;
+    int rc = -1;
 
     if (treeobj_decode_val (dirent, (void **)&data, &len) < 0) {
         diag ("%s: initial base64 decode failed", __FUNCTION__);
@@ -51,8 +54,9 @@ int check_raw_value (json_t *dirent, const char *expected, int expected_len)
     }
     if (len == expected_len
         && memcmp (data, expected, len) == 0)
-        return 0;
-    return -1;
+        rc = 0;
+    free (data);
+    return rc;
 }
 
 int main (int argc, char *argv[])
@@ -112,7 +116,7 @@ int main (int argc, char *argv[])
         "1: retrieved");
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "1: txn_decode_op works");
-    ok (!strcmp (key, "foo")
+    ok (streq (key, "foo")
         && flags == FLUX_KVS_APPEND
         && check_raw_value (dirent, "ABC", 3) == 0,
         "1: consolidated foo = ABC");
@@ -150,7 +154,7 @@ int main (int argc, char *argv[])
         "1: retrieved");
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "1: txn_decode_op works");
-    ok (!strcmp (key, "foo")
+    ok (streq (key, "foo")
         && flags == FLUX_KVS_APPEND
         && check_raw_value (dirent, "AC", 2) == 0,
         "1: consolidated foo = AC");
@@ -159,7 +163,7 @@ int main (int argc, char *argv[])
         "2: retrieved");
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "2: txn_decode_op works");
-    ok (!strcmp (key, "bar")
+    ok (streq (key, "bar")
         && flags == FLUX_KVS_APPEND
         && check_raw_value (dirent, "B", 1) == 0,
         "2: bar = B");
@@ -200,7 +204,7 @@ int main (int argc, char *argv[])
         "1: retrieved");
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "1: txn_decode_op works");
-    ok (!strcmp (key, "foo")
+    ok (streq (key, "foo")
         && flags == FLUX_KVS_APPEND
         && check_raw_value (dirent, "AC", 2) == 0,
         "1: consolidated foo = AC");
@@ -209,7 +213,7 @@ int main (int argc, char *argv[])
         "2: retrieved");
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "2: txn_decode_op works");
-    ok (!strcmp (key, "bar")
+    ok (streq (key, "bar")
         && flags == FLUX_KVS_APPEND
         && check_raw_value (dirent, "BD", 2) == 0,
         "2: bar = BD");
@@ -247,7 +251,7 @@ int main (int argc, char *argv[])
         "1: retrieved");
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "1: txn_decode_op works");
-    ok (!strcmp (key, "foo")
+    ok (streq (key, "foo")
         && flags == 0
         && check_raw_value (dirent, "A", 1) == 0,
         "1: consolidated foo = A");
@@ -256,7 +260,7 @@ int main (int argc, char *argv[])
         "2: retrieved");
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "2: txn_decode_op works");
-    ok (!strcmp (key, "foo")
+    ok (streq (key, "foo")
         && flags == FLUX_KVS_APPEND
         && check_raw_value (dirent, "BC", 2) == 0,
         "2: foo = BC");
@@ -313,7 +317,7 @@ int main (int argc, char *argv[])
         "1: retrieved");
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "1: txn_decode_op works");
-    ok (!strcmp (key, "foo")
+    ok (streq (key, "foo")
         && flags == FLUX_KVS_APPEND
         && check_raw_value (dirent, "A", 1) == 0,
         "1: consolidated foo = A");
@@ -351,11 +355,13 @@ int main (int argc, char *argv[])
         "1: retrieved");
     ok (txn_decode_op (entry, &key, &flags, &dirent) == 0,
         "1: txn_decode_op works");
-    ok (!strcmp (key, "foo")
+    ok (streq (key, "foo")
         && flags == FLUX_KVS_APPEND
         && check_null_value (dirent) == 0,
         "1: consolidated foo = A");
 
+    flux_kvs_txn_destroy (txn);
+
     /* Test 8: single append is a no-op */
 
     txn = flux_kvs_txn_create ();
diff --git a/src/common/libkvs/test/kvs_util.c b/src/common/libkvs/test/kvs_util.c
index 804eccf79b6b..c7a84ad849c8 100644
--- a/src/common/libkvs/test/kvs_util.c
+++ b/src/common/libkvs/test/kvs_util.c
@@ -14,9 +14,12 @@
 #include <stdbool.h>
 #include <string.h>
 #include <errno.h>
+#include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
-#include "src/common/libkvs/kvs_util_private.h"
+#include "ccan/str/str.h"
+
+#include "kvs_util_private.h"
 
 void kvs_util_normalize_key_path_tests (void)
 {
@@ -24,42 +27,42 @@ void kvs_util_normalize_key_path_tests (void)
     bool dirflag;
 
     s = kvs_util_normalize_key ("a.b.c..d.e", &dirflag);
-    ok (s != NULL && !strcmp (s, "a.b.c.d.e") && dirflag == false,
+    ok (s != NULL && streq (s, "a.b.c.d.e") && dirflag == false,
         "kvs_util_normalize_key transforms consecutive path separators to one");
     free (s);
 
     s = kvs_util_normalize_key (".a.b.c.d.e", &dirflag);
-    ok (s != NULL && !strcmp (s, "a.b.c.d.e") && dirflag == false,
+    ok (s != NULL && streq (s, "a.b.c.d.e") && dirflag == false,
         "kvs_util_normalize_key drops one leading path separator");
     free (s);
 
     s = kvs_util_normalize_key ("....a.b.c.d.e", &dirflag);
-    ok (s != NULL && !strcmp (s, "a.b.c.d.e") && dirflag == false,
+    ok (s != NULL && streq (s, "a.b.c.d.e") && dirflag == false,
         "kvs_util_normalize_key drops several leading path separators");
     free (s);
 
     s = kvs_util_normalize_key ("a.b.c.d.e.", &dirflag);
-    ok (s != NULL && !strcmp (s, "a.b.c.d.e") && dirflag == true,
+    ok (s != NULL && streq (s, "a.b.c.d.e") && dirflag == true,
         "kvs_util_normalize_key drops one trailing path separator");
     free (s);
 
     s = kvs_util_normalize_key ("a.b.c.d.e.....", &dirflag);
-    ok (s != NULL && !strcmp (s, "a.b.c.d.e") && dirflag == true,
+    ok (s != NULL && streq (s, "a.b.c.d.e") && dirflag == true,
         "kvs_util_normalize_key drops several trailing path separators");
     free (s);
 
     s = kvs_util_normalize_key (".a....b.c.....d..e.....", &dirflag);
-    ok (s != NULL && !strcmp (s, "a.b.c.d.e") && dirflag == true,
+    ok (s != NULL && streq (s, "a.b.c.d.e") && dirflag == true,
         "kvs_util_normalize_key fixes a big mess");
     free (s);
 
     s = kvs_util_normalize_key (".", &dirflag);
-    ok (s != NULL && !strcmp (s, "."),
+    ok (s != NULL && streq (s, "."),
         "kvs_util_normalize_key leaves one standalone separator as is");
     free (s);
 
     s = kvs_util_normalize_key ("....", &dirflag);
-    ok (s != NULL && !strcmp (s, "."),
+    ok (s != NULL && streq (s, "."),
         "kvs_util_normalize_key transforms several standalone separators to one");
     free (s);
 }
diff --git a/src/common/libkvs/test/treeobj.c b/src/common/libkvs/test/treeobj.c
index e73c549e5d37..8413c6a6ad42 100644
--- a/src/common/libkvs/test/treeobj.c
+++ b/src/common/libkvs/test/treeobj.c
@@ -8,15 +8,21 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 #include <errno.h>
-
-#include "src/common/libkvs/treeobj.h"
+#include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/sha1.h"
 #include "src/common/libutil/blobref.h"
+#include "ccan/str/str.h"
+
+#include "treeobj.h"
+
 
 const int large_dir_entries = 5000;
 json_t *create_large_dir (void)
@@ -83,7 +89,7 @@ void test_codec (void)
     p = treeobj_encode (cpy1);
     ok (p != NULL,
         "re-encoded %d-entry dir", large_dir_entries);
-    ok (strcmp (p, s) == 0,
+    ok (streq (p, s),
         "and they match");
     free (p);
 
@@ -126,17 +132,17 @@ void test_valref (void)
     ok (treeobj_get_count (valref) == 1,
         "treeobj_get_count returns 1");
     blobref = treeobj_get_blobref (valref, 0);
-    ok (blobref != NULL && !strcmp (blobref, blobrefs[0]),
+    ok (blobref != NULL && streq (blobref, blobrefs[0]),
         "treeobj_get_blobref [0] returns expected blobref");
     ok (treeobj_append_blobref (valref, blobrefs[1]) == 0,
         "treeobj_append_blobref works on 2nd blobref");
     ok (treeobj_get_count (valref) == 2,
         "treeobj_get_count returns 1");
     blobref = treeobj_get_blobref (valref, 0);
-    ok (blobref != NULL && !strcmp (blobref, blobrefs[0]),
+    ok (blobref != NULL && streq (blobref, blobrefs[0]),
         "treeobj_get_blobref [0] still returns expected blobref");
     blobref = treeobj_get_blobref (valref, 1);
-    ok (blobref != NULL && !strcmp (blobref, blobrefs[1]),
+    ok (blobref != NULL && streq (blobref, blobrefs[1]),
         "treeobj_get_blobref [1] returns expected blobref");
     diag_json (valref);
     json_decref (valref);
@@ -148,7 +154,7 @@ void test_valref (void)
     ok (treeobj_get_count (valref) == 1,
         "treeobj_get_count returns 1");
     blobref = treeobj_get_blobref (valref, 0);
-    ok (blobref != NULL && !strcmp (blobref, blobrefs[0]),
+    ok (blobref != NULL && streq (blobref, blobrefs[0]),
         "treeobj_get_blobref [0] returns expected blobref");
     diag_json (valref);
     json_decref (valref);
@@ -165,7 +171,7 @@ void test_valref (void)
     blobref = treeobj_get_blobref (valref, 0);
     for (i = 1; i < 4; i++) {
         blobref2 = treeobj_get_blobref (valref, i);
-        if (!blobref || !blobref2 || strcmp (blobref, blobref2) != 0)
+        if (!blobref || !blobref2 || !streq (blobref, blobref2))
             break;
     }
     ok (i == 4,
@@ -185,7 +191,7 @@ void test_val (void)
     json_t *val, *val2;
     char buf[32];
     char *outbuf;
-    int outlen;
+    size_t outlen;
 
     memset (buf, 'x', sizeof (buf));
 
@@ -254,17 +260,17 @@ void test_dirref (void)
     ok (treeobj_get_count (dirref) == 1,
         "treeobj_get_count returns 1");
     blobref = treeobj_get_blobref (dirref, 0);
-    ok (blobref != NULL && !strcmp (blobref, blobrefs[0]),
+    ok (blobref != NULL && streq (blobref, blobrefs[0]),
         "treeobj_get_blobref [0] returns expected blobref");
     ok (treeobj_append_blobref (dirref, blobrefs[1]) == 0,
         "treeobj_append_blobref works on 2nd blobref");
     ok (treeobj_get_count (dirref) == 2,
         "treeobj_get_count returns 1");
     blobref = treeobj_get_blobref (dirref, 0);
-    ok (blobref != NULL && !strcmp (blobref, blobrefs[0]),
+    ok (blobref != NULL && streq (blobref, blobrefs[0]),
         "treeobj_get_blobref [0] still returns expected blobref");
     blobref = treeobj_get_blobref (dirref, 1);
-    ok (blobref != NULL && !strcmp (blobref, blobrefs[1]),
+    ok (blobref != NULL && streq (blobref, blobrefs[1]),
         "treeobj_get_blobref [1] returns expected blobref");
     diag_json (dirref);
     json_decref (dirref);
@@ -276,7 +282,7 @@ void test_dirref (void)
     ok (treeobj_get_count (dirref) == 1,
         "treeobj_get_count returns 1");
     blobref = treeobj_get_blobref (dirref, 0);
-    ok (blobref != NULL && !strcmp (blobref, blobrefs[0]),
+    ok (blobref != NULL && streq (blobref, blobrefs[0]),
         "treeobj_get_blobref [0] returns expected blobref");
 
     diag_json (dirref);
@@ -686,7 +692,7 @@ void test_symlink (void)
         "treeobj_get_symlink works on symlink without namespace");
     ok (ns_str == NULL,
         "treeobj_get_symlink returns NULL for namespace");
-    ok (!strcmp (target_str, "a.b.c"),
+    ok (streq (target_str, "a.b.c"),
         "treeobj_get_symlink returns correct string for target");
 
     json_decref (o);
@@ -701,9 +707,9 @@ void test_symlink (void)
         "treeobj_get_data returned string");
     ok (treeobj_get_symlink (o, &ns_str, &target_str) == 0,
         "treeobj_get_symlink works on symlink with namespace");
-    ok (!strcmp (ns_str, "ns"),
+    ok (streq (ns_str, "ns"),
         "treeobj_get_symlink returns correct string for namespace");
-    ok (!strcmp (target_str, "d.e.f"),
+    ok (streq (target_str, "d.e.f"),
         "treeobj_get_symlink returns correct string for target");
 
     json_decref (o);
@@ -714,7 +720,7 @@ void test_corner_cases (void)
     json_t *val, *valref, *dir, *symlink;
     json_t *array, *object;
     char *outbuf;
-    int outlen;
+    size_t outlen;
 
     val = treeobj_create_val ("a", 1);
     if (!val)
@@ -801,6 +807,45 @@ void test_corner_cases (void)
     json_decref (symlink);
 }
 
+void test_type_name (void)
+{
+    json_t *val, *valref, *dir, *dirref, *symlink, *notatreeobj;
+    const char *s;
+
+    val = treeobj_create_val ("a", 1);
+    valref = treeobj_create_valref (NULL);
+    dir = treeobj_create_dir ();
+    dirref = treeobj_create_dirref (NULL);
+    symlink = treeobj_create_symlink (NULL, "some-string");
+    notatreeobj = json_object ();
+    if (!val || !valref || !dir || !dirref || !symlink || !notatreeobj)
+        BAIL_OUT ("can't continue without test value");
+
+    s = treeobj_type_name (val);
+    ok (streq (s, "val"), "treeobj_type_name returns val correctly");
+    s = treeobj_type_name (valref);
+    ok (streq (s, "valref"), "treeobj_type_name returns valref correctly");
+    s = treeobj_type_name (dir);
+    ok (streq (s, "dir"), "treeobj_type_name returns dir correctly");
+    s = treeobj_type_name (dirref);
+    ok (streq (s, "dirref"), "treeobj_type_name returns dirref correctly");
+    s = treeobj_type_name (symlink);
+    ok (streq (s, "symlink"), "treeobj_type_name returns symlink correctly");
+    s = treeobj_type_name (notatreeobj);
+    ok (streq (s, "unknown"),
+        "treeobj_type_name returns unknown for non-treeobj");
+    s = treeobj_type_name (NULL);
+    ok (streq (s, "unknown"),
+        "treeobj_type_name returns unknown on invalid input");
+
+    json_decref (val);
+    json_decref (valref);
+    json_decref (dir);
+    json_decref (dirref);
+    json_decref (symlink);
+    json_decref (notatreeobj);
+}
+
 int main(int argc, char** argv)
 {
     plan (NO_PLAN);
@@ -814,6 +859,7 @@ int main(int argc, char** argv)
     test_deep_copy ();
     test_symlink ();
     test_corner_cases ();
+    test_type_name ();
 
     test_codec ();
 
diff --git a/src/common/libkvs/treeobj.c b/src/common/libkvs/treeobj.c
index 15a983a27c24..e45efd453782 100644
--- a/src/common/libkvs/treeobj.c
+++ b/src/common/libkvs/treeobj.c
@@ -16,12 +16,15 @@
 #include <errno.h>
 #include <string.h>
 #include <assert.h>
-#include <sodium.h>
+#include <stdbool.h>
+#include <jansson.h>
 
-#include "treeobj.h"
-#include "src/common/libutil/macros.h"
+#include "ccan/base64/base64.h"
+#include "ccan/str/str.h"
 #include "src/common/libutil/blobref.h"
 
+#include "treeobj.h"
+
 static const int treeobj_version = 1;
 
 static int treeobj_unpack (json_t *obj, const char **typep, json_t **datap)
@@ -29,11 +32,13 @@ static int treeobj_unpack (json_t *obj, const char **typep, json_t **datap)
     json_t *data;
     int version;
     const char *type;
-    if (!obj || json_unpack (obj, "{s:i s:s s:o !}",
-                                  "ver", &version,
-                                  "type", &type,
-                                  "data", &data) < 0
-             || version != treeobj_version) {
+    if (!obj
+        || json_unpack (obj,
+                        "{s:i s:s s:o !}",
+                        "ver", &version,
+                        "type", &type,
+                        "data", &data) < 0
+        || version != treeobj_version) {
         errno = EINVAL;
         return -1;
     }
@@ -44,7 +49,8 @@ static int treeobj_unpack (json_t *obj, const char **typep, json_t **datap)
     return 0;
 }
 
-static int treeobj_peek (const json_t *obj, const char **typep,
+static int treeobj_peek (const json_t *obj,
+                         const char **typep,
                          const json_t **datap)
 {
     json_t *data;
@@ -54,11 +60,13 @@ static int treeobj_peek (const json_t *obj, const char **typep,
     /* N.B. it should be safe to cast away const on 'obj' as long as 'data'
      * parameter is not modified.  We make 'data' const to ensure that.
      */
-    if (!obj || json_unpack ((json_t *)obj, "{s:i s:s s:o !}",
-                                  "ver", &version,
-                                  "type", &type,
-                                  "data", &data) < 0
-             || version != treeobj_version) {
+    if (!obj
+        || json_unpack ((json_t *)obj,
+                        "{s:i s:s s:o !}",
+                        "ver", &version,
+                        "type", &type,
+                        "data", &data) < 0
+        || version != treeobj_version) {
         errno = EINVAL;
         return -1;
     }
@@ -77,7 +85,7 @@ int treeobj_validate (const json_t *obj)
 
     if (treeobj_peek (obj, &type, &data) < 0)
         goto inval;
-    if (!strcmp (type, "valref") || !strcmp (type, "dirref")) {
+    if (streq (type, "valref") || streq (type, "dirref")) {
         int i, len;
         if (!json_is_array (data))
             goto inval;
@@ -89,7 +97,7 @@ int treeobj_validate (const json_t *obj)
                 goto inval;
         }
     }
-    else if (!strcmp (type, "dir")) {
+    else if (streq (type, "dir")) {
         const char *key;
         if (!json_is_object (data))
             goto inval;
@@ -101,7 +109,7 @@ int treeobj_validate (const json_t *obj)
                 goto inval;
         }
     }
-    else if (!strcmp (type, "symlink")) {
+    else if (streq (type, "symlink")) {
         json_t *o;
         if (!json_is_object (data))
             goto inval;
@@ -115,7 +123,7 @@ int treeobj_validate (const json_t *obj)
                 goto inval;
         }
     }
-    else if (!strcmp (type, "val")) {
+    else if (streq (type, "val")) {
         /* is base64, should always be a string */
         if (!json_is_string (data))
             goto inval;
@@ -139,31 +147,31 @@ const char *treeobj_get_type (const json_t *obj)
 bool treeobj_is_symlink (const json_t *obj)
 {
     const char *type = treeobj_get_type (obj);
-    return type && !strcmp (type, "symlink");
+    return type && streq (type, "symlink");
 }
 
 bool treeobj_is_val (const json_t *obj)
 {
     const char *type = treeobj_get_type (obj);
-    return type && !strcmp (type, "val");
+    return type && streq (type, "val");
 }
 
 bool treeobj_is_valref (const json_t *obj)
 {
     const char *type = treeobj_get_type (obj);
-    return type && !strcmp (type, "valref");
+    return type && streq (type, "valref");
 }
 
 bool treeobj_is_dir (const json_t *obj)
 {
     const char *type = treeobj_get_type (obj);
-    return type && !strcmp (type, "dir");
+    return type && streq (type, "dir");
 }
 
 bool treeobj_is_dirref (const json_t *obj)
 {
     const char *type = treeobj_get_type (obj);
-    return type && !strcmp (type, "dirref");
+    return type && streq (type, "dirref");
 }
 
 json_t *treeobj_get_data (json_t *obj)
@@ -184,7 +192,7 @@ int treeobj_get_symlink (const json_t *obj,
     const char *n_str = NULL, *t_str = NULL;
 
     if (treeobj_peek (obj, &type, &data) < 0
-            || strcmp (type, "symlink") != 0) {
+            || !streq (type, "symlink")) {
         errno = EINVAL;
         return -1;
     }
@@ -210,28 +218,27 @@ int treeobj_get_symlink (const json_t *obj,
     return 0;
 }
 
-int treeobj_decode_val (const json_t *obj, void **dp, int *lp)
+int treeobj_decode_val (const json_t *obj, void **dp, size_t *lp)
 {
     const char *type, *xdatastr;
     const json_t *xdata;
-    size_t len, xlen;
+    size_t databuflen, xlen;
+    ssize_t datalen;
     char *data;
 
     if (treeobj_peek (obj, &type, &xdata) < 0
-            || strcmp (type, "val") != 0) {
+            || !streq (type, "val")) {
         errno = EINVAL;
         return -1;
     }
     xdatastr = json_string_value (xdata);
     xlen = strlen (xdatastr);
-    len = BASE64_DECODE_SIZE (xlen) + 1; // includes space for a trailing \0
+    databuflen = base64_decoded_length (xlen) + 1; // +1  for a trailing \0
 
-    if (len > 1) {
-        if (!(data = calloc (1, len)))
+    if (databuflen > 1) {
+        if (!(data = malloc (databuflen)))
             return -1;
-        if (sodium_base642bin ((unsigned char *)data, len, xdatastr, xlen,
-                               NULL, &len, NULL,
-                               sodium_base64_VARIANT_ORIGINAL) < 0) {
+        if ((datalen = base64_decode (data, databuflen, xdatastr, xlen)) < 0) {
             free (data);
             errno = EINVAL;
             return -1;
@@ -239,10 +246,10 @@ int treeobj_decode_val (const json_t *obj, void **dp, int *lp)
     }
     else {
         data = NULL;
-        len = 0;
+        datalen = 0;
     }
     if (lp)
-        *lp = len;
+        *lp = datalen;
     if (dp)
         *dp = data;
     else
@@ -258,13 +265,13 @@ int treeobj_get_count (const json_t *obj)
 
     if (treeobj_peek (obj, &type, &data) < 0)
         goto done;
-    if (!strcmp (type, "valref") || !strcmp (type, "dirref")) {
+    if (streq (type, "valref") || streq (type, "dirref")) {
         count = json_array_size (data);
     }
-    else if (!strcmp (type, "dir")) {
+    else if (streq (type, "dir")) {
         count = json_object_size (data);
     }
-    else if (!strcmp (type, "symlink") || !strcmp (type, "val")) {
+    else if (streq (type, "symlink") || streq (type, "val")) {
         count = 1;
     } else {
         errno = EINVAL; // unknown type
@@ -280,7 +287,7 @@ json_t *treeobj_get_entry (json_t *obj, const char *name)
     json_t *data, *obj2;
 
     if (treeobj_unpack (obj, &type, &data) < 0
-            || strcmp (type, "dir") != 0) {
+        || !streq (type, "dir")) {
         errno = EINVAL;
         return NULL;
     }
@@ -297,7 +304,7 @@ int treeobj_delete_entry (json_t *obj, const char *name)
     json_t *data;
 
     if (treeobj_unpack (obj, &type, &data) < 0
-            || strcmp (type, "dir") != 0) {
+        || !streq (type, "dir")) {
         errno = EINVAL;
         return -1;
     }
@@ -313,9 +320,11 @@ int treeobj_insert_entry (json_t *obj, const char *name, json_t *obj2)
     const char *type;
     json_t *data;
 
-    if (!name || !obj2 || treeobj_unpack (obj, &type, &data) < 0
-            || strcmp (type, "dir") != 0
-            || treeobj_validate (obj2) < 0) {
+    if (!name
+        || !obj2
+        || treeobj_unpack (obj, &type, &data) < 0
+        || !streq (type, "dir")
+        || treeobj_validate (obj2) < 0) {
         errno = EINVAL;
         return -1;
     }
@@ -335,9 +344,10 @@ int treeobj_insert_entry_novalidate (json_t *obj,
     const char *type;
     json_t *data;
 
-    if (!name || !obj2 || treeobj_unpack (obj, &type, &data) < 0
-            || strcmp (type, "dir") != 0
-            || treeobj_peek (obj2, NULL, NULL) < 0) {
+    if (!name
+        || !obj2 || treeobj_unpack (obj, &type, &data) < 0
+        || !streq (type, "dir")
+        || treeobj_peek (obj2, NULL, NULL) < 0) {
         errno = EINVAL;
         return -1;
     }
@@ -354,7 +364,7 @@ const json_t *treeobj_peek_entry (const json_t *obj, const char *name)
     const json_t *data, *obj2;
 
     if (treeobj_peek (obj, &type, &data) < 0
-            || strcmp (type, "dir") != 0) {
+        || !streq (type, "dir")) {
         errno = EINVAL;
         return NULL;
     }
@@ -415,10 +425,10 @@ int treeobj_append_blobref (json_t *obj, const char *blobref)
     json_t *data, *o;
     int rc = -1;
 
-    if (!blobref || blobref_validate (blobref) < 0
-                 || treeobj_unpack (obj, &type, &data) < 0
-                 || (strcmp (type, "dirref") != 0
-                    && strcmp (type, "valref") != 0)) {
+    if (!blobref
+        || blobref_validate (blobref) < 0
+        || treeobj_unpack (obj, &type, &data) < 0
+        || (!streq (type, "dirref") && !streq (type, "valref"))) {
         errno = EINVAL;
         goto done;
     }
@@ -443,8 +453,7 @@ const char *treeobj_get_blobref (const json_t *obj, int index)
     const char *type, *blobref = NULL;
 
     if (treeobj_peek (obj, &type, &data) < 0
-            || (strcmp (type, "dirref") != 0
-                && strcmp (type, "valref") != 0)) {
+        || (!streq (type, "dirref") && !streq (type, "valref"))) {
         errno = EINVAL;
         goto done;
     }
@@ -461,9 +470,10 @@ json_t *treeobj_create_dir (void)
 {
     json_t *obj;
 
-    if (!(obj = json_pack ("{s:i s:s s:{}}", "ver", treeobj_version,
-                                            "type", "dir",
-                                            "data"))) {
+    if (!(obj = json_pack ("{s:i s:s s:{}}",
+                           "ver", treeobj_version,
+                           "type", "dir",
+                           "data"))) {
         errno = ENOMEM;
         return NULL;
     }
@@ -486,7 +496,8 @@ json_t *treeobj_create_symlink (const char *ns, const char *target)
         }
     }
     else {
-        if (!(data = json_pack ("{s:s s:s}", "namespace", ns,
+        if (!(data = json_pack ("{s:s s:s}",
+                                "namespace", ns,
                                 "target", target))) {
             errno = ENOMEM;
             return NULL;
@@ -494,9 +505,10 @@ json_t *treeobj_create_symlink (const char *ns, const char *target)
     }
 
     /* obj takes reference to "data" */
-    if (!(obj = json_pack ("{s:i s:s s:o}", "ver", treeobj_version,
-                                            "type", "symlink",
-                                            "data", data))) {
+    if (!(obj = json_pack ("{s:i s:s s:o}",
+                           "ver", treeobj_version,
+                           "type", "symlink",
+                           "data", data))) {
         json_decref (data);
         errno = ENOMEM;
         return NULL;
@@ -505,23 +517,21 @@ json_t *treeobj_create_symlink (const char *ns, const char *target)
     return obj;
 }
 
-json_t *treeobj_create_val (const void *data, int len)
+json_t *treeobj_create_val (const void *data, size_t len)
 {
-    int xlen;
+    ssize_t xlen;
     char *xdata;
     json_t *obj = NULL;
 
-    xlen = sodium_base64_encoded_len (len, sodium_base64_VARIANT_ORIGINAL);
-    if (!(xdata = malloc (xlen))) {
-        errno = ENOMEM;
+    xlen = base64_encoded_length (len) + 1; /* +1 for NUL */
+    if (!(xdata = malloc (xlen)))
         goto done;
-    }
-    sodium_bin2base64 (xdata, xlen, (const unsigned char *)data, len,
-                              sodium_base64_VARIANT_ORIGINAL);
-
-    if (!(obj = json_pack ("{s:i s:s s:s}", "ver", treeobj_version,
-                                            "type", "val",
-                                            "data", xdata))) {
+    if (base64_encode (xdata, xlen, (char *)data, len) < 0)
+        goto done;
+    if (!(obj = json_pack ("{s:i s:s s:s}",
+                           "ver", treeobj_version,
+                           "type", "val",
+                           "data", xdata))) {
         errno = ENOMEM;
         goto done;
     }
@@ -534,14 +544,18 @@ json_t *treeobj_create_valref (const char *blobref)
 {
     json_t *obj;
 
-    if (blobref)
-        obj = json_pack ("{s:i s:s s:[s]}", "ver", treeobj_version,
-                                            "type", "valref",
-                                            "data", blobref);
-    else
-        obj = json_pack ("{s:i s:s s:[]}", "ver", treeobj_version,
-                                            "type", "valref",
-                                            "data");
+    if (blobref) {
+        obj = json_pack ("{s:i s:s s:[s]}",
+                         "ver", treeobj_version,
+                         "type", "valref",
+                         "data", blobref);
+    }
+    else {
+        obj = json_pack ("{s:i s:s s:[]}",
+                         "ver", treeobj_version,
+                         "type", "valref",
+                         "data");
+    }
     if (!obj) {
         errno = ENOMEM;
         return NULL;
@@ -553,14 +567,18 @@ json_t *treeobj_create_dirref (const char *blobref)
 {
     json_t *obj;
 
-    if (blobref)
-        obj = json_pack ("{s:i s:s s:[s]}", "ver", treeobj_version,
-                                            "type", "dirref",
-                                            "data", blobref);
-    else
-        obj = json_pack ("{s:i s:s s:[]}", "ver", treeobj_version,
-                                            "type", "dirref",
-                                            "data");
+    if (blobref) {
+        obj = json_pack ("{s:i s:s s:[s]}",
+                         "ver", treeobj_version,
+                         "type", "dirref",
+                         "data", blobref);
+    }
+    else {
+        obj = json_pack ("{s:i s:s s:[]}",
+                         "ver", treeobj_version,
+                         "type", "dirref",
+                         "data");
+    }
     if (!obj) {
         errno = ENOMEM;
         return NULL;
@@ -568,8 +586,10 @@ json_t *treeobj_create_dirref (const char *blobref)
     return obj;
 }
 
-json_t *treeobj_create_valref_buf (const char *hashtype, int maxblob,
-                                   void *data, int len)
+json_t *treeobj_create_valref_buf (const char *hashtype,
+                                   int maxblob,
+                                   void *data,
+                                   int len)
 {
     json_t *valref = NULL;
     char blobref[BLOBREF_MAX_STRING_SIZE];
@@ -581,7 +601,10 @@ json_t *treeobj_create_valref_buf (const char *hashtype, int maxblob,
         blob_len = len;
         if (maxblob > 0 && len > maxblob)
             blob_len = maxblob;
-        if (blobref_hash (hashtype, data, blob_len, blobref,
+        if (blobref_hash (hashtype,
+                          data,
+                          blob_len,
+                          blobref,
                           sizeof (blobref)) < 0)
             goto error;
         if (treeobj_append_blobref (valref, blobref) < 0)
@@ -610,7 +633,7 @@ json_t *treeobj_decodeb (const char *buf, size_t buflen)
 {
     json_t *obj = NULL;
     if (!(obj = json_loadb (buf, buflen, 0, NULL))
-            || treeobj_validate (obj) < 0) {
+        || treeobj_validate (obj) < 0) {
         errno = EPROTO;
         goto error;
     }
@@ -625,6 +648,21 @@ char *treeobj_encode (const json_t *obj)
     return json_dumps (obj, JSON_COMPACT|JSON_SORT_KEYS);
 }
 
+const char *treeobj_type_name (const json_t *obj)
+{
+    if (treeobj_is_symlink (obj))
+        return "symlink";
+    else if (treeobj_is_val (obj))
+        return "val";
+    else if (treeobj_is_valref (obj))
+        return "valref";
+    else if (treeobj_is_dir (obj))
+        return "dir";
+    else if (treeobj_is_dirref (obj))
+        return "dirref";
+    return "unknown";
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libkvs/treeobj.h b/src/common/libkvs/treeobj.h
index 6c887d2990ef..f0ec1c01ee0a 100644
--- a/src/common/libkvs/treeobj.h
+++ b/src/common/libkvs/treeobj.h
@@ -24,7 +24,7 @@
  * Return JSON object on success, NULL on failure with errno set.
  */
 json_t *treeobj_create_symlink (const char *ns, const char *target);
-json_t *treeobj_create_val (const void *data, int len);
+json_t *treeobj_create_val (const void *data, size_t len);
 json_t *treeobj_create_valref (const char *blobref);
 json_t *treeobj_create_dir (void);
 json_t *treeobj_create_dirref (const char *blobref);
@@ -67,7 +67,7 @@ int treeobj_get_symlink (const json_t *obj,
  * If len > 0, data will be followed by an extra NULL byte in memory.
  * Caller must free returned data.
  */
-int treeobj_decode_val (const json_t *obj, void **data, int *len);
+int treeobj_decode_val (const json_t *obj, void **data, size_t *len);
 
 /* get type-specific count.
  * For dirref/valref, this is the number of blobrefs.
@@ -78,7 +78,7 @@ int treeobj_decode_val (const json_t *obj, void **data, int *len);
 int treeobj_get_count (const json_t *obj);
 
 /* get/add/remove directory entry
- * Get returns JSON object (owned by 'obj', do not destory), NULL on error.
+ * Get returns JSON object (owned by 'obj', do not destroy), NULL on error.
  * insert takes a reference on 'obj2' (caller retains ownership).
  * insert/delete return 0 on success, -1 on error with errno set.
  */
@@ -125,8 +125,10 @@ const char *treeobj_get_blobref (const json_t *obj, int index);
  * 'hashtype' hash algorithm (e.g. "sha1").  If 'maxblob' > 0, split the
  * blob into maxblob size chunks.
  */
-json_t *treeobj_create_valref_buf (const char *hashtype, int maxblob,
-                                   void *data, int len);
+json_t *treeobj_create_valref_buf (const char *hashtype,
+                                   int maxblob,
+                                   void *data,
+                                   int len);
 
 /* Convert a treeobj to/from string.
  * The return value of treeobj_decode must be destroyed with json_decref().
@@ -136,6 +138,12 @@ json_t *treeobj_decode (const char *buf);
 json_t *treeobj_decodeb (const char *buf, size_t buflen);
 char *treeobj_encode (const json_t *obj);
 
+/* Get treeobj type name
+ * Returns "symlink", "val", "valref", "dir", "dirref" or NULL if
+ * invalid treeobj.
+ */
+const char *treeobj_type_name (const json_t *obj);
+
 #endif /* !_FLUX_KVS_TREEOBJ_H */
 
 /*
diff --git a/src/common/liblsd/Makefile.am b/src/common/liblsd/Makefile.am
index b91c78fdafc4..cd7eeb3ca3a9 100644
--- a/src/common/liblsd/Makefile.am
+++ b/src/common/liblsd/Makefile.am
@@ -1,18 +1,18 @@
 AM_CFLAGS = \
 	$(WARNING_CFLAGS) \
 	$(CODE_COVERAGE_CFLAGS) \
-	-Wno-parentheses -Wno-error=parentheses
+	-Wno-parentheses -Wno-error=parentheses \
+	-Wno-error=unused-but-set-variable \
+	-Wno-unused-but-set-variable
 
 AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
-	-DWITH_PTHREADS
+	$(CODE_COVERAGE_CPPFLAGS)
 
 noinst_LTLIBRARIES = liblsd.la
 
 liblsd_la_SOURCES = \
-	list.c \
-	list.h \
 	cbuf.c \
 	cbuf.h
diff --git a/src/common/liblsd/list.c b/src/common/liblsd/list.c
deleted file mode 100644
index 28b70547977f..000000000000
--- a/src/common/liblsd/list.c
+++ /dev/null
@@ -1,837 +0,0 @@
-/*****************************************************************************
- *  Written by Chris Dunlap <cdunlap@llnl.gov>.
- *  Copyright (C) 2007-2018 Lawrence Livermore National Security, LLC.
- *  Copyright (C) 2001-2007 The Regents of the University of California.
- *  UCRL-CODE-2002-009.
- *
- *  This file is part of ConMan: The Console Manager.
- *  For details, see <https://dun.github.io/conman/>.
- *
- *  ConMan is free software: you can redistribute it and/or modify it under
- *  the terms of the GNU General Public License as published by the Free
- *  Software Foundation, either version 3 of the License, or (at your option)
- *  any later version.
- *
- *  ConMan is distributed in the hope that it will be useful, but WITHOUT
- *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- *  for more details.
- *
- *  You should have received a copy of the GNU General Public License along
- *  with ConMan.  If not, see <http://www.gnu.org/licenses/>.
- *
- *  Additional permission under GNU GPL version 3 section 7:
- *
- *  The licensors of ConMan grant you additional permission to modify or
- *  combine with this file and convey/distribute the resulting work under
- *  the terms of the GNU Lesser General Public License as published by the
- *  Free Software Foundation, either version 2.1 of the License, or
- *  (at your option) any later version.
- *****************************************************************************/
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif /* HAVE_CONFIG_H */
-
-#ifdef WITH_PTHREADS
-#  include <pthread.h>
-#endif /* WITH_PTHREADS */
-
-#include <assert.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <string.h>
-#include "list.h"
-
-
-/*********************
- *  lsd_fatal_error  *
- *********************/
-
-#ifdef WITH_LSD_FATAL_ERROR_FUNC
-#  undef lsd_fatal_error
-   extern void lsd_fatal_error(char *file, int line, char *mesg);
-#else /* !WITH_LSD_FATAL_ERROR_FUNC */
-#  ifndef lsd_fatal_error
-#    include <errno.h>
-#    include <stdio.h>
-#    include <string.h>
-#    define lsd_fatal_error(file, line, mesg)                                 \
-       do {                                                                   \
-           fprintf(stderr, "ERROR: [%s:%d] %s: %s\n",                         \
-                   file, line, mesg, strerror(errno));                        \
-       } while (0)
-#  endif /* !lsd_fatal_error */
-#endif /* !WITH_LSD_FATAL_ERROR_FUNC */
-
-
-/*********************
- *  lsd_nomem_error  *
- *********************/
-
-#ifdef WITH_LSD_NOMEM_ERROR_FUNC
-#  undef lsd_nomem_error
-   extern void * lsd_nomem_error(char *file, int line, char *mesg);
-#else /* !WITH_LSD_NOMEM_ERROR_FUNC */
-#  ifndef lsd_nomem_error
-#    define lsd_nomem_error(file, line, mesg) (NULL)
-#  endif /* !lsd_nomem_error */
-#endif /* !WITH_LSD_NOMEM_ERROR_FUNC */
-
-
-/***************
- *  Constants  *
- ***************/
-
-#define LIST_ALLOC 32
-#define LIST_MAGIC 0xDEADBEEF
-
-
-/****************
- *  Data Types  *
- ****************/
-
-struct listNode {
-    void                 *data;         /* node's data                       */
-    struct listNode      *next;         /* next node in list                 */
-};
-
-struct listIterator {
-    struct list          *list;         /* the list being iterated           */
-    struct listNode      *pos;          /* the next node to be iterated      */
-    struct listNode     **prev;         /* addr of 'next' ptr to prv It node */
-    struct listIterator  *iNext;        /* iterator chain for list_destroy() */
-#ifndef NDEBUG
-    unsigned int          magic;        /* sentinel for asserting validity   */
-#endif /* !NDEBUG */
-};
-
-struct list {
-    struct listNode      *head;         /* head of the list                  */
-    struct listNode     **tail;         /* addr of last node's 'next' ptr    */
-    struct listIterator  *iNext;        /* iterator chain for list_destroy() */
-    ListDelF              fDel;         /* function to delete node data      */
-    int                   count;        /* number of nodes in list           */
-#ifdef WITH_PTHREADS
-    pthread_mutex_t       mutex;        /* mutex to protect access to list   */
-#endif /* WITH_PTHREADS */
-#ifndef NDEBUG
-    unsigned int          magic;        /* sentinel for asserting validity   */
-#endif /* !NDEBUG */
-};
-
-typedef struct listNode * ListNode;
-
-
-/****************
- *  Prototypes  *
- ****************/
-
-static void * list_node_create (List l, ListNode *pp, void *x);
-static void * list_node_destroy (List l, ListNode *pp);
-static List list_alloc (void);
-static void list_free (List l);
-static ListNode list_node_alloc (void);
-static void list_node_free (ListNode p);
-static ListIterator list_iterator_alloc (void);
-static void list_iterator_free (ListIterator i);
-static void * list_alloc_aux (int size, void *pfreelist);
-static void list_free_aux (void *x, void *pfreelist);
-
-
-/***************
- *  Variables  *
- ***************/
-
-static List list_free_lists = NULL;
-static ListNode list_free_nodes = NULL;
-static ListIterator list_free_iterators = NULL;
-
-#ifdef WITH_PTHREADS
-static pthread_mutex_t list_free_lock = PTHREAD_MUTEX_INITIALIZER;
-#endif /* WITH_PTHREADS */
-
-
-/************
- *  Macros  *
- ************/
-
-#ifdef WITH_PTHREADS
-
-#  define list_mutex_init(mutex)                                              \
-     do {                                                                     \
-         int e = pthread_mutex_init(mutex, NULL);                             \
-         if (e != 0) {                                                        \
-             errno = e;                                                       \
-             lsd_fatal_error(__FILE__, __LINE__, "list mutex init");          \
-             abort();                                                         \
-         }                                                                    \
-     } while (0)
-
-#  define list_mutex_lock(mutex)                                              \
-     do {                                                                     \
-         int e = pthread_mutex_lock(mutex);                                   \
-         if (e != 0) {                                                        \
-             errno = e;                                                       \
-             lsd_fatal_error(__FILE__, __LINE__, "list mutex lock");          \
-             abort();                                                         \
-         }                                                                    \
-     } while (0)
-
-#  define list_mutex_unlock(mutex)                                            \
-     do {                                                                     \
-         int e = pthread_mutex_unlock(mutex);                                 \
-         if (e != 0) {                                                        \
-             errno = e;                                                       \
-             lsd_fatal_error(__FILE__, __LINE__, "list mutex unlock");        \
-             abort();                                                         \
-         }                                                                    \
-     } while (0)
-
-#  define list_mutex_destroy(mutex)                                           \
-     do {                                                                     \
-         int e = pthread_mutex_destroy(mutex);                                \
-         if (e != 0) {                                                        \
-             errno = e;                                                       \
-             lsd_fatal_error(__FILE__, __LINE__, "list mutex destroy");       \
-             abort();                                                         \
-         }                                                                    \
-     } while (0)
-
-#  ifndef NDEBUG
-     static int list_mutex_is_locked (pthread_mutex_t *mutex);
-#  endif /* !NDEBUG */
-
-#else /* !WITH_PTHREADS */
-
-#  define list_mutex_init(mutex)
-#  define list_mutex_lock(mutex)
-#  define list_mutex_unlock(mutex)
-#  define list_mutex_destroy(mutex)
-#  define list_mutex_is_locked(mutex) (1)
-
-#endif /* !WITH_PTHREADS */
-
-
-/***************
- *  Functions  *
- ***************/
-
-List
-list_create (ListDelF f)
-{
-    List l;
-
-    if (!(l = list_alloc()))
-        return(lsd_nomem_error(__FILE__, __LINE__, "list create"));
-    l->head = NULL;
-    l->tail = &l->head;
-    l->iNext = NULL;
-    l->fDel = f;
-    l->count = 0;
-    list_mutex_init(&l->mutex);
-    assert(l->magic = LIST_MAGIC);      /* set magic via assert abuse */
-    return(l);
-}
-
-
-void
-list_destroy (List l)
-{
-    ListIterator i, iTmp;
-    ListNode p, pTmp;
-
-    assert(l != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    i = l->iNext;
-    while (i) {
-        assert(i->magic == LIST_MAGIC);
-        iTmp = i->iNext;
-        assert(i->magic = ~LIST_MAGIC); /* clear magic via assert abuse */
-        list_iterator_free(i);
-        i = iTmp;
-    }
-    p = l->head;
-    while (p) {
-        pTmp = p->next;
-        if (p->data && l->fDel)
-            l->fDel(p->data);
-        list_node_free(p);
-        p = pTmp;
-    }
-    assert(l->magic = ~LIST_MAGIC);     /* clear magic via assert abuse */
-    list_mutex_unlock(&l->mutex);
-    list_mutex_destroy(&l->mutex);
-    list_free(l);
-    return;
-}
-
-
-int
-list_is_empty (List l)
-{
-    int n;
-
-    assert(l != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    n = l->count;
-    list_mutex_unlock(&l->mutex);
-    return(n == 0);
-}
-
-
-int
-list_count (List l)
-{
-    int n;
-
-    assert(l != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    n = l->count;
-    list_mutex_unlock(&l->mutex);
-    return(n);
-}
-
-
-void *
-list_append (List l, void *x)
-{
-    void *v;
-
-    assert(l != NULL);
-    assert(x != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    v = list_node_create(l, l->tail, x);
-    list_mutex_unlock(&l->mutex);
-    return(v);
-}
-
-
-void *
-list_prepend (List l, void *x)
-{
-    void *v;
-
-    assert(l != NULL);
-    assert(x != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    v = list_node_create(l, &l->head, x);
-    list_mutex_unlock(&l->mutex);
-    return(v);
-}
-
-
-void *
-list_find_first (List l, ListFindF f, void *key)
-{
-    ListNode p;
-    void *v = NULL;
-
-    assert(l != NULL);
-    assert(f != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    for (p=l->head; p; p=p->next) {
-        if (f(p->data, key)) {
-            v = p->data;
-            break;
-        }
-    }
-    list_mutex_unlock(&l->mutex);
-    return(v);
-}
-
-
-int
-list_delete_all (List l, ListFindF f, void *key)
-{
-    ListNode *pp;
-    void *v;
-    int n = 0;
-
-    assert(l != NULL);
-    assert(f != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    pp = &l->head;
-    while (*pp) {
-        if (f((*pp)->data, key)) {
-            if ((v = list_node_destroy(l, pp))) {
-                if (l->fDel)
-                    l->fDel(v);
-                n++;
-            }
-        }
-        else {
-            pp = &(*pp)->next;
-        }
-    }
-    list_mutex_unlock(&l->mutex);
-    return(n);
-}
-
-
-int
-list_for_each (List l, ListForF f, void *arg)
-{
-    ListNode p;
-    int n = 0;
-
-    assert(l != NULL);
-    assert(f != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    for (p=l->head; p; p=p->next) {
-        n++;
-        if (f(p->data, arg) < 0) {
-            n = -n;
-            break;
-        }
-    }
-    list_mutex_unlock(&l->mutex);
-    return(n);
-}
-
-
-void
-list_sort (List l, ListCmpF f)
-{
-/*  Note: Time complexity O(n^2).
- */
-    ListNode *pp, *ppPrev, *ppPos, pTmp;
-    ListIterator i;
-
-    assert(l != NULL);
-    assert(f != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    if (l->count > 1) {
-        ppPrev = &l->head;
-        pp = &(*ppPrev)->next;
-        while (*pp) {
-            if (f((*pp)->data, (*ppPrev)->data) < 0) {
-                ppPos = &l->head;
-                while (f((*pp)->data, (*ppPos)->data) >= 0)
-                    ppPos = &(*ppPos)->next;
-                pTmp = (*pp)->next;
-                (*pp)->next = *ppPos;
-                *ppPos = *pp;
-                *pp = pTmp;
-                if (ppPrev == ppPos)
-                    ppPrev = &(*ppPrev)->next;
-            }
-            else {
-                ppPrev = pp;
-                pp = &(*pp)->next;
-            }
-        }
-        l->tail = pp;
-
-        for (i=l->iNext; i; i=i->iNext) {
-            assert(i->magic == LIST_MAGIC);
-            i->pos = i->list->head;
-            i->prev = &i->list->head;
-        }
-    }
-    list_mutex_unlock(&l->mutex);
-    return;
-}
-
-
-void *
-list_push (List l, void *x)
-{
-    void *v;
-
-    assert(l != NULL);
-    assert(x != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    v = list_node_create(l, &l->head, x);
-    list_mutex_unlock(&l->mutex);
-    return(v);
-}
-
-
-void *
-list_pop (List l)
-{
-    void *v;
-
-    assert(l != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    v = list_node_destroy(l, &l->head);
-    list_mutex_unlock(&l->mutex);
-    return(v);
-}
-
-
-void *
-list_peek (List l)
-{
-    void *v;
-
-    assert(l != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    v = (l->head) ? l->head->data : NULL;
-    list_mutex_unlock(&l->mutex);
-    return(v);
-}
-
-
-void *
-list_enqueue (List l, void *x)
-{
-    void *v;
-
-    assert(l != NULL);
-    assert(x != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    v = list_node_create(l, l->tail, x);
-    list_mutex_unlock(&l->mutex);
-    return(v);
-}
-
-
-void *
-list_dequeue (List l)
-{
-    void *v;
-
-    assert(l != NULL);
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    v = list_node_destroy(l, &l->head);
-    list_mutex_unlock(&l->mutex);
-    return(v);
-}
-
-
-ListIterator
-list_iterator_create (List l)
-{
-    ListIterator i;
-
-    assert(l != NULL);
-    if (!(i = list_iterator_alloc()))
-        return(lsd_nomem_error(__FILE__, __LINE__, "list iterator create"));
-    i->list = l;
-    list_mutex_lock(&l->mutex);
-    assert(l->magic == LIST_MAGIC);
-    i->pos = l->head;
-    i->prev = &l->head;
-    i->iNext = l->iNext;
-    l->iNext = i;
-    assert(i->magic = LIST_MAGIC);      /* set magic via assert abuse */
-    list_mutex_unlock(&l->mutex);
-    return(i);
-}
-
-
-void
-list_iterator_reset (ListIterator i)
-{
-    assert(i != NULL);
-    assert(i->magic == LIST_MAGIC);
-    list_mutex_lock(&i->list->mutex);
-    assert(i->list->magic == LIST_MAGIC);
-    i->pos = i->list->head;
-    i->prev = &i->list->head;
-    list_mutex_unlock(&i->list->mutex);
-    return;
-}
-
-
-void
-list_iterator_destroy (ListIterator i)
-{
-    ListIterator *pi;
-
-    assert(i != NULL);
-    assert(i->magic == LIST_MAGIC);
-    list_mutex_lock(&i->list->mutex);
-    assert(i->list->magic == LIST_MAGIC);
-    for (pi=&i->list->iNext; *pi; pi=&(*pi)->iNext) {
-        assert((*pi)->magic == LIST_MAGIC);
-        if (*pi == i) {
-            *pi = (*pi)->iNext;
-            break;
-        }
-    }
-    list_mutex_unlock(&i->list->mutex);
-    assert(i->magic = ~LIST_MAGIC);     /* clear magic via assert abuse */
-    list_iterator_free(i);
-    return;
-}
-
-
-void *
-list_next (ListIterator i)
-{
-    ListNode p;
-
-    assert(i != NULL);
-    assert(i->magic == LIST_MAGIC);
-    list_mutex_lock(&i->list->mutex);
-    assert(i->list->magic == LIST_MAGIC);
-    if ((p = i->pos))
-        i->pos = p->next;
-    if (*i->prev != p)
-        i->prev = &(*i->prev)->next;
-    list_mutex_unlock(&i->list->mutex);
-    return(p ? p->data : NULL);
-}
-
-
-void *
-list_insert (ListIterator i, void *x)
-{
-    void *v;
-
-    assert(i != NULL);
-    assert(x != NULL);
-    assert(i->magic == LIST_MAGIC);
-    list_mutex_lock(&i->list->mutex);
-    assert(i->list->magic == LIST_MAGIC);
-    v = list_node_create(i->list, i->prev, x);
-    list_mutex_unlock(&i->list->mutex);
-    return(v);
-}
-
-
-void *
-list_find (ListIterator i, ListFindF f, void *key)
-{
-    void *v;
-
-    assert(i != NULL);
-    assert(f != NULL);
-    assert(i->magic == LIST_MAGIC);
-    while ((v=list_next(i)) && !f(v,key)) {;}
-    return(v);
-}
-
-
-void *
-list_remove (ListIterator i)
-{
-    void *v = NULL;
-
-    assert(i != NULL);
-    assert(i->magic == LIST_MAGIC);
-    list_mutex_lock(&i->list->mutex);
-    assert(i->list->magic == LIST_MAGIC);
-    if (*i->prev != i->pos)
-        v = list_node_destroy(i->list, i->prev);
-    list_mutex_unlock(&i->list->mutex);
-    return(v);
-}
-
-
-int
-list_delete (ListIterator i)
-{
-    void *v;
-
-    assert(i != NULL);
-    assert(i->magic == LIST_MAGIC);
-    if ((v = list_remove(i))) {
-        if (i->list->fDel)
-            i->list->fDel(v);
-        return(1);
-    }
-    return(0);
-}
-
-
-static void *
-list_node_create (List l, ListNode *pp, void *x)
-{
-/*  Inserts data pointed to by [x] into list [l] after [pp],
- *    the address of the previous node's "next" ptr.
- *  Returns a ptr to data [x], or NULL if insertion fails.
- *  This routine assumes the list is already locked upon entry.
- */
-    ListNode p;
-    ListIterator i;
-
-    assert(l != NULL);
-    assert(l->magic == LIST_MAGIC);
-    assert(list_mutex_is_locked(&l->mutex));
-    assert(pp != NULL);
-    assert(x != NULL);
-    if (!(p = list_node_alloc()))
-        return(lsd_nomem_error(__FILE__, __LINE__, "list node create"));
-    p->data = x;
-    if (!(p->next = *pp))
-        l->tail = &p->next;
-    *pp = p;
-    l->count++;
-    for (i=l->iNext; i; i=i->iNext) {
-        assert(i->magic == LIST_MAGIC);
-        if (i->prev == pp)
-            i->prev = &p->next;
-        else if (i->pos == p->next)
-            i->pos = p;
-        assert((i->pos == *i->prev) || (i->pos == (*i->prev)->next));
-    }
-    return(x);
-}
-
-
-static void *
-list_node_destroy (List l, ListNode *pp)
-{
-/*  Removes the node pointed to by [*pp] from from list [l],
- *    where [pp] is the address of the previous node's "next" ptr.
- *  Returns the data ptr associated with list item being removed,
- *    or NULL if [*pp] points to the NULL element.
- *  This routine assumes the list is already locked upon entry.
- */
-    void *v;
-    ListNode p;
-    ListIterator i;
-
-    assert(l != NULL);
-    assert(l->magic == LIST_MAGIC);
-    assert(list_mutex_is_locked(&l->mutex));
-    assert(pp != NULL);
-    if (!(p = *pp))
-        return(NULL);
-    v = p->data;
-    if (!(*pp = p->next))
-        l->tail = pp;
-    l->count--;
-    for (i=l->iNext; i; i=i->iNext) {
-        assert(i->magic == LIST_MAGIC);
-        if (i->pos == p)
-            i->pos = p->next, i->prev = pp;
-        else if (i->prev == &p->next)
-            i->prev = pp;
-        assert((i->pos == *i->prev) || (i->pos == (*i->prev)->next));
-    }
-    list_node_free(p);
-    return(v);
-}
-
-
-static List
-list_alloc (void)
-{
-    return(list_alloc_aux(sizeof(struct list), &list_free_lists));
-}
-
-
-static void
-list_free (List l)
-{
-    list_free_aux(l, &list_free_lists);
-    return;
-}
-
-
-static ListNode
-list_node_alloc (void)
-{
-    return(list_alloc_aux(sizeof(struct listNode), &list_free_nodes));
-}
-
-
-static void
-list_node_free (ListNode p)
-{
-    list_free_aux(p, &list_free_nodes);
-    return;
-}
-
-
-static ListIterator
-list_iterator_alloc (void)
-{
-    return(list_alloc_aux(sizeof(struct listIterator), &list_free_iterators));
-}
-
-
-static void
-list_iterator_free (ListIterator i)
-{
-    list_free_aux(i, &list_free_iterators);
-    return;
-}
-
-
-static void *
-list_alloc_aux (int size, void *pfreelist)
-{
-/*  Allocates an object of [size] bytes from the freelist [*pfreelist].
- *  Memory is added to the freelist in chunks of size LIST_ALLOC.
- *  Returns a ptr to the object, or NULL if the memory request fails.
- */
-    void **px;
-    void **pfree = pfreelist;
-    void **plast;
-
-    assert(sizeof(char) == 1);
-    assert(size >= sizeof(void *));
-    assert(pfreelist != NULL);
-    assert(LIST_ALLOC > 0);
-    list_mutex_lock(&list_free_lock);
-    if (!*pfree) {
-        if ((*pfree = malloc(LIST_ALLOC * size))) {
-            px = *pfree;
-            plast = (void **) ((char *) *pfree + ((LIST_ALLOC - 1) * size));
-            while (px < plast)
-                *px = (char *) px + size, px = *px;
-            *plast = NULL;
-        }
-    }
-    if ((px = *pfree))
-        *pfree = *px;
-    else
-        errno = ENOMEM;
-    list_mutex_unlock(&list_free_lock);
-    return(px);
-}
-
-
-static void
-list_free_aux (void *x, void *pfreelist)
-{
-/*  Frees the object [x], returning it to the freelist [*pfreelist].
- */
-    void **px = x;
-    void **pfree = pfreelist;
-
-    assert(x != NULL);
-    assert(pfreelist != NULL);
-    list_mutex_lock(&list_free_lock);
-    *px = *pfree;
-    *pfree = px;
-    list_mutex_unlock(&list_free_lock);
-    return;
-}
-
-
-#ifndef NDEBUG
-#ifdef WITH_PTHREADS
-static int
-list_mutex_is_locked (pthread_mutex_t *mutex)
-{
-/*  Returns true if the mutex is locked; o/w, returns false.
- */
-    int rc;
-
-    assert(mutex != NULL);
-    rc = pthread_mutex_trylock(mutex);
-    return(rc == EBUSY ? 1 : 0);
-}
-#endif /* WITH_PTHREADS */
-#endif /* !NDEBUG */
diff --git a/src/common/liblsd/list.h b/src/common/liblsd/list.h
deleted file mode 100644
index 511113ff5708..000000000000
--- a/src/common/liblsd/list.h
+++ /dev/null
@@ -1,287 +0,0 @@
-/*****************************************************************************
- *  Written by Chris Dunlap <cdunlap@llnl.gov>.
- *  Copyright (C) 2007-2018 Lawrence Livermore National Security, LLC.
- *  Copyright (C) 2001-2007 The Regents of the University of California.
- *  UCRL-CODE-2002-009.
- *
- *  This file is part of ConMan: The Console Manager.
- *  For details, see <https://dun.github.io/conman/>.
- *
- *  ConMan is free software: you can redistribute it and/or modify it under
- *  the terms of the GNU General Public License as published by the Free
- *  Software Foundation, either version 3 of the License, or (at your option)
- *  any later version.
- *
- *  ConMan is distributed in the hope that it will be useful, but WITHOUT
- *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- *  for more details.
- *
- *  You should have received a copy of the GNU General Public License along
- *  with ConMan.  If not, see <http://www.gnu.org/licenses/>.
- *
- *  Additional permission under GNU GPL version 3 section 7:
- *
- *  The licensors of ConMan grant you additional permission to modify or
- *  combine with this file and convey/distribute the resulting work under
- *  the terms of the GNU Lesser General Public License as published by the
- *  Free Software Foundation, either version 2.1 of the License, or
- *  (at your option) any later version.
- *****************************************************************************/
-
-#ifndef LSD_LIST_H
-#define LSD_LIST_H
-
-
-/***********
- *  Notes  *
- ***********/
-/*
- *  If NDEBUG is not defined, internal debug code will be enabled.  This is
- *  intended for development use only and production code should define NDEBUG.
- *
- *  If WITH_LSD_FATAL_ERROR_FUNC is defined, the linker will expect to
- *  find an external lsd_fatal_error(file,line,mesg) function.  By default,
- *  lsd_fatal_error(file,line,mesg) is a macro definition that outputs an
- *  error message to stderr.  This macro may be redefined to invoke another
- *  routine instead.
- *
- *  If WITH_LSD_NOMEM_ERROR_FUNC is defined, the linker will expect to
- *  find an external lsd_nomem_error(file,line,mesg) function.  By default,
- *  lsd_nomem_error(file,line,mesg) is a macro definition that returns NULL.
- *  This macro may be redefined to invoke another routine instead.
- *
- *  If WITH_PTHREADS is defined, these routines will be thread-safe.
- */
-
-
-/****************
- *  Data Types  *
- ****************/
-
-typedef struct list * List;
-/*
- *  List opaque data type.
- */
-
-typedef struct listIterator * ListIterator;
-/*
- *  List Iterator opaque data type.
- */
-
-typedef void (*ListDelF) (void *x);
-/*
- *  Function prototype to deallocate data stored in a list.
- *    This function is responsible for freeing all memory associated
- *    with an item, including all subordinate items (if applicable).
- */
-
-typedef int (*ListCmpF) (void *x, void *y);
-/*
- *  Function prototype for comparing two items in a list.
- *  Returns less-than-zero if (x<y), zero if (x==y), and
- *    greather-than-zero if (x>y).
- */
-
-typedef int (*ListFindF) (void *x, void *key);
-/*
- *  Function prototype for matching items in a list.
- *  Returns non-zero if (x==key); o/w returns zero.
- */
-
-typedef int (*ListForF) (void *x, void *arg);
-/*
- *  Function prototype for operating on each item in a list.
- *  Returns less-than-zero on error.
- */
-
-
-/*******************************
- *  General-Purpose Functions  *
- *******************************/
-
-List list_create (ListDelF f);
-/*
- *  Creates and returns a new empty list, or lsd_nomem_error() on failure.
- *  The deletion function [f] is used to deallocate memory used by items
- *    in the list; if this is NULL, memory associated with these items
- *    will not be freed when the list is destroyed.
- *  Note: Abandoning a list without calling list_destroy() will result
- *    in a memory leak.
- */
-
-void list_destroy (List l);
-/*
- *  Destroys list [l], freeing memory used for list iterators and the
- *    list itself; if a deletion function was specified when the list
- *    was created, it will be called for each item in the list.
- */
-
-int list_is_empty (List l);
-/*
- *  Returns non-zero if list [l] is empty; o/w returns zero.
- */
-
-int list_count (List l);
-/*
- *  Returns the number of items in list [l].
- */
-
-
-/***************************
- *  List Access Functions  *
- ***************************/
-
-void * list_append (List l, void *x);
-/*
- *  Inserts data [x] at the end of list [l].
- *  Returns the data's ptr, or lsd_nomem_error() if insertion failed.
- */
-
-void * list_prepend (List l, void *x);
-/*
- *  Inserts data [x] at the beginning of list [l].
- *  Returns the data's ptr, or lsd_nomem_error() if insertion failed.
- */
-
-void * list_find_first (List l, ListFindF f, void *key);
-/*
- *  Traverses list [l] using [f] to match each item with [key].
- *  Returns a ptr to the first item for which the function [f]
- *    returns non-zero, or NULL if no such item is found.
- *  Note: This function differs from list_find() in that it does not require
- *    a list iterator; it should only be used when all list items are known
- *    to be unique (according to the function [f]).
- */
-
-int list_delete_all (List l, ListFindF f, void *key);
-/*
- *  Traverses list [l] using [f] to match each item with [key].
- *  Removes all items from the list for which the function [f] returns
- *    non-zero; if a deletion function was specified when the list was
- *    created, it will be called to deallocate each item being removed.
- *  Returns a count of the number of items removed from the list.
- */
-
-int list_for_each (List l, ListForF f, void *arg);
-/*
- *  For each item in list [l], invokes the function [f] with [arg].
- *  Returns a count of the number of items on which [f] was invoked.
- *  If [f] returns <0 for a given item, the iteration is aborted and the
- *    function returns the negative of that item's position in the list.
- */
-
-void list_sort (List l, ListCmpF f);
-/*
- *  Sorts list [l] into ascending order according to the function [f].
- *  Note: Sorting a list resets all iterators associated with the list.
- *  Note: The sort algorithm is stable.
- */
-
-
-/****************************
- *  Stack Access Functions  *
- ****************************/
-
-void * list_push (List l, void *x);
-/*
- *  Pushes data [x] onto the top of stack [l].
- *  Returns the data's ptr, or lsd_nomem_error() if insertion failed.
- */
-
-void * list_pop (List l);
-/*
- *  Pops the data item at the top of the stack [l].
- *  Returns the data's ptr, or NULL if the stack is empty.
- */
-
-void * list_peek (List l);
-/*
- *  Peeks at the data item at the top of the stack (or head of the queue) [l].
- *  Returns the data's ptr, or NULL if the stack (or queue) is empty.
- *  Note: The item is not removed from the list.
- */
-
-
-/****************************
- *  Queue Access Functions  *
- ****************************/
-
-void * list_enqueue (List l, void *x);
-/*
- *  Enqueues data [x] at the tail of queue [l].
- *  Returns the data's ptr, or lsd_nomem_error() if insertion failed.
- */
-
-void * list_dequeue (List l);
-/*
- *  Dequeues the data item at the head of the queue [l].
- *  Returns the data's ptr, or NULL if the queue is empty.
- */
-
-
-/*****************************
- *  List Iterator Functions  *
- *****************************/
-
-ListIterator list_iterator_create (List l);
-/*
- *  Creates and returns a list iterator for non-destructively traversing
- *    list [l], or lsd_nomem_error() on failure.
- */
-
-void list_iterator_reset (ListIterator i);
-/*
- *  Resets the list iterator [i] to start traversal at the beginning
- *    of the list.
- */
-
-void list_iterator_destroy (ListIterator i);
-/*
- *  Destroys the list iterator [i]; list iterators not explicitly destroyed
- *    in this manner will be destroyed when the list is deallocated via
- *    list_destroy().
- */
-
-void * list_next (ListIterator i);
-/*
- *  Returns a ptr to the next item's data,
- *    or NULL once the end of the list is reached.
- *  Example: i=list_iterator_create(i); while ((x=list_next(i))) {...}
- */
-
-void * list_insert (ListIterator i, void *x);
-/*
- *  Inserts data [x] immediately before the last item returned via list
- *    iterator [i]; once the list iterator reaches the end of the list,
- *    insertion is made at the list's end.
- *  Returns the data's ptr, or lsd_nomem_error() if insertion failed.
- */
-
-void * list_find (ListIterator i, ListFindF f, void *key);
-/*
- *  Traverses the list from the point of the list iterator [i]
- *    using [f] to match each item with [key].
- *  Returns a ptr to the next item for which the function [f]
- *    returns non-zero, or NULL once the end of the list is reached.
- *  Example: i=list_iterator_reset(i); while ((x=list_find(i,f,k))) {...}
- */
-
-void * list_remove (ListIterator i);
-/*
- *  Removes from the list the last item returned via list iterator [i]
- *    and returns the data's ptr.
- *  Note: The client is responsible for freeing the returned data.
- */
-
-int list_delete (ListIterator i);
-/*
- *  Removes from the list the last item returned via list iterator [i];
- *    if a deletion function was specified when the list was created,
- *    it will be called to deallocate the item being removed.
- *  Returns a count of the number of items removed from the list
- *    (ie, '1' if the item was removed, and '0' otherwise).
- */
-
-
-#endif /* !LSD_LIST_H */
diff --git a/src/common/libmissing/Makefile.am b/src/common/libmissing/Makefile.am
new file mode 100644
index 000000000000..1258758c9f68
--- /dev/null
+++ b/src/common/libmissing/Makefile.am
@@ -0,0 +1,57 @@
+# libmissing is for functions that may be missing from standard libraries
+# on some, but not all, platforms.  To handle a missing function named 'doit':
+#
+# 1. add 'doit' to AC_REPLACE_FUNCS in configure.ac
+# 2. add 'doit.c' and 'doit.h' to this directory
+# 3. add 'doit.h' to EXTRA_libmissing_la_SOURCES
+#
+# AC_REPLACE_FUNCS takes care of adding HAVE_DOIT to config.h,
+# doit.lo to LTLIBOBJS, and doit.c to DISTFILES.
+#
+# Where the missing function is used, add something like
+#
+#   #ifndef HAVE_DOIT
+#   #include "src/common/libmissing/doit.h"
+#   #endif
+#
+# or if there is an existing system include just for the missing function
+#
+#   #ifdef HAVE_DOIT
+#   #include <doit.h>
+#   #else
+#   #include "src/common/libmissing/doit.h"
+#   #endif
+#
+# Make sure the replacement function is compatible with the project license
+# and has proper attribution in the commit message that adds it.
+#
+# See also:
+# https://www.gnu.org/software/automake/manual/html_node/LIBOBJS.html
+# https://www.gnu.org/savannah-checkouts/gnu/autoconf/manual/autoconf-2.71/html_node/Generic-Functions.html
+#
+AM_CFLAGS = \
+        $(WARNING_CFLAGS) \
+        $(CODE_COVERAGE_CFLAGS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir)
+
+noinst_LTLIBRARIES = libmissing.la
+
+libmissing_la_LIBADD = $(LTLIBOBJS)
+libmissing_la_SOURCES =
+
+EXTRA_libmissing_la_SOURCES = \
+	strlcpy.h \
+	strlcat.h \
+	argz.h \
+	argz.c \
+	envz.h \
+	envz.c \
+	macros.h \
+	strerrorname_np.h \
+	json_object_update_recursive.h \
+	pipe2.h \
+	mempcpy.h \
+	get_current_dir_name.h
diff --git a/src/common/libmissing/argz.c b/src/common/libmissing/argz.c
new file mode 100644
index 000000000000..1b340a508ca9
--- /dev/null
+++ b/src/common/libmissing/argz.c
@@ -0,0 +1,408 @@
+/* Functions for dealing with '\0' separated arg vectors.
+   Copyright (C) 1995-1998, 2000-2002, 2006, 2008-2023 Free Software
+   Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   This file is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as
+   published by the Free Software Foundation; either version 2.1 of the
+   License, or (at your option) any later version.
+
+   This file is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with this program.  If not, see <https://www.gnu.org/licenses/>.  */
+
+#include <config.h>
+
+#include <argz.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#ifndef HAVE_MEMPCPY
+#include "mempcpy.h"
+#endif
+
+
+/* Add BUF, of length BUF_LEN to the argz vector in ARGZ & ARGZ_LEN.  */
+error_t
+argz_append (char **argz, size_t *argz_len, const char *buf, size_t buf_len)
+{
+  size_t new_argz_len = *argz_len + buf_len;
+  char *new_argz = realloc (*argz, new_argz_len);
+  if (new_argz)
+    {
+      memcpy (new_argz + *argz_len, buf, buf_len);
+      *argz = new_argz;
+      *argz_len = new_argz_len;
+      return 0;
+    }
+  else
+    return ENOMEM;
+}
+
+/* Add STR to the argz vector in ARGZ & ARGZ_LEN.  This should be moved into
+   argz.c in libshouldbelibc.  */
+error_t
+argz_add (char **argz, size_t *argz_len, const char *str)
+{
+  return argz_append (argz, argz_len, str, strlen (str) + 1);
+}
+
+
+
+error_t
+argz_add_sep (char **argz, size_t *argz_len, const char *string, int delim)
+{
+  size_t nlen = strlen (string) + 1;
+
+  if (nlen > 1)
+    {
+      const char *rp;
+      char *wp;
+
+      *argz = (char *) realloc (*argz, *argz_len + nlen);
+      if (*argz == NULL)
+        return ENOMEM;
+
+      wp = *argz + *argz_len;
+      rp = string;
+      do
+        if (*rp == delim)
+          {
+            if (wp > *argz && wp[-1] != '\0')
+              *wp++ = '\0';
+            else
+              --nlen;
+          }
+        else
+          *wp++ = *rp;
+      while (*rp++ != '\0');
+
+      *argz_len += nlen;
+    }
+
+  return 0;
+}
+
+
+
+error_t
+argz_create_sep (const char *string, int delim, char **argz, size_t *len)
+{
+  size_t nlen = strlen (string) + 1;
+
+  if (nlen > 1)
+    {
+      const char *rp;
+      char *wp;
+
+      *argz = (char *) malloc (nlen);
+      if (*argz == NULL)
+        return ENOMEM;
+
+      rp = string;
+      wp = *argz;
+      do
+        if (*rp == delim)
+          {
+            if (wp > *argz && wp[-1] != '\0')
+              *wp++ = '\0';
+            else
+              --nlen;
+          }
+        else
+          *wp++ = *rp;
+      while (*rp++ != '\0');
+
+      if (nlen == 0)
+        {
+          free (*argz);
+          *argz = NULL;
+          *len = 0;
+        }
+
+      *len = nlen;
+    }
+  else
+    {
+      *argz = NULL;
+      *len = 0;
+    }
+
+  return 0;
+}
+
+
+/* Insert ENTRY into ARGZ & ARGZ_LEN before BEFORE, which should be an
+   existing entry in ARGZ; if BEFORE is NULL, ENTRY is appended to the end.
+   Since ARGZ's first entry is the same as ARGZ, argz_insert (ARGZ, ARGZ_LEN,
+   ARGZ, ENTRY) will insert ENTRY at the beginning of ARGZ.  If BEFORE is not
+   in ARGZ, EINVAL is returned, else if memory can't be allocated for the new
+   ARGZ, ENOMEM is returned, else 0.  */
+error_t
+argz_insert (char **argz, size_t *argz_len, char *before, const char *entry)
+{
+  if (! before)
+    return argz_add (argz, argz_len, entry);
+
+  if (before < *argz || before >= *argz + *argz_len)
+    return EINVAL;
+
+  if (before > *argz)
+    /* Make sure before is actually the beginning of an entry.  */
+    while (before[-1])
+      before--;
+
+  {
+    size_t after_before = *argz_len - (before - *argz);
+    size_t entry_len = strlen  (entry) + 1;
+    size_t new_argz_len = *argz_len + entry_len;
+    char *new_argz = realloc (*argz, new_argz_len);
+
+    if (new_argz)
+      {
+        before = new_argz + (before - *argz);
+        memmove (before + entry_len, before, after_before);
+        memmove (before, entry, entry_len);
+        *argz = new_argz;
+        *argz_len = new_argz_len;
+        return 0;
+      }
+    else
+      return ENOMEM;
+  }
+}
+
+
+char *
+argz_next (const char *argz, size_t argz_len, const char *entry)
+{
+  if (entry)
+    {
+      if (entry < argz + argz_len)
+        entry = strchr (entry, '\0') + 1;
+
+      return entry >= argz + argz_len ? NULL : (char *) entry;
+    }
+  else
+    if (argz_len > 0)
+      return (char *) argz;
+    else
+      return NULL;
+}
+
+
+/* Make '\0' separated arg vector ARGZ printable by converting all the '\0's
+   except the last into the character SEP.  */
+void
+argz_stringify (char *argz, size_t len, int sep)
+{
+  if (len > 0)
+    while (1)
+      {
+        size_t part_len = strnlen (argz, len);
+        argz += part_len;
+        len -= part_len;
+        if (len-- <= 1)         /* includes final '\0' we want to stop at */
+          break;
+        *argz++ = sep;
+      }
+}
+
+
+/* Returns the number of strings in ARGZ.  */
+size_t
+argz_count (const char *argz, size_t len)
+{
+  size_t count = 0;
+  while (len > 0)
+    {
+      size_t part_len = strlen (argz);
+      argz += part_len + 1;
+      len -= part_len + 1;
+      count++;
+    }
+  return count;
+}
+
+
+/* Puts pointers to each string in ARGZ, plus a terminating 0 element, into
+   ARGV, which must be large enough to hold them all.  */
+void
+argz_extract (const char *argz, size_t len, char **argv)
+{
+  while (len > 0)
+    {
+      size_t part_len = strlen (argz);
+      *argv++ = (char *) argz;
+      argz += part_len + 1;
+      len -= part_len + 1;
+    }
+  *argv = 0;
+}
+
+
+/* Make a '\0' separated arg vector from a unix argv vector, returning it in
+   ARGZ, and the total length in LEN.  If a memory allocation error occurs,
+   ENOMEM is returned, otherwise 0.  */
+error_t
+argz_create (char *const argv[], char **argz, size_t *len)
+{
+  int argc;
+  size_t tlen = 0;
+  char *const *ap;
+  char *p;
+
+  for (argc = 0; argv[argc] != NULL; ++argc)
+    tlen += strlen (argv[argc]) + 1;
+
+  if (tlen == 0)
+    *argz = NULL;
+  else
+    {
+      *argz = malloc (tlen);
+      if (*argz == NULL)
+        return ENOMEM;
+
+      for (p = *argz, ap = argv; *ap; ++ap, ++p)
+        p = stpcpy (p, *ap);
+    }
+  *len = tlen;
+
+  return 0;
+}
+
+
+/* Delete ENTRY from ARGZ & ARGZ_LEN, if any.  */
+void
+argz_delete (char **argz, size_t *argz_len, char *entry)
+{
+  if (entry)
+    /* Get rid of the old value for NAME.  */
+    {
+      size_t entry_len = strlen (entry) + 1;
+      *argz_len -= entry_len;
+      memmove (entry, entry + entry_len, *argz_len - (entry - *argz));
+      if (*argz_len == 0)
+        {
+          free (*argz);
+          *argz = 0;
+        }
+    }
+}
+
+
+/* Append BUF, of length BUF_LEN to *TO, of length *TO_LEN, reallocating and
+   updating *TO & *TO_LEN appropriately.  If an allocation error occurs,
+   *TO's old value is freed, and *TO is set to 0.  */
+static void
+str_append (char **to, size_t *to_len, const char *buf, const size_t buf_len)
+{
+  size_t new_len = *to_len + buf_len;
+  char *new_to = realloc (*to, new_len + 1);
+
+  if (new_to)
+    {
+      *((char *) mempcpy (new_to + *to_len, buf, buf_len)) = '\0';
+      *to = new_to;
+      *to_len = new_len;
+    }
+  else
+    {
+      free (*to);
+      *to = 0;
+    }
+}
+
+/* Replace any occurrences of the string STR in ARGZ with WITH, reallocating
+   ARGZ as necessary.  If REPLACE_COUNT is non-zero, *REPLACE_COUNT will be
+   incremented by number of replacements performed.  */
+error_t
+argz_replace (char **argz, size_t *argz_len, const char *str, const char *with,
+                unsigned *replace_count)
+{
+  error_t err = 0;
+
+  if (str && *str)
+    {
+      char *arg = 0;
+      char *src = *argz;
+      size_t src_len = *argz_len;
+      char *dst = 0;
+      size_t dst_len = 0;
+      int delayed_copy = 1;     /* True while we've avoided copying anything.  */
+      size_t str_len = strlen (str), with_len = strlen (with);
+
+      while (!err && (arg = argz_next (src, src_len, arg)))
+        {
+          char *match = strstr (arg, str);
+          if (match)
+            {
+              char *from = match + str_len;
+              size_t to_len = match - arg;
+              char *to = strndup (arg, to_len);
+
+              while (to && from)
+                {
+                  str_append (&to, &to_len, with, with_len);
+                  if (to)
+                    {
+                      match = strstr (from, str);
+                      if (match)
+                        {
+                          str_append (&to, &to_len, from, match - from);
+                          from = match + str_len;
+                        }
+                      else
+                        {
+                          str_append (&to, &to_len, from, strlen (from));
+                          from = 0;
+                        }
+                    }
+                }
+
+              if (to)
+                {
+                  if (delayed_copy)
+                    /* We avoided copying SRC to DST until we found a match;
+                       now that we've done so, copy everything from the start
+                       of SRC.  */
+                    {
+                      if (arg > src)
+                        err = argz_append (&dst, &dst_len, src, (arg - src));
+                      delayed_copy = 0;
+                    }
+                  if (! err)
+                    err = argz_add (&dst, &dst_len, to);
+                  free (to);
+                }
+              else
+                err = ENOMEM;
+
+              if (replace_count)
+                (*replace_count)++;
+            }
+          else if (! delayed_copy)
+            err = argz_add (&dst, &dst_len, arg);
+        }
+
+      if (! err)
+        {
+          if (! delayed_copy)
+            /* We never found any instances of str.  */
+            {
+              free (src);
+              *argz = dst;
+              *argz_len = dst_len;
+            }
+        }
+      else if (dst_len > 0)
+        free (dst);
+    }
+
+  return err;
+}
diff --git a/src/common/libmissing/argz.h b/src/common/libmissing/argz.h
new file mode 100644
index 000000000000..3ba80a69ec85
--- /dev/null
+++ b/src/common/libmissing/argz.h
@@ -0,0 +1,130 @@
+/* Routines for dealing with '\0' separated arg vectors.
+   Copyright (C) 1995-2000, 2004, 2007, 2009-2023 Free Software Foundation,
+   Inc.
+   This file is part of the GNU C Library.
+
+   This file is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as
+   published by the Free Software Foundation; either version 2.1 of the
+   License, or (at your option) any later version.
+
+   This file is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with this program.  If not, see <https://www.gnu.org/licenses/>.  */
+
+#ifndef _ARGZ_H
+#define _ARGZ_H 1
+
+
+#define __need_error_t
+#include <errno.h>
+#include <string.h>             /* Need size_t.  */
+
+#ifndef __error_t_defined
+typedef int error_t;
+#endif
+
+
+
+/* Make a '\0' separated arg vector from a unix argv vector, returning it in
+   ARGZ, and the total length in LEN.  If a memory allocation error occurs,
+   ENOMEM is returned, otherwise 0.  The result can be destroyed using free.  */
+
+extern error_t argz_create (char *const /*argv*/[], char **restrict /*argz*/,
+                            size_t *restrict /*len*/);
+
+/* Make a '\0' separated arg vector from a SEP separated list in
+   STRING, returning it in ARGZ, and the total length in LEN.  If a
+   memory allocation error occurs, ENOMEM is returned, otherwise 0.
+   The result can be destroyed using free.  */
+
+extern error_t argz_create_sep (const char *restrict /*string*/,
+                                int /*sep*/, char **restrict /*argz*/,
+                                size_t *restrict /*len*/);
+
+/* Returns the number of strings in ARGZ.  */
+
+extern size_t argz_count (const char * /*argz*/, size_t /*len*/);
+
+/* Puts pointers to each string in ARGZ into ARGV, which must be large enough
+   to hold them all.  */
+
+extern void argz_extract (const char *restrict /*argz*/, size_t /*len*/,
+                          char **restrict /*argv*/);
+
+/* Make '\0' separated arg vector ARGZ printable by converting all the '\0's
+   except the last into the character SEP.  */
+
+extern void argz_stringify (char * /*argz*/, size_t /*len*/, int /*sep*/);
+
+/* Append BUF, of length BUF_LEN to the argz vector in ARGZ & ARGZ_LEN.  */
+
+extern error_t argz_append (char **restrict /*argz*/,
+                            size_t *restrict /*argz_len*/,
+                            const char *restrict /*buf*/, size_t /*buf_len*/);
+
+/* Append STR to the argz vector in ARGZ & ARGZ_LEN.  */
+
+extern error_t argz_add (char **restrict /*argz*/,
+                         size_t *restrict /*argz_len*/,
+                         const char *restrict str);
+
+/* Append SEP separated list in STRING to the argz vector in ARGZ &
+   ARGZ_LEN.  */
+
+extern error_t argz_add_sep (char **restrict /*argz*/,
+                             size_t *restrict /*argz_len*/,
+                             const char *restrict /*string*/, int /*delim*/);
+
+/* Delete ENTRY from ARGZ & ARGZ_LEN, if it appears there.  */
+
+extern void argz_delete (char **restrict /*argz*/,
+                         size_t *restrict /*argz_len*/,
+                         char *restrict /*entry*/);
+
+/* Insert ENTRY into ARGZ & ARGZ_LEN before BEFORE, which should be an
+   existing entry in ARGZ; if BEFORE is NULL, ENTRY is appended to the end.
+   Since ARGZ's first entry is the same as ARGZ, argz_insert (ARGZ, ARGZ_LEN,
+   ARGZ, ENTRY) will insert ENTRY at the beginning of ARGZ.  If BEFORE is not
+   in ARGZ, EINVAL is returned, else if memory can't be allocated for the new
+   ARGZ, ENOMEM is returned, else 0.  */
+
+extern error_t argz_insert (char **restrict /*argz*/,
+                            size_t *restrict /*argz_len*/,
+                            char *restrict /*before*/,
+                            const char *restrict /*entry*/);
+
+/* Replace any occurrences of the string STR in ARGZ with WITH, reallocating
+   ARGZ as necessary.  If REPLACE_COUNT is non-zero, *REPLACE_COUNT will be
+   incremented by number of replacements performed.  */
+
+extern error_t argz_replace (char **restrict /*argz*/,
+                             size_t *restrict /*argz_len*/,
+                             const char *restrict /*str*/,
+                             const char *restrict /*with*/,
+                             unsigned int *restrict /*replace_count*/);
+
+/* Returns the next entry in ARGZ & ARGZ_LEN after ENTRY, or NULL if there
+   are no more.  If entry is NULL, then the first entry is returned.  This
+   behavior allows two convenient iteration styles:
+
+    char *entry = 0;
+    while ((entry = argz_next (argz, argz_len, entry)))
+      ...;
+
+   or
+
+    char *entry;
+    for (entry = argz; entry; entry = argz_next (argz, argz_len, entry))
+      ...;
+*/
+
+extern char *argz_next (const char *restrict /*argz*/, size_t /*argz_len*/,
+                        const char *restrict /*entry*/);
+
+
+#endif /* argz.h */
diff --git a/src/common/libmissing/argz_add.c b/src/common/libmissing/argz_add.c
new file mode 100644
index 000000000000..2bda13789fb3
--- /dev/null
+++ b/src/common/libmissing/argz_add.c
@@ -0,0 +1,2 @@
+// build all of argz.c if argz_add() is detected missing by autotools
+#include "argz.c"
diff --git a/src/common/libmissing/envz.c b/src/common/libmissing/envz.c
new file mode 100644
index 000000000000..2e6767e46b19
--- /dev/null
+++ b/src/common/libmissing/envz.c
@@ -0,0 +1,169 @@
+/* Routines for dealing with '\0' separated environment vectors
+   Copyright (C) 1995-2023 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <envz.h>
+
+/* The character separating names from values in an envz.  */
+#define SEP '='
+
+/* Returns a pointer to the entry in ENVZ for NAME, or 0 if there is none.
+   If NAME contains the separator character, only the portion before it is
+   used in the comparison.  */
+char *
+envz_entry (const char *envz, size_t envz_len, const char *name)
+{
+  while (envz_len)
+    {
+      const char *p = name;
+      const char *entry = envz;	/* Start of this entry. */
+
+      /* See how far NAME and ENTRY match.  */
+      while (envz_len && *p == *envz && *p && *p != SEP)
+	p++, envz++, envz_len--;
+
+      if ((*envz == '\0' || *envz == SEP) && (*p == '\0' || *p == SEP))
+	/* Bingo! */
+	return (char *) entry;
+
+      /* No match, skip to the next entry.  */
+      while (envz_len && *envz)
+	envz++, envz_len--;
+      if (envz_len)
+	envz++, envz_len--;	/* skip '\0' */
+    }
+
+  return 0;
+}
+
+/* Returns a pointer to the value portion of the entry in ENVZ for NAME, or 0
+   if there is none.  */
+char *
+envz_get (const char *envz, size_t envz_len, const char *name)
+{
+  char *entry = envz_entry (envz, envz_len, name);
+  if (entry)
+    {
+      while (*entry && *entry != SEP)
+	entry++;
+      if (*entry)
+	entry++;
+      else
+	entry = 0;		/* A null entry.  */
+    }
+  return entry;
+}
+
+/* Remove the entry for NAME from ENVZ & ENVZ_LEN, if any.  */
+void
+envz_remove (char **envz, size_t *envz_len, const char *name)
+{
+  char *entry = envz_entry (*envz, *envz_len, name);
+  if (entry)
+    argz_delete (envz, envz_len, entry);
+}
+
+/* Adds an entry for NAME with value VALUE to ENVZ & ENVZ_LEN.  If an entry
+   with the same name already exists in ENVZ, it is removed.  If VALUE is
+   NULL, then the new entry will a special null one, for which envz_get will
+   return NULL, although envz_entry will still return an entry; this is handy
+   because when merging with another envz, the null entry can override an
+   entry in the other one.  Null entries can be removed with envz_strip ().  */
+error_t
+envz_add (char **envz, size_t *envz_len, const char *name, const char *value)
+{
+  envz_remove (envz, envz_len, name);
+
+  if (value)
+    /* Add the new value, if there is one.  */
+    {
+      size_t name_len = strlen (name);
+      size_t value_len = strlen (value);
+      size_t old_envz_len = *envz_len;
+      size_t new_envz_len = old_envz_len + name_len + 1 + value_len + 1;
+      char *new_envz = realloc (*envz, new_envz_len);
+
+      if (new_envz)
+	{
+	  memcpy (new_envz + old_envz_len, name, name_len);
+	  new_envz[old_envz_len + name_len] = SEP;
+	  memcpy (new_envz + old_envz_len + name_len + 1, value, value_len);
+	  new_envz[new_envz_len - 1] = 0;
+
+	  *envz = new_envz;
+	  *envz_len = new_envz_len;
+
+	  return 0;
+	}
+      else
+	return ENOMEM;
+    }
+  else
+    /* Add a null entry.  */
+    return argz_add (envz, envz_len, name);
+}
+
+/* Adds each entry in ENVZ2 to ENVZ & ENVZ_LEN, as if with envz_add().  If
+   OVERRIDE is true, then values in ENVZ2 will supersede those with the same
+   name in ENV, otherwise not.  */
+error_t
+envz_merge (char **envz, size_t *envz_len, const char *envz2,
+	    size_t envz2_len, int override)
+{
+  error_t err = 0;
+
+  while (envz2_len && ! err)
+    {
+      char *old = envz_entry (*envz, *envz_len, envz2);
+      size_t new_len = strlen (envz2) + 1;
+
+      if (! old)
+	err = argz_append (envz, envz_len, envz2, new_len);
+      else if (override)
+	{
+	  argz_delete (envz, envz_len, old);
+	  err = argz_append (envz, envz_len, envz2, new_len);
+	}
+
+      envz2 += new_len;
+      envz2_len -= new_len;
+    }
+
+  return err;
+}
+
+/* Remove null entries.  */
+void
+envz_strip (char **envz, size_t *envz_len)
+{
+  char *entry = *envz;
+  size_t left = *envz_len;
+  while (left)
+    {
+      size_t entry_len = strlen (entry) + 1;
+      left -= entry_len;
+      if (! strchr (entry, SEP))
+	/* Null entry. */
+	memmove (entry, entry + entry_len, left);
+      else
+	entry += entry_len;
+    }
+  *envz_len = entry - *envz;
+}
diff --git a/src/common/libmissing/envz.h b/src/common/libmissing/envz.h
new file mode 100644
index 000000000000..8f0905d67ab5
--- /dev/null
+++ b/src/common/libmissing/envz.h
@@ -0,0 +1,96 @@
+/* Routines for dealing with '\0' separated environment vectors
+   Copyright (C) 1995-2023 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#ifndef _ENVZ_H
+#define _ENVZ_H	1
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+
+/* Envz's are argz's too, and should be created etc., using the same
+   routines.  */
+#if HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "argz.h"
+#endif
+
+#ifndef __BEGIN_DECLS
+#define __BEGIN_DECLS
+#endif
+#ifndef __END_DECLS
+#define __END_DECLS
+#endif
+#ifndef __THROW
+#define __THROW
+#endif
+#ifndef __CATCH
+#define __CATCH
+#endif
+#ifndef __attribute_pure__
+#define __attribute_pure__
+#endif
+
+
+__BEGIN_DECLS
+
+/* Returns a pointer to the entry in ENVZ for NAME, or 0 if there is none.  */
+extern char *envz_entry (const char *__restrict __envz, size_t __envz_len,
+			 const char *__restrict __name)
+     __THROW __attribute_pure__;
+
+/* Returns a pointer to the value portion of the entry in ENVZ for NAME, or 0
+   if there is none.  */
+extern char *envz_get (const char *__restrict __envz, size_t __envz_len,
+		       const char *__restrict __name)
+     __THROW __attribute_pure__;
+
+/* Adds an entry for NAME with value VALUE to ENVZ & ENVZ_LEN.  If an entry
+   with the same name already exists in ENVZ, it is removed.  If VALUE is
+   NULL, then the new entry will a special null one, for which envz_get will
+   return NULL, although envz_entry will still return an entry; this is handy
+   because when merging with another envz, the null entry can override an
+   entry in the other one.  Null entries can be removed with envz_strip ().  */
+extern error_t envz_add (char **__restrict __envz,
+			 size_t *__restrict __envz_len,
+			 const char *__restrict __name,
+			 const char *__restrict __value) __THROW;
+
+/* Adds each entry in ENVZ2 to ENVZ & ENVZ_LEN, as if with envz_add().  If
+   OVERRIDE is true, then values in ENVZ2 will supersede those with the same
+   name in ENV, otherwise not.  */
+extern error_t envz_merge (char **__restrict __envz,
+			   size_t *__restrict __envz_len,
+			   const char *__restrict __envz2,
+			   size_t __envz2_len, int __override) __THROW;
+
+/* Remove the entry for NAME from ENVZ & ENVZ_LEN, if any.  */
+extern void envz_remove (char **__restrict __envz,
+			 size_t *__restrict __envz_len,
+			 const char *__restrict __name) __THROW;
+
+/* Remove null entries.  */
+extern void envz_strip (char **__restrict __envz,
+			size_t *__restrict __envz_len) __THROW;
+
+__END_DECLS
+
+#endif /* envz.h */
diff --git a/src/common/libmissing/envz_add.c b/src/common/libmissing/envz_add.c
new file mode 100644
index 000000000000..3530d0222348
--- /dev/null
+++ b/src/common/libmissing/envz_add.c
@@ -0,0 +1,2 @@
+// build all of envz.c if envz_add() is detected missing by autotools
+#include "envz.c"
diff --git a/src/common/libmissing/get_current_dir_name.c b/src/common/libmissing/get_current_dir_name.c
new file mode 100644
index 000000000000..34d26ec8f91b
--- /dev/null
+++ b/src/common/libmissing/get_current_dir_name.c
@@ -0,0 +1,33 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <limits.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+
+#include "get_current_dir_name.h"
+
+char *get_current_dir_name (void)
+{
+    char buf[PATH_MAX + 1];
+    char *name;
+    char *cpy;
+
+    if (!(name = getcwd (buf, sizeof (buf)))
+        || !(cpy = strdup (name)))
+        return NULL;
+    return cpy;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libmissing/get_current_dir_name.h b/src/common/libmissing/get_current_dir_name.h
new file mode 100644
index 000000000000..5ae7e301dc87
--- /dev/null
+++ b/src/common/libmissing/get_current_dir_name.h
@@ -0,0 +1,16 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _GET_CURRENT_DIR_NAME_H
+#define _GET_CURRENT_DIR_NAME_H 1
+
+char *get_current_dir_name (void);
+
+#endif // !_GET_CURRENT_DIR_NAME_H
diff --git a/src/common/libmissing/json_object_update_recursive.c b/src/common/libmissing/json_object_update_recursive.c
new file mode 100644
index 000000000000..eebbecce0236
--- /dev/null
+++ b/src/common/libmissing/json_object_update_recursive.c
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2009-2016 Petri Lehtinen <petri@digip.org>
+ *
+ * Jansson is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#include <jansson.h>
+
+/* Taken from jansson v2.13 source. This version does not do a loop
+ * check as is done in later versions of this function. The loop check
+ * requires access to jansson internals.
+ */
+int json_object_update_recursive (json_t *object, json_t *other)
+{
+    const char *key;
+    json_t *value;
+
+    if (!json_is_object (object) || !json_is_object (other))
+        return -1;
+
+    json_object_foreach (other, key, value) {
+        json_t *v = json_object_get (object, key);
+
+        if (json_is_object (v) && json_is_object (value))
+            json_object_update_recursive (v, value);
+        else
+            json_object_set_nocheck (object, key, value);
+    }
+
+    return 0;
+}
diff --git a/src/common/libmissing/json_object_update_recursive.h b/src/common/libmissing/json_object_update_recursive.h
new file mode 100644
index 000000000000..fa622333a220
--- /dev/null
+++ b/src/common/libmissing/json_object_update_recursive.h
@@ -0,0 +1,24 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef LIBMISSING_JANNSON_UPDATE_RECURSIVE_H
+#define LIBMISSING_JANNSON_UPDATE_RECURSIVE_H 1
+
+/* Like json_object_update(), but object values in *other* are
+ * recursively merged the corresponding values in *object* if they
+ * are also objects, instead of overwriting them. Returns a 0 on
+ * successs or -1 on error.
+ *
+ * Note: This version doesn't detect cycles like the version found
+ * in jansson >= 2.13.1.
+ */
+int json_object_update_recursive (json_t *object, json_t *other);
+
+#endif
diff --git a/src/common/libmissing/macros.h b/src/common/libmissing/macros.h
new file mode 100644
index 000000000000..061b877a211f
--- /dev/null
+++ b/src/common/libmissing/macros.h
@@ -0,0 +1,19 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef LIBMISSING_MACROS_H
+#define LIBMISSING_MACROS_H 1
+
+// borrowed from glibc-2.38.9000-255-g323f367cc4
+#ifndef __W_EXITCODE
+#define __W_EXITCODE(ret, sig)  ((ret) << 8 | (sig))
+#endif
+
+#endif
diff --git a/src/common/libmissing/mempcpy.c b/src/common/libmissing/mempcpy.c
new file mode 100644
index 000000000000..6f140c880687
--- /dev/null
+++ b/src/common/libmissing/mempcpy.c
@@ -0,0 +1,24 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <string.h>
+#include <errno.h>
+
+#include "mempcpy.h"
+
+void *mempcpy (void *dest, const void *src, size_t len)
+{
+    return memcpy (dest, src, len) + len;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libmissing/mempcpy.h b/src/common/libmissing/mempcpy.h
new file mode 100644
index 000000000000..ef5f260e214b
--- /dev/null
+++ b/src/common/libmissing/mempcpy.h
@@ -0,0 +1,16 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _MEMPCPY_H
+#define _MEMPCPY_H 1
+
+void *mempcpy (void *dest, const void *src, size_t len);
+
+#endif // !_MEMPCPY
diff --git a/src/common/libmissing/pipe2.c b/src/common/libmissing/pipe2.c
new file mode 100644
index 000000000000..1198455d5db1
--- /dev/null
+++ b/src/common/libmissing/pipe2.c
@@ -0,0 +1,58 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include "src/common/libutil/fdutils.h"
+
+#include "pipe2.h"
+
+static int setflags (int fd, int flags)
+{
+    int valid_flags = O_CLOEXEC | O_NONBLOCK;
+
+    if ((flags & ~valid_flags)) {
+        errno = EINVAL;
+        return -1;
+    }
+    if ((flags & O_CLOEXEC)) {
+        if (fd_set_cloexec (fd) < 0)
+            return -1;
+    }
+    if ((flags & O_NONBLOCK)) {
+        if (fd_set_nonblocking (fd) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+int pipe2 (int pipefd[2], int flags)
+{
+    int pfd[2];
+    if (pipe (pfd) < 0)
+        return -1;
+    if (setflags (pfd[0], flags) < 0 || setflags (pfd[1], flags) < 0) {
+        int saved_errno = errno;
+        (void)close (pfd[0]);
+        (void)close (pfd[1]);
+        errno = saved_errno;
+        return -1;
+    }
+    pipefd[0] = pfd[0];
+    pipefd[1] = pfd[1];
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libmissing/pipe2.h b/src/common/libmissing/pipe2.h
new file mode 100644
index 000000000000..4c9c346fb710
--- /dev/null
+++ b/src/common/libmissing/pipe2.h
@@ -0,0 +1,16 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _PIPE2_H
+#define _PIPE2_H 1
+
+int pipe2 (int pipefd[2], int flags);
+
+#endif // !_PIPE2_H
diff --git a/src/common/libmissing/strerrorname_np.c b/src/common/libmissing/strerrorname_np.c
new file mode 100644
index 000000000000..4445a88ac6f2
--- /dev/null
+++ b/src/common/libmissing/strerrorname_np.c
@@ -0,0 +1,1350 @@
+/* Name of system error code.
+   Copyright (C) 2020-2023 Free Software Foundation, Inc.
+
+   This file is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as
+   published by the Free Software Foundation; either version 2.1 of the
+   License, or (at your option) any later version.
+
+   This file is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with this program.  If not, see <https://www.gnu.org/licenses/>.  */
+
+/* Written by Bruno Haible <bruno@clisp.org>, 2020.  */
+
+#include <config.h>
+
+/* Specification.  */
+#include <string.h>
+
+#include <errno.h>
+
+const char *
+strerrorname_np (int errnum)
+{
+  switch (errnum)
+    {
+    /* Error codes specified by ISO C.  */
+    case EDOM:            return "EDOM";
+    case EILSEQ:          return "EILSEQ";
+    case ERANGE:          return "ERANGE";
+
+    /* Error codes specified by POSIX.
+       <https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html>  */
+    #if defined E2BIG
+    case E2BIG:           return "E2BIG";
+    #endif
+    #if defined EACCES
+    case EACCES:          return "EACCES";
+    #endif
+    #if defined EADDRINUSE
+    case EADDRINUSE:      return "EADDRINUSE";
+    #endif
+    #if defined EADDRNOTAVAIL
+    case EADDRNOTAVAIL:   return "EADDRNOTAVAIL";
+    #endif
+    #if defined EAFNOSUPPORT
+    case EAFNOSUPPORT:    return "EAFNOSUPPORT";
+    #endif
+    #if defined EAGAIN
+    case EAGAIN:          return "EAGAIN";
+    #endif
+    #if defined EALREADY
+    case EALREADY:        return "EALREADY";
+    #endif
+    #if defined EBADF
+    case EBADF:           return "EBADF";
+    #endif
+    #if defined EBADMSG
+    case EBADMSG:         return "EBADMSG";
+    #endif
+    #if defined EBUSY
+    case EBUSY:           return "EBUSY";
+    #endif
+    #if defined ECANCELED
+    case ECANCELED:       return "ECANCELED";
+    #endif
+    #if defined ECHILD
+    case ECHILD:          return "ECHILD";
+    #endif
+    #if defined ECONNABORTED
+    case ECONNABORTED:    return "ECONNABORTED";
+    #endif
+    #if defined ECONNREFUSED
+    case ECONNREFUSED:    return "ECONNREFUSED";
+    #endif
+    #if defined ECONNRESET
+    case ECONNRESET:      return "ECONNRESET";
+    #endif
+    #if defined EDEADLK
+    case EDEADLK:         return "EDEADLK";
+    #endif
+    #if defined EDESTADDRREQ
+    case EDESTADDRREQ:    return "EDESTADDRREQ";
+    #endif
+    #if defined EDQUOT
+    case EDQUOT:          return "EDQUOT";
+    #endif
+    #if defined EEXIST
+    case EEXIST:          return "EEXIST";
+    #endif
+    #if defined EFAULT
+    case EFAULT:          return "EFAULT";
+    #endif
+    #if defined EFBIG
+    case EFBIG:           return "EFBIG";
+    #endif
+    #if defined EHOSTUNREACH
+    case EHOSTUNREACH:    return "EHOSTUNREACH";
+    #endif
+    #if defined EIDRM
+    case EIDRM:           return "EIDRM";
+    #endif
+    #if defined EINPROGRESS
+    case EINPROGRESS:     return "EINPROGRESS";
+    #endif
+    #if defined EINTR
+    case EINTR:           return "EINTR";
+    #endif
+    #if defined EINVAL
+    case EINVAL:          return "EINVAL";
+    #endif
+    #if defined EIO
+    case EIO:             return "EIO";
+    #endif
+    #if defined EISCONN
+    case EISCONN:         return "EISCONN";
+    #endif
+    #if defined EISDIR
+    case EISDIR:          return "EISDIR";
+    #endif
+    #if defined ELOOP
+    case ELOOP:           return "ELOOP";
+    #endif
+    #if defined EMFILE
+    case EMFILE:          return "EMFILE";
+    #endif
+    #if defined EMLINK
+    case EMLINK:          return "EMLINK";
+    #endif
+    #if defined EMSGSIZE
+    case EMSGSIZE:        return "EMSGSIZE";
+    #endif
+    #if defined EMULTIHOP
+    case EMULTIHOP:       return "EMULTIHOP";
+    #endif
+    #if defined ENAMETOOLONG
+    case ENAMETOOLONG:    return "ENAMETOOLONG";
+    #endif
+    #if defined ENETDOWN
+    case ENETDOWN:        return "ENETDOWN";
+    #endif
+    #if defined ENETRESET
+    case ENETRESET:       return "ENETRESET";
+    #endif
+    #if defined ENETUNREACH
+    case ENETUNREACH:     return "ENETUNREACH";
+    #endif
+    #if defined ENFILE
+    case ENFILE:          return "ENFILE";
+    #endif
+    #if defined ENOBUFS
+    case ENOBUFS:         return "ENOBUFS";
+    #endif
+    #if defined ENODATA
+    case ENODATA:         return "ENODATA";
+    #endif
+    #if defined ENODEV
+    case ENODEV:          return "ENODEV";
+    #endif
+    #if defined ENOENT
+    case ENOENT:          return "ENOENT";
+    #endif
+    #if defined ENOEXEC
+    case ENOEXEC:         return "ENOEXEC";
+    #endif
+    #if defined ENOLCK
+    case ENOLCK:          return "ENOLCK";
+    #endif
+    #if defined ENOLINK
+    case ENOLINK:         return "ENOLINK";
+    #endif
+    #if defined ENOMEM
+    case ENOMEM:          return "ENOMEM";
+    #endif
+    #if defined ENOMSG
+    case ENOMSG:          return "ENOMSG";
+    #endif
+    #if defined ENOPROTOOPT
+    case ENOPROTOOPT:     return "ENOPROTOOPT";
+    #endif
+    #if defined ENOSPC
+    case ENOSPC:          return "ENOSPC";
+    #endif
+    #if defined ENOSR
+    case ENOSR:           return "ENOSR";
+    #endif
+    #if defined ENOSTR
+    case ENOSTR:          return "ENOSTR";
+    #endif
+    #if defined ENOSYS
+    case ENOSYS:          return "ENOSYS";
+    #endif
+    #if defined ENOTCONN
+    case ENOTCONN:        return "ENOTCONN";
+    #endif
+    #if defined ENOTDIR
+    case ENOTDIR:         return "ENOTDIR";
+    #endif
+    #if defined ENOTEMPTY && ENOTEMPTY != EEXIST
+    case ENOTEMPTY:       return "ENOTEMPTY";
+    #endif
+    #if defined ENOTRECOVERABLE
+    case ENOTRECOVERABLE: return "ENOTRECOVERABLE";
+    #endif
+    #if defined ENOTSOCK
+    case ENOTSOCK:        return "ENOTSOCK";
+    #endif
+    #if defined ENOTSUP && ENOTSUP != EOPNOTSUPP
+    case ENOTSUP:         return "ENOTSUP";
+    #endif
+    #if defined ENOTTY
+    case ENOTTY:          return "ENOTTY";
+    #endif
+    #if defined ENXIO
+    case ENXIO:           return "ENXIO";
+    #endif
+    #if defined EOPNOTSUPP
+    case EOPNOTSUPP:      return "EOPNOTSUPP";
+    #endif
+    #if defined EOVERFLOW
+    case EOVERFLOW:       return "EOVERFLOW";
+    #endif
+    #if defined EOWNERDEAD
+    case EOWNERDEAD:      return "EOWNERDEAD";
+    #endif
+    #if defined EPERM
+    case EPERM:           return "EPERM";
+    #endif
+    #if defined EPIPE
+    case EPIPE:           return "EPIPE";
+    #endif
+    #if defined EPROTO
+    case EPROTO:          return "EPROTO";
+    #endif
+    #if defined EPROTONOSUPPORT
+    case EPROTONOSUPPORT: return "EPROTONOSUPPORT";
+    #endif
+    #if defined EPROTOTYPE
+    case EPROTOTYPE:      return "EPROTOTYPE";
+    #endif
+    #if defined EROFS
+    case EROFS:           return "EROFS";
+    #endif
+    #if defined ESPIPE
+    case ESPIPE:          return "ESPIPE";
+    #endif
+    #if defined ESRCH
+    case ESRCH:           return "ESRCH";
+    #endif
+    #if defined ESTALE
+    case ESTALE:          return "ESTALE";
+    #endif
+    #if defined ETIME
+    case ETIME:           return "ETIME";
+    #endif
+    #if defined ETIMEDOUT
+    case ETIMEDOUT:       return "ETIMEDOUT";
+    #endif
+    #if defined ETXTBSY
+    case ETXTBSY:         return "ETXTBSY";
+    #endif
+    #if defined EWOULDBLOCK && EWOULDBLOCK != EAGAIN
+    case EWOULDBLOCK:     return "EWOULDBLOCK";
+    #endif
+    #if defined EXDEV
+    case EXDEV:           return "EXDEV";
+    #endif
+
+    /* Other error codes on other systems.  */
+    /* Solaris */
+    #if defined EADI
+    case EADI:            return "EADI";
+    #endif
+    /* Linux, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined EADV
+    case EADV:            return "EADV";
+    #endif
+    /* OSF/1 */
+    #if defined EAIO
+    case EAIO:            return "EAIO";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, Minix */
+    #if defined EAUTH
+    case EAUTH:           return "EAUTH";
+    #endif
+    /* GNU/Hurd */
+    #if defined EBACKGROUND
+    case EBACKGROUND:     return "EBACKGROUND";
+    #endif
+    /* Mac OS X */
+    #if defined EBADARCH
+    case EBADARCH:        return "EBADARCH";
+    #endif
+    /* Minix */
+    #if defined EBADCALL
+    case EBADCALL:        return "EBADCALL";
+    #endif
+    /* Minix */
+    #if defined EBADCPU
+    case EBADCPU:         return "EBADCPU";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined EBADE && EBADE != ECKSUM
+    case EBADE:           return "EBADE";
+    #endif
+    /* Minix */
+    #if defined EBADEPT
+    case EBADEPT:         return "EBADEPT";
+    #endif
+    /* Mac OS X */
+    #if defined EBADEXEC
+    case EBADEXEC:        return "EBADEXEC";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined EBADFD
+    case EBADFD:          return "EBADFD";
+    #endif
+    /* IRIX */
+    #if defined EBADFILT
+    case EBADFILT:        return "EBADFILT";
+    #endif
+    /* Minix */
+    #if defined EBADIOCTL
+    case EBADIOCTL:       return "EBADIOCTL";
+    #endif
+    /* Mac OS X */
+    #if defined EBADMACHO
+    case EBADMACHO:       return "EBADMACHO";
+    #endif
+    /* Minix */
+    #if defined EBADMODE
+    case EBADMODE:        return "EBADMODE";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined EBADR && EBADR != EFRAGS
+    case EBADR:           return "EBADR";
+    #endif
+    /* Minix */
+    #if defined EBADREQUEST
+    case EBADREQUEST:     return "EBADREQUEST";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, OSF/1, Minix */
+    #if defined EBADRPC
+    case EBADRPC:         return "EBADRPC";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined EBADRQC
+    case EBADRQC:         return "EBADRQC";
+    #endif
+    /* IRIX */
+    #if defined EBADRSPEC
+    case EBADRSPEC:       return "EBADRSPEC";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined EBADSLT
+    case EBADSLT:         return "EBADSLT";
+    #endif
+    /* IRIX */
+    #if defined EBADTSPEC
+    case EBADTSPEC:       return "EBADTSPEC";
+    #endif
+    /* HP-UX */
+    #if defined EBADVER
+    case EBADVER:         return "EBADVER";
+    #endif
+    /* IRIX */
+    #if defined EBDHDL
+    case EBDHDL:          return "EBDHDL";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined EBFONT
+    case EBFONT:          return "EBFONT";
+    #endif
+    /* IRIX */
+    #if defined EBUFSIZE
+    case EBUFSIZE:        return "EBUFSIZE";
+    #endif
+    /* Minix */
+    #if defined ECALLDENIED
+    case ECALLDENIED:     return "ECALLDENIED";
+    #endif
+    /* IRIX */
+    #if defined ECANTEXTENT
+    case ECANTEXTENT:     return "ECANTEXTENT";
+    #endif
+    /* FreeBSD */
+    #if defined ECAPMODE
+    case ECAPMODE:        return "ECAPMODE";
+    #endif
+    /* Cygwin */
+    #if defined ECASECLASH
+    case ECASECLASH:      return "ECASECLASH";
+    #endif
+    /* IRIX */
+    #if defined ECELLDOWN
+    case ECELLDOWN:       return "ECELLDOWN";
+    #endif
+    /* Linux, AIX, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined ECHRNG
+    case ECHRNG:          return "ECHRNG";
+    #endif
+    /* IRIX */
+    #if defined ECKPT
+    case ECKPT:           return "ECKPT";
+    #endif
+    /* Solaris */
+    #if defined ECKSUM
+    case ECKSUM:          return "ECKSUM";
+    #endif
+    /* IRIX */
+    #if defined ECLOCKCPU
+    case ECLOCKCPU:       return "ECLOCKCPU";
+    #endif
+    /* OSF/1 */
+    #if defined ECLONEME && ECLONEME != ERESTART
+    case ECLONEME:        return "ECLONEME";
+    #endif
+    /* Linux, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined ECOMM
+    case ECOMM:           return "ECOMM";
+    #endif
+    /* HP-UX */
+    #if defined ECONFIG
+    case ECONFIG:         return "ECONFIG";
+    #endif
+    /* IRIX */
+    #if defined ECONTROLLER
+    case ECONTROLLER:     return "ECONTROLLER";
+    #endif
+    /* AIX */
+    #if defined ECORRUPT
+    case ECORRUPT:        return "ECORRUPT";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED
+    case ED:              return "ED";
+    #endif
+    /* Minix */
+    #if defined EDEADEPT
+    case EDEADEPT:        return "EDEADEPT";
+    #endif
+    /* IRIX, Solaris, Cygwin */
+    #if defined EDEADLOCK && EDEADLOCK != EDEADLK
+    case EDEADLOCK:       return "EDEADLOCK";
+    #endif
+    /* Minix */
+    #if defined EDEADSRCDST
+    case EDEADSRCDST:     return "EDEADSRCDST";
+    #endif
+    /* IRIX */
+    #if defined EDELAY
+    case EDELAY:          return "EDELAY";
+    #endif
+    /* IRIX */
+    #if defined EDESTROYED
+    case EDESTROYED:      return "EDESTROYED";
+    #endif
+    /* Mac OS X */
+    #if defined EDEVERR
+    case EDEVERR:         return "EDEVERR";
+    #endif
+    /* GNU/Hurd */
+    #if defined EDIED
+    case EDIED:           return "EDIED";
+    #endif
+    /* IRIX */
+    #if defined EDIRCORRUPTED && EDIRCORRUPTED != EFSCORRUPTED
+    case EDIRCORRUPTED:   return "EDIRCORRUPTED";
+    #endif
+    /* FreeBSD */
+    #if defined EDIRIOCTL
+    case EDIRIOCTL:       return "EDIRIOCTL";
+    #endif
+    /* OSF/1 */
+    #if defined EDIRTY
+    case EDIRTY:          return "EDIRTY";
+    #endif
+    /* IRIX */
+    #if defined EDISJOINT
+    case EDISJOINT:       return "EDISJOINT";
+    #endif
+    /* AIX */
+    #if defined EDIST
+    case EDIST:           return "EDIST";
+    #endif
+    /* Minix */
+    #if defined EDONTREPLY
+    case EDONTREPLY:      return "EDONTREPLY";
+    #endif
+    /* FreeBSD */
+    #if defined EDOOFUS && EDOOFUS != EINVAL
+    case EDOOFUS:         return "EDOOFUS";
+    #endif
+    /* Linux, HP-UX, Cygwin */
+    #if defined EDOTDOT
+    case EDOTDOT:         return "EDOTDOT";
+    #endif
+    /* OSF/1 */
+    #if defined EDUPPKG
+    case EDUPPKG:         return "EDUPPKG";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED_ALREADY_OPEN
+    case ED_ALREADY_OPEN: return "ED_ALREADY_OPEN";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED_DEVICE_DOWN
+    case ED_DEVICE_DOWN:  return "ED_DEVICE_DOWN";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED_INVALID_OPERATION
+    case ED_INVALID_OPERATION:return "ED_INVALID_OPERATION";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED_INVALID_RECNUM
+    case ED_INVALID_RECNUM:return "ED_INVALID_RECNUM";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED_INVALID_SIZE
+    case ED_INVALID_SIZE: return "ED_INVALID_SIZE";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED_IO_ERROR
+    case ED_IO_ERROR:     return "ED_IO_ERROR";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED_NO_MEMORY
+    case ED_NO_MEMORY:    return "ED_NO_MEMORY";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED_NO_SUCH_DEVICE
+    case ED_NO_SUCH_DEVICE:return "ED_NO_SUCH_DEVICE";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED_READ_ONLY
+    case ED_READ_ONLY:    return "ED_READ_ONLY";
+    #endif
+    /* GNU/Hurd */
+    #if defined ED_WOULD_BLOCK
+    case ED_WOULD_BLOCK:  return "ED_WOULD_BLOCK";
+    #endif
+    /* IRIX */
+    #if defined EEMPTY
+    case EEMPTY:          return "EEMPTY";
+    #endif
+    /* OSF/1 */
+    #if defined EEMULATE
+    case EEMULATE:        return "EEMULATE";
+    #endif
+    /* IRIX */
+    #if defined EENDOFMINOR
+    case EENDOFMINOR:     return "EENDOFMINOR";
+    #endif
+    /* IRIX */
+    #if defined EENQUEUED
+    case EENQUEUED:       return "EENQUEUED";
+    #endif
+    /* OSF/1 */
+    #if defined EFAIL
+    case EFAIL:           return "EFAIL";
+    #endif
+    /* AIX */
+    #if defined EFORMAT
+    case EFORMAT:         return "EFORMAT";
+    #endif
+    /* Haiku */
+    #if defined EFPOS
+    case EFPOS:           return "EFPOS";
+    #endif
+    /* Solaris */
+    #if defined EFRAGS
+    case EFRAGS:          return "EFRAGS";
+    #endif
+    /* IRIX */
+    #if defined EFSCORRUPTED
+    case EFSCORRUPTED:    return "EFSCORRUPTED";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, OSF/1, Minix, Cygwin */
+    #if defined EFTYPE
+    case EFTYPE:          return "EFTYPE";
+    #endif
+    /* Minix */
+    #if defined EGENERIC
+    case EGENERIC:        return "EGENERIC";
+    #endif
+    /* GNU/Hurd */
+    #if defined EGRATUITOUS
+    case EGRATUITOUS:     return "EGRATUITOUS";
+    #endif
+    /* GNU/Hurd */
+    #if defined EGREGIOUS
+    case EGREGIOUS:       return "EGREGIOUS";
+    #endif
+    /* IRIX */
+    #if defined EGROUPLOOP
+    case EGROUPLOOP:      return "EGROUPLOOP";
+    #endif
+    /* Linux, Mac OS X, FreeBSD, NetBSD, OpenBSD, AIX, HP-UX, IRIX, OSF/1, Solaris, Minix, Haiku, Cygwin */
+    #if defined EHOSTDOWN
+    case EHOSTDOWN:       return "EHOSTDOWN";
+    #endif
+    /* Linux */
+    #if defined EHWPOISON
+    case EHWPOISON:       return "EHWPOISON";
+    #endif
+    /* GNU/Hurd */
+    #if defined EIEIO
+    case EIEIO:           return "EIEIO";
+    #endif
+    /* IRIX */
+    #if defined EINIT
+    case EINIT:           return "EINIT";
+    #endif
+    /* OSF/1 */
+    #if defined EINPROG
+    case EINPROG:         return "EINPROG";
+    #endif
+    /* IRIX */
+    #if defined EINVALMODE
+    case EINVALMODE:      return "EINVALMODE";
+    #endif
+    /* IRIX */
+    #if defined EINVALSTATE
+    case EINVALSTATE:     return "EINVALSTATE";
+    #endif
+    /* IRIX */
+    #if defined EINVALTIME
+    case EINVALTIME:      return "EINVALTIME";
+    #endif
+    /* IRIX */
+    #if defined EIORESID
+    case EIORESID:        return "EIORESID";
+    #endif
+    /* OpenBSD */
+    #if defined EIPSEC
+    case EIPSEC:          return "EIPSEC";
+    #endif
+    /* Linux, IRIX */
+    #if defined EISNAM
+    case EISNAM:          return "EISNAM";
+    #endif
+    /* IRIX */
+    #if defined EJOINED
+    case EJOINED:         return "EJOINED";
+    #endif
+    /* FreeBSD, OSF/1 */
+    #if defined EJUSTRETURN
+    case EJUSTRETURN:     return "EJUSTRETURN";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_ABORTED
+    case EKERN_ABORTED:   return "EKERN_ABORTED";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_FAILURE
+    case EKERN_FAILURE:   return "EKERN_FAILURE";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_INTERRUPTED
+    case EKERN_INTERRUPTED:return "EKERN_INTERRUPTED";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_INVALID_ADDRESS
+    case EKERN_INVALID_ADDRESS:return "EKERN_INVALID_ADDRESS";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_INVALID_ARGUMENT
+    case EKERN_INVALID_ARGUMENT:return "EKERN_INVALID_ARGUMENT";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_INVALID_CAPABILITY
+    case EKERN_INVALID_CAPABILITY:return "EKERN_INVALID_CAPABILITY";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_INVALID_HOST
+    case EKERN_INVALID_HOST:return "EKERN_INVALID_HOST";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_INVALID_NAME
+    case EKERN_INVALID_NAME:return "EKERN_INVALID_NAME";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_INVALID_RIGHT
+    case EKERN_INVALID_RIGHT:return "EKERN_INVALID_RIGHT";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_INVALID_TASK
+    case EKERN_INVALID_TASK:return "EKERN_INVALID_TASK";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_INVALID_VALUE
+    case EKERN_INVALID_VALUE:return "EKERN_INVALID_VALUE";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_MEMORY_ERROR
+    case EKERN_MEMORY_ERROR:return "EKERN_MEMORY_ERROR";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_MEMORY_FAILURE
+    case EKERN_MEMORY_FAILURE:return "EKERN_MEMORY_FAILURE";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_MEMORY_PRESENT
+    case EKERN_MEMORY_PRESENT:return "EKERN_MEMORY_PRESENT";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_NAME_EXISTS
+    case EKERN_NAME_EXISTS:return "EKERN_NAME_EXISTS";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_NOT_IN_SET
+    case EKERN_NOT_IN_SET:return "EKERN_NOT_IN_SET";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_NOT_RECEIVER
+    case EKERN_NOT_RECEIVER:return "EKERN_NOT_RECEIVER";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_NO_ACCESS
+    case EKERN_NO_ACCESS: return "EKERN_NO_ACCESS";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_NO_SPACE
+    case EKERN_NO_SPACE:  return "EKERN_NO_SPACE";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_PROTECTION_FAILURE
+    case EKERN_PROTECTION_FAILURE:return "EKERN_PROTECTION_FAILURE";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_RESOURCE_SHORTAGE
+    case EKERN_RESOURCE_SHORTAGE:return "EKERN_RESOURCE_SHORTAGE";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_RIGHT_EXISTS
+    case EKERN_RIGHT_EXISTS:return "EKERN_RIGHT_EXISTS";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_TERMINATED
+    case EKERN_TERMINATED:return "EKERN_TERMINATED";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_TIMEDOUT
+    case EKERN_TIMEDOUT:  return "EKERN_TIMEDOUT";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_UREFS_OVERFLOW
+    case EKERN_UREFS_OVERFLOW:return "EKERN_UREFS_OVERFLOW";
+    #endif
+    /* GNU/Hurd */
+    #if defined EKERN_WRITE_PROTECTION_FAILURE
+    case EKERN_WRITE_PROTECTION_FAILURE:return "EKERN_WRITE_PROTECTION_FAILURE";
+    #endif
+    /* Linux */
+    #if defined EKEYEXPIRED
+    case EKEYEXPIRED:     return "EKEYEXPIRED";
+    #endif
+    /* Linux */
+    #if defined EKEYREJECTED
+    case EKEYREJECTED:    return "EKEYREJECTED";
+    #endif
+    /* Linux */
+    #if defined EKEYREVOKED
+    case EKEYREVOKED:     return "EKEYREVOKED";
+    #endif
+    /* Linux, AIX, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined EL2HLT
+    case EL2HLT:          return "EL2HLT";
+    #endif
+    /* Linux, AIX, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined EL2NSYNC
+    case EL2NSYNC:        return "EL2NSYNC";
+    #endif
+    /* Linux, AIX, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined EL3HLT
+    case EL3HLT:          return "EL3HLT";
+    #endif
+    /* Linux, AIX, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined EL3RST
+    case EL3RST:          return "EL3RST";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, Minix */
+    #if defined ELAST && 0
+    case ELAST:           return "ELAST";
+    #endif
+    /* Cygwin */
+    #if defined ELBIN
+    case ELBIN:           return "ELBIN";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined ELIBACC
+    case ELIBACC:         return "ELIBACC";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined ELIBBAD
+    case ELIBBAD:         return "ELIBBAD";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined ELIBEXEC
+    case ELIBEXEC:        return "ELIBEXEC";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined ELIBMAX
+    case ELIBMAX:         return "ELIBMAX";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined ELIBSCN
+    case ELIBSCN:         return "ELIBSCN";
+    #endif
+    /* Linux, AIX, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined ELNRNG
+    case ELNRNG:          return "ELNRNG";
+    #endif
+    /* Minix */
+    #if defined ELOCKED
+    case ELOCKED:         return "ELOCKED";
+    #endif
+    /* Solaris */
+    #if defined ELOCKUNMAPPED
+    case ELOCKUNMAPPED:   return "ELOCKUNMAPPED";
+    #endif
+    /* IRIX */
+    #if defined ELOGINLIM
+    case ELOGINLIM:       return "ELOGINLIM";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_BODY_ERROR
+    case EMACH_RCV_BODY_ERROR:return "EMACH_RCV_BODY_ERROR";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_HEADER_ERROR
+    case EMACH_RCV_HEADER_ERROR:return "EMACH_RCV_HEADER_ERROR";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_INTERRUPTED
+    case EMACH_RCV_INTERRUPTED:return "EMACH_RCV_INTERRUPTED";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_INVALID_DATA
+    case EMACH_RCV_INVALID_DATA:return "EMACH_RCV_INVALID_DATA";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_INVALID_NAME
+    case EMACH_RCV_INVALID_NAME:return "EMACH_RCV_INVALID_NAME";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_INVALID_NOTIFY
+    case EMACH_RCV_INVALID_NOTIFY:return "EMACH_RCV_INVALID_NOTIFY";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_IN_PROGRESS
+    case EMACH_RCV_IN_PROGRESS:return "EMACH_RCV_IN_PROGRESS";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_IN_SET
+    case EMACH_RCV_IN_SET:return "EMACH_RCV_IN_SET";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_PORT_CHANGED
+    case EMACH_RCV_PORT_CHANGED:return "EMACH_RCV_PORT_CHANGED";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_PORT_DIED
+    case EMACH_RCV_PORT_DIED:return "EMACH_RCV_PORT_DIED";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_TIMED_OUT
+    case EMACH_RCV_TIMED_OUT:return "EMACH_RCV_TIMED_OUT";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_RCV_TOO_LARGE
+    case EMACH_RCV_TOO_LARGE:return "EMACH_RCV_TOO_LARGE";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_INTERRUPTED
+    case EMACH_SEND_INTERRUPTED:return "EMACH_SEND_INTERRUPTED";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_INVALID_DATA
+    case EMACH_SEND_INVALID_DATA:return "EMACH_SEND_INVALID_DATA";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_INVALID_DEST
+    case EMACH_SEND_INVALID_DEST:return "EMACH_SEND_INVALID_DEST";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_INVALID_HEADER
+    case EMACH_SEND_INVALID_HEADER:return "EMACH_SEND_INVALID_HEADER";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_INVALID_MEMORY
+    case EMACH_SEND_INVALID_MEMORY:return "EMACH_SEND_INVALID_MEMORY";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_INVALID_NOTIFY
+    case EMACH_SEND_INVALID_NOTIFY:return "EMACH_SEND_INVALID_NOTIFY";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_INVALID_REPLY
+    case EMACH_SEND_INVALID_REPLY:return "EMACH_SEND_INVALID_REPLY";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_INVALID_RIGHT
+    case EMACH_SEND_INVALID_RIGHT:return "EMACH_SEND_INVALID_RIGHT";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_INVALID_TYPE
+    case EMACH_SEND_INVALID_TYPE:return "EMACH_SEND_INVALID_TYPE";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_IN_PROGRESS
+    case EMACH_SEND_IN_PROGRESS:return "EMACH_SEND_IN_PROGRESS";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_MSG_TOO_SMALL
+    case EMACH_SEND_MSG_TOO_SMALL:return "EMACH_SEND_MSG_TOO_SMALL";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_NOTIFY_IN_PROGRESS
+    case EMACH_SEND_NOTIFY_IN_PROGRESS:return "EMACH_SEND_NOTIFY_IN_PROGRESS";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_NO_BUFFER
+    case EMACH_SEND_NO_BUFFER:return "EMACH_SEND_NO_BUFFER";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_NO_NOTIFY
+    case EMACH_SEND_NO_NOTIFY:return "EMACH_SEND_NO_NOTIFY";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_TIMED_OUT
+    case EMACH_SEND_TIMED_OUT:return "EMACH_SEND_TIMED_OUT";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMACH_SEND_WILL_NOTIFY
+    case EMACH_SEND_WILL_NOTIFY:return "EMACH_SEND_WILL_NOTIFY";
+    #endif
+    /* AIX, OSF/1 */
+    #if defined EMEDIA
+    case EMEDIA:          return "EMEDIA";
+    #endif
+    /* Linux, OpenBSD */
+    #if defined EMEDIUMTYPE
+    case EMEDIUMTYPE:     return "EMEDIUMTYPE";
+    #endif
+    /* IRIX */
+    #if defined EMEMRETRY
+    case EMEMRETRY:       return "EMEMRETRY";
+    #endif
+    /* IRIX */
+    #if defined EMIGRATED
+    case EMIGRATED:       return "EMIGRATED";
+    #endif
+    /* IRIX */
+    #if defined EMIGRATING
+    case EMIGRATING:      return "EMIGRATING";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMIG_ARRAY_TOO_LARGE
+    case EMIG_ARRAY_TOO_LARGE:return "EMIG_ARRAY_TOO_LARGE";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMIG_BAD_ARGUMENTS
+    case EMIG_BAD_ARGUMENTS:return "EMIG_BAD_ARGUMENTS";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMIG_BAD_ID
+    case EMIG_BAD_ID:     return "EMIG_BAD_ID";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMIG_DESTROY_REQUEST
+    case EMIG_DESTROY_REQUEST:return "EMIG_DESTROY_REQUEST";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMIG_EXCEPTION
+    case EMIG_EXCEPTION:  return "EMIG_EXCEPTION";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMIG_NO_REPLY
+    case EMIG_NO_REPLY:   return "EMIG_NO_REPLY";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMIG_REMOTE_ERROR
+    case EMIG_REMOTE_ERROR:return "EMIG_REMOTE_ERROR";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMIG_REPLY_MISMATCH
+    case EMIG_REPLY_MISMATCH:return "EMIG_REPLY_MISMATCH";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMIG_SERVER_DIED
+    case EMIG_SERVER_DIED:return "EMIG_SERVER_DIED";
+    #endif
+    /* GNU/Hurd */
+    #if defined EMIG_TYPE_ERROR
+    case EMIG_TYPE_ERROR: return "EMIG_TYPE_ERROR";
+    #endif
+    /* OSF/1 */
+    #if defined EMTIMERS
+    case EMTIMERS:        return "EMTIMERS";
+    #endif
+    /* IRIX */
+    #if defined EMUSTRUN
+    case EMUSTRUN:        return "EMUSTRUN";
+    #endif
+    /* Linux, IRIX */
+    #if defined ENAVAIL
+    case ENAVAIL:         return "ENAVAIL";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, Minix */
+    #if defined ENEEDAUTH
+    case ENEEDAUTH:       return "ENEEDAUTH";
+    #endif
+    /* IRIX */
+    #if defined ENFSREMOTE
+    case ENFSREMOTE:      return "ENFSREMOTE";
+    #endif
+    /* Cygwin */
+    #if defined ENMFILE
+    case ENMFILE:         return "ENMFILE";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined ENOANO && ENOANO != ENOKEY
+    case ENOANO:          return "ENOANO";
+    #endif
+    /* IRIX */
+    #if defined ENOATTACH
+    case ENOATTACH:       return "ENOATTACH";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, AIX, IRIX, Minix */
+    #if defined ENOATTR
+    case ENOATTR:         return "ENOATTR";
+    #endif
+    /* IRIX */
+    #if defined ENOBWD
+    case ENOBWD:          return "ENOBWD";
+    #endif
+    /* MirBSD */
+    #if defined ENOCOFFEE
+    case ENOCOFFEE:       return "ENOCOFFEE";
+    #endif
+    /* Minix */
+    #if defined ENOCONN
+    case ENOCONN:         return "ENOCONN";
+    #endif
+    /* AIX */
+    #if defined ENOCONNECT
+    case ENOCONNECT:      return "ENOCONNECT";
+    #endif
+    /* Linux, AIX, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined ENOCSI
+    case ENOCSI:          return "ENOCSI";
+    #endif
+    /* IRIX */
+    #if defined ENOEXIST
+    case ENOEXIST:        return "ENOEXIST";
+    #endif
+    /* IRIX */
+    #if defined ENOINTRGROUP
+    case ENOINTRGROUP:    return "ENOINTRGROUP";
+    #endif
+    /* FreeBSD */
+    #if defined ENOIOCTL
+    case ENOIOCTL:        return "ENOIOCTL";
+    #endif
+    /* Linux */
+    #if defined ENOKEY
+    case ENOKEY:          return "ENOKEY";
+    #endif
+    /* IRIX */
+    #if defined ENOLIMFILE
+    case ENOLIMFILE:      return "ENOLIMFILE";
+    #endif
+    /* HP-UX */
+    #if defined ENOLOAD
+    case ENOLOAD:         return "ENOLOAD";
+    #endif
+    /* IRIX */
+    #if defined ENOLOGIN
+    case ENOLOGIN:        return "ENOLOGIN";
+    #endif
+    /* HP-UX */
+    #if defined ENOMATCH
+    case ENOMATCH:        return "ENOMATCH";
+    #endif
+    /* Linux, OpenBSD, Cygwin */
+    #if defined ENOMEDIUM
+    case ENOMEDIUM:       return "ENOMEDIUM";
+    #endif
+    /* IRIX */
+    #if defined ENOMESSAGE
+    case ENOMESSAGE:      return "ENOMESSAGE";
+    #endif
+    /* Linux, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined ENONET
+    case ENONET:          return "ENONET";
+    #endif
+    /* Linux, HP-UX, IRIX, OSF/1, Solaris, Cygwin */
+    #if defined ENOPKG
+    case ENOPKG:          return "ENOPKG";
+    #endif
+    /* Mac OS X */
+    #if defined ENOPOLICY
+    case ENOPOLICY:       return "ENOPOLICY";
+    #endif
+    /* IRIX */
+    #if defined ENOPROC
+    case ENOPROC:         return "ENOPROC";
+    #endif
+    /* HP-UX */
+    #if defined ENOREG
+    case ENOREG:          return "ENOREG";
+    #endif
+    /* IRIX */
+    #if defined ENOSERVICE
+    case ENOSERVICE:      return "ENOSERVICE";
+    #endif
+    /* Cygwin */
+    #if defined ENOSHARE
+    case ENOSHARE:        return "ENOSHARE";
+    #endif
+    /* HP-UX, OSF/1 */
+    #if defined ENOSYM
+    case ENOSYM:          return "ENOSYM";
+    #endif
+    /* Solaris */
+    #if defined ENOTACTIVE
+    case ENOTACTIVE:      return "ENOTACTIVE";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, AIX, HP-UX, IRIX, OSF/1, Solaris, Minix, Cygwin */
+    #if defined ENOTBLK
+    case ENOTBLK:         return "ENOTBLK";
+    #endif
+    /* FreeBSD */
+    #if defined ENOTCAPABLE
+    case ENOTCAPABLE:     return "ENOTCAPABLE";
+    #endif
+    /* IRIX */
+    #if defined ENOTCONTROLLER
+    case ENOTCONTROLLER:  return "ENOTCONTROLLER";
+    #endif
+    /* IRIX */
+    #if defined ENOTENQUEUED
+    case ENOTENQUEUED:    return "ENOTENQUEUED";
+    #endif
+    /* IRIX */
+    #if defined ENOTJOINED
+    case ENOTJOINED:      return "ENOTJOINED";
+    #endif
+    /* Linux, IRIX */
+    #if defined ENOTNAM
+    case ENOTNAM:         return "ENOTNAM";
+    #endif
+    /* AIX, Minix */
+    #if defined ENOTREADY
+    case ENOTREADY:       return "ENOTREADY";
+    #endif
+    /* AIX */
+    #if defined ENOTRUST
+    case ENOTRUST:        return "ENOTRUST";
+    #endif
+    /* IRIX */
+    #if defined ENOTSTOPPED
+    case ENOTSTOPPED:     return "ENOTSTOPPED";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined ENOTUNIQ
+    case ENOTUNIQ:        return "ENOTUNIQ";
+    #endif
+    /* HP-UX */
+    #if defined ENOUNLD
+    case ENOUNLD:         return "ENOUNLD";
+    #endif
+    /* HP-UX */
+    #if defined ENOUNREG
+    case ENOUNREG:        return "ENOUNREG";
+    #endif
+    /* Minix */
+    #if defined ENOURG
+    case ENOURG:          return "ENOURG";
+    #endif
+    /* native Windows */
+    #if defined EOTHER
+    case EOTHER:          return "EOTHER";
+    #endif
+    /* Minix */
+    #if defined EPACKSIZE
+    case EPACKSIZE:       return "EPACKSIZE";
+    #endif
+    /* Linux, Mac OS X, FreeBSD, NetBSD, OpenBSD, AIX, HP-UX, IRIX, OSF/1, Solaris, Minix, Haiku, Cygwin */
+    #if defined EPFNOSUPPORT
+    case EPFNOSUPPORT:    return "EPFNOSUPPORT";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, AIX, IRIX, OSF/1, Minix, Cygwin */
+    #if defined EPROCLIM
+    case EPROCLIM:        return "EPROCLIM";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, OSF/1, Minix */
+    #if defined EPROCUNAVAIL
+    case EPROCUNAVAIL:    return "EPROCUNAVAIL";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, OSF/1, Minix */
+    #if defined EPROGMISMATCH
+    case EPROGMISMATCH:   return "EPROGMISMATCH";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, OSF/1, Minix */
+    #if defined EPROGUNAVAIL
+    case EPROGUNAVAIL:    return "EPROGUNAVAIL";
+    #endif
+    /* Mac OS X */
+    #if defined EPWROFF
+    case EPWROFF:         return "EPWROFF";
+    #endif
+    /* Mac OS X */
+    #if defined EQFULL
+    case EQFULL:          return "EQFULL";
+    #endif
+    /* HP-UX */
+    #if defined ERELOC
+    case ERELOC:          return "ERELOC";
+    #endif
+    /* OSF/1 */
+    #if defined ERELOCATED
+    case ERELOCATED:      return "ERELOCATED";
+    #endif
+    /* FreeBSD */
+    #if defined ERELOOKUP
+    case ERELOOKUP:       return "ERELOOKUP";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined EREMCHG
+    case EREMCHG:         return "EREMCHG";
+    #endif
+    /* IRIX */
+    #if defined EREMDEV
+    case EREMDEV:         return "EREMDEV";
+    #endif
+    /* Linux, Mac OS X, FreeBSD, NetBSD, OpenBSD, AIX, HP-UX, IRIX, OSF/1, Solaris, Minix, Haiku, Cygwin */
+    #if defined EREMOTE
+    case EREMOTE:         return "EREMOTE";
+    #endif
+    /* Linux, IRIX */
+    #if defined EREMOTEIO
+    case EREMOTEIO:       return "EREMOTEIO";
+    #endif
+    /* HP-UX */
+    #if defined EREMOTERELEASE
+    case EREMOTERELEASE:  return "EREMOTERELEASE";
+    #endif
+    /* Linux, FreeBSD, AIX, IRIX, OSF/1, Solaris, Minix */
+    #if defined ERESTART
+    case ERESTART:        return "ERESTART";
+    #endif
+    /* Linux */
+    #if defined ERFKILL
+    case ERFKILL:         return "ERFKILL";
+    #endif
+    /* Mac OS X, FreeBSD, NetBSD, OpenBSD, OSF/1, Minix */
+    #if defined ERPCMISMATCH
+    case ERPCMISMATCH:    return "ERPCMISMATCH";
+    #endif
+    /* AIX */
+    #if defined ESAD
+    case ESAD:            return "ESAD";
+    #endif
+    /* Mac OS X */
+    #if defined ESHLIBVERS
+    case ESHLIBVERS:      return "ESHLIBVERS";
+    #endif
+    /* Linux, Mac OS X, FreeBSD, NetBSD, OpenBSD, AIX, HP-UX, IRIX, OSF/1, Solaris, Minix, Haiku, Cygwin */
+    #if defined ESHUTDOWN
+    case ESHUTDOWN:       return "ESHUTDOWN";
+    #endif
+    /* Haiku */
+    #if defined ESIGPARM
+    case ESIGPARM:        return "ESIGPARM";
+    #endif
+    /* Linux, Mac OS X, FreeBSD, NetBSD, OpenBSD, AIX, HP-UX, IRIX, OSF/1, Solaris, Minix, Cygwin */
+    #if defined ESOCKTNOSUPPORT
+    case ESOCKTNOSUPPORT: return "ESOCKTNOSUPPORT";
+    #endif
+    /* AIX, OSF/1 */
+    #if defined ESOFT
+    case ESOFT:           return "ESOFT";
+    #endif
+    /* Linux, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined ESRMNT
+    case ESRMNT:          return "ESRMNT";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined ESTRPIPE
+    case ESTRPIPE:        return "ESTRPIPE";
+    #endif
+    /* OSF/1 */
+    #if defined ESUCCESS
+    case ESUCCESS:        return "ESUCCESS";
+    #endif
+    /* AIX */
+    #if defined ESYSERROR
+    case ESYSERROR:       return "ESYSERROR";
+    #endif
+    /* Linux, Mac OS X, FreeBSD, NetBSD, OpenBSD, AIX, HP-UX, IRIX, OSF/1, Solaris, Minix, Cygwin */
+    #if defined ETOOMANYREFS
+    case ETOOMANYREFS:    return "ETOOMANYREFS";
+    #endif
+    /* Minix */
+    #if defined ETRAPDENIED
+    case ETRAPDENIED:     return "ETRAPDENIED";
+    #endif
+    /* Linux, IRIX */
+    #if defined EUCLEAN
+    case EUCLEAN:         return "EUCLEAN";
+    #endif
+    /* Linux, AIX, HP-UX, IRIX, Solaris, Cygwin */
+    #if defined EUNATCH
+    case EUNATCH:         return "EUNATCH";
+    #endif
+    /* Minix */
+    #if defined EURG
+    case EURG:            return "EURG";
+    #endif
+    /* Linux, Mac OS X, FreeBSD, NetBSD, OpenBSD, AIX, HP-UX, IRIX, OSF/1, Solaris, Minix, Cygwin */
+    #if defined EUSERS
+    case EUSERS:          return "EUSERS";
+    #endif
+    /* OSF/1 */
+    #if defined EVERSION
+    case EVERSION:        return "EVERSION";
+    #endif
+    /* IRIX */
+    #if defined EWRONGFS
+    case EWRONGFS:        return "EWRONGFS";
+    #endif
+    /* AIX */
+    #if defined EWRPROTECT
+    case EWRPROTECT:      return "EWRPROTECT";
+    #endif
+    /* Linux, IRIX, Solaris, Cygwin */
+    #if defined EXFULL
+    case EXFULL:          return "EXFULL";
+    #endif
+
+    default:
+      return NULL;
+    }
+}
diff --git a/src/common/libmissing/strerrorname_np.h b/src/common/libmissing/strerrorname_np.h
new file mode 100644
index 000000000000..5cdf05ef8748
--- /dev/null
+++ b/src/common/libmissing/strerrorname_np.h
@@ -0,0 +1,18 @@
+/* A GNU-like <string.h>.
+
+   Copyright (C) 1995-1996, 2001-2023 Free Software Foundation, Inc.
+
+   This file is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as
+   published by the Free Software Foundation; either version 2.1 of the
+   License, or (at your option) any later version.
+
+   This file is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with this program.  If not, see <https://www.gnu.org/licenses/>.  */
+
+const char * strerrorname_np (int errnum);
diff --git a/src/common/libmissing/strlcat.c b/src/common/libmissing/strlcat.c
new file mode 100644
index 000000000000..2b6f1adbb4c4
--- /dev/null
+++ b/src/common/libmissing/strlcat.c
@@ -0,0 +1,59 @@
+/*	$OpenBSD: strlcat.c,v 1.11 2003/06/17 21:56:24 millert Exp $	*/
+
+/*
+ * Copyright (c) 1998 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char *rcsid = "$OpenBSD: strlcat.c,v 1.11 2003/06/17 21:56:24 millert Exp $";
+#endif /* LIBC_SCCS and not lint */
+
+#include <sys/types.h>
+#include <string.h>
+
+/*
+ * Appends src to string dst of size siz (unlike strncat, siz is the
+ * full size of dst, not space left).  At most siz-1 characters
+ * will be copied.  Always NUL terminates (unless siz <= strlen(dst)).
+ * Returns strlen(src) + MIN(siz, strlen(initial dst)).
+ * If retval >= siz, truncation occurred.
+ */
+size_t
+strlcat(char *dst, const char *src, size_t siz)
+{
+	register char *d = dst;
+	register const char *s = src;
+	register size_t n = siz;
+	size_t dlen;
+
+	/* Find the end of dst and adjust bytes left but don't go past end */
+	while (n-- != 0 && *d != '\0')
+		d++;
+	dlen = d - dst;
+	n = siz - dlen;
+
+	if (n == 0)
+		return(dlen + strlen(s));
+	while (*s != '\0') {
+		if (n != 1) {
+			*d++ = *s;
+			n--;
+		}
+		s++;
+	}
+	*d = '\0';
+
+	return(dlen + (s - src));	/* count does not include NUL */
+}
diff --git a/src/common/libmissing/strlcat.h b/src/common/libmissing/strlcat.h
new file mode 100644
index 000000000000..e2e2d3492d1a
--- /dev/null
+++ b/src/common/libmissing/strlcat.h
@@ -0,0 +1,17 @@
+#if HAVE_CONFIG_H
+#  include "config.h"
+#endif /* HAVE_CONFIG_H */
+
+#if !_MISSING_STRLCAT_H
+#define _MISSING_STRLCAT_H
+
+size_t strlcat(char *dst, const char *src, size_t siz);
+/*
+ *  Appends src to string dst of size siz (unlike strncat, siz is the
+ *    full size of dst, not space left).  At most siz-1 characters
+ *    will be copied.  Always NUL terminates (unless siz <= strlen(dst)).
+ *  Returns strlen(src) + MIN(siz, strlen(initial dst)).
+ *  If retval >= siz, truncation occurred.
+ */
+
+#endif // !_MISSING_STRLCAT
diff --git a/src/common/libmissing/strlcpy.c b/src/common/libmissing/strlcpy.c
new file mode 100644
index 000000000000..82cbfac016d6
--- /dev/null
+++ b/src/common/libmissing/strlcpy.c
@@ -0,0 +1,55 @@
+/*	$OpenBSD: strlcpy.c,v 1.8 2003/06/17 21:56:24 millert Exp $	*/
+
+/*
+ * Copyright (c) 1998 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char *rcsid = "$OpenBSD: strlcpy.c,v 1.8 2003/06/17 21:56:24 millert Exp $";
+#endif /* LIBC_SCCS and not lint */
+
+#include <sys/types.h>
+#include <string.h>
+
+/*
+ * Copy src to string dst of size siz.  At most siz-1 characters
+ * will be copied.  Always NUL terminates (unless siz == 0).
+ * Returns strlen(src); if retval >= siz, truncation occurred.
+ */
+size_t
+strlcpy(char *dst, const char *src, size_t siz)
+{
+	register char *d = dst;
+	register const char *s = src;
+	register size_t n = siz;
+
+	/* Copy as many bytes as will fit */
+	if (n != 0 && --n != 0) {
+		do {
+			if ((*d++ = *s++) == 0)
+				break;
+		} while (--n != 0);
+	}
+
+	/* Not enough room in dst, add NUL and traverse rest of src */
+	if (n == 0) {
+		if (siz != 0)
+			*d = '\0';		/* NUL-terminate dst */
+		while (*s++)
+			;
+	}
+
+	return(s - src - 1);	/* count does not include NUL */
+}
diff --git a/src/common/libmissing/strlcpy.h b/src/common/libmissing/strlcpy.h
new file mode 100644
index 000000000000..336ee26077cb
--- /dev/null
+++ b/src/common/libmissing/strlcpy.h
@@ -0,0 +1,14 @@
+#if HAVE_CONFIG_H
+#  include "config.h"
+#endif /* HAVE_CONFIG_H */
+
+#if !_UTIL_STRLCPY_H
+#define _UTIL_STRLCPY_H
+
+size_t strlcpy(char *dst, const char *src, size_t siz);
+/*
+ *  Copy src to string dst of size siz.  At most siz-1 characters
+ *    will be copied.  Always NUL terminates (unless siz == 0).
+ *  Returns strlen(src); if retval >= siz, truncation occurred.
+ */
+#endif /* !_UTIL_STRLCPY_H */
diff --git a/src/common/liboptparse/Makefile.am b/src/common/liboptparse/Makefile.am
index 300fcb622c9c..58ea4dcc2a2f 100644
--- a/src/common/liboptparse/Makefile.am
+++ b/src/common/liboptparse/Makefile.am
@@ -6,22 +6,16 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux
 
 noinst_LTLIBRARIES = liboptparse.la
 fluxinclude_HEADERS = optparse.h
 liboptparse_la_SOURCES = optparse.c getopt.c getopt_int.h getopt.h
 
-liboptparse_la_CPPFLAGS = \
-	$(AM_CPPFLAGS)
-liboptparse_la_LDFLAGS = \
-	-avoid-version -module -shared -export-dynamic \
-	$(AM_LDFLAGS)
-
-
 TESTS = test_optparse.t
 
 check_PROGRAMS = $(TESTS)
@@ -35,6 +29,7 @@ test_optparse_t_CPPFLAGS = $(AM_CPPFLAGS)
 test_optparse_t_LDADD = \
 	$(top_builddir)/src/common/libtap/libtap.la \
 	$(top_builddir)/src/common/liboptparse/liboptparse.la \
-	$(top_builddir)/src/common/liblsd/liblsd.la \
+	$(top_builddir)/src/common/libczmqcontainers/libczmqcontainers.la \
 	$(top_builddir)/src/common/libutil/libutil.la \
-	$(ZMQ_LIBS) $(LIBPTHREAD)
+	$(top_builddir)/src/common/libmissing/libmissing.la \
+	$(LIBPTHREAD)
diff --git a/src/common/liboptparse/getopt.c b/src/common/liboptparse/getopt.c
index efa34b358cd5..43d64e2f28da 100644
--- a/src/common/liboptparse/getopt.c
+++ b/src/common/liboptparse/getopt.c
@@ -59,10 +59,16 @@
    contain conflicting prototypes for getopt.  */
 # include <stdlib.h>
 # include <unistd.h>
+#else
+# include <alloca.h>
 #endif	/* GNU C library.  */
 
 #include <string.h>
 
+#ifndef HAVE_MEMPCPY
+#include "src/common/libmissing/mempcpy.h"
+#endif
+
 #ifdef VMS
 # include <unixlib.h>
 #endif
@@ -148,7 +154,7 @@ static struct _getopt_data getopt_data;
    whose names are inconsistent.  */
 
 #ifndef getenv
-extern char *getenv ();
+extern char *getenv (const char *name);
 #endif
 
 #endif /* not __GNU_LIBRARY__ */
diff --git a/src/common/liboptparse/getopt.h b/src/common/liboptparse/getopt.h
index e10b0389e9d5..b36c0e84fe7f 100644
--- a/src/common/liboptparse/getopt.h
+++ b/src/common/liboptparse/getopt.h
@@ -167,7 +167,7 @@ extern int __posix_getopt (int ___argc, char *const *___argv,
 #  endif
 # endif
 #else /* not __GNU_LIBRARY__ */
-extern int getopt ();
+extern int getopt (int argc, char *const argv[], const char *optstring);
 #endif /* __GNU_LIBRARY__ */
 
 #ifndef __need_getopt
diff --git a/src/common/liboptparse/optparse.c b/src/common/liboptparse/optparse.c
index ee1be88a2e14..b8690c7a6ad3 100644
--- a/src/common/liboptparse/optparse.c
+++ b/src/common/liboptparse/optparse.c
@@ -18,12 +18,18 @@
 #include <sys/param.h>
 #include <ctype.h>
 #include <stdarg.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <math.h>
 
-#include <czmq.h>
-
-#include "src/common/liblsd/list.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/fsd.h"
+#include "src/common/libutil/parse_size.h"
+#include "ccan/str/str.h"
+
 #include "optparse.h"
 #include "getopt.h"
 #include "getopt_int.h"
@@ -50,12 +56,15 @@ struct opt_parser {
     int            left_margin;     /* Size of --help output left margin    */
     int            option_width;    /* Width of --help output for optiion   */
     int            current_group;   /* Current option group number          */
-    List           option_list;     /* List of options for this program    */
+    zlist_t *      option_list;     /* List of options for this program     */
 
     unsigned int   skip_subcmds:1;  /* Do not Print subcommands in --help   */
     unsigned int   no_options:1;    /* Skip option processing for subcmd    */
     unsigned int   hidden:1;        /* If subcmd, skip in --help output     */
     unsigned int   posixly_correct:1; /* Value of GNU getopt posixly correct*/
+    unsigned int   sorted:1;        /* Options,subcmds are sorted in usage  */
+
+    int            seq;             /* If subcommand, insertion sequence    */
 
     zhash_t *      dhash;           /* Hash of ancillary data               */
 
@@ -69,9 +78,9 @@ struct opt_parser {
  */
 struct option_info {
     struct optparse_option * p_opt;   /* Copy of program option structure  */
-    List                    optargs;  /* If non-NULL, the option argument(s) */
-    const char *            optarg;   /* Pointer to last element in optargs  */
-    ListIterator            argi;     /* iterator for optargs */
+    zlist_t *               optargs;  /* If non-NULL, the option argument(s) */
+    const char *            optarg;   /* Pointer to last element in optargs */
+    int                     seq;      /* Sequence in which option was added  */
 
     unsigned int            found;    /* number of times we saw this option */
 
@@ -87,74 +96,78 @@ struct option_info {
 
 static void optparse_option_destroy (struct optparse_option *o)
 {
-    if (o == NULL)
-        return;
-    free ((void *)o->name);
-    free ((void *)o->arginfo);
-    free ((void *)o->usage);
-    free (o);
+    if (o) {
+        free ((void *)o->name);
+        free ((void *)o->arginfo);
+        free ((void *)o->usage);
+        free (o);
+    }
 }
 
 static struct optparse_option *
 optparse_option_dup (const struct optparse_option *src)
 {
-    struct optparse_option *o = malloc (sizeof (*o));
-    if (o != NULL) {
-        memset (o, 0, sizeof (*o));
-        if (src->name)
-            o->name =    strdup (src->name);
-        if (src->arginfo)
-            o->arginfo = strdup (src->arginfo);
-        if (src->usage)
-            o->usage =   strdup (src->usage);
-        o->key =     src->key;
-        o->group =   src->group;
-        o->has_arg = src->has_arg;
-        o->flags =   src->flags;
-        o->cb  =     src->cb;
-    }
+    struct optparse_option *o = calloc (1, sizeof (*o));
+    if (o == NULL)
+        return NULL;
+    if ((src->name
+        && !(o->name = strdup (src->name)))
+        || (src->arginfo && !(o->arginfo = strdup (src->arginfo)))
+        || (src->usage && !(o->usage = strdup (src->usage))))
+        goto err;
+
+    o->key = src->key;
+    o->group = src->group;
+    o->has_arg = src->has_arg;
+    o->flags = src->flags;
+    o->cb = src->cb;
+
     return (o);
-}
 
-static struct option_info *option_info_create (const struct optparse_option *o)
-{
-    struct option_info *c = malloc (sizeof (*c));
-    if (c != NULL) {
-        memset (c, 0, sizeof (*c));
-        c->found = 0;
-        c->optargs = NULL;
-        c->optarg = NULL;
-        c->argi = NULL;
-        c->p_opt = optparse_option_dup (o);
-        if (!o->name)
-            c->isdoc = 1;
-        if (o->flags & OPTPARSE_OPT_AUTOSPLIT)
-            c->autosplit = 1;
-        if (o->flags & OPTPARSE_OPT_HIDDEN)
-            c->hidden = 1;
-    }
-    return (c);
+err:
+    optparse_option_destroy (o);
+    return NULL;
 }
 
 static void option_info_destroy (struct option_info *c)
 {
     optparse_option_destroy (c->p_opt);
-    if (c->argi)
-        list_iterator_destroy (c->argi);
     if (c->optargs)
-        list_destroy (c->optargs);
+        zlist_destroy (&c->optargs);
     c->optarg = NULL;
     free (c);
 }
 
+static struct option_info *option_info_create (const struct optparse_option *o)
+{
+    struct option_info *c = calloc (1, sizeof (*c));
+    if (!c)
+        return NULL;
+
+    if (!(c->p_opt = optparse_option_dup (o)))
+        goto err;
+    if (!o->name)
+        c->isdoc = 1;
+    if (o->flags & OPTPARSE_OPT_AUTOSPLIT)
+        c->autosplit = 1;
+    if (o->flags & OPTPARSE_OPT_HIDDEN)
+        c->hidden = 1;
+    return (c);
+err:
+    option_info_destroy (c);
+    return NULL;
+}
+
 /*
  *  Sort function for option_info structures.
  *   We sort first by group, then sort "doc" strings first in group
  *   then options alphabetically by key if alphanumeric, otherwise
  *   by option name.
  */
-static int option_info_cmp (struct option_info *x, struct option_info *y)
+static int option_info_cmp (void *arg1, void *arg2)
 {
+    const struct option_info *x = arg1;
+    const struct option_info *y = arg2;
     const struct optparse_option *o1 = x->p_opt;
     const struct optparse_option *o2 = y->p_opt;
 
@@ -165,9 +178,9 @@ static int option_info_cmp (struct option_info *x, struct option_info *y)
             return (-1);
         else if (y->isdoc)
             return (1);
-        else if (strcmp (o1->name, "help") == 0)
+        else if (streq (o1->name, "help"))
             return -1;
-        else if (strcmp (o2->name, "help") == 0)
+        else if (streq (o2->name, "help"))
             return 1;
         else if (isalnum (o1->key) && isalnum (o2->key))
             return (o1->key - o2->key);
@@ -177,39 +190,69 @@ static int option_info_cmp (struct option_info *x, struct option_info *y)
     return (o1->group - o2->group);
 }
 
-
-/*
- *  List find function for option_info structures:
+/* As above but preserve original sequence of options
  */
-static int option_info_cmp_name (struct option_info *o, const char *name)
+static int option_info_seq (void *arg1, void *arg2)
 {
-    if (!o->p_opt->name)
-        return (0);
-    return (strcmp (o->p_opt->name, name) == 0);
+    const struct option_info *x = arg1;
+    const struct option_info *y = arg2;
+    const struct optparse_option *o1 = x->p_opt;
+    const struct optparse_option *o2 = y->p_opt;
+
+    if (o1->group == o2->group) {
+        if (x->isdoc && y->isdoc)
+            return (0);
+        else if (x->isdoc)
+            return (-1);
+        else if (y->isdoc)
+            return (1);
+        else
+            return x->seq - y->seq;
+    }
+    return (o1->group - o2->group);
 }
 
+
 /*
  *  Return option_info structure for option with [name]
  */
 static struct option_info *find_option_info (optparse_t *p, const char *name)
 {
-    return (list_find_first (p->option_list,
-                             (ListFindF) option_info_cmp_name,
-                             (void *) name));
+    struct option_info *o;
+    if (name == NULL)
+        return NULL;
+
+    o = zlist_first (p->option_list);
+    while (o) {
+        if (o->p_opt->name != NULL
+            && streq (o->p_opt->name, name))
+            return o;
+        o = zlist_next (p->option_list);
+    }
+    return NULL;
+}
+
+static struct option_info *find_option_by_val (optparse_t *p, int val)
+{
+    struct option_info *o = zlist_first (p->option_list);
+    while (o) {
+        if (o->p_opt->key == val)
+            return o;
+        o = zlist_next (p->option_list);
+    }
+    return NULL;
 }
 
 /*   Remove the options in table [opts], up to but not including [end]
  *    If [end] is NULL, remove all options.
  */
 static void option_table_remove (optparse_t *p,
-        struct optparse_option const opts[],
-        const struct optparse_option *end)
+                                 struct optparse_option const opts[],
+                                 const struct optparse_option *end)
 {
     const struct optparse_option *o = opts;
-
-    while (o->usage || (end && o != end))
+    while (o->usage && o != end)
         optparse_remove_option (p, (o++)->name);
-
     return;
 }
 
@@ -227,19 +270,22 @@ static optparse_err_t optparse_set_log_fn (optparse_t *p, opt_log_f fn)
     return (0);
 }
 
-static optparse_err_t optparse_set_fatalerr_fn (optparse_t *p, opt_fatalerr_f fn)
+static optparse_err_t optparse_set_fatalerr_fn (optparse_t *p,
+                                                opt_fatalerr_f fn)
 {
     p->fatalerr_fn = fn;
     return (0);
 }
 
-static optparse_err_t optparse_set_fatalerr_handle (optparse_t *p, void *handle)
+static optparse_err_t optparse_set_fatalerr_handle (optparse_t *p,
+                                                    void *handle)
 {
     p->fatalerr_handle = handle;
     return (0);
 }
 
-static optparse_err_t optparse_set_option_cb (optparse_t *p, const char *name,
+static optparse_err_t optparse_set_option_cb (optparse_t *p,
+                                              const char *name,
                                               optparse_cb_f fn)
 {
     struct option_info *o;
@@ -268,7 +314,7 @@ static int log_stderr (const char *fmt, ...)
 }
 
 /*
- * Default fatalerr funciton.
+ * Default fatalerr function.
  */
 static int fatal_exit (void *h, int exit_code)
 {
@@ -280,10 +326,12 @@ static const char * optparse_fullname (optparse_t *p)
 {
     if (!p->fullname) {
         char buf [1024];
-        snprintf (buf, sizeof (buf) - 1, "%s%s%s",
-                p->parent ? optparse_fullname (p->parent) :"",
-                p->parent ? " " : "",
-                p->program_name);
+        snprintf (buf,
+                  sizeof (buf) - 1,
+                  "%s%s%s",
+                  p->parent ? optparse_fullname (p->parent) :"",
+                  p->parent ? " " : "",
+                  p->program_name);
         p->fullname = strdup (buf);
     }
     return (p->fullname);
@@ -345,34 +393,34 @@ static int opt_init (struct option *opt, struct optparse_option *o)
 /*
  *  Find next eligible place to split a line.
  */
-static char *find_word_boundary(char *str, char *from, char **next)
+static char *find_word_boundary (char *str, char *from, char **next)
 {
-        char *p = from;
+    char *p = from;
 
-        /*
-         * Back up past any non-whitespace if we are pointing in
-         *  the middle of a word.
-         */
-        while ((p != str) && !isspace ((int)*p))
-                --p;
+    /*
+     * Back up past any non-whitespace if we are pointing in
+     *  the middle of a word.
+     */
+    while ((p != str) && !isspace ((int)*p))
+        --p;
 
-        /*
-         * Next holds next word boundary
-         */
-        *next = p+1;
+    /*
+     * Next holds next word boundary
+     */
+    *next = p+1;
 
-        /*
-         * Now move back to the end of the previous word
-         */
-        while ((p != str) && isspace ((int)*p))
-                --p;
+    /*
+     * Now move back to the end of the previous word
+     */
+    while ((p != str) && isspace ((int)*p))
+        --p;
 
-        if (p == str) {
-                *next = str;
-                return (NULL);
-        }
+    if (p == str) {
+        *next = str;
+        return (NULL);
+    }
 
-        return (p+1);
+    return (p+1);
 }
 
 
@@ -512,47 +560,57 @@ static int optparse_print_options (optparse_t *p)
 {
     int columns;
     struct option_info *o;
-    ListIterator i;
 
-    if (!p || !p->option_list || !list_count(p->option_list))
+    if (!p || !p->option_list || !zlist_size (p->option_list))
         return (0);
 
     /*
-     *  Sort option table by group then by name
+     *  Sort option table per sorted flag
      */
-    list_sort (p->option_list, (ListCmpF) option_info_cmp);
+    zlist_sort (p->option_list,
+                p->sorted ? option_info_cmp : option_info_seq);
 
     columns = get_term_columns();
-    i = list_iterator_create (p->option_list);
-    while ((o = list_next (i))) {
+    o = zlist_first (p->option_list);
+    while (o) {
         if (o->isdoc)
             optparse_doc_print (p, o->p_opt, columns);
         else if (!o->hidden)
             optparse_option_print (p, o->p_opt, columns);
+        o = zlist_next (p->option_list);
     }
-    list_iterator_destroy (i);
 
     return (0);
 }
 
-#if CZMQ_VERSION < CZMQ_MAKE_VERSION(3,0,1)
-static bool s_cmp (const char *s1, const char *s2)
+static int subcmd_name_cmp (void *s1, void *s2)
 {
-    return (strcmp (s1, s2) > 0);
+    const optparse_t *cmd1 = s1;
+    const optparse_t *cmd2 = s2;
+    return strcmp (cmd1->program_name, cmd2->program_name);
 }
-#else
-static int s_cmp (const char *s1, const char *s2)
+
+static int subcmd_seq_cmp (void *s1, void *s2)
 {
-    return strcmp (s1, s2);
+    const optparse_t *cmd1 = s1;
+    const optparse_t *cmd2 = s2;
+    return cmd1->seq - cmd2->seq;
 }
-#endif
 
 static zlist_t *subcmd_list_sorted (optparse_t *p)
 {
-    zlist_t *keys = zhash_keys (p->subcommands);
-    if (keys)
-        zlist_sort (keys, (zlist_compare_fn *) s_cmp);
-    return (keys);
+    optparse_t *cmd = NULL;
+    zlist_t *subcmds = zlist_new ();
+
+    cmd = zhash_first (p->subcommands);
+    while (cmd) {
+        zlist_append (subcmds, cmd);
+        cmd = zhash_next (p->subcommands);
+    }
+
+    zlist_sort (subcmds, p->sorted ? subcmd_name_cmp : subcmd_seq_cmp);
+
+    return (subcmds);
 }
 
 /*
@@ -562,9 +620,9 @@ static zlist_t *subcmd_list_sorted (optparse_t *p)
 static int print_usage_with_subcommands (optparse_t *parent)
 {
     int lines = 0;
-    const char *cmd;
     opt_log_f fp = parent->log_fn;
-    zlist_t *keys;
+    zlist_t *subcmds = NULL;
+    optparse_t *p = NULL;
     int nsubcmds = zhash_size (parent->subcommands);
     /*
      *  With subcommands, only print usage line for parent command
@@ -583,21 +641,20 @@ static int print_usage_with_subcommands (optparse_t *parent)
         return (1);
     }
 
-    if (!(keys = subcmd_list_sorted (parent)))
+    if (!(subcmds = subcmd_list_sorted (parent)))
         return (-1);
 
-    cmd = zlist_first (keys);
-    while (cmd) {
-        optparse_t *p = zhash_lookup (parent->subcommands, cmd);;
+    p = zlist_first (subcmds);
+    while (p) {
         if (!p->hidden) {
             (*fp) ("%5s: %s %s\n",
                     ++lines > 1 ? "or" : "Usage",
                     optparse_fullname (p),
                     p->usage ? p->usage : "[OPTIONS]");
         }
-        cmd = zlist_next (keys);
+        p = zlist_next (subcmds);
     }
-    zlist_destroy (&keys);
+    zlist_destroy (&subcmds);
     return (lines);
 }
 
@@ -640,10 +697,21 @@ void optparse_destroy (optparse_t *p)
         return;
 
     /* Unlink from parent */
-    if (p->parent && p->parent->subcommands)
+    if (p->parent && p->parent->subcommands) {
         zhash_delete (p->parent->subcommands, p->program_name);
+        /*
+         * zhash_delete() will call optparse_child_destroy(), which
+         * calls optparse_destroy() again. Return now and allow the
+         * rest of the optparse object to be freed on the next pass.
+         * (optparse_child_destroy() set p->parent = NULL)
+         *
+         * Note: This code path only occurs if a subcommand optparse
+         *  object is destroyed before its parent.
+         */
+        return;
+    }
 
-    list_destroy (p->option_list);
+    zlist_destroy (&p->option_list);
     zhash_destroy (&p->dhash);
     zhash_destroy (&p->subcommands);
     free (p->program_name);
@@ -664,19 +732,15 @@ optparse_t *optparse_create (const char *prog)
         .cb    = (optparse_cb_f) display_help,
     };
 
-    struct opt_parser *p = malloc (sizeof (*p));
+    struct opt_parser *p = calloc (1, sizeof (*p));
     if (!p)
         return NULL;
 
-    memset (p, 0, sizeof (*p));
-
     if (!(p->program_name = strdup (prog))) {
         free (p);
         return NULL;
     }
-    p->usage = NULL;
-    p->parent = NULL;
-    p->option_list = list_create ((ListDelF) option_info_destroy);
+    p->option_list = zlist_new ();
     p->dhash = zhash_new ();
     p->subcommands = zhash_new ();
     if (!p->option_list || !p->dhash || !p->subcommands) {
@@ -691,12 +755,15 @@ optparse_t *optparse_create (const char *prog)
     p->option_width = 25;
     p->option_index = -1;
     p->posixly_correct = 1;
+    p->sorted = 0;
 
     /*
      *  Register -h, --help
      */
     if (optparse_add_option (p, &help) != OPTPARSE_SUCCESS) {
-        fprintf (stderr, "failed to register --help option: %s\n", strerror (errno));
+        fprintf (stderr,
+                 "failed to register --help option: %s\n",
+                 strerror (errno));
         optparse_destroy (p);
         return (NULL);
     }
@@ -716,6 +783,7 @@ optparse_t *optparse_add_subcommand (optparse_t *p,
     new = optparse_create (name);
     if (new == NULL)
         return (NULL);
+    new->seq = zhash_size (p->subcommands);
     zhash_update (p->subcommands, name, (void *) new);
     zhash_freefn (p->subcommands, name, optparse_child_destroy);
     zhash_update (new->dhash, "optparse::cb", cb);
@@ -754,12 +822,15 @@ optparse_err_t optparse_reg_subcommand (optparse_t *p,
     new = optparse_add_subcommand (p, name, cb);
     if (new == NULL)
         return OPTPARSE_NOMEM;
-    if ((usage &&
-         (e = optparse_set (new, OPTPARSE_USAGE, usage)) != OPTPARSE_SUCCESS)
-     || (doc &&
-         (e = optparse_add_doc (new, doc, -1)) != OPTPARSE_SUCCESS)
-     || (opts &&
-         (e = optparse_add_option_table (new, opts) != OPTPARSE_SUCCESS))) {
+    if ((usage && (e = optparse_set (new,
+                                     OPTPARSE_USAGE,
+                                     usage)) != OPTPARSE_SUCCESS)
+        || (doc && (e = optparse_add_doc (new,
+                                          doc,
+                                          -1)) != OPTPARSE_SUCCESS)
+        || (opts
+            && (e = optparse_add_option_table (new,
+                                               opts) != OPTPARSE_SUCCESS))) {
         optparse_destroy (new);
         return (e);
     }
@@ -776,8 +847,13 @@ optparse_err_t optparse_reg_subcommands (optparse_t *p,
     optparse_err_t e;
     struct optparse_subcommand *cmd = &cmds[0];
     while (cmd->name) {
-        e = optparse_reg_subcommand (p, cmd->name, cmd->fn, cmd->usage,
-                                        cmd->doc, cmd->flags, cmd->opts);
+        e = optparse_reg_subcommand (p,
+                                     cmd->name,
+                                     cmd->fn,
+                                     cmd->usage,
+                                     cmd->doc,
+                                     cmd->flags,
+                                     cmd->opts);
         if (e != OPTPARSE_SUCCESS)
             return (e);
         cmd++;
@@ -812,9 +888,11 @@ bool optparse_hasopt (optparse_t *p, const char *name)
 {
     int n;
     if ((n = optparse_getopt (p, name, NULL)) < 0) {
-        optparse_fatalmsg (p, 1,
+        optparse_fatalmsg (p,
+                           1,
                            "%s: optparse error: no such argument '%s'\n",
-                           p->program_name, name);
+                           p->program_name,
+                           name);
         return false;
     }
     return (n > 0);
@@ -828,14 +906,18 @@ int optparse_get_int (optparse_t *p, const char *name, int default_value)
     char *endptr;
 
     if ((n = optparse_getopt (p, name, &s)) < 0) {
-        optparse_fatalmsg (p, 1,
+        optparse_fatalmsg (p,
+                           1,
                            "%s: optparse error: no such argument '%s'\n",
-                           p->program_name, name);
+                           p->program_name,
+                           name);
         return -1;
     }
     if (n == 0)
         return default_value;
-    if (s == NULL || strlen (s) == 0)
+    if (s == NULL)
+        return n;
+    if (strlen (s) == 0)
         goto badarg;
     errno = 0;
     l = strtol (s, &endptr, 10);
@@ -843,13 +925,16 @@ int optparse_get_int (optparse_t *p, const char *name, int default_value)
         goto badarg;
     return l;
 badarg:
-    optparse_fatalmsg (p, 1,
+    optparse_fatalmsg (p,
+                       1,
                        "%s: Option '%s' requires an integer argument\n",
-                       p->program_name, name);
+                       p->program_name,
+                       name);
     return -1;
 }
 
-double optparse_get_double (optparse_t *p, const char *name,
+double optparse_get_double (optparse_t *p,
+                            const char *name,
                             double default_value)
 {
     int n;
@@ -858,9 +943,11 @@ double optparse_get_double (optparse_t *p, const char *name,
     char *endptr;
 
     if ((n = optparse_getopt (p, name, &s)) < 0) {
-        optparse_fatalmsg (p, 1,
+        optparse_fatalmsg (p,
+                           1,
                            "%s: optparse error: no such argument '%s'\n",
-                           p->program_name, name);
+                           p->program_name,
+                           name);
         return -1;
     }
     if (n == 0)
@@ -873,13 +960,16 @@ double optparse_get_double (optparse_t *p, const char *name,
         goto badarg;
     return d;
 badarg:
-    optparse_fatalmsg (p, 1,
+    optparse_fatalmsg (p,
+                       1,
                        "%s: Option '%s' requires a floating point argument\n",
-                       p->program_name, name);
+                       p->program_name,
+                       name);
     return -1;
 }
 
-double optparse_get_duration (optparse_t *p, const char *name,
+double optparse_get_duration (optparse_t *p,
+                              const char *name,
                               double default_value)
 {
     int n;
@@ -887,31 +977,95 @@ double optparse_get_duration (optparse_t *p, const char *name,
     const char *s = NULL;
 
     if ((n = optparse_getopt (p, name, &s)) < 0) {
-        optparse_fatalmsg (p, 1,
+        optparse_fatalmsg (p,
+                           1,
                            "%s: optparse error: no such argument '%s'\n",
-                           p->program_name, name);
+                           p->program_name,
+                           name);
     }
     if (n == 0)
         return default_value;
     if (fsd_parse_duration (s, &d) < 0) {
-        optparse_fatalmsg (p, 1,
-                           "%s: Invalid argument for option '%s': '%s'",
-                           p->program_name, name, s);
+        optparse_fatalmsg (p,
+                           1,
+                           "%s: Invalid argument for option '%s': '%s'\n",
+                           p->program_name,
+                           name,
+                           s);
         return -1;
     }
     return d;
 }
 
-const char *optparse_get_str (optparse_t *p, const char *name,
+uint64_t optparse_get_size (optparse_t *p,
+                            const char *name,
+                            const char *default_value)
+{
+    int n;
+    uint64_t result;
+    const char *s = NULL;
+
+    if (default_value == NULL)
+        default_value = "0";
+
+    if ((n = optparse_getopt (p, name, &s)) < 0) {
+        optparse_fatalmsg (p,
+                           1,
+                           "%s: optparse error: no such argument '%s'\n",
+                           p->program_name,
+                           name);
+        return (uint64_t) -1;
+    }
+    if (n == 0)
+        s = default_value;
+    if (parse_size (s, &result) < 0) {
+        optparse_fatalmsg (p,
+                           1,
+                          "%s: invalid argument for option '%s': %s: %s\n",
+                          p->program_name,
+                          name,
+                          s,
+                          strerror (errno));
+        return (uint64_t) -1;
+    }
+    return result;
+}
+
+int optparse_get_size_int (optparse_t *p,
+                           const char *name,
+                           const char *default_value)
+{
+    uint64_t val = optparse_get_size (p, name, default_value);
+    if (val == (uint64_t)-1)
+        return -1;
+    if (val > INT_MAX) {
+        const char *s;
+        optparse_getopt (p, name, &s);
+        optparse_fatalmsg (p,
+                           1,
+                           "%s: %s: value %s too large (must be < %s)\n",
+                           p->program_name,
+                           name,
+                           s,
+                           encode_size (INT_MAX+1UL));
+        return -1;
+    }
+    return (int)val;
+}
+
+const char *optparse_get_str (optparse_t *p,
+                              const char *name,
                               const char *default_value)
 {
     int n;
     const char *s;
 
     if ((n = optparse_getopt (p, name, &s)) < 0) {
-        optparse_fatalmsg (p, 1,
+        optparse_fatalmsg (p,
+                           1,
                            "%s: optparse error: no such argument '%s'\n",
-                           p->program_name, name);
+                           p->program_name,
+                           name);
         return NULL;
     }
     if (n == 0)
@@ -922,13 +1076,16 @@ const char *optparse_get_str (optparse_t *p, const char *name,
 const char *optparse_getopt_next (optparse_t *p, const char *name)
 {
     struct option_info *c;
+    const char *current;
     if (!(c = find_option_info (p, name)))
         return (NULL);
     if (c->found == 0 || !c->optargs)
         return (NULL);
-    if (!c->argi)
-        c->argi = list_iterator_create (c->optargs);
-    return (list_next (c->argi));
+    /*  Return current item and advance list */
+    if (!(current = zlist_item (c->optargs)))
+        return NULL;
+    zlist_next (c->optargs);
+    return current;
 }
 
 int optparse_getopt_iterator_reset (optparse_t *p, const char *name)
@@ -938,13 +1095,12 @@ int optparse_getopt_iterator_reset (optparse_t *p, const char *name)
         return (-1);
     if (c->found == 0 || !c->optargs)
         return (0);
-    if (c->argi)
-        list_iterator_reset (c->argi);
-    return (list_count (c->optargs));
+    zlist_first (c->optargs);
+    return (zlist_size (c->optargs));
 }
 
 optparse_err_t optparse_add_option (optparse_t *p,
-        const struct optparse_option *o)
+                                    const struct optparse_option *o)
 {
     struct option_info *c;
 
@@ -957,35 +1113,44 @@ optparse_err_t optparse_add_option (optparse_t *p,
     if (!(c = option_info_create (o)))
         return OPTPARSE_NOMEM;
 
-    if (!list_append (p->option_list, c))
+    c->seq = zlist_size (p->option_list);
+
+    if (zlist_append (p->option_list, c) < 0)
         return OPTPARSE_NOMEM;
+    zlist_freefn (p->option_list,
+                  c,
+                  (zlist_free_fn *) option_info_destroy,
+                  true);
 
     return (OPTPARSE_SUCCESS);
 }
 
 optparse_err_t optparse_remove_option (optparse_t *p, const char *name)
 {
-    optparse_err_t rc = OPTPARSE_SUCCESS;
+    struct option_info *o = NULL;
 
-    int n = list_delete_all (p->option_list,
-                (ListFindF) option_info_cmp_name,
-                (void *) name);
+    if (!p || !name)
+        return OPTPARSE_FAILURE;
 
-    if (n != 1)
-        rc = OPTPARSE_FAILURE;
+    if (!(o = find_option_info (p, name)))
+        return OPTPARSE_FAILURE;
+    zlist_remove (p->option_list, o);
 
-    return (rc);
+    return OPTPARSE_SUCCESS;
 }
 
 optparse_err_t optparse_add_option_table (optparse_t *p,
-        struct optparse_option const opts[])
+                                          struct optparse_option const opts[])
 {
     optparse_err_t rc = OPTPARSE_SUCCESS;
     const struct optparse_option *o = opts;
 
+    if (!p)
+        return OPTPARSE_BAD_ARG;
+
     while (o->usage) {
         if ((rc = optparse_add_option (p, o++)) != OPTPARSE_SUCCESS) {
-            option_table_remove (p, opts, o);
+            option_table_remove (p, opts, o-1);
             return (rc);
         }
     }
@@ -1065,6 +1230,10 @@ optparse_err_t optparse_set (optparse_t *p, int item, ...)
         n = va_arg (vargs, int);
         p->posixly_correct = n;
         break;
+    case OPTPARSE_SORTED:
+        n = va_arg (vargs, int);
+        p->sorted = n;
+        break;
     default:
         e = OPTPARSE_BAD_ARG;
     }
@@ -1104,6 +1273,11 @@ optparse_err_t optparse_get (optparse_t *p, int item, ...)
     case OPTPARSE_LEFT_MARGIN:
     case OPTPARSE_OPTION_CB:
     case OPTPARSE_OPTION_WIDTH:
+    case OPTPARSE_PRINT_SUBCMDS:
+    case OPTPARSE_SUBCMD_NOOPTS:
+    case OPTPARSE_SUBCMD_HIDE:
+    case OPTPARSE_POSIXLY_CORRECT:
+    case OPTPARSE_SORTED:
         e = OPTPARSE_NOT_IMPL;
         break;
 
@@ -1126,10 +1300,17 @@ void *optparse_get_data (optparse_t *p, const char *s)
 
 static char * optstring_create ()
 {
-    char *optstring = malloc (4);
+    char *optstring = calloc (4, 1);
     if (optstring == NULL)
         return (NULL);
-    optstring[0] = '\0';
+    /* Per getopt(3) manpage
+     *
+     * "If the first character ... of optstring is a colon (':'), then
+     * getopt() returns ':' instead of '?' to indicate a missing
+     * option argument"
+     */
+    optstring[0] = ':';
+    optstring[1] = '\0';
     return (optstring);
 }
 
@@ -1149,6 +1330,8 @@ static char * optstring_append (char *optstring, struct optparse_option *o)
      *   an option with no argument (has_arg = 0), 2 characters
      *   for a required argument (has_arg = 1), and 3 chars for
      *   an optional argument "o::" (has_arg = 2)
+     *  N.B.: optional argument is applied to short options only if
+     *   OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG is set on flags
      */
     len = strlen (optstring);
     n = len + o->has_arg + 1;
@@ -1157,7 +1340,7 @@ static char * optstring_append (char *optstring, struct optparse_option *o)
 
     if (o->has_arg == 1)
         colons = ":";
-    else if (o->has_arg == 2)
+    else if (o->has_arg == 2 && o->flags & OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG)
         colons = "::";
 
     sprintf (optstring+len, "%c%s", o->key, colons);
@@ -1175,10 +1358,9 @@ static struct option * option_table_create (optparse_t *p, char **sp)
     struct option *opts;
     int n;
     int j;
-    ListIterator i;
 
-    n = list_count (p->option_list);
-    opts = malloc ((n + 1) * sizeof (struct option));
+    n = zlist_size (p->option_list);
+    opts = calloc ((n + 1), sizeof (struct option));
     if (opts == NULL)
         return (NULL);
 
@@ -1191,23 +1373,22 @@ static struct option * option_table_create (optparse_t *p, char **sp)
     }
 
     j = 0;
-    i = list_iterator_create (p->option_list);
-    while ((o = list_next (i))) {
-        if (o->isdoc)
-            continue;
-        /* Initialize option field from cached option structure */
-        opt_init (&opts[j++], o->p_opt);
-        if (sp) {
-            *sp = optstring_append (*sp, o->p_opt);
-            if (*sp == NULL) {
-                free (opts);
-                opts = NULL;
-                break;
+    o = zlist_first (p->option_list);
+    while (o) {
+        if (!o->isdoc) {
+            /* Initialize option field from cached option structure */
+            opt_init (&opts[j++], o->p_opt);
+            if (sp) {
+                *sp = optstring_append (*sp, o->p_opt);
+                if (*sp == NULL) {
+                    free (opts);
+                    opts = NULL;
+                    break;
+                }
             }
-
         }
+        o = zlist_next (p->option_list);
     }
-    list_iterator_destroy (i);
 
     /*
      *  Initialize final element of option array to zeros:
@@ -1218,55 +1399,75 @@ static struct option * option_table_create (optparse_t *p, char **sp)
     return (opts);
 }
 
-static int by_val (struct option_info *c, int *val)
-{
-    return (c->p_opt->key == *val);
-}
-
-static int opt_append_sep (struct option_info *opt, const char *s)
+static int opt_append_sep (struct option_info *opt, const char *str)
 {
     error_t e;
     char *arg = NULL;
     char *argz = NULL;
     size_t len = 0;
-    if ((e = argz_create_sep (s, ',', &argz, &len))) {
+    if ((e = argz_create_sep (str, ',', &argz, &len))) {
         errno = e;
         return (-1);
     }
-    while ((arg = argz_next (argz, len, arg)))
-        opt->optarg = list_append (opt->optargs, strdup (arg));
-
+    while ((arg = argz_next (argz, len, arg))) {
+        char *s = strdup (arg);
+        if (s == NULL)
+            return -1;
+        opt->optarg = s;
+        zlist_append (opt->optargs, s);
+        zlist_freefn (opt->optargs, s, free, true);
+    }
+    zlist_first (opt->optargs);
     free (argz);
     return (0);
 }
 
-static void opt_append_optarg (optparse_t *p, struct option_info *opt, const char *optarg)
+static void opt_append_optarg (optparse_t *p,
+                               struct option_info *opt,
+                               const char *optarg)
 {
     char *s;
     if (!opt->optargs)
-        opt->optargs = list_create ((ListDelF) free);
+        opt->optargs = zlist_new ();
     if (opt->autosplit) {
         if (opt_append_sep (opt, optarg) < 0)
-            optparse_fatalmsg (p, 1, "%s: append '%s': %s\n", p->program_name,
-                               optarg, strerror (errno));
+            optparse_fatalmsg (p,
+                               1,
+                               "%s: append '%s': %s\n",
+                               p->program_name,
+                               optarg,
+                               strerror (errno));
         return;
     }
     if ((s = strdup (optarg)) == NULL)
         optparse_fatalmsg (p, 1, "%s: out of memory\n", p->program_name);
-    list_append (opt->optargs, s);
+    zlist_append (opt->optargs, s);
+    zlist_freefn (opt->optargs, s, free, true);
     opt->optarg = s;
+    /*  Ensure iterator is reset on first append */
+    zlist_first (opt->optargs);
 }
 
 /*
  *  Call reentrant internal version of getopt_long() directly copied from
  *   glibc. See getopt.c and getopt_int.h in this directory.
  */
-static int getopt_long_r (int argc, char *const *argv, const char *options,
-                const struct option *long_options, int *opt_index,
-                struct _getopt_data *d, int posixly_correct)
-{
-  return _getopt_internal_r (argc, argv, options, long_options, opt_index,
-                             0, d, posixly_correct);
+static int getopt_long_r (int argc,
+                          char *const *argv,
+                          const char *options,
+                          const struct option *long_options,
+                          int *opt_index,
+                          struct _getopt_data *d,
+                          int posixly_correct)
+{
+  return _getopt_internal_r (argc,
+                             argv,
+                             options,
+                             long_options,
+                             opt_index,
+                             0,
+                             d,
+                             posixly_correct);
 }
 
 int optparse_parse_args (optparse_t *p, int argc, char *argv[])
@@ -1278,24 +1479,42 @@ int optparse_parse_args (optparse_t *p, int argc, char *argv[])
     char *optstring = NULL;
     struct option *optz = option_table_create (p, &optstring);
 
+    if (!optz)
+        return -1;
+
     fullname = optparse_fullname (p);
 
     /* Initialize getopt internal state data:
      */
     memset (&d, 0, sizeof (d));
 
-    while ((c = getopt_long_r (argc, argv, optstring, optz,
-                               &li, &d, p->posixly_correct)) >= 0) {
+    while ((c = getopt_long_r (argc,
+                               argv,
+                               optstring,
+                               optz,
+                               &li,
+                               &d,
+                               p->posixly_correct)) >= 0) {
         struct option_info *opt;
         struct optparse_option *o;
-        if (c == '?') {
+        if (c == ':') {
+            (*p->log_fn) ("%s: '%s' missing argument\n",
+                          fullname,
+                          argv[d.optind-1]);
+            d.optind = -1;
+            break;
+        }
+        else if (c == '?') {
             if (d.optopt != '\0')
                 (*p->log_fn) ("%s: unrecognized option '-%c'\n",
-                              fullname,  d.optopt);
+                              fullname,
+                              d.optopt);
             else
                 (*p->log_fn) ("%s: unrecognized option '%s'\n",
-                              fullname, argv[d.optind-1]);
-            (*p->log_fn) ("Try `%s --help' for more information.\n", fullname);
+                              fullname,
+                              argv[d.optind-1]);
+            (*p->log_fn) ("Try `%s --help' for more information.\n",
+                          fullname);
             d.optind = -1;
             break;
         }
@@ -1308,12 +1527,13 @@ int optparse_parse_args (optparse_t *p, int argc, char *argv[])
         if (li >= 0)
             opt = find_option_info (p, optz[li].name);
         else
-            opt = list_find_first (p->option_list, (ListFindF) by_val, &c);
+            opt = find_option_by_val (p, c);
 
         /* Reset li for next iteration */
         li = -1;
         if (opt == NULL) {
-            (*p->log_fn) ("ugh, didn't find option associated with char %c\n", c);
+            (*p->log_fn) ("ugh, didn't find option associated with char %c\n",
+                          c);
             continue;
         }
 
@@ -1361,8 +1581,10 @@ int optparse_run_subcommand (optparse_t *p, int argc, char *argv[])
         return optparse_fatalerr (sp, 1);
 
     if (!(cb = zhash_lookup (sp->dhash, "optparse::cb"))) {
-        return optparse_fatalmsg (p, 1,
-            "subcommand %s: failed to lookup callback!\n");
+        return optparse_fatalmsg (p,
+                                  1,
+                                  "subcommand %s: failed to lookup callback!\n",
+                                  av[0]);
     }
 
     return ((*cb) (sp, ac, av));
@@ -1385,7 +1607,6 @@ int optparse_fatal_usage (optparse_t *p, int code, const char *fmt, ...)
     return (*p->fatalerr_fn) (p->fatalerr_handle, code);
 }
 
-
 int optparse_option_index (optparse_t *p)
 {
     return (p->option_index);
@@ -1398,43 +1619,34 @@ int optparse_option_index (optparse_t *p)
 static void optparse_reset_one (optparse_t *p)
 {
     struct option_info *o;
-    ListIterator i;
 
     if (!p)
         return;
 
     p->option_index = -1;
 
-    if (!p->option_list || !list_count (p->option_list))
+    if (!p->option_list || !zlist_size (p->option_list))
         return;
 
-    i = list_iterator_create (p->option_list);
-    while ((o = list_next (i))) {
+    o = zlist_first (p->option_list);
+    while (o) {
         if (o->isdoc)
             continue;
         o->found = 0;
-        if (o->optargs) {
-            list_destroy (o->optargs);
-            o->optargs = NULL;
-        }
+        if (o->optargs)
+            zlist_destroy (&o->optargs);
         o->optarg = NULL;
+        o = zlist_next (p->option_list);
     }
-    list_iterator_destroy (i);
     return;
 }
 
 void optparse_reset (optparse_t *p)
 {
-    zlist_t *cmds = NULL;
-
-    if ((cmds = subcmd_list_sorted (p))) {
-        const char *cmd = zlist_first (cmds);
-        while (cmd) {
-            optparse_t *o = zhash_lookup (p->subcommands, cmd);
-            optparse_reset_one (o);
-            cmd = zlist_next (cmds);
-        }
-        zlist_destroy (&cmds);
+    optparse_t *cmd = zhash_first (p->subcommands);
+    while (cmd) {
+        optparse_reset_one (cmd);
+        cmd = zhash_next (p->subcommands);
     }
     optparse_reset_one (p);
 }
diff --git a/src/common/liboptparse/optparse.h b/src/common/liboptparse/optparse.h
index 4f064d0b4a56..3787a367a935 100644
--- a/src/common/liboptparse/optparse.h
+++ b/src/common/liboptparse/optparse.h
@@ -12,6 +12,7 @@
 #define _UTIL_OPTPARSE_H
 
 #include <stdbool.h>
+#include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -36,8 +37,9 @@ typedef int (*opt_fatalerr_f) (void *h, int exit_code);
 /*
  *  prototype for option callback hook
  */
-typedef int (*optparse_cb_f) (optparse_t *p, struct optparse_option *o,
-			      const char *optarg);
+typedef int (*optparse_cb_f) (optparse_t *p,
+                              struct optparse_option *o,
+                              const char *optarg);
 
 /*
  *  prototype for subcommand callback
@@ -71,6 +73,8 @@ typedef enum {
     OPTPARSE_SUBCMD_NOOPTS,/* Don't parse options for this subcommand       */
     OPTPARSE_SUBCMD_HIDE,  /* Don't output this subcmd in --help output     */
     OPTPARSE_POSIXLY_CORRECT, /* Set POSIXLY_CORRECT value                  */
+    OPTPARSE_SORTED,       /* Sort subcommands and options in help output   */
+    OPTPARSE_ITEM_END,
 } optparse_item_t;
 
 /*
@@ -79,11 +83,10 @@ typedef enum {
 struct optparse_option {
     const char *  name;    /*  Option name (e.g. "help" for --help)         */
     int           key;     /*  Option key  (e.g. 'h', or other number).
-				If !isalnum(key), then this option is
+                                If !isalnum(key), then this option is
                                 assumed to be a long option only.           */
 
-    int           has_arg; /*  0: no arg, 1: req'd arg, 2: optional arg
-                               3: list-arg (split on comma separate values) */
+    int           has_arg; /*  0: no arg, 1: req'd arg, 2: optional arg     */
     int           group;   /*  Grouping in --help output                    */
     int           flags;   /*  Extra flags. See Option FLAGS below          */
     const char *  arginfo; /*  arg info displayed after = in help output    */
@@ -129,6 +132,12 @@ struct optparse_subcommand {
  */
 #define OPTPARSE_OPT_HIDDEN       0x2
 
+/*
+ *  Apply optional argument to short options as well as long. The default
+ *   is that optional arguments only apply to longopts.
+ */
+#define OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG 0x4
+
 /******************************************************************************
  *  Subcommand FLAGS:
  *****************************************************************************/
@@ -189,12 +198,12 @@ optparse_t *optparse_get_parent (optparse_t *p);
  *    on failure.
  */
 optparse_err_t optparse_reg_subcommand (optparse_t *p,
-                                     const char *name,
-                                     optparse_subcmd_f cb,
-                                     const char *usage,
-                                     const char *doc,
-                                     int flags,
-                                     struct optparse_option const opts[]);
+                                        const char *name,
+                                        optparse_subcmd_f cb,
+                                        const char *usage,
+                                        const char *doc,
+                                        int flags,
+                                        struct optparse_option const opts[]);
 
 
 /*
@@ -205,7 +214,7 @@ optparse_err_t optparse_reg_subcommand (optparse_t *p,
  *    or optparse_err_t on failure.
  */
 optparse_err_t optparse_reg_subcommands (optparse_t *p,
-                                     struct optparse_subcommand cmds[]);
+                                         struct optparse_subcommand cmds[]);
 
 /*
  *   Destroy program options handle [p].
@@ -228,7 +237,7 @@ void optparse_reset (optparse_t *p);
  *
  *    OPTPARSE_NOMEM:  Failed to allocate memory for some options.
  *    OPTPARSE_EEXIST: An attempt to register a duplicate option was detected.
- *    OPTPARSE_EINVAL: The optparse_t *object is currupt or invalid.
+ *    OPTPARSE_EINVAL: The optparse_t *object is corrupt or invalid.
  */
 optparse_err_t optparse_add_option (optparse_t *p,
                                     const struct optparse_option *o);
@@ -248,11 +257,11 @@ optparse_err_t optparse_remove_option (optparse_t *p, const char *name);
  *    optparse_add_option().
  */
 optparse_err_t optparse_add_option_table (optparse_t *p,
-	                                  struct optparse_option const opts[]);
+                                          struct optparse_option const opts[]);
 
 /*
  *   Register a doc string [doc] for display in autogenerated --help output
- *    for program options object [p]. The doc string will preceed the
+ *    for program options object [p]. The doc string will precede the
  *    option output for group [group].
  */
 optparse_err_t optparse_add_doc (optparse_t *p, const char *doc, int group);
@@ -357,8 +366,10 @@ bool optparse_hasopt (optparse_t *p, const char *name);
 
 /*
  *   Return the option argument as an integer if 'name' was used,
- *    'default_value' if not.  If the option is unknown, or the argument
- *    could not be converted to an integer, call the fatal error function.
+ *    'default_value' if not.  If the option does not take an argument,
+ *    then returns the number of times the option was used.
+ *    If the option is unknown, or the argument could not be converted to
+ *    an integer, call the fatal error function.
  */
 int optparse_get_int (optparse_t *p, const char *name, int default_value);
 
@@ -369,22 +380,47 @@ int optparse_get_int (optparse_t *p, const char *name, int default_value);
  *   default_value. If there was an error parsing the duration string,
  *   call the fatal error function.
  */
-double optparse_get_duration (optparse_t *p, const char *name,
+double optparse_get_duration (optparse_t *p,
+                              const char *name,
                               double default_value);
 
+/*
+ *   Return the option argument parsed as a size in bytes (or other unit)
+ *    with optional multiplicative suffix: k or K=1024, M=1024*1024,
+ *    G=1024*1024*1024, and so on for T, P, and E. The end result is
+ *    truncated and returned as uint64_t. If there was an error parsing
+ *    the size string, then the fatal error function is called.
+ *
+ *   Return default value if option was unused. default_value=NULL
+ *    is equivalent to default_value="0".
+ */
+uint64_t optparse_get_size (optparse_t *p,
+                            const char *name,
+                            const char *default_value);
+
+/*
+ *  Like optparse_get_size(), but return an int and ensure the provided
+ *   argument does not overflow INT_MAX (calls fatal error function if so).
+ */
+int optparse_get_size_int (optparse_t *p,
+                           const char *name,
+                           const char *default_value);
+
 /*
  *   Return the option argument as a double if 'name' was used,
  *    'default_value' if not.  If the option is unknown, or the argument
  *    could not be converted to a double, call the fatal error function.
  */
-double optparse_get_double (optparse_t *p, const char *name,
+double optparse_get_double (optparse_t *p,
+                            const char *name,
                             double default_value);
 
 /*
  *   Return the option argument as a string if 'name' was used, 'default_value'
  *    if not.  If the option is unknown, call the fatal error function.
  */
-const char *optparse_get_str (optparse_t *p, const char *name,
+const char *optparse_get_str (optparse_t *p,
+                              const char *name,
                               const char *default_value);
 
 /*
diff --git a/src/common/liboptparse/test/optparse.c b/src/common/liboptparse/test/optparse.c
index 770bed7d80ca..398fd6a5352f 100644
--- a/src/common/liboptparse/test/optparse.c
+++ b/src/common/liboptparse/test/optparse.c
@@ -8,9 +8,17 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdarg.h>
+
 #include "src/common/libtap/tap.h"
 #include "src/common/liboptparse/optparse.h"
-#include "src/common/libutil/sds.h"
+#include "ccan/array_size/array_size.h"
 
 static void *myfatal_h = NULL;
 
@@ -20,22 +28,32 @@ int myfatal (void *h, int exit_code)
     return (0);
 }
 
-sds usage_out = NULL;
+char * usage_out = NULL;
+size_t usage_len = 0;
+
 void output_f (const char *fmt, ...)
 {
     va_list ap;
-    sds s = usage_out ? usage_out : sdsempty();
+    char s[4096];
+    int len;
     va_start (ap, fmt);
-    usage_out = sdscatvprintf (s, fmt, ap);
+    if ((len = vsnprintf (s, sizeof (s), fmt, ap)) >= sizeof (s))
+        BAIL_OUT ("unexpected snprintf failure");
     va_end (ap);
+    if (!(usage_out = realloc (usage_out, usage_len + len + 1)))
+        BAIL_OUT ("realloc failed usage_len = %d", usage_len);
+    usage_out[usage_len] = '\0';
+    strcat (usage_out, s);
+    usage_len += len + 1;
 }
 
 void usage_output_is (const char *expected, const char *msg)
 {
     ok (usage_out != NULL, "optparse_print_usage");
     is (usage_out, expected, msg);
-    sdsfree (usage_out);
+    free (usage_out);
     usage_out = NULL;
+    usage_len = 0;
 }
 
 void usage_ok (optparse_t *p, const char *expected, const char *msg)
@@ -80,10 +98,22 @@ void test_usage_output (void)
 
     usage_ok (p, "\
 Usage: prog-foo [OPTIONS]\n\
+  -h, --help             Display this message.\n\
+  -t, --test             Enable a test option.\n\
+  -T, --test2=N          Enable a test option N.\n",
+        "Usage output as expected");
+
+    e = optparse_set (p, OPTPARSE_SORTED, 1);
+    ok (e == OPTPARSE_SUCCESS, "optparse_set (SORTED)");
+    usage_ok (p, "\
+Usage: prog-foo [OPTIONS]\n\
   -h, --help             Display this message.\n\
   -T, --test2=N          Enable a test option N.\n\
   -t, --test             Enable a test option.\n",
-        "Usage output as expected");
+        "Usage output is now sorted as expected");
+
+    e = optparse_set (p, OPTPARSE_SORTED, 0);
+    ok (e == OPTPARSE_SUCCESS, "optparse_set (SORTED)");
 
     // Add a hidden (undocumented) option
     opt = ((struct optparse_option) {
@@ -94,14 +124,14 @@ Usage: prog-foo [OPTIONS]\n\
             });
     e = optparse_add_option (p, &opt);
     ok (e == OPTPARSE_SUCCESS, "optparse_add_option. group 1.");
+
     usage_ok (p, "\
 Usage: prog-foo [OPTIONS]\n\
   -h, --help             Display this message.\n\
-  -T, --test2=N          Enable a test option N.\n\
-  -t, --test             Enable a test option.\n",
+  -t, --test             Enable a test option.\n\
+  -T, --test2=N          Enable a test option N.\n",
         "Usage output as expected");
 
-
     // Adjust left margin
     e = optparse_set (p, OPTPARSE_LEFT_MARGIN, 0);
     ok (e == OPTPARSE_SUCCESS, "optparse_set (LEFT_MARGIN)");
@@ -109,8 +139,8 @@ Usage: prog-foo [OPTIONS]\n\
     usage_ok (p, "\
 Usage: prog-foo [OPTIONS]\n\
 -h, --help               Display this message.\n\
--T, --test2=N            Enable a test option N.\n\
--t, --test               Enable a test option.\n",
+-t, --test               Enable a test option.\n\
+-T, --test2=N            Enable a test option N.\n",
         "Usage output as expected w/ left margin");
 
     e = optparse_set (p, OPTPARSE_LEFT_MARGIN, 2);
@@ -249,12 +279,12 @@ This is some doc in header\n\
   -h, --help                  Display this message.\n\
   -T, --test2=N               Enable a test option N.\n\
 This is some doc for group 1\n\
-      --long-only             This option is long only\n\
   -A, --long-option=ARGINFO   Enable a long option with argument info ARGINFO.\n\
   -B, --option-B              This option has a very long description. It should\n\
                               be split across lines nicely.\n\
   -C, --option-C              ThisOptionHasAVeryLongWordInTheDescriptionThatSho-\n\
-                              uldBeBrokenAcrossLines.\n",
+                              uldBeBrokenAcrossLines.\n\
+      --long-only             This option is long only\n",
         "Usage output with long only option");
 
 
@@ -272,7 +302,7 @@ void test_option_cb (void)
     optparse_err_t e;
     optparse_t *p = optparse_create ("test-help");
     char *av[] = { "test-help", "-h", NULL };
-    int ac = sizeof (av) / sizeof (av[0]) - 1;
+    int ac = ARRAY_SIZE (av) - 1;
     int optindex;
 
     ok (p != NULL, "optparse_create");
@@ -340,25 +370,33 @@ void test_convenience_accessors (void)
 { .name = "dub", .key = 7, .has_arg = 1, .arginfo = "", .usage = "" },
 { .name = "ndb", .key = 8, .has_arg = 1, .arginfo = "", .usage = "" },
 { .name = "dur", .key = 9, .has_arg = 1, .arginfo = "", .usage = "" },
+{ .name = "size", .key = 10, .has_arg = 1, .arginfo = "", .usage = "" },
+{ .name = "sizeint", .key = 11, .has_arg = 1, .arginfo = "", .usage = "" },
         OPTPARSE_TABLE_END,
     };
 
     char *av[] = { "test", "--foo", "--baz=hello", "--mnf=7", "--neg=-4",
-                   "--dub=5.7", "--ndb=-3.2", "--dur=1.5m", NULL };
-    int ac = sizeof (av) / sizeof (av[0]) - 1;
+                   "--dub=5.7", "--ndb=-3.2", "--dur=1.5m", "--size=4G",
+                   "--sizeint=1.25G", NULL };
+    int ac = ARRAY_SIZE (av) - 1;
     int rc, optindex;
 
     optparse_t *p = optparse_create ("test");
     ok (p != NULL, "create object");
 
+    ok (optparse_set (p, OPTPARSE_LOG_FN, diag) == OPTPARSE_SUCCESS,
+        "optparse_set LOG_FN");
+
     rc = optparse_add_option_table (p, opts);
     ok (rc == OPTPARSE_SUCCESS, "register options");
 
-    ok (optparse_option_index (p) == -1, "optparse_option_index returns -1 before parse");
+    ok (optparse_option_index (p) == -1,
+        "optparse_option_index returns -1 before parse");
     optindex = optparse_parse_args (p, ac, av);
     ok (optindex == ac, "parse options, verify optindex");
 
-    ok (optparse_option_index (p) == optindex, "optparse_option_index works after parse");
+    ok (optparse_option_index (p) == optindex,
+        "optparse_option_index works after parse");
 
     /* hasopt
      */
@@ -374,20 +412,22 @@ void test_convenience_accessors (void)
      */
     dies_ok ({optparse_get_int (p, "no-exist", 0); },
             "get_int exits on unknown arg");
-    dies_ok ({optparse_get_int (p, "foo", 0); },
-            "get_int exits on option with no argument");
     dies_ok ({optparse_get_int (p, "baz", 0); },
             "get_int exits on option with wrong type argument (string)");
     dies_ok ({optparse_get_int (p, "dub", 0); },
             "get_int exits on option with wrong type argument (float)");
     lives_ok ({optparse_get_int (p, "bar", 0); },
             "get_int lives on known arg");
+    lives_ok ({optparse_get_int (p, "foo", 0); },
+            "get_int lives on option with no argument");
     ok (optparse_get_int (p, "bar", 42) == 42,
             "get_int returns default argument when arg not present");
     ok (optparse_get_int (p, "mnf", 42) == 7,
             "get_int returns arg when present");
     ok (optparse_get_int (p, "neg", 42) == -4,
             "get_int returns negative arg when present");
+    ok (optparse_get_int (p, "foo", 42) == 1,
+            "get_int returns option count with no arg");
 
     /* get_double
      */
@@ -429,6 +469,61 @@ void test_convenience_accessors (void)
     ok (optparse_get_duration (p, "dur", 42) == 90.,
             "get_duration returns duration arg when present");
 
+    /* get_size
+     */
+    dies_ok ({optparse_get_size (p, "no-exist", "0"); },
+            "get_size exits on unknown arg");
+    dies_ok ({optparse_get_size (p, "foo", "0"); },
+            "get_size exits on option with no argument");
+    dies_ok ({optparse_get_size (p, "baz", "0"); },
+            "get_size exits on option with wrong type argument (string)");
+    dies_ok ({optparse_get_size (p, "neg", "42"); },
+            "get_size exits on negative arg");
+    dies_ok ({optparse_get_size (p, "dur", "42"); },
+            "get_size exits on bad suffix");
+    dies_ok ({optparse_get_size (p, "bar", "1m"); },
+            "get_size exits on bad suffix in default");
+    lives_ok ({optparse_get_size (p, "size", "1k"); },
+            "get_size lives on known arg");
+
+    ok (optparse_get_size (p, "bar", "10M") == 10*1024*1024,
+            "get_size returns default argument when arg not present");
+    ok (optparse_get_size (p, "bar", NULL) == 0,
+            "get_size default_argument=NULL results in default=0 ");
+    ok (optparse_get_size (p, "mnf", "0") == 7,
+            "get_size returns arg when present");
+    ok (optparse_get_size (p, "size", "0") == 4*1024ULL*1024*1024,
+            "get_size returns size arg when present");
+
+    /* get_size_int
+     */
+    dies_ok ({optparse_get_size_int (p, "no-exist", "0"); },
+            "get_size_int exits on unknown arg");
+    dies_ok ({optparse_get_size_int (p, "foo", "0"); },
+            "get_size_int exits on option with no argument");
+    dies_ok ({optparse_get_size_int (p, "baz", "0"); },
+            "get_size_int exits on option with wrong type argument (string)");
+    dies_ok ({optparse_get_size_int (p, "neg", "42"); },
+            "get_size_int exits on negative arg");
+    dies_ok ({optparse_get_size_int (p, "dur", "42"); },
+            "get_size_int exits on bad suffix");
+    dies_ok ({optparse_get_size_int (p, "bar", "1m"); },
+            "get_size_int exits on bad suffix in default");
+    dies_ok ({optparse_get_size_int (p, "size", "1M"); },
+            "get_size_int exits on value too large");
+    lives_ok ({optparse_get_size_int (p, "mnf", "1k"); },
+            "get_size_int lives on known arg");
+
+    ok (optparse_get_size_int (p, "bar", "10M") == 10*1024*1024,
+            "get_size_int returns default argument when arg not present");
+    ok (optparse_get_size_int (p, "bar", NULL) == 0,
+            "get_size_int default_argument=NULL results in default=0 ");
+    ok (optparse_get_size_int (p, "mnf", "0") == 7,
+            "get_size_int returns arg when present");
+    ok (optparse_get_size_int (p, "sizeint", "0") == 1.25*1024L*1024*1024,
+            "get_size_int returns size arg when present");
+
+
     /* get_str
      */
     dies_ok ({optparse_get_str (p, "no-exist", NULL); },
@@ -470,7 +565,7 @@ void test_errors (void)
             });
 
     e = optparse_add_option (p, &opt);
-    ok (e == OPTPARSE_EEXIST, "optparse_add_option: Errror with EEXIST");
+    ok (e == OPTPARSE_EEXIST, "optparse_add_option: Error with EEXIST");
     e = optparse_add_option (NULL, &opt);
     ok (e == OPTPARSE_BAD_ARG, "optparse_add_option: BAD_ARG with invalid optparse_t");
 
@@ -516,7 +611,7 @@ void test_multiret (void)
                    "-r", "one", "-mone", "-m", "two",
                    "-o", "-rtwo", "--multi-ret=a,b,c",
                    NULL };
-    int ac = sizeof (av) / sizeof (av[0]) - 1;
+    int ac = ARRAY_SIZE (av) - 1;
     int optindex;
 
     ok (p != NULL, "optparse_create");
@@ -587,7 +682,7 @@ void test_long_only (void)
     char *av[] = { "long-only-test",
                    "-b", "one", "--again-long-only",
                    NULL };
-    int ac = sizeof (av) / sizeof (av[0]) - 1;
+    int ac = ARRAY_SIZE (av) - 1;
     int optindex;
 
     ok (p != NULL, "optparse_create");
@@ -609,7 +704,7 @@ void test_long_only (void)
 
     char *av2[] = { "long-only-test", "--again-long-only", "-bxxx",
                     "--long-only=foo", NULL };
-    ac = sizeof (av2) / sizeof(av2[0]) - 1;
+    ac = ARRAY_SIZE (av2) - 1;
 
     optindex = optparse_parse_args (p, ac, av2);
     ok (optindex == ac, "parse options, verify optindex");
@@ -645,7 +740,7 @@ void test_optional_argument (void)
     char *av[] = { "optarg",
                    "--optional-arg", "extra-args",
                    NULL };
-    int ac = sizeof (av) / sizeof (av[0]) - 1;
+    int ac = ARRAY_SIZE (av) - 1;
     int optindex;
 
     e = optparse_add_option_table (p, opts);
@@ -664,7 +759,7 @@ void test_optional_argument (void)
     char *av2[] = { "optarg",
                    "--optional-arg=foo", "extra-args",
                    NULL };
-    ac = sizeof (av2) / sizeof (av2[0]) - 1;
+    ac = ARRAY_SIZE (av2) - 1;
 
     optindex = optparse_parse_args (p, ac, av2);
     ok (optindex == (ac - 1), "parse options, verify optindex");
@@ -766,6 +861,7 @@ void test_subcommand (void)
     int called = 0;
     int n;
     optparse_t *b;
+    optparse_t *c;
     optparse_t *a = optparse_create ("test");
 
     ok (a != NULL, "optparse_create");
@@ -783,15 +879,33 @@ void test_subcommand (void)
     optparse_set_data (b, "called", &called);
     ok (optparse_get_data (b, "called") == &called, "optparse_set_data ()");
 
+    c = optparse_add_subcommand (a, "three", subcmd_two);
+    ok (c != NULL, "optparse_add_subcommand");
+
     e = optparse_set (a, OPTPARSE_LOG_FN, output_f);
     ok (e == OPTPARSE_SUCCESS, "optparse_set (LOG_FN)");
 
     usage_ok (a, "\
 Usage: test one [OPTIONS]\n\
    or: test two [OPTIONS]\n\
+   or: test three [OPTIONS]\n\
   -h, --help             Display this message.\n",
         "Usage output as expected with subcommands");
 
+    // Set OPTPARSE_SORTED:
+    e = optparse_set (a, OPTPARSE_SORTED, 1);
+    ok (e == OPTPARSE_SUCCESS, "optparse_set (SORTED)");
+
+    usage_ok (a, "\
+Usage: test one [OPTIONS]\n\
+   or: test three [OPTIONS]\n\
+   or: test two [OPTIONS]\n\
+  -h, --help             Display this message.\n",
+        "Usage output as expected with sorted subcommands");
+
+    e = optparse_set (a, OPTPARSE_SORTED, 0);
+    ok (e == OPTPARSE_SUCCESS, "optparse_set (SORTED, 0)");
+
     // Set OPTPARSE_PRINT_SUBCMDS false:
     e = optparse_set (a, OPTPARSE_PRINT_SUBCMDS, 0);
     ok (e == OPTPARSE_SUCCESS, "optparse_set (PRINT_SUBCMDS, 0)");
@@ -818,10 +932,8 @@ Usage: test two [OPTIONS]...\n\
   -t, --test-opt=N       Test option with numeric argument N\n",
         "Usage output as expected with subcommands");
 
-
-
     char *av[] = { "test", "one", NULL };
-    int ac = sizeof (av) / sizeof (av[0]) - 1;
+    int ac = ARRAY_SIZE (av) - 1;
 
     n = optparse_parse_args (a, ac, av);
     ok (n == 1, "optparse_parse_args");
@@ -830,7 +942,7 @@ Usage: test two [OPTIONS]...\n\
     ok (called == 1, "optparse_run_subcommand: called subcmd_one()");
 
     char *av2[] = { "test", "two", NULL };
-    ac = sizeof (av2) / sizeof (av2[0]) - 1;
+    ac = ARRAY_SIZE (av2) - 1;
 
     n = optparse_parse_args (a, ac, av2);
     ok (n == 1, "optparse_parse_args");
@@ -839,7 +951,7 @@ Usage: test two [OPTIONS]...\n\
     ok (called == 2, "optparse_run_subcommand: called subcmd_two()");
 
     char *av3[] = { "test", "two", "--test-opt", "3", NULL };
-    ac = sizeof (av3) / sizeof (av3[0]) - 1;
+    ac = ARRAY_SIZE (av3) - 1;
 
     // Run subcommand before parse also runs subcommand:
     //
@@ -849,7 +961,7 @@ Usage: test two [OPTIONS]...\n\
 
     // Test unknown option prints expected error:
     char *av4[] = { "test", "two", "--unknown", NULL };
-    ac = sizeof (av4) / sizeof (av4[0]) - 1;
+    ac = ARRAY_SIZE (av4) - 1;
 
     e = optparse_set (b, OPTPARSE_FATALERR_FN, do_nothing);
 
@@ -863,7 +975,7 @@ Try `test two --help' for more information.\n",
 
     // Test unknown short option prints expected error
     char *av41[] = { "test", "two", "-X", NULL };
-    ac = sizeof (av41) / sizeof (av41[0]) - 1;
+    ac = ARRAY_SIZE (av41) - 1;
 
     diag ("parsing test two -X");
     n = optparse_run_subcommand (a, ac, av41);
@@ -876,7 +988,7 @@ Try `test two --help' for more information.\n",
 
     // Test unknown short option with good option prints expected error
     char *av42[] = { "test", "two", "-Zt", "foo", NULL};
-    ac = sizeof (av42) / sizeof (av42[0]) - 1;
+    ac = ARRAY_SIZE (av42) - 1;
 
     diag ("parsing test two -Zt foo");
     n = optparse_run_subcommand (a, ac, av42);
@@ -888,9 +1000,21 @@ test two: unrecognized option '-Z'\n\
 Try `test two --help' for more information.\n",
     "bad argument error message is expected");
 
+    // Test missing argument prints expected error
+    char *av43[] = { "test", "two", "-t", NULL};
+    ac = ARRAY_SIZE (av43) - 1;
+
+    diag ("parsing test two -t");
+    n = optparse_run_subcommand (a, ac, av43);
+    ok (n == -1,
+        "optparse_run_subcommand with missing argument fails");
+
+    usage_output_is ("test two: '-t' missing argument\n",
+                     "missing argument error message is expected");
+
     // Test no subcommand (and subcommand required) prints error
     char *av5[] = { "test", NULL };
-    ac = sizeof (av5) / sizeof (av5[0]) - 1;
+    ac = ARRAY_SIZE (av5) - 1;
 
     // Set OPTPARSE_PRINT_SUBCMDS true:
     e = optparse_set (a, OPTPARSE_PRINT_SUBCMDS, 1);
@@ -905,6 +1029,7 @@ Try `test two --help' for more information.\n",
 test: missing subcommand\n\
 Usage: test one [OPTIONS]\n\
    or: test two [OPTIONS]\n\
+   or: test three [OPTIONS]\n\
   -h, --help             Display this message.\n",
     "missing subcommand error message is expected");
 
@@ -919,6 +1044,7 @@ Usage: test one [OPTIONS]\n\
     usage_ok (a, "\
 Usage: test one [OPTIONS]\n\
    or: test two [OPTIONS]\n\
+   or: test three [OPTIONS]\n\
   -h, --help             Display this message.\n",
     "Hidden subcommand doesn't appear in usage output");
 
@@ -927,9 +1053,10 @@ Usage: test one [OPTIONS]\n\
                       OPTPARSE_SUBCMD_HIDE, 0);
     ok (e == OPTPARSE_SUCCESS, "optparse_set (OPTPARSE_SUBCMD_HIDE, 0)");
     usage_ok (a, "\
-Usage: test hidden [OPTIONS]\n\
-   or: test one [OPTIONS]\n\
+Usage: test one [OPTIONS]\n\
    or: test two [OPTIONS]\n\
+   or: test three [OPTIONS]\n\
+   or: test hidden [OPTIONS]\n\
   -h, --help             Display this message.\n",
     "Unhidden subcommand now displayed in usage output");
 
@@ -940,6 +1067,7 @@ Usage: test hidden [OPTIONS]\n\
     usage_ok (a, "\
 Usage: test one [OPTIONS]\n\
    or: test two [OPTIONS]\n\
+   or: test three [OPTIONS]\n\
   -h, --help             Display this message.\n",
     "Unhidden subcommand now displayed in usage output");
 
@@ -953,7 +1081,7 @@ Usage: test one [OPTIONS]\n\
     optparse_set_data (d, "argc", &value);
 
     char *av6[] = { "test", "three", "--help", NULL };
-    ac = sizeof (av6) / sizeof (av6[0]) - 1;
+    ac = ARRAY_SIZE (av6) - 1;
 
     n = optparse_run_subcommand (a, ac, av6);
     ok (n == 0, "optparse_run_subcommand with OPTPARSE_SUBCMD_NOOPTS");
@@ -968,7 +1096,7 @@ void test_corner_case (void)
     optparse_err_t e;
     optparse_t *p = optparse_create ("optarg");
     char *av[] = { "cornercase", NULL };
-    int ac = sizeof (av) / sizeof (av[0]) - 1;
+    int ac = ARRAY_SIZE (av) - 1;
     int optindex;
 
     ok (p != NULL, "optparse_create");
@@ -1019,7 +1147,7 @@ void test_reset (void)
     ok (optparse_option_index (q) == -1, "subcmd: option index is -1");
 
     char *av[] = { "test", "-t", "2", "one", "--test-opt=5", NULL };
-    int ac = sizeof (av) / sizeof (av[0]) - 1;
+    int ac = ARRAY_SIZE (av) - 1;
 
     n = optparse_parse_args (p, ac, av);
     ok (n == 3, "optparse_parse_args() expected 3 got %d", n);
@@ -1069,7 +1197,7 @@ void test_non_option_arguments (void)
     };
     optparse_t *p = optparse_create ("non-option-arg");
     char *av[] = { "non-option-arg", "--test=foo", "--", "baz", NULL };
-    int ac = sizeof (av) / sizeof (*av) - 1;
+    int ac = ARRAY_SIZE (av) - 1;
     int optindex;
 
     ok (p != NULL, "optparse_create");
@@ -1083,7 +1211,7 @@ void test_non_option_arguments (void)
 
     optparse_reset (p);
     char *av2[] = { "non-option-arg", "foo", "bar", NULL };
-    ac = sizeof (av2) / sizeof (*av2) - 1;
+    ac = ARRAY_SIZE (av2) - 1;
     ok (optparse_parse_args (p, ac, av2) != -1, "optparse_parse_args");
     optindex = optparse_option_index (p);
     ok (optindex == 1, "argv with no options, optindex is 1");
@@ -1096,7 +1224,7 @@ void test_non_option_arguments (void)
     //
     optparse_reset (p);
     char *av3[] = { "non-option-arg", "-1234", NULL };
-    ac = sizeof (av3) / sizeof (*av3) - 1;
+    ac = ARRAY_SIZE (av3) - 1;
     ok (optparse_parse_args (p, ac, av3) != -1, "optparse_parse_args");
     optindex = optparse_option_index (p);
     ok (optindex == 1,
@@ -1105,7 +1233,7 @@ void test_non_option_arguments (void)
 
     optparse_reset (p);
     char *av4[] = { "non-option-arg", "1234", "--test=foo", NULL };
-    ac = sizeof (av4) / sizeof (*av4) - 1;
+    ac = ARRAY_SIZE (av4) - 1;
     ok (optparse_parse_args (p, ac, av4) != -1, "optparse_parse_args");
     optindex = optparse_option_index (p);
     ok (optindex == 1,
@@ -1130,24 +1258,230 @@ void test_non_option_arguments (void)
     optparse_destroy (p);
 }
 
+void test_add_option_table_failure ()
+{
+    optparse_err_t e;
+    struct optparse_option opts [] = {
+        { .name = "test",
+          .key  = 't',
+          .has_arg = 1,
+          .arginfo = "S",
+          .usage = "test option"
+        },
+        { .name = "test",
+          .key = 'x',
+          .has_arg = 0,
+          .usage = "test option with same name"
+        },
+        OPTPARSE_TABLE_END,
+    };
+    optparse_t *p = optparse_create ("invalid-table");
+    if (!p)
+        BAIL_OUT ("optparse_create");
 
-int main (int argc, char *argv[])
+    e = optparse_set (p, OPTPARSE_LOG_FN, output_f);
+    ok (e == OPTPARSE_SUCCESS, "optparse_set (LOG_FN)");
+
+    e = optparse_add_option_table (p, opts);
+    ok (e == OPTPARSE_EEXIST,
+        "option table with duplicate name fails with EEXIST");
+
+    usage_ok (p, "\
+Usage: invalid-table [OPTIONS]...\n\
+  -h, --help             Display this message.\n",
+        "No table options registered after optparse_add_option_table failure");
+
+    optparse_destroy (p);
+}
+
+void test_reg_subcommands ()
+{
+    optparse_t *p;
+    struct optparse_option opts [] = {
+        { .name = "test",
+          .key  = 't',
+          .has_arg = 1,
+          .arginfo = "S",
+          .usage = "test option"
+        },
+        OPTPARSE_TABLE_END,
+    };
+    struct optparse_subcommand subcmds[] = {
+        { "sub1",
+          "[OPTIONS]...",
+          "Subcommand one",
+          subcmd,
+          0,
+          opts,
+        },
+        { "sub2",
+          "[OPTIONS]...",
+          "Subcommand two",
+          subcmd,
+          0,
+          NULL,
+        },
+        OPTPARSE_SUBCMD_END
+    };
+
+    p = optparse_create ("reg-subcmds-test");
+    if (!p)
+        BAIL_OUT ("optparse_create");
+    ok (optparse_reg_subcommands (p, subcmds) == OPTPARSE_SUCCESS,
+        "optparse_reg_subcommands works");
+
+    optparse_destroy (p);
+}
+
+static void test_optparse_get ()
+{
+    optparse_t *p = optparse_create ("test-get");
+    if (!p)
+        BAIL_OUT ("optparse_create");
+
+    for (optparse_item_t i = OPTPARSE_USAGE; i < OPTPARSE_ITEM_END; i++)
+        ok (optparse_get (p, i) == OPTPARSE_NOT_IMPL,
+            "optparse_get %d returns NOT IMPL", i);
+
+    ok (optparse_get (p, OPTPARSE_ITEM_END) == OPTPARSE_BAD_ARG,
+        "optparse_get of invalid item returns BAD_ARG");
+
+    optparse_destroy (p);
+}
+
+static void test_optional_args ()
+{
+    char *av1[] = { "test-optional-args", "-xx", "-y2", NULL };
+    int ac1 =  ARRAY_SIZE (av1) - 1;
+
+    char *av2[] = { "test-optional-args", "--testx=2", "--testy=2", NULL };
+    int ac2 =  ARRAY_SIZE (av2) - 1;
+
+    char *av3[] = { "test-optional-args", "--testx", "--testy", NULL };
+    int ac3 =  ARRAY_SIZE (av3) - 1;
+
+    struct optparse_option opts [] = {
+        { .name = "testx",
+          .key  = 'x',
+          .has_arg = 2,
+          .arginfo = "N",
+          .usage = "optional arg on longopt only",
+          .flags = 0,
+        },
+        { .name = "testy",
+          .key  = 'y',
+          .has_arg = 2,
+          .arginfo = "N",
+          .usage = "optional arg on short and longopts",
+          .flags = OPTPARSE_OPT_SHORTOPT_OPTIONAL_ARG,
+        },
+        OPTPARSE_TABLE_END,
+    };
+    optparse_err_t e;
+    optparse_t *p = optparse_create ("test-optional-args");
+    if (!p)
+        BAIL_OUT ("optparse_create");
+
+    e = optparse_add_option_table (p, opts);
+    if (e != OPTPARSE_SUCCESS)
+        BAIL_OUT ("optparse_add_option_table");
+    ok (optparse_parse_args (p, ac1, av1) == ac1,
+        "optparse_parse_args");
+    ok (optparse_get_int (p, "testx", -1) == 2,
+        "shortopt with optional_arg: -xx works by default");
+    ok (optparse_get_int (p, "testy", -1) == 2,
+        "shortopt with optional_arg supported: -y2 works");
+    optparse_destroy (p);
+
+    p = optparse_create ("test-optional-args");
+    if (!p)
+        BAIL_OUT ("optparse_create");
+    e = optparse_add_option_table (p, opts);
+    if (e != OPTPARSE_SUCCESS)
+        BAIL_OUT ("optparse_add_option_table");
+    ok (optparse_parse_args (p, ac2, av2) == ac2,
+        "optparse_parse_args");
+    ok (optparse_get_int (p, "testx", -1) == 2,
+        "shortopt with optional_arg: --testx=2 works by default");
+    ok (optparse_get_int (p, "testy", -1) == 2,
+        "shortopt with optional_arg supported: ---testy=2 also works");
+    optparse_destroy (p);
+
+    p = optparse_create ("test-optional-args");
+    if (!p)
+        BAIL_OUT ("optparse_create");
+    e = optparse_add_option_table (p, opts);
+    if (e != OPTPARSE_SUCCESS)
+        BAIL_OUT ("optparse_add_option_table");
+    ok (optparse_parse_args (p, ac3, av3) == ac3,
+        "optparse_parse_args");
+    ok (optparse_get_int (p, "testx", -1) == 1,
+        "shortopt with optional_arg: --testx sets result to 1");
+    ok (optparse_get_int (p, "testy", -1) == 1,
+        "shortopt with optional_arg supported: ---testy also sets result to 1");
+    optparse_destroy (p);
+}
+
+static void test_issue5732 ()
 {
+    optparse_t *p;
+    struct optparse_option opts [] = {
+        { .name = "dup",
+          .key  = 'd',
+          .has_arg = 1,
+          .arginfo = "S",
+          .usage = "test option"
+        },
+        { .name = "dup",
+          .key  = 'd',
+          .has_arg = 1,
+          .arginfo = "S",
+          .usage = "test option"
+        },
+        OPTPARSE_TABLE_END,
+    };
+    struct optparse_subcommand subcmds[] = {
+        { "sub1",
+          "[OPTIONS]...",
+          "Subcommand one",
+          subcmd,
+          0,
+          opts,
+        },
+        OPTPARSE_SUBCMD_END
+    };
+
+    p = optparse_create ("issue-5732");
+    if (!p)
+        BAIL_OUT ("optparse_create");
+    lives_ok ({ optparse_reg_subcommands (p, subcmds); },
+              "optparse_reg_subcommands lives with duplicated options");
+
+    optparse_destroy (p);
+}
+
 
-    plan (259);
+int main (int argc, char *argv[])
+{
+    plan (324);
 
-    test_convenience_accessors (); /* 35 tests */
-    test_usage_output (); /* 42 tests */
+    test_convenience_accessors (); /* 60 tests */
+    test_usage_output (); /* 46 tests */
     test_option_cb ();  /* 16 tests */
     test_errors (); /* 9 tests */
     test_multiret (); /* 19 tests */
     test_data (); /* 8 tests */
-    test_subcommand (); /* 62 tests */
+    test_subcommand (); /* 70 tests */
     test_long_only (); /* 13 tests */
     test_optional_argument (); /* 9 tests */
     test_corner_case (); /* 3 tests */
     test_reset (); /* 9 tests */
     test_non_option_arguments (); /* 13 tests */
+    test_add_option_table_failure (); /* 4 tests */
+    test_reg_subcommands (); /* 1 test */
+    test_optparse_get (); /* 13 tests */
+    test_optional_args (); /* 9 tests */
+    test_issue5732 (); /* 1 test */
 
     done_testing ();
     return (0);
diff --git a/src/common/libpmi.map b/src/common/libpmi.map
deleted file mode 100644
index 742ac050544d..000000000000
--- a/src/common/libpmi.map
+++ /dev/null
@@ -1,7 +0,0 @@
-{ global:
-    PMI_*;
-    flux_pmi_library;
-    __asan*;
-    local: *;
-};
-
diff --git a/src/common/libpmi/Makefile.am b/src/common/libpmi/Makefile.am
index 77b9072480bc..528e808b842c 100644
--- a/src/common/libpmi/Makefile.am
+++ b/src/common/libpmi/Makefile.am
@@ -6,37 +6,44 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) \
-	$(LIBUUID_CFLAGS)
+	$(JANSSON_CFLAGS)
 
 noinst_LTLIBRARIES = \
 	libpmi_client.la \
-	libpmi_server.la
+	libpmi_server.la \
+	libpmi_common.la \
+	libupmi.la
 
-libpmi_common_sources = \
+libpmi_common_la_SOURCES = \
 	pmi_strerror.c \
 	pmi_strerror.h \
 	keyval.c \
 	keyval.h \
-	clique.c \
-	clique.h
+	sentinel.c
 
 libpmi_client_la_SOURCES = \
 	simple_client.c \
 	simple_client.h \
 	pmi.c \
-	pmi2.c \
-	dgetline.c \
-	dgetline.h \
-	$(libpmi_common_sources)
+	pmi2.c
 
 libpmi_server_la_SOURCES = \
 	simple_server.h \
-	simple_server.c \
-	$(libpmi_common_sources)
+	simple_server.c
+
+libupmi_la_SOURCES = \
+	upmi.h \
+	upmi.c \
+	upmi_plugin.h \
+	upmi_simple.c \
+	upmi_libpmi.c \
+	upmi_libpmi2.c \
+	upmi_single.c
 
 fluxinclude_HEADERS = \
 	pmi.h \
@@ -46,19 +53,16 @@ TESTS = test_keyval.t \
 	test_simple.t \
 	test_canonical.t \
 	test_canonical2.t \
-	test_clique.t
+	test_upmi.t
 
 test_ldadd = \
 	$(top_builddir)/src/common/libflux/libflux.la \
+	$(top_builddir)/src/common/libkvs/libkvs.la \
 	$(top_builddir)/src/common/libpmi/libpmi_client.la \
 	$(top_builddir)/src/common/libpmi/libpmi_server.la \
-	$(top_builddir)/src/common/libutil/libutil.la \
+	$(top_builddir)/src/common/libpmi/libpmi_common.la \
+	$(top_builddir)/src/common/libflux-internal.la \
 	$(top_builddir)/src/common/libtap/libtap.la \
-	$(top_builddir)/src/common/liblsd/liblsd.la \
-	$(top_builddir)/src/common/libtomlc99/libtomlc99.la \
-	$(top_builddir)/src/common/libev/libev.la \
-	$(ZMQ_LIBS) \
-	$(LIBUUID_LIBS) \
 	$(JANSSON_LIBS) \
 	$(LIBPTHREAD) \
 	$(LIBRT) \
@@ -66,12 +70,21 @@ test_ldadd = \
 
 test_cppflags = \
 	-I$(top_srcdir)/src/common/libtap \
+	-DUPMI_TEST_SEARCHPATH=\"$(builddir)/test/plugin/.libs\" \
 	$(AM_CPPFLAGS)
 
+test_ldflags = \
+	-no-install
+
 check_PROGRAMS = \
 	$(TESTS) \
 	test_pmi_info \
-	test_kvstest
+	test_pmi2_info \
+	test_kvstest \
+	test_kvstest2
+
+check_LTLIBRARIES = \
+        test/plugin/singlex.la
 
 TEST_EXTENSIONS = .t
 T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
@@ -102,18 +115,34 @@ test_canonical2_t_SOURCES = \
 test_canonical2_t_CPPFLAGS = $(test_cppflags)
 test_canonical2_t_LDADD = $(test_ldadd)
 
-
-test_clique_t_SOURCES = test/clique.c
-test_clique_t_CPPFLAGS = $(test_cppflags)
-test_clique_t_LDADD = $(test_ldadd)
+test_upmi_t_SOURCES = test/upmi.c
+test_upmi_t_CPPFLAGS = $(test_cppflags)
+test_upmi_t_LDADD = \
+	$(top_builddir)/src/common/libpmi/libupmi.la \
+	$(test_ldadd)
 
 test_pmi_info_SOURCES = test/pmi_info.c
 test_pmi_info_CPPFLAGS = $(test_cppflags)
 test_pmi_info_LDADD = $(test_ldadd)
 
+test_pmi2_info_SOURCES = test/pmi2_info.c
+test_pmi2_info_CPPFLAGS = $(test_cppflags)
+test_pmi2_info_LDADD = $(test_ldadd)
+
 test_kvstest_SOURCES = test/kvstest.c
 test_kvstest_CPPFLAGS = $(test_cppflags)
 test_kvstest_LDADD = $(test_ldadd)
 
+test_kvstest2_SOURCES = test/kvstest2.c
+test_kvstest2_CPPFLAGS = $(test_cppflags)
+test_kvstest2_LDADD = $(test_ldadd)
+
+test_plugin_singlex_la_SOURCES = test/plugin/singlex.c
+test_plugin_singlex_la_CPPFLAGS = $(test_cppflags)
+test_plugin_singlex_la_LIBADD = \
+	$(top_builddir)/src/common/libpmi/libupmi.la \
+	$(test_ldadd)
+test_plugin_singlex_la_LDFLAGS = -module -rpath /nowhere $(test_ldflags)
+
 EXTRA_DIST = \
     ltrace.conf
diff --git a/src/common/libpmi/clique.c b/src/common/libpmi/clique.c
deleted file mode 100644
index 3ef432a095b3..000000000000
--- a/src/common/libpmi/clique.c
+++ /dev/null
@@ -1,220 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdarg.h>
-#include <stdio.h>
-#include <errno.h>
-#include <string.h>
-#include <stdlib.h>
-#include <argz.h>
-
-#include "pmi.h"
-#include "clique.h"
-
-static int catprintf (char **buf, int *bufsz, const char *fmt, ...)
-{
-    va_list ap;
-    int n;
-
-    va_start (ap, fmt);
-    n = vsnprintf (*buf, *bufsz, fmt, ap);
-    va_end (ap);
-    if (n >= *bufsz)
-        return -1;
-    *bufsz -= n;
-    *buf += n;
-    return 0;
-}
-
-int pmi_process_mapping_encode (struct pmi_map_block *blocks,
-                                int nblocks,
-                                char *buf,
-                                int bufsz)
-{
-    int i;
-
-    if (catprintf (&buf, &bufsz, "(vector,") < 0)
-        return -1;
-    for (i = 0; i < nblocks; i++) {
-        if (catprintf (&buf, &bufsz, "%s(%d,%d,%d)",
-                       i > 0 ? "," : "",
-                       blocks[i].nodeid,
-                       blocks[i].nodes,
-                       blocks[i].procs) < 0)
-            return -1;
-    }
-    if (catprintf (&buf, &bufsz, ")") < 0)
-        return -1;
-    return 0;
-}
-
-static int parse_block (const char *s, struct pmi_map_block *block)
-{
-    char *endptr;
-
-    errno = 0;
-    block->nodeid = strtoul (s, &endptr, 10);
-    if (errno != 0 || *endptr != ',')
-        return PMI_FAIL;
-    s = endptr + 1;
-    errno = 0;
-    block->nodes = strtoul (s, &endptr, 10);
-    if (errno != 0 || *endptr != ',')
-        return PMI_FAIL;
-    s = endptr + 1;
-    errno = 0;
-    block->procs = strtoul (s, &endptr, 10);
-    if (errno != 0 || *endptr != ')')
-        return PMI_FAIL;
-    return PMI_SUCCESS;
-}
-
-int pmi_process_mapping_parse (const char *s,
-                               struct pmi_map_block **blocksp, int *nblocksp)
-{
-    char *argz = NULL;
-    size_t argz_len;
-    char *entry = NULL;
-    int nblocks, i;
-    struct pmi_map_block *blocks = NULL;
-    int rc = PMI_FAIL;
-
-    /* Special case empty string.
-     * Not an error: return no blocks.
-     */
-    if (*s == '\0') {
-        *blocksp = NULL;
-        *nblocksp = 0;
-        return PMI_SUCCESS;
-    }
-
-    if (argz_create_sep (s, '(', &argz, &argz_len) != 0) {
-        rc = PMI_ERR_NOMEM;
-        goto error;
-    }
-    nblocks = argz_count (argz, argz_len);
-    while ((entry = argz_next (argz, argz_len, entry))) {
-        nblocks--;
-        if (strstr (entry, "vector,"))
-            break;
-    }
-    if (nblocks == 0)
-        goto error;
-    if (!(blocks = calloc (nblocks, sizeof (blocks[0])))) {
-        rc = PMI_ERR_NOMEM;
-        goto error;
-    }
-    i = 0;
-    while ((entry = argz_next (argz, argz_len, entry))) {
-        if ((rc = parse_block (entry, &blocks[i++])) != PMI_SUCCESS)
-            goto error;
-    }
-    *nblocksp = nblocks;
-    *blocksp = blocks;
-    free (argz);
-    return PMI_SUCCESS;
-error:
-    if (blocks)
-        free (blocks);
-    if (argz)
-        free (argz);
-    return rc;
-}
-
-int pmi_process_mapping_find_nodeid (struct pmi_map_block *blocks, int nblocks,
-                                     int rank, int *nodeid)
-{
-    int i;
-    int brank = 0;
-
-    for (i = 0; i < nblocks; i++) {
-        int lsize = blocks[i].nodes * blocks[i].procs;
-        int lrank = rank - brank;
-
-        if (lrank >= 0 && lrank < lsize) {
-            *nodeid = blocks[i].nodeid + lrank / blocks[i].procs;
-            return PMI_SUCCESS;
-        }
-        brank += lsize;
-    }
-    return PMI_FAIL;
-}
-
-int pmi_process_mapping_find_nranks (struct pmi_map_block *blocks, int nblocks,
-                                     int nodeid, int size, int *nranksp)
-{
-    int i, j;
-    int brank = 0, nranks = 0;
-
-    for (i = 0; i < nblocks; i++) {
-        for (j = 0; j < blocks[i].nodes; j++) {
-            int lsize = blocks[i].procs;
-            if (brank + lsize > size)
-                lsize -= (brank - size);
-            if (blocks[i].nodeid + j == nodeid)
-                nranks += lsize;
-            brank += lsize;
-        }
-    }
-    *nranksp = nranks;
-    return PMI_SUCCESS;
-}
-
-int pmi_process_mapping_find_ranks (struct pmi_map_block *blocks, int nblocks,
-                                    int nodeid, int size,
-                                    int *ranks, int nranks)
-{
-    int i, j, k, nx = 0;
-    int brank = 0;
-
-    for (i = 0; i < nblocks; i++) {
-        for (j = 0; j < blocks[i].nodes; j++) {
-            int lsize = blocks[i].procs;
-            if (brank + lsize > size)
-                lsize -= (brank - size);
-            if (blocks[i].nodeid + j == nodeid) {
-                for (k = 0; k < lsize; k++) {
-                    if (nx >=  nranks)
-                        return PMI_ERR_INVALID_SIZE;
-                    ranks[nx++] = brank + k;
-                }
-            }
-            brank += lsize;
-        }
-    }
-    if (nx != nranks)
-        return PMI_ERR_INVALID_SIZE;
-    return PMI_SUCCESS;
-}
-
-char *pmi_cliquetostr (char *buf, int bufsz, int *ranks, int length)
-{
-    int n, i, count;
-
-    buf[0] = '\0';
-    for (i = 0, count  = 0; i < length; i++) {
-        n = snprintf (buf + count,
-                      bufsz - count,
-                      "%s%d",
-                      i > 0 ? "," : "",
-                      ranks[i]);
-        if (n >= bufsz - count)
-            return "overflow";
-        count += n;
-    }
-    return buf;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libpmi/clique.h b/src/common/libpmi/clique.h
deleted file mode 100644
index 2c32ba2bfe3c..000000000000
--- a/src/common/libpmi/clique.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_CORE_PMI_CLIQUE_H
-#define _FLUX_CORE_PMI_CLIQUE_H
-
-/* Parse the PMI_process_mapping attribute.
- *
- * The term "nodeid" below refers to a zero-origin logical nodeid within
- * the process group.  We can ask questions such
- *
- * - which nodeid will launch a given rank?
- * - how many procs will be launched on a given nodeid?
- * - which ranks will be launched on a given nodeid?
- *
- * N.B. due to the fixed PMI KVS value size, and the fact that a process
- * group can be mapped irregularly, some mappings may not be communicable
- * using this attribute.  Therefore, an empty value is to be interpreted
- * as "no mapping available", and should be handled as a non-fatal error.
- *
- * These functions return PMI result codes.
- */
-
-struct pmi_map_block {
-    int nodeid;
-    int nodes;
-    int procs;
-};
-
-/* Parse PMI_process_mapping value in 's' into an array of
- * struct pmi_map_blocks, returned in 'blocks', length in 'nblocks'.
- * The caller must free 'blocks'.
- */
-int pmi_process_mapping_parse (const char *s,
-                               struct pmi_map_block **blocks, int *nblocks);
-
-/* Generate PMI_process_mapping value string from array of pmi_map_blocks,
- * and place it in 'buf'.  Result will be null terminated.
- */
-int pmi_process_mapping_encode (struct pmi_map_block *blocks,
-                                int nblocks,
-                                char *buf,
-                                int bufsz);
-
-
-/* Determine the nodeid that will start 'rank', and return it in 'nodeid'.
- */
-int pmi_process_mapping_find_nodeid (struct pmi_map_block *blocks, int nblocks,
-                                     int rank, int *nodeid);
-
-/* Determine the number of ranks started by 'nodeid', and return it in 'nranks'.
- */
-int pmi_process_mapping_find_nranks (struct pmi_map_block *blocks, int nblocks,
-                                     int nodeid, int size, int *nranks);
-
-/* Determine the ranks that will be started by 'nodeid'.
- * The caller should supply a pre-allocated array in 'ranks' of length
- * 'nranks', sized according to find_nranks() above.
- */
-int pmi_process_mapping_find_ranks (struct pmi_map_block *blocks, int nblocks,
-                                    int nodeid, int size,
-                                    int *ranks, int nranks);
-
-
-/* Convert rank array to csv string.  If clique is empty, return empty string.
- * If string overflows, return "overflow".
- */
-char *pmi_cliquetostr (char *buf, int bufsz, int *ranks, int length);
-
-#endif /* _FLUX_CORE_PMI_CLIQUE_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libpmi/dgetline.c b/src/common/libpmi/dgetline.c
deleted file mode 100644
index dc2b0d415a48..000000000000
--- a/src/common/libpmi/dgetline.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-
-#include "dgetline.h"
-
-int dgetline (int fd, char *buf, int len)
-{
-    int i = 0;
-    while (i < len - 1) {
-        if (read (fd, &buf[i], 1) <= 0)
-            return -1;
-        if (buf[i++] == '\n')
-            break;
-    }
-    if (buf[i - 1] != '\n') {
-        errno = EPROTO;
-        return -1;
-    }
-    buf[i] = '\0';
-    return 0;
-}
-
-int dputline (int fd, const char *buf)
-{
-    int len = strlen (buf);
-    int n, count = 0;
-    while (count < len) {
-        if ((n = write (fd, buf + count, len - count)) < 0)
-            return n;
-        count += n;
-    }
-    return count;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libpmi/dgetline.h b/src/common/libpmi/dgetline.h
deleted file mode 100644
index 15d27d97df03..000000000000
--- a/src/common/libpmi/dgetline.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_LIBPMI_DGETLINE_H
-#define _FLUX_LIBPMI_DGETLINE_H
-
-int dgetline (int fd, char *buf, int len);
-int dputline (int fd, const char *buf);
-
-#endif /* !_FLUX_LIBPMI_DGETLINE_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libpmi/keyval.c b/src/common/libpmi/keyval.c
index 5f370592f379..b6f31f05b123 100644
--- a/src/common/libpmi/keyval.c
+++ b/src/common/libpmi/keyval.c
@@ -12,9 +12,11 @@
 # include "config.h"
 #endif
 #include <stdlib.h>
+#include <limits.h>
 #include <string.h>
 #include <ctype.h>
 #include <errno.h>
+#include "ccan/str/str.h"
 #include "keyval.h"
 
 static const char *parse_val (const char *s, const char *key)
@@ -36,12 +38,14 @@ int keyval_parse_uint (const char *s, const char *key, unsigned int *val)
 {
     const char *cp = parse_val (s, key);
     char *endptr;
-    unsigned int i;
+    unsigned long i;
     if (!cp)
         return EKV_NOKEY;
     errno = 0;
     i = strtoul (cp, &endptr, 10);
-    if (errno != 0 || (*endptr && !isspace (*endptr)))
+    if (errno != 0
+        || (*endptr && !isspace (*endptr))
+        || i > UINT_MAX)
         return EKV_VAL_PARSE;
     *val = i;
     return EKV_SUCCESS;
@@ -51,12 +55,15 @@ int keyval_parse_int (const char *s, const char *key, int *val)
 {
     const char *cp = parse_val (s, key);
     char *endptr;
-    int i;
+    long i;
     if (!cp)
         return EKV_NOKEY;
     errno = 0;
     i = strtol (cp, &endptr, 10);
-    if (errno != 0 || (*endptr && !isspace (*endptr)))
+    if (errno != 0
+        || (*endptr && !isspace (*endptr))
+        || i > INT_MAX
+        || i < INT_MIN)
         return EKV_VAL_PARSE;
     *val = i;
     return EKV_SUCCESS;
@@ -93,15 +100,22 @@ int keyval_parse_isword (const char *s, const char *key, const char *match)
 int keyval_parse_string (const char *s, const char *key, char *val, int len)
 {
     const char *cp = parse_val (s, key);
+    char *vp = val;
     if (!cp)
         return EKV_NOKEY;
     while (len > 0 && *cp && *cp != '\n') {
-        *val++ = *cp++;
+        *vp++ = *cp++;
         len--;
     }
     if (len == 0)
         return EKV_VAL_LEN;
-    *val++ = '\0';
+    *vp++ = '\0';
+   /* Quirk: mpiexec.hydra from mpich v4.2.0 and v4.1.1 appends "found=TRUE"
+    * to KVS get responses due to a presumed bug.  Ignore.
+    * See flux-framework/flux-core#6072.
+    */
+    if (streq (key, "value") && strends (val, " found=TRUE"))
+        val[strlen (val) - 11] = '\0';
     return EKV_SUCCESS;
 }
 
diff --git a/src/common/libpmi/keyval.h b/src/common/libpmi/keyval.h
index 57dc8de98782..c4266162d571 100644
--- a/src/common/libpmi/keyval.h
+++ b/src/common/libpmi/keyval.h
@@ -34,7 +34,7 @@ enum {
  *    last tuple in the input string.
  *
  * Functions return EKV_SUCCESS on success, one of the negative EKV_
- * values on falure.
+ * values on failure.
  */
 
 int keyval_parse_int (const char *s, const char *key, int *val);
diff --git a/src/common/libpmi/pmi.c b/src/common/libpmi/pmi.c
index 54643ac701e0..804b2b6dd127 100644
--- a/src/common/libpmi/pmi.c
+++ b/src/common/libpmi/pmi.c
@@ -36,7 +36,6 @@
 #include "pmi.h"
 #include "pmi_strerror.h"
 #include "simple_client.h"
-#include "clique.h"
 
 static struct pmi_simple_client *pmi_global_ctx;
 
@@ -89,14 +88,15 @@ int PMI_Finalize (void)
     return result;
 }
 
-int PMI_Abort (int exit_code, const char error_msg[])
+int PMI_Abort (int exit_code, const char *error_msg)
 {
     /* pmi_simple_client_abort() only returns on error, in which case
      * we fall back to printing the message on stderr and call exit()
      * (return code not checked because we don't do anything with it)
      */
     (void) pmi_simple_client_abort (pmi_global_ctx, exit_code, error_msg);
-    fprintf (stderr, "PMI_Abort: (%d) %s\n",
+    fprintf (stderr,
+             "PMI_Abort: (%d) %s\n",
              pmi_global_ctx ? pmi_global_ctx->rank : -1,
              error_msg);
     exit (exit_code);
@@ -134,7 +134,7 @@ int PMI_Get_appnum (int *appnum)
     return pmi_simple_client_get_appnum (pmi_global_ctx, appnum);
 }
 
-int PMI_KVS_Get_my_name (char kvsname[], int length)
+int PMI_KVS_Get_my_name (char *kvsname, int length)
 {
     return pmi_simple_client_kvs_get_my_name (pmi_global_ctx,
                                               kvsname,
@@ -171,13 +171,15 @@ int PMI_KVS_Get_value_length_max (int *length)
     return PMI_SUCCESS;
 }
 
-int PMI_KVS_Put (const char kvsname[], const char key[], const char value[])
+int PMI_KVS_Put (const char *kvsname, const char *key, const char *value)
 {
     return pmi_simple_client_kvs_put (pmi_global_ctx, kvsname, key, value);
 }
 
-int PMI_KVS_Get (const char kvsname[], const char key[],
-                 char value[], int length)
+int PMI_KVS_Get (const char *kvsname,
+                 const char *key,
+                 char *value,
+                 int length)
 {
     return pmi_simple_client_kvs_get (pmi_global_ctx,
                                       kvsname,
@@ -186,7 +188,7 @@ int PMI_KVS_Get (const char kvsname[], const char key[],
                                       length);
 }
 
-int PMI_KVS_Commit (const char kvsname[])
+int PMI_KVS_Commit (const char *kvsname)
 {
     if (!pmi_global_ctx || !pmi_global_ctx->initialized)
         return PMI_ERR_INIT;
@@ -200,30 +202,30 @@ int PMI_Barrier (void)
     return pmi_simple_client_barrier (pmi_global_ctx);
 }
 
-int PMI_Publish_name (const char service_name[], const char port[])
+int PMI_Publish_name (const char *service_name, const char *port)
 {
     return PMI_FAIL;
 }
 
-int PMI_Unpublish_name (const char service_name[])
+int PMI_Unpublish_name (const char *service_name)
 {
     return PMI_FAIL;
 }
 
-int PMI_Lookup_name (const char service_name[], char port[])
+int PMI_Lookup_name (const char *service_name, char *port)
 {
     return PMI_FAIL;
 }
 
 int PMI_Spawn_multiple(int count,
-                       const char * cmds[],
-                       const char ** argvs[],
-                       const int maxprocs[],
-                       const int info_keyval_sizesp[],
-                       const PMI_keyval_t * info_keyval_vectors[],
+                       const char **cmds,
+                       const char **argvs[],
+                       const int *maxprocs,
+                       const int *info_keyval_sizesp,
+                       const PMI_keyval_t **info_keyval_vectors,
                        int preput_keyval_size,
-                       const PMI_keyval_t preput_keyval_vector[],
-                       int errors[])
+                       const PMI_keyval_t *preput_keyval_vector,
+                       int *errors)
 {
     return PMI_FAIL;
 }
@@ -231,7 +233,7 @@ int PMI_Spawn_multiple(int count,
 /* Old API funcs - signatures needed for ABI compliance.
  */
 
-int PMI_Get_clique_ranks (int ranks[], int length)
+int PMI_Get_clique_ranks (int *ranks, int length)
 {
     return pmi_simple_client_get_clique_ranks (pmi_global_ctx, ranks, length);
 }
@@ -246,51 +248,62 @@ int PMI_Get_id_length_max (int *length)
     return PMI_KVS_Get_name_length_max (length);
 }
 
-int PMI_Get_id (char kvsname[], int length)
+int PMI_Get_id (char *kvsname, int length)
 {
     return PMI_KVS_Get_my_name (kvsname, length);
 }
 
-int PMI_Get_kvs_domain_id (char kvsname[], int length)
+int PMI_Get_kvs_domain_id (char *kvsname, int length)
 {
     return PMI_KVS_Get_my_name (kvsname, length);
 }
 
-int PMI_KVS_Create (char kvsname[], int length)
+int PMI_KVS_Create (char *kvsname, int length)
 {
     return PMI_FAIL;
 }
 
-int PMI_KVS_Destroy (const char kvsname[])
+int PMI_KVS_Destroy (const char *kvsname)
 {
     return PMI_FAIL;
 }
 
-int PMI_KVS_Iter_first (const char kvsname[], char key[], int key_len,
-                        char val[], int val_len)
+int PMI_KVS_Iter_first (const char *kvsname,
+                        char *key,
+                        int key_len,
+                        char *val,
+                        int val_len)
 {
     return PMI_FAIL;
 }
 
-int PMI_KVS_Iter_next (const char kvsname[], char key[], int key_len,
-                       char val[], int val_len)
+int PMI_KVS_Iter_next (const char *kvsname,
+                       char *key,
+                       int key_len,
+                       char *val,
+                       int val_len)
 {
     return PMI_FAIL;
 }
 
-int PMI_Parse_option (int num_args, char *args[], int *num_parsed,
-                      PMI_keyval_t **keyvalp, int *size)
+int PMI_Parse_option (int num_args,
+                      char **args,
+                      int *num_parsed,
+                      PMI_keyval_t **keyvalp,
+                      int *size)
 {
     return PMI_FAIL;
 }
 
-int PMI_Args_to_keyval (int *argcp, char *((*argvp)[]),
-                        PMI_keyval_t **keyvalp, int *size)
+int PMI_Args_to_keyval (int *argcp,
+                        char *((*argvp)[]),
+                        PMI_keyval_t **keyvalp,
+                        int *size)
 {
     return PMI_FAIL;
 }
 
-int PMI_Free_keyvals (PMI_keyval_t keyvalp[], int size)
+int PMI_Free_keyvals (PMI_keyval_t *keyvalp, int size)
 {
     return PMI_FAIL;
 }
diff --git a/src/common/libpmi/pmi.h b/src/common/libpmi/pmi.h
index a758777ed8f7..f9cd818fd87f 100644
--- a/src/common/libpmi/pmi.h
+++ b/src/common/libpmi/pmi.h
@@ -13,6 +13,10 @@
 #ifndef FLUX_PMI_H_INCLUDED
 #define FLUX_PMI_H_INCLUDED
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define PMI_SUCCESS                  0
 #define PMI_FAIL                    -1
 #define PMI_ERR_INIT                 1
@@ -32,68 +36,85 @@
 int PMI_Init (int *spawned);
 int PMI_Initialized (int *initialized);
 int PMI_Finalize (void);
-int PMI_Abort (int exit_code, const char error_msg[]);
+int PMI_Abort (int exit_code, const char *error_msg);
 
 int PMI_Get_size (int *size);
 int PMI_Get_rank (int *rank);
 int PMI_Get_universe_size (int *size);
 int PMI_Get_appnum (int *appnum);
 
-int PMI_Get_clique_ranks (int ranks[], int length);
+int PMI_Get_clique_ranks (int *ranks, int length);
 int PMI_Get_clique_size (int *size);
 
-int PMI_KVS_Get_my_name (char kvsname[], int length);
+int PMI_KVS_Get_my_name (char *kvsname, int length);
 
 int PMI_KVS_Get_name_length_max (int *length);
 int PMI_KVS_Get_key_length_max (int *length);
 int PMI_KVS_Get_value_length_max (int *length);
 
-int PMI_KVS_Put (const char kvsname[], const char key[], const char value[]);
-int PMI_KVS_Get (const char kvsname[], const char key[],
-                  char value[], int length);
-int PMI_KVS_Commit (const char kvsname[]);
+int PMI_KVS_Put (const char *kvsname, const char *key, const char *value);
+int PMI_KVS_Get (const char *kvsname,
+                 const char *key,
+                 char *value,
+                 int length);
+int PMI_KVS_Commit (const char *kvsname);
 int PMI_Barrier (void);
 
-int PMI_Publish_name (const char service_name[], const char port[]);
-int PMI_Unpublish_name (const char service_name[]);
-int PMI_Lookup_name (const char service_name[], char port[]);
+int PMI_Publish_name (const char *service_name, const char *port);
+int PMI_Unpublish_name (const char *service_name);
+int PMI_Lookup_name (const char *service_name, char *port);
 
 typedef struct {
-    const char * key;
-    char * val;
+    const char *key;
+    char *val;
 } PMI_keyval_t;
 
 int PMI_Spawn_multiple(int count,
-                       const char * cmds[],
-                       const char ** argvs[],
-                       const int maxprocs[],
-                       const int info_keyval_sizesp[],
-                       const PMI_keyval_t * info_keyval_vectors[],
+                       const char *cmds[],
+                       const char **argvs[],
+                       const int *maxprocs,
+                       const int *info_keyval_sizesp,
+                       const PMI_keyval_t **info_keyval_vectors,
                        int preput_keyval_size,
                        const PMI_keyval_t preput_keyval_vector[],
-                       int errors[]);
+                       int *errors);
 
 
 /* Old API funcs - signatures needed for ABI compliance.
  */
 int PMI_Get_id_length_max (int *length);
-int PMI_Get_id (char kvsname[], int length);
-int PMI_Get_kvs_domain_id (char kvsname[], int length);
-
-int PMI_KVS_Create (char kvsname[], int length);
-int PMI_KVS_Destroy (const char kvsname[]);
-int PMI_KVS_Iter_first (const char kvsname[], char key[], int key_len,
-                        char val[], int val_len);
-int PMI_KVS_Iter_next (const char kvsname[], char key[], int key_len,
-                       char val[], int val_len);
-
-int PMI_Parse_option (int num_args, char *args[], int *num_parsed,
-                      PMI_keyval_t **keyvalp, int *size);
-int PMI_Args_to_keyval (int *argcp, char *((*argvp)[]),
-                        PMI_keyval_t **keyvalp, int *size);
-int PMI_Free_keyvals (PMI_keyval_t keyvalp[], int size);
+int PMI_Get_id (char *kvsname, int length);
+int PMI_Get_kvs_domain_id (char *kvsname, int length);
+
+int PMI_KVS_Create (char *kvsname, int length);
+int PMI_KVS_Destroy (const char *kvsname);
+int PMI_KVS_Iter_first (const char *kvsname,
+                        char *key,
+                        int key_len,
+                        char *val,
+                        int val_len);
+int PMI_KVS_Iter_next (const char *kvsname,
+                       char *key,
+                       int key_len,
+                       char *val,
+                       int val_len);
+
+int PMI_Parse_option (int num_args,
+                      char **args,
+                      int *num_parsed,
+                      PMI_keyval_t **keyvalp,
+                      int *size);
+int PMI_Args_to_keyval (int *argcp,
+                        char *((*argvp)[]),
+                        PMI_keyval_t **keyvalp,
+                        int *size);
+int PMI_Free_keyvals (PMI_keyval_t *keyvalp, int size);
 int PMI_Get_options (char *str, int *length);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* !FLUX_PMI_H_INCLUDED */
 
 /*
diff --git a/src/common/libpmi/pmi2.c b/src/common/libpmi/pmi2.c
index cf2810d35c91..42f7cf72388d 100644
--- a/src/common/libpmi/pmi2.c
+++ b/src/common/libpmi/pmi2.c
@@ -42,12 +42,18 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <unistd.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
 #include <sys/types.h>
 #include <string.h>
 
+#include "ccan/str/str.h"
+#ifndef HAVE_STRLCPY
+#include "src/common/libmissing/strlcpy.h"
+#endif
+
 #include "pmi.h"
 #include "pmi2.h"
 #include "pmi_strerror.h"
@@ -114,14 +120,15 @@ int PMI2_Initialized (void)
     return 0;
 }
 
-int PMI2_Abort (int flag, const char msg[])
+int PMI2_Abort (int flag, const char *msg)
 {
     /* pmi_simple_client_abort() only returns on error, in which case
      * we fall back to printing the msg on stderr and call exit().
      * (return code not checked because we don't do anything with it)
      */
     (void) pmi_simple_client_abort (pmi_global_ctx, 1, msg);
-    fprintf (stderr, "PMI2_Abort: (%d) %s\n",
+    fprintf (stderr,
+             "PMI2_Abort: (%d) %s\n",
              pmi_global_ctx ? pmi_global_ctx->rank : -1,
              msg ? msg : "NULL");
     exit (1);
@@ -129,22 +136,26 @@ int PMI2_Abort (int flag, const char msg[])
     return PMI_SUCCESS;
 }
 
-int PMI2_Job_Spawn (int count, const char * cmds[],
-                    int argcs[], const char ** argvs[],
-                    const int maxprocs[],
-                    const int info_keyval_sizes[],
-                    const struct MPID_Info *info_keyval_vectors[],
+int PMI2_Job_Spawn (int count,
+                    const char **cmds,
+                    int *argcs,
+                    const char **argvs[],
+                    const int *maxprocs,
+                    const int *info_keyval_sizes,
+                    const struct MPID_Info **info_keyval_vectors,
                     int preput_keyval_size,
-                    const struct MPID_Info *preput_keyval_vector[],
-                    char jobId[], int jobIdSize,
-                    int errors[])
+                    const struct MPID_Info **preput_keyval_vector,
+                    char *jobId,
+                    int jobIdSize,
+                    int *errors)
 {
     return PMI_FAIL;
 }
 
 /* Look up kvsname on first request, then cache for subsequent requests.
  */
-static int get_cached_kvsname (struct pmi_simple_client *pmi, const char **name)
+static int get_cached_kvsname (struct pmi_simple_client *pmi,
+                               const char **name)
 {
     const char *auxkey = "flux::kvsname";
     char *kvsname;
@@ -175,7 +186,7 @@ static int get_cached_kvsname (struct pmi_simple_client *pmi, const char **name)
 
 /* MPICH: treats PMI2_Job_GetId() equivalent to PMI_KVS_Get_my_name().
  */
-int PMI2_Job_GetId (char jobid[], int jobid_size)
+int PMI2_Job_GetId (char *jobid, int jobid_size)
 {
     const char *kvsname;
     int result;
@@ -183,9 +194,8 @@ int PMI2_Job_GetId (char jobid[], int jobid_size)
     result = get_cached_kvsname (pmi_global_ctx, &kvsname);
     if (result != PMI2_SUCCESS)
         return result;
-    if (!jobid || jobid_size < (int)strlen (kvsname) + 1)
+    if (!jobid || strlcpy (jobid, kvsname, jobid_size) < strlen (kvsname))
         return PMI2_ERR_INVALID_ARGS;
-    strcpy (jobid, kvsname);
     return PMI2_SUCCESS;
 }
 
@@ -194,17 +204,17 @@ int PMI2_Job_GetRank (int *rank)
     return PMI2_FAIL;
 }
 
-int PMI2_Job_Connect (const char jobid[], PMI2_Connect_comm_t *conn)
+int PMI2_Job_Connect (const char *jobid, PMI2_Connect_comm_t *conn)
 {
     return PMI2_FAIL;
 }
 
-int PMI2_Job_Disconnect (const char jobid[])
+int PMI2_Job_Disconnect (const char *jobid)
 {
     return PMI2_FAIL;
 }
 
-int PMI2_KVS_Put (const char key[], const char value[])
+int PMI2_KVS_Put (const char *key, const char *value)
 {
     const char *kvsname;
     int result;
@@ -221,13 +231,18 @@ int PMI2_KVS_Put (const char key[], const char value[])
  */
 int PMI2_KVS_Get (const char *jobid,
                   int src_pmi_id,
-                  const char key[],
-                  char value [],
+                  const char *key,
+                  char *value,
                   int maxvalue,
                   int *vallen)
 {
     int result;
 
+    if (!jobid) {
+        result = get_cached_kvsname (pmi_global_ctx, &jobid);
+        if (result != PMI2_SUCCESS)
+            return result;
+    }
     result = pmi_simple_client_kvs_get (pmi_global_ctx,
                                         jobid,
                                         key,
@@ -248,28 +263,94 @@ int PMI2_Info_GetSize (int *size)
     return PMI2_FAIL;
 }
 
-int PMI2_Info_GetNodeAttr (const char name[],
-                           char value[], int valuelen, int *found, int waitfor)
+/* Cray MPI: look up a node-scope key stored with PMI2_Info_PutNodeAttr().
+ * If waitfor is nonzero, try once per second until the key is available.
+ */
+int PMI2_Info_GetNodeAttr (const char *name,
+                           char *value,
+                           int valuelen,
+                           int *found,
+                           int waitfor)
 {
-    return PMI2_FAIL;
+    const char *kvsname;
+    int result;
+    int tries = 0;
+    const int max_tries = 30;
+    char local_name[PMI2_MAX_KEYLEN + 8];
+
+    if (!name || !value)
+        return PMI2_ERR_INVALID_ARG;
+    result = get_cached_kvsname (pmi_global_ctx, &kvsname);
+    if (result != PMI2_SUCCESS)
+        return result;
+    if (snprintf (local_name,
+                  sizeof (local_name),
+                  "local::%s", name) >= sizeof (local_name))
+        return PMI2_ERR_INVALID_KEY_LENGTH;
+    do {
+        if (tries++ > 0)
+            sleep (1);
+        result = pmi_simple_client_kvs_get (pmi_global_ctx,
+                                            kvsname,
+                                            local_name,
+                                            value,
+                                            valuelen);
+        if (result != PMI2_ERR_INVALID_KEY && result != PMI2_SUCCESS)
+            return result;
+        tries++;
+    } while (result == PMI2_ERR_INVALID_KEY
+        && waitfor != 0
+        && tries < max_tries);
+    if (found) {
+        *found = (result == PMI2_SUCCESS) ? 1 : 0;
+        return PMI2_SUCCESS;
+    }
+    return result;
 }
 
-int PMI2_Info_GetNodeAttrIntArray (const char name[], int array[],
-                                   int arraylen, int *outlen, int *found)
+int PMI2_Info_GetNodeAttrIntArray (const char *name,
+                                   int *array,
+                                   int arraylen,
+                                   int *outlen,
+                                   int *found)
 {
     return PMI2_FAIL;
 }
 
-int PMI2_Info_PutNodeAttr (const char name[], const char value[])
+/* Cray MPI: prefix node local keys with local:: to tell the flux PMI plugin
+ * not to exchange them.  They are immediately available for kvs_get by procs
+ * on the same shell.
+ */
+int PMI2_Info_PutNodeAttr (const char *name, const char *value)
 {
-    return PMI2_FAIL;
+    const char *kvsname;
+    int result;
+    char local_name[PMI2_MAX_KEYLEN + 8];
+
+    if (!name || !value)
+        return PMI2_ERR_INVALID_ARG;
+    result = get_cached_kvsname (pmi_global_ctx, &kvsname);
+    if (result != PMI2_SUCCESS)
+        return result;
+
+    if (snprintf (local_name,
+                  sizeof (local_name),
+                  "local::%s", name) >= sizeof (local_name))
+        return PMI2_ERR_INVALID_KEY_LENGTH;
+
+    return pmi_simple_client_kvs_put (pmi_global_ctx,
+                                      kvsname,
+                                      local_name,
+                                      value);
 }
 
 /* MPICH: only fetches PMI_process_mapping and universeSize
  * with PMI2_Info_GetJobAttr().
  */
-int PMI2_Info_GetJobAttr (const char name[],
-                          char value[], int valuelen, int *found)
+int PMI2_Info_GetJobAttr (const char *name,
+                          char *value,
+                          int valuelen,
+                          int *found)
 {
     int result;
 
@@ -277,7 +358,7 @@ int PMI2_Info_GetJobAttr (const char name[],
         result = PMI2_ERR_INVALID_ARG;
         goto error;
     }
-    if (!strcmp (name, "PMI_process_mapping")) {
+    if (streq (name, "PMI_process_mapping")) {
         const char *kvsname;
 
         result = get_cached_kvsname (pmi_global_ctx, &kvsname);
@@ -291,7 +372,7 @@ int PMI2_Info_GetJobAttr (const char name[],
         if (result != PMI2_SUCCESS)
             goto error;
     }
-    else if (!strcmp (name, "universeSize")) {
+    else if (streq (name, "universeSize")) {
         int universe_size;
 
         result = pmi_simple_client_get_universe_size (pmi_global_ctx,
@@ -319,26 +400,31 @@ int PMI2_Info_GetJobAttr (const char name[],
     return result;
 }
 
-int PMI2_Info_GetJobAttrIntArray (const char name[], int array[],
-                                  int arraylen, int *outlen, int *found)
+int PMI2_Info_GetJobAttrIntArray (const char *name,
+                                  int *array,
+                                  int arraylen,
+                                  int *outlen,
+                                  int *found)
 {
     return PMI2_FAIL;
 }
 
-int PMI2_Nameserv_publish (const char service_name[],
-                           const struct MPID_Info *info_ptr, const char port[])
+int PMI2_Nameserv_publish (const char *service_name,
+                           const struct MPID_Info *info_ptr,
+                           const char *port)
 {
     return PMI2_FAIL;
 }
 
-int PMI2_Nameserv_lookup (const char service_name[],
+int PMI2_Nameserv_lookup (const char *service_name,
                           const struct MPID_Info *info_ptr,
-                          char port[], int portLen)
+                          char *port,
+                          int portLen)
 {
     return PMI2_FAIL;
 }
 
-int PMI2_Nameserv_unpublish (const char service_name[],
+int PMI2_Nameserv_unpublish (const char *service_name,
                              const struct MPID_Info *info_ptr)
 {
     return PMI2_FAIL;
diff --git a/src/common/libpmi/pmi2.h b/src/common/libpmi/pmi2.h
index e99445ca6e8f..c63292a9225c 100644
--- a/src/common/libpmi/pmi2.h
+++ b/src/common/libpmi/pmi2.h
@@ -11,6 +11,10 @@
 #ifndef FLUX_PMI2_H_INCLUDED
 #define FLUX_PMI2_H_INCLUDED
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define PMI2_SUCCESS                0
 #define PMI2_FAIL                   -1
 #define PMI2_ERR_INIT               1
@@ -29,6 +33,11 @@
 #define PMI2_ERR_OTHER              14
 
 #define PMI2_MAX_KEYLEN             64
+
+/* PMI2_MAX_VALLEN of 1024 is a de-facto standard and should not be
+ * increased. Experimentally increasing to 2048 was shown to cause
+ * crashes with mpich and mvapich2 MPI implementations.
+ */
 #define PMI2_MAX_VALLEN             1024
 #define PMI2_MAX_ATTRVALUE          1024
 #define PMI2_ID_NULL                -1
@@ -37,7 +46,7 @@
 int PMI2_Init (int *spawned, int *size, int *rank, int *appnum);
 int PMI2_Finalize (void);
 int PMI2_Initialized (void);
-int PMI2_Abort (int flag, const char msg[]);
+int PMI2_Abort (int flag, const char *msg);
 
 
 typedef struct PMI2_Connect_comm {
@@ -56,48 +65,70 @@ typedef struct MPID_Info {
     char                *value;
 } MPID_Info;
 
-int PMI2_Job_Spawn (int count, const char * cmds[],
-                    int argcs[], const char ** argvs[],
+int PMI2_Job_Spawn (int count,
+                    const char **cmds,
+                    int *argcs,
+                    const char **argvs[],
                     const int maxprocs[],
-                    const int info_keyval_sizes[],
-                    const struct MPID_Info *info_keyval_vectors[],
+                    const int *info_keyval_sizes,
+                    const struct MPID_Info **info_keyval_vectors,
                     int preput_keyval_size,
-                    const struct MPID_Info *preput_keyval_vector[],
-                    char jobId[], int jobIdSize,
-                    int errors[]);
-int PMI2_Job_GetId (char jobid[], int jobid_size);
+                    const struct MPID_Info **preput_keyval_vector,
+                    char *jobId,
+                    int jobIdSize,
+                    int *errors);
+int PMI2_Job_GetId (char *jobid, int jobid_size);
 int PMI2_Job_GetRank (int* rank);
-int PMI2_Job_Connect (const char jobid[], PMI2_Connect_comm_t *conn);
-int PMI2_Job_Disconnect (const char jobid[]);
+int PMI2_Job_Connect (const char *jobid, PMI2_Connect_comm_t *conn);
+int PMI2_Job_Disconnect (const char *jobid);
 
 
-int PMI2_KVS_Put (const char key[], const char value[]);
-int PMI2_KVS_Get (const char *jobid, int src_pmi_id,
-                  const char key[], char value [], int maxvalue, int *vallen);
+int PMI2_KVS_Put (const char *key, const char *value);
+int PMI2_KVS_Get (const char *jobid,
+                  int src_pmi_id,
+                  const char *key,
+                  char *value,
+                  int maxvalue,
+                  int *vallen);
 int PMI2_KVS_Fence (void);
 
 
-int PMI2_Info_GetSize (int* size);
-int PMI2_Info_GetNodeAttr (const char name[],
-                           char value[], int valuelen, int *found, int waitfor);
-int PMI2_Info_GetNodeAttrIntArray (const char name[], int array[],
-                                   int arraylen, int *outlen, int *found);
-int PMI2_Info_PutNodeAttr (const char name[], const char value[]);
-int PMI2_Info_GetJobAttr (const char name[],
-                          char value[], int valuelen, int *found);
-int PMI2_Info_GetJobAttrIntArray (const char name[], int array[],
-                                  int arraylen, int *outlen, int *found);
-
-
-int PMI2_Nameserv_publish (const char service_name[],
-                           const struct MPID_Info *info_ptr, const char port[]);
-int PMI2_Nameserv_lookup (const char service_name[],
+int PMI2_Info_GetSize (int *size);
+int PMI2_Info_GetNodeAttr (const char *name,
+                           char *value,
+                           int valuelen,
+                           int *found,
+                           int waitfor);
+int PMI2_Info_GetNodeAttrIntArray (const char *name,
+                                   int *array,
+                                   int arraylen,
+                                   int *outlen,
+                                   int *found);
+int PMI2_Info_PutNodeAttr (const char *name, const char *value);
+int PMI2_Info_GetJobAttr (const char *name,
+                          char *value,
+                          int valuelen,
+                          int *found);
+int PMI2_Info_GetJobAttrIntArray (const char *name,
+                                  int *array,
+                                  int arraylen,
+                                  int *outlen,
+                                  int *found);
+
+
+int PMI2_Nameserv_publish (const char *service_name,
+                           const struct MPID_Info *info_ptr,
+                           const char *port);
+int PMI2_Nameserv_lookup (const char *service_name,
                           const struct MPID_Info *info_ptr,
-                          char port[], int portLen);
-int PMI2_Nameserv_unpublish (const char service_name[],
+                          char *port,
+                          int portLen);
+int PMI2_Nameserv_unpublish (const char *service_name,
                              const struct MPID_Info *info_ptr);
 
-
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* !FLUX_PMI2_H_INCLUDED */
 
diff --git a/src/common/libpmi/pmi_strerror.c b/src/common/libpmi/pmi_strerror.c
index 38349dc6e822..1bd15c797ddc 100644
--- a/src/common/libpmi/pmi_strerror.c
+++ b/src/common/libpmi/pmi_strerror.c
@@ -15,7 +15,10 @@
 #include <errno.h>
 #include <string.h>
 
+#include "ccan/array_size/array_size.h"
+
 #include "pmi.h"
+#include "pmi2.h"
 #include "pmi_strerror.h"
 
 
@@ -40,15 +43,15 @@ static etab_t pmi_errors[] = {
     { PMI_ERR_INVALID_NUM_PARSED, "invalid num_parsed length argument" },
     { PMI_ERR_INVALID_KEYVALP,  "invalid keyvalp argument" },
     { PMI_ERR_INVALID_SIZE,     "invalid size argument" },
+    { PMI2_ERR_OTHER,           "other PMI2 error" },
 };
-static const int pmi_errors_len = sizeof (pmi_errors) / sizeof (pmi_errors[0]);
 
 const char *pmi_strerror (int rc)
 {
     static char unknown[] = "pmi error XXXXXXXXX";
     int i;
 
-    for (i = 0; i < pmi_errors_len; i++) {
+    for (i = 0; i < ARRAY_SIZE (pmi_errors); i++) {
         if (pmi_errors[i].errnum == rc)
             return pmi_errors[i].errstr;
     }
diff --git a/src/common/libpmi/sentinel.c b/src/common/libpmi/sentinel.c
new file mode 100644
index 000000000000..5633c102d575
--- /dev/null
+++ b/src/common/libpmi/sentinel.c
@@ -0,0 +1,20 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Define a symbol that can be used to tell the Flux pmi libs from others.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+void *flux_pmi_library;
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libpmi/simple_client.c b/src/common/libpmi/simple_client.c
index 7adeb687196e..95192d1527ba 100644
--- a/src/common/libpmi/simple_client.c
+++ b/src/common/libpmi/simple_client.c
@@ -19,13 +19,12 @@
 #include <string.h>
 #include <sys/param.h>
 #include <assert.h>
+#include <flux/taskmap.h>
 
 #include "src/common/libutil/aux.h"
 
 #include "simple_client.h"
 #include "simple_server.h"
-#include "clique.h"
-#include "dgetline.h"
 #include "keyval.h"
 #include "pmi.h"
 
@@ -38,9 +37,9 @@ int pmi_simple_client_init (struct pmi_simple_client *pmi)
 
     if (!pmi)
         return PMI_ERR_INIT;
-    if (dprintf (pmi->fd, "cmd=init pmi_version=1 pmi_subversion=1\n") < 0)
+    if (fprintf (pmi->f, "cmd=init pmi_version=1 pmi_subversion=1\n") < 0)
         goto done;
-    if (dgetline (pmi->fd, buf, sizeof (buf)) < 0)
+    if (!fgets (buf, sizeof (buf), pmi->f))
         goto done;
     if (keyval_parse_isword (buf, "cmd", "response_to_init") < 0)
         goto done;
@@ -49,14 +48,14 @@ int pmi_simple_client_init (struct pmi_simple_client *pmi)
         goto done;
     }
     if (keyval_parse_uint (buf, "pmi_version", &vers) < 0
-            || keyval_parse_uint (buf, "pmi_subversion", &subvers) < 0)
+        || keyval_parse_uint (buf, "pmi_subversion", &subvers) < 0)
         goto done;
     if (vers != 1 || subvers != 1)
         goto done;
 
-    if (dprintf (pmi->fd, "cmd=get_maxes\n") < 0)
+    if (fprintf (pmi->f, "cmd=get_maxes\n") < 0)
         goto done;
-    if (dgetline (pmi->fd, buf, sizeof (buf)) < 0)
+    if (!fgets (buf, sizeof (buf), pmi->f))
         goto done;
     if (keyval_parse_isword (buf, "cmd", "maxes") < 0)
         goto done;
@@ -65,8 +64,8 @@ int pmi_simple_client_init (struct pmi_simple_client *pmi)
         goto done;
     }
     if (keyval_parse_uint (buf, "kvsname_max", &pmi->kvsname_max) < 0
-            || keyval_parse_uint (buf, "keylen_max", &pmi->keylen_max) < 0
-            || keyval_parse_uint (buf, "vallen_max", &pmi->vallen_max) < 0)
+        || keyval_parse_uint (buf, "keylen_max", &pmi->keylen_max) < 0
+        || keyval_parse_uint (buf, "vallen_max", &pmi->vallen_max) < 0)
         goto done;
     pmi->buflen = pmi->keylen_max + pmi->vallen_max + pmi->kvsname_max
                                   + SIMPLE_MAX_PROTO_OVERHEAD;
@@ -87,9 +86,9 @@ int pmi_simple_client_finalize (struct pmi_simple_client *pmi)
 
     if (!pmi || !pmi->initialized)
         return PMI_ERR_INIT;
-    if (dprintf (pmi->fd, "cmd=finalize\n") < 0)
+    if (fprintf (pmi->f, "cmd=finalize\n") < 0)
         goto done;
-    if (dgetline (pmi->fd, pmi->buf, pmi->buflen) < 0)
+    if (!fgets (pmi->buf, pmi->buflen, pmi->f))
         goto done;
     if (keyval_parse_isword (pmi->buf, "cmd", "finalize_ack") < 0)
         goto done;
@@ -111,9 +110,9 @@ int pmi_simple_client_get_appnum (struct pmi_simple_client *pmi, int *appnum)
         return PMI_ERR_INIT;
     if (!appnum)
         return PMI_ERR_INVALID_ARG;
-    if (dprintf (pmi->fd, "cmd=get_appnum\n") < 0)
+    if (fprintf (pmi->f, "cmd=get_appnum\n") < 0)
         goto done;
-    if (dgetline (pmi->fd, pmi->buf, pmi->buflen) < 0)
+    if (!fgets (pmi->buf, pmi->buflen, pmi->f))
         goto done;
     if (keyval_parse_isword (pmi->buf, "cmd", "appnum") < 0)
         goto done;
@@ -138,9 +137,9 @@ int pmi_simple_client_get_universe_size (struct pmi_simple_client *pmi,
         return PMI_ERR_INIT;
     if (!universe_size)
         return PMI_ERR_INVALID_ARG;
-    if (dprintf (pmi->fd, "cmd=get_universe_size\n") < 0)
+    if (fprintf (pmi->f, "cmd=get_universe_size\n") < 0)
         goto done;
-    if (dgetline (pmi->fd, pmi->buf, pmi->buflen) < 0)
+    if (!fgets (pmi->buf, pmi->buflen, pmi->f))
         goto done;
     if (keyval_parse_isword (pmi->buf, "cmd", "universe_size") < 0)
         goto done;
@@ -162,9 +161,9 @@ int pmi_simple_client_barrier (struct pmi_simple_client *pmi)
 
     if (!pmi || !pmi->initialized)
         return PMI_ERR_INIT;
-    if (dprintf (pmi->fd, "cmd=barrier_in\n") < 0)
+    if (fprintf (pmi->f, "cmd=barrier_in\n") < 0)
         goto done;
-    if (dgetline (pmi->fd, pmi->buf, pmi->buflen) < 0)
+    if (!fgets (pmi->buf, pmi->buflen, pmi->f))
         goto done;
     if (keyval_parse_isword (pmi->buf, "cmd", "barrier_out") < 0)
         goto done;
@@ -188,9 +187,9 @@ int pmi_simple_client_kvs_get_my_name (struct pmi_simple_client *pmi,
         return PMI_ERR_INIT;
     if (!kvsname || length <= 0)
         return PMI_ERR_INVALID_ARG;
-    if (dprintf (pmi->fd, "cmd=get_my_kvsname\n") < 0)
+    if (fprintf (pmi->f, "cmd=get_my_kvsname\n") < 0)
         goto done;
-    if (dgetline (pmi->fd, pmi->buf, pmi->buflen) < 0)
+    if (!fgets (pmi->buf, pmi->buflen, pmi->f))
         goto done;
     if (keyval_parse_isword (pmi->buf, "cmd", "my_kvsname") < 0)
         goto done;
@@ -217,10 +216,10 @@ int pmi_simple_client_kvs_put (struct pmi_simple_client *pmi,
         return PMI_ERR_INIT;
     if (!kvsname || !key || !value)
         return PMI_ERR_INVALID_ARG;
-    if (dprintf (pmi->fd, "cmd=put kvsname=%s key=%s value=%s\n",
+    if (fprintf (pmi->f, "cmd=put kvsname=%s key=%s value=%s\n",
                  kvsname, key, value) < 0)
         goto done;
-    if (dgetline (pmi->fd, pmi->buf, pmi->buflen) < 0)
+    if (!fgets (pmi->buf, pmi->buflen, pmi->f))
         goto done;
     if (keyval_parse_isword (pmi->buf, "cmd", "put_result") < 0)
         goto done;
@@ -246,9 +245,9 @@ int pmi_simple_client_kvs_get (struct pmi_simple_client *pmi,
         return PMI_ERR_INIT;
     if (!kvsname || !key || !value || len <= 0)
         return PMI_ERR_INVALID_ARG;
-    if (dprintf (pmi->fd, "cmd=get kvsname=%s key=%s\n", kvsname, key) < 0)
+    if (fprintf (pmi->f, "cmd=get kvsname=%s key=%s\n", kvsname, key) < 0)
         goto done;
-    if (dgetline (pmi->fd, pmi->buf, pmi->buflen) < 0)
+    if (!fgets (pmi->buf, pmi->buflen, pmi->f))
         goto done;
     if (keyval_parse_isword (pmi->buf, "cmd", "get_result") < 0)
         goto done;
@@ -265,15 +264,12 @@ int pmi_simple_client_kvs_get (struct pmi_simple_client *pmi,
 
 /* Helper for get_clique_size(), get_clique_ranks().
  * Fetch 'PMI_process_mapping' from the KVS and parse.
- * On success, results are placed in blocks, nblocks, nodeid.
- * The caller must free 'blocks'.
+ * On success, a struct taskmap is stored in the pmi client aux hash
+ *  and returned to the caller.
  */
-static int fetch_process_mapping (struct pmi_simple_client *pmi,
-                                  struct pmi_map_block **blocks,
-                                  int *nblocks,
-                                  int *nodeid)
+static struct taskmap *fetch_taskmap (struct pmi_simple_client *pmi)
 {
-    const char *key = "PMI_process_mapping";
+    struct taskmap *map = NULL;
     int result;
     char *nom;
     char *val;
@@ -281,6 +277,9 @@ static int fetch_process_mapping (struct pmi_simple_client *pmi,
     assert (pmi != NULL);
     assert (pmi->initialized);
 
+    if ((map = pmi_simple_client_aux_get (pmi, "taskmap")))
+        return map;
+
     nom = calloc (1, pmi->kvsname_max);
     val = calloc (1, pmi->vallen_max);
     if (!nom || !val) {
@@ -290,70 +289,95 @@ static int fetch_process_mapping (struct pmi_simple_client *pmi,
     result = pmi_simple_client_kvs_get_my_name (pmi, nom, pmi->kvsname_max);
     if (result != PMI_SUCCESS)
         goto done;
-    result = pmi_simple_client_kvs_get (pmi, nom, key, val, pmi->vallen_max);
-    if (result != PMI_SUCCESS)
-        goto done;
-    result = pmi_process_mapping_parse (val, blocks, nblocks);
-    if (result != PMI_SUCCESS)
+    /*  First try flux.taskmap, falling back to PMI_process_mapping if it
+     *  does not exist (e.g. if process manager is not Flux).
+     */
+    result = pmi_simple_client_kvs_get (pmi,
+                                        nom,
+                                        "flux.taskmap",
+                                        val,
+                                        pmi->vallen_max);
+    if (result != PMI_SUCCESS) {
+        result = pmi_simple_client_kvs_get (pmi,
+                                            nom,
+                                            "PMI_process_mapping",
+                                            val,
+                                            pmi->vallen_max);
+        if (result != PMI_SUCCESS)
+            goto done;
+    }
+    if (!(map = taskmap_decode (val, NULL))) {
+        result = PMI_FAIL;
         goto done;
-    if (pmi_process_mapping_find_nodeid (*blocks, *nblocks,
-                                         pmi->rank, nodeid) != PMI_SUCCESS)
-        *nodeid = -1;
+    }
+    if (pmi_simple_client_aux_set (pmi,
+                                   "taskmap",
+                                   map,
+                                   (flux_free_f) taskmap_destroy) < 0) {
+        taskmap_destroy (map);
+        map = NULL;
+    }
 done:
     free (nom);
     free (val);
-    return result;
+    return map;
 }
 
 int pmi_simple_client_get_clique_size (struct pmi_simple_client *pmi,
                                        int *size)
 {
-    int result;
-    struct pmi_map_block *blocks = NULL;
-    int nblocks;
-    int nodeid;
+    int nodeid = -1;
+    struct taskmap *map;
 
     if (!pmi || !pmi->initialized)
         return PMI_ERR_INIT;
     if (!size)
         return PMI_ERR_INVALID_ARG;
-    result = fetch_process_mapping (pmi, &blocks, &nblocks, &nodeid);
-    if (result != PMI_SUCCESS || nodeid == -1) {
+    if (!(map = fetch_taskmap (pmi)) || taskmap_unknown (map)) {
         *size = 1;
-        result = PMI_SUCCESS;
+        return PMI_SUCCESS;
     }
-    else
-        result = pmi_process_mapping_find_nranks (blocks, nblocks, nodeid,
-                                                  pmi->size, size);
-    free (blocks);
-    return result;
+    if ((nodeid = taskmap_nodeid (map, pmi->rank)) < 0
+        || (*size = taskmap_ntasks (map, nodeid)) < 0)
+        return PMI_FAIL;
+    return PMI_SUCCESS;
 }
 
 int pmi_simple_client_get_clique_ranks (struct pmi_simple_client *pmi,
                                         int ranks[],
                                         int length)
 {
-    int result;
-    struct pmi_map_block *blocks = NULL;
-    int nblocks;
+    struct taskmap *map;
+    const struct idset *ids;
+    int ntasks;
     int nodeid;
+    unsigned int i;
+    int index = 0;
 
     if (!pmi || !pmi->initialized)
         return PMI_ERR_INIT;
     if (!ranks)
         return PMI_ERR_INVALID_ARG;
-    result = fetch_process_mapping (pmi, &blocks, &nblocks, &nodeid);
-    if (result != PMI_SUCCESS || nodeid == -1) {
+    map = fetch_taskmap (pmi);
+    if (!map || taskmap_unknown (map)) {
         if (length != 1)
             return PMI_ERR_INVALID_SIZE;
         *ranks = pmi->rank;
-        result = PMI_SUCCESS;
+        return PMI_SUCCESS;
     }
-    else
-        result = pmi_process_mapping_find_ranks (blocks, nblocks, nodeid,
-                                                 pmi->size, ranks, length);
-    free (blocks);
-    return result;
+    if ((nodeid = taskmap_nodeid (map, pmi->rank)) < 0
+        || (ntasks = taskmap_ntasks (map, nodeid)) < 0)
+        return PMI_FAIL;
+    if (ntasks > length)
+        return PMI_ERR_INVALID_SIZE;
+    if (!(ids = taskmap_taskids (map, nodeid)))
+        return PMI_FAIL;
+    i = idset_first (ids);
+    while (i != IDSET_INVALID_ID) {
+        ranks[index++] = i;
+        i = idset_next (ids, i);
+    }
+    return PMI_SUCCESS;
 }
 
 int pmi_simple_client_abort (struct pmi_simple_client *pmi,
@@ -361,13 +385,26 @@ int pmi_simple_client_abort (struct pmi_simple_client *pmi,
                              const char *msg)
 {
     int result = PMI_FAIL;
+    char *cpy = NULL;
 
     if (!pmi || !pmi->initialized)
         return PMI_ERR_INIT;
     if (exit_code < 0)
         return PMI_ERR_INVALID_ARG;
-    if (dprintf (pmi->fd,
-                 "cmd=abort exit_code=%d%s%s\n",
+    /* If message includes embedded \n's that would interfere with
+     * the wire protocol, replace them with spaces.
+     */
+    if (strchr (msg, '\n')) {
+        if (!(cpy = strdup (msg)))
+            return PMI_ERR_NOMEM;
+        for (char *cp = cpy; *cp != '\0'; cp++) {
+            if (*cp == '\n')
+                *cp = ' ';
+        }
+        msg = cpy;
+    }
+    if (fprintf (pmi->f,
+                 "cmd=abort exitcode=%d%s%s\n",
                  exit_code,
                  msg ? " error_msg=" : "",
                  msg ? msg : "") < 0)
@@ -375,6 +412,7 @@ int pmi_simple_client_abort (struct pmi_simple_client *pmi,
     exit (exit_code);
     /* NOTREACHED */
 done:
+    free (cpy);
     return result;
 }
 
@@ -405,8 +443,8 @@ void pmi_simple_client_destroy (struct pmi_simple_client *pmi)
     if (pmi) {
         int saved_errno = errno;
         aux_destroy (&pmi->aux);
-        if (pmi->fd != -1)
-            (void)close (pmi->fd);
+        if (pmi->f)
+            (void)fclose (pmi->f);
         free (pmi->buf);
         free (pmi);
         errno = saved_errno;
@@ -419,6 +457,7 @@ struct pmi_simple_client *pmi_simple_client_create_fd (const char *pmi_fd,
                                                        const char *pmi_spawned)
 {
     struct pmi_simple_client *pmi;
+    int fd = -1;
 
     if (!pmi_fd || !pmi_rank || !pmi_size) {
         errno = EINVAL;
@@ -427,10 +466,10 @@ struct pmi_simple_client *pmi_simple_client_create_fd (const char *pmi_fd,
     if (!(pmi = calloc (1, sizeof (*pmi))))
         return NULL;
     errno = 0;
-    pmi->fd = strtol (pmi_fd, NULL, 10);
+    fd = strtol (pmi_fd, NULL, 10);
     pmi->rank = strtol (pmi_rank, NULL, 10);
     pmi->size = strtol (pmi_size, NULL, 10);
-    if (errno != 0 || pmi->fd < 0 || pmi->rank < 0 || pmi->size < 1)
+    if (errno != 0 || fd < 0 || pmi->rank < 0 || pmi->size < 1)
         goto error;
     if (pmi_spawned) {
         errno = 0;
@@ -438,8 +477,12 @@ struct pmi_simple_client *pmi_simple_client_create_fd (const char *pmi_fd,
         if (errno != 0)
             goto error;
     }
+    if (!(pmi->f = fdopen (fd, "r+"))) // pmi->f takes ownership of fd
+        goto error;
     return pmi;
 error:
+    if (fd >= 0)
+        (void)close (fd);
     pmi_simple_client_destroy (pmi);
     return NULL;
 }
diff --git a/src/common/libpmi/simple_client.h b/src/common/libpmi/simple_client.h
index beb77c7b407f..86b2b57ef492 100644
--- a/src/common/libpmi/simple_client.h
+++ b/src/common/libpmi/simple_client.h
@@ -11,6 +11,7 @@
 #ifndef _FLUX_CORE_PMI_SIMPLE_CLIENT_H
 #define _FLUX_CORE_PMI_SIMPLE_CLIENT_H
 
+#include <stdio.h>
 #include "src/common/libutil/aux.h"
 #include "src/common/libflux/types.h"
 
@@ -29,7 +30,7 @@ struct pmi_simple_client {
     // for internal pmi_simple_client use only
     char *buf;
     int buflen;
-    int fd;
+    FILE *f;
     struct aux_item *aux;
 };
 
diff --git a/src/common/libpmi/simple_server.c b/src/common/libpmi/simple_server.c
index 2bac827728a1..d93ed22e400b 100644
--- a/src/common/libpmi/simple_server.c
+++ b/src/common/libpmi/simple_server.c
@@ -66,12 +66,15 @@
 #endif
 #include <stdio.h>
 #include <stdlib.h>
+#include <stdarg.h>
 #include <errno.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include <string.h>
 #include <sys/param.h>
-#include <czmq.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
 
 #include "simple_server.h"
 #include "keyval.h"
@@ -101,7 +104,8 @@ struct pmi_simple_server {
 
 
 static int pmi_simple_server_kvs_get_error (struct pmi_simple_server *pmi,
-                                            void *client, int result);
+                                            void *client,
+                                            int result);
 
 static struct client *client_create (int rank, void *arg);
 static void client_destroy (void **item);
@@ -112,7 +116,9 @@ static void client_hash_destroy (zhashx_t *zhx);
 
 
 static void trace (struct pmi_simple_server *pmi,
-                   void *client, const char *fmt, ...)
+                   void *client,
+                   const char *fmt,
+                   ...)
 {
     va_list ap;
 
@@ -126,6 +132,21 @@ static void trace (struct pmi_simple_server *pmi,
     }
 }
 
+static void warn (struct pmi_simple_server *pmi,
+                  void *client,
+                  const char *fmt,
+                  ...)
+{
+    char buf[1024];
+    va_list ap;
+
+    va_start (ap, fmt);
+    (void)vsnprintf (buf, sizeof (buf), fmt, ap);
+    va_end (ap);
+    if (pmi->ops.warn)
+        pmi->ops.warn (client, buf);
+}
+
 struct pmi_simple_server *pmi_simple_server_create (struct pmi_simple_ops ops,
                                                     int appnum,
                                                     int universe_size,
@@ -185,7 +206,8 @@ static int barrier_exit (struct pmi_simple_server *pmi, int rc)
     return ret;
 }
 
-static int client_respond (struct pmi_simple_server *pmi, void *client,
+static int client_respond (struct pmi_simple_server *pmi,
+                           void *client,
                            const char *resp)
 {
     if (resp[0] != '\0') {
@@ -197,7 +219,9 @@ static int client_respond (struct pmi_simple_server *pmi, void *client,
 }
 
 int pmi_simple_server_request (struct pmi_simple_server *pmi,
-                               const char *buf, void *client, int rank)
+                               const char *buf,
+                               void *client,
+                               int rank)
 {
     char resp[SIMPLE_MAX_PROTO_LINE+1];
     int rc = 0;
@@ -217,7 +241,7 @@ int pmi_simple_server_request (struct pmi_simple_server *pmi,
 
     /* spawn continuation (unimplemented) */
     if (cli->mcmd_started) {
-        if (strcmp (buf, "endcmd\n") != 0)
+        if (!streq (buf, "endcmd\n"))
             goto out_noresponse; // ignore protocol between mcmd and endcmd
         snprintf (resp, sizeof (resp), "cmd=spawn_result rc=-1\n");
         cli->mcmd_started = false;
@@ -231,34 +255,44 @@ int pmi_simple_server_request (struct pmi_simple_server *pmi,
             goto proto;
         if (pmi_version < 1 || (pmi_version == 1 && pmi_subversion < 1))
             snprintf (resp, sizeof (resp), "cmd=response_to_init rc=-1\n");
-        else
-            snprintf (resp, sizeof (resp), "cmd=response_to_init rc=0 "
+        else if (pmi_version == 2) {
+            if (cli->rank == 0) {
+                warn (pmi,
+                      cli,
+                      "is application unintentionally using slurm libpmi2.so?");
+            }
+            snprintf (resp, sizeof (resp), "cmd=response_to_init rc=-1\n");
+        }
+        else {
+            snprintf (resp,
+                      sizeof (resp),
+                      "cmd=response_to_init rc=0 "
                       "pmi_version=1 pmi_subversion=1\n");
+        }
     }
     /* maxes */
     else if (keyval_parse_isword (buf, "cmd", "get_maxes") == 0) {
-        snprintf (resp, sizeof (resp), "cmd=maxes rc=0 "
-                  "kvsname_max=%d keylen_max=%d vallen_max=%d\n",
-                  SIMPLE_KVS_NAME_MAX, SIMPLE_KVS_KEY_MAX, SIMPLE_KVS_VAL_MAX);
+        snprintf (resp,
+                  sizeof (resp),
+                  "cmd=maxes rc=0 kvsname_max=%d keylen_max=%d vallen_max=%d\n",
+                  SIMPLE_KVS_NAME_MAX,
+                  SIMPLE_KVS_KEY_MAX,
+                  SIMPLE_KVS_VAL_MAX);
     }
     /* abort */
     else if (keyval_parse_isword (buf, "cmd", "abort") == 0) {
         unsigned int code;
-        char *msg = "aborted";
-        char error_msg[SIMPLE_KVS_VAL_MAX];
+        char msg[SIMPLE_KVS_VAL_MAX];
 
-        /*  mpich circa 2014 sends `exit_code` but not `error_msg`.
-         *  Flux implementation sends both.
-         *  Older mpich and derivatives just exit from the task,
-         *   sending nothing.
+        /*  RFC 13 does not define any parameters to cmd=abort,
+         *  but exitcode=N is sent by mpich, and older flux
+         *  sent exit_code=N and error_msg=STR so accept those optionally.
          */
-        if (keyval_parse_uint (buf, "exit_code", &code) < 0)
-            goto proto;
-        if (keyval_parse_string (buf,
-                                 "error_msg",
-                                 error_msg,
-                                 sizeof (error_msg)) == 0)
-            msg = error_msg;
+        if (keyval_parse_uint (buf, "exitcode", &code) < 0
+            && keyval_parse_uint (buf, "exit_code", &code) < 0)
+            code = 1;
+        if (keyval_parse_string (buf, "error_msg", msg, sizeof (msg)) < 0)
+            snprintf (msg, sizeof (msg), "%s", "aborted");
         if (pmi->ops.abort)
             pmi->ops.abort (pmi->arg, cli, code, msg);
 
@@ -271,17 +305,22 @@ int pmi_simple_server_request (struct pmi_simple_server *pmi,
     }
     /* universe */
     else if (keyval_parse_isword (buf, "cmd", "get_universe_size") == 0) {
-        snprintf (resp, sizeof (resp), "cmd=universe_size rc=0 size=%d\n",
+        snprintf (resp,
+                  sizeof (resp),
+                  "cmd=universe_size rc=0 size=%d\n",
                   pmi->universe_size);
     }
     /* appnum */
     else if (keyval_parse_isword (buf, "cmd", "get_appnum") == 0) {
-        snprintf (resp, sizeof (resp), "cmd=appnum rc=0 appnum=%d\n",
+        snprintf (resp,
+                  sizeof (resp), "cmd=appnum rc=0 appnum=%d\n",
                   pmi->appnum);
     }
     /* kvsname */
     else if (keyval_parse_isword (buf, "cmd", "get_my_kvsname") == 0) {
-        snprintf (resp, sizeof (resp), "cmd=my_kvsname rc=0 kvsname=%s\n",
+        snprintf (resp,
+                  sizeof (resp),
+                  "cmd=my_kvsname rc=0 kvsname=%s\n",
                   pmi->kvsname);
     }
     /* put */
@@ -403,7 +442,8 @@ int pmi_simple_server_barrier_complete (struct pmi_simple_server *pmi, int rc)
 }
 
 static int pmi_simple_server_kvs_get_error (struct pmi_simple_server *pmi,
-                                            void *client, int result)
+                                            void *client,
+                                            int result)
 {
     char resp[SIMPLE_MAX_PROTO_LINE+1];
     snprintf (resp, sizeof (resp), "cmd=get_result rc=%d\n", result);
@@ -411,12 +451,15 @@ static int pmi_simple_server_kvs_get_error (struct pmi_simple_server *pmi,
 }
 
 int pmi_simple_server_kvs_get_complete (struct pmi_simple_server *pmi,
-                                        void *client, const char *val)
+                                        void *client,
+                                        const char *val)
 {
     char resp[SIMPLE_MAX_PROTO_LINE+1];
-    if (val == NULL)
-        return (pmi_simple_server_kvs_get_error (pmi, client,
+    if (val == NULL) {
+        return (pmi_simple_server_kvs_get_error (pmi,
+                                                 client,
                                                  PMI_ERR_INVALID_KEY));
+    }
     snprintf (resp, sizeof (resp), "cmd=get_result rc=0 value=%s\n", val);
     return (client_respond (pmi, client, resp));
 }
@@ -436,8 +479,10 @@ static struct client *client_create (int rank, void *arg)
 /* zhashx_destructor_fn footprint */
 static void client_destroy (void **item)
 {
-    free (*item);
-    *item = NULL;
+    if (item) {
+        free (*item);
+        *item = NULL;
+    }
 }
 
 /* zhashx_hash_fn footprint */
diff --git a/src/common/libpmi/simple_server.h b/src/common/libpmi/simple_server.h
index 26e383fa6ebf..d2fb5a3eba5a 100644
--- a/src/common/libpmi/simple_server.h
+++ b/src/common/libpmi/simple_server.h
@@ -14,6 +14,15 @@
 struct pmi_simple_server;
 
 #define SIMPLE_KVS_KEY_MAX         64
+
+/* Maximum size of a PMI KVS value. One might be tempted to increase
+ * this number to hold larger values, for example to hold an encoded
+ * PMI_process_mapping with a large count of tasks per node. However,
+ * experimentally, mpich and mvapich2 do not handle a larger max value
+ * correctly, and in many cases this causes a segfault in MPI. Therefore,
+ * it is suggested to leave SIMPLE_KVS_MAX at the de-facto standard of
+ * 1024 for now.
+ */
 #define SIMPLE_KVS_VAL_MAX         1024
 #define SIMPLE_KVS_NAME_MAX        64
 
@@ -29,14 +38,19 @@ struct pmi_simple_server;
  * Integer return: 0 on success, -1 on failure.
  */
 struct pmi_simple_ops {
-    int (*kvs_put)(void *arg, const char *kvsname,
-                   const char *key, const char *val);
+    int (*kvs_put)(void *arg,
+                   const char *kvsname,
+                   const char *key,
+                   const char *val);
     int (*kvs_get)(void *arg, void *cli, const char *kvsname, const char *key);
     int (*barrier_enter)(void *arg);
     int (*response_send)(void *client, const char *buf);
     void (*debug_trace)(void *client, const char *buf);
-    void (*abort) (void *arg, void *cli,
-                   int exit_code, const char error_message[]);
+    void (*abort) (void *arg,
+                   void *cli,
+                   int exit_code,
+                   const char error_message[]);
+    void (*warn)(void *client, const char *buf);
 };
 
 enum {
@@ -59,7 +73,9 @@ void pmi_simple_server_destroy (struct pmi_simple_server *pmi);
  * Returns 1 indicating finalized / close fd, 0 on success, -1 on failure.
  */
 int pmi_simple_server_request (struct pmi_simple_server *pmi,
-                               const char *buf, void *client, int rank);
+                               const char *buf,
+                               void *client,
+                               int rank);
 
 /* Finalize a barrier.  Set rc to 0 for success, -1 for failure.
  */
@@ -68,7 +84,8 @@ int pmi_simple_server_barrier_complete (struct pmi_simple_server *pmi, int rc);
 /* Finalize a kvs_get.
  */
 int pmi_simple_server_kvs_get_complete (struct pmi_simple_server *pmi,
-                                        void *client, const char *val);
+                                        void *client,
+                                        const char *val);
 
 #endif /* ! _FLUX_CORE_PMI_SIMPLE_SERVER_H */
 
diff --git a/src/common/libpmi/test/canonical.c b/src/common/libpmi/test/canonical.c
index 5275567f031e..fb35dff87b46 100644
--- a/src/common/libpmi/test/canonical.c
+++ b/src/common/libpmi/test/canonical.c
@@ -14,9 +14,9 @@
 #include "src/common/libutil/oom.h"
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libpmi/simple_client.h"
-#include "src/common/libpmi/dgetline.h"
 #include "src/common/libpmi/pmi.h"
 #include "src/common/libflux/reactor.h"
+#include "ccan/str/str.h"
 
 #include "src/common/libtap/tap.h"
 
@@ -274,7 +274,7 @@ int main (int argc, char *argv[])
         "PMI_KVS_Get length=-1 fails with PMI_ERR_INVALID_ARG");
 
     result = PMI_KVS_Get (kvsname, "foo", val, vallen_max);
-    ok (result == PMI_SUCCESS && !strcmp (val, "bar"),
+    ok (result == PMI_SUCCESS && streq (val, "bar"),
         "PMI_KVS_Get works and got expected value");
 
     /* clique
@@ -339,49 +339,49 @@ int main (int argc, char *argv[])
 
     result = PMI_KVS_Create (buf, sizeof (buf));
     ok (result == PMI_FAIL,
-        "PMI_KVS_Create (unimplemented) resturns PMI_FAIL");
+        "PMI_KVS_Create (unimplemented) returns PMI_FAIL");
 
     result = PMI_KVS_Destroy ("foo");
     ok (result == PMI_FAIL,
-        "PMI_KVS_Destroy (unimplemented) resturns PMI_FAIL");
+        "PMI_KVS_Destroy (unimplemented) returns PMI_FAIL");
 
     result = PMI_KVS_Iter_first ("foo", buf, sizeof (buf), buf, sizeof (buf));
     ok (result == PMI_FAIL,
-        "PMI_KVS_Iter_first (unimplemented) resturns PMI_FAIL");
+        "PMI_KVS_Iter_first (unimplemented) returns PMI_FAIL");
 
     result = PMI_KVS_Iter_next ("foo", buf, sizeof (buf), buf, sizeof (buf));
     ok (result == PMI_FAIL,
-        "PMI_KVS_Iter_next (unimplemented) resturns PMI_FAIL");
+        "PMI_KVS_Iter_next (unimplemented) returns PMI_FAIL");
 
     result = PMI_Parse_option (0, NULL, NULL, NULL, NULL);
     ok (result == PMI_FAIL,
-        "PMI_Parse_option (unimplemented) resturns PMI_FAIL");
+        "PMI_Parse_option (unimplemented) returns PMI_FAIL");
 
     result = PMI_Args_to_keyval (NULL, NULL, NULL, NULL);
     ok (result == PMI_FAIL,
-        "PMI_Args_to_keyval (unimplemented) resturns PMI_FAIL");
+        "PMI_Args_to_keyval (unimplemented) returns PMI_FAIL");
 
     result = PMI_Free_keyvals (NULL, 0);
     ok (result == PMI_FAIL,
-        "PMI_Free_keyvals (unimplemented) resturns PMI_FAIL");
+        "PMI_Free_keyvals (unimplemented) returns PMI_FAIL");
 
     result = PMI_Get_options (NULL, NULL);
     ok (result == PMI_FAIL,
-        "PMI_Get_options (unimplemented) resturns PMI_FAIL");
+        "PMI_Get_options (unimplemented) returns PMI_FAIL");
 
 
     /* aliases
      */
     result = PMI_Get_id_length_max (&n);
     ok (result == PMI_SUCCESS && n == kvsname_max,
-        "PMI_Get_id_lenght_max works and set idlen to kvsname_max");
+        "PMI_Get_id_length_max works and set idlen to kvsname_max");
 
     result = PMI_Get_id (buf, sizeof (buf));
-    ok (result == PMI_SUCCESS && !strcmp (buf, kvsname),
+    ok (result == PMI_SUCCESS && streq (buf, kvsname),
         "PMI_Get_id works and set buf to kvsname");
 
     result = PMI_Get_kvs_domain_id (buf, sizeof (buf));
-    ok (result == PMI_SUCCESS && !strcmp (buf, kvsname),
+    ok (result == PMI_SUCCESS && streq (buf, kvsname),
         "PMI_Get_kvs_domain_id works and set buf to kvsname");
 
     /* finalize
diff --git a/src/common/libpmi/test/canonical2.c b/src/common/libpmi/test/canonical2.c
index e9ed80dda8eb..04d7523aad4e 100644
--- a/src/common/libpmi/test/canonical2.c
+++ b/src/common/libpmi/test/canonical2.c
@@ -14,9 +14,9 @@
 #include "src/common/libutil/oom.h"
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libpmi/simple_client.h"
-#include "src/common/libpmi/dgetline.h"
 #include "src/common/libpmi/pmi2.h"
 #include "src/common/libflux/reactor.h"
+#include "ccan/str/str.h"
 
 #include "src/common/libtap/tap.h"
 
@@ -29,6 +29,7 @@ int main (int argc, char *argv[])
     int cfd[1];
     char jobid[PMI2_MAX_ATTRVALUE + 1];
     char val[PMI2_MAX_VALLEN + 1];
+    char longkey[2*PMI2_MAX_KEYLEN];
     char pmi_rank[16];
     char pmi_size[16];
     int result;
@@ -123,15 +124,69 @@ int main (int argc, char *argv[])
                                    val,
                                    sizeof (val),
                                    &found);
-    ok (result == PMI2_SUCCESS && found != 0 && !strcmp (val, "1"),
+    ok (result == PMI2_SUCCESS && found != 0 && streq (val, "1"),
        "PMI2_Info_GetJobAttr PMI_process_mapping works and found != 0");
 
     jobid[0] = '\0';
     result = PMI2_Job_GetId (jobid, sizeof (jobid));
     ok (result == PMI2_SUCCESS
-        && !strcmp (jobid, "bleepgorp"),
+        && streq (jobid, "bleepgorp"),
         "PMI2_Job_GetId works");
 
+    /* Exchange node scope data
+     */
+    result = PMI2_Info_PutNodeAttr ("attr1", "xyz");
+    ok (result == PMI2_SUCCESS,
+        "PMI2_Info_PutNodeAttr name=attr1 works");
+
+    found = 42;
+    result = PMI2_Info_GetNodeAttr ("attr1", val, sizeof (val), &found, 0);
+    ok (result == PMI2_SUCCESS
+        && found == 1
+        && streq (val, "xyz"),
+        "PMI2_Info_GetNodeAttr name=attr1 works");
+
+    found = 42;
+    result = PMI2_Info_GetNodeAttr ("attr1", val, sizeof (val), &found, 1);
+    ok (result == PMI2_SUCCESS
+        && found == 1
+        && streq (val, "xyz"),
+        "PMI2_Info_GetNodeAttr name=attr1 waitfor=1 works");
+
+    found = 42;
+    result = PMI2_Info_GetNodeAttr ("noexist", val, sizeof (val), &found, 0);
+    ok (result == PMI2_SUCCESS
+        && found == 0,
+        "PMI2_Info_GetNodeAttr name=noexist returns PMI2_SUCCESS with found=0");
+
+    result = PMI2_Info_GetNodeAttr ("noexist", val, sizeof (val), NULL, 0);
+    ok (result == PMI2_ERR_INVALID_KEY,
+        "PMI2_Info_GetNodeAttr name=noexist found=NULL returns"
+        " PMI2_ERR_INVALID_KEY");
+
+    memset (longkey, 'a', sizeof (longkey));
+    longkey[sizeof (longkey) - 1] = '\0';
+
+    result = PMI2_Info_GetNodeAttr (NULL, val, sizeof (val), NULL, 0);
+    ok (result == PMI2_ERR_INVALID_ARG,
+        "PMI2_Info_GetNodeAttr name=NULL returns PMI2_ERR_INVALID_ARG");
+    result = PMI2_Info_GetNodeAttr ("attr1", NULL, 0, NULL, 0);
+    ok (result == PMI2_ERR_INVALID_ARG,
+        "PMI2_Info_GetNodeAttr value=NULL returns PMI2_ERR_INVALID_ARG");
+    result = PMI2_Info_GetNodeAttr (longkey, val, sizeof (val), NULL, 0);
+    ok (result == PMI2_ERR_INVALID_KEY_LENGTH,
+        "PMI2_Info_GetNodeAttr name=longkey returns PMI2_ERR_INVALID_KEY_LENGTH");
+
+    result = PMI2_Info_PutNodeAttr (NULL, "xyz");
+    ok (result == PMI2_ERR_INVALID_ARG,
+        "PMI2_Info_PutNodeAttr name=NULL returns PMI2_ERR_INVALID_ARG");
+    result = PMI2_Info_PutNodeAttr ("attr2", NULL);
+    ok (result == PMI2_ERR_INVALID_ARG,
+        "PMI2_Info_PutNodeAttr value=NULL returns PMI2_ERR_INVALID_ARG");
+    result = PMI2_Info_PutNodeAttr (longkey, "xyz");
+    ok (result == PMI2_ERR_INVALID_KEY_LENGTH,
+        "PMI2_Info_PutNodeAttr name=longkey returns PMI2_ERR_INVALID_KEY_LENGTH");
+
     /* put foo=bar / fence / get foo
      */
     result = PMI2_KVS_Put (NULL, "bar");
@@ -151,12 +206,12 @@ int main (int argc, char *argv[])
         "PMI2_KVS_Fence works");
 
     result = PMI2_KVS_Get (NULL, 0, "foo", val, sizeof (val), &vallen);
-    ok (result == PMI2_ERR_INVALID_ARG,
-        "PMI2_KVS_Get jobid=NULL fails with PMI2_ERR_INVALID_ARG");
+    ok (result == PMI2_SUCCESS,
+        "PMI2_KVS_Get jobid=NULL works");
 
     result = PMI2_KVS_Get (jobid, 0, "foo", val, sizeof (val), &vallen);
     ok (result == PMI2_SUCCESS
-        && !strcmp (val, "bar")
+        && streq (val, "bar")
         && vallen == strlen (val),
         "PMI2_KVS_Get works and got expected value");
 
@@ -178,18 +233,10 @@ int main (int argc, char *argv[])
     ok (result == PMI2_FAIL,
         "PMI2_Info_GetSize (unimplemented) returns PMI2_FAIL");
 
-    result = PMI2_Info_GetNodeAttr (NULL, NULL, 0, NULL, 0);
-    ok (result == PMI2_FAIL,
-        "PMI2_Info_GetNodeAttr (unimplemented) returns PMI2_FAIL");
-
     result = PMI2_Info_GetNodeAttrIntArray (NULL, NULL, 0, NULL, NULL);
     ok (result == PMI2_FAIL,
         "PMI2_Info_GetNodeAttrIntArray (unimplemented) returns PMI2_FAIL");
 
-    result = PMI2_Info_PutNodeAttr (NULL, NULL);
-    ok (result == PMI2_FAIL,
-        "PMI2_Info_PutNodeAttr (unimplemented) returns PMI2_FAIL");
-
     result = PMI2_Info_GetJobAttrIntArray (NULL, NULL, 0, NULL, NULL);
     ok (result == PMI2_FAIL,
         "PMI2_Info_GetJobAttrIntArray (unimplemented) returns PMI2_FAIL");
diff --git a/src/common/libpmi/test/clique.c b/src/common/libpmi/test/clique.c
index a6b80235df3c..d578ea56993f 100644
--- a/src/common/libpmi/test/clique.c
+++ b/src/common/libpmi/test/clique.c
@@ -8,7 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include <czmq.h>
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 
 #include "src/common/libpmi/clique.h"
diff --git a/src/common/libpmi/test/keyval.c b/src/common/libpmi/test/keyval.c
index 5ac3759fede0..b241460a1185 100644
--- a/src/common/libpmi/test/keyval.c
+++ b/src/common/libpmi/test/keyval.c
@@ -8,12 +8,16 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include "src/common/libtap/tap.h"
-#include "src/common/libpmi/keyval.h"
-
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 #include <ctype.h>
 
+#include "src/common/libtap/tap.h"
+#include "src/common/libpmi/keyval.h"
+#include "ccan/str/str.h"
+
 static char *valid[] = {
     "key1=val1",
     "key1=val1 ",
@@ -54,6 +58,7 @@ static char *pmi[] = {
     "cmd=lookup_result rc=0 info=ok port=merp42\n",
     "cmd=unpublish_name service=zz\n",
     "cmd=unpublish_result rc=0 info=ok\n",
+    "cmd=get_result rc=0 msg=success value=a = b found=TRUE\n",
     NULL,
 };
 
@@ -79,6 +84,25 @@ static char *spawn[] = {
     NULL,
 };
 
+void check_overflow (void)
+{
+    int rc;
+    int i;
+    unsigned int u;
+
+    rc = keyval_parse_int ("x=4294967296", "x", &i);
+    ok (rc == EKV_VAL_PARSE,
+        "keyval_parse_int x=2^32 (overflow) fails with EKV_VAL_PARSE");
+
+    rc = keyval_parse_int ("x=-4294967296", "x", &i);
+    ok (rc == EKV_VAL_PARSE,
+        "keyval_parse_int x=-2^32 (overflow) fails with EKV_VAL_PARSE");
+
+    rc = keyval_parse_uint ("x=8589934592", "x", &u);
+    ok (rc == EKV_VAL_PARSE,
+        "keyval_parse_uint x=2^33 (overflow) fails with EKV_VAL_PARSE");
+}
+
 int main(int argc, char** argv)
 {
     char val[42];
@@ -88,18 +112,18 @@ int main(int argc, char** argv)
     plan (NO_PLAN);
 
     ok (keyval_parse_word (valid[0], "key1", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "val1"),
+        && streq (val, "val1"),
         "keyval_parse_word parsed the first key");
     ok (keyval_parse_word (valid[1], "key1", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "val1"),
+        && streq (val, "val1"),
         "keyval_parse_word parsed the first word, ignoring trailing space");
     ok (keyval_parse_word (valid[2], "key1", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "val1"),
+        && streq (val, "val1"),
         "keyval_parse_word parsed the first word, ignoring trailing newline");
     ok (keyval_parse_word (valid[2], "noexist", val, sizeof (val)) == EKV_NOKEY,
         "keyval_parse_word failed on nonexistent key");
     ok (keyval_parse_word (valid[3], "key2", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "val2"),
+        && streq (val, "val2"),
         "keyval_parse_word parsed the second key");
     ok (keyval_parse_uint (valid[4], "key3", &ui) == EKV_SUCCESS && ui == 42,
         "keyval_parse_uint worked");
@@ -108,13 +132,13 @@ int main(int argc, char** argv)
     ok (keyval_parse_int (valid[5], "key4", &i) == EKV_SUCCESS && i == -42,
         "keyval_parse_int worked on negative integer");
     ok (keyval_parse_word (valid[6], "key5", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "foo=bar"),
+        && streq (val, "foo=bar"),
         "keyval_parse_word handled value containing an equals");
     ok (keyval_parse_word (valid[6], "key6", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "baz"),
+        && streq (val, "baz"),
         "keyval_parse_word parsed word following value containing an equals");
     ok (keyval_parse_string (valid[7], "key7", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "x y z="),
+        && streq (val, "x y z="),
         "keyval_parse_string parsed string containing space and equals");
     ok (keyval_parse_int (valid[8], "key1", &i) == EKV_SUCCESS && i == 42,
         "keyval_parse_int parsed int not followed by white space");
@@ -125,183 +149,193 @@ int main(int argc, char** argv)
      */
 
     ok (keyval_parse_word (pmi[0], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "init")
+        && streq (val, "init")
         && keyval_parse_uint (pmi[0], "pmi_version", &ui) == EKV_SUCCESS && ui == 1
         && keyval_parse_uint (pmi[0], "pmi_subversion", &ui) == EKV_SUCCESS && ui == 1,
         "parsed pmi-1 init request");
     ok (keyval_parse_word (pmi[1], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "response_to_init")
+        && streq (val, "response_to_init")
         && keyval_parse_int (pmi[1], "rc", &i) == EKV_SUCCESS && i == 0
         && keyval_parse_uint (pmi[1], "pmi_version", &ui) == EKV_SUCCESS && ui == 1
         && keyval_parse_uint (pmi[1], "pmi_subversion", &ui) == EKV_SUCCESS && ui == 1,
         "parsed pmi-1 init response");
     ok (keyval_parse_word (pmi[2], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "get_maxes"),
+        && streq (val, "get_maxes"),
         "parsed pmi-1 maxes request");
     ok (keyval_parse_word (pmi[3], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "maxes")
+        && streq (val, "maxes")
         && keyval_parse_int (pmi[3], "rc", &i) == EKV_SUCCESS && i == 0
         && keyval_parse_uint (pmi[3], "kvsname_max", &ui) == EKV_SUCCESS && ui == 256
         && keyval_parse_uint (pmi[3], "keylen_max", &ui) == EKV_SUCCESS && ui == 256
         && keyval_parse_uint (pmi[3], "vallen_max", &ui) == EKV_SUCCESS && ui == 256,
         "parsed pmi-1 maxes response");
     ok (keyval_parse_word (pmi[4], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "get_universe_size"),
+        && streq (val, "get_universe_size"),
         "parsed pmi-1 universe_size request");
     ok (keyval_parse_word (pmi[5], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "universe_size")
+        && streq (val, "universe_size")
         && keyval_parse_int (pmi[5], "rc", &i) == EKV_SUCCESS && i == 0
         && keyval_parse_uint (pmi[5], "size", &ui) == EKV_SUCCESS && ui == 2,
         "parsed pmi-1 universe_size response");
     ok (keyval_parse_word (pmi[6], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "get_appnum"),
+        && streq (val, "get_appnum"),
         "parsed pmi-1 appnum request");
     ok (keyval_parse_word (pmi[7], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "appnum")
+        && streq (val, "appnum")
         && keyval_parse_int (pmi[7], "rc", &i) == EKV_SUCCESS && i == 0
         && keyval_parse_int (pmi[7], "appnum", &i) == EKV_SUCCESS && i == 0,
         "parsed pmi-1 appnum response");
     ok (keyval_parse_word (pmi[8], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "barrier_in"),
+        && streq (val, "barrier_in"),
         "parsed pmi-1 barrier request");
     ok (keyval_parse_word (pmi[9], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "barrier_out")
+        && streq (val, "barrier_out")
         && keyval_parse_int (pmi[9], "rc", &i) == EKV_SUCCESS && i == 0,
         "parsed pmi-1 barrier response");
     ok (keyval_parse_word (pmi[10], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "finalize"),
+        && streq (val, "finalize"),
         "parsed pmi-1 finalize request");
     ok (keyval_parse_word (pmi[11], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "finalize_ack")
+        && streq (val, "finalize_ack")
         && keyval_parse_int (pmi[11], "rc", &i) == EKV_SUCCESS && i == 0,
         "parsed pmi-1 finalize response");
     ok (keyval_parse_word (pmi[12], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "get_my_kvsname"),
+        && streq (val, "get_my_kvsname"),
         "parsed pmi-1 kvsname request");
     ok (keyval_parse_word (pmi[13], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "my_kvsname")
+        && streq (val, "my_kvsname")
         && keyval_parse_int (pmi[13], "rc", &i) == EKV_SUCCESS && i == 0
         && keyval_parse_word (pmi[13], "kvsname", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "lwj.1.pmi"),
+        && streq (val, "lwj.1.pmi"),
         "parsed pmi-1 kvsname response");
     ok (keyval_parse_word (pmi[14], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "put")
+        && streq (val, "put")
         && keyval_parse_word (pmi[14], "kvsname", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "lwj.1.pmi")
+        && streq (val, "lwj.1.pmi")
         && keyval_parse_word (pmi[14], "key", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "PM")
+        && streq (val, "PM")
         && keyval_parse_string (pmi[14], "value", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "/dev/shm/mpich_shar_tmpYbGKbb"),
+        && streq (val, "/dev/shm/mpich_shar_tmpYbGKbb"),
         "parsed pmi-1 put request");
     ok (keyval_parse_word (pmi[15], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "put_result")
+        && streq (val, "put_result")
         && keyval_parse_int (pmi[15], "rc", &i) == 0 && i == EKV_SUCCESS
         && keyval_parse_string (pmi[15], "msg", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "success"),
+        && streq (val, "success"),
         "parsed pmi-1 put response");
     ok (keyval_parse_word (pmi[16], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "get")
+        && streq (val, "get")
         && keyval_parse_word (pmi[16], "kvsname", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "lwj.1.pmi")
+        && streq (val, "lwj.1.pmi")
         && keyval_parse_word (pmi[16], "key", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "sh"),
+        && streq (val, "sh"),
         "parsed pmi-1 get request");
     ok (keyval_parse_word (pmi[17], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "get_result")
+        && streq (val, "get_result")
         && keyval_parse_int (pmi[17], "rc", &i) == EKV_SUCCESS && i == 0
         && keyval_parse_word (pmi[17], "msg", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "success")
+        && streq (val, "success")
         && keyval_parse_string (pmi[17], "value", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "/dev/shm/mpich_shar_tmpYbGKbb"),
+        && streq (val, "/dev/shm/mpich_shar_tmpYbGKbb"),
         "parsed pmi-1 get response");
     ok (keyval_parse_word (pmi[18], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "publish_name")
+        && streq (val, "publish_name")
         && keyval_parse_word (pmi[18], "service", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "zz")
+        && streq (val, "zz")
         && keyval_parse_word (pmi[18], "port", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "merp42"),
+        && streq (val, "merp42"),
         "parsed pmi-1 publish request");
     ok (keyval_parse_word (pmi[19], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "publish_result")
+        && streq (val, "publish_result")
         && keyval_parse_int (pmi[19], "rc", &i) == EKV_SUCCESS && i == 0
         && keyval_parse_word (pmi[19], "info", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "ok"),
+        && streq (val, "ok"),
         "parsed pmi-1 publish response");
     ok (keyval_parse_word (pmi[20], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "lookup_name")
+        && streq (val, "lookup_name")
         && keyval_parse_word (pmi[20], "service", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "zz"),
+        && streq (val, "zz"),
         "parsed pmi-1 lookup request");
     ok (keyval_parse_word (pmi[21], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "lookup_result")
+        && streq (val, "lookup_result")
         && keyval_parse_int (pmi[21], "rc", &i) == EKV_SUCCESS && i == 0
         && keyval_parse_word (pmi[21], "info", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "ok")
+        && streq (val, "ok")
         && keyval_parse_word (pmi[21], "port", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "merp42"),
+        && streq (val, "merp42"),
         "parsed pmi-1 lookup response");
     ok (keyval_parse_word (pmi[22], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "unpublish_name")
+        && streq (val, "unpublish_name")
         && keyval_parse_word (pmi[22], "service", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "zz"),
+        && streq (val, "zz"),
         "parsed pmi-1 unpublish request");
     ok (keyval_parse_word (pmi[23], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "unpublish_result")
+        && streq (val, "unpublish_result")
         && keyval_parse_int (pmi[23], "rc", &i) == EKV_SUCCESS && i == 0
         && keyval_parse_word (pmi[23], "info", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "ok"),
+        && streq (val, "ok"),
         "parsed pmi-1 unpublish response");
+    ok (keyval_parse_word (pmi[24], "cmd", val, sizeof (val)) == EKV_SUCCESS
+        && streq (val, "get_result")
+        && keyval_parse_int (pmi[24], "rc", &i) == EKV_SUCCESS && i == 0
+        && keyval_parse_word (pmi[24], "msg", val, sizeof (val)) == EKV_SUCCESS
+        && streq (val, "success")
+        && keyval_parse_string (pmi[24], "value", val, sizeof (val)) == EKV_SUCCESS
+        && streq (val, "a = b"),
+        "parsed pmi-1 lookup response with mpich v4.2.0 quirk");
 
     ok (keyval_parse_word (spawn[0], "mcmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "spawn"),
+        && streq (val, "spawn"),
         "parsed pmi-1 spawn mcmd request");
     ok (keyval_parse_uint (spawn[1], "nprocs", &ui) == EKV_SUCCESS && ui == 2,
         "parsed pmi-1 spawn nprocs request");
     ok (keyval_parse_word (spawn[2], "execname", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "workprog"),
+        && streq (val, "workprog"),
         "parsed pmi-1 spawn execname request");
     ok (keyval_parse_uint (spawn[3], "totspawns", &ui) == EKV_SUCCESS && ui == 2,
         "parsed pmi-1 spawn totspawns request");
     ok (keyval_parse_uint (spawn[4], "spawnssofar", &ui) == EKV_SUCCESS && ui == 0,
         "parsed pmi-1 spawn spawnssofar request");
     ok (keyval_parse_word (spawn[5], "arg0", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "workprog"),
+        && streq (val, "workprog"),
         "parsed pmi-1 spawn arg0 request");
     ok (keyval_parse_word (spawn[6], "arg1", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "--do-something=yes"),
+        && streq (val, "--do-something=yes"),
         "parsed pmi-1 spawn arg1 request");
     ok (keyval_parse_word (spawn[7], "arg2", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "-X"),
+        && streq (val, "-X"),
         "parsed pmi-1 spawn arg2 request");
     ok (keyval_parse_word (spawn[8], "arg3", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "inputdeck"),
+        && streq (val, "inputdeck"),
         "parsed pmi-1 spawn arg3 request");
     ok (keyval_parse_uint (spawn[9], "argcnt", &ui) == EKV_SUCCESS && ui == 4,
         "parsed pmi-1 spawn argcnt request");
     ok (keyval_parse_uint (spawn[10], "preput_num", &ui) == EKV_SUCCESS && ui == 1,
         "parsed pmi-1 spawn preput_num request");
     ok (keyval_parse_word (spawn[11], "preput_key_0", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "foo"),
+        && streq (val, "foo"),
         "parsed pmi-1 spawn preput_key_0 request");
     ok (keyval_parse_word (spawn[12], "preput_val_0", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "bar"),
+        && streq (val, "bar"),
         "parsed pmi-1 spawn preput_val_0 request");
     ok (keyval_parse_uint (spawn[13], "info_num", &ui) == EKV_SUCCESS && ui == 1,
         "parsed pmi-1 spawn info_num request");
     ok (keyval_parse_word (spawn[14], "info_key_0", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "baz"),
+        && streq (val, "baz"),
         "parsed pmi-1 spawn info_key_0 request");
     ok (keyval_parse_word (spawn[15], "info_val_0", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "zurn"),
+        && streq (val, "zurn"),
         "parsed pmi-1 spawn info_val_0 request");
-    /* skip endcmd - we'll just strcmp that one */
+    /* skip endcmd - we'll just streq that one */
     ok (keyval_parse_word (spawn[17], "cmd", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "spawn_result")
+        && streq (val, "spawn_result")
         && keyval_parse_int (spawn[17], "rc", &i) == EKV_SUCCESS && i == 0
         && keyval_parse_word (spawn[17], "errcodes", val, sizeof (val)) == EKV_SUCCESS
-        && !strcmp (val, "0,0"),
+        && streq (val, "0,0"),
         "parsed pmi-1 spawn response");
 
+    check_overflow();
+
     done_testing();
 }
 
diff --git a/src/common/libpmi/test/kvstest.c b/src/common/libpmi/test/kvstest.c
index 0b5beca3c2af..6f373fa958f3 100644
--- a/src/common/libpmi/test/kvstest.c
+++ b/src/common/libpmi/test/kvstest.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
@@ -22,12 +25,12 @@
 #include "src/common/libutil/monotime.h"
 #include "src/common/libpmi/pmi.h"
 #include "src/common/libpmi/pmi_strerror.h"
+#include "ccan/str/str.h"
 
-#define OPTIONS "nN:l:"
+#define OPTIONS "nN:"
 static const struct option longopts[] = {
     {"n-squared",    no_argument,        0, 'n'},
     {"key-count",    required_argument,  0, 'N'},
-    {"library",      required_argument,  0, 'l'},
     {0, 0, 0, 0},
 };
 
@@ -40,7 +43,6 @@ int main(int argc, char *argv[])
     bool nsquared = false;
     int ch;
     int i, j, keycount = 1;
-    char *library = NULL;
 
     while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
         switch (ch) {
@@ -50,19 +52,12 @@ int main(int argc, char *argv[])
             case 'N':   /* --key-count N */
                 keycount = strtoul (optarg, NULL, 10);
                 break;
-            case 'l':   /* --library */
-                library = optarg;
-                break;
         }
     }
 
     /* Initial handshake with PMI obtains
      *    rank, size, and some string max lengths
      */
-    if (library) {
-        unsetenv ("PMI_FD");
-        setenv ("PMI_LIBRARY", library, 1);
-    }
     e = PMI_Init (&spawned);
     if (e != PMI_SUCCESS)
         log_msg_exit ("PMI_Init: %s", pmi_strerror (e));
@@ -132,7 +127,7 @@ int main(int argc, char *argv[])
                 if (e != PMI_SUCCESS)
                     log_msg_exit ("%d: PMI_KVS_Get: %s", rank, pmi_strerror (e));
                 snprintf (val2, val_len, "sandwich.%d.%d", j, i);
-                if (strcmp (val, val2) != 0)
+                if (!streq (val, val2))
                     log_msg_exit ("%d: PMI_KVS_Get: exp %s got %s\n",
                              rank, val2, val);
             }
@@ -141,10 +136,10 @@ int main(int argc, char *argv[])
                       rank > 0 ? rank - 1 : size - 1, i);
             e = PMI_KVS_Get (kvsname, key, val, val_len);
             if (e != PMI_SUCCESS)
-                log_msg_exit ("%d: PMI_IVS_Get: %s", rank, pmi_strerror (e));
+                log_msg_exit ("%d: PMI_KVS_Get: %s", rank, pmi_strerror (e));
             snprintf (val2, val_len, "sandwich.%d.%d",
                       rank > 0 ? rank - 1 : size - 1, i);
-            if (strcmp (val, val2) != 0)
+            if (!streq (val, val2))
                 log_msg_exit ("%d: PMI_KVS_Get: exp %s got %s\n", rank, val2, val);
         }
     }
diff --git a/src/common/libpmi/test/kvstest2.c b/src/common/libpmi/test/kvstest2.c
new file mode 100644
index 000000000000..d66b4b06a886
--- /dev/null
+++ b/src/common/libpmi/test/kvstest2.c
@@ -0,0 +1,169 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <stdbool.h>
+#include <flux/taskmap.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libpmi/pmi2.h"
+#include "src/common/libpmi/pmi_strerror.h"
+#include "ccan/str/str.h"
+
+/* We don't have a pmi2_strerror() but the codes are mostly the same as PMI-1
+ */
+#define pmi2_strerror pmi_strerror
+
+static int find_id (const struct idset *ids, unsigned int id)
+{
+    unsigned int i;
+    int index = 0;
+
+    i = idset_first (ids);
+    while (i != IDSET_INVALID_ID) {
+        if (i == id)
+            return index;
+        i = idset_next (ids, i);
+        index++;
+    }
+    return -1;
+}
+
+static int get_neighbor (const struct idset *ids, unsigned int id)
+{
+    int i = idset_next (ids, id);
+    if (i == IDSET_INVALID_ID)
+        return idset_first (ids);
+    return i;
+}
+
+int main(int argc, char *argv[])
+{
+    int size, rank;
+    char jobid[PMI2_MAX_VALLEN];
+    char key[PMI2_MAX_KEYLEN];
+    char val[PMI2_MAX_VALLEN];
+    char attr[PMI2_MAX_ATTRVALUE];
+    char expected_attr[PMI2_MAX_ATTRVALUE];
+    char expected_val[PMI2_MAX_VALLEN];
+    const int keycount = 10;
+    struct taskmap *map;
+    const struct idset *taskids;
+    flux_error_t error;
+    int e;
+    int length;
+    int nodeid;
+    int clique_rank;
+    int clique_neighbor;
+
+    /* Initialize
+     */
+    e = PMI2_Init (NULL, &size, &rank, NULL);
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("PMI2_Init: %s", pmi2_strerror (e));
+    e = PMI2_Job_GetId (jobid, sizeof (jobid));
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("%d: PMI2_Job_Getid: %s", rank, pmi2_strerror (e));
+
+    /* Parse PMI_process_mapping, get this rank's nodeid and clique size
+     */
+    e = PMI2_Info_GetJobAttr ("PMI_process_mapping", val, sizeof (val), NULL);
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("%d: PMI2_Info_GetJobAttr PMI_process_mapping: %s",
+                      rank, pmi2_strerror (e));
+
+    if (!(map = taskmap_decode (val, &error)))
+        log_msg_exit ("%d: error parsing PMI_process_mapping: %s",
+                      rank, error.text);
+    if ((nodeid = taskmap_nodeid (map, rank)) < 0)
+        log_msg_exit ("%d: failed to get this rank's nodeid: %s",
+                      rank, strerror (errno));
+    if (!(taskids = taskmap_taskids (map, nodeid)))
+        log_msg_exit ("%d: failed to get taskids for node %d: %s",
+                      rank, nodeid, strerror (errno));
+
+    /* Set clique_rank to this rank's position in taskids
+     */
+    clique_rank = find_id (taskids, rank);
+    if (clique_rank == -1)
+        log_msg_exit ("%d: unable to determine clique rank", rank);
+
+    /* Exchange node-scope keys.
+     * Each rank puts one key, then fetches the key of clique neighbor.
+     * N.B. keys deliberately overlap across cliques.
+     */
+    snprintf (key, sizeof (key), "key-%d", clique_rank);
+    snprintf (attr, sizeof (attr), "val-%d", rank);
+    e = PMI2_Info_PutNodeAttr (key, attr);
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("%d: PMI2_Info_PutNodeAttr: %s", rank, pmi2_strerror (e));
+
+    clique_neighbor = get_neighbor (taskids, rank);
+    snprintf (key, sizeof (key), "key-%d", clique_neighbor);
+    snprintf (expected_attr, sizeof (expected_attr), "val-%d",
+              clique_neighbor);
+    e = PMI2_Info_GetNodeAttr (key, attr, sizeof (attr), NULL, 1);
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("%d: PMI2_Info_GetNodeAttr %s: %s",
+                      rank, key, pmi2_strerror (e));
+    if (!streq (attr, expected_attr))
+        log_msg_exit ("%d: PMI_Info_GetNodeAttr %s: exp %s got %s\n",
+                      rank, key, expected_val, val);
+
+    /* Put some keys; Fence; Get neighbor's keys.
+     */
+    for (int i = 0; i < keycount; i++) {
+        snprintf (key, sizeof (key), "key-%d-%d", rank, i);
+        snprintf (val, sizeof (val), "val-%d.%d", rank, i);
+        e = PMI2_KVS_Put (key, val);
+        if (e != PMI2_SUCCESS)
+            log_msg_exit ("%d: PMI2_KVS_Put: %s", rank, pmi2_strerror (e));
+    }
+    e = PMI2_KVS_Fence();
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("%d: PMI2_KVS_Fence: %s", rank, pmi2_strerror (e));
+    for (int i = 0; i < keycount; i++) {
+        snprintf (key, sizeof (key), "key-%d-%d",
+                  rank > 0 ? rank - 1 : size - 1, i);
+        e = PMI2_KVS_Get (jobid, 0, key, val, sizeof (val), &length);
+        if (e != PMI2_SUCCESS)
+            log_msg_exit ("%d: PMI2_KVS_Get: %s", rank, pmi2_strerror (e));
+        snprintf (expected_val, sizeof (expected_val), "val-%d.%d",
+                  rank > 0 ? rank - 1 : size - 1, i);
+        if (!streq (val, expected_val))
+            log_msg_exit ("%d: PMI_KVS_Get: exp %s got %s\n",
+                          rank, expected_val, val);
+        if (length != strlen (val))
+            log_msg_exit ("%d: PMI_KVS_Get %s: length %d != expected %zd",
+                          rank, key, length, strlen (val));
+    }
+
+    /* Finalize
+     */
+    e = PMI2_Finalize ();
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("%d: PMI2_Finalize: %s", rank, pmi2_strerror (e));
+
+
+    taskmap_destroy (map);
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libpmi/test/plugin/singlex.c b/src/common/libpmi/test/plugin/singlex.c
new file mode 100644
index 000000000000..5379ad6c6058
--- /dev/null
+++ b/src/common/libpmi/test/plugin/singlex.c
@@ -0,0 +1,29 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* singlex.c - dso wrapper for 'single' builtin plugin */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+int upmi_single_init (flux_plugin_t *p);
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (upmi_single_init (p) < 0)
+        return -1;
+    if (flux_plugin_set_name (p, "singlex") < 0) // override 'single'
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libpmi/test/pmi2_info.c b/src/common/libpmi/test/pmi2_info.c
new file mode 100644
index 000000000000..5b8964f1cb6f
--- /dev/null
+++ b/src/common/libpmi/test/pmi2_info.c
@@ -0,0 +1,91 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <stdbool.h>
+#include <getopt.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/xzmalloc.h"
+#include "src/common/libpmi/pmi2.h"
+#include "src/common/libpmi/pmi_strerror.h"
+
+/* We don't have a pmi2_strerror() but the codes are mostly the same as PMI-1
+ */
+#define pmi2_strerror pmi_strerror
+
+#define OPTIONS "a:"
+static const struct option longopts[] = {
+    {"abort",        required_argument,  0, 'a'},
+    {0, 0, 0, 0},
+};
+
+int main(int argc, char *argv[])
+{
+    int spawned, size, rank, appnum;
+    char jobid[PMI2_MAX_VALLEN];
+    char map[PMI2_MAX_ATTRVALUE];
+    char usize[PMI2_MAX_ATTRVALUE];
+    int e;
+    int ch;
+    int abort_rank = -1;
+
+    while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
+        switch (ch) {
+            case 'a':   /* --abort */
+                abort_rank = atoi (optarg);
+                break;
+        }
+    }
+    e = PMI2_Init (&spawned, &size, &rank, &appnum);
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("PMI2_Init: %s", pmi2_strerror (e));
+    if (PMI2_Initialized () == 0)
+        log_msg_exit ("%d: PMI2_Initialized returned 0", rank);
+
+    e = PMI2_Job_GetId (jobid, sizeof (jobid));
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("%d: PMI2_Job_Getid: %s", rank, pmi2_strerror (e));
+
+    e = PMI2_Info_GetJobAttr ("PMI_process_mapping", map, sizeof (map), NULL);
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("%d: PMI2_Info_GetJobAttr PMI_process_mapping: %s",
+                      rank, pmi2_strerror (e));
+
+    e = PMI2_Info_GetJobAttr ("universeSize", usize, sizeof (usize), NULL);
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("%d: PMI2_Info_GetJobAttr universeSize: %s",
+                      rank, pmi2_strerror (e));
+
+    printf ("%d: size=%d appnum=%d jobid=%s"
+            " PMI_process_mapping=%s universeSize=%s\n",
+            rank, size, appnum, jobid, map, usize);
+
+    if (abort_rank == rank)
+        PMI2_Abort (1, "This is a PMI2_Abort message.\nWith\nMultiple\nLines");
+
+    e = PMI2_Finalize ();
+    if (e != PMI2_SUCCESS)
+        log_msg_exit ("%d: PMI2_Finalize: %s", rank, pmi2_strerror (e));
+
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libpmi/test/pmi_info.c b/src/common/libpmi/test/pmi_info.c
index e5c7dd63ce8a..4016dfdfacc5 100644
--- a/src/common/libpmi/test/pmi_info.c
+++ b/src/common/libpmi/test/pmi_info.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
@@ -21,7 +24,6 @@
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libpmi/pmi.h"
 #include "src/common/libpmi/pmi_strerror.h"
-#include "src/common/libpmi/clique.h"
 
 #define OPTIONS "ca:"
 static const struct option longopts[] = {
@@ -30,6 +32,24 @@ static const struct option longopts[] = {
     {0, 0, 0, 0},
 };
 
+static char *pmi_cliquetostr (char *buf, int bufsz, int *ranks, int length)
+{
+    int n, i, count;
+
+    buf[0] = '\0';
+    for (i = 0, count  = 0; i < length; i++) {
+        n = snprintf (buf + count,
+                      bufsz - count,
+                      "%s%d",
+                      i > 0 ? "," : "",
+                      ranks[i]);
+        if (n >= bufsz - count)
+            return "overflow";
+        count += n;
+    }
+    return buf;
+}
+
 int main(int argc, char *argv[])
 {
     int rank, size, appnum;
@@ -92,7 +112,7 @@ int main(int argc, char *argv[])
         int clen;
         int *clique;
         char *s;
-        char buf[256];
+        char buf[4096];
 
         e = PMI_Get_clique_size (&clen);
         if (e != PMI_SUCCESS)
diff --git a/src/common/libpmi/test/server_thread.c b/src/common/libpmi/test/server_thread.c
index 319f298a9796..996b386fb0d1 100644
--- a/src/common/libpmi/test/server_thread.c
+++ b/src/common/libpmi/test/server_thread.c
@@ -14,14 +14,14 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <pthread.h>
-#include <czmq.h>
+#include <assert.h>
 #include <flux/core.h>
 
-#include "src/common/libutil/oom.h"
-#include "src/common/libutil/xzmalloc.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libpmi/simple_server.h"
-#include "src/common/libpmi/dgetline.h"
 #include "src/common/libpmi/pmi.h"
+#include "src/common/libutil/fdutils.h"
+#include "src/common/liblsd/cbuf.h"
 
 #include "src/common/libtap/tap.h"
 
@@ -29,9 +29,12 @@
 
 struct pmi_server_context;
 
+// server side context for one client
 struct client {
     int sfd;
     int rank;
+    cbuf_t cbuf;
+    char buf[SIMPLE_MAX_PROTO_LINE];
     struct pmi_server_context *ctx;
 };
 
@@ -87,31 +90,48 @@ static int s_send_response (void *client, const char *buf)
 {
     int *rfd = client;
 
-    return dputline (*rfd, buf);
+    return dprintf (*rfd, "%s", buf);
 }
 
-static void s_io_cb (flux_reactor_t *r, flux_watcher_t *w,
-                     int revents, void *arg)
+static void s_buf_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
 {
     struct client *cli = arg;
     struct pmi_server_context *ctx = cli->ctx;
-    int fd = flux_fd_watcher_get_fd (w);
-    int rc;
+    int nbytes;
 
     assert (ctx->magic == MAGIC_VALUE);
-    if (dgetline (fd, ctx->buf, sizeof (ctx->buf)) < 0) {
-        flux_reactor_stop_error (r);
-        return;
+
+    nbytes = cbuf_write_from_fd (cli->cbuf, cli->sfd, -1, NULL);
+    if (nbytes < 0) {
+        if (errno == EWOULDBLOCK || errno == EAGAIN)
+            return;
+        goto error;
     }
-    rc = pmi_simple_server_request (ctx->pmi, ctx->buf, cli, cli->rank);
-    if (rc < 0) {
-        flux_reactor_stop_error (r);
-        return;
+    if (nbytes == 0) { // premature EOF
+        errno = EIO;
+        goto error;
     }
-    if (rc == 1) {
-        close (fd);
-        flux_watcher_stop (w);
+    // truncation is not possible - buf is the same size as cbuf
+    while ((nbytes = cbuf_read_line (cli->cbuf,
+                                     cli->buf,
+                                     sizeof (cli->buf),
+                                     1)) != 0) {
+        if (nbytes < 0) // programming error
+            BAIL_OUT ("cbuf_read_line failed: %s", strerror (errno));
+        int rc = pmi_simple_server_request (ctx->pmi, cli->buf, cli, cli->rank);
+        if (rc < 0)
+            goto error;
+        if (rc == 1) {
+            flux_watcher_stop (w); // normal exit
+            break;
+        }
     }
+    return;
+error:
+    flux_reactor_stop_error (r);
 }
 
 static int s_barrier_enter (void *arg)
@@ -141,12 +161,13 @@ static void *server_thread (void *arg)
     if (!(w = calloc (ctx->size, sizeof (w[0]))))
         BAIL_OUT ("calloc failed");
     for (i = 0; i < ctx->size; i++) {
+        fd_set_nonblocking (ctx->cli[i].sfd);
         if (!(w[i] = flux_fd_watcher_create (reactor,
                                              ctx->cli[i].sfd,
                                              FLUX_POLLIN,
-                                             s_io_cb,
+                                             s_buf_cb,
                                              &ctx->cli[i])))
-            BAIL_OUT ("flux_fd_watcher_create failed");
+            BAIL_OUT ("could not create fd watcher: %s", strerror (errno));
         flux_watcher_start (w[i]);
     }
     if (flux_reactor_run (reactor, 0) < 0)
@@ -163,6 +184,11 @@ void s_trace (void *arg, const char *buf)
     diag ("%s", buf);
 }
 
+void s_warn (void *arg, const char *buf)
+{
+    diag ("WARN: %s", buf);
+}
+
 struct pmi_server_context *pmi_server_create (int *cfd, int size)
 {
     char pmi_fd[16];
@@ -172,6 +198,7 @@ struct pmi_server_context *pmi_server_create (int *cfd, int size)
         .barrier_enter = s_barrier_enter,
         .response_send = s_send_response,
         .debug_trace = s_trace,
+        .warn = s_warn,
     };
     struct pmi_server_context *ctx;
     int i;
@@ -191,12 +218,26 @@ struct pmi_server_context *pmi_server_create (int *cfd, int size)
 
     for (i = 0; i < size; i++) {
         int fd[2];
+#ifdef SOCK_CLOEXEC
         if (socketpair (PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0, fd) < 0)
             BAIL_OUT ("socketpair failed");
+#else
+        if (socketpair (PF_LOCAL, SOCK_STREAM, 0, fd) < 0
+            || fd_set_cloexec (fd[0]) < 0
+            || fd_set_cloexec (fd[1]) < 0)
+            BAIL_OUT ("socketpair failed");
+#endif
+
         cfd[i] = fd[0];
         ctx->cli[i].sfd = fd[1];
         ctx->cli[i].rank = i;
         ctx->cli[i].ctx = ctx;
+        if (!(ctx->cli[i].cbuf = cbuf_create (SIMPLE_MAX_PROTO_LINE,
+                                              SIMPLE_MAX_PROTO_LINE))
+            || cbuf_opt_set (ctx->cli[i].cbuf,
+                             CBUF_OPT_OVERWRITE,
+                             CBUF_NO_DROP) < 0)
+            BAIL_OUT ("could not create circular buffer for client %d", i);
     }
 
     ctx->pmi = pmi_simple_server_create (server_ops,
@@ -223,6 +264,8 @@ void pmi_server_destroy (struct pmi_server_context *ctx)
         BAIL_OUT ("pthread_join failed");
     pmi_simple_server_destroy (ctx->pmi);
     zhashx_destroy (&ctx->kvs);
+    for (int i = 0; i < ctx->size; i++)
+        cbuf_destroy (ctx->cli[i].cbuf);
     free (ctx->cli);
     ctx->magic = ~MAGIC_VALUE;
     free (ctx);
diff --git a/src/common/libpmi/test/simple.c b/src/common/libpmi/test/simple.c
index a9d76d0eb22c..a4b2f53ec40c 100644
--- a/src/common/libpmi/test/simple.c
+++ b/src/common/libpmi/test/simple.c
@@ -15,24 +15,25 @@
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libpmi/simple_client.h"
 #include "src/common/libpmi/simple_server.h"
-#include "src/common/libpmi/dgetline.h"
 #include "src/common/libpmi/keyval.h"
 #include "src/common/libpmi/pmi.h"
 #include "src/common/libflux/reactor.h"
+#include "ccan/str/str.h"
 
 #include "src/common/libtap/tap.h"
 
 #include "server_thread.h"
 
-int fake_publish (int fd, const char *service, const char *port)
+int fake_publish (FILE *fp, const char *service, const char *port)
 {
     char buf[SIMPLE_MAX_PROTO_LINE];
     int rc;
 
-    if (dprintf (fd, "cmd=publish_name service=%s port=%s\n",
+    if (fprintf (fp,
+                 "cmd=publish_name service=%s port=%s\n",
                  service, port) < 0)
         return PMI_FAIL;
-    if (dgetline (fd, buf, sizeof (buf)) < 0)
+    if (!fgets (buf, sizeof (buf), fp))
         return PMI_FAIL;
     if (keyval_parse_isword (buf, "cmd", "publish_result") < 0)
         return PMI_FAIL;
@@ -41,14 +42,14 @@ int fake_publish (int fd, const char *service, const char *port)
     return 0;
 }
 
-int fake_unpublish (int fd, const char *service)
+int fake_unpublish (FILE *fp, const char *service)
 {
     char buf[SIMPLE_MAX_PROTO_LINE];
     int rc;
 
-    if (dprintf (fd, "cmd=unpublish_name service=%s\n", service) < 0)
+    if (fprintf (fp, "cmd=unpublish_name service=%s\n", service) < 0)
         return PMI_FAIL;
-    if (dgetline (fd, buf, sizeof (buf)) < 0)
+    if (!fgets(buf, sizeof (buf), fp))
         return PMI_FAIL;
     if (keyval_parse_isword (buf, "cmd", "unpublish_result") < 0)
         return PMI_FAIL;
@@ -57,14 +58,14 @@ int fake_unpublish (int fd, const char *service)
     return 0;
 }
 
-int fake_lookup (int fd, const char *service)
+int fake_lookup (FILE *fp, const char *service)
 {
     char buf[SIMPLE_MAX_PROTO_LINE];
     int rc;
 
-    if (dprintf (fd, "cmd=lookup_name service=%s\n", service) < 0)
+    if (fprintf (fp, "cmd=lookup_name service=%s\n", service) < 0)
         return PMI_FAIL;
-    if (dgetline (fd, buf, sizeof (buf)) < 0)
+    if (!fgets (buf, sizeof (buf), fp))
         return PMI_FAIL;
     if (keyval_parse_isword (buf, "cmd", "lookup_result") < 0)
         return PMI_FAIL;
@@ -73,30 +74,23 @@ int fake_lookup (int fd, const char *service)
     return 0;
 }
 
-int fake_spawn (int fd)
+int fake_spawn (FILE *fp)
 {
     char buf[SIMPLE_MAX_PROTO_LINE];
     int rc;
 
-    if (dprintf (fd, "mcmd=spawn\n") < 0)
+    if (fprintf (fp,
+                 "mcmd=spawn\n"
+                 "nprocs=1\n"
+                 "execname=foo\n"
+                 "totspawns=1\n"
+                 "spawnssofar=1\n"
+                 "argcnt=0\n"
+                 "preput_num=0\n"
+                 "info_num=0\n"
+                 "endcmd\n") < 0)
         return PMI_FAIL;
-    if (dprintf (fd, "nprocs=1\n") < 0)
-        return PMI_FAIL;
-    if (dprintf (fd, "execname=foo\n") < 0)
-        return PMI_FAIL;
-    if (dprintf (fd, "totspawns=1\n") < 0)
-        return PMI_FAIL;
-    if (dprintf (fd, "spawnssofar=1\n") < 0)
-        return PMI_FAIL;
-    if (dprintf (fd, "argcnt=0\n") < 0)
-        return PMI_FAIL;
-    if (dprintf (fd, "preput_num=0\n") < 0)
-        return PMI_FAIL;
-    if (dprintf (fd, "info_num=0\n") < 0)
-        return PMI_FAIL;
-    if (dprintf (fd, "endcmd\n") < 0)
-        return PMI_FAIL;
-    if (dgetline (fd, buf, sizeof (buf)) < 0)
+    if (!fgets (buf, sizeof (buf), fp))
         return PMI_FAIL;
     if (keyval_parse_isword (buf, "cmd", "spawn_result") < 0)
         return PMI_FAIL;
@@ -105,6 +99,25 @@ int fake_spawn (int fd)
     return 0;
 }
 
+int fake_init (FILE *fp, int version, int subversion)
+{
+    char buf[SIMPLE_MAX_PROTO_LINE];
+    int rc;
+
+    if (fprintf (fp,
+                 "cmd=init pmi_version=%d pmi_subversion=%d\n",
+                 version,
+                 subversion) < 0)
+        return -1;
+    if (!fgets (buf, sizeof (buf), fp))
+        return -1;
+    if (keyval_parse_isword (buf, "cmd", "response_to_init") < 0)
+        return -1;
+    if (keyval_parse_int (buf, "rc", &rc) == 0 && rc != 0)
+        return rc;
+    return 0;
+}
+
 int main (int argc, char *argv[])
 {
     struct pmi_simple_client *cli;
@@ -117,10 +130,24 @@ int main (int argc, char *argv[])
     char pmi_fd[16];
     char pmi_rank[16];
     char pmi_size[16];
+    FILE *cfp;
 
     plan (NO_PLAN);
 
     srv = pmi_server_create (cfd, 1);
+    if (!(cfp = fdopen (dup (cfd[0]), "r+")))
+        BAIL_OUT ("fdopen/dup failed");
+
+    /* N.B. server allows multiple inits, so we take advantage
+     * of this to beat on its version negotiation without any
+     * setup/teardown of the server.
+     */
+    ok (fake_init (cfp, 1, 0) == -1,
+        "fake init for protocol 1.0 fails");
+    ok (fake_init (cfp, 2, 0) == -1,
+        "fake init for protocol 2.0 fails");
+    ok (fake_init (cfp, 1, 1) == 0,
+        "fake init for protocol 1.1 succeeds");
 
     /* create/init
      */
@@ -170,7 +197,7 @@ int main (int argc, char *argv[])
     val = xzmalloc (cli->vallen_max);
     ok (pmi_simple_client_kvs_get (cli, name, "foo",
                                    val, cli->vallen_max) == PMI_SUCCESS
-        && !strcmp (val, "bar"),
+        && streq (val, "bar"),
         "pmi_simple_client_kvs_get foo OK, val=%s", val);
 
     /* put long=... / get long
@@ -183,7 +210,7 @@ int main (int argc, char *argv[])
     ok (pmi_simple_client_kvs_get (cli, name, "long",
                                    val, cli->vallen_max) == PMI_SUCCESS
         && strnlen (val2, cli->vallen_max) < cli->vallen_max
-        && strcmp (val, val2) == 0,
+        && streq (val, val2),
         "pmi_simple_client_kvs_get long OK, val=xxx...");
 
     /* put: value too long
@@ -228,21 +255,23 @@ int main (int argc, char *argv[])
         "pmi_simple_client_barrier OK (rigged errors cleared)");
 
     /* publish */
-    ok (fake_publish (cfd[0], "foo", "bar") == PMI_FAIL,
+    ok (fake_publish (cfp, "foo", "bar") == PMI_FAIL,
         "publish fails (unimplemented)");
 
     /* unpublish */
-    ok (fake_unpublish (cfd[0], "foo") == PMI_FAIL,
+    ok (fake_unpublish (cfp, "foo") == PMI_FAIL,
         "unpublish fails (unimplemented)");
 
     /* lookup */
-    ok (fake_lookup (cfd[0], "foo") == PMI_FAIL,
+    ok (fake_lookup (cfp, "foo") == PMI_FAIL,
         "lookup fails (unimplemented)");
 
     /* spawn */
-    ok (fake_spawn (cfd[0]) == PMI_FAIL,
+    ok (fake_spawn (cfp) == PMI_FAIL,
         "spawn fails (unimplemented)");
 
+    fclose (cfp);
+
     /* finalize
      */
 
diff --git a/src/common/libpmi/test/upmi.c b/src/common/libpmi/test/upmi.c
new file mode 100644
index 000000000000..1e8b40aa947e
--- /dev/null
+++ b/src/common/libpmi/test/upmi.c
@@ -0,0 +1,185 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <string.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+#include "src/common/libpmi/upmi.h"
+
+void diag_trace (void *arg, const char *text)
+{
+    diag ("%s", text);
+}
+
+void test_single (void)
+{
+    struct upmi *upmi;
+    flux_error_t error;
+    struct upmi_info info;
+    const char *name;
+    char *val;
+
+    upmi = upmi_create ("single",
+                        UPMI_TRACE,
+                        diag_trace,
+                        NULL,
+                        &error);
+    ok (upmi != NULL,
+        "upmi_create spec=single works");
+
+    ok (upmi_initialize (upmi, &info, &error) == 0,
+        "upmi_initialize works");
+    ok (info.size == 1 && info.rank == 0,
+        "info rank==0, size==1");
+    is (info.name, "single",
+        "info name==single"); // normally jobid but not with spec=single
+    name = upmi_describe (upmi);
+    is (name, "single",
+        "upmi_describe returns single");
+
+    ok (upmi_put (upmi, "foo", "bar", &error) == 0,
+        "upmi_put key=foo val=bar works");
+
+    ok (upmi_barrier (upmi, &error) == 0,
+        "upmi_barrier works");
+
+    ok (upmi_get (upmi, "foo", -1, &val, &error) == 0,
+        "upmi_get key=foo works");
+    is (val, "bar",
+        "value==bar");
+    free (val);
+
+    error.text[0] = '\0';
+    ok (upmi_get (upmi, "notakey", -1, &val, &error) < 0,
+        "upmi_get key=notakey fails");
+    ok (strlen (error.text) > 0,
+        "error.text was set");
+
+    ok (upmi_finalize (upmi, &error) == 0,
+        "upmi_finalize works");
+
+    upmi_destroy (upmi);
+}
+
+void test_inval (void)
+{
+    struct upmi *upmi;
+    flux_error_t error;
+
+    error.text[0] = '\0';
+    upmi = upmi_create ("notpmi", 0, NULL, NULL, &error);
+    ok (upmi == NULL,
+        "upmi_create spec=notpmi fails");
+    diag ("%s", error.text);
+
+    error.text[0] = '\0';
+    upmi = upmi_create ("single", 0xffff, NULL, NULL, &error);
+    ok (upmi == NULL,
+        "upmi_create spec=single flags=0xffff fails");
+    diag ("%s", error.text);
+
+    upmi = upmi_create ("single", UPMI_TRACE, diag_trace, NULL, &error);
+    if (!upmi)
+        BAIL_OUT ("upmi_create spec=single failed");
+    ok (upmi_initialize (NULL, NULL, &error) < 0,
+        "upmi_initialize upmi=NULL fails");
+    if (upmi_initialize (upmi, NULL, &error) < 0)
+        BAIL_OUT ("upmi_initialize failed");
+    upmi_destroy (upmi);
+}
+
+/* The "singlex" dso (a clone of "single" with a different name) is
+ * built as a dso in UPMI_TEST_SEARCHPATH
+ */
+void test_dso (void)
+{
+    if (setenv ("FLUX_PMI_CLIENT_SEARCHPATH", UPMI_TEST_SEARCHPATH, true) < 0)
+        BAIL_OUT ("setenv failed");
+
+    struct upmi *upmi;
+    flux_error_t error;
+
+    upmi = upmi_create ("singlex",
+                        UPMI_TRACE,
+                        diag_trace,
+                        NULL,
+                        &error);
+    if (!upmi)
+        diag ("%s", error.text);
+    ok (upmi != NULL,
+        "upmi_create spec=singlex works");
+
+    upmi_destroy (upmi);
+
+    (void)unsetenv ("FLUX_PMI_CLIENT_SEARCHPATH");
+}
+
+/* Ensure the environment can alter the methods search path
+ */
+void test_env (void)
+{
+    if (setenv ("FLUX_PMI_CLIENT_METHODS", "unknown", true) < 0)
+        BAIL_OUT ("setenv failed");
+
+    struct upmi *upmi;
+    flux_error_t error;
+
+    upmi = upmi_create (NULL,
+                        UPMI_TRACE,
+                        diag_trace,
+                        NULL,
+                        &error);
+    if (!upmi)
+        diag ("%s", error.text);
+    ok (upmi == NULL,
+        "upmi_create tries only FLUX_PMI_CLIENT_METHODS");
+
+    if (setenv ("FLUX_PMI_CLIENT_SEARCHPATH", UPMI_TEST_SEARCHPATH, true) < 0)
+        BAIL_OUT ("setenv failed");
+    if (setenv ("FLUX_PMI_CLIENT_METHODS", "unknown singlex simple", true) < 0)
+        BAIL_OUT ("setenv failed");
+
+    upmi = upmi_create (NULL,
+                        UPMI_TRACE,
+                        diag_trace,
+                        NULL,
+                        &error);
+    if (!upmi)
+        diag ("%s", error.text);
+    ok (upmi != NULL && streq (upmi_describe (upmi), "singlex"),
+        "upmi_create respects FLUX_PMI_CLIENT_METHODS order");
+    upmi_destroy (upmi);
+
+    (void)unsetenv ("FLUX_PMI_CLIENT_METHODS");
+    (void)unsetenv ("FLUX_PMI_CLIENT_SEARCHPATH");
+}
+
+int main (int argc, char **argv)
+{
+    plan (NO_PLAN);
+
+    (void)unsetenv ("FLUX_PMI_CLIENT_SEARCHPATH");
+    (void)unsetenv ("FLUX_PMI_CLIENT_METHODS");
+
+    test_single ();
+    test_inval ();
+    test_dso ();
+    test_env ();
+
+    done_testing ();
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libpmi/upmi.c b/src/common/libpmi/upmi.c
new file mode 100644
index 000000000000..65aa267741da
--- /dev/null
+++ b/src/common/libpmi/upmi.c
@@ -0,0 +1,628 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* upmi.c - universal pmi client for internal use */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <assert.h>
+#ifdef HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <glob.h>
+
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "upmi.h"
+#include "upmi_plugin.h"
+
+struct upmi {
+    zlist_t *plugins;
+    flux_plugin_t *plugin;
+    flux_plugin_arg_t *args;
+    char *searchpath;
+    size_t searchpath_len;
+    char *methods;
+    size_t methods_len;
+    char *name;
+    void *ctx;
+    int flags;
+    upmi_trace_f trace_fun;
+    void *trace_arg;
+};
+
+void upmi_trace (struct upmi *upmi, const char *fmt, ...);
+static int upmi_preinit (struct upmi *upmi,
+                         int flags,
+                         const char *path,
+                         const char **note,
+                         flux_error_t *error);
+
+int upmi_simple_init (flux_plugin_t *p);
+int upmi_libpmi2_init (flux_plugin_t *p);
+int upmi_libpmi_init (flux_plugin_t *p);
+int upmi_single_init (flux_plugin_t *p);
+
+static flux_plugin_init_f builtins[] = {
+    &upmi_simple_init,
+    &upmi_libpmi2_init,
+    &upmi_libpmi_init,
+    &upmi_single_init,
+};
+
+static const char *default_methods = "simple libpmi2 libpmi single";
+
+void upmi_destroy (struct upmi *upmi)
+{
+    if (upmi) {
+        int saved_errno = errno;
+        free (upmi->methods);
+        free (upmi->searchpath);
+        free (upmi->name);
+        flux_plugin_arg_destroy (upmi->args);
+        zlist_destroy (&upmi->plugins);
+        free (upmi);
+        errno = saved_errno;
+    }
+}
+
+static int upmi_plugin_append (struct upmi *upmi, flux_plugin_t *p)
+{
+    if (zlist_append (upmi->plugins, p) < 0)
+        return -1;
+    zlist_freefn (upmi->plugins, p, (zlist_free_fn *)flux_plugin_destroy, true);
+    return 0;
+}
+
+static int upmi_register_external_glob (struct upmi *upmi,
+                                        const char *pattern,
+                                        flux_error_t *error)
+{
+    glob_t pglob;
+    int rc;
+
+    if ((rc = glob (pattern, 0, NULL, &pglob)) != 0) {
+        if (rc == GLOB_NOMATCH)
+            return 0;
+        return errprintf (error,
+                          "plugin glob error: %s",
+                          rc == GLOB_NOSPACE ? "out of memory" :
+                          rc == GLOB_ABORTED ? "read error" : "?");
+    }
+    for (int i = 0; i < pglob.gl_pathc; i++) {
+        flux_plugin_t *p;
+
+        if (!(p = flux_plugin_create ()))
+            goto nomem;
+        if (flux_plugin_load_dso (p, pglob.gl_pathv[i]) < 0) {
+            errprintf (error, "%s", flux_plugin_strerror (p));
+            flux_plugin_destroy (p);
+            goto error;
+        }
+        if (upmi_plugin_append (upmi, p) < 0) {
+            flux_plugin_destroy (p);
+            goto nomem;
+        }
+    }
+    globfree (&pglob);
+    return 0;
+nomem:
+    errprintf (error, "out of memory");
+error:
+    globfree (&pglob);
+    return -1;
+}
+
+static int upmi_register_external (struct upmi *upmi, flux_error_t *error)
+{
+    const char *item;
+    char pattern[1024];
+
+    item = argz_next (upmi->searchpath, upmi->searchpath_len, NULL);
+    while (item) {
+        snprintf (pattern, sizeof (pattern), "%s/*.so", item);
+        if (upmi_register_external_glob (upmi, pattern, error) < 0)
+            return -1;
+        item = argz_next (upmi->searchpath, upmi->searchpath_len, item);
+    }
+    return 0;
+}
+
+static int upmi_register_builtin (struct upmi *upmi)
+{
+    for (int i = 0; i < ARRAY_SIZE (builtins); i++) {
+        flux_plugin_init_f plugin_init = builtins[i];
+        flux_plugin_t *p;
+
+        if (!(p = flux_plugin_create ())
+            || plugin_init (p) < 0
+            || upmi_plugin_append (upmi, p) < 0) {
+            flux_plugin_destroy (p);
+            return -1;
+        }
+    }
+    return 0;
+}
+
+/* Instantiate 'struct upmi' without selecting a plugin.
+ */
+static struct upmi *upmi_create_uninit (const char *methods,
+                                        const char *searchpath,
+                                        int flags,
+                                        upmi_trace_f trace_fun,
+                                        void *trace_arg,
+                                        flux_error_t *error)
+{
+    struct upmi *upmi;
+
+    if (flags & ~(UPMI_TRACE | UPMI_LIBPMI_NOFLUX | UPMI_LIBPMI2_CRAY)) {
+        errprintf (error, "invalid argument");
+        return NULL;
+    }
+    if (!(upmi = calloc (1, sizeof (*upmi)))
+        || !(upmi->plugins = zlist_new ())
+        || (argz_create_sep (searchpath,
+                             ':',
+                             &upmi->searchpath,
+                             &upmi->searchpath_len) != 0)
+        || argz_create_sep (methods,
+                            ' ',
+                            &upmi->methods,
+                            &upmi->methods_len) != 0) {
+        errprintf (error, "out of memory");
+        goto error;
+    }
+    upmi->flags = flags;
+    upmi->trace_fun = trace_fun;
+    upmi->trace_arg = trace_arg;
+    if (upmi_register_builtin (upmi) < 0) {
+        errprintf (error, "error registering builtin plugins");
+        goto error;
+    }
+    return upmi;
+error:
+    upmi_destroy (upmi);
+    return NULL;
+}
+
+static bool match_scheme (const char *scheme, const char *uri)
+{
+    size_t slen = strlen (scheme);
+    if (slen < strlen (uri)
+        && uri[slen] == ':'
+        && strstarts (uri, scheme))
+            return true;
+    return streq (uri, scheme);
+}
+
+/* Search for a plugin by name.
+ * If a plugin name is not found among pre-loaded builtin plugins,
+ * load the external plugins and search again.
+ */
+static flux_plugin_t *lookup_plugin (struct upmi *upmi,
+                                     const char *uri,
+                                     flux_error_t *error)
+{
+    flux_plugin_t *p;
+    bool loaded = false;
+again:
+    p = zlist_first (upmi->plugins);
+    while (p) {
+        if (match_scheme (flux_plugin_get_name (p), uri))
+            break;
+        p = zlist_next (upmi->plugins);
+    }
+    if (!p && !loaded && upmi->searchpath) {
+        if (upmi_register_external (upmi, error) < 0)
+            return NULL;
+        loaded = true;
+        goto again;
+    }
+    if (!p)
+        errprintf (error, "plugin matching '%s' not found", uri);
+    return p;
+}
+
+struct upmi *upmi_create (const char *uri,
+                          int flags,
+                          upmi_trace_f trace_fun,
+                          void *trace_arg,
+                          flux_error_t *errp)
+{
+    struct upmi *upmi;
+    const char *searchpath;
+    const char *methods;
+
+    if (!(methods = getenv ("FLUX_PMI_CLIENT_METHODS")))
+        methods = default_methods;
+    if (!(searchpath = getenv ("FLUX_PMI_CLIENT_SEARCHPATH")))
+        searchpath = flux_conf_builtin_get ("upmi_pluginpath", FLUX_CONF_AUTO);
+
+    if (!(upmi = upmi_create_uninit (methods,
+                                     searchpath,
+                                     flags,
+                                     trace_fun,
+                                     trace_arg,
+                                     errp)))
+        return NULL;
+
+    if (uri) {
+        const char *path;
+        const char *note;
+
+        if (!(upmi->plugin = lookup_plugin (upmi, uri, errp)))
+            goto error;
+        if ((path = strchr (uri, ':')))
+            path++;
+        if (upmi_preinit (upmi, upmi->flags, path, &note, errp) < 0)
+            goto error;
+        if (note != NULL)
+            upmi_trace (upmi, "%s", note);
+    }
+    else {
+        flux_error_t error;
+        const char *note;
+
+        uri = argz_next (upmi->methods, upmi->methods_len, NULL);
+        while (uri) {
+            upmi_trace (upmi, "trying '%s'", uri);
+            if ((upmi->plugin = lookup_plugin (upmi, uri, &error))
+                && upmi_preinit (upmi, upmi->flags, NULL, &note, &error) == 0) {
+                upmi_trace (upmi, "%s", note ? note : "selected");
+                break;
+            }
+            upmi_trace (upmi, "%s", error.text);
+            errprintf (errp, "%s", error.text);
+            upmi->plugin = NULL;
+            uri = argz_next (upmi->methods, upmi->methods_len, uri);
+        }
+        if (!upmi->plugin) // 'error' was filled by last plugin in methods list
+            goto error;
+    }
+    return upmi;
+error:
+    upmi_destroy (upmi);
+    return NULL;
+}
+
+const char *upmi_describe (struct upmi *upmi)
+{
+    const char *name = NULL;
+    if (upmi)
+        name = flux_plugin_get_name (upmi->plugin);
+    return name;
+}
+
+/* Invoke plugin callback.
+ * This destroys and recreates upmi->args with IN arguments based on (fmt, ap).
+ * After the successful call, upmi->args contains OUT arguments.
+ */
+static int upmi_vcall (struct upmi *upmi,
+                       const char *name,
+                       flux_error_t *error,
+                       const char *fmt,
+                       va_list ap)
+{
+    int rc;
+
+    flux_plugin_arg_destroy (upmi->args);
+    if (!(upmi->args = flux_plugin_arg_create ())) {
+        errprintf (error, "out of memory");
+        return -1;
+    }
+    if (fmt) {
+        if (flux_plugin_arg_vpack (upmi->args,
+                                   FLUX_PLUGIN_ARG_IN,
+                                   fmt,
+                                   ap) < 0) {
+            errprintf (error, "%s", flux_plugin_arg_strerror (upmi->args));
+            return -1;
+        }
+    }
+    rc = flux_plugin_call (upmi->plugin, name, upmi->args);
+    if (rc == 0) {
+        errprintf (error, "%s not implemented", name);
+        return -1;
+    }
+    if (rc < 0) {
+        const char *errmsg;
+        if (flux_plugin_arg_unpack (upmi->args,
+                                    FLUX_PLUGIN_ARG_OUT,
+                                    "{s:s}",
+                                    "errmsg", &errmsg) < 0)
+            errmsg = strerror (errno != 0 ? errno : EINVAL);
+        errprintf (error, "%s", errmsg);
+        return -1;
+    }
+    return 0;
+}
+
+static int upmi_call (struct upmi *upmi,
+                      const char *name,
+                      flux_error_t *error,
+                      const char *fmt,
+                      ...)
+{
+    va_list ap;
+    int rc;
+
+    va_start (ap, fmt);
+    rc = upmi_vcall (upmi, name, error, fmt, ap);
+    va_end (ap);
+    return rc;
+}
+
+static int upmi_preinit (struct upmi *upmi,
+                         int flags,
+                         const char *path,
+                         const char **notep,
+                         flux_error_t *error)
+{
+    json_t *payload;
+    int rc;
+
+    if (!(payload = json_object ()))
+        goto nomem;
+    if ((flags & UPMI_LIBPMI_NOFLUX)) {
+        if (json_object_set (payload, "noflux", json_true ()) < 0)
+            goto nomem;
+    }
+    if ((flags & UPMI_LIBPMI2_CRAY)) {
+        if (json_object_set (payload, "craycray", json_true ()) < 0)
+            goto nomem;
+    }
+    if (path) {
+        json_t *o;
+        if (!(o = json_string (path))
+            || json_object_set_new (payload, "path", o) < 0) {
+            json_decref (o);
+            goto nomem;
+        }
+    }
+    rc = upmi_call (upmi, "upmi.preinit", error, "O", payload);
+    json_decref (payload);
+    if (rc == 0 && notep) {
+        const char *note = NULL;
+        (void)flux_plugin_arg_unpack (upmi->args,
+                                      FLUX_PLUGIN_ARG_OUT,
+                                      "{s:s}",
+                                      "note", &note);
+        *notep = note;
+    }
+    return rc;
+nomem:
+    json_decref (payload);
+    errno = ENOMEM;
+    return -1;
+}
+
+int upmi_initialize (struct upmi *upmi,
+                     struct upmi_info *infop,
+                     flux_error_t *errp)
+{
+    struct upmi_info info;
+    const char *name;
+    flux_error_t error;
+
+    if (!upmi) {
+        errprintf (errp, "invalid argument");
+        return -1;
+    }
+    if (upmi_call (upmi, "upmi.initialize", &error, NULL) < 0) {
+        errprintf (errp, "%s", error.text);
+        upmi_trace (upmi, "initialize: %s", error.text);
+        return -1;
+    }
+    if (flux_plugin_arg_unpack (upmi->args,
+                                FLUX_PLUGIN_ARG_OUT,
+                                "{s:i s:s s:i}",
+                                "rank", &info.rank,
+                                "name", &name,
+                                "size", &info.size) < 0) {
+        errprintf (errp, "%s", flux_plugin_arg_strerror (upmi->args));
+        return -1;
+    }
+    free (upmi->name);
+    if (!(upmi->name = strdup (name))) {
+        errprintf (errp, "out of memory");
+        return -1;
+    }
+    info.name = upmi->name;
+    upmi_trace (upmi,
+                "initialize: rank=%d size=%d name=%s: success",
+                info.rank,
+                info.size,
+                info.name);
+    if (infop)
+        *infop = info;
+    return 0;
+}
+
+int upmi_finalize (struct upmi *upmi, flux_error_t *errp)
+{
+    flux_error_t error;
+
+    if (!upmi) {
+        errprintf (errp, "invalid argument");
+        return -1;
+    }
+    if (upmi_call (upmi, "upmi.finalize", &error, NULL) < 0) {
+        errprintf (errp, "%s", error.text);
+        upmi_trace (upmi, "finalize: %s", error.text);
+        return -1;
+    }
+    upmi_trace (upmi, "finalize: success");
+    return 0;
+}
+
+int upmi_abort (struct upmi *upmi, const char *msg, flux_error_t *errp)
+{
+    flux_error_t error;
+
+    if (!upmi || !msg) {
+        errprintf (errp, "invalid argument\n");
+        return -1;
+    }
+    if (upmi_call (upmi,
+                   "upmi.abort",
+                   &error,
+                   "{s:s}",
+                   "msg", msg) < 0) {
+        errprintf (errp, "%s", error.text);
+        upmi_trace (upmi, "abort: %s", error.text);
+        return -1;
+    }
+    // possibly not reached
+    upmi_trace (upmi, "abort: success");
+    return 0;
+}
+
+int upmi_put (struct upmi *upmi,
+              const char *key,
+              const char *value,
+              flux_error_t *errp)
+{
+    flux_error_t error;
+
+    if (!upmi || !key || !value) {
+        errprintf (errp, "invalid argument");
+        return -1;
+    }
+    if (upmi_call (upmi,
+                   "upmi.put",
+                   &error,
+                   "{s:s s:s}",
+                   "key", key,
+                   "value", value) < 0) {
+        errprintf (errp, "%s", error.text);
+        upmi_trace (upmi, "put key=%s: %s", key, error.text);
+        return -1;
+    }
+    upmi_trace (upmi, "put key=%s value=%s: success", key, value);
+    return 0;
+}
+
+int upmi_get (struct upmi *upmi,
+              const char *key,
+              int rank,
+              char **valuep,
+              flux_error_t *errp)
+{
+    flux_error_t error;
+    const char *value;
+    char *cpy;
+
+    if (!upmi || !key || !valuep) {
+        errprintf (errp, "invalid argument");
+        return -1;
+    }
+    if (upmi_call (upmi,
+                   "upmi.get",
+                   &error,
+                   "{s:s s:i}",
+                   "key", key,
+                   "rank", rank) < 0) {
+        errprintf (errp, "%s", error.text);
+        upmi_trace (upmi, "get key=%s: %s", key, error.text);
+        return -1;
+    }
+    if (flux_plugin_arg_unpack (upmi->args,
+                                FLUX_PLUGIN_ARG_OUT,
+                                "{s:s}",
+                                "value", &value) < 0) {
+        errprintf (errp, "%s", flux_plugin_arg_strerror (upmi->args));
+        return -1;
+    }
+    upmi_trace (upmi, "get key=%s value=%s: success", key, value);
+    if (!(cpy = strdup (value))) {
+        errprintf (errp, "out of memory");
+        return -1;
+    }
+    *valuep = cpy;
+    return 0;
+}
+
+int upmi_barrier (struct upmi *upmi, flux_error_t *errp)
+{
+    flux_error_t error;
+
+    if (!upmi) {
+        errprintf (errp, "invalid argument");
+        return -1;
+    }
+    if (upmi_call (upmi, "upmi.barrier", &error, NULL) < 0) {
+        errprintf (errp, "%s", error.text);
+        upmi_trace (upmi, "barrier: %s", error.text);
+        return -1;
+    }
+    upmi_trace (upmi, "barrier: success");
+    return 0;
+}
+
+static void upmi_vtrace (struct upmi *upmi, const char *fmt, va_list ap)
+{
+    if ((upmi->flags & UPMI_TRACE) && upmi->trace_fun) {
+        char buf[1024];
+        const char *prefix = upmi_describe (upmi);
+        snprintf (buf,
+                  sizeof (buf),
+                  "%s: ",
+                  prefix ? prefix : "flux-pmi-client");
+        size_t len = strlen (buf);
+        vsnprintf (buf + len, sizeof (buf) - len, fmt, ap);
+        upmi->trace_fun (upmi->trace_arg, buf);
+    }
+}
+
+void upmi_trace (struct upmi *upmi, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start (ap, fmt);
+    upmi_vtrace (upmi, fmt, ap);
+    va_end (ap);
+}
+
+static void upmi_setverror (flux_plugin_t *p,
+                            flux_plugin_arg_t *args,
+                            const char *fmt,
+                            va_list ap)
+{
+    char buf[128];
+
+    (void)vsnprintf (buf, sizeof (buf), fmt, ap);
+    (void)flux_plugin_arg_pack (args,
+                                FLUX_PLUGIN_ARG_OUT,
+                                "{s:s}",
+                                "errmsg", buf);
+}
+
+int upmi_seterror (flux_plugin_t *p,
+                   flux_plugin_arg_t *args,
+                   const char *fmt,
+                   ...)
+{
+    va_list ap;
+
+    va_start (ap, fmt);
+    upmi_setverror (p, args, fmt, ap);
+    va_end (ap);
+    return -1;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libpmi/upmi.h b/src/common/libpmi/upmi.h
new file mode 100644
index 000000000000..5ed65d5b5da3
--- /dev/null
+++ b/src/common/libpmi/upmi.h
@@ -0,0 +1,62 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBPMI_UPMI_H
+#define _LIBPMI_UPMI_H 1
+
+#include <sys/types.h>
+#include <stdarg.h>
+#include <stdbool.h>
+
+#include "src/common/libflux/types.h"
+
+enum {
+    UPMI_TRACE = 1,         // call the trace callback for each operation
+    UPMI_LIBPMI_NOFLUX = 2, // libpmi should fail if Flux libflux.so is found
+    UPMI_LIBPMI2_CRAY = 4,  // force cray libpmi2 workarounds for testing
+};
+
+struct upmi_info {
+    int rank;
+    int size;
+    const char *name;
+};
+
+typedef void (*upmi_trace_f)(void *arg, const char *text);
+
+struct upmi *upmi_create (const char *spec,
+                          int flags,
+                          upmi_trace_f cb,
+                          void *arg,
+                          flux_error_t *error);
+void upmi_destroy (struct upmi *upmi);
+const char *upmi_describe (struct upmi *upmi);
+
+int upmi_initialize (struct upmi *upmi,
+                     struct upmi_info *info,
+                     flux_error_t *error);
+int upmi_finalize (struct upmi *upmi, flux_error_t *error);
+
+int upmi_put (struct upmi *upmi,
+              const char *key,
+              const char *value,
+              flux_error_t *error);
+int upmi_get (struct upmi *upmi,
+              const char *key,
+              int rank, // -1 if target rank is unknown
+              char **value, // caller must free
+              flux_error_t *error);
+int upmi_barrier (struct upmi *upmi,
+                  flux_error_t *error);
+int upmi_abort (struct upmi *upmi, const char *msg, flux_error_t *error);
+
+#endif /* !_LIBPMI_UPMI_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libpmi/upmi_libpmi.c b/src/common/libpmi/upmi_libpmi.c
new file mode 100644
index 000000000000..a699eb501ae3
--- /dev/null
+++ b/src/common/libpmi/upmi_libpmi.c
@@ -0,0 +1,355 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <errno.h>
+#include <dlfcn.h>
+#if HAVE_LINK_H
+#include <link.h>
+#endif
+
+#include "src/common/libutil/errprintf.h"
+#include "ccan/array_size/array_size.h"
+
+#include "simple_client.h"
+#include "pmi_strerror.h"
+#include "pmi.h"
+#include "upmi.h"
+#include "upmi_plugin.h"
+
+struct plugin_ctx {
+    void *dso;
+    int (*init) (int *spawned);
+    int (*finalize) (void);
+    int (*abort) (int exit_code, const char *error_msg);
+    int (*get_size) (int *size);
+    int (*get_rank) (int *rank);
+    int (*barrier) (void);
+    int (*kvs_get_my_name) (char *kvsname, int length);
+    int (*kvs_put) (const char *kvsname, const char *key, const char *value);
+    int (*kvs_commit) (const char *kvsname);
+    int (*kvs_get) (const char *kvsname, const char *key, char *value, int len);
+    int rank;
+    int size;
+    char kvsname[1024];
+};
+
+static const char *plugin_name = "libpmi";
+
+static const char *dlinfo_name (void *dso)
+{
+#if HAVE_LINK_H
+    struct link_map *p = NULL;
+    (void)dlinfo (dso, RTLD_DI_LINKMAP, &p);
+    if (p && p->l_name)
+        return p->l_name;
+#endif
+    return "unknown";
+}
+
+static int dlopen_wrap (const char *path,
+                        int flags,
+                        bool noflux,
+                        void **result,
+                        flux_error_t *error)
+{
+    void *dso;
+
+    dlerror ();
+    if (!(dso = dlopen (path, flags))) {
+        char *errstr = dlerror ();
+        if (errstr)
+            errprintf (error, "%s", errstr); // dlerror() already includes path
+        else
+            errprintf (error, "%s: dlopen failed", path);
+        return -1;
+    }
+    if (noflux) {
+        if (dlsym (dso, "flux_pmi_library") != NULL) {
+            errprintf (error,
+                       "%s: dlopen found Flux library (%s)",
+                       path,
+                       dlinfo_name (dso));
+            goto error;
+        }
+    }
+    *result = dso;
+    return 0;
+error:
+    dlclose (dso);
+    return -1;
+}
+
+static void plugin_ctx_destroy (struct plugin_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        if (ctx->dso)
+            dlclose (ctx->dso);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static struct plugin_ctx *plugin_ctx_create (const char *path,
+                                             bool noflux,
+                                             flux_error_t *error)
+{
+    struct plugin_ctx *ctx;
+
+    if (!(ctx = calloc (1, sizeof (*ctx)))) {
+        errprintf (error, "out of memory");
+        return NULL;
+    }
+    if (!path)
+        path = "libpmi.so";
+    // Use RTLD_GLOBAL due to flux-framework/flux-core#432
+    if (dlopen_wrap (path,
+                     RTLD_NOW | RTLD_GLOBAL,
+                     noflux,
+                     &ctx->dso,
+                     error) < 0)
+        goto error;
+    if (!(ctx->init = dlsym (ctx->dso, "PMI_Init"))
+        || !(ctx->finalize = dlsym (ctx->dso, "PMI_Finalize"))
+        || !(ctx->get_size = dlsym (ctx->dso, "PMI_Get_size"))
+        || !(ctx->get_rank = dlsym (ctx->dso, "PMI_Get_rank"))
+        || !(ctx->barrier = dlsym (ctx->dso, "PMI_Barrier"))
+        || !(ctx->kvs_get_my_name = dlsym (ctx->dso, "PMI_KVS_Get_my_name"))
+        || !(ctx->kvs_put = dlsym (ctx->dso, "PMI_KVS_Put"))
+        || !(ctx->kvs_commit = dlsym (ctx->dso, "PMI_KVS_Commit"))
+        || !(ctx->kvs_get = dlsym (ctx->dso, "PMI_KVS_Get"))
+        || !(ctx->abort = dlsym (ctx->dso, "PMI_Abort"))) {
+        errprintf (error, "%s:  missing required PMI_* symbols", path);
+        goto error;
+    }
+
+    /* Cray's libpmi requires workarounds implemented in the libpmi2 plugin.
+     * We shouldn't land here but if we do, fail early.
+     * See flux-framework/flux-core#504
+     */
+    if (dlsym (ctx->dso, "PMI_CRAY_Get_app_size") != NULL) {
+        errprintf (error, "refusing to use quirky cray libpmi.so");
+        goto error;
+    }
+
+    return ctx;
+error:
+    if (ctx->dso)
+        dlclose (ctx->dso);
+    free (ctx);
+    return NULL;
+}
+
+static int op_put (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    const char *key;
+    const char *value;
+    int result;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s s:s}",
+                                "key", &key,
+                                "value", &value) < 0)
+        return upmi_seterror (p, args, "error unpacking put arguments");
+
+    result = ctx->kvs_put (ctx->kvsname, key, value);
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    return 0;
+}
+
+static int op_get (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    int result;
+    const char *key;
+    char value[1024];
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s}",
+                                "key", &key) < 0)
+        return upmi_seterror (p, args, "error unpacking get arguments");
+
+    result = ctx->kvs_get (ctx->kvsname, key, value, sizeof (value));
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:s}",
+                              "value", value) < 0)
+        return -1;
+    return 0;
+}
+
+static int op_barrier (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    int result;
+
+    result = ctx->kvs_commit (ctx->kvsname);
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    result = ctx->barrier ();
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    return 0;
+}
+
+static int op_abort (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    int result;
+    const char *msg;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s}",
+                                "msg", &msg) < 0)
+        return upmi_seterror (p, args, "error unpacking abort arguments");
+    result = ctx->abort (1, msg);
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+    return 0;
+}
+
+static int op_initialize (flux_plugin_t *p,
+                          const char *topic,
+                          flux_plugin_arg_t *args,
+                          void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:i s:s s:i}",
+                              "rank", ctx->rank,
+                              "name", ctx->kvsname,
+                              "size", ctx->size) < 0)
+        return -1;
+    return 0;
+}
+
+
+static int op_finalize (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+
+    int result;
+
+    result = ctx->finalize ();
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    return 0;
+}
+
+static int op_preinit (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    struct plugin_ctx *ctx;
+    flux_error_t error;
+    const char *path = NULL;
+    int noflux = 0;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s?s s?b}",
+                                "path", &path,
+                                "noflux", &noflux) < 0)
+        return upmi_seterror (p, args, "error unpacking preinit arguments");
+    if (!(ctx = plugin_ctx_create (path, noflux, &error)))
+        return upmi_seterror (p, args, "%s", error.text);
+    if (flux_plugin_aux_set (p,
+                             plugin_name,
+                             ctx,
+                             (flux_free_f)plugin_ctx_destroy) < 0) {
+        plugin_ctx_destroy (ctx);
+        return upmi_seterror (p, args, "%s", strerror (errno));
+    }
+
+    /* Call PMI_Init() and basic info functions now so that upmi can fall
+     * through to the next plugin on failure.
+     */
+    int result;
+    int spawned;
+    const char *name = dlinfo_name (ctx->dso);
+
+    result = ctx->init (&spawned);
+    if (result != PMI_SUCCESS)
+        goto error;
+    result = ctx->kvs_get_my_name (ctx->kvsname, sizeof (ctx->kvsname));
+    if (result != PMI_SUCCESS)
+        goto error;
+    result = ctx->get_rank (&ctx->rank);
+    if (result != PMI_SUCCESS)
+        goto error;
+    result = ctx->get_size (&ctx->size);
+    if (result != PMI_SUCCESS)
+        goto error;
+
+    char note[1024];
+    snprintf (note, sizeof (note), "using %s", name);
+    flux_plugin_arg_pack (args,
+                          FLUX_PLUGIN_ARG_OUT,
+                          "{s:s}",
+                          "note", note);
+    return 0;
+error:
+    return upmi_seterror (p, args, "%s: %s", name, pmi_strerror (result));
+}
+
+static const struct flux_plugin_handler optab[] = {
+    { "upmi.put",           op_put,         NULL },
+    { "upmi.get",           op_get,         NULL },
+    { "upmi.barrier",       op_barrier,     NULL },
+    { "upmi.abort",         op_abort,       NULL },
+    { "upmi.initialize",    op_initialize,  NULL },
+    { "upmi.finalize",      op_finalize,    NULL },
+    { "upmi.preinit",       op_preinit,     NULL },
+    { 0 },
+};
+
+int upmi_libpmi_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, plugin_name, optab) < 0)
+        return -1;
+    return 0;
+}
+
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libpmi/upmi_libpmi2.c b/src/common/libpmi/upmi_libpmi2.c
new file mode 100644
index 000000000000..81502a9956ee
--- /dev/null
+++ b/src/common/libpmi/upmi_libpmi2.c
@@ -0,0 +1,466 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <errno.h>
+#include <dlfcn.h>
+#if HAVE_LINK_H
+#include <link.h>
+#endif
+
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "ccan/base64/base64.h"
+
+#include "simple_client.h"
+#include "pmi_strerror.h"
+#include "pmi2.h"
+#include "upmi.h"
+#include "upmi_plugin.h"
+
+#define LIBPMI2_IS_CRAY_CRAY 1
+
+struct plugin_ctx {
+    void *dso;
+    int (*init) (int *spawned, int *size, int *rank, int *appnum);
+    int (*finalize) (void);
+    int (*abort) (int flag, const char *msg);
+    int (*job_getid) (char *jobid, int jobid_size);
+    int (*kvs_put) (const char *key, const char *value);
+    int (*kvs_fence) (void);
+    int (*kvs_get) (const char *jobid,
+                    int src_pmi_id,
+                    const char *key,
+                    char *value,
+                    int maxvalue,
+                    int *vallen);
+    int (*getjobattr) (const char *name,
+                       const char *value,
+                       int valuelen,
+                       int *found);
+    int flags;
+    int size;
+    int rank;
+    char jobid[256];
+};
+
+static const char *plugin_name = "libpmi2";
+
+static const char *dlinfo_name (void *dso)
+{
+#if HAVE_LINK_H
+    struct link_map *p = NULL;
+    (void)dlinfo (dso, RTLD_DI_LINKMAP, &p);
+    if (p && p->l_name)
+        return p->l_name;
+#endif
+    return "unknown";
+}
+
+static int dlopen_wrap (const char *path,
+                        int flags,
+                        bool noflux,
+                        void **result,
+                        flux_error_t *error)
+{
+    void *dso;
+
+    dlerror ();
+    if (!(dso = dlopen (path, flags))) {
+        char *errstr = dlerror ();
+        if (errstr)
+            errprintf (error, "%s", errstr); // dlerror() already includes path
+        else
+            errprintf (error, "%s: dlopen failed", path);
+        return -1;
+    }
+    if (noflux) {
+        if (dlsym (dso, "flux_pmi_library") != NULL) {
+            errprintf (error,
+                       "%s: dlopen found Flux library (%s)",
+                       path,
+                       dlinfo_name (dso));
+            goto error;
+        }
+    }
+    *result = dso;
+    return 0;
+error:
+    dlclose (dso);
+    return -1;
+}
+
+static void plugin_ctx_destroy (struct plugin_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        if (ctx->dso)
+            dlclose (ctx->dso);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static struct plugin_ctx *plugin_ctx_create (const char *path,
+                                             bool noflux,
+                                             bool craycray,
+                                             flux_error_t *error)
+{
+    struct plugin_ctx *ctx;
+
+    if (!(ctx = calloc (1, sizeof (*ctx)))) {
+        errprintf (error, "out of memory");
+        return NULL;
+    }
+    if (!path)
+        path = "libpmi2.so";
+    // Use RTLD_GLOBAL due to flux-framework/flux-core#432
+    if (dlopen_wrap (path,
+                     RTLD_NOW | RTLD_GLOBAL,
+                     noflux,
+                     &ctx->dso,
+                     error) < 0)
+        goto error;
+    if (!(ctx->init = dlsym (ctx->dso, "PMI2_Init"))
+        || !(ctx->finalize = dlsym (ctx->dso, "PMI2_Finalize"))
+        || !(ctx->abort = dlsym (ctx->dso, "PMI2_Abort"))
+        || !(ctx->job_getid = dlsym (ctx->dso, "PMI2_Job_GetId"))
+        || !(ctx->kvs_put = dlsym (ctx->dso, "PMI2_KVS_Put"))
+        || !(ctx->kvs_fence = dlsym (ctx->dso, "PMI2_KVS_Fence"))
+        || !(ctx->kvs_get = dlsym (ctx->dso, "PMI2_KVS_Get"))
+        || !(ctx->getjobattr = dlsym (ctx->dso, "PMI2_Info_GetJobAttr"))) {
+        errprintf (error, "%s:  missing required PMI2_* symbols", path);
+        goto error;
+    }
+
+    if (dlsym (ctx->dso, "PMI_CRAY_Get_app_size") != NULL || craycray)
+        ctx->flags |= LIBPMI2_IS_CRAY_CRAY;
+
+    return ctx;
+error:
+    if (ctx->dso)
+        dlclose (ctx->dso);
+    free (ctx);
+    return NULL;
+}
+
+static void charsub (char *s, char c1, char c2)
+{
+    for (int i = 0; i < strlen (s); i++)
+        if (s[i] == c1)
+            s[i] = c2;
+}
+
+/* Base64 encode 's', then translate = to a character not in RFC 4648 base64
+ * alphabet.  Caller must free result.
+ */
+static char *encode_cray_value (const char *s)
+{
+    size_t len = strlen (s);
+    size_t bufsize = base64_encoded_length (len) + 1; // +1 for \0 termination
+    char *buf;
+
+    if (!(buf = malloc (bufsize)))
+        return NULL;
+    if (base64_encode (buf, bufsize, s, len) < 0) {
+        free (buf);
+        errno = EINVAL;
+        return NULL;
+    }
+    charsub (buf, '=', '*');
+    return buf;
+}
+
+/* Reverse the result of encode_cray_value().  Caller must free result.
+ */
+static char *decode_cray_value (const char *s)
+{
+    char *cpy;
+
+    if (!(cpy = strdup (s)))
+        return NULL;
+    charsub (cpy, '*', '=');
+
+    int len = strlen (cpy);
+    size_t bufsize = base64_decoded_length (len) + 1; // +1 for \0 termination
+    void *buf;
+
+    if (!(buf = malloc (bufsize)))
+        goto error;
+    if (base64_decode (buf, bufsize, cpy, len) < 0) {
+        errno = EINVAL;
+        goto error;
+    }
+    free (cpy);
+    return buf;
+error:
+    ERRNO_SAFE_WRAP (free, cpy);
+    ERRNO_SAFE_WRAP (free, buf);
+    return NULL;
+}
+
+static int op_put (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    const char *key;
+    const char *value;
+    char *xvalue = NULL;
+    int result;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s s:s}",
+                                "key", &key,
+                                "value", &value) < 0)
+        return upmi_seterror (p, args, "error unpacking put arguments");
+    /* Workaround for flux-framework/flux-core#5040.
+     * Cray's libpmi2.so doesn't like ; and = characters in KVS values.
+     */
+    if ((ctx->flags & LIBPMI2_IS_CRAY_CRAY)) {
+        if (!(xvalue = encode_cray_value (value)))
+            return upmi_seterror (p, args, "%s", strerror (errno));
+    }
+
+    result = ctx->kvs_put (key, xvalue ? xvalue : value);
+    if (result != PMI2_SUCCESS) {
+        free (xvalue);
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+    }
+    free (xvalue);
+    return 0;
+}
+
+static int op_get (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    int result;
+    const char *key;
+    char value[1024];
+    int vallen = 0; // ignored
+    char *xvalue = NULL;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s}",
+                                "key", &key) < 0)
+        return upmi_seterror (p, args, "error unpacking get arguments");
+
+    /* Divert PMI_process_mapping to a job attribute request.
+     */
+    if (streq (key, "PMI_process_mapping")) {
+        int found = 0;
+        result = ctx->getjobattr (key, value, sizeof (value), &found);
+        if (result == PMI2_SUCCESS && !found)
+            result = PMI2_ERR_INVALID_KEY;
+    }
+    /* Workaround for flux-framework/flux-core#5040.
+     * Cray's libpmi2.so prints to stderr when asked for a missing key.
+     * To avoid that unpleasantness, short circuit the request for "flux."
+     * prefixed keys, on the assumption that Cray's libpmi2.so won't ever be
+     * employed by Flux to launch Flux.
+     */
+    else if ((ctx->flags & LIBPMI2_IS_CRAY_CRAY)
+        && (strstarts (key, "flux.")))
+        result = PMI2_ERR_INVALID_KEY;
+    else {
+        result = ctx->kvs_get (NULL,            // this job's key-value space
+                               PMI2_ID_NULL,    // no hints for you
+                               key,
+                               value,
+                               sizeof (value),
+                               &vallen);
+        if (result == PMI2_SUCCESS) {
+            /* Workaround for flux-framework/flux-core#5040.
+             * Cray's libpmi2.so doesn't like ; and = characters in KVS values.
+             */
+            if ((ctx->flags & LIBPMI2_IS_CRAY_CRAY)) {
+                if (!(xvalue = decode_cray_value (value)))
+                    return upmi_seterror (p, args, "%s", strerror (errno));
+            }
+        }
+    }
+    if (result != PMI2_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:s}",
+                              "value", xvalue ? xvalue : value) < 0) {
+        free (xvalue);
+        return -1;
+    }
+    free (xvalue);
+    return 0;
+}
+
+static int op_barrier (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    int result;
+
+    result = ctx->kvs_fence ();
+    if (result != PMI2_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    return 0;
+}
+
+static int op_abort (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *data)
+
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    int flag = 1; // abort all processes in the job
+    const char *msg;
+    int result;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s}",
+                                "msg", &msg) < 0)
+        return upmi_seterror (p, args, "error unpacking abort arguments");
+    result = ctx->abort (flag, msg);
+    if (result != PMI2_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+    return 0;
+}
+
+static int op_initialize (flux_plugin_t *p,
+                          const char *topic,
+                          flux_plugin_arg_t *args,
+                          void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:i s:s s:i}",
+                              "rank", ctx->rank,
+                              "name", ctx->jobid,
+                              "size", ctx->size) < 0)
+        return -1;
+    return 0;
+}
+
+
+static int op_finalize (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+
+    int result;
+
+    result = ctx->finalize ();
+    if (result != PMI2_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    return 0;
+}
+
+static int op_preinit (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    struct plugin_ctx *ctx;
+    flux_error_t error;
+    const char *path = NULL;
+    int noflux = 0;
+    int craycray = 0;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s?s s?b s?b}",
+                                "path", &path,
+                                "noflux", &noflux,
+                                "craycray", &craycray) < 0)
+        return upmi_seterror (p, args, "error unpacking preinit arguments");
+    if (!(ctx = plugin_ctx_create (path, noflux, craycray, &error)))
+        return upmi_seterror (p, args, "%s", error.text);
+    if (flux_plugin_aux_set (p,
+                             plugin_name,
+                             ctx,
+                             (flux_free_f)plugin_ctx_destroy) < 0) {
+        plugin_ctx_destroy (ctx);
+        return upmi_seterror (p, args, "%s", strerror (errno));
+    }
+
+    /* Call PMI2_Init() and PMI2_Job_Getid() now so that upmi can fall
+     * through to the next plugin on failure.
+     */
+    int spawned;
+    int appnum;
+    int result;
+    const char *name = dlinfo_name (ctx->dso);
+
+    result = ctx->init (&spawned, &ctx->size, &ctx->rank, &appnum);
+    if (result != PMI2_SUCCESS)
+        goto error;
+    /* N.B. slurm's libpmi2 succeeds in PMI2_Init() but fails here
+     * outside of a slurm job.  See: flux-framework/flux-core#5057
+     */
+    result = ctx->job_getid (ctx->jobid, sizeof (ctx->jobid));
+    if (result != PMI2_SUCCESS)
+        goto error;
+
+    char note[1024];
+    snprintf (note,
+              sizeof (note),
+              "using %s%s",
+              name,
+              (ctx->flags & LIBPMI2_IS_CRAY_CRAY)
+                  ? " (cray quirks enabled)" : "");
+    flux_plugin_arg_pack (args,
+                          FLUX_PLUGIN_ARG_OUT,
+                          "{s:s}",
+                          "note", note);
+    return 0;
+error:
+    return upmi_seterror (p, args, "%s: %s", name, pmi_strerror (result));
+}
+
+static const struct flux_plugin_handler optab[] = {
+    { "upmi.put",           op_put,         NULL },
+    { "upmi.get",           op_get,         NULL },
+    { "upmi.barrier",       op_barrier,     NULL },
+    { "upmi.abort",         op_abort,       NULL },
+    { "upmi.initialize",    op_initialize,  NULL },
+    { "upmi.finalize",      op_finalize,    NULL },
+    { "upmi.preinit",       op_preinit,     NULL },
+    { 0 },
+};
+
+int upmi_libpmi2_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, plugin_name, optab) < 0)
+        return -1;
+    return 0;
+}
+
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libpmi/upmi_plugin.h b/src/common/libpmi/upmi_plugin.h
new file mode 100644
index 000000000000..b83edddcaa42
--- /dev/null
+++ b/src/common/libpmi/upmi_plugin.h
@@ -0,0 +1,26 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBPMI_UPMI_PLUGIN_H
+#define _LIBPMI_UPMI_PLUGIN_H 1
+
+#include <stdarg.h>
+#include <flux/core.h>
+
+int upmi_seterror (flux_plugin_t *p,
+                   flux_plugin_arg_t *args,
+                   const char *fmt,
+                   ...)
+__attribute__ ((format (printf, 3, 4)));
+
+
+#endif /* !_LIBPMI_UPMI_PLUGIN_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libpmi/upmi_simple.c b/src/common/libpmi/upmi_simple.c
new file mode 100644
index 000000000000..82be7f2e5e11
--- /dev/null
+++ b/src/common/libpmi/upmi_simple.c
@@ -0,0 +1,234 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <errno.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "ccan/array_size/array_size.h"
+
+#include "simple_client.h"
+#include "pmi_strerror.h"
+#include "pmi.h"
+#include "upmi.h"
+#include "upmi_plugin.h"
+
+struct plugin_ctx {
+    struct pmi_simple_client *client;
+    char kvsname[1024];
+};
+
+static const char *plugin_name = "simple";
+
+static void plugin_ctx_destroy (struct plugin_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        pmi_simple_client_destroy (ctx->client);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static struct plugin_ctx *plugin_ctx_create (void)
+{
+    struct plugin_ctx *ctx;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+    if (!(ctx->client = pmi_simple_client_create_fd (getenv ("PMI_FD"),
+                                                     getenv ("PMI_RANK"),
+                                                     getenv ("PMI_SIZE"),
+                                                     getenv ("PMI_SPAWNED")))) {
+        plugin_ctx_destroy (ctx);
+        return NULL;
+    }
+    return ctx;
+}
+
+static int op_put (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    const char *key;
+    const char *value;
+    int result;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s s:s}",
+                                "key", &key,
+                                "value", &value) < 0)
+        return upmi_seterror (p, args, "error unpacking put arguments");
+
+    result = pmi_simple_client_kvs_put (ctx->client, ctx->kvsname, key, value);
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    return 0;
+}
+
+static int op_get (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    int result;
+    const char *key;
+    char value[1024];
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s}",
+                                "key", &key) < 0)
+        return upmi_seterror (p, args, "error unpacking get arguments");
+
+    result = pmi_simple_client_kvs_get (ctx->client,
+                                        ctx->kvsname,
+                                        key,
+                                        value,
+                                        sizeof (value));
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:s}",
+                              "value", value) < 0)
+        return -1;
+    return 0;
+}
+
+static int op_barrier (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    int result;
+
+    result = pmi_simple_client_barrier (ctx->client);
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+    return 0;
+}
+
+static int op_abort (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    const char *msg;
+    int result;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s}",
+                                "msg", &msg) < 0)
+        return upmi_seterror (p, args, "error unpacking abort arguments");
+    result = pmi_simple_client_abort (ctx->client, 1, msg);
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+    return 0;
+}
+
+static int op_initialize (flux_plugin_t *p,
+                          const char *topic,
+                          flux_plugin_arg_t *args,
+                          void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    int result;
+
+    result = pmi_simple_client_init (ctx->client);
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    result = pmi_simple_client_kvs_get_my_name (ctx->client,
+                                                ctx->kvsname,
+                                                sizeof (ctx->kvsname));
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:i s:s s:i}",
+                              "rank", ctx->client->rank,
+                              "name", ctx->kvsname,
+                              "size", ctx->client->size) < 0)
+        return -1;
+    return 0;
+}
+
+static int op_finalize (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    int result;
+
+    result = pmi_simple_client_finalize (ctx->client);
+    if (result != PMI_SUCCESS)
+        return upmi_seterror (p, args, "%s", pmi_strerror (result));
+    return 0;
+}
+
+static int op_preinit (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    struct plugin_ctx *ctx;
+    const char *vars[] = { "PMI_FD", "PMI_RANK", "PMI_SIZE" }; // required
+
+    for (int i = 0; i < ARRAY_SIZE (vars); i++) {
+        if (!getenv (vars[i]))
+            return upmi_seterror (p, args, "%s not found in environ", vars[i]);
+    }
+    if (!(ctx = plugin_ctx_create ())
+        || flux_plugin_aux_set (p,
+                                plugin_name,
+                                ctx,
+                                (flux_free_f)plugin_ctx_destroy) < 0) {
+        plugin_ctx_destroy (ctx);
+        return upmi_seterror (p, args, "create context: %s", strerror (errno));
+    }
+    return 0;
+}
+
+static const struct flux_plugin_handler optab[] = {
+    { "upmi.put",           op_put,         NULL },
+    { "upmi.get",           op_get,         NULL },
+    { "upmi.barrier",       op_barrier,     NULL },
+    { "upmi.abort",         op_abort,       NULL },
+    { "upmi.initialize",    op_initialize,  NULL },
+    { "upmi.finalize",      op_finalize,    NULL },
+    { "upmi.preinit",       op_preinit,     NULL },
+    { 0 },
+};
+
+
+
+int upmi_simple_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, plugin_name, optab) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libpmi/upmi_single.c b/src/common/libpmi/upmi_single.c
new file mode 100644
index 000000000000..58a3e63c91f1
--- /dev/null
+++ b/src/common/libpmi/upmi_single.c
@@ -0,0 +1,183 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <errno.h>
+#include <jansson.h>
+
+#include "upmi.h"
+#include "upmi_plugin.h"
+
+struct plugin_ctx {
+    json_t *kvs;
+};
+
+static const char *plugin_name = "single";
+
+static void *plugin_ctx_create (void)
+{
+    struct plugin_ctx *ctx;
+
+    if (!(ctx = calloc (1, sizeof (*ctx)))
+        || !(ctx->kvs = json_object ())) {
+        return NULL;
+    }
+    return ctx;
+}
+
+static void plugin_ctx_destroy (struct plugin_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        json_decref (ctx->kvs);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static int op_put (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    const char *key;
+    const char *value;
+    json_t *o;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s s:s}",
+                                "key", &key,
+                                "value", &value) < 0)
+        return upmi_seterror (p, args, "error unpacking put arguments");
+    if (!(o = json_string (value))
+        || json_object_set_new (ctx->kvs, key, o) < 0) {
+        json_decref (o);
+        return upmi_seterror (p, args, "dictionary update error");
+    }
+    return 0;
+}
+
+static int op_get (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *data)
+{
+    struct plugin_ctx *ctx = flux_plugin_aux_get (p, plugin_name);
+    const char *key;
+    const char *value;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s}",
+                                "key", &key) < 0)
+        return upmi_seterror (p, args, "error unpacking get arguments");
+
+    if (json_unpack (ctx->kvs, "{s:s}", key, &value) < 0)
+        return upmi_seterror (p, args, "key not found");
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:s}",
+                              "value", value) < 0)
+        return -1;
+    return 0;
+}
+
+static int op_barrier (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    return 0;
+}
+
+static int op_abort (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *data)
+{
+    const char *msg;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s}",
+                                "msg", &msg) < 0)
+        return upmi_seterror (p, args, "error unpacking abort arguments");
+    fprintf (stderr, "%s\n", msg);
+    exit (1);
+    //NOTREACHED
+    return 0;
+}
+
+static int op_initialize (flux_plugin_t *p,
+                          const char *topic,
+                          flux_plugin_arg_t *args,
+                          void *data)
+{
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:i s:s s:i}",
+                              "rank", 0,
+                              "name", plugin_name,
+                              "size", 1) < 0)
+        return -1;
+    return 0;
+}
+
+static int op_finalize (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    return 0;
+}
+
+static int op_preinit (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    struct plugin_ctx *ctx;
+
+    if (!(ctx = plugin_ctx_create ())
+        || flux_plugin_aux_set (p,
+                                plugin_name,
+                                ctx,
+                                (flux_free_f)plugin_ctx_destroy) < 0) {
+        plugin_ctx_destroy (ctx);
+        return upmi_seterror (p, args, "could not create upmi plugin context");
+    }
+    return 0;
+}
+
+static const struct flux_plugin_handler optab[] = {
+    { "upmi.put",           op_put,         NULL },
+    { "upmi.get",           op_get,         NULL },
+    { "upmi.barrier",       op_barrier,     NULL },
+    { "upmi.abort",         op_abort,       NULL },
+    { "upmi.initialize",    op_initialize,  NULL },
+    { "upmi.finalize",      op_finalize,    NULL },
+    { "upmi.preinit",       op_preinit,     NULL },
+    { 0 },
+};
+
+int upmi_single_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, plugin_name, optab) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libpmi2.map b/src/common/libpmi2.map
deleted file mode 100644
index e7bb5a8cc9ad..000000000000
--- a/src/common/libpmi2.map
+++ /dev/null
@@ -1,7 +0,0 @@
-{ global:
-    PMI2_*;
-    flux_pmi_library;
-    __asan*;
-    local: *;
-};
-
diff --git a/src/common/librlist/Makefile.am b/src/common/librlist/Makefile.am
new file mode 100644
index 000000000000..9703294ff540
--- /dev/null
+++ b/src/common/librlist/Makefile.am
@@ -0,0 +1,95 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux \
+	$(JANSSON_CFLAGS) \
+	$(HWLOC_CFLAGS)
+
+noinst_LTLIBRARIES = \
+	librlist.la \
+	librlist-hwloc.la
+
+librlist_la_SOURCES = \
+	rnode.h \
+	rnode.c \
+	match.h \
+	match.c \
+	rlist.c \
+	rlist.h \
+	rlist_private.h
+
+librlist_hwloc_la_SOURCES = \
+	rhwloc.c \
+	rhwloc.h
+
+test_ldadd = \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(LIBPTHREAD) \
+	$(JANSSON_LIBS)
+
+test_cppflags = \
+	$(AM_CPPFLAGS)
+
+test_ldflags = \
+	-no-install
+
+TESTS = \
+	test_rnode.t \
+	test_match.t \
+	test_rlist.t \
+	test_rhwloc.t
+
+check_PROGRAMS = \
+	$(TESTS)
+
+test_rnode_t_SOURCES = \
+	test/rnode.c
+test_rnode_t_CPPFLAGS = \
+	$(test_cppflags)
+test_rnode_t_LDADD = \
+	librlist.la \
+	$(test_ldadd)
+test_rnode_t_LDFLAGS = \
+	$(test_ldflags)
+
+test_match_t_SOURCES = \
+	test/match.c
+test_match_t_CPPFLAGS = \
+	$(test_cppflags)
+test_match_t_LDADD = \
+	librlist.la \
+	$(test_ldadd)
+test_match_t_LDFLAGS = \
+	$(test_ldflags)
+
+test_rlist_t_SOURCES = \
+	test/rlist.c
+test_rlist_t_CPPFLAGS = \
+	$(test_cppflags)
+test_rlist_t_LDADD = \
+	librlist.la \
+	$(test_ldadd)
+test_rlist_t_LDFLAGS = \
+	$(test_ldflags)
+
+test_rhwloc_t_SOURCES = \
+	test/rhwloc.c
+test_rhwloc_t_CPPFLAGS = \
+	$(test_cppflags)
+test_rhwloc_t_LDADD = \
+	librlist.la \
+	librlist-hwloc.la \
+	$(test_ldadd) \
+	$(HWLOC_LIBS)
+test_rhwloc_t_LDFLAGS = \
+	$(test_ldflags)
diff --git a/src/common/librlist/match.c b/src/common/librlist/match.c
new file mode 100644
index 000000000000..6895c1b34254
--- /dev/null
+++ b/src/common/librlist/match.c
@@ -0,0 +1,424 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <jansson.h>
+#include <flux/hostlist.h>
+#include <flux/idset.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "match.h"
+
+typedef bool (*match_f) (struct job_constraint *, const struct rnode *);
+
+struct job_constraint {
+    zlistx_t *values;
+    match_f match;
+};
+
+static bool match_empty (struct job_constraint *c, const struct rnode *n)
+{
+    return true;
+}
+
+static struct job_constraint *job_constraint_new (flux_error_t *errp)
+{
+    struct job_constraint *c;
+    if (!(c = calloc (1, sizeof (*c)))
+        || !(c->values = zlistx_new ())) {
+        job_constraint_destroy (c);
+        errprintf (errp, "Out of memory");
+        return NULL;
+    }
+    c->match = match_empty;
+    return c;
+}
+
+static int add_idset_string (struct idset *idset, const char *s)
+{
+    int rc;
+    struct idset *ids;
+
+    if (!(ids = idset_decode (s)))
+        return -1;
+    rc = idset_add (idset, ids);
+    idset_destroy (ids);
+    return rc;
+}
+
+static struct idset *array_to_idset (json_t *idsets,
+                                     flux_error_t *errp)
+{
+    json_t *entry;
+    size_t index;
+    struct idset *idset = idset_create (0, IDSET_FLAG_AUTOGROW);
+
+    if (!idset)
+        return NULL;
+
+    json_array_foreach (idsets, index, entry) {
+        if (add_idset_string (idset, json_string_value (entry)) < 0) {
+            char *s = json_dumps (idsets, 0);
+            errprintf (errp,
+                       "invalid idset '%s' in %s",
+                       json_string_value (entry),
+                       s);
+            free (s);
+            goto error;
+        }
+    }
+    return idset;
+error:
+    idset_destroy (idset);
+    return NULL;
+}
+
+static void destruct_idset (void **item)
+{
+    if (item) {
+        idset_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static bool match_idset (struct job_constraint *c,
+                         const struct rnode *n)
+{
+    struct idset *idset = zlistx_first (c->values);
+    return idset_test (idset, n->rank);
+}
+
+static struct job_constraint *create_idset_constraint (json_t *values,
+                                                       flux_error_t *errp)
+{
+    struct job_constraint *c;
+    struct idset *idset = array_to_idset (values, errp);
+    if (!idset)
+        return NULL;
+    if (!(c = job_constraint_new (errp))
+        || !zlistx_add_end (c->values, idset)) {
+        job_constraint_destroy (c);
+        idset_destroy (idset);
+        return NULL;
+    }
+    zlistx_set_destructor (c->values, destruct_idset);
+    c->match = match_idset;
+    return c;
+}
+
+static struct hostlist *array_to_hostlist (json_t *hostlists,
+                                           flux_error_t *errp)
+{
+    json_t *entry;
+    size_t index;
+    struct hostlist *hl = hostlist_create ();
+
+    if (!hl)
+        return NULL;
+
+    json_array_foreach (hostlists, index, entry) {
+        if (hostlist_append (hl, json_string_value (entry)) < 0) {
+            char *s = json_dumps (hostlists, 0);
+            errprintf (errp,
+                       "invalid hostlist '%s' in %s",
+                       json_string_value (entry),
+                       s);
+            free (s);
+            goto error;
+        }
+    }
+    return hl;
+error:
+    hostlist_destroy (hl);
+    return NULL;
+}
+
+static void destruct_hostlist (void **item)
+{
+    if (item) {
+        hostlist_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static bool match_hostlist (struct job_constraint *c,
+                            const struct rnode *n)
+{
+    struct hostlist *hl = zlistx_first (c->values);
+    if (!hl || hostlist_find (hl, n->hostname) < 0)
+        return false;
+    return true;
+}
+
+static struct job_constraint *create_hostlist_constraint (json_t *values,
+                                                          flux_error_t *errp)
+{
+    struct job_constraint *c;
+    struct hostlist *hl = array_to_hostlist (values, errp);
+    if (!hl)
+        return NULL;
+    if (!(c = job_constraint_new (errp))
+        || !zlistx_add_end (c->values, hl)) {
+        job_constraint_destroy (c);
+        hostlist_destroy (hl);
+        return NULL;
+    }
+    zlistx_set_destructor (c->values, destruct_hostlist);
+    c->match = match_hostlist;
+    return c;
+}
+
+static bool rnode_has (const struct rnode *n, const char *property)
+{
+    const char *prop = property;
+    bool match = false;
+    bool negate = false;
+
+    if (!property)
+        return false;
+
+    if (prop[0] == '^') {
+        prop++;
+        negate = true;
+    }
+
+    if ((n->properties && zhashx_lookup (n->properties, prop))
+        || streq (n->hostname, prop))
+        match = true;
+
+    return negate ? !match : match;
+}
+
+static bool match_properties (struct job_constraint *c, const struct rnode *n)
+{
+    const char *property = zlistx_first (c->values);
+    while (property) {
+        if (!rnode_has (n, property))
+            return false;
+        property = zlistx_next (c->values);
+    }
+    return true;
+}
+
+bool job_constraint_match (struct job_constraint *c, const struct rnode *n)
+{
+    return c->match (c, n);
+}
+
+static char *property_query_string_invalid (const char *s)
+{
+    /*  Return first invalid character.
+     *  Invalid chaaracters are listed in RFC 20, but we specifically
+     *   allow "^" since it is used as shorthand for `not`.
+     */
+    return strpbrk (s, "!&'\"`|()");
+}
+
+static void free_item (void **item)
+{
+    if (item) {
+        free (*item);
+        *item = NULL;
+    }
+}
+
+static struct job_constraint *property_constraint (json_t *values,
+                                                   flux_error_t *errp)
+{
+    struct job_constraint *c;
+    json_t *entry;
+    size_t index;
+
+    if (!json_is_array (values)) {
+        errprintf (errp, "properties value must be an array");
+        return NULL;
+    }
+    if (!(c = job_constraint_new (errp))) {
+        errprintf (errp, "Out of memory");
+        return NULL;
+    }
+    zlistx_set_destructor (c->values, free_item);
+    c->match = match_properties;
+
+    json_array_foreach (values, index, entry) {
+        const char *value;
+        const char *invalid;
+        char *s;
+
+        if (!json_is_string (entry)) {
+            errprintf (errp, "non-string property specified");
+            goto err;
+        }
+        value = json_string_value (entry);
+        if ((invalid = property_query_string_invalid (value))) {
+            errprintf (errp,
+                       "invalid character '%c' in property \"%s\"",
+                       *invalid,
+                       value);
+            goto err;
+        }
+        if (!(s = strdup (value))) {
+            errprintf (errp, "strdup (\"%s\"): out of memory", value);
+            goto err;
+        }
+        if (!zlistx_add_end (c->values, s)) {
+            errprintf (errp, "zlistx_add_end: out of memory");
+            free (s);
+            goto err;
+        }
+    }
+    return c;
+err:
+    job_constraint_destroy (c);
+    return NULL;
+}
+
+static void job_constraint_destructor (void **item)
+{
+    if (item) {
+        job_constraint_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static bool match_and (struct job_constraint *c, const struct rnode *n)
+{
+    struct job_constraint *cp = zlistx_first (c->values);
+    while (cp) {
+        if (!cp->match (cp, n))
+            return false;
+        cp = zlistx_next (c->values);
+    }
+    return true;
+}
+
+static bool match_or (struct job_constraint *c, const struct rnode *n)
+{
+    struct job_constraint *cp = zlistx_first (c->values);
+    if (!cp)
+        return true;
+    while (cp) {
+        if (cp->match (cp, n))
+            return true;
+        cp = zlistx_next (c->values);
+    }
+    return false;
+}
+
+static bool match_not (struct job_constraint *c, const struct rnode *n)
+{
+    return !match_and (c, n);
+}
+
+static struct job_constraint *conditional_constraint (const char *type,
+                                                      json_t *values,
+                                                      flux_error_t *errp)
+{
+    json_t *entry;
+    size_t index;
+    struct job_constraint *c;
+
+    if (!json_is_array (values)) {
+        errprintf (errp, "%s operator value must be an array", type);
+        return NULL;
+    }
+
+    if (!(c = job_constraint_new (errp)))
+        return NULL;
+    if (streq (type, "and"))
+        c->match = match_and;
+    else if (streq (type, "or"))
+        c->match = match_or;
+    else if (streq (type, "not"))
+        c->match = match_not;
+    zlistx_set_destructor (c->values, job_constraint_destructor);
+
+    json_array_foreach (values, index, entry) {
+        struct job_constraint *cp = job_constraint_create (entry, errp);
+        if (!cp || !(zlistx_add_end (c->values, cp))) {
+            errprintf (errp, "Out of memory");
+            job_constraint_destroy (c);
+            job_constraint_destroy (cp);
+            return NULL;
+        }
+    }
+    return c;
+}
+
+void job_constraint_destroy (struct job_constraint *c)
+{
+    if (c) {
+        int saved_errno = errno;
+        zlistx_destroy (&c->values);
+        free (c);
+        errno = saved_errno;
+    }
+}
+
+struct job_constraint *job_constraint_create (json_t *constraint,
+                                              flux_error_t *errp)
+{
+    const char *op;
+    json_t *values;
+
+    if (!constraint || !json_is_object (constraint)) {
+        errprintf (errp, "constraint must be JSON object");
+        return NULL;
+    }
+    if (json_object_size (constraint) > 1) {
+        errprintf (errp, "constraint must only contain 1 element");
+        return NULL;
+    }
+    json_object_foreach (constraint, op, values) {
+        if (streq (op, "properties"))
+            return property_constraint (values, errp);
+        else if (streq (op, "hostlist"))
+            return create_hostlist_constraint (values, errp);
+        else if (streq (op, "ranks"))
+            return create_idset_constraint (values, errp);
+        else if (streq (op, "or") || streq (op, "and") || streq (op, "not"))
+            return conditional_constraint (op, values, errp);
+        else {
+            errprintf (errp, "unknown constraint operator: %s", op);
+            return NULL;
+        }
+    }
+    return job_constraint_new (errp);
+}
+
+bool rnode_match (const struct rnode *n,
+                  struct job_constraint *constraint)
+{
+    if (!n || !constraint)
+        return false;
+    return constraint->match (constraint, n);
+}
+
+struct rnode *rnode_copy_match (const struct rnode *orig,
+                                struct job_constraint *constraint)
+{
+    struct rnode *n = NULL;
+    if (rnode_match (orig, constraint)) {
+        if ((n = rnode_copy (orig)))
+            n->up = orig->up;
+    }
+    return n;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librlist/match.h b/src/common/librlist/match.h
new file mode 100644
index 000000000000..6e85a0f65dc5
--- /dev/null
+++ b/src/common/librlist/match.h
@@ -0,0 +1,41 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_RLIST_MATCH_H
+#define HAVE_RLIST_MATCH_H 1
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h> /* flux_error_t */
+
+#include "rnode.h"
+
+/*  Load and validate RFC 31 constraint spec 'constraint'
+ *
+ *  Returns a job constraint object if constraint is valid spec,
+ *  Returns NULL with error in errp if errp != NULL.
+ */
+struct job_constraint *job_constraint_create (json_t *constraint,
+                                              flux_error_t *errp);
+
+void job_constraint_destroy (struct job_constraint *c);
+
+/*  Return true if rnode 'n' matches constraints in RFC 31 constraint
+ *   specification 'constraint'.
+ */
+bool rnode_match (const struct rnode *n, struct job_constraint *constraint);
+
+/*  Copy an rnode only if it matches the RFC 31 constraints in `constraint` */
+struct rnode *rnode_copy_match (const struct rnode *n,
+                                struct job_constraint *constraint);
+
+#endif /* !HAVE_RLIST_MATCH_H */
diff --git a/src/common/librlist/rhwloc.c b/src/common/librlist/rhwloc.c
new file mode 100644
index 000000000000..d2a991941c62
--- /dev/null
+++ b/src/common/librlist/rhwloc.c
@@ -0,0 +1,349 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/utsname.h>
+
+#include <flux/idset.h>
+
+#include "ccan/str/str.h"
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/errno_safe.h"
+
+#include "rnode.h"
+#include "rlist.h"
+#include "rlist_private.h"
+#include "rhwloc.h"
+
+
+/*  Common hwloc_topology_init() and flags for Flux hwloc usage:
+ */
+static int topo_init_common (hwloc_topology_t *tp, unsigned long flags)
+{
+    if (hwloc_topology_init (tp) < 0)
+        return -1;
+#if HWLOC_API_VERSION < 0x20000
+    flags |= HWLOC_TOPOLOGY_FLAG_IO_DEVICES;
+    if (hwloc_topology_ignore_type (*tp, HWLOC_OBJ_CACHE) < 0)
+        return -1;
+#else
+    if (hwloc_topology_set_io_types_filter(*tp,
+                                           HWLOC_TYPE_FILTER_KEEP_IMPORTANT)
+        < 0)
+        return -1;
+    if (hwloc_topology_set_icache_types_filter(*tp,
+                                               HWLOC_TYPE_FILTER_KEEP_STRUCTURE)
+        < 0)
+        return -1;
+#endif
+    /*  N.B.: hwloc_topology_set_flags may cause memory leaks on some systems
+     */
+    if (hwloc_topology_set_flags (*tp, flags) < 0)
+        return -1;
+    return 0;
+}
+
+static int init_topo_from_xml (hwloc_topology_t *tp,
+                               const char *xml,
+                               unsigned long flags)
+{
+    int len = strlen (xml) + 1;
+
+    if (topo_init_common (tp, flags) < 0)
+        return -1;
+    if (hwloc_topology_set_xmlbuffer (*tp, xml, len) < 0) {
+        /* In some versions of hwloc/libxml, the NUL character on the XML
+         * buffer cannot be included in len. Therefore, if set_xmlbuffer fails
+         * above, retry with len-1.
+         */
+        if (hwloc_topology_set_xmlbuffer (*tp, xml, len - 1) < 0)
+            goto error;
+    }
+    if (hwloc_topology_load (*tp) < 0)
+        goto error;
+    return 0;
+error:
+    hwloc_topology_destroy (*tp);
+    return -1;
+}
+
+static int topo_restrict (hwloc_topology_t topo)
+{
+    hwloc_bitmap_t set = NULL;
+    int rc = -1;
+    if (!(set = hwloc_bitmap_alloc ())
+        || hwloc_get_cpubind (topo, set, HWLOC_CPUBIND_PROCESS) < 0
+        || hwloc_topology_restrict (topo, set, 0) < 0)
+        goto err;
+    rc = 0;
+err:
+    hwloc_bitmap_free (set);
+    return rc;
+}
+
+hwloc_topology_t rhwloc_xml_topology_load (const char *xml,
+                                           rhwloc_flags_t in_flags)
+{
+    hwloc_topology_t topo = NULL;
+    int flags = 0;
+    if (!(in_flags & RHWLOC_NO_RESTRICT))
+        flags |= HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM;
+    if (init_topo_from_xml (&topo, xml, flags) < 0)
+        return NULL;
+    if (!(in_flags & RHWLOC_NO_RESTRICT)
+        && topo_restrict (topo) < 0) {
+        hwloc_topology_destroy (topo);
+        return NULL;
+    }
+    return topo;
+}
+
+static char *topo_xml_export (hwloc_topology_t topo)
+{
+    char *buf = NULL;
+    int buflen;
+    char *result = NULL;
+
+    if (!topo)
+        return NULL;
+
+#if HWLOC_API_VERSION >= 0x20000
+    if (hwloc_topology_export_xmlbuffer (topo, &buf, &buflen, 0) < 0) {
+#else
+    if (hwloc_topology_export_xmlbuffer (topo, &buf, &buflen) < 0) {
+#endif
+        goto out;
+    }
+    result = strdup (buf);
+out:
+    if (buf)
+        hwloc_free_xmlbuffer (topo, buf);
+    return result;
+}
+
+/*  Restrict an XML topology by loading it with no flags (which automatically
+ *  restricts to current resource binding), then re-export to XML:
+ */
+char *rhwloc_topology_xml_restrict (const char *xml)
+{
+    char *result;
+    hwloc_topology_t topo;
+
+    if (!(topo = rhwloc_xml_topology_load (xml, 0)))
+        return NULL;
+    result = topo_xml_export (topo);
+    hwloc_topology_destroy (topo);
+    return result;
+}
+
+hwloc_topology_t rhwloc_xml_topology_load_file (const char *path,
+                                                rhwloc_flags_t flags)
+{
+    hwloc_topology_t topo;
+    int fd;
+    int rc;
+    char *buf;
+    if ((fd = open (path, O_RDONLY)) < 0)
+        return NULL;
+    rc = read_all (fd, (void **) &buf);
+    ERRNO_SAFE_WRAP (close, fd);
+    if (rc < 0)
+        return NULL;
+    /*  Load hwloc from XML file, add current system information from uname(2)
+     *  unless already set.
+     */
+    if ((topo = rhwloc_xml_topology_load (buf, flags))
+        && !rhwloc_hostname (topo)) {
+        struct utsname utsname;
+        int depth = hwloc_get_type_depth (topo, HWLOC_OBJ_MACHINE);
+        hwloc_obj_t obj = hwloc_get_obj_by_depth (topo, depth, 0);
+
+        /* Best effort */
+        if (uname (&utsname) == 0) {
+            hwloc_obj_add_info(obj, "OSName", utsname.sysname);
+            hwloc_obj_add_info(obj, "OSRelease", utsname.release);
+            hwloc_obj_add_info(obj, "OSVersion", utsname.version);
+            hwloc_obj_add_info(obj, "HostName", utsname.nodename);
+            hwloc_obj_add_info(obj, "Architecture", utsname.machine);
+        }
+    }
+    ERRNO_SAFE_WRAP (free, buf);
+    return topo;
+}
+
+hwloc_topology_t rhwloc_local_topology_load (rhwloc_flags_t flags)
+{
+    const char *xml;
+    hwloc_topology_t topo = NULL;
+    uint32_t hwloc_version = hwloc_get_api_version ();
+
+    if ((hwloc_version >> 16) != (HWLOC_API_VERSION >> 16))
+        return NULL;
+
+    /*  Allow FLUX_HWLOC_XMLFILE to force loading topology from a file
+     *  instead of the system. This is meant for testing usage only.
+     *  If loading from the XML file fails for any reason, fall back
+     *  to normal topology load.
+     */
+    if ((xml = getenv ("FLUX_HWLOC_XMLFILE"))
+        && (topo = rhwloc_xml_topology_load_file (xml, flags)))
+        return topo;
+
+    if (topo_init_common (&topo, 0) < 0)
+        goto err;
+#if HWLOC_API_VERSION >= 0x20100
+    /* gl probes the NV-CONTROL X server extension, and requires X auth
+     * to be properly set up or errors are emitted to stderr.
+     * Nvidia GPUs can still be discovered via opencl.
+     */
+    hwloc_topology_set_components (topo,
+                                   HWLOC_TOPOLOGY_COMPONENTS_FLAG_BLACKLIST,
+                                   "gl");
+#endif
+    if (hwloc_topology_load (topo) < 0)
+        goto err;
+    if (flags & RHWLOC_NO_RESTRICT)
+        return (topo);
+    if (topo_restrict (topo) < 0)
+        goto err;
+    return (topo);
+err:
+    hwloc_topology_destroy (topo);
+    return NULL;
+}
+
+char *rhwloc_local_topology_xml (rhwloc_flags_t rflags)
+{
+    char *result;
+    hwloc_topology_t topo = rhwloc_local_topology_load (rflags);
+    result = topo_xml_export (topo);
+    hwloc_topology_destroy (topo);
+    return result;
+}
+
+const char * rhwloc_hostname (hwloc_topology_t topo)
+{
+    int depth = hwloc_get_type_depth (topo, HWLOC_OBJ_MACHINE);
+    hwloc_obj_t obj = hwloc_get_obj_by_depth (topo, depth, 0);
+    if (obj)
+        return hwloc_obj_get_info_by_name(obj, "HostName");
+    return NULL;
+}
+
+/*  Generate a cpuset string for all cores in the current topology
+ */
+char *rhwloc_core_idset_string (hwloc_topology_t topo)
+{
+    char *result = NULL;
+    struct idset *ids = NULL;
+    int depth = hwloc_get_type_depth (topo, HWLOC_OBJ_CORE);
+
+    if (!(ids = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        goto out;
+
+    for (int i = 0; i < hwloc_get_nbobjs_by_depth(topo, depth); i++) {
+        hwloc_obj_t core = hwloc_get_obj_by_depth (topo, depth, i);
+        idset_set (ids, core->logical_index);
+    }
+
+    result = idset_encode (ids, IDSET_FLAG_RANGE);
+out:
+    idset_destroy (ids);
+    return result;
+}
+
+/*  Return true if the hwloc "backend" type string matches a GPU
+ *   which should be indexed as a compute GPU.
+ */
+static bool backend_is_coproc (const char *s)
+{
+    /* Only count cudaX, openclX, and rmsiX devices for now */
+    return (streq (s, "CUDA")
+            || streq (s, "OpenCL")
+            || streq (s, "RSMI"));
+}
+
+char * rhwloc_gpu_idset_string (hwloc_topology_t topo)
+{
+    int index;
+    char *result = NULL;
+    hwloc_obj_t obj = NULL;
+    struct idset *ids = idset_create (0, IDSET_FLAG_AUTOGROW);
+
+    if (!ids)
+        return NULL;
+
+    /*  Manually index GPUs -- os_index does not seem to be valid for
+     *  these devices in some cases, and logical index also seems
+     *  incorrect (?)
+     */
+    index = 0;
+    while ((obj = hwloc_get_next_osdev (topo, obj))) {
+        const char *s = hwloc_obj_get_info_by_name (obj, "Backend");
+        if (s && backend_is_coproc (s))
+            idset_set (ids, index++);
+    }
+    if (idset_count (ids) > 0)
+        result = idset_encode (ids, IDSET_FLAG_RANGE);
+    idset_destroy (ids);
+    return result;
+}
+
+struct rlist *rlist_from_hwloc (int rank, const char *xml)
+{
+    char *ids = NULL;
+    struct rnode *n = NULL;
+    hwloc_topology_t topo = NULL;
+    const char *name;
+    struct rlist *rl = rlist_create ();
+
+    if (!rl)
+        return NULL;
+
+    if (xml)
+        topo = rhwloc_xml_topology_load (xml, RHWLOC_NO_RESTRICT);
+    else
+        topo = rhwloc_local_topology_load (0);
+    if (!topo)
+        goto fail;
+    if (!(ids = rhwloc_core_idset_string (topo))
+        || !(name = rhwloc_hostname (topo)))
+        goto fail;
+
+    if (!(n = rnode_create (name, rank, ids))
+        || rlist_add_rnode (rl, n) < 0)
+        goto fail;
+
+    free (ids);
+
+    if ((ids = rhwloc_gpu_idset_string (topo))
+        && rnode_add_child (n, "gpu", ids) < 0)
+        goto fail;
+
+    hwloc_topology_destroy (topo);
+    free (ids);
+    return rl;
+fail:
+    rlist_destroy (rl);
+    rnode_destroy (n);
+    free (ids);
+    if (topo)
+        hwloc_topology_destroy (topo);
+    return NULL;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librlist/rhwloc.h b/src/common/librlist/rhwloc.h
new file mode 100644
index 000000000000..b77a001197ba
--- /dev/null
+++ b/src/common/librlist/rhwloc.h
@@ -0,0 +1,55 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_UTIL_RHWLOC_H
+#define HAVE_UTIL_RHWLOC_H 1
+
+#include <hwloc.h>
+
+typedef enum {
+    RHWLOC_NO_RESTRICT = 0x1
+} rhwloc_flags_t;
+
+/*  Load local topology with Flux standard flags and filtering
+ */
+hwloc_topology_t rhwloc_local_topology_load (rhwloc_flags_t flags);
+
+/*  As above, but return hwloc_topoology_t from XML
+ *  Topology is restricted to current CPU binding unless RHWLOC_NO_RESTRICT
+ *  flag is used.
+ */
+hwloc_topology_t rhwloc_xml_topology_load (const char *xml,
+                                           rhwloc_flags_t flags);
+
+/*  Load local topology and return XML as allocated string
+ */
+char *rhwloc_local_topology_xml (rhwloc_flags_t flags);
+
+/*  Restrict an XML topology to current CPU binding and return result.
+ */
+char *rhwloc_topology_xml_restrict (const char *xml);
+
+/*  Return HostName from an hwloc topology object
+ */
+const char *rhwloc_hostname (hwloc_topology_t topo);
+
+/*  Return idset string for all cores in hwloc topology object
+ */
+char * rhwloc_core_idset_string (hwloc_topology_t topo);
+
+/*  Return idset string for all GPUs in hwloc topology object
+ */
+char * rhwloc_gpu_idset_string (hwloc_topology_t topo);
+
+/*  Return rlist object from local hwloc topology, or from xml if non-NULL.
+ */
+struct rlist *rlist_from_hwloc (int my_rank, const char *xml);
+
+#endif /* !HAVE_UTIL_RHWLOC */
diff --git a/src/common/librlist/rlist.c b/src/common/librlist/rlist.c
new file mode 100644
index 000000000000..a453618dff19
--- /dev/null
+++ b/src/common/librlist/rlist.c
@@ -0,0 +1,2572 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdarg.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <errno.h>
+#include <string.h>
+#include <jansson.h>
+
+#include "src/common/libidset/idset.h"
+#include "src/common/libhostlist/hostlist.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "rnode.h"
+#include "match.h"
+#include "rlist.h"
+#include "rlist_private.h"
+
+static int by_rank (const void *item1, const void *item2);
+
+static size_t rank_hasher (const void *key)
+{
+    const int *id = key;
+    return *id;
+}
+
+static int rank_hash_key_cmp (const void *key1, const void *key2)
+{
+    const int *a = key1;
+    const int *b = key2;
+    return *a - *b;
+}
+
+static zhashx_t *rank_hash_create (void)
+{
+    zhashx_t *hash;
+
+    if (!(hash = zhashx_new ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    zhashx_set_key_hasher (hash, rank_hasher);
+    zhashx_set_key_comparator (hash, rank_hash_key_cmp);
+    zhashx_set_key_duplicator (hash, NULL);
+    zhashx_set_key_destructor (hash, NULL);
+
+    return hash;
+}
+
+static struct rnode * rank_hash_lookup (const struct rlist *rl, int rank)
+{
+    return zhashx_lookup (rl->rank_index, &rank);
+}
+
+static void rank_hash_delete (const struct rlist *rl, int rank)
+{
+    zhashx_delete (rl->rank_index, &rank);
+}
+
+static int rank_hash_insert (struct rlist *rl, struct rnode *n)
+{
+    return zhashx_insert (rl->rank_index, &n->rank, n);
+}
+
+static void rank_hash_purge (struct rlist *rl)
+{
+    zhashx_purge (rl->rank_index);
+}
+
+static int
+sprintfcat (char **s, size_t *sz, size_t *lenp, const char *fmt, ...)
+{
+    int done = false;
+    va_list ap;
+    int n = 0;
+    while (!done) {
+        int nleft = *sz-*lenp;
+        va_start (ap, fmt);
+        n = vsnprintf ((*s)+*lenp, nleft, fmt, ap);
+        if (n < 0 || n >= nleft) {
+            char *p;
+            *sz += 128;
+            if (!(p = realloc (*s, *sz)))
+                return -1;
+            *s = p;
+        }
+        else
+            done = true;
+        va_end (ap);
+    }
+    *lenp += n;
+    return (n);
+}
+
+void rlist_destroy (struct rlist *rl)
+{
+    if (rl) {
+        int saved_errno = errno;
+        zlistx_destroy (&rl->nodes);
+        zhashx_destroy (&rl->noremap);
+        zhashx_destroy (&rl->rank_index);
+        json_decref (rl->scheduling);
+        free (rl);
+        errno = saved_errno;
+    }
+}
+
+static void rn_free_fn (void **x)
+{
+    if (x) {
+        rnode_destroy (*(struct rnode **)x);
+        *x = NULL;
+    }
+}
+
+static void valfree (void **item)
+{
+    if (item) {
+        free (*item);
+        *item = NULL;
+    }
+}
+
+struct rlist *rlist_create (void)
+{
+    struct rlist *rl = calloc (1, sizeof (*rl));
+    if (!(rl->nodes = zlistx_new ()))
+        goto err;
+    zlistx_set_destructor (rl->nodes, rn_free_fn);
+
+    if (!(rl->rank_index = rank_hash_create ())
+        || !(rl->noremap = zhashx_new ()))
+        goto err;
+    zhashx_set_destructor (rl->noremap, valfree);
+    zhashx_set_duplicator (rl->noremap, (zhashx_duplicator_fn *) strdup);
+    zhashx_insert (rl->noremap, "gpu", "gpu");
+    return (rl);
+err:
+    rlist_destroy (rl);
+    return (NULL);
+}
+
+/*  Append two scheduling JSON objects s1 and s2.
+ *
+ *  These objects are supposed to be opaque, so for now we punt on
+ *   doing an actual merge and just return s1 if non-NULL or s2 if non-NULL.
+ *
+ *  In the future, perhaps a "deep merge" could be done here instead, though
+ *   the actual implementation of the scheduling key, JGF, do not make this
+ *   easy since the main components are two lists of nodes and edges.
+ */
+static json_t * scheduling_key_append (json_t *s1, json_t *s2)
+{
+    if (s1)
+        return json_incref (s1);
+    else if (s1 == NULL && s2)
+        return json_incref (s2);
+    else
+        return NULL;
+}
+
+static struct rnode *rlist_find_rank (const struct rlist *rl, uint32_t rank)
+{
+    return rank_hash_lookup (rl, rank);
+}
+
+static void rlist_update_totals (struct rlist *rl, struct rnode *n)
+{
+    rl->total += rnode_count (n);
+    if (n->up)
+        rl->avail += rnode_avail (n);
+}
+
+static int rlist_add_rnode_new (struct rlist *rl, struct rnode *n)
+{
+    void *handle;
+    if (!(handle = zlistx_add_end (rl->nodes, n)))
+        return -1;
+    if (rank_hash_insert (rl, n) < 0)
+        return -1;
+    rlist_update_totals (rl, n);
+    return 0;
+}
+
+/*  Add rnode 'n' to the rlist 'rl'. The memory for rnode n is stolen
+ *   by this function (either the rnode is consumed by the rlist, or
+ *   the rnode is destroyed after its resources are applied to 'rl')
+ */
+int rlist_add_rnode (struct rlist *rl, struct rnode *n)
+{
+    struct rnode *found = rlist_find_rank (rl, n->rank);
+    if (found) {
+        if (rnode_add (found, n) < 0)
+            return -1;
+        rlist_update_totals (rl, n);
+        rnode_destroy (n);
+    }
+    else if (rlist_add_rnode_new (rl, n) < 0)
+        return -1;
+    return 0;
+}
+
+typedef struct rnode * (*rnode_copy_f) (const struct rnode *, void *arg);
+
+static struct rlist *rlist_copy_internal (const struct rlist *orig,
+                                          rnode_copy_f cpfn,
+                                          void *arg)
+{
+    struct rnode *n;
+    struct rlist *rl = rlist_create ();
+    if (!rl)
+        return NULL;
+
+    n = zlistx_first (orig->nodes);
+    while (n) {
+        struct rnode *copy = (*cpfn) (n, arg);
+        if (copy && rlist_add_rnode_new (rl, copy) < 0) {
+            rnode_destroy (copy);
+            goto fail;
+        }
+        n = zlistx_next (orig->nodes);
+    }
+
+    /*  Copy entire opaque scheduling key unless rlist is empty
+     */
+    if (rlist_nnodes (rl) > 0)
+        rl->scheduling = scheduling_key_append (orig->scheduling, NULL);
+
+
+    /*  Copy noremap hash from original rlist
+     */
+    zhashx_destroy (&rl->noremap);
+    rl->noremap = zhashx_dup (orig->noremap);
+    if (!rl->noremap)
+        return NULL;
+
+    return rl;
+fail:
+    rlist_destroy (rl);
+    return NULL;
+}
+
+static struct rnode *copy_empty (const struct rnode *rnode, void *arg)
+{
+    return rnode_copy_empty (rnode);
+}
+
+struct rlist *rlist_copy_empty (const struct rlist *orig)
+{
+    return rlist_copy_internal (orig, copy_empty, NULL);
+}
+
+static struct rnode *copy_alloc (const struct rnode *rnode, void *arg)
+{
+    return rnode_copy_alloc (rnode);
+}
+
+struct rlist *rlist_copy_allocated (const struct rlist *orig)
+{
+    return rlist_copy_internal (orig, copy_alloc, NULL);
+}
+
+static struct rnode *copy_cores (const struct rnode *rnode, void *arg)
+{
+    return rnode_copy_cores (rnode);
+}
+
+struct rlist *rlist_copy_cores (const struct rlist *orig)
+{
+    return rlist_copy_internal (orig, copy_cores, NULL);
+}
+
+struct rlist *rlist_copy_down (const struct rlist *orig)
+{
+    struct rnode *n;
+    struct rlist *rl = rlist_create ();
+    if (!rl)
+        return NULL;
+    n = zlistx_first (orig->nodes);
+    while (n) {
+        if (!n->up) {
+            struct rnode *copy = rnode_copy_empty (n);
+            if (!copy || rlist_add_rnode_new (rl, copy) < 0)
+                goto fail;
+        }
+        n = zlistx_next (orig->nodes);
+    }
+
+    /*  Copy entire opaque scheduling key unless rlist is empty
+     */
+    if (rlist_nnodes (rl) > 0)
+        rl->scheduling = scheduling_key_append (orig->scheduling, NULL);
+
+    /*  Copy noremap hash from original rlist
+     */
+    zhashx_destroy (&rl->noremap);
+    rl->noremap = zhashx_dup (orig->noremap);
+    if (!rl->noremap)
+        return NULL;
+
+    return rl;
+fail:
+    rlist_destroy (rl);
+    return NULL;
+}
+
+struct rlist * rlist_copy_ranks (const struct rlist *rl, struct idset *ranks)
+{
+    unsigned int i;
+    struct rnode *n;
+    struct rlist *result = rlist_create ();
+    if (!result)
+        return NULL;
+
+    i = idset_first (ranks);
+    while (i != IDSET_INVALID_ID) {
+        if ((n = rlist_find_rank (rl, i))) {
+            struct rnode *copy = rnode_copy (n);
+            if (!copy || rlist_add_rnode_new (result, copy) < 0) {
+                rnode_destroy (copy);
+                goto err;
+            }
+        }
+        i = idset_next (ranks, i);
+    }
+
+    /*  Copy entire opaque scheduling key unless rlist is empty
+     */
+    if (rlist_nnodes (result) > 0)
+        result->scheduling = scheduling_key_append (rl->scheduling, NULL);
+
+    /*  Copy noremap hash from original rlist
+     */
+    zhashx_destroy (&result->noremap);
+    result->noremap = zhashx_dup (rl->noremap);
+    if (!result->noremap)
+        return NULL;
+
+    return result;
+err:
+    rlist_destroy (result);
+    return NULL;
+}
+
+struct rlist *rlist_copy_constraint (const struct rlist *orig,
+                                     json_t *constraint,
+                                     flux_error_t *errp)
+{
+    struct rlist *rl;
+    struct job_constraint *jc;
+
+    if (!(jc = job_constraint_create (constraint, errp))) {
+        errno = EINVAL;
+        return NULL;
+    }
+    rl = rlist_copy_internal (orig,
+                              (rnode_copy_f) rnode_copy_match,
+                              (void *) jc);
+    job_constraint_destroy (jc);
+    return rl;
+}
+
+struct rlist *rlist_copy_constraint_string (const struct rlist *orig,
+                                            const char *constraint,
+                                            flux_error_t *errp)
+{
+    struct rlist *rl;
+    json_error_t error;
+    json_t *o = json_loads (constraint, 0, &error);
+    if (!o) {
+        errprintf (errp, "%s", error.text);
+        return NULL;
+    }
+    rl = rlist_copy_constraint (orig, o, errp);
+    json_decref (o);
+    return rl;
+}
+
+static int rlist_remove_rank (struct rlist *rl, int rank)
+{
+    void *handle;
+    struct rnode *n;
+
+    if (!(n = rank_hash_lookup (rl, rank))
+        || !(handle = zlistx_find (rl->nodes, n))) {
+        errno = ENOENT;
+        return -1;
+    }
+    rank_hash_delete (rl, rank);
+    zlistx_delete (rl->nodes, handle);
+    return 0;
+}
+
+int rlist_remove_ranks (struct rlist *rl, struct idset *ranks)
+{
+    int count = 0;
+    unsigned int i;
+    i = idset_first (ranks);
+    while (i != IDSET_INVALID_ID) {
+        if (rlist_remove_rank (rl, i) == 0)
+            count++;
+        i = idset_next (ranks, i);
+    }
+    return count;
+}
+
+int rlist_remap (struct rlist *rl)
+{
+    uint32_t rank = 0;
+    struct rnode *n;
+
+    rank_hash_purge (rl);
+
+    /*   Sort list by ascending rank, then rerank starting at 0
+     */
+    zlistx_set_comparator (rl->nodes, by_rank);
+    zlistx_sort (rl->nodes);
+
+    n = zlistx_first (rl->nodes);
+    while (n) {
+        n->rank = rank++;
+        if (rank_hash_insert (rl, n) < 0)
+            return -1;
+        if (rnode_remap (n, rl->noremap) < 0)
+            return -1;
+        n = zlistx_next (rl->nodes);
+    }
+    return 0;
+}
+
+struct rnode * rlist_find_host (const struct rlist *rl, const char *host)
+{
+    struct rnode *n = zlistx_first (rl->nodes);
+    while (n) {
+        if (n->hostname && streq (n->hostname, host))
+            return n;
+        n = zlistx_next (rl->nodes);
+    }
+    errno = ENOENT;
+    return NULL;
+}
+
+static int rlist_rerank_hostlist (struct rlist *rl,
+                                  struct hostlist *hl,
+                                  flux_error_t *errp)
+{
+    uint32_t rank = 0;
+    const char *host = hostlist_first (hl);
+    while (host) {
+        struct rnode *n = rlist_find_host (rl, host);
+        if (!n) {
+            errprintf (errp, "Host %s not found in resources", host);
+            return -1;
+        }
+        n->rank = rank++;
+        if (rank_hash_insert (rl, n) < 0) {
+            errprintf (errp, "failed to hash rank %u", n->rank);
+            return -1;
+        }
+        host = hostlist_next (hl);
+    }
+    return 0;
+}
+
+int rlist_rerank (struct rlist *rl, const char *hosts, flux_error_t *errp)
+{
+    int rc = -1;
+    struct hostlist *hl = NULL;
+    struct hostlist *orig = NULL;
+
+    if (!(hl = hostlist_decode (hosts))) {
+        errprintf (errp, "hostlist_decode: %s: %s", hosts, strerror (errno));
+        return -1;
+    }
+
+    if (hostlist_count (hl) > rlist_nnodes (rl)) {
+        errprintf (errp,
+                   "Number of hosts (%d) is greater than node count (%zu)",
+                   hostlist_count (hl),
+                   rlist_nnodes (rl));
+        errno = EOVERFLOW;
+        goto done;
+    }
+    else if (hostlist_count (hl) < rlist_nnodes (rl)) {
+            errprintf (errp,
+                      "Number of hosts (%d) is less than node count (%zu)",
+                      hostlist_count (hl),
+                      rlist_nnodes (rl));
+        errno = ENOSPC;
+        goto done;
+    }
+
+    rank_hash_purge (rl);
+
+    /* Save original rank mapping in case of undo
+     */
+    if (!(orig = rlist_nodelist (rl)))
+        goto done;
+
+    /* Perform re-ranking based on hostlist hl. On failure, undo
+     *  by reranking with original hostlist.
+     */
+    if ((rc = rlist_rerank_hostlist (rl, hl, errp)) < 0) {
+        int saved_errno = errno;
+        (void) rlist_rerank_hostlist (rl, orig, NULL);
+        errno = saved_errno;
+    }
+done:
+    hostlist_destroy (orig);
+    hostlist_destroy (hl);
+    return rc;
+}
+
+static struct rnode *rlist_detach_rank (struct rlist *rl, uint32_t rank)
+{
+    struct rnode *n = rlist_find_rank (rl, rank);
+    if (n) {
+        zlistx_detach (rl->nodes, zlistx_find (rl->nodes, n));
+        rank_hash_delete (rl, rank);
+    }
+    return n;
+}
+
+struct rlist *rlist_diff (const struct rlist *rla, const struct rlist *rlb)
+{
+    struct rnode *n;
+    struct rlist *rl = rlist_create ();
+
+    if (!rl || rlist_append (rl, rla) < 0) {
+        rlist_destroy (rl);
+        return NULL;
+    }
+
+    n = zlistx_first (rlb->nodes);
+    while (n) {
+        /*  Attempt to find and "detach" the rank which we're diffing.
+         */
+        struct rnode *na = rlist_detach_rank (rl, n->rank);
+        if (na) {
+            /*  Diff the individual resource node.
+             *  If the result is empty, then do nothing since we've
+             *   already detached this node from the list. O/w, push
+             *   the result back onto the rlist.
+             */
+            struct rnode *result = rnode_diff (na, n);
+            if (!rnode_empty (result))
+                rlist_add_rnode (rl, result);
+            else
+                rnode_destroy (result);
+
+            /*  Always need to free the detached rnode */
+            rnode_destroy (na);
+        }
+        n = zlistx_next (rlb->nodes);
+    }
+    return rl;
+}
+
+struct rlist *rlist_union (const struct rlist *rla, const struct rlist *rlb)
+{
+    struct rlist *result = NULL;
+
+    /*  First take the set difference of b from a, such that there are no
+     *   common resources in 'rlb' and 'result'.
+     */
+    if (!(result = rlist_diff (rla, rlb)))
+        return NULL;
+
+    /*  Now append 'rlb' to 'result' to get the union of 'rla' + 'rlb':
+     */
+    if (rlist_append (result, rlb) < 0) {
+        rlist_destroy (result);
+        return NULL;
+    }
+
+    return result;
+}
+
+/*  Add resource set rlb to rla: rla becomes the union of a and b.
+ */
+int rlist_add (struct rlist *rla, const struct rlist *rlb)
+{
+    int rc;
+    struct rlist *diff = NULL;
+
+    /*  Take the set difference of a from b so there are no overlapping
+     *   resources with rla in `diff`
+     */
+    if (!(diff = rlist_diff (rlb, rla)))
+        return -1;
+
+    /*  Now append diff to rla
+     */
+    rc = rlist_append (rla, diff);
+    rlist_destroy (diff);
+    return rc;
+}
+
+struct rlist *rlist_intersect (const struct rlist *rla,
+                               const struct rlist *rlb)
+{
+    struct rnode *n;
+    struct rlist *result = rlist_create ();
+
+    if (!result)
+        return NULL;
+
+    n = zlistx_first (rlb->nodes);
+    while (n) {
+        struct rnode *na = rlist_find_rank (rla, n->rank);
+        struct rnode *nx = rnode_intersect (na, n);
+        if (nx != NULL
+            && !rnode_empty (nx)
+            && rlist_add_rnode (result, nx) < 0)
+            goto err;
+        n = zlistx_next (rlb->nodes);
+    }
+
+    /*  Copy opaque scheduling key unless result is empty
+     */
+    if (rlist_nnodes (result) > 0)
+        result->scheduling = scheduling_key_append (rla->scheduling, NULL);
+
+    return result;
+err:
+    rlist_destroy (result);
+    return NULL;
+}
+
+static void free_item (void **x)
+{
+    if (x) {
+        free (*x);
+        *x = NULL;
+    }
+}
+
+static zlistx_t *errlist_create ()
+{
+    zlistx_t *l = zlistx_new ();
+    if (!l)
+        return NULL;
+    zlistx_set_destructor (l, free_item);
+    return l;
+}
+
+static void errlist_destroy (zlistx_t *l)
+{
+    zlistx_destroy (&l);
+}
+
+static int errlist_append (zlistx_t *l, const char *fmt, ...)
+{
+    char *s = NULL;
+    va_list ap;
+    va_start (ap, fmt);
+    if (vasprintf (&s, fmt, ap) < 0)
+        return -1;
+    va_end (ap);
+    if (!zlistx_add_end (l, s)) {
+        free (s);
+        return -1;
+    }
+    return 0;
+}
+
+static int errlist_concat (zlistx_t *l, char *buf, size_t len)
+{
+    int n = 0;
+    char *s;
+
+    memset (buf, 0, len);
+
+    s = zlistx_first (l);
+    while (s) {
+        if ((int)len - n > 0)
+            strncpy (buf + n, s, len - n);
+        n += strlen (s);
+        s = zlistx_next (l);
+        if (s) {
+            strncat (buf, ", ", len - n);
+            n += 2;
+        }
+    }
+    return len;
+}
+
+static int rnode_namecmp (const void *s1, const void *s2)
+{
+    const char *a = s1;
+    const char *b = s2;
+
+    if (streq (a, "core"))
+        return -1;
+    else if (streq (b, "core"))
+        return 1;
+    else return strcmp (a, b);
+}
+
+static int rnode_sprintfcat (const struct rnode *n,
+                             char **dest,
+                             size_t *sizep,
+                             size_t *lenp)
+{
+    int rc = -1;
+    zlistx_t *keys = NULL;
+    char *ids = NULL;
+    char *name = NULL;
+    char *comma = "";
+
+    /*  Generate sorted list of resources on those ranks.
+     *  (Ensuring that "core" is always first)
+     */
+    keys = zhashx_keys (n->children);
+    zlistx_set_comparator (keys, rnode_namecmp);
+    zlistx_sort (keys);
+
+    /*  Output all resource strings as name[ids]
+     */
+    name = zlistx_first (keys);
+    while (name) {
+        struct rnode_child *c = zhashx_lookup (n->children, name);
+
+        /*  Skip empty sets */
+        if (idset_count (c->avail) > 0) {
+            ids = idset_encode (c->avail,
+                                IDSET_FLAG_RANGE | IDSET_FLAG_BRACKETS);
+            if (!ids)
+                goto fail;
+            if (sprintfcat (dest, sizep, lenp,
+                            "%s%s%s",
+                            comma,
+                            c->name,
+                            ids) < 0)
+                goto fail;
+
+            free (ids);
+            ids = NULL;
+            comma = ",";
+        }
+        name = zlistx_next (keys);
+    }
+    rc = 0;
+fail:
+    zlistx_destroy (&keys);
+    free (ids);
+    return rc;
+}
+
+static char * rnode_child_dumps (struct rnode *rnode)
+{
+    size_t n = 0;
+    size_t len = 0;
+    char *s = NULL;
+    if (rnode_sprintfcat (rnode, &s, &n, &len) < 0) {
+        free (s);
+        return NULL;
+    }
+    return s;
+}
+
+int rlist_verify (flux_error_t *errp,
+                  const struct rlist *expected,
+                  const struct rlist *rl)
+{
+    struct rnode *n = NULL;
+    struct rnode *exp = NULL;
+    struct rnode *diff = NULL;
+    zlistx_t *errors = NULL;
+    int saved_errno;
+    int rc = -1;
+
+    if (!(errors = errlist_create ())) {
+        errprintf (errp, "Internal error: Out of memory");
+        errno = ENOMEM;
+        goto done;
+    }
+
+    if (rlist_nnodes (rl) != 1) {
+        errlist_append (errors,
+                        "Verification supported on single rank only");
+        errno = EINVAL;
+        goto done;
+    }
+
+    n = zlistx_first (rl->nodes);
+    if (!(exp = rlist_find_rank (expected, n->rank))) {
+        errlist_append (errors,
+                        "rank %d not found in expected ranks",
+                        n->rank);
+        errno = EINVAL;
+        goto done;
+    }
+    if (rnode_hostname_cmp (n, exp) != 0) {
+        errlist_append (errors,
+                        "rank %d got hostname '%s', expected '%s'",
+                        n->rank,
+                        n->hostname ? n->hostname : "unknown",
+                        exp->hostname ? exp->hostname : "unknown");
+        goto done;
+    }
+    if (!(diff = rnode_diff (exp, n))) {
+        errlist_append (errors,
+                        "Internal error: rnode_diff failed: %s",
+                        strerror (errno));
+        goto done;
+    }
+    if (!rnode_empty (diff)) {
+        char *s = rnode_child_dumps (diff);
+        errlist_append (errors,
+                        "rank %d (%s) missing resources: %s",
+                        n->rank, n->hostname ? n->hostname : "unknown", s);
+        free (s);
+        goto done;
+    }
+    rnode_destroy (diff);
+    if (!(diff = rnode_diff (n, exp))) {
+        errlist_append (errors,
+                        "Internal error: rnode_diff failed: %s",
+                        strerror (errno));
+        goto done;
+    }
+    if (rnode_empty (diff))
+        rc = 0;
+    else {
+        char *s = rnode_child_dumps (diff);
+        errlist_append (errors,
+                        "rank %d (%s) has extra resources: %s",
+                        n->rank, n->hostname ? n->hostname : "unknown", s);
+        free (s);
+        rc = 1;
+    }
+done:
+    saved_errno = 0;
+    rnode_destroy (diff);
+    memset (errp->text, 0, sizeof (errp->text));
+    if (errors) {
+        errlist_concat (errors, errp->text, sizeof (errp->text));
+        errlist_destroy (errors);
+    }
+    errno = saved_errno;
+    return rc;
+}
+
+int rlist_append (struct rlist *rl, const struct rlist *rl2)
+{
+    json_t *o;
+    struct rnode *n = zlistx_first (rl2->nodes);
+    while (n) {
+        struct rnode *copy = rnode_copy_avail (n);
+        if (!copy || rlist_add_rnode (rl, copy) < 0) {
+            rnode_destroy (copy);
+            return -1;
+        }
+        n = zlistx_next (rl2->nodes);
+    }
+
+    o = scheduling_key_append (rl->scheduling, rl2->scheduling);
+    json_decref (rl->scheduling);
+    rl->scheduling = o;
+
+    return 0;
+}
+
+static int rlist_append_rank (struct rlist *rl,
+                              const char *hostname,
+                              unsigned int rank,
+                              json_t *children)
+{
+    struct rnode *n = rnode_create_children (hostname, rank, children);
+    if (!n || rlist_add_rnode (rl, n) < 0) {
+        rnode_destroy (n);
+        return -1;
+    }
+    return 0;
+}
+
+int rlist_append_rank_cores (struct rlist *rl,
+                             const char *hostname,
+                             unsigned int rank,
+                             const char *core_ids)
+{
+    int rc;
+    json_t *children = json_pack ("{s:s}", "core", core_ids);
+    if (!children)
+        return -1;
+    rc = rlist_append_rank (rl, hostname, rank, children);
+    json_decref (children);
+    return rc;
+}
+
+int rlist_rank_add_child (struct rlist *rl,
+                          unsigned int rank,
+                          const char *name,
+                          const char *ids)
+{
+    struct rnode *n = rlist_find_rank (rl, rank);
+    if (!n) {
+        errno = ENOENT;
+        return -1;
+    }
+    if (rnode_add_child (n, name, ids) == NULL)
+        return -1;
+    return 0;
+}
+
+static int rlist_append_ranks (struct rlist *rl,
+                               const char *rank,
+                               json_t *children)
+{
+    int rc = -1;
+    unsigned int i;
+    struct idset * ranks = idset_decode (rank);
+    if (!ranks)
+        return -1;
+    i = idset_first (ranks);
+    while (i != IDSET_INVALID_ID) {
+        if (rlist_append_rank (rl, NULL, i, children) < 0)
+            goto err;
+        i = idset_next (ranks, i);
+    }
+    rc = 0;
+err:
+    idset_destroy (ranks);
+    return rc;
+}
+
+static int rlist_append_cores (struct rlist *rl,
+                               const char *hostname,
+                               int rank,
+                               struct idset *idset)
+{
+    struct rnode *n = rnode_create_idset (hostname, rank, idset);
+    if (!n || rlist_add_rnode (rl, n) < 0) {
+        rnode_destroy (n);
+        return -1;
+    }
+    return 0;
+}
+
+static int rlist_append_rank_entry (struct rlist *rl,
+                                    json_t *entry,
+                                    json_error_t *ep)
+{
+    const char *ranks;
+    json_t *children;
+    if (json_unpack_ex (entry, ep, 0,
+                        "{s:s s:o}",
+                        "rank", &ranks,
+                        "children", &children) < 0) {
+        return -1;
+    }
+    return rlist_append_ranks (rl, ranks, children);
+}
+
+static struct hostlist * hostlist_from_array (json_t *o)
+{
+    size_t index;
+    json_t *val;
+
+    struct hostlist *hl = hostlist_create ();
+    if (hl == NULL)
+        return NULL;
+
+    json_array_foreach (o, index, val) {
+        const char *hosts = json_string_value (val);
+        if (hostlist_append (hl, hosts) < 0)
+            goto err;
+    }
+
+    return hl;
+err:
+    hostlist_destroy (hl);
+    return NULL;
+}
+
+static int rlist_assign_hostlist (struct rlist *rl, struct hostlist *hl)
+{
+    struct rnode *n;
+
+    if (!hl || hostlist_count (hl) != zlistx_size (rl->nodes))
+        return -1;
+
+    /*  Reset default sort to order nodes by "rank" */
+    zlistx_set_comparator (rl->nodes, by_rank);
+    zlistx_sort(rl->nodes);
+
+    /*  Consume a hostname for each node in the rlist */
+    n = zlistx_first (rl->nodes);
+    (void) hostlist_first (hl);
+    while (n) {
+        free (n->hostname);
+        if (!(n->hostname = strdup (hostlist_current (hl))))
+            return -1;
+        (void) hostlist_next (hl);
+        n = zlistx_next (rl->nodes);
+    }
+    return 0;
+}
+
+int rlist_assign_hosts (struct rlist *rl, const char *hosts)
+{
+    int rc;
+    struct hostlist *hl = hostlist_decode (hosts);
+    rc = rlist_assign_hostlist (rl, hl);
+    hostlist_destroy (hl);
+    return rc;
+}
+
+static int rlist_assign_nodelist (struct rlist *rl, json_t *nodelist)
+{
+    int rc = -1;
+    struct hostlist *hl = hostlist_from_array (nodelist);
+    rc = rlist_assign_hostlist (rl, hl);
+    hostlist_destroy (hl);
+    return rc;
+}
+
+static void property_destructor (void **arg)
+{
+    if (arg) {
+        idset_destroy (*(struct idset **) arg);
+        *arg = NULL;
+    }
+}
+
+static char * property_string_invalid (const char *s)
+{
+    return strpbrk (s, "^&'\"`|()");
+}
+
+int rlist_add_property (struct rlist *rl,
+                        flux_error_t *errp,
+                        const char *name,
+                        const char *targets)
+{
+    unsigned int i;
+    int count;
+    struct rnode *n;
+    struct idset *ids = NULL;
+    struct idset *unknown = NULL;
+    int rc = -1;
+    char *p;
+
+    if ((p = property_string_invalid (name))) {
+        errprintf (errp,
+                   "Invalid character '%c' in property \"%s\"",
+                   *p,
+                   name);
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (!targets || !(ids = idset_decode (targets))) {
+        errprintf (errp,
+                   "Invalid idset string '%s'",
+                   targets ? targets : "(null)");
+        errno = EINVAL;
+        goto out;
+    }
+
+    /*  Check for invalid ranks first, so we can fail early before
+     *   applying any properties.
+     */
+    if (!(unknown = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+        errprintf (errp, "Out of memory");
+        goto out;
+    }
+    i = idset_first (ids);
+    while (i != IDSET_INVALID_ID) {
+        if (!rlist_find_rank (rl, i)
+            && idset_set (unknown, i) < 0) {
+            errprintf (errp, "unknown rank %u", i);
+            errno = ENOENT;
+            goto out;
+        }
+        i = idset_next (ids, i);
+    }
+    if ((count = idset_count (unknown)) > 0) {
+        p = idset_encode (unknown, IDSET_FLAG_RANGE);
+        errprintf (errp,
+                   "%s%s not found in target resource list",
+                   p ? (count == 1 ? "rank " : "ranks ") : "some ranks",
+                   p ? p : "");
+        free (p);
+        errno = ENOENT;
+        goto out;
+    }
+
+    i = idset_first (ids);
+    while (i != IDSET_INVALID_ID) {
+        if ((n = rlist_find_rank (rl, i)))
+            if (rnode_set_property (n, name) < 0) {
+                errprintf (errp,
+                           "Failed to set property %s on rank %u",
+                           name,
+                           i);
+                goto out;
+            }
+        i = idset_next (ids, i);
+    }
+    rc = 0;
+out:
+    idset_destroy (ids);
+    idset_destroy (unknown);
+    return rc;
+}
+
+int rlist_assign_properties (struct rlist *rl,
+                             json_t *properties,
+                             flux_error_t *errp)
+{
+    json_t *val;
+    const char *name;
+    char *p;
+    int rc = -1;
+
+    if (!rl || !properties) {
+        errprintf (errp, "Invalid argument");
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!json_is_object (properties)) {
+        errprintf (errp, "properties must be an object");
+        errno = EINVAL;
+        return -1;
+    }
+
+    /*  First, iterate all property inputs to ensure they are valid.
+     *  This avoids the need to undo any applied properties on failure.
+     */
+    json_object_foreach (properties, name, val) {
+        struct idset *ids;
+        if (!json_is_string (val)) {
+            char *s = json_dumps (val, JSON_COMPACT | JSON_ENCODE_ANY);
+            errprintf (errp, "properties value '%s' not a string", s);
+            free (s);
+            errno = EINVAL;
+            goto error;
+        }
+        if ((p = property_string_invalid (name))) {
+            errprintf (errp,
+                       "invalid character '%c' in property \"%s\"",
+                        *p,
+                        name);
+            errno = EINVAL;
+            goto error;
+        }
+        if (!(ids = idset_decode (json_string_value (val)))) {
+            errprintf (errp,
+                       "invalid idset '%s' specified for property \"%s\"",
+                       json_string_value (val),
+                       name);
+            errno = EINVAL;
+            goto error;
+        }
+        idset_destroy (ids);
+    }
+
+    json_object_foreach (properties, name, val) {
+        /*  Note: validity of json_string_value (val)
+         *   checked above, no need to do it again here.
+         */
+        if (rlist_add_property (rl, errp, name, json_string_value (val)) < 0)
+            goto error;
+    }
+    rc = 0;
+error:
+    return rc;
+}
+
+struct rlist *rlist_from_json (json_t *o, json_error_t *errp)
+{
+    int i, version;
+    struct rlist *rl = NULL;
+    json_t *entry = NULL;
+    json_t *R_lite = NULL;
+    json_t *nodelist = NULL;
+    json_t *scheduling = NULL;
+    json_t *properties = NULL;
+    double starttime = -1.;
+    double expiration = -1.;
+    flux_error_t error;
+
+    if (json_unpack_ex (o, errp, 0,
+                        "{s:i s?O s:{s:o s?o s?o s?F s?F}}",
+                        "version", &version,
+                        "scheduling", &scheduling,
+                        "execution",
+                          "R_lite", &R_lite,
+                          "nodelist", &nodelist,
+                          "properties", &properties,
+                          "starttime", &starttime,
+                          "expiration", &expiration) < 0)
+        goto err;
+    if (version != 1) {
+        if (errp)
+            snprintf (errp->text,
+                      sizeof (errp->text),
+                      "invalid version=%d",
+                      version);
+        goto err;
+    }
+    if (!(rl = rlist_create ()))
+        goto err;
+
+    if (scheduling)
+        rl->scheduling = scheduling;
+    if (starttime > 0.)
+        rl->starttime = starttime;
+    if (expiration > 0.)
+        rl->expiration = expiration;
+
+    json_array_foreach (R_lite, i, entry) {
+        if (rlist_append_rank_entry (rl, entry, errp) < 0)
+            goto err;
+    }
+    if (nodelist && rlist_assign_nodelist (rl, nodelist) < 0)
+        goto err;
+    if (properties && rlist_assign_properties (rl, properties, &error) < 0) {
+        if (errp)
+            snprintf (errp->text, sizeof (errp->text), "%s", error.text);
+        goto err;
+    }
+    return (rl);
+err:
+    rlist_destroy (rl);
+    return (NULL);
+}
+
+struct rlist *rlist_from_R (const char *s)
+{
+    json_error_t err;
+    struct rlist *rl = NULL;
+    json_t *o = json_loads (s, 0, &err);
+    if (o)
+        rl = rlist_from_json (o, &err);
+    json_decref (o);
+    return rl;
+}
+
+/* Helper for rlist_compressed */
+struct multi_rnode {
+    struct idset *ids;
+    const struct rnode *rnode;
+};
+
+static int multi_rnode_cmp (struct multi_rnode *x, const struct rnode *n)
+{
+    int rv = rnode_cmp (x->rnode, n);
+
+    /* Only collapse nodes with same avail idset + same up/down status */
+    if (rv == 0 && n->up == x->rnode->up)
+        return 0;
+
+    /* O/w, order doesn't matter too much, but put up nodes first */
+    return n->up ? -1 : 1;
+}
+
+static void multi_rnode_destroy (struct multi_rnode **mrn)
+{
+    if (mrn && *mrn) {
+        (*mrn)->rnode = NULL;
+        idset_destroy ((*mrn)->ids);
+        free (*mrn);
+        *mrn = NULL;
+    }
+}
+
+struct multi_rnode * multi_rnode_create (struct rnode *rnode)
+{
+    struct multi_rnode *mrn = calloc (1, sizeof (*mrn));
+    if (mrn == NULL)
+        return NULL;
+    if (!(mrn->ids = idset_create (0, IDSET_FLAG_AUTOGROW))
+        || (idset_set (mrn->ids, rnode->rank) < 0))
+        goto fail;
+    mrn->rnode = rnode;
+    return (mrn);
+fail:
+    multi_rnode_destroy (&mrn);
+    return NULL;
+}
+
+json_t *multi_rnode_tojson (struct multi_rnode *mrn)
+{
+    return rnode_encode (mrn->rnode, mrn->ids);
+}
+
+
+static int multi_rnode_by_rank (const void *item1, const void *item2)
+{
+    const struct multi_rnode *mrn1 = item1;
+    const struct multi_rnode *mrn2 = item2;
+
+    unsigned int x = idset_first (mrn1->ids);
+    unsigned int y = idset_first (mrn2->ids);
+
+    return (x - y);
+}
+
+static zlistx_t * rlist_mrlist (const struct rlist *rl)
+{
+    struct rnode *n = NULL;
+    struct multi_rnode *mrn = NULL;
+    zlistx_t *l = zlistx_new ();
+
+    zlistx_set_comparator (l, (zlistx_comparator_fn *) multi_rnode_cmp);
+    zlistx_set_destructor (l, (zlistx_destructor_fn *) multi_rnode_destroy);
+
+    n = zlistx_first (rl->nodes);
+    while (n) {
+        if (zlistx_find (l, n)) {
+            if (!(mrn = zlistx_handle_item (zlistx_cursor (l)))
+                    || idset_set (mrn->ids, n->rank) < 0) {
+                goto fail;
+            }
+        }
+        else {
+            if (!(mrn = multi_rnode_create (n))
+                    || !zlistx_add_end (l, mrn)) {
+                goto fail;
+            }
+        }
+        n = zlistx_next (rl->nodes);
+    }
+    return (l);
+fail:
+    zlistx_destroy (&l);
+    return NULL;
+}
+
+static json_t * rlist_compressed (const struct rlist *rl)
+{
+    struct multi_rnode *mrn = NULL;
+    json_t *o = json_array ();
+    zlistx_t *l = rlist_mrlist (rl);
+
+    if (!l)
+        return NULL;
+    zlistx_set_comparator (l, (zlistx_comparator_fn *) multi_rnode_by_rank);
+    zlistx_sort (l);
+    mrn = zlistx_first (l);
+    while (mrn) {
+        if (rnode_avail_total (mrn->rnode) > 0) {
+            json_t *entry = multi_rnode_tojson (mrn);
+            if (!entry || json_array_append_new (o, entry) != 0) {
+                json_decref (entry);
+                goto fail;
+            }
+        }
+        mrn = zlistx_next (l);
+    }
+    zlistx_destroy (&l);
+    return (o);
+fail:
+    zlistx_destroy (&l);
+    json_decref (o);
+    return NULL;
+}
+
+static int mrnode_sprintfcat (struct multi_rnode *mrn,
+                              char **resultp,
+                              size_t *sizep,
+                              size_t *lenp)
+{
+    int flags = IDSET_FLAG_RANGE | IDSET_FLAG_BRACKETS;
+    int rc = -1;
+    char *ranks = NULL;
+
+    /* Do not output anything if there are no available resources */
+    if (rnode_avail_total (mrn->rnode) == 0)
+        return 0;
+
+    /*  First encode set of ranks into a string:
+     */
+    if (!(ranks = idset_encode (mrn->ids, flags)))
+        goto fail;
+    if (sprintfcat (resultp, sizep, lenp,
+                    "%srank%s/",
+                    (*resultp)[0] != '\0' ? " ": "",
+                    ranks) < 0)
+        goto fail;
+
+    if (rnode_sprintfcat (mrn->rnode, resultp, sizep, lenp) < 0)
+        goto fail;
+
+    rc = 0;
+fail:
+    free (ranks);
+    return rc;
+}
+
+char * rlist_dumps (const struct rlist *rl)
+{
+    char * result = NULL;
+    size_t len = 0;
+    size_t size = 64;
+    struct multi_rnode *mrn = NULL;
+    zlistx_t *l = NULL;
+
+    if (rl == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (!(l = rlist_mrlist (rl))
+        || !(result = calloc (size, sizeof (char))))
+        goto fail;
+
+    mrn = zlistx_first (l);
+    while (mrn) {
+        if (mrnode_sprintfcat (mrn, &result, &size, &len) < 0)
+            goto fail;
+        mrn = zlistx_next (l);
+    }
+    zlistx_destroy (&l);
+    return (result);
+fail:
+    free (result);
+    zlistx_destroy (&l);
+    return NULL;
+}
+
+static json_t *hostlist_to_nodelist (struct hostlist *hl)
+{
+    json_t *o = NULL;
+    char *hosts;
+    if ((hosts = hostlist_encode (hl)))
+        o = json_pack ("[s]", hosts);
+    free (hosts);
+    return o;
+}
+
+struct idset *rlist_ranks (const struct rlist *rl)
+{
+    struct rnode *n;
+    struct idset *ids = idset_create (0, IDSET_FLAG_AUTOGROW);
+    if (!ids)
+        return NULL;
+
+    n = zlistx_first (rl->nodes);
+    while (n) {
+        if (idset_set (ids, n->rank) < 0)
+            goto fail;
+        n = zlistx_next (rl->nodes);
+    }
+    return ids;
+fail:
+    idset_destroy (ids);
+    return NULL;
+}
+
+struct hostlist *rlist_nodelist (const struct rlist *rl)
+{
+    struct rnode *n;
+    struct hostlist *hl = hostlist_create ();
+
+    if (!hl)
+        return NULL;
+
+    /*  List must be sorted by rank before collecting nodelist
+     */
+    zlistx_set_comparator (rl->nodes, by_rank);
+    zlistx_sort (rl->nodes);
+
+    n = zlistx_first (rl->nodes);
+    while (n) {
+        if (!n->hostname || hostlist_append (hl, n->hostname) < 0)
+            goto fail;
+        n = zlistx_next (rl->nodes);
+    }
+    return hl;
+fail:
+    hostlist_destroy (hl);
+    return NULL;
+}
+
+static int rlist_idset_set_by_host (const struct rlist *rl,
+                                    struct idset *ids,
+                                    const char *host)
+{
+    int count = 0;
+    struct rnode *n = zlistx_first (rl->nodes);
+    while (n) {
+        if (n->hostname && streq (n->hostname, host)) {
+            if (idset_set (ids, n->rank) < 0)
+                return -1;
+            count++;
+        }
+        n = zlistx_next (rl->nodes);
+    }
+    return count;
+}
+
+struct idset *rlist_hosts_to_ranks (const struct rlist *rl,
+                                    const char *hosts,
+                                    flux_error_t *errp)
+{
+    const char *host;
+    struct idset *ids = NULL;
+    struct hostlist *hl = NULL;
+    struct hostlist *missing = NULL;
+
+    if (errp)
+        memset (errp->text, 0, sizeof (errp->text));
+
+    if (rl == NULL || hosts == NULL) {
+        errprintf (errp, "An expected argument was NULL");
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(hl = hostlist_decode (hosts))) {
+        errprintf (errp, "Hostlist cannot be decoded");
+        goto fail;
+    }
+    if (!(ids = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+        errprintf (errp, "idset_create: %s", strerror (errno));
+        goto fail;
+    }
+    if (!(missing = hostlist_create ())) {
+        errprintf (errp, "hostlist_create: %s", strerror (errno));
+        goto fail;
+    }
+    host = hostlist_first (hl);
+    while (host) {
+        int count = rlist_idset_set_by_host (rl, ids, host);
+        if (count < 0) {
+            errprintf (errp,
+                        "error adding host %s to idset: %s",
+                        host,
+                        strerror (errno));
+            goto fail;
+        } else if (!count && hostlist_append (missing, host) < 0) {
+            errprintf (errp,
+                        "failed to append missing host '%s'",
+                        host);
+            goto fail;
+        }
+        host = hostlist_next (hl);
+    }
+    if (hostlist_count (missing)) {
+        char *s = hostlist_encode (missing);
+        errprintf (errp, "invalid hosts: %s", s ? s : "");
+        free (s);
+        goto fail;
+    }
+    hostlist_destroy (hl);
+    hostlist_destroy (missing);
+    return ids;
+fail:
+    hostlist_destroy (hl);
+    hostlist_destroy (missing);
+    idset_destroy (ids);
+    return NULL;
+}
+
+int rlist_json_nodelist (const struct rlist *rl, json_t **result)
+{
+    struct hostlist *hl = rlist_nodelist (rl);
+    if (!hl)
+        return 0;
+    *result = hostlist_to_nodelist (hl);
+    hostlist_destroy (hl);
+    return 0;
+}
+
+static zhashx_t *rlist_properties (const struct rlist *rl)
+{
+    int saved_errno;
+    struct rnode *n;
+    zlistx_t *keys = NULL;
+    zhashx_t *properties = zhashx_new ();
+
+    if (!properties) {
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    zhashx_set_destructor (properties, property_destructor);
+
+    n = zlistx_first (rl->nodes);
+    while (n) {
+        if (n->properties) {
+            const char *name;
+            if (!(keys = zhashx_keys (n->properties))) {
+                errno = ENOMEM;
+                goto error;
+            }
+            name = zlistx_first (keys);
+            while (name) {
+                struct idset *ids = zhashx_lookup (properties, name);
+                if (!ids) {
+                    if (!(ids = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+                        errno = ENOMEM;
+                        goto error;
+                    }
+                    (void)zhashx_insert (properties, name, ids);
+                }
+                if (idset_set (ids, n->rank) < 0)
+                    goto error;
+                name = zlistx_next (keys);
+            }
+            zlistx_destroy (&keys);
+            keys = NULL;
+        }
+        n = zlistx_next (rl->nodes);
+    }
+
+    return properties;
+error:
+    saved_errno = errno;
+    zhashx_destroy (&properties);
+    zlistx_destroy (&keys);
+    errno = saved_errno;
+    return NULL;
+}
+
+static int rlist_json_properties (const struct rlist *rl, json_t **result)
+{
+    int saved_errno;
+    int rc = -1;
+    zhashx_t *properties = NULL;;
+    json_t *o = NULL;
+    struct idset *ids;
+
+    if (!rl || !result) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!(properties = rlist_properties (rl)))
+        return -1;
+
+    /*  Do not bother returning an empty JSON object
+     */
+    if (zhashx_size (properties) == 0) {
+        rc = 0;
+        goto out;
+    }
+
+    if (!(o = json_object ())) {
+        errno = ENOMEM;
+        goto out;
+    }
+    ids = zhashx_first (properties);
+    while (ids) {
+        const char *name = zhashx_cursor (properties);
+        char *s = idset_encode (ids, IDSET_FLAG_RANGE);
+        json_t *val = NULL;
+        if (!s
+            || !(val = json_string (s))
+            || json_object_set_new (o, name, val) < 0) {
+            free (s);
+            json_decref (val);
+            errno = ENOMEM;
+            goto out;
+        }
+        free (s);
+        ids = zhashx_next (properties);
+    }
+    /*  Assign o to result, but incref since we decref in exit path
+     */
+    json_incref (o);
+    *result = o;
+    rc = 0;
+out:
+    saved_errno = errno;
+    zhashx_destroy (&properties);
+    json_decref (o);
+    errno = saved_errno;
+    return rc;
+}
+
+char *rlist_properties_encode (const struct rlist *rl)
+{
+    char *result = NULL;
+    json_t *o = NULL;
+
+    if (rlist_json_properties (rl, &o) < 0)
+        return NULL;
+    if (o == NULL)
+        return (strdup ("{}"));
+    result = json_dumps (o, 0);
+    json_decref (o);
+    return result;
+}
+
+json_t *rlist_to_R (const struct rlist *rl)
+{
+    json_t *R = NULL;
+    json_t *R_lite = NULL;
+    json_t *nodelist = NULL;
+    json_t *properties = NULL;
+
+    if (!rl)
+        return NULL;
+
+    /*  Reset default sort to order nodes by "rank" */
+    zlistx_set_comparator (rl->nodes, by_rank);
+    zlistx_sort (rl->nodes);
+
+    if (!(R_lite = rlist_compressed (rl)))
+        goto fail;
+
+    if (rlist_json_nodelist (rl, &nodelist) < 0
+        || rlist_json_properties (rl, &properties) < 0)
+        goto fail;
+
+    if (!(R = json_pack ("{s:i, s:{s:o s:f s:f}}",
+                         "version", 1,
+                         "execution",
+                         "R_lite", R_lite,
+                         "starttime", rl->starttime,
+                         "expiration", rl->expiration)))
+        goto fail;
+    if (nodelist
+        && json_object_set_new (json_object_get (R, "execution"),
+                                "nodelist", nodelist) < 0)
+        goto fail;
+    if (properties
+        && json_object_set_new (json_object_get (R, "execution"),
+                                "properties", properties) < 0)
+        goto fail;
+    if (rl->scheduling
+        && json_object_set (R, "scheduling", rl->scheduling) < 0)
+        goto fail;
+
+    return (R);
+fail:
+    json_decref (R);
+    json_decref (nodelist);
+    return NULL;
+}
+
+char *rlist_encode (const struct rlist *rl)
+{
+    json_t *o;
+    char *R;
+    if (!rl)
+        return NULL;
+    if (!(o = rlist_to_R (rl)))
+        return NULL;
+    R = json_dumps (o, 0);
+    json_decref (o);
+    return R;
+}
+
+static int by_rank (const void *item1, const void *item2)
+{
+    const struct rnode *x = item1;
+    const struct rnode *y = item2;
+    return (x->rank - y->rank);
+}
+
+static int by_avail (const void *item1, const void *item2)
+{
+    int n;
+    const struct rnode *x = item1;
+    const struct rnode *y = item2;
+    if ((n = rnode_avail (x) - rnode_avail (y)) == 0)
+        n = by_rank (x, y);
+    return n;
+}
+
+static int by_used (const void *item1, const void *item2)
+{
+    int n;
+    const struct rnode *x = item1;
+    const struct rnode *y = item2;
+    if (x->up != y->up)
+        n = x->up ? -1 : 1;
+    else if ((n = rnode_avail (y) - rnode_avail (x)) == 0)
+        n = by_rank (x, y);
+    return n;
+}
+
+static int rlist_rnode_alloc (struct rlist *rl, struct rnode *n,
+                              int count, struct idset **idsetp)
+{
+    if (!n || rnode_alloc (n, count, idsetp) < 0)
+        return -1;
+    rl->avail -= idset_count (*idsetp);
+    return 0;
+}
+
+#if 0
+static uint32_t rlist_rnode_rank (struct rlist *rl)
+{
+    struct rnode *n = zlistx_item (rl->nodes);
+    if (n)
+        return n->rank;
+    else
+        return (uint32_t)-1;
+}
+#endif
+
+static struct rnode *rlist_first (struct rlist *rl)
+{
+    return zlistx_first (rl->nodes);
+}
+
+static struct rnode *rlist_next (struct rlist *rl)
+{
+    return zlistx_next (rl->nodes);
+}
+
+/*
+ *  Allocate the first available N slots of size cores_per_slot from
+ *   resource list rl after sorting the nodes with the current sort strategy.
+ */
+static struct rlist * rlist_alloc_first_fit (struct rlist *rl,
+                                             int cores_per_slot,
+                                             int slots)
+{
+    int rc;
+    struct idset *ids = NULL;
+    struct rnode *n = NULL;
+    struct rlist *result = NULL;
+
+    zlistx_sort (rl->nodes);
+
+    if (!(n = rlist_first (rl)))
+        return NULL;
+
+    if (!(result = rlist_create ()))
+        return NULL;
+
+    /* 2. assign slots to first nodes where they fit
+     */
+    while (n && slots) {
+        /*  Try to allocate a slot on this node. If we fail with ENOSPC,
+         *   then advance to the next node and try again.
+         */
+        if ((rc = rlist_rnode_alloc (rl, n, cores_per_slot, &ids)) < 0) {
+            if (errno != ENOSPC)
+                goto unwind;
+            n = rlist_next (rl);
+            continue;
+        }
+        /*  Append the allocated cores to the result set and continue
+         *   if needed
+         */
+        rc = rlist_append_cores (result, n->hostname, n->rank, ids);
+        idset_destroy (ids);
+        if (rc < 0)
+            goto unwind;
+        slots--;
+    }
+    if (slots != 0) {
+unwind:
+        rlist_free (rl, result);
+        rlist_destroy (result);
+        errno = ENOSPC;
+        return NULL;
+    }
+    return result;
+}
+
+/*
+ *  Allocate `slots` of size cores_per_slot from rlist `rl` and return
+ *   the result. Sorts the node list by smallest available first, so that
+ *   we get something like "best fit". (minimize nodes used)
+ */
+static struct rlist * rlist_alloc_best_fit (struct rlist *rl,
+                                            int cores_per_slot,
+                                            int slots)
+{
+    zlistx_set_comparator (rl->nodes, by_avail);
+    return rlist_alloc_first_fit (rl, cores_per_slot, slots);
+}
+
+/*
+ *  Allocate `slots` of size cores_per_slot from rlist `rl` and return
+ *   the result. Sorts the node list by least utilized first, so that
+ *   we get something like "worst fit". (Spread jobs across nodes)
+ */
+static struct rlist * rlist_alloc_worst_fit (struct rlist *rl,
+                                             int cores_per_slot,
+                                             int slots)
+{
+    zlistx_set_comparator (rl->nodes, by_used);
+    return rlist_alloc_first_fit (rl, cores_per_slot, slots);
+}
+
+
+static zlistx_t *rlist_get_nnodes (struct rlist *rl, int nnodes)
+{
+    struct rnode *n;
+    zlistx_t *l = zlistx_new ();
+    if (!l)
+        return NULL;
+    n = zlistx_first (rl->nodes);
+    while (nnodes > 0) {
+        if (n == NULL) {
+            errno = ENOSPC;
+            goto err;
+        }
+        if (n->up) {
+            if (!zlistx_add_end (l, n))
+                goto err;
+            nnodes--;
+        }
+        n = zlistx_next (rl->nodes);
+    }
+    return (l);
+err:
+    zlistx_destroy (&l);
+    return NULL;
+}
+
+/*  Allocate 'slots' of size 'cores_per_slot' across exactly `nnodes`.
+ *  Works by getting the first N least utilized nodes and spreading
+ *  the nslots evenly across the result.
+ */
+static struct rlist *rlist_alloc_nnodes (struct rlist *rl,
+                                         const struct rlist_alloc_info *ai)
+{
+    struct rlist *result = NULL;
+    struct rnode *n = NULL;
+    zlistx_t *cl = NULL;
+    int slots = ai->nslots;
+
+    if (rlist_nnodes (rl) < ai->nnodes) {
+        errno = ENOSPC;
+        return NULL;
+    }
+    if (ai->nslots < ai->nnodes) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(result = rlist_create ()))
+        return NULL;
+
+    /* 1. sort rank list by used cores ascending:
+     */
+    zlistx_set_comparator (rl->nodes, by_used);
+    zlistx_sort (rl->nodes);
+
+    if (ai->exclusive) {
+        int nleft = ai->nnodes;
+        struct rnode *cpy;
+        n = zlistx_first (rl->nodes);
+        while (n && nleft) {
+            /*
+             *  We can abort after we find the first non-idle or down node:
+             *
+             *  Note: by_used() sorts down nodes to back of list and
+             *   rnode_avail() returns 0 if node is not up
+             */
+            if (rnode_avail (n) < rnode_count (n))
+                goto unwind;
+
+            if (!(cpy = rnode_copy (n))
+                || rlist_add_rnode_new (result, cpy) < 0) {
+                rnode_destroy (cpy);
+                goto unwind;
+            }
+            rnode_alloc_idset (n, n->cores->ids);
+            nleft--;
+            n = zlistx_next (rl->nodes);
+        }
+        if (nleft) /* Unable to allocate all nodes exclusively */
+            goto unwind;
+        return result;
+    }
+
+    /* 2. get a list of the first up n nodes
+     */
+    if (!(cl = rlist_get_nnodes (rl, ai->nnodes)))
+        goto unwind;
+
+    /* We will sort candidate list by used cores on each iteration to
+     *  ensure even spread of slots across nodes
+     */
+    zlistx_set_comparator (cl, by_used);
+
+    /*
+     * 3. divide slots across all nodes, placing each slot
+     *    on most empty node first
+     */
+    while (slots > 0) {
+        int rc;
+        struct idset *ids = NULL;
+        n = zlistx_first (cl);
+        /*
+         * if we can't allocate on this node, give up. Since it is the
+         *  least loaded node from the least loaded nodelist, we know
+         *  we don't have enough resources to satisfy request.
+         */
+        if (rlist_rnode_alloc (rl, n, ai->slot_size, &ids) < 0)
+            goto unwind;
+        rc = rlist_append_cores (result, n->hostname, n->rank, ids);
+        idset_destroy (ids);
+        if (rc < 0)
+            goto unwind;
+
+        /*  If a node is empty, remove it from consideration.
+         *  O/w, force it to the back of the list to ensure all N
+         *   nodes are considered at least once.
+         */
+        zlistx_reorder (cl, zlistx_cursor (cl), false);
+        if (rnode_avail (n) == 0)
+            zlistx_detach (cl, zlistx_cursor (cl));
+        else
+            zlistx_move_end (cl, zlistx_cursor (cl));
+        slots--;
+    }
+    zlistx_destroy (&cl);
+    return result;
+unwind:
+    zlistx_destroy (&cl);
+    rlist_free (rl, result);
+    rlist_destroy (result);
+    errno = ENOSPC;
+    return NULL;
+}
+
+static struct rlist *rlist_try_alloc (struct rlist *rl,
+                                      const struct rlist_alloc_info *ai)
+{
+    struct rlist *result = NULL;
+    const char *mode = ai->mode;
+
+    if (!rl || !ai) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    /*  Reset default sort to order nodes by "rank" */
+    zlistx_set_comparator (rl->nodes, by_rank);
+
+    if (ai->nnodes > 0)
+        result = rlist_alloc_nnodes (rl, ai);
+    else if (mode == NULL || streq (mode, "worst-fit"))
+        result = rlist_alloc_worst_fit (rl, ai->slot_size, ai->nslots);
+    else if (mode && streq (mode, "best-fit"))
+        result = rlist_alloc_best_fit (rl, ai->slot_size, ai->nslots);
+    else if (mode && streq (mode, "first-fit"))
+        result = rlist_alloc_first_fit (rl, ai->slot_size, ai->nslots);
+    else
+        errno = EINVAL;
+    return result;
+}
+
+/*  Determine if allocation request is feasible for rlist `rl`.
+ */
+static bool rlist_alloc_feasible (const struct rlist *rl, const char *mode,
+                                  int nnodes, int slots, int slotsz)
+{
+    bool rc = false;
+    struct rlist *result = NULL;
+    struct rlist_alloc_info ai = {
+        .nnodes = nnodes,
+        .slot_size = slotsz,
+        .nslots = slots,
+        .mode = mode,
+    };
+    int saved_errno = errno;
+    struct rlist *all = rlist_copy_empty (rl);
+    if (all && (result = rlist_try_alloc (all, &ai)))
+        rc = true;
+    rlist_destroy (all);
+    rlist_destroy (result);
+    errno = saved_errno;
+    return rc;
+}
+
+static int alloc_info_check (struct rlist *rl,
+                             const struct rlist_alloc_info *ai,
+                             flux_error_t *errp)
+{
+    int slots = ai->nslots;
+    int nnodes = ai->nnodes;
+    int slotsz = ai->slot_size;
+    int total = slots * slotsz;
+
+    if (slots <= 0 || slotsz <= 0 || nnodes < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (ai->exclusive && ai->nnodes <= 0) {
+        errprintf (errp, "exclusive allocation only supported with nnodes");
+        errno = EINVAL;
+        return -1;
+    }
+    if (total > rl->total) {
+        errprintf (errp, "unsatisfiable request");
+        errno = EOVERFLOW;
+        return -1;
+    }
+    if (total > rl->avail) {
+        if (!rlist_alloc_feasible (rl,
+                                   ai->mode,
+                                   ai->nnodes,
+                                   ai->nslots,
+                                   ai->slot_size)) {
+            errprintf (errp, "unsatisfiable request");
+            errno = EOVERFLOW;
+        }
+        else
+            errno = ENOSPC;
+        return -1;
+    }
+    return 0;
+}
+
+static struct rlist *
+rlist_alloc_constrained (struct rlist *rl,
+                         const struct rlist_alloc_info *ai,
+                         flux_error_t *errp)
+{
+    struct rlist *result;
+    struct rlist *cpy;
+    int saved_errno;
+
+    if (!(cpy = rlist_copy_constraint (rl, ai->constraints, errp)))
+        return NULL;
+
+    if (rlist_count (cpy, "core") == 0) {
+        errprintf (errp, "no resources satisfy provided constraints");
+        errno = EOVERFLOW;
+    }
+
+    result = rlist_try_alloc (cpy, ai);
+    saved_errno = errno;
+
+    if (!result && errno == ENOSPC) {
+        if (!rlist_alloc_feasible (cpy,
+                                   ai->mode,
+                                   ai->nnodes,
+                                   ai->nslots,
+                                   ai->slot_size)) {
+            saved_errno = EOVERFLOW;
+            errprintf (errp, "unsatisfiable constrained request");
+        }
+    }
+    rlist_destroy (cpy);
+
+    if (result && rlist_set_allocated (rl, result) < 0) {
+        errprintf (errp, "rlist_set_allocated: %s", strerror (errno));
+        rlist_destroy (result);
+        result = NULL;
+    }
+
+    errno = saved_errno;
+    return result;
+}
+
+struct rlist *rlist_alloc (struct rlist *rl,
+                           const struct rlist_alloc_info *ai,
+                           flux_error_t *errp)
+{
+    struct rlist *result = NULL;
+
+    if (!rl || !ai) {
+        errno = EINVAL;
+        errprintf (errp, "Invalid argument");
+        return NULL;
+    }
+
+    if (alloc_info_check (rl, ai, errp) < 0)
+        return NULL;
+
+    if (ai->constraints)
+        result = rlist_alloc_constrained (rl, ai, errp);
+    else {
+        result = rlist_try_alloc (rl, ai);
+        if (!result)
+            errprintf (errp, "%s", strerror (errno));
+
+        if (!result && (errno == ENOSPC)) {
+            if (!rlist_alloc_feasible (rl,
+                                       ai->mode,
+                                       ai->nnodes,
+                                       ai->nslots,
+                                       ai->slot_size)) {
+                errprintf (errp, "unsatisfiable request");
+                errno = EOVERFLOW;
+            }
+        }
+    }
+    return result;
+}
+
+static int rlist_free_rnode (struct rlist *rl, struct rnode *n)
+{
+    struct rnode *rnode = rlist_find_rank (rl, n->rank);
+    if (!rnode) {
+        errno = ENOENT;
+        return -1;
+    }
+    if (rnode_free_idset (rnode, n->cores->ids) < 0)
+        return -1;
+    if (rnode->up)
+        rl->avail += idset_count (n->cores->ids);
+    return 0;
+}
+
+static int rlist_alloc_rnode (struct rlist *rl, struct rnode *n)
+{
+    struct rnode *rnode = rlist_find_rank (rl, n->rank);
+    if (!rnode) {
+        errno = ENOENT;
+        return -1;
+    }
+    if (rnode_alloc_idset (rnode, n->cores->avail) < 0)
+        return -1;
+    if (rnode->up)
+        rl->avail -= idset_count (n->cores->avail);
+    return 0;
+}
+
+int rlist_free (struct rlist *rl, struct rlist *alloc)
+{
+    zlistx_t *freed = NULL;
+    struct rnode *n = NULL;
+
+    if (!(freed = zlistx_new ()))
+        return -1;
+
+    n = zlistx_first (alloc->nodes);
+    while (n) {
+        if (rlist_free_rnode (rl, n) < 0
+            || !zlistx_add_end (freed, n))
+            goto cleanup;
+        n = zlistx_next (alloc->nodes);
+    }
+    zlistx_destroy (&freed);
+    return (0);
+cleanup:
+    /* re-allocate all freed items */
+    n = zlistx_first (freed);
+    while (n) {
+        rlist_alloc_rnode (rl, n);
+        n = zlistx_next (freed);
+    }
+    zlistx_destroy (&freed);
+    return (-1);
+}
+
+int rlist_set_allocated (struct rlist *rl, struct rlist *alloc)
+{
+    zlistx_t *allocd = NULL;
+    struct rnode *n = NULL;
+    if (!alloc || !(allocd = zlistx_new ()))
+        return -1;
+    n = zlistx_first (alloc->nodes);
+    while (n) {
+        if (rlist_alloc_rnode (rl, n) < 0
+            || !zlistx_add_end (allocd, n))
+            goto cleanup;
+        n = zlistx_next (alloc->nodes);
+    }
+    zlistx_destroy (&allocd);
+    return 0;
+cleanup:
+    n = zlistx_first (allocd);
+    while (n) {
+        rlist_free_rnode (rl, n);
+        n = zlistx_next (allocd);
+    }
+    zlistx_destroy (&allocd);
+    return -1;
+}
+
+size_t rlist_nnodes (const struct rlist *rl)
+{
+    return zlistx_size (rl->nodes);
+}
+
+size_t rlist_count (const struct rlist *rl, const char *type)
+{
+    struct rnode *n;
+    size_t count = 0;
+    n = zlistx_first (rl->nodes);
+    while (n) {
+        count += rnode_count_type (n, type);
+        n = zlistx_next (rl->nodes);
+    }
+    return count;
+}
+
+/* Mark all nodes in state 'up'. Count number of cores that changed
+ *  availability state.
+ */
+static int rlist_mark_all (struct rlist *rl, bool up)
+{
+    int count = 0;
+    struct rnode *n = zlistx_first (rl->nodes);
+    while (n) {
+        if (n->up != up)
+            count += idset_count (n->cores->avail);
+        n->up = up;
+        n = zlistx_next (rl->nodes);
+    }
+    return count;
+}
+
+static int rlist_mark_state (struct rlist *rl, bool up, const char *ids)
+{
+    int count = 0;
+    unsigned int i;
+    struct idset *idset = idset_decode (ids);
+    if (idset == NULL)
+        return -1;
+    i = idset_first (idset);
+    while (i != IDSET_INVALID_ID) {
+        struct rnode *n = rlist_find_rank (rl, i);
+        if (n) {
+            if (n->up != up)
+                count += idset_count (n->cores->avail);
+            n->up = up;
+        }
+        i = idset_next (idset, i);
+    }
+    idset_destroy (idset);
+    return count;
+}
+
+int rlist_mark_down (struct rlist *rl, const char *ids)
+{
+    int count;
+    if (streq (ids, "all"))
+        count = rlist_mark_all (rl, false);
+    else
+        count = rlist_mark_state (rl, false, ids);
+    rl->avail -= count;
+    return 0;
+}
+
+int rlist_mark_up (struct rlist *rl, const char *ids)
+{
+    int count;
+    if (streq (ids, "all"))
+        count = rlist_mark_all (rl, true);
+    else
+        count = rlist_mark_state (rl, true, ids);
+    rl->avail += count;
+    return 0;
+}
+
+/*  Check if a resource set provided by configuration is valid.
+ *  Returns -1 on failure with error in errp->text.
+ */
+static int rlist_config_check (struct rlist *rl, flux_error_t *errp)
+{
+    struct rnode *n;
+    struct hostlist *empty;
+    int rc = -1;
+
+    if (zlistx_size (rl->nodes) == 0)
+        return errprintf (errp, "no hosts configured");
+
+    if (!(empty = hostlist_create ()))
+        return errprintf (errp, "hostlist_create: Out of memory");
+
+    n = zlistx_first (rl->nodes);
+    while (n) {
+        if (rnode_avail_total (n) <= 0) {
+            if (hostlist_append (empty, n->hostname) < 0) {
+                errprintf (errp,
+                           "host %s was assigned no resources",
+                           n->hostname);
+                goto out;
+            }
+        }
+        n = zlistx_next (rl->nodes);
+    }
+    if (hostlist_count (empty) > 0) {
+        char *s = hostlist_encode (empty);
+        errprintf (errp, "resource.config: %s assigned no resources", s);
+        free (s);
+        goto out;
+    }
+    rc = 0;
+out:
+    hostlist_destroy (empty);
+    return rc;
+}
+
+/*  Process one entry from the resource.config array
+ */
+static int rlist_config_add_entry (struct rlist *rl,
+                                   struct hostlist *hostmap,
+                                   flux_error_t *errp,
+                                   int index,
+                                   const char *hosts,
+                                   const char *cores,
+                                   const char *gpus,
+                                   json_t *properties)
+{
+    struct hostlist *hl = NULL;
+    const char *host = NULL;
+    struct idset *coreids = NULL;
+    struct idset *gpuids = NULL;
+    struct idset *ranks = NULL;
+    int rc = -1;
+
+    if (!(hl = hostlist_decode (hosts))) {
+        errprintf (errp, "config[%d]: invalid hostlist '%s'", index, hosts);
+        goto error;
+    }
+    if (hostlist_count (hl) == 0) {
+        errprintf (errp, "config[%d]: empty hostlist specified", index);
+        goto error;
+    }
+    if (!(ranks = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+        errprintf (errp, "idset_create: %s", strerror (errno));
+        goto error;
+    }
+    if (cores && !(coreids = idset_decode (cores))) {
+        errprintf (errp, "config[%d]: invalid idset cores='%s'", index, cores);
+        goto error;
+    }
+    if (gpus && !(gpuids = idset_decode (gpus))) {
+        errprintf (errp, "config[%d]: invalid idset gpus='%s'", index, gpus);
+        goto error;
+    }
+    host = hostlist_first (hl);
+    while (host) {
+        struct rnode *n;
+        int rank = hostlist_find (hostmap, host);
+        if (rank < 0) {
+            /*
+             *  First time encountering this host. Append to host map
+             *   hostlist and assign a rank.
+             */
+            if (hostlist_append (hostmap, host) < 0) {
+                errprintf (errp, "failed to append %s to host map", host);
+                goto error;
+            }
+            rank = hostlist_count (hostmap) - 1;
+        }
+        if (idset_set (ranks, rank) < 0) {
+            errprintf (errp, "idset_set(ranks, %d): %s",
+                       rank,
+                       strerror (errno));
+            goto error;
+        }
+        if (!(n = rnode_new (host, rank))) {
+            errprintf (errp, "rnode_new: %s", strerror (errno));
+            goto error;
+        }
+        if (coreids && !rnode_add_child_idset (n, "core", coreids, coreids)) {
+            errprintf (errp, "rnode_add_child_idset: %s", strerror (errno));
+            goto error;
+        }
+        if (gpuids && !rnode_add_child_idset (n, "gpu", gpuids, gpuids)) {
+            errprintf (errp, "rnode_add_child_idset: %s", strerror (errno));
+            goto error;
+        }
+        if (properties) {
+            size_t idx;
+            json_t *o;
+
+            json_array_foreach (properties, idx, o) {
+                const char *property;
+                if (!(property = json_string_value (o))
+                    || property_string_invalid (property)) {
+                    char *s = json_dumps (o, JSON_ENCODE_ANY);
+                    errprintf (errp,
+                               "config[%d]: invalid property \"%s\"",
+                               index,
+                               s);
+                    free (s);
+                    goto error;
+                }
+                if (rnode_set_property (n, property) < 0) {
+                    errprintf (errp,
+                               "Failed to set property %s on rank %u",
+                                property,
+                                rank);
+                    goto error;
+                }
+            }
+        }
+        if (rlist_add_rnode (rl, n) < 0) {
+            errprintf (errp, "Unable to add rnode: %s", strerror (errno));
+            goto error;
+        }
+        host = hostlist_next (hl);
+    }
+    rc = 0;
+error:
+    hostlist_destroy (hl);
+    idset_destroy (ranks);
+    idset_destroy (coreids);
+    idset_destroy (gpuids);
+    return rc;
+}
+
+struct rlist *rlist_from_config (json_t *conf, flux_error_t *errp)
+{
+    size_t index;
+    json_t *entry;
+    struct rlist *rl = NULL;
+    struct hostlist *hl = NULL;
+
+    if (!conf || !json_is_array (conf)) {
+        errprintf (errp, "resource config must be an array");
+        return NULL;
+    }
+
+    if (!(hl = hostlist_create ())
+        || !(rl = rlist_create ())) {
+        errprintf (errp, "Out of memory");
+        goto error;
+    }
+
+    json_array_foreach (conf, index, entry) {
+        const char *hosts = NULL;
+        const char *cores = NULL;
+        const char *gpus = NULL;
+        json_t *properties = NULL;
+        json_error_t error;
+
+        if (json_unpack_ex (entry, &error, 0,
+                            "{s:s s?s s?s s?o !}",
+                            "hosts", &hosts,
+                            "cores", &cores,
+                            "gpus",  &gpus,
+                            "properties", &properties) < 0) {
+            errprintf (errp, "config[%zu]: %s", index, error.text);
+            goto error;
+        }
+        if (properties != NULL && !json_is_array (properties)) {
+            errprintf (errp,
+                       "config[%zu]: %s",
+                       index,
+                       "properties must be an array");
+            goto error;
+        }
+        if (rlist_config_add_entry (rl,
+                                    hl,
+                                    errp,
+                                    index,
+                                    hosts,
+                                    cores,
+                                    gpus,
+                                    properties) < 0)
+            goto error;
+    }
+
+    if (rlist_config_check (rl, errp) < 0)
+        goto error;
+
+    hostlist_destroy (hl);
+    return rl;
+error:
+    hostlist_destroy (hl);
+    rlist_destroy (rl);
+    return NULL;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librlist/rlist.h b/src/common/librlist/rlist.h
new file mode 100644
index 000000000000..d2fc7d6b8ad3
--- /dev/null
+++ b/src/common/librlist/rlist.h
@@ -0,0 +1,291 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_SCHED_RLIST_H
+#define HAVE_SCHED_RLIST_H 1
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <jansson.h>
+#include <flux/idset.h>
+#include <flux/hostlist.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libflux/types.h" /* flux_error_t */
+
+/* A list of resource nodes */
+struct rlist {
+    int total;
+    int avail;
+    zlistx_t *nodes;
+
+    zhashx_t *rank_index;
+
+    /*  hash of resources to ignore on remap */
+    zhashx_t *noremap;
+
+    /*  Hash of property->idset mapping */
+    zhashx_t *properties;
+
+    /*  Rv1 optional starttime, expiration:
+     */
+    double starttime;
+    double expiration;
+
+    /*  Opaque Rv1.scheduling key */
+    json_t *scheduling;
+};
+
+struct rlist_alloc_info {
+    int nnodes;
+    int slot_size;
+    int nslots;
+    const char *mode;
+    bool exclusive;
+    json_t *constraints;
+};
+
+/*  Create an empty rlist object */
+struct rlist *rlist_create (void);
+
+/*  Mark ranks down
+ */
+int rlist_mark_down (struct rlist *rl, const char *ids);
+
+/*  Mark ranks up
+ */
+int rlist_mark_up (struct rlist *rl, const char *ids);
+
+/*  Create a copy of rlist rl with all cores available */
+struct rlist *rlist_copy_empty (const struct rlist *rl);
+
+/*  Create a copy of rl including only down resources */
+struct rlist *rlist_copy_down (const struct rlist *orig);
+
+/*  Create a copy of rl including only allocated resources */
+struct rlist *rlist_copy_allocated (const struct rlist *orig);
+
+/*  Create a copy of rl including only the ranks in 'ranks' idset */
+struct rlist *rlist_copy_ranks (const struct rlist *rl, struct idset *ranks);
+
+struct rlist *rlist_copy_cores (const struct rlist *rl);
+
+/*  Create a copy of rl constrained by an RFC 31 constraint object
+ *
+ *  Returns a copy of rl with only those resource nodes that match
+ *   the provided constraint. The result 'struct rlist' may be empty
+ *   if no resources satisfy the constraint.
+ *
+ *  Returns NULL with `errp` set if the constraint object was invalid.
+ *
+ */
+struct rlist *rlist_copy_constraint (const struct rlist *rl,
+                                     json_t *constraint,
+                                     flux_error_t *errp);
+
+/*  Same as above, but takes a JSON string instead of json_t object.
+ */
+struct rlist *rlist_copy_constraint_string (const struct rlist *orig,
+                                            const char *constraint,
+                                            flux_error_t *errp);
+
+/*  Delete ranks in idset 'ranks' from rlist 'rl'
+ */
+int rlist_remove_ranks (struct rlist *rl, struct idset *ranks);
+
+/*  Re-map ranks and all resources (except those named in rl->noremap hash)
+ *   such that their IDs will be mapped 0 - count-1.
+ */
+int rlist_remap (struct rlist *rl);
+
+/*  Re-assign hostnames to rlist 'rl'. The number of hosts in the "hosts"
+ *   hostlist expression must match the size of rlist 'rl'.
+ */
+int rlist_assign_hosts (struct rlist *rl, const char *hosts);
+
+/*  Re-assign ranks based on the RFC29 hostlist 'hosts'. Ranks in 'rl'
+ *   will be remapped based on host order in 'hosts', i.e. the first
+ *   host will be rank 0, the next rank 1, and so on.
+ *
+ *  Returns 0 on success, and -1 with errno set for the following cases:
+ *   EOVERFLOW: the number of hostnames in 'hosts' is > nranks in 'rl'
+ *   ENOSPC:    the number of hostnames in 'hosts' is < nranks in 'rl'
+ *   ENOENT:    a hostname in 'hosts' was not found in 'rl'
+ *   ENOMEM:    out of memory
+ */
+int rlist_rerank (struct rlist *rl, const char *hosts, flux_error_t *error);
+
+/*  Destroy an rlist object */
+void rlist_destroy (struct rlist *rl);
+
+/*  Append a new resource node with hostname, rank, and core idset string
+ */
+int rlist_append_rank_cores (struct rlist *rl,
+                             const char *hostname,
+                             unsigned int rank,
+                             const char *core_ids);
+
+/*  Add child resource 'ids' with name 'name' to rank 'rank' in resource
+ *   list 'rl'.
+ */
+int rlist_rank_add_child (struct rlist *rl,
+                          unsigned int rank,
+                          const char *name,
+                          const char *ids);
+
+/*  Append rlist 'rl2' to 'rl'
+ */
+int rlist_append (struct rlist *rl, const struct rlist *rl2);
+
+/*  Like append, but it is not an error if resources in `rl` also
+ *   exist in `rl2`.
+ */
+int rlist_add (struct rlist *rl, const struct rlist *rl2);
+
+/*  Return the set difference of 'rlb' from 'rla'.
+ */
+struct rlist *rlist_diff (const struct rlist *rla, const struct rlist *rlb);
+
+/*  Return the union of 'rla' and 'rlb'
+ */
+struct rlist *rlist_union (const struct rlist *rla, const struct rlist *rlb);
+
+/*  Return the intersection of 'rla' and 'rlb'
+ */
+struct rlist *rlist_intersect (const struct rlist *rla,
+                              const struct rlist *rlb);
+
+/*  Return number of resource nodes in resource list `rl`
+ */
+size_t rlist_nnodes (const struct rlist *rl);
+
+size_t rlist_count (const struct rlist *rl, const char *type);
+
+
+/*  Return a hostlist of rlist hostnames
+ */
+struct hostlist * rlist_nodelist (const struct rlist *rl);
+
+/*  Return an idset of rlist ranks
+ */
+struct idset * rlist_ranks (const struct rlist *rl);
+
+
+/*  Return an idset of ranks corresponding to 'hosts' (a string encoded
+ *   in RFC29 hostlist format)
+ *
+ *  Multiple ranks may be returned per host in 'hosts' if ranks
+ *   share hostnames (e.g. multiple broker ranks per node)
+ *
+ *  Order of 'hosts' is ignored since the return type is an idset.
+ *
+ *  Returns success only if all hosts have one or more ranks in rlist.
+ *
+ *  Returns NULL on failure with error text in err if err is non-NULL.
+ */
+struct idset * rlist_hosts_to_ranks (const struct rlist *rl,
+                                     const char *hosts,
+                                     flux_error_t *err);
+
+/*
+ *  Serialize a resource list into v1 "R" format. This encodes only the
+ *   "available" ids in each resource node into execution.R_lite
+ */
+json_t * rlist_to_R (const struct rlist *rl);
+
+
+/*
+ *  Encode resource list into v1 "R" string format.
+ *  Identical to `R = rlist_to_R (rl); return json_dumps (R, 0);`.
+ */
+char *rlist_encode (const struct rlist *rl);
+
+/*
+ *  Dump short form description of rlist `rl` as a single line string.
+ *    Caller must free returned string.
+ */
+char *rlist_dumps (const struct rlist *rl);
+
+/*
+ *  De-serialize a v1 "R" format string into a new resource list object.
+ *  Returns a new resource list object on success, NULL on failure.
+ */
+struct rlist *rlist_from_R (const char *R);
+
+/*  Like rlist_from_R(), but takes a json_t * argument.
+ */
+struct rlist *rlist_from_json (json_t *o, json_error_t *err);
+
+/*  Verify resources in rlist 'actual' meet or exceed resources in
+ *   matching ranks of rlist 'expected'
+ *  Returns:
+ *
+ *    0: all resources in matching ranks of 'expected' are in 'actual'
+ *
+ *   -1: one or more resources in 'expected' do not appear in 'actual'
+ *        a human readable summary will be available in error.text if
+ *        error is non-NULL.
+ *
+ *    1: resources in 'actual' exceed those in 'expected'.
+ */
+int rlist_verify (flux_error_t *error,
+                  const struct rlist *expected,
+                  const struct rlist *actual);
+
+/*  Attempt to allocate nslots of slot_size across optional nnodes
+ *   from the resource list `rl` using algorithm `mode`.
+ *
+ *  Valid modes (nnodes == 0 only):
+ *   NULL or "worst-fit" - allocate from least-used nodes first
+ *   "best-fit"          - allocate from most-used nodes first
+ *   "first-fit"         - allocate first free slots found in rank order
+ *
+ *  Returns a new rlist representing the allocation on success,
+ *   NULL on failure with errno set.
+ *
+ *   ENOSPC - unable to fulfill allocation.
+ *   EINVAL - An argument was invalid.
+ */
+struct rlist * rlist_alloc (struct rlist *rl,
+                            const struct rlist_alloc_info *ai,
+                            flux_error_t *errp);
+
+/*  Mark rlist "alloc" as allocated in rlist "rl".
+ */
+int rlist_set_allocated (struct rlist *rl, struct rlist *alloc);
+
+/*  Free resource list `to_free` from resource list `rl`
+ */
+int rlist_free (struct rlist *rl, struct rlist *to_free);
+
+/*  Assign a single property 'name' to ranks in 'targets'
+ */
+int rlist_add_property (struct rlist *rl,
+                        flux_error_t *errp,
+                        const char *name,
+                        const char *targets);
+
+/*  Assign properties to targets
+ */
+int rlist_assign_properties (struct rlist *rl,
+                             json_t *properties,
+                             flux_error_t *errp);
+
+/*  Encode properties to a JSON string which conforms to RFC 20 properties
+ *   specification. Caller must free.
+ */
+char *rlist_properties_encode (const struct rlist *rl);
+
+struct rlist *rlist_from_config (json_t *conf, flux_error_t *errp);
+
+
+#endif /* !HAVE_SCHED_RLIST_H */
diff --git a/src/common/librlist/rlist_private.h b/src/common/librlist/rlist_private.h
new file mode 100644
index 000000000000..e3fa0fdc7724
--- /dev/null
+++ b/src/common/librlist/rlist_private.h
@@ -0,0 +1,16 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_SCHED_RLIST_PRIVATE_H
+#define HAVE_SCHED_RLIST_PRIVATE_H 1
+
+int rlist_add_rnode (struct rlist *rl, struct rnode *n);
+
+#endif /* !HAVE_SCHED_RLIST_PRIVATE_H */
diff --git a/src/common/librlist/rnode.c b/src/common/librlist/rnode.c
new file mode 100644
index 000000000000..9c3e8b1a5ef0
--- /dev/null
+++ b/src/common/librlist/rnode.c
@@ -0,0 +1,878 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <jansson.h>
+#include <flux/idset.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/ptrint/ptrint.h"
+#include "ccan/str/str.h"
+
+#include "rnode.h"
+
+static struct idset *util_idset_add_check (const struct idset *a,
+                                           const struct idset *b)
+{
+    if (idset_count (a) == 0)
+        return idset_copy (b);
+    if (idset_has_intersection (a, b)) {
+        errno = EEXIST;
+        return NULL;
+    }
+    return idset_union (a, b);
+}
+
+void rnode_destroy (struct rnode *n)
+{
+    if (n) {
+        int saved_errno = errno;
+        free (n->hostname);
+        zhashx_destroy (&n->children);
+        zhashx_destroy (&n->properties);
+        free (n);
+        errno = saved_errno;
+    }
+}
+
+static void rnode_child_destroy (struct rnode_child *c)
+{
+    if (c) {
+        int saved_errno = errno;
+        free (c->name);
+        idset_destroy (c->avail);
+        idset_destroy (c->ids);
+        free (c);
+        errno = saved_errno;
+    }
+}
+
+static struct rnode_child *rnode_child_idset (const char *name,
+                                              const struct idset *ids,
+                                              const struct idset *avail)
+{
+    struct rnode_child *c = calloc (1, sizeof (*c));
+    if (!(c->name = strdup (name)))
+        return NULL;
+     if (!(c->ids = idset_copy (ids))
+        || !(c->avail = idset_copy (avail)))
+        goto fail;
+    return c;
+fail:
+    rnode_child_destroy (c);
+    return NULL;
+}
+
+static struct rnode_child *rnode_child_copy (const struct rnode_child *c)
+{
+    return rnode_child_idset (c->name, c->ids, c->avail);
+}
+
+static int rnode_child_clear (struct rnode_child *c)
+{
+    /* clearing idsets manually is expensive. It is cheaper to destroy
+     * and recraete an empty idset. Update this code when that is no longer
+     * the case.
+     */
+    idset_destroy (c->avail);
+    idset_destroy (c->ids);
+    if (!(c->avail = idset_create (0, IDSET_FLAG_AUTOGROW))
+        || !(c->ids = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        return -1;
+    return 0;
+}
+
+static void rn_child_free (void **x)
+{
+    if (x) {
+        rnode_child_destroy (*x);
+        *x = NULL;
+    }
+}
+
+/*  Add IDs in idset 'new' to the rnode child 'c'.
+ *  It is an error if any of the IDs in 'new' are already set in 'c'.
+ */
+static int rnode_child_add_idset (struct rnode_child *c,
+                                   const struct idset *new)
+{
+    struct idset *tmp = NULL;
+    struct idset *ids = NULL;
+    struct idset *avail = NULL;
+    int rc = -1;
+
+    if (!(ids = util_idset_add_check (c->ids, new))
+        || !(avail = util_idset_add_check (c->avail, new)))
+        goto out;
+
+    tmp = c->ids;
+    c->ids = ids;
+    idset_destroy (tmp);
+    ids = NULL;
+
+    tmp = c->avail;
+    c->avail = avail;
+    idset_destroy (tmp);
+    avail = NULL;
+
+    rc = 0;
+out:
+    idset_destroy (ids);
+    idset_destroy (avail);
+    return rc;
+}
+
+/*  Add 'ids' to resource child 'name' in rnode 'n'. if a child with
+ *   'name' does not exist, then add a new child. If 'name' does exist
+ *   then add 'ids' to that child (it is an error if one or more ids
+ *   are already set in child 'name'.
+ */
+struct rnode_child * rnode_add_child_idset (struct rnode *n,
+                                            const char *name,
+                                            const struct idset *ids,
+                                            const struct idset *avail)
+{
+    struct rnode_child *c = zhashx_lookup (n->children, name);
+
+    if (c == NULL) {
+        c = rnode_child_idset (name, ids, avail);
+        if (!c || zhashx_insert (n->children, name, c) != 0) {
+            rnode_child_destroy (c);
+            return NULL;
+        }
+    }
+    else {
+        if (rnode_child_add_idset (c, ids) < 0)
+            c = NULL;
+    }
+    return c;
+}
+
+struct rnode_child * rnode_add_child (struct rnode *n,
+                                      const char *name,
+                                      const char *ids)
+{
+    struct rnode_child *c = NULL;
+    struct idset *new;
+    if (!(new = idset_decode (ids)))
+        return NULL;
+    c = rnode_add_child_idset (n, name, new, new);
+    idset_destroy (new);
+    return c;
+}
+
+/*  Create a new rnode with no child resources.
+ */
+struct rnode *rnode_new (const char *name, uint32_t rank)
+{
+    struct rnode *n = calloc (1, sizeof (*n));
+    if (n == NULL)
+        return NULL;
+    if (name && !(n->hostname = strdup (name)))
+        goto fail;
+    if (!(n->children = zhashx_new ()))
+        goto fail;
+    zhashx_set_destructor (n->children, rn_child_free);
+
+    /*  A child named "core" is always required, even if idset
+     *   is empty. Add it now:
+     */
+    if (!(n->cores = rnode_add_child (n, "core", "")))
+        goto fail;
+    n->rank = rank;
+    n->up = true;
+    return n;
+fail:
+    rnode_destroy (n);
+    return NULL;
+}
+
+int rnode_add (struct rnode *orig, struct rnode *n)
+{
+    int rc = 0;
+    struct rnode_child *c;
+    if (!orig || !n)
+        return -1;
+
+    c = zhashx_first (n->children);
+    while (c) {
+        if (!rnode_add_child_idset (orig, c->name, c->ids, c->avail))
+            return -1;
+        c = zhashx_next (n->children);
+    }
+    if (n->properties) {
+        zlistx_t *l = zhashx_keys (n->properties);
+        if (l != NULL) {
+            const char *property = zlistx_first (l);
+            while (property) {
+                if (rnode_set_property (orig, property) < 0)
+                    rc = -1;
+                property = zlistx_next (l);
+            }
+            zlistx_destroy (&l);
+        }
+        else
+            rc = -1;
+    }
+    return rc;
+}
+
+struct rnode *rnode_create (const char *name, uint32_t rank, const char *ids)
+{
+    struct rnode *n = rnode_new (name, rank);
+    if (n == NULL)
+        return NULL;
+    if (!(n->cores = rnode_add_child (n, "core", ids)))
+        goto fail;
+    assert (n->cores == zhashx_lookup (n->children, "core"));
+    return (n);
+fail:
+    rnode_destroy (n);
+    return NULL;
+}
+
+struct rnode *rnode_create_children (const char *name,
+                                     uint32_t rank,
+                                     json_t *children)
+{
+    const char *key = NULL;
+    json_t *val = NULL;
+
+    struct rnode *n = rnode_new (name, rank);
+    if (n == NULL)
+        return NULL;
+    json_object_foreach (children, key, val) {
+        const char *ids = json_string_value (val);
+        struct rnode_child *c = rnode_add_child (n, key, ids);
+        if (c == NULL)
+            goto fail;
+        if (streq (key, "core"))
+            n->cores = c;
+    }
+    return n;
+fail:
+    rnode_destroy (n);
+    return NULL;
+}
+
+struct rnode *rnode_create_idset (const char *name,
+                                  uint32_t rank,
+                                  struct idset *ids)
+{
+    struct rnode *n = rnode_new (name, rank);
+    if (n == NULL)
+        return NULL;
+    if (!(n->cores = rnode_add_child_idset (n, "core", ids, ids)))
+        goto fail;
+    return (n);
+fail:
+    rnode_destroy (n);
+    return NULL;
+}
+
+int rnode_set_property (struct rnode *n, const char *name)
+{
+    if (!n->properties && !(n->properties = zhashx_new ())) {
+        errno = ENOMEM;
+        return -1;
+    }
+    /*
+     *  zhashx_insert () supposedly returns -1 when 'name' already
+     *   exists in the hash, but setting an existing property is
+     *   not an error. Therefore, ignore this error.
+     */
+    (void) zhashx_insert (n->properties, name, int2ptr (1));
+    return 0;
+}
+
+void rnode_remove_property (struct rnode *n, const char *name)
+{
+    if (n->properties)
+        zhashx_delete (n->properties, name);
+}
+
+bool rnode_has_property (struct rnode *n, const char *name)
+{
+    return (n->properties && zhashx_lookup (n->properties, name));
+}
+
+static int rnode_set_empty (struct rnode *n)
+{
+    int count = 0;
+    struct rnode_child *c = zhashx_first (n->children);
+    while (c) {
+        idset_destroy (c->avail);
+        if (!(c->avail = idset_copy (c->ids)))
+            return -1;
+        count += idset_count (c->ids);
+        c = zhashx_next (n->children);
+    }
+    return count;
+}
+
+static int rnode_set_alloc (struct rnode *n)
+{
+    int count = 0;
+    struct rnode_child *c;
+
+    if (!n)
+        return -1;
+
+    c = zhashx_first (n->children);
+    while (c) {
+        if (idset_subtract (c->ids, c->avail) < 0)
+            return -1;
+        idset_destroy (c->avail);
+        if (!(c->avail = idset_copy (c->ids)))
+            return -1;
+        count += idset_count (c->ids);
+        c = zhashx_next (n->children);
+    }
+    return count;
+}
+
+static int rnode_set_avail (struct rnode *n)
+{
+    int count = 0;
+    struct rnode_child *c;
+
+    if (!n)
+        return -1;
+
+    c = zhashx_first (n->children);
+    while (c) {
+        idset_destroy (c->ids);
+        if (!(c->ids = idset_copy (c->avail)))
+            return -1;
+        count += idset_count (c->ids);
+        c = zhashx_next (n->children);
+    }
+    return count;
+}
+
+static zhashx_t *rnode_children_copy (const struct rnode *n)
+{
+    zhashx_t *copy;
+
+    /*  Temporarily set duplicator on orig.
+     *  (if duplicator permanently set then zhashx_insert() also
+     *   duplicates items, which we don't want
+     */
+    zhashx_set_duplicator (n->children,
+                           (zhashx_duplicator_fn *) rnode_child_copy);
+    copy = zhashx_dup (n->children);
+    zhashx_set_destructor (copy, rn_child_free);
+    zhashx_set_duplicator (copy, NULL);
+    zhashx_set_duplicator (n->children, NULL);
+    return copy;
+}
+
+struct rnode *rnode_copy (const struct rnode *orig)
+{
+    struct rnode *n = rnode_new (orig->hostname, orig->rank);
+    if (!n)
+        return NULL;
+    zhashx_destroy (&n->children);
+    if (!(n->children = rnode_children_copy (orig)))
+        goto fail;
+    if (!(n->cores = zhashx_lookup (n->children, "core")))
+        goto fail;
+    if (orig->properties
+        && !(n->properties = zhashx_dup (orig->properties)))
+        goto fail;
+    return n;
+fail:
+    rnode_destroy (n);
+    return NULL;
+}
+
+struct rnode *rnode_copy_cores (const struct rnode *orig)
+{
+    struct rnode *n = rnode_copy (orig);
+    if (n) {
+        const char *name;
+        zlistx_t *keys = zhashx_keys (n->children);
+        if (!keys)
+            goto error;
+        name = zlistx_first (keys);
+        while (name) {
+            if (!streq (name, "core"))
+                zhashx_delete (n->children, name);
+            name = zlistx_next (keys);
+        }
+        zlistx_destroy (&keys);
+        return n;
+    }
+error:
+    rnode_destroy (n);
+    return NULL;
+}
+
+struct rnode *rnode_copy_empty (const struct rnode *orig)
+{
+    struct rnode *n = rnode_copy (orig);
+    if (!n || rnode_set_empty (n) <= 0)
+        goto fail;
+    return n;
+fail:
+    rnode_destroy (n);
+    return NULL;
+}
+
+struct rnode *rnode_copy_avail (const struct rnode *orig)
+{
+    struct rnode *n = rnode_copy (orig);
+    if (!n || rnode_set_avail (n) <= 0)
+        goto fail;
+    return n;
+fail:
+    rnode_destroy (n);
+    return NULL;
+}
+
+struct rnode *rnode_copy_alloc (const struct rnode *orig)
+{
+    struct rnode *n = rnode_copy (orig);
+    if (!n || rnode_set_alloc (n) <= 0)
+        goto fail;
+    return n;
+fail:
+    rnode_destroy (n);
+    return NULL;
+}
+
+bool rnode_empty (const struct rnode *n)
+{
+    int count = 0;
+    struct rnode_child *c;
+
+    c = zhashx_first (n->children);
+    while (c) {
+        count += idset_count (c->ids);
+        c = zhashx_next (n->children);
+    }
+    return count == 0;
+}
+
+struct rnode *rnode_diff (const struct rnode *a, const struct rnode *b)
+{
+    struct rnode_child *c;
+    struct rnode *n = rnode_copy (a);
+    if (!n)
+        return NULL;
+    c = zhashx_first (b->children);
+    while (c) {
+        struct rnode_child *nc = zhashx_lookup (n->children, c->name);
+        if (nc) {
+            if (idset_equal (nc->ids, c->ids)) {
+                /* Optimization: if nc->ids == c->ids, then just replace
+                 * nc with empty idsets. This will be much faster than
+                 * idset_subtract().
+                 */
+                if (rnode_child_clear (nc) < 0)
+                    goto err;
+            }
+            else if (idset_subtract (nc->ids, c->ids) < 0
+                     || idset_subtract (nc->avail, c->avail) < 0)
+                goto err;
+
+            /*  For non-core resources, remove empty sets:
+             */
+            if (!streq (nc->name, "core")
+                && idset_count (nc->ids) == 0)
+                zhashx_delete (n->children, nc->name);
+        }
+        c = zhashx_next (b->children);
+    }
+    return n;
+err:
+    rnode_destroy (n);
+    return NULL;
+}
+
+int rnode_alloc (struct rnode *n, int count, struct idset **setp)
+{
+    struct idset *ids = NULL;
+    unsigned int i;
+    if (!n->up) {
+        errno = EHOSTDOWN;
+        return -1;
+    }
+    if (idset_count (n->cores->avail) < count) {
+        errno = ENOSPC;
+        return -1;
+    }
+    if (!(ids = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        return -1;
+    i = idset_first (n->cores->avail);
+    while (count--) {
+        idset_set (ids, i);
+        idset_clear (n->cores->avail, i);
+        i = idset_next (n->cores->avail, i);
+    }
+    if (setp != NULL)
+        *setp = ids;
+    return (0);
+}
+
+/*
+ *  Test if idset `ids` is a valid set of ids to allocate from the rnode `n`
+ *  Return true if the idset is valid, false otherwise.
+ */
+static bool alloc_ids_valid (struct rnode *n, struct idset *ids)
+{
+    unsigned int i = idset_first (ids);
+    while (i != IDSET_INVALID_ID) {
+        if (!idset_test (n->cores->ids, i)) {
+            errno = ENOENT;
+            return false;
+        }
+        if (!idset_test (n->cores->avail, i)) {
+            errno = EEXIST;
+            return false;
+        }
+        i = idset_next (ids, i);
+    }
+    return (true);
+}
+
+int rnode_alloc_idset (struct rnode *n, struct idset *ids)
+{
+    unsigned int i;
+    if (!ids) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!alloc_ids_valid (n, ids))
+        return -1;
+    i = idset_first (ids);
+    while (i != IDSET_INVALID_ID) {
+        idset_clear (n->cores->avail, i);
+        i = idset_next (ids, i);
+    }
+    return 0;
+}
+
+/*
+ *  Test if idset `ids` is a valid set of ids to free from the rnode `n`
+ *  Return true if the idset is valid, false otherwise.
+ */
+static bool free_ids_valid (struct rnode *n, struct idset *ids)
+{
+    unsigned int i = idset_first (ids);
+    while (i != IDSET_INVALID_ID) {
+        if (!idset_test (n->cores->ids, i)) {
+            errno = ENOENT;
+            return false;
+        }
+        if (idset_test (n->cores->avail, i)) {
+            errno = EEXIST;
+            return false;
+        }
+        i = idset_next (ids, i);
+    }
+    return (true);
+}
+
+int rnode_free_idset (struct rnode *n, struct idset *ids)
+{
+    unsigned int i;
+    if (!ids) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!free_ids_valid (n, ids))
+        return -1;
+    i = idset_first (ids);
+    while (i != IDSET_INVALID_ID) {
+        idset_set (n->cores->avail, i);
+        i = idset_next (ids, i);
+    }
+    return 0;
+}
+
+int rnode_free (struct rnode *n, const char *s)
+{
+    int saved_errno;
+    struct idset *ids = idset_decode (s);
+    int rc = rnode_free_idset (n, ids);
+    saved_errno = errno;
+    idset_destroy (ids);
+    errno = saved_errno;
+    return (rc);
+}
+
+int rnode_avail_total (const struct rnode *n)
+{
+    int count = 0;
+    struct rnode_child *c;
+
+    if (!n->up)
+        return 0;
+
+    c = zhashx_first (n->children);
+    while (c) {
+        count += idset_count (c->avail);
+        c = zhashx_next (n->children);
+    }
+    return count;
+}
+
+size_t rnode_avail (const struct rnode *n)
+{
+    if (n->up)
+        return (idset_count (n->cores->avail));
+    return 0;
+}
+
+size_t rnode_count (const struct rnode *n)
+{
+    return (idset_count (n->cores->ids));
+}
+
+size_t rnode_count_type (const struct rnode *n, const char *type)
+{
+    struct rnode_child *c = zhashx_lookup (n->children, type);
+    if (c)
+        return (idset_count (c->ids));
+    return 0;
+}
+
+/*  Compare two values from idset_first()/idset_next():
+ *  Returns:
+ *    0   : if x == y
+ *    > 0 : if x > y
+ *    < 0 : if x < y
+ *
+ *  Where IDSET_INVALID_ID is considered to come before all numbers.
+ */
+static int idset_val_cmp (unsigned int x, unsigned int y)
+{
+    if (x == y)
+        return 0;
+    else if (x == IDSET_INVALID_ID)
+        return -1;
+    else if (y == IDSET_INVALID_ID)
+        return  1;
+    else
+        return (x - y);
+}
+
+static int idset_cmp (struct idset *set1, struct idset *set2)
+{
+    int rv = 0;
+    if (!idset_equal (set1, set2)) {
+        /*  Sort on the first non-equal integer (see idset_val_cmp())
+         */
+        unsigned int a = idset_first (set1);
+        unsigned int b = idset_first (set2);
+        while ((rv = idset_val_cmp (a, b)) == 0) {
+            a = idset_next (set1, a);
+            b = idset_next (set2, b);
+        }
+    }
+    return rv;
+}
+
+int rnode_cmp (const struct rnode *a, const struct rnode *b)
+{
+    int rv = 0;
+    /*  All avail idsets must match */
+    struct rnode_child *ca;
+    struct rnode_child *cb;
+
+    /* If two nodes do not have the same number of children types,
+     *  then they are not identical. Order doesn't matter here...
+     */
+    if (zhashx_size (a->children) != zhashx_size (b->children))
+        return -1;
+    ca = zhashx_first (a->children);
+    while (ca) {
+        if (!(cb = zhashx_lookup (b->children, ca->name)))
+            return -1;
+        if ((rv = idset_cmp (ca->avail, cb->avail)) != 0)
+            break;
+        ca = zhashx_next (a->children);
+    }
+    return rv;
+}
+
+struct rnode_child * rnode_child_intersect (const struct rnode_child *a,
+                                            const struct rnode_child *b)
+{
+    struct rnode_child *c = NULL;
+    struct idset *ids = idset_intersect (a->ids, b->ids);
+    struct idset *avail = idset_intersect (a->avail, b->avail);
+    if (!ids || !avail)
+        goto out;
+    if (!idset_count (ids) && !idset_count (avail))
+        goto out;
+    c = rnode_child_idset (a->name, ids, avail);
+out:
+    idset_destroy (ids);
+    idset_destroy (avail);
+    return c;
+}
+
+int rnode_hostname_cmp (const struct rnode *a,
+                        const struct rnode *b)
+{
+    /*  N.B.: missing hostname is ignored for now for backwards
+     *   compatibility.
+     */
+    if (!a->hostname || !b->hostname)
+        return 0;
+    return (strcmp (a->hostname, b->hostname));
+}
+
+struct rnode *rnode_intersect (const struct rnode *a,
+                               const struct rnode *b)
+{
+    struct rnode *result = NULL;
+    struct rnode_child *cb = NULL;
+    struct rnode_child *ca = NULL;
+
+    if (!a || !b)
+        return NULL;
+
+    if (a->rank != b->rank
+        || rnode_hostname_cmp (a, b) != 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(result = rnode_new (a->hostname, a->rank)))
+        return NULL;
+
+    ca = zhashx_first (a->children);
+    while (ca) {
+        if ((cb = zhashx_lookup (b->children, ca->name))) {
+            struct rnode_child *c = rnode_child_intersect (ca, cb);
+            if (c &&
+                rnode_add_child_idset (result, c->name, c->ids, c->avail) < 0)
+                goto err;
+            rnode_child_destroy (c);
+        }
+        ca = zhashx_next (a->children);
+    }
+    return result;
+err:
+    rnode_destroy (result);
+    return NULL;
+}
+
+static int rnode_child_remap (struct rnode_child *c)
+{
+    unsigned int i;
+    unsigned int n;
+    size_t count = idset_count (c->ids);
+
+    /*  No need to remap if no IDs are set for this child, or
+     *   c->ids is already [0, count-1]
+     */
+    if (count == 0 
+        || (idset_first (c->ids) == 0
+            && idset_last (c->ids) == count - 1))
+        return 0;
+    
+    /*  First, remap c->avail using c->ids as a reference
+     */
+    n = 0;
+    i = idset_first (c->ids);
+    while (i != IDSET_INVALID_ID) {
+        if (idset_test (c->avail, i)) {
+            idset_clear (c->avail, i);
+            idset_set (c->avail, n);
+        }
+        i = idset_next (c->ids, i);
+        n++;
+    }
+
+    /*  Now, remap c->ids by simply setting ids 0 - count-1
+     */
+    if (idset_range_clear (c->ids, 0, idset_last (c->ids)) < 0)
+        return -1;
+    if (idset_range_set (c->ids, 0, count - 1) < 0)
+        return -1;
+    return 0;
+}
+
+int rnode_remap (struct rnode *n, zhashx_t *noremap)
+{
+    struct rnode_child *c;
+
+    c = zhashx_first (n->children);
+    while (c) {
+        if (!noremap || !zhashx_lookup (noremap, c->name))
+            if (rnode_child_remap (c) < 0)
+                return -1;
+        c = zhashx_next (n->children);
+    }
+    return 0;
+}
+
+static json_t *children_encode (const struct rnode *n)
+{
+    struct rnode_child *c;
+    json_t *o = json_object ();
+
+    c = zhashx_first (n->children);
+    while (c) {
+        if (idset_count (c->avail) > 0) {
+            char *ids = idset_encode (c->avail, IDSET_FLAG_RANGE);
+            json_t *val;
+            if (!ids)
+                goto fail;
+            if (!(val = json_string (ids))) {
+                free (ids);
+                goto fail;
+            }
+            free (ids);
+            if (json_object_set_new (o, c->name, val) < 0)
+                goto fail;
+        }
+        c = zhashx_next (n->children);
+    }
+    return o;
+fail:
+    json_decref (o);
+    return NULL;
+}
+
+json_t *rnode_encode (const struct rnode *n, const struct idset *ids)
+{
+    char *ranks = NULL;
+    json_t *children = NULL;
+    json_t *o = NULL;
+
+    if (!(ranks = idset_encode (ids, IDSET_FLAG_RANGE)))
+        return NULL;
+    if (!(children = children_encode (n)))
+        goto done;
+    if (!(o = json_pack ("{s:s s:o}",
+                         "rank", ranks,
+                         "children", children)))
+        goto done;
+    children = NULL;
+done:
+    json_decref (children);
+    free (ranks);
+    return o;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librlist/rnode.h b/src/common/librlist/rnode.h
new file mode 100644
index 000000000000..917547c4a89d
--- /dev/null
+++ b/src/common/librlist/rnode.h
@@ -0,0 +1,169 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_SCHED_RNODE_H
+#define HAVE_SCHED_RNODE_H 1
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <inttypes.h>
+#include <jansson.h>
+#include <flux/idset.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+struct rnode_child {
+    char *name;
+    struct idset *ids;
+    struct idset *avail;
+};
+
+/* Simple resource node object */
+struct rnode {
+    bool up;
+    char *hostname;
+    uint32_t rank;
+
+    struct rnode_child *cores;
+
+    /* non-core children */
+    zhashx_t *children;
+
+    zhashx_t *properties;
+};
+
+struct rnode *rnode_new (const char *name, uint32_t rank);
+
+struct rnode_child * rnode_add_child_idset (struct rnode *n,
+                                            const char *name,
+                                            const struct idset *ids,
+                                            const struct idset *avail);
+
+/*  Create a resource node object from an existing idset `set`
+ */
+struct rnode *rnode_create_idset (const char *name,
+                                  uint32_t rank,
+                                  struct idset *ids);
+
+/*  Create a resource node from a string representation of an idset.
+ */
+struct rnode *rnode_create (const char *name,
+                            uint32_t rank,
+                            const char *ids);
+
+/*  Create a resource node with `count` ids, starting at 0, i.e.
+ *   same as rnode_create (rank, "0-"..count-1).
+ */
+struct rnode *rnode_create_count (const char *name,
+                                  uint32_t rank,
+                                  int count);
+
+struct rnode *rnode_create_children (const char *name,
+                                     uint32_t rank,
+                                     json_t *children);
+
+struct rnode_child *rnode_add_child (struct rnode *n,
+                                     const char *name,
+                                     const char *ids);
+
+/*  Copy rnode 'n' */
+struct rnode *rnode_copy (const struct rnode *n);
+
+/*  Copy rnode 'n' with all allocated resources cleared */
+struct rnode *rnode_copy_empty (const struct rnode *n);
+
+/*  Copy only available resources from rnode 'n' */
+struct rnode *rnode_copy_avail (const struct rnode *n);
+
+/*  Copy only allocated resources from rnode 'n' */
+struct rnode *rnode_copy_alloc (const struct rnode *n);
+
+/*  Copy only all cores in rnode 'n' */
+struct rnode *rnode_copy_cores (const struct rnode *n);
+
+int rnode_add (struct rnode *orig, struct rnode *n);
+
+/*  Return an rnode object that is the set difference of 'b' from 'a'.
+ */
+struct rnode *rnode_diff (const struct rnode *a, const struct rnode *b);
+
+/*  Return an rnode object that is the set intersection of 'a' and 'b'.
+ */
+struct rnode *rnode_intersect (const struct rnode *a, const struct rnode *b);
+
+/*  Return true if rnode is empty -- i.e. it has no resources
+ */
+bool rnode_empty (const struct rnode *n);
+
+/*  Destroy rnode object
+ */
+void rnode_destroy (struct rnode *n);
+
+/*  Allocate `count` ids from rnode object `n`.
+ *   On success, return 0 and allocated ids in `setp`.
+ *   On failure, return -1 with errno set:
+ *   ENOSPC - there are not `count` ids available in `n`
+ *   EINVAL - Invalid arguments
+ */
+int rnode_alloc (struct rnode *n, int count, struct idset **setp);
+
+/*  Free the idset `ids` from resource node `n`.
+ *  Returns 0 on success, -1 if one or more ids is not allocated.
+ */
+int rnode_free (struct rnode *n, const char *ids);
+
+/*  As above, but free a struct idset instead of string.
+ */
+int rnode_free_idset (struct rnode *n, struct idset *ids);
+
+/*  Allocate specific idset `ids` from a resource node
+ */
+int rnode_alloc_idset (struct rnode *n, struct idset *ids);
+
+/*  Return the number of core ids available in resource node `n`.
+ */
+size_t rnode_avail (const struct rnode *n);
+
+/*  Return total of all available resources in 'n'. Returns 0 if node is
+ *    not in the up state.
+ */
+int rnode_avail_total (const struct rnode *n);
+
+/*  Return the total number of ids in resource node `n`.
+ */
+size_t rnode_count (const struct rnode *n);
+
+/*  Return the total number of ids for resource type in node 'n'
+ */
+size_t rnode_count_type (const struct rnode *n, const char *type);
+
+int rnode_cmp (const struct rnode *a, const struct rnode *b);
+
+/*  Return 0 if hostnames of rnode 'a' and 'b' match.
+ */
+int rnode_hostname_cmp (const struct rnode *a, const struct rnode *b);
+
+/*  Remap all resource IDs in rnode 'n' to zero origin
+ */
+int rnode_remap (struct rnode *n, zhashx_t *noremap);
+
+json_t *rnode_encode (const struct rnode *n, const struct idset *ids);
+
+/*  Set/remove/check for rnode properties
+ */
+int rnode_set_property (struct rnode *n, const char *name);
+
+void rnode_remove_property (struct rnode *n, const char *name);
+
+bool rnode_has_property (struct rnode *n, const char *name);
+
+#endif /* !HAVE_SCHED_RNODE_H */
diff --git a/src/common/librlist/test/match.c b/src/common/librlist/test/match.c
new file mode 100644
index 000000000000..ae092ec4a57f
--- /dev/null
+++ b/src/common/librlist/test/match.c
@@ -0,0 +1,314 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+
+#include "src/common/libtap/tap.h"
+#include "match.h"
+
+struct match_test {
+    const char *desc;
+    const char *json;
+    bool result;
+};
+
+struct validate_test {
+    const char *desc;
+    const char *json;
+    bool rc;
+    const char *err;
+};
+
+/*  These tests all assume a rnode object foo0 with properties xx and yy.
+ */
+struct match_test match_tests[] = {
+    { "empty json object matches everything", "{}", true },
+    { "empty and object matches everything",
+      "{ \"and\": [] }",
+      true,
+    },
+    { "empty or object matches everything",
+      "{ \"or\": [] }",
+      true,
+    },
+    { "empty not object matches nothing",
+      "{ \"not\": [] }",
+      false,
+    },
+    { "hostname property matches",
+       "{\"properties\": [\"foo0\"]}",
+       true
+    },
+    {  "empty properties dict matches everything",
+       "{\"properties\": [] }",
+       true
+    },
+    {  "property matches",
+       "{\"properties\": [\"xx\"]}",
+        true,
+    },
+    {  "logical not on property",
+       "{\"properties\": [\"^xx\"]}",
+        false,
+    },
+    {  "logical not on unset property",
+       "{\"properties\": [\"^zz\"]}",
+        true,
+    },
+    {  "property list matches like 'and'",
+       "{\"properties\": [\"xx\", \"yy\"]}",
+        true,
+    },
+    {  "property list match fails unless node has all",
+       "{\"properties\": [\"xx\", \"zz\"]}",
+        false,
+    },
+    { "property list match fails if property missing",
+       "{\"properties\": [\"zz\"]}",
+        false,
+    },
+    { "and with two true statements",
+      "{\"and\": [ {\"properties\": [\"xx\"]}, \
+                   {\"properties\": [\"yy\"]}  \
+                 ]}",
+        true,
+    },
+    { "and with one false statement",
+      "{\"and\": [ {\"properties\": [\"xx\"]}, \
+                   {\"properties\": [\"zz\"]}  \
+                 ]}",
+        false,
+    },
+    { "or with two true statements",
+      "{\"or\": [ {\"properties\": [\"xx\"]}, \
+                  {\"properties\": [\"yy\"]}  \
+                 ]}",
+        true,
+    },
+    { "or with one true statements",
+      "{\"or\": [ {\"properties\": [\"zz\"]}, \
+                  {\"properties\": [\"yy\"]}  \
+                 ]}",
+        true,
+    },
+    { "or with two false statements",
+      "{\"or\": [ {\"properties\": [\"zz\"]}, \
+                  {\"properties\": [\"aa\"]}  \
+                 ]}",
+        false,
+    },
+    { "not with or with one true statement",
+      "{\"not\": [ \
+        {\"or\": [ {\"properties\": [\"zz\"]}, \
+                   {\"properties\": [\"yy\"]}  \
+                 ]} \
+        ] \
+       }",
+       false,
+    },
+    { "hostlist operator works",
+      "{\"hostlist\": [\"foo[0-2]\"]}",
+      true,
+    },
+    { "hostlist operator works with non-matching hostlist",
+      "{\"hostlist\": [\"foo[1-3]\"]}",
+      false,
+    },
+    { "ranks operator works",
+      "{\"ranks\": [\"0,2\", \"1\"]}",
+      true,
+    },
+    { "ranks operator works with non-matching rank",
+      "{\"ranks\": [\"1-3\"]}",
+      false,
+    },
+    { NULL, NULL, false },
+};
+
+struct validate_test validate_tests[] = {
+    { "non-object fails",
+      "[]",
+      false,
+      "constraint must be JSON object",
+    },
+    { "Unknown operation fails",
+      "{ \"foo\": [] }",
+      false,
+      "unknown constraint operator: foo",
+    },
+    { "non-array argument to 'and' fails",
+      "{ \"and\": \"foo\" }",
+      false,
+      "and operator value must be an array",
+    },
+    { "non-array argument to 'or' fails",
+      "{ \"or\": \"foo\" }",
+      false,
+      "or operator value must be an array",
+    },
+    { "non-array argument to 'properties' fails",
+      "{ \"properties\": \"foo\" }",
+      false,
+      "properties value must be an array",
+    },
+    { "non-string property fails",
+      "{ \"properties\": [ \"foo\", 42 ] }",
+      false,
+      "non-string property specified",
+    },
+    { "invalid property string fails",
+      "{ \"properties\": [ \"foo\", \"bar&\" ] }",
+      false,
+      "invalid character '&' in property \"bar&\"",
+    },
+    { "empty object is valid constraint",
+      "{}",
+      true,
+      NULL
+    },
+    { "empty and object is valid constraint",
+      "{ \"and\": [] }",
+      true,
+      NULL
+    },
+    { "empty or object is valid constraint",
+      "{ \"or\": [] }",
+      true,
+      NULL
+    },
+    { "empty not object is valid constraint",
+      "{ \"not\": [] }",
+      true,
+      NULL
+    },
+    { "empty properties object is valid constraint",
+      "{ \"properties\": [] }",
+      true,
+      NULL
+    },
+    { "complex conditional works",
+      "{ \"and\": \
+         [ { \"or\": \
+             [ {\"properties\": [\"foo\"]}, \
+               {\"properties\": [\"bar\"]}  \
+             ] \
+           }, \
+           { \"and\": \
+             [ {\"properties\": [\"xx\"]}, \
+               {\"properties\": [\"yy\"]}  \
+             ] \
+           } \
+         ] \
+      }",
+      true,
+      NULL
+    },
+    { "hostlist can be included",
+      "{\"hostlist\": [\"foo[0-10]\"]}",
+      true,
+      NULL
+    },
+    { "invalid hostlist fails",
+      "{\"hostlist\": [\"foo0-10]\"]}",
+      false,
+      "invalid hostlist 'foo0-10]' in [\"foo0-10]\"]",
+    },
+    { "ranks can be included",
+      "{\"ranks\": [\"0-10\"]}",
+      true,
+      NULL
+    },
+    { "invalid ranks entry fails",
+      "{\"ranks\": [\"5,1-3\"]}",
+      false,
+      "invalid idset '5,1-3' in [\"5,1-3\"]",
+    },
+    { NULL, NULL, 0, NULL }
+};
+
+void test_match ()
+{
+    struct rnode *n;
+    if (!(n = rnode_create ("foo0", 0, "0-3")))
+        BAIL_OUT ("failed to create rnode object");
+
+    ok (rnode_set_property (n, "xx") == 0,
+        "rnode_set_property: foo0 has xx");
+    ok (rnode_set_property (n, "yy") == 0,
+        "rnode_set_property: foo0 has yy");
+
+    struct match_test *t = match_tests;
+    while (t->desc) {
+        flux_error_t error;
+        struct job_constraint *c;
+        json_t *o = json_loads (t->json, 0, NULL);
+        if (!o)
+            BAIL_OUT ("failed to parse json logic for '%s'", t->desc);
+        if (!(c = job_constraint_create (o, &error)))
+            BAIL_OUT ("%s: job_constraint_create: %s", t->desc, error.text);
+        ok (rnode_match (n, c) == t->result, "%s", t->desc);
+        json_decref (o);
+        job_constraint_destroy (c);
+        t++;
+    }
+    rnode_destroy (n);
+}
+
+void test_validate ()
+{
+    flux_error_t error;
+    struct validate_test *t = validate_tests;
+    while (t->desc) {
+        struct job_constraint *c;
+        json_t *o = json_loads (t->json, 0, NULL);
+        if (!o)
+            BAIL_OUT ("failed to parse json logic for '%s'", t->desc);
+        bool result = ((c = job_constraint_create (o, &error)) != NULL);
+        ok (result == t->rc, "%s", t->desc);
+        if (result != t->rc)
+            diag ("%s", error.text);
+        if (t->err)
+            is (error.text, t->err, "got expected error: %s", error.text);
+        json_decref (o);
+        job_constraint_destroy (c);
+        t++;
+    }
+}
+
+void test_invalid ()
+{
+    json_t *o = json_object ();
+    struct job_constraint *c = job_constraint_create (o, NULL);
+    if (!o || !c)
+        BAIL_OUT ("test_invalid: json_object() failed");
+    ok (!rnode_match (NULL, NULL),
+        "rnode_match (NULL, NULL) returns false");
+    ok (!rnode_match (NULL, c),
+        "rnode_match (NULL, c) returns false");
+    ok (rnode_copy_match (NULL, NULL) == NULL,
+        "rnode_copy_match (NULL, NULL) returns NULL");
+    json_decref (o);
+    job_constraint_destroy (c);
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+    test_match ();
+    test_validate ();
+    test_invalid ();
+    done_testing ();
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librlist/test/rhwloc.c b/src/common/librlist/test/rhwloc.c
new file mode 100644
index 000000000000..ab3684b1d03b
--- /dev/null
+++ b/src/common/librlist/test/rhwloc.c
@@ -0,0 +1,826 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/hostlist.h>
+
+#include "src/common/libtap/tap.h"
+#include "rlist.h"
+#include "rhwloc.h"
+
+const char xml1[] = "\
+<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
+<!DOCTYPE topology SYSTEM \"hwloc.dtd\">\n\
+<topology>\n\
+  <object type=\"Machine\" os_index=\"0\" cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff\" complete_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff\" online_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff\" allowed_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff\" nodeset=\"0xf0000000,,,,,,,0x00000101\" complete_nodeset=\"0xf0000000,,,,,,,0x00000101\" allowed_nodeset=\"0xf0000000,,,,,,,0x00000101\">\n\
+    <page_type size=\"65536\" count=\"0\"/>\n\
+    <page_type size=\"2097152\" count=\"0\"/>\n\
+    <page_type size=\"1073741824\" count=\"0\"/>\n\
+    <info name=\"PlatformName\" value=\"PowerNV\"/>\n\
+    <info name=\"PlatformModel\" value=\"PowerNV 8335-GTW\"/>\n\
+    <info name=\"Backend\" value=\"Linux\"/>\n\
+    <info name=\"LinuxCgroup\" value=\"/allocation_54106\"/>\n\
+    <info name=\"OSName\" value=\"Linux\"/>\n\
+    <info name=\"OSRelease\" value=\"4.14.0-49.6.1.el7a.ppc64le\"/>\n\
+    <info name=\"OSVersion\" value=\"#1 SMP Wed May 16 21:05:05 UTC 2018\"/>\n\
+    <info name=\"HostName\" value=\"sierra3179\"/>\n\
+    <info name=\"Architecture\" value=\"ppc64le\"/>\n\
+    <info name=\"hwlocVersion\" value=\"1.11.10\"/>\n\
+    <info name=\"ProcessName\" value=\"hwloc_xml\"/>\n\
+    <distances nbobjs=\"6\" relative_depth=\"2\" latency_base=\"10.000000\">\n\
+      <latency value=\"1.000000\"/>\n\
+      <latency value=\"4.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"4.000000\"/>\n\
+      <latency value=\"1.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"1.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"1.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"1.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"8.000000\"/>\n\
+      <latency value=\"1.000000\"/>\n\
+    </distances>\n\
+    <object type=\"Group\" cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff\" complete_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff\" online_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff\" allowed_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff\" nodeset=\"0x00000101\" complete_nodeset=\"0x00000101\" allowed_nodeset=\"0x00000101\" depth=\"0\">\n\
+      <object type=\"NUMANode\" os_index=\"0\" cpuset=\"0x00ffffff,0xffffffff,0xffffffff\" complete_cpuset=\"0x00ffffff,0xffffffff,0xffffffff\" online_cpuset=\"0x00ffffff,0xffffffff,0xffffffff\" allowed_cpuset=\"0x00ffffff,0xffffffff,0xffffffff\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" local_memory=\"137132310528\">\n\
+        <page_type size=\"65536\" count=\"2092473\"/>\n\
+        <page_type size=\"2097152\" count=\"0\"/>\n\
+        <page_type size=\"1073741824\" count=\"0\"/>\n\
+        <object type=\"Package\" os_index=\"0\" cpuset=\"0x00ffffff,0xffffffff,0xffffffff\" complete_cpuset=\"0x00ffffff,0xffffffff,0xffffffff\" online_cpuset=\"0x00ffffff,0xffffffff,0xffffffff\" allowed_cpuset=\"0x00ffffff,0xffffffff,0xffffffff\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+          <info name=\"CPUModel\" value=\"POWER9, altivec supported\"/>\n\
+          <info name=\"CPURevision\" value=\"2.1 (pvr 004e 1201)\"/>\n\
+          <object type=\"Cache\" cpuset=\"0x000000ff\" complete_cpuset=\"0x000000ff\" online_cpuset=\"0x000000ff\" allowed_cpuset=\"0x000000ff\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x000000ff\" complete_cpuset=\"0x000000ff\" online_cpuset=\"0x000000ff\" allowed_cpuset=\"0x000000ff\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0000000f\" complete_cpuset=\"0x0000000f\" online_cpuset=\"0x0000000f\" allowed_cpuset=\"0x0000000f\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"8\" cpuset=\"0x0000000f\" complete_cpuset=\"0x0000000f\" online_cpuset=\"0x0000000f\" allowed_cpuset=\"0x0000000f\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"0\" cpuset=\"0x00000001\" complete_cpuset=\"0x00000001\" online_cpuset=\"0x00000001\" allowed_cpuset=\"0x00000001\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"1\" cpuset=\"0x00000002\" complete_cpuset=\"0x00000002\" online_cpuset=\"0x00000002\" allowed_cpuset=\"0x00000002\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"2\" cpuset=\"0x00000004\" complete_cpuset=\"0x00000004\" online_cpuset=\"0x00000004\" allowed_cpuset=\"0x00000004\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"3\" cpuset=\"0x00000008\" complete_cpuset=\"0x00000008\" online_cpuset=\"0x00000008\" allowed_cpuset=\"0x00000008\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x000000f0\" complete_cpuset=\"0x000000f0\" online_cpuset=\"0x000000f0\" allowed_cpuset=\"0x000000f0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"12\" cpuset=\"0x000000f0\" complete_cpuset=\"0x000000f0\" online_cpuset=\"0x000000f0\" allowed_cpuset=\"0x000000f0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"4\" cpuset=\"0x00000010\" complete_cpuset=\"0x00000010\" online_cpuset=\"0x00000010\" allowed_cpuset=\"0x00000010\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"5\" cpuset=\"0x00000020\" complete_cpuset=\"0x00000020\" online_cpuset=\"0x00000020\" allowed_cpuset=\"0x00000020\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"6\" cpuset=\"0x00000040\" complete_cpuset=\"0x00000040\" online_cpuset=\"0x00000040\" allowed_cpuset=\"0x00000040\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"7\" cpuset=\"0x00000080\" complete_cpuset=\"0x00000080\" online_cpuset=\"0x00000080\" allowed_cpuset=\"0x00000080\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x0000ff00\" complete_cpuset=\"0x0000ff00\" online_cpuset=\"0x0000ff00\" allowed_cpuset=\"0x0000ff00\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x0000ff00\" complete_cpuset=\"0x0000ff00\" online_cpuset=\"0x0000ff00\" allowed_cpuset=\"0x0000ff00\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x00000f00\" complete_cpuset=\"0x00000f00\" online_cpuset=\"0x00000f00\" allowed_cpuset=\"0x00000f00\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"16\" cpuset=\"0x00000f00\" complete_cpuset=\"0x00000f00\" online_cpuset=\"0x00000f00\" allowed_cpuset=\"0x00000f00\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"8\" cpuset=\"0x00000100\" complete_cpuset=\"0x00000100\" online_cpuset=\"0x00000100\" allowed_cpuset=\"0x00000100\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"9\" cpuset=\"0x00000200\" complete_cpuset=\"0x00000200\" online_cpuset=\"0x00000200\" allowed_cpuset=\"0x00000200\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"10\" cpuset=\"0x00000400\" complete_cpuset=\"0x00000400\" online_cpuset=\"0x00000400\" allowed_cpuset=\"0x00000400\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"11\" cpuset=\"0x00000800\" complete_cpuset=\"0x00000800\" online_cpuset=\"0x00000800\" allowed_cpuset=\"0x00000800\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x0000f000\" complete_cpuset=\"0x0000f000\" online_cpuset=\"0x0000f000\" allowed_cpuset=\"0x0000f000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"20\" cpuset=\"0x0000f000\" complete_cpuset=\"0x0000f000\" online_cpuset=\"0x0000f000\" allowed_cpuset=\"0x0000f000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"12\" cpuset=\"0x00001000\" complete_cpuset=\"0x00001000\" online_cpuset=\"0x00001000\" allowed_cpuset=\"0x00001000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"13\" cpuset=\"0x00002000\" complete_cpuset=\"0x00002000\" online_cpuset=\"0x00002000\" allowed_cpuset=\"0x00002000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"14\" cpuset=\"0x00004000\" complete_cpuset=\"0x00004000\" online_cpuset=\"0x00004000\" allowed_cpuset=\"0x00004000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"15\" cpuset=\"0x00008000\" complete_cpuset=\"0x00008000\" online_cpuset=\"0x00008000\" allowed_cpuset=\"0x00008000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x00ff0000\" complete_cpuset=\"0x00ff0000\" online_cpuset=\"0x00ff0000\" allowed_cpuset=\"0x00ff0000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x00ff0000\" complete_cpuset=\"0x00ff0000\" online_cpuset=\"0x00ff0000\" allowed_cpuset=\"0x00ff0000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x000f0000\" complete_cpuset=\"0x000f0000\" online_cpuset=\"0x000f0000\" allowed_cpuset=\"0x000f0000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"24\" cpuset=\"0x000f0000\" complete_cpuset=\"0x000f0000\" online_cpuset=\"0x000f0000\" allowed_cpuset=\"0x000f0000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"16\" cpuset=\"0x00010000\" complete_cpuset=\"0x00010000\" online_cpuset=\"0x00010000\" allowed_cpuset=\"0x00010000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"17\" cpuset=\"0x00020000\" complete_cpuset=\"0x00020000\" online_cpuset=\"0x00020000\" allowed_cpuset=\"0x00020000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"18\" cpuset=\"0x00040000\" complete_cpuset=\"0x00040000\" online_cpuset=\"0x00040000\" allowed_cpuset=\"0x00040000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"19\" cpuset=\"0x00080000\" complete_cpuset=\"0x00080000\" online_cpuset=\"0x00080000\" allowed_cpuset=\"0x00080000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x00f00000\" complete_cpuset=\"0x00f00000\" online_cpuset=\"0x00f00000\" allowed_cpuset=\"0x00f00000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"28\" cpuset=\"0x00f00000\" complete_cpuset=\"0x00f00000\" online_cpuset=\"0x00f00000\" allowed_cpuset=\"0x00f00000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"20\" cpuset=\"0x00100000\" complete_cpuset=\"0x00100000\" online_cpuset=\"0x00100000\" allowed_cpuset=\"0x00100000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"21\" cpuset=\"0x00200000\" complete_cpuset=\"0x00200000\" online_cpuset=\"0x00200000\" allowed_cpuset=\"0x00200000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"22\" cpuset=\"0x00400000\" complete_cpuset=\"0x00400000\" online_cpuset=\"0x00400000\" allowed_cpuset=\"0x00400000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"23\" cpuset=\"0x00800000\" complete_cpuset=\"0x00800000\" online_cpuset=\"0x00800000\" allowed_cpuset=\"0x00800000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0xff000000\" complete_cpuset=\"0xff000000\" online_cpuset=\"0xff000000\" allowed_cpuset=\"0xff000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0xff000000\" complete_cpuset=\"0xff000000\" online_cpuset=\"0xff000000\" allowed_cpuset=\"0xff000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0f000000\" complete_cpuset=\"0x0f000000\" online_cpuset=\"0x0f000000\" allowed_cpuset=\"0x0f000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"32\" cpuset=\"0x0f000000\" complete_cpuset=\"0x0f000000\" online_cpuset=\"0x0f000000\" allowed_cpuset=\"0x0f000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"24\" cpuset=\"0x01000000\" complete_cpuset=\"0x01000000\" online_cpuset=\"0x01000000\" allowed_cpuset=\"0x01000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"25\" cpuset=\"0x02000000\" complete_cpuset=\"0x02000000\" online_cpuset=\"0x02000000\" allowed_cpuset=\"0x02000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"26\" cpuset=\"0x04000000\" complete_cpuset=\"0x04000000\" online_cpuset=\"0x04000000\" allowed_cpuset=\"0x04000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"27\" cpuset=\"0x08000000\" complete_cpuset=\"0x08000000\" online_cpuset=\"0x08000000\" allowed_cpuset=\"0x08000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0xf0000000\" complete_cpuset=\"0xf0000000\" online_cpuset=\"0xf0000000\" allowed_cpuset=\"0xf0000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"36\" cpuset=\"0xf0000000\" complete_cpuset=\"0xf0000000\" online_cpuset=\"0xf0000000\" allowed_cpuset=\"0xf0000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"28\" cpuset=\"0x10000000\" complete_cpuset=\"0x10000000\" online_cpuset=\"0x10000000\" allowed_cpuset=\"0x10000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"29\" cpuset=\"0x20000000\" complete_cpuset=\"0x20000000\" online_cpuset=\"0x20000000\" allowed_cpuset=\"0x20000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"30\" cpuset=\"0x40000000\" complete_cpuset=\"0x40000000\" online_cpuset=\"0x40000000\" allowed_cpuset=\"0x40000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"31\" cpuset=\"0x80000000\" complete_cpuset=\"0x80000000\" online_cpuset=\"0x80000000\" allowed_cpuset=\"0x80000000\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x000000ff,0x0\" complete_cpuset=\"0x000000ff,0x0\" online_cpuset=\"0x000000ff,0x0\" allowed_cpuset=\"0x000000ff,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x000000ff,0x0\" complete_cpuset=\"0x000000ff,0x0\" online_cpuset=\"0x000000ff,0x0\" allowed_cpuset=\"0x000000ff,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0000000f,0x0\" complete_cpuset=\"0x0000000f,0x0\" online_cpuset=\"0x0000000f,0x0\" allowed_cpuset=\"0x0000000f,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"40\" cpuset=\"0x0000000f,0x0\" complete_cpuset=\"0x0000000f,0x0\" online_cpuset=\"0x0000000f,0x0\" allowed_cpuset=\"0x0000000f,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"32\" cpuset=\"0x00000001,0x0\" complete_cpuset=\"0x00000001,0x0\" online_cpuset=\"0x00000001,0x0\" allowed_cpuset=\"0x00000001,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"33\" cpuset=\"0x00000002,0x0\" complete_cpuset=\"0x00000002,0x0\" online_cpuset=\"0x00000002,0x0\" allowed_cpuset=\"0x00000002,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"34\" cpuset=\"0x00000004,0x0\" complete_cpuset=\"0x00000004,0x0\" online_cpuset=\"0x00000004,0x0\" allowed_cpuset=\"0x00000004,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"35\" cpuset=\"0x00000008,0x0\" complete_cpuset=\"0x00000008,0x0\" online_cpuset=\"0x00000008,0x0\" allowed_cpuset=\"0x00000008,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x000000f0,0x0\" complete_cpuset=\"0x000000f0,0x0\" online_cpuset=\"0x000000f0,0x0\" allowed_cpuset=\"0x000000f0,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"44\" cpuset=\"0x000000f0,0x0\" complete_cpuset=\"0x000000f0,0x0\" online_cpuset=\"0x000000f0,0x0\" allowed_cpuset=\"0x000000f0,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"36\" cpuset=\"0x00000010,0x0\" complete_cpuset=\"0x00000010,0x0\" online_cpuset=\"0x00000010,0x0\" allowed_cpuset=\"0x00000010,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"37\" cpuset=\"0x00000020,0x0\" complete_cpuset=\"0x00000020,0x0\" online_cpuset=\"0x00000020,0x0\" allowed_cpuset=\"0x00000020,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"38\" cpuset=\"0x00000040,0x0\" complete_cpuset=\"0x00000040,0x0\" online_cpuset=\"0x00000040,0x0\" allowed_cpuset=\"0x00000040,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"39\" cpuset=\"0x00000080,0x0\" complete_cpuset=\"0x00000080,0x0\" online_cpuset=\"0x00000080,0x0\" allowed_cpuset=\"0x00000080,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x0000ff00,0x0\" complete_cpuset=\"0x0000ff00,0x0\" online_cpuset=\"0x0000ff00,0x0\" allowed_cpuset=\"0x0000ff00,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x0000ff00,0x0\" complete_cpuset=\"0x0000ff00,0x0\" online_cpuset=\"0x0000ff00,0x0\" allowed_cpuset=\"0x0000ff00,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x00000f00,0x0\" complete_cpuset=\"0x00000f00,0x0\" online_cpuset=\"0x00000f00,0x0\" allowed_cpuset=\"0x00000f00,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"48\" cpuset=\"0x00000f00,0x0\" complete_cpuset=\"0x00000f00,0x0\" online_cpuset=\"0x00000f00,0x0\" allowed_cpuset=\"0x00000f00,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"40\" cpuset=\"0x00000100,0x0\" complete_cpuset=\"0x00000100,0x0\" online_cpuset=\"0x00000100,0x0\" allowed_cpuset=\"0x00000100,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"41\" cpuset=\"0x00000200,0x0\" complete_cpuset=\"0x00000200,0x0\" online_cpuset=\"0x00000200,0x0\" allowed_cpuset=\"0x00000200,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"42\" cpuset=\"0x00000400,0x0\" complete_cpuset=\"0x00000400,0x0\" online_cpuset=\"0x00000400,0x0\" allowed_cpuset=\"0x00000400,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"43\" cpuset=\"0x00000800,0x0\" complete_cpuset=\"0x00000800,0x0\" online_cpuset=\"0x00000800,0x0\" allowed_cpuset=\"0x00000800,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x0000f000,0x0\" complete_cpuset=\"0x0000f000,0x0\" online_cpuset=\"0x0000f000,0x0\" allowed_cpuset=\"0x0000f000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"52\" cpuset=\"0x0000f000,0x0\" complete_cpuset=\"0x0000f000,0x0\" online_cpuset=\"0x0000f000,0x0\" allowed_cpuset=\"0x0000f000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"44\" cpuset=\"0x00001000,0x0\" complete_cpuset=\"0x00001000,0x0\" online_cpuset=\"0x00001000,0x0\" allowed_cpuset=\"0x00001000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"45\" cpuset=\"0x00002000,0x0\" complete_cpuset=\"0x00002000,0x0\" online_cpuset=\"0x00002000,0x0\" allowed_cpuset=\"0x00002000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"46\" cpuset=\"0x00004000,0x0\" complete_cpuset=\"0x00004000,0x0\" online_cpuset=\"0x00004000,0x0\" allowed_cpuset=\"0x00004000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"47\" cpuset=\"0x00008000,0x0\" complete_cpuset=\"0x00008000,0x0\" online_cpuset=\"0x00008000,0x0\" allowed_cpuset=\"0x00008000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x00ff0000,0x0\" complete_cpuset=\"0x00ff0000,0x0\" online_cpuset=\"0x00ff0000,0x0\" allowed_cpuset=\"0x00ff0000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x00ff0000,0x0\" complete_cpuset=\"0x00ff0000,0x0\" online_cpuset=\"0x00ff0000,0x0\" allowed_cpuset=\"0x00ff0000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x000f0000,0x0\" complete_cpuset=\"0x000f0000,0x0\" online_cpuset=\"0x000f0000,0x0\" allowed_cpuset=\"0x000f0000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"56\" cpuset=\"0x000f0000,0x0\" complete_cpuset=\"0x000f0000,0x0\" online_cpuset=\"0x000f0000,0x0\" allowed_cpuset=\"0x000f0000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"48\" cpuset=\"0x00010000,0x0\" complete_cpuset=\"0x00010000,0x0\" online_cpuset=\"0x00010000,0x0\" allowed_cpuset=\"0x00010000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"49\" cpuset=\"0x00020000,0x0\" complete_cpuset=\"0x00020000,0x0\" online_cpuset=\"0x00020000,0x0\" allowed_cpuset=\"0x00020000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"50\" cpuset=\"0x00040000,0x0\" complete_cpuset=\"0x00040000,0x0\" online_cpuset=\"0x00040000,0x0\" allowed_cpuset=\"0x00040000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"51\" cpuset=\"0x00080000,0x0\" complete_cpuset=\"0x00080000,0x0\" online_cpuset=\"0x00080000,0x0\" allowed_cpuset=\"0x00080000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x00f00000,0x0\" complete_cpuset=\"0x00f00000,0x0\" online_cpuset=\"0x00f00000,0x0\" allowed_cpuset=\"0x00f00000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"60\" cpuset=\"0x00f00000,0x0\" complete_cpuset=\"0x00f00000,0x0\" online_cpuset=\"0x00f00000,0x0\" allowed_cpuset=\"0x00f00000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"52\" cpuset=\"0x00100000,0x0\" complete_cpuset=\"0x00100000,0x0\" online_cpuset=\"0x00100000,0x0\" allowed_cpuset=\"0x00100000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"53\" cpuset=\"0x00200000,0x0\" complete_cpuset=\"0x00200000,0x0\" online_cpuset=\"0x00200000,0x0\" allowed_cpuset=\"0x00200000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"54\" cpuset=\"0x00400000,0x0\" complete_cpuset=\"0x00400000,0x0\" online_cpuset=\"0x00400000,0x0\" allowed_cpuset=\"0x00400000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"55\" cpuset=\"0x00800000,0x0\" complete_cpuset=\"0x00800000,0x0\" online_cpuset=\"0x00800000,0x0\" allowed_cpuset=\"0x00800000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0xff000000,0x0\" complete_cpuset=\"0xff000000,0x0\" online_cpuset=\"0xff000000,0x0\" allowed_cpuset=\"0xff000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0xff000000,0x0\" complete_cpuset=\"0xff000000,0x0\" online_cpuset=\"0xff000000,0x0\" allowed_cpuset=\"0xff000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0f000000,0x0\" complete_cpuset=\"0x0f000000,0x0\" online_cpuset=\"0x0f000000,0x0\" allowed_cpuset=\"0x0f000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"64\" cpuset=\"0x0f000000,0x0\" complete_cpuset=\"0x0f000000,0x0\" online_cpuset=\"0x0f000000,0x0\" allowed_cpuset=\"0x0f000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"56\" cpuset=\"0x01000000,0x0\" complete_cpuset=\"0x01000000,0x0\" online_cpuset=\"0x01000000,0x0\" allowed_cpuset=\"0x01000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"57\" cpuset=\"0x02000000,0x0\" complete_cpuset=\"0x02000000,0x0\" online_cpuset=\"0x02000000,0x0\" allowed_cpuset=\"0x02000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"58\" cpuset=\"0x04000000,0x0\" complete_cpuset=\"0x04000000,0x0\" online_cpuset=\"0x04000000,0x0\" allowed_cpuset=\"0x04000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"59\" cpuset=\"0x08000000,0x0\" complete_cpuset=\"0x08000000,0x0\" online_cpuset=\"0x08000000,0x0\" allowed_cpuset=\"0x08000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0xf0000000,0x0\" complete_cpuset=\"0xf0000000,0x0\" online_cpuset=\"0xf0000000,0x0\" allowed_cpuset=\"0xf0000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"68\" cpuset=\"0xf0000000,0x0\" complete_cpuset=\"0xf0000000,0x0\" online_cpuset=\"0xf0000000,0x0\" allowed_cpuset=\"0xf0000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"60\" cpuset=\"0x10000000,0x0\" complete_cpuset=\"0x10000000,0x0\" online_cpuset=\"0x10000000,0x0\" allowed_cpuset=\"0x10000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"61\" cpuset=\"0x20000000,0x0\" complete_cpuset=\"0x20000000,0x0\" online_cpuset=\"0x20000000,0x0\" allowed_cpuset=\"0x20000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"62\" cpuset=\"0x40000000,0x0\" complete_cpuset=\"0x40000000,0x0\" online_cpuset=\"0x40000000,0x0\" allowed_cpuset=\"0x40000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"63\" cpuset=\"0x80000000,0x0\" complete_cpuset=\"0x80000000,0x0\" online_cpuset=\"0x80000000,0x0\" allowed_cpuset=\"0x80000000,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x000000ff,,0x0\" complete_cpuset=\"0x000000ff,,0x0\" online_cpuset=\"0x000000ff,,0x0\" allowed_cpuset=\"0x000000ff,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x000000ff,,0x0\" complete_cpuset=\"0x000000ff,,0x0\" online_cpuset=\"0x000000ff,,0x0\" allowed_cpuset=\"0x000000ff,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0000000f,,0x0\" complete_cpuset=\"0x0000000f,,0x0\" online_cpuset=\"0x0000000f,,0x0\" allowed_cpuset=\"0x0000000f,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"72\" cpuset=\"0x0000000f,,0x0\" complete_cpuset=\"0x0000000f,,0x0\" online_cpuset=\"0x0000000f,,0x0\" allowed_cpuset=\"0x0000000f,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"64\" cpuset=\"0x00000001,,0x0\" complete_cpuset=\"0x00000001,,0x0\" online_cpuset=\"0x00000001,,0x0\" allowed_cpuset=\"0x00000001,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"65\" cpuset=\"0x00000002,,0x0\" complete_cpuset=\"0x00000002,,0x0\" online_cpuset=\"0x00000002,,0x0\" allowed_cpuset=\"0x00000002,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"66\" cpuset=\"0x00000004,,0x0\" complete_cpuset=\"0x00000004,,0x0\" online_cpuset=\"0x00000004,,0x0\" allowed_cpuset=\"0x00000004,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"67\" cpuset=\"0x00000008,,0x0\" complete_cpuset=\"0x00000008,,0x0\" online_cpuset=\"0x00000008,,0x0\" allowed_cpuset=\"0x00000008,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x000000f0,,0x0\" complete_cpuset=\"0x000000f0,,0x0\" online_cpuset=\"0x000000f0,,0x0\" allowed_cpuset=\"0x000000f0,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"76\" cpuset=\"0x000000f0,,0x0\" complete_cpuset=\"0x000000f0,,0x0\" online_cpuset=\"0x000000f0,,0x0\" allowed_cpuset=\"0x000000f0,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"68\" cpuset=\"0x00000010,,0x0\" complete_cpuset=\"0x00000010,,0x0\" online_cpuset=\"0x00000010,,0x0\" allowed_cpuset=\"0x00000010,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"69\" cpuset=\"0x00000020,,0x0\" complete_cpuset=\"0x00000020,,0x0\" online_cpuset=\"0x00000020,,0x0\" allowed_cpuset=\"0x00000020,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"70\" cpuset=\"0x00000040,,0x0\" complete_cpuset=\"0x00000040,,0x0\" online_cpuset=\"0x00000040,,0x0\" allowed_cpuset=\"0x00000040,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"71\" cpuset=\"0x00000080,,0x0\" complete_cpuset=\"0x00000080,,0x0\" online_cpuset=\"0x00000080,,0x0\" allowed_cpuset=\"0x00000080,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x0000ff00,,0x0\" complete_cpuset=\"0x0000ff00,,0x0\" online_cpuset=\"0x0000ff00,,0x0\" allowed_cpuset=\"0x0000ff00,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x0000ff00,,0x0\" complete_cpuset=\"0x0000ff00,,0x0\" online_cpuset=\"0x0000ff00,,0x0\" allowed_cpuset=\"0x0000ff00,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x00000f00,,0x0\" complete_cpuset=\"0x00000f00,,0x0\" online_cpuset=\"0x00000f00,,0x0\" allowed_cpuset=\"0x00000f00,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"80\" cpuset=\"0x00000f00,,0x0\" complete_cpuset=\"0x00000f00,,0x0\" online_cpuset=\"0x00000f00,,0x0\" allowed_cpuset=\"0x00000f00,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"72\" cpuset=\"0x00000100,,0x0\" complete_cpuset=\"0x00000100,,0x0\" online_cpuset=\"0x00000100,,0x0\" allowed_cpuset=\"0x00000100,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"73\" cpuset=\"0x00000200,,0x0\" complete_cpuset=\"0x00000200,,0x0\" online_cpuset=\"0x00000200,,0x0\" allowed_cpuset=\"0x00000200,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"74\" cpuset=\"0x00000400,,0x0\" complete_cpuset=\"0x00000400,,0x0\" online_cpuset=\"0x00000400,,0x0\" allowed_cpuset=\"0x00000400,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"75\" cpuset=\"0x00000800,,0x0\" complete_cpuset=\"0x00000800,,0x0\" online_cpuset=\"0x00000800,,0x0\" allowed_cpuset=\"0x00000800,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x0000f000,,0x0\" complete_cpuset=\"0x0000f000,,0x0\" online_cpuset=\"0x0000f000,,0x0\" allowed_cpuset=\"0x0000f000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"84\" cpuset=\"0x0000f000,,0x0\" complete_cpuset=\"0x0000f000,,0x0\" online_cpuset=\"0x0000f000,,0x0\" allowed_cpuset=\"0x0000f000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"76\" cpuset=\"0x00001000,,0x0\" complete_cpuset=\"0x00001000,,0x0\" online_cpuset=\"0x00001000,,0x0\" allowed_cpuset=\"0x00001000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"77\" cpuset=\"0x00002000,,0x0\" complete_cpuset=\"0x00002000,,0x0\" online_cpuset=\"0x00002000,,0x0\" allowed_cpuset=\"0x00002000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"78\" cpuset=\"0x00004000,,0x0\" complete_cpuset=\"0x00004000,,0x0\" online_cpuset=\"0x00004000,,0x0\" allowed_cpuset=\"0x00004000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"79\" cpuset=\"0x00008000,,0x0\" complete_cpuset=\"0x00008000,,0x0\" online_cpuset=\"0x00008000,,0x0\" allowed_cpuset=\"0x00008000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x00ff0000,,0x0\" complete_cpuset=\"0x00ff0000,,0x0\" online_cpuset=\"0x00ff0000,,0x0\" allowed_cpuset=\"0x00ff0000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x00ff0000,,0x0\" complete_cpuset=\"0x00ff0000,,0x0\" online_cpuset=\"0x00ff0000,,0x0\" allowed_cpuset=\"0x00ff0000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x000f0000,,0x0\" complete_cpuset=\"0x000f0000,,0x0\" online_cpuset=\"0x000f0000,,0x0\" allowed_cpuset=\"0x000f0000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"88\" cpuset=\"0x000f0000,,0x0\" complete_cpuset=\"0x000f0000,,0x0\" online_cpuset=\"0x000f0000,,0x0\" allowed_cpuset=\"0x000f0000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"80\" cpuset=\"0x00010000,,0x0\" complete_cpuset=\"0x00010000,,0x0\" online_cpuset=\"0x00010000,,0x0\" allowed_cpuset=\"0x00010000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"81\" cpuset=\"0x00020000,,0x0\" complete_cpuset=\"0x00020000,,0x0\" online_cpuset=\"0x00020000,,0x0\" allowed_cpuset=\"0x00020000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"82\" cpuset=\"0x00040000,,0x0\" complete_cpuset=\"0x00040000,,0x0\" online_cpuset=\"0x00040000,,0x0\" allowed_cpuset=\"0x00040000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"83\" cpuset=\"0x00080000,,0x0\" complete_cpuset=\"0x00080000,,0x0\" online_cpuset=\"0x00080000,,0x0\" allowed_cpuset=\"0x00080000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x00f00000,,0x0\" complete_cpuset=\"0x00f00000,,0x0\" online_cpuset=\"0x00f00000,,0x0\" allowed_cpuset=\"0x00f00000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"92\" cpuset=\"0x00f00000,,0x0\" complete_cpuset=\"0x00f00000,,0x0\" online_cpuset=\"0x00f00000,,0x0\" allowed_cpuset=\"0x00f00000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\">\n\
+                  <object type=\"PU\" os_index=\"84\" cpuset=\"0x00100000,,0x0\" complete_cpuset=\"0x00100000,,0x0\" online_cpuset=\"0x00100000,,0x0\" allowed_cpuset=\"0x00100000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"85\" cpuset=\"0x00200000,,0x0\" complete_cpuset=\"0x00200000,,0x0\" online_cpuset=\"0x00200000,,0x0\" allowed_cpuset=\"0x00200000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"86\" cpuset=\"0x00400000,,0x0\" complete_cpuset=\"0x00400000,,0x0\" online_cpuset=\"0x00400000,,0x0\" allowed_cpuset=\"0x00400000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                  <object type=\"PU\" os_index=\"87\" cpuset=\"0x00800000,,0x0\" complete_cpuset=\"0x00800000,,0x0\" online_cpuset=\"0x00800000,,0x0\" allowed_cpuset=\"0x00800000,,0x0\" nodeset=\"0x00000001\" complete_nodeset=\"0x00000001\" allowed_nodeset=\"0x00000001\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+        </object>\n\
+        <object type=\"Bridge\" os_index=\"0\" bridge_type=\"0-1\" depth=\"0\" bridge_pci=\"0000:[00-01]\">\n\
+          <object type=\"PCIDev\" os_index=\"4096\" name=\"Samsung Electronics Co Ltd NVMe SSD Controller 172Xa\" pci_busid=\"0000:01:00.0\" pci_type=\"0108 [144d:a822] [1014:0621] 01\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"Samsung Electronics Co Ltd\"/>\n\
+            <info name=\"PCIDevice\" value=\"NVMe SSD Controller 172Xa\"/>\n\
+            <object type=\"OSDev\" name=\"nvme0n1\" osdev_type=\"0\">\n\
+              <info name=\"LinuxDeviceID\" value=\"259:0\"/>\n\
+              <info name=\"SerialNumber\" value=\"S3RVNA0JA01484\"/>\n\
+              <info name=\"Type\" value=\"Disk\"/>\n\
+            </object>\n\
+          </object>\n\
+        </object>\n\
+        <object type=\"Bridge\" os_index=\"2\" bridge_type=\"0-1\" depth=\"0\" bridge_pci=\"0002:[00-02]\">\n\
+          <object type=\"PCIDev\" os_index=\"2105344\" name=\"ASPEED Technology, Inc. ASPEED Graphics Family\" pci_busid=\"0002:02:00.0\" pci_type=\"0300 [1a03:2000] [1a03:2000] 41\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"ASPEED Technology, Inc.\"/>\n\
+            <info name=\"PCIDevice\" value=\"ASPEED Graphics Family\"/>\n\
+            <object type=\"OSDev\" name=\"card4\" osdev_type=\"1\"/>\n\
+            <object type=\"OSDev\" name=\"controlD68\" osdev_type=\"1\"/>\n\
+          </object>\n\
+        </object>\n\
+        <object type=\"Bridge\" os_index=\"3\" bridge_type=\"0-1\" depth=\"0\" bridge_pci=\"0003:[00-01]\">\n\
+          <object type=\"PCIDev\" os_index=\"3149824\" name=\"Mellanox Technologies MT28800 Family [ConnectX-5 Ex]\" pci_busid=\"0003:01:00.0\" pci_type=\"0207 [15b3:1019] [1014:0617] 00\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"Mellanox Technologies\"/>\n\
+            <info name=\"PCIDevice\" value=\"MT28800 Family [ConnectX-5 Ex]\"/>\n\
+            <object type=\"OSDev\" name=\"hsi0\" osdev_type=\"2\">\n\
+              <info name=\"Address\" value=\"00:00:10:87:fe:80:00:00:00:00:00:00:ec:0d:9a:03:00:ca:a4:18\"/>\n\
+              <info name=\"Port\" value=\"1\"/>\n\
+            </object>\n\
+            <object type=\"OSDev\" name=\"mlx5_0\" osdev_type=\"3\">\n\
+              <info name=\"NodeGUID\" value=\"ec0d:9a03:00ca:a418\"/>\n\
+              <info name=\"SysImageGUID\" value=\"ec0d:9a03:00ca:a418\"/>\n\
+              <info name=\"Port1State\" value=\"4\"/>\n\
+              <info name=\"Port1LID\" value=\"0x32ed\"/>\n\
+              <info name=\"Port1LMC\" value=\"0\"/>\n\
+              <info name=\"Port1GID0\" value=\"fe80:0000:0000:0000:ec0d:9a03:00ca:a418\"/>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"PCIDev\" os_index=\"3149825\" name=\"Mellanox Technologies MT28800 Family [ConnectX-5 Ex]\" pci_busid=\"0003:01:00.1\" pci_type=\"0207 [15b3:1019] [1014:0617] 00\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"Mellanox Technologies\"/>\n\
+            <info name=\"PCIDevice\" value=\"MT28800 Family [ConnectX-5 Ex]\"/>\n\
+            <object type=\"OSDev\" name=\"hsi1\" osdev_type=\"2\">\n\
+              <info name=\"Address\" value=\"00:00:18:87:fe:80:00:00:00:00:00:00:ec:0d:9a:03:00:ca:a4:19\"/>\n\
+              <info name=\"Port\" value=\"1\"/>\n\
+            </object>\n\
+            <object type=\"OSDev\" name=\"mlx5_1\" osdev_type=\"3\">\n\
+              <info name=\"NodeGUID\" value=\"ec0d:9a03:00ca:a419\"/>\n\
+              <info name=\"SysImageGUID\" value=\"ec0d:9a03:00ca:a418\"/>\n\
+              <info name=\"Port1State\" value=\"4\"/>\n\
+              <info name=\"Port1LID\" value=\"0x32ee\"/>\n\
+              <info name=\"Port1LMC\" value=\"0\"/>\n\
+              <info name=\"Port1GID0\" value=\"fe80:0000:0000:0000:ec0d:9a03:00ca:a419\"/>\n\
+            </object>\n\
+          </object>\n\
+        </object>\n\
+        <object type=\"Bridge\" os_index=\"4\" bridge_type=\"0-1\" depth=\"0\" bridge_pci=\"0004:[00-0a]\">\n\
+          <object type=\"PCIDev\" os_index=\"4206592\" name=\"Marvell Technology Group Ltd. 88SE9235 PCIe 2.0 x2 4-port SATA 6 Gb/s Controller\" pci_busid=\"0004:03:00.0\" pci_type=\"0106 [1b4b:9235] [1014:0612] 11\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"Marvell Technology Group Ltd.\"/>\n\
+            <info name=\"PCIDevice\" value=\"88SE9235 PCIe 2.0 x2 4-port SATA 6 Gb/s Controller\"/>\n\
+          </object>\n\
+          <object type=\"PCIDev\" os_index=\"4210688\" name=\"NVIDIA Corporation GV100GL [Tesla V100 SXM2]\" pci_busid=\"0004:04:00.0\" pci_type=\"0302 [10de:1db1] [10de:1212] a1\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"NVIDIA Corporation\"/>\n\
+            <info name=\"PCIDevice\" value=\"GV100GL [Tesla V100 SXM2]\"/>\n\
+            <object type=\"OSDev\" name=\"renderD128\" osdev_type=\"1\"/>\n\
+            <object type=\"OSDev\" name=\"card0\" osdev_type=\"1\"/>\n\
+            <object type=\"OSDev\" name=\"cuda0\" osdev_type=\"5\">\n\
+              <info name=\"CoProcType\" value=\"CUDA\"/>\n\
+              <info name=\"Backend\" value=\"CUDA\"/>\n\
+              <info name=\"GPUVendor\" value=\"NVIDIA Corporation\"/>\n\
+              <info name=\"GPUModel\" value=\"Tesla V100-SXM2-16GB\"/>\n\
+              <info name=\"CUDAGlobalMemorySize\" value=\"16515072\"/>\n\
+              <info name=\"CUDAL2CacheSize\" value=\"6144\"/>\n\
+              <info name=\"CUDAMultiProcessors\" value=\"80\"/>\n\
+              <info name=\"CUDACoresPerMP\" value=\"64\"/>\n\
+              <info name=\"CUDASharedMemorySizePerMP\" value=\"48\"/>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"PCIDev\" os_index=\"4214784\" name=\"NVIDIA Corporation GV100GL [Tesla V100 SXM2]\" pci_busid=\"0004:05:00.0\" pci_type=\"0302 [10de:1db1] [10de:1212] a1\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"NVIDIA Corporation\"/>\n\
+            <info name=\"PCIDevice\" value=\"GV100GL [Tesla V100 SXM2]\"/>\n\
+            <object type=\"OSDev\" name=\"card1\" osdev_type=\"1\"/>\n\
+            <object type=\"OSDev\" name=\"renderD129\" osdev_type=\"1\"/>\n\
+            <object type=\"OSDev\" name=\"cuda1\" osdev_type=\"5\">\n\
+              <info name=\"CoProcType\" value=\"CUDA\"/>\n\
+              <info name=\"Backend\" value=\"CUDA\"/>\n\
+              <info name=\"GPUVendor\" value=\"NVIDIA Corporation\"/>\n\
+              <info name=\"GPUModel\" value=\"Tesla V100-SXM2-16GB\"/>\n\
+              <info name=\"CUDAGlobalMemorySize\" value=\"16515072\"/>\n\
+              <info name=\"CUDAL2CacheSize\" value=\"6144\"/>\n\
+              <info name=\"CUDAMultiProcessors\" value=\"80\"/>\n\
+              <info name=\"CUDACoresPerMP\" value=\"64\"/>\n\
+              <info name=\"CUDASharedMemorySizePerMP\" value=\"48\"/>\n\
+            </object>\n\
+          </object>\n\
+        </object>\n\
+        <object type=\"Bridge\" os_index=\"5\" bridge_type=\"0-1\" depth=\"0\" bridge_pci=\"0005:[00-01]\">\n\
+          <object type=\"PCIDev\" os_index=\"5246976\" name=\"Broadcom Limited NetXtreme BCM5719 Gigabit Ethernet PCIe\" pci_busid=\"0005:01:00.0\" pci_type=\"0200 [14e4:1657] [14e4:1981] 01\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"Broadcom Limited\"/>\n\
+            <info name=\"PCIDevice\" value=\"NetXtreme BCM5719 Gigabit Ethernet PCIe\"/>\n\
+            <object type=\"OSDev\" name=\"enP5p1s0f0\" osdev_type=\"2\">\n\
+              <info name=\"Address\" value=\"70:e2:84:14:9d:1f\"/>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"PCIDev\" os_index=\"5246977\" name=\"Broadcom Limited NetXtreme BCM5719 Gigabit Ethernet PCIe\" pci_busid=\"0005:01:00.1\" pci_type=\"0200 [14e4:1657] [14e4:1657] 01\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"Broadcom Limited\"/>\n\
+            <info name=\"PCIDevice\" value=\"NetXtreme BCM5719 Gigabit Ethernet PCIe\"/>\n\
+            <object type=\"OSDev\" name=\"enP5p1s0f1\" osdev_type=\"2\">\n\
+              <info name=\"Address\" value=\"70:e2:84:14:9d:20\"/>\n\
+            </object>\n\
+          </object>\n\
+        </object>\n\
+      </object>\n\
+      <object type=\"NUMANode\" os_index=\"8\" cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xff000000,,0x0\" complete_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xff000000,,0x0\" online_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xff000000,,0x0\" allowed_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xff000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" local_memory=\"137166979072\">\n\
+        <page_type size=\"65536\" count=\"2093002\"/>\n\
+        <page_type size=\"2097152\" count=\"0\"/>\n\
+        <page_type size=\"1073741824\" count=\"0\"/>\n\
+        <object type=\"Package\" os_index=\"8\" cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xff000000,,0x0\" complete_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xff000000,,0x0\" online_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xff000000,,0x0\" allowed_cpuset=\"0x0000ffff,0xffffffff,0xffffffff,0xff000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+          <info name=\"CPUModel\" value=\"POWER9, altivec supported\"/>\n\
+          <info name=\"CPURevision\" value=\"2.1 (pvr 004e 1201)\"/>\n\
+          <object type=\"Cache\" cpuset=\"0xff000000,,0x0\" complete_cpuset=\"0xff000000,,0x0\" online_cpuset=\"0xff000000,,0x0\" allowed_cpuset=\"0xff000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0xff000000,,0x0\" complete_cpuset=\"0xff000000,,0x0\" online_cpuset=\"0xff000000,,0x0\" allowed_cpuset=\"0xff000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0f000000,,0x0\" complete_cpuset=\"0x0f000000,,0x0\" online_cpuset=\"0x0f000000,,0x0\" allowed_cpuset=\"0x0f000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2048\" cpuset=\"0x0f000000,,0x0\" complete_cpuset=\"0x0f000000,,0x0\" online_cpuset=\"0x0f000000,,0x0\" allowed_cpuset=\"0x0f000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"88\" cpuset=\"0x01000000,,0x0\" complete_cpuset=\"0x01000000,,0x0\" online_cpuset=\"0x01000000,,0x0\" allowed_cpuset=\"0x01000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"89\" cpuset=\"0x02000000,,0x0\" complete_cpuset=\"0x02000000,,0x0\" online_cpuset=\"0x02000000,,0x0\" allowed_cpuset=\"0x02000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"90\" cpuset=\"0x04000000,,0x0\" complete_cpuset=\"0x04000000,,0x0\" online_cpuset=\"0x04000000,,0x0\" allowed_cpuset=\"0x04000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"91\" cpuset=\"0x08000000,,0x0\" complete_cpuset=\"0x08000000,,0x0\" online_cpuset=\"0x08000000,,0x0\" allowed_cpuset=\"0x08000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0xf0000000,,0x0\" complete_cpuset=\"0xf0000000,,0x0\" online_cpuset=\"0xf0000000,,0x0\" allowed_cpuset=\"0xf0000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2052\" cpuset=\"0xf0000000,,0x0\" complete_cpuset=\"0xf0000000,,0x0\" online_cpuset=\"0xf0000000,,0x0\" allowed_cpuset=\"0xf0000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"92\" cpuset=\"0x10000000,,0x0\" complete_cpuset=\"0x10000000,,0x0\" online_cpuset=\"0x10000000,,0x0\" allowed_cpuset=\"0x10000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"93\" cpuset=\"0x20000000,,0x0\" complete_cpuset=\"0x20000000,,0x0\" online_cpuset=\"0x20000000,,0x0\" allowed_cpuset=\"0x20000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"94\" cpuset=\"0x40000000,,0x0\" complete_cpuset=\"0x40000000,,0x0\" online_cpuset=\"0x40000000,,0x0\" allowed_cpuset=\"0x40000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"95\" cpuset=\"0x80000000,,0x0\" complete_cpuset=\"0x80000000,,0x0\" online_cpuset=\"0x80000000,,0x0\" allowed_cpuset=\"0x80000000,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x000000ff,,,0x0\" complete_cpuset=\"0x000000ff,,,0x0\" online_cpuset=\"0x000000ff,,,0x0\" allowed_cpuset=\"0x000000ff,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x000000ff,,,0x0\" complete_cpuset=\"0x000000ff,,,0x0\" online_cpuset=\"0x000000ff,,,0x0\" allowed_cpuset=\"0x000000ff,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0000000f,,,0x0\" complete_cpuset=\"0x0000000f,,,0x0\" online_cpuset=\"0x0000000f,,,0x0\" allowed_cpuset=\"0x0000000f,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2056\" cpuset=\"0x0000000f,,,0x0\" complete_cpuset=\"0x0000000f,,,0x0\" online_cpuset=\"0x0000000f,,,0x0\" allowed_cpuset=\"0x0000000f,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"96\" cpuset=\"0x00000001,,,0x0\" complete_cpuset=\"0x00000001,,,0x0\" online_cpuset=\"0x00000001,,,0x0\" allowed_cpuset=\"0x00000001,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"97\" cpuset=\"0x00000002,,,0x0\" complete_cpuset=\"0x00000002,,,0x0\" online_cpuset=\"0x00000002,,,0x0\" allowed_cpuset=\"0x00000002,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"98\" cpuset=\"0x00000004,,,0x0\" complete_cpuset=\"0x00000004,,,0x0\" online_cpuset=\"0x00000004,,,0x0\" allowed_cpuset=\"0x00000004,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"99\" cpuset=\"0x00000008,,,0x0\" complete_cpuset=\"0x00000008,,,0x0\" online_cpuset=\"0x00000008,,,0x0\" allowed_cpuset=\"0x00000008,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x000000f0,,,0x0\" complete_cpuset=\"0x000000f0,,,0x0\" online_cpuset=\"0x000000f0,,,0x0\" allowed_cpuset=\"0x000000f0,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2060\" cpuset=\"0x000000f0,,,0x0\" complete_cpuset=\"0x000000f0,,,0x0\" online_cpuset=\"0x000000f0,,,0x0\" allowed_cpuset=\"0x000000f0,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"100\" cpuset=\"0x00000010,,,0x0\" complete_cpuset=\"0x00000010,,,0x0\" online_cpuset=\"0x00000010,,,0x0\" allowed_cpuset=\"0x00000010,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"101\" cpuset=\"0x00000020,,,0x0\" complete_cpuset=\"0x00000020,,,0x0\" online_cpuset=\"0x00000020,,,0x0\" allowed_cpuset=\"0x00000020,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"102\" cpuset=\"0x00000040,,,0x0\" complete_cpuset=\"0x00000040,,,0x0\" online_cpuset=\"0x00000040,,,0x0\" allowed_cpuset=\"0x00000040,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"103\" cpuset=\"0x00000080,,,0x0\" complete_cpuset=\"0x00000080,,,0x0\" online_cpuset=\"0x00000080,,,0x0\" allowed_cpuset=\"0x00000080,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x0000ff00,,,0x0\" complete_cpuset=\"0x0000ff00,,,0x0\" online_cpuset=\"0x0000ff00,,,0x0\" allowed_cpuset=\"0x0000ff00,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x0000ff00,,,0x0\" complete_cpuset=\"0x0000ff00,,,0x0\" online_cpuset=\"0x0000ff00,,,0x0\" allowed_cpuset=\"0x0000ff00,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x00000f00,,,0x0\" complete_cpuset=\"0x00000f00,,,0x0\" online_cpuset=\"0x00000f00,,,0x0\" allowed_cpuset=\"0x00000f00,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2064\" cpuset=\"0x00000f00,,,0x0\" complete_cpuset=\"0x00000f00,,,0x0\" online_cpuset=\"0x00000f00,,,0x0\" allowed_cpuset=\"0x00000f00,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"104\" cpuset=\"0x00000100,,,0x0\" complete_cpuset=\"0x00000100,,,0x0\" online_cpuset=\"0x00000100,,,0x0\" allowed_cpuset=\"0x00000100,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"105\" cpuset=\"0x00000200,,,0x0\" complete_cpuset=\"0x00000200,,,0x0\" online_cpuset=\"0x00000200,,,0x0\" allowed_cpuset=\"0x00000200,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"106\" cpuset=\"0x00000400,,,0x0\" complete_cpuset=\"0x00000400,,,0x0\" online_cpuset=\"0x00000400,,,0x0\" allowed_cpuset=\"0x00000400,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"107\" cpuset=\"0x00000800,,,0x0\" complete_cpuset=\"0x00000800,,,0x0\" online_cpuset=\"0x00000800,,,0x0\" allowed_cpuset=\"0x00000800,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x0000f000,,,0x0\" complete_cpuset=\"0x0000f000,,,0x0\" online_cpuset=\"0x0000f000,,,0x0\" allowed_cpuset=\"0x0000f000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2068\" cpuset=\"0x0000f000,,,0x0\" complete_cpuset=\"0x0000f000,,,0x0\" online_cpuset=\"0x0000f000,,,0x0\" allowed_cpuset=\"0x0000f000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"108\" cpuset=\"0x00001000,,,0x0\" complete_cpuset=\"0x00001000,,,0x0\" online_cpuset=\"0x00001000,,,0x0\" allowed_cpuset=\"0x00001000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"109\" cpuset=\"0x00002000,,,0x0\" complete_cpuset=\"0x00002000,,,0x0\" online_cpuset=\"0x00002000,,,0x0\" allowed_cpuset=\"0x00002000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"110\" cpuset=\"0x00004000,,,0x0\" complete_cpuset=\"0x00004000,,,0x0\" online_cpuset=\"0x00004000,,,0x0\" allowed_cpuset=\"0x00004000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"111\" cpuset=\"0x00008000,,,0x0\" complete_cpuset=\"0x00008000,,,0x0\" online_cpuset=\"0x00008000,,,0x0\" allowed_cpuset=\"0x00008000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x00ff0000,,,0x0\" complete_cpuset=\"0x00ff0000,,,0x0\" online_cpuset=\"0x00ff0000,,,0x0\" allowed_cpuset=\"0x00ff0000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x00ff0000,,,0x0\" complete_cpuset=\"0x00ff0000,,,0x0\" online_cpuset=\"0x00ff0000,,,0x0\" allowed_cpuset=\"0x00ff0000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x000f0000,,,0x0\" complete_cpuset=\"0x000f0000,,,0x0\" online_cpuset=\"0x000f0000,,,0x0\" allowed_cpuset=\"0x000f0000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2072\" cpuset=\"0x000f0000,,,0x0\" complete_cpuset=\"0x000f0000,,,0x0\" online_cpuset=\"0x000f0000,,,0x0\" allowed_cpuset=\"0x000f0000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"112\" cpuset=\"0x00010000,,,0x0\" complete_cpuset=\"0x00010000,,,0x0\" online_cpuset=\"0x00010000,,,0x0\" allowed_cpuset=\"0x00010000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"113\" cpuset=\"0x00020000,,,0x0\" complete_cpuset=\"0x00020000,,,0x0\" online_cpuset=\"0x00020000,,,0x0\" allowed_cpuset=\"0x00020000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"114\" cpuset=\"0x00040000,,,0x0\" complete_cpuset=\"0x00040000,,,0x0\" online_cpuset=\"0x00040000,,,0x0\" allowed_cpuset=\"0x00040000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"115\" cpuset=\"0x00080000,,,0x0\" complete_cpuset=\"0x00080000,,,0x0\" online_cpuset=\"0x00080000,,,0x0\" allowed_cpuset=\"0x00080000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x00f00000,,,0x0\" complete_cpuset=\"0x00f00000,,,0x0\" online_cpuset=\"0x00f00000,,,0x0\" allowed_cpuset=\"0x00f00000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2076\" cpuset=\"0x00f00000,,,0x0\" complete_cpuset=\"0x00f00000,,,0x0\" online_cpuset=\"0x00f00000,,,0x0\" allowed_cpuset=\"0x00f00000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"116\" cpuset=\"0x00100000,,,0x0\" complete_cpuset=\"0x00100000,,,0x0\" online_cpuset=\"0x00100000,,,0x0\" allowed_cpuset=\"0x00100000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"117\" cpuset=\"0x00200000,,,0x0\" complete_cpuset=\"0x00200000,,,0x0\" online_cpuset=\"0x00200000,,,0x0\" allowed_cpuset=\"0x00200000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"118\" cpuset=\"0x00400000,,,0x0\" complete_cpuset=\"0x00400000,,,0x0\" online_cpuset=\"0x00400000,,,0x0\" allowed_cpuset=\"0x00400000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"119\" cpuset=\"0x00800000,,,0x0\" complete_cpuset=\"0x00800000,,,0x0\" online_cpuset=\"0x00800000,,,0x0\" allowed_cpuset=\"0x00800000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0xff000000,,,0x0\" complete_cpuset=\"0xff000000,,,0x0\" online_cpuset=\"0xff000000,,,0x0\" allowed_cpuset=\"0xff000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0xff000000,,,0x0\" complete_cpuset=\"0xff000000,,,0x0\" online_cpuset=\"0xff000000,,,0x0\" allowed_cpuset=\"0xff000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0f000000,,,0x0\" complete_cpuset=\"0x0f000000,,,0x0\" online_cpuset=\"0x0f000000,,,0x0\" allowed_cpuset=\"0x0f000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2080\" cpuset=\"0x0f000000,,,0x0\" complete_cpuset=\"0x0f000000,,,0x0\" online_cpuset=\"0x0f000000,,,0x0\" allowed_cpuset=\"0x0f000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"120\" cpuset=\"0x01000000,,,0x0\" complete_cpuset=\"0x01000000,,,0x0\" online_cpuset=\"0x01000000,,,0x0\" allowed_cpuset=\"0x01000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"121\" cpuset=\"0x02000000,,,0x0\" complete_cpuset=\"0x02000000,,,0x0\" online_cpuset=\"0x02000000,,,0x0\" allowed_cpuset=\"0x02000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"122\" cpuset=\"0x04000000,,,0x0\" complete_cpuset=\"0x04000000,,,0x0\" online_cpuset=\"0x04000000,,,0x0\" allowed_cpuset=\"0x04000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"123\" cpuset=\"0x08000000,,,0x0\" complete_cpuset=\"0x08000000,,,0x0\" online_cpuset=\"0x08000000,,,0x0\" allowed_cpuset=\"0x08000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0xf0000000,,,0x0\" complete_cpuset=\"0xf0000000,,,0x0\" online_cpuset=\"0xf0000000,,,0x0\" allowed_cpuset=\"0xf0000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2084\" cpuset=\"0xf0000000,,,0x0\" complete_cpuset=\"0xf0000000,,,0x0\" online_cpuset=\"0xf0000000,,,0x0\" allowed_cpuset=\"0xf0000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"124\" cpuset=\"0x10000000,,,0x0\" complete_cpuset=\"0x10000000,,,0x0\" online_cpuset=\"0x10000000,,,0x0\" allowed_cpuset=\"0x10000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"125\" cpuset=\"0x20000000,,,0x0\" complete_cpuset=\"0x20000000,,,0x0\" online_cpuset=\"0x20000000,,,0x0\" allowed_cpuset=\"0x20000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"126\" cpuset=\"0x40000000,,,0x0\" complete_cpuset=\"0x40000000,,,0x0\" online_cpuset=\"0x40000000,,,0x0\" allowed_cpuset=\"0x40000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"127\" cpuset=\"0x80000000,,,0x0\" complete_cpuset=\"0x80000000,,,0x0\" online_cpuset=\"0x80000000,,,0x0\" allowed_cpuset=\"0x80000000,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x000000ff,,,,0x0\" complete_cpuset=\"0x000000ff,,,,0x0\" online_cpuset=\"0x000000ff,,,,0x0\" allowed_cpuset=\"0x000000ff,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x000000ff,,,,0x0\" complete_cpuset=\"0x000000ff,,,,0x0\" online_cpuset=\"0x000000ff,,,,0x0\" allowed_cpuset=\"0x000000ff,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0000000f,,,,0x0\" complete_cpuset=\"0x0000000f,,,,0x0\" online_cpuset=\"0x0000000f,,,,0x0\" allowed_cpuset=\"0x0000000f,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2088\" cpuset=\"0x0000000f,,,,0x0\" complete_cpuset=\"0x0000000f,,,,0x0\" online_cpuset=\"0x0000000f,,,,0x0\" allowed_cpuset=\"0x0000000f,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"128\" cpuset=\"0x00000001,,,,0x0\" complete_cpuset=\"0x00000001,,,,0x0\" online_cpuset=\"0x00000001,,,,0x0\" allowed_cpuset=\"0x00000001,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"129\" cpuset=\"0x00000002,,,,0x0\" complete_cpuset=\"0x00000002,,,,0x0\" online_cpuset=\"0x00000002,,,,0x0\" allowed_cpuset=\"0x00000002,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"130\" cpuset=\"0x00000004,,,,0x0\" complete_cpuset=\"0x00000004,,,,0x0\" online_cpuset=\"0x00000004,,,,0x0\" allowed_cpuset=\"0x00000004,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"131\" cpuset=\"0x00000008,,,,0x0\" complete_cpuset=\"0x00000008,,,,0x0\" online_cpuset=\"0x00000008,,,,0x0\" allowed_cpuset=\"0x00000008,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x000000f0,,,,0x0\" complete_cpuset=\"0x000000f0,,,,0x0\" online_cpuset=\"0x000000f0,,,,0x0\" allowed_cpuset=\"0x000000f0,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2092\" cpuset=\"0x000000f0,,,,0x0\" complete_cpuset=\"0x000000f0,,,,0x0\" online_cpuset=\"0x000000f0,,,,0x0\" allowed_cpuset=\"0x000000f0,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"132\" cpuset=\"0x00000010,,,,0x0\" complete_cpuset=\"0x00000010,,,,0x0\" online_cpuset=\"0x00000010,,,,0x0\" allowed_cpuset=\"0x00000010,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"133\" cpuset=\"0x00000020,,,,0x0\" complete_cpuset=\"0x00000020,,,,0x0\" online_cpuset=\"0x00000020,,,,0x0\" allowed_cpuset=\"0x00000020,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"134\" cpuset=\"0x00000040,,,,0x0\" complete_cpuset=\"0x00000040,,,,0x0\" online_cpuset=\"0x00000040,,,,0x0\" allowed_cpuset=\"0x00000040,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"135\" cpuset=\"0x00000080,,,,0x0\" complete_cpuset=\"0x00000080,,,,0x0\" online_cpuset=\"0x00000080,,,,0x0\" allowed_cpuset=\"0x00000080,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x00000f00,,,,0x0\" complete_cpuset=\"0x00000f00,,,,0x0\" online_cpuset=\"0x00000f00,,,,0x0\" allowed_cpuset=\"0x00000f00,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x00000f00,,,,0x0\" complete_cpuset=\"0x00000f00,,,,0x0\" online_cpuset=\"0x00000f00,,,,0x0\" allowed_cpuset=\"0x00000f00,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x00000f00,,,,0x0\" complete_cpuset=\"0x00000f00,,,,0x0\" online_cpuset=\"0x00000f00,,,,0x0\" allowed_cpuset=\"0x00000f00,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2100\" cpuset=\"0x00000f00,,,,0x0\" complete_cpuset=\"0x00000f00,,,,0x0\" online_cpuset=\"0x00000f00,,,,0x0\" allowed_cpuset=\"0x00000f00,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"136\" cpuset=\"0x00000100,,,,0x0\" complete_cpuset=\"0x00000100,,,,0x0\" online_cpuset=\"0x00000100,,,,0x0\" allowed_cpuset=\"0x00000100,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"137\" cpuset=\"0x00000200,,,,0x0\" complete_cpuset=\"0x00000200,,,,0x0\" online_cpuset=\"0x00000200,,,,0x0\" allowed_cpuset=\"0x00000200,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"138\" cpuset=\"0x00000400,,,,0x0\" complete_cpuset=\"0x00000400,,,,0x0\" online_cpuset=\"0x00000400,,,,0x0\" allowed_cpuset=\"0x00000400,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"139\" cpuset=\"0x00000800,,,,0x0\" complete_cpuset=\"0x00000800,,,,0x0\" online_cpuset=\"0x00000800,,,,0x0\" allowed_cpuset=\"0x00000800,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x000ff000,,,,0x0\" complete_cpuset=\"0x000ff000,,,,0x0\" online_cpuset=\"0x000ff000,,,,0x0\" allowed_cpuset=\"0x000ff000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x000ff000,,,,0x0\" complete_cpuset=\"0x000ff000,,,,0x0\" online_cpuset=\"0x000ff000,,,,0x0\" allowed_cpuset=\"0x000ff000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0000f000,,,,0x0\" complete_cpuset=\"0x0000f000,,,,0x0\" online_cpuset=\"0x0000f000,,,,0x0\" allowed_cpuset=\"0x0000f000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2104\" cpuset=\"0x0000f000,,,,0x0\" complete_cpuset=\"0x0000f000,,,,0x0\" online_cpuset=\"0x0000f000,,,,0x0\" allowed_cpuset=\"0x0000f000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"140\" cpuset=\"0x00001000,,,,0x0\" complete_cpuset=\"0x00001000,,,,0x0\" online_cpuset=\"0x00001000,,,,0x0\" allowed_cpuset=\"0x00001000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"141\" cpuset=\"0x00002000,,,,0x0\" complete_cpuset=\"0x00002000,,,,0x0\" online_cpuset=\"0x00002000,,,,0x0\" allowed_cpuset=\"0x00002000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"142\" cpuset=\"0x00004000,,,,0x0\" complete_cpuset=\"0x00004000,,,,0x0\" online_cpuset=\"0x00004000,,,,0x0\" allowed_cpuset=\"0x00004000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"143\" cpuset=\"0x00008000,,,,0x0\" complete_cpuset=\"0x00008000,,,,0x0\" online_cpuset=\"0x00008000,,,,0x0\" allowed_cpuset=\"0x00008000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x000f0000,,,,0x0\" complete_cpuset=\"0x000f0000,,,,0x0\" online_cpuset=\"0x000f0000,,,,0x0\" allowed_cpuset=\"0x000f0000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2108\" cpuset=\"0x000f0000,,,,0x0\" complete_cpuset=\"0x000f0000,,,,0x0\" online_cpuset=\"0x000f0000,,,,0x0\" allowed_cpuset=\"0x000f0000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"144\" cpuset=\"0x00010000,,,,0x0\" complete_cpuset=\"0x00010000,,,,0x0\" online_cpuset=\"0x00010000,,,,0x0\" allowed_cpuset=\"0x00010000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"145\" cpuset=\"0x00020000,,,,0x0\" complete_cpuset=\"0x00020000,,,,0x0\" online_cpuset=\"0x00020000,,,,0x0\" allowed_cpuset=\"0x00020000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"146\" cpuset=\"0x00040000,,,,0x0\" complete_cpuset=\"0x00040000,,,,0x0\" online_cpuset=\"0x00040000,,,,0x0\" allowed_cpuset=\"0x00040000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"147\" cpuset=\"0x00080000,,,,0x0\" complete_cpuset=\"0x00080000,,,,0x0\" online_cpuset=\"0x00080000,,,,0x0\" allowed_cpuset=\"0x00080000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x00f00000,,,,0x0\" complete_cpuset=\"0x00f00000,,,,0x0\" online_cpuset=\"0x00f00000,,,,0x0\" allowed_cpuset=\"0x00f00000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x00f00000,,,,0x0\" complete_cpuset=\"0x00f00000,,,,0x0\" online_cpuset=\"0x00f00000,,,,0x0\" allowed_cpuset=\"0x00f00000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x00f00000,,,,0x0\" complete_cpuset=\"0x00f00000,,,,0x0\" online_cpuset=\"0x00f00000,,,,0x0\" allowed_cpuset=\"0x00f00000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2112\" cpuset=\"0x00f00000,,,,0x0\" complete_cpuset=\"0x00f00000,,,,0x0\" online_cpuset=\"0x00f00000,,,,0x0\" allowed_cpuset=\"0x00f00000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"148\" cpuset=\"0x00100000,,,,0x0\" complete_cpuset=\"0x00100000,,,,0x0\" online_cpuset=\"0x00100000,,,,0x0\" allowed_cpuset=\"0x00100000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"149\" cpuset=\"0x00200000,,,,0x0\" complete_cpuset=\"0x00200000,,,,0x0\" online_cpuset=\"0x00200000,,,,0x0\" allowed_cpuset=\"0x00200000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"150\" cpuset=\"0x00400000,,,,0x0\" complete_cpuset=\"0x00400000,,,,0x0\" online_cpuset=\"0x00400000,,,,0x0\" allowed_cpuset=\"0x00400000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"151\" cpuset=\"0x00800000,,,,0x0\" complete_cpuset=\"0x00800000,,,,0x0\" online_cpuset=\"0x00800000,,,,0x0\" allowed_cpuset=\"0x00800000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0xff000000,,,,0x0\" complete_cpuset=\"0xff000000,,,,0x0\" online_cpuset=\"0xff000000,,,,0x0\" allowed_cpuset=\"0xff000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0xff000000,,,,0x0\" complete_cpuset=\"0xff000000,,,,0x0\" online_cpuset=\"0xff000000,,,,0x0\" allowed_cpuset=\"0xff000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0f000000,,,,0x0\" complete_cpuset=\"0x0f000000,,,,0x0\" online_cpuset=\"0x0f000000,,,,0x0\" allowed_cpuset=\"0x0f000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2120\" cpuset=\"0x0f000000,,,,0x0\" complete_cpuset=\"0x0f000000,,,,0x0\" online_cpuset=\"0x0f000000,,,,0x0\" allowed_cpuset=\"0x0f000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"152\" cpuset=\"0x01000000,,,,0x0\" complete_cpuset=\"0x01000000,,,,0x0\" online_cpuset=\"0x01000000,,,,0x0\" allowed_cpuset=\"0x01000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"153\" cpuset=\"0x02000000,,,,0x0\" complete_cpuset=\"0x02000000,,,,0x0\" online_cpuset=\"0x02000000,,,,0x0\" allowed_cpuset=\"0x02000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"154\" cpuset=\"0x04000000,,,,0x0\" complete_cpuset=\"0x04000000,,,,0x0\" online_cpuset=\"0x04000000,,,,0x0\" allowed_cpuset=\"0x04000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"155\" cpuset=\"0x08000000,,,,0x0\" complete_cpuset=\"0x08000000,,,,0x0\" online_cpuset=\"0x08000000,,,,0x0\" allowed_cpuset=\"0x08000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0xf0000000,,,,0x0\" complete_cpuset=\"0xf0000000,,,,0x0\" online_cpuset=\"0xf0000000,,,,0x0\" allowed_cpuset=\"0xf0000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2124\" cpuset=\"0xf0000000,,,,0x0\" complete_cpuset=\"0xf0000000,,,,0x0\" online_cpuset=\"0xf0000000,,,,0x0\" allowed_cpuset=\"0xf0000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"156\" cpuset=\"0x10000000,,,,0x0\" complete_cpuset=\"0x10000000,,,,0x0\" online_cpuset=\"0x10000000,,,,0x0\" allowed_cpuset=\"0x10000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"157\" cpuset=\"0x20000000,,,,0x0\" complete_cpuset=\"0x20000000,,,,0x0\" online_cpuset=\"0x20000000,,,,0x0\" allowed_cpuset=\"0x20000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"158\" cpuset=\"0x40000000,,,,0x0\" complete_cpuset=\"0x40000000,,,,0x0\" online_cpuset=\"0x40000000,,,,0x0\" allowed_cpuset=\"0x40000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"159\" cpuset=\"0x80000000,,,,0x0\" complete_cpuset=\"0x80000000,,,,0x0\" online_cpuset=\"0x80000000,,,,0x0\" allowed_cpuset=\"0x80000000,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x000000ff,,,,,0x0\" complete_cpuset=\"0x000000ff,,,,,0x0\" online_cpuset=\"0x000000ff,,,,,0x0\" allowed_cpuset=\"0x000000ff,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x000000ff,,,,,0x0\" complete_cpuset=\"0x000000ff,,,,,0x0\" online_cpuset=\"0x000000ff,,,,,0x0\" allowed_cpuset=\"0x000000ff,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x0000000f,,,,,0x0\" complete_cpuset=\"0x0000000f,,,,,0x0\" online_cpuset=\"0x0000000f,,,,,0x0\" allowed_cpuset=\"0x0000000f,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2128\" cpuset=\"0x0000000f,,,,,0x0\" complete_cpuset=\"0x0000000f,,,,,0x0\" online_cpuset=\"0x0000000f,,,,,0x0\" allowed_cpuset=\"0x0000000f,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"160\" cpuset=\"0x00000001,,,,,0x0\" complete_cpuset=\"0x00000001,,,,,0x0\" online_cpuset=\"0x00000001,,,,,0x0\" allowed_cpuset=\"0x00000001,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"161\" cpuset=\"0x00000002,,,,,0x0\" complete_cpuset=\"0x00000002,,,,,0x0\" online_cpuset=\"0x00000002,,,,,0x0\" allowed_cpuset=\"0x00000002,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"162\" cpuset=\"0x00000004,,,,,0x0\" complete_cpuset=\"0x00000004,,,,,0x0\" online_cpuset=\"0x00000004,,,,,0x0\" allowed_cpuset=\"0x00000004,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"163\" cpuset=\"0x00000008,,,,,0x0\" complete_cpuset=\"0x00000008,,,,,0x0\" online_cpuset=\"0x00000008,,,,,0x0\" allowed_cpuset=\"0x00000008,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x000000f0,,,,,0x0\" complete_cpuset=\"0x000000f0,,,,,0x0\" online_cpuset=\"0x000000f0,,,,,0x0\" allowed_cpuset=\"0x000000f0,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2132\" cpuset=\"0x000000f0,,,,,0x0\" complete_cpuset=\"0x000000f0,,,,,0x0\" online_cpuset=\"0x000000f0,,,,,0x0\" allowed_cpuset=\"0x000000f0,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"164\" cpuset=\"0x00000010,,,,,0x0\" complete_cpuset=\"0x00000010,,,,,0x0\" online_cpuset=\"0x00000010,,,,,0x0\" allowed_cpuset=\"0x00000010,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"165\" cpuset=\"0x00000020,,,,,0x0\" complete_cpuset=\"0x00000020,,,,,0x0\" online_cpuset=\"0x00000020,,,,,0x0\" allowed_cpuset=\"0x00000020,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"166\" cpuset=\"0x00000040,,,,,0x0\" complete_cpuset=\"0x00000040,,,,,0x0\" online_cpuset=\"0x00000040,,,,,0x0\" allowed_cpuset=\"0x00000040,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"167\" cpuset=\"0x00000080,,,,,0x0\" complete_cpuset=\"0x00000080,,,,,0x0\" online_cpuset=\"0x00000080,,,,,0x0\" allowed_cpuset=\"0x00000080,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"Cache\" cpuset=\"0x0000ff00,,,,,0x0\" complete_cpuset=\"0x0000ff00,,,,,0x0\" online_cpuset=\"0x0000ff00,,,,,0x0\" allowed_cpuset=\"0x0000ff00,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"10485760\" depth=\"3\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+            <object type=\"Cache\" cpuset=\"0x0000ff00,,,,,0x0\" complete_cpuset=\"0x0000ff00,,,,,0x0\" online_cpuset=\"0x0000ff00,,,,,0x0\" allowed_cpuset=\"0x0000ff00,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"524288\" depth=\"2\" cache_linesize=\"0\" cache_associativity=\"0\" cache_type=\"0\">\n\
+              <object type=\"Cache\" cpuset=\"0x00000f00,,,,,0x0\" complete_cpuset=\"0x00000f00,,,,,0x0\" online_cpuset=\"0x00000f00,,,,,0x0\" allowed_cpuset=\"0x00000f00,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2136\" cpuset=\"0x00000f00,,,,,0x0\" complete_cpuset=\"0x00000f00,,,,,0x0\" online_cpuset=\"0x00000f00,,,,,0x0\" allowed_cpuset=\"0x00000f00,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"168\" cpuset=\"0x00000100,,,,,0x0\" complete_cpuset=\"0x00000100,,,,,0x0\" online_cpuset=\"0x00000100,,,,,0x0\" allowed_cpuset=\"0x00000100,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"169\" cpuset=\"0x00000200,,,,,0x0\" complete_cpuset=\"0x00000200,,,,,0x0\" online_cpuset=\"0x00000200,,,,,0x0\" allowed_cpuset=\"0x00000200,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"170\" cpuset=\"0x00000400,,,,,0x0\" complete_cpuset=\"0x00000400,,,,,0x0\" online_cpuset=\"0x00000400,,,,,0x0\" allowed_cpuset=\"0x00000400,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"171\" cpuset=\"0x00000800,,,,,0x0\" complete_cpuset=\"0x00000800,,,,,0x0\" online_cpuset=\"0x00000800,,,,,0x0\" allowed_cpuset=\"0x00000800,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+              <object type=\"Cache\" cpuset=\"0x0000f000,,,,,0x0\" complete_cpuset=\"0x0000f000,,,,,0x0\" online_cpuset=\"0x0000f000,,,,,0x0\" allowed_cpuset=\"0x0000f000,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\" cache_size=\"32768\" depth=\"1\" cache_linesize=\"128\" cache_associativity=\"32\" cache_type=\"1\">\n\
+                <object type=\"Core\" os_index=\"2140\" cpuset=\"0x0000f000,,,,,0x0\" complete_cpuset=\"0x0000f000,,,,,0x0\" online_cpuset=\"0x0000f000,,,,,0x0\" allowed_cpuset=\"0x0000f000,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\">\n\
+                  <object type=\"PU\" os_index=\"172\" cpuset=\"0x00001000,,,,,0x0\" complete_cpuset=\"0x00001000,,,,,0x0\" online_cpuset=\"0x00001000,,,,,0x0\" allowed_cpuset=\"0x00001000,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"173\" cpuset=\"0x00002000,,,,,0x0\" complete_cpuset=\"0x00002000,,,,,0x0\" online_cpuset=\"0x00002000,,,,,0x0\" allowed_cpuset=\"0x00002000,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"174\" cpuset=\"0x00004000,,,,,0x0\" complete_cpuset=\"0x00004000,,,,,0x0\" online_cpuset=\"0x00004000,,,,,0x0\" allowed_cpuset=\"0x00004000,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                  <object type=\"PU\" os_index=\"175\" cpuset=\"0x00008000,,,,,0x0\" complete_cpuset=\"0x00008000,,,,,0x0\" online_cpuset=\"0x00008000,,,,,0x0\" allowed_cpuset=\"0x00008000,,,,,0x0\" nodeset=\"0x00000100\" complete_nodeset=\"0x00000100\" allowed_nodeset=\"0x00000100\"/>\n\
+                </object>\n\
+              </object>\n\
+            </object>\n\
+          </object>\n\
+        </object>\n\
+        <object type=\"Bridge\" os_index=\"9\" bridge_type=\"0-1\" depth=\"0\" bridge_pci=\"0033:[00-01]\">\n\
+          <object type=\"PCIDev\" os_index=\"53481472\" name=\"Mellanox Technologies MT28800 Family [ConnectX-5 Ex]\" pci_busid=\"0033:01:00.0\" pci_type=\"0207 [15b3:1019] [1014:0617] 00\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"Mellanox Technologies\"/>\n\
+            <info name=\"PCIDevice\" value=\"MT28800 Family [ConnectX-5 Ex]\"/>\n\
+            <object type=\"OSDev\" name=\"hsi2\" osdev_type=\"2\">\n\
+              <info name=\"Address\" value=\"00:00:14:87:fe:80:00:00:00:00:00:00:ec:0d:9a:03:00:ca:a4:1a\"/>\n\
+              <info name=\"Port\" value=\"1\"/>\n\
+            </object>\n\
+            <object type=\"OSDev\" name=\"mlx5_2\" osdev_type=\"3\">\n\
+              <info name=\"NodeGUID\" value=\"ec0d:9a03:00ca:a41a\"/>\n\
+              <info name=\"SysImageGUID\" value=\"ec0d:9a03:00ca:a418\"/>\n\
+              <info name=\"Port1State\" value=\"4\"/>\n\
+              <info name=\"Port1LID\" value=\"0x335d\"/>\n\
+              <info name=\"Port1LMC\" value=\"0\"/>\n\
+              <info name=\"Port1GID0\" value=\"fe80:0000:0000:0000:ec0d:9a03:00ca:a41a\"/>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"PCIDev\" os_index=\"53481473\" name=\"Mellanox Technologies MT28800 Family [ConnectX-5 Ex]\" pci_busid=\"0033:01:00.1\" pci_type=\"0207 [15b3:1019] [1014:0617] 00\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"Mellanox Technologies\"/>\n\
+            <info name=\"PCIDevice\" value=\"MT28800 Family [ConnectX-5 Ex]\"/>\n\
+            <object type=\"OSDev\" name=\"hsi3\" osdev_type=\"2\">\n\
+              <info name=\"Address\" value=\"00:00:1c:87:fe:80:00:00:00:00:00:00:ec:0d:9a:03:00:ca:a4:1b\"/>\n\
+              <info name=\"Port\" value=\"1\"/>\n\
+            </object>\n\
+            <object type=\"OSDev\" name=\"mlx5_3\" osdev_type=\"3\">\n\
+              <info name=\"NodeGUID\" value=\"ec0d:9a03:00ca:a41b\"/>\n\
+              <info name=\"SysImageGUID\" value=\"ec0d:9a03:00ca:a418\"/>\n\
+              <info name=\"Port1State\" value=\"4\"/>\n\
+              <info name=\"Port1LID\" value=\"0x3333\"/>\n\
+              <info name=\"Port1LMC\" value=\"0\"/>\n\
+              <info name=\"Port1GID0\" value=\"fe80:0000:0000:0000:ec0d:9a03:00ca:a41b\"/>\n\
+            </object>\n\
+          </object>\n\
+        </object>\n\
+        <object type=\"Bridge\" os_index=\"11\" bridge_type=\"0-1\" depth=\"0\" bridge_pci=\"0035:[00-09]\">\n\
+          <object type=\"PCIDev\" os_index=\"55586816\" name=\"NVIDIA Corporation GV100GL [Tesla V100 SXM2]\" pci_busid=\"0035:03:00.0\" pci_type=\"0302 [10de:1db1] [10de:1212] a1\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"NVIDIA Corporation\"/>\n\
+            <info name=\"PCIDevice\" value=\"GV100GL [Tesla V100 SXM2]\"/>\n\
+            <object type=\"OSDev\" name=\"card2\" osdev_type=\"1\"/>\n\
+            <object type=\"OSDev\" name=\"renderD130\" osdev_type=\"1\"/>\n\
+            <object type=\"OSDev\" name=\"cuda2\" osdev_type=\"5\">\n\
+              <info name=\"CoProcType\" value=\"CUDA\"/>\n\
+              <info name=\"Backend\" value=\"CUDA\"/>\n\
+              <info name=\"GPUVendor\" value=\"NVIDIA Corporation\"/>\n\
+              <info name=\"GPUModel\" value=\"Tesla V100-SXM2-16GB\"/>\n\
+              <info name=\"CUDAGlobalMemorySize\" value=\"16515072\"/>\n\
+              <info name=\"CUDAL2CacheSize\" value=\"6144\"/>\n\
+              <info name=\"CUDAMultiProcessors\" value=\"80\"/>\n\
+              <info name=\"CUDACoresPerMP\" value=\"64\"/>\n\
+              <info name=\"CUDASharedMemorySizePerMP\" value=\"48\"/>\n\
+            </object>\n\
+          </object>\n\
+          <object type=\"PCIDev\" os_index=\"55590912\" name=\"NVIDIA Corporation GV100GL [Tesla V100 SXM2]\" pci_busid=\"0035:04:00.0\" pci_type=\"0302 [10de:1db1] [10de:1212] a1\" pci_link_speed=\"0.000000\">\n\
+            <info name=\"PCIVendor\" value=\"NVIDIA Corporation\"/>\n\
+            <info name=\"PCIDevice\" value=\"GV100GL [Tesla V100 SXM2]\"/>\n\
+            <object type=\"OSDev\" name=\"card3\" osdev_type=\"1\"/>\n\
+            <object type=\"OSDev\" name=\"renderD131\" osdev_type=\"1\"/>\n\
+            <object type=\"OSDev\" name=\"cuda3\" osdev_type=\"5\">\n\
+              <info name=\"CoProcType\" value=\"CUDA\"/>\n\
+              <info name=\"Backend\" value=\"CUDA\"/>\n\
+              <info name=\"GPUVendor\" value=\"NVIDIA Corporation\"/>\n\
+              <info name=\"GPUModel\" value=\"Tesla V100-SXM2-16GB\"/>\n\
+              <info name=\"CUDAGlobalMemorySize\" value=\"16515072\"/>\n\
+              <info name=\"CUDAL2CacheSize\" value=\"6144\"/>\n\
+              <info name=\"CUDAMultiProcessors\" value=\"80\"/>\n\
+              <info name=\"CUDACoresPerMP\" value=\"64\"/>\n\
+              <info name=\"CUDASharedMemorySizePerMP\" value=\"48\"/>\n\
+            </object>\n\
+          </object>\n\
+        </object>\n\
+      </object>\n\
+    </object>\n\
+    <object type=\"NUMANode\" os_index=\"252\" cpuset=\"0x0\" complete_cpuset=\"0x0\" online_cpuset=\"0x0\" allowed_cpuset=\"0x0\" nodeset=\"0x10000000,,,,,,,0x0\" complete_nodeset=\"0x10000000,,,,,,,0x0\" allowed_nodeset=\"0x10000000,,,,,,,0x0\" local_memory=\"16911433728\">\n\
+      <page_type size=\"65536\" count=\"258048\"/>\n\
+      <page_type size=\"2097152\" count=\"0\"/>\n\
+      <page_type size=\"1073741824\" count=\"0\"/>\n\
+    </object>\n\
+    <object type=\"NUMANode\" os_index=\"253\" cpuset=\"0x0\" complete_cpuset=\"0x0\" online_cpuset=\"0x0\" allowed_cpuset=\"0x0\" nodeset=\"0x20000000,,,,,,,0x0\" complete_nodeset=\"0x20000000,,,,,,,0x0\" allowed_nodeset=\"0x20000000,,,,,,,0x0\" local_memory=\"16911433728\">\n\
+      <page_type size=\"65536\" count=\"258048\"/>\n\
+      <page_type size=\"2097152\" count=\"0\"/>\n\
+      <page_type size=\"1073741824\" count=\"0\"/>\n\
+    </object>\n\
+    <object type=\"NUMANode\" os_index=\"254\" cpuset=\"0x0\" complete_cpuset=\"0x0\" online_cpuset=\"0x0\" allowed_cpuset=\"0x0\" nodeset=\"0x40000000,,,,,,,0x0\" complete_nodeset=\"0x40000000,,,,,,,0x0\" allowed_nodeset=\"0x40000000,,,,,,,0x0\" local_memory=\"16911433728\">\n\
+      <page_type size=\"65536\" count=\"258048\"/>\n\
+      <page_type size=\"2097152\" count=\"0\"/>\n\
+      <page_type size=\"1073741824\" count=\"0\"/>\n\
+    </object>\n\
+    <object type=\"NUMANode\" os_index=\"255\" cpuset=\"0x0\" complete_cpuset=\"0x0\" online_cpuset=\"0x0\" allowed_cpuset=\"0x0\" nodeset=\"0x80000000,,,,,,,0x0\" complete_nodeset=\"0x80000000,,,,,,,0x0\" allowed_nodeset=\"0x80000000,,,,,,,0x0\" local_memory=\"16911433728\">\n\
+      <page_type size=\"65536\" count=\"258048\"/>\n\
+      <page_type size=\"2097152\" count=\"0\"/>\n\
+      <page_type size=\"1073741824\" count=\"0\"/>\n\
+    </object>\n\
+  </object>\n\
+</topology>";
+
+void test_hwloc (const char *xml)
+{
+    char *s;
+    struct hostlist *hl;
+    json_t *R;
+    struct rlist *rl = rlist_from_hwloc (0, xml);
+
+    if (!rl)
+        BAIL_OUT ("rlist_from_hwloc failed!");
+    if (!(hl = rlist_nodelist (rl)))
+        BAIL_OUT ("rlist_nodelist failed");
+
+    ok (rlist_nnodes (rl) == 1 && rl->total > 0,
+        "rlist_from_hwloc() was able to gather hwloc from %s",
+        xml ? "xml" : "local");
+
+    s = rlist_dumps (rl);
+    diag ("%s", s);
+    free (s);
+
+    R = rlist_to_R (rl);
+    s = json_dumps (R, JSON_COMPACT);
+    json_decref (R);
+    diag ("%s", s);
+    free (s);
+
+    ok (hostlist_count (hl) == 1,
+        "rlist_nodelist works on rlist gathered via hwloc");
+
+    s = hostlist_encode (hl);
+    diag ("hosts=%s", s);
+    free (s);
+
+    rlist_destroy (rl);
+    hostlist_destroy (hl);
+}
+
+void test_xml ()
+{
+    flux_error_t error;
+    struct rlist *rl1 = NULL;
+    struct rlist *rl2 = NULL;
+    char *s1 = NULL;
+    char *s2 = NULL;
+    char *xml = rhwloc_local_topology_xml (0);
+    if (!xml)
+        BAIL_OUT ("rhwloc_local_topology_xml failed!");
+    pass ("rhwloc_topology_xml");
+
+    /*  rlist from local XML and rlist from local directly should match */
+
+    rl1 = rlist_from_hwloc (0, NULL);
+    rl2 = rlist_from_hwloc (0, xml);
+    if (!rl1 || !rl2)
+        BAIL_OUT ("rlist_from_hwloc failed!");
+
+    s1 = rlist_dumps (rl1);
+    s2 = rlist_dumps (rl2);
+    diag ("local hwloc got %s, local XML got %s", s1, s2);
+    free (s1);
+    free (s2);
+
+    ok (rlist_verify (&error, rl1, rl2) == 0,
+        "rlist from local hwloc and rlist from local XML match: %s",
+        error.text);
+
+    rlist_destroy (rl1);
+    rlist_destroy (rl2);
+    free (xml);
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    test_hwloc (NULL);
+    test_hwloc (xml1);
+    test_xml ();
+
+    done_testing ();
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librlist/test/rlist.c b/src/common/librlist/test/rlist.c
new file mode 100644
index 000000000000..eebb7ec50bd2
--- /dev/null
+++ b/src/common/librlist/test/rlist.c
@@ -0,0 +1,2187 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <errno.h>
+#include <jansson.h>
+
+#include "src/common/libtap/tap.h"
+#include "rlist.h"
+
+struct testalloc {
+    int nnodes;
+    int nslots;
+    int slot_size;
+    int exclusive;
+};
+
+struct rlist_test_entry {
+    const char *description;
+    const char *mode;
+    const char *down;
+    struct testalloc alloc;
+    const char *result;
+    const char *allocated;
+    const char *avail;
+    int expected_errno;
+    bool free;
+};
+
+#define RLIST_TEST_END { NULL, NULL, NULL, \
+                         { 0, 0, 0, 0 },      \
+                         NULL, NULL, NULL, \
+                         0, false }
+
+struct rlist_test_entry test_2n_4c[] = {
+    { "too large of slot returns EOVERFLOW", NULL, NULL,
+      { 0, 1, 5, 0 },
+      NULL,
+      "",
+      "rank[0-1]/core[0-3]",
+      EOVERFLOW, false },
+    { "too many slots returns error", NULL, NULL,
+      { 0, 9, 1, 0 },
+      NULL,
+      "",
+      "rank[0-1]/core[0-3]",
+      EOVERFLOW, false },
+    { "invalid number of nodes returns error", NULL, NULL,
+      { -1, 1, 1, 0 },
+      NULL,
+      "",
+      "rank[0-1]/core[0-3]",
+      EINVAL, false },
+    { "Too many nodes returns error", NULL, NULL,
+      { 3, 4, 1, 0 },
+      NULL,
+      "",
+      "rank[0-1]/core[0-3]",
+      EOVERFLOW, false },
+    { "nodes > slots returns error", NULL, NULL,
+      { 2, 1, 1, 0 },
+      NULL,
+      "",
+      "rank[0-1]/core[0-3]",
+      EINVAL, false },
+    { "invalid number of slots return error", NULL, NULL,
+      { 0, 0, 1, 0 },
+      NULL,
+      "",
+      "rank[0-1]/core[0-3]",
+      EINVAL, false },
+    { "invalid slot size returns error", NULL, NULL,
+      { 0, 1, -1, 0 },
+      NULL,
+      "",
+      "rank[0-1]/core[0-3]",
+      EINVAL, false },
+    { "allocate with all nodes down returns ENOSPC", NULL, "0-1",
+      { 0, 1, 1, 0 },
+      NULL,
+      "",
+      "",
+      ENOSPC, false },
+    { "allocating a single core gets expected result", NULL, NULL,
+      { 0, 1, 1, 0 },
+      "rank0/core0",
+      "rank0/core0",
+      "rank0/core[1-3] rank1/core[0-3]",
+      0, true },
+    { "allocating a single core with down rank", NULL, "0",
+      { 0, 1, 1, 0 },
+      "rank1/core0",
+      "rank1/core0",
+      "rank1/core[1-3]",
+      0, false },
+    { "allocating another core (all ranks up)", NULL, NULL,
+      { 0, 1, 1, 0 },
+      "rank0/core0",
+      "rank[0-1]/core0",
+      "rank[0-1]/core[1-3]",
+      0, false },
+    { "allocating another core gets expected result", NULL, NULL,
+      { 0, 1, 1, 0 },
+      "rank0/core1",
+      "rank0/core[0-1] rank1/core0",
+      "rank0/core[2-3] rank1/core[1-3]",
+      0, false },
+    { "allocate 1 slot of size 3 lands on correct node", NULL, NULL,
+      { 0, 1, 3, 0 },
+      "rank1/core[1-3]",
+      "rank0/core[0-1] rank1/core[0-3]",
+      "rank0/core[2-3]",
+      0, false },
+    { "allocate 4 slots of 1 core now returns ENOSPC", NULL, NULL,
+      { 0, 4, 1, 0 },
+      NULL,
+      "rank0/core[0-1] rank1/core[0-3]",
+      "rank0/core[2-3]",
+      ENOSPC, false },
+    { "allocate remaining 2 cores", NULL, NULL,
+      { 0, 1, 2, 0 },
+      "rank0/core[2-3]",
+      "rank[0-1]/core[0-3]",
+      "",
+       0, false },
+    RLIST_TEST_END,
+};
+
+struct rlist_test_entry test_6n_4c[] = {
+    { "best-fit: alloc 1 core", "best-fit", NULL,
+      { 0, 1, 1, 0 },
+      "rank0/core0",
+      "rank0/core0",
+      "rank0/core[1-3] rank[1-5]/core[0-3]",
+      0, false },
+    { "best-fit: alloc 1 slot/size 3 fits on rank0", "best-fit", NULL,
+      { 0, 1, 3, 0 },
+      "rank0/core[1-3]",
+      "rank0/core[0-3]",
+      "rank[1-5]/core[0-3]",
+      0, false },
+    { "best-fit: alloc 2 slots/size 2 fits on rank1","best-fit", NULL,
+      { 0, 2, 2, 0 },
+      "rank1/core[0-3]",
+      "rank[0-1]/core[0-3]",
+      "rank[2-5]/core[0-3]",
+      0, false },
+    { "best-fit: alloc 3 slot of size 1",            "best-fit", NULL,
+      { 0, 3, 1, 0 },
+      "rank2/core[0-2]",
+      "rank[0-1]/core[0-3] rank2/core[0-2]",
+      "rank2/core3 rank[3-5]/core[0-3]",
+      0, false },
+    { "best-fit alloc 3 slots of 1 core",            "best-fit", NULL,
+      { 0, 3, 1, 0 },
+      "rank2/core3 rank3/core[0-1]",
+      "rank[0-2]/core[0-3] rank3/core[0-1]",
+      "rank3/core[2-3] rank[4-5]/core[0-3]",
+      0, false },
+    RLIST_TEST_END,
+};
+
+struct rlist_test_entry test_1024n_4c[] = {
+    { "large: 512 nodes with 2 cores", NULL, NULL,
+      { 512, 512, 2, 0 },
+      "rank[0-511]/core[0-1]",
+      "rank[0-511]/core[0-1]",
+      "rank[0-511]/core[2-3] rank[512-1023]/core[0-3]",
+      0, false
+    },
+    { "large: 512 slots of 4 cores", NULL, NULL,
+      { 0, 512, 4, 0 },
+      "rank[512-1023]/core[0-3]",
+      "rank[0-511]/core[0-1] rank[512-1023]/core[0-3]",
+      "rank[0-511]/core[2-3]",
+      0, true
+    },
+    { "large: 1 core on 10 nodes", NULL, NULL,
+      { 10, 10, 1, 0 },
+      "rank[512-521]/core0",
+      "rank[0-511]/core[0-1] rank[512-521]/core0",
+      "rank[0-511]/core[2-3] rank[512-521]/core[1-3] rank[522-1023]/core[0-3]",
+      0, false },
+    { "large: alloc 2 cores on 128 nodes with free", NULL, NULL,
+      { 128, 256, 1, 0 },
+      "rank[522-649]/core[0-1]",
+      "rank[0-511,522-649]/core[0-1] rank[512-521]/core0",
+      "rank[0-511,522-649]/core[2-3] rank[512-521]/core[1-3] rank[650-1023]/core[0-3]",
+      0, true
+    },
+    RLIST_TEST_END,
+};
+
+
+struct rlist_test_entry test_exclusive[] = {
+    { "exclusive: exclusive without nnodes fails",
+      NULL,
+      NULL,
+      { 0, 1, 1, 1 },
+      NULL,
+      "",
+      "rank[0-3]/core[0-3]",
+      EINVAL,
+      false
+    },
+    { "exclusive: allocate one core first",
+      NULL,
+      NULL,
+      { 0, 1, 1, 0 },
+      "rank0/core0",
+      "rank0/core0",
+      "rank0/core[1-3] rank[1-3]/core[0-3]",
+      0,
+      false
+    },
+    { "exclusive: exclusively allocate 2 nodes",
+      NULL,
+      NULL,
+      { 2, 2, 1, 1 },
+      "rank[1-2]/core[0-3]",
+      "rank0/core0 rank[1-2]/core[0-3]",
+      "rank0/core[1-3] rank3/core[0-3]",
+      0,
+      false
+    },
+    { "exclusive: exclusively allocate 2 nodes fails",
+      NULL,
+      NULL,
+      { 2, 2, 1, 1 },
+      NULL,
+      "rank0/core0 rank[1-2]/core[0-3]",
+      "rank0/core[1-3] rank3/core[0-3]",
+      ENOSPC,
+      false
+    },
+    { "exclusive: but 1 node works",
+      NULL,
+      NULL,
+      { 1, 1, 1, 1 },
+      "rank3/core[0-3]",
+      "rank0/core0 rank[1-3]/core[0-3]",
+      "rank0/core[1-3]",
+      0,
+      false
+    },
+    { "exclusive: last 3 cores can be allocated non-exclusively",
+      NULL,
+      NULL,
+      { 0, 3, 1, 0 },
+      "rank0/core[1-3]",
+      "rank[0-3]/core[0-3]",
+      "",
+      0,
+      false,
+    },
+    RLIST_TEST_END,
+};
+
+
+char *R_create (const char *ranklist,
+                const char *corelist,
+                const char *gpus,
+                const char *nodelist,
+                const char *properties)
+{
+    char *retval;
+    json_t *o = NULL;
+    json_t *R_lite = NULL;
+
+    if (gpus) {
+        if (!(R_lite = json_pack ("{s:s,s:{s:s s:s}}",
+                        "rank", ranklist,
+                        "children",
+                          "core", corelist ? corelist : "",
+                          "gpu", gpus)))
+            goto err;
+    }
+    else {
+        if (!(R_lite = json_pack ("{s:s,s:{s:s}}",
+                        "rank", ranklist,
+                        "children", "core", corelist)))
+            goto err;
+    }
+    if (nodelist) {
+        if (!(o = json_pack ("{s:i, s:{s:[s], s:[O]}}",
+                   "version", 1,
+                   "execution",
+                     "nodelist", nodelist,
+                     "R_lite", R_lite)))
+            goto err;
+    }
+    else {
+        if (!(o = json_pack ("{s:i, s:{s:[O]}}",
+                   "version", 1,
+                   "execution", "R_lite", R_lite)))
+            goto err;
+    }
+    if (properties) {
+        json_error_t error;
+        json_t *execution;
+        json_t *prop = json_loads (properties, JSON_DECODE_ANY, &error);
+        if (!prop)
+            BAIL_OUT ("json_loads ('%s'): %s", properties, error.text);
+        if (!(execution = json_object_get (o, "execution")))
+            goto err;
+        json_object_set_new (execution, "properties", prop);
+    }
+    retval = json_dumps (o, JSON_COMPACT|JSON_ENCODE_ANY);
+    json_decref (o);
+    json_decref (R_lite);
+    return (retval);
+err:
+    json_decref (o);
+    json_decref (R_lite);
+    return NULL;
+}
+
+static struct rlist * rl_alloc (struct rlist *rl,
+                                const char *mode,
+                                int nnodes,
+                                int nslots,
+                                int slot_size,
+                                int exclusive)
+{
+    struct rlist_alloc_info ai = {
+        .mode = mode,
+        .nnodes = nnodes,
+        .nslots = nslots,
+        .slot_size = slot_size,
+        .exclusive = exclusive
+    };
+    flux_error_t error;
+    struct rlist *result = rlist_alloc (rl, &ai, &error);
+    if (!result)
+        diag ("rlist_alloc: %s", error.text);
+    return result;
+}
+
+static struct rlist * rlist_testalloc (struct rlist *rl,
+                                       struct rlist_test_entry *e)
+{
+    return rl_alloc (rl, e->mode,
+                     e->alloc.nnodes,
+                     e->alloc.nslots,
+                     e->alloc.slot_size,
+                     e->alloc.exclusive);
+}
+
+static char * rlist_tostring (struct rlist *rl, bool allocated)
+{
+    char *result;
+    struct rlist *l = NULL;
+    char *s = NULL;
+    json_t *R = NULL;
+
+    if (allocated) {
+        struct rlist *alloc = rlist_copy_allocated (rl);
+        if (!alloc)
+            BAIL_OUT ("rlist_copy_allocated failed! %s", strerror (errno));
+        R = rlist_to_R (alloc);
+        rlist_destroy (alloc);
+    }
+    else
+        R = rlist_to_R (rl);
+
+    if (!R || !(s = json_dumps (R, JSON_COMPACT)))
+        BAIL_OUT ("rlist_to_R* failed!");
+    if (!(l = rlist_from_R (s)))
+        BAIL_OUT ("rlist_from_R failed!");
+
+    result = rlist_dumps (l);
+    rlist_destroy (l);
+    free (s);
+    json_decref (R);
+    return result;
+}
+
+static char *R_create_num (int ranks, int cores)
+{
+    char corelist[64];
+    char ranklist[64];
+    if ((snprintf (corelist, sizeof (corelist)-1, "0-%d", cores-1) < 0)
+        || (snprintf (ranklist, sizeof (ranklist)-1, "0-%d", ranks -1) < 0))
+        return NULL;
+    return R_create (ranklist, corelist, NULL, NULL, NULL);
+}
+
+void run_test_entries (struct rlist_test_entry tests[], int ranks, int cores)
+{
+    struct rlist *rl = NULL;
+    struct rlist *alloc = NULL;
+    struct rlist_test_entry *e = NULL;
+    char *R = R_create_num (ranks, cores);
+    if (R == NULL)
+        BAIL_OUT ("R_create (ranks=%d, cores=%d) failed", ranks, cores);
+    if (!(rl = rlist_from_R (R)))
+        BAIL_OUT ("rlist_from_R (%s)", R);
+    free (R);
+
+    e = &tests[0];
+    while (e && e->description) {
+        int avail_start = rl->avail;
+
+        if (e->down)
+            ok (rlist_mark_down (rl, e->down) == 0,
+                "marking ranks %s down", e->down);
+
+        alloc = rlist_testalloc (rl, e);
+        if (e->result == NULL) {  // Test for expected failure
+            ok (alloc == NULL && errno == e->expected_errno,
+                "%s: errno=%d", e->description, errno);
+        }
+        else {
+            if (alloc) {
+                char *result = rlist_dumps (alloc);
+                is (result, e->result, "%s: %s", e->description, result);
+                if (e->allocated) {
+                    char *s = rlist_tostring (rl, true);
+                    diag ("total=%d, avail=%d", rl->total, rl->avail);
+                    is (s, e->allocated, "%s: alloc: %s", e->description, s);
+                    free (s);
+                }
+                if (e->avail) {
+                    char *s = rlist_tostring (rl, false);
+                    is (s, e->avail, "%s: avail: %s", e->description, s);
+                    free (s);
+                }
+                if (e->free) {
+                    ok (rlist_free (rl, alloc) == 0, "rlist_free (%s)", result);
+                    ok (avail_start == rl->avail, "freed all cores");
+                }
+                free (result);
+                rlist_destroy (alloc);
+            }
+            else {
+                fail ("%s: rlist_testalloc: %s",
+                      e->description,
+                      strerror (errno));
+            }
+        }
+
+        if (e->down)
+            ok (rlist_mark_up (rl, e->down) == 0,
+                "marking ranks %s back up", e->down);
+
+        char *s = rlist_dumps (rl);
+        diag ("avail=%s", s);
+        free (s);
+        e++;
+    }
+    rlist_destroy (rl);
+}
+
+static void test_simple (void)
+{
+    struct rlist *rl = NULL;
+    struct rlist *alloc = NULL;
+    struct rlist *copy = NULL;
+
+    if (!(rl = rlist_create ()))
+        BAIL_OUT ("Failed to create rlist");
+
+    ok (rl->total == 0 && rl->avail == 0,
+        "rlist_create creates empty list");
+    ok (rlist_append_rank_cores (rl, "host", 0, "0-3") == 0,
+        "rlist_append_rank_cores 0, 0-3");
+    ok (rl->total == 4 && rl->avail == 4,
+        "rlist: avail and total == 4");
+    ok (rlist_append_rank_cores (rl, "host", 1, "0-3") == 0,
+        "rlist_append_rank_cores 1, 0-3");
+    ok (rl->total == 8 && rl->avail == 8,
+        "rlist: avail and total == 4");
+    ok ((alloc = rl_alloc (rl, NULL, 0, 8, 1, 0)) != NULL,
+        "rlist: alloc all cores works");
+    ok (alloc->total == 8 && alloc->avail == 8,
+        "rlist: alloc: got %d/%d (expected 8/8)",
+        alloc->avail, alloc->total);
+    ok (rl->total == 8 && rl->avail == 0,
+        "rlist: avail == 0, total == 8");
+    ok ((copy = rlist_copy_empty (rl)) != NULL,
+        "rlist: rlist_copy_empty");
+    if (!copy)
+        BAIL_OUT ("rlist_copy_empty failed!");
+    ok (copy->total == 8 && copy->avail == 8,
+        "rlist: copy: total = %d, avail = %d", copy->total, copy->avail);
+
+    rlist_destroy (rl);
+    rlist_destroy (alloc);
+    rlist_destroy (copy);
+}
+
+const char R_issue2202[] = "{\n\
+  \"version\": 1,\n\
+  \"execution\": {\n\
+    \"R_lite\": [\n\
+      {\n\
+        \"rank\": \"0\",\n\
+        \"children\": {\n\
+          \"core\": \"0\"\n\
+        }\n\
+      },\n\
+      {\n\
+        \"rank\": \"1\",\n\
+        \"children\": {\n\
+          \"core\": \"1\"\n\
+        }\n\
+      },\n\
+      {\n\
+        \"rank\": \"2\",\n\
+        \"children\": {\n\
+          \"core\": \"2\"\n\
+        }\n\
+      },\n\
+      {\n\
+        \"rank\": \"3\",\n\
+        \"children\": {\n\
+          \"core\": \"3\"\n\
+        }\n\
+      }\n\
+    ]\n\
+  }\n\
+}";
+
+const char R_issue2202b[] = "{\
+  \"version\": 1,\n\
+  \"execution\": {\n\
+    \"R_lite\": [\n\
+      {\n\
+        \"rank\": \"0\",\n\
+        \"children\": {\n\
+          \"core\": \"0-1\"\n\
+        }\n\
+      },\n\
+      {\n\
+        \"rank\": \"1\",\n\
+        \"children\": {\n\
+          \"core\": \"0,2\"\n\
+        }\n\
+      },\n\
+      {\n\
+        \"rank\": \"2\",\n\
+        \"children\": {\n\
+          \"core\": \"0,3\"\n\
+        }\n\
+      },\n\
+      {\n\
+        \"rank\": \"3\",\n\
+        \"children\": {\n\
+          \"core\": \"3-4\"\n\
+        }\n\
+      }\n\
+    ]\n\
+  }\n\
+}";
+
+static void test_issue2202 (void)
+{
+    char *result = NULL;
+    struct rlist *a = NULL;
+
+    struct rlist *rl = rlist_from_R (R_issue2202);
+    if (!rl)
+        BAIL_OUT ("unable to create rlist from R_issue2202");
+    ok (rl != NULL, "issue2202: rlist_from_R");
+
+    result = rlist_dumps (rl);
+    is (result,
+        "rank0/core0 rank1/core1 rank2/core2 rank3/core3",
+        "issue2202: rlist_dumps works");
+    free (result);
+
+    a = rl_alloc (rl, "best-fit", 1, 1, 1, 0);
+    ok (a != NULL,
+        "issue2202: rlist_alloc worked");
+    if (a) {
+        result = rlist_dumps (a);
+        is (result, "rank0/core0", "issue2202: allocated %s", result);
+        free (result);
+        result = rlist_dumps (rl);
+        is (result,
+            "rank1/core1 rank2/core2 rank3/core3",
+            "issue2202: remaining: %s", result);
+        free (result);
+        ok (rlist_free (rl, a) == 0,
+            "issue2202: rlist_free worked: %s", strerror (errno));
+        result = rlist_dumps (rl);
+        is (result,
+            "rank0/core0 rank1/core1 rank2/core2 rank3/core3",
+            "issue2202: rlist now has all cores again");
+        free (result);
+        rlist_destroy (a);
+    }
+    rlist_destroy (rl);
+
+
+    /*  Part B:  test with multiple cores per rank, same cpuset size
+     */
+    rl = rlist_from_R (R_issue2202b);
+    if (!rl)
+        BAIL_OUT ("unable to create rlist from R_issue2202b");
+
+    ok (rl != NULL, "issue2202: rlist_from_R");
+    result = rlist_dumps (rl);
+    is (result,
+        "rank0/core[0-1] rank1/core[0,2] rank2/core[0,3] rank3/core[3-4]",
+        "issue2202b: rlist_dumps works");
+    free (result);
+
+    a = rl_alloc (rl, "best-fit", 1, 1, 1, 0);
+    ok (a != NULL,
+        "issue2202b: rlist_alloc worked");
+    if (a) {
+        result = rlist_dumps (a);
+        is (result, "rank0/core0", "issue2202b: allocated %s", result);
+        free (result);
+        result = rlist_dumps (rl);
+        is (result,
+            "rank0/core1 rank1/core[0,2] rank2/core[0,3] rank3/core[3-4]",
+            "issue2202b: remaining: %s", result);
+        free (result);
+        ok (rlist_free (rl, a) == 0,
+            "issue2202b: rlist_free worked: %s", strerror (errno));
+        result = rlist_dumps (rl);
+        is (result,
+            "rank0/core[0-1] rank1/core[0,2] rank2/core[0,3] rank3/core[3-4]",
+            "issue2202b: rlist now has all cores again");
+        free (result);
+        rlist_destroy (a);
+    }
+    rlist_destroy (rl);
+}
+
+const char R_issue2473[] = "{\
+  \"version\": 1,\n\
+  \"execution\": {\n\
+    \"R_lite\": [\n\
+      {\n\
+        \"rank\": \"0\",\n\
+        \"children\": {\n\
+          \"core\": \"0-3\"\n\
+        }\n\
+      },\n\
+      {\n\
+        \"rank\": \"1-2\",\n\
+        \"children\": {\n\
+          \"core\": \"0-1\"\n\
+        }\n\
+      }\n\
+    ]\n\
+  }\n\
+}";
+
+static void test_issue2473 (void)
+{
+    char *result;
+    struct rlist *rl;
+    struct rlist *a, *a2;
+
+    rl = rlist_from_R (R_issue2473);
+    if (rl == NULL)
+        BAIL_OUT ("unable to create rlist from R_issue2473");
+    ok (rl != NULL, "issue2473: rlist_from_R");
+
+    ok (rlist_nnodes (rl) == 3,
+        "issue2473: created rlist with 3 nodes");
+    result = rlist_dumps (rl);
+    is (result,
+        "rank0/core[0-3] rank[1-2]/core[0-1]",
+        "issue2473: rlist_dumps works");
+    free (result);
+
+    /* problem: allocated 3 cores on one node */
+    a = rl_alloc (rl, "worst-fit", 3, 3, 1, 0);
+    ok (a != NULL,
+        "issue2473: rlist_alloc nnodes=3 slots=3 slotsz=1 worked");
+    if (!a)
+        BAIL_OUT ("rlist_alloc failed");
+    ok (rlist_nnodes (a) == 3,
+        "issue2473: allocation has 3 nodes");
+
+    result = rlist_dumps (a);
+    is (result,
+        "rank[0-2]/core0",
+        "issue2473: rlist_dumps shows one core per node");
+    free (result);
+    rlist_free (rl, a);
+    rlist_destroy (a);
+
+    /* problem: unsatisfiable */
+    a = rl_alloc (rl, "worst-fit", 3, 8, 1, 0);
+    ok (a != NULL,
+        "issue2473: rlist_alloc nnodes=3 slots=8 slotsz=1 worked");
+    if (a) {
+        rlist_free (rl, a);
+        rlist_destroy (a);
+    }
+
+    /* not a problem but verify slightly counter-intuitive case discussed
+     * in the issue:
+     * - alloc 1 core on rank0
+     * - ask for 2 cores spread across 2 nodes
+     * - we should get cores on rank[0-1] not rank[1-2]
+     */
+    a = rl_alloc (rl, "worst-fit", 1, 1, 1, 0);
+    ok (a != NULL,
+        "issue2473: rlist_alloc nnodes=1 slots=1 slotsz=1 worked");
+    if (!a)
+        BAIL_OUT ("rlist_alloc failed");
+
+    result = rlist_dumps (rl);
+    is (result,
+        "rank0/core[1-3] rank[1-2]/core[0-1]",
+        "issue2473: one core was allocated from rank0");
+    free (result);
+
+    a2 = rl_alloc (rl, "worst-fit", 2, 2, 1, 0);
+    ok (a2 != NULL,
+        "issue2473: rlist_alloc nnodes=2 slots=2 slotsz=1 worked");
+    result = rlist_dumps (a2);
+    is (result,
+        "rank0/core1 rank1/core0",
+        "issue2473: allocated a core from used node, not starting new bin");
+    free (result);
+    rlist_free (rl, a);
+    rlist_destroy (a);
+    rlist_free (rl, a2);
+    rlist_destroy (a2);
+
+    rlist_destroy (rl);
+}
+
+static void test_dumps (void)
+{
+    char *result = NULL;
+    struct rlist *rl = NULL;
+
+    if (!(rl = rlist_create ()))
+        BAIL_OUT ("rlist_dumps: failed to create rlist");
+
+    ok (rlist_dumps (NULL) == NULL,
+        "rlist_dumps (NULL) == NULL");
+
+    result = rlist_dumps (rl);
+    is (result, "",
+        "rlist_dumps: empty list returns empty string");
+    free (result);
+
+    rlist_append_rank_cores (rl, "host", 0, "0-3");
+    result = rlist_dumps (rl);
+    is (result, "rank0/core[0-3]",
+        "rlist_dumps with one rank 4 cores gets expected result");
+    free (result);
+
+    rlist_append_rank_cores (rl, "host", 1, "0-7");
+    result = rlist_dumps (rl);
+    is (result, "rank0/core[0-3] rank1/core[0-7]",
+        "rlist_dumps with two ranks gets expected result");
+    free (result);
+
+    rlist_append_rank_cores (rl, "host", 1234567, "0-12345");
+    rlist_append_rank_cores (rl, "host", 1234568, "0-12346");
+    result = rlist_dumps (rl);
+    is (result, "rank0/core[0-3] rank1/core[0-7] "
+                "rank1234567/core[0-12345] rank1234568/core[0-12346]",
+        "rlist_dumps with long result");
+    free (result);
+    rlist_destroy (rl);
+}
+
+static void test_updown ()
+{
+    struct rlist *rl = NULL;
+    struct rlist *rl2 = NULL;
+    char *R = R_create ("0-3", "0-3", NULL, "host[0-3]", NULL);
+    rl = rlist_from_R (R);
+    if (rl == NULL)
+        BAIL_OUT ("rlist_from_R failed");
+    free (R);
+
+    ok (rl->avail == 16,
+        "rlist avail == 16");
+    ok (rlist_mark_down (rl, "all") == 0,
+        "rlist_mark_down: all works");
+    ok (rl->avail == 0,
+        "rlist avail == 0");
+    ok (rlist_mark_up (rl, "0-1") == 0,
+        "rlist_mark_up (0-1) works");
+    ok (rl->avail == 8,
+        "rl avail == 8");
+    ok (rlist_mark_up (rl, "all") == 0,
+        "rlist_mark_up (all) works");
+    ok (rl->avail == 16,
+        "rl avail == 16");
+
+    rl2 = rl_alloc (rl, NULL, 0, 4, 1, 0);
+    ok (rl2 != NULL,
+        "rlist_alloc() works when all nodes up");
+
+    ok (rl->avail == 12,
+        "rl avail == 12");
+
+    ok (rlist_mark_down (rl, "all") == 0,
+        "rlist_mark_down all with some resources allocated");
+
+    ok (rl->avail == 0,
+        "rl avail == 0");
+
+    ok (rlist_free (rl, rl2) == 0,
+        "rlist_free original");
+    rlist_destroy (rl2);
+
+    ok (rl->avail == 0,
+        "rl avail == %d", rl->avail);
+
+    ok (rlist_mark_up (rl, "0-2") == 0,
+        "rlist_mark_up all but rank 3 up");
+
+    ok (rl_alloc (rl, NULL, 4, 4, 1, 0) == NULL && errno == ENOSPC,
+        "allocation with 4 nodes fails with ENOSPC");
+
+    ok (rlist_mark_up (rl, "3") == 0,
+        "rlist_mark_up 3");
+    rl2 = rl_alloc (rl, NULL, 4, 4, 1, 0);
+
+    ok (rl2 != NULL,
+        "rlist_alloc() for 4 nodes now succeeds");
+
+    rlist_destroy (rl);
+    rlist_destroy (rl2);
+}
+
+struct append_test {
+    const char *ranksa;
+    const char *coresa;
+    const char *hostsa;
+
+    const char *ranksb;
+    const char *coresb;
+    const char *hostsb;
+
+    int total_cores;
+    int total_nodes;
+    const char *nodelist;
+};
+
+struct append_test append_tests[] = {
+    {
+        "1", "0-3", "foo15",
+        "0", "0-3", "foo16",
+        8,
+        2,
+        "foo[16,15]",
+    },
+    {
+        "0,2-3", "0-3", "foo[0,2-3]",
+        "1",     "0-3", "foo1",
+        16,
+        4,
+        "foo[0-3]",
+    },
+    {
+        "0", "0-3", "foo0",
+        "0", "4-7", "foo0",
+        8,
+        1,
+        "foo0",
+    },
+    {
+        "[0-1023]", "0-3", "foo[0-1023]",
+        "[1000-1024]",  "4-7", "foo[1000-1024]",
+        4196,
+        1025,
+        "foo[0-1024]",
+    },
+    { 0 },
+};
+
+void test_append (void)
+{
+    struct append_test *t = append_tests;
+
+    while (t && t->ranksa) {
+        struct rlist *rl = NULL;
+        struct rlist *rl2 = NULL;
+        struct hostlist *hl = NULL;
+        char *s1;
+        char *s2;
+        char *R1 = R_create (t->ranksa, t->coresa, NULL, t->hostsa, NULL);
+        char *R2 = R_create (t->ranksb, t->coresb, NULL, t->hostsb, NULL);
+
+        if (!R1 || !R2)
+            BAIL_OUT ("R_create() failed!");
+
+        rl = rlist_from_R (R1);
+        rl2 = rlist_from_R (R2);
+        if (!rl || !rl2)
+            BAIL_OUT ("rlist_from_R failed!");
+        free (R1);
+        free (R2);
+
+        s1 = rlist_dumps (rl);
+        s2 = rlist_dumps (rl2);
+
+
+        ok (rlist_append (rl, rl2) == 0,
+            "rlist_append: %s + %s", s1, s2);
+        rlist_destroy (rl2);
+        free (s1);
+        free (s2);
+
+        s1 = rlist_dumps (rl);
+        diag ("result = %s", s1);
+        free (s1);
+
+        ok (rl->total == t->total_cores,
+            "rlist_append: result has %d cores", rl->total);
+        ok (rlist_nnodes (rl) == t->total_nodes,
+            "rlist_append: result has %d nodes", rlist_nnodes (rl));
+
+        hl = rlist_nodelist (rl);
+        s1 = hostlist_encode (hl);
+        is (s1, t->nodelist,
+            "rlist_append: result has nodelist = %s", s1);
+        free (s1);
+        hostlist_destroy (hl);
+
+        json_t *R = rlist_to_R (rl);
+        R1 = json_dumps (R, JSON_COMPACT);
+        diag ("%s", R1);
+        json_decref (R);
+        free (R1);
+        rlist_destroy (rl);
+
+        t++;
+    }
+}
+
+struct append_test add_tests[] = {
+    {
+        "1", "0-3", "foo15",
+        "0", "0-3", "foo16",
+        8,
+        2,
+        "foo[16,15]",
+    },
+    {
+        "0-1", "0-3", "foo[16,15]",
+        "0", "0-3", "foo16",
+        8,
+        2,
+        "foo[16,15]",
+    },
+    {
+        "0,2-3", "0-3", "foo[0,2-3]",
+        "1",     "0-3", "foo1",
+        16,
+        4,
+        "foo[0-3]",
+    },
+    {
+        "0", "0-3", "foo0",
+        "0", "0-7", "foo0",
+        8,
+        1,
+        "foo0",
+    },
+    {
+        "[0-1023]", "0-3", "foo[0-1023]",
+        "[1000-1024]",  "4-7", "foo[1000-1024]",
+        4196,
+        1025,
+        "foo[0-1024]",
+    },
+    { 0 },
+};
+
+
+void test_add (void)
+{
+    struct append_test *t = add_tests;
+
+    while (t && t->ranksa) {
+        struct rlist *rl = NULL;
+        struct rlist *rl2 = NULL;
+        struct hostlist *hl = NULL;
+        char *s1;
+        char *s2;
+        char *R1 = R_create (t->ranksa, t->coresa, NULL, t->hostsa, NULL);
+        char *R2 = R_create (t->ranksb, t->coresb, NULL, t->hostsb, NULL);
+
+        if (!R1 || !R2)
+            BAIL_OUT ("R_create() failed!");
+
+        rl = rlist_from_R (R1);
+        rl2 = rlist_from_R (R2);
+        if (!rl || !rl2)
+            BAIL_OUT ("rlist_from_R failed!");
+        free (R1);
+        free (R2);
+
+        s1 = rlist_dumps (rl);
+        s2 = rlist_dumps (rl2);
+
+
+        ok (rlist_add (rl, rl2) == 0,
+            "rlist_add: %s + %s", s1, s2);
+        rlist_destroy (rl2);
+        free (s1);
+        free (s2);
+
+        s1 = rlist_dumps (rl);
+        diag ("result = %s", s1);
+        free (s1);
+
+        ok (rl->total == t->total_cores,
+            "rlist_add: result has %d cores", rl->total);
+        ok (rlist_nnodes (rl) == t->total_nodes,
+            "rlist_add: result has %d nodes", rlist_nnodes (rl));
+
+        hl = rlist_nodelist (rl);
+        s1 = hostlist_encode (hl);
+        is (s1, t->nodelist,
+            "rlist_add: result has nodelist = %s", s1);
+        free (s1);
+        hostlist_destroy (hl);
+
+        json_t *R = rlist_to_R (rl);
+        R1 = json_dumps (R, JSON_COMPACT);
+        diag ("%s", R1);
+        json_decref (R);
+        free (R1);
+        rlist_destroy (rl);
+
+        t++;
+    }
+}
+
+struct remap_test {
+    const char *ranks;
+    const char *cores;
+    const char *gpus;
+    const char *hosts;
+
+    const char *result;
+};
+
+struct remap_test remap_tests[] = {
+    {
+        "1,7,9,53", "0-3", NULL, "foo[1,7,9,53]",
+        "rank[0-3]/core[0-3]",
+    },
+    {
+        "1,7,9,53", "1,5,7,9", "1,3", "foo[1,7,9,53]",
+        "rank[0-3]/core[0-3],gpu[1,3]",
+    },
+    { 0 },
+};
+
+void test_remap ()
+{
+    struct remap_test *t = remap_tests;
+
+    while (t && t->ranks) {
+        char *before;
+        char *after;
+        struct rlist *rl;
+        char *R = R_create (t->ranks, t->cores, t->gpus, t->hosts, NULL);
+        if (!R)
+            BAIL_OUT ("R_create failed");
+        if (!(rl = rlist_from_R (R)))
+            BAIL_OUT ("rlist_from_R failed");
+
+        before = rlist_dumps (rl);
+        ok (rlist_remap (rl) == 0,
+                "rlist_remap (%s)", before);
+        after = rlist_dumps (rl);
+        is (after, t->result,
+                "result = %s", after);
+
+        free (before);
+        free (after);
+        rlist_destroy (rl);
+        free (R);
+
+        t++;
+    }
+}
+
+struct remap_test assign_hosts_tests[] = {
+    {
+        "1,7,9,53", "0-3", NULL, "foo[1,7,9,53]",
+        "rank[0-3]/core[0-3]",
+    },
+    {
+        "1,7,9,53", "1,5,7,9", "1,3", "foo[1,7,9,53]",
+        "rank[0-3]/core[0-3],gpu[1,3]",
+    },
+    { 0 },
+};
+
+void test_assign_hosts ()
+{
+    struct remap_test *t = assign_hosts_tests;
+
+    while (t && t->ranks) {
+        char *hosts = NULL;
+        struct hostlist *hl;
+        struct rlist *rl;
+        char *R = R_create (t->ranks, t->cores, t->gpus, NULL, NULL);
+        if (!R)
+            BAIL_OUT ("R_create failed");
+        if (!(rl = rlist_from_R (R)))
+            BAIL_OUT ("rlist_from_R failed");
+
+        ok (rlist_assign_hosts (rl, t->hosts) == 0,
+            "rlist_assign_hosts (%s)", t->hosts);
+
+        if (!(hl = rlist_nodelist (rl)))
+            BAIL_OUT ("rlist_nodelist failed");
+        if (!(hosts = hostlist_encode (hl)))
+            BAIL_OUT ("hostlist_encode failed");
+
+        is (hosts, t->hosts,
+            "reassign hosts to %s worked", hosts);
+
+        free (hosts);
+
+        hostlist_destroy (hl);
+        rlist_destroy (rl);
+        free (R);
+
+        t++;
+    }
+
+}
+
+
+void test_rerank ()
+{
+    struct hostlist *hl = NULL;
+    char *s = NULL;
+    struct rlist *rl = NULL;
+    flux_error_t err;
+    char *R = R_create ("0-15", "0-3", NULL, "foo[0-15]", NULL);
+    if (!R)
+        BAIL_OUT ("R_create failed");
+    if (!(rl = rlist_from_R (R)))
+        BAIL_OUT ("rlist_from_R failed");
+
+    ok (rlist_rerank (rl, "foo[1-15]", &err) < 0 && errno == ENOSPC,
+        "rlist_rerank with too few hosts returns ENOSPC");
+    is (err.text, "Number of hosts (15) is less than node count (16)",
+        "rlist_rerank error message is expected");
+    ok (rlist_rerank (rl, "foo[0-16]", &err) < 0 && errno == EOVERFLOW,
+        "rlist_rerank with too many hosts returns EOVERFLOW");
+    is (err.text, "Number of hosts (17) is greater than node count (16)",
+        "rlist_rerank error message is expected");
+    ok (rlist_rerank (rl, "foo[1-16]", &err) < 0 && errno == ENOENT,
+        "rlist_rerank with invalid host returns ENOENT");
+    is (err.text, "Host foo16 not found in resources",
+        "rlist_rerank error message is expected");
+    ok (rlist_rerank (rl, "foo[0-", &err) < 0 && errno == EINVAL,
+        "rlist_rerank fails with invalid hostlist");
+    is (err.text, "hostlist_decode: foo[0-: Invalid argument",
+        "rlist_rerank error message is expected");
+
+    if (!(hl = rlist_nodelist (rl)) || !(s = hostlist_encode (hl)))
+        BAIL_OUT ("rlist_nodelist/hostlist_encode failed!");
+    is (s, "foo[0-15]",
+        "before: hostlist is %s", s);
+    free (s);
+    hostlist_destroy (hl);
+
+    /* Swap rank 0 to rank 15 */
+    ok (rlist_rerank (rl, "foo[1-15,0]", NULL) == 0,
+        "rlist_rerank works");
+
+    if (!(hl = rlist_nodelist (rl)) || !(s = hostlist_encode (hl)))
+        BAIL_OUT ("rlist_nodelist/hostlist_encode failed!");
+    is (s, "foo[1-15,0]",
+        "after: hostlist is %s", s);
+    free (s);
+    hostlist_destroy (hl);
+
+    rlist_destroy (rl);
+    free (R);
+}
+
+struct op_test {
+    const char *ranksa;
+    const char *coresa;
+    const char *gpusa;
+    const char *hostsa;
+
+    const char *ranksb;
+    const char *coresb;
+    const char *gpusb;
+    const char *hostsb;
+
+    const char *result;
+};
+
+
+struct op_test diff_tests[] = {
+    {
+        "0", "0-3", NULL, "foo15",
+        "0", "0-3", NULL, "foo15",
+        "",
+    },
+    {
+        "0", "0-3", "0-1", "foo15",
+        "0", "0-3", "0-1", "foo15",
+        "",
+    },
+    {
+        "0", "0-3", NULL, "foo15",
+        "0", "0-1", NULL, "foo15",
+        "rank0/core[2-3]",
+    },
+    {
+        "0", "0-3", "0", "foo15",
+        "0", "0-3", NULL, "foo15",
+        "rank0/gpu0",
+    },
+    { 0 },
+};
+
+void test_diff ()
+{
+    struct op_test *t = diff_tests;
+
+    while (t && t->ranksa) {
+        struct rlist *rla = NULL;
+        struct rlist *rlb = NULL;
+        struct rlist *result;
+        char *a;
+        char *b;
+        char *s;
+        char *Ra = R_create (t->ranksa, t->coresa, t->gpusa, t->hostsa, NULL);
+        char *Rb = R_create (t->ranksb, t->coresb, t->gpusb, t->hostsb, NULL);
+
+        if (!Ra || !Rb)
+            BAIL_OUT ("R_create() failed!");
+
+        diag ("%s", Ra);
+
+        rla = rlist_from_R (Ra);
+        rlb = rlist_from_R (Rb);
+        if (!rla || !rlb)
+            BAIL_OUT ("rlist_from_R failed!");
+        free (Ra);
+        free (Rb);
+
+        a = rlist_dumps (rla);
+        b = rlist_dumps (rlb);
+
+        result = rlist_diff (rla, rlb);
+        if (!result)
+            BAIL_OUT ("rlist_diff (%s, %s) failed", a, b);
+
+        s = rlist_dumps (result);
+        is (s, t->result,
+            "rlist_diff: %s - %s = %s",
+            a, b, s);
+
+        free (a);
+        free (b);
+        free (s);
+        rlist_destroy (result);
+        rlist_destroy (rla);
+        rlist_destroy (rlb);
+
+        t++;
+    }
+}
+
+struct op_test union_tests[] = {
+    {
+        "0", "0-3", NULL, "foo15",
+        "0", "0-3", NULL, "foo15",
+        "rank0/core[0-3]",
+    },
+    {
+        "0", "0-3", "0-1", "foo15",
+        "1", "0-3", "0-1", "foo16",
+        "rank[0-1]/core[0-3],gpu[0-1]",
+    },
+    {
+        "0", "0-3", NULL, "foo15",
+        "0", NULL, "0", "foo15",
+        "rank0/core[0-3],gpu0",
+    },
+    { 0 },
+};
+
+
+void test_union ()
+{
+    struct op_test *t = union_tests;
+
+    while (t && t->ranksa) {
+        struct rlist *rla = NULL;
+        struct rlist *rlb = NULL;
+        struct rlist *result;
+        char *a;
+        char *b;
+        char *s;
+        char *Ra = R_create (t->ranksa, t->coresa, t->gpusa, t->hostsa, NULL);
+        char *Rb = R_create (t->ranksb, t->coresb, t->gpusb, t->hostsb, NULL);
+
+        if (!Ra || !Rb)
+            BAIL_OUT ("R_create() failed!");
+
+        rla = rlist_from_R (Ra);
+        rlb = rlist_from_R (Rb);
+        if (!rla || !rlb)
+            BAIL_OUT ("rlist_from_R failed!");
+        free (Ra);
+        free (Rb);
+
+        a = rlist_dumps (rla);
+        b = rlist_dumps (rlb);
+
+        result = rlist_union (rla, rlb);
+        if (!result)
+            BAIL_OUT ("rlist_union (%s, %s) failed", a, b);
+
+        s = rlist_dumps (result);
+        is (s, t->result,
+            "rlist_union: %s U %s = %s",
+            a, b, s);
+
+        free (a);
+        free (b);
+        free (s);
+        rlist_destroy (result);
+        rlist_destroy (rla);
+        rlist_destroy (rlb);
+
+        t++;
+    }
+
+}
+
+
+struct op_test intersect_tests[] = {
+    {
+        "0-10", "0-3", NULL, "foo[0-10]",
+        "9-15",   "1", NULL, "foo[9-15]",
+        "rank[9-10]/core1",
+    },
+    {
+        "0", "0-3", "0-1", "foo15",
+        "1", "0-3", "0-1", "foo16",
+        "",
+    },
+    { 0 },
+};
+
+
+void test_intersect ()
+{
+    struct op_test *t = intersect_tests;
+
+    while (t && t->ranksa) {
+        struct rlist *rla = NULL;
+        struct rlist *rlb = NULL;
+        struct rlist *result;
+        char *a;
+        char *b;
+        char *s;
+        char *Ra = R_create (t->ranksa, t->coresa, t->gpusa, t->hostsa, NULL);
+        char *Rb = R_create (t->ranksb, t->coresb, t->gpusb, t->hostsb, NULL);
+
+        if (!Ra || !Rb)
+            BAIL_OUT ("R_create() failed!");
+
+        rla = rlist_from_R (Ra);
+        rlb = rlist_from_R (Rb);
+        if (!rla || !rlb)
+            BAIL_OUT ("rlist_from_R failed!");
+        free (Ra);
+        free (Rb);
+
+        a = rlist_dumps (rla);
+        b = rlist_dumps (rlb);
+
+        result = rlist_intersect (rla, rlb);
+        if (!result)
+            BAIL_OUT ("rlist_intersect (%s, %s) failed", a, b);
+
+        s = rlist_dumps (result);
+        is (s, t->result,
+            "rlist_intersect: %s ∩ %s = %s",
+            a, b, s);
+
+        free (a);
+        free (b);
+        free (s);
+        rlist_destroy (result);
+        rlist_destroy (rla);
+        rlist_destroy (rlb);
+
+        t++;
+    }
+}
+
+void test_copy_ranks ()
+{
+    struct idset *ranks;
+    struct rlist *rl;
+    struct rlist *result;
+    char *s;
+    char *R = R_create ("0-5", "0-3", "0", "foo[0-5]", NULL);
+    if (!R)
+        BAIL_OUT ("R_create failed");
+    if (!(rl = rlist_from_R (R)))
+        BAIL_OUT ("rlist_from_R failed");
+    if (!(ranks = idset_decode ("1,3,5")))
+        BAIL_OUT ("idset_decode failed");
+
+    result = rlist_copy_ranks (rl, ranks);
+    if (!result)
+        BAIL_OUT ("rlist_copy_ranks failed");
+    ok (rlist_nnodes (result) == 3 && rlist_count (result, "core") == 12,
+        "rlist_copy_ranks worked");
+    s = rlist_dumps (result);
+    is (s, "rank[1,3,5]/core[0-3],gpu0",
+        "rlist_copy_ranks has expected result");
+    free (s);
+    rlist_destroy (result);
+    idset_destroy (ranks);
+
+    if (!(ranks = idset_decode ("5-9")))
+        BAIL_OUT ("idset_decode failed");
+    result = rlist_copy_ranks (rl, ranks);
+    if (!result)
+        BAIL_OUT ("rlist_copy_ranks failed");
+    ok (rlist_nnodes (result) == 1 && rlist_count (result, "core") == 4,
+        "rlist_copy_ranks worked");
+    s = rlist_dumps (result);
+    is (s, "rank5/core[0-3],gpu0",
+        "rlist_copy_ranks has expected result");
+    free (s);
+    rlist_destroy (result);
+    idset_destroy (ranks);
+
+    if (!(ranks = idset_decode ("9,20")))
+        BAIL_OUT ("idset_decode failed");
+    result = rlist_copy_ranks (rl, ranks);
+    if (!result)
+        BAIL_OUT ("rlist_copy_ranks failed");
+    ok (rlist_nnodes (result) == 0 && rlist_count (result, "core") == 0,
+        "rlist_copy_ranks worked");
+    s = rlist_dumps (result);
+    is (s, "",
+        "rlist_copy_ranks has expected result");
+    free (s);
+    rlist_destroy (result);
+    idset_destroy (ranks);
+    rlist_destroy (rl);
+    free (R);
+}
+
+void test_remove_ranks ()
+{
+    struct idset *ranks;
+    struct rlist *rl;
+    char *s;
+    char *R = R_create ("0-5", "0-3", "0", "foo[0-5]", NULL);
+    if (!R)
+        BAIL_OUT ("R_create failed");
+    if (!(rl = rlist_from_R (R)))
+        BAIL_OUT ("rlist_from_R failed");
+    if (!(ranks = idset_decode ("1,3,5")))
+        BAIL_OUT ("idset_decode failed");
+
+    ok (rlist_remove_ranks (rl, ranks) == idset_count (ranks),
+        "rlist_remove_ranks(1,3,5) works");
+
+    s = rlist_dumps (rl);
+    is (s, "rank[0,2,4]/core[0-3],gpu0",
+        "rlist_remove_ranks: %s", s);
+    free (s);
+    idset_destroy (ranks);
+    rlist_destroy (rl);
+
+    if (!(rl = rlist_from_R (R)))
+        BAIL_OUT ("rlist_from_R failed");
+    if (!(ranks = idset_decode ("5-9")))
+        BAIL_OUT ("idset_decode failed");
+    ok (rlist_remove_ranks (rl, ranks) == 1,
+        "rlist_remove_ranks (5-9)");
+    s = rlist_dumps (rl);
+    is (s, "rank[0-4]/core[0-3],gpu0",
+        "rlist_remove_ranks: %s", s);
+    free (s);
+    idset_destroy (ranks);
+    rlist_destroy (rl);
+
+    if (!(rl = rlist_from_R (R)))
+        BAIL_OUT ("rlist_from_R failed");
+    if (!(ranks = idset_decode ("9,20")))
+        BAIL_OUT ("idset_decode failed");
+    ok (rlist_remove_ranks (rl, ranks) == 0,
+        "rlist_remove_ranks (9,20) removed no ranks");
+    s = rlist_dumps (rl);
+    is (s, "rank[0-5]/core[0-3],gpu0",
+        "rlist_remove_ranks: %s", s);
+    free (s);
+
+    idset_destroy (ranks);
+    rlist_destroy (rl);
+    free (R);
+}
+
+
+struct verify_test {
+    const char *ranksa;
+    const char *coresa;
+    const char *gpusa;
+    const char *hostsa;
+
+    const char *ranksb;
+    const char *coresb;
+    const char *gpusb;
+    const char *hostsb;
+
+    int result;
+    const char *errmsg;
+};
+
+struct verify_test verify_tests[] = {
+    {
+        "0-5", "0-3", "0", "foo[0-5]",
+        "1",   "0-3", "0", "foo1",
+        0,
+        ""
+    },
+    {
+        "0-5", "0-3", "0", "foo[0-5]",
+        "1",   "0-3", "",  "foo1",
+        -1,
+        "rank 1 (foo1) missing resources: gpu0"
+    },
+    {
+        "0-5", "0-3", "0", "foo[0-5]",
+        "5",   "0-1", "0", "foo5",
+        -1,
+        "rank 5 (foo5) missing resources: core[2-3]"
+    },
+    {
+        "0-5", "0-3", "0", "foo[0-5]",
+        "5",   "0-3", "0", "foo7",
+        -1,
+        "rank 5 got hostname 'foo7', expected 'foo5'"
+    },
+    {
+        "0-5", "0-3", "0",   "foo[0-5]",
+        "0",   "0-7", "0-1", "foo0",
+        1,
+        "rank 0 (foo0) has extra resources: core[4-7],gpu1"
+    },
+    {
+        "0-5", "0-3", "0", "foo[0-5]",
+        "7",   "0-3", "0", "foo7",
+        -1,
+        "rank 7 not found in expected ranks"
+    },
+    {
+        "0-5", "0-3", "0", "foo[0-5]",
+        "0",   "0-3", "0",  NULL,
+         0,
+         "",
+
+    },
+    {
+        "0-5", "0-3", "0", NULL,
+        "0",   "0-3", "0", NULL,
+         0,
+         "",
+    },
+    {
+        "0-5", "0-3", "0", NULL,
+        "0",   "0-3", "0", "foo0",
+         0,
+         "",
+    },
+    { 0 },
+};
+
+void test_verify ()
+{
+    struct verify_test *t = verify_tests;
+
+    while (t && t->ranksa) {
+        int rc;
+        flux_error_t error;
+        struct rlist *rla = NULL;
+        struct rlist *rlb = NULL;
+        char *a;
+        char *b;
+        char *Ra = R_create (t->ranksa, t->coresa, t->gpusa, t->hostsa, NULL);
+        char *Rb = R_create (t->ranksb, t->coresb, t->gpusb, t->hostsb, NULL);
+
+        if (!Ra || !Rb)
+            BAIL_OUT ("R_create() failed!");
+
+        rla = rlist_from_R (Ra);
+        rlb = rlist_from_R (Rb);
+        if (!rla || !rlb)
+            BAIL_OUT ("rlist_from_R failed!");
+        free (Ra);
+        free (Rb);
+
+        a = rlist_dumps (rla);
+        b = rlist_dumps (rlb);
+
+        rc = rlist_verify (&error, rla, rlb);
+        ok (rc == t->result,
+            "rlist_verify: %s in %s = %d", b, a, rc);
+        is (error.text, t->errmsg,
+            "Got expected message: '%s'", error.text);
+
+        free (a);
+        free (b);
+        rlist_destroy (rla);
+        rlist_destroy (rlb);
+
+        t++;
+    }
+}
+
+void test_timelimits ()
+{
+    struct rlist *rl;
+    json_t *o;
+    char *R = R_create ("0-1", "0-3", NULL, "foo[0-1]", NULL);
+
+    if (!R)
+        BAIL_OUT ("R_create failed");
+    if (!(rl = rlist_from_R (R)))
+        BAIL_OUT ("rlist_from_R failed");
+
+    rl->starttime = 1234.;
+    rl->expiration = 2345.;
+
+    /*  Encode to R and ensure starttime/expiration are preserved */
+    if(!(o = rlist_to_R (rl)))
+        BAIL_OUT ("rlist_to_R failed");
+
+    rlist_destroy (rl);
+    free (R);
+
+    if (!(rl = rlist_from_json (o, NULL)))
+        BAIL_OUT ("rlist_from_json failed");
+
+    ok (rl->starttime == 1234. && rl->expiration == 2345.,
+        "starttime and expiration preserved during encode/decode");
+
+    json_decref (o);
+    rlist_destroy (rl);
+}
+
+struct hosts_to_ranks_test {
+    const char *input;
+    const char *ranks;
+    const char *hosts;
+    const char *result;
+    const char *error;
+};
+
+static struct hosts_to_ranks_test hosts_to_ranks_tests[] = {
+   { "foo[0-10]",
+     "0-10",
+     "foo[9-11]",
+     NULL,
+     "invalid hosts: foo11",
+    },
+    { "foo[0-10]",
+      "0-10",
+      "foo[a-b]",
+      NULL,
+      "Hostlist cannot be decoded",
+    },
+    { "foo[0-10]",
+      "0-10",
+      "foo[1,7]",
+      "1,7",
+      NULL,
+    },
+    { "foo10,foo[0-4],foo11,foo[5-9]",
+      "0-11",
+      "foo[1,9,4]",
+      "2,5,11",
+      NULL,
+    },
+    { "foo,foo,foo,foo",
+      "0-3",
+      "foo",
+      "0-3",
+      NULL,
+    },
+    { 0 },
+};
+
+void test_hosts_to_ranks (void)
+{
+    flux_error_t err;
+    struct hosts_to_ranks_test *t = hosts_to_ranks_tests;
+
+    ok (rlist_hosts_to_ranks (NULL, NULL, &err) == NULL,
+        "rlist_hosts_to_ranks returns NULL with NULL args");
+    is (err.text, "An expected argument was NULL",
+        "got expected error: %s", err.text);
+
+    while (t && t->input) {
+        char *R = NULL;
+        struct rlist *rl = NULL;
+        struct idset *ids = NULL;
+
+        if (!(R = R_create (t->ranks, "0-1", NULL, t->input, NULL)))
+            BAIL_OUT ("R_create");
+        if (!(rl = rlist_from_R (R)))
+            BAIL_OUT ("rlist_from_R");
+        ids = rlist_hosts_to_ranks (rl, t->hosts, &err);
+        if (t->result) {
+            char *s = idset_encode (ids, IDSET_FLAG_RANGE);
+            is (s, t->result,
+                "rlist_hosts_to_ranks (rl, %s) = %s",
+                t->hosts, s);
+            free (s);
+        }
+        else {
+            is (err.text, t->error,
+                "to_ranks (rl, %s) got expected error: %s",
+                t->hosts, err.text);
+        }
+        idset_destroy (ids);
+        rlist_destroy (rl);
+        free (R);
+        t++;
+    }
+}
+
+void test_issue4184 ()
+{
+    char *R;
+    struct rlist *rl = NULL;
+    struct rlist *alloc = NULL;
+
+    if (!(R = R_create_num (4, 4))
+        || !(rl = rlist_from_R (R)))
+        BAIL_OUT ("issue4184: failed to create rlist");
+
+    free (R);
+    if (!(R = R_create_num (4, 4))
+        || !(alloc = rlist_from_R (R)))
+        BAIL_OUT ("issue4184: failed to create alloc rlist");
+
+    ok (rlist_mark_down (rl, "all") == 0,
+        "rlist_mark_down");
+
+    ok (rl->avail == 0,
+        "rlist avail = %d (expected 0)", rl->avail);
+
+    ok (rlist_set_allocated (rl, alloc) == 0,
+        "rlist_set_allocated");
+
+    ok (rl->avail == 0,
+        "rlist avail = %d (expected 0)", rl->avail);
+
+    rlist_destroy (alloc);
+    rlist_destroy (rl);
+    free (R);
+}
+
+struct property_test {
+    const char *desc;
+    const char *ranks;
+    const char *cores;
+    const char *hosts;
+    const char *properties;
+
+    const char *decode_error;
+
+    const char *constraint;
+
+    const char *result;
+};
+
+struct property_test property_tests[] = {
+    {
+        "invalid properties",
+        "1-10", "0-1", "foo[1-10]",
+        "\"foo\"",
+        "properties must be an object",
+        NULL,
+        NULL,
+    },
+    {
+        "invalid properties",
+        "1-10", "0-1", "foo[1-10]",
+        "{ \"foo\": 1 }",
+        "properties value '1' not a string",
+        NULL,
+        NULL,
+    },
+    {
+        "invalid properties",
+        "1-10", "0-1", "foo[1-10]",
+        "{ \"foo\": \"1-30\" }",
+        "ranks 11-30 not found in target resource list",
+        NULL,
+        NULL,
+    },
+    {
+        "invalid properties",
+        "1-10", "0-1", "foo[1-10]",
+        "{ \"fo^o\": \"1-30\" }",
+        "invalid character '^' in property \"fo^o\"",
+        NULL,
+        NULL,
+    },
+    {
+        "invalid properties",
+        "1-10", "0-1", "foo[1-10]",
+        "{ \"foo\": \"x-y\" }",
+        "invalid idset 'x-y' specified for property \"foo\"",
+        NULL,
+        NULL,
+    },
+    {
+        "constraint: property=na",
+        "1-10", "0-1", "foo[1-10]",
+        "{\"foo\": \"1-3\", \"bar\": \"7\"}",
+        NULL,
+        "{\"properties\": [\"na\"]}",
+        "",
+    },
+    {
+        "constraint: property=foo",
+        "1-10", "0-1", "foo[1-10]",
+        "{\"foo\": \"1-3\", \"bar\": \"7\"}",
+        NULL,
+        "{\"properties\": [\"foo\"]}",
+        "rank[1-3]/core[0-1]",
+    },
+    {
+        "constraint: property=bar",
+        "1-10", "0-1", "foo[1-10]",
+        "{\"foo\": \"1-3\", \"bar\": \"7\"}",
+        NULL,
+        "{\"properties\": [\"bar\"]}",
+        "rank7/core[0-1]",
+    },
+    {
+        "constraint: property=^foo",
+        "1-10", "0-1", "foo[1-10]",
+        "{\"foo\": \"1-3\", \"bar\": \"7\"}",
+        NULL,
+        "{\"properties\": [\"^bar\"]}",
+        "rank[1-6,8-10]/core[0-1]",
+    },
+    {
+        "constraint: by hostname: foo5",
+        "1-10", "0-1", "foo[1-10]",
+        "{\"foo\": \"1-3\", \"bar\": \"7\"}",
+        NULL,
+        "{\"properties\": [\"foo5\"]}",
+        "rank5/core[0-1]",
+    },
+    {
+        "constraint: by hostname: ^foo5",
+        "1-10", "0-1", "foo[1-10]",
+        "{\"foo\": \"1-3\", \"bar\": \"7\"}",
+        NULL,
+        "{\"properties\": [\"^foo5\"]}",
+        "rank[1-4,6-10]/core[0-1]",
+    },
+    {
+        "constraint: by hostname: ^foo5",
+        "1-10", "0-1", "foo[1-10]",
+        "{\"foo\": \"1-3\", \"bar\": \"7\"}",
+        NULL,
+        "{\"properties\": [\"^foo5\"]}",
+        "rank[1-4,6-10]/core[0-1]",
+    },
+    {
+        "constraint: by hostlist: foo[2,3,7]",
+        "1-10", "0-1", "foo[1-10]",
+        "{\"foo\": \"1-3\", \"bar\": \"7\"}",
+        NULL,
+        "{\"hostlist\": [\"foo[2,3,7]\"]}",
+        "rank[2-3,7]/core[0-1]",
+    },
+    {
+        "constraint: by hostlist: not foo[2,3,7]",
+        "1-10", "0-1", "foo[1-10]",
+        "{\"foo\": \"1-3\", \"bar\": \"7\"}",
+        NULL,
+        "{ \"not\": [{\"hostlist\": [\"foo[2,3,7]\"]}] }",
+        "rank[1,4-6,8-10]/core[0-1]",
+    },
+    {
+        "constraint: by rank: not 0-4",
+        "1-10", "0-1", "foo[1-10]",
+        "{\"foo\": \"1-3\", \"bar\": \"7\"}",
+        NULL,
+        "{ \"not\": [{\"ranks\": [\"0-4\"]}] }",
+        "rank[5-10]/core[0-1]",
+    },
+    { 0 }
+};
+
+static void json_compare (const char *x, const char *y, const char *msg)
+{
+    json_t *ox = NULL;
+    json_t *oy = NULL;
+    json_error_t error;
+    if (!(ox = json_loads (x, JSON_DECODE_ANY, &error))
+        || !(oy = json_loads (y, JSON_DECODE_ANY, &error)))
+        BAIL_OUT ("json_loads '%s' or '%s' failed: %s", error.text);
+
+    ok (json_equal (ox, oy),
+        "%s: %s", msg, x);
+
+    json_decref (ox);
+    json_decref (oy);
+}
+
+/*  Note: this test only does some simple sanity checks.
+ *   More extensive testing will be contained in flux-R driven tests.
+ */
+void test_properties (void)
+{
+    struct rlist *rl;
+    struct rlist *cpy;
+    struct rlist *rlc;
+    char *R;
+    char *s;
+    json_t *Rj;
+    flux_error_t error;
+    json_error_t jerr;
+
+    ok (rlist_assign_properties (NULL, NULL, &error) < 0 && errno == EINVAL,
+        "rlist_assign_properties (NULL, NULL) fails");
+    is (error.text, "Invalid argument",
+        "fails with \"Invalid argument\"");
+
+    ok (rlist_properties_encode (NULL) == NULL && errno == EINVAL,
+        "rlist_properties_encode (NULL) returns EINVAL");
+
+    rl = rlist_create ();
+    s = rlist_properties_encode (rl);
+    is (s, "{}",
+        "rlist_properties_encode on empty rlist returns empty object");
+    free (s);
+
+    if (rlist_append_rank_cores (rl, "foo0", 0, "0-3") < 0)
+        BAIL_OUT ("rlist_append_rank_cores failed: %s", strerror (errno));
+
+    s = rlist_properties_encode (rl);
+    is (s, "{}",
+        "rlist_properties_encode with no properties returns empty object");
+    free (s);
+    rlist_destroy (rl);
+
+    struct property_test *t = property_tests;
+    while (t->desc) {
+        if (!(R = R_create (t->ranks,
+                            t->cores,
+                            NULL,
+                            t->hosts,
+                            t->properties)))
+            BAIL_OUT ("%s: R_create failed!", t->desc);
+
+        if (!(Rj = json_loads (R, 0, NULL)))
+            BAIL_OUT ("%s: json_loads (R) failed", t->desc);
+
+        if (!(rl  = rlist_from_json (Rj, &jerr))) {
+            if (t->decode_error) {
+                is (jerr.text, t->decode_error,
+                    "%s: %s",
+                    t->desc,
+                    jerr.text);
+                free (R);
+                json_decref (Rj);
+                ++t;
+                continue;
+            }
+            BAIL_OUT ("%s: rlist_from_R() failed!", t->desc);
+        }
+
+        /*  Return R from rl and ensure it can be decoded again.
+         */
+        free (R);
+        if (!(R = rlist_encode (rl)))
+            BAIL_OUT ("%s: rlist_encode() failed!", t->desc);
+        if (!(cpy = rlist_from_R (R)))
+            BAIL_OUT ("%s: rlist_from_R() after rlist_encode() failed!",
+                      t->desc);
+
+        /*  Use cpy in place of original rlist to ensure that encode/decode
+         *   preserves expected properties.
+         */
+        rlist_destroy (rl);
+        rl = cpy;
+
+        /*  Check that rlist_properties_encode() works
+         */
+        char *p = rlist_properties_encode (rl);
+        json_compare (p, t->properties,
+                      "rlist_properties_encode");
+        free (p);
+
+        rlc = rlist_copy_constraint_string (rl, t->constraint, &error);
+        ok (rlc != NULL,
+            "rlist_copy_constraint works: %s",
+            rlc ? "ok" : error.text);
+        s = rlist_dumps (rlc);
+        is (s, t->result, "%s: %s", t->desc, s);
+
+        free (R);
+        free (s);
+        json_decref (Rj);
+        rlist_destroy (rl);
+        rlist_destroy (rlc);
+        t++;
+    }
+
+}
+
+static void test_issue4290 (void)
+{
+    char *R;
+    char *s;
+    struct rlist *result;
+    struct rlist *rl;
+    flux_error_t error;
+    struct rlist_alloc_info ai = {
+        .nnodes = 4,
+        .slot_size = 1,
+        .nslots = 4,
+        .exclusive = true,
+    };
+
+    if (!(R = R_create ("0-3",
+                        "0-3",
+                        NULL,
+                        "foo[0-3]",
+                        NULL)))
+        BAIL_OUT ("issue4290: R_create");
+
+    if (!(rl = rlist_from_R (R)))
+        BAIL_OUT ("issue4290: rlist_from_R() failed");
+    if (rlist_mark_down (rl, "2") < 0)
+        BAIL_OUT ("issue4290: error marking rank 2 down");
+    result = rlist_alloc (rl, &ai, &error);
+    ok (!result && errno == ENOSPC,
+        "issue4290: alloc 4/4 nodes with node down fails with ENOSPC");
+    ok (rlist_mark_up (rl, "2") == 0,
+        "issue4290: marking rank 2 up");
+    ok ((result = rlist_alloc (rl, &ai, &error)) != NULL,
+        "issue4290: now allocation succeeds");
+    s = rlist_dumps (result);
+    diag ("%s", s);
+
+    free (s);
+    rlist_destroy (result);
+    rlist_destroy (rl);
+    free (R);
+}
+
+static void test_rlist_config_inval (void)
+{
+    flux_error_t error;
+    json_t *o;
+
+    ok (rlist_from_config (NULL, &error) == NULL,
+        "rlist_from_config (NULL) fails");
+    is (error.text, "resource config must be an array",
+        "error.text is expected: %s",
+        error.text);
+
+    if (!(o = json_object()))
+        BAIL_OUT ("test_rlist_config_inval: json_object: %s", strerror (errno));
+    ok (rlist_from_config (o, &error) == NULL,
+        "rlist_from_config() with empty object fails");
+    is (error.text, "resource config must be an array",
+        "error.text is expected: %s",
+        error.text);
+    json_decref (o);
+
+    if (!(o = json_array ()))
+        BAIL_OUT ("test_rlist_config_inval: json_array: %s", strerror (errno));
+    ok (rlist_from_config (o, &error) == NULL,
+        "rlist_from_config() with empty array fails");
+    is (error.text, "no hosts configured",
+        "error.text is expected: %s",
+        error.text);
+    json_decref (o);
+}
+
+static void test_issue_5868 (void)
+{
+    char *R;
+    char *s;
+    struct rlist *rl;
+    struct idset *ranks;
+
+    if (!(R = R_create ("0-3",
+                        "0-3",
+                        NULL,
+                        "foo[0-3]",
+                        NULL)))
+        BAIL_OUT ("issue5868: R_create");
+
+    if (!(rl = rlist_from_R (R)))
+        BAIL_OUT ("issue5868: rlist_from_R() failed");
+    /*  Remove ranks 0-1
+     */
+    if (!(ranks = idset_decode ("0-1")))
+        BAIL_OUT ("issue5868: idset_create failed");
+    if (rlist_remove_ranks (rl, ranks) < 0)
+        BAIL_OUT ("issue5868: rlist_remove_ranks failed");
+    idset_destroy (ranks);
+
+    ok (rlist_mark_down (rl, "0-2") == 0,
+        "issue5868: rlist_mark_down (0-2) ignores missing ranks");
+
+    s = rlist_dumps (rl);
+    diag ("%s", s);
+    is (s, "rank3/core[0-3]",
+        "issue5868: expected resources remain up");
+
+    free (s);
+    rlist_destroy (rl);
+    free (R);
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    test_simple ();
+    test_dumps ();
+    run_test_entries (test_2n_4c,       2, 4);
+    run_test_entries (test_6n_4c,       6, 4);
+    run_test_entries (test_1024n_4c, 1024, 4);
+    run_test_entries (test_exclusive,   4, 4);
+    test_issue2202 ();
+    test_issue2473 ();
+    test_updown ();
+    test_append ();
+    test_add ();
+    test_diff ();
+    test_union ();
+    test_intersect ();
+    test_copy_ranks ();
+    test_remove_ranks ();
+    test_verify ();
+    test_timelimits ();
+    test_remap ();
+    test_assign_hosts ();
+    test_rerank ();
+    test_hosts_to_ranks ();
+    test_issue4184 ();
+    test_properties ();
+    test_issue4290 ();
+    test_rlist_config_inval ();
+    test_issue_5868 ();
+    done_testing ();
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librlist/test/rnode.c b/src/common/librlist/test/rnode.c
new file mode 100644
index 000000000000..d16f32df1bad
--- /dev/null
+++ b/src/common/librlist/test/rnode.c
@@ -0,0 +1,361 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+
+#include "src/common/libtap/tap.h"
+#include "rnode.h"
+
+static void rnode_alloc_and_check (struct rnode *n, int count, char *expected)
+{
+    struct idset *ids = NULL;
+    char *result = NULL;
+    int avail = rnode_avail (n);
+    ok (rnode_alloc (n, count, &ids) == 0,
+        "rnode_alloc: count=%d", count);
+    ok (ids != NULL,
+        "rnode_alloc: returns non-null idset");
+    ok (idset_count (ids) == count,
+        "rnode_alloc: returned idset with expected count (%d)",
+        idset_count (ids));
+    if (!(result = idset_encode (ids, IDSET_FLAG_RANGE)))
+        BAIL_OUT ("failed to encode idset result");
+    is (result, expected,
+        "rnode_alloc: count=%d: returned expected result %s", count, result);
+    ok (rnode_avail (n) == avail - count,
+        "rnode_alloc: rnode_avail now %d, expected %d",
+        rnode_avail (n), avail - count);
+    idset_destroy (ids);
+    free (result);
+}
+
+static void rnode_avail_check (struct rnode *n, const char *expected)
+{
+    char *avail = idset_encode (n->cores->avail, IDSET_FLAG_RANGE);
+    if (avail == NULL)
+        BAIL_OUT ("failed to encode n->cores->avail");
+    is (avail, expected,
+        "rnode->avail is expected: %s", avail);
+    free (avail);
+}
+
+static void test_diff ()
+{
+    struct rnode *a = rnode_create ("foo", 0, "0-3");
+    struct rnode *b = rnode_create ("foo", 0, "0-3");
+    struct rnode *c = rnode_create ("foo", 0, "0-1");
+    struct rnode *result = NULL;
+
+    if (!a || !b || !c)
+        BAIL_OUT ("rnode_create failed!");
+
+    result = rnode_diff (a, b);
+    ok (result != NULL,
+        "rnode_diff (a, b) worked");
+    ok (rnode_empty (result),
+        "result is empty");
+    rnode_destroy (result);
+
+    result = rnode_diff (a, c);
+    ok (result != NULL,
+        "rnode_diff (a, c) works");
+    ok (!rnode_empty (result),
+        "rnode is not empty");
+    ok (rnode_avail_total (result) == 2,
+        "result has two available resources");
+    rnode_avail_check (result, "2-3");
+    rnode_destroy (result);
+
+    rnode_destroy (a);
+    rnode_destroy (b);
+    rnode_destroy (c);
+}
+
+static void test_intersect ()
+{
+    struct rnode *a = rnode_create ("foo", 0, "0-1");
+    struct rnode *b = rnode_create ("foo", 0, "1-3");
+    struct rnode *c = rnode_create ("foo", 0, "2-3");
+    struct rnode *result = NULL;
+
+    if (!a || !b || !c)
+        BAIL_OUT ("rnode_create failed");
+
+    result = rnode_intersect (a, b);
+    ok (result != NULL,
+        "rnode_intersect (a, b) worked");
+    if (!result)
+        BAIL_OUT ("rnode_intersect failed: %s", strerror (errno));
+    ok (!rnode_empty (result),
+        "result is not empty");
+    ok (rnode_count_type (result, "core") == 1,
+        "result has 1 core");
+    rnode_destroy (result);
+
+    result = rnode_intersect (a, c);
+    ok (result != NULL,
+        "rnode_intersect (a, c) worked");
+    if (!result)
+        BAIL_OUT ("rnode_intersect failed: %s", strerror (errno));
+    ok (rnode_empty (result),
+        "result is empty");
+    rnode_destroy (result);
+
+    rnode_destroy (a);
+    rnode_destroy (b);
+    rnode_destroy (c);
+}
+
+static void test_add_child ()
+{
+    struct rnode_child *c;
+    struct rnode *a = rnode_create ("foo", 0, "0-3");
+
+    if (!a)
+        BAIL_OUT ("rnode_create failed");
+
+    ok (rnode_count (a) == 4,
+        "rnode_create worked");
+    ok (rnode_count_type (a, "gpu") == 0,
+        "rnode_count_type (gpu) == 0");
+
+    if (!(c = rnode_add_child (a, "gpu", "0")))
+        BAIL_OUT ("rnode_add_child failed");
+
+    is (c->name, "gpu",
+        "rnode_add_child (gpu) works");
+    ok (idset_count (c->ids) == 1 && idset_count (c->avail) == 1,
+        "child has correct idsets");
+    ok (rnode_count_type (a, "gpu") == 1,
+        "rnode_count_type (gpu) == 1");
+
+    if (!(c = rnode_add_child (a, "core", "4-7")))
+        BAIL_OUT ("rnode_add_child failed");
+
+    is (c->name, "core",
+        "rnode_add_child (core) works");
+    ok (rnode_count (a) == 8,
+        "core count is now 8");
+    ok (rnode_avail_total (a) == 9,
+        "total available resources is 9");
+
+    ok (rnode_add_child (a, "gpu", "0-1") == NULL && errno == EEXIST,
+        "rnode_add_child fails with EEXIST if ids already exist in set");
+
+    rnode_destroy (a);
+}
+
+void test_copy ()
+{
+    struct rnode *n = NULL;
+    struct rnode *b = NULL;
+
+    if (!(n = rnode_create ("foo", 0, "0-3")))
+        BAIL_OUT ("failed to create an rnode object");
+    ok (rnode_add_child (n, "gpu", "0-1") != NULL,
+        "add two gpus to rnode");
+
+    ok ((b = rnode_copy (n)) != NULL,
+        "copy rnode");
+    ok (rnode_count_type (b, "core") == 4,
+        "rnode_count_type (gpu) == 4");
+    ok (rnode_count_type (b, "gpu") == 2,
+        "rnode_count_type (gpu) == 2");
+
+    rnode_destroy (b);
+    ok ((b = rnode_copy_avail (n)) != NULL,
+        "rnode_copy_avail");
+    ok (rnode_count_type (b, "core") == 4,
+        "rnode_count_type (gpu) == 4");
+    ok (rnode_count_type (b, "gpu") == 2,
+        "rnode_count_type (gpu) == 2");
+
+    rnode_destroy (b);
+    ok ((b = rnode_copy_cores (n)) != NULL,
+        "copy rnode (cores only)");
+    ok (rnode_count_type (b, "core") == 4,
+        "rnode_count_type (gpu) == 4");
+    ok (rnode_count_type (b, "gpu") == 0,
+        "rnode_count_type (gpu) == 0");
+
+    rnode_destroy (b);
+    rnode_destroy (n);
+}
+
+void test_rnode_cmp ()
+{
+    struct rnode *a = NULL;
+    struct rnode *b = NULL;
+
+    if (!(a = rnode_create ("foo", 0, "0-3"))
+        || !(b = rnode_create ("foo", 1, "0-3")))
+        BAIL_OUT ("failed to create rnode objects");
+
+    ok (rnode_cmp (a, b) == 0,
+        "rnode_cmp returns zero for nodes with identical children");
+
+    /*  Add gpus to rnode b only */
+    ok (rnode_add_child (b, "gpu", "0-1") != NULL,
+        "add two gpus to rnode");
+
+    ok (rnode_cmp (a, b) != 0,
+        "rnode_cmp returns nonzero for nodes with differing children");
+
+    rnode_destroy (a);
+    rnode_destroy (b);
+}
+
+void test_properties ()
+{
+    struct rnode *a;
+    struct rnode *b;
+
+    if (!(a = rnode_create ("foo", 0, "0-3")))
+        BAIL_OUT ("failed to create rnode object");
+
+    ok (rnode_set_property (a, "blingy") == 0,
+        "rnode_set_property works");
+    ok (rnode_set_property (a, "blingy") == 0,
+        "rnode_set_property again works");
+    ok (rnode_has_property (a, "blingy"),
+        "rnode_has_property works");
+    ok (!rnode_has_property (a, "dull"),
+        "rnode_has_property returns false if property not set");
+    if (!(b = rnode_copy (a)))
+        BAIL_OUT ("failed to copy rnode");
+    ok (b != NULL,
+        "rnode_copy with properties");
+    ok (rnode_has_property (b, "blingy"),
+        "rnode_has_property works on copy");
+    ok (!rnode_has_property (b, "dull"),
+        "rnode_has_property on copy returns false if property not set");
+    rnode_remove_property (a, "blingy");
+    ok (!rnode_has_property (a, "blingy"),
+        "rnode_has_property now returns false");
+
+    rnode_destroy (a);
+    rnode_destroy (b);
+}
+
+int main (int ac, char *av[])
+{
+    struct idset *ids = NULL;
+    struct rnode *n = NULL;
+
+    plan (NO_PLAN);
+
+    if (!(n = rnode_create ("foo", 0, "0-3")))
+        BAIL_OUT ("could not create an rnode object");
+    is (n->hostname, "foo",
+        "rnode is has hostname set");
+    ok (n->up,
+        "rnode is created in up state by default");
+    n->up = false;
+    ok (rnode_avail (n) == 0,
+        "rnode_avail == 0 for down rnode");
+
+    ok (rnode_alloc (n, 1, &ids) < 0 && errno == EHOSTDOWN,
+        "rnode_alloc on down host returns EHOSTDOWN");
+
+    n->up = true;
+    ok (rnode_avail (n) == 4,
+        "rnode_avail == 4");
+
+    ok (rnode_alloc (n, 5, &ids) < 0 && errno == ENOSPC,
+        "rnode_alloc too many cores returns errno ENOSPC");
+
+    rnode_alloc_and_check (n, 1, "0");
+    ok (rnode_avail (n) == 3,
+        "rnode_avail == 3");
+    rnode_avail_check (n, "1-3");
+
+    rnode_alloc_and_check (n, 1, "1");
+    ok (rnode_avail (n) == 2,
+        "rnode_avail == 2");
+    rnode_avail_check (n, "2-3");
+
+    rnode_alloc_and_check (n, 2, "2-3");
+    ok (rnode_avail (n) == 0,
+        "rnode_avail == 0");
+    rnode_avail_check (n, "");
+
+    ok (rnode_alloc (n, 1, &ids) < 0 && errno == ENOSPC && ids == NULL,
+        "rnode_alloc on empty rnode fails with ENOSPC");
+
+    ok (rnode_free (n, "3-4") < 0 && errno == ENOENT,
+        "rnode_free with invalid ids fails");
+    ok (rnode_avail (n) == 0,
+        "rnode_avail still is 0");
+    rnode_avail_check (n, "");
+
+    ok (rnode_free (n, "0-1") == 0,
+        "rnode_free (0-1) works");
+    ok (rnode_avail (n) == 2,
+        "rnode_avail now is 2");
+    rnode_avail_check (n, "0-1");
+    ok (rnode_free (n, "0") < 0 && errno == EEXIST,
+        "rnode_free of already available id fails");
+    ok (rnode_avail (n) == 2,
+        "rnode_avail is still 2");
+    ok (rnode_free (n, "3") == 0,
+        "rnode_free '3' works");
+    rnode_avail_check (n, "0-1,3");
+
+    rnode_alloc_and_check (n, 3, "0-1,3");
+
+    rnode_destroy (n);
+
+    struct idset *idset = idset_decode ("0-3");
+    n = rnode_create_idset ("foo", 3, idset);
+    idset_destroy (idset);
+    if (n == NULL)
+        BAIL_OUT ("rnode_create_idset failed");
+    is (n->hostname, "foo",
+        "rnode hostname set correctly");
+    ok (n->rank == 3, "rnode rank set correctly");
+    rnode_avail_check (n, "0-3");
+
+    struct idset *alloc = idset_decode ("1,3");
+    ok (rnode_alloc_idset (n, alloc) == 0,
+        "rnode_alloc_idset (1,3)");
+    rnode_avail_check (n, "0,2");
+    ok (rnode_alloc_idset (n, alloc) < 0 && errno == EEXIST,
+        "rnode_alloc_idset with idset already allocated returns EEXIST");
+
+    ok (rnode_free_idset (n, alloc) == 0,
+        "rnode_free_idset (1,3)");
+    rnode_avail_check (n, "0-3");
+
+    ok (rnode_free_idset (n, alloc) < 0 && errno == EEXIST,
+        "rnode_free_idset with idset already available returns EEXIST");
+
+    idset_destroy (alloc);
+    alloc = idset_decode ("4-7");
+    ok (rnode_alloc_idset (n, alloc) < 0 && errno == ENOENT,
+        "rnode_alloc_idset with invalid ids return ENOENT");
+    ok (rnode_free_idset (n, alloc) < 0 && errno == ENOENT,
+        "rnode_free_idset with invalid ids return ENOENT");
+
+    idset_destroy (alloc);
+    rnode_destroy (n);
+
+    test_diff ();
+    test_intersect ();
+    test_add_child ();
+    test_copy ();
+    test_properties ();
+    done_testing ();
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librouter/Makefile.am b/src/common/librouter/Makefile.am
index fff0b633e4b8..f5eb10ffef02 100644
--- a/src/common/librouter/Makefile.am
+++ b/src/common/librouter/Makefile.am
@@ -6,10 +6,11 @@ AM_LDFLAGS = \
         $(CODE_COVERAGE_LDFLAGS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) \
+	-I$(top_srcdir)/src/common/libccan \
 	$(LIBUUID_CFLAGS)
 
 noinst_LTLIBRARIES = \
@@ -29,7 +30,13 @@ librouter_la_SOURCES = \
 	servhash.h \
 	servhash.c \
 	router.h \
-	router.c
+	router.c \
+	usock_service.h \
+	usock_service.c \
+	msg_hash.h \
+	msg_hash.c \
+	rpc_track.h \
+	rpc_track.c
 
 TESTS = \
 	test_sendfd.t \
@@ -37,9 +44,14 @@ TESTS = \
 	test_auth.t \
 	test_usock.t \
 	test_usock_echo.t \
+	test_usock_epipe.t \
+	test_usock_emfile.t \
 	test_subhash.t \
 	test_router.t \
-	test_servhash.t
+	test_servhash.t \
+	test_usock_service.t \
+	test_msg_hash.t \
+	test_rpc_track.t
 
 check_PROGRAMS = \
         $(TESTS)
@@ -58,42 +70,78 @@ test_ldadd = \
 	$(builddir)/libtestutil.la \
         $(top_builddir)/src/common/librouter/librouter.la \
         $(top_builddir)/src/common/libtestutil/libtestutil.la \
-        $(top_builddir)/src/common/libflux-internal.la \
         $(top_builddir)/src/common/libflux-core.la \
+        $(top_builddir)/src/common/libflux-internal.la \
         $(top_builddir)/src/common/libtap/libtap.la
 
 test_cppflags = \
         $(AM_CPPFLAGS) \
         -I$(top_srcdir)/src/common/libtap
 
+test_ldflags = \
+	-no-install
+
 test_sendfd_t_SOURCES = test/sendfd.c
 test_sendfd_t_CPPFLAGS = $(test_cppflags)
 test_sendfd_t_LDADD = $(test_ldadd)
+test_sendfd_t_LDFLAGS = $(test_ldflags)
 
 test_disconnect_t_SOURCES = test/disconnect.c
 test_disconnect_t_CPPFLAGS = $(test_cppflags)
 test_disconnect_t_LDADD = $(test_ldadd)
+test_disconnect_t_LDFLAGS = $(test_ldflags)
 
 test_auth_t_SOURCES = test/auth.c
 test_auth_t_CPPFLAGS = $(test_cppflags)
 test_auth_t_LDADD = $(test_ldadd)
+test_auth_t_LDFLAGS = $(test_ldflags)
 
 test_usock_t_SOURCES = test/usock.c
 test_usock_t_CPPFLAGS = $(test_cppflags)
 test_usock_t_LDADD = $(test_ldadd)
+test_usock_t_LDFLAGS = $(test_ldflags)
 
 test_usock_echo_t_SOURCES = test/usock_echo.c
 test_usock_echo_t_CPPFLAGS = $(test_cppflags)
 test_usock_echo_t_LDADD = $(test_ldadd)
+test_usock_echo_t_LDFLAGS = $(test_ldflags)
+
+test_usock_epipe_t_SOURCES = test/usock_epipe.c
+test_usock_epipe_t_CPPFLAGS = $(test_cppflags)
+test_usock_epipe_t_LDADD = $(test_ldadd)
+test_usock_epipe_t_LDFLAGS = $(test_ldflags)
+
+test_usock_emfile_t_SOURCES = test/usock_emfile.c
+test_usock_emfile_t_CPPFLAGS = $(test_cppflags)
+test_usock_emfile_t_LDADD = $(test_ldadd)
+test_usock_emfile_t_LDFLAGS = $(test_ldflags)
 
 test_subhash_t_SOURCES = test/subhash.c
 test_subhash_t_CPPFLAGS = $(test_cppflags)
 test_subhash_t_LDADD = $(test_ldadd)
+test_subhash_t_LDFLAGS = $(test_ldflags)
 
 test_router_t_SOURCES = test/router.c
 test_router_t_CPPFLAGS = $(test_cppflags)
 test_router_t_LDADD = $(test_ldadd)
+test_router_t_LDFLAGS = $(test_ldflags)
 
 test_servhash_t_SOURCES = test/servhash.c
 test_servhash_t_CPPFLAGS = $(test_cppflags)
 test_servhash_t_LDADD = $(test_ldadd)
+test_servhash_t_LDFLAGS = $(test_ldflags)
+
+test_usock_service_t_SOURCES = test/usock_service.c
+test_usock_service_t_CPPFLAGS = $(test_cppflags)
+test_usock_service_t_LDADD = $(test_ldadd)
+test_usock_service_t_LDFLAGS = $(test_ldflags)
+
+test_msg_hash_t_SOURCES = test/msg_hash.c
+test_msg_hash_t_CPPFLAGS = $(test_cppflags)
+test_msg_hash_t_LDADD = $(test_ldadd)
+test_msg_hash_t_LDFLAGS = $(test_ldflags)
+
+test_rpc_track_t_SOURCES = test/rpc_track.c
+test_rpc_track_t_CPPFLAGS = $(test_cppflags)
+test_rpc_track_t_LDADD = $(test_ldadd)
+test_rpc_track_t_LDFLAGS = $(test_ldflags)
diff --git a/src/common/librouter/auth.c b/src/common/librouter/auth.c
index caaa054d22e2..34e920f9d77a 100644
--- a/src/common/librouter/auth.c
+++ b/src/common/librouter/auth.c
@@ -14,35 +14,6 @@
 #include <flux/core.h>
 #include "auth.h"
 
-flux_future_t *auth_lookup_rolemask (flux_t *h, uint32_t userid)
-{
-    flux_future_t *f;
-
-    if (!h) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if (!(f = flux_rpc_pack (h,
-                             "userdb.lookup",
-                             FLUX_NODEID_ANY,
-                             0,
-                             "{s:i}",
-                             "userid", userid)))
-        return NULL;
-    return f;
-}
-
-int auth_lookup_rolemask_get (flux_future_t *f, uint32_t *rolemask)
-{
-    if (!f || !rolemask) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (flux_rpc_get_unpack (f, "{s:i}", "rolemask", rolemask) < 0)
-        return -1;
-    return 0;
-}
-
 int auth_init_message (flux_msg_t *msg, const struct flux_msg_cred *conn)
 {
     if (!msg || !conn) {
@@ -61,7 +32,7 @@ int auth_init_message (flux_msg_t *msg, const struct flux_msg_cred *conn)
     /* Owner:
      * If message credentials have been set, we allow them to pass through.
      * Use case #1: owner message router components, where auth is "downstream"
-     * Use case #2: testing, to simulate guest accesss.
+     * Use case #2: testing, to simulate guest access.
      * If they have not been set, overwrite with connect creds, as above.
      */
     else {
diff --git a/src/common/librouter/auth.h b/src/common/librouter/auth.h
index cb2559ec61af..2d1aab11567f 100644
--- a/src/common/librouter/auth.h
+++ b/src/common/librouter/auth.h
@@ -13,11 +13,6 @@
 
 #include <flux/core.h>
 
-/* Look up user in the 'userdb' to determine assigned roles.
- */
-flux_future_t *auth_lookup_rolemask (flux_t *h, uint32_t userid);
-int auth_lookup_rolemask_get (flux_future_t *f, uint32_t *rolemask);
-
 /* Initialize received message creds based on the connected user's credentials.
  */
 int auth_init_message (flux_msg_t *msg, const struct flux_msg_cred *conn);
diff --git a/src/common/librouter/disconnect.c b/src/common/librouter/disconnect.c
index 8de09089a9ee..6aba1abe6b39 100644
--- a/src/common/librouter/disconnect.c
+++ b/src/common/librouter/disconnect.c
@@ -31,9 +31,9 @@
 #include "config.h"
 #endif
 #include <flux/core.h>
-#include <czmq.h>
 
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "disconnect.h"
 
@@ -106,7 +106,7 @@ int disconnect_hashkey (const flux_msg_t *msg, char *buf, int len)
 {
     const char *topic;
     uint32_t nodeid;
-    uint8_t flags;
+    int flags = 0;
     int used, n;
 
     if (!msg || !buf) {
@@ -114,18 +114,15 @@ int disconnect_hashkey (const flux_msg_t *msg, char *buf, int len)
         return -1;
     }
     if (flux_msg_get_topic (msg, &topic) < 0
-            || flux_msg_get_nodeid (msg, &nodeid) < 0
-            || flux_msg_get_flags (msg, &flags) < 0)
+        || flux_msg_get_nodeid (msg, &nodeid) < 0)
         return -1;
+    if (flux_msg_has_flag (msg, FLUX_MSGFLAG_UPSTREAM))
+        flags = FLUX_MSGFLAG_UPSTREAM;
     if ((used = disconnect_topic (topic, buf, len)) < 0)
         return -1;
     len -= used;
     buf += used;
-    n = snprintf (buf,
-                  len,
-                  ":%lu:%d",
-                  (unsigned long)nodeid,
-                  (int)flags & FLUX_MSGFLAG_UPSTREAM);
+    n = snprintf (buf, len, ":%lu:%d", (unsigned long)nodeid, flags);
     if (n >= len) {
         errno = EINVAL;
         return -1;
@@ -148,11 +145,14 @@ flux_msg_t *disconnect_msg (const flux_msg_t *msg)
         return NULL;
     if (!(cpy = flux_msg_copy (msg, false)))
         return NULL;
-    if (flux_msg_set_topic (cpy, distopic) < 0) {
-        flux_msg_destroy (cpy);
-        return NULL;
-    }
+    if (flux_msg_set_topic (cpy, distopic) < 0)
+        goto error;
+    if (flux_msg_set_noresponse (cpy) < 0)
+        goto error;
     return cpy;
+error:
+    flux_msg_destroy (cpy);
+    return NULL;
 }
 
 /* Insert a disconnect message cloned from 'msg' in the disconnect hash,
@@ -162,16 +162,15 @@ int disconnect_arm (struct disconnect *dcon, const flux_msg_t *msg)
 {
     char key[HASHKEY_BUFSIZE];
 
+    if (flux_msg_is_noresponse (msg))
+        return 0;
     if (disconnect_hashkey (msg, key, sizeof (key)) < 0)
         return -1;
     if (!zhashx_lookup (dcon->hash, key)) {
         flux_msg_t *dmsg;
         if (!(dmsg = disconnect_msg (msg)))
             return -1;
-        if (zhashx_insert (dcon->hash, key, dmsg) < 0) {
-            flux_msg_destroy (dmsg);
-            return -1;
-        }
+        (void)zhashx_insert (dcon->hash, key, dmsg);
     }
     return 0;
 }
diff --git a/src/common/librouter/msg_hash.c b/src/common/librouter/msg_hash.c
new file mode 100644
index 000000000000..cedea2b69c2c
--- /dev/null
+++ b/src/common/librouter/msg_hash.c
@@ -0,0 +1,110 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "msg_hash.h"
+
+/* get message matchtag, or if there is none, return FLUX_MATCHTAG_NONE */
+static uint32_t msg_hash_matchtag_get (const flux_msg_t *msg)
+{
+    uint32_t matchtag = FLUX_MATCHTAG_NONE;
+    (void)flux_msg_get_matchtag (msg, &matchtag);
+    return matchtag;
+}
+/* get request sender uuid, or if there is none, return empty string */
+static const char *msg_hash_uuid_get (const flux_msg_t *msg)
+{
+    const char *uuid = flux_msg_route_first (msg);
+    return uuid ? uuid : "";
+}
+
+/* Use "modified Bernstein hash" as employed by zhashx internally, but input
+ * is message uuid+matchtag instead of a simple NULL-terminated string.
+ * N.B. zhashx_hash_fn signature
+ */
+static size_t msg_hash_uuid_matchtag_hasher (const void *key)
+{
+    size_t key_hash = 0;
+    const char *cp;
+    uint32_t matchtag = msg_hash_matchtag_get (key);
+    int i;
+
+    cp = msg_hash_uuid_get (key);
+    while (*cp)
+        key_hash = 33 * key_hash ^ *cp++;
+    cp = (const char *)&matchtag;
+    for (i = 0; i < sizeof (matchtag); i++)
+        key_hash = 33 * key_hash ^ *cp++;
+    return key_hash;
+}
+
+#define NUMCMP(a,b) ((a)==(b)?0:((a)<(b)?-1:1))
+
+/* Compare hash keys, consisting of message uuid+matchtag.
+ * N.B. zhashx_comparator_fn signature
+ */
+static int msg_hash_uuid_matchtag_key_cmp (const void *key1, const void *key2)
+{
+    int ret;
+
+    ret = strcmp (msg_hash_uuid_get (key1),
+                  msg_hash_uuid_get (key2));
+    if (ret == 0) {
+        ret = NUMCMP (msg_hash_matchtag_get (key1),
+                      msg_hash_matchtag_get (key2));
+    }
+    return ret;
+}
+
+/* N.B. zhashx_destructor_fn signature
+ */
+static void msg_hash_destructor (void **item)
+{
+    if (item) {
+        flux_msg_decref (*item);
+        item = NULL;
+    }
+}
+
+zhashx_t *msg_hash_create (msg_hash_type_t type)
+{
+    zhashx_t *hash;
+
+    if (!(hash = zhashx_new ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    switch (type) {
+        case MSG_HASH_TYPE_UUID_MATCHTAG:
+            zhashx_set_key_hasher (hash, msg_hash_uuid_matchtag_hasher);
+            zhashx_set_key_comparator (hash, msg_hash_uuid_matchtag_key_cmp);
+            break;
+        default:
+            zhashx_destroy (&hash);
+            errno = EINVAL;
+            return NULL;
+    }
+    zhashx_set_key_duplicator (hash, NULL);
+    zhashx_set_key_destructor (hash, NULL);
+    zhashx_set_duplicator (hash, (zhashx_duplicator_fn *)flux_msg_incref);
+    zhashx_set_destructor (hash, msg_hash_destructor);
+
+    return hash;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/librouter/msg_hash.h b/src/common/librouter/msg_hash.h
new file mode 100644
index 000000000000..49ec03c0382a
--- /dev/null
+++ b/src/common/librouter/msg_hash.h
@@ -0,0 +1,41 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _ROUTER_MSG_HASH_H
+#define _ROUTER_MSG_HASH_H
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+typedef enum {
+    MSG_HASH_TYPE_UUID_MATCHTAG = 1,    // Hash request/response messages based
+                                        // on sender uuid+matchtag such that
+                                        // request and its response have the
+                                        // same hash key.
+
+} msg_hash_type_t;
+
+/* Create a zhashx_t for Flux messages.  The hash key is derived from
+ * info in the message, with key hasher and key comparator methods set up
+ * as appropriate for the hash type chosen at creation.
+ *
+ * The key duplicator and destructor are disabled, since the message contains
+ * all of the key information.
+ *
+ * The entry duplicator and destructor are set to flux_msg_incref() and
+ * flux_msg_decref() respectively.
+ */
+zhashx_t *msg_hash_create (msg_hash_type_t hashtype);
+
+#endif /* _ROUTER_MSG_HASH_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/librouter/router.c b/src/common/librouter/router.c
index f36c5dc73322..a95ae9f2a5cb 100644
--- a/src/common/librouter/router.c
+++ b/src/common/librouter/router.c
@@ -12,10 +12,11 @@
 #include "config.h"
 #endif
 #include <stdlib.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
 
 #include "router.h"
 #include "subhash.h"
@@ -92,7 +93,7 @@ static void router_entry_respond_byuuid (const flux_msg_t *msg,
         router_entry_respond (entry, msg, errnum);
 }
 
-/* Handle internal local.sub request.
+/* Handle internal local subscribe request.
  */
 static void local_sub_request (struct router_entry *entry, flux_msg_t *msg)
 {
@@ -108,7 +109,7 @@ static void local_sub_request (struct router_entry *entry, flux_msg_t *msg)
     router_entry_respond (entry, msg, errno);
 }
 
-/* Handle internal local.unsub request.
+/* Handle internal local unsubscribe request.
  */
 static void local_unsub_request (struct router_entry *entry, flux_msg_t *msg)
 {
@@ -172,25 +173,24 @@ void router_entry_recv (struct router_entry *entry, flux_msg_t *msg)
         return;
     switch (type) {
         case FLUX_MSGTYPE_REQUEST:
-            if (!strcmp (topic, "local.sub")) {
+            if (streq (topic, "event.subscribe")) {
                 local_sub_request (entry, msg);
                 break;
             }
-            if (!strcmp (topic, "local.unsub")) {
+            if (streq (topic, "event.unsubscribe")) {
                 local_unsub_request (entry, msg);
                 break;
             }
-            if (!strcmp (topic, "service.add")) {
+            if (streq (topic, "service.add")) {
                 service_add_request (entry, msg);
                 break;
             }
-            if (!strcmp (topic, "service.remove")) {
+            if (streq (topic, "service.remove")) {
                 service_remove_request (entry, msg);
                 break;
             }
-            if (flux_msg_enable_route (msg) < 0)
-                return;
-            if (flux_msg_push_route (msg, entry->uuid) < 0)
+            flux_msg_route_enable (msg);
+            if (flux_msg_route_push (msg, entry->uuid) < 0)
                 return;
             if (disconnect_arm (entry->dcon, msg) < 0)
                 return;
@@ -214,10 +214,6 @@ static int broker_subscribe (const char *topic, void *arg)
 
     if (flux_event_subscribe (rtr->h, topic) < 0)
         return -1;
-
-    /* N.B. t/t1008-proxy.t looks for this log message */
-    flux_log (rtr->h, LOG_DEBUG, "subscribe %s", topic);
-
     return 0;
 }
 
@@ -227,10 +223,6 @@ static int broker_unsubscribe (const char *topic, void *arg)
 
     if (!rtr->mute && flux_event_unsubscribe (rtr->h, topic) < 0)
         return -1;
-
-    /* N.B. t/t1008-proxy.t looks for this log message */
-    flux_log (rtr->h, LOG_DEBUG, "unsubscribe %s", topic);
-
     return 0;
 }
 
@@ -383,13 +375,11 @@ static void response_cb (flux_t *h,
     struct router *rtr = arg;
     struct router_entry *entry = NULL;
     flux_msg_t *cpy;
-    char *uuid = NULL;
+    const char *uuid = NULL;
 
     if (!(cpy = flux_msg_copy (msg, true)))
         goto error;
-    if (flux_msg_pop_route (cpy, &uuid) < 0) // may set uuid=NULL on success
-        goto error;
-    if (!uuid) {
+    if (!(uuid = flux_msg_route_last (cpy))) { // may set uuid=NULL no routes
         errno = EINVAL;
         goto error;
     }
@@ -397,16 +387,15 @@ static void response_cb (flux_t *h,
         errno = EHOSTUNREACH;
         goto error;
     }
+    if (flux_msg_route_delete_last (cpy) < 0)
+        goto error;
     if (entry->send (cpy, entry->arg) < 0) {
         flux_log_error (h, "router: response > client=%.5s", entry->uuid);
         goto error;
     }
-    free (uuid);
-    flux_msg_destroy (cpy);
-    return;
 error:
-    ERRNO_SAFE_WRAP (free, uuid);
     flux_msg_destroy (cpy);
+    return;
 }
 
 /* Receive event from broker.
@@ -493,6 +482,17 @@ void router_mute (struct router *rtr)
         rtr->mute = true;
 }
 
+int router_renew (struct router *rtr)
+{
+    if (rtr) {
+        if (subhash_renew (rtr->subscriptions) < 0)
+            return -1;
+        if (servhash_renew (rtr->services) < 0)
+            return -1;
+    }
+    return 0;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/librouter/router.h b/src/common/librouter/router.h
index 20d3b1b3449c..381de5ae9f91 100644
--- a/src/common/librouter/router.h
+++ b/src/common/librouter/router.h
@@ -36,6 +36,12 @@ struct router_entry *router_entry_add (struct router *rtr,
                                        void *arg);
 void router_entry_delete (struct router_entry *entry);
 
+/* Notify router that connection was lost to broker and restored
+ * so it can re-establish event subscriptions and service registrations.
+ * It does this synchronously.
+ */
+int router_renew (struct router *rtr);
+
 /* Create/destroy router.  'h' is the "upstream broker connection.
  */
 struct router *router_create (flux_t *h);
diff --git a/src/common/librouter/rpc_track.c b/src/common/librouter/rpc_track.c
new file mode 100644
index 000000000000..abaa57e1b273
--- /dev/null
+++ b/src/common/librouter/rpc_track.c
@@ -0,0 +1,151 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libccan/ccan/str/str.h"
+
+#include "rpc_track.h"
+#include "msg_hash.h"
+
+struct rpc_track {
+    zhashx_t *hash;
+    msg_hash_type_t type;
+};
+
+void rpc_track_destroy (struct rpc_track *rt)
+{
+    if (rt) {
+        int saved_errno = errno;
+        zhashx_destroy (&rt->hash);
+        free (rt);
+        errno = saved_errno;
+    }
+}
+
+struct rpc_track *rpc_track_create (msg_hash_type_t type)
+{
+    struct rpc_track *rt;
+
+    if (!(rt = calloc (1, sizeof (*rt))))
+        return NULL;
+    rt->type = type;
+    if (!(rt->hash = msg_hash_create (type)))
+        goto error;
+    return rt;
+error:
+    rpc_track_destroy (rt);
+    return NULL;
+}
+
+static bool response_is_error (const flux_msg_t *msg)
+{
+    int errnum;
+
+    if (flux_msg_get_errnum (msg, &errnum) < 0
+        || errnum == 0)
+        return false;
+    return true;
+}
+
+static bool request_is_disconnect (const flux_msg_t *msg)
+{
+    const char *topic;
+    const char *cp;
+
+    if (flux_msg_get_topic (msg, &topic) < 0
+        || !(cp = strstr (topic, ".disconnect"))
+        || strlen (cp) > 11)
+        return false;
+    return true;
+}
+
+/* Avoid putting messages in the hash that have ambiguous hash keys;
+ * specifically, avoid RFC 27 sched alloc RPCs, which are regular RPCs
+ * that don't use the matchtag field (setting it to FLUX_MATCHTAG_NONE),
+ * instead using payload elements to match requests and responses.
+ */
+static bool message_is_hashable (const flux_msg_t *msg)
+{
+    uint32_t matchtag;
+
+    if (flux_msg_get_matchtag (msg, &matchtag) < 0
+        || matchtag == FLUX_MATCHTAG_NONE)
+        return false;
+    return true;
+}
+
+static void rpc_track_disconnect (struct rpc_track *rt, const flux_msg_t *msg)
+{
+    const char *uuid;
+    zlistx_t *values;
+    const flux_msg_t *req;
+
+    if (!(uuid = flux_msg_route_first (msg))
+        || !(values = zhashx_values (rt->hash)))
+        return;
+    req = zlistx_first (values);
+    while (req) {
+        const char *uuid2 = flux_msg_route_first (req);
+        if (uuid2 && streq (uuid, uuid2))
+            zhashx_delete (rt->hash, req);
+        req = zlistx_next (values);
+    }
+    zlistx_destroy (&values);
+}
+
+void rpc_track_update (struct rpc_track *rt, const flux_msg_t *msg)
+{
+    int type;
+
+    if (!rt || !msg || flux_msg_get_type (msg, &type) < 0)
+        return;
+    switch (type) {
+        case FLUX_MSGTYPE_RESPONSE:
+            if (message_is_hashable (msg)
+                && (!flux_msg_is_streaming (msg) || response_is_error (msg)))
+                zhashx_delete (rt->hash, msg);
+            break;
+        case FLUX_MSGTYPE_REQUEST:
+            if (!flux_msg_is_noresponse (msg)
+                && message_is_hashable (msg))
+                zhashx_insert (rt->hash, msg, (flux_msg_t *)msg);
+            else if (request_is_disconnect (msg))
+                rpc_track_disconnect (rt, msg);
+            break;
+        default:
+            break;
+    }
+}
+
+void rpc_track_purge (struct rpc_track *rt, rpc_respond_f fun, void *arg)
+{
+    const flux_msg_t *msg;
+
+    if (rt) {
+        msg = zhashx_first (rt->hash);
+        while (msg) {
+            if (fun)
+                fun (msg, arg);
+            msg = zhashx_next (rt->hash);
+        }
+        zhashx_purge (rt->hash);
+    }
+}
+
+int rpc_track_count (struct rpc_track *rt)
+{
+    return rt ? zhashx_size (rt->hash) : 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/librouter/rpc_track.h b/src/common/librouter/rpc_track.h
new file mode 100644
index 000000000000..8541ba0866f0
--- /dev/null
+++ b/src/common/librouter/rpc_track.h
@@ -0,0 +1,44 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _ROUTER_RPC_TRACK_H
+#define _ROUTER_RPC_TRACK_H
+
+#include <flux/core.h>
+
+#include "msg_hash.h"
+
+typedef void (*rpc_respond_f)(const flux_msg_t *msg, void *arg);
+
+/* Create/destroy hash of messages.
+ * Set type=MSG_HASH_TYPE_UUID_MATCHTAG.
+ */
+struct rpc_track *rpc_track_create (msg_hash_type_t type);
+void rpc_track_destroy (struct rpc_track *rt);
+
+/* If msg is a request that requires a response, add it to the hash.
+ * If msg is a response that terminates a request in the hash (per RFC 6),
+ * remove the matching request from the hash.
+ * If msg is a disconnect request, remove all messages from the hash that
+ * were sent by the same uuid as the disconnect request.
+ */
+void rpc_track_update (struct rpc_track *rt, const flux_msg_t *msg);
+
+/* Call fun() for every hash entry, then purge all entries.
+ */
+void rpc_track_purge (struct rpc_track *rt, rpc_respond_f fun, void *arg);
+
+/* Return the number of RPCs currently being tracked.
+ */
+int rpc_track_count (struct rpc_track *rt);
+
+#endif /* _ROUTER_RPC_TRACK_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/librouter/sendfd.c b/src/common/librouter/sendfd.c
index ce91ec8aec2c..4ac27f22790f 100644
--- a/src/common/librouter/sendfd.c
+++ b/src/common/librouter/sendfd.c
@@ -80,7 +80,10 @@ int sendfd (int fd, const flux_msg_t *msg, struct iobuf *iobuf)
     if (!iobuf)
         iobuf_init (&local);
     if (!io->buf) {
-        io->size = flux_msg_encode_size (msg) + 8;
+        ssize_t s;
+        if ((s = flux_msg_encode_size (msg)) < 0)
+            goto done;
+        io->size = s + 8;
         if (io->size <= sizeof (io->buf_fixed))
             io->buf = io->buf_fixed;
         else if (!(io->buf = malloc (io->size)))
diff --git a/src/common/librouter/servhash.c b/src/common/librouter/servhash.c
index 67d96e8d40d7..42affd32832c 100644
--- a/src/common/librouter/servhash.c
+++ b/src/common/librouter/servhash.c
@@ -33,10 +33,11 @@
 #include "config.h"
 #endif
 #include <stdlib.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
 
 #include "servhash.h"
 
@@ -67,10 +68,7 @@ static flux_msg_t *request_copy_clear_routes (const flux_msg_t *msg)
     flux_msg_t *cpy;
     if (!(cpy = flux_msg_copy (msg, true)))
         return NULL;
-    if (flux_msg_clear_route (cpy) < 0 || flux_msg_enable_route (cpy) < 0) {
-        flux_msg_destroy (cpy);
-        return NULL;
-    }
+    flux_msg_route_clear (cpy);
     return cpy;
 }
 
@@ -240,7 +238,7 @@ int servhash_remove (struct servhash *sh,
         return -1;
     }
     if (!(entry = zhashx_lookup (sh->services, name))
-            || strcmp (entry->uuid, uuid) != 0
+            || !streq (entry->uuid, uuid)
             || entry->f_remove != NULL) {
         errno = ENOENT;
         return -1;
@@ -270,7 +268,7 @@ void servhash_disconnect (struct servhash *sh, const char *uuid)
         key = zlistx_first (keys);
         while (key) {
             entry = zhashx_lookup (sh->services, key);
-            if (!strcmp (entry->uuid, uuid))
+            if (streq (entry->uuid, uuid))
                 zhashx_delete (sh->services, key);
             key = zlistx_next (keys);
         }
@@ -338,6 +336,43 @@ struct servhash *servhash_create (flux_t *h)
     return NULL;
 }
 
+static int renew_service_registration (struct servhash *sh,
+                                       struct servhash_entry *entry)
+{
+    flux_future_t *f;
+    flux_msg_t *cpy;
+    int rc = -1;
+
+    if (!(cpy = request_copy_clear_routes (entry->add_request)))
+        return -1;
+    if (!(f = flux_rpc_message (sh->h, cpy, FLUX_NODEID_ANY, 0)))
+        goto done;
+    if (flux_future_get (f, NULL) < 0)
+        goto done;
+    rc = 0;
+done:
+    flux_future_destroy (f);
+    flux_msg_destroy (cpy);
+    return rc;
+}
+
+int servhash_renew (struct servhash *sh)
+{
+    struct servhash_entry *entry;
+
+    if (sh) {
+        entry = zhashx_first (sh->services);
+        while (entry) {
+            if (entry->live && !entry->f_remove) {
+                if (renew_service_registration (sh, entry) < 0)
+                    return -1;
+            }
+            entry = zhashx_next (sh->services);
+        }
+    }
+    return 0;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/librouter/servhash.h b/src/common/librouter/servhash.h
index a7bb30b4f007..0cdcb7b7441f 100644
--- a/src/common/librouter/servhash.h
+++ b/src/common/librouter/servhash.h
@@ -44,6 +44,8 @@ int servhash_match (struct servhash *sh,
 
 void servhash_disconnect (struct servhash *sh, const char *uuid);
 
+int servhash_renew (struct servhash *sh);
+
 #endif /* !_ROUTER_SERVHASH_H */
 
 /*
diff --git a/src/common/librouter/subhash.c b/src/common/librouter/subhash.c
index c8fd8742d886..6b0a1bcd48f5 100644
--- a/src/common/librouter/subhash.c
+++ b/src/common/librouter/subhash.c
@@ -18,7 +18,7 @@
  * its subscriptions.
  *
  * The router entry (client) has its sh->sub() / sh->unsub() callbacks wired
- * to the router's subhash_subcribe() / subhash_unsubscribe() functions,
+ * to the router's subhash_subscribe() / subhash_unsubscribe() functions,
  * while the router's sh->sub() / sh->unsub() callbacks are wired to the
  * real flux_event_subscribe() / flux_event_unsubscribe().
  *
@@ -36,10 +36,11 @@
 #include "config.h"
 #endif
 #include <stdlib.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
 
 #include "subhash.h"
 
@@ -94,18 +95,6 @@ static struct subhash_entry *subhash_entry_create (const char *topic)
     return NULL;
 }
 
-/* sub="" matches all
- * sub="foo" matches "foo", "foobar", "foo.bar"
- */
-static bool match_event (const char *sub_topic, const char *msg_topic)
-{
-    int len = strlen (sub_topic);
-
-    if (!strncmp (sub_topic, msg_topic, len))
-        return true;
-    return false;
-}
-
 bool subhash_topic_match (struct subhash *sh, const char *topic)
 {
     struct subhash_entry *entry;
@@ -113,7 +102,10 @@ bool subhash_topic_match (struct subhash *sh, const char *topic)
     if (sh && topic) {
         entry = zhashx_first (sh->subs);
         while (entry) {
-            if (match_event  (entry->topic, topic))
+            /* entry->topic="" matches all
+             * entry->topic="foo" matches "foo", "foobar", "foo.bar"
+             */
+            if (strstarts (topic, entry->topic))
                 return true;
             entry = zhashx_next (sh->subs);
         }
@@ -188,6 +180,20 @@ void subhash_set_unsubscribe (struct subhash *sh, subscribe_f cb, void *arg)
     }
 }
 
+int subhash_renew (struct subhash *sh)
+{
+    struct subhash_entry *entry;
+
+    if (sh) {
+        entry = zhashx_first (sh->subs);
+        while (entry) {
+            if (sh->sub (entry->topic, sh->sub_arg) < 0)
+                return -1;
+            entry = zhashx_next (sh->subs);
+        }
+    }
+    return 0;
+}
 
 void subhash_destroy (struct subhash *sh)
 {
diff --git a/src/common/librouter/subhash.h b/src/common/librouter/subhash.h
index 2e62697325fa..dcabbff53bff 100644
--- a/src/common/librouter/subhash.h
+++ b/src/common/librouter/subhash.h
@@ -25,7 +25,7 @@ bool subhash_topic_match (struct subhash *sh, const char *topic);
 int subhash_subscribe (struct subhash *sh, const char *topic);
 int subhash_unsubscribe (struct subhash *sh, const char *topic);
 
-
+int subhash_renew (struct subhash *sh);
 
 #endif /* !_ROUTER_SUBHASH_H */
 
diff --git a/src/common/librouter/test/auth.c b/src/common/librouter/test/auth.c
index 6d192bba5244..fafd604f567a 100644
--- a/src/common/librouter/test/auth.c
+++ b/src/common/librouter/test/auth.c
@@ -16,35 +16,12 @@
 #include "src/common/libtap/tap.h"
 #include "src/common/librouter/auth.h"
 
-void lookup (void)
-{
-    uint32_t rolemask;
-    flux_future_t *f;
-
-    errno = 0;
-    ok (auth_lookup_rolemask (NULL, 0) == NULL && errno == EINVAL,
-        "auth_lookup_rolemask h=NULL fails with EINVAL");
-
-    errno = 0;
-    ok (auth_lookup_rolemask_get (NULL, &rolemask) < 0 && errno == EINVAL,
-        "auth_lookup_rolemask_get f=NULL fails with EINVAL");
-
-    if (!(f = flux_future_create (NULL, NULL)))
-        BAIL_OUT ("flux_future_create failed");
-
-    errno = 0;
-    ok (auth_lookup_rolemask_get (NULL, &rolemask) < 0 && errno == EINVAL,
-        "auth_lookup_rolemask_get f=NULL fails with EINVAL");
-
-    flux_future_destroy (f);
-}
-
 int checkcred (const flux_msg_t *msg, struct flux_msg_cred *cred)
 {
     struct flux_msg_cred msgcred;
 
     if (flux_msg_get_cred (msg, &msgcred) < 0)
-        BAIL_OUT ("flux_msg_get_cred faild");
+        BAIL_OUT ("flux_msg_get_cred failed");
     if (msgcred.rolemask != cred->rolemask)
         return -1;
     if (msgcred.userid != cred->userid)
@@ -166,7 +143,6 @@ int main (int argc, char *argv[])
 {
     plan (NO_PLAN);
 
-    lookup ();
     init_message ();
     event_privacy ();
 
diff --git a/src/common/librouter/test/disconnect.c b/src/common/librouter/test/disconnect.c
index 0677f870cb03..12485c19f076 100644
--- a/src/common/librouter/test/disconnect.c
+++ b/src/common/librouter/test/disconnect.c
@@ -14,6 +14,7 @@
 #include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 #include "src/common/librouter/disconnect.h"
 
 /* Used for topic() and hashkeys() subtests */
@@ -21,20 +22,16 @@ struct stab {
     const char *topic;
     const char *out;
     uint32_t nodeid;
-    uint8_t flags;
+    int flags;
 };
 
 flux_msg_t *gen_request (const char *topic, uint32_t nodeid, uint8_t flags)
 {
-    uint8_t tmp;
     flux_msg_t *msg;
 
     if (!(msg = flux_request_encode (topic, NULL)))
         return NULL;
-    if (flux_msg_get_flags (msg, &tmp) < 0)
-        goto error;
-    tmp |= flags;
-    if (flux_msg_set_flags (msg, tmp) < 0)
+    if (flux_msg_set_flag (msg, flags) < 0)
         goto error;
     if (flux_msg_set_nodeid (msg, nodeid) < 0)
         goto error;
@@ -58,7 +55,7 @@ void topic (void)
 
     for (i = 0; topics[i].topic != NULL; i++) {
         ok (disconnect_topic (topics[i].topic, buf, sizeof (buf)) >= 0
-            && !strcmp (buf, topics[i].out),
+            && streq (buf, topics[i].out),
             "topic: %s => %s", topics[i].topic, topics[i].out);
     }
 
@@ -107,7 +104,7 @@ void hashkey (void)
             BAIL_OUT ("gen_request failed");
 
         ok (disconnect_hashkey (msg, buf, sizeof (buf)) >= 0
-            && !strcmp (buf, hashkeys[i].out),
+            && streq (buf, hashkeys[i].out),
             "hashkey: %s,%u,%u => %s",
             hashkeys[i].topic,
             (unsigned int)hashkeys[i].nodeid,
diff --git a/src/common/librouter/test/msg_hash.c b/src/common/librouter/test/msg_hash.c
new file mode 100644
index 000000000000..1423103eee0f
--- /dev/null
+++ b/src/common/librouter/test/msg_hash.c
@@ -0,0 +1,101 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <uuid.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/librouter/msg_hash.h"
+#include "src/common/libtap/tap.h"
+
+#ifndef UUID_STR_LEN
+#define UUID_STR_LEN 37     // defined in later libuuid headers
+#endif
+
+flux_msg_t *create_request (void)
+{
+    flux_msg_t *msg;
+    uuid_t uuid;
+    char uuid_str[UUID_STR_LEN];
+
+    if (!(msg = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("flux_request_create failed");
+    flux_msg_route_enable (msg);
+    uuid_generate (uuid);
+    uuid_unparse (uuid, uuid_str);
+    if (flux_msg_route_push (msg, uuid_str) < 0)
+        BAIL_OUT ("flux_msg_route_push failed");
+    return msg;
+}
+
+void test_basic (void)
+{
+    zhashx_t *zh;
+    flux_msg_t *req1;
+    flux_msg_t *req2;
+    flux_msg_t *rep1;
+    flux_msg_t *rep2;
+
+    errno = 0;
+    ok (msg_hash_create (42) == NULL && errno == EINVAL,
+        "msg_hash_create type=42 fails with EINVAL");
+
+    if (!(zh = msg_hash_create (MSG_HASH_TYPE_UUID_MATCHTAG)))
+        BAIL_OUT ("msg_hash_create failed");
+
+    req1 = create_request ();
+    req2 = create_request ();
+    rep1 = flux_response_derive (req1, 0);
+    rep2 = flux_response_derive (req2, 0);
+    if (!rep1 || !rep2)
+        BAIL_OUT ("flux_response_derive failed");
+
+    ok (zhashx_insert (zh, req1, req1) == 0,
+        "inserted first request");
+    ok (zhashx_insert (zh, req2, req2) == 0,
+        "inserted second request");
+    ok (zhashx_size (zh) == 2,
+        "hash size=2");
+
+    zhashx_delete (zh, req1);
+    ok (zhashx_size (zh) == 1,
+        "delete first request (from response), now hash size=1");
+
+    ok (zhashx_lookup (zh, rep1) == NULL,
+        "lookup of response 1 fails");
+    ok (zhashx_lookup (zh, rep2) != NULL,
+        "lookup of response 2 works");
+
+    flux_msg_decref (req1);
+    flux_msg_decref (req2);
+    flux_msg_decref (rep1);
+    flux_msg_decref (rep2);
+
+    zhashx_destroy (&zh);
+
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_basic ();
+
+    done_testing();
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/librouter/test/router.c b/src/common/librouter/test/router.c
index 04423a73d067..a4efc260b8c9 100644
--- a/src/common/librouter/test/router.c
+++ b/src/common/librouter/test/router.c
@@ -135,7 +135,7 @@ int basic_recv (const flux_msg_t *msg, void *arg)
 
     switch (type) {
         case FLUX_MSGTYPE_RESPONSE:
-            like (topic, "local.sub|local.unsub|service.add|service.remove|rtest.hello",
+            like (topic, "event.subscribe|event.unsubscribe|service.add|service.remove|rtest.hello",
                   "router-entry: response is %s", topic);
             break;
         case FLUX_MSGTYPE_EVENT:
@@ -179,7 +179,7 @@ void test_basic (flux_t *h)
      */
     if (!(request = flux_request_encode ("rtest.hello", NULL)))
         BAIL_OUT ("flux_request_encode failed");
-    router_entry_recv (entry, request); // router recives message from abcd
+    router_entry_recv (entry, request); // router receives message from abcd
     diag ("basic: sent rtest.hello request");
     flux_msg_destroy (request);
     ok (flux_reactor_run (r, 0) >= 0,
@@ -192,11 +192,11 @@ void test_basic (flux_t *h)
      * - basic_recv() is called in the context of router_entry_recv()
      *   in this case so don't start the reactor.
      */
-    if (!(request = flux_request_encode ("local.sub",
+    if (!(request = flux_request_encode ("event.subscribe",
                                          "{\"topic\":\"rtest\"}")))
         BAIL_OUT ("flux_request_encode failed");
-    router_entry_recv (entry, request); // router recives message from abcd
-    diag ("basic: sent local.sub request");
+    router_entry_recv (entry, request); // router receives message from abcd
+    diag ("basic: sent event.subscribe request");
     flux_msg_destroy (request);
 
     /* Send an rtest.pub request from client.
@@ -204,7 +204,7 @@ void test_basic (flux_t *h)
      */
     if (!(request = flux_request_encode ("rtest.pub", NULL)))
         BAIL_OUT ("flux_request_encode failed");
-    router_entry_recv (entry, request); // router recives message from abcd
+    router_entry_recv (entry, request); // router receives message from abcd
     diag ("basic: sent rtest.pub request");
     flux_msg_destroy (request);
     ok (flux_reactor_run (r, 0) >= 0,
@@ -212,11 +212,11 @@ void test_basic (flux_t *h)
 
     /* Now unsubscribe to rtest events.
      */
-    if (!(request = flux_request_encode ("local.unsub",
+    if (!(request = flux_request_encode ("event.unsubscribe",
                                          "{\"topic\":\"rtest\"}")))
         BAIL_OUT ("flux_request_encode failed");
-    router_entry_recv (entry, request); // router recives message from abcd
-    diag ("basic: sent local.unsub request");
+    router_entry_recv (entry, request); // router receives message from abcd
+    diag ("basic: sent event.unsubscribe request");
     flux_msg_destroy (request);
 
     /* Register testfu service.
@@ -228,7 +228,7 @@ void test_basic (flux_t *h)
                                          "{\"service\":\"testfu\"}")))
         BAIL_OUT ("flux_request_encode failed");
     router_entry_recv (entry, request); // router receives message from abcd
-    diag ("basic: sent local.sub request");
+    diag ("basic: sent service.add request");
     flux_msg_destroy (request);
     ok (flux_reactor_run (r, 0) >= 0, "basic: reactor processed one message");
 
@@ -238,7 +238,7 @@ void test_basic (flux_t *h)
      */
     if (!(request = flux_request_encode ("testfu.bar", NULL)))
         BAIL_OUT ("flux_request_encode failed");
-    router_entry_recv (entry, request); // router recives message from abcd
+    router_entry_recv (entry, request); // router receives message from abcd
     diag ("basic: sent testfu.bar request");
     flux_msg_destroy (request);
     ok (flux_reactor_run (r, 0) >= 0,
@@ -249,13 +249,20 @@ void test_basic (flux_t *h)
     if (!(request = flux_request_encode ("service.remove",
                                          "{\"service\":\"testfu\"}")))
         BAIL_OUT ("flux_request_encode failed");
-    router_entry_recv (entry, request); // router recives message from abcd
+    router_entry_recv (entry, request); // router receives message from abcd
     ok (flux_reactor_run (r, 0) >= 0, "basic: reactor processed one message");
+    flux_msg_destroy (request);
 
     router_entry_delete (entry);
     router_destroy (rtr);
 }
 
+void test_error (flux_t *h)
+{
+    ok (router_renew (NULL) == 0,
+        "router_renew rtr=NULL works as no-op");
+}
+
 int main (int argc, char *argv[])
 {
     flux_t *h;
@@ -263,12 +270,12 @@ int main (int argc, char *argv[])
     plan (NO_PLAN);
 
     diag ("starting test server");
-    test_server_environment_init ("test_router");
 
-    if (!(h = test_server_create (server_cb, NULL)))
+    if (!(h = test_server_create (FLUX_O_TEST_NOSUB, server_cb, NULL)))
         BAIL_OUT ("test_server_create failed");
 
     test_basic (h);
+    test_error (h);
 
     diag ("stopping test server");
     if (test_server_stop (h) < 0)
diff --git a/src/common/librouter/test/rpc_track.c b/src/common/librouter/test/rpc_track.c
new file mode 100644
index 000000000000..57f0dbdd7aa4
--- /dev/null
+++ b/src/common/librouter/test/rpc_track.c
@@ -0,0 +1,363 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <uuid.h>
+#include <flux/core.h>
+
+#include "src/common/librouter/rpc_track.h"
+#include "src/common/libtap/tap.h"
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/librouter/msg_hash.h"
+#include "src/common/libtap/tap.h"
+#include "ccan/array_size/array_size.h"
+
+#ifndef UUID_STR_LEN
+#define UUID_STR_LEN 37     // defined in later libuuid headers
+#endif
+
+/* Create a disconnect with same uuid as 'req'.
+ */
+flux_msg_t *create_disconnect (const flux_msg_t *req)
+{
+    const char *topic;
+    flux_msg_t *dis;
+    char buf[128];
+
+    if (!(dis = flux_msg_copy (req, false))
+        || flux_msg_get_topic (req, &topic) < 0
+        || snprintf (buf, sizeof (buf), "%s.disconnect", topic) >= sizeof (buf)
+        || flux_msg_set_topic (dis, buf) < 0
+        || flux_msg_set_noresponse (dis) < 0
+        || flux_msg_set_matchtag (dis, FLUX_MATCHTAG_NONE) < 0)
+        BAIL_OUT ("failed to create disconnect request");
+    return dis;
+}
+
+flux_msg_t *create_response (const flux_msg_t *req, int errnum)
+{
+    flux_msg_t *rep;
+
+    if (!(rep = flux_response_derive (req, errnum)))
+        BAIL_OUT ("flux_response_derive failed");
+    return rep;
+}
+
+flux_msg_t *create_request (uint32_t matchtag, int setflags, bool add_uuid)
+{
+    flux_msg_t *msg;
+    uuid_t uuid;
+    char uuid_str[UUID_STR_LEN];
+
+    if (!(msg = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("flux_request_create failed");
+    if (flux_msg_set_matchtag (msg, matchtag) < 0)
+        BAIL_OUT ("flux_msg_set_matchtag failed");
+    if (flux_msg_set_flag (msg, setflags) < 0)
+        BAIL_OUT ("flux_msg_set_flag failed");
+
+    flux_msg_route_enable (msg);
+    if (add_uuid) {
+        uuid_generate (uuid);
+        uuid_unparse (uuid, uuid_str);
+        if (flux_msg_route_push (msg, uuid_str) < 0)
+            BAIL_OUT ("flux_msg_route_push failed");
+        }
+    return msg;
+}
+
+void purge (const flux_msg_t *msg, void *arg)
+{
+    int *count = arg;
+    (*count)++;
+}
+
+void test_purge (void)
+{
+    struct rpc_track *rt;
+    int count;
+    flux_msg_t *msg[2];
+    int i;
+
+    msg[0] = create_request (1, 0, true);
+    msg[1] = create_request (2, FLUX_MSGFLAG_STREAMING, true);
+
+    if (!(rt = rpc_track_create (MSG_HASH_TYPE_UUID_MATCHTAG)))
+        BAIL_OUT ("rpc_track_create failed");
+
+    count = 0;
+    rpc_track_purge (rt, purge, &count);
+    ok (count == 0,
+        "rpc_track_purge does nothing on empty hash");
+
+    for (i = 0; i < ARRAY_SIZE (msg); i++)
+        rpc_track_update (rt, msg[i]);
+    ok (rpc_track_count (rt) == 2,
+        "rpc_track_update tracks 2 messages");
+
+    count = 0;
+    rpc_track_purge (rt, purge, &count);
+    ok (count == 2,
+        "rpc_track_purge called callback 2 times");
+    ok (rpc_track_count (rt) == 0,
+        "rpc_track_purge emptied hash");
+
+    rpc_track_destroy (rt);
+
+    for (i = 0; i < ARRAY_SIZE (msg); i++)
+        flux_msg_decref (msg[i]);
+}
+
+void test_basic (void)
+{
+    struct rpc_track *rt;
+    flux_msg_t *req[4];
+    flux_msg_t *rep[3];
+    int i;
+
+    req[0] = create_request (0, FLUX_MSGFLAG_NORESPONSE, true); // won't track
+    req[1] = create_request (1, 0, true);
+    req[2] = create_request (2, FLUX_MSGFLAG_STREAMING, true);
+    req[3] = flux_msg_copy (req[2], true); // same as 2 except new matchtag
+    if (!req[3])
+        BAIL_OUT ("flux_msg_copy failed");
+    if (flux_msg_set_matchtag (req[3], 3) < 0)
+        BAIL_OUT ("flux_msg_set_matchtag failed");
+
+    rep[0] = create_response (req[1], 0); // terminating (non-streaming)
+    rep[1] = create_response (req[2], 1); // terminating
+    rep[2] = create_response (req[3], 0); // not terminating (streaming)
+
+    if (!(rt = rpc_track_create (MSG_HASH_TYPE_UUID_MATCHTAG)))
+        BAIL_OUT ("rpc_track_create failed");
+
+    ok (rpc_track_count (rt) == 0,
+        "rpc_track_count returns 0 on empty hash");
+
+    for (i = 0; i < ARRAY_SIZE (req); i++)
+        rpc_track_update (rt, req[i]);
+    ok (rpc_track_count (rt) == 3,
+        "rpc_track_update works (3 of 4 requests tracked)");
+
+    for (i = 0; i < ARRAY_SIZE (rep); i++)
+        rpc_track_update (rt, rep[i]);
+    ok (rpc_track_count (rt) == 1,
+        "rpc_track_update works (2 requests terminated)");
+
+    rpc_track_destroy (rt);
+
+    for (i = 0; i < ARRAY_SIZE (req); i++)
+        flux_msg_decref (req[i]);
+    for (i = 0; i < ARRAY_SIZE (rep); i++)
+        flux_msg_decref (rep[i]);
+}
+
+void test_disconnect (void)
+{
+    struct rpc_track *rt;
+    flux_msg_t *req[4];
+    flux_msg_t *dis;
+    int i;
+
+    req[0] = create_request (0, FLUX_MSGFLAG_NORESPONSE, true); // won't track
+    req[1] = create_request (1, 0, true);
+    req[2] = create_request (2, FLUX_MSGFLAG_STREAMING, true);
+    req[3] = flux_msg_copy (req[2], true); // same as 2 except new matchtag
+    if (!req[3])
+        BAIL_OUT ("flux_msg_copy failed");
+    if (flux_msg_set_matchtag (req[3], 3) < 0)
+        BAIL_OUT ("flux_msg_set_matchtag failed");
+    dis = create_disconnect (req[2]);
+
+    if (!(rt = rpc_track_create (MSG_HASH_TYPE_UUID_MATCHTAG)))
+        BAIL_OUT ("rpc_track_create failed");
+
+    for (i = 0; i < ARRAY_SIZE (req); i++)
+        rpc_track_update (rt, req[i]);
+    ok (rpc_track_count (rt) == 3,
+        "rpc_track_update works (3 of 4 requests tracked)");
+
+    rpc_track_update (rt, dis);
+    ok (rpc_track_count (rt) == 1, // 2 of 3 match the disconnect
+        "rpc_track_update correctly processed disconnect request");
+
+    rpc_track_destroy (rt);
+
+    for (i = 0; i < ARRAY_SIZE (req); i++)
+        flux_msg_decref (req[i]);
+    flux_msg_decref (dis);
+}
+
+void test_badarg (void)
+{
+    struct rpc_track *rt;
+    flux_msg_t *msg;
+    uuid_t uuid;
+    char uuid_str[UUID_STR_LEN];
+
+    uuid_generate (uuid);
+    uuid_unparse (uuid, uuid_str);
+
+    errno = 0;
+    ok (rpc_track_create (42) == NULL && errno == EINVAL,
+        "rpc_track_create type=42 fails with EINVAL");
+
+    if (!(rt = rpc_track_create (MSG_HASH_TYPE_UUID_MATCHTAG)))
+        BAIL_OUT ("rpc_track_create failed");
+
+    rpc_track_update (rt, NULL);
+    ok (rpc_track_count (rt) == 0,
+        "rpc_track_update msg=NULL is a no-op");
+
+    if (!(msg = flux_request_encode ("foo", NULL))
+        || flux_msg_set_matchtag (msg, 1) < 0)
+        BAIL_OUT ("could not create test message");
+    rpc_track_update (rt, NULL);
+    ok (rpc_track_count (rt) == 0,
+        "rpc_track_update msg=(no sender) is a no-op");
+    flux_msg_decref (msg);
+
+    if (!(msg = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("could not create test message");
+    flux_msg_route_enable (msg);
+    if (flux_msg_route_push (msg, uuid_str) < 0)
+        BAIL_OUT ("could not tweak test message");
+    rpc_track_update (rt, NULL);
+    ok (rpc_track_count (rt) == 0,
+        "rpc_track_update msg=(no matchtag) is a no-op");
+    flux_msg_decref (msg);
+
+    if (!(msg = flux_event_encode ("meep", NULL)))
+        BAIL_OUT ("could not create test message");
+    rpc_track_update (rt, NULL);
+    ok (rpc_track_count (rt) == 0,
+        "rpc_track_update msg=event is a no-op");
+    flux_msg_decref (msg);
+
+    if (!(msg = flux_control_encode (42, 43)))
+        BAIL_OUT ("could not create test message");
+    rpc_track_update (rt, NULL);
+    ok (rpc_track_count (rt) == 0,
+        "rpc_track_update msg=control is a no-op");
+    flux_msg_decref (msg);
+
+    if (!(msg = flux_request_encode ("foo", NULL)))
+        BAIL_OUT ("could not create test message");
+    flux_msg_route_enable (msg);
+    if (flux_msg_route_push (msg, uuid_str) < 0
+        || flux_msg_set_matchtag (msg, 1) < 0)
+        BAIL_OUT ("could not tweak test message");
+    rpc_track_update (rt, msg);
+    if (rpc_track_count (rt) != 1)
+        BAIL_OUT ("could not track legit request");
+    flux_msg_decref (msg);  // reminder: hash has one entry
+
+    if (!(msg = flux_request_encode ("foo.disconnect", NULL))
+        || flux_msg_set_noresponse (msg) < 0)
+        BAIL_OUT ("could not create test disconnect");
+    rpc_track_update (rt, msg);
+    ok (rpc_track_count (rt) == 1,
+        "a disconnect without a uuid has no effect");
+    flux_msg_decref (msg);
+
+    rpc_track_destroy (rt);
+}
+
+/* Will it hash?
+ */
+void test_hashable (void)
+{
+    struct rpc_track *rt;
+    flux_msg_t *msg;
+    int count;
+
+    if (!(rt = rpc_track_create (MSG_HASH_TYPE_UUID_MATCHTAG)))
+        BAIL_OUT ("rpc_track_create failed");
+
+    count = rpc_track_count (rt);
+    msg = create_request (1, 0, false);
+    rpc_track_update (rt, msg);
+    ok (rpc_track_count (rt) - count == 1,
+        "message 1 with valid matchtag, missing uuid is tracked");
+    flux_msg_decref (msg);
+
+    count = rpc_track_count (rt);
+    msg = create_request (2, 0, false);
+    rpc_track_update (rt, msg);
+    ok (rpc_track_count (rt) - count == 1,
+        "message 2 with new matchtag, missing uuid is tracked");
+    flux_msg_decref (msg);
+
+    /* This one is like RFC 27 sched alloc RPC, which sets matchtag to
+     * FLUX_MATCHTAG_NONE, but does not set FLUX_MSGFLAG_NORESPONSE flag.
+     */
+    count = rpc_track_count (rt);
+    msg = create_request (FLUX_MATCHTAG_NONE, 0, true);
+    rpc_track_update (rt, msg);
+    ok (rpc_track_count (rt) - count == 0,
+        "message with no matchtag, valid uuid is not tracked");
+    flux_msg_decref (msg);
+
+    rpc_track_destroy (rt);
+}
+
+void test_nilarg (void)
+{
+    struct rpc_track *rt;
+    flux_msg_t *msg;
+
+    if (!(rt = rpc_track_create (MSG_HASH_TYPE_UUID_MATCHTAG)))
+        BAIL_OUT ("rpc_track_create failed");
+    msg = create_request (1, 0, true);
+    rpc_track_update (rt, msg);
+
+    ok (rpc_track_count (NULL) == 0,
+        "rpc_track_count rt=NULL returns 0");
+
+    lives_ok ({rpc_track_update (NULL, msg);},
+              "rpc_track_update rt=NULL doesn't crash");
+    lives_ok ({rpc_track_update (rt, NULL);},
+              "rpc_track_update msg=NULL doesn't crash");
+
+    lives_ok ({rpc_track_purge (NULL, purge, NULL);},
+              "rpc_track_purge rt=NULL doesn't crash");
+
+    lives_ok ({rpc_track_purge (rt, NULL, NULL);},
+              "rpc_track_purge func=NULL doesn't crash");
+    lives_ok ({rpc_track_destroy (NULL);},
+              "rpc_track_destroy rt=NULL doesn't crash");
+
+
+    flux_msg_decref (msg);
+    rpc_track_destroy (rt);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_basic ();
+    test_purge ();
+    test_disconnect ();
+    test_badarg ();
+    test_hashable ();
+    test_nilarg ();
+
+    done_testing();
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/librouter/test/sendfd.c b/src/common/librouter/test/sendfd.c
index 600b2f08ccb7..4d0d3f67139d 100644
--- a/src/common/librouter/test/sendfd.c
+++ b/src/common/librouter/test/sendfd.c
@@ -15,13 +15,18 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <unistd.h>
-#include <czmq.h>
+#ifndef HAVE_PIPE2
+#include "src/common/libmissing/pipe2.h"
+#endif
+
 
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/librouter/sendfd.h"
 #include "src/common/libutil/fdutils.h"
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 /* Send a small message over a blocking pipe.
  * We assume that there's enough buffer to do this in one go.
@@ -43,7 +48,7 @@ void test_basic (void)
         "recvfd works");
     ok (flux_request_decode (msg2, &topic, &payload) == 0,
         "received request can be decoded");
-    ok (!strcmp (topic, "foo.bar"),
+    ok (streq (topic, "foo.bar"),
         "decoded request has expected topic string");
     ok (payload == NULL,
         "decoded request has expected (lack of) payload");
@@ -63,12 +68,26 @@ void test_large (void)
     char buf[8192];
     const char *topic;
     const void *buf2;
-    int buf2len;
+    size_t buf2len;
+#if defined(F_GETPIPE_SZ)
+    int min_size = 16384;
+    int size;
+#endif
 
     memset (buf, 0x0f, sizeof (buf));
 
     if (pipe2 (pfd, O_CLOEXEC) < 0)
         BAIL_OUT ("pipe2 failed");
+
+#if defined(F_GETPIPE_SZ)
+    size = fcntl (pfd[1], F_GETPIPE_SZ);
+    if (size < min_size)
+        (void)fcntl (pfd[1], F_SETPIPE_SZ, min_size);
+    size = fcntl (pfd[1], F_GETPIPE_SZ);
+    skip (size < min_size, 4, "%d byte pipe is too small", size);
+#else
+    skip (true, 4, "F_GETPIPE_SZ not defined");
+#endif
     if (!(msg = flux_request_encode_raw ("foo.bar", buf, sizeof (buf))))
         BAIL_OUT ("flux_request_encode failed");
     ok (sendfd (pfd[1], msg, NULL) == 0,
@@ -77,7 +96,7 @@ void test_large (void)
         "recvfd works");
     ok (flux_request_decode_raw (msg2, &topic, &buf2, &buf2len) == 0,
         "received request can be decoded");
-    ok (!strcmp (topic, "foo.bar"),
+    ok (streq (topic, "foo.bar"),
         "decoded request has expected topic string");
     ok (buf2 != NULL
         && buf2len == sizeof (buf)
@@ -86,6 +105,9 @@ void test_large (void)
 
     flux_msg_destroy (msg);
     flux_msg_destroy (msg2);
+
+    end_skip;
+
     close (pfd[1]);
     close (pfd[0]);
 }
@@ -209,7 +231,7 @@ struct io *io_create (flux_reactor_t *r,
  * Set up nonblocking sender and receiver.
  * Run the reactor:
  * - sender sends all enqueued messages
- * - receiver enqueues all recived messages
+ * - receiver enqueues all received messages
  * Verify that messages are all received intact.
  */
 void test_nonblock (int size, int count)
@@ -258,14 +280,14 @@ void test_nonblock (int size, int count)
     while ((msg = zlist_pop (ior->queue))) {
         const char *topic;
         const void *buf2;
-        int buf2len;
+        size_t buf2len;
 
         if (flux_request_decode_raw (msg, &topic, &buf2, &buf2len) < 0) {
             diag ("flux_request_decode_raw: %s", flux_strerror (errno));
             errors++;
             goto next;
         }
-        if (strcmp (topic, "foo.bar") != 0) {
+        if (!streq (topic, "foo.bar")) {
             diag ("decoded wrong topic: %s", topic);
             errors++;
             goto next;
diff --git a/src/common/librouter/test/servhash.c b/src/common/librouter/test/servhash.c
index 54c4f209db74..36aad37727e5 100644
--- a/src/common/librouter/test/servhash.c
+++ b/src/common/librouter/test/servhash.c
@@ -12,10 +12,11 @@
 #include "config.h"
 #endif
 #include <flux/core.h>
-#include <czmq.h>
 
 #include "src/common/libtap/tap.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libtestutil/util.h"
+#include "ccan/str/str.h"
 #include "src/common/librouter/servhash.h"
 
 /*
@@ -180,6 +181,7 @@ void test_basic (flux_t *h)
     flux_msg_t *req2;
     flux_reactor_t *r;
     const char *uuid;
+    flux_future_t *f;
 
     if (!(add = flux_request_encode ("service.add", NULL))
             || flux_msg_pack (add, "{s:s}", "service", "fubar") < 0)
@@ -213,11 +215,28 @@ void test_basic (flux_t *h)
         && errno == EEXIST,
         "servhash_add for same service failed with EEXIST");
 
+    /* servhash_renew makes a synchronous RPC internally for any existing
+     * services.  The service thread should respond with EEXIST.
+     */
+    errno = 0;
+    ok (servhash_renew (sh) < 0 && errno == EEXIST,
+        "servhash_renew fails with EEXIST");
+
+    /* remove the service with a direct rpc, then call servhash_renew()
+     * to restore it.
+     */
+    if (!(f = flux_rpc_message (h, remove, FLUX_NODEID_ANY, 0))
+        || flux_rpc_get (f, NULL) < 0)
+        BAIL_OUT ("error removing fubar with direct RPC");
+    flux_future_destroy (f);
+    ok (servhash_renew (sh) == 0,
+        "servhash_renew works");
+
     /* match some messages */
     uuid = NULL;
     ok (servhash_match (sh, req, &uuid) == 0,
         "servhash_match matched request");
-    ok (uuid != NULL && !strcmp (uuid, "basic-uuid"),
+    ok (uuid != NULL && streq (uuid, "basic-uuid"),
         "and matched it to the correct uuid");
     errno = 0;
     ok (servhash_match (sh, req2, &uuid) < 0 && errno == ENOENT,
@@ -232,6 +251,10 @@ void test_basic (flux_t *h)
     ok (last_errnum == 0,
         "remove request was successful");
 
+    /* renew with no valid services is a no-op */
+    ok (servhash_renew (sh) == 0,
+        "servhash_renew works with empty servhash");
+
     flux_msg_destroy (add);
     flux_msg_destroy (remove);
     flux_msg_destroy (req);
@@ -246,9 +269,8 @@ int main (int argc, char *argv[])
     plan (NO_PLAN);
 
     diag ("starting test server");
-    test_server_environment_init ("test_router");
 
-    if (!(h = test_server_create (server_cb, NULL)))
+    if (!(h = test_server_create (0, server_cb, NULL)))
         BAIL_OUT ("test_server_create failed");
 
     test_basic (h);
@@ -258,7 +280,6 @@ int main (int argc, char *argv[])
     if (test_server_stop (h) < 0)
         BAIL_OUT ("test_server_stop failed");
     flux_close (h);
-
     done_testing ();
 
     return 0;
diff --git a/src/common/librouter/test/subhash.c b/src/common/librouter/test/subhash.c
index ed27c7f1875e..fc9335ec1c33 100644
--- a/src/common/librouter/test/subhash.c
+++ b/src/common/librouter/test/subhash.c
@@ -103,7 +103,8 @@ void test_callbacks (void)
     sub_count = 0;
     unsub_count = 0;
 
-    /* subhash destroy unsubscribes to remaining topics */
+    /* Subscribe to two topics.
+     */
     ok (subhash_subscribe (sub, "bar") == 0,
         "subhash_subscribe bar");
     ok (subhash_subscribe (sub, "baz") == 0,
@@ -111,6 +112,11 @@ void test_callbacks (void)
     ok (sub_count == 2,
         "sub callback called twice");
 
+    ok (subhash_renew (sub) == 0,
+        "subhash_renew works");
+    ok (sub_count == 4,
+        "sub callback called twice more");
+
     subhash_destroy (sub);
 
     ok (unsub_count == 2,
@@ -179,6 +185,9 @@ void test_errors (void)
     ok (subhash_topic_match (sub, NULL) == false,
         "subhash_topic_match topic=NULL returns false");
 
+    ok (subhash_renew (NULL) == 0,
+        "subhash_renew sub=NULL succeeds as a no-op");
+
     lives_ok ({ subhash_destroy (NULL);},
         "subhash_destroy sub=NULL doesn't crash");
 
diff --git a/src/common/librouter/test/usock.c b/src/common/librouter/test/usock.c
index 918b0458ccc2..c07664b8e996 100644
--- a/src/common/librouter/test/usock.c
+++ b/src/common/librouter/test/usock.c
@@ -152,6 +152,7 @@ void conn_invalid (void)
     ok (usock_conn_create (r, 0, -1) == NULL && errno == EINVAL,
         "usock_conn_create outfd=-1 fails with EINVAL");
 
+    flux_msg_destroy (msg);
     usock_conn_destroy (conn);
     (void)close (fd[0]);
     (void)close (fd[1]);
diff --git a/src/common/librouter/test/usock_echo.c b/src/common/librouter/test/usock_echo.c
index d446e73f8c7d..f36252b400f2 100644
--- a/src/common/librouter/test/usock_echo.c
+++ b/src/common/librouter/test/usock_echo.c
@@ -18,6 +18,7 @@
 #include "src/common/libutil/unlink_recursive.h"
 #include "src/common/libtestutil/util.h"
 #include "src/common/librouter/usock.h"
+#include "ccan/str/str.h"
 
 #include "usock_util.h"
 
@@ -138,7 +139,7 @@ static bool equal_message (const flux_msg_t *m1, const flux_msg_t *m2)
     int type1, type2;
     const char *topic1, *topic2;
     const void *buf1, *buf2;
-    int len1, len2;
+    size_t len1, len2;
 
     if (flux_msg_get_type (m1, &type1) < 0)
         return false;
@@ -150,7 +151,7 @@ static bool equal_message (const flux_msg_t *m1, const flux_msg_t *m2)
         return false;
     if (flux_msg_get_topic (m2, &topic2) < 0)
         return false;
-    if (strcmp (topic1, topic2) != 0)
+    if (!streq (topic1, topic2))
         return false;
     if (flux_msg_has_payload (m1) && !flux_msg_has_payload (m2))
         return false;
@@ -293,9 +294,8 @@ int main (int argc, char *argv[])
     signal (SIGPIPE, SIG_IGN);
 
     diag ("starting test server");
-    test_server_environment_init ("usock_server");
 
-    if (!(h = test_server_create (server_cb, NULL)))
+    if (!(h = test_server_create (0, server_cb, NULL)))
         BAIL_OUT ("test_server_create failed");
 
     test_early_disconnect (h);
diff --git a/src/common/librouter/test/usock_emfile.c b/src/common/librouter/test/usock_emfile.c
new file mode 100644
index 000000000000..d29b214b226c
--- /dev/null
+++ b/src/common/librouter/test/usock_emfile.c
@@ -0,0 +1,349 @@
+/************************************************************  \
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/param.h>
+#include <sys/resource.h>
+#include <pthread.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/unlink_recursive.h"
+#include "src/common/libutil/fdwalk.h"
+#include "src/common/libtestutil/util.h"
+#include "src/common/librouter/usock.h"
+
+#include "usock_util.h"
+
+/* EMFILE test.
+ *
+ * Start a usock server and allow one client to connect.
+ * Then set RLIMIT_NOFILE such that second client can connect,
+ * but the server will get EMFILE (e.g. current count + 2).
+ *
+ * Then allow first client to exit, freeing a few fds and letting
+ *  2nd connection succeed.
+ *
+ * Ensure that both clients connected and successfully sent a message
+ *  each, and count the number of times the server exited the reactor
+ *  and ensure that count is not unreasonable.
+ *
+ */
+
+struct test_params {
+    int ready;
+    int loop;
+    int recvd;
+};
+
+static char tmpdir[PATH_MAX + 1];
+static char sockpath[PATH_MAX + 1];
+
+pthread_mutex_t server_mutex = PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t server_cond = PTHREAD_COND_INITIALIZER;
+static int server_ready = 0;
+
+static void tmpdir_destroy (void)
+{
+    diag ("rm -r %s", tmpdir);
+    if (unlink_recursive (tmpdir) < 0)
+        BAIL_OUT ("unlink_recursive failed");
+}
+
+static void tmpdir_create (void)
+{
+    const char *tmp = getenv ("TMPDIR");
+
+    if (snprintf (tmpdir,
+                  sizeof (tmpdir),
+                  "%s/usock.XXXXXXX",
+                  tmp ? tmp : "/tmp") >= sizeof (tmpdir))
+        BAIL_OUT ("tmpdir_create buffer overflow");
+    if (!mkdtemp (tmpdir))
+        BAIL_OUT ("mkdtemp %s: %s", tmpdir, strerror (errno));
+    diag ("mkdir %s", tmpdir);
+}
+
+static void server_recv_cb (struct usock_conn *conn, flux_msg_t *msg, void *arg)
+{
+    struct test_params *tp = arg;
+    const char *topic = NULL;
+
+    if (flux_request_decode (msg, &topic, NULL) < 0)
+        diag ("usock_conn_send failed: %s", flux_strerror (errno));
+
+    if (usock_conn_send (conn, msg) < 0)
+        diag ("usock_conn_send failed: %s", flux_strerror (errno));
+
+    tp->recvd++;
+}
+
+static void server_error_cb (struct usock_conn *conn, int errnum, void *arg)
+{
+    diag ("server_error_cb uuid=%.5s: %s",
+         usock_conn_get_uuid (conn),
+         flux_strerror (errnum));
+
+    usock_conn_destroy (conn);
+}
+
+static void server_acceptor (struct usock_conn *conn, void *arg)
+{
+    const struct flux_msg_cred *cred;
+
+    cred = usock_conn_get_cred (conn);
+
+    usock_conn_set_error_cb (conn, server_error_cb, arg);
+    usock_conn_set_recv_cb (conn, server_recv_cb, arg);
+
+    usock_conn_accept (conn, cred);
+}
+
+static void check_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct test_params *params = arg;
+    params->loop++;
+}
+
+static int server_cb (flux_t *h, void *arg)
+{
+    flux_watcher_t *w;
+    flux_reactor_t *r = flux_get_reactor (h);
+    struct usock_server *server;
+    struct test_params *tp = arg;
+
+    if (!(server = usock_server_create (r, sockpath, 0644))) {
+        diag ("usock_server_create failed");
+        return -1;
+    }
+    usock_server_set_acceptor (server, server_acceptor, tp);
+
+    if (!(w = flux_check_watcher_create (r, check_cb, tp))) {
+        diag ("flux_check_watcher_create failed");
+        return -1;
+    }
+    flux_watcher_start (w);
+
+    pthread_mutex_lock (&server_mutex);
+    server_ready = 1;
+    pthread_cond_signal (&server_cond);
+    pthread_mutex_unlock (&server_mutex);
+
+    if (flux_reactor_run (r, 0) < 0) {
+        diag ("flux_reactor_run failed");
+        return -1;
+    }
+    flux_watcher_destroy (w);
+    usock_server_destroy (server);
+    return 0;
+}
+
+/* End Test Server
+ */
+
+/*  Client thread - create a connection, send a message, pause a configurable
+ *   amount, then exit.
+ */
+struct client_args {
+    pthread_t thread;
+    int id;
+    int ready;
+    int done;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+};
+
+static void *client_thread (void *arg)
+{
+    int fd;
+    flux_msg_t *msg;
+    struct usock_client *client;
+    struct client_args *args = arg;
+
+    /*  Due to raciness with server, usock_client_connect() may fail due
+     *  to ENFILE. Just retry until condition resolves itself.
+     */
+    int retries = 5;
+    while ((fd = usock_client_connect (sockpath, USOCK_RETRY_DEFAULT)) < 0
+           && retries--)
+        usleep (1000);
+
+    if (fd < 0)
+        BAIL_OUT ("usock_client_connect: %s", strerror (errno));
+
+    if ((client = usock_client_create (fd)) == NULL)
+        BAIL_OUT ("usock_client_create");
+   if (!(msg = flux_request_encode ("nil", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+    if (usock_client_send (client, msg, 0) < 0)
+        BAIL_OUT ("client %d: usock_client_send message works", args->id);
+
+    /* Signal main thread we're ready */
+    pthread_mutex_lock (&args->mutex);
+    args->ready = 1;
+    pthread_cond_signal (&args->cond);
+    pthread_mutex_unlock (&args->mutex);
+
+    /*  Wait for test program to tell this client to exit
+     */
+    pthread_mutex_lock (&args->mutex);
+    while (!args->done)
+        pthread_cond_wait (&args->cond, &args->mutex);
+    pthread_mutex_unlock (&args->mutex);
+
+    diag ("client %d: disconnecting", args->id);
+    usock_client_destroy (client);
+    (void) close (fd);
+    flux_msg_destroy (msg);
+
+    return NULL;
+}
+
+
+static void fd_count (void *arg, int fd)
+{
+    int *countp = arg;
+    (*countp)++;
+}
+
+static int fds_inuse (void)
+{
+    int count = 0;
+    if (fdwalk (fd_count, &count))
+        BAIL_OUT ("fdwalk");
+    return count;
+}
+
+static void wait_for_server (void)
+{
+    pthread_mutex_lock (&server_mutex);
+    while (!server_ready)
+        pthread_cond_wait (&server_cond, &server_mutex);
+    server_ready = 0;
+    pthread_mutex_unlock (&server_mutex);
+}
+
+static void wait_for_client (struct client_args *a)
+{
+    pthread_mutex_lock (&a->mutex);
+    while (!a->ready)
+        pthread_cond_wait (&a->cond, &a->mutex);
+    pthread_mutex_unlock (&a->mutex);
+}
+
+static void wait_for_client_complete (struct client_args *a)
+{
+        pthread_mutex_lock (&a->mutex);
+        a->done = 1;
+        pthread_cond_signal (&a->cond);
+        pthread_mutex_unlock (&a->mutex);
+        pthread_join (a->thread, NULL);
+}
+
+int main (int argc, char *argv[])
+{
+    int e;
+    struct test_params tp = {0};
+    struct client_args cargs[2];
+    flux_t *h;
+
+    plan (NO_PLAN);
+
+    tmpdir_create ();
+    if (snprintf (sockpath,
+                  sizeof (sockpath),
+                  "%s/server",
+                  tmpdir) >= sizeof (sockpath))
+        BAIL_OUT ("Failed to create server sockpath");
+
+    signal (SIGPIPE, SIG_IGN);
+
+    diag ("starting test server");
+
+    if (!(h = test_server_create (0, server_cb, &tp)))
+        BAIL_OUT ("test_server_create failed");
+
+    wait_for_server ();
+
+    diag ("fds_inuse = %d", fds_inuse ());
+
+    for (int i = 0; i < 2; i++) {
+        struct client_args *a = &cargs[i];
+        memset (a, 0, sizeof (struct client_args));
+        e = pthread_mutex_init (&a->mutex, NULL);
+        if (e != 0)
+            BAIL_OUT ("pthread_mutex_init failed: %s", strerror (e));
+        e = pthread_cond_init (&a->cond, NULL);
+        if (e != 0)
+            BAIL_OUT ("pthread_cond_init failed: %s", strerror (e));
+        a->id = i;
+        a->done = 0;
+
+        e = pthread_create (&a->thread, NULL, client_thread, a);
+        if (e != 0)
+            BAIL_OUT ("pthread_create client %d failed: %s", i, strerror (e));
+
+        if (i == 0) {
+            struct rlimit rlim;
+
+            /*  Wait for first client to connect, then decrease number of
+             *   open files to current + 2, leaving the next client
+             *   to hang due to EMFILE on the server side.
+             */
+            wait_for_client (a);
+            diag ("client0 started");
+            diag ("fds_inuse = %d", fds_inuse ());
+
+            rlim.rlim_cur = fds_inuse () + 2;
+            rlim.rlim_max = rlim.rlim_cur;
+            diag ("setting nofile limit to %d", rlim.rlim_cur);
+            if (setrlimit (RLIMIT_NOFILE, &rlim) < 0)
+                BAIL_OUT ("setrlimit: %s", strerror (errno));
+        }
+
+    }
+    diag ("fds_inuse = %d", fds_inuse ());
+    //diag ("usleep 10000");
+    usleep (10000);
+    diag ("fds_inuse = %d", fds_inuse ());
+
+    for (int i = 0; i < 2; i++) {
+        wait_for_client_complete (&cargs[i]);
+        diag ("client%d done", i);
+    }
+
+    diag ("stopping test server");
+    if (test_server_stop (h) < 0)
+        BAIL_OUT ("test_server_stop failed");
+    flux_close (h);
+
+    diag ("results: %d recvd %d loops",
+          tp.recvd, tp.loop);
+
+    ok (tp.recvd == 2,
+        "got expected messages from two clients");
+    ok (tp.loop < 20,
+        "number of loops is not unreasonable");
+
+    tmpdir_destroy ();
+
+    diag ("fds_inuse = %d", fds_inuse ());
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librouter/test/usock_epipe.c b/src/common/librouter/test/usock_epipe.c
new file mode 100644
index 000000000000..4b05f947c2cf
--- /dev/null
+++ b/src/common/librouter/test/usock_epipe.c
@@ -0,0 +1,261 @@
+/************************************************************  \
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/param.h>
+#include <pthread.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/unlink_recursive.h"
+#include "src/common/libtestutil/util.h"
+#include "src/common/librouter/usock.h"
+#include "ccan/str/str.h"
+
+#include "usock_util.h"
+
+/* EPIPE handling test:
+ * 
+ * Client sends multiple messages to server and immediately closes socket.
+ * Server reads messages and tries to echo each back to client with closed
+ *  connection.
+ * Server tracks count of received messages in shared "test_params"
+ *  structure.
+ * A global mutex "server_mutex" is locked before tests are run and
+ *  unlocked by the server only after each client connection exits,
+ *  allowing the test to synchronize with the server
+ */
+
+struct test_params {
+    int ready;
+    int expected;
+    int recvd;
+};
+
+pthread_mutex_t server_mutex = PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t server_cond = PTHREAD_COND_INITIALIZER;
+
+static char tmpdir[PATH_MAX + 1];
+
+static void tmpdir_destroy (void)
+{
+    diag ("rm -r %s", tmpdir);
+    if (unlink_recursive (tmpdir) < 0)
+        BAIL_OUT ("unlink_recursive failed");
+}
+
+static void tmpdir_create (void)
+{
+    const char *tmp = getenv ("TMPDIR");
+
+    if (snprintf (tmpdir,
+                  sizeof (tmpdir),
+                  "%s/usock.XXXXXXX",
+                  tmp ? tmp : "/tmp") >= sizeof (tmpdir))
+        BAIL_OUT ("tmpdir_create buffer overflow");
+    if (!mkdtemp (tmpdir))
+        BAIL_OUT ("mkdtemp %s: %s", tmpdir, strerror (errno));
+    diag ("mkdir %s", tmpdir);
+}
+
+static void server_recv_cb (struct usock_conn *conn, flux_msg_t *msg, void *arg)
+{
+    struct test_params *tp = arg;
+    const char *topic = NULL;
+
+    if (flux_request_decode (msg, &topic, NULL) < 0)
+        diag ("usock_conn_send failed: %s", flux_strerror (errno));
+
+    if (topic && streq (topic, "init")) {
+        if (flux_msg_unpack (msg, "{s:i}", "expected", &tp->expected) < 0)
+            diag ("flux_msg_pack: %s", flux_strerror (errno));
+        diag ("connection: uuid=%.5s: expect %d messages",
+              usock_conn_get_uuid (conn),
+              tp->expected);
+    }
+
+    if (usock_conn_send (conn, msg) < 0)
+        diag ("usock_conn_send failed: %s", flux_strerror (errno));
+
+    tp->recvd++;
+}
+
+static void server_error_cb (struct usock_conn *conn, int errnum, void *arg)
+{
+    diag ("server_error_cb uuid=%.5s: %s",
+         usock_conn_get_uuid (conn),
+         flux_strerror (errnum));
+
+    usock_conn_destroy (conn);
+}
+
+static void server_close_cb (struct usock_conn *conn, void *arg)
+{
+    struct test_params *tp = arg;
+    diag ("server_close_cb: uuid=%.5s: recvd %d/%d messages",
+        usock_conn_get_uuid (conn),
+        tp->recvd, tp->expected);
+    pthread_mutex_lock (&server_mutex);
+    tp->ready = 1;
+    pthread_cond_signal (&server_cond);
+    pthread_mutex_unlock (&server_mutex);
+}
+
+static void server_acceptor (struct usock_conn *conn, void *arg)
+{
+    const struct flux_msg_cred *cred;
+
+    cred = usock_conn_get_cred (conn);
+
+    usock_conn_set_error_cb (conn, server_error_cb, arg);
+    usock_conn_set_recv_cb (conn, server_recv_cb, arg);
+    usock_conn_set_close_cb (conn, server_close_cb, arg);
+
+    usock_conn_accept (conn, cred);
+}
+
+static int server_cb (flux_t *h, void *arg)
+{
+    flux_reactor_t *r = flux_get_reactor (h);
+    char sockpath[PATH_MAX + 1];
+    struct usock_server *server;
+
+    if (snprintf (sockpath,
+                  sizeof (sockpath),
+                  "%s/server",
+                  tmpdir) >= sizeof (sockpath)) {
+        diag ("usock_server_create buffer overflow");
+        return -1;
+    }
+    if (!(server = usock_server_create (r, sockpath, 0644))) {
+        diag ("usock_server_create failed");
+        return -1;
+    }
+    usock_server_set_acceptor (server, server_acceptor, arg);
+
+    if (flux_reactor_run (r, 0) < 0) {
+        diag ("flux_reactor_run failed");
+        return -1;
+    }
+    usock_server_destroy (server);
+    return 0;
+}
+
+/* End Test Server
+ */
+
+/* Wait on condition variable for server to mark test results ready.
+ * Then ensure expected messages == recvd messages
+ */
+static void check_result (struct test_params *tp)
+{
+    pthread_mutex_lock (&server_mutex);
+    while (!tp->ready)
+        pthread_cond_wait (&server_cond, &server_mutex);
+    ok (tp->expected == tp->recvd,
+        "got %d/%d messages",
+        tp->recvd,
+        tp->expected);
+    pthread_mutex_unlock (&server_mutex);
+}
+
+
+/* Send a burst of count small messages and closes connection.
+ * Assumes that the OS socket buffer is sufficient to contain all of it.
+ */
+static void test_send_and_exit (flux_t *h, int count)
+{
+    int i;
+    char sockpath[PATH_MAX + 1];
+    flux_msg_t *msg;
+    flux_msg_t *nmsg = NULL;
+    int fd;
+    struct usock_client *client;
+
+    if (!(msg = flux_request_encode ("init", NULL))
+        || !(nmsg = flux_request_encode ("nil", NULL)))
+        BAIL_OUT ("flux_request_encode failed");
+    if (flux_msg_pack (msg, "{s:i}", "expected", count) < 0)
+        BAIL_OUT ("flux_msg_pack failed");
+
+    if (snprintf (sockpath,
+                  sizeof (sockpath),
+                  "%s/server",
+                  tmpdir) >= sizeof (sockpath))
+        BAIL_OUT ("buffer overflow");
+    fd = usock_client_connect (sockpath, USOCK_RETRY_DEFAULT);
+    ok (fd >= 0,
+        "usock_client_connect %s works", sockpath);
+    ok ((client = usock_client_create (fd)) != NULL,
+        "usock_client_create works");
+
+    ok (usock_client_send (client, msg, 0) == 0,
+        "usock_client_send init message works: expected=%d", count);
+
+    for (i = 1; i < count; i++) {
+        ok (usock_client_send (client, nmsg, 0) == 0,
+            "usock_client_send[%d] works", i);
+    }
+
+    diag ("disconnecting");
+
+    usock_client_destroy (client);
+    (void)close (fd);
+    flux_msg_destroy (msg);
+    flux_msg_destroy (nmsg);
+}
+int main (int argc, char *argv[])
+{
+    struct test_params tp = {0};
+    flux_t *h;
+
+    plan (NO_PLAN);
+
+    tmpdir_create ();
+
+    signal (SIGPIPE, SIG_IGN);
+
+    diag ("starting test server");
+
+    if (!(h = test_server_create (0, server_cb, &tp)))
+        BAIL_OUT ("test_server_create failed");
+
+    test_send_and_exit (h, 1);
+    check_result (&tp);
+    memset (&tp, 0, sizeof (tp));
+
+    test_send_and_exit (h, 2);
+    check_result (&tp);
+    memset (&tp, 0, sizeof (tp));
+
+    test_send_and_exit (h, 5);
+    check_result (&tp);
+    memset (&tp, 0, sizeof (tp));
+
+    test_send_and_exit (h, 7);
+    check_result (&tp);
+    memset (&tp, 0, sizeof (tp));
+
+    diag ("stopping test server");
+    if (test_server_stop (h) < 0)
+        BAIL_OUT ("test_server_stop failed");
+    flux_close (h);
+
+    tmpdir_destroy ();
+    done_testing ();
+
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librouter/test/usock_service.c b/src/common/librouter/test/usock_service.c
new file mode 100644
index 000000000000..90a6f02f1b5b
--- /dev/null
+++ b/src/common/librouter/test/usock_service.c
@@ -0,0 +1,151 @@
+/************************************************************  \
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <limits.h>
+#include <pthread.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/unlink_recursive.h"
+#include "src/common/librouter/usock_service.h"
+
+struct server_context {
+    char sockpath[1024];
+    pthread_t t;
+    flux_t *h;
+    flux_reactor_t *r;
+    flux_msg_handler_t **handlers;
+};
+
+void tmpdir_destroy (const char *path)
+{
+    diag ("rm -r %s", path);
+    if (unlink_recursive (path) < 0)
+        BAIL_OUT ("unlink_recursive failed");
+}
+
+void tmpdir_create (char *buf, int size)
+{
+    const char *tmpdir = getenv ("TMPDIR");
+
+    if (snprintf (buf,
+                  size,
+                  "%s/usock.XXXXXXX",
+                  tmpdir ? tmpdir : "/tmp") >= size)
+        BAIL_OUT ("tmpdir_create buffer overflow");
+    if (!mkdtemp (buf))
+        BAIL_OUT ("mkdtemp %s: %s", buf, strerror (errno));
+    diag ("mkdir %s", buf);
+}
+
+void hello_cb (flux_t *h,
+               flux_msg_handler_t *mh,
+               const flux_msg_t *msg,
+               void *arg)
+{
+    diag ("hello");
+    if (flux_respond (h, msg, NULL) < 0)
+        BAIL_OUT ("flux_respond failed");
+}
+
+void disconnect_cb (flux_t *h,
+                    flux_msg_handler_t *mh,
+                    const flux_msg_t *msg,
+                    void *arg)
+{
+    const char *uuid;
+    if ((uuid = flux_msg_route_first (msg)))
+        diag ("disconnect from %.5s", uuid);
+    flux_reactor_stop (flux_get_reactor (h));
+}
+
+void *server_thread (void *arg)
+{
+    struct server_context *ctx = arg;
+
+    flux_reactor_run (ctx->r, 0);
+    return NULL;
+}
+
+const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST, "hello", hello_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "disconnect", disconnect_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void server_create (struct server_context *ctx, const char *tmpdir)
+{
+    if (!(ctx->r = flux_reactor_create (0)))
+        BAIL_OUT ("flux_reactor_create failed");
+    snprintf (ctx->sockpath, sizeof (ctx->sockpath), "%s/sock", tmpdir);
+    ctx->h = usock_service_create (ctx->r, ctx->sockpath, true);
+    ok (ctx->h != NULL,
+        "usock_service_create listening on %s", ctx->sockpath);
+    ok (flux_msg_handler_addvec (ctx->h, htab, &ctx, &ctx->handlers) == 0,
+        "successfully registered msg handlers");
+    ok (pthread_create (&ctx->t, NULL, server_thread, ctx) == 0,
+        "started server thread");
+}
+
+void server_destroy (struct server_context *ctx)
+{
+    ok (pthread_join (ctx->t, NULL) == 0,
+        "joined with server thread");
+    flux_msg_handler_delvec (ctx->handlers);
+    flux_close (ctx->h);
+    flux_reactor_destroy (ctx->r);
+}
+
+void simple_check (const char *tmpdir)
+{
+    char uri[2048];
+    struct server_context ctx;
+    flux_t *h;
+    flux_future_t *f;
+
+    memset (&ctx, 0, sizeof (ctx));
+    server_create (&ctx, tmpdir);
+
+    snprintf (uri, sizeof (uri), "local://%s", ctx.sockpath);
+    h = flux_open (uri, 0);
+    ok (h != NULL,
+        "client connected to server");
+
+    if (!(f = flux_rpc (h, "hello", NULL, 0, 0)))
+        BAIL_OUT ("error sending hello RPC");
+    ok (flux_rpc_get (f, NULL) == 0,
+        "got response to HELLO rpc");
+    flux_future_destroy (f);
+
+    flux_close (h); // triggers disconnect
+
+    server_destroy (&ctx);
+}
+
+int main (int argc, char *argv[])
+{
+    char tmpdir[PATH_MAX + 1];
+
+    plan (NO_PLAN);
+    tmpdir_create (tmpdir, sizeof (tmpdir));
+
+    simple_check (tmpdir);
+
+    tmpdir_destroy (tmpdir);
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/librouter/test/usock_util.c b/src/common/librouter/test/usock_util.c
index a893d1b5ee97..b074e5456f82 100644
--- a/src/common/librouter/test/usock_util.c
+++ b/src/common/librouter/test/usock_util.c
@@ -18,9 +18,9 @@
 #include <fcntl.h>
 #include <stdlib.h>
 #include <flux/core.h>
-#include <czmq.h>
 
 #include "src/common/libtap/tap.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/unlink_recursive.h"
 #include "src/common/librouter/usock.h"
 
diff --git a/src/common/librouter/usock.c b/src/common/librouter/usock.c
index 3f25458f516e..3ff1857c9923 100644
--- a/src/common/librouter/usock.c
+++ b/src/common/librouter/usock.c
@@ -39,15 +39,18 @@
 #include <sys/un.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
+#if HAVE_SYS_UCRED_H
+#include <sys/ucred.h>
+#endif
 #include <poll.h>
 #include <unistd.h>
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <czmq.h>
 #include <uuid.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/aux.h"
 #include "src/common/libutil/fdutils.h"
 #include "src/common/libutil/errno_safe.h"
@@ -83,6 +86,9 @@ struct usock_conn {
     struct usock_io out;
     zlist_t *outqueue;
 
+    usock_conn_close_f close_cb;
+    void *close_arg;
+
     usock_conn_error_f error_cb;
     void *error_arg;
 
@@ -125,6 +131,17 @@ void usock_conn_set_error_cb (struct usock_conn *conn,
     }
 }
 
+void usock_conn_set_close_cb (struct usock_conn *conn,
+                              usock_conn_close_f cb,
+                              void *arg)
+{
+    if (conn) {
+        conn->close_cb = cb;
+        conn->close_arg = arg;
+    }
+}
+
+
 void usock_conn_set_recv_cb (struct usock_conn *conn,
                              usock_conn_recv_f cb,
                              void *arg)
@@ -215,6 +232,15 @@ static void conn_read_cb (flux_reactor_t *r,
     conn_io_error (conn, errno);
 }
 
+static int conn_outqueue_drop (struct usock_conn *conn)
+{
+    flux_msg_t *msg = zlist_pop (conn->outqueue);
+    if (msg == NULL)
+        return 0;
+    flux_msg_decref (msg);
+    return 1;
+}
+
 static void conn_write_cb (flux_reactor_t *r,
                            flux_watcher_t *w,
                            int revents,
@@ -231,12 +257,23 @@ static void conn_write_cb (flux_reactor_t *r,
         const flux_msg_t *msg = zlist_head (conn->outqueue);
         if (msg) {
             if (sendfd (conn->out.fd, msg, &conn->out.iobuf) < 0) {
-                if (errno != EWOULDBLOCK && errno != EAGAIN)
+                if (errno == EPIPE) {
+                    /* Remote peer has closed connection.
+                     * However, there may still be pending messages sent
+                     * by peer, so do not destroy connection here. Instead,
+                     * drop all pending messages in the output queue, and
+                     * let connection be closed after EOF/ECONNRESET from
+                     * *read* side of connection.
+                     */
+                    while (conn_outqueue_drop (conn))
+                        ;
+                    flux_watcher_stop (conn->out.w);
+                }
+                else if (errno != EWOULDBLOCK && errno != EAGAIN)
                     goto error;
             }
             else {
-                (void)zlist_pop (conn->outqueue);
-                flux_msg_decref (msg);
+                (void) conn_outqueue_drop (conn);
                 if (zlist_size (conn->outqueue) == 0)
                     flux_watcher_stop (conn->out.w);
             }
@@ -292,6 +329,8 @@ void usock_conn_destroy (struct usock_conn *conn)
 {
     if (conn) {
         int saved_errno = errno;
+        if (conn->close_cb)
+            (*conn->close_cb) (conn, conn->close_arg);
         aux_destroy (&conn->aux);
         flux_watcher_destroy (conn->in.w);
         iobuf_clean (&conn->in.iobuf);
@@ -349,12 +388,16 @@ void usock_server_destroy (struct usock_server *server)
     }
 }
 
-static int get_socket_peercred (int fd, struct flux_msg_cred *cred)
+static int usock_get_cred (int fd, struct flux_msg_cred *cred)
 {
-    struct ucred ucred;
-    socklen_t crlen;
 
-    crlen = sizeof (ucred);
+    if (fd < 0 || !cred) {
+        errno = EINVAL;
+        return -1;
+    }
+#if defined(SO_PEERCRED)
+    struct ucred ucred;
+    socklen_t crlen = sizeof (ucred);
     if (getsockopt (fd,
                     SOL_SOCKET,
                     SO_PEERCRED,
@@ -366,6 +409,19 @@ static int get_socket_peercred (int fd, struct flux_msg_cred *cred)
         return -1;
     }
     cred->userid = ucred.uid;
+#elif defined(LOCAL_PEERCRED)
+    struct xucred ucred;
+    socklen_t crlen = sizeof (ucred);
+    if (getsockopt (fd, 0, LOCAL_PEERCRED, &ucred, &crlen) < 0)
+        return -1;
+    if (ucred.cr_version != XUCRED_VERSION) {
+        errno = EINVAL;
+        return -1;
+    }
+    cred->userid = ucred.cr_uid;
+#else
+#error Neither SO_PEERCRED nor LOCAL_PEERCRED are defined
+#endif
     cred->rolemask = FLUX_ROLE_NONE;
     return 0;
 }
@@ -420,10 +476,19 @@ static struct usock_conn *server_accept (struct usock_server *server,
     struct usock_conn *conn;
     int cfd;
 
+#ifdef SOCK_CLOEXEC
     if ((cfd = accept4 (server->fd, NULL, NULL, SOCK_CLOEXEC)) < 0)
         return NULL;
+#else
+    if ((cfd = accept (server->fd, NULL, NULL)) < 0)
+        return NULL;
+    if (fd_set_cloexec (cfd) < 0) {
+        ERRNO_SAFE_WRAP (close, cfd);
+        return NULL;
+    }
+#endif
     if (!(conn = usock_conn_create (r, cfd, cfd))
-                        || get_socket_peercred (cfd, &conn->cred) < 0) {
+        || usock_get_cred (cfd, &conn->cred) < 0) {
         ERRNO_SAFE_WRAP (close, cfd);
         return NULL;
     }
@@ -431,6 +496,15 @@ static struct usock_conn *server_accept (struct usock_server *server,
     return conn;
 }
 
+static void timeout_cb (flux_reactor_t *r,
+                        flux_watcher_t *w,
+                        int revents,
+                        void *arg)
+{
+    struct usock_server *server = arg;
+    flux_watcher_start (server->w);
+    flux_watcher_destroy (w);
+}
 
 static void server_cb (flux_reactor_t *r,
                        flux_watcher_t *w,
@@ -442,8 +516,24 @@ static void server_cb (flux_reactor_t *r,
     if ((revents & FLUX_POLLIN)) {
         struct usock_conn *conn;
 
-        if (!(conn = server_accept (server, r)))
+        if (!(conn = server_accept (server, r))) {
+            if (errno == ENFILE || errno == EMFILE) {
+                /*  Too many open files. Do not just go back to sleep in the
+                 *   reactor since we'll wake right back up again. Instead
+                 *   disable this callback until after a short pause, giving
+                 *   time for fds to be closed and have success on the next
+                 *   try...
+                 */
+                flux_watcher_t *tw = flux_timer_watcher_create (r,
+                                                                0.01,
+                                                                0,
+                                                                timeout_cb,
+                                                                server);
+                flux_watcher_start (tw);
+                flux_watcher_stop (w);
+            }
             return;
+        }
         if (zlist_append (server->connections, conn) < 0) {
             usock_conn_destroy (conn);
             return;
@@ -472,8 +562,14 @@ struct usock_server *usock_server_create (flux_reactor_t *r,
     }
     if (!(server = calloc (1, sizeof (*server))))
         return NULL;
+#ifdef SOCK_CLOEXEC
     if ((server->fd = socket (AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0)) < 0)
         goto error;
+#else
+    if ((server->fd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0
+        || fd_set_cloexec (server->fd) < 0)
+        goto error;
+#endif
     if (!(server->sockpath = strdup (sockpath)))
         goto error;
     if (remove (sockpath) < 0 && errno != ENOENT)
@@ -615,13 +711,23 @@ int usock_client_connect (const char *sockpath,
     useconds_t delay_usec = retry.min_delay * 1E6; // sec -> usec
     int retries = 0;
 
-    if (!sockpath || strlen (sockpath) == 0 || retry.max_retry < 0
-                  || retry.min_delay < 0 || retry.max_delay < 0) {
+    if (!sockpath
+        || strlen (sockpath) == 0
+        || retry.max_retry < 0
+        || retry.min_delay < 0
+        || retry.max_delay < 0) {
         errno = EINVAL;
         return -1;
     }
+#ifdef SOCK_CLOEXEC
     if ((fd = socket (AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0)) < 0)
         return -1;
+#else
+    if ((fd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0)
+        return -1;
+    if (fd_set_cloexec (fd) < 0)
+        goto error;
+#endif
 
     memset (&addr, 0, sizeof (addr));
     addr.sun_family = AF_UNIX;
diff --git a/src/common/librouter/usock.h b/src/common/librouter/usock.h
index a408cf19064b..5fc0a7271e4d 100644
--- a/src/common/librouter/usock.h
+++ b/src/common/librouter/usock.h
@@ -38,6 +38,8 @@ struct usock_retry_params {
 
 typedef void (*usock_acceptor_f)(struct usock_conn *conn, void *arg);
 
+typedef void (*usock_conn_close_f)(struct usock_conn *conn,
+                                   void *arg);
 typedef void (*usock_conn_error_f)(struct usock_conn *conn,
                                    int errnum,
                                    void *arg);
@@ -66,6 +68,10 @@ const struct flux_msg_cred *usock_conn_get_cred (struct usock_conn *conn);
 
 const char *usock_conn_get_uuid (struct usock_conn *conn);
 
+void usock_conn_set_close_cb (struct usock_conn *conn,
+                              usock_conn_close_f cb,
+                              void *arg);
+
 void usock_conn_set_error_cb (struct usock_conn *conn,
                               usock_conn_error_f cb,
                               void *arg);
diff --git a/src/common/librouter/usock_service.c b/src/common/librouter/usock_service.c
new file mode 100644
index 000000000000..bc8d8a3e2c55
--- /dev/null
+++ b/src/common/librouter/usock_service.c
@@ -0,0 +1,236 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <sys/types.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/errno_safe.h"
+#include "usock.h"
+
+#include "usock_service.h"
+
+static void service_destroy (void *impl);
+
+static const struct flux_handle_ops service_handle_ops;
+
+struct service {
+    bool verbose;
+    struct usock_server *usock_srv;
+    struct flux_msg_cred cred;
+    zhashx_t *connections; // uconn by uuid
+    flux_t *h;
+};
+
+/* zhashx_destructor_fn signature */
+static void connection_destructor (void **item)
+{
+    if (item) {
+        usock_conn_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static void notify_disconnect (struct service *ss, const char *uuid)
+{
+    flux_msg_t *msg;
+
+    /* flux_msg_route_enable() returns void.  To avoid creating an
+     * extra branch, call with C trick to avoid breaking up single if
+     * statement into multiple.
+     */
+    if (!(msg = flux_request_encode ("disconnect", NULL))
+        || flux_msg_set_noresponse (msg) < 0
+        || (flux_msg_route_enable (msg), false)
+        || flux_msg_set_cred (msg, ss->cred) < 0
+        || flux_msg_route_push (msg, uuid) < 0
+        || flux_requeue (ss->h, msg, FLUX_RQ_TAIL) < 0) {
+        if (ss->verbose)
+            log_msg ("error notifying server of %.5s disconnect", uuid);
+    }
+    flux_msg_decref (msg);
+}
+
+/* usock_conn_error_f signature */
+static void service_error (struct usock_conn *uconn, int errnum, void *arg)
+{
+    struct service *ss = arg;
+    const char *uuid = usock_conn_get_uuid (uconn);
+
+    if (ss->verbose) {
+        if (errnum != EPIPE && errnum != EPROTO && errnum != ECONNRESET) {
+            const struct flux_msg_cred *cred = usock_conn_get_cred (uconn);
+            log_errn (errnum, "client=%.5s userid=%u",
+                     uuid,
+                     (unsigned int)cred->userid);
+        }
+        log_msg ("bye %.5s", uuid);
+    }
+    notify_disconnect (ss, uuid); // notify server of disconnect
+    zhashx_delete (ss->connections, uuid);
+}
+
+/* usock_conn_recv_f signature */
+static void service_recv (struct usock_conn *uconn, flux_msg_t *msg, void *arg)
+{
+    const char *uuid = usock_conn_get_uuid (uconn);
+    struct service *ss = arg;
+    int type = 0;
+
+    /* flux_msg_route_enable() returns void.  To avoid creating an
+     * extra branch, call with C trick to avoid breaking up single if
+     * statement into multiple.
+     */
+    if (flux_msg_get_type (msg, &type) < 0
+        || type != FLUX_MSGTYPE_REQUEST
+        || (flux_msg_route_enable (msg), false)
+        || flux_msg_set_cred (msg, ss->cred) < 0
+        || flux_msg_route_push (msg, uuid) < 0
+        || flux_requeue (ss->h, msg, FLUX_RQ_TAIL) < 0)
+        goto drop;
+    return;
+drop:
+    if (ss->verbose)
+        log_msg ("drop %s from %.5s", flux_msg_typestr (type), uuid);
+}
+
+/* usock_acceptor_f signature */
+static void service_acceptor (struct usock_conn *uconn, void *arg)
+{
+    struct service *ss = arg;
+    const struct flux_msg_cred *cred = usock_conn_get_cred (uconn);
+    const char *uuid = usock_conn_get_uuid (uconn);
+
+    if (cred->userid != ss->cred.userid) {
+        errno = EPERM;
+        goto error;
+    }
+    if (zhashx_insert (ss->connections, uuid, uconn) < 0) {
+        errno = EEXIST;
+        goto error;
+    }
+    if (ss->verbose)
+        log_msg ("hi %.5s", uuid);
+    usock_conn_set_error_cb (uconn, service_error, ss);
+    usock_conn_set_recv_cb (uconn, service_recv, ss);
+    usock_conn_accept (uconn, cred);
+    return;
+error:
+    usock_conn_reject (uconn, errno);
+    usock_conn_destroy (uconn);
+}
+
+/* flux_handle_ops send signature */
+static int service_handle_send (void *impl, const flux_msg_t *msg, int flags)
+{
+    struct service *ss = impl;
+    int type = 0;
+    flux_msg_t *cpy = NULL;
+    const char *uuid;
+    struct usock_conn *uconn;
+
+    if (flux_msg_get_type (msg, &type) < 0)
+        return -1;
+    if (type != FLUX_MSGTYPE_RESPONSE) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(cpy = flux_msg_copy (msg, true)))
+        return -1;
+    if (!(uuid = flux_msg_route_last (cpy))) {
+        errno = EPROTO;
+        goto error;
+    }
+    if (flux_msg_set_cred (cpy, ss->cred) < 0)
+        goto error;
+    if (!(uconn = zhashx_lookup (ss->connections, uuid))) {
+        errno = ENOENT;
+        goto error;
+    }
+    if (flux_msg_route_delete_last (cpy) < 0)
+        goto error;
+    if (usock_conn_send (uconn, cpy) < 0)
+        goto error;
+    flux_msg_decref (cpy);
+    return 0;
+error:
+    flux_msg_decref (cpy);
+    return -1;
+}
+
+static struct service *service_create  (flux_reactor_t *r,
+                                        const char *sockpath,
+                                        bool verbose)
+{
+    struct service *ss;
+
+    if (!(ss = calloc (1, sizeof (*ss))))
+        return NULL;
+    ss->verbose = verbose;
+    if (!(ss->usock_srv = usock_server_create (r, sockpath, 0777)))
+        goto error;
+    usock_server_set_acceptor (ss->usock_srv, service_acceptor, ss);
+    if (!(ss->connections = zhashx_new ()))
+        goto error;
+    zhashx_set_key_duplicator (ss->connections, NULL);
+    zhashx_set_key_destructor (ss->connections, NULL);
+    zhashx_set_destructor (ss->connections, connection_destructor);
+    ss->cred.userid = getuid ();
+    ss->cred.rolemask = FLUX_ROLE_OWNER;
+    return ss;
+error:
+    service_destroy (ss);
+    return NULL;
+}
+
+/* flux_handle_ops impl_destroy signature */
+static void service_destroy (void *impl)
+{
+    struct service *ss = impl;
+    if (ss) {
+        int saved_errno = errno;
+        zhashx_destroy (&ss->connections);
+        usock_server_destroy (ss->usock_srv);
+        free (ss);
+        errno = saved_errno;
+    }
+}
+
+flux_t *usock_service_create (flux_reactor_t *r,
+                              const char *rundir,
+                              bool verbose)
+{
+    struct service *ss;
+
+    if (!(ss = service_create (r, rundir, verbose)))
+        return NULL;
+    if (!(ss->h = flux_handle_create (ss, &service_handle_ops, 0))
+            || flux_set_reactor (ss->h, r) < 0) {
+        service_destroy (ss);
+        return NULL;
+    }
+    return ss->h;
+}
+
+static const struct flux_handle_ops service_handle_ops = {
+    .send = service_handle_send,
+    .impl_destroy = service_destroy,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/librouter/usock_service.h b/src/common/librouter/usock_service.h
new file mode 100644
index 000000000000..375992013458
--- /dev/null
+++ b/src/common/librouter/usock_service.h
@@ -0,0 +1,41 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _ROUTER_USOCK_SERVICE_H
+#define _ROUTER_USOCK_SERVICE_H
+
+/* Create a flux_t handle representing a usock socket on 'sockpath'.
+ * This can be used to embed a faux flux service in a program.
+ * The server end can register message handlers as usual.
+ * The client end can flux_open() local://${sockpath} and make RPCs.
+ *
+ * Limitations:
+ * - connections from "guests" (uid != server uid) are rejected
+ * - event messages may not be published or subscribed to
+ * - clients may not register services
+ * - rank addressing is ignored
+ * - server flux_t handle requires async reactor operation
+ *   (one cannot call flux_recv() in a loop and expect it to make progress)
+ *
+ * When a client disconnects, a request is automatically sent to the server
+ * from the client's uuid.  This is similar to RFC 6 disconnects, except
+ * the topic string is always "disconnect", not "<service>.disconnect".
+ *
+ * The handle must be closed with flux_close().
+ */
+flux_t *usock_service_create (flux_reactor_t *r,
+                              const char *sockpath,
+                              bool verbose);
+
+#endif // _ROUTER_USOCK_SERVICE_H
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libschedutil/Makefile.am b/src/common/libschedutil/Makefile.am
index 66809cc8170b..1486b939136f 100644
--- a/src/common/libschedutil/Makefile.am
+++ b/src/common/libschedutil/Makefile.am
@@ -6,10 +6,12 @@ AM_LDFLAGS = \
         $(CODE_COVERAGE_LDFLAGS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
+	$(JANSSON_CFLAGS)
 
 noinst_LTLIBRARIES = \
 	libschedutil.la
@@ -35,6 +37,3 @@ libschedutil_la_SOURCES = \
 	alloc.c \
 	free.h \
 	free.c
-
-libschedutil_la_LIBADD = \
-	$(ZMQ_LIBS)
diff --git a/src/common/libschedutil/alloc.c b/src/common/libschedutil/alloc.c
index 76f2c0fcb0e1..e540bb90011e 100644
--- a/src/common/libschedutil/alloc.c
+++ b/src/common/libschedutil/alloc.c
@@ -12,65 +12,106 @@
 #include "config.h"
 #endif
 #include <flux/core.h>
+#include <jansson.h>
 
+#include "src/common/libutil/errno_safe.h"
 #include "schedutil_private.h"
 #include "init.h"
 #include "alloc.h"
 
-int schedutil_alloc_request_decode (const flux_msg_t *msg,
-                                    flux_jobid_t *id,
-                                    int *priority,
-                                    uint32_t *userid,
-                                    double *t_submit)
+static int schedutil_alloc_respond_pack (flux_t *h,
+                                         const flux_msg_t *msg,
+                                         int type,
+                                         const char *fmt,
+                                         ...)
 {
-    return flux_request_unpack (msg, NULL, "{s:I s:i s:i s:f}",
-                                           "id", id,
-                                           "priority", priority,
-                                           "userid", userid,
-                                           "t_submit", t_submit);
+    flux_jobid_t id;
+    va_list ap;
+    json_t *payload;
+
+    if (flux_request_unpack (msg, NULL, "{s:I}", "id", &id) < 0)
+        return -1;
+    if (!(payload = json_pack ("{s:I s:i}",
+                               "id", id,
+                               "type", type)))
+        goto nomem;
+    if (fmt) {
+        json_t *o;
+        va_start (ap, fmt);
+        o = json_vpack_ex (NULL, 0, fmt, ap);
+        va_end (ap);
+        if (!o || json_object_update (payload, o) < 0) {
+            json_decref (o);
+            goto nomem;
+        }
+        json_decref (o);
+    }
+    if (flux_respond_pack (h, msg, "O", payload) < 0)
+        goto error;
+    json_decref (payload);
+    return 0;
+nomem:
+    errno = ENOMEM;
+error:
+    ERRNO_SAFE_WRAP (json_decref, payload);
+    return -1;
 }
 
-static int schedutil_alloc_respond (flux_t *h, const flux_msg_t *msg,
-                                    int type, const char *note)
+int schedutil_alloc_respond_annotate_pack (schedutil_t *util,
+                                           const flux_msg_t *msg,
+                                           const char *fmt,
+                                           ...)
 {
-    flux_jobid_t id;
+    va_list ap;
+    json_t *annotations;
     int rc;
 
-    if (flux_request_unpack (msg, NULL, "{s:I}", "id", &id) < 0)
+    va_start (ap, fmt);
+    annotations = json_vpack_ex (NULL, 0, fmt, ap);
+    va_end (ap);
+    if (!annotations) {
+        errno = EINVAL;
         return -1;
-    if (note)
-        rc = flux_respond_pack (h, msg, "{s:I s:i s:s}",
-                                        "id", id,
-                                        "type", type,
-                                        "note", note);
-    else
-        rc = flux_respond_pack (h, msg, "{s:I s:i}",
-                                        "id", id,
-                                        "type", type);
+    }
+    rc = schedutil_alloc_respond_pack (util->h,
+                                       msg,
+                                       FLUX_SCHED_ALLOC_ANNOTATE,
+                                       "{s:O}",
+                                       "annotations", annotations);
+    ERRNO_SAFE_WRAP (json_decref, annotations);
     return rc;
 }
 
-int schedutil_alloc_respond_note (schedutil_t *util, const flux_msg_t *msg,
+int schedutil_alloc_respond_deny (schedutil_t *util,
+                                  const flux_msg_t *msg,
                                   const char *note)
 {
-    return schedutil_alloc_respond (util->h, msg, 1, note);
-}
-
-int schedutil_alloc_respond_denied (schedutil_t *util, const flux_msg_t *msg,
-                                    const char *note)
-{
-    return schedutil_alloc_respond (util->h, msg, 2, note);
+    if (note) {
+        return schedutil_alloc_respond_pack (util->h,
+                                             msg,
+                                             FLUX_SCHED_ALLOC_DENY,
+                                             "{s:s}",
+                                             "note", note);
+    }
+    return schedutil_alloc_respond_pack (util->h,
+                                         msg,
+                                         FLUX_SCHED_ALLOC_DENY,
+                                         NULL);
 }
 
 int schedutil_alloc_respond_cancel (schedutil_t *util, const flux_msg_t *msg)
 {
-    return schedutil_alloc_respond (util->h, msg, 3, NULL);
+    return schedutil_alloc_respond_pack (util->h,
+                                         msg,
+                                         FLUX_SCHED_ALLOC_CANCEL,
+                                         NULL);
 }
 
 struct alloc {
-    char *note;
+    json_t *annotations;
     const flux_msg_t *msg;
     flux_kvs_txn_t *txn;
+    json_t *R;
 };
 
 static void alloc_destroy (struct alloc *ctx)
@@ -79,14 +120,17 @@ static void alloc_destroy (struct alloc *ctx)
         int saved_errno = errno;
         flux_kvs_txn_destroy (ctx->txn);
         flux_msg_decref (ctx->msg);
-        free (ctx->note);
+        json_decref (ctx->annotations);
+        json_decref (ctx->R);
         free (ctx);
         errno = saved_errno;
     }
 }
 
-static struct alloc *alloc_create (const flux_msg_t *msg, const char *R,
-                                   const char *note)
+static struct alloc *alloc_create (const flux_msg_t *msg,
+                                   const char *R,
+                                   const char *fmt,
+                                   va_list ap)
 {
     struct alloc *ctx;
     flux_jobid_t id;
@@ -101,8 +145,14 @@ static struct alloc *alloc_create (const flux_msg_t *msg, const char *R,
     if (!(ctx = calloc (1, sizeof (*ctx))))
         return NULL;
     ctx->msg = flux_msg_incref (msg);
-    if (note && !(ctx->note = strdup (note)))
+    if (fmt) {
+        if (!(ctx->annotations = json_vpack_ex (NULL, 0, fmt, ap)))
+            goto error;
+    }
+    if (!(ctx->R = json_loads (R, 0, NULL))) {
+        errno = ENOMEM;
         goto error;
+    }
     if (!(ctx->txn = flux_kvs_txn_create ()))
         goto error;
     if (flux_kvs_txn_put (ctx->txn, 0, key, R) < 0)
@@ -118,49 +168,66 @@ static void alloc_continuation (flux_future_t *f, void *arg)
     schedutil_t *util = arg;
     flux_t *h = util->h;
     struct alloc *ctx = flux_future_aux_get (f, "flux::alloc_ctx");
+    json_t *payload = NULL;
 
+    remove_outstanding_future (util, f);
     if (flux_future_get (f, NULL) < 0) {
         flux_log_error (h, "commit R");
         goto error;
     }
-    schedutil_remove_outstanding_future (util, f);
-    if (schedutil_alloc_respond (h, ctx->msg, 0, ctx->note) < 0) {
-        flux_log_error (h, "alloc response");
+    if (!(payload = json_object ())
+        || (ctx->annotations && json_object_set (payload,
+                                                 "annotations",
+                                                 ctx->annotations) < 0)
+        || json_object_set (payload, "R", ctx->R) < 0) {
+        errno = ENOMEM;
+        flux_log_error (h, "error responding to alloc request");
+        goto error;
+    }
+    if (schedutil_alloc_respond_pack (h,
+                                      ctx->msg,
+                                      FLUX_SCHED_ALLOC_SUCCESS,
+                                      "O",
+                                      payload) < 0) {
+        flux_log_error (h, "error responding to alloc request");
         goto error;
     }
+    json_decref (payload);
     flux_future_destroy (f);
     return;
 error:
     flux_reactor_stop_error (flux_get_reactor (h)); // XXX
-    alloc_destroy (ctx);
+    ERRNO_SAFE_WRAP (json_decref, payload);
     flux_future_destroy (f);
 }
 
-int schedutil_alloc_respond_R (schedutil_t *util, const flux_msg_t *msg,
-                               const char *R, const char *note)
+int schedutil_alloc_respond_success_pack (schedutil_t *util,
+                                          const flux_msg_t *msg,
+                                          const char *R,
+                                          const char *fmt,
+                                          ...)
 {
     struct alloc *ctx;
     flux_future_t *f;
     flux_t *h = util->h;
+    va_list ap;
 
-    if (!(ctx = alloc_create (msg, R, note)))
+    va_start (ap, fmt);
+    ctx = alloc_create (msg, R, fmt, ap);
+    va_end (ap);
+    if (!ctx)
         return -1;
     if (!(f = flux_kvs_commit (h, NULL, 0, ctx->txn)))
         goto error;
-    if (flux_future_aux_set (f, "flux::alloc_ctx",
-                             ctx, (flux_free_f)alloc_destroy) < 0) {
+    if (flux_future_aux_set (f,
+                             "flux::alloc_ctx",
+                             ctx,
+                             (flux_free_f)alloc_destroy) < 0) {
         goto error;
     }
-    if (flux_future_then (f, -1, alloc_continuation, ctx) < 0)
+    if (flux_future_then (f, -1, alloc_continuation, util) < 0)
         goto error;
-    if (!schedutil_hang_responses (util)) {
-        if (flux_future_then (f, -1, alloc_continuation, util) < 0)
-            goto error;
-    }
-    /* else: intentionally do not register a continuation to force
-     * a permanent outstanding request for testing
-     */
-    schedutil_add_outstanding_future (util, f);
+    add_outstanding_future (util, f);
     return 0;
 error:
     alloc_destroy (ctx);
diff --git a/src/common/libschedutil/alloc.h b/src/common/libschedutil/alloc.h
index 879fdf5b89ea..745b32740847 100644
--- a/src/common/libschedutil/alloc.h
+++ b/src/common/libschedutil/alloc.h
@@ -16,42 +16,53 @@
 
 #include "init.h"
 
-/* Decode an alloc request message.
- * Return 0 on success, -1 on error with errno set.
- */
-int schedutil_alloc_request_decode (const flux_msg_t *msg,
-                                    flux_jobid_t *id,
-                                    int *priority,
-                                    uint32_t *userid,
-                                    double *t_submit);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+    FLUX_SCHED_ALLOC_SUCCESS  = 0,
+    FLUX_SCHED_ALLOC_ANNOTATE = 1,
+    FLUX_SCHED_ALLOC_DENY     = 2,
+    FLUX_SCHED_ALLOC_CANCEL   = 3,
+};
 
 /* Respond to alloc request message - update annotation.
  * A job's annotation may be updated any number of times before alloc request
- * is finally terminated with alloc_respond_denied() or alloc_respond_R().
+ * is finally terminated with alloc_respond_deny() or alloc_respond_success().
  * Return 0 on success, -1 on error with errno set.
  */
-int schedutil_alloc_respond_note (schedutil_t *util, const flux_msg_t *msg,
-                                  const char *note);
+int schedutil_alloc_respond_annotate_pack (schedutil_t *util,
+                                           const flux_msg_t *msg,
+                                           const char *fmt, ...);
 
 /* Respond to alloc request message - the job cannot run.
  * Include human readable error message in 'note'.
  * Return 0 on success, -1 on error with errno set.
  */
-int schedutil_alloc_respond_denied (schedutil_t *util, const flux_msg_t *msg,
-                                    const char *note);
+int schedutil_alloc_respond_deny (schedutil_t *util,
+                                  const flux_msg_t *msg,
+                                  const char *note);
 
-/* Respond to alloc request message - allocate R.
+/* Respond to alloc request message - success, allocate R.
  * R is committed to the KVS first, then the response is sent.
  * If something goes wrong after this function returns, the reactor is stopped.
  */
-int schedutil_alloc_respond_R (schedutil_t *util, const flux_msg_t *msg,
-                               const char *R, const char *note);
+int schedutil_alloc_respond_success_pack (schedutil_t *util,
+                                          const flux_msg_t *msg,
+                                          const char *R,
+                                          const char *fmt,
+                                          ...);
 
 /* Respond to an alloc request message - canceled.
  * N.B. 'msg' is the alloc request, not the cancel request.
  */
 int schedutil_alloc_respond_cancel (schedutil_t *util, const flux_msg_t *msg);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* !_FLUX_SCHEDUTIL_ALLOC_H */
 
 /*
diff --git a/src/common/libschedutil/free.c b/src/common/libschedutil/free.c
index 01e17e8a54b5..07bd89edd5a6 100644
--- a/src/common/libschedutil/free.c
+++ b/src/common/libschedutil/free.c
@@ -17,19 +17,9 @@
 #include "init.h"
 #include "free.h"
 
-
-int schedutil_free_request_decode (const flux_msg_t *msg, flux_jobid_t *id)
-{
-    return flux_request_unpack (msg, NULL, "{s:I}", "id", id);
-}
-
 int schedutil_free_respond (schedutil_t *util, const flux_msg_t *msg)
 {
-    flux_jobid_t id;
-
-    if (flux_request_unpack (msg, NULL, "{s:I}", "id", &id) < 0)
-        return -1;
-    return flux_respond_pack (util->h, msg, "{s:I}", "id", id);
+    return 0;
 }
 
 /*
diff --git a/src/common/libschedutil/free.h b/src/common/libschedutil/free.h
index 142946f97e66..3d015a927c53 100644
--- a/src/common/libschedutil/free.h
+++ b/src/common/libschedutil/free.h
@@ -15,15 +15,18 @@
 
 #include "init.h"
 
-/* Decode a free request.
- * Returns 0 on success, -1 on failure with errno set.
- */
-int schedutil_free_request_decode (const flux_msg_t *msg, flux_jobid_t *id);
+#ifdef __cplusplus
+extern "C" {
+#endif
 
-/* Respond to a free request.
+/* This is a no-op now that sched.free doesn't require a response.
  */
 int schedutil_free_respond (schedutil_t *util, const flux_msg_t *msg);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* !_FLUX_SCHEDUTIL_FREE_H */
 
 /*
diff --git a/src/common/libschedutil/hello.c b/src/common/libschedutil/hello.c
index 22796714b5c9..ca6460a4afa2 100644
--- a/src/common/libschedutil/hello.c
+++ b/src/common/libschedutil/hello.c
@@ -14,90 +14,96 @@
 #include <flux/core.h>
 #include <jansson.h>
 
+#include "src/common/libjob/idf58.h"
 #include "schedutil_private.h"
 #include "init.h"
 #include "hello.h"
 
-static int schedutil_hello_job (flux_t *h,
-                                flux_jobid_t id,
-                                int priority,
-                                uint32_t userid,
-                                double t_submit,
-                                schedutil_hello_cb_f *cb,
-                                void *arg)
+
+static void raise_exception (flux_t *h, flux_jobid_t id, const char *note)
 {
-    char key[64];
     flux_future_t *f;
-    const char *R;
 
-    if (!h) {
-        errno = EINVAL;
-        return -1;
+    flux_log (h,
+              LOG_INFO,
+              "raising fatal exception on running job id=%s",
+              idf58 (id));
+
+    if (!(f = flux_job_raise (h, id, "scheduler-restart", 0, note))
+        || flux_future_get (f, NULL) < 0) {
+        flux_log_error (h,
+                        "error raising fatal exception on %s: %s",
+                        idf58 (id),
+                        future_strerror (f, errno));
     }
+    flux_future_destroy (f);
+}
+
+static int schedutil_hello_job (schedutil_t *util,
+                                const flux_msg_t *msg)
+{
+    char key[64];
+    flux_future_t *f = NULL;
+    const char *R;
+    flux_jobid_t id;
 
+    if (flux_msg_unpack (msg, "{s:I}", "id", &id) < 0)
+        goto error;
     if (flux_job_kvs_key (key, sizeof (key), id, "R") < 0) {
         errno = EPROTO;
-        return -1;
+        goto error;
     }
-    if (!(f = flux_kvs_lookup (h, NULL, 0, key)))
-        return -1;
-    if (flux_kvs_lookup_get (f, &R) < 0)
+    if (!(f = flux_kvs_lookup (util->h, NULL, 0, key)))
         goto error;
-    if (cb (h, id, priority, userid, t_submit, R, arg) < 0)
+    if (flux_kvs_lookup_get (f, &R) < 0)
         goto error;
+    if (util->ops->hello (util->h,
+                          msg,
+                          R,
+                          util->cb_arg) < 0)
+        raise_exception (util->h,
+                         id,
+                         "failed to reallocate R for running job");
     flux_future_destroy (f);
     return 0;
 error:
-    flux_log_error (h, "hello: error loading R for id=%llu",
-                    (unsigned long long)id);
+    flux_log_error (util->h,
+                    "hello: error loading R for id=%s",
+                    idf58 (id));
     flux_future_destroy (f);
     return -1;
 }
 
-int schedutil_hello (schedutil_t *util, schedutil_hello_cb_f *cb, void *arg)
+int schedutil_hello (schedutil_t *util)
 {
     flux_future_t *f;
-    json_t *jobs;
-    json_t *entry;
-    size_t index;
+    int rc = -1;
 
-    if (!util || !cb) {
+    if (!util || !util->ops->hello) {
         errno = EINVAL;
         return -1;
     }
-    if (!(f = flux_rpc (util->h, "job-manager.sched-hello",
-                        NULL, FLUX_NODEID_ANY, 0)))
+    if (!(f = flux_rpc (util->h,
+                        "job-manager.sched-hello",
+                        NULL,
+                        FLUX_NODEID_ANY,
+                        FLUX_RPC_STREAMING)))
         return -1;
-    if (flux_rpc_get_unpack (f, "{s:o}", "alloc", &jobs) < 0)
-        goto error;
-    json_array_foreach (jobs, index, entry) {
-        flux_jobid_t id;
-        int priority;
-        uint32_t userid;
-        double t_submit;
-
-        if (json_unpack (entry, "{s:I s:i s:i s:f}",
-                                "id", &id,
-                                "priority", &priority,
-                                "userid", &userid,
-                                "t_submit", &t_submit) < 0) {
-            errno = EPROTO;
+    while (1) {
+        const flux_msg_t *msg;
+        if (flux_future_get (f, (const void **)&msg) < 0) {
+            if (errno == ENODATA)
+                break;
             goto error;
         }
-        if (schedutil_hello_job (util->h,
-                                 id,
-                                 priority,
-                                 userid,
-                                 t_submit,
-                                 cb,
-                                 arg) < 0)
+        if (schedutil_hello_job (util, msg) < 0)
             goto error;
+        flux_future_reset (f);
     }
-    flux_future_destroy (f);
-    return 0;
+    rc = 0;
 error:
     flux_future_destroy (f);
-    return -1;
+    return rc;
 }
 
 /*
diff --git a/src/common/libschedutil/hello.h b/src/common/libschedutil/hello.h
index 7c810b2cc063..3aecf2135264 100644
--- a/src/common/libschedutil/hello.h
+++ b/src/common/libschedutil/hello.h
@@ -15,25 +15,20 @@
 
 #include "init.h"
 
-/* Callback for ingesting R + metadata for jobs that have resources
- * Return 0 on success, -1 on failure with errno set.
- * Failure of the callback aborts iteration and causes schedutil_hello()
- * to return -1 with errno passed through.
- */
-typedef int (schedutil_hello_cb_f)(flux_t *h,
-                                   flux_jobid_t id,
-                                   int priority,
-                                   uint32_t userid,
-                                   double t_submit,
-                                   const char *R,
-                                   void *arg);
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 /* Send hello announcement to job-manager.
  * The job-manager responds with a list of jobs that have resources assigned.
- * This function looks up R for each job and passes R + metadata to 'cb'
- * with 'arg'.
+ * This function looks up R for each job and passes R + metadata to
+ * ops->hello callback.
  */
-int schedutil_hello (schedutil_t *util, schedutil_hello_cb_f *cb, void *arg);
+int schedutil_hello (schedutil_t *util);
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* !_FLUX_SCHEDUTIL_HELLO_H */
 
diff --git a/src/common/libschedutil/init.c b/src/common/libschedutil/init.c
index 225696e51377..c78de0667485 100644
--- a/src/common/libschedutil/init.c
+++ b/src/common/libschedutil/init.c
@@ -8,49 +8,55 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include <czmq.h>
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/aux.h"
 
 #include "init.h"
 #include "schedutil_private.h"
 
-/* flux module debug --setbit 0x8000 sched
- * flux module debug --clearbit 0x8000 sched
- */
-enum module_debug_flags {
-    /* alloc and free responses received while this is set
-     * will never get a response
-     */
-    DEBUG_HANG_RESPONSES = 0x8000, // 16th bit
-};
+/* destroy future - zlistx_dsetructor_t footprint */
+static void future_destructor (void **item)
+{
+    if (*item) {
+        flux_future_destroy (*item);
+        *item = NULL;
+    }
+}
 
 
 schedutil_t *schedutil_create (flux_t *h,
-                               schedutil_alloc_cb_f *alloc_cb,
-                               schedutil_free_cb_f *free_cb,
-                               schedutil_cancel_cb_f *cancel_cb,
-                               void *cb_arg)
+                               int flags,
+                               const struct schedutil_ops *ops,
+                               void *arg)
 {
     schedutil_t *util;
 
-    if (!h || !alloc_cb || !free_cb || !cancel_cb) {
+    /* ops->prioritize is optional */
+    if (!h
+        || !ops
+        || !ops->alloc
+        || !ops->free
+        || !ops->cancel) {
         errno = EINVAL;
         return NULL;
     }
-    if (!(util = calloc(1, sizeof(*util))))
+    if (!(util = calloc (1, sizeof (*util))))
         return NULL;
 
     util->h = h;
-    util->alloc_cb = alloc_cb;
-    util->free_cb = free_cb;
-    util->cancel_cb = cancel_cb;
-    util->cb_arg = cb_arg;
-    if ((util->outstanding_futures = zlistx_new ()) == NULL)
+    util->ops = ops;
+    util->flags = flags;
+    util->cb_arg = arg;
+    if (!(util->outstanding_futures = zlistx_new ()))
         goto error;
-    if (schedutil_ops_register (util) < 0)
+    zlistx_set_destructor (util->outstanding_futures, future_destructor);
+    if (ops_register (util) < 0)
         goto error;
 
     return util;
@@ -60,58 +66,25 @@ schedutil_t *schedutil_create (flux_t *h,
     return NULL;
 }
 
-static void respond_to_outstanding_msgs (schedutil_t *util)
-{
-    int rc = 0;
-    flux_future_t *fut;
-    flux_msg_t *msg;
-    for (fut = zlistx_first (util->outstanding_futures);
-         fut;
-         fut = zlistx_next (util->outstanding_futures))
-        {
-            msg = flux_future_aux_get (fut, "schedutil::msg");
-            rc = flux_respond_error (util->h,
-                                     msg,
-                                     ENOSYS,
-                                     "automatic ENOSYS response "
-                                     "from schedutil");
-            if (rc != 0) {
-                flux_log (util->h,
-                          LOG_ERR,
-                          "schedutil: error in responding to "
-                          "outstanding messages");
-            }
-            flux_future_destroy (fut);
-        }
-    zlistx_purge (util->outstanding_futures);
-}
-
 void schedutil_destroy (schedutil_t *util)
 {
     if (util) {
         int saved_errno = errno;
-        respond_to_outstanding_msgs (util);
         zlistx_destroy (&util->outstanding_futures);
-        schedutil_ops_unregister (util);
+        ops_unregister (util);
         free (util);
         errno = saved_errno;
     }
-    return;
-}
-
-bool schedutil_hang_responses (const schedutil_t *util)
-{
-    return flux_module_debug_test (util->h, DEBUG_HANG_RESPONSES, false);
 }
 
-int schedutil_add_outstanding_future (schedutil_t *util, flux_future_t *fut)
+int add_outstanding_future (schedutil_t *util, flux_future_t *fut)
 {
     if (zlistx_add_end (util->outstanding_futures, fut) == NULL)
         return -1;
     return 0;
 }
 
-int schedutil_remove_outstanding_future (schedutil_t *util, flux_future_t *fut)
+int remove_outstanding_future (schedutil_t *util, flux_future_t *fut)
 {
     if (!zlistx_find (util->outstanding_futures, fut))
         return -1;
diff --git a/src/common/libschedutil/init.h b/src/common/libschedutil/init.h
index cef65c397e48..a0903a26c5e5 100644
--- a/src/common/libschedutil/init.h
+++ b/src/common/libschedutil/init.h
@@ -15,25 +15,36 @@
 
 #include "ops.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef struct schedutil_ctx schedutil_t;
 
-/* Create a handle for the schedutil conveinence library.
+enum schedutil_flags {
+    SCHEDUTIL_FREE_NOLOOKUP = 1, // now the default so this flag is ignored
+};
+
+/* Create a handle for the schedutil convenience library.
  *
  * Used to track outstanding futures and register callbacks relevant for
  * schedulers and simulators.
  * Return NULL on error.
  */
 schedutil_t *schedutil_create (flux_t *h,
-                               schedutil_alloc_cb_f *alloc_cb,
-                               schedutil_free_cb_f *free_cb,
-                               schedutil_cancel_cb_f *cancel_cb,
-                               void *cb_arg);
+                               int flags,
+                               const struct schedutil_ops *ops,
+                               void *arg);
 
-/* Destory the handle for the schedutil conveinence library.
+/* Destroy the handle for the schedutil convenience library.
  *
  * Will automatically respond ENOSYS to any outstanding messages (e.g., free,
  * alloc).
  */
 void schedutil_destroy (schedutil_t* ctx);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* !_FLUX_SCHEDUTIL_INIT_H */
diff --git a/src/common/libschedutil/ops.c b/src/common/libschedutil/ops.c
index 2a390217ae1c..0c02fa1c8d20 100644
--- a/src/common/libschedutil/ops.c
+++ b/src/common/libschedutil/ops.c
@@ -14,172 +14,67 @@
 #include <stdlib.h>
 #include <flux/core.h>
 #include <jansson.h>
+#include <assert.h>
 
 #include "src/common/libutil/errno_safe.h"
 #include "schedutil_private.h"
 #include "init.h"
 #include "ops.h"
 
-static void alloc_continuation (flux_future_t *f, void *arg)
+static void alloc_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     schedutil_t *util = arg;
-    flux_t *h = util->h;
-    const flux_msg_t *msg = flux_future_aux_get (f, "schedutil::msg");
 
-    if (util == NULL) {
-        errno = EINVAL;
-        goto error;
-    }
-    const char *jobspec;
+    assert (util);
 
-    if (flux_kvs_lookup_get (f, &jobspec) < 0) {
-        flux_log_error (h, "sched.alloc lookup R");
-        goto error;
-    }
-    if (schedutil_remove_outstanding_future (util, f) < 0)
-        flux_log_error (h, "sched.alloc unable to remove outstanding future");
-    util->alloc_cb (h, msg, jobspec, util->cb_arg);
-    flux_future_destroy (f);
-    return;
-error:
-    flux_log_error (h, "sched.alloc");
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "sched.alloc respond_error");
-    flux_future_destroy (f);
+    util->ops->alloc (h, msg, util->cb_arg);
 }
 
-static void alloc_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+static void cancel_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
 {
     schedutil_t *util = arg;
-    flux_jobid_t id;
-    char key[64];
-    flux_future_t *f;
 
-    if (util == NULL) {
-        errno = EINVAL;
-        goto error;
-    }
-
-    if (flux_request_unpack (msg, NULL, "{s:I}", "id", &id) < 0)
-        goto error;
-    if (flux_job_kvs_key (key, sizeof (key), id, "jobspec") < 0) {
-        errno = EPROTO;
-        goto error;
-    }
-    if (!(f = flux_kvs_lookup (h, NULL, 0, key)))
-        goto error;
-    if (flux_future_aux_set (f,
-                             "schedutil::msg",
-                             (void *)flux_msg_incref (msg),
-                             (flux_free_f)flux_msg_decref) < 0) {
-        flux_msg_decref (msg);
-        goto error_future;
-    }
-    if (!schedutil_hang_responses (util)) {
-        if (flux_future_then (f, -1, alloc_continuation, util) < 0)
-            goto error_future;
-    }
-    // else: intentionally do not register a continuation to force a permanent
-    // outstanding request for testing
-    if (schedutil_add_outstanding_future (util, f) < 0)
-        flux_log_error (h, "sched.alloc unable to add outstanding future");
+    assert (util);
 
-    return;
-error_future:
-    flux_future_destroy (f);
-error:
-    flux_log_error (h, "sched.alloc");
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "sched.alloc respond_error");
+    util->ops->cancel (h, msg, util->cb_arg);
 }
 
-static void cancel_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
+static void free_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
 {
     schedutil_t *util = arg;
-    flux_jobid_t id;
 
-    if (flux_request_unpack (msg, NULL, "{s:I}", "id", &id) < 0) {
-        flux_log_error (h, "sched.cancel");
-        return;
-    }
-    /* N.B. in case scheduler hasn't transitioned to new cancel_cb,
-     * call as though this were the old exception_cb, with type=cancel,
-     * severity=0.
-     */
-    util->cancel_cb (h, id, "cancel", 0, util->cb_arg);
-}
+    assert (util);
 
-static void free_continuation (flux_future_t *f, void *arg)
-{
-    schedutil_t *util = arg;
-    const flux_msg_t *msg = flux_future_aux_get (f, "schedutil::msg");
-    flux_t *h = util->h;
-    const char *R;
-
-    if (flux_kvs_lookup_get (f, &R) < 0) {
-        flux_log_error (h, "sched.free lookup R");
-        goto error;
-    }
-    if (schedutil_remove_outstanding_future (util, f) < 0)
-        flux_log_error (h, "sched.free unable to remove outstanding future");
-    util->free_cb (h, msg, R, util->cb_arg);
-    flux_future_destroy (f);
-    return;
-error:
-    flux_log_error (h, "sched.free");
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "sched.free respond_error");
-    flux_future_destroy (f);
+    util->ops->free (h, msg, NULL, util->cb_arg);
 }
 
-static void free_cb (flux_t *h, flux_msg_handler_t *mh,
-                     const flux_msg_t *msg, void *arg)
+static void prioritize_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
 {
     schedutil_t *util = arg;
-    flux_jobid_t id;
-    flux_future_t *f;
-    char key[64];
 
-    if (flux_request_unpack (msg, NULL, "{s:I}", "id", &id) < 0)
-        goto error;
-    if (flux_job_kvs_key (key, sizeof (key), id, "R") < 0) {
-        errno = EPROTO;
-        goto error;
-    }
-    if (!(f = flux_kvs_lookup (h, NULL, 0, key)))
-        goto error;
-    if (flux_future_aux_set (f,
-                             "schedutil::msg",
-                             (void *)flux_msg_incref (msg),
-                             (flux_free_f)flux_msg_decref) < 0) {
-        flux_msg_decref (msg);
-        goto error_future;
-    }
-    if (!schedutil_hang_responses (util)) {
-        if (flux_future_then (f, -1, free_continuation, util) < 0)
-            goto error_future;
-    }
-    /* else: intentionally do not register a continuation to force
-     * a permanent outstanding request for testing
-     */
-    if (schedutil_add_outstanding_future (util, f) < 0)
-        flux_log_error (h, "sched.free unable to add outstanding future");
-
-    return;
-error_future:
-    flux_future_destroy (f);
-error:
-    flux_log_error (h, "sched.free");
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "sched.free respond_error");
+    assert (util);
+
+    if (util->ops->prioritize)
+        util->ops->prioritize (h, msg, util->cb_arg);
 }
 
 static const struct flux_msg_handler_spec htab[] = {
     { FLUX_MSGTYPE_REQUEST,  "sched.alloc", alloc_cb, 0},
     { FLUX_MSGTYPE_REQUEST,  "sched.cancel", cancel_cb, 0},
     { FLUX_MSGTYPE_REQUEST,  "sched.free", free_cb, 0},
+    { FLUX_MSGTYPE_REQUEST,  "sched.prioritize", prioritize_cb, 0},
     FLUX_MSGHANDLER_TABLE_END,
 };
 
@@ -200,26 +95,7 @@ static int service_register (flux_t *h)
     return 0;
 }
 
-/* Unregister dynamic service name 'sched'
- */
-static void service_unregister (flux_t *h)
-{
-    flux_future_t *f;
-
-    if (!(f = flux_service_unregister (h, "sched"))) {
-        flux_log_error (h, "service_unregister");
-        return;
-    }
-    if (flux_future_get (f, NULL) < 0) {
-        flux_log_error (h, "service_unregister");
-        flux_future_destroy (f);
-        return;
-    }
-    flux_log (h, LOG_DEBUG, "service_unregister");
-    flux_future_destroy (f);
-}
-
-int schedutil_ops_register (schedutil_t *util)
+int ops_register (schedutil_t *util)
 {
     flux_t *h = util->h;
 
@@ -227,22 +103,17 @@ int schedutil_ops_register (schedutil_t *util)
         errno = EINVAL;
         return -1;
     }
-    if (service_register (h) < 0)
-        goto error;
-    if (flux_msg_handler_addvec (h, htab, util, &util->handlers) < 0)
-        goto error;
+    if (service_register (h) < 0
+        || flux_msg_handler_addvec (h, htab, util, &util->handlers) < 0)
+        return -1;
     return 0;
-error:
-    schedutil_ops_unregister (util);
-    return -1;
 }
 
-void schedutil_ops_unregister (schedutil_t *util)
+void ops_unregister (schedutil_t *util)
 {
     if (!util)
         return;
 
-    service_unregister (util->h);
     flux_msg_handler_delvec (util->handlers);
  }
 
diff --git a/src/common/libschedutil/ops.h b/src/common/libschedutil/ops.h
index 462bd10686b7..e336cb72d9b3 100644
--- a/src/common/libschedutil/ops.h
+++ b/src/common/libschedutil/ops.h
@@ -13,39 +13,65 @@
 
 #include <flux/core.h>
 
-/* Callback for an alloc request.  jobspec is looked up as a convenience.
- * Decode msg with schedutil_alloc_request_decode().
- * 'msg' and 'jobspec' are only valid for hte duration of this call.
- * You should either respond to the request immediately (see alloc.h),
- * or cache this information for later response.
- */
-typedef void (schedutil_alloc_cb_f)(flux_t *h,
-                                    const flux_msg_t *msg,
-                                    const char *jobspec,
-                                    void *arg);
-
-/* Callback for a free request.  R is looked up as a convenience.
- * Decode msg with schedutil_free_request_decode().
- * 'msg' and 'R' are only valid for the duration of this call.
- * You should either respond to the request immediately (see free.h),
- * or cache this information for later response.
- */
-typedef void (schedutil_free_cb_f)(flux_t *h,
-                                   const flux_msg_t *msg,
-                                   const char *R,
-                                   void *arg);
-
-/* The job manager wants to cancel a pending alloc request.
- * The scheduler should look up the job in its queue.  If not found, do nothing.
- * If found, call schedutil_alloc_respond_cancel() and dequeue.
- * N.B. same calling footprint as former exception callback to facilitate
- * transition.
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* In the following callbacks, 'msg' is a request or response message from
+ * the job manager with payload defined by RFC 27.  The message's reference
+ * count is decremented when the callback returns.
  */
-typedef void (schedutil_cancel_cb_f)(flux_t *h,
-                                     flux_jobid_t id,
-                                     const char *unused_arg1,
-                                     int unused_arg2,
-                                     void *arg);
+struct schedutil_ops {
+    /* Callback for ingesting R + metadata for jobs that have resources
+     * Return 0 on success, -1 on failure with errno set.
+     */
+    int (*hello)(flux_t *h,
+                 const flux_msg_t *msg,
+                 const char *R,
+                 void *arg);
+
+    /* Callback for an alloc request.  'msg' is only valid for the
+     * duration of this call.  You should either respond to the
+     * request immediately (see alloc.h), or cache this information
+     * for later response.
+     */
+    void (*alloc)(flux_t *h,
+                  const flux_msg_t *msg,
+                  void *arg);
+
+    /* Callback for a free request.
+     * Currently 'R' is always NULL and may be unpacked as a JSON object
+     * under the "R" key in 'msg' if needed.
+     * 'msg' is only valid for the duration of this call.
+     * You should either respond to the request immediately (see
+     * free.h), or cache this information for later response.
+     */
+    void (*free)(flux_t *h,
+                 const flux_msg_t *msg,
+                 const char *R,
+                 void *arg);
+
+    /* The job manager wants to cancel a pending alloc request.  The
+     * scheduler should look up the job in its queue.  If not found,
+     * do nothing.  If found, call schedutil_alloc_respond_cancel()
+     * and dequeue.
+     */
+    void (*cancel)(flux_t *h,
+                   const flux_msg_t *msg,
+                   void *arg);
+
+    /* The job manager wants to change the priority one or more jobs.
+     * This callback is not required and can be set to NULL if the
+     * caller wishes to ignore these messages.
+     */
+    void (*prioritize)(flux_t *h,
+                       const flux_msg_t *msg,
+                       void *arg);
+};
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* !_FLUX_SCHEDUTIL_OPS_H */
 
diff --git a/src/common/libschedutil/ready.c b/src/common/libschedutil/ready.c
index 8f3d8d1eb90d..cf0b93915d8b 100644
--- a/src/common/libschedutil/ready.c
+++ b/src/common/libschedutil/ready.c
@@ -14,6 +14,8 @@
 #include <flux/core.h>
 #include <jansson.h>
 
+#include "ccan/str/str.h"
+
 #include "schedutil_private.h"
 #include "init.h"
 #include "ready.h"
@@ -21,16 +23,46 @@
 int schedutil_ready (schedutil_t *util, const char *mode, int *queue_depth)
 {
     flux_future_t *f;
+    int limit = 0;
     int count;
 
     if (!util || !mode) {
         errno = EINVAL;
         return -1;
     }
-    if (!(f = flux_rpc_pack (util->h, "job-manager.sched-ready",
-                             FLUX_NODEID_ANY, 0,
-                             "{s:s}", "mode", mode)))
+    if (strstarts (mode, "limited=")) {
+        char *endptr;
+        int n = strtol (mode+8, &endptr, 0);
+        if (*endptr != '\0' || n <= 0) {
+            errno = EINVAL;
+            return -1;
+        }
+        mode = "limited";
+        limit = n;
+    }
+    else if (!streq (mode, "unlimited")) {
+        errno = EINVAL;
         return -1;
+    }
+    if (limit) {
+        if (!(f = flux_rpc_pack (util->h,
+                                 "job-manager.sched-ready",
+                                 FLUX_NODEID_ANY,
+                                 0,
+                                 "{s:s s:i}",
+                                 "mode", mode,
+                                 "limit", limit)))
+            return -1;
+    }
+    else {
+        if (!(f = flux_rpc_pack (util->h,
+                                 "job-manager.sched-ready",
+                                 FLUX_NODEID_ANY,
+                                 0,
+                                 "{s:s}",
+                                 "mode", mode)))
+            return -1;
+    }
     if (flux_rpc_get_unpack (f, "{s:i}", "count", &count) < 0)
         goto error;
     if (queue_depth)
diff --git a/src/common/libschedutil/ready.h b/src/common/libschedutil/ready.h
index 3856f0ea7c7f..c7242baf8b93 100644
--- a/src/common/libschedutil/ready.h
+++ b/src/common/libschedutil/ready.h
@@ -15,13 +15,26 @@
 
 #include "init.h"
 
-/* Send ready request to job-manager, selecting interface 'mode'
- * ("single", "unlimited", ...).  'queue_depth', if non-NULL,
- * is set to the number of jobs in SCHED state that have not yet requested
- * resources.  Returns 0 on success, -1 on failure with errno set.
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Send ready request to job-manager, selecting interface 'mode'.
+ * Potential inputs:
+ *
+ * "unlimited"
+ * "limited=N" (N in range 1 - 2147483647)
+ *
+ * 'queue_depth', if non-NULL, is set to the number of jobs in SCHED
+ * state that have not yet requested resources.  Returns 0 on success,
+ * -1 on failure with errno set.
  */
 int schedutil_ready (schedutil_t *util, const char *mode, int *queue_depth);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* !_FLUX_SCHEDUTIL_READY_H */
 
 /*
diff --git a/src/common/libschedutil/schedutil_private.h b/src/common/libschedutil/schedutil_private.h
index 36473fc9d0c0..927f7ee9d792 100644
--- a/src/common/libschedutil/schedutil_private.h
+++ b/src/common/libschedutil/schedutil_private.h
@@ -11,43 +11,31 @@
 #ifndef HAVE_SCHEDUTIL_PRIVATE_H
 #define HAVE_SCHEDUTIL_PRIVATE_H 1
 
-#include <czmq.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
 #include "init.h"
 
 
 struct schedutil_ctx {
     flux_t *h;
     flux_msg_handler_t **handlers;
-    schedutil_alloc_cb_f *alloc_cb;
-    schedutil_free_cb_f *free_cb;
-    schedutil_cancel_cb_f *cancel_cb;
+    const struct schedutil_ops *ops;
+    int flags;
     void *cb_arg;
     zlistx_t *outstanding_futures;
 };
 
-/*
- * Add/remove futures that have associated outstandings messages whose response
- * is blocked on the future's fulfillment.  Schedutil will automatically reply
- * to the msg with ENOSYS and destroy the future when the scheduler gets
- * unloaded.
+/* Track futures that need to be destroyed on scheduler unload.
  * Return 0 on success and -1 on error.
  */
-int schedutil_add_outstanding_future (schedutil_t *util, flux_future_t *fut);
-int schedutil_remove_outstanding_future (schedutil_t *util,
-                                         flux_future_t *fut);
+int add_outstanding_future (schedutil_t *util, flux_future_t *fut);
+int remove_outstanding_future (schedutil_t *util, flux_future_t *fut);
 
 /* (Un-)register callbacks for alloc, free, cancel.
  */
-int schedutil_ops_register (schedutil_t *util);
-void schedutil_ops_unregister (schedutil_t *util);
-
-/* Testing interfaces
- *
- * Check to see if the scheduler has the debug flag set such
- * that responses should hang, forcing outstanding requests to exist.
- */
-bool schedutil_hang_responses (const schedutil_t *util);
+int ops_register (schedutil_t *util);
+void ops_unregister (schedutil_t *util);
 
 #endif /* HAVE_SCHEDUTIL_PRIVATE_H */
diff --git a/src/common/libsdexec/Makefile.am b/src/common/libsdexec/Makefile.am
new file mode 100644
index 000000000000..7a9e3603e92a
--- /dev/null
+++ b/src/common/libsdexec/Makefile.am
@@ -0,0 +1,110 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_builddir)/src/common/libflux \
+	-I$(top_srcdir)/src/common/libccan \
+	$(JANSSON_CFLAGS)
+
+noinst_LTLIBRARIES = libsdexec.la
+
+libsdexec_la_SOURCES = \
+	state.c \
+	state.h \
+	start.c \
+	start.h \
+	stop.h \
+	stop.c \
+	list.h \
+	list.c \
+	property.h \
+	property.c \
+	outbuf.h \
+	outbuf.c \
+	channel.h \
+	channel.c \
+	unit.h \
+	unit.c \
+	parse.h \
+	parse.c
+
+test_ldadd = \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(top_builddir)/src/common/libtestutil/libtestutil.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(LIBPTHREAD) \
+	$(JANSSON_LIBS)
+
+test_cppflags = \
+	$(AM_CPPFLAGS)
+
+test_ldflags = \
+	-no-install
+
+TESTS = \
+	test_parse.t \
+	test_channel.t \
+	test_channel_outbuf.t \
+	test_list.t \
+	test_property.t \
+	test_start.t \
+	test_state.t \
+	test_stop.t \
+	test_unit.t
+
+check_PROGRAMS = \
+	$(TESTS)
+
+test_parse_t_SOURCES = test/parse.c
+test_parse_t_CPPFLAGS = $(test_cppflags)
+test_parse_t_LDADD = libsdexec.la $(test_ldadd)
+test_parse_t_LDFLAGS = $(test_ldflags)
+
+test_channel_t_SOURCES = test/channel.c
+test_channel_t_CPPFLAGS = $(test_cppflags)
+test_channel_t_LDADD = libsdexec.la $(test_ldadd)
+test_channel_t_LDFLAGS = $(test_ldflags)
+
+test_channel_outbuf_t_SOURCES = test/channel_outbuf.c
+test_channel_outbuf_t_CPPFLAGS = $(test_cppflags)
+test_channel_outbuf_t_LDADD = libsdexec.la $(test_ldadd)
+test_channel_outbuf_t_LDFLAGS = $(test_ldflags)
+
+test_list_t_SOURCES = test/list.c
+test_list_t_CPPFLAGS = $(test_cppflags)
+test_list_t_LDADD = libsdexec.la $(test_ldadd)
+test_list_t_LDFLAGS = $(test_ldflags)
+
+test_property_t_SOURCES = test/property.c
+test_property_t_CPPFLAGS = $(test_cppflags)
+test_property_t_LDADD = libsdexec.la $(test_ldadd)
+test_property_t_LDFLAGS = $(test_ldflags)
+
+test_start_t_SOURCES = test/start.c
+test_start_t_CPPFLAGS = $(test_cppflags)
+test_start_t_LDADD = libsdexec.la $(test_ldadd)
+test_start_t_LDFLAGS = $(test_ldflags)
+
+test_state_t_SOURCES = test/state.c
+test_state_t_CPPFLAGS = $(test_cppflags)
+test_state_t_LDADD = libsdexec.la $(test_ldadd)
+test_state_t_LDFLAGS = $(test_ldflags)
+
+test_stop_t_SOURCES = test/stop.c
+test_stop_t_CPPFLAGS = $(test_cppflags)
+test_stop_t_LDADD = libsdexec.la $(test_ldadd)
+test_stop_t_LDFLAGS = $(test_ldflags)
+
+test_unit_t_SOURCES = test/unit.c
+test_unit_t_CPPFLAGS = $(test_cppflags)
+test_unit_t_LDADD = libsdexec.la $(test_ldadd)
+test_unit_t_LDFLAGS = $(test_ldflags)
diff --git a/src/common/libsdexec/channel.c b/src/common/libsdexec/channel.c
new file mode 100644
index 000000000000..c224335cb3c1
--- /dev/null
+++ b/src/common/libsdexec/channel.c
@@ -0,0 +1,374 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* channel.c - manage stdio
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <flux/core.h>
+
+#include "src/common/libioencode/ioencode.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/fdutils.h"
+#include "src/common/libsubprocess/subprocess_private.h" // for default bufsize
+
+#include "outbuf.h"
+#include "channel.h"
+
+struct channel {
+    flux_t *h;
+    char rankstr[16];
+    int fd[2];
+    flux_watcher_t *w;
+    bool eof_received;
+    bool eof_delivered;
+    struct outbuf *buf;
+    int flags;
+    char *name;
+    bool writable;
+    channel_output_f output_cb;
+    channel_error_f error_cb;
+    void *arg;
+    int refcount;
+};
+
+static struct channel *sdexec_channel_incref (struct channel *ch);
+static void sdexec_channel_decref (struct channel *ch);
+
+static int call_output_callback (struct channel *ch,
+                                 const char *data,
+                                 size_t length,
+                                 bool eof)
+{
+    json_t *io;
+    int rc = -1;
+
+    if (length == 0)
+        data = NULL; // appease ioencode()
+    if (!(io = ioencode (ch->name, ch->rankstr, data, length, eof)))
+        goto done;
+    if (ch->output_cb)
+        ch->output_cb (ch, io, ch->arg);
+    if (eof)
+        ch->eof_delivered = true;
+    rc = 0;
+done:
+    json_decref (io);
+    return rc;
+}
+
+static size_t nextline (const char *data, size_t len)
+{
+    for (size_t i = 0; i < len; i++) {
+        if (data[i] == '\n')
+            return i + 1;
+    }
+    return 0;
+}
+
+/* Flush one line, or one partial buffer if it meets criteria noted below.
+ * This function returns -1 on error, 0 if done, or 1 if it should be called
+ * again.
+ */
+static int flush_output_line (struct channel *ch)
+{
+    size_t len;
+    bool eof = false;
+
+    len = nextline (outbuf_tail (ch->buf), outbuf_used (ch->buf));
+    /* There is no complete line, but the buffer is full.
+     * No more data can be added to terminate the line so we must flush.
+     */
+    if (len == 0 && outbuf_full (ch->buf))
+        len = outbuf_used (ch->buf);
+    /* There is no complete line nor full buffer, but EOF has been reached.
+     * No more data will ever be added to terminate the line so we must flush.
+     */
+    if (len == 0 && ch->eof_received) {
+        len = outbuf_used (ch->buf);
+        eof = true;
+    }
+    if (len > 0 || eof) {
+        int rc = call_output_callback (ch, outbuf_tail (ch->buf), len, eof);
+        outbuf_mark_free (ch->buf, len);
+        if (rc < 0)
+            return -1;
+    }
+    if (len == 0 || eof)
+        return 0;
+    return 1;
+}
+
+/* Flush all data in the buffer.
+ */
+static int flush_output_raw (struct channel *ch)
+{
+    int n;
+    n = call_output_callback (ch,
+                              outbuf_tail (ch->buf),
+                              outbuf_used (ch->buf),
+                              ch->eof_received);
+    outbuf_mark_free (ch->buf, outbuf_used (ch->buf));
+    return n;
+}
+
+/* fd watcher for read end of channel file descriptor
+ */
+static void channel_output_cb (flux_reactor_t *r,
+                               flux_watcher_t *w,
+                               int revents,
+                               void *arg)
+{
+    struct channel *ch = arg;
+    ssize_t n;
+
+    /* Read a chunk of data into the buffer, not necessarily all that is ready.
+     * Let the event loop iterate and read more as needed.
+     */
+    n = read (ch->fd[0], outbuf_head (ch->buf), outbuf_free (ch->buf));
+    if (n < 0) {
+        if (errno == EAGAIN || errno == EWOULDBLOCK)
+            return; // spurious wakeup or revents without POLLIN?
+        if (ch->error_cb) {
+            flux_error_t error;
+            errprintf (&error,
+                       "error reading from %s: %s",
+                       ch->name,
+                       strerror (errno));
+            ch->error_cb (ch, &error, ch->arg);
+            // fall through and generate EOF
+        }
+    }
+    /* Since sdexec.exec clients are not finalized until the channel callback
+     * gets EOF, ensure that it always does, even if there was a read error.
+     */
+    if (n <= 0) {
+        ch->eof_received = true;
+        flux_watcher_stop (w);
+    }
+    else
+        outbuf_mark_used (ch->buf, n);
+    /* In case the channel output callback destroys the channel,
+     * hold a reference for the remainder of this function.
+     * See flux-framework/flux-core#6036.
+     */
+    sdexec_channel_incref (ch);
+    if ((ch->flags & CHANNEL_LINEBUF)) {
+        while ((n = flush_output_line (ch)) > 0)
+            ;
+    }
+    else
+        n = flush_output_raw (ch);
+    if (n < 0) {
+        if (ch->error_cb) {
+            flux_error_t error;
+            errprintf (&error,
+                       "error flushing data from %s: %s",
+                       ch->name,
+                       strerror (errno));
+            ch->error_cb (ch, &error, ch->arg);
+        }
+    }
+    outbuf_gc (ch->buf);
+    sdexec_channel_decref (ch);
+}
+
+int sdexec_channel_get_fd (struct channel *ch)
+{
+    return ch ? ch->fd[1] : -1;
+}
+
+const char *sdexec_channel_get_name (struct channel *ch)
+{
+    return ch ? ch->name : "unknown";
+}
+
+void sdexec_channel_close_fd (struct channel *ch)
+{
+    if (ch && ch->fd[1] >= 0) {
+        close (ch->fd[1]);
+        ch->fd[1] = -1;
+    }
+}
+
+void sdexec_channel_start_output (struct channel *ch)
+{
+    if (ch && !ch->eof_delivered)
+        flux_watcher_start (ch->w);
+}
+
+static struct channel *sdexec_channel_incref (struct channel *ch)
+{
+    if (ch)
+        ch->refcount++;
+    return ch;
+}
+
+static void sdexec_channel_decref (struct channel *ch)
+{
+    if (ch && --ch->refcount == 0) {
+        int saved_errno = errno;
+        if (ch->fd[0] >= 0)
+            close (ch->fd[0]);
+        if (ch->fd[1] >= 0)
+            close (ch->fd[1]);
+        flux_watcher_destroy (ch->w);
+        outbuf_destroy (ch->buf);
+        free (ch->name);
+        free (ch);
+        errno = saved_errno;
+    }
+}
+
+void sdexec_channel_destroy (struct channel *ch)
+{
+    sdexec_channel_decref (ch);
+}
+
+static struct channel *sdexec_channel_create (flux_t *h, const char *name)
+{
+    struct channel *ch;
+    uint32_t rank;
+
+    if (!h || !name) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(ch = calloc (1, sizeof (*ch))))
+        return NULL;
+    ch->refcount = 1;
+    ch->h = h;
+    ch->fd[0] = -1;
+    ch->fd[1] = -1;
+    if (!(ch->name = strdup (name)))
+        goto error;
+    if (flux_get_rank (h, &rank) < 0)
+        goto error;
+    snprintf (ch->rankstr, sizeof (ch->rankstr), "%u", (unsigned int)rank);
+    if (socketpair (PF_LOCAL, SOCK_STREAM, 0, ch->fd) < 0)
+        goto error;
+    return ch;
+error:
+    sdexec_channel_destroy (ch);
+    return NULL;
+}
+
+struct channel *sdexec_channel_create_output (flux_t *h,
+                                              const char *name,
+                                              size_t bufsize,
+                                              int flags,
+                                              channel_output_f output_cb,
+                                              channel_error_f error_cb,
+                                              void *arg)
+{
+    struct channel *ch;
+
+    if (!(ch = sdexec_channel_create (h, name)))
+        return NULL;
+    ch->output_cb = output_cb;
+    ch->error_cb = error_cb;
+    ch->arg = arg;
+    ch->flags = flags;
+    if (fd_set_nonblocking (ch->fd[0]) < 0)
+        goto error;
+    if (!(ch->w = flux_fd_watcher_create (flux_get_reactor (h),
+                                          ch->fd[0],
+                                          FLUX_POLLIN,
+                                          channel_output_cb,
+                                          ch)))
+        goto error;
+    if (bufsize == 0)
+        bufsize = SUBPROCESS_DEFAULT_BUFSIZE;
+    if (!(ch->buf = outbuf_create (bufsize)))
+        goto error;
+    return ch;
+error:
+    sdexec_channel_destroy (ch);
+    return NULL;
+}
+
+struct channel *sdexec_channel_create_input (flux_t *h, const char *name)
+{
+    struct channel *ch;
+
+    if (!(ch = sdexec_channel_create (h, name)))
+        return NULL;
+    ch->writable = true;
+    return ch;
+}
+
+int sdexec_channel_write (struct channel *ch, json_t *io)
+{
+    char *data;
+    int len;
+    bool eof;
+
+    if (!ch || !io) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (iodecode (io, NULL, NULL, &data, &len, &eof) < 0)
+        return -1;
+    if (!ch->writable || ch->fd[0] < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (data && len > 0) {
+        int count = 0;
+        while (count < len) {
+            ssize_t n;
+            if ((n = write (ch->fd[0], data + count, len - count)) < 0) {
+                ERRNO_SAFE_WRAP (free, data);
+                return -1;
+            }
+            count += n;
+        }
+        free (data);
+    }
+    if (eof) {
+        int fd = ch->fd[0];
+
+        ch->fd[0] = -1;
+        if (close (fd) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+json_t *sdexec_channel_get_stats (struct channel *ch)
+{
+    json_t *o = NULL;
+
+    if (ch) {
+        if (ch->writable) {
+            o = json_pack ("{s:i s:i}",
+                           "local_fd", ch->fd[0],
+                           "remote_fd", ch->fd[1]);
+        }
+        else {
+            o = json_pack ("{s:i s:i s:i s:i s:b}",
+                           "local_fd", ch->fd[0],
+                           "remote_fd", ch->fd[1],
+                           "buf_used", outbuf_used (ch->buf),
+                           "buf_free", outbuf_free (ch->buf),
+                           "eof", ch->eof_received);
+        }
+    }
+    return o;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/channel.h b/src/common/libsdexec/channel.h
new file mode 100644
index 000000000000..096adc2a3bc9
--- /dev/null
+++ b/src/common/libsdexec/channel.h
@@ -0,0 +1,83 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSDEXEC_CHANNEL_H
+#define _LIBSDEXEC_CHANNEL_H
+
+#include <jansson.h>
+#include <flux/core.h>
+
+struct channel;
+
+enum {
+    CHANNEL_LINEBUF = 1,
+};
+
+typedef void (*channel_output_f)(struct channel *ch, json_t *io, void *arg);
+typedef void (*channel_error_f)(struct channel *ch,
+                                flux_error_t *error,
+                                void *arg);
+
+/* Open a channel for output from the systemd unit.  When the unit has written
+ * some data, an internal fd watcher buffers it, then invokes output_cb.
+ * If there is a read error, the error_cb is also called for logging, then
+ * output_cb is called with EOF.
+ *
+ * Notes:
+ * - internal watcher is not started until sdexec_channel_start_output()
+ *   is called
+ * - data is line buffered if 'flags' includes CHANNEL_LINEBUF
+ * - a single callback may not represent all the data available at that moment
+ */
+struct channel *sdexec_channel_create_output (flux_t *h,
+                                              const char *name,
+                                              size_t bufsize,
+                                              int flags,
+                                              channel_output_f output_cb,
+                                              channel_error_f error_cb,
+                                              void *arg);
+
+/* Open a channel for input to the systemd unit.
+ * The channel may be written to using sdexec_channel_write().
+ */
+struct channel *sdexec_channel_create_input (flux_t *h, const char *name);
+
+/* Write to channel created with sdexec_channel_create_input ().
+ * The ioencoded object's rank and stream name are ignored.
+ * This is potentially a blocking operation if the socketpair cannot
+ * accept all the data.
+ */
+int sdexec_channel_write (struct channel *ch, json_t *io);
+
+/* Get fd for systemd end of socketpair.  Returns -1 if unset or ch==NULL.
+ */
+int sdexec_channel_get_fd (struct channel *ch);
+
+/* Get name of channel.
+ */
+const char *sdexec_channel_get_name (struct channel *ch);
+
+/* Close fd on systemd end of socketpair.
+ * Call this after systemd has received the fd and duped it - in the response
+ * handler for StartTransientUnit should be correct.
+ */
+void sdexec_channel_close_fd (struct channel *ch);
+
+/* Start watching for channel output.
+ */
+void sdexec_channel_start_output (struct channel *ch);
+
+void sdexec_channel_destroy (struct channel *ch);
+
+json_t *sdexec_channel_get_stats (struct channel *ch);
+
+#endif /* !_LIBSDEXEC_CHANNEL_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/list.c b/src/common/libsdexec/list.c
new file mode 100644
index 000000000000..531a77835f82
--- /dev/null
+++ b/src/common/libsdexec/list.c
@@ -0,0 +1,84 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* list.c - list units
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "ccan/ptrint/ptrint.h"
+
+#include "list.h"
+
+static int parse_unit (json_t *units, size_t index, struct unit_info *info)
+{
+    json_t *entry;
+
+    if (!units
+        || !info
+        || !(entry = json_array_get (units, index))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (json_unpack (entry,
+                     "[sssssssIss]",
+                     &info->name,
+                     &info->description,
+                     &info->load_state,
+                     &info->active_state,
+                     &info->sub_state,
+                     &info->name_follower,
+                     &info->path,
+                     &info->job_id,
+                     &info->job_type,
+                     &info->job_path) < 0) {
+        errno = EPROTO;
+        return -1;
+    }
+    return 0;
+}
+
+bool sdexec_list_units_next (flux_future_t *f, struct unit_info *infop)
+{
+    json_t *units;
+    struct unit_info info;
+    int index = ptr2int (flux_future_aux_get (f, "index")); // zero if not set
+
+    if (!infop
+        || flux_rpc_get_unpack (f, "{s:[o]}", "params", &units) < 0
+        || parse_unit (units, index, &info) < 0
+        || flux_future_aux_set (f, "index", int2ptr (index + 1), NULL) < 0)
+        return false;
+
+    *infop = info;
+    return true;
+}
+
+/* N.B. Input params:  states=[], patterns=[pattern].
+ */
+flux_future_t *sdexec_list_units (flux_t *h, uint32_t rank, const char *pattern)
+{
+    if (!h || !pattern) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return flux_rpc_pack (h,
+                          "sdbus.call",
+                          rank,
+                          0,
+                          "{s:s s:[[] [s]]}",
+                          "member", "ListUnitsByPatterns",
+                          "params", pattern);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/list.h b/src/common/libsdexec/list.h
new file mode 100644
index 000000000000..115767ff5d51
--- /dev/null
+++ b/src/common/libsdexec/list.h
@@ -0,0 +1,46 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSDEXEC_LIST_H
+#define _LIBSDEXEC_LIST_H
+
+#include <jansson.h>
+#include <flux/core.h>
+
+struct unit_info {
+    const char *name;
+    const char *description;
+    const char *load_state;
+    const char *active_state;
+    const char *sub_state;
+    const char *name_follower;  // "" if no unit whose state follows this one
+    const char *path;
+
+    json_int_t job_id;          // 0 if no job queued for the unit
+    const char *job_type;
+    const char *job_path;
+};
+
+/* Obtain the unit list, filtered by glob pattern.
+ * (E.g. use "*" for all).
+ */
+flux_future_t *sdexec_list_units (flux_t *h,
+                                  uint32_t rank,
+                                  const char *pattern);
+
+/* Iterate over returned list of units.
+ * Fill in 'info' with the next entry and return true,
+ * or return false if there are no more entries.
+ */
+bool sdexec_list_units_next (flux_future_t *f, struct unit_info *info);
+
+#endif /* !_LIBSDEXEC_LIST_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/outbuf.c b/src/common/libsdexec/outbuf.c
new file mode 100644
index 000000000000..80b08d88aa9f
--- /dev/null
+++ b/src/common/libsdexec/outbuf.c
@@ -0,0 +1,101 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* outbuf.c - output buffer for channel.c
+ *
+ * outbuf is a linear buffer which allows data to be removed in contiguous
+ * chunks of our choosing (for example lines) without copying, unlike cbuf.
+ * However, the buffer space has to be reclaimed after data has been taken
+ * out by calling output_gc(). This works here because the channel_output_cb()
+ * watcher aggressively flushes the buffer after putting data in it.  The gc
+ * can be called just before the watcher returns.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include "outbuf.h"
+
+struct outbuf {
+    char *data;
+    size_t size;
+    size_t offset;  // valid data begins at buf.data + buf.offset
+    size_t used;    // bytes used starting at buf.data + buf.offset
+};
+
+char *outbuf_head (struct outbuf *buf)
+{
+    return buf->data + buf->offset + buf->used;
+}
+
+size_t outbuf_free (struct outbuf *buf)
+{
+    return buf->size - (buf->offset + buf->used);
+}
+
+void outbuf_mark_used (struct outbuf *buf, size_t count)
+{
+    buf->used += count;
+}
+
+// "full" in the sense that even after gc there will be no room for new data
+bool outbuf_full (struct outbuf *buf)
+{
+    return (buf->size == buf->used) ? true : false;
+}
+
+char *outbuf_tail (struct outbuf *buf)
+{
+    return buf->data + buf->offset;
+}
+
+size_t outbuf_used (struct outbuf *buf)
+{
+    return buf->used;
+}
+
+void outbuf_mark_free (struct outbuf *buf, size_t count)
+{
+    buf->offset += count;
+    buf->used -= count;
+}
+
+void outbuf_gc (struct outbuf *buf)
+{
+    if (buf->offset > 0) {
+        memmove (buf->data, buf->data + buf->offset, buf->used);
+        buf->offset = 0;
+    }
+}
+
+struct outbuf *outbuf_create (size_t size)
+{
+    struct outbuf *buf;
+    if (!(buf = calloc (1, sizeof (*buf) + size)))
+        return NULL;
+    buf->data = (char *)(buf + 1);
+    buf->size = size;
+    return buf;
+}
+
+void outbuf_destroy (struct outbuf *buf)
+{
+    if (buf) {
+        int saved_errno = errno;
+        free (buf);
+        errno = saved_errno;
+    }
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/outbuf.h b/src/common/libsdexec/outbuf.h
new file mode 100644
index 000000000000..271f6be09511
--- /dev/null
+++ b/src/common/libsdexec/outbuf.h
@@ -0,0 +1,46 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSDEXEC_OUTBUF_H
+#define _LIBSDEXEC_OUTBUF_H
+
+#include <stdbool.h>
+#include <sys/types.h>
+
+/* The outbuf container was purpose-built for sdexec/channel.c.
+ *
+ * Putting data in the buffer:
+ * - write up to outbuf_free() bytes to the location returned by outbuf_head()
+ * - account for that with outbuf_mark_used().
+ *
+ * Taking data out of the buffer:
+ * - read up to outbuf_used() bytes from the location returned by outbuf_tail()
+ * - account for that with outbuf_mark_free().
+ *
+ * Call outbuf_gc() when done consuming data from the buffer.
+ */
+struct outbuf *outbuf_create (size_t size);
+void outbuf_destroy (struct outbuf *buf);
+
+char *outbuf_head (struct outbuf *buf);
+size_t outbuf_free (struct outbuf *buf);
+void outbuf_mark_used (struct outbuf *buf, size_t count);
+
+// full in the sense that the entire buffer is used, even after gc
+bool outbuf_full (struct outbuf *buf);
+
+char *outbuf_tail (struct outbuf *buf);
+size_t outbuf_used (struct outbuf *buf);
+void outbuf_mark_free (struct outbuf *buf, size_t count);
+void outbuf_gc (struct outbuf *buf);
+
+#endif /* !_LIBSDEXEC_OUTBUF_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/parse.c b/src/common/libsdexec/parse.c
new file mode 100644
index 000000000000..e8561ed758d7
--- /dev/null
+++ b/src/common/libsdexec/parse.c
@@ -0,0 +1,81 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <flux/idset.h>
+
+#include "ccan/str/str.h"
+#include "parse.h"
+
+int sdexec_parse_percent (const char *s, double *dp)
+{
+    char *endptr;
+    double d;
+
+    if (!s || !dp)
+        return -1;
+    errno = 0;
+    d = strtod (s, &endptr);
+    if (errno != 0 || endptr == s || strlen (endptr) != 1 || endptr[0] != '%')
+        return -1;
+    if (d < 0 || d > 100)
+        return -1;
+    *dp = d * 1E-2;
+    return 0;
+}
+
+/* Borrow some macro ideas from ccan/bitmap/bitmap.h.  It cannot be used here
+ * because it apparently swaps bytes of internal words on little endian for
+ * portability, which isn't compatible with the byte array requirement of
+ * systemd/dbus.
+ */
+#define BITMAP_NBYTES(nb)           (((nb) + 8 - 1) / 8)
+#define BITMAP_BYTE(bm,n)           (bm)[(n) / 8]
+#define BITMAP_BIT(n)               (1 << ((n) % 8))
+
+int sdexec_parse_bitmap (const char *s, uint8_t **bp, size_t *sp)
+{
+    struct idset *ids;
+    unsigned int id;
+    unsigned int nbits = 0;
+    uint8_t *bitmap = NULL;
+
+    if (!s || !bp || !sp)
+        return -1;
+    if (!(ids = idset_decode (s)))
+        return -1;
+
+    // allocate a bitmap large enough to fit idset
+    if ((id = idset_last (ids)) != IDSET_INVALID_ID)
+        nbits = id + 1;
+    if (nbits > 0) {
+        if (!(bitmap = calloc (1, BITMAP_NBYTES (nbits)))) {
+            idset_destroy (ids);
+            return -1;
+        }
+        id = idset_first (ids);
+        while (id != IDSET_INVALID_ID) {
+            BITMAP_BYTE (bitmap, id) |= BITMAP_BIT (id);
+            id = idset_next (ids, id);
+        }
+    }
+    *bp = bitmap;
+    *sp = BITMAP_NBYTES (nbits);
+    idset_destroy (ids);
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/parse.h b/src/common/libsdexec/parse.h
new file mode 100644
index 000000000000..f707a65fa2f9
--- /dev/null
+++ b/src/common/libsdexec/parse.h
@@ -0,0 +1,32 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSDEXEC_PARSE_H
+#define _LIBSDEXEC_PARSE_H
+
+#include <sys/types.h>
+#include <stdint.h>
+
+/* Parse 's' as a percentage between 0 and 100 with % suffix and
+ * assign the result, a 0 <= value <= 1.0, to 'dp'.
+ * Return 0 on success or -1 on failure.  Errno is not set.
+ */
+int sdexec_parse_percent (const char *s, double *dp);
+
+/* Parse 's' as an idset into bitmap assigned to 'bp' (caller must free).
+ * The size of the bitmap (in bytes) is assigned to 'sp'.
+ * N.B. an empty string translates to an empty (*bp=NULL, *sp=0) bitmap.
+ * Return 0 on success or -1 on failure.  Errno is not set.
+ */
+int sdexec_parse_bitmap (const char *s, uint8_t **bp, size_t *sp);
+
+#endif /* !_LIBSDEXEC_PARSE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/property.c b/src/common/libsdexec/property.c
new file mode 100644
index 000000000000..82ffdcac3b2a
--- /dev/null
+++ b/src/common/libsdexec/property.c
@@ -0,0 +1,204 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* property.c - query unit properties
+ *
+ * The Get method-reply includes a single property value, represented as a
+ * D-Bus variant type, which is a (type, value) tuple: [s,o]
+ *
+ * The GetAll method-reply and the PropertiesChanged signal include a
+ * dictionary of property values:
+ * {s:[s,o], s:[s,o], s:[s,o], ...}
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "ccan/str/str.h"
+#include "src/common/libutil/errno_safe.h"
+
+#include "property.h"
+
+static const char *serv_interface = "org.freedesktop.systemd1.Service";
+static const char *prop_interface = "org.freedesktop.DBus.Properties";
+
+flux_future_t *sdexec_property_get_all (flux_t *h,
+                                        uint32_t rank,
+                                        const char *path)
+{
+    flux_future_t *f;
+
+    if (!h || !path) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(f = flux_rpc_pack (h,
+                             "sdbus.call",
+                             rank,
+                             0,
+                             "{s:s s:s s:s s:[s]}",
+                             "path", path,
+                             "interface", prop_interface,
+                             "member", "GetAll",
+                             "params", serv_interface)))
+        return NULL;
+    return f;
+}
+
+flux_future_t *sdexec_property_get (flux_t *h,
+                                    uint32_t rank,
+                                    const char *path,
+                                    const char *name)
+{
+    flux_future_t *f;
+
+    if (!h || !path || !name) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(f = flux_rpc_pack (h,
+                             "sdbus.call",
+                             rank,
+                             0,
+                             "{s:s s:s s:s s:[ss]}",
+                             "path", path,
+                             "interface", prop_interface,
+                             "member", "Get",
+                             "params", serv_interface, name)))
+        return NULL;
+    return f;
+}
+
+flux_future_t *sdexec_property_changed (flux_t *h,
+                                        uint32_t rank,
+                                        const char *path)
+{
+    flux_future_t *f;
+    json_t *o;
+
+    if (!h) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(o = json_pack ("{s:s s:s}",
+                         "interface", prop_interface,
+                         "member", "PropertiesChanged")))
+        goto nomem;
+    if (path) {
+        json_t *val = json_string (path);
+        if (!val || json_object_set_new (o, "path", val) < 0) {
+            json_decref (val);
+            goto nomem;
+        }
+    }
+    if (!(f = flux_rpc_pack (h,
+                             "sdbus.subscribe",
+                             rank,
+                             FLUX_RPC_STREAMING,
+                             "O", o)))
+        goto error;
+    json_decref (o);
+    return f;
+nomem:
+    errno = ENOMEM;
+error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return NULL;
+}
+
+int sdexec_property_get_unpack (flux_future_t *f, const char *fmt, ...)
+{
+    const char *type; // ignored
+    json_t *val;
+    va_list ap;
+    int rc;
+
+    if (!f || !fmt) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_rpc_get_unpack (f, "{s:[[so]]}", "params", &type, &val) < 0)
+        return -1;
+    va_start (ap, fmt);
+    rc = json_vunpack_ex (val, NULL, 0, fmt, ap);
+    va_end (ap);
+    if (rc < 0) {
+        errno = EPROTO;
+        return -1;
+    }
+    return rc;
+}
+
+int sdexec_property_dict_unpack (json_t *dict,
+                                 const char *name,
+                                 const char *fmt,
+                                 ...)
+
+{
+    const char *type; // ignored
+    json_t *val;
+    va_list ap;
+    int rc;
+
+    if (!dict || !name || !fmt) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (json_unpack (dict, "{s:[so]}", name, &type, &val) < 0) {
+        errno = EPROTO;
+        return -1;
+    }
+    va_start (ap, fmt);
+    rc = json_vunpack_ex (val, NULL, 0, fmt, ap);
+    va_end (ap);
+    if (rc < 0) {
+        errno = EPROTO;
+        return -1;
+    }
+    return 0;
+}
+
+json_t *sdexec_property_get_all_dict (flux_future_t *f)
+
+{
+    json_t *dict;
+
+    if (flux_rpc_get_unpack (f,
+                             "{s:[o]}",
+                             "params", &dict) < 0)
+        return NULL;
+    return dict;
+}
+
+json_t *sdexec_property_changed_dict (flux_future_t *f)
+{
+    const char *iface;
+    json_t *dict;
+    json_t *inval;
+
+    if (flux_rpc_get_unpack (f,
+                             "{s:[s o o]}",
+                             "params", &iface, &dict, &inval) < 0)
+        return NULL;
+    return dict;
+}
+
+const char *sdexec_property_changed_path (flux_future_t *f)
+{
+    const char *path;
+
+    if (flux_rpc_get_unpack (f, "{s:s}", "path", &path) < 0)
+        return NULL;
+    return path;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/property.h b/src/common/libsdexec/property.h
new file mode 100644
index 000000000000..631859199ba4
--- /dev/null
+++ b/src/common/libsdexec/property.h
@@ -0,0 +1,57 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+j************************************************************/
+
+#ifndef _LIBSDEXEC_PROPERTY_H
+#define _LIBSDEXEC_PROPERTY_H
+
+#include <jansson.h>
+#include <flux/core.h>
+
+/* 'Get' method-call.
+ * Parse the returned value with sdexec_property_get_unpack().
+ */
+flux_future_t *sdexec_property_get (flux_t *h,
+                                    uint32_t rank,
+                                    const char *path,
+                                    const char *name);
+int sdexec_property_get_unpack (flux_future_t *f, const char *fmt, ...);
+
+/* 'GetAll' method-call.
+ * sdexec_property_get_all_dict() accesses the returned property dict,
+ * which can be further parsed with sdexec_property_dict_unpack().
+ */
+flux_future_t *sdexec_property_get_all (flux_t *h,
+                                        uint32_t rank,
+                                        const char *path);
+json_t *sdexec_property_get_all_dict (flux_future_t *f);
+
+/* 'PropertiesChanged' signal.
+ * sdexec_property_changed() subscribes to streaming property updates for the
+ * specified path.  Each response contains a property dict that may be accessed
+ * with sdexec_property_changed_dict().  The dict can be further parsed with
+ * sdexec_property_dict_unpack().  Use path=NULL for no path filter, then
+ * sdexec_property_changed_path() to get the path for each response.
+ */
+flux_future_t *sdexec_property_changed (flux_t *h,
+                                        uint32_t rank,
+                                        const char *path);
+json_t *sdexec_property_changed_dict (flux_future_t *f);
+const char *sdexec_property_changed_path (flux_future_t *f);
+
+/* Parse property 'name' from property dict.
+ */
+int sdexec_property_dict_unpack (json_t *dict,
+                                 const char *name,
+                                 const char *fmt,
+                                 ...);
+
+#endif /* !_LIBSDEXEC_PROPERTY_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/start.c b/src/common/libsdexec/start.c
new file mode 100644
index 000000000000..d2af62b3f341
--- /dev/null
+++ b/src/common/libsdexec/start.c
@@ -0,0 +1,419 @@
+/************************************************************\
+
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* start.c - start transient service unit from json-encoded flux_cmd_t
+ *
+ * Ref: https://www.freedesktop.org/wiki/Software/systemd/dbus/
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <ctype.h>
+#include <stdio.h>
+#include <jansson.h>
+#include <flux/core.h>
+#include <float.h> // for DBL_MAX
+
+#include "ccan/str/str.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/parse_size.h"
+
+#include "parse.h"
+#include "start.h"
+
+/* This dense JSON format string deserves some explanation!
+ * [s[sO]] is [key,[type,val]] std. form for a StartTransientUnit property.
+ * - key is "ExecStart", the property name
+ * - type is "a(sasb)", the D-Bus signature for value
+ * - val is [[sOb]], an array of command lines
+ * The command line [sOb] consists of
+ * - command name (argv[0])
+ * - argv array (of strings)
+ * - boolean ignore-failure flag (e.g. an ExecStart prefix of "-")
+ * This function assumes one command line, and ignore-failure=false.
+ */
+static int prop_add_execstart (json_t *prop, const char *name, json_t *cmdline)
+{
+    json_t *o = NULL;
+    const char *arg0;
+
+    if (json_unpack (cmdline, "[s]", &arg0) < 0
+        || !(o = json_pack ("[s[s[[sOb]]]]", name, "a(sasb)", arg0, cmdline, 0))
+        || json_array_append_new (prop, o) < 0) {
+        json_decref (o);
+        return -1;
+    }
+    return 0;
+}
+
+/* systemd fails a StartTransientUnit request if environment variable
+ * names start with a digit, or contain characters other than digits,
+ * letters, or '_'.
+ * https://github.com/systemd/systemd/blob/main/src/basic/env-util.c#L28
+ */
+static bool environment_name_ok (const char *name)
+{
+    if (strlen (name) == 0 || isdigit (name[0]))
+        return false;
+    for (int i = 0; i < strlen (name); i++) {
+        if (!isalnum (name[i]) && name[i] != '_')
+            return false;
+    }
+    return true;
+}
+
+/* The Environment property is an array of "key=value" strings, which
+ * is built up from the env dict received as part of the command.
+ */
+static int prop_add_env (json_t *prop, const char *name, json_t *dict)
+{
+    json_t *a;
+    const char *k;
+    json_t *vo;
+    json_t *env;
+    int rc = -1;
+
+    if (!(a = json_array ()))
+        return -1;
+    json_object_foreach (dict, k, vo) {
+        const char *v = json_string_value (vo);
+
+        if (environment_name_ok (k)) {
+            char *kv = NULL;
+            json_t *kvo = NULL;
+            if (asprintf (&kv, "%s=%s", k, v) < 0
+                || !(kvo = json_string (kv))
+                || json_array_append_new (a, kvo) < 0) {
+                json_decref (kvo);
+                free (kv);
+                goto out;
+            }
+            free (kv);
+        }
+    }
+    if (!(env = json_pack ("[s[sO]]", name, "as", a))
+        || json_array_append_new (prop, env) < 0) {
+        json_decref (env);
+        goto out;
+    }
+    rc = 0;
+out:
+    json_decref (a);
+    return rc;
+}
+
+static int prop_add_string (json_t *prop, const char *name, const char *val)
+{
+    json_t *o;
+
+    if (val) {
+        if (!(o = json_pack ("[s[ss]]", name, "s", val))
+            || json_array_append_new (prop, o) < 0) {
+            json_decref (o);
+            return -1;
+        }
+    }
+    return 0;
+}
+
+/* This assumes message source and destination are in the same process,
+ * as is the case with sdexec => sdbus broker modules.
+ */
+static int prop_add_fd (json_t *prop, const char *name, int val)
+{
+    json_t *o;
+
+    if (val >= 0) {
+        if (!(o = json_pack ("[s[si]]", name, "h", val))
+            || json_array_append_new (prop, o) < 0) {
+            json_decref (o);
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static int prop_add_bool (json_t *prop, const char *name, int val)
+{
+    json_t *o;
+
+    if (!(o = json_pack ("[s[sb]]", name, "b", val))
+        || json_array_append_new (prop, o) < 0) {
+        json_decref (o);
+        return -1;
+    }
+    return 0;
+}
+
+// per systemd.syntax(7), boolean values are: 1|yes|true|on, 0|no|false|off
+static bool is_true (const char *s)
+{
+    if (streq (s, "1")
+        || !strcasecmp (s, "yes")
+        || !strcasecmp (s, "true")
+        || !strcasecmp (s, "on"))
+        return true;
+    return false;
+}
+static bool is_false (const char *s)
+{
+    if (streq (s, "0")
+        || !strcasecmp (s, "no")
+        || !strcasecmp (s, "false")
+        || !strcasecmp (s, "off"))
+        return true;
+    return false;
+}
+
+static int prop_add_u32 (json_t *prop, const char *name, uint32_t val)
+{
+    json_int_t i = val;
+    json_t *o;
+
+    if (!(o = json_pack ("[s[sI]]", name, "u", i))
+        || json_array_append_new (prop, o) < 0) {
+        json_decref (o);
+        return -1;
+    }
+    return 0;
+}
+
+static int prop_add_u64 (json_t *prop, const char *name, uint64_t val)
+{
+    json_int_t i = (json_int_t)val;
+    json_t *o;
+
+    if (!(o = json_pack ("[s[sI]]", name, "t", i))
+        || json_array_append_new (prop, o) < 0) {
+        json_decref (o);
+        return -1;
+    }
+    return 0;
+}
+
+static int prop_add_bytearray (json_t *prop,
+                               const char *name,
+                               uint8_t *bytearray,
+                               size_t size)
+{
+    json_t *o;
+    json_t *a;
+
+    if (!(a = json_array ()))
+        return -1;
+    for (int i = 0; i < size; i++) {
+        json_t *vo;
+        if (!(vo = json_integer (bytearray[i]))
+            || json_array_append_new (a, vo) < 0) {
+            json_decref (vo);
+            json_decref (a);
+            return -1;
+        }
+    }
+    if (!(o = json_pack ("[s[sO]]", name, "ay", a))
+        || json_array_append_new (prop, o) < 0) {
+        json_decref (o);
+        json_decref (a);
+        return -1;
+    }
+    json_decref (a);
+    return 0;
+}
+
+/* Set a property by name. By default, values are strings  Those that are
+ * not require explicit conversion from string.
+ */
+static int prop_add (json_t *prop, const char *name, const char *val)
+{
+    if (strlen (name) == 0 || !val)
+        return 0;
+
+    if (streq (name, "MemoryHigh")
+        || streq (name, "MemoryMax")
+        || streq (name, "MemoryMin")
+        || streq (name, "MemoryLow")) {
+        double d;
+        uint64_t u;
+
+        if (sdexec_parse_percent (val, &d) == 0) {
+            char newname[64];
+            snprintf (newname, sizeof (newname), "%sScale", name);
+            if (prop_add_u32 (prop, newname, (uint32_t)(d * UINT32_MAX)) < 0)
+                return -1;
+        }
+        else if (parse_size (val, &u) == 0) {
+            if (prop_add_u64 (prop, name, u) < 0)
+                return -1;
+        }
+        else if (streq (val, "infinity")) {
+            if (prop_add_u64 (prop, name, UINT64_MAX) < 0)
+                return -1;
+        }
+        else
+            return -1;
+    }
+    else if (streq (name, "AllowedCPUs")) {
+        uint8_t *bitmap;
+        size_t size;
+
+        if (sdexec_parse_bitmap (val, &bitmap, &size) < 0)
+            return -1;
+        if (prop_add_bytearray (prop, name, bitmap, size) < 0) {
+            free (bitmap);
+            return -1;
+        }
+        free (bitmap);
+    }
+    else if (streq (name, "SendSIGKILL")) {
+        bool value;
+        if (is_false (val))
+            value = false;
+        else if (is_true (val))
+            value = true;
+        else
+            return -1;
+        if (prop_add_bool (prop, name, value) < 0)
+            return -1;
+    }
+    else {
+        if (prop_add_string (prop, name, val) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static json_t *prop_create (json_t *cmd,
+                            const char *type,
+                            int stdin_fd,
+                            int stdout_fd,
+                            int stderr_fd,
+                            flux_error_t *error)
+{
+    json_t *prop;
+    json_error_t jerror;
+    const char *cwd = NULL;
+    json_t *cmdline;
+    json_t *env;
+    json_t *opts;
+    const char *key;
+    json_t *val;
+
+    // Not unpacked: channels
+    if (json_unpack_ex (cmd,
+                        &jerror,
+                        0,
+                        "{s?s s:o s:o s:o}",
+                        "cwd", &cwd,
+                        "cmdline", &cmdline,
+                        "env", &env,
+                        "opts", &opts) < 0
+        || json_array_size (cmdline) == 0) {
+        errprintf (error, "error parsing command object: %s", jerror.text);
+        errno = EPROTO;
+        return NULL;
+    }
+    if (!(prop = json_array ())) {
+        errprintf (error, "out of memory");
+        errno = ENOMEM;
+        return NULL;
+    }
+    if (prop_add_execstart (prop, "ExecStart", cmdline) < 0
+        || prop_add_string (prop, "Type", type) < 0
+        || prop_add_string (prop, "WorkingDirectory", cwd) < 0
+        || prop_add_bool (prop, "RemainAfterExit", true) < 0
+        || prop_add_env (prop, "Environment", env) < 0
+        || prop_add_fd (prop, "StandardInputFileDescriptor", stdin_fd) < 0
+        || prop_add_fd (prop, "StandardOutputFileDescriptor", stdout_fd) < 0
+        || prop_add_fd (prop, "StandardErrorFileDescriptor", stderr_fd) < 0) {
+        errprintf (error, "error packing StartTransientUnit properties");
+        goto error;
+    }
+    // any subprocess opt prefixed with SDEXEC_PROP_ is taken for a property
+    json_object_foreach (opts, key, val) {
+        if (strstarts (key, "SDEXEC_PROP_")) {
+            if (prop_add (prop, key + 12, json_string_value (val)) < 0) {
+                errprintf (error, "%s: error setting property", key);
+                goto error;
+            }
+        }
+    }
+    return prop;
+error:
+    json_decref (prop);
+    errno = EINVAL;
+    return NULL;
+}
+
+flux_future_t *sdexec_start_transient_unit (flux_t *h,
+                                            uint32_t rank,
+                                            const char *mode,
+                                            const char *type,
+                                            json_t *cmd,
+                                            int stdin_fd,
+                                            int stdout_fd,
+                                            int stderr_fd,
+                                            flux_error_t *error)
+{
+    flux_future_t *f;
+    json_t *prop = NULL;
+    const char *name;
+
+    if (!h || !mode || !type || !cmd) {
+        errprintf (error, "invalid argument");
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(prop = prop_create (cmd,
+                              type,
+                              stdin_fd,
+                              stdout_fd,
+                              stderr_fd,
+                              error)))
+        return NULL;
+    if (json_unpack (cmd, "{s:{s:s}}", "opts", "SDEXEC_NAME", &name) < 0) {
+        errprintf (error, "SDEXEC_NAME subprocess command option is not set");
+        errno = EINVAL;
+        goto error;
+    }
+    /* N.B. the empty array tacked onto the end of the 'params' array below
+     * is the placeholder for aux unit info, unused here.
+     */
+    if (!(f = flux_rpc_pack (h,
+                             "sdbus.call",
+                             rank,
+                             0,
+                             "{s:s s:[ssO[]]}",
+                             "member", "StartTransientUnit",
+                             "params", name, mode, prop))) {
+        errprintf (error, "error sending StartTransientUnit RPC");
+        goto error;
+    }
+    json_decref (prop);
+    return f;
+error:
+    ERRNO_SAFE_WRAP (json_decref, prop);
+    return NULL;
+}
+
+int sdexec_start_transient_unit_get (flux_future_t *f, const char **jobp)
+{
+    const char *job;
+
+    if (flux_rpc_get_unpack (f, "{s:[s]}", "params", &job) < 0)
+        return -1;
+    if (jobp)
+        *jobp = job;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/start.h b/src/common/libsdexec/start.h
new file mode 100644
index 000000000000..1d9e007825bc
--- /dev/null
+++ b/src/common/libsdexec/start.h
@@ -0,0 +1,64 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSDEXEC_START_H
+#define _LIBSDEXEC_START_H
+
+#include <flux/core.h>
+
+/* Call systemd StartTransientUnit with parameters from libsubprocess style
+ * command object. The SDEXEC_NAME command option must be set to the unit
+ * name (with .service suffx).
+ *
+ * See https://www.freedesktop.org/wiki/Software/systemd/dbus/
+ * and systemd.service(5) for more info on mode and type parameters.
+ * mode may be one of:
+     replace, fail, isolate, ignore-dependencies, ignore-requirements
+ * type may be one of:
+ *   simple, exec, forking, oneshot, dbus, notify, notify-reload, idle
+ *
+ * stdin_fd, stdout_fd, and stderr_fd are file descriptors to be duped and
+ * passed to the new unit.  The dup should be complete on first fulfillment
+ * of the future and local copies can be closed at that time.  Set to -1 to
+ * indicate that systemd should manage a particular stdio stream.
+ *
+ * Service unit properties may be set with command options prefixed with
+ * SDEXEC_PROP_.  The following unit properties are explicitly parsed and
+ * converted to their native types:
+ *
+ * MemoryHigh, MemoryMax, MemoryLow, MemoryMin
+ *   Value may be "infinity", a percentage of physical memory ("98%"),
+ *   or a quantity with optional base 1024 K, M, G, or T suffix ("8g").
+ *   See also: systemd.resource-control(5).
+ *
+ * AllowedCPUs
+ *   Restrict execution to specific CPUs. Value is a Flux idset representing
+ *   a list of CPU indices.
+ *   See also: systemd.resource-control(5).
+ *
+ * Other service unit properties are assumed to be of type string and are
+ * set without conversion.  For example, Description may be set to a string
+ * that is shown in list-units output.
+ */
+flux_future_t *sdexec_start_transient_unit (flux_t *h,
+                                            uint32_t rank,
+                                            const char *mode,
+                                            const char *type,
+                                            json_t *cmd,
+                                            int stdin_fd,
+                                            int stdout_fd,
+                                            int stderr_fd,
+                                            flux_error_t *error);
+
+int sdexec_start_transient_unit_get (flux_future_t *f, const char **job);
+
+#endif /* !_LIBSDEXEC_START_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/state.c b/src/common/libsdexec/state.c
new file mode 100644
index 000000000000..80a097b6889c
--- /dev/null
+++ b/src/common/libsdexec/state.c
@@ -0,0 +1,88 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* state.c - unit state/substate
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+#include "state.h"
+
+struct state_tab {
+    const char *name;
+    int state;
+};
+
+static struct state_tab states[] = {
+    { "unknown", STATE_UNKNOWN },
+    { "activating", STATE_ACTIVATING },
+    { "active", STATE_ACTIVE },
+    { "deactivating", STATE_DEACTIVATING },
+    { "inactive", STATE_INACTIVE },
+    { "failed", STATE_FAILED },
+};
+
+static struct state_tab substates[] = {
+    { "unknown", SUBSTATE_UNKNOWN },
+    { "dead", SUBSTATE_DEAD },
+    { "start", SUBSTATE_START },
+    { "running", SUBSTATE_RUNNING },
+    { "exited", SUBSTATE_EXITED },
+    { "failed", SUBSTATE_FAILED },
+};
+
+static int strtostate (const char *s,
+                       struct state_tab *tab,
+                       size_t len)
+{
+    if (s) {
+        for (int i = 0; i < len; i++) {
+            if (streq (tab[i].name, s))
+                return tab[i].state;
+        }
+    }
+    return tab[0].state;
+}
+
+sdexec_state_t sdexec_strtostate (const char *s)
+{
+    return strtostate (s, states, ARRAY_SIZE (states));
+}
+
+sdexec_substate_t sdexec_strtosubstate (const char *s)
+{
+    return strtostate (s, substates, ARRAY_SIZE (substates));
+}
+
+static const char *statetostr (int state, struct state_tab *tab, size_t len)
+{
+    for (int i = 0; i < len; i++) {
+        if (tab[i].state == state)
+            return tab[i].name;
+    }
+    return tab[0].name;
+}
+
+const char *sdexec_statetostr (sdexec_state_t state)
+{
+    return statetostr (state, states, ARRAY_SIZE (states));
+}
+
+const char *sdexec_substatetostr (sdexec_substate_t substate)
+{
+    return statetostr (substate, substates, ARRAY_SIZE (substates));
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/state.h b/src/common/libsdexec/state.h
new file mode 100644
index 000000000000..f1ec3c4ad631
--- /dev/null
+++ b/src/common/libsdexec/state.h
@@ -0,0 +1,40 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSDEXEC_STATE_H
+#define _LIBSDEXEC_STATE_H
+
+typedef enum {
+    STATE_UNKNOWN,
+    STATE_INACTIVE,
+    STATE_ACTIVATING,
+    STATE_ACTIVE,
+    STATE_DEACTIVATING,
+    STATE_FAILED,
+} sdexec_state_t;
+
+typedef enum {
+    SUBSTATE_UNKNOWN,
+    SUBSTATE_DEAD,
+    SUBSTATE_START,
+    SUBSTATE_RUNNING,
+    SUBSTATE_EXITED,
+    SUBSTATE_FAILED,
+} sdexec_substate_t;
+
+const char *sdexec_statetostr (sdexec_state_t state);
+const char *sdexec_substatetostr (sdexec_substate_t substate);
+
+sdexec_state_t sdexec_strtostate (const char *s);
+sdexec_substate_t sdexec_strtosubstate (const char *s);
+
+#endif /* !_LIBSDEXEC_STATE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/stop.c b/src/common/libsdexec/stop.c
new file mode 100644
index 000000000000..924d978bb24d
--- /dev/null
+++ b/src/common/libsdexec/stop.c
@@ -0,0 +1,76 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* stop.c - reset/stop units
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "stop.h"
+
+flux_future_t *sdexec_stop_unit (flux_t *h,
+                                 uint32_t rank,
+                                 const char *name,
+                                 const char *mode)
+{
+    if (!h || !name || !mode) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return flux_rpc_pack (h,
+                          "sdbus.call",
+                          rank,
+                          0,
+                          "{s:s s:[ss]}",
+                          "member", "StopUnit",
+                          "params", name, mode);
+}
+
+flux_future_t *sdexec_reset_failed_unit (flux_t *h,
+                                         uint32_t rank,
+                                         const char *name)
+{
+    if (!h || !name) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return flux_rpc_pack (h,
+                          "sdbus.call",
+                          rank,
+                          0,
+                          "{s:s s:[s]}",
+                          "member", "ResetFailedUnit",
+                          "params", name);
+}
+
+flux_future_t *sdexec_kill_unit (flux_t *h,
+                                 uint32_t rank,
+                                 const char *name,
+                                 const char *who,
+                                 int signum)
+{
+    if (!h || !name || !who ) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return flux_rpc_pack (h,
+                          "sdbus.call",
+                          rank,
+                          0,
+                          "{s:s s:[ssi]}",
+                          "member", "KillUnit",
+                          "params", name, who, signum);
+
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/stop.h b/src/common/libsdexec/stop.h
new file mode 100644
index 000000000000..12a34fba256d
--- /dev/null
+++ b/src/common/libsdexec/stop.h
@@ -0,0 +1,37 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSDEXEC_STOP_H
+#define _LIBSDEXEC_STOP_H
+
+#include <flux/core.h>
+
+/* See https://www.freedesktop.org/wiki/Software/systemd/dbus/
+ * for more info on mode parameter.
+ * mode maybe one of: replace, fail, ignore-dependencies, ignore-requirements.
+ */
+flux_future_t *sdexec_stop_unit (flux_t *h,
+                                 uint32_t rank,
+                                 const char *name,
+                                 const char *mode);
+
+flux_future_t *sdexec_reset_failed_unit (flux_t *h,
+                                         uint32_t rank,
+                                         const char *name);
+
+flux_future_t *sdexec_kill_unit (flux_t *h,
+                                 uint32_t rank,
+                                 const char *name,
+                                 const char *who, // main/control/all
+                                 int signum);
+
+#endif /* !_LIBSDEXEC_STOP_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/test/channel.c b/src/common/libsdexec/test/channel.c
new file mode 100644
index 000000000000..8d3444596c82
--- /dev/null
+++ b/src/common/libsdexec/test/channel.c
@@ -0,0 +1,294 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "ccan/array_size/array_size.h"
+#include "src/common/libtap/tap.h"
+#include "src/common/libioencode/ioencode.h"
+#include "channel.h"
+
+bool input_called;
+bool input_eof_set;
+
+void input_cb (flux_reactor_t *r,
+               flux_watcher_t *w,
+               int revents,
+               void *arg)
+{
+    struct channel *ch = arg;
+    const char *name = sdexec_channel_get_name (ch);
+    int fd = sdexec_channel_get_fd (ch);
+    char buf[64];
+    int n;
+
+    n = read (fd, buf, sizeof (buf));
+    if (n < 0) {
+        diag ("%s: read error: %s", name, strerror (errno));
+    }
+    else if (n == 0) {
+        diag ("%s: EOF", name);
+        input_eof_set = true;
+    }
+    else
+        diag ("%s: read %d chars", name, n);
+    input_called = true;
+}
+
+void test_input (void)
+{
+    flux_t *h;
+    struct channel *ch;
+    flux_watcher_t *w;
+    int fd;
+    json_t *io;
+    json_t *io_eof;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop flux_t handle for testing");
+    if (flux_attr_set_cacheonly (h, "rank", "0") < 0)
+        BAIL_OUT ("could not set rank for testing");
+    ch = sdexec_channel_create_input (h, "in");
+    ok (ch != NULL,
+        "sdexec_channel_create_input works");
+    fd = sdexec_channel_get_fd (ch);
+    ok (fd >= 0,
+        "sdexec_channel_get_fd works");
+    w = flux_fd_watcher_create (flux_get_reactor (h),
+                                fd,
+                                FLUX_POLLIN,
+                                input_cb,
+                                ch);
+    if (!w)
+        BAIL_OUT ("could not create fd watcher");
+    if (!(io = ioencode ("foo", "0", "hello", 6, false)))
+        BAIL_OUT ("could not create json io object");
+    if (!(io_eof = ioencode ("foo", "0", NULL, 0, true)))
+        BAIL_OUT ("could not create json io_eof object");
+    flux_watcher_start (w);
+
+    input_called = false;
+    input_eof_set = false;
+    ok (sdexec_channel_write (ch, io) == 0,
+        "sdexec_channel_write works");
+    ok (flux_reactor_run (flux_get_reactor (h), FLUX_REACTOR_ONCE) >= 0,
+        "flux_reactor_run ran ONCE");
+    ok (input_called == true,
+        "input callback was called");
+    ok (input_eof_set == false,
+        "eof was not set");
+
+    input_called = false;
+    input_eof_set = false;
+    ok (sdexec_channel_write (ch, io_eof) == 0,
+        "sdexec_channel_write eof works");
+    ok (flux_reactor_run (flux_get_reactor (h), FLUX_REACTOR_ONCE) >= 0,
+        "flux_reactor_run ran ONCE");
+    ok (input_called == true,
+        "input callback was called");
+    ok (input_eof_set == true,
+        "eof was set");
+
+    json_decref (io_eof);
+    json_decref (io);
+    flux_watcher_destroy (w);
+    sdexec_channel_destroy (ch);
+    flux_close (h);
+}
+
+bool error_called;
+bool output_called;
+bool output_eof_set;
+
+void output_cb (struct channel *ch, json_t *io, void *arg)
+{
+    const char *stream;
+    int len;
+
+    if (iodecode (io, &stream, NULL, NULL, &len, &output_eof_set) < 0) {
+        diag ("%s: idoecode error: %s",
+              sdexec_channel_get_name (ch),
+              strerror (errno));
+    }
+    else {
+        diag ("%s output: stream=%s len=%d eof=%s",
+              sdexec_channel_get_name (ch),
+              stream,
+              len,
+              output_eof_set ? "true" : "false");
+    }
+    output_called = true;
+}
+
+void error_cb (struct channel *ch, flux_error_t *error, void *arg)
+{
+    diag ("%s error: %s",
+          sdexec_channel_get_name (ch),
+          error->text);
+    error_called = true;
+}
+
+void test_output (void)
+{
+    flux_t *h;
+    struct channel *ch;
+    int fd;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop flux_t handle for testing");
+    if (flux_attr_set_cacheonly (h, "rank", "0") < 0)
+        BAIL_OUT ("could not set rank for testing");
+    ch = sdexec_channel_create_output (h, "out", 0, 0, output_cb, error_cb, NULL);
+    ok (ch != NULL,
+        "sdexec_channel_crate_output works");
+    sdexec_channel_start_output (ch);
+    ok (true, "sdexec_channel_start_output called");
+
+    fd = sdexec_channel_get_fd (ch);
+    ok (fd >= 0,
+        "sdexec_channel_get_fd works");
+
+    output_called = false;
+    error_called = false;
+    output_eof_set = false;
+    ok (write (fd, "hello", 6) == 6,
+        "wrote 'hello' from unit");
+    ok (flux_reactor_run (flux_get_reactor (h), FLUX_REACTOR_ONCE) >= 0,
+        "flux_reactor_run ran ONCE");
+    ok (output_called == true,
+        "output callback was called");
+    ok (output_eof_set == false,
+        "eof was not set");
+    ok (error_called == false,
+        "error callback was not called");
+
+    output_called = false;
+    error_called = false;
+    output_eof_set = false;
+    sdexec_channel_close_fd (ch);
+    ok (true, "sdexec_channel_close_fd called");
+    ok (flux_reactor_run (flux_get_reactor (h), FLUX_REACTOR_ONCE) >= 0,
+        "flux_reactor_run ran ONCE");
+    ok (output_called == true,
+        "output callback was called");
+    ok (output_eof_set == true,
+        "eof was set");
+    ok (error_called == false,
+        "error callback was not called");
+
+    sdexec_channel_destroy (ch);
+    flux_close (h);
+}
+
+void destroy_output_cb (struct channel *ch, json_t *io, void *arg)
+{
+    sdexec_channel_destroy (ch);
+}
+
+void test_output_destroy_in_cb (void)
+{
+    flux_t *h;
+    struct channel *ch;
+    int fd;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop flux_t handle for testing");
+    if (flux_attr_set_cacheonly (h, "rank", "0") < 0)
+        BAIL_OUT ("could not set rank for testing");
+    ch = sdexec_channel_create_output (h,
+                                       "out",
+                                       0,
+                                       0, destroy_output_cb,
+                                       error_cb,
+                                       NULL);
+    ok (ch != NULL,
+        "sdexec_channel_crate_output works");
+    sdexec_channel_start_output (ch);
+    ok (true, "sdexec_channel_start_output called");
+
+    fd = sdexec_channel_get_fd (ch);
+    ok (fd >= 0,
+        "sdexec_channel_get_fd works");
+    sdexec_channel_close_fd (ch);
+    ok (true, "sdexec_channel_close_fd called");
+    ok (flux_reactor_run (flux_get_reactor (h), FLUX_REACTOR_ONCE) >= 0,
+        "flux_reactor_run ran ONCE");
+    flux_close (h);
+}
+
+void test_inval (void)
+{
+    struct channel *ch;
+    flux_t *h;
+    json_t *io;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop flux_t handle for testing");
+    if (!(io = ioencode ("foo", "0", NULL, 0, true)))
+        BAIL_OUT ("could not create json io object");
+
+    errno = 0;
+    ch = sdexec_channel_create_output (NULL, "foo", 0, 0, output_cb, error_cb, NULL);
+    ok (ch == NULL && errno == EINVAL,
+        "sdexec_channel_create_output h=NULL fails with EINVAL");
+    errno = 0;
+    ch = sdexec_channel_create_output (h, NULL, 0, 0, output_cb, error_cb, NULL);
+    ok (ch == NULL && errno == EINVAL,
+        "sdexec_channel_create_output name=NULL fails with EINVAL");
+
+    errno = 0;
+    ch = sdexec_channel_create_input (NULL, "foo");
+    ok (ch == NULL && errno == EINVAL,
+        "sdexec_channel_create_input h=NULL fails with EINVAL");
+    errno = 0;
+    ch = sdexec_channel_create_input (h, NULL);
+    ok (ch == NULL && errno == EINVAL,
+        "sdexec_channel_create_input name=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (sdexec_channel_write (NULL, io) < 0 && errno == EINVAL,
+        "sdexec_channel_write ch=NULL fails with EINVAL");
+
+    ok (sdexec_channel_get_fd (NULL) == -1,
+        "sdexec_channel_get_fd ch=NULL returns -1");
+
+    ok (sdexec_channel_get_name (NULL) != NULL,
+        "sdexec_channel_get_name ch=NULL returns non-NULL");
+
+    lives_ok ({sdexec_channel_start_output (NULL);},
+              "sdexec_channel_start_output ch=NULL doesn't crash");
+    lives_ok ({sdexec_channel_close_fd (NULL);},
+              "sdexec_channel_close_fd ch=NULL doesn't crash");
+    lives_ok ({sdexec_channel_destroy (NULL);},
+              "sdexec_channel_destroy ch=NULL doesn't crash");
+
+    json_decref (io);
+    flux_close (h);
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    test_input ();
+    test_output ();
+    test_output_destroy_in_cb ();
+    test_inval ();
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/test/channel_outbuf.c b/src/common/libsdexec/test/channel_outbuf.c
new file mode 100644
index 000000000000..ba57b48d4beb
--- /dev/null
+++ b/src/common/libsdexec/test/channel_outbuf.c
@@ -0,0 +1,246 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "ccan/array_size/array_size.h"
+#include "src/common/libtap/tap.h"
+#include "src/common/libioencode/ioencode.h"
+#include "channel.h"
+
+bool error_called;
+
+void error_cb (struct channel *ch, flux_error_t *error, void *arg)
+{
+    diag ("%s error: %s",
+          sdexec_channel_get_name (ch),
+          error->text);
+    error_called = true;
+}
+
+size_t raw_byte_count;
+
+void raw_output_cb (struct channel *ch, json_t *io, void *arg)
+{
+    flux_reactor_t *r = arg;
+    const char *stream;
+    int len;
+    bool eof;
+
+    if (iodecode (io, &stream, NULL, NULL, &len, &eof) < 0) {
+        diag ("%s: idoecode error: %s",
+              sdexec_channel_get_name (ch),
+              strerror (errno));
+    }
+    else {
+        diag ("%s output: stream=%s len=%d eof=%s",
+              sdexec_channel_get_name (ch),
+              stream,
+              len,
+              eof ? "true" : "false");
+        raw_byte_count += len;
+        if (eof)
+            flux_reactor_stop (r);
+    }
+}
+
+void test_raw (size_t bufsize, size_t datasize)
+{
+    flux_t *h;
+    struct channel *ch;
+    int fd;
+    char buf[datasize];
+    flux_reactor_t *r;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop flux_t handle for testing");
+    r = flux_get_reactor (h);
+    if (flux_attr_set_cacheonly (h, "rank", "0") < 0)
+        BAIL_OUT ("could not set rank for testing");
+    ch = sdexec_channel_create_output (h,
+                                       "raw",
+                                       bufsize,
+                                       0,
+                                       raw_output_cb,
+                                       error_cb,
+                                       r);
+    ok (ch != NULL,
+        "sdexec_channel_crate_output works");
+    sdexec_channel_start_output (ch);
+    ok (true, "sdexec_channel_start_output called");
+
+    fd = sdexec_channel_get_fd (ch);
+    ok (fd >= 0,
+        "sdexec_channel_get_fd works");
+
+    raw_byte_count = 0;
+    error_called = false;
+    // write can exceed TINY_OUTBUF size by approx O(PAGE_SIZE) socket buf
+    memset (buf, 'x', sizeof (buf));
+    ok (write (fd, buf, sizeof (buf)) == sizeof (buf),
+        "wrote %zu bytes of data from unit", sizeof (buf));
+    sdexec_channel_close_fd (ch);
+    ok (true, "sdexec_channel_close_fd called");
+    ok (flux_reactor_run (r, 0) >= 0,
+        "flux_reactor_run ran successfully");
+    ok (error_called == false,
+        "error callback was not called");
+    ok (raw_byte_count == sizeof (buf),
+        "all bytes were received");
+
+    sdexec_channel_destroy (ch);
+    flux_close (h);
+}
+
+size_t line_byte_count;
+size_t line_count;
+size_t line_calls;
+
+void line_output_cb (struct channel *ch, json_t *io, void *arg)
+{
+    flux_reactor_t *r = arg;
+    const char *stream;
+    char *data = NULL;
+    int len;
+    bool eof;
+
+    if (iodecode (io, &stream, NULL, &data, &len, &eof) < 0) {
+        diag ("%s: idoecode error: %s",
+              sdexec_channel_get_name (ch),
+              strerror (errno));
+    }
+    else {
+        diag ("%s output: stream=%s len=%d eof=%s",
+              sdexec_channel_get_name (ch),
+              stream,
+              len,
+              eof ? "true" : "false");
+        line_byte_count += len;
+        int count = 0;
+        for (int i = 0; i < len; i++) {
+            if (data[i] == '\n')
+                count++;
+        }
+        line_count += count;
+        line_calls++;
+        free (data);
+        if (eof)
+            flux_reactor_stop (r);
+    }
+}
+
+void test_line (size_t bufsize, size_t linelength, size_t datasize)
+{
+    flux_t *h;
+    struct channel *ch;
+    int fd;
+    char buf[linelength];
+    flux_reactor_t *r;
+    size_t dataused;
+
+    diag ("line test with bufsize=%zu linelength=%zu datasize=%zu",
+          bufsize,
+          linelength,
+          datasize);
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop flux_t handle for testing");
+    r = flux_get_reactor (h);
+    if (flux_attr_set_cacheonly (h, "rank", "0") < 0)
+        BAIL_OUT ("could not set rank for testing");
+    ch = sdexec_channel_create_output (h,
+                                       "line",
+                                       bufsize,
+                                       CHANNEL_LINEBUF,
+                                       line_output_cb,
+                                       error_cb,
+                                       r);
+    ok (ch != NULL,
+        "sdexec_channel_crate_output works");
+    sdexec_channel_start_output (ch);
+    ok (true, "sdexec_channel_start_output called");
+
+    fd = sdexec_channel_get_fd (ch);
+    ok (fd >= 0,
+        "sdexec_channel_get_fd works");
+
+    line_byte_count = 0;
+    line_count = 0;
+    line_calls = 0;
+    error_called = false;
+    memset (buf, 'x', sizeof (buf) - 1);
+    buf[sizeof (buf) - 1] = '\n';
+    dataused = 0;
+    while (dataused < datasize) {
+        size_t len = sizeof (buf);
+        if (len > datasize - dataused)
+            len = datasize - dataused;
+        // write can exceed TINY_OUTBUF size by approx O(PAGE_SIZE) socket buf
+        ok (write (fd, buf, len) == len,
+            "wrote %zu bytes of data from unit", len);
+        dataused += len;
+    }
+    sdexec_channel_close_fd (ch);
+    ok (true, "sdexec_channel_close_fd called");
+    ok (flux_reactor_run (r, 0) >= 0,
+        "flux_reactor_run ran successfully");
+    ok (error_called == false,
+        "error callback was not called");
+    ok (line_byte_count == datasize,
+        "all bytes were received");
+
+    diag ("lines %zu calls %zu", line_count, line_calls);
+    size_t expected_line_count = datasize / linelength;
+    size_t expected_calls = expected_line_count;
+
+    /* If the lines are larger than the buffer, then each full line will
+     * be transmitted in 2 callback - first one buffer's worth, then the
+     * terminated fragment.  This assumes linelength is at most bufsize*2.
+     */
+    if (bufsize < linelength)
+        expected_calls *= 2;
+
+    /* The final "line" isn't terminated if datasize is not a multiple of
+     * the linelength. The callback will get that + eof in one go.  Otherwise,
+     * the eof will come through on its own.  Either way, one extra call.
+     */
+    expected_calls += 1;
+
+    ok (line_count == expected_line_count,
+        "expected number of lines (%zu) were received", expected_line_count);
+    ok (line_calls == expected_calls,
+        "expected number of callbacks (%zu) were made", expected_calls);
+
+    sdexec_channel_destroy (ch);
+    flux_close (h);
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    test_raw (16, 47);
+    test_raw (4096, 3000);
+    test_raw (4096, 6000);
+
+    test_line (16, 4, 64); // 16 lines that fit perfectly
+    test_line (16, 4, 63); // 15 lines + last one truncated
+    test_line (15, 16, 32); // 2 lines split into 4 callbacks (short buffer)
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/test/list.c b/src/common/libsdexec/test/list.c
new file mode 100644
index 000000000000..3653a8e39657
--- /dev/null
+++ b/src/common/libsdexec/test/list.c
@@ -0,0 +1,58 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "list.h"
+
+void test_inval (void)
+{
+    flux_t *h;
+    struct unit_info info;
+    flux_future_t *f;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop flux_t handle for testing");
+    if (!(f = flux_future_create (NULL, 0)))
+        BAIL_OUT ("could not create future for testing");
+
+    errno = 0;
+    ok (sdexec_list_units (NULL, 0, "*") == NULL && errno == EINVAL,
+        "sdexec_list_units h=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_list_units (h, 0, NULL) == NULL && errno == EINVAL,
+        "sdexec_list_units pattern=NULL fails with EINVAL");
+
+    ok (sdexec_list_units_next (NULL, &info) == false,
+        "sdexec_list_units_next f=NULL returns false");
+    ok (sdexec_list_units_next (f, NULL) == false,
+        "sdexec_list_units_next infop=NULL returns false");
+
+    flux_future_destroy (f);
+    flux_close (h);
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    test_inval ();
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/test/parse.c b/src/common/libsdexec/test/parse.c
new file mode 100644
index 000000000000..55b4e9952e88
--- /dev/null
+++ b/src/common/libsdexec/test/parse.c
@@ -0,0 +1,128 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <stdbool.h>
+
+#include "ccan/array_size/array_size.h"
+#include "src/common/libtap/tap.h"
+#include "parse.h"
+
+struct tab_percent {
+    const char *s;
+    double val;
+    bool ok;
+};
+const struct tab_percent ptab[] = {
+    // bad
+    { "10", 0, false },
+    { "10%x", 0, false },
+    { "-10%", 0, false },
+    { "", 0, false },
+    { "%", 0, false },
+    { "x%", 0, false },
+    { "110%", 0, false },
+    // good
+    { "0%", 0, true},
+    { "10%", 0.1, true},
+    { "50%", 0.5, true },
+    { "100%", 1, true },
+};
+
+void test_percent (void)
+{
+    double d;
+    int rc;
+
+    lives_ok ({sdexec_parse_percent (NULL, &d);},
+        "sdexec_parse_percent input=NULL doesn't crash");
+    lives_ok ({sdexec_parse_percent ("x", NULL);},
+        "sdexec_parse_percent value=NULL doesn't crash");
+
+    for (int i = 0; i < ARRAY_SIZE (ptab); i++) {
+        d = 0;
+        rc = sdexec_parse_percent (ptab[i].s, &d);
+        if (ptab[i].ok) {
+            ok (rc == 0 && d == ptab[i].val,
+                "sdexec_parse_percent val=%s works", ptab[i].s);
+        }
+        else {
+            ok (rc == -1,
+                "sdexec_parse_percent val=%s fails", ptab[i].s);
+        }
+    }
+}
+
+struct tab_bitmap {
+    const char *s;
+    uint8_t val[4];
+    size_t val_size;
+    bool ok;
+};
+const struct tab_bitmap btab[] = {
+    // bad
+    { "1-",             { 0, 0, 0, 0 },     0, false },
+    { "x",              { 0, 0, 0, 0 },     0, false },
+    // good
+    { "",               { 0, 0, 0, 0 },     0, true },
+    { "0",              { 1, 0, 0, 0 },     1, true },
+    { "0-2,8",          { 7, 1, 0, 0 },     2, true },
+    { "8-15,16-23",     { 0, 255, 255, 0 }, 3, true },
+};
+
+void test_bitmap (void)
+{
+    uint8_t *bitmap;
+    size_t size;
+    int rc;
+
+    lives_ok ({sdexec_parse_bitmap (NULL, &bitmap, &size);},
+        "sdexec_parse_bitmap input=NULL doesn't crash");
+    lives_ok ({sdexec_parse_bitmap ("0", NULL, &size);},
+        "sdexec_parse_bitmap bitmap=NULL doesn't crash");
+    lives_ok ({sdexec_parse_bitmap ("0", &bitmap, NULL);},
+        "sdexec_parse_bitmap size=NULL doesn't crash");
+
+    for (int i = 0; i < ARRAY_SIZE (btab); i++) {
+        bitmap = NULL;
+        size = 0;
+        rc = sdexec_parse_bitmap (btab[i].s, &bitmap, &size);
+        if (btab[i].ok) {
+            ok (rc == 0
+                && size == btab[i].val_size
+                && (size < 1 || bitmap[0] == btab[i].val[0])
+                && (size < 2 || bitmap[1] == btab[i].val[1])
+                && (size < 3 || bitmap[2] == btab[i].val[2])
+                && (size < 4 || bitmap[3] == btab[i].val[3]),
+                "sdexec_parse_bitmap val=%s works", btab[i].s);
+        }
+        else {
+            ok (rc == -1,
+                "sdexec_parse_bitmap val=%s fails", btab[i].s);
+        }
+        free (bitmap);
+    }
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    test_percent ();
+    test_bitmap ();
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/test/property.c b/src/common/libsdexec/test/property.c
new file mode 100644
index 000000000000..e928ccbc199d
--- /dev/null
+++ b/src/common/libsdexec/test/property.c
@@ -0,0 +1,120 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "property.h"
+
+void test_dict (void)
+{
+    json_t *dict;
+    int val;
+
+    if (!(dict = json_pack ("{s:[si]}", "foo", "i", 42)))
+        BAIL_OUT ("could not create property dict for testing");
+
+    ok (sdexec_property_dict_unpack (dict, "foo", "i", &val) == 0
+        && val == 42,
+        "sdexec_property_dict_unpack works");
+    errno = 0;
+    ok (sdexec_property_dict_unpack (dict, "unknown", "i", &val) < 0
+        && errno == EPROTO,
+        "sdexec_property_dict_unpack name=unknown fails with EPROTO");
+
+    json_decref (dict);
+}
+
+void test_inval (void)
+{
+    flux_t *h;
+    flux_future_t *f;
+    json_t *dict;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop flux_t handle for testing");
+    if (!(f = flux_future_create (NULL, 0)))
+        BAIL_OUT ("could not create future for testing");
+    if (!(dict = json_pack ("{s:[si]}", "foo", "i", 42)))
+        BAIL_OUT ("could not create property dict for testing");
+
+    errno = 0;
+    ok (sdexec_property_get (NULL, 0, "foo", "bar") == NULL && errno == EINVAL,
+        "sdexec_property_get h=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_property_get (h, 0, NULL, "bar") == NULL && errno == EINVAL,
+        "sdexec_property_get path=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_property_get (h, 0, "foo", NULL) == NULL && errno == EINVAL,
+        "sdexec_property_get name=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (sdexec_property_get_unpack (NULL, "foo") < 0 && errno == EINVAL,
+        "sdexec_property_get_unpack f=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_property_get_unpack (f, NULL) < 0 && errno == EINVAL,
+        "sdexec_property_get_unpack fmt=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (sdexec_property_get_all (NULL, 0, "foo") == NULL && errno == EINVAL,
+        "sdexec_property_get_all h=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_property_get_all (h, 0, NULL) == NULL && errno == EINVAL,
+        "sdexec_property_get_all path=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (sdexec_property_get_all_dict (NULL) == NULL && errno == EINVAL,
+        "sdexec_property_get_all_dict f=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (sdexec_property_changed (NULL, 0, "foo") == NULL && errno == EINVAL,
+        "sdexec_property_changed h=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (sdexec_property_changed_dict (NULL) == NULL && errno == EINVAL,
+        "sdexec_property_changed_dict f=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_property_changed_path (NULL) == NULL && errno == EINVAL,
+        "sdexec_property_changed_path f=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (sdexec_property_dict_unpack (NULL, "foo", "bar") < 0
+        && errno == EINVAL,
+        "sdexec_property_dict_unpack dict=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_property_dict_unpack (dict, NULL, "bar") < 0
+        && errno == EINVAL,
+        "sdexec_property_dict_unpack name=NULL fails with EINVAL");
+    ok (sdexec_property_dict_unpack (dict, "foo", NULL) < 0
+        && errno == EINVAL,
+        "sdexec_property_dict_unpack fmt=NULL fails with EINVAL");
+
+    json_decref (dict);
+    flux_future_destroy (f);
+    flux_close (h);
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    test_dict ();
+    test_inval ();
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/test/start.c b/src/common/libsdexec/test/start.c
new file mode 100644
index 000000000000..9f136529242c
--- /dev/null
+++ b/src/common/libsdexec/test/start.c
@@ -0,0 +1,154 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libsubprocess/command_private.h"
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/jpath.h"
+#include "start.h"
+
+extern char **environ;
+
+void test_inval (void)
+{
+    char *av[] = { "/bin/ls", NULL };
+    int ac = 1;
+    flux_t *h;
+    flux_future_t *f;
+    flux_cmd_t *cmd;
+    json_t *cmd_o = NULL;
+    json_t *cmd_o_noname;
+    json_t *cmd_o_badprop;
+    json_t *o;
+    flux_error_t error;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop flux_t handle for testing");
+    if (!(f = flux_future_create (NULL, 0)))
+        BAIL_OUT ("could not create future for testing");
+    if (!(cmd = flux_cmd_create (ac, av, environ))
+        || flux_cmd_setopt (cmd, "SDEXEC_NAME", "foo") < 0
+        || !(cmd_o = cmd_tojson (cmd)))
+        BAIL_OUT ("could not create command object for testing");
+    if (!(cmd_o_noname = json_deep_copy (cmd_o))
+        || jpath_del (cmd_o_noname, "opts.SDEXEC_NAME") < 0)
+        BAIL_OUT ("error preparing test command without SDEXEC_NAME");
+    if (!(cmd_o_badprop = json_deep_copy (cmd_o))
+        || !(o = json_string ("badvalue"))
+        || jpath_set_new (cmd_o_badprop, "opts.SDEXEC_PROP_MemoryMax", o) < 0)
+        BAIL_OUT ("error preparing test command with bad property");
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok (sdexec_start_transient_unit (NULL,      // h
+                                     0,         // rank
+                                     "fail",    // mode
+                                     "simple",  // type
+                                     cmd_o,     // cmd
+                                     -1, -1, -1, // *_fd
+                                     &error) == NULL
+        && errno == EINVAL,
+        "sdexec_start_transient_unit h=NULL fails with EINVAL");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok (sdexec_start_transient_unit (h,         // h
+                                     0,         // rank
+                                     NULL,      // mode
+                                     "simple",  // type
+                                     cmd_o,     // cmd
+                                     -1, -1, -1, // *_fd
+                                     &error) == NULL
+        && errno == EINVAL,
+        "sdexec_start_transient_unit mode=NULL fails with EINVAL");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok (sdexec_start_transient_unit (h,         // h
+                                     0,         // rank
+                                     "fail",    // mode
+                                     NULL,      // type
+                                     cmd_o,     // cmd
+                                     -1, -1, -1, // *_fd
+                                     &error) == NULL
+        && errno == EINVAL,
+        "sdexec_start_transient_unit type=NULL fails with EINVAL");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok (sdexec_start_transient_unit (h,         // h
+                                     0,         // rank
+                                     "fail",    // mode
+                                     "simple",  // type
+                                     NULL,      // cmd
+                                     -1, -1, -1, // *_fd
+                                     &error) == NULL
+        && errno == EINVAL,
+        "sdexec_start_transient_unit cmd=NULL fails with EINVAL");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok (sdexec_start_transient_unit (h,         // h
+                                     0,         // rank
+                                     "fail",    // mode
+                                     "simple",  // type
+                                     cmd_o_noname, // cmd
+                                     -1, -1, -1, // *_fd
+                                     &error) == NULL
+        && errno == EINVAL,
+        "sdexec_start_transient_unit missing SDEXEC_NAME fails with EINVAL");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    ok (sdexec_start_transient_unit (h,         // h
+                                     0,         // rank
+                                     "fail",    // mode
+                                     "simple",  // type
+                                     cmd_o_badprop, // cmd
+                                     -1, -1, -1, // *_fd
+                                     &error) == NULL
+        && errno == EINVAL,
+        "sdexec_start_transient_unit with bad property fails with EINVAL");
+    diag ("%s", error.text);
+
+    errno = 0;
+    ok (sdexec_start_transient_unit_get (NULL, NULL) < 0 && errno == EINVAL,
+        "sdexec_start_transient_unit_get f=NULL fails with EINVAL");
+
+    json_decref (cmd_o_badprop);
+    json_decref (cmd_o_noname);
+    json_decref (cmd_o);
+    flux_cmd_destroy (cmd);
+    flux_future_destroy (f);
+    flux_close (h);
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    test_inval ();
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/test/state.c b/src/common/libsdexec/test/state.c
new file mode 100644
index 000000000000..b4c568f684ca
--- /dev/null
+++ b/src/common/libsdexec/test/state.c
@@ -0,0 +1,74 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdbool.h>
+#include <string.h>
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "src/common/libtap/tap.h"
+#include "state.h"
+
+struct state_tab {
+    const char *name;
+    int state;
+    bool reverse;
+};
+
+static struct state_tab states[] = {
+    { "unknown", STATE_UNKNOWN, true },
+    { "xyz", STATE_UNKNOWN, false },
+    { NULL, STATE_UNKNOWN, false },
+    { "activating", STATE_ACTIVATING, true },
+    { "active", STATE_ACTIVE, true },
+    { "deactivating", STATE_DEACTIVATING, true },
+    { "inactive", STATE_INACTIVE, true },
+    { "failed", STATE_FAILED, true },
+};
+
+static struct state_tab subs[] = {
+    { "unknown", SUBSTATE_UNKNOWN, true },
+    { "xyz", SUBSTATE_UNKNOWN, false },
+    { NULL, SUBSTATE_UNKNOWN, false },
+    { "dead", SUBSTATE_DEAD, true },
+    { "start", SUBSTATE_START, true },
+    { "running", SUBSTATE_RUNNING, true },
+    { "exited", SUBSTATE_EXITED, true },
+    { "failed", SUBSTATE_FAILED, true },
+};
+
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    for (int i = 0; i < ARRAY_SIZE (states); i++) {
+        ok (sdexec_strtostate (states[i].name) == states[i].state,
+            "sdexec_strtostate %s works", states[i].name);
+        if (states[i].reverse) {
+            ok (streq (sdexec_statetostr (states[i].state), states[i].name),
+                "sdexec_statetostr %s works", states[i].name);
+        }
+    }
+    for (int i = 0; i < ARRAY_SIZE (subs); i++) {
+        ok (sdexec_strtosubstate (subs[i].name) == subs[i].state,
+            "sdexec_strtosubstate %s works", subs[i].name);
+        if (subs[i].reverse) {
+            ok (streq (sdexec_substatetostr (subs[i].state), subs[i].name),
+                "sdexec_substatetostr %s works", subs[i].name);
+        }
+    }
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/test/stop.c b/src/common/libsdexec/test/stop.c
new file mode 100644
index 000000000000..25c8188e35a9
--- /dev/null
+++ b/src/common/libsdexec/test/stop.c
@@ -0,0 +1,68 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "stop.h"
+
+void test_inval (void)
+{
+    flux_t *h;
+
+    if (!(h = flux_open ("loop://", 0)))
+        BAIL_OUT ("could not create loop flux_t handle for testing");
+
+    errno = 0;
+    ok (sdexec_stop_unit (NULL, 0, "foo", "bar") == NULL && errno == EINVAL,
+        "sdexec_stop_unit h=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_stop_unit (h, 0, NULL, "bar") == NULL && errno == EINVAL,
+        "sdexec_stop_unit name=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_stop_unit (h, 0, "foo", NULL) == NULL && errno == EINVAL,
+        "sdexec_stop_unit mode=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (sdexec_reset_failed_unit (NULL, 0, "foo") == NULL && errno == EINVAL,
+        "sdexec_reset_failed_unit h=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_reset_failed_unit (h, 0, NULL) == NULL && errno == EINVAL,
+        "sdexec_reset_failed_unit name=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (sdexec_kill_unit (NULL, 0, "foo", "bar", 0) == NULL && errno == EINVAL,
+        "sdexec_kill_unit h=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_kill_unit (h, 0, NULL, "bar", 0) == NULL && errno == EINVAL,
+        "sdexec_kill_unit name=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_kill_unit (h, 0, "foo", NULL, 0) == NULL && errno == EINVAL,
+        "sdexec_kill_unit who=NULL fails with EINVAL");
+
+    flux_close (h);
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    test_inval ();
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/test/unit.c b/src/common/libsdexec/test/unit.c
new file mode 100644
index 000000000000..b61fe2896c40
--- /dev/null
+++ b/src/common/libsdexec/test/unit.c
@@ -0,0 +1,172 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/wait.h>
+#include <string.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+#include "list.h"
+#include "unit.h"
+
+void test_init (void)
+{
+    struct unit *unit;
+    const char *s;
+
+    unit = sdexec_unit_create ("foo.service");
+    ok (unit != NULL,
+        "sdexec_unit_create works");
+    ok (sdexec_unit_state (unit) == STATE_UNKNOWN,
+        "initial state is UNKNOWN");
+    ok (sdexec_unit_substate (unit) == SUBSTATE_UNKNOWN,
+        "initial substate is UNKNOWN");
+    ok (sdexec_unit_pid (unit) == -1,
+        "initial pid is -1");
+    s = sdexec_unit_name (unit);
+    ok (s && streq (s, "foo.service"),
+        "sdexec_unit_name returns original name");
+    s = sdexec_unit_path (unit);
+    ok (s != NULL && streq (s, "/org/freedesktop/systemd1/unit/foo.service"),
+        "sdexec_unit_path returns expected path");
+    ok (sdexec_unit_has_started (unit) == false,
+        "sdexec_unit_has_started returns false");
+    ok (sdexec_unit_has_finished (unit) == false,
+        "sdexec_unit_has_finished returns false");
+    ok (sdexec_unit_wait_status (unit) == -1,
+        "sdexec_unit_wait_status returns -1");
+    ok (sdexec_unit_systemd_error (unit) == -1,
+        "sdexec_unit_systemd_error returns -1");
+
+    sdexec_unit_destroy (unit);
+    ok (true, "sdexec_unit_destroy called");
+}
+
+void test_update (void)
+{
+    struct unit_info info = {
+        .active_state = "active",
+        .sub_state = "start",
+    };
+    struct unit *unit;
+    json_t *dict_pid;
+    json_t *dict_exit;
+
+    if (!(unit = sdexec_unit_create ("foo.service")))
+        BAIL_OUT ("could not create unit object for testing");
+    if (!(dict_pid = json_pack ("{s:[si]}", "ExecMainPID", "I", 42)))
+        BAIL_OUT ("could not create property dict with MainExitPid");
+    if (!(dict_exit = json_pack ("{s:[si] s:[si]}",
+                                 "ExecMainCode", "I", CLD_EXITED,
+                                 "ExecMainStatus", "I", 0)))
+        BAIL_OUT ("could not create property dict with"
+                  " ExecMainCode, ExecMainStatus for testing");
+
+    ok (sdexec_unit_update (unit, dict_pid) == true,
+        "sdexec_unit_update ExecMainPID=42 returns true");
+    ok (sdexec_unit_pid (unit) == 42,
+        "sdexec_unit_pid returns 42");
+
+    ok (sdexec_unit_update_frominfo (unit, &info) == true,
+        "sdexec_unit_update_frominfo active,start returns true");
+    ok (sdexec_unit_has_started (unit) == true,
+        "sdexec_unit_has_started returns true");
+
+    ok (sdexec_unit_update (unit, dict_exit) == true,
+        "sdexec_unit_update ExecMainCode=CLD_EXITED ExecMainStatus=0"
+        " returns true");
+    ok (sdexec_unit_has_finished (unit) == true,
+        "sdexec_unit_has_finished returns true");
+    ok (sdexec_unit_has_failed (unit) == false,
+        "sdexec_unit_has_finished returns true");
+    ok (sdexec_unit_wait_status (unit) == 0,
+        "sdexec_unit_wait_status returns 0");
+
+    json_decref (dict_exit);
+    json_decref (dict_pid);
+    sdexec_unit_destroy (unit);
+}
+
+void test_inval (void)
+{
+    struct unit *unit;
+    json_t *dict;
+    struct unit_info info = {
+        .active_state = "active",
+        .sub_state = "start",
+    };
+
+    if (!(unit = sdexec_unit_create ("foo.service")))
+        BAIL_OUT ("could not create unit object for testing");
+    if (!(dict = json_pack ("{s:[si]}", "foo", "i", 42)))
+        BAIL_OUT ("could not create property dict for testing");
+
+    errno = 0;
+    ok (sdexec_unit_create (NULL) == NULL && errno == EINVAL,
+        "sdexec_unit_create name=NULL fails with EINVAL");
+
+    ok (sdexec_unit_state (NULL) == STATE_UNKNOWN,
+        "sdexec_unit_state unit=NULL is UNKNOWN");
+    ok (sdexec_unit_substate (NULL) == SUBSTATE_UNKNOWN,
+        "sdexec_unit_substate unit=NULL is UNKNOWN");
+    ok (sdexec_unit_name (NULL) != NULL,
+        "sdexec_unit_name unit=NULL returns non-NULL");
+    ok (sdexec_unit_path (NULL) != NULL,
+        "sdexec_unit_path unit=NULL returns non-NULL");
+    ok (sdexec_unit_has_started (NULL) == false,
+        "sdexec_unit_has_started unit=NULL returns false");
+    ok (sdexec_unit_has_finished (NULL) == false,
+        "sdexec_unit_has_finished unit=NULL returns false");
+    ok (sdexec_unit_wait_status (NULL) == -1,
+        "sdexec_unit_wait_status unit=NULL returns -1");
+    ok (sdexec_unit_systemd_error (NULL) == -1,
+        "sdexec_unit_systemd_error unit=NULL returns -1");
+    ok (sdexec_unit_update (NULL, dict) == false,
+        "sdexec_unit_update unit=NULL returns false");
+    ok (sdexec_unit_update (unit, NULL) == false,
+        "sdexec_unit_update dict=NULL returns false");
+
+    ok (sdexec_unit_update_frominfo (NULL, &info) == false,
+        "sdexec_unit_update_frominfo unit=NULL returns false");
+    ok (sdexec_unit_update_frominfo (unit, NULL) == false,
+        "sdexec_unit_update_frominfo info=NULL returns false");
+
+    errno = 0;
+    ok (sdexec_unit_aux_set (NULL, "foo", "bar", NULL) < 0 && errno == EINVAL,
+        "sdexec_unit_aux_set unit=NULL fails with EINVAL");
+    errno = 0;
+    ok (sdexec_unit_aux_get (NULL, "foo") == NULL && errno == EINVAL,
+        "sdexec_unit_aux_get unit=NULL fails with EINVAL");
+
+    lives_ok ({sdexec_unit_destroy (NULL);},
+              "sdexec_unit_destroy unit=NULL doesn't crash");
+
+    json_decref (dict);
+    sdexec_unit_destroy (unit);
+}
+
+int main (int ac, char *av[])
+{
+    plan (NO_PLAN);
+
+    test_init ();
+    test_update ();
+    test_inval ();
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/unit.c b/src/common/libsdexec/unit.c
new file mode 100644
index 000000000000..9ee6b204b6c3
--- /dev/null
+++ b/src/common/libsdexec/unit.c
@@ -0,0 +1,258 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* unit.c - translate unit property updates to unit object changes
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <flux/core.h>
+
+#include "src/common/libmissing/macros.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/aux.h"
+#include "src/common/libutil/basename.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+#include "property.h"
+#include "list.h"
+#include "unit.h"
+
+struct unit {
+    char *path;
+    sdexec_state_t state;
+    sdexec_substate_t substate;
+    pid_t exec_main_pid;
+    int exec_main_code;
+    int exec_main_status;
+    bool exec_main_pid_is_set;
+    bool exec_main_status_is_set;
+
+    struct aux_item *aux;
+};
+
+void sdexec_unit_destroy (struct unit *unit)
+{
+    if (unit) {
+        int saved_errno = errno;
+        aux_destroy (&unit->aux);
+        free (unit->path);
+        free (unit);
+        errno = saved_errno;
+    }
+}
+
+void *sdexec_unit_aux_get (struct unit *unit, const char *name)
+{
+    if (!unit) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return aux_get (unit->aux, name);
+}
+
+int sdexec_unit_aux_set (struct unit *unit,
+                         const char *name,
+                         void *aux,
+                         flux_free_f destroy)
+{
+    if (!unit) {
+        errno = EINVAL;
+        return -1;
+    }
+    return aux_set (&unit->aux, name, aux, destroy);
+}
+
+const char *sdexec_unit_name (struct unit *unit)
+{
+    if (unit)
+        return basename_simple (unit->path);
+    return "internal error: unit is null";
+}
+
+const char *sdexec_unit_path (struct unit *unit)
+{
+    if (unit)
+        return unit->path;
+    return "internal error: unit is null";
+}
+
+pid_t sdexec_unit_pid (struct unit *unit)
+{
+    if (unit && unit->exec_main_pid_is_set)
+        return unit->exec_main_pid;
+    return -1;
+}
+
+sdexec_state_t sdexec_unit_state (struct unit *unit)
+{
+    if (unit)
+        return unit->state;
+    return STATE_UNKNOWN;
+}
+
+sdexec_substate_t sdexec_unit_substate (struct unit *unit)
+{
+    if (unit)
+        return unit->substate;
+    return SUBSTATE_UNKNOWN;
+}
+
+int sdexec_unit_wait_status (struct unit *unit)
+{
+    if (sdexec_unit_has_finished (unit)) {
+        if (unit->exec_main_code == CLD_KILLED)
+            return __W_EXITCODE (0, unit->exec_main_status);
+        else
+            return __W_EXITCODE (unit->exec_main_status, 0);
+    }
+    return -1;
+}
+
+int sdexec_unit_systemd_error (struct unit *unit)
+{
+    if (sdexec_unit_has_failed (unit))
+        return unit->exec_main_status;
+    return -1;
+}
+
+bool sdexec_unit_has_finished (struct unit *unit)
+{
+    if (unit) {
+        if (unit->exec_main_status_is_set &&
+            unit->exec_main_status < 200) // systemd errors are [200-243]
+            return true;
+    }
+    return false;
+}
+
+bool sdexec_unit_has_failed (struct unit *unit)
+{
+    if (unit) {
+        if (unit->exec_main_status_is_set &&
+            unit->exec_main_status >= 200) // systemd errors are [200-243]
+            return true;
+    }
+    return false;
+}
+
+bool sdexec_unit_has_started (struct unit *unit)
+{
+    if (unit) {
+        if (!unit->exec_main_pid_is_set)
+            return false;
+        /* Process was started if it's got an exit status,
+         * unless the exit status is a systemd error [200-243].
+         */
+        if ((unit->exec_main_status_is_set
+            && unit->exec_main_status < 200)
+            || unit->substate == SUBSTATE_START) {
+            return true;
+        }
+    }
+    return false;
+}
+
+struct unit *sdexec_unit_create (const char *name)
+{
+    struct unit *unit;
+
+    if (!name) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(unit = calloc (1, sizeof (*unit))))
+        return NULL;
+    if (asprintf (&unit->path, "/org/freedesktop/systemd1/unit/%s", name) < 0)
+        goto error;
+    unit->state = STATE_UNKNOWN;
+    unit->substate = SUBSTATE_UNKNOWN;
+    return unit;
+error:
+    sdexec_unit_destroy (unit);
+    return NULL;
+}
+
+bool sdexec_unit_update (struct unit *unit, json_t *dict)
+{
+    json_int_t i;
+    json_int_t j;
+    const char *s;
+    int changes = 0;
+
+    if (!unit || !dict)
+        return false;
+
+    /* The pid is for the forked child and so its availability does not
+     * necessarily mean the exec has succeeded.
+     */
+    if (sdexec_property_dict_unpack (dict, "ExecMainPID", "I", &i) == 0
+        && !unit->exec_main_pid_is_set) {
+        unit->exec_main_pid = i;
+        unit->exec_main_pid_is_set = true;
+        changes++;
+    }
+    /* These seem to be set as a pair, and appear early with values of zero,
+     * which is a valid status but not CLD_* code.  So don't set either unless
+     * the code is valid.  On exec failure, code=1 (CLD_EXITED), status=203.
+     */
+    if (sdexec_property_dict_unpack (dict, "ExecMainCode", "I", &i) == 0
+        && sdexec_property_dict_unpack (dict, "ExecMainStatus", "I", &j) == 0
+        && !unit->exec_main_status_is_set
+        && i > 0) {
+        unit->exec_main_code = i;
+        unit->exec_main_status = j;
+        unit->exec_main_status_is_set = true;
+        changes++;
+    }
+    if (sdexec_property_dict_unpack (dict, "SubState", "s", &s) == 0) {
+        sdexec_substate_t substate  = sdexec_strtosubstate (s);
+        if (unit->substate != substate) {
+            unit->substate = substate;
+            changes++;
+        }
+    }
+    if (sdexec_property_dict_unpack (dict, "ActiveState", "s", &s) == 0) {
+        sdexec_state_t state = sdexec_strtostate (s);
+        if (unit->state != state) {
+            unit->state = state;
+            changes++;
+        }
+    }
+    return (changes > 0 ? true : false);
+}
+
+bool sdexec_unit_update_frominfo (struct unit *unit, struct unit_info *info)
+{
+    sdexec_state_t state;
+    sdexec_substate_t substate;
+    int changes = 0;
+
+    if (!unit || !info)
+        return false;
+
+    state = sdexec_strtostate (info->active_state);
+    substate = sdexec_strtosubstate (info->sub_state);
+    if (unit->state != state) {
+        unit->state = state;
+        changes++;
+    }
+    if (unit->substate != substate) {
+        unit->substate = substate;
+        changes++;
+    }
+    return (changes > 0 ? true : false);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsdexec/unit.h b/src/common/libsdexec/unit.h
new file mode 100644
index 000000000000..70ea25f8f24d
--- /dev/null
+++ b/src/common/libsdexec/unit.h
@@ -0,0 +1,64 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSDEXEC_UNIT_H
+#define _LIBSDEXEC_UNIT_H
+
+#include <sys/types.h>
+#include <stdint.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "list.h"
+#include "state.h"
+
+/* Create/destroy a unit object.
+ */
+struct unit *sdexec_unit_create (const char *name);
+void sdexec_unit_destroy (struct unit *unit);
+
+/* Update unit object with property dict from sdexec_property_changed_dict()
+ * or sdexec_property_get_all_dict().  Return true if there was a change,
+ * false if the update was a no-op with respect to the unit object.
+ */
+bool sdexec_unit_update (struct unit *unit, json_t *property_dict);
+
+/* Like above but update unit with info from sdexec_list_units_next()
+ */
+bool sdexec_unit_update_frominfo (struct unit *unit, struct unit_info *info);
+
+/* Attach arbitrary data to unit.
+ */
+void *sdexec_unit_aux_get (struct unit *unit, const char *name);
+int sdexec_unit_aux_set (struct unit *unit,
+                         const char *name,
+                         void *aux,
+                         flux_free_f destroy);
+
+/* accessors */
+sdexec_state_t sdexec_unit_state (struct unit *unit);
+sdexec_substate_t sdexec_unit_substate (struct unit *unit);
+pid_t sdexec_unit_pid (struct unit *unit);
+const char *sdexec_unit_path (struct unit *unit);
+const char *sdexec_unit_name (struct unit *unit);
+
+// returns wait(2) status if unit_has_finished() == true, else -1.
+int sdexec_unit_wait_status (struct unit *unit);
+
+// returns error code if unit_has_failed() == true, else -1
+int sdexec_unit_systemd_error (struct unit *unit);
+
+bool sdexec_unit_has_started (struct unit *unit);
+bool sdexec_unit_has_finished (struct unit *unit);
+bool sdexec_unit_has_failed (struct unit *unit);
+
+#endif /* !_LIBSDEXEC_UNIT_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/Makefile.am b/src/common/libsubprocess/Makefile.am
index 44218625433b..c93a494ae8d4 100644
--- a/src/common/libsubprocess/Makefile.am
+++ b/src/common/libsubprocess/Makefile.am
@@ -6,19 +6,26 @@ AM_LDFLAGS = \
         $(CODE_COVERAGE_LDFLAGS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
+	-DLLOG_ENABLE_DEBUG=1 \
+	$(JANSSON_CFLAGS)
 
 noinst_LTLIBRARIES = \
 	libsubprocess.la
 
 libsubprocess_la_SOURCES = \
 	command.c \
-	command.h \
+	command_private.h \
 	local.c \
 	local.h \
+	fork.c \
+	fork.h \
+	posix_spawn.c \
+	posix_spawn.h \
 	remote.c \
 	remote.h \
 	server.c \
@@ -26,47 +33,142 @@ libsubprocess_la_SOURCES = \
 	util.c \
 	util.h \
 	subprocess.c \
-	subprocess_private.h
+	subprocess_private.h \
+	client.h \
+	client.c \
+	ev_fbuf_read.h \
+	ev_fbuf_read.c \
+	ev_fbuf_write.h \
+	ev_fbuf_write.c \
+	fbuf.h \
+	fbuf.c \
+	fbuf_watcher.h \
+	fbuf_watcher.c \
+	bulk-exec.h \
+	bulk-exec.c
 
 fluxcoreinclude_HEADERS = \
+	command.h \
 	subprocess.h
 
 TESTS = \
-	test_cmd.t \
-	test_subprocess.t
+	test_command.t \
+	test_subprocess.t \
+	test_stdio.t \
+	test_channel.t \
+	test_remote.t \
+	test_iostress.t \
+	test_iochan.t \
+	test_fbuf.t \
+	test_fbuf_watcher.t \
+	test_bulk-exec-einval.t
 
 check_PROGRAMS = \
 	$(TESTS) \
 	test_echo \
 	test_multi_echo \
-	test_fork_sleep
+	test_fork_sleep \
+	test_fdcopy \
+	bulk-exec
+
+check_LTLIBRARIES = test/libutil.la
+
 
 TEST_EXTENSIONS = .t
 T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
         $(top_srcdir)/config/tap-driver.sh
 
 test_ldadd = \
+	$(builddir)/test/libutil.la \
+        $(top_builddir)/src/common/libtestutil/libtestutil.la \
+        $(top_builddir)/src/common/libtap/libtap.la \
         $(top_builddir)/src/common/libsubprocess/libsubprocess.la \
-        $(top_builddir)/src/common/libflux-internal.la \
         $(top_builddir)/src/common/libflux-core.la \
-        $(top_builddir)/src/common/libtap/libtap.la
+        $(top_builddir)/src/common/libflux-internal.la
+
+test_ldflags = \
+	-no-install
 
 test_cppflags = \
         $(AM_CPPFLAGS) \
 	-I$(top_srcdir)/src/common/libtap
 
-test_cmd_t_SOURCES = test/cmd.c
-test_cmd_t_CPPFLAGS = $(test_cppflags)
-test_cmd_t_LDADD = $(test_ldadd)
+test_libutil_la_SOURCES = \
+	test/rcmdsrv.h \
+	test/rcmdsrv.c
+
+bulk_exec_SOURCES = test/bulk-exec.c
+bulk_exec_CPPFLAGS = $(test_cppflags)
+bulk_exec_LDADD = \
+	$(test_ldadd) \
+	$(top_builddir)/src/common/libflux-optparse.la
+
+bulk_exec_LDFLAGS = $(test_ldflags)
+
+test_command_t_SOURCES = test/command.c
+test_command_t_CPPFLAGS = $(test_cppflags)
+test_command_t_LDADD = $(test_ldadd)
+test_command_t_LDFLAGS = $(test_ldflags)
 
 test_subprocess_t_SOURCES = test/subprocess.c
 test_subprocess_t_CPPFLAGS = \
 	-DTEST_SUBPROCESS_DIR=\"$(top_builddir)/src/common/libsubprocess/\" \
 	$(test_cppflags)
 test_subprocess_t_LDADD = $(test_ldadd)
+test_subprocess_t_LDFLAGS = $(test_ldflags)
+
+test_stdio_t_SOURCES = test/stdio.c
+test_stdio_t_CPPFLAGS = \
+	-DTEST_SUBPROCESS_DIR=\"$(top_builddir)/src/common/libsubprocess/\" \
+	$(test_cppflags)
+test_stdio_t_LDADD = $(test_ldadd)
+test_stdio_t_LDFLAGS = $(test_ldflags)
+
+test_channel_t_SOURCES = test/channel.c
+test_channel_t_CPPFLAGS = \
+	-DTEST_SUBPROCESS_DIR=\"$(top_builddir)/src/common/libsubprocess/\" \
+	$(test_cppflags)
+test_channel_t_LDADD = $(test_ldadd)
+test_channel_t_LDFLAGS = $(test_ldflags)
+
+test_remote_t_SOURCES = test/remote.c
+test_remote_t_CPPFLAGS = \
+	-DTEST_SUBPROCESS_DIR=\"$(top_builddir)/src/common/libsubprocess/\" \
+	$(test_cppflags)
+test_remote_t_LDADD = $(test_ldadd)
+test_remote_t_LDFLAGS = $(test_ldflags)
+
+test_iostress_t_SOURCES = test/iostress.c
+test_iostress_t_CPPFLAGS = $(test_cppflags)
+test_iostress_t_LDADD = $(test_ldadd)
+test_iostress_t_LDFLAGS = $(test_ldflags)
+
+test_iochan_t_SOURCES = test/iochan.c
+test_iochan_t_CPPFLAGS = \
+	-DTEST_SUBPROCESS_DIR=\"$(top_builddir)/src/common/libsubprocess/\" \
+	$(test_cppflags)
+test_iochan_t_LDADD = $(test_ldadd)
+test_iochan_t_LDFLAGS = $(test_ldflags)
+
+test_fbuf_t_SOURCES = test/fbuf.c
+test_fbuf_t_CPPFLAGS = $(test_cppflags)
+test_fbuf_t_LDADD = $(test_ldadd)
+test_fbuf_t_LDFLAGS = $(test_ldflags)
+
+test_fbuf_watcher_t_SOURCES = test/fbuf_watcher.c
+test_fbuf_watcher_t_CPPFLAGS = $(test_cppflags)
+test_fbuf_watcher_t_LDADD = $(test_ldadd)
+test_fbuf_watcher_t_LDFLAGS = $(test_ldflags)
+
+test_bulk_exec_einval_t_SOURCES = test/bulk-exec-einval.c
+test_bulk_exec_einval_t_CPPFLAGS = $(test_cppflags)
+test_bulk_exec_einval_t_LDADD = $(test_ldadd)
+test_bulk_exec_einval_t_LDFLAGS = $(test_ldflags)
 
 test_echo_SOURCES = test/test_echo.c
 
+test_fdcopy_SOURCES = test/fdcopy.c
+
 test_multi_echo_SOURCES = test/test_multi_echo.c
 
 test_fork_sleep_SOURCES = test/test_fork_sleep.c
diff --git a/src/common/libsubprocess/bulk-exec.c b/src/common/libsubprocess/bulk-exec.c
new file mode 100644
index 000000000000..f4e7c7c4dd2a
--- /dev/null
+++ b/src/common/libsubprocess/bulk-exec.c
@@ -0,0 +1,774 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <sys/wait.h>
+#include "src/common/libmissing/macros.h"
+#define EXIT_CODE(x) __W_EXITCODE(x,0)
+
+#include <flux/core.h>
+#include <flux/idset.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/aux.h"
+#include "src/common/libjob/idf58.h"
+#include "ccan/str/str.h"
+#include "bulk-exec.h"
+
+struct exec_cmd {
+    struct idset *ranks;
+    flux_cmd_t *cmd;
+    int flags;
+};
+
+struct bulk_exec {
+    flux_t *h;
+
+    char *service;
+    flux_jobid_t id;
+    char *name;
+
+    struct aux_item *aux;
+
+    int max_start_per_loop;  /* Max subprocess started per event loop cb */
+    int total;               /* Total processes expected to run */
+    int started;             /* Number of processes that have reached start */
+    int complete;            /* Number of processes that have completed */
+
+    int exit_status;         /* Largest wait status of all complete procs */
+
+    unsigned int active:1;
+
+    flux_watcher_t *prep;
+    flux_watcher_t *check;
+    flux_watcher_t *idle;
+
+    struct idset *exit_batch;         /* Support for batched exit notify */
+    flux_watcher_t *exit_batch_timer; /* Timer for batched exit notify */
+
+    flux_subprocess_ops_t ops;
+
+
+    zlist_t *commands;
+    zlist_t *processes;
+
+    struct bulk_exec_ops *handlers;
+    void *arg;
+};
+
+extern char **environ;
+
+int bulk_exec_rc (struct bulk_exec *exec)
+{
+    if (!exec) {
+        errno = EINVAL;
+        return -1;
+    }
+    return exec->exit_status;
+}
+
+int bulk_exec_started_count (struct bulk_exec *exec)
+{
+    if (!exec || !exec->processes)
+        return 0;
+    return zlist_size (exec->processes);
+}
+
+int bulk_exec_total (struct bulk_exec *exec)
+{
+    if (!exec || !exec->processes)
+        return 0;
+    return exec->total;
+}
+
+int bulk_exec_complete (struct bulk_exec *exec)
+{
+    if (!exec || !exec->processes)
+        return 0;
+    return exec->complete;
+}
+
+int bulk_exec_active_count (struct bulk_exec *exec)
+{
+    if (!exec || !exec->processes)
+        return 0;
+    return exec->total - exec->complete;
+}
+
+struct idset *bulk_exec_active_ranks (struct bulk_exec *exec)
+{
+    flux_subprocess_t *p;
+    struct idset *ranks;
+
+    if (!exec || !exec->processes) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (!(ranks = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        return NULL;
+
+    p = zlist_first (exec->processes);
+    while (p) {
+        if (flux_subprocess_active (p)) {
+            int rank = flux_subprocess_rank (p);
+            if (rank >= 0 && idset_set (ranks, rank) < 0) {
+                goto error;
+            }
+        }
+        p = zlist_next (exec->processes);
+    }
+    return ranks;
+error:
+    idset_destroy (ranks);
+    return NULL;
+}
+
+int bulk_exec_write (struct bulk_exec *exec,
+                     const char *stream,
+                     const char *buf,
+                     size_t len)
+{
+    flux_subprocess_t *p;
+
+    if (!exec || !stream || !buf || len <= 0) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    p = zlist_first (exec->processes);
+    while (p) {
+        if (flux_subprocess_write (p, stream, buf, len) < len)
+            return -1;
+        p = zlist_next (exec->processes);
+    }
+    return 0;
+}
+
+int bulk_exec_close (struct bulk_exec *exec, const char *stream)
+{
+    flux_subprocess_t *p;
+
+    if (!exec || !stream) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    p = zlist_first (exec->processes);
+    while (p) {
+        if (flux_subprocess_close (p, stream) < 0)
+            return -1;
+        p = zlist_next (exec->processes);
+    }
+    return 0;
+}
+
+static int exec_exit_notify (struct bulk_exec *exec)
+{
+    if (exec->handlers->on_exit)
+        (*exec->handlers->on_exit) (exec, exec->arg, exec->exit_batch);
+    if (exec->exit_batch_timer) {
+        flux_watcher_destroy (exec->exit_batch_timer);
+        exec->exit_batch_timer = NULL;
+        idset_range_clear (exec->exit_batch, 0, INT_MAX);
+    }
+    return 0;
+}
+
+static void exit_batch_cb (flux_reactor_t *r,
+                           flux_watcher_t *w,
+                           int revents,
+                           void *arg)
+{
+    struct bulk_exec *exec = arg;
+    exec_exit_notify (exec);
+}
+
+/*  Append completed subprocess 'p' to the current batch for exit
+ *   notification. If this is the first exited process in the batch,
+ *   then start a timer which will fire and call the function to
+ *   notify bulk_exec user of the batch of subprocess exits.
+ *
+ *  This approach avoids unnecessarily calling into user's callback
+ *   multiple times when all tasks exit within 0.01s.
+ */
+static void exit_batch_append (struct bulk_exec *exec, flux_subprocess_t *p)
+{
+    int rank = flux_subprocess_rank (p);
+    if (idset_set (exec->exit_batch, rank) < 0) {
+        flux_log_error (exec->h, "exit_batch_append:idset_set");
+        return;
+    }
+    if (!exec->exit_batch_timer) {
+        flux_reactor_t *r = flux_get_reactor (exec->h);
+        /*  XXX: batch timer should eventually be configurable by caller */
+        exec->exit_batch_timer =
+            flux_timer_watcher_create (r, 0.01, 0.,
+                                       exit_batch_cb,
+                                       exec);
+        if (!exec->exit_batch_timer) {
+            flux_log_error (exec->h, "exit_batch_append:timer create");
+            return;
+        }
+        flux_watcher_start (exec->exit_batch_timer);
+    }
+}
+
+static void exec_add_completed (struct bulk_exec *exec, flux_subprocess_t *p)
+{
+    /* Append this process to the current batch for notification */
+    exit_batch_append (exec, p);
+
+    if (++exec->complete == exec->total) {
+        exec_exit_notify (exec);
+        if (exec->handlers->on_complete)
+            (*exec->handlers->on_complete) (exec, exec->arg);
+    }
+}
+
+static void exec_complete_cb (flux_subprocess_t *p)
+{
+    int status = flux_subprocess_status (p);
+    struct bulk_exec *exec = flux_subprocess_aux_get (p, "job-exec::exec");
+
+    if (status > exec->exit_status)
+        exec->exit_status = status;
+
+    exec_add_completed (exec, p);
+}
+
+static void exec_state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
+{
+    struct bulk_exec *exec = flux_subprocess_aux_get (p, "job-exec::exec");
+    if (state == FLUX_SUBPROCESS_RUNNING) {
+        if (++exec->started == exec->total) {
+            if (exec->handlers->on_start)
+                (*exec->handlers->on_start) (exec, exec->arg);
+        }
+    }
+    else if (state == FLUX_SUBPROCESS_FAILED) {
+        int errnum = flux_subprocess_fail_errno (p);
+        int code = EXIT_CODE(1);
+
+        if (errnum == EPERM || errnum == EACCES)
+            code = EXIT_CODE(126);
+        else if (errnum == ENOENT)
+            code = EXIT_CODE(127);
+        else if (errnum == EHOSTUNREACH) {
+            /*  Do not set a "failure" exit code for a lost job shell.
+             *  This is because if the child job is an instance of Flux
+             *  that wants to continue running after losing a broker, then
+             *  we don't want to force a nonzero instance exit code which
+             *  would make the job appear to have failed. If the instance
+             *  does exit due to a node failure, then a nonzero exit code
+             *  will be set later anyway by the resultant job exception.
+             */
+            code = 0;
+        }
+
+        if (code > exec->exit_status)
+            exec->exit_status = code;
+
+        if (exec->handlers->on_error)
+            (*exec->handlers->on_error) (exec, p, exec->arg);
+
+        exec_add_completed (exec, p);
+    }
+}
+
+static void exec_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    struct bulk_exec *exec = flux_subprocess_aux_get (p, "job-exec::exec");
+    const char *s;
+    int len;
+
+    if ((len = flux_subprocess_read (p, stream, &s)) < 0) {
+        flux_log_error (exec->h, "flux_subprocess_read");
+        return;
+    }
+    if (len) {
+        int rank = flux_subprocess_rank (p);
+        if (exec->handlers->on_output)
+            (*exec->handlers->on_output) (exec, p, stream, s, len, exec->arg);
+        else {
+            flux_log (exec->h,
+                      LOG_INFO,
+                      "rank %d: %s: %.*s",
+                      rank,
+                      stream,
+                      len,
+                      s);
+        }
+    }
+}
+
+static void exec_cmd_destroy (void *arg)
+{
+    struct exec_cmd *cmd = arg;
+    if (cmd) {
+        int saved_errno = errno;
+        idset_destroy (cmd->ranks);
+        flux_cmd_destroy (cmd->cmd);
+        free (cmd);
+        errno = saved_errno;
+    }
+}
+
+static struct exec_cmd *exec_cmd_create (const struct idset *ranks,
+                                         flux_cmd_t *cmd,
+                                         int flags)
+{
+    struct exec_cmd *c = calloc (1, sizeof (*c));
+    if (!c)
+        return NULL;
+    if (!(c->ranks = idset_copy (ranks))) {
+        fprintf (stderr, "exec_cmd_create: idset_copy failed");
+        goto err;
+    }
+    if (!(c->cmd = flux_cmd_copy (cmd))) {
+        fprintf (stderr, "exec_cmd_create: flux_cmd_copy failed");
+        goto err;
+    }
+    /* bulk-exec always uses unbuffered reads for performance */
+    c->flags = flags | FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF;
+    return (c);
+err:
+    exec_cmd_destroy (c);
+    return NULL;
+}
+
+static void subprocess_destroy_finish (flux_future_t *f, void *arg)
+{
+    flux_subprocess_t *p = arg;
+    if (flux_future_get (f, NULL) < 0) {
+        flux_t *h = flux_subprocess_aux_get (p, "flux_t");
+        flux_log_error (h,
+                        "subprocess_kill: %ju: %s",
+                        (uintmax_t) flux_subprocess_pid (p),
+                        future_strerror (f, errno));
+    }
+    flux_subprocess_destroy (p);
+    flux_future_destroy (f);
+}
+
+static int subprocess_destroy (flux_t *h, flux_subprocess_t *p)
+{
+    flux_future_t *f = flux_subprocess_kill (p, SIGKILL);
+    if (!f || flux_future_then (f, -1., subprocess_destroy_finish, p) < 0)
+        return -1;
+    return 0;
+}
+
+static int exec_start_cmd (struct bulk_exec *exec,
+                           struct exec_cmd *cmd,
+                           int max)
+{
+    int count = 0;
+    uint32_t rank;
+    rank = idset_first (cmd->ranks);
+    while (rank != IDSET_INVALID_ID && (max < 0 || count < max)) {
+        /* Set the unit name for the "sdexec" service.  This is done here
+         * for each rank instead of once in bulk_exec_push_cmd() to ensure
+         * the name is unique when there are multiple brokers per node.
+         * Ex: shell-0-fTE9HHdZvi3.service, imp-kill-1-fTE9HHdZvi3.service.
+         * (N.B. systemd doesn't like "ƒ" in the unit name hence f58plain).
+         */
+        if (streq (exec->service, "sdexec")) {
+            char idbuf[21];
+            char name[128];
+            if (flux_job_id_encode (exec->id,
+                                    "f58plain",
+                                    idbuf,
+                                    sizeof (idbuf)) < 0)
+                return -1;
+            snprintf (name,
+                      sizeof (name),
+                      "%s-%lu-%s.service",
+                      exec->name,
+                      (unsigned long)rank,
+                      idbuf);
+            if (flux_cmd_setopt (cmd->cmd, "SDEXEC_NAME", name) < 0
+                || flux_cmd_setopt (cmd->cmd,
+                                    "SDEXEC_PROP_Description",
+                                    "User workload") < 0) {
+                flux_log_error (exec->h, "Unable to set sdexec options");
+                return -1;
+            }
+        }
+        flux_subprocess_t *p = flux_rexec_ex (exec->h,
+                                              bulk_exec_service_name (exec),
+                                              rank,
+                                              cmd->flags,
+                                              cmd->cmd,
+                                              &exec->ops,
+                                              flux_llog,
+                                              exec->h);
+        if (!p)
+            return -1;
+        if (flux_subprocess_aux_set (p, "job-exec::exec", exec, NULL) < 0
+           || zlist_append (exec->processes, p) < 0) {
+            if (subprocess_destroy (exec->h, p) < 0)
+                flux_log_error (exec->h, "Unable to destroy pid %ju",
+                        (uintmax_t) flux_subprocess_pid (p));
+            return -1;
+        }
+        zlist_freefn (exec->processes, p,
+                     (zlist_free_fn *) flux_subprocess_destroy,
+                     true);
+
+        idset_clear (cmd->ranks, rank);
+        rank = idset_next (cmd->ranks, rank);
+        count++;
+    }
+    return count;
+}
+
+void bulk_exec_stop (struct bulk_exec *exec)
+{
+    flux_watcher_stop (exec->prep);
+    flux_watcher_stop (exec->check);
+}
+
+static int exec_start_cmds (struct bulk_exec *exec, int max)
+{
+    while (zlist_size (exec->commands) && (max != 0)) {
+        struct exec_cmd *cmd = zlist_first (exec->commands);
+        int rc = exec_start_cmd (exec, cmd, max);
+        if (rc < 0) {
+            flux_log_error (exec->h, "exec_start_cmd failed");
+            return -1;
+        }
+        if (idset_count (cmd->ranks) == 0)
+            zlist_remove (exec->commands, cmd);
+        if (max > 0)
+            max -= rc;
+
+    }
+    return 0;
+}
+
+
+static void prep_cb (flux_reactor_t *r,
+                     flux_watcher_t *w,
+                     int revents,
+                     void *arg)
+{
+    struct bulk_exec *exec = arg;
+
+    /* Don't block in reactor if there are commands to run */
+    if (zlist_size (exec->commands) > 0) {
+        flux_watcher_start (exec->idle);
+        flux_watcher_start (exec->check);
+    }
+    else
+        bulk_exec_stop (exec);
+}
+
+static void check_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct bulk_exec *exec = arg;
+    flux_watcher_stop (exec->idle);
+    flux_watcher_stop (exec->check);
+    if (exec_start_cmds (exec, exec->max_start_per_loop) < 0) {
+        bulk_exec_stop (exec);
+        if (exec->handlers->on_error)
+            (*exec->handlers->on_error) (exec, NULL, exec->arg);
+    }
+}
+
+void bulk_exec_destroy (struct bulk_exec *exec)
+{
+    if (exec) {
+        int saved_errno = errno;
+        zlist_destroy (&exec->processes);
+        zlist_destroy (&exec->commands);
+        idset_destroy (exec->exit_batch);
+        flux_watcher_destroy (exec->prep);
+        flux_watcher_destroy (exec->check);
+        flux_watcher_destroy (exec->idle);
+        flux_watcher_destroy (exec->exit_batch_timer);
+        aux_destroy (&exec->aux);
+        free (exec->name);
+        free (exec->service);
+        free (exec);
+        errno = saved_errno;
+    }
+}
+
+struct bulk_exec * bulk_exec_create (struct bulk_exec_ops *ops,
+                                     const char *service,
+                                     flux_jobid_t id,
+                                     const char *name,
+                                     void *arg)
+{
+    flux_subprocess_ops_t sp_ops = {
+        .on_completion =   exec_complete_cb,
+        .on_state_change = exec_state_cb,
+        .on_channel_out =  exec_output_cb,
+        .on_stdout =       exec_output_cb,
+        .on_stderr =       exec_output_cb,
+    };
+    struct bulk_exec *exec = calloc (1, sizeof (*exec));
+    if (!exec
+        || !(exec->service = strdup (service))
+        || !(exec->name = strdup (name)))
+        goto error;
+    exec->id = id;
+    exec->ops = sp_ops;
+    exec->handlers = ops;
+    exec->arg = arg;
+    exec->processes = zlist_new ();
+    exec->commands = zlist_new ();
+    exec->exit_batch = idset_create (0, IDSET_FLAG_AUTOGROW);
+    exec->max_start_per_loop = 1;
+
+    return exec;
+error:
+    bulk_exec_destroy (exec);
+    return NULL;
+}
+
+int bulk_exec_set_max_per_loop (struct bulk_exec *exec, int max)
+{
+    if (!exec || max == 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    exec->max_start_per_loop = max;
+    return 0;
+}
+
+int bulk_exec_push_cmd (struct bulk_exec *exec,
+                       const struct idset *ranks,
+                       flux_cmd_t *cmd,
+                       int flags)
+{
+    struct exec_cmd *c;
+
+    if (!exec || !ranks || !cmd) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!(c = exec_cmd_create (ranks, cmd, flags)))
+        return -1;
+
+    if (zlist_append (exec->commands, c) < 0) {
+        exec_cmd_destroy (c);
+        errno = ENOMEM;
+        return -1;
+    }
+    zlist_freefn (exec->commands, c, exec_cmd_destroy, true);
+
+    exec->total += idset_count (ranks);
+    if (exec->active) {
+        flux_watcher_start (exec->prep);
+        flux_watcher_start (exec->check);
+    }
+
+    return 0;
+}
+
+int bulk_exec_start (flux_t *h, struct bulk_exec *exec)
+{
+    flux_reactor_t *r;
+
+    if (!h || !exec) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    r = flux_get_reactor (h);
+    exec->h = h;
+    exec->prep = flux_prepare_watcher_create (r, prep_cb, exec);
+    exec->check = flux_check_watcher_create (r, check_cb, exec);
+    exec->idle = flux_idle_watcher_create (r, NULL, NULL);
+    if (!exec->prep || !exec->check || !exec->idle)
+        return -1;
+    flux_watcher_start (exec->prep);
+    exec->active = 1;
+    return 0;
+}
+
+/*  Cancel all pending commands.
+ */
+int bulk_exec_cancel (struct bulk_exec *exec)
+{
+    struct exec_cmd *cmd;
+
+    if (!exec) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!(cmd = zlist_first (exec->commands)))
+        return 0;
+
+    while (cmd) {
+        uint32_t rank = idset_first (cmd->ranks);
+        while (rank != IDSET_INVALID_ID) {
+            exec->complete++;
+            if (idset_set (exec->exit_batch, rank) < 0)
+                flux_log_error (exec->h, "bulk_exec_cancel: idset_set");
+            rank = idset_next (cmd->ranks, rank);
+        }
+        cmd = zlist_next (exec->commands);
+    }
+    zlist_purge (exec->commands);
+    exec_exit_notify (exec);
+
+    if (exec->complete == exec->total) {
+        if (exec->handlers->on_complete)
+            (*exec->handlers->on_complete) (exec, exec->arg);
+    }
+    return 0;
+}
+
+/*  Loop through all child futures and print rank-specific errors
+ */
+void bulk_exec_kill_log_error (flux_future_t *f, flux_jobid_t id)
+{
+    flux_t *h;
+    const char *name;
+
+    if (!f)
+        return;
+
+    h = flux_future_get_flux (f);
+    name = flux_future_first_child (f);
+    while (name) {
+        flux_future_t *cf = flux_future_get_child (f, name);
+        uint32_t rank = flux_rpc_get_nodeid (cf);
+        if (flux_future_is_ready (cf)
+            && flux_future_get (cf, NULL) < 0
+            && errno != ESRCH
+            && rank != FLUX_NODEID_ANY) {
+            flux_log (h,
+                      LOG_ERR,
+                      "%s: exec_kill: %s (rank %lu): %s",
+                      idf58 (id),
+                      flux_get_hostbyrank (h, rank),
+                      (unsigned long)rank,
+                      future_strerror (cf, errno));
+        }
+        name = flux_future_next_child (f);
+    }
+}
+
+flux_future_t *bulk_exec_kill (struct bulk_exec *exec,
+                               const struct idset *ranks,
+                               int signum)
+{
+    flux_subprocess_t *p;
+    flux_future_t *cf;
+
+    if (!exec || signum < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (!(cf = flux_future_wait_all_create ()))
+        return NULL;
+    flux_future_set_flux (cf, exec->h);
+
+    p = zlist_first (exec->processes);
+    while (p) {
+        if ((!ranks || idset_test (ranks, flux_subprocess_rank (p)))
+            && (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING
+            || flux_subprocess_state (p) == FLUX_SUBPROCESS_INIT)) {
+            flux_future_t *f = NULL;
+            char s[64];
+            if (!(f = flux_subprocess_kill (p, signum))) {
+                int err = errno;
+                const char *errstr = flux_strerror (errno);
+                if ((f = flux_future_create (NULL, NULL)))
+                    flux_future_fulfill_error (f, err, errstr);
+                else
+                    flux_future_fulfill_error (cf, err, "Internal error");
+            }
+            (void) snprintf (s,
+                             sizeof (s)-1,
+                             "%u",
+                             flux_subprocess_rank (p));
+            if (flux_future_push (cf, s, f) < 0) {
+                fprintf (stderr, "flux_future_push: %s\n", strerror (errno));
+                flux_future_destroy (f);
+            }
+        }
+        p = zlist_next (exec->processes);
+    }
+
+    /*  If no child futures were pushed into the wait_all future `cf`,
+     *   then no signals were sent and we should immediately return ENOENT.
+     */
+    if (!flux_future_first_child (cf)) {
+        flux_future_destroy (cf);
+        errno = ENOENT;
+        return NULL;
+    }
+
+    return cf;
+}
+
+int bulk_exec_aux_set (struct bulk_exec *exec,
+                       const char *key,
+                       void *val,
+                       flux_free_f free_fn)
+{
+    if (!exec) {
+        errno = EINVAL;
+        return -1;
+    }
+    return (aux_set (&exec->aux, key, val, free_fn));
+}
+
+void * bulk_exec_aux_get (struct bulk_exec *exec, const char *key)
+{
+    if (!exec) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return (aux_get (exec->aux, key));
+}
+
+const char *bulk_exec_service_name (struct bulk_exec *exec)
+{
+    if (!exec)
+        return NULL;
+    return exec->service;
+}
+
+flux_subprocess_t *bulk_exec_get_subprocess (struct bulk_exec *exec, int rank)
+{
+    flux_subprocess_t *p;
+
+    if (!exec || rank < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    p = zlist_first (exec->processes);
+    while (p) {
+        if (flux_subprocess_rank (p) == rank)
+            return p;
+        p = zlist_next (exec->processes);
+    }
+    errno = ENOENT;
+    return NULL;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libsubprocess/bulk-exec.h b/src/common/libsubprocess/bulk-exec.h
new file mode 100644
index 000000000000..417f3c54a507
--- /dev/null
+++ b/src/common/libsubprocess/bulk-exec.h
@@ -0,0 +1,123 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* "bulk" subprocess execution wrapper around libsubprocess API */
+
+#ifndef HAVE_JOB_EXEC_BULK_EXEC_H
+#define HAVE_JOB_EXEC_BULK_EXEC_H 1
+
+#include <flux/core.h>
+#include <flux/idset.h>
+
+struct bulk_exec;
+
+typedef void (*exec_cb_f)   (struct bulk_exec *, void *arg);
+
+typedef void (*exec_exit_f) (struct bulk_exec *, void *arg,
+                             const struct idset *ranks);
+
+typedef void (*exec_io_f)   (struct bulk_exec *,
+                             flux_subprocess_t *,
+                             const char *stream,
+                             const char *data,
+                             int data_len,
+                             void *arg);
+
+typedef void (*exec_error_f) (struct bulk_exec *,
+                              flux_subprocess_t *,
+                              void *arg);
+
+struct bulk_exec_ops {
+    exec_cb_f    on_start;    /* called when all processes are running  */
+    exec_exit_f  on_exit;     /* called when a set of tasks exits       */
+    exec_cb_f    on_complete; /* called when all processes are done     */
+    exec_io_f    on_output;   /* called on process output               */
+    exec_error_f on_error;    /* called on any fatal error              */
+};
+
+struct bulk_exec * bulk_exec_create (struct bulk_exec_ops *ops,
+                                     const char *service,
+                                     flux_jobid_t id,
+                                     const char *name,
+                                     void *arg);
+
+void *bulk_exec_aux_get (struct bulk_exec *exec, const char *key);
+
+int bulk_exec_aux_set (struct bulk_exec *exec,
+                       const char *key,
+                       void *val,
+                       flux_free_f free_fn);
+
+/*  Set maximum number of flux_subprocess_rexex(3) calls per event
+ *   loop iteration. (-1 for no max)
+ */
+int bulk_exec_set_max_per_loop (struct bulk_exec *exec, int max);
+
+void bulk_exec_destroy (struct bulk_exec *exec);
+
+int bulk_exec_push_cmd (struct bulk_exec *exec,
+                       const struct idset *ranks,
+                       flux_cmd_t *cmd,
+                       int flags);
+
+int bulk_exec_start (flux_t *h, struct bulk_exec *exec);
+
+/* Send signal to ranks. Set ranks=NULL for all.
+ * If an IMP path has been set then bulk_exec_imp_kill() will be used.
+ */
+flux_future_t * bulk_exec_kill (struct bulk_exec *exec,
+                                const struct idset *ranks,
+                                int signal);
+
+/*  Log per-rank kill errors for a failed bulk_exec_kill() RPC.
+ */
+void bulk_exec_kill_log_error (flux_future_t *f, flux_jobid_t id);
+
+flux_future_t *bulk_exec_imp_kill (struct bulk_exec *exec,
+                                   const char *imp_path,
+                                   const struct idset *ranks,
+                                   int signal);
+
+int bulk_exec_cancel (struct bulk_exec *exec);
+
+/* Returns max wait status returned from all exited processes */
+int bulk_exec_rc (struct bulk_exec *exec);
+
+/* Returns current number of processes that have been started */
+int bulk_exec_started_count (struct bulk_exec *exec);
+
+/* Return number of processes that are complete */
+int bulk_exec_complete (struct bulk_exec *exec);
+
+/* Return number of processes that are still active */
+int bulk_exec_active_count (struct bulk_exec *exec);
+
+/* Return idset of ranks on which processes are still active */
+struct idset *bulk_exec_active_ranks (struct bulk_exec *exec);
+
+int bulk_exec_write (struct bulk_exec *exec,
+                     const char *stream,
+                     const char *buf,
+                     size_t len);
+
+int bulk_exec_close (struct bulk_exec *exec, const char *stream);
+
+/* Returns total number of processes expected to run */
+int bulk_exec_total (struct bulk_exec *exec);
+
+/* Get subprocess remote exec service name (never returns NULL) */
+const char *bulk_exec_service_name (struct bulk_exec *exec);
+
+/*  Get the subprocess handle for a rank
+ */
+flux_subprocess_t *bulk_exec_get_subprocess (struct bulk_exec *exec,
+                                             int rank);
+
+#endif /* !HAVE_JOB_EXEC_BULK_EXEC_H */
diff --git a/src/common/libsubprocess/client.c b/src/common/libsubprocess/client.c
new file mode 100644
index 000000000000..e28db588cde0
--- /dev/null
+++ b/src/common/libsubprocess/client.c
@@ -0,0 +1,335 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <unistd.h>
+#include <stdlib.h>
+#include <flux/core.h>
+
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+#include "src/common/libioencode/ioencode.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+
+#include "command_private.h"
+#include "client.h"
+
+struct rexec_io {
+    json_t *obj;
+    const char *stream;
+    char *data;
+    int len;
+    bool eof;
+};
+
+struct rexec_response {
+    const char *type;
+    pid_t pid;
+    int status;
+    struct rexec_io io;
+    json_t *channels;
+};
+
+struct rexec_ctx {
+    json_t *cmd;
+    int flags;
+    struct rexec_response response;
+    uint32_t matchtag;
+    uint32_t rank;
+    char *service_name;
+};
+
+static void rexec_response_clear (struct rexec_response *resp)
+{
+    json_decref (resp->io.obj);
+    free (resp->io.data);
+    json_decref (resp->channels);
+    resp->channels = NULL;
+
+    memset (resp, 0, sizeof (*resp));
+
+    resp->pid = -1;
+}
+
+static void rexec_ctx_destroy (struct rexec_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        rexec_response_clear (&ctx->response);
+        json_decref (ctx->cmd);
+        free (ctx->service_name);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static struct rexec_ctx *rexec_ctx_create (flux_cmd_t *cmd,
+                                           const char *service_name,
+                                           uint32_t rank,
+                                           int flags)
+{
+    struct rexec_ctx *ctx;
+    int valid_flags = SUBPROCESS_REXEC_STDOUT
+        | SUBPROCESS_REXEC_STDERR
+        | SUBPROCESS_REXEC_CHANNEL
+        | SUBPROCESS_REXEC_WRITE_CREDIT;
+
+    if ((flags & ~valid_flags)) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+    if (!(ctx->cmd = cmd_tojson (cmd))
+        || !(ctx->service_name = strdup (service_name)))
+        goto error;
+    ctx->flags = flags;
+    ctx->response.pid = -1;
+    ctx->rank = rank;
+    return ctx;
+error:
+    rexec_ctx_destroy (ctx);
+    return NULL;
+}
+
+flux_future_t *subprocess_rexec (flux_t *h,
+                                 const char *service_name,
+                                 uint32_t rank,
+                                 flux_cmd_t *cmd,
+                                 int flags)
+{
+    flux_future_t *f = NULL;
+    struct rexec_ctx *ctx;
+    char *topic;
+
+    if (!h || !cmd || !service_name) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (asprintf (&topic, "%s.exec", service_name) < 0)
+        return NULL;
+    if (!(ctx = rexec_ctx_create (cmd, service_name, rank, flags)))
+        goto error;
+    if (!(f = flux_rpc_pack (h,
+                             topic,
+                             rank,
+                             FLUX_RPC_STREAMING,
+                             "{s:O s:i}",
+                             "cmd", ctx->cmd,
+                             "flags", ctx->flags))
+        || flux_future_aux_set (f,
+                                "flux::rexec",
+                                ctx,
+                                (flux_free_f)rexec_ctx_destroy) < 0) {
+        rexec_ctx_destroy (ctx);
+        goto error;
+    }
+    ctx->matchtag = flux_rpc_get_matchtag (f);
+    free (topic);
+    return f;
+error:
+    ERRNO_SAFE_WRAP (free, topic);
+    flux_future_destroy (f);
+    return NULL;
+}
+
+int subprocess_rexec_get (flux_future_t *f)
+{
+    struct rexec_ctx *ctx;
+
+    if (!(ctx = flux_future_aux_get (f, "flux::rexec"))) {
+        errno = EINVAL;
+        return -1;
+    }
+    rexec_response_clear (&ctx->response);
+    if (flux_rpc_get_unpack (f,
+                             "{s:s s?i s?i s?O s?O}",
+                             "type", &ctx->response.type,
+                             "pid", &ctx->response.pid,
+                             "status", &ctx->response.status,
+                             "io", &ctx->response.io.obj,
+                             "channels", &ctx->response.channels) < 0)
+        return -1;
+    if (streq (ctx->response.type, "output")) {
+        if (iodecode (ctx->response.io.obj,
+                      &ctx->response.io.stream,
+                      NULL,
+                      &ctx->response.io.data,
+                      &ctx->response.io.len,
+                      &ctx->response.io.eof) < 0)
+            return -1;
+    }
+    else if (streq (ctx->response.type, "add-credit")) {
+        const char *key;
+        json_t *value;
+
+        if (!ctx->response.channels
+            || !json_is_object (ctx->response.channels)) {
+            errno = EPROTO;
+            return -1;
+        }
+        json_object_foreach (ctx->response.channels, key, value) {
+            if (!json_is_integer (value)) {
+                errno = EPROTO;
+                return -1;
+            }
+        }
+    }
+    else if (!streq (ctx->response.type, "started")
+        && !streq (ctx->response.type, "stopped")
+        && !streq (ctx->response.type, "finished")) {
+        errno = EPROTO;
+        return -1;
+    }
+    return 0;
+}
+
+bool subprocess_rexec_is_started (flux_future_t *f, pid_t *pid)
+{
+    struct rexec_ctx *ctx;
+    if ((ctx = flux_future_aux_get (f, "flux::rexec"))
+        && ctx->response.type != NULL
+        && streq (ctx->response.type, "started")) {
+        if (pid)
+            *pid = ctx->response.pid;
+        return true;
+    }
+    return false;
+}
+
+bool subprocess_rexec_is_stopped (flux_future_t *f)
+{
+    struct rexec_ctx *ctx;
+    if ((ctx = flux_future_aux_get (f, "flux::rexec"))
+        && ctx->response.type != NULL
+        && streq (ctx->response.type, "stopped"))
+        return true;
+    return false;
+}
+
+bool subprocess_rexec_is_finished (flux_future_t *f, int *status)
+{
+    struct rexec_ctx *ctx;
+    if ((ctx = flux_future_aux_get (f, "flux::rexec"))
+        && ctx->response.type != NULL
+        && streq (ctx->response.type, "finished")) {
+        if (status)
+            *status = ctx->response.status;
+        return true;
+    }
+    return false;
+}
+
+bool subprocess_rexec_is_output (flux_future_t *f,
+                                 const char **stream,
+                                 const char **data,
+                                 int *len,
+                                 bool *eof)
+{
+    struct rexec_ctx *ctx;
+    if ((ctx = flux_future_aux_get (f, "flux::rexec"))
+        && ctx->response.type != NULL
+        && streq (ctx->response.type, "output")) {
+        if (stream)
+            *stream = ctx->response.io.stream;
+        if (data)
+            *data = ctx->response.io.data;
+        if (len)
+            *len = ctx->response.io.len;
+        if (eof)
+            *eof = ctx->response.io.eof;
+        return true;
+    }
+    return false;
+}
+
+bool subprocess_rexec_is_add_credit (flux_future_t *f, json_t **channels)
+{
+    struct rexec_ctx *ctx;
+    if ((ctx = flux_future_aux_get (f, "flux::rexec"))
+        && ctx->response.type != NULL
+        && streq (ctx->response.type, "add-credit")) {
+        if (channels)
+            (*channels) = ctx->response.channels;
+        return true;
+    }
+    return false;
+}
+
+int subprocess_write (flux_future_t *f_exec,
+                      const char *stream,
+                      const char *data,
+                      int len,
+                      bool eof)
+{
+    struct rexec_ctx *ctx = flux_future_aux_get (f_exec, "flux::rexec");
+    flux_t *h = flux_future_get_flux (f_exec);
+    flux_future_t *f = NULL;
+    json_t *io;
+    char *topic;
+    int rc = -1;
+
+    if (!stream || !ctx) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (asprintf (&topic, "%s.write", ctx->service_name) < 0)
+        return -1;
+    if (!(io = ioencode (stream, "0", data, len, eof))
+        || !(f = flux_rpc_pack (h,
+                                topic,
+                                ctx->rank,
+                                FLUX_RPC_NORESPONSE,
+                                "{s:i s:O}",
+                                "matchtag", ctx->matchtag,
+                                "io", io)))
+        goto out;
+    rc = 0;
+out:
+    flux_future_destroy (f);
+    ERRNO_SAFE_WRAP (json_decref, io);
+    ERRNO_SAFE_WRAP (free, topic);
+    return rc;
+}
+
+flux_future_t *subprocess_kill (flux_t *h,
+                                const char *service_name,
+                                uint32_t rank,
+                                pid_t pid,
+                                int signum)
+{
+    flux_future_t *f;
+    char *topic;
+
+    if (!h || !service_name) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (asprintf (&topic, "%s.kill", service_name) < 0)
+        return NULL;
+    if (!(f = flux_rpc_pack (h,
+                             topic,
+                             rank,
+                             0,
+                             "{s:i s:i}",
+                             "pid", pid,
+                             "signum", signum))) {
+        ERRNO_SAFE_WRAP (free, topic);
+        return NULL;
+    }
+    free (topic);
+    return f;
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/client.h b/src/common/libsubprocess/client.h
new file mode 100644
index 000000000000..baf82f004ef9
--- /dev/null
+++ b/src/common/libsubprocess/client.h
@@ -0,0 +1,60 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SUBPROCESS_CLIENT_H
+#define _SUBPROCESS_CLIENT_H
+
+#include <sys/types.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#include <flux/core.h>
+#include "subprocess.h"
+
+enum {
+    SUBPROCESS_REXEC_STDOUT = 1,
+    SUBPROCESS_REXEC_STDERR = 2,
+    SUBPROCESS_REXEC_CHANNEL = 4,
+    SUBPROCESS_REXEC_WRITE_CREDIT = 8,
+};
+
+flux_future_t *subprocess_rexec (flux_t *h,
+                                 const char *service_name,
+                                 uint32_t rank,
+                                 flux_cmd_t *cmd,
+                                 int flags);
+
+int subprocess_rexec_get (flux_future_t *f);
+bool subprocess_rexec_is_started (flux_future_t *f, pid_t *pid);
+bool subprocess_rexec_is_stopped (flux_future_t *f);
+bool subprocess_rexec_is_finished (flux_future_t *f, int *status);
+bool subprocess_rexec_is_output (flux_future_t *f,
+                                 const char **stream,
+                                 const char **buf,
+                                 int *len,
+                                 bool *eof);
+bool subprocess_rexec_is_add_credit (flux_future_t *f, json_t **channels);
+
+int subprocess_write (flux_future_t *f,
+                      const char *stream,
+                      const char *data,
+                      int len,
+                      bool eof);
+
+flux_future_t *subprocess_kill (flux_t *h,
+                                const char *service_name,
+                                uint32_t rank,
+                                pid_t pid,
+                                int signum);
+
+
+#endif /* !_SUBPROCESS_CLIENT_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/command.c b/src/common/libsubprocess/command.c
index 534f5457dc02..f47246020bfb 100644
--- a/src/common/libsubprocess/command.c
+++ b/src/common/libsubprocess/command.c
@@ -14,12 +14,25 @@
 
 #include <stdlib.h>
 #include <sys/types.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#ifdef HAVE_ENVZ_ADD
 #include <envz.h>
+#else
+#include "src/common/libmissing/envz.h"
+#endif
+#include <assert.h>
+#include <fnmatch.h>
 
 #include <jansson.h>
-#include <czmq.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
+#include "command_private.h"
 #include "command.h"
 
 struct flux_command {
@@ -68,8 +81,10 @@ static int init_argz (char **argzp, size_t *argz_lenp, char * const av[])
  *  Same as init_argz, but pass argument count (ac) and verify that
  *   the argument vector av has NULL as its final element.
  */
-static int init_argz_count (char **argzp, size_t *argz_lenp,
-                            int ac, char * const av[])
+static int init_argz_count (char **argzp,
+                            size_t *argz_lenp,
+                            int ac,
+                            char * const av[])
 {
     if (av && (av[ac] != NULL)) {
         errno = EINVAL;
@@ -81,8 +96,10 @@ static int init_argz_count (char **argzp, size_t *argz_lenp,
 /*
  *  Append string defined by [fmt, ap] to argz vector in argzp
  */
-static int argz_appendv (char **argzp, size_t *argz_lenp,
-                         const char *fmt,  va_list ap)
+static int argz_appendv (char **argzp,
+                         size_t *argz_lenp,
+                         const char *fmt,
+                         va_list ap)
 {
     int e;
     char *s;
@@ -306,7 +323,7 @@ static zhash_t *zhash_fromjson (json_t *o)
             goto fail;
         if (zhash_insert (h, key, (char *) json_string_value (val)) < 0) {
             /* Duplicate key. This can't happen unless json object is
-             *  corrupt, so give up and return error (EINVAL)
+             *  corrupt, so give up and return error (EPROTO)
              */
             goto fail;
         }
@@ -374,7 +391,7 @@ static const char * z_list_find (zlist_t *l, const char *s)
 {
     const char *v = zlist_first (l);
     while (v != NULL) {
-        if (strcmp (s, v) == 0)
+        if (streq (s, v))
             return (v);
         v = zlist_next (l);
     }
@@ -462,6 +479,19 @@ int flux_cmd_argc (const flux_cmd_t *cmd)
     return argz_count (cmd->argz, cmd->argz_len);
 }
 
+char *flux_cmd_stringify (const flux_cmd_t *cmd)
+{
+    if (cmd->argz_len == 0)
+        return strdup ("");
+
+    char *result;
+    if (!(result = malloc (cmd->argz_len)))
+        return NULL;
+    memcpy (result, cmd->argz, cmd->argz_len);
+    argz_stringify (result, cmd->argz_len, ' ');
+    return result;
+}
+
 const char *flux_cmd_arg (const flux_cmd_t *cmd, int n)
 {
     char *arg = NULL;
@@ -480,6 +510,34 @@ const char *flux_cmd_arg (const flux_cmd_t *cmd, int n)
     return arg;
 }
 
+int flux_cmd_argv_insert (flux_cmd_t *cmd, int n, const char *entry)
+{
+    const char *arg = flux_cmd_arg (cmd, n);
+
+    /*  Always allow n == 0, even if flux_cmd_arg (cmd, 0) returned NULL.
+     *  This means the argv is currently empty and insert is equivalent
+     *   to append. (argz_insert with entry == NULL is an implicit append)
+     */
+    if (arg == NULL && n > 0)
+        return -1;
+
+    if (argz_insert (&cmd->argz, &cmd->argz_len, (char *)arg, entry) != 0) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+int flux_cmd_argv_delete (flux_cmd_t *cmd, int n)
+{
+    const char *arg = flux_cmd_arg (cmd, n);
+    if (arg == NULL)
+        return -1;
+
+    argz_delete (&cmd->argz, &cmd->argz_len, (char *) arg);
+    return 0;
+}
+
 int flux_cmd_argv_appendf (flux_cmd_t *cmd, const char *fmt, ...)
 {
     int rc = 0;
@@ -508,22 +566,24 @@ int flux_cmd_argv_append (flux_cmd_t *cmd, const char *arg)
     return 0;
 }
 
-static int flux_cmd_setenv (flux_cmd_t *cmd, const char *k, const char *v,
+static int flux_cmd_setenv (flux_cmd_t *cmd,
+                            const char *k,
+                            const char *v,
                             int overwrite)
 {
-    if (!overwrite && envz_entry (cmd->envz, cmd->envz_len, k)) {
-        errno = EEXIST;
-        return -1;
-    }
-    if (envz_add (&cmd->envz, &cmd->envz_len, k, v) != 0) {
-        errno = ENOMEM;
-        return -1;
+    if (overwrite || !envz_entry (cmd->envz, cmd->envz_len, k)) {
+        if (envz_add (&cmd->envz, &cmd->envz_len, k, v) != 0) {
+            errno = ENOMEM;
+            return -1;
+        }
     }
     return 0;
 }
 
-int flux_cmd_setenvf (flux_cmd_t *cmd, int overwrite,
-                      const char *name, const char *fmt, ...)
+int flux_cmd_setenvf (flux_cmd_t *cmd,
+                      int overwrite,
+                      const char *name,
+                      const char *fmt, ...)
 {
     va_list ap;
     char *val;
@@ -539,9 +599,39 @@ int flux_cmd_setenvf (flux_cmd_t *cmd, int overwrite,
     return (rc);
 }
 
+static bool isa_glob (const char *s)
+{
+    if (strchr (s, '*') || strchr (s, '?') || strchr (s, '['))
+        return true;
+    return false;
+}
+
 void flux_cmd_unsetenv (flux_cmd_t *cmd, const char *name)
 {
-    envz_remove (&cmd->envz, &cmd->envz_len, name);
+    if (!cmd
+        || !name
+        || cmd->envz == NULL
+        || cmd->envz_len == 0)
+        return;
+
+    if (isa_glob (name)) {
+        char *cpy = NULL;
+        size_t cpy_len = 0;
+        char buf[1024];
+        char *s;
+
+        if (argz_append (&cpy, &cpy_len, cmd->envz, cmd->envz_len) == 0) {
+            char *entry = NULL;
+            while ((entry = argz_next (cpy, cpy_len, entry))) {
+                if ((s = env_entry_name (entry, buf, sizeof (buf)))
+                    && fnmatch (name, s, 0) == 0)
+                    envz_remove (&cmd->envz, &cmd->envz_len, s);
+            }
+            free (cpy);
+        }
+    }
+    else
+        envz_remove (&cmd->envz, &cmd->envz_len, name);
 }
 
 const char * flux_cmd_getenv (const flux_cmd_t *cmd, const char *name)
@@ -574,7 +664,7 @@ int flux_cmd_add_channel (flux_cmd_t *cmd, const char *name)
         return -1;
     }
     /* autofree is set on cmd->channels, so name is automatically strdup'd */
-    return zlist_append (cmd->channels, (char *) name);
+    return zlist_append (cmd->channels, (char *)name);
 }
 
 int flux_cmd_setopt (flux_cmd_t *cmd, const char *var, const char *val)
@@ -584,7 +674,7 @@ int flux_cmd_setopt (flux_cmd_t *cmd, const char *var, const char *val)
         return -1;
     }
     /* autofree is set on cmd->opts, so val is automatically strdup'd */
-    zhash_update (cmd->opts, var, (char *) val);
+    zhash_update (cmd->opts, var, (char *)val);
     return 0;
 }
 
@@ -615,35 +705,33 @@ flux_cmd_t * flux_cmd_copy (const flux_cmd_t *src)
     return NULL;
 }
 
-flux_cmd_t * flux_cmd_fromjson (const char *json_str, json_error_t *errp)
+flux_cmd_t *cmd_fromjson (json_t *o, json_error_t *errp)
 {
     int errnum;
-    json_t *o = NULL;
     json_t *jenv = NULL;
     json_t *jargv = NULL;
     json_t *jopts = NULL;
     json_t *jchans = NULL;
-    const char *cwd;
+    const char *cwd = NULL;
     flux_cmd_t *cmd = NULL;;
 
-    if (!(o = json_loads (json_str, 0, errp))) {
-        errnum = EPROTO;
-        goto fail;
-    }
     if (!(cmd = calloc (1, sizeof (*cmd)))) {
         errnum = ENOMEM;
         goto fail;
     }
-    if (json_unpack_ex (o, errp, 0, "{s:s, s:o, s:o, s:o, s:o}",
-                "cwd", &cwd,
-                "cmdline", &jargv,
-                "env", &jenv,
-                "opts", &jopts,
-                "channels", &jchans) < 0) {
+    if (json_unpack_ex (o,
+                        errp,
+                        0,
+                        "{s?s, s:o, s:o, s:o, s:o}",
+                        "cwd", &cwd,
+                        "cmdline", &jargv,
+                        "env", &jenv,
+                        "opts", &jopts,
+                        "channels", &jchans) < 0) {
         errnum = EPROTO;
         goto fail;
     }
-    if (!(cmd->cwd = strdup (cwd))
+    if ((cwd && !(cmd->cwd = strdup (cwd)))
         || (argz_fromjson (jargv, &cmd->argz, &cmd->argz_len) < 0)
         || (envz_fromjson (jenv, &cmd->envz, &cmd->envz_len) < 0)
         || !(cmd->opts = zhash_fromjson (jopts))
@@ -654,19 +742,16 @@ flux_cmd_t * flux_cmd_fromjson (const char *json_str, json_error_t *errp)
     /* All sub-objects of `o` inherit reference from root object so
      *  this decref should free jenv, jargv, ... etc.
      */
-    json_decref (o);
     return cmd;
 
 fail:
-    json_decref (o);
     flux_cmd_destroy (cmd);
     errno = errnum;
     return NULL;
 }
 
-char * flux_cmd_tojson (const flux_cmd_t *cmd)
+json_t *cmd_tojson (const flux_cmd_t *cmd)
 {
-    char *str = NULL;
     json_t *o = json_object ();
     json_t *a;
 
@@ -715,25 +800,23 @@ char * flux_cmd_tojson (const flux_cmd_t *cmd)
         json_decref (a);
         goto err;
     }
-    str = json_dumps (o, JSON_COMPACT);
-    json_decref (o);
-    return str;
+    return o;
 err:
     json_decref (o);
     return NULL;
 }
 
-char **flux_cmd_env_expand (flux_cmd_t *cmd)
+char **cmd_env_expand (flux_cmd_t *cmd)
 {
     return expand_argz (cmd->envz, cmd->envz_len);
 }
 
-char **flux_cmd_argv_expand (flux_cmd_t *cmd)
+char **cmd_argv_expand (flux_cmd_t *cmd)
 {
     return expand_argz (cmd->argz, cmd->argz_len);
 }
 
-int flux_cmd_set_env (flux_cmd_t *cmd, char **env)
+int cmd_set_env (flux_cmd_t *cmd, char **env)
 {
     size_t new_envz_len = 0;
     char *new_envz = NULL;
@@ -749,12 +832,12 @@ int flux_cmd_set_env (flux_cmd_t *cmd, char **env)
     return 0;
 }
 
-zlist_t *flux_cmd_channel_list (flux_cmd_t *cmd)
+zlist_t *cmd_channel_list (flux_cmd_t *cmd)
 {
     return cmd->channels;
 }
 
-int flux_cmd_find_opts (const flux_cmd_t *cmd, const char **substrings)
+int cmd_find_opts (const flux_cmd_t *cmd, const char **substrings)
 {
     void *iter;
     int rv = 0;
diff --git a/src/common/libsubprocess/command.h b/src/common/libsubprocess/command.h
index b98708b4b258..2ec3324cd0de 100644
--- a/src/common/libsubprocess/command.h
+++ b/src/common/libsubprocess/command.h
@@ -8,54 +8,161 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#ifndef _SUBPROCESS_CMD_H
-#define _SUBPROCESS_CMD_H
+#ifndef _SUBPROCESS_COMMAND_H
+#define _SUBPROCESS_COMMAND_H
 
-#include <jansson.h>
-#include <czmq.h>
+#include <sys/types.h>
 
-#include "subprocess.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 /*
- *  Internal only flux_cmd_t interfaces
+ *  flux_cmd_t: An object that defines a command to be run, either
+ *   remotely or as a child of the current process. Includes cmdline
+ *   arguments, environment, and working directory. A flux_cmd_t is
+ *   used to create Flux subprocesses.
  */
+typedef struct flux_command flux_cmd_t;
 
 /*
- *  Return JSON string representation of command object `cmd`
+ *  Create a cmd object, from which subprocesses can be created
  */
-char * flux_cmd_tojson (const flux_cmd_t *cmd);
+flux_cmd_t * flux_cmd_create (int argc, char *argv[], char **env);
 
 /*
- *  Return a newly allocated flux_cmd_t from a JSON string representation.
- *   Returns NULL on failure.
- *   If non-NULL, any jansson decode errors are returned in *errp.
+ *  Create a copy of a cmd object.
  */
-flux_cmd_t *flux_cmd_fromjson (const char *json_str, json_error_t *errp);
+flux_cmd_t * flux_cmd_copy (const flux_cmd_t *cmd);
 
 /*
- *  Return environment for flux_cmd_t as a NULL terminated string array.
+ *  Destroy and free command object `cmd`
  */
-char **flux_cmd_env_expand (flux_cmd_t *cmd);
+void flux_cmd_destroy (flux_cmd_t *cmd);
 
 /*
- *  Return argument vector for flux_cmd_t as NULL terminated string array.
+ *  Append formatted string to argv of `cmd`.
  */
-char **flux_cmd_argv_expand (flux_cmd_t *cmd);
+int flux_cmd_argv_appendf (flux_cmd_t *cmd,
+                           const char *fmt, ...)
+                           __attribute__ ((format (printf, 2, 3)));
 
 /*
- *  Set an entirely new environment, discarding internal one.
+ *  Append string to argv of `cmd`.
  */
-int flux_cmd_set_env (flux_cmd_t *cmd, char **env);
+int flux_cmd_argv_append (flux_cmd_t *cmd, const char *arg);
 
 /*
- *  Return list of channels.  Should not be destryed by caller.
+ *  Delete the nth argument in cmd's argv
  */
-zlist_t *flux_cmd_channel_list (flux_cmd_t *cmd);
+int flux_cmd_argv_delete (flux_cmd_t *cmd, int n);
 
 /*
- * Find opts that contain a specific substring.  Returns 1 if
- * substrings found, 0 if not.
+ *  Insert arg before the nth argument in cmd's argv
  */
-int flux_cmd_find_opts (const flux_cmd_t *cmd, const char **substrings);
+int flux_cmd_argv_insert (flux_cmd_t *cmd, int n, const char *arg);
 
-#endif /* !_SUBPROCESS_CMD_H */
+/*
+ *  Return the current argument count for `cmd`.
+ */
+int flux_cmd_argc (const flux_cmd_t *cmd);
+
+/*
+ *  Return the current argument at index n (range 0 to argc - 1)
+ */
+const char *flux_cmd_arg (const flux_cmd_t *cmd, int n);
+
+/*
+ *  Return a copy of the current cmd as a string. Caller must free
+ */
+char *flux_cmd_stringify (const flux_cmd_t *cmd);
+
+/*
+ *  Set a single environment variable (name) to formatted string `fmt`.
+ *   If `overwrite` is non-zero then overwrite any existing setting for `name`.
+ */
+int flux_cmd_setenvf (flux_cmd_t *cmd,
+                      int overwrite,
+                      const char *name,
+                      const char *fmt,
+                      ...)
+                      __attribute__ ((format (printf, 4, 5)));
+
+/*
+ *  Unset environment variable `name` in the command object `cmd`.
+ *   If `name` is a glob pattern, unset all matching variables.
+ */
+void flux_cmd_unsetenv (flux_cmd_t *cmd, const char *name);
+
+/*
+ *  Return current value for environment variable `name` as set in
+ *   command object `cmd`. If environment variable is not set then NULL
+ *   is returned.
+ */
+const char *flux_cmd_getenv (const flux_cmd_t *cmd, const char *name);
+
+/*
+ *  Set/get the working directory for the command `cmd`.
+ */
+int flux_cmd_setcwd (flux_cmd_t *cmd, const char *cwd);
+const char *flux_cmd_getcwd (const flux_cmd_t *cmd);
+
+/*
+ *  Request a channel for communication between process and caller.
+ *   Callers can write to the subproces via flux_subprocess_write()
+ *   and read from it via flux_subprocess_read(), which is typically
+ *   called from a callback set in 'on_channel_out'.
+ *
+ *  The `name` argument is also used as the name of the environment variable
+ *   in the subprocess environment that is set to the file descriptor number
+ *   of the process side of the socketpair. E.g. name = "FLUX_PMI_FD" would
+ *   result in the environment variable "FLUX_PMI_FD=N" set in the process
+ *   environment.
+ */
+int flux_cmd_add_channel (flux_cmd_t *cmd, const char *name);
+
+/*
+ *  Set generic string options for command object `cmd`. As with environment
+ *   variables, this function adds the option `var` with value `val` to
+ *   the options array for this command. This can be used to enable optional
+ *   behavior for executed processes (e.g. setpgrp(2))
+ *
+ *  String options, note that name indicates the 'name' argument used
+ *  in flux_cmd_add_channel() above.
+ *
+ *  "BUFSIZE" option
+ *
+ *   By default, stdio and channels use an internal buffer of 4 megs.
+ *   The buffer size can be adjusted with this option.
+ *
+ *   - name + "_BUFSIZE" - set buffer size on channel name
+ *   - stdin_BUFSIZE - set buffer size on stdin
+ *   - stdout_BUFSIZE - set buffer size on stdout
+ *   - stderr_BUFSIZE - set buffer size on stderr
+ *
+ *   The BUFSIZE string may be a floating point quantity scaled by
+ *   an optional suffix from the set 'kKMG'.
+ *
+ *  "LINE_BUFFER" option
+ *
+ *    By default, the 'on_output' callback
+ *    is called when a line of data is available (with the exception
+ *    with data after a subprocess has exited).  By setting this
+ *    option to "false", the output callback will be called whenever any amount
+ *    of data is available on the stream.  These options can also be set to
+ *    "true" to keep default behavior of line buffering.
+ *
+ *    - name + "_LINE_BUFFER" - configuring line buffering on channel name
+ *    - stdout_LINE_BUFFER - configure line buffering for stdout
+ *    - stderr_LINE_BUFFER - configure line buffering for stderr
+ */
+int flux_cmd_setopt (flux_cmd_t *cmd, const char *var, const char *val);
+const char *flux_cmd_getopt (flux_cmd_t *cmd, const char *var);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_SUBPROCESS_COMMAND_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/command_private.h b/src/common/libsubprocess/command_private.h
new file mode 100644
index 000000000000..d560e788b651
--- /dev/null
+++ b/src/common/libsubprocess/command_private.h
@@ -0,0 +1,65 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SUBPROCESS_COMMAND_PRIVATE_H
+#define _SUBPROCESS_COMMAND_PRIVATE_H
+
+#include <jansson.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "subprocess.h"
+#include "command.h"
+
+/*
+ *  Internal only flux_cmd_t interfaces
+ */
+
+/*
+ *  Return JSON representation of command object `cmd`
+ */
+json_t *cmd_tojson (const flux_cmd_t *cmd);
+
+/*
+ *  Return a newly allocated flux_cmd_t from a JSON representation.
+ *   Returns NULL on failure.
+ *   If non-NULL, any jansson decode errors are returned in *errp.
+ */
+flux_cmd_t *cmd_fromjson (json_t *o, json_error_t *errp);
+
+/*
+ *  Return environment for flux_cmd_t as a NULL terminated string array.
+ */
+char **cmd_env_expand (flux_cmd_t *cmd);
+
+/*
+ *  Return argument vector for flux_cmd_t as NULL terminated string array.
+ */
+char **cmd_argv_expand (flux_cmd_t *cmd);
+
+/*
+ *  Set an entirely new environment, discarding internal one.
+ */
+int cmd_set_env (flux_cmd_t *cmd, char **env);
+
+/*
+ *  Return list of channels.  Should not be destroyed by caller.
+ */
+zlist_t *cmd_channel_list (flux_cmd_t *cmd);
+
+/*
+ * Find opts that contain a specific substring.  Returns 1 if
+ * substrings found, 0 if not.
+ */
+int cmd_find_opts (const flux_cmd_t *cmd, const char **substrings);
+
+#endif /* !_SUBPROCESS_COMMAND_PRIVATE_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/ev_fbuf_read.c b/src/common/libsubprocess/ev_fbuf_read.c
new file mode 100644
index 000000000000..12ad03575f68
--- /dev/null
+++ b/src/common/libsubprocess/ev_fbuf_read.c
@@ -0,0 +1,199 @@
+/************************************************************\
+ * Copyright 2015 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stddef.h>
+#include <stdbool.h>
+
+#include "src/common/libev/ev.h"
+
+#include "ev_fbuf_read.h"
+
+static bool data_to_read (struct ev_fbuf_read *ebr, bool *is_eof)
+{
+    if (ebr->line) {
+        if (fbuf_has_line (ebr->fb))
+            return true;
+        else {
+            /* if no line, but the buffer is full, we have to flush */
+            if (!fbuf_space (ebr->fb))
+                return true;
+            /* if eof read, no lines, but left over data non-line data,
+             * this data should be flushed to the user */
+            if (ebr->eof_read && fbuf_bytes (ebr->fb))
+                return true;
+        }
+    }
+    else {
+        if (fbuf_bytes (ebr->fb) > 0)
+            return true;
+    }
+
+    if (ebr->eof_read && !ebr->eof_sent && !fbuf_bytes (ebr->fb)) {
+        if (is_eof)
+            (*is_eof) = true;
+        return true;
+    }
+
+    return false;
+}
+
+static void buffer_notify_cb (struct fbuf *fb, void *arg)
+{
+    struct ev_fbuf_read *ebr = arg;
+
+    /* space is available, start ev io watcher again, assuming watcher
+     * is not stopped by user */
+    if (ebr->start && fbuf_space (fb) > 0)
+        ev_io_start (ebr->loop, &(ebr->io_w));
+}
+
+static void prepare_cb (struct ev_loop *loop, ev_prepare *w, int revents)
+{
+    struct ev_fbuf_read *ebr = (struct ev_fbuf_read *)((char *)w
+                            - offsetof (struct ev_fbuf_read, prepare_w));
+
+    if (data_to_read (ebr, NULL) == true)
+        ev_idle_start (loop, &ebr->idle_w);
+}
+
+static void buffer_read_cb (struct ev_loop *loop, ev_io *iow, int revents)
+{
+    struct ev_fbuf_read *ebr = iow->data;
+
+    if (revents & EV_READ) {
+        int ret, space;
+
+        if ((space = fbuf_space (ebr->fb)) < 0)
+            return;
+
+        if ((ret = fbuf_write_from_fd (ebr->fb, ebr->fd, space)) < 0)
+            return;
+
+        if (!ret) {
+            ev_fbuf_read_decref (ebr);
+            (void)fbuf_readonly (ebr->fb);
+            ev_io_stop (ebr->loop, iow);
+        }
+        else if (ret == space) {
+            /* buffer full, buffer_notify_cb will be called
+             * to re-enable io reactor when space is available */
+            ev_io_stop (ebr->loop, iow);
+        }
+    }
+    else {
+        if (ebr->cb)
+            ebr->cb (loop, ebr, revents);
+    }
+}
+
+static void check_cb (struct ev_loop *loop, ev_check *w, int revents)
+{
+    struct ev_fbuf_read *ebr = (struct ev_fbuf_read *)((char *)w
+                            - offsetof (struct ev_fbuf_read, check_w));
+    bool is_eof = false;
+
+    ev_idle_stop (loop, &ebr->idle_w);
+
+    if (data_to_read (ebr, &is_eof) == true) {
+        if (ebr->cb)
+            ebr->cb (loop, ebr, EV_READ);
+
+        if (is_eof)
+            ebr->eof_sent = true;
+    }
+}
+
+int ev_fbuf_read_init (struct ev_fbuf_read *ebr,
+                       int fd,
+                       int size,
+                       ev_fbuf_read_f cb,
+                       struct ev_loop *loop)
+{
+    ebr->cb = cb;
+    ebr->fd = fd;
+    ebr->loop = loop;
+    ebr->start = false;
+    ebr->eof_read = false;
+    ebr->eof_sent = false;
+    ebr->refcnt = 1;
+
+    if (!(ebr->fb = fbuf_create (size)))
+        goto cleanup;
+    fbuf_set_notify (ebr->fb, buffer_notify_cb, ebr);
+
+    ev_prepare_init (&ebr->prepare_w, prepare_cb);
+    ev_check_init (&ebr->check_w, check_cb);
+    ev_idle_init (&ebr->idle_w, NULL);
+    ev_io_init (&ebr->io_w, buffer_read_cb, ebr->fd, EV_READ);
+    ebr->io_w.data = ebr;
+
+    return 0;
+
+cleanup:
+    ev_fbuf_read_cleanup (ebr);
+    return -1;
+}
+
+void ev_fbuf_read_cleanup (struct ev_fbuf_read *ebr)
+{
+    if (ebr) {
+        fbuf_destroy (ebr->fb);
+        ebr->fb = NULL;
+    }
+}
+
+void ev_fbuf_read_start (struct ev_loop *loop, struct ev_fbuf_read *ebr)
+{
+    if (!ebr->start) {
+        ebr->start = true;
+        ev_prepare_start (loop, &ebr->prepare_w);
+        ev_check_start (loop, &ebr->check_w);
+
+        if (fbuf_space (ebr->fb) > 0)
+            ev_io_start (ebr->loop, &(ebr->io_w));
+        /* else: buffer full, buffer_notify_cb will be called
+         * to re-enable io reactor when space is available */
+    }
+}
+
+void ev_fbuf_read_stop (struct ev_loop *loop, struct ev_fbuf_read *ebr)
+{
+    if (ebr->start) {
+        ev_prepare_stop (loop, &ebr->prepare_w);
+        ev_check_stop (loop, &ebr->check_w);
+        ev_io_stop (loop, &ebr->io_w);
+        ev_idle_stop (loop, &ebr->idle_w);
+        ebr->start = false;
+    }
+}
+
+bool ev_fbuf_read_is_active (struct ev_fbuf_read *ebr)
+{
+    return ev_is_active (&ebr->prepare_w);
+}
+
+void ev_fbuf_read_incref (struct ev_fbuf_read *ebr)
+{
+    ebr->refcnt++;
+}
+
+void ev_fbuf_read_decref (struct ev_fbuf_read *ebr)
+{
+    if (--ebr->refcnt == 0)
+        ebr->eof_read = true;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libsubprocess/ev_fbuf_read.h b/src/common/libsubprocess/ev_fbuf_read.h
new file mode 100644
index 000000000000..4dd1b781f54f
--- /dev/null
+++ b/src/common/libsubprocess/ev_fbuf_read.h
@@ -0,0 +1,54 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSUBPROCESS_EV_FBUF_READ_H
+#define _LIBSUBPROCESS_EV_FBUF_READ_H
+
+#include "src/common/libev/ev.h"
+#include "fbuf.h"
+
+struct ev_fbuf_read;
+
+typedef void (*ev_fbuf_read_f)(struct ev_loop *loop,
+                               struct ev_fbuf_read *ebr,
+                               int revents);
+
+struct ev_fbuf_read {
+    int              refcnt;
+    ev_io            io_w;
+    ev_prepare       prepare_w;
+    ev_idle          idle_w;
+    ev_check         check_w;
+    int              fd;
+    ev_fbuf_read_f   cb;
+    struct fbuf      *fb;
+    struct ev_loop   *loop;
+    bool             start;     /* flag, if user started reactor */
+    bool             eof_read;  /* flag, if EOF on stream seen */
+    bool             eof_sent;  /* flag, if EOF to user sent */
+    bool             line;      /* flag, if line buffered */
+    void             *data;
+};
+
+int ev_fbuf_read_init (struct ev_fbuf_read *ebr,
+                       int fd,
+                       int size,
+                       ev_fbuf_read_f cb,
+                       struct ev_loop *loop);
+void ev_fbuf_read_cleanup (struct ev_fbuf_read *ebr);
+void ev_fbuf_read_start (struct ev_loop *loop, struct ev_fbuf_read *ebr);
+void ev_fbuf_read_stop (struct ev_loop *loop, struct ev_fbuf_read *ebr);
+bool ev_fbuf_read_is_active (struct ev_fbuf_read *ebr);
+void ev_fbuf_read_incref (struct ev_fbuf_read *ebr);
+void ev_fbuf_read_decref (struct ev_fbuf_read *ebr);
+
+#endif /* !_LIBSUBPROCESS_EV_FBUF_READ_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/ev_fbuf_write.c b/src/common/libsubprocess/ev_fbuf_write.c
new file mode 100644
index 000000000000..25c28ef1925c
--- /dev/null
+++ b/src/common/libsubprocess/ev_fbuf_write.c
@@ -0,0 +1,152 @@
+/************************************************************\
+ * Copyright 2015 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stddef.h>
+#include <stdbool.h>
+#include <unistd.h>
+#include <errno.h>
+
+#include "src/common/libev/ev.h"
+
+#include "ev_fbuf_write.h"
+
+static void buffer_write_cb (struct ev_loop *loop, ev_io *iow, int revents)
+{
+    struct ev_fbuf_write *ebw = iow->data;
+
+    if (revents & EV_WRITE) {
+        int ret;
+
+        /* Send one time notification so user knows initial buffer
+         * size */
+        if (!ebw->initial_space) {
+            ebw->initial_space = true;
+            if (ebw->cb)
+                ebw->cb (loop, ebw, revents);
+        }
+
+        if ((ret = fbuf_read_to_fd (ebw->fb, ebw->fd, -1)) < 0) {
+            if (ebw->cb)
+                ebw->cb (loop, ebw, EV_ERROR);
+            return;
+        }
+
+        if (ret) {
+            if (ebw->cb)
+                ebw->cb (loop, ebw, revents);
+        }
+
+        if (!fbuf_bytes (ebw->fb) && ebw->eof) {
+            if (close (ebw->fd) < 0)
+                ebw->close_errno = errno;
+            ebw->fd = -1;
+            ebw->closed = true;
+            ebw->eof = false;
+            if (ebw->cb)
+                ebw->cb (loop, ebw, revents);
+        }
+
+        if (!fbuf_bytes (ebw->fb) && !ebw->eof)
+            ev_io_stop (ebw->loop, &(ebw->io_w));
+    }
+    else {
+        if (ebw->cb)
+            ebw->cb (loop, ebw, revents);
+    }
+}
+
+/* data is available, start ev io watcher assuming user has
+ * started the watcher.
+ */
+void ev_fbuf_write_wakeup (struct ev_fbuf_write *ebw)
+{
+    if (ebw->start)
+        ev_io_start (ebw->loop, &(ebw->io_w));
+}
+
+static void buffer_notify_cb (struct fbuf *fb, void *arg)
+{
+    struct ev_fbuf_write *ebw = arg;
+
+    if (fbuf_bytes (fb) > 0)
+        ev_fbuf_write_wakeup (ebw);
+}
+
+int ev_fbuf_write_init (struct ev_fbuf_write *ebw,
+                        int fd,
+                        int size,
+                        ev_fbuf_write_f cb,
+                        struct ev_loop *loop)
+{
+    ebw->cb = cb;
+    ebw->fd = fd;
+    ebw->loop = loop;
+    ebw->start = false;
+
+    if (!(ebw->fb = fbuf_create (size)))
+        goto cleanup;
+
+    /* When any data becomes available, call buffer_notify_cb,
+     * which will start io reactor
+     */
+    fbuf_set_notify (ebw->fb, buffer_notify_cb, ebw);
+
+    ev_io_init (&ebw->io_w, buffer_write_cb, ebw->fd, EV_WRITE);
+    ebw->io_w.data = ebw;
+
+    return 0;
+
+cleanup:
+    ev_fbuf_write_cleanup (ebw);
+    return -1;
+}
+
+void ev_fbuf_write_cleanup (struct ev_fbuf_write *ebw)
+{
+    if (ebw) {
+        fbuf_destroy (ebw->fb);
+        ebw->fb = NULL;
+    }
+}
+
+void ev_fbuf_write_start (struct ev_loop *loop, struct ev_fbuf_write *ebw)
+{
+    if (!ebw->start) {
+        ebw->start = true;
+        /* do not start watcher unless
+         * - we have not sent initial space
+         * - there is data to be written out
+         * - notify EOF
+         */
+        if (!ebw->initial_space || fbuf_bytes (ebw->fb) || ebw->eof)
+            ev_io_start (ebw->loop, &(ebw->io_w));
+    }
+}
+
+void ev_fbuf_write_stop (struct ev_loop *loop, struct ev_fbuf_write *ebw)
+{
+    if (ebw->start) {
+        ev_io_stop (loop, &ebw->io_w);
+        ebw->start = false;
+    }
+}
+
+bool ev_fbuf_write_is_active (struct ev_fbuf_write *ebw)
+{
+    return ev_is_active (&ebw->io_w);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libsubprocess/ev_fbuf_write.h b/src/common/libsubprocess/ev_fbuf_write.h
new file mode 100644
index 000000000000..89245b39defe
--- /dev/null
+++ b/src/common/libsubprocess/ev_fbuf_write.h
@@ -0,0 +1,49 @@
+/************************************************************\
+ * Copyright 2015 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSUBPROCESS_EV_FBUF_WRITE_H
+#define _LIBSUBPROCESS_EV_FBUF_WRITE_H
+
+#include "src/common/libev/ev.h"
+#include "fbuf.h"
+
+struct ev_fbuf_write;
+
+typedef void (*ev_fbuf_write_f)(struct ev_loop *loop,
+                                struct ev_fbuf_write *ebw,
+                                int revents);
+
+struct ev_fbuf_write {
+    ev_io             io_w;
+    int               fd;
+    ev_fbuf_write_f   cb;
+    struct fbuf       *fb;
+    struct ev_loop    *loop;
+    bool              start;    /* flag, if user started reactor */
+    bool              eof;      /* flag, eof written             */
+    bool              closed;   /* flag, fd has been closed      */
+    int               close_errno;  /* errno from close          */
+    bool              initial_space; /* flag, initial space notified */
+    void              *data;
+};
+
+int ev_fbuf_write_init (struct ev_fbuf_write *ebw,
+                        int fd,
+                        int size,
+                        ev_fbuf_write_f cb,
+                        struct ev_loop *loop);
+void ev_fbuf_write_cleanup (struct ev_fbuf_write *ebw);
+void ev_fbuf_write_start (struct ev_loop *loop, struct ev_fbuf_write *ebw);
+void ev_fbuf_write_stop (struct ev_loop *loop, struct ev_fbuf_write *ebw);
+bool ev_fbuf_write_is_active (struct ev_fbuf_write *ebw);
+void ev_fbuf_write_wakeup (struct ev_fbuf_write *ebw);
+#endif /* !_EV_BUFFER_WRITE_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/fbuf.c b/src/common/libsubprocess/fbuf.c
new file mode 100644
index 000000000000..ac041e8a9108
--- /dev/null
+++ b/src/common/libsubprocess/fbuf.c
@@ -0,0 +1,341 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdarg.h>
+
+#include "fbuf.h"
+
+#include "src/common/liblsd/cbuf.h"
+
+#define FBUF_MIN   4096
+
+struct fbuf {
+    int size;
+    bool readonly;
+    cbuf_t cbuf;
+    char *buf;                  /* internal buffer for user reads */
+    int buflen;
+    fbuf_notify_f cb;
+    void *cb_arg;
+};
+
+struct fbuf *fbuf_create (int size)
+{
+    struct fbuf *fb = NULL;
+    int minsize = FBUF_MIN;
+
+    if (size <= 0) {
+        errno = EINVAL;
+        goto cleanup;
+    }
+
+    if (!(fb = calloc (1, sizeof (*fb))))
+        goto cleanup;
+
+    fb->size = size;
+    if (size < FBUF_MIN)
+        minsize = size;
+    else
+        minsize = FBUF_MIN;
+    fb->readonly = false;
+
+    /* buffer can grow to size specified by user */
+    if (!(fb->cbuf = cbuf_create (minsize, fb->size)))
+        goto cleanup;
+
+    if (cbuf_opt_set (fb->cbuf, CBUF_OPT_OVERWRITE, CBUF_NO_DROP) < 0)
+        goto cleanup;
+
+    /* +1 for possible NUL on line reads */
+    fb->buflen = minsize + 1;
+
+    if (!(fb->buf = malloc (fb->buflen)))
+        goto cleanup;
+
+    return fb;
+
+cleanup:
+    fbuf_destroy (fb);
+    return NULL;
+}
+
+void fbuf_set_notify (struct fbuf *fb, fbuf_notify_f cb, void *arg)
+{
+    if (fb) {
+        fb->cb = cb;
+        fb->cb_arg = arg;
+    }
+}
+
+void fbuf_destroy (void *data)
+{
+    struct fbuf *fb = data;
+    if (fb) {
+        int saved_errno = errno;
+        if (fb->cbuf)
+            cbuf_destroy (fb->cbuf);
+        free (fb->buf);
+        free (fb);
+        errno = saved_errno;
+    }
+}
+
+int fbuf_size (struct fbuf *fb)
+{
+    if (!fb) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    return fb->size;
+}
+
+int fbuf_bytes (struct fbuf *fb)
+{
+    if (!fb) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    return cbuf_used (fb->cbuf);
+}
+
+int fbuf_space (struct fbuf *fb)
+{
+    if (!fb) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    return cbuf_free (fb->cbuf);
+}
+
+int fbuf_readonly (struct fbuf *fb)
+{
+    if (!fb) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    fb->readonly = true;
+    return 0;
+}
+
+bool fbuf_is_readonly (struct fbuf *fb)
+{
+    if (!fb) {
+        errno = EINVAL;
+        return false;
+    }
+    return fb->readonly;
+}
+
+static void fbuf_notify (struct fbuf *fb, int old_used)
+{
+    if (cbuf_used (fb->cbuf) != old_used) {
+        if (fb->cb)
+            fb->cb (fb, fb->cb_arg);
+    }
+}
+
+/* check if internal buffer can hold data from user */
+static int return_buffer_check (struct fbuf *fb)
+{
+    int used = cbuf_used (fb->cbuf);
+
+    if (used < 0)
+        return -1;
+
+    assert (used <= fb->size);
+
+    /* +1 for potential NUL char */
+    if (fb->buflen < (used + 1)) {
+        size_t newsize = fb->buflen;
+        char *newbuf;
+
+        while ((newsize < (used + 1))) {
+            newsize = (newsize - 1) * 2 + 1;
+            if (newsize > (fb->size + 1))
+                newsize = fb->size + 1;
+        }
+
+        if (!(newbuf = realloc (fb->buf, newsize)))
+            return -1;
+        fb->buf = newbuf;
+        fb->buflen = newsize;
+    }
+
+    return 0;
+}
+
+const void *fbuf_read (struct fbuf *fb, int len, int *lenp)
+{
+    int ret;
+
+    if (!fb) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (return_buffer_check (fb) < 0)
+        return NULL;
+
+    if (len < 0)
+        len = cbuf_used (fb->cbuf);
+
+    if (len > fb->buflen)
+        len = fb->buflen;
+
+    int old_used = cbuf_used (fb->cbuf);
+
+    if ((ret = cbuf_read (fb->cbuf, fb->buf, len)) < 0)
+        return NULL;
+    fb->buf[ret] = '\0';
+
+    if (lenp)
+        (*lenp) = ret;
+
+    fbuf_notify (fb, old_used);
+
+    return fb->buf;
+}
+
+int fbuf_write (struct fbuf *fb, const void *data, int len)
+{
+    int ret;
+
+    if (!fb || !data || len < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (fb->readonly) {
+        errno = EROFS;
+        return -1;
+    }
+
+    int old_used = cbuf_used (fb->cbuf);
+
+    if ((ret = cbuf_write (fb->cbuf, (void *)data, len, NULL)) < 0)
+        return -1;
+
+    fbuf_notify (fb, old_used);
+
+    return ret;
+}
+
+bool fbuf_has_line (struct fbuf *fb)
+{
+    char buf[1];
+    if (!fb) {
+        errno = EINVAL;
+        return false;
+    }
+    return (cbuf_peek_line (fb->cbuf, buf, 0, 1) > 0);
+}
+
+const void *fbuf_read_line (struct fbuf *fb, int *lenp)
+{
+    int ret;
+
+    if (!fb) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (return_buffer_check (fb) < 0)
+        return NULL;
+
+    int old_used = cbuf_used (fb->cbuf);
+
+    if ((ret = cbuf_read_line (fb->cbuf, fb->buf, fb->buflen, 1)) < 0)
+        return NULL;
+
+    if (lenp)
+        (*lenp) = ret;
+
+    fbuf_notify (fb, old_used);
+
+    return fb->buf;
+}
+
+const void *fbuf_read_trimmed_line (struct fbuf *fb, int *lenp)
+{
+    int tmp_lenp = 0;
+
+    if (!fbuf_read_line (fb, &tmp_lenp))
+        return NULL;
+
+    if (tmp_lenp) {
+        if (fb->buf[tmp_lenp - 1] == '\n') {
+            fb->buf[tmp_lenp - 1] = '\0';
+            tmp_lenp--;
+        }
+    }
+    if (lenp)
+        (*lenp) = tmp_lenp;
+
+    return fb->buf;
+}
+
+int fbuf_read_to_fd (struct fbuf *fb, int fd, int len)
+{
+    int ret;
+
+    if (!fb) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    int old_used = cbuf_used (fb->cbuf);
+
+    if ((ret = cbuf_read_to_fd (fb->cbuf, fd, len)) < 0)
+        return -1;
+
+    fbuf_notify (fb, old_used);
+
+    return ret;
+}
+
+int fbuf_write_from_fd (struct fbuf *fb, int fd, int len)
+{
+    int ret;
+
+    if (!fb) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (fb->readonly) {
+        errno = EROFS;
+        return -1;
+    }
+
+    int old_used = cbuf_used (fb->cbuf);
+
+    if ((ret = cbuf_write_from_fd (fb->cbuf, fd, len, NULL)) < 0)
+        return -1;
+
+    fbuf_notify (fb, old_used);
+
+    return ret;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libsubprocess/fbuf.h b/src/common/libsubprocess/fbuf.h
new file mode 100644
index 000000000000..5b5d38ddeae2
--- /dev/null
+++ b/src/common/libsubprocess/fbuf.h
@@ -0,0 +1,94 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSUBPROCESS_FBUF_H
+#define _LIBSUBPROCESS_FBUF_H
+
+#include <stdbool.h>
+
+/* Create buffer.
+ */
+struct fbuf *fbuf_create (int size);
+
+void fbuf_destroy (void *fb);
+
+/* Returns the buffer size, set when fbuf_create () was called */
+int fbuf_size (struct fbuf *fb);
+
+/* Returns the number of bytes currently stored in fbuf */
+int fbuf_bytes (struct fbuf *fb);
+
+/* Returns the number of bytes of space available in fbuf */
+int fbuf_space (struct fbuf *fb);
+
+/* Manage "readonly" status
+ * - fbuf_readonly() makes it so writes are no longer allowed
+ *   to the buffer.  Reads are allowed until the buffer is empty.
+ *   Changing a buffer to "readonly" can only be called once and
+ *   cannot be disabled.  This is a convenience status can be used to
+ *   indicate to users that the buffer is no longer usable.
+ * - fbuf_is_readonly() returns true if a buffer is readonly,
+ *    and false if not.
+ */
+int fbuf_readonly (struct fbuf *fb);
+bool fbuf_is_readonly (struct fbuf *fb);
+
+/* Read up to [len] bytes of data in the buffer and mark data as
+ * consumed.  Pointer to buffer is returned to user and optionally
+ * length read can be returned to user in [lenp].  The buffer will
+ * always be NUL terminated, so the user may treat returned ptr as a
+ * string.  User shall not free returned pointer.  If no data is
+ * available, returns pointer and length of 0.  Set [len] to -1 to
+ * read all data.
+ */
+const void *fbuf_read (struct fbuf *fb, int len, int *lenp);
+
+/* Write [len] bytes of data into the buffer.  Returns number of bytes
+ * written on success.
+ */
+int fbuf_write (struct fbuf *fb, const void *data, int len);
+
+/* Return true if buffer has at least one unread line */
+bool fbuf_has_line (struct fbuf *fb);
+
+/* Read a line in the buffer and mark data as consumed.  Return buffer
+ * will include newline.  Optionally return length of data returned in
+ * [lenp].  If no line is available, returns pointer and length of 0.
+ * Return NULL on error.
+ */
+const void *fbuf_read_line (struct fbuf *fb, int *lenp);
+
+/* Identical to fbuf_read_line(), but does not return trailing
+ * newline */
+const void *fbuf_read_trimmed_line (struct fbuf *fb, int *lenp);
+
+/* Read up to [len] bytes from buffer to file descriptor [fd] and mark
+ * data as consumed.  Set [len] to -1 to read all data.  Returns
+ * number of bytes read or -1 on error. */
+int fbuf_read_to_fd (struct fbuf *fb, int fd, int len);
+
+/* Write up to [len] bytes to buffer from file descriptor [fd].  Set
+ * [len] to -1 to read an appropriate chunk size.  Returns number of
+ * bytes written on success.
+ */
+int fbuf_write_from_fd (struct fbuf *fb, int fd, int len);
+
+/* FUTURE: append, prepend, printf, add_fbuf, etc. */
+
+typedef void (*fbuf_notify_f) (struct fbuf *fb, void *arg);
+
+/* Set notify callback for internal use by fbuf watchers.
+ * The callback is invoked when the amount of data in the buffer has changed.
+ */
+void fbuf_set_notify (struct fbuf *fb, fbuf_notify_f cb, void *arg);
+
+#endif /* !_LIBSUBPROCESS_FBUF_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/fbuf_watcher.c b/src/common/libsubprocess/fbuf_watcher.c
new file mode 100644
index 000000000000..987f9d1465d2
--- /dev/null
+++ b/src/common/libsubprocess/fbuf_watcher.c
@@ -0,0 +1,289 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <flux/core.h>
+
+#include "src/common/libev/ev.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/fdutils.h"
+#include "src/common/libflux/reactor_private.h"
+
+#include "fbuf_watcher.h"
+#include "ev_fbuf_read.h"
+#include "ev_fbuf_write.h"
+#include "fbuf.h"
+
+static void buffer_read_start (flux_watcher_t *w)
+{
+    struct ev_fbuf_read *ebr = (struct ev_fbuf_read *)w->data;
+    ev_fbuf_read_start (w->r->loop, ebr);
+}
+
+static void buffer_read_stop (flux_watcher_t *w)
+{
+    struct ev_fbuf_read *ebr = (struct ev_fbuf_read *)w->data;
+    ev_fbuf_read_stop (w->r->loop, ebr);
+}
+
+static bool buffer_read_is_active (flux_watcher_t *w)
+{
+    return ev_fbuf_read_is_active (w->data);
+}
+
+static void buffer_read_destroy (flux_watcher_t *w)
+{
+    struct ev_fbuf_read *ebr = (struct ev_fbuf_read *)w->data;
+    ev_fbuf_read_cleanup (ebr);
+}
+
+static void buffer_read_cb (struct ev_loop *loop,
+                            struct ev_fbuf_read *ebr,
+                            int revents)
+{
+    struct flux_watcher *w = ebr->data;
+    if (w->fn)
+        w->fn (ev_userdata (loop), w, libev_to_events (revents), w->arg);
+}
+
+static struct flux_watcher_ops buffer_read_watcher = {
+    .start = buffer_read_start,
+    .stop = buffer_read_stop,
+    .destroy = buffer_read_destroy,
+    .is_active = buffer_read_is_active,
+};
+
+flux_watcher_t *fbuf_read_watcher_create (flux_reactor_t *r,
+                                          int fd,
+                                          int size,
+                                          flux_watcher_f cb,
+                                          int flags,
+                                          void *arg)
+{
+    struct ev_fbuf_read *ebr;
+    flux_watcher_t *w = NULL;
+    int fd_flags;
+
+    if (fd < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if ((fd_flags = fd_get_flags (fd)) < 0)
+        return NULL;
+    if (!(fd_flags & O_NONBLOCK)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (!(w = watcher_create (r,
+                              sizeof (*ebr),
+                              &buffer_read_watcher,
+                              cb,
+                              arg)))
+        goto cleanup;
+
+    ebr = watcher_get_data (w);
+
+    if (ev_fbuf_read_init (ebr,
+                           fd,
+                           size,
+                           buffer_read_cb,
+                           r->loop) < 0)
+        goto cleanup;
+
+    if (flags & FBUF_WATCHER_LINE_BUFFER)
+        ebr->line = true;
+
+    ebr->data = w;
+
+    return w;
+
+cleanup:
+    flux_watcher_destroy (w);
+    return NULL;
+}
+
+struct fbuf *fbuf_read_watcher_get_buffer (flux_watcher_t *w)
+{
+    if (w)
+        return ((struct ev_fbuf_read *)(w->data))->fb;
+    return NULL;
+}
+
+const char *fbuf_read_watcher_get_data (flux_watcher_t *w, int *lenp)
+{
+    if (w) {
+        struct ev_fbuf_read *eb = w->data;
+        const char *data;
+        if (eb->line) {
+            if (!(data = fbuf_read_line (eb->fb, lenp)))
+                return NULL;
+            if (*lenp > 0)
+                return data;
+            /* if no space, have to flush data out */
+            if (!(*lenp) && !fbuf_space (eb->fb))
+                return fbuf_read (eb->fb, -1, lenp);
+        }
+        /* Not line-buffered, or reading last bit of data which does
+         * not contain a newline. Read any data:
+         */
+        return fbuf_read (eb->fb, -1, lenp);
+    }
+    errno = EINVAL;
+    return NULL;
+}
+
+void fbuf_read_watcher_incref (flux_watcher_t *w)
+{
+    if (w)
+        ev_fbuf_read_incref ((struct ev_fbuf_read *)w->data);
+}
+
+void fbuf_read_watcher_decref (flux_watcher_t *w)
+{
+    if (w)
+        ev_fbuf_read_decref ((struct ev_fbuf_read *)w->data);
+}
+
+static void buffer_write_start (flux_watcher_t *w)
+{
+    struct ev_fbuf_write *ebw = (struct ev_fbuf_write *)w->data;
+    ev_fbuf_write_start (w->r->loop, ebw);
+}
+
+static void buffer_write_stop (flux_watcher_t *w)
+{
+    struct ev_fbuf_write *ebw = (struct ev_fbuf_write *)w->data;
+    ev_fbuf_write_stop (w->r->loop, ebw);
+}
+
+static bool buffer_write_is_active (flux_watcher_t *w)
+{
+    return ev_fbuf_write_is_active (w->data);
+}
+
+static void buffer_write_destroy (flux_watcher_t *w)
+{
+    struct ev_fbuf_write *ebw = (struct ev_fbuf_write *)w->data;
+    ev_fbuf_write_cleanup (ebw);
+}
+
+static void buffer_write_cb (struct ev_loop *loop,
+                             struct ev_fbuf_write *ebw,
+                             int revents)
+{
+    struct flux_watcher *w = ebw->data;
+    if (w->fn)
+        w->fn (ev_userdata (loop), w, libev_to_events (revents), w->arg);
+}
+
+static struct flux_watcher_ops buffer_write_watcher = {
+    .start = buffer_write_start,
+    .stop = buffer_write_stop,
+    .destroy = buffer_write_destroy,
+    .is_active = buffer_write_is_active,
+};
+
+flux_watcher_t *fbuf_write_watcher_create (flux_reactor_t *r,
+                                           int fd,
+                                           int size,
+                                           flux_watcher_f cb,
+                                           int flags,
+                                           void *arg)
+{
+    struct ev_fbuf_write *ebw;
+    flux_watcher_t *w = NULL;
+    int fd_flags;
+
+    if (fd < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if ((fd_flags = fd_get_flags (fd)) < 0)
+        return NULL;
+    if (!(fd_flags & O_NONBLOCK)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (!(w = watcher_create (r,
+                              sizeof (*ebw),
+                              &buffer_write_watcher,
+                              cb,
+                              arg)))
+        goto cleanup;
+
+    ebw = watcher_get_data (w);
+
+    if (ev_fbuf_write_init (ebw,
+                            fd,
+                            size,
+                            buffer_write_cb,
+                            r->loop) < 0)
+        goto cleanup;
+
+    ebw->data = w;
+
+    return w;
+
+cleanup:
+    flux_watcher_destroy (w);
+    return NULL;
+}
+
+struct fbuf *fbuf_write_watcher_get_buffer (flux_watcher_t *w)
+{
+    if (w)
+        return ((struct ev_fbuf_write *)(w->data))->fb;
+    return NULL;
+}
+
+int fbuf_write_watcher_close (flux_watcher_t *w)
+{
+    struct ev_fbuf_write *evw;
+    if (!w) {
+        errno = EINVAL;
+        return (-1);
+    }
+    evw = w->data;
+    if (evw->eof) {
+        errno = EINPROGRESS;
+        return (-1);
+    }
+    if (evw->closed) {
+        errno = EINVAL;
+        return (-1);
+    }
+    evw->eof = true;
+    fbuf_readonly (evw->fb);
+    ev_fbuf_write_wakeup (evw);
+    return (0);
+}
+
+int fbuf_write_watcher_is_closed (flux_watcher_t *w, int *errp)
+{
+    if (w) {
+        struct ev_fbuf_write *evw = w->data;
+        if (evw->closed && errp != NULL)
+            *errp = evw->close_errno;
+        return (evw->closed);
+    }
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libsubprocess/fbuf_watcher.h b/src/common/libsubprocess/fbuf_watcher.h
new file mode 100644
index 000000000000..a6c0e328a96d
--- /dev/null
+++ b/src/common/libsubprocess/fbuf_watcher.h
@@ -0,0 +1,86 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _LIBSUBPROCESS_FBUF_WATCHER_H
+#define _LIBSUBPROCESS_FBUF_WATCHER_H
+
+#include <flux/core.h>
+#include "fbuf.h"
+
+enum {
+    FBUF_WATCHER_LINE_BUFFER = 1, /* line buffer data before invoking callback */
+};
+
+/* read watcher
+ *
+ * - data from fd copied into buffer
+ * - when data is available, triggers callback (FLUX_POLLIN)
+ * - on eof, callback will be called with an empty buffer (FLUX_POLLIN)
+ * - if line buffered, second to last callback may not contain a full line
+ * - users should read from the buffer or stop the watcher, to avoid
+ *   excessive event loop iterations without progress
+ */
+flux_watcher_t *fbuf_read_watcher_create (flux_reactor_t *r,
+                                          int fd,
+                                          int size,
+                                          flux_watcher_f cb,
+                                          int flags,
+                                          void *arg);
+
+struct fbuf *fbuf_read_watcher_get_buffer (flux_watcher_t *w);
+
+/* Get next chunk of data from a buffered read watcher. Gets the next
+ * line if the watcher is line buffered.
+ */
+const char *fbuf_read_watcher_get_data (flux_watcher_t *w, int *lenp);
+
+/* Take a reference on read watcher to prevent read of EOF
+ * EOF will be delayed until decref drops refcount to 0.
+ */
+void fbuf_read_watcher_incref (flux_watcher_t *w);
+void fbuf_read_watcher_decref (flux_watcher_t *w);
+
+/* write watcher
+ *
+ * - data from buffer written to fd
+ * - callback triggered after:
+ *   - buffer space available (FLUX_POLLOUT)
+ *   - fbuf_write_watcher_close() was called AND any buffered data has
+ *     been written out (FLUX_POLLOUT)
+ *   - error (FLUX_POLLERR)
+ */
+flux_watcher_t *fbuf_write_watcher_create (flux_reactor_t *r,
+                                           int fd,
+                                           int size,
+                                           flux_watcher_f cb,
+                                           int flags,
+                                           void *arg);
+
+struct fbuf *fbuf_write_watcher_get_buffer (flux_watcher_t *w);
+
+/* "write" EOF to buffer write watcher 'w'. The underlying fd will be closed
+ *  once the buffer is emptied. The underlying struct fbuf will be marked
+ *  readonly and subsequent fbuf_write* calls will return EROFS.
+ *
+ *  Once close(2) completes, the watcher callback is called with FLUX_POLLOUT.
+ *  Use fbuf_write_watcher_is_closed() to check for errors.
+ *
+ * Returns 0 on success, -1 on error with errno set.
+ */
+int fbuf_write_watcher_close (flux_watcher_t *w);
+
+/* Returns 1 if write watcher is closed, errnum from close in close_err */
+int fbuf_write_watcher_is_closed (flux_watcher_t *w, int *close_err);
+
+#endif /* !_LIBSUBPROCESS_FBUF_WATCHER_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libsubprocess/fork.c b/src/common/libsubprocess/fork.c
new file mode 100644
index 000000000000..5ebb4473be22
--- /dev/null
+++ b/src/common/libsubprocess/fork.c
@@ -0,0 +1,299 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <sys/wait.h>
+#include <unistd.h>
+#include <signal.h>
+#include <errno.h>
+
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/fdwalk.h"
+#include "src/common/libutil/llog.h"
+
+#include "subprocess_private.h"
+#include "command_private.h"
+
+extern char **environ;
+
+static int sigmask_unblock_all (void)
+{
+    sigset_t mask;
+    sigemptyset (&mask);
+    return sigprocmask (SIG_SETMASK, &mask, NULL);
+}
+
+static void close_parent_fds (flux_subprocess_t *p)
+{
+    struct subprocess_channel *c;
+    c = zhash_first (p->channels);
+    while (c) {
+        if (c->parent_fd >= 0) {
+            close (c->parent_fd);
+            c->parent_fd = -1;
+        }
+        c = zhash_next (p->channels);
+    }
+}
+
+static void closefd_child (void *arg, int fd)
+{
+    struct idset *ids = arg;
+    if (idset_test (ids, fd))
+        return;
+    close (fd);
+}
+
+/*  Signal parent that child is ready for exec(2) and wait for parent's
+ *   signal to proceed. This is done by writing 1 byte to child side of
+ *   socketpair, and waiting for parent to write one byte back.
+ *
+ * Call fprintf instead of llog_error(), errors in child should
+ *  go to parent error streams.
+ */
+static int local_child_ready (flux_subprocess_t *p)
+{
+    int n;
+    int fd = p->sync_fds[1];
+    char c = 0;
+
+    if (write (fd, &c, sizeof (c)) != 1) {
+        fprintf (stderr, "local_child_ready: write: %s\n", strerror (errno));
+        return -1;
+    }
+    if ((n = read (fd, &c, sizeof (c))) != 1) {
+        fprintf (stderr, "local_child_ready: read (fd=%d): rc=%d: %s\n",
+                 fd, n, strerror (errno));
+        return -1;
+    }
+    return 0;
+}
+
+static void local_child_report_exec_failed_errno (flux_subprocess_t *p, int e)
+{
+    int fd = p->sync_fds[1];
+    /* Call fprintf instead of llog_error(), errors in child
+     * should go to parent error streams. */
+    if (write (fd, &e, sizeof (e)) != sizeof (e))
+        fprintf (stderr, "local_child_report_exec_failed_errno: %s\n",
+                 strerror (errno));
+}
+
+#if CODE_COVERAGE_ENABLED
+void __gcov_dump (void);
+void __gcov_reset (void);
+#endif
+static int local_child (flux_subprocess_t *p)
+{
+    struct subprocess_channel *c;
+    int errnum;
+    char **argv;
+    const char *cwd;
+    struct idset *ids;
+
+    /* Throughout this function use _exit() instead of exit(), to
+     * avoid calling any atexit() routines of parent.
+     *
+     * Call fprintf instead of llog_error(), errors in child
+     * should go to parent error streams.
+     */
+
+    if (sigmask_unblock_all () < 0)
+        fprintf (stderr, "sigprocmask: %s\n", strerror (errno));
+
+    close_parent_fds (p);
+
+    if (!(p->flags & FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH)) {
+        if ((c = zhash_lookup (p->channels, "stdin"))) {
+            if (dup2 (c->child_fd, STDIN_FILENO) < 0) {
+                fprintf (stderr, "dup2: %s\n", strerror (errno));
+                _exit (1);
+            }
+        }
+
+        if ((c = zhash_lookup (p->channels, "stdout"))) {
+            if (dup2 (c->child_fd, STDOUT_FILENO) < 0) {
+                fprintf (stderr, "dup2: %s\n", strerror (errno));
+                _exit (1);
+            }
+        }
+        else
+            close (STDOUT_FILENO);
+
+        if ((c = zhash_lookup (p->channels, "stderr"))) {
+            if (dup2 (c->child_fd, STDERR_FILENO) < 0) {
+                fprintf (stderr, "dup2: %s\n", strerror (errno));
+                _exit (1);
+            }
+        }
+        else
+            close (STDERR_FILENO);
+    }
+
+    // Change working directory
+    if ((cwd = flux_cmd_getcwd (p->cmd)) && chdir (cwd) < 0) {
+        fprintf (stderr,
+                 "Could not change dir to %s: %s. Going to /tmp instead\n",
+                 cwd, strerror (errno));
+        if (chdir ("/tmp") < 0)
+            _exit (1);
+    }
+
+    // Send ready to parent
+    if (local_child_ready (p) < 0)
+        _exit (1);
+
+    // Close fds
+    if (!(ids = subprocess_childfds (p))
+        || fdwalk (closefd_child, (void *) ids) < 0) {
+        fprintf (stderr, "Failed closing all fds: %s", strerror (errno));
+        _exit (1);
+    }
+    idset_destroy (ids);
+
+    if (p->hooks.pre_exec) {
+        p->in_hook = true;
+        (*p->hooks.pre_exec) (p, p->hooks.pre_exec_arg);
+        p->in_hook = false;
+    }
+
+    if (!(p->flags & FLUX_SUBPROCESS_FLAGS_NO_SETPGRP)
+        && getpgrp () != getpid ()) {
+        if (setpgrp () < 0) {
+            fprintf (stderr, "setpgrp: %s\n", strerror (errno));
+            _exit (1);
+        }
+    }
+
+    environ = cmd_env_expand (p->cmd);
+    argv = cmd_argv_expand (p->cmd);
+    if (!environ || !argv) {
+        fprintf (stderr, "out of memory\n");
+        _exit (1);
+    }
+#if CODE_COVERAGE_ENABLED
+    __gcov_dump ();
+    __gcov_reset ();
+#endif
+    execvp (argv[0], argv);
+
+    errnum = errno;
+    /*
+     * NB: close stdout and stderr here to avoid flushing buffers at exit.
+     *  This can cause duplicate output if parent was running in fully
+     *  buffered mode, and there was buffered output.
+     */
+    close (STDOUT_FILENO);
+    local_child_report_exec_failed_errno (p, errnum);
+    close (STDERR_FILENO);
+#if CODE_COVERAGE_ENABLED
+    __gcov_dump ();
+    __gcov_reset ();
+#endif
+    /* exit code doesn't matter, can't be returned to user */
+    _exit (1);
+}
+
+/*  Wait for child to indicate it is ready for exec(2) by doing a blocking
+ *   read() of one byte on parent side of sync_fds.
+ */
+static int subprocess_parent_wait_on_child (flux_subprocess_t *p)
+{
+    char c;
+
+    if (read (p->sync_fds[0], &c, sizeof (c)) != 1) {
+        llog_debug (p,
+                    "subprocess_parent_wait_on_child: read: %s",
+                    strerror (errno));
+        return -1;
+    }
+    return 0;
+}
+
+/*  Signal child to proceed with exec(2) and read any error from exec
+ *   back on sync_fds.  Return < 0 on failure to signal, or > 0 errnum if
+ *   an exec error was returned from child.
+ */
+static int local_release_child (flux_subprocess_t *p)
+{
+    int fd = p->sync_fds[0];
+    char c = 0;
+    int e = 0;
+    ssize_t n;
+
+    if (write (fd, &c, sizeof (c)) != 1)
+        return -1;
+    if ((n = read (fd, &e, sizeof (e))) < 0)
+        return -1;
+    else if (n == sizeof (int)) {
+        // exec error received
+        return e;
+    }
+    /* else n == 0, child exec'ed and closed sync_fds[1] */
+
+    /* no longer need this fd */
+    close (p->sync_fds[0]);
+    p->sync_fds[0] = -1;
+    return 0;
+}
+
+static int local_exec (flux_subprocess_t *p)
+{
+    int ret;
+    /* N.B. We don't set p->failed_errno here, if locally launched via
+     * flux_local_exec(), will return -1 and errno to caller.  If
+     * called via server, p->failed_errno will be set by remote
+     * handler. */
+    if ((ret = local_release_child (p)) != 0) {
+        /*
+         *  Reap child immediately. Expectation from caller is that
+         *   failure to exec will not require subsequent reaping of
+         *   child.
+         */
+        int status;
+        pid_t pid;
+        if ((pid = waitpid (p->pid, &status, 0)) <= 0)
+            return -1;
+        p->status = status;
+
+        errno = ret;
+        return -1;
+    }
+    return 0;
+}
+
+int create_process_fork (flux_subprocess_t *p)
+{
+    if ((p->pid = fork ()) < 0)
+        return -1;
+
+    if (p->pid == 0)
+        local_child (p); /* No return */
+
+    p->pid_set = true;
+
+    /*  close child end of the sync_fd */
+    close (p->sync_fds[1]);
+    p->sync_fds[1] = -1;
+
+    if (subprocess_parent_wait_on_child (p) < 0)
+        return -1;
+
+    return local_exec (p);
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libsubprocess/fork.h b/src/common/libsubprocess/fork.h
new file mode 100644
index 000000000000..b4efb76ec1d4
--- /dev/null
+++ b/src/common/libsubprocess/fork.h
@@ -0,0 +1,20 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SUBPROCESS_FORK_H
+#define _SUBPROCESS_FORK_H
+
+#include "subprocess.h"
+
+int create_process_fork (flux_subprocess_t *p);
+
+#endif /* !_SUBPROCESS_FORK_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/local.c b/src/common/libsubprocess/local.c
index a59f793e5ee6..0432012a1669 100644
--- a/src/common/libsubprocess/local.c
+++ b/src/common/libsubprocess/local.c
@@ -13,22 +13,25 @@
 #endif
 
 #include <sys/types.h>
-#include <wait.h>
+#include <sys/socket.h>
 #include <unistd.h>
 #include <errno.h>
 
-#include <czmq.h>
-
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/fdwalk.h"
 #include "src/common/libutil/fdutils.h"
+#include "src/common/libutil/llog.h"
 
 #include "subprocess.h"
 #include "subprocess_private.h"
-#include "command.h"
+#include "command_private.h"
+#include "fbuf.h"
 #include "local.h"
+#include "fork.h"
+#include "posix_spawn.h"
 #include "util.h"
 
 static void local_channel_flush (struct subprocess_channel *c)
@@ -39,20 +42,25 @@ static void local_channel_flush (struct subprocess_channel *c)
     if (!(c->flags & CHANNEL_READ))
         return;
 
-    if (!c->eof_sent_to_caller && c->output_f) {
-        flux_buffer_t *fb;
+    if (!c->eof_sent_to_caller && c->output_cb) {
+        struct fbuf *fb;
         int len;
 
-        if (!(fb = flux_buffer_read_watcher_get_buffer (c->buffer_read_w))) {
-            flux_log_error (c->p->h, "flux_buffer_read_watcher_get_buffer");
+        /* always a chance caller may destroy subprocess in callback */
+        subprocess_incref (c->p);
+
+        if (!(fb = fbuf_read_watcher_get_buffer (c->buffer_read_w))) {
+            llog_error (c->p,
+                        "fbuf_read_watcher_get_buffer: %s",
+                        strerror (errno));
             return;
         }
 
-        while ((len = flux_buffer_bytes (fb)) > 0)
-            c->output_f (c->p, c->name);
+        while ((len = fbuf_bytes (fb)) > 0)
+            c->output_cb (c->p, c->name);
 
         /* eof call */
-        c->output_f (c->p, c->name);
+        c->output_cb (c->p, c->name);
 
         c->eof_sent_to_caller = true;
         c->p->channels_eof_sent++;
@@ -61,44 +69,111 @@ static void local_channel_flush (struct subprocess_channel *c)
 
         if (c->p->state == FLUX_SUBPROCESS_EXITED && c->eof_sent_to_caller)
             subprocess_check_completed (c->p);
+
+        subprocess_decref (c->p);
     }
 }
 
-static void local_in_cb (flux_reactor_t *r, flux_watcher_t *w,
-                         int revents, void *arg)
+static void local_in_cb (flux_reactor_t *r,
+                         flux_watcher_t *w,
+                         int revents,
+                         void *arg)
 {
     struct subprocess_channel *c = (struct subprocess_channel *)arg;
     int err = 0;
 
-    if (flux_buffer_write_watcher_is_closed (w, &err) == 1) {
-        if (err)
-            log_msg ("flux_buffer_write_watcher close error: %s",
-                     strerror (err));
-        else
-            c->parent_fd = -1;  /* closed by reactor */
-        flux_watcher_stop (w);  /* c->buffer_write_w */
-        local_channel_flush (c);
+    /* always a chance caller may destroy subprocess in callback */
+    subprocess_incref (c->p);
+
+    if (revents & FLUX_POLLOUT) {
+        if (fbuf_write_watcher_is_closed (w, &err) == 1) {
+            if (err) {
+                llog_error (c->p,
+                            "fbuf_write_watcher close error: %s",
+                            strerror (err));
+            }
+            else
+                c->parent_fd = -1;  /* closed by reactor */
+            flux_watcher_stop (w);  /* c->buffer_write_w */
+            local_channel_flush (c);
+        }
+        else {
+            /* If watcher is not closed, we were alerted to a change
+             * in buffer space
+             *
+             * - c->buffer_space is set to the buffer size on creation
+             * - c->buffer_space is reduced by flux_subprocess_write()
+             * - it is legal to call flux_subprocess_write() before
+             *   the first on_credit() callback
+             * - the fbuf write watcher specially raises POLLOUT
+             *   initially for initial credits
+             * - if data has already been written and moved out of the
+             *   buffer at that time, POLLOUT is raised again
+             *   (separately) after the initial callback returns
+             */
+            struct fbuf *fb;
+
+            if (!(fb = fbuf_write_watcher_get_buffer (c->buffer_write_w))) {
+                llog_error (c->p,
+                            "fbuf_write_watcher_get_buffer: %s",
+                            strerror (errno));
+                goto out;
+            }
+
+            if (!c->initial_credits_sent) {
+                c->initial_credits_sent = true;
+                if (c->p->ops.on_credit)
+                    c->p->ops.on_credit (c->p, c->name, fbuf_size (fb));
+            }
+            else {
+                int new_buffer_space = fbuf_space (fb);
+                /* Only report credits when space is reclaimed, not when
+                 * space is used up */
+                if (new_buffer_space > c->buffer_space) {
+                    int credits = new_buffer_space - c->buffer_space;
+                    /* N.B. Update buffer_space BEFORE the
+                     * on_credit callback, as there is a good chance
+                     * on_credit callback may call
+                     * flux_subprocess_write() */
+                    c->buffer_space = new_buffer_space;
+                    if (c->p->ops.on_credit)
+                        c->p->ops.on_credit (c->p, c->name, credits);
+                }
+            }
+        }
+    }
+    else {
+        llog_error (c->p,
+                    "fbuf_write_watcher: stream %s: %d: %s",
+                    c->name,
+                    revents,
+                    strerror (errno));
     }
-    else
-        flux_log_error (c->p->h, "flux_buffer_write_watcher: stream %s: %d:",
-                        c->name, revents);
+out:
+    subprocess_decref (c->p);
 }
 
 static void local_output (struct subprocess_channel *c,
-                          flux_watcher_t *w, int revents,
+                          flux_watcher_t *w,
+                          int revents,
                           flux_subprocess_output_f output_cb)
 {
+    /* always a chance caller may destroy subprocess in callback */
+    subprocess_incref (c->p);
+
     if (revents & FLUX_POLLIN) {
         bool eof_set = false;
         if (!c->eof_sent_to_caller) {
-            flux_buffer_t *fb;
+            struct fbuf *fb;
 
-            if (!(fb = flux_buffer_read_watcher_get_buffer (w))) {
-                flux_log_error (c->p->h, "flux_buffer_read_watcher_get_buffer");
-                return;
+            if (!(fb = fbuf_read_watcher_get_buffer (w))) {
+                llog_error (c->p,
+                            "fbuf_read_watcher_get_buffer: %s",
+                            strerror (errno));
+                goto out;
             }
 
-            if (!flux_buffer_bytes (fb)) {
+            if (!fbuf_bytes (fb)) {
                 c->eof_sent_to_caller = true;
                 eof_set = true;
                 c->p->channels_eof_sent++;
@@ -121,37 +196,42 @@ static void local_output (struct subprocess_channel *c,
             }
         }
     }
-    else
-        flux_log_error (c->p->h, "flux_buffer_read_watcher on %s: 0x%X:",
-                        c->name, revents);
+    else {
+        llog_error (c->p, "fbuf_read_watcher on %s: 0x%X:", c->name, revents);
+    }
 
     if (c->p->state == FLUX_SUBPROCESS_EXITED && c->eof_sent_to_caller)
         subprocess_check_completed (c->p);
+
+out:
+    subprocess_decref (c->p);
 }
 
-static void local_out_cb (flux_reactor_t *r, flux_watcher_t *w,
-                          int revents, void *arg)
+static void local_out_cb (flux_reactor_t *r,
+                          flux_watcher_t *w,
+                          int revents,
+                          void *arg)
 {
     struct subprocess_channel *c = (struct subprocess_channel *)arg;
     local_output (c, w, revents, c->p->ops.on_channel_out);
 }
 
 static void local_stdout_cb (flux_reactor_t *r, flux_watcher_t *w,
-                               int revents, void *arg)
+                             int revents, void *arg)
 {
     struct subprocess_channel *c = (struct subprocess_channel *)arg;
     local_output (c, w, revents, c->p->ops.on_stdout);
 }
 
 static void local_stderr_cb (flux_reactor_t *r, flux_watcher_t *w,
-                               int revents, void *arg)
+                             int revents, void *arg)
 {
     struct subprocess_channel *c = (struct subprocess_channel *)arg;
     local_output (c, w, revents, c->p->ops.on_stderr);
 }
 
 static int channel_local_setup (flux_subprocess_t *p,
-                                flux_subprocess_output_f output_f,
+                                flux_subprocess_output_f output_cb,
                                 flux_watcher_f in_cb,
                                 flux_watcher_f out_cb,
                                 const char *name,
@@ -164,13 +244,13 @@ static int channel_local_setup (flux_subprocess_t *p,
     int fd_flags;
     int buffer_size;
 
-    if (!(c = channel_create (p, output_f, name, channel_flags))) {
-        flux_log_error (p->h, "calloc");
+    if (!(c = channel_create (p, output_cb, name, channel_flags))) {
+        llog_debug (p, "channel_create %s: %s", name, strerror (errno));
         goto error;
     }
 
     if (socketpair (PF_LOCAL, SOCK_STREAM, 0, fds) < 0) {
-        flux_log_error (p->h, "socketpair");
+        llog_debug (p, "socketpair: %s", strerror (errno));
         goto error;
     }
 
@@ -184,24 +264,38 @@ static int channel_local_setup (flux_subprocess_t *p,
     fds[1] = -1;
 
     if ((fd_flags = fd_set_nonblocking (c->parent_fd)) < 0) {
-        flux_log_error (p->h, "fd_set_nonblocking");
+        llog_debug (p, "fd_set_nonblocking: %s", strerror (errno));
         goto error;
     }
 
     if ((buffer_size = cmd_option_bufsize (p, name)) < 0) {
-        flux_log_error (p->h, "cmd_option_bufsize");
+        llog_debug (p, "cmd_option_bufsize: %s", strerror (errno));
         goto error;
     }
 
     if ((channel_flags & CHANNEL_WRITE) && in_cb) {
-        c->buffer_write_w = flux_buffer_write_watcher_create (p->reactor,
-                                                              c->parent_fd,
-                                                              buffer_size,
-                                                              in_cb,
-                                                              0,
-                                                              c);
+        struct fbuf *fb;
+
+        c->buffer_write_w = fbuf_write_watcher_create (p->reactor,
+                                                       c->parent_fd,
+                                                       buffer_size,
+                                                       in_cb,
+                                                       0,
+                                                       c);
         if (!c->buffer_write_w) {
-            flux_log_error (p->h, "flux_buffer_write_watcher_create");
+            llog_debug (p, "fbuf_write_watcher_create: %s", strerror (errno));
+            goto error;
+        }
+
+        if (!(fb = fbuf_write_watcher_get_buffer (c->buffer_write_w))) {
+            llog_debug (p,
+                        "fbuf_write_watcher_get_buffer: %s",
+                        strerror (errno));
+            goto error;
+        }
+
+        if ((c->buffer_space = fbuf_size (fb)) < 0) {
+            llog_debug (p, "fbuf_size: %s", strerror (errno));
             goto error;
         }
     }
@@ -210,21 +304,21 @@ static int channel_local_setup (flux_subprocess_t *p,
         int wflag;
 
         if ((wflag = cmd_option_line_buffer (p, name)) < 0) {
-            flux_log_error (p->h, "cmd_option_line_buffer");
+            llog_debug (p, "cmd_option_line_buffer: %s", strerror (errno));
             goto error;
         }
 
         if (wflag)
             c->line_buffered = true;
 
-        c->buffer_read_w = flux_buffer_read_watcher_create (p->reactor,
-                                                            c->parent_fd,
-                                                            buffer_size,
-                                                            out_cb,
-                                                            wflag,
-                                                            c);
+        c->buffer_read_w = fbuf_read_watcher_create (p->reactor,
+                                                     c->parent_fd,
+                                                     buffer_size,
+                                                     out_cb,
+                                                     wflag,
+                                                     c);
         if (!c->buffer_read_w) {
-            flux_log_error (p->h, "flux_buffer_read_watcher_create");
+            llog_debug (p, "fbuf_read_watcher_create: %s", strerror (errno));
             goto error;
         }
 
@@ -233,7 +327,7 @@ static int channel_local_setup (flux_subprocess_t *p,
 
     if (channel_flags & CHANNEL_FD) {
         if (asprintf (&e, "%s", name) < 0) {
-            flux_log_error (p->h, "asprintf");
+            llog_debug (p, "asprintf: %s", strerror (errno));
             goto error;
         }
 
@@ -244,17 +338,18 @@ static int channel_local_setup (flux_subprocess_t *p,
                               e,
                               "%d",
                               c->child_fd) < 0) {
-            flux_log_error (p->h, "flux_cmd_setenvf");
+            llog_debug (p, "flux_cmd_setenvf: %s", strerror (errno));
             goto error;
         }
     }
 
     if (zhash_insert (p->channels, name, c) < 0) {
-        flux_log_error (p->h, "zhash_insert");
+        llog_debug (p, "zhash_insert failed");
+        errno = EEXIST;
         goto error;
     }
     if (!zhash_freefn (p->channels, name, channel_destroy)) {
-        flux_log_error (p->h, "zhash_freefn");
+        llog_debug (p, "zhash_freefn failed");
         goto error;
     }
 
@@ -316,17 +411,11 @@ static int local_setup_stdio (flux_subprocess_t *p)
 
 static int local_setup_channels (flux_subprocess_t *p)
 {
-    zlist_t *channels;
+    zlist_t *channels = cmd_channel_list (p->cmd);
     const char *name;
     int channel_flags = CHANNEL_READ | CHANNEL_WRITE | CHANNEL_FD;
-    int len;
-
-    if (!(channels = flux_cmd_channel_list (p->cmd))) {
-        flux_log_error (p->h, "flux_cmd_channel_list");
-        return -1;
-    }
 
-    if (!(len = zlist_size (channels)))
+    if (zlist_size (channels) == 0)
         return 0;
 
     if (!p->ops.on_channel_out)
@@ -347,30 +436,12 @@ static int local_setup_channels (flux_subprocess_t *p)
     return 0;
 }
 
-static int sigmask_unblock_all (void)
-{
-    sigset_t mask;
-    sigemptyset (&mask);
-    return sigprocmask (SIG_SETMASK, &mask, NULL);
-}
-
-static void close_fds (flux_subprocess_t *p, bool parent)
+static void close_child_fds (flux_subprocess_t *p)
 {
     struct subprocess_channel *c;
-    int f = parent ? 0 : 1;
-
-    close (p->sync_fds[f]);
-    p->sync_fds[f] = -1;
-
-    /* note, it is safe to iterate via zhash, child & parent will have
-     * different copies of zhash */
     c = zhash_first (p->channels);
     while (c) {
-        if (parent && c->parent_fd != -1) {
-            close (c->parent_fd);
-            c->parent_fd = -1;
-        }
-        else if (!parent && c->child_fd != -1) {
+        if (c->child_fd != -1) {
             close (c->child_fd);
             c->child_fd = -1;
         }
@@ -378,207 +449,31 @@ static void close_fds (flux_subprocess_t *p, bool parent)
     }
 }
 
-static void close_parent_fds (flux_subprocess_t *p)
-{
-    close_fds (p, true);
-}
-
-static void close_child_fds (flux_subprocess_t *p)
-{
-    close_fds (p, false);
-}
-
-static void closefd_child (void *arg, int fd)
-{
-    flux_subprocess_t *p = arg;
-    struct subprocess_channel *c;
-    if (fd < 3 || fd == p->sync_fds[1])
-        return;
-    c = zhash_first (p->channels);
-    while (c) {
-        if (c->child_fd == fd) {
-            (void) fd_unset_cloexec (fd);
-            return;
-        }
-        c = zhash_next (p->channels);
-    }
-    close (fd);
-}
-
-/*  Signal parent that child is ready for exec(2) and wait for parent's
- *   signal to proceed. This is done by writing 1 byte to child side of
- *   socketpair, and waiting for parent to write one byte back.
- *
- * Call fprintf instead of flux_log_error(), errors in child should
- *  go to parent error streams.
- */
-static int local_child_ready (flux_subprocess_t *p)
-{
-    int n;
-    int fd = p->sync_fds[1];
-    char c = 0;
-
-    if (write (fd, &c, sizeof (c)) != 1) {
-        fprintf (stderr, "local_child_ready: write: %s\n", strerror (errno));
-        return -1;
-    }
-    if ((n = read (fd, &c, sizeof (c))) != 1) {
-        fprintf (stderr, "local_child_ready: read (fd=%d): rc=%d: %s\n",
-                 fd, n, strerror (errno));
-        return -1;
-    }
-    return 0;
-}
-
-static void local_child_report_exec_failed_errno (flux_subprocess_t *p, int e)
-{
-    int fd = p->sync_fds[1];
-    /* Call fprintf instead of flux_log_error(), errors in child
-     * should go to parent error streams. */
-    if (write (fd, &e, sizeof (e)) != sizeof (e))
-        fprintf (stderr, "local_child_report_exec_failed_errno: %s\n",
-                 strerror (errno));
-}
-
-#if CODE_COVERAGE_ENABLED
-void __gcov_flush (void);
-#endif
-static int local_child (flux_subprocess_t *p)
-{
-    struct subprocess_channel *c;
-    int errnum;
-    char **argv;
-    const char *cwd;
-
-    /* Throughout this function use _exit() instead of exit(), to
-     * avoid calling any atexit() routines of parent.
-     *
-     * Call fprintf instead of flux_log_error(), errors in child
-     * should go to parent error streams.
-     */
-
-    if (sigmask_unblock_all () < 0)
-        fprintf (stderr, "sigprocmask: %s\n", strerror (errno));
-
-    close_parent_fds (p);
-
-    if (!(p->flags & FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH)) {
-        if ((c = zhash_lookup (p->channels, "stdin"))) {
-            if (dup2 (c->child_fd, STDIN_FILENO) < 0) {
-                fprintf (stderr, "dup2: %s\n", strerror (errno));
-                _exit (1);
-            }
-        }
-
-        if ((c = zhash_lookup (p->channels, "stdout"))) {
-            if (dup2 (c->child_fd, STDOUT_FILENO) < 0) {
-                fprintf (stderr, "dup2: %s\n", strerror (errno));
-                _exit (1);
-            }
-        }
-        else
-            close (STDOUT_FILENO);
-
-        if ((c = zhash_lookup (p->channels, "stderr"))) {
-            if (dup2 (c->child_fd, STDERR_FILENO) < 0) {
-                fprintf (stderr, "dup2: %s\n", strerror (errno));
-                _exit (1);
-            }
-        }
-        else
-            close (STDERR_FILENO);
-    }
-
-    // Change working directory
-    if ((cwd = flux_cmd_getcwd (p->cmd)) && chdir (cwd) < 0) {
-        fprintf (stderr,
-                 "Could not change dir to %s: %s. Going to /tmp instead\n",
-                 cwd, strerror (errno));
-        if (chdir ("/tmp") < 0)
-            _exit (1);
-    }
-
-    // Send ready to parent
-    if (local_child_ready (p) < 0)
-        _exit (1);
-
-    // Close fds
-    if (fdwalk (closefd_child, (void *) p) < 0) {
-        fprintf (stderr, "Failed closing all fds: %s", strerror (errno));
-        _exit (1);
-    }
-
-    if (p->hooks.pre_exec) {
-        /* always a chance caller may destroy subprocess in callback */
-        flux_subprocess_ref (p);
-        p->in_hook = true;
-        (*p->hooks.pre_exec) (p, p->hooks.pre_exec_arg);
-        p->in_hook = false;
-        flux_subprocess_unref (p);
-    }
-
-    if (p->flags & FLUX_SUBPROCESS_FLAGS_SETPGRP) {
-        if (setpgrp () < 0) {
-            fprintf (stderr, "setpgrp: %s\n", strerror (errno));
-            _exit (1);
-        }
-    }
-
-    environ = flux_cmd_env_expand (p->cmd);
-    argv = flux_cmd_argv_expand (p->cmd);
-    if (!environ || !argv) {
-        fprintf (stderr, "out of memory\n");
-        _exit (1);
-    }
-#if CODE_COVERAGE_ENABLED
-    __gcov_flush ();
-#endif
-    execvp (argv[0], argv);
-
-    errnum = errno;
-    /*
-     * NB: close stdout and stderr here to avoid flushing buffers at exit.
-     *  This can cause duplicate output if parent was running in fully
-     *  bufferred mode, and there was buffered output.
-     */
-    close (STDOUT_FILENO);
-    local_child_report_exec_failed_errno (p, errnum);
-    close (STDERR_FILENO);
-    /* exit code doesn't matter, can't be returned to user */
-    _exit (1);
-}
-
-/*  Wait for child to indicate it is ready for exec(2) by doing a blocking
- *   read() of one byte on parent side of sync_fds.
- */
-static int subprocess_parent_wait_on_child (flux_subprocess_t *p)
-{
-    char c;
-
-    if (read (p->sync_fds[0], &c, sizeof (c)) != 1) {
-        flux_log_error (p->h, "subprocess_parent_wait_on_child: read");
-        return -1;
-    }
-    return 0;
-}
-
-static void child_watch_cb (flux_reactor_t *r, flux_watcher_t *w,
-                            int revents, void *arg)
+static void child_watch_cb (flux_reactor_t *r,
+                            flux_watcher_t *w,
+                            int revents,
+                            void *arg)
 {
     flux_subprocess_t *p = arg;
     int status;
 
     if ((status = flux_child_watcher_get_rstatus (w)) < 0) {
-        flux_log_error (p->h, "flux_child_watcher_get_rstatus");
+        llog_error (p,
+                    "flux_child_watcher_get_rstatus: %s",
+                    strerror (errno));
         return;
     }
 
     p->status = status;
 
+    if (WIFSTOPPED (p->status)) {
+        if (p->ops.on_state_change)
+            (*p->ops.on_state_change) (p, FLUX_SUBPROCESS_STOPPED);
+    }
+
     if (WIFEXITED (p->status) || WIFSIGNALED (p->status)) {
 
-        /* remote/server code may have set EXEC_FAILED or
-         * FAILED on fatal errors.
+        /* remote/server code may have set FAILED on fatal errors.
          */
         if (p->state == FLUX_SUBPROCESS_RUNNING) {
             p->state = FLUX_SUBPROCESS_EXITED;
@@ -594,17 +489,18 @@ static void child_watch_cb (flux_reactor_t *r, flux_watcher_t *w,
         subprocess_check_completed (p);
 }
 
-static int local_fork (flux_subprocess_t *p)
+static int create_process (flux_subprocess_t *p)
 {
-    if ((p->pid = fork ()) < 0)
-        return -1;
-
-    if (p->pid == 0)
-        local_child (p); /* No return */
-
-    p->pid_set = true;
+    if (!(p->flags & FLUX_SUBPROCESS_FLAGS_FORK_EXEC)
+        && !p->hooks.pre_exec
+        && !flux_cmd_getcwd (p->cmd))
+        return create_process_spawn (p);
+    return create_process_fork (p);
+}
 
-    close_child_fds (p);
+static int start_local_watchers (flux_subprocess_t *p)
+{
+    struct subprocess_channel *c;
 
     /* no-op if reactor is !FLUX_REACTOR_SIGCHLD */
     if (!(p->child_w = flux_child_watcher_create (p->reactor,
@@ -612,97 +508,19 @@ static int local_fork (flux_subprocess_t *p)
                                                   true,
                                                   child_watch_cb,
                                                   p))) {
-        flux_log_error (p->h, "flux_child_watcher_create");
+        llog_debug (p, "flux_child_watcher_create: %s", strerror (errno));
         return -1;
     }
-
     flux_watcher_start (p->child_w);
 
-    if (subprocess_parent_wait_on_child (p) < 0)
-        return -1;
-
-    if (p->hooks.post_fork) {
-        /* always a chance caller may destroy subprocess in callback */
-        flux_subprocess_ref (p);
-        p->in_hook = true;
-        (*p->hooks.post_fork) (p, p->hooks.post_fork_arg);
-        p->in_hook = false;
-        flux_subprocess_unref (p);
-    }
-
-    return (0);
-}
-
-/*  Signal child to proceed with exec(2) and read any error from exec
- *   back on sync_fds.  Return < 0 on failure to signal, or > 0 errnum if
- *   an exec error was returned from child.
- */
-static int local_release_child (flux_subprocess_t *p)
-{
-    int fd = p->sync_fds[0];
-    char c = 0;
-    int e = 0;
-    ssize_t n;
-
-    if (write (fd, &c, sizeof (c)) != 1)
-        return -1;
-    if ((n = read (fd, &e, sizeof (e))) < 0)
-        return -1;
-    else if (n == sizeof (int)) {
-        // exec error received
-        return e;
-    }
-    /* else n == 0, child exec'ed and closed sync_fds[1] */
-
-    /* no longer need this fd */
-    close (p->sync_fds[0]);
-    p->sync_fds[0] = -1;
-    return 0;
-}
-
-static int local_exec (flux_subprocess_t *p)
-{
-    if ((p->exec_failed_errno = local_release_child (p)) != 0) {
-        /*
-         *  Reap child immediately. Expectation from caller is that
-         *   failure to exec will not require subsequent reaping of
-         *   child.
-         */
-        int status;
-        pid_t pid;
-        if ((pid = waitpid (p->pid, &status, 0)) <= 0)
-            return -1;
-        p->status = status;
-
-        /* spiritually FLUX_SUBPROCESS_EXEC_FAILED state at this
-         * point */
-        errno = p->exec_failed_errno;
-        return -1;
-    }
-    p->state = FLUX_SUBPROCESS_RUNNING;
-
-    return 0;
-}
-
-static int start_local_watchers (flux_subprocess_t *p)
-{
-    struct subprocess_channel *c;
-
     c = zhash_first (p->channels);
     while (c) {
-        int ret;
         flux_watcher_start (c->buffer_write_w);
-        if ((ret = cmd_option_stream_stop (p, c->name)) < 0)
-            return -1;
-        if (ret) {
-            flux_watcher_start (c->buffer_read_stopped_w);
-        }
-        else {
-            flux_watcher_start (c->buffer_read_w);
-            c->buffer_read_w_started = true;
-        }
+        flux_watcher_start (c->buffer_read_w);
+        c->buffer_read_w_started = true;
         c = zhash_next (p->channels);
     }
+
     return 0;
 }
 
@@ -712,10 +530,18 @@ int subprocess_local_setup (flux_subprocess_t *p)
         return -1;
     if (local_setup_channels (p) < 0)
         return -1;
-    if (local_fork (p) < 0)
-        return -1;
-    if (local_exec (p) < 0)
+    if (create_process (p) < 0)
         return -1;
+
+    p->state = FLUX_SUBPROCESS_RUNNING;
+    close_child_fds (p);
+
+    if (p->hooks.post_fork) {
+        p->in_hook = true;
+        (*p->hooks.post_fork) (p, p->hooks.post_fork_arg);
+        p->in_hook = false;
+    }
+
     if (start_local_watchers (p) < 0)
         return -1;
     return 0;
diff --git a/src/common/libsubprocess/local.h b/src/common/libsubprocess/local.h
index fb85fe3d1337..da24a8551aff 100644
--- a/src/common/libsubprocess/local.h
+++ b/src/common/libsubprocess/local.h
@@ -16,3 +16,5 @@
 int subprocess_local_setup (flux_subprocess_t *p);
 
 #endif /* !_SUBPROCESS_LOCAL_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/posix_spawn.c b/src/common/libsubprocess/posix_spawn.c
new file mode 100644
index 000000000000..f4306473bdd0
--- /dev/null
+++ b/src/common/libsubprocess/posix_spawn.c
@@ -0,0 +1,168 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <unistd.h>
+#include <signal.h>
+#include <spawn.h>
+
+#include <flux/core.h>
+#include <flux/idset.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/fdwalk.h"
+
+#include "subprocess_private.h"
+#include "command_private.h"
+
+struct spawn_close_arg {
+    posix_spawn_file_actions_t *fa;
+    struct idset *childfds;
+};
+
+static void spawn_closefd (void *arg, int fd)
+{
+    struct spawn_close_arg *sc = arg;
+    if (idset_test (sc->childfds, fd))
+        return;
+    posix_spawn_file_actions_addclose (sc->fa, fd);
+}
+
+/*  Setup posix_spawn_file_actions_t for this subprocess.
+ *  - dup stdin/out/err fds onto standard file descriptor numbers
+ *  - set up other file descriptors to close in the child
+ */
+static int spawn_setup_fds (flux_subprocess_t *p,
+                            posix_spawn_file_actions_t *fa)
+{
+    int rc = -1;
+    struct subprocess_channel *c;
+    struct spawn_close_arg sc;
+
+    sc.fa = fa;
+    if (!(sc.childfds = subprocess_childfds (p)))
+        return -1;
+
+    if (!(p->flags & FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH)) {
+        if ((c = zhash_lookup (p->channels, "stdin"))) {
+            if (posix_spawn_file_actions_adddup2 (fa,
+                                                  c->child_fd,
+                                                  STDIN_FILENO) < 0)
+                goto out;
+        }
+        if ((c = zhash_lookup (p->channels, "stdout"))) {
+            if (posix_spawn_file_actions_adddup2 (fa,
+                                                  c->child_fd,
+                                                  STDOUT_FILENO) < 0)
+                goto out;
+        }
+        else if (posix_spawn_file_actions_addclose (fa, STDOUT_FILENO) < 0)
+                goto out;
+
+        if ((c = zhash_lookup (p->channels, "stderr"))) {
+            if (posix_spawn_file_actions_adddup2 (fa,
+                                                  c->child_fd,
+                                                  STDERR_FILENO) < 0)
+                goto out;
+        }
+        else if (posix_spawn_file_actions_addclose (fa, STDERR_FILENO) < 0)
+                goto out;
+    }
+
+    if (fdwalk (spawn_closefd, (void *) &sc) < 0)
+        goto out;
+
+    rc = 0;
+out:
+    idset_destroy (sc.childfds);
+    return rc;
+}
+
+/*  Reset (most) signals to default mask and handlers in child.
+ */
+static int setup_signals (posix_spawnattr_t *attr)
+{
+    sigset_t mask;
+
+    if (sigemptyset (&mask) < 0
+        || posix_spawnattr_setsigmask (attr, &mask) < 0)
+        return -1;
+
+    /*  Iterate list of signals to reset.
+     *
+     *  Note: It has been experimentally determined that setting
+     *  unblockable signals like SIGKILL and SIGSTOP, as well as high
+     *  signal numbers (> 64)  in the sigdefault mask cause spawn
+     *  failures in the child process (exit code 127). Therefore, we
+     *  have to be more targeted than just using sigfillset(3).
+     */
+    for (int i = 1; i < SIGSYS; i++) {
+        if (i != SIGKILL && i != SIGSTOP)
+            if (sigaddset (&mask, i) < 0)
+                return -1;
+    }
+    return posix_spawnattr_setsigdefault (attr, &mask);
+}
+
+/*  Create a child process using posix_spawnp(3).
+ */
+int create_process_spawn (flux_subprocess_t *p)
+{
+    int rc = -1;
+    int retval;
+    int saved_errno;
+    posix_spawn_file_actions_t file_actions;
+    posix_spawnattr_t attr;
+    short flags = POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK;
+    char **env = cmd_env_expand (p->cmd);
+    char **argv = cmd_argv_expand (p->cmd);
+
+    posix_spawnattr_init (&attr);
+    posix_spawn_file_actions_init (&file_actions);
+
+    if (setup_signals (&attr) < 0)
+        goto out;
+
+    /*  If setpgrp(2) is desired for the child process, then add this
+     *  flag to the spawnattr flags.
+     */
+    if (!(p->flags & FLUX_SUBPROCESS_FLAGS_NO_SETPGRP))
+        flags |= POSIX_SPAWN_SETPGROUP;
+    posix_spawnattr_setflags (&attr, flags);
+
+    /*  Setup file descriptors in file_actions
+     */
+    spawn_setup_fds (p, &file_actions);
+
+    /*  Attempt to spawn a new child process */
+    retval = posix_spawnp (&p->pid, argv[0], &file_actions, &attr, argv, env);
+    if (retval != 0) {
+        errno = retval;
+        goto out;
+    }
+    p->pid_set = true;
+    rc = 0;
+out:
+    saved_errno = errno;
+    posix_spawnattr_destroy (&attr);
+    posix_spawn_file_actions_destroy (&file_actions);
+
+    free (env);
+    free (argv);
+
+    errno = saved_errno;
+    return rc;
+}
+
+/*  vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libsubprocess/posix_spawn.h b/src/common/libsubprocess/posix_spawn.h
new file mode 100644
index 000000000000..437674935a36
--- /dev/null
+++ b/src/common/libsubprocess/posix_spawn.h
@@ -0,0 +1,20 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SUBPROCESS_SPAWN_H
+#define _SUBPROCESS_SPAWN_H
+
+#include "subprocess.h"
+
+int create_process_spawn (flux_subprocess_t *p);
+
+#endif /* !_SUBPROCESS_SPAWN_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/remote.c b/src/common/libsubprocess/remote.c
index 1ef4dfab6f6f..85482779035d 100644
--- a/src/common/libsubprocess/remote.c
+++ b/src/common/libsubprocess/remote.c
@@ -13,37 +13,39 @@
 #endif
 
 #include <sys/types.h>
-#include <wait.h>
+#include <sys/wait.h>
 #include <unistd.h>
+#include <stdarg.h>
 #include <errno.h>
-
-#include <czmq.h>
-#include <sodium.h>
+#include <assert.h>
 
 #include <flux/core.h>
 
+#include "ccan/str/str.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/errprintf.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/fdwalk.h"
 #include "src/common/libutil/macros.h"
+#include "src/common/libutil/llog.h"
 #include "src/common/libioencode/ioencode.h"
 
 #include "subprocess.h"
 #include "subprocess_private.h"
-#include "command.h"
+#include "command_private.h"
 #include "remote.h"
 #include "util.h"
+#include "client.h"
+
+static void remote_kill_nowait (flux_subprocess_t *p, int signum);
 
-static void start_channel_watchers (flux_subprocess_t *p)
+static void set_failed (flux_subprocess_t *p, const char *fmt, ...)
 {
-    struct subprocess_channel *c;
-    c = zhash_first (p->channels);
-    while (c) {
-        flux_watcher_start (c->in_prep_w);
-        flux_watcher_start (c->in_check_w);
-        flux_watcher_start (c->out_prep_w);
-        flux_watcher_start (c->out_check_w);
-        c = zhash_next (p->channels);
-    }
+    va_list ap;
+    va_start (ap, fmt);
+    verrprintf (&p->failed_error, fmt, ap);
+    p->failed_errno = errno;
+    va_end (ap);
 }
 
 static void stop_channel_watchers (flux_subprocess_t *p, bool in, bool out)
@@ -51,11 +53,6 @@ static void stop_channel_watchers (flux_subprocess_t *p, bool in, bool out)
     struct subprocess_channel *c;
     c = zhash_first (p->channels);
     while (c) {
-        if (in) {
-            flux_watcher_stop (c->in_prep_w);
-            flux_watcher_stop (c->in_idle_w);
-            flux_watcher_stop (c->in_check_w);
-        }
         if (out) {
             flux_watcher_stop (c->out_prep_w);
             flux_watcher_stop (c->out_idle_w);
@@ -91,7 +88,8 @@ static void sigpending_cb (flux_future_t *f, void *arg)
      * remote kill(2) (f).
      */
     flux_future_fulfill_with (prev, f);
-    flux_future_destroy (f);
+
+    /* Note, f is not destroyed here since it is owned by prev */
 }
 
 static void fwd_pending_signal (flux_subprocess_t *p)
@@ -103,39 +101,55 @@ static void fwd_pending_signal (flux_subprocess_t *p)
         flux_future_t *f = flux_subprocess_kill (p, p->signal_pending);
         if (!f || (flux_future_then (f, -1., sigpending_cb, prev) < 0))
             flux_future_fulfill_error (prev, errno, NULL);
+
+        /*  If 'prev' is destroyed, then also destroy 'f'. Otherwise,
+         *  use-after-free will occur with 'prev' if it is destroyed between
+         *  now and when sigpending_cb() is run:
+         */
+        if (flux_future_aux_set (prev,
+                                 NULL,
+                                 f,
+                                 (flux_free_f) flux_future_destroy) < 0) {
+            flux_future_fulfill_error (prev, errno, NULL);
+            flux_future_destroy (f);
+        }
     }
     else {
         /* Remote process exited or failed, not able to send signal */
         flux_future_fulfill_error (prev, EINVAL, NULL);
     }
     p->signal_pending = 0;
+
+    /*  Now drop the reference on 'prev' added in add_pending_signal().
+     *  This may destroy 'f' created above if the caller destroyed 'prev'
+     *  before this callback was called.
+     */
+    flux_future_decref (prev);
 }
 
 static void process_new_state (flux_subprocess_t *p,
-                               flux_subprocess_state_t state,
-                               int rank, pid_t pid, int errnum, int status)
+                               flux_subprocess_state_t state)
 {
-    if (p->state == FLUX_SUBPROCESS_EXEC_FAILED
-        || p->state == FLUX_SUBPROCESS_FAILED)
+    if (p->state == FLUX_SUBPROCESS_FAILED)
         return;
 
+    if (state == FLUX_SUBPROCESS_STOPPED) {
+        if (p->ops.on_state_change) {
+            /* always a chance caller may destroy subprocess in
+             * callback */
+            subprocess_incref (p);
+            (*p->ops.on_state_change) (p, FLUX_SUBPROCESS_STOPPED);
+            subprocess_decref (p);
+        }
+        return;
+    }
+
     p->state = state;
 
-    if (p->state == FLUX_SUBPROCESS_RUNNING) {
-        p->pid = pid;
-        p->pid_set = true;
-        start_channel_watchers (p);
-    }
-    else if (state == FLUX_SUBPROCESS_EXEC_FAILED) {
-        p->exec_failed_errno = errnum;
-        stop_io_watchers (p);
-    }
-    else if (state == FLUX_SUBPROCESS_EXITED) {
-        p->status = status;
+    if (state == FLUX_SUBPROCESS_EXITED) {
         stop_in_watchers (p);
     }
     else if (state == FLUX_SUBPROCESS_FAILED) {
-        p->failed_errno = errnum;
         stop_io_watchers (p);
     }
 
@@ -146,145 +160,33 @@ static void process_new_state (flux_subprocess_t *p,
         state_change_start (p);
 }
 
-static void remote_in_prep_cb (flux_reactor_t *r,
-                               flux_watcher_t *w,
-                               int revents,
-                               void *arg)
+static void process_add_credit (flux_subprocess_t *p, json_t *channels)
 {
-    struct subprocess_channel *c = arg;
-
-    if (flux_buffer_bytes (c->write_buffer) > 0
-        || (c->closed && !c->write_eof_sent)
-        || (c->p->state == FLUX_SUBPROCESS_EXITED
-            || c->p->state == FLUX_SUBPROCESS_FAILED))
-        flux_watcher_start (c->in_idle_w);
-}
-
-static int remote_write (struct subprocess_channel *c)
-{
-    flux_future_t *f = NULL;
-    json_t *io = NULL;
-    const void *ptr;
-    int lenp;
-    bool eof = false;
-    int rv = -1;
-
-    if (!(ptr = flux_buffer_read (c->write_buffer, -1, &lenp))) {
-        flux_log_error (c->p->h, "flux_buffer_read");
-        goto error;
-    }
-
-    assert (lenp);
-
-    /* if closed / EOF about to be sent, can attach to this RPC to
-     * avoid extra RPC */
-    if (!flux_buffer_bytes (c->write_buffer)
-        && c->closed
-        && !c->write_eof_sent)
-        eof = true;
-
-    /* rank not needed, set to 0 */
-    if (!(io = ioencode (c->name, "0", ptr, lenp, eof))) {
-        flux_log_error (c->p->h, "ioencode");
-        goto error;
-    }
-
-    if (!(f = flux_rpc_pack (c->p->h, "cmb.rexec.write", c->p->rank,
-                             FLUX_RPC_NORESPONSE,
-                             "{ s:i s:O }",
-                             "pid", c->p->pid,
-                             "io", io))) {
-        flux_log_error (c->p->h, "flux_rpc_pack");
-        goto error;
-    }
-
-    if (eof)
-        c->write_eof_sent = true;
-    rv = 0;
-error:
-    /* no response */
-    flux_future_destroy (f);
-    json_decref (io);
-    return rv;
-}
-
-static int remote_close (struct subprocess_channel *c)
-{
-    flux_future_t *f = NULL;
-    json_t *io = NULL;
-    int rv = -1;
-
-    /* rank not needed, set to 0 */
-    if (!(io = ioencode (c->name, "0", NULL, 0, true))) {
-        flux_log_error (c->p->h, "ioencode");
-        goto error;
-    }
-
-    if (!(f = flux_rpc_pack (c->p->h, "cmb.rexec.write", c->p->rank,
-                             FLUX_RPC_NORESPONSE,
-                             "{ s:i s:O }",
-                             "pid", c->p->pid,
-                             "io", io))) {
-        flux_log_error (c->p->h, "flux_rpc_pack");
-        goto error;
+    if (p->ops.on_credit) {
+        const char *key;
+        json_t *value;
+        /* always a chance caller may destroy subprocess in callback */
+        subprocess_incref (p);
+        json_object_foreach (channels, key, value) {
+            int bytes = json_integer_value (value);
+            p->ops.on_credit (p, key, bytes);
+        }
+        subprocess_decref (p);
     }
-
-    rv = 0;
-error:
-    /* no response */
-    flux_future_destroy (f);
-    json_decref (io);
-
-    /* No need to do a "channel_flush", normal io reactor will handle
-     * flush of any data in read buffer */
-    return rv;
 }
 
-static void remote_in_check_cb (flux_reactor_t *r,
-                                flux_watcher_t *w,
-                                int revents,
-                                void *arg)
+static bool remote_out_data_available (struct subprocess_channel *c)
 {
-    struct subprocess_channel *c = arg;
-    flux_future_t *fkill;
-
-    flux_watcher_stop (c->in_idle_w);
-
-    if (flux_buffer_bytes (c->write_buffer) > 0) {
-        if (remote_write (c) < 0) {
-            flux_log_error (c->p->h, "remote_write");
-            goto error;
-        }
-    }
-
-    if (!flux_buffer_bytes (c->write_buffer)
-        && c->closed
-        && !c->write_eof_sent) {
-        if (remote_close (c) < 0) {
-            flux_log_error (c->p->h, "remote_close");
-            goto error;
-        }
-        c->write_eof_sent = true;
-    }
-
-    if (c->write_eof_sent
-        || c->p->state == FLUX_SUBPROCESS_EXITED
-        || c->p->state == FLUX_SUBPROCESS_FAILED) {
-        flux_watcher_stop (c->in_prep_w);
-        flux_watcher_stop (c->in_check_w);
-    }
-
-    return;
-
-error:
-    process_new_state (c->p, FLUX_SUBPROCESS_FAILED,
-                       c->p->rank, -1, errno, 0);
-    if (!(fkill = remote_kill (c->p, SIGKILL)))
-        flux_log_error (c->p->h, "%s: remote_kill", __FUNCTION__);
-    else
-        flux_future_destroy (fkill);
-    flux_future_destroy (c->p->f);
-    c->p->f = NULL;
+    /* no need to handle failure states, on fatal error, the
+     * io watchers are closed */
+    /* N.B. if line buffered and buffer full, gotta flush it
+     * regardless if there's a line or not */
+    if ((c->line_buffered
+         && (fbuf_has_line (c->read_buffer) || !fbuf_space (c->read_buffer)))
+        || (!c->line_buffered && fbuf_bytes (c->read_buffer) > 0)
+        || (c->read_eof_received && !c->eof_sent_to_caller))
+        return true;
+    return false;
 }
 
 static void remote_out_prep_cb (flux_reactor_t *r,
@@ -294,11 +196,7 @@ static void remote_out_prep_cb (flux_reactor_t *r,
 {
     struct subprocess_channel *c = arg;
 
-    /* no need to handle failure states, on fatal error, these
-     * reactors are closed */
-    if ((c->line_buffered && flux_buffer_has_line (c->read_buffer))
-        || (!c->line_buffered && flux_buffer_bytes (c->read_buffer) > 0)
-        || (c->read_eof_received && !c->eof_sent_to_caller))
+    if (remote_out_data_available (c))
         flux_watcher_start (c->out_idle_w);
 }
 
@@ -311,139 +209,120 @@ static void remote_out_check_cb (flux_reactor_t *r,
 
     flux_watcher_stop (c->out_idle_w);
 
+    /* always a chance caller may destroy subprocess in callback */
+    subprocess_incref (c->p);
+
     if ((c->line_buffered
-         && (flux_buffer_has_line (c->read_buffer)
+         && (fbuf_has_line (c->read_buffer)
+             || !fbuf_space (c->read_buffer)
              || (c->read_eof_received
-                 && flux_buffer_bytes (c->read_buffer) > 0)))
-        || (!c->line_buffered && flux_buffer_bytes (c->read_buffer) > 0)) {
-        c->output_f (c->p, c->name);
+                 && fbuf_bytes (c->read_buffer) > 0)))
+        || (!c->line_buffered && fbuf_bytes (c->read_buffer) > 0)) {
+        c->output_cb (c->p, c->name);
     }
 
-    if (!flux_buffer_bytes (c->read_buffer)
+    if (!fbuf_bytes (c->read_buffer)
         && c->read_eof_received
         && !c->eof_sent_to_caller) {
-        c->output_f (c->p, c->name);
+        c->output_cb (c->p, c->name);
         c->eof_sent_to_caller = true;
         c->p->channels_eof_sent++;
     }
 
-    /* no need to handle failure states, on fatal error, these
-     * reactors are closed */
-    if (c->eof_sent_to_caller) {
+    /* no need to handle failure states, on fatal error, the
+     * io watchers are closed */
+    if (!remote_out_data_available (c) || c->eof_sent_to_caller) {
+        /* if no data in buffer, shut down prep/check */
         flux_watcher_stop (c->out_prep_w);
         flux_watcher_stop (c->out_check_w);
 
-        /* close input side as well */
-        flux_watcher_stop (c->in_prep_w);
-        flux_watcher_stop (c->in_idle_w);
-        flux_watcher_stop (c->in_check_w);
-        c->closed = true;
+        /* close input side as well if eof sent to caller */
+        if (c->eof_sent_to_caller)
+            c->closed = true;
     }
 
     if (c->p->state == FLUX_SUBPROCESS_EXITED && c->eof_sent_to_caller)
         subprocess_check_completed (c->p);
+
+    subprocess_decref (c->p);
 }
 
 static int remote_channel_setup (flux_subprocess_t *p,
-                                 flux_subprocess_output_f output_f,
+                                 flux_subprocess_output_f output_cb,
                                  const char *name,
                                  int channel_flags)
 {
     struct subprocess_channel *c = NULL;
     char *e = NULL;
     int save_errno;
-    int buffer_size;
 
-    if (!(c = channel_create (p, output_f, name, channel_flags))) {
-        flux_log_error (p->h, "calloc");
+    if (!(c = channel_create (p, output_cb, name, channel_flags))) {
+        llog_debug (p, "channel_create: %s", strerror (errno));
         goto error;
     }
 
-    if ((buffer_size = cmd_option_bufsize (p, name)) < 0) {
-        flux_log_error (p->h, "cmd_option_bufsize");
-        goto error;
-    }
-
-    if (channel_flags & CHANNEL_WRITE) {
-        if (!(c->write_buffer = flux_buffer_create (buffer_size))) {
-            flux_log_error (p->h, "flux_buffer_create");
-            goto error;
-        }
-
-        if (!(c->in_prep_w = flux_prepare_watcher_create (p->reactor,
-                                                          remote_in_prep_cb,
-                                                          c))) {
-            flux_log_error (p->h, "flux_prepare_watcher_create");
-            goto error;
-        }
-
-        if (!(c->in_idle_w = flux_idle_watcher_create (p->reactor,
-                                                       NULL,
-                                                       c))) {
-            flux_log_error (p->h, "flux_idle_watcher_create");
-            goto error;
-        }
-
-        if (!(c->in_check_w = flux_check_watcher_create (p->reactor,
-                                                         remote_in_check_cb,
-                                                         c))) {
-            flux_log_error (p->h, "flux_check_watcher_create");
-            goto error;
-        }
-
-        /* do not start these watchers till later, cannot send data to
-         * remote until it has reached running state
-         */
-    }
-
     if (channel_flags & CHANNEL_READ) {
         int wflag;
 
         if ((wflag = cmd_option_line_buffer (p, name)) < 0) {
-            flux_log_error (p->h, "cmd_option_line_buffer");
+            llog_debug (p, "cmd_option_line_buffer: %s", strerror (errno));
             goto error;
         }
 
         if (wflag)
             c->line_buffered = true;
 
-        if (!(c->read_buffer = flux_buffer_create (buffer_size))) {
-            flux_log_error (p->h, "flux_buffer_create");
-            goto error;
+        if (!(p->flags & FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF)) {
+            int buffer_size;
+            if ((buffer_size = cmd_option_bufsize (p, name)) < 0) {
+                llog_debug (p, "cmd_option_bufsize: %s", strerror (errno));
+                goto error;
+            }
+            if (!(c->read_buffer = fbuf_create (buffer_size))) {
+                llog_debug (p, "fbuf_create: %s", strerror (errno));
+                goto error;
+            }
+
+            if (!(c->out_prep_w = flux_prepare_watcher_create (p->reactor,
+                                                               remote_out_prep_cb,
+                                                               c))) {
+                llog_debug (p, "flux_prepare_watcher_create: %s", strerror (errno));
+                goto error;
+            }
+
+            if (!(c->out_idle_w = flux_idle_watcher_create (p->reactor,
+                                                            NULL,
+                                                            c))) {
+                llog_debug (p, "flux_idle_watcher_create: %s", strerror (errno));
+                goto error;
+            }
+
+            if (!(c->out_check_w = flux_check_watcher_create (p->reactor,
+                                                              remote_out_check_cb,
+                                                              c))) {
+                llog_debug (p, "flux_check_watcher_create: %s", strerror (errno));
+                goto error;
+            }
+            /* the output check should be called before other check
+             * callbacks, to ensure that the output buffer is emptied
+             * before any check callbacks that may move data into the
+             * buffer.  So we up the priority of the output check
+             * watcher.
+             */
+            flux_watcher_set_priority (c->out_check_w, 1);
+            /* don't start these watchers until we've reached the running
+             * state */
         }
         p->channels_eof_expected++;
-
-        if (!(c->out_prep_w = flux_prepare_watcher_create (p->reactor,
-                                                           remote_out_prep_cb,
-                                                           c))) {
-            flux_log_error (p->h, "flux_prepare_watcher_create");
-            goto error;
-        }
-
-        if (!(c->out_idle_w = flux_idle_watcher_create (p->reactor,
-                                                        NULL,
-                                                        c))) {
-            flux_log_error (p->h, "flux_idle_watcher_create");
-            goto error;
-        }
-
-        if (!(c->out_check_w = flux_check_watcher_create (p->reactor,
-                                                          remote_out_check_cb,
-                                                          c))) {
-            flux_log_error (p->h, "flux_check_watcher_create");
-            goto error;
-        }
-
-        /* don't start these watchers until we've reached the running
-         * state */
     }
 
     if (zhash_insert (p->channels, name, c) < 0) {
-        flux_log_error (p->h, "zhash_insert");
+        llog_debug (p, "zhash_insert failed");
+        errno = EEXIST;
         goto error;
     }
     if (!zhash_freefn (p->channels, name, channel_destroy)) {
-        flux_log_error (p->h, "zhash_freefn");
+        llog_debug (p, "zhash_freefn failed");
         goto error;
     }
 
@@ -495,17 +374,11 @@ static int remote_setup_stdio (flux_subprocess_t *p)
 
 static int remote_setup_channels (flux_subprocess_t *p)
 {
-    zlist_t *channels;
+    zlist_t *channels = cmd_channel_list (p->cmd);
     const char *name;
     int channel_flags = CHANNEL_READ | CHANNEL_WRITE | CHANNEL_FD;
-    int len;
-
-    if (!(channels = flux_cmd_channel_list (p->cmd))) {
-        flux_log_error (p->h, "flux_cmd_channel_list");
-        return -1;
-    }
 
-    if (!(len = zlist_size (channels)))
+    if (zlist_size (channels) == 0)
         return 0;
 
     if (!p->ops.on_channel_out)
@@ -524,277 +397,223 @@ static int remote_setup_channels (flux_subprocess_t *p)
     return 0;
 }
 
-int subprocess_remote_setup (flux_subprocess_t *p)
+int subprocess_remote_setup (flux_subprocess_t *p, const char *service_name)
 {
     if (remote_setup_stdio (p) < 0)
         return -1;
     if (remote_setup_channels (p) < 0)
         return -1;
+    if (!(p->service_name = strdup (service_name)))
+        return -1;
     return 0;
 }
 
-static int remote_state (flux_subprocess_t *p, flux_future_t *f,
-                         int rank)
+static int remote_output_local_unbuf (flux_subprocess_t *p,
+                                      const char *stream,
+                                      const char *data,
+                                      int len,
+                                      bool eof)
 {
-    flux_subprocess_state_t state;
-    pid_t pid = -1;
-    int errnum = 0;
-    int status = 0;
+    struct subprocess_channel *c;
+    int rv = -1;
 
-    if (flux_rpc_get_unpack (f, "{ s:i }", "state", &state) < 0) {
-        flux_log_error (p->h, "%s: flux_rpc_get_unpack", __FUNCTION__);
-        return -1;
-    }
+    /* always a chance caller may destroy subprocess in callback */
+    subprocess_incref (p);
 
-    if (state == FLUX_SUBPROCESS_RUNNING) {
-        if (flux_rpc_get_unpack (f, "{ s:i }", "pid", &pid) < 0) {
-            flux_log_error (p->h, "%s: flux_rpc_get_unpack", __FUNCTION__);
-            return -1;
-        }
+    if (!(c = zhash_lookup (p->channels, stream))) {
+        llog_debug (p,
+                    "Error returning %d bytes received from remote"
+                    " subprocess pid %d %s: unknown channel name",
+                    len,
+                    (int)flux_subprocess_pid (p),
+                    stream);
+        errno = EPROTO;
+        set_failed (p, "error returning unknown channel %s", stream);
+        goto out;
     }
 
-    if (state == FLUX_SUBPROCESS_EXEC_FAILED
-        || state == FLUX_SUBPROCESS_FAILED) {
-        if (flux_rpc_get_unpack (f, "{ s:i }", "errno", &errnum) < 0) {
-            flux_log_error (p->h, "%s: flux_rpc_get_unpack", __FUNCTION__);
-            return -1;
-        }
-    }
+    if (data && len) {
+        c->unbuf_data = data;
+        c->unbuf_len = len;
+        if (eof)
+            c->read_eof_received = true;
+        c->output_cb (c->p, c->name);
+    }
+    /* N.B. any data not consumed by the user is lost, so if eof is
+     * seen, we send it immediately */
+    if (eof && !c->eof_sent_to_caller) {
+        c->read_eof_received = true;
+        c->unbuf_data = NULL;
+        c->unbuf_len = 0;
 
-    if (state == FLUX_SUBPROCESS_EXITED) {
-        if (flux_rpc_get_unpack (f, "{ s:i }", "status", &status) < 0) {
-            flux_log_error (p->h, "%s: flux_rpc_get_unpack", __FUNCTION__);
-            return -1;
-        }
-    }
+        c->output_cb (c->p, c->name);
 
-    process_new_state (p, state, rank, pid, errnum, status);
+        c->eof_sent_to_caller = true;
+        c->p->channels_eof_sent++;
+    }
 
-    return 0;
+    rv = 0;
+out:
+    subprocess_decref (p);
+    return rv;
 }
 
-static int remote_output (flux_subprocess_t *p, flux_future_t *f,
-                          int rank, pid_t pid)
+static int remote_output_buffered (flux_subprocess_t *p,
+                                   const char *stream,
+                                   const char *data,
+                                   int len,
+                                   bool eof)
 {
     struct subprocess_channel *c;
-    const char *stream = NULL;
-    char *data = NULL;
-    int len = 0;
-    bool eof = false;
-    json_t *io = NULL;
-    int rv = -1;
-
-    if (flux_rpc_get_unpack (f, "{ s:o }", "io", &io)) {
-        flux_log_error (p->h, "flux_rpc_get_unpack EPROTO io");
-        goto cleanup;
-    }
-
-    if (iodecode (io, &stream, NULL, &data, &len, &eof) < 0) {
-        flux_log_error (p->h, "iodecode");
-        goto cleanup;
-    }
 
     if (!(c = zhash_lookup (p->channels, stream))) {
-        flux_log_error (p->h, "invalid channel received: rank = %d, pid = %d, stream = %s",
-                 rank, pid, stream);
+        llog_debug (p,
+                    "Error buffering %d bytes received from remote"
+                    " subprocess pid %d %s: unknown channel name",
+                    len,
+                    (int)flux_subprocess_pid (p),
+                    stream);
         errno = EPROTO;
-        goto cleanup;
+        set_failed (p, "error buffering unknown channel %s", stream);
+        return -1;
     }
 
     if (data && len) {
         int tmp;
 
-        if ((tmp = flux_buffer_write (c->read_buffer, data, len)) < 0) {
-            flux_log_error (p->h, "flux_buffer_write");
-            goto cleanup;
+        tmp = fbuf_write (c->read_buffer, data, len);
+        if (tmp >= 0 && tmp < len) {
+            errno = ENOSPC; // short write is promoted to fatal error
+            tmp = -1;
         }
-
-        /* add list of msgs if there is overflow? */
-
-        if (tmp != len) {
-            flux_log_error (p->h, "channel buffer error: rank = %d pid = %d, stream = %s, len = %d",
-                            rank, pid, stream, len);
-            errno = EOVERFLOW;
-            goto cleanup;
+        if (tmp < 0) {
+            llog_debug (p,
+                        "Error buffering %d bytes received from remote"
+                        " subprocess pid %d %s: %s",
+                        len,
+                        (int)flux_subprocess_pid (p),
+                        stream,
+                        strerror (errno));
+            set_failed (p, "error buffering %d bytes of data", len);
+            return -1;
         }
     }
     if (eof) {
         c->read_eof_received = true;
-        if (flux_buffer_readonly (c->read_buffer) < 0)
-            flux_log_error (p->h, "flux_buffer_readonly");
+        if (fbuf_readonly (c->read_buffer) < 0)
+            llog_debug (p, "fbuf_readonly: %s", strerror (errno));
     }
-
-    rv = 0;
-cleanup:
-    free (data);
-    return rv;
-}
-
-static void remote_completion (flux_subprocess_t *p)
-{
-    p->remote_completed = true;
-    /* TBON inorder delivery of messages should guarantee we received
-     * FLUX_SUBPROCESS_EXITED before this.
-     */
-    subprocess_check_completed (p);
+    if (remote_out_data_available (c)) {
+        /* read buffer has stuff in it, start watchers */
+        flux_watcher_start (c->out_prep_w);
+        flux_watcher_start (c->out_check_w);
+    }
+    return 0;
 }
 
-static void remote_exec_cb (flux_future_t *f, void *arg)
+static void rexec_continuation (flux_future_t *f, void *arg)
 {
     flux_subprocess_t *p = arg;
-    const char *type;
-    int rank;
-    pid_t pid;
-
-    if (flux_rpc_get_unpack (f, "{ s:s s:i }",
-                             "type", &type,
-                             "rank", &rank) < 0) {
-        flux_log_error (p->h, "%s: flux_rpc_get_unpack", __FUNCTION__);
-        goto error;
-    }
-
-    if (!strcmp (type, "state")) {
-        if (remote_state (p, f, rank) < 0)
-            goto error;
-        if (p->state == FLUX_SUBPROCESS_EXEC_FAILED
-            || p->state == FLUX_SUBPROCESS_FAILED) {
-            flux_future_destroy (f);
-            p->f = NULL;
+    const char *stream;
+    const char *data;
+    json_t *channels = NULL;
+    int len;
+    bool eof;
+
+    if (subprocess_rexec_get (f) < 0) {
+        if (errno == ENODATA) {
+            p->remote_completed = true;
+            /* Per RFC42, when remote processes are launched, the
+             * process should return finished (i.e. state EXITED)
+             * before returning ENODATA.  It is otherwise considered a
+             * protocol error.
+             *
+             * N.B. There is evidence that the sdexec module has
+             * violated the protocol before #5956.
+             */
+            if (p->state != FLUX_SUBPROCESS_EXITED) {
+                errno = EPROTO;
+                set_failed (p, "%s", strerror (errno));
+                goto error;
+            }
+            subprocess_check_completed (p);
+            return;
         }
-        else
-            flux_future_reset (f);
+        set_failed (p, "%s", future_strerror (f, errno));
+        goto error;
     }
-    else if (!strcmp (type, "output")) {
-        if (flux_rpc_get_unpack (f, "{ s:i }", "pid", &pid) < 0) {
-            flux_log_error (p->h, "%s: flux_rpc_get_unpack", __FUNCTION__);
-            goto error;
-        }
-        if (remote_output (p, f, rank, pid) < 0)
-            goto error;
-        flux_future_reset (f);
+    if (subprocess_rexec_is_started (f, &p->pid)) {
+        p->pid_set = true;
+        process_new_state (p, FLUX_SUBPROCESS_RUNNING);
     }
-    else if (!strcmp (type, "complete")) {
-        remote_completion (p);
-        flux_future_destroy (f);
-        p->f = NULL;
+    else if (subprocess_rexec_is_stopped (f)) {
+        process_new_state (p, FLUX_SUBPROCESS_STOPPED);
     }
-    else {
-        flux_log_error (p->h, "%s: EPROTO", __FUNCTION__);
-        errno = EPROTO;
-        goto error;
+    else if (subprocess_rexec_is_finished (f, &p->status)) {
+        process_new_state (p, FLUX_SUBPROCESS_EXITED);
     }
-
-    return;
-
-error:
-    if (p->state == FLUX_SUBPROCESS_RUNNING) {
-        flux_future_t *fkill;
-        if (!(fkill = remote_kill (p, SIGKILL)))
-            flux_log_error (p->h, "%s: remote_kill", __FUNCTION__);
-        else
-            flux_future_destroy (fkill);
-    }
-    process_new_state (p, FLUX_SUBPROCESS_FAILED,
-                       p->rank, -1, errno, 0);
-    flux_future_destroy (f);
-    p->f = NULL;
-}
-
-static void remote_continuation_cb (flux_future_t *f, void *arg)
-{
-    flux_subprocess_t *p = arg;
-    const char *type;
-    int rank;
-    int save_errno;
-
-    if (flux_rpc_get_unpack (f, "{ s:s s:i }",
-                             "type", &type,
-                             "rank", &rank) < 0) {
-        flux_log_error (p->h, "%s: flux_rpc_get_unpack", __FUNCTION__);
-        goto error;
+    else if (subprocess_rexec_is_add_credit (f, &channels)) {
+        process_add_credit (p, channels);
     }
-
-    if (!strcmp (type, "start")) {
-        flux_future_reset (f);
-        if (flux_future_then (f, -1., remote_exec_cb, p) < 0) {
-            flux_log_error (p->h, "flux_future_then");
-            goto error;
+    else if (subprocess_rexec_is_output (f, &stream, &data, &len, &eof)) {
+        if (p->flags & FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF) {
+            if (remote_output_local_unbuf (p, stream, data, len, eof) < 0)
+                goto error;
+        }
+        else {
+            if (remote_output_buffered (p, stream, data, len, eof) < 0)
+                goto error;
         }
     }
-    else {
-        flux_log_error (p->h, "%s: EPROTO", __FUNCTION__);
-        errno = EPROTO;
-        goto error;
-    }
-
+    flux_future_reset (f);
     return;
 
 error:
-    /* error here is fatal, we can't do anything else b/c we lack a
-     * PID or anything similar.
+    /* c->p->failed_errno and c->p->failed_error expected to be
+     * set before this point (typically via set_failed())
      */
-    process_new_state (p, FLUX_SUBPROCESS_FAILED, p->rank, -1, errno, 0);
-    save_errno = errno;
-    flux_future_destroy (p->f);
-    p->f = NULL;
-    errno = save_errno;
-    return;
+    process_new_state (p, FLUX_SUBPROCESS_FAILED);
+    remote_kill_nowait (p, SIGKILL);
 }
 
 int remote_exec (flux_subprocess_t *p)
 {
-    flux_future_t *f = NULL;
-    char *cmd_str = NULL;
-    int save_errno;
-
-    if (!(cmd_str = flux_cmd_tojson (p->cmd))) {
-        flux_log_error (p->h, "flux_cmd_tojson");
-        goto error;
-    }
-
-    /* completion & state_change cbs always required b/c we use it
-     * internally in this code.  But output callbacks are optional, we
-     * don't care if user doesn't want it.
-     */
-    if (!(f = flux_rpc_pack (p->h, "cmb.rexec", p->rank, 0,
-                             "{s:s s:i s:i s:i}",
-                             "cmd", cmd_str,
-                             "on_channel_out", p->ops.on_channel_out ? 1 : 0,
-                             "on_stdout", p->ops.on_stdout ? 1 : 0,
-                             "on_stderr", p->ops.on_stderr ? 1 : 0))) {
-        flux_log_error (p->h, "flux_rpc");
-        goto error;
-    }
-
-    if (flux_future_then (f, -1., remote_continuation_cb, p) < 0) {
-        flux_log_error (p->h, "flux_future_then");
-        goto error;
+    flux_future_t *f;
+    int flags = 0;
+
+    if (zlist_size (cmd_channel_list (p->cmd)) > 0)
+        flags |= SUBPROCESS_REXEC_CHANNEL;
+    if (p->ops.on_stdout)
+        flags |= SUBPROCESS_REXEC_STDOUT;
+    if (p->ops.on_stderr)
+        flags |= SUBPROCESS_REXEC_STDERR;
+    if (p->ops.on_credit)
+        flags |= SUBPROCESS_REXEC_WRITE_CREDIT;
+
+    if (!(f = subprocess_rexec (p->h, p->service_name, p->rank, p->cmd, flags))
+        || flux_future_then (f, -1., rexec_continuation, p) < 0) {
+        llog_debug (p,
+                    "error sending rexec.exec request: %s",
+                    strerror (errno));
+        flux_future_destroy (f);
+        return -1;
     }
-
     p->f = f;
-    free (cmd_str);
     return 0;
-
- error:
-    save_errno = errno;
-    flux_future_destroy (f);
-    free (cmd_str);
-    errno = save_errno;
-    return -1;
 }
 
 flux_future_t *remote_kill (flux_subprocess_t *p, int signum)
 {
-    flux_future_t *f;
+    return subprocess_kill (p->h, p->service_name, p->rank, p->pid, signum);
+}
 
-    if (!(f = flux_rpc_pack (p->h, "cmb.rexec.signal", p->rank, 0,
-                             "{s:i s:i}",
-                             "pid", p->pid,
-                             "signum", signum))) {
-        flux_log_error (p->h, "%s: flux_rpc_pack", __FUNCTION__);
-        return NULL;
+static void remote_kill_nowait (flux_subprocess_t *p, int signum)
+{
+    if (p->pid_set) {
+        flux_future_t *f;
+        f = remote_kill (p, signum);
+        flux_future_destroy (f);
     }
-    return f;
 }
 
 /*
diff --git a/src/common/libsubprocess/remote.h b/src/common/libsubprocess/remote.h
index 442017501b87..bffa230cf8e5 100644
--- a/src/common/libsubprocess/remote.h
+++ b/src/common/libsubprocess/remote.h
@@ -13,10 +13,12 @@
 
 #include "subprocess.h"
 
-int subprocess_remote_setup (flux_subprocess_t *p);
+int subprocess_remote_setup (flux_subprocess_t *p, const char *service_name);
 
 int remote_exec (flux_subprocess_t *p);
 
 flux_future_t *remote_kill (flux_subprocess_t *p, int signum);
 
 #endif /* !_SUBPROCESS_REMOTE_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/server.c b/src/common/libsubprocess/server.c
index a5a3c5f6f413..05719a9a33bc 100644
--- a/src/common/libsubprocess/server.c
+++ b/src/common/libsubprocess/server.c
@@ -12,159 +12,152 @@
 # include "config.h"
 #endif
 
-#include <sys/types.h>
-#include <wait.h>
-#include <unistd.h>
+#include <unistd.h> // defines environ
+#include <signal.h>
 #include <errno.h>
-
-#include <czmq.h>
-#include <sodium.h>
-
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/errno_safe.h"
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/fdwalk.h"
-#include "src/common/libutil/macros.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/llog.h"
 #include "src/common/libioencode/ioencode.h"
+#include "ccan/str/str.h"
 
 #include "subprocess.h"
 #include "subprocess_private.h"
-#include "command.h"
-#include "remote.h"
+#include "command_private.h"
 #include "server.h"
+#include "client.h"
 #include "util.h"
 
-static const char *auxkey = "flux::rexec";
+extern char **environ;
 
-struct rexec {
-    const flux_msg_t *msg;          // rexec request message
-    flux_subprocess_server_t *s;    // server context
+/* Keys used to store subprocess server, exec request
+ * (i.e. rexec.exec), and 'subprocesses' zlistx handle in the
+ * subprocess object.
+ */
+static const char *srvkey = "flux::server";
+static const char *msgkey = "flux::request";
+static const char *lstkey = "flux::handle";
+
+struct subprocess_server {
+    flux_t *h;
+    char *service_name;
+    char *local_uri;
+    uint32_t rank;
+    subprocess_log_f llog;
+    void *llog_data;
+    zlistx_t *subprocesses;
+    flux_msg_handler_t **handlers;
+    subprocess_server_auth_f auth_cb;
+    void *arg;
+    // The shutdown future is created when user calls shutdown,
+    //  and fulfilled once subprocesses list becomes empty.
+    flux_future_t *shutdown;
 };
 
-static void rexec_destroy (struct rexec *rex)
-{
-    if (rex) {
-        flux_msg_decref (rex->msg);
-        ERRNO_SAFE_WRAP (free, rex);
-    }
-}
+static void server_kill (flux_subprocess_t *p, int signum);
 
-static struct rexec *rexec_create (const flux_msg_t *msg,
-                                   flux_subprocess_server_t *s)
+// zlistx_destructor_fn footprint
+static void proc_destructor (void **item)
 {
-    struct rexec *rex;
-
-    if ((rex = calloc (1, sizeof (*rex)))) {
-        rex->msg = flux_msg_incref (msg);
-        rex->s = s;
+    if (item) {
+        subprocess_decref (*item);
+        *item = NULL;
     }
-    return rex;
-}
-
-static void subprocesses_free_fn (void *arg)
-{
-    flux_subprocess_t *p = arg;
-
-    flux_subprocess_unref (p);
 }
 
-static int store_pid (flux_subprocess_server_t *s, flux_subprocess_t *p)
+static int proc_save (subprocess_server_t *s, flux_subprocess_t *p)
 {
-    pid_t pid = flux_subprocess_pid (p);
-    char *str = NULL;
-    int rv = -1;
-    void *ret = NULL;
+    void *handle;
 
-    if (asprintf (&str, "%d", pid) < 0) {
-        flux_log_error (s->h, "%s: asprintf", __FUNCTION__);
-        goto cleanup;
+    if (!(handle = zlistx_add_end (s->subprocesses, p))) {
+        errno = ENOMEM;
+        return -1;
     }
-
-    if (zhash_insert (s->subprocesses, str, p) < 0) {
-        flux_log_error (s->h, "%s: zhash_insert", __FUNCTION__);
-        goto cleanup;
+    if (flux_subprocess_aux_set (p, lstkey, handle, NULL) < 0) {
+        int saved_errno = errno;
+        zlistx_detach (s->subprocesses, handle);
+        errno = saved_errno;
+        return -1;
     }
-
-    ret = zhash_freefn (s->subprocesses, str, subprocesses_free_fn);
-    assert (ret);
-
-    rv = 0;
-cleanup:
-    free (str);
-    return rv;
+    return 0;
 }
 
-static void remove_pid (flux_subprocess_server_t *s, flux_subprocess_t *p)
+static void proc_delete (subprocess_server_t *s, flux_subprocess_t *p)
 {
-    pid_t pid = flux_subprocess_pid (p);
-    char *str = NULL;
+    int saved_errno = errno;
+    void *handle = flux_subprocess_aux_get (p, lstkey);
 
-    if (asprintf (&str, "%d", pid) < 0) {
-        flux_log_error (s->h, "%s: asprintf", __FUNCTION__);
-        goto cleanup;
-    }
-
-    zhash_delete (s->subprocesses, str);
+    zlistx_delete (s->subprocesses, handle);
 
-    if (!zhash_size (s->subprocesses) && s->terminate_prep_w) {
-        flux_watcher_start (s->terminate_prep_w);
-        flux_watcher_start (s->terminate_check_w);
-    }
+    if (zlistx_size (s->subprocesses) == 0 && s->shutdown)
+        flux_future_fulfill (s->shutdown, NULL, NULL);
 
-cleanup:
-    free (str);
+    errno = saved_errno;
 }
 
-static flux_subprocess_t *lookup_pid (flux_subprocess_server_t *s, pid_t pid)
+static flux_subprocess_t *proc_find_bypid (subprocess_server_t *s, pid_t pid)
 {
-    flux_subprocess_t *p = NULL;
-    char *str = NULL;
-    int save_errno;
-
-    if (asprintf (&str, "%d", pid) < 0)
-        goto cleanup;
+    flux_subprocess_t *p;
 
-    if (!(p = zhash_lookup (s->subprocesses, str))) {
-        errno = ENOENT;
-        goto cleanup;
+    p = zlistx_first (s->subprocesses);
+    while (p) {
+        if (flux_subprocess_pid (p) == pid)
+            return p;
+        p = zlistx_next (s->subprocesses);
     }
-
-cleanup:
-    save_errno = errno;
-    free (str);
-    errno = save_errno;
-    return p;
+    errno = ESRCH;
+    return NULL;
 }
 
-static void subprocess_cleanup (flux_subprocess_t *p)
+/* Find a <service>.exec message with the same sender as msg and matchtag as
+ * specified in the request matchtag field.
+ * N.B. flux_cancel_match() happens to be helpful because RFC 42 subprocess
+ * write works like RFC 6 cancel.
+ */
+static flux_subprocess_t *proc_find_byclient (subprocess_server_t *s,
+                                              const flux_msg_t *request)
 {
-    struct rexec *rex = flux_subprocess_aux_get (p, auxkey);
+    flux_subprocess_t *p;
 
-    assert (rex != NULL);
+    p = zlistx_first (s->subprocesses);
+    while (p) {
+        const flux_msg_t *msg;
 
-    remove_pid (rex->s, p);
+        if ((msg = flux_subprocess_aux_get (p, msgkey))
+            && flux_cancel_match (request, msg))
+            return p;
+        p = zlistx_next (s->subprocesses);
+    }
+    errno = ESRCH;
+    return NULL;
 }
 
-static void rexec_completion_cb (flux_subprocess_t *p)
-{
-    struct rexec *rex = flux_subprocess_aux_get (p, auxkey);
 
-    assert (rex != NULL);
+static void proc_completion_cb (flux_subprocess_t *p)
+{
+    subprocess_server_t *s = flux_subprocess_aux_get (p, srvkey);
+    const flux_msg_t *request = flux_subprocess_aux_get (p, msgkey);
 
     if (p->state != FLUX_SUBPROCESS_FAILED) {
         /* no fallback if this fails */
-        if (flux_respond_pack (rex->s->h, rex->msg, "{s:s s:i}",
-                               "type", "complete",
-                               "rank", rex->s->rank) < 0)
-            flux_log_error (rex->s->h, "%s: flux_respond_pack", __FUNCTION__);
+        if (flux_respond_error (s->h, request, ENODATA, NULL) < 0) {
+            llog_error (s,
+                        "error responding to %s.exec request: %s",
+                        s->service_name,
+                        strerror (errno));
+        }
     }
 
-    subprocess_cleanup (p);
+    proc_delete (s, p);
 }
 
-static void internal_fatal (flux_subprocess_server_t *s, flux_subprocess_t *p)
+static void proc_internal_fatal (flux_subprocess_t *p)
 {
+    subprocess_server_t *s = flux_subprocess_aux_get (p, srvkey);
+
     if (p->state == FLUX_SUBPROCESS_FAILED)
         return;
 
@@ -174,68 +167,80 @@ static void internal_fatal (flux_subprocess_server_t *s, flux_subprocess_t *p)
      */
     p->state = FLUX_SUBPROCESS_FAILED;
     p->failed_errno = errno;
+    errprintf (&p->failed_error, "internal fatal error: %s", strerror (errno));
     state_change_start (p);
 
     /* if we fail here, probably not much can be done */
     if (killpg (p->pid, SIGKILL) < 0) {
-        if (errno != ESRCH)
-            flux_log_error (s->h, "%s: kill", __FUNCTION__);
+        if (errno != ESRCH) {
+            llog_error (s,
+                        "killpg %d SIGKILL: %s",
+                        (int)p->pid,
+                        strerror (errno));
+        }
     }
 }
 
-static void rexec_state_change_cb (flux_subprocess_t *p,
-                                   flux_subprocess_state_t state)
+static void proc_state_change_cb (flux_subprocess_t *p,
+                                  flux_subprocess_state_t state)
 {
-    struct rexec *rex = flux_subprocess_aux_get (p, auxkey);
-
-    assert (rex != NULL);
+    subprocess_server_t *s = flux_subprocess_aux_get (p, srvkey);
+    const flux_msg_t *request = flux_subprocess_aux_get (p, msgkey);
+    int rc = 0;
 
     if (state == FLUX_SUBPROCESS_RUNNING) {
-        if (store_pid (rex->s, p) < 0)
-            goto error;
-        if (flux_respond_pack (rex->s->h, rex->msg, "{s:s s:i s:i s:i}",
-                               "type", "state",
-                               "rank", rex->s->rank,
-                               "pid", flux_subprocess_pid (p),
-                               "state", state) < 0) {
-            flux_log_error (rex->s->h, "%s: flux_respond_pack", __FUNCTION__);
-        }
-    } else if (state == FLUX_SUBPROCESS_EXITED) {
-        if (flux_respond_pack (rex->s->h, rex->msg, "{s:s s:i s:i s:i}",
-                               "type", "state",
-                               "rank", rex->s->rank,
-                               "state", state,
-                               "status", flux_subprocess_status (p)) < 0) {
-            flux_log_error (rex->s->h, "%s: flux_respond_pack", __FUNCTION__);
-        }
-    } else if (state == FLUX_SUBPROCESS_FAILED) {
-        if (flux_respond_pack (rex->s->h, rex->msg, "{s:s s:i s:i s:i}",
-                               "type", "state",
-                               "rank", rex->s->rank,
-                               "state", FLUX_SUBPROCESS_FAILED,
-                               "errno", p->failed_errno) < 0) {
-            flux_log_error (rex->s->h, "%s: flux_respond_pack", __FUNCTION__);
-        }
-        subprocess_cleanup (p);
+        rc = flux_respond_pack (s->h,
+                                request,
+                                "{s:s s:i}",
+                                "type", "started",
+                                "pid", flux_subprocess_pid (p));
+    }
+    else if (state == FLUX_SUBPROCESS_EXITED) {
+        rc = flux_respond_pack (s->h,
+                                request,
+                                "{s:s s:i}",
+                                "type", "finished",
+                                "status", flux_subprocess_status (p));
+    }
+    else if (state == FLUX_SUBPROCESS_STOPPED) {
+        rc = flux_respond_pack (s->h,
+                                request,
+                                "{s:s}",
+                                "type", "stopped");
+    }
+    else if (state == FLUX_SUBPROCESS_FAILED) {
+        const char *errmsg = NULL;
+        if (p->failed_error.text[0] != '\0')
+            errmsg = p->failed_error.text;
+        rc = flux_respond_error (s->h,
+                                 request,
+                                 p->failed_errno,
+                                 errmsg);
+        proc_delete (s, p); // N.B. proc_delete preserves errno
     } else {
         errno = EPROTO;
-        flux_log_error (rex->s->h, "%s: illegal state", __FUNCTION__);
+        llog_error (s, "subprocess entered illegal state %d", state);
         goto error;
     }
-
+    if (rc < 0) {
+        llog_error (s,
+                    "error responding to %s.exec request: %s",
+                    s->service_name,
+                    strerror (errno));
+    }
     return;
 
 error:
-    internal_fatal (rex->s, p);
+    proc_internal_fatal (p);
 }
 
-static int rexec_output (flux_subprocess_t *p,
-                         const char *stream,
-                         flux_subprocess_server_t *s,
-                         const flux_msg_t *msg,
-                         const char *data,
-                         int len,
-                         bool eof)
+static int proc_output (flux_subprocess_t *p,
+                        const char *stream,
+                        subprocess_server_t *s,
+                        const flux_msg_t *msg,
+                        const char *data,
+                        int len,
+                        bool eof)
 {
     json_t *io = NULL;
     char rankstr[64];
@@ -243,16 +248,20 @@ static int rexec_output (flux_subprocess_t *p,
 
     snprintf (rankstr, sizeof (rankstr), "%d", s->rank);
     if (!(io = ioencode (stream, rankstr, data, len, eof))) {
-        flux_log_error (s->h, "%s: ioencode", __FUNCTION__);
+        llog_error (s, "ioencode %s: %s", stream, strerror (errno));
         goto error;
     }
 
-    if (flux_respond_pack (s->h, msg, "{s:s s:i s:i s:O}",
+    if (flux_respond_pack (s->h,
+                           msg,
+                           "{s:s s:i s:O}",
                            "type", "output",
-                           "rank", s->rank,
                            "pid", flux_subprocess_pid (p),
                            "io", io) < 0) {
-        flux_log_error (s->h, "%s: flux_respond_pack", __FUNCTION__);
+        llog_error (s,
+                    "error responding to %s.exec request: %s",
+                    s->service_name,
+                    strerror (errno));
         goto error;
     }
 
@@ -262,225 +271,231 @@ static int rexec_output (flux_subprocess_t *p,
     return rv;
 }
 
-static void rexec_output_cb (flux_subprocess_t *p, const char *stream)
+static void proc_output_cb (flux_subprocess_t *p, const char *stream)
 {
-    struct rexec *rex = flux_subprocess_aux_get (p, auxkey);
-    const char *ptr;
-    int lenp;
+    subprocess_server_t *s = flux_subprocess_aux_get (p, srvkey);
+    const flux_msg_t *request = flux_subprocess_aux_get (p, msgkey);
+    const char *buf;
+    int len;
 
-    assert (rex != NULL);
-
-    if (!(ptr = flux_subprocess_read (p, stream, -1, &lenp))) {
-        flux_log_error (rex->s->h, "%s: flux_subprocess_read", __FUNCTION__);
+    len = flux_subprocess_getline (p, stream, &buf);
+    if (len < 0 && errno == EPERM) // not line buffered
+        len = flux_subprocess_read (p, stream, &buf);
+    if (len < 0) {
+        llog_error (s,
+                    "error reading from subprocess stream %s: %s",
+                    stream,
+                    strerror (errno));
         goto error;
     }
 
-    if (lenp) {
-        if (rexec_output (p, stream, rex->s, rex->msg, ptr, lenp, false) < 0)
+    if (len) {
+        if (proc_output (p, stream, s, request, buf, len, false) < 0)
             goto error;
     }
     else {
-        if (rexec_output (p, stream, rex->s, rex->msg, NULL, 0, true) < 0)
+        if (proc_output (p, stream, s, request, NULL, 0, true) < 0)
             goto error;
     }
 
     return;
 
 error:
-    internal_fatal (rex->s, p);
+    proc_internal_fatal (p);
+}
+
+static void proc_credit_cb (flux_subprocess_t *p, const char *stream, int bytes)
+{
+    subprocess_server_t *s = flux_subprocess_aux_get (p, srvkey);
+    const flux_msg_t *request = flux_subprocess_aux_get (p, msgkey);
+
+    if (flux_respond_pack (s->h,
+                           request,
+                           "{s:s s:{s:i}}",
+                           "type", "add-credit",
+                           "channels",
+                             stream, bytes) < 0) {
+        llog_error (s,
+                    "error responding to %s.exec request: %s",
+                    s->service_name,
+                    strerror (errno));
+        goto error;
+    }
+
+    return;
+
+error:
+    proc_internal_fatal (p);
 }
 
-static void server_exec_cb (flux_t *h, flux_msg_handler_t *mh,
-                              const flux_msg_t *msg, void *arg)
+static void server_exec_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
 {
-    flux_subprocess_server_t *s = arg;
-    const char *cmd_str;
+    subprocess_server_t *s = arg;
+    json_t *cmd_obj;
     flux_cmd_t *cmd = NULL;
-    struct rexec *rex;
     flux_subprocess_t *p = NULL;
     flux_subprocess_ops_t ops = {
-        .on_completion = rexec_completion_cb,
-        .on_state_change = rexec_state_change_cb,
-        .on_channel_out = rexec_output_cb,
-        .on_stdout = rexec_output_cb,
-        .on_stderr = rexec_output_cb,
+        .on_completion = proc_completion_cb,
+        .on_state_change = proc_state_change_cb,
+        .on_channel_out = proc_output_cb,
+        .on_stdout = proc_output_cb,
+        .on_stderr = proc_output_cb,
+        .on_credit = proc_credit_cb,
     };
-    int on_channel_out, on_stdout, on_stderr;
     char **env = NULL;
-
-    if (flux_request_unpack (msg, NULL, "{s:s s:i s:i s:i}",
-                             "cmd", &cmd_str,
-                             "on_channel_out", &on_channel_out,
-                             "on_stdout", &on_stdout,
-                             "on_stderr", &on_stderr))
+    const char *errmsg = NULL;
+    flux_error_t error;
+    int flags;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:o s:i}",
+                             "cmd", &cmd_obj,
+                             "flags", &flags) < 0)
         goto error;
-
-    if (!on_channel_out)
+    if (s->shutdown) {
+        errmsg = "subprocess server is shutting down";
+        errno = ENOSYS;
+        goto error;
+    }
+    if (s->auth_cb && (*s->auth_cb) (msg, s->arg, &error) < 0) {
+        errmsg = error.text;
+        errno = EPERM;
+        goto error;
+    }
+    if (!(flags & SUBPROCESS_REXEC_CHANNEL))
         ops.on_channel_out = NULL;
-    if (!on_stdout)
+    if (!(flags & SUBPROCESS_REXEC_STDOUT))
         ops.on_stdout = NULL;
-    if (!on_stderr)
+    if (!(flags & SUBPROCESS_REXEC_STDERR))
         ops.on_stderr = NULL;
+    if (!(flags & SUBPROCESS_REXEC_WRITE_CREDIT))
+        ops.on_credit = NULL;
 
-    if (!(cmd = flux_cmd_fromjson (cmd_str, NULL)))
+    if (!(cmd = cmd_fromjson (cmd_obj, NULL))) {
+        errmsg = "error parsing command string";
         goto error;
+    }
 
     if (!flux_cmd_argc (cmd)) {
         errno = EPROTO;
+        errmsg = "command string is empty";
         goto error;
     }
 
-    if (!flux_cmd_getcwd (cmd)) {
-        errno = EPROTO;
+    /* if no environment sent, use local server environment */
+    if (!(env = cmd_env_expand (cmd))
+        || (env[0] == NULL && cmd_set_env (cmd, environ))
+        || flux_cmd_setenvf (cmd, 1, "FLUX_URI", "%s", s->local_uri) < 0) {
+        errmsg = "error setting up command environment";
         goto error;
     }
 
-    if (!(env = flux_cmd_env_expand (cmd)))
+    /* Never propagate FLUX_PROXY_REMOTE to processes started from
+     * a subprocess server.
+     */
+    flux_cmd_unsetenv (cmd, "FLUX_PROXY_REMOTE");
+
+    if (!(p = flux_local_exec_ex (flux_get_reactor (s->h),
+                                  0,
+                                  cmd,
+                                  &ops,
+                                  NULL,
+                                  s->llog,
+                                  s->llog_data))) {
+        errprintf (&error, "error launching process: %s", strerror (errno));
+        errmsg = error.text;
         goto error;
-
-    /* if no environment sent, use local server environment */
-    if (env[0] == NULL) {
-        if (flux_cmd_set_env (cmd, environ) < 0) {
-            flux_log_error (s->h, "%s: flux_cmd_set_env", __FUNCTION__);
-            goto error;
-        }
     }
 
-    if (flux_cmd_setenvf (cmd, 1, "FLUX_URI", "%s", s->local_uri) < 0)
-        goto error;
-
-    if (flux_respond_pack (s->h, msg, "{s:s s:i}",
-                           "type", "start",
-                           "rank", s->rank) < 0) {
-        flux_log_error (s->h, "%s: flux_respond_pack", __FUNCTION__);
+    if (flux_subprocess_aux_set (p,
+                                 msgkey,
+                                 (void *)flux_msg_incref (msg),
+                                 (flux_free_f)flux_msg_decref) < 0) {
+        flux_msg_decref (msg);
         goto error;
     }
-
-    if (!(p = flux_exec (s->h,
-                         FLUX_SUBPROCESS_FLAGS_SETPGRP,
-                         cmd,
-                         &ops,
-                         NULL))) {
-        /* error here, generate FLUX_SUBPROCESS_EXEC_FAILED state */
-        if (flux_respond_pack (h, msg, "{s:s s:i s:i s:i}",
-                               "type", "state",
-                               "rank", s->rank,
-                               "state", FLUX_SUBPROCESS_EXEC_FAILED,
-                               "errno", errno) < 0) {
-            flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-            goto error;
-        }
-        goto cleanup;
-    }
-
-    if (!(rex = rexec_create (msg, s)))
+    if (flux_subprocess_aux_set (p, srvkey, s, NULL) < 0)
         goto error;
-    if (flux_subprocess_aux_set (p,
-                                auxkey,
-                                rex,
-                                (flux_free_f)rexec_destroy) < 0) {
-        rexec_destroy (rex);
+    if (proc_save (s, p) < 0)
         goto error;
-    }
 
     flux_cmd_destroy (cmd);
     free (env);
     return;
 
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-cleanup:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0) {
+        llog_error (s,
+                    "error responding to %s.exec request: %s",
+                    s->service_name,
+                    strerror (errno));
+    }
     flux_cmd_destroy (cmd);
     free (env);
-    flux_subprocess_unref (p);
-}
-
-static int write_subprocess (flux_subprocess_server_t *s, flux_subprocess_t *p,
-                             const char *stream, const char *data, int len)
-{
-    int tmp;
-
-    if ((tmp = flux_subprocess_write (p, stream, data, len)) < 0) {
-        flux_log_error (s->h, "%s: flux_subprocess_write", __FUNCTION__);
-        return -1;
-    }
-
-    /* add list of msgs if there is overflow? */
-
-    if (tmp != len) {
-        flux_log_error (s->h, "channel buffer error: rank = %d pid = %d, stream = %s, len = %d",
-                        s->rank, flux_subprocess_pid (p), stream, len);
-        errno = EOVERFLOW;
-        return -1;
-    }
-
-    return 0;
-}
-
-static int close_subprocess (flux_subprocess_server_t *s, flux_subprocess_t *p,
-                             const char *stream)
-{
-    if (flux_subprocess_close (p, stream) < 0) {
-        flux_log_error (s->h, "%s: flux_subprocess_close", __FUNCTION__);
-        return -1;
-    }
-
-    return 0;
+    subprocess_decref (p);
 }
 
-static void server_write_cb (flux_t *h, flux_msg_handler_t *mh,
-                             const flux_msg_t *msg, void *arg)
+static void server_write_cb (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
 {
     flux_subprocess_t *p;
-    flux_subprocess_server_t *s = arg;
+    subprocess_server_t *s = arg;
     const char *stream = NULL;
     char *data = NULL;
     int len = 0;
     bool eof = false;
-    pid_t pid;
+    int matchtag;
     json_t *io = NULL;
-
-    if (flux_request_unpack (msg, NULL, "{ s:i s:o }",
-                             "pid", &pid,
-                             "io", &io) < 0) {
-        /* can't handle error, no pid to sent errno back to, so just
-         * return */
-        flux_log_error (s->h, "%s: flux_request_unpack", __FUNCTION__);
-        return;
-    }
-
-    if (iodecode (io, &stream, NULL, &data, &len, &eof) < 0) {
-        flux_log_error (s->h, "%s: iodecode", __FUNCTION__);
-        return;
+    flux_error_t error;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{ s:i s:o }",
+                             "matchtag", &matchtag,
+                             "io", &io) < 0
+        || iodecode (io, &stream, NULL, &data, &len, &eof) < 0) {
+        llog_error (s,
+                    "Error decoding %s.write request: %s",
+                    s->service_name,
+                    strerror (errno));
+        goto out;
     }
-
-    if (!(p = lookup_pid (s, pid))) {
-        /* can't handle error, no pid to send errno back to, so just
-         * return
-         *
-         * It's common on EOF to be sent and server has already
-         * removed process from hash.  Don't output error in that
-         * case.
-         */
-        if (!(errno == ENOENT && eof))
-            flux_log_error (s->h, "%s: lookup_pid", __FUNCTION__);
+    if (s->auth_cb && (*s->auth_cb) (msg, s->arg, &error) < 0) {
+        llog_error (s, "%s.write: %s", s->service_name, error.text);
         goto out;
     }
 
-    /* Chance subprocess exited/killed/etc. since user write request
-     * was sent.
+    /* If the subprocess can't be found or is no longer running, just silently
+     * drop the data. This is expected if tasks are killed or exit with data
+     * in flight, and is not necessarily an error, and can be common enough
+     * that the log messages end up being a nuisance.
      */
-    if (p->state != FLUX_SUBPROCESS_RUNNING)
+    if (!(p = proc_find_byclient (s, msg))
+        || p->state == FLUX_SUBPROCESS_FAILED
+        || p->state == FLUX_SUBPROCESS_EXITED)
         goto out;
 
     if (data && len) {
-        if (write_subprocess (s, p, stream, data, len) < 0)
+        int rc = flux_subprocess_write (p, stream, data, len);
+        if (rc < 0) {
+            llog_error (s,
+                        "Error writing %d bytes to subprocess %s",
+                        len,
+                        stream);
             goto error;
+        }
     }
     if (eof) {
-        if (close_subprocess (s, p, stream) < 0)
+        if (flux_subprocess_close (p, stream) < 0) {
+            llog_error (s, "Error writing EOF to subprocess %s", stream);
             goto error;
+        }
     }
 
 out:
@@ -489,313 +504,286 @@ static void server_write_cb (flux_t *h, flux_msg_handler_t *mh,
 
 error:
     free (data);
-    internal_fatal (s, p);
+    proc_internal_fatal (p);
 }
 
-static void server_signal_cb (flux_t *h, flux_msg_handler_t *mh,
-                              const flux_msg_t *msg, void *arg)
+static void server_kill_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
 {
-    flux_subprocess_server_t *s = arg;
+    subprocess_server_t *s = arg;
     pid_t pid;
     int signum;
+    flux_error_t error;
+    const char *errmsg = NULL;
 
-    errno = 0;
-
-    if (flux_request_unpack (msg, NULL, "{ s:i s:i }",
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{ s:i s:i }",
                              "pid", &pid,
-                             "signum", &signum) < 0) {
-        flux_log_error (s->h, "%s: flux_request_unpack", __FUNCTION__);
-        errno = EPROTO;
+                             "signum", &signum) < 0)
         goto error;
-    }
-
-    if (!lookup_pid (s, pid))
+    if (s->auth_cb && (*s->auth_cb) (msg, s->arg, &error) < 0) {
+        errmsg = error.text;
+        errno = EPERM;
         goto error;
-
-    if (killpg (pid, signum) < 0) {
-        flux_log_error (s->h, "kill");
+    }
+    if (!proc_find_bypid (s, pid)
+        || killpg (pid, signum) < 0)
         goto error;
+    if (flux_respond (h, msg, NULL) < 0) {
+        llog_error (s,
+                    "error responding to %s.kill request: %s",
+                    s->service_name,
+                    strerror (errno));
     }
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
     return;
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    if (flux_respond_error (h, msg, errno, errmsg) < 0) {
+        llog_error (s,
+                    "error responding to %s.kill request: %s",
+                    s->service_name,
+                    strerror (errno));
+    }
 }
 
-char *subprocess_sender (flux_subprocess_t *p)
+static const char *subprocess_sender (flux_subprocess_t *p)
 {
-    struct rexec *rex = flux_subprocess_aux_get (p, auxkey);
-    char *sender;
-
-    if (!rex || flux_msg_get_route_first (rex->msg, &sender) < 0)
-        return NULL;
-
-    return sender;
+    const flux_msg_t *msg = flux_subprocess_aux_get (p, msgkey);
+    return flux_msg_route_first (msg);
 }
 
 static json_t *process_info (flux_subprocess_t *p)
 {
     flux_cmd_t *cmd;
-    char *cmd_str = NULL;
-    char *sender = NULL;
     json_t *info = NULL;
+    char *s;
 
-    if (!(cmd = flux_subprocess_get_cmd (p)))
-        goto cleanup;
-
-    if (!(cmd_str = flux_cmd_tojson (cmd)))
-        goto cleanup;
-
-    if (!(sender = subprocess_sender (p))) {
-        errno = ENOENT;
-        goto cleanup;
-    }
-
-    /* very limited returned, just for testing */
+    if (!(cmd = flux_subprocess_get_cmd (p))
+            || !(s = flux_cmd_stringify (cmd)))
+        return NULL;
     if (!(info = json_pack ("{s:i s:s}",
                             "pid", flux_subprocess_pid (p),
-                            "sender", sender))) {
+                            "cmd", flux_cmd_arg (cmd, 0)))) {
+        free (s);
         errno = ENOMEM;
-        goto cleanup;
+        return NULL;
     }
-
-cleanup:
-    free (sender);
-    free (cmd_str);
+    free (s);
     return info;
 }
 
-static void server_processes_cb (flux_t *h, flux_msg_handler_t *mh,
-                                 const flux_msg_t *msg, void *arg)
+static void server_list_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
 {
-    flux_subprocess_server_t *s = arg;
+    subprocess_server_t *s = arg;
     flux_subprocess_t *p;
     json_t *procs = NULL;
+    flux_error_t error;
+    const char *errmsg = NULL;
 
-    if (!(procs = json_array ())) {
-        errno = ENOMEM;
+    if (s->auth_cb && (*s->auth_cb) (msg, s->arg, &error) < 0) {
+        errmsg = error.text;
+        errno = EPERM;
         goto error;
     }
-
-    p = zhash_first (s->subprocesses);
+    if (!(procs = json_array ()))
+        goto nomem;
+    p = zlistx_first (s->subprocesses);
     while (p) {
         json_t *o = NULL;
         if (!(o = process_info (p))
             || json_array_append_new (procs, o) < 0) {
             json_decref (o);
-            errno = ENOMEM;
-            goto error;
+            goto nomem;
         }
-        p = zhash_next (s->subprocesses);
+        p = zlistx_next (s->subprocesses);
     }
-
     if (flux_respond_pack (h, msg, "{s:i s:o}", "rank", s->rank,
-                           "procs", procs) < 0)
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+                           "procs", procs) < 0) {
+        llog_error (s,
+                    "error responding to %s.list request: %s",
+                    s->service_name,
+                    strerror (errno));
+    }
     return;
-
+nomem:
+    errno = ENOMEM;
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    if (flux_respond_error (h, msg, errno, errmsg) < 0) {
+        llog_error (s,
+                    "error responding to %s.list request: %s",
+                    s->service_name,
+                    strerror (errno));
+    }
     json_decref (procs);
 }
 
-int server_start (flux_subprocess_server_t *s, const char *prefix)
+static void server_disconnect_cb (flux_t *h,
+                                  flux_msg_handler_t *mh,
+                                  const flux_msg_t *msg,
+                                  void *arg)
 {
-    /* rexec.processes is primarily for testing */
-    struct flux_msg_handler_spec htab[] = {
-        { FLUX_MSGTYPE_REQUEST, "rexec",        server_exec_cb, 0 },
-        { FLUX_MSGTYPE_REQUEST, "rexec.write",  server_write_cb, 0 },
-        { FLUX_MSGTYPE_REQUEST, "rexec.signal", server_signal_cb, 0 },
-        { FLUX_MSGTYPE_REQUEST, "rexec.processes", server_processes_cb, 0 },
-        FLUX_MSGHANDLER_TABLE_END,
-    };
-    char *topic_globs[4] = {NULL, NULL, NULL, NULL};
-    int rv = -1;
-
-    assert (prefix);
+    subprocess_server_t *s = arg;
+    const char *sender;
 
-    if (asprintf (&topic_globs[0], "%s.rexec", prefix) < 0)
-        goto cleanup;
-    if (asprintf (&topic_globs[1], "%s.rexec.write", prefix) < 0)
-        goto cleanup;
-    if (asprintf (&topic_globs[2], "%s.rexec.signal", prefix) < 0)
-        goto cleanup;
-    if (asprintf (&topic_globs[3], "%s.rexec.processes", prefix) < 0)
-        goto cleanup;
-
-    htab[0].topic_glob = (const char *)topic_globs[0];
-    htab[1].topic_glob = (const char *)topic_globs[1];
-    htab[2].topic_glob = (const char *)topic_globs[2];
-    htab[3].topic_glob = (const char *)topic_globs[3];
-
-    if (flux_msg_handler_addvec (s->h, htab, s, &s->handlers) < 0)
-        goto cleanup;
-
-    rv = 0;
-cleanup:
-    free (topic_globs[0]);
-    free (topic_globs[1]);
-    free (topic_globs[2]);
-    free (topic_globs[3]);
-    return rv;
+    if ((sender = flux_msg_route_first (msg))) {
+        flux_subprocess_t *p;
+        p = zlistx_first (s->subprocesses);
+        while (p) {
+            const char *uuid = subprocess_sender (p);
+            if (sender && streq (uuid, sender))
+                server_kill (p, SIGKILL);
+            p = zlistx_next (s->subprocesses);
+        }
+    }
 }
 
-void server_stop (flux_subprocess_server_t *s)
-{
-    flux_msg_handler_delvec (s->handlers);
-}
+static struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,
+      "exec",
+      server_exec_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "write",
+      server_write_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "kill",
+      server_kill_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "list",
+      server_list_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "disconnect",
+      server_disconnect_cb,
+      0
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
 
-static void server_signal_subprocess (flux_subprocess_t *p, int signum)
+static void server_kill (flux_subprocess_t *p, int signum)
 {
     flux_future_t *f;
     if (!(f = flux_subprocess_kill (p, signum))) {
-        struct rexec *rex = flux_subprocess_aux_get (p, auxkey);
-
-        flux_log_error (rex->s->h, "%s: flux_subprocess_kill", __FUNCTION__);
+        subprocess_server_t *s = flux_subprocess_aux_get (p, srvkey);
+        llog_error (s,
+                    "subprocess_kill %d %d: %s",
+                    p->pid,
+                    signum,
+                    strerror (errno));
         return;
     }
     flux_future_destroy (f);
 }
 
-int server_signal_subprocesses (flux_subprocess_server_t *s, int signum)
+static int server_killall (subprocess_server_t *s, int signum)
 {
     flux_subprocess_t *p;
 
-    p = zhash_first (s->subprocesses);
+    p = zlistx_first (s->subprocesses);
     while (p) {
-        server_signal_subprocess (p, signum);
-        p = zhash_next (s->subprocesses);
+        server_kill (p, signum);
+        p = zlistx_next (s->subprocesses);
     }
 
     return 0;
 }
 
-int server_terminate_subprocesses (flux_subprocess_server_t *s)
-{
-    server_signal_subprocesses (s, SIGKILL);
-    return 0;
-}
-
-static void terminate_uuid (flux_subprocess_t *p, const char *id)
+void subprocess_server_destroy (subprocess_server_t *s)
 {
-    char *sender;
-
-    if (!(sender = subprocess_sender (p)))
-        return;
-
-    if (!strcmp (id, sender))
-        server_signal_subprocess (p, SIGKILL);
-
-    free (sender);
+    if (s) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (s->handlers);
+        server_killall (s, SIGKILL);
+        zlistx_destroy (&s->subprocesses);
+        flux_future_destroy (s->shutdown);
+        free (s->service_name);
+        free (s->local_uri);
+        free (s);
+        errno = saved_errno;
+    }
 }
 
-int server_terminate_by_uuid (flux_subprocess_server_t *s,
-                              const char *id)
+subprocess_server_t *subprocess_server_create (flux_t *h,
+                                               const char *service_name,
+                                               const char *local_uri,
+                                               subprocess_log_f log_fn,
+                                               void *log_data)
 {
-    flux_subprocess_t *p;
+    subprocess_server_t *s;
 
-    p = zhash_first (s->subprocesses);
-    while (p) {
-        terminate_uuid (p, id);
-        p = zhash_next (s->subprocesses);
+    if (!h || !local_uri || !service_name) {
+        errno = EINVAL;
+        return NULL;
     }
+    if (!(s = calloc (1, sizeof (*s))))
+        return NULL;
 
-    return 0;
-}
-
-static void terminate_prep_cb (flux_reactor_t *r,
-                               flux_watcher_t *w,
-                               int revents,
-                               void *arg)
-{
-    flux_subprocess_server_t *s = arg;
-    flux_watcher_start (s->terminate_idle_w);
-}
-
-static void terminate_cb (flux_reactor_t *r,
-                          flux_watcher_t *w,
-                          int revents,
-                          void *arg)
-{
-    flux_subprocess_server_t *s = arg;
-    flux_watcher_stop (s->terminate_timer_w);
-    flux_watcher_stop (s->terminate_prep_w);
-    flux_watcher_stop (s->terminate_idle_w);
-    flux_watcher_stop (s->terminate_check_w);
-    flux_reactor_stop (s->r);
-}
+    s->h = h;
 
-void server_terminate_cleanup (flux_subprocess_server_t *s)
-{
-    flux_watcher_destroy (s->terminate_timer_w);
-    flux_watcher_destroy (s->terminate_prep_w);
-    flux_watcher_destroy (s->terminate_idle_w);
-    flux_watcher_destroy (s->terminate_check_w);
-    s->terminate_timer_w = NULL;
-    s->terminate_prep_w = NULL;
-    s->terminate_idle_w = NULL;
-    s->terminate_check_w = NULL;
-}
+    s->llog = log_fn;
+    s->llog_data = log_data;
 
-int server_terminate_setup (flux_subprocess_server_t *s,
-                            double wait_time)
-{
-    s->terminate_timer_w = flux_timer_watcher_create (s->r,
-                                                      wait_time, 0.,
-                                                      terminate_cb,
-                                                      s);
-    if (!s->terminate_timer_w) {
-        flux_log_error (s->h, "flux_timer_watcher_create");
+    if (!(s->subprocesses = zlistx_new ()))
         goto error;
-    }
-
-    if (s->terminate_prep_w)
-        return 0;
-
-    s->terminate_prep_w = flux_prepare_watcher_create (s->r,
-                                                       terminate_prep_cb,
-                                                       s);
-    if (!s->terminate_prep_w) {
-        flux_log_error (s->h, "flux_prepare_watcher_create");
+    zlistx_set_destructor (s->subprocesses, proc_destructor);
+    if (!(s->service_name = strdup (service_name)))
         goto error;
-    }
-
-    s->terminate_idle_w = flux_idle_watcher_create (s->r,
-                                                    NULL,
-                                                    s);
-    if (!s->terminate_idle_w) {
-        flux_log_error (s->h, "flux_idle_watcher_create");
+    if (!(s->local_uri = strdup (local_uri)))
         goto error;
-    }
-
-    s->terminate_check_w = flux_check_watcher_create (s->r,
-                                                      terminate_cb,
-                                                      s);
-    if (!s->terminate_check_w) {
-        flux_log_error (s->h, "flux_check_watcher_create");
+    if (flux_get_rank (h, &s->rank) < 0)
+        goto error;
+    if (flux_msg_handler_addvec_ex (s->h,
+                                    service_name,
+                                    htab,
+                                    s,
+                                    &s->handlers) < 0)
         goto error;
-    }
 
-    return 0;
+    return s;
 
 error:
-    server_terminate_cleanup (s);
-    return -1;
+    subprocess_server_destroy (s);
+    return NULL;
 }
 
-int server_terminate_wait (flux_subprocess_server_t *s)
+void subprocess_server_set_auth_cb (subprocess_server_t *s,
+                                    subprocess_server_auth_f fn,
+                                    void *arg)
 {
-    flux_watcher_start (s->terminate_timer_w);
+    s->auth_cb = fn;
+    s->arg = arg;
+}
 
-    if (flux_reactor_run (s->r, 0) < 0) {
-        flux_log_error (s->h, "flux_reactor_run");
-        return -1;
-    }
+flux_future_t *subprocess_server_shutdown (subprocess_server_t *s, int signum)
+{
+    flux_future_t *f;
 
-    return 0;
+    if (!s || s->shutdown != NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(f = flux_future_create (NULL, NULL)))
+        return NULL;
+    flux_future_set_reactor (f, flux_get_reactor (s->h));
+    flux_future_set_flux (f, s->h);
+    flux_future_incref (f);
+    s->shutdown = f;
+    if (zlistx_size (s->subprocesses) == 0)
+        flux_future_fulfill (f, NULL, NULL);
+    else
+        server_killall (s, signum);
+    return f;
 }
 
 /*
diff --git a/src/common/libsubprocess/server.h b/src/common/libsubprocess/server.h
index 2c8a018571da..e8ea032bd302 100644
--- a/src/common/libsubprocess/server.h
+++ b/src/common/libsubprocess/server.h
@@ -13,22 +13,46 @@
 
 #include "subprocess.h"
 
-int server_start (flux_subprocess_server_t *s, const char *prefix);
-
-void server_stop (flux_subprocess_server_t *s);
-
-int server_signal_subprocesses (flux_subprocess_server_t *s, int signum);
-
-int server_terminate_subprocesses (flux_subprocess_server_t *s);
-
-int server_terminate_by_uuid (flux_subprocess_server_t *s,
-                              const char *id);
-
-int server_terminate_setup (flux_subprocess_server_t *s,
-                            double wait_time);
-
-void server_terminate_cleanup (flux_subprocess_server_t *s);
-
-int server_terminate_wait (flux_subprocess_server_t *s);
+typedef struct subprocess_server subprocess_server_t;
+
+typedef int (*subprocess_server_auth_f) (const flux_msg_t *msg,
+                                         void *arg,
+                                         flux_error_t *error);
+
+/* Create a subprocess server.  The handle 'h' must contain a reactor
+ * created with the FLUX_REACTOR_SIGCHLD flag.  Note that there can be
+ * only one reactor per process with this flag set.  Also, it may be wise
+ * to block SIGPIPE to avoid termination when writing to stdin of a subprocess
+ * that has terminated.
+ */
+subprocess_server_t *subprocess_server_create (flux_t *h,
+                                               const char *service_name,
+                                               const char *local_uri,
+                                               subprocess_log_f log_fn,
+                                               void *log_data);
+
+/* Register a callback to allow/deny each rexec request.
+ * The callback should return 0 to allow.  It should return -1 with a
+ * message in 'error' to deny.
+ */
+void subprocess_server_set_auth_cb (subprocess_server_t *s,
+                                    subprocess_server_auth_f fn,
+                                    void *arg);
+
+/* Destroy a subprocess server.  This sends a SIGKILL to any remaining
+ * subprocesses, then destroys them.
+ */
+void subprocess_server_destroy (subprocess_server_t *s);
+
+/* Send all subprocesses a signal and return a future that is fulfilled
+ * when all subprocesses have exited.  New rexec.exec requests will fail.
+ * This future is fulfilled immediately if there are no subprocesses, but if
+ * there are some, the orig. reactor must be allowed to run in order for the
+ * shutdown to make progress.  Therefore this future should be tracked with
+ * flux_future_then(), not flux_future_get() which would deadlock.
+ */
+flux_future_t *subprocess_server_shutdown (subprocess_server_t *s, int signum);
 
 #endif /* !_SUBPROCESS_SERVER_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/subprocess.c b/src/common/libsubprocess/subprocess.c
index e28023d8a49b..fc8b15a8cb30 100644
--- a/src/common/libsubprocess/subprocess.c
+++ b/src/common/libsubprocess/subprocess.c
@@ -13,24 +13,29 @@
 #endif
 
 #include <sys/types.h>
-#include <wait.h>
+#include <sys/socket.h>
+#include <sys/wait.h>
 #include <unistd.h>
 #include <errno.h>
-
-#include <czmq.h>
+#include <signal.h>
+#include <assert.h>
 
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/fdwalk.h"
 #include "src/common/libutil/aux.h"
+#include "src/common/libutil/fdutils.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
 
 #include "subprocess.h"
 #include "subprocess_private.h"
-#include "command.h"
+#include "command_private.h"
 #include "local.h"
 #include "remote.h"
-#include "server.h"
+#include "client.h"
 #include "util.h"
 
 /*
@@ -40,9 +45,9 @@
 void channel_destroy (void *arg)
 {
     struct subprocess_channel *c = arg;
-    if (c && c->magic == CHANNEL_MAGIC) {
-        if (c->name)
-            free (c->name);
+    if (c) {
+        int saved_errno = errno;
+        free (c->name);
 
         if (c->parent_fd != -1)
             close (c->parent_fd);
@@ -53,72 +58,71 @@ void channel_destroy (void *arg)
         flux_watcher_destroy (c->buffer_read_stopped_w);
         c->buffer_read_w_started = false;
 
-        flux_buffer_destroy (c->write_buffer);
-        flux_buffer_destroy (c->read_buffer);
-        flux_watcher_destroy (c->in_prep_w);
-        flux_watcher_destroy (c->in_idle_w);
-        flux_watcher_destroy (c->in_check_w);
+        fbuf_destroy (c->read_buffer);
         flux_watcher_destroy (c->out_prep_w);
         flux_watcher_destroy (c->out_idle_w);
         flux_watcher_destroy (c->out_check_w);
 
-        c->magic = ~CHANNEL_MAGIC;
         free (c);
+        errno = saved_errno;
     }
 }
 
 struct subprocess_channel *channel_create (flux_subprocess_t *p,
-                                           flux_subprocess_output_f output_f,
+                                           flux_subprocess_output_f output_cb,
                                            const char *name,
                                            int flags)
 {
-    struct subprocess_channel *c = calloc (1, sizeof (*c));
-    int save_errno;
+    struct subprocess_channel *c;
 
-    if (!c)
+    if (!(c = calloc (1, sizeof (*c))))
         return NULL;
-
-    c->magic = CHANNEL_MAGIC;
-
     c->p = p;
-    c->output_f = output_f;
+    c->output_cb = output_cb;
+    c->parent_fd = -1;
+    c->child_fd = -1;
     if (!(c->name = strdup (name)))
         goto error;
     c->flags = flags;
-
-    c->eof_sent_to_caller = false;
-    c->closed = false;
-
-    c->parent_fd = -1;
-    c->child_fd = -1;
-    c->buffer_write_w = NULL;
-    c->buffer_read_w = NULL;
-    c->buffer_read_stopped_w = NULL;
-    c->buffer_read_w_started = false;
-
-    c->write_buffer = NULL;
-    c->read_buffer = NULL;
-    c->write_eof_sent = false;
-    c->read_eof_received = false;
-    c->in_prep_w = NULL;
-    c->in_idle_w = NULL;
-    c->in_check_w = NULL;
-    c->out_prep_w = NULL;
-    c->out_idle_w = NULL;
-    c->out_check_w = NULL;
-
     return c;
-
 error:
-    save_errno = errno;
     channel_destroy (c);
-    errno = save_errno;
     return NULL;
 }
 
+/*  Return the set of valid child file descriptors as an idset
+ */
+struct idset *subprocess_childfds (flux_subprocess_t *p)
+{
+    struct subprocess_channel *c;
+    struct idset *ids;
+    const char *stdchan[] = { "stdin", "stdout", "stderr" };
+
+    /*  fds 0,1,2 always remain open in the child (stdin,out,err)
+     */
+    if (!(ids = idset_decode ("0-2")))
+        return NULL;
+
+    if (p->sync_fds[1] > 0)
+        idset_set (ids, p->sync_fds[1]);
+
+    c = zhash_first (p->channels);
+    while (c) {
+        // invalidate (don't protect) file descriptors that are duped to stdio
+        for (int i = 0; i < ARRAY_SIZE (stdchan); i++) {
+            if (streq (c->name, stdchan[i]))
+                goto next;
+        }
+        idset_set (ids, c->child_fd);
+next:
+        c = zhash_next (p->channels);
+    }
+    return ids;
+}
+
 static void subprocess_free (flux_subprocess_t *p)
 {
-    if (p && p->magic == SUBPROCESS_MAGIC) {
+    if (p) {
         int saved_errno = errno;
         flux_cmd_destroy (p->cmd);
 
@@ -140,28 +144,32 @@ static void subprocess_free (flux_subprocess_t *p)
 
         if (p->f)
             flux_future_destroy (p->f);
+        free (p->service_name);
 
-        p->magic = ~SUBPROCESS_MAGIC;
         free (p);
         errno = saved_errno;
     }
 }
 
-static flux_subprocess_t * subprocess_create (flux_t *h,
-                                              flux_reactor_t *r,
-                                              int flags,
-                                              const flux_cmd_t *cmd,
-                                              const flux_subprocess_ops_t *ops,
-                                              const flux_subprocess_hooks_t *hooks,
-                                              int rank,
-                                              bool local)
+static flux_subprocess_t *subprocess_create (
+    flux_t *h,
+    flux_reactor_t *r,
+    int flags,
+    const flux_cmd_t *cmd,
+    const flux_subprocess_ops_t *ops,
+    const flux_subprocess_hooks_t *hooks,
+    int rank,
+    bool local,
+    subprocess_log_f log_fn,
+    void *log_data)
 {
     flux_subprocess_t *p = calloc (1, sizeof (*p));
 
     if (!p)
         return NULL;
 
-    p->magic = SUBPROCESS_MAGIC;
+    p->llog = log_fn;
+    p->llog_data = log_data;
 
     /* init fds, so on error we don't accidentally close stdin
      * (i.e. fd == 0)
@@ -170,8 +178,15 @@ static flux_subprocess_t * subprocess_create (flux_t *h,
 
     /* set CLOEXEC on sync_fds, so on exec(), child sync_fd is closed
      * and seen by parent */
+#if SOCK_CLOEXEC
     if (socketpair (PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0, p->sync_fds) < 0)
         goto error;
+#else
+    if (socketpair (PF_LOCAL, SOCK_STREAM, 0, p->sync_fds) < 0
+        || fd_set_cloexec (p->sync_fds[0]) < 0
+        || fd_set_cloexec (p->sync_fds[1]) < 0)
+        goto error;
+#endif
 
     if (!(p->channels = zhash_new ()))
         goto error;
@@ -203,55 +218,6 @@ static flux_subprocess_t * subprocess_create (flux_t *h,
     return NULL;
 }
 
-static void subprocess_server_destroy (void *arg)
-{
-    flux_subprocess_server_t *s = arg;
-    if (s && s->magic == SUBPROCESS_SERVER_MAGIC) {
-        /* s->handlers handled in server_stop, this is for destroying
-         * things only
-         */
-        zhash_destroy (&s->subprocesses);
-        free (s->local_uri);
-
-        flux_watcher_destroy (s->terminate_timer_w);
-        flux_watcher_destroy (s->terminate_prep_w);
-        flux_watcher_destroy (s->terminate_idle_w);
-        flux_watcher_destroy (s->terminate_check_w);
-
-        s->magic = ~SUBPROCESS_SERVER_MAGIC;
-        free (s);
-    }
-}
-
-static flux_subprocess_server_t *subprocess_server_create (flux_t *h,
-                                                           const char *local_uri,
-                                                           int rank)
-{
-    flux_subprocess_server_t *s = calloc (1, sizeof (*s));
-    int save_errno;
-
-    if (!s)
-        return NULL;
-
-    s->magic = SUBPROCESS_SERVER_MAGIC;
-    s->h = h;
-    if (!(s->r = flux_get_reactor (h)))
-        goto error;
-    if (!(s->subprocesses = zhash_new ()))
-        goto error;
-    if (!(s->local_uri = strdup (local_uri)))
-        goto error;
-    s->rank = rank;
-
-    return s;
-
-error:
-    save_errno = errno;
-    subprocess_server_destroy (s);
-    errno = save_errno;
-    return NULL;
-}
-
 /*
  * Accessors
  */
@@ -262,116 +228,31 @@ int subprocess_status (flux_subprocess_t *p)
     return p->status;
 }
 
-/*
- *  General support:
- */
-
-flux_subprocess_server_t *flux_subprocess_server_start (flux_t *h,
-                                                        const char *prefix,
-                                                        const char *local_uri,
-                                                        uint32_t rank)
-{
-    flux_subprocess_server_t *s = NULL;
-    int save_errno;
-
-    if (!h || !prefix || !local_uri) {
-        errno = EINVAL;
-        goto error;
-    }
-
-    if (!(s = subprocess_server_create (h, local_uri, rank)))
-        goto error;
-
-    if (server_start (s, prefix) < 0)
-        goto error;
-
-    return s;
-
-error:
-    save_errno = errno;
-    subprocess_server_destroy (s);
-    errno = save_errno;
-    return NULL;
-}
-
-void flux_subprocess_server_stop (flux_subprocess_server_t *s)
-{
-    if (s && s->magic == SUBPROCESS_SERVER_MAGIC) {
-        server_stop (s);
-        server_terminate_subprocesses (s);
-        subprocess_server_destroy (s);
-    }
-}
-
-int flux_subprocess_server_subprocesses_kill (flux_subprocess_server_t *s,
-                                              int signum,
-                                              double wait_time)
-{
-    int rv = -1;
-
-    if (!s || s->magic != SUBPROCESS_SERVER_MAGIC) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if (!zhash_size (s->subprocesses))
-        return 0;
-
-    if (server_terminate_setup (s, wait_time) < 0)
-        goto error;
-
-    if (server_signal_subprocesses (s, signum) < 0)
-        goto error;
-
-    if (server_terminate_wait (s) < 0)
-        goto error;
-
-    rv = 0;
-error:
-    server_terminate_cleanup (s);
-    return rv;
-}
-
-int flux_subprocess_server_terminate_by_uuid (flux_subprocess_server_t *s,
-                                              const char *id)
-{
-    if (!s || s->magic != SUBPROCESS_SERVER_MAGIC || !id) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    return server_terminate_by_uuid (s, id);
-}
-
-/*
- * Convenience Functions:
- */
-
-void flux_standard_output (flux_subprocess_t *p, const char *stream)
+void subprocess_standard_output (flux_subprocess_t *p, const char *stream)
 {
     /* everything except stderr goes to stdout */
     FILE *fstream = !strcasecmp (stream, "stderr") ? stderr : stdout;
-    const char *ptr;
-    int lenp;
+    const char *buf;
+    int len;
 
     /* Do not use flux_subprocess_getline(), this should work
      * regardless if stream is line buffered or not */
 
-    if (!(ptr = flux_subprocess_read_line (p, stream, &lenp))) {
-        log_err ("flux_standard_output: read_line");
+    if ((len = flux_subprocess_read_line (p, stream, &buf)) < 0) {
+        log_err ("subprocess_standard_output: read_line");
         return;
     }
 
     /* we're at the end of the stream, read any lingering data */
-    if (!lenp && flux_subprocess_read_stream_closed (p, stream) > 0) {
-        if (!(ptr = flux_subprocess_read (p, stream, -1, &lenp))) {
-            log_err ("flux_standard_output: read_line");
+    if (len == 0 && flux_subprocess_read_stream_closed (p, stream)) {
+        if ((len = flux_subprocess_read (p, stream, &buf)) < 0) {
+            log_err ("subprocess_standard_output: read");
             return;
         }
     }
 
-    if (lenp)
-        fwrite (ptr, lenp, 1, fstream);
+    if (len)
+        fwrite (buf, len, 1, fstream);
 }
 
 /*
@@ -380,7 +261,11 @@ void flux_standard_output (flux_subprocess_t *p, const char *stream)
 
 void subprocess_check_completed (flux_subprocess_t *p)
 {
-    assert (p->state == FLUX_SUBPROCESS_EXITED);
+    if (p->state != FLUX_SUBPROCESS_EXITED) {
+        log_err ("subprocess_check_completed: unexpected state %s",
+                 flux_subprocess_state_string (p->state));
+        return;
+    }
 
     /* we're also waiting for the "complete" to come from the remote end */
     if (!p->local && !p->remote_completed)
@@ -417,27 +302,21 @@ static void state_change_prep_cb (flux_reactor_t *r,
 
 static flux_subprocess_state_t state_change_next (flux_subprocess_t *p)
 {
-    assert (p->state != FLUX_SUBPROCESS_FAILED);
-
-    switch (p->state_reported) {
-    case FLUX_SUBPROCESS_INIT:
-        /* next state must be RUNNING or EXEC_FAILED */
-        if (p->state == FLUX_SUBPROCESS_EXEC_FAILED)
-            return FLUX_SUBPROCESS_EXEC_FAILED;
-        else /* p->state == FLUX_SUBPROCESS_RUNNING
-                || p->state == FLUX_SUBPROCESS_EXITED */
-            return FLUX_SUBPROCESS_RUNNING;
-    case FLUX_SUBPROCESS_RUNNING:
+    /* N.B. possible transition to FLUX_SUBPROCESS_STOPPED not handled
+     * here, see issue #5083
+     */
+    assert (p->state_reported != p->state);
+    assert (p->state_reported == FLUX_SUBPROCESS_INIT
+            || p->state_reported == FLUX_SUBPROCESS_RUNNING);
+
+    if (p->state_reported == FLUX_SUBPROCESS_INIT)
+        /* next state must be RUNNING */
+        return FLUX_SUBPROCESS_RUNNING;
+    else if (p->state_reported == FLUX_SUBPROCESS_RUNNING)
         /* next state is EXITED */
         return FLUX_SUBPROCESS_EXITED;
-    case FLUX_SUBPROCESS_EXEC_FAILED:
-    case FLUX_SUBPROCESS_EXITED:
-    case FLUX_SUBPROCESS_FAILED:
-        break;
-    }
-
     /* shouldn't be possible to reach here */
-    assert (0);
+    return p->state_reported;
 }
 
 static void state_change_check_cb (flux_reactor_t *r,
@@ -451,7 +330,7 @@ static void state_change_check_cb (flux_reactor_t *r,
     flux_watcher_stop (p->state_idle_w);
 
     /* always a chance caller may destroy subprocess in callback */
-    flux_subprocess_ref (p);
+    subprocess_incref (p);
 
     if (p->state_reported != p->state) {
         /* this is the ubiquitous fail state for internal failures,
@@ -468,8 +347,7 @@ static void state_change_check_cb (flux_reactor_t *r,
     }
 
     /* once we hit one of these states, no more state changes */
-    if (p->state_reported == FLUX_SUBPROCESS_EXEC_FAILED
-        || p->state_reported == FLUX_SUBPROCESS_EXITED
+    if (p->state_reported == FLUX_SUBPROCESS_EXITED
         || p->state_reported == FLUX_SUBPROCESS_FAILED) {
         flux_watcher_stop (p->state_prep_w);
         flux_watcher_stop (p->state_check_w);
@@ -482,7 +360,7 @@ static void state_change_check_cb (flux_reactor_t *r,
     if (p->state_reported == FLUX_SUBPROCESS_EXITED)
         subprocess_check_completed (p);
 
-    flux_subprocess_unref (p);
+    subprocess_decref (p);
 }
 
 static int subprocess_setup_state_change (flux_subprocess_t *p)
@@ -516,9 +394,9 @@ static int subprocess_setup_state_change (flux_subprocess_t *p)
 }
 
 static void completed_prep_cb (flux_reactor_t *r,
-                                  flux_watcher_t *w,
-                                  int revents,
-                                  void *arg)
+                               flux_watcher_t *w,
+                               int revents,
+                               void *arg)
 {
     flux_subprocess_t *p = arg;
 
@@ -539,7 +417,7 @@ static void completed_check_cb (flux_reactor_t *r,
     flux_watcher_stop (p->completed_idle_w);
 
     /* always a chance caller may destroy subprocess in callback */
-    flux_subprocess_ref (p);
+    subprocess_incref (p);
 
     /* There is a small "racy" component, where the state we're at may
      * not yet align with the state that has been reported to the
@@ -558,7 +436,7 @@ static void completed_check_cb (flux_reactor_t *r,
         flux_watcher_stop (p->completed_check_w);
     }
 
-    flux_subprocess_unref (p);
+    subprocess_decref (p);
 }
 
 static int subprocess_setup_completed (flux_subprocess_t *p)
@@ -593,14 +471,18 @@ static int subprocess_setup_completed (flux_subprocess_t *p)
     return 0;
 }
 
-static flux_subprocess_t * flux_exec_wrap (flux_t *h, flux_reactor_t *r, int flags,
-                                           const flux_cmd_t *cmd,
-                                           const flux_subprocess_ops_t *ops,
-                                           const flux_subprocess_hooks_t *hooks)
+flux_subprocess_t *flux_local_exec_ex (flux_reactor_t *r,
+                                       int flags,
+                                       const flux_cmd_t *cmd,
+                                       const flux_subprocess_ops_t *ops,
+                                       const flux_subprocess_hooks_t *hooks,
+                                       subprocess_log_f log_fn,
+                                       void *log_data)
 {
     flux_subprocess_t *p = NULL;
     int valid_flags = (FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH
-                       | FLUX_SUBPROCESS_FLAGS_SETPGRP);
+                       | FLUX_SUBPROCESS_FLAGS_NO_SETPGRP
+                       | FLUX_SUBPROCESS_FLAGS_FORK_EXEC);
 
     if (!r || !cmd) {
         errno = EINVAL;
@@ -612,7 +494,22 @@ static flux_subprocess_t * flux_exec_wrap (flux_t *h, flux_reactor_t *r, int fla
         return NULL;
     }
 
-    if (!(p = subprocess_create (h, r, flags, cmd, ops, hooks, -1, true)))
+    /* user required to set some args */
+    if (!flux_cmd_argc (cmd)) {
+        errno = EINVAL;
+        goto error;
+    }
+
+    if (!(p = subprocess_create (NULL,
+                                 r,
+                                 flags,
+                                 cmd,
+                                 ops,
+                                 hooks,
+                                 -1,
+                                 true,
+                                 log_fn,
+                                 log_data)))
         goto error;
 
     if (subprocess_local_setup (p) < 0)
@@ -629,62 +526,42 @@ static flux_subprocess_t * flux_exec_wrap (flux_t *h, flux_reactor_t *r, int fla
     return p;
 
 error:
-    flux_subprocess_unref (p);
+    subprocess_decref (p);
     return NULL;
 }
 
-flux_subprocess_t * flux_exec (flux_t *h, int flags,
-                               const flux_cmd_t *cmd,
-                               const flux_subprocess_ops_t *ops,
-                               const flux_subprocess_hooks_t *hooks)
-{
-    flux_reactor_t *r;
-
-    if (!h) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    if (!(r = flux_get_reactor (h)))
-        return NULL;
-
-    return flux_exec_wrap (h, r, flags, cmd, ops, hooks);
-}
-
-flux_subprocess_t * flux_local_exec (flux_reactor_t *r, int flags,
+flux_subprocess_t * flux_local_exec (flux_reactor_t *r,
+                                     int flags,
                                      const flux_cmd_t *cmd,
-                                     const flux_subprocess_ops_t *ops,
-                                     const flux_subprocess_hooks_t *hooks)
+                                     const flux_subprocess_ops_t *ops)
 {
-    return flux_exec_wrap (NULL, r, flags, cmd, ops, hooks);
+    return flux_local_exec_ex (r, flags, cmd, ops, NULL, NULL, NULL);
 }
 
-static int check_local_only_cmd_options (const flux_cmd_t *cmd)
-{
-    /* check for options that do not apply to remote subprocesses */
-    const char *substrings[] = { "STREAM_STOP", NULL };
-
-    return flux_cmd_find_opts (cmd, substrings);
-}
-
-flux_subprocess_t *flux_rexec (flux_t *h, int rank, int flags,
-                               const flux_cmd_t *cmd,
-                               const flux_subprocess_ops_t *ops)
+flux_subprocess_t *flux_rexec_ex (flux_t *h,
+                                  const char *service_name,
+                                  int rank,
+                                  int flags,
+                                  const flux_cmd_t *cmd,
+                                  const flux_subprocess_ops_t *ops,
+                                  subprocess_log_f log_fn,
+                                  void *log_data)
 {
     flux_subprocess_t *p = NULL;
     flux_reactor_t *r;
+    int valid_flags = FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF;
 
     if (!h
         || (rank < 0
             && rank != FLUX_NODEID_ANY
             && rank != FLUX_NODEID_UPSTREAM)
-        || !cmd) {
+        || !cmd
+        || !service_name) {
         errno = EINVAL;
         return NULL;
     }
 
-    /* no flags supported yet */
-    if (flags) {
+    if (flags & ~valid_flags) {
         errno = EINVAL;
         return NULL;
     }
@@ -695,25 +572,22 @@ flux_subprocess_t *flux_rexec (flux_t *h, int rank, int flags,
         goto error;
     }
 
-    /* user required to set cwd */
-    if (!flux_cmd_getcwd (cmd)) {
-        errno = EINVAL;
-        goto error;
-    }
-
-    /* make sure user didn't set local only cmd options */
-    if (check_local_only_cmd_options (cmd)) {
-        errno = EINVAL;
-        goto error;
-    }
-
     if (!(r = flux_get_reactor (h)))
         goto error;
 
-    if (!(p = subprocess_create (h, r, flags, cmd, ops, NULL, rank, false)))
+    if (!(p = subprocess_create (h,
+                                 r,
+                                 flags,
+                                 cmd,
+                                 ops,
+                                 NULL,
+                                 rank,
+                                 false,
+                                 log_fn,
+                                 log_data)))
         goto error;
 
-    if (subprocess_remote_setup (p) < 0)
+    if (subprocess_remote_setup (p, service_name) < 0)
         goto error;
 
     if (subprocess_setup_state_change (p) < 0)
@@ -728,49 +602,41 @@ flux_subprocess_t *flux_rexec (flux_t *h, int rank, int flags,
     return p;
 
 error:
-    flux_subprocess_unref (p);
+    subprocess_decref (p);
     return NULL;
 }
 
-int flux_subprocess_stream_start (flux_subprocess_t *p, const char *stream)
+flux_subprocess_t *flux_rexec (flux_t *h,
+                               int rank,
+                               int flags,
+                               const flux_cmd_t *cmd,
+                               const flux_subprocess_ops_t *ops)
 {
-    struct subprocess_channel *c;
-    flux_buffer_t *fb;
-
-    if (!p || !stream
-           || p->magic != SUBPROCESS_MAGIC
-           || (p->local && p->in_hook)) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    c = zhash_lookup (p->channels, stream);
-    if (!c || !(c->flags & CHANNEL_READ)) {
-        errno = EINVAL;
-        return -1;
-    }
+    return flux_rexec_ex (h, "rexec", rank, flags, cmd, ops, NULL, NULL);
+}
 
-    if (p->local) {
-        if (c->buffer_read_w_started)
-            return 0;
+void flux_subprocess_stream_start (flux_subprocess_t *p, const char *stream)
+{
+    struct subprocess_channel *c;
 
-        if (!(fb = flux_buffer_read_watcher_get_buffer (c->buffer_read_w)))
-            return -1;
+    if (!p
+        || !stream
+        || !p->local
+        || p->in_hook
+        || !(c = zhash_lookup (p->channels, stream))
+        || !(c->flags & CHANNEL_READ)
+        || c->buffer_read_w_started
+        || !fbuf_read_watcher_get_buffer (c->buffer_read_w))
+        return;
 
-        if (!c->buffer_read_stopped_w) {
-            /* use check watcher instead of idle watcher, as idle watcher
-             * could spin reactor */
-            c->buffer_read_stopped_w = flux_check_watcher_create (p->reactor,
-                                                                  NULL,
-                                                                  c);
-            if (!c->buffer_read_stopped_w)
-                return -1;
-        }
-    }
-    else {
-        /* not supported on remote right now */
-        errno = EINVAL;
-        return -1;
+    if (!c->buffer_read_stopped_w) {
+        /* use check watcher instead of idle watcher, as idle watcher
+         * could spin reactor */
+        c->buffer_read_stopped_w = flux_check_watcher_create (p->reactor,
+                                                              NULL,
+                                                              c);
+        if (!c->buffer_read_stopped_w)
+            return;
     }
 
     /* Note that in local.c, we never stop buffer_read_w
@@ -782,83 +648,37 @@ int flux_subprocess_stream_start (flux_subprocess_t *p, const char *stream)
     flux_watcher_start (c->buffer_read_w);
     c->buffer_read_w_started = true;
     flux_watcher_stop (c->buffer_read_stopped_w);
-    return 0;
 }
 
-int flux_subprocess_stream_stop (flux_subprocess_t *p, const char *stream)
+void flux_subprocess_stream_stop (flux_subprocess_t *p, const char *stream)
 {
     struct subprocess_channel *c;
-    flux_buffer_t *fb;
 
-    if (!p || !stream
-           || p->magic != SUBPROCESS_MAGIC
-           || (p->local && p->in_hook)) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    c = zhash_lookup (p->channels, stream);
-    if (!c || !(c->flags & CHANNEL_READ)) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if (p->local) {
-        if (!c->buffer_read_w_started)
-            return 0;
-
-        if (!(fb = flux_buffer_read_watcher_get_buffer (c->buffer_read_w)))
-            return -1;
-    }
-    else {
-        /* not supported on remote right now */
-        errno = EINVAL;
-        return -1;
-    }
+    if (!p
+        || !stream
+        || !p->local
+        || p->in_hook
+        || !(c = zhash_lookup (p->channels, stream))
+        || !(c->flags & CHANNEL_READ)
+        || !c->buffer_read_w_started
+        || !fbuf_read_watcher_get_buffer (c->buffer_read_w))
+        return;
 
     flux_watcher_stop (c->buffer_read_w);
     c->buffer_read_w_started = false;
     flux_watcher_start (c->buffer_read_stopped_w);
-    return 0;
 }
 
-int flux_subprocess_stream_status (flux_subprocess_t *p, const char *stream)
+int flux_subprocess_write (flux_subprocess_t *p,
+                           const char *stream,
+                           const char *buf,
+                           size_t len)
 {
     struct subprocess_channel *c;
+    struct fbuf *fb;
     int ret;
 
     if (!p || !stream
-           || p->magic != SUBPROCESS_MAGIC
-           || (p->local && p->in_hook)) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    c = zhash_lookup (p->channels, stream);
-    if (!c || !(c->flags & CHANNEL_READ)) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if (p->local)
-        ret = c->buffer_read_w_started ? 1 : 0;
-    else {
-        /* fb = c->read_buffer; */
-        assert (0);
-    }
-
-    return ret;
-}
-
-int flux_subprocess_write (flux_subprocess_t *p, const char *stream,
-                           const char *buf, size_t len)
-{
-    struct subprocess_channel *c;
-    flux_buffer_t *fb;
-    int ret;
-
-    if (!p || !stream
-           || p->magic != SUBPROCESS_MAGIC
            || (p->local && p->in_hook)) {
         errno = EINVAL;
         return -1;
@@ -885,15 +705,19 @@ int flux_subprocess_write (flux_subprocess_t *p, const char *stream,
             errno = EPIPE;
             return -1;
         }
-        if (!(fb = flux_buffer_write_watcher_get_buffer (c->buffer_write_w))) {
-            log_err ("flux_buffer_write_watcher_get_buffer");
+        if (!(fb = fbuf_write_watcher_get_buffer (c->buffer_write_w))) {
+            log_err ("fbuf_write_watcher_get_buffer");
             return -1;
         }
-
-        if ((ret = flux_buffer_write (fb, buf, len)) < 0) {
-            log_err ("flux_buffer_write");
+        if (fbuf_space (fb) < len) {
+            errno = ENOSPC;
+            return -1;
+        }
+        if ((ret = fbuf_write (fb, buf, len)) < 0) {
+            log_err ("fbuf_write");
             return -1;
         }
+        c->buffer_space -= ret;
     }
     else {
         if (p->state != FLUX_SUBPROCESS_INIT
@@ -901,10 +725,12 @@ int flux_subprocess_write (flux_subprocess_t *p, const char *stream,
             errno = EPIPE;
             return -1;
         }
-        if ((ret = flux_buffer_write (c->write_buffer, buf, len)) < 0) {
-            log_err ("flux_buffer_write");
+        if (subprocess_write (p->f, c->name, buf, len, false) < 0) {
+            log_err ("error sending rexec.write request: %s",
+                     strerror (errno));
             return -1;
         }
+        ret = len;
     }
 
     return ret;
@@ -915,7 +741,6 @@ int flux_subprocess_close (flux_subprocess_t *p, const char *stream)
     struct subprocess_channel *c;
 
     if (!p || !stream
-           || p->magic != SUBPROCESS_MAGIC
            || (p->local && p->in_hook)) {
         errno = EINVAL;
         return -1;
@@ -932,160 +757,176 @@ int flux_subprocess_close (flux_subprocess_t *p, const char *stream)
 
     if (p->local) {
         if (p->state == FLUX_SUBPROCESS_RUNNING) {
-            if (flux_buffer_write_watcher_close (c->buffer_write_w) < 0) {
-                log_err ("flux_buffer_write_watcher_close");
+            if (fbuf_write_watcher_close (c->buffer_write_w) < 0) {
+                log_err ("fbuf_write_watcher_close");
                 return -1;
             }
         }
-        /* else p->state == FLUX_SUBPROCESS_EXEC_FAILED
-           || p->state == FLUX_SUBPROCESS_EXITED
+        /* else p->state == FLUX_SUBPROCESS_EXITED
            || p->state == FLUX_SUBPROCESS_FAILED
         */
         c->closed = true;
     }
     else {
-        /* doesn't matter about state, b/c reactors will send closed.
-         * If those reactors are already turned off, it's b/c
-         * subprocess failed/exited.
-         */
+        if (subprocess_write (p->f, c->name, NULL, 0, true) < 0) {
+            log_err ("error sending rexec.write request: %s",
+                     strerror (errno));
+            return -1;
+        }
         c->closed = true;
     }
 
     return 0;
 }
 
-static const char *subprocess_read (flux_subprocess_t *p,
-                                    const char *stream,
-                                    int len, int *lenp,
-                                    bool read_line,
-                                    bool trimmed,
-                                    bool line_buffered_required,
-                                    bool *readonly)
+static int subprocess_read (flux_subprocess_t *p,
+                            const char *stream,
+                            const char **bufp,
+                            bool read_line,
+                            bool trimmed,
+                            bool line_buffered_required,
+                            bool *readonly)
 {
     struct subprocess_channel *c;
-    flux_buffer_t *fb;
+    struct fbuf *fb;
     const char *ptr;
+    int len;
 
     if (!p || !stream
-           || p->magic != SUBPROCESS_MAGIC
            || (p->local && p->in_hook)) {
         errno = EINVAL;
-        return NULL;
-    }
-
-    if (!read_line && len == 0) {
-        errno = EINVAL;
-        return NULL;
+        return -1;
     }
 
     c = zhash_lookup (p->channels, stream);
     if (!c || !(c->flags & CHANNEL_READ)) {
         errno = EINVAL;
-        return NULL;
+        return -1;
     }
 
     if (line_buffered_required && !c->line_buffered) {
         errno = EPERM;
-        return NULL;
+        return -1;
+    }
+
+    if (p->flags & FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF) {
+        if (bufp && c->unbuf_len)
+            (*bufp) = c->unbuf_data;
+        return c->unbuf_len;
     }
 
     if (p->local) {
-        if (!(fb = flux_buffer_read_watcher_get_buffer (c->buffer_read_w)))
-            return NULL;
+        if (!(fb = fbuf_read_watcher_get_buffer (c->buffer_read_w)))
+            return -1;
     }
     else
         fb = c->read_buffer;
 
     /* if readonly marked, indicates EOF received */
     if (readonly)
-        (*readonly) = flux_buffer_is_readonly (fb);
+        (*readonly) = fbuf_is_readonly (fb);
 
     if (read_line) {
         if (trimmed) {
-            if (!(ptr = flux_buffer_read_trimmed_line (fb, lenp)))
-                return NULL;
+            if (!(ptr = fbuf_read_trimmed_line (fb, &len)))
+                return -1;
         }
         else {
-            if (!(ptr = flux_buffer_read_line (fb, lenp)))
-                return NULL;
+            if (!(ptr = fbuf_read_line (fb, &len)))
+                return -1;
+        }
+        /* Special case if buffer full, we gotta flush it out even if
+         * there is no line
+         */
+        if (!len && !fbuf_space (fb)) {
+            if (!(ptr = fbuf_read (fb, -1, &len)))
+                return -1;
         }
     }
     else {
-        if (!(ptr = flux_buffer_read (fb, len, lenp)))
-            return NULL;
+        if (!(ptr = fbuf_read (fb, -1, &len)))
+            return -1;
     }
 
-    return ptr;
+    if (bufp && len > 0)
+        (*bufp) = ptr;
+    return len;
 }
 
-const char *flux_subprocess_read (flux_subprocess_t *p,
-                                  const char *stream,
-                                  int len, int *lenp)
+int flux_subprocess_read (flux_subprocess_t *p,
+                          const char *stream,
+                          const char **bufp)
 {
-    return subprocess_read (p, stream, len, lenp, false, false, false, NULL);
+    return subprocess_read (p, stream, bufp, false, false, false, NULL);
 }
 
-const char *flux_subprocess_read_line (flux_subprocess_t *p,
-                                       const char *stream,
-                                       int *lenp)
+
+int flux_subprocess_read_line (flux_subprocess_t *p,
+                               const char *stream,
+                               const char **bufp)
 {
-    return subprocess_read (p, stream, 0, lenp, true, false, false, NULL);
+    if (p && (p->flags & FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF)) {
+        errno = EPERM;
+        return -1;
+    }
+    return subprocess_read (p, stream, bufp, true, false, false, NULL);
 }
 
-const char *flux_subprocess_read_trimmed_line (flux_subprocess_t *p,
-                                               const char *stream,
-                                               int *lenp)
+int flux_subprocess_read_trimmed_line (flux_subprocess_t *p,
+                                       const char *stream,
+                                       const char **bufp)
 {
-    return subprocess_read (p, stream, 0, lenp, true, true, false, NULL);
+    if (p && (p->flags & FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF)) {
+        errno = EPERM;
+        return -1;
+    }
+    return subprocess_read (p, stream, bufp, true, true, false, NULL);
 }
 
-int flux_subprocess_read_stream_closed (flux_subprocess_t *p, const char *stream)
+bool flux_subprocess_read_stream_closed (flux_subprocess_t *p,
+                                         const char *stream)
 {
     struct subprocess_channel *c;
-    flux_buffer_t *fb;
+    struct fbuf *fb;
 
-    if (!p || !stream
-           || p->magic != SUBPROCESS_MAGIC
-           || (p->local && p->in_hook)) {
-        errno = EINVAL;
-        return -1;
-    }
+    if (!p
+        || !stream
+        || (p->local && p->in_hook)
+        || !(c = zhash_lookup (p->channels, stream))
+        || !(c->flags & CHANNEL_READ))
+        return false;
 
-    c = zhash_lookup (p->channels, stream);
-    if (!c || !(c->flags & CHANNEL_READ)) {
-        errno = EINVAL;
-        return -1;
-    }
+    if (p->flags & FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF)
+        return c->read_eof_received;
 
-    if (p->local) {
-        if (!(fb = flux_buffer_read_watcher_get_buffer (c->buffer_read_w)))
-            return -1;
-    }
+    if (p->local)
+        fb = fbuf_read_watcher_get_buffer (c->buffer_read_w);
     else
         fb = c->read_buffer;
 
-    return flux_buffer_is_readonly (fb);
+    return fb ? fbuf_is_readonly (fb) : false;
 }
 
-const char *flux_subprocess_getline (flux_subprocess_t *p,
-                                     const char *stream,
-                                     int *lenp)
+int flux_subprocess_getline (flux_subprocess_t *p,
+                             const char *stream,
+                             const char **bufp)
 {
-    const char *ptr;
     int len;
     bool readonly;
 
-    ptr = subprocess_read (p, stream, 0, &len, true, false, true, &readonly);
+    if (p && (p->flags & FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF)) {
+        errno = EPERM;
+        return -1;
+    }
+
+    len = subprocess_read (p, stream, bufp, true, false, true, &readonly);
 
     /* if no lines available and EOF received, read whatever is
      * lingering in the buffer */
-    if (ptr && len == 0 && readonly)
-        ptr = flux_subprocess_read (p, stream, -1, &len);
+    if (len == 0 && readonly)
+        len = flux_subprocess_read (p, stream, bufp);
 
-    if (lenp)
-        (*lenp) = len;
-
-    return ptr;
+    return len;
 }
 
 static flux_future_t *add_pending_signal (flux_subprocess_t *p, int signum)
@@ -1100,6 +941,11 @@ static flux_future_t *add_pending_signal (flux_subprocess_t *p, int signum)
     if ((f = flux_future_create (NULL, NULL))) {
         flux_subprocess_aux_set (p, "sp::signal_future", f, NULL);
         p->signal_pending = signum;
+        /*  Take a reference on the returned future in case the caller
+         *  destroys it between now and when the signal is actually sent.
+         *  This reference will be dropped in fwd_pending_signal()
+         */
+        flux_future_incref (f);
     }
     return f;
 }
@@ -1108,7 +954,7 @@ flux_future_t *flux_subprocess_kill (flux_subprocess_t *p, int signum)
 {
     flux_future_t *f = NULL;
 
-    if (!p || p->magic != SUBPROCESS_MAGIC || (p->local && p->in_hook)
+    if (!p || (p->local && p->in_hook)
         || !signum) {
         errno = EINVAL;
         return NULL;
@@ -1116,18 +962,17 @@ flux_future_t *flux_subprocess_kill (flux_subprocess_t *p, int signum)
 
     if (p->state != FLUX_SUBPROCESS_RUNNING
         && p->state != FLUX_SUBPROCESS_INIT) {
-        /* XXX right errno? */
-        errno = EINVAL;
+        errno = ESRCH;
         return NULL;
     }
 
     if (p->local) {
         int ret;
         if (p->pid <= (pid_t) 0) {
-            errno = EINVAL;
+            errno = ESRCH;
             return NULL;
         }
-        if (p->flags & FLUX_SUBPROCESS_FLAGS_SETPGRP)
+        if (!(p->flags & FLUX_SUBPROCESS_FLAGS_NO_SETPGRP))
             ret = killpg (p->pid, signum);
         else
             ret = kill (p->pid, signum);
@@ -1152,50 +997,67 @@ flux_future_t *flux_subprocess_kill (flux_subprocess_t *p, int signum)
     return f;
 }
 
-void flux_subprocess_ref (flux_subprocess_t *p)
+void subprocess_incref (flux_subprocess_t *p)
 {
-    if (p && p->magic == SUBPROCESS_MAGIC)
+    if (p) {
+        if (p->local && p->in_hook)
+            return;
         p->refcount++;
+    }
 }
 
-void flux_subprocess_unref (flux_subprocess_t *p)
+void subprocess_decref (flux_subprocess_t *p)
 {
-    if (p && p->magic == SUBPROCESS_MAGIC) {
+    if (p) {
+        if (p->local && p->in_hook)
+            return;
         if (--p->refcount == 0)
             subprocess_free (p);
     }
 }
 
+void flux_subprocess_destroy (flux_subprocess_t *p)
+{
+    subprocess_decref (p);
+}
+
 flux_subprocess_state_t flux_subprocess_state (flux_subprocess_t *p)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC) {
+    if (!p) {
         errno = EINVAL;
         return -1;
     }
     return p->state;
 }
 
+bool flux_subprocess_active (flux_subprocess_t *p)
+{
+    /*  A subprocess is still active if it has not failed or completed.
+     */
+    return (p && p->state != FLUX_SUBPROCESS_FAILED && !p->completed);
+}
+
 const char *flux_subprocess_state_string (flux_subprocess_state_t state)
 {
     switch (state)
     {
     case FLUX_SUBPROCESS_INIT:
         return "Init";
-    case FLUX_SUBPROCESS_EXEC_FAILED:
-        return "Exec Failed";
     case FLUX_SUBPROCESS_RUNNING:
         return "Running";
     case FLUX_SUBPROCESS_EXITED:
         return "Exited";
     case FLUX_SUBPROCESS_FAILED:
         return "Failed";
+    case FLUX_SUBPROCESS_STOPPED:
+        return "Stopped";
     }
     return NULL;
 }
 
 int flux_subprocess_rank (flux_subprocess_t *p)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC) {
+    if (!p) {
         errno = EINVAL;
         return -1;
     }
@@ -1208,24 +1070,31 @@ int flux_subprocess_rank (flux_subprocess_t *p)
 
 int flux_subprocess_fail_errno (flux_subprocess_t *p)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC) {
+    if (!p) {
         errno = EINVAL;
         return -1;
     }
-    if (p->state != FLUX_SUBPROCESS_EXEC_FAILED
-        && p->state != FLUX_SUBPROCESS_FAILED) {
+    if (p->state != FLUX_SUBPROCESS_FAILED) {
         errno = EINVAL;
         return -1;
     }
-    if (p->state == FLUX_SUBPROCESS_EXEC_FAILED)
-        return p->exec_failed_errno;
-    else
-        return p->failed_errno;
+    return p->failed_errno;
+}
+
+const char *flux_subprocess_fail_error (flux_subprocess_t *p)
+{
+    if (!p)
+        return "internal error: subprocess is NULL";
+    if (p->state != FLUX_SUBPROCESS_FAILED)
+        return "internal error: subprocess is not in FAILED state";
+    if (p->failed_error.text[0] == '\0')
+        return strerror (p->failed_errno);
+    return p->failed_error.text;
 }
 
 int flux_subprocess_status (flux_subprocess_t *p)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC) {
+    if (!p) {
         errno = EINVAL;
         return -1;
     }
@@ -1238,7 +1107,7 @@ int flux_subprocess_status (flux_subprocess_t *p)
 
 int flux_subprocess_exit_code (flux_subprocess_t *p)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC) {
+    if (!p) {
         errno = EINVAL;
         return -1;
     }
@@ -1255,7 +1124,7 @@ int flux_subprocess_exit_code (flux_subprocess_t *p)
 
 int flux_subprocess_signaled (flux_subprocess_t *p)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC) {
+    if (!p) {
         errno = EINVAL;
         return -1;
     }
@@ -1272,7 +1141,7 @@ int flux_subprocess_signaled (flux_subprocess_t *p)
 
 pid_t flux_subprocess_pid (flux_subprocess_t *p)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC) {
+    if (!p) {
         errno = EINVAL;
         return -1;
     }
@@ -1289,7 +1158,7 @@ pid_t flux_subprocess_pid (flux_subprocess_t *p)
 
 flux_cmd_t * flux_subprocess_get_cmd (flux_subprocess_t *p)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC) {
+    if (!p) {
         errno = EINVAL;
         return NULL;
     }
@@ -1298,7 +1167,7 @@ flux_cmd_t * flux_subprocess_get_cmd (flux_subprocess_t *p)
 
 flux_reactor_t * flux_subprocess_get_reactor (flux_subprocess_t *p)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC) {
+    if (!p) {
         errno = EINVAL;
         return NULL;
     }
@@ -1306,9 +1175,11 @@ flux_reactor_t * flux_subprocess_get_reactor (flux_subprocess_t *p)
 }
 
 int flux_subprocess_aux_set (flux_subprocess_t *p,
-                             const char *name, void *x, flux_free_f free_fn)
+                             const char *name,
+                             void *x,
+                             flux_free_f free_fn)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC || (p->local && p->in_hook)) {
+    if (!p || (p->local && p->in_hook)) {
         errno = EINVAL;
         return -1;
     }
@@ -1317,13 +1188,44 @@ int flux_subprocess_aux_set (flux_subprocess_t *p,
 
 void * flux_subprocess_aux_get (flux_subprocess_t *p, const char *name)
 {
-    if (!p || p->magic != SUBPROCESS_MAGIC || (p->local && p->in_hook)) {
+    if (!p || (p->local && p->in_hook)) {
         errno = EINVAL;
         return NULL;
     }
     return aux_get (p->aux, name);
 }
 
+int flux_set_default_subprocess_log (flux_t *h,
+                                     subprocess_log_f log_fn,
+                                     void *log_data)
+{
+    if (flux_aux_set (h, "flux::subprocess_llog_fn", log_fn, NULL) < 0
+        || flux_aux_set (h, "flux::subprocess_llog_data", log_data, NULL) < 0)
+        return -1;
+    return 0;
+}
+
+void flux_subprocess_channel_incref (flux_subprocess_t *p, const char *name)
+{
+    struct subprocess_channel *c;
+    if (!p || !p->local)
+        return;
+    if (!(c = zhash_lookup (p->channels, name)))
+        return;
+    fbuf_read_watcher_incref (c->buffer_read_w);
+}
+
+void flux_subprocess_channel_decref (flux_subprocess_t *p, const char *name)
+{
+    struct subprocess_channel *c;
+    if (!p || !p->local)
+        return;
+    if (!(c = zhash_lookup (p->channels, name)))
+        return;
+    fbuf_read_watcher_decref (c->buffer_read_w);
+}
+
+
 /*
  * vi: ts=4 sw=4 expandtab
  */
diff --git a/src/common/libsubprocess/subprocess.h b/src/common/libsubprocess/subprocess.h
index 314d568c4cb0..0b4c498abd8b 100644
--- a/src/common/libsubprocess/subprocess.h
+++ b/src/common/libsubprocess/subprocess.h
@@ -15,13 +15,11 @@
 
 #include <flux/core.h>
 
-/*
- *  flux_cmd_t: An object that defines a command to be run, either
- *   remotely or as a child of the current process. Includes cmdline
- *   arguments, environment, and working directory. A flux_cmd_t is
- *   used to create Flux subprocesses.
- */
-typedef struct flux_command flux_cmd_t;
+#include "command.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 /*
  *  flux_subprocess_t: A subprocess is an instantiation of a command
@@ -31,9 +29,6 @@ typedef struct flux_command flux_cmd_t;
  */
 typedef struct flux_subprocess flux_subprocess_t;
 
-/*  flux_subprocess_server_t: Handler for a subprocess remote server */
-typedef struct flux_subprocess_server flux_subprocess_server_t;
-
 /*
  * Subprocess states, on changes, will lead to calls to
  * on_state_change below.
@@ -41,31 +36,43 @@ typedef struct flux_subprocess_server flux_subprocess_server_t;
  * Possible state changes:
  *
  * init -> running
- * init -> exec failed
  * running -> exited
  * any state -> failed
  */
 typedef enum {
     FLUX_SUBPROCESS_INIT,         /* initial state */
-    FLUX_SUBPROCESS_EXEC_FAILED,  /* exec(2) has failed, only for rexec() */
     FLUX_SUBPROCESS_RUNNING,      /* exec(2) has been called */
     FLUX_SUBPROCESS_EXITED,       /* process has exited */
-    FLUX_SUBPROCESS_FAILED,       /* internal failure, catch all for
-                                   * all other errors */
+    FLUX_SUBPROCESS_FAILED,       /* exec failure or other non-child error */
+    FLUX_SUBPROCESS_STOPPED,      /* process was stopped */
 } flux_subprocess_state_t;
 
 /*
  * Subprocess flags
  */
 enum {
-    /* flux_exec(): let parent stdin, stdout, stderr, carry to child.
-     * Do not create "stdin", "stdout", or "stderr" channels.  Subsequently,
-     * flux_subprocess_write()/close()/read()/read_line() will fail on
-     * streams of "stdin", "stdout", or "stderr".
+    /* flux_local_exec(): let parent stdin, stdout, stderr, carry to
+     * child.  Do not create "stdin", "stdout", or "stderr" channels.
+     * Subsequently, flux_subprocess_write()/close()/read()/read_line()
+     * will fail on streams of "stdin", "stdout", or "stderr".
      */
     FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH = 1,
-    /* flux_exec(): call setpgrp() before exec(2) */
-    FLUX_SUBPROCESS_FLAGS_SETPGRP = 2,
+    /* flux_local_exec(): do not call setpgrp() before exec(2) */
+    FLUX_SUBPROCESS_FLAGS_NO_SETPGRP = 2,
+    /* flux_local_exec(): use fork(2)/exec(2) even if posix_spawn(3)
+     * available */
+    FLUX_SUBPROCESS_FLAGS_FORK_EXEC = 4,
+    /* flux_rexec(): In order to improve performance, do not locally
+     * copy and buffer output from the remote subprocess.  Immediately
+     * call output callbacks.  Users should call
+     * flux_subprocess_read() to retrieve the data.  If
+     * flux_subprocess_read() is not called, data will be lost.  Data
+     * will not be NUL terminated.  flux_subprocess_read() should be
+     * called only once.  If called more than once, the same data is
+     * returned.  Use of other read functions will result in a EPERM
+     * error.
+     */
+    FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF = 8,
 };
 
 /*
@@ -77,22 +84,32 @@ typedef void (*flux_subprocess_output_f) (flux_subprocess_t *p,
                                           const char *stream);
 typedef void (*flux_subprocess_state_f) (flux_subprocess_t *p,
                                          flux_subprocess_state_t state);
+typedef void (*flux_subprocess_credit_f) (flux_subprocess_t *p,
+                                          const char *stream,
+                                          int bytes);
 typedef void (*flux_subprocess_hook_f) (flux_subprocess_t *p, void *arg);
 
 /*
  *  Functions for event-driven subprocess handling:
  *
+ *  When output callbacks are called, flux_subprocess_read(),
+ *  flux_subprocess_read_line() and similar functions should be used
+ *  to read buffered data.  If this is not done, it can lead to
+ *  excessive callbacks and code "spinning".
+ *
+ *  The first call to on_credit will contain the full buffer size.
+ *
  */
 typedef struct {
     flux_subprocess_f on_completion;    /* Process exited and all I/O
                                          * complete, will not be
-                                         * called if EXEC_FAILED or
-                                         * FAILED states reached.
+                                         * called if FAILED state reached.
                                          */
     flux_subprocess_state_f on_state_change;  /* Process state change        */
     flux_subprocess_output_f on_channel_out; /* Read from channel when ready */
     flux_subprocess_output_f on_stdout; /* Read of stdout is ready           */
     flux_subprocess_output_f on_stderr; /* Read of stderr is ready           */
+    flux_subprocess_credit_f on_credit; /* Write buffer space available      */
 } flux_subprocess_ops_t;
 
 /*
@@ -107,184 +124,17 @@ typedef struct {
 } flux_subprocess_hooks_t;
 
 /*
- *  General support:
- */
-
-/*  Start a subprocess server on the handle `h`. Registers message
- *   handlers, etc for remote execution. "prefix" is the topic prefix
- *   used to listen for this service, e.g. `broker` would listen
- *   for `broker.exec`.
- */
-flux_subprocess_server_t *flux_subprocess_server_start (flux_t *h,
-                                                        const char *prefix,
-                                                        const char *local_uri,
-                                                        uint32_t rank);
-
-/*  Stop a subprocess server / cleanup flux_subprocess_server_t.  Will
- *  send a SIGKILL to all remaining subprocesses.
- */
-void flux_subprocess_server_stop (flux_subprocess_server_t *s);
-
-/* Send all subprocesses signal and wait up to wait_time seconds for
- * all subprocesses to complete.  This is typically called to send
- * SIGTERM before calling flux_subprocess_server_stop(), allowing
- * users to send a signal to inform subprocesses to complete / cleanup
- * before they are sent SIGKILL.
- *
- * This function will enter the reactor to wait for subprocesses to
- * complete, should only be called on cleanup path when primary
- * reactor has exited.
- */
-int flux_subprocess_server_subprocesses_kill (flux_subprocess_server_t *s,
-                                              int signum,
-                                              double wait_time);
-
-/* Terminate all subprocesses started by a sender id */
-int flux_subprocess_server_terminate_by_uuid (flux_subprocess_server_t *s,
-                                              const char *id);
-
-/*
- * Convenience Functions:
- */
-
-/*  General output callback that will send output from the subprocess
- *  to stdout or stderr.  Set the `on_stdout` and/or `on_stderr`
- *  callbacks in flux_subprocess_ops_t and this function will output
- *  to stdout/stderr respectively.  You can also set 'on_channel_out'
- *  to this function, which will send all channel output to stdout.
- */
-void flux_standard_output (flux_subprocess_t *p, const char *stream);
-
-/*
- *  Commands:
- */
-
-/*
- *  Create a cmd object, from which subprocesses can be created
- */
-flux_cmd_t * flux_cmd_create (int argc, char *argv[], char **env);
-
-/*
- *  Create a copy of a cmd object.
- */
-flux_cmd_t * flux_cmd_copy (const flux_cmd_t *cmd);
-
-/*
- *  Destroy and free command object `cmd`
- */
-void flux_cmd_destroy (flux_cmd_t *cmd);
-
-/*
- *  Append formatted string to argv of `cmd`.
- */
-int flux_cmd_argv_appendf (flux_cmd_t *cmd, const char *fmt, ...)
-                           __attribute__ ((format (printf, 2, 3)));
-
-/*
- *  Append string to argv of `cmd`.
- */
-int flux_cmd_argv_append (flux_cmd_t *cmd, const char *arg);
-
-/*
- *  Return the current argument count for `cmd`.
- */
-int flux_cmd_argc (const flux_cmd_t *cmd);
-
-/*
- *  Return the current argument at index n (range 0 to argc - 1)
- */
-const char *flux_cmd_arg (const flux_cmd_t *cmd, int n);
-
-/*
- *  Set a single environment variable (name) to formatted string `fmt`.
- *   If `overwrite` is non-zero then overwrite any existing setting for `name`.
- */
-int flux_cmd_setenvf (flux_cmd_t *cmd, int overwrite,
-		      const char *name, const char *fmt, ...)
-                      __attribute__ ((format (printf, 4, 5)));
-
-/*
- *  Unset environment variable `name` in the command object `cmd`.
- */
-void flux_cmd_unsetenv (flux_cmd_t *cmd, const char *name);
-
-/*
- *  Return current value for environment variable `name` as set in
- *   command object `cmd`. If environment variable is not set then NULL
- *   is returned.
- */
-const char *flux_cmd_getenv (const flux_cmd_t *cmd, const char *name);
-
-/*
- *  Set/get the working directory for the command `cmd`.
+ *  llog-compatible callback
  */
-int flux_cmd_setcwd (flux_cmd_t *cmd, const char *cwd);
-const char *flux_cmd_getcwd (const flux_cmd_t *cmd);
+typedef void (*subprocess_log_f)(void *arg,
+                                 const char *file,
+                                 int line,
+                                 const char *func,
+                                 const char *subsys,
+                                 int level,
+                                 const char *fmt,
+                                 va_list args);
 
-/*
- *  Request a channel for communication between process and caller.
- *   Callers can write to the subproces via flux_subprocess_write()
- *   and read from it via flux_subprocess_read(), which is typically
- *   called from a callback set in 'on_channel_out'.
- *
- *  The `name` argument is also used as the name of the environment variable
- *   in the subprocess environment that is set to the file descriptor number
- *   of the process side of the socketpair. E.g. name = "FLUX_PMI_FD" would
- *   result in the environment variable "FLUX_PMI_FD=N" set in the process
- *   environment.
- */
-int flux_cmd_add_channel (flux_cmd_t *cmd, const char *name);
-
-/*
- *  Set generic string options for command object `cmd`. As with environment
- *   variables, this function adds the option `var` with value `val` to
- *   the options array for this command. This can be used to enable optional
- *   behavior for executed processes (e.g. setpgrp(2))
- *
- *  String options, note that name indicates the 'name' argument used
- *  in flux_cmd_add_channel() above.
- *
- *  "BUFSIZE" option
- *
- *   By default, stdio and channels use an internal buffer of 4 megs.
- *   The buffer size can be adjusted with this option.
- *
- *   - name + "_BUFSIZE" - set buffer size on channel name
- *   - stdin_BUFSIZE - set buffer size on stdin
- *   - stdout_BUFSIZE - set buffer size on stdout
- *   - stderr_BUFSIZE - set buffer size on stderr
- *
- *  "LINE_BUFFER" option
- *
- *    By default, output callbacks such as 'on_stdout' and 'on_stderr'
- *    are called when a line of data is available (with the exception
- *    with data after a subprocess has exited).  By setting this
- *    option to "false", output callbacks will be called whenever any
- *    amount of data is available.  These options can also be set to
- *    "true" to keep default behavior of line buffering.
- *
- *    - name + "_LINE_BUFFER" - configuring line buffering on channel name
- *    - stdout_LINE_BUFFER - configure line buffering for stdout
- *    - stderr_LINE_BUFFER - configure line buffering for stderr
- *
- *  "STREAM_STOP" option
- *
- *    By default, the output callbacks such as 'on_stdout' and
- *    'on_stderr' can immediately begin receiving stdout/stderr data
- *    once a subprocess has started.  There are circumstances where a
- *    caller may wish to wait and can have these callbacks stopped by
- *    default and restarted later by flux_subprocess_stream_start().
- *    By setting this option to "true", output callbacks will be
- *    stopped by default.  These options can also be set to "false" to
- *    keep default behavior.  Note that these options only apply to
- *    local subprocesses.
- *
- *    - name + "_STREAM_STOP" - configure start/stop on channel name
- *    - stdout_STREAM_STOP - configure start/stop for stdout
- *    - stderr_STREAM_STOP - configure start/stop for stderr
- */
-int flux_cmd_setopt (flux_cmd_t *cmd, const char *var, const char *val);
-const char *flux_cmd_getopt (flux_cmd_t *cmd, const char *var);
 
 /*
  *  Subprocesses:
@@ -292,8 +142,8 @@ const char *flux_cmd_getopt (flux_cmd_t *cmd, const char *var);
 
 /*
  *  Asynchronously create a new subprocess described by command object
- *   `cmd`.  flux_exec() and flux_local_exec() create a new subprocess
- *   locally.  flux_rexec() creates a new subprocess on Flux rank
+ *   `cmd`.  flux_local_exec() create a new subprocess locally.
+ *   flux_rexec() creates a new subprocess on Flux rank
  *   `rank`. Callbacks in `ops` structure that are non-NULL will be
  *   called to process state changes, I/O, and completion.
  *
@@ -306,40 +156,54 @@ const char *flux_cmd_getopt (flux_cmd_t *cmd, const char *var);
  *   by the time the call returns.
  *
  */
-flux_subprocess_t *flux_exec (flux_t *h, int flags,
-                              const flux_cmd_t *cmd,
-                              const flux_subprocess_ops_t *ops,
-                              const flux_subprocess_hooks_t *hooks);
-
-flux_subprocess_t *flux_local_exec (flux_reactor_t *r, int flags,
+flux_subprocess_t *flux_local_exec (flux_reactor_t *r,
+                                    int flags,
                                     const flux_cmd_t *cmd,
-                                    const flux_subprocess_ops_t *ops,
-                                    const flux_subprocess_hooks_t *hooks);
-
-flux_subprocess_t *flux_rexec (flux_t *h, int rank, int flags,
+                                    const flux_subprocess_ops_t *ops);
+
+flux_subprocess_t *flux_local_exec_ex (flux_reactor_t *r,
+                                       int flags,
+                                       const flux_cmd_t *cmd,
+                                       const flux_subprocess_ops_t *ops,
+                                       const flux_subprocess_hooks_t *hooks,
+                                       subprocess_log_f log_fn,
+                                       void *log_data);
+
+flux_subprocess_t *flux_rexec (flux_t *h,
+                               int rank,
+                               int flags,
                                const flux_cmd_t *cmd,
                                const flux_subprocess_ops_t *ops);
 
+flux_subprocess_t *flux_rexec_ex (flux_t *h,
+                                  const char *service_name,
+                                  int rank,
+                                  int flags,
+                                  const flux_cmd_t *cmd,
+                                  const flux_subprocess_ops_t *ops,
+                                  subprocess_log_f log_fn,
+                                  void *log_data);
+
+
 /* Start / stop a read stream temporarily on local processes.  This
- * may be useful for flow control.  If you desire to have a stream not
- * call 'on_stdout' or 'on_stderr' when the local subprocess has
- * started, see STREAM_STOP configuration above.
- *
- * start and stop return 0 for success, -1 on error
- * status returns > 0 for started, 0 for stopped, -1 on error
+ * may be useful for flow control.
  */
-int flux_subprocess_stream_start (flux_subprocess_t *p, const char *stream);
-int flux_subprocess_stream_stop (flux_subprocess_t *p, const char *stream);
-int flux_subprocess_stream_status (flux_subprocess_t *p, const char *stream);
+void flux_subprocess_stream_start (flux_subprocess_t *p, const char *stream);
+void flux_subprocess_stream_stop (flux_subprocess_t *p, const char *stream);
 
 /*
  *  Write data to "stream" stream of subprocess `p`.  'stream' can be
  *  "stdin" or the name of a stream specified with flux_cmd_add_channel().
  *
- *  Returns the total amount of data successfully buffered.
+ *  Returns the total amount of data successfully buffered or -1 on error
+ *  with errno set. Note: this function will not return a short write. If
+ *  all `len` bytes cannot fit in the destination buffer, then no bytes
+ *  will be written and -1 will be returned with errno=ENOSPC.
  */
-int flux_subprocess_write (flux_subprocess_t *p, const char *stream,
-                           const char *buf, size_t len);
+int flux_subprocess_write (flux_subprocess_t *p,
+                           const char *stream,
+                           const char *buf,
+                           size_t len);
 
 /*
  *  Close "stream" stream of subprocess `p` and schedule EOF to be sent.
@@ -349,41 +213,57 @@ int flux_subprocess_write (flux_subprocess_t *p, const char *stream,
 int flux_subprocess_close (flux_subprocess_t *p, const char *stream);
 
 /*
- *  Read up to `len` bytes of unread data from stream `stream`.  To
- *   read all data, specify 'len' of -1.  'stream' can be "stdout",
+ *  Read unread data from stream `stream`.  'stream' can be "stdout",
  *   "stderr", or the name of a stream specified with flux_cmd_add_channel().
  *
- *   Returns pointer to buffer on success and NULL on error with errno
- *   set.  Buffer is guaranteed to be NUL terminated.  User shall not
- *   free returned pointer.  Length of buffer returned can optionally
- *   returned in 'lenp'.  A length of 0 indicates that the subprocess
- *   has closed this stream.
+ *   Returns length of data on success and -1 on error with errno set.
+ *   Buffer of data is returned in bufp.  Buffer is guaranteed to be
+ *   NUL terminated unless the FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF flag
+ *   has been specified.  User shall not free returned pointer.
+ *
+ *   In most cases, a length of 0 indicates that the subprocess has
+ *   closed this stream.  A length of 0 could be returned if read
+ *   functions are called multiple times within a single output
+ *   callback, so it is generally recommended to call this function
+ *   once per output callback.  flux_subprocess_read_stream_closed()
+ *   can always be used to verify if the stream is in fact closed.
  */
-const char *flux_subprocess_read (flux_subprocess_t *p,
-                                  const char *stream,
-                                  int len, int *lenp);
+int flux_subprocess_read (flux_subprocess_t *p,
+                          const char *stream,
+                          const char **bufp);
 
 /*
- *  Read line unread data from stream `stream`.  'stream' can be
+ *  Read line of unread data from stream `stream`.  'stream' can be
  *   "stdout", "stderr", or the name of a stream specified with
  *   flux_cmd_add_channel().
  *
- *   Returns pointer to buffer on success and NULL on error with errno
- *   set.  Buffer will include newline character and is guaranteed to
- *   be NUL terminated.  If no line is available, returns pointer and
- *   length of zero.  User shall not free returned pointer.  Length of
- *   buffer returned can optionally returned in 'lenp'.
+ *   Returns length of data on success and -1 on error with errno set.
+ *   Buffer with line is returned in bufp.  Buffer will include
+ *   newline character and is guaranteed to be NUL terminated.  If no
+ *   line is available, returns length of zero.  User shall not free
+ *   returned pointer.
+ *
+ *   A length of zero may be returned if the stream is closed OR if
+ *   the stream is line buffered and a line is not yet available. Use
+ *   flux_subprocess_read_stream_closed() to distinguish between the
+ *   two.
+ *
+ *   This function may return an incomplete line when:
+ *
+ *   1) the stream has closed and the last output is not a line
+ *   2) a single line of output exceeds the size of an internal output
+ *      buffer (see BUFSIZE option).
  */
-const char *flux_subprocess_read_line (flux_subprocess_t *p,
-                                       const char *stream,
-                                       int *lenp);
+int flux_subprocess_read_line (flux_subprocess_t *p,
+                               const char *stream,
+                               const char **bufp);
 
 /* Identical to flux_subprocess_read_line(), but does not return
  * trailing newline.
  */
-const char *flux_subprocess_read_trimmed_line (flux_subprocess_t *p,
-                                               const char *stream,
-                                               int *lenp);
+int flux_subprocess_read_trimmed_line (flux_subprocess_t *p,
+                                       const char *stream,
+                                       const char **bufp);
 
 /* Determine if the read stream has is closed / received an EOF.  This
  * function can be useful if you are reading lines via
@@ -391,11 +271,10 @@ const char *flux_subprocess_read_trimmed_line (flux_subprocess_t *p,
  * in output callbacks.  Those functions will return length 0 when no
  * lines are available, making it difficult to determine if the stream
  * has been closed and there is any non-newline terminated data left
- * available for reading with flux_subprocess_read().  Returns > 0 on
- * closed / eof seen, 0 if not, -1 on error.
+ * available for reading with flux_subprocess_read().
  */
-int flux_subprocess_read_stream_closed (flux_subprocess_t *p,
-                                        const char *stream);
+bool flux_subprocess_read_stream_closed (flux_subprocess_t *p,
+                                         const char *stream);
 
 /* flux_subprocess_getline() is a special case function
  * that behaves identically to flux_subprocess_read_line() but handles
@@ -408,14 +287,13 @@ int flux_subprocess_read_stream_closed (flux_subprocess_t *p,
  *   last data on the stream does not terminate in a newline
  *   character, this function will return that last data without the
  *   trailing newline.
- * - if the stream has been closed / reached EOF, lenp will be set to
- *   0.
+ * - if the stream has been closed / reached EOF, 0 will be returned.
  * - if the stream is not line buffered, NULL and errno = EPERM will
  *   be returned.
  */
-const char *flux_subprocess_getline (flux_subprocess_t *p,
-                                     const char *stream,
-                                     int *lenp);
+int flux_subprocess_getline (flux_subprocess_t *p,
+                             const char *stream,
+                             const char **bufp);
 
 /*
  *  Create RPC to send signal `signo` to subprocess `p`.
@@ -426,12 +304,11 @@ const char *flux_subprocess_getline (flux_subprocess_t *p,
 flux_future_t *flux_subprocess_kill (flux_subprocess_t *p, int signo);
 
 /*
- *  Add/remove a reference to subprocess object `p`. The subprocess object
- *   is destroyed once the last reference is removed.
+ *  Remove a reference to subprocess object `p`. The subprocess object
+ *   is destroyed once the last reference is removed.  This call
+ *   silently deso nothing if called within a hook.
  */
-void flux_subprocess_ref (flux_subprocess_t *p);
-void flux_subprocess_unref (flux_subprocess_t *p);
-#define flux_subprocess_destroy(x) flux_subprocess_unref(x)
+void flux_subprocess_destroy (flux_subprocess_t *p);
 
 /*  Return current state value of subprocess.  Note this may differ
  *  than state returned in on_state_change callback, as a subprocess
@@ -445,13 +322,23 @@ flux_subprocess_state_t flux_subprocess_state (flux_subprocess_t *p);
  */
 const char *flux_subprocess_state_string (flux_subprocess_state_t state);
 
+/*  Return true if subprocess p is still active, i.e. it is waiting to
+ *  start, still running, or waiting for eof on all streams.
+ */
+bool flux_subprocess_active (flux_subprocess_t *p);
+
 int flux_subprocess_rank (flux_subprocess_t *p);
 
-/* Returns the errno causing the FLUX_SUBPROCESS_EXEC_FAILED or
- * FLUX_SUBPROCESS_FAILED states to be reached.
+/* Returns the errno causing the FLUX_SUBPROCESS_FAILED state to be reached.
  */
 int flux_subprocess_fail_errno (flux_subprocess_t *p);
 
+/* Returns error message describing why FLUX_SUBPROCESS_FAILED state was
+ * reached.  If error message was not set, will return strerror() of
+ * errno returned from flux_subprocess_fail_errno().
+ */
+const char *flux_subprocess_fail_error (flux_subprocess_t *p);
+
 /* Returns exit status as returned from wait(2).  Works only for
  * FLUX_SUBPROCESS_EXITED state. */
 int flux_subprocess_status (flux_subprocess_t *p);
@@ -480,7 +367,9 @@ flux_reactor_t * flux_subprocess_get_reactor (flux_subprocess_t *p);
  *  Returns 0 on success
  */
 int flux_subprocess_aux_set (flux_subprocess_t *p,
-                             const char *name, void *ctx, flux_free_f free);
+                             const char *name,
+                             void *ctx,
+                             flux_free_f free);
 
 /*
  *  Return pointer to any context associated with `p` under `name`. If
@@ -488,4 +377,18 @@ int flux_subprocess_aux_set (flux_subprocess_t *p,
  */
 void *flux_subprocess_aux_get (flux_subprocess_t *p, const char *name);
 
+/*
+ *  Take/drop a reference on a subprocess output channel `name` (e.g. "stdout"
+ *   or "stderr"). EOF will not be produced from this channel the reference
+ *   count drops to zero.
+ */
+void flux_subprocess_channel_incref (flux_subprocess_t *p, const char *name);
+void flux_subprocess_channel_decref (flux_subprocess_t *p, const char *name);
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* !_FLUX_CORE_SUBPROCESS_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/subprocess_private.h b/src/common/libsubprocess/subprocess_private.h
index d000d027dca4..d13859c9e917 100644
--- a/src/common/libsubprocess/subprocess_private.h
+++ b/src/common/libsubprocess/subprocess_private.h
@@ -11,25 +11,20 @@
 #ifndef _SUBPROCESS_PRIVATE_H
 #define _SUBPROCESS_PRIVATE_H
 
+#include <flux/idset.h>
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "subprocess.h"
-
-#define SUBPROCESS_MAGIC           0xbeefcafe
-
-#define SUBPROCESS_SERVER_MAGIC    0xbeefbeef
+#include "fbuf_watcher.h"
 
 #define SUBPROCESS_DEFAULT_BUFSIZE 4194304
 
-#define CHANNEL_MAGIC    0xcafebeef
-
 #define CHANNEL_READ  0x01
 #define CHANNEL_WRITE 0x02
 #define CHANNEL_FD    0x04
 
 struct subprocess_channel {
-    int magic;
-
     flux_subprocess_t *p;
-    flux_subprocess_output_f output_f;
+    flux_subprocess_output_f output_cb;
     char *name;
     int flags;
 
@@ -41,6 +36,8 @@ struct subprocess_channel {
     int parent_fd;
     int child_fd;
     flux_watcher_t *buffer_write_w;
+    int buffer_space;
+    bool initial_credits_sent;     /* accounting for on_credit callback */
     flux_watcher_t *buffer_read_w;
     /* buffer_read_stopped_w is a "sub-in" watcher if buffer_read_w is
      * stopped.  We need to put something into the reactor so we know
@@ -50,30 +47,29 @@ struct subprocess_channel {
     bool buffer_read_w_started;
 
     /* remote */
-    flux_buffer_t *write_buffer;
-    flux_buffer_t *read_buffer;
-    bool write_eof_sent;
+    struct fbuf *read_buffer;
     bool read_eof_received;
-    flux_watcher_t *in_prep_w;
-    flux_watcher_t *in_idle_w;
-    flux_watcher_t *in_check_w;
     flux_watcher_t *out_prep_w;
     flux_watcher_t *out_idle_w;
     flux_watcher_t *out_check_w;
+    /* for FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF */
+    const char *unbuf_data;
+    int unbuf_len;
 
     /* misc */
     bool line_buffered;         /* for buffer_read_w / read_buffer */
 };
 
 struct flux_subprocess {
-    int magic;
-
     flux_t *h;
     flux_reactor_t *reactor;
     uint32_t rank;
     int flags;
     bool local;                     /* This is a local process, not remote. */
 
+    subprocess_log_f llog;
+    void *llog_data;
+
     int refcount;
     pid_t pid;
     bool pid_set;
@@ -82,14 +78,13 @@ struct flux_subprocess {
 
     flux_cmd_t *cmd;                /* readonly/o copy of the command     */
 
-    struct aux_item *aux;           /* auxillary data                     */
+    struct aux_item *aux;           /* auxiliary data                     */
 
     zhash_t *channels;              /* hash index by name to channel info */
     int channels_eof_expected;      /* number of eofs to expect */
     int channels_eof_sent;          /* counter to avoid loop checks */
 
     int status;      /* Raw status from waitpid(2), valid if exited       */
-    int exec_failed_errno;  /* Holds errno from exec(2) if exec() failed  */
 
     flux_subprocess_state_t state;
     flux_subprocess_state_t state_reported; /* for on_state_change */
@@ -112,28 +107,14 @@ struct flux_subprocess {
 
     /* remote */
 
+    char *service_name;
     flux_future_t *f;           /* primary future reactor */
     bool remote_completed;      /* if remote has completed */
     int failed_errno;           /* Holds errno if FAILED state reached */
+    flux_error_t failed_error;  /* Holds detailed message for failed_errno */
     int signal_pending;         /* signal sent while starting */
 };
 
-struct flux_subprocess_server {
-    int magic;
-    flux_t *h;
-    flux_reactor_t *r;
-    char *local_uri;
-    uint32_t rank;
-    zhash_t *subprocesses;
-    flux_msg_handler_t **handlers;
-
-    /* for teardown / termination */
-    flux_watcher_t *terminate_timer_w;
-    flux_watcher_t *terminate_prep_w;
-    flux_watcher_t *terminate_idle_w;
-    flux_watcher_t *terminate_check_w;
-};
-
 void subprocess_check_completed (flux_subprocess_t *p);
 
 void state_change_start (flux_subprocess_t *p);
@@ -141,8 +122,17 @@ void state_change_start (flux_subprocess_t *p);
 void channel_destroy (void *arg);
 
 struct subprocess_channel *channel_create (flux_subprocess_t *p,
-                                           flux_subprocess_output_f output_f,
+                                           flux_subprocess_output_f output_cb,
                                            const char *name,
                                            int flags);
 
+struct idset * subprocess_childfds (flux_subprocess_t *p);
+
+void subprocess_incref (flux_subprocess_t *p);
+void subprocess_decref (flux_subprocess_t *p);
+
+void subprocess_standard_output (flux_subprocess_t *p, const char *name);
+
 #endif /* !_SUBPROCESS_PRIVATE_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/test/bulk-exec-einval.c b/src/common/libsubprocess/test/bulk-exec-einval.c
new file mode 100644
index 000000000000..0667aea96527
--- /dev/null
+++ b/src/common/libsubprocess/test/bulk-exec-einval.c
@@ -0,0 +1,69 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libsubprocess/bulk-exec.h"
+#include "ccan/str/str.h"
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    ok (bulk_exec_aux_get (NULL, NULL) == NULL && errno == EINVAL,
+        "bulk_exec_aux_get (NULL, NULL) returns EINVAL");
+    ok (bulk_exec_aux_set (NULL, NULL, NULL, NULL) < 0 && errno == EINVAL,
+        "bulk_exec_aux_set (NULL, ..) returns EINVAL");
+    ok (bulk_exec_set_max_per_loop (NULL, 1) < 0 && errno == EINVAL,
+        "bulk_exec_set_max_per_loop (NULL, 1) returns EINVAL");
+    ok (bulk_exec_push_cmd (NULL, NULL, NULL, 0) < 0 && errno == EINVAL,
+        "bulk_exec_push_cmd (NULL, ...) returns EINVAL");
+    ok (bulk_exec_start (NULL, NULL) < 0 && errno == EINVAL,
+        "bulk_exec_start (NULL, NULL) returns EINVAL");
+    ok (bulk_exec_kill (NULL, NULL, 0) == NULL && errno == EINVAL,
+        "bulk_exec_kill (NULL, NULL, 0) returns EINVAL");
+    ok (bulk_exec_cancel (NULL) < 0 && errno == EINVAL,
+        "bulk_exec_cancel (NULL) returns EINVAL");
+    ok (bulk_exec_rc (NULL) < 0 && errno == EINVAL,
+        "bulk_exec_rc (NULL) returns EINVAL");
+    ok (bulk_exec_started_count (NULL) == 0,
+        "bulk_exec_started_count (NULL) == 0");
+    ok (bulk_exec_active_count (NULL) == 0,
+        "bulk_exec_active_count (NULL) == 0");
+    ok (bulk_exec_complete (NULL) == 0,
+        "bulk_exec_complete (NULL) == 0");
+    ok (bulk_exec_total (NULL) == 0,
+        "bulk_exec_total (NULL) == 0");
+    ok (bulk_exec_active_ranks (NULL) == NULL && errno == EINVAL,
+        "bulk_exec_active_ranks (NULL) returns EINVAL");
+    ok (bulk_exec_write (NULL, NULL, NULL, 0) < 0 && errno == EINVAL,
+        "bulk_exec_write (NULL, ...) returns EINVAL");
+    ok (bulk_exec_close (NULL, NULL) < 0 && errno == EINVAL,
+        "bulk_exec_close (NULL, NULL) returns EINVAL");
+    ok (bulk_exec_service_name (NULL) == NULL && errno == EINVAL,
+        "bulk_exec_service_name (NULL) returns EINVAL");
+    ok (bulk_exec_get_subprocess (NULL, 0) == NULL && errno == EINVAL,
+        "bulk_exec_get_subprocess (NULL, 0) returns EINVAL");
+    lives_ok ({bulk_exec_destroy (NULL);},
+              "bulk_exec_destroy (NULL) does not die");
+    lives_ok ({bulk_exec_kill_log_error (NULL, 1);},
+              "bulk_exec_kill_log_error (NULL, ...) does not die");
+
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libsubprocess/test/bulk-exec.c b/src/common/libsubprocess/test/bulk-exec.c
new file mode 100644
index 000000000000..88b24d2be212
--- /dev/null
+++ b/src/common/libsubprocess/test/bulk-exec.c
@@ -0,0 +1,282 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/*  Test driver for job-exec/bulk-exec.[ch] bulk subprocess
+ *   execution API.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <assert.h>
+#include <signal.h>
+#ifndef HAVE_GET_CURRENT_DIR_NAME
+#include "src/common/libmissing/get_current_dir_name.h"
+#endif
+
+#include <flux/core.h>
+#include <flux/idset.h>
+#include <flux/optparse.h>
+
+#include "bulk-exec.h"
+#include "src/common/libutil/log.h"
+#include "ccan/str/str.h"
+
+extern char **environ;
+static int cancel_after = 0;
+
+void started (struct bulk_exec *exec, void *arg)
+{
+    log_msg ("started");
+}
+
+void complete (struct bulk_exec *exec, void *arg)
+{
+    flux_t *h = arg;
+    log_msg ("complete");
+    flux_reactor_stop (flux_get_reactor (h));
+}
+
+void exited (struct bulk_exec *exec, void *arg, const struct idset *ids)
+{
+    char *s = idset_encode (ids, IDSET_FLAG_RANGE);
+    log_msg ("ranks %s: exited", s);
+    free (s);
+}
+
+void on_error (struct bulk_exec *exec, flux_subprocess_t *p, void *arg)
+{
+    if (p) {
+        flux_subprocess_state_t state = flux_subprocess_state (p);
+        log_msg ("%d: pid %ju: %s", flux_subprocess_rank (p),
+                (uintmax_t) flux_subprocess_pid (p),
+                flux_subprocess_state_string (state));
+    }
+    flux_future_t *f = bulk_exec_kill (exec, NULL, 9);
+    if (flux_future_get (f, NULL) < 0)
+        log_err_exit ("bulk_exec_kill");
+}
+
+void on_output (struct bulk_exec *exec, flux_subprocess_t *p,
+                const char *stream, const char *data,
+                int data_len, void *arg)
+{
+    int rank = flux_subprocess_rank (p);
+    FILE *fp = streq (stream, "stdout") ? stdout : stderr;
+    fprintf (fp, "%d: %s", rank, data);
+}
+
+static void kill_cb (flux_future_t *f, void *arg)
+{
+    if (flux_future_get (f, NULL) < 0)
+        log_err ("bulk_exec_kill");
+    flux_future_destroy (f);
+}
+
+static void signal_cb (flux_reactor_t *r, flux_watcher_t *w,
+                       int revents, void *arg)
+{
+    struct bulk_exec *exec = arg;
+    int signum = flux_signal_watcher_get_signum (w);
+
+    log_msg ("sending signal %d to all tasks\n", signum);
+    flux_future_t *f = bulk_exec_kill (exec, NULL, signum);
+    if (!f || (flux_future_then (f, -1., kill_cb, exec) < 0))
+        log_err ("SIGINT: failed to forward signal %d", signum);
+    flux_watcher_stop (w);
+}
+
+static void check_cancel_cb (flux_reactor_t *r, flux_watcher_t *w,
+                             int revents, void *arg)
+{
+    struct bulk_exec *exec = arg;
+    if (cancel_after && bulk_exec_started_count (exec) >= cancel_after) {
+        log_msg ("cancelling remaining commands");
+        bulk_exec_cancel (exec);
+        flux_watcher_stop (w);
+    }
+}
+
+static unsigned int idset_pop (struct idset *idset)
+{
+    unsigned int id = idset_first (idset);
+    if (id == IDSET_INVALID_ID)
+        return id;
+    (void) idset_clear (idset, id);
+    return id;
+}
+
+static struct idset * idset_pop_n (struct idset *idset, int n)
+{
+    unsigned int id;
+    struct idset *ids = idset_create (0, IDSET_FLAG_AUTOGROW);
+    if (ids == NULL)
+        return NULL;
+    while ((n-- > 0) && ((id = idset_pop (idset)) != IDSET_INVALID_ID)) {
+        if (idset_set (ids, id) < 0)
+            goto err;
+    }
+    return ids;
+err:
+    idset_destroy (ids);
+    return NULL;
+}
+
+void push_commands (struct bulk_exec *exec,
+                    struct idset *idset,
+                    int ncmds,
+                    int ac, char **av)
+{
+    flux_cmd_t *cmd = flux_cmd_create (ac, av, environ);
+    char *cwd = get_current_dir_name ();
+    struct idset *ids = NULL;
+    int count;
+    int per_cmd;
+
+    if (cmd == NULL)
+        log_err_exit ("flux_cmd_create");
+
+    if ((count = idset_count (idset)) < ncmds)
+        log_err_exit ("Can't split %d ranks into %d cmds", count, ncmds);
+    flux_cmd_setcwd (cmd, cwd);
+    free (cwd);
+
+    per_cmd = count/ncmds;
+    while (idset_count (idset)) {
+        struct idset *ids = idset_pop_n (idset, per_cmd);
+        if (bulk_exec_push_cmd (exec, ids, cmd, 0) < 0)
+            log_err_exit ("bulk_exec_push_cmd");
+        idset_destroy (ids);
+    }
+    idset_destroy (ids);
+    flux_cmd_destroy (cmd);
+}
+
+int main (int ac, char **av)
+{
+    struct bulk_exec *exec = NULL;
+    optparse_t *p = NULL;
+    const char *ranks = NULL;
+    struct idset *idset = NULL;
+    flux_t *h = NULL;
+    uint32_t size;
+    int ncmds = 1;
+    int optindex = -1;
+
+    struct optparse_option opts[] = {
+        { .name = "rank",
+          .key = 'r',
+          .has_arg = 1,
+          .arginfo = "IDS",
+          .usage = "Target ranks in IDS"
+        },
+        { .name = "mpl",
+          .key = 'm',
+          .has_arg = 1,
+          .arginfo = "COUNT",
+          .usage = "Max procs to start per loop iteration"
+        },
+        { .name = "ncmds",
+          .key = 'n',
+          .has_arg = 1,
+          .arginfo = "N",
+          .usage = "Internally, split into N 'cmds'"
+        },
+        { .name = "cancel-after",
+          .key  = 'c',
+          .has_arg = 1,
+          .arginfo = "NCMDS",
+          .usage = "Cancel after NCMDS cmds have been launched"
+        },
+        OPTPARSE_TABLE_END
+    };
+
+    struct bulk_exec_ops ops = {
+        .on_start    = started,
+        .on_exit     = exited,
+        .on_complete = complete,
+        .on_error    = on_error,
+        .on_output   = on_output,
+    };
+
+    if (!(p = optparse_create ("execer")))
+        log_err_exit ("optparse_create");
+    if (optparse_add_option_table (p, opts) != OPTPARSE_SUCCESS)
+        log_msg_exit ("optparse_add_option_table");
+    if ((optindex = optparse_parse_args (p, ac, av)) < 0)
+        exit (1);
+    av += optindex;
+    ac -= optindex;
+
+    if (ac == 0) {
+        optparse_print_usage (p);
+        exit (1);
+    }
+
+    if (!optparse_getopt (p, "rank", &ranks))
+        ranks = "all";
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (flux_get_size (h, &size) < 0)
+        log_err_exit ("flux_get_size");
+
+    if (streq (ranks, "all")) {
+        if (!(idset = idset_create (size, 0)))
+            log_err_exit ("idset_create");
+        if (idset_range_set (idset, 0, size - 1) < 0)
+            log_err_exit ("idset_range_set");
+    }
+    else if (!(idset = idset_decode (ranks)))
+        log_err_exit ("idset_decode (%s)", ranks);
+
+    if (!(exec = bulk_exec_create (&ops, "rexec", 1234, "shell", h)))
+        log_err_exit ("bulk_exec_create");
+
+    if (bulk_exec_set_max_per_loop (exec, optparse_get_int (p, "mpl", -1)) < 0)
+        log_err_exit ("bulk_exec_set_max_per_loop");
+
+    ncmds = optparse_get_int (p, "ncmds", 1);
+
+    push_commands (exec, idset, ncmds, ac, av);
+
+    if (bulk_exec_start (h, exec) < 0)
+        log_err_exit ("bulk_exec_start");
+
+    flux_watcher_t *w = flux_signal_watcher_create (flux_get_reactor (h),
+                                                    SIGINT,
+                                                    signal_cb,
+                                                    exec);
+    flux_watcher_start (w);
+
+    flux_watcher_t *cw = NULL;
+    if ((cancel_after = optparse_get_int (p, "cancel-after", 0)) > 0) {
+        cw = flux_check_watcher_create (flux_get_reactor (h),
+                                        check_cancel_cb,
+                                        exec);
+        flux_watcher_start (cw);
+    }
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        log_err_exit ("flux_reactor_run");
+
+    flux_watcher_destroy (w);
+    flux_watcher_destroy (cw);
+    idset_destroy (idset);
+    bulk_exec_destroy (exec);
+    flux_close (h);
+    optparse_destroy (p);
+
+    return 0;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libsubprocess/test/channel.c b/src/common/libsubprocess/test/channel.c
new file mode 100644
index 000000000000..27e2ff5d2bec
--- /dev/null
+++ b/src/common/libsubprocess/test/channel.c
@@ -0,0 +1,509 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <assert.h>
+#include <flux/core.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <unistd.h>
+#include <sys/mman.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libsubprocess/subprocess.h"
+#include "src/common/libsubprocess/subprocess_private.h"
+#include "src/common/libsubprocess/server.h"
+#include "ccan/str/str.h"
+
+extern char **environ;
+
+int completion_cb_count;
+int channel_fd_env_cb_count;
+int channel_in_cb_count;
+int channel_in_and_out_cb_count;
+int multiple_lines_channel_cb_count;
+int channel_nul_terminate_cb_count;
+
+static int fdcount (void)
+{
+    int fd, fdlimit = sysconf (_SC_OPEN_MAX);
+    int count = 0;
+    for (fd = 0; fd < fdlimit; fd++) {
+        if (fcntl (fd, F_GETFD) != -1)
+            count++;
+    }
+    return count;
+}
+
+void completion_cb (flux_subprocess_t *p)
+{
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_EXITED,
+        "subprocess state == EXITED in completion handler");
+    ok (flux_subprocess_status (p) != -1,
+        "subprocess status is valid");
+    ok (flux_subprocess_exit_code (p) == 0,
+        "subprocess exit code is 0, got %d", flux_subprocess_exit_code (p));
+    completion_cb_count++;
+}
+
+void channel_fd_env_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+
+    ok (!strcasecmp (stream, "stdout"),
+        "channel_fd_env_cb called with correct stream");
+
+    if (!channel_fd_env_cb_count) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (strstarts (buf, "FOO="),
+            "environment variable FOO created in subprocess");
+        /* no length check, can't predict channel FD value */
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+
+    channel_fd_env_cb_count++;
+}
+
+void test_channel_fd_env (flux_reactor_t *r)
+{
+    char *av[] = { "/usr/bin/env", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_add_channel (cmd, "FOO") == 0,
+        "flux_cmd_add_channel success adding channel FOO");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = channel_fd_env_cb
+    };
+    completion_cb_count = 0;
+    channel_fd_env_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (channel_fd_env_cb_count == 2, "channel fd callback called 2 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void channel_in_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+
+    ok (!strcasecmp (stream, "stdout"),
+        "channel_in_cb called with correct stream");
+
+    if (!channel_in_cb_count) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len == 7
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (!memcmp (buf, "foobar\n", 7),
+            "read on channel returned correct data");
+
+        ok (flux_subprocess_close (p, "TEST_CHANNEL") == 0,
+            "flux_subprocess_close success");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+
+    channel_in_cb_count++;
+}
+
+void test_channel_fd_in (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-c", "TEST_CHANNEL", "-O", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
+        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_channel_out = NULL,
+        .on_stdout = channel_in_cb,
+        .on_stderr = subprocess_standard_output
+    };
+    completion_cb_count = 0;
+    channel_in_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    ok (flux_subprocess_write (p, "TEST_CHANNEL", "foobar", 6) == 6,
+        "flux_subprocess_write success");
+
+    /* close after we get output */
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (channel_in_cb_count == 2, "channel in callback called 2 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void channel_in_and_out_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+
+    ok (!strcasecmp (stream, "TEST_CHANNEL"),
+        "channel_in_and_out_cb called with correct stream");
+
+    if (!channel_in_and_out_cb_count) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len == 7
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (!memcmp (buf, "foobaz\n", 7),
+            "read on channel returned correct data");
+
+        ok (flux_subprocess_close (p, "TEST_CHANNEL") == 0,
+            "flux_subprocess_close success");
+    }
+    else {
+        /* no check of flux_subprocess_read_stream_closed(), we aren't
+         * closing channel in test below */
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+
+    channel_in_and_out_cb_count++;
+}
+
+void test_channel_fd_in_and_out (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-c", "TEST_CHANNEL", "-C", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
+        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_channel_out = channel_in_and_out_cb,
+        .on_stdout = subprocess_standard_output,
+        .on_stderr = subprocess_standard_output
+    };
+    completion_cb_count = 0;
+    channel_in_and_out_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    ok (flux_subprocess_write (p, "TEST_CHANNEL", "foobaz", 6) == 6,
+        "flux_subprocess_write success");
+
+    /* don't call flux_subprocess_close() here, we'll race with data
+     * coming back, call in callback */
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (channel_in_and_out_cb_count == 2, "channel out callback called 2 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void channel_multiple_lines_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+
+    ok (!strcasecmp (stream, "TEST_CHANNEL"),
+        "channel_multiple_lines_cb called with correct stream");
+
+    if (multiple_lines_channel_cb_count == 0) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (streq (buf, "bob\n"),
+            "flux_subprocess_read_line returned correct data");
+    }
+    else if (multiple_lines_channel_cb_count == 1) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (streq (buf, "dan\n"),
+            "flux_subprocess_read_line returned correct data");
+    }
+    else if (multiple_lines_channel_cb_count == 2) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (streq (buf, "jo\n"),
+            "flux_subprocess_read_line returned correct data");
+
+        ok (flux_subprocess_close (p, "TEST_CHANNEL") == 0,
+            "flux_subprocess_close success");
+    }
+    else {
+        /* no check of flux_subprocess_read_stream_closed(), we aren't
+         * closing channel in test below */
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+
+    multiple_lines_channel_cb_count++;
+}
+
+void test_channel_multiple_lines (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-c", "TEST_CHANNEL", "-C", "-n", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
+        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_channel_out = channel_multiple_lines_cb,
+        .on_stdout = subprocess_standard_output,
+        .on_stderr = subprocess_standard_output
+    };
+    completion_cb_count = 0;
+    multiple_lines_channel_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    ok (flux_subprocess_write (p, "TEST_CHANNEL", "bob\n", 4) == 4,
+        "flux_subprocess_write success");
+
+    ok (flux_subprocess_write (p, "TEST_CHANNEL", "dan\n", 4) == 4,
+        "flux_subprocess_write success");
+
+    ok (flux_subprocess_write (p, "TEST_CHANNEL", "jo\n", 3) == 3,
+        "flux_subprocess_write success");
+
+    /* don't call flux_subprocess_close() here, we'll race with data
+     * coming back, call in callback */
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (multiple_lines_channel_cb_count == 4, "channel output callback called 4 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void channel_nul_terminate_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+
+    if (!channel_nul_terminate_cb_count) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len == 7
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (!memcmp (buf, "foobaz\n\0", 8),
+            "read on channel returned correct data");
+
+        ok (flux_subprocess_close (p, "TEST_CHANNEL") == 0,
+            "flux_subprocess_close success");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+
+    channel_nul_terminate_cb_count++;
+}
+
+void test_bufsize (flux_reactor_t *r)
+{
+    char *av[] = { "true", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (1, av, environ)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
+        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
+
+    ok (flux_cmd_setopt (cmd, "stdin_BUFSIZE", "1024") == 0,
+        "flux_cmd_setopt set stdin_BUFSIZE success");
+
+    ok (flux_cmd_setopt (cmd, "stdout_BUFSIZE", "1024") == 0,
+        "flux_cmd_setopt set stdout_BUFSIZE success");
+
+    ok (flux_cmd_setopt (cmd, "stderr_BUFSIZE", "1024") == 0,
+        "flux_cmd_setopt set stderr_BUFSIZE success");
+
+    ok (flux_cmd_setopt (cmd, "TEST_CHANNEL_BUFSIZE", "1024") == 0,
+        "flux_cmd_setopt set TEST_CHANNEL_BUFSIZE success");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_channel_out = subprocess_standard_output,
+        .on_stdout = subprocess_standard_output,
+        .on_stderr = subprocess_standard_output
+    };
+    completion_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void test_bufsize_error (flux_reactor_t *r)
+{
+    char *av[] = { "true", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_channel_out = subprocess_standard_output,
+        .on_stdout = subprocess_standard_output,
+        .on_stderr = subprocess_standard_output
+    };
+
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
+        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
+
+    ok (flux_cmd_setopt (cmd, "TEST_CHANNEL_BUFSIZE", "ABCD") == 0,
+        "flux_cmd_setopt set TEST_CHANNEL_BUFSIZE success");
+
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p == NULL
+        && errno == EINVAL,
+        "flux_local_exec fails with EINVAL due to bad bufsize input");
+
+    flux_cmd_destroy (cmd);
+
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
+        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
+
+    ok (flux_cmd_setopt (cmd, "TEST_CHANNEL_BUFSIZE", "0") == 0,
+        "flux_cmd_setopt set TEST_CHANNEL_BUFSIZE success");
+
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p == NULL
+        && errno == EINVAL,
+        "flux_local_exec fails with EINVAL due to bufsize zero");
+
+    flux_cmd_destroy (cmd);
+}
+
+int main (int argc, char *argv[])
+{
+    flux_reactor_t *r;
+    int start_fdcount, end_fdcount;
+
+    plan (NO_PLAN);
+
+    // Create shared reactor for all tests
+    ok ((r = flux_reactor_create (FLUX_REACTOR_SIGCHLD)) != NULL,
+        "flux_reactor_create");
+
+    start_fdcount = fdcount ();
+
+    diag ("channel_fd_env");
+    test_channel_fd_env (r);
+    diag ("channel_fd_in");
+    test_channel_fd_in (r);
+    diag ("channel_fd_in_and_out");
+    test_channel_fd_in_and_out (r);
+    diag ("channel_multiple_lines");
+    test_channel_multiple_lines (r);
+    diag ("bufsize");
+    test_bufsize (r);
+    diag ("bufsize_error");
+    test_bufsize_error (r);
+
+    end_fdcount = fdcount ();
+
+    ok (start_fdcount == end_fdcount,
+        "no file descriptors leaked");
+
+    flux_reactor_destroy (r);
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libsubprocess/test/cmd.c b/src/common/libsubprocess/test/cmd.c
deleted file mode 100644
index 810301f3685b..000000000000
--- a/src/common/libsubprocess/test/cmd.c
+++ /dev/null
@@ -1,284 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <assert.h>
-#include <jansson.h>
-
-#include "src/common/libtap/tap.h"
-#include "src/common/libsubprocess/command.h"
-
-/*
- *  Check basic flux_cmd_create () with args
- */
-void check_basic_create ()
-{
-    char **av;
-    char * argv[] = {
-        "test",
-        "--option=foo",
-        "bar",
-        NULL
-    };
-    int argc = (sizeof (argv)/sizeof (argv[0])) - 1;
-    char * env[] = {
-        "FOO=bar",
-        "PATH=/bin",
-        NULL
-    };
-    flux_cmd_t *cmd;
-
-    diag ("simple flux_cmd_create (argc, argv, env)");
-    cmd = flux_cmd_create (argc, argv, env);
-    ok (cmd != NULL, "flux_cmd_create ()");
-    av = flux_cmd_argv_expand (cmd);
-    ok (av != NULL, "flux_cmd_argv_expand ()");
-    is (av[0], "test", "av[0] == test");
-    is (av[1], "--option=foo", "av[1] == --option=foo");
-    is (av[2], "bar", "av[2] == bar");
-    ok (av[3] == NULL, "av[3] == NULL");
-    free (av);
-    is (flux_cmd_getenv (cmd, "FOO"), "bar", "flux_cmd_getenv");
-    is (flux_cmd_getenv (cmd, "PATH"), "/bin", "flux_cmd_getenv");
-
-    flux_cmd_destroy (cmd);
-}
-
-
-void check_empty_cmd_attributes (flux_cmd_t *cmd)
-{
-    char **argv, **env;
-
-    ok (flux_cmd_argc (cmd) == 0, "flux_cmd_argc");
-
-    argv = flux_cmd_argv_expand (cmd);
-    ok (argv != NULL, "flux_cmd_argv_expand returned an argv");
-    ok (argv[0] == NULL, "argv is properly NULL terminated");
-    free (argv);
-
-    env = flux_cmd_env_expand (cmd);
-    ok (env != NULL, "flux_cmd_env_expand works");
-    ok (env[0] == NULL, "flux_cmd_env_expand properly terminates env");
-    free (env);
-
-    ok (flux_cmd_getcwd (cmd) == NULL,
-        "flux_cmd_getcwd returns NULL");
-}
-
-/*
- * Set some basic known cmd attributes for testing
- */
-void set_cmd_attributes (flux_cmd_t *cmd)
-{
-    assert (flux_cmd_argc (cmd) == 0);
-
-    // Append to argv
-    ok (flux_cmd_argv_append (cmd, "command") >= 0,
-        "flux_cmd_argv_append");
-    ok (flux_cmd_argv_append (cmd, "foo") >= 0,
-        "flux_cmd_argv_append");
-    ok (flux_cmd_argv_appendf (cmd, "%s", "bar") >= 0,
-        "flux_cmd_argv_appendf");
-
-    // Test setenvf
-    ok (flux_cmd_setenvf (cmd, 0, "PATH", "/bin:/usr/bin") >= 0,
-        "flux_cmd_setenvf (PATH)");
-
-    ok (flux_cmd_setcwd (cmd, "/tmp") >= 0,
-        "flux_cmd_setcwd (/tmp)");
-    ok (flux_cmd_add_channel (cmd, "MY_FD") >= 0,
-        "flux_cmd_add_channel");
-    ok (flux_cmd_setopt (cmd, "OPTION", "VALUE") >= 0,
-        "flux_cmd_setopt");
-}
-
-/* set alternate way, to ensure alternate ways also work */
-void set_cmd_attributes2 (flux_cmd_t *cmd)
-{
-    char *env[] = { "PATH=/bin:/usr/bin", NULL };
-
-    ok (flux_cmd_set_env (cmd, env) == 0,
-        "flux_cmd_set_env");
-}
-
-void check_cmd_attributes (flux_cmd_t *cmd)
-{
-    char **argv, **env;
-    const char *arg = NULL;
-
-    ok (flux_cmd_argc (cmd) == 3, "flux_cmd_argc");
-
-    argv = flux_cmd_argv_expand (cmd);
-    ok (argv != NULL, "flux_cmd_argv_expand returned an argv");
-    ok (argv[3] == NULL, "argv is properly NULL terminated");
-    is (argv[0], "command", "argv[0] is correct");
-    is (argv[1], "foo", "argv[1] is correct");
-    is (argv[2], "bar", "argv[2] is correct");
-    free (argv);
-
-    ok (flux_cmd_arg (cmd, 3) == NULL
-        && errno == EINVAL,
-        "flux_cmd_arg returns EINVAL on bad range");
-    arg = flux_cmd_arg (cmd, 0);
-    ok (arg != NULL
-        && !strcmp (arg, "command"),
-        "flux_cmd_arg returns correct argv[0]");
-    arg = flux_cmd_arg (cmd, 1);
-    ok (arg != NULL
-        && !strcmp (arg, "foo"),
-        "flux_cmd_arg returns correct argv[1]");
-    arg = flux_cmd_arg (cmd, 2);
-    ok (arg != NULL
-        && !strcmp (arg, "bar"),
-        "flux_cmd_arg returns correct argv[2]");
-
-    is (flux_cmd_getenv (cmd, "PATH"), "/bin:/usr/bin",
-        "flux_cmd_getenv");
-
-    env = flux_cmd_env_expand (cmd);
-    ok (env != NULL, "flux_cmd_env_expand works");
-    ok (env[1] == NULL, "flux_cmd_env_expand properly terminates env");
-    is (env[0], "PATH=/bin:/usr/bin",
-        "first entry of env is as expected");
-    free (env);
-
-    is (flux_cmd_getcwd (cmd), "/tmp",
-        "flux_cmd_getcwd");
-    is (flux_cmd_getopt (cmd, "OPTION"), "VALUE",
-        "flux_cmd_getopt (cmd, 'OPTION') == VALUE");
-}
-
-void test_find_opts (void)
-{
-    flux_cmd_t *cmd;
-    const char *substrings1[] = { "FOO", NULL };
-    const char *substrings2[] = { "DUH", "BAZ", "UHH", NULL };
-    const char *substrings3[] = { "OOPS",  NULL };
-    const char *substrings4[] = { NULL };
-
-    cmd = flux_cmd_create (0, NULL, NULL);
-    ok (cmd != NULL,
-        "flux_cmd_create works");
-
-    ok (flux_cmd_setopt (cmd, "a_FOO", "val") == 0,
-        "flux_cmd_setopt works");
-    ok (flux_cmd_setopt (cmd, "a_BAR", "val") == 0,
-        "flux_cmd_setopt works");
-    ok (flux_cmd_setopt (cmd, "b_BAR", "val") == 0,
-        "flux_cmd_setopt works");
-    ok (flux_cmd_setopt (cmd, "a_BAZ", "val") == 0,
-        "flux_cmd_setopt works");
-    ok (flux_cmd_setopt (cmd, "b_BAZ", "val") == 0,
-        "flux_cmd_setopt works");
-
-    ok (flux_cmd_find_opts (cmd, substrings1) == 1,
-        "flux_cmd_find_opts finds substrings");
-
-    ok (flux_cmd_find_opts (cmd, substrings2) == 1,
-        "flux_cmd_find_opts finds substrings");
-
-    ok (flux_cmd_find_opts (cmd, substrings3) == 0,
-        "flux_cmd_find_opts doesn't find substrings");
-
-    ok (flux_cmd_find_opts (cmd, substrings4) == 0,
-        "flux_cmd_find_opts doesn't find substrings");
-}
-
-int main (int argc, char *argv[])
-{
-    char *s;
-    flux_cmd_t *cmd, *copy;
-
-    plan (NO_PLAN);
-
-    diag ("Basic flux_cmd_create");
-    check_basic_create ();
-
-    diag ("Create a flux_cmd_t and fill it with known values");
-    // Create an empty command then fill it with nonsense:
-    cmd = flux_cmd_create (0, NULL, NULL);
-    ok (cmd != NULL, "flux_cmd_create (0, NULL, NULL)");
-    check_empty_cmd_attributes (cmd);
-    set_cmd_attributes (cmd);
-
-    diag ("Ensure flux_cmd_t contains expected values and test interfaces");
-    // Check the nonsense
-    check_cmd_attributes (cmd);
-
-    set_cmd_attributes2 (cmd);
-
-    diag ("Ensure flux_cmd_t contains expected values again");
-    check_cmd_attributes (cmd);
-
-    // Test unsetenv with throwaway var
-    diag ("Test setenv/getenv/unsetenv");
-    ok (flux_cmd_setenvf (cmd, 1, "FOO", "%d", 42) >= 0,
-        "flux_cmd_setenvf (FOO=42)");
-    is (flux_cmd_getenv (cmd, "FOO"), "42",
-        "flux_cmd_getenv (FOO) == 42");
-    flux_cmd_unsetenv (cmd, "FOO");
-    ok (flux_cmd_getenv (cmd, "FOO") == NULL,
-        "flux_cmd_unsetenv works");
-
-    // Test env overwrite
-    ok (flux_cmd_setenvf (cmd, 0, "FOO", "%d", 42) >= 0,
-        "flux_cmd_setenvf (FOO=42)");
-    is (flux_cmd_getenv (cmd, "FOO"), "42",
-        "flux_cmd_getenv (FOO) == 42");
-    ok (flux_cmd_setenvf (cmd, 0, "FOO", "%d", 24) < 0,
-        "flux_cmd_setenvf (FOO=24) no overwrite fails");
-    ok (flux_cmd_setenvf (cmd, 1, "FOO", "%d", 24) >= 0,
-        "flux_cmd_setenvf (FOO=24, overwrite=true)");
-    is (flux_cmd_getenv (cmd, "FOO"), "24",
-        "flux_cmd_getenv (FOO) == 24");
-    flux_cmd_unsetenv (cmd, "FOO");
-
-    // Test opt overwrite
-    ok (flux_cmd_setopt (cmd, "FOO", "BAR") >= 0,
-        "flux_cmd_setopt");
-    is (flux_cmd_getopt (cmd, "FOO"), "BAR",
-        "flux_cmd_getopt (cmd, 'FOO') == BAR");
-    ok (flux_cmd_setopt (cmd, "FOO", "BAZ") >= 0,
-        "flux_cmd_setopt");
-    is (flux_cmd_getopt (cmd, "FOO"), "BAZ",
-        "flux_cmd_getopt (cmd, 'FOO') == BAZ");
-
-    diag ("Copy a flux_cmd_t and and ensure it matches source cmd");
-    copy = flux_cmd_copy (cmd);
-    ok (copy != NULL, "flux_cmd_copy");
-    check_cmd_attributes (copy);
-    flux_cmd_destroy (copy);
-
-    diag ("Convert flux_cmd_t to/from JSON");
-    s = flux_cmd_tojson (cmd);
-    ok (s != NULL, "flux_cmd_tojson (%d bytes)", strlen (s));
-    if (s) {
-        json_error_t error;
-        diag (s);
-        copy = flux_cmd_fromjson (s, &error);
-        free (s);
-        ok (copy != NULL, "flux_cmd_fromjson returned a new cmd");
-        if (copy) {
-            check_cmd_attributes (copy);
-            flux_cmd_destroy (copy);
-        }
-        else
-            diag ("%d:%d: %s", error.line, error.column, error.text);
-    }
-    flux_cmd_destroy (cmd);
-
-    test_find_opts ();
-
-    done_testing ();
-    return 0;
-}
-
-/*
- * vi: ts=4 sw=4 expandtab
- */
diff --git a/src/common/libsubprocess/test/command.c b/src/common/libsubprocess/test/command.c
new file mode 100644
index 000000000000..27a6c8fe57ab
--- /dev/null
+++ b/src/common/libsubprocess/test/command.c
@@ -0,0 +1,467 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <assert.h>
+#include <jansson.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libsubprocess/command.h"
+#include "src/common/libsubprocess/command_private.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+
+/*
+ *  Check basic flux_cmd_create () with args
+ */
+void check_basic_create ()
+{
+    char **av;
+    char * argv[] = {
+        "test",
+        "--option=foo",
+        "bar",
+        NULL
+    };
+    int argc = ARRAY_SIZE (argv) - 1;
+    char * env[] = {
+        "FOO=bar",
+        "PATH=/bin",
+        NULL
+    };
+    flux_cmd_t *cmd;
+
+    diag ("simple flux_cmd_create (argc, argv, env)");
+    cmd = flux_cmd_create (argc, argv, env);
+    ok (cmd != NULL, "flux_cmd_create ()");
+    av = cmd_argv_expand (cmd);
+    ok (av != NULL, "cmd_argv_expand ()");
+    is (av[0], "test", "av[0] == test");
+    is (av[1], "--option=foo", "av[1] == --option=foo");
+    is (av[2], "bar", "av[2] == bar");
+    ok (av[3] == NULL, "av[3] == NULL");
+    free (av);
+    is (flux_cmd_getenv (cmd, "FOO"), "bar", "flux_cmd_getenv");
+    is (flux_cmd_getenv (cmd, "PATH"), "/bin", "flux_cmd_getenv");
+
+    flux_cmd_destroy (cmd);
+}
+
+
+void check_empty_cmd_attributes (flux_cmd_t *cmd)
+{
+    char **argv, **env;
+
+    ok (flux_cmd_argc (cmd) == 0, "flux_cmd_argc");
+
+    argv = cmd_argv_expand (cmd);
+    ok (argv != NULL, "cmd_argv_expand returned an argv");
+    ok (argv[0] == NULL, "argv is properly NULL terminated");
+    free (argv);
+
+    env = cmd_env_expand (cmd);
+    ok (env != NULL, "cmd_env_expand works");
+    ok (env[0] == NULL, "cmd_env_expand properly terminates env");
+    free (env);
+
+    ok (flux_cmd_getcwd (cmd) == NULL,
+        "flux_cmd_getcwd returns NULL");
+}
+
+/*
+ * Set some basic known cmd attributes for testing
+ */
+void set_cmd_attributes (flux_cmd_t *cmd)
+{
+    assert (flux_cmd_argc (cmd) == 0);
+
+    // Append to argv
+    ok (flux_cmd_argv_append (cmd, "command") >= 0,
+        "flux_cmd_argv_append");
+    ok (flux_cmd_argv_append (cmd, "foo") >= 0,
+        "flux_cmd_argv_append");
+    ok (flux_cmd_argv_appendf (cmd, "%s", "bar") >= 0,
+        "flux_cmd_argv_appendf");
+
+    // Test setenvf
+    ok (flux_cmd_setenvf (cmd, 0, "PATH", "/bin:/usr/bin") >= 0,
+        "flux_cmd_setenvf (PATH)");
+
+    ok (flux_cmd_setcwd (cmd, "/tmp") >= 0,
+        "flux_cmd_setcwd (/tmp)");
+    ok (flux_cmd_add_channel (cmd, "MY_FD") >= 0,
+        "flux_cmd_add_channel");
+    ok (flux_cmd_setopt (cmd, "OPTION", "VALUE") >= 0,
+        "flux_cmd_setopt");
+}
+
+/* set alternate way, to ensure alternate ways also work */
+void set_cmd_attributes2 (flux_cmd_t *cmd)
+{
+    char *env[] = { "PATH=/bin:/usr/bin", NULL };
+
+    ok (cmd_set_env (cmd, env) == 0,
+        "cmd_set_env");
+}
+
+void check_cmd_attributes (flux_cmd_t *cmd)
+{
+    char **argv, **env;
+    const char *arg = NULL;
+
+    ok (flux_cmd_argc (cmd) == 3, "flux_cmd_argc");
+
+    argv = cmd_argv_expand (cmd);
+    ok (argv != NULL, "cmd_argv_expand returned an argv");
+    ok (argv[3] == NULL, "argv is properly NULL terminated");
+    is (argv[0], "command", "argv[0] is correct");
+    is (argv[1], "foo", "argv[1] is correct");
+    is (argv[2], "bar", "argv[2] is correct");
+    free (argv);
+
+    ok (flux_cmd_arg (cmd, 3) == NULL
+        && errno == EINVAL,
+        "flux_cmd_arg returns EINVAL on bad range");
+    arg = flux_cmd_arg (cmd, 0);
+    ok (arg != NULL
+        && streq (arg, "command"),
+        "flux_cmd_arg returns correct argv[0]");
+    arg = flux_cmd_arg (cmd, 1);
+    ok (arg != NULL
+        && streq (arg, "foo"),
+        "flux_cmd_arg returns correct argv[1]");
+    arg = flux_cmd_arg (cmd, 2);
+    ok (arg != NULL
+        && streq (arg, "bar"),
+        "flux_cmd_arg returns correct argv[2]");
+
+    is (flux_cmd_getenv (cmd, "PATH"), "/bin:/usr/bin",
+        "flux_cmd_getenv");
+
+    env = cmd_env_expand (cmd);
+    ok (env != NULL, "cmd_env_expand works");
+    ok (env[1] == NULL, "cmd_env_expand properly terminates env");
+    is (env[0], "PATH=/bin:/usr/bin",
+        "first entry of env is as expected");
+    free (env);
+
+    is (flux_cmd_getcwd (cmd), "/tmp",
+        "flux_cmd_getcwd");
+    is (flux_cmd_getopt (cmd, "OPTION"), "VALUE",
+        "flux_cmd_getopt (cmd, 'OPTION') == VALUE");
+}
+
+void test_find_opts (void)
+{
+    flux_cmd_t *cmd;
+    const char *substrings1[] = { "FOO", NULL };
+    const char *substrings2[] = { "DUH", "BAZ", "UHH", NULL };
+    const char *substrings3[] = { "OOPS",  NULL };
+    const char *substrings4[] = { NULL };
+
+    cmd = flux_cmd_create (0, NULL, NULL);
+    ok (cmd != NULL,
+        "flux_cmd_create works");
+
+    ok (flux_cmd_setopt (cmd, "a_FOO", "val") == 0,
+        "flux_cmd_setopt works");
+    ok (flux_cmd_setopt (cmd, "a_BAR", "val") == 0,
+        "flux_cmd_setopt works");
+    ok (flux_cmd_setopt (cmd, "b_BAR", "val") == 0,
+        "flux_cmd_setopt works");
+    ok (flux_cmd_setopt (cmd, "a_BAZ", "val") == 0,
+        "flux_cmd_setopt works");
+    ok (flux_cmd_setopt (cmd, "b_BAZ", "val") == 0,
+        "flux_cmd_setopt works");
+
+    ok (cmd_find_opts (cmd, substrings1) == 1,
+        "cmd_find_opts finds substrings");
+
+    ok (cmd_find_opts (cmd, substrings2) == 1,
+        "cmd_find_opts finds substrings");
+
+    ok (cmd_find_opts (cmd, substrings3) == 0,
+        "cmd_find_opts doesn't find substrings");
+
+    ok (cmd_find_opts (cmd, substrings4) == 0,
+        "cmd_find_opts doesn't find substrings");
+
+    flux_cmd_destroy (cmd);
+}
+
+void test_stringify (void)
+{
+    flux_cmd_t *cmd;
+    char *s;
+    char * argv[] = {
+        "test",
+        "--option=foo",
+        "-c",
+        "5",
+        "bar",
+        NULL
+    };
+    int argc = ARRAY_SIZE (argv) - 1;
+    char * env[] = {
+        "FOO=bar",
+        "PATH=/bin",
+        NULL
+    };
+
+    ok ((cmd = flux_cmd_create (0, NULL, NULL)) != NULL,
+        "flux_cmd_create empty");
+
+    s = flux_cmd_stringify (cmd);
+    ok (s != NULL,
+        "flux_cmd_stringify on empty cmd works");
+    is (s, "",
+        "flux_cmd_stringify on empty cmd returns empty string");
+    free (s);
+    flux_cmd_destroy (cmd);
+
+   ok ((cmd = flux_cmd_create (argc, argv, env)) != NULL,
+        "flux_cmd_create");
+   if (!cmd)
+       BAIL_OUT ("flux_cmd_create failed");
+
+    s = flux_cmd_stringify (cmd);
+    ok (s != NULL,
+        "flux_cmd_stringify works");
+    is (s, "test --option=foo -c 5 bar",
+        "flux_cmd_stringify returns expected string");
+    free (s);
+    flux_cmd_destroy (cmd);
+}
+
+void test_arg_insert_delete (void)
+{
+    flux_cmd_t *cmd;
+    char **av;
+    char * argv[] = {
+        "test",
+        "--option=foo",
+        "-c",
+        "5",
+        "bar",
+        NULL
+    };
+    int argc = ARRAY_SIZE (argv) - 1;
+    char * env[] = {
+        "FOO=bar",
+        "PATH=/bin",
+        NULL
+    };
+
+    ok ((cmd = flux_cmd_create (0, NULL, NULL)) != NULL,
+        "flux_cmd_create empty");
+    ok (flux_cmd_argv_delete (cmd, 0) < 0 && errno == EINVAL,
+        "flux_cmd_delete 0 on empty cmd returns EINVAL");
+    ok (flux_cmd_argv_insert (cmd, 0, "foo") == 0,
+        "flux_cmd_insert (cmd, 0) inserts at front of empty cmd");
+    ok (flux_cmd_argc (cmd) == 1,
+        "flux_cmd argc == 1");
+    is (flux_cmd_arg (cmd, 0), "foo",
+        "flux_cmd_arg returns foo for arg0");
+    flux_cmd_destroy (cmd);
+
+
+    ok ((cmd = flux_cmd_create (argc, argv, env)) != NULL,
+        "flux_cmd_create");
+    if (!cmd)
+        BAIL_OUT ("flux_cmd_create failed");
+
+    ok (flux_cmd_argc (cmd) == argc,
+        "flux_cmd_argc == %d (expected %d)",
+        flux_cmd_argc (cmd), argc);
+
+    ok (flux_cmd_argv_delete (cmd, 10) < 0 && errno == EINVAL,
+        "flux_cmd_argv_delete returns EINVAL for invalid index");
+
+    ok (flux_cmd_argv_delete (cmd, 0) == 0,
+        "flux_cmd_argv_delete first entry");
+    ok (flux_cmd_argc (cmd) == argc - 1,
+        "flux_cmd_argc is now %d (expected %d)",
+        flux_cmd_argc (cmd), argc - 1);
+    ok (flux_cmd_argv_insert (cmd, 0, "inserted") == 0,
+        "flux_cmd_argv_insert (cmd, 0, inserted)");
+    ok (flux_cmd_argc (cmd) == argc,
+        "flux_cmd_argc is now %d (expected %d)",
+        flux_cmd_argc (cmd), argc);
+    is (flux_cmd_arg (cmd, 0), "inserted",
+        "first argument is now `inserted`");
+
+    ok (flux_cmd_argv_delete (cmd, 2) == 0,
+        "flux_cmd_argv_delete from middle of argv works");
+    ok (flux_cmd_argc (cmd) == argc - 1,
+        "flux_cmd_argc is now %d (expected %d)",
+        flux_cmd_argc (cmd), argc - 1);
+    ok (flux_cmd_argv_insert (cmd, 2, "-d") == 0,
+        "flux_cmd_argv_insert (cmd, 2, -d)");
+    is (flux_cmd_arg (cmd, 2), "-d",
+        "arg 3 is now `-d`");
+
+    av = cmd_argv_expand (cmd);
+    ok (av != NULL, "cmd_argv_expand ()");
+    is (av[0], "inserted", "av[0] == inserted");
+    is (av[1], "--option=foo", "av[1] == --option=foo");
+    is (av[2], "-d", "av[2] == -d");
+    is (av[3], "5", "av[3] == 5");
+    is (av[4], "bar", "av[4] == bar");
+    ok (av[5] == NULL, "av[5] == NULL");
+
+    free (av);
+    flux_cmd_destroy (cmd);
+}
+
+void test_env (void)
+{
+    flux_cmd_t *cmd;
+
+    if (!(cmd = flux_cmd_create (0, NULL, NULL)))
+        BAIL_OUT ("failed to create command object");
+
+    // Test unsetenv with throwaway var
+    diag ("Test setenv/getenv/unsetenv");
+    ok (flux_cmd_setenvf (cmd, 1, "FOO", "%d", 42) >= 0,
+        "flux_cmd_setenvf (FOO=42)");
+    is (flux_cmd_getenv (cmd, "FOO"), "42",
+        "flux_cmd_getenv (FOO) == 42");
+    flux_cmd_unsetenv (cmd, "FOO");
+    ok (flux_cmd_getenv (cmd, "FOO") == NULL,
+        "flux_cmd_unsetenv works");
+
+    // Test env overwrite
+    ok (flux_cmd_setenvf (cmd, 0, "FOO", "%d", 42) >= 0,
+        "flux_cmd_setenvf (FOO=42)");
+    is (flux_cmd_getenv (cmd, "FOO"), "42",
+        "flux_cmd_getenv (FOO) == 42");
+    ok (flux_cmd_setenvf (cmd, 0, "FOO", "%d", 24) == 0,
+        "flux_cmd_setenvf (FOO=24) no overwrite succeeds");
+    is (flux_cmd_getenv (cmd, "FOO"), "42",
+        "flux_cmd_getenv (FOO) == 42 (still)");
+    ok (flux_cmd_setenvf (cmd, 1, "FOO", "%d", 24) >= 0,
+        "flux_cmd_setenvf (FOO=24, overwrite=true)");
+    is (flux_cmd_getenv (cmd, "FOO"), "24",
+        "flux_cmd_getenv (FOO) == 24");
+    flux_cmd_unsetenv (cmd, "FOO");
+
+    flux_cmd_destroy (cmd);
+}
+
+void test_env_glob (void)
+{
+    flux_cmd_t *cmd;
+
+    if (!(cmd = flux_cmd_create (0, NULL, NULL)))
+        BAIL_OUT ("failed to create command object");
+    lives_ok ({flux_cmd_unsetenv (NULL, "FOO");},
+        "flux_cmd_unset (NULL, FOO) doesn't crash");
+    lives_ok ({flux_cmd_unsetenv (cmd, NULL);},
+        "flux_cmd_unset (cmd, NULL) doesn't crash");
+    lives_ok ({flux_cmd_unsetenv (cmd, "FOO");},
+        "flux_cmd_unset (cmd, FOO) doesn't crash on empty cmd env");
+    ok (flux_cmd_setenvf (cmd, 0, "NOMATCH_FOO", "%d", 1) >= 0,
+        "flux_cmd_setenvf (NOMATCH_FOO=1)");
+    ok (flux_cmd_setenvf (cmd, 0, "MATCH_FOO", "%d", 2) >= 0,
+        "flux_cmd_setenvf (MATCH_FOO=2)");
+    ok (flux_cmd_setenvf (cmd, 0, "NOMATCH_BAR", "%d", 3) >= 0,
+        "flux_cmd_setenvf (NOMATCH_BAR=3)");
+    ok (flux_cmd_setenvf (cmd, 0, "MATCH_BAR", "%d", 4) >= 0,
+        "flux_cmd_setenvf (MATCH_BAR=4)");
+    flux_cmd_unsetenv (cmd, "MATCH_*");
+    diag ("flux_cmd_unsetenv (MATCH_*)");
+    is (flux_cmd_getenv (cmd, "NOMATCH_FOO"), "1",
+        "flux_cmd_getenv (NOMATCH_FOO == 1");
+    ok (flux_cmd_getenv (cmd, "MATCH_FOO") == NULL,
+        "flux_cmd_getenv (MATCH_FOO == NULL)");
+    is (flux_cmd_getenv (cmd, "NOMATCH_BAR"), "3",
+        "flux_cmd_getenv (NOMATCH_BAR == 3");
+    ok (flux_cmd_getenv (cmd, "MATCH_BAR") == NULL,
+        "flux_cmd_getenv (MATCH_BAR == NULL)");
+    flux_cmd_destroy (cmd);
+}
+
+int main (int argc, char *argv[])
+{
+    json_t *o;
+    flux_cmd_t *cmd, *copy;
+
+    plan (NO_PLAN);
+
+    diag ("Basic flux_cmd_create");
+    check_basic_create ();
+
+    diag ("Create a flux_cmd_t and fill it with known values");
+    // Create an empty command then fill it with nonsense:
+    cmd = flux_cmd_create (0, NULL, NULL);
+    ok (cmd != NULL, "flux_cmd_create (0, NULL, NULL)");
+    check_empty_cmd_attributes (cmd);
+    set_cmd_attributes (cmd);
+
+    diag ("Ensure flux_cmd_t contains expected values and test interfaces");
+    // Check the nonsense
+    check_cmd_attributes (cmd);
+
+    set_cmd_attributes2 (cmd);
+
+    diag ("Ensure flux_cmd_t contains expected values again");
+    check_cmd_attributes (cmd);
+
+    // Test opt overwrite
+    ok (flux_cmd_setopt (cmd, "FOO", "BAR") >= 0,
+        "flux_cmd_setopt");
+    is (flux_cmd_getopt (cmd, "FOO"), "BAR",
+        "flux_cmd_getopt (cmd, 'FOO') == BAR");
+    ok (flux_cmd_setopt (cmd, "FOO", "BAZ") >= 0,
+        "flux_cmd_setopt");
+    is (flux_cmd_getopt (cmd, "FOO"), "BAZ",
+        "flux_cmd_getopt (cmd, 'FOO') == BAZ");
+
+    diag ("Copy a flux_cmd_t and and ensure it matches source cmd");
+    copy = flux_cmd_copy (cmd);
+    ok (copy != NULL, "flux_cmd_copy");
+    check_cmd_attributes (copy);
+    flux_cmd_destroy (copy);
+
+    diag ("Convert flux_cmd_t to/from JSON");
+    o = cmd_tojson (cmd);
+    ok (o != NULL, "cmd_tojson works");
+    if (o) {
+        json_error_t error;
+        copy = cmd_fromjson (o, &error);
+        json_decref (o);
+        ok (copy != NULL, "cmd_fromjson returned a new cmd");
+        if (copy) {
+            check_cmd_attributes (copy);
+            flux_cmd_destroy (copy);
+        }
+        else
+            diag ("%d:%d: %s", error.line, error.column, error.text);
+    }
+    flux_cmd_destroy (cmd);
+
+    test_env ();
+    test_env_glob ();
+
+    test_find_opts ();
+
+    test_arg_insert_delete ();
+
+    test_stringify ();
+
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libsubprocess/test/fbuf.c b/src/common/libsubprocess/test/fbuf.c
new file mode 100644
index 000000000000..319d9c45f41a
--- /dev/null
+++ b/src/common/libsubprocess/test/fbuf.c
@@ -0,0 +1,447 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+#include "src/common/libtap/tap.h"
+
+#include "fbuf.h"
+
+#define FBUF_TEST_MAXSIZE 1048576
+
+void basic (void)
+{
+    struct fbuf *fb;
+    int pipefds[2];
+    char buf[1024];
+    const char *ptr;
+    int len;
+
+    ok (pipe (pipefds) == 0,
+        "pipe succeeded");
+
+    ok ((fb = fbuf_create (FBUF_TEST_MAXSIZE)) != NULL,
+        "fbuf_create works");
+
+    ok (fbuf_size (fb) == FBUF_TEST_MAXSIZE,
+        "fbuf_size returns correct size");
+
+    ok (fbuf_bytes (fb) == 0,
+        "fbuf_bytes initially returns 0");
+
+    ok (fbuf_space (fb) == FBUF_TEST_MAXSIZE,
+        "fbuf_space initially returns FBUF_TEST_MAXSIZE");
+
+    /* write and read tests */
+
+    ok (fbuf_write (fb, "foo", 3) == 3,
+        "fbuf_write works");
+
+    ok (fbuf_bytes (fb) == 3,
+        "fbuf_bytes returns length of bytes written");
+
+    ok (fbuf_space (fb) == (FBUF_TEST_MAXSIZE - 3),
+        "fbuf_space returns length of space left");
+
+    ok ((ptr = fbuf_read (fb, 2, &len)) != NULL
+        && len == 2,
+        "fbuf_read with specific length works");
+
+    ok (!memcmp (ptr, "fo", 2),
+        "fbuf_read returns expected data");
+
+    ok (fbuf_bytes (fb) == 1,
+        "fbuf_bytes returns new length after read");
+
+    ok ((ptr = fbuf_read (fb, -1, &len)) != NULL
+        && len == 1,
+        "fbuf_read with length -1 works");
+
+    ok (!memcmp (ptr, "o", 1),
+        "fbuf_read returns expected data");
+
+    ok (fbuf_bytes (fb) == 0,
+        "fbuf_bytes returns 0 with all bytes read");
+
+    ok (fbuf_space (fb) == FBUF_TEST_MAXSIZE,
+        "fbuf_space initially returns FBUF_TEST_MAXSIZE");
+
+    /* read_line tests */
+
+    ok (fbuf_write (fb, "foo\n", 4) == 4,
+        "fbuf_write works");
+
+    ok (fbuf_bytes (fb) == 4,
+        "fbuf_bytes returns length of bytes written");
+
+    ok (fbuf_space (fb) == (FBUF_TEST_MAXSIZE - 4),
+        "fbuf_space returns length of space left");
+
+    ok ((ptr = fbuf_read_line (fb, &len)) != NULL
+        && len == 4,
+        "fbuf_read_line works");
+
+    ok (!memcmp (ptr, "foo\n", 4),
+        "fbuf_read_line returns expected data");
+
+    ok (fbuf_bytes (fb) == 0,
+        "fbuf_bytes returns 0 after read_line");
+
+    ok (fbuf_space (fb) == FBUF_TEST_MAXSIZE,
+        "fbuf_space initially returns FBUF_TEST_MAXSIZE");
+
+    /* read_to_fd tests */
+
+    ok (fbuf_write (fb, "foo", 3) == 3,
+        "fbuf_write works");
+
+    ok (fbuf_read_to_fd (fb, pipefds[1], 2) == 2,
+        "fbuf_read_to_fd specific length works");
+
+    memset (buf, '\0', 1024);
+    ok (read (pipefds[0], buf, 1024) == 2,
+        "read correct number of bytes");
+
+    ok (memcmp (buf, "fo", 2) == 0,
+        "read returned correct data");
+
+    ok (fbuf_bytes (fb) == 1,
+        "fbuf_bytes returns correct length after read");
+
+    ok (fbuf_read_to_fd (fb, pipefds[1], -1) == 1,
+        "fbuf_read_to_fd length -1 works");
+
+    memset (buf, '\0', 1024);
+    ok (read (pipefds[0], buf, 1024) == 1,
+        "read correct number of bytes");
+
+    ok (memcmp (buf, "o", 1) == 0,
+        "read returned correct data");
+
+    ok (fbuf_bytes (fb) == 0,
+        "fbuf_bytes returns correct length after read");
+
+    /* write_from_fd and read tests */
+
+    ok (write (pipefds[1], "foo", 3) == 3,
+        "write to pipe works");
+
+    ok (fbuf_write_from_fd (fb, pipefds[0], -1) == 3,
+        "fbuf_write_from_fd works");
+
+    ok ((ptr = fbuf_read (fb, 2, &len)) != NULL
+        && len == 2,
+        "fbuf_read with specific length works");
+
+    ok (!memcmp (ptr, "fo", 2),
+        "fbuf_read returns expected data");
+
+    ok (fbuf_bytes (fb) == 1,
+        "fbuf_bytes returns new length after read");
+
+    ok ((ptr = fbuf_read (fb, -1, &len)) != NULL
+        && len == 1,
+        "fbuf_peek with length -1 works");
+
+    ok (!memcmp (ptr, "o", 1),
+        "fbuf_peek returns expected data");
+
+    ok (fbuf_bytes (fb) == 0,
+        "fbuf_bytes returns 0 with all bytes read");
+
+    fbuf_destroy (fb);
+    close (pipefds[0]);
+    close (pipefds[1]);
+}
+
+void notify_cb (struct fbuf *fb, void *arg)
+{
+    int *count = arg;
+    (*count)++;
+}
+
+void notify_callback (void)
+{
+    struct fbuf *fb;
+    int count;
+    int len;
+
+    ok ((fb = fbuf_create (16)) != NULL,
+        "fbuf_create 16 byte buffer works");
+    fbuf_set_notify (fb, notify_cb, &count);
+
+    count = 0;
+
+    ok (fbuf_write (fb, "foobar", 6) == 6,
+        "fbuf_write 6 bytes");
+
+    ok (count == 1,
+        "notify was called on transition from empty");
+
+    ok (fbuf_write (fb, "foo", 3) == 3,
+        "fbuf_write 3 bytes");
+
+    ok (count == 2,
+        "notify was called again");
+
+    ok (fbuf_write (fb, "1234567", 7) == 7,
+        "fbuf_write 7 bytes success");
+
+    ok (count == 3,
+        "notify was called again on transition to full");
+
+    ok (fbuf_read (fb, 1, &len) != NULL && len == 1,
+        "fbuf_read cleared one byte");
+
+    ok (count == 4,
+        "notify was called again on transition from full");
+
+    ok (fbuf_read (fb, -1, &len) != NULL && len == 15,
+        "fbuf_read cleared all data");
+
+    ok (count == 5,
+        "notify was called on transition to empty");
+
+    fbuf_destroy (fb);
+}
+
+void corner_case (void)
+{
+    struct fbuf *fb;
+    const char *ptr;
+    int len;
+
+    ok (fbuf_create (-1) == NULL
+        && errno == EINVAL,
+        "fbuf_create fails on bad input -1");
+
+    ok (fbuf_create (0) == NULL
+        && errno == EINVAL,
+        "fbuf_create fails on bad input 0");
+
+    /* all functions fail on NULL fb pointer */
+    ok (fbuf_size (NULL) < 0
+        && errno == EINVAL,
+        "fbuf_size fails on NULL pointer");
+    ok (fbuf_bytes (NULL) < 0
+        && errno == EINVAL,
+        "fbuf_bytes fails on NULL pointer");
+    ok (fbuf_space (NULL) < 0
+        && errno == EINVAL,
+        "fbuf_space fails on NULL pointer");
+    ok (fbuf_readonly (NULL) < 0
+        && errno == EINVAL,
+        "fbuf_readonly fails on NULL pointer");
+    errno = 0;
+    ok (!fbuf_is_readonly (NULL) && errno == EINVAL,
+        "fbuf_is_readonly returns false on NULL pointer");
+    ok (fbuf_read (NULL, -1, NULL) == NULL
+        && errno == EINVAL,
+        "fbuf_read fails on NULL pointer");
+    ok (fbuf_write (NULL, NULL, 0) < 0
+        && errno == EINVAL,
+        "fbuf_write fails on NULL pointer");
+    errno = 0;
+    ok (!fbuf_has_line (NULL)
+        && errno == EINVAL,
+        "fbuf_has_line returns false with errno set on NULL pointer");
+    ok (fbuf_read_line (NULL, NULL) == NULL
+        && errno == EINVAL,
+        "fbuf_read_line fails on NULL pointer");
+    ok (fbuf_read_trimmed_line (NULL, NULL) == NULL
+        && errno == EINVAL,
+        "fbuf_read_trimmed_line fails on NULL pointer");
+    ok (fbuf_read_to_fd (NULL, 0, 0) < 0
+        && errno == EINVAL,
+        "fbuf_read_to_fd fails on NULL pointer");
+    ok (fbuf_write_from_fd (NULL, 0, 0) < 0
+        && errno == EINVAL,
+        "fbuf_write_from_fd fails on NULL pointer");
+
+    ok ((fb = fbuf_create (FBUF_TEST_MAXSIZE)) != NULL,
+        "fbuf_create works");
+
+    ok ((ptr = fbuf_read (fb, -1, &len)) != NULL,
+        "fbuf_read works when no data available");
+    ok (len == 0,
+        "fbuf_read returns length 0 when no data available");
+
+    ok ((ptr = fbuf_read_line (fb, &len)) != NULL,
+        "fbuf_read_line works when no data available");
+    ok (len == 0,
+        "fbuf_read_line returns length 0 when no data available");
+    ok ((ptr = fbuf_read_trimmed_line (fb, &len)) != NULL,
+        "fbuf_read_trimmed_line works when no data available");
+    ok (len == 0,
+        "fbuf_read_trimmed_line returns length 0 when no data available");
+
+    /* write corner case tests */
+
+    ok (fbuf_write (fb, NULL, 0) < 0
+        && errno == EINVAL,
+        "fbuf_write fails on bad input");
+    ok (fbuf_write (fb, "foo", -1) < 0
+        && errno == EINVAL,
+        "fbuf_write fails on bad input");
+    ok (fbuf_write_from_fd (fb, -1, 0) < 0
+        && errno == EINVAL,
+        "fbuf_write_from_fd fails on bad input");
+
+    /* fbuf_destroy works with NULL */
+    fbuf_destroy (NULL);
+
+    fbuf_destroy (fb);
+}
+
+void full_buffer (void)
+{
+    struct fbuf *fb;
+    int len;
+
+    ok ((fb = fbuf_create (4)) != NULL,
+        "fbuf_create works");
+
+    ok (fbuf_write (fb, "1234", 4) == 4,
+        "fbuf_write success");
+
+    ok (fbuf_bytes (fb) == 4,
+        "fbuf_bytes returns length of bytes written");
+
+    ok (fbuf_space (fb) == 0,
+        "fbuf_space returns length of space left");
+
+    ok (fbuf_write (fb, "5", 1) < 0
+        && errno == ENOSPC,
+        "fbuf_write fails with ENOSPC if exceeding buffer size");
+
+    ok (fbuf_read (fb, -1, &len) != NULL && len == 4,
+        "fbuf_read drops all data");
+
+    fbuf_destroy (fb);
+}
+
+void readonly_buffer (void)
+{
+    struct fbuf *fb;
+    int pipefds[2];
+
+    ok (pipe (pipefds) == 0,
+        "pipe succeeded");
+
+    ok ((fb = fbuf_create (FBUF_TEST_MAXSIZE)) != NULL,
+        "fbuf_create works");
+
+    ok (!fbuf_is_readonly (fb),
+        "flux buffer is not readonly on creation");
+
+    ok (fbuf_readonly (fb) == 0,
+        "flux buffer readonly set");
+
+    ok (fbuf_is_readonly (fb),
+        "flux buffer is readonly after setting");
+
+    fbuf_destroy (fb);
+
+    ok ((fb = fbuf_create (FBUF_TEST_MAXSIZE)) != NULL,
+        "fbuf_create works");
+
+    ok (fbuf_write (fb, "foobar", 6) == 6,
+        "fbuf_write success");
+
+    ok (fbuf_readonly (fb) == 0,
+        "flux buffer readonly set");
+
+    ok (fbuf_write (fb, "foobar", 6) < 0
+        && errno == EROFS,
+        "fbuf_write fails b/c readonly is set");
+
+    ok (write (pipefds[1], "foo", 3) == 3,
+        "write to pipe works");
+
+    ok (fbuf_write_from_fd (fb, pipefds[0], -1) < 0
+        && errno == EROFS,
+        "fbuf_write_from_fd fails b/c readonly is set");
+
+    fbuf_destroy (fb);
+    close (pipefds[0]);
+    close (pipefds[1]);
+}
+
+/* tests to ensure internal buffers grow appropriately.  Current
+ * buffer default min is 4096, so we need to test > 4096 bytes of
+ * data.
+ */
+void large_data (void)
+{
+    struct fbuf *fb;
+    const char *data = "0123456789ABCDEF0123456789ABCDEF";
+    const char *ptr;
+    int len;
+    int i;
+
+    ok ((fb = fbuf_create (FBUF_TEST_MAXSIZE)) != NULL,
+        "fbuf_create works");
+
+    ok (fbuf_size (fb) == FBUF_TEST_MAXSIZE,
+        "fbuf_size returns correct size");
+
+    ok (fbuf_bytes (fb) == 0,
+        "fbuf_bytes initially returns 0");
+
+    ok (fbuf_space (fb) == FBUF_TEST_MAXSIZE,
+        "fbuf_space initially returns FBUF_TEST_MAXSIZE");
+
+    for (i = 0; i < 256; i++) {
+        if (fbuf_write (fb, data, 32) != 32)
+            ok (false, "fbuf_write fail: %s", strerror (errno));
+    }
+
+    ok (fbuf_space (fb) == (FBUF_TEST_MAXSIZE - 8192),
+        "fbuf_space returns length of space left");
+
+    ok ((ptr = fbuf_read (fb, -1, &len)) != NULL
+        && len == 8192,
+        "fbuf_read with length -1 works");
+
+    for (i = 0; i < 256; i++) {
+        if (memcmp (ptr + (i * 32), data, 32) != 0)
+            ok (false, "fbuf_read returned bad data");
+    }
+
+    fbuf_destroy (fb);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    basic ();
+    notify_callback ();
+    corner_case ();
+    full_buffer ();
+    readonly_buffer ();
+    large_data ();
+
+    done_testing();
+
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libsubprocess/test/fbuf_watcher.c b/src/common/libsubprocess/test/fbuf_watcher.c
new file mode 100644
index 000000000000..9fdf7ae982d2
--- /dev/null
+++ b/src/common/libsubprocess/test/fbuf_watcher.c
@@ -0,0 +1,1063 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/fdutils.h"
+#include "src/common/libtap/tap.h"
+
+#include "fbuf_watcher.h"
+
+static void buffer_read (flux_reactor_t *r, flux_watcher_t *w,
+                        int revents, void *arg)
+{
+    int *count = arg;
+
+    if (revents & FLUX_POLLERR) {
+        ok (false,
+            "buffer: read callback incorrectly called with FLUX_POLLERR");
+    }
+    else if (revents & FLUX_POLLIN) {
+        struct fbuf *fb = fbuf_read_watcher_get_buffer (w);
+        const void *ptr;
+        int len;
+
+        ok ((ptr = fbuf_read (fb, -1, &len)) != NULL,
+            "buffer: read from buffer success");
+
+        ok (len == 6,
+            "buffer: read returned correct length");
+
+        ok (!memcmp (ptr, "foobar", 6),
+            "buffer: read returned correct data");
+    }
+    else {
+        ok (false,
+            "buffer: read callback failed to return FLUX_POLLIN: %d", revents);
+    }
+
+    (*count)++;
+    flux_watcher_stop (w);
+    return;
+}
+
+static void buffer_read_data_unbuffered (flux_reactor_t *r,
+                                         flux_watcher_t *w,
+                                         int revents,
+                                         void *arg)
+{
+    int *count = arg;
+
+    if (revents & FLUX_POLLERR) {
+        ok (false,
+            "buffer: read callback incorrectly called with FLUX_POLLERR");
+    }
+    else if (revents & FLUX_POLLIN) {
+        const void *ptr;
+        int len;
+
+        ok ((ptr = fbuf_read_watcher_get_data (w, &len)) != NULL,
+            "buffer: read data from buffer success");
+
+        ok (len == 6,
+            "buffer: read data returned correct length");
+
+        ok (!memcmp (ptr, "foobar", 6),
+            "buffer: read data returned correct data");
+    }
+    else {
+        ok (false,
+            "buffer: read callback failed to return FLUX_POLLIN: %d", revents);
+    }
+
+    (*count)++;
+    flux_watcher_stop (w);
+    return;
+}
+
+
+static void buffer_read_line (flux_reactor_t *r, flux_watcher_t *w,
+                              int revents, void *arg)
+{
+    int *count = arg;
+
+    if (revents & FLUX_POLLERR) {
+        ok (false,
+            "buffer: read line callback incorrectly called with FLUX_POLLERR");
+    }
+    else if (revents & FLUX_POLLIN) {
+        struct fbuf *fb = fbuf_read_watcher_get_buffer (w);
+        const void *ptr;
+        int len;
+
+        ok ((ptr = fbuf_read_line (fb, &len)) != NULL,
+            "buffer: read line from buffer success");
+
+        ok (len == 4,
+            "buffer: read line returned correct length");
+
+        if ((*count) == 0) {
+            ok (!memcmp (ptr, "foo\n", 4),
+                "buffer: read line returned correct data");
+        }
+        else {
+            ok (!memcmp (ptr, "bar\n", 4),
+                "buffer: read line returned correct data");
+        }
+    }
+    else {
+        ok (false,
+            "buffer: read line callback failed to return FLUX_POLLIN: %d", revents);
+    }
+
+    (*count)++;
+    if ((*count) == 2)
+        flux_watcher_stop (w);
+    return;
+}
+
+static void buffer_read_data (flux_reactor_t *r, flux_watcher_t *w,
+                              int revents, void *arg)
+{
+    int *count = arg;
+
+    if (revents & FLUX_POLLERR) {
+        ok (false,
+            "buffer: read line callback incorrectly called with FLUX_POLLERR");
+    }
+    else if (revents & FLUX_POLLIN) {
+        const void *ptr;
+        int len;
+
+        ok ((ptr = fbuf_read_watcher_get_data (w, &len)) != NULL,
+            "buffer: read data from buffer success");
+
+        ok (len == 4,
+            "buffer: read data returned correct length");
+
+        if ((*count) == 0) {
+            ok (!memcmp (ptr, "foo\n", 4),
+                "buffer: read data returned correct data");
+        }
+        else {
+            ok (!memcmp (ptr, "bar\n", 4),
+                "buffer: read data returned correct data");
+        }
+    }
+    else {
+        ok (false,
+            "buffer: read line callback failed to return FLUX_POLLIN: %d", revents);
+    }
+
+    (*count)++;
+    if ((*count) == 2)
+        flux_watcher_stop (w);
+    return;
+}
+
+
+static void buffer_write (flux_reactor_t *r, flux_watcher_t *w,
+                          int revents, void *arg)
+{
+    int *count = arg;
+
+    if (revents & FLUX_POLLERR) {
+        ok (false,
+            "buffer: write callback called with FLUX_POLLERR");
+    }
+    else {
+        /* First callback is so user knows initial buffer size */
+        if ((*count) == 0) {
+            struct fbuf *fb = fbuf_write_watcher_get_buffer (w);
+            int space = fbuf_size (fb);
+            ok (space == 1024,
+                "buffer: write callback gets correct buffer size");
+        }
+        /* Second callback is when space is reclaimed */
+        else if ((*count) == 1) {
+            struct fbuf *fb = fbuf_write_watcher_get_buffer (w);
+            int space = fbuf_space (fb);
+            ok (space == 1024,
+                "buffer: write callback gets correct amount of space");
+        }
+        else {
+            ok (fbuf_write_watcher_is_closed (w, NULL),
+                "buffer: write callback called after close");
+        }
+    }
+
+    (*count)++;
+    if ((*count) == 1)
+        flux_watcher_stop (w);
+    return;
+}
+
+static void buffer_read_fill (flux_reactor_t *r, flux_watcher_t *w,
+                              int revents, void *arg)
+{
+    int *count = arg;
+
+    if (revents & FLUX_POLLERR) {
+        ok (false,
+            "buffer: read callback incorrectly called with FLUX_POLLERR");
+    }
+    else if (revents & FLUX_POLLIN) {
+        struct fbuf *fb = fbuf_read_watcher_get_buffer (w);
+        const void *ptr;
+        int len;
+
+        ok ((ptr = fbuf_read (fb, 6, &len)) != NULL,
+            "buffer: read from buffer success");
+
+        ok (len == 6,
+            "buffer: read returned correct length");
+
+        ok (!memcmp (ptr, "foobar", 6),
+            "buffer: read returned correct data");
+    }
+    else {
+        ok (false,
+            "buffer: read callback failed to return FLUX_POLLIN: %d", revents);
+    }
+
+    (*count)++;
+    if ((*count) == 3)
+        flux_watcher_stop (w);
+    return;
+}
+
+static void buffer_read_overflow (flux_reactor_t *r, flux_watcher_t *w,
+                                  int revents, void *arg)
+{
+    int *count = arg;
+
+    if (revents & FLUX_POLLERR) {
+        ok (false,
+            "buffer overflow test: read callback incorrectly called with FLUX_POLLERR");
+    }
+    else if (revents & FLUX_POLLIN) {
+        struct fbuf *fb = fbuf_read_watcher_get_buffer (w);
+        const void *ptr;
+        int len;
+
+        ok ((ptr = fbuf_read (fb, 6, &len)) != NULL,
+            "buffer overflow test: read from buffer success");
+
+        ok (len == 6,
+            "buffer overflow test: read returned correct length");
+
+        ok (!memcmp (ptr, "foobar", 6),
+            "buffer overflow test: read returned correct data");
+    }
+    else {
+        ok (false,
+            "buffer overflow test: read callback failed to return FLUX_POLLIN: %d", revents);
+    }
+
+    (*count)++;
+    if ((*count) == 3)
+        flux_watcher_stop (w);
+    return;
+}
+
+int create_socketpair_nonblock (int *fd)
+{
+#ifdef SOCK_NONBLOCK
+    if (socketpair (PF_LOCAL, SOCK_STREAM|SOCK_NONBLOCK, 0, fd) < 0)
+        return -1;
+#else
+    if (socketpair (PF_LOCAL, SOCK_STREAM, 0, fd) < 0
+        || fd_set_nonblocking (fd[0]) < 0
+        || fd_set_nonblocking (fd[1]) < 0)
+        return -1;
+#endif
+    return 0;
+}
+
+static void test_buffer (flux_reactor_t *reactor)
+{
+    int errnum = 0;
+    int fd[2];
+    int pfds[2];
+    flux_watcher_t *w;
+    struct fbuf *fb;
+    int count;
+    char buf[1024];
+
+    ok (create_socketpair_nonblock (fd) == 0,
+        "buffer: successfully created socketpair");
+
+    /* read buffer test */
+
+    count = 0;
+    w = fbuf_read_watcher_create (reactor,
+                                  fd[0],
+                                  1024,
+                                  buffer_read,
+                                  0,
+                                  &count);
+    ok (w != NULL,
+        "buffer: read created");
+
+    fb = fbuf_read_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer: buffer retrieved");
+
+    ok (write (fd[1], "foobar", 6) == 6,
+        "buffer: write to socketpair success");
+
+    flux_watcher_start (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer: reactor ran to completion");
+
+    ok (count == 1,
+        "buffer: read callback successfully called");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    /* read buffer test with fbuf_read_watcher_get_data() */
+
+    count = 0;
+    w = fbuf_read_watcher_create (reactor,
+                                  fd[0],
+                                  1024,
+                                  buffer_read_data_unbuffered,
+                                  0,
+                                  &count);
+    ok (w != NULL,
+        "buffer: read created");
+
+    fb = fbuf_read_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer: buffer retrieved");
+
+    ok (write (fd[1], "foobar", 6) == 6,
+        "buffer: write to socketpair success");
+
+    flux_watcher_start (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer: reactor ran to completion");
+
+    ok (count == 1,
+        "buffer: read callback successfully called");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+
+    /* read line buffer test */
+
+    count = 0;
+    w = fbuf_read_watcher_create (reactor,
+                                  fd[0],
+                                  1024,
+                                  buffer_read_line,
+                                  FBUF_WATCHER_LINE_BUFFER,
+                                  &count);
+    ok (w != NULL,
+        "buffer: read line created");
+
+    fb = fbuf_read_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer: buffer retrieved");
+
+    ok (write (fd[1], "foo\nbar\n", 8) == 8,
+        "buffer: write to socketpair success");
+
+    flux_watcher_start (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer: reactor ran to completion");
+
+    ok (count == 2,
+        "buffer: read line callback successfully called twice");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    /* read line with fbuf_read_watcher_get_data() */
+    count = 0;
+    w = fbuf_read_watcher_create (reactor,
+                                  fd[0],
+                                  1024,
+                                  buffer_read_data,
+                                  FBUF_WATCHER_LINE_BUFFER,
+                                  &count);
+    ok (w != NULL,
+        "buffer: read line created");
+
+    fb = fbuf_read_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer: buffer retrieved");
+
+    ok (write (fd[1], "foo\nbar\n", 8) == 8,
+        "buffer: write to socketpair success");
+
+    flux_watcher_start (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer: reactor ran to completion");
+
+    ok (count == 2,
+        "buffer: read line callback successfully called twice");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+
+    /* write buffer test */
+
+    count = 0;
+    w = fbuf_write_watcher_create (reactor,
+                                   fd[0],
+                                   1024,
+                                   buffer_write,
+                                   0,
+                                   &count);
+    ok (w != NULL,
+        "buffer: write created");
+
+    ok (flux_watcher_is_active (w) == false,
+        "flux_watcher_is_active() returns false on write buffer after create");
+
+    fb = fbuf_write_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer: buffer retrieved");
+
+    flux_watcher_start (w);
+
+    ok (flux_watcher_is_active (w) == true,
+        "flux_watcher_is_active() returns true on write buffer after start");
+
+    ok (fbuf_write (fb, "bazbar", 6) == 6,
+        "buffer: write to buffer success");
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer: reactor ran to completion");
+
+    ok (count == 2,
+        "buffer: write callback called 2 times");
+
+    ok (read (fd[1], buf, 1024) == 6,
+        "buffer: read from socketpair success");
+
+    ok (!memcmp (buf, "bazbar", 6),
+        "buffer: read from socketpair returned correct data");
+
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    /* write buffer test, write to buffer before start */
+
+    count = 0;
+    w = fbuf_write_watcher_create (reactor,
+                                   fd[0],
+                                   1024,
+                                   buffer_write,
+                                   0,
+                                   &count);
+    ok (w != NULL,
+        "buffer: write created");
+
+    fb = fbuf_write_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer: buffer retrieved");
+
+    ok (fbuf_write (fb, "foobaz", 6) == 6,
+        "buffer: write to buffer success");
+
+    flux_watcher_start (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer: reactor ran to completion");
+
+    ok (count == 2,
+        "buffer: write callback called 2 times");
+
+    ok (read (fd[1], buf, 1024) == 6,
+        "buffer: read from socketpair success");
+
+    ok (!memcmp (buf, "foobaz", 6),
+        "buffer: read from socketpair returned correct data");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    /* read buffer test, fill buffer before start */
+
+    count = 0;
+    w = fbuf_read_watcher_create (reactor,
+                                  fd[0],
+                                  12, /* 12 bytes = 2 "foobars"s */
+                                  buffer_read_fill,
+                                  0,
+                                  &count);
+    ok (w != NULL,
+        "buffer: read created");
+
+    fb = fbuf_read_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer: buffer retrieved");
+
+    ok (fbuf_write (fb, "foobarfoobar", 12) == 12,
+        "buffer: write to buffer success");
+
+    ok (write (fd[1], "foobar", 6) == 6,
+        "buffer: write to socketpair success");
+
+    flux_watcher_start (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer: reactor ran to completion");
+
+    ok (count == 3,
+        "buffer: read callback successfully called 3 times");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    /* read line buffer corner case test - fill buffer to max still works */
+
+    count = 0;
+    w = fbuf_read_watcher_create (reactor,
+                                  fd[0],
+                                  12, /* 12 bytes = 2 "foobar"s */
+                                  buffer_read_overflow,
+                                  0,
+                                  &count);
+    ok (w != NULL,
+        "buffer overflow test: read line created");
+
+    fb = fbuf_read_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer overflow test: buffer retrieved");
+
+    ok (write (fd[1], "foobarfoobarfoobar", 18) == 18,
+        "buffer overflow test: write to socketpair success");
+
+    flux_watcher_start (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer overflow test: reactor ran to completion");
+
+    ok (count == 3,
+        "buffer overflow test: read line callback successfully called three times");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    /* write buffer watcher close() testcase */
+
+    ok (fbuf_write_watcher_close (NULL) == -1 && errno == EINVAL,
+        "buffer: fbuf_write_watcher_close handles NULL argument");
+
+    count = 0;
+    ok (pipe (pfds) == 0,
+        "buffer: hey I can has a pipe!");
+
+    w = fbuf_write_watcher_create (reactor,
+                                   pfds[1],
+                                   1024,
+                                   buffer_write,
+                                   0,
+                                   &count);
+    ok (w == NULL && errno == EINVAL,
+        "buffer: write_watcher_create fails with EINVAL if fd !nonblocking");
+
+    ok (fd_set_nonblocking (pfds[1]) >= 0,
+        "buffer: fd_set_nonblocking");
+
+    w = fbuf_write_watcher_create (reactor,
+                                   pfds[1],
+                                   1024,
+                                   buffer_write,
+                                   0,
+                                   &count);
+    ok (w != NULL,
+        "buffer: write watcher close: watcher created");
+    fb = fbuf_write_watcher_get_buffer (w);
+    ok (fb != NULL,
+        "buffer: write watcher close: buffer retrieved");
+
+    ok (fbuf_write (fb, "foobaz", 6) == 6,
+        "buffer: write to buffer success");
+
+    ok (fbuf_write_watcher_is_closed (w, NULL) == 0,
+        "buffer: fbuf_write_watcher_is_closed returns false");
+    ok (fbuf_write_watcher_close (w) == 0,
+        "buffer: fbuf_write_watcher_close: Success");
+    ok (fbuf_write_watcher_is_closed (w, NULL) == 0,
+        "buffer: watcher still not closed (close(2) not called yet)");
+    ok (fbuf_write_watcher_close (w) == -1 && errno == EINPROGRESS,
+        "buffer: fbuf_write_watcher_close: In progress");
+
+    ok (fbuf_write (fb, "shouldfail", 10) == -1 && errno == EROFS,
+        "buffer: fbuf_write after close fails with EROFS");
+
+    flux_watcher_start (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer: reactor ran to completion");
+
+    ok (count == 3,
+        "buffer: write callback called 3 times");
+    ok (fbuf_write_watcher_is_closed (w, &errnum) == 1 && errnum == 0,
+        "buffer: fbuf_write_watcher_is_closed returns true");
+    ok (fbuf_write_watcher_close (w) == -1 && errno == EINVAL,
+        "buffer: fbuf_write_watcher_close after close returns EINVAL");
+
+    ok (read (pfds[0], buf, 1024) == 6,
+        "buffer: read from pipe success");
+
+    ok (!memcmp (buf, "foobaz", 6),
+        "buffer: read from pipe returned correct data");
+
+    ok (read (pfds[0], buf, 1024) == 0,
+        "buffer: read from pipe got EOF");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    close (pfds[0]);
+    close (fd[0]);
+    close (fd[1]);
+}
+
+struct buffer_fd_close
+{
+    flux_watcher_t *w;
+    int count;
+    int fd;
+};
+
+static void buffer_decref (flux_reactor_t *r,
+                           flux_watcher_t *w,
+                           int revents,
+                           void *arg)
+{
+    struct buffer_fd_close *bfc = arg;
+    bfc->count++;
+    fbuf_read_watcher_decref (bfc->w);
+    ok (true, "fbuf_read_watcher_decref");
+    flux_watcher_destroy (w);
+}
+
+static void buffer_read_fd_decref (flux_reactor_t *r,
+                                   flux_watcher_t *w,
+                                   int revents,
+                                   void *arg)
+{
+    struct buffer_fd_close *bfc = arg;
+    struct fbuf *fb;
+    const void *ptr;
+    int len;
+
+    if (revents & FLUX_POLLERR) {
+        fail ("buffer decref: got FLUX_POLLERR");
+        return;
+    }
+    if (!(revents & FLUX_POLLIN)) {
+        fail ("buffer decref: got FLUX_POLLERR");
+        return;
+    }
+
+    fb = fbuf_read_watcher_get_buffer (w);
+    ok ((ptr = fbuf_read (fb, -1, &len)) != NULL,
+        "buffer decref: read from buffer success");
+    if (!bfc->count) {
+        flux_watcher_t *w;
+        ok (len == 6,
+            "buffer decref: read returned correct length");
+        ok (!memcmp (ptr, "foobar", 6),
+            "buffer decref: read returned correct data");
+        diag ("closing write side of read buffer");
+        close (bfc->fd);
+
+        /* Schedule decref of read buffer
+         */
+        w = flux_timer_watcher_create (r, 0.01, 0., buffer_decref, bfc);
+        flux_watcher_start (w);
+    }
+    else {
+        ok (bfc->count == 2,
+            "buffer decref: EOF called only after manual decref");
+        ok ((ptr = fbuf_read (fb, -1, &len)) != NULL,
+            "buffer decref: read from buffer success");
+
+        ok (len == 0,
+            "buffer decref: read returned 0, socketpair is closed");
+        flux_watcher_stop (w);
+    }
+    bfc->count++;
+}
+
+static void buffer_read_fd_close (flux_reactor_t *r, flux_watcher_t *w,
+                                  int revents, void *arg)
+{
+    struct buffer_fd_close *bfc = arg;
+
+    if (revents & FLUX_POLLERR) {
+        ok (false,
+            "buffer corner case: read callback incorrectly called with FLUX_POLLERR");
+    }
+    else if (revents & FLUX_POLLIN) {
+        struct fbuf *fb = fbuf_read_watcher_get_buffer (w);
+        const void *ptr;
+        int len;
+
+        if (!bfc->count) {
+            ok ((ptr = fbuf_read (fb, -1, &len)) != NULL,
+                "buffer corner case: read from buffer success");
+
+            ok (len == 6,
+                "buffer corner case: read returned correct length");
+
+            ok (!memcmp (ptr, "foobar", 6),
+                "buffer corner case: read returned correct data");
+
+            close (bfc->fd);
+        }
+        else {
+            ok ((ptr = fbuf_read (fb, -1, &len)) != NULL,
+                "buffer corner case: read from buffer success");
+
+            ok (len == 0,
+                "buffer corner case: read returned 0, socketpair is closed");
+        }
+    }
+    else {
+        ok (false,
+            "buffer corner case: read callback failed to return FLUX_POLLIN: %d", revents);
+    }
+
+    bfc->count++;
+    if (bfc->count == 2)
+        flux_watcher_stop (w);
+    return;
+}
+
+static void buffer_read_line_fd_close (flux_reactor_t *r, flux_watcher_t *w,
+                                       int revents, void *arg)
+{
+    struct buffer_fd_close *bfc = arg;
+
+    if (revents & FLUX_POLLERR) {
+        ok (false,
+            "buffer corner case: read line callback incorrectly called with FLUX_POLLERR");
+    }
+    else if (revents & FLUX_POLLIN) {
+        struct fbuf *fb = fbuf_read_watcher_get_buffer (w);
+        const void *ptr;
+        int len;
+
+        if (!bfc->count) {
+            ok ((ptr = fbuf_read_line (fb, &len)) != NULL,
+                "buffer corner case: read line from buffer success");
+
+            ok (len == 7,
+                "buffer corner case: read line returned correct length");
+
+            ok (!memcmp (ptr, "foobar\n", 7),
+                "buffer corner case: read line returned correct data");
+
+            close (bfc->fd);
+        }
+        else {
+            ok ((ptr = fbuf_read_line (fb, &len)) != NULL,
+                "buffer corner case: read line from buffer success");
+
+            ok (len == 0,
+                "buffer corner case: read line returned 0, socketpair is closed");
+        }
+    }
+    else {
+        ok (false,
+            "buffer corner case: read line callback failed to return FLUX_POLLIN: %d", revents);
+    }
+
+    bfc->count++;
+    if (bfc->count == 2)
+        flux_watcher_stop (w);
+    return;
+}
+
+static void buffer_read_line_fd_close_and_left_over_data (flux_reactor_t *r,
+                                                          flux_watcher_t *w,
+                                                          int revents,
+                                                          void *arg)
+{
+    struct buffer_fd_close *bfc = arg;
+
+    if (revents & FLUX_POLLERR) {
+        ok (false,
+            "buffer corner case: read line callback incorrectly called with FLUX_POLLERR");
+    }
+    else if (revents & FLUX_POLLIN) {
+        struct fbuf *fb = fbuf_read_watcher_get_buffer (w);
+        const void *ptr;
+        int len;
+
+        if (!bfc->count) {
+            ok ((ptr = fbuf_read_line (fb, &len)) != NULL,
+                "buffer corner case: read line from buffer success");
+
+            ok (len == 7,
+                "buffer corner case: read line returned correct length");
+
+            ok (!memcmp (ptr, "foobar\n", 7),
+                "buffer corner case: read line returned correct data");
+
+            close (bfc->fd);
+        }
+        else if (bfc->count == 1) {
+            ok ((ptr = fbuf_read_line (fb, &len)) != NULL,
+                "buffer corner case: read line from buffer success");
+
+            ok (len == 0,
+                "buffer corner case: read line says no lines available");
+
+            ok ((ptr = fbuf_read (fb, -1, &len)) != NULL,
+                "buffer corner case: read from buffer success");
+
+            ok (len == 3,
+                "buffer corner case: read line returned correct length");
+
+            ok (!memcmp (ptr, "foo", 3),
+                "buffer corner case: read line returned correct data");
+        }
+        else {
+            ok ((ptr = fbuf_read_line (fb, &len)) != NULL,
+                "buffer corner case: read line from buffer success");
+
+            ok (len == 0,
+                "buffer corner case: read line returned 0, socketpair is closed");
+        }
+    }
+    else {
+        ok (false,
+            "buffer corner case: read line callback failed to return FLUX_POLLIN: %d", revents);
+    }
+
+    bfc->count++;
+    if (bfc->count == 3)
+        flux_watcher_stop (w);
+    return;
+}
+
+static void test_buffer_refcnt (flux_reactor_t *reactor)
+{
+    int fd[2];
+    flux_watcher_t *w;
+    struct buffer_fd_close bfc;
+
+    /* read buffer decref test - other end closes stream */
+
+    ok (create_socketpair_nonblock (fd) == 0,
+        "buffer decref: successfully created socketpair");
+
+    bfc.count = 0;
+    bfc.fd = fd[1];
+    w = fbuf_read_watcher_create (reactor,
+                                  fd[0],
+                                  1024,
+                                  buffer_read_fd_decref,
+                                  0,
+                                  &bfc);
+    ok (w != NULL,
+        "buffer decref: read created");
+    bfc.w = w;
+
+    ok (write (fd[1], "foobar", 6) == 6,
+        "buffer decref: write to socketpair success");
+
+    flux_watcher_start (w);
+
+    diag ("calling fbuf_read_watcher_incref");
+    fbuf_read_watcher_incref (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer decref: reactor ran to completion");
+
+    ok (bfc.count == 3,
+        "buffer decref: read callback successfully called thrice");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    close (fd[0]);
+}
+
+static void test_buffer_corner_case (flux_reactor_t *reactor)
+{
+    int fd[2];
+    flux_watcher_t *w;
+    struct fbuf *fb;
+    struct buffer_fd_close bfc;
+
+    /* read buffer corner case test - other end closes stream */
+
+    ok (create_socketpair_nonblock (fd) == 0,
+        "buffer corner case: successfully created socketpair");
+
+    bfc.count = 0;
+    bfc.fd = fd[1];
+    w = fbuf_read_watcher_create (reactor,
+                                  fd[0],
+                                  1024,
+                                  buffer_read_fd_close,
+                                  0,
+                                  &bfc);
+    ok (w != NULL,
+        "buffer corner case: read created");
+
+    ok (flux_watcher_is_active (w) == false,
+        "flux_watcher_is_active() returns false on read buffer after create");
+
+    fb = fbuf_read_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer corner case: buffer retrieved");
+
+    ok (write (fd[1], "foobar", 6) == 6,
+        "buffer corner case: write to socketpair success");
+
+    flux_watcher_start (w);
+    ok (flux_watcher_is_active (w) == true,
+        "flux_watcher_is_active() returns true on read buffer after start");
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer corner case: reactor ran to completion");
+
+    ok (bfc.count == 2,
+        "buffer corner case: read callback successfully called twice");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    close (fd[0]);
+
+    /* read line buffer corner case test - other end closes stream */
+
+    ok (create_socketpair_nonblock (fd) == 0,
+        "buffer corner case: successfully created socketpair");
+
+    bfc.count = 0;
+    bfc.fd = fd[1];
+    w = fbuf_read_watcher_create (reactor,
+                                  fd[0],
+                                  1024,
+                                  buffer_read_line_fd_close,
+                                  FBUF_WATCHER_LINE_BUFFER,
+                                  &bfc);
+    ok (w != NULL,
+        "buffer corner case: read line created");
+
+    fb = fbuf_read_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer corner case: buffer retrieved");
+
+    ok (write (fd[1], "foobar\n", 7) == 7,
+        "buffer corner case: write to socketpair success");
+
+    flux_watcher_start (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer corner case: reactor ran to completion");
+
+    ok (bfc.count == 2,
+        "buffer corner case: read line callback successfully called twice");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    close (fd[0]);
+
+    /* read line buffer corner case test - left over data not a line */
+
+    ok (create_socketpair_nonblock (fd) == 0,
+        "buffer corner case: successfully created socketpair");
+
+    bfc.count = 0;
+    bfc.fd = fd[1];
+    w = fbuf_read_watcher_create (reactor,
+                                  fd[0],
+                                  1024,
+                                  buffer_read_line_fd_close_and_left_over_data,
+                                  FBUF_WATCHER_LINE_BUFFER,
+                                  &bfc);
+    ok (w != NULL,
+        "buffer corner case: read line created");
+
+    fb = fbuf_read_watcher_get_buffer (w);
+
+    ok (fb != NULL,
+        "buffer corner case: buffer retrieved");
+
+    ok (write (fd[1], "foobar\nfoo", 10) == 10,
+        "buffer corner case: write to socketpair success");
+
+    flux_watcher_start (w);
+
+    ok (flux_reactor_run (reactor, 0) == 0,
+        "buffer corner case: reactor ran to completion");
+
+    ok (bfc.count == 3,
+        "buffer corner case: read line callback successfully called three times");
+
+    flux_watcher_stop (w);
+    flux_watcher_destroy (w);
+
+    close (fd[0]);
+    close (fd[1]);
+}
+
+int main (int argc, char *argv[])
+{
+    flux_reactor_t *reactor;
+
+    plan (NO_PLAN);
+
+    ok ((reactor = flux_reactor_create (0)) != NULL,
+        "created reactor");
+    if (!reactor)
+        BAIL_OUT ("can't continue without reactor");
+
+    test_buffer (reactor);
+    test_buffer_refcnt (reactor);
+    test_buffer_corner_case (reactor);
+
+    flux_reactor_destroy (reactor);
+
+    done_testing();
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libsubprocess/test/fdcopy.c b/src/common/libsubprocess/test/fdcopy.c
new file mode 100644
index 000000000000..8b8e0ffe4bd4
--- /dev/null
+++ b/src/common/libsubprocess/test/fdcopy.c
@@ -0,0 +1,84 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* fdcopy.c - copy one file descriptor to another */
+
+#if HAVE_CONFIG_H
+# include "config.h"
+#endif
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+
+int getenv_int (const char *name, int *valp)
+{
+    const char *s;
+    int val = -1;
+
+    if (!(s = getenv (name)))
+        return -1;
+    errno = 0;
+    val = strtoul (s, NULL, 10);
+    if (errno != 0)
+        return -1;
+    *valp = val;
+    return 0;
+}
+
+void fdcopy (int fd_in, int fd_out)
+{
+    int rlen;
+    int wlen;
+    char buf[1024];
+
+    for (;;) {
+        rlen = read (fd_in, buf, sizeof (buf));
+        if (rlen < 0) {
+            perror ("read");
+            exit (1);
+        }
+        if (rlen == 0)
+            break;
+        wlen = 0;
+        while (wlen < rlen) {
+            int n = write (fd_out, buf + wlen, rlen - wlen);
+            if (n < 0) {
+                perror ("write");
+                exit (1);
+            }
+            wlen += n;
+        }
+    }
+}
+
+int main (int argc, char **argv)
+{
+    int fd_in;
+    int fd_out;
+
+    if (argc != 3
+        || getenv_int (argv[1], &fd_in) < 0
+        || getenv_int (argv[2], &fd_out) < 0) {
+        fprintf (stderr,
+                 "Usage: fdcopy INCHAN OUTCHAN\n"
+                 "  where *CHAN are env vars set to file descriptor numbers\n");
+        exit (1);
+    }
+
+    fdcopy (fd_in, fd_out);
+
+    exit (0);
+}
+
+
+// vi: ts=4 sw=4 expandtab
+
diff --git a/src/common/libsubprocess/test/iochan.c b/src/common/libsubprocess/test/iochan.c
new file mode 100644
index 000000000000..8227fac31cf4
--- /dev/null
+++ b/src/common/libsubprocess/test/iochan.c
@@ -0,0 +1,277 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h> // environ def
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "src/common/libtap/tap.h"
+#include "src/common/libtestutil/util.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libsubprocess/server.h"
+#include "src/common/libsubprocess/subprocess_private.h"
+#include "src/common/libioencode/ioencode.h"
+#include "src/common/libutil/stdlog.h"
+
+#include "rcmdsrv.h"
+
+struct iochan_ctx {
+    flux_t *h;
+    flux_subprocess_t *p;
+    flux_watcher_t *source;
+    flux_watcher_t *timer;
+    pid_t pid;
+    int recvcount;
+    int sendcount;
+    int count;
+    int refcount;
+    const char *name;
+};
+
+extern char **environ;
+
+const int linesize = 80;
+const char *test_fdcopy = TEST_SUBPROCESS_DIR "test_fdcopy";
+
+static void iochan_timer_cb (flux_reactor_t *r,
+                               flux_watcher_t *w,
+                               int revents,
+                               void *arg)
+{
+    diag ("doomsday has arrived");
+    flux_reactor_stop_error (r);
+}
+
+static void iochan_start_doomsday (struct iochan_ctx *ctx, double t)
+{
+    flux_timer_watcher_reset (ctx->timer, t, 0.);
+    flux_watcher_start (ctx->timer);
+}
+
+static void iochan_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    struct iochan_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+    const char *line;
+    int len;
+
+    if ((len = flux_subprocess_read_line (p, stream, &line)) < 0)
+        diag ("%s: %s", stream, strerror (errno));
+    else if (len == 0)
+        diag ("%s: EOF", stream);
+    else if (streq (stream, "stderr") || streq (stream, "stdout"))
+        diag ("%s: %s", stream, line);
+    else if (streq (stream, "IOCHAN_OUT"))
+        ctx->recvcount += len;
+}
+
+static void iochan_completion_cb (flux_subprocess_t *p)
+{
+    struct iochan_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+
+    diag ("%s: completion callback", ctx->name);
+
+    diag ("%s: stopping reactor", ctx->name);
+    flux_reactor_stop (flux_get_reactor (ctx->h));
+}
+
+static void iochan_state_cb (flux_subprocess_t *p,
+                               flux_subprocess_state_t state)
+{
+    struct iochan_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+
+    diag ("%s state callback state=%s",
+          ctx->name,
+          flux_subprocess_state_string (state));
+
+    switch (state) {
+        case FLUX_SUBPROCESS_INIT:
+        case FLUX_SUBPROCESS_RUNNING:
+            ctx->pid = flux_subprocess_pid (p);
+            flux_watcher_start (ctx->source); // start sourcing data
+            break;
+        case FLUX_SUBPROCESS_STOPPED:
+            break;
+        case FLUX_SUBPROCESS_EXITED: {
+            int status = flux_subprocess_status (p);
+            if (WIFEXITED (status))
+                diag ("%s: exit %d", ctx->name, WEXITSTATUS (status));
+            else if (WIFSIGNALED (status))
+                diag ("%s: %s", ctx->name, strsignal (WTERMSIG (status)));
+            // completion callback will exit the reactor, but just in case
+            iochan_start_doomsday (ctx, 2.);
+            // if testing refcnt, release stdout reference now
+            if (streq (ctx->name, "refcnt")) {
+                ctx->refcount--;
+                flux_subprocess_channel_decref (ctx->p, "stdout");
+            }
+            break;
+        }
+        case FLUX_SUBPROCESS_FAILED:
+            diag ("%s: %s",
+                  ctx->name,
+                  strerror (flux_subprocess_fail_errno (p)));
+            diag ("%s: stopping reactor", ctx->name);
+            flux_reactor_stop_error (flux_get_reactor (ctx->h));
+            break;
+    }
+}
+
+static void iochan_source_cb (flux_reactor_t *r,
+                              flux_watcher_t *w,
+                              int revents,
+                              void *arg)
+{
+    struct iochan_ctx *ctx = arg;
+    char buf[linesize];
+    int len;
+    int n = linesize;
+
+    if (n > ctx->count - ctx->sendcount)
+        n = ctx->count - ctx->sendcount;
+
+    memset (buf, 'F', n - 1);
+    buf[n - 1] = '\n';
+
+    len = flux_subprocess_write (ctx->p, "IOCHAN_IN", buf, n);
+    if (len < 0) {
+        diag ("%s: source: %s", ctx->name, strerror (errno));
+        goto error;
+    }
+    if (len < n) {
+        diag ("%s: source: short write", ctx->name);
+        errno = ENOSPC;
+        goto error;
+    }
+    ctx->sendcount += len;
+    if (ctx->sendcount == ctx->count) {
+        if (flux_subprocess_close (ctx->p, "IOCHAN_IN") < 0) {
+            diag ("%s: source: %s", ctx->name, strerror (errno));
+            goto error;
+        }
+        flux_watcher_stop (w);
+    }
+    return;
+error:
+    //flux_reactor_stop_error (r);
+    iochan_start_doomsday (ctx, 2.);
+}
+
+flux_subprocess_ops_t iochan_ops = {
+    .on_completion      = iochan_completion_cb,
+    .on_state_change    = iochan_state_cb,
+    .on_stdout          = iochan_output_cb,
+    .on_stderr          = iochan_output_cb,
+    .on_channel_out     = iochan_output_cb,
+};
+
+bool iochan_run_check (flux_t *h, const char *name, int count)
+{
+    char *cat_av[] = {
+        (char *)test_fdcopy,
+        "IOCHAN_IN",
+        "IOCHAN_OUT",
+        NULL,
+    };
+    flux_cmd_t *cmd;
+    struct iochan_ctx ctx;
+    int rc;
+    bool ret = true;
+
+    memset (&ctx, 0, sizeof (ctx));
+    ctx.h = h;
+    ctx.count = count;
+    ctx.name = name;
+
+    if (!(cmd = flux_cmd_create (ARRAY_SIZE (cat_av) - 1, cat_av, environ)))
+        BAIL_OUT ("flux_cmd_create failed");
+    if (flux_cmd_add_channel (cmd, "IOCHAN_IN") < 0
+        || flux_cmd_add_channel (cmd, "IOCHAN_OUT") < 0)
+        BAIL_OUT ("flux_cmd_add_channel failed");
+    if (!(ctx.p = flux_rexec_ex (h,
+                                 "rexec",
+                                 FLUX_NODEID_ANY,
+                                 0,
+                                 cmd,
+                                 &iochan_ops,
+                                 tap_logger,
+                                 NULL)))
+        BAIL_OUT ("flux_rexec_ex failed");
+    if (streq (ctx.name, "refcnt")) {
+        ctx.refcount++;
+        flux_subprocess_channel_incref (ctx.p, "stdout");
+    }
+    if (flux_subprocess_aux_set (ctx.p, "ctx", &ctx, NULL) < 0)
+        BAIL_OUT ("flux_subprocess_aux_set failed");
+
+    if (!(ctx.source = flux_prepare_watcher_create (flux_get_reactor (h),
+                                                    iochan_source_cb,
+                                                    &ctx)))
+        BAIL_OUT ("could not create prepare watcher");
+    if (!(ctx.timer = flux_timer_watcher_create (flux_get_reactor (h),
+                                                 0.,
+                                                 0.,
+                                                 iochan_timer_cb,
+                                                 &ctx)))
+        BAIL_OUT ("could not create timer watcher");
+
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    if (rc < 0) {
+        diag ("%s: flux_reactor_run: %s", name, strerror (errno));
+        ret = false;
+    }
+
+    diag ("%s: processed %d of %d bytes", name, ctx.recvcount, ctx.sendcount);
+    if (ctx.recvcount < ctx.sendcount)
+        ret = false;
+
+    if (streq (ctx.name, "refcnt")) {
+        diag ("%s: final refcount: %d", ctx.name, ctx.refcount);
+        if (ctx.refcount != 0)
+            ret = false;
+    }
+
+    flux_watcher_destroy (ctx.source);
+    flux_watcher_destroy (ctx.timer);
+    diag ("%s: destroying subprocess", name);
+    flux_subprocess_destroy (ctx.p);
+    flux_cmd_destroy (cmd);
+
+    return ret;
+}
+
+int main (int argc, char *argv[])
+{
+    flux_t *h;
+
+    plan (NO_PLAN);
+
+    h = rcmdsrv_create ("rexec");
+
+    ok (iochan_run_check (h, "simple", linesize * 100),
+        "simple check worked");
+    ok (iochan_run_check (h, "simple", linesize * 1000),
+        "medium check worked");
+    ok (iochan_run_check (h, "simple", linesize * 10000),
+        "large check worked");
+    ok (iochan_run_check (h, "refcnt", linesize * 10),
+        "refcount check worked");
+    test_server_stop (h);
+    flux_close (h);
+
+    done_testing ();
+    return 0;
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/test/iostress.c b/src/common/libsubprocess/test/iostress.c
new file mode 100644
index 000000000000..3e3fb8a70fa5
--- /dev/null
+++ b/src/common/libsubprocess/test/iostress.c
@@ -0,0 +1,444 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h> // environ def
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "src/common/libtap/tap.h"
+#include "src/common/libtestutil/util.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libsubprocess/server.h"
+#include "src/common/libsubprocess/subprocess_private.h"
+#include "src/common/libioencode/ioencode.h"
+#include "src/common/libutil/stdlog.h"
+
+#include "rcmdsrv.h"
+
+enum {
+    WRITE_API,
+    WRITE_DIRECT,
+    WRITE_CREDIT,
+};
+
+struct iostress_ctx {
+    flux_t *h;
+    flux_subprocess_t *p;
+    flux_watcher_t *source;
+    flux_watcher_t *timer;
+    pid_t pid;
+    size_t linesize;
+    char *buf;
+    int buf_index;
+    int linerecv;
+    int batchcount;
+    int batchlines;
+    int batchcursor;
+    int batchlinescursor;
+    int outputcount;
+    int write_type;
+    int stdin_credits;
+    const char *name;
+};
+
+extern char **environ;
+
+static void iostress_timer_cb (flux_reactor_t *r,
+                               flux_watcher_t *w,
+                               int revents,
+                               void *arg)
+{
+    diag ("doomsday has arrived");
+    flux_reactor_stop_error (r);
+}
+
+static void iostress_start_doomsday (struct iostress_ctx *ctx, double t)
+{
+    flux_timer_watcher_reset (ctx->timer, t, 0.);
+    flux_watcher_start (ctx->timer);
+}
+
+static void iostress_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    struct iostress_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+    const char *line;
+    int len;
+
+    if ((len = flux_subprocess_read_line (p, stream, &line)) < 0)
+        diag ("%s: %s", stream, strerror (errno));
+    else if (len == 0)
+        diag ("%s: EOF", stream);
+    else {
+        // diag ("%s: %d bytes", stream, len);
+        if (strstr (line, "\n"))
+            ctx->linerecv++;
+    }
+    ctx->outputcount++;
+}
+
+static void iostress_completion_cb (flux_subprocess_t *p)
+{
+    struct iostress_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+
+    diag ("%s: completion callback", ctx->name);
+
+    diag ("%s: stopping reactor", ctx->name);
+    flux_reactor_stop (flux_get_reactor (ctx->h));
+}
+
+static void iostress_state_cb (flux_subprocess_t *p,
+                               flux_subprocess_state_t state)
+{
+    struct iostress_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+
+    diag ("%s state callback state=%s",
+          ctx->name,
+          flux_subprocess_state_string (state));
+
+    switch (state) {
+        case FLUX_SUBPROCESS_INIT:
+        case FLUX_SUBPROCESS_RUNNING:
+            ctx->pid = flux_subprocess_pid (p);
+            /* if credit based write, writing will be taken care of in
+             * iostress_credit_cb()
+             */
+            if (ctx->write_type == WRITE_API
+                || ctx->write_type == WRITE_DIRECT)
+                flux_watcher_start (ctx->source); // start sourcing data
+            break;
+        case FLUX_SUBPROCESS_STOPPED:
+            break;
+        case FLUX_SUBPROCESS_EXITED: {
+            int status = flux_subprocess_status (p);
+            if (WIFEXITED (status))
+                diag ("%s: exit %d", ctx->name, WEXITSTATUS (status));
+            else if (WIFSIGNALED (status))
+                diag ("%s: %s", ctx->name, strsignal (WTERMSIG (status)));
+            // completion callback will exit the reactor, but just in case
+            iostress_start_doomsday (ctx, 2.);
+            break;
+        }
+        case FLUX_SUBPROCESS_FAILED:
+            diag ("%s: %s",
+                  ctx->name,
+                  strerror (flux_subprocess_fail_errno (p)));
+            diag ("%s: stopping reactor", ctx->name);
+            flux_reactor_stop_error (flux_get_reactor (ctx->h));
+            break;
+    }
+}
+
+static int rexec_write (flux_t *h, uint32_t matchtag, const char *buf, int len)
+{
+    flux_future_t *f;
+    json_t *io;
+    bool eof = len > 0 ? false : true;
+
+    if (!(io = ioencode ("stdin", "0", buf, len, eof)))
+        return -1;
+    if (!(f = flux_rpc_pack (h,
+                             "rexec.write",
+                             0,
+                             FLUX_RPC_NORESPONSE,
+                             "{s:i s:O}",
+                             "matchtag", matchtag,
+                             "io", io))) {
+        json_decref (io);
+        return -1;
+    }
+    flux_future_destroy (f);
+    json_decref (io);
+    return 0;
+}
+
+static void iostress_source_cb (flux_reactor_t *r,
+                                flux_watcher_t *w,
+                                int revents,
+                                void *arg)
+{
+    struct iostress_ctx *ctx = arg;
+    uint32_t matchtag;
+
+    matchtag = flux_rpc_get_matchtag (ctx->p->f);
+
+    for (int i = 0; i < ctx->batchlines; i++) {
+        if (ctx->write_type == WRITE_DIRECT) {
+            if (rexec_write (ctx->h, matchtag, ctx->buf, ctx->linesize) < 0)
+                BAIL_OUT ("rexec_write failed");
+        }
+        else if (ctx->write_type == WRITE_API) {
+            int len;
+            len = flux_subprocess_write (ctx->p,
+                                         "stdin",
+                                         ctx->buf,
+                                         ctx->linesize);
+            if (len < 0) {
+                diag ("%s: source: %s", ctx->name, strerror (errno));
+                goto error;
+            }
+            if (len < ctx->linesize) {
+                diag ("%s: source: short write", ctx->name);
+                errno = ENOSPC;
+                goto error;
+            }
+        }
+    }
+    if (++ctx->batchcursor == ctx->batchcount) {
+        if (flux_subprocess_close (ctx->p, "stdin") < 0) {
+            diag ("%s: source: %s", ctx->name, strerror (errno));
+            goto error;
+        }
+        flux_watcher_stop (w);
+    }
+    return;
+error:
+    //flux_reactor_stop_error (r);
+    iostress_start_doomsday (ctx, 2.);
+}
+
+static void iostress_credit_cb (flux_subprocess_t *p,
+                                const char *stream,
+                                int bytes)
+{
+    struct iostress_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+
+    if (ctx->write_type != WRITE_CREDIT)
+        return;
+
+    // diag ("%s credit cb stream=%s bytes=%d", ctx->name, stream, bytes);
+
+    ctx->stdin_credits += bytes;
+
+    while (ctx->batchcursor < ctx->batchcount) {
+        while (ctx->batchlinescursor < ctx->batchlines) {
+            int wlen, len;
+
+            if ((ctx->linesize - ctx->buf_index) < ctx->stdin_credits)
+                wlen = ctx->linesize - ctx->buf_index;
+            else
+                wlen = ctx->stdin_credits;
+            len = flux_subprocess_write (ctx->p,
+                                         "stdin",
+                                         &ctx->buf[ctx->buf_index],
+                                         wlen);
+            if (len < 0) {
+                diag ("%s: source: %s", ctx->name, strerror (errno));
+                goto error;
+            }
+            ctx->stdin_credits -= len;
+            ctx->buf_index += len;
+            if (ctx->buf_index == ctx->linesize) {
+                ctx->buf_index = 0;
+                ctx->batchlinescursor++;
+            }
+            if (ctx->stdin_credits == 0)
+                break;
+        }
+        if (ctx->batchlinescursor == ctx->batchlines) {
+            ctx->batchlinescursor = 0;
+            if (++ctx->batchcursor == ctx->batchcount) {
+                if (flux_subprocess_close (ctx->p, "stdin") < 0) {
+                    diag ("%s: source: %s", ctx->name, strerror (errno));
+                    goto error;
+                }
+                break;
+            }
+        }
+        if (ctx->stdin_credits == 0)
+            break;
+    }
+    return;
+error:
+    iostress_start_doomsday (ctx, 2.);
+}
+
+flux_subprocess_ops_t iostress_ops = {
+    .on_completion      = iostress_completion_cb,
+    .on_state_change    = iostress_state_cb,
+    .on_stdout          = iostress_output_cb,
+    .on_credit          = iostress_credit_cb,
+};
+
+bool iostress_run_check (flux_t *h,
+                         const char *name,
+                         int write_type,
+                         int stdin_bufsize,
+                         int stdout_bufsize,
+                         int batchcount,
+                         int batchlines,
+                         size_t linesize)
+{
+    char *cat_av[] = { "cat", NULL };
+    flux_cmd_t *cmd;
+    struct iostress_ctx ctx;
+    int rc;
+    bool ret = true;
+    char nbuf[16];
+
+    memset (&ctx, 0, sizeof (ctx));
+    ctx.h = h;
+    ctx.batchcount = batchcount;
+    ctx.batchlines = batchlines;
+    ctx.linesize = linesize;
+    ctx.name = name;
+    ctx.write_type = write_type;
+    ctx.stdin_credits = 0;
+
+    if (!(ctx.buf = malloc (ctx.linesize)))
+        BAIL_OUT ("out of memory");
+    memset (ctx.buf, 'F', ctx.linesize - 1);
+    ctx.buf[ctx.linesize - 1] = '\n';
+
+    if (!(cmd = flux_cmd_create (ARRAY_SIZE (cat_av) - 1, cat_av, environ)))
+        BAIL_OUT ("flux_cmd_create failed");
+    if (stdin_bufsize > 0) {
+        snprintf (nbuf, sizeof (nbuf), "%d", stdin_bufsize);
+        if (flux_cmd_setopt (cmd, "stdin_BUFSIZE", nbuf) < 0)
+            BAIL_OUT ("flux_cmd_setopt failed");
+    }
+    if (stdout_bufsize > 0) {
+        snprintf (nbuf, sizeof (nbuf), "%d", stdout_bufsize);
+        if (flux_cmd_setopt (cmd, "stdout_BUFSIZE", nbuf) < 0)
+            BAIL_OUT ("flux_cmd_setopt failed");
+    }
+    if (!(ctx.p = flux_rexec_ex (h,
+                                 "rexec",
+                                 FLUX_NODEID_ANY,
+                                 0,
+                                 cmd,
+                                 &iostress_ops,
+                                 tap_logger,
+                                 NULL)))
+        BAIL_OUT ("flux_rexec failed");
+    if (flux_subprocess_aux_set (ctx.p, "ctx", &ctx, NULL) < 0)
+        BAIL_OUT ("flux_subprocess_aux_set failed");
+
+    if (!(ctx.source = flux_prepare_watcher_create (flux_get_reactor (h),
+                                                    iostress_source_cb,
+                                                    &ctx)))
+        BAIL_OUT ("could not create prepare watcher");
+    if (!(ctx.timer = flux_timer_watcher_create (flux_get_reactor (h),
+                                                 0.,
+                                                 0.,
+                                                 iostress_timer_cb,
+                                                 &ctx)))
+        BAIL_OUT ("could not create timer watcher");
+
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    if (rc < 0) {
+        diag ("%s: flux_reactor_run: %s", name, strerror (errno));
+        ret = false;
+    }
+
+    diag ("%s: processed %d of %d lines, %d calls to output cb", name,
+          ctx.linerecv, ctx.batchcount * ctx.batchlines, ctx.outputcount);
+    if (ctx.linerecv < ctx.batchcount * ctx.batchlines)
+        ret = false;
+
+    flux_watcher_destroy (ctx.source);
+    flux_watcher_destroy (ctx.timer);
+    diag ("%s: destroying subprocess", name);
+    flux_subprocess_destroy (ctx.p);
+    free (ctx.buf);
+    flux_cmd_destroy (cmd);
+
+    return ret;
+}
+
+int main (int argc, char *argv[])
+{
+    flux_t *h;
+
+    plan (NO_PLAN);
+
+    h = rcmdsrv_create ("rexec");
+
+    ok (iostress_run_check (h,
+                            "balanced",
+                            WRITE_API,
+                            0,
+                            0,
+                            8,
+                            8,
+                            80),
+        "balanced worked");
+
+    // stdout buffer is overrun
+
+    // When the line size is greater than the buffer size, all the
+    // data is transferred. flux_subprocess_read_line() will receive a
+    // "line" that is not terminated with \n
+    ok (iostress_run_check (h,
+                            "tinystdout",
+                            WRITE_API,
+                            0,
+                            128,
+                            1,
+                            1,
+                            256),
+        "tinystdout works");
+
+    // local stdin buffer is overrun (immediately)
+    // remote stdin buffer is also overwritten
+    ok (!iostress_run_check (h,
+                             "tinystdin",
+                             WRITE_API,
+                             128,
+                             0,
+                             1,
+                             1,
+                             4096),
+        "tinystdin failed as expected");
+
+    // remote stdin buffer is overwritten using direct RPC
+    ok (!iostress_run_check (h,
+                             "tinystdin-direct",
+                             WRITE_DIRECT,
+                             128,
+                             0,
+                             1,
+                             1,
+                             4096),
+        "tinystdin-direct failed as expected");
+
+    // remote stdin buffer managed via credits should work
+    ok (iostress_run_check (h,
+                            "tinystdin-credit",
+                            WRITE_CREDIT,
+                            128,
+                            0,
+                            1,
+                            1,
+                            4096),
+        "tinystdin-credit works as expected");
+
+    // credits w/ more data
+    ok (iostress_run_check (h,
+                            "tinystdin-credit-more",
+                            WRITE_CREDIT,
+                            128,
+                            0,
+                            8,
+                            8,
+                            4096),
+        "tinystdin-credit-more works as expected");
+
+    test_server_stop (h);
+    flux_close (h);
+    done_testing ();
+    return 0;
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/test/rcmdsrv.c b/src/common/libsubprocess/test/rcmdsrv.c
new file mode 100644
index 000000000000..530767510b57
--- /dev/null
+++ b/src/common/libsubprocess/test/rcmdsrv.c
@@ -0,0 +1,92 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h> // environ def
+#include <signal.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "src/common/libtap/tap.h"
+#include "src/common/libtestutil/util.h"
+#include "src/common/libsubprocess/server.h"
+#include "src/common/libioencode/ioencode.h"
+#include "src/common/libutil/stdlog.h"
+
+void tap_logger (void *arg,
+                 const char *file,
+                 int line,
+                 const char *func,
+                 const char *subsys,
+                 int level,
+                 const char *fmt,
+                 va_list ap)
+{
+    char buf[4096];
+    int len = sizeof (buf);
+    if (vsnprintf (buf, len, fmt, ap) >= len) {
+        buf[len-1] = '\0';
+        buf[len-2] = '+';
+    }
+    diag ("%s: %s:%d %s(): %s",
+          level == LOG_ERR ? "ERR" : "LOG",
+          file, line, func, buf);
+}
+
+static int test_server (flux_t *h, void *arg)
+{
+    const char *service_name = arg;
+    int rc = -1;
+    subprocess_server_t *srv = NULL;
+
+    if (flux_attr_set_cacheonly (h, "rank", "0") < 0) {
+        diag ("flux_attr_set_cacheonly failed");
+        goto done;
+    }
+    if (!(srv = subprocess_server_create (h,
+                                          service_name,
+                                          "smurf",
+                                          tap_logger,
+                                          NULL))) {
+        diag ("subprocess_server_create failed");
+        goto done;
+    }
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
+        diag ("flux_reactor_run failed");
+        goto done;
+    }
+    diag ("destroying subprocess server");
+    subprocess_server_destroy (srv);
+    diag ("server reactor exiting");
+    rc = 0;
+done:
+    return rc;
+}
+
+flux_t *rcmdsrv_create (const char *service_name)
+{
+    flux_t *h;
+
+    // Without this, process may be terminated by SIGPIPE when writing
+    // to stdin of subprocess that has terminated
+    signal (SIGPIPE, SIG_IGN);
+
+    // N.B. test reactor is created with FLUX_REACTOR_SIGCHLD flag
+    if (!(h = test_server_create (0, test_server, (char *)service_name)))
+        BAIL_OUT ("test_server_create failed");
+
+    return h;
+};
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/test/rcmdsrv.h b/src/common/libsubprocess/test/rcmdsrv.h
new file mode 100644
index 000000000000..9027d5c190f9
--- /dev/null
+++ b/src/common/libsubprocess/test/rcmdsrv.h
@@ -0,0 +1,32 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SUBPROCESS_TEST_RCMDSRV_H
+#define _SUBPROCESS_TEST_RCMDSRV_H
+
+/* Start subprocess server.  Returns one end of back-to-back flux_t test
+ * handle.  Call test_server_stop (h) when done to join with server thread.
+ */
+flux_t *rcmdsrv_create (const char *service_name);
+
+/* llog-compatible logger
+ */
+void tap_logger (void *arg,
+                 const char *file,
+                 int line,
+                 const char *func,
+                 const char *subsys,
+                 int level,
+                 const char *fmt,
+                 va_list ap);
+
+#endif // !_SUBPROCESS_TEST_RCMDSRV_H
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/test/remote.c b/src/common/libsubprocess/test/remote.c
new file mode 100644
index 000000000000..0a07dfa30e39
--- /dev/null
+++ b/src/common/libsubprocess/test/remote.c
@@ -0,0 +1,778 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h> // environ def
+#include <signal.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "src/common/libtap/tap.h"
+#include "src/common/libtestutil/util.h"
+#include "src/common/libsubprocess/server.h"
+#include "src/common/libioencode/ioencode.h"
+#include "src/common/libutil/stdlog.h"
+
+#include "rcmdsrv.h"
+
+#define SERVER_NAME "test-remote"
+
+struct simple_scorecard {
+    unsigned int completion:1;
+    unsigned int exit_nonzero:1;
+    unsigned int signaled:1;
+
+    // states
+    unsigned int init:1;
+    unsigned int running:1;
+    unsigned int failed:1;
+    unsigned int exited:1;
+    unsigned int stopped:1;
+
+    // output
+    unsigned int stdout_eof:1;
+    unsigned int stderr_eof:1;
+    unsigned int stdout_error:1;
+    unsigned int stderr_error:1;
+    int stdout_lines;
+    int stderr_lines;
+};
+
+struct simple_ctx {
+    flux_t *h;
+    struct simple_scorecard scorecard;
+};
+
+extern char **environ;
+
+void corner_case_test (flux_t *h)
+{
+    char *true_av[] = { "true", NULL };
+    flux_subprocess_t *p;
+    flux_cmd_t *cmd;
+
+    cmd = flux_cmd_create (1, true_av, environ);
+    if (!cmd)
+        BAIL_OUT ("flux_cmd_create failed");
+
+    /* N.B. Issue #5968: flags are not passed from `flux_rexec` to
+     * remote subprocesses.  As of this test addition, errors may
+     * occur on other flags, but the only actual flag that should not
+     * be allowed on remote subprocesses is
+     * FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH, which makes no sense
+     * for non-local subprocesses.
+     *
+     * Although other flags would elicit failures, we do not test them.
+     */
+    p = flux_rexec (h,
+                    FLUX_NODEID_ANY,
+                    FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH,
+                    cmd,
+                    NULL);
+    ok (p == NULL && errno == EINVAL,
+        "flux_rexec failed for unsupported flag");
+    flux_cmd_destroy (cmd);
+}
+
+static void simple_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    struct simple_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+    const char *line;
+    int len;
+
+    if ((len = flux_subprocess_read_line (p, stream, &line)) < 0)
+        diag ("%s: %s", stream, strerror (errno));
+    else if (len == 0)
+        diag ("%s: EOF", stream);
+    else
+        diag ("%s: %d bytes", stream, len);
+
+    if (streq (stream, "stdout")) {
+        if (len < 0)
+            ctx->scorecard.stdout_error = 1;
+        else if (len == 0)
+            ctx->scorecard.stdout_eof = 1;
+        else
+            ctx->scorecard.stdout_lines++;
+    }
+    else if (streq (stream, "stderr")) {
+        if (len < 0)
+            ctx->scorecard.stderr_error = 1;
+        else if (len == 0)
+            ctx->scorecard.stderr_eof = 1;
+        else
+            ctx->scorecard.stderr_lines++;
+    }
+}
+
+static void simple_state_cb (flux_subprocess_t *p,
+                             flux_subprocess_state_t state)
+{
+    struct simple_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+
+    diag ("state callback state=%s", flux_subprocess_state_string (state));
+
+    switch (state) {
+        case FLUX_SUBPROCESS_INIT:
+            ctx->scorecard.init = 1;
+            break;
+        case FLUX_SUBPROCESS_RUNNING:
+            ctx->scorecard.running= 1;
+            break;
+        case FLUX_SUBPROCESS_EXITED:
+            ctx->scorecard.exited = 1;
+            break;
+        case FLUX_SUBPROCESS_FAILED:
+            ctx->scorecard.failed = 1;
+            diag ("stopping reactor");
+            flux_reactor_stop (flux_get_reactor (ctx->h));
+            break;
+        case FLUX_SUBPROCESS_STOPPED:
+            ctx->scorecard.stopped = 1;
+            break;
+    }
+}
+
+static void simple_completion_cb (flux_subprocess_t *p)
+{
+    struct simple_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+
+    diag ("completion callback");
+
+    ctx->scorecard.completion = 1;
+    if (flux_subprocess_exit_code (p) > 0)
+        ctx->scorecard.exit_nonzero = 1;
+    if (flux_subprocess_signaled (p) >= 0)
+        ctx->scorecard.signaled = 1;
+
+    diag ("stopping reactor");
+    flux_reactor_stop (flux_get_reactor (ctx->h));
+}
+
+flux_subprocess_ops_t simple_ops = {
+    .on_completion      = simple_completion_cb,
+    .on_state_change    = simple_state_cb,
+    .on_stdout          = simple_output_cb,
+    .on_stderr          = simple_output_cb,
+};
+
+
+void simple_run_check (flux_t *h,
+                       const char *prefix,
+                       int ac,
+                       char **av,
+                       struct simple_scorecard *exp)
+{
+    flux_subprocess_t *p;
+    flux_cmd_t *cmd;
+    struct simple_ctx ctx;
+    int rc;
+
+    cmd = flux_cmd_create (ac, av, environ);
+    if (!cmd)
+        BAIL_OUT ("flux_cmd_create failed");
+
+    memset (&ctx, 0, sizeof (ctx));
+    ctx.h = h;
+    p = flux_rexec_ex (h,
+                       SERVER_NAME,
+                       FLUX_NODEID_ANY,
+                       0,
+                       cmd,
+                       &simple_ops,
+                       tap_logger,
+                       NULL);
+    ok (p != NULL,
+        "%s: flux_rexec_ex returned a subprocess object", prefix);
+    if (!p)
+        BAIL_OUT ("flux_rexec_ex failed");
+    if (flux_subprocess_aux_set (p, "ctx", &ctx, NULL) < 0)
+        BAIL_OUT ("flux_subprocess_aux_set failed");
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    ok (rc >= 0,
+        "%s: client reactor ran successfully", prefix);
+    ok (ctx.scorecard.init == exp->init,
+        "%s: subprocess state=INIT was %sreported",
+        prefix, exp->init ? "" : "not ");
+    ok (ctx.scorecard.running == exp->running,
+        "%s: subprocess state=RUNNING was %sreported",
+        prefix, exp->running ? "" : "not ");
+    ok (ctx.scorecard.exited == exp->exited,
+        "%s: subprocess state=EXITED was %sreported",
+        prefix, exp->exited ? "" : "not ");
+    ok (ctx.scorecard.failed == exp->failed,
+        "%s: subprocess state=FAILED was %sreported",
+        prefix, exp->failed ? "" : "not ");
+    ok (ctx.scorecard.stopped == exp->stopped,
+        "%s: subprocess state=STOPPED was %sreported",
+        prefix, exp->stopped ? "" : "not ");
+    ok (ctx.scorecard.completion == exp->completion,
+        "%s: subprocess completion callback was %sinvoked",
+        prefix, exp->completion ? "" : "not ");
+    ok (ctx.scorecard.exit_nonzero == exp->exit_nonzero,
+        "%s: subprocess did%s exit with nonzero exit code",
+        prefix, exp->exit_nonzero ? "" : " not");
+    ok (ctx.scorecard.signaled == exp->signaled,
+        "%s: subprocess was%s signaled",
+        prefix, exp->signaled ? "" : " not");
+    ok (ctx.scorecard.stdout_lines == exp->stdout_lines,
+        "%s: subprocess stdout got %d lines",
+        prefix, exp->stdout_lines);
+    ok (ctx.scorecard.stdout_eof == exp->stdout_eof,
+        "%s: subprocess stdout %s EOF",
+        prefix, exp->stdout_eof ? "got" : "did not get");
+    ok (ctx.scorecard.stdout_error == exp->stdout_error,
+        "%s: subprocess stdout %s error",
+        prefix, exp->stdout_error ? "got" : "did not get");
+    flux_cmd_destroy (cmd);
+    flux_subprocess_destroy (p);
+}
+
+void simple_test (flux_t *h)
+{
+    struct simple_scorecard exp;
+
+    char *true_av[] = { "true", NULL };
+    memset (&exp, 0, sizeof (exp));
+    exp.running = 1;
+    exp.exited = 1;
+    exp.completion = 1;
+    exp.stdout_eof = 1;
+    exp.stderr_eof = 1;
+    simple_run_check (h,
+                      "true",
+                      ARRAY_SIZE (true_av) - 1,
+                      true_av,
+                      &exp);
+
+    char *false_av[] = { "false", NULL };
+    memset (&exp, 0, sizeof (exp));
+    exp.running = 1;
+    exp.exited = 1;
+    exp.completion = 1;
+    exp.exit_nonzero = 1;
+    exp.stdout_eof = 1;
+    exp.stderr_eof = 1;
+    simple_run_check (h,
+                      "false",
+                      ARRAY_SIZE (false_av) - 1,
+                      false_av,
+                      &exp);
+#if 0
+    // This fails differently on el7 - need to investigate
+    char *nocmd_av[] = { "/nocmd", NULL };
+    memset (&exp, 0, sizeof (exp));
+    exp.failed = 1;
+    simple_run_check (h,
+                      "/nocmd",
+                      ARRAY_SIZE (nocmd_av) - 1,
+                      nocmd_av,
+                      &exp);
+#endif
+
+    char *echo_av[] = { "/bin/sh", "-c", "echo hello", NULL };
+    memset (&exp, 0, sizeof (exp));
+    exp.running = 1;
+    exp.exited = 1;
+    exp.completion = 1;
+    exp.stdout_lines = 1;
+    exp.stdout_eof = 1;
+    exp.stderr_eof = 1;
+    simple_run_check (h,
+                      "echo stdout",
+                      ARRAY_SIZE (echo_av) - 1,
+                      echo_av,
+                      &exp);
+
+    char *echo2_av[] = { "/bin/sh", "-c", "echo hello >&2", NULL };
+    memset (&exp, 0, sizeof (exp));
+    exp.running = 1;
+    exp.exited = 1;
+    exp.completion = 1;
+    exp.stderr_lines = 1;
+    exp.stdout_eof = 1;
+    exp.stderr_eof = 1;
+    simple_run_check (h,
+                      "echo stderr",
+                      ARRAY_SIZE (echo2_av) - 1,
+                      echo2_av,
+                      &exp);
+}
+
+void simple_pre_running_write_close_output_cb (flux_subprocess_t *p,
+                                               const char *stream)
+{
+    struct simple_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+    const char *line = NULL;
+    char cmpbuf[1024];
+    int len;
+
+    if (!streq (stream, "stdout"))
+        BAIL_OUT ("unexpected stream: %s", stream);
+
+    if (ctx->scorecard.stdout_lines == 0) {
+        len = flux_subprocess_read (p, stream, &line);
+        ok (len > 0
+            && line != NULL,
+            "flux_subprocess_read success");
+
+        /* 1 + 3 + 1 for ':', "foo", "\n" */
+        ok (len == (strlen (stream) + 1 + 3 + 1),
+            "flux_subprocess_read returned correct data len");
+
+        sprintf (cmpbuf, "%s:foo\n", stream);
+        ok (streq (line, cmpbuf),
+            "flux_subprocess_read returned correct data");
+
+        ctx->scorecard.stdout_lines++;
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &line);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+        ctx->scorecard.stdout_eof++;
+    }
+}
+
+void simple_pre_running_write_close (flux_t *h)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", NULL };
+    flux_subprocess_ops_t ops = {
+        .on_completion      = simple_completion_cb,
+        .on_stdout          = simple_pre_running_write_close_output_cb,
+    };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+    struct simple_ctx ctx;
+    int rc;
+
+    cmd = flux_cmd_create (3, av, environ);
+    if (!cmd)
+        BAIL_OUT ("flux_cmd_create failed");
+
+    memset (&ctx, 0, sizeof (ctx));
+    ctx.h = h;
+    p = flux_rexec_ex (h,
+                       SERVER_NAME,
+                       FLUX_NODEID_ANY,
+                       0,
+                       cmd,
+                       &ops,
+                       tap_logger,
+                       NULL);
+    ok (p != NULL,
+        "unbuf basic read: flux_rexec_ex returned a subprocess object");
+    if (!p)
+        BAIL_OUT ("flux_rexec_ex failed");
+    if (flux_subprocess_aux_set (p, "ctx", &ctx, NULL) < 0)
+        BAIL_OUT ("flux_subprocess_aux_set failed");
+
+    /* write & close BEFORE flux_reactor_run() */
+    ok (flux_subprocess_write (p, "stdin", "foo", 3) == 3,
+        "flux_subprocess_write success");
+
+    ok (flux_subprocess_close (p, "stdin") == 0,
+        "flux_subprocess_close success");
+
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    ok (rc >= 0, "unbuf basic read: client reactor ran successfully");
+    ok (ctx.scorecard.completion == 1, "completion callback called 1 time");
+    ok (ctx.scorecard.stdout_lines == 1, "stdout lines valid");
+    ok (ctx.scorecard.stdout_eof == 1, "stdout eof count valid");
+
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void simple_pre_running_close_output_cb (flux_subprocess_t *p,
+                                         const char *stream)
+{
+    struct simple_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+    const char *line = NULL;
+    int len;
+
+    if (!streq (stream, "stdout"))
+        BAIL_OUT ("unexpected stream: %s", stream);
+
+    ok (flux_subprocess_read_stream_closed (p, stream),
+        "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+    len = flux_subprocess_read (p, stream, &line);
+    ok (len == 0,
+        "flux_subprocess_read on %s read EOF", stream);
+    ctx->scorecard.stdout_eof++;
+}
+
+void simple_pre_running_close (flux_t *h)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", NULL };
+    flux_subprocess_ops_t ops = {
+        .on_completion      = simple_completion_cb,
+        .on_stdout          = simple_pre_running_close_output_cb,
+    };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+    struct simple_ctx ctx;
+    int rc;
+
+    cmd = flux_cmd_create (3, av, environ);
+    if (!cmd)
+        BAIL_OUT ("flux_cmd_create failed");
+
+    memset (&ctx, 0, sizeof (ctx));
+    ctx.h = h;
+    p = flux_rexec_ex (h,
+                       SERVER_NAME,
+                       FLUX_NODEID_ANY,
+                       0,
+                       cmd,
+                       &ops,
+                       tap_logger,
+                       NULL);
+    ok (p != NULL,
+        "unbuf basic read: flux_rexec_ex returned a subprocess object");
+    if (!p)
+        BAIL_OUT ("flux_rexec_ex failed");
+    if (flux_subprocess_aux_set (p, "ctx", &ctx, NULL) < 0)
+        BAIL_OUT ("flux_subprocess_aux_set failed");
+
+    /* close BEFORE flux_reactor_run() */
+    ok (flux_subprocess_close (p, "stdin") == 0,
+        "flux_subprocess_close success");
+
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    ok (rc >= 0, "unbuf basic read: client reactor ran successfully");
+    ok (ctx.scorecard.completion == 1, "completion callback called 1 time");
+    ok (ctx.scorecard.stdout_lines == 0, "stdout lines valid");
+    ok (ctx.scorecard.stdout_eof == 1, "stdout eof count valid");
+
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void local_unbuf_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    struct simple_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+    const char *line = NULL;
+    char cmpbuf[1024];
+    int len;
+
+    if (!streq (stream, "stdout"))
+        BAIL_OUT ("unexpected stream: %s", stream);
+
+    if (ctx->scorecard.stdout_lines == 0) {
+        errno = 0;
+        len = flux_subprocess_read_line (p, stream, &line);
+        ok (len < 0
+            && errno == EPERM,
+            "flux_subprocess_read_line fails w/ EPERM w/ LOCAL_UNBUF");
+
+        errno = 0;
+        len = flux_subprocess_read_trimmed_line (p, stream, &line);
+        ok (len < 0
+            && errno == EPERM,
+            "flux_subprocess_read_trimmed_line fails w/ EPERM w/ LOCAL_UNBUF");
+
+        errno = 0;
+        len = flux_subprocess_getline (p, stream, &line);
+        ok (len < 0
+            && errno == EPERM,
+            "flux_subprocess_getline fails w/ EPERM w/ LOCAL_UNBUF");
+
+        len = flux_subprocess_read (p, stream, &line);
+        ok (len > 0
+            && line != NULL,
+            "flux_subprocess_read success");
+
+        /* 1 + 2 + 1 for ':', "hi", "\n" */
+        ok (len == (strlen (stream) + 1 + 2 + 1),
+            "flux_subprocess_read returned correct data len");
+
+        /* N.B. not guarantee on NUL termination, use memcmp() not streq() */
+        sprintf (cmpbuf, "%s:hi\n", stream);
+        ok (memcmp (line, cmpbuf, len) == 0,
+            "flux_subprocess_read returned correct data");
+
+        ctx->scorecard.stdout_lines++;
+
+        len = flux_subprocess_read (p, stream, &line);
+        ok (len > 0
+            && line != NULL,
+            "flux_subprocess_read success on second call");
+
+        ok (len == (strlen (stream) + 1 + 2 + 1),
+            "flux_subprocess_read returned correct data len on second call");
+
+        ok (memcmp (line, cmpbuf, len) == 0,
+            "flux_subprocess_read returned correct data on second call");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &line);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+        ctx->scorecard.stdout_eof++;
+    }
+}
+
+void local_unbuf_test (flux_t *h)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "hi", NULL };
+    flux_subprocess_ops_t ops = {
+        .on_completion      = simple_completion_cb,
+        .on_stdout          = local_unbuf_output_cb,
+    };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+    struct simple_ctx ctx;
+    int rc;
+
+    cmd = flux_cmd_create (4, av, environ);
+    if (!cmd)
+        BAIL_OUT ("flux_cmd_create failed");
+
+    memset (&ctx, 0, sizeof (ctx));
+    ctx.h = h;
+    p = flux_rexec_ex (h,
+                       SERVER_NAME,
+                       FLUX_NODEID_ANY,
+                       FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF,
+                       cmd,
+                       &ops,
+                       tap_logger,
+                       NULL);
+    ok (p != NULL,
+        "unbuf basic read: flux_rexec_ex returned a subprocess object");
+    if (flux_subprocess_aux_set (p, "ctx", &ctx, NULL) < 0)
+        BAIL_OUT ("flux_subprocess_aux_set failed");
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    ok (rc >= 0, "unbuf basic read: client reactor ran successfully");
+    ok (ctx.scorecard.completion == 1, "completion callback called 1 time");
+    ok (ctx.scorecard.stdout_lines == 1, "stdout lines valid");
+    ok (ctx.scorecard.stdout_eof == 1, "stdout eof count valid");
+
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void local_unbuf_multiline_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    struct simple_ctx *ctx = flux_subprocess_aux_get (p, "ctx");
+    const char *line = NULL;
+    int len;
+
+    if (!streq (stream, "stdout"))
+        BAIL_OUT ("unexpected stream: %s", stream);
+
+    if (ctx->scorecard.stdout_lines < 2) {
+        len = flux_subprocess_read (p, stream, &line);
+        ok (len > 0
+            && line != NULL,
+            "flux_subprocess_read success");
+
+        /* 3 for "hi" and "\n" */
+        ok (len == 3,
+            "flux_subprocess_read returned correct data len");
+
+        /* N.B. not guarantee on NUL termination, use memcmp() not streq() */
+        ok (memcmp (line, "hi\n", len) == 0,
+            "flux_subprocess_read returned correct data");
+
+        ctx->scorecard.stdout_lines++;
+
+        len = flux_subprocess_read (p, stream, &line);
+        ok (len > 0
+            && line != NULL,
+            "flux_subprocess_read success on second call");
+
+        ok (len == 3,
+            "flux_subprocess_read returned correct data len on second call");
+
+        ok (memcmp (line, "hi\n", len) == 0,
+            "flux_subprocess_read returned correct data on second call");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &line);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+        ctx->scorecard.stdout_eof++;
+    }
+}
+
+void local_unbuf_multiline_test (flux_t *h)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "-n", NULL };
+    flux_subprocess_ops_t ops = {
+        .on_completion      = simple_completion_cb,
+        .on_stdout          = local_unbuf_multiline_output_cb,
+    };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+    struct simple_ctx ctx;
+    int rc;
+
+    cmd = flux_cmd_create (3, av, environ);
+    if (!cmd)
+        BAIL_OUT ("flux_cmd_create failed");
+
+    memset (&ctx, 0, sizeof (ctx));
+    ctx.h = h;
+    p = flux_rexec_ex (h,
+                       SERVER_NAME,
+                       FLUX_NODEID_ANY,
+                       FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF,
+                       cmd,
+                       &ops,
+                       tap_logger,
+                       NULL);
+    ok (p != NULL,
+        "unbuf basic read: flux_rexec_ex returned a subprocess object");
+    if (flux_subprocess_aux_set (p, "ctx", &ctx, NULL) < 0)
+        BAIL_OUT ("flux_subprocess_aux_set failed");
+    ok (flux_subprocess_write (p, "stdin", "hi\nhi\n", 6) == 6,
+        "flux_subprocess_write success");
+    ok (flux_subprocess_close (p, "stdin") == 0,
+        "flux_subprocess_close success");
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    ok (rc >= 0, "unbuf basic read: client reactor ran successfully");
+    ok (ctx.scorecard.completion == 1, "completion callback called 1 time");
+    ok (ctx.scorecard.stdout_lines == 2, "stdout lines valid");
+    ok (ctx.scorecard.stdout_eof == 1, "stdout eof count valid");
+
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+/* In SIGSTOP test, a 'cat' subprocess is sent SIGSTOP upon starting.
+ * If remote SIGSTOP handling works, the state callback is called again
+ * with state == STOPPED, which triggers closure of stdin and natural
+ * termination of the process, which causes the reactor to exit.
+ */
+
+static void stop_state_cb (flux_subprocess_t *p,
+                           flux_subprocess_state_t state)
+{
+    flux_reactor_t *r = flux_subprocess_aux_get (p, "reactor");
+
+    diag ("state callback state=%s", flux_subprocess_state_string (state));
+    if (state == FLUX_SUBPROCESS_RUNNING) {
+        pid_t pid = flux_subprocess_pid (p);
+        if (pid < 0 || kill (pid, SIGSTOP) < 0) {
+            diag ("could not stop test proc: %s", strerror (errno));
+            flux_reactor_stop_error (r);
+        }
+    }
+    else if (state == FLUX_SUBPROCESS_STOPPED) {
+        pid_t pid = flux_subprocess_pid (p);
+        if (pid < 0 || kill (pid, SIGCONT) < 0) {
+            diag ("could not continue test proc: %s", strerror (errno));
+            flux_reactor_stop_error (r);
+        }
+        if (flux_subprocess_close (p, "stdin") < 0) {
+            diag ("could not close remote stdin");
+            flux_reactor_stop_error (r);
+        }
+    }
+}
+
+static void stop_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *line;
+    int len;
+
+    if ((len = flux_subprocess_read_line (p, stream, &line)) < 0)
+        diag ("%s: %s", stream, strerror (errno));
+    else if (len == 0)
+        diag ("%s: EOF", stream);
+    else
+        diag ("%s: %d bytes", stream, len);
+}
+
+flux_subprocess_ops_t stoptest_ops = {
+    .on_state_change    = stop_state_cb,
+    .on_stdout          = stop_output_cb,
+    .on_stderr          = stop_output_cb,
+};
+
+void sigstop_test (flux_t *h)
+{
+    char *av[] = { "/bin/cat", NULL };
+    flux_subprocess_t *p;
+    flux_cmd_t *cmd;
+    int rc;
+
+    cmd = flux_cmd_create (ARRAY_SIZE (av) - 1, av, environ);
+    if (!cmd)
+        BAIL_OUT ("flux_cmd_create failed");
+
+    p = flux_rexec_ex (h,
+                       SERVER_NAME,
+                       FLUX_NODEID_ANY,
+                       0,
+                       cmd,
+                       &stoptest_ops,
+                       tap_logger,
+                       NULL);
+    ok (p != NULL,
+        "stoptest: created subprocess");
+    if (flux_subprocess_aux_set (p, "reactor", flux_get_reactor (h), NULL) < 0)
+        BAIL_OUT ("could not stash reactor in subprocess aux container");
+
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    ok (rc >= 0,
+        "stoptest: reactor ran successfully");
+
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+int main (int argc, char *argv[])
+{
+    flux_t *h;
+
+    plan (NO_PLAN);
+
+    h = rcmdsrv_create (SERVER_NAME);
+
+    diag ("corner_case_test");
+    corner_case_test (h);
+    diag ("simple_test");
+    simple_test (h);
+    diag ("simple_pre_running_write_close");
+    simple_pre_running_write_close (h);
+    diag ("simple_pre_running_close");
+    simple_pre_running_close (h);
+    diag ("local_unbuf_test");
+    local_unbuf_test (h);
+    diag ("local_unbuf_multiline_test");
+    local_unbuf_multiline_test (h);
+    diag ("sigstop_test");
+    sigstop_test (h);
+
+    test_server_stop (h);
+    flux_close (h);
+
+    done_testing ();
+    return 0;
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libsubprocess/test/stdio.c b/src/common/libsubprocess/test/stdio.c
new file mode 100644
index 000000000000..b5c94eadd0b8
--- /dev/null
+++ b/src/common/libsubprocess/test/stdio.c
@@ -0,0 +1,1670 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <assert.h>
+#include <flux/core.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <unistd.h>
+#include <sys/mman.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libsubprocess/subprocess.h"
+#include "src/common/libsubprocess/subprocess_private.h"
+#include "src/common/libsubprocess/server.h"
+#include "ccan/str/str.h"
+
+extern char **environ;
+
+int completion_cb_count;
+int completion_fail_cb_count;
+int stdout_output_cb_count;
+int stderr_output_cb_count;
+int stdout_output_cb_len_count;
+int stderr_output_cb_len_count;
+int output_default_stream_cb_count;
+int multiple_lines_stdout_output_cb_count;
+int multiple_lines_stderr_output_cb_count;
+int stdin_closed_stdout_cb_count;
+int stdin_closed_stderr_cb_count;
+int timer_cb_count;
+int credit_cb_count;
+char inputbuf[1024];
+int inputbuf_index;
+int inputbuf_len;
+char outputbuf[1024];
+int outputbuf_len;
+
+static int fdcount (void)
+{
+    int fd, fdlimit = sysconf (_SC_OPEN_MAX);
+    int count = 0;
+    for (fd = 0; fd < fdlimit; fd++) {
+        if (fcntl (fd, F_GETFD) != -1)
+            count++;
+    }
+    return count;
+}
+
+void completion_cb (flux_subprocess_t *p)
+{
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_EXITED,
+        "subprocess state == EXITED in completion handler");
+    ok (flux_subprocess_status (p) != -1,
+        "subprocess status is valid");
+    ok (flux_subprocess_exit_code (p) == 0,
+        "subprocess exit code is 0, got %d", flux_subprocess_exit_code (p));
+    completion_cb_count++;
+}
+
+void output_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    char cmpbuf[1024];
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdout_output_cb_count;
+    else if (!strcasecmp (stream, "stderr"))
+        counter = &stderr_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    if ((*counter) == 0) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        sprintf (cmpbuf, "%s:hi\n", stream);
+
+        ok (streq (buf, cmpbuf),
+            "flux_subprocess_read_line returned correct data");
+        /* 1 + 2 + 1 for ':', "hi", '\n' */
+        ok (len == (strlen (stream) + 1 + 2 + 1),
+            "flux_subprocess_read_line returned correct data len");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+
+    (*counter)++;
+}
+
+void test_basic_stdout (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
+    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void output_no_readline_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    char cmpbuf[1024];
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdout_output_cb_count;
+    else if (!strcasecmp (stream, "stderr"))
+        counter = &stderr_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    len = flux_subprocess_read (p, stream, &buf);
+    ok (len >= 0,
+        "flux_subprocess_read on %s success", stream);
+
+    if (len > 0) {
+        memcpy (outputbuf + outputbuf_len, buf, len);
+        outputbuf_len += len;
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        sprintf (cmpbuf, "%s:hi\n", stream);
+        ok (streq (outputbuf, cmpbuf),
+            "flux_subprocess_read returned correct data");
+        /* 1 + 2 + 1 for ':', "hi", '\n' */
+        ok (outputbuf_len == (strlen (stream) + 1 + 2 + 1),
+            "flux_subprocess_read returned correct amount of data");
+    }
+
+    (*counter)++;
+}
+
+/* use flux_subprocess_read() instead of flux_subprocess_read_line() */
+void test_basic_stdout_no_readline (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_no_readline_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    memset (outputbuf, '\0', sizeof (outputbuf));
+    outputbuf_len = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count >= 2, "stdout output callback called atleast 2 times");
+    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void test_basic_stderr (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-E", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stderr = output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+    ok ((flux_subprocess_pid (p) > (pid_t) 0),
+        "flux_local_exec() started pid %ld", (pid_t) flux_subprocess_pid (p));
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count == 0, "stdout output callback called 0 times");
+    ok (stderr_output_cb_count == 2, "stderr output callback called 2 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void test_basic_stdout_and_stderr (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "-E", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_cb,
+        .on_stderr = output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
+    ok (stderr_output_cb_count == 2, "stderr output callback called 2 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void test_basic_default_output (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "-E", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = subprocess_standard_output,
+        .on_stderr = subprocess_standard_output
+    };
+    completion_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void output_default_stream_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    char cmpbuf[1024];
+    int len;
+
+    if (output_default_stream_cb_count == 0) {
+        len = flux_subprocess_read_line (p, "stdout", &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", "stdout");
+
+        sprintf (cmpbuf, "%s:hi\n", stream);
+
+        ok (streq (buf, cmpbuf),
+            "flux_subprocess_read_line returned correct data");
+        /* 1 + 2 + 1 for ':', "hi", '\n' */
+        ok (len == (strlen (stream) + 1 + 2 + 1),
+            "flux_subprocess_read_line returned correct data len");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", "stdout");
+
+        len = flux_subprocess_read (p, "stdout", &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", "stdout");
+    }
+
+    output_default_stream_cb_count++;
+}
+
+void test_basic_stdin (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O",  NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (3, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    ok (flux_subprocess_write (p, "stdin", "hi", 2) == 2,
+        "flux_subprocess_write success");
+
+    ok (flux_subprocess_close (p, "stdin") == 0,
+        "flux_subprocess_close success");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void output_no_newline_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    char cmpbuf[1024];
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdout_output_cb_count;
+    else if (!strcasecmp (stream, "stderr"))
+        counter = &stderr_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    if ((*counter) == 0) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read_line on %s read 0 lines", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read on %s read success", stream);
+
+        sprintf (cmpbuf, "%s:hi", stream);
+
+        ok (streq (buf, cmpbuf),
+            "flux_subprocess_read returned correct data");
+        /* 1 + 2 for ':', "hi" */
+        ok (len == (strlen (stream) + 1 + 2),
+            "flux_subprocess_read_line returned correct data len");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+
+    (*counter)++;
+}
+
+void test_basic_no_newline (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "-E", "-n", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (6, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_no_newline_cb,
+        .on_stderr = output_no_newline_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
+    ok (stderr_output_cb_count == 2, "stderr output callback called 2 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void output_trimmed_line_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    char cmpbuf[1024];
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdout_output_cb_count;
+    else if (!strcasecmp (stream, "stderr"))
+        counter = &stderr_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    if ((*counter) == 0) {
+        len = flux_subprocess_read_trimmed_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_trimmed_line on %s success", stream);
+
+        sprintf (cmpbuf, "%s:hi", stream);
+
+        ok (streq (buf, cmpbuf),
+            "flux_subprocess_read_trimmed_line returned correct data");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+
+    (*counter)++;
+}
+
+void test_basic_trimmed_line (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "-E", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_trimmed_line_cb,
+        .on_stderr = output_trimmed_line_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
+    ok (stderr_output_cb_count == 2, "stderr output callback called 2 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void multiple_lines_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &multiple_lines_stdout_output_cb_count;
+    else if (!strcasecmp (stream, "stderr"))
+        counter = &multiple_lines_stderr_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    if ((*counter) == 0) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (streq (buf, "foo\n"),
+            "flux_subprocess_read_line returned correct data");
+        ok (len == 4,
+            "flux_subprocess_read_line returned correct data len");
+    }
+    else if ((*counter) == 1) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (streq (buf, "bar\n"),
+            "flux_subprocess_read_line returned correct data");
+        ok (len == 4,
+            "flux_subprocess_read_line returned correct data len");
+    }
+    else if ((*counter) == 2) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (streq (buf, "bo\n"),
+            "flux_subprocess_read_line returned correct data");
+        ok (len == 3,
+            "flux_subprocess_read_line returned correct data len");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+
+    (*counter)++;
+}
+
+void test_basic_multiple_lines (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "-E", "-n", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = multiple_lines_output_cb,
+        .on_stderr = multiple_lines_output_cb
+    };
+    completion_cb_count = 0;
+    multiple_lines_stdout_output_cb_count = 0;
+    multiple_lines_stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    ok (flux_subprocess_write (p, "stdin", "foo\n", 4) == 4,
+        "flux_subprocess_write success");
+
+    ok (flux_subprocess_write (p, "stdin", "bar\n", 4) == 4,
+        "flux_subprocess_write success");
+
+    ok (flux_subprocess_write (p, "stdin", "bo\n", 3) == 3,
+        "flux_subprocess_write success");
+
+    ok (flux_subprocess_close (p, "stdin") == 0,
+        "flux_subprocess_close success");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (multiple_lines_stdout_output_cb_count == 4, "stdout output callback called 4 times");
+    ok (multiple_lines_stderr_output_cb_count == 4, "stderr output callback called 4 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void stdin_closed_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdin_closed_stdout_cb_count;
+    else if (!strcasecmp (stream, "stderr"))
+        counter = &stdin_closed_stderr_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    ok (flux_subprocess_read_stream_closed (p, stream),
+        "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+    len = flux_subprocess_read (p, stream, &buf);
+    ok (len == 0,
+        "flux_subprocess_read on %s read EOF", stream);
+
+    (*counter)++;
+}
+
+void test_basic_stdin_closed (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "-E", "-n", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = stdin_closed_cb,
+        .on_stderr = stdin_closed_cb
+    };
+    completion_cb_count = 0;
+    stdin_closed_stdout_cb_count = 0;
+    stdin_closed_stderr_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    ok (flux_subprocess_close (p, "stdin") == 0,
+        "flux_subprocess_close success");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdin_closed_stdout_cb_count == 1, "stdout output callback called 1 time");
+    ok (stdin_closed_stderr_cb_count == 1, "stderr output callback called 1 time");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void output_read_line_until_eof_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdout_output_cb_count;
+    else if (!strcasecmp (stream, "stderr"))
+        counter = &stderr_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    len = flux_subprocess_getline (p, stream, &buf);
+    if ((*counter) == 0) {
+        ok (len > 0,
+            "flux_subprocess_getline on %s success", stream);
+        ok (streq (buf, "foo\n"),
+            "flux_subprocess_getline returned correct data");
+        ok (len == 4,
+            "flux_subprocess_getline returned correct data len");
+    }
+    else if ((*counter) == 1) {
+        ok (len > 0,
+            "flux_subprocess_getline on %s success", stream);
+        ok (streq (buf, "bar"),
+            "flux_subprocess_getline returned correct data");
+        ok (len == 3,
+            "flux_subprocess_getline returned correct data len");
+    }
+    else {
+        ok (len == 0,
+            "flux_subprocess_getline returned EOF");
+    }
+
+    (*counter)++;
+}
+
+void test_basic_read_line_until_eof (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "-E", "-n", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_read_line_until_eof_cb,
+        .on_stderr = output_read_line_until_eof_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    ok (flux_subprocess_write (p, "stdin", "foo\n", 4) == 4,
+        "flux_subprocess_write success");
+
+    ok (flux_subprocess_write (p, "stdin", "bar", 3) == 3,
+        "flux_subprocess_write success");
+
+    ok (flux_subprocess_close (p, "stdin") == 0,
+        "flux_subprocess_close success");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count == 3, "stdout output callback called 3 times");
+    ok (stderr_output_cb_count == 3, "stderr output callback called 3 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void output_read_line_until_eof_error_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdout_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    if ((*counter) == 0) {
+        len = flux_subprocess_getline (p, stream, &buf);
+        ok (len < 0 && errno == EPERM,
+            "flux_subprocess_getline returns EPERM "
+            "on non line-buffered stream");
+
+        /* drain whatever is in the buffer, we don't care about
+         * contents for this test */
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read on %s success", stream);
+    }
+    else {
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+    (*counter)++;
+}
+
+void test_basic_read_line_until_eof_error (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (3, av, environ)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "false") == 0,
+        "flux_cmd_setopt set stdout_LINE_BUFFER success");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_read_line_until_eof_error_cb,
+        .on_stderr = NULL
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
+    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void test_write_after_close (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "-E", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (3, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    ok (flux_subprocess_write (p, "stdin", "hi", 2) == 2,
+        "flux_subprocess_write success");
+
+    ok (flux_subprocess_close (p, "stdin") == 0,
+        "flux_subprocess_close success");
+
+    ok (flux_subprocess_write (p, "stdin", "hi", 2) < 0
+        && errno == EPIPE,
+        "flux_subprocess_write failed with EPIPE after a close");
+
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void test_write_enospc (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "-E", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (3, av, environ)) != NULL, "flux_cmd_create");
+    ok (flux_cmd_setopt (cmd, "stdin_BUFSIZE", "5") == 0,
+        "set stdin buffer size to 5 bytes");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    ok (flux_subprocess_write (p, "stdin", "hi\n", 3) == 3,
+        "flux_subprocess_write success");
+    ok (flux_subprocess_write (p, "stdin", "hello\n", 6) < 0
+        && errno == ENOSPC,
+        "flux_subprocess_write returns ENOSPC if buffer exceeded");
+
+    ok (flux_subprocess_close (p, "stdin") == 0,
+        "flux_subprocess_close success");
+
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+#if 0
+/* disable test.  libtap has an issue with fallthrough
+ * stdout/stderr in forked process */
+void test_flag_stdio_fallthrough (flux_reactor_t *r)
+{
+    char *av[] = { "echo", "foo", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (2, av, NULL)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb
+    };
+    completion_cb_count = 0;
+    p = flux_local_exec (r,
+                         FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH,
+                         cmd,
+                         &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+#endif
+
+/* Line buffering tests are technically racy.  If the stdout in the
+ * test_multi_echo command occurs fast enough, a single on_stdout
+ * callback could occur.  But hopefully by repeating the word "hi" a
+ * lot of times, the probability of that occurring is zero if line
+ * buffering is not working.
+ *
+ * I pick 2200 to make sure we output enough to surpass 4096 bytes of
+ * output (i.e. 2200 * 2 bytes > 4096 bytes).
+ */
+
+void line_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdout_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    if ((*counter) == 0) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len == 4401
+            && buf != NULL,
+            "flux_subprocess_read_line read line correctly");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+
+    (*counter)++;
+}
+
+void test_line_buffer_default (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-c", "2200", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = line_output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    /* == 2 times means we got a single line and EOF */
+    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
+    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void test_line_buffer_enable (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-c", "2200", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "true") == 0,
+        "flux_cmd_setopt set stdout_LINE_BUFFER success");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = line_output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    /* == 2 times means we got a single line and EOF */
+    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
+    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void count_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdout_output_cb_count;
+    else if (!strcasecmp (stream, "stderr"))
+        counter = &stderr_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    (void)flux_subprocess_read_line (p, stream, NULL);
+    (*counter)++;
+}
+
+void test_line_buffer_disable (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-c", "2200", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "false") == 0,
+        "flux_cmd_setopt set stdout_LINE_BUFFER success");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = count_output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    /* we care about greater than two, that it's not a single line and EOF */
+    ok (stdout_output_cb_count > 2, "stdout output callback got more than 2 calls: %d", stdout_output_cb_count);
+    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void test_line_buffer_error (flux_reactor_t *r)
+{
+    char *av[] = { "true", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "ABCD") == 0,
+        "flux_cmd_setopt set stdout_LINE_BUFFER success");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_channel_out = subprocess_standard_output,
+        .on_stdout = subprocess_standard_output,
+        .on_stderr = subprocess_standard_output
+    };
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p == NULL
+        && errno == EINVAL,
+        "flux_local_exec fails with EINVAL due to bad line_buffer input");
+
+    flux_cmd_destroy (cmd);
+}
+
+void test_stream_start_stop_basic (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "-E", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = output_cb,
+        .on_stderr = output_cb,
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
+    ok (stderr_output_cb_count == 2, "stderr output callback called 2 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void start_stdout_after_stderr_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+    int *counter;
+    int *len_counter;
+
+    if (!strcasecmp (stream, "stdout")) {
+        counter = &stdout_output_cb_count;
+        len_counter = &stdout_output_cb_len_count;
+    }
+    else if (!strcasecmp (stream, "stderr")) {
+        counter = &stderr_output_cb_count;
+        len_counter = &stderr_output_cb_len_count;
+    }
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    len = flux_subprocess_read (p, stream, &buf);
+    (*counter)++;
+    (*len_counter)+= len;
+
+    if (len > 0 && buf != NULL && (*len_counter) == 10001) {
+        if (streq (stream, "stderr")) {
+            ok (stdout_output_cb_count == 0
+                && stdout_output_cb_len_count == 0,
+                "received all stderr data and stdout output is still 0");
+            flux_subprocess_stream_start (p, "stdout");
+            diag ("flux_subprocess_stream_start on stdout");
+        }
+    }
+}
+
+/* How this tests works is we output "hi" alot of times without line
+ * buffering on both stdout and stderr.  After starting the
+ * subprocess, we immediately disable the stdout stream.  The goal is
+ * we get all the stderr via callback, then re-enable the stdout
+ * stream, and get the rest of the stdout.
+ *
+ * This test is racy, as its always possible stderr just arrives
+ * before stdout under normal circumstances, but the probability of
+ * that occurring is low given how much we output.
+ */
+void test_stream_start_stop_initial_stop (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-E", "-c", "5000", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (6, av, environ)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "false") == 0,
+        "flux_cmd_setopt set stdout_LINE_BUFFER success");
+
+    ok (flux_cmd_setopt (cmd, "stderr_LINE_BUFFER", "false") == 0,
+        "flux_cmd_setopt set stderr_LINE_BUFFER success");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = start_stdout_after_stderr_cb,
+        .on_stderr = start_stdout_after_stderr_cb,
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    stdout_output_cb_len_count = 0;
+    stderr_output_cb_len_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    flux_subprocess_stream_stop (p, "stdout");
+    diag ("flux_subprocess_stream_stop on stdout");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    /* potential for == 2, b/c could all be buffered before stdout
+     * callback is re-started */
+    ok (stdout_output_cb_count >= 2, "stdout output callback called >= 2 times: %d",
+        stdout_output_cb_count);
+    /* we would hope stderr is called > 2 times, but there's
+     * potentially racy behavior and its only called 2 times.  This
+     * isn't seen in practice. */
+    ok (stderr_output_cb_count > 2, "stderr output callback called > 2 times: %d",
+        stderr_output_cb_count);
+    ok (stdout_output_cb_len_count == 10001, "stdout_output_cb_len_count is 10001");
+    ok (stderr_output_cb_len_count == 10001, "stderr_output_cb_len_count is 10001");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void mid_stop_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
+                        int revents, void *arg)
+{
+    flux_subprocess_t *p = arg;
+    ok (stdout_output_cb_count == 1,
+        "stdout callback has not been called since timer activated");
+    flux_subprocess_stream_start (p, "stdout");
+    diag ("flux_subprocess_stream_start on stdout");
+    timer_cb_count++;
+    flux_watcher_stop (w);
+}
+
+void mid_stop_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdout_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    len = flux_subprocess_read (p, stream, &buf);
+    if (stdout_output_cb_count == 0) {
+        flux_watcher_t *tw = NULL;
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read read data on stdout: %d", len);
+        flux_subprocess_stream_stop (p, "stdout");
+        diag ("flux_subprocess_stream_stop on stdout");
+        ok ((tw = flux_subprocess_aux_get (p, "tw")) != NULL,
+            "flux_subprocess_aux_get timer success");
+        flux_watcher_start (tw);
+    }
+    else if (stdout_output_cb_count == 1) {
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read read data on stdout: %d", len);
+        ok (timer_cb_count == 1,
+            "next stdout callback called after time callback called");
+    }
+    (*counter)++;
+}
+
+/* How this tests works is we output "hi" alot of times without line
+ * buffering on stdout.  After the first callback, we stop the output
+ * stream, and setup a timer.  For a bit of time, we should see no
+ * more stdout, and after the timer expires, we'll re-enable the
+ * stdout stream.
+ *
+ * This test is racy, as its always possible stdout is just delayed,
+ * but the probability of that occurring is low given how much we
+ * output.
+ */
+void test_stream_start_stop_mid_stop (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-c", "5000", "hi", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+    flux_watcher_t *tw = NULL;
+
+    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "false") == 0,
+        "flux_cmd_setopt set stdout_LINE_BUFFER success");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = mid_stop_cb,
+        .on_stderr = NULL,
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    timer_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok ((tw = flux_timer_watcher_create (r, 2.0, 0.0,
+                                         mid_stop_timer_cb, p)) != NULL,
+        "flux_timer_watcher_create success");
+
+    ok (!flux_subprocess_aux_set (p, "tw", tw, NULL),
+        "flux_subprocess_aux_set timer success");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    /* could be == to 3 if output occurs fast enough, but chances are
+     * it'll be > 3 */
+    ok (stdout_output_cb_count >= 3, "stdout output callback called >= 3 times: %d",
+        stdout_output_cb_count);
+    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
+    ok (timer_cb_count == 1, "timer callback called 1 time");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+    flux_watcher_destroy (tw);
+}
+
+void overflow_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+
+    if (strcasecmp (stream, "stdout") != 0) {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    /* first callback should return "0123" for 4 byte buffer.
+     * second callback should return "456\n" in 4 byte buffer
+     */
+    if (stdout_output_cb_count == 0) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (streq (buf, "0123"),
+            "flux_subprocess_read_line returned correct data");
+        ok (len == 4,
+            "flux_subprocess_read_line returned correct data len");
+    }
+    else if (stdout_output_cb_count == 1) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (streq (buf, "456\n"),
+            "flux_subprocess_read_line returned correct data");
+        ok (len == 4,
+            "flux_subprocess_read_line returned correct data len");
+    }
+    else {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+    stdout_output_cb_count++;
+}
+
+/* Set buffer size to 4 and have 7 bytes of output (8 including newline) */
+void test_long_line (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "0123456", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (3, av, environ)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_setopt (cmd, "stdout_BUFSIZE", "4") == 0,
+        "flux_cmd_setopt set stdout_BUFSIZE success");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = overflow_output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count == 3, "stdout output callback called 3 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+void credit_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+
+    if (strcasecmp (stream, "stdout")) {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    len = flux_subprocess_read (p, stream, &buf);
+    ok (len >= 0,
+        "flux_subprocess_read on %s success", stream);
+
+    if (len > 0) {
+        memcpy (outputbuf + outputbuf_len, buf, len);
+        outputbuf_len += len;
+    }
+    else {
+        char cmpbuf[1024];
+
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        sprintf (cmpbuf, "abcdefghijklmnopqrstuvwxyz0123456789\n");
+        ok (streq (outputbuf, cmpbuf),
+            "flux_subprocess_read returned correct data");
+        /* 26 (ABCs) + 10 (1-10) + 1 for `\n' */
+        ok (outputbuf_len == (26 + 10 + 1),
+            "flux_subprocess_read returned correct amount of data");
+    }
+    stdout_output_cb_count++;
+}
+
+void credit_cb (flux_subprocess_t *p, const char *stream, int bytes)
+{
+    int *credits = flux_subprocess_aux_get (p, "credits");
+    int len;
+    int ret;
+
+    assert (credits);
+
+    diag ("on_credit: credit of %d bytes", bytes);
+
+    (*credits) += bytes;
+
+    if ((inputbuf_len - inputbuf_index) > 0) {
+        /* If we "borrowed" credits, credits may not be > 0 */
+        if ((*credits) <= 0)
+            goto out;
+
+        if ((inputbuf_len - inputbuf_index) > (*credits))
+            len = (*credits);
+        else
+            len = (inputbuf_len - inputbuf_index);
+
+        ret = flux_subprocess_write (p, "stdin", &inputbuf[inputbuf_index], len);
+        ok (ret == len,
+            "flux_subprocess_write success");
+
+        (*credits) -= ret;
+        inputbuf_index += ret;
+    }
+    else {
+        ok (flux_subprocess_close (p, "stdin") == 0,
+            "flux_subprocess_close success");
+    }
+
+out:
+    credit_cb_count++;
+}
+
+void test_on_credit (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O",  NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+    int credits = 0;
+    int ret;
+
+    ok ((cmd = flux_cmd_create (2, av, environ)) != NULL, "flux_cmd_create");
+    ok (flux_cmd_setopt (cmd, "stdin_BUFSIZE", "8") == 0,
+        "set stdin buffer size to 8 bytes");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = credit_output_cb,
+        .on_credit = credit_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    credit_cb_count = 0;
+    sprintf (inputbuf, "abcdefghijklmnopqrstuvwxyz0123456789");
+    inputbuf_index = 0;
+    inputbuf_len = (26 + 10);
+    memset (outputbuf, '\0', sizeof (outputbuf));
+    outputbuf_len = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+    ret = flux_subprocess_aux_set (p, "credits", &credits, NULL);
+    ok (ret == 0, "flux_subprocess_aux_set works");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    errno = 0;
+    ret = flux_subprocess_write (p, "stdin", &inputbuf[inputbuf_index], 10);
+    ok (ret < 0 && errno == ENOSPC,
+        "flux_subprocess_write fails with too much data");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count >= 2, "stdout output callback called >= 2 times");
+    ok (credit_cb_count == 6, "credit callback called 6 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+/* very similar to above test but we send initial write by "borrowing"
+ * credits
+ */
+void test_on_credit_borrow_credits (flux_reactor_t *r)
+{
+    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O",  NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+    int credits = 0;
+    int ret;
+
+    ok ((cmd = flux_cmd_create (2, av, environ)) != NULL, "flux_cmd_create");
+    ok (flux_cmd_setopt (cmd, "stdin_BUFSIZE", "8") == 0,
+        "set stdin buffer size to 8 bytes");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = completion_cb,
+        .on_stdout = credit_output_cb,
+        .on_credit = credit_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    credit_cb_count = 0;
+    sprintf (inputbuf, "abcdefghijklmnopqrstuvwxyz0123456789");
+    inputbuf_index = 0;
+    inputbuf_len = (26 + 10);
+    memset (outputbuf, '\0', sizeof (outputbuf));
+    outputbuf_len = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+    ret = flux_subprocess_aux_set (p, "credits", &credits, NULL);
+    ok (ret == 0, "flux_subprocess_aux_set works");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+
+    errno = 0;
+    ret = flux_subprocess_write (p, "stdin", &inputbuf[inputbuf_index], 8);
+    ok (ret == 8,
+        "flux_subprocess_write first 8 bytes");
+    credits -= ret;
+    inputbuf_index += ret;
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (stdout_output_cb_count >= 2, "stdout output callback called >= 2 times");
+    ok (credit_cb_count == 6, "credit callback called 6 times");
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
+int main (int argc, char *argv[])
+{
+    flux_reactor_t *r;
+    int start_fdcount, end_fdcount;
+
+    plan (NO_PLAN);
+
+    // Create shared reactor for all tests
+    ok ((r = flux_reactor_create (FLUX_REACTOR_SIGCHLD)) != NULL,
+        "flux_reactor_create");
+
+    start_fdcount = fdcount ();
+
+    diag ("basic_stdout");
+    test_basic_stdout (r);
+    diag ("basic_stdout_no_readline");
+    test_basic_stdout_no_readline (r);
+    diag ("basic_stderr");
+    test_basic_stderr (r);
+    diag ("basic_stdout_and_stderr");
+    test_basic_stdout_and_stderr (r);
+    diag ("basic_default_output");
+    test_basic_default_output (r);
+    diag ("basic_stdin");
+    test_basic_stdin (r);
+    diag ("basic_no_newline");
+    test_basic_no_newline (r);
+    diag ("basic_trimmed_line");
+    test_basic_trimmed_line (r);
+    diag ("basic_multiple_lines");
+    test_basic_multiple_lines (r);
+    diag ("basic_stdin_closed");
+    test_basic_stdin_closed (r);
+    diag ("basic_read_line_until_eof");
+    test_basic_read_line_until_eof (r);
+    diag ("basic_read_line_until_eof_error");
+    test_basic_read_line_until_eof_error (r);
+    diag ("write_after_close");
+    test_write_after_close (r);
+    diag ("write_enospc");
+    test_write_enospc (r);
+#if 0
+    diag ("flag_stdio_fallthrough");
+    test_flag_stdio_fallthrough (r);
+#endif
+    diag ("line_buffer_default");
+    test_line_buffer_default (r);
+    diag ("line_buffer_enable");
+    test_line_buffer_enable (r);
+    diag ("line_buffer_disable");
+    test_line_buffer_disable (r);
+    diag ("line_buffer_error");
+    test_line_buffer_error (r);
+    diag ("stream_start_stop_basic");
+    test_stream_start_stop_basic (r);
+    diag ("stream_start_stop_initial_stop");
+    test_stream_start_stop_initial_stop (r);
+    diag ("stream_start_stop_mid_stop");
+    test_stream_start_stop_mid_stop (r);
+    diag ("long_line");
+    test_long_line (r);
+    diag ("on_credit");
+    test_on_credit (r);
+    diag ("on_credit_borrow_credits");
+    test_on_credit_borrow_credits (r);
+
+    end_fdcount = fdcount ();
+
+    ok (start_fdcount == end_fdcount,
+        "no file descriptors leaked");
+
+    flux_reactor_destroy (r);
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libsubprocess/test/subprocess.c b/src/common/libsubprocess/test/subprocess.c
index b18d86be7172..8ebd0b7ecd8a 100644
--- a/src/common/libsubprocess/test/subprocess.c
+++ b/src/common/libsubprocess/test/subprocess.c
@@ -8,8 +8,10 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <assert.h>
-
 #include <flux/core.h>
 #include <unistd.h>
 #include <fcntl.h>
@@ -19,6 +21,10 @@
 
 #include "src/common/libtap/tap.h"
 #include "src/common/libsubprocess/subprocess.h"
+#include "src/common/libsubprocess/subprocess_private.h"
+#include "src/common/libsubprocess/server.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
 
 extern char **environ;
 
@@ -26,13 +32,6 @@ int completion_cb_count;
 int completion_fail_cb_count;
 int stdout_output_cb_count;
 int stderr_output_cb_count;
-int stdout_output_cb_len_count;
-int stderr_output_cb_len_count;
-int output_default_stream_cb_count;
-int multiple_lines_stdout_output_cb_count;
-int multiple_lines_stderr_output_cb_count;
-int stdin_closed_stdout_cb_count;
-int stdin_closed_stderr_cb_count;
 int env_passed_cb_count;
 int completion_sigterm_cb_count;
 int output_processes_cb_count;
@@ -41,11 +40,7 @@ int child_pid;
 int stdout_eof_cb_count;
 int stderr_eof_cb_count;
 int state_change_cb_count;
-int channel_fd_env_cb_count;
-int channel_in_cb_count;
-int channel_in_and_out_cb_count;
-int multiple_lines_channel_cb_count;
-int channel_nul_terminate_cb_count;
+int stopped_cb_count;
 int timer_cb_count;
 
 static int fdcount (void)
@@ -66,107 +61,31 @@ void completion_cb (flux_subprocess_t *p)
     ok (flux_subprocess_status (p) != -1,
         "subprocess status is valid");
     ok (flux_subprocess_exit_code (p) == 0,
-        "subprocess exit code is 0");
+        "subprocess exit code is 0, got %d", flux_subprocess_exit_code (p));
     completion_cb_count++;
 }
 
-void test_basic (flux_reactor_t *r)
-{
-    char *av[] = { "/bin/true", NULL };
-    flux_cmd_t *cmd, *cmd2;
-    flux_reactor_t *r2;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb
-    };
-    completion_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-    ok ((flux_subprocess_pid (p) > (pid_t) 0),
-        "flux_local_exec() started pid %ld", (pid_t) flux_subprocess_pid (p));
-    ok ((cmd2 = flux_subprocess_get_cmd (p)) != NULL,
-        "flux_subprocess_get_cmd success");
-    ok ((r2 = flux_subprocess_get_reactor (p)) != NULL,
-        "flux_subprocess_get_reactor success");
-    ok (r == r2,
-        "flux_subprocess_get_reactor returns correct reactor");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void completion_fail_cb (flux_subprocess_t *p)
-{
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_EXITED,
-        "subprocess state == EXITED in completion handler");
-    ok (flux_subprocess_status (p) != -1,
-        "subprocess status is valid");
-    ok (flux_subprocess_exit_code (p) == 1,
-        "subprocess exit code is 1");
-    completion_fail_cb_count++;
-}
-
-void test_basic_fail (flux_reactor_t *r)
-{
-    char *av[] = { "/bin/false", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_fail_cb
-    };
-    completion_fail_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_fail_cb_count == 1, "completion fail callback called 1 time");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void test_basic_errors (flux_reactor_t *r)
+void test_corner_cases (flux_reactor_t *r)
 {
     flux_t *h = NULL;
-    char *avgood[] = { "/bin/true", NULL };
+    char *avgood[] = { "true", NULL };
     char *avbad[] = { NULL };
     flux_cmd_t *cmd;
 
     ok ((h = flux_open ("loop://", 0)) != NULL,
         "flux_open on loop works");
 
-    ok (!flux_subprocess_server_start (NULL, NULL, NULL, 0)
-        && errno == EINVAL,
-        "flux_subprocess_server_start fails with NULL pointer inputs");
-    ok (flux_subprocess_server_terminate_by_uuid (NULL, NULL) < 0
+    ok (!subprocess_server_create (NULL, NULL, NULL, NULL, NULL)
         && errno == EINVAL,
-        "flux_subprocess_server_terminate_by_uuid fails with NULL pointer inputs");
-    ok (flux_subprocess_server_subprocesses_kill (NULL, 0, 0.) < 0
+        "subprocess_server_create fails with NULL pointer inputs");
+    ok (subprocess_server_shutdown (NULL, 0) == NULL
         && errno == EINVAL,
-        "flux_subprocess_server_subprocesses_kill fails with NULL pointer inputs");
+        "subprocess_server_shutdown fails with NULL pointer input");
 
-    ok (flux_exec (NULL, 0, NULL, NULL, NULL) == NULL
-        && errno == EINVAL,
-        "flux_exec fails with NULL pointer inputs");
-    ok (flux_local_exec (NULL, 0, NULL, NULL, NULL) == NULL
+    ok (flux_local_exec (NULL, 0, NULL, NULL) == NULL
         && errno == EINVAL,
         "flux_local_exec fails with NULL pointer inputs");
-    ok (flux_local_exec (r, 1234, NULL, NULL, NULL) == NULL
+    ok (flux_local_exec (r, 1234, NULL, NULL) == NULL
         && errno == EINVAL,
         "flux_local_exec fails with invalid flag");
     ok (flux_rexec (NULL, 0, 0, NULL, NULL) == NULL
@@ -178,107 +97,98 @@ void test_basic_errors (flux_reactor_t *r)
 
     ok ((cmd = flux_cmd_create (0, avbad, NULL)) != NULL,
         "flux_cmd_create with 0 args works");
+    ok (flux_local_exec (r, 0, cmd, NULL) == NULL
+        && errno == EINVAL,
+        "flux_local_exec fails with cmd with zero args");
     ok (flux_rexec (h, 0, 0, cmd, NULL) == NULL
         && errno == EINVAL,
         "flux_rexec fails with cmd with zero args");
     flux_cmd_destroy (cmd);
 
     ok ((cmd = flux_cmd_create (1, avgood, NULL)) != NULL,
-        "flux_cmd_create with 0 args works");
+        "flux_cmd_create with true works");
     ok (flux_rexec (h, -10, 0, cmd, NULL) == NULL
         && errno == EINVAL,
         "flux_rexec fails with cmd with invalid rank");
     flux_cmd_destroy (cmd);
 
-    ok ((cmd = flux_cmd_create (1, avgood, NULL)) != NULL,
-        "flux_cmd_create with 0 args works");
-    ok (flux_rexec (h, 0, 0, cmd, NULL) == NULL
-        && errno == EINVAL,
-        "flux_rexec fails with cmd with no cwd");
-    flux_cmd_destroy (cmd);
-
-    ok ((cmd = flux_cmd_create (1, avgood, NULL)) != NULL,
-        "flux_cmd_create with 0 args works");
-    ok (flux_cmd_setcwd (cmd, "foobar") == 0,
-        "flux_cmd_setcwd works");
-    ok (flux_cmd_setopt (cmd, "stdout_STREAM_STOP", "true") == 0,
-        "flux_cmd_setopt works");
-    ok (flux_rexec (h, 0, 0, cmd, NULL) == NULL
-        && errno == EINVAL,
-        "flux_rexec fails with cmd with STREAM_STOP option");
-    flux_cmd_destroy (cmd);
-
-    ok (flux_subprocess_stream_start (NULL, NULL) < 0
-        && errno == EINVAL,
-        "flux_subprocess_stream_start fails with NULL pointer inputs");
-    ok (flux_subprocess_stream_stop (NULL, NULL) < 0
-        && errno == EINVAL,
-        "flux_subprocess_stream_stop fails with NULL pointer inputs");
-    ok (flux_subprocess_stream_status (NULL, NULL) < 0
-        && errno == EINVAL,
-        "flux_subprocess_stream_status fails with NULL pointer inputs");
+    lives_ok ({flux_subprocess_stream_start (NULL, NULL);},
+        "flux_subprocess_stream_start doesn't crash with NULL pointer inputs");
+    lives_ok ({flux_subprocess_stream_stop (NULL, NULL);},
+        "flux_subprocess_stream_stop doesn't crash with NULL pointer inputs");
 
     ok (flux_subprocess_write (NULL, "stdin", "foo", 3) < 0
         && errno == EINVAL,
-        "flux_subprocess_write fails with NULL pointer inputs");
+        "flux_subprocess_write fails with NULL pointer input");
     ok (flux_subprocess_close (NULL, "stdin") < 0
         && errno == EINVAL,
-        "flux_subprocess_close fails with NULL pointer inputs");
-    ok (flux_subprocess_read (NULL, "stdout", -1, NULL) == NULL
+        "flux_subprocess_close fails with NULL pointer input");
+    ok (flux_subprocess_read (NULL, "stdout", NULL) < 0
         && errno == EINVAL,
         "flux_subprocess_read fails with NULL pointer inputs");
-    ok (flux_subprocess_read_line (NULL, "stdout", NULL) == NULL
+    ok (flux_subprocess_read_line (NULL, "stdout", NULL) < 0
         && errno == EINVAL,
         "flux_subprocess_read_line fails with NULL pointer inputs");
-    ok (flux_subprocess_read_trimmed_line (NULL, "stdout", NULL) == NULL
+    ok (flux_subprocess_read_trimmed_line (NULL, "stdout", NULL) < 0
         && errno == EINVAL,
         "flux_subprocess_read_trimmed_line fails with NULL pointer inputs");
-    ok (flux_subprocess_read_stream_closed (NULL, "stdout") < 0
-        && errno == EINVAL,
-        "flux_subprocess_read_stream_closed fails with NULL pointer inputs");
+    ok (flux_subprocess_read_stream_closed (NULL, "stdout") == false,
+        "flux_subprocess_read_stream_closed returns false with NULL pointer input");
     ok (flux_subprocess_kill (NULL, 0) == NULL
         && errno == EINVAL,
-        "flux_subprocess_kill fails with NULL pointer inputs");
+        "flux_subprocess_kill fails with NULL pointer input");
     ok ((int)flux_subprocess_state (NULL) < 0
         && errno == EINVAL,
-        "flux_subprocess_state fails with NULL pointer inputs");
+        "flux_subprocess_state fails with NULL pointer input");
     ok (flux_subprocess_rank (NULL) < 0
         && errno == EINVAL,
-        "flux_subprocess_rank fails with NULL pointer inputs");
+        "flux_subprocess_rank fails with NULL pointer input");
     ok (flux_subprocess_fail_errno (NULL) < 0
         && errno == EINVAL,
-        "flux_subprocess_fail_errno fails with NULL pointer inputs");
+        "flux_subprocess_fail_errno fails with NULL pointer input");
+    ok (flux_subprocess_fail_error (NULL) != NULL,
+        "flux_subprocess_fail_error works with NULL pointer input");
     ok (flux_subprocess_status (NULL) < 0
         && errno == EINVAL,
-        "flux_subprocess_status fails with NULL pointer inputs");
+        "flux_subprocess_status fails with NULL pointer input");
     ok (flux_subprocess_exit_code (NULL) < 0
         && errno == EINVAL,
-        "flux_subprocess_exit_code fails with NULL pointer inputs");
+        "flux_subprocess_exit_code fails with NULL pointer input");
     ok (flux_subprocess_signaled (NULL) < 0
         && errno == EINVAL,
-        "flux_subprocess_signaled fails with NULL pointer inputs");
+        "flux_subprocess_signaled fails with NULL pointer input");
     ok (flux_subprocess_pid (NULL) < 0
         && errno == EINVAL,
-        "flux_subprocess_pid fails with NULL pointer inputs");
+        "flux_subprocess_pid fails with NULL pointer input");
     ok (flux_subprocess_get_cmd (NULL) == NULL
         && errno == EINVAL,
-        "flux_subprocess_get_cmd fails with NULL pointer inputs");
+        "flux_subprocess_get_cmd fails with NULL pointer input");
     ok (flux_subprocess_get_reactor (NULL) == NULL
         && errno == EINVAL,
-        "flux_subprocess_get_reactor fails with NULL pointer inputs");
+        "flux_subprocess_get_reactor fails with NULL pointer input");
     ok (flux_subprocess_aux_set (NULL, "foo", "bar", NULL) < 0
         && errno == EINVAL,
-        "flux_subprocess_aux_set fails with NULL pointer inputs");
+        "flux_subprocess_aux_set fails with NULL pointer input");
     ok (flux_subprocess_aux_get (NULL, "foo") == NULL
         && errno == EINVAL,
-        "flux_subprocess_aux_get fails with NULL pointer inputs");
+        "flux_subprocess_aux_get fails with NULL pointer input");
+
+    ok ((cmd = flux_cmd_create (1, avgood, NULL)) != NULL,
+        "flux_cmd_create with true works");
+    ok (flux_local_exec (r,
+                         FLUX_SUBPROCESS_FLAGS_LOCAL_UNBUF,
+                         cmd,
+                         NULL) == NULL
+        && errno == EINVAL,
+        "flux_local_exec fails with invalid flag");
+    flux_cmd_destroy (cmd);
 
     flux_close (h);
 }
 
-void test_errors (flux_reactor_t *r)
+void test_post_exec_errors (flux_reactor_t *r)
 {
-    char *av[] = { "/bin/true", NULL };
+    char *av[] = { "true", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
 
@@ -288,20 +198,11 @@ void test_errors (flux_reactor_t *r)
         .on_completion = completion_cb
     };
     completion_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    p = flux_local_exec (r, 0, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
         "subprocess state == RUNNING after flux_local_exec");
-    ok (flux_subprocess_stream_start (p, NULL) < 0
-        && errno == EINVAL,
-        "flux_subprocess_stream_start returns EINVAL on bad stream");
-    ok (flux_subprocess_stream_stop (p, NULL) < 0
-        && errno == EINVAL,
-        "flux_subprocess_stream_stop returns EINVAL on bad stream");
-    ok (flux_subprocess_stream_status (p, NULL) < 0
-        && errno == EINVAL,
-        "flux_subprocess_stream_status returns EINVAL on bad stream");
     ok (flux_subprocess_write (p, NULL, NULL, 0) < 0
         && errno == EINVAL,
         "flux_subprocess_write returns EINVAL on bad input");
@@ -311,21 +212,20 @@ void test_errors (flux_reactor_t *r)
     ok (flux_subprocess_close (p, "foo") < 0
         && errno == EINVAL,
         "flux_subprocess_close returns EINVAL on bad stream");
-    ok (flux_subprocess_read (p, NULL, 0, NULL) == NULL
+    ok (flux_subprocess_read (p, NULL, NULL) < 0
         && errno == EINVAL,
         "flux_subprocess_read returns EINVAL on bad input");
-    ok (flux_subprocess_read (p, "foo", -1, NULL) == NULL
+    ok (flux_subprocess_read (p, "foo", NULL) < 0
         && errno == EINVAL,
         "flux_subprocess_read returns EINVAL on bad stream");
-    ok (flux_subprocess_read_line (p, "foo", NULL) == NULL
+    ok (flux_subprocess_read_line (p, "foo", NULL) < 0
         && errno == EINVAL,
         "flux_subprocess_read_line returns EINVAL on bad stream");
-    ok (flux_subprocess_read_trimmed_line (p, "foo", NULL) == NULL
+    ok (flux_subprocess_read_trimmed_line (p, "foo", NULL) < 0
         && errno == EINVAL,
         "flux_subprocess_read_trimmed_line returns EINVAL on bad stream");
-    ok (flux_subprocess_read_stream_closed (p, "foo") < 0
-        && errno == EINVAL,
-        "flux_subprocess_read_stream_closed returns EINVAL on bad stream");
+    ok (flux_subprocess_read_stream_closed (p, "foo") == false,
+        "flux_subprocess_read_stream_closed returns false on bad stream");
     ok (flux_subprocess_kill (p, 0) == NULL
         && errno == EINVAL,
         "flux_subprocess_kill returns EINVAL on illegal signum");
@@ -333,6 +233,8 @@ void test_errors (flux_reactor_t *r)
         "flux_subprocess_rank fails b/c subprocess is local");
     ok (flux_subprocess_fail_errno (p) < 0,
         "subprocess fail errno fails b/c subprocess not failed");
+    ok (flux_subprocess_fail_error (p) != NULL,
+        "subprocess fail error works when subprocess not failed");
     ok (flux_subprocess_status (p) < 0,
         "subprocess status fails b/c subprocess not yet exited");
     ok (flux_subprocess_exit_code (p) < 0,
@@ -352,128 +254,89 @@ void test_errors (flux_reactor_t *r)
     flux_cmd_destroy (cmd);
 }
 
-void output_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    char cmpbuf[1024];
-    int lenp = 0;
-    int *counter;
-
-    if (!strcasecmp (stream, "stdout"))
-        counter = &stdout_output_cb_count;
-    else if (!strcasecmp (stream, "stderr"))
-        counter = &stderr_output_cb_count;
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    if ((*counter) == 0) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read_line on %s success", stream);
-
-        sprintf (cmpbuf, "%s:hi\n", stream);
-
-        ok (!strcmp (ptr, cmpbuf),
-            "flux_subprocess_read_line returned correct data");
-        /* 1 + 2 + 1 for ':', "hi", '\n' */
-        ok (lenp == (strlen (stream) + 1 + 2 + 1),
-            "flux_subprocess_read_line returned correct data len");
-    }
-    else {
-        ok (flux_subprocess_read_stream_closed (p, stream) > 0,
-            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
-
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
-
-    (*counter)++;
-}
-
-void test_basic_stdout (flux_reactor_t *r)
+void test_basic (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "hi", NULL };
-    flux_cmd_t *cmd;
+    char *av[] = { "true", NULL };
+    flux_cmd_t *cmd, *cmd2;
+    flux_reactor_t *r2;
     flux_subprocess_t *p = NULL;
 
-    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
 
     flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = output_cb
+        .on_completion = completion_cb
     };
     completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    p = flux_local_exec (r, 0, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
         "subprocess state == RUNNING after flux_local_exec");
+    ok ((flux_subprocess_pid (p) > (pid_t) 0),
+        "flux_local_exec() started pid %ld", (pid_t) flux_subprocess_pid (p));
+    ok ((cmd2 = flux_subprocess_get_cmd (p)) != NULL,
+        "flux_subprocess_get_cmd success");
+    ok ((r2 = flux_subprocess_get_reactor (p)) != NULL,
+        "flux_subprocess_get_reactor success");
+    ok (r == r2,
+        "flux_subprocess_get_reactor returns correct reactor");
 
     int rc = flux_reactor_run (r, 0);
     ok (rc == 0, "flux_reactor_run returned zero status");
     ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
-    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
     flux_subprocess_destroy (p);
     flux_cmd_destroy (cmd);
 }
 
-void test_basic_stderr (flux_reactor_t *r)
+void completion_fail_cb (flux_subprocess_t *p)
+{
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_EXITED,
+        "subprocess state == EXITED in completion handler");
+    ok (flux_subprocess_status (p) != -1,
+        "subprocess status is valid");
+    ok (flux_subprocess_exit_code (p) == 1,
+        "subprocess exit code is 1");
+    completion_fail_cb_count++;
+}
+
+void test_basic_fail (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-E", "hi", NULL };
+    char *av[] = { "false", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
 
-    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
 
     flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stderr = output_cb
+        .on_completion = completion_fail_cb
     };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    completion_fail_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
         "subprocess state == RUNNING after flux_local_exec");
-    ok ((flux_subprocess_pid (p) > (pid_t) 0),
-        "flux_local_exec() started pid %ld", (pid_t) flux_subprocess_pid (p));
 
     int rc = flux_reactor_run (r, 0);
     ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdout_output_cb_count == 0, "stdout output callback called 0 times");
-    ok (stderr_output_cb_count == 2, "stderr output callback called 2 times");
+    ok (completion_fail_cb_count == 1, "completion fail callback called 1 time");
     flux_subprocess_destroy (p);
     flux_cmd_destroy (cmd);
 }
 
-void test_basic_stdout_and_stderr (flux_reactor_t *r)
+void test_flag_no_setpgrp (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "-E", "hi", NULL };
+    char *av[] = { "true", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
 
-    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
 
     flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = output_cb,
-        .on_stderr = output_cb
+        .on_completion = completion_cb
     };
     completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    p = flux_local_exec (r, FLUX_SUBPROCESS_FLAGS_NO_SETPGRP, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
@@ -482,27 +345,23 @@ void test_basic_stdout_and_stderr (flux_reactor_t *r)
     int rc = flux_reactor_run (r, 0);
     ok (rc == 0, "flux_reactor_run returned zero status");
     ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
-    ok (stderr_output_cb_count == 2, "stderr output callback called 2 times");
     flux_subprocess_destroy (p);
     flux_cmd_destroy (cmd);
 }
 
-void test_basic_default_output (flux_reactor_t *r)
+void test_flag_fork_exec (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "-E", "hi", NULL };
+    char *av[] = { "true", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
 
-    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
 
     flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = flux_standard_output,
-        .on_stderr = flux_standard_output
+        .on_completion = completion_cb
     };
     completion_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    p = flux_local_exec (r, FLUX_SUBPROCESS_FLAGS_FORK_EXEC, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
@@ -515,2008 +374,470 @@ void test_basic_default_output (flux_reactor_t *r)
     flux_cmd_destroy (cmd);
 }
 
-void output_default_stream_cb (flux_subprocess_t *p, const char *stream)
+void env_passed_cb (flux_subprocess_t *p, const char *stream)
 {
-    const char *ptr;
-    char cmpbuf[1024];
-    int lenp = 0;
-
-    if (output_default_stream_cb_count == 0) {
-        ptr = flux_subprocess_read_line (p, "stdout", &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read_line on %s success", "stdout");
-
-        sprintf (cmpbuf, "%s:hi\n", stream);
-
-        ok (!strcmp (ptr, cmpbuf),
-            "flux_subprocess_read_line returned correct data");
-        /* 1 + 2 + 1 for ':', "hi", '\n' */
-        ok (lenp == (strlen (stream) + 1 + 2 + 1),
+    const char *buf = NULL;
+    int len;
+
+    ok (!strcasecmp (stream, "stdout"),
+        "env_passed_cb called with correct stream");
+
+    if (!env_passed_cb_count) {
+        len = flux_subprocess_read_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_line on %s success", stream);
+
+        ok (strstarts (buf, "FOOBAR=foobaz"),
+            "environment variable FOOBAR in subprocess");
+        ok (len == 14,
             "flux_subprocess_read_line returned correct data len");
     }
     else {
-        ok (flux_subprocess_read_stream_closed (p, stream) > 0,
-            "flux_subprocess_read_stream_closed saw EOF on %s", "stdout");
-
-        ptr = flux_subprocess_read (p, "stdout", -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", "stdout");
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
     }
 
-    output_default_stream_cb_count++;
+    env_passed_cb_count++;
 }
 
-void test_basic_stdin (flux_reactor_t *r)
+void test_env_passed (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O",  NULL };
+    char *av[] = { "/usr/bin/env", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
 
-    ok ((cmd = flux_cmd_create (3, av, environ)) != NULL, "flux_cmd_create");
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
+
+    ok (flux_cmd_setenvf (cmd, 1, "FOOBAR", "foobaz") == 0,
+        "flux_cmd_setenvf");
 
     flux_subprocess_ops_t ops = {
         .on_completion = completion_cb,
-        .on_stdout = output_cb
+        .on_stdout = env_passed_cb
     };
     completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    env_passed_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
         "subprocess state == RUNNING after flux_local_exec");
 
-    ok (flux_subprocess_write (p, "stdin", "hi", 2) == 2,
-        "flux_subprocess_write success");
-
-    ok (flux_subprocess_close (p, "stdin") == 0,
-        "flux_subprocess_close success");
-
     int rc = flux_reactor_run (r, 0);
     ok (rc == 0, "flux_reactor_run returned zero status");
     ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
+    ok (env_passed_cb_count == 2, "channel fd callback called 2 times");
     flux_subprocess_destroy (p);
     flux_cmd_destroy (cmd);
 }
 
-void output_no_newline_cb (flux_subprocess_t *p, const char *stream)
+void completion_sigterm_cb (flux_subprocess_t *p)
 {
-    const char *ptr;
-    char cmpbuf[1024];
-    int lenp = 0;
-    int *counter;
-
-    if (!strcasecmp (stream, "stdout"))
-        counter = &stdout_output_cb_count;
-    else if (!strcasecmp (stream, "stderr"))
-        counter = &stderr_output_cb_count;
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    if ((*counter) == 0) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read_line on %s read 0 lines", stream);
-
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read on %s read success", stream);
-
-        sprintf (cmpbuf, "%s:hi", stream);
-
-        ok (!strcmp (ptr, cmpbuf),
-            "flux_subprocess_read returned correct data");
-        /* 1 + 2 + 1 for ':', "hi" */
-        ok (lenp == (strlen (stream) + 1 + 2),
-            "flux_subprocess_read_line returned correct data len");
-    }
-    else {
-        ok (flux_subprocess_read_stream_closed (p, stream) > 0,
-            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_EXITED,
+        "subprocess state == EXITED in completion handler");
+    ok (flux_subprocess_status (p) != -1,
+        "subprocess status is valid");
+    ok (flux_subprocess_signaled (p) == SIGTERM,
+        "subprocess terminated by SIGTERM");
+    flux_reactor_stop (flux_subprocess_get_reactor (p));
+    completion_sigterm_cb_count++;
 
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
+    errno = 0;
+    ok (flux_subprocess_kill (p, SIGTERM) == NULL && errno == ESRCH,
+        "flux_subprocess_kill fails with ESRCH");
 
-    (*counter)++;
 }
 
-void test_basic_no_newline (flux_reactor_t *r)
+void test_kill (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "-E", "-n", "hi", NULL };
-    flux_cmd_t *cmd;
+    char *av[]  = { "/bin/sleep", "1000", NULL };
+    flux_cmd_t *cmd = NULL;
     flux_subprocess_t *p = NULL;
+    flux_future_t *f = NULL;
 
-    ok ((cmd = flux_cmd_create (6, av, environ)) != NULL, "flux_cmd_create");
+    ok ((cmd = flux_cmd_create (2, av, NULL)) != NULL, "flux_cmd_create");
 
     flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = output_no_newline_cb,
-        .on_stderr = output_no_newline_cb
+        .on_completion = completion_sigterm_cb
     };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    completion_sigterm_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
         "subprocess state == RUNNING after flux_local_exec");
+    f = flux_subprocess_kill (p, SIGTERM);
+    ok (f != NULL, "flux_subprocess_kill returns future_t");
+    ok (flux_future_wait_for (f, 0.) == 0,
+        "future fulfilled immediately for local process");
+
+    ok (flux_future_get (f, NULL) == 0, "flux_future_get (f) returns 0");
+    ok (flux_reactor_run (r, 0) == 0, "reactor_run exits normally");
+    ok (completion_sigterm_cb_count == 1, "completion sigterm callback called 1 time");
 
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
-    ok (stderr_output_cb_count == 2, "stderr output callback called 2 times");
     flux_subprocess_destroy (p);
+    flux_future_destroy (f);
     flux_cmd_destroy (cmd);
 }
 
-void output_trimmed_line_cb (flux_subprocess_t *p, const char *stream)
+void output_processes_cb (flux_subprocess_t *p, const char *stream)
 {
-    const char *ptr;
-    char cmpbuf[1024];
-    int lenp = 0;
-    int *counter;
+    const char *buf = NULL;
+    int len;
 
-    if (!strcasecmp (stream, "stdout"))
-        counter = &stdout_output_cb_count;
-    else if (!strcasecmp (stream, "stderr"))
-        counter = &stderr_output_cb_count;
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    if ((*counter) == 0) {
-        ptr = flux_subprocess_read_trimmed_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read_trimmed_line on %s success", stream);
+    if (output_processes_cb_count == 0
+        || output_processes_cb_count == 1) {
+        len = flux_subprocess_read_trimmed_line (p, stream, &buf);
+        ok (len > 0
+            && buf != NULL,
+            "flux_subprocess_read_trimmed_line read valid data");
 
-        sprintf (cmpbuf, "%s:hi", stream);
+        if (len > 0 && buf) {
+            if (output_processes_cb_count == 0)
+                parent_pid = atoi (buf);
+            else
+                child_pid = atoi (buf);
+        }
 
-        ok (!strcmp (ptr, cmpbuf),
-            "flux_subprocess_read_trimmed_line returned correct data");
+        if (output_processes_cb_count == 1) {
+            flux_future_t *f = NULL;
+            f = flux_subprocess_kill (p, SIGTERM);
+            ok (f != NULL, "flux_subprocess_kill returns future_t");
+            /* ignore response, we're not going to wait for it */
+            flux_future_destroy (f);
+        }
     }
     else {
-        ok (flux_subprocess_read_stream_closed (p, stream) > 0,
+        ok (flux_subprocess_read_stream_closed (p, stream),
             "flux_subprocess_read_stream_closed saw EOF on %s", stream);
 
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
             "flux_subprocess_read on %s read EOF", stream);
     }
 
-    (*counter)++;
-}
-
-void test_basic_trimmed_line (flux_reactor_t *r)
-{
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "-E", "hi", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = output_trimmed_line_cb,
-        .on_stderr = output_trimmed_line_cb
-    };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
-    ok (stderr_output_cb_count == 2, "stderr output callback called 2 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
+    output_processes_cb_count++;
 }
 
-void multiple_lines_output_cb (flux_subprocess_t *p, const char *stream)
+int wait_kill (int pid)
 {
-    const char *ptr;
-    int lenp = 0;
-    int *counter;
-
-    if (!strcasecmp (stream, "stdout"))
-        counter = &multiple_lines_stdout_output_cb_count;
-    else if (!strcasecmp (stream, "stderr"))
-        counter = &multiple_lines_stderr_output_cb_count;
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    if ((*counter) == 0) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!strcmp (ptr, "foo\n"),
-            "flux_subprocess_read_line returned correct data");
-        ok (lenp == 4,
-            "flux_subprocess_read_line returned correct data len");
-    }
-    else if ((*counter) == 1) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!strcmp (ptr, "bar\n"),
-            "flux_subprocess_read_line returned correct data");
-        ok (lenp == 4,
-            "flux_subprocess_read_line returned correct data len");
-    }
-    else if ((*counter) == 2) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!strcmp (ptr, "bo\n"),
-            "flux_subprocess_read_line returned correct data");
-        ok (lenp == 3,
-            "flux_subprocess_read_line returned correct data len");
-    }
-    else {
-        ok (flux_subprocess_read_stream_closed (p, stream) > 0,
-            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+    int ret;
+    int i;
 
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
+    /* we'll try for at most a second (10 * .1 seconds) */
+    ret = kill (parent_pid, 0);
+    for (i = 0; i < 10 && ret == 0; i++) {
+        usleep (100000);
+        ret = kill (parent_pid, 0);
     }
-
-    (*counter)++;
+    return ret;
 }
 
-void test_basic_multiple_lines (flux_reactor_t *r)
+void test_kill_setpgrp (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "-E", "-n", NULL };
-    flux_cmd_t *cmd;
+    char *av[]  = { TEST_SUBPROCESS_DIR "test_fork_sleep", "100", NULL };
+    flux_cmd_t *cmd = NULL;
     flux_subprocess_t *p = NULL;
+    int ret;
 
-    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
+    ok ((cmd = flux_cmd_create (2, av, environ)) != NULL, "flux_cmd_create");
 
     flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = multiple_lines_output_cb,
-        .on_stderr = multiple_lines_output_cb
+        .on_completion = completion_sigterm_cb,
+        .on_stdout = output_processes_cb,
     };
-    completion_cb_count = 0;
-    multiple_lines_stdout_output_cb_count = 0;
-    multiple_lines_stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    completion_sigterm_cb_count = 0;
+    output_processes_cb_count = 0;
+    parent_pid = -1;
+    child_pid = -1;
+    p = flux_local_exec (r, 0, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
         "subprocess state == RUNNING after flux_local_exec");
 
-    ok (flux_subprocess_write (p, "stdin", "foo\n", 4) == 4,
-        "flux_subprocess_write success");
-
-    ok (flux_subprocess_write (p, "stdin", "bar\n", 4) == 4,
-        "flux_subprocess_write success");
-
-    ok (flux_subprocess_write (p, "stdin", "bo\n", 3) == 3,
-        "flux_subprocess_write success");
-
-    ok (flux_subprocess_close (p, "stdin") == 0,
-        "flux_subprocess_close success");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (multiple_lines_stdout_output_cb_count == 4, "stdout output callback called 4 times");
-    ok (multiple_lines_stderr_output_cb_count == 4, "stderr output callback called 4 times");
+    ok (flux_reactor_run (r, 0) == 0, "reactor_run exits normally");
+    ok (completion_sigterm_cb_count == 1, "completion sigterm callback called 1 time");
+    ok (output_processes_cb_count == 3, "output processes callback called 3 times");
+    /* checking if a pid has been killed at this point is a tad racy,
+     * so if necessary loop a second to wait for the kill to happen
+     */
+    ret = wait_kill (parent_pid);
+    ok (ret < 0
+        && errno == ESRCH,
+        "kill fails with ESRCH, parent pid killed %d", parent_pid);
+    ret = wait_kill (child_pid);
+    ok (ret < 0
+        && errno == ESRCH,
+        "kill fails with ESRCH, child pid killed %d", child_pid);
     flux_subprocess_destroy (p);
     flux_cmd_destroy (cmd);
 }
 
-void stdin_closed_cb (flux_subprocess_t *p, const char *stream)
+void eof_cb (flux_subprocess_t *p, const char *stream)
 {
-    const char *ptr;
-    int lenp = 0;
+    const char *buf = NULL;
+    int len;
     int *counter;
 
     if (!strcasecmp (stream, "stdout"))
-        counter = &stdin_closed_stdout_cb_count;
+        counter = &stdout_eof_cb_count;
     else if (!strcasecmp (stream, "stderr"))
-        counter = &stdin_closed_stderr_cb_count;
+        counter = &stderr_eof_cb_count;
     else {
         ok (false, "unexpected stream %s", stream);
         return;
     }
 
-    ok (flux_subprocess_read_stream_closed (p, stream) > 0,
+    ok (flux_subprocess_read_stream_closed (p, stream),
         "flux_subprocess_read_stream_closed saw EOF on %s", stream);
 
-    ptr = flux_subprocess_read (p, stream, -1, &lenp);
-    ok (ptr != NULL
-        && lenp == 0,
+    len = flux_subprocess_read (p, stream, &buf);
+    ok (len == 0,
         "flux_subprocess_read on %s read EOF", stream);
 
     (*counter)++;
 }
 
-void test_basic_stdin_closed (flux_reactor_t *r)
-{
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "-E", "-n", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = stdin_closed_cb,
-        .on_stderr = stdin_closed_cb
-    };
-    completion_cb_count = 0;
-    stdin_closed_stdout_cb_count = 0;
-    stdin_closed_stderr_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    ok (flux_subprocess_close (p, "stdin") == 0,
-        "flux_subprocess_close success");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdin_closed_stdout_cb_count == 1, "stdout output callback called 1 time");
-    ok (stdin_closed_stderr_cb_count == 1, "stderr output callback called 1 time");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void output_read_line_until_eof_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-    int *counter;
-
-    if (!strcasecmp (stream, "stdout"))
-        counter = &stdout_output_cb_count;
-    else if (!strcasecmp (stream, "stderr"))
-        counter = &stderr_output_cb_count;
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    ptr = flux_subprocess_getline (p, stream, &lenp);
-    if ((*counter) == 0) {
-        ok (ptr != NULL,
-            "flux_subprocess_getline on %s success", stream);
-        ok (!strcmp (ptr, "foo\n"),
-            "flux_subprocess_getline returned correct data");
-        ok (lenp == 4,
-            "flux_subprocess_getline returned correct data len");
-    }
-    else if ((*counter) == 1) {
-        ok (ptr != NULL,
-            "flux_subprocess_getline on %s success", stream);
-        ok (!strcmp (ptr, "bar"),
-            "flux_subprocess_getline returned correct data");
-        ok (lenp == 3,
-            "flux_subprocess_getline returned correct data len");
-    }
-    else {
-        ok (ptr != NULL,
-            "flux_subprocess_getline on %s success", stream);
-        ok (lenp == 0,
-            "flux_subprocess_getline returned EOF");
-    }
-
-    (*counter)++;
-}
-
-void test_basic_read_line_until_eof (flux_reactor_t *r)
-{
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "-E", "-n", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = output_read_line_until_eof_cb,
-        .on_stderr = output_read_line_until_eof_cb
-    };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    ok (flux_subprocess_write (p, "stdin", "foo\n", 4) == 4,
-        "flux_subprocess_write success");
-
-    ok (flux_subprocess_write (p, "stdin", "bar", 3) == 3,
-        "flux_subprocess_write success");
-
-    ok (flux_subprocess_close (p, "stdin") == 0,
-        "flux_subprocess_close success");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdout_output_cb_count == 3, "stdout output callback called 3 times");
-    ok (stderr_output_cb_count == 3, "stderr output callback called 3 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void output_read_line_until_eof_error_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-    int *counter;
-
-    if (!strcasecmp (stream, "stdout"))
-        counter = &stdout_output_cb_count;
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    if ((*counter) == 0) {
-        ptr = flux_subprocess_getline (p, stream, &lenp);
-        ok (!ptr && errno == EPERM,
-            "flux_subprocess_getline returns EPERM "
-            "on non line-buffered stream");
-
-        /* drain whatever is in the buffer, we don't care about
-         * contents for this test */
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL && lenp > 0,
-            "flux_subprocess_read on %s success", stream);
-    }
-    else {
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
-    (*counter)++;
-}
-
-void test_basic_read_line_until_eof_error (flux_reactor_t *r)
+void test_kill_eofs (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "hi", NULL };
-    flux_cmd_t *cmd;
+    char *av[]  = { "/bin/sleep", "1000", NULL };
+    flux_cmd_t *cmd = NULL;
     flux_subprocess_t *p = NULL;
+    flux_future_t *f = NULL;
 
-    ok ((cmd = flux_cmd_create (3, av, environ)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "false") == 0,
-        "flux_cmd_setopt set stdout_LINE_BUFFER success");
+    ok ((cmd = flux_cmd_create (2, av, NULL)) != NULL, "flux_cmd_create");
 
     flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = output_read_line_until_eof_error_cb,
-        .on_stderr = NULL
+        .on_completion = completion_sigterm_cb,
+        .on_stdout = eof_cb,
+        .on_stderr = eof_cb,
     };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    completion_sigterm_cb_count = 0;
+    stdout_eof_cb_count = 0;
+    stderr_eof_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
         "subprocess state == RUNNING after flux_local_exec");
+    f = flux_subprocess_kill (p, SIGTERM);
+    ok (f != NULL, "flux_subprocess_kill returns future_t");
+    ok (flux_future_wait_for (f, 0.) == 0,
+        "future fulfilled immediately for local process");
 
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
-    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
+    ok (flux_future_get (f, NULL) == 0, "flux_future_get (f) returns 0");
+    ok (flux_reactor_run (r, 0) == 0, "reactor_run exits normally");
+    ok (completion_sigterm_cb_count == 1, "completion sigterm callback called 1 time");
+    ok (stdout_eof_cb_count == 1, "stdout eof callback called 1 times");
+    ok (stderr_eof_cb_count == 1, "stderr eof callback called 1 times");
     flux_subprocess_destroy (p);
+    flux_future_destroy (f);
     flux_cmd_destroy (cmd);
 }
 
-void test_write_after_close (flux_reactor_t *r)
+void state_change_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-O", "-E", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (3, av, environ)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = output_cb
-    };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    ok (flux_subprocess_write (p, "stdin", "hi", 2) == 2,
-        "flux_subprocess_write success");
-
-    ok (flux_subprocess_close (p, "stdin") == 0,
-        "flux_subprocess_close success");
-
-    ok (flux_subprocess_write (p, "stdin", "hi", 2) < 0
-        && errno == EPIPE,
-        "flux_subprocess_write failed with EPIPE after a close");
-
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-#if 0
-/* disable test.  libtap has an issue with fallthrough
- * stdout/stderr in forked process */
-void test_flag_stdio_fallthrough (flux_reactor_t *r)
-{
-    char *av[] = { "echo", "foo", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (2, av, NULL)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb
-    };
-    completion_cb_count = 0;
-    p = flux_local_exec (r,
-                         FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH,
-                         cmd,
-                         &ops,
-                         NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-#endif
-
-void test_flag_setpgrp (flux_reactor_t *r)
-{
-    char *av[] = { "/bin/true", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb
-    };
-    completion_cb_count = 0;
-    p = flux_local_exec (r, FLUX_SUBPROCESS_FLAGS_SETPGRP, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void env_passed_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-
-    ok (!strcasecmp (stream, "stdout"),
-        "env_passed_cb called with correct stream");
-
-    if (!env_passed_cb_count) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr
-            && lenp > 0,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!strncmp (ptr, "FOOBAR=foobaz", 13),
-            "environment variable FOOBAR in subprocess");
-        ok (lenp == 14,
-            "flux_subprocess_read_line returned correct data len");
-    }
-    else {
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
-
-    env_passed_cb_count++;
-}
-
-void test_env_passed (flux_reactor_t *r)
-{
-    char *av[] = { "/usr/bin/env", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_setenvf (cmd, 1, "FOOBAR", "foobaz") == 0,
-        "flux_cmd_setenvf");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = env_passed_cb
-    };
-    completion_cb_count = 0;
-    env_passed_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (env_passed_cb_count == 2, "channel fd callback called 2 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void completion_sigterm_cb (flux_subprocess_t *p)
-{
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_EXITED,
-        "subprocess state == EXITED in completion handler");
-    ok (flux_subprocess_status (p) != -1,
-        "subprocess status is valid");
-    ok (flux_subprocess_signaled (p) == SIGTERM,
-        "subprocess terminated by SIGTERM");
-    flux_reactor_stop (flux_subprocess_get_reactor (p));
-    completion_sigterm_cb_count++;
-}
-
-void test_kill (flux_reactor_t *r)
-{
-    char *av[]  = { "/bin/sleep", "1000", NULL };
-    flux_cmd_t *cmd = NULL;
-    flux_subprocess_t *p = NULL;
-    flux_future_t *f = NULL;
-
-    ok ((cmd = flux_cmd_create (2, av, NULL)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_sigterm_cb
-    };
-    completion_sigterm_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-    f = flux_subprocess_kill (p, SIGTERM);
-    ok (f != NULL, "flux_subprocess_kill returns future_t");
-    ok (flux_future_wait_for (f, 0.) == 0,
-        "future fulfilled immediately for local process");
-
-    ok (flux_future_get (f, NULL) == 0, "flux_future_get (f) returns 0");
-    ok (flux_reactor_run (r, 0) == 0, "reactor_run exits normally");
-    ok (completion_sigterm_cb_count == 1, "completion sigterm callback called 1 time");
-    flux_subprocess_destroy (p);
-    flux_future_destroy (f);
-    flux_cmd_destroy (cmd);
-}
-
-void output_processes_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp;
-
-    if (output_processes_cb_count == 0
-        || output_processes_cb_count == 1) {
-        ptr = flux_subprocess_read_trimmed_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read_trimmed_line read valid data");
-
-        if (ptr && lenp) {
-            if (output_processes_cb_count == 0)
-                parent_pid = atoi (ptr);
-            else
-                child_pid = atoi (ptr);
-        }
-
-        if (output_processes_cb_count == 1) {
-            flux_future_t *f = NULL;
-            f = flux_subprocess_kill (p, SIGTERM);
-            ok (f != NULL, "flux_subprocess_kill returns future_t");
-            /* ignore response, we're not going to wait for it */
-            flux_future_destroy (f);
-        }
-    }
-    else {
-        ok (flux_subprocess_read_stream_closed (p, stream) > 0,
-            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
-
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
-
-    output_processes_cb_count++;
-}
-
-int wait_kill (int pid)
-{
-    int ret;
-    int i;
-
-    /* we'll try for at most a second (10 * .1 seconds) */
-    ret = kill (parent_pid, 0);
-    for (i = 0; i < 10 && ret == 0; i++) {
-        usleep (100000);
-        ret = kill (parent_pid, 0);
-    }
-    return ret;
-}
-
-void test_kill_setpgrp (flux_reactor_t *r)
-{
-    char *av[]  = { TEST_SUBPROCESS_DIR "test_fork_sleep", "100", NULL };
-    flux_cmd_t *cmd = NULL;
-    flux_subprocess_t *p = NULL;
-    int ret;
-
-    ok ((cmd = flux_cmd_create (2, av, environ)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_sigterm_cb,
-        .on_stdout = output_processes_cb,
-    };
-    completion_sigterm_cb_count = 0;
-    output_processes_cb_count = 0;
-    parent_pid = -1;
-    child_pid = -1;
-    p = flux_local_exec (r, FLUX_SUBPROCESS_FLAGS_SETPGRP, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    ok (flux_reactor_run (r, 0) == 0, "reactor_run exits normally");
-    ok (completion_sigterm_cb_count == 1, "completion sigterm callback called 1 time");
-    ok (output_processes_cb_count == 3, "output processes callback called 3 times");
-    /* checking if a pid has been killed at this point is a tad racy,
-     * so if necessary loop a second to wait for the kill to happen
-     */
-    ret = wait_kill (parent_pid);
-    ok (ret < 0
-        && errno == ESRCH,
-        "kill fails with ESRCH, parent pid killed %d", parent_pid);
-    ret = wait_kill (child_pid);
-    ok (ret < 0
-        && errno == ESRCH,
-        "kill fails with ESRCH, child pid killed %d", child_pid);
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void eof_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-    int *counter;
-
-    if (!strcasecmp (stream, "stdout"))
-        counter = &stdout_eof_cb_count;
-    else if (!strcasecmp (stream, "stderr"))
-        counter = &stderr_eof_cb_count;
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    ok (flux_subprocess_read_stream_closed (p, stream) > 0,
-        "flux_subprocess_read_stream_closed saw EOF on %s", stream);
-
-    ptr = flux_subprocess_read (p, stream, -1, &lenp);
-    ok (ptr != NULL
-        && lenp == 0,
-        "flux_subprocess_read on %s read EOF", stream);
-
-    (*counter)++;
-}
-
-void test_kill_eofs (flux_reactor_t *r)
-{
-    char *av[]  = { "/bin/sleep", "1000", NULL };
-    flux_cmd_t *cmd = NULL;
-    flux_subprocess_t *p = NULL;
-    flux_future_t *f = NULL;
-
-    ok ((cmd = flux_cmd_create (2, av, NULL)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_sigterm_cb,
-        .on_stdout = eof_cb,
-        .on_stderr = eof_cb,
-    };
-    completion_sigterm_cb_count = 0;
-    stdout_eof_cb_count = 0;
-    stderr_eof_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-    f = flux_subprocess_kill (p, SIGTERM);
-    ok (f != NULL, "flux_subprocess_kill returns future_t");
-    ok (flux_future_wait_for (f, 0.) == 0,
-        "future fulfilled immediately for local process");
-
-    ok (flux_future_get (f, NULL) == 0, "flux_future_get (f) returns 0");
-    ok (flux_reactor_run (r, 0) == 0, "reactor_run exits normally");
-    ok (completion_sigterm_cb_count == 1, "completion sigterm callback called 1 time");
-    ok (stdout_eof_cb_count == 1, "stdout eof callback called 1 times");
-    ok (stderr_eof_cb_count == 1, "stderr eof callback called 1 times");
-    flux_subprocess_destroy (p);
-    flux_future_destroy (f);
-    flux_cmd_destroy (cmd);
-}
-
-void state_change_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
-{
-    if (state_change_cb_count == 0)
-        ok (state == FLUX_SUBPROCESS_RUNNING,
-            "subprocess state == RUNNING in state change handler");
-    else
-        ok (state == FLUX_SUBPROCESS_EXITED,
-            "subprocess state == EXITED in state change handler");
-    state_change_cb_count++;
-}
-
-void test_state_change (flux_reactor_t *r)
-{
-    char *av[] = { "/bin/true", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_state_change = state_change_cb
-    };
-    completion_cb_count = 0;
-    state_change_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (state_change_cb_count == 2, "state change callback called 3 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void test_state_strings (void)
-{
-    ok (!strcasecmp (flux_subprocess_state_string (FLUX_SUBPROCESS_INIT), "Init"),
-        "flux_subprocess_state_string returns correct string");
-    ok (!strcasecmp (flux_subprocess_state_string (FLUX_SUBPROCESS_RUNNING), "Running"),
-        "flux_subprocess_state_string returns correct string");
-    ok (!strcasecmp (flux_subprocess_state_string (FLUX_SUBPROCESS_EXITED), "Exited"),
-        "flux_subprocess_state_string returns correct string");
-    ok (!strcasecmp (flux_subprocess_state_string (FLUX_SUBPROCESS_EXEC_FAILED), "Exec Failed"),
-        "flux_subprocess_state_string returns correct string");
-    ok (!flux_subprocess_state_string (100),
-        "flux_subprocess_state_string returns NULL on bad state");
-}
-
-void test_exec_fail (flux_reactor_t *r)
-{
-    char *av_eacces[]  = { "/", NULL };
-    char *av_enoent[]  = { "/usr/bin/foobarbaz", NULL };
-    flux_cmd_t *cmd = NULL;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (1, av_eacces, NULL)) != NULL, "flux_cmd_create");
-
-    p = flux_local_exec (r, 0, cmd, NULL, NULL);
-    ok (p == NULL
-        && errno == EACCES,
-        "flux_local_exec failed with EACCES");
-
-    flux_cmd_destroy (cmd);
-
-    ok ((cmd = flux_cmd_create (1, av_enoent, NULL)) != NULL, "flux_cmd_create");
-
-    p = flux_local_exec (r, 0, cmd, NULL, NULL);
-    ok (p == NULL
-        && errno == ENOENT,
-        "flux_local_exec failed with ENOENT");
-
-    flux_cmd_destroy (cmd);
-}
-
-void test_context (flux_reactor_t *r)
-{
-    char *av[] = { "/bin/true", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-    char *extra = "mydata";
-    char *tmp;
-
-    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb
-    };
-    completion_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-    ok (flux_subprocess_aux_set (p, "extra", extra, NULL) == 0,
-        "flux_subprocess_aux_set success");
-    ok ((tmp = flux_subprocess_aux_get (p, "extra")) != NULL,
-        "flux_subprocess_aux_get success");
-    ok (tmp == extra,
-        "flux_subprocess_aux_get returned correct pointer");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void test_refcount (flux_reactor_t *r)
-{
-    char *av[] = { "/bin/true", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-    char *extra = "mydata";
-    char *tmp;
-
-    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb
-    };
-    completion_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-    ok (flux_subprocess_aux_set (p, "extra", extra, NULL) == 0,
-        "flux_subprocess_aux_set success");
-    flux_subprocess_ref (p);
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    flux_subprocess_unref (p);
-
-    /* normally this should fail, but we've increased the refcount so
-     * subprocess should not be destroyed */
-    ok ((tmp = flux_subprocess_aux_get (p, "extra")) != NULL,
-        "flux_subprocess_aux_get success");
-    ok (tmp == extra,
-        "flux_subprocess_aux_get returned correct pointer");
-
-    flux_subprocess_unref (p);
-    flux_cmd_destroy (cmd);
-}
-
-void channel_fd_env_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-
-    ok (!strcasecmp (stream, "stdout"),
-        "channel_fd_env_cb called with correct stream");
-
-    if (!channel_fd_env_cb_count) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr
-            && lenp > 0,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!strncmp (ptr, "FOO=", 4),
-            "environment variable FOO created in subprocess");
-        /* no length check, can't predict channel FD value */
-    }
-    else {
-        ok (flux_subprocess_read_stream_closed (p, stream) > 0,
-            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
-
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
-
-    channel_fd_env_cb_count++;
-}
-
-void test_channel_fd_env (flux_reactor_t *r)
-{
-    char *av[] = { "/usr/bin/env", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_add_channel (cmd, "FOO") == 0,
-        "flux_cmd_add_channel success adding channel FOO");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = channel_fd_env_cb
-    };
-    completion_cb_count = 0;
-    channel_fd_env_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (channel_fd_env_cb_count == 2, "channel fd callback called 2 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void channel_in_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-
-    ok (!strcasecmp (stream, "stdout"),
-        "channel_in_cb called with correct stream");
-
-    if (!channel_in_cb_count) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp == 7,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!memcmp (ptr, "foobar\n", 7),
-            "read on channel returned correct data");
-
-        ok (flux_subprocess_close (p, "TEST_CHANNEL") == 0,
-            "flux_subprocess_close success");
-    }
-    else {
-        ok (flux_subprocess_read_stream_closed (p, stream) > 0,
-            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
-
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
-
-    channel_in_cb_count++;
-}
-
-void test_channel_fd_in (flux_reactor_t *r)
-{
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-c", "TEST_CHANNEL", "-O", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
-        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_channel_out = NULL,
-        .on_stdout = channel_in_cb,
-        .on_stderr = flux_standard_output
-    };
-    completion_cb_count = 0;
-    channel_in_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    ok (flux_subprocess_write (p, "TEST_CHANNEL", "foobar", 6) == 6,
-        "flux_subprocess_write success");
-
-    /* close after we get output */
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (channel_in_cb_count == 2, "channel in callback called 2 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void channel_in_and_out_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-
-    ok (!strcasecmp (stream, "TEST_CHANNEL"),
-        "channel_in_and_out_cb called with correct stream");
-
-    if (!channel_in_and_out_cb_count) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp == 7,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!memcmp (ptr, "foobaz\n", 7),
-            "read on channel returned correct data");
-
-        ok (flux_subprocess_close (p, "TEST_CHANNEL") == 0,
-            "flux_subprocess_close success");
-    }
-    else {
-        /* no check of flux_subprocess_read_stream_closed(), we aren't
-         * closing channel in test below */
-
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
-
-    channel_in_and_out_cb_count++;
-}
-
-void test_channel_fd_in_and_out (flux_reactor_t *r)
-{
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-c", "TEST_CHANNEL", "-C", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (4, av, environ)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
-        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_channel_out = channel_in_and_out_cb,
-        .on_stdout = flux_standard_output,
-        .on_stderr = flux_standard_output
-    };
-    completion_cb_count = 0;
-    channel_in_and_out_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    ok (flux_subprocess_write (p, "TEST_CHANNEL", "foobaz", 6) == 6,
-        "flux_subprocess_write success");
-
-    /* don't call flux_subprocess_close() here, we'll race with data
-     * coming back, call in callback */
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (channel_in_and_out_cb_count == 2, "channel out callback called 2 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void channel_multiple_lines_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-
-    ok (!strcasecmp (stream, "TEST_CHANNEL"),
-        "channel_multiple_lines_cb called with correct stream");
-
-    if (multiple_lines_channel_cb_count == 0) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!strcmp (ptr, "bob\n"),
-            "flux_subprocess_read_line returned correct data");
-    }
-    else if (multiple_lines_channel_cb_count == 1) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!strcmp (ptr, "dan\n"),
-            "flux_subprocess_read_line returned correct data");
-    }
-    else if (multiple_lines_channel_cb_count == 2) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp > 0,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!strcmp (ptr, "jo\n"),
-            "flux_subprocess_read_line returned correct data");
-
-        ok (flux_subprocess_close (p, "TEST_CHANNEL") == 0,
-            "flux_subprocess_close success");
-    }
-    else {
-        /* no check of flux_subprocess_read_stream_closed(), we aren't
-         * closing channel in test below */
-
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
-
-    multiple_lines_channel_cb_count++;
-}
-
-void test_channel_multiple_lines (flux_reactor_t *r)
-{
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-c", "TEST_CHANNEL", "-C", "-n", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
-        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_channel_out = channel_multiple_lines_cb,
-        .on_stdout = flux_standard_output,
-        .on_stderr = flux_standard_output
-    };
-    completion_cb_count = 0;
-    multiple_lines_channel_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    ok (flux_subprocess_write (p, "TEST_CHANNEL", "bob\n", 4) == 4,
-        "flux_subprocess_write success");
-
-    ok (flux_subprocess_write (p, "TEST_CHANNEL", "dan\n", 4) == 4,
-        "flux_subprocess_write success");
-
-    ok (flux_subprocess_write (p, "TEST_CHANNEL", "jo\n", 3) == 3,
-        "flux_subprocess_write success");
-
-    /* don't call flux_subprocess_close() here, we'll race with data
-     * coming back, call in callback */
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (multiple_lines_channel_cb_count == 4, "channel output callback called 4 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void channel_nul_terminate_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-
-    if (!channel_nul_terminate_cb_count) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL
-            && lenp == 7,
-            "flux_subprocess_read_line on %s success", stream);
-
-        ok (!memcmp (ptr, "foobaz\n\0", 8),
-            "read on channel returned correct data");
-
-        ok (flux_subprocess_close (p, "TEST_CHANNEL") == 0,
-            "flux_subprocess_close success");
-    }
-    else {
-        ok (flux_subprocess_read_stream_closed (p, stream) > 0,
-            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
-
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL
-            && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
-
-    channel_nul_terminate_cb_count++;
-}
-
-void test_bufsize (flux_reactor_t *r)
-{
-    char *av[] = { "/bin/true", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (1, av, environ)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
-        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
-
-    ok (flux_cmd_setopt (cmd, "stdin_BUFSIZE", "1024") == 0,
-        "flux_cmd_setopt set stdin_BUFSIZE success");
-
-    ok (flux_cmd_setopt (cmd, "stdout_BUFSIZE", "1024") == 0,
-        "flux_cmd_setopt set stdout_BUFSIZE success");
-
-    ok (flux_cmd_setopt (cmd, "stderr_BUFSIZE", "1024") == 0,
-        "flux_cmd_setopt set stderr_BUFSIZE success");
-
-    ok (flux_cmd_setopt (cmd, "TEST_CHANNEL_BUFSIZE", "1024") == 0,
-        "flux_cmd_setopt set TEST_CHANNEL_BUFSIZE success");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_channel_out = flux_standard_output,
-        .on_stdout = flux_standard_output,
-        .on_stderr = flux_standard_output
-    };
-    completion_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void test_bufsize_error (flux_reactor_t *r)
-{
-    char *av[] = { "/bin/true", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_add_channel (cmd, "TEST_CHANNEL") == 0,
-        "flux_cmd_add_channel success adding channel TEST_CHANNEL");
-
-    ok (flux_cmd_setopt (cmd, "TEST_CHANNEL_BUFSIZE", "ABCD") == 0,
-        "flux_cmd_setopt set TEST_CHANNEL_BUFSIZE success");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_channel_out = flux_standard_output,
-        .on_stdout = flux_standard_output,
-        .on_stderr = flux_standard_output
-    };
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p == NULL
-        && errno == EINVAL,
-        "flux_local_exec fails with EINVAL due to bad bufsize input");
-
-    flux_cmd_destroy (cmd);
-}
-
-/* Line buffering tests are technically racy.  If the stdout in the
- * test_multi_echo command occurs fast enough, a single on_stdout
- * callback could occur.  But hopefully by repeating the word "hi" a
- * lot of times, the probability of that occuring is zero if line
- * buffering is not working.
- *
- * I pick 2200 to make sure we output enough to surpass 4096 bytes of
- * output (i.e. 2200 * 2 bytes > 4096 bytes).
- */
-
-void line_output_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-    int *counter;
-
-    if (!strcasecmp (stream, "stdout"))
-        counter = &stdout_output_cb_count;
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    if ((*counter) == 0) {
-        ptr = flux_subprocess_read_line (p, stream, &lenp);
-        ok (ptr != NULL && lenp == 4401,
-            "flux_subprocess_read_line read line correctly");
-    }
-    else {
-        ok (flux_subprocess_read_stream_closed (p, stream) > 0,
-            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
-
-        ptr = flux_subprocess_read (p, stream, -1, &lenp);
-        ok (ptr != NULL && lenp == 0,
-            "flux_subprocess_read on %s read EOF", stream);
-    }
-
-    (*counter)++;
-}
-
-void test_line_buffer_default (flux_reactor_t *r)
-{
-    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-c", "2200", "hi", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = line_output_cb
-    };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    /* == 2 times means we got a single line and EOF */
-    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
-    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void test_line_buffer_enable (flux_reactor_t *r)
-{
-    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-c", "2200", "hi", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "true") == 0,
-        "flux_cmd_setopt set stdout_LINE_BUFFER success");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = line_output_cb
-    };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    /* == 2 times means we got a single line and EOF */
-    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
-    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void count_output_cb (flux_subprocess_t *p, const char *stream)
-{
-    int *counter;
-
-    if (!strcasecmp (stream, "stdout"))
-        counter = &stdout_output_cb_count;
-    else if (!strcasecmp (stream, "stderr"))
-        counter = &stderr_output_cb_count;
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    (void)flux_subprocess_read_line (p, stream, NULL);
-    (*counter)++;
-}
-
-void test_line_buffer_disable (flux_reactor_t *r)
-{
-    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-c", "2200", "hi", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "false") == 0,
-        "flux_cmd_setopt set stdout_LINE_BUFFER success");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = count_output_cb
-    };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    /* we care about greater than two, that it's not a single line and EOF */
-    ok (stdout_output_cb_count > 2, "stdout output callback got more than 2 calls: %d", stdout_output_cb_count);
-    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void test_line_buffer_error (flux_reactor_t *r)
-{
-    char *av[] = { "/bin/true", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "ABCD") == 0,
-        "flux_cmd_setopt set stdout_LINE_BUFFER success");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_channel_out = flux_standard_output,
-        .on_stdout = flux_standard_output,
-        .on_stderr = flux_standard_output
-    };
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p == NULL
-        && errno == EINVAL,
-        "flux_local_exec fails with EINVAL due to bad line_buffer input");
-
-    flux_cmd_destroy (cmd);
-}
-
-void test_stream_start_stop_basic (flux_reactor_t *r)
-{
-    char *av[] = { TEST_SUBPROCESS_DIR "test_echo", "-P", "-O", "-E", "hi", NULL };
-    flux_cmd_t *cmd;
-    flux_subprocess_t *p = NULL;
-
-    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
-
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = output_cb,
-        .on_stderr = output_cb,
-    };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
-
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
-
-    ok (flux_subprocess_stream_status (p, "stdout") > 0,
-        "flux_subprocess_stream_status says stdout is started");
-    ok (flux_subprocess_stream_status (p, "stderr") > 0,
-        "flux_subprocess_stream_status says stderr is started");
-
-    ok (!flux_subprocess_stream_stop (p, "stdout"),
-        "flux_subprocess_stream_stop on stdout success");
-    ok (!flux_subprocess_stream_stop (p, "stderr"),
-        "flux_subprocess_stream_stop on stderr success");
-
-    ok (flux_subprocess_stream_status (p, "stdout") == 0,
-        "flux_subprocess_stream_status says stdout is stopped");
-    ok (flux_subprocess_stream_status (p, "stderr") == 0,
-        "flux_subprocess_stream_status says stderr is stopped");
-
-    ok (!flux_subprocess_stream_start (p, "stdout"),
-        "flux_subprocess_stream_start on stdout success");
-    ok (!flux_subprocess_stream_start (p, "stderr"),
-        "flux_subprocess_stream_start on stderr success");
-
-    ok (flux_subprocess_stream_status (p, "stdout") > 0,
-        "flux_subprocess_stream_status says stdout is started");
-    ok (flux_subprocess_stream_status (p, "stderr") > 0,
-        "flux_subprocess_stream_status says stderr is started");
-
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdout_output_cb_count == 2, "stdout output callback called 2 times");
-    ok (stderr_output_cb_count == 2, "stderr output callback called 2 times");
-    flux_subprocess_destroy (p);
-    flux_cmd_destroy (cmd);
-}
-
-void start_stdout_after_stderr_cb (flux_subprocess_t *p, const char *stream)
-{
-    const char *ptr;
-    int lenp = 0;
-    int *counter;
-    int *len_counter;
-
-    if (!strcasecmp (stream, "stdout")) {
-        counter = &stdout_output_cb_count;
-        len_counter = &stdout_output_cb_len_count;
-    }
-    else if (!strcasecmp (stream, "stderr")) {
-        counter = &stderr_output_cb_count;
-        len_counter = &stderr_output_cb_len_count;
-    }
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    ptr = flux_subprocess_read (p, stream, -1, &lenp);
-    (*counter)++;
-    (*len_counter)+= lenp;
-
-    if (ptr && lenp && (*len_counter) == 10001) {
-        if (!strcmp (stream, "stderr")) {
-            ok (stdout_output_cb_count == 0
-                && stdout_output_cb_len_count == 0,
-                "received all stderr data and stdout output is still 0");
-            ok (!flux_subprocess_stream_start (p, "stdout"),
-                "flux_subprocess_stream_start on stdout success");
-        }
-    }
+    if (state_change_cb_count == 0)
+        ok (state == FLUX_SUBPROCESS_RUNNING,
+            "subprocess state == RUNNING in state change handler");
+    else
+        ok (state == FLUX_SUBPROCESS_EXITED,
+            "subprocess state == EXITED in state change handler");
+    state_change_cb_count++;
 }
 
-/* How this tests works is we output "hi" alot of times without line
- * buffering on both stdout and stderr.  After starting the
- * subprocess, we immediately disable the stdout stream.  The goal is
- * we get all the stderr via callback, then re-enable the stdout
- * stream, and get the rest of the stdout.
- *
- * This test is racy, as its always possible stderr just arrives
- * before stdout under normal circumstances, but the probability of
- * that occuring is low given how much we output.
- */
-void test_stream_start_stop_initial_stop (flux_reactor_t *r)
+void test_state_change (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-E", "-c", "5000", "hi", NULL };
+    char *av[] = { "true", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
 
-    ok ((cmd = flux_cmd_create (6, av, environ)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "false") == 0,
-        "flux_cmd_setopt set stdout_LINE_BUFFER success");
-
-    ok (flux_cmd_setopt (cmd, "stderr_LINE_BUFFER", "false") == 0,
-        "flux_cmd_setopt set stderr_LINE_BUFFER success");
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
 
     flux_subprocess_ops_t ops = {
         .on_completion = completion_cb,
-        .on_stdout = start_stdout_after_stderr_cb,
-        .on_stderr = start_stdout_after_stderr_cb,
+        .on_state_change = state_change_cb
     };
     completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    stdout_output_cb_len_count = 0;
-    stderr_output_cb_len_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    state_change_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
         "subprocess state == RUNNING after flux_local_exec");
 
-    ok (!flux_subprocess_stream_stop (p, "stdout"),
-        "flux_subprocess_stream_stop on stdout success");
-
     int rc = flux_reactor_run (r, 0);
     ok (rc == 0, "flux_reactor_run returned zero status");
     ok (completion_cb_count == 1, "completion callback called 1 time");
-    /* potential for == 2, b/c could all be buffered before stdout
-     * callback is re-started */
-    ok (stdout_output_cb_count >= 2, "stdout output callback called >= 2 times: %d",
-        stdout_output_cb_count);
-    /* we would hope stderr is called > 2 times, but there's
-     * potentially racy behavior and its only called 2 times.  This
-     * isn't seen in practice. */
-    ok (stderr_output_cb_count > 2, "stderr output callback called > 2 times: %d",
-        stderr_output_cb_count);
-    ok (stdout_output_cb_len_count == 10001, "stdout_output_cb_len_count is 10001");
-    ok (stderr_output_cb_len_count == 10001, "stderr_output_cb_len_count is 10001");
+    ok (state_change_cb_count == 2, "state change callback called 3 times");
     flux_subprocess_destroy (p);
     flux_cmd_destroy (cmd);
 }
 
-void mid_stop_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                        int revents, void *arg)
-{
-    flux_subprocess_t *p = arg;
-    ok (stdout_output_cb_count == 1,
-        "stdout callback has not been called since timer activated");
-    ok (!flux_subprocess_stream_start (p, "stdout"),
-        "flux_subprocess_stream_start on stdout success");
-    timer_cb_count++;
-    flux_watcher_stop (w);
-}
-
-void mid_stop_cb (flux_subprocess_t *p, const char *stream)
+void state_change_stopped_cb (flux_subprocess_t *p,
+                              flux_subprocess_state_t state)
 {
-    const char *ptr;
-    int lenp = 0;
-    int *counter;
-
-    if (!strcasecmp (stream, "stdout"))
-        counter = &stdout_output_cb_count;
-    else {
-        ok (false, "unexpected stream %s", stream);
-        return;
-    }
-
-    ptr = flux_subprocess_read (p, stream, -1, &lenp);
-    if (stdout_output_cb_count == 0) {
-        flux_watcher_t *tw = NULL;
-        ok (ptr && lenp > 0,
-            "flux_subprocess_read read data on stdout: %d", lenp);
-        ok (!flux_subprocess_stream_stop (p, "stdout"),
-            "flux_subprocess_stream_stop on stdout success");
-        ok ((tw = flux_subprocess_aux_get (p, "tw")) != NULL,
-            "flux_subprocess_aux_get timer success");
-        flux_watcher_start (tw);
-    }
-    else if (stdout_output_cb_count == 1) {
-        ok (ptr && lenp > 0,
-            "flux_subprocess_read read data on stdout: %d", lenp);
-        ok (timer_cb_count == 1,
-            "next stdout callback called after time callback called");
+    diag ("state_change_stopped: state = %s",
+          flux_subprocess_state_string (state));
+    if (state == FLUX_SUBPROCESS_STOPPED) {
+        flux_future_t *f = flux_subprocess_kill (p, SIGKILL);
+        ok (true, "subprocess state == STOPPED in state change handler");
+        flux_future_destroy (f);
+        stopped_cb_count++;
     }
-    (*counter)++;
 }
 
-/* How this tests works is we output "hi" alot of times without line
- * buffering on stdout.  After the first callback, we stop the output
- * stream, and setup a timer.  For a bit of time, we should see no
- * more stdout, and after the timer expires, we'll re-eanble the
- * stdout stream.
- *
- * This test is racy, as its always possible stdout is just delayed,
- * but the probability of that occuring is low given how much we
- * output.
- */
-void test_stream_start_stop_mid_stop (flux_reactor_t *r)
+void test_state_change_stopped (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-c", "5000", "hi", NULL };
+    char *av[] = { "/bin/sleep", "30", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
-    flux_watcher_t *tw = NULL;
-
-    ok ((cmd = flux_cmd_create (5, av, environ)) != NULL, "flux_cmd_create");
 
-    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "false") == 0,
-        "flux_cmd_setopt set stdout_LINE_BUFFER success");
+    ok ((cmd = flux_cmd_create (2, av, NULL)) != NULL, "flux_cmd_create");
 
     flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = mid_stop_cb,
-        .on_stderr = NULL,
+        .on_state_change = state_change_stopped_cb
     };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    timer_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    stopped_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
-    ok ((tw = flux_timer_watcher_create (r, 2.0, 0.0,
-                                         mid_stop_timer_cb, p)) != NULL,
-        "flux_timer_watcher_create success");
-
-    ok (!flux_subprocess_aux_set (p, "tw", tw, NULL),
-        "flux_subprocess_aux_set timer success");
-
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
         "subprocess state == RUNNING after flux_local_exec");
 
+    flux_future_t *f = flux_subprocess_kill (p, SIGSTOP);
+    ok (f != NULL,
+        "flux_subprocess_kill SIGSTOP");
+    flux_future_destroy (f);
+
     int rc = flux_reactor_run (r, 0);
     ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    /* could be == to 3 if output occurs fast enough, but chances are
-     * it'll be > 3 */
-    ok (stdout_output_cb_count >= 3, "stdout output callback called >= 3 times: %d",
-        stdout_output_cb_count);
-    ok (stderr_output_cb_count == 0, "stderr output callback called 0 times");
-    ok (timer_cb_count == 1, "timer callback called 1 time");
+    ok (stopped_cb_count == 1, "subprocess was stopped");
     flux_subprocess_destroy (p);
     flux_cmd_destroy (cmd);
-    flux_watcher_destroy (tw);
 }
 
-/* How this tests works is we output "hi" alot of times without line
- * buffering on both stdout and stderr.  We initially disable stdout
- * callbacks via STREAM_STOP.  So the goal is we get all the
- * stderr via callback and no stdout via callback.  After we get all
- * the stderr, we can re-enable stdout and get all of that data.
- *
- * This test is racy, as its always possible stderr just arrives
- * before stdout under normal circumstances, but the probability of
- * that occuring is low given how much we output.
-  */
-void test_stream_stop_enable (flux_reactor_t *r)
+void test_state_strings (void)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-E", "-c", "5000", "hi", NULL };
-    flux_cmd_t *cmd;
+    ok (!strcasecmp (flux_subprocess_state_string (FLUX_SUBPROCESS_INIT), "Init"),
+        "flux_subprocess_state_string returns correct string");
+    ok (!strcasecmp (flux_subprocess_state_string (FLUX_SUBPROCESS_RUNNING), "Running"),
+        "flux_subprocess_state_string returns correct string");
+    ok (!strcasecmp (flux_subprocess_state_string (FLUX_SUBPROCESS_EXITED), "Exited"),
+        "flux_subprocess_state_string returns correct string");
+    ok (!flux_subprocess_state_string (100),
+        "flux_subprocess_state_string returns NULL on bad state");
+    is (flux_subprocess_state_string (FLUX_SUBPROCESS_STOPPED),
+        "Stopped");
+}
+
+void test_exec_fail (flux_reactor_t *r)
+{
+    char path [4096];
+    char *av_eacces[]  = { "/", NULL };
+    char *av_enoent[]  = { "/usr/bin/foobarbaz", NULL };
+    flux_cmd_t *cmd = NULL;
     flux_subprocess_t *p = NULL;
 
-    ok ((cmd = flux_cmd_create (6, av, environ)) != NULL, "flux_cmd_create");
+    ok ((cmd = flux_cmd_create (1, av_eacces, NULL)) != NULL, "flux_cmd_create");
+
+    /*  Set cwd to force use of fork/exec */
+    ok (flux_cmd_setcwd (cmd, getcwd (path, sizeof (path))) == 0,
+        "flux_cmd_setcwd");
 
-    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "false") == 0,
-        "flux_cmd_setopt set stdout_LINE_BUFFER success");
+    p = flux_local_exec (r, 0, cmd, NULL);
+    ok (p == NULL
+        && errno == EACCES,
+        "flux_local_exec failed with EACCES");
 
-    ok (flux_cmd_setopt (cmd, "stderr_LINE_BUFFER", "false") == 0,
-        "flux_cmd_setopt set stderr_LINE_BUFFER success");
+    flux_cmd_destroy (cmd);
 
-    ok (flux_cmd_setopt (cmd, "stdout_STREAM_STOP", "true") == 0,
-        "flux_cmd_setopt set stdout_STREAM_STOP success");
+    ok ((cmd = flux_cmd_create (1, av_enoent, NULL)) != NULL, "flux_cmd_create");
 
-    flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = start_stdout_after_stderr_cb,
-        .on_stderr = start_stdout_after_stderr_cb
-    };
-    completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    stdout_output_cb_len_count = 0;
-    stderr_output_cb_len_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p != NULL, "flux_local_exec");
+    /*  Set cwd to force use of fork/exec */
+    ok (flux_cmd_setcwd (cmd, getcwd (path, sizeof (path))) == 0,
+        "flux_cmd_setcwd");
 
-    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
-        "subprocess state == RUNNING after flux_local_exec");
+    p = flux_local_exec (r, 0, cmd, NULL);
+    ok (p == NULL
+        && errno == ENOENT,
+        "flux_local_exec failed with ENOENT");
 
-    int rc = flux_reactor_run (r, 0);
-    ok (rc == 0, "flux_reactor_run returned zero status");
-    ok (completion_cb_count == 1, "completion callback called 1 time");
-    /* potential for == 2, b/c could all be buffered before stdout
-     * callback is started */
-    ok (stdout_output_cb_count >= 2, "stdout output callback called >= 2 times: %d",
-        stdout_output_cb_count);
-    /* we would hope stderr is called > 2 times, but there's
-     * potentially racy behavior and its only called 2 times.  This
-     * isn't seen in practice. */
-    ok (stderr_output_cb_count > 2, "stderr output callback called > 2 times: %d",
-        stderr_output_cb_count);
-    ok (stdout_output_cb_len_count == 10001, "stdout_output_cb_len_count is 10001");
-    ok (stderr_output_cb_len_count == 10001, "stderr_output_cb_len_count is 10001");
-    flux_subprocess_destroy (p);
     flux_cmd_destroy (cmd);
 }
 
-/* disabled the test should work like a normal test */
-void test_stream_stop_disable (flux_reactor_t *r)
+void test_context (flux_reactor_t *r)
 {
-    char *av[] = { TEST_SUBPROCESS_DIR "test_multi_echo", "-O", "-E", "-c", "5000", "hi", NULL };
+    char *av[] = { "true", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
+    char *extra = "mydata";
+    char *tmp;
 
-    ok ((cmd = flux_cmd_create (6, av, environ)) != NULL, "flux_cmd_create");
-
-    ok (flux_cmd_setopt (cmd, "stdout_LINE_BUFFER", "false") == 0,
-        "flux_cmd_setopt set stdout_LINE_BUFFER success");
-
-    ok (flux_cmd_setopt (cmd, "stderr_LINE_BUFFER", "false") == 0,
-        "flux_cmd_setopt set stderr_LINE_BUFFER success");
-
-    ok (flux_cmd_setopt (cmd, "stdout_STREAM_STOP", "false") == 0,
-        "flux_cmd_setopt set stdout_STREAM_STOP success");
-
-    ok (flux_cmd_setopt (cmd, "stderr_STREAM_STOP", "false") == 0,
-        "flux_cmd_setopt set stderr_STREAM_STOP success");
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
 
     flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_stdout = count_output_cb,
-        .on_stderr = count_output_cb
+        .on_completion = completion_cb
     };
     completion_cb_count = 0;
-    stdout_output_cb_count = 0;
-    stderr_output_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
+    p = flux_local_exec (r, 0, cmd, &ops);
     ok (p != NULL, "flux_local_exec");
 
     ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
         "subprocess state == RUNNING after flux_local_exec");
+    ok (flux_subprocess_aux_set (p, "extra", extra, NULL) == 0,
+        "flux_subprocess_aux_set success");
+    ok ((tmp = flux_subprocess_aux_get (p, "extra")) != NULL,
+        "flux_subprocess_aux_get success");
+    ok (tmp == extra,
+        "flux_subprocess_aux_get returned correct pointer");
 
     int rc = flux_reactor_run (r, 0);
     ok (rc == 0, "flux_reactor_run returned zero status");
     ok (completion_cb_count == 1, "completion callback called 1 time");
-    ok (stdout_output_cb_count > 2, "stdout output callback called > 2 times: %d",
-        stdout_output_cb_count);
-    ok (stderr_output_cb_count > 2, "stderr output callback called > 2 times: %d",
-        stderr_output_cb_count);
     flux_subprocess_destroy (p);
     flux_cmd_destroy (cmd);
 }
 
-void test_stream_stop_error (flux_reactor_t *r)
+void test_refcount (flux_reactor_t *r)
 {
-    char *av[] = { "/bin/true", NULL };
+    char *av[] = { "true", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
+    char *extra = "mydata";
+    char *tmp;
 
     ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
 
-    ok (flux_cmd_setopt (cmd, "stdout_STREAM_STOP", "ABCD") == 0,
-        "flux_cmd_setopt set stdout_STREAM_STOP success");
-
     flux_subprocess_ops_t ops = {
-        .on_completion = completion_cb,
-        .on_channel_out = flux_standard_output,
-        .on_stdout = flux_standard_output,
-        .on_stderr = flux_standard_output
+        .on_completion = completion_cb
     };
-    p = flux_local_exec (r, 0, cmd, &ops, NULL);
-    ok (p == NULL
-        && errno == EINVAL,
-        "flux_local_exec fails with EINVAL due to bad stream_stop input");
+    completion_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+    ok (flux_subprocess_aux_set (p, "extra", extra, NULL) == 0,
+        "flux_subprocess_aux_set success");
+    subprocess_incref (p);
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    subprocess_decref (p);
 
+    /* normally this should fail, but we've increased the refcount so
+     * subprocess should not be destroyed */
+    ok ((tmp = flux_subprocess_aux_get (p, "extra")) != NULL,
+        "flux_subprocess_aux_get success");
+    ok (tmp == extra,
+        "flux_subprocess_aux_get returned correct pointer");
+
+    flux_subprocess_destroy (p);
     flux_cmd_destroy (cmd);
 }
 
@@ -2528,7 +849,7 @@ void shmem_hook_cb (flux_subprocess_t *p, void *arg)
 
 void test_pre_exec_hook (flux_reactor_t *r)
 {
-    char *av[] = { "/bin/true", NULL };
+    char *av[] = { "true", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
     int *shmem_count;
@@ -2553,8 +874,14 @@ void test_pre_exec_hook (flux_reactor_t *r)
         .pre_exec_arg = shmem_count
     };
     completion_cb_count = 0;
-    p = flux_local_exec (r, FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH, cmd, &ops, &hooks);
-    ok (p != NULL, "flux_local_exec");
+    p = flux_local_exec_ex (r,
+                            FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH,
+                            cmd,
+                            &ops,
+                            &hooks,
+                            NULL,
+                            NULL);
+    ok (p != NULL, "flux_local_exec_ex");
 
     int rc = flux_reactor_run (r, 0);
     ok (rc == 0, "flux_reactor_run returned zero status");
@@ -2573,7 +900,7 @@ void count_hook_cb (flux_subprocess_t *p, void *arg)
 
 void test_post_fork_hook (flux_reactor_t *r)
 {
-    char *av[] = { "/bin/true", NULL };
+    char *av[] = { "true", NULL };
     flux_cmd_t *cmd;
     flux_subprocess_t *p = NULL;
     int hook_count = 0;
@@ -2588,8 +915,8 @@ void test_post_fork_hook (flux_reactor_t *r)
         .post_fork_arg = &hook_count
     };
     completion_cb_count = 0;
-    p = flux_local_exec (r, 0, cmd, &ops, &hooks);
-    ok (p != NULL, "flux_local_exec");
+    p = flux_local_exec_ex (r, 0, cmd, &ops, &hooks, NULL, NULL);
+    ok (p != NULL, "flux_local_exec_ex");
 
     int rc = flux_reactor_run (r, 0);
     ok (rc == 0, "flux_reactor_run returned zero status");
@@ -2599,6 +926,215 @@ void test_post_fork_hook (flux_reactor_t *r)
     flux_cmd_destroy (cmd);
 }
 
+void destroy_in_completion_cb (flux_subprocess_t *p)
+{
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_EXITED,
+        "subprocess state == EXITED in completion handler");
+    ok (flux_subprocess_status (p) != -1,
+        "subprocess status is valid");
+    ok (flux_subprocess_exit_code (p) == 0,
+        "subprocess exit code is 0");
+    completion_cb_count++;
+    flux_subprocess_destroy (p);
+}
+
+void test_destroy_in_completion (flux_reactor_t *r)
+{
+    char *av[] = { "true", NULL };
+    flux_cmd_t *cmd, *cmd2;
+    flux_reactor_t *r2;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = destroy_in_completion_cb
+    };
+    completion_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    ok (p != NULL, "flux_local_exec");
+
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+        "subprocess state == RUNNING after flux_local_exec");
+    ok ((flux_subprocess_pid (p) > (pid_t) 0),
+        "flux_local_exec() started pid %ld", (pid_t) flux_subprocess_pid (p));
+    ok ((cmd2 = flux_subprocess_get_cmd (p)) != NULL,
+        "flux_subprocess_get_cmd success");
+    ok ((r2 = flux_subprocess_get_reactor (p)) != NULL,
+        "flux_subprocess_get_reactor success");
+    ok (r == r2,
+        "flux_subprocess_get_reactor returns correct reactor");
+
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    flux_cmd_destroy (cmd);
+}
+
+void fail_completion_cb (flux_subprocess_t *p)
+{
+    ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_EXITED,
+        "subprocess state == EXITED in completion handler");
+    ok (flux_subprocess_status (p) != -1,
+        "subprocess status is valid");
+    ok (flux_subprocess_exit_code (p) == 127,
+        "subprocess exit code is 127, got %d", flux_subprocess_exit_code (p));
+    completion_cb_count++;
+}
+
+void fail_output_cb (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+    int *counter;
+
+    if (!strcasecmp (stream, "stdout"))
+        counter = &stdout_output_cb_count;
+    else if (!strcasecmp (stream, "stderr"))
+        counter = &stderr_output_cb_count;
+    else {
+        ok (false, "unexpected stream %s", stream);
+        return;
+    }
+
+    if ((*counter) == 0) {
+        ok (flux_subprocess_read_stream_closed (p, stream),
+            "flux_subprocess_read_stream_closed saw EOF on %s", stream);
+
+        len = flux_subprocess_read (p, stream, &buf);
+        ok (len == 0,
+            "flux_subprocess_read on %s read EOF", stream);
+    }
+    else
+        ok (false, "fail_output_cb called multiple times");
+
+    (*counter)++;
+}
+
+void test_fail_notacommand (flux_reactor_t *r)
+{
+    char *av[] = { "notacommand", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = fail_completion_cb,
+        .on_stdout = fail_output_cb,
+        .on_stderr = fail_output_cb
+    };
+    completion_cb_count = 0;
+    stdout_output_cb_count = 0;
+    stderr_output_cb_count = 0;
+    p = flux_local_exec (r, 0, cmd, &ops);
+    /* Per manpage:
+     *
+     * If posix_spawn() or posix_spawnp() fail for any of the reasons
+     *  that would cause fork() or one of the exec family of functions
+     *  to fail, an error value shall be returned as described by
+     *  fork() and exec, respectively (or, if the error occurs after
+     *  the calling process successfully returns, the child process
+     *  shall exit with exit status 127).
+     *
+     * So we can't assume flux_local_exec() returns an error on posix_spawn().
+     */
+    if (p == NULL) {
+        ok (p == NULL, "flux_local_exec failed");
+        ok (errno == ENOENT, "flux_local_exec returned ENOENT");
+    }
+    else {
+        ok (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING,
+            "subprocess state == RUNNING after flux_local_exec");
+
+        int rc = flux_reactor_run (r, 0);
+        ok (rc == 0, "flux_reactor_run returned zero status");
+        ok (completion_cb_count == 1, "completion callback called 1 time");
+        ok (stdout_output_cb_count == 1, "stdout output callback called 1 times");
+        ok (stderr_output_cb_count == 1, "stderr output callback called 1 times");
+        flux_subprocess_destroy (p);
+    }
+    flux_cmd_destroy (cmd);
+}
+
+void test_fail_notacommand_fork (flux_reactor_t *r)
+{
+    char *av[] = { "notacommand", NULL };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (1, av, NULL)) != NULL, "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_completion = fail_completion_cb,
+    };
+    p = flux_local_exec (r, FLUX_SUBPROCESS_FLAGS_FORK_EXEC, cmd, &ops);
+    ok (p == NULL, "flux_local_exec failed");
+    ok (errno == ENOENT, "flux_local_exec returned ENOENT");
+    flux_cmd_destroy (cmd);
+}
+
+int fdcleanup_fdcount;
+
+void fdcleanup_output (flux_subprocess_t *p, const char *stream)
+{
+    const char *buf = NULL;
+    int len;
+
+    len = flux_subprocess_read_line (p, stream, &buf);
+
+    if (len > 0 && buf != NULL) {
+        diag ("%s: %.*s", stream, len, buf);
+        if (streq (stream, "stdout"))
+            fdcleanup_fdcount = strtoul (buf, NULL, 10);
+    }
+}
+
+/* This test ensures that subprocs aren't gifted with bonus file descriptors.
+ * N.B. Occasionally an extra file descriptor matches the glob.  Assume this
+ * is the "syncfd" and we are racing with its removal. Therefore, allow the
+ * fd count to match expected_fdcount or one more than that.
+ */
+void test_fdcleanup (flux_reactor_t *r,
+                     const char *desc,
+                     int flags,
+                     int expected_fdcount)
+{
+    char *av[] = {
+        "sh",
+        "-c",
+        "ls -1 /proc/$$/fd | wc -w",
+        NULL
+    };
+    flux_cmd_t *cmd;
+    flux_subprocess_t *p = NULL;
+
+    ok ((cmd = flux_cmd_create (ARRAY_SIZE (av) - 1, av, NULL)) != NULL,
+        "flux_cmd_create");
+
+    flux_subprocess_ops_t ops = {
+        .on_stdout =  fdcleanup_output,
+        .on_stderr =  fdcleanup_output,
+        .on_completion = completion_cb,
+    };
+    p = flux_local_exec (r, flags, cmd, &ops);
+    ok (p != NULL, "flux_local_exec %s", desc);
+    completion_cb_count = 0;
+    fdcleanup_fdcount = 0;
+    int rc = flux_reactor_run (r, 0);
+    ok (rc == 0, "flux_reactor_run returned zero status");
+    ok (completion_cb_count == 1, "completion callback called 1 time");
+    ok (fdcleanup_fdcount == expected_fdcount
+        || fdcleanup_fdcount == expected_fdcount + 1,
+        "%d file descriptors are open (expected %d-%d)",
+        fdcleanup_fdcount,
+        expected_fdcount,
+        expected_fdcount + 1);
+
+    flux_subprocess_destroy (p);
+    flux_cmd_destroy (cmd);
+}
+
 int main (int argc, char *argv[])
 {
     flux_reactor_t *r;
@@ -2612,46 +1148,21 @@ int main (int argc, char *argv[])
 
     start_fdcount = fdcount ();
 
+    diag ("corner_cases");
+    test_corner_cases (r);
+    diag ("post_exec_errors");
+    test_post_exec_errors (r);
+
     diag ("basic");
     test_basic (r);
     diag ("basic_fail");
     test_basic_fail (r);
-    diag ("basic_errors");
-    test_basic_errors (r);
-    diag ("errors");
-    test_errors (r);
-    diag ("basic_stdout");
-    test_basic_stdout (r);
-    diag ("basic_stderr");
-    test_basic_stderr (r);
-    diag ("basic_stdout_and_stderr");
-    test_basic_stdout_and_stderr (r);
-    diag ("basic_default_output");
-    test_basic_default_output (r);
-    diag ("basic_stdin");
-    test_basic_stdin (r);
-    diag ("basic_no_newline");
-    test_basic_no_newline (r);
-    diag ("basic_trimmed_line");
-    test_basic_trimmed_line (r);
-    diag ("basic_multiple_lines");
-    test_basic_multiple_lines (r);
-    diag ("basic_stdin_closed");
-    test_basic_stdin_closed (r);
-    diag ("basic_read_line_until_eof");
-    test_basic_read_line_until_eof (r);
-    diag ("basic_read_line_until_eof_error");
-    test_basic_read_line_until_eof_error (r);
-    diag ("write_after_close");
-    test_write_after_close (r);
     diag ("env_passed");
     test_env_passed (r);
-#if 0
-    diag ("flag_stdio_fallthrough");
-    test_flag_stdio_fallthrough (r);
-#endif
-    diag ("flag_setpgrp");
-    test_flag_setpgrp (r);
+    diag ("flag_no_setpgrp");
+    test_flag_no_setpgrp (r);
+    diag ("flag_fork_exec");
+    test_flag_fork_exec (r);
     diag ("kill");
     test_kill (r);
     diag ("kill_setpgrp");
@@ -2660,6 +1171,8 @@ int main (int argc, char *argv[])
     test_kill_eofs (r);
     diag ("state_change");
     test_state_change (r);
+    diag ("state_change_stopped");
+    test_state_change_stopped (r);
     diag ("state_strings");
     test_state_strings ();
     diag ("exec_fail");
@@ -2668,42 +1181,20 @@ int main (int argc, char *argv[])
     test_context (r);
     diag ("refcount");
     test_refcount (r);
-    diag ("channel_fd_env");
-    test_channel_fd_env (r);
-    diag ("channel_fd_in");
-    test_channel_fd_in (r);
-    diag ("channel_fd_in_and_out");
-    test_channel_fd_in_and_out (r);
-    diag ("channel_multiple_lines");
-    test_channel_multiple_lines (r);
-    diag ("bufsize");
-    test_bufsize (r);
-    diag ("bufsize_error");
-    test_bufsize_error (r);
-    diag ("line_buffer_default");
-    test_line_buffer_default (r);
-    diag ("line_buffer_enable");
-    test_line_buffer_enable (r);
-    diag ("line_buffer_disable");
-    test_line_buffer_disable (r);
-    diag ("line_buffer_error");
-    test_line_buffer_error (r);
-    diag ("stream_start_stop_basic");
-    test_stream_start_stop_basic (r);
-    diag ("stream_start_stop_initial_stop");
-    test_stream_start_stop_initial_stop (r);
-    diag ("stream_start_stop_mid_stop");
-    test_stream_start_stop_mid_stop (r);
-    diag ("stream_stop_enable");
-    test_stream_stop_enable (r);
-    diag ("stream_stop_disable");
-    test_stream_stop_disable (r);
-    diag ("stream_stop_error");
-    test_stream_stop_error (r);
     diag ("pre_exec_hook");
     test_pre_exec_hook (r);
     diag ("post_fork_hook");
     test_post_fork_hook (r);
+    diag ("test_destroy_in_completion");
+    test_destroy_in_completion (r);
+    diag ("fail_notacommand");
+    test_fail_notacommand (r);
+    diag ("fail_notacommand_fork");
+    test_fail_notacommand_fork (r);
+    diag ("test_fdcleanup fork-exec");
+    test_fdcleanup (r, "fork-exec", FLUX_SUBPROCESS_FLAGS_FORK_EXEC, 3);
+    diag ("test_fdcleanup posix-spawn");
+    test_fdcleanup (r, "posix-spawn", 0, 3);
 
     end_fdcount = fdcount ();
 
diff --git a/src/common/libsubprocess/test/test_echo.c b/src/common/libsubprocess/test/test_echo.c
index 651dd05bd728..1e3be8fe90a9 100644
--- a/src/common/libsubprocess/test/test_echo.c
+++ b/src/common/libsubprocess/test/test_echo.c
@@ -10,6 +10,18 @@
 
 /* simple tool that outputs args to stdout/stderr or both depending on
  * options
+ *
+ * 'O' - output to stdout
+ * 'E' - output to stderr
+ * 'n' - no newline, do not output newline after output
+ * 'C' - output to channel, get fd via 'c' option
+ * 'c' - name environment variable that contains channel fd number
+ * 'P' - prefix output with stream name
+ * 'b' - max to output before breaking out
+ *
+ * if args on command line, output those args, else read from fd for
+ * input (default fd = STDIN_FILENO)
+ *
  */
 #include <stdio.h>
 #include <stdlib.h>
@@ -102,11 +114,8 @@ main (int argc, char *argv[])
 
     if (channel) {
         const char *fdstr;
-        char channelstr[1024];
-
-        sprintf (channelstr, "%s", channel_name);
 
-        if (!(fdstr = getenv (channelstr))) {
+        if (!(fdstr = getenv (channel_name))) {
             perror ("getenv");
             exit (1);
         }
diff --git a/src/common/libsubprocess/util.c b/src/common/libsubprocess/util.c
index fec6bd994254..1fca89586169 100644
--- a/src/common/libsubprocess/util.c
+++ b/src/common/libsubprocess/util.c
@@ -13,15 +13,14 @@
 #endif
 
 #include <sys/types.h>
-#include <wait.h>
 #include <unistd.h>
 #include <errno.h>
 
-#include <czmq.h>
-
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/fdwalk.h"
+#include "src/common/libutil/parse_size.h"
 
 #include "subprocess.h"
 #include "subprocess_private.h"
@@ -53,16 +52,18 @@ int cmd_option_bufsize (flux_subprocess_t *p, const char *name)
         goto cleanup;
 
     if ((val = flux_cmd_getopt (p->cmd, var))) {
-        char *endptr;
-        errno = 0;
-        rv = strtol (val, &endptr, 10);
-        if (errno
-            || endptr[0] != '\0'
-            || rv <= 0) {
-            rv = -1;
+        uint64_t size;
+        if (parse_size (val, &size) < 0)
+            goto cleanup;
+        if (size == 0) {
             errno = EINVAL;
             goto cleanup;
         }
+        if (size > INT_MAX) {
+            errno = EOVERFLOW;
+            goto cleanup;
+        }
+        rv = (int) size;
     }
     else
         rv = SUBPROCESS_DEFAULT_BUFSIZE;
@@ -85,37 +86,12 @@ int cmd_option_line_buffer (flux_subprocess_t *p, const char *name)
         if (!strcasecmp (val, "false"))
             rv = 0;
         else if (!strcasecmp (val, "true"))
-            rv = FLUX_WATCHER_LINE_BUFFER;
-        else
-            errno = EINVAL;
-    }
-    else
-        rv = FLUX_WATCHER_LINE_BUFFER;
-
-cleanup:
-    free (var);
-    return rv;
-}
-
-int cmd_option_stream_stop (flux_subprocess_t *p, const char *name)
-{
-    char *var;
-    const char *val;
-    int rv = -1;
-
-    if (asprintf (&var, "%s_STREAM_STOP", name) < 0)
-        goto cleanup;
-
-    if ((val = flux_cmd_getopt (p->cmd, var))) {
-        if (!strcasecmp (val, "true"))
-            rv = 1;
-        if (!strcasecmp (val, "false"))
-            rv = 0;
+            rv = FBUF_WATCHER_LINE_BUFFER;
         else
             errno = EINVAL;
     }
     else
-        rv = 0;
+        rv = FBUF_WATCHER_LINE_BUFFER;
 
 cleanup:
     free (var);
diff --git a/src/common/libsubprocess/util.h b/src/common/libsubprocess/util.h
index 969e3c8eb41d..c40e7da6004d 100644
--- a/src/common/libsubprocess/util.h
+++ b/src/common/libsubprocess/util.h
@@ -21,6 +21,6 @@ int cmd_option_bufsize (flux_subprocess_t *p, const char *name);
 
 int cmd_option_line_buffer (flux_subprocess_t *p, const char *name);
 
-int cmd_option_stream_stop (flux_subprocess_t *p, const char *name);
-
 #endif /* !_SUBPROCESS_UTIL_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libtap/Makefile.am b/src/common/libtap/Makefile.am
index 33483b6ee0e3..0bc55f985312 100644
--- a/src/common/libtap/Makefile.am
+++ b/src/common/libtap/Makefile.am
@@ -5,7 +5,8 @@ AM_CFLAGS = \
 AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
-AM_CPPFLAGS =
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS)
 
 check_LTLIBRARIES = libtap.la
 
diff --git a/src/common/libtaskmap/Makefile.am b/src/common/libtaskmap/Makefile.am
new file mode 100644
index 000000000000..46ddf75dd6f6
--- /dev/null
+++ b/src/common/libtaskmap/Makefile.am
@@ -0,0 +1,41 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux \
+	$(JANSSON_CFLAGS)
+
+noinst_LTLIBRARIES = \
+	libtaskmap.la
+fluxinclude_HEADERS = \
+	taskmap.h
+libtaskmap_la_SOURCES = \
+	taskmap_private.h \
+	taskmap.c
+
+TESTS = test_taskmap.t
+
+check_PROGRAMS = \
+	$(TESTS)
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+       $(top_srcdir)/config/tap-driver.sh
+
+test_taskmap_t_SOURCES = test/taskmap.c
+test_taskmap_t_CPPFLAGS = $(AM_CPPFLAGS)
+test_taskmap_t_LDADD = \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libtaskmap/libtaskmap.la \
+	$(top_builddir)/src/common/libutil/libutil.la \
+	$(top_builddir)/src/common/libidset/libidset.la \
+	$(top_builddir)/src/common/libczmqcontainers/libczmqcontainers.la \
+	$(JANSSON_LIBS)
diff --git a/src/common/libtaskmap/taskmap.c b/src/common/libtaskmap/taskmap.c
new file mode 100644
index 000000000000..8f1ab84e920d
--- /dev/null
+++ b/src/common/libtaskmap/taskmap.c
@@ -0,0 +1,989 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <ctype.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/idset.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/lru_cache.h"
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
+#include "taskmap.h"
+
+struct taskmap_block {
+    int start;
+    int nnodes;
+    int ppn;
+    int repeat;
+};
+
+struct taskmap {
+    zlistx_t *blocklist;
+    lru_cache_t *idsets;
+};
+
+static struct taskmap_block * taskmap_block_create (int nodeid,
+                                                    int nnodes,
+                                                    int ppn,
+                                                    int repeat)
+{
+    struct taskmap_block *block = calloc (1, sizeof (*block));
+    if (!block)
+        return NULL;
+
+    block->start = nodeid;
+    block->nnodes = nnodes;
+    block->ppn = ppn;
+    block->repeat = repeat;
+    return block;
+}
+
+static void taskmap_block_destroy (struct taskmap_block *block)
+{
+    if (block) {
+        int saved_errno = errno;
+        free (block);
+        errno = saved_errno;
+    }
+}
+
+static int taskmap_block_end (struct taskmap_block *block)
+{
+    return block->start + block->nnodes - 1;
+}
+
+static struct taskmap_block *taskmap_block_from_json (json_t *entry,
+                                                      flux_error_t *errp)
+{
+    int nodeid;
+    int nnodes;
+    int ppn;
+    int repeat;
+    json_error_t error;
+    struct taskmap_block *block;
+
+    if (json_unpack_ex (entry, &error, JSON_DECODE_ANY,
+                        "[iiii]",
+                        &nodeid,
+                        &nnodes,
+                        &ppn,
+                        &repeat) < 0) {
+        errprintf (errp, "error in taskmap entry: %s", error.text);
+        return NULL;
+    }
+    if (nodeid < 0 || nnodes <= 0 || ppn <= 0 || repeat <= 0) {
+        errprintf (errp,
+                   "invalid entry [%d,%d,%d,%d]",
+                   nodeid,
+                   nnodes,
+                   ppn,
+                   repeat);
+        return NULL;
+    }
+    if (!(block = taskmap_block_create (nodeid, nnodes, ppn, repeat)))
+        errprintf (errp, "Out of memory");
+    return block;
+}
+
+static void taskmap_block_destructor (void **item)
+{
+    if (item) {
+        taskmap_block_destroy (*item);
+        *item = NULL;
+    }
+}
+
+void taskmap_destroy (struct taskmap *map)
+{
+    if (map) {
+        int saved_errno = errno;
+        zlistx_destroy (&map->blocklist);
+        lru_cache_destroy (map->idsets);
+        free (map);
+        errno = saved_errno;
+    }
+}
+
+struct taskmap *taskmap_create (void)
+{
+    struct taskmap *map = NULL;
+
+    if (!(map = calloc (1, sizeof (*map)))
+        || !(map->blocklist = zlistx_new ())
+        || !(map->idsets = lru_cache_create (16))) {
+        errno = ENOMEM;
+        goto error;
+    }
+    zlistx_set_destructor (map->blocklist, taskmap_block_destructor);
+    lru_cache_set_free_f (map->idsets, (lru_cache_free_f) idset_destroy);
+    return map;
+error:
+    taskmap_destroy (map);
+    return NULL;
+}
+
+bool taskmap_unknown (const struct taskmap *map)
+{
+    /*  A zero-length mapping indicates that the task map is unknown */
+    return zlistx_size (map->blocklist) == 0;
+}
+
+static char *to_string (int n, char *buf, int len)
+{
+    (void) snprintf (buf, len, "%d", n);
+    return buf;
+}
+
+static void cache_idset (const struct taskmap *map,
+                         int nodeid,
+                         struct idset *idset)
+{
+    char id[24];
+    (void) lru_cache_put (map->idsets,
+                          to_string (nodeid, id, sizeof (id)),
+                          idset);
+}
+
+static void decache_idset (struct taskmap *map, int nodeid)
+{
+    char id[24];
+    (void) lru_cache_remove (map->idsets,
+                             to_string (nodeid, id, sizeof (id)));
+}
+
+/*  Wrapper for zlistx_add_end that sets errno.
+ *  Note: zlistx_add_end() asserts on failure, so there is currently no
+ *  way this function can fail. Here we assume if in the future it does
+ *  then ENOMEM is the likely cause.
+ */
+static int append_to_zlistx (zlistx_t *l, void *item)
+{
+    if (!zlistx_add_end (l, item)) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+static void taskmap_find_repeats (struct taskmap *map)
+{
+    struct taskmap_block *block;
+    struct taskmap_block *prev = NULL;
+
+    prev = zlistx_first (map->blocklist);
+    block = zlistx_next (map->blocklist);
+    while (block) {
+        if (block->start == prev->start
+            && block->nnodes == prev->nnodes
+            && block->ppn == prev->ppn) {
+            prev->repeat += block->repeat;
+            zlistx_delete (map->blocklist, zlistx_cursor (map->blocklist));
+        }
+        else
+            prev = block;
+        block = zlistx_next (map->blocklist);
+    }
+}
+
+int taskmap_append (struct taskmap *map, int nodeid, int nnodes, int ppn)
+{
+    struct taskmap_block *block;
+
+    if (!map || nodeid < 0 || nnodes <= 0 || ppn <= 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    decache_idset (map, nodeid);
+    if ((block = zlistx_tail (map->blocklist))) {
+        /*  If previous block ends at nodeid - 1, and has the same ppn
+         *  and a repeat of 1, then add nnodes to the previous block
+         *  instead of appending a new block.
+         */
+        if (nodeid == taskmap_block_end (block) + 1
+            && ppn == block->ppn
+            && block->repeat == 1) {
+            block->nnodes += nnodes;
+            /*  Check for any new repeated blocks, then return
+             */
+            taskmap_find_repeats (map);
+            return 0;
+        }
+        /*  If previous block and this block are a single, identical
+         *  node, then increment block->ppn by ppn.
+         */
+        if (block->start == nodeid
+            && block->nnodes == 1
+            && nnodes == 1) {
+            block->ppn += ppn;
+            taskmap_find_repeats (map);
+            return 0;
+        }
+        /*  O/w, if previous block matches (nodeid, nnodes, ppn), then
+         *  increment the repeat of the previous block.
+         */
+        if (block->start == nodeid
+            && block->nnodes == nnodes
+            && block->ppn == ppn) {
+            block->repeat++;
+            return 0;
+        }
+        /* Else fall through and append a new block
+         */
+    }
+    if (!(block = taskmap_block_create (nodeid, nnodes, ppn, 1))
+        || append_to_zlistx (map->blocklist, block) < 0) {
+        taskmap_block_destroy (block);
+        return -1;
+    }
+    return 0;
+}
+
+static struct taskmap *taskmap_decode_array (json_t *o, flux_error_t *errp)
+{
+    size_t index;
+    json_t *entry;
+    struct taskmap *map = NULL;
+
+    if (!json_is_array (o)) {
+        errprintf (errp, "taskmap must be an array");
+        goto err;
+    }
+
+    if (!(map = taskmap_create ())) {
+        errprintf (errp, "Out of memory");
+        goto err;
+    }
+
+    json_array_foreach (o, index, entry) {
+        struct taskmap_block *block;
+        if (!json_is_array (entry)) {
+            errprintf (errp, "entry %zu in taskmap is not an array", index);
+            goto err;
+        }
+        if (!(block = taskmap_block_from_json (entry, errp))
+            || append_to_zlistx (map->blocklist, block) < 0)
+            goto err;
+    }
+    return map;
+err:
+    taskmap_destroy (map);
+    return NULL;
+}
+
+struct taskmap *taskmap_decode_json (json_t *o, flux_error_t *errp)
+{
+    struct taskmap *map = NULL;
+    json_t *array;
+
+    err_init (errp);
+
+    if (!o) {
+        errprintf (errp, "Invalid argument");
+        goto error;
+    }
+
+    array = o;
+    if (json_is_object (o)) {
+        int version;
+        json_error_t error;
+        if (json_unpack_ex (o, &error, 0,
+                            "{s:i s:o}",
+                            "version", &version,
+                            "map", &array) < 0) {
+            errprintf (errp, "%s", error.text);
+            goto error;
+        }
+        if (version != 1) {
+            errprintf (errp, "expected version=1, got %d", version);
+            goto error;
+        }
+    }
+    else if (!json_is_array (o)) {
+        errprintf (errp, "taskmap must be an object or array");
+        goto error;
+    }
+    map = taskmap_decode_array (array, errp);
+error:
+    return map;
+}
+
+static int parse_pmi_block (const char *s, int *nodeid, int *count, int *ppn)
+{
+    char *endptr;
+    errno = 0;
+    *nodeid = strtoul (s, &endptr, 10);
+    if (errno != 0 || *endptr != ',')
+        return -1;
+    s = endptr + 1;
+    *count = strtoul (s, &endptr, 10);
+    if (errno != 0 || *endptr != ',')
+        return -1;
+    s = endptr + 1;
+    *ppn = strtoul (s, &endptr, 10);
+    if (errno != 0 || *endptr != ')')
+        return -1;
+    return 0;
+}
+
+static bool is_empty (const char *s)
+{
+    while (isspace(*s))
+        s++;
+    return *s == '\0';
+}
+
+static struct taskmap *taskmap_decode_pmi (const char *s, flux_error_t *errp)
+{
+    char *tok;
+    char *p;
+    char *q;
+    char *cpy = NULL;
+    struct taskmap *map = NULL;
+    bool got_sentinel = false;
+
+    if (!s) {
+        errprintf (errp, "Invalid argument");
+        return NULL;
+    }
+
+    /* Empty PMI_process_mapping is allowed: return empty taskmap
+     */
+    if (strlen (s) == 0)
+        return taskmap_create ();
+
+    if (!(map = taskmap_create ())
+        || !(cpy = strdup (s))) {
+        errprintf (errp, "Out of memory");
+        goto error;
+    }
+
+    p = cpy;
+    while ((tok = strtok_r (p, "(", &q))) {
+        int nodeid = -1;
+        int count = -1;
+        int ppn = -1;
+
+        while (isspace (*tok))
+            tok++;
+
+        if (strstarts (tok, "vector,"))
+            got_sentinel = true;
+        else if (!is_empty (tok)) {
+            if (!got_sentinel) {
+                errprintf (errp, "vector prefix must precede blocklist");
+                goto error;
+            }
+            if (parse_pmi_block (tok, &nodeid, &count, &ppn) < 0) {
+                errprintf (errp, "unable to parse block: (%s", tok);
+                goto error;
+            }
+            if (nodeid < 0 || count <= 0 || ppn <= 0) {
+                errprintf (errp, "invalid number in block: (%s", tok);
+                goto error;
+            }
+            if (taskmap_append (map, nodeid, count, ppn) < 0) {
+                errprintf (errp, "taskmap_append: %s", strerror (errno));
+                goto error;
+            }
+        }
+        p = NULL;
+    }
+    if (taskmap_total_ntasks (map) == 0) {
+        errprintf (errp, "no tasks found in PMI_process_mapping");
+        goto error;
+    }
+    free (cpy);
+    return map;
+error:
+    taskmap_destroy (map);
+    free (cpy);
+    return NULL;
+}
+
+struct raw_task {
+    int taskid;
+    int nodeid;
+    int repeat;
+};
+
+static void item_destructor (void **item)
+{
+    if (item) {
+        free (*item);
+        *item = NULL;
+    }
+}
+
+static int taskid_cmp (const void *a, const void *b)
+{
+    const struct raw_task *t1 = a;
+    const struct raw_task *t2 = b;
+    return (t1->taskid - t2->taskid);
+}
+
+static int raw_task_append (zlistx_t *l, int taskid, int nodeid, int repeat)
+{
+    struct raw_task *t = calloc (1, sizeof (*t));
+    if (!t)
+        return -1;
+    t->taskid = taskid;
+    t->nodeid = nodeid;
+    t->repeat = repeat;
+    if (!zlistx_add_end (l, t)) {
+        free (t);
+        return -1;
+    }
+    return 0;
+}
+
+static zlistx_t *raw_task_list_create (void)
+{
+    zlistx_t *l;
+    if (!(l = zlistx_new ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    zlistx_set_destructor (l, item_destructor);
+    zlistx_set_comparator (l, &taskid_cmp);
+    return l;
+}
+
+static int raw_task_list_append (zlistx_t *l,
+                                 const char *s,
+                                 int nodeid,
+                                 flux_error_t *errp)
+{
+    int rc = -1;
+    unsigned int id;
+    idset_error_t error;
+    struct idset *ids;
+
+    if (!(ids = idset_decode_ex (s, -1, 0, IDSET_FLAG_AUTOGROW, &error))) {
+        errprintf (errp, "%s", error.text);
+        goto error;
+    }
+    id = idset_first (ids);
+    while (id != IDSET_INVALID_ID) {
+        unsigned int next = idset_next (ids, id);
+        int repeat = 1;
+        while (next == id + repeat) {
+            next = idset_next (ids, next);
+            repeat++;
+        }
+        if (raw_task_append (l, id, nodeid, repeat) < 0) {
+            errprintf (errp, "Out of memory");
+            goto error;
+        }
+        id = next;
+    }
+    rc = 0;
+error:
+    idset_destroy (ids);
+    return rc;
+}
+
+static int raw_task_check (struct raw_task *a,
+                           struct raw_task *b,
+                           flux_error_t *errp)
+{
+    struct raw_task t_init = { .taskid = -1, .repeat = 1 };
+    int start, end1, end2, end;
+
+    if (a == NULL)
+        a = &t_init;
+
+    /*  Note: a->taskid <= b->taskid since taskmap_decode_raw() sorts
+     *   raw_task objects.
+     */
+    start = b->taskid;
+    end1 = a->taskid + a->repeat - 1;
+    end2 = b->taskid + b->repeat - 1;
+    end = end1 <= end2 ? end1 : end2;
+
+    /* If end - start is nonzero then we have overlap. report it.
+     */
+    int overlap = end - start;
+    if (overlap >= 0) {
+        /* taskid overlap detected, report as error
+         */
+        if (overlap == 0)
+            errprintf (errp, "duplicate taskid specified: %d", start);
+        else
+            errprintf (errp, "duplicate taskids specified: %d-%d", start, end);
+        return -1;
+    }
+    /*  Now check that tasks are consecutive. It is an error if not since
+     *  holes in taskids in a taskmap are not allowed
+     */
+    if (overlap != -1) {
+        if (overlap == -2)
+            return errprintf (errp, "missing taskid: %d", end + 1);
+        else
+            return errprintf (errp,
+                              "missing taskids: %d-%d",
+                              end + 1,
+                              end - overlap - 1);
+    }
+    return 0;
+}
+
+static struct taskmap *taskmap_decode_raw (const char *s, flux_error_t *errp)
+{
+    char *tok;
+    char *p;
+    char *q;
+    char *cpy = NULL;
+    struct taskmap *map = NULL;
+    zlistx_t *l = NULL;
+    int nodeid = 0;
+    struct raw_task *t, *prev;
+
+    if (!s || strlen (s) == 0) {
+        errprintf (errp, "Invalid argument");
+        return NULL;
+    }
+    if (!(map = taskmap_create ())
+        || !(cpy = strdup (s))
+        || !(l = raw_task_list_create ())) {
+        errprintf (errp, "Out of memory");
+        goto error;
+    }
+
+    p = cpy;
+
+    while ((tok = strtok_r (p, ";", &q))) {
+        if (raw_task_list_append (l, tok, nodeid++, errp) < 0)
+            goto error;
+        p = NULL;
+    }
+
+    /* sort by taskid */
+    zlistx_sort (l);
+    t = zlistx_first (l);
+    prev = NULL;
+
+    while (t) {
+        if (raw_task_check (prev, t, errp) < 0)
+            goto error;
+        if (taskmap_append (map, t->nodeid, 1, t->repeat) < 0) {
+            errprintf (errp, "taskmap_append: %s", strerror (errno));
+            goto error;
+        }
+        prev = t;
+        t = zlistx_next (l);
+    }
+    zlistx_destroy (&l);
+    free (cpy);
+    return map;
+error:
+    zlistx_destroy (&l);
+    taskmap_destroy (map);
+    free (cpy);
+    return NULL;
+}
+
+struct taskmap *taskmap_decode (const char *s, flux_error_t *errp)
+{
+    struct taskmap *map = NULL;
+    json_t *o = NULL;
+    json_error_t error;
+
+    err_init (errp);
+
+    if (s == NULL) {
+        errprintf (errp, "Invalid argument");
+        goto out;
+    }
+
+    /*  Empty string or string containing "vector," may be a valid
+     *  PMI_process_mapping. Pass to taskmap_decode_pmi().
+     */
+    if (strlen (s) == 0
+        || strstr (s, "vector,"))
+        return taskmap_decode_pmi (s, errp);
+
+    /*  A string without special characters might be a raw taskmap:
+     */
+    if (!strpbrk (s, "({[]})"))
+        return taskmap_decode_raw (s, errp);
+
+    /*  O/w, decode as RFC 34 Taskmap
+     */
+    if (!(o = json_loads (s, JSON_DECODE_ANY, &error))) {
+        errprintf (errp, "%s", error.text);
+        goto out;
+    }
+    map = taskmap_decode_json (o, errp);
+out:
+    json_decref (o);
+    return map;
+}
+
+static struct idset *lookup_idset (const struct taskmap *map, int nodeid)
+{
+    char id[24];
+    return lru_cache_get (map->idsets, to_string (nodeid, id, sizeof (id)));
+}
+
+const struct idset *taskmap_taskids (const struct taskmap *map, int nodeid)
+{
+    int current;
+    struct taskmap_block *block;
+    struct idset *taskids;
+
+    if (!map || nodeid < 0 || taskmap_unknown (map)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if ((taskids = lookup_idset (map, nodeid)))
+        return taskids;
+
+    if (!(taskids = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        return NULL;
+
+    current = 0;
+    block = zlistx_first (map->blocklist);
+    while (block) {
+        for (int n = 0; n < block->repeat; n++) {
+            int start = block->start;
+            if (nodeid >= start && nodeid <= taskmap_block_end (block)) {
+                int offset = nodeid - start;
+                start = current + (offset * block->ppn);
+                idset_range_set (taskids, start, start + block->ppn - 1);
+            }
+            current += block->nnodes * block->ppn;
+        }
+        block = zlistx_next (map->blocklist);
+    }
+
+    if (idset_count (taskids) == 0) {
+        idset_destroy (taskids);
+        errno = ENOENT;
+        return NULL;
+    }
+
+    cache_idset (map, nodeid, taskids);
+    return taskids;
+}
+
+int taskmap_nodeid (const struct taskmap *map, int taskid)
+{
+    struct taskmap_block *block;
+    int current = 0;
+
+    if (!map || taskid < 0 || taskmap_unknown (map)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    block = zlistx_first (map->blocklist);
+    while (block) {
+        for (int n = 0; n < block->repeat; n++) {
+            int last = current + block->nnodes * block->ppn - 1;
+            if (taskid <= last) {
+                int distance = taskid - current;
+                return block->start + (distance / block->ppn);
+            }
+            current = last + 1;
+        }
+        block = zlistx_next (map->blocklist);
+    }
+    errno = ENOENT;
+    return -1;
+}
+
+int taskmap_ntasks (const struct taskmap *map, int nodeid)
+{
+    const struct idset *taskids = taskmap_taskids (map, nodeid);
+    if (!taskids)
+        return -1;
+    return idset_count (taskids);
+}
+
+int taskmap_nnodes (const struct taskmap *map)
+{
+    struct taskmap_block *block;
+    int n;
+
+    if (!map || taskmap_unknown (map)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    n = 0;
+    block = zlistx_first (map->blocklist);
+    while (block) {
+        int end = block->start + block->nnodes;
+        if (n < end)
+            n = end;
+        block = zlistx_next (map->blocklist);
+    }
+    return n;
+}
+
+int taskmap_total_ntasks (const struct taskmap *map)
+{
+    struct taskmap_block *block;
+    int n;
+
+    if (!map || taskmap_unknown (map)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    n = 0;
+    block = zlistx_first (map->blocklist);
+    while (block) {
+        n += block->nnodes * block->repeat * block->ppn;
+        block = zlistx_next (map->blocklist);
+    }
+    return n;
+}
+
+static json_t *taskmap_block_encode (struct taskmap_block *block)
+{
+    return json_pack ("[iiii]",
+                      block->start,
+                      block->nnodes,
+                      block->ppn,
+                      block->repeat);
+}
+
+json_t *taskmap_encode_json (const struct taskmap *map, int flags)
+{
+    struct taskmap_block *block;
+    json_t *blocks = NULL;
+    json_t *taskmap = NULL;
+
+    if (!(blocks = json_array ()))
+        goto error;
+
+    block = zlistx_first (map->blocklist);
+    while (block) {
+        json_t *o = taskmap_block_encode (block);
+        if (!o)
+            goto error;
+        if (json_array_append_new (blocks, o) < 0) {
+            json_decref (o);
+            goto error;
+        }
+        block = zlistx_next (map->blocklist);
+    }
+    if (!(flags & TASKMAP_ENCODE_WRAPPED))
+        return blocks;
+    if (!(taskmap = json_pack ("{s:i s:o}",
+                               "version", 1,
+                               "map", blocks)))
+        goto error;
+    return taskmap;
+error:
+    json_decref (blocks);
+    json_decref (taskmap);
+
+    /* Note: Only possible reason for error exit is ENOMEM
+     */
+    errno = ENOMEM;
+    return NULL;
+}
+
+static bool valid_encode_flags (int flags)
+{
+    int count = 0;
+    int possible_flags = TASKMAP_ENCODE_WRAPPED
+                         | TASKMAP_ENCODE_PMI
+                         | TASKMAP_ENCODE_RAW
+                         | TASKMAP_ENCODE_RAW_DERANGED;
+    if ((flags & possible_flags) != flags)
+        return false;
+    while (flags) {
+        flags = flags & (flags - 1); /* clear the least significant bit set */
+        count++;
+    }
+    if (count > 1)
+        return false;
+    return true;
+}
+
+static char *taskmap_encode_map (const struct taskmap *map, int flags)
+{
+    char *s;
+    json_t *taskmap;
+    if (!(taskmap = taskmap_encode_json (map, flags)))
+        return NULL;
+    if (!(s = json_dumps (taskmap, JSON_COMPACT)))
+        errno = ENOMEM;
+    ERRNO_SAFE_WRAP (json_decref, taskmap);
+    return s;
+}
+
+static char *list_join (zlistx_t *l, char *sep)
+{
+    char *result = NULL;
+    char *s;
+    int seplen = strlen (sep);
+    int len = 0;
+    int n = 0;
+
+    /* special case: zero length list returns zero length string
+    */
+    if (zlistx_size (l) == 0)
+        return strdup ("");
+
+    s = zlistx_first (l);
+    while (s) {
+        len += strlen (s) + seplen;
+        s = zlistx_next (l);
+    }
+    if (!(result = malloc (len+1)))
+        return NULL;
+
+    s = zlistx_first (l);
+    n = 0;
+    while (s) {
+        n += sprintf (result+n, "%s%s", s, sep);
+        s = zlistx_next (l);
+    }
+    result[len - seplen] = '\0';
+    return result;
+}
+
+static char *taskmap_encode_raw (const struct taskmap *map, int flags)
+{
+    char *result = NULL;
+    zlistx_t *l;
+    int nnodes;
+    int saved_errno;
+
+    if (!(l = zlistx_new ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    zlistx_set_destructor (l, item_destructor);
+
+    nnodes = taskmap_nnodes (map);
+    for (int i = 0; i < nnodes; i++) {
+        const struct idset *ids = NULL;
+        char *s = NULL;
+        if (!(ids = taskmap_taskids (map, i))
+            || !(s = idset_encode (ids, flags))
+            || append_to_zlistx (l, s) < 0) {
+            ERRNO_SAFE_WRAP (free, s);
+            goto error;
+        }
+    }
+    result = list_join (l, ";");
+error:
+    saved_errno = errno;
+    zlistx_destroy (&l);
+    errno = saved_errno;
+    return result;
+}
+
+static char *taskmap_encode_pmi (const struct taskmap *map)
+{
+    char *result = NULL;
+    char *s;
+    struct taskmap_block *block;
+    zlistx_t *l;
+    int saved_errno;
+
+    if (taskmap_unknown (map))
+        return strdup ("");
+
+    if (!(l = zlistx_new ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    zlistx_set_destructor (l, item_destructor);
+
+    block = zlistx_first (map->blocklist);
+    while (block) {
+        for (int i = 0; i < block->repeat; i++)
+            if (asprintf (&s,
+                          "(%d,%d,%d)",
+                          block->start,
+                          block->nnodes,
+                          block->ppn) < 0
+                || append_to_zlistx (l, s) < 0)
+                goto error;
+        block = zlistx_next (map->blocklist);
+    }
+    if (!(s = list_join (l, ",")))
+        goto error;
+    if (asprintf (&result, "(vector,%s)", s) < 0)
+        result = NULL;
+error:
+    saved_errno = errno;
+    free (s);
+    zlistx_destroy (&l);
+    errno = saved_errno;
+    return result;
+}
+
+char *taskmap_encode (const struct taskmap *map, int flags)
+{
+    if (!map || !valid_encode_flags (flags)) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (flags & TASKMAP_ENCODE_RAW)
+        return taskmap_encode_raw (map, IDSET_FLAG_RANGE);
+    if (flags & TASKMAP_ENCODE_RAW_DERANGED)
+        return taskmap_encode_raw (map, 0);
+    if (flags & TASKMAP_ENCODE_PMI)
+        return taskmap_encode_pmi (map);
+    return taskmap_encode_map (map, flags);
+}
+
+int taskmap_check (const struct taskmap *old,
+                   const struct taskmap *new,
+                   flux_error_t *errp)
+{
+    int nnodes_old, nnodes_new;
+    int ntasks_old, ntasks_new;
+    if (!old || !new)
+        return errprintf (errp, "Invalid argument");
+    nnodes_old = taskmap_nnodes (old);
+    nnodes_new = taskmap_nnodes (new);
+    if (nnodes_old != nnodes_new)
+        return errprintf (errp,
+                          "got %d nodes, expected %d",
+                          nnodes_new,
+                          nnodes_old);
+    ntasks_old = taskmap_total_ntasks (old);
+    ntasks_new = taskmap_total_ntasks (new);
+    if (ntasks_old != ntasks_new)
+        return errprintf (errp,
+                          "got %d total tasks, expected %d",
+                          ntasks_new,
+                          ntasks_old);
+    for (int i = 0; i < nnodes_old; i++) {
+        ntasks_old = taskmap_ntasks (old, i);
+        ntasks_new = taskmap_ntasks (new, i);
+        if (ntasks_old != ntasks_new)
+            return errprintf (errp,
+                              "node %d has %d tasks, expected %d",
+                              i,
+                              ntasks_new,
+                              ntasks_old);
+    }
+    return 0;
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libtaskmap/taskmap.h b/src/common/libtaskmap/taskmap.h
new file mode 100644
index 000000000000..c4b3a4292995
--- /dev/null
+++ b/src/common/libtaskmap/taskmap.h
@@ -0,0 +1,109 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_TASKMAP_H
+#define _UTIL_TASKMAP_H
+
+#include <flux/core.h>
+#include <flux/idset.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum taskmap_flags {
+    TASKMAP_ENCODE_WRAPPED = 1,      /* Encode as RFC 34 wrapped object      */
+    TASKMAP_ENCODE_PMI =     1 << 1, /* Encode as PMI-1 PMI_process_mapping  */
+    TASKMAP_ENCODE_RAW =     1 << 2, /* Encode as semicolon-delimited taskids*/
+    TASKMAP_ENCODE_RAW_DERANGED = 1 << 3, /* Encode as raw without ranges    */
+};
+
+/*  Create an empty taskmap
+ *  Returns taskmap on success, NULL on failure with errno set.
+ */
+struct taskmap *taskmap_create ();
+
+/*  Destroy a taskmap
+ */
+void taskmap_destroy (struct taskmap *map);
+
+/*  Append a block of tasks to a taskmap starting at 'nodeid', for 'nnodes'
+ *   with 'ppn' tasks per node.
+ *  Returns 0 on success, -1 on failure with errno set.
+ */
+int taskmap_append (struct taskmap *map, int nodeid, int nnodes, int ppn);
+
+/*  Decode string 'map' into taskmap object.
+ *  The string may be a JSON array, RFC 34 wrapped object, a mapping
+ *   encoded in PMI-1 PMI_process_mapping form described in RFC 13, or
+ *   a raw, semicolon-delimited list of taskids.
+ *  Returns taskmap on success, or NULL on error with error string in 'errp'.
+ */
+struct taskmap *taskmap_decode (const char *map, flux_error_t *errp);
+
+/*  Encode taskmap 'map' to a string, which the caller must free.
+ *  'flags' may indicate
+ *    TASKMAP_ENCODE_WRAPPED to create an RFC 34 wrapped taskmap object.
+ *    TASKMAP_ENCODE_PMI to create a PMI-1 PMI_process_mapping encoding
+ *    TASKMAP_ENCODE_RAW to create a semicolon-delimited list of taskids
+ *  The default is to encode as a JSON array.
+ *  Returns string on success, or NULL on failure with errno set.
+ */
+char *taskmap_encode (const struct taskmap *map, int flags);
+
+/*  Return true if the task mapping is unknown.
+ */
+bool taskmap_unknown (const struct taskmap *map);
+
+/*  Return an idset of taskids encoded in 'map' for nodeid 'nodeid'.
+ *  Returns an idset on success, which may only be valid until the next
+ *   call to taskmap_taskids(), caller should use idset_copy() if necessary.
+ *  Returns NULL on error with errno set.
+ */
+const struct idset *taskmap_taskids (const struct taskmap *map, int nodeid);
+
+/*  Return the nodeid which contains task id 'taskid' in taskmap 'map'.
+ *  Returns the nodeid or -1 on failure with errno set.
+ */
+int taskmap_nodeid (const struct taskmap *map, int taskid);
+
+/*  Return the total number of tasks assigned to node 'nodeid' in 'map'.
+ *  Returns a task count or -1 on failure with errno set.
+ */
+int taskmap_ntasks (const struct taskmap *map, int nodeid);
+
+/*  Return the total number of nodes in 'map'.
+ *  Returns a count of nodes on success, -1 on failure with errno set.
+ */
+int taskmap_nnodes (const struct taskmap *map);
+
+/*  Return the total number of tasks in 'map'.
+ *  Returns a count of tasks on success, -1 on failure with errno set.
+ */
+int taskmap_total_ntasks (const struct taskmap *map);
+
+/*  Check if 'a' and 'b' taskmaps are compatible, i.e. they
+ *  have equivalent numbers of total tasks, total nodes, and tasks assigned
+ *  to each node.
+ *  Returns 0 on success, -1 with error message in 'errp' on failure.
+ */
+int taskmap_check (const struct taskmap *a,
+                   const struct taskmap *b,
+                   flux_error_t *errp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_UTIL_TASKMAP_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libtaskmap/taskmap_private.h b/src/common/libtaskmap/taskmap_private.h
new file mode 100644
index 000000000000..087210647bd2
--- /dev/null
+++ b/src/common/libtaskmap/taskmap_private.h
@@ -0,0 +1,25 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_TASKMAP_PRIVATE_H
+#define _UTIL_TASKMAP_PRIVATE_H
+
+#include <jansson.h>
+#include <flux/taskmap.h>
+
+json_t *taskmap_encode_json (const struct taskmap *map, int flags);
+
+struct taskmap *taskmap_decode_json (json_t *o, flux_error_t *errp);
+
+#endif /* !_UTIL_TASKMAP_PRIVATE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libtaskmap/test/taskmap.c b/src/common/libtaskmap/test/taskmap.c
new file mode 100644
index 000000000000..b0eff19cd89b
--- /dev/null
+++ b/src/common/libtaskmap/test/taskmap.c
@@ -0,0 +1,615 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+
+#include <flux/taskmap.h>
+#include "taskmap_private.h"
+
+#include "src/common/libtap/tap.h"
+
+struct test_args {
+    const char *input;
+    int nnodes;
+    int total_nnodes;
+    int total_ntasks;
+    char *idsets[24];
+};
+
+struct test_vector {
+    const char *taskmap;
+    const char *expected;
+};
+
+struct test_vector rfc34_test_vectors[] = {
+    { "[]", "" },
+    { "[[0,1,1,1]]",
+      "0" },
+    { "[[0,2,1,1]]",
+      "0;1" },
+    { "[[0,1,2,1]]",
+      "0-1" },
+    { "[[0,2,2,1]]",
+      "0-1;2-3" },
+    { "[[0,2,1,2]]",
+      "0,2;1,3" },
+    { "[[1,1,1,1],[0,1,1,1]]",
+      "1;0" },
+    { "[[0,4,4,1]]",
+      "0-3;4-7;8-11;12-15" },
+    { "[[0,4,1,4]]",
+      "0,4,8,12;1,5,9,13;2,6,10,14;3,7,11,15" },
+    { "[[0,4,2,2]]",
+      "0-1,8-9;2-3,10-11;4-5,12-13;6-7,14-15" },
+    { "[[0,4,2,1],[4,2,4,1]]",
+      "0-1;2-3;4-5;6-7;8-11;12-15" },
+    { "[[0,6,1,2],[4,2,1,2]]",
+      "0,6;1,7;2,8;3,9;4,10,12,14;5,11,13,15" },
+    { "[[5,1,4,1],[4,1,4,1],[3,1,2,1],[2,1,2,1],[1,1,2,1],[0,1,2,1]]",
+      "14-15;12-13;10-11;8-9;4-7;0-3" },
+    { "[[0,5,2,1],[6,1,2,1],[5,1,2,1],[7,1,2,1]]",
+      "0-1;2-3;4-5;6-7;8-9;12-13;10-11;14-15" },
+    { "[[3,1,4,1],[2,1,4,1],[1,1,4,1],[0,1,4,1]]",
+      "12-15;8-11;4-7;0-3" },
+    { NULL, NULL },
+};
+
+static void rfc34_tests ()
+{
+    struct test_vector *t;
+    for (t = &rfc34_test_vectors[0]; t->taskmap != NULL; t++) {
+        char *s;
+        struct taskmap *map = taskmap_decode (t->taskmap, NULL);
+        if (!map)
+            BAIL_OUT("taskmap_decode failed!");
+        ok (map != NULL,
+            "taskmap_decode (%s)",
+            t->taskmap);
+        ok ((s = taskmap_encode (map, TASKMAP_ENCODE_RAW)) != NULL,
+            "taskmap_encode_raw works");
+        is (s, t->expected,
+            "taskmap raw=%s",
+            s);
+        if (strlen (s))
+            ok (taskmap_unknown (map) == false,
+                "taskmap is known");
+        taskmap_destroy (map);
+        free (s);
+
+        /*  Try raw back to taskmap:
+         */
+        map = taskmap_decode (t->expected, NULL);
+        ok (map != NULL,
+            "taskmap_decode (%s)",
+            t->expected);
+        if (map) {
+            ok ((s = taskmap_encode (map, 0)) != NULL,
+            "taskmap_encode works");
+            is (s, t->taskmap,
+                "taskmap=%s",
+                s);
+            taskmap_destroy (map);
+            free (s);
+        }
+    }
+}
+
+struct test_vector pmi_tests[] = {
+    { "[]", "" },
+    { "[[0,4,4,1]]", "(vector,(0,4,4))" },
+    { "[[0,4,2,1],[4,2,4,1]]", "(vector,(0,4,2),(4,2,4))" },
+    { "[[0,4,1,4]]", "(vector,(0,4,1),(0,4,1),(0,4,1),(0,4,1))" },
+    { "[[0,4096,256,1]]", "(vector,(0,4096,256))" },
+    { NULL, NULL },
+};
+
+struct test_vector pmi_decode_tests[] = {
+    { "", "[]" },
+    { "(vector,(0,1,4))", "[[0,1,4,1]]" },
+    { "(vector,(0,2,2))", "[[0,2,2,1]]" },
+    { "(vector,(0,16,16))", "[[0,16,16,1]]" },
+    { "(vector,(0,8,16),(0,4,32))", "[[0,8,16,1],[0,4,32,1]]" },
+    { "(vector,(0,4,2),(1,3,1))", "[[0,4,2,1],[1,3,1,1]]" },
+    { "(vector,(0,4,1),(0,4,1),(0,4,1),(0,4,1))", "[[0,4,1,4]]" },
+    { "(vector,(0,4,4),(0,4,1))", "[[0,4,4,1],[0,4,1,1]]" },
+    { "    (vector, (0,4,4), (0,4,1), )", "[[0,4,4,1],[0,4,1,1]]" },
+    { "(vector, (1,1,1), (0,2,2))", "[[1,1,1,1],[0,2,2,1]]" },
+    { "(vector, (1,1,1), (0,2,2),)", "[[1,1,1,1],[0,2,2,1]]" },
+    { "(vector, (0,1,1), (1,5,3), (6,2, 5))",
+      "[[0,1,1,1],[1,5,3,1],[6,2,5,1]]" },
+    { NULL, NULL },
+};
+
+struct test_vector pmi_invalid[] = {
+    { "vector, (1,1))", "unable to parse block: (1,1))" },
+    { "(vector, (1.11, 2.2))", "unable to parse block: (1.11, 2.2))" },
+    { "(vector, (1,1,0))", "invalid number in block: (1,1,0))" },
+    { "((1,1,1))", "invalid token near '('" },
+    { "((1,1,1), vector,)", "vector prefix must precede blocklist" },
+    { NULL, NULL },
+};
+
+static void pmi_mapping_tests ()
+{
+    struct test_vector *t;
+    for (t = &pmi_tests[0]; t->taskmap != NULL; t++) {
+        char *s;
+        struct taskmap *map2;
+        struct taskmap *map = taskmap_decode (t->taskmap, NULL);
+        if (!map)
+            BAIL_OUT("taskmap_decode failed!");
+        ok (map != NULL,
+            "taskmap_decode (%s)",
+            t->taskmap);
+        ok ((s = taskmap_encode (map, TASKMAP_ENCODE_PMI)) != NULL,
+            "taskmap_encode_pmi works");
+        is (s, t->expected,
+            "taskmap pmi=%s",
+            s);
+        ok ((map2 = taskmap_decode (s, NULL)) != NULL,
+            "taskmap_decode (%s)",
+            s);
+        free (s);
+        ok ((s = taskmap_encode (map, 0)) != NULL,
+            "taskmap_encode works");
+        is (s, t->taskmap,
+            "taskmap=%s",
+            s);
+        taskmap_destroy (map);
+        taskmap_destroy (map2);
+        free (s);
+    }
+
+    for (t = &pmi_decode_tests[0]; t->taskmap != NULL; t++) {
+        char *s;
+        struct taskmap *map = taskmap_decode (t->taskmap, NULL);
+        if (!map)
+            BAIL_OUT ("taskmap_decode failed!");
+        ok (map != NULL,
+            "taskmap_decode (%s)",
+            t->taskmap);
+        ok ((s = taskmap_encode (map, 0)) != NULL,
+            "taskmap_encode works");
+        is (s, t->expected,
+            "taskmap map=%s",
+            s);
+        taskmap_destroy (map);
+        free (s);
+    }
+
+    for (t = &pmi_invalid[0]; t->taskmap != NULL; t++) {
+        flux_error_t error;
+        ok (taskmap_decode (t->taskmap, &error) == NULL,
+            "taskmap_decode (%s) fails",
+            t->taskmap);
+        is (error.text, t->expected,
+            "got error %s",
+            error.text);
+    }
+}
+
+struct test_args tests[] = {
+    { "[[0,2,2,1]]", 2, 2, 4, { "0-1", "2-3" } },
+    { "[[0,2,1,2]]", 2, 2, 4, { "0,2", "1,3" } },
+    { "[[0,16,16,1]]", 16, 16, 256,
+      { "0-15", "16-31", "32-47", "48-63", "64-79", "80-95",
+        "96-111", "112-127", "128-143", "144-159", "160-175",
+        "176-191", "192-207", "208-223", "224-239", "240-255",
+        NULL,
+      },
+    },
+    { "[[0,8,16,1],[8,4,32,1]]", 12, 12, 256,
+      { "0-15", "16-31", "32-47", "48-63", "64-79", "80-95",
+        "96-111", "112-127", "128-159", "160-191", "192-223", "224-255",
+        NULL
+      },
+    },
+    { "[[0,4096,1,256]]", 2, 4096, 1048576,
+      { "0,4096,8192,12288,16384,20480,24576,28672,32768,36864,40960,45056,49152,53248,57344,61440,65536,69632,73728,77824,81920,86016,90112,94208,98304,102400,106496,110592,114688,118784,122880,126976,131072,135168,139264,143360,147456,151552,155648,159744,163840,167936,172032,176128,180224,184320,188416,192512,196608,200704,204800,208896,212992,217088,221184,225280,229376,233472,237568,241664,245760,249856,253952,258048,262144,266240,270336,274432,278528,282624,286720,290816,294912,299008,303104,307200,311296,315392,319488,323584,327680,331776,335872,339968,344064,348160,352256,356352,360448,364544,368640,372736,376832,380928,385024,389120,393216,397312,401408,405504,409600,413696,417792,421888,425984,430080,434176,438272,442368,446464,450560,454656,458752,462848,466944,471040,475136,479232,483328,487424,491520,495616,499712,503808,507904,512000,516096,520192,524288,528384,532480,536576,540672,544768,548864,552960,557056,561152,565248,569344,573440,577536,581632,585728,589824,593920,598016,602112,606208,610304,614400,618496,622592,626688,630784,634880,638976,643072,647168,651264,655360,659456,663552,667648,671744,675840,679936,684032,688128,692224,696320,700416,704512,708608,712704,716800,720896,724992,729088,733184,737280,741376,745472,749568,753664,757760,761856,765952,770048,774144,778240,782336,786432,790528,794624,798720,802816,806912,811008,815104,819200,823296,827392,831488,835584,839680,843776,847872,851968,856064,860160,864256,868352,872448,876544,880640,884736,888832,892928,897024,901120,905216,909312,913408,917504,921600,925696,929792,933888,937984,942080,946176,950272,954368,958464,962560,966656,970752,974848,978944,983040,987136,991232,995328,999424,1003520,1007616,1011712,1015808,1019904,1024000,1028096,1032192,1036288,1040384,1044480",
+       "1,4097,8193,12289,16385,20481,24577,28673,32769,36865,40961,45057,49153,53249,57345,61441,65537,69633,73729,77825,81921,86017,90113,94209,98305,102401,106497,110593,114689,118785,122881,126977,131073,135169,139265,143361,147457,151553,155649,159745,163841,167937,172033,176129,180225,184321,188417,192513,196609,200705,204801,208897,212993,217089,221185,225281,229377,233473,237569,241665,245761,249857,253953,258049,262145,266241,270337,274433,278529,282625,286721,290817,294913,299009,303105,307201,311297,315393,319489,323585,327681,331777,335873,339969,344065,348161,352257,356353,360449,364545,368641,372737,376833,380929,385025,389121,393217,397313,401409,405505,409601,413697,417793,421889,425985,430081,434177,438273,442369,446465,450561,454657,458753,462849,466945,471041,475137,479233,483329,487425,491521,495617,499713,503809,507905,512001,516097,520193,524289,528385,532481,536577,540673,544769,548865,552961,557057,561153,565249,569345,573441,577537,581633,585729,589825,593921,598017,602113,606209,610305,614401,618497,622593,626689,630785,634881,638977,643073,647169,651265,655361,659457,663553,667649,671745,675841,679937,684033,688129,692225,696321,700417,704513,708609,712705,716801,720897,724993,729089,733185,737281,741377,745473,749569,753665,757761,761857,765953,770049,774145,778241,782337,786433,790529,794625,798721,802817,806913,811009,815105,819201,823297,827393,831489,835585,839681,843777,847873,851969,856065,860161,864257,868353,872449,876545,880641,884737,888833,892929,897025,901121,905217,909313,913409,917505,921601,925697,929793,933889,937985,942081,946177,950273,954369,958465,962561,966657,970753,974849,978945,983041,987137,991233,995329,999425,1003521,1007617,1011713,1015809,1019905,1024001,1028097,1032193,1036289,1040385,1044481",
+       NULL },
+    },
+    { NULL, 0, 0, 0, {0} },
+};
+
+static bool check_all_tasks (struct taskmap *map,
+                             const struct idset *taskids,
+                             int nodeid)
+{
+    unsigned int i = idset_first (taskids);
+
+    while (i != IDSET_INVALID_ID) {
+        int n = taskmap_nodeid (map, i);
+        if (n != nodeid) {
+            fail ("task %u is on node %d (expected %d)",
+                  i,
+                  n,
+                  nodeid);
+            return false;
+        }
+        i = idset_next (taskids, i);
+    }
+    return true;
+}
+
+static void main_tests ()
+{
+    struct test_args *t;
+
+    for (t = &tests[0]; t->input != NULL; t++) {
+        flux_error_t error;
+        char *s;
+        struct taskmap *map = taskmap_decode (t->input, &error);
+        if (!map)
+            BAIL_OUT ("taskmap_decode(%s): %s", t->input, error.text);
+        ok (map != NULL,
+            "taskmap_decode (%s)",
+            t->input);
+        ok (taskmap_nnodes (map) == t->total_nnodes,
+            "taskmap_nnodes returned %d (expected %d)",
+            taskmap_nnodes (map),
+            t->total_nnodes);
+        ok (taskmap_total_ntasks (map) == t->total_ntasks,
+            "taskmap_total_ntasks returned %d (expected %d)",
+            taskmap_total_ntasks (map),
+            t->total_ntasks);
+        ok ((s = taskmap_encode (map, 0)) != NULL,
+            "taskmap_encode works");
+        is (s, t->input,
+            "taskmap_encode returns expected string: %s", s);
+        free (s);
+        for (int i = 0; i < t->nnodes; i++) {
+            const struct idset *taskids = taskmap_taskids (map, i);
+            if (!taskids)
+                BAIL_OUT ("taskmap_taskids (%s, %d) failed", t->input, i);
+            char *s = idset_encode (taskids, IDSET_FLAG_RANGE);
+            is (s, t->idsets[i],
+                "node %d idset is %s", i, s);
+
+            ok (check_all_tasks (map, taskids, i),
+                "%d tasksids on nodeid %d",
+                idset_count (taskids), i);
+
+            free (s);
+        }
+        taskmap_destroy (map);
+    }
+}
+
+static const char *invalid[] = {
+    "}",
+    "{}",
+    "{\"version\":1}",
+    "{\"version\":1,\"map\":{}}",
+    "{\"version\":2,\"map\":[[1,1,1,1]]}",
+    "{\"version\":1,\"map\":[[]]}",
+    "{\"version\":1,\"map\":[[\"1\",\"1\",\"1\"]]}",
+    "[[-1,1,1,1]]",
+    "[[0,1,1,1],[-1,1,1,1]]",
+    "[[0,1,1,1],1]",
+    NULL
+};
+
+static void error_tests ()
+{
+    struct taskmap *map;
+    flux_error_t error;
+
+    if (!(map = taskmap_create ()))
+        BAIL_OUT ("taskmap_create");
+
+    /* Test "unknown" task map errors */
+    ok (taskmap_unknown (map),
+        "taskmap_unknown returns true for empty task map");
+    ok (taskmap_nnodes (map) < 0 && errno == EINVAL,
+        "taskmap_nnodes on unknown taskmap returns EINVAL");
+    ok (taskmap_total_ntasks (map) < 0 && errno == EINVAL,
+        "taskmap_nnodes on unknown taskmap returns EINVAL");
+    ok (taskmap_nodeid (map, 0) < 0 && errno == EINVAL,
+        "taskmap_nodeid on unknown taskmap returns EINVAL");
+    ok (taskmap_taskids (map, 0) == NULL && errno == EINVAL,
+        "taskmap_taskids on unknown taskmap returns EINVAL");
+
+    /* Add one task to taskmap so it is no longer unknown */
+    ok (taskmap_append (map, 0, 1, 1) == 0,
+        "add one task to taskmap so it is no longer unknown");
+
+    ok (taskmap_encode (NULL, 0) == NULL && errno == EINVAL,
+        "taskmap_encode (NULL) returns EINVAL");
+    ok (taskmap_encode (map, 0xff) == NULL && errno == EINVAL,
+        "taskmap_encode (map, 0xff) returns EINVAL");
+    ok (taskmap_encode (map, TASKMAP_ENCODE_RAW | TASKMAP_ENCODE_PMI) == NULL
+        && errno == EINVAL,
+        "taskmap_encode (map, MULTIPLE_ENCODINGS) returns EINVAL");
+
+    ok (taskmap_taskids (map, -1) == NULL && errno == EINVAL,
+        "taskmap_taskids (map, -1) returns EINVAL");
+    ok (taskmap_taskids (map, 1) == NULL && errno == ENOENT,
+        "taskmap_taskids (map, 1) returns ENOENT");
+
+    ok (taskmap_nodeid (NULL, 0) < 0 && errno == EINVAL,
+        "Ttaskmap_nodeid (NULL, 0) returns EINVAL");
+    ok (taskmap_nodeid (map, -1) < 0 && errno == EINVAL,
+        "Ttaskmap_nodeid (map, -1) returns EINVAL");
+
+    ok (taskmap_ntasks (NULL, 0) < 0 && errno == EINVAL,
+        "taskmap_ntasks (NULL) returns EINVAL");
+    ok (taskmap_ntasks (map, -1) < 0 && errno == EINVAL,
+        "taskmap_ntasks (map, -1) returns EINVAL");
+    ok (taskmap_ntasks (map, 1) < 0 && errno == ENOENT,
+        "taskmap_ntasks (map, 1) returns ENOENT");
+
+    ok (taskmap_nnodes (NULL) < 0 && errno == EINVAL,
+        "taskmap_nnodes (NULL) returns EINVAL");
+    ok (taskmap_total_ntasks (NULL) < 0 && errno == EINVAL,
+        "taskmap_total_ntasks (NULL) returns EINVAL");
+
+    ok (taskmap_decode (NULL, &error) == NULL,
+        "taskmap_decode (NULL) fails");
+    is (error.text, "Invalid argument",
+        "taskmap_decode (NULL) sets error.text=%s",
+        error.text);
+
+    ok (taskmap_decode_json (NULL, &error) == NULL,
+        "taskmap_decode_json (NULL) fails");
+    is (error.text, "Invalid argument",
+        "taskmape_decode_json (NULL) sets error.text=%s",
+        error.text);
+
+    /* Do not try to match jansson errors exactly */
+    for (const char **input = &invalid[0]; *input != NULL; input++) {
+        ok (taskmap_decode (*input, &error) == NULL,
+            "taskmap_decode (%s) fails with %s",
+            *input,
+            error.text);
+    }
+
+    ok (taskmap_append (NULL, 0, 0, 0) < 0 && errno == EINVAL,
+        "taskmap_append (NULL) returns EINVAL");
+    ok (taskmap_append (map, 0, 0, 0) < 0 && errno == EINVAL,
+        "taskmap_append (NULL, 0, 0, 0) returns EINVAL");
+
+    taskmap_destroy (map);
+}
+
+void append_tests ()
+{
+    int n;
+    char *s;
+    struct taskmap *map = taskmap_create ();
+    if (!map)
+        BAIL_OUT ("taskmap_create");
+
+    for (int i = 0; i < 4; i++) {
+        ok (taskmap_append (map, i, 1, 4) == 0,
+            "taskmap_append (%d, 1, 4)",
+            i);
+    }
+    n = taskmap_nnodes (map);
+    ok (n == 4,
+        "taskmap_nnodes() == 4 (got %d)",
+        n);
+    n = taskmap_total_ntasks (map);
+    ok (n == 16,
+        "taskmap_nnodes() == 16 (got %d)",
+        n);
+    s = taskmap_encode (map, 0);
+    ok (s != NULL,
+        "taskmap_encode works");
+    is (s, "[[0,4,4,1]]",
+        "map = %s",
+        s);
+    free (s);
+
+    /* Add another couple nodes with higher tasks-per-node count */
+    for (int i = 4; i < 6; i++) {
+        ok (taskmap_append (map, i, 1, 8) == 0,
+            "taskmap_append (%d, 1, 8)",
+            i);
+    }
+
+    n = taskmap_nnodes (map);
+    ok (n == 6,
+        "taskmap_nnodes() == 6 (got %d)",
+        n);
+    n = taskmap_total_ntasks (map);
+    ok (n == 32,
+        "taskmap_nnodes() == 32 (got %d)",
+        n);
+    s = taskmap_encode (map, 0);
+    ok (s != NULL,
+        "taskmap_encode works");
+    is (s, "[[0,4,4,1],[4,2,8,1]]",
+        "map = %s",
+        s);
+    free (s);
+
+    /* Add one more block of nodes that matches previous block */
+    ok (taskmap_append (map, 4, 2, 8) == 0,
+        "taskmap_append (4, 2, 8)");
+    s = taskmap_encode (map, 0);
+    ok (s != NULL,
+        "taskmap_encode works");
+    is (s, "[[0,4,4,1],[4,2,8,2]]",
+        "map = %s",
+        s);
+
+    free (s);
+    taskmap_destroy (map);
+}
+
+void append_cyclic_test ()
+{
+    int n;
+    char *s;
+    struct taskmap *map = taskmap_create ();
+    if (!map)
+        BAIL_OUT ("taskmap_create");
+
+    for (int i = 0; i < 4; i++) {
+        for (int j = 0; j < 4; j++) {
+            ok (taskmap_append (map, j, 1, 1) == 0,
+                "taskmap_append (%d, 1, 1)", j);
+        }
+    }
+    n = taskmap_nnodes (map);
+    ok (n == 4,
+        "taskmap_nnodes() == 4 (got %d)",
+        n);
+    n = taskmap_total_ntasks (map);
+    ok (n == 16,
+        "taskmap_nnodes() == 16 (got %d)",
+        n);
+    s = taskmap_encode (map, 0);
+    ok (s != NULL,
+        "taskmap_encode works");
+    is (s, "[[0,4,1,4]]",
+        "map = %s",
+        s);
+    free (s);
+    taskmap_destroy (map);
+}
+
+void append_cyclic_one ()
+{
+    int n;
+    char *s;
+    struct taskmap *map = taskmap_create ();
+    if (!map)
+        BAIL_OUT ("taskmap_create");
+
+    for (int i = 0; i < 4; i++) {
+        ok (taskmap_append (map, 0, 1, 1) == 0,
+            "taskmap_append (0, 1, 1)");
+    }
+    n = taskmap_nnodes (map);
+    ok (n == 1,
+        "taskmap_nnodes() == 1 (got %d)",
+        n);
+    n = taskmap_total_ntasks (map);
+    ok (n == 4,
+        "taskmap_nnodes() == 16 (got %d)",
+        n);
+    s = taskmap_encode (map, 0);
+    ok (s != NULL,
+        "taskmap_encode works");
+    is (s, "[[0,1,4,1]]",
+        "map = %s",
+        s);
+    free (s);
+    taskmap_destroy (map);
+
+}
+
+struct check_test {
+    const char *a;
+    const char *b;
+    int rc;
+    const char *errmsg;
+};
+
+struct check_test check_tests[] = {
+    { "[[0,4,4,1]]", "[[0,4,1,4]]", 0, NULL },
+    { "[[0,4,4,1]]", "[[0,4,2,2]]", 0, NULL },
+    { "[[0,4,4,1]]", "[[0,4,3,1],[0,4,1,1]]", 0, NULL },
+    { "[[0,4,4,1]]", "[[0,4,3,1]]", -1,
+      "got 12 total tasks, expected 16" },
+    { "[[0,4,4,1]]", "[[0,2,8,1]]", -1,
+      "got 2 nodes, expected 4" },
+    { "[[0,4,4,1]]", "[[0,2,4,1],[2,1,3,1],[3,1,5,1]]", -1,
+      "node 2 has 3 tasks, expected 4" },
+    { NULL, NULL, 0, NULL },
+};
+
+static void test_check ()
+{
+    for (struct check_test *t = &check_tests[0]; t->a != NULL; t++) {
+        flux_error_t error;
+        struct taskmap *a = taskmap_decode (t->a, &error);
+        struct taskmap *b = taskmap_decode (t->b, &error);
+        if (!a || !b)
+            BAIL_OUT ("taskmap_decode failed: %s", error.text);
+        ok (taskmap_check (a, b, &error) == t->rc,
+            "taskmap_check ('%s','%s') == %d",
+            t->a,
+            t->b,
+            t->rc);
+        if (t->errmsg)
+            is (error.text, t->errmsg,
+                "got expected error message: %s", error.text);
+        taskmap_destroy (a);
+        taskmap_destroy (b);
+    }
+}
+
+void test_deranged (void)
+{
+    struct taskmap *map;
+    flux_error_t error;
+    char *s;
+
+    if (!(map = taskmap_decode ("[[0,4,4,1]]", &error)))
+        BAIL_OUT ("taskmap_decode: %s", error.text);
+    ok ((s = taskmap_encode (map, TASKMAP_ENCODE_RAW_DERANGED)) != NULL,
+        "taskmap_encode RAW_DERANGED works");
+    is (s, "0,1,2,3;4,5,6,7;8,9,10,11;12,13,14,15",
+        "and result is deranged");
+    free (s);
+    taskmap_destroy (map);
+}
+
+struct test_vector raw_tests[] = {
+    { "-1",        "error parsing range '-1'" },
+    { "1-3;a-b",   "error parsing range 'a-b'" },
+    { "1,1",       "range '1' is out of order" },
+    { "0-1;1-2",   "duplicate taskid specified: 1" },
+    { "5-15;0-10", "duplicate taskids specified: 5-10" },
+    { "1",         "missing taskid: 0" },
+    { "3-4;0-1",   "missing taskid: 2" },
+    { "0-1;10-11", "missing taskids: 2-9" },
+    { NULL, NULL },
+};
+
+static void test_raw_decode_errors (void)
+{
+    struct test_vector *t;
+    for (t = &raw_tests[0]; t->taskmap != NULL; t++) {
+        flux_error_t error;
+        ok (taskmap_decode (t->taskmap, &error) == NULL,
+            "taskmap_decode (%s) fails",
+            t->taskmap);
+        is (error.text, t->expected,
+            "taskmap_decode: %s",
+            error.text);
+    }
+}
+
+int main (int ac, char **av)
+{
+    plan (NO_PLAN);
+    main_tests ();
+    rfc34_tests ();
+    pmi_mapping_tests ();
+    error_tests ();
+    append_tests ();
+    append_cyclic_test ();
+    append_cyclic_one ();
+    test_check ();
+    test_deranged ();
+    test_raw_decode_errors ();
+    done_testing ();
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libterminus/Makefile.am b/src/common/libterminus/Makefile.am
new file mode 100644
index 000000000000..44fbd3797d02
--- /dev/null
+++ b/src/common/libterminus/Makefile.am
@@ -0,0 +1,60 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux \
+	$(JANSSON_CFLAGS) \
+	$(FLUX_SECURITY_CFLAGS)
+
+noinst_LTLIBRARIES = libterminus.la
+
+libterminus_la_SOURCES = \
+	pty.c \
+	pty.h \
+	client.c \
+	terminus.h \
+	terminus.c
+
+TESTS = \
+	test_pty.t \
+	test_terminus.t
+
+check_PROGRAMS = \
+	$(TESTS)
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+        $(top_srcdir)/config/tap-driver.sh
+
+test_ldadd = \
+	$(top_builddir)/src/common/libterminus/libterminus.la \
+	$(top_builddir)/src/common/libutil/libutil.la \
+	$(top_builddir)/src/common/libtestutil/libtestutil.la \
+        $(top_builddir)/src/common/libflux-core.la \
+        $(top_builddir)/src/common/libflux-internal.la \
+        $(top_builddir)/src/common/libtap/libtap.la
+
+test_ldflags = \
+	-no-install
+
+test_cppflags = \
+	$(AM_CPPFLAGS) \
+	-I$(top_srcdir)/src/common/libtap
+
+test_pty_t_SOURCES = test/pty.c
+test_pty_t_CPPFLAGS = $(test_cppflags)
+test_pty_t_LDADD = $(test_ldadd)
+test_pty_t_LDFLAGS = $(test_ldflags)
+
+test_terminus_t_SOURCES = test/terminus.c
+test_terminus_t_CPPFLAGS = $(test_cppflags)
+test_terminus_t_LDADD = $(test_ldadd)
+test_terminus_t_LDFLAGS = $(test_ldflags)
diff --git a/src/common/libterminus/client.c b/src/common/libterminus/client.c
new file mode 100644
index 000000000000..9dcc3e94f9c5
--- /dev/null
+++ b/src/common/libterminus/client.c
@@ -0,0 +1,581 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <unistd.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <signal.h>
+#include <termios.h>
+
+#include <jansson.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/fdutils.h"
+#include "src/common/libutil/llog.h"
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
+
+#include "pty.h"
+
+static struct termios orig_term;
+static bool term_needs_restoration = false;
+
+struct exit_waiter {
+    flux_pty_client_exit_f cb;
+    void *arg;
+};
+
+struct flux_pty_client {
+    flux_t *h;
+
+    pty_log_f llog;
+    void *llog_data;
+
+    int flags;
+    int rank;
+    char *service;
+    bool attached;
+
+    flux_watcher_t *fdw;  /* fd watcher for STDIN */
+    flux_watcher_t *sw;   /* signal watcher       */
+    flux_watcher_t *kaw;  /* keepalive timer      */
+
+    flux_future_t *rpc_f; /* streaming RPC future */
+
+    struct termios term;
+
+    zlist_t *exit_waiters;
+    int wait_status;
+    char *exit_message;
+};
+
+static int get_winsize (struct winsize *ws)
+{
+    return ioctl (STDIN_FILENO, TIOCGWINSZ, ws);
+}
+
+void flux_pty_client_destroy (struct flux_pty_client *c)
+{
+    if (c) {
+        int saved_errno = errno;
+        flux_watcher_destroy (c->fdw);
+        flux_watcher_destroy (c->sw);
+        flux_watcher_destroy (c->kaw);
+        flux_future_destroy (c->rpc_f);
+        zlist_destroy (&c->exit_waiters);
+        free (c->exit_message);
+        free (c->service);
+        free (c);
+        errno = saved_errno;
+    }
+}
+
+int flux_pty_client_exit_status (struct flux_pty_client *c,
+                                 int *statusp)
+{
+    if (!c || !statusp) {
+        errno = EINVAL;
+        return -1;
+    }
+    *statusp = c->wait_status;
+    return 0;
+}
+
+struct flux_pty_client *flux_pty_client_create (void)
+{
+    struct flux_pty_client *c = calloc (1, sizeof (*c));
+    if (!c)
+        return NULL;
+    c->exit_waiters = zlist_new ();
+    if (!c->exit_waiters) {
+        flux_pty_client_destroy (c);
+        return NULL;
+    }
+    return c;
+}
+
+static void notify_exit (struct flux_pty_client *c)
+{
+    struct exit_waiter *w;
+    while ((w = zlist_pop (c->exit_waiters))) {
+        (*w->cb) (c, w->arg);
+        free (w);
+    }
+}
+
+int flux_pty_client_notify_exit (struct flux_pty_client *c,
+                                 flux_pty_client_exit_f fn,
+                                 void *arg)
+{
+    struct exit_waiter *w;
+
+    if (!c || !fn) {
+        errno = EINVAL;
+        return -1;
+    }
+    w = calloc (1, sizeof (*w));
+    if (!w)
+        return -1;
+    w->cb = fn;
+    w->arg = arg;
+    if (zlist_append (c->exit_waiters, w) < 0) {
+        free (w);
+        errno = ENOMEM;
+        return -1;
+    }
+    zlist_freefn (c->exit_waiters, w, (zlist_free_fn *) free, true);
+    return 0;
+}
+
+static int invalid_flags (int flags)
+{
+    const int valid_flags =
+        FLUX_PTY_CLIENT_ATTACH_SYNC
+        | FLUX_PTY_CLIENT_CLEAR_SCREEN
+        | FLUX_PTY_CLIENT_NOTIFY_ON_ATTACH
+        | FLUX_PTY_CLIENT_NOTIFY_ON_DETACH
+        | FLUX_PTY_CLIENT_NORAW
+        | FLUX_PTY_CLIENT_STDIN_PIPE;
+    return (flags & ~valid_flags);
+}
+
+int flux_pty_client_set_flags (struct flux_pty_client *c, int flags)
+{
+    if (!c || invalid_flags (flags)) {
+        errno = EINVAL;
+        return -1;
+    }
+    c->flags = flags;
+    return 0;
+}
+
+int flux_pty_client_get_flags (struct flux_pty_client *c)
+{
+    if (!c) {
+        errno = EINVAL;
+        return -1;
+    }
+    return c->flags;
+}
+
+void flux_pty_client_set_log (struct flux_pty_client *c,
+                              pty_log_f log,
+                              void *log_data)
+{
+    if (c) {
+        c->llog = log;
+        c->llog_data = log_data;
+    }
+}
+
+static void flux_pty_client_stop (struct flux_pty_client *c)
+{
+    flux_watcher_stop (c->fdw);
+    flux_watcher_stop (c->sw);
+    flux_watcher_stop (c->kaw);
+}
+
+static int flux_pty_client_set_server (struct flux_pty_client *c,
+                                       int rank,
+                                       const char *service)
+{
+    if (!(c->service = strdup (service)))
+        return -1;
+    c->rank = rank;
+    return 0;
+}
+
+static void cls (void)
+{
+    /* ANSI clear screen + Home */
+    printf ("\033[2J\033[;H");
+    fflush (stdout);
+}
+
+void flux_pty_client_restore_terminal (void)
+{
+    if (term_needs_restoration) {
+        tcsetattr (STDIN_FILENO, TCSADRAIN, &orig_term);
+        /* https://en.wikipedia.org/wiki/ANSI_escape_code
+         * Best effort: attempt to ensure cursor is visible:
+         */
+        printf ("\033[?25h");
+        fflush (stdout);
+        term_needs_restoration = false;
+    }
+}
+
+static int setup_terminal (struct flux_pty_client *c)
+{
+    if (tcgetattr (STDIN_FILENO, &orig_term) < 0)
+        return -1;
+
+    c->term = orig_term;
+
+    cfmakeraw (&c->term);
+    c->term.c_cc[VLNEXT] = _POSIX_VDISABLE;
+    c->term.c_cc[VMIN] = 1;
+    c->term.c_cc[VTIME] = 0;
+    if (tcsetattr (STDIN_FILENO, TCSANOW, &c->term) < 0) {
+        llog_warning (c, "failed to setup terminal\n");
+        return -1;
+    }
+    term_needs_restoration = true;
+    atexit (flux_pty_client_restore_terminal);
+    return 0;
+}
+
+static void pty_client_attached (struct flux_pty_client *c)
+{
+    /*  Setup terminal, start watching stdin for data */
+    if (!(c->flags & FLUX_PTY_CLIENT_NORAW))
+        (void) setup_terminal (c);
+    if (c->flags & FLUX_PTY_CLIENT_CLEAR_SCREEN)
+        cls ();
+    if (c->flags & FLUX_PTY_CLIENT_NOTIFY_ON_ATTACH) {
+        printf ("[attached]\r\n");
+    }
+    flux_watcher_start (c->fdw);
+    flux_watcher_start (c->kaw);
+    if (!(c->flags & FLUX_PTY_CLIENT_STDIN_PIPE))
+        flux_watcher_start (c->sw);
+    c->attached = true;
+}
+
+static void pty_client_data (struct flux_pty_client *c, flux_future_t *f)
+{
+    const flux_msg_t *msg;
+    char *data;
+    size_t len;
+    flux_error_t error;
+
+    if (flux_future_get (f, (const void **)&msg) < 0) {
+        llog_error (c, "data response: %s", future_strerror (f, errno));
+        return;
+    }
+    if (pty_data_unpack (msg, &error, &data, &len) < 0) {
+        llog_error (c, "unpack: %s", error.text);
+        return;
+    }
+    if (write (STDOUT_FILENO, data, len) < 0) {
+        llog_error (c, "write %zu bytes: %s", len, strerror (errno));
+        return;
+    }
+}
+
+static void client_resize_cb (flux_future_t *f, void *arg)
+{
+    if (flux_future_get (f, NULL) < 0) {
+        struct flux_pty_client *c = flux_future_aux_get (f, "pty_client");
+        if (c)
+            llog_error (c, "resize: %s", future_strerror (f, errno));
+    }
+    flux_future_destroy (f);
+}
+
+/*  Server requested winsize
+ */
+static void pty_client_resize (struct flux_pty_client *c)
+{
+    flux_future_t *f;
+    struct winsize ws;
+    if (get_winsize (&ws) < 0) {
+        llog_error (c, "get winsize failed: %s", strerror (errno));
+        return;
+    }
+    if (!(f = flux_rpc_pack (c->h, c->service, c->rank, 0,
+                             "{s:s s:{s:i s:i}}",
+                             "type", "resize",
+                             "winsize",
+                               "rows", ws.ws_row,
+                               "cols", ws.ws_col))) {
+        llog_error (c, "flux_rpc_pack type=resize: %s", flux_strerror (errno));
+        return;
+    }
+    (void) flux_future_aux_set (f, "pty_client", c, NULL);
+    if (flux_future_then (f, -1., client_resize_cb, c) < 0)
+        llog_error (c, "flux_future_then: %s", flux_strerror (errno));
+}
+
+static void pty_die (struct flux_pty_client *c, int code, const char *message)
+{
+    flux_pty_client_stop (c);
+    /*  Only overwrite c->wait_status for code > 0. O/w, we collect the
+     *   actual task exit status
+     */
+    if (code)
+        c->wait_status = code << 8;
+    if (message) {
+        free (c->exit_message);
+        c->exit_message = strdup (message);
+    }
+    if (c->attached && (c->flags & FLUX_PTY_CLIENT_NOTIFY_ON_DETACH)) {
+        printf ("\033[999H[detached: %s]\033[K\n\r",
+                c->exit_message ? c->exit_message : "unknown reason");
+        fflush (stdout);
+    }
+    notify_exit (c);
+}
+
+static void pty_client_exit (struct flux_pty_client *c, flux_future_t *f)
+{
+    const char *message;
+
+    if (flux_rpc_get_unpack (f, "{s:s s:i}",
+                                "message", &message,
+                                "status", &c->wait_status) < 0) {
+        llog_error (c, "rpc unpack: %s", future_strerror (f, errno));
+        message="unknown reason";
+    }
+    free (c->exit_message);
+    c->exit_message = strdup (message);
+    flux_pty_client_stop (c);
+}
+
+static void pty_server_cb (flux_future_t *f, void *arg)
+{
+    struct flux_pty_client *c = arg;
+    const char *type;
+    if (flux_rpc_get_unpack (f, "{s:s}", "type", &type) < 0) {
+        const char *message = NULL;
+        int code = 1;
+        if (errno == ENOSYS)
+            message = "No such session";
+        else if (errno != ENODATA)
+            message = future_strerror (f, errno);
+        else
+            code = 0;
+        pty_die (c, code, message);
+        flux_future_destroy (c->rpc_f);
+        c->rpc_f = NULL;
+        return;
+    }
+    if (streq (type, "attach"))
+        pty_client_attached (c);
+    else if (streq (type, "data"))
+        pty_client_data (c, f);
+    else if (streq (type, "resize"))
+        pty_client_resize (c);
+    else if (streq (type, "exit"))
+        pty_client_exit (c, f);
+    else {
+        llog_error (c, "unknown server response type=%s", type);
+        pty_die (c, 1, "Protocol error");
+        flux_future_destroy (c->rpc_f);
+        c->rpc_f = NULL;
+        return;
+    }
+    flux_future_reset (f);
+}
+
+int flux_pty_client_detach (struct flux_pty_client *c)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_rpc_pack (c->h, c->service, c->rank, 0,
+                             "{s:s}", "type", "detach"))) {
+        llog_error (c, "flux_rpc_pack: %s", flux_strerror (errno));
+        return -1;
+    }
+    flux_future_destroy (f);
+    return 0;
+}
+
+static void data_write_cb (flux_future_t *f, void *arg)
+{
+    if (flux_future_get (f, NULL) < 0) {
+        struct flux_pty_client *c = flux_future_aux_get (f, "pty_client");
+        if (c) {
+            flux_pty_client_detach (c);
+            if (errno == ENOSYS)
+                pty_die (c, 1, "remote pty disappeared");
+            else
+                pty_die (c, 1, "error writing to remote pty");
+        }
+    }
+    flux_future_destroy (f);
+}
+
+flux_future_t *flux_pty_client_write (struct flux_pty_client *c,
+                                      const void *buf,
+                                      ssize_t len)
+{
+    flux_future_t *f;
+    json_t *o = NULL;
+
+    if (!c || !buf || len < 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(o = pty_data_encode (buf, len)) )
+        return NULL;
+    f = flux_rpc_pack (c->h, c->service, c->rank, 0, "O", o);
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return f;
+}
+
+static void pty_read_cb (flux_reactor_t *r,
+                         flux_watcher_t *w,
+                         int revents,
+                         void *arg)
+{
+    struct flux_pty_client *c = arg;
+    flux_future_t *f;
+    char buf[4096];
+    ssize_t len;
+
+    if ((len = read (STDIN_FILENO, buf, sizeof (buf))) < 0) {
+        if (errno != EAGAIN && errno != EINTR) {
+            llog_fatal (c, "read: %s", strerror (errno));
+            pty_client_exit (c, NULL);
+        }
+        return;
+    }
+    if (len == 0) {
+        flux_pty_client_stop (c);
+        if (c->flags & FLUX_PTY_CLIENT_STDIN_PIPE)
+            flux_pty_client_detach (c);
+        return;
+    }
+    if (buf[0] == '') { /* request detach */
+        (void ) flux_pty_client_detach (c);
+        return;
+    }
+    if (!(f = flux_pty_client_write (c, buf, len))) {
+        llog_error (c, "flux_pty_client_write: %s", strerror (errno));
+        return;
+    }
+    (void) flux_future_aux_set (f, "pty_client", c, NULL);
+    if (flux_future_then (f, -1., data_write_cb, c) < 0) {
+        llog_error (c, "flux_future_then: %s", strerror (errno));
+        flux_future_destroy (f);
+    }
+}
+
+static void sigwinch_cb (flux_reactor_t *r,
+                         flux_watcher_t *w,
+                         int revents,
+                         void *arg)
+{
+    pty_client_resize ((struct flux_pty_client *)arg);
+}
+
+static void keepalive_cb (flux_reactor_t *r,
+                          flux_watcher_t *w,
+                          int revents,
+                          void *arg)
+{
+    struct flux_pty_client *c = arg;
+    flux_future_t *f;
+    char *buf = "";
+
+    if (!(f = flux_pty_client_write (c, buf, 0))) {
+        llog_error (c, "flux_pty_client_write: %s", strerror (errno));
+        return;
+    }
+    (void) flux_future_aux_set (f, "pty_client", c, NULL);
+    if (flux_future_then (f, -1., data_write_cb, c) < 0) {
+        llog_error (c, "flux_future_then: %s", strerror (errno));
+        flux_future_destroy (f);
+    }
+}
+
+bool flux_pty_client_attached (struct flux_pty_client *c)
+{
+    return c && c->attached;
+}
+
+int flux_pty_client_attach (struct flux_pty_client *c,
+                            flux_t *h,
+                            int rank,
+                            const char *service)
+{
+    flux_future_t *f;
+    struct winsize ws;
+    const char *mode;
+
+    if (!c || !h || !service) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (isatty (STDIN_FILENO) && get_winsize (&ws) < 0)
+        return -1;
+
+    /*  In some test conditions, get_winsize returns 0 for
+     *   rows or columns. Fix that here. It shouldn't be an
+     *   issue with a real tty.
+     */
+    if (ws.ws_row <= 0)
+        ws.ws_row = 1;
+    if (ws.ws_col <= 0)
+        ws.ws_col = 1;
+
+    if (flux_pty_client_set_server (c, rank, service) < 0)
+        return -1;
+
+    c->h = h;
+
+    c->fdw = flux_fd_watcher_create (flux_get_reactor (h),
+                                     STDIN_FILENO,
+                                     FLUX_POLLIN,
+                                     pty_read_cb,
+                                     c);
+    c->sw = flux_signal_watcher_create (flux_get_reactor (h),
+                                        SIGWINCH,
+                                        sigwinch_cb,
+                                        c);
+    c->kaw = flux_timer_watcher_create (flux_get_reactor (h),
+                                        1., 1.,
+                                        keepalive_cb,
+                                        c);
+    if (!c->fdw || !c->sw || !c->kaw)
+        return -1;
+
+    mode = c->flags & FLUX_PTY_CLIENT_STDIN_PIPE ? "wo" : "rw";
+
+    if (!(f = flux_rpc_pack (h, service, rank, FLUX_RPC_STREAMING,
+                             "{s:s s:s s:{s:i s:i}}",
+                             "type", "attach",
+                             "mode", mode,
+                             "winsize",
+                              "rows", ws.ws_row,
+                              "cols", ws.ws_col))) {
+            llog_error (c, "flux_rpc_pack: %s", flux_strerror (errno));
+            return -1;
+    }
+    if (c->flags & FLUX_PTY_CLIENT_ATTACH_SYNC) {
+        const char *type;
+        if (flux_rpc_get_unpack (f, "{s:s}", "type", &type) < 0
+            || !streq (type, "attach")) {
+            flux_future_destroy (f);
+            return -1;
+        }
+        pty_client_attached (c);
+        flux_future_reset (f);
+    }
+    if (flux_future_then (f, -1, pty_server_cb, c) < 0) {
+        flux_future_destroy (f);
+        return -1;
+    }
+    c->rpc_f = f;
+    return 0;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
+
diff --git a/src/common/libterminus/pty.c b/src/common/libterminus/pty.c
new file mode 100644
index 000000000000..dd8c4e5dad8f
--- /dev/null
+++ b/src/common/libterminus/pty.c
@@ -0,0 +1,847 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/*  pseudoterminal multiplexer like dtach, but using flux service
+ *   endpoint and messages rather than unix domain sockets.
+ *
+ *   Run and attach to a process anywhere in your Flux instance.
+ *
+ *  PROTOCOL:
+ *
+ *  Client attach to server:
+ *  { "type":"attach", "mode":s, "winsize":{"rows":i,"columns":i}}
+ *  where mode is one of "rw", "ro", or "rw"
+ *
+ *  Server response to attach:
+ *  { "type":"attach" }
+ *
+ *  Resize request: (client->server or server->client)
+ *  { "type":"resize", "winsize"?{"rows":i,"columns":i} }
+ *
+ *  Client/server write raw data to tty (string is utf-8)
+ *  { "type":"data", "data":s% }
+ *
+ *  Client detach:
+ *  { "type":"detach" }
+ *
+ *  Server tell client to exit (if process exited, include exit status):
+ *  { "type":"exit", "message":s, "status":i }
+ *
+ *  ENODATA: End of streaming RPC
+ *
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/types.h>
+#include <signal.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <termios.h>
+
+#include <jansson.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/fdutils.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/aux.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+#include "ccan/base64/base64.h"
+
+#define LLOG_SUBSYSTEM "pty"
+#include "src/common/libutil/llog.h"
+
+#include "pty.h"
+
+struct pty_client {
+    char *uuid;
+    const flux_msg_t *req;
+
+    bool write_enabled;
+    bool read_enabled;
+};
+
+struct flux_pty {
+    flux_t *h;
+
+    pty_log_f llog;
+    void *llog_data;
+
+    int leader;
+    char *follower;
+    flux_watcher_t *fdw;
+
+    bool wait_for_client;
+    bool wait_on_close;
+    bool exited;
+    int status;
+
+    zlist_t *clients;
+
+    pty_monitor_f monitor;
+    pty_complete_f complete;
+    struct aux_item *aux;
+};
+
+static void pty_client_destroy (struct pty_client *c)
+{
+    if (c) {
+        int saved_errno = errno;
+        flux_msg_decref (c->req);
+        free (c->uuid);
+        free (c);
+        errno = saved_errno;
+    }
+}
+
+static struct pty_client *pty_client_create (const flux_msg_t *msg)
+{
+    const char *uuid;
+    struct pty_client *c = NULL;
+
+    if (!(c = calloc (1, sizeof (*c))))
+        return NULL;
+    if (!(uuid = flux_msg_route_first (msg))) {
+        errno = EPROTO;
+        goto error;
+    }
+    c->req = flux_msg_incref (msg);
+    if (!(c->uuid = strdup (uuid)))
+        goto error;
+    return c;
+error:
+    pty_client_destroy (c);
+    return NULL;
+}
+
+static struct pty_client *pty_client_find (struct flux_pty *pty,
+                                           const char *uuid)
+{
+    struct pty_client *c = zlist_first (pty->clients);
+    while (c) {
+        if (streq (c->uuid, uuid))
+            break;
+        c = zlist_next (pty->clients);
+    }
+    return c;
+}
+
+static struct pty_client *pty_client_find_sender (struct flux_pty *pty,
+                                                  const flux_msg_t *msg)
+{
+    struct pty_client *c = NULL;
+    const char *uuid;
+
+    if (!(uuid = flux_msg_route_first (msg))) {
+        llog_error (pty, "flux_msg_get_route_first: uuid is NULL!");
+        return NULL;
+    }
+    c = pty_client_find (pty, uuid);
+
+    return c;
+}
+
+static int pty_client_send_exit (struct flux_pty *pty,
+                                 struct pty_client *c,
+                                 const char *message,
+                                 int status)
+{
+    if (flux_respond_pack (pty->h, c->req,
+                              "{s:s s:s? s:i}",
+                              "type", "exit",
+                              "message", message,
+                              "status", status) < 0)
+        return -1;
+    /* End of stream */
+    return flux_respond_error (pty->h, c->req, ENODATA, NULL);
+}
+
+static int pty_clients_notify_exit (struct flux_pty *pty)
+{
+    struct pty_client *c = zlist_first (pty->clients);
+    while (c) {
+        if (pty_client_send_exit (pty, c, "session exiting", pty->status) < 0)
+            llog_error (pty, "send_exit: %s", flux_strerror (errno));
+        c = zlist_next (pty->clients);
+    }
+    return 0;
+}
+
+static int pty_client_detach (struct flux_pty *pty, struct pty_client *c)
+{
+    if (c) {
+        pty_client_send_exit (pty, c, "Client requested detach", 0);
+        zlist_remove (pty->clients, c);
+        pty_client_destroy (c);
+    }
+    /* XXX: Resize remaining clients? */
+    return 0;
+}
+
+int flux_pty_disconnect_client (struct flux_pty *pty, const char *sender)
+{
+    if (!pty || !sender) {
+        errno = EINVAL;
+        return -1;
+    }
+    return pty_client_detach (pty, pty_client_find (pty, sender));
+}
+
+int flux_pty_kill (struct flux_pty *pty, int sig)
+{
+    pid_t pgrp = -1;
+    if (!pty || sig <= 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    /*  Disable wait-on-client if being killed (except for terminal resize) */
+    if (sig != SIGWINCH) {
+        pty->wait_for_client = false;
+        pty->wait_on_close = false;
+    }
+#ifdef TIOCSIG
+    if (ioctl (pty->leader, TIOCSIG, sig) >= 0)
+        return 0;
+    llog_debug (pty, "ioctl (TIOCSIG): %s", strerror (errno));
+#endif
+    if (ioctl (pty->leader, TIOCGPGRP, &pgrp) >= 0
+        && pgrp > 0
+        && kill (-pgrp, sig) >= 0)
+        return 0;
+    llog_debug (pty, "ioctl (TIOCPGRP): %s", strerror (errno));
+    return -1;
+}
+
+static void pty_clients_destroy (struct flux_pty *pty)
+{
+    struct pty_client *c = zlist_first (pty->clients);
+    while (c) {
+        pty_client_destroy (c);
+        c = zlist_next (pty->clients);
+    }
+}
+
+void flux_pty_destroy (struct flux_pty *pty)
+{
+    if (pty) {
+        flux_watcher_destroy (pty->fdw);
+        pty_clients_notify_exit (pty);
+        pty_clients_destroy (pty);
+        zlist_destroy (&pty->clients);
+        if (pty->leader >= 0)
+            close (pty->leader);
+        free (pty->follower);
+        aux_destroy (&pty->aux);
+        free (pty);
+    }
+}
+
+static void check_pty_complete (struct flux_pty *pty)
+{
+    llog_debug (pty, "wait_for_client=%d wait_on_close=%d exited=%d",
+                pty->wait_for_client, pty->wait_on_close, pty->exited);
+    if (!pty->wait_for_client
+        && !pty->wait_on_close
+        && pty->exited)
+        (*pty->complete) (pty);
+}
+
+void flux_pty_exited (struct flux_pty *pty, int status)
+{
+    if (pty) {
+        pty->status = status;
+        pty->exited = true;
+        check_pty_complete (pty);
+    }
+}
+
+static struct flux_pty * flux_pty_create ()
+{
+    struct flux_pty *pty = calloc (1, sizeof (*pty));
+    if (!pty)
+        return NULL;
+    pty->leader = -1;
+    pty->clients = zlist_new ();
+    return pty;
+}
+
+int flux_pty_client_count (struct flux_pty *pty)
+{
+    if (pty) {
+        return zlist_size (pty->clients);
+    }
+    return 0;
+}
+
+void flux_pty_wait_for_client (struct flux_pty *pty)
+{
+    if (pty)
+        pty->wait_for_client = true;
+}
+
+void flux_pty_wait_on_close (struct flux_pty *pty)
+{
+    if (pty)
+        pty->wait_on_close = true;
+}
+
+struct flux_pty * flux_pty_open ()
+{
+    struct flux_pty *pty = flux_pty_create ();
+    const char *name;
+    struct winsize ws = { 0 };
+    if (!pty)
+        return NULL;
+    if ((pty->leader = posix_openpt (O_RDWR | O_NOCTTY |O_CLOEXEC)) < 0
+        || grantpt (pty->leader) < 0
+        || unlockpt (pty->leader) < 0
+        || !(name = ptsname (pty->leader))
+        || !(pty->follower = strdup (name)))
+        goto err;
+
+    /*  Set a default winsize, so it isn't 0,0 */
+    ws.ws_row = 25;
+    ws.ws_col = 80;
+    if (ioctl (pty->leader, TIOCSWINSZ, &ws) < 0)
+        goto err;
+
+    pty->complete = flux_pty_destroy;
+
+    return pty;
+err:
+    flux_pty_destroy (pty);
+    return NULL;
+}
+
+void flux_pty_set_log (struct flux_pty *pty,
+                       pty_log_f log,
+                       void *log_data)
+{
+    if (pty) {
+        pty->llog = log;
+        pty->llog_data = log_data;
+    }
+}
+
+void flux_pty_monitor (struct flux_pty *pty, pty_monitor_f fn)
+{
+    if (!pty)
+        return;
+
+    pty->monitor = fn;
+    /*  If a monitor function is provided, and there are currently no
+     *   other clients, ensure the pty fd_watcher is started.
+     */
+    if (fn != NULL
+        && !pty->wait_for_client
+        && zlist_size (pty->clients) == 0) {
+        flux_watcher_start (pty->fdw);
+    }
+}
+
+void flux_pty_set_complete_cb (struct flux_pty *pty, pty_complete_f fn)
+{
+    if (pty)
+        pty->complete = fn;
+}
+
+int flux_pty_leader_fd (struct flux_pty *pty)
+{
+    if (!pty) {
+        errno = EINVAL;
+        return -1;
+    }
+    return pty->leader;
+}
+
+const char *flux_pty_name (struct flux_pty *pty)
+{
+    if (!pty) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return pty->follower;
+}
+
+int flux_pty_attach (struct flux_pty *pty)
+{
+    int fd;
+    if (!pty || !pty->follower) {
+        errno = EINVAL;
+        return -1;
+    }
+    if ((fd = open (pty->follower, O_RDWR|O_NOCTTY)) < 0)
+        return -1;
+
+    /*  New session so we can attach this process to tty */
+    (void) setsid ();
+
+    /*  Make the follower pty (known in documentation as the slave
+     *  side of pty) our controlling terminal */
+    if (ioctl (fd, TIOCSCTTY, NULL) < 0)
+        return -1;
+
+    /*  dup pty/in/out onto tty fd */
+    if (dup2 (fd, STDIN_FILENO) != STDIN_FILENO
+        || dup2 (fd, STDOUT_FILENO) != STDOUT_FILENO
+        || dup2 (fd, STDERR_FILENO) != STDERR_FILENO) {
+        llog_error (pty, "dup2: %s", strerror (errno));
+        return -1;
+    }
+    if (fd > 2)
+        (void) close (fd);
+    if (pty->leader >= 0)
+        (void) close (pty->leader);
+    return 0;
+}
+
+static int encode_base64 (const char *data, int len, char **destp, int *lenp)
+{
+    ssize_t n;
+    char *dest = NULL;
+    size_t destlen = base64_encoded_length (len) + 1;
+
+    if (!(dest = malloc (destlen))
+        || ((n = base64_encode (dest, destlen, data, len)) < 0))
+        return -1;
+    *destp = dest;
+    *lenp = n;
+    return 0;
+}
+
+static json_t *pty_data_encode_base64 (const void *data, int len)
+{
+    json_t *o = NULL;
+    char *b64data = NULL;
+    int b64len = 0;
+
+    if (encode_base64 (data, len, &b64data, &b64len) < 0)
+        return NULL;
+
+    o = json_pack ("{s:s s:s s:s#}",
+                   "type", "data",
+                   "encoding", "base64",
+                   "data", b64data, b64len);
+
+    ERRNO_SAFE_WRAP (free, b64data);
+    return o;
+}
+
+json_t *pty_data_encode (const void *data, int len)
+{
+    json_t *o;
+
+    if (!(o = json_pack ("{s:s s:s#}",
+                         "type", "data",
+                         "data", (char *) data, len))) {
+        /*
+         *  json_pack() may fail if there are bytes that cannot be encoded
+         *  as UTF-8, e.g. U+0000 is specifically not allowed in jansson.
+         *  Try encoding as base64 instead.
+         */
+        o = pty_data_encode_base64 (data, len);
+    }
+    return o;
+}
+
+void pty_client_send_data (struct flux_pty *pty, void *data, int len)
+{
+    struct pty_client *c = zlist_first (pty->clients);
+
+    if (pty->monitor)
+        (*pty->monitor) (pty, data, len);
+
+    while (c) {
+        if (c->read_enabled) {
+            json_t *o = pty_data_encode (data, len);
+            if (!o || flux_respond_pack (pty->h, c->req, "O", o) < 0)
+                llog_error (pty, "send data: %s", strerror (errno));
+            json_decref (o);
+        }
+        c = zlist_next (pty->clients);
+    }
+}
+
+void pty_client_monitor_send_eof (struct flux_pty *pty)
+{
+    if (pty->monitor)
+        (*pty->monitor) (pty, NULL, 0);
+}
+
+static void pty_read (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct flux_pty *pty = arg;
+    ssize_t n;
+    char buf [4096];
+
+    /* XXX: notify all clients and exit */
+    if (revents & FLUX_POLLERR)
+        return;
+
+    n = read (pty->leader, buf, sizeof (buf));
+    if (n < 0) {
+        if (errno == EAGAIN || errno == EINTR)
+            return;
+        /*
+         *  pty: EIO indicates pty follower has closed.
+         *   stop the fd watcher, disable wait_on_close and
+         *   check to see if the pty has met all conditions
+         *   for completion:
+         */
+        if (errno == EIO) {
+            flux_watcher_stop (pty->fdw);
+            pty->wait_on_close = false;
+            check_pty_complete (pty);
+            pty_client_monitor_send_eof (pty);
+            return;
+        }
+        llog_error (pty, "read: %s", strerror (errno));
+        return;
+    }
+    else if (n > 0)
+        pty_client_send_data (pty, buf, n);
+}
+
+static int pty_resize (struct flux_pty *pty, const flux_msg_t *msg)
+{
+    struct winsize ws = { 0 };
+
+    if (flux_msg_unpack (msg,
+                         "{s:{s:i s:i}}",
+                         "winsize",
+                         "rows", &ws.ws_row,
+                         "cols", &ws.ws_col) < 0) {
+        llog_error (pty, "msg_unpack failed: %s", strerror (errno));
+        return -1;
+    }
+    llog_debug (pty, "resize: %dx%d", ws.ws_row, ws.ws_col);
+    if (ws.ws_row <= 0 || ws.ws_col <= 0) {
+        errno = EINVAL;
+        llog_error (pty, "bad resize: row=%d, col=%d", ws.ws_row, ws.ws_col);
+        return -1;
+    }
+    if (ioctl (pty->leader, TIOCSWINSZ, &ws) < 0) {
+        llog_error (pty, "ioctl: TIOCSWINSZ: %s", strerror (errno));
+        return -1;
+    }
+
+    /*  notify foreground process to redraw (best effort) */
+    (void) flux_pty_kill (pty, SIGWINCH);
+
+    return 0;
+}
+
+int flux_pty_add_exit_watcher (struct flux_pty *pty, const flux_msg_t *msg)
+{
+    struct pty_client *c = pty_client_create (msg);
+    if (!c)
+        goto err;
+    if (zlist_append (pty->clients, c) < 0) {
+        errno = ENOMEM;
+        goto err;
+    }
+    return 0;
+err:
+    pty_client_destroy (c);
+    return -1;
+}
+
+static int pty_client_set_mode (struct flux_pty *pty,
+                                struct pty_client *c,
+                                const flux_msg_t *msg)
+{
+    const char *mode;
+    if (flux_msg_unpack (msg, "{s:s}", "mode", &mode) < 0)
+        return -1;
+    /*  Valid modes are currently only "ro", "wo", "rw" */
+    if (streq (mode, "rw"))
+        c->read_enabled = c->write_enabled = true;
+    else if (streq (mode, "wo"))
+        c->write_enabled = true;
+    else if (streq (mode, "ro"))
+        c->read_enabled = true;
+    else {
+        llog_error (pty, "client=%s: invalid mode: %s", c->uuid, mode);
+        errno = EPROTO;
+        return -1;
+    }
+    return 0;
+}
+
+static int pty_attach (struct flux_pty *pty, const flux_msg_t *msg)
+{
+    int saved_errno;
+    struct pty_client *c = pty_client_create (msg);
+
+    if (!c)
+        goto err;
+    if (pty_client_set_mode (pty, c, msg) < 0)
+        goto err;
+
+    /*  Only start watching tty fd when first client attaches */
+    if (zlist_size (pty->clients) == 0 && c->read_enabled) {
+        flux_watcher_start (pty->fdw);
+        /* Done waiting for first client */
+        pty->wait_for_client = false;
+    }
+    if (zlist_append (pty->clients, c) < 0) {
+        errno = ENOMEM;
+        goto err;
+    }
+    if (c->read_enabled
+        && c->write_enabled
+        && pty_resize (pty, msg) < 0)
+        goto err;
+    if (flux_respond_pack (pty->h, msg, "{s:s}", "type", "attach") < 0)
+        goto err;
+    return 0;
+err:
+    saved_errno = errno;
+    if (c)
+        zlist_remove (pty->clients, c);
+    pty_client_destroy (c);
+    errno = saved_errno;
+    return -1;
+}
+
+static int decode_base64 (char *src,
+                          size_t srclen,
+                          char **datap,
+                          size_t *lenp)
+{
+    ssize_t rc;
+    char *data;
+    size_t size = base64_decoded_length (srclen) + 1;
+
+    if (!(data = malloc (size)))
+        return -1;
+    if ((rc = base64_decode (data, size, src, srclen)) < 0) {
+        ERRNO_SAFE_WRAP (free, data);
+        return -1;
+    }
+    *lenp = rc;
+    *datap = data;
+    return 0;
+}
+
+int pty_data_unpack (const flux_msg_t *msg,
+                     flux_error_t *errp,
+                     char **datap,
+                     size_t *lenp)
+{
+    const char *encoding = NULL;
+    void *data;
+    size_t len;
+    json_t *o;
+    json_error_t error;
+
+    /* Note: Use json_unpack_ex(3) in order to pass the JSON_ALLOW_NUL flag.
+     *  This is necessary since pty data packed with 's#' may contain NUL
+     *  characters, especially in the case of ^@/Ctrl-Space/etc, which is
+     *  encoded as a NUL character (empty string with len=1).
+     */
+    if (flux_msg_unpack (msg, "o", &o) < 0)
+        return errprintf (errp,
+                          "failed to unpack data msg: %s",
+                          strerror (errno));
+    if (json_unpack_ex (o,
+                        NULL,
+                        JSON_ALLOW_NUL,
+                        "{s:s% s?s}",
+                        "data", &data, &len,
+                        "encoding", &encoding) < 0) {
+        errno = EPROTO;
+        return errprintf (errp,
+                          "failed to unpack data msg: %s",
+                          error.text);
+    }
+    if (encoding) {
+        if (streq (encoding, "base64")) {
+            char *b64data = NULL;
+            size_t b64len;
+            if (decode_base64 (data, len, &b64data, &b64len) < 0) {
+                return errprintf (errp,
+                                  "failed to decode %zu bytes of base64",
+                                  len);
+            }
+            if (flux_msg_aux_set (msg, NULL, b64data, free) < 0) {
+                ERRNO_SAFE_WRAP (free, b64data);
+                return errprintf (errp,
+                                  "flux_msg_aux_set: %s",
+                                  strerror (errno));
+            }
+            data = b64data;
+            len = b64len;
+        }
+        else {
+            errno = EPROTO;
+            return errprintf (errp,
+                              "invalid pty data encoding: %s",
+                              encoding);
+        }
+    }
+    *datap = data;
+    *lenp = len;
+    return 0;
+}
+
+static int pty_write (struct flux_pty *pty, const flux_msg_t *msg)
+{
+    char *data;
+    size_t len;
+    flux_error_t error;
+
+    if (pty_data_unpack (msg, &error, &data, &len) < 0) {
+        llog_error (pty, "%s", error.text);
+        return -1;
+    }
+    if (write (pty->leader, data, len) < 0) {
+        llog_error (pty, "write: %s", strerror (errno));
+        return -1;
+    }
+    return 0;
+}
+
+int flux_pty_sendmsg (struct flux_pty *pty, const flux_msg_t *msg)
+{
+    struct pty_client *c;
+    uint32_t userid;
+    const char *type;
+
+    if (!pty || !pty->h || !msg) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (flux_msg_get_userid (msg, &userid) < 0)
+        goto err;
+    if (userid != getuid ()) {
+        errno = EPERM;
+        goto err;
+    }
+    if (flux_request_unpack (msg, NULL, "{s:s}", "type", &type) < 0) {
+        llog_error (pty, "request_unpack: failed to get message type");
+        goto err;
+    }
+    llog_debug (pty, "msg: userid=%u type=%s", userid, type);
+    c = pty_client_find_sender (pty, msg);
+
+    if (streq (type, "attach")) {
+        /* It is an error for the same client to attach more than once */
+        if (c != NULL) {
+            errno = EEXIST;
+            goto err;
+        }
+        if (pty_attach (pty, msg) < 0)
+            goto err;
+        /* pty_attach() starts a streaming response. Skip singleton
+         * response below
+         */
+        check_pty_complete (pty);
+        return 0;
+    }
+
+    /*  It is an error for the remaining message types to come from
+     *   a sender that is not already attached.
+     */
+    if (c == NULL) {
+        errno = ENOENT;
+        goto err;
+    }
+    else if (streq (type, "resize")) {
+       if (pty_resize (pty, msg) < 0)
+            goto err;
+    }
+    else if (streq (type, "data")) {
+        if (c->write_enabled && pty_write (pty, msg) < 0) {
+            llog_error (pty, "pty_write: %s", strerror (errno));
+            goto err;
+        }
+    }
+    else if (streq (type, "detach")) {
+        if (pty_client_detach (pty, c) < 0)
+            goto err;
+        if (zlist_size (pty->clients) == 0)
+            flux_watcher_stop (pty->fdw);
+    }
+    else {
+        const char *topic;
+        flux_msg_get_topic (msg, &topic);
+        llog_error (pty, "unhandled message type=%s", type);
+        errno = ENOSYS;
+        goto err;
+    }
+    if (flux_respond (pty->h, msg, NULL) < 0)
+        llog_error (pty, "flux_respond: %s", flux_strerror (errno));
+    return 0;
+err:
+    if (flux_respond_error (pty->h, msg, errno, NULL) < 0)
+        llog_error (pty, "flux_respond_error: %s", flux_strerror (errno));
+    return 0;
+}
+
+int flux_pty_set_flux (struct flux_pty *pty, flux_t *h)
+{
+    if (!pty || !h) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    pty->h = h;
+
+    /*  Create pty fd watcher here, but do not start it until the first
+     *   client attaches
+     */
+    pty->fdw = flux_fd_watcher_create (flux_get_reactor (h),
+                                       pty->leader,
+                                       FLUX_POLLIN,
+                                       pty_read,
+                                       pty);
+    if (!pty->fdw)
+        return -1;
+
+    fd_set_nonblocking (pty->leader);
+
+    return 0;
+}
+
+int flux_pty_aux_set (struct flux_pty *pty,
+                      const char *key,
+                      void *val,
+                      flux_free_f destroy)
+{
+    if (!pty) {
+        errno = EINVAL;
+        return -1;
+    }
+    return aux_set (&pty->aux, key, val, destroy);
+}
+
+void * flux_pty_aux_get (struct flux_pty *pty, const char *name)
+{
+    if (!pty) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return aux_get (pty->aux, name);
+
+}
+
+
+/* vi: ts=4 sw=4 expandtab
+ */
+
diff --git a/src/common/libterminus/pty.h b/src/common/libterminus/pty.h
new file mode 100644
index 000000000000..e0947b1e45cf
--- /dev/null
+++ b/src/common/libterminus/pty.h
@@ -0,0 +1,226 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Simple pty/terminal client/server over flux messages */
+
+#ifndef FLUX_PTY_H
+#define FLUX_PTY_H
+
+#include <jansson.h>
+#include <flux/core.h>
+
+typedef void (*pty_log_f) (void *arg,
+                           const char *file,
+                           int line,
+                           const char *func,
+                           const char *subsys,
+                           int level,
+                           const char *fmt,
+                           va_list args);
+
+
+struct flux_pty;
+struct flux_pty_client;
+
+typedef void (*pty_monitor_f) (struct flux_pty *pty,
+                               void *data,
+                               int len);
+
+typedef void (*pty_complete_f) (struct flux_pty *pty);
+
+/* server:
+ */
+
+
+/*  Open a new server-side pty handle.
+ */
+struct flux_pty *flux_pty_open (void);
+
+void flux_pty_destroy (struct flux_pty *pty);
+
+/*
+ * Notify pty that associated process has exited.
+ *
+ * This may trigger the exit callback immediately, unless data
+ *  is still being read from the pty, or the pty is waiting
+ *  for the first client to attach.
+ */
+void flux_pty_exited (struct flux_pty *pty, int status);
+
+/* Set wait-for-client flag on 'pty'. This disables read from the pty
+ *  fd until the first client attaches so that no data is lost. Use
+ *  of flux_pty_monitor() does not count as a client in this case, which
+ *  allows a pty to be monitored, while still waiting for the first
+ *  real client to attach.
+ */
+void flux_pty_wait_for_client (struct flux_pty *pty);
+
+/* Set wait-on-close flag on 'pty'. This ensures the pty does not
+ *  "complete" until all data has been read from the pty.
+ *  (only should be set if a process is attached to the pty with
+ *   flux_pty_attach())
+ */
+void flux_pty_wait_on_close (struct flux_pty *pty);
+
+/*  Send signal `sig` to the foreground process group of `pty`.
+ */
+int  flux_pty_kill (struct flux_pty *pty, int sig);
+
+/*  Set internal logger for this pty instance.
+ */
+void flux_pty_set_log (struct flux_pty *pty,
+                       pty_log_f log,
+                       void *log_data);
+
+/*  Set a callback to receive data events locally.
+ *  (Useful if we want to locally monitor the pty for logging, etc.)
+ */
+void flux_pty_monitor (struct flux_pty *pty, pty_monitor_f fn);
+
+/*  Set a callback which is triggered when a pty has exited,
+ *   has read all data from the pty, and is not waiting for any
+ *   client to attach. The default handler calls flux_pty_destroy ().
+ */
+void flux_pty_set_complete_cb (struct flux_pty *pty, pty_complete_f fn);
+
+int flux_pty_leader_fd (struct flux_pty *pty);
+
+/*  Return the follower name for this pty
+ */
+const char *flux_pty_name (struct flux_pty *pty);
+
+/*  Attach the current process to follower end of pty
+ *  (e.g. called from pre_exec hook of a flux_subprocess_t)
+ */
+int flux_pty_attach (struct flux_pty *pty);
+
+int flux_pty_set_flux (struct flux_pty *pty, flux_t *h);
+
+/*  Return the current number of connected clients for pty
+ */
+int flux_pty_client_count (struct flux_pty *pty);
+
+/*  Send a request msg to the pty server pty. Used in a msg_handler
+ *   for the pty topic string for this pty service.
+ */
+int flux_pty_sendmsg (struct flux_pty *pty, const flux_msg_t *msg);
+
+/*  Add new client that receives no data, only waits for pty to exit
+ */
+int flux_pty_add_exit_watcher (struct flux_pty *pty, const flux_msg_t *msg);
+
+/*  Disconnect any client matching sender
+ */
+int flux_pty_disconnect_client (struct flux_pty *pty, const char *sender);
+
+
+/* client:
+ */
+
+enum {
+    FLUX_PTY_CLIENT_ATTACH_SYNC =      1,  /* Synchronous attach            */
+    FLUX_PTY_CLIENT_CLEAR_SCREEN =     2,  /* Clear screen on attach        */
+    FLUX_PTY_CLIENT_NOTIFY_ON_ATTACH = 4,  /* Informational msg on attach   */
+    FLUX_PTY_CLIENT_NOTIFY_ON_DETACH = 8,  /* Informational msg on attach   */
+    FLUX_PTY_CLIENT_NORAW =           16,  /* Do not set raw mode on attach */
+    FLUX_PTY_CLIENT_STDIN_PIPE =      32,  /* Pipe stdin to remote on attach*/
+};
+
+/*  Create a new pty client
+ */
+struct flux_pty_client *flux_pty_client_create (void);
+
+void flux_pty_client_destroy (struct flux_pty_client *c);
+
+int flux_pty_client_set_flags (struct flux_pty_client *c,
+                               int flags);
+int flux_pty_client_get_flags (struct flux_pty_client *c);
+
+/*  Set internal pty client logging function
+ */
+void flux_pty_client_set_log (struct flux_pty_client *c,
+                              pty_log_f log,
+                              void *log_data);
+
+
+/*  Attach pty client to server at rank,service endpoint. The current tty
+ *   will be put into "raw" mode. Terminal will be restored when process
+ *   exits, or flux_pty_client_restore_terminal() is called.
+ *
+ *  Will block until first attach response if FLUX_PTY_CLIENT_ATTACH_SYNC
+ *   was set in flags.
+ *
+ *  Screen is cleared on attach if FLUX_PTY_CLIENT_CLEAR_SCREEN is set.
+ */
+int flux_pty_client_attach (struct flux_pty_client *c,
+                            flux_t *h,
+                            int rank,
+                            const char *service);
+
+/*  Return true if the pty has completed an attach, i.e. the attach
+ *   request has gotten its first response.
+ */
+bool flux_pty_client_attached (struct flux_pty_client *pty);
+
+/*  Write data out-of-band to remote pty.
+ */
+flux_future_t *flux_pty_client_write (struct flux_pty_client *c,
+                                      const void *buf,
+                                      ssize_t len);
+
+/*  Send request to pty server to detach the current client.
+ *  Client will call exit callback when detach is complete.
+ */
+int flux_pty_client_detach (struct flux_pty_client *c);
+
+
+typedef void (*flux_pty_client_exit_f) (struct flux_pty_client *c,
+                                        void *arg);
+
+/*  Call supplied function when pty client `c` "exits" i.e. detaches
+ *   or remote pty exits.
+ */
+int flux_pty_client_notify_exit (struct flux_pty_client *c,
+                                 flux_pty_client_exit_f fn,
+                                 void *arg);
+
+int flux_pty_client_exit_status (struct flux_pty_client *c, int *statusp);
+
+void flux_pty_client_restore_terminal (void);
+
+int flux_pty_aux_set (struct flux_pty *pty,
+                      const char *key,
+                      void *val,
+                      flux_free_f destroy);
+
+void * flux_pty_aux_get (struct flux_pty *pty, const char *name);
+
+
+/*  Function exported for testing only
+ */
+void pty_client_send_data (struct flux_pty *pty, void *data, int len);
+
+
+/*  Common function for unpacking pty data msg.
+ */
+int pty_data_unpack (const flux_msg_t *msg,
+                     flux_error_t *errp,
+                     char **datap,
+                     size_t *lenp);
+
+/*  Common function for encoding pty data msg.
+ */
+json_t *pty_data_encode (const void *data, int len);
+
+#endif /* !FLUX_PTY_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libterminus/terminus.c b/src/common/libterminus/terminus.c
new file mode 100644
index 000000000000..830b05065341
--- /dev/null
+++ b/src/common/libterminus/terminus.c
@@ -0,0 +1,926 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* libterminus - Termin(al) User Services for Flux
+ *
+ * Manages multiple Flux pty sessions behind a common *-terminus
+ *  service endpoint. Supports;
+ *
+ * - Create new terminal session: *terminus.new
+ *
+ *   IN:   { "name":s "cmd":[] "environ":{} "cwd":s }
+ *   OUT:  { "name":s "pty_service":s "id":i }
+ *
+ * - List current terminal sessions: *terminus.list
+ *
+ *   IN:   {}
+ *   OUT:  { "server":{ "service":s "rank":i "ctime":f }
+ *           "sessions":[ { "id":i "name":s "clients"i
+ *                          "pid"i "ctime":f }, ...
+ *                      ]
+ *         }
+ *
+ * - Kill terminal sessions by ID: *terminus.kill
+ *   If 'wait', then response will be delayed until session exits
+ *
+ *   IN:   { "id":i "signal":i "wait"?i }
+ *   OUT:  {}
+ *
+ * - Kill all sessions: *terminus.kill-server
+ *
+ *   IN:   {}
+ *   OUT:  {} (response after all sessions exit)
+ *
+ *
+ * Sessions are managed on *terminus.ID service endpoints.
+ * Once the session ID is known, a client may connect directly to
+ * to the pty server at this service, using the protocol detailed
+ * in pty.c.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <signal.h>
+
+#include <jansson.h>
+
+#include <flux/idset.h>
+
+#define LLOG_SUBSYSTEM "pty"
+
+#include "src/common/libutil/llog.h"
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libterminus/pty.h"
+
+#include "terminus.h"
+
+extern char **environ;
+
+struct terminus_session {
+    struct flux_terminus_server *server;
+    int refcount;
+
+    char *name;
+    char topic[128];
+    flux_msg_handler_t *mh;
+
+    int id;
+    double ctime;
+
+    flux_subprocess_t *p;
+    flux_cmd_t *cmd;
+
+    bool wait_on_attach; /* wait at least one attach before reaping */
+    bool exited;         /* true if subprocess exited */
+
+    struct flux_pty *pty;
+};
+
+struct empty_waiter {
+    flux_terminus_server_empty_f cb;
+    void *arg;
+};
+
+struct flux_terminus_server {
+    flux_t *h;
+    uint32_t rank;
+
+    flux_msg_handler_t **handlers;
+
+    terminus_log_f llog;
+    void *llog_data;
+
+    char service[128];
+
+    struct idset *idset;
+
+    double ctime;
+    zlist_t *sessions;
+    zlist_t *empty_waiters;
+};
+
+static void terminus_session_incref (struct terminus_session *s)
+{
+    if (s)
+        s->refcount++;
+}
+
+static void terminus_session_decref (struct terminus_session *s)
+{
+    if (s && --s->refcount == 0) {
+        if (s->id >= 0)
+            idset_clear (s->server->idset, s->id);
+        flux_pty_destroy (s->pty);
+        flux_msg_handler_destroy (s->mh);
+        free (s->name);
+        flux_subprocess_destroy (s->p);
+        free (s);
+    }
+}
+
+static void terminus_session_destroy (struct terminus_session *s)
+{
+    terminus_session_decref (s);
+}
+
+static void notify_empty_waiters (struct flux_terminus_server *ts)
+{
+    struct empty_waiter *w;
+    while ((w = zlist_pop (ts->empty_waiters))) {
+        (*w->cb) (ts, w->arg);
+        free (w);
+    }
+}
+
+static void server_remove_session (struct flux_terminus_server *ts,
+                                   struct terminus_session *s)
+{
+    if (s) {
+        zlist_remove (ts->sessions, s);
+        if (zlist_size (ts->sessions) == 0)
+            notify_empty_waiters (ts);
+    }
+}
+
+static void session_msg_handler (flux_t *h,
+                                 flux_msg_handler_t *mh,
+                                 const flux_msg_t *msg,
+                                 void *arg)
+{
+    struct terminus_session *s = arg;
+    terminus_session_incref (s);
+    if (flux_pty_sendmsg (s->pty, msg) < 0)
+        llog_error (s->server, "flux_pty_sendmsg: %s", strerror (errno));
+
+    /*  If session is waiting for first attach, and there is 1 or
+     *   more clients attached now, then wait can be disabled
+     */
+    if (s->wait_on_attach && flux_pty_client_count (s->pty) > 0)
+        s->wait_on_attach = false;
+    terminus_session_decref (s);
+}
+
+static int terminus_msg_handler_start (struct terminus_session *s)
+{
+    struct flux_match match = FLUX_MATCH_REQUEST;
+
+    match.topic_glob = s->topic;
+    if (!(s->mh = flux_msg_handler_create (s->server->h,
+                                           match,
+                                           session_msg_handler,
+                                           s)))
+        return -1;
+    flux_msg_handler_allow_rolemask (s->mh, FLUX_ROLE_USER);
+    flux_msg_handler_start (s->mh);
+    return 0;
+}
+
+static void session_complete (struct flux_pty *pty)
+{
+    struct terminus_session *s = flux_pty_aux_get (pty, "terminus_session");
+    server_remove_session (s->server, s);
+}
+
+static struct terminus_session *
+terminus_session_create (struct flux_terminus_server *ts,
+                         int id,
+                         const char *name,
+                         bool wait)
+{
+    struct terminus_session *s = calloc (1, sizeof (*s));
+    if (!s)
+        return NULL;
+
+    s->refcount = 1;
+    s->id = id;
+    s->server = ts;
+    s->wait_on_attach = wait;
+    s->ctime = flux_reactor_now (flux_get_reactor (ts->h));
+    if (name && !(s->name = strdup (name)))
+        goto error;
+    if (!(s->pty = flux_pty_open ()))
+        goto error;
+    if (flux_pty_set_flux (s->pty, ts->h) < 0
+        || flux_pty_aux_set (s->pty, "terminus_session", s, NULL) < 0)
+        goto error;
+    flux_pty_wait_on_close (s->pty);
+    flux_pty_set_complete_cb (s->pty, session_complete);
+    if (s->wait_on_attach)
+        flux_pty_wait_for_client (s->pty);
+    if (ts->llog)
+        flux_pty_set_log (s->pty, ts->llog, ts->llog_data);
+    if (snprintf (s->topic,
+                  sizeof (s->topic),
+                  "%s.%d",
+                  s->server->service,
+                  s->id) >= sizeof (s->topic)) {
+        errno = EOVERFLOW;
+        goto error;
+    }
+    if (terminus_msg_handler_start (s) < 0)
+        goto error;
+    if (zlist_append (ts->sessions, s) < 0) {
+        errno = ENOMEM;
+        goto error;
+    }
+    if (!zlist_freefn (ts->sessions,
+                       s,
+                       (zlist_free_fn *) terminus_session_destroy,
+                       true)) {
+        errno = ENOENT;
+        goto error;
+    }
+    return s;
+error:
+    /*  On error we _possibly_ need remove session from zlist,
+     *   but definitely need to manually destroy session, since
+     *   zlist_freefn() is guaranteed not to have been called if
+     *   we got here.
+     */
+    if (s) {
+        int saved_errno = errno;
+        zlist_remove (ts->sessions, s);
+        terminus_session_destroy (s);
+        errno = saved_errno;
+    }
+    return NULL;
+}
+
+static int terminus_session_kill (struct terminus_session *s, int signum)
+{
+    /*  When killing a session, clear the wait flag so we don't hang
+     *   waiting on the first attach.
+     */
+    s->wait_on_attach = false;
+
+    /*  Session may have already exited if wait_on_attach.
+     *  Close the pty now to avoid a hang.
+     */
+    if (s->exited) {
+        server_remove_session (s->server, s);
+        return 0;
+    }
+    /*  First kill processes using pty, then signal process group,
+     *   though they may be one in the same
+     */
+    if (flux_pty_kill (s->pty, signum) < 0
+        || kill (-flux_subprocess_pid (s->p), signum) < 0)
+        return -1;
+
+    return 0;
+}
+
+#if CODE_COVERAGE_ENABLED
+extern void __gcov_dump ();
+extern void __gcov_reset ();
+#endif
+static void terminus_pty_attach (flux_subprocess_t *p, void *arg)
+{
+    struct terminus_session *s = arg;
+    if (flux_pty_attach (s->pty) < 0) {
+        llog_fatal (s->server, "terminus: pty attach: %s\n", strerror (errno));
+#if CODE_COVERAGE_ENABLED
+        __gcov_dump ();
+        __gcov_reset ();
+#endif
+        _exit (1);
+    }
+}
+
+int flux_terminus_server_session_close (struct flux_terminus_server *ts,
+                                        struct flux_pty *pty,
+                                        int status)
+{
+    struct terminus_session *s;
+
+    if (!ts || !pty || status < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    s = zlist_first (ts->sessions);
+    while (s) {
+        if (s->pty == pty)
+            break;
+        s = zlist_next (ts->sessions);
+    }
+    if (s) {
+        s->exited = true;
+        flux_pty_exited (s->pty, status);
+        return 0;
+    }
+    errno = ENOENT;
+    return -1;
+}
+
+static void terminus_session_exit (flux_subprocess_t *p)
+{
+    struct terminus_session *s = flux_subprocess_aux_get (p, "terminus");
+    llog_debug (s->server, "session %d exit: pid=%ld status=%d",
+                s->id,
+                (long) flux_subprocess_pid (p),
+                flux_subprocess_status (p));
+    s->exited = true;
+    flux_pty_exited (s->pty, flux_subprocess_status (p));
+}
+
+static int terminus_session_start (struct terminus_session *s,
+                                   flux_cmd_t *cmd)
+{
+    int flags = FLUX_SUBPROCESS_FLAGS_STDIO_FALLTHROUGH;
+    flux_subprocess_hooks_t hooks = {
+        .pre_exec = terminus_pty_attach,
+        .pre_exec_arg = s,
+    };
+    flux_subprocess_ops_t ops = {
+        .on_completion = terminus_session_exit,
+    };
+    s->cmd = cmd;
+    s->p = flux_local_exec_ex (flux_get_reactor (s->server->h),
+                               flags,
+                               cmd,
+                               &ops,
+                               &hooks,
+                               NULL,
+                               NULL);
+    if (!s->p)
+        goto cleanup;
+    if (flux_subprocess_aux_set (s->p, "terminus", s, NULL) < 0)
+        goto cleanup;
+    return 0;
+cleanup:
+    flux_subprocess_destroy (s->p);
+    s->p = NULL;
+    return -1;
+}
+
+static void flux_terminus_server_stop (struct flux_terminus_server *ts)
+{
+    flux_msg_handler_delvec (ts->handlers);
+    ts->handlers = NULL;
+}
+
+int flux_terminus_server_notify_empty (struct flux_terminus_server *ts,
+                                       flux_terminus_server_empty_f cb,
+                                       void *arg)
+{
+    struct empty_waiter *w;
+
+    if (!ts || !cb) {
+        errno = EINVAL;
+        return -1;
+    }
+    w = calloc (1, sizeof (*w));
+    if (!w)
+        return -1;
+    w->cb = cb;
+    w->arg = arg;
+    if (zlist_append (ts->empty_waiters, w) < 0) {
+        free (w);
+        return -1;
+    }
+    zlist_freefn (ts->empty_waiters, w, (zlist_free_fn *) free, true);
+    return 0;
+}
+
+void flux_terminus_server_destroy (struct flux_terminus_server *t)
+{
+    if (t) {
+        flux_terminus_server_stop (t);
+        zlist_destroy (&t->empty_waiters);
+        zlist_destroy (&t->sessions);
+        idset_destroy (t->idset);
+        free (t);
+    }
+}
+
+struct terminus_session * session_lookup (struct flux_terminus_server *ts,
+                                          int id)
+{
+    struct terminus_session *s = zlist_first (ts->sessions);
+    while (s) {
+        if (s->id == id)
+            return s;
+        s = zlist_next (ts->sessions);
+    }
+    return NULL;
+}
+
+/*  Build a flux_cmd_t from the msg 'msg'.
+ *  All of cmd, environ, and cwd are optional, with defaults '$SHELL'
+ *   current environment, and current working directory respectively.
+ */
+static flux_cmd_t *make_cmd (const flux_msg_t *msg)
+{
+    int i;
+    char cwd_buf [4096];
+    json_t *val;
+    json_t *cmd_array = NULL;
+    const char *cwd = NULL;
+    json_t *env = NULL;
+    flux_cmd_t *cmd = NULL;
+    const char *shell = getenv ("SHELL");
+
+    if (flux_msg_unpack (msg,
+                         "{s?o s?o s?s}",
+                         "cmd", &cmd_array,
+                         "environ", &env,
+                         "cwd", &cwd) < 0)
+        return NULL;
+
+    if ((cmd_array && !json_is_array (cmd_array))
+        || (env && !json_is_object (env))) {
+        errno = EPROTO;
+        goto err;
+    }
+
+    if (!(cmd = flux_cmd_create (0, NULL, env ? NULL: environ)))
+        goto err;
+
+    if (cmd_array && json_array_size (cmd_array) > 0) {
+        json_array_foreach (cmd_array, i, val) {
+            if (flux_cmd_argv_append (cmd, json_string_value (val)) < 0)
+                goto err;
+        }
+    }
+    else if (flux_cmd_argv_append (cmd, shell ? shell : "bash") < 0)
+        goto err;
+
+    if (!cwd)
+        cwd = getcwd (cwd_buf, sizeof (cwd_buf));
+    if (flux_cmd_setcwd (cmd, cwd) < 0)
+        goto err;
+
+    if (env) {
+        const char *key;
+        json_t *val;
+        json_object_foreach (env, key, val) {
+            const char *s = json_string_value (val);
+            if (flux_cmd_setenvf (cmd, 1, key, "%s", s) < 0)
+                goto err;
+        }
+    }
+
+    return cmd;
+err:
+    flux_cmd_destroy (cmd);
+    return NULL;
+}
+
+static int session_id (struct flux_terminus_server *ts)
+{
+    unsigned int i = 0;
+    while (idset_test (ts->idset, i))
+        ++i;
+    if (idset_set (ts->idset, i) < 0)
+        return -1;
+    return i;
+}
+
+static char *make_errmsg (char *buf, int buflen, const char *fmt, ...)
+{
+    va_list ap;
+    int saved_errno = errno;
+    va_start (ap, fmt);
+    vsnprintf (buf, buflen, fmt, ap);
+    va_end (ap);
+    errno = saved_errno;
+    return buf;
+}
+
+struct flux_pty *
+flux_terminus_server_session_open (struct flux_terminus_server *ts,
+                                   int id,
+                                   const char *name)
+{
+    struct terminus_session *s = NULL;
+
+    if (!ts || id < 0 || !name) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (idset_test (ts->idset, id)) {
+        errno = EEXIST;
+        return NULL;
+    }
+    if (idset_set (ts->idset, id) < 0)
+        goto error;
+    if (!(s = terminus_session_create (ts, id, name, false)))
+        goto error;
+    return s->pty;
+error:
+    idset_clear (ts->idset, id);
+    return NULL;
+}
+
+static int check_userid (const flux_msg_t *msg)
+{
+    uint32_t userid;
+
+    if (flux_msg_get_userid (msg, &userid) < 0)
+        return -1;
+    if (userid != getuid ()) {
+        errno = EPERM;
+        return -1;
+    }
+    return 0;
+}
+
+static void new_session (flux_t *h,
+                         flux_msg_handler_t *mh,
+                         const flux_msg_t *msg,
+                         void *arg)
+{
+    struct flux_terminus_server *ts = arg;
+    char errbuf [128];
+    const char *errmsg = NULL;
+    const char *name = NULL;
+    flux_cmd_t *cmd;
+    int id;
+    bool wait = true;
+    struct terminus_session *s = NULL;
+
+    if (check_userid (msg) < 0)
+        goto error;
+
+    if (flux_request_unpack (msg, NULL,
+                             "{s?s}",
+                             "name", &name) < 0) {
+        errno = EPROTO;
+        goto error;
+    }
+    if (!(cmd = make_cmd (msg))) {
+        errno = EPROTO;
+        errmsg = "failed to parse cmd field";
+        goto error;
+    }
+    if (!name || strlen (name) == 0)
+        name = flux_cmd_arg (cmd, 0);
+    if ((id = session_id (ts)) < 0) {
+        errmsg = "unable to get new session id";
+        goto error;
+    }
+    if (flux_cmd_setenvf (cmd, 1, "FLUX_TERMINUS_SESSION", "%d", id) < 0) {
+        errmsg = "failed to set FLUX_TERMINUS_SESSION in environment";
+        goto error;
+    }
+    if (!(s = terminus_session_create (ts, id, name, wait)))
+        goto error;
+    if (terminus_session_start (s, cmd) < 0) {
+        errmsg = make_errmsg (errbuf,
+                              sizeof (errbuf),
+                              "failed to run %s",
+                              flux_cmd_arg (cmd, 0));
+        goto error;
+    }
+    if (flux_respond_pack (h, msg,
+                          "{s:s s:s s:i}",
+                          "name", name,
+                          "pty_service", s->topic,
+                          "id", s->id) < 0) {
+        llog_error (ts, "flux_respond_pack: %s", strerror (errno));
+    }
+    flux_cmd_destroy (cmd);
+    return;
+error:
+    /* N.B.: triggers destruction of s
+     */
+    server_remove_session (ts, s);
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        llog_error (ts, "flux_respond_error: %s", strerror (errno));
+}
+
+static int list_append_session (json_t *l, struct terminus_session *s)
+{
+    json_t *o;
+
+    if (!(o = json_pack ("{s:i s:s s:i s:i s:i s:f}",
+                         "id", s->id,
+                         "name", s->name,
+                         "clients", flux_pty_client_count (s->pty),
+                         "pid", flux_subprocess_pid (s->p),
+                         "exited", s->exited,
+                         "ctime", s->ctime))) {
+        errno = ENOMEM;
+        return -1;
+    }
+    if (json_array_append_new (l, o) < 0) {
+        json_decref (o);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+json_t *server_info (struct flux_terminus_server *ts)
+{
+    return json_pack ("{s:s s:i s:f}",
+                      "service", ts->service,
+                      "rank", ts->rank,
+                      "ctime", ts->ctime);
+}
+
+static void list_sessions (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct flux_terminus_server *ts = arg;
+    struct terminus_session *s;
+    json_t *sessions = NULL;
+    json_t *info = NULL;
+
+    if (check_userid (msg) < 0)
+        goto error;
+
+    if (!(sessions = json_array ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    s = zlist_first (ts->sessions);
+    while (s) {
+        if (list_append_session (sessions, s) < 0)
+            goto error;
+        s = zlist_next (ts->sessions);
+    }
+    if (!(info = server_info (ts)))
+        goto error;
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:o s:o}",
+                           "sessions", sessions,
+                           "server", info) < 0)
+        llog_error (ts, "flux_respond_pack: %s", strerror (errno));
+    return;
+error:
+    json_decref (sessions);
+    json_decref (info);
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+         llog_error (ts, "flux_respond_error: %s", strerror (errno));
+}
+
+static void kill_sessions (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct flux_terminus_server *ts = arg;
+    struct terminus_session *s;
+    int id;
+    int signum;
+    int wait = false;
+    char *errmsg = NULL;
+
+    if (check_userid (msg) < 0)
+        goto error;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s:i s?i}",
+                             "id", &id,
+                             "signal", &signum,
+                             "wait", &wait) < 0)
+        goto error;
+    if (!(s = session_lookup (ts, id))) {
+        errno = ENOENT;
+        goto error;
+    }
+    if (terminus_session_kill (s, signum) < 0)
+        goto error;
+    /*
+     *  If 'wait' flag was specified, then attach a new client to pty
+     *   that will respond once the pty has fully exited. O/w simply
+     *   respond with Success now.
+     */
+    if (wait) {
+        if (flux_pty_add_exit_watcher (s->pty, msg) < 0)
+            goto error;
+    }
+    else if (flux_respond (ts->h, msg, NULL) < 0)
+        llog_error (ts, "flux_respond: %s", strerror (errno));
+    return;
+error:
+    if (flux_respond_error (ts->h, msg, errno, errmsg) < 0)
+        llog_error (ts, "flux_respond_error: %s", strerror (errno));
+}
+
+static void kill_server_exit (struct flux_terminus_server *ts, void *arg)
+{
+    const flux_msg_t *msg = arg;
+    flux_respond (ts->h, msg, NULL);
+    flux_msg_decref (msg);
+    flux_terminus_server_stop (ts);
+}
+
+static void kill_server (flux_t *h,
+                         flux_msg_handler_t *mh,
+                         const flux_msg_t *msg,
+                         void *arg)
+{
+    struct flux_terminus_server *ts = arg;
+    struct terminus_session *s;
+
+    if (check_userid (msg) < 0)
+        goto error;
+
+    /*  If no active sessions, exit server immediately */
+    if (zlist_size (ts->sessions) == 0) {
+        kill_server_exit (ts, (void *) flux_msg_incref (msg));
+        return;
+    }
+
+    /*  Grab reference to message so we can notify when all sessions
+     *   have been killed
+     */
+    flux_msg_incref (msg);
+
+    /*  Register empty callback so that server is stopped and response
+     *   goes out when last session exits.
+     */
+    if (flux_terminus_server_notify_empty (ts,
+                                           kill_server_exit,
+                                           (void *) msg) < 0) {
+        flux_msg_decref (msg);
+        goto error;
+    }
+
+    /*  Kill all active sessions */
+    s = zlist_first (ts->sessions);
+    while (s) {
+        (void) terminus_session_kill (s, SIGKILL);
+        s = zlist_next (ts->sessions);
+    }
+    return;
+error:
+    if (flux_respond_error (ts->h, msg, errno, NULL) < 0)
+        llog_error (ts, "flux_respond_error: %s", strerror (errno));
+}
+
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct flux_terminus_server *ts = arg;
+    struct terminus_session *s;
+    const char *sender;
+
+    if (!(sender = flux_msg_route_first (msg))) {
+        llog_error (ts, "flux_msg_get_route_first: uuid is NULL!");
+        return;
+    }
+    s = zlist_first (ts->sessions);
+    while (s) {
+        flux_pty_disconnect_client (s->pty, sender);
+        s = zlist_next (ts->sessions);
+    }
+}
+
+static const struct flux_msg_handler_spec handler_tab[] = {
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "list",
+        list_sessions,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "new",
+        new_session,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kill",
+        kill_sessions,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kill-server",
+        kill_server,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "disconnect",
+        disconnect_cb,
+        0
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+static inline int
+msg_handler_count (const struct flux_msg_handler_spec htab[])
+{
+    int count = 0;
+    while (htab[count].topic_glob)
+        count++;
+    return count;
+}
+
+static int start_msghandlers (struct flux_terminus_server *ts,
+                              const struct flux_msg_handler_spec tab[])
+{
+    int i;
+    char topic[128];
+    int len = sizeof (topic);
+    struct flux_match match = FLUX_MATCH_REQUEST;
+    int count = msg_handler_count (tab);
+
+    ts->handlers = calloc (count+1, sizeof (*ts->handlers));
+    for (i = 0; i < count; i++) {
+        flux_msg_handler_t *mh = NULL;
+        if (snprintf (topic,
+                      len,
+                      "%s.%s", ts->service,
+                      tab[i].topic_glob) >= len)
+            goto error;
+        match.topic_glob = topic;
+        match.typemask = tab[i].typemask;
+        if (!(mh = flux_msg_handler_create (ts->h, match, tab[i].cb, ts)))
+            goto error;
+        flux_msg_handler_allow_rolemask (mh, tab[i].rolemask);
+        flux_msg_handler_start (mh);
+        ts->handlers[i] = mh;
+    }
+    return 0;
+error:
+    flux_msg_handler_delvec (ts->handlers);
+    return -1;
+}
+
+flux_future_t *
+flux_terminus_server_unregister (struct flux_terminus_server *ts)
+{
+    return flux_service_unregister (ts->h, ts->service);
+}
+
+struct flux_terminus_server *
+flux_terminus_server_create (flux_t *h, const char *service)
+{
+    struct flux_terminus_server *ts;
+
+    if (!h || !service) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(ts = calloc (1, sizeof (*ts)))
+        || !(ts->empty_waiters = zlist_new ())
+        || !(ts->sessions = zlist_new ())) {
+        goto err;
+    }
+    ts->h = h;
+    ts->ctime = flux_reactor_now (flux_get_reactor (h));
+
+    /*  In test mode, avoid flux_get_rank(3) as it will hang
+     */
+    if (getenv ("FLUX_TERMINUS_TEST_SERVER"))
+        ts->rank = -1;
+    else if (flux_get_rank (h, &ts->rank) < 0)
+        goto err;
+
+    if (!(ts->idset = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        goto err;
+
+    if (strlen (service) > sizeof (ts->service) - 1)
+        goto err;
+    strcpy (ts->service, service);
+
+    if (start_msghandlers (ts, handler_tab) < 0)
+        goto err;
+
+    return ts;
+err:
+    flux_terminus_server_destroy (ts);
+    return NULL;
+}
+
+void flux_terminus_server_set_log (struct flux_terminus_server *ts,
+                                   terminus_log_f log_fn,
+                                   void *log_arg)
+{
+    if (ts) {
+        ts->llog = log_fn;
+        ts->llog_data = log_arg;
+    }
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
+
diff --git a/src/common/libterminus/terminus.h b/src/common/libterminus/terminus.h
new file mode 100644
index 000000000000..0ab3201ba519
--- /dev/null
+++ b/src/common/libterminus/terminus.h
@@ -0,0 +1,86 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef FLUX_TERMINUS_H
+#define FLUX_TERMINUS_H
+
+#include <stdarg.h>
+
+#include <flux/core.h>
+
+
+struct flux_terminus_server;
+
+/*  Logging function prototype */
+typedef void (*terminus_log_f) (void *arg,
+                                const char *file,
+                                int line,
+                                const char *func,
+                                const char *subsys,
+                                int level,
+                                const char *fmt,
+                                va_list args);
+
+/*  Callback prototype for use when a terminus server becomes "empty",
+ *   i.e. the last running session exits. (Used for optional cleanup).
+ */
+typedef void (*flux_terminus_server_empty_f) (struct flux_terminus_server *ts,
+                                              void *arg);
+
+
+/*  Create a flux_terminus_server listening at topic `service`
+ */
+struct flux_terminus_server *
+flux_terminus_server_create (flux_t *h, const char *service);
+
+void flux_terminus_server_destroy (struct flux_terminus_server *ts);
+
+/*  Set internal libterminus logging function to 'log_fn'. If unset,
+ *   then logging from the library is disabled.
+ */
+void flux_terminus_server_set_log (struct flux_terminus_server *ts,
+                                   terminus_log_f log_fn,
+                                   void *log_data);
+
+/*  Call function `fn` when terminus server next becomes empty, i.e. goes
+ *   from having 1 or more sessions to no sessions. (if there are no
+ *   current sessions, then the empty cb will *not* be called immediately,
+ *   only after at least one session is created and destroyed)
+ *
+ *  Callbacks are oneshot, i.e. they are removed after being called.
+ */
+int flux_terminus_server_notify_empty (struct flux_terminus_server *ts,
+                                       flux_terminus_server_empty_f fn,
+                                       void *arg);
+
+/*  Open a session directly in the server ts (as oppopsed to via the
+ *   protocol.
+ */
+struct flux_pty *
+flux_terminus_server_session_open (struct flux_terminus_server *ts,
+                                   int id,
+                                   const char *name);
+
+int flux_terminus_server_session_close (struct flux_terminus_server *ts,
+                                        struct flux_pty *pty,
+                                        int status);
+
+/*  Unregister the terminus server service.
+ *  Returns a future that will be fulfilled when the service is successfully
+ *  unregistered.
+ */
+flux_future_t *
+flux_terminus_server_unregister (struct flux_terminus_server *ts);
+
+#endif /* !FLUX_TERMINUS_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libterminus/test/pty.c b/src/common/libterminus/test/pty.c
new file mode 100644
index 000000000000..43150495e510
--- /dev/null
+++ b/src/common/libterminus/test/pty.c
@@ -0,0 +1,505 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include <flux/core.h>
+#include "src/common/libterminus/pty.h"
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libtestutil/util.h"
+
+static void test_invalid_args ()
+{
+    struct flux_pty *pty = flux_pty_open ();
+    struct flux_pty_client *c = flux_pty_client_create ();
+
+    if (pty == NULL || c == NULL)
+        BAIL_OUT ("Failed to create pty client/server!");
+
+    lives_ok ({flux_pty_set_log (NULL, NULL, NULL);},
+              "flux_pty_set_log does nothing with NULL args");
+    lives_ok ({flux_pty_destroy (NULL);},
+              "flux_pty_destroy does nothing with NULL arg");
+
+    ok (flux_pty_kill (NULL, SIGINT) < 0 && errno == EINVAL,
+        "flux_pty_kill() with NULL pty returns EINVAL");
+    ok (flux_pty_kill (pty, -1) < 0 && errno == EINVAL,
+        "flux_pty_kill() with invalid signal returns EINVAL");
+    ok (flux_pty_leader_fd (NULL) < 0 && errno == EINVAL,
+        "flux_pty_leader_fd() returns EINVAL with NULL arg");
+    ok (flux_pty_name (NULL) == NULL && errno == EINVAL,
+        "flux_pty_name() returns EINVAL with NULL arg");
+    ok (flux_pty_attach (NULL) < 0 && errno == EINVAL,
+        "flux_pty_attach() returns EINVAL with NULL arg");
+    ok (flux_pty_client_attached (NULL) == false,
+        "flux_pty_client_attached() returns false with NULL arg");
+    ok (flux_pty_set_flux (NULL, NULL) < 0 && errno == EINVAL,
+        "flux_pty_set_flux() returns EINVAL with NULL args");
+    ok (flux_pty_set_flux (pty, NULL) < 0 && errno == EINVAL,
+        "flux_pty_set_flux() returns EINVAL with NULL flux handle");
+    ok (flux_pty_client_count (NULL) == 0,
+        "flux_pty_client_count returns 0 for NULL pty");
+    ok (flux_pty_disconnect_client (NULL, NULL) < 0 && errno == EINVAL,
+        "flux_pty_disconnect_client returns EINVAL on NULL args");
+    ok (flux_pty_disconnect_client (pty, NULL) < 0 && errno == EINVAL,
+        "flux_pty_disconnect_client returns EINVAL on NULL sender");
+
+    lives_ok ({flux_pty_client_set_log (NULL, NULL, NULL);},
+              "flux_pty_client_set_log does nothing with NULL args");
+    lives_ok ({flux_pty_client_destroy (NULL);},
+              "flux_pty_client_destroy (NULL) does nothing");
+
+    ok (flux_pty_client_attach (c, NULL, 0, NULL) < 0
+        && errno == EINVAL,
+        "flux_pty_client_attach returns EINVAL with NULL args");
+    ok (flux_pty_client_attach (NULL, NULL, 0, NULL) < 0
+        && errno == EINVAL,
+        "flux_pty_client_attach returns EINVAL with NULL handle");
+
+    ok (flux_pty_client_notify_exit (NULL, NULL, NULL) < 0 && errno == EINVAL,
+        "flux_pty_client_notify_exit() returns EINVAL on NULL client");
+    ok (flux_pty_client_notify_exit (c, NULL, NULL) < 0 && errno == EINVAL,
+        "flux_pty_client_notify_exit() returns EINVAL on NULL args");
+
+    ok (flux_pty_client_write (NULL, NULL, 0) == NULL && errno == EINVAL,
+        "flux_pty_client_write() returns EINVAL on NULL args");
+
+    ok (flux_pty_client_set_flags (NULL, 0) < 0 && errno == EINVAL,
+        "flux_pty_client_set_flags returns EINVAL on NULL arg");
+    ok (flux_pty_client_set_flags (c, -1) < 0 && errno == EINVAL,
+        "flux_pty_client_set_flags returns EINVAL on bad flags");
+
+    ok (flux_pty_client_get_flags (NULL) < 0 && errno == EINVAL,
+        "flux_pty_client_gett_flags returns EINVAL on NULL arg");
+
+    ok (flux_pty_client_exit_status (NULL, NULL) < 0 && errno == EINVAL,
+        "flux_pty_client_get_exit_status returns EINVAL on NULL args");
+    ok (flux_pty_client_exit_status (c, NULL) < 0 && errno == EINVAL,
+        "flux_pty_client_get_exit_status returns EINVAL on NULL statusp");
+
+    lives_ok ({flux_pty_monitor (NULL, NULL);},
+             "flux_pty_monitor can safely be called with NULL");
+
+    ok (flux_pty_aux_set (NULL, NULL, NULL, NULL) < 0 && errno == EINVAL,
+        "flux_pty_aux_set (NULL) fails");
+    ok (flux_pty_aux_get (NULL, NULL) == NULL && errno == EINVAL,
+        "flux_pty_aux_get (NULL, NULL) fails");
+
+    flux_pty_destroy (pty);
+    flux_pty_client_destroy (c);
+}
+
+static void test_empty_server ()
+{
+    struct flux_pty *pty = flux_pty_open ();
+
+    ok (pty != NULL,
+        "flux_pty_open works");
+    ok (flux_pty_leader_fd (pty) >= 0,
+        "pty leader fd is valid");
+    ok (flux_pty_client_count (pty) == 0,
+        "pty client count is 0 for newly created pty server");
+    flux_pty_destroy (pty);
+}
+
+static void tap_logger (void *arg,
+                        const char *file,
+                        int line,
+                        const char *func,
+                        const char *subsys,
+                        int level,
+                        const char *fmt,
+                        va_list ap)
+{
+    struct flux_pty *pty = arg;
+    char buf [4096];
+    int len = sizeof (buf);
+    if (vsnprintf (buf, len, fmt, ap) >= len) {
+        buf[len-1] = '\0';
+        buf[len-2] = '+';
+    }
+    diag ("pty: %s: %s:%d %s(): %s",
+          flux_pty_name (pty), file, line, func, buf);
+}
+
+static void pty_server_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct flux_pty *pty = arg;
+    if (flux_pty_sendmsg (pty, msg) < 0)
+        fail ("flux_pty_sendmsg returned -1: %s", strerror (errno));
+}
+
+static int pty_server (flux_t *h, void *arg)
+{
+    int rc = -1;
+    struct flux_pty * pty = NULL;
+    flux_msg_handler_t *mh = NULL;
+    struct flux_match match = FLUX_MATCH_REQUEST;
+
+    pty = flux_pty_open ();
+    if (!pty)
+        goto out;
+
+    flux_pty_set_log (pty, tap_logger, pty);
+    flux_pty_set_flux (pty, h);
+    diag ("pty_server: opened %s", flux_pty_name (pty));
+
+    mh = flux_msg_handler_create (h, match, pty_server_cb, pty);
+    if (!mh)
+        goto out;
+    flux_msg_handler_start (mh);
+
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    diag ("pty server exiting");
+out:
+    flux_msg_handler_destroy (mh);
+    flux_pty_destroy (pty);
+    return rc;
+}
+
+static void test_basic_protocol (void)
+{
+    flux_t *h = test_server_create (0, pty_server, NULL);
+    flux_future_t *f = NULL;
+    flux_future_t *f_attach = NULL;
+
+    /* invalid message, no msg type: */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0, "{}")) != NULL,
+        "request: empty payload");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == EPROTO,
+        "response: EPROTO");
+    flux_future_destroy (f);
+
+
+    /* attach without terminal size: */
+    ok ((f = flux_rpc_pack (h, "pty", 0, FLUX_RPC_STREAMING,
+                            "{ s:s s:s }",
+                            "type", "attach",
+                            "mode", "rw")) != NULL,
+        "request: type attach, no winsize");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == EPROTO,
+        "response: EPROTO");
+    flux_future_destroy (f);
+
+
+    /* attach without mode: */
+    ok ((f = flux_rpc_pack (h, "pty", 0, FLUX_RPC_STREAMING,
+                            "{ s:s s:{s:i s:i} }",
+                            "type", "attach",
+                            "winsize",
+                               "rows", 25,
+                               "cols", 80)) != NULL,
+        "request: type attach, no mode");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == EPROTO,
+        "response: EPROTO");
+    flux_future_destroy (f);
+
+
+    /* attach: invalid mode: */
+    ok ((f = flux_rpc_pack (h, "pty", 0, FLUX_RPC_STREAMING,
+                            "{ s:s s:s s:{s:i s:i} }",
+                            "type", "attach",
+                            "mode", "x",
+                            "winsize",
+                               "rows", 25,
+                               "cols", 80)) != NULL,
+        "request: type attach, bad mode");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == EPROTO,
+        "response: EPROTO");
+    flux_future_destroy (f);
+
+    /* write from unattached client: */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0,
+                            "{ s:s s:s }",
+                            "type", "data",
+                            "data", "\r")) != NULL,
+        "request: type data, unconnected client");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == ENOENT,
+        "response: ENOENT");
+    flux_future_destroy (f);
+
+    /* resize from unattached client: */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0,
+                            "{s:s s:{s:i s:i}}",
+                            "type", "resize",
+                            "winsize",
+                               "rows", 25,
+                               "cols", 80)) != NULL,
+        "request: type resize, unconnected client");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == ENOENT,
+        "response: ENOENT");
+    flux_future_destroy (f);
+
+
+    /* detach from unattached client: */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0,
+                            "{s:s}", "type", "detach")) != NULL,
+        "request: type detach, unconnected client");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == ENOENT,
+        "response: ENOENT");
+    flux_future_destroy (f);
+
+
+    /* attach a client: */
+    ok ((f_attach = flux_rpc_pack (h, "pty", 0, FLUX_RPC_STREAMING,
+                                   "{s:s s:s s:{s:i s:i}}",
+                                   "type", "attach",
+                                   "mode", "rw",
+                                   "winsize",
+                                      "rows", 25,
+                                      "cols", 80)) != NULL,
+        "request: type attach");
+
+    const char *type = NULL;
+    ok (flux_rpc_get_unpack (f_attach, "{s:s}", "type", &type) == 0,
+        "response: OK errno=%s", strerror (errno));
+    is (type, "attach",
+        "response: type=attach");
+    flux_future_reset (f_attach);
+
+
+    /* attach client again should fail: */
+    ok ((f = flux_rpc_pack (h, "pty", 0, FLUX_RPC_STREAMING,
+                                   "{s:s s:s s:{s:i s:i}}",
+                                   "type", "attach",
+                                   "mode", "rw",
+                                   "winsize",
+                                      "rows", 25,
+                                      "cols", 80)) != NULL,
+        "request: type attach from same client");
+
+    ok (flux_rpc_get (f, NULL) < 0 && errno == EEXIST,
+        "response: EEXIST");
+    flux_future_destroy (f);
+
+
+   /* resize from attached client, invalid size */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0,
+                            "{s:s s:{s:i s:i}}",
+                            "type", "resize",
+                            "winsize",
+                               "rows", 0,
+                               "cols", 0)) != NULL,
+        "request: type resize, invalid winsize {0, 0}");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == EINVAL,
+        "response: EINVAL");
+    flux_future_destroy (f);
+
+
+   /* resize from attached client, valid size */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0,
+                            "{s:s s:{s:i s:i}}",
+                            "type", "resize",
+                            "winsize",
+                               "rows", 25,
+                               "cols", 80)) != NULL,
+        "request: type resize, valid winsize {25, 80}");
+    ok (flux_rpc_get (f, NULL) == 0,
+        "response: OK");
+    flux_future_destroy (f);
+
+
+    /* write from attached client, invalid message */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0,
+                            "{s:s s:s}",
+                            "type", "data",
+                            "foo", "")) != NULL,
+        "request: type data, invalid payload");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == EPROTO,
+        "response: EPROTO");
+    flux_future_destroy (f);
+
+
+    /* write from attached client, invalid data */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0,
+                            "{s:s s:i}",
+                            "type", "data",
+                            "data", 2)) != NULL,
+        "request: type data, invalid data type");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == EPROTO,
+        "response: EPROTO");
+    flux_future_destroy (f);
+
+
+    /* invalid msg type from attached client */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0,
+                            "{s:s}",
+                            "type", "foo")) != NULL,
+        "request: type invalid");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == ENOSYS,
+        "response: ENOSYS");
+    flux_future_destroy (f);
+
+
+    /* invalid msg type from attached client */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0,
+                            "{s:s}",
+                            "type", "foo")) != NULL,
+        "request: type invalid");
+    ok (flux_rpc_get (f, NULL) < 0 && errno == ENOSYS,
+        "response: ENOSYS");
+    flux_future_destroy (f);
+
+
+    /* detach client */
+    ok ((f = flux_rpc_pack (h, "pty", 0, 0,
+                            "{s:s}",
+                            "type", "detach")) != NULL,
+        "request: type invalid");
+    ok (flux_rpc_get (f, NULL) == 0,
+        "response: OK");
+    flux_future_destroy (f);
+
+    const char *message = NULL;
+    ok (flux_rpc_get_unpack (f_attach,
+                             "{s:s s:s}",
+                             "type", &type,
+                             "message", &message) == 0,
+        "response to attach multi-response rpc");
+    is (type, "exit",
+        "response: type = exit");
+    is (message, "Client requested detach",
+        "response: message = 'Client requested detach'");
+    flux_future_reset (f_attach);
+
+    ok (flux_rpc_get (f_attach, NULL) < 0 && errno == ENODATA,
+        "response: ENODATA");
+    flux_future_destroy (f_attach);
+
+    test_server_stop (h);
+    flux_close (h);
+}
+
+static void pty_exit_cb (struct flux_pty_client *c, void *arg)
+{
+    flux_t *h = arg;
+    flux_reactor_stop (flux_get_reactor (h));
+}
+
+static void test_client (void)
+{
+    flux_t *h = test_server_create (0, pty_server, NULL);
+    flux_future_t *f = NULL;
+    int rc;
+    int flags = FLUX_PTY_CLIENT_ATTACH_SYNC
+                | FLUX_PTY_CLIENT_NORAW;
+
+    struct flux_pty_client *c = flux_pty_client_create ();
+    if (!c)
+        BAIL_OUT ("flux_pty_client_create failed");
+
+    ok (flux_pty_client_get_flags (c) == 0,
+        "initial pty client flags are 0");
+    ok (flux_pty_client_set_flags (c, -1) < 0 && errno == EINVAL,
+        "flux_pty_client_set_flags with invalid flags returns EINVAL");
+    ok (flux_pty_client_set_flags (c, flags) == 0,
+        "set client flags");
+
+    ok (flux_pty_client_notify_exit (c, pty_exit_cb, h) == 0,
+        "flux_pty_client_notify_exit");
+
+    ok (flux_pty_client_attached (c) == false,
+        "flux_pty_client_attached is false");
+
+    ok (flux_pty_client_attach (c, h, 0, "pty") == 0,
+        "flux_pty_client_attach");
+
+    ok (flux_pty_client_attached (c),
+        "flux_pty_client_attached is true after synchronous attach");
+
+    f = flux_pty_client_write (c, "foo\r", 4);
+    ok (f != NULL,
+        "flux_pty_client_write");
+    rc = flux_future_get (f, NULL);
+    ok (rc == 0,
+        "flux_pty_client_write: %s",
+        rc == 0 ? "Success" : strerror (errno));
+    flux_future_destroy (f);
+
+    f = flux_pty_client_write (c, "bar\0\r\n", 6);
+    ok (f != NULL,
+        "flux_pty_client_write with U+0000");
+    ok (rc == 0,
+        "flux_pty_client_write: %s",
+        rc == 0 ? "Success" : strerror (errno));
+    flux_future_destroy (f);
+
+    ok (flux_pty_client_detach (c) == 0,
+        "flux_pty_client_detach");
+
+    /* Run reactor until pty client exits */
+    flux_reactor_run (flux_get_reactor (h), 0);
+
+    test_server_stop (h);
+    flux_pty_client_destroy (c);
+    flux_close (h);
+}
+
+static void monitor_cb (struct flux_pty *pty, void *data, int len)
+{
+    int *totalp = flux_pty_aux_get (pty, "total");
+    *totalp += len;
+    diag ("monitor_cb got %ld bytes", len);
+}
+
+void test_monitor ()
+{
+    int total = 0;
+    flux_t *h = flux_open ("loop://", 0);
+    struct flux_pty *pty = flux_pty_open ();
+
+    if (!h || !pty)
+        BAIL_OUT ("Unable to create test handle and pty");
+
+    ok (flux_pty_set_flux (pty, h) == 0,
+        "flux_pty_set_flux");
+
+    ok (flux_pty_aux_set (pty, "total", &total, NULL) == 0,
+        "flux_pty_aux_set");
+
+    diag ("starting pty monitor");
+    flux_pty_monitor (pty, monitor_cb);
+
+    pty_client_send_data (pty, "hello", 6);
+    pty_client_send_data (pty, "world", 6);
+
+    ok (total == 12,
+        "monitor received 12 bytes");
+
+    flux_pty_destroy (pty);
+    flux_close (h);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_invalid_args ();
+    test_empty_server ();
+    test_basic_protocol ();
+    test_client ();
+    test_monitor ();
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libterminus/test/terminus.c b/src/common/libterminus/test/terminus.c
new file mode 100644
index 000000000000..e2d5245bd2e8
--- /dev/null
+++ b/src/common/libterminus/test/terminus.c
@@ -0,0 +1,489 @@
+/************************************************************  \
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include "src/common/libterminus/terminus.h"
+#include "src/common/libterminus/pty.h"
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libtestutil/util.h"
+
+static void test_invalid_args (void)
+{
+    ok (flux_terminus_server_create (NULL, NULL) == NULL && errno == EINVAL,
+        "flux_terminus_server_create with NULL args returns EINVAL");
+
+    lives_ok ({flux_terminus_server_destroy (NULL);},
+        "flux_terminus_server_destroy (NULL) does nothing");
+    lives_ok ({flux_terminus_server_set_log (NULL, NULL, NULL);},
+        "flux_terminus_server_set_log (NULL) does nothing");
+
+    ok (flux_terminus_server_notify_empty (NULL, NULL, NULL) < 0
+        && errno == EINVAL,
+        "flux_terminus_server_notify_empty returns EINVAL on NULL args");
+
+    ok (flux_terminus_server_session_open (NULL, 0, NULL) == NULL,
+        "flux_terminus_server_session_open with NULL args returns NULL");
+    ok (flux_terminus_server_session_close (NULL, NULL, 0) == -1
+        && errno == EINVAL,
+        "flux_terminus_server_session_close with NULL args returns EINVAL");
+}
+
+static void tap_logger (void *arg,
+                        const char *file,
+                        int line,
+                        const char *func,
+                        const char *subsys,
+                        int level,
+                        const char *fmt,
+                        va_list ap)
+{
+    char buf [4096];
+    int len = sizeof (buf);
+    if (vsnprintf (buf, len, fmt, ap) >= len) {
+        buf[len-1] = '\0';
+        buf[len-2] = '+';
+    }
+    diag ("%s:%d %s(): %s", file, line, func, buf);
+}
+
+static int terminus_server (flux_t *h, void *arg)
+{
+    int rc = -1;
+    struct flux_terminus_server *t;
+
+    /*  N.B.: test_server handle `h` already has reactor with SIGCHLD
+     *   flag set.
+     */
+    t = flux_terminus_server_create (h, "terminus");
+    if (!t)
+        BAIL_OUT ("flux_terminus_server_create");
+
+    flux_terminus_server_set_log (t, tap_logger, NULL);
+
+    rc = flux_reactor_run (flux_get_reactor (h), 0);
+    flux_terminus_server_destroy (t);
+    //flux_close (h);
+    return rc;
+}
+
+static void test_kill_server_empty (void)
+{
+    int rc;
+    flux_future_t *f = NULL;
+    flux_t *h = test_server_create (0, terminus_server, NULL);
+
+    /* kill-server
+     */
+    f = flux_rpc_pack (h, "terminus.kill-server", 0, 0, "{}");
+    ok (f != NULL,
+        "terminus.kill-server");
+    rc = flux_rpc_get (f, NULL);
+    ok (rc == 0,
+        "terminus.kill-server: OK");
+    flux_future_destroy (f);
+
+    /* list, now fails
+     */
+    f = flux_rpc_pack (h, "terminus.list", 0, 0, "{}");
+    ok (f != NULL,
+        "terminus.list");
+    rc = flux_rpc_get (f, NULL);
+    ok (rc < 0 && errno == ENOSYS,
+        "terminus.list: ENOSYS");
+    flux_future_destroy (f);
+
+    test_server_stop (h);
+    flux_close (h);
+}
+
+static void test_protocol (void)
+{
+    int rc;
+    json_t *o = NULL;
+    flux_future_t *f = NULL;
+    flux_t *h = test_server_create (0, terminus_server, NULL);
+
+    const char *service = NULL;
+    const char *name = NULL;
+    const char *message = NULL;
+    const char *type = NULL;
+    int rank = -1;
+    int status = -1;
+    int id;
+    double ctime;
+
+    /* list, no sessions
+     */
+    f = flux_rpc_pack (h, "terminus.list", 0, 0, "{}");
+    ok (f != NULL,
+        "terminus.list");
+    rc = flux_rpc_get_unpack (f,
+                              "{s:{s:s s:i s:f} s:[]}",
+                              "server",
+                                 "service", &service,
+                                 "rank", &rank,
+                                 "ctime", &ctime,
+                              "sessions");
+    ok (rc == 0,
+        "terminus.list: OK");
+    is (service, "terminus",
+        "terminus.list returned service = terminus");
+    ok (rank == -1,
+        "terminus.list returned expected rank");
+    flux_future_destroy (f);
+
+
+    /* new, add a session, invalid proto
+     */
+    f = flux_rpc_pack (h, "terminus.new",
+                       0, 0,
+                      "{s:s}",
+                      "cmd", "/bin/bash");
+    ok (f != NULL,
+        "terminus.new: invalid proto");
+    errno = 0;
+    rc = flux_rpc_get_unpack (f, NULL);
+    ok (rc < 0 && errno == EPROTO,
+        "terminus.new (invalid proto): %s", strerror (errno));
+    flux_future_destroy (f);
+
+
+    /* new, add a session, no args
+     */
+    f = flux_rpc_pack (h, "terminus.new", 0, 0, "{}");
+    ok (f != NULL,
+        "terminus.new: no args");
+    errno = 0;
+    name = NULL;
+    service = NULL;
+    rc = flux_rpc_get_unpack (f,
+                              "{s:s s:s s:i}",
+                              "name", &name,
+                              "pty_service", &service,
+                              "id", &id);
+    ok (rc == 0,
+        "terminus.new (no args): %s", strerror (errno));
+    is (name, getenv ("SHELL"),
+        "terminus.new (no args): name is %s", getenv ("SHELL"));
+    ok (id == 0,
+        "terminus.new (no args): id is 0");
+    is (service, "terminus.0",
+        "terminus.new (no args): service is terminus.0");
+    flux_future_destroy (f);
+
+
+    /* new, add a session, full args
+     */
+    errno = 0;
+    f = flux_rpc_pack (h,
+                       "terminus.new",
+                        0, 0,
+                        "{s:s s:[ss] s:{s:s s:s}}",
+                        "name", "test-name",
+                        "cmd", "sleep", "1000",
+                        "environ",
+                           "PATH", "/bin:/usr/bin",
+                           "HOME", "/home/user1");
+    ok (f != NULL,
+        "terminus.new: full args");
+    service = NULL;
+    name = NULL;
+    errno = 0;
+    rc = flux_rpc_get_unpack (f,
+                              "{s:s s:s s:i}",
+                              "name", &name,
+                              "pty_service", &service,
+                              "id", &id);
+    ok (rc == 0,
+        "terminus.new (full args): %s", future_strerror (f, errno));
+    is (name, "test-name",
+        "terminus.new (full args): name is %s", "test-name");
+    ok (id == 1,
+        "terminus.new (full args): id is 1");
+    is (service, "terminus.1",
+        "terminus.new (full args): service is terminus.1");
+    flux_future_destroy (f);
+
+
+    /* new, add a session, cmd only
+     */
+    f = flux_rpc_pack (h,
+                       "terminus.new",
+                        0, 0,
+                        "{s:[ss]}",
+                        "cmd", "sleep", "1000");
+    ok (f != NULL,
+        "terminus.new: cmd only");
+    rc = flux_rpc_get_unpack (f,
+                              "{s:s s:s s:i}",
+                              "name", &name,
+                              "pty_service", &service,
+                              "id", &id);
+    ok (rc == 0,
+        "terminus.new (cmd only): OK");
+    is (name, "sleep",
+        "terminus.new (cmd only): name is %s", "sleep");
+    ok (id == 2,
+        "terminus.new (cmd only): id is 2");
+    is (service, "terminus.2",
+        "terminus.new (cmd only): service is terminus.2");
+    flux_future_destroy (f);
+
+
+    /* list, 3 sessions
+     */
+    f = flux_rpc_pack (h, "terminus.list", 0, 0, "{}");
+    ok (f != NULL,
+        "terminus.list");
+    service = NULL;
+    name = NULL;
+    errno = 0;
+    rc = flux_rpc_get_unpack (f,
+                              "{s:{s:s s:i s:f} s:o}",
+                              "server",
+                                 "service", &service,
+                                 "rank", &rank,
+                                 "ctime", &ctime,
+                              "sessions", &o);
+    ok (rc == 0,
+        "terminus.list: OK");
+    is (service, "terminus",
+        "terminus.list returned service = terminus");
+    ok (rank == -1,
+        "terminus.list returned expected rank");
+    ok (o != NULL && json_is_array (o),
+        "terminus.list returned sessions list");
+    ok (json_array_size (o) == 3,
+        "terminus.list returned 3 sessions");
+    flux_future_destroy (f);
+
+
+    /* kill session
+     */
+    f = flux_rpc_pack (h,
+                       "terminus.kill",
+                       0, FLUX_RPC_STREAMING,
+                       "{s:i s:i s:i}",
+                       "id", 0,
+                       "signal", SIGKILL,
+                       "wait", 1);
+    ok (f != NULL,
+        "terminus.kill (wait)");
+    errno = 0;
+    rc = flux_rpc_get_unpack (f,
+                              "{s:s s:s s:i}",
+                              "type", &type,
+                              "message", &message,
+                              "status", &status);
+    ok (rc == 0,
+        "terminus.kill (wait): OK");
+    is (type, "exit",
+        "terminus.kill (wait): response is of type exit");
+    ok (status == 0x9,
+        "terminus.kill (wait): status == 0x9");
+    flux_future_reset (f);
+    rc = flux_rpc_get (f, NULL);
+    ok (rc < 0 && errno == ENODATA,
+        "terminus.kill (wait): ENODATA (end of streaming response)");
+    flux_future_destroy (f);
+
+
+    /* kill: invalid session
+     */
+    f = flux_rpc_pack (h,
+                       "terminus.kill",
+                       0, 0,
+                       "{s:i s:i}",
+                       "id", 0,
+                       "signal", SIGKILL);
+    ok (f != NULL,
+        "terminus.kill (invalid session)");
+    errno = 0;
+    rc = flux_rpc_get (f, NULL);
+    ok (rc == -1 && errno == ENOENT,
+        "terminus.kill: ENOENT (got %s)", strerror (errno));
+    flux_future_destroy (f);
+
+
+    /* list, 2 sessions
+     */
+    f = flux_rpc_pack (h, "terminus.list", 0, 0, "{}");
+    ok (f != NULL,
+        "terminus.list");
+    service = NULL;
+    name = NULL;
+    errno = 0;
+    o = NULL;
+    rc = flux_rpc_get_unpack (f,
+                              "{s:{s:s s:i s:f} s:o !}",
+                              "server",
+                                 "service", &service,
+                                 "rank", &rank,
+                                 "ctime", &ctime,
+                              "sessions", &o);
+    ok (rc == 0,
+        "terminus.list: OK");
+    ok (o && json_is_array (o) && json_array_size (o) == 2,
+        "terminus.list: now returns 2 sessions");
+    flux_future_destroy (f);
+
+
+    /* kill session (no wait)
+     */
+    f = flux_rpc_pack (h,
+                       "terminus.kill",
+                       0, 0,
+                       "{s:i s:i}",
+                       "id", 1,
+                       "signal", SIGKILL);
+    ok (f != NULL,
+        "terminus.kill (no wait)");
+    errno = 0;
+    rc = flux_rpc_get (f, NULL);
+    ok (rc == 0,
+        "terminus.kill (no wait): OK");
+    flux_future_destroy (f);
+
+
+    /* kill-server
+     */
+    f = flux_rpc_pack (h, "terminus.kill-server", 0, 0, "{}");
+    ok (f != NULL,
+        "terminus.kill-server");
+    errno = 0;
+    rc = flux_rpc_get (f, NULL);
+    ok (rc == 0,
+        "terminus.kill-server: OK (%s)",
+        rc == 0 ? "Success": strerror (errno));
+    flux_future_destroy (f);
+
+
+    /* list, now fails
+     */
+    f = flux_rpc_pack (h, "terminus.list", 0, 0, "{}");
+    ok (f != NULL,
+        "terminus.list");
+    rc = flux_rpc_get (f, NULL);
+    ok (rc < 0 && errno == ENOSYS,
+        "terminus.list: ENOSYS");
+    flux_future_destroy (f);
+
+    test_server_stop (h);
+    flux_close (h);
+}
+
+void test_open_close_session (void)
+{
+    int rc;
+    struct flux_pty *pty = NULL;
+    struct flux_pty *pty0 = NULL;
+    struct flux_pty *pty1 = NULL;
+    struct flux_terminus_server *t = NULL;
+    struct flux_terminus_server *t2 = NULL;
+    flux_t *h = flux_open ("loop://", 0);
+    flux_reactor_t *r = flux_reactor_create (FLUX_REACTOR_SIGCHLD);
+
+    if (!h || !r)
+        BAIL_OUT ("Failed to create loopback handle/reactor");
+    flux_set_reactor (h, r);
+    flux_aux_set (h, NULL, r, (flux_free_f) flux_reactor_destroy);
+
+    t = flux_terminus_server_create (h, "terminus");
+    ok (t != NULL,
+        "flux_terminus_server_create()");
+    t2 = flux_terminus_server_create (h, "terminus2");
+    ok (t2 != NULL,
+        "flux_terminus_server_create()");
+
+    flux_terminus_server_set_log (t, tap_logger, NULL);
+    flux_terminus_server_set_log (t2, tap_logger, NULL);
+
+    pty = flux_terminus_server_session_open (t, -1, "test session");
+    ok (pty == NULL && errno == EINVAL,
+        "flux_terminus_server_session_open with invalid id returns EINVAL");
+    pty = flux_terminus_server_session_open (t, 0, NULL);
+    ok (pty == NULL && errno == EINVAL,
+        "flux_terminus_server_session_open with NULL name returns EINVAL");
+
+    pty0 = flux_terminus_server_session_open (t, 0, "test session");
+    ok (pty0 != NULL,
+        "flux_terminus_server_session_open works");
+
+    pty1 = flux_terminus_server_session_open (t, 1, "another test session");
+    ok (pty1 != NULL,
+        "flux_terminus_server_session_open again works");
+    rc = flux_terminus_server_session_close (t, pty1, 0);
+    ok (pty1 != NULL,
+        "flux_terminus_server_close");
+
+    pty = flux_terminus_server_session_open (t, 0, "duplicate");
+    ok (pty == NULL && errno == EEXIST,
+        "flux_terminus_server_session_open with duplicate id returns EEXIST");
+
+    rc = flux_terminus_server_session_close (t, NULL, 0);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_terminus_session_close with NULL pty returns EINVAL");
+
+    rc = flux_terminus_server_session_close (t, pty0, -1);
+    ok (rc < 0 && errno == EINVAL,
+        "flux_terminus_session_close with invalid status returns EINVAL");
+
+    rc = flux_terminus_server_session_close (t, pty0, 0);
+    ok (rc == 0,
+        "flux_terminus_session_close works");
+
+    pty0 = flux_terminus_server_session_open (t2, 0, "session0");
+    ok (pty0 != NULL,
+        "flux_terminus_server_session_open on second server");
+    rc = flux_terminus_server_session_close (t, pty0, 0);
+    ok (rc < 0 && errno == ENOENT,
+        "flux_terminus_server_session_close wrong server returns ENOENT");
+    rc = flux_terminus_server_session_close (t2, pty0, 0);
+    ok (rc == 0,
+        "flux_terminus_server_session_close right server works");
+
+    flux_terminus_server_destroy (t);
+    flux_terminus_server_destroy (t2);
+    flux_close (h);
+}
+
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+    /* Make sure SHELL is set in environment.
+     */
+    if (getenv ("SHELL") == NULL)
+        setenv ("SHELL", "/bin/sh", 1);
+
+    /*  set rank == -1 for testing. Avoid flux_get_rank(3) */
+    setenv ("FLUX_TERMINUS_TEST_SERVER", "t", 1);
+
+    test_invalid_args ();
+    test_kill_server_empty ();
+    test_protocol ();
+    test_open_close_session ();
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libtestutil/Makefile.am b/src/common/libtestutil/Makefile.am
index b4c82ae01825..b722c82c5b6c 100644
--- a/src/common/libtestutil/Makefile.am
+++ b/src/common/libtestutil/Makefile.am
@@ -6,10 +6,11 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) \
+	-I$(top_srcdir)/src/common/libccan \
 	$(LIBUUID_CFLAGS)
 
 check_LTLIBRARIES = libtestutil.la
diff --git a/src/common/libtestutil/util.c b/src/common/libtestutil/util.c
index 932d6dbdcb77..0c130972c754 100644
--- a/src/common/libtestutil/util.c
+++ b/src/common/libtestutil/util.c
@@ -12,20 +12,18 @@
 #include "config.h"
 #endif
 #include <flux/core.h>
-#include <czmq.h>
 #include <uuid.h>
 #include <pthread.h>
 
-#include "util.h"
-
+#include "ccan/str/str.h"
 #include "src/common/libtap/tap.h"
-#include "src/common/libutil/msglist.h"
+
+#include "util.h"
 
 #ifndef UUID_STR_LEN
 #define UUID_STR_LEN 37     // defined in later libuuid headers
 #endif
 
-
 struct test_server {
     flux_t *c;
     flux_t *s;
@@ -39,10 +37,6 @@ struct test_server {
     char uuid_str[UUID_STR_LEN];
 };
 
-static flux_t *test_connector_create (const char *shmem_name,
-                                      bool server, int flags);
-
-
 void shutdown_cb (flux_t *h, flux_msg_handler_t *mh,
                   const flux_msg_t *msg, void *arg)
 {
@@ -89,7 +83,7 @@ static void diag_cb (flux_t *h, flux_msg_handler_t *mh,
 
     if (flux_msg_get_type (msg, &msgtype) < 0)
         goto badmsg;
-    if (msgtype != FLUX_MSGTYPE_KEEPALIVE) {
+    if (msgtype != FLUX_MSGTYPE_CONTROL) {
         if (flux_msg_get_topic (msg, &topic) < 0)
             goto badmsg;
     }
@@ -121,12 +115,13 @@ static void test_server_destroy (struct test_server *a)
     }
 }
 
-flux_t *test_server_create (test_server_f cb, void *arg)
+flux_t *test_server_create (int cflags, test_server_f cb, void *arg)
 {
     int e;
     struct test_server *a;
-    int cflags = 0; // client connector flags
     int sflags = 0; // server connector flags
+    char uri[64];
+    flux_reactor_t *r;
 
     if (!(a = calloc (1, sizeof (*a))))
         BAIL_OUT ("calloc");
@@ -139,12 +134,19 @@ flux_t *test_server_create (test_server_f cb, void *arg)
     if (getenv ("FLUX_HANDLE_TRACE"))
         cflags |= FLUX_O_TRACE;
 
-    /* Create back-to-back wired flux_t handles
+
+    /* Create back-to-back wired flux_t handles.
+     * Give the server side a SIGCHLD capable reactor for subprocess testing.
      */
-    if (!(a->s = test_connector_create (a->uuid_str, true, sflags)))
-        BAIL_OUT ("test_connector_create server");
-    if (!(a->c = test_connector_create (a->uuid_str, false, cflags)))
-        BAIL_OUT ("test_connector_create client");
+    snprintf (uri, sizeof (uri), "interthread://%s", a->uuid_str);
+    if (!(a->s = flux_open (uri, sflags))
+        || !(r = flux_reactor_create (FLUX_REACTOR_SIGCHLD))
+        || flux_set_reactor (a->s, r) < 0
+        || flux_aux_set (a->s, NULL, r, (flux_free_f)flux_reactor_destroy) < 0
+        || flux_opt_set (a->s, FLUX_OPT_ROUTER_NAME, "server", 7) < 0)
+        BAIL_OUT ("could not create server interthread handle");
+    if (!(a->c = flux_open (uri, cflags)))
+        BAIL_OUT ("could not create client interthread handle");
 
     /* If no callback, register watcher for all messages so we can log them.
      * N.B. this has to go in before shutdown else shutdown will be masked.
@@ -176,291 +178,6 @@ flux_t *test_server_create (test_server_f cb, void *arg)
     return a->c;
 }
 
-void test_server_environment_init (const char *test_name)
-{
-    zsys_init ();
-    zsys_set_logstream (stderr);
-    zsys_set_logident (test_name);
-    zsys_handler_set (NULL);
-    zsys_set_linger (5); // msec
-}
-
-/* Test connector implementation
- */
-
-struct test_connector {
-    zsock_t *sock;
-    flux_t *h;
-    struct flux_msg_cred cred;
-};
-
-static int test_connector_pollevents (void *impl)
-{
-    struct test_connector *tcon = impl;
-    int e = zsock_events (tcon->sock);
-    int revents = 0;
-
-    if (e & ZMQ_POLLIN)
-        revents |= FLUX_POLLIN;
-    if (e & ZMQ_POLLOUT)
-        revents |= FLUX_POLLOUT;
-    if (e & ZMQ_POLLERR)
-        revents |= FLUX_POLLERR;
-
-    return revents;
-}
-
-static int test_connector_pollfd (void *impl)
-{
-    struct test_connector *tcon = impl;
-
-    return zsock_fd (tcon->sock);
-}
-
-static int test_connector_send (void *impl, const flux_msg_t *msg, int flags)
-{
-    struct test_connector *tcon = impl;
-    flux_msg_t *cpy;
-
-    if (!(cpy = flux_msg_copy (msg, true)))
-        return -1;
-    if (flux_msg_set_cred (cpy, tcon->cred) < 0)
-        goto error;
-    if (flux_msg_sendzsock (tcon->sock, cpy) < 0)
-        goto error;
-    flux_msg_destroy (cpy);
-    return 0;
-error:
-    flux_msg_destroy (cpy);
-    return -1;
-}
-
-static flux_msg_t *test_connector_recv (void *impl, int flags)
-{
-    struct test_connector *tcon = impl;
-
-    if ((flags & FLUX_O_NONBLOCK)) {
-        zmq_pollitem_t zp = {
-            .events = ZMQ_POLLIN,
-            .socket = zsock_resolve (tcon->sock),
-            .revents = 0,
-            .fd = -1,
-        };
-        int n;
-        if ((n = zmq_poll (&zp, 1, 0L)) <= 0) {
-            if (n == 0)
-                errno = EWOULDBLOCK;
-            return NULL;
-        }
-    }
-    return flux_msg_recvzsock (tcon->sock);
-}
-
-static void test_connector_fini (void *impl)
-{
-    struct test_connector *tcon = impl;
-
-    zsock_destroy (&tcon->sock);
-    free (tcon);
-}
-
-static const struct flux_handle_ops handle_ops = {
-    .pollfd = test_connector_pollfd,
-    .pollevents = test_connector_pollevents,
-    .send = test_connector_send,
-    .recv = test_connector_recv,
-    .getopt = NULL,
-    .setopt = NULL,
-    .event_subscribe = NULL,
-    .event_unsubscribe = NULL,
-    .impl_destroy = test_connector_fini,
-};
-
-static flux_t *test_connector_create (const char *shmem_name,
-                                      bool server, int flags)
-{
-    struct test_connector *tcon;
-
-    if (!(tcon = calloc (1, sizeof (*tcon))))
-        BAIL_OUT ("calloc");
-    tcon->cred.userid = geteuid ();
-    tcon->cred.rolemask = FLUX_ROLE_OWNER;
-    if (!(tcon->sock = zsock_new_pair (NULL)))
-        BAIL_OUT ("zsock_new_pair");
-    zsock_set_unbounded (tcon->sock);
-    if (server) {
-        if (zsock_bind (tcon->sock, "inproc://%s", shmem_name) < 0)
-            BAIL_OUT ("zsock_bind %s", shmem_name);
-    }
-    else {
-        if (zsock_connect (tcon->sock, "inproc://%s", shmem_name) < 0)
-            BAIL_OUT ("zsock_connect %s", shmem_name);
-    }
-    if (!(tcon->h = flux_handle_create (tcon, &handle_ops, flags)))
-        BAIL_OUT ("flux_handle_create");
-    return tcon->h;
-}
-
-/* Loopback connector implementation
- */
-
-struct loopback_connector {
-    msglist_t *queue;
-    flux_t *h;
-    int pollfd;
-    int pollevents;
-    struct flux_msg_cred cred;
-};
-
-static int loopback_connector_pollevents (void *impl)
-{
-    struct loopback_connector *lcon = impl;
-    int e;
-    int revents = 0;
-
-    if ((e = msglist_pollevents (lcon->queue)) < 0)
-        return -1;
-    if (e & POLLIN)
-        revents |= FLUX_POLLIN;
-    if (e & POLLOUT)
-        revents |= FLUX_POLLOUT;
-    if (e & POLLERR)
-        revents |= FLUX_POLLERR;
-
-    return revents;
-}
-
-static int loopback_connector_pollfd (void *impl)
-{
-    struct loopback_connector *lcon = impl;
-
-    return msglist_pollfd (lcon->queue);
-}
-
-static int loopback_connector_send (void *impl, const flux_msg_t *msg,
-                                    int flags)
-{
-    struct loopback_connector *lcon = impl;
-    struct flux_msg_cred cred;
-    flux_msg_t *cpy;
-
-    if (flux_msg_get_cred (msg, &cred) < 0)
-        return -1;
-    if (!(cpy = flux_msg_copy (msg, true)))
-        return -1;
-    if (cred.userid == FLUX_USERID_UNKNOWN) {
-        if (flux_msg_set_userid (cpy, lcon->cred.userid) < 0)
-            goto error;
-    }
-    if (cred.rolemask == FLUX_ROLE_NONE) {
-        if (flux_msg_set_rolemask (cpy, lcon->cred.rolemask) < 0)
-            goto error;
-    }
-    if (msglist_append (lcon->queue, cpy) < 0) // steals 'cpy'
-        goto error;
-    return 0;
-error:
-    flux_msg_destroy (cpy);
-    return -1;
-}
-
-static flux_msg_t *loopback_connector_recv (void *impl, int flags)
-{
-    struct loopback_connector *lcon = impl;
-    flux_msg_t *msg;
-
-    if (!(msg = msglist_pop (lcon->queue))) {
-        errno = EWOULDBLOCK;
-        return NULL;
-    }
-    return msg;
-}
-
-static int loopback_connector_getopt (void *impl, const char *option,
-                                      void *val, size_t size)
-{
-    struct loopback_connector *lcon = impl;
-
-    if (option && !strcmp (option, FLUX_OPT_TESTING_USERID)) {
-        if (size != sizeof (lcon->cred.userid))
-            goto error;
-        memcpy (val, &lcon->cred.userid, size);
-    }
-    else if (option && !strcmp (option, FLUX_OPT_TESTING_ROLEMASK)) {
-        if (size != sizeof (lcon->cred.rolemask))
-            goto error;
-        memcpy (val, &lcon->cred.rolemask, size);
-    }
-    else
-        goto error;
-    return 0;
-error:
-    errno = EINVAL;
-    return -1;
-}
-
-static int loopback_connector_setopt (void *impl, const char *option,
-                                      const void *val, size_t size)
-{
-    struct loopback_connector *lcon = impl;
-    size_t val_size;
-
-    if (option && !strcmp (option, FLUX_OPT_TESTING_USERID)) {
-        val_size = sizeof (lcon->cred.userid);
-        if (size != val_size)
-            goto error;
-        memcpy (&lcon->cred.userid, val, val_size);
-    }
-    else if (option && !strcmp (option, FLUX_OPT_TESTING_ROLEMASK)) {
-        val_size = sizeof (lcon->cred.rolemask);
-        if (size != val_size)
-            goto error;
-        memcpy (&lcon->cred.rolemask, val, val_size);
-    }
-    else
-        goto error;
-    return 0;
-error:
-    errno = EINVAL;
-    return -1;
-}
-
-static void loopback_connector_fini (void *impl)
-{
-    struct loopback_connector *lcon = impl;
-
-    msglist_destroy (lcon->queue);
-    free (lcon);
-}
-
-static const struct flux_handle_ops loopback_ops = {
-    .pollfd = loopback_connector_pollfd,
-    .pollevents = loopback_connector_pollevents,
-    .send = loopback_connector_send,
-    .recv = loopback_connector_recv,
-    .getopt = loopback_connector_getopt,
-    .setopt = loopback_connector_setopt,
-    .event_subscribe = NULL,
-    .event_unsubscribe = NULL,
-    .impl_destroy = loopback_connector_fini,
-};
-
-flux_t *loopback_create (int flags)
-{
-    struct loopback_connector *lcon;
-
-    if (!(lcon = calloc (1, sizeof (*lcon))))
-        BAIL_OUT ("calloc");
-    lcon->cred.userid = geteuid ();
-    lcon->cred.rolemask = FLUX_ROLE_OWNER;
-    if (!(lcon->queue = msglist_create ((msglist_free_f)flux_msg_destroy)))
-        BAIL_OUT ("msglist_create");
-
-    if (!(lcon->h = flux_handle_create (lcon, &loopback_ops, flags)))
-        BAIL_OUT ("flux_handle_create");
-    return lcon->h;
-}
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libtestutil/util.h b/src/common/libtestutil/util.h
index 528b8bd87f82..da767031a2f4 100644
--- a/src/common/libtestutil/util.h
+++ b/src/common/libtestutil/util.h
@@ -17,35 +17,15 @@
  *
  * Caveats:
  * 1) subscribe/unsubscribe requests are not supported
- * 2) all messages are sent with credentials userid=geteuid(), rolemask=OWNER
+ * 2) all messages are sent with credentials userid=getuid(), rolemask=OWNER
  * 3) broker attributes (such as rank and size) are unavailable
  * 4) message nodeid is ignored
  *
- * Unit tests that use the test server should call
- * test_server_environment_init() once prior to creating the first server
- * to initialize czmq's runtime.
- *
  * If callback is NULL, a default callback is run that logs each
  * message received with diag().
  */
 typedef int (*test_server_f)(flux_t *h, void *arg);
 
-flux_t *test_server_create (test_server_f cb, void *arg);
+flux_t *test_server_create (int flags, test_server_f cb, void *arg);
 
 int test_server_stop (flux_t *c);
-
-void test_server_environment_init (const char *test_name);
-
-
-/* Create a loopback connector for testing.
- * The net effect is much the same as flux_open("local://") except
- * the implementation is self contained here.  Close with flux_close().
- *
- * Like loop://, this support test manipulation of credentials:
- *   flux_opt_set (h, FLUX_OPT_TESTING_USERID, &userid, sizeof (userid);
- *   flux_opt_set (h, FLUX_OPT_TESTING_ROLEMASK, &rolemask, sizeof (rolemask))
- *
- * N.B. No need to call test_server_environment_init() if this is the
- * only component used from this module.
- */
-flux_t *loopback_create (int flags);
diff --git a/src/common/libtomlc99/Makefile.am b/src/common/libtomlc99/Makefile.am
index a95ff30e9faf..a589ad6cd0d3 100644
--- a/src/common/libtomlc99/Makefile.am
+++ b/src/common/libtomlc99/Makefile.am
@@ -7,7 +7,12 @@ AM_CFLAGS = \
 AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
-AM_CPPFLAGS =
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS)
+
+EXTRA_DIST = \
+	LICENSE \
+	README.md
 
 noinst_LTLIBRARIES = libtomlc99.la
 
@@ -33,7 +38,8 @@ test_cppflags = \
 
 test_ldadd = \
 	$(top_builddir)/src/common/libtomlc99/libtomlc99.la \
-	$(top_builddir)/src/common/libtap/libtap.la
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libutil/libutil.la
 
 toml_cat_SOURCES = toml_cat.c
 toml_cat_LDADD = $(test_ldadd)
@@ -47,7 +53,7 @@ test_toml_t_SOURCES = test/toml.c
 test_toml_t_LDADD = $(test_ldadd)
 test_toml_t_CPPFLAGS = $(test_cppflags)
 
-EXTRA_DIST = \
+EXTRA_DIST += \
 	BurntSushi_input/invalid/array-mixed-types-arrays-and-ints.toml \
 	BurntSushi_input/invalid/array-mixed-types-ints-and-floats.toml \
 	BurntSushi_input/invalid/array-mixed-types-strings-and-ints.toml \
diff --git a/src/common/libtomlc99/test/toml.c b/src/common/libtomlc99/test/toml.c
index 0a8a21f49c18..638138d4fc65 100644
--- a/src/common/libtomlc99/test/toml.c
+++ b/src/common/libtomlc99/test/toml.c
@@ -22,6 +22,7 @@
 #include <locale.h>
 
 #include "src/common/libtap/tap.h"
+#include "src/common/libutil/basename.h"
 #include "toml.h"
 
 #define EX1 "\
@@ -214,7 +215,7 @@ struct entry {
     char *reason;
 };
 
-const struct entry bad_input_blacklist[] = {
+const struct entry bad_input_blocklist[] = {
     { NULL, NULL },
 };
 
@@ -243,11 +244,11 @@ void parse_bad_input (void)
 
     for (i = 0; i < results.gl_pathc; i++) {
         char errbuf[255];
-        char *name = basename (results.gl_pathv[i]);
+        char *name = basename_simple (results.gl_pathv[i]);
         const char *reason;
-        bool blacklisted = matchtab (name, bad_input_blacklist, &reason);
+        bool blocklist = matchtab (name, bad_input_blocklist, &reason);
 
-        skip (blacklisted, 1, "%s: %s", name, reason);
+        skip (blocklist, 1, "%s: %s", name, reason);
         ok (parse_bad_file (results.gl_pathv[i], errbuf, 255) == true,
             "%s: %s", name, errbuf);
         end_skip;
@@ -299,7 +300,7 @@ void parse_good_input (void)
     for (i = 0; i < results.gl_pathc; i++) {
         char errbuf[255];
         ok (parse_good_file (results.gl_pathv[i], errbuf, 255) == true,
-            "%s: %s", basename (results.gl_pathv[i]), errbuf);
+            "%s: %s", basename_simple (results.gl_pathv[i]), errbuf);
     }
     globfree (&results);
 }
diff --git a/src/common/libutil/Makefile.am b/src/common/libutil/Makefile.am
index f34e8fc601ef..b9c993995258 100644
--- a/src/common/libutil/Makefile.am
+++ b/src/common/libutil/Makefile.am
@@ -1,8 +1,6 @@
 AM_CFLAGS = \
 	$(WARNING_CFLAGS) \
 	$(CODE_COVERAGE_CFLAGS) \
-	$(ZMQ_CFLAGS) \
-	$(LIBUUID_CFLAGS) \
 	-Wno-strict-aliasing -Wno-error=strict-aliasing \
 	-Wno-parentheses -Wno-error=parentheses
 
@@ -10,16 +8,21 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-	-DABS_TOP_BUILDDIR=\"$(abs_top_builddir)\"
+	-DABS_TOP_BUILDDIR=\"$(abs_top_builddir)\" \
+	$(JANSSON_CFLAGS)
 
 noinst_LTLIBRARIES = libutil.la
 
 libutil_la_SOURCES = \
 	ipaddr.c \
 	ipaddr.h \
+	cidr.c \
+	cidr.h \
 	log.c \
 	log.h \
 	xzmalloc.c \
@@ -30,21 +33,12 @@ libutil_la_SOURCES = \
 	setenvf.h \
 	tstat.c \
 	tstat.h \
-	veb.c \
-	veb.h \
 	read_all.c \
 	read_all.h \
-	ev_zmq.c \
-	ev_zmq.h \
-	msglist.c \
-	msglist.h \
 	cleanup.c \
 	cleanup.h \
 	unlink_recursive.c \
 	unlink_recursive.h \
-	sds.c \
-	sds.h \
-	sdsalloc.h \
 	iterators.h \
 	macros.h \
 	sha1.h \
@@ -87,24 +81,41 @@ libutil_la_SOURCES = \
 	fdutils.h \
 	fsd.c \
 	fsd.h \
-	zsecurity.c \
-	zsecurity.h \
 	errno_safe.h \
 	intree.c \
-	intree.h
-
-EXTRA_DIST = veb_mach.c
-
-TESTS = test_ev.t \
-	test_msglist.t \
-	test_sha1.t \
+	intree.h \
+	llog.h \
+	grudgeset.c \
+	grudgeset.h \
+	jpath.c \
+	jpath.h \
+	uri.c \
+	uri.h \
+	errprintf.c \
+	errprintf.h \
+	hola.c \
+	hola.h \
+	slice.c \
+	slice.h \
+	strstrip.c \
+	strstrip.h \
+	basemoji.h \
+	basemoji.c \
+	sigutil.h \
+	sigutil.c \
+	parse_size.h \
+	parse_size.c \
+	basename.h \
+	basename.c \
+	ansi_color.h
+
+TESTS = test_sha1.t \
 	test_sha256.t \
 	test_popen2.t \
 	test_kary.t \
 	test_cronodate.t \
 	test_wallclock.t \
 	test_stdlog.t \
-	test_veb.t \
 	test_lru_cache.t \
 	test_unlink.t \
 	test_cleanup.t \
@@ -113,23 +124,33 @@ TESTS = test_ev.t \
 	test_read_all.t \
 	test_tomltk.t \
 	test_ipaddr.t \
+	test_cidr.t \
 	test_fluid.t \
 	test_aux.t \
 	test_fdutils.t \
 	test_fsd.t \
-	test_zsecurity.t \
 	test_intree.t \
-	test_fdwalk.t
-
+	test_fdwalk.t \
+	test_grudgeset.t \
+	test_jpath.t \
+	test_errprintf.t \
+	test_hola.t \
+	test_strstrip.t \
+	test_slice.t \
+	test_timestamp.t \
+	test_environment.t \
+	test_basemoji.t \
+	test_sigutil.t \
+	test_parse_size.t
 
 test_ldadd = \
 	$(top_builddir)/src/common/libutil/libutil.la \
 	$(top_builddir)/src/common/libtap/libtap.la \
 	$(top_builddir)/src/common/liblsd/liblsd.la \
-	$(top_builddir)/src/common/libev/libev.la \
 	$(top_builddir)/src/common/libtomlc99/libtomlc99.la \
-	$(ZMQ_LIBS) \
-	$(LIBUUID_LIBS) \
+	$(top_builddir)/src/common/libczmqcontainers/libczmqcontainers.la \
+	$(top_builddir)/src/common/libccan/libccan.la \
+	$(top_builddir)/src/common/libmissing/libmissing.la \
 	$(LIBPTHREAD) \
 	$(LIBRT) \
 	$(JANSSON_LIBS)
@@ -138,20 +159,14 @@ test_cppflags = \
 	-I$(top_srcdir)/src/common/libtap \
 	$(AM_CPPFLAGS) $(JANSSON_CFLAGS)
 
-check_PROGRAMS = $(TESTS)
+check_PROGRAMS = \
+	$(TESTS) \
+	test_getaddr
 
 TEST_EXTENSIONS = .t
 T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
        $(top_srcdir)/config/tap-driver.sh
 
-test_ev_t_SOURCES = test/ev.c
-test_ev_t_CPPFLAGS = $(test_cppflags)
-test_ev_t_LDADD = $(test_ldadd)
-
-test_msglist_t_SOURCES = test/msglist.c
-test_msglist_t_CPPFLAGS = $(test_cppflags)
-test_msglist_t_LDADD = $(test_ldadd)
-
 test_sha1_t_SOURCES = test/sha1.c
 test_sha1_t_CPPFLAGS = $(test_cppflags)
 test_sha1_t_LDADD = $(test_ldadd)
@@ -173,8 +188,6 @@ test_cronodate_t_CPPFLAGS = $(test_cppflags)
 test_cronodate_t_LDADD = \
 	$(builddir)/cronodate.lo \
 	$(top_builddir)/src/common/libidset/libidset.la \
-	$(builddir)/veb.lo \
-	$(builddir)/xzmalloc.lo \
 	$(top_builddir)/src/common/libtap/libtap.la
 
 test_wallclock_t_SOURCES = test/wallclock.c
@@ -185,25 +198,21 @@ test_stdlog_t_SOURCES = test/stdlog.c
 test_stdlog_t_CPPFLAGS = $(test_cppflags)
 test_stdlog_t_LDADD = $(test_ldadd)
 
-test_veb_t_SOURCES = test/veb.c
-test_veb_t_CPPFLAGS = $(test_cppflags)
-test_veb_t_LDADD = $(test_ldadd)
-
 test_lru_cache_t_SOURCES = test/lru_cache.c
 test_lru_cache_t_CPPFLAGS = $(test_cppflags)
 test_lru_cache_t_LDADD = $(test_ldadd)
 
 test_blobref_t_SOURCES = test/blobref.c
-test_blobref_t_CPPFLAGS = $(test_cppflags) $(JANSSON_CFLAGS)
-test_blobref_t_LDADD = $(test_ldadd) $(JANSSON_LIBS)
+test_blobref_t_CPPFLAGS = $(test_cppflags)
+test_blobref_t_LDADD = $(test_ldadd)
 
 test_unlink_t_SOURCES = test/unlink.c
-test_unlink_t_CPPFLAGS = $(test_cppflags) $(JANSSON_CFLAGS)
-test_unlink_t_LDADD = $(test_ldadd) $(JANSSON_LIBS)
+test_unlink_t_CPPFLAGS = $(test_cppflags)
+test_unlink_t_LDADD = $(test_ldadd)
 
 test_cleanup_t_SOURCES = test/cleanup.c
-test_cleanup_t_CPPFLAGS = $(test_cppflags) $(JANSSON_CFLAGS)
-test_cleanup_t_LDADD = $(test_ldadd) $(JANSSON_LIBS)
+test_cleanup_t_CPPFLAGS = $(test_cppflags)
+test_cleanup_t_LDADD = $(test_ldadd)
 
 test_dirwalk_t_SOURCES = test/dirwalk.c
 test_dirwalk_t_CPPFLAGS = $(test_cppflags)
@@ -221,6 +230,14 @@ test_ipaddr_t_SOURCES = test/ipaddr.c
 test_ipaddr_t_CPPFLAGS = $(test_cppflags)
 test_ipaddr_t_LDADD = $(test_ldadd)
 
+test_getaddr_SOURCES = test/getaddr.c
+test_getaddr_CPPFLAGS = $(test_cppflags)
+test_getaddr_LDADD = $(test_ldadd)
+
+test_cidr_t_SOURCES = test/cidr.c
+test_cidr_t_CPPFLAGS = $(test_cppflags)
+test_cidr_t_LDADD = $(test_ldadd)
+
 test_fluid_t_SOURCES = test/fluid.c
 test_fluid_t_CPPFLAGS = $(test_cppflags)
 test_fluid_t_LDADD = $(test_ldadd)
@@ -237,10 +254,6 @@ test_fsd_t_SOURCES = test/fsd.c
 test_fsd_t_CPPFLAGS = $(test_cppflags)
 test_fsd_t_LDADD = $(test_ldadd)
 
-test_zsecurity_t_SOURCES = test/zsecurity.c
-test_zsecurity_t_CPPFLAGS = $(test_cppflags)
-test_zsecurity_t_LDADD = $(test_ldadd)
-
 test_intree_t_SOURCES = test/intree.c
 test_intree_t_CPPFLAGS = $(test_cppflags)
 test_intree_t_LDADD = $(test_ldadd)
@@ -248,3 +261,47 @@ test_intree_t_LDADD = $(test_ldadd)
 test_fdwalk_t_SOURCES = test/fdwalk.c
 test_fdwalk_t_CPPFLAGS = $(test_cppflags)
 test_fdwalk_t_LDADD = $(test_ldadd)
+
+test_grudgeset_t_SOURCES = test/grudgeset.c
+test_grudgeset_t_CPPFLAGS = $(test_cppflags)
+test_grudgeset_t_LDADD = $(test_ldadd)
+
+test_jpath_t_SOURCES = test/jpath.c
+test_jpath_t_CPPFLAGS = $(test_cppflags)
+test_jpath_t_LDADD = $(test_ldadd)
+
+test_errprintf_t_SOURCES = test/errprintf.c
+test_errprintf_t_CPPFLAGS = $(test_cppflags)
+test_errprintf_t_LDADD = $(test_ldadd)
+
+test_strstrip_t_SOURCES = test/strstrip.c
+test_strstrip_t_CPPFLAGS = $(test_cppflags)
+test_strstrip_t_LDADD = $(test_ldadd)
+
+test_hola_t_SOURCES = test/hola.c
+test_hola_t_CPPFLAGS = $(test_cppflags)
+test_hola_t_LDADD = $(test_ldadd)
+
+test_slice_t_SOURCES = test/slice.c
+test_slice_t_CPPFLAGS = $(test_cppflags)
+test_slice_t_LDADD = $(test_ldadd)
+
+test_timestamp_t_SOURCES = test/timestamp.c
+test_timestamp_t_CPPFLAGS = $(test_cppflags)
+test_timestamp_t_LDADD = $(test_ldadd)
+
+test_environment_t_SOURCES = test/environment.c
+test_environment_t_CPPFLAGS = $(test_cppflags)
+test_environment_t_LDADD = $(test_ldadd)
+
+test_basemoji_t_SOURCES = test/basemoji.c
+test_basemoji_t_CPPFLAGS = $(test_cppflags)
+test_basemoji_t_LDADD = $(test_ldadd)
+
+test_sigutil_t_SOURCES = test/sigutil.c
+test_sigutil_t_CPPFLAGS = $(test_cppflags)
+test_sigutil_t_LDADD = $(test_ldadd)
+
+test_parse_size_t_SOURCES = test/parse_size.c
+test_parse_size_t_CPPFLAGS = $(test_cppflags)
+test_parse_size_t_LDADD = $(test_ldadd)
diff --git a/src/common/libutil/ansi_color.h b/src/common/libutil/ansi_color.h
new file mode 100644
index 000000000000..d1fb151727e8
--- /dev/null
+++ b/src/common/libutil/ansi_color.h
@@ -0,0 +1,35 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_ANSI_COLOR_H
+#define _UTIL_ANSI_COLOR_H
+
+#define ANSI_COLOR_RED        "\x1b[31m"
+#define ANSI_COLOR_GREEN      "\x1b[32m"
+#define ANSI_COLOR_YELLOW     "\x1b[33m"
+#define ANSI_COLOR_BLUE       "\x1b[34m"
+#define ANSI_COLOR_MAGENTA    "\x1b[35m"
+#define ANSI_COLOR_CYAN       "\x1b[36m"
+#define ANSI_COLOR_GRAY       "\x1b[37m"
+#define ANSI_COLOR_DEFAULT    "\x1b[39m"
+#define ANSI_COLOR_DARK_GRAY  "\x1b[90m"
+
+#define ANSI_COLOR_BOLD_BLUE  "\x1b[01;34m"
+
+#define ANSI_COLOR_RESET      "\x1b[0m"
+#define ANSI_COLOR_BOLD       "\x1b[1m"
+#define ANSI_COLOR_HALFBRIGHT "\x1b[2m"
+#define ANSI_COLOR_REVERSE    "\x1b[7m"
+
+#define ANSI_COLOR_RESET      "\x1b[0m"
+
+#endif /* !_UTIL_ANSI_COLOR_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/aux.c b/src/common/libutil/aux.c
index 7805e308ba56..482c685c7e9d 100644
--- a/src/common/libutil/aux.c
+++ b/src/common/libutil/aux.c
@@ -15,6 +15,8 @@
 #include <errno.h>
 #include <stdlib.h>
 
+#include "ccan/str/str.h"
+
 #include "aux.h"
 
 struct aux_item {
@@ -33,33 +35,35 @@ static void aux_item_destroy (struct aux_item *aux)
         int saved_errno = errno;
         if (aux->free_fn && aux->val)
             aux->free_fn (aux->val);
-        free (aux->key);
         free (aux);
         errno = saved_errno;
     }
 }
 
 /* Create an aux item.
+ * The key (if any) is copied to the space following the item struct so the
+ * item and the key can be co-located in memory, and allocated with one malloc.
  * Return item on success, NULL on failure with errno set (ENOMEM).
  */
 static struct aux_item *aux_item_create (const char *key,
                                          void *val, aux_free_f free_fn)
 {
     struct aux_item *aux;
+    int keysize = key ? strlen (key) + 1 : 0;
 
-    if (!(aux = calloc (1, sizeof (*aux))))
+    if (!(aux = calloc (1, sizeof (*aux) + keysize)))
         return NULL;
-    if (key && !(aux->key = strdup (key)))
-        goto error;
+    if (key) {
+        aux->key = (char *)(aux + 1);
+        strcpy (aux->key, key);
+    }
     aux->val = val;
     aux->free_fn = free_fn;
     return aux;
-error:
-    aux_item_destroy (aux);
-    return NULL;
 }
 
 /* Delete from 'head' an aux item that was stored under 'key', if any.
+ * Quit search once an item with a NULL key is found, since these come last.
  * 'head' is an in/out parameter.
  */
 static void aux_item_delete (struct aux_item **head, const char *key)
@@ -67,8 +71,8 @@ static void aux_item_delete (struct aux_item **head, const char *key)
     if (key && head) {
         struct aux_item *item;
 
-        while ((item = *head)) {
-            if (item->key && !strcmp (item->key, key)) {
+        while ((item = *head) && item->key) {
+            if (streq (item->key, key)) {
                 *head = item->next;
                 aux_item_destroy (item);
                 break;
@@ -79,13 +83,14 @@ static void aux_item_delete (struct aux_item **head, const char *key)
 }
 
 /* Find in 'head' an aux item stored under 'key'.
+ * Quit search once an item with a NULL key is found, since these come last.
  * Returns item on success, NULL on failure.
  */
 static struct aux_item *aux_item_find (struct aux_item *head, const char *key)
 {
     if (key) {
-        while (head) {
-            if (head->key && !strcmp (key, head->key))
+        while (head && head->key) {
+            if (streq (key, head->key))
                 return head;
             head = head->next;
         }
@@ -105,6 +110,18 @@ static void aux_item_insert (struct aux_item **head, struct aux_item *item)
     }
 }
 
+/* Insert item at the end of 'head'.
+ * 'head' is an in/out parameter.
+ */
+static void aux_item_append (struct aux_item **head, struct aux_item *item)
+{
+    if (head && item) {
+        while (*head)
+            head = &(*head)->next;
+        *head = item;
+    }
+}
+
 /* Look up 'key' in 'head'.
  * Returns value on success, NULL on failure with errno set (EINVAL, ENOENT).
  */
@@ -125,6 +142,8 @@ void *aux_get (struct aux_item *head, const char *key)
 
 /* Insert ('key', 'value', 'free_fn') tuple in 'head'.
  * If 'key' is present in list, remove it first.
+ * If 'key' is NULL, append item to the list rather than prepend,
+ * so aux_get doesn't have to search keyless items.
  * 'head' is an in/out parameter.
  * Returns 0 on success, -1 on failure with errno set (EINVAL, ENOMEM).
  */
@@ -141,11 +160,30 @@ int aux_set (struct aux_item **head,
     if (val) {
         if (!(item = aux_item_create (key, val, free_fn)))
             return -1;
-        aux_item_insert (head, item);
+        if (key)
+            aux_item_insert (head, item);
+        else
+            aux_item_append (head, item);
     }
     return 0;
 }
 
+void aux_delete (struct aux_item **head, const void *val)
+{
+    struct aux_item *item;
+
+    if (head && val) {
+        while ((item = *head)) {
+            if (item->val == val) {
+                *head = item->next;
+                aux_item_destroy (item);
+                break;
+            }
+            head = &item->next;
+        }
+    }
+}
+
 /* Destroy aux list 'head', calling destructors on items that have them.
  */
 void aux_destroy (struct aux_item **head)
diff --git a/src/common/libutil/aux.h b/src/common/libutil/aux.h
index 8d02b1c1d9be..281147fd9029 100644
--- a/src/common/libutil/aux.h
+++ b/src/common/libutil/aux.h
@@ -36,6 +36,8 @@ struct aux_item;
 int aux_set (struct aux_item **aux, const char *key,
              void *val, aux_free_f free_fn);
 
+void aux_delete (struct aux_item **aux, const void *val);
+
 void *aux_get (struct aux_item *aux, const char *key);
 
 void aux_destroy (struct aux_item **aux);
diff --git a/src/common/libutil/basemoji.c b/src/common/libutil/basemoji.c
new file mode 100644
index 000000000000..7d71cbaba5aa
--- /dev/null
+++ b/src/common/libutil/basemoji.c
@@ -0,0 +1,228 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* basemoji.c - an emoji encoding for unsigned 64 bit integers
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <stdbool.h>
+
+#include "ccan/array_size/array_size.h"
+#include "basemoji.h"
+
+/* Minimum length of a b576 string is 1 emoji, or 4 bytes */
+#define BASEMOJI_MINLEN 4
+
+/* Maximum number of emoji "digits" in a basemoji string is
+ *
+ *  ceil (ln (2^64-1)/ln (576)) = 7
+ *
+ * 4 bytes per emoji, so 4*7 = 28 bytes.
+ */
+#define BASEMOJI_MAXLEN 28
+
+/*  The following is a Selection of 576 emoji in CLDR[1] collation order[2]
+ *  taken from the version 2010 Unicode emoji set[3]. Note: Selected code
+ *  points are all represented in 4 bytes, which is assumed in the
+ *  implementation in this module. Additionally, every character in this
+ *  selected set has a common first two bytes of F0 9F in UTF-8 encoding,
+ *  which aids in detection of a valid basemoji string.
+ *
+ *  1. https://cldr.unicode.org
+ *  2. https://unicode.org/emoji/charts-12.1/emoji-ordering.txt
+ *  3. https://unicode.org/emoji/charts/emoji-versions.html
+ *
+ */
+const char *emojis[] = {
+"😃", "😄", "😁", "😆", "😅", "😂", "😉", "😊", "😍", "😘", "😚", "😋",
+"😜", "😝", "😏", "😒", "😌", "😔", "😪", "😷", "😵", "😲", "😳", "😨",
+"😰", "😥", "😢", "😭", "😱", "😖", "😣", "😞", "😓", "😩", "😫", "😤",
+"😡", "😠", "👿", "💀", "💩", "👹", "👺", "👻", "👽", "👾", "😺", "😸",
+"😹", "😻", "😼", "😽", "🙀", "😿", "😾", "🙈", "🙉", "🙊", "💌", "💘",
+"💝", "💖", "💗", "💓", "💞", "💕", "💟", "💔", "💛", "💚", "💙", "💜",
+"💋", "💯", "💢", "💥", "💫", "💦", "💨", "💬", "💤", "👋", "👌", "👈",
+"👉", "👆", "👇", "👍", "👎", "👊", "👏", "🙌", "👐", "🙏", "💅", "💪",
+"👂", "👃", "👀", "👅", "👄", "👶", "👦", "👧", "👱", "👨", "👩", "👴",
+"👵", "🙍", "🙎", "🙅", "🙆", "💁", "🙋", "🙇", "👮", "💂", "👷", "👸",
+"👳", "👲", "👰", "👼", "🎅", "💆", "💇", "🚶", "🏃", "💃", "👯", "🏂",
+"🏄", "🏊", "🛀", "👫", "💏", "💑", "👪", "👤", "👣", "🐵", "🐒", "🐶",
+"🐩", "🐺", "🐱", "🐯", "🐴", "🐎", "🐮", "🐷", "🐗", "🐽", "🐑", "🐫",
+"🐘", "🐭", "🐹", "🐰", "🐻", "🐨", "🐼", "🐾", "🐔", "🐣", "🐤", "🐥",
+"🐦", "🐧", "🐸", "🐢", "🐍", "🐲", "🐳", "🐬", "🐟", "🐠", "🐡", "🐙",
+"🐚", "🐌", "🐛", "🐜", "🐝", "🐞", "💐", "🌸", "💮", "🌹", "🌺", "🌻",
+"🌼", "🌷", "🌱", "🌴", "🌵", "🌾", "🌿", "🍀", "🍁", "🍂", "🍃", "🍄",
+"🍇", "🍈", "🍉", "🍊", "🍌", "🍍", "🍎", "🍏", "🍑", "🍒", "🍓", "🍅",
+"🍆", "🌽", "🌰", "🍞", "🍖", "🍗", "🍔", "🍟", "🍕", "🍳", "🍲", "🍱",
+"🍘", "🍙", "🍚", "🍛", "🍜", "🍝", "🍠", "🍢", "🍣", "🍤", "🍥", "🍡",
+"🍦", "🍧", "🍨", "🍩", "🍪", "🎂", "🍰", "🍫", "🍬", "🍭", "🍮", "🍯",
+"🍵", "🍶", "🍷", "🍸", "🍹", "🍺", "🍻", "🍴", "🔪", "🌏", "🗾", "🌋",
+"🗻", "🏠", "🏡", "🏢", "🏣", "🏥", "🏦", "🏨", "🏩", "🏪", "🏫", "🏬",
+"🏭", "🏯", "🏰", "💒", "🗼", "🗽", "🌁", "🌃", "🌄", "🌅", "🌆", "🌇",
+"🌉", "🎠", "🎡", "🎢", "💈", "🎪", "🚃", "🚄", "🚅", "🚇", "🚉", "🚌",
+"🚑", "🚒", "🚓", "🚕", "🚗", "🚙", "🚚", "🚲", "🚏", "🚨", "🚥", "🚧",
+"🚤", "🚢", "💺", "🚀", "🕛", "🕐", "🕑", "🕒", "🕓", "🕔", "🕕", "🕖",
+"🕗", "🕘", "🕙", "🕚", "🌑", "🌓", "🌔", "🌕", "🌙", "🌛", "🌟", "🌠",
+"🌌", "🌀", "🌈", "🌂", "🔥", "💧", "🌊", "🎃", "🎄", "🎆", "🎇", "🎈",
+"🎉", "🎊", "🎋", "🎍", "🎎", "🎏", "🎐", "🎑", "🎀", "🎁", "🎫", "🏆",
+"🏀", "🏈", "🎾", "🎳", "🎣", "🎽", "🎿", "🎯", "🔫", "🎱", "🔮", "🎮",
+"🎰", "🎲", "🃏", "🀄", "🎴", "🎭", "🎨", "👓", "👔", "👕", "👖", "👗",
+"👘", "👙", "👚", "👛", "👜", "👝", "🎒", "👞", "👟", "👠", "👡", "👢",
+"👑", "👒", "🎩", "🎓", "💄", "💍", "💎", "🔊", "📢", "📣", "🔔", "🎼",
+"🎵", "🎶", "🎤", "🎧", "📻", "🎷", "🎸", "🎹", "🎺", "🎻", "📱", "📲",
+"📞", "📟", "📠", "🔋", "🔌", "💻", "💽", "💾", "💿", "📀", "🎥", "🎬",
+"📺", "📷", "📹", "📼", "🔍", "🔎", "💡", "🔦", "🏮", "📔", "📕", "📖",
+"📗", "📘", "📙", "📚", "📓", "📒", "📃", "📜", "📄", "📰", "📑", "🔖",
+"💰", "💴", "💵", "💸", "💳", "💹", "📧", "📨", "📩", "📤", "📥", "📦",
+"📫", "📪", "📮", "📝", "💼", "📁", "📂", "📅", "📆", "📇", "📈", "📉",
+"📊", "📋", "📌", "📍", "📎", "📏", "📐", "🔒", "🔓", "🔏", "🔐", "🔑",
+"🔨", "💣", "🔧", "🔩", "🔗", "📡", "💉", "💊", "🚪", "🚽", "🚬", "🗿",
+"🏧", "🚹", "🚺", "🚻", "🚼", "🚾", "🚫", "🚭", "🔞", "🔃", "🔙", "🔚",
+"🔛", "🔜", "🔝", "🔯", "🔼", "🔽", "🎦", "📶", "📳", "📴", "💱", "💲",
+"🔱", "📛", "🔰", "🔟", "🔠", "🔡", "🔢", "🔣", "🔤", "🆎", "🆑", "🆒",
+"🆓", "🆔", "🆕", "🆖", "🆗", "🆘", "🆙", "🆚", "🈁", "🈶", "🈯", "🉐",
+"🈹", "🈚", "🈲", "🉑", "🈸", "🈴", "🈳", "🈺", "🈵", "🔴", "🔵", "🔶",
+"🔷", "🔸", "🔹", "🔺", "🔻", "💠", "🔘", "🔳", "🔲", "🏁", "🚩", "🎌",
+};
+
+bool is_basemoji_string (const char *s)
+{
+    int len = strlen (s);
+
+    /* This code assumes length of emoji array is 576
+     * Generate error at build time if this becomes untrue:
+     */
+    BUILD_ASSERT(ARRAY_SIZE(emojis) == 576);
+
+    /* Check for expected length of a basemoji string, and if the
+     * first two bytes match the expected UTF-8 encoding.
+     * This doesn't guarantee that `s` is a valid basemoji string,
+     * but this will catch most obvious cases and other invalid strings
+     * are left to be detected in decode.
+     */
+    if (len >= BASEMOJI_MINLEN
+        && len <= BASEMOJI_MAXLEN
+        && len % 4 == 0
+        && (uint8_t)s[0] == 0xf0
+        && (uint8_t)s[1] == 0x9f)
+        return true;
+    return false;
+}
+
+/* Encode id into buf in reverse (i.e. higher order bytes are encoded
+ * and placed first into 'buf' since we're doing progressive division.)
+ */
+static int emoji_revenc (char *buf, int buflen, uint64_t id)
+{
+    int index = 0;
+    memset (buf, 0, buflen);
+    if (id == 0) {
+        memcpy (buf, emojis[0], 4);
+        return 4;
+    }
+    while (id > 0) {
+        int rem = id % 576;
+        memcpy (buf+index, emojis[rem], 4);
+        index += 4;
+        id = id / 576;
+    }
+    return index;
+}
+
+int uint64_basemoji_encode (uint64_t id, char *buf, int buflen)
+{
+    int count;
+    int n;
+    char reverse[BASEMOJI_MAXLEN+1];
+
+    if (buf == NULL || buflen <= 0) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    /* Encode bytes to emoji (in reverse), which also gives us a count
+     * of the total bytes required for this encoding.
+     */
+    if ((count = emoji_revenc (reverse, sizeof (reverse), id)) < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    /*  Check for overflow of provided buffer:
+     *  Need space for count bytes for emoji + NUL
+     */
+    if (count + 1 > buflen) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+
+    memset (buf, 0, buflen);
+    n = 0;
+
+    /* Copy 4-byte emojis back in order so that most significant bits are
+     * on the left:
+     */
+    for (int i = count - 4; i >= 0; i-=4) {
+        memcpy (buf+n, reverse+i, 4);
+        n+=4;
+    }
+    return 0;
+}
+
+
+static int basemoji_lookup (const char *c, int *result)
+{
+    for (int i = 0; i < 576; i++) {
+        if (memcmp (c, emojis[i], 4) == 0) {
+            *result = i;
+            return 0;
+        }
+    }
+    errno = EINVAL;
+    return -1;
+}
+
+int uint64_basemoji_decode (const char *str, uint64_t *idp)
+{
+    uint64_t id = 0;
+    uint64_t scale = 1;
+    int len;
+
+    if (str == NULL
+        || idp == NULL
+        || !is_basemoji_string (str)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    /* Move through basemoji string in reverse since least significant
+     * bits are at the end. Since all emoji are 4 bytes, start at 4 from
+     * the end to point to the final emoji.
+     */
+    len = strlen (str);
+    for (int i = len - 4; i >=  0; i-=4) {
+        int c;
+        if (basemoji_lookup (str+i, &c) < 0) {
+            errno = EINVAL;
+            return -1;
+        }
+        id += c * scale;
+        scale *= 576;
+    }
+    *idp = id;
+    return 0;
+}
diff --git a/src/common/libutil/basemoji.h b/src/common/libutil/basemoji.h
new file mode 100644
index 000000000000..488b82edc297
--- /dev/null
+++ b/src/common/libutil/basemoji.h
@@ -0,0 +1,46 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_BASEMOJI_H
+#define _UTIL_BASEMOJI_H
+
+#include <stdint.h>
+#include <stdbool.h>
+
+/*  basemoji - an implementation the RFC 19 FLUID emoji encoding
+ */
+
+/* Convert a 64 bit unsigned integer to basemoji, placing the result
+ * in buffer 'buf' of size 'buflen'.
+ *
+ * Returns 0 on success, -1 on failure with errno set:
+ * EINVAL: Invalid arguments
+ * EOVERFLOW: buffer too small for encoded string
+ */
+int uint64_basemoji_encode (uint64_t id, char *buf, int buflen);
+
+/* Decode a string in basemoji to an unsigned 64 bit integer.
+ *
+ * Returns 0 on success, -1 on failure with errno set:
+ * EINVAL: Invalid arguments
+ */
+int uint64_basemoji_decode (const char *str, uint64_t *idp);
+
+/*  Return true if 's' could be a basemoji string, i.e. it falls
+ *  within the minimum and maximum lengths, and starts with the
+ *  expected bytes.
+ */
+bool is_basemoji_string (const char *s);
+
+#endif /* !_UTIL_BASEMOJI_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/basename.c b/src/common/libutil/basename.c
new file mode 100644
index 000000000000..bdd90654c206
--- /dev/null
+++ b/src/common/libutil/basename.c
@@ -0,0 +1,28 @@
+/************************************************************\
+ * Copyright 2016 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <string.h>
+
+#include "basename.h"
+
+/* This is what glibc basename(3) does more or less.
+ * https://github.com/lattera/glibc/blob/master/string/basename.c
+ */
+
+char *basename_simple (const char *path)
+{
+    char *p = strrchr (path, '/');
+    return p ? p + 1 : (char *)path;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/basename.h b/src/common/libutil/basename.h
new file mode 100644
index 000000000000..315be034c85d
--- /dev/null
+++ b/src/common/libutil/basename.h
@@ -0,0 +1,18 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_BASENAME_H
+#define _UTIL_BASENAME_H
+
+char *basename_simple (const char *path);
+
+#endif /* !_UTIL_BASENAME_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/blobref.c b/src/common/libutil/blobref.c
index f02678333eff..b1af0658cea1 100644
--- a/src/common/libutil/blobref.c
+++ b/src/common/libutil/blobref.c
@@ -16,6 +16,10 @@
 #include <stdbool.h>
 #include <errno.h>
 #include <assert.h>
+#include <stdio.h>
+
+#include "ccan/str/str.h"
+#include "ccan/str/hex/hex.h"
 
 #include "blobref.h"
 #include "sha1.h"
@@ -42,13 +46,22 @@
 #error BLOBREF_MAX_DIGEST_SIZE is too small
 #endif
 
-static void sha1_hash (const void *data, int data_len, void *hash, int hash_len);
-static void sha256_hash (const void *data, int data_len, void *hash, int hash_len);
+static void sha1_hash (const void *data,
+                       size_t data_len,
+                       void *hash,
+                       size_t hash_len);
+static void sha256_hash (const void *data,
+                         size_t data_len,
+                         void *hash,
+                         size_t hash_len);
 
 struct blobhash {
     char *name;
-    int hashlen;
-    void (*hashfun)(const void *data, int data_len, void *hash, int hash_len);
+    size_t hashlen;
+    void (*hashfun)(const void *data,
+                    size_t data_len,
+                    void *hash,
+                    size_t hash_len);
 };
 
 static struct blobhash blobtab[] = {
@@ -63,7 +76,10 @@ static struct blobhash blobtab[] = {
     { NULL, 0, 0 },
 };
 
-static void sha1_hash (const void *data, int data_len, void *hash, int hash_len)
+static void sha1_hash (const void *data,
+                       size_t data_len,
+                       void *hash,
+                       size_t hash_len)
 {
     SHA1_CTX ctx;
 
@@ -73,7 +89,10 @@ static void sha1_hash (const void *data, int data_len, void *hash, int hash_len)
     SHA1_Final (&ctx, hash);
 }
 
-static void sha256_hash (const void *data, int data_len, void *hash, int hash_len)
+static void sha256_hash (const void *data,
+                         size_t data_len,
+                         void *hash,
+                         size_t hash_len)
 {
     SHA256_CTX ctx;
 
@@ -85,12 +104,14 @@ static void sha256_hash (const void *data, int data_len, void *hash, int hash_le
 
 /* true if s1 contains "s2-" prefix
  */
-static int prefixmatch (const char *s1, const char *s2)
+static bool prefixmatch (const char *s1, const char *s2)
 {
-    int len = strlen (s2);
-    if (strlen (s1) < len + 1 || s1[len] != '-')
-        return 0;
-    return !strncmp (s1, s2, len);
+    if (!strstarts (s1, s2))
+        return false;
+    s1 += strlen (s2);
+    if (*s1 != '-')
+        return false;
+    return true;
 }
 
 static struct blobhash *lookup_blobhash (const char *name)
@@ -98,28 +119,11 @@ static struct blobhash *lookup_blobhash (const char *name)
     struct blobhash *bh;
 
     for (bh = &blobtab[0]; bh->name != NULL; bh++)
-        if (!strcmp (name, bh->name) || prefixmatch (name, bh->name))
+        if (streq (name, bh->name) || prefixmatch (name, bh->name))
             return bh;
     return NULL;
 }
 
-static uint8_t xtoint (char c)
-{
-    if (c >= '0' && c <= '9')
-        return c - '0';
-    if (c >= 'A' && c <= 'F')
-        return c - 'A' + 0xA;
-    /* (c >= 'a' && c <= 'f') */
-    return c - 'a' + 0xA;
-}
-
-static char inttox (uint8_t i)
-{
-    if (i <= 9)
-        return '0' + i;
-    return 'a' + i - 0xa;
-}
-
 static bool isxdigit_lower (char c)
 {
     if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))
@@ -127,25 +131,19 @@ static bool isxdigit_lower (char c)
     return false;
 }
 
-int blobref_strtohash (const char *blobref, void *hash, int size)
+ssize_t blobref_strtohash (const char *blobref, void *hash, size_t size)
 {
     struct blobhash *bh;
-    uint8_t *ihash = (uint8_t *)hash;
-    int i;
+    size_t len = strlen (blobref);
+    size_t offset;
 
     if (!(bh = lookup_blobhash (blobref)) || size < bh->hashlen)
         goto inval;
-    if (strlen (blobref) != bh->hashlen*2 + strlen (bh->name) + 1)
+    offset = strlen (bh->name) + 1;
+    if (len - offset + 1 != hex_str_size (bh->hashlen))
+        goto inval;
+    if (!hex_decode (blobref + offset, len - offset, hash, bh->hashlen))
         goto inval;
-    blobref += strlen (bh->name) + 1;
-    for (i = 0; i < bh->hashlen; i++) {
-        if (!isxdigit_lower (blobref[i*2]))
-            goto inval;
-        ihash[i] = xtoint (blobref[i*2]) << 4;
-        if (!isxdigit_lower (blobref[i*2 + 1]))
-            goto inval;
-        ihash[i] |= xtoint (blobref[i*2 + 1]);
-    }
     return bh->hashlen;
 inval:
     errno = EINVAL;
@@ -153,32 +151,33 @@ int blobref_strtohash (const char *blobref, void *hash, int size)
 }
 
 static int hashtostr (struct blobhash *bh,
-                      const void *hash, int len,
-                      char *blobref, int blobref_len)
+                      const void *hash,
+                      size_t len,
+                      char *blobref,
+                      size_t blobref_len)
 {
-    uint8_t *ihash = (uint8_t *)hash;
-    int i;
+    size_t offset;
 
-    if (len != bh->hashlen
-        || !blobref
-        || blobref_len < bh->hashlen*2 + strlen (bh->name) + 2) {
-        errno = EINVAL;
-        return -1;
-    }
+    if (len != bh->hashlen || !blobref)
+        goto inval;
+    offset = strlen (bh->name) + 1;
+    if (blobref_len < hex_str_size (bh->hashlen) + offset - 1)
+        goto inval;
     strcpy (blobref, bh->name);
     strcat (blobref, "-");
-    blobref += strlen (bh->name) + 1;
-    for (i = 0; i < bh->hashlen; i++) {
-        blobref[i*2] = inttox (ihash[i] >> 4);
-        blobref[i*2 + 1] = inttox (ihash[i] & 0xf);
-    }
-    blobref[i*2] = '\0';
+    if (!hex_encode (hash, len, blobref + offset, blobref_len - offset))
+        goto inval;
     return 0;
+inval:
+    errno = EINVAL;
+    return -1;
 }
 
 int blobref_hashtostr (const char *hashtype,
-                       const void *hash, int len,
-                       void *blobref, int blobref_len)
+                       const void *hash,
+                       size_t len,
+                       void *blobref,
+                       size_t blobref_len)
 {
     struct blobhash *bh;
 
@@ -191,8 +190,10 @@ int blobref_hashtostr (const char *hashtype,
 
 
 int blobref_hash (const char *hashtype,
-                  const void *data, int len,
-                  void *blobref, int blobref_len)
+                  const void *data,
+                  size_t len,
+                  void *blobref,
+                  size_t blobref_len)
 {
     struct blobhash *bh;
     uint8_t hash[BLOBREF_MAX_DIGEST_SIZE];
@@ -205,32 +206,60 @@ int blobref_hash (const char *hashtype,
     return hashtostr (bh, hash, bh->hashlen, blobref, blobref_len);
 }
 
-int blobref_validate (const char *blobref)
+ssize_t blobref_hash_raw (const char *hashtype,
+                          const void *data,
+                          size_t len,
+                          void *hash,
+                          size_t hash_len)
 {
     struct blobhash *bh;
 
-    if (!blobref || !(bh = lookup_blobhash (blobref))
-                 || strlen (blobref) != bh->hashlen*2 + strlen (bh->name) + 1) {
+    if (!hashtype
+        || !(bh = lookup_blobhash (hashtype))
+        || hash_len < bh->hashlen
+        || !hash) {
         errno = EINVAL;
         return -1;
     }
-    blobref += strlen (bh->name) + 1;
+    bh->hashfun (data, len, hash, bh->hashlen);
+    return bh->hashlen;
+}
+
+int blobref_validate (const char *blobref)
+{
+    struct blobhash *bh;
+    size_t len;
+    size_t offset;
+
+    if (!blobref || !(bh = lookup_blobhash (blobref)))
+        goto inval;
+    len = strlen (blobref);
+    offset = strlen (bh->name) + 1;
+    if (len - offset + 1 != hex_str_size (bh->hashlen))
+        goto inval;
+    blobref += offset;
     while (*blobref) {
-        if (!isxdigit_lower (*blobref++)) {
-            errno = EINVAL;
-            return -1;
-        }
+        if (!isxdigit_lower (*blobref++))
+            goto inval;
     }
     return 0;
+inval:
+    errno = EINVAL;
+    return -1;
 }
 
-int blobref_validate_hashtype (const char *name)
+ssize_t blobref_validate_hashtype (const char *name)
 {
-    if (name == NULL || !lookup_blobhash (name))
+    struct blobhash *bh;
+
+    if (name == NULL || !(bh = lookup_blobhash (name))) {
+        errno = EINVAL;
         return -1;
-    return 0;
+    }
+    return bh->hashlen;
 }
 
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libutil/blobref.h b/src/common/libutil/blobref.h
index b07a7da3fbb9..628a63760c5f 100644
--- a/src/common/libutil/blobref.h
+++ b/src/common/libutil/blobref.h
@@ -20,32 +20,49 @@
  * The hash algorithm is selected by the blobref prefix.
  * Returns hash length on success, or -1 on error, with errno set.
  */
-int blobref_strtohash (const char *blobref, void *hash, int size);
+ssize_t blobref_strtohash (const char *blobref,
+                           void *hash,
+                           size_t size);
 
 /* Convert a hash digest to null-terminated blobref string in 'blobref'.
  * The hash algorithm is selected by 'hashtype', e.g. "sha1".
  * Returns 0 on success, -1 on error, with errno set.
  */
 int blobref_hashtostr (const char *hashtype,
-                       const void *hash, int len,
-                       void *blobref, int blobref_len);
+                       const void *hash,
+                       size_t len,
+                       void *blobref,
+                       size_t blobref_len);
 
 /* Compute hash over data and return null-terminated blobref string in
  * 'blobref'.  The hash algorithm is selected by 'hashtype', e.g. "sha1".
  * Returns 0 on success, -1 on error with errno set.
  */
 int blobref_hash (const char *hashtype,
-                  const void *data, int len,
-                  void *blobref, int blobref_len);
+                  const void *data,
+                  size_t len,
+                  void *blobref,
+                  size_t blobref_len);
+
+/* Compute hash over data and store it in 'hash'.
+ * The hash algorithm is selected by 'hashtype', e.g. "sha1".
+ * Returns hash size on success, -1 on error with errno set.
+ */
+ssize_t blobref_hash_raw (const char *hashtype,
+                          const void *data,
+                          size_t len,
+                          void *hash,
+                          size_t hash_len);
 
 /* Check validity of blobref string.
  */
 int blobref_validate (const char *blobref);
 
 /* Check the validity of hash type (by name)
+ * If valid, the digest size is returned.
+ * If invalid, -1 is returned with errno set.
  */
-int blobref_validate_hashtype (const char *name);
-
+ssize_t blobref_validate_hashtype (const char *name);
 
 #endif /* _UTIL_BLOBREF_H */
 /*
diff --git a/src/common/libutil/cidr.c b/src/common/libutil/cidr.c
new file mode 100644
index 000000000000..a7a43d23d98d
--- /dev/null
+++ b/src/common/libutil/cidr.c
@@ -0,0 +1,95 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* cidr.c - parse RFC 4632 CIDR notation */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <arpa/inet.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <string.h>
+#include <errno.h>
+
+#include "errno_safe.h"
+#include "cidr.h"
+
+/* Destructively parse /N from the end of 's'.
+ * Leaves 's' containing only the string before the / character.
+ * If /N is missing, return max_value.
+ */
+static int parse_netprefix (char *s, int max_value)
+{
+    char *cp;
+    char *endptr;
+    int n = max_value;
+
+    if ((cp = strrchr (s, '/'))) {
+        *cp++ = '\0';
+        errno = 0;
+        n = strtoul (cp, &endptr, 10);
+        if (errno != 0 || *endptr != '\0' || n > max_value) {
+            errno = EINVAL;
+            return -1;
+        }
+    }
+    return n;
+}
+
+static uint32_t netprefix_to_netmask4 (int prefix)
+{
+    if (prefix > 0)
+        return htonl (~((1 << (32 - prefix)) - 1));
+    return 0;
+}
+
+int cidr_parse4 (struct cidr4 *cidrp, const char *s)
+{
+    char *cpy;
+    struct cidr4 cidr;
+    int prefix;
+    int n;
+
+    if (!cidrp || !s) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(cpy = strdup (s)))
+        return -1;
+    if ((prefix = parse_netprefix (cpy, 32)) < 0)
+        goto error;
+    cidr.mask.s_addr = netprefix_to_netmask4 (prefix);
+    if ((n = inet_pton (AF_INET, cpy, &cidr.addr)) < 0)
+        goto error;
+    if (n == 0) {
+        errno = EINVAL;
+        goto error;
+    }
+
+    free (cpy);
+    *cidrp = cidr;
+    return 0;
+error:
+    ERRNO_SAFE_WRAP (free, cpy);
+    return -1;
+}
+
+bool cidr_match4 (struct cidr4 *cidr, struct in_addr *addr)
+{
+    if (!cidr || !addr)
+        return false;
+    uint32_t mask = cidr->mask.s_addr;
+    if ((addr->s_addr & mask) == (cidr->addr.s_addr & mask))
+        return true;
+    return false;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/cidr.h b/src/common/libutil/cidr.h
new file mode 100644
index 000000000000..0d59f2cb7547
--- /dev/null
+++ b/src/common/libutil/cidr.h
@@ -0,0 +1,27 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_CIDR_H
+#define _UTIL_CIDR_H
+
+#include <arpa/inet.h>
+#include <stdbool.h>
+
+struct cidr4 {
+    struct in_addr addr;
+    struct in_addr mask;
+};
+
+int cidr_parse4 (struct cidr4 *cidr, const char *s);
+bool cidr_match4 (struct cidr4 *cidr, struct in_addr *addr);
+
+#endif /* !_UTIL_CIDR_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/cleanup.c b/src/common/libutil/cleanup.c
index f0863826da95..46fe374d0931 100644
--- a/src/common/libutil/cleanup.c
+++ b/src/common/libutil/cleanup.c
@@ -12,16 +12,16 @@
 #  include <config.h>
 #endif /* HAVE_CONFIG_H */
 
-#include "cleanup.h"
-#include "unlink_recursive.h"
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <libgen.h>
 #include <unistd.h>
 #include <pthread.h>
 
-#include <czmq.h>
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "cleanup.h"
+#include "unlink_recursive.h"
 
 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
 static pid_t cleaner_pid = 0;
diff --git a/src/common/libutil/cronodate.c b/src/common/libutil/cronodate.c
index f3acc547fbb4..ffb2978e42de 100644
--- a/src/common/libutil/cronodate.c
+++ b/src/common/libutil/cronodate.c
@@ -14,10 +14,13 @@
 #include <time.h>
 #include <sys/time.h>
 #include <ctype.h>
-#include <czmq.h>
+#include <errno.h>
+#include <assert.h>
+#include <stdlib.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libidset/idset.h"
-#include "src/common/libutil/xzmalloc.h"
+#include "ccan/str/str.h"
 
 #include "cronodate.h"
 
@@ -174,9 +177,11 @@ void cronodate_destroy (cronodate_t *d)
 cronodate_t * cronodate_create ()
 {
     int i;
-    cronodate_t *d = xzmalloc (sizeof (*d));
+    cronodate_t *d;
+
+    if (!(d = calloc (1, sizeof (*d))))
+        return NULL;
 
-    memset (d, 0, sizeof (*d));
     for (i = 0; i < TM_MAX_ITEM; i++) {
         struct idset *n = idset_create (tm_unit_max (i) + 1,
                                         IDSET_FLAG_AUTOGROW);
@@ -218,7 +223,7 @@ static int tm_string2int (const char *s, tm_unit_t u)
 static int get_range (const char *r, tm_unit_t u, int *lo, int *hi)
 {
     char *p;
-    if (strcmp (r, "*") == 0) {
+    if (streq (r, "*")) {
         *lo = tm_unit_min (u);
         *hi = tm_unit_max (u);
     }
diff --git a/src/common/libutil/dirwalk.c b/src/common/libutil/dirwalk.c
index 693cc8584cec..0a5b8a13c216 100644
--- a/src/common/libutil/dirwalk.c
+++ b/src/common/libutil/dirwalk.c
@@ -10,7 +10,9 @@
 
 /* dirwalk.c - simple interface to recurse a directory tree */
 
-#define _GNU_SOURCE
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -18,9 +20,15 @@
 #include <unistd.h>
 #include <libgen.h>
 #include <fnmatch.h>
+#include <assert.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <fcntl.h>
 
-#include <czmq.h>
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
 
+#include "basename.h"
 #include "dirwalk.h"
 
 struct direntry {
@@ -146,7 +154,7 @@ const char * dirwalk_name (dirwalk_t *d)
     if (!d->current)
         return NULL;
     if (!d->current->basename)
-        d->current->basename = basename (d->current->path);
+        d->current->basename = basename_simple (d->current->path);
     return d->current->basename;
 }
 
@@ -180,7 +188,7 @@ int dirwalk_isdir (dirwalk_t *d)
 
 static int is_dotted_dir (struct dirent *dent)
 {
-    if (!strcmp (dent->d_name, ".") || !strcmp (dent->d_name, ".."))
+    if (streq (dent->d_name, ".") || streq (dent->d_name, ".."))
         return 1;
     return 0;
 }
@@ -226,7 +234,8 @@ static int dirwalk_traverse (dirwalk_t *d, dirwalk_filter_f fn, void *arg)
                 dirwalk_stop (d, errno);
             continue;
         }
-        if (S_ISDIR (d->current->sb.st_mode)) {
+        if (S_ISDIR (d->current->sb.st_mode)
+            && !(d->flags & DIRWALK_NORECURSE)) {
             /*
              *  Save current direntry onto stack and call traverse()
              */
@@ -234,7 +243,7 @@ static int dirwalk_traverse (dirwalk_t *d, dirwalk_filter_f fn, void *arg)
             (void) dirwalk_traverse (d, fn, arg);
             d->current = zlist_pop (d->dirstack);
         }
-        else /* Not A directory, simply visit this file */
+        else /* Not a directory or NORECURSE, simply visit this object */
             dirwalk_visit (d, fn, arg);
         direntry_destroy (d->current);
         d->current = NULL;
diff --git a/src/common/libutil/dirwalk.h b/src/common/libutil/dirwalk.h
index 0f4fc985f2d0..50d0ce8f04d4 100644
--- a/src/common/libutil/dirwalk.h
+++ b/src/common/libutil/dirwalk.h
@@ -11,7 +11,7 @@
 #ifndef HAVE_UTIL_DIRWALK_H
 #define HAVE_UTIL_DIRWALK_H
 
-#include <czmq.h> /* zlist_t */
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 typedef struct dirwalk dirwalk_t;
 
@@ -19,6 +19,7 @@ enum {
     DIRWALK_DEPTH =    1<<0,  /* Traverse in depth-first order             */
     DIRWALK_REALPATH = 1<<1,  /* Resolve all paths with realpath(3)        */
     DIRWALK_FIND_DIR = 1<<2,  /* Do not skip directories in dirwalk_find() */
+    DIRWALK_NORECURSE = 1<<3, /* Do not descend into directories           */
 };
 
 /*
diff --git a/src/common/libutil/environment.c b/src/common/libutil/environment.c
index 6f7251ee1297..78f543e58a26 100644
--- a/src/common/libutil/environment.c
+++ b/src/common/libutil/environment.c
@@ -16,13 +16,18 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <stdlib.h>
-#include <czmq.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/oom.h"
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/iterators.h"
+#include "ccan/str/str.h"
 
 
 #include "environment.h"
@@ -70,7 +75,7 @@ char *find_env_item(struct env_item *i, const char *s)
 {
     char *entry = 0;
     while ((entry = argz_next (i->argz, i->argz_len, entry))) {
-        if (!strcmp(entry, s))
+        if (streq (entry, s))
             return entry;
     }
     return NULL;
@@ -146,8 +151,23 @@ static void environment_push_inner (struct environment *e,
     if (split) {
         char *split_value = NULL;
         size_t split_value_len = 0;
+        char *entry = NULL;
+
         argz_create_sep (value, item->sep, &split_value, &split_value_len);
-        char *entry = 0;
+        if (before && argz_count (split_value, split_value_len) > 1) {
+            /*  If inserting a split list "before", we need to reverse
+             *   the list, o/w the split list is pushed in the wrong
+             *   order (last element ends up as the first element)
+             */
+            char *rev = NULL;
+            size_t rev_len = 0;
+            while ((entry = argz_next (split_value, split_value_len, entry)))
+                argz_insert (&rev, &rev_len, rev, entry);
+            free (split_value);
+            split_value = rev;
+            split_value_len = rev_len;
+        }
+        entry = NULL;
         while((entry = argz_next (split_value, split_value_len, entry))) {
             char *found;
             if ((!strlen(entry)))
@@ -208,6 +228,8 @@ void environment_from_env (struct environment *e,
                            char separator)
 {
     const char *env = getenv (key);
+    if (!env && !default_base)
+        return;
     if (!env)
         env = default_base;
     environment_set (e, key, env, separator);
@@ -237,6 +259,38 @@ const char *environment_cursor (struct environment *e)
     return zhash_cursor (e->environment);
 }
 
+const char *environment_var_next (struct environment *e,
+                                  const char *key,
+                                  const char *entry)
+{
+    struct env_item *item = zhash_lookup (e->environment, key);
+    if (!item)
+        return NULL;
+    return argz_next (item->argz, item->argz_len, entry);
+}
+
+int environment_insert (struct environment *e,
+                        const char *key,
+                        char *before,
+                        const char *value)
+{
+    error_t err;
+    struct env_item *item = zhash_lookup (e->environment, key);
+    if (!item) {
+        errno = ENOENT;
+        return -1;
+    }
+    if ((err = argz_insert (&item->argz,
+                            &item->argz_len,
+                            before,
+                            value) != 0)) {
+        errno = err;
+        return -1;
+    }
+    return 0;
+}
+
+
 void environment_apply (struct environment *e)
 {
     const char *key;
diff --git a/src/common/libutil/environment.h b/src/common/libutil/environment.h
index dc130f270b96..b4d436a8224b 100644
--- a/src/common/libutil/environment.h
+++ b/src/common/libutil/environment.h
@@ -42,12 +42,25 @@ const char *environment_next (struct environment *e);
  * @brief Get the value of the current cursor element, matching the key from
  * either *_first or *_next
  *
- * @param e the envirnoment to operate on
+ * @param e the environment to operate on
  *
  * @return The value of the current environment element
  */
 const char *environment_cursor (struct environment *e);
 
+/**
+ * @brief Iterate over a multi-element environment variable *_key
+ *
+ * @param e the environment to operate on
+ * @param key the environment variable to iterate over
+ * @param entry the current entry, set to NULL to begin iteration
+ *
+ * @return The value of the current environment variable element
+ */
+const char *environment_var_next (struct environment *e,
+                                  const char *key,
+                                  const char *entry);
+
 /**
  * @brief Apply the changes encoded in this environment to the environment of
  * the current process.
@@ -86,6 +99,22 @@ void environment_push_back (struct environment *e,
                             const char *key,
                             const char *value);
 
+/**
+ * @brief Push "value" before position "before" in the environment variable
+ *  "key."
+ *
+ * @param e the environment in which to add this element
+ * @param key the environment variable name
+ * @param before the element before which to insert this value
+ * @param value the value to insert
+ *
+ * @return 0 on success -1 with errno set on failure.
+ */
+int environment_insert (struct environment *e,
+                        const char *key,
+                        char *before,
+                        const char *value);
+
 /**
  * @brief Add the specified value to the front of the target key without
  * de-duplication, it will still be separated from the rest of the value by sep,
@@ -146,7 +175,8 @@ void environment_unset (struct environment *e, const char *key);
  * @param e the environment to act on
  * @param key the key to initialize
  * @param default_base the default value to use if the environment variable is
- * unset
+ *  unset. If default_base == NULL then do not set a default (if the key
+ *  is unset, leave it unset).
  * @param separator the separator between tokens in the value
  */
 void environment_from_env (struct environment *e,
diff --git a/src/common/libutil/errprintf.c b/src/common/libutil/errprintf.c
new file mode 100644
index 000000000000..3716af71e5c4
--- /dev/null
+++ b/src/common/libutil/errprintf.c
@@ -0,0 +1,46 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <errno.h>
+
+#include "errprintf.h"
+
+int verrprintf (flux_error_t *errp, const char *fmt, va_list ap)
+{
+    if (errp) {
+        int saved_errno = errno;
+        memset (errp->text, 0, sizeof (errp->text));
+        if (fmt) {
+            int n;
+            n = vsnprintf (errp->text, sizeof (errp->text), fmt, ap);
+            if (n > sizeof (errp->text))
+                errp->text[sizeof (errp->text) - 2] = '+';
+        }
+        errno = saved_errno;
+    }
+    return -1;
+}
+
+int errprintf (flux_error_t *errp, const char *fmt, ...)
+{
+    va_list ap;
+    va_start (ap, fmt);
+    verrprintf (errp, fmt, ap);
+    va_end (ap);
+    return -1;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/errprintf.h b/src/common/libutil/errprintf.h
new file mode 100644
index 000000000000..e0b31fc925f1
--- /dev/null
+++ b/src/common/libutil/errprintf.h
@@ -0,0 +1,41 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_ERRPRINTF_H
+#define _UTIL_ERRPRINTF_H
+
+#include <stdarg.h>
+#include <string.h>
+
+#include "src/common/libflux/types.h" /* flux_error_t */
+
+/*
+ *  Utility function for printing an error to a flux_error_t container.
+ *  This function always returns -1 as a convenience in error handling
+ *   functions, e.g.
+ *
+ *  return errprintf (errp, "Function failed");
+ *
+ */
+int errprintf (flux_error_t *errp, const char *fmt, ...)
+     __attribute__ ((format (printf, 2, 3)));
+
+int verrprintf (flux_error_t *errp, const char *fmt, va_list ap);
+
+
+static inline void err_init (flux_error_t *errp)
+{
+    if (errp)
+        memset (errp->text, 0, sizeof (errp->text));
+}
+
+#endif /* !_UTIL_ERRPRINTF_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/fdwalk.c b/src/common/libutil/fdwalk.c
index 735f22d37de8..78a77ce8c6ab 100644
--- a/src/common/libutil/fdwalk.c
+++ b/src/common/libutil/fdwalk.c
@@ -93,6 +93,7 @@ struct linux_dirent64
 
 #include "fdwalk.h"
 
+#ifdef __linux__
 // Parses a file descriptor number in base 10, requiring the strict format used
 // in /proc/*/fd. Returns the value, or -1 if not a valid string.
 static int parse_fd(const char *s) {
@@ -111,6 +112,7 @@ static int parse_fd(const char *s) {
     } while (*s);
     return val;
 }
+#endif /* __linux__ */
 
 int _fdwalk_portable (void (*func)(void *, int), void *data)
 {
@@ -121,15 +123,16 @@ int _fdwalk_portable (void (*func)(void *, int), void *data)
 
 }
 
-int fdwalk (void (*func)(void *, int), void *data) {
-    int fd;
-    int rc = 0;
+int fdwalk (void (*func)(void *, int), void *data){
 
     /* On Linux use getdents64 to avoid malloc in opendir() and
      *  still walk only open fds. If not on linux or open() of /proc/self
      *  fails, fall back to iterating over all possible fds.
      */
 #ifdef __linux__
+    int fd;
+    int rc = 0;
+
     int dir_fd = open ("/proc/self/fd", O_RDONLY | O_DIRECTORY);
     if (dir_fd >= 0) {
         char buf [4096];
diff --git a/src/common/libutil/fluid.c b/src/common/libutil/fluid.c
index 0e1c93089e8f..ca16044dcb26 100644
--- a/src/common/libutil/fluid.c
+++ b/src/common/libutil/fluid.c
@@ -18,15 +18,74 @@
 #include <time.h>
 #include <unistd.h>
 #include <stdint.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <ctype.h>
+#include <langinfo.h>
+#include <locale.h>
 
+#include "ccan/str/str.h"
 #include "fluid.h"
 #include "mnemonic.h"
+#include "basemoji.h"
 
 /* fluid: [ts:40 id:14 seq:10] */
 static const int bits_per_ts = 40;
 static const int bits_per_id = 14;
 static const int bits_per_seq = 10;
 
+/* Max base58 string length for F58 encoding */
+#define MAX_B58_STRLEN 12
+
+#if ASSUME_BROKEN_LOCALE
+static const char f58_prefix[] = "f";
+static const char f58_alt_prefix[] = "ƒ";
+#else
+static const char f58_prefix[] = "ƒ";
+static const char f58_alt_prefix[] = "f";
+#endif /* ASSUME_BROKEN_LOCALE */
+
+/*  b58digits_map courtesy of libbase58:
+ *
+ *  https://github.com/bitcoin/libbase58.git
+ *
+
+Copyright (c) 2014 Luke Dashjr
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+*/
+
+static const int8_t b58digits_map[] = {
+    -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+    -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+    -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+    -1, 0, 1, 2, 3, 4, 5, 6,  7, 8,-1,-1,-1,-1,-1,-1,
+    -1, 9,10,11,12,13,14,15, 16,-1,17,18,19,20,21,-1,
+    22,23,24,25,26,27,28,29, 30,31,32,-1,-1,-1,-1,-1,
+    -1,33,34,35,36,37,38,39, 40,41,42,43,-1,44,45,46,
+    47,48,49,50,51,52,53,54, 55,56,57,-1,-1,-1,-1,-1,
+};
+
+static const char b58digits[] =
+    "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
+
 static int current_ds (uint64_t *ds)
 {
     struct timespec ts;
@@ -37,41 +96,197 @@ static int current_ds (uint64_t *ds)
     return 0;
 }
 
-int fluid_init (struct fluid_generator *gen, uint32_t id)
+int fluid_init (struct fluid_generator *gen, uint32_t id, uint64_t timestamp)
 {
-    if (current_ds (&gen->epoch) < 0)
+    if (current_ds (&gen->clock_zero) < 0)
         return -1;
     if (id >= (1ULL<<bits_per_id))
         return -1;
     gen->id = id;
     gen->seq = 0;
-    gen->last_ds = 0;
+    gen->clock_offset = timestamp;
+    gen->timestamp = timestamp;
     return 0;
 }
 
-int fluid_generate (struct fluid_generator *gen, fluid_t *fluid)
+static int update_timestamp (struct fluid_generator *gen)
 {
-    uint64_t s;
+    uint64_t clock;
+    uint64_t timestamp;
 
-    if (current_ds (&s) < 0)
+    if (current_ds (&clock) < 0)
         return -1;
-    if (s == gen->last_ds)
-        gen->seq++;
-    else {
+    timestamp = clock - gen->clock_zero + gen->clock_offset;
+    if (timestamp >= (1ULL<<bits_per_ts))
+        return -1; // (unlikely) lifetime of FLUID sequence is over
+    if (timestamp > gen->timestamp) {
         gen->seq = 0;
-        gen->last_ds = s;
+        gen->timestamp = timestamp;
     }
-    if ((s - gen->epoch) >= (1ULL<<bits_per_ts))
+    return 0;
+}
+
+int fluid_save_timestamp (struct fluid_generator *gen, uint64_t *timestamp)
+{
+    if (update_timestamp (gen) < 0)
+        return -1;
+    *timestamp = gen->timestamp;
+    return 0;
+}
+
+uint64_t fluid_get_timestamp (fluid_t fluid)
+{
+    return fluid >> (bits_per_seq + bits_per_id);
+}
+
+/* If sequence bits were exhausted (already 1024 allocated in this timestamp),
+ * busy-wait, calling update_timestamp() until seq is cleared.
+ * The busy-wait time is bounded by the timestamp quanta (1 msec).
+ */
+int fluid_generate (struct fluid_generator *gen, fluid_t *fluid)
+{
+    do {
+        if (update_timestamp (gen) < 0)
+            return -1;
+    } while (gen->seq + 1 >= (1ULL<<bits_per_seq));
+    *fluid = (gen->timestamp << (bits_per_seq + bits_per_id)
+                    | (gen->id << bits_per_seq)
+                    | gen->seq);
+    gen->seq++;
+    return 0;
+}
+
+/*  F58 encoding.
+ */
+/*  Compute base58 encoding of id in *reverse*
+ *  Return number of characters written into buf
+ */
+static int b58revenc (char *buf, int bufsz, fluid_t id)
+{
+    int index = 0;
+    memset (buf, 0, bufsz);
+    if (id == 0) {
+        buf[0] = b58digits[0];
+        return 1;
+    }
+    while (id > 0) {
+        int rem = id % 58;
+        buf[index++] = b58digits[rem];
+        id = id / 58;
+    }
+    return index;
+}
+
+static inline int is_utf8_locale (void)
+{
+    /* Check for UTF-8, only if the current encoding is multibyte (not C.UTF-8
+     * or similar), but allow ascii encoding to be enforced if
+     * FLUX_F58_FORCE_ASCII is set.
+     */
+    if (MB_CUR_MAX > 1 && streq (nl_langinfo (CODESET), "UTF-8")
+        && !getenv ("FLUX_F58_FORCE_ASCII"))
+        return 1;
+    return 0;
+}
+
+static int fluid_f58_encode (char *buf, int bufsz, fluid_t id, bool plain)
+{
+    int count;
+    const char *prefix = f58_prefix;
+    char b58reverse[13];
+    int index = 0;
+
+    if (buf == NULL || bufsz <= 0) {
+        errno = EINVAL;
+        return -1;
+    }
+
+#if !ASSUME_BROKEN_LOCALE
+    /* Use alternate "f" prefix if locale is not multibyte */
+    if (!is_utf8_locale() || plain)
+        prefix = f58_alt_prefix;
+#endif
+
+    if (bufsz <= strlen (prefix) + 1) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    if ((count = b58revenc (b58reverse, sizeof (b58reverse), id)) < 0) {
+        errno = EINVAL;
         return -1;
-    if (gen->seq >= (1ULL<<bits_per_seq)) {
-        usleep (200);
-        return fluid_generate (gen, fluid);
     }
-    *fluid = ((s - gen->epoch) << (bits_per_seq + bits_per_id)
-                    | (gen->id << bits_per_seq) | gen->seq);
+
+    /* Copy prefix to buf and zero remaining bytes */
+    (void) strncpy (buf, prefix, bufsz);
+    index = strlen (buf);
+
+    if (index + count + 1 > bufsz) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+
+    for (int i = count - 1; i >= 0; i--)
+        buf[index++] = b58reverse[i];
+
     return 0;
 }
 
+static int b58decode (const char *str, uint64_t *idp)
+{
+    int64_t id = 0;
+    int64_t scale = 1;
+    int len = strlen (str);
+    if (len == 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    for (int i = len - 1; i >= 0; i--) {
+        int8_t c = b58digits_map[(int8_t)str[i]];
+        if (c == -1) {
+            /* invalid base58 digit */
+            errno = EINVAL;
+            return -1;
+        }
+        id += c * scale;
+        scale *= 58;
+    }
+    *idp = id;
+    return 0;
+}
+
+static int fluid_is_f58 (const char *str)
+{
+    if (str == NULL || str[0] == '\0')
+        return 0;
+    if (strstarts (str, f58_prefix))
+        return strlen (f58_prefix);
+    if (strstarts (str, f58_alt_prefix))
+        return strlen (f58_alt_prefix);
+    return 0;
+}
+
+static int fluid_f58_decode (fluid_t *idptr, const char *str)
+{
+    int prefix = 0;
+    const char *b58str = NULL;
+
+    if (idptr == NULL || str == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    if ((prefix = fluid_is_f58 (str)) == 0) {
+        /* no prefix match, not valid f58 */
+        errno = EINVAL;
+        return -1;
+    }
+    b58str = str+prefix;
+    if (strlen (b58str) > MAX_B58_STRLEN) {
+        errno = EINVAL;
+        return -1;
+    }
+    return b58decode (str+prefix, idptr);
+}
+
 static int fluid_decode_dothex (const char *s, fluid_t *fluid)
 {
     int i;
@@ -116,6 +331,18 @@ int fluid_encode (char *buf, int bufsz, fluid_t fluid,
                             buf, bufsz, MN_FDEFAULT) != MN_OK)
                 return -1;
             break;
+        case FLUID_STRING_F58:
+            if (fluid_f58_encode (buf, bufsz, fluid, false) < 0)
+                return -1;
+            break;
+        case FLUID_STRING_F58_PLAIN:
+            if (fluid_f58_encode (buf, bufsz, fluid, true) < 0)
+                return -1;
+            break;
+        case FLUID_STRING_EMOJI:
+            if (uint64_basemoji_encode (fluid, buf, bufsz) < 0)
+                return -1;
+            break;
     }
     return 0;
 }
@@ -153,11 +380,21 @@ int fluid_decode (const char *s, fluid_t *fluidp, fluid_string_type_t type)
              * Also, 's' is not modified so it is safe to cast away const.
              */
             rc = mn_decode ((char *)s, (void *)&fluid, sizeof (fluid_t));
-            if (rc != 8)
+            if (rc != 8) {
+                errno = EINVAL;
+                return -1;
+            }
+            break;
+        case FLUID_STRING_F58:
+            if (fluid_f58_decode (&fluid, s) < 0)
+                return -1;
+            break;
+        case FLUID_STRING_EMOJI:
+            if (uint64_basemoji_decode (s, &fluid) < 0)
                 return -1;
             break;
-
         default:
+            errno = EINVAL;
             return -1;
     }
     if (fluid_validate (fluid) < 0)
@@ -166,6 +403,83 @@ int fluid_decode (const char *s, fluid_t *fluidp, fluid_string_type_t type)
     return 0;
 }
 
+static bool fluid_is_dothex (const char *s)
+{
+    return (strchr (s, '.') != NULL);
+}
+
+static bool fluid_is_words (const char *s)
+{
+    return (strchr (s, '-') != NULL);
+}
+
+fluid_string_type_t fluid_string_detect_type (const char *s)
+{
+    /* N.B.: An F58 encoded FLUID may start with 'f', which also could
+     *  be true for dothex or words representations. Therefore, always
+     *  check for these encodings first, since F58 must not have '.'
+     *  or '-' characters, which distinguish dothex and mnemonic.
+     */
+    if (fluid_is_dothex (s))
+        return FLUID_STRING_DOTHEX;
+    if (fluid_is_words (s))
+        return FLUID_STRING_MNEMONIC;
+    if (fluid_is_f58 (s) > 0)
+        return FLUID_STRING_F58;
+    if (is_basemoji_string (s))
+        return FLUID_STRING_EMOJI;
+    return 0;
+}
+
+static bool is_trailing_space (const char *p)
+{
+    while (*p != '\0' && isspace (*p))
+        p++;
+    return (*p == '\0');
+}
+
+int fluid_parse (const char *s, fluid_t *fluidp)
+{
+    int base = 10;
+    unsigned long long l;
+    char *endptr;
+    fluid_string_type_t type;
+
+    if (s == NULL || s[0] == '\0') {
+        errno = EINVAL;
+        return -1;
+    }
+
+    /*  Skip leading whitespace
+     */
+    while (*s != '\0' && isspace (*s))
+        s++;
+
+    if ((type = fluid_string_detect_type (s)) != 0)
+        return fluid_decode (s, fluidp, type);
+
+    /* O/w, FLUID encoded as an integer, either base16 (prefix="0x")
+     *  or base10 (no prefix).
+     */
+    if (strstarts (s, "0x"))
+        base = 16;
+    errno = 0;
+    l = strtoull (s, &endptr, base);
+    if (errno != 0)
+        return -1;
+    /*  Ignore trailing whitespace */
+    if (!is_trailing_space(endptr)) {
+        errno = EINVAL;
+        return -1;
+    }
+    *fluidp = l;
+
+    if (fluid_validate (*fluidp) < 0)
+        return -1;
+
+    return 0;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libutil/fluid.h b/src/common/libutil/fluid.h
index d2d70bfb0496..a3719d40090a 100644
--- a/src/common/libutil/fluid.h
+++ b/src/common/libutil/fluid.h
@@ -22,21 +22,26 @@
 typedef enum {
     FLUID_STRING_DOTHEX = 1,    // x.x.x.x
     FLUID_STRING_MNEMONIC = 2,  // mnemonicode x-x-x--x-x-x
+    FLUID_STRING_F58 = 3,       // FLUID base58 enc: ƒXXXX or fXXXX
+    FLUID_STRING_EMOJI = 4,     // FLUID basemoji enc: 😪🏭🐭🍑👨
+    FLUID_STRING_F58_PLAIN = 5, // FLUID base58 enc: fXXXX
 } fluid_string_type_t;
 
 struct fluid_generator {
     uint16_t id;
-    uint64_t epoch;
     uint16_t seq;
-    uint64_t last_ds;
+    uint64_t clock_zero;        // local clock value at fluid_init()
+    uint64_t clock_offset;      // clock offset due to starting timestamp
+    uint64_t timestamp;
 };
 
 typedef uint64_t fluid_t;
 
-/* Returns 0 on success, -1 on failure.
+/* Initialize generator 'id' with starting 'timestamp'.
+ * Returns 0 on success, -1 on failure.
  * Failures include id out of range, clock_gettime() error.
  */
-int fluid_init (struct fluid_generator *gen, uint32_t id);
+int fluid_init (struct fluid_generator *gen, uint32_t id, uint64_t timestamp);
 
 /* Returns 0 on success, -1 on failure.
  * Failures include timestamp out of range, clock_gettime() error.
@@ -45,6 +50,15 @@ int fluid_init (struct fluid_generator *gen, uint32_t id);
  */
 int fluid_generate (struct fluid_generator *gen, fluid_t *fluid);
 
+/* Update and retrieve the internal timestamp.
+ * Returns 0 on success, -1 on failure.
+ */
+int fluid_save_timestamp (struct fluid_generator *gen, uint64_t *timestamp);
+
+/* Extract timestamp from a fluid.
+ */
+uint64_t fluid_get_timestamp (fluid_t fluid);
+
 /* Convert 'fluid' to NULL-terminated string 'buf' of specified type.
  * Return 0 on success, -1 on failure.
  */
@@ -57,6 +71,20 @@ int fluid_encode (char *buf, int bufsz, fluid_t fluid,
 int fluid_decode (const char *s, fluid_t *fluid,
                   fluid_string_type_t type);
 
+/* Attempt to detect the string type of encoded FLUID in `s`.
+ *  returns the string type or 0 if not one the defined encodings above.
+ *  (FLUID may still be encoded as integer in decimal or hex)
+ */
+fluid_string_type_t fluid_string_detect_type (const char *s);
+
+/* Convert NULL-terminated string 's' to 'fluid' by auto-detecting
+ *  the encoding in 's'.
+ * Supported encodings include any fluid_string_type_t, or an integer
+ *  in decimal or hexadecimal prefixed with "0x".
+ * Return 0 on success, -1 on failure.
+ */
+int fluid_parse (const char *s, fluid_t *fluid);
+
 #endif /* !_UTIL_FLUID_H */
 
 /*
diff --git a/src/common/libutil/fsd.c b/src/common/libutil/fsd.c
index 829621099827..64f81d3e0750 100644
--- a/src/common/libutil/fsd.c
+++ b/src/common/libutil/fsd.c
@@ -18,6 +18,7 @@
 #include <stdlib.h>
 
 #include "fsd.h"
+#include "ccan/str/str.h"
 
 static int is_invalid_duration (double d)
 {
@@ -25,8 +26,8 @@ static int is_invalid_duration (double d)
         case FP_NORMAL:     // [[fallthrough]]
         case FP_SUBNORMAL:  // [[fallthrough]]
         case FP_ZERO:       // [[fallthrough]]
-            break;          // OK
         case FP_INFINITE:   // [[fallthrough]]
+            break;          // OK
         case FP_NAN:        // [[fallthrough]]
         default:            // something else, bad
             errno = EINVAL;
@@ -51,33 +52,30 @@ int fsd_parse_duration (const char *s, double *dp)
     }
     d = strtod (s, &p);
 
-    if (is_invalid_duration (d)) {
-        return -1;
-    }
-
-    if (*p && *(p + 1)) {
-        errno = EINVAL;
+    if (is_invalid_duration (d))
         return -1;
-    }
 
     if (*p != '\0') {
-        unsigned int multiplier = 0;
-        switch (*p) {
-            case 0:
-            case 's':
-                multiplier = 1;
-                break;
-            case 'm':
-                multiplier = 60;
-                break;
-            case 'h':
-                multiplier = 60 * 60;
-                break;
-            case 'd':
-                multiplier = 60 * 60 * 24;
-                break;
+        double multiplier = 0.;
+
+        /*  units not allowed for inf/infinity
+         */
+        if (isinf (d)) {
+            errno = EINVAL;
+            return -1;
         }
-        if (multiplier == 0) {
+
+        if (streq (p, "ms"))
+            multiplier = .001;
+        else if (streq (p, "s"))
+            multiplier = 1;
+        else if (streq (p, "m"))
+            multiplier = 60;
+        else if (streq (p, "h"))
+            multiplier = 60 * 60;
+        else if (streq (p, "d"))
+            multiplier = 60 * 60 * 24;
+        else  {
             errno = EINVAL;
             return -1;
         }
@@ -87,20 +85,49 @@ int fsd_parse_duration (const char *s, double *dp)
     return 0;
 }
 
-int fsd_format_duration (char *buf, size_t len, double duration)
+int fsd_format_duration_ex (char *buf,
+                            size_t len,
+                            double duration,
+                            int precision)
 {
-    if (buf == NULL || len <= 0 || is_invalid_duration(duration)) {
+    if (buf == NULL || len <= 0) {
         errno = EINVAL;
         return -1;
     }
-    if (duration < 60.)
-        return snprintf (buf, len, "%gs", duration);
+
+    /*  First check for infinity special case
+     */
+    if (isinf (duration))
+        return snprintf (buf, len, "%s", "infinity");
+
+    if (is_invalid_duration (duration)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    /*  We'd rather present a result in seconds if possible, since that
+     *  is the base unit of FSD. However, if the duration is very small,
+     *  present in milliseconds since the result will be easier for a
+     *  human to read. E.g. 62.1ms vs 0.0621s, or more importantly
+     *  0.0123ms vs 1.23e-05s.
+     */
+    if (duration < 0.1 && duration != 0.)
+        return snprintf (buf, len, "%.*gms", precision, duration * 1000.);
+    else if (duration < 60.)
+        return snprintf (buf, len, "%.*gs", precision, duration);
     else if (duration < 60. * 60.)
-        return snprintf (buf, len, "%gm", duration / 60.);
+        return snprintf (buf, len, "%.*gm", precision, duration / 60.);
     else if (duration < 60. * 60. * 24.)
-        return snprintf (buf, len, "%gh", duration / (60. * 60.));
-    else
-        return snprintf (buf, len, "%gd", duration / (60. * 60. * 24.));
+        return snprintf (buf, len, "%.*gh", precision, duration / (60. * 60.));
+    else {
+        return snprintf (buf, len, "%.*gd", precision,
+                         duration / (60. * 60. * 24.));
+    }
+}
+
+int fsd_format_duration (char *buf, size_t len, double duration)
+{
+    return fsd_format_duration_ex (buf, len, duration, 6);
 }
 
 /*
diff --git a/src/common/libutil/fsd.h b/src/common/libutil/fsd.h
index b7de2933a8be..1e026f856c1f 100644
--- a/src/common/libutil/fsd.h
+++ b/src/common/libutil/fsd.h
@@ -23,6 +23,10 @@ int fsd_parse_duration (const char *s, double *dp);
  */
 int fsd_format_duration (char *buf, size_t len, double duration);
 
+/*  Same as above, but precision may be specified.
+ */
+int fsd_format_duration_ex (char *buf, size_t len, double duration, int prec);
+
 #endif /* !_UTIL_FSD_H */
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/common/libutil/grudgeset.c b/src/common/libutil/grudgeset.c
new file mode 100644
index 000000000000..7243f8f08797
--- /dev/null
+++ b/src/common/libutil/grudgeset.c
@@ -0,0 +1,135 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <string.h>
+#include <errno.h>
+
+#include "ccan/str/str.h"
+
+#include <jansson.h>
+
+struct grudgeset {
+    json_t *set;
+    json_t *grudges;
+};
+
+void grudgeset_destroy (struct grudgeset *gset)
+{
+    if (gset) {
+        int saved_errno = errno;
+        json_decref (gset->set);
+        json_decref (gset->grudges);
+        free (gset);
+        errno = saved_errno;
+    }
+}
+
+static struct grudgeset *grudgeset_create (void)
+{
+    struct grudgeset *gset = calloc (1, sizeof (*gset));
+    if (!gset
+        || !(gset->set = json_array ())
+        || !(gset->grudges = json_object ()))
+        goto error;
+    return gset;
+error:
+    grudgeset_destroy (gset);
+    return NULL;
+}
+
+int grudgeset_used (struct grudgeset *gset, const char *val)
+{
+    if (!gset || !val)
+        return 0;
+    return (json_object_get (gset->grudges, val) != NULL);
+}
+
+static int json_array_find (json_t *array, const char *val)
+{
+    size_t index;
+    json_t *entry;
+    json_array_foreach (array, index, entry) {
+        const char *s = json_string_value (entry);
+        if (val && streq (val, s)) {
+            return (int) index;
+        }
+    }
+    return -1;
+}
+
+int grudgeset_contains (struct grudgeset *gset, const char *val)
+{
+    if (!gset
+        || !val
+        || json_array_find (gset->set, val) < 0)
+        return 0;
+    return 1;
+}
+
+int grudgeset_add (struct grudgeset **gsetp, const char *val)
+{
+    json_t *o = NULL;
+
+    if (!gsetp || !val) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!*gsetp) {
+        if (!(*gsetp = grudgeset_create ()))
+            return -1;
+    }
+    if (grudgeset_used (*gsetp, val)) {
+        errno = EEXIST;
+        return -1;
+    }
+    if (!(o = json_string (val))
+        || json_array_append_new ((*gsetp)->set, o) < 0) {
+        json_decref (o);
+        errno = ENOMEM;
+        return -1;
+    }
+    /* val is guaranteed not to exist in set->grudges */
+    (void) json_object_set_new ((*gsetp)->grudges, val, json_null ());
+    return 0;
+}
+
+int grudgeset_remove (struct grudgeset *gset, const char *val)
+{
+    int index;
+    if (val == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (gset && (index = json_array_find (gset->set, val)) >= 0)
+        return json_array_remove (gset->set, index);
+    errno = ENOENT;
+    return -1;
+}
+
+json_t *grudgeset_tojson (struct grudgeset *gset)
+{
+    if (gset)
+        return gset->set;
+    return NULL;
+}
+
+int grudgeset_size (struct grudgeset *gset)
+{
+    if (gset == NULL)
+        return 0;
+    return json_array_size (gset->set);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/grudgeset.h b/src/common/libutil/grudgeset.h
new file mode 100644
index 000000000000..ed3442a95acc
--- /dev/null
+++ b/src/common/libutil/grudgeset.h
@@ -0,0 +1,56 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_GRUDGESET_H
+#define _UTIL_GRUDGESET_H
+
+/*  "grudge" set implementation. A set which only allows values to
+ *   be inserted once, even if they are subsequently removed.
+ */
+
+struct grudgeset;
+
+/* Add the string 'val' to set `*gsetp`. If *gsetp is NULL
+ * a new grudge set will be created with a single entry.
+ *
+ *  If a value was previously added to the set, then the append
+ *   will fail with -1 and errno set to EEXIST.
+ */
+int grudgeset_add (struct grudgeset **gsetp, const char *val);
+
+/*  Remove matching entry 'val' from gset. It is assumed there are
+ *   no duplicates.
+ */
+int grudgeset_remove (struct grudgeset *gset, const char *val);
+
+/*  Return number of elements in grudgeset
+ *  If gset is NULL then the size is 0.
+ */
+int grudgeset_size (struct grudgeset *gset);
+
+/*  Return 1 if val has been used in grudgeset, 0 otherwise
+ */
+int grudgeset_used (struct grudgeset *gset, const char *val);
+
+/*  Return 1 if set currently contains val, 0 otherwise
+ */
+int grudgeset_contains (struct grudgeset *gset, const char *val);
+
+/*  Return the current set in grudgeset 'gset'
+ *  The returned object is a JSON array.
+ *  Note: this should not be modified by caller, but is returned
+ *   non-const so it can be json_incref'd for nocopy use in message
+ *   payloads, etc.
+ */
+json_t *grudgeset_tojson (struct grudgeset *gset);
+
+void grudgeset_destroy (struct grudgeset *gset);
+
+#endif /* !_UTIL_GRUDGE_SET_H */
diff --git a/src/common/libutil/hola.c b/src/common/libutil/hola.c
new file mode 100644
index 000000000000..6483cb8514d9
--- /dev/null
+++ b/src/common/libutil/hola.c
@@ -0,0 +1,359 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* hola.c - hash of lists abstraciton
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <errno.h>
+
+#include "hola.h"
+
+struct hola {
+    zhashx_t *hash; // hash items are type (zlistx_t *)
+    zlistx_t *keys; // for iteration
+    zlistx_destructor_fn *list_destructor;
+    zlistx_duplicator_fn *list_duplicator;
+    zlistx_comparator_fn *list_comparator;
+    unsigned int keys_valid:1;
+    unsigned int flags;
+};
+
+void hola_destroy (struct hola *hola)
+{
+    if (hola) {
+        int saved_errno = errno;
+        zlistx_destroy (&hola->keys);
+        zhashx_destroy (&hola->hash);
+        errno = saved_errno;
+        free (hola);
+    }
+}
+
+struct hola *hola_create (int flags)
+{
+    struct hola *hola;
+
+    if ((flags & ~(HOLA_AUTOCREATE | HOLA_AUTODESTROY)) != 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(hola = calloc (1, sizeof (*hola))))
+        return NULL;
+    hola->flags = flags;
+    if (!(hola->hash = zhashx_new ()))
+        goto error;
+    zhashx_set_destructor (hola->hash, (zhashx_destructor_fn *)zlistx_destroy);
+    return hola;
+error:
+    hola_destroy (hola);
+    return NULL;
+}
+
+void hola_set_list_destructor (struct hola *hola, zlistx_destructor_fn fun)
+{
+    if (hola)
+        hola->list_destructor = fun;
+}
+void hola_set_list_duplicator (struct hola *hola, zlistx_duplicator_fn fun)
+{
+    if (hola)
+        hola->list_duplicator = fun;
+}
+void hola_set_list_comparator (struct hola *hola, zlistx_comparator_fn fun)
+{
+    if (hola)
+        hola->list_comparator = fun;
+}
+void hola_set_hash_key_destructor (struct hola *hola, zhashx_destructor_fn fun)
+{
+    if (hola)
+        zhashx_set_key_destructor (hola->hash, fun);
+}
+void hola_set_hash_key_duplicator (struct hola *hola, zhashx_duplicator_fn fun)
+{
+    if (hola)
+        zhashx_set_key_duplicator (hola->hash, fun);
+}
+void hola_set_hash_key_comparator (struct hola *hola, zhashx_comparator_fn fun)
+{
+    if (hola)
+        zhashx_set_key_comparator (hola->hash, fun);
+}
+void hola_set_hash_key_hasher (struct hola *hola, zhashx_hash_fn fun)
+{
+    if (hola)
+        zhashx_set_key_hasher (hola->hash, fun);
+}
+
+zlistx_t *hola_hash_lookup (struct hola *hola, const void *key)
+{
+    zlistx_t *l;
+
+    if (!hola || !key) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(l = zhashx_lookup (hola->hash, key))) {
+        errno = ENOENT;
+        return NULL;
+    }
+    return l;
+}
+
+static zlistx_t *hash_add (struct hola *hola, const void *key)
+{
+    zlistx_t *l;
+
+    if (!(l = zlistx_new ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    zlistx_set_destructor (l, hola->list_destructor);
+    zlistx_set_duplicator (l, hola->list_duplicator);
+    zlistx_set_comparator (l, hola->list_comparator);
+    if (zhashx_insert (hola->hash, key, l) < 0) {
+        zlistx_destroy (&l);
+        errno = EEXIST;
+        return NULL;
+    }
+    hola->keys_valid = 0;
+    return l;
+}
+
+int hola_hash_add (struct hola *hola, const void *key)
+{
+    if (!hola || !key) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (hash_add (hola, key) == NULL)
+        return -1;
+    return 0;
+}
+
+int hola_hash_delete (struct hola *hola, const void *key)
+{
+    if (!hola || !key) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!zhashx_lookup (hola->hash, key)) {
+        errno = ENOENT;
+        return -1;
+    }
+    zhashx_delete (hola->hash, key);
+    hola->keys_valid = 0;
+    return 0;
+}
+
+const void *hola_hash_first (struct hola *hola)
+{
+    const void *key = NULL;
+    if (hola) {
+        if (!hola->keys_valid) {
+            zlistx_destroy (&hola->keys);
+            if ((hola->keys = zhashx_keys (hola->hash)))
+                hola->keys_valid = 1;
+        }
+        if (hola->keys)
+            key = zlistx_first (hola->keys);
+    }
+    return key;
+}
+
+const void *hola_hash_next (struct hola *hola)
+{
+    const void *key = NULL;
+    if (hola && hola->keys) {
+        key = zlistx_next (hola->keys);
+    }
+    return key;
+}
+
+size_t hola_hash_size (struct hola *hola)
+{
+    return hola ? zhashx_size (hola->hash) : 0;
+}
+
+void *hola_list_add_end (struct hola *hola, const void *key, void *item)
+{
+    zlistx_t *l;
+    void *handle;
+
+    if (!hola || !key || !item) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(l = zhashx_lookup (hola->hash, key))) {
+        if ((hola->flags & HOLA_AUTOCREATE)) {
+            if (!(l = hash_add (hola, key)))
+                return NULL;
+        }
+        else {
+            errno = ENOENT;
+            return NULL;
+        }
+    }
+    if ((handle = zlistx_add_end (l, item)) == NULL) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    return handle;
+}
+
+void *hola_list_insert (struct hola *hola,
+                        const void *key,
+                        void *item,
+                        bool low_value)
+{
+    zlistx_t *l;
+    void *handle;
+
+    if (!hola || !key || !item) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(l = zhashx_lookup (hola->hash, key))) {
+        if ((hola->flags & HOLA_AUTOCREATE)) {
+            if (!(l = hash_add (hola, key)))
+                return NULL;
+        }
+        else {
+            errno = ENOENT;
+            return NULL;
+        }
+    }
+    if ((handle = zlistx_insert (l, item, low_value)) == NULL) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    return handle;
+}
+
+void *hola_list_find (struct hola *hola,
+                      const void *key,
+                      void *item)
+{
+    zlistx_t *l;
+    void *handle;
+
+    if (!hola || !key || !item) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(l = zhashx_lookup (hola->hash, key))
+        || !(handle = zlistx_find (l, item))) {
+        errno = ENOENT;
+        return NULL;
+    }
+    return handle;
+}
+
+int hola_list_delete (struct hola *hola, const void *key, void *handle)
+{
+    zlistx_t *l;
+
+    if (!hola || !key || !handle) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(l = zhashx_lookup (hola->hash, key))
+        || zlistx_delete (l, handle) < 0) {
+        errno = ENOENT;
+        return -1;
+    }
+    if ((hola->flags & HOLA_AUTODESTROY)) {
+        if (zlistx_size (l) == 0) {
+            zhashx_delete (hola->hash, key);
+            hola->keys_valid = 0;
+        }
+    }
+    return 0;
+}
+
+void *hola_list_first (struct hola *hola, const void *key)
+{
+    void *item = NULL;
+
+    if (hola && key) {
+        zlistx_t *l;
+        if ((l = zhashx_lookup (hola->hash, key)))
+            item = zlistx_first (l);
+    }
+    return item;
+}
+
+void *hola_list_next (struct hola *hola, const void *key)
+{
+    void *item = NULL;
+
+    if (hola && key) {
+        zlistx_t *l;
+        if ((l = zhashx_lookup (hola->hash, key)))
+            item = zlistx_next (l);
+    }
+    return item;
+}
+
+void *hola_list_prev (struct hola *hola, const void *key)
+{
+    void *item = NULL;
+
+    if (hola && key) {
+        zlistx_t *l;
+        if ((l = zhashx_lookup (hola->hash, key)))
+            item = zlistx_prev (l);
+    }
+    return item;
+}
+
+void *hola_list_last (struct hola *hola, const void *key)
+{
+    void *item = NULL;
+
+    if (hola && key) {
+        zlistx_t *l;
+        if ((l = zhashx_lookup (hola->hash, key)))
+            item = zlistx_last (l);
+    }
+    return item;
+}
+
+
+void *hola_list_cursor (struct hola *hola, const void *key)
+{
+    void *handle = NULL;
+
+    if (hola && key) {
+        zlistx_t *l;
+        if ((l = zhashx_lookup (hola->hash, key)))
+            handle = zlistx_cursor (l);
+    }
+    return handle;
+}
+
+size_t hola_list_size (struct hola *hola, const void *key)
+{
+    size_t size = 0;
+
+    if (hola && key) {
+        zlistx_t *l;
+        if ((l = zhashx_lookup (hola->hash, key)))
+            size = zlistx_size (l);
+    }
+    return size;
+}
+
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/hola.h b/src/common/libutil/hola.h
new file mode 100644
index 000000000000..a8c0944f19ae
--- /dev/null
+++ b/src/common/libutil/hola.h
@@ -0,0 +1,67 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_HOLA_H
+#define _UTIL_HOLA_H
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+/* Lists are not internally created/destroyed automatically by default.
+ * hola_hash_add() / hola_hash_delete() must be called to create/destroy them.
+ * These flags allow automatic behavior to be enabled at list creation.
+ */
+enum {
+    HOLA_AUTOCREATE = 1,    // create list on first addition
+    HOLA_AUTODESTROY = 2,   // destroy list on last removal
+};
+
+struct hola *hola_create (int flags);
+void hola_destroy (struct hola *hola);
+
+void hola_set_list_destructor (struct hola *hola, zlistx_destructor_fn fun);
+void hola_set_list_duplicator (struct hola *hola, zlistx_duplicator_fn fun);
+void hola_set_list_comparator (struct hola *hola, zlistx_comparator_fn fun);
+
+void hola_set_hash_key_destructor (struct hola *hola, zhashx_destructor_fn fun);
+void hola_set_hash_key_duplicator (struct hola *hola, zhashx_duplicator_fn fun);
+void hola_set_hash_key_comparator (struct hola *hola, zhashx_comparator_fn fun);
+void hola_set_hash_key_hasher (struct hola *hola, zhashx_hash_fn fun);
+
+const void *hola_hash_first (struct hola *hola);
+const void *hola_hash_next (struct hola *hola);
+int hola_hash_add (struct hola *hola, const void *key);
+int hola_hash_delete (struct hola *hola, const void *key);
+size_t hola_hash_size (struct hola *hola);
+zlistx_t *hola_hash_lookup (struct hola *hola, const void *key);
+
+// returns list item
+void *hola_list_first (struct hola *hola, const void *key);
+void *hola_list_next (struct hola *hola, const void *key);
+void *hola_list_prev (struct hola *hola, const void *key);
+void *hola_list_last (struct hola *hola, const void *key);
+
+// returns list handle
+void *hola_list_add_end (struct hola *hola, const void *key, void *item);
+void *hola_list_cursor (struct hola *hola, const void *key);
+void *hola_list_insert (struct hola *hola,
+                        const void *key,
+                        void *item,
+                        bool low_value);
+void *hola_list_find (struct hola *hola,
+                      const void *key,
+                      void *item);
+
+int hola_list_delete (struct hola *hola, const void *key, void *handle);
+size_t hola_list_size (struct hola *hola, const void *key);
+
+
+#endif /* !_UTIL_HOLA_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/intree.c b/src/common/libutil/intree.c
index 0af123509dc3..95c84372972f 100644
--- a/src/common/libutil/intree.c
+++ b/src/common/libutil/intree.c
@@ -20,6 +20,9 @@
 #include <libgen.h>    /* dirname(3) */
 #include <pthread.h>
 
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
+
 
 /*  Strip trailing ".libs", otherwise do nothing
  */
@@ -36,6 +39,52 @@ static char *strip_trailing_dot_libs (char *dir)
     return (dir);
 }
 
+#if __APPLE__
+#include <mach-o/dyld.h>
+static char *nsget_wrap (void)
+{
+    char *path = NULL;
+    uint32_t size = 0;
+
+    (void)_NSGetExecutablePath (path, &size); // get the required buffer size
+    if (!(path = calloc (1, size))
+        || _NSGetExecutablePath (path, &size) < 0) {
+        ERRNO_SAFE_WRAP (free, path);
+        return NULL;
+    }
+    return path;
+}
+static int executable_self (char *buf, size_t bufsize)
+{
+    char *path;
+    char *newpath = NULL;
+    int rc = -1;
+
+    if (!(path = nsget_wrap ())
+        || !(newpath = realpath (path, NULL))) {
+        goto done;
+    }
+    if (strlcpy (buf, newpath, bufsize) >= bufsize) {
+        errno = EOVERFLOW;
+        goto done;
+    }
+    rc = 0;
+done:
+    ERRNO_SAFE_WRAP (free, newpath);
+    ERRNO_SAFE_WRAP (free, path);
+    return rc;
+}
+#else
+static int executable_self (char *buf, size_t bufsize)
+{
+    ssize_t len = readlink ("/proc/self/exe", buf, bufsize - 1);
+    if (len < 0)
+        return -1;
+    buf[len] = '\0';
+    return 0;
+}
+#endif
+
 /*  Return directory containing this executable.
  */
 const char *executable_selfdir (void)
@@ -45,8 +94,7 @@ const char *executable_selfdir (void)
     static char *current_exe_dir = NULL;
     pthread_mutex_lock (&selfdir_lock);
     if (!current_exe_dir) {
-        memset (current_exe_path, 0, sizeof (current_exe_path));
-        if (readlink ("/proc/self/exe", current_exe_path, MAXPATHLEN - 1) < 0)
+        if (executable_self (current_exe_path, sizeof (current_exe_path)) < 0)
             goto out;
         current_exe_dir = strip_trailing_dot_libs (dirname (current_exe_path));
     }
@@ -80,7 +128,7 @@ static int is_intree (void)
             return 0;
         return -1;
     }
-    if (strncmp (builddir, selfdir, strlen (builddir)) == 0)
+    if (strstarts (selfdir, builddir))
         ret = 1;
     free (builddir);
     return ret;
diff --git a/src/common/libutil/ipaddr.c b/src/common/libutil/ipaddr.c
index 8160288c6210..a71c7f582956 100644
--- a/src/common/libutil/ipaddr.c
+++ b/src/common/libutil/ipaddr.c
@@ -11,45 +11,209 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <sys/param.h>
+#include <limits.h>
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <ifaddrs.h>
+#include <inttypes.h>
 #include <netdb.h>
 #include <stdarg.h>
 #include <string.h>
+#include <stdbool.h>
 #include <stdio.h>
-#include <argz.h>
+
+#include "ccan/str/str.h"
+#include "errprintf.h"
 
 #include "log.h"
+#include "cidr.h"
 #include "ipaddr.h"
 
-int ipaddr_getprimary (char *buf, int len, char *errstr, int errstrsz)
+/* Identify an IPv6 link-local address so it can be skipped.
+ * The leftmost 10 bits of the 128 bit address will be 0xfe80.
+ * At present, Flux cannot use link-local addresses for PMI bootstrap,
+ * as the scope (e.g. %index or %iface-name) is not valid off the local node.
+ * See also flux-framework/flux-core#3378
+ */
+static bool is_linklocal6 (struct sockaddr_in6 *sin)
+{
+    if (sin->sin6_addr.s6_addr[0] == 0xfe
+            && (sin->sin6_addr.s6_addr[1] & 0xc0) == 0x80)
+        return true;
+    return false;
+}
+
+static int getprimary_iface4 (char *buf, size_t size, flux_error_t *error)
 {
-    char hostname[HOST_NAME_MAX + 1];
+    const char *path = "/proc/net/route";
+    FILE *f;
+    unsigned long dest;
+    char line[256];
+
+    if (!(f = fopen (path, "r"))) {
+        errprintf (error, "%s: %s", path, strerror (errno));
+        return -1;
+    }
+    while (fgets (line, sizeof (line), f)) {
+        char fmt[24];
+        /*  Format guaranteed to fit in 24 bytes (assuming 10 digit max) */
+        (void) snprintf (fmt,
+                         sizeof(fmt),
+                         "%%%us\t%%lx",
+                         (unsigned) size - 1);
+        if (sscanf (line, fmt, buf, &dest) == 2 && dest == 0) {
+            fclose (f);
+            return 0;
+        }
+    }
+    fclose (f);
+    errprintf (error, "%s: no default route", path);
+    return -1;
+}
+
+static struct ifaddrs *find_ifaddr (struct ifaddrs *ifaddr,
+                                    const char *name,
+                                    int family)
+{
+    struct ifaddrs *ifa;
+    struct cidr4 cidr;
+
+    for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
+        if (streq (ifa->ifa_name, name)
+            && ifa->ifa_addr != NULL
+            && ifa->ifa_addr->sa_family == family
+            && (ifa->ifa_addr->sa_family != AF_INET6
+                || !is_linklocal6 ((struct sockaddr_in6 *)ifa->ifa_addr)))
+            break;
+    }
+    if (ifa)
+        return ifa;
+
+    /* We didn't find an exact interface match for 'name' above, so
+     * try parsing 'name' as a CIDR and match the interface address.
+     * Only ipv4 is supported at this point.
+     */
+    if (family == AF_INET6 || cidr_parse4 (&cidr, name) < 0)
+        return NULL;
+
+    for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
+        if (ifa->ifa_addr == NULL || ifa->ifa_addr->sa_family != AF_INET)
+            continue;
+        struct sockaddr_in *sin = (struct sockaddr_in *)ifa->ifa_addr;
+        if (cidr_match4 (&cidr, &sin->sin_addr))
+            break;
+    }
+
+    return ifa;
+}
+
+static int getnamed_ifaddr (char *buf,
+                            int len,
+                            const char *name,
+                            int prefer_family,
+                            flux_error_t *error)
+{
+    struct ifaddrs *ifaddr;
+    struct ifaddrs *ifa;
+    int e;
+
+    if (getifaddrs (&ifaddr) < 0) {
+        errprintf (error, "getifaddrs: %s", strerror (errno));
+        return -1;
+    }
+    if (!(ifa = find_ifaddr (ifaddr, name, prefer_family))) {
+        prefer_family = (prefer_family == AF_INET) ? AF_INET6 : AF_INET;
+        ifa = find_ifaddr (ifaddr, name, prefer_family);
+    }
+    if (!ifa) {
+        errprintf (error, "could not find address associated with %s", name);
+        freeifaddrs (ifaddr);
+        return -1;
+    }
+    e = getnameinfo (ifa->ifa_addr,
+                     ifa->ifa_addr->sa_family == AF_INET
+                         ? sizeof (struct sockaddr_in)
+                         : sizeof (struct sockaddr_in6),
+                     buf, // <== result copied here
+                     len,
+                     NULL,
+                     0,
+                     NI_NUMERICHOST);
+    if (e) {
+        errprintf (error, "getnameinfo: %s", gai_strerror (e));
+        freeifaddrs (ifaddr);
+        return -1;
+    }
+    freeifaddrs (ifaddr);
+    return 0;
+}
+
+static int getprimary_ifaddr (char *buf,
+                              int len,
+                              int prefer_family,
+                              flux_error_t *error)
+{
+    char name[64];
+    if (getprimary_iface4 (name, sizeof (name), error) < 0)
+        return -1;
+    return getnamed_ifaddr (buf, len, name, prefer_family, error);
+}
+
+static struct addrinfo *find_addrinfo (struct addrinfo *addrinfo, int family)
+{
+    struct addrinfo *ai;
+
+    for (ai = addrinfo; ai != NULL; ai = ai->ai_next) {
+        if (ai->ai_family == family
+            && (ai->ai_family != AF_INET6
+                || !is_linklocal6 ((struct sockaddr_in6 *)ai->ai_addr)))
+            break;
+    }
+    return ai;
+}
+
+static int getprimary_hostaddr (char *buf,
+                                int len,
+                                int prefer_family,
+                                flux_error_t *error)
+{
+    char hostname[_POSIX_HOST_NAME_MAX + 1];
     struct addrinfo hints, *res = NULL;
+    struct addrinfo *rp;
     int e;
 
     if (gethostname (hostname, sizeof (hostname)) < 0) {
-        if (errstr)
-            snprintf (errstr, errstrsz, "gethostname: %s", strerror (errno));
+        errprintf (error, "gethostname: %s", strerror (errno));
         return -1;
     }
     memset (&hints, 0, sizeof (hints));
-    hints.ai_family = PF_UNSPEC;
+    hints.ai_family = AF_UNSPEC;
     hints.ai_socktype = SOCK_STREAM;
 
-    if ((e = getaddrinfo (hostname, NULL, &hints, &res)) || res == NULL) {
-        if (errstr)
-            snprintf (errstr, errstrsz, "getaddrinfo %s: %s",
-                      hostname, gai_strerror (e));
+    e = getaddrinfo (hostname, NULL, &hints, &res);
+    if (e) {
+        errprintf (error, "getaddrinfo %s: %s", hostname, gai_strerror (e));
+        return -1;
+    }
+    if (!(rp = find_addrinfo (res, prefer_family))) {
+        prefer_family = (prefer_family == AF_INET) ? AF_INET6 : AF_INET;
+        rp = find_addrinfo (res, prefer_family);
+    }
+    if (!rp) {
+        errprintf (error, "could not find address of %s", hostname);
+        freeaddrinfo (res);
         return -1;
     }
-    if ((e = getnameinfo (res->ai_addr, res->ai_addrlen, buf, len,
-                          NULL, 0, NI_NUMERICHOST))) {
-        if (errstr)
-            snprintf (errstr, errstrsz, "getnameinfo: %s", gai_strerror (e));
+    e = getnameinfo (rp->ai_addr,
+                     rp->ai_addrlen,
+                     buf, // <== result copied here
+                     len,
+                     NULL,
+                     0,
+                     NI_NUMERICHOST);
+    if (e) {
+        errprintf (error, "getnameinfo: %s", gai_strerror (e));
         freeaddrinfo (res);
         return -1;
     }
@@ -57,48 +221,25 @@ int ipaddr_getprimary (char *buf, int len, char *errstr, int errstrsz)
     return 0;
 }
 
-int ipaddr_getall (char **addrs, size_t *addrssz, char *errstr, int errstrsz)
+int ipaddr_getprimary (char *buf,
+                       int len,
+                       int flags,
+                       const char *interface,
+                       flux_error_t *error)
 {
-    struct ifaddrs *ifaddr = NULL;
-    struct ifaddrs *ifa;
-    int family;
-    int e;
-    char host[NI_MAXHOST];
+    int prefer_family = (flags & IPADDR_V6) ? AF_INET6 : AF_INET;
     int rc = -1;
 
-    if (getifaddrs (&ifaddr) < 0) {
-        if (errstr)
-            snprintf (errstr, errstrsz, "getifaddrs: %s", strerror (errno));
-        return -1;
+    if (interface) {
+        rc = getnamed_ifaddr (buf, len, interface, prefer_family, error);
     }
-    /* Ref: getifaddrs(3) example */
-    for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
-        if (ifa->ifa_addr == NULL)
-            continue;
-        family = ifa->ifa_addr->sa_family;
-        if (family != AF_INET && family != AF_INET6)
-            continue;
-        e = getnameinfo (ifa->ifa_addr,
-                         family == AF_INET ? sizeof (struct sockaddr_in)
-                                           : sizeof (struct sockaddr_in6),
-                         host, NI_MAXHOST,
-                         NULL, 0, NI_NUMERICHOST);
-        if (e != 0) {
-            if (errstr)
-                snprintf (errstr, errstrsz, "getnameinfo: %s",
-                          gai_strerror (e));
-            goto done;
-        }
-        if ((e = argz_add (addrs, addrssz, host)) != 0) {
-            if (errstr)
-                snprintf (errstr, errstrsz, "argz_add: %s", strerror (errno));
-            goto done;
+    else {
+        if (!(flags & IPADDR_HOSTNAME))
+            rc = getprimary_ifaddr (buf, len, prefer_family, error);
+        if (rc < 0) {
+            rc = getprimary_hostaddr (buf, len, prefer_family, error);
         }
     }
-    rc = 0;
-done:
-    if (ifaddr)
-        freeifaddrs (ifaddr);
     return rc;
 }
 
diff --git a/src/common/libutil/ipaddr.h b/src/common/libutil/ipaddr.h
index 1c9bfe491833..5215b6ef2f6c 100644
--- a/src/common/libutil/ipaddr.h
+++ b/src/common/libutil/ipaddr.h
@@ -8,28 +8,48 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#ifndef _UTIL_GETIP_H
-#define _UTIL_GETIP_H
+#ifndef _UTIL_IPADDR_H
+#define _UTIL_IPADDR_H
 
-#include <sys/types.h>
+#include <sys/socket.h> // for AF_INET, AF_INET6
+#include "src/common/libflux/types.h" // for flux_error_t
 
-/* Get ip address associated with primary hostname.
- * Return it as a string in buf (up to len bytes, always null terminated)
- * Return 0 on success, -1 on error with error message written to errstr
- * if non-NULL.
- */
-int ipaddr_getprimary (char *buf, int len, char *errstr, int errstrsz);
+enum {
+    IPADDR_V6 = 1,
+    IPADDR_HOSTNAME = 2,
+};
 
-
-/* Get a list of all stringified ip addresses associated with interfaces on
- * the local host.  The result is appended to 'addrs', 'addrsz', an argz list
- * that must be initialized as described in argz_add(3).  Both AF_INET
- * and AF_INET6 address families are included in the list.  Return 0 on
- * success, -1 on error with error message written to errstr if non-NULL.
+/* Guess at a usable network address for the local node using one
+ * of these methods:
+ * 1. Find the interface associated with the default route, then
+ *    look up address of that interface.
+ * 2. Look up address associated with the hostname
+ * 3. Look up address associated with a specific interface.
+ *
+ * Main use case: determine bind address for a PMI-bootstrapped flux broker.
+ *
+ * Flags and the optional 'interface' param alter the default behavior:
+ *
+ * IPADDR_IPV6
+ *   if set, IPv6 addresses are preferred, with fallback to IPv4
+ *   if unset, IPv4 addresses are preferred, with fallback to IPv6
+ * IPADDR_HOSTNAME
+ *   if set, only method 2 is tried above
+ *   if unset, first method 1 is tried, then if that fails, method 2 is tried
+ * 'interface'
+ *   if set, only method 3 is tried above
+ *
+ * Return address as a string in buf (up to len bytes, always null terminated)
+ * Return 0 on success, -1 on error with error message written to 'error'
+ * if non-NULL.
  */
-int ipaddr_getall (char **addrs, size_t *addrssz, char *errstr, int errstrsz);
+int ipaddr_getprimary (char *buf,
+                       int len,
+                       int flags,
+                       const char *interface,
+                       flux_error_t *error);
 
-#endif /* !_UTIL_GETIP_H */
+#endif /* !_UTIL_IPADDR_H */
 
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/common/libutil/iterators.h b/src/common/libutil/iterators.h
index 03f787fe0053..72ddc67ac3e9 100644
--- a/src/common/libutil/iterators.h
+++ b/src/common/libutil/iterators.h
@@ -11,6 +11,8 @@
 #ifndef FLUX_ITERATORS_H
 #define FLUX_ITERATORS_H
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
 #define FOREACH_ZLIST(LIST, VAR)   \
     for((VAR) = zlist_first(LIST); \
             VAR;                   \
diff --git a/src/common/libutil/jpath.c b/src/common/libutil/jpath.c
new file mode 100644
index 000000000000..083ce8cc509c
--- /dev/null
+++ b/src/common/libutil/jpath.c
@@ -0,0 +1,258 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <string.h>
+#include <jansson.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
+
+#include "jpath.h"
+
+static int update_object_recursive (json_t *orig, json_t *val)
+{
+    const char *key;
+    json_t *value;
+
+    json_object_foreach (val, key, value) {
+        json_t *orig_value = json_object_get (orig, key);
+
+        if (json_is_object (value)) {
+            if (!json_is_object (orig_value)) {
+                json_t *o = json_object ();
+                if (!o || json_object_set_new (orig, key, o) < 0) {
+                    errno = ENOMEM;
+                    json_decref (o);
+                    return -1;
+                }
+                orig_value = o;
+            }
+            if (update_object_recursive (orig_value, value) < 0)
+                return -1;
+        }
+        else if (json_object_set (orig, key, value) < 0) {
+            errno = ENOMEM;
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static int jpath_set_destructive (json_t *o,
+                                  int replace,
+                                  char *path,
+                                  json_t *val)
+{
+    char *cp;
+    json_t *dir;
+
+    if ((cp = strchr (path, '.'))) {
+        *cp++ = '\0';
+        if (strlen (path) == 0) {
+            errno = EINVAL;
+            return -1;
+        }
+        if (!(dir = json_object_get (o, path))) {
+            if (!(dir = json_object ()))
+                goto nomem;
+            if (json_object_set_new (o, path, dir) < 0) {
+                json_decref (dir);
+                goto nomem;
+            }
+        }
+        return jpath_set_destructive (dir, replace, cp, val);
+    }
+
+    if (strlen (path) == 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (replace || !(dir = json_object_get (o, path))) {
+        if (json_object_set (o, path, val) < 0)
+            goto nomem;
+    }
+    else {
+        if (update_object_recursive (dir, val) < 0)
+            goto nomem;
+    }
+    return 0;
+nomem:
+    errno = ENOMEM;
+    return -1;
+}
+
+static int jpath_del_destructive (json_t *o, char *path)
+{
+    char *cp;
+    json_t *dir;
+
+    if ((cp = strchr (path, '.'))) {
+        *cp++ = '\0';
+        if (strlen (path) == 0) {
+            errno = EINVAL;
+            return -1;
+        }
+        if (!(dir = json_object_get (o, path)))
+            return 0;
+        return jpath_del_destructive (dir, cp);
+    }
+
+    if (strlen (path) == 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    (void)json_object_del (o, path);
+    return 0;
+}
+
+static json_t *jpath_get_destructive (json_t *o, char *path)
+{
+    char *cp;
+    json_t *dir;
+    json_t *val;
+
+    if ((cp = strchr (path, '.'))) {
+        *cp++ = '\0';
+        if (strlen (path) == 0) {
+            errno = EINVAL;
+            return NULL;
+        }
+        if (!(dir = json_object_get (o, path))) {
+            errno = ENOENT;
+            return NULL;
+        }
+        return jpath_get_destructive (dir, cp);
+    }
+
+    if (strlen (path) == 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(val = json_object_get (o, path))) {
+        errno = ENOENT;
+        return NULL;
+    }
+    return val;
+}
+
+static int jpath_do_set (json_t *o, int replace, const char *path, json_t *val)
+{
+    char *cpy;
+    int rc;
+
+    if (!o || !path || !val) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(cpy = strdup (path)))
+        return -1;
+    rc = jpath_set_destructive (o, replace, cpy, val);
+    ERRNO_SAFE_WRAP (free, cpy);
+    return rc;
+
+}
+
+int jpath_set (json_t *o, const char *path, json_t *val)
+{
+    return jpath_do_set (o, 1, path, val);
+}
+
+json_t * jpath_set_new (json_t *o, const char *path, json_t *val)
+{
+    json_t *new = NULL;
+
+    if (!path || !val) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!o) {
+        if (!(new = json_object ())) {
+            errno = ENOMEM;
+            return NULL;
+        }
+        o = new;
+    }
+    if (jpath_set (o, path, val) < 0) {
+        ERRNO_SAFE_WRAP (json_decref, new);
+        return NULL;
+    }
+    /*  Steal reference to val */
+    json_decref (val);
+    return o;
+}
+
+int jpath_update (json_t *o, const char *path, json_t *val)
+{
+    /* Special case, allow "." to update current object */
+    if (streq (path, "."))
+        return update_object_recursive (o, val);
+    return jpath_do_set (o, 0, path, val);
+}
+
+int jpath_del (json_t *o, const char *path)
+{
+    char *cpy;
+    int rc;
+
+    if (!o || !path) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(cpy = strdup (path)))
+        return -1;
+    rc = jpath_del_destructive (o, cpy);
+    ERRNO_SAFE_WRAP (free, cpy);
+    return rc;
+}
+
+json_t *jpath_get (json_t *o, const char *path)
+{
+    char *cpy;
+    json_t *ret;
+
+    if (!o || !path) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(cpy = strdup (path)))
+        return NULL;
+    ret = jpath_get_destructive (o, cpy);
+    ERRNO_SAFE_WRAP (free, cpy);
+    return ret;
+}
+
+int jpath_clear_null (json_t *o)
+{
+    const char *key;
+    json_t *value;
+    void *tmp;
+
+    if (!o) {
+        errno = EINVAL;
+        return -1;
+    }
+    json_object_foreach_safe (o, tmp, key, value)  {
+        if (json_is_object (value)) {
+            if (jpath_clear_null (value) < 0)
+                return -1;
+            if (json_object_size (value) == 0)
+                (void) json_object_del (o, key); /* ignore key-not-found */
+        }
+        else if (json_is_null (value))
+            (void) json_object_del (o, key); /* ignore key-not-found */
+    }
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/jpath.h b/src/common/libutil/jpath.h
new file mode 100644
index 000000000000..5d61c6163a6e
--- /dev/null
+++ b/src/common/libutil/jpath.h
@@ -0,0 +1,49 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_JPATH_H
+#define _UTIL_JPATH_H
+
+#include <jansson.h>
+
+/* Set 'path' to 'val' in object 'o', taking a reference on 'val'.
+ * Return 0 on success, -1 on failure with errno set.
+ */
+int jpath_set (json_t *o, const char *path, json_t *val);
+
+/* Like japth_set(), but if 'o' is NULL create an empty object
+ * and add 'path'. Also doesn't take a reference on 'val'.
+ * Returns result or NULL on failure.
+ */
+json_t * jpath_set_new (json_t *o, const char *path, json_t *val);
+
+/* Update 'path' in object 'o' with 'val', taking a reference on 'val'.
+ * As a special case, a 'path' of '.' updates 'o' with 'val'.
+ * Return 0 on success, -1 on failure with errno set.
+ */
+int jpath_update (json_t *o, const char *path, json_t *val);
+
+/* Delete 'path' from object 'o'.
+ * Return 0 on success, -1 on failure with errno set.
+ */
+int jpath_del (json_t *o, const char *path);
+
+/* Get 'path' from object 'o', without taking a reference on returned value.
+ * Return value on success, NULL on failure with errno set.
+ */
+json_t *jpath_get (json_t *o, const char *path);
+
+/* Delete all values set to JSON null recursively in 'o'.
+ */
+int jpath_clear_null (json_t *o);
+
+#endif /* !_UTIL_JPATH_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/llog.h b/src/common/libutil/llog.h
new file mode 100644
index 000000000000..55ccdc8532ca
--- /dev/null
+++ b/src/common/libutil/llog.h
@@ -0,0 +1,232 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Flux internal library logging interface.
+ *
+ * Mostly taken from libtsm shl-llog.h, with updates for Flux
+ * typedef naming styles.
+ *
+ * https://www.freedesktop.org/wiki/Software/kmscon/libtsm/
+ *
+ * SHL - Library Log/Debug Interface
+ *
+ * Copyright (c) 2010-2013 David Herrmann <dh.herrmann@gmail.com>
+ * Dedicated to the Public Domain
+ *
+ * libtsm COPYRIGHT notice:
+
+This software is licensed under the terms of the MIT license. Please see each
+source file for the related copyright notice and license.
+
+If a file does not contain a copyright notice, the following license shall
+apply:
+
+	Copyright (c) 2011-2013 David Herrmann <dh.herrmann@gmail.com>
+
+	Permission is hereby granted, free of charge, to any person obtaining
+	a copy of this software and associated documentation files
+	(the "Software"), to deal in the Software without restriction, including
+	without limitation the rights to use, copy, modify, merge, publish,
+	distribute, sublicense, and/or sell copies of the Software, and to
+	permit persons to whom the Software is furnished to do so, subject to
+	the following conditions:
+
+	The above copyright notice and this permission notice shall be included
+	in all copies or substantial portions of the Software.
+
+	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+	OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+	MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+	IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+	CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+	TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+	SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ */
+
+/*
+ * Library Log/Debug Interface
+ * Libraries should always avoid producing side-effects. This includes writing
+ * log-messages of any kind. However, you often don't want to disable debugging
+ * entirely, therefore, the core objects often contain a pointer to a function
+ * which performs logging. If that pointer is NULL (default), logging is
+ * disabled.
+ *
+ * This header should never be installed into the system! This is _no_ public
+ * header. Instead, copy it into your application if you want and use it there.
+ * Your public library API should include something like this:
+ *
+ *   typedef void (*MYPREFIX_log_t) (void *data,
+ *                                   const char *file,
+ *                                   int line,
+ *                                   const char *func,
+ *                                   const char *subs,
+ *                                   unsigned int sev,
+ *                                   const char *format,
+ *                                   va_list args);
+ *
+ * And then the user can supply such a function when creating a new context
+ * object of your library or simply supply NULL. Internally, you have a field of
+ * type "MYPREFIX_log_t llog" in your main structure. If you pass this to the
+ * convenience helpers like llog_dbg(), llog_warn() etc. it will automatically
+ * use the "llog" field to print the message. If it is NULL, nothing is done.
+ *
+ * The arguments of the log-function are defined as:
+ *   data: User-supplied data field that is passed straight through.
+ *   file: Zero terminated string of the file-name where the log-message
+ *         occurred. Can be NULL.
+ *   line: Line number of @file where the message occurred. Set to 0 or smaller
+ *         if not available.
+ *   func: Function name where the log-message occurred. Can be NULL.
+ *   subs: Subsystem where the message occurred (zero terminated). Can be NULL.
+ *   sev: Severity of log-message. An integer between 0 and 7 as defined below.
+ *        These are identical to the linux-kernel severities so there is no need
+ *        to include these in your public API. Every app can define them
+ *        themselves, if they need it.
+ *   format: Format string. Must not be NULL.
+ *   args: Argument array
+ *
+ * The user should also be able to optionally provide a data field which is
+ * always passed unmodified as first parameter to the log-function. This allows
+ * to add context to the logger.
+ */
+
+#ifndef _UTIL_LLOG_H
+#define _UTIL_LLOG_H
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include "errno_safe.h"
+
+enum llog_severity {
+        LLOG_FATAL = 0,
+        LLOG_ALERT = 1,
+        LLOG_CRITICAL = 2,
+        LLOG_ERROR = 3,
+        LLOG_WARNING = 4,
+        LLOG_NOTICE = 5,
+        LLOG_INFO = 6,
+        LLOG_DEBUG = 7,
+        LLOG_SEV_NUM,
+};
+
+typedef void (*llog_writer_f) (void *arg,
+                               const char *file,
+                               int line,
+                               const char *func,
+                               const char *subsys,
+                               int level,
+                               const char *fmt,
+                               va_list args);
+
+static inline  __attribute__((format(printf, 8, 9)))
+void llog_format (llog_writer_f llog,
+                               void *arg,
+                               const char *file,
+                               int line,
+                               const char *func,
+                               const char *subsys,
+                               int level,
+                               const char *fmt,
+                               ...)
+{
+    if (llog) {
+        int saved_errno = errno;
+        va_list ap;
+        va_start (ap, fmt);
+        errno = saved_errno;
+        llog (arg, file, line, func, subsys, level, fmt, ap);
+        va_end (ap);
+        errno = saved_errno;
+    }
+}
+
+#ifndef LLOG_SUBSYSTEM
+static const char *LLOG_SUBSYSTEM __attribute__((__unused__));
+#endif
+
+#define LLOG_DEFAULT __FILE__, __LINE__, __func__, LLOG_SUBSYSTEM
+
+#define llog_printf(obj, sev, format, ...) \
+        llog_format((obj)->llog, \
+                    (obj)->llog_data, \
+                    LLOG_DEFAULT, \
+                    (sev), \
+                    (format), \
+                    ##__VA_ARGS__)
+#define llog_dprintf(obj, data, sev, format, ...) \
+        llog_format((obj), \
+                    (data), \
+                    LLOG_DEFAULT, \
+                    (sev), \
+                    (format), \
+                    ##__VA_ARGS__)
+
+static inline __attribute__((format(printf, 4, 5)))
+void llog_dummyf(llog_writer_f llog, void *data, unsigned int sev,
+                 const char *format, ...)
+{
+}
+
+/*
+ * Helpers
+ * They pick up all the default values and submit the message to the
+ * llog-subsystem. The llog_debug() function will discard the message unless
+ * LLOG_ENABLE_DEBUG is defined.
+ */
+#ifdef LLOG_ENABLE_DEBUG
+        #define llog_ddebug(obj, data, format, ...) \
+                llog_dprintf((obj), (data), LLOG_DEBUG, (format), ##__VA_ARGS__)
+        #define llog_debug(obj, format, ...) \
+                llog_ddebug((obj)->llog, (obj)->llog_data, (format), ##__VA_ARGS__)
+#else
+        #define llog_ddebug(obj, data, format, ...) \
+                llog_dummyf((obj), (data), LLOG_DEBUG, (format), ##__VA_ARGS__)
+        #define llog_debug(obj, format, ...) \
+                llog_ddebug((obj)->llog, (obj)->llog_data, (format), ##__VA_ARGS__)
+#endif
+
+
+#define llog_info(obj, format, ...) \
+        llog_printf((obj), LLOG_INFO, (format), ##__VA_ARGS__)
+#define llog_dinfo(obj, data, format, ...) \
+        llog_dprintf((obj), (data), LLOG_INFO, (format), ##__VA_ARGS__)
+#define llog_notice(obj, format, ...) \
+        llog_printf((obj), LLOG_NOTICE, (format), ##__VA_ARGS__)
+#define llog_dnotice(obj, data, format, ...) \
+        llog_dprintf((obj), (data), LLOG_NOTICE, (format), ##__VA_ARGS__)
+#define llog_warning(obj, format, ...) \
+        llog_printf((obj), LLOG_WARNING, (format), ##__VA_ARGS__)
+#define llog_dwarning(obj, data, format, ...) \
+        llog_dprintf((obj), (data), LLOG_WARNING, (format), ##__VA_ARGS__)
+#define llog_error(obj, format, ...) \
+        llog_printf((obj), LLOG_ERROR, (format), ##__VA_ARGS__)
+#define llog_derror(obj, data, format, ...) \
+        llog_dprintf((obj), (data), LLOG_ERROR, (format), ##__VA_ARGS__)
+#define llog_critical(obj, format, ...) \
+        llog_printf((obj), LLOG_CRITICAL, (format), ##__VA_ARGS__)
+#define llog_dcritical(obj, data, format, ...) \
+        llog_dprintf((obj), (data), LLOG_CRITICAL, (format), ##__VA_ARGS__)
+#define llog_alert(obj, format, ...) \
+        llog_printf((obj), LLOG_ALERT, (format), ##__VA_ARGS__)
+#define llog_dalert(obj, data, format, ...) \
+        llog_dprintf((obj), (data), LLOG_ALERT, (format), ##__VA_ARGS__)
+#define llog_fatal(obj, format, ...) \
+        llog_printf((obj), LLOG_FATAL, (format), ##__VA_ARGS__)
+#define llog_dfatal(obj, data, format, ...) \
+        llog_dprintf((obj), (data), LLOG_FATAL, (format), ##__VA_ARGS__)
+
+
+#endif /* !_UTIL_LLOG_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/log.c b/src/common/libutil/log.c
index aff81550e9c7..d31415f96304 100644
--- a/src/common/libutil/log.c
+++ b/src/common/libutil/log.c
@@ -22,8 +22,8 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <signal.h>
-#include <zmq.h>
 
+#include "basename.h"
 #include "log.h"
 
 extern char *__progname;
@@ -35,7 +35,7 @@ log_init (char *p)
     if (!p)
         prog = __progname;
     else
-        prog = basename (p);
+        prog = basename_simple (p);
 }
 
 void
@@ -48,16 +48,7 @@ _verr (int errnum, const char *fmt, va_list ap)
 {
     char *msg = NULL;
     char buf[128];
-    const char *s;
-
-    /* zeromq-4.2.1 reports EHOSTUNREACH as "Host unreachable",
-     * but "No route to host" is canonical on Linux and we have some
-     * tests that depend on it, so remap here.
-     */
-    if (errnum == EHOSTUNREACH)
-        s = "No route to host";
-    else
-        s = zmq_strerror (errnum);
+    const char *s = strerror (errnum);
 
     if (!prog)
         log_init (NULL);
diff --git a/src/common/libutil/lru_cache.c b/src/common/libutil/lru_cache.c
index 416acff00a11..8d132b2c63cd 100644
--- a/src/common/libutil/lru_cache.c
+++ b/src/common/libutil/lru_cache.c
@@ -13,8 +13,11 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <stdlib.h>
 #include <unistd.h>
-#include <czmq.h>
+#include <errno.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "lru_cache.h"
 
diff --git a/src/common/libutil/macros.h b/src/common/libutil/macros.h
index 44dbe987da03..96295664c8bc 100644
--- a/src/common/libutil/macros.h
+++ b/src/common/libutil/macros.h
@@ -15,13 +15,4 @@
 #define STRINGIFY(X) REAL_STRINGIFY (X)
 #define SIZEOF_FIELD(type, field) sizeof (((type *)0)->field)
 
-/* Maximum size of buffer needed to decode a base64 string of length 'x',
- * where 4 characters are decoded into 3 bytes.  Add 3 bytes to ensure a
- * partial 4-byte chunk will be accounted for during integer division.
- * This size is safe for use with all (4) libsodium base64 variants.
- * N.B. unlike @dun's base64 implementation from the munge project,
- * this size does not include room for a \0 string terminator.
- */
-#define BASE64_DECODE_SIZE(x) ((((x) + 3) / 4) * 3)
-
 #endif
diff --git a/src/common/libutil/mnemonic.c b/src/common/libutil/mnemonic.c
index 08f19bcb6b56..8ec57fa803d8 100644
--- a/src/common/libutil/mnemonic.c
+++ b/src/common/libutil/mnemonic.c
@@ -265,7 +265,7 @@ mn_decode_word_index (mn_index index, void *vdest, int destsize, int *offset)
       return *offset;
     }
 
-  groupofs = *offset & ~3;	/* Offset of 4 byte group containing offet */
+  groupofs = *offset & ~3;	/* Offset of 4 byte group containing offset */
   x = 0;
   for (i = 0; i < 4; i++)
     if (groupofs + i < destsize)	/* Ignore any bytes outside buffer */
diff --git a/src/common/libutil/msglist.c b/src/common/libutil/msglist.c
deleted file mode 100644
index e71468d0621e..000000000000
--- a/src/common/libutil/msglist.c
+++ /dev/null
@@ -1,181 +0,0 @@
-/************************************************************\
- * Copyright 2015 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <unistd.h>
-#include <poll.h>
-#include <sys/eventfd.h>
-#include <errno.h>
-#include <czmq.h>
-
-#include "msglist.h"
-
-struct msglist_struct {
-    zlist_t *zl;
-    msglist_free_f destructor;
-    int pollevents;
-    int pollfd;
-    uint64_t event;
-};
-
-static int raise_event (msglist_t *l)
-{
-    if (l->pollfd >= 0 && l->event == 0) {
-        l->event = 1;
-        if (write (l->pollfd, &l->event, sizeof (l->event)) < 0)
-            return -1;
-    }
-    return 0;
-}
-
-static int clear_event (msglist_t *l)
-{
-    if (l->pollfd >= 0 && l->event == 1) {
-        if (read (l->pollfd, &l->event, sizeof (l->event)) < 0) {
-            if (errno != EAGAIN  && errno != EWOULDBLOCK)
-                return -1;
-            errno = 0;
-        }
-        l->event = 0;
-    }
-    return 0;
-}
-
-void msglist_destroy (msglist_t *l)
-{
-    if (l) {
-        if (l->zl) {
-            void *item;
-            while ((item = zlist_pop (l->zl)))
-                if (l->destructor)
-                    l->destructor (item);
-            zlist_destroy (&l->zl);
-        }
-        if (l->pollfd >= 0)
-            close (l->pollfd);
-        free (l);
-    }
-}
-
-msglist_t *msglist_create (msglist_free_f fun)
-{
-    msglist_t *l;
-
-    if (!(l = malloc (sizeof (*l)))) {
-        errno = ENOMEM;
-        goto error;
-    }
-    memset (l, 0, sizeof (*l));
-    l->pollfd = -1;
-    if (!(l->zl = zlist_new ())) {
-        errno = ENOMEM;
-        goto error;
-    }
-    l->pollevents = POLLOUT;
-    l->destructor = fun;
-    return l;
-error:
-    msglist_destroy (l);
-    return NULL;
-}
-
-void *msglist_pop (msglist_t *l)
-{
-    void *item = zlist_pop (l->zl);
-
-    if ((l->pollevents & POLLIN) && zlist_size (l->zl) == 0)
-        l->pollevents &= ~POLLIN;
-    return item;
-}
-
-int msglist_push (msglist_t *l, void *item)
-{
-    int rc = -1;
-
-    if (!(l->pollevents & POLLIN)) {
-        l->pollevents |= POLLIN;
-        if (raise_event (l) < 0)
-            goto done;
-    }
-    if (zlist_push (l->zl, item) < 0) {
-        l->pollevents |= POLLERR;
-        (void)raise_event (l);
-        errno = ENOMEM;
-        goto done;
-    }
-    rc = 0;
-done:
-    return rc;
-}
-
-int msglist_append (msglist_t *l, void *item)
-{
-    int rc = -1;
-
-    if (!(l->pollevents & POLLIN)) {
-        l->pollevents |= POLLIN;
-        if (raise_event (l) < 0)
-            goto done;
-    }
-    if (zlist_append (l->zl, item) < 0) {
-        l->pollevents |= POLLERR;
-        (void)raise_event (l);
-        errno = ENOMEM;
-        goto done;
-    }
-    rc = 0;
-done:
-    return rc;
-}
-
-void *msglist_first (msglist_t *l)
-{
-    return zlist_first (l->zl);
-}
-
-void *msglist_next (msglist_t *l)
-{
-    return zlist_next (l->zl);
-}
-
-void msglist_remove (msglist_t *l, void *item)
-{
-    zlist_remove (l->zl, item);
-    if ((l->pollevents & POLLIN) && zlist_size (l->zl) == 0)
-        l->pollevents &= ~POLLIN;
-}
-
-int msglist_count (msglist_t *l)
-{
-    return zlist_size (l->zl);
-}
-
-int msglist_pollfd (msglist_t *l)
-{
-    if (l->pollfd < 0) {
-        l->event = l->pollevents ? 1 : 0;
-        l->pollfd = eventfd (l->pollevents, EFD_NONBLOCK);
-    }
-    return l->pollfd;
-}
-
-int msglist_pollevents (msglist_t *l)
-{
-    if (clear_event (l) < 0)
-        return -1;
-    return l->pollevents;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/src/common/libutil/msglist.h b/src/common/libutil/msglist.h
deleted file mode 100644
index 26acaea74518..000000000000
--- a/src/common/libutil/msglist.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/************************************************************\
- * Copyright 2015 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _UTIL_MSGLIST_H
-#define _UTIL_MSGLIST_H
-
-#include <poll.h>
-
-typedef struct msglist_struct msglist_t;
-
-typedef void (*msglist_free_f)(void *item);
-
-/* Create/destroy list.
- * If 'fun' is non-NULL, msglist_destroy () will use it to destroy any
- * items on the list at that time.
- */
-msglist_t *msglist_create (msglist_free_f fun);
-void msglist_destroy (msglist_t *l);
-
-/* Add/remove items from the list.
- * Push/append returns 0 on success, -1 on error with errno set.
- */
-void *msglist_pop (msglist_t *l);
-int msglist_push (msglist_t *l, void *item);
-int msglist_append (msglist_t *l, void *item);
-
-/* Iteration, removal, count.
- */
-void *msglist_first (msglist_t *l);
-void *msglist_next (msglist_t *l);
-void msglist_remove (msglist_t *l, void *item);
-int msglist_count (msglist_t *l);
-
-/* Get the list 'pollevents' bitmask.
- * POLLIN = items can be removed with msglist_pop()
- * POLLOUT = items can be added wtih msglist_push() / msglist_append().
- * POLLERR = the msglist code encountered an error (eventfd related or ENOMEM)
- * Returns pollevents on success, -1 on error with errno set.
- */
-int msglist_pollevents (msglist_t *l);
-
-/* Obtain a file descriptor that will be readable when one of the pollevents
- * bits has been raised (edge triggered).  This file descriptor belongs
- * to msglist_t and should not be operated on except to integrate msglist_t
- * into a poll/event loop.  Returns fd on success, -1 on error with errno set.
- */
-int msglist_pollfd (msglist_t *l);
-
-#endif /* !_UTIL_MSGLIST_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/src/common/libutil/parse_size.c b/src/common/libutil/parse_size.c
new file mode 100644
index 000000000000..1183e79fffc5
--- /dev/null
+++ b/src/common/libutil/parse_size.c
@@ -0,0 +1,161 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdint.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+#include <math.h>
+
+#include <ccan/array_size/array_size.h>
+#include <ccan/str/str.h>
+
+#include "parse_size.h"
+
+struct scale {
+    const char *s;
+    uint64_t scale;
+};
+
+static struct scale mtab[] = {
+    { "", 1 },
+    { "k", 1024 },
+    { "K", 1024 }, // upper case K is not the SI prefix but is unambiguous
+    { "M", 1024*1024 },
+    { "G", 1024UL*1024*1024 },
+    { "T", 1024ULL*1024*1024*1024 },
+    { "P", 1024ULL*1024*1024*1024*1024 },
+    { "E", 1024ULL*1024*1024*1024*1024*1024 },
+};
+
+static int lookup_scale (const char *s, uint64_t *vp)
+{
+    for (int i = 0; i < ARRAY_SIZE (mtab); i++) {
+        if (streq (mtab[i].s, s)) {
+            *vp = mtab[i].scale;
+            return 0;
+        }
+    }
+    return -1;
+}
+
+static bool invalid_fp_size (double val)
+{
+    switch (fpclassify (val)) {
+        case FP_NORMAL:     // [[fallthrough]]
+        case FP_SUBNORMAL:  // [[fallthrough]]
+        case FP_ZERO:       // [[fallthrough]]
+            break;          // OK
+        case FP_NAN:        // [[fallthrough]]
+        case FP_INFINITE:   // [[fallthrough]]
+        default:            // something else, bad
+            return true;
+    }
+    if (val < 0.)
+        return true;
+    return false;
+}
+
+static int parse_as_integer (const char *s, uint64_t *up)
+{
+    char *endptr;
+    uint64_t u;
+    uint64_t scale;
+
+    // strtoull() allows a leading minus sign but we do not
+    if (strchr (s, '-')) {
+        errno = EINVAL;
+        return -1;
+    }
+    errno = 0;
+    u = strtoull (s, &endptr, 0);
+    if (errno != 0
+        || endptr == s
+        || lookup_scale (endptr, &scale) < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    uint64_t result = u * scale;
+    if (result < u) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    *up = result;
+    return 0;
+}
+
+static int parse_as_double (const char *s, uint64_t *up)
+{
+    char *endptr;
+    double d;
+    uint64_t scale;
+
+    errno = 0;
+    d = strtold (s, &endptr);
+    if (errno != 0
+        || endptr == s
+        || lookup_scale (endptr, &scale) < 0
+        || invalid_fp_size (d)) {
+        errno = EINVAL;
+        return -1;
+    }
+    double result = floor (d * scale);
+    if (result > (double)UINT64_MAX) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    *up = (uint64_t)result;
+    return 0;
+}
+
+int parse_size (const char *s, uint64_t *vp)
+{
+    if (!s || !vp) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (parse_as_integer (s, vp) < 0
+        && parse_as_double (s, vp) < 0)
+        return -1;
+    return 0;
+}
+
+const char *encode_size (uint64_t size)
+{
+    /* Allocate a thread-local buffer to make this function easy to use
+     * in output formats (its intended use case).
+     *
+     * Note: The maximum printable digits for a double is 15 (DBL_DIG).
+     *  We also account for an optional decimal point, suffix, and required
+     *  space for NUL to get a buffer size of 18. (We ignore the fact that
+     *  a precision is specified in the %g format below for safety).
+     */
+    static __thread char buf[18];
+    const char* suffix[] = {"", "K", "M", "G", "T", "P", "E"};
+    int i = 0;
+    double value = size;
+    while (value >= 1024) {
+        value /= 1024;
+        i++;
+    }
+    /* Note: UINT64_MAX is 16E so there is no possibility that 'i' will
+     * overflow the suffix array.
+     */
+    (void) snprintf (buf, sizeof (buf), "%.8g%s", value, suffix[i]);
+    return buf;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/parse_size.h b/src/common/libutil/parse_size.h
new file mode 100644
index 000000000000..9ff289c8136e
--- /dev/null
+++ b/src/common/libutil/parse_size.h
@@ -0,0 +1,47 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_PARSE_SIZE_H
+#define _UTIL_PARSE_SIZE_H
+
+#include <stdint.h>
+
+/* Parse 's' as a floating point quantity scaled by optional suffix:
+ *
+ *  k,K   2^10 (1024)
+ *  M     2^20
+ *  G     2^30
+ *  T     2^40
+ *  P     2^50
+ *  E     2^60
+ *
+ * The numeric part is parsed with first strtoull(3) then strtod(3),
+ * so all input supported by those functions should work including
+ * decimal (255), hex (0xf), octal (0377 prefix), exponent (2.55E2), etc.
+ *
+ * Assign the result to 'vp' and return 0 on success,
+ * or return -1 on failure with errno set (EINVAL, EOVERFLOW).
+ */
+int parse_size (const char *s, uint64_t *vp);
+
+/* Format 'size' as a human readable string using suffixes documented
+ * above for parse_size(). Note that due to use of double precision
+ * arithmetic and because the result is rounded to 8 significant figures
+ * the returned string may be imprecise. Passing the result of encode_size()
+ * to parse_size() may not result in the same value for 'size'.
+ *
+ * The result is only good until the next call to encode_size() from the
+ * current thread.
+ */
+const char *encode_size (uint64_t size);
+
+#endif /* !_UTIL_PARSE_SIZE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/popen2.c b/src/common/libutil/popen2.c
index 572283ba2064..e58cbddf7bf6 100644
--- a/src/common/libutil/popen2.c
+++ b/src/common/libutil/popen2.c
@@ -21,18 +21,19 @@
 #include <stdlib.h>
 #include <errno.h>
 #include <string.h>
-#include <assert.h>
 #include <stdio.h>
+#ifndef HAVE_PIPE2
+#include "src/common/libmissing/pipe2.h"
+#endif
 
 #include "popen2.h"
 #include "fdwalk.h"
 #include "fdutils.h"
 
-#define PXOPEN_CHILD_MAGIC 0xc00ceeee
-
 struct popen2_child {
-    int magic;
+    int flags;
     int fd[2];
+    int efd[2];
     int ctl[2];
     pid_t pid;
 };
@@ -44,33 +45,45 @@ enum {
 
 int popen2_get_fd (struct popen2_child *p)
 {
-    if (!p || p->magic != PXOPEN_CHILD_MAGIC) {
+    if (!p) {
         errno = EINVAL;
         return -1;
     }
     return p->fd[SP_PARENT];
 }
 
+int popen2_get_stderr_fd (struct popen2_child *p)
+{
+    if (!p) {
+        errno = EINVAL;
+        return -1;
+    }
+    return p->efd[SP_PARENT];
+}
+
 static void popen2_child_close_fd (void *arg, int fd)
 {
     struct popen2_child *p = arg;
-    if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO
-                                                  && fd != p->ctl[SP_CHILD])
+    if (fd != STDIN_FILENO
+        && fd != STDOUT_FILENO
+        && fd != STDERR_FILENO
+        && fd != p->ctl[SP_CHILD])
         (void)close (fd);
 }
 
 static void child (struct popen2_child *p, const char *path, char *const argv[])
 {
     int saved_errno;
+    int efd = p->efd[SP_CHILD];
 
-    (void)close (STDIN_FILENO);
-    (void)close (STDOUT_FILENO);
-    if (    dup2 (p->fd[SP_CHILD], STDIN_FILENO) < 0
-         || dup2 (p->fd[SP_CHILD], STDOUT_FILENO) < 0) {
+    if (dup2 (p->fd[SP_CHILD], STDIN_FILENO) < 0
+        || dup2 (p->fd[SP_CHILD], STDOUT_FILENO) < 0
+        || (efd >= 0 && dup2 (efd, STDERR_FILENO) < 0)) {
         saved_errno = errno;
         goto error;
     }
     (void)close (p->fd[SP_CHILD]);
+    (void)close (p->efd[SP_CHILD]);
 
     if (fdwalk (popen2_child_close_fd, p)) {
         saved_errno = errno;
@@ -85,19 +98,28 @@ static void child (struct popen2_child *p, const char *path, char *const argv[])
     exit (0);
 }
 
-struct popen2_child *popen2 (const char *path, char *const argv[])
+struct popen2_child *popen2 (const char *path,
+                             char *const argv[],
+                             int flags)
 {
     struct popen2_child *p = NULL;
     int n, saved_errno;
+    const int valid_flags = POPEN2_CAPTURE_STDERR;
 
-    if (!(p = malloc (sizeof (*p)))) {
+    if ((flags & valid_flags) != flags) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (!(p = calloc (1, sizeof (*p)))) {
         saved_errno = ENOMEM;
         goto error;
     }
-    memset (p, 0, sizeof (*p));
-    p->magic = PXOPEN_CHILD_MAGIC;
+    p->flags = flags;
     p->fd[SP_CHILD] = -1;
     p->fd[SP_PARENT] = -1;
+    p->efd[SP_CHILD] = -1;
+    p->efd[SP_PARENT] = -1;
     p->ctl[SP_CHILD] = -1;
     p->ctl[SP_PARENT] = -1;
     if (socketpair (PF_LOCAL, SOCK_STREAM, 0, p->fd) < 0) {
@@ -112,6 +134,11 @@ struct popen2_child *popen2 (const char *path, char *const argv[])
         saved_errno = errno;
         goto error;
     }
+    if ((p->flags & POPEN2_CAPTURE_STDERR)
+        && pipe2 (p->efd, O_CLOEXEC) < 0) {
+        saved_errno = errno;
+        goto error;
+    }
     signal(SIGPIPE, SIG_IGN);
     switch ((p->pid = fork ())) {
         case -1:    /* fork error */
@@ -124,8 +151,10 @@ struct popen2_child *popen2 (const char *path, char *const argv[])
             break;
     }
     (void)close (p->fd[SP_CHILD]);
+    (void)close (p->efd[SP_CHILD]);
     (void)close (p->ctl[SP_CHILD]);
     p->fd[SP_CHILD] = -1;
+    p->efd[SP_CHILD] = -1;
     p->ctl[SP_CHILD] = -1;
 
     /* Handshake on ctl pipe to make sure exec worked.
@@ -159,11 +188,6 @@ int pclose2 (struct popen2_child *p)
     int rc = 0;
 
     if (p) {
-        if (p->magic != PXOPEN_CHILD_MAGIC) {
-            saved_errno = EINVAL;
-            rc = -1;
-            goto done;
-        }
         if (p->fd[SP_PARENT] && shutdown (p->fd[SP_PARENT], SHUT_WR) < 0) {
             saved_errno = errno;
             rc = -1;
@@ -173,23 +197,25 @@ int pclose2 (struct popen2_child *p)
                 saved_errno = errno;
                 rc = -1;
             } else {
-                if (!WIFEXITED (status) || WEXITSTATUS (status) != 0) {
+                if (!WIFEXITED (status)) {
                     saved_errno = EIO;
                     rc = -1;
                 }
+                else
+                    rc = status;
             }
         }
         if ((p->fd[SP_PARENT] >= 0 && close (p->fd[SP_PARENT]) < 0)
-         || (p->fd[SP_CHILD] >= 0 && close (p->fd[SP_CHILD]) < 0)
-         || (p->ctl[SP_PARENT] >= 0 && close (p->ctl[SP_PARENT]) < 0)
-         || (p->ctl[SP_CHILD] >= 0 && close (p->ctl[SP_CHILD]) < 0)) {
+            || (p->fd[SP_CHILD] >= 0 && close (p->fd[SP_CHILD]) < 0)
+            || (p->efd[SP_PARENT] >= 0 && close (p->efd[SP_PARENT]) < 0)
+            || (p->efd[SP_CHILD] >= 0 && close (p->efd[SP_CHILD]) < 0)
+            || (p->ctl[SP_PARENT] >= 0 && close (p->ctl[SP_PARENT]) < 0)
+            || (p->ctl[SP_CHILD] >= 0 && close (p->ctl[SP_CHILD]) < 0)) {
             saved_errno = errno;
             rc = -1;
         }
-        p->magic = ~PXOPEN_CHILD_MAGIC;
         free (p);
     }
-done:
     if (rc == -1)
         errno = saved_errno;
     return rc;
diff --git a/src/common/libutil/popen2.h b/src/common/libutil/popen2.h
index d696c4df50ca..d1fe879bee7e 100644
--- a/src/common/libutil/popen2.h
+++ b/src/common/libutil/popen2.h
@@ -11,11 +11,18 @@
 #ifndef _UTIL_POPEN2_H
 #define _UTIL_POPEN2_H
 
+enum {
+    POPEN2_CAPTURE_STDERR = 0x1,
+};
+
 struct popen2_child;
 
-struct popen2_child *popen2 (const char *path, char *const argv[]);
+struct popen2_child *popen2 (const char *path,
+                             char *const argv[],
+                             int flags);
 
 int popen2_get_fd (struct popen2_child *p);
+int popen2_get_stderr_fd (struct popen2_child *p);
 
 int pclose2 (struct popen2_child *p);
 
diff --git a/src/common/libutil/sds.c b/src/common/libutil/sds.c
deleted file mode 100644
index 4d8ea1044c87..000000000000
--- a/src/common/libutil/sds.c
+++ /dev/null
@@ -1,1265 +0,0 @@
-/* SDSLib 2.0 -- A C dynamic strings library
- *
- * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2015, Oran Agra
- * Copyright (c) 2015, Redis Labs, Inc
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-#include <assert.h>
-#include "sds.h"
-#include "sdsalloc.h"
-
-static inline int sdsHdrSize(char type) {
-    switch(type&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            return sizeof(struct sdshdr5);
-        case SDS_TYPE_8:
-            return sizeof(struct sdshdr8);
-        case SDS_TYPE_16:
-            return sizeof(struct sdshdr16);
-        case SDS_TYPE_32:
-            return sizeof(struct sdshdr32);
-        case SDS_TYPE_64:
-            return sizeof(struct sdshdr64);
-    }
-    return 0;
-}
-
-static inline char sdsReqType(size_t string_size) {
-    if (string_size < 32)
-        return SDS_TYPE_5;
-    if (string_size < 0xff)
-        return SDS_TYPE_8;
-    if (string_size < 0xffff)
-        return SDS_TYPE_16;
-    if (string_size < 0xffffffff)
-        return SDS_TYPE_32;
-    return SDS_TYPE_64;
-}
-
-/* Create a new sds string with the content specified by the 'init' pointer
- * and 'initlen'.
- * If NULL is used for 'init' the string is initialized with zero bytes.
- *
- * The string is always null-termined (all the sds strings are, always) so
- * even if you create an sds string with:
- *
- * mystring = sdsnewlen("abc",3);
- *
- * You can print the string with printf() as there is an implicit \0 at the
- * end of the string. However the string is binary safe and can contain
- * \0 characters in the middle, as the length is stored in the sds header. */
-sds sdsnewlen(const void *init, size_t initlen) {
-    void *sh;
-    sds s;
-    char type = sdsReqType(initlen);
-    /* Empty strings are usually created in order to append. Use type 8
-     * since type 5 is not good at this. */
-    if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8;
-    int hdrlen = sdsHdrSize(type);
-    unsigned char *fp; /* flags pointer. */
-
-    sh = s_malloc(hdrlen+initlen+1);
-    if (!init)
-        memset(sh, 0, hdrlen+initlen+1);
-    if (sh == NULL) return NULL;
-    s = (char*)sh+hdrlen;
-    fp = ((unsigned char*)s)-1;
-    switch(type) {
-        case SDS_TYPE_5: {
-            *fp = type | (initlen << SDS_TYPE_BITS);
-            break;
-        }
-        case SDS_TYPE_8: {
-            SDS_HDR_VAR(8,s);
-            sh->len = initlen;
-            sh->alloc = initlen;
-            *fp = type;
-            break;
-        }
-        case SDS_TYPE_16: {
-            SDS_HDR_VAR(16,s);
-            sh->len = initlen;
-            sh->alloc = initlen;
-            *fp = type;
-            break;
-        }
-        case SDS_TYPE_32: {
-            SDS_HDR_VAR(32,s);
-            sh->len = initlen;
-            sh->alloc = initlen;
-            *fp = type;
-            break;
-        }
-        case SDS_TYPE_64: {
-            SDS_HDR_VAR(64,s);
-            sh->len = initlen;
-            sh->alloc = initlen;
-            *fp = type;
-            break;
-        }
-    }
-    if (initlen && init)
-        memcpy(s, init, initlen);
-    s[initlen] = '\0';
-    return s;
-}
-
-/* Create an empty (zero length) sds string. Even in this case the string
- * always has an implicit null term. */
-sds sdsempty(void) {
-    return sdsnewlen("",0);
-}
-
-/* Create a new sds string starting from a null terminated C string. */
-sds sdsnew(const char *init) {
-    size_t initlen = (init == NULL) ? 0 : strlen(init);
-    return sdsnewlen(init, initlen);
-}
-
-/* Duplicate an sds string. */
-sds sdsdup(const sds s) {
-    return sdsnewlen(s, sdslen(s));
-}
-
-/* Free an sds string. No operation is performed if 's' is NULL. */
-void sdsfree(sds s) {
-    if (s == NULL) return;
-    s_free((char*)s-sdsHdrSize(s[-1]));
-}
-
-/* Set the sds string length to the length as obtained with strlen(), so
- * considering as content only up to the first null term character.
- *
- * This function is useful when the sds string is hacked manually in some
- * way, like in the following example:
- *
- * s = sdsnew("foobar");
- * s[2] = '\0';
- * sdsupdatelen(s);
- * printf("%d\n", sdslen(s));
- *
- * The output will be "2", but if we comment out the call to sdsupdatelen()
- * the output will be "6" as the string was modified but the logical length
- * remains 6 bytes. */
-void sdsupdatelen(sds s) {
-    int reallen = strlen(s);
-    sdssetlen(s, reallen);
-}
-
-/* Modify an sds string in-place to make it empty (zero length).
- * However all the existing buffer is not discarded but set as free space
- * so that next append operations will not require allocations up to the
- * number of bytes previously available. */
-void sdsclear(sds s) {
-    sdssetlen(s, 0);
-    s[0] = '\0';
-}
-
-/* Enlarge the free space at the end of the sds string so that the caller
- * is sure that after calling this function can overwrite up to addlen
- * bytes after the end of the string, plus one more byte for nul term.
- *
- * Note: this does not change the *length* of the sds string as returned
- * by sdslen(), but only the free buffer space we have. */
-sds sdsMakeRoomFor(sds s, size_t addlen) {
-    void *sh, *newsh;
-    size_t avail = sdsavail(s);
-    size_t len, newlen;
-    char type, oldtype = s[-1] & SDS_TYPE_MASK;
-    int hdrlen;
-
-    /* Return ASAP if there is enough space left. */
-    if (avail >= addlen) return s;
-
-    len = sdslen(s);
-    sh = (char*)s-sdsHdrSize(oldtype);
-    newlen = (len+addlen);
-    if (newlen < SDS_MAX_PREALLOC)
-        newlen *= 2;
-    else
-        newlen += SDS_MAX_PREALLOC;
-
-    type = sdsReqType(newlen);
-
-    /* Don't use type 5: the user is appending to the string and type 5 is
-     * not able to remember empty space, so sdsMakeRoomFor() must be called
-     * at every appending operation. */
-    if (type == SDS_TYPE_5) type = SDS_TYPE_8;
-
-    hdrlen = sdsHdrSize(type);
-    if (oldtype==type) {
-        newsh = s_realloc(sh, hdrlen+newlen+1);
-        if (newsh == NULL) return NULL;
-        s = (char*)newsh+hdrlen;
-    } else {
-        /* Since the header size changes, need to move the string forward,
-         * and can't use realloc */
-        newsh = s_malloc(hdrlen+newlen+1);
-        if (newsh == NULL) return NULL;
-        memcpy((char*)newsh+hdrlen, s, len+1);
-        s_free(sh);
-        s = (char*)newsh+hdrlen;
-        s[-1] = type;
-        sdssetlen(s, len);
-    }
-    sdssetalloc(s, newlen);
-    return s;
-}
-
-/* Reallocate the sds string so that it has no free space at the end. The
- * contained string remains not altered, but next concatenation operations
- * will require a reallocation.
- *
- * After the call, the passed sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call. */
-sds sdsRemoveFreeSpace(sds s) {
-    void *sh, *newsh;
-    char type, oldtype = s[-1] & SDS_TYPE_MASK;
-    int hdrlen;
-    size_t len = sdslen(s);
-    sh = (char*)s-sdsHdrSize(oldtype);
-
-    type = sdsReqType(len);
-    hdrlen = sdsHdrSize(type);
-    if (oldtype==type) {
-        newsh = s_realloc(sh, hdrlen+len+1);
-        if (newsh == NULL) return NULL;
-        s = (char*)newsh+hdrlen;
-    } else {
-        newsh = s_malloc(hdrlen+len+1);
-        if (newsh == NULL) return NULL;
-        memcpy((char*)newsh+hdrlen, s, len+1);
-        s_free(sh);
-        s = (char*)newsh+hdrlen;
-        s[-1] = type;
-        sdssetlen(s, len);
-    }
-    sdssetalloc(s, len);
-    return s;
-}
-
-/* Return the total size of the allocation of the specifed sds string,
- * including:
- * 1) The sds header before the pointer.
- * 2) The string.
- * 3) The free buffer at the end if any.
- * 4) The implicit null term.
- */
-size_t sdsAllocSize(sds s) {
-    size_t alloc = sdsalloc(s);
-    return sdsHdrSize(s[-1])+alloc+1;
-}
-
-/* Return the pointer of the actual SDS allocation (normally SDS strings
- * are referenced by the start of the string buffer). */
-void *sdsAllocPtr(sds s) {
-    return (void*) (s-sdsHdrSize(s[-1]));
-}
-
-/* Increment the sds length and decrements the left free space at the
- * end of the string according to 'incr'. Also set the null term
- * in the new end of the string.
- *
- * This function is used in order to fix the string length after the
- * user calls sdsMakeRoomFor(), writes something after the end of
- * the current string, and finally needs to set the new length.
- *
- * Note: it is possible to use a negative increment in order to
- * right-trim the string.
- *
- * Usage example:
- *
- * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the
- * following schema, to cat bytes coming from the kernel to the end of an
- * sds string without copying into an intermediate buffer:
- *
- * oldlen = sdslen(s);
- * s = sdsMakeRoomFor(s, BUFFER_SIZE);
- * nread = read(fd, s+oldlen, BUFFER_SIZE);
- * ... check for nread <= 0 and handle it ...
- * sdsIncrLen(s, nread);
- */
-void sdsIncrLen(sds s, int incr) {
-    unsigned char flags = s[-1];
-    size_t len;
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5: {
-            unsigned char *fp = ((unsigned char*)s)-1;
-            unsigned char oldlen = SDS_TYPE_5_LEN(flags);
-            assert((incr > 0 && oldlen+incr < 32) || (incr < 0 && oldlen >= (unsigned int)(-incr)));
-            *fp = SDS_TYPE_5 | ((oldlen+incr) << SDS_TYPE_BITS);
-            len = oldlen+incr;
-            break;
-        }
-        case SDS_TYPE_8: {
-            SDS_HDR_VAR(8,s);
-            assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
-            len = (sh->len += incr);
-            break;
-        }
-        case SDS_TYPE_16: {
-            SDS_HDR_VAR(16,s);
-            assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
-            len = (sh->len += incr);
-            break;
-        }
-        case SDS_TYPE_32: {
-            SDS_HDR_VAR(32,s);
-            assert((incr >= 0 && sh->alloc-sh->len >= (unsigned int)incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
-            len = (sh->len += incr);
-            break;
-        }
-        case SDS_TYPE_64: {
-            SDS_HDR_VAR(64,s);
-            assert((incr >= 0 && sh->alloc-sh->len >= (uint64_t)incr) || (incr < 0 && sh->len >= (uint64_t)(-incr)));
-            len = (sh->len += incr);
-            break;
-        }
-        default: len = 0; /* Just to avoid compilation warnings. */
-    }
-    s[len] = '\0';
-}
-
-/* Grow the sds to have the specified length. Bytes that were not part of
- * the original length of the sds will be set to zero.
- *
- * if the specified length is smaller than the current length, no operation
- * is performed. */
-sds sdsgrowzero(sds s, size_t len) {
-    size_t curlen = sdslen(s);
-
-    if (len <= curlen) return s;
-    s = sdsMakeRoomFor(s,len-curlen);
-    if (s == NULL) return NULL;
-
-    /* Make sure added region doesn't contain garbage */
-    memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */
-    sdssetlen(s, len);
-    return s;
-}
-
-/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the
- * end of the specified sds string 's'.
- *
- * After the call, the passed sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call. */
-sds sdscatlen(sds s, const void *t, size_t len) {
-    size_t curlen = sdslen(s);
-
-    s = sdsMakeRoomFor(s,len);
-    if (s == NULL) return NULL;
-    memcpy(s+curlen, t, len);
-    sdssetlen(s, curlen+len);
-    s[curlen+len] = '\0';
-    return s;
-}
-
-/* Append the specified null termianted C string to the sds string 's'.
- *
- * After the call, the passed sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call. */
-sds sdscat(sds s, const char *t) {
-    return sdscatlen(s, t, strlen(t));
-}
-
-/* Append the specified sds 't' to the existing sds 's'.
- *
- * After the call, the modified sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call. */
-sds sdscatsds(sds s, const sds t) {
-    return sdscatlen(s, t, sdslen(t));
-}
-
-/* Destructively modify the sds string 's' to hold the specified binary
- * safe string pointed by 't' of length 'len' bytes. */
-sds sdscpylen(sds s, const char *t, size_t len) {
-    if (sdsalloc(s) < len) {
-        s = sdsMakeRoomFor(s,len-sdslen(s));
-        if (s == NULL) return NULL;
-    }
-    memcpy(s, t, len);
-    s[len] = '\0';
-    sdssetlen(s, len);
-    return s;
-}
-
-/* Like sdscpylen() but 't' must be a null-termined string so that the length
- * of the string is obtained with strlen(). */
-sds sdscpy(sds s, const char *t) {
-    return sdscpylen(s, t, strlen(t));
-}
-
-/* Helper for sdscatlonglong() doing the actual number -> string
- * conversion. 's' must point to a string with room for at least
- * SDS_LLSTR_SIZE bytes.
- *
- * The function returns the length of the null-terminated string
- * representation stored at 's'. */
-#define SDS_LLSTR_SIZE 21
-int sdsll2str(char *s, long long value) {
-    char *p, aux;
-    unsigned long long v;
-    size_t l;
-
-    /* Generate the string representation, this method produces
-     * an reversed string. */
-    v = (value < 0) ? -value : value;
-    p = s;
-    do {
-        *p++ = '0'+(v%10);
-        v /= 10;
-    } while(v);
-    if (value < 0) *p++ = '-';
-
-    /* Compute length and add null term. */
-    l = p-s;
-    *p = '\0';
-
-    /* Reverse the string. */
-    p--;
-    while(s < p) {
-        aux = *s;
-        *s = *p;
-        *p = aux;
-        s++;
-        p--;
-    }
-    return l;
-}
-
-/* Identical sdsll2str(), but for unsigned long long type. */
-int sdsull2str(char *s, unsigned long long v) {
-    char *p, aux;
-    size_t l;
-
-    /* Generate the string representation, this method produces
-     * an reversed string. */
-    p = s;
-    do {
-        *p++ = '0'+(v%10);
-        v /= 10;
-    } while(v);
-
-    /* Compute length and add null term. */
-    l = p-s;
-    *p = '\0';
-
-    /* Reverse the string. */
-    p--;
-    while(s < p) {
-        aux = *s;
-        *s = *p;
-        *p = aux;
-        s++;
-        p--;
-    }
-    return l;
-}
-
-/* Create an sds string from a long long value. It is much faster than:
- *
- * sdscatprintf(sdsempty(),"%lld\n", value);
- */
-sds sdsfromlonglong(long long value) {
-    char buf[SDS_LLSTR_SIZE];
-    int len = sdsll2str(buf,value);
-
-    return sdsnewlen(buf,len);
-}
-
-/* Like sdscatprintf() but gets va_list instead of being variadic. */
-sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
-    va_list cpy;
-    char staticbuf[1024], *buf = staticbuf, *t;
-    size_t buflen = strlen(fmt)*2;
-
-    /* We try to start using a static buffer for speed.
-     * If not possible we revert to heap allocation. */
-    if (buflen > sizeof(staticbuf)) {
-        buf = s_malloc(buflen);
-        if (buf == NULL) return NULL;
-    } else {
-        buflen = sizeof(staticbuf);
-    }
-
-    /* Try with buffers two times bigger every time we fail to
-     * fit the string in the current buffer size. */
-    while(1) {
-        buf[buflen-2] = '\0';
-        va_copy(cpy,ap);
-        vsnprintf(buf, buflen, fmt, cpy);
-        va_end(cpy);
-        if (buf[buflen-2] != '\0') {
-            if (buf != staticbuf) s_free(buf);
-            buflen *= 2;
-            buf = s_malloc(buflen);
-            if (buf == NULL) return NULL;
-            continue;
-        }
-        break;
-    }
-
-    /* Finally concat the obtained string to the SDS string and return it. */
-    t = sdscat(s, buf);
-    if (buf != staticbuf) s_free(buf);
-    return t;
-}
-
-/* Append to the sds string 's' a string obtained using printf-alike format
- * specifier.
- *
- * After the call, the modified sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call.
- *
- * Example:
- *
- * s = sdsnew("Sum is: ");
- * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b).
- *
- * Often you need to create a string from scratch with the printf-alike
- * format. When this is the need, just use sdsempty() as the target string:
- *
- * s = sdscatprintf(sdsempty(), "... your format ...", args);
- */
-sds sdscatprintf(sds s, const char *fmt, ...) {
-    va_list ap;
-    char *t;
-    va_start(ap, fmt);
-    t = sdscatvprintf(s,fmt,ap);
-    va_end(ap);
-    return t;
-}
-
-/* This function is similar to sdscatprintf, but much faster as it does
- * not rely on sprintf() family functions implemented by the libc that
- * are often very slow. Moreover directly handling the sds string as
- * new data is concatenated provides a performance improvement.
- *
- * However this function only handles an incompatible subset of printf-alike
- * format specifiers:
- *
- * %s - C String
- * %S - SDS string
- * %i - signed int
- * %I - 64 bit signed integer (long long, int64_t)
- * %u - unsigned int
- * %U - 64 bit unsigned integer (unsigned long long, uint64_t)
- * %% - Verbatim "%" character.
- */
-sds sdscatfmt(sds s, char const *fmt, ...) {
-    size_t initlen = sdslen(s);
-    const char *f = fmt;
-    int i;
-    va_list ap;
-
-    va_start(ap,fmt);
-    f = fmt;    /* Next format specifier byte to process. */
-    i = initlen; /* Position of the next byte to write to dest str. */
-    while(*f) {
-        char next, *str;
-        size_t l;
-        long long num;
-        unsigned long long unum;
-
-        /* Make sure there is always space for at least 1 char. */
-        if (sdsavail(s)==0) {
-            s = sdsMakeRoomFor(s,1);
-        }
-
-        switch(*f) {
-        case '%':
-            next = *(f+1);
-            f++;
-            switch(next) {
-            case 's':
-            case 'S':
-                str = va_arg(ap,char*);
-                l = (next == 's') ? strlen(str) : sdslen(str);
-                if (sdsavail(s) < l) {
-                    s = sdsMakeRoomFor(s,l);
-                }
-                memcpy(s+i,str,l);
-                sdsinclen(s,l);
-                i += l;
-                break;
-            case 'i':
-            case 'I':
-                if (next == 'i')
-                    num = va_arg(ap,int);
-                else
-                    num = va_arg(ap,long long);
-                {
-                    char buf[SDS_LLSTR_SIZE];
-                    l = sdsll2str(buf,num);
-                    if (sdsavail(s) < l) {
-                        s = sdsMakeRoomFor(s,l);
-                    }
-                    memcpy(s+i,buf,l);
-                    sdsinclen(s,l);
-                    i += l;
-                }
-                break;
-            case 'u':
-            case 'U':
-                if (next == 'u')
-                    unum = va_arg(ap,unsigned int);
-                else
-                    unum = va_arg(ap,unsigned long long);
-                {
-                    char buf[SDS_LLSTR_SIZE];
-                    l = sdsull2str(buf,unum);
-                    if (sdsavail(s) < l) {
-                        s = sdsMakeRoomFor(s,l);
-                    }
-                    memcpy(s+i,buf,l);
-                    sdsinclen(s,l);
-                    i += l;
-                }
-                break;
-            default: /* Handle %% and generally %<unknown>. */
-                s[i++] = next;
-                sdsinclen(s,1);
-                break;
-            }
-            break;
-        default:
-            s[i++] = *f;
-            sdsinclen(s,1);
-            break;
-        }
-        f++;
-    }
-    va_end(ap);
-
-    /* Add null-term */
-    s[i] = '\0';
-    return s;
-}
-
-/* Remove the part of the string from left and from right composed just of
- * contiguous characters found in 'cset', that is a null terminted C string.
- *
- * After the call, the modified sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call.
- *
- * Example:
- *
- * s = sdsnew("AA...AA.a.aa.aHelloWorld     :::");
- * s = sdstrim(s,"Aa. :");
- * printf("%s\n", s);
- *
- * Output will be just "Hello World".
- */
-sds sdstrim(sds s, const char *cset) {
-    char *start, *end, *sp, *ep;
-    size_t len;
-
-    sp = start = s;
-    ep = end = s+sdslen(s)-1;
-    while(sp <= end && strchr(cset, *sp)) sp++;
-    while(ep > sp && strchr(cset, *ep)) ep--;
-    len = (sp > ep) ? 0 : ((ep-sp)+1);
-    if (s != sp) memmove(s, sp, len);
-    s[len] = '\0';
-    sdssetlen(s,len);
-    return s;
-}
-
-/* Turn the string into a smaller (or equal) string containing only the
- * substring specified by the 'start' and 'end' indexes.
- *
- * start and end can be negative, where -1 means the last character of the
- * string, -2 the penultimate character, and so forth.
- *
- * The interval is inclusive, so the start and end characters will be part
- * of the resulting string.
- *
- * The string is modified in-place.
- *
- * Example:
- *
- * s = sdsnew("Hello World");
- * sdsrange(s,1,-1); => "ello World"
- */
-void sdsrange(sds s, int start, int end) {
-    size_t newlen, len = sdslen(s);
-
-    if (len == 0) return;
-    if (start < 0) {
-        start = len+start;
-        if (start < 0) start = 0;
-    }
-    if (end < 0) {
-        end = len+end;
-        if (end < 0) end = 0;
-    }
-    newlen = (start > end) ? 0 : (end-start)+1;
-    if (newlen != 0) {
-        if (start >= (signed)len) {
-            newlen = 0;
-        } else if (end >= (signed)len) {
-            end = len-1;
-            newlen = (start > end) ? 0 : (end-start)+1;
-        }
-    } else {
-        start = 0;
-    }
-    if (start && newlen) memmove(s, s+start, newlen);
-    s[newlen] = 0;
-    sdssetlen(s,newlen);
-}
-
-/* Apply tolower() to every character of the sds string 's'. */
-void sdstolower(sds s) {
-    int len = sdslen(s), j;
-
-    for (j = 0; j < len; j++) s[j] = tolower(s[j]);
-}
-
-/* Apply toupper() to every character of the sds string 's'. */
-void sdstoupper(sds s) {
-    int len = sdslen(s), j;
-
-    for (j = 0; j < len; j++) s[j] = toupper(s[j]);
-}
-
-/* Compare two sds strings s1 and s2 with memcmp().
- *
- * Return value:
- *
- *     positive if s1 > s2.
- *     negative if s1 < s2.
- *     0 if s1 and s2 are exactly the same binary string.
- *
- * If two strings share exactly the same prefix, but one of the two has
- * additional characters, the longer string is considered to be greater than
- * the smaller one. */
-int sdscmp(const sds s1, const sds s2) {
-    size_t l1, l2, minlen;
-    int cmp;
-
-    l1 = sdslen(s1);
-    l2 = sdslen(s2);
-    minlen = (l1 < l2) ? l1 : l2;
-    cmp = memcmp(s1,s2,minlen);
-    if (cmp == 0) return l1-l2;
-    return cmp;
-}
-
-/* Split 's' with separator in 'sep'. An array
- * of sds strings is returned. *count will be set
- * by reference to the number of tokens returned.
- *
- * On out of memory, zero length string, zero length
- * separator, NULL is returned.
- *
- * Note that 'sep' is able to split a string using
- * a multi-character separator. For example
- * sdssplit("foo_-_bar","_-_"); will return two
- * elements "foo" and "bar".
- *
- * This version of the function is binary-safe but
- * requires length arguments. sdssplit() is just the
- * same function but for zero-terminated strings.
- */
-sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) {
-    int elements = 0, slots = 5, start = 0, j;
-    sds *tokens;
-
-    if (seplen < 1 || len < 0) return NULL;
-
-    tokens = s_malloc(sizeof(sds)*slots);
-    if (tokens == NULL) return NULL;
-
-    if (len == 0) {
-        *count = 0;
-        return tokens;
-    }
-    for (j = 0; j < (len-(seplen-1)); j++) {
-        /* make sure there is room for the next element and the final one */
-        if (slots < elements+2) {
-            sds *newtokens;
-
-            slots *= 2;
-            newtokens = s_realloc(tokens,sizeof(sds)*slots);
-            if (newtokens == NULL) goto cleanup;
-            tokens = newtokens;
-        }
-        /* search the separator */
-        if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) {
-            tokens[elements] = sdsnewlen(s+start,j-start);
-            if (tokens[elements] == NULL) goto cleanup;
-            elements++;
-            start = j+seplen;
-            j = j+seplen-1; /* skip the separator */
-        }
-    }
-    /* Add the final element. We are sure there is room in the tokens array. */
-    tokens[elements] = sdsnewlen(s+start,len-start);
-    if (tokens[elements] == NULL) goto cleanup;
-    elements++;
-    *count = elements;
-    return tokens;
-
-cleanup:
-    {
-        int i;
-        for (i = 0; i < elements; i++) sdsfree(tokens[i]);
-        s_free(tokens);
-        *count = 0;
-        return NULL;
-    }
-}
-
-/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */
-void sdsfreesplitres(sds *tokens, int count) {
-    if (!tokens) return;
-    while(count--)
-        sdsfree(tokens[count]);
-    s_free(tokens);
-}
-
-/* Append to the sds string "s" an escaped string representation where
- * all the non-printable characters (tested with isprint()) are turned into
- * escapes in the form "\n\r\a...." or "\x<hex-number>".
- *
- * After the call, the modified sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call. */
-sds sdscatrepr(sds s, const char *p, size_t len) {
-    s = sdscatlen(s,"\"",1);
-    while(len--) {
-        switch(*p) {
-        case '\\':
-        case '"':
-            s = sdscatprintf(s,"\\%c",*p);
-            break;
-        case '\n': s = sdscatlen(s,"\\n",2); break;
-        case '\r': s = sdscatlen(s,"\\r",2); break;
-        case '\t': s = sdscatlen(s,"\\t",2); break;
-        case '\a': s = sdscatlen(s,"\\a",2); break;
-        case '\b': s = sdscatlen(s,"\\b",2); break;
-        default:
-            if (isprint(*p))
-                s = sdscatprintf(s,"%c",*p);
-            else
-                s = sdscatprintf(s,"\\x%02x",(unsigned char)*p);
-            break;
-        }
-        p++;
-    }
-    return sdscatlen(s,"\"",1);
-}
-
-/* Helper function for sdssplitargs() that returns non zero if 'c'
- * is a valid hex digit. */
-int is_hex_digit(char c) {
-    return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
-           (c >= 'A' && c <= 'F');
-}
-
-/* Helper function for sdssplitargs() that converts a hex digit into an
- * integer from 0 to 15 */
-int hex_digit_to_int(char c) {
-    switch(c) {
-    case '0': return 0;
-    case '1': return 1;
-    case '2': return 2;
-    case '3': return 3;
-    case '4': return 4;
-    case '5': return 5;
-    case '6': return 6;
-    case '7': return 7;
-    case '8': return 8;
-    case '9': return 9;
-    case 'a': case 'A': return 10;
-    case 'b': case 'B': return 11;
-    case 'c': case 'C': return 12;
-    case 'd': case 'D': return 13;
-    case 'e': case 'E': return 14;
-    case 'f': case 'F': return 15;
-    default: return 0;
-    }
-}
-
-/* Split a line into arguments, where every argument can be in the
- * following programming-language REPL-alike form:
- *
- * foo bar "newline are supported\n" and "\xff\x00otherstuff"
- *
- * The number of arguments is stored into *argc, and an array
- * of sds is returned.
- *
- * The caller should free the resulting array of sds strings with
- * sdsfreesplitres().
- *
- * Note that sdscatrepr() is able to convert back a string into
- * a quoted string in the same format sdssplitargs() is able to parse.
- *
- * The function returns the allocated tokens on success, even when the
- * input string is empty, or NULL if the input contains unbalanced
- * quotes or closed quotes followed by non space characters
- * as in: "foo"bar or "foo'
- */
-sds *sdssplitargs(const char *line, int *argc) {
-    const char *p = line;
-    char *current = NULL;
-    char **vector = NULL;
-
-    *argc = 0;
-    while(1) {
-        /* skip blanks */
-        while(*p && isspace(*p)) p++;
-        if (*p) {
-            /* get a token */
-            int inq=0;  /* set to 1 if we are in "quotes" */
-            int insq=0; /* set to 1 if we are in 'single quotes' */
-            int done=0;
-
-            if (current == NULL) current = sdsempty();
-            while(!done) {
-                if (inq) {
-                    if (*p == '\\' && *(p+1) == 'x' &&
-                                             is_hex_digit(*(p+2)) &&
-                                             is_hex_digit(*(p+3)))
-                    {
-                        unsigned char byte;
-
-                        byte = (hex_digit_to_int(*(p+2))*16)+
-                                hex_digit_to_int(*(p+3));
-                        current = sdscatlen(current,(char*)&byte,1);
-                        p += 3;
-                    } else if (*p == '\\' && *(p+1)) {
-                        char c;
-
-                        p++;
-                        switch(*p) {
-                        case 'n': c = '\n'; break;
-                        case 'r': c = '\r'; break;
-                        case 't': c = '\t'; break;
-                        case 'b': c = '\b'; break;
-                        case 'a': c = '\a'; break;
-                        default: c = *p; break;
-                        }
-                        current = sdscatlen(current,&c,1);
-                    } else if (*p == '"') {
-                        /* closing quote must be followed by a space or
-                         * nothing at all. */
-                        if (*(p+1) && !isspace(*(p+1))) goto err;
-                        done=1;
-                    } else if (!*p) {
-                        /* unterminated quotes */
-                        goto err;
-                    } else {
-                        current = sdscatlen(current,p,1);
-                    }
-                } else if (insq) {
-                    if (*p == '\\' && *(p+1) == '\'') {
-                        p++;
-                        current = sdscatlen(current,"'",1);
-                    } else if (*p == '\'') {
-                        /* closing quote must be followed by a space or
-                         * nothing at all. */
-                        if (*(p+1) && !isspace(*(p+1))) goto err;
-                        done=1;
-                    } else if (!*p) {
-                        /* unterminated quotes */
-                        goto err;
-                    } else {
-                        current = sdscatlen(current,p,1);
-                    }
-                } else {
-                    switch(*p) {
-                    case ' ':
-                    case '\n':
-                    case '\r':
-                    case '\t':
-                    case '\0':
-                        done=1;
-                        break;
-                    case '"':
-                        inq=1;
-                        break;
-                    case '\'':
-                        insq=1;
-                        break;
-                    default:
-                        current = sdscatlen(current,p,1);
-                        break;
-                    }
-                }
-                if (*p) p++;
-            }
-            /* add the token to the vector */
-            vector = s_realloc(vector,((*argc)+1)*sizeof(char*));
-            vector[*argc] = current;
-            (*argc)++;
-            current = NULL;
-        } else {
-            /* Even on empty input string return something not NULL. */
-            if (vector == NULL) vector = s_malloc(sizeof(void*));
-            return vector;
-        }
-    }
-
-err:
-    while((*argc)--)
-        sdsfree(vector[*argc]);
-    s_free(vector);
-    if (current) sdsfree(current);
-    *argc = 0;
-    return NULL;
-}
-
-/* Modify the string substituting all the occurrences of the set of
- * characters specified in the 'from' string to the corresponding character
- * in the 'to' array.
- *
- * For instance: sdsmapchars(mystring, "ho", "01", 2)
- * will have the effect of turning the string "hello" into "0ell1".
- *
- * The function returns the sds string pointer, that is always the same
- * as the input pointer since no resize is needed. */
-sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) {
-    size_t j, i, l = sdslen(s);
-
-    for (j = 0; j < l; j++) {
-        for (i = 0; i < setlen; i++) {
-            if (s[j] == from[i]) {
-                s[j] = to[i];
-                break;
-            }
-        }
-    }
-    return s;
-}
-
-/* Join an array of C strings using the specified separator (also a C string).
- * Returns the result as an sds string. */
-sds sdsjoin(char **argv, int argc, const char *sep) {
-    sds join = sdsempty();
-    int j;
-
-    for (j = 0; j < argc; j++) {
-        join = sdscat(join, argv[j]);
-        if (j != argc-1) join = sdscat(join,sep);
-    }
-    return join;
-}
-
-/* Like sdsjoin, but joins an array of SDS strings. */
-sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) {
-    sds join = sdsempty();
-    int j;
-
-    for (j = 0; j < argc; j++) {
-        join = sdscatsds(join, argv[j]);
-        if (j != argc-1) join = sdscatlen(join,sep,seplen);
-    }
-    return join;
-}
-
-#if defined(SDS_TEST_MAIN)
-#include <stdio.h>
-#include "testhelp.h"
-#include "limits.h"
-
-#define UNUSED(x) (void)(x)
-int sdsTest(void) {
-    {
-        sds x = sdsnew("foo"), y;
-
-        test_cond("Create a string and obtain the length",
-            sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0)
-
-        sdsfree(x);
-        x = sdsnewlen("foo",2);
-        test_cond("Create a string with specified length",
-            sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0)
-
-        x = sdscat(x,"bar");
-        test_cond("Strings concatenation",
-            sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0);
-
-        x = sdscpy(x,"a");
-        test_cond("sdscpy() against an originally longer string",
-            sdslen(x) == 1 && memcmp(x,"a\0",2) == 0)
-
-        x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk");
-        test_cond("sdscpy() against an originally shorter string",
-            sdslen(x) == 33 &&
-            memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0)
-
-        sdsfree(x);
-        x = sdscatprintf(sdsempty(),"%d",123);
-        test_cond("sdscatprintf() seems working in the base case",
-            sdslen(x) == 3 && memcmp(x,"123\0",4) == 0)
-
-        sdsfree(x);
-        x = sdsnew("--");
-        x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX);
-        test_cond("sdscatfmt() seems working in the base case",
-            sdslen(x) == 60 &&
-            memcmp(x,"--Hello Hi! World -9223372036854775808,"
-                     "9223372036854775807--",60) == 0)
-        printf("[%s]\n",x);
-
-        sdsfree(x);
-        x = sdsnew("--");
-        x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX);
-        test_cond("sdscatfmt() seems working with unsigned numbers",
-            sdslen(x) == 35 &&
-            memcmp(x,"--4294967295,18446744073709551615--",35) == 0)
-
-        sdsfree(x);
-        x = sdsnew(" x ");
-        sdstrim(x," x");
-        test_cond("sdstrim() works when all chars match",
-            sdslen(x) == 0)
-
-        sdsfree(x);
-        x = sdsnew(" x ");
-        sdstrim(x," ");
-        test_cond("sdstrim() works when a single char remains",
-            sdslen(x) == 1 && x[0] == 'x')
-
-        sdsfree(x);
-        x = sdsnew("xxciaoyyy");
-        sdstrim(x,"xy");
-        test_cond("sdstrim() correctly trims characters",
-            sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0)
-
-        y = sdsdup(x);
-        sdsrange(y,1,1);
-        test_cond("sdsrange(...,1,1)",
-            sdslen(y) == 1 && memcmp(y,"i\0",2) == 0)
-
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,1,-1);
-        test_cond("sdsrange(...,1,-1)",
-            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
-
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,-2,-1);
-        test_cond("sdsrange(...,-2,-1)",
-            sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0)
-
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,2,1);
-        test_cond("sdsrange(...,2,1)",
-            sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
-
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,1,100);
-        test_cond("sdsrange(...,1,100)",
-            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
-
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,100,100);
-        test_cond("sdsrange(...,100,100)",
-            sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
-
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnew("foo");
-        y = sdsnew("foa");
-        test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0)
-
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnew("bar");
-        y = sdsnew("bar");
-        test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0)
-
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnew("aar");
-        y = sdsnew("bar");
-        test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0)
-
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnewlen("\a\n\0foo\r",7);
-        y = sdscatrepr(sdsempty(),x,sdslen(x));
-        test_cond("sdscatrepr(...data...)",
-            memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0)
-
-        {
-            unsigned int oldfree;
-            char *p;
-            int step = 10, j, i;
-
-            sdsfree(x);
-            sdsfree(y);
-            x = sdsnew("0");
-            test_cond("sdsnew() free/len buffers", sdslen(x) == 1 && sdsavail(x) == 0);
-
-            /* Run the test a few times in order to hit the first two
-             * SDS header types. */
-            for (i = 0; i < 10; i++) {
-                int oldlen = sdslen(x);
-                x = sdsMakeRoomFor(x,step);
-                int type = x[-1]&SDS_TYPE_MASK;
-
-                test_cond("sdsMakeRoomFor() len", sdslen(x) == oldlen);
-                if (type != SDS_TYPE_5) {
-                    test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step);
-                    oldfree = sdsavail(x);
-                }
-                p = x+oldlen;
-                for (j = 0; j < step; j++) {
-                    p[j] = 'A'+j;
-                }
-                sdsIncrLen(x,step);
-            }
-            test_cond("sdsMakeRoomFor() content",
-                memcmp("0ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJ",x,101) == 0);
-            test_cond("sdsMakeRoomFor() final length",sdslen(x)==101);
-
-            sdsfree(x);
-        }
-    }
-    test_report()
-    return 0;
-}
-#endif
-
-#ifdef SDS_TEST_MAIN
-int main(void) {
-    return sdsTest();
-}
-#endif
diff --git a/src/common/libutil/sds.h b/src/common/libutil/sds.h
deleted file mode 100644
index 0bc4e45b674e..000000000000
--- a/src/common/libutil/sds.h
+++ /dev/null
@@ -1,265 +0,0 @@
-/* SDSLib 2.0 -- A C dynamic strings library
- *
- * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2015, Oran Agra
- * Copyright (c) 2015, Redis Labs, Inc
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __SDS_H
-#define __SDS_H
-
-#define SDS_MAX_PREALLOC (1024*1024)
-
-#include <sys/types.h>
-#include <stdarg.h>
-#include <stdint.h>
-
-typedef char *sds;
-
-/* Note: sdshdr5 is never used, we just access the flags byte directly.
- * However is here to document the layout of type 5 SDS strings. */
-struct __attribute__ ((__packed__)) sdshdr5 {
-    unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
-    char buf[];
-};
-struct __attribute__ ((__packed__)) sdshdr8 {
-    uint8_t len; /* used */
-    uint8_t alloc; /* excluding the header and null terminator */
-    unsigned char flags; /* 3 lsb of type, 5 unused bits */
-    char buf[];
-};
-struct __attribute__ ((__packed__)) sdshdr16 {
-    uint16_t len; /* used */
-    uint16_t alloc; /* excluding the header and null terminator */
-    unsigned char flags; /* 3 lsb of type, 5 unused bits */
-    char buf[];
-};
-struct __attribute__ ((__packed__)) sdshdr32 {
-    uint32_t len; /* used */
-    uint32_t alloc; /* excluding the header and null terminator */
-    unsigned char flags; /* 3 lsb of type, 5 unused bits */
-    char buf[];
-};
-struct __attribute__ ((__packed__)) sdshdr64 {
-    uint64_t len; /* used */
-    uint64_t alloc; /* excluding the header and null terminator */
-    unsigned char flags; /* 3 lsb of type, 5 unused bits */
-    char buf[];
-};
-
-#define SDS_TYPE_5  0
-#define SDS_TYPE_8  1
-#define SDS_TYPE_16 2
-#define SDS_TYPE_32 3
-#define SDS_TYPE_64 4
-#define SDS_TYPE_MASK 7
-#define SDS_TYPE_BITS 3
-#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (void*)((s)-(sizeof(struct sdshdr##T)));
-#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
-#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
-
-static inline size_t sdslen(const sds s) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            return SDS_TYPE_5_LEN(flags);
-        case SDS_TYPE_8:
-            return SDS_HDR(8,s)->len;
-        case SDS_TYPE_16:
-            return SDS_HDR(16,s)->len;
-        case SDS_TYPE_32:
-            return SDS_HDR(32,s)->len;
-        case SDS_TYPE_64:
-            return SDS_HDR(64,s)->len;
-    }
-    return 0;
-}
-
-static inline size_t sdsavail(const sds s) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5: {
-            return 0;
-        }
-        case SDS_TYPE_8: {
-            SDS_HDR_VAR(8,s);
-            return sh->alloc - sh->len;
-        }
-        case SDS_TYPE_16: {
-            SDS_HDR_VAR(16,s);
-            return sh->alloc - sh->len;
-        }
-        case SDS_TYPE_32: {
-            SDS_HDR_VAR(32,s);
-            return sh->alloc - sh->len;
-        }
-        case SDS_TYPE_64: {
-            SDS_HDR_VAR(64,s);
-            return sh->alloc - sh->len;
-        }
-    }
-    return 0;
-}
-
-static inline void sdssetlen(sds s, size_t newlen) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            {
-                unsigned char *fp = ((unsigned char*)s)-1;
-                *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
-            }
-            break;
-        case SDS_TYPE_8:
-            SDS_HDR(8,s)->len = newlen;
-            break;
-        case SDS_TYPE_16:
-            SDS_HDR(16,s)->len = newlen;
-            break;
-        case SDS_TYPE_32:
-            SDS_HDR(32,s)->len = newlen;
-            break;
-        case SDS_TYPE_64:
-            SDS_HDR(64,s)->len = newlen;
-            break;
-    }
-}
-
-static inline void sdsinclen(sds s, size_t inc) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            {
-                unsigned char *fp = ((unsigned char*)s)-1;
-                unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc;
-                *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
-            }
-            break;
-        case SDS_TYPE_8:
-            SDS_HDR(8,s)->len += inc;
-            break;
-        case SDS_TYPE_16:
-            SDS_HDR(16,s)->len += inc;
-            break;
-        case SDS_TYPE_32:
-            SDS_HDR(32,s)->len += inc;
-            break;
-        case SDS_TYPE_64:
-            SDS_HDR(64,s)->len += inc;
-            break;
-    }
-}
-
-/* sdsalloc() = sdsavail() + sdslen() */
-static inline size_t sdsalloc(const sds s) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            return SDS_TYPE_5_LEN(flags);
-        case SDS_TYPE_8:
-            return SDS_HDR(8,s)->alloc;
-        case SDS_TYPE_16:
-            return SDS_HDR(16,s)->alloc;
-        case SDS_TYPE_32:
-            return SDS_HDR(32,s)->alloc;
-        case SDS_TYPE_64:
-            return SDS_HDR(64,s)->alloc;
-    }
-    return 0;
-}
-
-static inline void sdssetalloc(sds s, size_t newlen) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            /* Nothing to do, this type has no total allocation info. */
-            break;
-        case SDS_TYPE_8:
-            SDS_HDR(8,s)->alloc = newlen;
-            break;
-        case SDS_TYPE_16:
-            SDS_HDR(16,s)->alloc = newlen;
-            break;
-        case SDS_TYPE_32:
-            SDS_HDR(32,s)->alloc = newlen;
-            break;
-        case SDS_TYPE_64:
-            SDS_HDR(64,s)->alloc = newlen;
-            break;
-    }
-}
-
-sds sdsnewlen(const void *init, size_t initlen);
-sds sdsnew(const char *init);
-sds sdsempty(void);
-sds sdsdup(const sds s);
-void sdsfree(sds s);
-sds sdsgrowzero(sds s, size_t len);
-sds sdscatlen(sds s, const void *t, size_t len);
-sds sdscat(sds s, const char *t);
-sds sdscatsds(sds s, const sds t);
-sds sdscpylen(sds s, const char *t, size_t len);
-sds sdscpy(sds s, const char *t);
-
-sds sdscatvprintf(sds s, const char *fmt, va_list ap);
-#ifdef __GNUC__
-sds sdscatprintf(sds s, const char *fmt, ...)
-    __attribute__((format(printf, 2, 3)));
-#else
-sds sdscatprintf(sds s, const char *fmt, ...);
-#endif
-
-sds sdscatfmt(sds s, char const *fmt, ...);
-sds sdstrim(sds s, const char *cset);
-void sdsrange(sds s, int start, int end);
-void sdsupdatelen(sds s);
-void sdsclear(sds s);
-int sdscmp(const sds s1, const sds s2);
-sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count);
-void sdsfreesplitres(sds *tokens, int count);
-void sdstolower(sds s);
-void sdstoupper(sds s);
-sds sdsfromlonglong(long long value);
-sds sdscatrepr(sds s, const char *p, size_t len);
-sds *sdssplitargs(const char *line, int *argc);
-sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen);
-sds sdsjoin(char **argv, int argc, const char *sep);
-sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
-
-/* Low level functions exposed to the user API */
-sds sdsMakeRoomFor(sds s, size_t addlen);
-void sdsIncrLen(sds s, int incr);
-sds sdsRemoveFreeSpace(sds s);
-size_t sdsAllocSize(sds s);
-void *sdsAllocPtr(sds s);
-
-#ifdef REDIS_TEST
-int sdsTest(int argc, char *argv[]);
-#endif
-
-#endif
diff --git a/src/common/libutil/sdsalloc.h b/src/common/libutil/sdsalloc.h
deleted file mode 100644
index f43023c48438..000000000000
--- a/src/common/libutil/sdsalloc.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/* SDSLib 2.0 -- A C dynamic strings library
- *
- * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2015, Oran Agra
- * Copyright (c) 2015, Redis Labs, Inc
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-/* SDS allocator selection.
- *
- * This file is used in order to change the SDS allocator at compile time.
- * Just define the following defines to what you want to use. Also add
- * the include of your alternate allocator if needed (not needed in order
- * to use the default libc allocator). */
-
-#define s_malloc malloc
-#define s_realloc realloc
-#define s_free free
diff --git a/src/common/libutil/sigutil.c b/src/common/libutil/sigutil.c
new file mode 100644
index 000000000000..5c05b9379a0b
--- /dev/null
+++ b/src/common/libutil/sigutil.c
@@ -0,0 +1,133 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+#include "sigutil.h"
+
+struct signal_info {
+    int signum;
+    const char *name;
+};
+
+#define SIGDEF(x) { x, #x }
+
+/* Note:
+ *
+ * This list was generated on Debian 11 with the cmdline:
+ *
+ *  $ kill -l \
+ *      | sed 's/[0-9]*)//g' \
+ *      | xargs -n1 printf '    SIGDEF(%s),\n' \
+ *      | grep -v SIGRT
+ */
+static const struct signal_info signals[] = {
+    SIGDEF(SIGHUP),
+    SIGDEF(SIGINT),
+    SIGDEF(SIGQUIT),
+    SIGDEF(SIGILL),
+    SIGDEF(SIGTRAP),
+    SIGDEF(SIGABRT),
+    SIGDEF(SIGBUS),
+    SIGDEF(SIGFPE),
+    SIGDEF(SIGKILL),
+    SIGDEF(SIGUSR1),
+    SIGDEF(SIGSEGV),
+    SIGDEF(SIGUSR2),
+    SIGDEF(SIGPIPE),
+    SIGDEF(SIGALRM),
+    SIGDEF(SIGTERM),
+#ifdef SIGSTKFLT
+    SIGDEF(SIGSTKFLT),
+#endif
+    SIGDEF(SIGCHLD),
+    SIGDEF(SIGCONT),
+    SIGDEF(SIGSTOP),
+    SIGDEF(SIGTSTP),
+    SIGDEF(SIGTTIN),
+    SIGDEF(SIGTTOU),
+    SIGDEF(SIGURG),
+    SIGDEF(SIGXCPU),
+    SIGDEF(SIGXFSZ),
+    SIGDEF(SIGVTALRM),
+    SIGDEF(SIGPROF),
+    SIGDEF(SIGWINCH),
+    SIGDEF(SIGIO),
+#ifdef SIGPWR
+    SIGDEF(SIGPWR),
+#endif
+    SIGDEF(SIGSYS),
+};
+
+static bool strisnumber (const char *s, int *result)
+{
+    char *endptr;
+    long int l;
+
+    errno = 0;
+    l = strtol (s, &endptr, 10);
+    if (errno || *endptr != '\0') {
+        return false;
+    }
+    *result = (int) l;
+    return true;
+}
+
+int sigutil_signum (const char *s)
+{
+    int signum;
+    if (s == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (strisnumber (s, &signum)) {
+        if (signum <= 0) {
+            errno = EINVAL;
+            return -1;
+        }
+        return signum;
+    }
+    for (int i = 0; i < ARRAY_SIZE(signals); i++) {
+        if (streq (s, signals[i].name)
+            || streq (s, signals[i].name+3))
+            return signals[i].signum;
+    }
+    errno = ENOENT;
+    return -1;
+}
+
+const char *sigutil_signame (int signum)
+{
+    if (signum <= 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    for (int i = 0; i < ARRAY_SIZE(signals); i++) {
+        if (signals[i].signum == signum)
+            return signals[i].name;
+    }
+    errno = ENOENT;
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/sigutil.h b/src/common/libutil/sigutil.h
new file mode 100644
index 000000000000..cac4be473370
--- /dev/null
+++ b/src/common/libutil/sigutil.h
@@ -0,0 +1,26 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_SIGUTIL_H
+#define _UTIL_SIGUTIL_H
+
+/* Return signal number given a string, e.g. "SIGINT" or "INT"
+ */
+int sigutil_signum (const char *s);
+
+/* Return signal name given number, e.g. 10 -> "SIGUSR1"
+ */
+const char *sigutil_signame (int signum);
+
+#endif /* !_UTIL_SIGUTIL_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/slice.c b/src/common/libutil/slice.c
new file mode 100644
index 000000000000..5874d674b391
--- /dev/null
+++ b/src/common/libutil/slice.c
@@ -0,0 +1,166 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* slice.c - python style array slicing
+ *
+ * https://python-reference.readthedocs.io/en/latest/docs/brackets/slicing.html
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <string.h>
+
+#include "slice.h"
+
+static int parse_enclosing (char **cp, char begin, char end)
+{
+    size_t slen = strlen (*cp);
+
+    if (slen < 2)
+        return -1;
+    if ((*cp)[0] != begin || (*cp)[slen - 1] != end)
+        return -1;
+    (*cp)[slen - 1] = '\0';
+    (*cp)++;
+    return 0;
+}
+
+// returns 1 if value was provided, 0 if default was assigned, -1 if error
+static int parse_int (char **cp, int def, char sep, int *value)
+{
+    int v;
+    char *endptr;
+
+    errno = 0;
+    v = strtol (*cp, &endptr, 10);
+    // standard says EINVAL may be set if zero is returned for nothing parsed
+    if (v != 0 && errno != 0)
+        return -1;
+    if (endptr == *cp) {    // no digits
+        if ((*cp)[0] == sep)
+            (*cp)++;
+        else if ((*cp)[0] != '\0')
+            return -1;
+        *value = def;
+        return 0;
+    }
+    if (*endptr == '\0') {  // entire string consumed
+        *cp = endptr;
+        *value = v;
+        return 1;
+    }
+    if (*endptr == sep) {   // string consumed to separator
+        *cp = endptr + 1;
+        *value = v;
+        return 1;
+    }
+    return -1;
+}
+
+// return true if index has surpassed bounds of array or slice
+static bool overrun (struct slice *sl, int i)
+{
+    if (sl->step > 0 && (i >= sl->stop || i >= sl->length))
+        return true;
+    if (sl->step < 0 && (i <= sl->stop || i < 0))
+        return true;
+    return false;
+}
+
+// set cursor to first slice index that's in array bounds
+static void cursor_first (struct slice *sl)
+{
+    sl->cursor = sl->start;
+    while (!overrun (sl, sl->cursor)) {
+        if (sl->cursor >= 0 && sl->cursor < sl->length)
+            return;
+        sl->cursor += sl->step;
+    }
+    sl->cursor = -1;
+}
+
+// set cursor to next slice index
+static void cursor_next (struct slice *sl)
+{
+    if (sl->cursor != -1) {
+        do {
+            sl->cursor += sl->step;
+            if (overrun (sl, sl->cursor)) {
+                sl->cursor = -1;
+                return;
+            }
+        } while (sl->cursor < 0 || sl->cursor >= sl->length);
+    }
+}
+
+int slice_first (struct slice *sl)
+{
+    if (!sl)
+        return -1;
+    cursor_first (sl);
+    return slice_next (sl);
+}
+
+int slice_next (struct slice *sl)
+{
+    if (!sl)
+        return -1;
+    int i = sl->cursor;
+    cursor_next (sl);
+    return i;
+}
+
+int slice_parse (struct slice *sl, const char *s, size_t array_length)
+{
+    char *cpy;
+    char *cp;
+    int rc1, rc2;
+
+    if (!sl || !s || !strchr (s, ':'))
+        return -1;
+    if (!(cpy = strdup (s)))
+        return -1;
+    sl->length = array_length;
+    cp = cpy;
+    if (parse_enclosing (&cp, '[', ']') < 0)
+        goto error;
+    if ((rc1 = parse_int (&cp, 0, ':', &sl->start)) < 0)
+        goto error;
+    if ((rc2 = parse_int (&cp, array_length, ':', &sl->stop)) < 0)
+        goto error;
+    if (parse_int (&cp, 1, ':', &sl->step) < 0)
+        goto error;
+    if (sl->step == 0)
+        goto error;
+    // transform negative indices to positive
+    if (sl->start < 0)
+        sl->start = sl->length + sl->start;
+    if (sl->stop < 0)
+        sl->stop = sl->length + sl->stop;
+    // fix up default step/stop assigned above if step is negative
+    if (sl->step < 0) {
+        if (rc1 == 0)
+            sl->start = array_length - 1;
+        if (rc2 == 0)
+            sl->stop = -1;
+    }
+    cursor_first (sl);
+    free (cpy);
+    return 0;
+error:
+    free (cpy);
+    return -1;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/slice.h b/src/common/libutil/slice.h
new file mode 100644
index 000000000000..440f69add87e
--- /dev/null
+++ b/src/common/libutil/slice.h
@@ -0,0 +1,36 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_SLICE_H
+#define _UTIL_SLICE_H
+
+struct slice {
+    int start;
+    int stop;
+    int step;
+
+    size_t length;
+    int cursor;
+};
+
+/* Parse 's' as a python style array slice, e.g. [start:stop:step].
+ * 'array_length' is the length of the array to be sliced.
+ * Returns 0 on success, -1 on failure.  Errno is undefined on failure.
+ */
+int slice_parse (struct slice *sl, const char *s, size_t array_length);
+
+/* Built in iterator returns zero-origin sliced array indices (-1 at end).
+ */
+int slice_first (struct slice *sl);
+int slice_next (struct slice *sl);
+
+#endif /* !_UTIL_SLICE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/stdlog.c b/src/common/libutil/stdlog.c
index 8fb6de644a8c..e289471cf6cc 100644
--- a/src/common/libutil/stdlog.c
+++ b/src/common/libutil/stdlog.c
@@ -52,11 +52,14 @@ static int next_str (char **p, char **result)
     return 0;
 }
 
-static int next_structured_data (const char *buf, int len, int *offp,
-                                 const char **sp, int *slenp)
+static int next_structured_data (const char *buf,
+                                 size_t len,
+                                 size_t *offp,
+                                 const char **sp,
+                                 size_t *slenp)
 {
-    int off = *offp;
-    int this = *offp;
+    size_t off = *offp;
+    size_t this = *offp;
     int level = 0;
 
     while (off < len) {
@@ -78,14 +81,18 @@ static int next_structured_data (const char *buf, int len, int *offp,
     return 0;
 }
 
-int stdlog_decode (const char *buf, int len, struct stdlog_header *hdr,
-                   const char **sdp, int *sdlenp,
-                   const char **msgp, int *msglenp)
+int stdlog_decode (const char *buf,
+                   size_t len,
+                   struct stdlog_header *hdr,
+                   const char **sdp,
+                   size_t *sdlenp,
+                   const char **msgp,
+                   size_t *msglenp)
 {
-    int hdr_len = STDLOG_MAX_HEADER;
+    size_t hdr_len = STDLOG_MAX_HEADER;
     char *p = &hdr->buf[0];
     const char *sd;
-    int off, sdlen;
+    size_t off, sdlen;
 
     if (hdr_len > len)
         hdr_len = len;
@@ -126,14 +133,14 @@ int stdlog_decode (const char *buf, int len, struct stdlog_header *hdr,
     return 0;
 }
 
-char *stdlog_split_message (const char *buf, int *len, const char *sep)
+char *stdlog_split_message (const char *buf, size_t *len, const char *sep)
 {
     struct stdlog_header hdr;
     const char *msg;
-    int msglen;
-    int off;
+    size_t msglen;
+    size_t off;
     char *xtra;
-    int xtra_len;
+    size_t xtra_len;
 
     if (stdlog_decode (buf, *len, &hdr, NULL, NULL, &msg, &msglen) < 0)
         return NULL;
@@ -158,19 +165,31 @@ char *stdlog_split_message (const char *buf, int *len, const char *sep)
     return xtra;
 }
 
-int stdlog_vencodef (char *buf, int len, struct stdlog_header *hdr,
-                     const char *sd, const char *fmt, va_list ap)
+int stdlog_vencodef (char *buf,
+                     size_t len,
+                     struct stdlog_header *hdr,
+                     const char *sd,
+                     const char *fmt,
+                     va_list ap)
 {
     int m, n, i;
     int rc; // includes any overflow
 
-    m = snprintf (buf, len, "<%d>%d %.*s %.*s %.*s %.*s %.*s %s ",
-                  hdr->pri, hdr->version,
-                  STDLOG_MAX_TIMESTAMP, hdr->timestamp,
-                  STDLOG_MAX_HOSTNAME, hdr->hostname,
-                  STDLOG_MAX_APPNAME, hdr->appname,
-                  STDLOG_MAX_PROCID, hdr->procid,
-                  STDLOG_MAX_MSGID, hdr->msgid,
+    m = snprintf (buf,
+                  len,
+                  "<%d>%d %.*s %.*s %.*s %.*s %.*s %s ",
+                  hdr->pri,
+                  hdr->version,
+                  STDLOG_MAX_TIMESTAMP,
+                  hdr->timestamp,
+                  STDLOG_MAX_HOSTNAME,
+                  hdr->hostname,
+                  STDLOG_MAX_APPNAME,
+                  hdr->appname,
+                  STDLOG_MAX_PROCID,
+                  hdr->procid,
+                  STDLOG_MAX_MSGID,
+                  hdr->msgid,
                   sd);
     rc = m;
     if (m > len)
@@ -180,8 +199,6 @@ int stdlog_vencodef (char *buf, int len, struct stdlog_header *hdr,
     rc += n;
     if (n > len - m)
         n = len - m;
-    for (i = 0; i < n; i++)
-        buf[m + i] &= 0x7f; // ensure only ascii chars are logged
     for (i = n - 1; i >= 0; i--) {
         if (buf[m + i] != '\r' && buf[m + i] != '\n')
             break;
@@ -191,8 +208,12 @@ int stdlog_vencodef (char *buf, int len, struct stdlog_header *hdr,
     return rc;
 }
 
-int stdlog_encodef (char *buf, int len, struct stdlog_header *hdr,
-                    const char *sd, const char *fmt, ...)
+int stdlog_encodef (char *buf,
+                    size_t len,
+                    struct stdlog_header *hdr,
+                    const char *sd,
+                    const char *fmt,
+                    ...)
 {
     va_list ap;
     int rc;
@@ -203,8 +224,11 @@ int stdlog_encodef (char *buf, int len, struct stdlog_header *hdr,
     return rc;
 }
 
-int stdlog_encode (char *buf, int len, struct stdlog_header *hdr,
-                   const char *sd, const char *msg)
+int stdlog_encode (char *buf,
+                   size_t len,
+                   struct stdlog_header *hdr,
+                   const char *sd,
+                   const char *msg)
 {
     return stdlog_encodef (buf, len, hdr, sd, "%s", msg);
 }
diff --git a/src/common/libutil/stdlog.h b/src/common/libutil/stdlog.h
index 5d7194aebdad..540f057a74f9 100644
--- a/src/common/libutil/stdlog.h
+++ b/src/common/libutil/stdlog.h
@@ -43,17 +43,33 @@ struct stdlog_header {
     char *msgid;
 };
 
-int stdlog_decode (const char *buf, int len, struct stdlog_header *hdr,
-                   const char **sd, int *sdlen, const char **msg, int *msglen);
-
-int stdlog_encode (char *buf, int len, struct stdlog_header *hdr,
-                   const char *sd, const char *msg);
-
-int stdlog_vencodef (char *buf, int len, struct stdlog_header *hdr,
-                     const char *sd, const char *fmt, va_list ap);
-
-int stdlog_encodef (char *buf, int len, struct stdlog_header *hdr,
-                    const char *sd, const char *fmt, ...);
+int stdlog_decode (const char *buf,
+                   size_t len,
+                   struct stdlog_header *hdr,
+                   const char **sd,
+                   size_t *sdlen,
+                   const char **msg,
+                   size_t *msglen);
+
+int stdlog_encode (char *buf,
+                   size_t len,
+                   struct stdlog_header *hdr,
+                   const char *sd,
+                   const char *msg);
+
+int stdlog_vencodef (char *buf,
+                     size_t len,
+                     struct stdlog_header *hdr,
+                     const char *sd,
+                     const char *fmt,
+                     va_list ap);
+
+int stdlog_encodef (char *buf,
+                    size_t len,
+                    struct stdlog_header *hdr,
+                    const char *sd,
+                    const char *fmt,
+                    ...);
 
 /* If encoded stdlog message in buf, *len contains chars from 'sep'
  * (in the message part), truncate the original message and return
@@ -61,7 +77,7 @@ int stdlog_encodef (char *buf, int len, struct stdlog_header *hdr,
  * Returns NULL if no 'sep' chars or on alloc failure.
  * If non-NULL, caller must free returned value.
  */
-char *stdlog_split_message (const char *buf, int *len, const char *sep);
+char *stdlog_split_message (const char *buf, size_t *len, const char *sep);
 
 void stdlog_init (struct stdlog_header *hdr);
 
diff --git a/src/common/libutil/strstrip.c b/src/common/libutil/strstrip.c
new file mode 100644
index 000000000000..52706755ab42
--- /dev/null
+++ b/src/common/libutil/strstrip.c
@@ -0,0 +1,79 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <errno.h>
+#include <ctype.h>
+
+char *strstrip (char *s)
+{
+    size_t size;
+    char *end;
+
+    if (!s) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    size = strlen (s);
+
+    if (!size)
+        return s;
+
+    end = s + size - 1;
+    while (end >= s && isspace ((unsigned char) *end))
+        end--;
+    *(end + 1) = '\0';
+
+    while (*s && isspace((unsigned char) *s))
+        s++;
+
+    return s;
+}
+
+char *strstrip_copy (const char *s)
+{
+    size_t size;
+    char *end;
+    char *result = NULL;
+
+    if (!s) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (s[0] == '\0')
+        return strdup (s);
+
+    /* Work from front to back so we do not have to copy all of 's'
+     */
+    while (*s && isspace ((unsigned char) *s))
+        s++;
+
+    if (!(result = strdup (s)))
+        return NULL;
+
+    if ((size = strlen (result)) == 0)
+        return result;
+
+    end = result + size - 1;
+    while (end >= result && isspace ((unsigned char) *end))
+        end--;
+    *(end + 1) = '\0';
+
+    return result;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libutil/strstrip.h b/src/common/libutil/strstrip.h
new file mode 100644
index 000000000000..bb479a75f422
--- /dev/null
+++ b/src/common/libutil/strstrip.h
@@ -0,0 +1,25 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_STRSTRIP_H
+#define _UTIL_STRSTRIP_H
+
+/*  Strip leading and trailing whitespace from string 's'.
+ *  Returns a pointer inside of 's'.
+ */
+char *strstrip (char *s);
+
+/*  Like strstrip(), but returns a copy of stripped 's'
+ */
+char *strstrip_copy (char *s);
+
+#endif /* !_UTIL_STRSTRIP_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/test/aux.c b/src/common/libutil/test/aux.c
index ef4f92aef940..6bab1f3ac0c7 100644
--- a/src/common/libutil/test/aux.c
+++ b/src/common/libutil/test/aux.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 #include <errno.h>
 
@@ -176,6 +179,38 @@ void simple_test (void)
         "aux_destroy aux=NULL doesn't crash");
 }
 
+void test_delete (void)
+{
+    struct aux_item *aux = NULL;
+    int items[8];
+    int i;
+
+    for (i = 0; i < 8; i++)
+        if (aux_set (&aux, NULL, &items[i], myfree) < 0)
+            BAIL_OUT ("aux_set failed on item %d", i);
+
+    myfree_count = 0;
+    aux_delete (NULL, "foo");
+    ok (myfree_count == 0,
+        "aux_delete aux=NULL does nothing");
+
+    myfree_count = 0;
+    aux_delete (&aux, NULL);
+    ok (myfree_count == 0,
+        "aux_delete val=NULL does nothing");
+
+    myfree_count = 0;
+    aux_delete (&aux, &i);
+    ok (myfree_count == 0,
+        "aux_delete val=unknown does nothing");
+
+    myfree_count = 0;
+    for (i = 0; i < 8; i++)
+        aux_delete (&aux, &items[i]);
+    ok (myfree_count == 8,
+        "aux_delete works with valid pointer");
+}
+
 int main (int argc, char *argv[])
 {
     plan (NO_PLAN);
@@ -183,6 +218,7 @@ int main (int argc, char *argv[])
     simple_test ();
     aux_destroy_no_get_self ();
     aux_destroy_set_ok ();
+    test_delete ();
 
     done_testing ();
 
diff --git a/src/common/libutil/test/basemoji.c b/src/common/libutil/test/basemoji.c
new file mode 100644
index 000000000000..513b662a172a
--- /dev/null
+++ b/src/common/libutil/test/basemoji.c
@@ -0,0 +1,114 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <string.h>
+#include <locale.h>
+#include <stdint.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/basemoji.h"
+#include "ccan/str/str.h"
+
+static const char invalid[] = "ƒ1234";
+static const char valid[] = "😪🏭🐭🍑👨";
+
+struct basemoji_test {
+    uint64_t id;
+    const char *result;
+} basemoji_tests[] =  {
+    { 0,                       "😃" },
+    { 1,                       "😄" },
+    { 1234,                    "😁👌" },
+    { 65535,                   "💁📚" },
+    { 12342435,                "😠🙇📍" },
+    { 2034152287593,           "😪🏭🐭🍑👨" },
+    { 21900760568561664,       "🎆🍧🎆🐾🕓😃" },
+    { 18446743892750589633ULL, "🚹💗🔥😜💟🎱💃" },
+    { 18446744073709551615ULL, "🚹💗💧👗😷📷📚" },
+    { 0,                        NULL },
+};
+
+void test_basic (void)
+{
+    struct basemoji_test *tp = &basemoji_tests[0];
+    char buf[30];
+
+    while (tp->result != NULL) {
+        uint64_t id;
+
+        ok (uint64_basemoji_encode (tp->id, buf, sizeof (buf)) == 0,
+            "uint64_basemoji_encode (%ju)",
+            tp->id);
+        ok (strcmp (buf, tp->result) == 0,
+            "result: %s, wanted: %s",
+            buf,
+            tp->result);
+        ok (is_basemoji_string (tp->result),
+            "is_basemoji_string (%s) works",
+            tp->result);
+        ok (uint64_basemoji_decode (buf, &id) == 0,
+            "uint64_basemoji_decode worked");
+        ok (id == tp->id,
+            "expected %ju got %ju",
+            (uintmax_t) tp->id,
+            (uintmax_t) id);
+        ++tp;
+    }
+}
+
+void test_errors (void)
+{
+    char buf[30];
+    uint64_t id;
+
+    ok (uint64_basemoji_encode (0, NULL, 0) < 0 && errno == EINVAL,
+        "uint64_basemoji_encode (0, NULL, 0) fails with EINVAL");
+    ok (uint64_basemoji_encode (0, buf, -1) < 0 && errno == EINVAL,
+        "uint64_basemoji_encode (0, buf, -1) fails with EINVAL");
+
+    ok (uint64_basemoji_encode (0, buf, 3) < 0 && errno == EOVERFLOW,
+        "uint64_basemoji_encode (0, buf, 3) fails with EOVERFLOW");
+    ok (uint64_basemoji_encode (UINT64_MAX, buf, 28) < 0
+        && errno == EOVERFLOW,
+        "uint64_basemoji_encode (UINT64_MAX, buf, 28) fails with EOVERFLOW");
+
+    ok (uint64_basemoji_decode (NULL, NULL) < 0 && errno == EINVAL,
+        "uint64_basemoji_decode (NULL, NULL) fails with EINVAL");
+    ok (uint64_basemoji_decode ("",  NULL) < 0 && errno == EINVAL,
+        "uint64_basemoji_decode (\"\", NULL) fails with EINVAL");
+    ok (uint64_basemoji_decode (valid,  NULL) < 0 && errno == EINVAL,
+        "uint64_basemoji_decode (valid, NULL) fails with EINVAL");
+    ok (uint64_basemoji_decode ("f",  &id) < 0 && errno == EINVAL,
+        "uint64_basemoji_decode (\"f\", NULL) fails with EINVAL");
+    ok (uint64_basemoji_decode (invalid,  &id) < 0 && errno == EINVAL,
+        "uint64_basemoji_decode (invalid, &id) fails with EINVAL");
+
+    ok (!is_basemoji_string (invalid),
+        "is_basemoji_string (invalid) returns false");
+    ok (!is_basemoji_string (""),
+        "is_basemoji_string (\"\") returns false");
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+    test_errors ();
+    test_basic ();
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/test/blobref.c b/src/common/libutil/test/blobref.c
index b6cb83e11f51..71047c8cc524 100644
--- a/src/common/libutil/test/blobref.c
+++ b/src/common/libutil/test/blobref.c
@@ -8,12 +8,17 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 #include <errno.h>
+
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/blobref.h"
 #include "src/common/libutil/sha1.h"
 #include "src/common/libutil/sha256.h"
+#include "ccan/str/str.h"
 
 const char *badref[] = {
     "nerf-4d4ed591f7d26abd8145650f334d283bdb661765", // unknown hash
@@ -54,6 +59,20 @@ int main(int argc, char** argv)
         && errno == EINVAL,
         "blobref_hash fails EINVAL with invalid ref length");
 
+    errno = 0;
+    ok (blobref_hash_raw ("nerf", data, sizeof (data),
+                          digest, sizeof (digest)) < 0
+        && errno == EINVAL,
+        "blobref_hash_raw fails EINVAL with unknown hash name");
+    errno = 0;
+    ok (blobref_hash_raw ("sha1", data, sizeof (data), NULL, 0) < 0
+        && errno == EINVAL,
+        "blobref_hash_raw fails EINVAL with NULL hash pointer");
+    errno = 0;
+    ok (blobref_hash_raw ("sha1", data, sizeof (data), digest, 1) < 0
+        && errno == EINVAL,
+        "blobref_hash_raw fails EINVAL with invalid hash length");
+
     errno = 0;
     ok (blobref_strtohash (badref[0], digest, sizeof (digest)) < 0
         && errno == EINVAL,
@@ -105,13 +124,22 @@ int main(int argc, char** argv)
         "blobref_hash sha1 works");
     diag ("%s", ref);
 
+    ok (blobref_hash_raw ("sha1",
+                          NULL, 0,
+                          digest, sizeof (digest)) == SHA1_DIGEST_SIZE,
+        "blobref_hash_raw sha1 handles zero length data");
+    ok (blobref_hash_raw ("sha1",
+                          data, sizeof (data),
+                          digest, sizeof (digest)) == SHA1_DIGEST_SIZE,
+        "blobref_hash_raw sha1 works");
+
     ok (blobref_strtohash (ref, digest, sizeof (digest)) == SHA1_DIGEST_SIZE,
         "blobref_strtohash returns expected size hash");
     ok (blobref_hashtostr ("sha1", digest, SHA1_DIGEST_SIZE, ref2,
                            sizeof (ref2)) == 0,
         "blobref_hashtostr back again works");
     diag ("%s", ref2);
-    ok (strcmp (ref, ref2) == 0,
+    ok (streq (ref, ref2),
         "and blobrefs match");
 
     /* sha256 */
@@ -122,13 +150,22 @@ int main(int argc, char** argv)
         "blobref_hash sha256 works");
     diag ("%s", ref);
 
+    ok (blobref_hash_raw ("sha256",
+                          NULL, 0,
+                          digest, sizeof (digest)) == SHA256_BLOCK_SIZE,
+        "blobref_hash_raw sha256 handles zero length data");
+    ok (blobref_hash_raw ("sha256",
+                          data, sizeof (data),
+                          digest, sizeof (digest)) == SHA256_BLOCK_SIZE,
+        "blobref_hash_raw sha256 works");
+
     ok (blobref_strtohash (ref, digest, sizeof (digest)) == SHA256_BLOCK_SIZE,
         "blobref_strtohash returns expected size hash");
     ok (blobref_hashtostr ("sha256", digest, SHA256_BLOCK_SIZE, ref2,
                            sizeof (ref2)) == 0,
         "blobref_hashtostr back again works");
     diag ("%s", ref2);
-    ok (strcmp (ref, ref2) == 0,
+    ok (streq (ref, ref2),
         "and blobrefs match");
 
     /* blobref_validate */
@@ -148,9 +185,9 @@ int main(int argc, char** argv)
     }
 
     /* blobref_validate_hashtype */
-    ok (blobref_validate_hashtype ("sha1") == 0,
+    ok (blobref_validate_hashtype ("sha1") == SHA1_DIGEST_SIZE,
         "blobref_validate_hashtype sha1 is valid");
-    ok (blobref_validate_hashtype ("sha256") == 0,
+    ok (blobref_validate_hashtype ("sha256") == SHA256_BLOCK_SIZE,
         "blobref_validate_hashtype sha256 is valid");
     ok (blobref_validate_hashtype ("nerf") == -1,
         "blobref_validate_hashtype nerf is invalid");
diff --git a/src/common/libutil/test/cidr.c b/src/common/libutil/test/cidr.c
new file mode 100644
index 000000000000..e19b8541a590
--- /dev/null
+++ b/src/common/libutil/test/cidr.c
@@ -0,0 +1,127 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/param.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/cidr.h"
+#include "ccan/array_size/array_size.h"
+
+struct cidr_test {
+    const char *input;
+    const char *addr;
+    const char *mask;
+    const char *match;
+    const char *nomatch;
+};
+
+struct cidr_test testvec[] = {
+    {   .input = "0.0.0.0/16",
+        .addr = "0.0.0.0",
+        .mask = "255.255.0.0",
+        .match = "0.0.255.255",
+        .nomatch = "1.1.1.1"
+    },
+    {   .input = "255.255.255.255/8",
+        .addr = "255.255.255.255",
+        .mask = "255.0.0.0",
+        .match = "255.1.1.1",
+        .nomatch = "254.1.1.1"
+    },
+    {   .input = "192.168.0.0/24",
+        .addr = "192.168.0.0",
+        .mask = "255.255.255.0",
+        .match = "192.168.0.1",
+        .nomatch = "192.168.1.1"
+    },
+    {   .input = "192.168.0.1/32", // "host route" in RFC 4632
+        .addr = "192.168.0.1",
+        .mask = "255.255.255.255",
+        .match = "192.168.0.1",
+        .nomatch = "192.168.0.2"
+    },
+    {   .input = "192.168.0.0",
+        .addr = "192.168.0.0",
+        .mask = "255.255.255.255",
+        .match = "192.168.0.0",
+        .nomatch = "192.168.0.1"
+    },
+    {   .input = "0.0.0.0/0",   // "default route" in RFC 4632
+        .addr = "0.0.0.0",
+        .mask = "0.0.0.0",
+        .match = "255.255.255.255",
+        NULL    // skip
+    },
+};
+
+bool addr_is (struct in_addr *addr, const char *s)
+{
+    struct in_addr a;
+    if (inet_pton (AF_INET, s, &a) < 0 || a.s_addr != addr->s_addr)
+        return false;
+    return true;
+}
+
+bool match_addr (struct cidr4 *cidr, const char *s)
+{
+    struct in_addr a;
+    if (inet_pton (AF_INET, s, &a) < 0 || !cidr_match4 (cidr, &a))
+        return false;
+    return true;
+}
+
+int main (int argc, char** argv)
+{
+    int n;
+    struct cidr4 cidr;
+    struct in_addr addr;
+
+    plan (NO_PLAN);
+
+    for (int i = 0; i < ARRAY_SIZE (testvec); i++) {
+        n = cidr_parse4 (&cidr, testvec[i].input);
+        ok (n == 0
+            && addr_is (&cidr.addr, testvec[i].addr)
+            && addr_is (&cidr.mask, testvec[i].mask),
+            "%s => %s/%s",
+            testvec[i].input,
+            testvec[i].addr,
+            testvec[i].mask);
+
+        if (testvec[i].match) {
+            ok (match_addr (&cidr, testvec[i].match) == true,
+                "%s matches", testvec[i].match);
+        }
+        if (testvec[i].nomatch) {
+            ok (match_addr (&cidr, testvec[i].nomatch) == false,
+                "%.8x does not match", testvec[i].nomatch);
+        }
+    }
+
+    errno = 0;
+    ok (cidr_parse4 (NULL, "0.0.0.0") < 0 && errno == EINVAL,
+        "cidr_parse4 cidr=NULL fails with EINVAL");
+    errno = 0;
+    ok (cidr_parse4 (&cidr, NULL) < 0 && errno == EINVAL,
+        "cidr_parse4 s=NULL fails with EINVAL");
+
+    ok (cidr_match4 (NULL, &addr) == false,
+        "cidr_match4 cidr=NULL returns false");
+    ok (cidr_match4 (&cidr, NULL) == false,
+        "cidr_match4 addr=NULL returns false");
+
+    done_testing ();
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/test/cleanup.c b/src/common/libutil/test/cleanup.c
index e6a3be575f7d..76591c8e9874 100644
--- a/src/common/libutil/test/cleanup.c
+++ b/src/common/libutil/test/cleanup.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <limits.h>
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -16,7 +19,6 @@
 #include <string.h>
 #include <errno.h>
 
-
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/unlink_recursive.h"
 #include "src/common/libutil/cleanup.h"
diff --git a/src/common/libutil/test/cronodate.c b/src/common/libutil/test/cronodate.c
index 4331d4cfaba7..bbded8d3624c 100644
--- a/src/common/libutil/test/cronodate.c
+++ b/src/common/libutil/test/cronodate.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #define _XOPEN_SOURCE 700
 #include <stdlib.h>
 #include <time.h>
diff --git a/src/common/libutil/test/dirwalk.c b/src/common/libutil/test/dirwalk.c
index e98ace8ecac8..9e214e705a2a 100644
--- a/src/common/libutil/test/dirwalk.c
+++ b/src/common/libutil/test/dirwalk.c
@@ -8,7 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#define _GNU_SOURCE
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <limits.h>
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -18,13 +20,17 @@
 #include <stdarg.h>
 #include <errno.h>
 #include <stdio.h>
+#ifndef HAVE_GET_CURRENT_DIR_NAME
+#include "src/common/libmissing/get_current_dir_name.h"
+#endif
 
 #include <dirent.h>
 
-#include <czmq.h>
-
 #include "src/common/libtap/tap.h"
-#include "src/common/libutil/dirwalk.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+#include "basename.h"
+#include "dirwalk.h"
 
 static int makepath (const char *fmt, ...)
 {
@@ -102,7 +108,7 @@ static char * create_test_dir ()
 
 static int find_dir (dirwalk_t *d, void *arg)
 {
-    return (dirwalk_isdir (d) ? 1 : 0); 
+    return (dirwalk_isdir (d) ? 1 : 0);
 }
 
 static int return_err (dirwalk_t *d, void *arg)
@@ -158,7 +164,6 @@ int check_zlist_order (zlist_t *l, const char *base, char *expected[])
     dir = zlist_first (l);
     while (dir) {
         diag ("zlist_order: %d: %s", i, dir);
-        int result;
         char *exp;
         if (expected[i] == NULL) {
             diag ("check_zlist: more results than expected=%d\n", i-1);
@@ -167,8 +172,7 @@ int check_zlist_order (zlist_t *l, const char *base, char *expected[])
         if (asprintf (&exp, "%s%s", base, expected [i]) < 0)
             BAIL_OUT ("asprintf");
 
-        result = strcmp (exp, dir);
-        if (result != 0) {
+        if (!streq (exp, dir)) {
             diag ("check_zlist: %d: expected %s got %s", i, exp, dir);
             free (exp);
             return 0;
@@ -232,6 +236,9 @@ int main(int argc, char** argv)
     n = dirwalk (tmp, 0, NULL, NULL);
     ok (n == 4, "dirwalk of deeper dirtree");
 
+    n = dirwalk (tmp, DIRWALK_NORECURSE, NULL, NULL);
+    ok (n == 2, "dirwalk of deeper dirtree with NORECURSE works");
+
     if (makepath ("%s/a/b/c/d", tmp) < 0)
         BAIL_OUT ("makepath failed");
 
@@ -262,7 +269,7 @@ int main(int argc, char** argv)
     l = dirwalk_find (tmp, 0, "foo", 1, NULL, 0);
     ok (l != NULL, "dirwalk_find");
     ok (l && zlist_size (l) == 1, "dirwalk_find stopped at 1 result");
-    ok (strcmp (basename (zlist_first (l)), "foo") == 0,
+    ok (streq (basename_simple (zlist_first (l)), "foo"),
         "breadth-first search got expected match");
     zlist_destroy (&l);
 
@@ -280,7 +287,7 @@ int main(int argc, char** argv)
     ok (l && zlist_size (l) == 3, "find with search path found all matches");
     zlist_destroy (&l);
     free (s);
- 
+
     /* depth-first find */
     l = dirwalk_find (tmp, DIRWALK_DEPTH, "foo", 0, NULL, 0);
     ok (l != NULL, "dirwalk with find callback");
@@ -304,6 +311,16 @@ int main(int argc, char** argv)
         "depth-first visited directories in correct order");
     zlist_destroy (&l);
 
+    l = dirwalk_find (tmp,
+                      DIRWALK_FIND_DIR | DIRWALK_NORECURSE,
+                      "*",
+                      0,
+                      find_dir,
+                      NULL);
+    ok (l != NULL && zlist_size (l) == 2,
+        "dirwalk_find FIND_DIR|NORECURSE visits top level directories");
+    zlist_destroy (&l);
+
     flags = DIRWALK_FIND_DIR;
     l = dirwalk_find (tmp, flags, "*", 0, find_dir, NULL);
     ok (l && zlist_size (l) >  0, "dirwalk to find all dirs works");
diff --git a/src/common/libutil/test/environment.c b/src/common/libutil/test/environment.c
new file mode 100644
index 000000000000..30741035cc48
--- /dev/null
+++ b/src/common/libutil/test/environment.c
@@ -0,0 +1,84 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+
+#include "ccan/str/str.h"
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/environment.h"
+
+static void test_var_next ()
+{
+    const char *entry = NULL;
+    struct environment *e = environment_create ();
+    if (e == NULL)
+        BAIL_OUT ("Failed to create environment object!");
+    ok (environment_var_next (e, "PATH", NULL) == NULL,
+        "environment_var_next () returns NULL for missing env var");
+    environment_set (e, "PATH", "/bin:/usr/bin:/usr/local/bin", ':');
+    diag ("set PATH=/bin:/usr/bin:/usr/local/bin");
+    ok ((entry = environment_var_next (e, "PATH", entry)) != NULL,
+        "environment_var_next () works");
+    is (entry, "/bin",
+        "environment_var_next returns first element");
+    ok ((entry = environment_var_next (e, "PATH", entry)) != NULL,
+        "environment_var_next () works");
+    is (entry, "/usr/bin",
+        "environment_var_next returns next element");
+    ok ((entry = environment_var_next (e, "PATH", entry)) != NULL,
+        "environment_var_next () works");
+    is (entry, "/usr/local/bin",
+        "environment_var_next returns last element");
+    ok (!(entry = environment_var_next (e, "PATH", entry)),
+        "environment_var_next () returns NULL after last element");
+    environment_destroy (e);
+}
+
+static void test_insert ()
+{
+    const char *entry = NULL;
+    struct environment *e = environment_create ();
+    if (e == NULL)
+        BAIL_OUT ("Failed to create environment object!");
+    ok (environment_insert (e, "PATH", "/bin", "/foo") < 0 && errno == ENOENT,
+        "environment_insert on missing key returns ENOENT");
+    environment_set (e, "PATH", "/bin:/usr/bin:/usr/local/bin", ':');
+    diag ("set PATH=/bin:/usr/bin:/usr/local/bin");
+    diag ("searching for entry=/usr/bin");
+    while ((entry = environment_var_next (e, "PATH", entry)))
+        if (streq (entry, "/usr/bin"))
+            break;
+    diag ("entry=%s", entry);
+    ok (environment_insert (e, "PATH", (char *) entry, "/new/path") == 0,
+        "environment_insert /new/path before /usr/bin return success");
+    is (environment_get (e, "PATH"),
+        "/bin:/new/path:/usr/bin:/usr/local/bin",
+        "PATH is now /bin:/new/path:/usr/bin:/usr/local/bin");
+    environment_destroy (e);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_var_next ();
+    test_insert ();
+
+    done_testing ();
+
+    return 0;
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/test/errprintf.c b/src/common/libutil/test/errprintf.c
new file mode 100644
index 000000000000..bfc6480f83e6
--- /dev/null
+++ b/src/common/libutil/test/errprintf.c
@@ -0,0 +1,75 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <errno.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/errprintf.h"
+
+int main(int argc, char** argv)
+{
+    flux_error_t error;
+    char longstring[256];
+
+    lives_ok ({err_init (NULL);},
+              "err_init with NULL args doesn't crash");
+
+    lives_ok ({errprintf (NULL, NULL);},
+              "errprintf with no args doesn't crash");
+    lives_ok ({errprintf (&error, NULL);},
+              "errprintf with NULL format doesn't crash");
+    is (error.text, "",
+        "and returned error is empty");
+
+    errprintf (&error, "foo");
+    is (error.text, "foo",
+        "errprintf with static format works");
+
+    err_init (&error);
+    is (error.text, "",
+        "err_init zeros error.text buffer");
+
+    errno = 64;
+    errprintf (&error, "foo");
+    ok (errno == 64,
+        "errprintf preserves error");
+
+    errprintf (&error, "%s: %s", "foo", "bar");
+    is (error.text, "foo: bar",
+        "errprintf with simple format works");
+
+    for (int i = 0; i < sizeof (longstring) - 1; i++)
+        longstring[i] = 'x';
+    longstring[sizeof (longstring) - 1] = '\0';
+
+    errprintf (&error, "%s", longstring);
+    ok (strlen (error.text) == sizeof (error.text) - 1,
+        "errprintf with too long format properly truncates");
+    ok (error.text[sizeof (error.text) - 2] == '+',
+        "errprintf notes trucation with a '+'");
+    longstring[sizeof (error.text) - 1] = '\0';
+    longstring[sizeof (error.text) - 2] = '+';
+    is (error.text, longstring,
+        "error is expected");
+
+    ok (errprintf (&error, "Test error") == -1,
+        "errprintf() always returns -1");
+
+    done_testing();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/test/ev.c b/src/common/libutil/test/ev.c
deleted file mode 100644
index 36b0fca187a8..000000000000
--- a/src/common/libutil/test/ev.c
+++ /dev/null
@@ -1,282 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <czmq.h>
-#include <poll.h>
-
-#include "src/common/libutil/oom.h"
-#include "src/common/libev/ev.h"
-#include "src/common/libutil/ev_zmq.h"
-#include "src/common/libtap/tap.h"
-
-void timer_arg_cb (struct ev_loop *loop, ev_timer *w, int revents)
-{
-    int *i = w->data;
-    if (i && ++(*i) == 100)
-        ev_break (loop, EVBREAK_ALL);
-}
-
-void timer_cb (struct ev_loop *loop, ev_timer *w, int revents)
-{
-}
-
-void test_libev_timer (void)
-{
-    struct ev_loop *loop;
-    ev_timer w;
-    int i;
-
-    ok ((loop = ev_loop_new (EVFLAG_AUTO)) != NULL,
-        "ev_loop_new works");
-    ok (ev_run (loop, 0) == 0,
-        "ev_run returns 0 with no watchers configured");
-
-    ev_timer_init (&w, timer_cb, 1E-1, 0.);
-    ev_timer_start (loop, &w);
-    ok (ev_run (loop, 0) == 0,
-        "ev_run returns 0 after no-repeat timer fires once");
-    ev_timer_stop (loop, &w);
-
-    i = 0;
-    ev_timer_init (&w, timer_arg_cb, 1E-1, 0.);
-    w.data = &i;
-    ev_timer_start (loop, &w);
-    ok (ev_run (loop, 0) == 0 && i == 1,
-        "passing arbitrary data using w->data works");
-    ev_timer_stop (loop, &w);
-
-    i = 0;
-    ev_timer_init (&w, timer_arg_cb, 1E-3, 1E-3);
-    w.data = &i;
-    ev_timer_start (loop, &w);
-    ok (ev_run (loop, 0) != 0 && i == 100,
-        "ev_break causes ev_run to return nonzero");
-    ev_timer_stop (loop, &w);
-
-    ev_loop_destroy (loop);
-}
-
-void zero_cb (struct ev_loop *loop, ev_io *w, int revents)
-{
-    int *i = w->data;
-    ssize_t n;
-    char buf[1024];
-
-    if ((n = read (w->fd, buf, sizeof (buf))) < 0) {
-        fprintf (stderr, "read error on /dev/zero: %s\n", strerror (errno));
-        return;
-    }
-    if (n < sizeof (buf)) {
-        fprintf (stderr, "short read on /dev/zero\n");
-        return;
-    }
-    if (i && ++(*i) == 100)
-        ev_break (loop, EVBREAK_ALL);
-}
-
-void test_libev_io (void)
-{
-    struct ev_loop *loop;
-    ev_io w, w2;
-    int i, fd, fd2;
-
-    ok ((loop = ev_loop_new (EVFLAG_AUTO)) != NULL,
-        "ev_loop_new works");
-
-    /* handle 100 read events from /dev/zero with two workers */
-    fd = open ("/dev/zero", O_RDONLY);
-    fd2 = open ("/dev/zero", O_RDONLY);
-    ok (fd >= 0 && fd2 >= 0,
-        "opened /dev/zero twice");
-
-    i = 0;
-    ev_io_init (&w, zero_cb, fd, EV_READ);
-    w.data = &i;
-    ev_io_init (&w2, zero_cb, fd2, EV_READ);
-    w2.data = &i;
-    ev_io_start (loop, &w);
-    ev_io_start (loop, &w2);
-    ok (ev_run (loop, 0) != 0 && i == 100,
-        "ev_run ran two /dev/zero readers a total of 100 times");
-    ev_io_stop (loop, &w);
-    ev_io_stop (loop, &w2);
-
-    close (fd);
-    close (fd2);
-
-    ev_loop_destroy (loop);
-}
-
-/* test that zmq arcana we built ev_zmq on functions as advertised,
- * mainly the ZMQ_FD and ZMQ_EVENTS socket options that zmq_poll uses.
- */
-void test_zmq_events (void)
-{
-    void *zctx;
-    void *zin, *zout;
-    int fd;
-    char *s = NULL;
-
-    ok ((zctx = zmq_init (1)) != NULL,
-        "initialized zmq context");
-    ok ((zout = zmq_socket (zctx, ZMQ_PAIR)) != NULL
-        && zmq_bind (zout, "inproc://eventloop_test") == 0,
-        "PAIR socket bind ok");
-    ok ((zin = zmq_socket (zctx, ZMQ_PAIR)) != NULL
-        && zmq_connect (zin, "inproc://eventloop_test") == 0,
-        "PAIR socket connect ok");
-    size_t fd_size = sizeof (fd);
-    ok (zmq_getsockopt (zin, ZMQ_FD, &fd, &fd_size) == 0 && fd >= 0,
-        "zmq_getsockopt ZMQ_FD returned valid file descriptor");
-    /* ZMQ_EVENTS must be called after ZMQ_FD and before each poll()
-     * to "reset" event trigger.  For more details see Issue #524.
-     */
-    uint32_t zevents;
-    size_t zevents_size = sizeof (zevents);
-    ok (zmq_getsockopt (zin, ZMQ_EVENTS, &zevents, &zevents_size) == 0
-        && !(zevents & ZMQ_POLLIN),
-        "zmq_getsockopt ZMQ_EVENTS says PAIR socket not ready to recv");
-    // this test is somewhat questionable as there may be false positives
-    struct pollfd pfd = { .fd = fd, .events = POLLIN };
-    ok (poll (&pfd, 1, 10) == 0,
-        "poll says edge triggered mailbox descriptor is not ready");
-    ok (zstr_send (zout, "TEST") == 0,
-        "sent a message over PAIR sockets");
-    ok (poll (&pfd, 1, 10) == 1
-        && (pfd.revents & POLLIN),
-        "poll says edge triggered mailbox descriptor is ready");
-    ok (zmq_getsockopt (zin, ZMQ_EVENTS, &zevents, &zevents_size) == 0
-        && (zevents & ZMQ_POLLIN),
-        "zmq_getsockopt ZMQ_EVENTS says PAIR socket ready to recv");
-    zmq_pollitem_t zp = { .socket = zin, .events = ZMQ_POLLIN };
-    ok (zmq_poll (&zp, 1, 10) == 1 && zp.revents == ZMQ_POLLIN,
-        "zmq_poll says PAIR socket ready to recv");
-    ok ((s = zstr_recv (zin)) != NULL,
-        "received message over PAIR sockets");
-    ok (zmq_getsockopt (zin, ZMQ_EVENTS, &zevents, &zevents_size) == 0
-        && !(zevents & ZMQ_POLLIN),
-        "zmq_getsockopt ZMQ_EVENTS says PAIR socket not ready to recv");
-    ok (zmq_poll (&zp, 1, 10) == 0,
-        "zmq_poll says PAIR socket not ready to recv");
-    if (s)
-        free (s);
-    zmq_close (zin);
-    zmq_close (zout);
-    zmq_ctx_destroy (zctx);
-}
-
-void zsock_tx_cb (struct ev_loop *loop, ev_zmq *w, int revents)
-{
-    static int count = 50; /* send two per invocation */
-
-    if ((revents & EV_WRITE)) {
-        if (zstr_send (w->zsock, "PING") < 0)
-            fprintf (stderr, "zstr_send: %s", strerror (errno));
-        if (zstr_send (w->zsock, "PING") < 0)
-            fprintf (stderr, "zstr_send: %s", strerror (errno));
-        if (--count == 0)
-            ev_zmq_stop (loop, w);
-    }
-    if ((revents & EV_ERROR))
-        ev_break (loop, EVBREAK_ALL);
-}
-
-void zsock_rx_cb (struct ev_loop *loop, ev_zmq *w, int revents)
-{
-    int *iter = w->data;
-    char *s;
-    static int count = 100;
-
-    if ((revents & EV_READ)) {
-        (*iter)++;
-        if (!(s = zstr_recv (w->zsock)))
-            fprintf (stderr, "zstr_recv: %s", strerror (errno));
-        else
-            free (s);
-        if (--count == 0)
-            ev_zmq_stop (loop, w);
-    }
-    if ((revents & EV_ERROR))
-        ev_break (loop, EVBREAK_ALL);
-}
-
-
-/* send 100 messages over PAIR sockets
- * sender in one event handler, receiver in another
- */
-void test_ev_zmq (void)
-{
-    struct ev_loop *loop;
-    void *zctx;
-    void *zin, *zout;
-    int i;
-    ev_zmq win, wout;
-
-    ok ((loop = ev_loop_new (EVFLAG_AUTO)) != NULL,
-        "ev_loop_new works");
-    ok ((zctx = zmq_init (1)) != NULL,
-        "initialized zmq context");
-    ok ((zout = zmq_socket (zctx, ZMQ_PAIR)) != NULL
-        && zmq_bind (zout, "inproc://eventloop_test") == 0,
-        "PAIR socket bind ok");
-    ok ((zin = zmq_socket (zctx, ZMQ_PAIR)) != NULL
-        && zmq_connect (zin, "inproc://eventloop_test") == 0,
-        "PAIR socket connect ok");
-
-    i = 0;
-    ev_zmq_init (&win, zsock_rx_cb, zin, EV_READ);
-    win.data = &i;
-    ev_zmq_init (&wout, zsock_tx_cb, zout, EV_WRITE);
-
-    ev_zmq_start (loop, &win);
-    ev_zmq_start (loop, &wout);
-
-    ok (ev_run (loop, 0) == 0,
-        "both watchers removed themselves and ev_run exited");
-    ev_zmq_stop (loop, &win);
-    ev_zmq_stop (loop, &wout);
-    cmp_ok (i, "==", 100,
-        "ev_zmq handler ran 100 times");
-
-    ev_loop_destroy (loop);
-
-    zmq_close (zin);
-    zmq_close (zout);
-    zmq_ctx_destroy (zctx);
-}
-
-void list_timer_cb (struct ev_loop *loop, ev_timer *w, int revents)
-{
-    static int i = 100;
-    zlist_t *l = w->data;
-    if (--i == 0) {
-        ev_break (loop, EVBREAK_ALL);
-    } else {
-        zmsg_t *zmsg;
-        if (!(zmsg = zmsg_new ()) || zlist_append (l, zmsg) < 0)
-            oom ();
-    }
-}
-
-int main (int argc, char *argv[])
-{
-    plan (27);
-
-    test_libev_timer (); // 5
-    test_libev_io (); // 3
-    test_zmq_events (); // 13
-    test_ev_zmq (); // 6
-
-    done_testing ();
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libutil/test/fdutils.c b/src/common/libutil/test/fdutils.c
index 73161fe9809c..963bdc32edd8 100644
--- a/src/common/libutil/test/fdutils.c
+++ b/src/common/libutil/test/fdutils.c
@@ -8,10 +8,14 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <unistd.h>
 #include <errno.h>
 #include <string.h>
 #include <fcntl.h>
+
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/fdutils.h"
 
diff --git a/src/common/libutil/test/fdwalk.c b/src/common/libutil/test/fdwalk.c
index eaf3e8982a62..88f23da833e1 100644
--- a/src/common/libutil/test/fdwalk.c
+++ b/src/common/libutil/test/fdwalk.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <unistd.h>
 #include <errno.h>
 #include <string.h>
@@ -43,6 +46,14 @@ static void set_fd (void *data, int fd)
     fds[fd]++;
 }
 
+static void set_fd_if_open (void *data, int fd)
+{
+    if (fcntl (fd, F_GETFL) < 0 && errno == EBADF)
+        return;
+    int *fds = data;
+    fds[fd]++;
+}
+
 static int * get_open_fds (int maxfd)
 {
     /* Valgrind may show open fds > maxfd, so double the space
@@ -50,7 +61,7 @@ static int * get_open_fds (int maxfd)
      */
     int * fds = calloc (maxfd * 2, sizeof (int));
     if (fds)
-        ok (fdwalk (set_fd, fds) == 0,
+        ok (fdwalk (set_fd_if_open, fds) == 0,
             "fdwalk () worked");
     return fds;
 }
diff --git a/src/common/libutil/test/fluid.c b/src/common/libutil/test/fluid.c
index e18774989d88..cfcb13ffb5d8 100644
--- a/src/common/libutil/test/fluid.c
+++ b/src/common/libutil/test/fluid.c
@@ -8,10 +8,232 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <string.h>
+#include <locale.h>
+
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/fluid.h"
+#include "ccan/str/str.h"
 
-int main (int argc, char *argv[])
+struct f58_test {
+    fluid_t id;
+    const char *f58;
+    const char *f58_alt;
+};
+
+struct f58_test f58_tests [] = {
+#if !ASSUME_BROKEN_LOCALE
+    { 0, "ƒ1", "f1" },
+    { 1, "ƒ2", "f2" },
+    { 57, "ƒz", "fz" },
+    { 1234, "ƒNH", "fNH" },
+    { 1888, "ƒZZ", "fZZ" },
+    { 3363, "ƒzz", "fzz" },
+    { 3364, "ƒ211", "f211" },
+    { 4369, "ƒ2JL", "f2JL" },
+    { 65535, "ƒLUv", "fLUv" },
+    { 4294967295, "ƒ7YXq9G", "f7YXq9G" },
+    { 633528662, "ƒxyzzy", "fxyzzy" },
+    { 6731191091817518LL, "ƒuZZybuNNy", "fuZZybuNNy" },
+    { 18446744073709551614UL, "ƒjpXCZedGfVP", "fjpXCZedGfVP" },
+    { 18446744073709551615UL, "ƒjpXCZedGfVQ", "fjpXCZedGfVQ" },
+#endif
+    { 0, NULL, NULL },
+};
+
+struct f58_test f58_alt_tests [] = {
+    { 0, "f1", NULL },
+    { 0, "f111", NULL },
+    { 1, "f2", NULL },
+    { 57, "fz", NULL },
+    { 1234, "fNH", NULL },
+    { 1888, "fZZ", NULL },
+    { 3363, "fzz", NULL },
+    { 3364, "f211", NULL },
+    { 4369, "f2JL", NULL },
+    { 65535, "fLUv", NULL },
+    { 4294967295, "f7YXq9G", NULL },
+    { 633528662, "fxyzzy", NULL },
+    { 6731191091817518LL, "fuZZybuNNy", NULL },
+    { 18446744073709551614UL, "fjpXCZedGfVP", NULL },
+    { 18446744073709551615UL, "fjpXCZedGfVQ", NULL },
+    { 0, NULL, NULL },
+};
+
+void test_f58 (void)
+{
+    fluid_string_type_t type = FLUID_STRING_F58;
+    char buf[16];
+    fluid_t id;
+    struct f58_test *tp = f58_tests;
+    while (tp->f58 != NULL) {
+        ok (fluid_encode (buf, sizeof(buf), tp->id, type) == 0,
+            "f58_encode (%ju)", tp->id);
+        if (streq (buf, tp->f58)
+            || streq (buf, tp->f58_alt))
+            pass ("f58_encode %ju -> %s", tp->id, buf);
+        else
+            fail ("f58_encode %ju: got %s expected %s",
+                  tp->id, buf, tp->f58);
+        ok (fluid_decode (tp->f58, &id, type) == 0,
+            "f58_decode (%s)", tp->f58);
+        ok (id == tp->id,
+            "%s -> %ju", tp->f58, (uintmax_t) id);
+        tp++;
+    }
+    tp = f58_alt_tests;
+    while (tp->f58 != NULL) {
+        ok (fluid_decode (tp->f58, &id, type) == 0,
+            "f58_decode (%s)", tp->f58);
+        ok (id == tp->id,
+            "%s -> %ju", tp->f58, (uintmax_t) id);
+        tp++;
+    }
+
+#if !ASSUME_BROKEN_LOCALE
+    if (setenv ("FLUX_F58_FORCE_ASCII", "1", 1) < 0)
+        BAIL_OUT ("Failed to setenv FLUX_F58_FORCE_ASCII");
+    ok (fluid_encode (buf, sizeof (buf), f58_tests->id, type) == 0,
+        "fluid_encode with FLUX_F58_FORCE_ASCII works");
+    is (buf, f58_tests->f58_alt,
+        "fluid_encode with FLUX_F58_FORCE_ASCII used ascii prefix");
+    if (unsetenv ("FLUX_F58_FORCE_ASCII") < 0)
+        BAIL_OUT ("Failed to unsetenv FLUX_F58_FORCE_ASCII");
+
+    ok (fluid_encode (buf,
+                      sizeof (buf),
+                      f58_tests->id,
+                      FLUID_STRING_F58_PLAIN) == 0,
+        "fluid_encode FLUX_STRING_F58_PLAIN works");
+    is (buf, f58_tests->f58_alt,
+        "fluid_encode FLUID_STRING_F58_PLAIN used ascii prefix");
+#endif
+
+    ok (fluid_encode (buf, 1, 1, type) < 0 && errno == EOVERFLOW,
+        "fluid_encode (buf, 1, 1, F58) returns EOVERFLOW");
+    errno = 0;
+    ok (fluid_encode (buf, 4, 65535, type) < 0 && errno == EOVERFLOW,
+        "fluid_encode (buf, 4, 65535, F58) returns EOVERFLOW: %s", strerror (errno));
+
+    ok (fluid_decode ("1234", &id, type) < 0 && errno == EINVAL,
+        "fluid_decode ('aaa', FLUID_STRING_F58) returns EINVAL");
+    ok (fluid_decode ("aaa", &id, type) < 0 && errno == EINVAL,
+        "fluid_decode ('aaa', FLUID_STRING_F58) returns EINVAL");
+    ok (fluid_decode ("f", &id, type) < 0 && errno == EINVAL,
+        "fluid_decode ('f', FLUID_STRING_F58) returns EINVAL");
+    ok (fluid_decode ("flux", &id, type) < 0 && errno == EINVAL,
+        "fluid_decode ('flux', FLUID_STRING_F58) returns EINVAL");
+    ok (fluid_decode ("f1230", &id, type) < 0 && errno == EINVAL,
+        "fluid_decode ('f1230', FLUID_STRING_F58) returns EINVAL");
+    ok (fluid_decode ("x1", &id, type) < 0 && errno == EINVAL,
+        "fluid_decode ('x1', FLUID_STRING_F58) returns EINVAL");
+}
+
+struct fluid_parse_test {
+    fluid_t id;
+    const char *input;
+};
+
+struct fluid_parse_test fluid_parse_tests [] = {
+#if !ASSUME_BROEN_LOCALE
+    { 0, "ƒ1" },
+    { 1, "ƒ2" },
+    { 57, "ƒz" },
+    { 1234, "ƒNH" },
+    { 1888, "ƒZZ" },
+    { 3363, "ƒzz" },
+    { 3364, "ƒ211" },
+    { 4369, "ƒ2JL" },
+    { 65535, "ƒLUv" },
+    { 4294967295, "ƒ7YXq9G" },
+    { 633528662, "ƒxyzzy" },
+    { 6731191091817518LL, "ƒuZZybuNNy" },
+    { 18446744073709551614UL, "ƒjpXCZedGfVP" },
+    { 18446744073709551615UL, "ƒjpXCZedGfVQ" },
+#endif
+    { 0, "f1" },
+    { 1, "f2" },
+    { 4294967295, "f7YXq9G" },
+    { 633528662, "fxyzzy" },
+    { 18446744073709551614UL, "fjpXCZedGfVP" },
+    { 18446744073709551615UL, "fjpXCZedGfVQ" },
+    { 1234, "1234" },
+    { 1888, "1888" },
+    { 3363, "3363" },
+    { 3364, "3364" },
+    { 4369, "4369" },
+    { 6731191091817518LL, "6731191091817518" },
+    { 18446744073709551614UL, "18446744073709551614" },
+    { 18446744073709551615UL, "18446744073709551615" },
+    { 0, "0x0" },
+    { 1, "0x1" },
+    { 57, "0x39" },
+    { 1234, "0x4d2" },
+    { 1888, "0x760" },
+    { 3363, "0xd23" },
+    { 4369, "0x1111" },
+    { 65535, "0xffff" },
+    { 4294967295, "0xffffffff" },
+    { 633528662,  "0x25c2e156" },
+    { 6731191091817518LL, "0x17e9fb8df16c2e" },
+    { 18446744073709551615UL, "0xffffffffffffffff" },
+    { 0, "0.0.0.0" },
+    { 1, "0000.0000.0000.0001" },
+    { 57, "0.0.0.0039" },
+    { 1234, "0000.0000.0000.04d2" },
+    { 1888, "0000.0000.0000.0760" },
+    { 4369, "0000.0000.0000.1111" },
+    { 65535, "0.0.0.ffff" },
+    { 4294967295, "0000.0000.ffff.ffff" },
+    { 18446744073709551615UL, "ffff.ffff.ffff.ffff" },
+    { 0, "😃" },
+    { 1, "😄" },
+    { 57, "🙊" },
+    { 1234, "😁👌" },
+    { 1888, "😆🐻" },
+    { 4369, "😊🌀" },
+    { 65535, "💁📚" },
+    { 4294967295, "😳🏪🍖🍸" },
+    { 18446744073709551615UL, "🚹💗💧👗😷📷📚" },
+    { 0, NULL },
+};
+
+static void test_fluid_parse (void)
+{
+    fluid_t id;
+    struct fluid_parse_test *tp = fluid_parse_tests;
+    while (tp->input != NULL) {
+        id = 0;
+        ok (fluid_parse (tp->input, &id) == 0,
+            "fluid_parse (%s) works", tp->input);
+        ok (id == tp->id,
+            "%s -> %ju", tp->input, (uintmax_t) id);
+        tp++;
+    }
+
+    ok (fluid_parse (" 0xffff   ", &id) == 0,
+        "flux_parse() works with leading/trailing whitespace");
+    ok (id == 65535,
+        "flux_parse with whitespace works");
+
+    id = 0;
+    ok (fluid_parse (NULL, &id) < 0 && errno == EINVAL,
+        "fluid_parse returns EINVAL for with NULL string");
+    ok (fluid_parse ("", &id) < 0 && errno == EINVAL,
+        "fluid_parse returns EINVAL for with empty string");
+    ok (fluid_parse ("boo", &id) < 0 && errno == EINVAL,
+        "fluid_parse returns EINVAL for 'boo'");
+    ok (fluid_parse ("f", &id) < 0 && errno == EINVAL,
+        "fluid_parse returns EINVAL for 'f'");
+    ok (fluid_parse ("-1", &id) < 0 && errno == EINVAL,
+        "fluid_parse returns EINVAL for '-1'");
+}
+
+void test_basic (void)
 {
     struct fluid_generator gen;
     fluid_t id, id2;
@@ -21,10 +243,8 @@ int main (int argc, char *argv[])
     int encode_errors;
     int decode_errors;
 
-    plan (NO_PLAN);
-
-    ok (fluid_init (&gen, 0) == 0,
-        "fluid_init id=0 works");
+    ok (fluid_init (&gen, 0, 0) == 0,
+        "fluid_init id=0 timestamp=0 works");
 
     /* Probably all zeroes, or (unlikely) with slightly advanced timestamp.
      */
@@ -33,7 +253,7 @@ int main (int argc, char *argv[])
     ok (fluid_encode (buf, sizeof (buf), id, FLUID_STRING_DOTHEX) == 0,
         "fluid_encode type=DOTHEX works");
     ok (fluid_decode (buf, &id2, FLUID_STRING_DOTHEX) == 0 && id == id2,
-        "fluid_decode type=MNEMONIC works");
+        "fluid_decode type=DOTHEX works");
     diag ("%s", buf);
 
     ok (fluid_encode (buf, sizeof (buf), id, FLUID_STRING_MNEMONIC) == 0,
@@ -42,17 +262,25 @@ int main (int argc, char *argv[])
         "fluid_decode type=MNEMONIC works");
     diag ("%s", buf);
 
+    ok (fluid_encode (buf, sizeof (buf), id, FLUID_STRING_EMOJI) == 0,
+        "fluid_encode type=EMOJI works");
+    ok (fluid_decode (buf, &id2, FLUID_STRING_EMOJI) == 0 && id == id2,
+        "fluid_decode type=EMOJI works");
+    diag ("%s", buf);
+
     /* With artificially tweaked generator state
      */
+    const uint64_t time_34y = 1000ULL*60*60*24*365*34;
+    ok (fluid_init (&gen, 0, time_34y) == 0,
+        "fluid_init id=0 timestamp=34y works");
     gen.id = 16383;
-    gen.epoch -= 1000ULL*60*60*24*365*34; // 34 years
     gen.seq = 1023;
     ok (fluid_generate (&gen, &id) == 0,
         "fluid_generate works 34 years in the future");
     ok (fluid_encode (buf, sizeof (buf), id, FLUID_STRING_DOTHEX) == 0,
         "fluid_encode type=DOTHEX works");
     ok (fluid_decode (buf, &id2, FLUID_STRING_DOTHEX) == 0 && id == id2,
-        "fluid_decode type=MNEMONIC works");
+        "fluid_decode type=DOTHEX works");
     diag ("%s", buf);
 
     ok (fluid_encode (buf, sizeof (buf), id, FLUID_STRING_MNEMONIC) == 0,
@@ -61,8 +289,15 @@ int main (int argc, char *argv[])
         "fluid_decode type=MNEMONIC works");
     diag ("%s", buf);
 
+    ok (fluid_encode (buf, sizeof (buf), id, FLUID_STRING_EMOJI) == 0,
+        "fluid_encode type=EMOJI works");
+    ok (fluid_decode (buf, &id2, FLUID_STRING_EMOJI) == 0 && id == id2,
+        "fluid_decode type=EMOJI works");
+    diag ("%s", buf);
+
+
     /* Generate 64K id's as rapidly as possible.
-     * Probably will induce usleep and take around 64 milliseconds.
+     * Probably will cover running out of seq bits.
      */
     generate_errors = 0;
     encode_errors = 0;
@@ -82,8 +317,7 @@ int main (int argc, char *argv[])
     ok (decode_errors == 0,
         "fluid_decode type=DOTHEX worked 64K times");
 
-    /* Continue for another 4K with NMEMONIC encoding, which is slower
-     * and probably won't induce usleep.
+    /* Continue for another 4K with NMEMONIC encoding (slower).
      */
     generate_errors = 0;
     encode_errors = 0;
@@ -103,6 +337,51 @@ int main (int argc, char *argv[])
     ok (decode_errors == 0,
         "fluid_decode type=MNEMONIC worked 4K times");
 
+    /* Continue for another 4K with EMOJI encoding (slower).
+     */
+    generate_errors = 0;
+    encode_errors = 0;
+    decode_errors = 0;
+    for (i = 0; i < 4096; i++) {
+        if (fluid_generate (&gen, &id) < 0)
+            generate_errors++;
+        if (fluid_encode (buf, sizeof (buf), id, FLUID_STRING_EMOJI) < 0)
+            encode_errors++;
+        if (fluid_decode (buf, &id2, FLUID_STRING_EMOJI) < 0 || id != id2)
+            decode_errors++;
+    }
+    ok (generate_errors == 0,
+        "fluid_generate worked 4K times");
+    ok (encode_errors == 0,
+        "fluid_encode type=EMOJI worked 4K times");
+    ok (decode_errors == 0,
+        "fluid_decode type=EMOJI worked 4K times");
+
+
+    /* Generate 64K FLUIDs, restarting generator each time from timestamp
+     * extracted from generated FLUID + 1.  Verify number always increases.
+     */
+    fluid_t lastid = 0;
+    uint64_t ts;
+    int errors = 0;
+    for (i = 0; i < 65536; i++) {
+        if (fluid_generate (&gen, &id) < 0)
+            BAIL_OUT ("fluid_generate unexpectedly failed");
+        if (lastid >= id)
+            errors++;
+        lastid = id;
+        ts = fluid_get_timestamp (id);
+        if (fluid_init (&gen, 0, ts + 1) < 0)
+            BAIL_OUT ("fluid_init unexpectedly failed");
+    }
+    ok (errors == 0,
+        "restarted generator 64K times without going backwards");
+
+    /* Get timestsamp with fluid_save()
+     */
+    ok (fluid_save_timestamp (&gen, &ts) == 0 && ts == gen.timestamp,
+        "fluid_save_timestamp worked");
+
     /* Decode bad input must fail
      */
     ok (fluid_decode ("bogus", &id, FLUID_STRING_DOTHEX) < 0,
@@ -111,6 +390,20 @@ int main (int argc, char *argv[])
         "fluid_decode type=MNEMONIC fails on input=bogus");
     ok (fluid_decode ("a-a-a--a-a-a", &id, FLUID_STRING_MNEMONIC) < 0,
         "fluid_decode type=MNEMONIC fails on unknown words xx-xx-xx--xx-xx-xx");
+    ok (fluid_decode ("bogus", &id, FLUID_STRING_EMOJI) < 0,
+        "fluid_decode type=EMOJI fails on ascii string");
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    /* Locale initialization required for fluid_f58_encode() */
+    setlocale (LC_ALL, "");
+
+    test_basic ();
+    test_f58 ();
+    test_fluid_parse ();
 
     done_testing ();
     return 0;
diff --git a/src/common/libutil/test/fsd.c b/src/common/libutil/test/fsd.c
index bd64f29d92d8..59382100d5fa 100644
--- a/src/common/libutil/test/fsd.c
+++ b/src/common/libutil/test/fsd.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <math.h>
 #include <string.h>
 #include <errno.h>
@@ -15,6 +18,40 @@
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/fsd.h"
 
+struct test_vector {
+    const char *input;
+    double result;
+};
+
+struct test_vector rfc23_tests[] = {
+    { "2ms",          0.002    },
+    { "0.1s",         0.1      },
+    { "30",           30.      },
+    { "1.2h",         4320.    },
+    { "5m",           300.     },
+    { "0s",           0.       },
+    { "5d",           432000.  },
+    { "inf",          INFINITY },
+    { "INF",          INFINITY },
+    { "infinity",     INFINITY },
+    { NULL,           0.       },
+};
+
+static void run_rfc23_tests ()
+{
+    struct test_vector *tp = rfc23_tests;
+    while (tp->input != NULL) {
+        double d;
+        ok (fsd_parse_duration (tp->input, &d) == 0,
+            "rfc23: fsd_parse_duration (%s)",
+            tp->input);
+        ok (d == tp->result,
+            "rfc23: result is %.3f",
+            d);
+        tp++;
+    }
+}
+
 int main(int argc, char** argv)
 {
     double d;
@@ -37,11 +74,22 @@ int main(int argc, char** argv)
         "fsd_parse_duration (NaNs) is an error");
     ok (fsd_parse_duration ("infinites", &d) < 0 && errno == EINVAL,
         "fsd_parse_duration (infinites) is an error");
+    ok (fsd_parse_duration ("infd", &d) < 0 && errno == EINVAL,
+        "fsd_parse_duration (infd) is an error");
+
+    ok (fsd_parse_duration ("infinity", &d) == 0,
+        "fsd_parse_duration (\"infinity\") returns success");
+    ok (isinf (d),
+        "isinf (result) is true");
 
     ok (fsd_parse_duration ("0", &d) == 0,
         "fsd_parse_duration (0) returns success");
     ok (d == 0., "got d == %g", d);
 
+    ok (fsd_parse_duration ("0ms", &d) == 0,
+        "fsd_parse_duration (0ms) returns success");
+    ok (d == 0., "got d == %g", d);
+
     ok (fsd_parse_duration ("0s", &d) == 0,
         "fsd_parse_duration (0s) returns success");
     ok (d == 0., "got d == %g", d);
@@ -58,6 +106,14 @@ int main(int argc, char** argv)
         "fsd_parse_duration (0d) returns success");
     ok (d == 0., "got d == %g", d);
 
+    ok (fsd_parse_duration ("500ms", &d) == 0,
+        "fsd_parse_duration (500ms) returns success");
+    ok (d == 0.5, "got d == %g", d);
+
+    ok (fsd_parse_duration ("0.2ms", &d) == 0,
+        "fsd_parse_duration (0.2ms) returns success");
+    ok (d == 0.0002, "got d == %g", d);
+
     ok (fsd_parse_duration ("0.5", &d) == 0,
         "fsd_parse_duration (0.5) returns success");
     ok (d == 0.5, "got d == %g", d);
@@ -69,7 +125,7 @@ int main(int argc, char** argv)
     ok (fsd_parse_duration ("0.5m", &d) == 0,
         "fsd_parse_duration (0.5m) returns success");
     ok (d == 30., "got d == %g", d);
-    
+
     ok (fsd_parse_duration ("0.5h", &d) == 0,
         "fsd_parse_duration (0.5h) returns success");
     ok (d == .5 * 60. * 60., "got d == %g", d);
@@ -81,10 +137,7 @@ int main(int argc, char** argv)
     ok (fsd_format_duration (buf, sizeof (buf), NAN) < 0 && errno == EINVAL,
         "fsd_format_duration with NaN duration. returns EINVAL");
 
-    ok (fsd_format_duration (buf, sizeof (buf), INFINITY) < 0 && errno == EINVAL,
-        "fsd_format_duration with INF duration. returns EINVAL");
-
-    ok (fsd_format_duration (buf, sizeof (buf), -1.) < 0 && errno == EINVAL,
+   ok (fsd_format_duration (buf, sizeof (buf), -1.) < 0 && errno == EINVAL,
         "fsd_format_duration with duration < 0. returns EINVAL");
 
     ok (fsd_format_duration (buf, 0, 1.) < 0 && errno == EINVAL,
@@ -92,10 +145,18 @@ int main(int argc, char** argv)
 
     ok (fsd_format_duration (NULL, 1024, 1000.) < 0 && errno == EINVAL,
         "fsd_format_duration with buf == NULL returns EINVAL");
-    
+
+    ok (fsd_format_duration (buf, sizeof (buf), INFINITY) > 0,
+        "fsd_format_duration (INFINITY) works");
+    is (buf, "infinity", "returns expected string = %s", buf);
+
     ok (fsd_format_duration (buf, sizeof (buf), .001),
         "fsd_format_duration (.001) works");
-    is (buf, "0.001s", "returns expected string = %s", buf);
+    is (buf, "1ms", "returns expected string = %s", buf);
+
+    ok (fsd_format_duration (buf, sizeof (buf), 0.01),
+        "fsd_format_duration (0.5) works");
+    is (buf, "10ms", "returns expected string = %s", buf);
 
     ok (fsd_format_duration (buf, sizeof (buf), 5.),
         "fsd_format_duration (5.0) works");
@@ -125,6 +186,12 @@ int main(int argc, char** argv)
         "fsd_format_duration (86400.) works");
     is (buf, "1.2d", "returns expected string = %s", buf);
 
+    ok (fsd_format_duration_ex (buf, sizeof (buf), 62.0, 1),
+        "fsd_format_duration_ex (62., 1) works");
+    is (buf, "1m", "returns expected string = %s", buf);
+
+    run_rfc23_tests ();
+
     done_testing();
 }
 
diff --git a/src/common/libutil/test/getaddr.c b/src/common/libutil/test/getaddr.c
new file mode 100644
index 000000000000..a307dff43ff0
--- /dev/null
+++ b/src/common/libutil/test/getaddr.c
@@ -0,0 +1,45 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <limits.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/ipaddr.h"
+#include "src/common/libutil/log.h"
+
+
+int main(int argc, char** argv)
+{
+    char buf[_POSIX_HOST_NAME_MAX + 1];
+    flux_error_t error;
+    char *name = getenv ("FLUX_IPADDR_INTERFACE");
+    int flags = 0;
+
+    log_init ("getaddr");
+
+    if (argc > 2)
+        log_msg_exit ("too many arguments");
+    if (argc == 2)
+        name = argv[1];
+    if (getenv ("FLUX_IPADDR_HOSTNAME"))
+        flags |= IPADDR_HOSTNAME;
+    if (getenv ("FLUX_IPADDR_V6"))
+        flags |= IPADDR_V6;
+
+    if (ipaddr_getprimary (buf, sizeof (buf), flags, name, &error) < 0)
+        log_msg_exit ("%s", error.text);
+
+    printf ("%s\n", buf);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/test/grudgeset.c b/src/common/libutil/test/grudgeset.c
new file mode 100644
index 000000000000..804d80adc613
--- /dev/null
+++ b/src/common/libutil/test/grudgeset.c
@@ -0,0 +1,100 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <string.h>
+#include <errno.h>
+#include <jansson.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/grudgeset.h"
+
+int main (int argc, char *argv[])
+{
+    struct grudgeset *gs = NULL;
+
+    plan (NO_PLAN);
+
+    ok (grudgeset_add (NULL, NULL) < 0 && errno == EINVAL,
+        "grudgeset_add (NULL, NULL) returns EINVAL");
+    ok (grudgeset_add (&gs, NULL) < 0 && errno == EINVAL,
+        "grudgeset_add (&o, NULL) returns EINVAL");
+    ok (grudgeset_remove (NULL, NULL) < 0 && errno == EINVAL,
+        "grudgeset_remove (NULL, NULL) returns EINVAL");
+
+    ok (grudgeset_remove (NULL, "foo") < 0 && errno == ENOENT,
+        "grudgeset_remove(NULL, \"foo\") returns ENOENT");
+    ok (grudgeset_size (NULL) == 0,
+        "grudgeset_size (NULL) == 0");
+
+    ok (grudgeset_used (NULL, "foo") == 0,
+        "grudgeset_used (NULL, \"foo\") returns 0");
+    ok (grudgeset_contains (NULL, NULL) == 0,
+        "grudgeset_contains (NULL, NULL) returns 0");
+    ok (grudgeset_contains (NULL, "foo") == 0,
+        "grudgeset_contains (NULL, \"foo\") returns 0");
+    ok (grudgeset_tojson (NULL) == NULL,
+        "grudgeset_tojson (NULL) returns NULL");
+
+    ok (grudgeset_add (&gs, "foo") == 0,
+        "grudgeset_add works with NULL object");
+    ok (gs != NULL,
+        "grudgeset is now non-NULL");
+    ok (grudgeset_size (gs) == 1,
+        "set is of size 1");
+    ok (grudgeset_contains (gs, "foo") == 1,
+        "grudgeset_contains (foo) works");
+
+    ok (grudgeset_add (&gs, "foo") < 0 && errno == EEXIST,
+        "grudgeset_add of existing value returns EEXIST");
+
+    ok (grudgeset_add (&gs, "baz") == 0,
+        "grudgeset_add of a second value works");
+    ok (grudgeset_size (gs) == 2,
+        "grudgeset is of size 2");
+    ok (grudgeset_contains (gs, "baz") == 1,
+        "grudgeset_contains (baz) works");
+
+    ok (grudgeset_tojson (gs) != NULL,
+        "grudgeset_tojson() is non-NULL");
+
+    ok (grudgeset_remove (gs, "xxyyzz") < 0 && errno == ENOENT,
+        "grudgeset_remove of nonexistent entry returns ENOENT");
+
+    ok (grudgeset_remove (gs, "foo") == 0,
+        "grudgeset_remove of first item in set works");
+    ok (grudgeset_size (gs) == 1,
+        "set is of size 1");
+    ok (grudgeset_contains (gs, "foo") == 0,
+        "grudgeset no longer contains foo");
+    ok (grudgeset_used (gs, "foo") == 1,
+        "but grudgeset has marked foo as \"used\"");
+
+    ok (grudgeset_add (&gs, "foo") < 0 && errno == EEXIST,
+        "grudgeset_add of removed value still fails");
+
+    ok (grudgeset_remove (gs, "baz") == 0,
+        "grudgeset_remove of second element works");
+    ok (grudgeset_size (gs) == 0,
+        "grudgeset_size is now 0");
+    ok (grudgeset_used (gs, "baz"),
+        "grudgeset_used (baz) == 1");
+
+    grudgeset_destroy (gs);
+
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/test/hola.c b/src/common/libutil/test/hola.c
new file mode 100644
index 000000000000..5111acbc6516
--- /dev/null
+++ b/src/common/libutil/test/hola.c
@@ -0,0 +1,436 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "src/common/libutil/hola.h"
+
+void key_destructor (void **item)
+{
+    if (item) {
+        free (*item);
+        *item = NULL;
+    }
+}
+void *key_duplicator (const void *item)
+{
+    return strdup (item);
+}
+int key_comparator (const void *item1, const void *item2)
+{
+    return strcmp (item1, item2);
+}
+size_t
+key_hasher (const void *key)
+{
+    const char *cp = key;
+    size_t hash = 0;
+    while (*cp)
+        hash = 33 * hash ^ *cp++;
+    return hash;
+}
+
+void test_hash (void)
+{
+    struct hola *h;
+    const char *key;
+    zlistx_t *l;
+
+    ok ((h = hola_create (0)) != NULL,
+        "hola_create works");
+
+    /* set callbacks identical to internal zhashx defaults
+     * just to cover functions
+     */
+    hola_set_hash_key_destructor (h, key_destructor);
+    hola_set_hash_key_duplicator (h, key_duplicator);
+    hola_set_hash_key_comparator (h, key_comparator);
+    hola_set_hash_key_hasher (h, key_hasher);
+
+    /* empty hash */
+    ok (hola_hash_size (h) == 0,
+        "hola_hash_size is 0");
+    key = hola_hash_first (h);
+    ok (key == NULL,
+        "hola_hash_first returns NULL");
+    key = hola_hash_next (h);
+    ok (key == NULL,
+        "hola_hash_next returns NULL");
+    errno = 0;
+    l = hola_hash_lookup (h, "foo");
+    ok (l == NULL && errno == ENOENT,
+        "hola_hash_lookup key=foo fails with ENOENT");
+    errno = 0;
+    ok (hola_hash_delete (h, "foo") < 0 && errno == ENOENT,
+        "hola_hash_delete key=foo fails with ENOENT");
+
+    /* one item */
+    ok (hola_hash_add (h, "item1") == 0,
+        "hola_hash_add key=item1 works");
+    ok (hola_hash_size (h) == 1,
+        "hola_hash_size is 1");
+    errno = 0;
+    ok (hola_hash_add (h, "item1") < 0
+        && errno == EEXIST,
+        "hola_hash_add key=item1 fails with EEXIST");
+    key = hola_hash_first (h);
+    ok (key && streq (key, "item1"),
+        "hola_hash_first returns item1");
+    key = hola_hash_next (h);
+    ok (key == NULL,
+        "hola_hash_next returns NULL");
+    l = hola_hash_lookup (h, "item1");
+    ok (l != NULL,
+        "hola_hash_lookup key=item1 works");
+    ok (hola_hash_delete (h, "item1") == 0
+        && hola_hash_size (h) == 0,
+        "hola_hash_delete key=item1 works");
+
+    /* two items */
+    ok (hola_hash_add (h, "item1") == 0,
+        "hola_hash_add key=item1 works");
+    ok (hola_hash_add (h, "item2") == 0,
+        "hola_hash_add key=item2 works");
+    ok (hola_hash_size (h) == 2,
+        "hola_hash_size is 2");
+    key = hola_hash_first (h);
+    ok (key && (streq (key, "item1") || streq (key, "item2")),
+        "hola_hash_first returns a valid key");
+    key = hola_hash_next (h);
+    ok (key && (streq (key, "item1") || streq (key, "item2")),
+        "hola_hash_next returns a valid key");
+    key = hola_hash_next (h);
+    ok (key == NULL,
+        "hola_hash_next returns NULL");
+    l = hola_hash_lookup (h, "item1");
+    ok (l != NULL,
+        "hola_hash_lookup key=item1 works");
+    l = hola_hash_lookup (h, "item2");
+    ok (l != NULL,
+        "hola_hash_lookup key=item2 works");
+    ok (hola_hash_delete (h, "item1") == 0
+        && hola_hash_size (h) == 1,
+        "hola_hash_delete key=item1 works");
+    key = hola_hash_first (h);
+    ok (key && streq (key, "item2"),
+        "hola_hash_first returns item2");
+    key = hola_hash_next (h);
+    ok (key == NULL,
+        "hola_hash_next returns NULL");
+
+    hola_destroy (h);
+}
+
+int item_comparator (const void *item1, const void *item2)
+{
+    return strcmp (item1, item2);
+}
+
+void test_auto (void)
+{
+    struct hola *h;
+    void *item1;
+    void *item2;
+    void *item3;
+    void *handle;
+
+    ok ((h = hola_create (HOLA_AUTOCREATE | HOLA_AUTODESTROY)) != NULL,
+        "hola_create AUTOCREATE | AUTODDESTROY works");
+    hola_set_list_comparator (h, item_comparator);
+
+    item1 = hola_list_add_end (h, "blue", "item1");
+    ok (item1 != NULL,
+        "hola_add_end key=blue value=item1 works");
+    item2 = hola_list_add_end (h, "red", "item2");
+    ok (item2 != NULL,
+        "hola_add_end key=red value=item2 works");
+    item3 = hola_list_add_end (h, "red", "item3");
+    ok (item3 != NULL,
+        "hola_add_end key=red value=item3 works");
+    ok (hola_hash_size (h) == 2,
+        "hola_hash_size is 2");
+    ok (hola_list_size (h, "blue") == 1,
+        "hola_list_size is key=blue is 1");
+    ok (hola_list_size (h, "red") == 2,
+        "hola_list_size is key=red is 2");
+    ok (hola_list_delete (h, "red", item3) == 0
+        && hola_list_size (h, "red") == 1,
+        "hola_list_delete key=red item3 works");
+    ok (hola_list_delete (h, "red", item2) == 0
+        && hola_list_size (h, "red") == 0,
+        "hola_list_delete key=red item3 works");
+    ok (hola_hash_size (h) == 1,
+        "hola_hash_size is 1");
+    ok (hola_list_delete (h, "blue", item1) == 0
+        && hola_list_size (h, "blue") == 0,
+        "hola_list_delete key=blue item1 works");
+    ok (hola_hash_size (h) == 0,
+        "hola_hash_size is 0");
+    item1 = "item1";
+    errno = 0;
+    ok (hola_list_find (h, "blue", item1) == NULL && errno == ENOENT,
+        "hola_list_find fails with ENOENT on missing item");
+    ok ((handle = hola_list_insert (h, "blue", item1, false)) != NULL,
+        "hola_list_insert key=blue value=item1 works");
+    ok (hola_list_find (h, "blue", item1) == handle,
+        "hola_list_find finds it again");
+    ok (hola_list_insert (h, "blue", "item0", false) != NULL,
+        "hola_list_insert key=blue value=item0 works");
+    ok (hola_list_insert (h, "blue", "item2", false) != NULL,
+        "hola_list_insert key=blue value=item2 works");
+    ok (hola_list_size (h, "blue") == 3,
+        "hola_list_size key=blue is 3");
+    const char *val;
+    ok ((val = hola_list_first (h, "blue")) != NULL
+        && streq (val, "item0"),
+        "first item is item0");
+    ok ((val = hola_list_next (h, "blue")) != NULL
+        && streq (val, "item1"),
+        "first item is item1");
+    ok ((val = hola_list_next (h, "blue")) != NULL
+        && streq (val, "item2"),
+        "first item is item2");
+
+    hola_destroy (h);
+}
+
+struct test_input {
+    const char *key;
+    char *val;
+};
+
+struct test_input test1[] = {
+    { "blue", "item1" },
+    { "blue", "item2" },
+    { "blue", "item3" },
+    { "red", "item4" },
+    { "red", "item5" },
+    { "green", "item6" },
+};
+
+int find_entry (struct test_input *t,
+                size_t len,
+                const char *key,
+                const char *val)
+{
+    if (key && val) {
+        for (int i = 0; i < len; i++) {
+            if (streq (key, t[i].key) && streq (val, t[i].val))
+                return i;
+        }
+    }
+    return -1;
+}
+
+bool test_iter_one (struct test_input *t, size_t len)
+{
+    struct hola *h;
+    const char *key;
+    const char *val;
+    bool *checklist;
+    bool result = true;
+
+    if (!(checklist = calloc (len, sizeof (checklist[0]))))
+        BAIL_OUT ("out of memory");
+    h = hola_create (HOLA_AUTOCREATE);
+    if (!h)
+        BAIL_OUT ("hola_create failed");
+    for (int i = 0; i < len; i++) {
+        if (!hola_list_add_end (h, t[i].key, t[i].val))
+            BAIL_OUT ("could not populate test object for iteration");
+    }
+    key = hola_hash_first (h);
+    while (key) {
+        size_t count = 0;
+        val = hola_list_first (h, key);
+        while (val) {
+            int index = find_entry (t, len, key, val);
+            if (hola_list_cursor (h, key) == NULL) // cursor must not be NULL
+                break;
+            if (index != -1)
+                checklist[index] = true;
+            count++;
+            val = hola_list_next (h, key);
+        }
+        if (hola_list_cursor (h, key) != NULL) // cursor must be NULL
+            break;
+        /* iterate backwards for fun */
+        val = hola_list_last (h, key);
+        while (val) {
+            count--;
+            val = hola_list_prev (h, key);
+        }
+        if (count > 0) {
+            diag ("reverse iteration failed");
+            result = false;
+        }
+
+        key = hola_hash_next (h);
+    }
+    for (int i = 0; i < len; i++)
+        if (!checklist[i])
+            result = false;
+    hola_destroy (h);
+    free (checklist);
+    return result;
+}
+
+void test_iter (void)
+{
+    ok (test_iter_one (test1, ARRAY_SIZE (test1)) == true,
+        "iteration works");
+}
+
+void test_inval (void)
+{
+    struct hola *h;
+    if (!(h = hola_create (0)))
+        BAIL_OUT ("hola_create failed");
+
+    errno = 0;
+    ok (hola_create (0xff) == NULL && errno == EINVAL,
+        "hola_create flags=0xff fails with EINVAL");
+
+    errno = 0;
+    ok (hola_hash_lookup (NULL, "foo") == NULL && errno == EINVAL,
+        "hola_hash_lookup h=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (hola_hash_lookup (h, NULL) == NULL && errno == EINVAL,
+        "hola_hash_lookup key=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (hola_hash_add (NULL, "foo") < 0 && errno == EINVAL,
+        "hola_hash_add h=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (hola_hash_add (h, NULL) < 0 && errno == EINVAL,
+        "hola_hash_add key=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (hola_hash_delete (NULL, "foo") < 0 && errno == EINVAL,
+        "hola_hash_delete h=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (hola_hash_delete (h, NULL) < 0 && errno == EINVAL,
+        "hola_hash_delete key=NULL fails with EINVAL");
+
+    ok (hola_hash_first (NULL) == NULL,
+        "hola_hash_first h=NULL returns NULL");
+    ok (hola_hash_next (NULL) == NULL,
+        "hola_hash_next h=NULL returns NULL");
+    ok (hola_hash_size (NULL) == 0,
+        "hola_hash_size h=NULL returns 0");
+
+    errno = 0;
+    ok (hola_list_add_end (NULL, "foo", "foo") == NULL && errno == EINVAL,
+        "hola_list_add_end h=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (hola_list_add_end (h, NULL, "foo") == NULL && errno == EINVAL,
+        "hola_list_add_end key=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (hola_list_add_end (h, "foo", NULL) == NULL && errno == EINVAL,
+        "hola_list_add_end item=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (hola_list_add_end (h, "noexist", "bar") == NULL && errno == ENOENT,
+        "hola_list_add_end key=nonexistent list fails with ENOENT");
+
+    errno = 0;
+    ok (hola_list_insert (NULL, "foo", "bar", false) == NULL && errno == EINVAL,
+        "hola_list_insert h=NULL fails with EINVAL");
+    errno = 0;
+    ok (hola_list_insert (h, NULL, "bar", false) == NULL && errno == EINVAL,
+        "hola_list_insert key=NULL fails with EINVAL");
+    errno = 0;
+    ok (hola_list_insert (h, "foo", NULL, true) == NULL && errno == EINVAL,
+        "hola_list_insert item=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (hola_list_delete (NULL, "foo", "bar") < 0 && errno == EINVAL,
+        "hola_list_delete h=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (hola_list_delete (h, NULL, "bar") < 0 && errno == EINVAL,
+        "hola_list_delete key=NULL fails with EINVAL");
+    errno = 0;
+    ok (hola_list_delete (h, "foo", NULL) < 0 && errno == EINVAL,
+        "hola_list_delete handle=NULL fails with EINVAL");
+    errno = 0;
+    ok (hola_list_delete (h, "foo", &errno) < 0 && errno == ENOENT,
+        "hola_list_delete key=unknown fails with ENOENT");
+
+    ok (hola_list_first (NULL, "foo") == NULL,
+        "hola_list_first h=NULL returns NULL");
+    ok (hola_list_first (h, NULL) == NULL,
+        "hola_list_first key=NULL returns NULL");
+    ok (hola_list_next (NULL, "foo") == NULL,
+        "hola_list_next h=NULL returns NULL");
+    ok (hola_list_next (h, NULL) == NULL,
+        "hola_list_next key=NULL returns NULL");
+    ok (hola_list_last (NULL, "foo") == NULL,
+        "hola_list_last h=NULL returns NULL");
+    ok (hola_list_last (h, NULL) == NULL,
+        "hola_list_last key=NULL returns NULL");
+    ok (hola_list_prev (NULL, "foo") == NULL,
+        "hola_list_prev h=NULL returns NULL");
+    ok (hola_list_prev (h, NULL) == NULL,
+        "hola_list_next key=NULL returns NULL");
+    ok (hola_list_cursor (NULL, "foo") == NULL,
+        "hola_list_cursor h=NULL returns NULL");
+    ok (hola_list_cursor (h, NULL) == NULL,
+        "hola_list_cursor key=NULL returns NULL");
+
+    errno = 0;
+    ok (hola_list_find (NULL, "foo", "bar") == NULL && errno == EINVAL,
+        "hola_list_find h=NULL fails with EINVAL");
+    errno = 0;
+    ok (hola_list_find (h, NULL, "bar") == NULL && errno == EINVAL,
+        "hola_list_find key=NULL fails with EINVAL");
+    errno = 0;
+    ok (hola_list_find (h, "foo", NULL) == NULL && errno == EINVAL,
+        "hola_list_find item=NULL fails with EINVAL");
+
+    lives_ok ({hola_set_hash_key_destructor (NULL, key_destructor);},
+        "holas_set_hash_key_destructor h=NULL doesn't crash");
+    lives_ok ({hola_set_hash_key_duplicator (NULL, key_duplicator);},
+        "holas_set_hash_key_duplicator h=NULL doesn't crash");
+    lives_ok ({hola_set_hash_key_comparator (NULL, key_comparator);},
+        "holas_set_hash_key_comparator h=NULL doesn't crash");
+    lives_ok ({hola_set_hash_key_hasher (NULL, key_hasher);},
+        "holas_set_hash_key_hasher h=NULL doesn't crash");
+
+    lives_ok ({hola_destroy (NULL);},
+        "hola_destroy h=NULL doesn't crash");
+
+    hola_destroy (h);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_hash ();
+    test_auto ();
+    test_iter ();
+    test_inval ();
+
+    done_testing ();
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/test/intree.c b/src/common/libutil/test/intree.c
index c2841d2e6401..046b26072d38 100644
--- a/src/common/libutil/test/intree.c
+++ b/src/common/libutil/test/intree.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 #include <errno.h>
 #include <pthread.h>
@@ -17,6 +20,63 @@
 
 #define NTHREADS 16
 
+/* pthread_barrier_t is an optional POSIX feature and is not present
+ * on macos.  If unavailable, a simple replacement is provided below.
+ */
+#if !defined _POSIX_BARRIERS || _POSIX_BARRIERS <= 0
+
+typedef struct {
+    int max;
+    int count;
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
+} pthread_barrier_t;
+
+#ifndef PTHREAD_BARRIER_SERIAL_THREAD
+#define PTHREAD_BARRIER_SERIAL_THREAD -1
+#endif
+
+int pthread_barrier_destroy (pthread_barrier_t *barrier)
+{
+    if (!barrier)
+        return EINVAL;
+    pthread_mutex_destroy (&barrier->lock);
+    pthread_cond_destroy (&barrier->cond);
+    return 0;
+}
+
+int pthread_barrier_init (pthread_barrier_t *barrier,
+                          void *barrier_attr,
+                          unsigned count)
+{
+    if (!barrier || barrier_attr != NULL)
+        return EINVAL;
+    barrier->max = count;
+    barrier->count = 0;
+    pthread_mutex_init (&barrier->lock, NULL);
+    pthread_cond_init (&barrier->cond, NULL);
+    return 0;
+}
+
+int pthread_barrier_wait (pthread_barrier_t *barrier)
+{
+    int ret = EINVAL;
+    if (barrier) {
+        pthread_mutex_lock (&barrier->lock);
+        if (++barrier->count == barrier->max) {
+            pthread_cond_broadcast (&barrier->cond);
+            ret = PTHREAD_BARRIER_SERIAL_THREAD;
+        }
+        else if (barrier->count < barrier->max) {
+            pthread_cond_wait (&barrier->cond, &barrier->lock);
+            ret = 0;
+        }
+        pthread_mutex_unlock (&barrier->lock);
+    }
+    return ret;
+}
+#endif
+
 static pthread_barrier_t barrier;
 
 static void *thd_intree (void *arg)
@@ -62,7 +122,7 @@ int main (int argc, char *argv[])
     }
 
     ok (pass == 1,
-        "%d threads ran executable_is_intree sucessfully", NTHREADS);
+        "%d threads ran executable_is_intree successfully", NTHREADS);
 
     pthread_barrier_destroy (&barrier);
     done_testing ();
diff --git a/src/common/libutil/test/ipaddr.c b/src/common/libutil/test/ipaddr.c
index e867a90d7c7b..f90138a0429d 100644
--- a/src/common/libutil/test/ipaddr.c
+++ b/src/common/libutil/test/ipaddr.c
@@ -8,32 +8,47 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-#include <sys/param.h>
-#include <argz.h>
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <limits.h>
+#include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/ipaddr.h"
 
+
 int main(int argc, char** argv)
 {
-    char host[MAXHOSTNAMELEN + 1];
-    char errstr[200];
-    char *addrs = NULL;
-    size_t addrs_len = 0;
-    const char *entry = NULL;
+    char host[_POSIX_HOST_NAME_MAX + 1];
+    flux_error_t error;
+    int n;
 
     plan (NO_PLAN);
 
-    ok (ipaddr_getprimary (host, sizeof (host), errstr, sizeof (errstr)) == 0,
-        "ipaddr_getprimary works");
-    diag ("primary: %s", host);
-
-    ok (ipaddr_getall (&addrs, &addrs_len, errstr, sizeof (errstr)) == 0,
-        "ipaddrs_getall works");
-    while ((entry = argz_next (addrs, addrs_len, entry)))
-        diag ("%s", entry);
-
-    free (addrs);
+    n = ipaddr_getprimary (host, sizeof (host), 0, NULL, &error);
+    ok (n == 0,
+        "ipaddr_getprimary (hostname=0 v6=0) works");
+    diag ("%s", n == 0 ? host : error.text);
+
+    n = ipaddr_getprimary (host, sizeof (host), IPADDR_V6, NULL, &error);
+    ok (n == 0,
+        "ipaddr_getprimary (hostname=0 v6=1) works");
+    diag ("%s", n == 0 ? host : error.text);
+
+    n = ipaddr_getprimary (host, sizeof (host), IPADDR_HOSTNAME, NULL, &error);
+    ok (n == 0,
+        "ipaddr_getprimary (hostname=1 v6=0) works");
+    diag ("%s", n == 0 ? host : error.text);
+
+    n = ipaddr_getprimary (host,
+                           sizeof (host),
+                           IPADDR_HOSTNAME | IPADDR_V6,
+                           NULL,
+                           &error);
+    ok (n == 0,
+        "ipaddr_getprimary (hostname=1 v6=1)");
+    diag ("%s", n == 0 ? host : error.text);
 
     done_testing();
 }
diff --git a/src/common/libutil/test/jpath.c b/src/common/libutil/test/jpath.c
new file mode 100644
index 000000000000..3f283384f965
--- /dev/null
+++ b/src/common/libutil/test/jpath.c
@@ -0,0 +1,259 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/param.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "src/common/libtap/tap.h"
+#include "jpath.h"
+
+void diag_json (json_t *o)
+{
+    char *s = json_dumps (o, JSON_COMPACT);
+    diag ("%s", s);
+    free (s);
+}
+
+void badargs (void)
+{
+    json_t *o;
+
+    if (!(o = json_object ()))
+        BAIL_OUT ("json_object() failed");
+
+    errno = 0;
+    ok (jpath_get (NULL, "foo") ==NULL && errno == EINVAL,
+        "jpath_get o=NULL fails with EINVAL");
+    errno = 0;
+    ok (jpath_del (NULL, "foo") < 0 && errno == EINVAL,
+        "jpath_del o=NULL fails with EINVAL");
+    errno = 0;
+    ok (jpath_set (NULL, "foo", json_null ()) < 0 && errno == EINVAL,
+        "jpath_set o=NULL fails with EINVAL");
+    errno = 0;
+    ok (jpath_update (NULL, "foo", json_null ()) < 0 && errno == EINVAL,
+        "jpath_update o=NULL fails with EINVAL");
+    errno = 0;
+    ok (jpath_clear_null (NULL) < 0 && errno == EINVAL,
+        "jpath_clear_null o=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (jpath_get (o, NULL) == NULL && errno == EINVAL,
+        "jpath_get path=NULL fails with EINVAL");
+    errno = 0;
+    ok (jpath_del (o, NULL) < 0 && errno == EINVAL,
+        "jpath_del path=NULL fails with EINVAL");
+    errno = 0;
+    ok (jpath_set (o, NULL, json_null ()) < 0 && errno == EINVAL,
+        "jpath_set path=NULL fails with EINVAL");
+    errno = 0;
+    ok (jpath_set_new (NULL, NULL, json_null ()) == NULL && errno == EINVAL,
+        "jpath_set_new path=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (jpath_set (o, "foo", NULL) < 0 && errno == EINVAL,
+        "jpath_set val=NULL fails with EINVAL");
+
+    json_decref (o);
+}
+
+void basic (void)
+{
+    json_t *o;
+    json_t *val[4];
+    json_t *tmp;
+    int i;
+
+    o = json_object ();
+    val[0] = json_object ();
+    val[1] = json_real (3.14);
+    val[2] = json_string ("foo");
+    val[3] = json_pack ("{s:{s:s}}", "c", "f", "bar");
+    if (!o || !val[0] || !val[1] || !val[2] || !val[3])
+        BAIL_OUT ("error creating test objects");
+
+    ok (jpath_set (o, "a.c.d", val[0]) == 0,
+        "jpath_set a.c.d=object works");
+    ok (jpath_set (o, "a.c.e", val[1]) == 0,
+        "jpath_set a.c.e=3.14 works");
+    ok (jpath_set (o, "a.b", val[2]) == 0,
+        "jpath_set a.b=\"foo\" works");
+    ok (jpath_update (o, "a", val[3]) == 0,
+        "jpath_set set a=object works");
+
+    diag_json (o);
+
+    tmp = jpath_get (o, "a.c.d");
+    ok (tmp
+        && json_is_object (tmp),
+        "jpath_get a.c.d returned expected value");
+
+    tmp = jpath_get (o, "a.c.e");
+    ok (tmp
+        && json_is_real (tmp)
+        && json_real_value (tmp) == 3.14,
+        "jpath_get a.c.e returned expected value");
+
+    tmp = jpath_get (o, "a.b");
+    ok (tmp
+        && json_is_string (tmp)
+        && streq (json_string_value (tmp), "foo"),
+        "jpath_get a.b returned expected value");
+
+    tmp = jpath_get (o, "a.c.f");
+    ok (tmp
+        && json_is_string (tmp)
+        && streq (json_string_value (tmp), "bar"),
+        "jpath_get a.c.f returned expected value");
+
+    diag_json (o);
+
+    ok (jpath_del (o, "a.b") == 0,
+        "jpath_del a.b works");
+    errno = 0;
+    ok (jpath_get (o, "a.b") == NULL && errno == ENOENT,
+        "jpath_get a.b fails with ENOENT");
+
+    ok (jpath_del (o, "a.c") == 0,
+        "jpath_del a.c works");
+    errno = 0;
+    ok (jpath_get (o, "a.c.e") == NULL && errno == ENOENT,
+        "jpath_get a.c.e fails with ENOENT");
+    errno = 0;
+    ok (jpath_get (o, "a.c.d") == NULL && errno == ENOENT,
+        "jpath_get a.c.d fails with ENOENT");
+
+    diag_json (o);
+
+    ok (jpath_del (o, "a.c.d") == 0,
+        "jpath_del on nonexistent path does not fail");
+
+    for (i = 0; i < ARRAY_SIZE (val); i++)
+        json_decref (val[i]);
+    json_decref (o);
+}
+
+void null (void)
+{
+    json_t *o;
+    json_t *val = json_pack ("{s:{s:{s:i}} s:{s:n s:n s:s}}",
+                             "user",
+                               "mykey",
+                                 "baz", 42,
+                             "sched",
+                               "reason_pending",
+                               "jobs_ahead",
+                               "resource_summary", "rank0/core0");
+
+    if (!(o = json_object ()) || !val)
+        BAIL_OUT ("error setting up test objects failed");
+
+    ok (jpath_update (o, ".", val) == 0,
+        "jpath_update with null works");
+
+    diag_json (o);
+
+    ok (jpath_clear_null (o) == 0,
+        "jpath_clear_null works");
+
+    diag_json (o);
+
+    errno = 0;
+    ok (jpath_get (o, "sched.jobs_ahead") == NULL && errno == ENOENT,
+        "null sched.jobs_ahead is eliminated after jpath_clear_null()");
+    errno = 0;
+    ok (jpath_get (o, "sched.reason_pending") == NULL && errno == ENOENT,
+        "null sched.reason_pending is eliminated after jpath_clear_null()");
+
+    json_decref (o);
+    json_decref (val);
+}
+
+void update_new (void)
+{
+    json_t *o = NULL;
+    json_t *val = json_string ("bar");
+
+    if (!val)
+        BAIL_OUT ("error setting up test object");
+
+    o = jpath_set_new (o, "a.b.c", val);
+    ok (o != NULL,
+        "json_update_new (NULL, ...) creates new object");
+    diag_json (o);
+    json_decref (o);
+}
+
+void edge (void)
+{
+    json_t *o;
+
+    if (!(o = json_object ())
+        || jpath_set (o, "foo.bar", json_null ()) < 0)
+        BAIL_OUT ("error setting up test objects failed");
+
+    errno = 0;
+    ok (jpath_del (o, ".foo") < 0 && errno == EINVAL,
+        "jpath_del .foo fails with EINVAL");
+    errno = 0;
+    ok (jpath_del (o, "foo..bar") < 0 && errno == EINVAL,
+        "jpath_del foo..bar fails with EINVAL");
+    errno = 0;
+    ok (jpath_del (o, "foo.") < 0 && errno == EINVAL,
+        "jpath_del foo. fails with EINVAL");
+
+    errno = 0;
+    ok (jpath_get (o, ".foo") == NULL && errno == EINVAL,
+        "jpath_get .foo fails with EINVAL");
+    errno = 0;
+    ok (jpath_get (o, "foo..bar") == NULL && errno == EINVAL,
+        "jpath_get foo..bar fails with EINVAL");
+    errno = 0;
+    ok (jpath_get (o, "foo.") == NULL && errno == EINVAL,
+        "jpath_get foo. fails with EINVAL");
+
+    errno = 0;
+    ok (jpath_set (o, ".foo", json_null ()) < 0 && errno == EINVAL,
+        "jpath_set .foo fails with EINVAL");
+    errno = 0;
+    ok (jpath_set (o, "foo..bar", json_null ()) < 0 && errno == EINVAL,
+        "jpath_set foo..bar fails with EINVAL");
+    errno = 0;
+    ok (jpath_set (o, "foo.", json_null ()) < 0 && errno == EINVAL,
+        "jpath_set foo. fails with EINVAL");
+
+    json_decref (o);
+}
+
+int main (int argc, char *argv[])
+{
+
+    plan (NO_PLAN);
+
+    badargs ();
+    basic ();
+    null ();
+    update_new ();
+    edge ();
+
+    done_testing ();
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/common/libutil/test/kary.c b/src/common/libutil/test/kary.c
index b2ed85b62f71..25caf8c3e2ce 100644
--- a/src/common/libutil/test/kary.c
+++ b/src/common/libutil/test/kary.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/kary.h"
 
diff --git a/src/common/libutil/test/lru_cache.c b/src/common/libutil/test/lru_cache.c
index 5d355dc44353..bf658a0fa262 100644
--- a/src/common/libutil/test/lru_cache.c
+++ b/src/common/libutil/test/lru_cache.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
 
 #include "src/common/libtap/tap.h"
diff --git a/src/common/libutil/test/msglist.c b/src/common/libutil/test/msglist.c
deleted file mode 100644
index 3e15c036b57c..000000000000
--- a/src/common/libutil/test/msglist.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <pthread.h>
-#include <errno.h>
-#include <string.h>
-#include <signal.h>
-#include <alloca.h>
-
-#include "src/common/libtap/tap.h"
-#include "src/common/libutil/msglist.h"
-#include "src/common/libutil/xzmalloc.h"
-
-int main (int argc, char *argv[])
-{
-    msglist_t *ml;
-    int e;
-    char *msg;
-    struct pollfd pfd;
-
-    plan (19);
-
-    ok ((ml = msglist_create (free)) != NULL,
-        "msglist_create works");
-    ok ((e = msglist_pollevents (ml)) >= 0 && e == POLLOUT,
-        "msglist_pollevents on empty msglist returns POLLOUT");
-    ok (msglist_append (ml, xstrdup ("foo")) == 0,
-        "msglist_append 'foo' works");
-    ok ((e = msglist_pollevents (ml)) >= 0 && e == (POLLOUT | POLLIN),
-        "msglist_pollevents on non-empty msglist returns POLLOUT | POLLIN");
-    ok (msglist_push (ml, xstrdup ("bar")) == 0,
-        "msglist_push 'bar' works");
-    ok ((e = msglist_pollevents (ml)) >= 0 && e == (POLLOUT | POLLIN),
-        "msglist_pollevents still returns POLLOUT | POLLIN");
-    ok ((msg = msglist_pop (ml)) != NULL && !strcmp (msg, "bar"),
-        "msglist_pop returns 'bar'");
-    ok ((e = msglist_pollevents (ml)) >= 0 && e == (POLLOUT | POLLIN),
-        "msglist_pollevents still returns POLLOUT | POLLIN");
-    free (msg);
-
-    ok ((msg = msglist_pop (ml)) != NULL && !strcmp (msg, "foo"),
-        "msglist_pop returns 'foo'");
-    ok ((e = msglist_pollevents (ml)) >= 0 && e == POLLOUT,
-        "msglist_pollevents on empty msglist returns POLLOUT");
-    // cppcheck-suppress doubleFree
-    free (msg);
-
-    ok ((pfd.fd = msglist_pollfd (ml)) >= 0,
-        "msglist_pollfd works");
-    pfd.events = POLLIN,
-    pfd.revents = 0,
-    ok (poll (&pfd, 1, 0) == 1 && pfd.revents == POLLIN,
-        "pollfd suggests we read pollevents");
-    ok ((e = msglist_pollevents (ml)) >= 0 && e == POLLOUT,
-        "msglist_pollevents on empty msglist returns POLLOUT");
-    pfd.events = POLLIN,
-    pfd.revents = 0,
-    ok (poll (&pfd, 1, 0) == 0,
-        "pollfd is no longer ready");
-    ok (msglist_push (ml, xstrdup ("foo")) == 0,
-        "msglist_push 'foo' works");
-    pfd.events = POLLIN,
-    pfd.revents = 0,
-    ok (poll (&pfd, 1, 0) == 1 && pfd.revents == POLLIN,
-        "pollfd suggests we read pollevents");
-    ok ((e = msglist_pollevents (ml)) >= 0 && e == (POLLOUT | POLLIN),
-        "msglist_pollevents on non-empty msglist returns POLLOUT | POLLIN");
-    pfd.events = POLLIN,
-    pfd.revents = 0,
-    ok (poll (&pfd, 1, 0) == 0,
-        "pollfd is no longer ready");
-    ok ((e = msglist_pollevents (ml)) >= 0 && e == (POLLOUT | POLLIN),
-        "msglist_pollevents still returns POLLOUT | POLLIN");
-
-    msglist_destroy (ml);
-
-    done_testing ();
-    return (0);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libutil/test/parse_size.c b/src/common/libutil/test/parse_size.c
new file mode 100644
index 000000000000..ac5436ead2d1
--- /dev/null
+++ b/src/common/libutil/test/parse_size.c
@@ -0,0 +1,130 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <string.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "ccan/array_size/array_size.h"
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/parse_size.h"
+
+struct entry {
+    const char *s;
+    uint64_t val;
+    int errnum;
+};
+
+const struct entry testvec[] = {
+    // bad
+    { "xx", 0, EINVAL },
+    { "", 0, EINVAL },
+    { "1q", 0, EINVAL },
+    { "1kb", 0, EINVAL },
+    { "-1", 0, EINVAL },
+    { "1E20", 0, EOVERFLOW },
+    { "M", 0, EINVAL },
+    { "1m", 0, EINVAL },
+    { "1g", 0, EINVAL },
+    { "nan", 0, EINVAL },
+    { "inf", 0, EINVAL },
+    { "1b", 0, EINVAL },
+    // good
+    { "0", 0, 0 },
+    { "0K", 0, 0 },
+    { "077", 63, 0 },
+    { "0xff", 255, 0 },
+    { "+42", 42, 0 },
+    { "1", 1, 0 },
+    { "1E2", 100, 0 },
+    { "4k", 4096, 0 },
+    { "1M", 1048576, 0 },
+    { "2G", 2147483648, 0 },
+    { "0.5k", 512, 0 },
+    { "4T", 4398046511104, 0 },
+    { "18446744073709551615", UINT64_MAX, 0 },
+    { "  42", 42, 0 },
+    { "1P", 1125899906842624, 0 },
+    { "0.5E", 576460752303423488, 0 },
+};
+
+static void test_parse (void)
+{
+    uint64_t val;
+    int rc;
+
+    lives_ok ({parse_size (NULL, &val);},
+        "parse_size input=NULL doesn't crash");
+    lives_ok ({parse_size ("x", NULL);},
+        "parse_size value=NULL doesn't crash");
+
+    for (int i = 0; i < ARRAY_SIZE (testvec); i++) {
+        val = 0;
+        errno = 0;
+        rc = parse_size (testvec[i].s, &val);
+        if (testvec[i].errnum == 0) {
+            ok (rc == 0 && val == testvec[i].val,
+                "parse_size val=%s works", testvec[i].s);
+            if (rc == 0 && val != testvec[i].val)
+                diag ("got %ju", (uintmax_t)val);
+        }
+        else {
+            ok (rc == -1 && errno == testvec[i].errnum,
+                "parse_size val=%s fails with errno=%d",
+                testvec[i].s,
+                testvec[i].errnum);
+        }
+    }
+}
+
+const struct entry encode_tests[] = {
+    { "0", 0, 0 },
+    { "1K", 1024, 0 },
+    { "1.5K", 1024*1.5, 0 },
+    { "4K", 4096, 0 },
+    { "1M", 1048576, 0 },
+    { "1.000001M", 1048577, 0 },
+    { "8.75M", 1024*1024*8.75, 0 },
+    { "2G", 2147483648, 0 },
+    { "512", 512, 0 },
+    { "1.22G", 1024*1024*1024*1.22, 0 },
+    { "4T", 4398046511104, 0 },
+    { "4.04T", 4398046511104*1.01, 0 },
+    { "8.5P", 1125899906842624UL * 8.5, 0 },
+    { "16E", UINT64_MAX, 0 },
+};
+
+static void test_encode (void)
+{
+    for (int i = 0; i < ARRAY_SIZE (encode_tests); i++) {
+        const struct entry *te = &encode_tests[i];
+        const char *result = encode_size (te->val);
+        is (result, te->s,
+            "encode_size (%ju) = %s",
+            (uintmax_t) te->val,
+            result);
+    }
+}
+
+int main (int argc, char **argv)
+{
+    plan (NO_PLAN);
+    test_parse ();
+    test_encode ();
+    done_testing ();
+    return 0;
+}
+
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/common/libutil/test/popen2.c b/src/common/libutil/test/popen2.c
index 7d0132649eec..e315990781ff 100644
--- a/src/common/libutil/test/popen2.c
+++ b/src/common/libutil/test/popen2.c
@@ -8,14 +8,47 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 #include <errno.h>
 #include <stdint.h>
+#include <sys/wait.h>
 
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/popen2.h"
 #include "src/common/libutil/read_all.h"
 
+static void test_popen2_stderr ()
+{
+    struct popen2_child *p;
+    char *av[] = { "cat",  "/nosuchfile", NULL };
+    char *buf;
+    int efd;
+
+    ok (popen2 ("cat", av, 42) == NULL && errno == EINVAL,
+        "popen2() with invalid flags returns EINVAL");
+
+    ok ((p = popen2 ("cat", av, POPEN2_CAPTURE_STDERR)) != NULL,
+        "popen2() with POPEN2_CAPTURE_STDERR works");
+    ok (pclose2 (p) == 0x100,
+        "immediate pclose2 returns failed exit status of command");
+
+    ok ((p = popen2 ("cat", av, POPEN2_CAPTURE_STDERR)) != NULL,
+        "popen2() with POPEN2_CAPTURE_STDERR works");
+    ok ((efd = popen2_get_stderr_fd (p)) >= 0,
+        "popen2_get_stderr_fd() works");
+    ok (read_all (efd, (void **) &buf) > 0,
+        "read from stderr fd worked");
+    ok (pclose2 (p) == 0x100,
+        "pclose2 returns failed exit status of command");
+
+    like (buf, ".*: No such file or directory",
+        "stderr contained expected error");
+    free (buf);
+}
+
 int main(int argc, char** argv)
 {
     struct popen2_child *p;
@@ -27,13 +60,13 @@ int main(int argc, char** argv)
     plan (NO_PLAN);
 
     /* open/close */
-    ok ((p = popen2 ("cat", av)) != NULL,
+    ok ((p = popen2 ("cat", av, 0)) != NULL,
         "popen2 cat worked");
     ok (pclose2 (p) == 0,
         "immediate pclose2 OK");
 
     /* open/write/close */
-    ok ((p = popen2 ("cat", av)) != NULL,
+    ok ((p = popen2 ("cat", av, 0)) != NULL,
         "popen2 cat worked");
     fd = popen2_get_fd (p);
     ok (fd >= 0,
@@ -44,7 +77,7 @@ int main(int argc, char** argv)
         "pclose2 with read data pending OK");
 
     /* open/write/read/close */
-    ok ((p = popen2 ("cat", av)) != NULL,
+    ok ((p = popen2 ("cat", av, 0)) != NULL,
         "popen2 cat worked");
     fd = popen2_get_fd (p);
     ok (fd >= 0,
@@ -59,17 +92,17 @@ int main(int argc, char** argv)
 
     /* open failure */
     errno = 0;
-    p = popen2 ("/noexist", av);
+    p = popen2 ("/noexist", av, 0);
     ok (p == NULL && errno == ENOENT,
         "popen2 /noexist failed with ENOENT");
 
     /* open/close (child exit error) */
-    ok ((p = popen2 ("/bin/false", av)) != NULL,
-        "popen2 /bin/false OK");
-    errno = 0;
-    ok (pclose2 (p) < 0 && errno == EIO,
-        "pclose2 failed with EIO");
+    ok ((p = popen2 ("false", (char*[]){ "false", NULL }, 0)) != NULL,
+        "popen2 false OK");
+    ok (pclose2 (p) == 0x100,
+        "pclose2 returns child exit code 1");
 
+    test_popen2_stderr ();
     done_testing();
 }
 
diff --git a/src/common/libutil/test/sigutil.c b/src/common/libutil/test/sigutil.c
new file mode 100644
index 000000000000..f5df847a8839
--- /dev/null
+++ b/src/common/libutil/test/sigutil.c
@@ -0,0 +1,80 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <string.h>
+#include <stdint.h>
+
+#include "sigutil.h"
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+
+static void test_errors ()
+{
+    errno = 0;
+    ok (sigutil_signum (NULL) < 0 && errno == EINVAL,
+        "sigutil_signum (NULL) returns EINVAL");
+    errno = 0;
+    ok (sigutil_signum ("0") < 0 && errno == EINVAL,
+        "sigutil_signum (\"0\") returns EINVAL");
+    errno = 0;
+    ok (sigutil_signum ("-12") < 0 && errno == EINVAL,
+        "sigutil_signum (\"-12\") returns EINVAL");
+    errno = 0;
+    ok (sigutil_signum ("SIGFOO") < 0 && errno == ENOENT,
+        "sigutil_signum() with invalid name returns ENOENT");
+
+    errno = 0;
+    ok (sigutil_signame (0) == NULL && errno == EINVAL,
+        "sigutil_signame (0) returns EINVAL");
+    errno = 0;
+    ok (sigutil_signame (-1) == NULL && errno == EINVAL,
+        "sigutil_signame (0) returns EINVAL");
+    errno = 0;
+    ok (sigutil_signame (12345) == NULL && errno == ENOENT,
+        "sigutil_signame (12345) returns EINVAL");
+}
+
+static void test_basic ()
+{
+    ok (sigutil_signum ("1") == 1,
+        "sigutil_signum() works with string that is a number");
+    ok (sigutil_signum ("SIGKILL") == SIGKILL,
+        "sigutil_signum (\"SIGKILL\") works");
+    ok (sigutil_signum ("KILL") == SIGKILL,
+        "sigutil_signum (\"KILL\") works");
+    ok (sigutil_signum ("SIGSYS") == SIGSYS,
+        "sigutil_signum (\"SIGSYS\") works");
+    ok (sigutil_signum ("SYS") == SIGSYS,
+        "sigutil_signum (\"SYS\") works");
+
+    is (sigutil_signame (SIGKILL), "SIGKILL",
+        "sigutil_signame (SIGKILL) works");
+    is (sigutil_signame (SIGHUP), "SIGHUP",
+        "sigutil_signame (SIGHUP) works");
+    is (sigutil_signame (SIGSYS), "SIGSYS",
+        "sigutil_signame (SIGSYS) works");
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+    test_errors ();
+    test_basic ();
+    done_testing ();
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/test/slice.c b/src/common/libutil/test/slice.c
new file mode 100644
index 000000000000..9e838d3f5af9
--- /dev/null
+++ b/src/common/libutil/test/slice.c
@@ -0,0 +1,161 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <stdbool.h>
+#include <string.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
+#include "src/common/libutil/slice.h"
+
+const char *testinput = "ABCD";
+
+struct testent {
+    const char *s;
+    struct slice slice;
+    const char *result;
+};
+
+struct testent testvec[] = {
+    { "[0:2]",    { .start = 0, .stop = 2,  .step = 1 },   "AB" },
+    { "[0:4:2]",  { .start = 0, .stop = 4,  .step = 2 },   "AC" },
+    { "[1:]",     { .start = 1, .stop = 4,  .step = 1 },   "BCD" },
+    { "[:3]",     { .start = 0, .stop = 3,  .step = 1 },   "ABC" },
+    { "[1:3]",    { .start = 1, .stop = 3,  .step = 1 },   "BC" },
+    { "[1:3:]",   { .start = 1, .stop = 3,  .step = 1 },   "BC" },
+    { "[1:99]",   { .start = 1, .stop = 99, .step = 1 },   "BCD" },
+    { "[::2]",    { .start = 0, .stop = 4,  .step = 2 },   "AC" },
+    { "[::]",     { .start = 0, .stop = 4,  .step = 1 },   "ABCD" },
+    { "[:]",      { .start = 0, .stop = 4,  .step = 1 },   "ABCD" },
+    { "[8:]",     { .start = 8, .stop = 4,  .step = 1 },   "" },
+    { "[3:1]",    { .start = 3, .stop = 1,  .step = 1 },   "" },
+    { "[::-1]",   { .start = 3, .stop = -1, .step = -1 },  "DCBA" },
+    { "[-1:0:-1]",{ .start = 3, .stop = 0,  .step = -1 },  "DCB" },
+    { "[-3:-1]",  { .start = 1, .stop = 3,  .step = 1 },   "BC" },
+    { "[99:0:-1]", { .start = 99,.stop = 0, .step = -1 },  "DCB" },
+    { "[0:4:-1]", { .start = 0, .stop = 4,  .step = -1 },  "" },
+};
+
+const char *badvec[] = {
+    ":",
+    "[:",
+    ":]",
+    "[:]x",
+    "x[:]",
+    "[]",
+};
+
+bool check_parse (struct testent test, const char *in)
+{
+    size_t slen = strlen (in);
+    struct slice sl;
+
+    if (slice_parse (&sl, test.s, slen) < 0) {
+        diag ("parse %s failed", test.s);
+        return false;
+    }
+    if (sl.start != test.slice.start) {
+        diag ("parse %s: start=%d != %d", test.s, sl.start, test.slice.start);
+        return false;
+    }
+    if (sl.stop != test.slice.stop) {
+        diag ("parse %s: stop=%d != %d", test.s, sl.stop, test.slice.stop);
+        return false;
+    }
+    if (sl.step != test.slice.step) {
+        diag ("parse %s: step=%d != %d", test.s, sl.step, test.slice.step);
+        return false;
+    }
+
+    return true;
+}
+
+static int string_slice (struct slice *sl, const char *in, char **out)
+{
+    size_t slen = strlen (in);
+    char *s;
+    char *cp;
+    int i;
+
+    if (!(s = calloc (1, slen + 1)))
+        BAIL_OUT ("out of memory");
+    cp = s;
+    i = slice_first (sl);
+    while (i >= 0) {
+        if (i >= slen)
+            BAIL_OUT ("unexpected slice_first/next index %d", i);
+        *cp++ = in[i];
+        i = slice_next (sl);
+    }
+    *out = s;
+    return 0;
+}
+
+bool check_slice (struct testent test)
+{
+    struct slice sl;
+    char *result;
+
+    if (slice_parse (&sl, test.s, strlen (testinput)) < 0) {
+        diag ("parse %s failed", test.s);
+        return false;
+    }
+    if (string_slice (&sl, testinput, &result) < 0) {
+        diag ("slice %s failed", test.s);
+        return false;
+    }
+    if (!streq (result, test.result)) {
+        diag ("slice %s: %s != %s", test.s, result, test.result);
+        free (result);
+        return false;
+    }
+    free (result);
+    return true;
+}
+
+int main (int argc, char *argv[])
+{
+    struct slice sl;
+
+    plan (NO_PLAN);
+
+    for (int i = 0; i < ARRAY_SIZE (testvec); i++) {
+        ok (check_parse (testvec[i], testinput),
+            "parsed \"%s\"", testvec[i].s);
+    }
+
+    for (int i = 0; i < ARRAY_SIZE (testvec); i++) {
+        ok (check_slice (testvec[i]),
+            "sliced \"%s\"", testvec[i].s);
+    }
+
+    for (int i = 0; i < ARRAY_SIZE (badvec); i++) {
+        ok (slice_parse (&sl, badvec[i], strlen (testinput)) < 0,
+           "rejected \"%s\"", badvec[i]);
+    }
+
+    ok (slice_parse (NULL, "[:]", 4) < 0,
+        "slice_parse sl=NULL fails");
+    ok (slice_parse (&sl, NULL, 4) < 0,
+        "slice_parse s=NULL fails");
+    ok (slice_first (NULL) == -1,
+        "slice_first sl=NULL returns -1");
+    ok (slice_next (NULL) == -1,
+        "slice_next sl=NULL returns -1");
+
+    done_testing ();
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/test/stdlog.c b/src/common/libutil/test/stdlog.c
index b0d34e2dcba8..ac28db9533ab 100644
--- a/src/common/libutil/test/stdlog.c
+++ b/src/common/libutil/test/stdlog.c
@@ -8,12 +8,16 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <string.h>
+#include <ctype.h>
+
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/wallclock.h"
 #include "src/common/libutil/stdlog.h"
-
-#include <string.h>
-#include <ctype.h>
+#include "ccan/str/str.h"
 
 static char *valid[] = {
     "<1>1 - - - - - - message",
@@ -28,7 +32,7 @@ static char *valid[] = {
     "<42>1 2016-06-12T22:59:59.816857Z 0 - - - - message",
     "<42>1 2016-06-12T22:59:59.816857Z 1 - - - - message",
     "<42>1 2016-06-12T22:59:59.816857Z 4294967295 - - - - message",
-    "<42>1 2016-06-12T22:59:59.816857Z this-is-a-really-long-hostname-field-well-we-have-255-chars-avaialable-so-maybe-not-that-long-huh - - - - message",
+    "<42>1 2016-06-12T22:59:59.816857Z this-is-a-really-long-hostname-field-well-we-have-255-chars-available-so-maybe-not-that-long-huh - - - - message",
     "<42>1 2016-06-12T22:59:59.816857Z 0 logger - - - message",
     "<42>1 2016-06-12T22:59:59.816857Z 0 procid-000@@@-aaa - - - message",
     "<42>1 2016-06-12T22:59:59.816857Z 0 logger procid - - message",
@@ -44,31 +48,34 @@ void test_split (void)
     char buf[2048];
     char *xtra;
     struct stdlog_header hdr;
-    int len;
-    int msglen;
+    int n;
+    size_t len;
+    size_t msglen;
     const char *msg;
 
     stdlog_init (&hdr);
 
     /* orig=foo\nbar\nbaz */
-    len = stdlog_encode (buf, sizeof (buf), &hdr, STDLOG_NILVALUE,
+    n = stdlog_encode (buf, sizeof (buf), &hdr, STDLOG_NILVALUE,
                          "foo\nbar\nbaz");
-    ok (len >= 0,
+    ok (n >= 0,
         "stdlog_encode encoded foo\\nbar\\nbaz");
+    len = n;
     xtra = stdlog_split_message (buf, &len, "\r\n");
-    ok (xtra != NULL && !strcmp (xtra, "bar\nbaz"),
+    ok (xtra != NULL && streq (xtra, "bar\nbaz"),
         "stdlog_split_message got bar\\nbaz");
     ok (stdlog_decode (buf, len, &hdr, NULL, NULL, &msg, &msglen) == 0
         && msg != NULL && msglen == 3 && !strncmp (msg, "foo", msglen),
         "and truncated orig message to foo");
 
     /* xtra=bar\nbaz */
-    len = stdlog_encode (buf, sizeof (buf), &hdr, STDLOG_NILVALUE, xtra);
-    ok (len >= 0,
+    n = stdlog_encode (buf, sizeof (buf), &hdr, STDLOG_NILVALUE, xtra);
+    ok (n >= 0,
         "stdlog_encode encoded bar\\nbaz");
+    len = n;
     free (xtra);
     xtra = stdlog_split_message (buf, &len, "\r\n");
-    ok (xtra != NULL && !strcmp (xtra, "baz"),
+    ok (xtra != NULL && streq (xtra, "baz"),
         "stdlog_split_message got baz");
     diag ("xtra='%s'", xtra);
     ok (stdlog_decode (buf, len, &hdr, NULL, NULL, &msg, &msglen) == 0
@@ -76,9 +83,10 @@ void test_split (void)
         "and truncated orig message to bar");
 
     /* xtra=baz */
-    len = stdlog_encode (buf, sizeof (buf), &hdr, STDLOG_NILVALUE, xtra);
-    ok (len >= 0,
+    n = stdlog_encode (buf, sizeof (buf), &hdr, STDLOG_NILVALUE, xtra);
+    ok (n >= 0,
         "stdlog_encode encoded baz");
+    len = n;
     free (xtra);
     xtra = stdlog_split_message (buf, &len, "\r\n");
     ok (xtra == NULL,
@@ -89,15 +97,19 @@ int main(int argc, char** argv)
 {
     char buf[2048];
     struct stdlog_header hdr, cln;
-    int n, len;
+    int n;
+    size_t len;
     const char *sd, *msg;
-    int sdlen, msglen;
+    size_t sdlen, msglen;
 
     plan (NO_PLAN);
 
     stdlog_init (&hdr);
-    len = stdlog_encode (buf, sizeof (buf), &hdr,
-                         STDLOG_NILVALUE, STDLOG_NILVALUE);
+    len = stdlog_encode (buf,
+                         sizeof (buf),
+                         &hdr,
+                         STDLOG_NILVALUE,
+                         STDLOG_NILVALUE);
     ok (len >= 0,
         "stdlog_init encoded defaults");
     diag ("%.*s", len, buf);
@@ -113,38 +125,67 @@ int main(int argc, char** argv)
         "stdlog_decode decoded pri");
     ok (hdr.version == cln.version,
         "stdlog_decode decoded version");
-    ok (strcmp (hdr.timestamp, cln.timestamp) == 0,
+    ok (streq (hdr.timestamp, cln.timestamp),
         "stdlog_decode decoded timestamp");
-    ok (strcmp (hdr.hostname, cln.hostname) == 0,
+    ok (streq (hdr.hostname, cln.hostname),
         "stdlog_decode decoded hostname") ;
-    ok (strcmp (hdr.appname, cln.appname) == 0,
+    ok (streq (hdr.appname, cln.appname),
         "stdlog_decode decoded appname") ;
-    ok (strcmp (hdr.procid, cln.procid) == 0,
+    ok (streq (hdr.procid, cln.procid),
         "stdlog_decode decoded procid") ;
-    ok (strcmp (hdr.msgid, cln.msgid) == 0,
+    ok (streq (hdr.msgid, cln.msgid),
         "stdlog_decode decoded msgid") ;
-    ok (sdlen == strlen (STDLOG_NILVALUE) && strncmp (sd, STDLOG_NILVALUE, sdlen) == 0,
+    ok (sdlen == strlen (STDLOG_NILVALUE)
+        && strncmp (sd, STDLOG_NILVALUE, sdlen) == 0,
         "stdlog_decode decoded structured data");
-    ok (msglen == strlen (STDLOG_NILVALUE) && strncmp (msg, STDLOG_NILVALUE, msglen) == 0,
+    ok (msglen == strlen (STDLOG_NILVALUE)
+        && strncmp (msg, STDLOG_NILVALUE, msglen) == 0,
         "stdlog_decode decoded message");
 
     /* Check that trailing \n or \r in message are dropped
      */
     stdlog_init (&hdr);
-    len = stdlog_encode (buf, sizeof (buf), &hdr,
+    len = stdlog_encode (buf,
+                         sizeof (buf),
+                         &hdr,
                          STDLOG_NILVALUE,
                          "Hello whorl\n\r\n");
     ok (len >= 0,
         "stdlog_encode worked with message");
     diag ("%.*s", len, buf);
     n = stdlog_decode (buf, len, &hdr, &sd, &sdlen, &msg, &msglen);
-    ok (n == 0 && strncmp (msg, "Hello whorl", msglen) == 0,
+    ok (n == 0
+        && strncmp (msg, "Hello whorl", msglen) == 0,
         "trailing cr/lf chars were truncated");
 
+    /* Check that valid UTF-8 non-ascii characters are preserved
+     */
+    const char data[] = "jobid ƒ6LEmNENaf9 😄 😹";
+    len = stdlog_encode (buf,
+                         sizeof (buf),
+                         &hdr,
+                         STDLOG_NILVALUE,
+                         data);
+    ok (len >= 0,
+        "stdlog_encode worked with %s", data);
+    n = stdlog_decode (buf, len, &hdr, &sd, &sdlen, &msg, &msglen);
+    ok (n == 0,
+        "stdlog_decode worked");
+    is (data, msg,
+        "non-ascii characters were preserved");
+
     int i = 0;
     while (valid[i] != NULL) {
-        n = stdlog_decode (valid[i], strlen (valid[i]), &hdr, &sd, &sdlen, &msg, &msglen);
-        ok (n == 0 && msglen == strlen ("message") && strncmp (msg, "message", msglen) == 0,
+        n = stdlog_decode (valid[i],
+                           strlen (valid[i]),
+                           &hdr,
+                           &sd,
+                           &sdlen,
+                           &msg,
+                           &msglen);
+        ok (n == 0
+            && msglen == strlen ("message")
+            && strncmp (msg, "message", msglen) == 0,
             "successfully decoded %s", valid[i]);
         i++;
     }
diff --git a/src/common/libutil/test/strstrip.c b/src/common/libutil/test/strstrip.c
new file mode 100644
index 000000000000..30ab2509d9b9
--- /dev/null
+++ b/src/common/libutil/test/strstrip.c
@@ -0,0 +1,86 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <string.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/strstrip.h"
+
+struct str_test {
+    const char *input;
+    const char *printable;
+    const char *expected;
+};
+
+struct str_test tests[] = {
+    { "",               "",                 ""  },
+    { "   ",            "",                 ""  },
+    { "\t",             "",                 ""  },
+    { "a",              "a",                "a" },
+    { "no thing",       "no thing",         "no thing" },
+    { "   no thing",    "   no thing",      "no thing" },
+    { "   no thing\n",  "   no thing\\n",   "no thing" },
+    { "   no thing  \n","   no thing \\n",  "no thing" },
+    { "a     ",         "     a",           "a" },
+    { "\na   ",         "\\n  a",           "a" },
+    { NULL, NULL, NULL },
+};
+
+int main(int argc, char** argv)
+{
+    struct str_test *st = tests;
+
+    plan (NO_PLAN);
+
+    ok (strstrip (NULL) == NULL && errno == EINVAL,
+        "strstrip (NULL) returns EINVAL");
+    ok (strstrip_copy (NULL) == NULL && errno == EINVAL,
+        "strstrip_copy (NULL) returns EINVAL");
+
+    while (st->input != NULL) {
+        char *result;
+        char *s = strdup (st->input);
+        if (!s)
+            BAIL_OUT ("Failed to copy input string %s", st->input);
+
+        /* strstrip() */
+        ok ((result = strstrip (s)) != NULL,
+            "strstrip (\"%s\")",
+            st->printable);
+        is (result, st->expected,
+           "got expected result");
+        free (s);
+
+        /* strstrip_copy() */
+        if (!(s = strdup (st->input)))
+            BAIL_OUT ("Failed to copy input string %s", st->input);
+        ok ((result = strstrip_copy (s)) != NULL,
+            "strstrip_copy (\"%s\")",
+            st->printable);
+        is (result, st->expected,
+           "got expected result");
+        is (s, st->input,
+           "original string unmodified");
+        free (s);
+        free (result);
+
+        st++;
+    }
+
+    done_testing();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libutil/test/timestamp.c b/src/common/libutil/test/timestamp.c
new file mode 100644
index 000000000000..6e5c0448b8f1
--- /dev/null
+++ b/src/common/libutil/test/timestamp.c
@@ -0,0 +1,306 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/timestamp.h"
+
+struct test_entry {
+    const char *entry;
+    time_t ts;
+    int sec;
+    int min;
+    int hour;
+    int mday;
+    int mon;
+    int year;
+    suseconds_t us;
+};
+
+/* N.B.: All expected outputs assume TZ=PST8PDT
+ */
+struct test_entry tests[] = {
+    { "2017-03-17T04:11:45.948349Z",
+      1489723905,
+      45, 11, 21, 16, 3, 2017, 948349
+    },
+    { "2020-06-05T23:34:22.960708Z",
+      1591400062,
+      22, 34, 16, 5, 6, 2020, 960708
+    },
+    { "1977-10-18T15:30:37.53737Z",
+      246036637,
+      37, 30, 8, 18, 10, 1977, 537370
+    },
+    { "1971-11-02T15:18:03.191981Z",
+      57943083,
+      3, 18, 7, 2, 11, 1971, 191981
+    },
+    { "1996-12-17T15:23:31.253948Z",
+      850836211,
+      31, 23, 7, 17, 12, 1996, 253948
+    },
+    { "2013-10-11T11:46:10.907826Z",
+      1381491970,
+      10, 46, 4, 11, 10, 2013, 907826
+    },
+    { "2011-02-03T07:44:19.881821Z",
+      1296719059,
+      19, 44, 23, 2, 2, 2011, 881821
+    },
+    { "1979-07-28T05:59:14.035254Z",
+      301989554,
+      14, 59, 22, 27, 7, 1979, 35254
+    },
+    { "1977-10-22T14:17:21.905639Z",
+      246377841,
+      21, 17, 7, 22, 10, 1977, 905639
+    },
+    { "2013-02-27T20:00:39.353657Z",
+      1361995239,
+      39, 0, 12, 27, 2, 2013, 353657
+    },
+    { "2023-04-08T23:14:34.029081Z",
+      1680995674,
+      34, 14, 16, 8, 4, 2023, 29081
+    },
+    { "2013-01-29T02:36:38.527697Z",
+      1359426998,
+      38, 36, 18, 28, 1, 2013, 527697
+    },
+    { "1996-11-12T23:58:38.277011Z",
+      847843118,
+      38, 58, 15, 12, 11, 1996, 277011
+    },
+    { "2007-01-27T18:13:58.749355Z",
+      1169921638,
+      58, 13, 10, 27, 1, 2007, 749355
+    },
+    { "1985-01-11T05:51:23.032399Z",
+      474270683,
+      23, 51, 21, 10, 1, 1985, 32399
+    },
+    { "1971-06-26T06:41:19.743417Z",
+      46766479,
+      19, 41, 23, 25, 6, 1971, 743417
+    },
+    { "1996-08-05T05:31:01.268064Z",
+      839223061,
+      1, 31, 22, 4, 8, 1996, 268064
+    },
+    { "2000-02-23T12:13:17.427706Z",
+      951307997,
+      17, 13, 4, 23, 2, 2000, 427706
+    },
+    { "1985-04-07T00:31:25.608501Z",
+      481681885,
+      25, 31, 16, 6, 4, 1985, 608501
+    },
+    { "1970-04-21T12:58:31.529143Z",
+      9550711,
+      31, 58, 4, 21, 4, 1970, 529143
+    },
+    { "1978-11-22T13:16:29.795159Z",
+      280588589,
+      29, 16, 5, 22, 11, 1978, 795159
+    },
+    { "1984-11-07T12:10:05.840087Z",
+      468677405,
+      5, 10, 4, 7, 11, 1984, 840087
+    },
+    { "1987-11-06T22:33:15.153931Z",
+      563236395,
+      15, 33, 14, 6, 11, 1987, 153931
+    },
+    { "1979-11-23T00:55:52.367158Z",
+      312166552,
+      52, 55, 16, 22, 11, 1979, 367158
+    },
+    { "1972-10-19T17:02:31.682269Z",
+      88362151,
+      31, 2, 10, 19, 10, 1972, 682269
+    },
+    { "2001-12-27T10:13:29.52Z",
+      1009448009,
+      29, 13, 2, 27, 12, 2001, 520000
+    },
+    { "1984-10-30T10:49:56.3Z",
+      467981396,
+      56, 49, 2, 30, 10, 1984, 300000
+    },
+    { "1989-04-14T05:06:09.000003Z",
+      608533569,
+      9, 6, 22, 13, 4, 1989, 3
+    },
+    { "1983-03-16T23:04:03.00003Z",
+      416703843,
+      3, 4, 15, 16, 3, 1983, 30
+    },
+    { "1988-05-11T02:47:16.003Z",
+      579322036,
+      16, 47, 19, 10, 5, 1988, 3000
+    },
+    { "1970-01-01T00:00:00.836367Z",
+      0,
+      0, 0, 16, 31, 12, 1969, 836367
+    },
+    { "1970-01-01T00:00:00.000000Z",
+      0,
+      0, 0, 16, 31, 12, 1969, 0
+    },
+    { "2011-08-28T16:30:40.000000Z",
+      1314549040,
+      40, 30, 9, 28, 8, 2011, 0
+    },
+    { "1970-01-01T00:00:00Z",
+      0,
+      0, 0, 16, 31, 12, 1969, 0
+    },
+    { "2017-01-14T05:18:47Z",
+      1484371127,
+      47, 18, 21, 13, 1, 2017, 0
+    },
+    { NULL, 0, 0, 0, 0, 0, 0, 0, 0 }
+};
+
+static void test_invalid ()
+{
+    struct tm tm;
+    struct timeval tv;
+    struct test_entry *te = &tests[0];
+
+    ok (timestamp_parse ("", &tm, &tv) < 0,
+        "timestamp_parse empty string fails");
+    ok (timestamp_parse ("1:00", &tm, &tv) < 0,
+        "timestamp_parse on invalid timestamp fails");
+    ok (timestamp_parse ("1969-01-01T00:00:00Z", &tm, &tv) < 0,
+        "timestamp_parse on too old timestamp fails");
+
+    ok (timestamp_parse (NULL, NULL, NULL) < 0 && errno == EINVAL,
+        "timestamp_parse (NULL, NULL, NULL) fails with EINVAL");
+    ok (timestamp_parse (NULL, &tm, &tv) < 0 && errno == EINVAL,
+        "timestamp_parse (NULL, &tm, &tv) fails with EINVAL");
+    ok (timestamp_parse (te->entry, NULL, NULL) < 0 && errno == EINVAL,
+        "timestamp_parse (NULL, &tm, &tv) fails with EINVAL");
+
+    ok (timestamp_from_double (-1., &tm, &tv) < 0 && errno == EINVAL,
+        "timestamp_from_double (-1, &tm, &tv) fails with EINVAL");
+    ok (timestamp_from_double (0., NULL, NULL) < 0 && errno == EINVAL,
+        "timestamp_from_double (0., NULL, NULL) fails with EINVAL");
+
+    ok (timestamp_parse (te->entry, &tm, NULL) == 0,
+        "timestamp_parse (ts, &tm, NULL) works");
+    ok (tm.tm_year == te->year - 1900
+        && tm.tm_mon == te->mon - 1
+        && tm.tm_mday == te->mday
+        && tm.tm_min == te->min
+        && tm.tm_sec == te->sec,
+        "timestamp is expected values");
+
+    ok (timestamp_parse (te->entry, NULL, &tv) == 0,
+        "timestamp_parse (ts, NULL, &tv) works");
+    ok (tv.tv_sec == te->ts
+        && tv.tv_usec == te->us,
+        "timsestamp is expected value");
+}
+
+static void test_entry_check (struct test_entry *test,
+                              struct tm tm,
+                              struct timeval tv)
+{
+    ok (tm.tm_sec == test->sec,
+        "tm_sec == %d (expected %d)", tm.tm_sec, test->sec);
+    ok (tm.tm_min == test->min,
+        "tm_min == %d (expected %d)", tm.tm_min, test->min);
+    /* N.B.: We do not test tm_hour since this may be influenced
+     * by incorrect, missing, or updated DST values in the local
+     * system's tzdata.
+    ok (tm.tm_mday == test->mday,
+        "tm_mday == %d (expected %d)", tm.tm_mday, test->mday);
+     */
+    /* tm_mon is months since Jan 0-11
+     */
+    ok (tm.tm_mon == test->mon - 1,
+        "tm_mon == %d (expected %d)", tm.tm_mon, test->mon - 1);
+    /* tm_year is number of years since 1900
+     */
+    ok (tm.tm_year == test->year - 1900,
+        "tm_year == %d (expected %d)", tm.tm_year, test->year - 1900);
+
+    ok (tv.tv_sec == test->ts,
+        "tv_sec == %u (expected %u)", tv.tv_sec, test->ts);
+    ok (tv.tv_usec == test->us,
+        "tv_usec == %u (expected %u)", tv.tv_usec, test->us);
+}
+
+static void test_all ()
+{
+    struct test_entry *test;
+    struct tm tm;
+    struct timeval tv;
+
+    test = tests;
+    while (test->entry) {
+        char buf[1024];
+        ok (timestamp_parse (test->entry, &tm, &tv) == 0,
+            "timestamp_parse: %s", test->entry);
+        timestamp_tostr ((time_t) tv.tv_sec, buf, sizeof (buf));
+        diag ("%s", buf);
+        test_entry_check (test, tm, tv);
+
+        /* Now test timestamp_from_double()
+         */
+        double ts = tv.tv_sec + ((double) tv.tv_usec / 1000000);
+
+        memset (&tm, 0, sizeof (tm));
+        memset (&tv, 0, sizeof (tv));
+        ok (timestamp_from_double (ts, &tm, &tv) == 0,
+            "timestamp_from_double (%f) works",
+            ts);
+        test_entry_check (test, tm, tv);
+
+        ++test;
+    }
+}
+
+void test_tzoffset (void)
+{
+    struct tm tm;
+
+    ok (timestamp_tzoffset (NULL, NULL, 0) < 0 && errno == EINVAL,
+        "timestamp_tzoffset (NULL, NULL, 0) returns EINVAL");
+
+    memset (&tm, 0, sizeof (tm));
+    ok (timestamp_tzoffset (&tm, NULL, 0) < 0 && errno == EINVAL,
+        "timestamp_tzoffset (&tm, NULL, 0) returns EINVAL");
+}
+
+int main (int argc, char *argv[])
+{
+
+    plan (NO_PLAN);
+
+    /* All expected outputs assume a timezone of PST8PDT */
+    setenv ("TZ", "PST8PDT", 1);
+
+    test_all ();
+    test_invalid ();
+    test_tzoffset ();
+
+    done_testing ();
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/test/tomltk.c b/src/common/libutil/test/tomltk.c
index d55994c95384..147c359939db 100644
--- a/src/common/libutil/test/tomltk.c
+++ b/src/common/libutil/test/tomltk.c
@@ -21,6 +21,7 @@
 #include <jansson.h>
 
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 #include "src/common/libtomlc99/toml.h"
 #include "tomltk.h"
 
@@ -72,10 +73,10 @@ static bool check_ts (json_t *ts, const char *timestr)
         return false;
     if (!gmtime_r (&t, &tm))
         return false;
-    if (strftime (buf, sizeof (buf), "%FT%TZ", &tm) == 0)
+    if (strftime (buf, sizeof (buf), "%Y-%m-%dT%TZ", &tm) == 0)
         return false;
     diag ("%s: %s ?= %s", __FUNCTION__, buf, timestr);
-    return !strcmp (buf, timestr);
+    return streq (buf, timestr);
 }
 
 void test_json_ts(void)
@@ -126,7 +127,7 @@ void test_tojson_t1 (void)
                      "ts", &ts);
     ok (rc == 0,
         "t1: unpack successful");
-    ok (i == 1 && d == 3.14 && s != NULL && !strcmp (s, "foo") && b != 0
+    ok (i == 1 && d == 3.14 && s != NULL && streq (s, "foo") && b != 0
         && check_ts (ts, "1979-05-27T07:32:00Z"),
         "t1: has expected values");
     json_decref (obj);
@@ -209,7 +210,7 @@ void test_parse_lineno (void)
     ok (error.lineno == 4,
         "bad1: error.lineno is 4");
     const char *msg = "unterminated s-quote";
-    ok (!strcmp (error.errbuf, msg),
+    ok (streq (error.errbuf, msg),
         "bad1: error is \"%s\"", msg); // no "line %d: " prefix
 }
 
diff --git a/src/common/libutil/test/unlink.c b/src/common/libutil/test/unlink.c
index 7534f44516b6..ae3c352cacb7 100644
--- a/src/common/libutil/test/unlink.c
+++ b/src/common/libutil/test/unlink.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <limits.h>
 #include <sys/types.h>
 #include <sys/stat.h>
diff --git a/src/common/libutil/test/wallclock.c b/src/common/libutil/test/wallclock.c
index 5b63dbdef2b6..267d14ebafc3 100644
--- a/src/common/libutil/test/wallclock.c
+++ b/src/common/libutil/test/wallclock.c
@@ -8,13 +8,16 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <string.h>
+#include <ctype.h>
+
 #include "src/common/libtap/tap.h"
 #include "src/common/libutil/wallclock.h"
 #include "src/common/libutil/stdlog.h"
 
-#include <string.h>
-#include <ctype.h>
-
 int main(int argc, char** argv)
 {
     char buf[WALLCLOCK_MAXLEN];
diff --git a/src/common/libutil/test/zsecurity.c b/src/common/libutil/test/zsecurity.c
deleted file mode 100644
index 19ca14622b90..000000000000
--- a/src/common/libutil/test/zsecurity.c
+++ /dev/null
@@ -1,451 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <limits.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <string.h>
-#include <errno.h>
-#include <dirent.h>
-#include <czmq.h>
-
-#include "src/common/libutil/zsecurity.h"
-#include "src/common/libtap/tap.h"
-#include "src/common/libutil/unlink_recursive.h"
-
-void test_ctor_dtor (void)
-{
-    zsecurity_t *sec;
-    const char *s;
-
-    lives_ok ({zsecurity_destroy (NULL);},
-            "zsecurity_destroy accepts a NULL argument");
-
-    ok ((sec = zsecurity_create (0, "/tmp")) != NULL,
-            "zsecurity_create with no selected method works");
-    ok ((s = zsecurity_errstr (sec)) != NULL && !strcmp (s, "Success"),
-            "zsecurity_errstr returns 'Success'");
-    ok ((s = zsecurity_get_directory (sec)) != NULL && !strcmp (s, "/tmp"),
-            "zsecurity_get_directory returns configured confdir");
-    ok (zsecurity_type_enabled (sec, ZSECURITY_TYPE_PLAIN) == false,
-            "zsecurity_type_enabled ZSECURITY_TYPE_PLAIN false");
-    ok (zsecurity_type_enabled (sec, ZSECURITY_TYPE_CURVE) == false,
-            "zsecurity_type_enabled ZSECURITY_TYPE_CURVE false");
-    zsecurity_destroy (sec);
-
-    ok ((sec = zsecurity_create (0, NULL)) != NULL,
-            "zsecurity_create with NULL confdir works");
-    ok (zsecurity_get_directory (sec) == NULL,
-            "zsecurity_get_directory returns configured NULL");
-    zsecurity_destroy (sec);
-
-    errno = 0;
-    sec = zsecurity_create (ZSECURITY_TYPE_CURVE | ZSECURITY_TYPE_PLAIN, NULL);
-    ok (sec == NULL && errno == EINVAL,
-            "zsecurity_create PLAIN|CURVE returns EINVAL");
-
-    ok ((sec = zsecurity_create (ZSECURITY_TYPE_PLAIN, NULL)) != NULL,
-            "zsecurity_create PLAIN works");
-    ok (zsecurity_type_enabled (sec, ZSECURITY_TYPE_PLAIN) == true,
-            "zsecurity_type_enabled ZSECURITY_TYPE_PLAIN true");
-    ok (zsecurity_type_enabled (sec, ZSECURITY_TYPE_CURVE) == false,
-            "zsecurity_type_enabled ZSECURITY_TYPE_CURVE false");
-    zsecurity_destroy (sec);
-}
-
-void test_keygen (void)
-{
-    zsecurity_t *sec;
-    const char *tmp = getenv ("TMPDIR");
-    char path[PATH_MAX];
-    struct stat sb;
-
-    /* NULL confdir.
-     */
-    sec = zsecurity_create (0, NULL);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    errno = 0;
-    ok (zsecurity_keygen (sec) < 0 && errno == EINVAL,
-            "zsecurity_keygen fails with EINVAL if confdir not set");
-    zsecurity_destroy (sec);
-
-    /* Nonexistent confdir.
-     *
-     * errno has multiple possibilities depending on system, EACCES,
-     * EROFS, EPERM, etc.  Simply check for failure and errno != 0.
-     */
-    sec = zsecurity_create (0, "/noexist");
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    errno = 0;
-    ok (zsecurity_keygen (sec) < 0 && errno != 0,
-            "zsecurity_keygen fails with errno != 0 if confdir does not exist");
-    zsecurity_destroy (sec);
-
-    /* Same with FORCE flag.
-     */
-    sec = zsecurity_create (ZSECURITY_KEYGEN_FORCE, "/noexist");
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    errno = 0;
-    ok (zsecurity_keygen (sec) < 0 && errno != 0,
-            "zsecurity_keygen (force) fails with errno != 0 if confdir does not exist");
-    zsecurity_destroy (sec);
-
-    /* No security modes selected.
-     */
-    snprintf (path, sizeof (path), "%s/sectest.XXXXXX", tmp ? tmp : "/tmp");
-    if (!mkdtemp (path))
-        BAIL_OUT ("could not create tmp directory");
-    sec = zsecurity_create (0, path);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    ok (zsecurity_keygen (sec) == 0,
-            "zsecurity_keygen with no security modes works");
-    ok ((stat (path, &sb) == 0 && S_ISDIR (sb.st_mode)
-                && (sb.st_mode & (S_IRWXU|S_IRWXG|S_IRWXO)) == 0700),
-            "confdir is a directory with mode 0700");
-    ok (unlink_recursive (path) == 1,
-            "unlinked 1 file/dir");
-    zsecurity_destroy (sec);
-
-    /* Wrong confdir perms
-     */
-    snprintf (path, sizeof (path), "%s/sectest.XXXXXX", tmp ? tmp : "/tmp");
-    if (!mkdtemp (path))
-        BAIL_OUT ("could not create tmp directory");
-    sec = zsecurity_create (0, path);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    if (chmod (path, 0755) < 0)
-        BAIL_OUT ("chmod %s: %s", path, strerror (errno));
-    errno = 0;
-    ok (zsecurity_keygen (sec) < 0 && errno == EPERM,
-            "zsecurity_keygen with bad mode confdir fails with EPERM");
-    ok (unlink_recursive (path) == 1,
-            "unlinked 1 file/dir");
-    zsecurity_destroy (sec);
-
-    /* PLAIN
-     */
-    snprintf (path, sizeof (path), "%s/sectest.XXXXXX", tmp ? tmp : "/tmp");
-    if (!mkdtemp (path))
-        BAIL_OUT ("could not create tmp directory");
-    sec = zsecurity_create (ZSECURITY_TYPE_PLAIN, path);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    ok (zsecurity_keygen (sec) == 0,
-            "zsecurity_keygen PLAIN works");
-    ok (unlink_recursive (path) == 2,
-            "unlinked 2 file/dir");
-    zsecurity_destroy (sec);
-
-    /* CURVE
-     */
-    snprintf (path, sizeof (path), "%s/sectest.XXXXXX", tmp ? tmp : "/tmp");
-    if (!mkdtemp (path))
-        BAIL_OUT ("could not create tmp directory");
-    sec = zsecurity_create (ZSECURITY_TYPE_CURVE, path);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    ok (zsecurity_keygen (sec) == 0,
-            "zsecurity_keygen CURVE works");
-    ok (unlink_recursive (path) == 6,
-            "unlinked 6 file/dir");
-    zsecurity_destroy (sec);
-
-    /* CURVE overwrite
-     */
-    snprintf (path, sizeof (path), "%s/sectest.XXXXXX", tmp ? tmp : "/tmp");
-    if (!mkdtemp (path))
-        BAIL_OUT ("could not create tmp directory");
-    sec = zsecurity_create (ZSECURITY_TYPE_CURVE, path);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    if (zsecurity_keygen (sec) < 0)
-        BAIL_OUT ("zsecurity_keygen CURVE failed");
-    errno = 0;
-    ok (zsecurity_keygen (sec) < 0 && errno == EEXIST,
-            "zsecurity_keygen CURVE-overwrite fails with EEXIST");
-    ok (unlink_recursive (path) == 6,
-            "unlinked 6 file/dir");
-    zsecurity_destroy (sec);
-
-    /* Same with FORCE
-     */
-    snprintf (path, sizeof (path), "%s/sectest.XXXXXX", tmp ? tmp : "/tmp");
-    if (!mkdtemp (path))
-        BAIL_OUT ("could not create tmp directory");
-    sec = zsecurity_create (ZSECURITY_TYPE_CURVE | ZSECURITY_KEYGEN_FORCE, path);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    if (zsecurity_keygen (sec) < 0)
-        BAIL_OUT ("zsecurity_keygen CURVE failed");
-    errno = 0;
-    ok (zsecurity_keygen (sec) == 0,
-            "zsecurity_keygen (force) CURVE-overwrite works");
-    ok (unlink_recursive (path) == 6,
-            "unlinked 6 file/dir");
-    zsecurity_destroy (sec);
-
-    /* PLAIN overwrite
-     */
-    snprintf (path, sizeof (path), "%s/sectest.XXXXXX", tmp ? tmp : "/tmp");
-    if (!mkdtemp (path))
-        BAIL_OUT ("could not create tmp directory");
-    sec = zsecurity_create (ZSECURITY_TYPE_PLAIN, path);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    if (zsecurity_keygen (sec) < 0)
-        BAIL_OUT ("zsecurity_keygen PLAIN failed");
-    errno = 0;
-    ok (zsecurity_keygen (sec) < 0 && errno == EEXIST,
-            "zsecurity_keygen PLAIN-overwrite fails with EEXIST");
-    ok (unlink_recursive (path) == 2,
-            "unlinked 2 file/dir");
-    zsecurity_destroy (sec);
-
-    /* Same with FORCE
-     */
-    snprintf (path, sizeof (path), "%s/sectest.XXXXXX", tmp ? tmp : "/tmp");
-    if (!mkdtemp (path))
-        BAIL_OUT ("could not create tmp directory");
-    sec = zsecurity_create (ZSECURITY_TYPE_PLAIN | ZSECURITY_KEYGEN_FORCE, path);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create failed");
-    if (zsecurity_keygen (sec) < 0)
-        BAIL_OUT ("zsecurity_keygen PLAIN failed");
-    errno = 0;
-    ok (zsecurity_keygen (sec) == 0,
-            "zsecurity_keygen (force) PLAIN-overwrite works");
-    ok (unlink_recursive (path) == 2,
-            "unlinked 2 file/dir");
-    zsecurity_destroy (sec);
-}
-
-void test_plain (void)
-{
-    zsecurity_t *sec;
-    const char *tmp = getenv ("TMPDIR");
-    char path[PATH_MAX];
-    zsock_t *cli, *srv, *rdy, *rogue;
-    zpoller_t *srv_poller;
-    int srv_port;
-    char *s;
-
-    snprintf (path, sizeof (path), "%s/sectest.XXXXXX", tmp ? tmp : "/tmp");
-    if (!mkdtemp (path))
-        BAIL_OUT ("could not create tmp directory");
-    sec = zsecurity_create (ZSECURITY_TYPE_PLAIN | ZSECURITY_VERBOSE, path);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create PLAIN failed");
-    if (zsecurity_keygen (sec) < 0)
-        BAIL_OUT ("zsecurity_keygen PLAIN failed");
-    ok (zsecurity_comms_init (sec) == 0,
-            "zsecurity_comms_init PLAIN works");
-
-    /* set up server */
-    if (!(srv = zsock_new_pull (NULL)))
-        BAIL_OUT ("zsock_new: %s", zmq_strerror (errno));
-    ok (zsecurity_ssockinit (sec, srv) == 0,
-            "zsecurity_ssockinit works");
-    srv_port = zsock_bind (srv, "tcp://127.0.0.1:*");
-    ok (srv_port >= 0,
-            "server bound to localhost on port %d", srv_port);
-    if (!(srv_poller = zpoller_new (srv, NULL)))
-        BAIL_OUT ("poller_new failed");
-
-    /* set up client */
-    if (!(cli = zsock_new_push (NULL)))
-        BAIL_OUT ("zsock_new: %s", zmq_strerror (errno));
-    ok (zsecurity_csockinit (sec, cli) == 0,
-            "zsecurity_csockinit works");
-    ok (zsock_connect (cli, "tcp://127.0.0.1:%d", srv_port) >= 0,
-            "client connected to server");
-    ok (zstr_sendx (cli, "Hi", NULL) == 0,
-            "client sent Hi");
-    rdy = zpoller_wait (srv_poller, 1000);
-    ok (rdy == srv,
-            "server ready within 1s timeout");
-    s = NULL;
-    ok (rdy != NULL && zstr_recvx (srv, &s, NULL) == 1
-            && s != NULL && !strcmp (s, "Hi"),
-            "server received Hi");
-    free (s);
-
-    /* rogue client tries to send with no security setup */
-    if (!(rogue = zsock_new_push (NULL)))
-        BAIL_OUT ("zsock_new: %s", zmq_strerror (errno));
-    ok (zsock_connect (rogue, "tcp://127.0.0.1:%d", srv_port) >= 0,
-        "rogue connected to server with no security");
-    ok (zstr_sendx (rogue, "Blimey!", NULL) == 0,
-            "rogue sent Blimey!");
-    rdy = zpoller_wait (srv_poller, 200);
-    ok (rdy == NULL && zpoller_expired (srv_poller),
-            "server not ready within 0.2s timeout");
-    zsock_destroy (&rogue);
-
-    /* rogue client tries to send with wrong PLAIN password */
-    if (!(rogue = zsock_new_push (NULL)))
-        BAIL_OUT ("zsock_new: %s", zmq_strerror (errno));
-    zsock_set_plain_username (rogue, "client");
-    zsock_set_plain_password (rogue, "not-the-correct-password");
-    ok (zsock_connect (rogue, "tcp://127.0.0.1:%d", srv_port) >= 0,
-        "rogue connected to server using wrong password");
-    ok (zstr_sendx (rogue, "Skallywag!", NULL) == 0,
-            "rogue sent Skallywag!");
-    rdy = zpoller_wait (srv_poller, 200);
-    ok (rdy == NULL && zpoller_expired (srv_poller),
-            "server not ready within 0.2s timeout");
-    zsock_destroy (&rogue);
-
-    zsock_destroy (&cli);
-    zpoller_destroy (&srv_poller);
-    zsock_destroy (&srv);
-    zsecurity_destroy (sec);
-    unlink_recursive (path);
-}
-
-void test_curve (void)
-{
-    zsecurity_t *sec;
-    const char *tmp = getenv ("TMPDIR");
-    char path[PATH_MAX];
-    zsock_t *cli, *srv, *rdy, *rogue;
-    zpoller_t *srv_poller;
-    int srv_port;
-    char *s;
-    zcert_t *rogue_cert;
-
-    snprintf (path, sizeof (path), "%s/sectest.XXXXXX", tmp ? tmp : "/tmp");
-    if (!mkdtemp (path))
-        BAIL_OUT ("could not create tmp directory");
-    sec = zsecurity_create (ZSECURITY_TYPE_CURVE | ZSECURITY_VERBOSE, path);
-    if (!sec)
-        BAIL_OUT ("zsecurity_create CURVE failed");
-    if (zsecurity_keygen (sec) < 0)
-        BAIL_OUT ("zsecurity_keygen CURVE failed");
-    ok (zsecurity_comms_init (sec) == 0,
-            "zsecurity_comms_init CURVE works");
-
-    /* set up server */
-    if (!(srv = zsock_new_pull (NULL)))
-        BAIL_OUT ("zsock_new: %s", zmq_strerror (errno));
-    ok (zsecurity_ssockinit (sec, srv) == 0,
-            "zsecurity_ssockinit works");
-    srv_port = zsock_bind (srv, "tcp://127.0.0.1:*");
-    ok (srv_port >= 0,
-            "server bound to localhost on port %d", srv_port);
-    if (!(srv_poller = zpoller_new (srv, NULL)))
-        BAIL_OUT ("poller_new failed");
-
-    /* set up client */
-    if (!(cli = zsock_new_push (NULL)))
-        BAIL_OUT ("zsock_new: %s", zmq_strerror (errno));
-    ok (zsecurity_csockinit (sec, cli) == 0,
-            "zsecurity_csockinit works");
-    ok (zsock_connect (cli, "tcp://127.0.0.1:%d", srv_port) >= 0,
-            "client connected to server");
-
-    /* client sends Greetings! */
-    ok (zstr_sendx (cli, "Greetings!", NULL) == 0,
-            "client sent Greetings!");
-    rdy = zpoller_wait (srv_poller, 1000);
-    ok (rdy == srv,
-            "server ready within 1s timeout");
-    s = NULL;
-    ok (rdy != NULL && zstr_recvx (srv, &s, NULL) == 1
-            && s != NULL && !strcmp (s, "Greetings!"),
-            "server received Greetings!");
-    free (s);
-
-    /* rogue client tries to send with no security setup */
-    if (!(rogue = zsock_new_push (NULL)))
-        BAIL_OUT ("zsock_new: %s", zmq_strerror (errno));
-    ok (zsock_connect (rogue, "tcp://127.0.0.1:%d", srv_port) >= 0,
-        "rogue connected to server with no security");
-    ok (zstr_sendx (rogue, "Avast!", NULL) == 0,
-            "rogue sent Avast");
-    rdy = zpoller_wait (srv_poller, 200);
-    ok (rdy == NULL && zpoller_expired (srv_poller),
-            "server not ready within 0.2s timeout");
-    zsock_destroy (&rogue);
-
-    /* rogue client tries to send with correct server public key,
-     * but unknown client (server doesn't have public key in "certstore")
-     */
-    if (!(rogue_cert = zcert_new ()))
-        BAIL_OUT ("zcert_new: %s", zmq_strerror (errno));
-    if (!(rogue = zsock_new_push (NULL)))
-        BAIL_OUT ("zsock_new: %s", zmq_strerror (errno));
-    zsock_set_zap_domain (rogue, "flux"); // same as zsecurity_t hardwired
-    zcert_apply (rogue_cert, rogue);
-    /* read server public key from file */
-    char server_file[PATH_MAX];
-    int n;
-    n = snprintf (server_file, sizeof (server_file), "%s/curve/server", path);
-    if ((n < 0) || (n >= sizeof (server_file)))
-        BAIL_OUT ("snprintf failed in creation of server_file");
-    zcert_t *server_cert = zcert_load (server_file);
-    if (!server_cert)
-        BAIL_OUT ("zcert_load %s: %s", server_file, zmq_strerror (errno));
-    const char *server_key = zcert_public_txt (server_cert);
-    zsock_set_curve_serverkey (rogue, server_key);
-    /* now connect */
-    ok (zsock_connect (rogue, "tcp://127.0.0.1:%d", srv_port) >= 0,
-        "rogue connected to server using right server, wrong client key");
-    ok (zstr_sendx (rogue, "Haar!", NULL) == 0,
-            "rogue sent Haar!");
-    rdy = zpoller_wait (srv_poller, 200);
-    ok (rdy == NULL && zpoller_expired (srv_poller),
-            "server not ready within 0.2s timeout");
-    zcert_destroy (&rogue_cert);
-    zcert_destroy (&server_cert);
-    zsock_destroy (&rogue);
-
-    zsock_destroy (&cli);
-    zpoller_destroy (&srv_poller);
-    zsock_destroy (&srv);
-    zsecurity_destroy (sec);
-    unlink_recursive (path);
-}
-
-void alarm_callback (int arg)
-{
-    diag ("test timed out");
-    exit (1);
-}
-
-int main (int argc, char *argv[])
-{
-    plan (NO_PLAN);
-
-    signal (SIGALRM, alarm_callback);
-    alarm (30);
-
-    test_ctor_dtor ();
-    test_keygen ();
-    test_plain ();
-    test_curve ();
-
-    done_testing ();
-    return (0);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libutil/timestamp.c b/src/common/libutil/timestamp.c
index f3f5caf92d08..6d0c969302ab 100644
--- a/src/common/libutil/timestamp.c
+++ b/src/common/libutil/timestamp.c
@@ -12,16 +12,41 @@
 #  include <config.h>
 #endif /* HAVE_CONFIG_H */
 
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
 #include <time.h>
 
 #include "timestamp.h"
+#include "ccan/str/str.h"
+
+/*
+ *  GNU libc has timegm(3), but the manual states:
+ *
+ *   These functions [timelocal(), timegm()] are nonstandard GNU extensions
+ *    that are also present on the BSDs.  Avoid their use.
+ *
+ *  This "portable" version was found on sourceware.org, and appears to work:
+ *
+ *   https://patchwork.sourceware.org/project/glibc/patch/20211011115406.11430-2-alx.manpages@gmail.com/
+ *
+ */
+static time_t portable_timegm (struct tm *tm)
+{
+    time_t t;
+
+    tm->tm_isdst = 0;
+    if ((t = mktime (tm)) == (time_t) -1)
+        return t;
+    return t - timezone;
+}
 
 int timestamp_tostr (time_t t, char *buf, int size)
 {
     struct tm tm;
     if (t < 0 || !gmtime_r (&t, &tm))
         return -1;
-    if (strftime (buf, size, "%FT%TZ", &tm) == 0)
+    if (strftime (buf, size, "%Y-%m-%dT%TZ", &tm) == 0)
         return -1;
     return 0;
 }
@@ -30,15 +55,128 @@ int timestamp_fromstr (const char *s, time_t *tp)
 {
     struct tm tm;
     time_t t;
-    if (!strptime (s, "%FT%TZ", &tm))
+    if (!strptime (s, "%Y-%m-%dT%TZ", &tm))
         return -1;
-    if ((t = timegm (&tm)) < 0)
+    if ((t = portable_timegm (&tm)) < 0)
         return -1;
     if (tp)
         *tp = t;
     return 0;
 }
 
+int timestamp_parse (const char *s,
+                     struct tm *tm,
+                     struct timeval *tv)
+{
+    char *extra;
+    struct tm gm_tm;
+    time_t t;
+
+    if (s == NULL || (!tm && !tv)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!(extra = strptime (s, "%Y-%m-%dT%T", &gm_tm))
+        || (t = portable_timegm (&gm_tm)) < (time_t) -1) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (tm && !(localtime_r (&t, tm)))
+            return -1;
+
+    if (tv)
+        tv->tv_sec = t;
+
+    if (tv && extra[0] == '.') {
+        char *endptr;
+        double d;
+
+        errno = 0;
+        d = strtod (extra, &endptr);
+
+        /*  Note: in this implementation, there should be a "Z" after the
+         *  timestamp to indicate UTC or "Zulu" time.
+         */
+        if (errno != 0 || *endptr != 'Z')
+            return -1;
+
+        /*  Note: cast to integer type truncates. To handle underflow from
+         *  double arithmetic (e.g. result = 1234.999), add 0.5 and then
+         *  allow the truncation to simulate floor(3).
+         */
+        tv->tv_usec = (d * 1000000) + 0.5;
+    }
+    return 0;
+}
+
+int timestamp_from_double (double ts, struct tm *tm, struct timeval *tv)
+{
+    if (ts < 0. || (!tm && !tv)) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (tm) {
+        time_t t = (time_t) ts;
+        memset (tm, 0, sizeof (*tm));
+        if (!localtime_r (&t, tm))
+            return -1;
+    }
+    if (tv) {
+        tv->tv_sec = ts;
+        /*  Note: cast to integer type truncates. To handle underflow from
+         *  double arithmetic (e.g. result = 1234.999), add 0.5 and then
+         *  allow the truncation to simulate floor(3).
+         */
+        tv->tv_usec = ((ts - tv->tv_sec) * 1000000) + 0.5;
+    }
+    return 0;
+}
+
+int timestamp_tzoffset (struct tm *tm, char *buf, int len)
+{
+    if (!tm || !buf || len <= 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (strftime (buf, len, "%z", tm) == 0)
+        return -1;
+    /* Special case: use "Z" for UTC for backwards compatibility
+     */
+    if (streq (buf, "+0000")) {
+        buf[0] = 'Z';
+        buf[1] = '\0';
+        return 0;
+    }
+    /* O/w, insert `:` in offset if it is of the form [+-]NNNN for
+     * enhanced readability.
+     */
+    if (strlen (buf) == 5
+        && len >= 7
+        && (buf[0] == '-' || buf[0] == '+')) {
+        char minutes[3];
+
+        /* Save last two characters of current tz string:
+         * (Note: due to strlen check above, we know this only copies
+         *  2 characters, so it is safe to use strcpy())
+         */
+        strcpy (minutes, buf+3);
+
+        /* Insert colon after [+-]NN
+         */
+        buf[3] = ':';
+
+        /* Copy minutes back to end of string.
+         * (Note: we know minutes only contains 2 characters, so it is
+         *  safe to use strcpy())
+         */
+        strcpy (buf+4, minutes);
+    }
+    return 0;
+}
+
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libutil/timestamp.h b/src/common/libutil/timestamp.h
index c0e0c43f7196..b901965fb7b7 100644
--- a/src/common/libutil/timestamp.h
+++ b/src/common/libutil/timestamp.h
@@ -11,6 +11,7 @@
 #ifndef _UTIL_TIMESTAMP_H
 #define _UTIL_TIMESTAMP_H
 
+#include <sys/time.h>
 #include <time.h>
 
 /* Convert time_t (GMT) to ISO 8601 timestamp string,
@@ -22,6 +23,31 @@ int timestamp_tostr (time_t t, char *buf, int size);
  */
 int timestamp_fromstr (const char *s, time_t *tp);
 
+/* Convert from ISO 8601 timestamp string, including optional
+ * microsecond precision, to struct tm, timeval pair.
+ *
+ * e.g. "2022-10-15T14:43:18.159009Z"
+ *
+ * At least one of 'tm' or 'tv' must be provided.
+ */
+int timestamp_parse (const char *s,
+                     struct tm *tm,
+                     struct timeval *tv);
+
+
+/* Convert a double precision timestamp to struct tm and timeval.
+ * At least one of 'tm' or 'tv' must be provided.
+ */
+int timestamp_from_double (double ts, struct tm *tm, struct timeval *tv);
+
+/* Get the current timezone offset for `tm` in the form [+-]HH:MM
+ * and place it into the supplied buffer. As a special case, +00:00
+ * is converted to "Z" (Zulu time) for backwards compatibility when
+ * the current timezone is UTC.
+ *
+ * Returns -1 on failure, 0 for success.
+ */
+int timestamp_tzoffset (struct tm *tm, char *buf, int size);
 
 #endif /* !_UTIL_TIMESTAMP_H */
 
diff --git a/src/common/libutil/tomltk.c b/src/common/libutil/tomltk.c
index 690f0043687a..5cc20f343b02 100644
--- a/src/common/libutil/tomltk.c
+++ b/src/common/libutil/tomltk.c
@@ -20,6 +20,7 @@
 #include <jansson.h>
 
 #include "src/common/libtomlc99/toml.h"
+#include "ccan/str/str.h"
 #include "timestamp.h"
 #include "tomltk.h"
 
@@ -56,9 +57,9 @@ static void errfromtoml (struct tomltk_error *error,
     if (error) {
         char *msg = errstr;
         int lineno = -1;
-        if (!strncmp (errstr, "line ", 5)) {
+        if (strstarts (errstr, "line ")) {
             lineno = strtoul (errstr + 5, &msg, 10);
-            if (!strncmp (msg, ": ", 2))
+            if (strstarts (msg, ": "))
                 msg += 2;
         }
         return errprintf (error, filename, lineno, "%s", msg);
diff --git a/src/common/libutil/unlink_recursive.c b/src/common/libutil/unlink_recursive.c
index bb9804b55da6..657300dd7d42 100644
--- a/src/common/libutil/unlink_recursive.c
+++ b/src/common/libutil/unlink_recursive.c
@@ -20,8 +20,10 @@
 #include <dirent.h>
 #include <libgen.h>
 #include <unistd.h>
+#include <errno.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
-#include <czmq.h>
 #include "unlink_recursive.h"
 #include "dirwalk.h"
 
diff --git a/src/common/libutil/uri.c b/src/common/libutil/uri.c
new file mode 100644
index 000000000000..8ba4bbdd19fc
--- /dev/null
+++ b/src/common/libutil/uri.c
@@ -0,0 +1,116 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "src/common/libyuarel/yuarel.h"
+#include "ccan/str/str.h"
+#include "errprintf.h"
+#include "popen2.h"
+#include "read_all.h"
+#include "log.h"
+#include "strstrip.h"
+#include "errno_safe.h"
+#include "uri.h"
+
+static void nullify_newline (char *str)
+{
+    int n;
+    if (str && (n = strlen (str)) > 0) {
+        if (str[n-1] == '\n')
+            str[n-1] = '\0';
+    }
+}
+
+static char *uri_to_local (struct yuarel *yuri)
+{
+    char *uri = NULL;
+    if (asprintf (&uri, "local:///%s", yuri->path) < 0)
+        return NULL;
+    return uri;
+}
+
+char *uri_remote_get_authority (const char *uri)
+{
+    struct yuarel yuri;
+    char *cpy;
+    char *result = NULL;
+
+    if (!uri)
+        return NULL;
+
+    cpy = strdup (uri);
+    if (yuarel_parse (&yuri, cpy) == 0
+        && yuri.scheme
+        && streq (yuri.scheme, "ssh")) {
+        if (asprintf (&result,
+                      "%s%s%s",
+                      yuri.username ? yuri.username : "",
+                      yuri.username ? "@" : "",
+                      yuri.host) < 0)
+            result = NULL;
+    }
+    ERRNO_SAFE_WRAP (free, cpy);
+    return result;
+}
+
+char *uri_resolve (const char *uri, flux_error_t *errp)
+{
+    struct popen2_child *child = NULL;
+    struct yuarel yuri;
+    char *result = NULL;
+    char *errors = NULL;
+    char *argv[] = { "flux", "uri", (char *) uri, NULL };
+    int flags = errp != NULL ? POPEN2_CAPTURE_STDERR : 0;
+
+    char *cpy = strdup (uri);
+    if (!cpy)
+        return NULL;
+    if (yuarel_parse (&yuri, cpy) == 0 && yuri.scheme) {
+        if (streq (yuri.scheme, "ssh")
+            || streq (yuri.scheme, "local")) {
+            if (getenv ("FLUX_URI_RESOLVE_LOCAL"))
+                result = uri_to_local (&yuri);
+            else
+                result = strdup (uri);
+            free (cpy);
+            return result;
+        }
+    }
+    free (cpy);
+
+    if (!(child = popen2 ("flux", argv, flags))
+        || (read_all (popen2_get_fd (child), (void **)&result) < 0))
+        goto out;
+    nullify_newline (result);
+
+    if (errp) {
+        /* stderr capture requested */
+        if (read_all (popen2_get_stderr_fd (child), (void **)&errors) > 0) {
+            errprintf (errp, "%s", strstrip (errors));
+            free (errors);
+        }
+    }
+out:
+    if (pclose2 (child) != 0) {
+        /* flux-uri returned error */
+        free (result);
+        result = NULL;
+    }
+    return result;
+}
+
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/uri.h b/src/common/libutil/uri.h
new file mode 100644
index 000000000000..dbb9c889631b
--- /dev/null
+++ b/src/common/libutil/uri.h
@@ -0,0 +1,44 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _UTIL_URI_H
+#define _UTIL_URI_H
+
+#include <flux/core.h>
+
+/*  Resolve a target or "high-level" URI with the flux-uri(1) command,
+ *   returning the result. If the URI is already a native Flux URI (e.g.
+ *   `local://` or `ssh://`), then `flux uri` is *not* called and instead
+ *   the target is returned unmodified to avoid the extra overhead of
+ *   running a subprocess.
+ *
+ *  On failure, NULL is returned. If errp is not NULL, then stderr from
+ *   the underlying command will be copied there (possibly truncated).
+ *   Otherwise, stderr is not redirected or consumed, so the expectation
+ *   is that the underlying `flux uri` error will already be copied to the
+ *   callers tty.
+ *
+ *  Caller must free the returned string on success.
+ *
+ *  Note: this function uses popen2() to execute flux-uri as a subprocess,
+ *   so care should be taken in when and how this function is called.
+ */
+char *uri_resolve (const char *target, flux_error_t *errp);
+
+/*  Return the authority part of a remote URI, e.g. [username@]host
+ *  Returns NULL if uri is NULL or not a remote URI.
+ *
+ *  Caller must free returned value.
+ */
+char *uri_remote_get_authority (const char *uri);
+
+#endif /* !_UTIL_URI_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/wallclock.c b/src/common/libutil/wallclock.c
index c36f16349778..0fde546a687a 100644
--- a/src/common/libutil/wallclock.c
+++ b/src/common/libutil/wallclock.c
@@ -44,7 +44,7 @@ int wallclock_get_zulu (char *buf, size_t len)
         errno = EINVAL;
         return -1;
     }
-    if (strftime (buf, len, "%FT%T", &tm) == 0) {
+    if (strftime (buf, len, "%Y-%m-%dT%T", &tm) == 0) {
         errno = EINVAL;
         return -1;
     }
diff --git a/src/common/libutil/xzmalloc.c b/src/common/libutil/xzmalloc.c
index b4c067cdc7a9..8102c0effeab 100644
--- a/src/common/libutil/xzmalloc.c
+++ b/src/common/libutil/xzmalloc.c
@@ -29,14 +29,6 @@ void *xzmalloc (size_t size)
     return new;
 }
 
-void *xrealloc (void *ptr, size_t size)
-{
-    void *new = realloc (ptr, size);
-    if (!new)
-        oom ();
-    return new;
-}
-
 char *xstrdup (const char *s)
 {
     char *cpy = strdup (s);
@@ -66,18 +58,6 @@ char *xasprintf (const char *fmt, ...)
     return s;
 }
 
-char *xstrsub (const char *str, char a, char b)
-{
-    char *cpy = xstrdup (str);
-    char *s = cpy;
-    while (*s) {
-        if (*s == a)
-            *s = b;
-        s++;
-    }
-    return cpy;
-}
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libutil/xzmalloc.h b/src/common/libutil/xzmalloc.h
index f3a104b9479c..9cc87a26f16a 100644
--- a/src/common/libutil/xzmalloc.h
+++ b/src/common/libutil/xzmalloc.h
@@ -17,12 +17,10 @@
 /* Memory allocation functions that call oom() on allocation error.
  */
 void *xzmalloc (size_t size);
-void *xrealloc (void *ptr, size_t size);
 char *xstrdup (const char *s);
 char *xvasprintf(const char *fmt, va_list ap);
 char *xasprintf (const char *fmt, ...)
      __attribute__ ((format (printf, 1, 2)));
-char *xstrsub (const char *str, char a, char b);
 
 #endif /* !_UTIL_XZMALLOC_H */
 /*
diff --git a/src/common/libutil/zsecurity.c b/src/common/libutil/zsecurity.c
deleted file mode 100644
index 370c484a9623..000000000000
--- a/src/common/libutil/zsecurity.c
+++ /dev/null
@@ -1,491 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* zsecurity.c - flux zeromq security functions */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <getopt.h>
-#include <libgen.h>
-#include <czmq.h>
-#include <uuid.h>
-
-#include "zsecurity.h"
-
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/oom.h"
-#include "src/common/libutil/xzmalloc.h"
-
-#ifndef UUID_STR_LEN
-#define UUID_STR_LEN 37     // defined in later libuuid headers
-#endif
-
-#define FLUX_ZAP_DOMAIN "flux"
-
-struct zsecurity_struct {
-    zactor_t *auth;
-    int typemask;
-    zcert_t *srv_cert;
-    zcert_t *cli_cert;
-    char *conf_dir;
-    char *curve_dir;
-    char *passwd_file;
-    char *errstr;
-    char *confstr;
-    uid_t uid;
-    uid_t gid;
-};
-
-static int checksecdirs (zsecurity_t *c, bool create);
-static zcert_t *getcurve (zsecurity_t *c, const char *role);
-static int gencurve (zsecurity_t *c, const char *role);
-static char *getpasswd (zsecurity_t *c, const char *user);
-static int genpasswd (zsecurity_t *c, const char *user);
-
-const char *zsecurity_errstr (zsecurity_t *c)
-{
-    return (c->errstr ? c->errstr : "Success");
-}
-
-const char *zsecurity_confstr (zsecurity_t *c)
-{
-    if (c->confstr)
-        free (c->confstr);
-    if (asprintf (&c->confstr, "Security: epgm=off, tcp/ipc=%s",
-               (c->typemask & ZSECURITY_TYPE_PLAIN) ? "PLAIN"
-             : (c->typemask & ZSECURITY_TYPE_CURVE) ? "CURVE" : "off") < 0)
-        oom ();
-    return c->confstr;
-}
-
-static void seterrstr (zsecurity_t *c, const char *fmt, ...)
-{
-    va_list ap;
-
-    if (c->errstr)
-        free (c->errstr);
-    va_start (ap, fmt);
-    if (vasprintf (&c->errstr, fmt, ap) < 0)
-        oom ();
-    va_end (ap);
-}
-
-void zsecurity_destroy (zsecurity_t *c)
-{
-    if (c) {
-        free (c->conf_dir);
-        free (c->curve_dir);
-        free (c->passwd_file);
-        zcert_destroy (&c->cli_cert);
-        zcert_destroy (&c->srv_cert);
-        free (c->errstr);
-        free (c->confstr);
-        zactor_destroy (&c->auth);
-        free (c);
-    }
-}
-
-zsecurity_t *zsecurity_create (int typemask, const char *confdir)
-{
-    zsecurity_t *c = calloc (1, sizeof (*c));
-
-    if ((typemask & ZSECURITY_TYPE_CURVE) && (typemask & ZSECURITY_TYPE_PLAIN)) {
-        errno = EINVAL;
-        goto error;
-    }
-    if (!c) {
-        errno = ENOMEM;
-        goto error;
-    }
-    if (confdir) {
-        if (!(c->conf_dir = strdup (confdir))) {
-            errno = ENOMEM;
-            goto error;
-        }
-    }
-    c->uid = getuid ();
-    c->gid = getgid ();
-    c->typemask = typemask;
-    return c;
-error:
-    if (c)
-        free (c->conf_dir);
-    free (c);
-    return NULL;
-}
-
-const char *zsecurity_get_directory (zsecurity_t *c)
-{
-    return c->conf_dir;
-}
-
-bool zsecurity_type_enabled (zsecurity_t *c, int tm)
-{
-    bool ret;
-    ret = ((c->typemask & tm) == tm);
-    return ret;
-}
-
-int zsecurity_keygen (zsecurity_t *c)
-{
-    int rc = -1;
-    if (checksecdirs (c, true) < 0)
-        goto done;
-    if ((c->typemask & ZSECURITY_TYPE_CURVE)) {
-        if (gencurve (c, "client") < 0)
-            goto done;
-        if (gencurve (c, "server") < 0)
-            goto done;
-    }
-    if ((c->typemask & ZSECURITY_TYPE_PLAIN)) {
-        if (genpasswd (c, "client") < 0)
-            goto done;
-    }
-    rc = 0;
-done:
-    return rc;
-}
-
-int zsecurity_comms_init (zsecurity_t *c)
-{
-    if (c->auth == NULL && ((c->typemask & ZSECURITY_TYPE_CURVE)
-                        || (c->typemask & ZSECURITY_TYPE_PLAIN))) {
-        if (checksecdirs (c, false) < 0)
-            goto error;
-        if (!(c->auth = zactor_new (zauth, NULL))) {
-            seterrstr (c, "zactor_new (zauth): %s", zmq_strerror (errno));
-            goto error;
-        }
-        if ((c->typemask & ZSECURITY_VERBOSE)) {
-            if (zstr_sendx (c->auth, "VERBOSE", NULL) < 0)
-                goto error;
-            if (zsock_wait (c->auth) < 0)
-                goto error;
-        }
-        if ((c->typemask & ZSECURITY_TYPE_CURVE)) {
-            if (!zsys_has_curve ()) {
-                seterrstr (c, "libczmq was not built with CURVE support!");
-                errno = EINVAL;
-                goto error;
-            }
-            if (!(c->cli_cert = getcurve (c, "client")))
-                goto error;
-            if (!(c->srv_cert = getcurve (c, "server")))
-                goto error;
-            /* Authorize only the clients with certs in $confdir/curve
-             * (server must find public key of new client here)
-             */
-            if (zstr_sendx (c->auth, "CURVE", c->curve_dir, NULL) < 0)
-                goto error;
-            if (zsock_wait (c->auth) < 0)
-                goto error;
-        }
-        if ((c->typemask & ZSECURITY_TYPE_PLAIN)) {
-            if (zstr_sendx (c->auth, "PLAIN", c->passwd_file, NULL) < 0)
-                goto error;
-            if (zsock_wait (c->auth) < 0)
-                goto error;
-        }
-    }
-    return 0;
-error:
-    return -1;
-}
-
-int zsecurity_csockinit (zsecurity_t *c, void *sock)
-{
-    int rc = -1;
-
-    if ((c->typemask & ZSECURITY_TYPE_CURVE)) {
-        zsock_set_zap_domain (sock, FLUX_ZAP_DOMAIN);
-        zcert_apply (c->cli_cert, sock);
-        zsock_set_curve_serverkey (sock, zcert_public_txt (c->srv_cert));
-    } else if ((c->typemask & ZSECURITY_TYPE_PLAIN)) {
-        char *passwd = NULL;
-        if (!(passwd = getpasswd (c, "client"))) {
-            seterrstr (c, "client not found in %s", c->passwd_file);
-            goto done;
-        }
-        zsock_set_plain_username (sock, "client");
-        zsock_set_plain_password (sock, passwd);
-        free (passwd);
-    }
-    rc = 0;
-done:
-    return rc;
-}
-
-int zsecurity_ssockinit (zsecurity_t *c, void *sock)
-{
-    if ((c->typemask & (ZSECURITY_TYPE_CURVE))) {
-        zsock_set_zap_domain (sock, FLUX_ZAP_DOMAIN);
-        zcert_apply (c->srv_cert, sock);
-        zsock_set_curve_server (sock, 1);
-    } else if ((c->typemask & (ZSECURITY_TYPE_PLAIN))) {
-        zsock_set_plain_server (sock, 1);
-    }
-    return 0;
-}
-
-static int checksecdir (zsecurity_t *c, const char *path, bool create)
-{
-    struct stat sb;
-    int rc = -1;
-
-stat_again:
-    if (lstat (path, &sb) < 0) {
-        if (errno == ENOENT) {
-            if (create) {
-                if (mkdir (path, 0700) < 0) {
-                    seterrstr (c, "mkdir %s: %s", path, strerror (errno));
-                    goto done;
-                }
-                create = false;
-                goto stat_again;
-            } else {
-                seterrstr (c, "The directory '%s' does not exist.  Have you run \"flux keygen\"?", path);
-            }
-        } else
-            seterrstr (c, "lstat %s: %s", path, strerror (errno));
-        goto done;
-    }
-    if (!S_ISDIR (sb.st_mode)) {
-        errno = ENOTDIR;
-        seterrstr (c, "%s: %s", path, strerror (errno));
-        goto done;
-    }
-    if ((sb.st_mode & (S_IRWXU|S_IRWXG|S_IRWXO)) != 0700) {
-        seterrstr (c, "%s: mode should be 0700", path);
-        errno = EPERM;
-        goto done;
-    }
-    if ((sb.st_uid != c->uid)) {
-        seterrstr (c, "%s: owner should be you", path);
-        errno = EPERM;
-        goto done;
-    }
-    rc = 0;
-done:
-    return rc;
-}
-
-static int checksecdirs (zsecurity_t *c, bool create)
-{
-    if (!c->conf_dir) {
-        seterrstr (c, "config directory is not set");
-        errno = EINVAL;
-        return -1;
-    }
-    if (checksecdir (c, c->conf_dir, create) < 0)
-        return -1;
-    if ((c->typemask & ZSECURITY_TYPE_CURVE)) {
-        if (!c->curve_dir) {
-            if (asprintf (&c->curve_dir, "%s/curve", c->conf_dir) < 0) {
-                errno = ENOMEM;
-                return -1;
-            }
-        }
-        if (checksecdir (c, c->curve_dir, create) < 0)
-            return -1;
-    }
-    if ((c->typemask & ZSECURITY_TYPE_PLAIN)) {
-        if (!c->passwd_file) {
-            if (asprintf (&c->passwd_file, "%s/passwd", c->conf_dir) < 0) {
-                errno = ENOMEM;
-                return -1;
-            }
-        }
-    }
-    return 0;
-}
-
-static char * ctime_iso8601_now (char *buf, size_t sz)
-{
-    struct tm tm;
-    time_t now = time (NULL);
-
-    memset (buf, 0, sz);
-
-    if (!localtime_r (&now, &tm))
-        return (NULL);
-    strftime (buf, sz, "%FT%T", &tm);
-
-    return (buf);
-}
-
-static zcert_t *zcert_curve_new (zsecurity_t *c)
-{
-    zcert_t *new;
-    char sec[41];
-    char pub[41];
-    uint8_t s[32];
-    uint8_t p[32];
-
-    if (zmq_curve_keypair (pub, sec) < 0) {
-        if (errno == ENOTSUP)
-            seterrstr (c,
-                "No CURVE support in libzmq (not compiled with libsodium?)");
-        else
-            seterrstr (c,
-                "Unknown error generating CURVE keypair");
-        return NULL;
-    }
-
-    if (!zmq_z85_decode (s, sec) || !zmq_z85_decode (p, pub)) {
-        seterrstr (c, "zcert_curve_new: Failed to decode keys");
-        return NULL;
-    }
-
-    if (!(new = zcert_new_from (p, s)))
-        oom ();
-
-    return new;
-}
-
-static int gencurve (zsecurity_t *c, const char *role)
-{
-    char *path = NULL, *priv = NULL;;
-    zcert_t *cert = NULL;
-    char buf[64];
-    struct stat sb;
-    int rc = -1;
-
-    if (asprintf (&path, "%s/%s", c->curve_dir, role) < 0)
-        oom ();
-    if (asprintf (&priv, "%s/%s_private", c->curve_dir, role) < 0)
-        oom ();
-    if ((c->typemask & ZSECURITY_KEYGEN_FORCE)) {
-        (void)unlink (path);
-        (void)unlink (priv);
-    }
-    if (stat (path, &sb) == 0) {
-        seterrstr (c, "%s exists, try --force", path);
-        errno = EEXIST;
-        goto done;
-    }
-    if (stat (priv, &sb) == 0) {
-        seterrstr (c, "%s exists, try --force", priv);
-        errno = EEXIST;
-        goto done;
-    }
-
-    if (!(cert = zcert_curve_new (c)))
-        goto done; /* error message set in zcert_curve_new() */
-
-    zcert_set_meta (cert, "time", "%s", ctime_iso8601_now (buf, sizeof (buf)));
-    zcert_set_meta (cert, "role", "%s", role);
-    if ((c->typemask & ZSECURITY_VERBOSE)) {
-        printf ("Saving %s\n", path);
-        printf ("Saving %s\n", priv);
-    }
-    if (zcert_save (cert, path) < 0) {
-        seterrstr (c, "zcert_save %s: %s", path, zmq_strerror (errno));
-        goto done;
-    }
-    rc = 0;
-done:
-    if (cert)
-        zcert_destroy (&cert);
-    if (path)
-        free (path);
-    if (priv)
-        free (priv);
-    return rc;
-}
-
-static zcert_t *getcurve (zsecurity_t *c, const char *role)
-{
-    char s[PATH_MAX];
-    zcert_t *cert = NULL;
-
-    if (snprintf (s, sizeof (s), "%s/%s", c->curve_dir, role) >= sizeof (s)) {
-        errno = EINVAL;
-        goto error;
-    }
-    if (!(cert = zcert_load (s)))
-        seterrstr (c, "zcert_load %s: %s", s, zmq_strerror (errno));
-    return cert;
-error:
-    return NULL;
-}
-
-static char *getpasswd (zsecurity_t *c, const char *user)
-{
-    zhash_t *passwds = NULL;
-    const char *pass;
-    char *s = NULL;
-
-    if (!(passwds = zhash_new ())) {
-        errno = ENOMEM;
-        goto error;
-    }
-    zhash_autofree (passwds);
-    if (zhash_load (passwds, c->passwd_file) < 0)
-        goto error;
-    if (!(pass = zhash_lookup (passwds, user))) {
-        errno = ENOENT;
-        goto error;
-    }
-    if (!(s = strdup (pass))) {
-        errno = ENOMEM;
-        goto error;
-    }
-    zhash_destroy (&passwds);
-    return s;
-error:
-    zhash_destroy (&passwds);
-    return NULL;
-}
-
-static int genpasswd (zsecurity_t *c, const char *user)
-{
-    struct stat sb;
-    zhash_t *passwds = NULL;
-    uuid_t uuid;
-    char uuid_str[UUID_STR_LEN];
-    mode_t old_mask;
-    int rc = -1;
-
-    uuid_generate (uuid);
-    uuid_unparse (uuid, uuid_str);
-    if ((c->typemask & ZSECURITY_KEYGEN_FORCE))
-        (void)unlink (c->passwd_file);
-    if (stat (c->passwd_file, &sb) == 0) {
-        seterrstr (c, "%s exists, try --force", c->passwd_file);
-        errno = EEXIST;
-        goto done;
-    }
-    if (!(passwds = zhash_new ()))
-        oom ();
-    zhash_update (passwds, user, uuid_str);
-    if ((c->typemask & ZSECURITY_VERBOSE))
-        printf ("Saving %s\n", c->passwd_file);
-    old_mask = umask (077);
-    rc = zhash_save (passwds, c->passwd_file);
-    umask (old_mask);
-    if (rc < 0) {
-        seterrstr (c, "zhash_save %s: %s", c->passwd_file, zmq_strerror (errno));
-        goto done;
-    }
-    /* FIXME: check created file mode */
-    rc = 0;
-done:
-    if (passwds)
-        zhash_destroy (&passwds);
-    return rc;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libutil/zsecurity.h b/src/common/libutil/zsecurity.h
deleted file mode 100644
index 3d1c578d0566..000000000000
--- a/src/common/libutil/zsecurity.h
+++ /dev/null
@@ -1,120 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _UTIL_ZSECURITY_H
-#define _UTIL_ZSECURITY_H
-
-#include <stdbool.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct zsecurity_struct zsecurity_t;
-
-enum {
-    /* enabled security modes */
-    ZSECURITY_TYPE_PLAIN = 1, // cannot be used with CURVE
-    ZSECURITY_TYPE_CURVE = 2, // cannot be used with PLAIN
-
-    /* flags */
-    ZSECURITY_VERBOSE = 0x20,
-    ZSECURITY_KEYGEN_FORCE = 0x40,
-};
-
-/* Create a security context.
- * 'typemask' (may be 0) selects the security mode and optional flags.
- * 'confdir' (may be NULL) selects a key directory.
- * This function only allocates the context and does not do anything
- * to initialize the selected security modes.  zsecurity_keygen()
- * or zsecurity_comms_init() may be called next.
- * Returns context on success, or NULL on failure with errno set.
- */
-zsecurity_t *zsecurity_create (int typemask, const char *confdir);
-void zsecurity_destroy (zsecurity_t *c);
-
-/* Test whether a particular security mode is enabled
- * in the security context.
- */
-bool zsecurity_type_enabled (zsecurity_t *c, int typemask);
-
-/* Get config directory used by security context.
- * May be NULL if none was configured.
- */
-const char *zsecurity_get_directory (zsecurity_t *c);
-
-/* Generate a user's keys for the configured security modes,
- * storing them in the security context's 'confdir'.
- * If the ZSECURITY_KEYGEN_FORCE flag is set, existing keys
- * are overwritten; otherwise the existence of keys is treated as
- * an error. This function is a no-op if no keys are required
- * by the configured security modes.
- * Returns 0 on success, or -1 on failure with errno set.
- */
-int zsecurity_keygen (zsecurity_t *c);
-
-/* Initialize the security context for communication.
- * For PLAIN and CURVE, a zauth
- * actor for ZAP processing is started.  Since there can be only one registered
- * zauth actor per zeromq process, this function may only be called once
- * per process.  For PLAIN, the actor is configured to allow only connections
- * from clients who can send the 'client' password stored in 'confdir'.
- * For CURVE, the actor is configured to allow only connections from
- * clients whose public keys are stored in 'confdir'.
- * The actor is not strictly necessary for client-only contexts but
- * at this point all security contexts are both client and server capable.
- * Returns 0 on success, or -1 on failure with errno set.
- */
-int zsecurity_comms_init (zsecurity_t *c);
-
-/* Enable the configured security mode (client role) on a
- * zeromq socket.  For PLAIN, the client password is
- * obtained from 'confdir' and associated with the socket.
- * For CURVE, the server public key and client keypair are
- * obtained from 'confdir' and associated with the socket.
- * This is a no-op if neither CURVE nor PLAIN is enabled.
- * Generallay the client role calls "connect" but this is not a
- * hard requirement for the SMTP security handshake.
- * Returns 0 on success, or -1 on failure with errno set.
- */
-int zsecurity_csockinit (zsecurity_t *c, void *sock);
-
-/* Enable the configured security mode (server role) on a
- * zeromq socket.  For PLAIN, plain auth is enabled for the
- * socket via the zauth actor.  For CURVE, the server keypair
- * is obtained from 'confdir' and associated with the socket,
- * and curve auth is enabled for the socket via the zauth actor.
- * This is a no-op if neither CURVE nor PLAIN is enabled.
- * Generally the server role calls "bind" but this is not a
- * hard requirement for the ZMTP security handshake.
- * Returns 0 on success, or -1 on failure with errno set.
- */
-int zsecurity_ssockinit (zsecurity_t *c, void *sock);
-
-/* Retrieve a string describing the last error.
- * This value is valid after one of the above calls returns -1.
- * The caller should not free this string.
- */
-const char *zsecurity_errstr (zsecurity_t *c);
-
-/* Retrieve a string describing the security modes selected.
- * The caller should not free this string.
- */
-const char *zsecurity_confstr (zsecurity_t *c);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* !_UTIL_ZSECURITY_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/common/libyuarel/Makefile.am b/src/common/libyuarel/Makefile.am
index d80d44bb5b03..295b6e22fb30 100644
--- a/src/common/libyuarel/Makefile.am
+++ b/src/common/libyuarel/Makefile.am
@@ -6,6 +6,9 @@ AM_CFLAGS = \
 AM_LDFLAGS = \
         $(CODE_COVERAGE_LIBS)
 
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS)
+
 noinst_LTLIBRARIES = libyuarel.la
 
 libyuarel_la_SOURCES = \
diff --git a/src/common/libzmqutil/Makefile.am b/src/common/libzmqutil/Makefile.am
new file mode 100644
index 000000000000..db4ab41952d6
--- /dev/null
+++ b/src/common/libzmqutil/Makefile.am
@@ -0,0 +1,103 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LDFLAGS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux \
+	$(LIBUUID_CFLAGS) \
+	$(ZMQ_CFLAGS)
+
+noinst_LTLIBRARIES = \
+	libzmqutil.la
+
+libzmqutil_la_SOURCES = \
+	msg_zsock.h \
+	msg_zsock.c \
+	reactor.h \
+	reactor.c \
+	ev_zmq.h \
+	ev_zmq.c \
+	zap.h \
+	zap.c \
+	monitor.h \
+	monitor.c \
+	sockopt.h \
+	sockopt.c \
+	mpart.h \
+	mpart.c \
+	cert.h \
+	cert.c
+
+TESTS = test_msg_zsock.t \
+	test_reactor.t \
+	test_ev.t \
+	test_zap.t \
+	test_monitor.t \
+	test_mpart.t \
+	test_cert.t
+
+check_PROGRAMS = \
+	$(TESTS)
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+	$(top_srcdir)/config/tap-driver.sh
+
+test_ldadd = \
+	$(top_builddir)/src/common/libzmqutil/libzmqutil.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux/libflux.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(LIBUUID_LIBS) \
+	$(ZMQ_LIBS)
+
+test_cppflags = \
+	-I$(top_srcdir)/src/common/libtap \
+	-I$(srcdir) \
+	$(AM_CPPFLAGS)
+
+test_ldflags = \
+	-no-install
+
+test_msg_zsock_t_SOURCES = test/msg_zsock.c
+test_msg_zsock_t_CPPFLAGS = $(test_cppflags)
+test_msg_zsock_t_LDADD = $(test_ldadd)
+test_msg_zsock_t_LDFLAGS = $(test_ldflags)
+
+test_reactor_t_SOURCES = test/reactor.c
+test_reactor_t_CPPFLAGS = $(test_cppflags)
+test_reactor_t_LDADD = $(test_ldadd)
+test_reactor_t_LDFLAGS = $(test_ldflags)
+
+test_ev_t_SOURCES = test/ev.c
+test_ev_t_CPPFLAGS = $(test_cppflags)
+test_ev_t_LDADD = $(test_ldadd)
+test_ev_t_LDFLAGS = $(test_ldflags)
+
+test_zap_t_SOURCES = test/zap.c
+test_zap_t_CPPFLAGS = $(test_cppflags)
+test_zap_t_LDADD = $(test_ldadd)
+test_zap_t_LDFLAGS = $(test_ldflags)
+
+test_monitor_t_SOURCES = test/monitor.c
+test_monitor_t_CPPFLAGS = $(test_cppflags)
+test_monitor_t_LDADD = $(test_ldadd)
+test_monitor_t_LDFLAGS = $(test_ldflags)
+
+test_mpart_t_SOURCES = test/mpart.c
+test_mpart_t_CPPFLAGS = $(test_cppflags)
+test_mpart_t_LDADD = $(test_ldadd)
+test_mpart_t_LDFLAGS = $(test_ldflags)
+
+test_cert_t_SOURCES = test/cert.c
+test_cert_t_CPPFLAGS = $(test_cppflags)
+test_cert_t_LDADD = $(test_ldadd)
+test_cert_t_LDFLAGS = $(test_ldflags)
diff --git a/src/common/libzmqutil/cert.c b/src/common/libzmqutil/cert.c
new file mode 100644
index 000000000000..fd6cb7e7c30f
--- /dev/null
+++ b/src/common/libzmqutil/cert.c
@@ -0,0 +1,448 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* cert.c - manage zeromq curve certificates
+ *
+ * Note that ZeroMQ curve certs generated by CZMQ use the ZPL config
+ * format.  Flux no longer uses CZMQ but retains compatibility with certs
+ * generated by earlier Flux versions that did.  Although Flux certs remain
+ * in ZPL form, this cert parser is far more restrictive, and therefore
+ * simpler, than the generic ZPL parser in CZMQ.
+ */
+
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <zmq.h>
+#include <string.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/strstrip.h"
+#ifndef HAVE_STRLCPY
+#include "src/common/libmissing/strlcpy.h"
+#endif
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
+#include "sockopt.h"
+#include "cert.h"
+
+/* CURVEZMQ public and private long-term keys are always exactly 32 octets.
+ * See 0MQ RFC 26/CURVEZMQ (https://rfc.zeromq.org/spec/26/)
+ */
+#define KEYSIZE 32
+#define TXTSIZE 41 // Z85-encoded key size + \0
+
+struct cert {
+    uint8_t public_key[KEYSIZE];
+    uint8_t secret_key[KEYSIZE];
+    char public_txt[TXTSIZE];
+    char secret_txt[TXTSIZE];
+    zhash_t *metadata;
+    bool secret_valid;
+    bool public_valid;
+};
+
+void cert_destroy (struct cert *cert)
+{
+    if (cert) {
+        int saved_errno = errno;
+        zhash_destroy (&cert->metadata);
+        free (cert);
+        errno = saved_errno;
+    }
+}
+
+static struct cert *cert_create_empty (void)
+{
+    struct cert *cert;
+
+    if (!(cert = calloc (1, sizeof (*cert))))
+        return NULL;
+    if (!(cert->metadata = zhash_new ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    zhash_autofree (cert->metadata);
+    return cert;
+error:
+    cert_destroy (cert);
+    return NULL;
+}
+
+struct cert *cert_create (void)
+{
+    struct cert *cert;
+
+    if (!(cert = cert_create_empty ()))
+        goto error;
+    if (zmq_curve_keypair (cert->public_txt, cert->secret_txt) < 0)
+        goto error;
+    if (!zmq_z85_decode (cert->public_key, cert->public_txt)
+        || !zmq_z85_decode (cert->secret_key, cert->secret_txt)) {
+        errno = EINVAL;
+        goto error;
+    }
+    cert->public_valid = true;
+    cert->secret_valid = true;
+    return cert;
+error:
+    cert_destroy (cert);
+    return NULL;
+}
+
+static int copy_z85_key (char *dst, const char *src, int size)
+{
+    const char *xtra = ".-:+=^!/*?&<>()[]{}@%$#";
+    const char *cp = src;
+
+    if (!cp || strlen (cp) != size - 1)
+        return -1;
+    while (*cp) {
+        if (!isalnum (*cp) && !strchr (xtra, *cp))
+            return -1;
+        cp++;
+    }
+    (void)strlcpy (dst, src, size);
+    return 0;
+}
+
+// assumes both keys are valid
+static bool valid_keypair (struct cert *cert)
+{
+#if (ZMQ_VERSION >= ZMQ_MAKE_VERSION(4,2,1))
+    char pub[TXTSIZE];
+    if (zmq_curve_public (pub, cert->secret_txt) < 0
+        || !streq (pub, cert->public_txt))
+        return false;
+#endif
+    return true;
+}
+
+struct cert *cert_create_from (const char *public_txt, const char *secret_txt)
+{
+    struct cert *cert;
+
+    if (!(cert = cert_create_empty ()))
+        return NULL;
+    if (public_txt) {
+        if (copy_z85_key (cert->public_txt, public_txt, TXTSIZE) < 0
+            || !zmq_z85_decode (cert->public_key, cert->public_txt))
+            goto inval;
+        cert->public_valid = true;
+    }
+    if (secret_txt) {
+        if (copy_z85_key (cert->secret_txt, secret_txt, TXTSIZE) < 0
+            || !zmq_z85_decode (cert->secret_key, cert->secret_txt))
+            goto inval;
+        cert->secret_valid = true;
+    }
+    if (cert->public_valid && cert->secret_valid) {
+        if (!valid_keypair (cert))
+            goto inval;
+    }
+    return cert;
+inval:
+    errno = EINVAL;
+    cert_destroy (cert);
+    return NULL;
+}
+
+int cert_meta_set (struct cert *cert, const char *key, const char *val)
+{
+    if (!cert || !key || strlen (key) == 0 || !val) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (zhash_insert (cert->metadata, key, (void *)val) < 0) {
+        errno = EEXIST;
+        return -1;
+    }
+    return 0;
+}
+
+const char *cert_meta_get (struct cert *cert, const char *name)
+{
+    if (!cert || !name)
+        return NULL;
+    return zhash_lookup (cert->metadata, name);
+}
+
+static bool meta_equal (zhash_t *meta1, zhash_t *meta2)
+{
+    char *val1;
+    char *val2;
+
+    if (zhash_size (meta1) != zhash_size (meta2))
+        return false;
+    val1 = zhash_first (meta1);
+    while (val1) {
+        val2 = zhash_lookup (meta2, zhash_cursor (meta1));
+        if (!val2 || !streq (val1, val2))
+            return false;
+        val1 = zhash_next (meta1);
+    }
+    return true;
+}
+
+const char *cert_public_txt (struct cert *cert)
+{
+    return (cert && cert->public_valid) ? cert->public_txt : NULL;
+}
+
+const char *cert_secret_txt (struct cert *cert)
+{
+    return (cert && cert->secret_valid) ? cert->secret_txt : NULL;
+}
+
+bool cert_equal (struct cert *c1, struct cert *c2)
+{
+    if (!c1
+        || !c2
+        || c1->public_valid != c2->public_valid
+        || c1->secret_valid != c2->secret_valid
+        || !meta_equal (c1->metadata, c2->metadata))
+        return false;
+    if (c1->public_valid) {
+        if (memcmp (c1->public_key, c2->public_key, KEYSIZE) != 0)
+            return false;
+    }
+    if (c1->secret_valid) {
+        if (memcmp (c1->secret_key, c2->secret_key, KEYSIZE) != 0)
+            return false;
+    }
+    return true;
+}
+
+// File format is compatible with CZMQ zcert, which uses ZPL (0MQ RFC 4).
+int cert_write (struct cert *cert, FILE *f)
+{
+    const char *entry;
+
+    if (!cert || !cert->public_valid || !cert->secret_valid || !f) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (fprintf (f,
+                 "#  ZeroMQ CURVE **Secret** Certificate\n"
+                 "#  DO NOT DISTRIBUTE\n"
+                 "\n"
+                 "metadata\n") < 0)
+        return -1;
+    entry = zhash_first (cert->metadata);
+    while (entry) {
+        if (fprintf (f,
+                     "    %s = \"%s\"\n",
+                     zhash_cursor (cert->metadata),
+                     entry) < 0)
+            return -1;
+        entry = zhash_next (cert->metadata);
+    }
+    if (fprintf (f,
+                 "curve\n"
+                 "    public-key = \"%s\"\n"
+                 "    secret-key = \"%s\"\n",
+                 cert->public_txt,
+                 cert->secret_txt) < 0)
+        return -1;
+    return 0;
+}
+
+// "val" => val (modifies s)
+static char *parse_quoted (char *s)
+{
+    size_t len = strlen (s);
+    if (len < 2)
+        return NULL;
+    if (s[0] != '"' || s[len - 1] != '"')
+        return NULL;
+    s[len - 1] = '\0';
+    memmove (&s[0], &s[1], len - 1);
+    return s;
+}
+
+static bool valid_key (const char *s)
+{
+    const char *xtra = "-.";
+    const char *cp = s;
+
+    if (!cp || strlen (cp) == 0)
+        return false;
+    while (*cp) {
+        if (!isalnum (*cp) && !strchr (xtra, *cp))
+            return false;
+        cp++;
+    }
+    return true;
+}
+
+static bool valid_value (const char *s)
+{
+    const char *xtra = ".-:+=^!/*?&<>()[]{}@%$# \t";
+    const char *cp = s;
+
+    if (!cp)
+        return false;
+    while (*cp) {
+        if (!isalnum (*cp) && !strchr (xtra, *cp))
+            return false;
+        cp++;
+    }
+    return true;
+}
+
+// parse key = "val" (modifies 's')
+static int parse_keyval (char *s, const char **keyp, const char **valp)
+{
+    char *key = s;
+    char *val;
+
+    if (!(val = strchr (s, '=')))
+        goto error;
+    *val++ = '\0';
+    key = strstrip (key);
+    if (!valid_key (key))
+        goto error;
+    val = parse_quoted (strstrip (val));
+    if (!valid_value (val)) {
+        goto error;
+    }
+    *keyp = key;
+    *valp = val;
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+// modifies 's'
+static int parse_metadata (struct cert *cert, char *s)
+{
+    const char *key;
+    const char *val;
+
+    if (parse_keyval (s, &key, &val) < 0)
+        return -1;
+    if (zhash_insert (cert->metadata, key, (void *)val) < 0) {
+        errno = EEXIST;
+        return -1;
+    }
+    return 0;
+}
+
+// modifies 's'
+static int parse_curve (struct cert *cert, char *s)
+{
+    const char *key;
+    const char *val;
+
+    if (parse_keyval (s, &key, &val) < 0)
+        return -1;
+    if (streq (key, "public-key")) {
+        if (copy_z85_key (cert->public_txt, val, TXTSIZE) < 0
+            || !zmq_z85_decode (cert->public_key, cert->public_txt))
+            goto error;
+        cert->public_valid = true;
+    }
+    else if (streq (key, "secret-key")) {
+        if (copy_z85_key (cert->secret_txt, val, TXTSIZE) < 0
+            || !zmq_z85_decode (cert->secret_key, cert->secret_txt))
+            goto error;
+        cert->secret_valid = true;
+    }
+    else
+        goto error;
+    return 0;
+error:
+    errno = EINVAL;
+    return -1;
+}
+
+// Parse the certificate as we've been previously generating them in Flux
+// with CZMQ zcert_t.  This is obviously not a general ZPL parser.
+struct cert *cert_read (FILE *f)
+{
+    struct cert *cert;
+    char buf[256];
+    const char *section = "none";
+    char *line;
+
+    if (!f) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(cert = cert_create_empty ()))
+        return NULL;
+    while ((line = fgets (buf, sizeof (buf), f))) {
+        if (strlen (line) == 0)
+            continue;
+        bool indent = (line[0] == ' ' || line[0] == '\t' ) ? true : false;
+        /* Discard comments and skip blank lines.
+         * Strip leading and trailing white space.
+         */
+        char *stripped = strstrip (line);
+        if (strlen (stripped) == 0 || stripped[0] == '#')
+            continue;
+
+        /* There are two possible sections: metadata and curve.
+         * Everything else is a key = "value" pair.
+         */
+        if (!indent && streq (stripped, "metadata"))
+            section = "metadata";
+        else if (!indent && streq (stripped, "curve")) {
+            section = "curve";
+        }
+        else if (indent && streq (section, "metadata")) {
+            if (parse_metadata (cert, stripped) < 0)
+                goto error;
+        }
+        else if (indent && streq (section, "curve")) {
+            if (parse_curve (cert, stripped) < 0)
+                goto error;
+        }
+        else
+            goto inval;
+    }
+    if (ferror (f))
+        goto error;
+    if (!cert->public_valid || !cert->secret_valid
+        || !valid_keypair (cert))
+        goto inval;
+    return cert;
+inval:
+    errno = EINVAL;
+error:
+    cert_destroy (cert);
+    return NULL;
+}
+
+int cert_apply (struct cert *cert, void *sock)
+{
+    if (!cert || !cert->public_valid || !cert->secret_valid) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (zmq_setsockopt (sock,
+                        ZMQ_CURVE_PUBLICKEY,
+                        cert->public_key,
+                        KEYSIZE) < 0
+        || zmq_setsockopt (sock,
+                           ZMQ_CURVE_SECRETKEY,
+                           cert->secret_key,
+                           KEYSIZE) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libzmqutil/cert.h b/src/common/libzmqutil/cert.h
new file mode 100644
index 000000000000..e67c8614932d
--- /dev/null
+++ b/src/common/libzmqutil/cert.h
@@ -0,0 +1,36 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _ZMQUTIL_CERT_H
+#define _ZMQUTIL_CERT_H
+
+#include <stdio.h>
+#include <stdbool.h>
+
+struct cert *cert_create (void);
+struct cert *cert_create_from (const char *public_txt, const char *secret_txt);
+void cert_destroy (struct cert *cert);
+
+int cert_write (struct cert *cert, FILE *f);
+struct cert *cert_read (FILE *f);
+
+bool cert_equal (struct cert *cert1, struct cert *cert2);
+
+const char *cert_meta_get (struct cert *cert, const char *name);
+int cert_meta_set (struct cert *cert, const char *key, const char *val);
+
+const char *cert_public_txt (struct cert *cert);
+const char *cert_secret_txt (struct cert *cert);
+
+int cert_apply (struct cert *cert, void *sock);
+
+#endif /* !_ZMQUTIL_CERT_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libutil/ev_zmq.c b/src/common/libzmqutil/ev_zmq.c
similarity index 84%
rename from src/common/libutil/ev_zmq.c
rename to src/common/libzmqutil/ev_zmq.c
index 055ab7af675e..e8e1c768d3ca 100644
--- a/src/common/libutil/ev_zmq.c
+++ b/src/common/libzmqutil/ev_zmq.c
@@ -28,23 +28,27 @@
  *   on the next mailbox event
  */
 
-#include <czmq.h>
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <zmq.h>
 #include <stdbool.h>
 
 #include "src/common/libev/ev.h"
-#include "src/common/libutil/ev_zmq.h"
+#include "ev_zmq.h"
 
 static void prepare_cb (struct ev_loop *loop, ev_prepare *w, int revents)
 {
     ev_zmq *zw = (ev_zmq *)((char *)w - offsetof (ev_zmq, prepare_w));
     uint32_t zevents = 0;
     size_t zevents_size = sizeof (zevents);
-    void *handle = zsock_resolve (zw->zsock);
 
-    if (handle == NULL)
+    if (zw->zsock == NULL)
         ev_idle_start (loop, &zw->idle_w);
-    else if (zmq_getsockopt (handle, ZMQ_EVENTS, &zevents, &zevents_size) < 0)
+    else if (zmq_getsockopt (zw->zsock,
+                             ZMQ_EVENTS,
+                             &zevents,
+                             &zevents_size) < 0)
         ev_idle_start (loop, &zw->idle_w);
     else if ((revents = ztoe (zevents) & zw->events))
         ev_idle_start (loop, &zw->idle_w);
@@ -57,17 +61,19 @@ static void check_cb (struct ev_loop *loop, ev_check *w, int revents)
     ev_zmq *zw = (ev_zmq *)((char *)w - offsetof (ev_zmq, check_w));
     uint32_t zevents = 0;
     size_t zevents_size = sizeof (zevents);
-    void *handle = zsock_resolve (zw->zsock);
 
     ev_io_stop (loop, &zw->io_w);
     ev_idle_stop (loop, &zw->idle_w);
 
-    if (handle == NULL)
+    if (zw->zsock == NULL)
         zw->cb (loop, zw, EV_ERROR);
     else if (ev_is_pending (&zw->io_w)
             && ev_clear_pending (loop, &zw->io_w) & EV_ERROR)
         zw->cb (loop, zw, EV_ERROR);
-    else if (zmq_getsockopt (handle, ZMQ_EVENTS, &zevents, &zevents_size) < 0)
+    else if (zmq_getsockopt (zw->zsock,
+                             ZMQ_EVENTS,
+                             &zevents,
+                             &zevents_size) < 0)
         zw->cb (loop, zw, EV_ERROR);
     else if ((revents = ztoe (zevents) & zw->events))
         zw->cb (loop, zw, revents);
@@ -79,11 +85,10 @@ int ev_zmq_init (ev_zmq *w, ev_zmq_cb cb, void *zsock, int events)
     w->zsock = zsock;
     w->events = events;
     size_t fd_size = sizeof (w->fd);
-    void *handle = zsock_resolve (zsock);
 
-    if (handle == NULL)
+    if (zsock == NULL)
         return -1;
-    if (zmq_getsockopt (handle, ZMQ_FD, &w->fd, &fd_size) < 0)
+    if (zmq_getsockopt (zsock, ZMQ_FD, &w->fd, &fd_size) < 0)
         return -1;
 
     ev_prepare_init (&w->prepare_w, prepare_cb);
@@ -108,6 +113,11 @@ void ev_zmq_stop (struct ev_loop *loop, ev_zmq *w)
     ev_idle_stop (loop, &w->idle_w);
 }
 
+bool ev_zmq_is_active (ev_zmq *w)
+{
+    return ev_is_active (&w->prepare_w);
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/common/libutil/ev_zmq.h b/src/common/libzmqutil/ev_zmq.h
similarity index 95%
rename from src/common/libutil/ev_zmq.h
rename to src/common/libzmqutil/ev_zmq.h
index 0f288739afe1..f5a2a7b2c7d7 100644
--- a/src/common/libutil/ev_zmq.h
+++ b/src/common/libzmqutil/ev_zmq.h
@@ -11,6 +11,9 @@
 #ifndef _EV_ZMQ_H
 #define _EV_ZMQ_H
 
+#include <stdbool.h>
+#include "src/common/libev/ev.h"
+
 typedef struct ev_zmq_struct ev_zmq;
 typedef void (*ev_zmq_cb)(struct ev_loop *loop, ev_zmq *w, int revents);
 
@@ -29,6 +32,7 @@ struct ev_zmq_struct {
 int ev_zmq_init (ev_zmq *w, ev_zmq_cb cb, void *zsock, int events);
 void ev_zmq_start (struct ev_loop *loop, ev_zmq *w);
 void ev_zmq_stop (struct ev_loop *loop, ev_zmq *w);
+bool ev_zmq_is_active (ev_zmq *w);
 
 /* Convert zeromq poll bits to libev's, for construction of 'events'
  * when registering a watcher.
diff --git a/src/common/libzmqutil/monitor.c b/src/common/libzmqutil/monitor.c
new file mode 100644
index 000000000000..812daf1c048c
--- /dev/null
+++ b/src/common/libzmqutil/monitor.c
@@ -0,0 +1,336 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <zmq.h>
+#include <stdint.h>
+#include <uuid.h>
+
+#include "ccan/array_size/array_size.h"
+
+#include "reactor.h"
+#include "sockopt.h"
+#include "monitor.h"
+
+#ifndef UUID_STR_LEN
+#define UUID_STR_LEN 37     // defined in later libuuid headers
+#endif
+
+#if (ZMQ_VERSION > ZMQ_MAKE_VERSION (4, 1, 4))
+/* N.B. 4.1.4 has a bad bug
+ */
+struct zmqutil_monitor {
+    void *sock;
+    char endpoint[UUID_STR_LEN + 64];
+    flux_watcher_t *w;
+    zmqutil_monitor_f fun;
+    void *arg;
+    bool stopped;
+};
+
+static struct {
+    uint16_t event;
+    enum {
+        VAL_NONE,
+        VAL_ERRNO,
+        VAL_PROTO,
+        VAL_ZAPNUM,
+    } valtype;
+    const char *desc;
+} nametab[] = {
+    { ZMQ_EVENT_CONNECTED, VAL_NONE,"connected" },
+    { ZMQ_EVENT_CONNECT_DELAYED, VAL_NONE, "connect delayed" },
+    { ZMQ_EVENT_CONNECT_RETRIED, VAL_NONE, "connect retried" },
+    { ZMQ_EVENT_LISTENING, VAL_NONE, "listening" },
+    { ZMQ_EVENT_BIND_FAILED, VAL_ERRNO, "bind failed" },
+    { ZMQ_EVENT_ACCEPTED, VAL_NONE, "accepted" },
+    { ZMQ_EVENT_ACCEPT_FAILED, VAL_ERRNO, "accept failed" },
+    { ZMQ_EVENT_CLOSED, VAL_NONE, "closed" },
+    { ZMQ_EVENT_CLOSE_FAILED, VAL_ERRNO, "close failed" },
+    { ZMQ_EVENT_DISCONNECTED, VAL_NONE, "disconnected" },
+    { ZMQ_EVENT_MONITOR_STOPPED, VAL_NONE, "monitor stopped" },
+#ifdef ZMQ_EVENT_HANDSHAKE_FAILED_NO_DETAIL
+    { ZMQ_EVENT_HANDSHAKE_FAILED_NO_DETAIL, VAL_ERRNO, "handshake failed" },
+#endif
+#ifdef ZMQ_EVENT_HANDSHAKE_SUCCEEDED
+    { ZMQ_EVENT_HANDSHAKE_SUCCEEDED, VAL_NONE, "handshake succeeded" },
+#endif
+#ifdef ZMQ_EVENT_HANDSHAKE_FAILED_PROTOCOL
+    { ZMQ_EVENT_HANDSHAKE_FAILED_PROTOCOL, VAL_PROTO,
+      "handshake failed protocol" },
+#endif
+#ifdef ZMQ_EVENT_HANDSHAKE_FAILED_AUTH
+    { ZMQ_EVENT_HANDSHAKE_FAILED_AUTH, VAL_ZAPNUM,
+      "handshake failed auth" },
+#endif
+};
+
+static struct {
+    uint32_t value;
+    const char *desc;
+} prototab[] = {
+#ifdef ZMQ_EVENT_HANDSHAKE_FAILED_PROTOCOL
+    { ZMQ_PROTOCOL_ERROR_ZMTP_UNSPECIFIED, "ZMTP unspecified" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_UNEXPECTED_COMMAND, "ZMTP unexpected command" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_INVALID_SEQUENCE, "ZMTP invalid sequence" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_KEY_EXCHANGE, "ZMTP key exchange" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_UNSPECIFIED,
+      "ZMTP malformed command unspecified" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_MESSAGE,
+      "ZMTP malformed command message" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_HELLO,
+      "ZMTP malformed command hello" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_INITIATE,
+      "ZMTP malformed command initiate" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_ERROR,
+      "ZMTP malformed command error" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_READY,
+      "ZMTP malformed command ready" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_WELCOME,
+      "ZMTP malformed command welcome" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_INVALID_METADATA, "ZMTP invalid metadata" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_CRYPTOGRAPHIC, "ZMTP cryptographic" },
+    { ZMQ_PROTOCOL_ERROR_ZMTP_MECHANISM_MISMATCH, "ZMTP mechanism mismatch" },
+    { ZMQ_PROTOCOL_ERROR_ZAP_UNSPECIFIED, "ZAP unspecified" },
+    { ZMQ_PROTOCOL_ERROR_ZAP_MALFORMED_REPLY, "ZAP malformed reply" },
+    { ZMQ_PROTOCOL_ERROR_ZAP_BAD_REQUEST_ID, "ZAP bad request id" },
+    { ZMQ_PROTOCOL_ERROR_ZAP_BAD_VERSION, "ZAP bad version" },
+    { ZMQ_PROTOCOL_ERROR_ZAP_INVALID_STATUS_CODE, "ZAP invalid status code" },
+    { ZMQ_PROTOCOL_ERROR_ZAP_INVALID_METADATA, "ZAP invalid metadata" },
+#endif
+};
+
+static const char *eventstr (struct monitor_event *mevent)
+{
+    int i;
+    for (i = 0; i < ARRAY_SIZE (nametab); i++) {
+        if (mevent->event == nametab[i].event)
+            return nametab[i].desc;
+    }
+    return "unknown socket event";
+}
+
+static const char *valuestr (struct monitor_event *mevent)
+{
+    int i;
+    int valtype = VAL_NONE;
+    static char buf[128];
+
+    /* look up valtype for event */
+    for (i = 0; i < ARRAY_SIZE (nametab); i++) {
+        if (mevent->event == nametab[i].event)
+            valtype = nametab[i].valtype;
+    }
+    /* decode value depending on valtype */
+    if (valtype == VAL_ERRNO)
+        return strerror (mevent->value);
+    if (valtype == VAL_ZAPNUM) {
+        snprintf (buf, sizeof (buf), "ZAP status code %d", mevent->value);
+        return buf;
+    }
+    if (valtype == VAL_PROTO) {
+        for (i = 0; i < ARRAY_SIZE (prototab); i++) {
+            if (mevent->value == prototab[i].value)
+                return prototab[i].desc;
+        }
+        snprintf (buf, sizeof (buf), "unknown protocol error %d", mevent->value);
+        return buf;
+    }
+    return "";
+}
+
+bool zmqutil_monitor_iserror (struct monitor_event *mevent)
+{
+    if (mevent) {
+        switch (mevent->event) {
+            case ZMQ_EVENT_ACCEPT_FAILED:
+            case ZMQ_EVENT_CLOSE_FAILED:
+#ifdef ZMQ_EVENT_HANDSHAKE_FAILED_NO_DETAIL
+            case ZMQ_EVENT_HANDSHAKE_FAILED_NO_DETAIL:
+#endif
+#ifdef ZMQ_EVENT_HANDSHAKE_FAILED_PROTOCOL
+            case ZMQ_EVENT_HANDSHAKE_FAILED_PROTOCOL:
+#endif
+#ifdef ZMQ_EVENT_HANDSHAKE_FAILED_AUTH
+            case ZMQ_EVENT_HANDSHAKE_FAILED_AUTH:
+#endif
+                return true;
+        }
+    }
+    return false;
+}
+
+static int recv_frame1 (void *sock, uint16_t *event, uint32_t *value)
+{
+    uint8_t buf[6];
+    if (zmq_recv (sock, buf, sizeof (buf), 0) != 6)
+        return -1;
+    *event = *(uint16_t *)buf;
+    *value = *(uint32_t *)(buf + 2);
+    return 0;
+}
+
+static int recv_frame2 (void *sock, char *buf, int size)
+{
+    int more;
+    if (zgetsockopt_int (sock, ZMQ_RCVMORE, &more) < 0 || !more)
+        return -1;
+    memset (buf, 0, size);
+    if (zmq_recv (sock, buf, size, 0) < 0)
+        return -1;
+    return 0;
+}
+
+int zmqutil_monitor_get (struct zmqutil_monitor *mon,
+                         struct monitor_event *mevent)
+{
+    if (!mon || !mevent) {
+        errno = EINVAL;
+        return -1;
+    }
+    /* receive event+value frame */
+    if (recv_frame1 (mon->sock, &mevent->event, &mevent->value) < 0)
+        return -1;
+
+    /* receive endpoint frame */
+    if (recv_frame2 (mon->sock,
+                     mevent->endpoint,
+                     sizeof (mevent->endpoint)) < 0)
+        return -1;
+
+    /* note end of monitor stream for zmqutil_monitor_destroy() */
+    if (mevent->event == ZMQ_EVENT_MONITOR_STOPPED)
+        mon->stopped = true;
+
+    /* decode event, value */
+    mevent->event_str = eventstr (mevent);
+    mevent->value_str = valuestr (mevent);
+    return 0;
+}
+
+static void monitor_callback (flux_reactor_t *r,
+                              flux_watcher_t *w,
+                              int revents,
+                              void *arg)
+{
+    struct zmqutil_monitor *mon = arg;
+
+    if (mon->fun)
+        mon->fun (mon, mon->arg);
+}
+
+/* Read messages from the monitor socket until the final MONITOR_STOPPED
+ * message is read.  This presumes that the socket being monitored is
+ * closed before zmqutil_monitor_destroy() is called.  If the monitor socket
+ * is destroyed before consuming this event, it may cause the 0MQ I/O thread
+ * to block when the PAIR socket it is writing to enters the mute state.
+ */
+static void monitor_purge (struct zmqutil_monitor *mon)
+{
+    struct monitor_event event;
+
+    while (!mon->stopped) {
+        if (zmqutil_monitor_get (mon, &event) < 0)
+            break;
+    }
+}
+
+void zmqutil_monitor_destroy (struct zmqutil_monitor *mon)
+{
+    if (mon) {
+        int saved_errno = errno;
+        flux_watcher_destroy (mon->w);
+        if (mon->sock) {
+            monitor_purge (mon);
+            (void)zmq_disconnect (mon->sock, mon->endpoint);
+            (void)zmq_close (mon->sock);
+        }
+        free (mon);
+        errno = saved_errno;
+    }
+}
+
+struct zmqutil_monitor *zmqutil_monitor_create (void *zctx,
+                                                void *sock,
+                                                flux_reactor_t *r,
+                                                zmqutil_monitor_f fun,
+                                                void *arg)
+{
+    struct zmqutil_monitor *mon;
+    uuid_t uuid;
+    char uuid_str[UUID_STR_LEN];
+
+    if (!zctx || !sock || !r) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(mon = calloc (1, sizeof (*mon))))
+        return NULL;
+    mon->fun = fun;
+    mon->arg = arg;
+
+    /* Generate a unique inproc endpoint for monitoring this socket.
+     */
+    uuid_generate (uuid);
+    uuid_unparse (uuid, uuid_str);
+    snprintf (mon->endpoint, sizeof (mon->endpoint), "inproc://%s", uuid_str);
+
+    /* Arrange for local callback to run on each monitor event.
+     * It will call the user's callback.
+     */
+    if (zmq_socket_monitor (sock,
+                            mon->endpoint,
+                            ZMQ_EVENT_ALL) < 0
+        || !(mon->sock = zmq_socket (zctx, ZMQ_PAIR))
+        || zmq_connect (mon->sock, mon->endpoint) < 0
+        || !(mon->w = zmqutil_watcher_create (r,
+                                              mon->sock,
+                                              FLUX_POLLIN,
+                                              monitor_callback,
+                                              mon)))
+        goto error;
+    if (zsetsockopt_int (mon->sock, ZMQ_LINGER, 0) < 0
+        || zsetsockopt_int (mon->sock, ZMQ_RCVHWM, 0) < 0
+        || zsetsockopt_int (mon->sock, ZMQ_SNDHWM, 0) < 0)
+        goto error;
+    flux_watcher_start (mon->w);
+    return mon;
+error:
+    zmqutil_monitor_destroy (mon);
+    return NULL;
+}
+#else
+/* Monitoring is disabled due to libzmq being too old.
+ */
+struct zmqutil_monitor *zmqutil_monitor_create (void *zctx,
+                                                void *sock,
+                                                flux_reactor_t *r,
+                                                zmqutil_monitor_f fun,
+                                                void *arg)
+{
+    return NULL;
+}
+void zmqutil_monitor_destroy (struct zmqutil_monitor *mon)
+{
+}
+int zmqutil_monitor_get (struct zmqutil_monitor *mon,
+                         struct monitor_event *mevent)
+{
+    return -1;
+}
+bool zmqutil_monitor_iserror (struct monitor_event *mevent)
+{
+    return false;
+}
+#endif
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libzmqutil/monitor.h b/src/common/libzmqutil/monitor.h
new file mode 100644
index 000000000000..6fcae243ff8f
--- /dev/null
+++ b/src/common/libzmqutil/monitor.h
@@ -0,0 +1,52 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _ZMQUTIL_MONITOR_H
+#define _ZMQUTIL_MONITOR_H
+
+#include <flux/core.h>
+
+struct monitor_event {
+    uint16_t event;
+    uint32_t value;
+    char endpoint[256];
+    const char *event_str;
+    const char *value_str;
+};
+
+struct zmqutil_monitor;
+
+typedef void (*zmqutil_monitor_f)(struct zmqutil_monitor *mon, void *arg);
+
+/* Arrange for 'fun' to be called each time there is an event on 'sock'.
+ * Create must be called before connect/bind, and destroy must be called
+ * after close/destroy.
+ * N.B. this will fail if an old/buggy version of libzmq is used.
+ */
+struct zmqutil_monitor *zmqutil_monitor_create (void *zctx,
+                                                void *sock,
+                                                flux_reactor_t *r,
+                                                zmqutil_monitor_f fun,
+                                                void *arg);
+void zmqutil_monitor_destroy (struct zmqutil_monitor *mon);
+
+/* Receive an event from the monitor socket.
+ * This should be called once each time the the monitor callback is invoked.
+ */
+int zmqutil_monitor_get (struct zmqutil_monitor *mon,
+                         struct monitor_event *mevent);
+
+/* Returns true if the socket event likely should be logged at error severity.
+ */
+bool zmqutil_monitor_iserror (struct monitor_event *mevent);
+
+#endif // !_ZMQUTIL_MONITOR
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libzmqutil/mpart.c b/src/common/libzmqutil/mpart.c
new file mode 100644
index 000000000000..a6ff76e5dcf1
--- /dev/null
+++ b/src/common/libzmqutil/mpart.c
@@ -0,0 +1,201 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <zmq.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "sockopt.h"
+#include "mpart.h"
+
+static void part_destroy (zmq_msg_t *part)
+{
+    if (part) {
+        int saved_errno = errno;
+        zmq_msg_close (part);
+        free (part);
+        errno = saved_errno;
+    }
+}
+
+static zmq_msg_t *part_create (const void *data, size_t size)
+{
+    zmq_msg_t *part;
+
+    if (!(part = calloc (1, sizeof (*part))))
+        return NULL;
+    if (size > 0) {
+        if (zmq_msg_init_size (part, size) < 0)
+            goto error;
+        if (data)
+            memcpy (zmq_msg_data (part), data, size);
+    }
+    else
+        (void)zmq_msg_init (part); // documented as always returning 0
+    return part;
+error:
+    part_destroy (part);
+    return NULL;
+}
+
+static zmq_msg_t *part_recv (void *sock)
+{
+    zmq_msg_t *part;
+
+    if (!(part = part_create (NULL, 0)))
+        return NULL;
+    if (zmq_msg_recv (part, sock, 0) < 0) {
+        part_destroy (part);
+        return NULL;
+    }
+    return part;
+}
+
+static bool part_streq (zmq_msg_t *part, const char *s)
+{
+    if (part
+        && s
+        && zmq_msg_size (part) == strlen (s)
+        && memcmp (zmq_msg_data (part), s, zmq_msg_size (part)) == 0)
+        return true;
+    return false;
+}
+
+void mpart_destroy (zlist_t *mpart)
+{
+    if (mpart) {
+        int saved_errno = errno;
+        zlist_destroy (&mpart);
+        errno = saved_errno;
+    }
+}
+
+zlist_t *mpart_create (void)
+{
+    zlist_t *mpart;
+
+    if (!(mpart = zlist_new ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    return mpart;
+}
+
+static int mpart_append (zlist_t *mpart, zmq_msg_t *part)
+{
+    if (zlist_append (mpart, part) < 0) {
+        errno = ENOMEM;
+        return -1;
+    }
+    zlist_freefn (mpart, part, (zlist_free_fn *)part_destroy, true);
+    return 0;
+}
+
+int mpart_addmem (zlist_t *mpart, const void *buf, size_t size)
+{
+    zmq_msg_t *part;
+
+    if (!mpart) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(part = part_create (buf, size)))
+        return -1;
+    if (mpart_append (mpart, part) < 0) {
+        part_destroy (part);
+        return -1;
+    }
+    return 0;
+}
+
+int mpart_addstr (zlist_t *mpart, const char *s)
+{
+    if (!mpart || !s) {
+        errno = EINVAL;
+        return -1;
+    }
+    return mpart_addmem (mpart, s, strlen (s));
+}
+
+zlist_t *mpart_recv (void *sock)
+{
+    zlist_t *mpart;
+    int more;
+
+    if (!(mpart = mpart_create ()))
+        return NULL;
+    do {
+        zmq_msg_t *part;
+        if (!(part = part_recv (sock)))
+            goto error;
+        if (mpart_append (mpart, part) < 0) {
+            part_destroy (part);
+            goto error;
+        }
+        if (zgetsockopt_int (sock, ZMQ_RCVMORE, &more) < 0)
+            goto error;
+    } while (more);
+    return mpart;
+error:
+    mpart_destroy (mpart);
+    return NULL;
+}
+
+int mpart_send (void *sock, zlist_t *mpart)
+{
+    if (!mpart) {
+        errno = EINVAL;
+        return -1;
+    }
+    int count = 0;
+    int parts = zlist_size (mpart);
+    zmq_msg_t *part = zlist_first (mpart);
+    while (part) {
+        int flags = 0;
+        if (++count < parts)
+            flags |= ZMQ_SNDMORE;
+        if (zmq_msg_send (part, sock, flags) < 0)
+            return -1;
+        part = zlist_next (mpart);
+    }
+    return 0;
+}
+
+zmq_msg_t *mpart_get (zlist_t *mpart, int index)
+{
+    if (mpart) {
+        zmq_msg_t *part;
+        int count = 0;
+
+        part = zlist_first (mpart);
+        while (part) {
+            if (count++ == index)
+                return part;
+            part = zlist_next (mpart);
+        }
+    }
+    return NULL;
+}
+
+bool mpart_streq (zlist_t *mpart, int index, const char *s)
+{
+    if (mpart && s) {
+        zmq_msg_t *part = mpart_get (mpart, index);
+        if (part && part_streq (part, s))
+            return true;
+    }
+    return false;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libzmqutil/mpart.h b/src/common/libzmqutil/mpart.h
new file mode 100644
index 000000000000..4adc1e505b38
--- /dev/null
+++ b/src/common/libzmqutil/mpart.h
@@ -0,0 +1,32 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _ZMQUTIL_MPART_H
+#define _ZMQUTIL_MPART_H
+
+#include <stdbool.h>
+#include <zmq.h>
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+/* helpers for multi-part messages as zlist of zmq_msg_t
+ * (like stripped down zmsg_t)
+ */
+void mpart_destroy (zlist_t *mpart);
+zlist_t *mpart_create (void);
+int mpart_addmem (zlist_t *mpart, const void *buf, size_t size);
+int mpart_addstr (zlist_t *mpart, const char *s);
+zlist_t *mpart_recv (void *sock);
+int mpart_send (void *sock, zlist_t *mpart);
+zmq_msg_t *mpart_get (zlist_t *mpart, int index);
+bool mpart_streq (zlist_t *mpart, int index, const char *s);
+
+#endif // !_ZMQUTIL_MPART_H
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libzmqutil/msg_zsock.c b/src/common/libzmqutil/msg_zsock.c
new file mode 100644
index 000000000000..2637f66d28c3
--- /dev/null
+++ b/src/common/libzmqutil/msg_zsock.c
@@ -0,0 +1,140 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <stdbool.h>
+#include <string.h>
+#include <assert.h>
+#include <inttypes.h>
+#include <zmq.h>
+#include <flux/core.h>
+
+#include "src/common/libflux/message_iovec.h"
+#include "src/common/libflux/message_proto.h"
+#include "src/common/libutil/errno_safe.h"
+
+#include "sockopt.h"
+#include "msg_zsock.h"
+
+int zmqutil_msg_send_ex (void *sock, const flux_msg_t *msg, bool nonblock)
+{
+    int flags = ZMQ_SNDMORE;
+    struct msg_iovec *iov = NULL;
+    int iovcnt;
+    uint8_t proto[PROTO_SIZE];
+    int count = 0;
+    int rc = -1;
+
+    if (!sock || !msg) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (msg_to_iovec (msg, proto, PROTO_SIZE, &iov, &iovcnt) < 0)
+        goto error;
+
+    if (nonblock)
+        flags |= ZMQ_DONTWAIT;
+
+    while (count < iovcnt) {
+        if ((count + 1) == iovcnt)
+            flags &= ~ZMQ_SNDMORE;
+        if (zmq_send (sock,
+                      iov[count].data,
+                      iov[count].size,
+                      flags) < 0)
+            goto error;
+        count++;
+    }
+    rc = 0;
+error:
+    ERRNO_SAFE_WRAP (free, iov);
+    return rc;
+}
+
+int zmqutil_msg_send (void *sock, const flux_msg_t *msg)
+{
+    return zmqutil_msg_send_ex (sock, msg, false);
+}
+
+flux_msg_t *zmqutil_msg_recv (void *sock)
+{
+    struct msg_iovec *iov = NULL;
+    int iovlen = 0;
+    int iovcnt = 0;
+    flux_msg_t *msg;
+    flux_msg_t *rv = NULL;
+
+    if (!sock) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    /* N.B. we need to store a zmq_msg_t for each iovec entry so that
+     * the memory is available during the call to iovec_to_msg().  We
+     * use the msg_iovec's "transport_data" field to store the entry
+     * and then clear/free it later.
+     */
+    while (true) {
+        zmq_msg_t *msgdata;
+        if (iovlen <= iovcnt) {
+            struct msg_iovec *tmp;
+            iovlen += IOVECINCR;
+            if (!(tmp = realloc (iov, sizeof (*iov) * iovlen)))
+                goto error;
+            iov = tmp;
+        }
+        if (!(msgdata = malloc (sizeof (zmq_msg_t))))
+            goto error;
+        zmq_msg_init (msgdata);
+        if (zmq_recvmsg (sock, msgdata, 0) < 0) {
+            int save_errno = errno;
+            zmq_msg_close (msgdata);
+            free (msgdata);
+            errno = save_errno;
+            goto error;
+        }
+        iov[iovcnt].transport_data = msgdata;
+        iov[iovcnt].data = zmq_msg_data (msgdata);
+        iov[iovcnt].size = zmq_msg_size (msgdata);
+        iovcnt++;
+
+        int rcvmore;
+        if (zgetsockopt_int (sock, ZMQ_RCVMORE, &rcvmore) < 0)
+            goto error;
+        if (!rcvmore)
+            break;
+    }
+
+    if (!(msg = iovec_to_msg (iov, iovcnt)))
+        goto error;
+    rv = msg;
+error:
+    if (iov) {
+        int save_errno = errno;
+        int i;
+        for (i = 0; i < iovcnt; i++) {
+            zmq_msg_t *msgdata = iov[i].transport_data;
+            zmq_msg_close (msgdata);
+            free (msgdata);
+        }
+        free (iov);
+        errno = save_errno;
+    }
+    return rv;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libzmqutil/msg_zsock.h b/src/common/libzmqutil/msg_zsock.h
new file mode 100644
index 000000000000..304d236affac
--- /dev/null
+++ b/src/common/libzmqutil/msg_zsock.h
@@ -0,0 +1,43 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _ZMQUTIL_MSG_ZSOCK_H
+#define _ZMQUTIL_MSG_ZSOCK_H
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "src/common/libflux/message.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Send message to zeromq socket.
+ * Returns 0 on success, -1 on failure with errno set.
+ */
+int zmqutil_msg_send (void *dest, const flux_msg_t *msg);
+int zmqutil_msg_send_ex (void *dest, const flux_msg_t *msg, bool nonblock);
+
+/* Receive a message from zeromq socket.
+ * Returns message on success, NULL on failure with errno set.
+ */
+flux_msg_t *zmqutil_msg_recv (void *dest);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_ZMQUTIL_MSG_ZSOCK_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libzmqutil/reactor.c b/src/common/libzmqutil/reactor.c
new file mode 100644
index 000000000000..a0794ce1147a
--- /dev/null
+++ b/src/common/libzmqutil/reactor.c
@@ -0,0 +1,88 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <stdbool.h>
+#include <string.h>
+#include <assert.h>
+#include <zmq.h>
+#include <flux/core.h>
+
+#include "src/common/libev/ev.h"
+#include "src/common/libflux/reactor_private.h"
+
+#include "ev_zmq.h"
+#include "reactor.h"
+
+/* 0MQ sockets
+ */
+
+static void zmq_start (flux_watcher_t *w)
+{
+    ev_zmq_start (w->r->loop, (ev_zmq *)w->data);
+}
+
+static void zmq_stop (flux_watcher_t *w)
+{
+    ev_zmq_stop (w->r->loop, (ev_zmq *)w->data);
+}
+
+static bool zmq_is_active (flux_watcher_t *w)
+{
+    return ev_zmq_is_active (w->data);
+}
+
+static void zmq_cb (struct ev_loop *loop, ev_zmq *pw, int revents)
+{
+    struct flux_watcher *w = pw->data;
+    if (w->fn)
+        w->fn (ev_userdata (loop), w, libev_to_events (revents), w->arg);
+}
+
+static struct flux_watcher_ops zmq_watcher  = {
+    .start = zmq_start,
+    .stop = zmq_stop,
+    .destroy = NULL,
+    .is_active = zmq_is_active,
+};
+
+flux_watcher_t *zmqutil_watcher_create (flux_reactor_t *r,
+                                        void *zsock, int events,
+                                        flux_watcher_f cb, void *arg)
+{
+    ev_zmq *zw;
+    flux_watcher_t *w;
+
+    if (!(w = watcher_create (r, sizeof (*zw), &zmq_watcher, cb, arg)))
+        return NULL;
+    zw = watcher_get_data (w);
+    ev_zmq_init (zw, zmq_cb, zsock, events_to_libev (events) & ~EV_ERROR);
+    zw->data = w;
+
+    return w;
+}
+
+void *zmqutil_watcher_get_zsock (flux_watcher_t *w)
+{
+    if (watcher_get_ops (w) != &zmq_watcher) {
+        errno = EINVAL;
+        return NULL;
+    }
+    ev_zmq *zw = w->data;
+    return zw->zsock;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libzmqutil/reactor.h b/src/common/libzmqutil/reactor.h
new file mode 100644
index 000000000000..86f2874476cb
--- /dev/null
+++ b/src/common/libzmqutil/reactor.h
@@ -0,0 +1,40 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _ZMQUTIL_REACTOR_H
+#define _ZMQUTIL_REACTOR_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <flux/core.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* zmq socket
+ */
+
+flux_watcher_t *zmqutil_watcher_create (flux_reactor_t *r,
+                                        void *zsock, int events,
+                                        flux_watcher_f cb, void *arg);
+
+void *zmqutil_watcher_get_zsock (flux_watcher_t *w);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_ZMQUTIL_REACTOR_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libzmqutil/sockopt.c b/src/common/libzmqutil/sockopt.c
new file mode 100644
index 000000000000..0c665e86cfc7
--- /dev/null
+++ b/src/common/libzmqutil/sockopt.c
@@ -0,0 +1,53 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <zmq.h>
+#include <string.h>
+
+#include "sockopt.h"
+
+int zsetsockopt_int (void *sock, int option_name, int value)
+{
+    return zmq_setsockopt (sock, option_name, &value, sizeof (value));
+}
+
+int zgetsockopt_int (void *sock, int option_name, int *value)
+{
+    int val;
+    size_t size = sizeof (val);
+    if (zmq_getsockopt (sock, option_name, &val, &size) < 0)
+        return -1;
+    *value = val;
+    return 0;
+}
+
+int zgetsockopt_str (void *sock, int option_name, char **value)
+{
+    char val[1024];
+    size_t size = sizeof (val);
+    char *cpy;
+
+    if (zmq_getsockopt (sock, option_name, &val, &size) < 0)
+        return -1;
+    if (!(cpy = strdup (val)))
+        return -1;
+    *value = cpy;
+    return 0;
+}
+
+int zsetsockopt_str (void *sock, int option_name, const char *value)
+{
+    return zmq_setsockopt (sock, option_name, value, strlen (value));
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libzmqutil/sockopt.h b/src/common/libzmqutil/sockopt.h
new file mode 100644
index 000000000000..7d8778dafa17
--- /dev/null
+++ b/src/common/libzmqutil/sockopt.h
@@ -0,0 +1,22 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _ZMQUTIL_SOCKOPT_H
+#define _ZMQUTIL_SOCKOPT_H
+
+int zsetsockopt_int (void *sock, int option_name, int value);
+int zgetsockopt_int (void *sock, int option_name, int *value);
+
+int zsetsockopt_str (void *sock, int option_name, const char *value);
+int zgetsockopt_str (void *sock, int option_name, char **value);
+
+#endif /* !_ZMQUTIL_SOCKOPT_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libzmqutil/test/cert.c b/src/common/libzmqutil/test/cert.c
new file mode 100644
index 000000000000..f8b1d1cfeca5
--- /dev/null
+++ b/src/common/libzmqutil/test/cert.c
@@ -0,0 +1,437 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdbool.h>
+#include <errno.h>
+#include <zmq.h>
+
+#include "tap.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+#include "cert.h"
+
+// valid keypair
+#define PAIR1_PUB "FYFE.@650VuUqRGygAtG.RC$A<cid63q(WEnR+&y"
+#define PAIR1_SEC "^Oto<5V$^d/V}kj6^Q[LRwdgAY5K3gE{gN2+1ox)"
+
+// valid keypair, sec has an embedded # character
+#define PAIR2_PUB "Viyg(M7U*Q6ZI=D6(q?]amgXrQ{[IQnEY{IF5*X)"
+#define PAIR2_SEC "7F)3@>MGW.+)..qc)8R#lfL31*^QX<GXufgpVtbA"
+
+// contains invalid Z85 char (space) in a 5-char chunk
+//   see 0MQ RFC 32/Z85 (https://rfc.zeromq.org/spec/32/)
+#define NOTZ85 "Vtb A"
+
+struct test_vec {
+    const char *name;
+    char *input;
+};
+
+static struct test_vec goodvec[] = {
+    {
+        .name = "czmq zcert sample",
+        .input =
+    "#   ****  Generated on 2023-09-16 23:15:27 by CZMQ  ****\n"
+    "#   ZeroMQ CURVE **Secret** Certificate\n"
+    "#   DO NOT PROVIDE THIS FILE TO OTHER USERS nor change its permissions.\n"
+    "\n"
+    "metadata\n"
+    "    name = \"picl0\"\n"
+    "    keygen.czmq-version = \"4.2.1\"\n"
+    "    keygen.sodium-version = \"1.0.18\"\n"
+    "    keygen.flux-core-version = \"0.54.0\"\n"
+    "    keygen.hostname = \"picl0\"\n"
+    "    keygen.time = \"2023-09-16T23:15:27\"\n"
+    "    keygen.userid = \"5588\"\n"
+    "    keygen.zmq-version = \"4.3.4\"\n"
+    "curve\n"
+    "    public-key = \"8)TKx/<plQR>gO0.HCH/AsS3n[QeKMOy@}$)=GVu\"\n"
+    "    secret-key = \"225YW{2q$:dqH]7cCbZW4a-}5Al/)0vkb>cE)o}Z\"\n"
+    },
+    {
+         .name = "cert with blank lines",
+         .input =
+    "curve\n"
+    " public-key = \"" PAIR1_PUB "\"\n"
+    "\n"
+    " secret-key = \"" PAIR1_SEC "\"\n"
+    "\n"
+    },
+    {
+         .name = "cert with indented inline comments",
+         .input =
+    "metadata\n"
+    "	# comment \n"
+    "curve\n"
+    "# comment \n"
+    " public-key = \"" PAIR1_PUB "\"\n"
+    " secret-key = \"" PAIR1_SEC "\"\n"
+    },
+    {
+         .name = "cert with # in z85 key",
+         .input =
+    "curve\n"
+    " public-key = \"" PAIR2_PUB "\"\n"
+    " secret-key = \"" PAIR2_SEC "\"\n"
+    },
+};
+
+static struct test_vec badvec[] = {
+    {
+        .name = "empty input",
+        .input = ""
+    },
+    {
+        .name = "cert with missing curve section",
+        .input =
+    "metadata\n"
+    },
+    {
+        .name = "cert with empty curve section",
+        .input =
+    "metadata\n"
+    "curve\n"
+    },
+    {
+        .name = "cert with extra section",
+        .input =
+    "metadata\n"
+    "unknown\n"
+    "curve\n"
+    " public-key = \"" PAIR1_PUB "\"\n"
+    " secret-key = \"" PAIR1_SEC "\"\n"
+    },
+    {
+        .name = "cert with curve section indented",
+        .input =
+    " curve\n"
+    " public-key = \"" PAIR1_PUB "\"\n"
+    " secret-key = \"" PAIR1_SEC "\"\n"
+    },
+    {
+        .name = "cert with keys not indented",
+        .input =
+    "curve\n"
+    "public-key = \"" PAIR1_PUB "\"\n"
+    "secret-key = \"" PAIR1_SEC "\"\n"
+    },
+    {
+        .name = "cert with public key missing",
+        .input =
+    "curve\n"
+    " secret-key = \"" PAIR1_SEC "\"\n"
+    },
+    {
+        .name = "cert with secret key missing",
+        .input =
+    "curve\n"
+    " public-key = \"" PAIR1_PUB "\"\n"
+    },
+    {
+        .name = "cert with public key containing illegal Z85",
+        .input =
+    "curve\n"
+    " public-key = \"" "FYFE.@650VuUqRGygAtG.RC$A<cid63q(WE" NOTZ85 "\"\n"
+    " secret-key = \"" PAIR1_SEC "\"\n"
+    },
+    {
+        .name = "cert with public missing end quote",
+        .input =
+    "metadata\n"
+    "curve\n"
+    " public-key = \"" PAIR1_PUB "\n"
+    " secret-key = \"" PAIR1_SEC "\"\n"
+    },
+    {
+        .name = "cert with public key too long",
+        .input =
+    "metadata\n"
+    "curve\n"
+    " public-key = \"" PAIR1_PUB PAIR1_PUB "\"\n"
+    " secret-key = \"" PAIR1_SEC "\"\n"
+    },
+#if (ZMQ_VERSION >= ZMQ_MAKE_VERSION(4,2,1))
+    {
+        .name = "cert with mismatched keypair",
+        .input =
+    "metadata\n"
+    "curve\n"
+    " public-key = \"" "YYFE.@650VuUqRGygAtG.RC$A<cid63q(WEnR+&y" "\"\n"
+    " secret-key = \"" PAIR1_SEC "\"\n"
+    },
+#endif
+};
+
+void test_basic (void)
+{
+    struct cert *cert;
+    struct cert *cert2;
+    size_t memsize = 8182;
+    void *membuf;
+    FILE *f;
+    const char *val;
+
+    cert = cert_create ();
+    ok (cert != NULL,
+        "cert_create works");
+    ok (cert_meta_set (cert, "foo", "bar") == 0,
+        "cert_meta_set foo=bar works");
+    val = cert_meta_get (cert, "foo");
+    ok (val != NULL && streq (val, "bar"),
+        "cert_meta_get foo returns bar");
+    ok (cert_meta_get (cert, "notakey") == NULL,
+        "cert_meta_get notakey returns NULL");
+
+    if (!(membuf = calloc (1, memsize)))
+        BAIL_OUT ("out of memory");
+    if (!(f = fmemopen (membuf, memsize, "r+")))
+        BAIL_OUT ("fmemopen failed: %s", strerror (errno));
+    ok (cert_write (cert, f) == 0,
+        "cert_write works");
+
+    if (fseek (f, 0L, SEEK_SET) < 0)
+        BAIL_OUT ("fseek 0 failed");
+    cert2 = cert_read (f);
+    ok (cert2 != NULL,
+        "cert_read works");
+    ok (cert_equal (cert, cert2),
+        "cert_equal says we read back what we wrote");
+    cert_destroy (cert2);
+
+    ok ((cert2 = cert_create ()) != NULL,
+        "cert_create created a second cert");
+    ok (!cert_equal (cert, cert2),
+        "cert_equal says it is different from the first cert");
+    cert_destroy (cert2);
+
+    fclose (f);
+    cert_destroy (cert);
+
+    cert = cert_create_from (NULL, NULL);
+    ok (cert != NULL,
+        "cert_create_from pub=NULL sec=NULL works");
+    ok (cert_public_txt (cert) == NULL,
+        "cert_public_txt returns NULL");
+    ok (cert_secret_txt (cert) == NULL,
+        "cert_secret_txt returns NULL");
+    cert_destroy (cert);
+
+    cert = cert_create_from (PAIR1_PUB, PAIR1_SEC);
+    ok (cert != NULL,
+        "cert_create_from works");
+    ok (cert_public_txt (cert) != NULL
+        && streq (cert_public_txt (cert), PAIR1_PUB),
+        "cert_public_txt is the public key");
+    ok (cert_secret_txt (cert) != NULL
+        && streq (cert_secret_txt (cert), PAIR1_SEC),
+        "cert_secret_txt is the secret key");
+    struct cert *cpub;
+    struct cert *csec;
+    cpub = cert_create_from (PAIR1_PUB, NULL);
+    ok (cpub != NULL,
+        "cert_create_from sec=NULL works");
+    ok (cert_public_txt (cpub) != NULL
+        && streq (cert_public_txt (cpub), PAIR1_PUB),
+        "cert_public_txt is the public key");
+    ok (cert_secret_txt (cpub) == NULL,
+        "cert_secret_txt returns NULL");
+    csec = cert_create_from (NULL, PAIR1_SEC);
+    ok (csec != NULL,
+        "cert_create_from pub=NULL works");
+    ok (cert_public_txt (csec) == NULL,
+        "cert_public_txt returns NULL");
+    ok (cert_secret_txt (csec) != NULL
+        && streq (cert_secret_txt (csec), PAIR1_SEC),
+        "cert_secret_txt is the secret key");
+    ok (!cert_equal (cert, cpub)
+        && !cert_equal (cert, csec)
+        && !cert_equal (cpub, csec),
+        "cert_equal handles partial certs OK");
+    cert_destroy (cpub);
+    cert_destroy (csec);
+    cert_destroy (cert);
+
+
+    free (membuf);
+}
+
+bool test_good_one (char *s)
+{
+    FILE *f_in;
+    FILE *f_inout = NULL;
+    struct cert *cert = NULL;
+    struct cert *cert2 = NULL;
+    bool result = false;
+    size_t memsize = 8182;
+    void *membuf = NULL;
+
+    /* read cert from 's' */
+    if (!(f_in = fmemopen (s, strlen (s) + 1, "r")))
+        BAIL_OUT ("fmemopen of good input failed: %s", strerror (errno));
+    if (!(cert = cert_read (f_in))) {
+        diag ("cert_read input failed\n");
+        goto done;
+    }
+
+    /* write cert to file */
+    if (!(membuf = calloc (1, memsize)))
+        BAIL_OUT ("out of memory");
+    if (!(f_inout = fmemopen (membuf, memsize, "w+")))
+        BAIL_OUT ("fmemopen failed: %s", strerror (errno));
+    if (cert_write (cert, f_inout) < 0) {
+        diag ("cert_write failed");
+        goto done;
+    }
+
+    if (fseek (f_inout, 0L, SEEK_SET) < 0)
+        BAIL_OUT ("fseek 0 failed");
+
+    /* read cert from file */
+    if (!(cert2 = cert_read (f_inout))) {
+        diag ("cert_read tmp failed");
+        goto done;
+    }
+
+    /* compare certs */
+    result = cert_equal (cert, cert2);
+
+done:
+    if (f_in)
+        fclose (f_in);
+    if (f_inout)
+        fclose (f_inout);
+    free (membuf);
+    cert_destroy (cert);
+    cert_destroy (cert2);
+    return result;
+}
+
+bool test_bad_one (char *s)
+{
+    FILE *f;
+    struct cert *cert;
+
+    if (!(f = fmemopen (s, strlen (s) + 1, "r")))
+        BAIL_OUT ("fmemopen of bad input failed: %s", strerror (errno));
+    cert = cert_read (f);
+    fclose (f);
+    if (cert) {
+        diag ("cert_read unexpectedly succeeded\n");
+        cert_destroy (cert);
+        return false;
+    }
+    return true;
+}
+
+void test_vec (void)
+{
+    for (int i = 0; i < ARRAY_SIZE (goodvec); i++) {
+        ok (test_good_one (goodvec[i].input),
+           "%s can be read/written/read", goodvec[i].name);
+    }
+    for (int i = 0; i < ARRAY_SIZE (badvec); i++) {
+        ok (test_bad_one (badvec[i].input),
+           "%s fails as expected", badvec[i].name);
+    }
+}
+
+void test_inval (void)
+{
+    struct cert *cert;
+    struct cert *cpub;
+
+    if (!(cert = cert_create ()))
+        BAIL_OUT ("could not create cert");
+    if (!(cpub = cert_create_from (cert_public_txt (cert), NULL)))
+        BAIL_OUT ("could not create cert");
+
+    errno = 0;
+    ok (cert_meta_set (NULL, "foo", "bar") == -1 && errno == EINVAL,
+        "cert_meta_set cert=NULL fails with EINVAL");
+    errno = 0;
+    ok (cert_meta_set (cert, NULL, "bar") == -1 && errno == EINVAL,
+        "cert_meta_set key=NULL fails with EINVAL");
+    errno = 0;
+    ok (cert_meta_set (cert, "", "bar") == -1 && errno == EINVAL,
+        "cert_meta_set key=\"\" fails with EINVAL");
+    errno = 0;
+    ok (cert_meta_set (cert, "foo", NULL) == -1 && errno == EINVAL,
+        "cert_meta_set val=NULL fails with EINVAL");
+
+    const char *shortkey = "S:do@!Xbon>XQ$e!SK";
+    errno = 0;
+    ok (cert_create_from (shortkey, NULL) == NULL && errno == EINVAL,
+        "cert_create_from pub=shortkey fails with EINVAL");
+    errno = 0;
+    ok (cert_create_from (NULL, shortkey) == NULL && errno == EINVAL,
+        "cert_create_from sec=shortkey fails with EINVAL");
+
+    const char *badz85 = "s70JW1{s!)bET!S&yF=7z=b{%+<2Nu1zO31tCad\002";
+    errno = 0;
+    ok (cert_create_from (badz85, NULL) == NULL && errno == EINVAL,
+        "cert_create_from pub=badz85 fails with EINVAL");
+    errno = 0;
+    ok (cert_create_from (NULL, badz85) == NULL && errno == EINVAL,
+        "cert_create_from sec=badz85 fails with EINVAL");
+
+    errno = 0;
+    ok (cert_write (NULL, stderr) < 0 && errno == EINVAL,
+        "cert_write cert=NULL fails with EINVAL");
+    errno = 0;
+    ok (cert_write (cpub, stderr) < 0 && errno == EINVAL,
+        "cert_write cert=NULL fails with EINVAL");
+    errno = 0;
+    ok (cert_write (cert, NULL) < 0 && errno == EINVAL,
+        "cert_write f=NULL fails with EINVAL");
+    errno = 0;
+    ok (cert_write (cpub, stderr) < 0 && errno == EINVAL,
+        "cert_write cert=partial fails with EINVAL");
+
+    lives_ok ({cert_destroy (NULL);},
+              "cert_destroy cert=NULL does not crash");
+
+    ok (cert_public_txt (NULL) == NULL,
+        "cert_public_txt cert=NULL returns NULL");
+    ok (cert_secret_txt (NULL) == NULL,
+        "cert_secret_txt cert=NULL returns NULL");
+
+    errno = EINVAL;
+    ok (cert_apply (NULL, NULL) < 0 && errno == EINVAL,
+        "cert_apply cert=NULL fails with EINVAL");
+    errno = EINVAL;
+    ok (cert_apply (cpub, NULL) < 0 && errno == EINVAL,
+        "cert_apply cert=partial fails with EINVAL");
+    errno = EINVAL;
+    ok (cert_apply (cert, NULL) < 0 && errno == ENOTSOCK,
+        "cert_apply sock=NULL fails with ENOTSOCK");
+
+    cert_destroy (cpub);
+    cert_destroy (cert);
+}
+
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_basic ();
+    test_vec ();
+    test_inval ();
+
+    done_testing();
+
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libzmqutil/test/ev.c b/src/common/libzmqutil/test/ev.c
new file mode 100644
index 000000000000..ffac32c92748
--- /dev/null
+++ b/src/common/libzmqutil/test/ev.c
@@ -0,0 +1,118 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* ev.c - standalone test of libev ev_zmq watcher (no flux) */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <poll.h>
+#include <zmq.h>
+#include <stdio.h>
+
+#include "src/common/libzmqutil/ev_zmq.h"
+#include "src/common/libtap/tap.h"
+
+static void *zctx;
+
+void zsock_tx_cb (struct ev_loop *loop, ev_zmq *w, int revents)
+{
+    static int count = 50; /* send two per invocation */
+
+    if ((revents & EV_WRITE)) {
+        if (zmq_send (w->zsock, "PING", 4, 0) < 0)
+            fprintf (stderr, "zmq_send: %s", strerror (errno));
+        if (zmq_send (w->zsock, "PING", 4, 0) < 0)
+            fprintf (stderr, "zmq_send: %s", strerror (errno));
+        if (--count == 0)
+            ev_zmq_stop (loop, w);
+    }
+    if ((revents & EV_ERROR))
+        ev_break (loop, EVBREAK_ALL);
+}
+
+void zsock_rx_cb (struct ev_loop *loop, ev_zmq *w, int revents)
+{
+    int *iter = w->data;
+    char buf[128];
+    static int count = 100;
+
+    if ((revents & EV_READ)) {
+        (*iter)++;
+        if (zmq_recv (w->zsock, buf, sizeof (buf), 0) < 0)
+            fprintf (stderr, "zstr_recv: %s", strerror (errno));
+        if (--count == 0)
+            ev_zmq_stop (loop, w);
+    }
+    if ((revents & EV_ERROR))
+        ev_break (loop, EVBREAK_ALL);
+}
+
+
+/* send 100 messages over PAIR sockets
+ * sender in one event handler, receiver in another
+ */
+void test_ev_zmq (void)
+{
+    struct ev_loop *loop;
+    void *zctx;
+    void *zin, *zout;
+    int i;
+    ev_zmq win, wout;
+
+    ok ((loop = ev_loop_new (EVFLAG_AUTO)) != NULL,
+        "ev_loop_new works");
+    ok ((zctx = zmq_init (1)) != NULL,
+        "initialized zmq context");
+    ok ((zout = zmq_socket (zctx, ZMQ_PAIR)) != NULL
+        && zmq_bind (zout, "inproc://eventloop_test") == 0,
+        "PAIR socket bind ok");
+    ok ((zin = zmq_socket (zctx, ZMQ_PAIR)) != NULL
+        && zmq_connect (zin, "inproc://eventloop_test") == 0,
+        "PAIR socket connect ok");
+
+    i = 0;
+    ev_zmq_init (&win, zsock_rx_cb, zin, EV_READ);
+    win.data = &i;
+    ev_zmq_init (&wout, zsock_tx_cb, zout, EV_WRITE);
+
+    ev_zmq_start (loop, &win);
+    ev_zmq_start (loop, &wout);
+
+    ok (ev_run (loop, 0) == 0,
+        "both watchers removed themselves and ev_run exited");
+    ev_zmq_stop (loop, &win);
+    ev_zmq_stop (loop, &wout);
+    cmp_ok (i, "==", 100,
+        "ev_zmq handler ran 100 times");
+
+    ev_loop_destroy (loop);
+
+    zmq_close (zin);
+    zmq_close (zout);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    if (!(zctx = zmq_ctx_new ()))
+        BAIL_OUT ("could not create zeromq context");
+
+    test_ev_zmq ();
+
+    zmq_ctx_term (zctx);
+
+    done_testing ();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libzmqutil/test/monitor.c b/src/common/libzmqutil/test/monitor.c
new file mode 100644
index 000000000000..46c4b16ad77b
--- /dev/null
+++ b/src/common/libzmqutil/test/monitor.c
@@ -0,0 +1,50 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h>
+#include "monitor.h"
+
+#include "src/common/libtap/tap.h"
+
+void test_badargs (void)
+{
+    /* Note: these are stubbed for older libzmq (e.g. centos 7),
+     * so checking for errno == EINVAL is not going to happen there.
+     */
+    ok (zmqutil_monitor_create (NULL, NULL, NULL, NULL, NULL) == NULL,
+        "zmqutil_monitor_create sock=NULL fails");
+
+    lives_ok({zmqutil_monitor_destroy (NULL);},
+        "zmqutil_monitor_destroy sock=NULL doesn't crash");
+
+    ok (zmqutil_monitor_get (NULL, NULL) < 0,
+        "zmqutil_monitor_get mon=NULL fails");
+
+    lives_ok({zmqutil_monitor_iserror (NULL);},
+        "zmqutil_monitor_iserror mevent=NULL doesn't crash");
+
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_badargs ();
+
+    done_testing ();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libzmqutil/test/mpart.c b/src/common/libzmqutil/test/mpart.c
new file mode 100644
index 000000000000..69ed59ad8c8a
--- /dev/null
+++ b/src/common/libzmqutil/test/mpart.c
@@ -0,0 +1,136 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h>
+#include <zmq.h>
+#include "mpart.h"
+
+#include "src/common/libtap/tap.h"
+
+static void *zctx;
+
+static void zsocketpair (void **sock, const char *uri)
+{
+    sock[0] = zmq_socket (zctx, ZMQ_PAIR);
+    sock[1] = zmq_socket (zctx, ZMQ_PAIR);
+    if (!sock[0]
+        || !sock[1]
+        || zmq_bind (sock[0], uri) < 0
+        || zmq_connect (sock[1], uri) < 0)
+        BAIL_OUT ("could not create 0MQ socketpair");
+}
+
+void test_mpart (void)
+{
+    void *sock[2];
+    zlist_t *mpart_snd;
+    zlist_t *mpart_rcv;
+
+    zsocketpair (sock, "inproc://test_mpart");
+
+    mpart_snd = mpart_create ();
+    ok (mpart_snd != NULL,
+        "mpart_create works");
+    ok (mpart_addstr (mpart_snd, "foo") == 0
+        && zlist_size (mpart_snd) == 1,
+        "mpart_addstr works");
+    ok (mpart_addmem (mpart_snd, "bar", 3) == 0
+        && zlist_size (mpart_snd) == 2,
+        "mpart_addmem works");
+    ok (mpart_addmem (mpart_snd, NULL, 0) == 0
+        && zlist_size (mpart_snd) == 3,
+        "mpart_addmem buf=NULL size=0 works");
+    ok (mpart_send (sock[1], mpart_snd) == 0,
+        "mpart_send works");
+    mpart_rcv = mpart_recv (sock[0]);
+    ok (mpart_rcv != NULL,
+        "mpart_recv works");
+    ok (zlist_size (mpart_rcv) == 3
+        && mpart_streq (mpart_rcv, 0, "foo")
+        && mpart_streq (mpart_rcv, 1, "bar")
+        && zmq_msg_size (mpart_get (mpart_rcv, 2)) == 0,
+        "send and recv messages are identical");
+
+    errno = 42;
+    mpart_destroy (mpart_snd);
+    mpart_destroy (mpart_rcv);
+    ok (errno == 42,
+        "mpart_destroy doesn't clobber errno");
+
+    zmq_close (sock[0]);
+    zmq_close (sock[1]);
+}
+
+void test_mpart_inval (void)
+{
+    zlist_t *mpart;
+    void *sock[2];
+
+    zsocketpair (sock, "inproc://test_mpart_inval");
+
+    if (!(mpart = mpart_create ())
+        || mpart_addstr (mpart, "x"))
+        BAIL_OUT ("mpart_create failed");
+
+    errno = 0;
+    ok (mpart_addmem (NULL, NULL, 0) < 0 && errno == EINVAL,
+        "mpart_addmem mpart=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (mpart_addstr (NULL, "foo") < 0 && errno == EINVAL,
+        "mpart_addstr mpart=NULL fails with EINVAL");
+    errno = 0;
+    ok (mpart_addstr (mpart, NULL) < 0 && errno == EINVAL,
+        "mpart_addstr s=NULL fails with EINVAL");
+
+    errno = 0;
+    ok (mpart_recv (NULL) == NULL && errno == ENOTSOCK,
+        "mpart_recv sock=NULL fails with ENOTSOCK");
+
+    errno = 0;
+    ok (mpart_send (NULL, mpart) < 0 && errno == ENOTSOCK,
+        "mpart_send sock=NULL fails with ENOTSOCK");
+
+    errno = 0;
+    ok (mpart_send (sock[1], NULL) < 0 && errno == EINVAL,
+        "mpart_send mpart=NULL fails with EINVAL");
+
+    ok (mpart_get (NULL, 0) == NULL,
+        "mpart_get mpart=NULL returns NULL");
+    ok (mpart_streq (NULL, 0, "foo") == false,
+        "mpart_streq mpart=NULL returns false");
+
+    mpart_destroy (mpart);
+
+    zmq_close (sock[0]);
+    zmq_close (sock[1]);
+
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    if (!(zctx = zmq_ctx_new ()))
+        BAIL_OUT ("could not create zeromq context");
+
+    test_mpart ();
+    test_mpart_inval ();
+
+    zmq_ctx_term (zctx);
+
+    done_testing ();
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/common/libzmqutil/test/msg_zsock.c b/src/common/libzmqutil/test/msg_zsock.c
new file mode 100644
index 000000000000..d5344e92be7c
--- /dev/null
+++ b/src/common/libzmqutil/test/msg_zsock.c
@@ -0,0 +1,105 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdbool.h>
+#include <zmq.h>
+#include <errno.h>
+#include <stdio.h>
+#include <flux/core.h>
+
+#include "src/common/libzmqutil/msg_zsock.h"
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+
+#include "sockopt.h"
+
+static void *zctx;
+
+void check_sendzsock (void)
+{
+    void *zsock[2] = { NULL, NULL };
+    flux_msg_t *msg, *msg2;
+    const char *topic;
+    int type;
+    const char *uri = "inproc://test";
+
+    ok ((zsock[0] = zmq_socket (zctx, ZMQ_PAIR)) != NULL
+        && zmq_bind (zsock[0], uri) == 0
+        && (zsock[1] = zmq_socket (zctx, ZMQ_PAIR)) != NULL
+        && zmq_connect( zsock[1], uri) == 0,
+        "got inproc socket pair");
+
+    if (zsetsockopt_int (zsock[0], ZMQ_LINGER, 5) < 0
+        || zsetsockopt_int (zsock[1], ZMQ_LINGER, 5) < 0)
+        BAIL_OUT ("could not set ZMQ_LINGER socket option");
+
+    ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL
+            && flux_msg_set_topic (msg, "foo.bar") == 0,
+        "created test message");
+
+    /* corner case tests */
+    ok (zmqutil_msg_send (NULL, msg) < 0 && errno == EINVAL,
+        "zmqutil_msg_send returns < 0 and EINVAL on dest = NULL");
+    ok (zmqutil_msg_send_ex (NULL, msg, true) < 0 && errno == EINVAL,
+        "zmqutil_msg_send_ex returns < 0 and EINVAL on dest = NULL");
+    ok (zmqutil_msg_recv (NULL) == NULL && errno == EINVAL,
+        "zmqutil_msg_recv returns NULL and EINVAL on dest = NULL");
+
+    ok (zmqutil_msg_send (zsock[1], msg) == 0,
+        "zmqutil_msg_send works");
+    ok ((msg2 = zmqutil_msg_recv (zsock[0])) != NULL,
+        "zmqutil_msg_recv works");
+    ok (flux_msg_get_type (msg2, &type) == 0 && type == FLUX_MSGTYPE_REQUEST
+            && flux_msg_get_topic (msg2, &topic) == 0
+            && streq (topic, "foo.bar")
+            && flux_msg_has_payload (msg2) == false,
+        "decoded message looks like what was sent");
+    flux_msg_destroy (msg2);
+
+    /* Send it again.
+     */
+    ok (zmqutil_msg_send (zsock[1], msg) == 0,
+        "try2: zmqutil_msg_send works");
+    ok ((msg2 = zmqutil_msg_recv (zsock[0])) != NULL,
+        "try2: zmqutil_msg_recv works");
+    ok (flux_msg_get_type (msg2, &type) == 0 && type == FLUX_MSGTYPE_REQUEST
+            && flux_msg_get_topic (msg2, &topic) == 0
+            && streq (topic, "foo.bar")
+            && flux_msg_has_payload (msg2) == false,
+        "try2: decoded message looks like what was sent");
+    flux_msg_destroy (msg2);
+    flux_msg_destroy (msg);
+
+    zmq_close (zsock[0]);
+    zmq_close (zsock[1]);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    if (!(zctx = zmq_ctx_new ()))
+        BAIL_OUT ("could not create zeromq context");
+
+    check_sendzsock ();
+
+    zmq_ctx_term (zctx);
+
+    done_testing();
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libzmqutil/test/reactor.c b/src/common/libzmqutil/test/reactor.c
new file mode 100644
index 000000000000..bcdb9cee019d
--- /dev/null
+++ b/src/common/libzmqutil/test/reactor.c
@@ -0,0 +1,143 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <zmq.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <stdlib.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+
+#include "src/common/libzmqutil/reactor.h"
+
+static const size_t zmqwriter_msgcount = 1024;
+static void *zctx;
+
+static void zmqwriter (flux_reactor_t *r, flux_watcher_t *w,
+                       int revents, void *arg)
+{
+    void *sock = zmqutil_watcher_get_zsock (w);
+    static int count = 0;
+    if (revents & FLUX_POLLERR) {
+        fprintf (stderr, "%s: FLUX_POLLERR is set\n", __FUNCTION__);
+        goto error;
+    }
+    if (revents & FLUX_POLLOUT) {
+        uint8_t blob[64] = { 0 };
+        if (zmq_send (sock, blob, sizeof (blob), 0) < 0) {
+            diag ("zmq_send: %s", strerror (errno));
+            goto error;
+        }
+        count++;
+        if (count == zmqwriter_msgcount)
+            flux_watcher_stop (w);
+    }
+    return;
+error:
+    flux_reactor_stop_error (r);
+}
+
+static void zmqreader (flux_reactor_t *r, flux_watcher_t *w,
+                       int revents, void *arg)
+{
+    void *sock = zmqutil_watcher_get_zsock (w);
+    static int count = 0;
+    if (revents & FLUX_POLLERR) {
+        fprintf (stderr, "%s: FLUX_POLLERR is set\n", __FUNCTION__);
+        goto error;
+    }
+    if (revents & FLUX_POLLIN) {
+        char buf[64];
+        int rc;
+        if ((rc = zmq_recv (sock, buf, sizeof (buf), 0)) < 0) {
+            diag ("zmq_recv: %s", strerror (errno));
+            goto error;
+        }
+        if (rc != 64) {
+            diag ("zmq_reciv: got %d bytes, expected 64", rc);
+            goto error;
+        }
+        count++;
+        if (count == zmqwriter_msgcount)
+            flux_watcher_stop (w);
+    }
+    return;
+error:
+    flux_reactor_stop_error (r);
+}
+
+static void test_zmq (flux_reactor_t *reactor)
+{
+    void *zs[2];
+    flux_watcher_t *r, *w;
+    const char *uri = "inproc://test_zmq";
+
+    zs[0] = zmq_socket (zctx, ZMQ_PAIR);
+    zs[1] = zmq_socket (zctx, ZMQ_PAIR);
+    ok (zs[0] && zs[1]
+        && zmq_bind (zs[0], uri) == 0
+        && zmq_connect (zs[1], uri) == 0,
+        "zmq: connected ZMQ_PAIR sockets over inproc");
+    r = zmqutil_watcher_create (reactor, zs[0], FLUX_POLLIN, zmqreader, NULL);
+    w = zmqutil_watcher_create (reactor, zs[1], FLUX_POLLOUT, zmqwriter, NULL);
+    ok (r != NULL && w != NULL,
+        "zmq: nonblocking reader and writer created");
+    ok (flux_watcher_is_active (r) == false,
+        "flux_watcher_is_active() returns false on create");
+    flux_watcher_start (r);
+    ok (flux_watcher_is_active (r) == true,
+        "flux_watcher_is_active() returns true after start");
+    flux_watcher_start (w);
+    ok (flux_reactor_run  (reactor, 0) == 0,
+        "zmq: reactor ran to completion after %d messages", zmqwriter_msgcount);
+    flux_watcher_stop (r);
+    ok (flux_watcher_is_active (r) == false,
+        "flux_watcher_is_active() returns false after stop");
+    flux_watcher_stop (w);
+    flux_watcher_destroy (r);
+    flux_watcher_destroy (w);
+
+    zmq_close (zs[0]);
+    zmq_close (zs[1]);
+}
+
+int main (int argc, char *argv[])
+{
+    flux_reactor_t *reactor;
+
+    plan (NO_PLAN);
+
+    if (!(zctx = zmq_ctx_new ()))
+        BAIL_OUT ("cannot create zmq context");
+
+    ok ((reactor = flux_reactor_create (0)) != NULL,
+        "created reactor");
+    if (!reactor)
+        BAIL_OUT ("can't continue without reactor");
+
+    test_zmq (reactor);
+
+    flux_reactor_destroy (reactor);
+
+    zmq_ctx_term (zctx);
+
+    done_testing();
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libzmqutil/test/zap.c b/src/common/libzmqutil/test/zap.c
new file mode 100644
index 000000000000..ab427236a13c
--- /dev/null
+++ b/src/common/libzmqutil/test/zap.c
@@ -0,0 +1,51 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "src/common/libtap/tap.h"
+
+#include <flux/core.h>
+#include "zap.h"
+
+void test_badargs (void)
+{
+    errno = 0;
+    ok (zmqutil_zap_create (NULL, NULL) == NULL && errno == EINVAL,
+        "zmqutil_zap_create zctx=NULL reactor=NULL fails with EINVAL");
+
+    lives_ok ({zmqutil_zap_destroy (NULL);},
+        "zmqutil_zap_destroy zap=NULL doesn't crash");
+    lives_ok ({zmqutil_zap_set_logger (NULL, NULL, NULL);},
+        "zmqutil_zap_set_logger zap=NULL doesn't crash");
+
+    errno = 0;
+    ok (zmqutil_zap_authorize (NULL, NULL, NULL) < 0 && errno == EINVAL,
+        "zmqutil_zap_authorize zap=NULL fails with EINVAL");
+}
+
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_badargs ();
+
+    done_testing();
+
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/common/libzmqutil/zap.c b/src/common/libzmqutil/zap.c
new file mode 100644
index 000000000000..69711789a50a
--- /dev/null
+++ b/src/common/libzmqutil/zap.c
@@ -0,0 +1,231 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* zap.c - zeromq auth proto (ZAP) server, embeddable in a flux reactor loop
+ *
+ * See 0MQ RFC 27:  https://rfc.zeromq.org/spec/27/
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <zmq.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "reactor.h"
+#include "sockopt.h"
+#include "mpart.h"
+#include "cert.h"
+#include "zap.h"
+
+#define ZAP_ENDPOINT "inproc://zeromq.zap.01"
+
+struct zmqutil_zap {
+    zhashx_t *certstore; // public_txt => authorized cert
+    void *sock;
+    flux_watcher_t *w;
+    zaplog_f logger;
+    void *logger_arg;
+};
+
+/* Get a public CURVE key (binary form) from message part at index 'n',
+ * and convert it to 40 byte text.  Return 0 on success, -1 on failure.
+ */
+static int get_mpart_pubkey (zlist_t *mpart, int n, char *pubkey_txt)
+{
+    zmq_msg_t *part = mpart_get (mpart, n);
+    if (!part || zmq_msg_size (part) != 32)
+        return -1;
+    zmq_z85_encode (pubkey_txt, zmq_msg_data (part), zmq_msg_size (part));
+    return 0;
+}
+
+/* Take a copy of message part at index 'n' of 'src' and append it to 'dst'.
+ * Return 0 on success, -1 on failure.
+ */
+static int add_mpart_copy (zlist_t *dst, zlist_t *src, int n)
+{
+    zmq_msg_t *part = mpart_get (src, n);
+    if (!part)
+        return -1;
+    return mpart_addmem (dst, zmq_msg_data (part), zmq_msg_size (part));
+}
+
+static void logger (struct zmqutil_zap *zap,
+                    int severity,
+                    const char *fmt, ...)
+{
+    if (zap->logger) {
+        va_list ap;
+        char buf[256];
+
+        va_start (ap, fmt);
+        vsnprintf (buf, sizeof (buf), fmt, ap);
+        va_end (ap);
+
+        zap->logger (severity, buf, zap->logger_arg);
+    }
+}
+
+/* ZAP 1.0 messages have the following parts
+ * REQUEST                              RESPONSE
+ *   0: version                           0: version
+ *   1: sequence                          1: sequence
+ *   2: domain                            2: status_code
+ *   3: address                           3: status_text
+ *   4: identity                          4: user_id
+ *   5: mechanism                         5: metadata
+ *   6: client_key
+ */
+static void zap_cb (flux_reactor_t *r,
+                    flux_watcher_t *w,
+                    int revents,
+                    void *arg)
+{
+    struct zmqutil_zap *zap = arg;
+    zlist_t *req = NULL;
+    zlist_t *rep = NULL;
+    char pubkey[41];
+    const char *status_code = "400";
+    const char *status_text = "No access";
+    const char *user_id = "";
+    struct cert *cert;
+
+    if ((req = mpart_recv (zap->sock))) {
+        if (!mpart_streq (req, 0, "1.0")
+            || !mpart_streq (req, 5, "CURVE")
+            || get_mpart_pubkey (req, 6, pubkey) < 0) {
+            logger (zap, LOG_ERR, "ZAP request decode error");
+            goto done;
+        }
+        if ((cert = zhashx_lookup (zap->certstore, pubkey)) != NULL) {
+            status_code = "200";
+            status_text = "OK";
+            user_id = pubkey;
+        }
+        else
+            logger (zap, LOG_ERR, "overlay auth %s", status_text);
+
+        if (!(rep = mpart_create ()))
+            goto done;
+        if (add_mpart_copy (rep, req, 0) < 0
+            || add_mpart_copy (rep, req, 1) < 0
+            || mpart_addstr (rep, status_code) < 0
+            || mpart_addstr (rep, status_text) < 0
+            || mpart_addstr (rep, user_id) < 0
+            || mpart_addmem (rep, NULL, 0) < 0) {
+            logger (zap, LOG_ERR, "ZAP response encode error");
+            goto done;
+        }
+        if (mpart_send (zap->sock, rep) < 0)
+            logger (zap, LOG_ERR, "ZAP send error");
+    }
+done:
+    mpart_destroy (req);
+    mpart_destroy (rep);
+}
+
+/* Create a cert and add it to in-memory store.
+ */
+int zmqutil_zap_authorize (struct zmqutil_zap *zap,
+                           const char *name,
+                           const char *pubkey)
+{
+    struct cert *cert;
+
+    if (!zap || !name) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(cert = cert_create_from (pubkey, NULL))
+        || cert_meta_set (cert, "name", name) < 0)
+        return -1;
+
+    // hash takes ownership of cert
+    if ((zhashx_insert (zap->certstore, cert_public_txt (cert), cert)) < 0) {
+        cert_destroy (cert);
+        errno = EEXIST;
+        return -1;
+    }
+    return 0;
+}
+
+void zmqutil_zap_set_logger (struct zmqutil_zap *zap, zaplog_f fun, void *arg)
+{
+    if (zap) {
+        zap->logger = fun;
+        zap->logger_arg = arg;
+    }
+}
+
+void zmqutil_zap_destroy (struct zmqutil_zap *zap)
+{
+    if (zap) {
+        int saved_errno = errno;
+        flux_watcher_destroy (zap->w);
+        if (zap->sock) {
+            zmq_unbind (zap->sock, ZAP_ENDPOINT);
+            zmq_close (zap->sock);
+        }
+        zhashx_destroy (&zap->certstore);
+        free (zap);
+        errno = saved_errno;
+    }
+}
+
+// zhashx_destructor_fn footprint
+void cert_destructor (void **item)
+{
+    if (item) {
+        struct cert *cert = *item;
+        cert_destroy (cert);
+        *item = NULL;
+    }
+}
+
+struct zmqutil_zap *zmqutil_zap_create (void *zctx, flux_reactor_t *r)
+{
+    struct zmqutil_zap *zap;
+
+    if (!r || !zctx) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(zap = calloc (1, sizeof (*zap))))
+        return NULL;
+    if (!(zap->certstore = zhashx_new ())) {
+        errno = EINVAL;
+        goto error;
+    }
+    zhashx_set_key_duplicator (zap->certstore, NULL);
+    zhashx_set_key_destructor (zap->certstore, NULL);
+    zhashx_set_destructor (zap->certstore, cert_destructor);
+    if (!(zap->sock = zmq_socket (zctx, ZMQ_REP)))
+        goto error;
+    if (zmq_bind (zap->sock, ZAP_ENDPOINT) < 0)
+        goto error;
+    if (!(zap->w = zmqutil_watcher_create (r,
+                                           zap->sock,
+                                           FLUX_POLLIN,
+                                           zap_cb,
+                                           zap)))
+        goto error;
+    flux_watcher_start (zap->w);
+    return zap;
+error:
+    zmqutil_zap_destroy (zap);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/libzmqutil/zap.h b/src/common/libzmqutil/zap.h
new file mode 100644
index 000000000000..b9e92089bb24
--- /dev/null
+++ b/src/common/libzmqutil/zap.h
@@ -0,0 +1,33 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _ZMQUTIL_ZAP_H
+#define _ZMQUTIL_ZAP_H
+
+#include <flux/core.h>
+
+struct zmqutil_zap;
+
+typedef void (*zaplog_f)(int severity, const char *message, void *arg);
+
+int zmqutil_zap_authorize (struct zmqutil_zap *zap,
+                           const char *name,
+                           const char *pubkey);
+
+void zmqutil_zap_set_logger (struct zmqutil_zap *zap, zaplog_f fun, void *arg);
+
+void zmqutil_zap_destroy (struct zmqutil_zap *zap);
+struct zmqutil_zap *zmqutil_zap_create (void *zctx, flux_reactor_t *r);
+
+#endif // !_ZMQUTIL_ZAP_H
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/common/schedutil.h b/src/common/schedutil.h
index 5db3682a4cd8..2489b1bc9cb1 100644
--- a/src/common/schedutil.h
+++ b/src/common/schedutil.h
@@ -11,10 +11,6 @@
 #ifndef _FLUX_SCHEDUTIL_H
 #define _FLUX_SCHEDUTIL_H
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 #include "schedutil/init.h"
 #include "schedutil/hello.h"
 #include "schedutil/ready.h"
@@ -22,10 +18,6 @@ extern "C" {
 #include "schedutil/free.h"
 #include "schedutil/ops.h"
 
-#ifdef __cplusplus
-}
-#endif
-
 #endif /* !_FLUX_SCHEDUTIL_H */
 
 /*
diff --git a/src/connectors/Makefile.am b/src/connectors/Makefile.am
index 8718b9bb2639..240397c2e9e6 100644
--- a/src/connectors/Makefile.am
+++ b/src/connectors/Makefile.am
@@ -1 +1,28 @@
-SUBDIRS = local shmem loop ssh
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_builddir)/src/common/libflux \
+	-I$(top_srcdir)/src/common/libccan
+
+fluxconnector_LTLIBRARIES = \
+	ssh.la
+
+connector_ldflags = -module $(san_ld_zdef_flag) \
+	-export-symbols-regex '^connector_init$$' \
+	--disable-static -avoid-version -shared -export-dynamic \
+	$(ld_gc_sections)
+
+ssh_la_SOURCES = \
+	ssh/ssh.c
+ssh_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+ssh_la_LDFLAGS = $(connector_ldflags)
diff --git a/src/connectors/local/Makefile.am b/src/connectors/local/Makefile.am
deleted file mode 100644
index 6d96b320c60a..000000000000
--- a/src/connectors/local/Makefile.am
+++ /dev/null
@@ -1,27 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) \
-	$(LIBUUID_CFLAGS)
-
-fluxconnector_LTLIBRARIES = local.la
-
-local_la_SOURCES = local.c
-
-local_la_LDFLAGS = -module $(san_ld_zdef_flag) \
-	-export-symbols-regex '^connector_init$$' \
-	--disable-static -avoid-version -shared -export-dynamic
-
-local_la_LIBADD = \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(ZMQ_LIBS) \
-	$(LIBUUID_LIBS)
diff --git a/src/connectors/local/local.c b/src/connectors/local/local.c
deleted file mode 100644
index 25d06e2e86ad..000000000000
--- a/src/connectors/local/local.c
+++ /dev/null
@@ -1,245 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <assert.h>
-#include <errno.h>
-#include <stdbool.h>
-#include <sys/types.h>
-#include <sys/param.h>
-#include <sys/un.h>
-#include <sys/socket.h>
-#include <signal.h>
-#include <poll.h>
-#include <unistd.h>
-#include <flux/core.h>
-
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/errno_safe.h"
-#include "src/common/librouter/usock.h"
-
-struct local_connector {
-    struct usock_client *uclient;
-    uint32_t testing_userid;
-    uint32_t testing_rolemask;
-    flux_t *h;
-    int fd;
-};
-
-static const struct flux_handle_ops handle_ops;
-
-static int op_pollevents (void *impl)
-{
-    struct local_connector *ctx = impl;
-
-    return usock_client_pollevents (ctx->uclient);
-}
-
-static int op_pollfd (void *impl)
-{
-    struct local_connector *ctx = impl;
-
-    return usock_client_pollfd (ctx->uclient);
-}
-
-/* Special send function for testing that sets the userid/rolemask to
- * values set with flux_opt_set().  The connector-local module
- * overwrites these credentials for guests, but allows pass-through for
- * instance owner.  This is useful for service access control testing.
- */
-static int send_testing (struct local_connector *ctx,
-                         const flux_msg_t *msg,
-                         int flags)
-{
-    flux_msg_t *cpy;
-
-    if (!(cpy = flux_msg_copy (msg, true)))
-        return -1;
-    if (flux_msg_set_userid (cpy, ctx->testing_userid) < 0)
-        goto error;
-    if (flux_msg_set_rolemask (cpy, ctx->testing_rolemask) < 0)
-        goto error;
-    if (usock_client_send (ctx->uclient, cpy, flags) < 0)
-        goto error;
-    flux_msg_destroy (cpy);
-    return 0;
-error:
-    flux_msg_destroy (cpy);
-    return -1;
-}
-
-static int op_send (void *impl, const flux_msg_t *msg, int flags)
-{
-    struct local_connector *ctx = impl;
-
-    if (ctx->testing_userid != FLUX_USERID_UNKNOWN
-                                || ctx->testing_rolemask != FLUX_ROLE_NONE)
-        return send_testing (ctx, msg, flags);
-
-    return usock_client_send (ctx->uclient, msg, flags);
-}
-
-static flux_msg_t *op_recv (void *impl, int flags)
-{
-    struct local_connector *ctx = impl;
-
-    return usock_client_recv (ctx->uclient, flags);
-}
-
-static int op_event_subscribe (void *impl, const char *topic)
-{
-    struct local_connector *ctx = impl;
-    flux_future_t *f;
-
-    if (!(f = flux_rpc_pack (ctx->h,
-                             "local.sub",
-                             FLUX_NODEID_ANY,
-                             0,
-                             "{s:s}",
-                             "topic", topic)))
-        return -1;
-    if (flux_future_get (f, NULL) < 0) {
-        flux_future_destroy (f);
-        return -1;
-    }
-    flux_future_destroy (f);
-    return 0;
-}
-
-static int op_event_unsubscribe (void *impl, const char *topic)
-{
-    struct local_connector *ctx = impl;
-    flux_future_t *f;
-
-    if (!(f = flux_rpc_pack (ctx->h,
-                             "local.unsub",
-                             FLUX_NODEID_ANY,
-                             0,
-                             "{s:s}",
-                             "topic", topic)))
-        return -1;
-    if (flux_future_get (f, NULL) < 0) {
-        flux_future_destroy (f);
-        return -1;
-    }
-    flux_future_destroy (f);
-    return 0;
-}
-
-static int op_setopt (void *impl, const char *option,
-                      const void *val, size_t size)
-{
-    struct local_connector *ctx = impl;
-    size_t val_size;
-    int rc = -1;
-
-    if (option && !strcmp (option, FLUX_OPT_TESTING_USERID)) {
-        val_size = sizeof (ctx->testing_userid);
-        if (size != val_size) {
-            errno = EINVAL;
-            goto done;
-        }
-        memcpy (&ctx->testing_userid, val, val_size);
-    } else if (option && !strcmp (option, FLUX_OPT_TESTING_ROLEMASK)) {
-        val_size = sizeof (ctx->testing_rolemask);
-        if (size != val_size) {
-            errno = EINVAL;
-            goto done;
-        }
-        memcpy (&ctx->testing_rolemask, val, val_size);
-    } else {
-        errno = EINVAL;
-        goto done;
-    }
-    rc = 0;
-done:
-    return rc;
-}
-
-static void op_fini (void *impl)
-{
-    struct local_connector *ctx = impl;
-
-    if (ctx) {
-        int saved_errno = errno;
-        usock_client_destroy (ctx->uclient);
-        if (ctx->fd >= 0)
-            ERRNO_SAFE_WRAP (close, ctx->fd);
-        free (ctx);
-        errno = saved_errno;
-    }
-}
-
-static int override_retry_count (struct usock_retry_params *retry)
-{
-    const char *s;
-
-    if ((s = getenv ("FLUX_LOCAL_CONNECTOR_RETRY_COUNT"))) {
-        char *endptr;
-        int n;
-
-        errno = 0;
-        n = strtol (s, &endptr, 10);
-        if (errno != 0 || *endptr != '\0') {
-            errno = EINVAL;
-            return  -1;
-        }
-        retry->max_retry = n;
-    }
-    return 0;
-}
-
-/* Path is interpreted as the directory containing the unix domain socket.
- */
-flux_t *connector_init (const char *path, int flags)
-{
-    struct local_connector *ctx;
-    struct usock_retry_params retry = USOCK_RETRY_DEFAULT;
-
-    if (!path || override_retry_count (&retry) < 0) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if (!(ctx = calloc (1, sizeof (*ctx))))
-        return NULL;
-
-    ctx->testing_userid = FLUX_USERID_UNKNOWN;
-    ctx->testing_rolemask = FLUX_ROLE_NONE;
-
-    ctx->fd = usock_client_connect (path, retry);
-    if (ctx->fd < 0)
-        goto error;
-    if (!(ctx->uclient = usock_client_create (ctx->fd)))
-        goto error;
-    if (!(ctx->h = flux_handle_create (ctx, &handle_ops, flags)))
-        goto error;
-    return ctx->h;
-error:
-    op_fini (ctx);
-    return NULL;
-}
-
-static const struct flux_handle_ops handle_ops = {
-    .pollfd = op_pollfd,
-    .pollevents = op_pollevents,
-    .send = op_send,
-    .recv = op_recv,
-    .event_subscribe = op_event_subscribe,
-    .event_unsubscribe = op_event_unsubscribe,
-    .setopt = op_setopt,
-    .getopt = NULL,
-    .impl_destroy = op_fini,
-};
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/connectors/loop/Makefile.am b/src/connectors/loop/Makefile.am
deleted file mode 100644
index 7a1d7b1efc41..000000000000
--- a/src/connectors/loop/Makefile.am
+++ /dev/null
@@ -1,27 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) \
-	$(LIBUUID_CFLAGS)
-
-fluxconnector_LTLIBRARIES = loop.la
-
-loop_la_SOURCES = loop.c
-
-loop_la_LDFLAGS = -module $(san_ld_zdef_flag) \
-	-export-symbols-regex '^connector_init$$' \
-	--disable-static -avoid-version -shared -export-dynamic
-
-loop_la_LIBADD = \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(ZMQ_LIBS) \
-	$(LIBUUID_LIBS)
diff --git a/src/connectors/loop/loop.c b/src/connectors/loop/loop.c
deleted file mode 100644
index 1c4c1b25a201..000000000000
--- a/src/connectors/loop/loop.c
+++ /dev/null
@@ -1,153 +0,0 @@
-/************************************************************\
- * Copyright 2015 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* loop connector - mainly for testing */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <assert.h>
-#include <errno.h>
-#include <unistd.h>
-
-#include <flux/core.h>
-
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/msglist.h"
-
-#define CTX_MAGIC   0xf434aaa0
-typedef struct {
-    int magic;
-    flux_t *h;
-
-    struct flux_msg_cred cred;
-
-    msglist_t *queue;
-} loop_ctx_t;
-
-
-static const struct flux_handle_ops handle_ops;
-
-const char *fake_uuid = "12345678123456781234567812345678";
-
-static int op_pollevents (void *impl)
-{
-    loop_ctx_t *c = impl;
-    int e, revents = 0;
-
-    if ((e = msglist_pollevents (c->queue)) < 0)
-        return e;
-    if (e & POLLIN)
-        revents |= FLUX_POLLIN;
-    if (e & POLLOUT)
-        revents |= FLUX_POLLOUT;
-    if (e & POLLERR)
-        revents |= FLUX_POLLERR;
-    return revents;
-}
-
-static int op_pollfd (void *impl)
-{
-    loop_ctx_t *c = impl;
-    return msglist_pollfd (c->queue);
-}
-
-static int op_send (void *impl, const flux_msg_t *msg, int flags)
-{
-    loop_ctx_t *c = impl;
-    assert (c->magic == CTX_MAGIC);
-    flux_msg_t *cpy = NULL;
-    struct flux_msg_cred cred;
-    int rc = -1;
-
-    if (!(cpy = flux_msg_copy (msg, true)))
-        goto done;
-    if (flux_msg_get_cred (cpy, &cred) < 0)
-        goto done;
-    if (cred.userid == FLUX_USERID_UNKNOWN)
-        cred.userid = c->cred.userid;
-    if (cred.rolemask == FLUX_ROLE_NONE)
-        cred.rolemask = c->cred.rolemask;
-    if (flux_msg_set_cred (cpy, cred) < 0)
-        goto done;
-    if (msglist_append (c->queue, cpy) < 0)
-        goto done;
-    cpy = NULL; /* c->queue now owns cpy */
-    rc = 0;
-done:
-    if (cpy)
-        flux_msg_destroy (cpy);
-    return rc;
-}
-
-static flux_msg_t *op_recv (void *impl, int flags)
-{
-    loop_ctx_t *c = impl;
-    assert (c->magic == CTX_MAGIC);
-    flux_msg_t *msg = msglist_pop (c->queue);
-    if (!msg)
-        errno = EWOULDBLOCK;
-    return msg;
-}
-
-static void op_fini (void *impl)
-{
-    loop_ctx_t *c = impl;
-    assert (c->magic == CTX_MAGIC);
-
-    msglist_destroy (c->queue);
-    c->magic = ~CTX_MAGIC;
-    free (c);
-}
-
-flux_t *connector_init (const char *path, int flags)
-{
-    loop_ctx_t *c = malloc (sizeof (*c));
-    if (!c) {
-        errno = ENOMEM;
-        goto error;
-    }
-    memset (c, 0, sizeof (*c));
-    c->magic = CTX_MAGIC;
-    if (!(c->queue = msglist_create ((msglist_free_f)flux_msg_destroy)))
-        goto error;
-    if (!(c->h = flux_handle_create (c, &handle_ops, flags)))
-        goto error;
-    /* Fake out size, rank, tbon-arity attributes for testing.
-     */
-    if (flux_attr_set_cacheonly(c->h, "rank", "0") < 0
-                || flux_attr_set_cacheonly (c->h, "size", "1") < 0
-                || flux_attr_set_cacheonly (c->h, "tbon-arity", "2") < 0)
-        goto error;
-    c->cred.userid = geteuid ();
-    c->cred.rolemask = FLUX_ROLE_OWNER;
-    return c->h;
-error:
-    if (c) {
-        int saved_errno = errno;
-        op_fini (c);
-        errno = saved_errno;
-    }
-    return NULL;
-}
-
-static const struct flux_handle_ops handle_ops = {
-    .pollfd = op_pollfd,
-    .pollevents = op_pollevents,
-    .send = op_send,
-    .recv = op_recv,
-    .getopt = NULL,
-    .setopt = NULL,
-    .impl_destroy = op_fini,
-};
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/connectors/shmem/Makefile.am b/src/connectors/shmem/Makefile.am
deleted file mode 100644
index d8c77e94f66b..000000000000
--- a/src/connectors/shmem/Makefile.am
+++ /dev/null
@@ -1,27 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) \
-	$(LIBUUID_CFLAGS)
-
-fluxconnector_LTLIBRARIES = shmem.la
-
-shmem_la_SOURCES = shmem.c
-
-shmem_la_LDFLAGS = -module $(san_ld_zdef_flag) \
-	-export-symbols-regex '^connector_init$$' \
-	--disable-static -avoid-version -shared -export-dynamic
-
-shmem_la_LIBADD = \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(ZMQ_LIBS) \
-	$(LIBUUID_LIBS)
diff --git a/src/connectors/shmem/shmem.c b/src/connectors/shmem/shmem.c
deleted file mode 100644
index 773e44983527..000000000000
--- a/src/connectors/shmem/shmem.c
+++ /dev/null
@@ -1,224 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <assert.h>
-#include <errno.h>
-#include <czmq.h>
-#include <argz.h>
-#if HAVE_CALIPER
-#include <caliper/cali.h>
-#endif
-#include <flux/core.h>
-
-#include "src/common/libutil/log.h"
-
-#define MODHANDLE_MAGIC    0xfeefbe02
-typedef struct {
-    int magic;
-    zsock_t *sock;
-    char *uuid;
-    flux_t *h;
-    char *argz;
-    size_t argz_len;
-} shmem_ctx_t;
-
-static const struct flux_handle_ops handle_ops;
-
-static int op_pollevents (void *impl)
-{
-    shmem_ctx_t *ctx = impl;
-    assert (ctx->magic == MODHANDLE_MAGIC);
-    uint32_t e;
-    int revents = 0;
-
-    e = zsock_events (ctx->sock);
-    if (e & ZMQ_POLLIN)
-        revents |= FLUX_POLLIN;
-    if (e & ZMQ_POLLOUT)
-        revents |= FLUX_POLLOUT;
-    if (e & ZMQ_POLLERR)
-        revents |= FLUX_POLLERR;
-
-    return revents;
-}
-
-static int op_pollfd (void *impl)
-{
-    shmem_ctx_t *ctx = impl;
-    assert (ctx->magic == MODHANDLE_MAGIC);
-
-    return zsock_fd (ctx->sock);
-}
-
-static int op_send (void *impl, const flux_msg_t *msg, int flags)
-{
-    shmem_ctx_t *ctx = impl;
-    assert (ctx->magic == MODHANDLE_MAGIC);
-
-    return flux_msg_sendzsock (ctx->sock, msg);
-}
-
-static flux_msg_t *op_recv (void *impl, int flags)
-{
-    shmem_ctx_t *ctx = impl;
-    assert (ctx->magic == MODHANDLE_MAGIC);
-    zmq_pollitem_t zp = {
-        .events = ZMQ_POLLIN,
-        .socket = zsock_resolve (ctx->sock),
-        .revents = 0,
-        .fd = -1,
-    };
-    flux_msg_t *msg = NULL;
-
-    if ((flags & FLUX_O_NONBLOCK)) {
-        int n;
-        if ((n = zmq_poll (&zp, 1, 0L)) <= 0) {
-            if (n == 0)
-                errno = EWOULDBLOCK;
-            goto done;
-        }
-    }
-    msg = flux_msg_recvzsock (ctx->sock);
-done:
-    return msg;
-}
-
-static int op_event_subscribe (void *impl, const char *topic)
-{
-    shmem_ctx_t *ctx = impl;
-    assert (ctx->magic == MODHANDLE_MAGIC);
-    flux_future_t *f;
-    int rc = -1;
-
-    if (!(f = flux_rpc_pack (ctx->h, "cmb.sub", FLUX_NODEID_ANY, 0,
-                             "{ s:s }", "topic", topic)))
-        goto done;
-    if (flux_future_get (f, NULL) < 0)
-        goto done;
-    rc = 0;
-done:
-    flux_future_destroy (f);
-    return rc;
-}
-
-static int op_event_unsubscribe (void *impl, const char *topic)
-{
-    shmem_ctx_t *ctx = impl;
-    assert (ctx->magic == MODHANDLE_MAGIC);
-    flux_future_t *f = NULL;
-    int rc = -1;
-
-    if (!(f = flux_rpc_pack (ctx->h, "cmb.unsub", FLUX_NODEID_ANY, 0,
-                             "{ s:s }", "topic", topic)))
-        goto done;
-    if (flux_future_get (f, NULL) < 0)
-        goto done;
-    rc = 0;
-done:
-    flux_future_destroy (f);
-    return rc;
-}
-
-
-static void op_fini (void *impl)
-{
-    shmem_ctx_t *ctx = impl;
-    assert (ctx->magic == MODHANDLE_MAGIC);
-    zsock_destroy (&ctx->sock);
-    free (ctx->argz);
-    ctx->magic = ~MODHANDLE_MAGIC;
-    free (ctx);
-}
-
-flux_t *connector_init (const char *path, int flags)
-{
-#if HAVE_CALIPER
-    cali_id_t uuid   = cali_create_attribute ("flux.uuid",
-                                              CALI_TYPE_STRING,
-                                              CALI_ATTR_SKIP_EVENTS);
-    size_t length = strlen(path);
-    cali_push_snapshot ( CALI_SCOPE_PROCESS | CALI_SCOPE_THREAD,
-                         1, &uuid, (const void **)&path, &length);
-#endif
-
-    shmem_ctx_t *ctx = NULL;
-    char *item;
-    int e;
-    int bind_socket = 0; // if set, call bind on socket, else connect
-
-    if (!path) {
-        errno = EINVAL;
-        goto error;
-    }
-    if (!(ctx = calloc (1, sizeof (*ctx)))) {
-        errno = ENOMEM;
-        goto error;
-    }
-    ctx->magic = MODHANDLE_MAGIC;
-    if ((e = argz_create_sep (path, '&', &ctx->argz, &ctx->argz_len)) != 0) {
-        errno = e;
-        goto error;
-    }
-    ctx->uuid = item = argz_next (ctx->argz, ctx->argz_len, NULL);
-    if (!ctx->uuid) {
-        errno = EINVAL;
-        goto error;
-    }
-    while ((item = argz_next (ctx->argz, ctx->argz_len, item))) {
-        if (!strcmp (item, "bind"))
-            bind_socket = 1;
-        else if (!strcmp (item, "connect"))
-            bind_socket = 0;
-        else {
-            errno = EINVAL;
-            goto error;
-        }
-    }
-    if (!(ctx->sock = zsock_new_pair (NULL)))
-        goto error;
-    zsock_set_unbounded (ctx->sock);
-    if (bind_socket) {
-        if (zsock_bind (ctx->sock, "inproc://%s", ctx->uuid) < 0)
-            goto error;
-    }
-    else {
-        if (zsock_connect (ctx->sock, "inproc://%s", ctx->uuid) < 0)
-            goto error;
-    }
-    if (!(ctx->h = flux_handle_create (ctx, &handle_ops, flags)))
-        goto error;
-    return ctx->h;
-error:
-    if (ctx) {
-        int saved_errno = errno;
-        op_fini (ctx);
-        errno = saved_errno;
-    }
-    return NULL;
-}
-
-static const struct flux_handle_ops handle_ops = {
-    .pollfd = op_pollfd,
-    .pollevents = op_pollevents,
-    .send = op_send,
-    .recv = op_recv,
-    .getopt = NULL,
-    .setopt = NULL,
-    .event_subscribe = op_event_subscribe,
-    .event_unsubscribe = op_event_unsubscribe,
-    .impl_destroy = op_fini,
-};
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/connectors/ssh/Makefile.am b/src/connectors/ssh/Makefile.am
deleted file mode 100644
index a6b3082e2a1d..000000000000
--- a/src/connectors/ssh/Makefile.am
+++ /dev/null
@@ -1,27 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) \
-	$(LIBUUID_CFLAGS)
-
-fluxconnector_LTLIBRARIES = ssh.la
-
-ssh_la_SOURCES = ssh.c
-
-ssh_la_LDFLAGS = -module $(san_ld_zdef_flag) \
-	-export-symbols-regex '^connector_init$$' \
-	--disable-static -avoid-version -shared -export-dynamic
-
-ssh_la_LIBADD = \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(ZMQ_LIBS) \
-	$(LIBUUID_LIBS)
diff --git a/src/connectors/ssh/ssh.c b/src/connectors/ssh/ssh.c
index 229bd478c0fb..52553ce3630c 100644
--- a/src/connectors/ssh/ssh.c
+++ b/src/connectors/ssh/ssh.c
@@ -11,23 +11,28 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <assert.h>
-#include <errno.h>
-#include <stdbool.h>
-#include <sys/types.h>
 #include <sys/param.h>
-#include <sys/un.h>
-#include <sys/socket.h>
-#include <signal.h>
-#include <poll.h>
 #include <unistd.h>
+#ifdef HAVE_ARGZ_ADD
 #include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <libgen.h>
 #include <flux/core.h>
 
-#include "src/common/libutil/log.h"
 #include "src/common/libutil/popen2.h"
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/strstrip.h"
 #include "src/common/libyuarel/yuarel.h"
+#ifndef HAVE_STRLCPY
+#include "src/common/libmissing/strlcpy.h"
+#endif
+#ifndef HAVE_STRLCAT
+#include "src/common/libmissing/strlcat.h"
+#endif
 #include "src/common/librouter/usock.h"
 
 struct ssh_connector {
@@ -66,40 +71,6 @@ static flux_msg_t *op_recv (void *impl, int flags)
     return usock_client_recv (ctx->uclient, flags);
 }
 
-static int op_event_subscribe (void *impl, const char *topic)
-{
-    struct ssh_connector *ctx = impl;
-    flux_future_t *f;
-    int rc = 0;
-
-    if (!(f = flux_rpc_pack (ctx->h, "local.sub", FLUX_NODEID_ANY, 0,
-                             "{ s:s }", "topic", topic)))
-        goto done;
-    if (flux_future_get (f, NULL) < 0)
-        goto done;
-    rc = 0;
-done:
-    flux_future_destroy (f);
-    return rc;
-}
-
-static int op_event_unsubscribe (void *impl, const char *topic)
-{
-    struct ssh_connector *ctx = impl;
-    flux_future_t *f;
-    int rc = 0;
-
-    if (!(f = flux_rpc_pack (ctx->h, "local.unsub", FLUX_NODEID_ANY, 0,
-                             "{ s:s }", "topic", topic)))
-        goto done;
-    if (flux_future_get (f, NULL) < 0)
-        goto done;
-    rc = 0;
-done:
-    flux_future_destroy (f);
-    return rc;
-}
-
 static void op_fini (void *impl)
 {
     struct ssh_connector *ctx = impl;
@@ -113,7 +84,7 @@ static void op_fini (void *impl)
     }
 }
 
-static char *which (const char *prog, char *buf, size_t size)
+static const char *which_dir (const char *prog, char *buf, size_t size)
 {
     char *path = getenv ("PATH");
     char *cpy = path ? strdup (path) : NULL;
@@ -124,9 +95,10 @@ static char *which (const char *prog, char *buf, size_t size)
     if (cpy) {
         while ((dir = strtok_r (a1, ":", &saveptr))) {
             snprintf (buf, size, "%s/%s", dir, prog);
-            if (stat (buf, &sb) == 0 && S_ISREG (sb.st_mode)
-                                     && access (buf, X_OK) == 0) {
-                result = buf;
+            if (stat (buf, &sb) == 0
+                && S_ISREG (sb.st_mode)
+                && access (buf, X_OK) == 0) {
+                result = dirname (buf);
                 break;
             }
             a1 = NULL;
@@ -136,6 +108,46 @@ static char *which (const char *prog, char *buf, size_t size)
     return result;
 }
 
+static int make_path (char *path, size_t size, const char *sockpath)
+{
+    char *sockpath_cpy;
+    char buf[1024];
+    const char *rundir;
+    const char *bindir;
+    int rc = -1;
+
+    if (strlcpy (path, "PATH=", size) >= size
+        || !(sockpath_cpy = strdup (sockpath)))
+        return -1;
+
+    // append rundir/bin
+    rundir = dirname (sockpath_cpy);
+    if (rundir[0] != '/') {
+        if (strlcat (path, "/", size) >= size)
+            goto error;
+    }
+    if (strlcat (path, rundir, size) >= size)
+        goto error;
+    if (strlcat (path, "/bin", size) >= size)
+        goto error;
+
+    // append directory in which flux(1) was found locally
+    if ((bindir = which_dir ("flux", buf, sizeof (buf)))) {
+        if (strlcat (path, ":", size) >= size)
+            goto error;
+        if (strlcat (path, bindir, size) >= size)
+            goto error;
+    }
+
+    // append system bin so libtool wrappers can work if necessary
+    if (strlcat (path, ":/bin:/usr/bin", size) >= size)
+        goto error;
+    rc = 0;
+error:
+    free (sockpath_cpy);
+    return rc;
+}
+
 /* uri_path is interpreted as:
  *   [user@]hostname[:port]/unix-path
  * Sets *argvp, *argbuf (caller must free).
@@ -145,6 +157,7 @@ static char *which (const char *prog, char *buf, size_t size)
 int build_ssh_command (const char *uri_path,
                        const char *ssh_cmd,
                        const char *flux_cmd,
+                       const char *ld_lib_path,
                        char ***argvp,
                        char **argbuf)
 {
@@ -188,6 +201,27 @@ int build_ssh_command (const char *uri_path,
         if (argz_add (&argz, &argz_len, yuri.host) != 0)
             goto nomem;
     }
+
+    /* [env] */
+    if (ld_lib_path || !flux_cmd) {
+        if (argz_add (&argz, &argz_len, "env") != 0)
+            goto nomem;
+    }
+    /* [PATH=remote_path] */
+    if (!flux_cmd) {
+        if (make_path (buf, sizeof (buf), yuri.path) == 0) {
+            if (argz_add (&argz, &argz_len, buf) != 0)
+                goto nomem;
+        }
+        flux_cmd = "flux";
+    }
+    /* [LD_LIBRARY_PATH=ld_lib_path] */
+    if (ld_lib_path) {
+        (void)snprintf (buf, sizeof (buf), "LD_LIBRARY_PATH=%s", ld_lib_path);
+        if (argz_add (&argz, &argz_len, buf) != 0)
+            goto nomem;
+    }
+
     /* flux-relay */
     if (argz_add (&argz, &argz_len, flux_cmd) != 0)
         goto nomem;
@@ -220,14 +254,15 @@ int build_ssh_command (const char *uri_path,
     return -1;
 }
 
-flux_t *connector_init (const char *path, int flags)
+flux_t *connector_init (const char *path, int flags, flux_error_t *errp)
 {
     struct ssh_connector *ctx;
-    char buf[PATH_MAX + 1];
     const char *ssh_cmd;
     const char *flux_cmd;
+    const char *ld_lib_path;
     char *argbuf = NULL;
     char **argv = NULL;
+    int popen_flags = 0;
 
     if (!(ctx = calloc (1, sizeof (*ctx))))
         return NULL;
@@ -237,29 +272,49 @@ flux_t *connector_init (const char *path, int flags)
      */
     if (!(ssh_cmd = getenv ("FLUX_SSH")))
         ssh_cmd = PATH_SSH;
-    /* FLUX_SSH_RCMD may be used to select a different path to the flux
-     * command front end than the default.  The default is to use the one
-     * from the client's PATH.
+    /* FLUX_SSH_RCMD may be used to force a specific path to the flux
+     * command front end.
      */
     flux_cmd = getenv ("FLUX_SSH_RCMD");
-    if (!flux_cmd)
-        flux_cmd = which ("flux", buf, sizeof (buf));
-    if (!flux_cmd)
-        flux_cmd = "flux"; // maybe this will work for installed version
+
+    /* ssh and rsh do not forward environment variables, thus LD_LIBRARY_PATH
+     * is not guaranteed to be set on the remote node where the flux command is
+     * run.  If the flux command is linked against libraries that can only be
+     * found when LD_LIBRARY_PATH is set, then the flux command will fail to
+     * run over ssh.  Grab the client-side LD_LIBRARY_PATH so that we can
+     * manually forward it. See flux-core issue #3457 for more details.
+     */
+    ld_lib_path = getenv ("LD_LIBRARY_PATH");
 
     /* Construct argv for ssh command from uri "path" (non-scheme part)
      * and flux and ssh command paths.
      */
-    if (build_ssh_command (path, ssh_cmd, flux_cmd, &argv, &argbuf) < 0)
+    if (build_ssh_command (path,
+                           ssh_cmd,
+                           flux_cmd,
+                           ld_lib_path,
+                           &argv,
+                           &argbuf) < 0)
         goto error;
 
     /* Start the ssh command
      */
-    if (!(ctx->p = popen2 (ssh_cmd, argv))) {
+    if (errp)
+        popen_flags = POPEN2_CAPTURE_STDERR;
+    if (!(ctx->p = popen2 (ssh_cmd, argv, popen_flags))) {
         /* If popen fails because ssh cannot be found, flux_open()
          * will just fail with errno = ENOENT, which is not all that helpful.
          * Emit a hint on stderr even though this is perhaps not ideal.
+         * (if errp is non-NULL then use that instead)
          */
+        if (errp) {
+            errprintf (errp,
+                       "ssh-connector: %s: %s\n"
+                       "Hint: set FLUX_SSH in environment to override",
+                       ssh_cmd,
+                       strerror (errno));
+            goto error;
+        }
         fprintf (stderr, "ssh-connector: %s: %s\n", ssh_cmd, strerror (errno));
         fprintf (stderr, "Hint: set FLUX_SSH in environment to override\n");
         goto error;
@@ -270,8 +325,13 @@ flux_t *connector_init (const char *path, int flags)
      * but performing this handshake forces some errors to be handled here
      * inside flux_open() rather than in some less obvious context later.
      */
-    if (!(ctx->uclient = usock_client_create (popen2_get_fd (ctx->p))))
+    if (!(ctx->uclient = usock_client_create (popen2_get_fd (ctx->p)))) {
+        char *data = NULL;
+        if (read_all (popen2_get_stderr_fd (ctx->p), (void **) &data) > 0)
+            errprintf (errp, "%s", strstrip (data));
+        free (data);
         goto error;
+    }
     if (!(ctx->h = flux_handle_create (ctx, &handle_ops, flags)))
         goto error;
     free (argbuf);
@@ -294,8 +354,6 @@ static const struct flux_handle_ops handle_ops = {
     .pollevents = op_pollevents,
     .send = op_send,
     .recv = op_recv,
-    .event_subscribe = op_event_subscribe,
-    .event_unsubscribe = op_event_unsubscribe,
     .impl_destroy = op_fini,
 };
 
diff --git a/src/include/flux/core.h b/src/include/flux/core.h
index 7d357100a892..6bd7b2176138 100644
--- a/src/include/flux/core.h
+++ b/src/include/flux/core.h
@@ -18,5 +18,6 @@
 #include "src/common/libkvs/kvs.h"
 #include "src/common/libsubprocess/subprocess.h"
 #include "src/common/libjob/job.h"
+#include "src/common/libjob/jobspec1.h"
 
 #endif /* FLUX_CORE_H */
diff --git a/src/include/flux/hostlist.h b/src/include/flux/hostlist.h
new file mode 100644
index 000000000000..70986da7dd76
--- /dev/null
+++ b/src/include/flux/hostlist.h
@@ -0,0 +1,14 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Allow in-tree programs to #include <flux/hostlist.h> like out-of-tree would.
+ */
+
+#include "src/common/libhostlist/hostlist.h"
diff --git a/src/include/flux/jobtap.h b/src/include/flux/jobtap.h
new file mode 100644
index 000000000000..5676d92ddfd9
--- /dev/null
+++ b/src/include/flux/jobtap.h
@@ -0,0 +1,14 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Allow in-tree programs to #include <flux/jobtap.h> like out-of-tree would.
+ */
+
+#include "src/modules/job-manager/jobtap.h"
diff --git a/src/include/flux/taskmap.h b/src/include/flux/taskmap.h
new file mode 100644
index 000000000000..88a039554853
--- /dev/null
+++ b/src/include/flux/taskmap.h
@@ -0,0 +1,14 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Allow in-tree programs to #include <flux/taskmap.h> like out-of-tree would.
+ */
+
+#include "src/common/libtaskmap/taskmap.h"
diff --git a/src/modules/Makefile.am b/src/modules/Makefile.am
index 9072602e5a2f..f1a1d7bd3e20 100644
--- a/src/modules/Makefile.am
+++ b/src/modules/Makefile.am
@@ -1,15 +1,291 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_builddir)/src/common/libflux \
+	-I$(top_srcdir)/src/common/libccan \
+	$(JANSSON_CFLAGS)
+
 SUBDIRS = \
- barrier \
- connector-local \
- kvs \
- kvs-watch \
- content-sqlite \
- cron \
- aggregator \
- userdb \
- pymod \
- job-ingest \
- job-manager \
- job-info \
- job-exec \
- sched-simple
+	kvs \
+	content-files \
+	job-ingest \
+	job-manager \
+	job-list \
+	job-exec \
+	resource \
+	sdbus
+
+if ENABLE_CONTENT_S3
+SUBDIRS += content-s3
+endif
+
+fluxmod_LTLIBRARIES = \
+	barrier.la \
+	connector-local.la \
+	content.la \
+	content-files.la \
+	content-sqlite.la \
+	cron.la \
+	heartbeat.la \
+	job-exec.la \
+	job-info.la \
+	job-list.la \
+	job-ingest.la \
+	job-manager.la \
+	kvs.la \
+	kvs-watch.la \
+	resource.la \
+	sched-simple.la \
+	sdexec.la \
+	sdbus.la
+
+if ENABLE_CONTENT_S3
+fluxmod_LTLIBRARIES += content-s3.la
+endif
+
+# N.B. SOURCES should be empty when module subdir is listed in
+# SUBDIRS above.  This avoids distcheck problems with older automake.
+# Hint: build convenience library in subdir and reference that in LIBADD.
+
+barrier_la_SOURCES = \
+	barrier/barrier.c
+barrier_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+barrier_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+connector_local_la_SOURCES = \
+	connector-local/local.c
+connector_local_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+connector_local_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+content_la_SOURCES = \
+	content/main.c \
+	content/cache.c \
+	content/cache.h \
+	content/mmap.c \
+	content/mmap.h \
+	content/checkpoint.c \
+	content/checkpoint.h
+content_la_LIBADD = \
+	$(top_builddir)/src/common/libfilemap/libfilemap.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(LIBARCHIVE_LIBS)
+content_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+content_files_la_SOURCES =
+content_files_la_LIBADD = \
+	$(builddir)/content-files/libcontent-files.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+content_files_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+if ENABLE_CONTENT_S3
+content_s3_la_SOURCES =
+content_s3_la_LIBADD = \
+	$(builddir)/content-s3/libcontent-s3.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(LIBS3)
+content_s3_la_LDFLAGS = $(fluxmod_ldflags) -module
+endif
+
+content_sqlite_la_SOURCES = \
+        content-sqlite/content-sqlite.c
+content_sqlite_la_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(SQLITE_CFLAGS) \
+	$(LZ4_CFLAGS)
+content_sqlite_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(SQLITE_LIBS) \
+	$(LZ4_LIBS)
+content_sqlite_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+cron_la_SOURCES = \
+        cron/cron.c \
+	cron/task.h \
+	cron/task.c \
+	cron/entry.h \
+	cron/types.h \
+	cron/types.c \
+	cron/interval.c \
+	cron/event.c \
+	cron/datetime.c
+cron_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(JANSSON_LIBS)
+cron_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+heartbeat_la_SOURCES = \
+	heartbeat/heartbeat.c
+heartbeat_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+heartbeat_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+job_exec_la_SOURCES =
+job_exec_la_LIBADD = \
+	$(builddir)/job-exec/libjob-exec.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(JANSSON_LIBS)
+job_exec_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+job_info_la_SOURCES = \
+	job-info/job-info.h \
+	job-info/job-info.c \
+	job-info/allow.h \
+	job-info/allow.c \
+	job-info/lookup.h \
+	job-info/lookup.c \
+	job-info/watch.h \
+	job-info/watch.c \
+	job-info/guest_watch.h \
+	job-info/guest_watch.c \
+	job-info/update.h \
+	job-info/update.c \
+	job-info/util.h \
+	job-info/util.c
+job_info_la_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(FLUX_SECURITY_CFLAGS)
+job_info_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(JANSSON_LIBS)
+job_info_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+job_list_la_SOURCES =
+job_list_la_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(FLUX_SECURITY_CFLAGS) \
+	$(HWLOC_CFLAGS)
+job_list_la_LIBADD = \
+	$(builddir)/job-list/libjob-list.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-optparse.la \
+	$(top_builddir)/src/common/librlist/librlist.la \
+	$(JANSSON_LIBS)
+job_list_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+job_ingest_la_SOURCES =
+job_ingest_la_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(HWLOC_CFLAGS)
+job_ingest_la_LIBADD = \
+	$(builddir)/job-ingest/libingest.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-optparse.la \
+	$(JANSSON_LIBS) \
+	$(FLUX_SECURITY_LIBS)
+job_ingest_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+job_manager_la_SOURCES =
+job_manager_la_LIBADD = \
+	$(builddir)/job-manager/libjob-manager.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-optparse.la \
+	$(top_builddir)/src/common/librlist/librlist.la \
+	$(JANSSON_LIBS)
+job_manager_la_LDFLAGS = \
+	$(fluxlib_ldflags) \
+	-avoid-version \
+	-export-symbols-regex \
+	'^(flux_jobtap.*|mod_main)$$' \
+	-module
+
+kvs_la_SOURCES =
+kvs_la_LIBADD = \
+	$(builddir)/kvs/libkvs.la \
+	$(top_builddir)/src/common/libkvs/libkvs.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(JANSSON_LIBS)
+kvs_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+kvs_watch_la_SOURCES = \
+	kvs-watch/kvs-watch.c
+kvs_watch_la_LIBADD = \
+	$(top_builddir)/src/common/libkvs/libkvs.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-optparse.la \
+	$(JANSSON_LIBS)
+kvs_watch_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+resource_la_SOURCES =
+resource_la_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(HWLOC_CFLAGS)
+resource_la_LIBADD = \
+	$(builddir)/resource/libresource.la \
+	$(top_builddir)/src/common/librlist/librlist-hwloc.la \
+	$(top_builddir)/src/common/librlist/librlist.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(JANSSON_LIBS) \
+	$(HWLOC_LIBS)
+resource_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+sched_simple_la_SOURCES = \
+	sched-simple/sched.c
+sched_simple_la_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(HWLOC_CFLAGS)
+sched_simple_la_LIBADD = \
+	$(top_builddir)/src/common/librlist/librlist.la \
+	$(top_builddir)/src/common/libschedutil/libschedutil.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-optparse.la \
+	$(JANSSON_LIBS)
+sched_simple_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+sdexec_la_SOURCES = \
+	sdexec/sdexec.c
+sdexec_la_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(LIBUUID_CFLAGS)
+sdexec_la_LIBADD = \
+	$(top_builddir)/src/common/libsdexec/libsdexec.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(LIBUUID_LIBS) \
+	$(JANSSON_LIBS)
+sdexec_la_LDFLAGS = $(fluxmod_ldflags) -module
+
+sdbus_la_SOURCES =
+sdbus_la_LIBADD = \
+	$(builddir)/sdbus/libsdbus.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(JANSSON_LIBS)
+if HAVE_LIBSYSTEMD
+sdbus_la_LIBADD += $(LIBSYSTEMD_LIBS)
+endif
+sdbus_la_LDFLAGS = $(fluxmod_ldflags) -module
diff --git a/src/modules/aggregator/Makefile.am b/src/modules/aggregator/Makefile.am
deleted file mode 100644
index 68a53447b8a1..000000000000
--- a/src/modules/aggregator/Makefile.am
+++ /dev/null
@@ -1,23 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
-
-#
-# Comms module
-#
-fluxmod_LTLIBRARIES = aggregator.la
-
-aggregator_la_SOURCES = aggregator.c
-aggregator_la_LDFLAGS = $(fluxmod_ldflags) -module
-aggregator_la_LIBADD = $(top_builddir)/src/common/libflux-internal.la \
-		 $(top_builddir)/src/common/libflux-core.la \
-		 $(ZMQ_LIBS)
diff --git a/src/modules/aggregator/aggregator.c b/src/modules/aggregator/aggregator.c
deleted file mode 100644
index c801f8e36da2..000000000000
--- a/src/modules/aggregator/aggregator.c
+++ /dev/null
@@ -1,698 +0,0 @@
-/************************************************************\
- * Copyright 2016 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* aggregator.c - reduction based numerical aggreagator */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdio.h>
-#include <flux/core.h>
-#include <czmq.h>
-#include <jansson.h>
-
-#include "src/common/libidset/idset.h"
-
-struct aggregator {
-    flux_t *h;
-    uint32_t rank;
-    double default_timeout;
-    double timer_scale;
-    zhash_t *aggregates;
-};
-
-/*
- *  Single entry in an aggregate: a list of ids with a common value.
- */
-struct aggregate_entry {
-    struct idset *ids;
-    json_t *value;
-};
-
-
-/*
- *  Representation of an aggregate. A unique kvs key, along with a
- *   list of aggregate entries as above. Each aggregate tracks its
- *   summary stats, current count and expected total of entries.
- */
-struct aggregate {
-    struct aggregator *ctx;  /* Pointer back to containing aggregator        */
-    flux_watcher_t *tw;      /* timeout watcher                              */
-    double timeout;          /* timeout                                      */
-    int sink_retries;        /* number of times left to try to sink to kvs   */
-    uint32_t fwd_count;      /* forward at this many                         */
-    char *key;               /* KVS key into which to sink the aggregate     */
-    uint32_t count;          /* count of current total entries               */
-    uint32_t total;          /* expected total entries (used for sink)       */
-    zlist_t *entries;        /* list of individual entries                   */
-    json_t *summary;         /* optional summary stats for this aggregate    */
-};
-
-static void aggregate_entry_destroy (struct aggregate_entry *ae)
-{
-    if (ae) {
-        int saved_errno = errno;
-        idset_destroy (ae->ids);
-        free (ae);
-        errno = saved_errno;
-    }
-}
-
-static struct aggregate_entry * aggregate_entry_create (void)
-{
-    struct aggregate_entry *ae = calloc (1, sizeof (*ae));
-    if (!ae)
-        return (NULL);
-    if (!(ae->ids = idset_create (0, IDSET_FLAG_AUTOGROW))) {
-        aggregate_entry_destroy (ae);
-        return (NULL);
-    }
-    return (ae);
-}
-
-
-/*  Search this aggregates entries for a value. Return entry if found
- */
-static struct aggregate_entry *
-    aggregate_entry_find (struct aggregate *ag, json_t *value)
-{
-    struct aggregate_entry *ae = zlist_first (ag->entries);
-    while (ae) {
-        if (json_equal (ae->value, value))
-            return (ae);
-        ae = zlist_next (ag->entries);
-    }
-    return (NULL);
-}
-
-static int summarize_real (struct aggregate *ag, json_t *value)
-{
-    double v = json_real_value (value);
-    double min, max;
-    if (!ag->summary) {
-        ag->summary = json_pack ("{s:f,s:f}", "min", v, "max", v);
-        return ag->summary ? 0 : -1;
-    }
-    if (json_unpack (ag->summary, "{s:F,s:F}", "min", &min, "max", &max) < 0) {
-        flux_log (ag->ctx->h, LOG_ERR, "summarize_real: json_unpack failed");
-        return (-1);
-    }
-    if (((max < v) && (json_object_set (ag->summary, "max", value) < 0))
-        || ((min > v) && (json_object_set (ag->summary, "min", value) < 0))) {
-        flux_log_error (ag->ctx->h, "summarize_real: json_object_set");
-        return (-1);
-    }
-    return (0);
-}
-
-static int summarize_int (struct aggregate *ag, json_t *value)
-{
-    int64_t v = json_integer_value (value);
-    int64_t min, max;
-    if (!ag->summary) {
-        ag->summary = json_pack ("{s:I,s:I}", "min", v, "max", v);
-        return ag->summary ? 0 : -1;
-    }
-    if (json_unpack (ag->summary, "{s:I,s:I}", "min", &min, "max", &max) < 0) {
-        flux_log_error (ag->ctx->h, "summarize_int: json_unpack");
-        return (-1);
-    }
-    if (((max < v) && (json_object_set (ag->summary, "max", value) < 0))
-        || ((min > v) && (json_object_set (ag->summary, "min", value) < 0))) {
-        flux_log_error (ag->ctx->h, "summarize_int: json_object_set");
-        return (-1);
-    }
-    return (0);
-}
-
-static int aggregate_update_summary (struct aggregate *ag, json_t *value)
-{
-    switch (json_typeof (value)) {
-        case JSON_INTEGER:
-            return summarize_int (ag, value);
-        case JSON_REAL:
-            return summarize_real (ag, value);
-        case JSON_STRING:
-        case JSON_OBJECT:
-        case JSON_ARRAY:
-        case JSON_TRUE:
-        case JSON_FALSE:
-        case JSON_NULL:
-            /* Currently no summary stats for these types */
-            return (0);
-
-    }
-    return (0);
-}
-
-/*  Add a new aggregate entry to this aggregate.
- *   Update summary stats if update == true.
- */
-static struct aggregate_entry *
-    aggregate_entry_add (struct aggregate *ag, json_t *value)
-{
-    struct aggregate_entry *ae = aggregate_entry_create ();
-    if (ae) {
-        json_incref (value);
-        ae->value = value;
-
-        /* Update aggregate summary statistics on rank 0 only */
-        if (ag->ctx->rank == 0 && aggregate_update_summary (ag, value) < 0)
-            flux_log_error (ag->ctx->h, "aggregate_update_summary");
-        zlist_push (ag->entries, ae);
-    }
-    return (ae);
-}
-
-int add_string_to_idset (struct idset *idset, const char *s)
-{
-    struct idset *nids;
-    unsigned int id;
-    int rc = -1;
-
-    if (!(nids = idset_decode (s)))
-        return (-1);
-    id = idset_first (nids);
-    while (id != IDSET_INVALID_ID) {
-        if (idset_set (idset, id) < 0)
-            goto done;
-        id = idset_next (nids, id);
-    }
-    rc = 0;
-done:
-    idset_destroy (nids);
-    return rc;
-}
-
-/*  Push a new (ids, value) pair onto aggregate `ag`.
- *   If an existing matching entry is found, add ids to its nodeset.
- *   o/w, add a new entry. In either case update current count with
- *   the number of `ids` added.
- */
-static int aggregate_push (struct aggregate *ag, json_t *value, const char *ids)
-{
-    int count;
-    struct aggregate_entry *ae = aggregate_entry_find (ag, value);
-    if ((ae == NULL) && !(ae = aggregate_entry_add (ag, value)))
-        return (-1);
-
-    count = idset_count (ae->ids);
-    if (add_string_to_idset (ae->ids, ids) < 0)
-        return (-1);
-
-    /* Update count */
-    ag->count += (idset_count (ae->ids) - count);
-
-    return (0);
-}
-
-/*  Push JSON object of aggregate entries onto aggregate `ag`
- */
-static int aggregate_push_json (struct aggregate *ag,
-                                json_t *entries)
-{
-    const char *ids;
-    json_t *val;
-
-    json_object_foreach (entries, ids, val) {
-        if (aggregate_push (ag, val, ids) < 0) {
-            flux_log_error (ag->ctx->h, "aggregate_push failed");
-            return (-1);
-        }
-    }
-    return (0);
-}
-
-static int set_json_object_new_idset_key (json_t *o, struct idset *key,
-                                          json_t *value)
-{
-    char *s;
-    int rc;
-
-    if (!(s = idset_encode (key, IDSET_FLAG_RANGE | IDSET_FLAG_BRACKETS)))
-        return (-1);
-    rc = json_object_set_new (o, s, value);
-    free (s);
-    return (rc);
-}
-
-/*  Return json object containing all "entries" from the current
- *   aggregate object `ag`
- */
-static json_t *aggregate_entries_tojson (struct aggregate *ag)
-{
-    struct aggregate_entry *ae;
-    json_t *entries = NULL;
-
-    if (!(entries = json_object ()))
-        return NULL;
-
-    ae = zlist_first (ag->entries);
-    while (ae) {
-        if (set_json_object_new_idset_key (entries, ae->ids, ae->value) < 0)
-            goto error;
-        ae = zlist_next (ag->entries);
-    }
-    return (entries);
-error:
-    json_decref (entries);
-    return (NULL);
-}
-
-static void forward_continuation (flux_future_t *f, void *arg)
-{
-    flux_t *h = flux_future_get_flux (f);
-    struct aggregate *ag = arg;
-    if (flux_rpc_get (f, NULL) < 0)
-        flux_log_error (h, "aggregator.push: key=%s", ag->key);
-    flux_future_destroy (f);
-}
-
-/*
- *  Forward aggregate `ag` upstream
- */
-static int aggregate_forward (flux_t *h, struct aggregate *ag)
-{
-    int rc = 0;
-    flux_future_t *f;
-    json_t *o = aggregate_entries_tojson (ag);
-
-    if (o == NULL) {
-        flux_log (h, LOG_ERR, "forward: aggregate_entries_tojson failed");
-        return (-1);
-    }
-    flux_log (h, LOG_DEBUG, "forward: %s: count=%d total=%d",
-                 ag->key, ag->count, ag->total);
-    if (!(f = flux_rpc_pack (h, "aggregator.push", FLUX_NODEID_UPSTREAM, 0,
-                                "{s:s,s:i,s:i,s:f,s:o}",
-                                "key", ag->key,
-                                "count", ag->count,
-                                "total", ag->total,
-                                "timeout", ag->timeout,
-                                "entries", o)) ||
-        (flux_future_then (f, -1., forward_continuation, (void *) ag) < 0)) {
-        flux_log_error (h, "flux_rpc: aggregator.push");
-        flux_future_destroy (f);
-        rc = -1;
-    }
-    return (rc);
-}
-
-static void aggregate_sink_abort (flux_t *h, struct aggregate *ag)
-{
-    flux_msg_t *msg = NULL;
-    char *topic = NULL;
-
-    flux_log (h, LOG_ERR, "sink: aborting aggregate %s\n", ag->key);
-
-    if ((asprintf (&topic, "aggregator.abort.%s", ag->key)) < 0) {
-        flux_log_error (h, "sink_abort: asprintf");
-        goto out;
-    }
-    if ((msg = flux_event_encode (topic, "{ }")) == NULL) {
-        flux_log_error (h, "flux_event_encode");
-        goto out;
-    }
-    if (flux_send (h, msg, 0) < 0)
-        flux_log_error (h, "flux_event_encode");
-out:
-    free (topic);
-    flux_msg_destroy (msg);
-}
-
-static void aggregate_sink (flux_t *h, struct aggregate *ag);
-
-static void aggregate_sink_again (flux_reactor_t *r, flux_watcher_t *w,
-                                 int revents, void *arg)
-{
-    struct aggregate *ag = arg;
-    aggregate_sink (ag->ctx->h, ag);
-    flux_watcher_destroy (w);
-}
-
-static int sink_retry (flux_t *h, struct aggregate *ag)
-{
-    flux_watcher_t *w;
-    double t = ag->timeout;
-    if (t <= 1e-3)
-        t = .250;
-
-    /* Return with error if we're out of retries */
-    if (--ag->sink_retries <= 0)
-        return (-1);
-
-    flux_log (h, LOG_DEBUG, "sink: %s: retry  in %.3fs", ag->key, t);
-    w = flux_timer_watcher_create (flux_get_reactor (h),
-                                   t, 0.,
-                                   aggregate_sink_again,
-                                   (void *) ag);
-    if (w == NULL) {
-        flux_log_error (h, "sink_retry: flux_timer_watcher_create");
-        return (-1);
-    }
-    flux_watcher_start (w);
-    return (0);
-}
-
-static void sink_continuation (flux_future_t *f, void *arg)
-{
-    flux_t *h = flux_future_get_flux (f);
-    struct aggregate *ag = arg;
-
-    int rc = flux_future_get (f, NULL);
-    flux_future_destroy (f);
-    if (rc < 0) {
-        /*  Schedule a retry, if  succesful return immediately, otherwise
-         *   abort the current aggregate and remove it.
-         */
-        if (sink_retry (h, ag) == 0)
-            return;
-        aggregate_sink_abort (h, ag);
-    }
-    zhash_delete (ag->ctx->aggregates, ag->key);
-    return;
-}
-
-static char *aggregate_to_string (struct aggregate *ag)
-{
-    char *s = NULL;
-    const char *name;
-    json_t *val, *o;
-    json_t *entries = aggregate_entries_tojson (ag);
-
-    if (entries == NULL)
-        return (NULL);
-
-    o = json_pack ("{s:i,s:i,s:o}",
-                   "total", ag->total,
-                   "count", ag->count,
-                   "entries", entries);
-    if (o == NULL)
-        return (NULL);
-
-    /*  Encode summary stats at top level of json representation
-     *   for backwards compatibility
-     */
-    if (ag->summary) {
-        json_object_foreach (ag->summary, name, val)
-            json_object_set (o, name, val);
-    }
-    s = json_dumps (o, JSON_COMPACT);
-    json_decref (o);
-    return (s);
-}
-
-
-static void aggregate_sink (flux_t *h, struct aggregate *ag)
-{
-    int rc = -1;
-    char *agstr = NULL;
-    flux_kvs_txn_t *txn = NULL;
-    flux_future_t *f = NULL;
-
-    flux_log (h, LOG_DEBUG, "sink: %s: count=%d total=%d",
-                ag->key, ag->count, ag->total);
-
-    /* Fail on key == "." */
-    if (strcmp (ag->key, ".") == 0) {
-        flux_log (h, LOG_ERR, "sink: refusing to sink to rootdir");
-        goto out;
-    }
-    if (!(agstr = aggregate_to_string (ag))) {
-        flux_log (h, LOG_ERR, "sink: aggregate_to_string failed");
-        goto out;
-    }
-    if (!(txn = flux_kvs_txn_create ())) {
-        flux_log_error (h, "sink: flux_kvs_txn_create");
-        goto out;
-    }
-    if (flux_kvs_txn_put (txn, 0, ag->key, agstr) < 0) {
-        flux_log_error (h, "sink: flux_kvs_txn_put");
-        goto out;
-    }
-    if (!(f = flux_kvs_commit (h, NULL, 0, txn))
-        || flux_future_then (f, -1., sink_continuation, (void *)ag) < 0) {
-        flux_log_error (h, "sink: flux_kvs_commit");
-        flux_future_destroy (f);
-        goto out;
-    }
-    rc = 0;
-out:
-    flux_kvs_txn_destroy (txn);
-    free (agstr);
-    if ((rc < 0) && (sink_retry (h, ag) < 0)) {
-        aggregate_sink_abort (h, ag);
-        zhash_delete (ag->ctx->aggregates, ag->key);
-    }
-}
-
-/*
- *  Flush aggregate `ag` -- forward entry upstream and destroy it locally.
- */
-static int aggregate_flush (struct aggregate *ag)
-{
-    flux_t *h = ag->ctx->h;
-    int rc;
-    assert (ag->ctx->rank != 0);
-    rc = aggregate_forward (h, ag);
-    zhash_delete (ag->ctx->aggregates, ag->key);
-    return (rc);
-}
-
-static void aggregate_destroy (struct aggregate *ag)
-{
-    struct aggregate_entry *ae = zlist_first (ag->entries);
-    while (ae) {
-        aggregate_entry_destroy (ae);
-        ae = zlist_next (ag->entries);
-    }
-    zlist_destroy (&ag->entries);
-    json_decref (ag->summary);
-    flux_watcher_destroy (ag->tw);
-    free (ag->key);
-    free (ag);
-}
-
-static void timer_cb (flux_reactor_t *r, flux_watcher_t *tw,
-                      int revents, void *arg)
-{
-    struct aggregate *ag = arg;
-    flux_t *h = ag->ctx->h;
-    if (aggregate_flush (ag) < 0)
-        flux_log_error (h, "aggregate_flush");
-}
-
-static void aggregate_timer_start (struct aggregate *ag,
-                                   double timeout)
-{
-    assert (ag && ag->ctx && ag->ctx->h);
-    struct aggregator *ctx = ag->ctx;
-    if (ctx->rank != 0) {
-        flux_t *h = ctx->h;
-        flux_reactor_t *r = flux_get_reactor (h);
-        ag->tw = flux_timer_watcher_create (r, timeout, 0.,
-                                            timer_cb, (void *) ag);
-        if (ag->tw == NULL) {
-            flux_log_error (h, "flux_timer_watcher_create");
-            return;
-        }
-        flux_watcher_start (ag->tw);
-    }
-}
-
-static struct aggregate *
-    aggregate_create (struct aggregator *ctx, const char *key)
-{
-    flux_t *h = ctx->h;
-
-    struct aggregate *ag = calloc (1, sizeof (*ag));
-    if (ag == NULL)
-        return NULL;
-
-    ag->ctx = ctx;
-    if (!(ag->key = strdup (key)) || !(ag->entries = zlist_new ())) {
-        flux_log_error (h, "aggregate_create: memory allocation error");
-        aggregate_destroy (ag);
-        return (NULL);
-    }
-    ag->sink_retries = 2;
-    return (ag);
-}
-
-static void aggregator_destroy (struct aggregator *ctx)
-{
-    if (ctx) {
-        zhash_destroy (&ctx->aggregates);
-        free (ctx);
-    }
-}
-
-static int attr_get_int (flux_t *h, const char *attr)
-{
-    unsigned long l;
-    char *p;
-    const char *s = flux_attr_get (h, attr);
-    if (!s)
-        return (-1);
-    errno = 0;
-    l = strtoul (s, &p, 10);
-    if (*p != '\0' || errno != 0) {
-        flux_log_error (h, "flux_attr_get (%s) = %s", attr, s);
-        return (-1);
-    }
-    return (l);
-}
-
-static double timer_scale (flux_t *h)
-{
-    long level, maxlevel;
-    if (((level = attr_get_int (h, "tbon.level")) < 0) ||
-        ((maxlevel = attr_get_int (h, "tbon.maxlevel")) < 0)) {
-        return (1.);
-    }
-    return (maxlevel - level + 1.);
-}
-
-static struct aggregator * aggregator_create (flux_t *h)
-{
-    struct aggregator * ctx = calloc (1, sizeof (*ctx));
-    if (ctx == NULL)
-        return (NULL);
-    ctx->h = h;
-    if (flux_get_rank (h, &ctx->rank) < 0) {
-        flux_log_error (h, "flux_get_rank");
-        goto error;
-    }
-    ctx->default_timeout = 0.01;
-    ctx->timer_scale = timer_scale (h);
-    if (!(ctx->aggregates = zhash_new ())) {
-        flux_log_error (h, "zhash_new");
-        goto error;
-    }
-    return (ctx);
-error:
-    aggregator_destroy (ctx);
-    return (NULL);
-}
-
-/*
- *  Add a new aggregate to aggregator `ctx`. Insert into entries
- *   hash, start the aggregate timeout, scaled by the current
- *   aggregator timeout scale.
- */
-static struct aggregate *
-aggregator_new_aggregate (struct aggregator *ctx, const char *key,
-                          int64_t total,
-                          double timeout)
-{
-    struct aggregate *ag = aggregate_create (ctx, key);
-    if (ag == NULL)
-        return (NULL);
-
-    if (zhash_insert (ctx->aggregates, key, ag) < 0) {
-        aggregate_destroy (ag);
-        return (NULL);
-    }
-    zhash_freefn (ctx->aggregates, key, (zhash_free_fn *) aggregate_destroy);
-    ag->timeout = timeout;
-    ag->total = total;
-    aggregate_timer_start (ag, timeout * ctx->timer_scale);
-    return (ag);
-}
-
-
-/*
- *  Callback for "aggregator.push"
- */
-static void push_cb (flux_t *h, flux_msg_handler_t *mh,
-                     const flux_msg_t *msg, void *arg)
-{
-    struct aggregator *ctx = arg;
-    struct aggregate *ag = NULL;
-    const char *key;
-    double timeout = ctx->default_timeout;
-    int64_t fwd_count = 0;
-    int64_t total = 0;
-    json_t *entries = NULL;
-
-    if (flux_msg_unpack (msg, "{s:s,s:I,s:o,s?F,s?I}",
-                              "key", &key,
-                              "total", &total,
-                              "entries", &entries,
-                              "timeout", &timeout,
-                              "fwd_count", &fwd_count) < 0)
-        goto error;
-
-    if (!(ag = zhash_lookup (ctx->aggregates, key)) &&
-        !(ag = aggregator_new_aggregate (ctx, key, total, timeout))) {
-        flux_log_error (ctx->h, "failed to get new aggregate");
-        goto error;
-    }
-
-    if (fwd_count > 0)
-        ag->fwd_count = fwd_count;
-
-    if (aggregate_push_json (ag, entries) < 0) {
-        flux_log_error (h, "aggregate_push_json: failed");
-        goto error;
-    }
-
-    flux_log (ctx->h, LOG_DEBUG, "push: %s: count=%d fwd_count=%d total=%d",
-                      ag->key, ag->count, ag->fwd_count, ag->total);
-    if (ctx->rank > 0) {
-        if ((ag->count == ag->total || ag->count == ag->fwd_count || timeout == 0.))
-            if (aggregate_flush (ag) < 0)
-                goto error;
-    }
-    else if (ag->count == ag->total)
-        aggregate_sink (h, ag);
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "aggregator.push: flux_respond");
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "aggregator.push: flux_respond_error");
-}
-
-
-static const struct flux_msg_handler_spec htab[] = {
-    //{ FLUX_MSGTYPE_EVENT,      "hb",               hb_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST,   "aggregator.push",  push_cb, 0 },
-    FLUX_MSGHANDLER_TABLE_END,
-};
-
-int mod_main (flux_t *h, int argc, char **argv)
-{
-    int rc = -1;
-    flux_msg_handler_t **handlers = NULL;
-    struct aggregator *ctx = aggregator_create (h);
-    if (!ctx)
-        goto done;
-
-    if (flux_msg_handler_addvec (h, htab, ctx, &handlers) < 0) {
-        flux_log_error (h, "flux_msg_handler_advec");
-        goto done;
-    }
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
-        flux_log_error (h, "flux_reactor_run");
-        goto done;
-    }
-    rc = 0;
-done:
-    flux_msg_handler_delvec (handlers);
-    aggregator_destroy (ctx);
-    return rc;
-}
-
-MOD_NAME ("aggregator");
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/barrier/Makefile.am b/src/modules/barrier/Makefile.am
deleted file mode 100644
index 0799d6de2f46..000000000000
--- a/src/modules/barrier/Makefile.am
+++ /dev/null
@@ -1,24 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
-
-#
-# Comms module
-#
-fluxmod_LTLIBRARIES = barrier.la
-
-barrier_la_SOURCES = barrier.c
-barrier_la_LDFLAGS = $(fluxmod_ldflags) -module
-barrier_la_LIBADD = $(fluxmod_libadd) \
-		    $(top_builddir)/src/common/libflux-internal.la \
-		    $(top_builddir)/src/common/libflux-core.la \
-		    $(ZMQ_LIBS)
diff --git a/src/modules/barrier/barrier.c b/src/modules/barrier/barrier.c
index c655a9f4f36c..903d07a179a7 100644
--- a/src/modules/barrier/barrier.c
+++ b/src/modules/barrier/barrier.c
@@ -40,12 +40,14 @@
 #include <stdio.h>
 #include <assert.h>
 #include <stdlib.h>
+#include <inttypes.h>
 #include <string.h>
 #include <errno.h>
 #include <stdbool.h>
+#include <stdbool.h>
 #include <flux/core.h>
-#include <czmq.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/errno_safe.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/iterators.h"
@@ -156,7 +158,7 @@ static struct barrier *barrier_create (struct barrier_ctx *ctx,
 }
 
 static int barrier_add_client (struct barrier *b,
-                               char *sender,
+                               const char *sender,
                                const flux_msg_t *msg)
 {
     if (zhash_insert (b->clients, sender, (void *)flux_msg_incref (msg)) < 0) {
@@ -316,7 +318,7 @@ static void enter_request_cb (flux_t *h, flux_msg_handler_t *mh,
 {
     struct barrier_ctx *ctx = arg;
     struct barrier *b;
-    char *sender = NULL;
+    const char *sender;
     const char *name;
     int nprocs;
     uint32_t owner;
@@ -327,8 +329,10 @@ static void enter_request_cb (flux_t *h, flux_msg_handler_t *mh,
                              "name", &name,
                              "nprocs", &nprocs) < 0)
         goto error;
-    if (flux_msg_get_route_first (msg, &sender) < 0)
+    if (!(sender = flux_msg_route_first (msg))) {
+        errno = EPROTO;
         goto error;
+    }
     if (flux_msg_get_userid (msg, &owner) < 0)
         goto error;
     if (!(b = barrier_lookup_create (ctx, name, nprocs, owner)))
@@ -344,11 +348,9 @@ static void enter_request_cb (flux_t *h, flux_msg_handler_t *mh,
     }
     if (barrier_update (b, 1) < 0)
         goto error;
-    free (sender);
     return;
 error:
     flux_respond_error (ctx->h, msg, errno, NULL);
-    free (sender);
 }
 
 /* Upon client disconnect, abort any pending barriers it was
@@ -358,15 +360,17 @@ static void disconnect_request_cb (flux_t *h, flux_msg_handler_t *mh,
                                    const flux_msg_t *msg, void *arg)
 {
     struct barrier_ctx *ctx = arg;
-    char *sender = NULL;
+    const char *sender;
     const char *key;
     struct barrier *b;
     struct flux_msg_cred cred;
 
     if (flux_msg_get_cred (msg, &cred) < 0)
         goto error;
-    if (flux_msg_get_route_first (msg, &sender) < 0)
+    if (!(sender = flux_msg_route_first (msg))) {
+        errno = EPROTO;
         goto error;
+    }
     FOREACH_ZHASH (ctx->barriers, key, b) {
         if (zhash_lookup (b->clients, sender)) {
             if (flux_msg_cred_authorize (cred, b->owner) < 0) {
@@ -380,11 +384,9 @@ static void disconnect_request_cb (flux_t *h, flux_msg_handler_t *mh,
                 flux_log_error (h, "exit_event_send");
         }
     }
-    free (sender);
     return;
 error:
     flux_log_error (h, "barrier.disconnect");
-    free (sender);
 }
 
 static int exit_event_send (flux_t *h,
@@ -508,8 +510,6 @@ int mod_main (flux_t *h, int argc, char **argv)
     return rc;
 }
 
-MOD_NAME ("barrier");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/connector-local/Makefile.am b/src/modules/connector-local/Makefile.am
deleted file mode 100644
index 77d5b56cc117..000000000000
--- a/src/modules/connector-local/Makefile.am
+++ /dev/null
@@ -1,23 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
-
-#
-# Comms module
-#
-fluxmod_LTLIBRARIES = connector-local.la
-
-connector_local_la_SOURCES = local.c
-connector_local_la_LDFLAGS = $(fluxmod_ldflags) -module
-connector_local_la_LIBADD = $(top_builddir)/src/common/libflux-internal.la \
-			    $(top_builddir)/src/common/libflux-core.la \
-			    $(ZMQ_LIBS)
diff --git a/src/modules/connector-local/local.c b/src/modules/connector-local/local.c
index 25bb0106d10c..300d53b12d70 100644
--- a/src/modules/connector-local/local.c
+++ b/src/modules/connector-local/local.c
@@ -25,17 +25,17 @@
 #include <sys/un.h>
 #include <sys/socket.h>
 #include <ctype.h>
-#include <czmq.h>
 #include <inttypes.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/cleanup.h"
+#include "src/common/libutil/errprintf.h"
 #include "src/common/librouter/usock.h"
 #include "src/common/librouter/router.h"
 
 enum {
     DEBUG_AUTHFAIL_ONESHOT = 1, /* force auth to fail one time */
-    DEBUG_USERDB_ONESHOT = 2,   /* force userdb lookup of instance owner */
     DEBUG_OWNERDROP_ONESHOT = 4,/* drop OWNER role to USER on next connection */
 };
 
@@ -44,6 +44,9 @@ struct connector_local {
     struct router *router;
     flux_t *h;
     uid_t instance_owner;
+    int allow_guest_user;
+    int allow_root_owner;
+    flux_msg_handler_t **handlers;
 };
 
 /* A 'struct route_entry' is attached to the 'struct usock_conn' aux hash
@@ -53,57 +56,62 @@ struct connector_local {
 static const char *route_auxkey = "flux::route";
 
 
-static int client_authenticate (flux_t *h,
-                                uint32_t instance_owner,
+static int client_authenticate (struct connector_local *ctx,
                                 uid_t cuid,
                                 struct flux_msg_cred *cred)
 {
-    uint32_t rolemask;
-    flux_future_t *f;
+    uint32_t rolemask = FLUX_ROLE_NONE;
 
-    if (flux_module_debug_test (h, DEBUG_AUTHFAIL_ONESHOT, true)) {
-        flux_log (h, LOG_ERR, "connect by uid=%d denied by debug flag",
+    /* Test hook: when set, deny one connection.
+     */
+    if (flux_module_debug_test (ctx->h, DEBUG_AUTHFAIL_ONESHOT, true)) {
+        flux_log (ctx->h,
+                  LOG_ERR,
+                  "connect by uid=%d denied by debug flag",
                   cuid);
         errno = EPERM;
         goto error;
     }
-    if (!flux_module_debug_test (h, DEBUG_USERDB_ONESHOT, true)) {
-        if (cuid == instance_owner) {
-            rolemask = FLUX_ROLE_OWNER;
-            goto success_nolog;
-        }
-    }
-    if (!(f = auth_lookup_rolemask (h, cuid)))
-        goto error;
-    if (auth_lookup_rolemask_get (f, &rolemask) < 0) {
-        flux_future_destroy (f);
-        errno = EPERM;
-        goto error;
-    }
-    flux_future_destroy (f);
+    /* Assign roles based on connecting uid and configured policy.
+     */
+    if (cuid == ctx->instance_owner)
+        rolemask = FLUX_ROLE_OWNER;
+    else if (ctx->allow_root_owner && cuid == 0)
+        rolemask = FLUX_ROLE_OWNER;
+    else if (ctx->allow_guest_user)
+        rolemask = FLUX_ROLE_USER;
+
     if (rolemask == FLUX_ROLE_NONE) {
-        flux_log (h, LOG_ERR, "%s: uid=%d no assigned roles",
-                  __FUNCTION__, cuid);
+        flux_log (ctx->h,
+                  LOG_ERR,
+                  "%s: uid=%d no assigned roles",
+                  __FUNCTION__,
+                  cuid);
         errno = EPERM;
         goto error;
     }
-    flux_log (h, LOG_INFO, "%s: uid=%d allowed rolemask=0x%x",
-              __FUNCTION__, cuid, rolemask);
-success_nolog:
-    if (flux_module_debug_test (h, DEBUG_OWNERDROP_ONESHOT, true)
-                            && (rolemask & FLUX_ROLE_OWNER)) {
-        cred->rolemask = FLUX_ROLE_USER;
-        cred->userid = FLUX_USERID_UNKNOWN;
-    } else {
-        cred->userid = cuid;
-        cred->rolemask = rolemask;
+    /* Tack on FLUX_ROLE_LOCAL to indicate that this message was
+     * accepted by the local connector.  This role is cleared when
+     * the message is received by another broker.
+     */
+    rolemask |= FLUX_ROLE_LOCAL;
+    /* Test hook: drop owner cred for one connection.
+     */
+    if (flux_module_debug_test (ctx->h, DEBUG_OWNERDROP_ONESHOT, true)) {
+        if ((rolemask & FLUX_ROLE_OWNER)) {
+            rolemask = FLUX_ROLE_USER;
+            cuid = FLUX_USERID_UNKNOWN;
+        }
     }
+
+    cred->userid = cuid;
+    cred->rolemask = rolemask;
     return 0;
 error:
     return -1;
 }
 
-/* Usock client encouters an error.
+/* Usock client encounters an error.
  */
 static void uconn_error (struct usock_conn *uconn, int errnum, void *arg)
 {
@@ -163,8 +171,7 @@ static void acceptor_cb (struct usock_conn *uconn, void *arg)
     struct router_entry *entry;
 
     initial_cred = usock_conn_get_cred (uconn);
-    if (client_authenticate (ctx->h,
-                             ctx->instance_owner,
+    if (client_authenticate (ctx,
                              initial_cred->userid,
                              &cred) < 0)
         goto error;
@@ -189,17 +196,104 @@ static void acceptor_cb (struct usock_conn *uconn, void *arg)
     usock_conn_destroy (uconn);
 }
 
+/* Parse [access] table.
+ * Access policy is instance owner only, unless configured otherwise:
+ *
+ * allow-guest-user = true
+ *   Allow users other than instance owner to connect with FLUX_ROLE_USER
+ *
+ * allow-root-owner = true
+ *   Allow root user to have instance owner role
+ *
+ * Missing [access] keys are interpreted as false.
+ * [access] keys other than the above are not allowed.
+ */
+int parse_config (struct connector_local *ctx,
+                  const flux_conf_t *conf,
+                  flux_error_t *errp)
+{
+    flux_error_t error;
+    int allow_guest_user = 0;
+    int allow_root_owner = 0;
+
+    if (flux_conf_unpack (conf,
+                          &error,
+                          "{s?{s?b s?b !}}",
+                          "access",
+                            "allow-guest-user",
+                            &allow_guest_user,
+                            "allow-root-owner",
+                            &allow_root_owner) < 0) {
+        errprintf (errp,
+                   "error parsing [access] configuration: %s",
+                   error.text);
+        return -1;
+    }
+    ctx->allow_guest_user = allow_guest_user;
+    ctx->allow_root_owner = allow_root_owner;
+    flux_log (ctx->h,
+              LOG_DEBUG,
+              "allow-guest-user=%s",
+              ctx->allow_guest_user ? "true" : "false");
+    flux_log (ctx->h,
+              LOG_DEBUG,
+              "allow-root-owner=%s",
+              ctx->allow_root_owner ? "true" : "false");
+    return 0;
+}
+
+static void reload_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    struct connector_local *ctx = arg;
+    const flux_conf_t *conf;
+    flux_error_t error;
+    const char *errstr = NULL;
+
+    if (flux_conf_reload_decode (msg, &conf) < 0)
+        goto error;
+    if (parse_config (ctx, conf, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (flux_set_conf (h, flux_conf_incref (conf)) < 0) {
+        errstr = "error updating cached configuration";
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,  "connector-local.config-reload", reload_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
 int mod_main (flux_t *h, int argc, char **argv)
 {
     struct connector_local ctx;
     const char *local_uri = NULL;
     char *tmpdir;
     const char *sockpath;
+    flux_error_t error;
     int rc = -1;
 
     memset (&ctx, 0, sizeof (ctx));
     ctx.h = h;
-    ctx.instance_owner = geteuid ();
+    ctx.instance_owner = getuid ();
+
+    /* Parse configuration
+     */
+    if (parse_config (&ctx, flux_get_conf (h), &error) < 0) {
+        flux_log (h, LOG_ERR, "%s", error.text);
+        goto done;
+    }
 
     /* Create router
      */
@@ -214,6 +308,7 @@ int mod_main (flux_t *h, int argc, char **argv)
     }
     if (!(tmpdir = strstr (local_uri, "local://"))) {
         flux_log (h, LOG_ERR, "malformed local-uri");
+        errno = EINVAL;
         goto done;
     }
     sockpath = tmpdir + 8;
@@ -229,6 +324,9 @@ int mod_main (flux_t *h, int argc, char **argv)
     cleanup_push_string (cleanup_file, sockpath);
     usock_server_set_acceptor (ctx.server, acceptor_cb, &ctx);
 
+    if (flux_msg_handler_addvec (h, htab, &ctx, &ctx.handlers) < 0)
+        goto done;
+
     /* Start reactor
      */
     if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
@@ -239,13 +337,12 @@ int mod_main (flux_t *h, int argc, char **argv)
     router_mute (ctx.router); // issue #1025 - disable unsub during shutdown
     rc = 0;
 done:
+    flux_msg_handler_delvec (ctx.handlers);
     usock_server_destroy (ctx.server); // destroy before router
     router_destroy (ctx.router);
     return rc;
 }
 
-MOD_NAME ("connector-local");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/content-files/Makefile.am b/src/modules/content-files/Makefile.am
new file mode 100644
index 000000000000..b5638687a30a
--- /dev/null
+++ b/src/modules/content-files/Makefile.am
@@ -0,0 +1,60 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux \
+	$(JANSSON_CFLAGS)
+
+noinst_LTLIBRARIES = libcontent-files.la
+
+libcontent_files_la_SOURCES = \
+	content-files.c \
+	filedb.h \
+	filedb.c
+
+TESTS = test_filedb.t
+
+test_ldadd = \
+	$(builddir)/libcontent-files.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(LIBPTHREAD)
+
+test_ldflags = \
+	-no-install
+
+test_cppflags = $(AM_CPPFLAGS)
+
+check_PROGRAMS = \
+	test_load \
+	test_store \
+	test_filedb.t
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+	$(top_srcdir)/config/tap-driver.sh
+
+
+test_load_SOURCES = test/load.c
+test_load_CPPFLAGS = $(test_cppflags)
+test_load_LDADD = $(test_ldadd)
+test_load_LDFLAGS = $(test_ldflags)
+
+test_store_SOURCES = test/store.c
+test_store_CPPFLAGS = $(test_cppflags)
+test_store_LDADD = $(test_ldadd)
+test_store_LDFLAGS = $(test_ldflags)
+
+test_filedb_t_SOURCES = test/filedb.c
+test_filedb_t_CPPFLAGS = $(test_cppflags)
+test_filedb_t_LDADD =  $(test_ldadd)
+test_filedb_t_LDFLAGS = $(test_ldflags)
diff --git a/src/modules/content-files/content-files.c b/src/modules/content-files/content-files.c
new file mode 100644
index 000000000000..b5c3fad8db6e
--- /dev/null
+++ b/src/modules/content-files/content-files.c
@@ -0,0 +1,425 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* content-files.c - content addressable storage with files back end
+ *
+ * This is mainly for demo/experimentation purposes.
+ * The "store" is a flat directory with blobrefs as filenames.
+ * As such, it is hungry for inodes and may run the file system out of them
+ * if used in anger!
+ *
+ * There are four main operations (RPC handlers):
+ *
+ * content-backing.load:
+ * Given a hash, lookup blob and return it or a "not found" error.
+ *
+ * content-backing.store:
+ * Given a blob, store it and return its hash
+ *
+ * content-backing.checkpoint-get:
+ * Given a string key, lookup string value and return it or a "not found" error.
+ *
+ * content-backing.checkpoint-put:
+ * Given a string key and string value, store it and return.
+ * If the key exists, overwrite.
+ *
+ * The content operations are per RFC 10 and are the main storage behind
+ * the Flux KVS.
+ *
+ * The content-backing.checkpoint operations allow the current KVS
+ * root reference to be saved/restored along with the content so it
+ * can persist across a Flux instance restart.  Multiple KVS
+ * namespaces (each with an independent root) are technically
+ * supported, although currently only the main KVS namespace is
+ * saved/restored by the KVS module.
+ *
+ * The main client of this module is the rank 0 content-cache.  The content
+ * cache is hierarchical:  each broker resolves missing content-cache entries
+ * by asking its TBON parent if it has the missing item.  Rank 0, the TBON
+ * root, asks the content backing store module.
+ *
+ * Once loaded this module can also be exercised directly using
+ * flux-content(1) with the --bypass-cache option.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libutil/blobref.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/dirwalk.h"
+#include "src/common/libutil/unlink_recursive.h"
+#include "ccan/str/str.h"
+
+#include "src/common/libcontent/content-util.h"
+
+#include "filedb.h"
+
+struct content_files {
+    flux_msg_handler_t **handlers;
+    char *dbpath;
+    flux_t *h;
+    char *hashfun;
+    int hash_size;
+};
+
+static int file_count_cb (dirwalk_t *d, void *arg)
+{
+    int *count = arg;
+
+    if (!dirwalk_isdir (d))
+        (*count)++;
+    return 0;
+}
+
+static int get_object_count (const char *path)
+{
+    int count = 0;
+    if (dirwalk (path, 0, file_count_cb, &count) < 0)
+        return -1;
+    return count;
+}
+
+static void stats_get_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
+{
+    struct content_files *ctx = arg;
+    int count;
+
+    if ((count = get_object_count (ctx->dbpath)) < 0)
+        goto error;
+
+    if (flux_respond_pack (h, msg, "{s:i}", "object_count", count) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+}
+
+
+/* Handle a content-backing.load request from the rank 0 broker's
+ * content-cache service.  The raw request payload is a hash digest.
+ * The raw response payload is the blob content.
+ * These payloads are specified in RFC 10.
+ */
+static void load_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    struct content_files *ctx = arg;
+    const void *hash;
+    size_t hash_size;
+    char blobref[BLOBREF_MAX_STRING_SIZE];
+    void *data = NULL;
+    size_t size;
+    const char *errstr = NULL;
+
+    if (flux_request_decode_raw (msg, NULL, &hash, &hash_size) < 0)
+        goto error;
+    if (hash_size != ctx->hash_size) {
+        errno = EPROTO;
+        goto error;
+    }
+    if (blobref_hashtostr (ctx->hashfun,
+                           hash,
+                           hash_size,
+                           blobref,
+                           sizeof (blobref)) < 0)
+        goto error;
+    if (filedb_get (ctx->dbpath, blobref, &data, &size, &errstr) < 0)
+        goto error;
+    if (flux_respond_raw (h, msg, data, size) < 0)
+        flux_log_error (h, "error responding to load request");
+    free (data);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to load request");
+    free (data);
+}
+
+/* Handle a content-backing.store request from the rank 0 broker's
+ * content-cache service.  The raw request payload is the blob content.
+ * The raw response payload is hash digest.
+ * These payloads are specified in RFC 10.
+ */
+void store_cb (flux_t *h,
+               flux_msg_handler_t *mh,
+               const flux_msg_t *msg,
+               void *arg)
+{
+    struct content_files *ctx = arg;
+    const void *data;
+    size_t size;
+    char blobref[BLOBREF_MAX_STRING_SIZE];
+    char hash[BLOBREF_MAX_DIGEST_SIZE];
+    int hash_size;
+    const char *errstr = NULL;
+
+    if (flux_request_decode_raw (msg, NULL, &data, &size) < 0)
+        goto error;
+    if ((hash_size = blobref_hash_raw (ctx->hashfun,
+                                       data,
+                                       size,
+                                       hash,
+                                       sizeof (hash))) < 0)
+        goto error;
+    if (blobref_hashtostr (ctx->hashfun,
+                           hash,
+                           hash_size,
+                           blobref,
+                           sizeof (blobref)) < 0)
+        goto error;
+    if (filedb_put (ctx->dbpath, blobref, data, size, &errstr) < 0)
+        goto error;
+    if (flux_respond_raw (h, msg, hash, hash_size) < 0)
+        flux_log_error (h, "error responding to store request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to store request");
+}
+
+/* Handle a content-backing.checkpoint-get request from the rank 0 kvs module.
+ * The KVS stores its last root reference here for restart purposes.
+ *
+ * N.B. filedb_get() calls read_all() which ensures that the returned buffer
+ * is padded with an extra NULL not included in the returned length,
+ * so it is safe to use the result as a string argument in flux_respond_pack().
+ */
+void checkpoint_get_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
+{
+    struct content_files *ctx = arg;
+    const char *key;
+    void *data = NULL;
+    size_t size;
+    json_t *o = NULL;
+    const char *errstr = NULL;
+    json_error_t error;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "key", &key) < 0)
+        goto error;
+    if (filedb_get (ctx->dbpath, key, &data, &size, &errstr) < 0)
+        goto error;
+    /* recovery from version 0 checkpoint blobref not supported */
+    if (!(o = json_loadb (data, size, 0, &error))) {
+        errstr = error.text;
+        errno = EINVAL;
+        goto error;
+    }
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:O}",
+                           "value",
+                           o) < 0)
+        flux_log_error (h, "error responding to checkpoint-get request");
+    free (data);
+    json_decref (o);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to checkpoint-get request");
+    free (data);
+    json_decref (o);
+}
+
+/* Handle a content-backing.checkpoint-put request from the rank 0 kvs module.
+ * The KVS stores its last root reference here for restart purposes.
+ */
+void checkpoint_put_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
+{
+    struct content_files *ctx = arg;
+    const char *key;
+    json_t *o;
+    char *value = NULL;
+    const char *errstr = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s:o}",
+                             "key",
+                             &key,
+                             "value",
+                             &o) < 0)
+        goto error;
+    if (!(value = json_dumps (o, JSON_COMPACT))) {
+        errstr = "failed to encode checkpoint value";
+        errno = EINVAL;
+        goto error;
+    }
+    if (filedb_put (ctx->dbpath, key, value, strlen (value), &errstr) < 0)
+        goto error;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to checkpoint-put request");
+    free (value);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to checkpoint-put request");
+    free (value);
+}
+
+/* Destroy module context.
+ */
+static void content_files_destroy (struct content_files *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (ctx->handlers);
+        free (ctx->dbpath);
+        free (ctx->hashfun);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+/* Table of message handler callbacks registered below.
+ * The topic strings in the table consist of <service name>.<method>.
+ */
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST, "content-backing.load",    load_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-backing.store",   store_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-backing.checkpoint-get", checkpoint_get_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-backing.checkpoint-put", checkpoint_put_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-files.stats-get",
+      stats_get_cb, FLUX_ROLE_USER },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+/* Create module context and perform some initialization.
+ */
+static struct content_files *content_files_create (flux_t *h, bool truncate)
+{
+    struct content_files *ctx;
+    const char *dbdir;
+    const char *s;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+    ctx->h = h;
+
+    /* Some tunables:
+     * - the hash function, e.g. sha1, sha256
+     * - path to sqlite file
+     */
+    if (!(s = flux_attr_get (h, "content.hash"))
+        || !(ctx->hashfun = strdup (s))
+        || (ctx->hash_size = blobref_validate_hashtype (s)) < 0) {
+        flux_log_error (h, "content.hash");
+        goto error;
+    }
+
+    /* Prefer 'statedir' as the location for the content.files directory,
+     * if set.  Otherwise use 'rundir'.  If the directory exists, the
+     * instance is restarting.
+     */
+    if (!(dbdir = flux_attr_get (h, "statedir")))
+        dbdir = flux_attr_get (h, "rundir");
+    if (!dbdir) {
+        flux_log_error (h, "neither statedir nor rundir are set");
+        goto error;
+    }
+    if (asprintf (&ctx->dbpath, "%s/content.files", dbdir) < 0)
+        goto error;
+    if (truncate)
+        (void)unlink_recursive (ctx->dbpath);
+    if (mkdir (ctx->dbpath, 0700) < 0 && errno != EEXIST) {
+        flux_log_error (h, "could not create %s", ctx->dbpath);
+        goto error;
+    }
+    if (flux_msg_handler_addvec (h, htab, ctx, &ctx->handlers) < 0)
+        goto error;
+    return ctx;
+error:
+    content_files_destroy (ctx);
+    return NULL;
+}
+
+static int parse_args (flux_t *h,
+                       int argc,
+                       char **argv,
+                       bool *testing,
+                       bool *truncate)
+{
+    int i;
+    for (i = 0; i < argc; i++) {
+        if (streq (argv[i], "testing"))
+            *testing = true;
+        else if (streq (argv[i], "truncate"))
+            *truncate = true;
+        else {
+            flux_log (h, LOG_ERR, "Unknown module option: %s", argv[i]);
+            errno = EINVAL;
+            return -1;
+        }
+    }
+    return 0;
+}
+
+/* The module thread enters here with broker handle 'h' pre-connected.
+ * The pattern used by most flux modules is to perform some initialization
+ * including installing message handlers, then enter the flux reactor loop.
+ * When the broker sends handle 'h' request messages that we registered
+ * to receive during initialization, the reactor ensures that our message
+ * handlers are called to deal with them.
+ *
+ * The reactor loop runs until it is stopped, e.g. with
+ * 'flux module remove <modname>' is run on this module.
+ *
+ * This function should return 0, or -1 on failure with errno set.
+ */
+int mod_main (flux_t *h, int argc, char **argv)
+{
+    struct content_files *ctx;
+    bool testing = false;
+    bool truncate = false;
+    int rc = -1;
+
+    if (parse_args (h, argc, argv, &testing, &truncate) < 0)
+        return -1;
+    if (!(ctx = content_files_create (h, truncate))) {
+        flux_log_error (h, "content_files_create failed");
+        return -1;
+    }
+    if (content_register_service (h, "content-backing") < 0)
+        goto done;
+    if (!testing) {
+        if (content_register_backing_store (h, "content-files") < 0)
+            goto done;
+    }
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
+        flux_log_error (h, "flux_reactor_run");
+        goto done_unreg;
+    }
+    rc = 0;
+done_unreg:
+    if (!testing)
+        (void)content_unregister_backing_store (h);
+done:
+    content_files_destroy (ctx);
+    return rc;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/content-files/filedb.c b/src/modules/content-files/filedb.c
new file mode 100644
index 000000000000..a6e8d225a557
--- /dev/null
+++ b/src/modules/content-files/filedb.c
@@ -0,0 +1,104 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+#include <stdio.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
+
+#include "filedb.h"
+
+
+int filedb_get (const char *dbpath,
+                const char *key,
+                void **datap,
+                size_t *sizep,
+                const char **errstr)
+{
+    char path[1024];
+    int fd;
+    void *data;
+    ssize_t size;
+
+    if (strlen (key) == 0 || strchr (key, '/') || streq (key, "..")
+                          || streq (key, ".")) {
+        errno = EINVAL;
+        if (errstr)
+            *errstr = "invalid key name";
+        return -1;
+    }
+    if (snprintf (path, sizeof (path), "%s/%s", dbpath, key) >= sizeof (path)) {
+        errno = EOVERFLOW;
+        if (errstr)
+            *errstr = "key name too long for internal buffer";
+        return -1;
+    }
+    if ((fd = open (path, O_RDONLY)) < 0)
+        return -1;
+    if ((size = read_all (fd, &data)) < 0) {
+        ERRNO_SAFE_WRAP (close, fd);
+        return -1;
+    }
+    if (close (fd) < 0) {
+        ERRNO_SAFE_WRAP (free, data);
+        return -1;
+    }
+    *datap = data;
+    *sizep = size;
+    return 0;
+}
+
+int filedb_put (const char *dbpath,
+                const char *key,
+                const void *data,
+                size_t size,
+                const char **errstr)
+{
+    char path[1024];
+    int fd;
+
+    if (strlen (key) == 0 || strchr (key, '/') || streq (key, "..")
+                          || streq (key, ".")) {
+        errno = EINVAL;
+        if (errstr)
+            *errstr = "invalid key";
+        return -1;
+    }
+    if (snprintf (path, sizeof (path), "%s/%s", dbpath, key) >= sizeof (path)) {
+        errno = EOVERFLOW;
+        if (errstr)
+            *errstr = "key name too long for internal buffer";
+        return -1;
+    }
+    if ((fd = open (path, O_WRONLY | O_CREAT | O_TRUNC, 0666)) < 0)
+        return -1;
+    if (write_all (fd, data, size) < 0) {
+        ERRNO_SAFE_WRAP (close, fd);
+        return -1;
+    }
+    if (close (fd) < 0)
+        return -1;
+    return 0;
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/content-files/filedb.h b/src/modules/content-files/filedb.h
new file mode 100644
index 000000000000..25721bb6ab63
--- /dev/null
+++ b/src/modules/content-files/filedb.h
@@ -0,0 +1,44 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _CONTENT_FILES_FILEDB_H
+#define _CONTENT_FILES_FILEDB_H
+
+/* Read file named 'key' from the dbpath directory.
+ * On success, 'datap' and 'sizep' are assigned the contents and size
+ * and 0 is returned (*datap must be freed).
+ * On failure, -1 is returned with errno set.
+ * Pass '*errstr' in pre-set to NULL and if a human readable error message
+ * is appropriate, it is assigned on error (do not free).
+ */
+int filedb_get (const char *dbpath,
+                const char *key,
+                void **datap,
+                size_t *sizep,
+                const char **errstr);
+
+
+/* Put file named 'key' with content 'data' and length 'size' to the
+ * dbpath directory.  On success, 0 is returned.
+ * On failure, -1 is returned with errno set.
+ * Pass '*errstr' in pre-set to NULL and if a human readable error message
+ * is appropriate, it is assigned on error (do not free).
+ */
+int filedb_put (const char *dbpath,
+                const char *key,
+                const void *data,
+                size_t size,
+                const char **errstr);
+
+#endif /* !_CONTENT_FILES_FILEDB_H */
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/content-files/test/filedb.c b/src/modules/content-files/test/filedb.c
new file mode 100644
index 000000000000..bda8a495b92d
--- /dev/null
+++ b/src/modules/content-files/test/filedb.c
@@ -0,0 +1,129 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/modules/content-files/filedb.h"
+#include "src/common/libutil/unlink_recursive.h"
+
+void test_badargs (const char *dbpath)
+{
+    void *data;
+    size_t size;
+    const char *errstr;
+    char longkey[8192];
+
+    memset (longkey, 'x', sizeof (longkey));
+    longkey[sizeof (longkey) - 1] = '\0';
+
+    /* get */
+
+    errno = 0;
+    errstr = NULL;
+    ok (filedb_get (dbpath, "/", &data, &size, &errstr) < 0 && errno == EINVAL,
+        "filedb_get key=\"/\" failed with EINVAL");
+    ok (errstr != NULL,
+        "and error string was set");
+
+    errno = 0;
+    errstr = NULL;
+    ok (filedb_get (dbpath, longkey, &data, &size, &errstr) < 0
+        && errno == EOVERFLOW,
+        "filedb_get key=<long> failed with EOVERFLOW");
+    ok (errstr != NULL,
+        "and error string was set");
+
+    errno = 0;
+    ok (filedb_get (dbpath, "noexist", &data, &size, &errstr) < 0
+        && errno == ENOENT,
+        "filedb_get key=\"\" failed with ENOENT");
+
+    /* put */
+
+    errno = 0;
+    errstr = NULL;
+    ok (filedb_put (dbpath, "", "", 1, &errstr) < 0 && errno == EINVAL,
+        "filedb_put key=\"\" failed with EINVAL");
+    ok (errstr != NULL,
+        "and error string was set");
+
+    errno = 0;
+    errstr = NULL;
+    ok (filedb_put (dbpath, longkey, "", 1, &errstr) < 0
+        && errno == EOVERFLOW,
+        "filedb_put key=<long> failed with EOVERFLOW");
+    ok (errstr != NULL,
+        "and error string was set");
+}
+
+void test_simple (const char *dbpath)
+{
+    char val1[] = { 'a', 'b', 'c' };
+    char val2[] = { 'z', 'y', 'x', 'w', 'v', 'u'};
+    const char *errstr;
+    void *data;
+    size_t size;
+
+    /* simple put, get */
+
+    ok (filedb_put (dbpath, "key1", val1, sizeof (val1), &errstr) == 0,
+        "filedb_put key1={abc} works");
+    size = 0;
+    data = NULL;
+    ok (filedb_get (dbpath, "key1", &data, &size, &errstr) == 0,
+        "filedb_get key1 works");
+    ok (data && size == sizeof (val1) && memcmp (data, val1, size) == 0,
+        "and returned data matches");
+    free (data);
+
+    /* overwrite key is allowed (e.g. for checkpoint support) */
+
+    ok (filedb_put (dbpath, "key1", val2, sizeof (val2), &errstr) == 0,
+        "filedb_put key1={zyxwvu} works (overwrite)");
+    ok (filedb_get (dbpath, "key1", &data, &size, &errstr) == 0,
+        "filedb_get key1 works");
+    ok (data && size == sizeof (val2) && memcmp (data, val2, size) == 0,
+        "and returned the updated data");
+    free (data);
+}
+
+int main (int argc, char *argv[])
+{
+    char dir[1024];
+    const char *tmp = getenv ("TMPDIR");
+
+    plan (NO_PLAN);
+
+    if (!tmp)
+        tmp = "/tmp";
+    if (snprintf (dir, sizeof (dir), "%s/filedb.XXXXXX", tmp) >= sizeof (dir))
+        BAIL_OUT ("internal buffer ovverflow");
+    if (!mkdtemp (dir))
+        BAIL_OUT ("mkdtemp failed");
+    diag ("mkdir %s", dir);
+
+    test_badargs (dir);
+    test_simple (dir);
+
+    if (unlink_recursive (dir) < 0)
+        BAIL_OUT ("unlink_recursive failed");
+
+    done_testing ();
+    return (0);
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/modules/content-files/test/load.c b/src/modules/content-files/test/load.c
new file mode 100644
index 000000000000..f1f79ff2be71
--- /dev/null
+++ b/src/modules/content-files/test/load.c
@@ -0,0 +1,40 @@
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/log.h"
+
+#include "src/modules/content-files/filedb.h"
+
+int main (int argc, char **argv)
+{
+    const char *errstr = NULL;
+    void *data;
+    size_t size;
+
+    if (argc != 3) {
+        fprintf (stderr, "Usage: test_load dbpath key >output\n");
+        exit (1);
+    }
+    if (filedb_get (argv[1], argv[2], &data, &size, &errstr) < 0)
+        log_msg_exit ("filedb_get: %s", errstr ? errstr : strerror (errno));
+
+    log_msg ("%zu bytes", size);
+
+    if (write_all (STDOUT_FILENO, data, size) < 0)
+        log_err_exit ("writing to stdout");
+
+    free (data);
+    exit (0);
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
+
diff --git a/src/modules/content-files/test/store.c b/src/modules/content-files/test/store.c
new file mode 100644
index 000000000000..8fbe3a572e96
--- /dev/null
+++ b/src/modules/content-files/test/store.c
@@ -0,0 +1,40 @@
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/log.h"
+
+#include "src/modules/content-files/filedb.h"
+
+int main (int argc, char **argv)
+{
+    const char *errstr = NULL;
+    void *data;
+    size_t size;
+
+    if (argc != 3) {
+        fprintf (stderr, "Usage: test_store dbpath key <input\n");
+        exit (1);
+    }
+    size = read_all (STDIN_FILENO, &data);
+    if (size < 0)
+        log_err_exit ("error reading stdin");
+
+    if (filedb_put (argv[1], argv[2], data, size, &errstr) < 0)
+        log_msg_exit ("filedb_put : %s", errstr ? errstr : "failed");
+
+    free (data);
+
+    exit (0);
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
+
diff --git a/src/modules/content-s3/Makefile.am b/src/modules/content-s3/Makefile.am
new file mode 100644
index 000000000000..8d1d457d3f5c
--- /dev/null
+++ b/src/modules/content-s3/Makefile.am
@@ -0,0 +1,46 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux
+
+noinst_LTLIBRARIES = libcontent-s3.la
+
+libcontent_s3_la_SOURCES = \
+	content-s3.c \
+	s3.c \
+	s3.h
+
+test_ldadd = \
+	$(builddir)/libcontent-s3.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(LIBPTHREAD) $(LIBS3)
+
+test_ldflags = \
+	-no-install
+
+test_cppflags = $(AM_CPPFLAGS)
+
+check_PROGRAMS = \
+	test_load \
+	test_store
+
+test_load_SOURCES = test/load.c
+test_load_CPPFLAGS = $(test_cppflags)
+test_load_LDADD = $(test_ldadd)
+test_load_LDFLAGS = $(test_ldflags)
+
+test_store_SOURCES = test/store.c
+test_store_CPPFLAGS = $(test_cppflags)
+test_store_LDADD = $(test_ldadd)
+test_store_LDFLAGS = $(test_ldflags)
diff --git a/src/modules/content-s3/content-s3.c b/src/modules/content-s3/content-s3.c
new file mode 100644
index 000000000000..4f39e140851f
--- /dev/null
+++ b/src/modules/content-s3/content-s3.c
@@ -0,0 +1,535 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <assert.h>
+
+#include "src/common/libutil/blobref.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libutil/errprintf.h"
+
+#include "src/common/libcontent/content-util.h"
+
+#include "src/common/libtomlc99/toml.h"
+#include "src/common/libutil/tomltk.h"
+
+#include "src/common/libyuarel/yuarel.h"
+#include "ccan/str/str.h"
+
+#include "s3.h"
+
+struct content_s3 {
+    flux_msg_handler_t **handlers;
+    struct s3_config *cfg;
+    flux_t *h;
+    char *hashfun;
+    int hash_size;
+};
+
+static void s3_config_destroy (struct s3_config *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        free (ctx->bucket);
+        free (ctx->hostname);
+        free (ctx->access_key);
+        free (ctx->secret_key);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+/* Destroy module context.
+ */
+static void content_s3_destroy (struct content_s3 *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (ctx->handlers);
+        s3_config_destroy (ctx->cfg);
+        free (ctx->hashfun);
+        free (ctx);
+        errno = saved_errno;
+    }
+
+    s3_cleanup ();
+}
+
+static int parse_credentials (struct s3_config *cfg,
+                              const char *cred_file,
+                              flux_error_t *errp)
+{
+    struct tomltk_error toml_error;
+    toml_table_t *tbl;
+    const char *raw;
+    char *access_key;
+    char *secret_key;
+    int saved_errno;
+
+    if (!(tbl = tomltk_parse_file (cred_file, &toml_error))) {
+        errno = EINVAL;
+        errprintf (errp, "toml parse failed: %s", toml_error.errbuf);
+        goto error;
+    }
+
+    if (!(raw = toml_raw_in (tbl, "secret-access-key"))) {
+        errno = EINVAL;
+        errprintf (errp, "failed to parse secret key");
+        goto error;
+    }
+
+    if (toml_rtos (raw, &secret_key)) {
+        errno = EINVAL;
+        errprintf (errp, "failed to parse secret key");
+        goto error;
+    }
+
+    if (!(raw = toml_raw_in (tbl, "access-key-id"))) {
+        free (secret_key);
+        errno = EINVAL;
+        errprintf (errp, "failed to parse access key");
+        goto error;
+    }
+
+    if (toml_rtos (raw, &access_key)) {
+        free (secret_key);
+        errno = EINVAL;
+        errprintf (errp, "failed to parse access key");
+        goto error;
+    }
+
+    cfg->secret_key = secret_key;
+    cfg->access_key = access_key;
+
+    return 0;
+
+error:
+    saved_errno = errno;
+    toml_free (tbl);
+    errno = saved_errno;
+    return -1;
+}
+
+static char *hostport (const char *host, int port)
+{
+    char *s;
+    if (port == 0) {
+        if (!(s = strdup (host)))
+            return NULL;
+    }
+    else {
+        if (asprintf (&s, "%s:%d", host, port) < 0)
+            return NULL;
+    }
+    return s;
+}
+
+static struct s3_config *parse_config (const flux_conf_t *conf,
+                                       flux_error_t *errp)
+{
+    struct s3_config *cfg;
+    flux_error_t error;
+    const char *uri = NULL;
+    const char *bucket = NULL;
+    const char *cred_file = NULL;
+    int is_virtual_host = 0;
+    struct yuarel yuri;
+    char *cpy = NULL;
+    int saved_errno;
+
+    if (!(cfg = calloc (1, sizeof (*cfg))))
+        return NULL;
+
+    cfg->retries = 5;
+    cfg->is_secure = 0;
+
+    if (flux_conf_unpack (conf,
+                          &error,
+                          "{s:{s:s, s:s, s:s, s?b !} }",
+                          "content-s3",
+                          "credential-file",
+                          &cred_file,
+                          "bucket",
+                          &bucket,
+                          "uri",
+                          &uri,
+                          "virtual-host-style",
+                          &is_virtual_host) < 0) {
+        errprintf (errp, "%s", error.text);
+        goto error;
+    }
+
+    if (!(cpy = strdup (uri)))
+        goto error;
+
+    if (yuarel_parse (&yuri, cpy) < 0) {
+        errprintf (errp, "failed to parse uri");
+        errno = EINVAL;
+        goto error;
+    }
+
+    if (!(cfg->hostname = hostport (yuri.host, yuri.port))) {
+        errprintf (errp, "failed to form hostname");
+        errno = ENOMEM;
+        goto error;
+    }
+
+    if (!(cfg->bucket = strdup (bucket)))
+        goto error;
+
+    if (strstarts (yuri.scheme, "https"))
+        cfg->is_secure = 1;
+
+    cfg->is_virtual_host = is_virtual_host;
+
+    if (parse_credentials (cfg, cred_file, errp))
+        goto error;
+
+    free (cpy);
+    return cfg;
+
+error:
+    saved_errno = errno;
+    free (cpy);
+    errno = saved_errno;
+    s3_config_destroy (cfg);
+    return NULL;
+}
+
+/* Broker is sending us a new config object because 'flux config reload'
+ * was run.  Parse it and respond with human readable errors.
+ * If events are posted, block until they complete so that:
+ * - any KVS commit errors are captured by 'flux config reload'
+ * - tests can look for eventlog entry after running 'flux config reload'
+ */
+static void config_reload_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct s3_config *cfg;
+    const flux_conf_t *conf;
+    flux_error_t error;
+    const char *errstr = NULL;
+
+    if (flux_conf_reload_decode (msg, &conf) < 0)
+        goto error;
+    if (!(cfg = parse_config (conf, &error)) ){
+        errstr = error.text;
+        goto error;
+    }
+    free (cfg);
+    flux_log (h, LOG_WARNING, "config-reload: changes will not take effect "
+                              "until next flux restart");
+
+    if (flux_set_conf (h, flux_conf_incref (conf)) < 0) {
+        errstr = "error updating cached configuration";
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+}
+
+/* Handle a content-backing.load request from the rank 0 broker's
+ * content-cache service.  The raw request payload is a hash digest,
+ * The raw response payload is the blob content.  These payloads are specified
+ * in RFC 10.
+ */
+static void load_cb (flux_t *h, flux_msg_handler_t *mh, const flux_msg_t *msg, void *arg)
+{
+    struct content_s3 *ctx = arg;
+    const void *hash;
+    size_t hash_size;
+    char blobref[BLOBREF_MAX_STRING_SIZE];
+    void *data = NULL;
+    size_t size;
+    const char *errstr = NULL;
+
+    if (flux_request_decode_raw (msg, NULL, &hash, &hash_size) < 0)
+        goto error;
+    if (hash_size != ctx->hash_size) {
+        errno = EPROTO;
+        errstr = "incorrect hash size";
+        goto error;
+    }
+    if (blobref_hashtostr (ctx->hashfun,
+                           hash,
+                           hash_size,
+                           blobref,
+                           sizeof (blobref)) < 0)
+        goto error;
+    if (s3_get (ctx->cfg, blobref, &data, &size, &errstr) < 0)
+        goto error;
+    if (flux_respond_raw (h, msg, data, size) < 0)
+        flux_log_error (h, "error responding to load request");
+    free (data);
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to load request");
+    free (data);
+}
+
+/* Handle a content-backing.store request from the rank 0 broker's
+ * content-cache service.  The raw request payload is the blob content.
+ * The raw response payload is a hash digest.
+ * These payloads are specified in RFC 10.
+ */
+void store_cb (flux_t *h,
+               flux_msg_handler_t *mh,
+               const flux_msg_t *msg,
+               void *arg)
+{
+    struct content_s3 *ctx = arg;
+    const void *data;
+    size_t size;
+    char blobref[BLOBREF_MAX_STRING_SIZE];
+    uint8_t hash[BLOBREF_MAX_DIGEST_SIZE];
+    int hash_size;
+    const char *errstr = NULL;
+
+    if (flux_request_decode_raw (msg, NULL, &data, &size) < 0)
+        goto error;
+    if ((hash_size = blobref_hash_raw (ctx->hashfun,
+                                       data,
+                                       size,
+                                       hash,
+                                       sizeof (hash))) < 0
+        || blobref_hashtostr (ctx->hashfun,
+                              hash,
+                              hash_size,
+                              blobref,
+                              sizeof (blobref)) < 0)
+        goto error;
+    assert (hash_size == ctx->hash_size);
+    if (s3_put (ctx->cfg, blobref, data, size, &errstr) < 0)
+        goto error;
+    if (flux_respond_raw (h, msg, hash, hash_size) < 0)
+        flux_log_error (h, "error responding to store request");
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to store request");
+}
+
+/* Handle a content-backing.checkpoint-get request from the rank 0 kvs module.
+ * The KVS stores its last root reference here for restart purposes.
+ *
+ * N.B. filedb_get() calls read_all() which ensures that the returned buffer
+ * is padded with an extra NULL not included in the returned length,
+ * so it is safe to use the result as a string argument in flux_respond_pack().
+ */
+void checkpoint_get_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
+{
+    const char *errstr = NULL;
+    struct content_s3 *ctx = arg;
+    const char *key;
+    void *data = NULL;
+    size_t size;
+    json_t *o = NULL;
+    json_error_t error;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "key", &key) < 0)
+        goto error;
+
+    if (s3_get (ctx->cfg, key, &data, &size, &errstr) < 0)
+        goto error;
+
+    if (!(o = json_loadb (data, size, 0, &error))) {
+        /* recovery from version 0 checkpoint blobref not supported */
+        errstr = error.text;
+        errno = EINVAL;
+        goto error;
+    }
+
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:O}",
+                           "value",
+                           o) < 0) {
+        errno = EIO;
+        flux_log_error (h, "error responding to checkpoint-get request (pack)");
+    }
+    free (data);
+    json_decref (o);
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to checkpoint-get request");
+    free (data);
+    json_decref (o);
+}
+
+/* Handle a content-backing.checkpoint-put request from the rank 0 kvs module.
+ * The KVS stores its last root reference here for restart purposes.
+ */
+void checkpoint_put_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
+{
+    struct content_s3 *ctx = arg;
+    const char *key;
+    json_t *o;
+    char *value = NULL;
+    const char *errstr = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s:o}",
+                             "key",
+                             &key,
+                             "value",
+                             &o) < 0)
+        goto error;
+    if (!(value = json_dumps (o, JSON_COMPACT))) {
+        errstr = "failed to encode checkpoint value";
+        errno = EINVAL;
+        goto error;
+    }
+    if (s3_put (ctx->cfg, key, value, strlen (value), &errstr) < 0)
+        goto error;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to checkpoint-put request (pack)");
+    free (value);
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to checkpoint-put request");
+    free (value);
+}
+
+/* Table of message handler callbacks registered below.
+ * The topic strings in the table consist of <service name>.<method>.
+ */
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST, "content-backing.load",    load_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-backing.store",   store_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-backing.checkpoint-get",
+                            checkpoint_get_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-backing.checkpoint-put",
+                            checkpoint_put_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-s3.config-reload", config_reload_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+/* Create the s3 context, initialize the connection, and
+ * create the working bucket
+ */
+static struct content_s3 *content_s3_create (flux_t *h)
+{
+    const char *errstr = NULL;
+    flux_error_t error;
+    struct content_s3 *ctx;
+    const char *s;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+    ctx->h = h;
+
+    if (!(s = flux_attr_get (h, "content.hash"))
+        || !(ctx->hashfun = strdup (s))
+        || (ctx->hash_size = blobref_validate_hashtype (s)) < 0) {
+        flux_log_error (h, "content.hash");
+        goto error;
+    }
+
+    if (!(ctx->cfg = parse_config (flux_get_conf (h), &error))) {
+        errstr = error.text;
+        flux_log (h, LOG_ERR, "content-s3 parsing config file: %s", errstr);
+        goto error;
+    }
+
+    if (s3_init (ctx->cfg, &errstr) < 0) {
+        flux_log (h, LOG_ERR, "content-s3 init: %s", errstr);
+        goto error;
+    }
+
+    if (s3_bucket_create (ctx->cfg, &errstr) < 0) {
+        flux_log (h, LOG_ERR, "content-s3 create bucket: %s", errstr);
+        goto error;
+    }
+
+    if (flux_msg_handler_addvec (h, htab, ctx, &ctx->handlers) < 0)
+        goto error;
+
+    return ctx;
+
+error:
+    content_s3_destroy (ctx);
+    return NULL;
+}
+
+static int parse_args (flux_t *h, int argc, char **argv)
+{
+    for (int i = 0; i < argc; i++) {
+        if (streq (argv[i], "truncate")) {
+            flux_log (h,
+                      LOG_ERR,
+                      "truncate is not implemented.  Use S3 console"
+                      " or other external mechanism to empty bucket.");
+            return -1;
+        }
+        else {
+            flux_log (h, LOG_ERR, "Unknown module option: %s", argv[i]);
+            errno = EINVAL;
+            return -1;
+        }
+    }
+    return 0;
+}
+
+int mod_main (flux_t *h, int argc, char **argv)
+{
+    struct content_s3 *ctx;
+    int rc = -1;
+
+    if (parse_args (h, argc, argv) < 0)
+        return -1;
+    if (!(ctx = content_s3_create (h))) {
+        flux_log_error (h, "content_s3_create failed");
+        return -1;
+    }
+    if (content_register_service (h, "content-backing") < 0)
+        goto done;
+    if (content_register_backing_store (h, "content-s3") < 0)
+        goto done;
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
+        flux_log_error (h, "flux_reactor_run");
+        goto done_unreg;
+    }
+    rc = 0;
+done_unreg:
+    (void)content_unregister_backing_store (h);
+done:
+    content_s3_destroy (ctx);
+    return rc;
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/content-s3/s3.c b/src/modules/content-s3/s3.c
new file mode 100644
index 000000000000..04d703777bdd
--- /dev/null
+++ b/src/modules/content-s3/s3.c
@@ -0,0 +1,326 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <string.h>
+#include <errno.h>
+#include <stdio.h>
+#include <libs3.h>
+#include <stdlib.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
+
+#include "s3.h"
+
+#if HAVE_S3_AUTH_REGION
+#define S3_create_bucket(proto, access, secret, host, buck, acl, loc, req, cb, data) \
+    S3_create_bucket(proto, access, secret, NULL, host, buck, acl, loc, req, cb, data)
+#endif
+
+#if HAVE_S3_TIMEOUT_ARG
+#define S3_create_bucket(proto, access, secret, host, buck, acl, loc, req, cb, data) \
+    S3_create_bucket(proto, access, secret, NULL, host, buck, NULL, acl, loc, req, 0, cb, data)
+#define S3_put_object(ctx, key, size, prop, req, cb, data) \
+    S3_put_object(ctx, key, size, prop, req, 0, cb, data)
+#define S3_get_object(ctx, key, cond, start, cnt, req, cb, data) \
+    S3_get_object(ctx, key, cond, start, cnt, req, 0, cb, data)
+#endif
+
+static S3Protocol protocol = S3ProtocolHTTP;
+static S3UriStyle uri_style = S3UriStylePath;
+
+/* Data needed by the get object callback function
+ */
+struct cb_data {
+    size_t size;
+    void *data;
+    int count;
+    S3Status status;
+};
+
+static S3Status response_props_cb (const S3ResponseProperties *properties,
+                                   void *data)
+{
+    return S3StatusOK;
+}
+
+static void response_complete_cb (S3Status status,
+                                  const S3ErrorDetails *error,
+                                  void *data)
+{
+    struct cb_data *ctx = data;
+    ctx->status = status;
+}
+
+/* Writes a chunk of 'data' to s3, returning
+ * the size of the data written.
+ */
+static int put_object_cb (int buff_size, char *buff, void *data)
+{
+    struct cb_data *ctx = data;
+    int size = (buff_size < ctx->size - ctx->count ? buff_size
+                                                   : ctx->size - ctx->count);
+
+    memcpy (buff, ctx->data + ctx->count, size);
+    ctx->count += size;
+
+    return size;
+}
+
+/* Gets the object from s3 storing it in in 'data->datap' as well
+ * as updating 'data->sizep' to hold the size of the data read.
+ */
+static S3Status get_object_cb (int buff_size, const char *buff, void *data)
+{
+    void *tmp = NULL;
+    struct cb_data *ctx = data;
+
+    if (! (tmp = realloc (ctx->data, buff_size + ctx->size)))
+        return S3StatusOutOfMemory;
+    ctx->data = tmp;
+
+    memcpy (ctx->data + ctx->size, buff, buff_size);
+    ctx->size += buff_size;
+
+    return S3StatusOK;
+}
+
+int s3_init (struct s3_config *cfg, const char **errstr)
+{
+    S3Status status = S3_initialize ("s3", S3_INIT_ALL, cfg->hostname);
+
+    if (cfg->is_virtual_host)
+        uri_style = S3UriStyleVirtualHost;
+    if (cfg->is_secure)
+        protocol = S3ProtocolHTTPS;
+
+    if (status != S3StatusOK) {
+        errno = ECONNREFUSED;
+        if (errstr)
+            *errstr = S3_get_status_name (status);
+
+        return -1;
+    }
+
+    return 0;
+}
+
+void s3_cleanup (void)
+{
+    S3_deinitialize ();
+}
+
+int s3_bucket_create (struct s3_config *cfg, const char **errstr)
+{
+    int retries = cfg->retries;
+    S3Status status = S3_validate_bucket_name (cfg->bucket, uri_style);
+
+    if (status != S3StatusOK) {
+        errno = ECONNREFUSED;
+        if (errstr)
+            *errstr = S3_get_status_name (status);
+
+        return -1;
+    }
+
+    S3ResponseHandler resp_hndl = {
+        .propertiesCallback = &response_props_cb,
+        .completeCallback = &response_complete_cb
+    };
+
+    struct cb_data ctx = {
+        .size = 0,
+        .data = NULL,
+        .count = 0,
+        .status = status,
+    };
+
+    do {
+        S3_create_bucket (protocol,
+                          cfg->access_key,
+                          cfg->secret_key,
+                          NULL, // hostName (NULL=use hostName passed
+                                //  to S3_initialize())
+                          cfg->bucket,
+                          S3CannedAclPrivate,
+                          NULL, // locationConstraint
+                          NULL, // requestContext (NULL for synchronous
+                                //  operation)
+                          &resp_hndl,
+                          &ctx);
+        retries--;
+    } while (S3_status_is_retryable (ctx.status) && retries > 0);
+
+    if (ctx.status != S3StatusOK
+        && ctx.status != S3StatusErrorBucketAlreadyOwnedByYou) {
+        errno = EREMOTEIO;
+        if (errstr)
+            *errstr = S3_get_status_name (status);
+
+        return -1;
+    }
+
+    return 0;
+}
+
+int s3_put (struct s3_config *cfg,
+            const char *key,
+            const void *data,
+            size_t size,
+            const char **errstr)
+{
+    int retries = cfg->retries;
+    S3Status status = S3StatusOK;
+
+    S3ResponseHandler resp_hndl = {
+        .propertiesCallback = &response_props_cb,
+        .completeCallback = &response_complete_cb
+    };
+
+    S3BucketContext bucket_ctx = {
+        .hostName = NULL,
+        .bucketName = cfg->bucket,
+        .protocol = protocol,
+        .uriStyle = uri_style,
+        .accessKeyId = cfg->access_key,
+        .secretAccessKey = cfg->secret_key
+    };
+
+    S3PutObjectHandler put_obj_hndl ={
+        .responseHandler = resp_hndl,
+        .putObjectDataCallback = &put_object_cb
+    };
+
+    if (strlen (key) == 0
+        || strchr (key, '/')
+        || streq (key, "..")
+        || streq (key, ".")) {
+        errno = EINVAL;
+        if (errstr)
+            *errstr = "invalid key";
+
+        return -1;
+    }
+
+    struct cb_data ctx = {
+        .size = size,
+        .data = (void *) data,
+        .count = 0,
+        .status = status
+    };
+
+    do {
+        S3_put_object (&bucket_ctx,
+                       key,
+                       size,
+                       NULL, // putProperties (NULL for none)
+                       NULL, // requestContext (NULL for synchronous operation)
+                       &put_obj_hndl,
+                       &ctx);
+        retries--;
+    } while (S3_status_is_retryable (ctx.status) && retries > 0);
+
+    if (ctx.status != S3StatusOK) {
+        errno = EREMOTEIO;
+        if (errstr)
+            *errstr = S3_get_status_name (ctx.status);
+
+        return -1;
+    }
+
+    return 0;
+}
+
+int s3_get (struct s3_config *cfg,
+            const char *key,
+            void **datap,
+            size_t *sizep,
+            const char **errstr)
+{
+    int retries = cfg->retries;
+    size_t size = 0;
+    S3Status status = S3StatusOK;
+
+    S3ResponseHandler resp_hndl = {
+        .propertiesCallback = &response_props_cb,
+        .completeCallback = &response_complete_cb
+    };
+
+    S3BucketContext bucket_ctx = {
+        .hostName = NULL,
+        .bucketName = cfg->bucket,
+        .protocol = protocol,
+        .uriStyle = uri_style,
+        .accessKeyId = cfg->access_key,
+        .secretAccessKey = cfg->secret_key
+    };
+
+
+    S3GetObjectHandler get_obj_hndl = {
+        .responseHandler = resp_hndl,
+        .getObjectDataCallback =  &get_object_cb
+    };
+
+    struct cb_data ctx = {
+        .size = size,
+        .data = NULL,
+        .count = 0,
+        .status = status
+    };
+
+    if (strlen (key) == 0
+        || strchr (key, '/')
+        || streq (key, "..")
+        || streq (key, ".")) {
+        errno = EINVAL;
+        if (errstr)
+            *errstr = "invalid key";
+
+        return -1;
+    }
+
+    do {
+        S3_get_object (&bucket_ctx,
+                       key,
+                       NULL, // getConditions (NULL for none)
+                       0,    // startByte
+                       0,    // byteCount (0 indicates the entire object
+                             //  should be read)
+                       NULL, // requestContext (NULL for synchronous operation)
+                       &get_obj_hndl, &ctx);
+        retries--;
+    } while (S3_status_is_retryable (ctx.status) && retries > 0);
+
+    if (ctx.status != S3StatusOK) {
+        free (ctx.data);
+        if (ctx.status == S3StatusErrorNoSuchKey)
+            errno = ENOENT;
+        else
+            errno = EREMOTEIO;
+
+        if (errstr)
+            *errstr = S3_get_status_name (ctx.status);
+
+        return -1;
+    }
+
+    *datap = ctx.data;
+    *sizep = ctx.size;
+
+    return 0;
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/content-s3/s3.h b/src/modules/content-s3/s3.h
new file mode 100644
index 000000000000..95d140278efb
--- /dev/null
+++ b/src/modules/content-s3/s3.h
@@ -0,0 +1,63 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _CONTENT_S3_S3_H
+#define _CONTENT_S3_S3_H
+
+/* Configuration info needed for all s3 calls
+ */
+struct s3_config {
+    int retries;        // number of times to retry each operation
+    int is_secure;
+    int is_virtual_host;
+    char *bucket;       // the bucket name for the instance to use
+    char *access_key;   // access key id string
+    char *secret_key;   // secret access key id string
+    char *hostname;     // hostname string
+};
+
+/* All int-returning functions below return 0 on success.  On failure,
+ * they return -1 with errno set.  In addition, if 'errstr' is non-NULL
+ * on failure, it is assigned an S3 status string.
+ */
+
+/* Initialize the s3 connection.
+ */
+int s3_init (struct s3_config *cfg, const char **errstr);
+
+/* Close down the s3 connection.
+ */
+void s3_cleanup (void);
+
+/* Create a bucket to be used for subsequent put/get operations.
+ */
+int s3_bucket_create (struct s3_config *cfg, const char **errstr);
+
+/* Write 'data' of length 'size' to object named 'key'.
+ */
+int s3_put (struct s3_config *cfg,
+            const char *key,
+            const void *data,
+            size_t size,
+            const char **errstr);
+
+/* Read 'datap', 'sizep' from object named 'key'.
+ */
+int s3_get (struct s3_config *cfg,
+            const char *key,
+            void **datap,
+            size_t *sizep,
+            const char **errstr);
+
+#endif
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/content-s3/test/load.c b/src/modules/content-s3/test/load.c
new file mode 100644
index 000000000000..63473154f13d
--- /dev/null
+++ b/src/modules/content-s3/test/load.c
@@ -0,0 +1,61 @@
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/log.h"
+
+#include "src/modules/content-s3/s3.h"
+
+int main (int argc, char **argv)
+{
+    const char *errstr = NULL;
+    void *data = NULL;
+    size_t size;
+
+
+    struct s3_config *cfg;
+
+    if (!(cfg = calloc (1, sizeof (*cfg))))
+        fprintf(stderr, "calloc error");
+    
+    cfg->retries = 5;
+    cfg->is_virtual_host = 0;
+    cfg->is_secure = 0;
+    cfg->bucket = getenv("S3_BUCKET");
+    cfg->access_key = getenv("S3_ACCESS_KEY_ID");
+    cfg->secret_key = getenv("S3_SECRET_ACCESS_KEY");
+    cfg->hostname = getenv("S3_HOSTNAME");
+
+    if (s3_init (cfg, &errstr) < 0) {
+        fprintf(stderr, "S3 init error\n%s\n", errstr);
+    }
+
+    if (s3_bucket_create (cfg, &errstr) < 0) {
+        fprintf(stderr, "S3 create bucket error\n%s\n", errstr);
+    }
+
+    if (argc != 2) {
+        fprintf (stderr, "Usage: test_load key >output\n");
+        exit (1);
+    }
+    if (s3_get (cfg, argv[1], &data, &size, &errstr) < 0)
+        log_msg_exit ("s3_get: %s", errstr ? errstr : strerror (errno));
+
+    log_msg ("%zu bytes", size);
+
+    if (write_all (STDOUT_FILENO, data, size) < 0)
+        log_err_exit ("writing to stdout");
+
+    free (data);
+    exit (0);
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/content-s3/test/store.c b/src/modules/content-s3/test/store.c
new file mode 100644
index 000000000000..8d668a60907a
--- /dev/null
+++ b/src/modules/content-s3/test/store.c
@@ -0,0 +1,61 @@
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/log.h"
+
+#include "src/modules/content-s3/s3.h"
+
+int main (int argc, char **argv)
+{
+    const char *errstr = NULL;
+    void *data;
+    size_t size;
+
+    struct s3_config *cfg;
+
+    if (!(cfg = calloc (1, sizeof (*cfg))))
+        fprintf(stderr, "calloc error");
+    
+    cfg->retries = 5;
+    cfg->is_virtual_host = 0;
+    cfg->is_secure = 0;
+    cfg->bucket = getenv("S3_BUCKET");
+    cfg->access_key = getenv("S3_ACCESS_KEY_ID");
+    cfg->secret_key = getenv("S3_SECRET_ACCESS_KEY");
+    cfg->hostname = getenv("S3_HOSTNAME");
+
+
+    if (s3_init (cfg, &errstr) < 0) {
+        fprintf(stderr, "S3 init error\n%s\n", errstr);
+    }
+
+    if (s3_bucket_create (cfg, &errstr) < 0) {
+        fprintf(stderr, "S3 create bucket error\n%s\n", errstr);
+    }
+
+    if (argc != 2) {
+        fprintf (stderr, "Usage: test_store key <input\n");
+        exit (1);
+    }
+    size = read_all (STDIN_FILENO, &data);
+    if (size < 0)
+        log_err_exit ("error reading stdin");
+
+    if (s3_put (cfg, argv[1], data, size, &errstr) < 0)
+        log_msg_exit ("s3_put : %s", errstr ? errstr : "failed");
+
+    free (data);
+
+    exit (0);
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/content-sqlite/Makefile.am b/src/modules/content-sqlite/Makefile.am
deleted file mode 100644
index 8bcfc728e8ec..000000000000
--- a/src/modules/content-sqlite/Makefile.am
+++ /dev/null
@@ -1,23 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) $(SQLITE_CFLAGS) \
-	$(LZ4_CFLAGS)
-
-fluxmod_LTLIBRARIES = content-sqlite.la
-
-content_sqlite_la_SOURCES = \
-	content-sqlite.c
-
-content_sqlite_la_LDFLAGS = $(fluxmod_ldflags) -module
-content_sqlite_la_LIBADD = $(top_builddir)/src/common/libflux-internal.la \
-		$(top_builddir)/src/common/libflux-core.la \
-		$(ZMQ_LIBS) $(SQLITE_LIBS) $(LZ4_LIBS)
diff --git a/src/modules/content-sqlite/content-sqlite.c b/src/modules/content-sqlite/content-sqlite.c
index 919c80e5978c..312d8060e1dc 100644
--- a/src/modules/content-sqlite/content-sqlite.c
+++ b/src/modules/content-sqlite/content-sqlite.c
@@ -13,21 +13,30 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/statvfs.h>
 #include <sqlite3.h>
-#include <czmq.h>
 #include <lz4.h>
 #include <flux/core.h>
+#include <jansson.h>
+#include <assert.h>
 
 #include "src/common/libutil/blobref.h"
-#include "src/common/libutil/cleanup.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/tstat.h"
+#include "src/common/libutil/monotime.h"
+
+#include "src/common/libcontent/content-util.h"
+#include "ccan/str/str.h"
 
 const size_t lzo_buf_chunksize = 1024*1024;
 const size_t compression_threshold = 256; /* compress blobs >= this size */
 
 const char *sql_create_table = "CREATE TABLE if not exists objects("
-                               "  hash CHAR(20) PRIMARY KEY,"
+                               "  hash BLOB PRIMARY KEY,"
                                "  size INT,"
                                "  object BLOB"
                                ");";
@@ -35,6 +44,7 @@ const char *sql_load = "SELECT object,size FROM objects"
                        "  WHERE hash = ?1 LIMIT 1";
 const char *sql_store = "INSERT INTO objects (hash,size,object) "
                         "  values (?1, ?2, ?3)";
+const char *sql_objects_count = "SELECT count(1) FROM objects";
 
 const char *sql_create_table_checkpt = "CREATE TABLE if not exists checkpt("
                                        "  key TEXT UNIQUE,"
@@ -45,6 +55,11 @@ const char *sql_checkpt_get = "SELECT value FROM checkpt"
 const char *sql_checkpt_put = "REPLACE INTO checkpt (key,value) "
                               "  values (?1, ?2)";
 
+struct content_stats {
+    tstat_t load;
+    tstat_t store;
+};
+
 struct content_sqlite {
     flux_msg_handler_t **handlers;
     char *dbfile;
@@ -54,12 +69,26 @@ struct content_sqlite {
     sqlite3_stmt *checkpt_get_stmt;
     sqlite3_stmt *checkpt_put_stmt;
     flux_t *h;
-    const char *hashfun;
-    uint32_t blob_size_limit;
+    char *hashfun;
+    int hash_size;
     size_t lzo_bufsize;
     void *lzo_buf;
+    struct content_stats stats;
+    char *journal_mode;
+    char *synchronous;
+    bool truncate;
 };
 
+static int set_config (char **conf, const char *val)
+{
+    char *tmp;
+    if (!(tmp = strdup (val)))
+        return -1;
+    free (*conf);
+    *conf = tmp;
+    return 0;
+}
+
 static void log_sqlite_error (struct content_sqlite *ctx, const char *fmt, ...)
 {
     char buf[64];
@@ -91,7 +120,7 @@ static void set_errno_from_sqlite_error (struct content_sqlite *ctx)
         case SQLITE_NOMEM:      /* cannot allocate memory */
             errno = ENOMEM;
             break;
-        case SQLITE_ABORT:      /* statment is not authorized */
+        case SQLITE_ABORT:      /* statement is not authorized */
         case SQLITE_PERM:       /* access mode for new db cannot be provided */
         case SQLITE_READONLY:   /* attempt to alter data with no permission */
             errno = EPERM;
@@ -129,25 +158,19 @@ static int grow_lzo_buf (struct content_sqlite *ctx, size_t size)
  * which invalidates returned data.
  */
 static int content_sqlite_load (struct content_sqlite *ctx,
-                                const char *blobref,
+                                const void *hash,
+                                int hash_size,
                                 const void **datap,
                                 int *sizep)
 {
-    uint8_t hash[BLOBREF_MAX_DIGEST_SIZE];
-    int hash_len;
     const void *data = NULL;
     int size = 0;
     int uncompressed_size;
 
-    if ((hash_len = blobref_strtohash (blobref, hash, sizeof (hash))) < 0) {
-        errno = ENOENT;
-        flux_log_error (ctx->h, "load: unexpected foreign blobref");
-        return -1;
-    }
     if (sqlite3_bind_text (ctx->load_stmt,
                            1,
                            (char *)hash,
-                           hash_len,
+                           hash_size,
                            SQLITE_STATIC) != SQLITE_OK) {
         log_sqlite_error (ctx, "load: binding key");
         set_errno_from_sqlite_error (ctx);
@@ -200,31 +223,25 @@ static int content_sqlite_load (struct content_sqlite *ctx,
 }
 
 /* Store blob to objects table, compressing if necessary.
- * Blobref resulting from hash over 'data' is stored to 'blobref'.
- * Returns 0 on success, -1 on error with errno set.
+ * hash over 'data' is stored to 'hash'.
+ * Returns hash size on success, -1 on error with errno set.
  */
 static int content_sqlite_store (struct content_sqlite *ctx,
                                  const void *data,
                                  int size,
-                                 char *blobref,
-                                 int blobrefsz)
+                                 void *hash,
+                                 int hash_len)
 {
-    uint8_t hash[BLOBREF_MAX_DIGEST_SIZE];
-    int hash_len;
     int uncompressed_size = -1;
+    int hash_size;
 
-    if (size > ctx->blob_size_limit) {
-        errno = EFBIG;
-        return -1;
-    }
-    if (blobref_hash (ctx->hashfun,
-                      (uint8_t *)data,
-                      size,
-                      blobref,
-                      blobrefsz) < 0)
-        return -1;
-    if ((hash_len = blobref_strtohash (blobref, hash, sizeof (hash))) < 0)
+    if ((hash_size = blobref_hash_raw (ctx->hashfun,
+                                       data,
+                                       size,
+                                       hash,
+                                       hash_len)) < 0)
         return -1;
+    assert (hash_size == ctx->hash_size);
     if (size >= compression_threshold) {
         int r;
         int out_len = LZ4_compressBound(size);
@@ -241,8 +258,8 @@ static int content_sqlite_store (struct content_sqlite *ctx,
     }
     if (sqlite3_bind_text (ctx->store_stmt,
                            1,
-                           (char *)hash,
-                           hash_len,
+                           hash,
+                           hash_size,
                            SQLITE_STATIC) != SQLITE_OK) {
         log_sqlite_error (ctx, "store: binding key");
         set_errno_from_sqlite_error (ctx);
@@ -264,6 +281,10 @@ static int content_sqlite_store (struct content_sqlite *ctx,
         set_errno_from_sqlite_error (ctx);
         goto error;
     }
+    /* N.B. ignore SQLITE_CONSTRAINT errors - it means the insert failed
+     * because it violated the implicit primary key uniqueness constraint.
+     * Blob and blobref are indeed stored and storage is conserved - success!
+     */
     if (sqlite3_step (ctx->store_stmt) != SQLITE_DONE
                     && sqlite3_errcode (ctx->db) != SQLITE_CONSTRAINT) {
         log_sqlite_error (ctx, "store: executing stmt");
@@ -271,7 +292,7 @@ static int content_sqlite_store (struct content_sqlite *ctx,
         goto error;
     }
     sqlite3_reset (ctx->store_stmt);
-    return 0;
+    return hash_size;
 error:
     ERRNO_SAFE_WRAP (sqlite3_reset, ctx->store_stmt);
     return -1;
@@ -283,25 +304,25 @@ static void load_cb (flux_t *h,
                      void *arg)
 {
     struct content_sqlite *ctx = arg;
-    const char *blobref;
-    int blobref_size;
+    const void *hash;
+    size_t hash_size;
     const void *data;
     int size;
+    struct timespec t0;
 
     if (flux_request_decode_raw (msg,
                                  NULL,
-                                 (const void **)&blobref,
-                                 &blobref_size) < 0) {
-        flux_log_error (h, "load: request decode failed");
+                                 &hash,
+                                 &hash_size) < 0)
         goto error;
-    }
-    if (!blobref || blobref[blobref_size - 1] != '\0') {
+    if (hash_size != ctx->hash_size) {
         errno = EPROTO;
-        flux_log_error (h, "load: malformed blobref");
         goto error;
     }
-    if (content_sqlite_load (ctx, blobref, &data, &size) < 0)
+    monotime (&t0);
+    if (content_sqlite_load (ctx, hash, hash_size, &data, &size) < 0)
         goto error;
+    tstat_push (&ctx->stats.load, monotime_since (t0));
     if (flux_respond_raw (h, msg, data, size) < 0)
         flux_log_error (h, "load: flux_respond_raw");
     (void )sqlite3_reset (ctx->load_stmt);
@@ -318,16 +339,24 @@ void store_cb (flux_t *h,
 {
     struct content_sqlite *ctx = arg;
     const void *data;
-    int size;
-    char blobref[BLOBREF_MAX_STRING_SIZE];
+    size_t size;
+    uint8_t hash[BLOBREF_MAX_DIGEST_SIZE];
+    int hash_size;
+    struct timespec t0;
 
     if (flux_request_decode_raw (msg, NULL, &data, &size) < 0) {
         flux_log_error (h, "store: request decode failed");
         goto error;
     }
-    if (content_sqlite_store (ctx, data, size, blobref, sizeof (blobref)) < 0)
+    monotime (&t0);
+    if ((hash_size = content_sqlite_store (ctx,
+                                           data,
+                                           size,
+                                           hash,
+                                           sizeof (hash))) < 0)
         goto error;
-    if (flux_respond_raw (h, msg, blobref, strlen (blobref) + 1) < 0)
+    tstat_push (&ctx->stats.store, monotime_since (t0));
+    if (flux_respond_raw (h, msg, hash, hash_size) < 0)
         flux_log_error (h, "store: flux_respond_raw");
     return;
 error:
@@ -342,6 +371,10 @@ void checkpoint_get_cb (flux_t *h,
 {
     struct content_sqlite *ctx = arg;
     const char *key;
+    char *s;
+    json_t *o = NULL;
+    const char *errstr = NULL;
+    json_error_t error;
 
     if (flux_request_unpack (msg, NULL, "{s:s}", "key", &key) < 0)
         goto error;
@@ -358,18 +391,39 @@ void checkpoint_get_cb (flux_t *h,
         errno = ENOENT;
         goto error;
     }
+    s = (char *)sqlite3_column_text (ctx->checkpt_get_stmt, 0);
+    if (!(o = json_loads (s, 0, &error))) {
+        if (blobref_validate (s) < 0) {
+            errstr = error.text;
+            errno = EINVAL;
+            goto error;
+        }
+        /* assume "version 0" if value is a bare blobref and return it
+         * in a json envelope */
+        if (!(o = json_pack ("{s:i s:s s:i s:f}",
+                             "version", 0,
+                             "rootref", s,
+                             "sequence", 0,
+                             "timestamp", 0.))) {
+            errstr = "failed to encode blobref in json envelope";
+            errno = EINVAL;
+            goto error;
+        }
+    }
     if (flux_respond_pack (h,
                            msg,
-                           "{s:s}",
+                           "{s:O}",
                            "value",
-                           sqlite3_column_text (ctx->checkpt_get_stmt, 0)) < 0)
+                           o) < 0)
         flux_log_error (h, "flux_respond_pack");
     (void )sqlite3_reset (ctx->checkpt_get_stmt);
+    json_decref (o);
     return;
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
         flux_log_error (h, "flux_respond_error");
     (void )sqlite3_reset (ctx->checkpt_get_stmt);
+    json_decref (o);
 }
 
 void checkpoint_put_cb (flux_t *h,
@@ -379,17 +433,20 @@ void checkpoint_put_cb (flux_t *h,
 {
     struct content_sqlite *ctx = arg;
     const char *key;
-    const char *value;
+    json_t *o;
+    char *value = NULL;
+    const char *errstr = NULL;
 
     if (flux_request_unpack (msg,
                              NULL,
-                             "{s:s s:s}",
+                             "{s:s s:o}",
                              "key",
                              &key,
                              "value",
-                             &value) < 0)
+                             &o) < 0)
         goto error;
-    if (strlen (key) == 0) {
+    if (!(value = json_dumps (o, JSON_COMPACT))) {
+        errstr = "failed to encode checkpoint value";
         errno = EINVAL;
         goto error;
     }
@@ -404,7 +461,7 @@ void checkpoint_put_cb (flux_t *h,
     }
     if (sqlite3_bind_text (ctx->checkpt_put_stmt,
                            2,
-                           (char *)value,
+                           value,
                            strlen (value),
                            SQLITE_STATIC) != SQLITE_OK) {
         log_sqlite_error (ctx, "checkpt_put: binding value");
@@ -420,53 +477,13 @@ void checkpoint_put_cb (flux_t *h,
     if (flux_respond (h, msg, NULL) < 0)
         flux_log_error (h, "flux_respond");
     (void )sqlite3_reset (ctx->checkpt_put_stmt);
+    free (value);
     return;
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
         flux_log_error (h, "flux_respond_error");
     (void )sqlite3_reset (ctx->checkpt_put_stmt);
-}
-
-
-int register_backing_store (flux_t *h, const char *name)
-{
-    flux_future_t *f;
-    int rc;
-
-    if (!(f = flux_rpc_pack (h,
-                             "content.register-backing",
-                             0,
-                             0,
-                             "{s:s}",
-                             "name",
-                             name)))
-        return -1;
-    rc = flux_future_get (f, NULL);
-    flux_future_destroy (f);
-    return rc;
-}
-
-int unregister_backing_store (flux_t *h)
-{
-    flux_future_t *f;
-    int rc;
-
-    if (!(f = flux_rpc (h, "content.unregister-backing", NULL, 0, 0)))
-        return -1;
-    rc = flux_future_get (f, NULL);
-    flux_future_destroy (f);
-    return rc;
-}
-
-static int register_service (flux_t *h, const char *name)
-{
-    int rc;
-    flux_future_t *f;
-    if (!(f = flux_service_register (h, name)))
-        return -1;
-    rc = flux_future_get (f, NULL);
-    flux_future_destroy (f);
-    return rc;
+    free (value);
 }
 
 static void content_sqlite_closedb (struct content_sqlite *ctx)
@@ -497,26 +514,132 @@ static void content_sqlite_closedb (struct content_sqlite *ctx)
     }
 }
 
+/* sqlite3_exec() callback from sql_objects_count query.
+ * On success, return 0 and set *arg to the count result.
+ * On error, return -1 which causes sqlite3_exec() to fail with SQLITE_ABORT.
+ */
+static int set_count (void *arg, int ncols, char **cols, char **col_names)
+{
+    int *result = arg;
+    int count = 0;
+    int rc = -1;
+
+    if (ncols == 1) {
+        errno = 0;
+        count = strtoul (cols[0], NULL, 10);
+        if (errno == 0) {
+            *result = count;
+            rc = 0;
+        }
+    }
+    return rc; // returning -1 causes SQLITE_ABORT
+}
+
+static json_t *pack_tstat (tstat_t *ts)
+{
+    json_t *o;
+    if (!(o = json_pack ("{s:i s:f s:f s:f s:f}",
+                         "count", tstat_count (ts),
+                         "min", tstat_min (ts),
+                         "max", tstat_max (ts),
+                         "mean", tstat_mean (ts),
+                          "stddev", tstat_stddev (ts)))) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    return o;
+}
+
+static unsigned long long get_file_size (const char *path)
+{
+    struct stat sb;
+
+    if (stat (path, &sb) < 0)
+        return 0;
+    return sb.st_size;
+}
+
+static unsigned long long get_fs_free (const char *path)
+{
+    struct statvfs sb;
+
+    if (statvfs (path, &sb) < 0)
+        return 0;
+    return sb.f_bsize * sb.f_bavail;
+}
+
+void stats_get_cb (flux_t *h,
+                   flux_msg_handler_t *mh,
+                   const flux_msg_t *msg,
+                   void *arg)
+{
+    struct content_sqlite *ctx = arg;
+    int count;
+    const char *errmsg = NULL;
+    json_t *load_time = NULL;
+    json_t *store_time = NULL;
+
+    if (sqlite3_exec (ctx->db,
+                      sql_objects_count,
+                      set_count,
+                      &count,
+                      NULL) != SQLITE_OK) {
+        errmsg = sqlite3_errmsg (ctx->db);
+        errno = EPERM;
+        goto error;
+    }
+    if (!(load_time = pack_tstat (&ctx->stats.load))
+        || !(store_time = pack_tstat (&ctx->stats.store)))
+        goto error;
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:i s:I s:I s:O s:O s:{s:s s:s}}",
+                           "object_count", count,
+                           "dbfile_size", get_file_size (ctx->dbfile),
+                           "dbfile_free", get_fs_free (ctx->dbfile),
+                           "load_time", load_time,
+                           "store_time", store_time,
+                           "config",
+                             "journal_mode", ctx->journal_mode,
+                             "synchronous", ctx->synchronous) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+    json_decref (load_time);
+    json_decref (store_time);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+    json_decref (load_time);
+    json_decref (store_time);
+}
+
 /* Open the database file ctx->dbfile and set up the database.
  */
-static int content_sqlite_opendb (struct content_sqlite *ctx)
+static int content_sqlite_opendb (struct content_sqlite *ctx, bool truncate)
 {
     int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
+    char s[128];
+    int count;
+
+    if (truncate)
+        (void)unlink (ctx->dbfile);
 
     if (sqlite3_open_v2 (ctx->dbfile, &ctx->db, flags, NULL) != SQLITE_OK) {
         log_sqlite_error (ctx, "opening %s", ctx->dbfile);
         goto error;
     }
+    snprintf (s, sizeof (s), "PRAGMA journal_mode=%s", ctx->journal_mode);
     if (sqlite3_exec (ctx->db,
-                      "PRAGMA journal_mode=OFF",
+                      s,
                       NULL,
                       NULL,
                       NULL) != SQLITE_OK) {
         log_sqlite_error (ctx, "setting sqlite 'journal_mode' pragma");
         goto error;
     }
+    snprintf (s, sizeof (s), "PRAGMA synchronous=%s", ctx->synchronous);
     if (sqlite3_exec (ctx->db,
-                      "PRAGMA synchronous=OFF",
+                      s,
                       NULL,
                       NULL,
                       NULL) != SQLITE_OK) {
@@ -531,6 +654,14 @@ static int content_sqlite_opendb (struct content_sqlite *ctx)
         log_sqlite_error (ctx, "setting sqlite 'locking_mode' pragma");
         goto error;
     }
+    if (sqlite3_exec (ctx->db,
+                      "PRAGMA quick_check",
+                      NULL,
+                      NULL,
+                      NULL) != SQLITE_OK) {
+        log_sqlite_error (ctx, "setting sqlite 'quick_check' pragma");
+        goto error;
+    }
     if (sqlite3_exec (ctx->db,
                       sql_create_table,
                       NULL,
@@ -579,6 +710,21 @@ static int content_sqlite_opendb (struct content_sqlite *ctx)
         log_sqlite_error (ctx, "preparing checkpt_put stmt");
         goto error;
     }
+    if (sqlite3_exec (ctx->db,
+                      sql_objects_count,
+                      set_count,
+                      &count,
+                      NULL) != SQLITE_OK) {
+        log_sqlite_error (ctx, "querying objects count");
+        goto error;
+    }
+    flux_log (ctx->h,
+              LOG_DEBUG,
+              "%s (%d objects) journal_mode=%s synchronous=%s",
+              ctx->dbfile,
+              count,
+              ctx->journal_mode,
+              ctx->synchronous);
     return 0;
 error:
     set_errno_from_sqlite_error (ctx);
@@ -592,6 +738,9 @@ static void content_sqlite_destroy (struct content_sqlite *ctx)
         flux_msg_handler_delvec (ctx->handlers);
         free (ctx->dbfile);
         free (ctx->lzo_buf);
+        free (ctx->hashfun);
+        free (ctx->journal_mode);
+        free (ctx->synchronous);
         free (ctx);
         errno = saved_errno;
     }
@@ -600,16 +749,20 @@ static void content_sqlite_destroy (struct content_sqlite *ctx)
 static const struct flux_msg_handler_spec htab[] = {
     { FLUX_MSGTYPE_REQUEST, "content-backing.load",    load_cb, 0 },
     { FLUX_MSGTYPE_REQUEST, "content-backing.store",   store_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs-checkpoint.get", checkpoint_get_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs-checkpoint.put", checkpoint_put_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-backing.checkpoint-get",
+                            checkpoint_get_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-backing.checkpoint-put",
+                            checkpoint_put_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "content-sqlite.stats-get",
+                            stats_get_cb, FLUX_ROLE_USER },
     FLUX_MSGHANDLER_TABLE_END,
 };
 
 static struct content_sqlite *content_sqlite_create (flux_t *h)
 {
     struct content_sqlite *ctx;
-    const char *backing_path;
-    const char *tmp;
+    const char *dbdir;
+    const char *s;
 
     if (!(ctx = calloc (1, sizeof (*ctx))))
         return NULL;
@@ -617,48 +770,52 @@ static struct content_sqlite *content_sqlite_create (flux_t *h)
         goto error;
     ctx->lzo_bufsize = lzo_buf_chunksize;
     ctx->h = h;
+    if (set_config (&ctx->journal_mode, "WAL") < 0)
+        goto error;
+    if (set_config (&ctx->synchronous, "NORMAL") < 0)
+        goto error;
 
     /* Some tunables:
      * - the hash function, e.g. sha1, sha256
      * - the maximum blob size
      * - path to sqlite file
      */
-    if (!(ctx->hashfun = flux_attr_get (h, "content.hash"))) {
+    if (!(s = flux_attr_get (h, "content.hash"))
+        || !(ctx->hashfun = strdup (s))
+        || (ctx->hash_size = blobref_validate_hashtype (s)) < 0) {
         flux_log_error (h, "content.hash");
         goto error;
     }
-    if (!(tmp = flux_attr_get (h, "content.blob-size-limit"))) {
-        flux_log_error (h, "content.blob-size-limit");
-        goto error;
-    }
-    ctx->blob_size_limit = strtoul (tmp, NULL, 10);
 
-    /* If 'content.backing-path' attribute is already set, then:
-     * - value is the sqlite backing file
-     * - if it exists, preserve existing content; else create empty
-     * - ensure that file perists when the instance exits
-     * Otherwise:
-     * - ${rundir}/content.sqlite is the backing file
-     * - ensure that file is cleaned up when the instance exits
-     * - set 'content.backing-path' to this name
+    /* Prefer 'statedir' as the location for content.sqlite file, if set.
+     * Otherwise use 'rundir', and enable pragmas that increase performance
+     * but risk database corruption on a crash (since rundir is temporary
+     * and the database is not being preserved after a crash anyway).
      */
-    backing_path = flux_attr_get (h, "content.backing-path");
-    if (backing_path) {
-        if (!(ctx->dbfile = strdup (backing_path)))
+    if (!(dbdir = flux_attr_get (h, "statedir"))) {
+        dbdir = flux_attr_get (h, "rundir");
+        if (set_config (&ctx->journal_mode, "OFF") < 0)
+            goto error;
+        if (set_config (&ctx->synchronous, "OFF") < 0)
             goto error;
     }
-    else {
-        const char *rundir = flux_attr_get (h, "rundir");
-        if (!rundir) {
-            flux_log_error (h, "rundir");
+    if (!dbdir) {
+        flux_log_error (h, "neither statedir nor rundir are set");
+        goto error;
+    }
+    if (asprintf (&ctx->dbfile, "%s/content.sqlite", dbdir) < 0)
+        goto error;
+
+    /* If dbfile exists, we are restarting.
+     * If existing dbfile does not have the right permissions, fail early.
+     */
+    if (access (ctx->dbfile, F_OK) == 0) {
+        if (access (ctx->dbfile, R_OK | W_OK) < 0) {
+            flux_log_error (h, "%s", ctx->dbfile);
             goto error;
         }
-        if (asprintf (&ctx->dbfile, "%s/content.sqlite", rundir) < 0)
-            goto error;
-        if (flux_attr_set (h, "content.backing-path", ctx->dbfile) < 0)
-            goto error;
-        cleanup_push_string (cleanup_file, ctx->dbfile);
     }
+
     if (flux_msg_handler_addvec (h, htab, ctx, &ctx->handlers) < 0)
         goto error;
     return ctx;
@@ -667,44 +824,137 @@ static struct content_sqlite *content_sqlite_create (flux_t *h)
     return NULL;
 }
 
+static bool journal_mode_valid (const char *s)
+{
+    /* N.B. sqlite is case sensitive by default, we assume it here */
+    if (!streq (s, "DELETE")
+        && !streq (s, "TRUNCATE")
+        && !streq (s, "PERSIST")
+        && !streq (s, "MEMORY")
+        && !streq (s, "WAL")
+        && !streq (s, "OFF"))
+        return false;
+    return true;
+}
+
+static bool synchronous_valid (const char *s)
+{
+    /* N.B. sqlite is case sensitive by default, we assume it here */
+    if (!streq (s, "EXTRA")
+        && !streq (s, "FULL")
+        && !streq (s, "NORMAL")
+        && !streq (s, "OFF"))
+        return false;
+    return true;
+}
+
+static int process_config (struct content_sqlite *ctx,
+                           const flux_conf_t *conf)
+{
+    flux_error_t error;
+    const char *journal_mode = NULL;
+    const char *synchronous = NULL;
+
+    if (flux_conf_unpack (conf,
+                          &error,
+                          "{s?{s?s s?s}}",
+                          "content-sqlite",
+                            "journal_mode", &journal_mode,
+                            "synchronous", &synchronous) < 0) {
+        flux_log_error (ctx->h, "%s", error.text);
+        return -1;
+    }
+    if (journal_mode) {
+        if (!journal_mode_valid (journal_mode)) {
+            flux_log (ctx->h, LOG_ERR, "invalid journal_mode config");
+            errno = EINVAL;
+            return -1;
+        }
+        if (set_config (&ctx->journal_mode, journal_mode) < 0)
+            return -1;
+    }
+    if (synchronous) {
+        if (!synchronous_valid (synchronous)) {
+            flux_log (ctx->h, LOG_ERR, "invalid synchronous config");
+            errno = EINVAL;
+            return -1;
+        }
+        if (set_config (&ctx->synchronous, synchronous) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static int process_args (struct content_sqlite *ctx,
+                         int argc,
+                         char **argv,
+                         bool *truncate)
+{
+    int i;
+    for (i = 0; i < argc; i++) {
+        if (strstarts (argv[i], "journal_mode=")) {
+            if (!journal_mode_valid (argv[i] + 13)) {
+                flux_log (ctx->h, LOG_ERR, "invalid journal_mode specified");
+                errno = EINVAL;
+                return -1;
+            }
+            if (set_config (&ctx->journal_mode, argv[i] + 13) < 0)
+                return -1;
+        }
+        else if (strstarts (argv[i], "synchronous=")) {
+            if (!synchronous_valid (argv[i] + 12)) {
+                flux_log (ctx->h, LOG_ERR, "invalid synchronous specified");
+                errno = EINVAL;
+                return -1;
+            }
+            if (set_config (&ctx->synchronous, argv[i] + 12) < 0)
+                return -1;
+        }
+        else if (streq ("truncate", argv[i])) {
+            *truncate = true;
+        }
+        else {
+            flux_log (ctx->h, LOG_ERR, "Unknown module option: '%s'", argv[i]);
+            errno = EINVAL;
+            return -1;
+        }
+    }
+    return 0;
+}
+
 int mod_main (flux_t *h, int argc, char **argv)
 {
     struct content_sqlite *ctx;
+    bool truncate = false;
+    int rc = -1;
 
     if (!(ctx = content_sqlite_create (h))) {
         flux_log_error (h, "content_sqlite_create failed");
         return -1;
     }
-    if (content_sqlite_opendb(ctx) < 0)
+    if (process_config (ctx, flux_get_conf (h)) < 0)
         goto done;
-    if (register_backing_store (h, "content-sqlite") < 0) {
-        flux_log_error (h, "registering backing store");
+    if (process_args (ctx, argc, argv, &truncate) < 0)
         goto done;
-    }
-    if (register_service (h, "content-backing") < 0) {
-        flux_log_error (h, "service.add: content-backing");
+    if (content_sqlite_opendb (ctx, truncate) < 0)
         goto done;
-    }
-    if (register_service (h, "kvs-checkpoint") < 0) {
-        flux_log_error (h, "service.add: kvs-checkpiont");
+    if (content_register_service (h, "content-backing") < 0)
+        goto done;
+    if (content_register_backing_store (h, "content-sqlite") < 0)
         goto done;
-    }
     if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
         flux_log_error (h, "flux_reactor_run");
-        goto done;
-    }
-    if (unregister_backing_store (h) < 0) {
-        flux_log_error (h, "unregistering backing store");
-        goto done;
+        goto done_unreg;
     }
+    rc = 0;
+done_unreg:
+    (void)content_unregister_backing_store (h);
 done:
     content_sqlite_closedb (ctx);
     content_sqlite_destroy (ctx);
-    return 0;
+    return rc;
 }
 
-MOD_NAME ("content-sqlite");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/content/cache.c b/src/modules/content/cache.c
new file mode 100644
index 000000000000..b543f397fa7a
--- /dev/null
+++ b/src/modules/content/cache.c
@@ -0,0 +1,1193 @@
+/************************************************************\
+ * Copyright 2015 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* See RFC 10 */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <inttypes.h>
+#include <assert.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libccan/ccan/list/list.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/blobref.h"
+#include "src/common/libutil/iterators.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libcontent/content.h"
+#include "ccan/str/str.h"
+
+#include "cache.h"
+#include "checkpoint.h"
+#include "mmap.h"
+
+/* A periodic callback purges the cache of least recently used entries.
+ * The callback is synchronized with the instance heartbeat, with a
+ * sync period upper bound set to 'sync_max' seconds.
+ */
+static double sync_max = 10.;
+
+static const char *default_hash = "sha1";
+
+static const uint32_t default_cache_purge_target_size = 1024*1024*16;
+static const uint32_t default_cache_purge_old_entry = 10; // seconds
+
+/* Raise the max blob size value to 1GB so that large KVS values
+ * (including KVS directories) can be supported while the KVS transitions
+ * to the RFC 11 treeobj data representation.
+ */
+//static const uint32_t default_blob_size_limit = 1048576; /* RFC 10 */
+static const uint32_t default_blob_size_limit = 1048576*1024;
+
+static const uint32_t default_flush_batch_limit = 256;
+
+/* Hash digests are used as zhashx keys.  The digest size needs to be
+ * available to zhashx comparator so make this global.
+ */
+static int content_hash_size;
+
+struct msgstack {
+    const flux_msg_t *msg;
+    struct msgstack *next;
+};
+
+struct cache_entry {
+    const void *data;
+    size_t len;
+    void *data_container;
+    void *hash;                     // key storage is contiguous with struct
+    uint8_t valid:1;                // entry contains valid data
+    uint8_t dirty:1;                // entry needs to be stored upstream
+                                    //   or to backing store (rank 0)
+    uint8_t ephemeral:1;            // clean entry is not on backing store
+    uint8_t load_pending:1;
+    uint8_t store_pending:1;
+    uint8_t mmapped:1;
+    struct msgstack *load_requests;
+    struct msgstack *store_requests;
+    double lastused;
+
+    struct list_node list;
+};
+
+struct content_cache {
+    flux_t *h;
+    flux_reactor_t *reactor;
+    flux_msg_handler_t **handlers;
+    flux_future_t *f_sync;
+    uint32_t rank;
+    zhashx_t *entries;
+    uint8_t backing:1;              // 'content.backing' service available
+    char *backing_name;
+    char *hash_name;
+    struct msgstack *flush_requests;
+
+    struct list_head lru;           // LRU is for valid, clean entries only
+    struct list_head flush;         // dirties queued due to batch limit
+
+    uint32_t blob_size_limit;
+    uint32_t flush_batch_limit;
+    uint32_t flush_batch_count;
+    int flush_errno;
+
+    uint32_t purge_target_size;
+    uint32_t purge_old_entry;
+
+    uint64_t acct_size;             // total size of all cache entries
+    uint32_t acct_valid;            // count of valid cache entries
+    uint32_t acct_dirty;            // count of dirty cache entries
+
+    struct content_checkpoint *checkpoint;
+    struct content_mmap *mmap;
+};
+
+static void flush_respond (struct content_cache *cache);
+static int cache_flush (struct content_cache *cache);
+
+static int msgstack_push (struct msgstack **msp, const flux_msg_t *msg)
+{
+    struct msgstack *ms;
+    if (!(ms = malloc (sizeof (*ms))))
+        return -1;
+    ms->msg = flux_msg_incref (msg);
+    ms->next = *msp;
+    *msp = ms;
+    return 0;
+}
+
+static const flux_msg_t *msgstack_pop (struct msgstack **msp)
+{
+    struct msgstack *ms;
+    const flux_msg_t *msg = NULL;
+
+    if ((ms = *msp)) {
+        *msp = ms->next;
+        msg = ms->msg;
+        free (ms);
+    }
+    return msg;
+}
+
+static void msgstack_destroy (struct msgstack **msp)
+{
+    const flux_msg_t *msg;
+    while ((msg = msgstack_pop (msp)))
+        flux_msg_decref (msg);
+}
+
+
+/* Respond identically to a list of requests.
+ * The list is always run to completion.
+ * On error, log at LOG_ERR level.
+ */
+static void request_list_respond_raw (struct msgstack **l,
+                                      flux_t *h,
+                                      int flag,
+                                      const void *data,
+                                      int len,
+                                      const char *type)
+{
+    const flux_msg_t *msg;
+    while ((msg = msgstack_pop (l))) {
+        flux_msg_t *response;
+        if (!(response = flux_response_derive (msg, 0))
+            || flux_msg_set_payload (response, data, len) < 0
+            || flux_msg_set_flag (response, flag) < 0
+            || flux_send (h, response, 0) < 0)
+            flux_log_error (h, "%s (%s):", __FUNCTION__, type);
+        flux_msg_decref (response);
+        flux_msg_decref (msg);
+    }
+}
+
+/* Same as above only send errnum, errmsg response
+ */
+static void request_list_respond_error (struct msgstack **l,
+                                        flux_t *h,
+                                        int errnum,
+                                        const char *errmsg,
+                                        const char *type)
+{
+    const flux_msg_t *msg;
+    while ((msg = msgstack_pop (l))) {
+        if (flux_respond_error (h, msg, errnum, errmsg) < 0)
+            flux_log_error (h, "%s (%s):", __FUNCTION__, type);
+        flux_msg_decref (msg);
+    }
+}
+
+/* Destroy a cache entry
+ */
+static void cache_entry_destroy (struct cache_entry *e)
+{
+    if (e) {
+        int saved_errno = errno;
+        msgstack_destroy (&e->load_requests);
+        msgstack_destroy (&e->store_requests);
+        if (e->mmapped)
+            content_mmap_region_decref (e->data_container);
+        else
+            flux_msg_decref (e->data_container);
+        free (e);
+        errno = saved_errno;
+    }
+}
+
+/* zhashx_destructor_fn footprint
+ */
+static void cache_entry_destructor (void **item)
+{
+    if (item) {
+        cache_entry_destroy (*item);
+        *item = NULL;
+    }
+}
+/* zhashx_hash_fn footprint
+ */
+static size_t cache_entry_hasher (const void *key)
+{
+    return *(size_t *)key;
+}
+/* zhashx_comparator_fn footprint
+ */
+static int cache_entry_comparator (const void *item1, const void *item2)
+{
+    return memcmp (item1, item2, content_hash_size);
+}
+
+/* Create a cache entry.
+ * Entries are created with no data (e.g. "invalid").
+ * Returns entry on success, NULL with errno set on failure.
+ */
+static struct cache_entry *cache_entry_create (const void *hash)
+{
+    struct cache_entry *e;
+
+    if (!(e = calloc (1, sizeof (*e) + content_hash_size)))
+        return NULL;
+    e->hash = (char *)(e + 1);
+    memcpy (e->hash, hash, content_hash_size);
+    list_node_init (&e->list);
+    return e;
+}
+
+static void cache_entry_dirty_clear (struct content_cache *cache,
+                                     struct cache_entry *e)
+{
+    if (e->dirty) {
+        cache->acct_dirty--;
+        e->dirty = 0;
+
+        assert (e->valid);
+        list_add (&cache->lru, &e->list);
+        e->lastused = flux_reactor_now (cache->reactor);
+
+        request_list_respond_raw (&e->store_requests,
+                                  cache->h,
+                                  0,
+                                  e->hash,
+                                  content_hash_size,
+                                  "store");
+    }
+}
+
+
+/* Create and insert a cache entry.
+ * Returns 0 on success, -1 on failure with errno set.
+ */
+static struct cache_entry *cache_entry_insert (struct content_cache *cache,
+                                               const void *hash,
+                                               int hash_size)
+{
+    struct cache_entry *e;
+
+    if (hash_size != content_hash_size) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(e = cache_entry_create (hash)))
+        return NULL;
+    if (zhashx_insert (cache->entries, e->hash, e) < 0) {
+        errno = EEXIST;
+        cache_entry_destroy (e);
+        return NULL;
+    }
+    return e;
+}
+
+/* Look up a cache entry.
+ * Move to front of LRU because it was looked up.
+ * Returns entry on success, NULL on failure.
+ * N.B. errno is not set
+ */
+static struct cache_entry *cache_entry_lookup (struct content_cache *cache,
+                                               const void *hash,
+                                               int hash_size)
+{
+    struct cache_entry *e;
+
+    if (hash_size != content_hash_size)
+        return NULL;
+    if (!(e = zhashx_lookup (cache->entries, hash)))
+        return NULL;
+
+    if (e->valid && !e->dirty) {
+        list_del_from (&cache->lru, &e->list);
+        list_add (&cache->lru, &e->list);
+        e->lastused = flux_reactor_now (cache->reactor);
+    }
+
+    return e;
+}
+
+/* Remove a cache entry.
+ */
+static void cache_entry_remove (struct content_cache *cache,
+                                struct cache_entry *e)
+{
+    assert (e->load_requests == NULL);
+    assert (e->store_requests == NULL);
+    assert (!e->dirty);
+    list_del (&e->list);
+    if (e->valid) {
+        cache->acct_size -= e->len;
+        cache->acct_valid--;
+    }
+    zhashx_delete (cache->entries, e->hash);
+}
+
+/* Load operation
+ *
+ * If a cache entry is already present and valid, response is immediate.
+ * Otherwise request is queued on the invalid cache entry, and a new
+ * request is sent to the next level of TBON, or on rank 0, to the
+ * content.backing service.  At most a single request is sent per cache entry.
+ * Once the response is received, identical responses are sent to all
+ * parked requests, and cache entry is made valid or removed if there was
+ * an error such as ENOENT.
+ */
+
+static void cache_load_continuation (flux_future_t *f, void *arg)
+{
+    struct content_cache *cache = arg;
+    struct cache_entry *e = flux_future_aux_get (f, "entry");
+    const flux_msg_t *msg;
+    const char *errmsg = NULL;
+
+    e->load_pending = 0;
+    if (flux_future_get (f, (const void **)&msg) < 0) {
+        if (errno == ENOSYS && cache->rank == 0)
+            errno = ENOENT;
+        if (errno != ENOENT)
+            flux_log_error (cache->h, "content load");
+        errmsg = flux_future_error_string (f);
+        goto error;
+    }
+    /* N.B. the entry may already be valid if a store filled it while
+     * we were waiting for this load completion.  Do nothing in that case.
+     * Any pending load requests would have been answered already.
+     */
+    if (!e->valid) {
+        assert (!e->data_container);
+        assert (!e->dirty);
+        if (flux_response_decode_raw (msg, NULL, &e->data, &e->len) < 0) {
+            flux_log_error (cache->h, "content load");
+            goto error;
+        }
+        e->data_container = (void *)flux_msg_incref (msg);
+        e->valid = 1;
+        if (flux_msg_has_flag (msg, FLUX_MSGFLAG_USER1))
+            e->ephemeral = 1;
+        cache->acct_valid++;
+        cache->acct_size += e->len;
+        list_add (&cache->lru, &e->list);
+        e->lastused = flux_reactor_now (cache->reactor);
+        request_list_respond_raw (&e->load_requests,
+                                  cache->h,
+                                  e->ephemeral ? FLUX_MSGFLAG_USER1 : 0,
+                                  e->data,
+                                  e->len,
+                                  "load");
+    }
+    flux_future_destroy (f);
+    return;
+error:
+    request_list_respond_error (&e->load_requests,
+                                cache->h,
+                                errno,
+                                errmsg,
+                                "load");
+    cache_entry_remove (cache, e);
+    flux_future_destroy (f);
+}
+
+static int cache_load (struct content_cache *cache, struct cache_entry *e)
+{
+    flux_future_t *f;
+    int flags = CONTENT_FLAG_UPSTREAM;
+
+    if (e->load_pending)
+        return 0;
+    if (cache->rank == 0)
+        flags = CONTENT_FLAG_CACHE_BYPASS;
+    if (!(f = content_load_byhash (cache->h, e->hash, content_hash_size, flags))
+        || flux_future_aux_set (f, "entry", e, NULL) < 0
+        || flux_future_then (f, -1., cache_load_continuation, cache) < 0) {
+        flux_log_error (cache->h, "content load");
+        flux_future_destroy (f);
+        return -1;
+    }
+    e->load_pending = 1;
+    return 0;
+}
+
+static void content_load_request (flux_t *h,
+                                  flux_msg_handler_t *mh,
+                                  const flux_msg_t *msg,
+                                  void *arg)
+{
+    struct content_cache *cache = arg;
+    const void *hash;
+    size_t hash_size;
+    struct cache_entry *e;
+    const char *errmsg = NULL;
+
+    if (flux_request_decode_raw (msg, NULL, &hash, &hash_size) < 0)
+        goto error;
+    if (hash_size != content_hash_size) {
+        errno = EPROTO;
+        goto error;
+    }
+    if (!(e = cache_entry_lookup (cache, hash, hash_size))) {
+        struct content_region *region = NULL;
+        const void *data = NULL;
+        int len = 0;
+
+        if (cache->rank == 0) {
+            region = content_mmap_region_lookup (cache->mmap,
+                                                 hash,
+                                                 hash_size,
+                                                 &data,
+                                                 &len);
+            if (!region && !cache->backing) {
+                errno = ENOENT;
+                goto error;
+            }
+        }
+        if (!(e = cache_entry_insert (cache, hash, hash_size))) {
+            flux_log_error (h, "content load");
+            goto error;
+        }
+        if (region) {
+            e->data_container = content_mmap_region_incref (region);
+            e->data = data;
+            e->len = len;
+            e->valid = 1;
+            e->ephemeral = 1;
+            e->mmapped = 1;
+            cache->acct_valid++;
+            cache->acct_size += e->len;
+            list_add (&cache->lru, &e->list);
+            e->lastused = flux_reactor_now (cache->reactor);
+        }
+    }
+    if (!e->valid) {
+        if (cache_load (cache, e) < 0)
+            goto error;
+        if (msgstack_push (&e->load_requests, msg) < 0) {
+            flux_log_error (h, "content load");
+            goto error;
+        }
+        return; /* RPC continuation will respond to msg */
+    }
+    if (e->valid && e->mmapped) { // rank 0 only
+        if (!content_mmap_validate (e->data_container,
+                                    e->hash,
+                                    content_hash_size,
+                                    e->data,
+                                    e->len)) {
+            errmsg = "mapped file content has changed";
+            errno = EINVAL;
+            goto error;
+        }
+    }
+
+    /* Send load response with FLUX_MSGFLAG_USER1 representing the
+     * ephemeral flag, if set.
+     */
+    flux_msg_t *response;
+    if (!(response = flux_response_derive (msg, 0))
+        || flux_msg_set_payload (response, e->data, e->len) < 0
+        || (e->ephemeral
+            && flux_msg_set_flag (response, FLUX_MSGFLAG_USER1) < 0)
+        || flux_send (h, response, 0) < 0) {
+        flux_log_error (h, "content load: error sending response");
+    }
+    flux_msg_decref (response);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "content load: flux_respond_error");
+}
+
+/* Store operation
+ *
+ * If a cache entry is already valid and not dirty, response is immediate.
+ * If cache entry is invalid, it is made valid (responding to any queued
+ * load requests), and then dirty.
+ *
+ * Dirty cache is write-through for ranks > 0;  that is, a request is queued
+ * and a single store request per cache entry is sent to the next level
+ * of TBON.  Once present in the rank 0 cache, requests are unwound and
+ * responded to at each level.
+ *
+ * Dirty cache is write-back for rank 0; that is, the response is immediate
+ * even though the entry may be dirty with respect to a 'content.backing'
+ * service.  This allows the cache to be updated at memory speeds,
+ * while holding the invariant that after a store RPC returns, the entry may
+ * be loaded from any rank.  The optional content.backing service can
+ * offload rank 0 hash entries at a slower pace.
+ */
+
+/* If cache has been flushed, respond to flush requests, if any.
+ * If there are still dirty entries in the cache->flush queue waiting to
+ * be stored, call cache_flush() to see if we can start any more.
+ */
+static void cache_resume_flush (struct content_cache *cache)
+{
+    if (cache->acct_dirty == 0 || (cache->rank == 0 && !cache->backing))
+        flush_respond (cache);
+    else
+        (void)cache_flush (cache); /* resume flushing, subject to limits */
+}
+
+static void cache_store_continuation (flux_future_t *f, void *arg)
+{
+    struct content_cache *cache = arg;
+    struct cache_entry *e = flux_future_aux_get (f, "entry");
+    const void *hash;
+    size_t hash_size;
+
+    e->store_pending = 0;
+    assert (cache->flush_batch_count > 0);
+    cache->flush_batch_count--;
+    if (content_store_get_hash (f, &hash, &hash_size) < 0) {
+        if (cache->rank == 0 && errno == ENOSYS) {
+            flux_log (cache->h,
+                      LOG_DEBUG,
+                      "content store: %s",
+                      "backing store service unavailable");
+        }
+        else {
+            flux_log (cache->h,
+                      LOG_CRIT,
+                      "content store: %s",
+                      strerror (errno));
+        }
+        goto error;
+    }
+    if (hash_size != content_hash_size
+        || memcmp (hash, e->hash, content_hash_size) != 0) {
+        errno = EIO;
+        goto error;
+    }
+    cache_entry_dirty_clear (cache, e);
+    /* clear flush errno if backing store functional/recovered */
+    cache->flush_errno = 0;
+    flux_future_destroy (f);
+    cache_resume_flush (cache);
+    return;
+error:
+    request_list_respond_error (&e->store_requests,
+                                cache->h,
+                                errno,
+                                NULL,
+                                "store");
+    /* all flush requests are assumed to fail with same errno */
+    request_list_respond_error (&cache->flush_requests,
+                                cache->h,
+                                errno,
+                                NULL,
+                                "flush");
+    cache->flush_errno = errno;
+    flux_future_destroy (f);
+    cache_resume_flush (cache);
+}
+
+/* Issue #4482, there is a small chance a dirty entry could be added
+ * to the flush list twice which can lead to list corruption.  As an
+ * extra measure, perform a delete from the list first.  If the node
+ * is not on the list, the delete is a no-op.
+ */
+static void flush_list_append (struct content_cache *cache,
+                               struct cache_entry *e)
+{
+    list_del (&e->list);
+    list_add_tail (&cache->flush, &e->list);
+}
+
+static int cache_store (struct content_cache *cache, struct cache_entry *e)
+{
+    flux_future_t *f;
+    int flags = CONTENT_FLAG_UPSTREAM;
+
+    assert (e->valid);
+
+    if (e->store_pending)
+        return 0;
+    if (cache->rank == 0) {
+        if (cache->flush_batch_count >= cache->flush_batch_limit) {
+            flush_list_append (cache, e);
+            return 0;
+        }
+        flags = CONTENT_FLAG_CACHE_BYPASS;
+    }
+    if (!(f = content_store (cache->h, e->data, e->len, flags))
+        || flux_future_aux_set (f, "entry", e, NULL) < 0
+        || flux_future_then (f, -1., cache_store_continuation, cache) < 0) {
+        flux_log_error (cache->h, "content store");
+        flux_future_destroy (f);
+        return -1;
+    }
+    e->store_pending = 1;
+    cache->flush_batch_count++;
+    return 0;
+}
+
+static void content_store_request (flux_t *h,
+                                   flux_msg_handler_t *mh,
+                                   const flux_msg_t *msg,
+                                   void *arg)
+{
+    struct content_cache *cache = arg;
+    const void *data;
+    size_t len;
+    struct cache_entry *e = NULL;
+    uint8_t hash[BLOBREF_MAX_DIGEST_SIZE];
+    int hash_size;
+
+    if (flux_request_decode_raw (msg, NULL, &data, &len) < 0)
+        goto error;
+    if (len > cache->blob_size_limit) {
+        errno = EFBIG;
+        goto error;
+    }
+    if ((hash_size = blobref_hash_raw (cache->hash_name,
+                                       data,
+                                       len,
+                                       hash,
+                                       sizeof (hash))) < 0)
+        goto error;
+    /* If existing entry has the ephemeral bit set, remove it and let it be
+     * replaced with a new entry.  N.B. it can be assumed that an entry with
+     * the ephemeral bit set is valid and not dirty.
+     */
+    if ((e = cache_entry_lookup (cache, hash, hash_size))
+        && e->ephemeral) {
+        cache_entry_remove (cache, e);
+        e = NULL;
+    }
+    if (!e) {
+        if (!(e = cache_entry_insert (cache, hash, hash_size)))
+            goto error;
+    }
+    /* Fill invalid cache entry, which may have been just created above,
+     * or could be there because a load was requested and it still awaits
+     * a response from up.  For the latter case, respond to any pending load
+     * requests after we fill the entry.
+     */
+    if (!e->valid) {
+        assert (!e->data_container);
+        e->data = data;
+        e->len = len;
+        e->data_container = (void *)flux_msg_incref (msg);
+        e->valid = 1;
+        e->dirty = 1;
+        cache->acct_valid++;
+        cache->acct_size += e->len;
+        cache->acct_dirty++;
+        request_list_respond_raw (&e->load_requests,
+                                  cache->h,
+                                  0,
+                                  e->data,
+                                  e->len,
+                                  "load");
+    }
+    if (e->dirty) {
+        if (cache->rank > 0 || cache->backing) {
+            if (cache_store (cache, e) < 0)
+                goto error;
+            if (cache->rank > 0) {  /* write-through */
+                if (msgstack_push (&e->store_requests, msg) < 0)
+                    goto error;
+                return;
+            }
+        }
+        /* On rank 0, save to flush list in event backing module
+         * loaded later.  Note that dirty entries are not removed
+         * during purge or dropcache, so this does not alter
+         * behavior */
+        if (cache->rank == 0 && !cache->backing)
+            flush_list_append (cache, e);
+    }
+    if (flux_respond_raw (h, msg, hash, hash_size) < 0)
+        flux_log_error (h, "content store: flux_respond_raw");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "content store: flux_respond_error");
+}
+
+/* Backing store is enabled/disabled by modules that provide the
+ * 'content.backing' service.  At module load time, the backing module
+ * informs the content service of its availability, and entries are
+ * asynchronously duplicated on the backing store and made eligible for
+ * dropping from the rank 0 cache.
+ */
+
+static int cache_flush (struct content_cache *cache)
+{
+    struct cache_entry *e;
+    int last_errno = 0;
+    int rc = 0;
+
+    while (cache->flush_batch_count < cache->flush_batch_limit) {
+        if (!(e = list_top (&cache->flush, struct cache_entry, list)))
+            break;
+        if (cache_store (cache, e) < 0) { // incr flush_batch_count
+            last_errno = errno;           //   and continuation will decr
+            rc = -1;
+            /* A few errors we will consider "unrecoverable", so break
+             * out */
+            if (errno == ENOSYS
+                || errno == ENOMEM)
+                break;
+        }
+        (void)list_pop (&cache->flush, struct cache_entry, list);
+    }
+    if (rc < 0)
+        errno = last_errno;
+    return rc;
+}
+
+static void content_register_backing_request (flux_t *h,
+                                              flux_msg_handler_t *mh,
+                                              const flux_msg_t *msg,
+                                              void *arg)
+{
+    struct content_cache *cache = arg;
+    const char *name;
+    const char *errstr = NULL;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "name", &name) < 0)
+        goto error;
+    if (cache->rank != 0) {
+        errno = EINVAL;
+        errstr = "content backing store can only be registered on rank 0";
+        goto error;
+    }
+    if (cache->backing) {
+        errno = EBUSY;
+        errstr = "content backing store is already active";
+        goto error;
+    }
+    /* cache->backing_name is either set to the initial value of the
+     * "content.backing-module" attribute (e.g. from broker command line),
+     * or to the first-registered backing store name.  Once set, it cannot
+     * be changed.
+     */
+    if (!cache->backing_name) {
+        if (!(cache->backing_name = strdup (name))
+            || flux_attr_set (h,
+                              "content.backing-module",
+                              cache->backing_name) < 0)
+            goto error;
+    }
+    if (!streq (cache->backing_name, name)) {
+        errno = EINVAL;
+        errstr = "content backing store cannot be changed on the fly";
+        goto error;
+    }
+    cache->backing = 1;
+    flux_log (h, LOG_DEBUG, "content backing store: enabled %s", name);
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to register-backing request");
+    (void)cache_flush (cache);
+    (void)checkpoints_flush (cache->checkpoint);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to register-backing request");
+};
+
+static void content_unregister_backing_request (flux_t *h,
+                                                flux_msg_handler_t *mh,
+                                                const flux_msg_t *msg,
+                                                void *arg)
+{
+    struct content_cache *cache = arg;
+    const char *errstr = NULL;
+
+    if (!cache->backing) {
+        errno = EINVAL;
+        errstr = "content backing store is not active";
+        goto error;
+    }
+    cache->backing = 0;
+    flux_log (h, LOG_DEBUG, "content backing store: disabled");
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to unregister-backing request");
+    if (cache->acct_dirty > 0)
+        flux_log (h, LOG_ERR, "%d unflushables", cache->acct_dirty);
+    /* If backing store is unloaded with pending flush requests, ensure that
+     * they receive an error response.
+     */
+    if (cache->flush_requests) {
+        request_list_respond_error (&cache->flush_requests,
+                                    cache->h,
+                                    ENOSYS,
+                                    NULL,
+                                    "flush");
+    }
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to unregister-backing request");
+}
+
+/* Forcibly drop all entries from the cache that can be dropped
+ * without data loss.  Use the LRU for this since all entries are
+ * valid and clean.
+ */
+
+static void content_dropcache_request (flux_t *h,
+                                       flux_msg_handler_t *mh,
+                                       const flux_msg_t *msg,
+                                       void *arg)
+{
+    struct content_cache *cache = arg;
+    int orig_size;
+    struct cache_entry *e = NULL;
+    struct cache_entry *next;
+
+    orig_size = zhashx_size (cache->entries);
+
+    list_for_each_safe (&cache->lru, e, next, list) {
+        cache_entry_remove (cache, e);
+    }
+
+    flux_log (h, LOG_DEBUG, "content dropcache %d/%d",
+              orig_size - (int)zhashx_size (cache->entries), orig_size);
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "content dropcache");
+}
+
+/* Return stats about the cache.
+ */
+
+static void content_stats_request (flux_t *h,
+                                   flux_msg_handler_t *mh,
+                                   const flux_msg_t *msg,
+                                   void *arg)
+{
+    struct content_cache *cache = arg;
+    json_t *o = content_mmap_get_stats (cache->mmap);
+
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:i s:i s:i s:I s:i s:O}",
+                           "count", zhashx_size (cache->entries),
+                           "valid", cache->acct_valid,
+                           "dirty", cache->acct_dirty,
+                           "size", cache->acct_size,
+                           "flush-batch-count", cache->flush_batch_count,
+                           "mmap", o ? o : json_null ()) < 0)
+        flux_log_error (h, "content stats");
+    json_decref (o);
+}
+
+/* Handle request to store all dirty entries.  The store requests are batched
+ * and handled asynchronously.  flush_respond() may be called immediately
+ * if there are no dirty entries, or later from cache_resume_flush().
+ * If 'backing' is false on rank 0, we go ahead and try to issue the store
+ * requests and handle the ENOSYS errors that result.
+ */
+
+/* This is called when outstanding store ops have completed.  */
+static void flush_respond (struct content_cache *cache)
+{
+    if (!cache->acct_dirty) {
+        request_list_respond_raw (&cache->flush_requests,
+                                  cache->h,
+                                  0,
+                                  NULL,
+                                  0,
+                                  "flush");
+    }
+    else {
+        errno = EIO;
+        if (cache->rank == 0 && !cache->backing)
+            errno = ENOSYS;
+        request_list_respond_error (&cache->flush_requests,
+                                    cache->h,
+                                    errno,
+                                    NULL,
+                                    "flush");
+    }
+}
+
+static void content_flush_request (flux_t *h,
+                                   flux_msg_handler_t *mh,
+                                   const flux_msg_t *msg,
+                                   void *arg)
+{
+    struct content_cache *cache = arg;
+
+    if (cache->rank == 0 && !cache->backing) {
+        errno = ENOSYS;
+        goto error;
+    }
+    if (cache->acct_dirty > 0) {
+        if (cache_flush (cache) < 0)
+            goto error;
+        /* if flush_batch_count == 0, no stores are in progress.  If
+         * there is a problem with the backing store, we must return
+         * error here.  We assume that the last store error is the
+         * primary storage error (i.e. ENOSPC, ENOSYS, etc.).
+         */
+        if (!cache->flush_batch_count && cache->flush_errno) {
+            errno = cache->flush_errno;
+            goto error;
+        }
+        if (msgstack_push (&cache->flush_requests, msg) < 0)
+            goto error;
+        return;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to content flush");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to content flush");
+}
+
+/* Heartbeat drives periodic cache purge
+ */
+
+static void cache_purge (struct content_cache *cache)
+{
+    double now = flux_reactor_now (cache->reactor);
+    struct cache_entry *e = NULL;
+    struct cache_entry *next;
+
+    list_for_each_rev_safe (&cache->lru, e, next, list) {
+        if (cache->acct_size <= cache->purge_target_size
+            || now - e->lastused < cache->purge_old_entry)
+            break;
+        assert (e->valid);
+        assert (!e->dirty);
+        cache_entry_remove (cache, e);
+    }
+}
+
+static void update_stats (struct content_cache *cache)
+{
+    flux_stats_gauge_set (cache->h, "content-cache.count",
+        (int) zhashx_size (cache->entries));
+    flux_stats_gauge_set (cache->h, "content-cache.valid",
+        cache->acct_valid);
+    flux_stats_gauge_set (cache->h, "content-cache.dirty",
+        cache->acct_dirty);
+    flux_stats_gauge_set (cache->h, "content-cache.size",
+        cache->acct_size);
+    flux_stats_gauge_set (cache->h, "content-cache.flush-batch-count",
+        cache->flush_batch_count);
+}
+
+static void sync_cb (flux_future_t *f, void *arg)
+{
+    struct content_cache *cache = arg;
+
+    if (flux_stats_enabled (cache->h, NULL))
+        update_stats (cache);
+
+    cache_purge (cache);
+
+    flux_future_reset (f);
+}
+
+bool content_cache_backing_loaded (struct content_cache *cache)
+{
+    return cache->backing;
+}
+
+/* Initialization
+ */
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.load",
+        content_load_request,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.store",
+        content_store_request,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.unregister-backing",
+        content_unregister_backing_request,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.register-backing",
+        content_register_backing_request,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.dropcache",
+        content_dropcache_request,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.stats-get",
+        content_stats_request,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.flush",
+        content_flush_request,
+        0
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+static int get_hash_name (struct content_cache *cache)
+{
+    const char *s;
+
+    if (!(s = flux_attr_get (cache->h, "content.hash")))
+        s = default_hash;
+    if ((content_hash_size = blobref_validate_hashtype (s)) < 0
+        || !(cache->hash_name = strdup (s))) {
+        flux_log_error (cache->h, "%s: unknown hash type", s);
+        return -1;
+    }
+    if (flux_attr_set (cache->h, "content.hash", cache->hash_name) < 0) {
+        flux_log_error (cache->h, "setattr content.hash");
+        return -1;
+    }
+    return 0;
+}
+
+static int parse_u32 (const char *s, uint32_t *val)
+{
+    unsigned long u;
+    char *endptr;
+
+    errno = 0;
+    u = strtoul (s, &endptr, 10);
+    if (errno != 0 || *endptr != '\0' || u > UINT32_MAX)
+        return -1;
+    *val = u;
+    return 0;
+}
+
+int parse_args (struct content_cache *cache, int argc, char **argv)
+{
+    uint32_t val;
+
+    for (int i = 0; i < argc; i++) {
+        if (strstarts (argv[i], "purge-target-size=")) {
+            if (parse_u32 (argv[i] + 18, &val) < 0) {
+                flux_log (cache->h, LOG_ERR, "error parsing %s", argv[i]);
+                return -1;
+            }
+            cache->purge_target_size = val;
+        }
+        else if (strstarts (argv[i], "purge-old-entry=")) {
+            if (parse_u32 (argv[i] + 16, &val) < 0) {
+                flux_log (cache->h, LOG_ERR, "error parsing %s", argv[i]);
+                return -1;
+            }
+            cache->purge_old_entry = val;
+        }
+        else if (strstarts (argv[i], "flush-batch-limit=")) {
+            if (parse_u32 (argv[i] + 18, &val) < 0) {
+                flux_log (cache->h, LOG_ERR, "error parsing %s", argv[i]);
+                return -1;
+            }
+            cache->purge_old_entry = val;
+        }
+        else if (strstarts (argv[i], "blob-size-limit=")) {
+            if (parse_u32 (argv[i] + 16, &val) < 0) {
+                flux_log (cache->h, LOG_ERR, "error parsing %s", argv[i]);
+                return -1;
+            }
+            cache->blob_size_limit = val;
+        }
+        else {
+            flux_log (cache->h, LOG_ERR, "unknown module option: %s", argv[i]);
+            return -1;
+        }
+    }
+    return 0;
+}
+
+void content_cache_destroy (struct content_cache *cache)
+{
+    if (cache) {
+        int saved_errno = errno;
+        flux_future_destroy (cache->f_sync);
+        flux_msg_handler_delvec (cache->handlers);
+        free (cache->backing_name);
+        zhashx_destroy (&cache->entries);
+        msgstack_destroy (&cache->flush_requests);
+        content_checkpoint_destroy (cache->checkpoint);
+        content_mmap_destroy (cache->mmap);
+        free (cache->hash_name);
+        free (cache);
+        errno = saved_errno;
+    }
+}
+
+struct content_cache *content_cache_create (flux_t *h, int argc, char **argv)
+{
+    struct content_cache *cache;
+
+    if (!(cache = calloc (1, sizeof (*cache))))
+        return NULL;
+    if (!(cache->entries = zhashx_new ()))
+        goto nomem;
+    cache->h = h;
+    cache->reactor = flux_get_reactor (h);
+
+    zhashx_set_destructor (cache->entries, cache_entry_destructor);
+    zhashx_set_key_hasher (cache->entries, cache_entry_hasher);
+    zhashx_set_key_comparator (cache->entries, cache_entry_comparator);
+    zhashx_set_key_destructor (cache->entries, NULL); // key is part of entry
+    zhashx_set_key_duplicator (cache->entries, NULL); // key is part of entry
+
+    cache->rank = FLUX_NODEID_ANY;
+    cache->blob_size_limit = default_blob_size_limit;
+    cache->flush_batch_limit = default_flush_batch_limit;
+    cache->purge_target_size = default_cache_purge_target_size;
+    cache->purge_old_entry = default_cache_purge_old_entry;
+    /* Some tunables may be set on the module command line (mainly for test).
+     */
+    if (parse_args (cache, argc, argv) < 0) {
+        errno = EINVAL;
+        goto error;
+    }
+    if (get_hash_name (cache) < 0)
+        goto error;
+
+    list_head_init (&cache->lru);
+    list_head_init (&cache->flush);
+
+    if (flux_get_rank (h, &cache->rank) < 0)
+        goto error;
+
+    if (!(cache->checkpoint = content_checkpoint_create (h,
+                                                         cache->rank,
+                                                         cache)))
+        goto error;
+    if (cache->rank == 0) {
+        if (!(cache->mmap = content_mmap_create (h,
+                                                 cache->hash_name,
+                                                 content_hash_size)))
+            goto error;
+    }
+    if (flux_msg_handler_addvec (h, htab, cache, &cache->handlers) < 0)
+        goto error;
+    if (!(cache->f_sync = flux_sync_create (h, 0))
+        || flux_future_then (cache->f_sync, sync_max, sync_cb, cache) < 0)
+        goto error;
+    return cache;
+nomem:
+    errno = ENOMEM;
+error:
+    content_cache_destroy (cache);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/content/cache.h b/src/modules/content/cache.h
new file mode 100644
index 000000000000..8fe79baa33c9
--- /dev/null
+++ b/src/modules/content/cache.h
@@ -0,0 +1,24 @@
+/************************************************************\
+ * Copyright 2015 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _CONTENT_CACHE_H
+#define _CONTENT_CACHE_H 1
+
+struct content_cache;
+
+struct content_cache *content_cache_create (flux_t *h, int argc, char **argv);
+void content_cache_destroy (struct content_cache *cache);
+bool content_cache_backing_loaded (struct content_cache *cache);
+
+#endif /* !_CONTENT_CACHE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/content/checkpoint.c b/src/modules/content/checkpoint.c
new file mode 100644
index 000000000000..ad181eadef23
--- /dev/null
+++ b/src/modules/content/checkpoint.c
@@ -0,0 +1,439 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* See RFC 10 */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <inttypes.h>
+#include <assert.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "checkpoint.h"
+#include "cache.h"
+
+struct content_checkpoint {
+    flux_t *h;
+    flux_msg_handler_t **handlers;
+    uint32_t rank;
+    struct content_cache *cache;
+    zhashx_t *hash;
+    unsigned int hash_dirty;
+};
+
+struct checkpoint_data {
+    struct content_checkpoint *checkpoint;
+    json_t *value;
+    uint8_t dirty:1;
+    bool in_progress;
+    int refcount;
+};
+
+static struct checkpoint_data *
+checkpoint_data_incref (struct checkpoint_data *data)
+{
+    if (data)
+        data->refcount++;
+    return data;
+}
+
+static void checkpoint_data_decref (struct checkpoint_data *data)
+{
+    if (data && --data->refcount == 0) {
+        if (data->dirty)
+            data->checkpoint->hash_dirty--;
+        json_decref (data->value);
+        free (data);
+    }
+}
+
+/* zhashx_destructor_fn */
+static void checkpoint_data_decref_wrapper (void **arg)
+{
+    if (arg) {
+        struct checkpoint_data *data = *arg;
+        checkpoint_data_decref (data);
+    }
+}
+
+static struct checkpoint_data *
+checkpoint_data_create (struct content_checkpoint *checkpoint,
+                        json_t *value)
+{
+    struct checkpoint_data *data = NULL;
+
+    if (!(data = calloc (1, sizeof (*data))))
+        return NULL;
+    data->checkpoint = checkpoint;
+    data->value = json_incref (value);
+    data->refcount = 1;
+    return data;
+}
+
+static int checkpoint_data_update (struct content_checkpoint *checkpoint,
+                                   const char *key,
+                                   json_t *value)
+{
+    struct checkpoint_data *data = NULL;
+
+    if (!(data = checkpoint_data_create (checkpoint, value)))
+        return -1;
+
+    zhashx_update (checkpoint->hash, key, data);
+    data->dirty = 1;
+    checkpoint->hash_dirty++;
+    return 0;
+}
+
+static void checkpoint_get_continuation (flux_future_t *f, void *arg)
+{
+    struct content_checkpoint *checkpoint = arg;
+    const flux_msg_t *msg = flux_future_aux_get (f, "msg");
+    const char *key;
+    json_t *value = NULL;
+
+    assert (msg);
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "key", &key) < 0)
+        goto error;
+
+    if (flux_rpc_get_unpack (f, "{s:o}", "value", &value) < 0)
+        goto error;
+
+    if (checkpoint_data_update (checkpoint, key, value) < 0)
+        goto error;
+
+    if (flux_respond_pack (checkpoint->h, msg, "{s:O}", "value", value) < 0)
+        flux_log_error (checkpoint->h, "error responding to checkpoint-get");
+
+    flux_future_destroy (f);
+    return;
+
+error:
+    if (flux_respond_error (checkpoint->h, msg, errno, NULL) < 0)
+        flux_log_error (checkpoint->h, "error responding to checkpoint-get");
+    flux_future_destroy (f);
+}
+
+static int checkpoint_get_forward (struct content_checkpoint *checkpoint,
+                                   const flux_msg_t *msg,
+                                   const char *key,
+                                   const char **errstr)
+{
+    const char *topic = "content.checkpoint-get";
+    uint32_t rank = FLUX_NODEID_UPSTREAM;
+    flux_future_t *f = NULL;
+
+    /* if we're on rank 0, go directly to backing store */
+    if (checkpoint->rank == 0) {
+        topic = "content-backing.checkpoint-get";
+        rank = 0;
+    }
+
+    if (!(f = flux_rpc_pack (checkpoint->h,
+                             topic,
+                             rank,
+                             0,
+                             "{s:s}",
+                             "key", key))
+        || flux_future_then (f,
+                             -1,
+                             checkpoint_get_continuation,
+                             checkpoint) < 0)
+        goto error;
+
+    if (flux_future_aux_set (f,
+                             "msg",
+                             (void *)flux_msg_incref (msg),
+                             (flux_free_f)flux_msg_decref) < 0) {
+        flux_msg_decref (msg);
+        goto error;
+    }
+
+    return 0;
+
+error:
+    (*errstr) = "error starting checkpoint-get RPC";
+    flux_future_destroy (f);
+    return -1;
+}
+
+void content_checkpoint_get_request (flux_t *h, flux_msg_handler_t *mh,
+                                     const flux_msg_t *msg, void *arg)
+{
+    struct content_checkpoint *checkpoint = arg;
+    const char *key;
+    const char *errstr = NULL;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "key", &key) < 0)
+        goto error;
+
+    if (checkpoint->rank == 0
+        && !content_cache_backing_loaded (checkpoint->cache)) {
+        struct checkpoint_data *data = zhashx_lookup (checkpoint->hash, key);
+        if (!data) {
+            errstr = "checkpoint key unavailable";
+            errno = ENOENT;
+            goto error;
+        }
+        if (flux_respond_pack (h, msg,
+                               "{s:O}",
+                               "value", data->value) < 0)
+            flux_log_error (h, "error responding to checkpoint-get");
+        return;
+    }
+
+    if (checkpoint_get_forward (checkpoint,
+                                msg,
+                                key,
+                                &errstr) < 0)
+        goto error;
+
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to checkpoint-get request");
+}
+
+static void checkpoint_put_continuation (flux_future_t *f, void *arg)
+{
+    struct content_checkpoint *checkpoint = arg;
+    const flux_msg_t *msg = flux_future_aux_get (f, "msg");
+    const char *s;
+
+    assert (msg);
+
+    if (flux_rpc_get (f, &s) < 0)
+        goto error;
+
+    if (flux_respond (checkpoint->h, msg, s) < 0)
+        flux_log_error (checkpoint->h, "error responding to checkpoint-put");
+    flux_future_destroy (f);
+    return;
+
+error:
+    if (flux_respond_error (checkpoint->h, msg, errno, NULL) < 0)
+        flux_log_error (checkpoint->h, "error responding to checkpoint-put");
+    flux_future_destroy (f);
+}
+
+static int checkpoint_put_forward (struct content_checkpoint *checkpoint,
+                                   const flux_msg_t *msg,
+                                   const char *key,
+                                   json_t *value,
+                                   const char **errstr)
+{
+    const char *topic = "content.checkpoint-put";
+    uint32_t rank = FLUX_NODEID_UPSTREAM;
+    flux_future_t *f = NULL;
+
+    /* if we're on rank 0, go directly to backing store */
+    if (checkpoint->rank == 0) {
+        topic = "content-backing.checkpoint-put";
+        rank = 0;
+    }
+
+    if (!(f = flux_rpc_pack (checkpoint->h, topic, rank, 0,
+                             "{s:s s:O}",
+                             "key", key,
+                             "value", value))
+        || flux_future_then (f,
+                             -1,
+                             checkpoint_put_continuation,
+                             checkpoint) < 0)
+        goto error;
+
+    if (flux_future_aux_set (f,
+                             "msg",
+                             (void *)flux_msg_incref (msg),
+                             (flux_free_f)flux_msg_decref) < 0) {
+        flux_msg_decref (msg);
+        goto error;
+    }
+
+    return 0;
+
+error:
+    (*errstr) = "error starting checkpoint-put RPC";
+    flux_future_destroy (f);
+    return -1;
+}
+
+void content_checkpoint_put_request (flux_t *h, flux_msg_handler_t *mh,
+                                     const flux_msg_t *msg, void *arg)
+{
+    struct content_checkpoint *checkpoint = arg;
+    const char *key;
+    json_t *value;
+    const char *errstr = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s:o}",
+                             "key",
+                             &key,
+                             "value",
+                             &value) < 0)
+        goto error;
+
+    if (checkpoint->rank == 0) {
+        if (checkpoint_data_update (checkpoint, key, value) < 0)
+            goto error;
+
+        if (!content_cache_backing_loaded (checkpoint->cache)) {
+            if (flux_respond (h, msg, NULL) < 0)
+                flux_log_error (checkpoint->h, "error responding to checkpoint-put");
+            return;
+        }
+    }
+
+    if (checkpoint_put_forward (checkpoint,
+                                msg,
+                                key,
+                                value,
+                                &errstr) < 0)
+        goto error;
+
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to checkpoint-put request");
+}
+
+static void checkpoint_flush_continuation (flux_future_t *f, void *arg)
+{
+    struct checkpoint_data *data = arg;
+    int rv;
+
+    assert (data);
+    if ((rv = flux_rpc_get (f, NULL)) < 0)
+        flux_log_error (data->checkpoint->h, "checkpoint flush rpc");
+    if (!rv) {
+        data->dirty = 0;
+        data->checkpoint->hash_dirty--;
+    }
+    data->in_progress = false;
+    checkpoint_data_decref (data);
+    flux_future_destroy (f);
+}
+
+static int checkpoint_flush (struct content_checkpoint *checkpoint,
+                             struct checkpoint_data *data)
+{
+    if (data->dirty && !data->in_progress) {
+        const char *key = zhashx_cursor (checkpoint->hash);
+        const char *topic = "content-backing.checkpoint-put";
+        flux_future_t *f;
+        if (!(f = flux_rpc_pack (checkpoint->h, topic, 0, 0,
+                                 "{s:s s:O}",
+                                 "key", key,
+                                 "value", data->value))
+            || flux_future_then (f,
+                                 -1,
+                                 checkpoint_flush_continuation,
+                                 (void *)checkpoint_data_incref (data)) < 0) {
+            flux_log_error (checkpoint->h, "%s: checkpoint flush", __FUNCTION__);
+            flux_future_destroy (f);
+            return -1;
+        }
+        data->in_progress = true;
+    }
+    return 0;
+}
+
+int checkpoints_flush (struct content_checkpoint *checkpoint)
+{
+    int last_errno = 0;
+    int rc = 0;
+
+    if (checkpoint->hash_dirty > 0) {
+        struct checkpoint_data *data = zhashx_first (checkpoint->hash);
+        while (data) {
+            if (checkpoint_flush (checkpoint, data) < 0) {
+                last_errno = errno;
+                rc = -1;
+                /* A few errors we will consider "unrecoverable", so
+                 * break out */
+                if (errno == ENOSYS
+                    || errno == ENOMEM)
+                    break;
+            }
+            data = zhashx_next (checkpoint->hash);
+        }
+    }
+    if (rc < 0)
+        errno = last_errno;
+    return rc;
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.checkpoint-get",
+        content_checkpoint_get_request,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.checkpoint-put",
+        content_checkpoint_put_request,
+        0
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void content_checkpoint_destroy (struct content_checkpoint *checkpoint)
+{
+    if (checkpoint) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (checkpoint->handlers);
+        zhashx_destroy (&checkpoint->hash);
+        free (checkpoint);
+        errno = saved_errno;
+    }
+}
+
+struct content_checkpoint *content_checkpoint_create (
+    flux_t *h,
+    uint32_t rank,
+    struct content_cache *cache)
+{
+    struct content_checkpoint *checkpoint;
+
+    if (!(checkpoint = calloc (1, sizeof (*checkpoint))))
+        return NULL;
+    checkpoint->h = h;
+    checkpoint->rank = rank;
+    checkpoint->cache = cache;
+
+    if (!(checkpoint->hash = zhashx_new ()))
+        goto nomem;
+    zhashx_set_destructor (checkpoint->hash, checkpoint_data_decref_wrapper);
+
+    if (flux_msg_handler_addvec (h, htab, checkpoint, &checkpoint->handlers) < 0)
+        goto error;
+    return checkpoint;
+
+nomem:
+    errno = ENOMEM;
+error:
+    content_checkpoint_destroy (checkpoint);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/content/checkpoint.h b/src/modules/content/checkpoint.h
new file mode 100644
index 000000000000..7bd0c5ad6622
--- /dev/null
+++ b/src/modules/content/checkpoint.h
@@ -0,0 +1,28 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _CONTENT_CHECKPOINT_H
+#define _CONTENT_CHECKPOINT_H 1
+
+#include "cache.h"
+
+struct content_checkpoint *content_checkpoint_create (
+    flux_t *h,
+    uint32_t rank,
+    struct content_cache *cache);
+void content_checkpoint_destroy (struct content_checkpoint *checkpoint);
+
+int checkpoints_flush (struct content_checkpoint *checkpoint);
+
+#endif /* !_CONTENT_CHECKPOINT_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/content/main.c b/src/modules/content/main.c
new file mode 100644
index 000000000000..8909abf3e63f
--- /dev/null
+++ b/src/modules/content/main.c
@@ -0,0 +1,42 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* main.c - module main() for content module
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "cache.h"
+
+int mod_main (flux_t *h, int argc, char **argv)
+{
+    struct content_cache *cache;
+    int rc = -1;
+
+    if (!(cache = content_cache_create (h, argc, argv))) {
+        flux_log_error (h, "error initializing content cache");
+        goto done;
+    }
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
+        flux_log_error (h, "reactor exited abnormally");
+        goto done;
+    }
+    rc = 0;
+done:
+    content_cache_destroy (cache);
+    return rc;
+}
+
+MOD_NAME ("content");
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/content/mmap.c b/src/modules/content/mmap.c
new file mode 100644
index 000000000000..5af38dd3b284
--- /dev/null
+++ b/src/modules/content/mmap.c
@@ -0,0 +1,617 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* content-mmap.c - map files into content cache on rank 0
+ *
+ * Each file is represented by a 'struct content_region' that includes
+ * a 'fileref' object containing the file's metadata and blobrefs for content.
+ * The region also contains a pointer to mmap(2)ed memory for the file's
+ * content.
+ *
+ * All files have one or more tags, so the regions are placed in a
+ * hash-of-lists where the list names are tags, and the entries are struct
+ * content_regions.  When files are mapped, the requestor provides a tag.
+ * When files are removed, the requestor provides (only) one or more tags.
+ *
+ * The content-cache calls content_mmap_region_lookup() on rank 0 when it
+ * doesn't have a requested blobref in cache, and only consults the backing
+ * store when that fails.  If content_mmap_region_lookup() succeeds, the
+ * content-cache takes a reference on the struct content_region.  When we
+ * request to unmap a region, the munmap(2) and free of the struct is delayed
+ * until all content-cache references are dropped.
+ *
+ * Slightly tricky optimization:
+ * To speed up content_mmap_region_lookup() we have mm->cache, which is used
+ * to find a content_region given a hash.  The cache contains hash keys for
+ * mmapped data.  A given hash may appear in multiple files or parts of the
+ * same file, so when a file is mapped, we put all its hashes in mm->cache
+ * except those that are already mapped.  If nothing is unmapped, then we know
+ * all the blobrefs for all the files will remain valid.  However when
+ * something is unmapped we could be losing pieces of unrelated files.  Since
+ * unmaps are bulk operations involving tags, we just walk the entire
+ * hash-of-lists at that time and restore any missing cache entries.
+ *
+ * Safety issue:
+ * The content addressable storage model relies on the fact that once hashed,
+ * data does not change.  However, this cannot be assured when the data is
+ * mmapped from a file that may not be protected from updates.  To avoid
+ * propagating bad data in the cache, content_mmap_validate() is called each
+ * time an mmapped cache entry is accessed.  This function recomputes the
+ * hash to make sure the content has not changed.  If the data has changed,
+ * the content-cache returns an error to the requestor.  In addition, mmapped
+ * pages could become invalid if the size of a mapped file is reduced.
+ * Accessing invalid pages could cause the broker to crash with SIGBUS.  To
+ * mitigate this, content_mmap_validate() also calls stat(2) on the file to
+ * make sure the memory region is still valid.  This is not foolproof because
+ * it is inherently a time-of-check-time-of-use problem.  In fact we rate
+ * limit the calls to stat(2) to avoid a "stat storm" when a file with many
+ * blobrefs is accessed, which increases the window where it could have
+ * changed.  But it's likely better than not checking at all.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <jansson.h>
+#include <assert.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/hola.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/blobref.h"
+#include "src/common/libutil/monotime.h"
+#include "src/common/libfilemap/fileref.h"
+#include "ccan/ptrint/ptrint.h"
+#include "ccan/str/str.h"
+
+#include "mmap.h"
+
+struct cache_entry {
+    struct content_region *reg;
+    const void *data;                       // pointer into reg->mapinfo.base
+    size_t size;                            //
+    void *hash;                             // contiguous with struct
+};
+
+struct content_region {
+    struct blobvec_mapinfo mapinfo;
+    int refcount;
+    json_t *fileref;
+
+    struct content_mmap *mm;
+
+    char *fullpath;                         // full path for stat(2) checking
+    struct timespec last_check;             // rate limit stat(2) checking
+};
+
+struct content_mmap {
+    flux_t *h;
+    uint32_t rank;
+    char *hash_name;
+    flux_msg_handler_t **handlers;
+    struct hola *tags;                      // tagged bundles of files/regions
+    zhashx_t *cache;                        // hash digest => cache entry
+};
+
+static int content_hash_size;
+
+static const double max_check_age = 5; // seconds since last region stat(2)
+
+static void cache_entry_destroy (struct cache_entry *e)
+{
+    if (e) {
+        int saved_errno = errno;
+        free (e);
+        errno = saved_errno;
+    }
+}
+
+static struct cache_entry *cache_entry_create (const void *digest)
+{
+    struct cache_entry *e;
+
+    if (!(e = calloc (1, sizeof (*e) + content_hash_size)))
+        return NULL;
+    e->hash = (char *)(e + 1);
+    memcpy (e->hash, digest, content_hash_size);
+    return e;
+}
+
+/* Add entry to cache.
+ * If entry exists, return success.  The blobref must be valid in the
+ * cache - where it comes from is unimportant.
+ * N.B. this is a potential hot spot so defer memory allocation until
+ * after lookup.
+ */
+static int cache_entry_add (struct content_region *reg,
+                            const void *data,
+                            size_t size,
+                            const char *blobref)
+{
+    struct cache_entry *e;
+    char digest[BLOBREF_MAX_DIGEST_SIZE];
+
+    if (blobref_strtohash (blobref, digest, sizeof (digest)) < 0)
+        return -1;
+    if (zhashx_lookup (reg->mm->cache, digest) < 0)
+        return 0;
+    if (!(e = cache_entry_create (digest)))
+        return -1;
+    e->reg = reg;
+    e->data = data;
+    e->size = size;
+    if (zhashx_insert (reg->mm->cache, e->hash, e) < 0) {
+        cache_entry_destroy (e);
+        return 0;
+    }
+    return 0;
+}
+
+/* Remove entry from cache IFF it belongs to this region.
+ */
+static int cache_entry_remove (struct content_mmap *mm,
+                               struct content_region *reg,
+                               const char *blobref)
+{
+    struct cache_entry *e;
+    char digest[BLOBREF_MAX_DIGEST_SIZE];
+
+    if (blobref_strtohash (blobref, digest, sizeof (digest)) < 0)
+        return -1;
+    if ((e = zhashx_lookup (mm->cache, digest)) // calls destructor
+        && reg == e->reg)
+        zhashx_delete (mm->cache, digest);
+    return 0;
+}
+
+/* Remove all cache entries associated with region (blobvec + fileref).
+ */
+static void region_cache_remove (struct content_region *reg)
+{
+    if (reg->fileref) {
+        int saved_errno = errno;
+        const char *encoding = NULL;
+        json_t *data = NULL;
+
+        if (json_unpack (reg->fileref,
+                         "{s?s s?o}",
+                         "encoding", &encoding,
+                         "data", &data) == 0
+            && data != NULL
+            && encoding != NULL
+            && streq (encoding, "blobvec")) {
+            size_t index;
+            json_t *entry;
+
+            json_array_foreach (data, index, entry) {
+                json_int_t offset;
+                json_int_t size;
+                const char *blobref;
+
+                if (json_unpack (entry,
+                                 "[I,I,s]",
+                                 &offset,
+                                 &size,
+                                 &blobref) == 0)
+                    cache_entry_remove (reg->mm, reg, blobref);
+            }
+        }
+        errno = saved_errno;
+    }
+}
+
+/* Add cache entries for entries associated with region
+ */
+static int region_cache_add (struct content_region *reg)
+{
+    size_t index;
+    const char *encoding = NULL;
+    json_t *data = NULL;
+    json_t *entry;
+
+    if (json_unpack (reg->fileref,
+                     "{s?s s?o}",
+                     "encoding", &encoding,
+                     "data", &data) < 0)
+        goto inval;
+    if (data && encoding && streq (encoding, "blobvec")) {
+        json_array_foreach (data, index, entry) {
+            json_int_t offset;
+            json_int_t size;
+            const char *blobref;
+            if (json_unpack (entry, "[I,I,s]", &offset, &size, &blobref) < 0)
+                goto inval;
+            if (cache_entry_add (reg,
+                                 reg->mapinfo.base + offset,
+                                 size,
+                                 blobref) < 0)
+                return -1;
+        }
+    }
+    return 0;
+inval:
+    errno = EINVAL;
+    return -1;
+}
+
+/* After a region is unmapped, other regions may have blobrefs that are no
+ * longer represented in the cache.  This scans all mapped regions and fills
+ * in missing cache entries.  Design tradeoff:  mapping and lookup are fast,
+ * and the cache implementation is lightweight and simple, at the expense of
+ * unmap efficiency.
+ */
+static int plug_cache_holes (struct content_mmap *mm)
+{
+    const char *name;
+    struct content_region *reg;
+
+    name = hola_hash_first (mm->tags);
+    while (name) {
+        reg = hola_list_first (mm->tags, name);
+        while (reg) {
+            if (region_cache_add (reg) < 0)
+                return -1;
+            reg = hola_list_next (mm->tags, name);
+        }
+        name = hola_hash_next (mm->tags);
+    }
+    return 0;
+}
+
+// zhashx_destructor_fn footprint
+static void cache_entry_destructor (void **item)
+{
+    if (item) {
+        cache_entry_destroy (*item);
+        *item = NULL;
+    }
+}
+// zhashx_hash_fn footprint
+static size_t cache_entry_hasher (const void *key)
+{
+    return *(size_t *)key;
+}
+// zhashx_comparator_fn footprint
+static int cache_entry_comparator (const void *item1, const void *item2)
+{
+    return memcmp (item1, item2, content_hash_size);
+}
+
+void content_mmap_region_decref (struct content_region *reg)
+{
+    if (reg && --reg->refcount == 0) {
+        int saved_errno = errno;
+        region_cache_remove (reg);
+        if (reg->mapinfo.base != MAP_FAILED)
+            (void)munmap (reg->mapinfo.base, reg->mapinfo.size);
+        json_decref (reg->fileref);
+        free (reg->fullpath);
+        free (reg);
+        errno = saved_errno;
+    }
+}
+
+// zhashx_destructor_fn footprint
+static void content_mmap_region_destructor (void **item)
+{
+    if (item) {
+        content_mmap_region_decref (*item);
+        *item = NULL;
+    }
+}
+
+struct content_region *content_mmap_region_incref (struct content_region *reg)
+{
+    if (reg)
+        reg->refcount++;
+    return reg;
+}
+
+/* Validate mmapped blob before use, checking for:
+ * - size has changed so mmmapped pages are no longer valid (SIGBUS if used!)
+ * - content no longer matches hash
+ * To avoid repeatedly calling stat(2) on a file, skip it if last check was
+ * within max_check_age seconds.
+ */
+bool content_mmap_validate (struct content_region *reg,
+                            const void *hash,
+                            int hash_size,
+                            const void *data,
+                            int data_size)
+{
+    char hash2[BLOBREF_MAX_DIGEST_SIZE];
+
+    assert (reg->mapinfo.base != NULL);
+    assert (data >= reg->mapinfo.base);
+    assert (data + data_size <= reg->mapinfo.base + reg->mapinfo.size);
+
+    if (monotime_since (reg->last_check)/1000 >= max_check_age) {
+        struct stat sb;
+
+        if (stat (reg->fullpath, &sb) < 0
+            || sb.st_size < reg->mapinfo.size)
+            return false;
+
+        monotime (&reg->last_check);
+    }
+
+    if (blobref_hash_raw (reg->mm->hash_name,
+                          data,
+                          data_size,
+                          hash2,
+                          sizeof (hash2)) != hash_size
+            || memcmp (hash, hash2, hash_size) != 0)
+            return false;
+    return true;
+}
+
+struct content_region *content_mmap_region_lookup (struct content_mmap *mm,
+                                                   const void *hash,
+                                                   int hash_size,
+                                                   const void **data,
+                                                   int *data_size)
+{
+    struct cache_entry *e;
+
+    if (hash_size != content_hash_size
+        || !(e = zhashx_lookup (mm->cache, hash)))
+        return NULL;
+    *data = e->data;
+    *data_size = e->size;
+    return e->reg;
+}
+
+static struct content_region *content_mmap_region_create (
+                                                 struct content_mmap *mm,
+                                                 const char *path,
+                                                 int chunksize,
+                                                 flux_error_t *error)
+{
+    struct blobvec_param param = {
+        .hashtype = mm->hash_name,
+        .chunksize = chunksize,
+        .small_file_threshold = 0, // always choose blobvec encoding here
+    };
+    struct content_region *reg;
+
+    if (!(reg = calloc (1, sizeof (*reg)))) {
+        errprintf (error, "out of memory");
+        goto error;
+    }
+    reg->refcount = 1;
+    reg->mm = mm;
+    reg->mapinfo.base = MAP_FAILED;
+    if (!(reg->fullpath = strdup (path)))
+        goto error;
+
+    if (!(reg->fileref = fileref_create_ex (path,
+                                            &param,
+                                            &reg->mapinfo,
+                                            error)))
+        goto error;
+    /* fileref_create_ex() accepts all file types, but flux-archive(1) should
+     * not be requesting that files be mapped which do not meet criteria.
+     */
+    if (reg->mapinfo.base == MAP_FAILED) {
+        errprintf (error, "%s: not suitable for mapping", path);
+        errno = EINVAL;
+        goto error;
+    }
+    if (region_cache_add (reg) < 0) {
+        errprintf (error,
+                   "%s: error caching region blobrefs: %s",
+                   path,
+                   strerror (errno));
+        goto error;
+    }
+    return reg;
+error:
+    content_mmap_region_decref (reg);
+    return NULL;
+}
+
+static void content_mmap_add_cb (flux_t *h,
+                                 flux_msg_handler_t *mh,
+                                 const flux_msg_t *msg,
+                                 void *arg)
+{
+    struct content_mmap *mm = arg;
+    const char *path;
+    int chunksize;
+    const char *tag;
+    struct content_region *reg = NULL;
+    flux_error_t error;
+    const char *errmsg = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s:i s:s}",
+                             "path", &path,
+                             "chunksize", &chunksize,
+                             "tag", &tag) < 0)
+        goto error;
+    if (mm->rank != 0) {
+        errmsg = "content may only be mmapped on rank 0";
+        goto inval;
+    }
+    if (path[0] != '/') {
+        errmsg = "path must be fully qualified";
+        goto inval;
+    }
+    if (!(reg = content_mmap_region_create (mm, path, chunksize, &error))) {
+        errmsg = error.text;
+        goto error;
+    }
+    // takes a reference on region
+    if (!hola_list_add_end (mm->tags, tag, reg))
+        goto error;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to content.mmap-add request");
+    content_mmap_region_decref (reg);
+    return;
+inval:
+    errno = EINVAL;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to content.mmap-add request");
+    content_mmap_region_decref (reg);
+}
+
+static void content_mmap_remove_cb (flux_t *h,
+                                    flux_msg_handler_t *mh,
+                                    const flux_msg_t *msg,
+                                    void *arg)
+{
+    struct content_mmap *mm = arg;
+    const char *errmsg = NULL;
+    const char *tag;
+    int unmap_count = 0;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "tag", &tag) < 0)
+        goto error;
+    if (mm->rank != 0) {
+        errmsg = "content can only be mmapped on rank 0";
+        goto inval;
+    }
+    if (hola_hash_delete (mm->tags, tag) == 0)
+        unmap_count++;
+    if (unmap_count > 0) {
+        if (plug_cache_holes (mm) < 0) {
+            errmsg = "error filling missing cache entries after unmap";
+            goto error;
+        }
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to content.mmap-remove request");
+    return;
+inval:
+    errno = EINVAL;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to content.mmap-remove request");
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.mmap-add",
+        content_mmap_add_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "content.mmap-remove",
+        content_mmap_remove_cb,
+        0
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+json_t *content_mmap_get_stats (struct content_mmap *mm)
+{
+    const char *key;
+    json_t *o;
+    json_t *mmap;
+
+    if (!(o = json_object ()))
+        goto nomem;
+    key = hola_hash_first (mm->tags);
+    while (key) {
+        struct content_region *reg;
+        json_t *a;
+        if (!(a = json_array ()))
+            goto nomem;
+        reg = hola_list_first (mm->tags, key);
+        while (reg) {
+            json_t *s;
+            if (!(s = json_string (reg->fullpath))
+                || json_array_append_new (a, s) < 0) {
+                json_decref (s);
+                json_decref (a);
+                goto nomem;
+            }
+            reg = hola_list_next (mm->tags, key);
+        }
+        if (json_object_set_new (o, key, a) < 0) {
+            json_decref (a);
+            goto nomem;
+        }
+        key = hola_hash_next (mm->tags);
+    }
+    if (!(mmap = json_pack ("{s:O s:I}",
+                            "tags", o,
+                            "blobs", zhashx_size (mm->cache))))
+        goto nomem;
+    json_decref (o);
+    return mmap;
+nomem:
+    json_decref (o);
+    return NULL;
+}
+
+void content_mmap_destroy (struct content_mmap *mm)
+{
+    if (mm) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (mm->handlers);
+        hola_destroy (mm->tags);
+        zhashx_destroy (&mm->cache);
+        free (mm->hash_name);
+        free (mm);
+        errno = saved_errno;
+    }
+}
+
+struct content_mmap *content_mmap_create (flux_t *h,
+                                          const char *hash_name,
+                                          int hash_size)
+{
+    struct content_mmap *mm;
+
+    if (!(mm = calloc (1, sizeof (*mm))))
+        return NULL;
+    content_hash_size = hash_size;
+    if (flux_get_rank (h, &mm->rank) < 0)
+        goto error;
+    if (!(mm->hash_name = strdup (hash_name)))
+        goto error;
+    mm->h = h;
+    if (flux_msg_handler_addvec (h, htab, mm, &mm->handlers) < 0)
+        goto error;
+    if (!(mm->tags = hola_create (HOLA_AUTOCREATE)))
+        goto error;
+    hola_set_list_destructor (mm->tags, content_mmap_region_destructor);
+    hola_set_list_duplicator (mm->tags,
+                         (zlistx_duplicator_fn *)content_mmap_region_incref);
+    if (!(mm->cache = zhashx_new ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    zhashx_set_destructor (mm->cache, cache_entry_destructor);
+    zhashx_set_key_hasher (mm->cache, cache_entry_hasher);
+    zhashx_set_key_comparator (mm->cache, cache_entry_comparator);
+    zhashx_set_key_destructor (mm->cache, NULL); // key is part of entry
+    zhashx_set_key_duplicator (mm->cache, NULL); // key is part of entry
+    return mm;
+error:
+    content_mmap_destroy (mm);
+    return NULL;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/content/mmap.h b/src/modules/content/mmap.h
new file mode 100644
index 000000000000..9869bf2482c5
--- /dev/null
+++ b/src/modules/content/mmap.h
@@ -0,0 +1,43 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _CONTENT_MMAP_H
+#define _CONTENT_MMAP_H 1
+
+#include <stdbool.h>
+#include <jansson.h>
+
+#include "cache.h"
+
+struct content_mmap *content_mmap_create (flux_t *h,
+                                          const char *hash_name,
+                                          int hash_size);
+void content_mmap_destroy (struct content_mmap *mm);
+
+json_t *content_mmap_get_stats (struct content_mmap *mm);
+
+struct content_region *content_mmap_region_lookup (struct content_mmap *mm,
+                                                   const void *hash,
+                                                   int hash_size,
+                                                   const void **data,
+                                                   int *data_size);
+
+bool content_mmap_validate (struct content_region *reg,
+                            const void *hash,
+                            int hash_size,
+                            const void *data,
+                            int data_size);
+
+void content_mmap_region_decref (struct content_region *reg);
+struct content_region *content_mmap_region_incref (struct content_region *reg);
+
+#endif /* !_CONTENT_MMAP_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/cron/Makefile.am b/src/modules/cron/Makefile.am
deleted file mode 100644
index b8541a421779..000000000000
--- a/src/modules/cron/Makefile.am
+++ /dev/null
@@ -1,33 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
-
-#
-# Comms module
-#
-fluxmod_LTLIBRARIES = cron.la
-
-cron_la_SOURCES = \
-	task.h \
-	task.c \
-	entry.h \
-	types.h \
-	types.c \
-	interval.c \
-	event.c \
-	datetime.c \
-	cron.c
-
-cron_la_LDFLAGS = $(fluxmod_ldflags) -module
-cron_la_LIBADD = $(top_builddir)/src/common/libflux-internal.la \
-		 $(top_builddir)/src/common/libflux-core.la \
-		 $(ZMQ_LIBS)
diff --git a/src/modules/cron/cron.c b/src/modules/cron/cron.c
index 9544998defa9..50a13692657b 100644
--- a/src/modules/cron/cron.c
+++ b/src/modules/cron/cron.c
@@ -28,12 +28,17 @@
 #include <stdbool.h>
 #include <sys/time.h>
 #include <ctype.h>
+#include <signal.h>
 #include <flux/core.h>
-#include <czmq.h>
 #include <jansson.h>
+#ifndef HAVE_GET_CURRENT_DIR_NAME
+#include "src/common/libmissing/get_current_dir_name.h"
+#endif
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/fsd.h"
+#include "ccan/str/str.h"
 
 #include "task.h"
 #include "entry.h"
@@ -59,10 +64,14 @@ struct cron_ctx {
 static cron_entry_t *cron_entry_create (cron_ctx_t *ctx, const flux_msg_t *msg);
 static void cron_entry_destroy (cron_entry_t *e);
 static int cron_entry_stop (cron_entry_t *e);
-static void cron_entry_finished_handler (flux_t *h, cron_task_t *t,
-    void *arg);
-static void cron_entry_io_cb (flux_t *h, cron_task_t *t, void *arg,
-    bool is_stderr, const char *data, int datalen);
+static void cron_entry_finished_handler (flux_t *h,
+                                         cron_task_t *t,
+                                         void *arg);
+static void cron_entry_io_cb (flux_t *h,
+                              cron_task_t *t,
+                              void *arg,
+                              bool is_stderr,
+                              const char *data, int datalen);
 static int cron_entry_run_task (cron_entry_t *e);
 static int cron_entry_defer (cron_entry_t *e);
 
@@ -81,19 +90,11 @@ double get_timestamp (void)
     return ((double) tm.tv_sec + (tm.tv_nsec/1.0e9));
 }
 
-static void timeout_cb (flux_t *h, cron_task_t *t, void *arg)
-{
-    cron_entry_t *e = arg;
-    flux_log (h, LOG_INFO, "cron-%ju: task timeout at %.2fs. Killing",
-              e->id, e->timeout);
-    cron_task_kill (t, SIGTERM);
-}
-
 static int cron_entry_run_task (cron_entry_t *e)
 {
     flux_t *h = e->ctx->h;
     if (cron_task_run (e->task, e->rank, e->command, e->cwd, e->env) < 0) {
-        flux_log_error (h, "cron-%ju: cron_task_run", e->id);
+        flux_log_error (h, "cron-%ju: cron_task_run", (uintmax_t)e->id);
         /* Run "finished" handler since this task is done */
         cron_entry_finished_handler (h, e->task, e);
         return (-1);
@@ -115,8 +116,11 @@ int cron_entry_schedule_task (cron_entry_t *e)
     /* Refuse to run more than one task at once
      */
     if (e->task) {
-        flux_log (h, LOG_INFO, "cron-%ju: %s: task still running or scheduled",
-                  e->id, e->name);
+        flux_log (h,
+                  LOG_INFO,
+                  "cron-%ju: %s: task still running or scheduled",
+                  (uintmax_t)e->id,
+                  e->name);
         return (0);
     }
     if (!(e->task = cron_task_new (h, cron_entry_finished_handler, e))) {
@@ -125,7 +129,7 @@ int cron_entry_schedule_task (cron_entry_t *e)
     }
     cron_task_on_io (e->task, cron_entry_io_cb);
     if (e->timeout >= 0.0)
-        cron_task_set_timeout (e->task, e->timeout, timeout_cb);
+        cron_task_set_timeout (e->task, e->timeout, NULL);
 
     /*   if we've reached our (non-zero) repeat count, prematurely stop
      *     the current entry (i.e. remove it from event loop, but leave
@@ -145,8 +149,14 @@ static void cron_entry_io_cb (flux_t *h, cron_task_t *t, void *arg,
 {
     cron_entry_t *e = arg;
     int level = is_stderr ? LOG_ERR : LOG_INFO;
-    flux_log (h, level, "cron-%ju[%s]: rank=%d: command=\"%s\": \"%s\"",
-              e->id, e->name, e->rank, e->command, data);
+    flux_log (h,
+              level,
+              "cron-%ju[%s]: rank=%d: command=\"%s\": \"%s\"",
+              (uintmax_t)e->id,
+              e->name,
+              e->rank,
+              e->command,
+              data);
 }
 
 /* Push task t onto the front of the completed tasks list for
@@ -172,9 +182,11 @@ static void cron_entry_failure (cron_entry_t *e)
     e->stats.failure++;
     e->stats.failcount++;
     if (e->stop_on_failure && e->stats.failcount >= e->stop_on_failure) {
-        flux_log (e->ctx->h, LOG_ERR,
-                "cron-%ju: exceeded failure limit of %d. stopping",
-                e->id, e->stop_on_failure);
+        flux_log (e->ctx->h,
+                  LOG_ERR,
+                  "cron-%ju: exceeded failure limit of %d. stopping",
+                  (uintmax_t)e->id,
+                  e->stop_on_failure);
         cron_entry_stop (e);
     }
 }
@@ -183,17 +195,26 @@ static void cron_entry_finished_handler (flux_t *h, cron_task_t *t, void *arg)
 {
     cron_entry_t *e = arg;
 
-    if (strcmp (cron_task_state (t), "Exec Failure") == 0) {
-        flux_log_error (h, "cron-%ju: failed to run %s", e->id, e->command);
+    if (streq (cron_task_state (t), "Exec Failure")) {
+        flux_log_error (h,
+                        "cron-%ju: failed to run %s",
+                        (uintmax_t)e->id,
+                        e->command);
         cron_entry_failure (e);
     }
-    else if (strcmp (cron_task_state (t), "Rexec Failure") == 0) {
-        flux_log_error (h, "cron-%ju: failure running %s", e->id, e->command);
+    else if (streq (cron_task_state (t), "Rexec Failure")) {
+        flux_log_error (h,
+                        "cron-%ju: failure running %s",
+                        (uintmax_t)e->id,
+                        e->command);
         cron_entry_failure (e);
     }
     else if (cron_task_status (t) != 0) {
-        flux_log (h, LOG_ERR, "cron-%ju: \"%s\": Failed: %s",
-                 e->id, e->command, cron_task_state (t));
+        flux_log (h,
+                  LOG_ERR, "cron-%ju: \"%s\": Failed: %s",
+                  (uintmax_t)e->id,
+                  e->command,
+                  cron_task_state (t));
         cron_entry_failure (e);
     }
     else
@@ -230,8 +251,10 @@ static int cron_entry_stop (cron_entry_t *e)
 /*
  * Callback used to stop a cron entry safely.
  */
-static void entry_stop_cb (flux_reactor_t *r, flux_watcher_t *w,
-                           int revents, void *arg)
+static void entry_stop_cb (flux_reactor_t *r,
+                           flux_watcher_t *w,
+                           int revents,
+                           void *arg)
 {
     cron_entry_stop (arg);
     flux_watcher_stop (w);
@@ -239,14 +262,15 @@ static void entry_stop_cb (flux_reactor_t *r, flux_watcher_t *w,
 }
 
 /* Stop cron entry `e` "safely" by waiting until the next
- *  "prepare" callback. Temporary watcher created here wil lbe
+ *  "prepare" callback. Temporary watcher created here will be
  *  destroyed within prepare_cb.
  */
 int cron_entry_stop_safe (cron_entry_t *e)
 {
     flux_reactor_t *r = flux_get_reactor (e->ctx->h);
     flux_watcher_t *w = flux_prepare_watcher_create (r,
-                            entry_stop_cb, e);
+                                                     entry_stop_cb,
+                                                     e);
     if (!w)
         return (-1);
     flux_watcher_start (w);
@@ -318,8 +342,10 @@ static void cron_entry_destroy (cron_entry_t *e)
     free (e);
 }
 
-static void deferred_cb (flux_t *h, flux_msg_handler_t *mh,
-                         const flux_msg_t *msg, void *arg)
+static void deferred_cb (flux_t *h,
+                         flux_msg_handler_t *mh,
+                         const flux_msg_t *msg,
+                         void *arg)
 {
     cron_ctx_t *ctx = arg;
     cron_entry_t *e;
@@ -346,8 +372,11 @@ static int cron_entry_defer (cron_entry_t *e)
     if (zlist_push (ctx->deferred, e) < 0)
         return (-1);
     e->stats.deferred++;
-    flux_log (ctx->h, LOG_DEBUG, "deferring cron-%ju to next %s event",
-             e->id, ctx->sync_event);
+    flux_log (ctx->h,
+              LOG_DEBUG,
+              "deferring cron-%ju to next %s event",
+              (uintmax_t)e->id,
+              ctx->sync_event);
 
     if (zlist_size (ctx->deferred) == 1)
         flux_msg_handler_start (ctx->mh);
@@ -376,11 +405,12 @@ static cron_entry_t *cron_entry_create (cron_ctx_t *ctx, const flux_msg_t *msg)
     int saved_errno = EPROTO;
 
     /* Get required fields "type", "name" and "command" */
-    if (flux_msg_unpack (msg, "{ s:s, s:s, s:s, s:O }",
-            "type", &type,
-            "name", &name,
-            "command", &command,
-            "args", &args) < 0) {
+    if (flux_msg_unpack (msg,
+                         "{ s:s, s:s, s:s, s:O }",
+                         "type", &type,
+                         "name", &name,
+                         "command", &command,
+                         "args", &args) < 0) {
         flux_log_error (h, "cron.create: Failed to get name/command/args");
         goto done;
     }
@@ -409,23 +439,21 @@ static cron_entry_t *cron_entry_create (cron_ctx_t *ctx, const flux_msg_t *msg)
     e->stop_on_failure = 0;    /* Whether the cron job is stopped on failure */
     e->timeout = -1.0;         /* Task timeout (default -1, no timeout)      */
 
-    if (flux_msg_unpack (msg, "{ s?O, s?s, s?i, s?i, s?i, s?i, s?F }",
-            "environ",            &e->env,
-            "cwd",                &cwd,
-            "repeat",             &e->repeat,
-            "rank",               &e->rank,
-            "task-history-count", &e->task_history_count,
-            "stop-on-failure",    &e->stop_on_failure,
-            "timeout",            &e->timeout) < 0) {
+    if (flux_msg_unpack (msg,
+                         "{ s?O, s?s, s?i, s?i, s?i, s?i, s?F }",
+                         "environ", &e->env,
+                         "cwd", &cwd,
+                         "repeat", &e->repeat,
+                         "rank", &e->rank,
+                         "task-history-count", &e->task_history_count,
+                         "stop-on-failure", &e->stop_on_failure,
+                         "timeout", &e->timeout) < 0) {
         saved_errno = EPROTO;
         flux_log_error (h, "cron.create: flux_msg_unpack");
         goto out_err;
     }
 
-    if (!cwd)
-        cwd = ctx->cwd;
-
-    if ((e->cwd = strdup (cwd)) == NULL) {
+    if (cwd && (e->cwd = strdup (cwd)) == NULL) {
         flux_log_error (h, "cron.create: strdup (cwd)");
         errno = ENOMEM;
         goto out_err;
@@ -504,16 +532,20 @@ static int cron_ctx_sync_event_init (cron_ctx_t *ctx, const char *topic)
 {
     struct flux_match match = FLUX_MATCH_EVENT;
 
-    flux_log (ctx->h, LOG_INFO,
-        "synchronizing cron tasks to event %s", topic);
+    flux_log (ctx->h,
+              LOG_INFO,
+              "synchronizing cron tasks to event %s",
+              topic);
 
     if ((ctx->sync_event = strdup (topic)) == NULL) {
         flux_log_error (ctx->h, "sync_event_init: strdup");
         return (-1);
     }
     match.topic_glob = ctx->sync_event;
-    ctx->mh = flux_msg_handler_create (ctx->h, match,
-                                       deferred_cb, (void *) ctx);
+    ctx->mh = flux_msg_handler_create (ctx->h,
+                                       match,
+                                       deferred_cb,
+                                       (void *) ctx);
     if (!ctx->mh) {
         flux_log_error (ctx->h, "sync_event_init: msg_handler_create");
         return (-1);
@@ -522,7 +554,7 @@ static int cron_ctx_sync_event_init (cron_ctx_t *ctx, const char *topic)
         flux_log_error (ctx->h, "sync_event_init: subscribe (%s)", topic);
         return (-1);
     }
-    /* Do not start the handler until we have entries on the the list */
+    /* Do not start the handler until we have entries on the list */
     return (0);
 }
 
@@ -586,13 +618,13 @@ static json_t *cron_entry_to_json (cron_entry_t *e)
      *  Common entry contents:
      */
     if (!(o = json_pack ("{ s:I, s:i, s:s, s:s, s:i, s:b, s:s }",
-                   "id", (json_int_t) e->id,
-                   "rank", e->rank,
-                   "name", e->name,
-                   "command", e->command,
-                   "repeat", e->repeat,
-                   "stopped", e->stopped,
-                   "type", e->typename)))
+                         "id", (json_int_t) e->id,
+                         "rank", e->rank,
+                         "name", e->name,
+                         "command", e->command,
+                         "repeat", e->repeat,
+                         "stopped", e->stopped,
+                         "type", e->typename)))
         return NULL;
 
     if (e->timeout >= 0.0)
@@ -636,8 +668,10 @@ static json_t *cron_entry_to_json (cron_entry_t *e)
 /*
  *  Handle cron.create: create a new cron entry
  */
-static void cron_create_handler (flux_t *h, flux_msg_handler_t *w,
-                                  const flux_msg_t *msg, void *arg)
+static void cron_create_handler (flux_t *h,
+                                 flux_msg_handler_t *w,
+                                 const flux_msg_t *msg,
+                                 void *arg)
 {
     cron_entry_t *e;
     cron_ctx_t *ctx = arg;
@@ -665,13 +699,16 @@ static void cron_create_handler (flux_t *h, flux_msg_handler_t *w,
         flux_log_error (h, "cron.request: flux_respond_error");
 }
 
-static void cron_sync_handler (flux_t *h, flux_msg_handler_t *w,
-                                const flux_msg_t *msg, void *arg)
+static void cron_sync_handler (flux_t *h,
+                               flux_msg_handler_t *w,
+                               const flux_msg_t *msg,
+                               void *arg)
 {
     cron_ctx_t *ctx = arg;
     const char *topic;
     int disable;
     double epsilon;
+    bool sync_event_before = ctx->sync_event ? true : false;
 
     if (flux_request_unpack (msg, NULL, "{}") < 0)
         goto error;
@@ -685,17 +722,35 @@ static void cron_sync_handler (flux_t *h, flux_msg_handler_t *w,
     if (topic) {
         if (cron_ctx_sync_event_init (ctx, topic) < 0)
             goto error;
+        /* If we changed the sync event, restart the message handler
+         * if there are any current deferred jobs
+         */
+        if (zlist_size (ctx->deferred) > 0)
+            flux_msg_handler_start (ctx->mh);
     }
 
     if (!flux_request_unpack (msg, NULL, "{ s:F }", "sync_epsilon", &epsilon))
         ctx->sync_epsilon = epsilon;
 
     if (ctx->sync_event) {
-        if (flux_respond_pack (h, msg, "{ s:s s:f }",
+        if (flux_respond_pack (h,
+                               msg,
+                               "{ s:s s:f }",
                                "sync_event", ctx->sync_event,
                                "sync_epsilon", ctx->sync_epsilon) < 0)
             flux_log_error (h, "cron.request: flux_respond_pack");
     } else {
+        if (sync_event_before) {
+            /* If we just disabled a sync event, any cron jobs on the
+             * deferred list can never be executed (i.e. the deferred
+             * callback can never be triggered now).  These deferred
+             * jobs would have already been executed if there wasn't a
+             * sync event, so just execute them right now.
+             */
+            cron_entry_t *e;
+            while ((e = zlist_pop (ctx->deferred)))
+                cron_entry_run_task (e);
+        }
         if (flux_respond_pack (h, msg, "{ s:b }", "sync_disabled", true) < 0)
             flux_log_error (h, "cron.request: flux_respond_pack");
     }
@@ -718,8 +773,10 @@ static cron_entry_t *cron_ctx_find_entry (cron_ctx_t *ctx, int64_t id)
  *  Return a cron entry referenced by request in flux message msg.
  *  [service] is name of service for logging purposes.
  */
-static cron_entry_t *entry_from_request (flux_t *h, const flux_msg_t *msg,
-                                         cron_ctx_t *ctx, const char *service)
+static cron_entry_t *entry_from_request (flux_t *h,
+                                         const flux_msg_t *msg,
+                                         cron_ctx_t *ctx,
+                                         const char *service)
 {
     int64_t id;
 
@@ -735,8 +792,10 @@ static cron_entry_t *entry_from_request (flux_t *h, const flux_msg_t *msg,
 /*
  *  "cron.delete" handler
  */
-static void cron_delete_handler (flux_t *h, flux_msg_handler_t *w,
-                                 const flux_msg_t *msg, void *arg)
+static void cron_delete_handler (flux_t *h,
+                                 flux_msg_handler_t *w,
+                                 const flux_msg_t *msg,
+                                 void *arg)
 {
     cron_entry_t *e;
     cron_ctx_t *ctx = arg;
@@ -769,8 +828,10 @@ static void cron_delete_handler (flux_t *h, flux_msg_handler_t *w,
 /*
  *  "cron.stop" handler: stop a cron entry until restarted
  */
-static void cron_stop_handler (flux_t *h, flux_msg_handler_t *w,
-                               const flux_msg_t *msg, void *arg)
+static void cron_stop_handler (flux_t *h,
+                               flux_msg_handler_t *w,
+                               const flux_msg_t *msg,
+                               void *arg)
 {
     cron_entry_t *e;
     cron_ctx_t *ctx = arg;
@@ -797,8 +858,10 @@ static void cron_stop_handler (flux_t *h, flux_msg_handler_t *w,
 /*
  *  "cron.start" handler: start a stopped cron entry
  */
-static void cron_start_handler (flux_t *h, flux_msg_handler_t *w,
-                               const flux_msg_t *msg, void *arg)
+static void cron_start_handler (flux_t *h,
+                                flux_msg_handler_t *w,
+                                const flux_msg_t *msg,
+                                void *arg)
 {
     cron_entry_t *e;
     cron_ctx_t *ctx = arg;
@@ -826,8 +889,10 @@ static void cron_start_handler (flux_t *h, flux_msg_handler_t *w,
 /*
  *  Handle "cron.list" -- dump a list of current cron entries via JSON
  */
-static void cron_ls_handler (flux_t *h, flux_msg_handler_t *w,
-                             const flux_msg_t *msg, void *arg)
+static void cron_ls_handler (flux_t *h,
+                             flux_msg_handler_t *w,
+                             const flux_msg_t *msg,
+                             void *arg)
 {
     cron_ctx_t *ctx = arg;
     cron_entry_t *e = NULL;
@@ -872,20 +937,24 @@ static const struct flux_msg_handler_spec htab[] = {
     FLUX_MSGHANDLER_TABLE_END,
 };
 
-static void process_args (cron_ctx_t *ctx, int ac, char **av)
+static int process_args (cron_ctx_t *ctx, int ac, char **av)
 {
     int i;
     for (i = 0; i < ac; i++) {
-        if (strncmp (av[i], "sync=", 5) == 0)
+        if (strstarts (av[i], "sync="))
             cron_ctx_sync_event_init (ctx, (av[i])+5);
-        else if (strncmp (av[i], "sync_epsilon=", 13) == 0) {
+        else if (strstarts (av[i], "sync_epsilon=")) {
             char *s = (av[i])+13;
             if (fsd_parse_duration (s, &ctx->sync_epsilon) < 0)
                 flux_log_error (ctx->h, "option %s ignored", av[i]);
         }
-        else
+        else {
             flux_log (ctx->h, LOG_ERR, "Unknown option `%s'", av[i]);
+            errno = EINVAL;
+            return -1;
+        }
     }
+    return 0;
 }
 
 
@@ -897,7 +966,8 @@ int mod_main (flux_t *h, int ac, char **av)
     if (ctx == NULL)
         return -1;
 
-    process_args (ctx, ac, av);
+    if (process_args (ctx, ac, av) < 0)
+        goto done;
 
     if (flux_msg_handler_addvec (h, htab, ctx, &handlers) < 0) {
         flux_log_error (h, "flux_msg_handler_addvec");
@@ -911,8 +981,6 @@ int mod_main (flux_t *h, int ac, char **av)
     return rc;
 }
 
-MOD_NAME ("cron");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/cron/datetime.c b/src/modules/cron/datetime.c
index 93f33663c897..7577418acc1d 100644
--- a/src/modules/cron/datetime.c
+++ b/src/modules/cron/datetime.c
@@ -15,9 +15,9 @@
 #include <sys/time.h>
 #include <ctype.h>
 #include <flux/core.h>
-#include <czmq.h>
 
 #include "src/common/libutil/cronodate.h"
+#include "src/common/libutil/errno_safe.h"
 
 #include "entry.h"
 
@@ -39,7 +39,10 @@ struct datetime_entry * datetime_entry_create ()
 {
     struct datetime_entry *dt = calloc (1, sizeof (*dt));
     if (dt) {
-        dt->d = cronodate_create ();
+        if (!(dt->d = cronodate_create ())) {
+            ERRNO_SAFE_WRAP (free, dt);
+            return NULL;
+        }
         /*  Fill cronodate set initially. The cronodate object will
          *  be refined when json arguments from user are processed
          */
@@ -53,6 +56,9 @@ static struct datetime_entry * datetime_entry_from_json (json_t *o)
     int i, rc = 0;
     struct datetime_entry *dt = datetime_entry_create ();
 
+    if (!dt)
+        return NULL;
+
     for (i = 0; i < TM_MAX_ITEM; i++) {
         json_t *val;
         /*  Time unit members of the json arguments are optional.
@@ -108,7 +114,8 @@ static double reschedule_cb (flux_watcher_t *w, double now, void *arg)
          */
         if (e->repeat == 0 || e->repeat < e->stats.count + 1) {
             flux_log_error (dt->h,
-                    "cron-%ju: Unable to get next wakeup. Stopping.", e->id);
+                            "cron-%ju: Unable to get next wakeup. Stopping.",
+                            (uintmax_t)e->id);
         }
         cron_entry_stop_safe (e);
         return now + 1.e19;
@@ -123,9 +130,10 @@ static void *cron_datetime_create (flux_t *h, cron_entry_t *e, json_t *arg)
         return (NULL);
     dt->h = h;
     dt->w = flux_periodic_watcher_create (flux_get_reactor (h),
-                                         0., 0.,
-                                         reschedule_cb,
-                                         datetime_cb, (void *) e);
+                                          0.,
+                                          0.,
+                                          reschedule_cb,
+                                          datetime_cb, (void *) e);
     if (dt->w == NULL) {
         flux_log_error (h, "periodic_watcher_create");
         datetime_entry_destroy (dt);
diff --git a/src/modules/cron/entry.h b/src/modules/cron/entry.h
index f91794b07a15..e6b868cb6409 100644
--- a/src/modules/cron/entry.h
+++ b/src/modules/cron/entry.h
@@ -11,10 +11,11 @@
 #ifndef HAVE_CRON_ENTRY_H
 # define HAVE_CRON_ENTRY_H
 
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
 typedef struct cron_ctx cron_ctx_t;
 typedef struct cron_entry cron_entry_t;
 
diff --git a/src/modules/cron/event.c b/src/modules/cron/event.c
index 81981680fd0b..90688ef1fcba 100644
--- a/src/modules/cron/event.c
+++ b/src/modules/cron/event.c
@@ -26,8 +26,10 @@ struct cron_event {
     char *event;
 };
 
-static void ev_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                         int revents, void *arg)
+static void ev_timer_cb (flux_reactor_t *r,
+                         flux_watcher_t *w,
+                         int revents,
+                         void *arg)
 {
     cron_entry_t *e = arg;
     struct cron_event *ev = cron_entry_type_data (e);
@@ -37,8 +39,10 @@ static void ev_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
     ev->paused = 0;
 }
 
-static void event_handler (flux_t *h, flux_msg_handler_t *w,
-                           const flux_msg_t *msg, void *arg)
+static void event_handler (flux_t *h,
+                           flux_msg_handler_t *w,
+                           const flux_msg_t *msg,
+                           void *arg)
 {
     cron_entry_t *e = arg;
     struct cron_event *ev = cron_entry_type_data (e);
@@ -63,8 +67,11 @@ static void event_handler (flux_t *h, flux_msg_handler_t *w,
         if (remaining > 1e-5) {
             flux_reactor_t *r = flux_get_reactor (h);
             flux_watcher_t *w;
-            if (!(w = flux_timer_watcher_create (r, remaining, 0.,
-                                                 ev_timer_cb, (void *) e)))
+            if (!(w = flux_timer_watcher_create (r,
+                                                 remaining,
+                                                 0.,
+                                                 ev_timer_cb,
+                                                 (void *) e)))
                 flux_log_error (h, "timer_watcher_create");
             else {
                 /* Pause the event watcher. Continue to count events but
@@ -72,9 +79,11 @@ static void event_handler (flux_t *h, flux_msg_handler_t *w,
                  */
                 ev->paused = 1;
                 flux_watcher_start (w);
-                flux_log (h, LOG_DEBUG,
+                flux_log (h,
+                          LOG_DEBUG,
                           "cron-%ju: delaying %4.03fs due to min interval",
-                          e->id, remaining);
+                          (uintmax_t)e->id,
+                          remaining);
             }
             return;
         }
@@ -105,11 +114,12 @@ static void *cron_event_create (flux_t *h, cron_entry_t *e, json_t *arg)
     const char *event;
     struct flux_match match = FLUX_MATCH_EVENT;
 
-    if (json_unpack (arg, "{ s:s, s?i, s?i, s?F }",
-                          "topic", &event,
-                          "nth",   &nth,
-                          "after", &after,
-                          "min_interval", &min_interval) < 0) {
+    if (json_unpack (arg,
+                     "{s:s s?i s?i s?F}",
+                     "topic", &event,
+                     "nth",   &nth,
+                     "after", &after,
+                     "min_interval", &min_interval) < 0) {
         flux_log_error (h, "cron event: json_unpack");
         errno = EPROTO;
         return NULL;
diff --git a/src/modules/cron/interval.c b/src/modules/cron/interval.c
index 2f387232fac7..644591d04729 100644
--- a/src/modules/cron/interval.c
+++ b/src/modules/cron/interval.c
@@ -26,8 +26,10 @@ struct cron_interval {
 };
 
 
-static void interval_handler (flux_reactor_t *r, flux_watcher_t *w,
-                              int revents, void *arg)
+static void interval_handler (flux_reactor_t *r,
+                              flux_watcher_t *w,
+                              int revents,
+                              void *arg)
 {
     cron_entry_schedule_task ((cron_entry_t *)arg);
 }
@@ -40,9 +42,10 @@ static void *cron_interval_create (flux_t *h, cron_entry_t *e, json_t *arg)
     /*  Unpack 'interval' and 'after' arguments. If after was not specified,
      *   (and thus is still < 0.0), then it is set to interval by default.
      */
-    if (json_unpack (arg, "{ s:F, s?F }",
-                          "interval", &i,
-                          "after", &after) < 0) {
+    if (json_unpack (arg,
+                     "{s:F s?F}",
+                     "interval", &i,
+                     "after", &after) < 0) {
         return NULL;
     }
     if (after < 0.0)
@@ -55,7 +58,9 @@ static void *cron_interval_create (flux_t *h, cron_entry_t *e, json_t *arg)
     iv->seconds = i;
     iv->after = after;
     iv->w = flux_timer_watcher_create (flux_get_reactor (h),
-                                       after, i, interval_handler,
+                                       after,
+                                       i,
+                                       interval_handler,
                                        (void *) e);
     if (!iv->w) {
         flux_log_error (h, "cron_interval: flux_timer_watcher_create");
@@ -86,9 +91,9 @@ static void cron_interval_stop (void *arg)
 static json_t *cron_interval_to_json (void *arg)
 {
     struct cron_interval *iv = arg;
-    return json_pack ("{ s:f, s:f, s:f }",
-                      "interval",    iv->seconds,
-                      "after",       iv->after,
+    return json_pack ("{s:f s:f s:f}",
+                      "interval", iv->seconds,
+                      "after", iv->after,
                       "next_wakeup", flux_watcher_next_wakeup (iv->w));
 }
 
diff --git a/src/modules/cron/task.c b/src/modules/cron/task.c
index 2c99c39a98da..33d97934c4d3 100644
--- a/src/modules/cron/task.c
+++ b/src/modules/cron/task.c
@@ -20,6 +20,7 @@
 #include <assert.h>
 
 #include "src/common/libutil/log.h"
+#include "ccan/str/str.h"
 
 #include "task.h"
 
@@ -29,14 +30,13 @@ struct cron_task {
 
     int                   rank;   /* rank on which task is being run        */
     pid_t                 pid;    /* remote process id                      */
-    char *                state;  /* state string returned by cmb.exec      */
+    char *                state;  /* state string returned by broker.exec   */
 
     double               timeout;
     flux_watcher_t *   timeout_w;
 
     int                   status; /* exit status if state is Exited         */
     int              rexec_errno; /* any errno returned by rexec service    */
-    int               exec_errno; /* any errno returned by remote exec(2)   */
 
     struct timespec   createtime; /* Time at which task was created         */
     struct timespec    starttime; /* Time at which exec request was sent    */
@@ -151,13 +151,6 @@ void cron_task_set_timeout (cron_task_t *t, double to, cron_task_state_f cb)
         cron_task_timeout_start (t);
 }
 
-static void cron_task_exec_failed (cron_task_t *t, int errnum)
-{
-    t->exec_failed = 1;
-    t->exec_errno = errnum;
-    cron_task_state_update (t, "Exec Failure");
-}
-
 static void cron_task_rexec_failed (cron_task_t *t, int errnum)
 {
     t->rexec_failed = 1;
@@ -188,24 +181,21 @@ static void completion_cb (flux_subprocess_t *p)
     cron_task_handle_finished (p, t);
 }
 
-static void state_change_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
+static void state_change_cb (flux_subprocess_t *p,
+                             flux_subprocess_state_t state)
 {
     cron_task_t *t = flux_subprocess_aux_get (p, "task");
 
     assert (t);
 
-    cron_task_state_update (t, flux_subprocess_state_string (state));
+    cron_task_state_update (t, "%s", flux_subprocess_state_string (state));
 
     if (state == FLUX_SUBPROCESS_RUNNING) {
         clock_gettime (CLOCK_REALTIME, &t->runningtime);
         t->running = 1;
         t->pid = flux_subprocess_pid (p);
         t->rank = flux_subprocess_rank (p);
-    }
-    else if (state == FLUX_SUBPROCESS_EXEC_FAILED) {
-        cron_task_exec_failed (t, flux_subprocess_fail_errno (p));
-        cron_task_handle_finished (p, t);
-        errno = t->exec_errno;
+        cron_task_timeout_start (t);
     }
     else if (state == FLUX_SUBPROCESS_FAILED) {
         cron_task_rexec_failed (t, flux_subprocess_fail_errno (p));
@@ -228,31 +218,33 @@ static void state_change_cb (flux_subprocess_t *p, flux_subprocess_state_t state
 static void io_cb (flux_subprocess_t *p, const char *stream)
 {
     cron_task_t *t = flux_subprocess_aux_get (p, "task");
-    const char *ptr = NULL;
-    int lenp;
+    const char *buf;
+    int len;
     bool is_stderr = false;
 
     assert (t);
 
-    if (!strcmp (stream, "stderr"))
+    if (streq (stream, "stderr"))
         is_stderr = true;
 
-    if (!(ptr = flux_subprocess_read_trimmed_line (p, stream, &lenp))) {
-        flux_log_error (t->h, "%s: flux_subprocess_read_trimmed_line",
+    if ((len = flux_subprocess_read_trimmed_line (p, stream, &buf)) < 0) {
+        flux_log_error (t->h,
+                        "%s: flux_subprocess_read_trimmed_line",
                         __FUNCTION__);
         return;
     }
 
-    if (!lenp) {
-        if (!(ptr = flux_subprocess_read (p, stream, -1, &lenp))) {
-            flux_log_error (t->h, "%s: flux_subprocess_read",
+    if (!len) {
+        if ((len = flux_subprocess_read (p, stream, &buf)) < 0) {
+            flux_log_error (t->h,
+                            "%s: flux_subprocess_read",
                             __FUNCTION__);
             return;
         }
     }
 
-    if (t->io_cb && lenp)
-        (*t->io_cb) (t->h, t, t->arg, is_stderr, ptr, lenp);
+    if (t->io_cb && len)
+        (*t->io_cb) (t->h, t, t->arg, is_stderr, buf, len);
 }
 
 int cron_task_kill (cron_task_t *t, int sig)
@@ -276,12 +268,11 @@ int cron_task_kill (cron_task_t *t, int sig)
 
 
 static flux_cmd_t *exec_cmd_create (struct cron_task *t,
-    const char *command,
-    const char *cwd,
-    json_t *env)
+                                    const char *command,
+                                    const char *cwd,
+                                    json_t *env)
 {
     flux_cmd_t *cmd = NULL;
-    char *tmp_cwd = NULL;
 
     if (!(cmd = flux_cmd_create (0, NULL, NULL))) {
         flux_log_error (t->h, "exec_cmd_create: flux_cmd_create");
@@ -293,7 +284,7 @@ static flux_cmd_t *exec_cmd_create (struct cron_task *t,
         flux_log_error (t->h, "exec_cmd_create: flux_cmd_argv_append");
         goto error;
     }
-    if (flux_cmd_setcwd (cmd, cwd) < 0) {
+    if (cwd && flux_cmd_setcwd (cmd, cwd) < 0) {
         flux_log_error (t->h, "exec_cmd_create: flux_cmd_setcwd");
         goto error;
     }
@@ -316,17 +307,17 @@ static flux_cmd_t *exec_cmd_create (struct cron_task *t,
         }
     }
 
-    free (tmp_cwd);
     return (cmd);
  error:
-    free (tmp_cwd);
     flux_cmd_destroy (cmd);
     return (NULL);
 }
 
 int cron_task_run (cron_task_t *t,
-    int rank, const char *command, const char *cwd,
-    json_t *env)
+                   int rank,
+                   const char *command,
+                   const char *cwd,
+                   json_t *env)
 {
     flux_t *h = t->h;
     flux_subprocess_t *p = NULL;
@@ -343,8 +334,8 @@ int cron_task_run (cron_task_t *t,
     if (!(cmd = exec_cmd_create (t, command, cwd, env)))
         goto done;
 
-    if (!(p = flux_rexec (h, rank, 0, cmd, &ops))) {
-        cron_task_exec_failed (t, errno);
+    if (!(p = flux_rexec_ex (h, "rexec", rank, 0, cmd, &ops, flux_llog, h))) {
+        cron_task_rexec_failed (t, errno);
         goto done;
     }
 
@@ -356,8 +347,6 @@ int cron_task_run (cron_task_t *t,
     t->started = 1;
     clock_gettime (CLOCK_REALTIME, &t->starttime);
     cron_task_state_update (t, "Started");
-    if (t->timeout >= 0.0)
-        cron_task_timeout_start (t);
 
     t->p = p;
     rc = 0;
@@ -400,8 +389,6 @@ static const char * cron_task_state_string (cron_task_t *t)
 {
     if (t->rexec_errno)
         return ("Rexec Failure");
-    if (t->exec_errno)
-        return ("Exec Failure");
     if (!t->started)
         return ("Deferred");
     if (!t->exited)
@@ -441,8 +428,6 @@ json_t *cron_task_to_json (struct cron_task *t)
 
     if (t->rexec_errno)
         json_object_set_new (o, "rexec_errno", json_integer (t->rexec_errno));
-    if (t->exec_errno)
-        json_object_set_new (o, "exec_errno", json_integer (t->exec_errno));
     if (t->timedout)
         json_object_set_new (o, "timedout", json_boolean (true));
     if (cron_task_finished (t)) {
diff --git a/src/modules/cron/task.h b/src/modules/cron/task.h
index e094b3097f68..f36ef28d1b05 100644
--- a/src/modules/cron/task.h
+++ b/src/modules/cron/task.h
@@ -20,8 +20,12 @@ typedef struct cron_task cron_task_t;
 
 /*  io callback fn for cron task
  */
-typedef void (*cron_task_io_f) (flux_t *h, cron_task_t *t, void *arg,
-                                bool is_stderr, const char *data, int datalen);
+typedef void (*cron_task_io_f) (flux_t *h,
+                                cron_task_t *t,
+                                void *arg,
+                                bool is_stderr,
+                                const char *data,
+                                int datalen);
 
 /*  task state change handler for cron task, check state with
  *   cron_task_state().
diff --git a/src/modules/cron/types.c b/src/modules/cron/types.c
index 63249f4e1025..b7c83eda8a35 100644
--- a/src/modules/cron/types.c
+++ b/src/modules/cron/types.c
@@ -8,8 +8,12 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <string.h>
 #include "entry.h"
+#include "ccan/str/str.h"
 
 /* List of external operations structures defined in
  *  type-specific source files:
@@ -33,7 +37,7 @@ int cron_type_operations_lookup (const char *name,
 {
     struct cron_typeinfo *type = cron_types;
     while (type && type->name) {
-        if (strcmp (name, type->name) == 0) {
+        if (streq (name, type->name)) {
             *ops = *type->ops;
             return (0);
         }
diff --git a/src/modules/cron/types.h b/src/modules/cron/types.h
index 7d8b3a520f12..52e2f1e9febb 100644
--- a/src/modules/cron/types.h
+++ b/src/modules/cron/types.h
@@ -16,6 +16,6 @@
  *  Lookup cron entry operations for cron entry type named "name"
  */
 int cron_type_operations_lookup (const char *name,
-    struct cron_entry_ops *ops);
+                                 struct cron_entry_ops *ops);
 
 #endif /* !HAVE_CRON_TYPES_H */
diff --git a/src/modules/heartbeat/heartbeat.c b/src/modules/heartbeat/heartbeat.c
new file mode 100644
index 000000000000..cd9b964b0a28
--- /dev/null
+++ b/src/modules/heartbeat/heartbeat.c
@@ -0,0 +1,173 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* heartbeat.c - publish regular heartbeat messages
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h>
+
+#include <src/common/libutil/fsd.h>
+#include "ccan/str/str.h"
+
+static const double default_period = 2.0;
+
+struct heartbeat {
+    flux_t *h;
+    double period;
+    flux_watcher_t *timer;
+    flux_msg_handler_t **handlers;
+    flux_future_t *f;
+};
+
+static void heartbeat_get_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct heartbeat *hb = arg;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:f}",
+                           "period",
+                           hb->period) < 0)
+        flux_log_error (h, "error responding to heartbeat.get request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to heartbeat.get request");
+}
+
+static void publish_continuation (flux_future_t *f, void *arg)
+{
+    struct heartbeat *hb = arg;
+
+    if (flux_future_get (f, NULL) < 0)
+        flux_log_error (hb->h, "error publishing heartbeat");
+
+    flux_future_destroy (f);
+    hb->f = NULL;
+}
+
+static void timer_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct heartbeat *hb = arg;
+
+    flux_future_destroy (hb->f);
+
+    if (!(hb->f = flux_event_publish (hb->h, "heartbeat.pulse", 0, NULL))) {
+        flux_log_error (hb->h, "error sending publish request");
+        return;
+    }
+    if (flux_future_then (hb->f, -1, publish_continuation, hb) < 0) {
+        flux_log_error (hb->h, "error setting up continuation");
+        flux_future_destroy (hb->f);
+        hb->f = NULL;
+    }
+}
+
+static int parse_args (int argc, char **argv, struct heartbeat *hb)
+{
+    int i;
+
+    for (i = 0; i < argc; i++) {
+        if (strstarts (argv[i], "period=")) {
+            if (fsd_parse_duration (argv[i] + 7, &hb->period) < 0) {
+                flux_log_error (hb->h, "error parsing period value");
+                return -1;
+            }
+        }
+        else {
+            flux_log (hb->h, LOG_ERR, "unknown option: %s", argv[i]);
+            goto inval;
+        }
+    }
+    return 0;
+inval:
+    errno = EINVAL;
+    return -1;
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {   FLUX_MSGTYPE_REQUEST,
+        "heartbeat.get",
+        heartbeat_get_cb,
+        FLUX_ROLE_USER
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+static void heartbeat_destroy (struct heartbeat *hb)
+{
+    if (hb) {
+        int saved_errno = errno;
+        flux_future_destroy (hb->f);
+        flux_msg_handler_delvec (hb->handlers);
+        flux_watcher_destroy (hb->timer);
+        free (hb);
+        errno = saved_errno;
+    }
+}
+
+static struct heartbeat *heartbeat_create (flux_t *h)
+{
+    struct heartbeat *hb;
+
+    if (!(hb = calloc (1, sizeof (*hb))))
+        return NULL;
+    hb->h = h;
+    hb->period = default_period;
+    if (flux_msg_handler_addvec (hb->h, htab, hb, &hb->handlers) < 0)
+        goto error;
+    return hb;
+error:
+    heartbeat_destroy (hb);
+    return NULL;
+}
+
+int mod_main (flux_t *h, int argc, char **argv)
+{
+    struct heartbeat *hb;
+    flux_reactor_t *r = flux_get_reactor (h);
+
+    if (!(hb = heartbeat_create (h)))
+        return -1;
+    if (parse_args (argc, argv, hb) < 0)
+        goto error;
+    if (!(hb->timer = flux_timer_watcher_create (r,
+                                                 0.,
+                                                 hb->period,
+                                                 timer_cb,
+                                                 hb)))
+        goto error;
+    flux_watcher_start (hb->timer);
+    if (flux_reactor_run (r, 0) < 0) {
+        flux_log_error (h, "flux_reactor_run");
+        goto error;
+    }
+    heartbeat_destroy (hb);
+    return 0;
+error:
+    heartbeat_destroy (hb);
+    return -1;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-exec/Makefile.am b/src/modules/job-exec/Makefile.am
index 45b2fffb4d57..ca1eb4dbd92a 100644
--- a/src/modules/job-exec/Makefile.am
+++ b/src/modules/job-exec/Makefile.am
@@ -6,61 +6,37 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) $(JANSSON_CFLAGS)
-
-fluxmod_LTLIBRARIES = \
-	job-exec.la
+	$(JANSSON_CFLAGS)
 
 noinst_LTLIBRARIES = \
-	libbulk-exec.la
-
-noinst_PROGRAMS = \
-	bulk-exec
+	libjob-exec.la
 
-libbulk_exec_la_SOURCES = \
-	bulk-exec.h \
-	bulk-exec.c
-
-job_exec_la_SOURCES = \
+libjob_exec_la_SOURCES = \
 	job-exec.h \
 	job-exec.c \
+	checkpoint.h \
+	checkpoint.c \
+	exec_config.h \
+	exec_config.c \
 	rset.c \
 	rset.h \
 	testexec.c \
 	exec.c
 
-job_exec_la_LDFLAGS = \
-	$(fluxmod_ldflags) \
-	-module
-
-job_exec_la_LIBADD = \
-	$(fluxmod_libadd) \
-	$(top_builddir)/src/common/libjob/libjob.la \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	libbulk-exec.la \
-	$(ZMQ_LIBS)
-
-bulk_exec_SOURCES = \
-	test/bulk-exec.c
-
-bulk_exec_LDADD = \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(top_builddir)/src/common/libflux-idset.la \
-	$(top_builddir)/src/common/libflux-optparse.la \
-	libbulk-exec.la \
-	$(top_builddir)/src/common/libutil/libutil.la \
-	$(ZMQ_LIBS)
-
 test_ldadd = \
+	$(builddir)/libjob-exec.la \
 	$(top_builddir)/src/common/libtap/libtap.la \
-	$(top_builddir)/src/common/libflux-internal.la \
 	$(top_builddir)/src/common/libflux-core.la \
-	$(ZMQ_LIBS) $(LIBPTHREAD) $(JANSSON_LIBS)
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(LIBPTHREAD) $(JANSSON_LIBS)
+
+test_ldflags = \
+	-no-install
 
 test_cppflags = \
 	$(AM_CPPFLAGS)
@@ -72,10 +48,10 @@ check_PROGRAMS = \
 	$(TESTS)
 
 test_rset_t_SOURCES = \
-	rset.c \
-	rset.h \
 	test/rset.c
 test_rset_t_CPPFLAGS = \
 	$(test_cppflags)
 test_rset_t_LDADD = \
 	$(test_ldadd)
+test_rset_t_LDFLAGS = \
+	$(test_ldflags)
diff --git a/src/modules/job-exec/bulk-exec.c b/src/modules/job-exec/bulk-exec.c
deleted file mode 100644
index a1b6c3dc17ed..000000000000
--- a/src/modules/job-exec/bulk-exec.c
+++ /dev/null
@@ -1,494 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-# include "config.h"
-#endif
-
-#include <sys/wait.h>
-#define EXIT_CODE(x) __W_EXITCODE(x,0)
-
-#include <flux/core.h>
-#include <flux/idset.h>
-#include <czmq.h>
-
-#include "src/common/libutil/aux.h"
-#include "bulk-exec.h"
-
-struct exec_cmd {
-    struct idset *ranks;
-    flux_cmd_t *cmd;
-    int flags;
-};
-
-struct bulk_exec {
-    flux_t *h;
-
-    struct aux_item *aux;
-
-    int max_start_per_loop;  /* Max subprocess started per event loop cb */
-    int total;               /* Total processes expected to run */
-    int started;             /* Number of processes that have reached start */
-    int complete;            /* Number of processes that have completed */
-
-    int exit_status;         /* Largest wait status of all complete procs */
-
-    unsigned int active:1;
-
-    flux_watcher_t *prep;
-    flux_watcher_t *check;
-    flux_watcher_t *idle;
-
-    struct idset *exit_batch;         /* Support for batched exit notify */
-    flux_watcher_t *exit_batch_timer; /* Timer for batched exit notify */
-
-    flux_subprocess_ops_t ops;
-
-    zlist_t *commands;
-    zlist_t *processes;
-
-    struct bulk_exec_ops *handlers;
-    void *arg;
-};
-
-int bulk_exec_rc (struct bulk_exec *exec)
-{
-    return (exec->exit_status);
-}
-
-int bulk_exec_current (struct bulk_exec *exec)
-{
-    return zlist_size (exec->processes);
-}
-
-int bulk_exec_total (struct bulk_exec *exec)
-{
-    return exec->total;
-}
-
-static int exec_exit_notify (struct bulk_exec *exec)
-{
-    if (exec->handlers->on_exit)
-        (*exec->handlers->on_exit) (exec, exec->arg, exec->exit_batch);
-    if (exec->exit_batch_timer) {
-        flux_watcher_destroy (exec->exit_batch_timer);
-        exec->exit_batch_timer = NULL;
-        idset_range_clear (exec->exit_batch, 0, INT_MAX);
-    }
-    return 0;
-}
-
-static void exit_batch_cb (flux_reactor_t *r, flux_watcher_t *w,
-                          int revents, void *arg)
-{
-    struct bulk_exec *exec = arg;
-    exec_exit_notify (exec);
-}
-
-/*  Append completed subprocess 'p' to the current batch for exit
- *   notification. If this is the first exited process in the batch,
- *   then start a timer which will fire and call the function to
- *   notify bulk_exec user of the batch of subprocess exits.
- *
- *  This appraoch avoids unecessarily calling into user's callback
- *   multiple times when all tasks exit within 0.01s.
- */
-static void exit_batch_append (struct bulk_exec *exec, flux_subprocess_t *p)
-{
-    int rank = flux_subprocess_rank (p);
-    if (idset_set (exec->exit_batch, rank) < 0) {
-        flux_log_error (exec->h, "exit_batch_append:idset_set");
-        return;
-    }
-    if (!exec->exit_batch_timer) {
-        flux_reactor_t *r = flux_get_reactor (exec->h);
-        /*  XXX: batch timer should eventually be configurable by caller */
-        exec->exit_batch_timer =
-            flux_timer_watcher_create (r, 0.01, 0.,
-                                       exit_batch_cb,
-                                       exec);
-        if (!exec->exit_batch_timer) {
-            flux_log_error (exec->h, "exit_batch_append:timer create");
-            return;
-        }
-        flux_watcher_start (exec->exit_batch_timer);
-    }
-}
-
-static void exec_add_completed (struct bulk_exec *exec, flux_subprocess_t *p)
-{
-    /* Append this process to the current batch for notification */
-    exit_batch_append (exec, p);
-
-    if (++exec->complete == exec->total) {
-        exec_exit_notify (exec);
-        if (exec->handlers->on_complete)
-            (*exec->handlers->on_complete) (exec, exec->arg);
-    }
-}
-
-static void exec_complete_cb (flux_subprocess_t *p)
-{
-    int status = flux_subprocess_status (p);
-    struct bulk_exec *exec = flux_subprocess_aux_get (p, "job-exec::exec");
-
-    if (status > exec->exit_status)
-        exec->exit_status = status;
-
-    exec_add_completed (exec, p);
-}
-
-static void exec_state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
-{
-    struct bulk_exec *exec = flux_subprocess_aux_get (p, "job-exec::exec");
-    if (state == FLUX_SUBPROCESS_RUNNING) {
-        if (++exec->started == exec->total) {
-            if (exec->handlers->on_start)
-                (*exec->handlers->on_start) (exec, exec->arg);
-        }
-    }
-    else if (state == FLUX_SUBPROCESS_FAILED
-            || state == FLUX_SUBPROCESS_EXEC_FAILED) {
-        int errnum = flux_subprocess_fail_errno (p);
-        int code = EXIT_CODE(1);
-
-        if (errnum == EPERM || errnum == EACCES)
-            code = EXIT_CODE(126);
-        else if (errnum == ENOENT)
-            code = EXIT_CODE(127);
-        else if (errnum == EHOSTUNREACH)
-            code = EXIT_CODE(68);
-
-        if (code > exec->exit_status)
-            exec->exit_status = code;
-
-        if (exec->handlers->on_error)
-            (*exec->handlers->on_error) (exec, p, exec->arg);
-
-        exec_add_completed (exec, p);
-    }
-}
-
-static void exec_output_cb (flux_subprocess_t *p, const char *stream)
-{
-    struct bulk_exec *exec = flux_subprocess_aux_get (p, "job-exec::exec");
-    const char *s;
-    int len;
-
-    if (!(s = flux_subprocess_getline (p, stream, &len))) {
-        flux_log_error (exec->h, "flux_subprocess_getline");
-        return;
-    }
-    if (len) {
-        int rank = flux_subprocess_rank (p);
-        if (exec->handlers->on_output)
-            (*exec->handlers->on_output) (exec, p, stream, s, len, exec->arg);
-        else
-            flux_log (exec->h, LOG_INFO, "rank %d: %s: %s", rank, stream, s);
-    }
-}
-
-static void exec_cmd_destroy (void *arg)
-{
-    struct exec_cmd *cmd = arg;
-    idset_destroy (cmd->ranks);
-    flux_cmd_destroy (cmd->cmd);
-    free (cmd);
-}
-
-static struct exec_cmd *exec_cmd_create (const struct idset *ranks,
-                                         flux_cmd_t *cmd,
-                                         int flags)
-{
-    struct exec_cmd *c = calloc (1, sizeof (*c));
-    if (!c)
-        return NULL;
-    if (!(c->ranks = idset_copy (ranks)))
-        goto err;
-    if (!(c->cmd = flux_cmd_copy (cmd)))
-        goto err;
-    c->flags = flags;
-    return (c);
-err:
-    exec_cmd_destroy (c);
-    return NULL;
-}
-
-static void subprocess_destroy_finish (flux_future_t *f, void *arg)
-{
-    flux_subprocess_t *p = arg;
-    if (flux_future_get (f, NULL) < 0) {
-        flux_t *h = flux_subprocess_aux_get (p, "flux_t");
-        flux_log_error (h, "subprocess_kill: %ju: %s",
-                        (uintmax_t) flux_subprocess_pid,
-                        flux_strerror (errno));
-    }
-    flux_subprocess_destroy (p);
-    flux_future_destroy (f);
-}
-
-static int subprocess_destroy (flux_t *h, flux_subprocess_t *p)
-{
-    flux_future_t *f = flux_subprocess_kill (p, SIGKILL);
-    if (!f || flux_future_then (f, -1., subprocess_destroy_finish, p) < 0)
-        return -1;
-    return 0;
-}
-
-static int exec_start_cmd (struct bulk_exec *exec,
-                           struct exec_cmd *cmd,
-                           int max)
-{
-    int count = 0;
-    uint32_t rank;
-    rank = idset_first (cmd->ranks);
-    while (rank != IDSET_INVALID_ID && (max < 0 || count < max)) {
-        flux_subprocess_t *p = flux_rexec (exec->h,
-                                           rank,
-                                           cmd->flags,
-                                           cmd->cmd,
-                                           &exec->ops);
-        if (!p)
-            return -1;
-        if (flux_subprocess_aux_set (p, "job-exec::exec", exec, NULL) < 0
-           || zlist_append (exec->processes, p) < 0) {
-            if (subprocess_destroy (exec->h, p) < 0)
-                flux_log_error (exec->h, "Unable to destroy pid %ju",
-                        (uintmax_t) flux_subprocess_pid (p));
-            return -1;
-        }
-        zlist_freefn (exec->processes, p,
-                     (zlist_free_fn *) flux_subprocess_unref,
-                     true);
-
-        idset_clear (cmd->ranks, rank);
-        rank = idset_next (cmd->ranks, rank);
-        count++;
-    }
-    return count;
-}
-
-void bulk_exec_stop (struct bulk_exec *exec)
-{
-    flux_watcher_stop (exec->prep);
-    flux_watcher_stop (exec->check);
-}
-
-static int exec_start_cmds (struct bulk_exec *exec, int max)
-{
-    while (zlist_size (exec->commands) && (max != 0)) {
-        struct exec_cmd *cmd = zlist_first (exec->commands);
-        int rc = exec_start_cmd (exec, cmd, max);
-        if (rc < 0) {
-            flux_log_error (exec->h, "exec_start_cmd failed");
-            return -1;
-        }
-        if (idset_count (cmd->ranks) == 0)
-            zlist_remove (exec->commands, cmd);
-        if (max > 0)
-            max -= rc;
-
-    }
-    return 0;
-}
-
-static void prep_cb (flux_reactor_t *r, flux_watcher_t *w,
-                     int revents, void *arg)
-{
-    struct bulk_exec *exec = arg;
-
-    /* Don't block in reactor if there are commands to run */
-    if (zlist_size (exec->commands) > 0) {
-        flux_watcher_start (exec->idle);
-        flux_watcher_start (exec->check);
-    }
-    else
-        bulk_exec_stop (exec);
-}
-
-static void check_cb (flux_reactor_t *r, flux_watcher_t *w,
-                     int revents, void *arg)
-{
-    struct bulk_exec *exec = arg;
-    flux_watcher_stop (exec->idle);
-    flux_watcher_stop (exec->check);
-    if (exec_start_cmds (exec, exec->max_start_per_loop) < 0) {
-        bulk_exec_stop (exec);
-        if (exec->handlers->on_error)
-            (*exec->handlers->on_error) (exec, NULL, exec->arg);
-    }
-}
-
-void bulk_exec_destroy (struct bulk_exec *exec)
-{
-    zlist_destroy (&exec->processes);
-    zlist_destroy (&exec->commands);
-    idset_destroy (exec->exit_batch);
-    flux_watcher_destroy (exec->prep);
-    flux_watcher_destroy (exec->check);
-    flux_watcher_destroy (exec->idle);
-    aux_destroy (&exec->aux);
-    free (exec);
-}
-
-struct bulk_exec * bulk_exec_create (struct bulk_exec_ops *ops, void *arg)
-{
-    flux_subprocess_ops_t sp_ops = {
-        .on_completion =   exec_complete_cb,
-        .on_state_change = exec_state_cb,
-        .on_stdout =       exec_output_cb,
-        .on_stderr =       exec_output_cb,
-    };
-    struct bulk_exec *exec = calloc (1, sizeof (*exec));
-    if (!exec)
-        return NULL;
-    exec->ops = sp_ops;
-    exec->handlers = ops;
-    exec->arg = arg;
-    exec->processes = zlist_new ();
-    exec->commands = zlist_new ();
-    exec->exit_batch = idset_create (0, IDSET_FLAG_AUTOGROW);
-    exec->max_start_per_loop = 1;
-
-    return exec;
-}
-
-int bulk_exec_set_max_per_loop (struct bulk_exec *exec, int max)
-{
-    if (max == 0) {
-        errno = EINVAL;
-        return -1;
-    }
-    exec->max_start_per_loop = max;
-    return 0;
-}
-
-int bulk_exec_push_cmd (struct bulk_exec *exec,
-                       const struct idset *ranks,
-                       flux_cmd_t *cmd,
-                       int flags)
-{
-    struct exec_cmd *c = exec_cmd_create (ranks, cmd, flags);
-    if (!c)
-        return -1;
-
-    if (zlist_append (exec->commands, c) < 0) {
-        exec_cmd_destroy (c);
-        return -1;
-    }
-    zlist_freefn (exec->commands, c, exec_cmd_destroy, true);
-
-    exec->total += idset_count (ranks);
-    if (exec->active) {
-        flux_watcher_start (exec->prep);
-        flux_watcher_start (exec->check);
-    }
-
-    return 0;
-}
-
-int bulk_exec_start (flux_t *h, struct bulk_exec *exec)
-{
-    flux_reactor_t *r = flux_get_reactor (h);
-    exec->h = h;
-    exec->prep = flux_prepare_watcher_create (r, prep_cb, exec);
-    exec->check = flux_check_watcher_create (r, check_cb, exec);
-    exec->idle = flux_idle_watcher_create (r, NULL, NULL);
-    if (!exec->prep || !exec->check || !exec->idle)
-        return -1;
-    flux_watcher_start (exec->prep);
-    exec->active = 1;
-    return 0;
-}
-
-/*  Cancel all pending commands.
- */
-int bulk_exec_cancel (struct bulk_exec *exec)
-{
-    struct exec_cmd *cmd = zlist_first (exec->commands);
-    if (!cmd)
-        return 0;
-
-    while (cmd) {
-        uint32_t rank = idset_first (cmd->ranks);
-        while (rank != IDSET_INVALID_ID) {
-            exec->complete++;
-            if (idset_set (exec->exit_batch, rank) < 0)
-                flux_log_error (exec->h, "bulk_exec_cance: idset_set");
-            rank = idset_next (cmd->ranks, rank);
-        }
-        cmd = zlist_next (exec->commands);
-    }
-    zlist_purge (exec->commands);
-    exec_exit_notify (exec);
-
-    if (exec->complete == exec->total) {
-        if (exec->handlers->on_complete)
-            (*exec->handlers->on_complete) (exec, exec->arg);
-    }
-    return 0;
-}
-
-flux_future_t *bulk_exec_kill (struct bulk_exec *exec, int signum)
-{
-    flux_subprocess_t *p = zlist_first (exec->processes);
-    flux_future_t *cf = NULL;
-
-    if (!(cf = flux_future_wait_all_create ()))
-        return NULL;
-    flux_future_set_flux (cf, exec->h);
-
-    while (p) {
-        if (flux_subprocess_state (p) == FLUX_SUBPROCESS_RUNNING
-            || flux_subprocess_state (p) == FLUX_SUBPROCESS_INIT) {
-            flux_future_t *f = NULL;
-            char s[64];
-            if (!(f = flux_subprocess_kill (p, signum))) {
-                int err = errno;
-                const char *errstr = flux_strerror (errno);
-                if ((f = flux_future_create (NULL, NULL)))
-                    flux_future_fulfill_error (f, err, errstr);
-                else
-                    flux_future_fulfill_error (cf, err, "Internal error");
-            }
-            (void) snprintf (s, sizeof (s)-1, "%u",
-                            flux_subprocess_rank (p));
-            if (flux_future_push (cf, s, f) < 0) {
-                fprintf (stderr, "flux_future_push: %s\n", strerror (errno));
-                flux_future_destroy (f);
-            }
-        }
-        p = zlist_next (exec->processes);
-    }
-
-    /*  If no child futures were pushed into the wait_all future `cf`,
-     *   then no signals were sent and we should immediately return ENOENT.
-     */
-    if (!flux_future_first_child (cf)) {
-        flux_future_destroy (cf);
-        errno = ENOENT;
-        return NULL;
-    }
-
-    return cf;
-}
-
-int bulk_exec_aux_set (struct bulk_exec *exec, const char *key,
-                       void *val, flux_free_f free_fn)
-{
-    return (aux_set (&exec->aux, key, val, free_fn));
-}
-
-void * bulk_exec_aux_get (struct bulk_exec *exec, const char *key)
-{
-    return (aux_get (exec->aux, key));
-}
-/* vi: ts=4 sw=4 expandtab
- */
diff --git a/src/modules/job-exec/bulk-exec.h b/src/modules/job-exec/bulk-exec.h
deleted file mode 100644
index 522a6b37137f..000000000000
--- a/src/modules/job-exec/bulk-exec.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* "bulk" subprocess execution wrapper around libsubprocess API */
-
-#ifndef HAVE_JOB_EXEC_BULK_EXEC_H
-#define HAVE_JOB_EXEC_BULK_EXEC_H 1
-
-#include <flux/core.h>
-
-struct bulk_exec;
-
-typedef void (*exec_cb_f)   (struct bulk_exec *, void *arg);
-
-typedef void (*exec_exit_f) (struct bulk_exec *, void *arg,
-                             const struct idset *ranks);
-
-typedef void (*exec_io_f)   (struct bulk_exec *,
-                             flux_subprocess_t *,
-                             const char *stream,
-			     const char *data,
-			     int data_len,
-                             void *arg);
-
-typedef void (*exec_error_f) (struct bulk_exec *,
-                              flux_subprocess_t *,
-                              void *arg);
-
-struct bulk_exec_ops {
-    exec_cb_f    on_start;    /* called when all processes are running  */
-    exec_exit_f  on_exit;     /* called when a set of tasks exits       */
-    exec_cb_f    on_complete; /* called when all processes are done     */
-    exec_io_f    on_output;   /* called on process output               */
-    exec_error_f on_error;    /* called on any fatal error              */
-};
-
-struct bulk_exec * bulk_exec_create (struct bulk_exec_ops *ops, void *arg);
-
-void *bulk_exec_aux_get (struct bulk_exec *exec, const char *key);
-
-int bulk_exec_aux_set (struct bulk_exec *exec,
-                       const char *key,
-                       void *val,
-                       flux_free_f free_fn);
-
-/*  Set maximum number of flux_subprocess_rexex(3) calls per event
- *   loop iteration. (-1 for no max)
- */
-int bulk_exec_set_max_per_loop (struct bulk_exec *exec, int max);
-
-void bulk_exec_destroy (struct bulk_exec *exec);
-
-int bulk_exec_push_cmd (struct bulk_exec *exec,
-                       const struct idset *ranks,
-                       flux_cmd_t *cmd,
-                       int flags);
-
-int bulk_exec_start (flux_t *h, struct bulk_exec *exec);
-
-flux_future_t * bulk_exec_kill (struct bulk_exec *exec, int signal);
-
-int bulk_exec_cancel (struct bulk_exec *exec);
-
-/* Returns max wait status returned from all exited processes */
-int bulk_exec_rc (struct bulk_exec *exec);
-
-/* Returns current number of processes starting/running */
-int bulk_exec_current (struct bulk_exec *exec);
-
-/* Returns total number of processes expected to run */
-int bulk_exec_total (struct bulk_exec *exec);
-
-#endif /* !HAVE_JOB_EXEC_BULK_EXEC_H */
diff --git a/src/modules/job-exec/checkpoint.c b/src/modules/job-exec/checkpoint.c
new file mode 100644
index 000000000000..055cb64396e8
--- /dev/null
+++ b/src/modules/job-exec/checkpoint.c
@@ -0,0 +1,253 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Prototype checkpoint of running jobs KVS root refs
+ *
+ * DESCRIPTION
+ *
+ * Handle checkpoint of running job's guest KVS namescapes into the
+ * primary KVS.  This will allow the namespaces to be recreated if
+ * a job manager is brought down than back up.
+ *
+ * Also provide helper functions to get rootrefs from the checkpointed
+ * object.
+ *
+ * OPERATION
+ *
+ * Get the KVS rootrefs for all running jobs and commit to
+ * "job-exec.kvs-namespaces".
+ *
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <assert.h>
+#include <unistd.h>
+#include <errno.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "job-exec.h"
+#include "checkpoint.h"
+
+flux_future_t *checkpoint_get_rootrefs (flux_t *h)
+{
+    if (!h) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    return flux_kvs_lookup (h,
+                            NULL,
+                            0,
+                            "job-exec.kvs-namespaces");
+}
+
+char *checkpoint_find_rootref (flux_future_t *f,
+                               flux_jobid_t id,
+                               uint32_t owner)
+{
+    int saved_errno;
+    flux_t *h = flux_future_get_flux (f);
+    char *rv = NULL;
+    const char *rootrefs;
+    json_error_t error;
+    json_t *o = NULL;
+    size_t index;
+    json_t *value;
+
+    if (flux_kvs_lookup_get (f, &rootrefs) < 0) {
+        flux_log_error (h, "checkpoint_get_rootref: flux_kvs_lookup_get");
+        goto cleanup;
+    }
+
+    if (!(o = json_loads (rootrefs, 0, &error))) {
+        flux_log (h, LOG_ERR, "json_loads rootrefs: %s", error.text);
+        goto cleanup;
+    }
+
+    json_array_foreach (o, index, value) {
+        flux_jobid_t l_id;
+        uint32_t l_owner;
+        const char *rootref;
+
+        if (json_unpack (value,
+                         "{s:I s:i s:s}",
+                         "id", &l_id,
+                         "owner", &l_owner,
+                         "kvsroot", &rootref) < 0) {
+            flux_log (h, LOG_ERR, "json_unpack rootref: %s", error.text);
+            goto cleanup;
+        }
+        if (l_id == id && l_owner == owner) {
+            if (!(rv = strdup (rootref)))
+                goto cleanup;
+            break;
+        }
+    }
+
+cleanup:
+    saved_errno = errno;
+    json_decref (o);
+    errno = saved_errno;
+    return rv;
+}
+
+static int lookup_nsroots (flux_t *h, zhashx_t *jobs, flux_future_t **fp)
+{
+    struct jobinfo *job = zhashx_first (jobs);
+    flux_future_t *fall = NULL;
+    flux_future_t *f = NULL;
+
+    while (job) {
+        if (job->running) {
+            if (!fall) {
+                if (!(fall = flux_future_wait_all_create ()))
+                    goto cleanup;
+                flux_future_set_flux (fall, h);
+            }
+            if (!(f = flux_kvs_getroot (h, job->ns, 0)))
+                goto cleanup;
+            if (flux_future_aux_set (f, "jobinfo", job, NULL) < 0)
+                goto cleanup;
+            if (flux_future_push (fall, job->ns, f) < 0)
+                goto cleanup;
+            f = NULL;
+        }
+        job = zhashx_next (jobs);
+    }
+
+    (*fp) = fall;
+    return 0;
+
+cleanup:
+    flux_future_destroy (f);
+    flux_future_destroy (fall);
+    return -1;
+}
+
+static json_t *get_nsroots (flux_t *h, flux_future_t *fall)
+{
+    const char *child;
+    json_t *nsdata = NULL;
+    int saved_errno;
+
+    if (!(nsdata = json_array ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    child = flux_future_first_child (fall);
+    while (child) {
+        flux_future_t *f = flux_future_get_child (fall, child);
+        struct jobinfo *job;
+        const char *blobref = NULL;
+        json_t *o = NULL;
+        if (!f)
+            goto cleanup;
+        if (!(job = flux_future_aux_get (f, "jobinfo")))
+            goto cleanup;
+        if (flux_kvs_getroot_get_blobref (f, &blobref) < 0)
+            goto cleanup;
+        if (!(o = json_pack ("{s:I s:i s:s}",
+                             "id", job->id,
+                             "owner", job->userid,
+                             "kvsroot", blobref))) {
+            errno = ENOMEM;
+            goto cleanup;
+        }
+        if (json_array_append (nsdata, o) < 0) {
+            json_decref (o);
+            errno = ENOMEM;
+            goto cleanup;
+        }
+        json_decref (o);
+        child = flux_future_next_child (fall);
+    }
+
+    return nsdata;
+cleanup:
+    saved_errno = errno;
+    json_decref (nsdata);
+    errno = saved_errno;
+    return NULL;
+}
+
+static int checkpoint_commit (flux_t *h, json_t *nsdata)
+{
+    flux_future_t *f = NULL;
+    flux_kvs_txn_t *txn = NULL;
+    char *s = NULL;
+    int rv = -1;
+
+    if (!(s = json_dumps (nsdata, JSON_COMPACT))) {
+        errno = ENOMEM;
+        goto cleanup;
+    }
+
+    if (!(txn = flux_kvs_txn_create ()))
+        goto cleanup;
+
+    if (flux_kvs_txn_put (txn,
+                          0,
+                          "job-exec.kvs-namespaces",
+                          s) < 0)
+        goto cleanup;
+
+    if (!(f = flux_kvs_commit (h, NULL, 0, txn)))
+        goto cleanup;
+
+    if (flux_future_get (f, NULL) < 0)
+        goto cleanup;
+
+    rv = 0;
+cleanup:
+    flux_future_destroy (f);
+    flux_kvs_txn_destroy (txn);
+    free (s);
+    return rv;
+}
+
+void checkpoint_running (flux_t *h, zhashx_t *jobs)
+{
+    flux_future_t *lookupf = NULL;
+    json_t *nsdata = NULL;
+
+    if (!h || !jobs)
+        return;
+
+    if (lookup_nsroots (h, jobs, &lookupf) < 0) {
+        flux_log_error (h, "failed to lookup ns root refs");
+        goto cleanup;
+    }
+
+    if (!lookupf)
+        return;
+
+    if (!(nsdata = get_nsroots (h, lookupf))) {
+        flux_log_error (h, "failure getting ns root refs");
+        goto cleanup;
+    }
+
+    if (checkpoint_commit (h, nsdata) < 0) {
+        flux_log_error (h, "failure committing ns checkpoint data");
+        goto cleanup;
+    }
+
+cleanup:
+    json_decref (nsdata);
+    flux_future_destroy (lookupf);
+}
+
+/*
+ * vi: tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-exec/checkpoint.h b/src/modules/job-exec/checkpoint.h
new file mode 100644
index 000000000000..c4d036f5faac
--- /dev/null
+++ b/src/modules/job-exec/checkpoint.h
@@ -0,0 +1,30 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_JOB_EXEC_CHECKPOINT_H
+#define HAVE_JOB_EXEC_CHECKPOINT_H 1
+
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "job-exec.h"
+
+flux_future_t *checkpoint_get_rootrefs (flux_t *h);
+
+char *checkpoint_find_rootref (flux_future_t *f,
+                               flux_jobid_t id,
+                               uint32_t owner);
+
+void checkpoint_running (flux_t *h, zhashx_t *jobs);
+
+#endif /* !HAVE_JOB_EXEC_CHECKPOINT_EXEC_H */
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/job-exec/exec.c b/src/modules/job-exec/exec.c
index 01cbd331e00b..d1a058754c8a 100644
--- a/src/modules/job-exec/exec.c
+++ b/src/modules/job-exec/exec.c
@@ -20,8 +20,12 @@
  * attributes.system.exec.bulkexec object. Supported keys include
  *
  * {
- *    "mock_exception":s       - Generate a mock execption in phase:
+ *    "mock_exception":s       - Generate a mock exception in phase:
  *                               "init", or "starting"
+ *    "service":s              - Specify service to use for launching remote
+ *                               subprocesses: "rexec" or "sdexec".
+ *    "barrier-timeout":F      - Specify timeout for start barrier in floating
+ *                               point seconds.
  * }
  *
  */
@@ -31,84 +35,155 @@
 #endif
 
 #include <unistd.h>
+#include <string.h>
+
+#include "src/common/libjob/idf58.h"
+#include "ccan/str/str.h"
+#include "src/common/libutil/basename.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libsubprocess/bulk-exec.h"
 
 #include "job-exec.h"
-#include "bulk-exec.h"
+#include "exec_config.h"
 #include "rset.h"
 
+/*  Numeric severity used for a non-fatal, critical job exception:
+ *  (e.g. node failure)
+ */
+#define FLUX_JOB_EXCEPTION_CRIT 2
+
 extern char **environ;
-static const char *default_cwd = "/tmp";
-static const char *default_job_shell = NULL;
 
-/* Configuration for "bulk" execution implementation. Used only for testing
- *  for now.
- */
-struct exec_conf {
-    const char *        mock_exception;   /* fake exception */
+struct exec_ctx {
+    struct jobinfo *job;
+
+    const char * mock_exception;   /* fake exception */
+    struct idset *barrier_pending_ranks;
+    int barrier_enter_count;
+    int barrier_completion_count;
+    int exit_count;
+
+    flux_watcher_t *shell_barrier_timer;
 };
 
-static void exec_conf_destroy (struct exec_conf *tc)
+static void barrier_timer_cb (flux_reactor_t *r,
+                              flux_watcher_t *w,
+                              int revents,
+                              void *arg)
 {
-    free (tc);
-}
+    struct exec_ctx *ctx = arg;
+    struct jobinfo *job = ctx->job;
+    struct bulk_exec *exec = ctx->job->data;
+    char *ranks;
+
+    if (!(ranks = idset_encode (ctx->barrier_pending_ranks,
+                                IDSET_FLAG_RANGE))) {
+        flux_log_error (job->h,
+                        "failed to encode barrier pending ranks for job %s",
+                        idf58 (job->id));
+        return;
+    }
 
-static struct exec_conf *exec_conf_create (json_t *jobspec)
-{
-    struct exec_conf *conf = calloc (1, sizeof (*conf));
-    if (conf == NULL)
-        return NULL;
-    (void) json_unpack (jobspec, "{s:{s:{s:{s:{s:s}}}}}",
-                                 "attributes", "system", "exec",
-                                     "bulkexec",
-                                         "mock_exception",
-                                         &conf->mock_exception);
-    return conf;
+    (void) jobinfo_drain_ranks (job,
+                                ranks,
+                                "job %s start timeout: %s",
+                                idf58 (job->id),
+                                "possible node hang");
+
+    jobinfo_fatal_error (job,
+                         0,
+                         "%s waiting for %zu/%d nodes (rank%s %s)",
+                         "start barrier timeout",
+                         idset_count (ctx->barrier_pending_ranks),
+                         bulk_exec_total (exec),
+                         idset_count (ctx->barrier_pending_ranks) > 1 ?"s":"",
+                         ranks);
+    free (ranks);
 }
 
-static const char * exec_mock_exception (struct bulk_exec *exec)
-{
-    struct exec_conf *conf = bulk_exec_aux_get (exec, "conf");
-    if (!conf || !conf->mock_exception)
-        return "none";
-    return conf->mock_exception;
-}
 
-static const char *jobspec_get_job_shell (json_t *jobspec)
+static void exec_ctx_destroy (struct exec_ctx *tc)
 {
-    const char *path = NULL;
-    (void) json_unpack (jobspec, "{s:{s:{s:{s:s}}}}",
-                                 "attributes", "system", "exec",
-                                 "job_shell", &path);
-    return path;
+    if (tc) {
+        int saved_errno = errno;
+        idset_destroy (tc->barrier_pending_ranks);
+        flux_watcher_destroy (tc->shell_barrier_timer);
+        free (tc);
+        errno = saved_errno;
+    }
 }
 
-static const char *job_shell_path (struct jobinfo *job)
+static struct exec_ctx *exec_ctx_create (struct jobinfo *job,
+                                         const struct idset *ranks,
+                                         flux_error_t *errp)
 {
-    const char *path = jobspec_get_job_shell (job->jobspec);
-    return path ? path : default_job_shell;
-}
+    json_error_t error;
+    const char *service;
+    struct exec_ctx *ctx = calloc (1, sizeof (*ctx));
+    flux_reactor_t *r = flux_get_reactor (job->h);
+    double barrier_timeout = config_get_default_barrier_timeout ();
+
+    if (!r || !ctx || !(ctx->barrier_pending_ranks = idset_copy (ranks))) {
+        errprintf (errp, "Out of memory");
+        goto error;
+    }
 
-static const char *jobspec_get_cwd (json_t *jobspec)
-{
-    const char *cwd = NULL;
-    (void) json_unpack (jobspec, "{s:{s:{s:s}}}",
-                                 "attributes", "system",
-                                 "cwd", &cwd);
-    return cwd;
+    ctx->job = job;
+
+    /* Note: service unpacked below but unused to allow use of strict (!)
+     * unpacking.
+     */
+    if (json_unpack_ex (job->jobspec,
+                        &error,
+                        0,
+                        "{s?{s?{s?{s?{s?s s?s s?F !}}}}}",
+                        "attributes",
+                          "system",
+                            "exec",
+                              "bulkexec",
+                                "service", &service,
+                                "mock_exception", &ctx->mock_exception,
+                                "barrier-timeout", &barrier_timeout) < 0) {
+        errprintf (errp,
+                   "failed to unpack system.exec.bulkexec for %s: %s",
+                    idf58 (job->id),
+                    error.text);
+        goto error;
+    }
+
+    if (barrier_timeout > 0.) {
+        ctx->shell_barrier_timer = flux_timer_watcher_create (r,
+                                                              barrier_timeout,
+                                                              0.,
+                                                              barrier_timer_cb,
+                                                              ctx);
+        if (!ctx->shell_barrier_timer) {
+            errprintf (errp,
+                       "%s: failed to create barrier timer",
+                       idf58 (ctx->job->id));
+            goto error;
+        }
+    }
+
+    return ctx;
+error:
+    exec_ctx_destroy (ctx);
+    return NULL;
 }
 
-static const char *job_get_cwd (struct jobinfo *job)
+static const char * exec_mock_exception (struct bulk_exec *exec)
 {
-    const char *cwd = jobspec_get_cwd (job->jobspec);
-    if (!cwd)
-        cwd = default_cwd;
-    return (cwd);
+    struct exec_ctx *ctx = bulk_exec_aux_get (exec, "ctx");
+    if (!ctx || !ctx->mock_exception)
+        return "none";
+    return ctx->mock_exception;
 }
 
 static void start_cb (struct bulk_exec *exec, void *arg)
 {
     struct jobinfo *job = arg;
-    jobinfo_started (job, NULL);
+    jobinfo_started (job);
 }
 
 static void complete_cb (struct bulk_exec *exec, void *arg)
@@ -119,31 +194,335 @@ static void complete_cb (struct bulk_exec *exec, void *arg)
                             bulk_exec_rc (exec));
 }
 
-static void output_cb (struct bulk_exec *exec, flux_subprocess_t *p,
+static void barrier_timer_stop (struct exec_ctx *ctx)
+{
+    flux_watcher_stop (ctx->shell_barrier_timer);
+}
+
+static void barrier_timer_start (struct exec_ctx *ctx)
+{
+    /* Only ever create one barrier timer (for the first shell barrier)
+     */
+    if (ctx->barrier_completion_count == 0)
+        flux_watcher_start (ctx->shell_barrier_timer);
+}
+
+
+static int exec_barrier_enter (struct bulk_exec *exec, int rank)
+{
+    struct exec_ctx *ctx = bulk_exec_aux_get (exec, "ctx");
+
+    if (!ctx)
+        return -1;
+
+    (void) idset_clear (ctx->barrier_pending_ranks, rank);
+    if (++ctx->barrier_enter_count == bulk_exec_total (exec)) {
+        if (bulk_exec_write (exec, "stdin", "exit=0\n", 7) < 0)
+            return -1;
+        ctx->barrier_enter_count = 0;
+        ctx->barrier_completion_count++;
+        barrier_timer_stop (ctx);
+    }
+    else if (ctx->barrier_enter_count == 1 && ctx->exit_count > 0) {
+        /*
+         *  Terminate barrier with error immediately when a barrier is
+         *   started after one or more shells have already exited. The
+         *   case where a shell exits while a barrier is already in progress
+         *   is handled in exit_cb().
+         */
+        if (bulk_exec_write (exec, "stdin", "exit=1\n", 7) < 0)
+            return -1;
+    }
+
+    /*  When the first shell enters the barrier, start a timer after
+     *   which the job will be terminated if all shells have not reached
+     *   the barrier.
+     */
+    if (ctx->barrier_enter_count == 1)
+        barrier_timer_start (ctx);
+
+    return 0;
+}
+
+static void output_cb (struct bulk_exec *exec,
+                       flux_subprocess_t *p,
                        const char *stream,
                        const char *data,
-                       int data_len,
+                       int len,
                        void *arg)
 {
     struct jobinfo *job = arg;
-    flux_log (job->h, LOG_INFO, "%ju: %d: %s: %s",
-                      (uintmax_t) job->id,
-                      flux_subprocess_rank (p),
-                      stream, data);
+    const char *cmd = flux_cmd_arg (flux_subprocess_get_cmd (p), 0);
+    int rank = flux_subprocess_rank (p);
+
+    if (streq (stream, "stdout")) {
+        if (len == 6
+            && strncmp (data, "enter\n", 6) == 0
+            && exec_barrier_enter (exec, rank) < 0) {
+            jobinfo_fatal_error (job,
+                                 errno,
+                                 "Failed to handle barrier");
+        }
+        return;
+    }
+    jobinfo_log_output (job,
+                        flux_subprocess_rank (p),
+                        basename_simple (cmd),
+                        stream,
+                        data,
+                        len);
+}
+
+static int lost_shell (struct jobinfo *job,
+                       bool critical,
+                       int shell_rank,
+                       const char *fmt,
+                       ...)
+{
+    flux_future_t *f;
+    char msgbuf[160];
+    int msglen = sizeof (msgbuf);
+    char *msg = msgbuf;
+    va_list ap;
+    int severity = critical ? 0 : FLUX_JOB_EXCEPTION_CRIT;
+
+    if (fmt) {
+        va_start (ap, fmt);
+        if (vsnprintf (msg, msglen, fmt, ap) >= msglen)
+            (void) snprintf (msg, msglen, "%s", "lost contact with job shell");
+        va_end (ap);
+    }
+
+    if (!critical) {
+        /* Raise a non-fatal job exception if the lost shell was not critical.
+         * The job exec service will raise a fatal exception later for
+         * critical shells.
+         */
+        jobinfo_raise (job,
+                       "node-failure",
+                       FLUX_JOB_EXCEPTION_CRIT,
+                       "%s",
+                       msg);
+        /* If an exception was raised, do not duplicate the message
+         * to the shell exception service since the message will already
+         * be displayed as part of the exception note:
+         */
+        msg = "";
+    }
+
+    /* Also notify job shell rank 0 of exception
+     */
+    if (!(f = jobinfo_shell_rpc_pack (job,
+                                      "exception",
+                                      "{s:s s:i s:i s:s}",
+                                      "type", "lost-shell",
+                                      "severity", severity,
+                                      "shell_rank", shell_rank,
+                                      "message", msg)))
+            return -1;
+    /*  Do not wait for response. If a shell is lost because the job
+     *  is terminating, then the rank 0 shell may also have exited by the
+     *  time this message is sent, so a response may never come. This
+     *  could leak the future (and the job reference taken by
+     *  jobinfo_shell_rpc_pack())
+     */
+    flux_future_destroy (f);
+    return 0;
+}
+
+static bool is_critical_rank (struct jobinfo *job, int shell_rank)
+{
+    return idset_test (job->critical_ranks, shell_rank);
 }
 
 static void error_cb (struct bulk_exec *exec, flux_subprocess_t *p, void *arg)
 {
     struct jobinfo *job = arg;
-    const char *arg0 = flux_cmd_arg (flux_subprocess_get_cmd (p), 0);
-    jobinfo_fatal_error (job, flux_subprocess_fail_errno (p),
-                              "cmd=%s: rank=%d failed",
-                              arg0, flux_subprocess_rank (p));
+    flux_cmd_t *cmd = flux_subprocess_get_cmd (p);
+    int errnum = flux_subprocess_fail_errno (p);
+    int rank = flux_subprocess_rank (p);
+    int shell_rank = resource_set_rank_index (job->R, rank);
+    const char *hostname = flux_get_hostbyrank (job->h, rank);
+
+    /*  cmd may be NULL here if exec implementation failed to
+     *   create flux_cmd_t
+     */
+    if (cmd) {
+        if (errnum == EHOSTUNREACH) {
+            bool critical = is_critical_rank (job, shell_rank);
+
+            /*  Always notify rank 0 shell of a lost shell.
+             */
+            lost_shell (job,
+                        critical,
+                        shell_rank,
+                        "node failure on %s (shell rank %d)",
+                        hostname,
+                        shell_rank);
+
+            /*  Raise a fatal error and terminate job immediately if
+             *  the lost shell was critical.
+             */
+            if (critical)
+                jobinfo_fatal_error (job,
+                                     0,
+                                     "node failure on %s (rank %d)",
+                                     hostname,
+                                     rank);
+        }
+        else if (errnum == ENOSYS) {
+            jobinfo_fatal_error (job,
+                                 0,
+                                 "%s service is not loaded on %s (rank %d)",
+                                 bulk_exec_service_name (exec),
+                                 hostname,
+                                 rank);
+        }
+        else {
+            jobinfo_fatal_error (job,
+                                 errnum,
+                                 "%s on broker %s (rank %d): %s",
+                                 "job shell exec error",
+                                 hostname,
+                                 rank,
+                                 flux_cmd_arg (cmd, 0));
+        }
+    }
+    else
+        jobinfo_fatal_error (job,
+                             flux_subprocess_fail_errno (p),
+                             "job shell exec error on %s (rank %d)",
+                             hostname,
+                             rank);
+}
+
+
+static void exit_cb (struct bulk_exec *exec,
+                     void *arg,
+                     const struct idset *ranks)
+{
+    struct jobinfo *job = arg;
+    struct exec_ctx *ctx = bulk_exec_aux_get (exec, "ctx");
+
+    /*  Nothing to do here if the job consists of only one shell.
+     *  (or, if we fail to to get ctx object (highly unlikely))
+     */
+    if (bulk_exec_total (exec) == 1
+        || !(ctx = bulk_exec_aux_get (exec, "ctx")))
+        return;
+
+    ctx->exit_count++;
+
+    /*  Check if a shell is exiting before the first barrier, in which
+     *   case we raise a job exception because the shell or IMP may not
+     *   have had a chance to do so.
+     */
+    if (ctx->barrier_completion_count == 0) {
+        char *ids = idset_encode (ranks, IDSET_FLAG_RANGE);
+        char *hosts = flux_hostmap_lookup (job->h, ids, NULL);
+        jobinfo_fatal_error (job, 0,
+                             "%s (rank%s %s) terminated before first barrier",
+                              hosts ? hosts : "(unknown)",
+                              idset_count (ranks) ? "s" : "",
+                              ids ? ids : "(unknown)");
+        free (ids);
+        free (hosts);
+    }
+
+    /*  If a shell exited before the first barrier or there is a
+     *   barrier in progress (enter_count > 0), then terminate the
+     *   current/next barrier immediately with error. This will allow
+     *   shells currently waiting or entering the barrier in the future
+     *   to exit immediately, rather than being killed by exec system.
+     */
+    if (ctx->barrier_completion_count == 0
+        || ctx->barrier_enter_count > 0) {
+        if (bulk_exec_write (exec, "stdin", "exit=1\n", 7) < 0)
+            jobinfo_fatal_error (job,
+                                 0,
+                                 "failed to terminate barrier: %s",
+                                 strerror (errno));
+    }
+
+    /*  If a shell exits due to signal report the shell as lost to
+     *  the leader shell. This avoids potential hangs in the leader
+     *  shell if it is waiting for data from job shells that did not
+     *  exit cleanly.
+     */
+    unsigned int rank = idset_first (ranks);
+    while (rank != IDSET_INVALID_ID) {
+        flux_subprocess_t *p = bulk_exec_get_subprocess (exec, rank);
+        int signo = flux_subprocess_signaled (p);
+        int shell_rank = resource_set_rank_index (job->R, rank);
+        if (p && signo > 0) {
+            if (shell_rank != 0)
+                lost_shell (job,
+                            is_critical_rank (job, shell_rank),
+                            shell_rank,
+                            "shell rank %d (on %s): %s",
+                            shell_rank,
+                            flux_get_hostbyrank (job->h, rank),
+                            strsignal (signo));
+            else {
+                /*  Job can't continue without the leader shell, which has
+                 *  terminated unexpectedly. Cancel the job now to avoid
+                 *  a potential hang.
+                 */
+                jobinfo_fatal_error (job,
+                                     0,
+                                     "shell rank 0 (on %s): %s",
+                                     flux_get_hostbyrank (job->h, rank),
+                                     strsignal (signo));
+            }
+        }
+        rank = idset_next (ranks, rank);
+    }
+}
+
+static int parse_service_option (json_t *jobspec,
+                                 const char **service,
+                                 flux_error_t *error)
+{
+    const char *s = config_get_exec_service (); // default
+    bool override = config_get_exec_service_override ();
+    json_error_t e;
+
+    if (jobspec) {
+        const char *s2 = NULL;
+        if (json_unpack_ex (jobspec,
+                            &e,
+                            0,
+                            "{s:{s?{s?{s?{s?s}}}}}",
+                            "attributes",   // key is required per RFC 14
+                              "system",     // key is optional per RFC 14
+                                "exec",
+                                  "bulkexec",
+                                    "service", &s2) < 0) {
+            errprintf (error, "error parsing bulkexec.service: %s", e.text);
+            errno = EINVAL;
+            return -1;
+        }
+        if (s2) {
+            if (!override && !streq (s, s2)) {
+                errprintf (error, "exec service override is not permitted");
+                errno = EINVAL;
+                return -1;
+            }
+            s = s2;
+        }
+    }
+    if (!streq (s, "rexec") && !streq (s, "sdexec")) {
+        errprintf (error, "unknown bulkexec.service value: %s", s);
+        errno = EINVAL;
+        return -1;
+    }
+    *service = s;
+    return 0;
 }
 
 static struct bulk_exec_ops exec_ops = {
     .on_start =     start_cb,
-    .on_exit =      NULL,
+    .on_exit =      exit_cb,
     .on_complete =  complete_cb,
     .on_output =    output_cb,
     .on_error =     error_cb
@@ -152,24 +531,43 @@ static struct bulk_exec_ops exec_ops = {
 static int exec_init (struct jobinfo *job)
 {
     flux_cmd_t *cmd = NULL;
-    struct exec_conf *conf = NULL;
+    struct exec_ctx *ctx = NULL;
     struct bulk_exec *exec = NULL;
     const struct idset *ranks = NULL;
+    const char *imp_path = NULL;
+    const char *service;
+    flux_error_t error;
+
+    if (job->multiuser && !(imp_path = config_get_imp_path ())) {
+        flux_log (job->h,
+                  LOG_ERR,
+                  "unable run multiuser job with no IMP configured!");
+        goto err;
+    }
 
     if (!(ranks = resource_set_ranks (job->R))) {
         flux_log_error (job->h, "exec_init: resource_set_ranks");
         goto err;
     }
-    if (!(exec = bulk_exec_create (&exec_ops, job))) {
+    if (parse_service_option (job->jobspec, &service, &error) < 0) {
+        flux_log (job->h, LOG_ERR, "exec_init: %s" , error.text);
+        goto err;
+    }
+    if (!(exec = bulk_exec_create (&exec_ops,
+                                   service,
+                                   job->id,
+                                   job->multiuser ? "imp-shell" : "shell",
+                                   job))) {
         flux_log_error (job->h, "exec_init: bulk_exec_create");
         goto err;
     }
-    if (!(conf = exec_conf_create (job->jobspec))) {
-        flux_log_error (job->h, "exec_init: exec_conf_create");
+    if (!(ctx = exec_ctx_create (job, ranks, &error))) {
+        flux_log (job->h, LOG_ERR, "exec_ctx_create: %s", error.text);
         goto err;
     }
-    if (bulk_exec_aux_set (exec, "conf", conf,
-                          (flux_free_f) exec_conf_destroy) < 0) {
+    if (bulk_exec_aux_set (exec, "ctx", ctx,
+                          (flux_free_f) exec_ctx_destroy) < 0) {
+        exec_ctx_destroy (ctx);
         flux_log_error (job->h, "exec_init: bulk_exec_aux_set");
         goto err;
     }
@@ -177,19 +575,60 @@ static int exec_init (struct jobinfo *job)
         flux_log_error (job->h, "exec_init: flux_cmd_create");
         goto err;
     }
+    /* Set any configured exec.sdexec-properties.
+     */
+    json_t *props;
+    if (streq (service, "sdexec")
+        && (props = config_get_sdexec_properties ())) {
+        const char *k;
+        json_t *v;
+        json_object_foreach (props, k, v) {
+            char name[128];
+            snprintf (name, sizeof (name), "SDEXEC_PROP_%s", k);
+            if (flux_cmd_setopt (cmd, name, json_string_value (v)) < 0) {
+                flux_log_error (job->h, "Unable to set sdexec options");
+                return -1;
+            }
+        }
+    }
     if (flux_cmd_setenvf (cmd, 1, "FLUX_KVS_NAMESPACE", "%s", job->ns) < 0) {
         flux_log_error (job->h, "exec_init: flux_cmd_setenvf");
         goto err;
     }
-    if (flux_cmd_argv_append (cmd, job_shell_path (job)) < 0
+    if (job->multiuser) {
+        if (flux_cmd_setenvf (cmd,
+                              1,
+                              "FLUX_IMP_EXEC_HELPER",
+                              "flux imp-exec-helper %ju",
+                              (uintmax_t) job->id) < 0) {
+            flux_log_error (job->h, "exec_init: flux_cmd_setenvf");
+            goto err;
+        }
+        /* The systemd user instance running as user flux is not privileged
+         * to signal guest processes, therefore only signal the IMP and
+         * never use SIGKILL.  See flux-framework/flux-core#6399
+         */
+        if (streq (service, "sdexec")) {
+            if (flux_cmd_setopt (cmd, "SDEXEC_PROP_KillMode", "process") < 0
+                || flux_cmd_setopt (cmd,
+                                    "SDEXEC_PROP_SendSIGKILL",
+                                    "off") < 0) {
+                flux_log_error (job->h,
+                                "Unable to set multiuser sdexec options");
+                return -1;
+            }
+        }
+        if (flux_cmd_argv_append (cmd, config_get_imp_path ()) < 0
+            || flux_cmd_argv_append (cmd, "exec") < 0) {
+            flux_log_error (job->h, "exec_init: flux_cmd_argv_append");
+            goto err;
+        }
+    }
+    if (flux_cmd_argv_append (cmd, config_get_job_shell (job)) < 0
         || flux_cmd_argv_appendf (cmd, "%ju", (uintmax_t) job->id) < 0) {
         flux_log_error (job->h, "exec_init: flux_cmd_argv_append");
         goto err;
     }
-    if (flux_cmd_setcwd (cmd, job_get_cwd (job)) < 0) {
-        flux_log_error (job->h, "exec_init: flux_cmd_setcwd");
-        goto err;
-    }
     if (bulk_exec_push_cmd (exec, ranks, cmd, 0) < 0) {
         flux_log_error (job->h, "exec_init: bulk_exec_push_cmd");
         goto err;
@@ -203,18 +642,20 @@ static int exec_init (struct jobinfo *job)
     return -1;
 }
 
-static void exec_check_cb (flux_reactor_t *r, flux_watcher_t *w,
-                           int revents, void *arg)
+static void exec_check_cb (flux_reactor_t *r,
+                           flux_watcher_t *w,
+                           int revents,
+                           void *arg)
 {
     struct jobinfo *job = arg;
     struct bulk_exec *exec = job->data;
-    if (bulk_exec_current (exec) >= 1) {
+    if (bulk_exec_started_count (exec) >= 1) {
         jobinfo_fatal_error (job, 0, "mock starting exception generated");
         flux_log (job->h,
                   LOG_DEBUG,
                   "mock exception for starting job total=%d, current=%d",
                   bulk_exec_total (exec),
-                  bulk_exec_current (exec));
+                  bulk_exec_started_count (exec));
         flux_watcher_destroy (w);
     }
 }
@@ -222,8 +663,14 @@ static void exec_check_cb (flux_reactor_t *r, flux_watcher_t *w,
 static int exec_start (struct jobinfo *job)
 {
     struct bulk_exec *exec = job->data;
+    struct exec_ctx *ctx = bulk_exec_aux_get (exec, "ctx");
+
+    if (!exec || !(ctx = bulk_exec_aux_get (exec, "ctx"))) {
+        jobinfo_fatal_error (job, errno, "failed to get bulk-exec ctx");
+        return -1;
+    }
 
-    if (strcmp (exec_mock_exception (exec), "init") == 0) {
+    if (streq (exec_mock_exception (exec), "init")) {
         /* If creating an "init" mock exception, generate it and
          *  then return to simulate an exception that came in before
          *  we could actually start the job
@@ -231,7 +678,7 @@ static int exec_start (struct jobinfo *job)
         jobinfo_fatal_error (job, 0, "mock init exception generated");
         return 0;
     }
-    else if (strcmp (exec_mock_exception (exec), "starting") == 0) {
+    else if (streq (exec_mock_exception (exec), "starting")) {
         /*  If we're going to mock an exception in "starting" phase, then
          *   set up a check watcher to cancel the job when some shells have
          *   started but (potentially) not all.
@@ -249,7 +696,7 @@ static void exec_kill_cb (flux_future_t *f, void *arg)
 {
     struct jobinfo *job = arg;
     if (flux_future_get (f, NULL) < 0 && errno != ENOENT)
-        flux_log_error (job->h, "%ju: exec_kill", (uintmax_t) job->id);
+        bulk_exec_kill_log_error (f, job->id);
     jobinfo_decref (job);
     flux_future_destroy (f);
 }
@@ -257,21 +704,19 @@ static void exec_kill_cb (flux_future_t *f, void *arg)
 static int exec_kill (struct jobinfo *job, int signum)
 {
     struct  bulk_exec *exec = job->data;
-    flux_future_t *f = bulk_exec_kill (exec, signum);
-    if (!f) {
+    flux_future_t *f;
+
+    if (!(f = bulk_exec_kill (exec, NULL, signum))) {
         if (errno != ENOENT)
-            flux_log_error (job->h, "%ju: bulk_exec_kill", job->id);
+            flux_log_error (job->h, "%s: bulk_exec_kill", idf58 (job->id));
         return 0;
     }
 
-    flux_log (job->h, LOG_DEBUG,
-              "exec_kill: %ju: signal %d",
-              (uintmax_t) job->id,
-              signum);
-
     jobinfo_incref (job);
     if (flux_future_then (f, 3., exec_kill_cb, job) < 0) {
-        flux_log_error (job->h, "%ju: exec_kill: flux_future_then", job->id);
+        flux_log_error (job->h,
+                        "%s: exec_kill: flux_future_then",
+                        idf58 (job->id));
         flux_future_destroy (f);
         return -1;
     }
@@ -284,13 +729,6 @@ static int exec_cancel (struct jobinfo *job)
     return bulk_exec_cancel (exec);
 }
 
-static int exec_cleanup (struct jobinfo *job, const struct idset *idset)
-{
-    /* No epilog supported */
-    jobinfo_cleanup_complete (job, idset, 0);
-    return 0;
-}
-
 static void exec_exit (struct jobinfo *job)
 {
     struct bulk_exec *exec = job->data;
@@ -298,38 +736,72 @@ static void exec_exit (struct jobinfo *job)
     job->data = NULL;
 }
 
-/*  Configure the exec module.
- *  Read the default job shell path from config. Allow override on cmdline
- */
-static int exec_config (flux_t *h, int argc, char **argv)
+static int exec_config (flux_t *h,
+                        const flux_conf_t *conf,
+                        int argc,
+                        char **argv,
+                        flux_error_t *errp)
 {
-    flux_conf_error_t err;
+    return config_setup (h, conf, argc, argv, errp);
+}
 
-    /*  Set default job shell path from builtin configuration,
-     *   allow override via configuration, then cmdline.
-     */
-    default_job_shell = flux_conf_builtin_get ("shell_path", FLUX_CONF_AUTO);
-
-
-    /*  Check configuration for exec.job-shell */
-    if (flux_conf_unpack (flux_get_conf (h, NULL),
-                          &err,
-                          "{s?:{s?s}}",
-                          "exec",
-                            "job-shell", &default_job_shell) < 0) {
-        flux_log (h, LOG_ERR,
-                  "error reading config value exec.job-shell: %s",
-                  err.errbuf);
-        return -1;
-    }
+static json_t *exec_config_stats (void)
+{
+    json_t *o = NULL;
+    json_t *conf = NULL;
 
-    /* Finally, override on cmdline if job-shell=%s appears */
-    for (int i = 0; i < argc; i++) {
-        if (strncmp (argv[i], "job-shell=", 10) == 0)
-            default_job_shell = argv[i]+10;
+    if (!(o = json_object ())) {
+        errno = ENOMEM;
+        goto error;
     }
-    flux_log (h, LOG_DEBUG, "using default shell path %s", default_job_shell);
-    return 0;
+
+    if (config_get_stats (&conf) < 0)
+        goto error;
+
+    if (json_object_set_new (o, "config", conf) < 0)
+        goto error;
+
+    return o;
+error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    ERRNO_SAFE_WRAP (json_decref, conf);
+    return NULL;
+}
+
+static json_t *exec_job_stats (struct jobinfo *job)
+{
+    struct bulk_exec *exec = job->data;
+    struct idset *active_ranks;
+    char *s = NULL;
+    json_t *o;
+    int total = bulk_exec_total (exec);
+    int active = bulk_exec_active_count (exec);
+
+    if ((active_ranks = bulk_exec_active_ranks (exec)))
+        s = idset_encode (active_ranks, IDSET_FLAG_RANGE);
+
+    o = json_pack ("{s:i s:i s:s}",
+                   "total_shells", total,
+                   "active_shells", active,
+                   "active_ranks", s ? s : "");
+    free (s);
+    idset_destroy (active_ranks);
+    return o;
+}
+
+static json_t *exec_stats (struct jobinfo *job)
+{
+    if (job)
+        return exec_job_stats (job);
+    else
+        return exec_config_stats ();
+}
+
+static struct idset *active_ranks (struct jobinfo *job)
+{
+    if (job)
+        return bulk_exec_active_ranks ((struct bulk_exec *) job->data);
+    return NULL;
 }
 
 struct exec_implementation bulkexec = {
@@ -340,7 +812,8 @@ struct exec_implementation bulkexec = {
     .start =    exec_start,
     .kill =     exec_kill,
     .cancel =   exec_cancel,
-    .cleanup =  exec_cleanup
+    .stats =    exec_stats,
+    .active_ranks = active_ranks,
 };
 
 /* vi: ts=4 sw=4 expandtab
diff --git a/src/modules/job-exec/exec_config.c b/src/modules/job-exec/exec_config.c
new file mode 100644
index 000000000000..1eaba425e03a
--- /dev/null
+++ b/src/modules/job-exec/exec_config.c
@@ -0,0 +1,290 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Flux bulk-exec configuration code */
+
+#if HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <jansson.h>
+#include <unistd.h>
+
+#if HAVE_FLUX_SECURITY
+#include <flux/security/version.h>
+#endif
+
+#include "exec_config.h"
+#include "ccan/str/str.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/jpath.h"
+
+static const char *default_cwd = "/tmp";
+
+struct exec_config {
+    const char *default_job_shell;
+    const char *flux_imp_path;
+    const char *exec_service;
+    int exec_service_override;
+    json_t *sdexec_properties;
+    double default_barrier_timeout;
+};
+
+/* Global configs initialized in config_init() */
+static struct exec_config exec_conf;
+
+static const char *jobspec_get_job_shell (json_t *jobspec)
+{
+    const char *path = NULL;
+    if (jobspec)
+        (void) json_unpack (jobspec,
+                            "{s:{s:{s:{s:s}}}}",
+                            "attributes",
+                             "system",
+                              "exec",
+                               "job_shell", &path);
+    return path;
+}
+
+const char *config_get_job_shell (struct jobinfo *job)
+{
+    const char *path = NULL;
+    if (job)
+        path = jobspec_get_job_shell (job->jobspec);
+    return path ? path : exec_conf.default_job_shell;
+}
+
+static const char *jobspec_get_cwd (json_t *jobspec)
+{
+    const char *cwd = NULL;
+    if (jobspec)
+        (void) json_unpack (jobspec,
+                            "{s:{s:{s:s}}}",
+                             "attributes",
+                              "system",
+                               "cwd", &cwd);
+    return cwd;
+}
+
+const char *config_get_cwd (struct jobinfo *job)
+{
+    const char *cwd = NULL;
+    if (job) {
+        if (job->multiuser)
+            cwd = "/";
+        else if (!(cwd = jobspec_get_cwd (job->jobspec)))
+            cwd = default_cwd;
+    }
+    return cwd;
+}
+
+const char *config_get_imp_path (void)
+{
+    return exec_conf.flux_imp_path;
+}
+
+const char *config_get_exec_service (void)
+{
+    return exec_conf.exec_service;
+}
+
+bool config_get_exec_service_override (void)
+{
+    return exec_conf.exec_service_override;
+}
+
+json_t *config_get_sdexec_properties (void)
+{
+    return exec_conf.sdexec_properties;
+}
+
+double config_get_default_barrier_timeout (void)
+{
+    return exec_conf.default_barrier_timeout;
+}
+
+int config_get_stats (json_t **config_stats)
+{
+    json_t *o = NULL;
+
+    if (!(o = json_pack ("{s:s? s:s? s:s? s:s? s:i s:f}",
+                         "default_cwd", default_cwd,
+                         "default_job_shell", exec_conf.default_job_shell,
+                         "flux_imp_path", exec_conf.flux_imp_path,
+                         "exec_service", exec_conf.exec_service,
+                         "exec_service_override",
+                         exec_conf.exec_service_override,
+                         "default_barrier_timeout",
+                         exec_conf.default_barrier_timeout))) {
+        errno = ENOMEM;
+        return -1;
+    }
+
+    if (exec_conf.sdexec_properties) {
+        if (json_object_set (o,
+                             "sdexec_properties",
+                             exec_conf.sdexec_properties) < 0)
+            goto error;
+    }
+
+    (void) jpath_clear_null (o);
+
+    (*config_stats) = o;
+    return 0;
+
+error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return -1;
+}
+
+static void exec_config_init (struct exec_config *ec)
+{
+    ec->default_job_shell = flux_conf_builtin_get ("shell_path", FLUX_CONF_AUTO);
+    ec->flux_imp_path = NULL;
+    ec->exec_service = "rexec";
+    ec->exec_service_override = 0;
+    ec->sdexec_properties = NULL;
+    ec->default_barrier_timeout = 1800.;
+}
+
+/*  Initialize configurations for use by job-exec bulk-exec
+ *  implementation
+ */
+int config_setup (flux_t *h,
+                  const flux_conf_t *conf,
+                  int argc,
+                  char **argv,
+                  flux_error_t *errp)
+{
+    struct exec_config tmpconf;
+    const char *barrier_timeout = NULL;
+    flux_error_t err;
+
+    /* Per trws comment in 97421e88987535260b10d6a19551cea625f26ce4
+     *
+     * The musl libc loader doesn't actually unload objects on
+     * dlclose, so a subsequent dlopen doesn't re-clear globals and
+     * similar.
+     *
+     * So we must re-initialize globals everytime we reload the module.
+     */
+    exec_config_init (&tmpconf);
+
+    /*  Check configuration for exec.job-shell */
+    if (flux_conf_unpack (conf,
+                          &err,
+                          "{s?{s?s}}",
+                          "exec",
+                            "job-shell", &tmpconf.default_job_shell) < 0) {
+        errprintf (errp,
+                   "error reading config value exec.job-shell: %s",
+                   err.text);
+        return -1;
+    }
+
+    /*  Check configuration for exec.imp */
+    if (flux_conf_unpack (conf,
+                          &err,
+                          "{s?{s?s}}",
+                          "exec",
+                            "imp", &tmpconf.flux_imp_path) < 0) {
+        errprintf (errp,
+                   "error reading config value exec.imp: %s",
+                   err.text);
+        return -1;
+    }
+
+    /*  Check configuration for exec.service and exec.service-override */
+    if (flux_conf_unpack (conf,
+                          &err,
+                          "{s?{s?s s?b}}",
+                          "exec",
+                            "service", &tmpconf.exec_service,
+                            "service-override",
+                              &tmpconf.exec_service_override) < 0) {
+        errprintf (errp,
+                   "error reading config value exec.service: %s",
+                   err.text);
+        return -1;
+    }
+
+    /*  Check configuration for exec.sdexec-properties */
+    if (flux_conf_unpack (conf,
+                          &err,
+                          "{s?{s?o}}",
+                          "exec",
+                            "sdexec-properties",
+                              &tmpconf.sdexec_properties) < 0) {
+        errprintf (errp,
+                  "error reading config table exec.sdexec-properties: %s",
+                  err.text);
+        return -1;
+    }
+    if (tmpconf.sdexec_properties) {
+        const char *k;
+        json_t *v;
+
+        if (!json_is_object (tmpconf.sdexec_properties)) {
+            errprintf (errp, "exec.sdexec-properties is not a table");
+            errno = EINVAL;
+            return -1;
+        }
+        json_object_foreach (tmpconf.sdexec_properties, k, v) {
+            if (!json_is_string (v)) {
+                errprintf (errp,
+                          "exec.sdexec-properties.%s is not a string",
+                          k);
+                errno = EINVAL;
+                return -1;
+            }
+        }
+    }
+
+    /*  Check configuration for exec.barrier-timeout */
+    if (flux_conf_unpack (conf,
+                          &err,
+                          "{s?{s?s}}",
+                          "exec",
+                            "barrier-timeout", &barrier_timeout) < 0) {
+        errprintf (errp,
+                   "error reading config value exec.barrier-timeout: %s",
+                   err.text);
+        return -1;
+    }
+    if (barrier_timeout
+        && fsd_parse_duration (barrier_timeout,
+                               &tmpconf.default_barrier_timeout) < 0) {
+        errprintf (errp,
+                   "invalid duration '%s' specified for exec.barrier-timeout",
+                   barrier_timeout);
+        return -1;
+    }
+
+
+    if (argv && argc) {
+        /* Finally, override values on cmdline */
+        for (int i = 0; i < argc; i++) {
+            if (strstarts (argv[i], "job-shell="))
+                tmpconf.default_job_shell = argv[i]+10;
+            else if (strstarts (argv[i], "imp="))
+                tmpconf.flux_imp_path = argv[i]+4;
+            else if (strstarts (argv[i], "service="))
+                tmpconf.exec_service = argv[i]+8;
+        }
+    }
+
+    exec_conf = tmpconf;
+    return 0;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/job-exec/exec_config.h b/src/modules/job-exec/exec_config.h
new file mode 100644
index 000000000000..18dab00c1d19
--- /dev/null
+++ b/src/modules/job-exec/exec_config.h
@@ -0,0 +1,50 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_JOB_EXEC_CONFIG_H
+#define HAVE_JOB_EXEC_CONFIG_H 1
+
+#include <jansson.h>
+#include <stdbool.h>
+#include <flux/core.h>
+
+#include "job-exec.h"
+
+/* Configuration getters.  It is not safe to hold on to returned
+ * values.  Callers should strdup() / json_incref() / etc. values
+ * they wish to access for later use.
+ */
+
+const char *config_get_job_shell (struct jobinfo *job);
+
+const char *config_get_cwd (struct jobinfo *job);
+
+const char *config_get_imp_path (void);
+
+const char *config_get_exec_service (void);
+
+json_t *config_get_sdexec_properties (void);
+
+bool config_get_exec_service_override (void);
+
+double config_get_default_barrier_timeout (void);
+
+int config_get_stats (json_t **config_stats);
+
+int config_setup (flux_t *h,
+                  const flux_conf_t *conf,
+                  int argc,
+                  char **argv,
+                  flux_error_t *errp);
+
+#endif /* !HAVE_JOB_EXEC_CONFIG_EXEC_H */
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/job-exec/job-exec.c b/src/modules/job-exec/job-exec.c
index 7afff4a8bcd1..646acd13ef4e 100644
--- a/src/modules/job-exec/job-exec.c
+++ b/src/modules/job-exec/job-exec.c
@@ -17,8 +17,8 @@
  * real work. Execution is simulated by setting a timer for the duration
  * specified in either the jobspec system.duration attribute or a test
  * duration in system.exec.test.run_duration.  The module can optionally
- * simulate an epilog/cleanup stage, and/or mock exceptions during run
- * or initialization. See TEST CONFIGURATION below.
+ * simulate mock exceptions during run or initialization.
+ * See TEST CONFIGURATION below.
  *
  * OPERATION
  *
@@ -26,7 +26,7 @@
  *
  * JOB INIT:
  *
- * On reciept of a start request, the exec service enters initialization
+ * On receipt of a start request, the exec service enters initialization
  * phase of the job, where the jobspec and R are fetched from the KVS,
  * and the guest namespace is created and linked from the primary
  * namespace. A guest.exec.eventlog is created with an initial "init"
@@ -39,7 +39,7 @@
  * JOB STARTING/RUNNING:
  *
  * After initialization is complete, the exec service emits a "starting"
- * event to the exec eventlog and calls the implementaton "start" method.
+ * event to the exec eventlog and calls the implementation "start" method.
  * Once all job shells or equivalent are running, the exec implementation
  * should invoke jobinfo_started(), which emits a "running" event to the
  * exec eventlog and sends the "start" response to the job-manager.
@@ -48,17 +48,12 @@
  *
  * As tasks/job shells exit, the exec implementation should call
  * jobinfo_tasks_complete(), which emits a "complete" event to the exec
- * eventlog, sends a "finish" response to the job-manager, emits a
- * "cleanup.start" event in the exec eventlog, and finally invokes
- * the exec implementation's "cleanup" method on the completed ranks.
- * (NB: currently a subset of ranks is not supported)
+ * eventlog, sends a "finish" response to the job-manager.
  *
  * JOB FINALIZATION:
  *
- * Once cleanup tasks have completed, the exec implementation should call
- * jobinfo_cleanup_complete(), which emits "cleanup.finish" to the exec
- * eventlog, and then calls jobinfo_finalize, which performs the following
- * tasks:
+ * jobinfo_finalize() is called after the "finish" event, which performs
+ * the following tasks:
  *
  *  - terminating "done" event is posted to the exec.eventlog
  *  - the guest namespace, now quiesced, is copied to the primary namespace
@@ -74,14 +69,13 @@
  *
  * {
  *   "run_duration":s,      - alternate/override attributes.system.duration
- *   "cleanup_duration":s   - enable a fake job epilog and set its duration
  *   "wait_status":i        - report this value as status in the "finish" resp
  *   "mock_exception":s     - mock an exception during this phase of job
  *                             execution (currently "init" and "run")
  * }
  *
  * The "bulk" execution implementation supports testing and other
- * paramters under attributes.system.exec.bulkexec, including:
+ * parameters under attributes.system.exec.bulkexec, including:
  *
  * {
  *   "mock_exception":s     - cancel job after a certain number of shells
@@ -93,16 +87,32 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
+#include <assert.h>
+#include <unistd.h>
+#include <signal.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libjob/job_hash.h"
+#include "src/common/libjob/idf58.h"
 #include "src/common/libeventlog/eventlog.h"
+#include "src/common/libeventlog/eventlogger.h"
 #include "src/common/libutil/fsd.h"
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/sigutil.h"
+#include "ccan/str/str.h"
+
 #include "job-exec.h"
+#include "checkpoint.h"
+#include "exec_config.h"
+
+static double kill_timeout;
+static int max_kill_count;
+static int term_signal;
+static int kill_signal;
 
-static double kill_timeout=5.0;
+#define DEBUG_FAIL_EXPIRATION 1
 
 extern struct exec_implementation testexec;
 extern struct exec_implementation bulkexec;
@@ -115,6 +125,8 @@ static struct exec_implementation * implementations[] = {
 
 struct job_exec_ctx {
     flux_t *              h;
+    int                   argc; /* needed for later reparse */
+    char **               argv; /* needed for later reparse */
     flux_msg_handler_t ** handlers;
     zhashx_t *            jobs;
 };
@@ -128,7 +140,11 @@ void jobinfo_decref (struct jobinfo *job)
 {
     if (job && (--job->refcount == 0)) {
         int saved_errno = errno;
+        idset_destroy (job->critical_ranks);
+        eventlogger_destroy (job->ev);
         flux_watcher_destroy (job->kill_timer);
+        flux_watcher_destroy (job->kill_shell_timer);
+        flux_watcher_destroy (job->expiration_timer);
         zhashx_delete (job->ctx->jobs, &job->id);
         if (job->impl && job->impl->exit)
             (*job->impl->exit) (job);
@@ -137,6 +153,7 @@ void jobinfo_decref (struct jobinfo *job)
         job->req = NULL;
         resource_set_destroy (job->R);
         json_decref (job->jobspec);
+        free (job->rootref);
         free (job);
         errno = saved_errno;
     }
@@ -149,6 +166,35 @@ static struct jobinfo * jobinfo_new (void)
     return job;
 }
 
+flux_future_t *jobinfo_shell_rpc_pack (struct jobinfo *job,
+                                       const char *topic,
+                                       const char *fmt,
+                                       ...)
+{
+    va_list ap;
+    char *shell_topic = NULL;
+    flux_future_t *f = NULL;
+    uint32_t rank;
+    char idbuf[21];
+
+    if (flux_job_id_encode (job->id, "f58plain", idbuf, sizeof (idbuf)) < 0
+        || asprintf (&shell_topic,
+                     "%ju-shell-%s.%s",
+                     (uintmax_t) job->userid,
+                     idbuf,
+                     topic) < 0
+        || ((rank = resource_set_nth_rank (job->R, 0)) == IDSET_INVALID_ID))
+        goto out;
+    va_start (ap, fmt);
+    f = flux_rpc_vpack (job->ctx->h, shell_topic, rank, 0, fmt, ap);
+    va_end (ap);
+    flux_future_aux_set (f, "jobinfo", job, (flux_free_f) jobinfo_decref);
+    jobinfo_incref (job);
+out:
+    ERRNO_SAFE_WRAP (free, shell_topic);
+    return f;
+}
+
 /*  Emit an event to the exec system eventlog and return a future from
  *   flux_kvs_commit().
  */
@@ -204,34 +250,15 @@ static flux_future_t * jobinfo_emit_event_pack (struct jobinfo *job,
     return f;
 }
 
-static void emit_event_continuation (flux_future_t *f, void *arg)
-{
-    struct jobinfo *job = arg;
-    if (flux_future_get (f, NULL) < 0)
-        flux_log_error (job->ctx->h, "%ju: emit_event", job->id);
-    flux_future_destroy (f);
-    jobinfo_decref (job);
-}
-
 static int jobinfo_emit_event_vpack_nowait (struct jobinfo *job,
                                             const char *name,
                                             const char *fmt,
                                             va_list ap)
 {
-    flux_future_t *f = jobinfo_emit_event_vpack (job, name, fmt, ap);
-    if (f == NULL)
-        return -1;
-    jobinfo_incref (job);
-    if (flux_future_then (f, -1., emit_event_continuation, job) < 0) {
-        flux_log_error (job->ctx->h, "jobinfo_emit_event");
-        goto error;
-    }
-    return 0;
-error:
-    jobinfo_decref (job);
-    flux_future_destroy (f);
-    return -1;
-
+    return eventlogger_append_vpack (job->ev,
+                                     0,
+                                     "exec.eventlog",
+                                     name, fmt, ap);
 }
 
 /*
@@ -251,9 +278,13 @@ int jobinfo_emit_event_pack_nowait (struct jobinfo *job,
     return rc;
 }
 
-static int jobid_respond_error (flux_t *h, flux_jobid_t id,
-                                const flux_msg_t *msg,
-                                int errnum, const char *text)
+static int jobid_exception (flux_t *h,
+                            flux_jobid_t id,
+                            const flux_msg_t *msg,
+                            const char *type,
+                            int severity,
+                            int errnum,
+                            const char *text)
 {
     char note [256];
     if (errnum)
@@ -263,19 +294,32 @@ static int jobid_respond_error (flux_t *h, flux_jobid_t id,
                                         strerror (errnum));
     else
         snprintf (note, sizeof (note), "%s", text ? text : "");
+
+    flux_log (h,
+              LOG_INFO,
+              "job-exception: id=%s: %s",
+              idf58 (id),
+              note);
     return flux_respond_pack (h, msg, "{s:I s:s s:{s:i s:s s:s}}",
                                       "id", id,
                                       "type", "exception",
                                       "data",
-                                      "severity", 0,
-                                      "type", "exec",
+                                      "severity", severity,
+                                      "type", type,
                                       "note", note);
 }
 
-static int jobinfo_respond_error (struct jobinfo *job, int errnum,
+static int jobinfo_respond_error (struct jobinfo *job,
+                                  int errnum,
                                   const char *msg)
 {
-    return jobid_respond_error (job->ctx->h, job->id, job->req, errnum, msg);
+    return jobid_exception (job->ctx->h,
+                            job->id,
+                            job->req,
+                            "exec",
+                            0,
+                            errnum,
+                            msg);
 }
 
 static int jobinfo_send_release (struct jobinfo *job,
@@ -284,67 +328,335 @@ static int jobinfo_send_release (struct jobinfo *job,
     int rc;
     flux_t *h = job->ctx->h;
     // XXX: idset ignored for now. Always release all resources
-    rc = flux_respond_pack (h, job->req, "{s:I s:s s{s:s s:b}}",
-                                         "id", job->id,
-                                         "type", "release",
-                                         "data", "ranks", "all",
-                                                 "final", true);
+    rc = flux_respond_pack (h,
+                            job->req,
+                            "{s:I s:s s{s:s s:b}}",
+                            "id", job->id,
+                            "type", "release",
+                            "data",
+                              "ranks", "all",
+                              "final", true);
     return rc;
 }
 
-static int jobinfo_respond (flux_t *h, struct jobinfo *job,
-                            const char *event, int status)
+static int jobinfo_respond (flux_t *h,
+                            struct jobinfo *job,
+                            const char *event)
 {
-    return flux_respond_pack (h, job->req, "{s:I s:s s:{}}",
-                                           "id", job->id,
-                                           "type", event,
-                                           "data");
+    return flux_respond_pack (h,
+                              job->req,
+                              "{s:I s:s s:{}}",
+                              "id", job->id,
+                              "type", event,
+                              "data");
 }
 
 static void jobinfo_complete (struct jobinfo *job, const struct idset *ranks)
 {
     flux_t *h = job->ctx->h;
     job->running = 0;
+
+    if (job->exception_in_progress && job->wait_status == 0)
+        job->wait_status = 1<<8;
+
     if (h && job->req) {
         jobinfo_emit_event_pack_nowait (job, "complete",
                                         "{ s:i }",
                                         "status", job->wait_status);
-        if (flux_respond_pack (h, job->req, "{s:I s:s s:{s:i}}",
-                                            "id", job->id,
-                                            "type", "finish",
-                                            "data",
-                                            "status", job->wait_status) < 0)
+        if (flux_respond_pack (h,
+                               job->req,
+                               "{s:I s:s s:{s:i}}",
+                               "id", job->id,
+                               "type", "finish",
+                               "data",
+                                 "status", job->wait_status) < 0)
             flux_log_error (h, "jobinfo_complete: flux_respond");
     }
 }
 
-void jobinfo_started (struct jobinfo *job, const char *fmt, ...)
+int jobinfo_drain_ranks (struct jobinfo *job,
+                         const char *ranks,
+                         const char *fmt,
+                         ...)
+{
+    va_list ap;
+    flux_future_t *f = NULL;
+    char reason[256];
+    int rc = -1;
+
+    /* vsnprintf(3) May truncate (unlikely), but that's ok 255 chars should
+     * get the gist across...
+     */
+    va_start (ap, fmt);
+    (void) vsnprintf (reason, sizeof (reason), fmt, ap);
+    va_end (ap);
+
+    if (!(f = flux_rpc_pack (job->h,
+                             "resource.drain",
+                             0,
+                             0,
+                             "{s:s s:s s:s}",
+                             "targets", ranks,
+                             "reason", reason,
+                             "mode", "update"))) {
+        flux_log (job->h,
+                  LOG_ERR,
+                  "Failed to drain broker ranks %s for job %s",
+                  ranks,
+                  idf58 (job->id));
+        goto error;
+    }
+    rc = 0;
+error:
+    flux_future_destroy (f);
+    return rc;
+}
+
+static int drain_active_ranks (struct jobinfo *job, struct idset *active_ranks)
+{
+    char *ranks = NULL;
+    int rc;
+
+    if (!(ranks = idset_encode (active_ranks, IDSET_FLAG_RANGE))) {
+        flux_log_error (job->h,
+                        "drain_active_ranks: Failed get rank string for job %s",
+                        idf58 (job->id));
+        return -1;
+    }
+    rc = jobinfo_drain_ranks (job,
+                              ranks,
+                              "unkillable processes for job %s",
+                              idf58 (job->id));
+    ERRNO_SAFE_WRAP (free, ranks);
+    return rc;
+}
+
+static void kill_shell_timer_cb (flux_reactor_t  *r,
+                                 flux_watcher_t *w,
+                                 int revents,
+                                 void *arg)
+{
+    struct jobinfo *job = arg;
+    struct idset *active_ranks;
+    int actual_kill_signal = kill_signal;
+
+    /* RFC 15 states that the IMP handles SIGUSR1 by sending SIGKILL to
+     * the entire cgroup.  Sending SIGKILL to the IMP is not productive.
+     */
+    if (job->multiuser)
+        actual_kill_signal = SIGUSR1;
+
+    flux_log (job->h,
+              LOG_DEBUG,
+              "Sending %s to %s for job %s",
+              sigutil_signame (actual_kill_signal),
+              job->multiuser ? "IMP" : "job shell",
+              idf58 (job->id));
+    (*job->impl->kill) (job, actual_kill_signal);
+    job->kill_shell_count++;
+
+    /* Since we've transitioned to killing the shell directly, stop the
+     * flux_job_kill(3) timer:
+     */
+    flux_watcher_stop (job->kill_timer);
+
+    /*  Reuse job->kill_timer to create an exponential backoff
+     */
+    if ((job->kill_timeout = job->kill_timeout * 2) > 300.)
+        job->kill_timeout = 300.;
+    flux_timer_watcher_reset (w, job->kill_timeout, 0.);
+    flux_watcher_start (w);
+
+    /* Check if we've exceeded the maximum number of kill attempts.
+     * Drain ranks that are still active if so. If the drain succeeds,
+     * then force remaining tasks to be complete, which will then cause
+     * this job object to be destroyed (which terminates any active kill
+     * timers).
+     *
+     * If the drain fails (unlikely), then the job stays active and we'll
+     * try to kill it again (and drain ranks) the next time the kill timer
+     * fires.
+     */
+    if (job->kill_shell_count >= max_kill_count
+        && job->impl->active_ranks
+        && (active_ranks = (*job->impl->active_ranks) (job))) {
+        flux_log (job->h,
+                  LOG_DEBUG,
+                  "job %s exceeded max kill count. Draining active ranks",
+                  idf58 (job->id));
+        if (drain_active_ranks (job, active_ranks) == 0) {
+            /* Force remaining tasks to be complete
+             */
+            jobinfo_tasks_complete (job, active_ranks, 1);
+        }
+        idset_destroy (active_ranks);
+    }
+}
+
+static void kill_timer_cb (flux_reactor_t *r,
+                           flux_watcher_t *w,
+                           int revents,
+                           void *arg)
+{
+    struct jobinfo *job = arg;
+    flux_future_t *f;
+    flux_log (job->h,
+              LOG_DEBUG,
+              "Sending %s to job %s",
+              sigutil_signame (kill_signal),
+              idf58 (job->id));
+    if (!(f = flux_job_kill (job->h, job->id, kill_signal))) {
+        flux_log_error (job->h,
+                        "flux_job_kill (%s, %s)",
+                        idf58 (job->id),
+                        sigutil_signame (kill_signal));
+        return;
+    }
+    job->kill_count++;
+    /* Open loop */
+    flux_future_destroy (f);
+}
+
+
+static void jobinfo_killtimer_start (struct jobinfo *job, double after)
+{
+    flux_reactor_t *r = flux_get_reactor (job->h);
+
+    /* Only start kill timer if not already running */
+    if (job->kill_timer == NULL) {
+        job->kill_timer = flux_timer_watcher_create (r,
+                                                     after,
+                                                     after,
+                                                     kill_timer_cb,
+                                                     job);
+        flux_watcher_start (job->kill_timer);
+    }
+    if (job->kill_shell_timer == NULL) {
+        job->kill_shell_timer = flux_timer_watcher_create (r,
+                                                           after*5,
+                                                           0.,
+                                                           kill_shell_timer_cb,
+                                                           job);
+        flux_watcher_start (job->kill_shell_timer);
+    }
+
+}
+
+static void timelimit_cb (flux_reactor_t *r,
+                          flux_watcher_t *w,
+                          int revents,
+                          void *arg)
+{
+    struct jobinfo *job = arg;
+
+    /*  Timelimit reached. Generate "timeout" exception and send SIGALRM.
+     *  Wait for a gracetime then forcibly terminate job.
+     */
+    if (jobid_exception (job->h,
+                         job->id,
+                         job->req,
+                         "timeout",
+                         0,
+                         0,
+                         "resource allocation expired") < 0)
+        flux_log_error (job->h,
+                        "failed to generate timeout exception for %s",
+                        idf58 (job->id));
+    (*job->impl->kill) (job, SIGALRM);
+    flux_watcher_stop (w);
+    job->exception_in_progress = 1;
+    jobinfo_killtimer_start (job, job->kill_timeout);
+}
+
+static int jobinfo_set_expiration (struct jobinfo *job)
+{
+    flux_watcher_t *w = NULL;
+    double now;
+    double expiration = resource_set_expiration (job->R);
+    double starttime = resource_set_starttime (job->R);
+    double offset;
+
+    if (expiration < 0.) {
+        jobinfo_fatal_error (job,
+                             EINVAL,
+                             "Invalid resource set expiration %.2f",
+                             expiration);
+        return -1;
+    }
+
+    flux_watcher_destroy (job->expiration_timer);
+    job->expiration_timer = NULL;
+
+    /* Timelimit disabled if expiration is set to 0.
+     */
+    if (expiration == 0.)
+        return 0;
+
+    /* N.B. Use of flux_reactor_time(3) here instead of flux_reactor_now(3)
+     *  is purposeful, Since this is used to find the time the job has
+     *  remaining, we should be as accurate as possible.
+     */
+    now = flux_reactor_time ();
+    if (job->t0 == 0.)
+       job->t0 = now;
+
+    /* Adjust expiration time based on delay between when scheduler
+     *  created R and when we received this job. O/w jobs may be
+     *  terminated due to timeouts prematurely when the system
+     *  is very busy, which can cause long delays between alloc and
+     *  start events.
+     */
+    if (starttime > 0.)
+        expiration += job->t0 - starttime;
+
+    offset = expiration - now;
+    if (offset < 0.) {
+        /*  Expiration has already passed. Just set offset to 0. so that
+         *  the expiration timer will fire immediately.
+         */
+        offset = 0.;
+    }
+    if (!(w = flux_timer_watcher_create (flux_get_reactor(job->h),
+                                         offset,
+                                         0.,
+                                         timelimit_cb,
+                                         job))) {
+        jobinfo_fatal_error (job, errno, "unable to start expiration timer");
+        return -1;
+    }
+    flux_watcher_start (w);
+    job->expiration_timer = w;
+    return 0;
+}
+
+
+void jobinfo_started (struct jobinfo *job)
 {
     flux_t *h = job->ctx->h;
     if (h && job->req) {
-        va_list ap;
-        va_start (ap, fmt);
+        if (jobinfo_set_expiration (job) < 0)
+            flux_log_error (h,
+                            "failed to set expiration for %s",
+                            idf58 (job->id));
         job->running = 1;
-        va_end (ap);
-        if (jobinfo_respond (h, job, "start", 0) < 0)
+        if (jobinfo_respond (h, job, "start") < 0)
             flux_log_error (h, "jobinfo_started: flux_respond");
     }
 }
 
-static void kill_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                           int revents, void *arg)
+void jobinfo_reattached (struct jobinfo *job)
 {
-    struct jobinfo *job = arg;
-    flux_log (job->h,
-              LOG_DEBUG,
-              "Sending SIGKILL to job %ju",
-              (uintmax_t) job->id);
-    (*job->impl->kill) (job, SIGKILL);
+    flux_t *h = job->ctx->h;
+    if (h && job->req) {
+        job->running = 1;
+        if (jobinfo_respond (h, job, "reattached") < 0)
+            flux_log_error (h, "jobinfo_reattach: flux_respond");
+    }
 }
 
 /*  Cancel any pending shells to execute with implementations cancel
- *   method, send SIGTERM to executing shells to notify them to terminate,
- *   schedule SIGKILL to be sent after kill_timeout seconds.
+ *   method, send term_signal to executing shells to notify them to terminate,
+ *   schedule kill_signal to be sent after kill_timeout seconds.
  */
 static void jobinfo_cancel (struct jobinfo *job)
 {
@@ -355,17 +667,16 @@ static void jobinfo_cancel (struct jobinfo *job)
     if (job->impl->cancel)
         (*job->impl->cancel) (job);
 
-    (*job->impl->kill) (job, SIGTERM);
-    job->kill_timer = flux_timer_watcher_create (flux_get_reactor (job->h),
-                                                 job->kill_timeout, 0.,
-                                                 kill_timer_cb, job);
-    flux_watcher_start (job->kill_timer);
+    (*job->impl->kill) (job, term_signal);
+    jobinfo_killtimer_start (job, job->kill_timeout);
 }
 
 static int jobinfo_finalize (struct jobinfo *job);
 
-static void jobinfo_fatal_verror (struct jobinfo *job, int errnum,
-                                  const char *fmt, va_list ap)
+static void jobinfo_fatal_verror (struct jobinfo *job,
+                                  int errnum,
+                                  const char *fmt,
+                                  va_list ap)
 {
     int n;
     char msg [256];
@@ -378,7 +689,6 @@ static void jobinfo_fatal_verror (struct jobinfo *job, int errnum,
         msg [msglen-2] = '+';
         msg [msglen-1] = '\0';
     }
-    jobinfo_emit_event_pack_nowait (job, "exception", "{ s:s }", "note", msg);
     /* If exception_in_progress set, then no need to respond with another
      *  exception back to job manager. O/w, DO respond to job-manager
      *  and set exception-in-progress.
@@ -400,7 +710,8 @@ static void jobinfo_fatal_verror (struct jobinfo *job, int errnum,
     }
 }
 
-void jobinfo_fatal_error (struct jobinfo *job, int errnum,
+void jobinfo_fatal_error (struct jobinfo *job,
+                          int errnum,
                           const char *fmt, ...)
 {
     flux_t *h = job->ctx->h;
@@ -414,6 +725,68 @@ void jobinfo_fatal_error (struct jobinfo *job, int errnum,
     errno = saved_errno;
 }
 
+static void jobinfo_vraise (struct jobinfo *job,
+                            const char *type,
+                            int severity,
+                            const char *fmt,
+                            va_list ap)
+{
+    int n;
+    char msg [256];
+    int msglen = sizeof (msg);
+    flux_t *h = job->ctx->h;
+
+    if ((n = vsnprintf (msg, msglen, fmt, ap)) < 0)
+        strcpy (msg, "vsnprintf error");
+    else if (n >= msglen) {
+        msg [msglen-2] = '+';
+        msg [msglen-1] = '\0';
+    }
+    if (jobid_exception (h, job->id, job->req, type, severity, 0, msg) < 0)
+        flux_log_error (h,
+                        "error raising exception type=%s severity=%d: %s",
+                        type,
+                        severity,
+                        msg);
+}
+
+void jobinfo_raise (struct jobinfo *job,
+                    const char *type,
+                    int severity,
+                    const char *fmt, ...)
+{
+    va_list ap;
+    va_start (ap, fmt);
+    jobinfo_vraise (job, type, severity, fmt, ap);
+    va_end (ap);
+}
+
+void jobinfo_log_output (struct jobinfo *job,
+                         int rank,
+                         const char *component,
+                         const char *stream,
+                         const char *data,
+                         int len)
+{
+    char buf[16];
+    if (len == 0 || !data || !stream)
+        return;
+    if (snprintf (buf, sizeof (buf), "%d", rank) >= sizeof (buf))
+        flux_log_error (job->h, "jobinfo_log_output: snprintf");
+    if (eventlogger_append_pack (job->ev, 0,
+                                 "exec.eventlog",
+                                 "log",
+                                 "{ s:s, s:s s:s s:s# }",
+                                 "component", component,
+                                 "stream", stream,
+                                 "rank", buf,
+                                 "data", data, len) < 0)
+        flux_log_error (job->h,
+                        "eventlog_append failed: %s: message=%s",
+                        idf58 (job->id),
+                        data);
+}
+
 static void namespace_delete (flux_future_t *f, void *arg)
 {
     struct jobinfo *job = arg;
@@ -459,111 +832,36 @@ static flux_future_t * namespace_move (struct jobinfo *job)
 {
     flux_t *h = job->ctx->h;
     flux_future_t *f = NULL;
-    flux_future_t *fnext = NULL;
+    flux_future_t *f1 = NULL;
+    flux_future_t *f2 = NULL;
 
-    if (!(f = jobinfo_emit_event_pack (job, "done", NULL))) {
+    if (jobinfo_emit_event_pack_nowait (job, "done", NULL) < 0)
+        flux_log_error (h, "emit_event");
+    /*
+     *  Ensure the final eventlog entry ("done", from above), is committed
+     *   to then eventlog before performing the next steps. This ensures
+     *   the eventlog is quiesced before the namespace is moved and becomes
+     *   read-only.
+     */
+    if (!(f = eventlogger_commit (job->ev))) {
         flux_log_error (h, "namespace_move: jobinfo_emit_event");
         goto error;
     }
-    if (   !(fnext = flux_future_and_then (f, namespace_copy, job))
-        || !(fnext = flux_future_and_then (f=fnext, namespace_delete, job))) {
+    if (!(f1 = flux_future_and_then (f, namespace_copy, job))
+        || !(f1 = flux_future_or_then (f, namespace_copy, job))
+        || !(f2 = flux_future_and_then (f1, namespace_delete, job))
+        || !(f2 = flux_future_or_then (f1, namespace_delete, job))) {
         flux_log_error (h, "namespace_move: flux_future_and_then");
         goto error;
     }
-    return fnext;
+    return f2;
 error:
     flux_future_destroy (f);
-    flux_future_destroy (fnext);
+    flux_future_destroy (f1);
+    flux_future_destroy (f2);
     return NULL;
 }
 
-static void cleanup_complete_cb (flux_future_t *f, void *arg)
-{
-    struct jobinfo *job = arg;
-    if (flux_future_get (f, NULL) < 0)
-        flux_log_error (job->h, "cleanup_complete_cb: event_pack");
-    flux_future_destroy (f);
-
-    /* XXX: when cleanup ranks are tracked, only finalize once all
-     *  involved ranks have completed cleanup. For now though, only
-     *  one cleanup_complete call is expected.
-     */
-    if (jobinfo_finalize (job) < 0) {
-        flux_log_error (job->h, "cleanup_complete_cb: jobinfo_finalize");
-        jobinfo_decref (job);
-    }
-}
-
-/* Notify job-exec that any "cleanup" tasks including epilog have
- *  completed on ranks with return code `rc`.
- *
- * XXX: ranks ignored for now
- */
-void jobinfo_cleanup_complete (struct jobinfo *job,
-                               const struct idset *ranks,
-                               int rc)
-{
-    flux_future_t *f = NULL;
-
-    /*  XXX: It isn't clear what to do if a cleanup task fails.
-     *   For now, log the return code from the cleanup composite
-     *   future and errno if rc < 0 for informational purposes,
-     *   but do not generate an exception.
-     */
-    if (rc < 0)
-        f = jobinfo_emit_event_pack (job, "cleanup.finish",
-                                     "{ s:s s:i s:s }",
-                                     "ranks", "all",
-                                     "rc", rc,
-                                     "note", strerror (errno));
-    else
-        f = jobinfo_emit_event_pack (job, "cleanup.finish",
-                                     "{ s:s s:i }",
-                                     "ranks", "all",
-                                     "rc", rc);
-   if (flux_future_then (f, -1., cleanup_complete_cb, job) < 0) {
-        jobinfo_respond_error (job, errno, "cleanup complete error");
-        flux_future_destroy (f);
-    }
-}
-
-static void jobinfo_start_cleanup_cb (flux_future_t *f, void *arg)
-{
-    struct jobinfo *job = arg;
-
-    /*  Log error if cleanup.start event failed */
-    if (flux_future_get (f, NULL) < 0)
-        flux_log_error (job->h, "jobinfo_emit_event (cleanup.start)");
-    flux_future_destroy (f);
-
-    if ((*job->impl->cleanup) (job, NULL) < 0)
-        flux_log_error (job->h, "%s: cleanup()", job->impl->name);
-}
-
-/*  Initiate cleanup on ranks in idset, if necessary */
-static void jobinfo_start_cleanup (struct jobinfo *job,
-                                   const struct idset *idset)
-{
-    flux_future_t *f = NULL;
-
-    /* XXX: idset ignored for now */
-
-    if (!(f = jobinfo_emit_event_pack (job, "cleanup.start",
-                                            "{ s:s }",
-                                            "ranks", "all"))) {
-        flux_log_error (job->h, "jobinfo_emit_event_pack");
-        goto error;
-    }
-    if (flux_future_then (f, -1., jobinfo_start_cleanup_cb, job) < 0) {
-        flux_log_error (job->h, "flux_future_then");
-        goto error;
-    }
-    return;
-error:
-    jobinfo_respond_error (job, errno, "cleanup start error");
-    flux_future_destroy (f);
-}
-
 /* Notify job-exec that shells on ranks are complete.
  * XXX: currently assumes ranks equivalent to "all"
  */
@@ -580,8 +878,10 @@ void jobinfo_tasks_complete (struct jobinfo *job,
      */
     jobinfo_complete (job, ranks);
 
-    /* Start cleanup tasks on completed ranks */
-    jobinfo_start_cleanup (job, ranks);
+    if (jobinfo_finalize (job) < 0) {
+        flux_log_error (job->h, "tasks_complete: jobinfo_finalize");
+        jobinfo_decref (job);
+    }
 }
 
 
@@ -607,8 +907,7 @@ static void namespace_move_cb (flux_future_t *f, void *arg)
 /*
  *  All job shells have exited or we've hit an exception:
  *   start finalization steps:
- *   1. Ensure all cleanup tasks have completed
- *   2. Move namespace into primary namespace, emitting final event to log
+ *   1. Move namespace into primary namespace, emitting final event to log
  */
 static int jobinfo_finalize (struct jobinfo *job)
 {
@@ -635,7 +934,10 @@ static int jobinfo_finalize (struct jobinfo *job)
 
 static int jobinfo_start_execution (struct jobinfo *job)
 {
-    jobinfo_emit_event_pack_nowait (job, "starting", NULL);
+    if (job->reattach)
+        jobinfo_emit_event_pack_nowait (job, "re-starting", NULL);
+    else
+        jobinfo_emit_event_pack_nowait (job, "starting", NULL);
     /* Set started flag before calling 'start' method because we want to
      *  be sure to clean up properly if an exception occurs
      */
@@ -647,31 +949,6 @@ static int jobinfo_start_execution (struct jobinfo *job)
     return 0;
 }
 
-/*  Lookup key 'key' under jobid 'id' kvs dir:
- */
-static flux_future_t *flux_jobid_kvs_lookup (flux_t *h, flux_jobid_t id,
-                                             int flags, const char *key)
-{
-    char path [256];
-    if (flux_job_kvs_key (path, sizeof (path), id, key) < 0)
-        return NULL;
-    return flux_kvs_lookup (h, NULL, flags, path);
-}
-
-/*
- *  Call lookup_get on a child named 'name' of the composite future 'f'
- */
-static const char * jobinfo_kvs_lookup_get (flux_future_t *f, const char *name)
-{
-    const char *result;
-    flux_future_t *child = flux_future_get_child (f, name);
-    if (child == NULL)
-        return NULL;
-    if (flux_kvs_lookup_get (child, &result) < 0)
-        return NULL;
-    return result;
-}
-
 static int jobinfo_load_implementation (struct jobinfo *job)
 {
     int i = 0;
@@ -695,44 +972,23 @@ static int jobinfo_load_implementation (struct jobinfo *job)
     return -1;
 }
 
-/*  Completion for jobinfo_initialize(), finish init of jobinfo using
+/*  Completion for jobinfo_start_init (), finish init of jobinfo using
  *   data fetched from KVS
  */
 static void jobinfo_start_continue (flux_future_t *f, void *arg)
 {
-    json_error_t error;
-    const char *R = NULL;
-    const char *jobspec = NULL;
     struct jobinfo *job = arg;
 
     if (flux_future_get (flux_future_get_child (f, "ns"), NULL) < 0) {
         jobinfo_fatal_error (job, errno, "failed to create guest ns");
         goto done;
     }
-    job->has_namespace = 1;
 
     /*  If an exception was received during startup, no need to continue
      *   with startup
      */
     if (job->exception_in_progress)
         goto done;
-
-    if (!(jobspec = jobinfo_kvs_lookup_get (f, "jobspec"))) {
-        jobinfo_fatal_error (job, errno, "unable to fetch jobspec");
-        goto done;
-    }
-    if (!(R = jobinfo_kvs_lookup_get (f, "R"))) {
-        jobinfo_fatal_error (job, errno, "job does not have allocation");
-        goto done;
-    }
-    if (!(job->R = resource_set_create (R, &error))) {
-        jobinfo_fatal_error (job, errno, "reading R: %s", error.text);
-        goto done;
-    }
-    if (!(job->jobspec = json_loads (jobspec, 0, &error))) {
-        jobinfo_fatal_error (job, errno, "reading jobspec: %s", error.text);
-        goto done;
-    }
     if (jobinfo_load_implementation (job) < 0) {
         jobinfo_fatal_error (job, errno, "failed to initialize implementation");
         goto done;
@@ -788,7 +1044,9 @@ static void namespace_link (flux_future_t *fprev, void *arg)
         goto error;
     }
     flux_future_set_flux (cf, h);
-    if (!(f = jobinfo_emit_event_pack (job, "init", NULL))
+    if (!(f = jobinfo_emit_event_pack (job,
+                                       job->reattach ? "reattach" : "init",
+                                       NULL))
         || flux_future_push (cf, "emit event", f) < 0)
         goto error;
 
@@ -812,8 +1070,22 @@ static flux_future_t *ns_create_and_link (flux_t *h,
     flux_future_t *f = NULL;
     flux_future_t *f2 = NULL;
 
-    if (!(f = flux_kvs_namespace_create (h, job->ns, job->userid, flags))
-        || !(f2 = flux_future_and_then (f, namespace_link, job))) {
+    if (job->reattach && job->rootref)
+        f = flux_kvs_namespace_create_with (h,
+                                            job->ns,
+                                            job->rootref,
+                                            job->userid,
+                                            flags);
+    else
+        f = flux_kvs_namespace_create (h, job->ns, job->userid, flags);
+
+    /*  Set job->has_namespace flag immediately after sending the namespace
+     *  create RPC. This avoids the potential to leave orphaned namespaces
+     *  if the job is canceled before the response is received.
+     */
+    job->has_namespace = 1;
+
+    if (!f || !(f2 = flux_future_and_then (f, namespace_link, job))) {
         flux_log_error (h, "namespace_move: flux_future_and_then");
         flux_future_destroy (f);
         return NULL;
@@ -821,6 +1093,54 @@ static flux_future_t *ns_create_and_link (flux_t *h,
     return f2;
 }
 
+static void get_rootref_cb (flux_future_t *fprev, void *arg)
+{
+    int saved_errno;
+    flux_t *h = flux_future_get_flux (fprev);
+    struct jobinfo *job = arg;
+    flux_future_t *f = NULL;
+
+    if (!(job->rootref = checkpoint_find_rootref (fprev,
+                                                  job->id,
+                                                  job->userid)))
+        flux_log (job->h,
+                  LOG_DEBUG,
+                  "checkpoint rootref not found: %s",
+                  idf58 (job->id));
+
+    /* if rootref not found, still create namespace */
+    if (!(f = ns_create_and_link (h, job, 0)))
+        goto error;
+
+    flux_future_continue (fprev, f);
+    flux_future_destroy (fprev);
+    return;
+error:
+    saved_errno = errno;
+    flux_future_destroy (f);
+    flux_future_continue_error (fprev, saved_errno, NULL);
+    flux_future_destroy (fprev);
+}
+
+static flux_future_t *ns_get_rootref (flux_t *h,
+                                      struct jobinfo *job,
+                                      int flags)
+{
+    flux_future_t *f = NULL;
+    flux_future_t *f2 = NULL;
+
+    if (!(f = checkpoint_get_rootrefs (h))) {
+        flux_log_error (h, "ns_get_rootref: checkpoint_get_rootrefs");
+        return NULL;
+    }
+    if (!f || !(f2 = flux_future_and_then (f, get_rootref_cb, job))) {
+        flux_log_error (h, "ns_get_rootref: flux_future_and_then");
+        flux_future_destroy (f);
+        return NULL;
+    }
+    return f2;
+}
+
 /*  Asynchronously fetch job data from KVS and create namespace.
  */
 static flux_future_t *jobinfo_start_init (struct jobinfo *job)
@@ -830,31 +1150,58 @@ static flux_future_t *jobinfo_start_init (struct jobinfo *job)
     flux_future_t *f = flux_future_wait_all_create ();
     flux_future_set_flux (f, job->ctx->h);
 
-    if (!(f_kvs = flux_jobid_kvs_lookup (h, job->id, 0, "R"))
-        || flux_future_push (f, "R", f_kvs) < 0)
-        goto err;
-    if (!(f_kvs = flux_jobid_kvs_lookup (h, job->id, 0, "jobspec"))
-        || flux_future_push (f, "jobspec", f_kvs) < 0)
-        goto err;
-    if (!(f_kvs = ns_create_and_link (h, job, 0))
-        || flux_future_push (f, "ns", f_kvs))
+    if (job->reattach)
+        f_kvs = ns_get_rootref (h, job, 0);
+    else
+        f_kvs = ns_create_and_link (h, job, 0);
+
+    if (flux_future_push (f, "ns", f_kvs) < 0)
         goto err;
 
     return f;
 err:
     flux_log_error (job->ctx->h, "jobinfo_kvs_lookup/namespace_create");
+    flux_future_destroy (f_kvs);
     flux_future_destroy (f);
     return NULL;
 }
 
+static void evlog_err (struct eventlogger *ev, void *arg, int err, json_t *e)
+{
+    struct jobinfo *job = arg;
+    char *s = json_dumps (e, JSON_COMPACT);
+    flux_log_error (job->h,
+                    "eventlog error: %s: entry=%s",
+                    idf58 (job->id),
+                    s);
+    free (s);
+}
+
+static void evlog_busy (struct eventlogger *ev, void *arg)
+{
+    jobinfo_incref ((struct jobinfo *) arg);
+}
+
+static void evlog_idle (struct eventlogger *ev, void *arg)
+{
+    jobinfo_decref ((struct jobinfo *) arg);
+}
+
 static int job_start (struct job_exec_ctx *ctx, const flux_msg_t *msg)
 {
+    struct eventlogger_ops ev_ops = {
+        .err = evlog_err,
+        .busy = evlog_busy,
+        .idle = evlog_idle
+    };
     flux_future_t *f = NULL;
     struct jobinfo *job;
+    json_t *R;
 
     if (!(job = jobinfo_new ()))
         return -1;
 
+
     /* Copy flux handle for each job to allow implementation access.
      * (This could also be done with an accessor, but choose the simpler
      *  approach for now)
@@ -866,13 +1213,35 @@ static int job_start (struct job_exec_ctx *ctx, const flux_msg_t *msg)
 
     job->ctx = ctx;
 
-    if (flux_request_unpack (job->req, NULL, "{s:I, s:i}",
-                                             "id", &job->id,
-                                             "userid", &job->userid) < 0) {
+    if (flux_request_unpack (job->req,
+                             NULL,
+                             "{s:I s:i s:O s:b s:o}",
+                             "id", &job->id,
+                             "userid", &job->userid,
+                             "jobspec", &job->jobspec,
+                             "reattach", &job->reattach,
+                             "R", &R) < 0) {
         flux_log_error (ctx->h, "start: flux_request_unpack");
         jobinfo_decref (job);
         return -1;
     }
+    json_error_t error;
+    if (!(job->R = resource_set_create_fromjson (R, &error))) {
+        flux_log (ctx->h, LOG_ERR, "reading R: %s", error.text);
+        jobinfo_decref (job);
+        return -1;
+    }
+    size_t size = idset_count (resource_set_ranks (job->R));
+    if (!(job->critical_ranks = idset_create (0, IDSET_FLAG_AUTOGROW))
+        || idset_range_set (job->critical_ranks, 0, size - 1) < 0) {
+        flux_log_error (ctx->h, "initializing critical ranks");
+        jobinfo_decref (job);
+        return -1;
+    }
+
+    if (job->userid != getuid ())
+        job->multiuser = 1;
+
     /*  Take a reference until initialization complete in case an
      *   exception is generated during this phase
      */
@@ -881,12 +1250,21 @@ static int job_start (struct job_exec_ctx *ctx, const flux_msg_t *msg)
     if (flux_job_kvs_namespace (job->ns, sizeof (job->ns), job->id) < 0) {
         jobinfo_fatal_error (job, errno, "failed to create ns name for job");
         flux_log_error (ctx->h, "job_ns_create");
-        return -1;
+        goto error;
     }
+    job->ev = eventlogger_create (job->h, 0.01, &ev_ops, job);
+    if (!job->ev || eventlogger_setns (job->ev, job->ns) < 0) {
+        flux_log_error (job->h,
+                        "eventlogger_create/setns for job %s failed",
+                        idf58 (job->id));
+        goto error;
+    }
+
     if (zhashx_insert (ctx->jobs, &job->id, job) < 0) {
+        errno = EEXIST;
         flux_log_error (ctx->h, "zhashx_insert");
         jobinfo_fatal_error (job, errno, "failed to hash job");
-        return -1;
+        goto error;
     }
     if (!(f = jobinfo_start_init (job))) {
         flux_log_error (ctx->h, "start: jobinfo_kvs_lookup");
@@ -899,11 +1277,14 @@ static int job_start (struct job_exec_ctx *ctx, const flux_msg_t *msg)
     return 0;
 error:
     jobinfo_fatal_error (job, errno, "job start failure");
+    jobinfo_decref (job);
     return -1;
 }
 
-static void start_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+static void start_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     struct job_exec_ctx *ctx = arg;
 
@@ -917,8 +1298,10 @@ static void start_cb (flux_t *h, flux_msg_handler_t *mh,
     }
 }
 
-static void exception_cb (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg)
+static void exception_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
 {
     struct job_exec_ctx *ctx = arg;
     flux_jobid_t id;
@@ -926,23 +1309,115 @@ static void exception_cb (flux_t *h, flux_msg_handler_t *mh,
     const char *type = NULL;
     struct jobinfo *job = NULL;
 
-    if (flux_event_unpack (msg, NULL, "{s:I s:s s:i}",
-                                      "id", &id,
-                                      "type", &type,
-                                      "severity", &severity) < 0) {
+    if (flux_event_unpack (msg,
+                           NULL,
+                           "{s:I s:s s:i}",
+                           "id", &id,
+                           "type", &type,
+                           "severity", &severity) < 0) {
         flux_log_error (h, "job-exception event");
         return;
     }
     if (severity == 0
         && (job = zhashx_lookup (ctx->jobs, &id))
-        && (!job->finalizing)
-        && (!job->exception_in_progress)) {
+        && (!job->finalizing)) {
+        if (job->exception_in_progress) {
+            /*  Resend term_signal even if exception in progress */
+            (*job->impl->kill) (job, term_signal);
+            return;
+        }
+        /*  !job->exception_in_progress:
+         *
+         *  Set job->exception_in_progress so that jobinfo_fatal_error()
+         *   doesn't dump a duplicate exception into the eventlog.
+         */
         job->exception_in_progress = 1;
-        flux_log (h, LOG_DEBUG, "exec aborted: id=%ld", id);
+        flux_log (h, LOG_DEBUG, "exec aborted: id=%s", idf58 (id));
         jobinfo_fatal_error (job, 0, "aborted due to exception type=%s", type);
     }
 }
 
+static void critical_ranks_cb (flux_t *h,
+                               flux_msg_handler_t *mh,
+                               const flux_msg_t *msg,
+                               void *arg)
+{
+    struct job_exec_ctx *ctx = arg;
+    flux_jobid_t id;
+    const char *ranks;
+    struct idset *idset;
+    struct jobinfo *job;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:s}",
+                             "id", &id,
+                             "ranks", &ranks) < 0)
+        goto error;
+
+    if (!(job = zhashx_lookup (ctx->jobs, &id))) {
+        errno = ENOENT;
+        goto error;
+    }
+    if (flux_msg_authorize (msg, job->userid) < 0
+        || !(idset = idset_decode (ranks)))
+        goto error;
+
+    idset_destroy (job->critical_ranks);
+    job->critical_ranks = idset;
+
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
+
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+static void expiration_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct job_exec_ctx *ctx = arg;
+    flux_jobid_t id;
+    double expiration;
+    struct jobinfo *job;
+
+    /*  Respond with error if module debug flag set for testing purposes
+     */
+    if (flux_module_debug_test (h, DEBUG_FAIL_EXPIRATION, false)) {
+        errno = EINVAL;
+        goto error;
+    }
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:F}",
+                             "id", &id,
+                             "expiration", &expiration) < 0)
+        goto error;
+
+    if (!(job = zhashx_lookup (ctx->jobs, &id))) {
+        errno = ENOENT;
+        goto error;
+    }
+    resource_set_update_expiration (job->R, expiration);
+    if (jobinfo_set_expiration (job) < 0)
+        goto error;
+    flux_log (h,
+              LOG_DEBUG,
+              "updated expiration of %s to %.2f",
+              idf58 (job->id),
+              resource_set_expiration (job->R));
+    if (flux_respond (ctx->h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to expiration update RPC");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "expiration_cb: flux_respond_error");
+}
+
 static void job_exec_ctx_destroy (struct job_exec_ctx *ctx)
 {
     if (ctx == NULL)
@@ -952,12 +1427,16 @@ static void job_exec_ctx_destroy (struct job_exec_ctx *ctx)
     free (ctx);
 }
 
-static struct job_exec_ctx * job_exec_ctx_create (flux_t *h)
+static struct job_exec_ctx * job_exec_ctx_create (flux_t *h,
+                                                  int argc,
+                                                  char **argv)
 {
     struct job_exec_ctx *ctx = calloc (1, sizeof (*ctx));
     if (ctx == NULL)
         return NULL;
     ctx->h = h;
+    ctx->argc = argc;
+    ctx->argv = argv;
     if (!(ctx->jobs = job_hash_create ())) {
         ERRNO_SAFE_WRAP (free, ctx);
         return NULL;
@@ -982,48 +1461,77 @@ static int exec_hello (flux_t *h, const char *service)
     return rc;
 }
 
-/*  Initialize job-exec module from defaults, config, cmdline,
- *   in that order. Currently only the kill-timeout is
- *   set here.
+/*  Initialize job-exec module global config from defaults, config, cmdline,
+ *   in that order.
  */
-static int job_exec_initialize (flux_t *h, int argc, char **argv)
+static int job_exec_set_config_globals (flux_t *h,
+                                        const flux_conf_t *conf,
+                                        int argc,
+                                        char **argv,
+                                        flux_error_t *errp)
 {
-    const flux_conf_t *conf;
-    flux_conf_error_t err;
     const char *kto = NULL;
-
-    if (!(conf = flux_get_conf (h, &err))) {
-        flux_log (h, LOG_ERR,
-                  "config file error: %s:%d: %s",
-                   err.filename,
-                   err.lineno,
-                   err.errbuf);
-        return -1;
-    }
+    const char *tsignal = NULL;
+    const char *ksignal = NULL;
+    flux_error_t error;
+
+    /* Per trws comment in 97421e88987535260b10d6a19551cea625f26ce4
+     *
+     * The musl libc loader doesn't actually unload objects on
+     * dlclose, so a subsequent dlopen doesn't re-clear globals and
+     * similar.
+     *
+     * So we must re-initialize globals everytime we reload the module.
+     */
+    kill_timeout = 5.0;
+    max_kill_count = 8;
+    term_signal = SIGTERM;
+    kill_signal = SIGKILL;
 
     if (flux_conf_unpack (conf,
-                          &err,
-                          "{s?{s?s}}",
+                          &error,
+                          "{s?{s?s s?s s?s s?i}}",
                           "exec",
-                            "kill-timeout", &kto) < 0) {
-        flux_log (h, LOG_ERR,
-                  "error reading config value exec.kill-timeout: %s",
-                  err.errbuf);
-        return -1;
-    }
+                            "kill-timeout", &kto,
+                            "term-signal", &tsignal,
+                            "kill-signal", &ksignal,
+                            "max-kill-count", &max_kill_count) < 0)
+        return errprintf (errp,
+                          "Error reading [exec] table: %s",
+                          error.text);
+
     /* Override via commandline */
     for (int i = 0; i < argc; i++) {
-        if (strncmp (argv[i], "kill-timeout=", 13) == 0)
+        if (strstarts (argv[i], "kill-timeout="))
             kto = argv[i] + 13;
+        else if (strstarts (argv[i], "kill-signal="))
+            ksignal = argv[i] + 12;
+        else if (strstarts (argv[i], "term-signal="))
+            tsignal = argv[i] + 12;
+        else if (strstarts (argv[i], "max-kill-count="))
+            max_kill_count = atoi(argv[i] + 15);
     }
 
     if (kto) {
         if (fsd_parse_duration (kto, &kill_timeout) < 0) {
-            flux_log_error (h, "invalid kill-timeout: %s", kto);
+            errprintf (errp, "invalid kill-timeout: %s", kto);
+            errno = EINVAL;
+            return -1;
+        }
+    }
+    if (ksignal) {
+        if ((kill_signal = sigutil_signum (ksignal)) < 0) {
+            errprintf (errp, "invalid kill-signal: %s", ksignal);
+            errno = EINVAL;
+            return -1;
+        }
+    }
+    if (tsignal) {
+        if ((term_signal = sigutil_signum (tsignal)) < 0) {
+            errprintf (errp, "invalid term-signal: %s", tsignal);
             errno = EINVAL;
             return -1;
         }
-        flux_log (h, LOG_INFO, "using kill-timeout of %.4gs", kill_timeout);
     }
     return 0;
 }
@@ -1031,18 +1539,235 @@ static int job_exec_initialize (flux_t *h, int argc, char **argv)
 static int configure_implementations (flux_t *h, int argc, char **argv)
 {
     struct exec_implementation *impl;
+    const flux_conf_t *conf;
+    flux_error_t err;
     int i = 0;
+    if (!(conf = flux_get_conf (h))) {
+        flux_log_error (h, "error retrieving flux conf");
+        return -1;
+    }
     while ((impl = implementations[i]) && impl->name) {
-        if (impl->config && (*impl->config) (h, argc, argv) < 0)
+        if (impl->config && (*impl->config) (h, conf, argc, argv, &err) < 0) {
+            flux_log (h, LOG_ERR, "%s", err.text);
             return -1;
+        }
         i++;
     }
     return 0;
 }
 
+static int remove_running_ns (struct job_exec_ctx *ctx)
+{
+    struct jobinfo *job = zhashx_first (ctx->jobs);
+    flux_future_t *fall = NULL;
+    flux_future_t *f = NULL;
+    int rv = -1;
+
+    while (job) {
+        if (job->running) {
+            if (!fall) {
+                if (!(fall = flux_future_wait_all_create ()))
+                    goto cleanup;
+                flux_future_set_flux (fall, ctx->h);
+            }
+            if (!(f = flux_kvs_namespace_remove (ctx->h, job->ns)))
+                goto cleanup;
+            if (flux_future_push (fall, job->ns, f) < 0)
+                goto cleanup;
+            f = NULL;
+        }
+        job = zhashx_next (ctx->jobs);
+    }
+
+    if (fall) {
+        if (flux_future_wait_for (fall, -1.) < 0)
+            goto cleanup;
+    }
+
+    rv = 0;
+cleanup:
+    flux_future_destroy (f);
+    flux_future_destroy (fall);
+    return rv;
+}
+
+static int unload_implementations (struct job_exec_ctx *ctx)
+{
+    struct exec_implementation *impl;
+    int i = 0;
+    if (ctx && ctx->jobs) {
+        checkpoint_running (ctx->h, ctx->jobs);
+        if (remove_running_ns (ctx) < 0)
+            flux_log_error (ctx->h, "failed to remove guest namespaces");
+    }
+    while ((impl = implementations[i]) && impl->name) {
+        if (impl->unload)
+             (*impl->unload) ();
+        i++;
+    }
+    return 0;
+}
+
+static json_t *running_job_stats (struct job_exec_ctx *ctx)
+{
+    struct jobinfo *job = zhashx_first (ctx->jobs);
+    json_t *o = NULL;
+
+    if (!(o = json_object ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    while (job) {
+        json_t *entry;
+        char *critical_ranks;
+        json_t *impl_stats = NULL;
+
+        if (!(critical_ranks = idset_encode (job->critical_ranks,
+                                             IDSET_FLAG_RANGE)))
+            goto error;
+
+        entry = json_pack ("{s:s s:s s:s s:i s:i s:i s:i s:i s:i s:f s:i s:i}",
+                           "implementation",
+                           job->impl ? job->impl->name : "none",
+                           "ns", job->ns,
+                           "critical_ranks", critical_ranks,
+                           "multiuser", job->multiuser,
+                           "has_namespace", job->has_namespace,
+                           "exception_in_progress", job->exception_in_progress,
+                           "started", job->started,
+                           "running", job->running,
+                           "finalizing", job->finalizing,
+                           "kill_timeout", job->kill_timeout,
+                           "kill_count", job->kill_count,
+                           "kill_shell_count", job->kill_shell_count);
+
+        free (critical_ranks);
+        if (!entry) {
+            errno = ENOMEM;
+            goto error;
+        }
+        if (job->impl
+            && job->impl->stats
+            && (impl_stats = (*job->impl->stats) (job))
+            && json_object_update_missing (entry, impl_stats) < 0) {
+            json_decref (entry);
+            json_decref (impl_stats);
+            errno = ENOMEM;
+            goto error;
+        }
+        json_decref (impl_stats);
+        if (json_object_set_new (o, idf58 (job->id), entry) < 0) {
+            json_decref (entry);
+            errno = ENOMEM;
+            goto error;
+        }
+        job = zhashx_next (ctx->jobs);
+    }
+    return o;
+error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return NULL;
+}
+
+static void stats_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
+{
+    struct job_exec_ctx *ctx = arg;
+    struct exec_implementation *impl;
+    json_t *o = NULL;
+    json_t *jobs;
+    int i = 0;
+
+    if (!(o = json_pack ("{s:f s:s s:s s:i}",
+                         "kill-timeout", kill_timeout,
+                         "term-signal", sigutil_signame (term_signal),
+                         "kill-signal", sigutil_signame (kill_signal),
+                         "max-kill-count", max_kill_count))) {
+        errno = ENOMEM;
+        goto error;
+    }
+    if (!(jobs = running_job_stats (ctx))
+        || json_object_set_new (o, "jobs", jobs)) {
+        json_decref (jobs);
+        errno = ENOMEM;
+        goto error;
+    }
+    while ((impl = implementations[i]) && impl->name) {
+        json_t *stats = NULL;
+        if (impl->stats && (stats = (*impl->stats) (NULL))) {
+            if (json_object_set_new (o, impl->name, stats) < 0) {
+                json_decref (stats);
+                errno = ENOMEM;
+                goto error;
+            }
+        }
+        i++;
+    }
+    if (flux_respond_pack (h, msg, "o", o) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+    json_decref (o);
+}
+
+static void config_reload_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct job_exec_ctx *ctx = arg;
+    const flux_conf_t *conf;
+    struct exec_implementation *impl;
+    flux_error_t err;
+    int i = 0;
+
+    if (flux_conf_reload_decode (msg, &conf) < 0) {
+        errprintf (&err, "Error parsing new config: %s", strerror (errno));
+        goto error;
+    }
+
+    if (job_exec_set_config_globals (h,
+                                     conf,
+                                     ctx->argc,
+                                     ctx->argv,
+                                     &err) < 0)
+        goto error;
+
+    while ((impl = implementations[i]) && impl->name) {
+        if (impl->config) {
+            if ((*impl->config) (h, conf, ctx->argc, ctx->argv, &err) < 0)
+                goto error;
+        }
+        i++;
+    }
+    if (flux_set_conf (h, flux_conf_incref (conf)) < 0) {
+        errprintf (&err, "error updating cached configuration");
+        flux_conf_decref (conf);
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, err.text) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+}
+
 static const struct flux_msg_handler_spec htab[]  = {
     { FLUX_MSGTYPE_REQUEST, "job-exec.start", start_cb,     0 },
     { FLUX_MSGTYPE_EVENT,   "job-exception",  exception_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST,
+      "job-exec.critical-ranks",
+       critical_ranks_cb,
+       FLUX_ROLE_USER
+    },
+    { FLUX_MSGTYPE_REQUEST, "job-exec.expiration", expiration_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "job-exec.stats-get", stats_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "job-exec.config-reload", config_reload_cb, 0 },
     FLUX_MSGHANDLER_TABLE_END
 };
 
@@ -1050,14 +1775,22 @@ int mod_main (flux_t *h, int argc, char **argv)
 {
     int saved_errno = 0;
     int rc = -1;
-    struct job_exec_ctx *ctx = job_exec_ctx_create (h);
-
-    if (job_exec_initialize (h, argc, argv) < 0
-        || configure_implementations (h, argc, argv) < 0) {
+    bool subscribed = false;
+    flux_error_t error;
+    struct job_exec_ctx *ctx = job_exec_ctx_create (h, argc, argv);
+
+    if (job_exec_set_config_globals (h,
+                                     flux_get_conf (h),
+                                     argc,
+                                     argv,
+                                     &error) < 0) {
+        flux_log_error (h, "job-exec: error parsing config: %s", error.text);
+        goto out;
+    }
+    if (configure_implementations (h, argc, argv) < 0) {
         flux_log_error (h, "job-exec: module initialization failed");
         goto out;
     }
-
     if (flux_msg_handler_addvec (h, htab, ctx, &ctx->handlers) < 0) {
         flux_log_error (h, "flux_msg_handler_addvec");
         goto out;
@@ -1066,21 +1799,22 @@ int mod_main (flux_t *h, int argc, char **argv)
         flux_log_error (h, "flux_event_subscribe");
         goto out;
     }
+    subscribed = true;
     if (exec_hello (h, "job-exec") < 0)
         goto out;
 
     rc = flux_reactor_run (flux_get_reactor (h), 0);
 out:
+    unload_implementations (ctx);
+
     saved_errno = errno;
-    if (flux_event_unsubscribe (h, "job-exception") < 0)
+    if (subscribed && flux_event_unsubscribe (h, "job-exception") < 0)
         flux_log_error (h, "flux_event_unsubscribe ('job-exception')");
     job_exec_ctx_destroy (ctx);
     errno = saved_errno;
     return rc;
 }
 
-MOD_NAME ("job-exec");
-
 /*
  * vi: tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/job-exec/job-exec.h b/src/modules/job-exec/job-exec.h
index e34fc2691216..b79da965c3d5 100644
--- a/src/modules/job-exec/job-exec.h
+++ b/src/modules/job-exec/job-exec.h
@@ -23,9 +23,12 @@ struct jobinfo;
 
 /*  Exec implementation interface:
  *
- *  An exec implementation must include the methods below:
+ *  An exec implementation must include the methods below (except where noted):
  *
- *   - config:  (optional) called once at module load for configuraiton
+ *   - config:  (optional) called at module load for configuration and config
+ *              reload.
+ *
+ *   - unload:  (optional) called once at module unload
  *
  *   - init:    allow selection and initialization of an exec implementation
  *              from jobspec and/or R. An implementation should return 0 to
@@ -42,17 +45,26 @@ struct jobinfo;
  *
  *   - cancel:  cancel any pending work (i.e. shells yet to be executed)
  *
- *   - cleanup: Initiate any cleanup (epilog) on ranks.
+ *   - stats:   (optional) get json object of exec implementation stats
+ *
+ *   - active_ranks:
+ *              (optional) get the set of ranks with active job shells.
  */
 struct exec_implementation {
     const char *name;
-    int  (*config)  (flux_t *h, int argc, char **argv);
+    int  (*config)  (flux_t *h,
+                     const flux_conf_t *conf,
+                     int argc,
+                     char **argv,
+                     flux_error_t *errp);
+    void (*unload)  (void);
     int  (*init)    (struct jobinfo *job);
     void (*exit)    (struct jobinfo *job);
     int  (*start)   (struct jobinfo *job);
     int  (*kill)    (struct jobinfo *job, int signum);
     int  (*cancel)  (struct jobinfo *job);
-    int  (*cleanup) (struct jobinfo *job, const struct idset *ranks);
+    json_t * (*stats) (struct jobinfo *job);
+    struct idset * (*active_ranks) (struct jobinfo *job);
 };
 
 /*  Exec job information */
@@ -60,6 +72,7 @@ struct jobinfo {
     flux_t *              h;
     flux_jobid_t          id;
     char                  ns [64];   /* namespace string */
+    char                * rootref;   /* ns rootref if restart */
     const flux_msg_t *    req;       /* initial request */
     uint32_t              userid;    /* requesting userid */
     int                   flags;     /* job flags */
@@ -67,6 +80,9 @@ struct jobinfo {
     struct resource_set * R;         /* Fetched and parsed resource set R */
     json_t *              jobspec;   /* Fetched jobspec */
 
+    struct idset *        critical_ranks;  /* critical shell ranks */
+
+    uint8_t               multiuser:1;
     uint8_t               has_namespace:1;
     uint8_t               exception_in_progress:1;
 
@@ -74,10 +90,19 @@ struct jobinfo {
     uint8_t               running:1;     /* all shells are running */
     uint8_t               finalizing:1;  /* in process of cleanup */
 
+    int                   reattach;      /* job-manager reattach attempt */
     int                   wait_status;
 
+    struct eventlogger *  ev;           /* event batcher */
+
     double                kill_timeout; /* grace time between sigterm,kill */
     flux_watcher_t       *kill_timer;
+    int                   kill_count;
+    flux_watcher_t       *kill_shell_timer;
+    int                   kill_shell_count;
+    flux_watcher_t       *expiration_timer;
+
+    double                t0;        /* timestamp we initially saw this job */
 
     /* Exec implementation for this job */
     struct exec_implementation *impl;
@@ -96,8 +121,11 @@ int jobinfo_emit_event_pack_nowait (struct jobinfo *job,
                                      const char *name,
                                      const char *fmt, ...);
 
-/* Emit  start event with optional note in jansson pack format */
-void jobinfo_started (struct jobinfo *job, const char *fmt, ...);
+/* Emit  start event */
+void jobinfo_started (struct jobinfo *job);
+
+/* Emit  reattached event */
+void jobinfo_reattached (struct jobinfo *job);
 
 /* Notify job-exec that ranks in idset `ranks` have completed
  *  with the given wait status
@@ -106,18 +134,35 @@ void jobinfo_tasks_complete (struct jobinfo *job,
                              const struct idset *ranks,
                              int wait_status);
 
-/* Notify job-exec that ranks in idset `ranks` have finished epilog,
- *  and resources can be released
+/* Notify job-exec of fatal error. Triggers kill/finalize.
  */
-void jobinfo_cleanup_complete (struct jobinfo *job,
-                               const struct idset *ranks,
-                               int rc);
+void jobinfo_fatal_error (struct jobinfo *job, int errnum,
+                          const char *fmt, ...);
+
+void jobinfo_raise (struct jobinfo *job,
+                    const char *type,
+                    int severity,
+                    const char *fmt, ...);
 
+int jobinfo_drain_ranks (struct jobinfo *job,
+                         const char *ranks,
+                         const char *fmt,
+                         ...);
 
-/* Notify job-exec of fatal error. Triggers kill/cleanup.
+/* Append a log output message to exec.eventlog for job
  */
-void jobinfo_fatal_error (struct jobinfo *job, int errnum,
-                          const char *fmt, ...);
+void jobinfo_log_output (struct jobinfo *job,
+                         int rank,
+                         const char *component,
+                         const char *stream,
+                         const char *data,
+                         int len);
+
+
+flux_future_t *jobinfo_shell_rpc_pack (struct jobinfo *job,
+                                       const char *topic,
+                                       const char *fmt,
+                                       ...);
 
 #endif /* !HAVE_JOB_EXEC_EXEC_H */
 
diff --git a/src/modules/job-exec/rset.c b/src/modules/job-exec/rset.c
index 7698a34fd298..2fcbdb79bcfe 100644
--- a/src/modules/job-exec/rset.c
+++ b/src/modules/job-exec/rset.c
@@ -8,7 +8,11 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
+#include "src/common/libutil/errno_safe.h"
 #include "rset.h"
 
 struct resource_set {
@@ -30,28 +34,24 @@ void resource_set_destroy (struct resource_set *r)
     }
 }
 
-static int idset_add_set (struct idset *set, struct idset *new)
+json_t *resource_set_get_json (struct resource_set *r)
 {
-    unsigned int i = idset_first (new);
-    while (i != IDSET_INVALID_ID) {
-        if (idset_test (set, i)) {
-            errno = EEXIST;
-            return -1;
-        }
-        if (idset_set (set, i) < 0)
-            return -1;
-        i = idset_next (new, i);
-    }
-    return 0;
+    return r ? r->R : NULL;
 }
 
-static int idset_set_string (struct idset *idset, const char *ids)
+static int util_idset_set_string (struct idset *idset, const char *ids)
 {
-    int rc;
-    struct idset *new = idset_decode (ids);
-    if (!new)
+    struct idset *new;
+    int rc = -1;
+
+    if (!(new = idset_decode (ids)))
         return -1;
-    rc = idset_add_set (idset, new);
+    if (idset_has_intersection (idset, new)) {
+        errno = EEXIST;
+        goto done;
+    }
+    rc = idset_add (idset, new);
+done:
     idset_destroy (new);
     return rc;
 }
@@ -71,7 +71,7 @@ static struct idset *rset_ranks (struct resource_set *r)
         return NULL;
     json_array_foreach (r->R_lite, i, entry) {
         if ((json_unpack_ex (entry, NULL, 0, "{s:s}", "rank", &ranks) < 0)
-            || (idset_set_string (idset, ranks) < 0))
+            || (util_idset_set_string (idset, ranks) < 0))
             goto err;
     }
     return idset;
@@ -86,40 +86,52 @@ static int rset_read_time_window (struct resource_set *r, json_error_t *errp)
         errno = EINVAL;
         return -1;
     }
-    /*  Default values: < 0 indicates "unset"
+    /*  Default values: 0. indicates "unset"
      */
-    r->expiration = -1.;
-    r-> starttime = -1.;
-    if (json_unpack_ex (r->R, errp, 0,
+    r->expiration = 0.;
+    r->starttime = 0.;
+    if (json_unpack_ex (r->R,
+                        errp,
+                        0,
                         "{s:{s?F s?F}}",
                         "execution",
-                        "starttime",  &r->starttime,
-                        "expiration", &r->expiration) < 0)
+                         "starttime",  &r->starttime,
+                         "expiration", &r->expiration) < 0)
         return -1;
     return 0;
 }
 
-struct resource_set * resource_set_create (const char *R, json_error_t *errp)
+struct resource_set *resource_set_create_fromjson (json_t *R,
+                                                   json_error_t *errp)
 {
     int version = 0;
     struct resource_set *r = calloc (1, sizeof (*r));
-    if (!(r->R = json_loads (R, 0, errp)))
+    if (!r) {
+        snprintf (errp->text, sizeof (errp->text), "out of memory");
         goto err;
-    if (json_unpack_ex (r->R, errp, 0, "{s:i s:{s:o}}",
-                                       "version", &version,
-                                       "execution",
-                                       "R_lite", &r->R_lite) < 0)
+    }
+    r->R = json_incref (R);
+    if (json_unpack_ex (r->R,
+                        errp,
+                        0,
+                        "{s:i s:{s:o}}",
+                        "version", &version,
+                        "execution",
+                        "R_lite", &r->R_lite) < 0)
         goto err;
     if (version != 1) {
         if (errp)
-            snprintf (errp->text, sizeof (errp->text),
-                    "invalid version: %d", version);
+            snprintf (errp->text,
+                      sizeof (errp->text),
+                      "invalid version: %d",
+                      version);
         goto err;
     }
     if (!(r->ranks = rset_ranks (r))) {
         if (errp)
-            snprintf (errp->text, sizeof (errp->text),
-                    "R_lite: failed to read target rank list");
+            snprintf (errp->text,
+                      sizeof (errp->text),
+                      "R_lite: failed to read target rank list");
         goto err;
     }
     if (rset_read_time_window (r, errp) < 0)
@@ -130,7 +142,19 @@ struct resource_set * resource_set_create (const char *R, json_error_t *errp)
     return NULL;
 }
 
-const struct idset * resource_set_ranks (struct resource_set *r)
+struct resource_set *resource_set_create (const char *R, json_error_t *errp)
+{
+    json_t *o;
+    struct resource_set *rset;
+
+    if (!(o = json_loads (R, 0, errp)))
+        return NULL;
+    rset = resource_set_create_fromjson (o, errp);
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return rset;
+}
+
+const struct idset *resource_set_ranks (struct resource_set *r)
 {
     return r->ranks;
 }
@@ -145,6 +169,49 @@ double resource_set_expiration (struct resource_set *r)
     return r->expiration;
 }
 
+void resource_set_update_expiration (struct resource_set *r,
+                                     double expiration)
+{
+    r->expiration = expiration;
+}
+
+uint32_t resource_set_nth_rank (struct resource_set *r, int n)
+{
+    uint32_t rank;
+
+    if (r == NULL || n < 0) {
+        errno = EINVAL;
+        return IDSET_INVALID_ID;
+    }
+
+    rank = idset_first (r->ranks);
+    while (n-- && rank != IDSET_INVALID_ID)
+        rank = idset_next (r->ranks, rank);
+    if (rank == IDSET_INVALID_ID)
+        errno = ENOENT;
+    return rank;
+}
+
+uint32_t resource_set_rank_index (struct resource_set *r, uint32_t rank)
+{
+    uint32_t i, n;
+
+    if (r == NULL) {
+        errno = EINVAL;
+        return IDSET_INVALID_ID;
+    }
+
+    i = 0;
+    n = idset_first (r->ranks);
+    while (n != IDSET_INVALID_ID) {
+        if (n == rank)
+            return i;
+        i++;
+        n = idset_next (r->ranks, n);
+    }
+    errno = ENOENT;
+    return IDSET_INVALID_ID;
+}
 
 /* vi: ts=4 sw=4 expandtab
  */
diff --git a/src/modules/job-exec/rset.h b/src/modules/job-exec/rset.h
index 4df5a10402c7..de98f2cf3d98 100644
--- a/src/modules/job-exec/rset.h
+++ b/src/modules/job-exec/rset.h
@@ -12,19 +12,32 @@
 #define HAVE_JOB_EXEC_RSET_H 1
 #include <flux/idset.h>
 #include <jansson.h>
+#include <stdint.h>
 
 struct resource_set;
 
 struct resource_set *resource_set_create (const char *R, json_error_t *errp);
 
+struct resource_set *resource_set_create_fromjson (json_t *R,
+		                                   json_error_t *errp);
+
 void resource_set_destroy (struct resource_set *rset);
 
-const struct idset * resource_set_ranks (struct resource_set *rset);
+json_t *resource_set_get_json (struct resource_set *rset);
+
+const struct idset *resource_set_ranks (struct resource_set *rset);
+
+uint32_t resource_set_nth_rank (struct resource_set *r, int n);
+
+uint32_t resource_set_rank_index (struct resource_set *r, uint32_t rank);
 
 double resource_set_starttime (struct resource_set *rset);
 
 double resource_set_expiration (struct resource_set *rset);
 
+void resource_set_update_expiration (struct resource_set *rset,
+                                     double expiration);
+
 #endif /* !HAVE_JOB_EXEC_RSET_H */
 
 
diff --git a/src/modules/job-exec/test/bulk-exec.c b/src/modules/job-exec/test/bulk-exec.c
deleted file mode 100644
index 44d39ec78ca4..000000000000
--- a/src/modules/job-exec/test/bulk-exec.c
+++ /dev/null
@@ -1,275 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/*  Test driver for job-exec/bulk-exec.[ch] bulk subprocess
- *   execution API.
- */
-
-#define _GNU_SOURCE 1
-#include <unistd.h>
-#include <assert.h>
-
-#include <flux/core.h>
-#include <flux/idset.h>
-#include <flux/optparse.h>
-
-#include "bulk-exec.h"
-#include "src/common/libutil/log.h"
-
-extern char **environ;
-static int cancel_after = 0;
-
-void started (struct bulk_exec *exec, void *arg)
-{
-    log_msg ("started");
-}
-
-void complete (struct bulk_exec *exec, void *arg)
-{
-    flux_t *h = arg;
-    log_msg ("complete");
-    flux_reactor_stop (flux_get_reactor (h));
-}
-
-void exited (struct bulk_exec *exec, void *arg, const struct idset *ids)
-{
-    char *s = idset_encode (ids, IDSET_FLAG_RANGE);
-    log_msg ("ranks %s: exited", s);
-    free (s);
-}
-
-void on_error (struct bulk_exec *exec, flux_subprocess_t *p, void *arg)
-{
-    if (p) {
-        flux_subprocess_state_t state = flux_subprocess_state (p);
-        log_msg ("%d: pid %ju: %s", flux_subprocess_rank (p),
-                (uintmax_t) flux_subprocess_pid (p),
-                flux_subprocess_state_string (state));
-    }
-    flux_future_t *f = bulk_exec_kill (exec, 9);
-    if (flux_future_get (f, NULL) < 0)
-        log_err_exit ("bulk_exec_kill");
-}
-
-void on_output (struct bulk_exec *exec, flux_subprocess_t *p,
-                const char *stream, const char *data,
-                int data_len, void *arg)
-{
-    int rank = flux_subprocess_rank (p);
-    FILE *fp = strcmp (stream, "stdout") == 0 ? stdout : stderr;
-    fprintf (fp, "%d: %s", rank, data);
-}
-
-static void kill_cb (flux_future_t *f, void *arg)
-{
-    if (flux_future_get (f, NULL) < 0)
-        log_err ("bulk_exec_kill");
-    flux_future_destroy (f);
-}
-
-static void signal_cb (flux_reactor_t *r, flux_watcher_t *w,
-                       int revents, void *arg)
-{
-    struct bulk_exec *exec = arg;
-    int signum = flux_signal_watcher_get_signum (w);
-
-    log_msg ("sending signal %d to all tasks\n", signum);
-    flux_future_t *f = bulk_exec_kill (exec, signum);
-    if (!f || (flux_future_then (f, -1., kill_cb, exec) < 0))
-        log_err ("SIGINT: failed to forward signal %d", signum);
-    flux_watcher_stop (w);
-}
-
-static void check_cancel_cb (flux_reactor_t *r, flux_watcher_t *w,
-                             int revents, void *arg)
-{
-    struct bulk_exec *exec = arg;
-    if (cancel_after && bulk_exec_current (exec) >= cancel_after) {
-        log_msg ("cancelling remaining commands");
-        bulk_exec_cancel (exec);
-        flux_watcher_stop (w);
-    }
-}
-
-static unsigned int idset_pop (struct idset *idset)
-{
-    unsigned int id = idset_first (idset);
-    if (id == IDSET_INVALID_ID)
-        return id;
-    (void) idset_clear (idset, id);
-    return id;
-}
-
-static struct idset * idset_pop_n (struct idset *idset, int n)
-{
-    unsigned int id;
-    struct idset *ids = idset_create (0, IDSET_FLAG_AUTOGROW);
-    if (ids == NULL)
-        return NULL;
-    while ((n-- > 0) && ((id = idset_pop (idset)) != IDSET_INVALID_ID)) {
-        if (idset_set (ids, id) < 0)
-            goto err;
-    }
-    return ids;
-err:
-    idset_destroy (ids);
-    return NULL;
-}
-
-void push_commands (struct bulk_exec *exec,
-                    struct idset *idset,
-                    int ncmds,
-                    int ac, char **av)
-{
-    flux_cmd_t *cmd = flux_cmd_create (ac, av, environ);
-    char *cwd = get_current_dir_name ();
-    struct idset *ids = NULL;
-    int count;
-    int per_cmd;
-
-    if (cmd == NULL)
-        log_err_exit ("flux_cmd_create");
-
-    if ((count = idset_count (idset)) < ncmds)
-        log_err_exit ("Can't split %d ranks into %d cmds", count, ncmds);
-    flux_cmd_setcwd (cmd, cwd);
-    free (cwd);
-
-    per_cmd = count/ncmds;
-    while (idset_count (idset)) {
-        struct idset *ids = idset_pop_n (idset, per_cmd);
-        if (bulk_exec_push_cmd (exec, ids, cmd, 0) < 0)
-            log_err_exit ("bulk_exec_push_cmd");
-        idset_destroy (ids);
-    }
-    idset_destroy (ids);
-    flux_cmd_destroy (cmd);
-}
-
-int main (int ac, char **av)
-{
-    struct bulk_exec *exec = NULL;
-    optparse_t *p = NULL;
-    const char *ranks = NULL;
-    struct idset *idset = NULL;
-    flux_t *h = NULL;
-    uint32_t size;
-    int ncmds = 1;
-    int optindex = -1;
-
-    struct optparse_option opts[] = {
-        { .name = "rank",
-          .key = 'r',
-          .has_arg = 1,
-          .arginfo = "IDS",
-          .usage = "Target ranks in IDS"
-        },
-        { .name = "mpl",
-          .key = 'm',
-          .has_arg = 1,
-          .arginfo = "COUNT",
-          .usage = "Max procs to start per loop iteration"
-        },
-        { .name = "ncmds",
-          .key = 'n',
-          .has_arg = 1,
-          .arginfo = "N",
-          .usage = "Internally, split into N 'cmds'"
-        },
-        { .name = "cancel-after",
-          .key  = 'c',
-          .has_arg = 1,
-          .arginfo = "NCMDS",
-          .usage = "Cancel after NCMDS cmds have been launched"
-        },
-        OPTPARSE_TABLE_END
-    };
-
-    struct bulk_exec_ops ops = {
-        .on_start    = started,
-        .on_exit     = exited,
-        .on_complete = complete,
-        .on_error    = on_error,
-        .on_output   = on_output,
-    };
-
-    if (!(p = optparse_create ("execer")))
-        log_err_exit ("optparse_create");
-    if (optparse_add_option_table (p, opts) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_add_option_table");
-    if ((optindex = optparse_parse_args (p, ac, av)) < 0)
-        exit (1);
-    av += optindex;
-    ac -= optindex;
-
-    if (ac == 0) {
-        optparse_print_usage (p);
-        exit (1);
-    }
-
-    if (!optparse_getopt (p, "rank", &ranks))
-        ranks = "all";
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (flux_get_size (h, &size) < 0)
-        log_err_exit ("flux_get_size");
-
-    if (strcmp (ranks, "all") == 0) {
-        if (!(idset = idset_create (size, 0)))
-            log_err_exit ("idset_create");
-        if (idset_range_set (idset, 0, size - 1) < 0)
-            log_err_exit ("idset_range_set");
-    }
-    else if (!(idset = idset_decode (ranks)))
-        log_err_exit ("idset_decode (%s)", ranks);
-
-    if (!(exec = bulk_exec_create (&ops, h)))
-        log_err_exit ("bulk_exec_create");
-
-    if (bulk_exec_set_max_per_loop (exec, optparse_get_int (p, "mpl", -1)) < 0)
-        log_err_exit ("bulk_exec_set_max_per_loop");
-
-    ncmds = optparse_get_int (p, "ncmds", 1);
-
-    push_commands (exec, idset, ncmds, ac, av);
-
-    if (bulk_exec_start (h, exec) < 0)
-        log_err_exit ("bulk_exec_start");
-
-    flux_watcher_t *w = flux_signal_watcher_create (flux_get_reactor (h),
-                                                    SIGINT,
-                                                    signal_cb,
-                                                    exec);
-    flux_watcher_start (w);
-
-    flux_watcher_t *cw = NULL;
-    if ((cancel_after = optparse_get_int (p, "cancel-after", 0)) > 0) {
-        cw = flux_check_watcher_create (flux_get_reactor (h),
-                                        check_cancel_cb,
-                                        exec);
-        flux_watcher_start (cw);
-    }
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
-        log_err_exit ("flux_reactor_run");
-
-    flux_watcher_destroy (w);
-    flux_watcher_destroy (cw);
-    idset_destroy (idset);
-    bulk_exec_destroy (exec);
-    flux_close (h);
-    optparse_destroy (p);
-
-    return 0;
-}
-
-/* vi: ts=4 sw=4 expandtab
- */
diff --git a/src/modules/job-exec/test/rset.c b/src/modules/job-exec/test/rset.c
index 74b7824f8c70..23e99d2ced5d 100644
--- a/src/modules/job-exec/test/rset.c
+++ b/src/modules/job-exec/test/rset.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <errno.h>
 #include <string.h>
 
@@ -38,6 +41,19 @@ struct resource_set_test {
     "    } " \
     "}"
 
+#define BASIC_ALT_RANKS \
+    "{ \"version\": 1," \
+    "  \"execution\": { " \
+    "    \"starttime\":  12345, " \
+    "    \"expiration\": 12445, " \
+    "    \"R_lite\": " \
+    "       [ {\"rank\": \"7,14,21\", " \
+    "          \"children\": { \"core\": \"0-3\" } " \
+    "         } " \
+    "       ] " \
+    "    } " \
+    "}"
+
 #define BAD_VERSION \
     "{ \"version\": 2," \
     "  \"execution\": { " \
@@ -86,9 +102,48 @@ struct resource_set_test tests[] = {
       "0-2", 12345., 12445.,
       NULL
     },
+    { "basic alternate ranks R check",
+      BASIC_ALT_RANKS,
+      "7,14,21", 12345., 12445.,
+      NULL
+    },
     RESOURCE_SET_TEST_END
 };
 
+void test_rank_conversions ()
+{
+    json_error_t err;
+    struct resource_set *r = resource_set_create (BASIC_ALT_RANKS, &err);
+
+    if (r == NULL)
+        BAIL_OUT ("resource_set_create: %s", err.text);
+
+    ok (resource_set_nth_rank (NULL, 0) == IDSET_INVALID_ID && errno == EINVAL,
+        "resource_set_nth_rank (NULL, 0) returns EINVAL");
+    ok (resource_set_nth_rank (r, -1) == IDSET_INVALID_ID && errno == EINVAL,
+        "resource_set_nth_rank (r, -1) returns EINVAL");
+
+    ok (resource_set_nth_rank (r, 3) == IDSET_INVALID_ID && errno == ENOENT,
+        "resource_set_nth_rank too big a rank returns ENOENT");
+
+    ok (resource_set_nth_rank (r, 0) == 7
+        && resource_set_nth_rank (r, 1) == 14
+        && resource_set_nth_rank (r, 2) == 21,
+        "resource_set_nth_rank works");
+
+    ok (resource_set_rank_index (NULL, 0) == IDSET_INVALID_ID
+        && errno == EINVAL,
+        "resource_set_rank_index (NULL, 0) returns EINVAL");
+    ok (resource_set_rank_index (r, 3) == IDSET_INVALID_ID
+        && errno == ENOENT,
+        "resource_set_rank_index with invalid rank returns ENOENT");
+
+    ok (resource_set_rank_index (r, 7) == 0
+        && resource_set_rank_index (r, 14) == 1
+        && resource_set_rank_index (r, 21) == 2,
+        "resource_set_rank_index works");
+}
+
 int main (int ac, char *av[])
 {
     struct resource_set_test *e = NULL;
@@ -124,6 +179,10 @@ int main (int ac, char *av[])
                 ok (expiration == e->expiration,
                     "%s: expect expiration %.2f (got %.2f)", e->descr,
                     e->expiration, expiration);
+                resource_set_update_expiration (r, 120.);
+                ok (resource_set_expiration (r) == 120.,
+                    "%s: resource_set_update_expiration() works",
+                    e->descr);
             }
             else {
                 fail ("%s: %d:[%s]",
@@ -134,6 +193,8 @@ int main (int ac, char *av[])
         e++;
     }
 
+    test_rank_conversions ();
+
     done_testing ();
 }
 
diff --git a/src/modules/job-exec/testexec.c b/src/modules/job-exec/testexec.c
index c296e8e25182..edbd57ff331a 100644
--- a/src/modules/job-exec/testexec.c
+++ b/src/modules/job-exec/testexec.c
@@ -12,7 +12,7 @@
  *
  * DESCRIPTION
  *
- * This exec module implments timer driven test execution without any
+ * This exec module implements timer driven test execution without any
  * job-shells for testing and demonstration purposes. The module is
  * activated either when it is the only active exec implementation
  * loaded, or if the exec test configuration block is present in the
@@ -25,10 +25,16 @@
  *
  * {
  *   "run_duration":s,      - alternate/override attributes.system.duration
- *   "cleanup_duration":s   - enable a fake job epilog and set its duration
  *   "wait_status":i        - report this value as status in the "finish" resp
  *   "mock_exception":s     - mock an exception during this phase of job
  *                             execution (currently "init" and "run")
+ *   "override":i           - exec override mode: wait for RPC to emit start
+ *                             event a testexec job. If job has unlimited
+ *                             duration, then also wait for finish RPC or
+ *                             job exception for job finish event.
+ *   "reattach_finish":i    - if reattached, assume job has finished
+ *                             and no longer has remaining time to run.
+ *                             Useful for testing job reattach.
  * }
  *
  */
@@ -37,28 +43,52 @@
 # include "config.h"
 #endif
 
+#include <time.h>
+#include <sys/time.h>
+
 #include "src/common/libutil/fsd.h"
+#include "src/common/libjob/job_hash.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
 #include "job-exec.h"
 
+struct testexec_ctx {
+    flux_t *h;
+    flux_msg_handler_t *mh;  /* testexec RPC handler */
+    flux_msg_handler_t *dh;  /* disconnect handler */
+    zhashx_t *jobs;
+    bool allow_guests;
+};
+
 struct testconf {
     bool                  enabled;          /* test execution enabled       */
+    int                   override;         /* wait for RPC for start event */
+    int                   reattach_finish;  /* if reattached, just finish job */
     double                run_duration;     /* duration of fake job in sec  */
-    double                cleanup_duration; /* if > 0., duration of epilog  */
     int                   wait_status;      /* reported status for "finish" */
-    const char *          mock_exception;   /* fake excetion at this site   */
+    const char *          mock_exception;   /* fake exception at this site  */
 };
 
 struct testexec {
+    struct jobinfo *job;
     struct testconf conf;
     struct idset *ranks;
     flux_watcher_t *timer;
 };
 
-static struct testexec * testexec_create (struct testconf conf)
+
+static struct testexec_ctx *testexec_ctx = NULL;
+
+
+static struct testexec * testexec_create (struct jobinfo *job,
+                                          struct testconf conf)
 {
     struct testexec *te = calloc (1, sizeof (*te));
     if (te == NULL)
         return NULL;
+    te->job = job;
     te->conf = conf;
     return (te);
 }
@@ -82,28 +112,34 @@ static double jobspec_duration (flux_t *h, json_t *jobspec)
 
 static int init_testconf (flux_t *h, struct testconf *conf, json_t *jobspec)
 {
-    const char *tclean = NULL;
     const char *trun = NULL;
     json_t *test = NULL;
     json_error_t err;
 
     /* get/set defaults */
     conf->run_duration = jobspec_duration (h, jobspec);
-    conf->cleanup_duration = -1.;
+    conf->override = 0;
+    conf->reattach_finish = 0;
     conf->wait_status = 0;
     conf->mock_exception = NULL;
     conf->enabled = false;
 
-    if (json_unpack_ex (jobspec, &err, 0,
-                     "{s:{s:{s:{s:o}}}}",
-                     "attributes", "system", "exec",
-                     "test", &test) < 0)
+    if (json_unpack_ex (jobspec,
+                        &err,
+                        0,
+                        "{s:{s:{s:{s:o}}}}",
+                        "attributes",
+                         "system",
+                          "exec",
+                           "test", &test) < 0)
         return 0;
     conf->enabled = true;
-    if (json_unpack_ex (test, &err, 0,
-                        "{s?s s?s s?i s?s}",
+    if (json_unpack_ex (test,
+                        &err, 0,
+                        "{s?s s?i s?i s?i s?s}",
                         "run_duration", &trun,
-                        "cleanup_duration", &tclean,
+                        "override", &conf->override,
+                        "reattach_finish", &conf->reattach_finish,
                         "wait_status", &conf->wait_status,
                         "mock_exception", &conf->mock_exception) < 0) {
         flux_log (h, LOG_ERR, "init_testconf: %s", err.text);
@@ -111,8 +147,6 @@ static int init_testconf (flux_t *h, struct testconf *conf, json_t *jobspec)
     }
     if (trun && fsd_parse_duration (trun, &conf->run_duration) < 0)
         flux_log (h, LOG_ERR, "Unable to parse run duration: %s", trun);
-    if (tclean && fsd_parse_duration (tclean, &conf->cleanup_duration) < 0)
-        flux_log (h, LOG_ERR, "Unable to parse cleanup duration: %s", tclean);
     return 0;
 }
 
@@ -121,22 +155,22 @@ static int init_testconf (flux_t *h, struct testconf *conf, json_t *jobspec)
 static bool testconf_mock_exception (struct testconf *conf, const char *where)
 {
     const char *s = conf->mock_exception;
-    return (s && strcmp (s, where) == 0);
+    return (s && streq (s, where));
 }
 
-/* Timer callback, post the "finish" event and start "cleanup" tasks.
+/* Timer callback, post the "finish" event and notify tasks are complete
  */
 static void timer_cb (flux_reactor_t *r,
                       flux_watcher_t *w,
-                      int revents, void *arg)
+                      int revents,
+                      void *arg)
 {
-    struct jobinfo *job = arg;
-    struct testexec *te = job->data;
+    struct testexec *te = arg;
 
     /* Notify job-exec that tasks have completed:
      */
-    jobinfo_tasks_complete (job,
-                            resource_set_ranks (job->R),
+    jobinfo_tasks_complete (te->job,
+                            resource_set_ranks (te->job->R),
                             te->conf.wait_status);
 }
 
@@ -145,90 +179,34 @@ static void timer_cb (flux_reactor_t *r,
  *   is sent when the timer fires (simulating the exit of the final
  *   job shell.)
  */
-static int start_timer (flux_t *h, struct testexec *te, struct jobinfo *job)
+static int start_timer (flux_t *h, struct testexec *te, double t)
 {
     flux_reactor_t *r = flux_get_reactor (h);
-    double t = te->conf.run_duration;
 
     /*  For now, if a job duration wasn't found, complete job almost
      *   immediately.
      */
     if (t < 0.)
         t = 1.e-5;
-    if (t > 0.) {
-        char timebuf[256];
-        te->timer = flux_timer_watcher_create (r, t, 0., timer_cb, job);
-        if (!te->timer) {
-            flux_log_error (h, "jobinfo_start: timer_create");
-            return -1;
+    if (t >= 0.) {
+        if (t > 0.) {
+            te->timer = flux_timer_watcher_create (r, t, 0., timer_cb, te);
+            if (!te->timer) {
+                flux_log_error (h, "jobinfo_start: timer_create");
+                return -1;
+            }
+            flux_watcher_start (te->timer);
         }
-        flux_watcher_start (te->timer);
-        snprintf (timebuf, sizeof (timebuf), "%.6fs", t);
-        jobinfo_started (job, "{ s:s }", "timer", timebuf);
+        if (te->job->reattach)
+            jobinfo_reattached (te->job);
+        else
+            jobinfo_started (te->job);
     }
     else
         return -1;
     return 0;
 }
 
-
-static void cleanup_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
-                             int revents, void *arg)
-{
-    flux_future_fulfill ((flux_future_t *) arg, NULL, NULL);
-    flux_watcher_destroy (w);
-}
-
-static flux_future_t * fake_cleanup (struct jobinfo *job)
-{
-    struct testexec *te = job->data;
-    flux_t *h = job->h;
-    flux_reactor_t *r = flux_get_reactor (h);
-    flux_future_t *f = NULL;
-    flux_watcher_t *timer = NULL;
-    double t = te->conf.cleanup_duration;
-
-    if (!(f = flux_future_create (NULL, NULL)))
-        return NULL;
-    flux_future_set_flux (f, h);
-
-    /* If no cleanup, immediately fulfill future */
-    if (t <= 0.) {
-        flux_future_fulfill (f, NULL, NULL);
-        return f;
-    }
-
-    /*  O/w, start timer to simulate cleanup */
-    if (!(timer = flux_timer_watcher_create (r, t, 0.,
-                                             cleanup_timer_cb, f))) {
-        flux_log_error (h, "flux_timer_watcher_create");
-        flux_future_fulfill_error (f, errno, "flux_timer_watcher_create");
-    }
-    flux_watcher_start (timer);
-    return f;
-}
-
-static void fake_cleanup_cb (flux_future_t *f, void *arg)
-{
-    struct jobinfo *job = arg;
-    struct idset *idset = flux_future_aux_get (f, "idset");
-    /*  XXX: ranks idset not fully supported and may be NULL */
-    jobinfo_cleanup_complete (job, idset, 0);
-    flux_future_destroy (f);
-}
-
-static int fake_cleanup_start (struct jobinfo *job, const struct idset *ranks)
-{
-    struct idset *idset = ranks ? idset_copy (ranks) : NULL;
-    flux_future_t *f = fake_cleanup (job);
-    if (!f || flux_future_then (f, -1., fake_cleanup_cb, job) < 0) {
-        flux_future_destroy (f);
-        return -1;
-    }
-    flux_future_aux_set (f, "idset", idset, (flux_free_f) idset_destroy);
-    return (0);
-}
-
 static int testexec_init (struct jobinfo *job)
 {
     flux_t *h = job->h;
@@ -241,7 +219,13 @@ static int testexec_init (struct jobinfo *job)
     }
     else if (!conf.enabled)
         return 0;
-    if (!(te = testexec_create (conf))) {
+    if (job->multiuser && !testexec_ctx->allow_guests) {
+        jobinfo_fatal_error (job,
+                             0,
+                             "guests may not use test exec implementation");
+        return -1;
+    }
+    if (!(te = testexec_create (job, conf))) {
         jobinfo_fatal_error (job, errno, "failed to init test exec module");
         return -1;
     }
@@ -251,20 +235,112 @@ static int testexec_init (struct jobinfo *job)
         testexec_destroy (te);
         return -1;
     }
+    if (zhashx_insert (testexec_ctx->jobs, &job->id, te) < 0) {
+        jobinfo_fatal_error (job, 0, "testexec: zhashx_insert failed");
+        testexec_destroy (te);
+        return -1;
+    }
     return 1;
 }
 
+static int testexec_reattach_starttime (struct jobinfo *job,
+                                        const char *eventlog,
+                                        time_t *startp)
+{
+    json_t *o = NULL;
+    size_t index;
+    json_t *value;
+    int rv = -1;
+
+    if (!(o = eventlog_decode (eventlog))) {
+        jobinfo_fatal_error (job, errno, "eventlog_decode");
+        goto cleanup;
+    }
+
+    json_array_foreach (o, index, value) {
+        double timestamp;
+        const char *name;
+
+        if (eventlog_entry_parse (value, &timestamp, &name, NULL) < 0) {
+            jobinfo_fatal_error (job, errno, "eventlog_entry_parse");
+            goto cleanup;
+        }
+        if (streq (name, "start")) {
+            (*startp) = (time_t)timestamp;
+            break;
+        }
+    }
+
+    rv = 0;
+cleanup:
+    json_decref (o);
+    return rv;
+}
+
+static int testexec_reattach (struct testexec *te)
+{
+    const char *value;
+    flux_future_t *f = NULL;
+    time_t start = 0;
+    struct timespec now;
+    time_t runtimeleft = -1;
+    int rv = -1;
+    char ekey[256];
+
+    if (flux_job_kvs_key (ekey, sizeof (ekey), te->job->id, "eventlog") < 0) {
+        jobinfo_fatal_error (te->job, errno, "flux_job_kvs_key");
+        goto cleanup;
+    }
+    if (!(f = flux_kvs_lookup (te->job->h,
+                               NULL,
+                               0,
+                               ekey))) {
+        jobinfo_fatal_error (te->job, errno, "flux_kvs_lookup");
+        goto cleanup;
+    }
+    if (flux_kvs_lookup_get (f, &value) < 0) {
+        jobinfo_fatal_error (te->job, errno, "flux_kvs_lookup_get starttimes");
+        goto cleanup;
+    }
+    if (testexec_reattach_starttime (te->job, value, &start) < 0)
+        goto cleanup;
+    if (!te->conf.reattach_finish) {
+        /* just use seconds, we approximate runtime left */
+        clock_gettime (CLOCK_REALTIME, &now);
+        if ((now.tv_sec - start) <= te->conf.run_duration)
+            runtimeleft = (start + te->conf.run_duration) - now.tv_sec;
+    }
+    if (start_timer (te->job->h,
+                     te,
+                     runtimeleft) < 0) {
+        jobinfo_fatal_error (te->job, errno, "unable to restart timer");
+        goto cleanup;
+    }
+    rv = 0;
+cleanup:
+    flux_future_destroy (f);
+    return rv;
+}
+
 static int testexec_start (struct jobinfo *job)
 {
     struct testexec *te = job->data;
 
-    if (start_timer (job->h, te, job) < 0) {
-        jobinfo_fatal_error (job, errno, "unable to start test exec timer");
-        return -1;
+    if (job->reattach) {
+        if (testexec_reattach (te) < 0)
+            return -1;
     }
-    if (testconf_mock_exception (&te->conf, "run")) {
-        jobinfo_fatal_error (job, 0, "mock run exception generated");
-        return -1;
+    else {
+        if (!te->conf.override && start_timer (job->h,
+                                               te,
+                                               te->conf.run_duration) < 0) {
+            jobinfo_fatal_error (job, errno, "unable to start test exec timer");
+            return -1;
+        }
+        if (testconf_mock_exception (&te->conf, "run")) {
+            jobinfo_fatal_error (job, 0, "mock run exception generated");
+            return -1;
+        }
     }
     return 0;
 }
@@ -280,30 +356,171 @@ static int testexec_kill (struct jobinfo *job, int signum)
      *  sent to all ranks would terminate processes that would exit and
      *  report wait status through normal channels.
      */
-    if (job->running)
+    if (job->started)
         jobinfo_tasks_complete (job, te->ranks, signum);
     return 0;
 }
 
-static int testexec_cleanup (struct jobinfo *job, const struct idset *idset)
+static void testexec_exit (struct jobinfo *job)
 {
-    return fake_cleanup_start (job, idset);
+    zhashx_delete (testexec_ctx->jobs, &job->id);
+    job->data = NULL;
 }
 
-static void testexec_exit (struct jobinfo *job)
+static void testexec_request_cb (flux_t *h,
+                                 flux_msg_handler_t *mh,
+                                 const flux_msg_t *msg,
+                                 void *arg)
 {
-    struct testexec *te = job->data;
-    testexec_destroy (te);
-    job->data = NULL;
+    const char *errmsg = NULL;
+    struct testexec_ctx *ctx = arg;
+    struct testexec *te;
+    const char *event;
+    flux_jobid_t id;
+    uint32_t owner;
+    int code = 0;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s:I s?i}",
+                             "event", &event,
+                             "jobid", &id,
+                             "status", &code) < 0)
+        goto error;
+    if (!(te = zhashx_lookup (ctx->jobs, &id))) {
+        errmsg = "Job not found";
+        errno = ENOENT;
+        goto error;
+    }
+    if (flux_msg_get_userid (msg, &owner) < 0
+        || owner != te->job->userid) {
+        errmsg = "Permission denied";
+        errno = EPERM;
+        goto error;
+    }
+    if (!te->conf.override) {
+        errmsg = "Job not in exec override mode";
+        errno = EINVAL;
+        goto error;
+    }
+    if (streq (event, "start")) {
+        if (te->job->running) {
+            errmsg = "Job already running";
+            errno = EINVAL;
+            goto error;
+        }
+        if (start_timer (h, te, te->conf.run_duration) < 0)
+            goto error;
+    }
+    else if (streq (event, "finish")) {
+        if (!te->job->running) {
+            errmsg = "Job not running";
+            errno = EINVAL;
+            goto error;
+        }
+        flux_watcher_stop (te->timer);
+        jobinfo_tasks_complete (te->job, te->ranks, code);
+    }
+    else {
+        errmsg = "Invalid event";
+        errno = EINVAL;
+        goto error;
+    }
+
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
+
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+static void testexec_ctx_destroy (struct testexec_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        flux_msg_handler_destroy (ctx->mh);
+        flux_msg_handler_destroy (ctx->dh);
+        zhashx_destroy (&ctx->jobs);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static void testexec_destructor (void **item)
+{
+    if (item) {
+        struct testexec *te = *item;
+        testexec_destroy (te);
+        *item = NULL;
+    }
+}
+
+static struct testexec_ctx *testexec_ctx_create (flux_t *h)
+{
+    struct flux_match match = FLUX_MATCH_REQUEST;
+    struct testexec_ctx *ctx = calloc (1, sizeof (*ctx));
+    if (!ctx)
+        return NULL;
+    ctx->h = h;
+
+    match.topic_glob = "job-exec.override";
+    ctx->mh = flux_msg_handler_create (h,
+                                       match,
+                                       testexec_request_cb,
+                                       ctx);
+    if (!ctx->mh
+        || !(ctx->jobs = job_hash_create ()))
+        goto error;
+    zhashx_set_destructor (ctx->jobs, testexec_destructor);
+
+    flux_msg_handler_allow_rolemask (ctx->mh, FLUX_ROLE_USER);
+    flux_msg_handler_start (ctx->mh);
+
+    return ctx;
+
+error:
+    testexec_ctx_destroy (ctx);
+    return NULL;
+}
+
+static int testexec_config (flux_t *h,
+                            const flux_conf_t *conf,
+                            int argc,
+                            char **argv,
+                            flux_error_t *errp)
+{
+    if (!testexec_ctx) {
+        if (!(testexec_ctx = testexec_ctx_create (h)))
+            return -1;
+    }
+    if (flux_conf_unpack (conf,
+                          errp,
+                          "{s?{s?{s?b}}}",
+                          "exec",
+                           "testexec",
+                             "allow-guests", &testexec_ctx->allow_guests) < 0)
+        return -1;
+    return 0;
+}
+
+static void testexec_unload (void)
+{
+    testexec_ctx_destroy (testexec_ctx);
+    // Ensure a reload will not re-use a freed value
+    testexec_ctx = NULL;
 }
 
 struct exec_implementation testexec = {
     .name =     "testexec",
+    .config =   testexec_config,
+    .unload =   testexec_unload,
     .init =     testexec_init,
     .exit =     testexec_exit,
     .start =    testexec_start,
     .kill =     testexec_kill,
-    .cleanup =  testexec_cleanup
+    .stats =    NULL,
 };
 
 /* vi: ts=4 sw=4 expandtab
diff --git a/src/modules/job-info/Makefile.am b/src/modules/job-info/Makefile.am
deleted file mode 100644
index 99ef031ed823..000000000000
--- a/src/modules/job-info/Makefile.am
+++ /dev/null
@@ -1,42 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) $(FLUX_SECURITY_CFLAGS) $(YAMLCPP_CFLAGS)
-
-fluxmod_LTLIBRARIES = job-info.la
-
-job_info_la_SOURCES = \
-	job-info.c \
-	info.h \
-	allow.h \
-	allow.c \
-	job_state.h \
-	job_state.c \
-	list.h \
-	list.c \
-	lookup.h \
-	lookup.c \
-	watch.h \
-	watch.c \
-	guest_watch.h \
-	guest_watch.c \
-	job_util.h \
-	job_util.c \
-	idsync.h \
-	idsync.c
-
-job_info_la_LDFLAGS = $(fluxmod_ldflags) -module
-job_info_la_LIBADD = $(fluxmod_libadd) \
-	$(top_builddir)/src/common/libjob/libjob.la \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(top_builddir)/src/common/libflux-optparse.la \
-	$(ZMQ_LIBS)
diff --git a/src/modules/job-info/allow.c b/src/modules/job-info/allow.c
index 8b09a542f6bd..4c8256c2186b 100644
--- a/src/modules/job-info/allow.c
+++ b/src/modules/job-info/allow.c
@@ -16,66 +16,91 @@
 #include <jansson.h>
 #include <flux/core.h>
 
-#include "info.h"
+#include "job-info.h"
 #include "allow.h"
 
 #include "src/common/libeventlog/eventlog.h"
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
 
 /* Parse the submit userid from the event log.
- * Assume "submit" is the first event.
+ * RFC 18 defines the structure of eventlogs.
+ * RFC 21 requires that the first entry is "submit" and defines its context.
  */
-static int eventlog_get_userid (struct info_ctx *ctx, const char *s,
-                                int *useridp)
+static int eventlog_get_userid (struct info_ctx *ctx,
+                                const char *s,
+                                uint32_t *useridp)
 {
-    json_t *a = NULL;
-    json_t *entry = NULL;
-    const char *name = NULL;
-    json_t *context = NULL;
+    json_t *o = NULL;
+    const char *name;
+    int userid;
     int rv = -1;
 
-    if (!(a = eventlog_decode (s))) {
-        flux_log_error (ctx->h, "%s: eventlog_decode", __FUNCTION__);
-        goto error;
-    }
-    if (!(entry = json_array_get (a, 0))) {
-        errno = EINVAL;
-        goto error;
-    }
-    if (eventlog_entry_parse (entry, NULL, &name, &context) < 0) {
-        flux_log_error (ctx->h, "%s: eventlog_decode", __FUNCTION__);
-        goto error;
-    }
-    if (strcmp (name, "submit") != 0 || !context) {
-        flux_log_error (ctx->h, "%s: invalid event", __FUNCTION__);
-        errno = EINVAL;
-        goto error;
-    }
-    if (json_unpack (context, "{ s:i }", "userid", useridp) < 0) {
+    if (!s
+        || !(o = json_loads (s, JSON_DISABLE_EOF_CHECK, NULL))
+        || json_unpack (o,
+                        "{s:s s:{s:i}}",
+                        "name", &name,
+                        "context",
+                          "userid", &userid) < 0
+        || !streq (name, "submit")) {
         errno = EPROTO;
         goto error;
     }
+    (*useridp) = userid;
     rv = 0;
 error:
-    json_decref (a);
+    ERRNO_SAFE_WRAP (json_decref, o);
     return rv;
 }
 
-/* Optimization:
- * Avoid calling eventlog_get_userid() if message cred has OWNER role.
- */
-int eventlog_allow (struct info_ctx *ctx, const flux_msg_t *msg,
-                    const char *s)
+static void store_lru (struct info_ctx *ctx, flux_jobid_t id, uint32_t userid)
 {
-    struct flux_msg_cred cred;
-    int job_user;
+    char key[64];
+    uint32_t *userid_ptr = NULL;
 
-    if (flux_msg_get_cred (msg, &cred) < 0)
+    snprintf (key, 64, "%ju", (uintmax_t)id);
+
+    if (!(userid_ptr = calloc (1, sizeof (userid))))
+        return;
+    (*userid_ptr) = userid;
+
+    if (lru_cache_put (ctx->owner_lru, key, userid_ptr) < 0) {
+        if (errno != EEXIST)
+            flux_log_error (ctx->h, "%s: lru_cache_put", __FUNCTION__);
+        free (userid_ptr);
+        return;
+    }
+    return;
+}
+
+int eventlog_allow (struct info_ctx *ctx,
+                    const flux_msg_t *msg,
+                    flux_jobid_t id,
+                    const char *s)
+{
+    uint32_t userid;
+    if (eventlog_get_userid (ctx, s, &userid) < 0)
         return -1;
-    if (!(cred.rolemask & FLUX_ROLE_OWNER)) {
-        if (eventlog_get_userid (ctx, s, &job_user) < 0)
-            return -1;
-        if (flux_msg_cred_authorize (cred, job_user) < 0)
+    store_lru (ctx, id, userid);
+    if (flux_msg_authorize (msg, userid) < 0)
+        return -1;
+    return 0;
+}
+
+int eventlog_allow_lru (struct info_ctx *ctx,
+                        const flux_msg_t *msg,
+                        flux_jobid_t id)
+{
+    char key[64];
+    uint32_t *userid_ptr;
+
+    snprintf (key, 64, "%ju", (uintmax_t)id);
+
+    if ((userid_ptr = lru_cache_get (ctx->owner_lru, key))) {
+        if (flux_msg_authorize (msg, (*userid_ptr)) < 0)
             return -1;
+        return 1;
     }
     return 0;
 }
diff --git a/src/modules/job-info/allow.h b/src/modules/job-info/allow.h
index 35e62157769e..f78e05bd0dda 100644
--- a/src/modules/job-info/allow.h
+++ b/src/modules/job-info/allow.h
@@ -13,15 +13,28 @@
 
 #include <flux/core.h>
 
-#include "info.h"
+#include "job-info.h"
 
 /* Determine if user who sent request 'msg' is allowed to
  * access job eventlog 's'.  Assume first event is the "submit"
- * event which records the job owner.
+ * event which records the job owner.  Will cache recently looked
+ * up job owners in an LRU cache.
  */
-int eventlog_allow (struct info_ctx *ctx, const flux_msg_t *msg,
+int eventlog_allow (struct info_ctx *ctx,
+                    const flux_msg_t *msg,
+                    flux_jobid_t id,
                     const char *s);
 
+/* Determine if user who sent request 'msg' is allowed to access job
+ * eventlog via LRU cache.  Returns 1 if access allowed, 0 if
+ * indeterminate, -1 on error (including EPERM if access not allowed).
+ * If 0 returned, typically that means the eventlog needs to be looked
+ * up and then eventlog_allow() has to be called.
+ */
+int eventlog_allow_lru (struct info_ctx *ctx,
+                        const flux_msg_t *msg,
+                        flux_jobid_t id);
+
 #endif /* ! _FLUX_JOB_INFO_ALLOW_H */
 
 /*
diff --git a/src/modules/job-info/guest_watch.c b/src/modules/job-info/guest_watch.c
index 1f022ca5fc3e..f007d47d8dbc 100644
--- a/src/modules/job-info/guest_watch.c
+++ b/src/modules/job-info/guest_watch.c
@@ -8,23 +8,27 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
-/* guest_watch.c - handle job-info.guest-eventlog-watch &
- * job-info.guest-eventlog-watch-cancel for job-info */
+/* guest_watch.c - handle guest eventlog logic for
+ * job-info.eventlog-watch & job-info.eventlog-watch-cancel for
+ * job-info */
 
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
+#include <limits.h>
 #include <jansson.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libjob/job.h"
 #include "src/common/libeventlog/eventlog.h"
+#include "ccan/str/str.h"
 
-#include "info.h"
+#include "job-info.h"
+#include "util.h"
 #include "watch.h"
 
-/* The callback for job-info.guest-eventlog-watch handles all
+/* This code (entrypoint guest_watch()) handles all
  * of the tricky / racy things related to reading an eventlog from the
  * guest namespace.  Effectively it is a state machine, checking the
  * main job eventlog (via job-info.lookup) to determine what state the
@@ -59,7 +63,7 @@
  *     before we start reading it via a call in #3.
  *
  * 4) If the namespace has not yet been created (event "start" has not
- *    ocurred), must wait for the guest namespace to be created
+ *    occurred), must wait for the guest namespace to be created
  *    (wait_guest_namespace()), then eventually follow the path of
  *    watching events in the guest namespace (#3).
  */
@@ -71,7 +75,8 @@ struct guest_watch_ctx {
     flux_jobid_t id;
     char *path;
     int flags;
-    bool cancel;
+    bool eventlog_watch_canceled;
+    bool cancel;                /* cancel or disconnect */
 
     /* transition possibilities
      *
@@ -128,6 +133,7 @@ static void guest_watch_ctx_destroy (void *data)
 {
     if (data) {
         struct guest_watch_ctx *gw = data;
+        int save_errno = errno;
         flux_msg_decref (gw->msg);
         free (gw->path);
         flux_future_destroy (gw->get_main_eventlog_f);
@@ -135,6 +141,7 @@ static void guest_watch_ctx_destroy (void *data)
         flux_future_destroy (gw->guest_namespace_watch_f);
         flux_future_destroy (gw->main_namespace_lookup_f);
         free (gw);
+        errno = save_errno;
     }
 }
 
@@ -145,7 +152,6 @@ static struct guest_watch_ctx *guest_watch_ctx_create (struct info_ctx *ctx,
                                                        int flags)
 {
     struct guest_watch_ctx *gw = calloc (1, sizeof (*gw));
-    int saved_errno;
 
     if (!gw)
         return NULL;
@@ -169,60 +175,20 @@ static struct guest_watch_ctx *guest_watch_ctx_create (struct info_ctx *ctx,
     return gw;
 
 error:
-    saved_errno = errno;
     guest_watch_ctx_destroy (gw);
-    errno = saved_errno;
     return NULL;
 }
 
-/* we want to copy credentials, etc. from the original
- * message when we redirect to other job-info targets.
- */
-static flux_msg_t *guest_msg_pack (struct guest_watch_ctx *gw,
-                                   const char *topic,
-                                   const char *fmt,
-                                   ...)
-{
-    flux_msg_t *newmsg = NULL;
-    json_t *payload = NULL;
-    char *payloadstr = NULL;
-    flux_msg_t *rv = NULL;
-    int save_errno;
-    va_list ap;
-
-    va_start (ap, fmt);
-
-    if (!(newmsg = flux_request_encode (topic, NULL)))
-        goto error;
-    if (flux_msg_set_cred (newmsg, gw->cred) < 0)
-        goto error;
-    if (!(payload = json_vpack_ex (NULL, 0, fmt, ap)))
-        goto error;
-    if (!(payloadstr = json_dumps (payload, JSON_COMPACT))) {
-        errno = ENOMEM;
-        goto error;
-    }
-    if (flux_msg_set_string (newmsg, payloadstr) < 0)
-        goto error;
-
-    rv = newmsg;
-error:
-    save_errno = errno;
-    if (!rv)
-        flux_msg_destroy (newmsg);
-    json_decref (payload);
-    free (payloadstr);
-    va_end (ap);
-    errno = save_errno;
-    return rv;
-}
-
-static int send_cancel (struct guest_watch_ctx *gw, flux_future_t *f)
+static int send_eventlog_watch_cancel (struct guest_watch_ctx *gw,
+                                       flux_future_t *f,
+                                       bool cancel)
 {
-    if (!gw->cancel) {
+    if (!gw->eventlog_watch_canceled) {
         flux_future_t *f2;
         int matchtag;
 
+        gw->cancel = cancel;
+
         if (!f) {
             if (gw->state == GUEST_WATCH_STATE_WAIT_GUEST_NAMESPACE)
                 f = gw->wait_guest_namespace_f;
@@ -231,20 +197,23 @@ static int send_cancel (struct guest_watch_ctx *gw, flux_future_t *f)
             else if (gw->state == GUEST_WATCH_STATE_MAIN_NAMESPACE_LOOKUP) {
                 /* Since this is a lookup, we don't need to perform an actual
                  * cancel to "job-info.eventlog-watch-cancel".  Just return
-                 * ENODATA to the caller.
+                 * ENODATA to the caller if necessary.
                  */
-                gw->cancel = true;
-                if (flux_respond_error (gw->ctx->h,
-                                        gw->msg,
-                                        ENODATA,
-                                        NULL) < 0)
-                    flux_log_error (gw->ctx->h, "%s: flux_respond_error",
-                                    __FUNCTION__);
+                gw->eventlog_watch_canceled = true;
+                if (gw->cancel) {
+                    if (flux_respond_error (gw->ctx->h,
+                                            gw->msg,
+                                            ENODATA,
+                                            NULL) < 0)
+                        flux_log_error (gw->ctx->h, "%s: flux_respond_error",
+                                        __FUNCTION__);
+                }
                 return 0;
             }
             else {
-                /* gw->state == GUEST_WATCH_STATE_INIT */
-                gw->cancel = true;
+                /* gw->state == GUEST_WATCH_STATE_INIT, eventlog-watch
+                 * never started so sort of "auto-canceled" */
+                gw->eventlog_watch_canceled = true;
                 return 0;
             }
         }
@@ -261,7 +230,7 @@ static int send_cancel (struct guest_watch_ctx *gw, flux_future_t *f)
             return -1;
         }
         flux_future_destroy (f2);
-        gw->cancel = true;
+        gw->eventlog_watch_canceled = true;
     }
     return 0;
 }
@@ -272,12 +241,12 @@ static int get_main_eventlog (struct guest_watch_ctx *gw)
     flux_msg_t *msg = NULL;
     int save_errno, rv = -1;
 
-    if (!(msg = guest_msg_pack (gw,
-                                topic,
-                                "{s:I s:[s] s:i}",
-                                "id", gw->id,
-                                "keys", "eventlog",
-                                "flags", 0)))
+    if (!(msg = cred_msg_pack (topic,
+                               gw->cred,
+                               "{s:I s:[s] s:i}",
+                               "id", gw->id,
+                               "keys", "eventlog",
+                               "flags", 0)))
         goto error;
 
     if (!(gw->get_main_eventlog_f = flux_rpc_message (gw->ctx->h,
@@ -328,13 +297,13 @@ static int check_guest_namespace_status (struct guest_watch_ctx *gw,
         json_t *context = NULL;
         if (eventlog_entry_parse (event, NULL, &name, &context) < 0)
             goto error;
-        if (!strcmp (name, "start"))
+        if (streq (name, "start"))
             gw->guest_started = true;
-        if (!strcmp (name, "release")) {
+        if (streq (name, "release")) {
             void *iter = json_object_iter (context);
             while (iter && !gw->guest_released) {
                 const char *key = json_object_iter_key (iter);
-                if (!strcmp (key, "final")) {
+                if (streq (key, "final")) {
                     json_t *value = json_object_iter_value (iter);
                     if (json_is_boolean (value) && json_is_true (value))
                         gw->guest_released = true;
@@ -362,12 +331,20 @@ static void get_main_eventlog_continuation (flux_future_t *f, void *arg)
         goto error;
     }
 
-    if (gw->cancel) {
-        if (flux_respond_error (ctx->h, gw->msg, ENODATA, NULL) < 0)
-            flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+    if (gw->eventlog_watch_canceled) {
+        if (gw->cancel) {
+            if (flux_respond_error (ctx->h, gw->msg, ENODATA, NULL) < 0)
+                flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+        }
         goto done;
     }
 
+    /* N.B. Check for whether requester should be allowed to read this
+     * eventlog could be done here (eventlog_allow ()), however since
+     * it will be done in the primary watch code anyways, we let the
+     * check fallthrough to be done there.
+     */
+
     if (check_guest_namespace_status (gw, s) < 0)
         goto error;
 
@@ -406,12 +383,12 @@ static int wait_guest_namespace (struct guest_watch_ctx *gw)
     flux_msg_t *msg = NULL;
     int save_errno, rv = -1;
 
-    if (!(msg = guest_msg_pack (gw,
-                                topic,
-                                "{s:I s:s s:i}",
-                                "id", gw->id,
-                                "path", "eventlog",
-                                "flags", 0)))
+    if (!(msg = cred_msg_pack (topic,
+                               gw->cred,
+                               "{s:I s:s s:i}",
+                               "id", gw->id,
+                               "path", "eventlog",
+                               "flags", 0)))
         goto error;
 
     if (!(gw->wait_guest_namespace_f = flux_rpc_message (gw->ctx->h,
@@ -457,7 +434,7 @@ static int check_guest_namespace_created (struct guest_watch_ctx *gw,
         goto error;
     }
 
-    if (!strcmp (name, "start"))
+    if (streq (name, "start"))
         gw->guest_started = true;
 
     /* Do not need to check for "clean", if "start" never occurs, will
@@ -488,9 +465,11 @@ static void wait_guest_namespace_continuation (flux_future_t *f, void *arg)
             if (gw->guest_started) {
                 /* check for racy cancel - user canceled while this
                  * error was in transit */
-                if (gw->cancel) {
+                if (gw->eventlog_watch_canceled) {
                     errno = ENODATA;
-                    goto error;
+                    if (gw->cancel)
+                        goto error;
+                    goto cleanup;
                 }
                 if (guest_namespace_watch (gw) < 0)
                     goto error;
@@ -503,9 +482,11 @@ static void wait_guest_namespace_continuation (flux_future_t *f, void *arg)
         goto error;
     }
 
-    if (gw->cancel) {
+    if (gw->eventlog_watch_canceled) {
         errno = ENODATA;
-        goto error;
+        if (gw->cancel)
+            goto error;
+        goto cleanup;
     }
 
     if (flux_job_event_watch_get (f, &event) < 0) {
@@ -521,8 +502,8 @@ static void wait_guest_namespace_continuation (flux_future_t *f, void *arg)
         flux_future_t *f2;
 
         /* cancel this watcher, and once its canceled, watch the guest
-         * namespace.  Don't call send_cancel(), this is not an error
-         * or "full" cancel */
+         * namespace.  Don't call send_eventlog_watch_cancel(), this
+         * is not an error or "full" cancel */
         if (!(f2 = flux_rpc_pack (gw->ctx->h,
                                   "job-info.eventlog-watch-cancel",
                                   FLUX_NODEID_ANY,
@@ -541,16 +522,18 @@ static void wait_guest_namespace_continuation (flux_future_t *f, void *arg)
 error_cancel:
     /* If we haven't sent a cancellation yet, must do so so that
      * the future's matchtag will eventually be freed */
-    if (!gw->cancel) {
+    if (!gw->eventlog_watch_canceled) {
         int save_errno = errno;
-        (void) send_cancel (gw, gw->wait_guest_namespace_f);
+        (void) send_eventlog_watch_cancel (gw,
+                                           gw->wait_guest_namespace_f,
+                                           false);
         errno = save_errno;
     }
 
 error:
     if (flux_respond_error (ctx->h, gw->msg, errno, NULL) < 0)
         flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
-
+cleanup:
     /* flux future destroyed in guest_watch_ctx_destroy, which is
      * called via zlist_remove() */
     zlist_remove (ctx->guest_watchers, gw);
@@ -564,13 +547,13 @@ static int guest_namespace_watch (struct guest_watch_ctx *gw)
     int save_errno;
     int rv = -1;
 
-    if (!(msg = guest_msg_pack (gw,
-                                topic,
-                                "{s:I s:b s:s s:i}",
-                                "id", gw->id,
-                                "guest", true,
-                                "path", gw->path,
-                                "flags", 0)))
+    if (!(msg = cred_msg_pack (topic,
+                               gw->cred,
+                               "{s:I s:b s:s s:i}",
+                               "id", gw->id,
+                               "guest", true,
+                               "path", gw->path,
+                               "flags", gw->flags)))
         goto error;
 
     if (!(gw->guest_namespace_watch_f = flux_rpc_message (gw->ctx->h,
@@ -624,18 +607,20 @@ static void guest_namespace_watch_continuation (flux_future_t *f, void *arg)
              */
             /* check for racy cancel - user canceled while this
              * error was in transit */
-            if (gw->cancel) {
+            if (gw->eventlog_watch_canceled) {
                 errno = ENODATA;
-                goto error;
+                if (gw->cancel)
+                    goto error;
+                goto cleanup;
             }
             if (main_namespace_lookup (gw) < 0)
                 goto error;
             return;
         }
         else {
-            /* We assume ENODATA always comes from a user cancellation,
-             * or similar error.  There is no circumstance where would
-             * desire to ENODATA this stream.
+            /* Generally speaking we assume ENODATA always comes from
+             * a user cancellation, or similar error.  There is no
+             * circumstance where would desire to ENODATA this stream.
              */
             if (errno != ENOENT && errno != ENODATA)
                 flux_log_error (ctx->h, "%s: flux_rpc_get", __FUNCTION__);
@@ -643,34 +628,35 @@ static void guest_namespace_watch_continuation (flux_future_t *f, void *arg)
         }
     }
 
-    if (gw->cancel) {
-        errno = ENODATA;
-        goto error;
+    if (gw->eventlog_watch_canceled) {
+        if (gw->cancel) {
+            errno = ENODATA;
+            goto error;
+        }
+        goto cleanup;
     }
 
     if (flux_respond_pack (ctx->h, gw->msg, "{s:s}", "event", event) < 0) {
         flux_log_error (ctx->h, "%s: flux_respond_pack",
                         __FUNCTION__);
-        goto error_cancel;
+
+        /* If we haven't sent a cancellation yet, must do so so that
+         * the future's matchtag will eventually be freed */
+        if (!gw->eventlog_watch_canceled)
+            (void) send_eventlog_watch_cancel (gw,
+                                               gw->guest_namespace_watch_f,
+                                               false);
+        goto cleanup;
     }
 
     gw->offset += strlen (event);
     flux_future_reset (f);
     return;
 
-error_cancel:
-    /* If we haven't sent a cancellation yet, must do so so that
-     * the future's matchtag will eventually be freed */
-    if (!gw->cancel) {
-        int save_errno = errno;
-        (void) send_cancel (gw, gw->guest_namespace_watch_f);
-        errno = save_errno;
-    }
-
 error:
     if (flux_respond_error (ctx->h, gw->msg, errno, NULL) < 0)
         flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
-
+cleanup:
     /* flux future destroyed in guest_watch_ctx_destroy, which is called
      * via zlist_remove() */
     zlist_remove (ctx->guest_watchers, gw);
@@ -707,18 +693,18 @@ static int main_namespace_lookup (struct guest_watch_ctx *gw)
      * know that the eventlog is complete, so no need to do a "watch",
      * do a lookup instead */
 
-    if (!(msg = guest_msg_pack (gw,
-                                topic,
-                                "{s:I s:[s] s:i}",
-                                "id", gw->id,
-                                "keys", path,
-                                "flags", 0)))
+    if (!(msg = cred_msg_pack (topic,
+                               gw->cred,
+                               "{s:I s:[s] s:i}",
+                               "id", gw->id,
+                               "keys", path,
+                               "flags", 0)))
         goto error;
 
     if (!(gw->main_namespace_lookup_f = flux_rpc_message (gw->ctx->h,
-                                                         msg,
-                                                         FLUX_NODEID_ANY,
-                                                         0))) {
+                                                          msg,
+                                                          FLUX_NODEID_ANY,
+                                                          0))) {
         flux_log_error (gw->ctx->h, "%s: flux_rpc_message", __FUNCTION__);
         goto error;
     }
@@ -741,19 +727,6 @@ static int main_namespace_lookup (struct guest_watch_ctx *gw)
     return rv;
 }
 
-static bool eventlog_parse_next (const char **pp, const char **tok,
-                                 size_t *toklen)
-{
-    char *term;
-
-    if (!(term = strchr (*pp, '\n')))
-        return false;
-    *tok = *pp;
-    *toklen = term - *pp + 1;
-    *pp = term + 1;
-    return true;
-}
-
 static void main_namespace_lookup_continuation (flux_future_t *f, void *arg)
 {
     struct guest_watch_ctx *gw = arg;
@@ -773,19 +746,20 @@ static void main_namespace_lookup_continuation (flux_future_t *f, void *arg)
         goto error;
     }
 
-    if (gw->cancel) {
-        /* already sent ENODATA via send_cancel(), so just cleanup */
+    if (gw->eventlog_watch_canceled) {
+        /* already sent ENODATA via send_eventlog_watch_cancel(), so
+         * just cleanup */
         goto cleanup;
     }
 
     input = s + gw->offset;
-    while (eventlog_parse_next (&input, &tok, &toklen)) {
+    while (get_next_eventlog_entry (&input, &tok, &toklen)) {
         if (flux_respond_pack (ctx->h, gw->msg,
                                "{s:s#}",
                                "event", tok, toklen) < 0) {
             flux_log_error (ctx->h, "%s: flux_respond_pack",
                             __FUNCTION__);
-            goto error;
+            goto cleanup;
         }
     }
 
@@ -802,29 +776,13 @@ static void main_namespace_lookup_continuation (flux_future_t *f, void *arg)
     zlist_remove (ctx->guest_watchers, gw);
 }
 
-void guest_watch_cb (flux_t *h, flux_msg_handler_t *mh,
-                     const flux_msg_t *msg, void *arg)
+int guest_watch (struct info_ctx *ctx,
+                 const flux_msg_t *msg,
+                 flux_jobid_t id,
+                 const char *path,
+                 int flags)
 {
-    struct info_ctx *ctx = arg;
     struct guest_watch_ctx *gw = NULL;
-    flux_jobid_t id;
-    const char *path = NULL;
-    int flags;
-    const char *errmsg = NULL;
-
-    if (flux_request_unpack (msg, NULL, "{s:I s:s s:i}",
-                             "id", &id,
-                             "path", &path,
-                             "flags", &flags) < 0) {
-        flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
-        goto error;
-    }
-    if (!flux_msg_is_streaming (msg)) {
-        errno = EPROTO;
-        errmsg = "guest-eventlog-watch request rejected without streaming "
-                 "RPC flag";
-        goto error;
-    }
 
     if (!(gw = guest_watch_ctx_create (ctx, msg, id, path, flags)))
         goto error;
@@ -833,77 +791,54 @@ void guest_watch_cb (flux_t *h, flux_msg_handler_t *mh,
         goto error;
 
     if (zlist_append (ctx->guest_watchers, gw) < 0) {
-        flux_log_error (h, "%s: zlist_append", __FUNCTION__);
+        flux_log_error (ctx->h, "%s: zlist_append", __FUNCTION__);
         goto error;
     }
     zlist_freefn (ctx->guest_watchers, gw, guest_watch_ctx_destroy, true);
     gw = NULL;
 
-    return;
+    return 0;
 
 error:
-    if (flux_respond_error (h, msg, errno, errmsg) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
     guest_watch_ctx_destroy (gw);
+    return -1;
 }
 
-/* Cancel guest_watch 'gw' if it matches (sender, matchtag).
- * matchtag=FLUX_MATCHTAG_NONE matches any matchtag.
+/* Cancel guest_watch 'gw' if it matches message
  */
 static void guest_watch_cancel (struct info_ctx *ctx,
                                 struct guest_watch_ctx *gw,
-                                const char *sender, uint32_t matchtag)
+                                const flux_msg_t *msg,
+                                bool cancel)
 {
-    uint32_t t;
-    char *s;
-
-    if (matchtag != FLUX_MATCHTAG_NONE
-        && (flux_msg_get_matchtag (gw->msg, &t) < 0 || matchtag != t))
-        return;
-    if (flux_msg_get_route_first (gw->msg, &s) < 0)
-        return;
-    if (!strcmp (sender, s))
-        send_cancel (gw, NULL);
-    free (s);
+    bool match;
+    if (cancel)
+        match = flux_cancel_match (msg, gw->msg);
+    else
+        match = flux_disconnect_match (msg, gw->msg);
+    if (match)
+        send_eventlog_watch_cancel (gw, NULL, cancel);
 }
 
 void guest_watchers_cancel (struct info_ctx *ctx,
-                            const char *sender, uint32_t matchtag)
+                            const flux_msg_t *msg,
+                            bool cancel)
 {
     struct guest_watch_ctx *gw;
 
     gw = zlist_first (ctx->guest_watchers);
     while (gw) {
-        guest_watch_cancel (ctx, gw, sender, matchtag);
+        guest_watch_cancel (ctx, gw, msg, cancel);
         gw = zlist_next (ctx->guest_watchers);
     }
 }
 
-void guest_watch_cancel_cb (flux_t *h, flux_msg_handler_t *mh,
-                            const flux_msg_t *msg, void *arg)
-{
-    struct info_ctx *ctx = arg;
-    uint32_t matchtag;
-    char *sender;
-
-    if (flux_request_unpack (msg, NULL, "{s:i}", "matchtag", &matchtag) < 0) {
-        flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
-        return;
-    }
-    if (flux_msg_get_route_first (msg, &sender) < 0) {
-        flux_log_error (h, "%s: flux_msg_get_route_first", __FUNCTION__);
-        return;
-    }
-    guest_watchers_cancel (ctx, sender, matchtag);
-    free (sender);
-}
-
 void guest_watch_cleanup (struct info_ctx *ctx)
 {
     struct guest_watch_ctx *gw;
 
     while ((gw = zlist_pop (ctx->guest_watchers))) {
-        send_cancel (gw, NULL);
+        send_eventlog_watch_cancel (gw, NULL, false);
 
         if (flux_respond_error (ctx->h, gw->msg, ENOSYS, NULL) < 0)
             flux_log_error (ctx->h, "%s: flux_respond_error",
diff --git a/src/modules/job-info/guest_watch.h b/src/modules/job-info/guest_watch.h
index 9b707a79194e..b7a71b8903c6 100644
--- a/src/modules/job-info/guest_watch.h
+++ b/src/modules/job-info/guest_watch.h
@@ -13,19 +13,25 @@
 
 #include <flux/core.h>
 
-void guest_watch_cb (flux_t *h, flux_msg_handler_t *mh,
-                     const flux_msg_t *msg, void *arg);
+#include "job-info.h"
 
-void guest_watch_cancel_cb (flux_t *h, flux_msg_handler_t *mh,
-                            const flux_msg_t *msg, void *arg);
+int guest_watch (struct info_ctx *ctx,
+                 const flux_msg_t *msg,
+                 flux_jobid_t id,
+                 const char *path,
+                 int flags);
 
-/* Cancel all lookups that match (sender, matchtag). */
+/* Cancel all lookups that match msg.
+ * match credentials & matchtag if cancel true
+ * match credentials if cancel false
+ */
 void guest_watchers_cancel (struct info_ctx *ctx,
-                            const char *sender, uint32_t matchtag);
+                            const flux_msg_t *msg,
+                            bool cancel);
 
 void guest_watch_cleanup (struct info_ctx *ctx);
 
-#endif /* ! _FLUX_JOB_INFO_EVENTLOG_GUEST_WATCH_H */
+#endif /* ! _FLUX_JOB_INFO_GUEST_WATCH_H */
 
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/modules/job-info/idsync.c b/src/modules/job-info/idsync.c
deleted file mode 100644
index ba71f010af7d..000000000000
--- a/src/modules/job-info/idsync.c
+++ /dev/null
@@ -1,109 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* job_util.c - job utility functions */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <czmq.h>
-#include <jansson.h>
-#include <flux/core.h>
-
-#include "src/common/libutil/errno_safe.h"
-#include "src/common/libjob/job_hash.h"
-
-#include "idsync.h"
-#include "job_state.h"
-
-void idsync_data_destroy (void *data)
-{
-    if (data) {
-        struct idsync_data *isd = data;
-        flux_msg_destroy (isd->msg);
-        json_decref (isd->attrs);
-        flux_future_destroy (isd->f_lookup);
-        free (isd);
-    }
-}
-
-void idsync_data_destroy_wrapper (void **data)
-{
-    if (data) {
-        struct idsync_data **isd = (struct idsync_data **) data;
-        idsync_data_destroy (*isd);
-    }
-}
-
-struct idsync_data *idsync_data_create (struct info_ctx *ctx,
-                                        flux_jobid_t id,
-                                        const flux_msg_t *msg,
-                                        json_t *attrs,
-                                        flux_future_t *f_lookup)
-{
-    struct idsync_data *isd = NULL;
-    int saved_errno;
-
-    isd = calloc (1, sizeof (*isd));
-    if (!isd)
-        goto error_enomem;
-    isd->ctx = ctx;
-    isd->id = id;
-    if (!(isd->msg = flux_msg_copy (msg, false)))
-        goto error;
-    isd->attrs = json_incref (attrs);
-    isd->f_lookup = f_lookup;
-    return isd;
-
- error_enomem:
-    errno = ENOMEM;
- error:
-    saved_errno = errno;
-    idsync_data_destroy (isd);
-    errno = saved_errno;
-    return NULL;
-}
-
-void idsync_waits_list_destroy (void **data)
-{
-    if (data)
-        zlistx_destroy ((zlistx_t **) data);
-}
-
-int idsync_setup (struct info_ctx *ctx)
-{
-    if (!(ctx->idsync_lookups = zlistx_new ()))
-        return -1;
-    zlistx_set_destructor (ctx->idsync_lookups, idsync_data_destroy_wrapper);
-    if (!(ctx->idsync_waits = job_hash_create ()))
-        return -1;
-    zhashx_set_destructor (ctx->idsync_waits, idsync_waits_list_destroy);
-    return 0;
-}
-
-void idsync_cleanup (struct info_ctx *ctx)
-{
-    struct idsync_data *isd;
-    isd = zlistx_first (ctx->idsync_lookups);
-    while (isd) {
-        if (isd->f_lookup) {
-            if (flux_future_get (isd->f_lookup, NULL) < 0)
-                flux_log_error (ctx->h, "%s: flux_future_get",
-                                __FUNCTION__);
-        }
-        isd = zlistx_next (ctx->idsync_lookups);
-    }
-    zlistx_destroy (&ctx->idsync_lookups);
-    zhashx_destroy (&ctx->idsync_waits);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-info/idsync.h b/src/modules/job-info/idsync.h
deleted file mode 100644
index ce51556c58fc..000000000000
--- a/src/modules/job-info/idsync.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_JOB_INFO_IDSYNC_H
-#define _FLUX_JOB_INFO_IDSYNC_H
-
-#include <flux/core.h>
-
-#include "info.h"
-#include "job_state.h"
-
-struct idsync_data {
-    struct info_ctx *ctx;
-    flux_jobid_t id;
-    flux_msg_t *msg;
-    json_t *attrs;
-
-    flux_future_t *f_lookup;
-};
-
-int idsync_setup (struct info_ctx *ctx);
-
-void idsync_cleanup (struct info_ctx *ctx);
-
-void idsync_data_destroy (void *data);
-
-void idsync_data_destroy_wrapper (void **data);
-
-struct idsync_data *idsync_data_create (struct info_ctx *ctx,
-                                        flux_jobid_t id,
-                                        const flux_msg_t *msg,
-                                        json_t *attrs,
-                                        flux_future_t *f_lookup);
-
-#endif /* ! _FLUX_JOB_INFO_IDSYNC_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-info/info.h b/src/modules/job-info/info.h
deleted file mode 100644
index de91f83b7114..000000000000
--- a/src/modules/job-info/info.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_JOB_INFO_INFO_H
-#define _FLUX_JOB_INFO_INFO_H
-
-#include <flux/core.h>
-#include <czmq.h>
-
-#include "job_state.h"
-
-struct info_ctx {
-    flux_t *h;
-    flux_msg_handler_t **handlers;
-    zlist_t *lookups;
-    zlist_t *watchers;
-    zlist_t *guest_watchers;
-    struct job_state_ctx *jsctx;
-    zlistx_t *idsync_lookups;
-    zhashx_t *idsync_waits;
-};
-
-#endif /* _FLUX_JOB_INFO_INFO_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/src/modules/job-info/job-info.c b/src/modules/job-info/job-info.c
index bc30eb39fed6..730bc4c7f1cd 100644
--- a/src/modules/job-info/job-info.c
+++ b/src/modules/job-info/job-info.c
@@ -11,87 +11,47 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
 #include <flux/core.h>
 
-#include "info.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "job-info.h"
 #include "allow.h"
-#include "job_state.h"
-#include "list.h"
 #include "lookup.h"
 #include "watch.h"
 #include "guest_watch.h"
-#include "idsync.h"
+#include "update.h"
 
-static void disconnect_cb (flux_t *h, flux_msg_handler_t *mh,
-                           const flux_msg_t *msg, void *arg)
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
 {
     struct info_ctx *ctx = arg;
-    char *sender;
-
-    if (flux_request_decode (msg, NULL, NULL) < 0) {
-        flux_log_error (h, "%s: flux_request_decode", __FUNCTION__);
-        return;
-    }
-    if (flux_msg_get_route_first (msg, &sender) < 0) {
-        flux_log_error (h, "%s: flux_msg_get_route_first", __FUNCTION__);
-        return;
-    }
-    watchers_cancel (ctx, sender, FLUX_MATCHTAG_NONE);
-    guest_watchers_cancel (ctx, sender, FLUX_MATCHTAG_NONE);
-    free (sender);
+    watchers_cancel (ctx, msg, false);
+    guest_watchers_cancel (ctx, msg, false);
+    update_watchers_cancel (ctx, msg, false);
 }
 
-static void stats_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+static void stats_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     struct info_ctx *ctx = arg;
     int lookups = zlist_size (ctx->lookups);
     int watchers = zlist_size (ctx->watchers);
     int guest_watchers = zlist_size (ctx->guest_watchers);
-    int pending = zlistx_size (ctx->jsctx->pending);
-    int running = zlistx_size (ctx->jsctx->running);
-    int inactive = zlistx_size (ctx->jsctx->inactive);
-    int idsync_lookups = zlistx_size (ctx->idsync_lookups);
-    int idsync_waits = zhashx_size (ctx->idsync_waits);
-    if (flux_respond_pack (h, msg, "{s:i s:i s:i s:{s:i s:i s:i} s:{s:i s:i}}",
+    int update_lookups = 0;     /* no longer supported */
+    int update_watchers = update_watch_count (ctx);
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:i s:i s:i s:i s:i}",
                            "lookups", lookups,
                            "watchers", watchers,
                            "guest_watchers", guest_watchers,
-                           "jobs",
-                           "pending", pending,
-                           "running", running,
-                           "inactive", inactive,
-                           "idsync",
-                           "lookups", idsync_lookups,
-                           "waits", idsync_waits) < 0) {
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-        goto error;
-    }
-
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-static void job_stats_cb (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg)
-{
-    struct info_ctx *ctx = arg;
-    int total = ctx->jsctx->depend_count + ctx->jsctx->sched_count +
-        ctx->jsctx->run_count + ctx->jsctx->cleanup_count +
-        ctx->jsctx->inactive_count;
-    if (flux_respond_pack (h,
-                           msg,
-                           "{s:{s:i s:i s:i s:i s:i s:i}}",
-                           "job_states",
-                           "depend", ctx->jsctx->depend_count,
-                           "sched", ctx->jsctx->sched_count,
-                           "run", ctx->jsctx->run_count,
-                           "cleanup", ctx->jsctx->cleanup_count,
-                           "inactive", ctx->jsctx->inactive_count,
-                           "total", total) < 0) {
+                           "update_lookups", update_lookups,
+                           "update_watchers", update_watchers) < 0) {
         flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
         goto error;
     }
@@ -119,65 +79,30 @@ static const struct flux_msg_handler_spec htab[] = {
       .rolemask     = FLUX_ROLE_USER
     },
     { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "job-info.guest-eventlog-watch",
-      .cb           = guest_watch_cb,
-      .rolemask     = FLUX_ROLE_USER
-    },
-    { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "job-info.guest-eventlog-watch-cancel",
-      .cb           = guest_watch_cancel_cb,
+      .topic_glob   = "job-info.update-lookup",
+      .cb           = update_lookup_cb,
       .rolemask     = FLUX_ROLE_USER
     },
     { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "job-info.list",
-      .cb           = list_cb,
+      .topic_glob   = "job-info.update-watch",
+      .cb           = update_watch_cb,
       .rolemask     = FLUX_ROLE_USER
     },
     { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "job-info.list-inactive",
-      .cb           = list_inactive_cb,
-      .rolemask     = FLUX_ROLE_USER
-    },
-    { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "job-info.list-id",
-      .cb           = list_id_cb,
-      .rolemask     = FLUX_ROLE_USER
-    },
-    { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "job-info.list-attrs",
-      .cb           = list_attrs_cb,
-      .rolemask     = FLUX_ROLE_USER
-    },
-    { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "job-info.job-state-pause",
-      .cb           = job_state_pause_cb,
-      .rolemask     = FLUX_ROLE_USER
-    },
-    { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "job-info.job-state-unpause",
-      .cb           = job_state_unpause_cb,
-      .rolemask     = FLUX_ROLE_USER
-    },
-    { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "job-info.job-stats",
-      .cb           = job_stats_cb,
+      .topic_glob   = "job-info.update-watch-cancel",
+      .cb           = update_watch_cancel_cb,
       .rolemask     = FLUX_ROLE_USER
     },
     { .typemask     = FLUX_MSGTYPE_REQUEST,
       .topic_glob   = "job-info.disconnect",
       .cb           = disconnect_cb,
-      .rolemask     = 0
+      .rolemask     = FLUX_ROLE_USER
     },
     { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "job-info.stats.get",
+      .topic_glob   = "job-info.stats-get",
       .cb           = stats_cb,
       .rolemask     = 0
     },
-    { .typemask     = FLUX_MSGTYPE_EVENT,
-      .topic_glob   = "job-state",
-      .cb           = job_state_cb,
-      .rolemask     = 0
-    },
     FLUX_MSGHANDLER_TABLE_END,
 };
 
@@ -186,6 +111,7 @@ static void info_ctx_destroy (struct info_ctx *ctx)
     if (ctx) {
         int saved_errno = errno;
         flux_msg_handler_delvec (ctx->handlers);
+        lru_cache_destroy (ctx->owner_lru);
         /* freefn set on lookup entries will destroy list entries */
         if (ctx->lookups)
             zlist_destroy (&ctx->lookups);
@@ -197,10 +123,12 @@ static void info_ctx_destroy (struct info_ctx *ctx)
             guest_watch_cleanup (ctx);
             zlist_destroy (&ctx->guest_watchers);
         }
-        if (ctx->jsctx)
-            job_state_destroy (ctx->jsctx);
-        if (ctx->idsync_lookups)
-            idsync_cleanup (ctx);
+        if (ctx->update_watchers) {
+            update_watch_cleanup (ctx);
+            zlist_destroy (&ctx->update_watchers);
+        }
+        if (ctx->index_uw)
+            zhashx_destroy (&ctx->index_uw);
         free (ctx);
         errno = saved_errno;
     }
@@ -214,15 +142,20 @@ static struct info_ctx *info_ctx_create (flux_t *h)
     ctx->h = h;
     if (flux_msg_handler_addvec (h, htab, ctx, &ctx->handlers) < 0)
         goto error;
+    if (!(ctx->owner_lru = lru_cache_create (OWNER_LRU_MAXSIZE)))
+        goto error;
+    lru_cache_set_free_f (ctx->owner_lru, (lru_cache_free_f)free);
     if (!(ctx->lookups = zlist_new ()))
         goto error;
     if (!(ctx->watchers = zlist_new ()))
         goto error;
     if (!(ctx->guest_watchers = zlist_new ()))
         goto error;
-    if (!(ctx->jsctx = job_state_create (h)))
+    if (!(ctx->update_watchers = zlist_new ()))
         goto error;
-    if (idsync_setup (ctx) < 0)
+    /* no destructor for index_uw, destruction handled on
+     * update_watchers list */
+    if (!(ctx->index_uw = zhashx_new ()))
         goto error;
     return ctx;
 error:
@@ -239,8 +172,6 @@ int mod_main (flux_t *h, int argc, char **argv)
         flux_log_error (h, "initialization error");
         goto done;
     }
-    if (job_state_init_from_kvs (ctx) < 0)
-        goto done;
     if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
         goto done;
     rc = 0;
@@ -249,8 +180,6 @@ int mod_main (flux_t *h, int argc, char **argv)
     return rc;
 }
 
-MOD_NAME ("job-info");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/job-info/job-info.h b/src/modules/job-info/job-info.h
new file mode 100644
index 000000000000..8761d5131dd0
--- /dev/null
+++ b/src/modules/job-info/job-info.h
@@ -0,0 +1,38 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_INFO_H
+#define _FLUX_JOB_INFO_H
+
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libjob/job_hash.h"
+#include "src/common/libutil/lru_cache.h"
+
+#define OWNER_LRU_MAXSIZE 1000
+
+struct info_ctx {
+    flux_t *h;
+    flux_msg_handler_t **handlers;
+    lru_cache_t *owner_lru; /* jobid -> owner LRU */
+    zlist_t *lookups;
+    zlist_t *watchers;
+    zlist_t *guest_watchers;
+    zlist_t *update_watchers;
+    zhashx_t *index_uw;        /* update_watchers lookup */
+};
+
+#endif /* _FLUX_JOB_INFO_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/modules/job-info/job_state.c b/src/modules/job-info/job_state.c
deleted file mode 100644
index cd03f2b80ffb..000000000000
--- a/src/modules/job-info/job_state.c
+++ /dev/null
@@ -1,1369 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* job_state.c - store information on state of jobs */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <czmq.h>
-#include <jansson.h>
-#include <flux/core.h>
-
-#include "src/common/libeventlog/eventlog.h"
-#include "src/common/libutil/fluid.h"
-#include "src/common/libjob/job_hash.h"
-#include "src/common/libidset/idset.h"
-
-#include "job_state.h"
-#include "idsync.h"
-#include "job_util.h"
-
-#define NUMCMP(a,b) ((a)==(b)?0:((a)<(b)?-1:1))
-
-struct state_transition {
-    flux_job_state_t state;
-    bool processed;
-    double timestamp;
-};
-
-static void process_next_state (struct info_ctx *ctx, struct job *job);
-
-/* Compare items for sorting in list, priority first (higher priority
- * before lower priority), t_submit second (earlier submission time
- * first) N.B. zlistx_comparator_fn signature
- */
-static int job_priority_cmp (const void *a1, const void *a2)
-{
-    const struct job *j1 = a1;
-    const struct job *j2 = a2;
-    int rc;
-
-    if ((rc = (-1)*NUMCMP (j1->priority, j2->priority)) == 0)
-        rc = NUMCMP (j1->t_submit, j2->t_submit);
-    return rc;
-}
-
-/* Compare items for sorting in list by timestamp (note that sorting
- * is in reverse order, most recently (i.e. bigger timestamp)
- * running/completed comes first).  N.B. zlistx_comparator_fn
- * signature
- */
-static int job_running_cmp (const void *a1, const void *a2)
-{
-    const struct job *j1 = a1;
-    const struct job *j2 = a2;
-
-    return NUMCMP (j2->t_run, j1->t_run);
-}
-
-static int job_inactive_cmp (const void *a1, const void *a2)
-{
-    const struct job *j1 = a1;
-    const struct job *j2 = a2;
-
-    return NUMCMP (j2->t_inactive, j1->t_inactive);
-}
-
-static void job_destroy (void *data)
-{
-    struct job *job = data;
-    if (job) {
-        json_decref (job->jobspec_job);
-        json_decref (job->jobspec_cmd);
-        json_decref (job->R);
-        free (job->ranks);
-        zlist_destroy (&job->next_states);
-        free (job);
-    }
-}
-
-static void job_destroy_wrapper (void **data)
-{
-    struct job **job = (struct job **)data;
-    job_destroy (*job);
-}
-
-void flux_msg_destroy_wrapper (void **data)
-{
-    if (data) {
-        flux_msg_t **ptr = (flux_msg_t **)data;
-        flux_msg_destroy (*ptr);
-    }
-}
-
-static struct job *job_create (struct info_ctx *ctx, flux_jobid_t id)
-{
-    struct job *job = NULL;
-
-    if (!(job = calloc (1, sizeof (*job))))
-        return NULL;
-    job->ctx = ctx;
-    job->id = id;
-    job->state = FLUX_JOB_NEW;
-
-    if (!(job->next_states = zlist_new ())) {
-        errno = ENOMEM;
-        job_destroy (job);
-        return NULL;
-    }
-
-    return job;
-}
-
-struct job_state_ctx *job_state_create (flux_t *h)
-{
-    struct job_state_ctx *jsctx = NULL;
-    int saved_errno;
-
-    if (!(jsctx = calloc (1, sizeof (*jsctx)))) {
-        flux_log_error (h, "calloc");
-        return NULL;
-    }
-    jsctx->h = h;
-
-    /* Index is the primary data structure holding the job data
-     * structures.  It is responsible for destruction.  Lists only
-     * contain desired sort of jobs.
-     */
-
-    if (!(jsctx->index = job_hash_create ()))
-        goto error;
-    zhashx_set_destructor (jsctx->index, job_destroy_wrapper);
-
-    if (!(jsctx->pending = zlistx_new ()))
-        goto error;
-    zlistx_set_comparator (jsctx->pending, job_priority_cmp);
-
-    if (!(jsctx->running = zlistx_new ()))
-        goto error;
-    zlistx_set_comparator (jsctx->running, job_running_cmp);
-
-    if (!(jsctx->inactive = zlistx_new ()))
-        goto error;
-    zlistx_set_comparator (jsctx->inactive, job_inactive_cmp);
-
-    if (!(jsctx->processing = zlistx_new ()))
-        goto error;
-
-    if (!(jsctx->futures = zlistx_new ()))
-        goto error;
-
-    if (!(jsctx->transitions = zlistx_new ()))
-        goto error;
-    zlistx_set_destructor (jsctx->transitions, flux_msg_destroy_wrapper);
-
-    if (flux_event_subscribe (h, "job-state") < 0) {
-        flux_log_error (h, "flux_event_subscribe");
-        goto error;
-    }
-
-    return jsctx;
-
-error:
-    saved_errno = errno;
-    job_state_destroy (jsctx);
-    errno = saved_errno;
-    return NULL;
-}
-
-void job_state_destroy (void *data)
-{
-    struct job_state_ctx *jsctx = data;
-    if (jsctx) {
-        /* Don't destroy processing until futures are complete */
-        if (jsctx->futures) {
-            flux_future_t *f;
-            f = zlistx_first (jsctx->futures);
-            while (f) {
-                if (flux_future_get (f, NULL) < 0)
-                    flux_log_error (jsctx->h, "%s: flux_future_get",
-                                    __FUNCTION__);
-                flux_future_destroy (f);
-                f = zlistx_next (jsctx->futures);
-            }
-            zlistx_destroy (&jsctx->futures);
-        }
-        /* Destroy index last, as it is the one that will actually
-         * destroy the job objects */
-        if (jsctx->processing)
-            zlistx_destroy (&jsctx->processing);
-        if (jsctx->inactive)
-            zlistx_destroy (&jsctx->inactive);
-        if (jsctx->running)
-            zlistx_destroy (&jsctx->running);
-        if (jsctx->pending)
-            zlistx_destroy (&jsctx->pending);
-        if (jsctx->index)
-            zhashx_destroy (&jsctx->index);
-        if (jsctx->transitions)
-            zlistx_destroy (&jsctx->transitions);
-        (void)flux_event_unsubscribe (jsctx->h, "job-state");
-        free (jsctx);
-    }
-}
-
-/* zlistx_insert() and zlistx_reorder() take a 'low_value' parameter
- * which indicates which end of the list to search from.
- * false=search begins at tail (lowest priority, youngest)
- * true=search begins at head (highest priority, oldest)
- * Attempt to minimize search distance based on job priority.
- */
-static bool search_direction (struct job *job)
-{
-    if (job->priority > FLUX_JOB_PRIORITY_DEFAULT)
-        return true;
-    else
-        return false;
-}
-
-static int *state_counter (struct info_ctx *ctx,
-                           struct job *job,
-                           flux_job_state_t state)
-{
-    if (state == FLUX_JOB_NEW)
-        return NULL;
-    else if (state == FLUX_JOB_DEPEND)
-        return &ctx->jsctx->depend_count;
-    else if (state == FLUX_JOB_SCHED)
-        return &ctx->jsctx->sched_count;
-    else if (state == FLUX_JOB_RUN)
-        return &ctx->jsctx->run_count;
-    else if (state == FLUX_JOB_CLEANUP)
-        return &ctx->jsctx->cleanup_count;
-    else if (state == FLUX_JOB_INACTIVE)
-        return &ctx->jsctx->inactive_count;
-
-    flux_log_error (ctx->h, "illegal state transition for job %llu: %d",
-                    (unsigned long long)job->id, state);
-    return NULL;
-}
-
-static void update_job_state (struct info_ctx *ctx,
-                              struct job *job,
-                              flux_job_state_t new_state,
-                              double timestamp)
-{
-    int *decrement;
-    int *increment;
-
-    decrement = state_counter (ctx, job, job->state);
-    increment = state_counter (ctx, job, new_state);
-    job->state = new_state;
-    if (job->state == FLUX_JOB_DEPEND)
-        job->t_submit = timestamp;
-    else if (job->state == FLUX_JOB_SCHED)
-        job->t_sched = timestamp;
-    else if (job->state == FLUX_JOB_RUN)
-        job->t_run = timestamp;
-    else if (job->state == FLUX_JOB_CLEANUP)
-        job->t_cleanup = timestamp;
-    else if (job->state == FLUX_JOB_INACTIVE)
-        job->t_inactive = timestamp;
-    job->states_mask |= job->state;
-    if (decrement)
-        (*decrement)--;
-    if (increment)
-        (*increment)++;
-}
-
-static void job_insert_list (struct job_state_ctx *jsctx,
-                             struct job *job,
-                             flux_job_state_t newstate)
-{
-    /* Note: comparator is set for running & inactive lists, but the
-     * sort calls are not called on zlistx_add_start() */
-    if (newstate == FLUX_JOB_DEPEND
-        || newstate == FLUX_JOB_SCHED) {
-        if (!(job->list_handle = zlistx_insert (jsctx->pending,
-                                                job,
-                                                search_direction (job))))
-            flux_log_error (jsctx->h, "%s: zlistx_insert",
-                            __FUNCTION__);
-    }
-    else if (newstate == FLUX_JOB_RUN
-             || newstate == FLUX_JOB_CLEANUP) {
-        if (!(job->list_handle = zlistx_add_start (jsctx->running,
-                                                   job)))
-            flux_log_error (jsctx->h, "%s: zlistx_add_start",
-                            __FUNCTION__);
-    }
-    else { /* newstate == FLUX_JOB_INACTIVE */
-        if (!(job->list_handle = zlistx_add_start (jsctx->inactive,
-                                                   job)))
-            flux_log_error (jsctx->h, "%s: zlistx_add_start",
-                            __FUNCTION__);
-    }
-}
-
-/* remove job from one list and move it to another based on the
- * newstate */
-static void job_change_list (struct job_state_ctx *jsctx,
-                             struct job *job,
-                             zlistx_t *oldlist,
-                             flux_job_state_t newstate)
-{
-    if (zlistx_detach (oldlist, job->list_handle) < 0)
-        flux_log_error (jsctx->h, "%s: zlistx_detach",
-                        __FUNCTION__);
-    job->list_handle = NULL;
-
-    job_insert_list (jsctx, job, newstate);
-}
-
-static zlistx_t *get_list (struct job_state_ctx *jsctx, flux_job_state_t state)
-{
-    if (state == FLUX_JOB_NEW)
-        return jsctx->processing;
-    else if (state == FLUX_JOB_DEPEND
-             || state == FLUX_JOB_SCHED)
-        return jsctx->pending;
-    else if (state == FLUX_JOB_RUN
-             || state == FLUX_JOB_CLEANUP)
-        return jsctx->running;
-    else /* state == FLUX_JOB_INACTIVE */
-        return jsctx->inactive;
-}
-
-static void update_job_state_and_list (struct info_ctx *ctx,
-                                       struct job *job,
-                                       flux_job_state_t newstate,
-                                       double timestamp)
-{
-    zlistx_t *oldlist, *newlist;
-    struct job_state_ctx *jsctx = job->ctx->jsctx;
-
-    oldlist = get_list (jsctx, job->state);
-    newlist = get_list (jsctx, newstate);
-
-    /* must call before job_change_list(), to ensure timestamps are
-     * set before any sorting based on timestamps are done
-     */
-    update_job_state (ctx, job, newstate, timestamp);
-
-    if (oldlist != newlist)
-        job_change_list (jsctx, job, oldlist, newstate);
-}
-
-static void list_id_respond (struct info_ctx *ctx,
-                             struct idsync_data *isd,
-                             struct job *job)
-{
-    json_t *o;
-
-    if (!(o = job_to_json (job, isd->attrs)))
-        goto error;
-
-    if (flux_respond_pack (ctx->h, isd->msg, "{s:O}", "job", o) < 0) {
-        flux_log_error (ctx->h, "%s: flux_respond_pack", __FUNCTION__);
-        goto error;
-    }
-
-    json_decref (o);
-    return;
-
-error:
-    if (flux_respond_error (ctx->h, isd->msg, errno, NULL) < 0)
-        flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
-    json_decref (o);
-}
-
-static void check_waiting_id (struct info_ctx *ctx,
-                              struct job *job)
-{
-    zlistx_t *list_isd;
-
-    if ((list_isd = zhashx_lookup (ctx->idsync_waits, &job->id))) {
-        struct idsync_data *isd;
-        isd = zlistx_first (list_isd);
-        while (isd) {
-            list_id_respond (ctx, isd, job);
-            isd = zlistx_next (list_isd);
-        }
-        zhashx_delete (ctx->idsync_waits, &job->id);
-    }
-}
-
-static int eventlog_lookup_parse (struct info_ctx *ctx,
-                                  struct job *job,
-                                  const char *s)
-{
-    json_t *a = NULL;
-    size_t index;
-    json_t *value;
-    int rc = -1;
-
-    if (!(a = eventlog_decode (s))) {
-        flux_log_error (ctx->h, "%s: error parsing eventlog for %llu",
-                        __FUNCTION__, (unsigned long long)job->id);
-        goto out;
-    }
-
-    json_array_foreach (a, index, value) {
-        const char *name;
-        double timestamp;
-        json_t *context = NULL;
-
-        if (eventlog_entry_parse (value, &timestamp, &name, &context) < 0) {
-            flux_log_error (ctx->h, "%s: error parsing entry for %llu",
-                            __FUNCTION__, (unsigned long long)job->id);
-            goto out;
-        }
-
-        if (!strcmp (name, "submit")) {
-            if (!context) {
-                flux_log_error (ctx->h, "%s: no submit context for %llu",
-                                __FUNCTION__, (unsigned long long)job->id);
-                goto out;
-            }
-
-            if (json_unpack (context, "{ s:i s:i s:i }",
-                             "priority", &job->priority,
-                             "userid", &job->userid,
-                             "flags", &job->flags) < 0) {
-                flux_log_error (ctx->h, "%s: submit context for %llu invalid",
-                                __FUNCTION__, (unsigned long long)job->id);
-                goto out;
-            }
-            break;
-        }
-    }
-
-    rc = 0;
-out:
-    json_decref (a);
-    return rc;
-}
-
-struct res_level {
-    const char *type;
-    int count;
-    json_t *with;
-};
-
-static int parse_res_level (struct info_ctx *ctx,
-                            struct job *job,
-                            json_t *o,
-                            struct res_level *resp)
-{
-    json_error_t error;
-    struct res_level res;
-
-    res.with = NULL;
-    /* For jobspec version 1, expect exactly one array element per level.
-     */
-    if (json_unpack_ex (o, &error, 0,
-                        "[{s:s s:i s?o}]",
-                        "type", &res.type,
-                        "count", &res.count,
-                        "with", &res.with) < 0) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu invalid jobspec: %s",
-                  __FUNCTION__, (unsigned long long)job->id, error.text);
-        return -1;
-    }
-    *resp = res;
-    return 0;
-}
-
-/* Return basename of path if there is a '/' in path.  Otherwise return
- * full path */
-const char *
-parse_job_name (const char *path)
-{
-    char *p = strrchr (path, '/');
-    if (p) {
-        p++;
-        /* user mistake, specified a directory with trailing '/',
-         * return full path */
-        if (*p == '\0')
-            return path;
-        return p;
-    }
-    return path;
-}
-
-static int jobspec_parse (struct info_ctx *ctx,
-                          struct job *job,
-                          const char *s)
-{
-    json_error_t error;
-    json_t *jobspec = NULL;
-    json_t *tasks, *resources, *command, *jobspec_job = NULL;
-    int rc = -1;
-
-    if (!(jobspec = json_loads (s, 0, &error))) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu invalid jobspec: %s",
-                  __FUNCTION__, (unsigned long long)job->id, error.text);
-        goto error;
-    }
-
-    if (json_unpack_ex (jobspec, &error, 0,
-                        "{s:{s:{s?:o}}}",
-                        "attributes",
-                        "system",
-                        "job",
-                        &jobspec_job) < 0) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu invalid jobspec: %s",
-                  __FUNCTION__, (unsigned long long)job->id, error.text);
-        goto error;
-    }
-
-    if (jobspec_job) {
-        if (!json_is_object (jobspec_job)) {
-            flux_log (ctx->h, LOG_ERR,
-                      "%s: job %llu invalid jobspec",
-                      __FUNCTION__, (unsigned long long)job->id);
-            goto error;
-        }
-        job->jobspec_job = json_incref (jobspec_job);
-    }
-
-    if (json_unpack_ex (jobspec, &error, 0,
-                        "{s:o}",
-                        "tasks", &tasks) < 0) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu invalid jobspec: %s",
-                  __FUNCTION__, (unsigned long long)job->id, error.text);
-        goto error;
-    }
-    if (json_unpack_ex (tasks, &error, 0,
-                        "[{s:o}]",
-                        "command", &command) < 0) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu invalid jobspec: %s",
-                  __FUNCTION__, (unsigned long long)job->id, error.text);
-        goto error;
-    }
-
-    if (!json_is_array (command)) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu invalid jobspec",
-                  __FUNCTION__, (unsigned long long)job->id);
-        goto error;
-    }
-
-    job->jobspec_cmd = json_incref (command);
-
-    if (job->jobspec_job) {
-        if (json_unpack_ex (job->jobspec_job, &error, 0,
-                            "{s?:s}",
-                            "name", &job->name) < 0) {
-            flux_log (ctx->h, LOG_ERR,
-                      "%s: job %llu invalid job dictionary: %s",
-                      __FUNCTION__, (unsigned long long)job->id, error.text);
-            goto error;
-        }
-    }
-
-    /* If user did not specify job.name, we treat arg 0 of the command
-     * as the job name */
-    if (!job->name) {
-        json_t *arg0 = json_array_get (job->jobspec_cmd, 0);
-        if (!arg0 || !json_is_string (arg0)) {
-            flux_log (ctx->h, LOG_ERR,
-                      "%s: job %llu invalid job command",
-                      __FUNCTION__, (unsigned long long)job->id);
-            goto error;
-        }
-        job->name = parse_job_name (json_string_value (arg0));
-        assert (job->name);
-    }
-
-    if (json_unpack_ex (jobspec, &error, 0,
-                        "{s:o}",
-                        "resources", &resources) < 0) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu invalid jobspec: %s",
-                  __FUNCTION__, (unsigned long long)job->id, error.text);
-        goto error;
-    }
-
-    /* Set job->ntasks
-     */
-    if (json_unpack_ex (tasks, NULL, 0,
-                        "[{s:{s:i}}]",
-                        "count", "total", &job->ntasks) < 0) {
-        int per_slot, slot_count = 0;
-        struct res_level res[3];
-
-        if (json_unpack_ex (tasks, NULL, 0,
-                            "[{s:{s:i}}]",
-                            "count", "per_slot", &per_slot) < 0) {
-            flux_log (ctx->h, LOG_ERR,
-                      "%s: job %llu invalid jobspec: %s",
-                      __FUNCTION__, (unsigned long long)job->id, error.text);
-            goto error;
-        }
-        if (per_slot != 1) {
-            flux_log (ctx->h, LOG_ERR,
-                      "%s: job %llu: per_slot count: expected 1 got %d: %s",
-                      __FUNCTION__, (unsigned long long)job->id, per_slot,
-                      error.text);
-            goto error;
-        }
-        /* For jobspec version 1, expect either:
-         * - node->slot->core->NIL
-         * - slot->core->NIL
-         * Set job->slot_count and job->cores_per_slot.
-         */
-        memset (res, 0, sizeof (res));
-        if (parse_res_level (ctx, job, resources, &res[0]) < 0)
-            goto error;
-        if (res[0].with && parse_res_level (ctx, job, res[0].with, &res[1]) < 0)
-            goto error;
-        if (res[1].with && parse_res_level (ctx, job, res[1].with, &res[2]) < 0)
-            goto error;
-        if (res[0].type != NULL && !strcmp (res[0].type, "slot")
-            && res[1].type != NULL && !strcmp (res[1].type, "core")
-            && res[1].with == NULL) {
-            slot_count = res[0].count;
-        }
-        else if (res[0].type != NULL && !strcmp (res[0].type, "node")
-                 && res[1].type != NULL && !strcmp (res[1].type, "slot")
-                 && res[2].type != NULL && !strcmp (res[2].type, "core")
-                 && res[2].with == NULL) {
-            slot_count = res[0].count * res[1].count;
-        }
-        else {
-            flux_log (ctx->h, LOG_ERR,
-                      "%s: job %llu: Unexpected resources: %s->%s->%s%s",
-                      __FUNCTION__,
-                      (unsigned long long)job->id,
-                      res[0].type ? res[0].type : "NULL",
-                      res[1].type ? res[1].type : "NULL",
-                      res[2].type ? res[2].type : "NULL",
-                      res[2].with ? "->..." : NULL);
-            goto error;
-        }
-        job->ntasks = slot_count;
-    }
-
-    rc = 0;
-error:
-    json_decref (jobspec);
-    return rc;
-}
-
-static void state_depend_lookup_continuation (flux_future_t *f, void *arg)
-{
-    struct job *job = arg;
-    struct info_ctx *ctx = job->ctx;
-    struct state_transition *st;
-    const char *s;
-    void *handle;
-
-    if (flux_rpc_get_unpack (f, "{s:s}", "eventlog", &s) < 0) {
-        flux_log_error (ctx->h, "%s: error eventlog for %llu",
-                        __FUNCTION__, (unsigned long long)job->id);
-        goto out;
-    }
-
-    if (eventlog_lookup_parse (ctx, job, s) < 0)
-        goto out;
-
-    if (flux_rpc_get_unpack (f, "{s:s}", "jobspec", &s) < 0) {
-        flux_log_error (ctx->h, "%s: error jobspec for %llu",
-                        __FUNCTION__, (unsigned long long)job->id);
-        goto out;
-    }
-
-    if (jobspec_parse (ctx, job, s) < 0)
-        goto out;
-
-    st = zlist_head (job->next_states);
-    assert (st);
-    update_job_state_and_list (ctx, job, st->state, st->timestamp);
-    check_waiting_id (ctx, job);
-    zlist_remove (job->next_states, st);
-    process_next_state (ctx, job);
-
-out:
-    handle = zlistx_find (ctx->jsctx->futures, f);
-    if (handle)
-        zlistx_detach (ctx->jsctx->futures, handle);
-    flux_future_destroy (f);
-}
-
-static flux_future_t *state_depend_lookup (struct job_state_ctx *jsctx,
-                                           struct job *job)
-{
-    flux_future_t *f = NULL;
-    int saved_errno;
-
-    if (!(f = flux_rpc_pack (jsctx->h, "job-info.lookup", FLUX_NODEID_ANY, 0,
-                             "{s:I s:[ss] s:i}",
-                             "id", job->id,
-                             "keys", "eventlog", "jobspec",
-                             "flags", 0))) {
-        flux_log_error (jsctx->h, "%s: flux_rpc_pack", __FUNCTION__);
-        goto error;
-    }
-
-    if (flux_future_then (f, -1, state_depend_lookup_continuation, job) < 0) {
-        flux_log_error (jsctx->h, "%s: flux_future_then", __FUNCTION__);
-        goto error;
-    }
-
-    return f;
-
- error:
-    saved_errno = errno;
-    flux_future_destroy (f);
-    errno = saved_errno;
-    return NULL;
-}
-
-static int idset_add_set (struct idset *set, struct idset *new)
-{
-    unsigned int i = idset_first (new);
-    while (i != IDSET_INVALID_ID) {
-        if (idset_test (set, i)) {
-            errno = EEXIST;
-            return -1;
-        }
-        if (idset_set (set, i) < 0)
-            return -1;
-        i = idset_next (new, i);
-    }
-    return 0;
-}
-
-static int idset_set_string (struct idset *idset, const char *ids)
-{
-    int rc;
-    struct idset *new = idset_decode (ids);
-    if (!new)
-        return -1;
-    rc = idset_add_set (idset, new);
-    idset_destroy (new);
-    return rc;
-}
-
-static int parse_rlite (struct info_ctx *ctx,
-                        struct job *job,
-                        const json_t *R_lite)
-{
-    struct idset *idset = NULL;
-    size_t index;
-    json_t *value;
-    int saved_errno, rc = -1;
-    int flags = IDSET_FLAG_BRACKETS | IDSET_FLAG_RANGE;
-
-    if (!(idset = idset_create (0, IDSET_FLAG_AUTOGROW)))
-        return -1;
-    json_array_foreach (R_lite, index, value) {
-        const char *ranks = NULL;
-        if ((json_unpack_ex (value, NULL, 0, "{s:s}", "rank", &ranks) < 0)
-            || (idset_set_string (idset, ranks) < 0))
-            goto err;
-    }
-
-    job->nnodes = idset_count (idset);
-    if (!(job->ranks = idset_encode (idset, flags)))
-        goto err;
-
-    rc = 0;
-err:
-    saved_errno = errno;
-    idset_destroy (idset);
-    errno = saved_errno;
-    return rc;
-}
-
-static int R_lookup_parse (struct info_ctx *ctx,
-                           struct job *job,
-                           const char *s)
-{
-    json_error_t error;
-    json_t *R_lite = NULL;
-    int version;
-    int rc = -1;
-
-    if (!(job->R = json_loads (s, 0, &error))) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu invalid R: %s",
-                  __FUNCTION__, (unsigned long long)job->id, error.text);
-        goto error;
-    }
-
-    if (json_unpack_ex (job->R, &error, 0,
-                        "{s:i s:{s:o}}",
-                        "version", &version,
-                        "execution",
-                        "R_lite", &R_lite) < 0) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu invalid R: %s",
-                  __FUNCTION__, (unsigned long long)job->id, error.text);
-        goto error;
-    }
-    if (version != 1) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu invalid R version: %d",
-                  __FUNCTION__, (unsigned long long)job->id, version);
-        goto error;
-    }
-    if (parse_rlite (ctx, job, R_lite) < 0) {
-        flux_log (ctx->h, LOG_ERR,
-                  "%s: job %llu parse_rlite: %s",
-                  __FUNCTION__, (unsigned long long)job->id, strerror (errno));
-        goto error;
-    }
-
-    rc = 0;
-error:
-    return rc;
-}
-
-static void state_run_lookup_continuation (flux_future_t *f, void *arg)
-{
-    struct job *job = arg;
-    struct info_ctx *ctx = job->ctx;
-    struct state_transition *st;
-    const char *s;
-    void *handle;
-
-    if (flux_rpc_get_unpack (f, "{s:s}", "R", &s) < 0) {
-        flux_log_error (ctx->h, "%s: error eventlog for %llu",
-                        __FUNCTION__, (unsigned long long)job->id);
-        goto out;
-    }
-
-    if (R_lookup_parse (ctx, job, s) < 0)
-        goto out;
-
-    st = zlist_head (job->next_states);
-    assert (st);
-    update_job_state_and_list (ctx, job, st->state, st->timestamp);
-    zlist_remove (job->next_states, st);
-    process_next_state (ctx, job);
-
-out:
-    handle = zlistx_find (ctx->jsctx->futures, f);
-    if (handle)
-        zlistx_detach (ctx->jsctx->futures, handle);
-    flux_future_destroy (f);
-}
-
-static flux_future_t *state_run_lookup (struct job_state_ctx *jsctx,
-                                           struct job *job)
-{
-    flux_future_t *f = NULL;
-    int saved_errno;
-
-    if (!(f = flux_rpc_pack (jsctx->h, "job-info.lookup", FLUX_NODEID_ANY, 0,
-                             "{s:I s:[s] s:i}",
-                             "id", job->id,
-                             "keys", "R",
-                             "flags", 0))) {
-        flux_log_error (jsctx->h, "%s: flux_rpc_pack", __FUNCTION__);
-        goto error;
-    }
-
-    if (flux_future_then (f, -1, state_run_lookup_continuation, job) < 0) {
-        flux_log_error (jsctx->h, "%s: flux_future_then", __FUNCTION__);
-        goto error;
-    }
-
-    return f;
-
- error:
-    saved_errno = errno;
-    flux_future_destroy (f);
-    errno = saved_errno;
-    return NULL;
-}
-
-static void state_transition_destroy (void *data)
-{
-    struct state_transition *st = data;
-    if (st)
-        free (st);
-}
-
-static int add_state_transition (struct job *job,
-                                 flux_job_state_t newstate,
-                                 double timestamp)
-{
-    struct state_transition *st = NULL;
-    int saved_errno;
-
-    if (!(st = calloc (1, sizeof (*st))))
-        return -1;
-    st->state = newstate;
-    st->processed = false;
-    st->timestamp = timestamp;
-
-    if (zlist_append (job->next_states, st) < 0) {
-        errno = ENOMEM;
-        goto cleanup;
-    }
-    zlist_freefn (job->next_states, st, state_transition_destroy, true);
-
-    return 0;
-
- cleanup:
-    saved_errno = errno;
-    state_transition_destroy (st);
-    errno = saved_errno;
-    return -1;
-}
-
-static void process_next_state (struct info_ctx *ctx, struct job *job)
-{
-    struct state_transition *st;
-    struct job_state_ctx *jsctx = job->ctx->jsctx;
-
-    while ((st = zlist_head (job->next_states))
-           && !st->processed) {
-        if (st->state == FLUX_JOB_DEPEND
-            || st->state == FLUX_JOB_RUN) {
-            flux_future_t *f = NULL;
-
-            if (st->state == FLUX_JOB_DEPEND) {
-                /* get initial job information, such as userid,
-                 * priority, t_submit, flags, and jobspec info */
-                if (!(f = state_depend_lookup (jsctx, job))) {
-                    flux_log_error (jsctx->h, "%s: state_depend_lookup", __FUNCTION__);
-                    return;
-                }
-            }
-            else { /* st->state == FLUX_JOB_RUN */
-                /* get R to get node count, etc. */
-                if (!(f = state_run_lookup (jsctx, job))) {
-                    flux_log_error (jsctx->h, "%s: state_run_lookup", __FUNCTION__);
-                    return;
-                }
-            }
-
-            if (!zlistx_add_end (jsctx->futures, f)) {
-                flux_log_error (jsctx->h, "%s: zlistx_add_end", __FUNCTION__);
-                flux_future_destroy (f);
-                return;
-            }
-
-            st->processed = true;
-            break;
-        }
-        else {
-            /* FLUX_JOB_SCHED */
-            /* FLUX_JOB_CLEANUP */
-            /* FLUX_JOB_INACTIVE */
-            update_job_state_and_list (ctx, job, st->state, st->timestamp);
-            zlist_remove (job->next_states, st);
-        }
-    }
-}
-
-static void update_jobs (struct info_ctx *ctx, json_t *transitions)
-{
-    struct job_state_ctx *jsctx = ctx->jsctx;
-    size_t index;
-    json_t *value;
-
-    if (!json_is_array (transitions)) {
-        flux_log_error (ctx->h, "%s: transitions EPROTO", __FUNCTION__);
-        return;
-    }
-
-    json_array_foreach (transitions, index, value) {
-        struct job *job;
-        json_t *o;
-        flux_jobid_t id;
-        flux_job_state_t state;
-        double timestamp;
-
-        if (!json_is_array (value)) {
-            flux_log_error (jsctx->h, "%s: transition EPROTO", __FUNCTION__);
-            return;
-        }
-
-        if (!(o = json_array_get (value, 0))
-            || !json_is_integer (o)) {
-            flux_log_error (jsctx->h, "%s: transition EPROTO", __FUNCTION__);
-            return;
-        }
-
-        id = json_integer_value (o);
-
-        if (!(o = json_array_get (value, 1))
-            || !json_is_string (o)) {
-            flux_log_error (jsctx->h, "%s: transition EPROTO", __FUNCTION__);
-            return;
-        }
-
-        if (flux_job_strtostate (json_string_value (o), &state) < 0) {
-            flux_log_error (jsctx->h, "%s: transition EPROTO", __FUNCTION__);
-            return;
-        }
-
-        if (!(o = json_array_get (value, 2))
-            || !json_is_real (o)) {
-            flux_log_error (jsctx->h, "%s: transition EPROTO", __FUNCTION__);
-            return;
-        }
-
-        timestamp = json_real_value (o);
-
-        if (!(job = zhashx_lookup (jsctx->index, &id))) {
-            if (!(job = job_create (ctx, id))){
-                flux_log_error (jsctx->h, "%s: job_create", __FUNCTION__);
-                return;
-            }
-            if (zhashx_insert (jsctx->index, &job->id, job) < 0) {
-                flux_log_error (jsctx->h, "%s: zhashx_insert", __FUNCTION__);
-                job_destroy (job);
-                return;
-            }
-            /* job always starts off on processing list */
-            if (!(job->list_handle = zlistx_add_end (jsctx->processing, job))) {
-                flux_log_error (jsctx->h, "%s: zlistx_add_end", __FUNCTION__);
-                return;
-            }
-        }
-
-        if (add_state_transition (job, state, timestamp) < 0) {
-            flux_log_error (jsctx->h, "%s: add_state_transition",
-                            __FUNCTION__);
-            return;
-        }
-
-        process_next_state (ctx, job);
-    }
-
-}
-
-void job_state_cb (flux_t *h, flux_msg_handler_t *mh,
-                   const flux_msg_t *msg, void *arg)
-{
-    struct info_ctx *ctx = arg;
-    json_t *transitions;
-
-    if (ctx->jsctx->pause) {
-        flux_msg_t *cpy;
-
-        if (!(cpy = flux_msg_copy (msg, true))) {
-            flux_log_error (h, "%s: flux_msg_copy", __FUNCTION__);
-            return;
-        }
-        if (!zlistx_add_end (ctx->jsctx->transitions, cpy)) {
-            flux_log_error (h, "%s: zlistx_add_end", __FUNCTION__);
-            flux_msg_destroy (cpy);
-        }
-    }
-    else {
-        if (flux_event_unpack (msg, NULL, "{s:o}",
-                               "transitions",
-                               &transitions) < 0) {
-            flux_log_error (h, "%s: flux_event_unpack", __FUNCTION__);
-            return;
-        }
-
-        update_jobs (ctx, transitions);
-    }
-
-    return;
-}
-
-void job_state_pause_cb (flux_t *h, flux_msg_handler_t *mh,
-                         const flux_msg_t *msg, void *arg)
-{
-    struct info_ctx *ctx = arg;
-
-    ctx->jsctx->pause = true;
-
-    if (flux_respond (h, msg, NULL) < 0) {
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-        goto error;
-    }
-
-    return;
-
- error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-void job_state_unpause_cb (flux_t *h, flux_msg_handler_t *mh,
-                           const flux_msg_t *msg, void *arg)
-{
-    struct info_ctx *ctx = arg;
-    flux_msg_t *tmsg;
-
-    ctx->jsctx->pause = false;
-
-    tmsg = zlistx_first (ctx->jsctx->transitions);
-    while (tmsg) {
-        job_state_cb (h, mh, tmsg, ctx);
-        tmsg = zlistx_next (ctx->jsctx->transitions);
-    }
-
-    if (flux_respond (h, msg, NULL) < 0) {
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-        goto error;
-    }
-
-    zlistx_purge (ctx->jsctx->transitions);
-    return;
-
- error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    zlistx_purge (ctx->jsctx->transitions);
-}
-
-static struct job *eventlog_restart_parse (struct info_ctx *ctx,
-                                           const char *eventlog,
-                                           flux_jobid_t id)
-{
-    struct job *job = NULL;
-    json_t *a = NULL;
-    size_t index;
-    json_t *value;
-
-    if (!(job = job_create (ctx, id)))
-        goto error;
-
-    if (!(a = eventlog_decode (eventlog))) {
-        flux_log_error (ctx->h, "%s: error parsing eventlog for %llu",
-                        __FUNCTION__, (unsigned long long)job->id);
-        goto error;
-    }
-
-    json_array_foreach (a, index, value) {
-        const char *name;
-        double timestamp;
-        json_t *context = NULL;
-
-        if (eventlog_entry_parse (value, &timestamp, &name, &context) < 0) {
-            flux_log_error (ctx->h, "%s: error parsing entry for %llu",
-                            __FUNCTION__, (unsigned long long)job->id);
-            goto error;
-        }
-
-        if (!strcmp (name, "submit")) {
-            if (!context) {
-                flux_log_error (ctx->h, "%s: no submit context for %llu",
-                                __FUNCTION__, (unsigned long long)job->id);
-                goto error;
-            }
-
-            if (json_unpack (context, "{ s:i s:i s:i }",
-                             "priority", &job->priority,
-                             "userid", &job->userid,
-                             "flags", &job->flags) < 0) {
-                flux_log_error (ctx->h, "%s: submit context for %llu invalid",
-                                __FUNCTION__, (unsigned long long)job->id);
-                goto error;
-            }
-            update_job_state (ctx, job, FLUX_JOB_DEPEND, timestamp);
-        }
-        else if (!strcmp (name, "depend")) {
-            update_job_state (ctx, job, FLUX_JOB_SCHED, timestamp);
-        }
-        else if (!strcmp (name, "priority")) {
-            if (json_unpack (context, "{ s:i }",
-                                      "priority", &job->priority) < 0) {
-                flux_log_error (ctx->h, "%s: priority context for %llu invalid",
-                                __FUNCTION__, (unsigned long long)job->id);
-                goto error;
-            }
-        }
-        else if (!strcmp (name, "exception")) {
-            int severity;
-            if (json_unpack (context, "{ s:i }", "severity", &severity) < 0) {
-                flux_log_error (ctx->h, "%s: exception context for %llu invalid",
-                                __FUNCTION__, (unsigned long long)job->id);
-                goto error;
-            }
-            if (severity == 0)
-                update_job_state (ctx, job, FLUX_JOB_CLEANUP, timestamp);
-        }
-        else if (!strcmp (name, "alloc")) {
-            if (job->state == FLUX_JOB_SCHED)
-                update_job_state (ctx, job, FLUX_JOB_RUN, timestamp);
-        }
-        else if (!strcmp (name, "finish")) {
-            if (job->state == FLUX_JOB_RUN)
-                update_job_state (ctx, job, FLUX_JOB_CLEANUP, timestamp);
-        }
-        else if (!strcmp (name, "clean")) {
-            update_job_state (ctx, job, FLUX_JOB_INACTIVE, timestamp);
-        }
-    }
-
-    if (job->state == FLUX_JOB_NEW) {
-        flux_log_error (ctx->h, "%s: eventlog has no transition events",
-                        __FUNCTION__);
-        goto error;
-    }
-
-    json_decref (a);
-    return job;
-
-error:
-    job_destroy (job);
-    json_decref (a);
-    return NULL;
-}
-
-static int depthfirst_count_depth (const char *s)
-{
-    int count = 0;
-    while (*s) {
-        if (*s++ == '.')
-            count++;
-    }
-    return count;
-}
-
-static int depthfirst_map_one (struct info_ctx *ctx, const char *key,
-                               int dirskip)
-{
-    struct job *job = NULL;
-    flux_jobid_t id;
-    flux_future_t *f1 = NULL;
-    flux_future_t *f2 = NULL;
-    flux_future_t *f3 = NULL;
-    const char *eventlog, *jobspec, *R;
-    char path[64];
-    int rc = -1;
-
-    if (strlen (key) <= dirskip) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (fluid_decode (key + dirskip + 1, &id, FLUID_STRING_DOTHEX) < 0)
-        return -1;
-    if (flux_job_kvs_key (path, sizeof (path), id, "eventlog") < 0) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (!(f1 = flux_kvs_lookup (ctx->h, NULL, 0, path)))
-        goto done;
-    if (flux_kvs_lookup_get (f1, &eventlog) < 0)
-        goto done;
-
-    if (!(job = eventlog_restart_parse (ctx, eventlog, id)))
-        goto done;
-
-    if (flux_job_kvs_key (path, sizeof (path), id, "jobspec") < 0) {
-        errno = EINVAL;
-        goto done;
-    }
-    if (!(f2 = flux_kvs_lookup (ctx->h, NULL, 0, path)))
-        goto done;
-    if (flux_kvs_lookup_get (f2, &jobspec) < 0)
-        goto done;
-
-    if (jobspec_parse (ctx, job, jobspec) < 0)
-        goto done;
-
-    if (job->states_mask & FLUX_JOB_RUN) {
-        if (flux_job_kvs_key (path, sizeof (path), id, "R") < 0) {
-            errno = EINVAL;
-            return -1;
-        }
-        if (!(f3 = flux_kvs_lookup (ctx->h, NULL, 0, path)))
-            goto done;
-        if (flux_kvs_lookup_get (f3, &R) < 0)
-            goto done;
-
-        if (R_lookup_parse (ctx, job, R) < 0)
-            goto done;
-    }
-
-    if (zhashx_insert (ctx->jsctx->index, &job->id, job) < 0) {
-        flux_log_error (ctx->h, "%s: zhashx_insert", __FUNCTION__);
-        goto done;
-    }
-    job_insert_list (ctx->jsctx, job, job->state);
-
-    rc = 1;
-done:
-    if (rc < 0)
-        job_destroy (job);
-    flux_future_destroy (f1);
-    flux_future_destroy (f2);
-    flux_future_destroy (f3);
-    return rc;
-}
-
-static int depthfirst_map (struct info_ctx *ctx, const char *key,
-                           int dirskip)
-{
-    flux_future_t *f;
-    const flux_kvsdir_t *dir;
-    flux_kvsitr_t *itr;
-    const char *name;
-    int path_level;
-    int count = 0;
-    int rc = -1;
-
-    path_level = depthfirst_count_depth (key + dirskip);
-    if (!(f = flux_kvs_lookup (ctx->h, NULL, FLUX_KVS_READDIR, key)))
-        return -1;
-    if (flux_kvs_lookup_get_dir (f, &dir) < 0) {
-        if (errno == ENOENT && path_level == 0)
-            rc = 0;
-        goto done;
-    }
-    if (!(itr = flux_kvsitr_create (dir)))
-        goto done;
-    while ((name = flux_kvsitr_next (itr))) {
-        char *nkey;
-        int n;
-        if (!flux_kvsdir_isdir (dir, name))
-            continue;
-        if (!(nkey = flux_kvsdir_key_at (dir, name)))
-            goto done_destroyitr;
-        if (path_level == 3) // orig 'key' = .A.B.C, thus 'nkey' is complete
-            n = depthfirst_map_one (ctx, nkey, dirskip);
-        else
-            n = depthfirst_map (ctx, nkey, dirskip);
-        if (n < 0) {
-            int saved_errno = errno;
-            free (nkey);
-            errno = saved_errno;
-            goto done_destroyitr;
-        }
-        count += n;
-        free (nkey);
-    }
-    rc = count;
-done_destroyitr:
-    flux_kvsitr_destroy (itr);
-done:
-    flux_future_destroy (f);
-    return rc;
-}
-
-/* Read jobs present in the KVS at startup. */
-int job_state_init_from_kvs (struct info_ctx *ctx)
-{
-    const char *dirname = "job";
-    int dirskip = strlen (dirname);
-    int count;
-
-    count = depthfirst_map (ctx, dirname, dirskip);
-    if (count < 0)
-        return -1;
-    flux_log (ctx->h, LOG_DEBUG, "%s: read %d jobs", __FUNCTION__, count);
-
-    zlistx_sort (ctx->jsctx->running);
-    zlistx_sort (ctx->jsctx->inactive);
-    return 0;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-info/job_state.h b/src/modules/job-info/job_state.h
deleted file mode 100644
index fe0a6c4a21e3..000000000000
--- a/src/modules/job-info/job_state.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_JOB_INFO_JOB_STATE_H
-#define _FLUX_JOB_INFO_JOB_STATE_H
-
-#include <flux/core.h>
-#include <jansson.h>
-
-#include "info.h"
-
-/* To handle the common case of user queries on job state, we will
- * store jobs in three different lists.
- *
- * - pending - these are jobs that have not yet in the RUN state, they
- *   are sorted based on job priority (highest first), then job
- *   submission time (earlier submission time first).
- * - running - these are jobs that have transitioned to the RUN state.
- *   They are sorted by initial run start time (later run start
- *   times first).
- * - inactive - these are jobs that are in the INACTIVE state, they
- *   are sorted by job completion time (later completion times
- *   first)
- *
- * There is also an additional list `processing` that stores jobs that
- * cannot yet be stored on one of the lists above.
- *
- * The list `futures` is used to store in process futures.
- */
-
-struct job_state_ctx {
-    flux_t *h;
-    zhashx_t *index;
-    zlistx_t *pending;
-    zlistx_t *running;
-    zlistx_t *inactive;
-    zlistx_t *processing;
-    zlistx_t *futures;
-
-    /* count current jobs in what states */
-    int depend_count;
-    int sched_count;
-    int run_count;
-    int cleanup_count;
-    int inactive_count;
-
-    /* debug/testing - if paused store job transitions on list for
-     * processing later */
-    bool pause;
-    zlistx_t *transitions;
-};
-
-struct job {
-    struct info_ctx *ctx;
-
-    flux_jobid_t id;
-    uint32_t userid;
-    int priority;
-    double t_submit;
-    int flags;
-    flux_job_state_t state;
-    const char *name;
-    int ntasks;
-    int nnodes;
-    char *ranks;
-
-    /* cache of job information */
-    json_t *jobspec_job;
-    json_t *jobspec_cmd;
-    json_t *R;
-
-    /* Track which states we have seen and have completed transition
-     * to.  We do not immediately update to the new state and place
-     * onto a new list until we have retrieved any necessary data
-     * associated to that state.  For example, when the 'depend' state
-     * has been seen, we don't immediately place it on the `pending`
-     * list.  We wait until we've retrieved data such as userid,
-     * priority, etc.
-     *
-     * Track which states we've seen via the states_mask;
-     */
-    zlist_t *next_states;
-    unsigned int states_mask;
-    void *list_handle;
-
-    /* timestamp of when we enter the state
-     *
-     * associated eventlog entries when restarting
-     *
-     * depend - "submit"
-     * sched - "depend"
-     * run - "alloc"
-     * cleanup - "finish" or "exception" w/ severity == 0
-     * inactive - "clean"
-     */
-    // t_depend is identical to t_submit above, use that
-    // double t_depend;
-    double t_sched;
-    double t_run;
-    double t_cleanup;
-    double t_inactive;
-};
-
-struct job_state_ctx *job_state_create (flux_t *h);
-
-void job_state_destroy (void *data);
-
-void job_state_cb (flux_t *h, flux_msg_handler_t *mh,
-                   const flux_msg_t *msg, void *arg);
-
-void job_state_pause_cb (flux_t *h, flux_msg_handler_t *mh,
-                         const flux_msg_t *msg, void *arg);
-
-void job_state_unpause_cb (flux_t *h, flux_msg_handler_t *mh,
-                           const flux_msg_t *msg, void *arg);
-
-int job_state_init_from_kvs (struct info_ctx *ctx);
-
-#endif /* ! _FLUX_JOB_INFO_JOB_STATE_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-info/job_util.c b/src/modules/job-info/job_util.c
deleted file mode 100644
index 538091be4c06..000000000000
--- a/src/modules/job-info/job_util.c
+++ /dev/null
@@ -1,126 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* job_util.c - job utility functions */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <czmq.h>
-#include <jansson.h>
-#include <flux/core.h>
-
-#include "src/common/libutil/errno_safe.h"
-
-#include "job_util.h"
-#include "job_state.h"
-
-/* For a given job, create a JSON object containing the jobid and any
- * additional requested attributes and their values.  Returns JSON
- * object which the caller must free.  On error, return NULL with
- * errno set:
- *
- * EPROTO - malformed attrs array
- * ENOMEM - out of memory
- */
-json_t *job_to_json (struct job *job, json_t *attrs)
-{
-    size_t index;
-    json_t *value;
-    json_t *o;
-    json_t *val = NULL;
-
-    if (!(o = json_object ()))
-        goto error_nomem;
-    if (!(val = json_integer (job->id)))
-        goto error_nomem;
-    if (json_object_set_new (o, "id", val) < 0) {
-        json_decref (val);
-        goto error_nomem;
-    }
-    json_array_foreach (attrs, index, value) {
-        const char *attr = json_string_value (value);
-        if (!attr) {
-            errno = EINVAL;
-            goto error;
-        }
-        if (!strcmp (attr, "userid")) {
-            val = json_integer (job->userid);
-        }
-        else if (!strcmp (attr, "priority")) {
-            val = json_integer (job->priority);
-        }
-        else if (!strcmp (attr, "t_submit")
-                 || !strcmp (attr, "t_depend")) {
-            if (!(job->states_mask & FLUX_JOB_DEPEND))
-                continue;
-            val = json_real (job->t_submit);
-        }
-        else if (!strcmp (attr, "t_sched")) {
-            if (!(job->states_mask & FLUX_JOB_SCHED))
-                continue;
-            val = json_real (job->t_sched);
-        }
-        else if (!strcmp (attr, "t_run")) {
-            if (!(job->states_mask & FLUX_JOB_RUN))
-                continue;
-            val = json_real (job->t_run);
-        }
-        else if (!strcmp (attr, "t_cleanup")) {
-            if (!(job->states_mask & FLUX_JOB_CLEANUP))
-                continue;
-            val = json_real (job->t_cleanup);
-        }
-        else if (!strcmp (attr, "t_inactive")) {
-            if (!(job->states_mask & FLUX_JOB_INACTIVE))
-                continue;
-            val = json_real (job->t_inactive);
-        }
-        else if (!strcmp (attr, "state")) {
-            val = json_integer (job->state);
-        }
-        else if (!strcmp (attr, "name")) {
-            val = json_string (job->name);
-        }
-        else if (!strcmp (attr, "ntasks")) {
-            val = json_integer (job->ntasks);
-        }
-        else if (!strcmp (attr, "nnodes")) {
-            if (!(job->states_mask & FLUX_JOB_RUN))
-                continue;
-            val = json_integer (job->nnodes);
-        }
-        else if (!strcmp (attr, "ranks")) {
-            if (!(job->states_mask & FLUX_JOB_RUN))
-                continue;
-            val = json_string (job->ranks);
-        }
-        else {
-            errno = EINVAL;
-            goto error;
-        }
-        if (val == NULL)
-            goto error_nomem;
-        if (json_object_set_new (o, attr, val) < 0) {
-            json_decref (val);
-            goto error_nomem;
-        }
-    }
-    return o;
- error_nomem:
-    errno = ENOMEM;
- error:
-    ERRNO_SAFE_WRAP (json_decref, o);
-    return NULL;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-info/job_util.h b/src/modules/job-info/job_util.h
deleted file mode 100644
index 8d79798a5d6b..000000000000
--- a/src/modules/job-info/job_util.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_JOB_INFO_JOB_UTIL_H
-#define _FLUX_JOB_INFO_JOB_UTIL_H
-
-#include <flux/core.h>
-
-#include "job_state.h"
-
-json_t *job_to_json (struct job *job, json_t *attrs);
-
-#endif /* ! _FLUX_JOB_INFO_JOB_UTIL_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-info/list.c b/src/modules/job-info/list.c
deleted file mode 100644
index e9241c1e5a8f..000000000000
--- a/src/modules/job-info/list.c
+++ /dev/null
@@ -1,517 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* list.c - list jobs */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <czmq.h>
-#include <jansson.h>
-#include <flux/core.h>
-
-#include "src/common/libutil/errno_safe.h"
-
-#include "idsync.h"
-#include "list.h"
-#include "job_util.h"
-#include "job_state.h"
-
-json_t *get_job_by_id (struct info_ctx *ctx,
-                       const flux_msg_t *msg,
-                       flux_jobid_t id,
-                       json_t *attrs,
-                       bool *stall);
-
-/* Put jobs from list onto jobs array, breaking if max_entries has
- * been reached. Returns 1 if jobs array is full, 0 if continue, -1
- * one error with errno set:
- *
- * ENOMEM - out of memory
- */
-int get_jobs_from_list (json_t *jobs,
-                        zlistx_t *list,
-                        int max_entries,
-                        json_t *attrs,
-                        uint32_t userid,
-                        int states)
-{
-    struct job *job;
-
-    job = zlistx_first (list);
-    while (job) {
-        if (job->state & states) {
-            if (userid == FLUX_USERID_UNKNOWN || job->userid == userid) {
-                json_t *o;
-                if (!(o = job_to_json (job, attrs)))
-                    return -1;
-                if (json_array_append_new (jobs, o) < 0) {
-                    json_decref (o);
-                    errno = ENOMEM;
-                    return -1;
-                }
-                if (json_array_size (jobs) == max_entries)
-                    return 1;
-            }
-        }
-        job = zlistx_next (list);
-    }
-
-    return 0;
-}
-
-/* Create a JSON array of 'job' objects.  'max_entries' determines the
- * max number of jobs to return, 0=unlimited.  Returns JSON object
- * which the caller must free.  On error, return NULL with errno set:
- *
- * EPROTO - malformed or empty attrs array, max_entries out of range
- * ENOMEM - out of memory
- */
-json_t *get_jobs (struct info_ctx *ctx,
-                  int max_entries,
-                  json_t *attrs,
-                  uint32_t userid,
-                  int states)
-{
-    json_t *jobs = NULL;
-    int saved_errno;
-    int ret = 0;
-
-    if (!(jobs = json_array ()))
-        goto error_nomem;
-
-    /* We return jobs in the following order, pending, running,
-     * inactive */
-
-    if (states & FLUX_JOB_PENDING) {
-        if ((ret = get_jobs_from_list (jobs,
-                                       ctx->jsctx->pending,
-                                       max_entries,
-                                       attrs,
-                                       userid,
-                                       states)) < 0)
-            goto error;
-    }
-
-    if (states & FLUX_JOB_RUNNING) {
-        if (!ret) {
-            if ((ret = get_jobs_from_list (jobs,
-                                           ctx->jsctx->running,
-                                           max_entries,
-                                           attrs,
-                                           userid,
-                                           states)) < 0)
-                goto error;
-        }
-    }
-
-    if (states & FLUX_JOB_INACTIVE) {
-        if (!ret) {
-            if ((ret = get_jobs_from_list (jobs,
-                                           ctx->jsctx->inactive,
-                                           max_entries,
-                                           attrs,
-                                           userid,
-                                           states)) < 0)
-                goto error;
-        }
-    }
-
-    return jobs;
-
-error_nomem:
-    errno = ENOMEM;
-error:
-    saved_errno = errno;
-    json_decref (jobs);
-    errno = saved_errno;
-    return NULL;
-}
-
-void list_cb (flux_t *h, flux_msg_handler_t *mh,
-              const flux_msg_t *msg, void *arg)
-{
-    struct info_ctx *ctx = arg;
-    json_t *jobs = NULL;
-    json_t *attrs;
-    int max_entries;
-    uint32_t userid;
-    int states;
-
-    if (flux_request_unpack (msg, NULL, "{s:i s:o s:i s:i}",
-                             "max_entries", &max_entries,
-                             "attrs", &attrs,
-                             "userid", &userid,
-                             "states", &states) < 0)
-        goto error;
-
-    if (max_entries < 0 || !json_is_array (attrs)) {
-        errno = EPROTO;
-        goto error;
-    }
-
-    /* If user sets no states, assume they want all information */
-    if (!states)
-        states = (FLUX_JOB_PENDING
-                  | FLUX_JOB_RUNNING
-                  | FLUX_JOB_INACTIVE);
-
-    if (!(jobs = get_jobs (ctx, max_entries, attrs, userid, states)))
-        goto error;
-
-    if (flux_respond_pack (h, msg, "{s:O}", "jobs", jobs) < 0) {
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-        goto error;
-    }
-
-    json_decref (jobs);
-    return;
-
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    json_decref (jobs);
-}
-
-/* Create a JSON array of 'job' objects.  'since' limits entries
- * returned, only returning entries with 't_inactive' newer than the
- * timestamp.  Returns JSON object which the caller must free.  On
- * error, return NULL with errno set:
- *
- * EPROTO - malformed or empty attrs array
- * ENOMEM - out of memory
- */
-json_t *get_inactive_jobs (struct info_ctx *ctx,
-                           int max_entries,
-                           double since,
-                           json_t *attrs)
-{
-    json_t *jobs = NULL;
-    struct job *job;
-    int saved_errno;
-
-    if (!(jobs = json_array ()))
-        goto error_nomem;
-
-    job = zlistx_first (ctx->jsctx->inactive);
-    while (job && (job->t_inactive > since)) {
-        json_t *o;
-        if (!(o = job_to_json (job, attrs)))
-            goto error;
-        if (json_array_append_new (jobs, o) < 0) {
-            json_decref (o);
-            errno = ENOMEM;
-            goto error;
-        }
-        if (json_array_size (jobs) == max_entries)
-            goto out;
-        job = zlistx_next (ctx->jsctx->inactive);
-    }
-
-out:
-    return jobs;
-
-error_nomem:
-    errno = ENOMEM;
-error:
-    saved_errno = errno;
-    json_decref (jobs);
-    errno = saved_errno;
-    return NULL;
-}
-
-void list_inactive_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
-{
-    struct info_ctx *ctx = arg;
-    json_t *jobs = NULL;
-    int max_entries;
-    double since;
-    json_t *attrs;
-
-    if (flux_request_unpack (msg, NULL, "{s:i s:F s:o}",
-                             "max_entries", &max_entries,
-                             "since", &since,
-                             "attrs", &attrs) < 0)
-        goto error;
-
-    if (max_entries < 0 || !json_is_array (attrs)) {
-        errno = EPROTO;
-        goto error;
-    }
-
-    if (!(jobs = get_inactive_jobs (ctx, max_entries, since, attrs)))
-        goto error;
-
-    if (flux_respond_pack (h, msg, "{s:O}", "jobs", jobs) < 0) {
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-        goto error;
-    }
-
-    json_decref (jobs);
-    return;
-
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    json_decref (jobs);
-}
-
-int wait_id_valid (struct info_ctx *ctx, struct idsync_data *isd)
-{
-    zlistx_t *list_isd;
-    void *handle;
-    int saved_errno;
-
-    if ((handle = zlistx_find (ctx->idsync_lookups, isd))) {
-        /* detach will not call zlistx destructor */
-        zlistx_detach (ctx->idsync_lookups, handle);
-    }
-
-    /* idsync_waits holds lists of ids waiting on, b/c multiplers callers
-     * could wait on same id */
-    if (!(list_isd = zhashx_lookup (ctx->idsync_waits, &isd->id))) {
-        if (!(list_isd = zlistx_new ())) {
-            flux_log_error (isd->ctx->h, "%s: zlistx_new", __FUNCTION__);
-            goto error_destroy;
-        }
-        zlistx_set_destructor (list_isd, idsync_data_destroy_wrapper);
-
-        if (zhashx_insert (ctx->idsync_waits, &isd->id, list_isd) < 0) {
-            flux_log_error (isd->ctx->h, "%s: zhashx_insert", __FUNCTION__);
-            goto error_destroy;
-        }
-    }
-
-    if (!zlistx_add_end (list_isd, isd)) {
-        flux_log_error (isd->ctx->h, "%s: zlistx_add_end", __FUNCTION__);
-        goto error_destroy;
-    }
-
-    return 0;
-
-error_destroy:
-    saved_errno = errno;
-    idsync_data_destroy (isd);
-    errno = saved_errno;
-    return -1;
-}
-
-void check_id_valid_continuation (flux_future_t *f, void *arg)
-{
-    struct idsync_data *isd = arg;
-    struct info_ctx *ctx = isd->ctx;
-    void *handle;
-
-    if (flux_future_get (f, NULL) < 0) {
-        if (flux_respond_error (ctx->h, isd->msg, errno, NULL) < 0)
-            flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
-        goto cleanup;
-    }
-    else {
-        /* Job ID is legal.  Chance job-info has seen ID since this
-         * lookup was done */
-        struct job *job;
-        if (!(job = zhashx_lookup (ctx->jsctx->index, &isd->id))
-            || job->state == FLUX_JOB_NEW) {
-            /* Must wait for job-info to see state change */
-            if (wait_id_valid (ctx, isd) < 0)
-                flux_log_error (ctx->h, "%s: wait_id_valid", __FUNCTION__);
-            goto cleanup;
-        }
-        else {
-            json_t *o;
-            if (!(o = get_job_by_id (ctx, isd->msg, isd->id, isd->attrs, NULL))) {
-                flux_log_error (ctx->h, "%s: get_job_by_id", __FUNCTION__);
-                goto cleanup;
-            }
-            if (flux_respond_pack (ctx->h, isd->msg, "{s:O}", "job", o) < 0) {
-                flux_log_error (ctx->h, "%s: flux_respond_pack", __FUNCTION__);
-                goto cleanup;
-            }
-        }
-    }
-
-cleanup:
-    /* delete will destroy struct idsync_data and future within it */
-    handle = zlistx_find (ctx->idsync_lookups, isd);
-    if (handle)
-        zlistx_delete (ctx->idsync_lookups, handle);
-    return;
-}
-
-int check_id_valid (struct info_ctx *ctx,
-                    const flux_msg_t *msg,
-                    flux_jobid_t id,
-                    json_t *attrs)
-{
-    flux_future_t *f = NULL;
-    struct idsync_data *isd = NULL;
-    char path[256];
-    int saved_errno;
-
-    /* Check to see if the ID is legal, job-info may have not yet
-     * seen the ID publication yet */
-    if (flux_job_kvs_key (path, sizeof (path), id, NULL) < 0)
-        goto error;
-
-    if (!(f = flux_kvs_lookup (ctx->h, NULL, FLUX_KVS_READDIR, path))) {
-        flux_log_error (ctx->h, "%s: flux_kvs_lookup", __FUNCTION__);
-        goto error;
-    }
-
-    if (!(isd = idsync_data_create (ctx, id, msg, attrs, f)))
-        goto error;
-
-    /* future now owned by struct idsync_data */
-    f = NULL;
-
-    if (flux_future_then (isd->f_lookup,
-                          -1,
-                          check_id_valid_continuation,
-                          isd) < 0) {
-        flux_log_error (ctx->h, "%s: flux_future_then", __FUNCTION__);
-        goto error;
-    }
-
-    if (!zlistx_add_end (ctx->idsync_lookups, isd)) {
-        flux_log_error (ctx->h, "%s: zlistx_add_end", __FUNCTION__);
-        goto error;
-    }
-
-    return 0;
-
-error:
-    saved_errno = errno;
-    flux_future_destroy (f);
-    idsync_data_destroy (isd);
-    errno = saved_errno;
-    return -1;
-}
-
-/* Returns JSON object which the caller must free.  On error, return
- * NULL with errno set:
- *
- * EPROTO - malformed or empty id or attrs array
- * EINVAL - invalid id
- * ENOMEM - out of memory
- */
-json_t *get_job_by_id (struct info_ctx *ctx,
-                       const flux_msg_t *msg,
-                       flux_jobid_t id,
-                       json_t *attrs,
-                       bool *stall)
-{
-    struct job *job;
-
-    if (!(job = zhashx_lookup (ctx->jsctx->index, &id))) {
-        if (stall) {
-            if (check_id_valid (ctx, msg, id, attrs) < 0) {
-                flux_log_error (ctx->h, "%s: check_id_valid", __FUNCTION__);
-                return NULL;
-            }
-            (*stall) = true;
-        }
-        return NULL;
-    }
-
-    if (job->state == FLUX_JOB_NEW) {
-        if (stall) {
-            struct idsync_data *isd;
-            if (!(isd = idsync_data_create (ctx, id, msg, attrs, NULL))) {
-                flux_log_error (ctx->h, "%s: idsync_data_create", __FUNCTION__);
-                return NULL;
-            }
-            /* Must wait for job-info to see state change */
-            if (wait_id_valid (ctx, isd) < 0) {
-                flux_log_error (ctx->h, "%s: wait_id_valid", __FUNCTION__);
-                return NULL;
-            }
-            (*stall) = true;
-        }
-        return NULL;
-    }
-
-    return job_to_json (job, attrs);
-}
-
-void list_id_cb (flux_t *h, flux_msg_handler_t *mh,
-                  const flux_msg_t *msg, void *arg)
-{
-    struct info_ctx *ctx = arg;
-    json_t *job = NULL;
-    flux_jobid_t id;
-    json_t *attrs;
-    bool stall = false;
-
-    if (flux_request_unpack (msg, NULL, "{s:I s:o}",
-                             "id", &id,
-                             "attrs", &attrs) < 0)
-        goto error;
-
-    if (!json_is_array (attrs)) {
-        errno = EPROTO;
-        goto error;
-    }
-
-    if (!(job = get_job_by_id (ctx, msg, id, attrs, &stall))) {
-        /* response handled after KVS lookup complete */
-        if (stall)
-            goto stall;
-        goto error;
-    }
-
-    if (flux_respond_pack (h, msg, "{s:O}", "job", job) < 0) {
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-        goto error;
-    }
-
-    json_decref (job);
-stall:
-    return;
-
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    json_decref (job);
-}
-
-void list_attrs_cb (flux_t *h, flux_msg_handler_t *mh,
-                    const flux_msg_t *msg, void *arg)
-{
-    if (flux_respond_pack (h, msg, "{s:[s,s,s,s,s,s,s,s,s,s,s,s]}",
-                           "attrs",
-                           "userid",
-                           "priority",
-                           "t_submit",
-                           "t_depend",
-                           "t_sched",
-                           "t_run",
-                           "t_cleanup",
-                           "t_inactive",
-                           "state",
-                           "name",
-                           "ntasks",
-                           "nnodes",
-                           "ranks") < 0) {
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-        goto error;
-    }
-
-    return;
-
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-info/list.h b/src/modules/job-info/list.h
deleted file mode 100644
index 66d169b12f29..000000000000
--- a/src/modules/job-info/list.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_JOB_INFO_LIST_H
-#define _FLUX_JOB_INFO_LIST_H
-
-#include <flux/core.h>
-
-#include "info.h"
-
-void list_cb (flux_t *h, flux_msg_handler_t *mh,
-              const flux_msg_t *msg, void *arg);
-
-void list_inactive_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg);
-
-void list_id_cb (flux_t *h, flux_msg_handler_t *mh,
-                 const flux_msg_t *msg, void *arg);
-
-void list_attrs_cb (flux_t *h, flux_msg_handler_t *mh,
-                    const flux_msg_t *msg, void *arg);
-
-#endif /* ! _FLUX_JOB_INFO_LIST_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-info/lookup.c b/src/modules/job-info/lookup.c
index db6f8d25be8a..54a05796b6c1 100644
--- a/src/modules/job-info/lookup.c
+++ b/src/modules/job-info/lookup.c
@@ -13,20 +13,28 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
+#include <assert.h>
 
-#include "info.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "job-info.h"
 #include "lookup.h"
+#include "update.h"
 #include "allow.h"
+#include "util.h"
 
 struct lookup_ctx {
     struct info_ctx *ctx;
     const flux_msg_t *msg;
     flux_jobid_t id;
     json_t *keys;
-    bool check_eventlog;
+    bool lookup_eventlog;
     int flags;
     flux_future_t *f;
     bool allow;
@@ -36,12 +44,15 @@ static void info_lookup_continuation (flux_future_t *fall, void *arg);
 
 static void lookup_ctx_destroy (void *data)
 {
-    if (data) {
-        struct lookup_ctx *ctx = data;
+    struct lookup_ctx *ctx = data;
+
+    if (ctx) {
+        int saved_errno = errno;
         flux_msg_decref (ctx->msg);
         json_decref (ctx->keys);
         flux_future_destroy (ctx->f);
         free (ctx);
+        errno = saved_errno;
     }
 }
 
@@ -52,7 +63,6 @@ static struct lookup_ctx *lookup_ctx_create (struct info_ctx *ctx,
                                              int flags)
 {
     struct lookup_ctx *l = calloc (1, sizeof (*l));
-    int saved_errno;
 
     if (!l)
         return NULL;
@@ -71,9 +81,7 @@ static struct lookup_ctx *lookup_ctx_create (struct info_ctx *ctx,
     return l;
 
 error:
-    saved_errno = errno;
     lookup_ctx_destroy (l);
-    errno = saved_errno;
     return NULL;
 }
 
@@ -84,68 +92,128 @@ static int lookup_key (struct lookup_ctx *l,
     flux_future_t *f = NULL;
     char path[64];
 
-    if (flux_job_kvs_key (path, sizeof (path), l->id, key) < 0) {
-        flux_log_error (l->ctx->h, "%s: flux_job_kvs_key", __FUNCTION__);
-        goto error;
+    /* Check for duplicate key, return if already looked up */
+    if (flux_future_get_child (fall, key) != NULL)
+        return 0;
+
+    if (flux_job_kvs_key (path, sizeof (path), l->id, key) < 0
+        || !(f = flux_kvs_lookup (l->ctx->h, NULL, 0, path))
+        || flux_future_push (fall, key, f) < 0) {
+        flux_future_destroy (f);
+        return -1;
     }
+    return 0;
+}
 
-    if (!(f = flux_kvs_lookup (l->ctx->h, NULL, 0, path))) {
-        flux_log_error (l->ctx->h, "%s: flux_kvs_lookup", __FUNCTION__);
-        goto error;
+static int lookup_keys (struct lookup_ctx *l)
+{
+    flux_future_t *fall = NULL;
+    size_t index;
+    json_t *key;
+
+    if (!(fall = flux_future_wait_all_create ()))
+        return -1;
+    flux_future_set_flux (fall, l->ctx->h);
+
+    if (l->lookup_eventlog) {
+        if (lookup_key (l, fall, "eventlog") < 0)
+            goto error;
     }
 
-    if (flux_future_push (fall, key, f) < 0) {
-        flux_log_error (l->ctx->h, "%s: flux_future_push", __FUNCTION__);
-        goto error;
+    json_array_foreach (l->keys, index, key) {
+        if (lookup_key (l, fall, json_string_value (key)) < 0)
+            goto error;
     }
 
+    if (flux_future_then (fall, -1, info_lookup_continuation, l) < 0)
+        goto error;
+
+    l->f = fall;
     return 0;
 
 error:
-    flux_future_destroy (f);
+    flux_future_destroy (fall);
     return -1;
 }
 
-static int lookup_keys (struct lookup_ctx *l)
+static int lookup_current (struct lookup_ctx *l,
+                           flux_future_t *fall,
+                           const char *key,
+                           const char *value,
+                           char **current_value)
 {
-    flux_future_t *fall = NULL;
+    flux_future_t *f_eventlog;
+    const char *s_eventlog;
+    json_t *value_object = NULL;
+    json_t *eventlog = NULL;
     size_t index;
-    json_t *key;
+    json_t *entry;
+    const char *update_event_name = NULL;
+    char *value_object_str = NULL;
+    int save_errno;
+
+    if (streq (key, "R"))
+        update_event_name = "resource-update";
+    else if (streq (key, "jobspec"))
+        update_event_name = "jobspec-update";
+
+    if (!(value_object = json_loads (value, 0, NULL))) {
+        errno = EINVAL;
+        goto error;
+    }
 
-    if (!(fall = flux_future_wait_all_create ())) {
-        flux_log_error (l->ctx->h, "%s: flux_wait_all_create", __FUNCTION__);
+    if (!(f_eventlog = flux_future_get_child (fall, "eventlog"))) {
+        flux_log_error (l->ctx->h,
+                        "%s: flux_future_get_child",
+                        __FUNCTION__);
         goto error;
     }
-    flux_future_set_flux (fall, l->ctx->h);
 
-    if (l->check_eventlog) {
-        if (lookup_key (l, fall, "eventlog") < 0)
-            goto error;
+    if (flux_kvs_lookup_get (f_eventlog, &s_eventlog) < 0) {
+        if (errno != ENOENT) {
+            flux_log_error (l->ctx->h,
+                            "%s: flux_kvs_lookup_get",
+                            __FUNCTION__);
+        }
+        goto error;
     }
 
-    json_array_foreach(l->keys, index, key) {
-        const char *keystr;
-        if (!(keystr = json_string_value (key))) {
-            errno = EINVAL;
+    if (!(eventlog = eventlog_decode (s_eventlog))) {
+        errno = EINVAL;
+        goto error;
+    }
+
+    json_array_foreach (eventlog, index, entry) {
+        const char *name;
+        json_t *context = NULL;
+        if (eventlog_entry_parse (entry, NULL, &name, &context) < 0)
             goto error;
+        if (streq (name, update_event_name)) {
+            if (streq (key, "R"))
+                apply_updates_R (l->ctx->h, l->id, key, value_object, context);
+            else if (streq (key, "jobspec"))
+                apply_updates_jobspec (l->ctx->h,
+                                       l->id,
+                                       key,
+                                       value_object,
+                                       context);
         }
-        if (lookup_key (l, fall, keystr) < 0)
-            goto error;
     }
 
-    if (flux_future_then (fall,
-                          -1,
-                          info_lookup_continuation,
-                          l) < 0) {
-        flux_log_error (l->ctx->h, "%s: flux_future_then", __FUNCTION__);
+    if (!(value_object_str = json_dumps (value_object, 0)))
         goto error;
-    }
 
-    l->f = fall;
+    (*current_value) = value_object_str;
+    json_decref (eventlog);
+    json_decref (value_object);
     return 0;
 
 error:
-    flux_future_destroy (fall);
+    save_errno = errno;
+    json_decref (eventlog);
+    json_decref (value_object);
+    free (value_object_str);
+    errno = save_errno;
     return -1;
 }
 
@@ -154,160 +222,428 @@ static void info_lookup_continuation (flux_future_t *fall, void *arg)
     struct lookup_ctx *l = arg;
     struct info_ctx *ctx = l->ctx;
     const char *s;
+    char *current_value = NULL;
     size_t index;
     json_t *key;
     json_t *o = NULL;
-    char *data = NULL;
+    json_t *tmp = NULL;
+    flux_error_t error;
 
     if (!l->allow) {
         flux_future_t *f;
 
         if (!(f = flux_future_get_child (fall, "eventlog"))) {
-            flux_log_error (ctx->h, "%s: flux_future_get_child", __FUNCTION__);
+            errprintf (&error,
+                       "internal error: flux_future_get_child eventlog: %s",
+                       strerror (errno));
             goto error;
         }
 
         if (flux_kvs_lookup_get (f, &s) < 0) {
-            if (errno != ENOENT)
-                flux_log_error (l->ctx->h, "%s: flux_kvs_lookup_get", __FUNCTION__);
+            errprintf (&error,
+                       "%s",
+                       errno == ENOENT ? "invalid job id" : strerror (errno));
             goto error;
         }
 
-        if (eventlog_allow (ctx, l->msg, s) < 0)
+        if (eventlog_allow (ctx, l->msg, l->id, s) < 0) {
+            char *errmsg;
+            if (errno == EPERM)
+                errmsg = "access is restricted to job/instance owner";
+            else
+                errmsg = "error parsing eventlog";
+            errprintf (&error, "%s", errmsg);
             goto error;
+        }
         l->allow = true;
     }
 
-    if (!(o = json_object ()))
+    if (!(o = json_object ())
+        || !(tmp = json_integer (l->id))
+        || json_object_set_new (o, "id", tmp) < 0) {
+        errprintf (&error, "error creating response object");
+        json_decref (tmp);
         goto enomem;
+    }
 
-    json_array_foreach(l->keys, index, key) {
+    json_array_foreach (l->keys, index, key) {
         flux_future_t *f;
-        const char *keystr;
-        json_t *str = NULL;
+        const char *keystr = json_string_value (key); /* validated earlier */
+        json_t *val = NULL;
 
-        if (!(keystr = json_string_value (key))) {
-            errno = EINVAL;
+        if (!(f = flux_future_get_child (fall, keystr))) {
+            errprintf (&error,
+                       "internal error: flux_future_get_child %s: %s",
+                       keystr,
+                       strerror (errno));
             goto error;
         }
 
-        if (!(f = flux_future_get_child (fall, keystr))) {
-            flux_log_error (ctx->h, "%s: flux_future_get_child", __FUNCTION__);
+        if (flux_kvs_lookup_get (f, &s) < 0) {
+            errprintf (&error,
+                       "%s: %s",
+                       keystr,
+                       errno == ENOENT ? "key not found" : strerror (errno));
             goto error;
         }
 
-        if (flux_kvs_lookup_get (f, &s) < 0) {
-            if (errno != ENOENT)
-                flux_log_error (l->ctx->h, "%s: flux_kvs_lookup_get", __FUNCTION__);
+        /* treat empty value as invalid */
+        if (!s) {
+            errprintf (&error, "%s: value is unexpectedly empty", keystr);
+            errno = EPROTO;
             goto error;
         }
 
-        if (!(str = json_string (s)))
-            goto enomem;
+        if ((l->flags & FLUX_JOB_LOOKUP_CURRENT)
+            && (streq (keystr, "R") || streq (keystr, "jobspec"))) {
+            if (lookup_current (l, fall, keystr, s, &current_value) < 0) {
+                errprintf (&error,
+                           "%s: error applying eventlog to original value: %s",
+                           keystr,
+                           strerror (errno));
+                goto error;
+            }
+            s = current_value;
+        }
 
-        if (json_object_set_new (o, keystr, str) < 0) {
-            json_decref (str);
+        /* check for JSON_DECODE flag last, as changes above could affect
+         * desired value */
+        if ((l->flags & FLUX_JOB_LOOKUP_JSON_DECODE)
+            && (streq (keystr, "jobspec") || streq (keystr, "R"))) {
+            /* We assume if it was stored in the KVS it's valid JSON,
+             * so failure is ENOMEM */
+            val = json_loads (s, 0, NULL);
+        }
+        else
+            val = json_string (s);
+        if (!val || json_object_set_new (o, keystr, val) < 0) {
+            json_decref (val);
+            errprintf (&error, "%s: error adding value to response", keystr);
             goto enomem;
         }
+
+        free (current_value);
+        current_value = NULL;
     }
 
     /* must have been allowed earlier or above, otherwise should have
      * taken error path */
     assert (l->allow);
 
-    if (!(data = json_dumps (o, JSON_COMPACT)))
-        goto enomem;
-
-    if (flux_respond (ctx->h, l->msg, data) < 0) {
+    if (flux_respond_pack (ctx->h, l->msg, "O", o) < 0)
         flux_log_error (ctx->h, "%s: flux_respond", __FUNCTION__);
-        goto error;
-    }
 
     goto done;
 
 enomem:
     errno = ENOMEM;
 error:
-    if (flux_respond_error (ctx->h, l->msg, errno, NULL) < 0)
+    if (flux_respond_error (ctx->h, l->msg, errno, error.text) < 0)
         flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
 
 done:
     /* flux future destroyed in lookup_ctx_destroy, which is called
      * via zlist_remove() */
     json_decref (o);
-    free (data);
+    free (current_value);
     zlist_remove (ctx->lookups, l);
 }
 
-/* If keys array doesn't contain eventlog, flag that we'll need to do
- * an eventlog check.
+/* If we need the eventlog for an allow check or for update-lookup
+ * we need to add it to the key lookup list.
  */
-static int check_keys_for_eventlog (struct lookup_ctx *l)
+static void check_to_lookup_eventlog (struct lookup_ctx *l)
 {
-    size_t index;
+    if (!l->allow || (l->flags & FLUX_JOB_LOOKUP_CURRENT)) {
+        size_t index;
+        json_t *key;
+        json_array_foreach (l->keys, index, key) {
+            if (streq (json_string_value (key), "eventlog"))
+                return;
+        }
+        l->lookup_eventlog = true;
+    }
+}
+
+static json_t *get_json_string (json_t *o)
+{
+    char *s = json_dumps (o, JSON_ENCODE_ANY);
+    json_t *tmp = NULL;
+    /* We assume json is internally valid, thus this is an ENOMEM error */
+    if (!s) {
+        errno = ENOMEM;
+        goto cleanup;
+    }
+    if (!(tmp = json_string (s))) {
+        errno = ENOMEM;
+        goto cleanup;
+    }
+cleanup:
+    free (s);
+    return tmp;
+}
+
+/* returns -1 on error, 1 on cached response returned, 0 on no cache */
+static int lookup_cached (struct lookup_ctx *l)
+{
+    json_t *current_object = NULL;
     json_t *key;
+    const char *key_str;
+    int ret, rv = -1;
+
+    /* Special optimization, looking for a single updated value that
+     * could be cached via an update-watch
+     *
+     * - Caller must want current / updated value
+     * - This lookup is already allowed (i.e. if we have to do a
+     *   "allow" KVS lookup, there is little benefit to returning the
+     *   cached value).
+     * - The caller only wants one key (i.e. if we have to do lookup
+     *   on another value anyways, there is little benefit to
+     *   returning the cached value).
+     */
+
+    if (!(l->flags & FLUX_JOB_LOOKUP_CURRENT)
+        || !l->allow
+        || json_array_size (l->keys) != 1)
+        return 0;
+
+    key = json_array_get (l->keys, 0);
+    if (!key) {
+        errno = EINVAL;
+        goto cleanup;
+    }
 
-    json_array_foreach(l->keys, index, key) {
-        const char *keystr;
-        if (!(keystr = json_string_value (key))) {
-            errno = EINVAL;
-            return -1;
+    key_str = json_string_value (key);
+
+    if (!streq (key_str, "R") && !streq (key_str, "jobspec"))
+        return 0;
+
+    if ((ret = update_watch_get_cached (l->ctx,
+                                        l->id,
+                                        key_str,
+                                        &current_object)) < 0)
+        goto cleanup;
+
+    if (ret) {
+        if (l->flags & FLUX_JOB_LOOKUP_JSON_DECODE) {
+            if (flux_respond_pack (l->ctx->h,
+                                   l->msg,
+                                   "{s:I s:O}",
+                                   "id", l->id,
+                                   key_str, current_object) < 0) {
+                flux_log_error (l->ctx->h, "%s: flux_respond", __FUNCTION__);
+                goto cleanup;
+            }
+            rv = 1;
+            goto cleanup;
+        }
+        else {
+            json_t *o = get_json_string (current_object);
+            if (!o) {
+                errno = ENOMEM;
+                goto cleanup;
+            }
+            if (flux_respond_pack (l->ctx->h,
+                                   l->msg,
+                                   "{s:I s:O}",
+                                   "id", l->id,
+                                   key_str, o) < 0) {
+                json_decref (o);
+                flux_log_error (l->ctx->h, "%s: flux_respond", __FUNCTION__);
+                goto cleanup;
+            }
+            rv = 1;
+            json_decref (o);
+            goto cleanup;
         }
-        if (!strcmp (keystr, "eventlog"))
-            return 0;
     }
 
-    l->check_eventlog = true;
+    rv = 0;
+cleanup:
+    json_decref (current_object);
+    return rv;
+}
+
+static int lookup (flux_t *h,
+                   const flux_msg_t *msg,
+                   struct info_ctx *ctx,
+                   flux_jobid_t id,
+                   json_t *keys,
+                   int flags,
+                   flux_error_t *error)
+{
+    struct lookup_ctx *l = NULL;
+    int ret;
+
+    if (!(l = lookup_ctx_create (ctx, msg, id, keys, flags))) {
+        errprintf (error,
+                   "could not create lookup context: %s",
+                   strerror (errno));
+        goto error;
+    }
+
+    /* If authorization is indeterminate at this stage (l->allow == false),
+     * look up the eventlog and authorize in the continuation.  N.B. we could
+     * summarily allow the instance owner without looking up the eventlog,
+     * but then we could not differentiate an invalid job ID and a missing key.
+     * Since keys are looked up in parallel, this should not be too costly.
+     * See also: flux-framework/flux-core#6325
+     */
+    switch (eventlog_allow_lru (l->ctx, l->msg, l->id)) {
+        case -1:    // entry found - DENY
+            errprintf (error, "access is restricted to job/instance owner");
+            goto error;
+        case 1:     // entry found - ALLOW
+            l->allow = true;
+            break;
+        case 0:     // indeterminate
+            break;
+    }
+
+    if ((ret = lookup_cached (l)) < 0) {
+        errprintf (error,
+                   "internal error attempting to use update-watch cache: %s",
+                   strerror (errno));
+        goto error;
+    }
+
+    if (ret) {
+        lookup_ctx_destroy (l);
+        return 0;
+    }
+
+    check_to_lookup_eventlog (l);
+
+    if (lookup_keys (l) < 0) {
+        errprintf (error,
+                   "error sending KVS lookup request(s): %s",
+                   strerror (errno));
+        goto error;
+    }
+
+    if (zlist_append (ctx->lookups, l) < 0) {
+        errprintf (error,
+                   "internal error saving lookup context: out of memory");
+        errno = ENOMEM;
+        goto error;
+    }
+    zlist_freefn (ctx->lookups, l, lookup_ctx_destroy, true);
     return 0;
+
+error:
+    lookup_ctx_destroy (l);
+    return -1;
 }
 
-void lookup_cb (flux_t *h, flux_msg_handler_t *mh,
-                const flux_msg_t *msg, void *arg)
+void lookup_cb (flux_t *h,
+                flux_msg_handler_t *mh,
+                const flux_msg_t *msg,
+                void *arg)
 {
     struct info_ctx *ctx = arg;
-    struct lookup_ctx *l = NULL;
+    size_t index;
+    json_t *key;
     json_t *keys;
     flux_jobid_t id;
-    uint32_t rolemask;
     int flags;
+    int valid_flags = FLUX_JOB_LOOKUP_JSON_DECODE | FLUX_JOB_LOOKUP_CURRENT;
+    flux_error_t error;
+    const char *errmsg = NULL;
 
-    if (flux_request_unpack (msg, NULL, "{s:I s:o s:i}",
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:o s:i}",
                              "id", &id,
                              "keys", &keys,
-                             "flags", &flags) < 0) {
-        flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
+                             "flags", &flags) < 0)
         goto error;
-    }
 
-    if (!(l = lookup_ctx_create (ctx, msg, id, keys, flags)))
+    if (flags & ~valid_flags) {
+        errno = EPROTO;
+        errmsg = "lookup request rejected with invalid flag";
         goto error;
+    }
 
-    if (flux_msg_get_rolemask (msg, &rolemask) < 0)
+    /* validate keys is an array and all fields are strings */
+    if (!json_is_array (keys)) {
+        errno = EPROTO;
         goto error;
+    }
 
-    /* if rpc from owner, no need to do guest access check */
-    if ((rolemask & FLUX_ROLE_OWNER))
-        l->allow = true;
-    else {
-        if (check_keys_for_eventlog (l) < 0)
+    json_array_foreach (keys, index, key) {
+        if (!json_is_string (key)) {
+            errno = EPROTO;
             goto error;
+        }
     }
 
-    if (lookup_keys (l) < 0)
+    if (lookup (h, msg, ctx, id, keys, flags, &error) < 0) {
+        errmsg = error.text;
         goto error;
+    }
 
-    if (zlist_append (ctx->lookups, l) < 0) {
-        flux_log_error (h, "%s: zlist_append", __FUNCTION__);
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+/* legacy rpc target */
+void update_lookup_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    struct info_ctx *ctx = arg;
+    flux_jobid_t id;
+    const char *key = NULL;
+    json_t *keys = NULL;
+    int flags;
+    int valid_flags = 0;
+    flux_error_t error;
+    const char *errmsg = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:s s:i}",
+                             "id", &id,
+                             "key", &key,
+                             "flags", &flags) < 0)
+        goto error;
+    if ((flags & ~valid_flags)) {
+        errno = EPROTO;
+        errmsg = "update-lookup request rejected with invalid flag";
+        goto error;
+    }
+    if (!streq (key, "R")) {
+        errno = EINVAL;
+        errmsg = "update-lookup unsupported key specified";
+        goto error;
+    }
+
+    if (!(keys = json_pack ("[s]", key))) {
+        errno = ENOMEM;
         goto error;
     }
-    zlist_freefn (ctx->lookups, l, lookup_ctx_destroy, true);
 
+    if (lookup (h,
+                msg,
+                ctx,
+                id,
+                keys,
+                FLUX_JOB_LOOKUP_JSON_DECODE | FLUX_JOB_LOOKUP_CURRENT,
+                &error) < 0) {
+        errmsg = error.text;
+        goto error;
+    }
     return;
 
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    lookup_ctx_destroy (l);
+    json_decref (keys);
 }
 
 /*
diff --git a/src/modules/job-info/lookup.h b/src/modules/job-info/lookup.h
index b2b0d44f1a25..4b69238e7e59 100644
--- a/src/modules/job-info/lookup.h
+++ b/src/modules/job-info/lookup.h
@@ -13,10 +13,16 @@
 
 #include <flux/core.h>
 
-#include "info.h"
+void lookup_cb (flux_t *h,
+                flux_msg_handler_t *mh,
+                const flux_msg_t *msg,
+                void *arg);
 
-void lookup_cb (flux_t *h, flux_msg_handler_t *mh,
-                const flux_msg_t *msg, void *arg);
+/* legacy rpc target */
+void update_lookup_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg);
 
 #endif /* ! _FLUX_JOB_INFO_LOOKUP_H */
 
diff --git a/src/modules/job-info/update.c b/src/modules/job-info/update.c
new file mode 100644
index 000000000000..cb2a3d4557eb
--- /dev/null
+++ b/src/modules/job-info/update.c
@@ -0,0 +1,647 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* update.c - handle job-info.update-watch for job-info */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libjob/job.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "ccan/str/str.h"
+
+#include "job-info.h"
+#include "update.h"
+#include "util.h"
+
+struct update_ctx {
+    struct info_ctx *ctx;
+    struct flux_msglist *msglist;
+    uint32_t userid;
+    flux_jobid_t id;
+    char *key;
+    int flags;
+    const char *update_name;
+    flux_future_t *lookup_f;
+    flux_future_t *eventlog_watch_f;
+    bool eventlog_watch_canceled;
+    json_t *update_object;
+    int initial_update_count;
+    int watch_update_count;
+    char *index_key;
+};
+
+static void update_ctx_destroy (void *data)
+{
+    if (data) {
+        struct update_ctx *uc = data;
+        int save_errno = errno;
+        flux_msglist_destroy (uc->msglist);
+        free (uc->key);
+        flux_future_destroy (uc->lookup_f);
+        flux_future_destroy (uc->eventlog_watch_f);
+        json_decref (uc->update_object);
+        free (uc->index_key);
+        free (uc);
+        errno = save_errno;
+    }
+}
+
+static char *get_index_key (flux_jobid_t id, const char *key)
+{
+    char *s;
+    if (asprintf (&s, "%ju-%s", (uintmax_t)id, key) < 0)
+        return NULL;
+    return s;
+}
+
+static struct update_ctx *update_ctx_create (struct info_ctx *ctx,
+                                             const flux_msg_t *msg,
+                                             flux_jobid_t id,
+                                             const char *key,
+                                             int flags)
+{
+    struct update_ctx *uc = calloc (1, sizeof (*uc));
+
+    if (!uc)
+        return NULL;
+
+    uc->ctx = ctx;
+    uc->id = id;
+    if (!(uc->key = strdup (key)))
+        goto error;
+    if (streq (key, "R"))
+        uc->update_name = "resource-update";
+    else if (streq (key, "jobspec"))
+        uc->update_name = "jobspec-update";
+    else {
+        errno = EINVAL;
+        goto error;
+    }
+    uc->flags = flags;
+
+    /* for lookups, the msglist will never be > 1 in length */
+    if (!(uc->msglist = flux_msglist_create ()))
+        goto error;
+    flux_msglist_append (uc->msglist, msg);
+
+    /* use jobid + key as lookup key, in future we may support other
+     * keys other than R
+     */
+    if (!(uc->index_key = get_index_key (uc->id, uc->key)))
+        goto error;
+
+    return uc;
+
+error:
+    update_ctx_destroy (uc);
+    return NULL;
+}
+
+static void eventlog_watch_cancel (struct update_ctx *uc)
+{
+    flux_future_t *f;
+    int matchtag;
+
+    /* in some cases, possible eventlog watch hasn't started yet */
+    if (!uc->eventlog_watch_f || uc->eventlog_watch_canceled)
+        return;
+
+    matchtag = (int)flux_rpc_get_matchtag (uc->eventlog_watch_f);
+
+    if (!(f = flux_rpc_pack (uc->ctx->h,
+                             "job-info.eventlog-watch-cancel",
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_NORESPONSE,
+                             "{s:i}",
+                             "matchtag", matchtag))) {
+        flux_log_error (uc->ctx->h, "%s: flux_rpc_pack", __FUNCTION__);
+        return;
+    }
+    flux_future_destroy (f);
+    uc->eventlog_watch_canceled = true;
+}
+
+static void eventlog_continuation (flux_future_t *f, void *arg)
+{
+    struct update_ctx *uc = arg;
+    struct info_ctx *ctx = uc->ctx;
+    const char *s;
+    json_t *event = NULL;
+    const char *name;
+    json_t *context = NULL;
+    const char *errmsg = NULL;
+    const flux_msg_t *msg;
+
+    if (flux_rpc_get (f, NULL) < 0) {
+        /* ENODATA is normal when job finishes or we've sent cancel */
+        if (errno != ENODATA)
+            flux_log_error (ctx->h, "%s: job-info.eventlog-watch", __FUNCTION__);
+        goto error;
+    }
+
+    /* if count == 0, all callers canceled streams */
+    if (flux_msglist_count (uc->msglist) == 0)
+        goto cleanup;
+
+    if (flux_job_event_watch_get (f, &s) < 0) {
+        flux_log_error (ctx->h, "%s: flux_job_event_watch_get", __FUNCTION__);
+        eventlog_watch_cancel (uc);
+        goto cleanup;
+    }
+
+    if (!(event = eventlog_entry_decode (s))) {
+        flux_log_error (uc->ctx->h, "%s: eventlog_entry_decode", __FUNCTION__);
+        eventlog_watch_cancel (uc);
+        goto cleanup;
+    }
+
+    if (eventlog_entry_parse (event, NULL, &name, &context) < 0) {
+        flux_log_error (uc->ctx->h, "%s: eventlog_entry_decode", __FUNCTION__);
+        eventlog_watch_cancel (uc);
+        goto cleanup;
+    }
+
+    if (context && streq (name, uc->update_name)) {
+        uc->watch_update_count++;
+
+        /* don't apply update events that we've already applied from
+         * initial lookup */
+        if (uc->watch_update_count > uc->initial_update_count) {
+            if (streq (uc->key, "R"))
+                apply_updates_R (uc->ctx->h,
+                                 uc->id,
+                                 uc->key,
+                                 uc->update_object,
+                                 context);
+            else if (streq (uc->key, "jobspec"))
+                apply_updates_jobspec (uc->ctx->h,
+                                       uc->id,
+                                       uc->key,
+                                       uc->update_object,
+                                       context);
+
+            msg = flux_msglist_first (uc->msglist);
+            while (msg) {
+                if (flux_respond_pack (uc->ctx->h,
+                                       msg,
+                                       "{s:O}",
+                                       uc->key, uc->update_object) < 0) {
+                    flux_log_error (ctx->h, "%s: flux_respond", __FUNCTION__);
+                    eventlog_watch_cancel (uc);
+                    goto cleanup;
+                }
+                msg = flux_msglist_next (uc->msglist);
+            }
+        }
+    }
+
+    flux_future_reset (f);
+    json_decref (event);
+    return;
+
+error:
+    msg = flux_msglist_first (uc->msglist);
+    while (msg) {
+        if (flux_respond_error (ctx->h, msg, errno, errmsg) < 0)
+            flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+        msg = flux_msglist_next (uc->msglist);
+    }
+
+cleanup:
+    /* flux future destroyed in update_ctx_destroy, which is
+     * called via zlist_remove() */
+    zhashx_delete (ctx->index_uw, uc->index_key);
+    zlist_remove (ctx->update_watchers, uc);
+}
+
+static int eventlog_watch (struct update_ctx *uc)
+{
+    const char *topic = "job-info.eventlog-watch";
+    int rpc_flags = FLUX_RPC_STREAMING;
+    flux_msg_t *msg = NULL;
+    int save_errno, rc = -1;
+
+    if (!(uc->eventlog_watch_f = flux_rpc_pack (uc->ctx->h,
+                                                topic,
+                                                FLUX_NODEID_ANY,
+                                                rpc_flags,
+                                                "{s:I s:s s:i}",
+                                                "id", uc->id,
+                                                "path", "eventlog",
+                                                "flags", 0))) {
+        flux_log_error (uc->ctx->h, "%s: flux_rpc_pack", __FUNCTION__);
+        goto error;
+    }
+
+    if (flux_future_then (uc->eventlog_watch_f,
+                          -1,
+                          eventlog_continuation,
+                          uc) < 0) {
+        /* future cleanup handled with context destruction */
+        flux_log_error (uc->ctx->h, "%s: flux_future_then", __FUNCTION__);
+        goto error;
+    }
+
+    rc = 0;
+error:
+    save_errno = errno;
+    flux_msg_destroy (msg);
+    errno = save_errno;
+    return rc;
+}
+
+static void lookup_continuation (flux_future_t *f, void *arg)
+{
+    struct update_ctx *uc = arg;
+    struct info_ctx *ctx = uc->ctx;
+    const char *key_str;
+    const char *eventlog_str;
+    json_t *eventlog = NULL;
+    size_t index;
+    json_t *entry;
+    const char *errmsg = NULL;
+    bool job_ended = false;
+    bool submit_parsed = false;
+    const flux_msg_t *msg;
+
+    if (flux_rpc_get_unpack (f,
+                             "{s:s s:s}",
+                             uc->key, &key_str,
+                             "eventlog", &eventlog_str) < 0) {
+        if (errno != ENOENT && errno != EPERM)
+            flux_log_error (ctx->h, "%s: flux_rpc_get_unpack", __FUNCTION__);
+        goto error;
+    }
+
+    /* if count == 0, all callers canceled streams */
+    if (flux_msglist_count (uc->msglist) == 0)
+        goto cleanup;
+
+    if (!(uc->update_object = json_loads (key_str, 0, NULL))) {
+        errno = EINVAL;
+        errmsg = "lookup value cannot be parsed";
+        goto error;
+    }
+
+    if (!(eventlog = eventlog_decode (eventlog_str))) {
+        errno = EINVAL;
+        errmsg = "lookup eventlog cannot be parsed";
+        goto error;
+    }
+    json_array_foreach (eventlog, index, entry) {
+        const char *name;
+        json_t *context = NULL;
+        if (eventlog_entry_parse (entry, NULL, &name, &context) < 0) {
+            errmsg = "error parsing eventlog";
+            goto error;
+        }
+        if (streq (name, "submit")) {
+            if (!context) {
+                errno = EPROTO;
+                goto error;
+            }
+            if (json_unpack (context, "{ s:i }", "userid", &uc->userid) < 0) {
+                errno = EPROTO;
+                goto error;
+            }
+            submit_parsed = true;
+        }
+        else if (streq (name, uc->update_name)) {
+            if (streq (uc->key, "R"))
+                apply_updates_R (uc->ctx->h,
+                                 uc->id,
+                                 uc->key,
+                                 uc->update_object,
+                                 context);
+            else if (streq (uc->key, "jobspec"))
+                apply_updates_jobspec (uc->ctx->h,
+                                       uc->id,
+                                       uc->key,
+                                       uc->update_object,
+                                       context);
+            uc->initial_update_count++;
+        }
+        else if (streq (name, "clean"))
+            job_ended = true;
+    }
+
+    /* double check, generally speaking should be impossible */
+    if (!submit_parsed) {
+        errno = EPROTO;
+        goto error;
+    }
+
+    msg = flux_msglist_first (uc->msglist);
+    while (msg) {
+        /* caller can't access this data, this is not a "fatal" error,
+         * so send error to this one message and continue on the
+         * msglist
+         */
+        if (flux_msg_authorize (msg, uc->userid) < 0) {
+            if (flux_respond_error (ctx->h, msg, errno, NULL) < 0)
+                flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+            flux_msglist_delete (uc->msglist);
+            goto next;
+        }
+
+        if (flux_respond_pack (uc->ctx->h, msg, "{s:O}",
+                               uc->key, uc->update_object) < 0) {
+            flux_log_error (ctx->h, "%s: flux_respond", __FUNCTION__);
+            goto cleanup;
+        }
+
+    next:
+        msg = flux_msglist_next (uc->msglist);
+    }
+
+    /* due to security check above, possible no more messages in this
+     * watcher */
+    if (flux_msglist_count (uc->msglist) == 0)
+        goto cleanup;
+
+    /* this job has ended, no need to watch the eventlog for future
+     * updates */
+    if (job_ended) {
+        errno = ENODATA;
+        goto error;
+    }
+
+    /* we've confirmed key is readable and sent to caller, now watch
+     * eventlog for future changes */
+    if (eventlog_watch (uc) < 0)
+        goto error;
+
+    json_decref (eventlog);
+    return;
+
+error:
+    msg = flux_msglist_first (uc->msglist);
+    while (msg) {
+        if (flux_respond_error (ctx->h, msg, errno, errmsg) < 0)
+            flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+        msg = flux_msglist_next (uc->msglist);
+    }
+
+cleanup:
+    /* flux future destroyed in update_ctx_destroy, which is called
+     * via zlist_remove() */
+    zhashx_delete (ctx->index_uw, uc->index_key);
+    zlist_remove (ctx->update_watchers, uc);
+    json_decref (eventlog);
+}
+
+static int update_lookup (struct info_ctx *ctx,
+                          const flux_msg_t *msg,
+                          flux_jobid_t id,
+                          const char *key,
+                          int flags)
+{
+    struct update_ctx *uc = NULL;
+    const char *topic = "job-info.lookup";
+
+    if (!(uc = update_ctx_create (ctx,
+                                  msg,
+                                  id,
+                                  key,
+                                  flags)))
+        goto error;
+
+    if (!(uc->lookup_f = flux_rpc_pack (uc->ctx->h,
+                                        topic,
+                                        FLUX_NODEID_ANY,
+                                        0,
+                                        "{s:I s:[ss] s:i}",
+                                        "id", uc->id,
+                                        "keys", uc->key, "eventlog",
+                                        "flags", 0))) {
+        flux_log_error (uc->ctx->h, "%s: flux_rpc_pack", __FUNCTION__);
+        goto error;
+    }
+
+    if (flux_future_then (uc->lookup_f,
+                          -1,
+                          lookup_continuation,
+                          uc) < 0) {
+        /* future cleanup handled with context destruction */
+        flux_log_error (uc->ctx->h, "%s: flux_future_then", __FUNCTION__);
+        goto error;
+    }
+
+    if (zlist_append (ctx->update_watchers, uc) < 0) {
+        flux_log_error (ctx->h, "%s: zlist_append", __FUNCTION__);
+        goto error;
+    }
+    zlist_freefn (ctx->update_watchers, uc, update_ctx_destroy, true);
+
+    if (zhashx_insert (ctx->index_uw, uc->index_key, uc) < 0) {
+        flux_log_error (ctx->h, "%s: zhashx_insert", __FUNCTION__);
+        goto error_list;
+    }
+
+    return 0;
+
+error_list:
+    zlist_remove (ctx->update_watchers, uc);
+    return -1;
+
+error:
+    update_ctx_destroy (uc);
+    return -1;
+}
+
+void update_watch_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
+{
+    struct info_ctx *ctx = arg;
+    struct update_ctx *uc = NULL;
+    flux_jobid_t id;
+    const char *key = NULL;
+    int flags;
+    int valid_flags = 0;
+    const char *errmsg = NULL;
+    char *index_key = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:s s:i}",
+                             "id", &id,
+                             "key", &key,
+                             "flags", &flags) < 0)
+        goto error;
+    if ((flags & ~valid_flags)) {
+        errno = EPROTO;
+        errmsg = "update-watch request rejected with invalid flag";
+        goto error;
+    }
+    if (!flux_msg_is_streaming (msg)) {
+        errno = EPROTO;
+        errmsg = "update-watch request rejected without streaming RPC flag";
+        goto error;
+    }
+    if (!streq (key, "R") && !streq (key, "jobspec")) {
+        errno = EINVAL;
+        errmsg = "update-watch unsupported key specified";
+        goto error;
+    }
+
+    if (!(index_key = get_index_key (id, key)))
+        goto error;
+
+    /* if no watchers for this jobid yet, start it */
+    if (!(uc = zhashx_lookup (ctx->index_uw, index_key))) {
+        if (update_lookup (ctx,
+                           msg,
+                           id,
+                           key,
+                           flags) < 0)
+            goto error;
+    }
+    else {
+        if (uc->update_object) {
+            if (flux_msg_authorize (msg, uc->userid) < 0)
+                goto error;
+            if (flux_respond_pack (uc->ctx->h,
+                                   msg,
+                                   "{s:O}",
+                                   uc->key, uc->update_object) < 0) {
+                flux_log_error (ctx->h, "%s: flux_respond", __FUNCTION__);
+                goto cleanup;
+            }
+        }
+        /* if uc->update_object has not been set, the initial lookup
+         * has not completed.  The security check will be done in
+         * watch_lookup_continuation when the initial lookup completes.
+         */
+        flux_msglist_append (uc->msglist, msg);
+    }
+
+    free (index_key);
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+cleanup:
+    free (index_key);
+}
+
+int update_watch_get_cached (struct info_ctx *ctx,
+                             flux_jobid_t id,
+                             const char *key,
+                             json_t **current_object)
+{
+    struct update_ctx *uc;
+    char *index_key;
+    int rv = 0;
+
+    /* if somebody is already watching this jobid + key, we read the
+     * cached value */
+
+    if (!(index_key = get_index_key (id, key))) {
+        errno = ENOMEM;
+        return -1;
+    }
+
+    if ((uc = zhashx_lookup (ctx->index_uw, index_key))
+        && uc->update_object) {
+        (*current_object) = json_incref (uc->update_object);
+        rv = 1;
+    }
+
+    free (index_key);
+    return rv;
+}
+
+/* Cancel update_watch if it matches message.
+ */
+static void update_watch_cancel (struct update_ctx *uc,
+                                 const flux_msg_t *msg,
+                                 bool cancel)
+{
+    if (cancel) {
+        if (flux_msglist_cancel (uc->ctx->h, uc->msglist, msg) < 0)
+            flux_log_error (uc->ctx->h,
+                            "error handling job-info.update-watch-cancel");
+    }
+    else {
+        if (flux_msglist_disconnect (uc->msglist, msg) < 0)
+            flux_log_error (uc->ctx->h,
+                            "error handling job-info.update-watch disconnect");
+    }
+
+    if (flux_msglist_count (uc->msglist) == 0)
+        eventlog_watch_cancel (uc);
+}
+
+void update_watchers_cancel (struct info_ctx *ctx,
+                             const flux_msg_t *msg,
+                             bool cancel)
+{
+    struct update_ctx *uc;
+
+    uc = zlist_first (ctx->update_watchers);
+    while (uc) {
+        update_watch_cancel (uc, msg, cancel);
+        uc = zlist_next (ctx->update_watchers);
+    }
+}
+
+void update_watch_cancel_cb (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
+{
+    struct info_ctx *ctx = arg;
+    update_watchers_cancel (ctx, msg, true);
+}
+
+void update_watch_cleanup (struct info_ctx *ctx)
+{
+    struct update_ctx *uc;
+
+    while ((uc = zlist_pop (ctx->update_watchers))) {
+        const flux_msg_t *msg;
+        eventlog_watch_cancel (uc);
+        msg = flux_msglist_first (uc->msglist);
+        while (msg) {
+            if (flux_respond_error (ctx->h, msg, ENOSYS, NULL) < 0) {
+                flux_log_error (ctx->h,
+                                "%s: flux_respond_error",
+                                __FUNCTION__);
+            }
+            msg = flux_msglist_next (uc->msglist);
+        }
+        update_ctx_destroy (uc);
+    }
+}
+
+int update_watch_count (struct info_ctx *ctx)
+{
+    struct update_ctx *uc;
+    int count = 0;
+
+    uc = zlist_first (ctx->update_watchers);
+    while (uc) {
+        count += flux_msglist_count (uc->msglist);
+        uc = zlist_next (ctx->update_watchers);
+    }
+    return count;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-info/update.h b/src/modules/job-info/update.h
new file mode 100644
index 000000000000..eb5151415a82
--- /dev/null
+++ b/src/modules/job-info/update.h
@@ -0,0 +1,49 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_INFO_UPDATE_H
+#define _FLUX_JOB_INFO_UPDATE_H
+
+#include <flux/core.h>
+#include <jansson.h>
+
+void update_watch_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg);
+
+void update_watch_cancel_cb (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg);
+
+/* returns 1 on found, 0 if not, -1 on error */
+int update_watch_get_cached (struct info_ctx *ctx,
+                             flux_jobid_t id,
+                             const char *key,
+                             json_t **current_object);
+
+/* Cancel all update watches that match msg.
+ * match credentials & matchtag if cancel true
+ * match credentials if cancel false
+ */
+void update_watchers_cancel (struct info_ctx *ctx,
+                             const flux_msg_t *msg,
+                             bool cancel);
+
+void update_watch_cleanup (struct info_ctx *ctx);
+
+int update_watch_count (struct info_ctx *ctx);
+
+#endif /* ! _FLUX_JOB_INFO_UPDATE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-info/util.c b/src/modules/job-info/util.c
new file mode 100644
index 000000000000..95a34d126cb7
--- /dev/null
+++ b/src/modules/job-info/util.c
@@ -0,0 +1,157 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <string.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libutil/jpath.h"
+
+#include "ccan/str/str.h"
+
+flux_msg_t *cred_msg_pack (const char *topic,
+                           struct flux_msg_cred cred,
+                           const char *fmt,
+                           ...)
+{
+    flux_msg_t *newmsg = NULL;
+    flux_msg_t *rv = NULL;
+    int save_errno;
+    va_list ap;
+
+    va_start (ap, fmt);
+
+    if (!(newmsg = flux_request_encode (topic, NULL)))
+        goto error;
+    if (flux_msg_set_cred (newmsg, cred) < 0)
+        goto error;
+    if (flux_msg_vpack (newmsg, fmt, ap) < 0)
+        goto error;
+    rv = newmsg;
+error:
+    save_errno = errno;
+    if (!rv)
+        flux_msg_destroy (newmsg);
+    va_end (ap);
+    errno = save_errno;
+    return rv;
+}
+
+bool get_next_eventlog_entry (const char **pp,
+                              const char **tok,
+                              size_t *toklen)
+{
+    char *term;
+
+    if (!(term = strchr (*pp, '\n')))
+        return false;
+    *tok = *pp;
+    *toklen = term - *pp + 1;
+    *pp = term + 1;
+    return true;
+}
+
+int parse_eventlog_entry (flux_t *h,
+                          const char *tok,
+                          size_t toklen,
+                          json_t **entry,
+                          const char **name,
+                          json_t **context)
+{
+    char *str = NULL;
+    json_t *o = NULL;
+    int saved_errno, rc = -1;
+
+    if (!(str = strndup (tok, toklen))) {
+        flux_log_error (h, "%s: strndup", __FUNCTION__);
+        goto error;
+    }
+
+    if (!(o = eventlog_entry_decode (str))) {
+        flux_log_error (h, "%s: eventlog_entry_decode", __FUNCTION__);
+        goto error;
+    }
+
+    if (eventlog_entry_parse (o, NULL, name, context) < 0) {
+        flux_log_error (h, "%s: eventlog_entry_parse", __FUNCTION__);
+        goto error;
+    }
+
+    (*entry) = o;
+    free (str);
+    return 0;
+
+error:
+    saved_errno = errno;
+    free (str);
+    json_decref (o);
+    errno = saved_errno;
+    return rc;
+}
+
+void apply_updates_R (flux_t *h,
+                      flux_jobid_t id,
+                      const char *key,
+                      json_t *R,
+                      json_t *context)
+{
+    const char *ckey;
+    json_t *value;
+
+    json_object_foreach (context, ckey, value) {
+        /* RFC 21 resource-update event only allows update
+         * to:
+         * - expiration
+         */
+        if (streq (ckey, "expiration"))
+            if (jpath_set (R,
+                           "execution.expiration",
+                           value) < 0) {
+                flux_log (h,
+                          LOG_INFO,
+                          "%s: failed to update job %s %s",
+                          __FUNCTION__,
+                          idf58 (id),
+                          key);
+                }
+    }
+}
+
+void apply_updates_jobspec (flux_t *h,
+                            flux_jobid_t id,
+                            const char *key,
+                            json_t *jobspec,
+                            json_t *context)
+{
+    const char *ckey;
+    json_t *value;
+
+    json_object_foreach (context, ckey, value) {
+        if (jpath_set (jobspec,
+                       ckey,
+                       value) < 0) {
+            flux_log (h,
+                      LOG_INFO,
+                      "%s: failed to update job %s %s",
+                      __FUNCTION__,
+                      idf58 (id),
+                      key);
+        }
+    }
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-info/util.h b/src/modules/job-info/util.h
new file mode 100644
index 000000000000..51f442a544d1
--- /dev/null
+++ b/src/modules/job-info/util.h
@@ -0,0 +1,57 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_INFO_UTIL_H
+#define _FLUX_JOB_INFO_UTIL_H
+
+#include <jansson.h>
+#include <flux/core.h>
+
+/* we want to copy credentials, etc. from the original
+ * message when we send RPCs to other job-info targets.
+ */
+flux_msg_t *cred_msg_pack (const char *topic,
+                           struct flux_msg_cred cred,
+                           const char *fmt,
+                           ...);
+
+/* helper to parse next eventlog entry when whole eventlog is read */
+bool get_next_eventlog_entry (const char **pp,
+                              const char **tok,
+                              size_t *toklen);
+
+/* parse chunk from eventlog_parse_next, 'entry' is required and
+ * should be json_decref'ed after use */
+int parse_eventlog_entry (flux_t *h,
+                          const char *tok,
+                          size_t toklen,
+                          json_t **entry,
+                          const char **name,
+                          json_t **context);
+
+/* apply context updates to the R object */
+void apply_updates_R (flux_t *h,
+                      flux_jobid_t id,
+                      const char *key,
+                      json_t *R,
+                      json_t *context);
+
+/* apply context updates to the jobspec object */
+void apply_updates_jobspec (flux_t *h,
+                            flux_jobid_t id,
+                            const char *key,
+                            json_t *jobspec,
+                            json_t *context);
+
+#endif /* ! _FLUX_JOB_INFO_UTIL_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-info/watch.c b/src/modules/job-info/watch.c
index ca1423c1591a..36877a8a9111 100644
--- a/src/modules/job-info/watch.c
+++ b/src/modules/job-info/watch.c
@@ -14,16 +14,18 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libjob/job.h"
-#include "src/common/libeventlog/eventlog.h"
+#include "ccan/str/str.h"
 
-#include "info.h"
+#include "job-info.h"
 #include "watch.h"
+#include "guest_watch.h"
 #include "allow.h"
+#include "util.h"
 
 struct watch_ctx {
     struct info_ctx *ctx;
@@ -35,6 +37,7 @@ struct watch_ctx {
     flux_future_t *check_f;
     flux_future_t *watch_f;
     bool allow;
+    bool kvs_watch_canceled;
     bool cancel;
 };
 
@@ -45,11 +48,13 @@ static void watch_ctx_destroy (void *data)
 {
     if (data) {
         struct watch_ctx *ctx = data;
+        int save_errno = errno;
         flux_msg_decref (ctx->msg);
         free (ctx->path);
         flux_future_destroy (ctx->check_f);
         flux_future_destroy (ctx->watch_f);
         free (ctx);
+        errno = save_errno;
     }
 }
 
@@ -61,7 +66,6 @@ static struct watch_ctx *watch_ctx_create (struct info_ctx *ctx,
                                            int flags)
 {
     struct watch_ctx *w = calloc (1, sizeof (*w));
-    int saved_errno;
 
     if (!w)
         return NULL;
@@ -80,9 +84,7 @@ static struct watch_ctx *watch_ctx_create (struct info_ctx *ctx,
     return w;
 
 error:
-    saved_errno = errno;
     watch_ctx_destroy (w);
-    errno = saved_errno;
     return NULL;
 }
 
@@ -117,6 +119,9 @@ static int watch_key (struct watch_ctx *w)
     char *pathptr = NULL;
     int flags = (FLUX_KVS_WATCH | FLUX_KVS_WATCH_APPEND);
 
+    if (w->flags & FLUX_JOB_EVENT_WATCH_WAITCREATE)
+        flags |= FLUX_KVS_WAITCREATE;
+
     if (w->guest) {
         if (flux_job_kvs_namespace (ns, sizeof (ns), w->id) < 0) {
             flux_log_error (w->ctx->h, "%s: flux_job_kvs_namespace",
@@ -127,7 +132,10 @@ static int watch_key (struct watch_ctx *w)
         pathptr = w->path;
     }
     else {
-        if (flux_job_kvs_key (fullpath, sizeof (fullpath), w->id, w->path) < 0) {
+        if (flux_job_kvs_key (fullpath,
+                              sizeof (fullpath),
+                              w->id,
+                              w->path) < 0) {
             flux_log_error (w->ctx->h, "%s: flux_job_kvs_key", __FUNCTION__);
             return -1;
         }
@@ -161,16 +169,18 @@ static void check_eventlog_continuation (flux_future_t *f, void *arg)
     }
 
     if (!w->allow) {
-        if (eventlog_allow (ctx, w->msg, s) < 0)
+        if (eventlog_allow (ctx, w->msg, w->id, s) < 0)
             goto error;
         w->allow = true;
     }
 
     /* There is a chance user canceled before we began legitimately
      * "watching" the desired eventlog */
-    if (w->cancel) {
-        if (flux_respond_error (ctx->h, w->msg, ENODATA, NULL) < 0)
-            flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+    if (w->kvs_watch_canceled) {
+        if (w->cancel) {
+            if (flux_respond_error (ctx->h, w->msg, ENODATA, NULL) < 0)
+                flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+        }
         goto done;
     }
 
@@ -188,52 +198,20 @@ static void check_eventlog_continuation (flux_future_t *f, void *arg)
     zlist_remove (ctx->watchers, w);
 }
 
-static bool eventlog_parse_next (const char **pp, const char **tok,
-                                 size_t *toklen)
-{
-    char *term;
-
-    if (!(term = strchr (*pp, '\n')))
-        return false;
-    *tok = *pp;
-    *toklen = term - *pp + 1;
-    *pp = term + 1;
-    return true;
-}
-
 static int check_eventlog_end (struct watch_ctx *w,
                                const char *tok,
                                size_t toklen)
 {
-    char *str = NULL;
-    json_t *o = NULL;
-    const char *name = NULL;
-    int saved_errno, rc = -1;
-
-    if (!(str = strndup (tok, toklen))) {
-        flux_log_error (w->ctx->h, "%s: strndup", __FUNCTION__);
-        goto error;
-    }
-
-    if (!(o = eventlog_entry_decode (str))) {
-        flux_log_error (w->ctx->h, "%s: eventlog_entry_decode", __FUNCTION__);
-        goto error;
-    }
+    const char *name;
+    json_t *entry = NULL;
+    int rc = 0;
 
-    if (eventlog_entry_parse (o, NULL, &name, NULL) < 0) {
-        flux_log_error (w->ctx->h, "%s: eventlog_entry_parse", __FUNCTION__);
-        goto error;
-    }
+    if (parse_eventlog_entry (w->ctx->h, tok, toklen, &entry, &name, NULL) < 0)
+        return -1;
 
-    if (!strcmp (name, "clean"))
+    if (streq (name, "clean"))
         rc = 1;
-    else
-        rc = 0;
-error:
-    saved_errno = errno;
-    free (str);
-    json_decref (o);
-    errno = saved_errno;
+    json_decref (entry);
     return rc;
 }
 
@@ -245,32 +223,63 @@ static void watch_continuation (flux_future_t *f, void *arg)
     const char *input;
     const char *tok;
     size_t toklen;
+    const char *errmsg = NULL;
 
     if (flux_kvs_lookup_get (f, &s) < 0) {
         if (errno != ENOENT && errno != ENODATA && errno != ENOTSUP)
             flux_log_error (ctx->h, "%s: flux_kvs_lookup_get", __FUNCTION__);
+        if (errno == ENODATA && w->kvs_watch_canceled && !w->cancel)
+            goto cleanup;
         goto error;
     }
 
-    if (w->cancel) {
-        errno = ENODATA;
+    /* Issue #4612 - zero length append illegal for an eventlog.  This
+     * most likely occurred through an illegal overwrite of the whole
+     * eventlog.
+     */
+    if (!s) {
+        errmsg = "illegal append of zero bytes";
+        errno = EINVAL;
         goto error;
     }
 
+    if (w->kvs_watch_canceled) {
+        if (w->cancel) {
+            errno = ENODATA;
+            goto error;
+        }
+        goto cleanup;
+    }
+
     if (!w->allow) {
-        if (eventlog_allow (ctx, w->msg, s) < 0)
-            goto error_cancel;
+        if (eventlog_allow (ctx, w->msg, w->id, s) < 0) {
+            if (!w->kvs_watch_canceled) {
+                if (flux_kvs_lookup_cancel (w->watch_f) < 0)
+                    flux_log_error (ctx->h,
+                                    "%s: flux_kvs_lookup_cancel",
+                                    __FUNCTION__);
+            }
+            goto cleanup;
+        }
         w->allow = true;
     }
 
     input = s;
-    while (eventlog_parse_next (&input, &tok, &toklen)) {
-        if (flux_respond_pack (ctx->h, w->msg,
+    while (get_next_eventlog_entry (&input, &tok, &toklen)) {
+        if (flux_respond_pack (ctx->h,
+                               w->msg,
                                "{s:s#}",
                                "event", tok, toklen) < 0) {
-            flux_log_error (ctx->h, "%s: flux_respond_pack",
+            flux_log_error (ctx->h,
+                            "%s: flux_respond_pack",
                             __FUNCTION__);
-            goto error_cancel;
+            if (!w->kvs_watch_canceled) {
+                if (flux_kvs_lookup_cancel (w->watch_f) < 0)
+                    flux_log_error (ctx->h,
+                                    "%s: flux_kvs_lookup_cancel",
+                                    __FUNCTION__);
+            }
+            goto cleanup;
         }
 
         /* When watching the main job eventlog, we return ENODATA back
@@ -279,10 +288,11 @@ static void watch_continuation (flux_future_t *f, void *arg)
          * An alternate main KVS namespace eventlog does not have a
          * known ruleset, so it will hang.
          */
-        if (!w->guest && !strcmp (w->path, "eventlog")) {
+        if (!w->guest && streq (w->path, "eventlog")) {
             if (check_eventlog_end (w, tok, toklen) > 0) {
                 if (flux_kvs_lookup_cancel (w->watch_f) < 0) {
-                    flux_log_error (ctx->h, "%s: flux_kvs_lookup_cancel",
+                    flux_log_error (ctx->h,
+                                    "%s: flux_kvs_lookup_cancel",
                                     __FUNCTION__);
                     goto error;
                 }
@@ -297,28 +307,82 @@ static void watch_continuation (flux_future_t *f, void *arg)
     flux_future_reset (f);
     return;
 
-error_cancel:
-    /* If we haven't sent a cancellation yet, must do so so that
-     * the future's matchtag will eventually be freed */
-    if (!w->cancel) {
-        int save_errno = errno;
-        if (flux_kvs_lookup_cancel (w->watch_f) < 0)
-            flux_log_error (ctx->h, "%s: flux_kvs_lookup_cancel",
-                            __FUNCTION__);
-        errno = save_errno;
-    }
-
 error:
-    if (flux_respond_error (ctx->h, w->msg, errno, NULL) < 0)
+    if (flux_respond_error (ctx->h, w->msg, errno, errmsg) < 0)
         flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
-
+cleanup:
     /* flux future destroyed in watch_ctx_destroy, which is called
      * via zlist_remove() */
     zlist_remove (ctx->watchers, w);
 }
 
-void watch_cb (flux_t *h, flux_msg_handler_t *mh,
-               const flux_msg_t *msg, void *arg)
+static int watch (struct info_ctx *ctx,
+                  const flux_msg_t *msg,
+                  flux_jobid_t id,
+                  const char *path,
+                  int flags,
+                  bool guest)
+{
+    struct watch_ctx *w = NULL;
+    uint32_t rolemask;
+
+    if (!(w = watch_ctx_create (ctx, msg, id, guest, path, flags)))
+        goto error;
+
+    /* if user requested an alternate path and that alternate path is
+     * not the main eventlog, we have to check the main eventlog for
+     * access first.
+     *
+     * if rpc from owner, no need to do guest access check.  Likewise
+     * if the cached check indicates we can read the alternate path.
+     */
+
+    if (flux_msg_get_rolemask (msg, &rolemask) < 0)
+        goto error;
+
+    if ((rolemask & FLUX_ROLE_OWNER))
+        w->allow = true;
+
+    if (!w->allow) {
+        int ret;
+        if ((ret = eventlog_allow_lru (w->ctx,
+                                       w->msg,
+                                       w->id)) < 0)
+            return -1;
+
+        if (ret)
+            w->allow = true;
+    }
+
+    if (path
+        && !streq (path, "eventlog")
+        && !w->allow) {
+        if (check_eventlog (w) < 0)
+            goto error;
+    }
+    else {
+        if (watch_key (w) < 0)
+            goto error;
+    }
+
+    if (zlist_append (ctx->watchers, w) < 0) {
+        flux_log_error (ctx->h, "%s: zlist_append", __FUNCTION__);
+        goto error;
+    }
+    zlist_freefn (ctx->watchers, w, watch_ctx_destroy, true);
+    w = NULL;
+
+    return 0;
+
+error:
+    watch_ctx_destroy (w);
+    return -1;
+}
+
+void watch_cb (flux_t *h,
+               flux_msg_handler_t *mh,
+               const flux_msg_t *msg,
+               void *arg)
 {
     struct info_ctx *ctx = arg;
     struct watch_ctx *w = NULL;
@@ -326,13 +390,19 @@ void watch_cb (flux_t *h, flux_msg_handler_t *mh,
     int guest = 0;
     const char *path = NULL;
     int flags;
+    int valid_flags = FLUX_JOB_EVENT_WATCH_WAITCREATE;
     const char *errmsg = NULL;
 
-    if (flux_request_unpack (msg, NULL, "{s:I s:s s:i}",
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:s s:i}",
                              "id", &id,
                              "path", &path,
-                             "flags", &flags) < 0) {
-        flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
+                             "flags", &flags) < 0)
+        goto error;
+    if ((flags & ~valid_flags)) {
+        errno = EPROTO;
+        errmsg = "eventlog-watch request rejected with invalid flag";
         goto error;
     }
     if (!flux_msg_is_streaming (msg)) {
@@ -340,31 +410,20 @@ void watch_cb (flux_t *h, flux_msg_handler_t *mh,
         errmsg = "eventlog-watch request rejected without streaming RPC flag";
         goto error;
     }
+    /* guest flag indicates to read path from guest namespace */
     (void)flux_request_unpack (msg, NULL, "{s:b}", "guest", &guest);
 
-    if (!(w = watch_ctx_create (ctx, msg, id, guest, path, flags)))
-        goto error;
-
-    /* if user requested an alternate path and that alternate path is
-     * not the main eventlog, we have to check the main eventlog for
-     * access first.
-     */
-    if (path && strcasecmp (path, "eventlog")) {
-        if (check_eventlog (w) < 0)
+    /* if watching a "guest" path, forward to guest watcher for
+     * handling */
+    if (strstarts (path, "guest.")) {
+        if (guest_watch (ctx, msg, id, path + 6, flags) < 0)
             goto error;
     }
     else {
-        if (watch_key (w) < 0)
+        if (watch (ctx, msg, id, path, flags, guest) < 0)
             goto error;
     }
 
-    if (zlist_append (ctx->watchers, w) < 0) {
-        flux_log_error (h, "%s: zlist_append", __FUNCTION__);
-        goto error;
-    }
-    zlist_freefn (ctx->watchers, w, watch_ctx_destroy, true);
-    w = NULL;
-
     return;
 
 error:
@@ -373,42 +432,40 @@ void watch_cb (flux_t *h, flux_msg_handler_t *mh,
     watch_ctx_destroy (w);
 }
 
-/* Cancel watch 'w' if it matches (sender, matchtag).
- * matchtag=FLUX_MATCHTAG_NONE matches any matchtag.
+/* Cancel watch 'w' if it matches message.
  */
-static void watch_cancel (struct info_ctx *ctx,
-                          struct watch_ctx *w,
-                          const char *sender, uint32_t matchtag)
+static void send_kvs_watch_cancel (struct info_ctx *ctx,
+                                   struct watch_ctx *w,
+                                   const flux_msg_t *msg,
+                                   bool cancel)
 {
-    uint32_t t;
-    char *s;
-
-    if (matchtag != FLUX_MATCHTAG_NONE
-        && (flux_msg_get_matchtag (w->msg, &t) < 0 || matchtag != t))
-        return;
-    if (flux_msg_get_route_first (w->msg, &s) < 0)
-        return;
-    if (!strcmp (sender, s)) {
-        w->cancel = true;
+    bool match;
+    if (cancel)
+        match = flux_cancel_match (msg, w->msg);
+    else
+        match = flux_disconnect_match (msg, w->msg);
+    if (match) {
+        w->kvs_watch_canceled = true;
+        w->cancel = cancel;
 
         /* if the watching hasn't started yet, no need to cancel */
         if (w->watch_f) {
-            if (flux_kvs_lookup_cancel (w->watch_f) < 0)
-                flux_log_error (ctx->h, "%s: flux_kvs_lookup_cancel",
+            if (flux_kvs_lookup_cancel (w->watch_f) < 0) {
+                flux_log_error (ctx->h,
+                                "%s: flux_kvs_lookup_cancel",
                                 __FUNCTION__);
+            }
         }
     }
-    free (s);
 }
 
-void watchers_cancel (struct info_ctx *ctx,
-                      const char *sender, uint32_t matchtag)
+void watchers_cancel (struct info_ctx *ctx, const flux_msg_t *msg, bool cancel)
 {
     struct watch_ctx *w;
 
     w = zlist_first (ctx->watchers);
     while (w) {
-        watch_cancel (ctx, w, sender, matchtag);
+        send_kvs_watch_cancel (ctx, w, msg, cancel);
         w = zlist_next (ctx->watchers);
     }
 }
@@ -417,19 +474,8 @@ void watch_cancel_cb (flux_t *h, flux_msg_handler_t *mh,
                       const flux_msg_t *msg, void *arg)
 {
     struct info_ctx *ctx = arg;
-    uint32_t matchtag;
-    char *sender;
-
-    if (flux_request_unpack (msg, NULL, "{s:i}", "matchtag", &matchtag) < 0) {
-        flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
-        return;
-    }
-    if (flux_msg_get_route_first (msg, &sender) < 0) {
-        flux_log_error (h, "%s: flux_msg_get_route_first", __FUNCTION__);
-        return;
-    }
-    watchers_cancel (ctx, sender, matchtag);
-    free (sender);
+    watchers_cancel (ctx, msg, true);
+    guest_watchers_cancel (ctx, msg, true);
 }
 
 void watch_cleanup (struct info_ctx *ctx)
@@ -438,13 +484,14 @@ void watch_cleanup (struct info_ctx *ctx)
 
     while ((w = zlist_pop (ctx->watchers))) {
         if (w->watch_f) {
-            if (flux_kvs_lookup_cancel (w->watch_f) < 0)
-                flux_log_error (ctx->h, "%s: flux_kvs_lookup_cancel",
+            if (flux_kvs_lookup_cancel (w->watch_f) < 0) {
+                flux_log_error (ctx->h,
+                                "%s: flux_kvs_lookup_cancel",
                                 __FUNCTION__);
+            }
         }
         if (flux_respond_error (ctx->h, w->msg, ENOSYS, NULL) < 0)
-            flux_log_error (ctx->h, "%s: flux_respond_error",
-                            __FUNCTION__);
+            flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
         watch_ctx_destroy (w);
     }
 }
diff --git a/src/modules/job-info/watch.h b/src/modules/job-info/watch.h
index 58db3ca62b27..fd5916f1de73 100644
--- a/src/modules/job-info/watch.h
+++ b/src/modules/job-info/watch.h
@@ -13,19 +13,27 @@
 
 #include <flux/core.h>
 
-void watch_cb (flux_t *h, flux_msg_handler_t *mh,
-               const flux_msg_t *msg, void *arg);
-
-void watch_cancel_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg);
-
-/* Cancel all lookups that match (sender, matchtag). */
+void watch_cb (flux_t *h,
+               flux_msg_handler_t *mh,
+               const flux_msg_t *msg,
+               void *arg);
+
+void watch_cancel_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg);
+
+/* Cancel all lookups that match msg.
+ * match credentials & matchtag if cancel true
+ * match credentials if cancel false
+ */
 void watchers_cancel (struct info_ctx *ctx,
-                      const char *sender, uint32_t matchtag);
+                      const flux_msg_t *msg,
+                      bool cancel);
 
 void watch_cleanup (struct info_ctx *ctx);
 
-#endif /* ! _FLUX_JOB_INFO_EVENTLOG_WATCH_H */
+#endif /* ! _FLUX_JOB_INFO_WATCH_H */
 
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/modules/job-ingest/Makefile.am b/src/modules/job-ingest/Makefile.am
index ac5a4e336b98..98dbf4810fc8 100644
--- a/src/modules/job-ingest/Makefile.am
+++ b/src/modules/job-ingest/Makefile.am
@@ -6,36 +6,60 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) $(FLUX_SECURITY_CFLAGS) $(YAMLCPP_CFLAGS) \
+	$(FLUX_SECURITY_CFLAGS) \
 	$(JANSSON_CFLAGS)
 
-fluxmod_LTLIBRARIES = job-ingest.la
+noinst_LTLIBRARIES = libingest.la
 
-job_ingest_la_SOURCES = \
+libingest_la_SOURCES = \
 	job-ingest.c \
-	validate.c \
-	validate.h \
+	workcrew.c \
+	workcrew.h \
 	worker.c \
 	worker.h \
-	types.h
-
-job_ingest_la_LDFLAGS = $(fluxmod_ldflags) -module
-job_ingest_la_LIBADD = $(fluxmod_libadd) \
-		    $(top_builddir)/src/common/libjob/libjob.la \
-		    $(top_builddir)/src/common/libflux-internal.la \
-		    $(top_builddir)/src/common/libflux-core.la \
-		    $(top_builddir)/src/common/libflux-optparse.la \
-		    $(FLUX_SECURITY_LIBS) \
-		    $(ZMQ_LIBS)
-
-dist_fluxlibexec_SCRIPTS = \
-	validators/validate-schema.py \
-	validators/validate-jobspec.py
-
-fluxschemadir = $(datadir)/flux/schema/jobspec/
-dist_fluxschema_DATA = \
-	schemas/jobspec.jsonschema \
-	schemas/jobspec_v1.jsonschema
+	types.h \
+	util.h \
+	util.c \
+	job.h \
+	job.c \
+	pipeline.h \
+	pipeline.c
+
+TESTS = \
+	test_util.t \
+	test_job.t
+
+test_ldadd = \
+	$(builddir)/libingest.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(JANSSON_LIBS)
+
+test_cppflags = \
+	$(AM_CPPFLAGS)
+
+test_ldflags = \
+	-no-install
+
+check_PROGRAMS = $(TESTS)
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+	$(top_srcdir)/config/tap-driver.sh
+
+test_util_t_SOURCES = test/util.c
+test_util_t_CPPFLAGS = $(test_cppflags)
+test_util_t_LDADD = $(test_ldadd)
+test_util_t_LDFLAGS = $(test_ldflags)
+
+test_job_t_SOURCES = test/job.c
+test_job_t_CPPFLAGS = $(test_cppflags)
+test_job_t_LDADD = $(test_ldadd)
+test_job_t_LDFLAGS = $(test_ldflags)
diff --git a/src/modules/job-ingest/job-ingest.c b/src/modules/job-ingest/job-ingest.c
index 2276cce7295e..990411e8b998 100644
--- a/src/modules/job-ingest/job-ingest.c
+++ b/src/modules/job-ingest/job-ingest.c
@@ -11,8 +11,9 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <time.h>
 #include <unistd.h>
-#include <czmq.h>
+#include <sys/types.h>
 #include <jansson.h>
 #include <flux/core.h>
 #if HAVE_FLUX_SECURITY
@@ -20,11 +21,18 @@
 #include <flux/security/sign.h>
 #endif
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/fluid.h"
-#include "src/common/libjob/sign_none.h"
-#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libutil/jpath.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/parse_size.h"
+#include "src/common/libjob/job_hash.h"
+#include "src/common/libfluxutil/policy.h"
+#include "ccan/str/str.h"
 
-#include "validate.h"
+#include "util.h"
+#include "job.h"
+#include "pipeline.h"
 
 /* job-ingest takes in signed jobspec submitted through flux_job_submit(),
  * performing the following tasks for each job:
@@ -48,7 +56,7 @@
  *   job.0000.0004.b200.0000
  *
  * The job-ingest module can be loaded on rank 0, or on many ranks across
- * the instance, rank < max FLUID id of 16384.  Each rank is relatively
+ * the instance, rank < max FLUID id - 1.  Each rank is relatively
  * independent and KVS commit scalability will ultimately limit the max
  * ingest rate for an instance.
  *
@@ -66,20 +74,28 @@
  * Too large, and individual job submit latency will suffer.
  * Too small, and KVS commit overhead will increase.
  */
-const double batch_timeout = 0.01;
+static const double batch_timeout = 0.01;
 
-/* Timeout (seconds) to wait for validators to terminate when
- * stopped by closing their stdin.  If the timer pops, stop the reactor
- * and allow validate_destroy() to signal them.
+/* There can be 2^14 FLUID generators per RFC 19.
+ * Reserve the top 16 for future use.
+ * This value may be set on the command line for testing.
  */
-const double shutdown_timeout = 5.;
+static int max_fluid_generator_id = 16384 - 16 - 1;
 
+/* By default, root (userid=0) jobs are rejected at submission
+ * unless the instance owner is also root. However, for testing
+ * purposes it may be useful to allow root jobs:
+ */
+static bool allow_root_jobs = false;
 
 struct job_ingest_ctx {
     flux_t *h;
-    struct validate *validate;
+    struct pipeline *pipeline;
+    uid_t owner;
 #if HAVE_FLUX_SECURITY
     flux_security_t *sec;
+#else
+    void *sec;
 #endif
     struct fluid_generator gen;
     flux_msg_handler_t **handlers;
@@ -87,24 +103,10 @@ struct job_ingest_ctx {
     struct batch *batch;
     flux_watcher_t *timer;
 
-    bool shutdown;              // no new jobs are accepted in shutdown mode
-    int shutdown_process_count; // number of validators executing at shutdown
-    flux_watcher_t *shutdown_timer;
-};
-
-struct job {
-    fluid_t id;         // jobid
-
-    const flux_msg_t *msg; // submit request message
-    const char *J;      // signed jobspec
-    struct flux_msg_cred cred;    // submitting user's creds
-    int priority;       // requested job priority
-    int flags;          // submit flags
+    int batch_count;            // if nonzero, batch by count not timer
+    const char *buffer_size;
 
-    char *jobspec;      // jobspec, not \0 terminated (unwrapped from signed)
-    int jobspecsz;      // jobspec string length
-
-    struct job_ingest_ctx *ctx;
+    bool shutdown;
 };
 
 struct batch {
@@ -114,51 +116,15 @@ struct batch {
     json_t *joblist;
 };
 
-static int make_key (char *buf, int bufsz, struct job *job, const char *name);
-
-/* Free decoded jobspec after it has been transferred to the batch txn,
- * to conserve memory.
- */
-static void job_clean (struct job *job)
-{
-    if (job) {
-        free (job->jobspec);
-        job->jobspec = NULL;
-    }
-}
-
-static void job_destroy (struct job *job)
-{
-    if (job) {
-        int saved_errno = errno;
-        free (job->jobspec);
-        flux_msg_decref (job->msg);
-        free (job);
-        errno = saved_errno;
-    }
-}
-
-static struct job *job_create (const flux_msg_t *msg,
-                               struct job_ingest_ctx *ctx)
-{
-    struct job *job;
+struct batch_response {
+    flux_future_t *f;
+    bool batch_failed;
+    int errnum;
+    const char *errmsg;
+    zhashx_t *errors;
+};
 
-    if (!(job = calloc (1, sizeof (*job))))
-        return NULL;
-    job->msg = flux_msg_incref (msg);
-    if (flux_request_unpack (job->msg, NULL, "{s:s s:i s:i}",
-                             "J", &job->J,
-                             "priority", &job->priority,
-                             "flags", &job->flags) < 0)
-        goto error;
-    if (flux_msg_get_cred (job->msg, &job->cred) < 0)
-        goto error;
-    job->ctx = ctx;
-    return job;
-error:
-    job_destroy (job);
-    return NULL;
-}
+static int make_key (char *buf, int bufsz, struct job *job, const char *name);
 
 static void batch_destroy (struct batch *batch)
 {
@@ -202,8 +168,87 @@ static struct batch *batch_create (struct job_ingest_ctx *ctx)
     return NULL;
 }
 
-/* Respond to all requestors (for each job) with errnum and errstr (required).
- */
+static void batch_response_destroy (struct batch_response *bresp)
+{
+    if (bresp) {
+        zhashx_destroy (&bresp->errors);
+        flux_future_destroy (bresp->f);
+        free (bresp);
+    }
+}
+
+static void *jobid_duplicator (const void *item)
+{
+    flux_jobid_t *id = calloc (1, sizeof (flux_jobid_t));
+    if (id)
+        *id = *((flux_jobid_t *)item);
+    return id;
+}
+
+static void jobid_destructor (void **item)
+{
+    if (item) {
+        free (*item);
+        *item = NULL;
+    }
+}
+
+static struct batch_response *batch_response_create (flux_future_t *f)
+{
+    struct batch_response *bresp = NULL;
+    json_t *o = NULL;
+    json_t *entry = NULL;
+    size_t index;
+
+    if (!(bresp = calloc (1, sizeof (*bresp)))
+        || !(bresp->errors = job_hash_create ()))
+        goto error;
+    zhashx_set_key_duplicator (bresp->errors, jobid_duplicator);
+    zhashx_set_key_destructor (bresp->errors, jobid_destructor);
+    bresp->f = f;
+    flux_future_incref (f);
+
+    /*  We differentiate future fulfilled with error (entire batch failed)
+     *   and failure to unpack payload (EPROTO). This is why "future_get"
+     *   is called twice below:
+     */
+    if (flux_rpc_get (f, NULL) < 0) {
+        bresp->errnum = errno;
+        bresp->errmsg = future_strerror (f, errno);
+        bresp->batch_failed = true;
+        return bresp;
+    }
+    if (flux_rpc_get_unpack (f, "{s?o}", "errors", &o) < 0) {
+        errno = EPROTO;
+        goto error;
+    }
+    /*  Empty payload indicates the whole batch was successful
+     */
+    if (o == NULL)
+        return bresp;
+
+    /*  O/w, there were zero or more failures sent in the errors array.
+     *   Capture these in the response errors hash.
+     */
+    json_array_foreach (o, index, entry) {
+        flux_jobid_t id;
+        char *errmsg;
+        if (json_unpack (entry, "[Is]", &id, &errmsg) < 0) {
+            errno = EPROTO;
+            goto error;
+        }
+        if (zhashx_insert (bresp->errors, &id, errmsg) < 0) {
+            /* jobid duplicated? Should not happen */
+            errno = EPROTO;
+            goto error;
+        }
+    }
+    return bresp;
+error:
+    batch_response_destroy (bresp);
+    return NULL;
+}
+
 static void batch_respond_error (struct batch *batch,
                                  int errnum, const char *errstr)
 {
@@ -216,14 +261,26 @@ static void batch_respond_error (struct batch *batch,
     }
 }
 
-/* Respond to all requestors (for each job) with their id.
+/* Respond to all requestors (for each job) with their id or an error if
+ *  job submit failed
  */
-static void batch_respond_success (struct batch *batch)
+static void batch_respond (struct batch *batch, struct batch_response *br)
 {
     flux_t *h = batch->ctx->h;
+    const char *errmsg;
     struct job *job = zlist_first (batch->jobs);
+
+    if (br->batch_failed) {
+        batch_respond_error (batch, br->errnum, br->errmsg);
+        return;
+    }
+
     while (job) {
-        if (flux_respond_pack (h, job->msg, "{s:I}", "id", job->id) < 0)
+        if ((errmsg = zhashx_lookup (br->errors, &job->id))) {
+            if (flux_respond_error (h, job->msg, EINVAL, errmsg) < 0)
+                flux_log_error (h, "batch_respond: flux_respond_error");
+        }
+        else if (flux_respond_pack (h, job->msg, "{s:I}", "id", job->id) < 0)
             flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
         job = zlist_next (batch->jobs);
     }
@@ -238,30 +295,38 @@ static void batch_cleanup_continuation (flux_future_t *f, void *arg)
     flux_future_destroy (f);
 }
 
-/* Remove KVS job entries previously committed for all jobs in batch.
+/* Remove KVS job entries previously committed for all failed jobs in batch.
  */
-static int batch_cleanup (struct batch *batch)
+static int batch_cleanup (struct batch *batch, struct batch_response *br)
 {
     flux_t *h = batch->ctx->h;
     flux_kvs_txn_t *txn;
     struct job *job;
     flux_future_t *f = NULL;
     char key[64];
+    int count = 0;
 
     if (!(txn = flux_kvs_txn_create ()))
         return -1;
     job = zlist_first (batch->jobs);
     while (job) {
-        if (make_key (key, sizeof (key), job, NULL) < 0)
+        if (br == NULL
+            || br->batch_failed
+            || zhashx_lookup (br->errors, &job->id)) {
+            if (make_key (key, sizeof (key), job, NULL) < 0)
+                goto error;
+            if (flux_kvs_txn_unlink (txn, 0, key) < 0)
+                goto error;
+            count++;
+        }
+        job = zlist_next (batch->jobs);
+    }
+    if (count > 0) {
+        if (!(f = flux_kvs_commit (h, NULL, 0, txn)))
             goto error;
-        if (flux_kvs_txn_unlink (txn, 0, key) < 0)
+        if (flux_future_then (f, -1., batch_cleanup_continuation, NULL) < 0)
             goto error;
-        job = zlist_next (batch->jobs);
     }
-    if (!(f = flux_kvs_commit (h, NULL, 0, txn)))
-        goto error;
-    if (flux_future_then (f, -1., batch_cleanup_continuation, NULL) < 0)
-        goto error;
     flux_kvs_txn_destroy (txn);
     return 0;
 error:
@@ -276,17 +341,23 @@ static int batch_cleanup (struct batch *batch)
 static void batch_announce_continuation (flux_future_t *f, void *arg)
 {
     struct batch *batch = arg;
+    struct batch_response *bresp;
     flux_t *h = batch->ctx->h;
 
-    if (flux_future_get (f, NULL) < 0) {
-        batch_respond_error (batch, errno, future_strerror (f, errno));
-        if (batch_cleanup (batch) < 0)
-            flux_log_error (h, "%s: KVS cleanup failure", __FUNCTION__);
-    }
+    if (!(bresp = batch_response_create (f)))
+        batch_respond_error (batch,
+                             errno,
+                             "Failed to process batch response");
     else
-        batch_respond_success (batch);
+        batch_respond (batch, bresp);
+
+    /*  Clean up any state in KVS for failed jobs
+     */
+    if (batch_cleanup (batch, bresp) < 0)
+        flux_log_error (h, "%s: KVS cleanup failure", __FUNCTION__);
 
     batch_destroy (batch);
+    batch_response_destroy (bresp);
     flux_future_destroy (f);
 }
 
@@ -307,7 +378,7 @@ static void batch_announce (struct batch *batch)
 error:
     flux_log_error (h, "%s: error sending RPC", __FUNCTION__);
     batch_respond_error (batch, errno, "error sending job-manager.submit RPC");
-    if (batch_cleanup (batch) < 0)
+    if (batch_cleanup (batch, NULL) < 0)
         flux_log_error (h, "%s: KVS cleanup failure", __FUNCTION__);
     batch_destroy (batch);
     flux_future_destroy (f);
@@ -330,15 +401,13 @@ static void batch_flush_continuation (flux_future_t *f, void *arg)
     flux_future_destroy (f);
 }
 
-/* batch timer - expires 'batch_timeout' seconds after batch was created.
+/*
  * Replace ctx->batch with a NULL, and pass 'batch' off to a chain of
  * continuations that commit its data to the KVS, respond to requestors,
  * and announce the new jobids.
  */
-static void batch_flush (flux_reactor_t *r, flux_watcher_t *w,
-                         int revents, void *arg)
+static void batch_flush (struct job_ingest_ctx *ctx)
 {
-    struct job_ingest_ctx *ctx = arg;
     struct batch *batch;
     flux_future_t *f;
 
@@ -352,7 +421,7 @@ static void batch_flush (flux_reactor_t *r, flux_watcher_t *w,
     if (flux_future_then (f, -1., batch_flush_continuation, batch) < 0) {
         batch_respond_error (batch, errno, "flux_future_then (kvs) failed");
         flux_future_destroy (f);
-        if (batch_cleanup (batch) < 0)
+        if (batch_cleanup (batch, NULL) < 0)
             flux_log_error (ctx->h, "%s: KVS cleanup failure", __FUNCTION__);
         goto error;
     }
@@ -361,6 +430,16 @@ static void batch_flush (flux_reactor_t *r, flux_watcher_t *w,
     batch_destroy (batch);
 }
 
+/* batch timer - expires 'batch_timeout' seconds after batch was created.
+ */
+static void batch_timer_cb (flux_reactor_t *r,
+                            flux_watcher_t *w,
+                            int revents,
+                            void *arg)
+{
+    batch_flush ((struct job_ingest_ctx *) arg);
+}
+
 /* Format key within the KVS directory of 'job'.
  */
 static int make_key (char *buf, int bufsz, struct job *job, const char *name)
@@ -372,6 +451,17 @@ static int make_key (char *buf, int bufsz, struct job *job, const char *name)
     return 0;
 }
 
+static double get_timestamp_now (void)
+{
+    struct timespec ts;
+    double now;
+
+    (void)clock_gettime (CLOCK_REALTIME, &ts);
+    now = ts.tv_sec;
+    now += (1E-9 * ts.tv_nsec);
+    return now;
+}
+
 /* Add 'job' to 'batch'.
  * On error, ensure that no remnants of job made into KVS transaction.
  */
@@ -380,9 +470,6 @@ static int batch_add_job (struct batch *batch, struct job *job)
     char key[64];
     int saved_errno;
     json_t *jobentry;
-    json_t *entry = NULL;
-    char *entrystr = NULL;
-    double t;
 
     if (zlist_append (batch->jobs, job) < 0) {
         errno = ENOMEM;
@@ -394,46 +481,30 @@ static int batch_add_job (struct batch *batch, struct job *job)
         goto error;
     if (make_key (key, sizeof (key), job, "jobspec") < 0)
         goto error;
-    if (flux_kvs_txn_put_raw (batch->txn, 0, key,
-                              job->jobspec, job->jobspecsz) < 0)
-        goto error;
-    entry = eventlog_entry_pack (0., "submit",
-                                 "{ s:i s:i s:i }",
-                                 "userid", job->cred.userid,
-                                 "priority", job->priority,
-                                 "flags", job->flags);
-    if (!entry)
-        goto error;
-    if (!(entrystr = eventlog_entry_encode (entry)))
-        goto error;
-    if (make_key (key, sizeof (key), job, "eventlog") < 0)
-        goto error;
-    if (flux_kvs_txn_put (batch->txn, FLUX_KVS_APPEND, key, entrystr) < 0)
-        goto error;
-    /* get created timestamp in eventlog entry for job */
-    if (eventlog_entry_parse (entry, &t, NULL, NULL) < 0)
+    /* Drop environment from the jobspec to reduce its bulk.
+     * If needed, it can be extracted from J.
+     * See also flux-framework/flux-core#4520
+     */
+    jpath_del (job->jobspec, "attributes.system.environment");
+    if (flux_kvs_txn_pack (batch->txn, 0, key, "O", job->jobspec) < 0)
         goto error;
-    if (!(jobentry = json_pack ("{s:I s:i s:i s:f s:i}",
+    if (!(jobentry = json_pack ("{s:I s:I s:i s:f s:i, s:O}",
                                 "id", job->id,
-                                "userid", job->cred.userid,
-                                "priority", job->priority,
-                                "t_submit", t,
-                                "flags", job->flags)))
+                                "userid", (json_int_t) job->cred.userid,
+                                "urgency", job->urgency,
+                                "t_submit", get_timestamp_now (),
+                                "flags", job->flags,
+                                "jobspec", job->jobspec)))
         goto nomem;
     if (json_array_append_new (batch->joblist, jobentry) < 0) {
         json_decref (jobentry);
         goto nomem;
     }
-    free (entrystr);
-    json_decref (entry);
-    job_clean (job); // batch->txn now holds this info
     return 0;
 nomem:
     errno = ENOMEM;
 error:
     saved_errno = errno;
-    free (entrystr);
-    json_decref (entry);
     zlist_remove (batch->jobs, job);
     if (make_key (key, sizeof (key), job, NULL) == 0)
         (void)flux_kvs_txn_unlink (batch->txn, 0, key);
@@ -441,10 +512,35 @@ static int batch_add_job (struct batch *batch, struct job *job)
     return -1;
 }
 
-void validate_continuation (flux_future_t *f, void *arg)
+static int ingest_add_job (struct job_ingest_ctx *ctx, struct job *job)
+{
+    if (fluid_generate (&ctx->gen, &job->id) < 0)
+        return -1;
+
+    /* Add job to the current "batch" of new jobs, creating the batch if
+     * one doesn't exist already.  Submit is finalized upon timer expiration.
+     */
+    if (!ctx->batch) {
+        if (!(ctx->batch = batch_create (ctx)))
+            return -1;
+        if (!ctx->batch_count) {
+            flux_timer_watcher_reset (ctx->timer, batch_timeout, 0.);
+            flux_watcher_start (ctx->timer);
+        }
+    }
+    if (batch_add_job (ctx->batch, job) < 0)
+        return -1;
+
+    if (ctx->batch_count
+        && zlist_size (ctx->batch->jobs) == ctx->batch_count)
+        batch_flush (ctx);
+    return 0;
+}
+
+void pipeline_continuation (flux_future_t *f, void *arg)
 {
     struct job *job = arg;
-    struct job_ingest_ctx *ctx = job->ctx;
+    struct job_ingest_ctx *ctx = flux_future_aux_get (f, "ctx");
     flux_t *h = flux_future_get_flux (f);
     const char *errmsg = NULL;
 
@@ -454,19 +550,10 @@ void validate_continuation (flux_future_t *f, void *arg)
         errmsg = future_strerror (f, errno);
         goto error;
     }
-    if (fluid_generate (&ctx->gen, &job->id) < 0)
-        goto error;
-    /* Add job to the current "batch" of new jobs, creating the batch if
-     * one doesn't exist already.  Submit is finalized upon timer expiration.
-     */
-    if (!ctx->batch) {
-        if (!(ctx->batch = batch_create (ctx)))
-            goto error;
-        flux_timer_watcher_reset (ctx->timer, batch_timeout, 0.);
-        flux_watcher_start (ctx->timer);
-    }
-    if (batch_add_job (ctx->batch, job) < 0)
+
+    if (ingest_add_job (ctx, job) < 0)
         goto error;
+
     flux_future_destroy (f);
     return;
 error:
@@ -476,16 +563,6 @@ void validate_continuation (flux_future_t *f, void *arg)
     flux_future_destroy (f);
 }
 
-static int valid_flags (int flags)
-{
-    int allowed = FLUX_JOB_DEBUG | FLUX_JOB_WAITABLE;
-    if ((flags & ~allowed)) {
-        errno = EPROTO;
-        return -1;
-    }
-    return 0;
-}
-
 /* Handle "job-ingest.submit" request to add a new job.
  */
 static void submit_cb (flux_t *h, flux_msg_handler_t *mh,
@@ -494,108 +571,38 @@ static void submit_cb (flux_t *h, flux_msg_handler_t *mh,
     struct job_ingest_ctx *ctx = arg;
     struct job *job = NULL;
     const char *errmsg = NULL;
-    char errbuf[256];
-    int64_t userid_signer;
-    const char *mech_type;
+    flux_error_t error;
     flux_future_t *f = NULL;
 
     if (ctx->shutdown) {
         errno = ENOSYS;
         goto error;
     }
-
-    /* Parse request.
-     */
-    if (!(job = job_create (msg, ctx)))
-        goto error;
-    /* Validate submit flags.
-     */
-    if (valid_flags (job->flags) < 0)
-        goto error;
-    /* Validate requested job priority.
-     */
-    if (job->priority < FLUX_JOB_PRIORITY_MIN
-            || job->priority > FLUX_JOB_PRIORITY_MAX) {
-        snprintf (errbuf, sizeof (errbuf), "priority range is [%d:%d]",
-                  FLUX_JOB_PRIORITY_MIN, FLUX_JOB_PRIORITY_MAX);
-        errmsg = errbuf;
-        errno = EINVAL;
-        goto error;
-    }
-    if (!(job->cred.rolemask & FLUX_ROLE_OWNER)
-           && job->priority > FLUX_JOB_PRIORITY_DEFAULT) {
-        snprintf (errbuf, sizeof (errbuf),
-                  "only the instance owner can submit with priority >%d",
-                  FLUX_JOB_PRIORITY_DEFAULT);
-        errmsg = errbuf;
-        errno = EINVAL;
+    if (!(job = job_create_from_request (msg, ctx->sec, &error))) {
+        errmsg = error.text;
         goto error;
     }
-    /* Only owner can set FLUX_JOB_WAITABLE.
+    /* Do not allow root user to submit jobs in a multi-user instance.
+     * The jobs will fail at runtime anyway.
      */
-    if (!(job->cred.rolemask & FLUX_ROLE_OWNER)
-            && (job->flags & FLUX_JOB_WAITABLE)) {
-        snprintf (errbuf,
-                  sizeof (errbuf),
-                  "only the instance onwer can submit with FLUX_JOB_WAITABLE");
-        errmsg = errbuf;
-        errno = EINVAL;
-        goto error;
-    }
-    /* Validate jobspec signature, and unwrap(J) -> jobspec,  jobspecsz.
-     * Userid claimed by signature must match authenticated job->cred.userid.
-     * If not the instance owner, a strong signature is required
-     * to give the IMP permission to launch processes on behalf of the user.
-     */
-#if HAVE_FLUX_SECURITY
-    const void *jobspec;
-    if (flux_sign_unwrap_anymech (ctx->sec, job->J, &jobspec, &job->jobspecsz,
-                                  &mech_type, &userid_signer,
-                                  FLUX_SIGN_NOVERIFY) < 0) {
-        errmsg = flux_security_last_error (ctx->sec);
+    if (ctx->owner != 0 && !allow_root_jobs && job->cred.userid == 0) {
+        errmsg = "submission of jobs as user root not supported";
         goto error;
     }
-    if (!(job->jobspec = malloc (job->jobspecsz)))
-        goto error;
-    memcpy (job->jobspec, jobspec, job->jobspecsz);
-#else
-    uint32_t userid_signer_u32;
-    /* Simplified unwrap only understands mech=none.
-     * Unlike flux-security version, returned payload must be freed,
-     * and returned userid is a uint32_t.
-     */
-    if (sign_none_unwrap (job->J, (void **)&job->jobspec, &job->jobspecsz,
-                          &userid_signer_u32) < 0) {
-        errmsg = "could not unwrap jobspec";
+    if (pipeline_process_job (ctx->pipeline, job, &f, &error) < 0) {
+        errmsg = error.text;
         goto error;
     }
-    mech_type = "none";
-    userid_signer = userid_signer_u32;
-#endif
-    if (userid_signer != job->cred.userid) {
-        snprintf (errbuf, sizeof (errbuf),
-                  "signer=%lu != requestor=%lu",
-                  (unsigned long)userid_signer,
-                  (unsigned long)job->cred.userid);
-        errmsg = errbuf;
-        errno = EPERM;
-        goto error;
+    if (f) {
+        if (flux_future_then (f, -1., pipeline_continuation, job) < 0
+            || flux_future_aux_set (f, "ctx", ctx, NULL) < 0) {
+            goto error;
+        }
     }
-    if (!(job->cred.rolemask & FLUX_ROLE_OWNER)
-                                && !strcmp (mech_type, "none")) {
-        snprintf (errbuf, sizeof (errbuf),
-                  "only instance owner can use sign-type=none");
-        errmsg = errbuf;
-        errno = EPERM;
-        goto error;
+    else {
+        if (ingest_add_job (ctx, job) < 0)
+            goto error;
     }
-    /* Validate jobspec asynchronously.
-     * Continue submission process in validate_continuation().
-     */
-    if (!(f = validate_jobspec (ctx->validate, job->jobspec, job->jobspecsz)))
-        goto error;
-    if (flux_future_then (f, -1., validate_continuation, job) < 0)
-        goto error;
     return;
 error:
     if (flux_respond_error (h, msg, errno, errmsg) < 0)
@@ -604,33 +611,9 @@ static void submit_cb (flux_t *h, flux_msg_handler_t *mh,
     flux_future_destroy (f);
 }
 
-static void exit_cb (void *arg)
-{
-    struct job_ingest_ctx *ctx = arg;
-
-    if (--ctx->shutdown_process_count == 0) {
-        flux_watcher_stop (ctx->shutdown_timer);
-        flux_reactor_stop (flux_get_reactor (ctx->h));
-    }
-}
-
-static void shutdown_timeout_cb (flux_reactor_t *r,
-                                 flux_watcher_t *w,
-                                 int revents,
-                                 void *arg)
-{
-    struct job_ingest_ctx *ctx = arg;
-
-    flux_log (ctx->h,
-              LOG_ERR,
-              "shutdown timed out with %d validators active",
-              ctx->shutdown_process_count);
-    flux_reactor_stop (flux_get_reactor (ctx->h));
-}
-
 /* Override built-in shutdown handler that calls flux_reactor_stop().
- * Since libsubprocess client is not able ot run outside of the reactor,
- * take care of cleaning up validator before exiting reactor.
+ * Since libsubprocess clients must run in reactive mode,
+ * take care of cleaning up the pipeline before exiting reactor.
  */
 static void shutdown_cb (flux_t *h,
                          flux_msg_handler_t *mh,
@@ -640,118 +623,320 @@ static void shutdown_cb (flux_t *h,
     struct job_ingest_ctx *ctx = arg;
 
     ctx->shutdown = true; // fail any new submit requests
-    ctx->shutdown_process_count = validate_stop_notify (ctx->validate,
-                                                        exit_cb,
-                                                        ctx);
-    if (ctx->shutdown_process_count == 0)
-        flux_reactor_stop (flux_get_reactor (h));
-    else {
-        flux_timer_watcher_reset (ctx->shutdown_timer, shutdown_timeout, 0.);
-        flux_watcher_start (ctx->shutdown_timer);
-    }
+
+    pipeline_shutdown (ctx->pipeline);
 }
 
-static const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_REQUEST,  "job-ingest.submit", submit_cb, FLUX_ROLE_USER },
-    { FLUX_MSGTYPE_REQUEST,  "job-ingest.shutdown", shutdown_cb, 0 },
-    FLUX_MSGHANDLER_TABLE_END,
-};
+static void getinfo_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
+{
+    struct job_ingest_ctx *ctx = arg;
+    uint64_t timestamp;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (fluid_save_timestamp (&ctx->gen, &timestamp) < 0) {
+        errno = EOVERFLOW; // punt: which is more likely, clock_gettime()
+                           //       failure or flux running for 35 years?
+        goto error;
+    }
+    if (flux_respond_pack (h, msg, "{s:I}", "timestamp", timestamp) < 0)
+        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
 
-/* Configure the validator.
- * Use compiled in path and string for validator program and args,
- * unless overridden with validator=path or schema=path on module load
- * command line.
+/*  Configure job ingest from flux_conf_t and/or  argc, argv.
+ *
+ *  Supported configuration:
+ *
+ *  [ingest]
+ *  batch-count = N
+ *  buffer-size = "40M"
+ *
+ *  [ingest.validator]
+ *  disable = false
+ *  plugins = [ "jobspec" ]
+ *  args = []
+ *
+ *  [ingest.validator.plugin-name]
+ *  # plugin-specific configuration
+ *
+ *  [ingest.frobnicator.plugin-name]
+ #  # plugin-specific configuration
+ *
  */
-int validate_initialize (flux_t *h,
-                         int argc,
-                         char **argv,
-                         struct validate **validate)
+static int job_ingest_configure (struct job_ingest_ctx *ctx,
+                                 const flux_conf_t *conf,
+                                 int argc,
+                                 char **argv,
+                                 flux_error_t *error)
 {
-    const char *usage_message = "Usage: flux module load [OPTIONS] job-ingest "
-                                " [validator-args=ARGS] [validator=PATH]";
-    const char *valpath;
-    const char *valargs;
-    struct validate *v;
-    int i;
-
-    valpath = flux_conf_builtin_get ("jobspec_validate_path", FLUX_CONF_AUTO);
-    valargs = flux_conf_builtin_get ("jobspec_validator_args", FLUX_CONF_AUTO);
-    for (i = 0; i < argc; i++) {
-        if (!strncmp (argv[i], "validator-args=", 15)) {
-            valargs = argv[i] + 15;
+    flux_error_t conf_error;
+    const char *buffer_size = NULL;
+    const char *max_fluid_id = NULL;
+
+    if (policy_validate (conf, error) < 0)
+        return -1;
+    if (flux_conf_unpack (conf,
+                          &conf_error,
+                          "{s?{s?i s?s}}",
+                          "ingest",
+                            "batch-count", &ctx->batch_count,
+                            "buffer-size", &buffer_size) < 0) {
+        errprintf (error,
+                  "error reading [ingest] config table: %s",
+                  conf_error.text);
+        return -1;
+    }
+    for (int i = 0; i < argc; i++) {
+        if (strstarts (argv[i], "validator-args=")
+            || strstarts (argv[i], "validator-plugins=")
+            || streq (argv[i], "disable-validator")) {
+            /* handled in pipeline.c */
         }
-        else if (!strncmp (argv[i], "validator=", 10)) {
-            valpath = argv[i] + 10;
-            if (access (valpath, X_OK) < 0) {
-                flux_log_error (h, "validator %s", valpath);
+        else if (strstarts (argv[i], "batch-count=")) {
+            char *endptr;
+            errno = 0;
+            ctx->batch_count = strtol (argv[i]+12, &endptr, 0);
+            if (errno != 0 || *endptr != '\0' || ctx->batch_count < 0) {
+                errprintf (error, "Invalid batch-count: %s", argv[i]);
+                errno = EINVAL;
                 return -1;
             }
         }
+        else if (strstarts (argv[i], "buffer-size=")) {
+            buffer_size = argv[i]+12;
+        }
+        else if (strstarts (argv[i], "max-fluid-generator-id=")) {
+            max_fluid_id = argv[i] + 23;
+        }
+        else if (streq (argv[i], "allow-root-jobs")) {
+            allow_root_jobs = true;
+        }
         else {
-            flux_log (h, LOG_ERR, "invalid option %s", argv[i]);
-            flux_log (h, LOG_ERR, "%s", usage_message);
+            errprintf (error, "Invalid option: %s", argv[i]);
             errno = EINVAL;
             return -1;
         }
     }
-    if (!(v = validate_create (h, valpath, valargs))) {
-        flux_log_error (h, "validate_create");
-        return -1;
+    if (buffer_size) {
+        uint64_t val;
+        if (parse_size (buffer_size, &val) < 0
+            || val > INT_MAX)
+            return errprintf (error, "Invalid buffer-size: '%s'", buffer_size);
+        ctx->buffer_size = buffer_size;
+        flux_log (ctx->h,
+                  LOG_DEBUG,
+                  "worker input buffer set to %s",
+                  ctx->buffer_size);
+    }
+    if (max_fluid_id) {
+        uint64_t val;
+        if (parse_size (max_fluid_id, &val) < 0 || val > 16384 - 1)
+            return errprintf (error,
+                              "Invalid max-fluid-generator-id: '%s'",
+                              max_fluid_id);
+        max_fluid_generator_id = val;
+    }
+    return pipeline_configure (ctx->pipeline,
+                               conf,
+                               argc,
+                               argv,
+                               ctx->buffer_size,
+                               error);
+}
+
+static void reload_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    struct job_ingest_ctx *ctx = arg;
+    const flux_conf_t *conf;
+    flux_error_t error;
+    const char *errstr = NULL;
+
+    if (flux_conf_reload_decode (msg, &conf) < 0) {
+        errstr = "Failed to parse config-reload request";
+        goto error;
     }
-    *validate = v;
-    return 0;
+    if (job_ingest_configure (ctx, conf, 0, NULL, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to config-reload request");
 }
 
-int mod_main (flux_t *h, int argc, char **argv)
+static void stats_get_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
+{
+    struct job_ingest_ctx *ctx = arg;
+    json_t *pstats = NULL;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    pstats = pipeline_stats_get (ctx->pipeline);
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:O}",
+                           "pipeline", pstats) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+    json_decref (pstats);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+}
+
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,  "job-ingest.getinfo", getinfo_cb, 0},
+    { FLUX_MSGTYPE_REQUEST,  "job-ingest.submit", submit_cb, FLUX_ROLE_USER },
+    { FLUX_MSGTYPE_REQUEST,  "job-ingest.shutdown", shutdown_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST,  "job-ingest.config-reload", reload_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST,  "job-ingest.stats-get",
+      stats_get_cb, FLUX_ROLE_USER },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+int job_ingest_ctx_init (struct job_ingest_ctx *ctx,
+                         flux_t *h,
+                         int argc,
+                         char **argv)
 {
     flux_reactor_t *r = flux_get_reactor (h);
-    int rc = -1;
-    struct job_ingest_ctx ctx;
-    uint32_t rank;
+    memset (ctx, 0, sizeof (*ctx));
+    ctx->h = h;
+    flux_error_t error;
+
+    ctx->owner = getuid ();
+
+    /*  Default worker input buffer size is 10MB */
+    ctx->buffer_size = "10M";
 
-    memset (&ctx, 0, sizeof (ctx));
-    ctx.h = h;
+    if (!(ctx->pipeline = pipeline_create (h))) {
+        flux_log_error (h, "error initializing job preprocessing pipeline");
+        return -1;
+    }
+    if (job_ingest_configure (ctx, flux_get_conf (h), argc, argv, &error) < 0) {
+        flux_log (h, LOG_ERR, "%s", error.text);
+        return -1;
+    }
 #if HAVE_FLUX_SECURITY
-    if (!(ctx.sec = flux_security_create (0))) {
+    if (!(ctx->sec = flux_security_create (0))) {
         flux_log_error (h, "flux_security_create");
-        goto done;
+        return -1;
     }
-    if (flux_security_configure (ctx.sec, NULL) < 0) {
+    if (flux_security_configure (ctx->sec, NULL) < 0) {
         flux_log_error (h, "flux_security_configure: %s",
-                        flux_security_last_error (ctx.sec));
-        goto done;
+                        flux_security_last_error (ctx->sec));
+        return -1;
     }
 #endif
-    if (flux_msg_handler_addvec (h, htab, &ctx, &ctx.handlers) < 0) {
+    if (flux_msg_handler_addvec (h, htab, ctx, &ctx->handlers) < 0) {
         flux_log_error (h, "flux_msghandler_add");
-        goto done;
+        return -1;
     }
-    if (!(ctx.timer = flux_timer_watcher_create (r, 0., 0.,
-                                                 batch_flush, &ctx))) {
+    if (!(ctx->timer = flux_timer_watcher_create (r, 0., 0.,
+                                                  batch_timer_cb,
+                                                  ctx))) {
         flux_log_error (h, "flux_timer_watcher_create");
-        goto done;
+        return -1;
     }
-    if (!(ctx.shutdown_timer = flux_timer_watcher_create (r,
-                                                          0.,
-                                                          0.,
-                                                          shutdown_timeout_cb,
-                                                          &ctx))) {
-        flux_log_error (h, "flux_timer_watcher_create");
+    return 0;
+}
+
+int mod_main (flux_t *h, int argc, char **argv)
+{
+    flux_reactor_t *r = flux_get_reactor (h);
+    int rc = -1;
+    struct job_ingest_ctx ctx;
+    uint32_t rank;
+
+    if (job_ingest_ctx_init (&ctx, h, argc, argv) < 0) {
+        flux_log (h, LOG_ERR, "Failed to initialize job-ingest ctx");
         goto done;
     }
+
     if (flux_get_rank (h, &rank) < 0) {
         flux_log_error (h, "flux_get_rank");
         goto done;
     }
-    /* fluid_init() will fail on rank > 16K.
-     * Just skip loading the job module on those ranks.
+    /* If the rank exceeds the maximum FLUID generator ID, then return success
+     * and let job ingest be handled upstream.
      */
-    if (fluid_init (&ctx.gen, rank) < 0) {
-        flux_log (h, LOG_ERR, "fluid_init failed");
-        errno = EINVAL;
-    }
-    if (validate_initialize (h, argc, argv, &ctx.validate) < 0)
+    if (rank > max_fluid_generator_id) {
+        flux_log (h,
+                  LOG_DEBUG,
+                  "job-ingest cannot allocate job IDs on ranks > %d."
+                  " Exiting - upstream will handle ingest requests.",
+                  max_fluid_generator_id);
+        rc = 0;
         goto done;
+    }
+    /* Initialize FLUID generator.
+     * On rank 0, derive the starting timestamp from the job manager's
+     * 'max_jobid' plus one.  On other ranks, ask upstream job-ingest.
+     */
+    if (rank == 0) {
+        flux_future_t *f;
+        flux_jobid_t max_jobid;
+
+        if (!(f = flux_rpc (h, "job-manager.getinfo", NULL, 0, 0))) {
+            flux_log_error (h, "flux_rpc");
+            goto done;
+        }
+        if (flux_rpc_get_unpack (f, "{s:I}", "max_jobid", &max_jobid) < 0) {
+            if (errno == ENOSYS)
+                flux_log_error (h, "job-manager must be loaded first");
+            else
+                flux_log_error (h, "job-manager.getinfo");
+            flux_future_destroy (f);
+            goto done;
+        }
+        flux_future_destroy (f);
+        if (fluid_init (&ctx.gen, 0, fluid_get_timestamp (max_jobid) + 1) < 0) {
+            flux_log (h, LOG_ERR, "fluid_init failed");
+            errno = EINVAL;
+            goto done;
+        }
+    }
+    else {
+        flux_future_t *f;
+        uint64_t timestamp;
+
+        if (!(f = flux_rpc (h, "job-ingest.getinfo", NULL, 0, 0))) {
+            flux_log_error (h, "flux_rpc");
+            goto done;
+        }
+        if (flux_rpc_get_unpack (f, "{s:I}", "timestamp", &timestamp) < 0) {
+            if (errno == ENOSYS)
+                flux_log_error (h, "job-ingest must be loaded on rank 0 first");
+            else
+                flux_log_error (h, "job-ingest.getinfo");
+            flux_future_destroy (f);
+            goto done;
+        }
+        flux_future_destroy (f);
+        if (fluid_init (&ctx.gen, rank, timestamp) < 0) {
+            flux_log (h, LOG_ERR, "fluid_init failed");
+            errno = EINVAL;
+            goto done;
+        }
+    }
+    flux_log (h, LOG_DEBUG, "fluid ts=%jums", (uintmax_t)ctx.gen.timestamp);
     if (flux_reactor_run (r, 0) < 0) {
         flux_log_error (h, "flux_reactor_run");
         goto done;
@@ -760,16 +945,13 @@ int mod_main (flux_t *h, int argc, char **argv)
 done:
     flux_msg_handler_delvec (ctx.handlers);
     flux_watcher_destroy (ctx.timer);
-    flux_watcher_destroy (ctx.shutdown_timer);
 #if HAVE_FLUX_SECURITY
     flux_security_destroy (ctx.sec);
 #endif
-    validate_destroy (ctx.validate);
+    pipeline_destroy (ctx.pipeline);
     return rc;
 }
 
-MOD_NAME ("job-ingest");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/job-ingest/job.c b/src/modules/job-ingest/job.c
new file mode 100644
index 000000000000..e0b3d2d31a68
--- /dev/null
+++ b/src/modules/job-ingest/job.c
@@ -0,0 +1,194 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <time.h>
+#include <unistd.h>
+#include <jansson.h>
+#include <flux/core.h>
+#if HAVE_FLUX_SECURITY
+#include <flux/security/context.h>
+#include <flux/security/sign.h>
+#endif
+
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libjob/sign_none.h"
+#include "ccan/str/str.h"
+
+#include "job.h"
+
+void job_destroy (struct job *job)
+{
+    if (job) {
+        int saved_errno = errno;
+        flux_msg_decref (job->msg);
+        json_decref (job->jobspec);
+        free (job);
+        errno = saved_errno;
+    }
+}
+
+static int valid_flags (int flags)
+{
+    int allowed = FLUX_JOB_DEBUG | FLUX_JOB_WAITABLE | FLUX_JOB_NOVALIDATE;
+    if ((flags & ~allowed)) {
+        errno = EPROTO;
+        return -1;
+    }
+    return 0;
+}
+
+struct job *job_create_from_request (const flux_msg_t *msg,
+                                     void *security_context,
+                                     flux_error_t *error)
+{
+    struct job *job;
+    int64_t userid_signer;
+    const char *mech_type;
+    json_error_t json_error;
+    const char *jobspec_str;
+    int jobspec_strsize;
+    char *jobspec_buf = NULL;
+
+    if (!(job = calloc (1, sizeof (*job)))) {
+        errprintf (error, "out of memory decoding job request");
+        return NULL;
+    }
+    job->msg = flux_msg_incref (msg);
+    if (flux_request_unpack (job->msg,
+                             NULL,
+                             "{s:s s:i s:i}",
+                             "J", &job->J,
+                             "urgency", &job->urgency,
+                             "flags", &job->flags) < 0
+        || flux_msg_get_cred (job->msg, &job->cred) < 0) {
+        errprintf (error, "error decoding job request: %s", strerror (errno));
+        goto error;
+    }
+    if (valid_flags (job->flags) < 0) {
+        errprintf (error, "invalid job flags");
+        goto error;
+    }
+    if (!(job->cred.rolemask & FLUX_ROLE_OWNER)
+        && (job->flags & FLUX_JOB_NOVALIDATE)) {
+        errprintf (error,
+                   "only the instance owner can submit "
+                   "with FLUX_JOB_NOVALIDATE");
+        errno = EPERM;
+        goto error;
+    }
+    if (job->urgency < FLUX_JOB_URGENCY_MIN
+        || job->urgency > FLUX_JOB_URGENCY_MAX) {
+        errprintf (error, "urgency range is [%d:%d]",
+                   FLUX_JOB_URGENCY_MIN, FLUX_JOB_URGENCY_MAX);
+        goto inval;
+    }
+    if (!(job->cred.rolemask & FLUX_ROLE_OWNER)
+        && job->urgency > FLUX_JOB_URGENCY_DEFAULT) {
+        errprintf (error,
+                   "only the instance owner can submit with urgency >%d",
+                   FLUX_JOB_URGENCY_DEFAULT);
+        goto inval;
+    }
+    if (!(job->cred.rolemask & FLUX_ROLE_OWNER)
+        && (job->flags & FLUX_JOB_WAITABLE)) {
+        errprintf (error,
+                   "only the instance owner can submit with FLUX_JOB_WAITABLE");
+        goto inval;
+    }
+    /* Validate jobspec signature, and unwrap(J) -> jobspec_str, _strsize.
+     * Userid claimed by signature must match authenticated job->cred.userid.
+     * If not the instance owner, a strong signature is required
+     * to give the IMP permission to launch processes on behalf of the user.
+     */
+#if HAVE_FLUX_SECURITY
+    if (flux_sign_unwrap_anymech (security_context,
+                                  job->J,
+                                  (const void **)&jobspec_str,
+                                  &jobspec_strsize,
+                                  &mech_type,
+                                  &userid_signer,
+                                  FLUX_SIGN_NOVERIFY) < 0) {
+        errprintf (error, "%s", flux_security_last_error (security_context));
+        goto error;
+    }
+#else
+    uint32_t userid_signer_u32;
+    /* Simplified unwrap only understands mech=none.
+     * Unlike flux-security version, returned payload must be freed,
+     * and returned userid is a uint32_t.
+     */
+    if (sign_none_unwrap (job->J,
+                          (void **)&jobspec_buf,
+                          &jobspec_strsize,
+                          &userid_signer_u32) < 0) {
+        errprintf (error, "could not unwrap jobspec: %s", strerror (errno));
+        goto error;
+    }
+    jobspec_str = jobspec_buf;
+    mech_type = "none";
+    userid_signer = userid_signer_u32;
+#endif
+    if (userid_signer != job->cred.userid) {
+        errprintf (error,
+                  "signer=%lu != requestor=%lu",
+                  (unsigned long)userid_signer,
+                  (unsigned long)job->cred.userid);
+        errno = EPERM;
+        goto error;
+    }
+    if (!(job->cred.rolemask & FLUX_ROLE_OWNER)
+        && streq (mech_type, "none")) {
+        errprintf (error, "only instance owner can use sign-type=none");
+        errno = EPERM;
+        goto error;
+    }
+    if (!(job->jobspec = json_loadb (jobspec_str,
+                                     jobspec_strsize,
+                                     0,
+                                     &json_error))) {
+        errprintf (error, "jobspec: invalid JSON: %s", json_error.text);
+        goto inval;
+    }
+    free (jobspec_buf);
+    return job;
+inval:
+    errno = EINVAL;
+error:
+    ERRNO_SAFE_WRAP (free, jobspec_buf);
+    job_destroy (job);
+    return NULL;
+}
+
+json_t *job_json_object (struct job *job, flux_error_t *error)
+{
+    json_error_t json_error;
+    json_t *o;
+
+    if (!(o = json_pack_ex (&json_error,
+                            0,
+                            "{s:O s:I s:i s:i s:i}",
+                            "jobspec", job->jobspec,
+                            "userid", (json_int_t) job->cred.userid,
+                            "rolemask", job->cred.rolemask,
+                            "urgency", job->urgency,
+                            "flags", job->flags))) {
+        errprintf (error,
+                   "Error creating JSON job object: %s", json_error.text);
+        errno = EINVAL;
+        return NULL;
+    }
+    return o;
+}
+
+// vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/modules/job-ingest/job.h b/src/modules/job-ingest/job.h
new file mode 100644
index 000000000000..ae5ee4871814
--- /dev/null
+++ b/src/modules/job-ingest/job.h
@@ -0,0 +1,39 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _JOB_INGEST_JOB_H_
+#define _JOB_INGEST_JOB_H_
+
+#include <jansson.h>
+#include <flux/core.h>
+
+struct job {
+    flux_jobid_t id;
+
+    const flux_msg_t *msg; // submit request message
+    const char *J;      // signed jobspec
+    struct flux_msg_cred cred;    // submitting user's creds
+    int urgency;        // requested job urgency
+    int flags;          // submit flags
+    json_t *jobspec;    // jobspec modified after unwrap from J
+};
+
+
+void job_destroy (struct job *job);
+
+struct job *job_create_from_request (const flux_msg_t *msg,
+                                     void *security_context,
+                                     flux_error_t *error);
+
+json_t *job_json_object (struct job *job, flux_error_t *error);
+
+#endif /* !_JOB_INGEST_JOB_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-ingest/pipeline.c b/src/modules/job-ingest/pipeline.c
new file mode 100644
index 000000000000..56c3c0b836af
--- /dev/null
+++ b/src/modules/job-ingest/pipeline.c
@@ -0,0 +1,447 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* pipeline.c - run jobspec through ingest pipeline: frobnicator | validator */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
+
+#include "util.h"
+#include "workcrew.h"
+#include "pipeline.h"
+
+struct pipeline {
+    flux_t *h;
+    struct workcrew *validate;
+    struct workcrew *frobnicate;
+    int process_count;
+    flux_watcher_t *shutdown_timer;
+    bool validator_bypass;
+    bool frobnicate_enable;
+};
+
+static const char *cmd_validator = "job-validator";
+static const char *cmd_frobnicator = "job-frobnicator";
+
+
+/* Timeout (seconds) to wait for workers to terminate when
+ * stopped by closing their stdin.  If the timer expires, stop the reactor
+ * and allow workcrew_destroy() to signal them.
+ */
+static const double shutdown_timeout = 5.;
+
+static void exit_cb (void *arg)
+{
+    struct pipeline *pl = arg;
+
+    if (--pl->process_count == 0) {
+        flux_watcher_stop (pl->shutdown_timer);
+        flux_reactor_stop (flux_get_reactor (pl->h));
+    }
+}
+
+static void shutdown_timeout_cb (flux_reactor_t *r,
+                                 flux_watcher_t *w,
+                                 int revents,
+                                 void *arg)
+{
+    struct pipeline *pl = arg;
+
+    flux_log (pl->h,
+              LOG_ERR,
+              "shutdown timed out with %d workers active",
+              pl->process_count);
+    flux_reactor_stop (r);
+}
+
+void pipeline_shutdown (struct pipeline *pl)
+{
+    pl->process_count = workcrew_stop_notify (pl->validate, exit_cb, pl);
+    pl->process_count += workcrew_stop_notify (pl->frobnicate, exit_cb, pl);
+    if (pl->process_count == 0)
+        flux_reactor_stop (flux_get_reactor (pl->h));
+    else {
+        flux_timer_watcher_reset (pl->shutdown_timer, shutdown_timeout, 0.);
+        flux_watcher_start (pl->shutdown_timer);
+    }
+
+}
+
+static bool validator_bypass (struct pipeline *pl, struct job *job)
+{
+    if ((pl->validator_bypass || (job->flags & FLUX_JOB_NOVALIDATE)))
+        return true;
+    return false;
+}
+
+static flux_future_t *validate_job (struct pipeline *pl,
+                                    struct job *job,
+                                    flux_error_t *error)
+{
+    json_t *input;
+    flux_future_t *f;
+
+    if (!(input = job_json_object (job, error)))
+        return NULL;
+    if (!(f = workcrew_process_job (pl->validate, input))) {
+        errprintf (error, "Error passing job to validator");
+        goto error;
+    }
+    json_decref (input);
+    return f;
+error:
+    ERRNO_SAFE_WRAP (json_decref, input);
+    return NULL;
+}
+
+static flux_future_t *frobnicate_job (struct pipeline *pl,
+                                      struct job *job,
+                                      flux_error_t *error)
+{
+    json_t *input;
+    flux_future_t *f;
+
+    if (!(input = job_json_object (job, error)))
+        return NULL;
+    if (!(f = workcrew_process_job (pl->frobnicate, input))) {
+        errprintf (error, "Error passing job to frobnicator");
+        goto error;
+    }
+    json_decref (input);
+    return f;
+error:
+    ERRNO_SAFE_WRAP (json_decref, input);
+    return NULL;
+}
+
+static void frobnicate_continuation (flux_future_t *f1, void *arg)
+{
+    struct pipeline *pl = arg;
+    struct job *job = flux_future_aux_get (f1, "job");
+    const char *s;
+    json_t *jobspec;
+    const char *errmsg = NULL;
+    flux_error_t error;
+
+    if (flux_future_get (f1, (const void **)&s) < 0) {
+        errmsg = future_strerror (f1, errno);
+        goto error;
+    }
+
+    if (!(jobspec = json_loads (s, 0, NULL))) {
+        errmsg = "error decoding jobspec from frobnicator";
+        errno = EINVAL;
+        goto error;
+    }
+    json_decref (job->jobspec);
+    job->jobspec = jobspec;
+
+    if (!validator_bypass (pl, job)) {
+        flux_future_t *f2;
+
+        if (!(f2 = validate_job (pl, job, &error))) {
+            errmsg = error.text;
+            goto error;
+        }
+        if (flux_future_continue (f1, f2) < 0) {
+            flux_future_destroy (f2);
+            errmsg = "error continuing validator";
+            goto error;
+        }
+    }
+    goto done;
+error:
+    flux_future_continue_error (f1, errno, errmsg);
+done:
+    flux_future_destroy (f1);
+}
+
+/* N.B. this function could be a little simpler if futures for the pipeline
+ * stages were unconditionally chained;  instead, minimize overhead for:
+ * - frobnicator not configured
+ * - frobnicator not configured AND validator bypassed
+ */
+int pipeline_process_job (struct pipeline *pl,
+                          struct job *job,
+                          flux_future_t **fp,
+                          flux_error_t *error)
+{
+    if (pl->frobnicate_enable) {
+        flux_future_t *f1;
+        flux_future_t *f_comp;
+
+        if (!(f1 = frobnicate_job (pl, job, error))
+            || flux_future_aux_set (f1, "job", job, NULL) < 0
+            || !(f_comp = flux_future_and_then (f1,
+                                                frobnicate_continuation,
+                                                pl))) {
+            flux_future_destroy (f1);
+            return -1;
+        }
+        *fp = f_comp;
+    }
+    else {
+        flux_future_t *f;
+
+        if (validator_bypass (pl, job))
+            *fp = NULL;
+        else {
+            if (!(f = validate_job (pl, job, error)))
+                return -1;
+            *fp = f;
+        }
+    }
+    return 0;
+}
+
+static int unpack_ingest_subtable (json_t *o,
+                                   const char *name,
+                                   char **pluginsp,
+                                   char **argsp,
+                                   bool *disablep,
+                                   flux_error_t *error)
+{
+    json_error_t json_error;
+    json_t *op = NULL;
+    json_t *oa = NULL;
+    char *plugins = NULL;
+    char *args = NULL;
+    int disable = 0;
+
+    if (o) {
+        if (json_unpack_ex (o,
+                            &json_error,
+                            0,
+                            "{s?{s?o s?o s?b}}",
+                            name,
+                              "args", &oa,
+                              "plugins", &op,
+                              "disable", &disable) < 0) {
+            errprintf (error,
+                       "error parsing [ingest.%s] config table: %s",
+                       name,
+                       json_error.text);
+            errno = EINVAL;
+            return -1;
+        }
+        if (!disablep && disable) {
+            errprintf (error, "[ingest.%s]: 'disable' key is unknown", name);
+            errno = EINVAL;
+            return -1;
+        }
+        if (op) {
+            if (!(plugins = util_join_arguments (op))) {
+                errprintf (error, "error in [ingest.%s] plugins array", name);
+                goto error;
+            }
+        }
+        if (oa) {
+            if (!(args = util_join_arguments (oa))) {
+                errprintf (error, "error in [ingest.%s] args array", name);
+                goto error;
+            }
+        }
+    }
+    *pluginsp = plugins;
+    *argsp = args;
+    *disablep = disable ? true : false;
+    return 0;
+error:
+    ERRNO_SAFE_WRAP (free, args);
+    ERRNO_SAFE_WRAP (free, plugins);
+    return -1;
+}
+
+int pipeline_configure (struct pipeline *pl,
+                        const flux_conf_t *conf,
+                        int argc,
+                        char **argv,
+                        const char *bufsize,
+                        flux_error_t *error)
+{
+    flux_error_t conf_error;
+    json_t *ingest = NULL;
+    char *validator_plugins = NULL;
+    char *validator_args = NULL;
+    char *frobnicator_plugins = NULL;
+    char *frobnicator_args = NULL;
+    bool frobnicator_bypass = false;
+    int rc = -1;
+
+    /* Process toml
+     */
+    if (flux_conf_unpack (conf,
+                          &conf_error,
+                          "{s?o}",
+                          "ingest", &ingest) < 0) {
+        errprintf (error,
+                   "error parsing [ingest] config table: %s",
+                   conf_error.text);
+        return -1;
+    }
+    if (unpack_ingest_subtable (ingest,
+                                "validator",
+                                &validator_plugins,
+                                &validator_args,
+                                &pl->validator_bypass,
+                                error) < 0)
+        return -1;
+    if (unpack_ingest_subtable (ingest,
+                                "frobnicator",
+                                &frobnicator_plugins,
+                                &frobnicator_args,
+                                &frobnicator_bypass,
+                                error) < 0)
+        return -1;
+
+    /* Process module command line
+     */
+    for (int i = 0; i < argc; i++) {
+        if (strstarts (argv[i], "validator-args=")) {
+            free (validator_args);
+            validator_args = strdup (argv[i] + 15);
+        }
+        else if (strstarts (argv[i], "validator-plugins=")) {
+            free (validator_plugins);
+            validator_plugins = strdup (argv[i] + 18);
+        }
+        else if (streq (argv[i], "disable-validator"))
+            pl->validator_bypass = true;
+    }
+
+    /* Enable the frobnicator if not bypassed AND either explicitly configured
+     * or implicitly required by queues or jobspec defaults configuration.
+     * See flux-framework/flux-core#4598.
+     */
+    pl->frobnicate_enable = false;
+    if (!frobnicator_bypass) {
+        if (frobnicator_plugins && strlen (frobnicator_plugins) > 0)
+            pl->frobnicate_enable = true;
+        else {
+            json_t *defaults = NULL;
+            json_t *queues = NULL;
+            (void)flux_conf_unpack (conf,
+                                    NULL,
+                                    "{s?{s?{s?o}} s?o}",
+                                    "policy",
+                                      "jobspec",
+                                        "defaults", &defaults,
+                                    "queues", &queues);
+            if (defaults || queues)
+                pl->frobnicate_enable = true;
+        }
+    }
+
+    if (workcrew_configure (pl->frobnicate,
+                            cmd_frobnicator,
+                            frobnicator_plugins,
+                            frobnicator_args,
+                            bufsize) < 0) {
+        errprintf (error,
+                   "Error (re-)configuring frobnicator workcrew: %s",
+                   strerror (errno));
+        goto error;
+    }
+
+    // Checked for by t2111-job-ingest-config.t
+    flux_log (pl->h,
+              LOG_DEBUG,
+              "configuring validator with plugins=%s, args=%s (%s)",
+              validator_plugins,
+              validator_args,
+              pl->validator_bypass ? "disabled" : "enabled");
+    if (workcrew_configure (pl->validate,
+                            cmd_validator,
+                            validator_plugins,
+                            validator_args,
+                            bufsize) < 0) {
+        errprintf (error,
+                   "Error (re-)configuring validator workcrew: %s",
+                   strerror (errno));
+        goto error;
+    }
+    rc = 0;
+error:
+    ERRNO_SAFE_WRAP (free, validator_plugins);
+    ERRNO_SAFE_WRAP (free, validator_args);
+    ERRNO_SAFE_WRAP (free, frobnicator_plugins);
+    ERRNO_SAFE_WRAP (free, frobnicator_args);
+    return rc;
+}
+
+json_t *pipeline_stats_get (struct pipeline *pl)
+{
+    json_t *o = NULL;
+    if (pl) {
+        json_t *fo = workcrew_stats_get (pl->frobnicate);
+        json_t *vo = workcrew_stats_get (pl->validate);
+        o = json_pack ("{s:O s:O}",
+                       "frobnicator", fo,
+                       "validator", vo);
+        json_decref (fo);
+        json_decref (vo);
+    }
+    return o ? o : json_null ();
+}
+
+void pipeline_destroy (struct pipeline *pl)
+{
+    if (pl) {
+        int saved_errno = errno;
+        workcrew_destroy (pl->validate);
+        workcrew_destroy (pl->frobnicate);
+        flux_watcher_destroy (pl->shutdown_timer);
+        free (pl);
+        errno = saved_errno;
+    }
+}
+
+struct pipeline *pipeline_create (flux_t *h)
+{
+    struct pipeline *pl;
+    flux_reactor_t *r = flux_get_reactor (h);
+
+    if (!(pl = calloc (1, sizeof (*pl))))
+        return NULL;
+    pl->h = h;
+    if (!(pl->shutdown_timer = flux_timer_watcher_create (r,
+                                                          0.,
+                                                          0.,
+                                                          shutdown_timeout_cb,
+                                                          pl)))
+        goto error;
+    if (!(pl->validate = workcrew_create (pl->h))
+        || workcrew_configure (pl->validate,
+                               cmd_validator,
+                               NULL,
+                               NULL,
+                               NULL) < 0)
+        goto error;
+    if (!(pl->frobnicate = workcrew_create (pl->h))
+        || workcrew_configure (pl->frobnicate,
+                               cmd_frobnicator,
+                               NULL,
+                               NULL,
+                               NULL) < 0)
+        goto error;
+    return pl;
+error:
+    pipeline_destroy (pl);
+    return NULL;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-ingest/pipeline.h b/src/modules/job-ingest/pipeline.h
new file mode 100644
index 000000000000..aca1e0233be2
--- /dev/null
+++ b/src/modules/job-ingest/pipeline.h
@@ -0,0 +1,40 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _JOB_INGEST_PIPELINE_H
+#define _JOB_INGEST_PIPELINE_H
+
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "job.h"
+
+struct pipeline *pipeline_create (flux_t *h);
+void pipeline_destroy (struct pipeline *pl);
+
+void pipeline_shutdown (struct pipeline *pl);
+
+int pipeline_configure (struct pipeline *pl,
+                        const flux_conf_t *conf,
+                        int argc,
+                        char **argv,
+                        const char *bufsize,
+                        flux_error_t *error);
+
+int pipeline_process_job (struct pipeline *pl,
+                          struct job *job,
+                          flux_future_t **fp,
+                          flux_error_t *error);
+
+json_t *pipeline_stats_get (struct pipeline *pl);
+
+#endif /* !_JOB_INGEST_PIPELINE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-ingest/schemas/jobspec.jsonschema b/src/modules/job-ingest/schemas/jobspec.jsonschema
deleted file mode 100644
index f3796cb28f05..000000000000
--- a/src/modules/job-ingest/schemas/jobspec.jsonschema
+++ /dev/null
@@ -1,144 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "$id": "http://github.com/flux-framework/rfc/tree/master/data/spec_14/schema.json",
-  "title": "canonical-jobspec",
-
-  "description":         "Flux canonical jobspec",
-
-  "definitions": {
-    "complex_range": {
-      "description": "a complex range of numbers",
-      "type": "object",
-      "properties":{
-        "min": { "type": "integer", "minimum" : 1 },
-        "max": { "type": "integer", "minimum" : 1 },
-        "operator": { "type": "string", "enum": ["+", "*", "^"] },
-        "operand": { "type": "integer", "minimum" : 1 }
-      },
-      "required": ["min"],
-      "dependencies": {
-         "max":      { "required": ["operator", "operand"] },
-         "operator": { "required": ["max", "operand"] },
-         "operand":  { "required": ["max", "operator"] }
-      },
-      "additionalProperties": false
-    },
-    "resource_vertex_base": {
-      "description": "base schema for slot/other resource vertex",
-      "type": "object",
-      "required": ["type", "count"],
-      "properties": {
-        "type": { "type": "string" },
-        "count": {
-          "oneOf": [
-            { "type": "integer", "minimum" : 1 },
-            { "$ref": "#/definitions/complex_range" }
-          ]
-        },
-        "exclusive": { "type": "boolean" },
-        "with": {
-          "type": "array",
-          "items": { "$ref": "#/definitions/resource_vertex" }
-        },
-        "id": { "type": "string" },
-        "unit": { "type": "string" },
-        "label": { "type": "string" }
-      },
-      "additionalProperties": false
-    },
-    "resource_vertex_slot": {
-      "description": "special slot resource type - label assigns to task slot",
-      "allOf": [
-        { "$ref": "#/definitions/resource_vertex_base" },
-        {
-          "properties": {
-            "type": { "enum": ["slot"] }
-          },
-          "required": ["label"]
-        }
-      ]
-    },
-    "resource_vertex_other": {
-      "description": "other (non-slot) resource type",
-      "allOf": [
-        { "$ref": "#/definitions/resource_vertex_base" },
-        {
-          "properties": {
-            "type": { "not": { "enum": ["slot"] } }
-          }
-        }
-      ]
-    },
-    "resource_vertex": {
-      "oneOf":[
-        { "$ref": "#/definitions/resource_vertex_slot" },
-        { "$ref": "#/definitions/resource_vertex_other" }
-      ]
-    }
-  },
-
-  "type": "object",
-  "required": ["version", "resources", "attributes", "tasks"],
-  "properties": {
-    "version": {
-      "description": "the jobspec version",
-      "type": "integer"
-    },
-    "resources": {
-      "description": "requested resources",
-      "type": "array",
-      "minItems": 1,
-      "items": { "$ref": "#/definitions/resource_vertex" }
-    },
-    "attributes": {
-      "description": "system and user attributes",
-      "type": ["object", "null"],
-      "properties": {
-        "system": {
-          "type": "object",
-          "properties": {
-            "duration": { "type": "number", "minimum": 0 },
-            "cwd": { "type": "string" },
-            "environment": { "type": "object" }
-          }
-        },
-        "user": {
-          "type": "object"
-        }
-      },
-      "additionalProperties": false
-    },
-    "tasks": {
-      "description": "task configuration",
-      "type": "array",
-      "items": {
-        "type": "object",
-        "required": ["command", "slot", "count" ],
-        "properties": {
-          "command": {
-            "type": "array",
-            "minItems": 1,
-            "items": { "type": "string" }
-          },
-          "slot": { "type": "string" },
-          "count": {
-            "type": "object",
-            "properties": {
-              "per_slot": { "type": "integer", "minimum" : 1 },
-              "total": { "type": "integer", "minimum" : 1 }
-            }
-          },
-          "distribution": { "type": "string" },
-          "attributes": {
-            "type": "object",
-            "properties": {
-              "environment": { "type" : "object"}
-            },
-	        "additionalProperties": { "type": "string" }
-          }
-        },
-	"additionalProperties": false
-      }
-    }
-  }
-}
diff --git a/src/modules/job-ingest/schemas/jobspec_v1.jsonschema b/src/modules/job-ingest/schemas/jobspec_v1.jsonschema
deleted file mode 100644
index 4ec181438b4d..000000000000
--- a/src/modules/job-ingest/schemas/jobspec_v1.jsonschema
+++ /dev/null
@@ -1,130 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "$id": "http://github.com/flux-framework/rfc/tree/master/data/spec_24/schema.json",
-  "title": "jobspec-01",
-
-  "description":         "Flux jobspec version 1",
-
-  "definitions": {
-    "intranode_resource_vertex": {
-      "description": "schema for resource vertices within a node, cannot have child vertices",
-      "type": "object",
-      "required": ["type", "count"],
-      "properties": {
-        "type": { "enum": ["core", "gpu"]},
-        "count": { "type": "integer", "minimum" : 1 },
-        "unit": { "type": "string" }
-      },
-      "additionalProperties": false
-    },
-    "node_vertex": {
-      "description": "schema for the node resource vertex",
-      "type": "object",
-      "required": ["type", "count", "with"],
-      "properties": {
-        "type": { "enum" : ["node"] },
-        "count": { "type": "integer", "minimum" : 1 },
-        "unit": { "type": "string" },
-        "with": {
-          "type": "array",
-          "minItems": 1,
-          "maxItems": 1,
-          "items": {
-            "oneOf": [
-              {"$ref": "#/definitions/slot_vertex"}
-            ]
-          }
-        }
-      },
-      "additionalProperties": false
-    },
-    "slot_vertex": {
-      "description": "special slot resource type - label assigns to task slot",
-      "type": "object",
-      "required": ["type", "count", "with", "label"],
-      "properties": {
-        "type": { "enum" : ["slot"] },
-        "count": { "type": "integer", "minimum" : 1 },
-        "unit": { "type": "string" },
-        "label": { "type": "string" },
-        "exclusive": { "type": "boolean" },
-        "with": {
-          "type": "array",
-          "minItems": 1,
-          "maxItems": 2,
-          "items": {
-            "oneOf": [
-              {"$ref": "#/definitions/intranode_resource_vertex"}
-            ]
-          }
-        }
-      },
-      "additionalProperties": false
-    }
-  },
-  "type": "object",
-  "required": ["version", "resources", "attributes", "tasks"],
-  "properties": {
-    "version": {
-      "description": "the jobspec version",
-      "type": "integer",
-      "enum": [1]
-    },
-    "resources": {
-      "description": "requested resources",
-      "type": "array",
-      "minItems": 1,
-      "maxItems": 1,
-      "items": {
-        "oneOf": [
-          { "$ref": "#/definitions/node_vertex" },
-          { "$ref": "#/definitions/slot_vertex" }
-        ]
-      }
-    },
-    "attributes": {
-      "description": "system and user attributes",
-      "type": ["object", "null"],
-      "properties": {
-        "system": {
-          "type": "object",
-          "properties": {
-            "duration": { "type": "number", "minimum": 0 },
-            "cwd": { "type": "string" },
-            "environment": { "type": "object" }
-          }
-        },
-        "user": {
-          "type": "object"
-        }
-      },
-      "additionalProperties": false
-    },
-    "tasks": {
-      "description": "task configuration",
-      "type": "array",
-      "maxItems": 1,
-      "items": {
-        "type": "object",
-        "required": ["command", "slot", "count" ],
-        "properties": {
-          "command": {
-            "type": "array",
-            "minItems": 1,
-            "items": { "type": "string" }
-          },
-          "slot": { "type": "string" },
-          "count": {
-            "type": "object",
-            "additionalProperties": false,
-            "properties": {
-              "per_slot": { "type": "integer", "minimum" : 1 },
-              "total": { "type": "integer", "minimum" : 1 }
-            }
-          }
-        },
-	    "additionalProperties": false
-      }
-    }
-  }
-}
diff --git a/src/modules/job-ingest/test/job.c b/src/modules/job-ingest/test/job.c
new file mode 100644
index 000000000000..95f6c030350d
--- /dev/null
+++ b/src/modules/job-ingest/test/job.c
@@ -0,0 +1,365 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <string.h>
+#include <errno.h>
+#include <stdarg.h>
+#if HAVE_FLUX_SECURITY
+#include <flux/security/sign.h>
+#endif
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libjob/sign_none.h"
+
+#include "job.h"
+
+
+flux_msg_t *pack_request (bool owner, const char *fmt, ...)
+{
+    va_list ap;
+    flux_msg_t *msg;
+    int rc;
+    struct flux_msg_cred cred = {
+        .userid = getuid(),
+        .rolemask = owner ? FLUX_ROLE_OWNER : FLUX_ROLE_USER,
+    };
+
+    if (!(msg = flux_request_encode ("job-ingest.submit", NULL)))
+        BAIL_OUT ("could not create request");
+    va_start (ap, fmt);
+    rc = flux_msg_vpack (msg, fmt, ap);
+    va_end (ap);
+    if (rc < 0)
+        BAIL_OUT ("could not pack request");
+    if (flux_msg_set_cred (msg, cred) < 0)
+        BAIL_OUT ("could not set request cred");
+    return msg;
+}
+
+void test_job_flags_guest (void *sec, const char *J_signed)
+{
+    struct job *job;
+    flux_error_t error;
+    flux_msg_t *msg;
+
+    skip (J_signed == NULL, 1,
+         "guest flags=WAITABLE (no guest support)");
+    msg = pack_request (false,
+                        "{s:s s:i s:i}",
+                        "J", J_signed,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", FLUX_JOB_WAITABLE);
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_request flags=WAITABLE fails with EINVAL for guest");
+    flux_msg_decref (msg);
+    end_skip;
+
+    skip (J_signed == NULL, 1,
+         "guest flags=NOVALIDATE (no guest support)");
+    msg = pack_request (false,
+                        "{s:s s:i s:i}",
+                        "J", J_signed,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", FLUX_JOB_NOVALIDATE);
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EPERM,
+        "job_create_from_request flags=NOVALIDATE fails with EPERM for guest");
+    flux_msg_decref (msg);
+    end_skip;
+}
+
+void test_job_flags_owner (void *sec, const char *J_none)
+{
+    struct job *job;
+    flux_error_t error;
+    flux_msg_t *msg;
+
+    msg = pack_request (true,
+                        "{s:s s:i s:i}",
+                        "J", J_none,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", 0xffff);
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EPROTO,
+        "job_create_from_request flags=0xffff fails with EPROTO");
+    flux_msg_decref (msg);
+
+    msg = pack_request (true,
+                        "{s:s s:i s:i}",
+                        "J", J_none,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", FLUX_JOB_WAITABLE);
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job != NULL,
+        "job_create_from_request flags=WAITABLE works for owner");
+    job_destroy (job);
+    flux_msg_decref (msg);
+
+    msg = pack_request (true,
+                        "{s:s s:i s:i}",
+                        "J", J_none,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", FLUX_JOB_NOVALIDATE);
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job != NULL,
+        "job_create_from_request flags=NOVALIDATE works for owner");
+    job_destroy (job);
+    flux_msg_decref (msg);
+}
+
+void test_job_urgency_guest (void *sec, const char *J_signed)
+{
+    struct job *job;
+    flux_error_t error;
+    flux_msg_t *msg;
+
+    skip (J_signed == NULL, 1,
+         "guest urgency=MAX (no guest support)");
+    msg = pack_request (false,
+                        "{s:s s:i s:i}",
+                        "J", J_signed,
+                        "urgency", FLUX_JOB_URGENCY_MAX,
+                        "flags", 0);
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_request urgency=MAX fails with EINVAL for guest");
+    flux_msg_decref (msg);
+    end_skip;
+}
+
+void test_job_urgency_owner (void *sec, const char *J_none)
+{
+    struct job *job;
+    flux_error_t error;
+    flux_msg_t *msg;
+
+    msg = pack_request (true,
+                        "{s:s s:i s:i}",
+                        "J", J_none,
+                        "urgency", 9999,
+                        "flags", 0);
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_request urgency=9999 fails with EINVAL");
+    flux_msg_decref (msg);
+
+    msg = pack_request (true,
+                        "{s:s s:i s:i}",
+                        "J", J_none,
+                        "urgency", FLUX_JOB_URGENCY_MAX,
+                        "flags", 0);
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job != NULL,
+        "job_create_from_request urgency=MAX works for owner");
+    job_destroy (job);
+    flux_msg_decref (msg);
+}
+
+void test_job_basic_guest (void *sec, const char *J_signed, const char *J_none)
+{
+    struct job *job;
+    flux_error_t error;
+    flux_msg_t *msg;
+
+    skip (J_signed == NULL, 1,
+         "guest basic check (no guest support)");
+    msg = pack_request (false,
+                        "{s:s s:i s:i}",
+                        "J", J_signed,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", 0);
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job != NULL,
+        "job_create_from_request works for guest");
+    job_destroy (job);
+    flux_msg_decref (msg);
+    end_skip;
+
+    msg = pack_request (false,
+                        "{s:s s:i s:i}",
+                        "J", J_none,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", 0);
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EPERM,
+        "job_create_from_request sign_type=none fails for guest");
+    flux_msg_decref (msg);
+}
+
+void test_job_basic_owner (void *sec, const char *J_none, const char *J_bad)
+{
+    struct job *job;
+    flux_error_t error;
+    flux_msg_t *msg;
+    json_t *o;
+
+    errno = 0;
+    if (!(job = job_create_from_request (NULL, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_request msg=NULL fails with EINVAL");
+
+    if (!(msg = flux_request_encode ("topic", "xyz")))
+        BAIL_OUT ("could not create message");
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EPROTO,
+        "job_create_from_request non-json-payload fails with EPROTO");
+    flux_msg_decref (msg);
+
+    msg = pack_request (true,
+                        "{s:s s:i s:i}",
+                        "J", J_none,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", 0);
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job != NULL,
+        "job_create_from_request works for owner");
+    o = job_json_object (job, &error);
+    ok (o != NULL,
+        "job_json_object works");
+    json_decref (o);
+    json_decref (job->jobspec);
+    job->jobspec = NULL;
+    errno = 0;
+    if (!(o = job_json_object (job, &error)))
+        diag ("job_json_object: %s", error.text);
+    ok (o == NULL && errno == EINVAL,
+        "job_json_object fails on incomplete job struct");
+
+    job_destroy (job);
+    flux_msg_decref (msg);
+
+    struct flux_msg_cred cred = {
+        .userid = getuid() + 1,
+        .rolemask = FLUX_ROLE_OWNER
+    };
+    msg = pack_request (true,
+                        "{s:s s:i s:i}",
+                        "J", J_none,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", 0);
+    if (flux_msg_set_cred (msg, cred) < 0)
+        BAIL_OUT ("could not override message cred");
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EPERM,
+        "job_create_from_request submitter != signer fails with EPERM");
+    flux_msg_decref (msg);
+
+    msg = pack_request (true,
+                        "{s:s% s:i s:i}",
+                        "J", J_none, strlen (J_none) - 8,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", 0);
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_request J=damaged fails with EINVAL");
+    flux_msg_decref (msg);
+
+    msg = pack_request (true,
+                        "{s:s s:i s:i}",
+                        "J", J_bad,
+                        "urgency", FLUX_JOB_URGENCY_DEFAULT,
+                        "flags", 0);
+    errno = 0;
+    if (!(job = job_create_from_request (msg, sec, &error)))
+        diag ("%s", error.text);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_request J=bad-contents fails with EINVAL");
+    flux_msg_decref (msg);
+
+    lives_ok ({job_destroy (NULL);},
+              "job_destroy NULL doesn't crash");
+}
+
+int main (int argc, char *argv[])
+{
+#if HAVE_FLUX_SECURITY
+    static flux_security_t *sec = NULL;
+    static const char *sec_config;
+#else
+    static void *sec = NULL;
+#endif
+    const char *jobspec = "{}"; // fake it
+    char *J_none;
+    char *J_bad;
+    char *J_signed = NULL;
+
+    plan (NO_PLAN);
+
+#if HAVE_FLUX_SECURITY
+    const char *J;
+    if (!(sec = flux_security_create (0)))
+        BAIL_OUT ("flux_security_create: %s", strerror (errno));
+    if (flux_security_configure (sec, sec_config) < 0)
+        BAIL_OUT ("security config %s", flux_security_last_error (sec));
+
+    /*  Only enable guest tests if flux_sign_wrap(3) succeeds:
+     */
+    if ((J = flux_sign_wrap (sec, jobspec, strlen (jobspec), NULL, 0)))
+        if (!(J_signed = strdup (J)))
+            BAIL_OUT ("could not strdup signed J");
+#endif
+    if (!(J_none = sign_none_wrap (jobspec, strlen (jobspec), getuid ())))
+        BAIL_OUT ("failed to sign jobspec with none mech: %s",
+                  strerror (errno));
+    if (!(J_bad = sign_none_wrap ("{", 1, getuid ())))
+        BAIL_OUT ("failed to sign bad jobspec with none mech: %s",
+                  strerror (errno));
+
+    test_job_basic_owner (sec, J_none, J_bad);
+    test_job_basic_guest (sec, J_signed, J_none);
+
+    test_job_flags_owner (sec, J_none);
+    test_job_flags_guest (sec, J_signed);
+
+    test_job_urgency_owner (sec, J_none);
+    test_job_urgency_guest (sec, J_signed);
+
+#if HAVE_FLUX_SECURITY
+    flux_security_destroy (sec);
+#endif
+    free (J_bad);
+    free (J_none);
+    free (J_signed);
+
+    done_testing ();
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-ingest/test/util.c b/src/modules/job-ingest/test/util.c
new file mode 100644
index 000000000000..c558b8c365da
--- /dev/null
+++ b/src/modules/job-ingest/test/util.c
@@ -0,0 +1,84 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <string.h>
+#include <errno.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+
+#include "util.h"
+
+void test_join (void)
+{
+    json_t *o;
+    char *s;
+
+    if (!(o = json_pack ("[s]", "foo")))
+        BAIL_OUT ("could not create json array");
+    s = util_join_arguments (o);
+    ok (s && streq (s, "foo"),
+        "util_join_arguments [foo] works");
+    free (s);
+    json_decref (o);
+
+    if (!(o = json_pack ("[sss]", "foo", "bar", "baz")))
+        BAIL_OUT ("could not create json array");
+    s = util_join_arguments (o);
+    ok (s && streq (s, "foo,bar,baz"),
+        "util_join_arguments [foo,bar,baz] works");
+    free (s);
+    json_decref (o);
+
+    if (!(o = json_array ()))
+        BAIL_OUT ("could not create json array");
+    s = util_join_arguments (o);
+    ok (s && streq (s, ""),
+        "util_join_arguments [] works");
+    free (s);
+    json_decref (o);
+
+    if (!(o = json_object ()))
+        BAIL_OUT ("could not create json object");
+    errno = 0;
+    s = util_join_arguments (o);
+    ok (s == NULL && errno == EINVAL,
+        "util_join_arguments object fails with EINVAL");
+    json_decref (o);
+
+    if (!(o = json_pack ("[i]", 42)))
+        BAIL_OUT ("could not create json array");
+    errno = 0;
+    s = util_join_arguments (o);
+    ok (s == NULL && errno == EINVAL,
+        "util_join_arguments [42] fails with EINVAL");
+    json_decref (o);
+
+    errno = 0;
+    s = util_join_arguments (NULL);
+    ok (s == NULL && errno == EINVAL,
+        "util_join_arguments NULL fails with EINVAL");
+}
+
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_join ();
+
+    done_testing ();
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-ingest/util.c b/src/modules/job-ingest/util.c
new file mode 100644
index 000000000000..d61ec1097b63
--- /dev/null
+++ b/src/modules/job-ingest/util.c
@@ -0,0 +1,56 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <string.h>
+#include <jansson.h>
+
+#include "util.h"
+
+char *util_join_arguments (json_t *o)
+{
+    int n = 0;
+    size_t index;
+    json_t *value;
+    const char *arg;
+    char *result;
+
+    if (!json_is_array (o)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    json_array_foreach (o, index, value) {
+        if (index > 0)
+            n++; // space for comma
+        if (!(arg = json_string_value (value))) {
+            errno = EINVAL;
+            return NULL;
+        }
+        n += strlen (arg);
+    }
+    n++; // space for terminating \0
+
+    if (!(result = calloc (1, n)))
+        return NULL;
+
+    json_array_foreach (o, index, value) {
+        if (index > 0)
+            strcat (result, ",");
+        strcat (result, json_string_value (value));
+    }
+
+    return result;
+}
+
+// vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/modules/job-ingest/util.h b/src/modules/job-ingest/util.h
new file mode 100644
index 000000000000..9beb8d3d710f
--- /dev/null
+++ b/src/modules/job-ingest/util.h
@@ -0,0 +1,20 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _JOB_INGEST_UTIL_H_
+#define _JOB_INGEST_UTIL_H_
+
+#include <jansson.h>
+
+char *util_join_arguments (json_t *o);
+
+#endif /* !_JOB_INGEST_UTIL_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-ingest/validate.c b/src/modules/job-ingest/validate.c
deleted file mode 100644
index a4ec603c7139..000000000000
--- a/src/modules/job-ingest/validate.c
+++ /dev/null
@@ -1,245 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* validate - asynchronous jobspec validation interface
- *
- * Spawn worker(s) to validate jobspec.  Up to 'DEFAULT_WORKER_COUNT'
- * workers may be active at one time.  They are started lazily, on demand,
- * and stop after a period of inactivity (see "tunables" below).
- *
- * The validator executable and its command line, including the
- * location of jobspec.jsonschema, are currently hardwired.
- *
- * Jobspec is expected to be in encoded JSON form, with or without
- * whitespace or NULL termination.  The encoding is normalized before
- * it is sent to the worker on a single line.
- *
- * The future is fulfilled with the result of validation.  On success,
- * the container will be empty.  On failure, the reason the jobspec
- * did not pass validation (suitable for returning to the submitting user)
- * will be assigned to the future's extended error string.
- */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <unistd.h>
-#include <argz.h>
-#include <czmq.h>
-#include <jansson.h>
-#include <flux/core.h>
-
-#include "validate.h"
-#include "worker.h"
-
-/* Tunables:
- */
-
-/* The maximum number of concurrent workers.
- */
-#define MAX_WORKER_COUNT 4
-
-/* Start a new worker if backlog reaches this level for all active workers.
- */
-const int worker_queue_threshold = 32;
-
-/* Workers exit once they have been inactive for this many seconds.
- */
-const double worker_inactivity_timeout = 5.0;
-
-
-
-struct validate {
-    flux_t *h;
-    struct worker *worker[MAX_WORKER_COUNT];
-};
-
-static void validate_killall (struct validate *v)
-{
-    flux_future_t *cf = flux_future_wait_all_create ();
-    flux_future_t *f;
-    int i;
-    if (!cf) {
-        flux_log_error (v->h, "validate_destroy: flux_future_wait_all_create");
-        return;
-    }
-    flux_future_set_flux (cf, v->h);
-    for (i = 0; i < MAX_WORKER_COUNT; i++) {
-        if ((f = worker_kill (v->worker[i], SIGKILL)))
-            flux_future_push (cf, NULL, f);
-    }
-    /* Wait for up to 5s for response that signals have been delivered
-     *  to all workers before continuing. This should ensure no workers
-     *  are left around after removal of the job-ingest module.
-     *  (report, but otherwise ignore errors)
-     */
-    if (flux_future_wait_for (cf, 5.) < 0
-        || flux_future_get (cf, NULL) < 0)
-        flux_log_error (v->h, "validate_destroy: killing workers");
-    flux_future_destroy (cf);
-}
-
-int validate_stop_notify (struct validate *v, process_exit_f cb, void *arg)
-{
-    int i;
-    int count;
-
-    count = 0;
-    for (i = 0; i < MAX_WORKER_COUNT; i++)
-        count += worker_stop_notify (v->worker[i], cb, arg);
-    return count;
-}
-
-void validate_destroy (struct validate *v)
-{
-    if (v) {
-        int saved_errno = errno;
-        int i;
-        validate_killall (v);
-        for (i = 0; i < MAX_WORKER_COUNT; i++)
-            worker_destroy (v->worker[i]);
-        free (v);
-        errno = saved_errno;
-    }
-}
-
-static bool str_ends_with (const char *str, const char *suffix)
-{
-    int str_len = strlen (str);
-    int suffix_len = strlen (suffix);
-
-    return (str_len >= suffix_len) && \
-        (!strncmp ((str + str_len) - suffix_len, suffix, suffix_len));
-}
-
-struct validate *validate_create (flux_t *h,
-                                  const char *validate_path,
-                                  const char *validator_args)
-{
-    struct validate *v;
-    char *argv[5];
-    int argc = 0;
-    int i;
-    char *validator_argz = NULL;
-    char *validator_arg = NULL;
-    size_t validator_argz_len = 0;
-
-    if (!(v = calloc (1, sizeof (*v))))
-        return NULL;
-    v->h = h;
-
-    assert (validate_path != NULL);
-
-    if (str_ends_with (validate_path, ".py"))
-        argv[argc++] = PYTHON_INTERPRETER;
-    argv[argc++] = (char *)validate_path;
-    if (validator_args != NULL) {
-        // Parse the comma-separated argument list passed in when loading the
-        // job-ingest module.  For example:
-        // module load job-ingest validator-args=--schema,/path/to/schema.json
-        if (argz_create_sep (validator_args,
-                             ',',
-                             &validator_argz,
-                             &validator_argz_len) != 0) {
-            goto error;
-        }
-        validator_arg = argz_next (validator_argz,
-                                   validator_argz_len,
-                                   NULL);
-        while (validator_arg != NULL) {
-            argv[argc++] = validator_arg;
-            validator_arg = argz_next (validator_argz,
-                                       validator_argz_len,
-                                       validator_arg);
-        }
-    }
-    argv[argc] = NULL;
-
-    for (i = 0; i < MAX_WORKER_COUNT; i++) {
-        if (!(v->worker[i] = worker_create (h, worker_inactivity_timeout,
-                                            validate_path,
-                                            argc, argv)))
-            goto error;
-    }
-    free (validator_argz);
-    return v;
-error:
-    free (validator_argz);
-    validate_destroy (v);
-    return NULL;
-}
-
-/* Select worker with least backlog.  If none is running, or the best
- * has a backlog at or beyond threshold, activate a new one, if possible.
- */
-struct worker *select_best_worker (struct validate *v)
-{
-    struct worker *best = NULL;
-    struct worker *idle = NULL;
-    int i;
-
-    for (i = 0; i < MAX_WORKER_COUNT; i++) {
-        if (worker_is_running (v->worker[i])) {
-            if (!best || (worker_queue_depth (v->worker[i])
-                        < worker_queue_depth (best)))
-                best = v->worker[i];
-        }
-        else if (!idle)
-            idle = v->worker[i];
-    }
-    if (idle && (!best || worker_queue_depth (best) >= worker_queue_threshold))
-        best = idle;
-
-    return best;
-}
-
-flux_future_t *validate_jobspec (struct validate *v, const char *buf, int len)
-{
-    flux_future_t *f;
-    json_t *o;
-    json_error_t error;
-    char *s;
-    int saved_errno;
-    struct worker *w;
-
-    /* Make sure jobspec decodes as JSON (no YAML allowed here).
-     * Capture any JSON parsing errors by returning them in a future.
-     * Then re-encode in compact form to eliminate any white space (esp \n).
-     */
-    if (!(o = json_loadb (buf, len, 0, &error))) {
-        char errbuf[256];
-        if (!(f = flux_future_create (NULL, NULL)))
-            return NULL;
-        flux_future_set_flux (f, v->h);
-        (void)snprintf (errbuf, sizeof (errbuf),
-                       "jobspec: invalid JSON: %s", error.text);
-        flux_future_fulfill_error (f, EINVAL, errbuf);
-        return f;
-    }
-    if (!(s = json_dumps (o, JSON_COMPACT)))
-        goto error;
-    w = select_best_worker (v);
-    assert (w != NULL);
-    if (!(f = worker_request (w, s)))
-        goto error;
-    free (s);
-    json_decref (o);
-    return f;
-error:
-    saved_errno = errno;
-    free (s);
-    json_decref (o);
-    errno = saved_errno;
-    return NULL;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-ingest/validate.h b/src/modules/job-ingest/validate.h
deleted file mode 100644
index e7fdd078a53b..000000000000
--- a/src/modules/job-ingest/validate.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _JOB_INGEST_VALIDATE_H
-#define _JOB_INGEST_VALIDATE_H
-
-#include <flux/core.h>
-
-#include "types.h"
-
-struct validate *v;
-
-/* Submit jobspec ('buf, 'len') for validation.
- * Future is fulfilled once validation is complete.
- */
-flux_future_t *validate_jobspec (struct validate *v, const char *buf, int len);
-
-/* Tell validators to stop.
- * Return a count of running processes.
- * If nonzero, arrange for callback to be called each time a process exits.
- */
-int validate_stop_notify (struct validate *v, process_exit_f cb, void *arg);
-
-struct validate *validate_create (flux_t *h,
-                                  const char *validate_path,
-                                  const char *validator_args);
-
-void validate_destroy (struct validate *v);
-
-#endif /* !_JOB_INGEST_VALIDATE_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-ingest/validators/validate-jobspec.py b/src/modules/job-ingest/validators/validate-jobspec.py
deleted file mode 100755
index 82bc38f47046..000000000000
--- a/src/modules/job-ingest/validators/validate-jobspec.py
+++ /dev/null
@@ -1,60 +0,0 @@
-##############################################################
-#  Copyright 2018 Lawrence Livermore National Security, LLC
-#  (c.f. AUTHORS, NOTICE.LLNS, COPYING)
-#
-#  This file is part of the Flux resource manager framework.
-#  For details, see https://github.com/flux-framework.
-#
-#  SPDX-License-Identifier: LGPL-3.0
-##############################################################
-
-from __future__ import print_function
-
-import sys
-import json
-import argparse
-
-from flux.job import validate_jobspec
-
-
-def emit(object_dict):
-    s = json.dumps(object_dict, separators=(",", ":"))
-    print(s)
-    sys.stdout.flush()
-
-
-print("ready", file=sys.stderr)
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--require-version", type=int)
-args = parser.parse_args()
-
-if args.require_version is not None:
-    if args.require_version < 1:
-        print(
-            "Required version too low: {} is < 1".format(args.require_version),
-            file=sys.stderr,
-        )
-        exit(1)
-    elif args.require_version > 1:
-        print(
-            "Required version too high: {} is > 1".format(args.require_version),
-            file=sys.stderr,
-        )
-        exit(1)
-
-while True:
-    line = sys.stdin.readline()
-    if line == "":
-        break
-    errnum, errstr = (0, None)
-    try:
-        validate_jobspec(line, args.require_version)
-    except (ValueError, TypeError, EnvironmentError) as e:
-        errnum, errstr = (1, str(e))
-    if errstr != None:
-        emit({"errnum": errnum, "errstr": errstr})
-    else:
-        emit({"errnum": errnum})
-
-print("exiting", file=sys.stderr)
diff --git a/src/modules/job-ingest/validators/validate-schema.py b/src/modules/job-ingest/validators/validate-schema.py
deleted file mode 100755
index f973f8372316..000000000000
--- a/src/modules/job-ingest/validators/validate-schema.py
+++ /dev/null
@@ -1,60 +0,0 @@
-##############################################################
-#  Copyright 2018 Lawrence Livermore National Security, LLC
-#  (c.f. AUTHORS, NOTICE.LLNS, COPYING)
-#
-#  This file is part of the Flux resource manager framework.
-#  For details, see https://github.com/flux-framework.
-#
-#  SPDX-License-Identifier: LGPL-3.0
-##############################################################
-
-from __future__ import print_function
-
-import sys
-import argparse
-import json
-import jsonschema
-
-
-def emit(object_dict):
-    s = json.dumps(object_dict, separators=(",", ":"))
-    print(s)
-    sys.stdout.flush()
-
-
-def validate(schema, line):
-    try:
-        jobspec = json.loads(line)
-        jsonschema.validate(jobspec, schema)
-    except ValueError as e:
-        return (1, str(e))
-    except jsonschema.exceptions.ValidationError as e:
-        return (1, e.message.replace("u'", "'"))
-    return (0, None)
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--schema", "-s", type=str, required=True)
-args = parser.parse_args()
-
-try:
-    with open(args.schema) as fd:
-        schema = json.load(fd)
-except (OSError, IOError) as e:
-    sys.exit(args.schema + ": " + e.strerror)
-except ValueError as e:
-    sys.exit(args.schema + ": " + str(e))
-
-print("ready", file=sys.stderr)
-
-while True:
-    line = sys.stdin.readline()
-    if line == "":
-        break
-    errnum, errstr = validate(schema, line)
-    if errstr != None:
-        emit({"errnum": errnum, "errstr": errstr})
-    else:
-        emit({"errnum": errnum})
-
-print("exiting", file=sys.stderr)
diff --git a/src/modules/job-ingest/workcrew.c b/src/modules/job-ingest/workcrew.c
new file mode 100644
index 000000000000..e12ad0268f23
--- /dev/null
+++ b/src/modules/job-ingest/workcrew.c
@@ -0,0 +1,288 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* workcrew - asynchronous worker interface
+ *
+ * Spawn worker(s) to process jobspec.  Up to 'DEFAULT_WORKER_COUNT'
+ * workers may be active at one time.  They are started lazily, on demand,
+ * selected based on least backlog, and stopped after a period of inactivity.
+ *
+ * Jobspec input is provided to workcrew_process_job() as a JSON object,
+ * and is internally encoded as a single-line string for a worker.
+ *
+ * The future returned by workcrew_process_job() is fulfilled with the result
+ * of worker execution (success or failure and optional JSON object).
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#ifdef HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <jansson.h>
+#include <assert.h>
+#include <signal.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/errno_safe.h"
+
+#include "workcrew.h"
+#include "worker.h"
+
+/* The maximum number of concurrent workers.
+ */
+#define WORKCREW_SIZE 4
+
+/* Start a new worker if backlog reaches this level for all active workers.
+ */
+static const int workcrew_max_backlog = 32;
+
+/* Workers exit once they have been inactive for this many seconds.
+ */
+static const double workcrew_inactivity_timeout = 5.0;
+
+struct workcrew {
+    flux_t *h;
+    struct worker *worker[WORKCREW_SIZE];
+};
+
+static void workcrew_killall (struct workcrew *crew)
+{
+    flux_future_t *cf = NULL;
+    flux_future_t *f;
+    int i;
+
+    if (crew == NULL)
+        return;
+    if (!(cf = flux_future_wait_all_create ())) {
+        flux_log_error (crew->h, "workcrew: error setting up for killall");
+        return;
+    }
+    flux_future_set_flux (cf, crew->h);
+    for (i = 0; i < WORKCREW_SIZE; i++) {
+        if ((f = worker_kill (crew->worker[i], SIGKILL)))
+            flux_future_push (cf, NULL, f);
+    }
+    /* Wait for up to 5s for response that signals have been delivered
+     *  to all workers before continuing. This should ensure no workers
+     *  are left around after removal of the job-ingest module.
+     *  (report, but otherwise ignore errors)
+     */
+    if (flux_future_wait_for (cf, 5.) < 0
+        || flux_future_get (cf, NULL) < 0) {
+        flux_log_error (crew->h,
+                        "workcrew: killall failed: %s",
+                        future_strerror (cf, errno));
+    }
+    flux_future_destroy (cf);
+}
+
+int workcrew_stop_notify (struct workcrew *crew, process_exit_f cb, void *arg)
+{
+    int i;
+    int count;
+
+    if (crew == NULL)
+        return 0;
+
+    count = 0;
+    for (i = 0; i < WORKCREW_SIZE; i++)
+        count += worker_stop_notify (crew->worker[i], cb, arg);
+    return count;
+}
+
+void workcrew_destroy (struct workcrew *crew)
+{
+    if (crew) {
+        int saved_errno = errno;
+        int i;
+        workcrew_killall (crew);
+        for (i = 0; i < WORKCREW_SIZE; i++)
+            worker_destroy (crew->worker[i]);
+        free (crew);
+        errno = saved_errno;
+    }
+}
+
+static int create_worker_argz (char **argzp,
+                               size_t *argz_lenp,
+                               const char *cmdname,
+                               const char *plugins,
+                               const char *args)
+{
+    int e;
+    if ((e = argz_add (argzp, argz_lenp, "flux"))
+            || (e = argz_add (argzp, argz_lenp, cmdname)))
+        goto error;
+
+    if (plugins) {
+        if ((e = argz_add (argzp, argz_lenp, "--plugins"))
+            || (e = argz_add (argzp, argz_lenp, plugins))) {
+            goto error;
+        }
+    }
+    if (args
+        && (e = argz_add_sep (argzp, argz_lenp, args, ','))) {
+        goto error;
+    }
+    return 0;
+error:
+    errno = e;
+    return -1;
+}
+
+int workcrew_configure (struct workcrew *crew,
+                        const char *cmdname,
+                        const char *plugins,
+                        const char *args,
+                        const char *bufsize)
+{
+    int rc = -1;
+    int argc;
+    char **argv = NULL;
+    char *argz = NULL;
+    size_t argz_len = 0;
+
+    if (create_worker_argz (&argz, &argz_len, cmdname, plugins, args) < 0)
+        goto error;
+
+    argc = argz_count (argz, argz_len);
+    if (!(argv = calloc (1, sizeof (char *) * (argc + 1)))) {
+        flux_log_error (crew->h, "failed to create argv");
+        goto error;
+    }
+    argz_extract (argz, argz_len, argv);
+
+    for (int i = 0; i < WORKCREW_SIZE; i++) {
+        if (!crew->worker[i]) {
+            char name[256];
+            (void) snprintf (name, sizeof (name), "%s[%d]", cmdname, i);
+            if (!(crew->worker[i] = worker_create (crew->h,
+                                                   workcrew_inactivity_timeout,
+                                                   name)))
+                goto error;
+        }
+        if (worker_set_cmdline (crew->worker[i], argc, argv) < 0)
+            goto error;
+        if (bufsize && worker_set_bufsize (crew->worker[i], bufsize) < 0)
+            goto error;
+    }
+    /* Close stdin of current workers and allow them to restart on demand.
+     * This forces them to re-acquire their configuration, if any.
+     */
+    workcrew_stop_notify (crew, NULL, NULL);
+    rc = 0;
+error:
+    ERRNO_SAFE_WRAP (free, argv);
+    ERRNO_SAFE_WRAP (free, argz);
+    return rc;
+}
+
+struct workcrew *workcrew_create (flux_t *h)
+{
+    struct workcrew *crew;
+    if (!(crew = calloc (1, sizeof (*crew))))
+        return NULL;
+    crew->h = h;
+    return crew;
+}
+
+/* Select worker with least backlog.  If none is running, or the best
+ * has a backlog at or beyond threshold, select a non-running worker which
+ * will be started by worker_request().
+ */
+static struct worker *select_best_worker (struct workcrew *crew)
+{
+    struct worker *best = NULL;
+    struct worker *idle = NULL;
+    int i;
+
+    for (i = 0; i < WORKCREW_SIZE; i++) {
+        if (worker_is_running (crew->worker[i])) {
+            if (!best || (worker_queue_depth (crew->worker[i])
+                        < worker_queue_depth (best)))
+                best = crew->worker[i];
+        }
+        else if (!idle)
+            idle = crew->worker[i];
+    }
+    if (idle && (!best || worker_queue_depth (best) >= workcrew_max_backlog))
+        best = idle;
+
+    return best;
+}
+
+/* Re-encode job info in compact form to eliminate any white space (esp \n),
+ * then pass it to least busy worker, returning a future.
+ */
+flux_future_t *workcrew_process_job (struct workcrew *crew, json_t *job)
+{
+    flux_future_t *f;
+    char *s;
+    struct worker *w;
+
+    if (!(s = json_dumps (job, JSON_COMPACT))) {
+        errno = ENOMEM;
+        goto error;
+    }
+    w = select_best_worker (crew);
+    assert (w != NULL);
+    if (!(f = worker_request (w, s)))
+        goto error;
+    free (s);
+    return f;
+error:
+    ERRNO_SAFE_WRAP (free, s);
+    return NULL;
+}
+
+json_t *workcrew_stats_get (struct workcrew *crew)
+{
+    json_t *o = NULL;
+
+    if (crew) {
+        int running = 0;
+        int requests = 0;
+        int errors = 0;
+        int backlog = 0;
+        int trash = 0;
+        json_t *pids = json_array ();
+
+        for (int i = 0; i < WORKCREW_SIZE; i++) {
+            running += worker_is_running (crew->worker[i]) ? 1 : 0;
+            requests += worker_request_count (crew->worker[i]);
+            errors += worker_error_count (crew->worker[i]);
+            trash += worker_trash_count (crew->worker[i]);
+            backlog += worker_queue_depth (crew->worker[i]);
+            if (pids) {
+                json_t *pid = json_integer (worker_pid (crew->worker[i]));
+                if (json_array_append_new (pids, pid ? pid : json_null ()) < 0)
+                    json_decref (pid);
+            }
+        }
+        o = json_pack ("{s:i s:i s:i s:i s:i s:O}",
+                       "running", running,
+                       "requests", requests,
+                       "errors", errors,
+                       "trash", trash,
+                       "backlog", backlog,
+                       "pids", pids ? pids : json_null ());
+        json_decref (pids);
+    }
+    return o ? o : json_null ();
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-ingest/workcrew.h b/src/modules/job-ingest/workcrew.h
new file mode 100644
index 000000000000..b4968794fcbc
--- /dev/null
+++ b/src/modules/job-ingest/workcrew.h
@@ -0,0 +1,62 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _JOB_INGEST_WORKCREW_H
+#define _JOB_INGEST_WORKCREW_H
+
+#include <flux/core.h>
+
+#include "types.h"
+
+struct workcrew;
+
+json_t *workcrew_stats_get (struct workcrew *crew);
+
+/* Submit one job to workcrew.
+ * Future is fulfilled once processing is complete.
+ */
+flux_future_t *workcrew_process_job (struct workcrew *crew, json_t *job);
+
+/* Tell workcrew to stop.
+ * Return a count of running processes.
+ * If nonzero, arrange for callback to be called each time a process exits.
+ */
+int workcrew_stop_notify (struct workcrew *crew, process_exit_f cb, void *arg);
+
+struct workcrew *workcrew_create (flux_t *h);
+
+/* (Re-)configure work crew command.  This must be called initially and then
+ * may be called again when the config changes.  Workers pick up changes on
+ * the next restart.  The worker command line will be:
+ *
+ * flux <cmdname> [--plugins <plugins>] [<args>]
+ *
+ * plugins should be a comma-delimited list of plugin names, or NULL.
+ * It is passed through as one command line argument with delimiters intact.
+ *
+ * args should be a comma-delimited list of additional arguments, or NULL.
+ * The list is split into separate command line arguments.
+ *
+ * bufsize should be a string buffer size represented as a floating point
+ * value with optional scale suffix [kKMG].
+ */
+int workcrew_configure (struct workcrew *crew,
+                        const char *cmdname,
+                        const char *plugins,
+                        const char *args,
+                        const char *bufsize);
+
+void workcrew_destroy (struct workcrew *crew);
+
+#endif /* !_JOB_INGEST_WORKCREW_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-ingest/worker.c b/src/modules/job-ingest/worker.c
index 795a662e16fb..b394fa8287b9 100644
--- a/src/modules/job-ingest/worker.c
+++ b/src/modules/job-ingest/worker.c
@@ -42,11 +42,17 @@
 #include "config.h"
 #endif
 #include <unistd.h>
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/basename.h"
+#include "ccan/str/str.h"
+
 #include "worker.h"
 
+extern char **environ;
+
 const char *worker_auxkey = "flux::worker";
 
 struct worker {
@@ -60,13 +66,33 @@ struct worker {
     zlist_t *trash;
     process_exit_f exit_cb;
     void *exit_arg;
+    int request_count;
+    int error_count;
 };
 
 static int worker_start (struct worker *w);
 static void worker_stop (struct worker *w);
+static void worker_unexpected_exit (struct worker *w);
+
+static void worker_cleanup_process (struct worker *w, flux_subprocess_t *p)
+{
+    flux_subprocess_destroy (p);
+    zlist_remove (w->trash, p);
+
+    /*  Be sure to nullify w->p if this worker unexpectedly exited
+     *  (i.e., worker_stop() wasn't called on it)
+     */
+    if (w->p == p)
+        w->p = NULL;
+
+    /*  Call worker_stop_notify() callback, if any
+     */
+    if (w->exit_cb)
+        w->exit_cb (w->exit_arg);
+}
 
 /* Subprocess completed.
- * Destroy the subprocess, but don't use w->p since that may be a diferent
+ * Destroy the subprocess, but don't use w->p since that may be a different
  * one, if worker_stop() was followed immediately by worker_start().
  * Remove from w->trash to avoid double-free in worker_destroy()
  */
@@ -76,29 +102,14 @@ static void worker_completion_cb (flux_subprocess_t *p)
     int rc;
 
     if ((rc = flux_subprocess_exit_code (p)) >= 0) {
-        if (rc == 0)
-            flux_log (w->h, LOG_DEBUG, "%s: exited normally", w->name);
-        else {
+        if (rc != 0)
             flux_log (w->h, LOG_ERR, "%s: exited with rc=%d", w->name, rc);
-        }
     }
     else if ((rc = flux_subprocess_signaled (p)) >= 0)
         flux_log (w->h, LOG_ERR, "%s: killed by %s", w->name, strsignal (rc));
     else
         flux_log (w->h, LOG_ERR, "%s: completed (not signal or exit)", w->name);
-    flux_subprocess_destroy (p);
-    zlist_remove (w->trash, p);
-
-    /*  Be sure to nullify w->p if this worker unexpectedly exited
-     *  (i.e., worker_stop() wasn't called on it)
-     */
-    if (w->p == p)
-        w->p = NULL;
-
-    /*  Call worker_stop_notify() callback, if any
-     */
-    if (w->exit_cb)
-        w->exit_cb (w->exit_arg);
+    worker_cleanup_process (w, p);
 }
 
 /* Subprocess state change.
@@ -110,16 +121,19 @@ static void worker_state_cb (flux_subprocess_t *p,
 
     switch (state) {
         case FLUX_SUBPROCESS_RUNNING:
-            flux_log (w->h, LOG_DEBUG, "%s: running (pid=%d)", w->name,
-                      (int)flux_subprocess_pid (p));
             break;
-        case FLUX_SUBPROCESS_EXEC_FAILED:
         case FLUX_SUBPROCESS_FAILED:
-            flux_log (w->h, LOG_ERR, "%s: %s", w->name,
-                      flux_subprocess_state_string (state));
+            flux_log (w->h,
+                      LOG_ERR,
+                      "%s: %s: %s", w->name,
+                      flux_subprocess_state_string (state),
+                      strerror (flux_subprocess_fail_errno (p)));
+            worker_unexpected_exit (w);
+            worker_cleanup_process (w, p);
             break;
         case FLUX_SUBPROCESS_EXITED:
         case FLUX_SUBPROCESS_INIT:
+        case FLUX_SUBPROCESS_STOPPED:
             break; // ignore
     }
 }
@@ -128,7 +142,6 @@ static void worker_timeout (flux_reactor_t *r, flux_watcher_t *timer,
                             int revents, void *arg)
 {
     struct worker *w = arg;
-    flux_log (w->h, LOG_DEBUG, "%s: inactivity timeout", w->name);
     worker_stop (w);
 }
 
@@ -165,9 +178,11 @@ static void worker_fulfill_future (struct worker *w, flux_future_t *f, const cha
         errnum = EINVAL;
         goto error;
     }
-    if (json_unpack (o, "{s:i s?:s s?:o}", "errnum", &errnum,
-                                           "errstr", &errstr,
-                                           "data", &data) < 0) {
+    if (json_unpack (o,
+                     "{s:i s?s s?o}",
+                     "errnum", &errnum,
+                     "errstr", &errstr,
+                     "data", &data) < 0) {
         flux_log (w->h, LOG_ERR, "%s: json_unpack '%s' failed", w->name, s);
         errnum = EINVAL;
         goto error;
@@ -185,6 +200,7 @@ static void worker_fulfill_future (struct worker *w, flux_future_t *f, const cha
     json_decref (o);
     return;
 error:
+    w->error_count++;
     flux_future_fulfill_error (f, errnum, errstr);
     json_decref (o);
 }
@@ -194,7 +210,7 @@ static void worker_unexpected_exit (struct worker *w)
     flux_future_t *f;
     const char *json_err =
         "{\"errnum\":71,"
-        "\"errstr\":\"Unrecoverable error: validator unexpectedly exited\"}";
+        "\"errstr\":\"Unrecoverable error: worker unexpectedly exited\"}";
 
     /*  Respond to any pending requests immediately with error above.
      *  The remainder of worker cleanup will happen in the exit callback.
@@ -202,6 +218,7 @@ static void worker_unexpected_exit (struct worker *w)
     while ((f = zlist_pop (w->queue))) {
         worker_fulfill_future (w, f, json_err);
         flux_future_decref (f);
+        w->error_count++;
     }
 }
 
@@ -215,7 +232,7 @@ static void worker_output_cb (flux_subprocess_t *p, const char *stream)
     const char *s;
     int len;
 
-    if (!(s = flux_subprocess_read_trimmed_line (p, stream, &len))) {
+    if ((len = flux_subprocess_read_trimmed_line (p, stream, &s)) < 0) {
         flux_log_error (w->h, "%s: subprocess_read_trimmed_line", w->name);
         return;
     }
@@ -233,12 +250,12 @@ static void worker_output_cb (flux_subprocess_t *p, const char *stream)
          * all output complete.
          */
         if (w->p == p &&
-            !strcmp (stream, "stdout") &&
+            streq (stream, "stdout") &&
             worker_queue_depth (w) > 0)
             worker_unexpected_exit (w);
         return;
     }
-    if (!strcmp (stream, "stdout")) {
+    if (streq (stream, "stdout")) {
         flux_future_t *f;
 
         if (!(f = zlist_pop (w->queue))) {
@@ -251,7 +268,7 @@ static void worker_output_cb (flux_subprocess_t *p, const char *stream)
         if (zlist_size (w->queue) == 0)
             worker_inactive (w);
     }
-    else if (!strcmp (stream, "stderr")) {
+    else if (streq (stream, "stderr")) {
         flux_log (w->h, LOG_DEBUG, "%s: %s", w->name, s ? s : "");
     }
 }
@@ -289,6 +306,7 @@ flux_future_t *worker_request (struct worker *w, const char *s)
         goto error;
     flux_future_incref (f); // queue takes a reference on the future
     free (buf);
+    w->request_count++;
     return f;
 error:
     saved_errno = errno;
@@ -331,24 +349,59 @@ int worker_stop_notify (struct worker *w, process_exit_f cb, void *arg)
     return count;
 }
 
-flux_future_t *worker_kill (struct worker *w, int signo)
+static void worker_kill_add (struct worker *w,
+                             flux_future_t **cf,
+                             flux_subprocess_t *p,
+                             int signo)
 {
+    long pid = flux_subprocess_pid (p);
     flux_future_t *f = NULL;
-    if (w->p) {
-        flux_log (w->h, LOG_DEBUG,
-                  "killing %s (pid=%ld)",
-                  w->name,
-                  (long) flux_subprocess_pid (w->p));
-        f = flux_subprocess_kill (w->p, signo);
+
+    flux_log (w->h,
+              LOG_DEBUG,
+              "killing %s (%spid=%ld)",
+              w->name,
+              w->p == p ? "" : "trash ",
+              pid);
+    if ((!*cf && !(*cf = flux_future_wait_all_create ()))
+        || !(f = flux_subprocess_kill (p, signo))
+        || flux_future_push (*cf, NULL, f) < 0) {
+        flux_log_error (w->h, "kill %s (pid=%ld)", w->name, pid);
+        flux_future_destroy (f);
     }
-    return f;
+}
+
+flux_future_t *worker_kill (struct worker *w, int signo)
+{
+    flux_future_t *cf = NULL;
+    flux_subprocess_t *p;
+
+    if (w->p)
+        worker_kill_add (w, &cf, w->p, signo);
+    p = zlist_first (w->trash);
+    while (p) {
+        worker_kill_add (w, &cf, p, signo);
+        p = zlist_next (w->trash);
+    }
+    // N.B. cf could be empty if worker_kill_add() fails to add future
+    if (cf && !flux_future_first_child (cf)) {
+        flux_future_destroy (cf);
+        return NULL;
+    }
+    return cf;
 }
 
 static int worker_start (struct worker *w)
 {
     if (!w->p) {
-        if (!(w->p = flux_rexec (w->h, FLUX_NODEID_ANY, 0,
-                                 w->cmd, &worker_ops))) {
+        if (!(w->p = flux_rexec_ex (w->h,
+                                    "rexec",
+                                    FLUX_NODEID_ANY,
+                                    0,
+                                    w->cmd,
+                                    &worker_ops,
+                                    flux_llog,
+                                    w->h))) {
             return -1;
         }
         if (flux_subprocess_aux_set (w->p, worker_auxkey, w, NULL) < 0) {
@@ -361,12 +414,32 @@ static int worker_start (struct worker *w)
 
 int worker_queue_depth (struct worker *w)
 {
-    return zlist_size (w->queue);
+    return w ? zlist_size (w->queue) : 0;
+}
+
+int worker_request_count (struct worker *w)
+{
+    return w ? w->request_count : 0;
+}
+
+int worker_error_count (struct worker *w)
+{
+    return w ? w->error_count : 0;
+}
+
+int worker_trash_count (struct worker *w)
+{
+    return w ? zlist_size (w->trash) : 0;
 }
 
 bool worker_is_running (struct worker *w)
 {
-    return (w->p ? true : false);
+    return (w && w->p ? true : false);
+}
+
+pid_t worker_pid (struct worker *w)
+{
+    return (w && w->p) ? flux_subprocess_pid (w->p) : 0;
 }
 
 void worker_destroy (struct worker *w)
@@ -391,35 +464,47 @@ void worker_destroy (struct worker *w)
     }
 }
 
+int worker_set_cmdline (struct worker *w, int argc, char **argv)
+{
+    flux_cmd_destroy (w->cmd);
+
+    if (!(w->cmd = flux_cmd_create (argc, argv, environ))) {
+        flux_log_error (w->h, "flux_cmd_create");
+        return -1;
+    }
+    return 0;
+}
+
+int worker_set_bufsize (struct worker *w, const char *bufsize)
+{
+    int rc = 0;
+    if (bufsize)
+        rc = flux_cmd_setopt (w->cmd, "stdin_BUFSIZE", bufsize);
+    return rc;
+}
+
 struct worker *worker_create (flux_t *h, double inactivity_timeout,
-                              const char *name,
-                              int argc, char **argv)
+                              const char *name)
 {
     struct worker *w;
-    char path[PATH_MAX + 1];
     flux_reactor_t *r = flux_get_reactor (h);
 
     if (!(w = calloc (1, sizeof (*w))))
         return NULL;
     w->h = h;
     w->inactivity_timeout = inactivity_timeout;
-    if (!(w->timer = flux_timer_watcher_create (r, inactivity_timeout,
-                                                0., worker_timeout, w)))
+    if (!(w->timer = flux_timer_watcher_create (r,
+                                                inactivity_timeout,
+                                                0.,
+                                                worker_timeout,
+                                                w)))
         goto error;
     if (!(w->trash = zlist_new()))
         goto error;
-    if (!(w->name = strdup (basename (name))))
+    if (!(w->name = strdup (basename_simple (name))))
         goto error;
     if (!(w->queue = zlist_new ()))
         goto error;
-    if (!(w->cmd = flux_cmd_create (argc, argv, environ))) {
-        flux_log_error (h, "flux_cmd_create");
-        goto error;
-    }
-    if (flux_cmd_setcwd (w->cmd, getcwd (path, sizeof (path))) < 0) {
-        flux_log_error (h, "flux_cmd_setcwd");
-        goto error;
-    }
     return w;
 error:
     worker_destroy (w);
diff --git a/src/modules/job-ingest/worker.h b/src/modules/job-ingest/worker.h
index 94d822e7992b..45b6f9ae77eb 100644
--- a/src/modules/job-ingest/worker.h
+++ b/src/modules/job-ingest/worker.h
@@ -11,6 +11,7 @@
 #ifndef _JOB_INGEST_WORKER_H
 #define _JOB_INGEST_WORKER_H
 
+#include <sys/types.h>
 #include <flux/core.h>
 
 #include "types.h"
@@ -21,14 +22,29 @@ struct worker;
 flux_future_t *worker_request (struct worker *w, const char *s);
 
 int worker_queue_depth (struct worker *w);
+int worker_request_count (struct worker *w);
+int worker_error_count (struct worker *w);
+int worker_trash_count (struct worker *w);
 bool worker_is_running (struct worker *w);
+pid_t worker_pid (struct worker *w);
 
 flux_future_t *worker_kill (struct worker *w, int signo);
 void worker_destroy (struct worker *w);
-struct worker *worker_create (flux_t *h, double inactivity_timeout,
-                              const char *worker_name,
-                              int argc, char **argv);
 
+struct worker *worker_create (flux_t *h,
+                              double inactivity_timeout,
+                              const char *worker_name);
+
+/*  (re)set cmdline for worker `w`. The new command will be used the
+ *   next time the worker starts.
+ */
+int worker_set_cmdline (struct worker *w, int argc, char **argv);
+
+/*  (re)set stdin buffer size for worker `w`.
+ *  `bufsize` may be a floating point value with optional scale suffix:
+ *  'kKMG'
+ */
+int worker_set_bufsize (struct worker *w, const char *bufsize);
 
 /* Tell worker to stop.
  * Return a count of running processes.
diff --git a/src/modules/job-list/Makefile.am b/src/modules/job-list/Makefile.am
new file mode 100644
index 000000000000..800fac39f7d7
--- /dev/null
+++ b/src/modules/job-list/Makefile.am
@@ -0,0 +1,144 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux \
+	$(FLUX_SECURITY_CFLAGS) \
+	$(JANSSON_CFLAGS)
+
+noinst_LTLIBRARIES = libjob-list.la
+
+libjob_list_la_SOURCES = \
+	job-list.c \
+	job-list.h \
+	job_state.h \
+	job_state.c \
+	job_data.h \
+	job_data.c \
+	list.h \
+	list.c \
+	job_util.h \
+	job_util.c \
+	idsync.h \
+	idsync.c \
+	stats.h \
+	stats.c \
+	match.h \
+	match.c \
+	state_match.h \
+	state_match.c \
+	match_util.h \
+	match_util.c
+
+TESTS = \
+	test_job_data.t \
+	test_match.t \
+	test_state_match.t
+
+test_ldadd = \
+	$(builddir)/libjob-list.la \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(top_builddir)/src/common/librlist/librlist.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(JANSSON_LIBS)
+
+test_cppflags = \
+	$(AM_CPPFLAGS)
+
+test_ldflags = \
+	-no-install
+
+check_PROGRAMS = $(TESTS)
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+       $(top_srcdir)/config/tap-driver.sh
+
+test_job_data_t_SOURCES = test/job_data.c
+test_job_data_t_CPPFLAGS = \
+	-DTEST_SRCDIR=\"$(top_srcdir)/src/modules/job-list/test\" \
+	$(test_cppflags)
+test_job_data_t_LDADD = \
+	$(test_ldadd)
+test_job_data_t_LDFLAGS = \
+	$(test_ldflags)
+
+test_match_t_SOURCES = test/match.c
+test_match_t_CPPFLAGS = \
+	$(test_cppflags)
+test_match_t_LDADD = \
+	$(test_ldadd)
+test_match_t_LDFLAGS = \
+	$(test_ldflags)
+
+test_state_match_t_SOURCES = test/state_match.c
+test_state_match_t_CPPFLAGS = \
+	$(test_cppflags)
+test_state_match_t_LDADD = \
+	$(test_ldadd)
+test_state_match_t_LDFLAGS = \
+	$(test_ldflags)
+
+EXTRA_DIST = \
+	test/R/1node_1core.R \
+	test/R/1node_4core.R \
+	test/R/4node_1core.R \
+	test/R/4node_4core.R \
+	test/R/invalid_R_lite.R \
+	test/R/invalid_json.R \
+	test/R/invalid_nodelist.R \
+	test/R/invalid_version.R \
+	test/R/missing_R_lite.R \
+	test/R/missing_expiration.R \
+	test/R/missing_nodelist.R \
+	test/R/missing_starttime.R \
+	test/R/missing_version.R \
+	test/jobspec/1node.jobspec \
+	test/jobspec/1node_1slot_nonexclusive.jobspec \
+	test/jobspec/1node_perresourcecore4.jobspec \
+	test/jobspec/1node_perresourcenode4.jobspec \
+	test/jobspec/1slot.jobspec \
+	test/jobspec/1slot_4core.jobspec \
+	test/jobspec/1slot_perresourcecore4.jobspec \
+	test/jobspec/1slot_project_bank.jobspec \
+	test/jobspec/4node.jobspec \
+	test/jobspec/4node_1slot_nonexclusive.jobspec \
+	test/jobspec/4node_4slot_nonexclusive.jobspec \
+	test/jobspec/4node_perresourcecore4.jobspec \
+	test/jobspec/4node_perresourcenode4.jobspec \
+	test/jobspec/4slot.jobspec \
+	test/jobspec/4slot_perresourcecore4.jobspec \
+	test/jobspec/duration_alt.jobspec \
+	test/jobspec/queue_specified.jobspec \
+	test/jobspec/cwd_not_specified.jobspec \
+	test/jobspec/invalid_attributes_system_job.jobspec \
+	test/jobspec/invalid_attributes_system_missing_duration.jobspec \
+	test/jobspec/invalid_command_array.jobspec \
+	test/jobspec/invalid_command_string.jobspec \
+	test/jobspec/invalid_json.jobspec \
+	test/jobspec/invalid_per_resource_missing_type.jobspec \
+	test/jobspec/invalid_resources.jobspec \
+	test/jobspec/invalid_resources_invalid_count.jobspec \
+	test/jobspec/invalid_resources_invalid_type.jobspec \
+	test/jobspec/invalid_resources_missing_count.jobspec \
+	test/jobspec/invalid_resources_missing_type.jobspec \
+	test/jobspec/invalid_resources_nocores.jobspec \
+	test/jobspec/invalid_resources_noslots.jobspec \
+	test/jobspec/invalid_tasks_array.jobspec \
+	test/jobspec/invalid_tasks_missing_command.jobspec \
+	test/jobspec/invalid_version.jobspec \
+	test/jobspec/job_name_alt.jobspec \
+	test/jobspec/missing_attributes.jobspec \
+	test/jobspec/missing_resources.jobspec \
+	test/jobspec/missing_tasks.jobspec \
+	test/jobspec/missing_version.jobspec
diff --git a/src/modules/job-list/idsync.c b/src/modules/job-list/idsync.c
new file mode 100644
index 000000000000..5cf417d63033
--- /dev/null
+++ b/src/modules/job-list/idsync.c
@@ -0,0 +1,300 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* idsync.c - code to sync job ids if job-list not yet aware of them */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libjob/job_hash.h"
+
+#include "idsync.h"
+#include "job_util.h"
+
+/* Used in waits hash, need to store job id within data structure for lookup */
+struct idsync_wait_list {
+    zlistx_t *l;
+    flux_jobid_t id;
+};
+
+void idsync_data_destroy (void *data)
+{
+    if (data) {
+        struct idsync_data *isd = data;
+        int save_errno = errno;
+        flux_msg_destroy (isd->msg);
+        json_decref (isd->attrs);
+        flux_future_destroy (isd->f_lookup);
+        free (isd);
+        errno = save_errno;
+    }
+}
+
+/* czmq_destructor */
+static void idsync_data_destroy_wrapper (void **data)
+{
+    if (data) {
+        idsync_data_destroy (*data);
+        *data = NULL;
+    }
+}
+
+static struct idsync_data *idsync_data_create (flux_t *h,
+                                               flux_jobid_t id,
+                                               const flux_msg_t *msg,
+                                               json_t *attrs,
+                                               flux_job_state_t state,
+                                               flux_future_t *f_lookup)
+{
+    struct idsync_data *isd = NULL;
+
+    isd = calloc (1, sizeof (*isd));
+    if (!isd)
+        goto error_enomem;
+    isd->h = h;
+    isd->id = id;
+    if (!(isd->msg = flux_msg_copy (msg, false)))
+        goto error;
+    isd->attrs = json_incref (attrs);
+    isd->state = state;
+    isd->f_lookup = f_lookup;
+    return isd;
+
+ error_enomem:
+    errno = ENOMEM;
+ error:
+    idsync_data_destroy (isd);
+    return NULL;
+}
+
+static void idsync_wait_list_destroy (void **data)
+{
+    if (data) {
+        struct idsync_wait_list *iwl = *data;
+        if (iwl) {
+            zlistx_destroy (&iwl->l);
+            free (iwl);
+        }
+        *data = NULL;
+    }
+}
+
+struct idsync_ctx *idsync_ctx_create (flux_t *h)
+{
+    struct idsync_ctx *isctx = NULL;
+    int saved_errno;
+
+    if (!(isctx = calloc (1, sizeof (*isctx))))
+        return NULL;
+    isctx->h = h;
+
+    if (!(isctx->lookups = zlistx_new ()))
+        goto error;
+
+    zlistx_set_destructor (isctx->lookups, idsync_data_destroy_wrapper);
+
+    if (!(isctx->waits = job_hash_create ()))
+        goto error;
+
+    zhashx_set_destructor (isctx->waits, idsync_wait_list_destroy);
+
+    return isctx;
+
+error:
+    saved_errno = errno;
+    idsync_ctx_destroy (isctx);
+    errno = saved_errno;
+    return NULL;
+}
+
+void idsync_ctx_destroy (struct idsync_ctx *isctx)
+{
+    if (isctx) {
+        struct idsync_data *isd;
+        isd = zlistx_first (isctx->lookups);
+        while (isd) {
+            if (isd->f_lookup) {
+                if (flux_future_get (isd->f_lookup, NULL) < 0)
+                    flux_log_error (isctx->h, "%s: flux_future_get",
+                                    __FUNCTION__);
+            }
+            isd = zlistx_next (isctx->lookups);
+        }
+        zlistx_destroy (&isctx->lookups);
+        zhashx_destroy (&isctx->waits);
+        free (isctx);
+    }
+}
+
+struct idsync_data *idsync_check_id_valid (struct idsync_ctx *isctx,
+                                           flux_jobid_t id,
+                                           const flux_msg_t *msg,
+                                           json_t *attrs,
+                                           flux_job_state_t state)
+{
+    flux_future_t *f = NULL;
+    struct idsync_data *isd = NULL;
+    char path[256];
+
+    /* Check to see if the ID is legal, job-list may have not yet
+     * seen the ID publication yet */
+    if (flux_job_kvs_key (path, sizeof (path), id, NULL) < 0)
+        goto error;
+
+    if (!(f = flux_kvs_lookup (isctx->h, NULL, FLUX_KVS_READDIR, path))) {
+        flux_log_error (isctx->h, "%s: flux_kvs_lookup", __FUNCTION__);
+        goto error;
+    }
+
+    if (!(isd = idsync_data_create (isctx->h, id, msg, attrs, state, f)))
+        goto error;
+
+    /* future now owned by struct idsync_data */
+    f = NULL;
+
+    if (!zlistx_add_end (isctx->lookups, isd)) {
+        flux_log (isctx->h, LOG_ERR, "%s: zlistx_add_end", __FUNCTION__);
+        errno = ENOMEM;
+        goto error;
+    }
+
+    return isd;
+
+error:
+    flux_future_destroy (f);
+    idsync_data_destroy (isd);
+    return NULL;
+}
+
+void idsync_check_id_valid_cleanup (struct idsync_ctx *isctx,
+                                    struct idsync_data *isd)
+{
+    /* delete will destroy struct idsync_data and future within it */
+    void *handle = zlistx_find (isctx->lookups, isd);
+    if (handle)
+        zlistx_delete (isctx->lookups, handle);
+}
+
+static int idsync_add_waiter (struct idsync_ctx *isctx,
+                              struct idsync_data *isd)
+{
+    struct idsync_wait_list *iwl = NULL;
+
+    /* isctx->waits holds lists of ids waiting on, b/c multiple callers
+     * could wait on same id */
+    if (!(iwl = zhashx_lookup (isctx->waits, &isd->id))) {
+        iwl = calloc (1, sizeof (*iwl));
+        if (!iwl)
+            goto enomem;
+
+        if (!(iwl->l = zlistx_new ()))
+            goto enomem;
+        zlistx_set_destructor (iwl->l, idsync_data_destroy_wrapper);
+        iwl->id = isd->id;
+
+        (void)zhashx_insert (isctx->waits, &iwl->id, iwl);
+    }
+
+    if (!zlistx_add_end (iwl->l, isd))
+        goto enomem;
+
+    return 0;
+
+enomem:
+    idsync_wait_list_destroy ((void **)&iwl);
+    errno = ENOMEM;
+    return -1;
+}
+
+int idsync_wait_valid (struct idsync_ctx *isctx, struct idsync_data *isd)
+{
+    void *handle;
+
+    /* make sure isd isn't on lookups list, if so remove it */
+    if ((handle = zlistx_find (isctx->lookups, isd))) {
+        /* detach will not call zlistx destructor */
+        zlistx_detach (isctx->lookups, handle);
+    }
+
+    return idsync_add_waiter (isctx, isd);
+}
+
+
+int idsync_wait_valid_id (struct idsync_ctx *isctx,
+                          flux_jobid_t id,
+                          const flux_msg_t *msg,
+                          json_t *attrs,
+                          flux_job_state_t state)
+{
+    struct idsync_data *isd = NULL;
+
+    if (!(isd = idsync_data_create (isctx->h, id, msg, attrs, state, NULL)))
+        return -1;
+
+    return idsync_add_waiter (isctx, isd);
+}
+
+static void idsync_data_respond (struct idsync_ctx *isctx,
+                                 struct idsync_data *isd,
+                                 struct job *job)
+{
+    flux_error_t err;
+    json_t *o;
+
+    if (!(o = job_to_json (job, isd->attrs, &err)))
+        goto error;
+
+    if (flux_respond_pack (isctx->h, isd->msg, "{s:O}", "job", o) < 0)
+        flux_log_error (isctx->h, "%s: flux_respond_pack", __FUNCTION__);
+
+    json_decref (o);
+    return;
+
+error:
+    if (flux_respond_error (isctx->h, isd->msg, errno, err.text) < 0)
+        flux_log_error (isctx->h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+void idsync_check_waiting_id (struct idsync_ctx *isctx, struct job *job)
+{
+    struct idsync_wait_list *iwl;
+
+    if ((iwl = zhashx_lookup (isctx->waits, &job->id))) {
+        struct idsync_data *isd;
+        isd = zlistx_first (iwl->l);
+        while (isd) {
+            /* Some job states can be missed.  For example a job that
+             * is canceled before it runs will never reach the
+             * FLUX_JOB_STATE_RUN state.  To ensure jobs waiting on
+             * states that are missed will eventually get a response, always
+             * respond once the job has reached the inactive state.
+             */
+            if (!isd->state
+                || (isd->state & job->states_mask)
+                || (isd->state && job->state == FLUX_JOB_STATE_INACTIVE)) {
+                struct idsync_data *tmp;
+                idsync_data_respond (isctx, isd, job);
+                tmp = zlistx_detach_cur (iwl->l);
+                idsync_data_destroy (tmp);
+            }
+            isd = zlistx_next (iwl->l);
+        }
+        if (!zlistx_size (iwl->l))
+            zhashx_delete (isctx->waits, &job->id);
+    }
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/idsync.h b/src/modules/job-list/idsync.h
new file mode 100644
index 000000000000..4adc24e9bb5e
--- /dev/null
+++ b/src/modules/job-list/idsync.h
@@ -0,0 +1,81 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_LIST_IDSYNC_H
+#define _FLUX_JOB_LIST_IDSYNC_H
+
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "job_data.h"
+
+struct idsync_ctx {
+    flux_t *h;
+    zlistx_t *lookups;
+    zhashx_t *waits;
+};
+
+struct idsync_data {
+    flux_t *h;
+    flux_jobid_t id;
+    flux_msg_t *msg;
+    json_t *attrs;
+    flux_job_state_t state;
+
+    flux_future_t *f_lookup;
+};
+
+struct idsync_ctx *idsync_ctx_create (flux_t *h);
+
+void idsync_ctx_destroy (struct idsync_ctx *isctx);
+
+void idsync_data_destroy (void *data);
+
+/* lookup id in KVS to check if it is valid, futures will be tracked /
+ * managed in lookups list.  Future returned in idsync_data pointer
+ * under 'f_lookup'.
+ */
+struct idsync_data *idsync_check_id_valid (struct idsync_ctx *isctx,
+                                           flux_jobid_t id,
+                                           const flux_msg_t *msg,
+                                           json_t *attrs,
+                                           flux_job_state_t state);
+
+
+/* free / cleanup 'struct idsync_data' after
+ * idsync_check_id_valid().  Don't call this if you re-use
+ * 'struct idsync_data' with idsync_wait_valid().
+ */
+void idsync_check_id_valid_cleanup (struct idsync_ctx *isctx,
+                                    struct idsync_data *isd);
+
+/* idsync_wait_valid*() add to job id waits hash, waiting for id to be
+ * legal in job-list.  idsync_check_waiting_id() will be able to
+ * respond to message at later time when job id becomes available.
+ */
+
+int idsync_wait_valid (struct idsync_ctx *isctx, struct idsync_data *isd);
+
+int idsync_wait_valid_id (struct idsync_ctx *isctx,
+                          flux_jobid_t id,
+                          const flux_msg_t *msg,
+                          json_t *attrs,
+                          flux_job_state_t state);
+
+/* check if 'job' is in waits list, if so respond to original
+ * message */
+void idsync_check_waiting_id (struct idsync_ctx *isctx, struct job *job);
+
+#endif /* ! _FLUX_JOB_LIST_IDSYNC_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/job-list.c b/src/modules/job-list/job-list.c
new file mode 100644
index 000000000000..476553b9e4f3
--- /dev/null
+++ b/src/modules/job-list/job-list.c
@@ -0,0 +1,265 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "job-list.h"
+#include "job_state.h"
+#include "job_data.h"
+#include "list.h"
+#include "idsync.h"
+#include "stats.h"
+
+static const char *attrs[] = {
+    "userid", "urgency", "priority", "t_submit",
+    "t_depend", "t_run", "t_cleanup", "t_inactive",
+    "state", "name", "cwd", "queue", "project", "bank",
+    "ntasks", "ncores", "duration", "nnodes",
+    "ranks", "nodelist", "success", "exception_occurred",
+    "exception_type", "exception_severity",
+    "exception_note", "result", "expiration",
+    "annotations", "waitstatus", "dependencies",
+    NULL
+};
+
+const char **job_attrs (void)
+{
+    return attrs;
+}
+
+static void stats_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
+{
+    struct list_ctx *ctx = arg;
+
+    if (!ctx->jsctx->initialized) {
+        if (flux_msglist_append (ctx->deferred_requests, msg) < 0)
+            goto error;
+        return;
+    }
+
+    int pending = zlistx_size (ctx->jsctx->pending);
+    int running = zlistx_size (ctx->jsctx->running);
+    int inactive = zlistx_size (ctx->jsctx->inactive);
+    int idsync_lookups = zlistx_size (ctx->isctx->lookups);
+    int idsync_waits = zhashx_size (ctx->isctx->waits);
+    int stats_watchers = job_stats_watchers (ctx->jsctx->statsctx);
+    if (flux_respond_pack (h, msg, "{s:{s:i s:i s:i} s:{s:i s:i} s:i}",
+                           "jobs",
+                           "pending", pending,
+                           "running", running,
+                           "inactive", inactive,
+                           "idsync",
+                           "lookups", idsync_lookups,
+                           "waits", idsync_waits,
+                           "stats_watchers", stats_watchers) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+}
+
+static void purge_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
+{
+    struct list_ctx *ctx = arg;
+    json_t *jobs;
+    size_t index;
+    json_t *entry;
+    int count = 0;
+
+    if (flux_event_unpack (msg, NULL, "{s:o}", "jobs", &jobs) < 0)
+        flux_log_error (h, "job-purge-inactive message");
+    json_array_foreach (jobs, index, entry) {
+        flux_jobid_t id = json_integer_value (entry);
+        struct job *job;
+
+        if ((job = zhashx_lookup (ctx->jsctx->index, &id))) {
+            if (job->state != FLUX_JOB_STATE_INACTIVE)
+                continue;
+            job_stats_purge (ctx->jsctx->statsctx, job);
+            if (job->list_handle)
+                zlistx_delete (ctx->jsctx->inactive, job->list_handle);
+            zhashx_delete (ctx->jsctx->index, &id);
+            count++;
+        }
+    }
+    flux_log (h, LOG_DEBUG, "purged %d inactive jobs", count);
+}
+
+void requeue_deferred_requests (struct list_ctx *ctx)
+{
+    const flux_msg_t *msg;
+
+    while ((msg = flux_msglist_pop (ctx->deferred_requests))) {
+        if (flux_requeue (ctx->h, msg, FLUX_RQ_TAIL) < 0)
+            flux_log_error (ctx->h, "error requeuing deferred request");
+        flux_msg_decref (msg);
+    }
+}
+
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct list_ctx *ctx = arg;
+    job_stats_disconnect (ctx->jsctx->statsctx, msg);
+}
+
+static void config_reload_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct list_ctx *ctx = arg;
+    const flux_conf_t *conf;
+    flux_error_t error;
+    const char *errstr = NULL;
+
+    if (flux_conf_reload_decode (msg, &conf) < 0)
+        goto error;
+    if (job_state_config_reload (ctx->jsctx, conf, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (job_match_config_reload (ctx->mctx, conf, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { .typemask     = FLUX_MSGTYPE_REQUEST,
+      .topic_glob   = "job-list.list",
+      .cb           = list_cb,
+      .rolemask     = FLUX_ROLE_USER
+    },
+    { .typemask     = FLUX_MSGTYPE_REQUEST,
+      .topic_glob   = "job-list.list-id",
+      .cb           = list_id_cb,
+      .rolemask     = FLUX_ROLE_USER
+    },
+    { .typemask     = FLUX_MSGTYPE_REQUEST,
+      .topic_glob   = "job-list.list-attrs",
+      .cb           = list_attrs_cb,
+      .rolemask     = FLUX_ROLE_USER
+    },
+    { .typemask     = FLUX_MSGTYPE_REQUEST,
+      .topic_glob   = "job-list.job-state-pause",
+      .cb           = job_state_pause_cb,
+      .rolemask     = FLUX_ROLE_USER
+    },
+    { .typemask     = FLUX_MSGTYPE_REQUEST,
+      .topic_glob   = "job-list.job-state-unpause",
+      .cb           = job_state_unpause_cb,
+      .rolemask     = FLUX_ROLE_USER
+    },
+    { .typemask     = FLUX_MSGTYPE_REQUEST,
+      .topic_glob   = "job-list.stats-get",
+      .cb           = stats_cb,
+      .rolemask     = FLUX_ROLE_USER,
+    },
+    { .typemask     = FLUX_MSGTYPE_EVENT,
+      .topic_glob   = "job-purge-inactive",
+      .cb           = purge_cb,
+      .rolemask     = 0
+    },
+    { .typemask     = FLUX_MSGTYPE_REQUEST,
+      .topic_glob   = "job-list.disconnect",
+      .cb           = disconnect_cb,
+      .rolemask     = FLUX_ROLE_USER,
+    },
+    {
+      .typemask     = FLUX_MSGTYPE_REQUEST,
+      .topic_glob   = "job-list.config-reload",
+      .cb           = config_reload_cb,
+      .rolemask     = 0
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+static void list_ctx_destroy (struct list_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (ctx->handlers);
+        flux_msglist_destroy (ctx->deferred_requests);
+        if (ctx->jsctx)
+            job_state_destroy (ctx->jsctx);
+        if (ctx->isctx)
+            idsync_ctx_destroy (ctx->isctx);
+        if (ctx->mctx)
+            match_ctx_destroy (ctx->mctx);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static struct list_ctx *list_ctx_create (flux_t *h)
+{
+    struct list_ctx *ctx = calloc (1, sizeof (*ctx));
+    if (!ctx)
+        return NULL;
+    ctx->h = h;
+    if (flux_event_subscribe (h, "job-purge-inactive") < 0)
+        goto error;
+    if (flux_msg_handler_addvec (h, htab, ctx, &ctx->handlers) < 0)
+        goto error;
+    if (!(ctx->isctx = idsync_ctx_create (ctx->h)))
+        goto error;
+    if (!(ctx->jsctx = job_state_create (ctx)))
+        goto error;
+    if (!(ctx->deferred_requests = flux_msglist_create ()))
+        goto error;
+    if (!(ctx->mctx = match_ctx_create (ctx->h)))
+        goto error;
+    return ctx;
+error:
+    list_ctx_destroy (ctx);
+    return NULL;
+}
+
+int mod_main (flux_t *h, int argc, char **argv)
+{
+    struct list_ctx *ctx;
+    int rc = -1;
+
+    if (!(ctx = list_ctx_create (h))) {
+        flux_log_error (h, "initialization error");
+        goto done;
+    }
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        goto done;
+    rc = 0;
+done:
+    list_ctx_destroy (ctx);
+    return rc;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/job-list.h b/src/modules/job-list/job-list.h
new file mode 100644
index 000000000000..6a239275171d
--- /dev/null
+++ b/src/modules/job-list/job-list.h
@@ -0,0 +1,40 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_LIST_H
+#define _FLUX_JOB_LIST_H
+
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "job_state.h"
+#include "idsync.h"
+#include "match.h"
+
+struct list_ctx {
+    flux_t *h;
+    flux_msg_handler_t **handlers;
+    struct job_state_ctx *jsctx;
+    struct idsync_ctx *isctx;
+    struct flux_msglist *deferred_requests;
+    struct match_ctx *mctx;
+};
+
+const char **job_attrs (void);
+
+void requeue_deferred_requests (struct list_ctx *ctx);
+
+#endif /* _FLUX_JOB_LIST_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/modules/job-list/job_data.c b/src/modules/job-list/job_data.c
new file mode 100644
index 000000000000..733d2f8e5963
--- /dev/null
+++ b/src/modules/job-list/job_data.c
@@ -0,0 +1,551 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* job_data.c - primary struct job helper functions */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <assert.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/librlist/rlist.h"
+#include "src/common/librlist/rnode.h"
+#include "src/common/libccan/ccan/str/str.h"
+#include "src/common/libjob/jj.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libutil/jpath.h"
+#include "ccan/str/str.h"
+
+#include "job_data.h"
+
+void job_destroy (void *data)
+{
+    struct job *job = data;
+    if (job) {
+        int save_errno = errno;
+        free (job->ranks);
+        free (job->nodelist);
+        hostlist_destroy (job->nodelist_hl);
+        idset_destroy (job->ranks_idset);
+        json_decref (job->annotations);
+        grudgeset_destroy (job->dependencies);
+        json_decref (job->jobspec);
+        json_decref (job->R);
+        json_decref (job->exception_context);
+        free (job);
+        errno = save_errno;
+    }
+}
+
+struct job *job_create (flux_t *h, flux_jobid_t id)
+{
+    struct job *job = NULL;
+
+    if (!(job = calloc (1, sizeof (*job))))
+        return NULL;
+    job->h = h;
+    job->id = id;
+    job->userid = FLUX_USERID_UNKNOWN;
+    job->urgency = -1;
+    /* pending jobs that are not yet assigned a priority shall be
+     * listed after those who do, so we set the job priority to MIN */
+    job->priority = FLUX_JOB_PRIORITY_MIN;
+    job->state = FLUX_JOB_STATE_NEW;
+    job->ntasks = -1;
+    job->ncores = -1;
+    job->duration = -1.0;
+    job->nnodes = -1;
+    job->expiration = -1.0;
+    job->wait_status = -1;
+    job->result = FLUX_JOB_RESULT_FAILED;
+    job->states_mask = FLUX_JOB_STATE_NEW;
+    job->states_events_mask = FLUX_JOB_STATE_NEW;
+    return job;
+}
+
+/* Return basename of path if there is a '/' in path.  Otherwise return
+ * full path */
+static const char *parse_job_name (const char *path)
+{
+    char *p = strrchr (path, '/');
+    if (p) {
+        p++;
+        /* user mistake, specified a directory with trailing '/',
+         * return full path */
+        if (*p == '\0')
+            return path;
+        return p;
+    }
+    return path;
+}
+
+static int parse_jobspec_job_name (struct job *job,
+                                   json_t *jobspec_job)
+{
+    json_error_t error;
+
+    if (jobspec_job) {
+        if (json_unpack_ex (jobspec_job, &error, 0,
+                            "{s?s}",
+                            "name", &job->name) < 0) {
+            flux_log (job->h, LOG_ERR,
+                      "%s: job %s invalid job dictionary: %s",
+                      __FUNCTION__, idf58 (job->id), error.text);
+            return -1;
+        }
+    }
+    else
+        job->name = NULL;
+
+    /* If user did not specify job.name, we treat arg 0 of the command
+     * as the job name */
+    if (!job->name) {
+        json_t *command = NULL;
+        json_t *arg0;
+        json_t *tasks;
+
+        if (json_unpack_ex (job->jobspec, &error, 0,
+                            "{s:o}",
+                            "tasks", &tasks) < 0) {
+            flux_log (job->h, LOG_ERR,
+                      "%s: job %s invalid jobspec: %s",
+                      __FUNCTION__, idf58 (job->id), error.text);
+            return -1;
+        }
+
+        if (json_unpack_ex (tasks, &error, 0,
+                            "[{s:o}]",
+                            "command", &command) < 0) {
+            flux_log (job->h, LOG_ERR,
+                      "%s: job %s invalid jobspec: %s",
+                      __FUNCTION__, idf58 (job->id), error.text);
+            return -1;
+        }
+
+        if (!json_is_array (command)) {
+            flux_log (job->h, LOG_ERR,
+                      "%s: job %s invalid jobspec",
+                      __FUNCTION__, idf58 (job->id));
+            return -1;
+        }
+
+        arg0 = json_array_get (command, 0);
+        if (!arg0 || !json_is_string (arg0)) {
+            flux_log (job->h, LOG_ERR,
+                      "%s: job %s invalid job command",
+                      __FUNCTION__, idf58 (job->id));
+            return -1;
+        }
+        job->name = parse_job_name (json_string_value (arg0));
+        assert (job->name);
+    }
+
+    return 0;
+}
+
+static int parse_attributes_dict (struct job *job)
+{
+    json_error_t error;
+    json_t *jobspec_job = NULL;
+
+    if (json_unpack_ex (job->jobspec, &error, 0,
+                        "{s:{s?{s?o}}}",
+                        "attributes",
+                        "system",
+                        "job",
+                        &jobspec_job) < 0) {
+        flux_log (job->h, LOG_ERR,
+                  "%s: job %s invalid jobspec: %s",
+                  __FUNCTION__, idf58 (job->id), error.text);
+        return -1;
+    }
+
+    if (jobspec_job) {
+        if (!json_is_object (jobspec_job)) {
+            flux_log (job->h, LOG_ERR,
+                      "%s: job %s invalid jobspec",
+                      __FUNCTION__, idf58 (job->id));
+            return -1;
+        }
+    }
+
+    if (parse_jobspec_job_name (job, jobspec_job) < 0)
+        return -1;
+
+    /* N.B. attributes.system.duration is required in jobspec version 1 */
+    /* N.B. cwd & queue are optional, reset to NULL before parse in case
+     * not listed
+     */
+    job->cwd = NULL;
+    job->queue = NULL;
+    if (json_unpack_ex (job->jobspec, &error, 0,
+                        "{s:{s?{s?s s?s s:F s?s s?s}}}",
+                        "attributes",
+                        "system",
+                        "cwd", &job->cwd,
+                        "queue", &job->queue,
+                        "duration", &job->duration,
+                        "project", &job->project,
+                        "bank", &job->bank) < 0) {
+        flux_log (job->h, LOG_ERR,
+                  "%s: job %s invalid jobspec: %s",
+                  __FUNCTION__, idf58 (job->id), error.text);
+        return -1;
+    }
+
+    return 0;
+}
+
+static int parse_jobspec_nnodes (struct job *job, struct jj_counts *jj)
+{
+    /* Set job->nnodes if it is available, otherwise it will be set
+     * later when R is available.
+     */
+    if (jj->nnodes > 0)
+        job->nnodes = jj->nnodes;
+    else
+        job->nnodes = -1;
+
+    return 0;
+}
+
+static int parse_per_resource (struct job *job,
+                               const char **type,
+                               int *count)
+{
+    json_error_t error;
+    json_t *o = NULL;
+
+    if (json_unpack_ex (job->jobspec, &error, 0,
+                        "{s:{s?{s?{s?{s?o}}}}}",
+                        "attributes",
+                          "system",
+                            "shell",
+                              "options",
+                                "per-resource", &o) < 0) {
+        flux_log (job->h, LOG_ERR,
+                  "%s: job %s invalid jobspec: %s",
+                  __FUNCTION__, idf58 (job->id), error.text);
+        return -1;
+    }
+
+    (*count) = 1;
+    if (o) {
+        if (json_unpack_ex (o, &error, 0,
+                            "{s:s s?i}",
+                            "type", type,
+                            "count", count) < 0) {
+            flux_log (job->h, LOG_ERR,
+                      "%s: job %s invalid per-resource spec: %s",
+                      __FUNCTION__, idf58 (job->id), error.text);
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+static int parse_jobspec_ntasks (struct job *job, struct jj_counts *jj)
+{
+    const char *type = NULL;
+    int count = 0;
+
+    /* per-resource is used to overcome short-term gaps in
+     * Jobspec V1.  Remove per-resource logic below when it
+     * has been retired
+     */
+
+    if (parse_per_resource (job, &type, &count) < 0)
+        return -1;
+
+    if (type && count > 0) {
+        /* if per-resource type == nodes and nodes specified
+         * (node->slot->core), this is a special case of ntasks.
+         */
+        if (streq (type, "node") && jj->nnodes > 0) {
+            job->ntasks = jj->nnodes * count;
+            return 0;
+        }
+        if (streq (type, "core")) {
+            if (jj->nnodes == 0)
+                job->ntasks = jj->nslots * jj->slot_size * count;
+            else {
+                /* if nnodes > 0, can't determine until nodes
+                 * allocated and number of cores on node(s) are known.
+                 * Set a flag / count to retrieve data later when
+                 * R has been retrieved.
+                 */
+                job->ntasks_per_core_on_node_count = count;
+                job->ntasks = -1;
+            }
+            return 0;
+        }
+    }
+
+    if (json_unpack_ex (job->jobspec, NULL, 0,
+                        "{s:[{s:{s:i}}]}",
+                        "tasks",
+                        "count",
+                        "total", &job->ntasks) < 0)
+        job->ntasks = jj->nslots;
+    return 0;
+}
+
+static int parse_jobspec_ncores (struct job *job, struct jj_counts *jj)
+{
+    /* number of cores can't be determined yet, calculate later when R
+     * is parsed */
+    if (jj->nnodes > 0 && jj->exclusive) {
+        job->ncores = -1;
+        return 0;
+    }
+
+    /* nslots already accounts for nnodes if available */
+    job->ncores = jj->nslots * jj->slot_size;
+    return 0;
+}
+
+static int load_jobspec (struct job *job, const char *s, bool allow_nonfatal)
+{
+    json_error_t error;
+
+    if (!(job->jobspec = json_loads (s, 0, &error))) {
+        flux_log (job->h, LOG_ERR,
+                  "%s: job %s invalid jobspec: %s",
+                  __FUNCTION__, idf58 (job->id), error.text);
+        return allow_nonfatal ? 0 : -1;
+    }
+    return 0;
+}
+
+static int parse_jobspec (struct job *job, bool allow_nonfatal)
+{
+    struct jj_counts jj;
+
+    if (parse_attributes_dict (job) < 0)
+        goto nonfatal_error;
+
+    if (jj_get_counts_json (job->jobspec, &jj) < 0) {
+        flux_log (job->h, LOG_ERR,
+                  "%s: job %s invalid jobspec; %s",
+                  __FUNCTION__, idf58 (job->id), jj.error);
+        goto nonfatal_error;
+    }
+
+    if (parse_jobspec_nnodes (job, &jj) < 0)
+        goto nonfatal_error;
+
+    if (parse_jobspec_ntasks (job, &jj) < 0)
+        goto nonfatal_error;
+
+    if (parse_jobspec_ncores (job, &jj) < 0)
+        goto nonfatal_error;
+
+    return 0;
+
+    /* nonfatal error - jobspec illegal, but we'll continue on.  job
+     * listing will return whatever data is available */
+nonfatal_error:
+    return allow_nonfatal ? 0 : -1;
+}
+
+int job_parse_jobspec_cached (struct job *job, json_t *updates)
+{
+    if (!job->jobspec) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (parse_jobspec (job, true) < 0)
+        return -1;
+    return job_jobspec_update (job, updates);
+}
+
+int job_parse_jobspec (struct job *job, const char *s, json_t *updates)
+{
+    if (load_jobspec (job, s, true) < 0)
+        return -1;
+    return job_parse_jobspec_cached (job, updates);
+}
+
+int job_parse_jobspec_fatal (struct job *job, const char *s, json_t *updates)
+{
+    if (load_jobspec (job, s, false) < 0)
+        return -1;
+    if (parse_jobspec (job, false) < 0)
+        return -1;
+    return job_jobspec_update (job, updates);
+}
+
+static int load_R (struct job *job, const char *s, bool allow_nonfatal)
+{
+    json_error_t error;
+
+    if (!(job->R = json_loads (s, 0, &error))) {
+        flux_log (job->h, LOG_ERR,
+                  "%s: job %s invalid R: %s",
+                  __FUNCTION__, idf58 (job->id), error.text);
+        return allow_nonfatal ? 0 : -1;
+    }
+    return 0;
+}
+
+static int parse_R (struct job *job, bool allow_nonfatal)
+{
+    struct rlist *rl = NULL;
+    struct idset *idset = NULL;
+    struct hostlist *hl = NULL;
+    json_error_t error;
+    int flags = IDSET_FLAG_BRACKETS | IDSET_FLAG_RANGE;
+    int core_count = 0;
+    struct rnode *rnode;
+    int saved_errno, rc = -1;
+    char *tmp;
+
+    if (!(rl = rlist_from_json (job->R, &error))) {
+        flux_log_error (job->h, "rlist_from_json: %s", error.text);
+        goto nonfatal_error;
+    }
+
+    job->expiration = rl->expiration;
+
+    if (!(idset = rlist_ranks (rl)))
+        goto nonfatal_error;
+
+    job->nnodes = idset_count (idset);
+    if (!(tmp = idset_encode (idset, flags)))
+        goto nonfatal_error;
+    free (job->ranks);
+    job->ranks = tmp;
+
+    /* reading nodelist from R directly would avoid the creation /
+     * destruction of a hostlist.  However, we get a hostlist to
+     * ensure that the nodelist we return to users is consistently
+     * formatted.
+     */
+    if (!(hl = rlist_nodelist (rl)))
+        goto nonfatal_error;
+
+    if (!(tmp = hostlist_encode (hl)))
+        goto nonfatal_error;
+    free (job->nodelist);
+    job->nodelist = tmp;
+
+    rnode = zlistx_first (rl->nodes);
+    while (rnode) {
+        core_count += idset_count (rnode->cores->ids);
+        rnode = zlistx_next (rl->nodes);
+    }
+    job->ncores = core_count;
+
+    if (job->ntasks_per_core_on_node_count > 0)
+        job->ntasks = core_count * job->ntasks_per_core_on_node_count;
+
+    rc = 0;
+    goto cleanup;
+
+    /* nonfatal error - invalid R, but we'll continue on.  job listing
+     * will get initialized data */
+nonfatal_error:
+    rc = allow_nonfatal ? 0 : -1;
+cleanup:
+    saved_errno = errno;
+    hostlist_destroy (hl);
+    idset_destroy (idset);
+    rlist_destroy (rl);
+    errno = saved_errno;
+    return rc;
+}
+
+int job_parse_R_cached (struct job *job, json_t *updates)
+{
+    if (!job->R) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (parse_R (job, true) < 0)
+        return -1;
+    return job_R_update (job, updates);
+}
+
+int job_parse_R (struct job *job, const char *s, json_t *updates)
+{
+    if (load_R (job, s, true) < 0)
+        return -1;
+    return job_parse_R_cached (job, updates);
+}
+
+int job_parse_R_fatal (struct job *job, const char *s, json_t *updates)
+{
+    if (load_R (job, s, false) < 0)
+        return -1;
+    if (parse_R (job, false) < 0)
+        return -1;
+    return job_R_update (job, updates);
+}
+
+int job_jobspec_update (struct job *job, json_t *updates)
+{
+    const char *key;
+    json_t *value;
+
+    if (!updates)
+        return 0;
+
+    /* To be on safe side, we should probably copy job->jobspec and
+     * only apply updates if they succeed and are parsed.  However, we
+     * don't do that given low odds of invalid updates ever happening.
+     */
+    json_object_foreach (updates, key, value) {
+        /* In jobspec V1 only valid keys in a jobspec are resources,
+         * tasks, and attributes
+         */
+        if ((!streq (key, "resources")
+             && !strstarts (key, "resources.")
+             && !streq (key, "tasks")
+             && !strstarts (key, "tasks.")
+             && !streq (key, "attributes")
+             && !strstarts (key, "attributes."))
+            || jpath_set (job->jobspec, key, value) < 0)
+            flux_log (job->h, LOG_INFO,
+                      "%s: job %s failed to update jobspec key %s",
+                      __FUNCTION__, idf58 (job->id), key);
+    }
+    return parse_jobspec (job, false);
+}
+
+int job_R_update (struct job *job, json_t *updates)
+{
+    const char *key;
+    json_t *value;
+
+    if (!updates)
+        return 0;
+
+    json_object_foreach (updates, key, value) {
+        /* RFC 21 resource-update event only allows update
+         * to:
+         * - expiration
+         */
+        if (streq (key, "expiration"))
+            if (jpath_set (job->R, "execution.expiration", value) < 0)
+                flux_log (job->h, LOG_INFO,
+                          "%s: job %s failed to update R key %s",
+                          __FUNCTION__, idf58 (job->id), key);
+    }
+
+    return parse_R (job, false);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/job_data.h b/src/modules/job-list/job_data.h
new file mode 100644
index 000000000000..4729e0d0431d
--- /dev/null
+++ b/src/modules/job-list/job_data.h
@@ -0,0 +1,140 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_LIST_JOB_DATA_H
+#define _FLUX_JOB_LIST_JOB_DATA_H
+
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libhostlist/hostlist.h"
+#include "src/common/libidset/idset.h"
+#include "src/common/libutil/grudgeset.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+/* timestamp of when we enter the state
+ *
+ * associated eventlog entries when restarting
+ *
+ * t_submit = "submit"
+ * t_depend - "validate"
+ * t_priority - "priority" (not saved, can be entered multiple times)
+ * t_sched - "depend" (not saved, can be entered multiple times)
+ * t_run - "alloc"
+ * t_cleanup - "finish" or "exception" w/ severity == 0
+ * t_inactive - "clean"
+ */
+struct job {
+    flux_t *h;
+
+    flux_jobid_t id;
+    uint32_t userid;
+    int urgency;
+    int64_t priority;
+    double t_submit;
+    double t_depend;
+    double t_run;
+    double t_cleanup;
+    double t_inactive;
+    flux_job_state_t state;
+    const char *name;
+    const char *queue;
+    const char *cwd;
+    const char *project;
+    const char *bank;
+    int ntasks;
+    int ntasks_per_core_on_node_count;  /* flag for ntasks calculation */
+    int ncores;
+    double duration;
+    int nnodes;
+    char *ranks;
+    char *nodelist;
+    struct hostlist *nodelist_hl; /* cache of nodelist in hl form */
+    struct idset *ranks_idset;    /* cache of ranks in idset form */
+    double expiration;
+    int wait_status;
+    bool success;
+    bool exception_occurred;
+    int exception_severity;
+    const char *exception_type;
+    const char *exception_note;
+    flux_job_result_t result;
+    json_t *annotations;
+    struct grudgeset *dependencies;
+
+    /* cache of job information */
+    json_t *jobspec;
+    json_t *R;
+    json_t *exception_context;
+
+    /* Track which states we have seen and have completed transition
+     * to.  States we've processed via the states_mask and states seen
+     * via events stream in states_events_mask.
+     */
+    unsigned int states_mask;
+    unsigned int states_events_mask;
+    void *list_handle;
+
+    int submit_version;         /* version number in submit context */
+};
+
+void job_destroy (void *data);
+
+struct job *job_create (flux_t *h, flux_jobid_t id);
+
+/* Parse and internally cache jobspec.  Set values for:
+ * - job name
+ * - queue
+ * - ntasks
+ * - nnodes (if available)
+ * - ncores (if possible)
+ * - duration
+ *
+ * Optionally pass in "updates", an object with path:value updates to
+ * the jobspec.
+ */
+int job_parse_jobspec (struct job *job, const char *s, json_t *updates);
+int job_parse_jobspec_cached (struct job *job, json_t *updates);
+
+/* identical to above, but all nonfatal errors will return error.
+ * Primarily used for testing.
+ */
+int job_parse_jobspec_fatal (struct job *job, const char *s, json_t *updates);
+
+/* Update jobspec with period delimited paths
+ * (i.e. "attributes.system.duration") and value.
+ */
+int job_jobspec_update (struct job *job, json_t *updates);
+
+/* Parse and internally cache R.  Set values for:
+ * - expiration
+ * - nnodes
+ * - nodelist
+ * - ncores
+ * - ntasks (if necessary)
+ */
+int job_parse_R (struct job *job, const char *s, json_t *updates);
+int job_parse_R_cached (struct job *job, json_t *updates);
+
+/* identical to above, but all nonfatal errors will return error.
+ * Primarily used for testing.
+ */
+int job_parse_R_fatal (struct job *job, const char *s, json_t *updates);
+
+/* Update R with RFC21 defined keys
+ * (i.e. "expiration") and value.
+ */
+int job_R_update (struct job *job, json_t *updates);
+
+#endif /* ! _FLUX_JOB_LIST_JOB_DATA_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/job_state.c b/src/modules/job-list/job_state.c
new file mode 100644
index 000000000000..2ce0519ef03d
--- /dev/null
+++ b/src/modules/job-list/job_state.c
@@ -0,0 +1,1194 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* job_state.c - store information on state of jobs */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <assert.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libutil/fluid.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/jpath.h"
+#include "src/common/libutil/grudgeset.h"
+#include "src/common/libjob/job_hash.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libidset/idset.h"
+#include "ccan/str/str.h"
+
+#include "job-list.h"
+#include "job_state.h"
+#include "job_data.h"
+#include "idsync.h"
+#include "job_util.h"
+
+#define NUMCMP(a,b) ((a)==(b)?0:((a)<(b)?-1:1))
+
+/* REVERT - flag indicates state transition is a revert, avoid certain
+ * checks, clear certain bitmasks on revert
+ *
+ * CONDITIONAL - flag indicates state transition is dependent on
+ * current state.
+ */
+#define STATE_TRANSITION_FLAG_REVERT 0x1
+#define STATE_TRANSITION_FLAG_CONDITIONAL 0x2
+
+static int submit_context_parse (flux_t *h,
+                                 struct job *job,
+                                 json_t *context);
+static int priority_context_parse (flux_t *h,
+                                   struct job *job,
+                                   json_t *context);
+static int finish_context_parse (flux_t *h,
+                                 struct job *job,
+                                 json_t *context);
+static int urgency_context_parse (flux_t *h,
+                                  struct job *job,
+                                  json_t *context);
+static int exception_context_parse (flux_t *h,
+                                    struct job *job,
+                                    json_t *context,
+                                    int *severityP);
+static int dependency_context_parse (flux_t *h,
+                                     struct job *job,
+                                     const char *cmd,
+                                     json_t *context);
+static int memo_update (flux_t *h,
+                        struct job *job,
+                        json_t *context);
+
+static int journal_process_events (struct job_state_ctx *jsctx,
+                                   const flux_msg_t *msg);
+
+static void update_jobspec (struct job_state_ctx *jsctx,
+                            struct job *job,
+                            json_t *context,
+                            bool update_stats);
+
+/* Compare items for sorting in list, priority first (higher priority
+ * before lower priority), job id second N.B. zlistx_comparator_fn signature
+ */
+static int job_urgency_cmp (const void *a1, const void *a2)
+{
+    const struct job *j1 = a1;
+    const struct job *j2 = a2;
+    int rc;
+
+    if ((rc = (-1)*NUMCMP (j1->priority, j2->priority)) == 0)
+        rc = NUMCMP (j1->id, j2->id);
+    return rc;
+}
+
+/* Compare items for sorting in list by timestamp (note that sorting
+ * is in reverse order, most recently (i.e. bigger timestamp)
+ * running/completed comes first).  N.B. zlistx_comparator_fn
+ * signature
+ */
+static int job_running_cmp (const void *a1, const void *a2)
+{
+    const struct job *j1 = a1;
+    const struct job *j2 = a2;
+
+    return NUMCMP (j2->t_run, j1->t_run);
+}
+
+static int job_inactive_cmp (const void *a1, const void *a2)
+{
+    const struct job *j1 = a1;
+    const struct job *j2 = a2;
+
+    return NUMCMP (j2->t_inactive, j1->t_inactive);
+}
+
+static void job_destroy_wrapper (void **data)
+{
+    struct job **job = (struct job **)data;
+    job_destroy (*job);
+}
+
+/* zlistx_insert() and zlistx_reorder() take a 'low_value' parameter
+ * which indicates which end of the list to search from.
+ * false=search begins at tail (lowest urgency, youngest)
+ * true=search begins at head (highest urgency, oldest)
+ * Attempt to minimize search distance based on job urgency.
+ */
+static bool search_direction (struct job *job)
+{
+    if (job->priority > (FLUX_JOB_PRIORITY_MAX / 2))
+        return true;
+    else
+        return false;
+}
+
+static void set_submit_timestamp (struct job *job, double timestamp)
+{
+    job->t_submit = timestamp;
+}
+
+static void update_job_state (struct job_state_ctx *jsctx,
+                              struct job *job,
+                              flux_job_state_t new_state,
+                              double timestamp)
+{
+    job_stats_update (jsctx->statsctx, job, new_state);
+
+    job->state = new_state;
+    if (job->state == FLUX_JOB_STATE_DEPEND)
+        job->t_depend = timestamp;
+    else if (job->state == FLUX_JOB_STATE_RUN)
+        job->t_run = timestamp;
+    else if (job->state == FLUX_JOB_STATE_CLEANUP)
+        job->t_cleanup = timestamp;
+    else if (job->state == FLUX_JOB_STATE_INACTIVE)
+        job->t_inactive = timestamp;
+    job->states_mask |= job->state;
+}
+
+static int job_insert_list (struct job_state_ctx *jsctx,
+                            struct job *job,
+                            flux_job_state_t newstate)
+{
+    if (newstate == FLUX_JOB_STATE_DEPEND
+        || newstate == FLUX_JOB_STATE_PRIORITY
+        || newstate == FLUX_JOB_STATE_SCHED) {
+        if (!(job->list_handle = zlistx_insert (jsctx->pending,
+                                                job,
+                                                search_direction (job))))
+            goto enomem;
+    }
+    else if (newstate == FLUX_JOB_STATE_RUN
+             || newstate == FLUX_JOB_STATE_CLEANUP) {
+        if (!(job->list_handle = zlistx_insert (jsctx->running, job, true)))
+            goto enomem;
+    }
+    else { /* newstate == FLUX_JOB_STATE_INACTIVE */
+        if (!(job->list_handle = zlistx_insert (jsctx->inactive, job, true)))
+            goto enomem;
+    }
+
+    return 0;
+
+enomem:
+    errno = ENOMEM;
+    return -1;
+}
+
+/* remove job from one list and move it to another based on the
+ * newstate */
+static void job_change_list (struct job_state_ctx *jsctx,
+                             struct job *job,
+                             zlistx_t *oldlist,
+                             flux_job_state_t newstate)
+{
+    if (zlistx_detach (oldlist, job->list_handle) < 0)
+        flux_log (jsctx->h,
+                  LOG_ERR,
+                  "%s: zlistx_detach: out of memory",
+                  __FUNCTION__);
+    job->list_handle = NULL;
+
+    if (job_insert_list (jsctx, job, newstate) < 0)
+        flux_log_error (jsctx->h,
+                        "error moving job to new list on state transition to %s",
+                        flux_job_statetostr (newstate, "L"));
+}
+
+static zlistx_t *get_list (struct job_state_ctx *jsctx, flux_job_state_t state)
+{
+    if (state == FLUX_JOB_STATE_NEW)
+        return jsctx->processing;
+    else if (state == FLUX_JOB_STATE_DEPEND
+             || state == FLUX_JOB_STATE_PRIORITY
+             || state == FLUX_JOB_STATE_SCHED)
+        return jsctx->pending;
+    else if (state == FLUX_JOB_STATE_RUN
+             || state == FLUX_JOB_STATE_CLEANUP)
+        return jsctx->running;
+    else /* state == FLUX_JOB_STATE_INACTIVE */
+        return jsctx->inactive;
+}
+
+static void update_job_state_and_list (struct job_state_ctx *jsctx,
+                                       struct job *job,
+                                       flux_job_state_t newstate,
+                                       double timestamp)
+{
+    zlistx_t *oldlist, *newlist;
+
+    oldlist = get_list (jsctx, job->state);
+    newlist = get_list (jsctx, newstate);
+
+    /* must call before job_change_list(), to ensure timestamps are
+     * set before any sorting based on timestamps are done
+     */
+    update_job_state (jsctx, job, newstate, timestamp);
+
+    /* when FLUX_JOB_STATE_SCHED is reached, the queue priority has
+     * been determined, meaning we can now sort the job on the pending
+     * list amongst jobs with queue priorities
+     */
+    if (oldlist != newlist)
+        job_change_list (jsctx, job, oldlist, newstate);
+    else if (oldlist == jsctx->pending
+             && newstate == FLUX_JOB_STATE_SCHED)
+        zlistx_reorder (jsctx->pending,
+                        job->list_handle,
+                        search_direction (job));
+
+    idsync_check_waiting_id (jsctx->ctx->isctx, job);
+}
+
+/* calculate any remaining fields */
+static void eventlog_inactive_complete (struct job *job)
+{
+    /* Default result is failed, overridden below */
+    if (job->success)
+        job->result = FLUX_JOB_RESULT_COMPLETED;
+    else if (job->exception_occurred) {
+        if (streq (job->exception_type, "cancel"))
+            job->result = FLUX_JOB_RESULT_CANCELED;
+        else if (streq (job->exception_type, "timeout"))
+            job->result = FLUX_JOB_RESULT_TIMEOUT;
+    }
+}
+
+static void process_state_transition_update (struct job_state_ctx *jsctx,
+                                             struct job *job,
+                                             flux_job_state_t state,
+                                             double timestamp,
+                                             int flags,
+                                             flux_job_state_t expected_state)
+{
+    if ((flags & STATE_TRANSITION_FLAG_REVERT)) {
+        /* only revert if the current state is what is expected */
+        if (job->state == expected_state) {
+            job->states_mask &= ~job->state;
+            job->states_mask &= ~state;
+            update_job_state_and_list (jsctx, job, state, timestamp);
+        }
+        else
+            return;
+    }
+    else if ((flags & STATE_TRANSITION_FLAG_CONDITIONAL)) {
+        /* if current state isn't what we expected, move on */
+        if (job->state != expected_state)
+            return;
+    }
+    if (state == FLUX_JOB_STATE_DEPEND) {
+        // process job->jobspec which was obtained from journal
+        if (job_parse_jobspec_cached (job, NULL) < 0) {
+            flux_log_error (jsctx->h,
+                            "%s: error parsing jobspec",
+                            idf58 (job->id));
+        }
+        update_job_state_and_list (jsctx, job, state, timestamp);
+    }
+    else if (state == FLUX_JOB_STATE_RUN) {
+        // process job->R which was obtained from journal
+        if (job_parse_R_cached (job, NULL) < 0) {
+            flux_log_error (jsctx->h,
+                            "%s: error parsing R",
+                            idf58 (job->id));
+        }
+        update_job_state_and_list (jsctx, job, state, timestamp);
+    }
+    else {
+        /* FLUX_JOB_STATE_PRIORITY */
+        /* FLUX_JOB_STATE_SCHED */
+        /* FLUX_JOB_STATE_CLEANUP */
+        /* FLUX_JOB_STATE_INACTIVE */
+
+        if (state == FLUX_JOB_STATE_INACTIVE)
+            eventlog_inactive_complete (job);
+
+        update_job_state_and_list (jsctx, job, state, timestamp);
+    }
+}
+
+static void update_jobspec (struct job_state_ctx *jsctx,
+                            struct job *job,
+                            json_t *context,
+                            bool update_stats)
+{
+    /* It is theoretically possible an update could occur before the
+     * jobspec is available.  We don't handle it, just log an error.
+     */
+    if (!job->jobspec) {
+        flux_log (jsctx->h, LOG_ERR,
+                  "%s: job %s received jobspec update before jobspec",
+                  __FUNCTION__, idf58 (job->id));
+        return;
+    }
+
+    /* jobspec-update has the potential to change the job queue,
+     * remove the queue specific stats and re-add after the update.
+     */
+    if (update_stats)
+        job_stats_remove_queue (jsctx->statsctx, job);
+
+    job_jobspec_update (job, context);
+
+    if (update_stats)
+        job_stats_add_queue (jsctx->statsctx, job);
+}
+
+static void update_resource (struct job_state_ctx *jsctx,
+                             struct job *job,
+                             json_t *context)
+{
+    /* R should always be available at this point, outside of
+     * testing scenarios.
+     */
+    if (!job->R) {
+        flux_log (jsctx->h, LOG_ERR,
+                  "%s: job %s received resource update before R",
+                  __FUNCTION__, idf58 (job->id));
+        return;
+    }
+
+    job_R_update (job, context);
+}
+
+void job_state_pause_cb (flux_t *h, flux_msg_handler_t *mh,
+                         const flux_msg_t *msg, void *arg)
+{
+    struct list_ctx *ctx = arg;
+    if (!ctx->jsctx->initialized) {
+        if (flux_msglist_append (ctx->deferred_requests, msg) < 0)
+            goto error;
+        return;
+    }
+    ctx->jsctx->pause = true;
+
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to pause request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to pause request");
+}
+
+void job_state_unpause_cb (flux_t *h, flux_msg_handler_t *mh,
+                           const flux_msg_t *msg, void *arg)
+{
+    struct list_ctx *ctx = arg;
+    const flux_msg_t *resp;
+
+    if (!ctx->jsctx->initialized) {
+        if (flux_msglist_append (ctx->deferred_requests, msg) < 0)
+            goto error;
+        return;
+    }
+    resp = flux_msglist_first (ctx->jsctx->backlog);
+    while (resp) {
+        if (journal_process_events (ctx->jsctx, resp) < 0)
+            goto error;
+        flux_msglist_delete (ctx->jsctx->backlog);
+        resp = flux_msglist_next (ctx->jsctx->backlog);
+    }
+
+    ctx->jsctx->pause = false;
+
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to unpause request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to unpause request");
+}
+
+static int job_transition_state (struct job_state_ctx *jsctx,
+                                 struct job *job,
+                                 flux_job_state_t newstate,
+                                 double timestamp,
+                                 int flags,
+                                 flux_job_state_t expected_state)
+{
+    if (!((flags & STATE_TRANSITION_FLAG_REVERT)
+          || (flags & STATE_TRANSITION_FLAG_CONDITIONAL))
+        && (newstate & job->states_events_mask))
+        return 0;
+
+    job->states_events_mask |= newstate;
+
+    process_state_transition_update (jsctx,
+                                     job,
+                                     newstate,
+                                     timestamp,
+                                     flags,
+                                     expected_state);
+    return 0;
+}
+
+static int journal_advance_job (struct job_state_ctx *jsctx,
+                                struct job *job,
+                                flux_job_state_t newstate,
+                                double timestamp)
+{
+    return job_transition_state (jsctx, job, newstate, timestamp, 0, 0);
+}
+
+static int journal_revert_job (struct job_state_ctx *jsctx,
+                               struct job *job,
+                               double timestamp)
+{
+    /* The flux-restart event is currently only posted to jobs in
+     * SCHED state since that is the only state transition defined
+     * for the event in RFC21.  In the future, other transitions
+     * may be defined.
+     */
+    return job_transition_state (jsctx,
+                                 job,
+                                 FLUX_JOB_STATE_PRIORITY,
+                                 timestamp,
+                                 STATE_TRANSITION_FLAG_REVERT,
+                                 FLUX_JOB_STATE_SCHED);
+}
+
+static int submit_context_parse (flux_t *h,
+                                 struct job *job,
+                                 json_t *context)
+{
+    int urgency;
+    int userid;
+    int version = -1;
+
+    if (!context
+        || json_unpack (context,
+                        "{ s:i s:i s?i }",
+                        "urgency", &urgency,
+                        "userid", &userid,
+                        "version", &version) < 0) {
+        flux_log (h, LOG_ERR, "%s: submit context invalid: %s",
+                  __FUNCTION__, idf58 (job->id));
+        errno = EPROTO;
+        return -1;
+    }
+
+    job->userid = userid;
+    job->urgency = urgency;
+    job->submit_version = version;
+    return 0;
+}
+
+static int journal_submit_event (struct job_state_ctx *jsctx,
+                                 struct job *job,
+                                 flux_jobid_t id,
+                                 double timestamp,
+                                 json_t *context,
+                                 json_t *jobspec)
+{
+    if (!job) {
+        if (!(job = job_create (jsctx->h, id)))
+            return -1;
+        if (jobspec)
+            job->jobspec = json_incref (jobspec);
+        if (zhashx_insert (jsctx->index, &job->id, job) < 0) {
+            job_destroy (job);
+            errno = EEXIST;
+            return -1;
+        }
+        /* job always starts off on processing list */
+        if (!(job->list_handle = zlistx_add_end (jsctx->processing, job))) {
+            errno = ENOMEM;
+            return -1;
+        }
+    }
+
+    if (submit_context_parse (jsctx->h, job, context) < 0)
+        return -1;
+    set_submit_timestamp (job, timestamp);
+
+    return 0;
+}
+
+static int priority_context_parse (flux_t *h,
+                                   struct job *job,
+                                   json_t *context)
+{
+    if (!context
+        || json_unpack (context,
+                        "{ s:I }",
+                        "priority", (json_int_t *)&job->priority) < 0) {
+        flux_log (h, LOG_ERR, "%s: priority context invalid: %s",
+                  __FUNCTION__, idf58 (job->id));
+        errno = EPROTO;
+        return -1;
+    }
+    return 0;
+}
+
+static int journal_priority_event (struct job_state_ctx *jsctx,
+                                   struct job *job,
+                                   double timestamp,
+                                   json_t *context)
+{
+    int64_t orig_priority = job->priority;
+
+    if (priority_context_parse (jsctx->h, job, context) < 0)
+        return -1;
+
+    if (job->state & FLUX_JOB_STATE_PENDING
+        && job->priority != orig_priority)
+        zlistx_reorder (jsctx->pending,
+                        job->list_handle,
+                        search_direction (job));
+
+    return job_transition_state (jsctx,
+                                 job,
+                                 FLUX_JOB_STATE_SCHED,
+                                 timestamp,
+                                 STATE_TRANSITION_FLAG_CONDITIONAL,
+                                 FLUX_JOB_STATE_PRIORITY);
+}
+
+static int finish_context_parse (flux_t *h,
+                                 struct job *job,
+                                 json_t *context)
+{
+    if (!context
+        || json_unpack (context,
+                        "{ s:i }",
+                        "status", &job->wait_status) < 0) {
+        flux_log (h, LOG_ERR, "%s: finish context invalid: %s",
+                  __FUNCTION__, idf58 (job->id));
+        errno = EPROTO;
+        return -1;
+    }
+
+    /*
+     *  A job is successful only if it finished with status == 0
+     *  *and* there were no fatal job exceptions:
+     */
+    if (job->wait_status == 0
+        && !(job->exception_occurred && job->exception_severity == 0))
+        job->success = true;
+
+    return 0;
+}
+
+static int journal_finish_event (struct job_state_ctx *jsctx,
+                                 struct job *job,
+                                 double timestamp,
+                                 json_t *context)
+{
+    if (finish_context_parse (jsctx->h, job, context) < 0)
+        return -1;
+
+    return job_transition_state (jsctx,
+                                 job,
+                                 FLUX_JOB_STATE_CLEANUP,
+                                 timestamp,
+                                 0,
+                                 0);
+}
+
+static int urgency_context_parse (flux_t *h,
+                                  struct job *job,
+                                  json_t *context)
+{
+    int urgency;
+
+    if (!context
+        || json_unpack (context, "{ s:i }", "urgency", &urgency) < 0
+        || (urgency < FLUX_JOB_URGENCY_MIN
+            || urgency > FLUX_JOB_URGENCY_MAX)) {
+        flux_log (h, LOG_ERR, "%s: urgency context invalid: %s",
+                  __FUNCTION__, idf58 (job->id));
+        errno = EPROTO;
+        return -1;
+    }
+
+    job->urgency = urgency;
+    return 0;
+}
+
+static int journal_urgency_event (struct job_state_ctx *jsctx,
+                                  struct job *job,
+                                  json_t *context)
+{
+    return urgency_context_parse (jsctx->h, job, context);
+}
+
+static int exception_context_parse (flux_t *h,
+                                    struct job *job,
+                                    json_t *context,
+                                    int *severityP)
+{
+    const char *type;
+    int severity;
+    const char *note;
+
+    if (!context
+        || json_unpack (context,
+                        "{s:s s:i s:s}",
+                        "type", &type,
+                        "severity", &severity,
+                        "note", &note) < 0) {
+        flux_log (h, LOG_ERR, "%s: exception context invalid: %s",
+                  __FUNCTION__, idf58 (job->id));
+        errno = EPROTO;
+        return -1;
+    }
+
+    if (!job->exception_occurred
+        || severity < job->exception_severity) {
+        job->exception_occurred = true;
+        job->exception_severity = severity;
+        job->exception_type = type;
+        job->exception_note = note;
+        json_decref (job->exception_context);
+        job->exception_context = json_incref (context);
+    }
+
+    if (severityP)
+        (*severityP) = severity;
+    return 0;
+}
+
+
+static int dependency_add (struct job *job,
+                           const char *description)
+{
+    if (grudgeset_add (&job->dependencies, description) < 0
+        && errno != EEXIST)
+        /*  Log non-EEXIST errors, but it is not fatal */
+        flux_log_error (job->h,
+                        "job %s: dependency-add",
+                         idf58 (job->id));
+    return 0;
+}
+
+static int dependency_remove (struct job *job,
+                              const char *description)
+{
+    int rc = grudgeset_remove (job->dependencies, description);
+    if (rc < 0 && errno == ENOENT) {
+        /*  No matching dependency is non-fatal error */
+        flux_log (job->h,
+                  LOG_DEBUG,
+                  "job %s: dependency-remove '%s' not found",
+                  idf58 (job->id),
+                  description);
+        rc = 0;
+    }
+    return rc;
+}
+
+static int dependency_context_parse (flux_t *h,
+                                     struct job *job,
+                                     const char *cmd,
+                                     json_t *context)
+{
+    int rc;
+    const char *description = NULL;
+
+    if (!context
+        || json_unpack (context,
+                        "{s:s}",
+                        "description", &description) < 0) {
+        flux_log (h, LOG_ERR,
+                  "job %s: dependency-%s context invalid",
+                  idf58 (job->id),
+                  cmd);
+        errno = EPROTO;
+        return -1;
+    }
+
+    if (streq (cmd, "add"))
+        rc = dependency_add (job, description);
+    else if (streq (cmd, "remove"))
+        rc = dependency_remove (job, description);
+    else {
+        flux_log (h, LOG_ERR,
+                  "job %s: invalid dependency event: dependency-%s",
+                  idf58 (job->id),
+                  cmd);
+        return -1;
+    }
+    return rc;
+}
+
+static int memo_update (flux_t *h,
+                        struct job *job,
+                        json_t *o)
+{
+    if (!o) {
+        flux_log (h, LOG_ERR, "%s: invalid memo context", idf58 (job->id));
+        errno = EPROTO;
+        return -1;
+    }
+    if (!job->annotations && !(job->annotations = json_object ())) {
+        errno = ENOMEM;
+        return -1;
+    }
+    if (jpath_update (job->annotations, "user", o) < 0
+        || jpath_clear_null (job->annotations) < 0)
+        return -1;
+    if (json_object_size (job->annotations) == 0) {
+        json_decref (job->annotations);
+        job->annotations = NULL;
+    }
+    return 0;
+}
+
+static int journal_exception_event (struct job_state_ctx *jsctx,
+                                    struct job *job,
+                                    double timestamp,
+                                    json_t *context)
+{
+    int severity;
+
+    if (exception_context_parse (jsctx->h, job, context, &severity) < 0)
+        return -1;
+
+    if (severity == 0)
+        return job_transition_state (jsctx,
+                                     job,
+                                     FLUX_JOB_STATE_CLEANUP,
+                                     timestamp,
+                                     0,
+                                     0);
+
+    return 0;
+}
+
+static int journal_annotations_event (struct job_state_ctx *jsctx,
+                                      struct job *job,
+                                      json_t *context)
+{
+    json_t *annotations = NULL;
+
+    if (!context
+        || json_unpack (context, "{ s:o }", "annotations", &annotations) < 0) {
+        flux_log (jsctx->h, LOG_ERR,
+                  "%s: annotations event context invalid: %s",
+                  __FUNCTION__, idf58 (job->id));
+        errno = EPROTO;
+        return -1;
+    }
+    json_decref (job->annotations);
+    if (json_is_null (annotations))
+        job->annotations = NULL;
+    else
+        job->annotations = json_incref (annotations);
+
+    return 0;
+}
+
+static int journal_jobspec_update_event (struct job_state_ctx *jsctx,
+                                         struct job *job,
+                                         json_t *context)
+{
+    if (!context) {
+        flux_log (jsctx->h, LOG_ERR,
+                  "%s: jobspec-update event context invalid: %s",
+                  __FUNCTION__, idf58 (job->id));
+        errno = EPROTO;
+        return -1;
+    }
+    /* Generally speaking, after a job is running, jobspec-update
+     * events should have no effect.  Note that in some cases,
+     * such as job duration, jobspec-updates can alter a job's
+     * behavior, but it is via an update to R.  In this case, we
+     * elect to not update the job duration seen by the user in
+     * the jobspec.  The effect will be seen changes in R (in this
+     * example, via the job expiration time in R).
+     */
+    if (job->state < FLUX_JOB_STATE_RUN)
+        update_jobspec (jsctx, job, context, true);
+    return 0;
+}
+
+static int journal_resource_update_event (struct job_state_ctx *jsctx,
+                                         struct job *job,
+                                         json_t *context)
+{
+    if (!context) {
+        flux_log (jsctx->h, LOG_ERR,
+                  "%s: resource-update event context invalid: %s",
+                  __FUNCTION__, idf58 (job->id));
+        errno = EPROTO;
+        return -1;
+    }
+    /* Generally speaking, resource-update events only have an effect
+     * when a job is running. */
+    if (job->state == FLUX_JOB_STATE_RUN)
+        update_resource (jsctx, job, context);
+    return 0;
+}
+
+static int journal_dependency_event (struct job_state_ctx *jsctx,
+                                     struct job *job,
+                                     const char *cmd,
+                                     json_t *context)
+{
+    return dependency_context_parse (jsctx->h, job, cmd, context);
+}
+
+static int journal_process_event (struct job_state_ctx *jsctx,
+                                  flux_jobid_t id,
+                                  json_t *event,
+                                  json_t *jobspec,
+                                  json_t *R)
+{
+    double timestamp;
+    const char *name;
+    struct job *job;
+    json_t *context = NULL;
+
+    if (eventlog_entry_parse (event, &timestamp, &name, &context) < 0) {
+        flux_log (jsctx->h, LOG_ERR, "%s: error parsing record",
+                  __FUNCTION__);
+        errno = EPROTO;
+        return -1;
+    }
+
+    job = zhashx_lookup (jsctx->index, &id);
+    if (job) {
+        if (!job->R && R)
+            job->R = json_incref (R);
+    }
+
+    /* The "submit" event is now posted before the job transitions out of NEW
+     * on the "validate" event.  If "invalidate" is posted instead, job
+     * submission failed and the job is removed from the KVS.  Drop the
+     * nascent job info.
+     */
+    if (job && streq (name, "invalidate")) {
+        if (job->list_handle) {
+            zlistx_detach (jsctx->processing, job->list_handle);
+            job->list_handle = NULL;
+        }
+        zhashx_delete (jsctx->index, &job->id);
+        /* N.B. since invalid job ids are not released to the submitter, there
+         * should be no pending ctx->isctx->lookups requests to clean up here.
+         * A test in t2212-job-manager-plugins.t does query invalid ids, but
+         * it is careful to ensure that it does so only _after_ the invalidate
+         * event has been processed here.
+         */
+        return 0;
+    }
+
+    /*  Job not found is non-fatal, do not return an error.
+     *  No need to proceed unless this is the first event (submit),
+     *   but log an error since this is an unexpected condition.
+     */
+    if (!job && !streq (name, "submit")) {
+        flux_log (jsctx->h,
+                  LOG_ERR,
+                  "event %s: job %s not in hash",
+                  name,
+                  idf58 (id));
+        return 0;
+    }
+
+    if (streq (name, "submit")) {
+        if (journal_submit_event (jsctx,
+                                  job,
+                                  id,
+                                  timestamp,
+                                  context,
+                                  jobspec) < 0)
+            return -1;
+    }
+    else if (streq (name, "validate")) {
+        if (journal_advance_job (jsctx,
+                                 job,
+                                 FLUX_JOB_STATE_DEPEND,
+                                 timestamp) < 0)
+            return -1;
+    }
+    else if (streq (name, "depend")) {
+        if (journal_advance_job (jsctx,
+                                 job,
+                                 FLUX_JOB_STATE_PRIORITY,
+                                 timestamp) < 0)
+            return -1;
+    }
+    else if (streq (name, "priority")) {
+        if (journal_priority_event (jsctx,
+                                    job,
+                                    timestamp,
+                                    context) < 0)
+            return -1;
+    }
+    else if (streq (name, "alloc")) {
+        /* alloc event contains annotations, but we only update
+         * annotations via "annotations" events */
+        if (journal_advance_job (jsctx,
+                                 job,
+                                 FLUX_JOB_STATE_RUN,
+                                 timestamp) < 0)
+            return -1;
+    }
+    else if (streq (name, "finish")) {
+        if (journal_finish_event (jsctx,
+                                  job,
+                                  timestamp,
+                                  context) < 0)
+            return -1;
+    }
+    else if (streq (name, "clean")) {
+        if (journal_advance_job (jsctx,
+                                 job,
+                                 FLUX_JOB_STATE_INACTIVE,
+                                 timestamp) < 0)
+            return -1;
+    }
+    else if (streq (name, "urgency")) {
+        if (journal_urgency_event (jsctx,
+                                   job,
+                                   context) < 0)
+            return -1;
+    }
+    else if (streq (name, "exception")) {
+        if (journal_exception_event (jsctx,
+                                     job,
+                                     timestamp,
+                                     context) < 0)
+            return -1;
+    }
+    else if (streq (name, "annotations")) {
+        if (journal_annotations_event (jsctx,
+                                       job,
+                                       context) < 0)
+            return -1;
+    }
+    else if (streq (name, "jobspec-update")) {
+        if (journal_jobspec_update_event (jsctx, job, context) < 0)
+            return -1;
+    }
+    else if (streq (name, "resource-update")) {
+        if (journal_resource_update_event (jsctx, job, context) < 0)
+            return -1;
+    }
+    else if (streq (name, "memo")) {
+        if (memo_update (jsctx->h, job, context) < 0)
+            return -1;
+    }
+    else if (strstarts (name, "dependency-")) {
+        if (journal_dependency_event (jsctx, job, name+11, context) < 0)
+            return -1;
+    }
+    else if (streq (name, "flux-restart")) {
+        /* This can only be seen while processing the journal backlog
+         * as it is posted by the job manager during its KVS restart,
+         * which happens synchronously before the journal RPC is processed.
+         */
+        if (journal_revert_job (jsctx,
+                                job,
+                                timestamp) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static int journal_process_events (struct job_state_ctx *jsctx,
+                                   const flux_msg_t *msg)
+{
+    flux_jobid_t id;
+    json_t *events;
+    size_t index;
+    json_t *value;
+    json_t *jobspec = NULL;
+    json_t *R = NULL;
+
+    if (flux_msg_unpack (msg,
+                        "{s:I s:o s?o s?o}",
+                        "id", &id,
+                        "events", &events,
+                        "jobspec", &jobspec,
+                        "R", &R) < 0)
+        return -1;
+    if (!json_is_array (events)) {
+        errno = EPROTO;
+        return -1;
+    }
+    json_array_foreach (events, index, value) {
+        if (journal_process_event (jsctx, id, value, jobspec, R) < 0)
+            return -1;
+    }
+
+    return 0;
+}
+
+static void job_events_journal_continuation (flux_future_t *f, void *arg)
+{
+    struct job_state_ctx *jsctx = arg;
+    const flux_msg_t *msg;
+    flux_jobid_t id;
+
+    if (flux_rpc_get_unpack (f, "{s:I}", "id", &id) < 0
+        || flux_future_get (f, (const void **)&msg) < 0) {
+        if (errno == ENODATA) {
+            flux_log (jsctx->h, LOG_INFO, "journal: EOF (exiting)");
+            flux_reactor_stop (flux_get_reactor (jsctx->h));
+            return;
+        }
+        flux_log (jsctx->h, LOG_ERR, "journal: %s", future_strerror (f, errno));
+        goto error;
+    }
+
+    /* Detect sentinel that delimits old and new events.  If there are no
+     * more old events, process backlog and begin processing new events
+     * as they arrive.
+     */
+    if (id == FLUX_JOBID_ANY) {
+        while ((msg = flux_msglist_pop (jsctx->backlog))) {
+            int rc = journal_process_events (jsctx, msg);
+            flux_msg_decref (msg);
+            if (rc < 0) {
+                flux_log_error (jsctx->h, "error processing journal backlog");
+                goto error;
+            }
+        }
+        jsctx->initialized = true;
+        requeue_deferred_requests (jsctx->ctx);
+        flux_future_reset (f);
+        return;
+    }
+    if (!jsctx->initialized || jsctx->pause) {
+        if (flux_msglist_append (jsctx->backlog, msg) < 0) {
+            flux_log_error (jsctx->h, "error storing journal backlog");
+            goto error;
+        }
+    }
+    else {
+        if (journal_process_events (jsctx, msg) < 0) {
+            flux_log_error (jsctx->h, "error processing events");
+            goto error;
+        }
+    }
+
+    flux_future_reset (f);
+    return;
+
+error:
+    /* future will be cleaned up in shutdown path */
+    flux_reactor_stop_error (flux_get_reactor (jsctx->h));
+    return;
+}
+
+static flux_future_t *job_events_journal (struct job_state_ctx *jsctx)
+{
+    flux_future_t *f;
+
+    /* Set full=true so that inactive jobs are included.
+     * Don't set allow/deny so that we receive all events.
+     */
+    if (!(f = flux_rpc_pack (jsctx->h,
+                             "job-manager.events-journal",
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_STREAMING,
+                             "{s:b}",
+                             "full", 1))
+        || flux_future_then (f,
+                             -1,
+                             job_events_journal_continuation,
+                             jsctx) < 0) {
+        flux_log (jsctx->h,
+                  LOG_ERR,
+                  "error synchronizing with job manager journal: %s",
+                  future_strerror (f, errno));
+        goto error;
+    }
+    return f;
+error:
+    flux_future_destroy (f);
+    return NULL;
+}
+
+struct job_state_ctx *job_state_create (struct list_ctx *ctx)
+{
+    struct job_state_ctx *jsctx = NULL;
+
+    if (!(jsctx = calloc (1, sizeof (*jsctx)))) {
+        flux_log_error (ctx->h, "calloc");
+        return NULL;
+    }
+    jsctx->h = ctx->h;
+    jsctx->ctx = ctx;
+
+    /* Index is the primary data structure holding the job data
+     * structures.  It is responsible for destruction.  Lists only
+     * contain desired sort of jobs.
+     */
+
+    if (!(jsctx->index = job_hash_create ()))
+        goto error;
+    zhashx_set_destructor (jsctx->index, job_destroy_wrapper);
+
+    if (!(jsctx->pending = zlistx_new ()))
+        goto error;
+    zlistx_set_comparator (jsctx->pending, job_urgency_cmp);
+
+    if (!(jsctx->running = zlistx_new ()))
+        goto error;
+    zlistx_set_comparator (jsctx->running, job_running_cmp);
+
+    if (!(jsctx->inactive = zlistx_new ()))
+        goto error;
+    zlistx_set_comparator (jsctx->inactive, job_inactive_cmp);
+
+    if (!(jsctx->processing = zlistx_new ()))
+        goto error;
+
+    if (!(jsctx->statsctx = job_stats_ctx_create (jsctx->h)))
+        goto error;
+
+    if (!(jsctx->backlog = flux_msglist_create ()))
+        goto error;
+
+    if (!(jsctx->events = job_events_journal (jsctx)))
+        goto error;
+
+    return jsctx;
+
+error:
+    job_state_destroy (jsctx);
+    return NULL;
+}
+
+void job_state_destroy (void *data)
+{
+    struct job_state_ctx *jsctx = data;
+    if (jsctx) {
+        int saved_errno = errno;
+        /* Destroy index last, as it is the one that will actually
+         * destroy the job objects */
+        zlistx_destroy (&jsctx->processing);
+        zlistx_destroy (&jsctx->inactive);
+        zlistx_destroy (&jsctx->running);
+        zlistx_destroy (&jsctx->pending);
+        zhashx_destroy (&jsctx->index);
+        job_stats_ctx_destroy (jsctx->statsctx);
+        flux_msglist_destroy (jsctx->backlog);
+        flux_future_destroy (jsctx->events);
+        free (jsctx);
+        errno = saved_errno;
+    }
+}
+
+int job_state_config_reload (struct job_state_ctx *jsctx,
+                             const flux_conf_t *conf,
+                             flux_error_t *errp)
+{
+    return job_stats_config_reload (jsctx->statsctx, conf, errp);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/job_state.h b/src/modules/job-list/job_state.h
new file mode 100644
index 000000000000..56d6f6ca0a72
--- /dev/null
+++ b/src/modules/job-list/job_state.h
@@ -0,0 +1,80 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_LIST_JOB_STATE_H
+#define _FLUX_JOB_LIST_JOB_STATE_H
+
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "idsync.h"
+#include "stats.h"
+
+/* To handle the common case of user queries on job state, we will
+ * store jobs in three different lists.
+ *
+ * - pending - these are jobs that have not yet in the RUN state, they
+ *   are sorted based on job urgency (highest first), then job
+ *   submission time (earlier submission time first).
+ * - running - these are jobs that have transitioned to the RUN state.
+ *   They are sorted by initial run start time (later run start
+ *   times first).
+ * - inactive - these are jobs that are in the INACTIVE state, they
+ *   are sorted by job completion time (later completion times
+ *   first)
+ *
+ * There is also an additional list `processing` that stores jobs that
+ * cannot yet be stored on one of the lists above.
+ */
+
+struct job_state_ctx {
+    flux_t *h;
+    struct list_ctx *ctx;
+
+    zhashx_t *index;
+    zlistx_t *pending;
+    zlistx_t *running;
+    zlistx_t *inactive;
+    zlistx_t *processing;
+
+    /*  Job statistics: */
+    struct job_stats_ctx *statsctx;
+
+    /* debug/testing - journal responses queued during pause */
+    bool pause;
+    struct flux_msglist *backlog;
+
+    /* stream of job events from the job-manager */
+    flux_future_t *events;
+
+    bool initialized;
+};
+
+struct job_state_ctx *job_state_create (struct list_ctx *ctx);
+
+void job_state_destroy (void *data);
+
+void job_state_pause_cb (flux_t *h, flux_msg_handler_t *mh,
+                         const flux_msg_t *msg, void *arg);
+
+void job_state_unpause_cb (flux_t *h, flux_msg_handler_t *mh,
+                           const flux_msg_t *msg, void *arg);
+
+int job_state_config_reload (struct job_state_ctx *jsctx,
+                             const flux_conf_t *conf,
+                             flux_error_t *errp);
+
+#endif /* ! _FLUX_JOB_LIST_JOB_STATE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/job_util.c b/src/modules/job-list/job_util.c
new file mode 100644
index 000000000000..163ab84d9462
--- /dev/null
+++ b/src/modules/job-list/job_util.c
@@ -0,0 +1,295 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* job_util.c - job utility functions */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <stdarg.h>
+#include <flux/core.h>
+#include <assert.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "job-list.h"
+#include "job_util.h"
+
+static int store_attr (struct job *job,
+                       const char *attr,
+                       json_t *o,
+                       flux_error_t *errp)
+{
+    json_t *val = NULL;
+
+    if (streq (attr, "userid")) {
+        val = json_integer (job->userid);
+    }
+    else if (streq (attr, "urgency")) {
+        val = json_integer (job->urgency);
+    }
+    else if (streq (attr, "priority")) {
+        if (!(job->states_mask & FLUX_JOB_STATE_SCHED))
+            return 0;
+        val = json_integer (job->priority);
+    }
+    else if (streq (attr, "t_submit")) {
+        val = json_real (job->t_submit);
+    }
+    else if (streq (attr, "t_depend")) {
+        /* if submit_version < 1, it means it was not set.  This is
+         * before the introduction of event `validate` after 0.41.1.
+         * Before the introduction of this event, t_submit and
+         * t_depend are the same.  So return the value of t_submit
+         */
+        if (job->submit_version < 1) {
+            val = json_real (job->t_submit);
+            goto out;
+        }
+        if (!(job->states_mask & FLUX_JOB_STATE_DEPEND))
+            return 0;
+        val = json_real (job->t_depend);
+    }
+    else if (streq (attr, "t_run")) {
+        if (!(job->states_mask & FLUX_JOB_STATE_RUN))
+            return 0;
+        val = json_real (job->t_run);
+    }
+    else if (streq (attr, "t_cleanup")) {
+        if (!(job->states_mask & FLUX_JOB_STATE_CLEANUP))
+            return 0;
+        val = json_real (job->t_cleanup);
+    }
+    else if (streq (attr, "t_inactive")) {
+        if (!(job->states_mask & FLUX_JOB_STATE_INACTIVE))
+            return 0;
+        val = json_real (job->t_inactive);
+    }
+    else if (streq (attr, "state")) {
+        val = json_integer (job->state);
+    }
+    else if (streq (attr, "name")) {
+        /* job->name potentially NULL if jobspec invalid */
+        if (!job->name)
+            return 0;
+        val = json_string (job->name);
+    }
+    else if (streq (attr, "cwd")) {
+        /* job->cwd potentially NULL, is optional in jobspec */
+        if (!job->cwd)
+            return 0;
+        val = json_string (job->cwd);
+    }
+    else if (streq (attr, "queue")) {
+        /* job->queue potentially NULL if:
+         * - unspecified
+         * - jobspec invalid
+         */
+        if (!job->queue)
+            return 0;
+        val = json_string (job->queue);
+    }
+    else if (streq (attr, "project")) {
+        /* job->project potentially NULL, usually set via jobspec-update event */
+        if (!job->project)
+            return 0;
+        val = json_string (job->project);
+    }
+    else if (streq (attr, "bank")) {
+        /* job->bank potentially NULL, usually set via jobspec-update event */
+        if (!job->bank)
+            return 0;
+        val = json_string (job->bank);
+    }
+    else if (streq (attr, "ntasks")) {
+        /* job->ntasks potentially < 0 if jobspec invalid */
+        if (job->ntasks < 0)
+            return 0;
+        val = json_integer (job->ntasks);
+    }
+    else if (streq (attr, "ncores")) {
+        /* job->ncores potentially < 0 if not set yet or R invalid,
+         * may be set in DEPEND or RUN state */
+        if (job->ncores < 0)
+            return 0;
+        val = json_integer (job->ncores);
+    }
+    else if (streq (attr, "duration")) {
+        /* job->duration potentially < 0 if jobspec invalid */
+        if (job->duration < 0)
+            return 0;
+        val = json_real (job->duration);
+    }
+    else if (streq (attr, "nnodes")) {
+        /* job->nnodes < 0 if not set yet or R invalid, may be set in
+         * DEPEND or RUN state */
+        if (job->nnodes < 0)
+            return 0;
+        val = json_integer (job->nnodes);
+    }
+    else if (streq (attr, "ranks")) {
+        /* job->ranks potentially NULL if R invalid */
+        if (!(job->states_mask & FLUX_JOB_STATE_RUN)
+            || !job->ranks)
+            return 0;
+        val = json_string (job->ranks);
+    }
+    else if (streq (attr, "nodelist")) {
+        /* job->nodelist potentially NULL if R invalid */
+        if (!(job->states_mask & FLUX_JOB_STATE_RUN)
+            || !job->nodelist)
+            return 0;
+        val = json_string (job->nodelist);
+    }
+    else if (streq (attr, "expiration")) {
+        /* job->expiration potentially < 0 if R invalid */
+        if (!(job->states_mask & FLUX_JOB_STATE_RUN)
+            || job->expiration < 0)
+            return 0;
+        val = json_real (job->expiration);
+    }
+    else if (streq (attr, "waitstatus")) {
+        if (job->wait_status < 0)
+            return 0;
+        val = json_integer (job->wait_status);
+    }
+    else if (streq (attr, "success")) {
+        if (!(job->states_mask & FLUX_JOB_STATE_INACTIVE))
+            return 0;
+        val = json_boolean (job->success);
+    }
+    else if (streq (attr, "exception_occurred")) {
+        if (!(job->states_mask & FLUX_JOB_STATE_INACTIVE))
+            return 0;
+        val = json_boolean (job->exception_occurred);
+    }
+    else if (streq (attr, "exception_severity")) {
+        if (!(job->states_mask & FLUX_JOB_STATE_INACTIVE)
+            || !job->exception_occurred)
+            return 0;
+        val = json_integer (job->exception_severity);
+    }
+    else if (streq (attr, "exception_type")) {
+        if (!(job->states_mask & FLUX_JOB_STATE_INACTIVE)
+            || !job->exception_occurred)
+            return 0;
+        val = json_string (job->exception_type);
+    }
+    else if (streq (attr, "exception_note")) {
+        if (!(job->states_mask & FLUX_JOB_STATE_INACTIVE)
+            || !job->exception_occurred
+            || !job->exception_note)
+            return 0;
+        val = json_string (job->exception_note);
+    }
+    else if (streq (attr, "result")) {
+        if (!(job->states_mask & FLUX_JOB_STATE_INACTIVE))
+            return 0;
+        val = json_integer (job->result);
+    }
+    else if (streq (attr, "annotations")) {
+        if (!job->annotations)
+            return 0;
+        val = json_incref (job->annotations);
+    }
+    else if (streq (attr, "dependencies")) {
+        if (!job->dependencies)
+            return 0;
+        val = json_incref (grudgeset_tojson (job->dependencies));
+    }
+    else {
+        errprintf (errp, "%s is not a valid attribute", attr);
+        errno = EINVAL;
+        return -1;
+    }
+out:
+    if (val == NULL) {
+        errno = ENOMEM;
+        return -1;
+    }
+    if (json_object_set_new (o, attr, val) < 0) {
+        json_decref (val);
+        errno = ENOMEM;
+        return -1;
+    }
+
+    return 0;
+}
+
+int store_all_attr (struct job *job, json_t *o, flux_error_t *errp)
+{
+    const char **ptr = job_attrs ();
+
+    assert (ptr);
+
+    while (*ptr) {
+        if (store_attr (job, *ptr, o, errp) < 0)
+            return -1;
+        ptr++;
+    }
+
+    return 0;
+}
+
+/* For a given job, create a JSON object containing the jobid and any
+ * additional requested attributes and their values.  Returns JSON
+ * object which the caller must free.  On error, return NULL with
+ * errno set:
+ *
+ * EPROTO - malformed attrs array
+ * ENOMEM - out of memory
+ */
+json_t *job_to_json (struct job *job, json_t *attrs, flux_error_t *errp)
+{
+    json_t *val = NULL;
+    size_t index;
+    json_t *value;
+    json_t *o;
+
+    memset (errp, 0, sizeof (*errp));
+
+    if (!(o = json_object ()))
+        goto error_nomem;
+    if (!(val = json_integer (job->id)))
+        goto error_nomem;
+    if (json_object_set_new (o, "id", val) < 0) {
+        json_decref (val);
+        goto error_nomem;
+    }
+    json_array_foreach (attrs, index, value) {
+        const char *attr = json_string_value (value);
+        if (!attr) {
+            errprintf (errp, "attr has no string value");
+            errno = EINVAL;
+            goto error;
+        }
+        if (streq (attr, "all")) {
+            if (store_all_attr (job, o, errp) < 0)
+                goto error;
+        }
+        else {
+            if (store_attr (job, attr, o, errp) < 0)
+                goto error;
+        }
+    }
+    return o;
+ error_nomem:
+    errno = ENOMEM;
+ error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/job_util.h b/src/modules/job-list/job_util.h
new file mode 100644
index 000000000000..8e09fc47e722
--- /dev/null
+++ b/src/modules/job-list/job_util.h
@@ -0,0 +1,24 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_LIST_JOB_UTIL_H
+#define _FLUX_JOB_LIST_JOB_UTIL_H
+
+#include <flux/core.h>
+
+#include "job_data.h"
+
+json_t *job_to_json (struct job *job, json_t *attrs, flux_error_t *errp);
+
+#endif /* ! _FLUX_JOB_LIST_JOB_UTIL_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/list.c b/src/modules/job-list/list.c
new file mode 100644
index 000000000000..e952e4fb0185
--- /dev/null
+++ b/src/modules/job-list/list.c
@@ -0,0 +1,626 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* list.c - list jobs */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <assert.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
+#include "job-list.h"
+#include "idsync.h"
+#include "list.h"
+#include "job_util.h"
+#include "job_data.h"
+#include "match.h"
+#include "state_match.h"
+
+json_t *get_job_by_id (struct job_state_ctx *jsctx,
+                       flux_error_t *errp,
+                       const flux_msg_t *msg,
+                       flux_jobid_t id,
+                       json_t *attrs,
+                       flux_job_state_t state,
+                       bool *stall);
+
+/* Put jobs from list onto jobs array, breaking if max_entries has
+ * been reached. Returns 1 if jobs array is full, 0 if continue, -1
+ * one error with errno set:
+ *
+ * ENOMEM - out of memory
+ */
+int get_jobs_from_list (json_t *jobs,
+                        flux_error_t *errp,
+                        zlistx_t *list,
+                        int max_entries,
+                        json_t *attrs,
+                        double since,
+                        struct list_constraint *c)
+{
+    struct job *job;
+
+    job = zlistx_first (list);
+    while (job) {
+        int ret;
+
+        /*  If job->t_inactive > 0. we're on the inactive jobs list and jobs are
+         *  sorted on the inactive list, larger t_inactive first.
+         *
+         *  If job->t_inactive > since, this is a job that could potentially be returned
+         *
+         *  So if job->t_inactive <= since, then we're done b/c the rest of the inactive
+         *   jobs cannot be returned.
+         */
+        if (job->t_inactive > 0. && job->t_inactive <= since)
+            break;
+
+        if ((ret = job_match (job, c, errp)) < 0)
+            return -1;
+        if (ret) {
+            json_t *o;
+            if (!(o = job_to_json (job, attrs, errp)))
+                return -1;
+            if (json_array_append_new (jobs, o) < 0) {
+                json_decref (o);
+                errno = ENOMEM;
+                return -1;
+            }
+            if (json_array_size (jobs) == max_entries)
+                return 1;
+        }
+        job = zlistx_next (list);
+    }
+
+    return 0;
+}
+
+/* Create a JSON array of 'job' objects.  'max_entries' determines the
+ * max number of jobs to return, 0=unlimited. 'since' limits jobs returned
+ * to those with t_inactive greater than timestamp.  Returns JSON object
+ * which the caller must free.  On error, return NULL with errno set:
+ *
+ * EPROTO - malformed or empty attrs array, max_entries out of range
+ * ENOMEM - out of memory
+ */
+json_t *get_jobs (struct job_state_ctx *jsctx,
+                  flux_error_t *errp,
+                  int max_entries,
+                  double since,
+                  json_t *attrs,
+                  struct list_constraint *c,
+                  struct state_constraint *statec)
+{
+    json_t *jobs = NULL;
+    int saved_errno;
+    int ret = 0;
+
+    if (!(jobs = json_array ()))
+        goto error_nomem;
+
+    /* We return jobs in the following order, pending, running,
+     * inactive */
+
+    if (state_match (FLUX_JOB_STATE_PENDING, statec)) {
+        if ((ret = get_jobs_from_list (jobs,
+                                       errp,
+                                       jsctx->pending,
+                                       max_entries,
+                                       attrs,
+                                       0.,
+                                       c)) < 0)
+            goto error;
+    }
+
+    if (state_match (FLUX_JOB_STATE_RUNNING, statec)) {
+        if (!ret) {
+            if ((ret = get_jobs_from_list (jobs,
+                                           errp,
+                                           jsctx->running,
+                                           max_entries,
+                                           attrs,
+                                           0.,
+                                           c)) < 0)
+                goto error;
+        }
+    }
+
+    if (state_match (FLUX_JOB_STATE_INACTIVE, statec)) {
+        if (!ret) {
+            if ((ret = get_jobs_from_list (jobs,
+                                           errp,
+                                           jsctx->inactive,
+                                           max_entries,
+                                           attrs,
+                                           since,
+                                           c)) < 0)
+                goto error;
+        }
+    }
+
+    return jobs;
+
+error_nomem:
+    errno = ENOMEM;
+error:
+    saved_errno = errno;
+    json_decref (jobs);
+    errno = saved_errno;
+    return NULL;
+}
+
+static int legacy_list_rpc (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            struct list_ctx *ctx,
+                            int *max_entries,
+                            json_t **attrs,
+                            double *since,
+                            json_t **legacy_constraint)
+{
+    uint32_t userid;
+    int states;
+    int results;
+    const char *name = NULL;
+    const char *queue = NULL;
+    json_t *a = NULL;
+    json_t *o;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s:o s:i s:i s:i s?F s?s s?s}",
+                             "max_entries", max_entries,
+                             "attrs", attrs,
+                             "userid", &userid,
+                             "states", &states,
+                             "results", &results,
+                             "since", since,
+                             "name", &name,
+                             "queue", &queue) < 0)
+        return -1;
+
+    /* Create constraint object given legacy inputs */
+
+    if (!(a = json_array ()))
+        goto error;
+
+    if (userid != FLUX_USERID_UNKNOWN) {
+        o = json_pack ("{s:[i]}", "userid", userid);
+        if (!o || json_array_append_new (a, o) < 0) {
+            json_decref (o);
+            goto error;
+        }
+    }
+
+    if (name) {
+        o = json_pack ("{s:[s]}", "name", name);
+        if (!o || json_array_append_new (a, o) < 0) {
+            json_decref (o);
+            goto error;
+        }
+    }
+
+    if (queue) {
+        o = json_pack ("{s:[s]}", "queue", queue);
+        if (!o || json_array_append_new (a, o) < 0) {
+            json_decref (o);
+            goto error;
+        }
+    }
+
+    /* N.B. in older code, if states == 0, then set states=<all the states>.
+     * The equivalent in constraints is to not set a constraint. */
+    if (states) {
+        o = json_pack ("{s:[i]}", "states", states);
+        if (!o || json_array_append_new (a, o) < 0) {
+            json_decref (o);
+            goto error;
+        }
+    }
+
+    /* N.B. in older code, if results == 0, then set states=<all the results>.
+     * The equivalent in constraints is to not set a constraint. */
+    if (results) {
+        o = json_pack ("{s:[i]}", "results", results);
+        if (!o || json_array_append_new (a, o) < 0) {
+            json_decref (o);
+            goto error;
+        }
+    }
+
+    if (!((*legacy_constraint) = json_pack ("{s:o}", "and", a)))
+        goto error;
+
+    return 0;
+
+error:
+    json_decref (a);
+    return -1;
+}
+
+void list_cb (flux_t *h,
+              flux_msg_handler_t *mh,
+              const flux_msg_t *msg,
+              void *arg)
+{
+    struct list_ctx *ctx = arg;
+    flux_error_t err;
+    json_t *jobs;
+    json_t *attrs;
+    int max_entries;
+    double since = 0.;
+    json_t *constraint = NULL;
+    json_t *legacy_constraint = NULL;
+    struct list_constraint *c = NULL;
+    struct state_constraint *statec = NULL;
+    flux_error_t error;
+
+    if (!ctx->jsctx->initialized) {
+        if (flux_msglist_append (ctx->deferred_requests, msg) < 0)
+            goto error;
+        return;
+    }
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s:o s?F s?o}",
+                             "max_entries", &max_entries,
+                             "attrs", &attrs,
+                             "since", &since,
+                             "constraint", &constraint) < 0) {
+        errprintf (&err, "invalid payload: %s", flux_msg_last_error (msg));
+        errno = EPROTO;
+        goto error;
+    }
+    if (!constraint) {
+        /* Double check for legacy RPC fields since "constraint"
+         * object is optional in new protocol. */
+        int tmp_max_entries;
+        json_t *tmp_attrs;
+        double tmp_since = 0.;
+        if (!legacy_list_rpc (h,
+                              mh,
+                              msg,
+                              ctx,
+                              &tmp_max_entries,
+                              &tmp_attrs,
+                              &tmp_since,
+                              &legacy_constraint)) {
+            max_entries = tmp_max_entries;
+            attrs = tmp_attrs;
+            since = tmp_since;
+            constraint = legacy_constraint;
+        }
+    }
+    if (max_entries < 0) {
+        errprintf (&err, "invalid payload: max_entries < 0 not allowed");
+        errno = EPROTO;
+        goto error;
+    }
+    if (since < 0.) {
+        errprintf (&err, "invalid payload: since < 0.0 not allowed");
+        errno = EPROTO;
+        goto error;
+    }
+    if (!json_is_array (attrs)) {
+        errprintf (&err, "invalid payload: attrs must be an array");
+        errno = EPROTO;
+        goto error;
+    }
+    if (!(c = list_constraint_create (ctx->mctx, constraint, &error))) {
+        errprintf (&err,
+                   "invalid payload: constraint object invalid: %s",
+                   error.text);
+        errno = EPROTO;
+        goto error;
+    }
+    if (!(statec = state_constraint_create (constraint, &error))) {
+        errprintf (&err,
+                   "invalid payload: constraint object invalid: %s",
+                   error.text);
+        errno = EPROTO;
+        goto error;
+    }
+
+    if (!(jobs = get_jobs (ctx->jsctx, &err, max_entries, since,
+                           attrs, c, statec)))
+        goto error;
+
+    if (flux_respond_pack (h, msg, "{s:O}", "jobs", jobs) < 0)
+        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+
+    json_decref (jobs);
+    list_constraint_destroy (c);
+    state_constraint_destroy (statec);
+    json_decref (legacy_constraint);
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, err.text) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    list_constraint_destroy (c);
+    state_constraint_destroy (statec);
+    json_decref (legacy_constraint);
+}
+
+void check_id_valid_continuation (flux_future_t *f, void *arg)
+{
+    struct idsync_data *isd = arg;
+    struct job_state_ctx *jsctx = flux_future_aux_get (f, "job_state_ctx");
+
+    assert (jsctx);
+
+    if (flux_future_get (f, NULL) < 0) {
+        if (flux_respond_error (jsctx->h, isd->msg, errno, NULL) < 0)
+            flux_log_error (jsctx->h, "%s: flux_respond_error", __FUNCTION__);
+        goto cleanup;
+    }
+    else {
+        /* Job ID is legal.  Chance job-list has seen ID since this
+         * lookup was done */
+        struct job *job;
+        if (!(job = zhashx_lookup (jsctx->index, &isd->id))
+            || job->state == FLUX_JOB_STATE_NEW) {
+            /* Must wait for job-list to see state change */
+            if (idsync_wait_valid (jsctx->ctx->isctx, isd) < 0)
+                flux_log_error (jsctx->h, "%s: idsync_wait_valid", __FUNCTION__);
+            return;
+        }
+        else {
+            json_t *o;
+            if (!(o = get_job_by_id (jsctx,
+                                     NULL,
+                                     isd->msg,
+                                     isd->id,
+                                     isd->attrs,
+                                     isd->state,
+                                     NULL))) {
+                flux_log_error (jsctx->h, "%s: get_job_by_id", __FUNCTION__);
+                goto cleanup;
+            }
+            if (flux_respond_pack (jsctx->h, isd->msg, "{s:O}", "job", o) < 0) {
+                json_decref (o);
+                flux_log_error (jsctx->h,
+                                "%s: flux_respond_pack",
+                                __FUNCTION__);
+                goto cleanup;
+            }
+            json_decref (o);
+        }
+    }
+
+cleanup:
+    /* will free isd memory */
+    idsync_check_id_valid_cleanup (jsctx->ctx->isctx, isd);
+    return;
+}
+
+int check_id_valid (struct job_state_ctx *jsctx,
+                    const flux_msg_t *msg,
+                    flux_jobid_t id,
+                    json_t *attrs,
+                    flux_job_state_t state)
+{
+    struct idsync_data *isd = NULL;
+
+    if (!(isd = idsync_check_id_valid (jsctx->ctx->isctx,
+                                       id,
+                                       msg,
+                                       attrs,
+                                       state))
+        || flux_future_aux_set (isd->f_lookup,
+                                "job_state_ctx",
+                                jsctx,
+                                NULL) < 0
+        || flux_future_then (isd->f_lookup,
+                             -1,
+                             check_id_valid_continuation,
+                             isd) < 0) {
+        idsync_data_destroy (isd);
+        return -1;
+    }
+
+    return 0;
+}
+
+/* Returns JSON object which the caller must free.  On error, return
+ * NULL with errno set:
+ *
+ * EPROTO - malformed or empty id or attrs array
+ * EINVAL - invalid id
+ * ENOMEM - out of memory
+ */
+json_t *get_job_by_id (struct job_state_ctx *jsctx,
+                       flux_error_t *errp,
+                       const flux_msg_t *msg,
+                       flux_jobid_t id,
+                       json_t *attrs,
+                       flux_job_state_t state,
+                       bool *stall)
+{
+    struct job *job;
+
+    if (!(job = zhashx_lookup (jsctx->index, &id))) {
+        if (stall) {
+            if (check_id_valid (jsctx, msg, id, attrs, state) < 0) {
+                flux_log_error (jsctx->h, "%s: check_id_valid", __FUNCTION__);
+                return NULL;
+            }
+            (*stall) = true;
+        }
+        return NULL;
+    }
+
+    /*  Always return job in inactive state, even if a requested state was
+     *  provided. This avoids no response when a job does not enter a given
+     *  state before becoming inactive, e.g. when a pending job is canceled.
+     */
+    if (job->state == FLUX_JOB_STATE_INACTIVE)
+        return job_to_json (job, attrs, errp);
+
+    /*  Otherwise, wait for given state if the job is still NEW or a specific
+     *  state was requested that has not yet in the job states mask:
+     */
+    if ((state && !(job->states_mask & state))
+        || job->state == FLUX_JOB_STATE_NEW) {
+        if (stall) {
+            /* Must wait for job-list to see state change */
+            if (idsync_wait_valid_id (jsctx->ctx->isctx,
+                                      id,
+                                      msg,
+                                      attrs,
+                                      state) < 0) {
+                flux_log_error (jsctx->h,
+                                "%s: idsync_wait_valid_id",
+                                __FUNCTION__);
+                return NULL;
+            }
+            (*stall) = true;
+        }
+        return NULL;
+    }
+
+    return job_to_json (job, attrs, errp);
+}
+
+void list_id_cb (flux_t *h,
+                 flux_msg_handler_t *mh,
+                 const flux_msg_t *msg,
+                 void *arg)
+{
+    struct list_ctx *ctx = arg;
+    flux_error_t err = {{0}};
+    json_t *job;
+    flux_jobid_t id;
+    json_t *attrs;
+    int state = 0;
+    int valid_states = FLUX_JOB_STATE_ACTIVE | FLUX_JOB_STATE_INACTIVE;
+    bool stall = false;
+
+    if (!ctx->jsctx->initialized) {
+        if (flux_msglist_append (ctx->deferred_requests, msg) < 0)
+            goto error;
+        return;
+    }
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:o s?i}",
+                             "id", &id,
+                             "attrs", &attrs,
+                             "state", &state) < 0) {
+        errprintf (&err, "invalid payload: %s", flux_msg_last_error (msg));
+        errno = EPROTO;
+        goto error;
+    }
+
+    if (!json_is_array (attrs)) {
+        errprintf (&err, "invalid payload: attrs must be an array");
+        errno = EPROTO;
+        goto error;
+    }
+
+    if (state && (state & ~valid_states)) {
+        errprintf (&err, "invalid payload: invalid state specified");
+        errno = EPROTO;
+        goto error;
+    }
+
+    if (!(job = get_job_by_id (ctx->jsctx,
+                               &err,
+                               msg,
+                               id,
+                               attrs,
+                               state,
+                               &stall))) {
+        /* response handled after KVS lookup complete */
+        if (stall)
+            goto stall;
+        goto error;
+    }
+
+    if (flux_respond_pack (h, msg, "{s:O}", "job", job) < 0)
+        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+
+    json_decref (job);
+stall:
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, err.text) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+static int list_attrs_append (json_t *a, const char *attr)
+{
+    json_t *o = json_string (attr);
+    if (!o) {
+        errno = ENOMEM;
+        return -1;
+    }
+    if (json_array_append_new (a, o) < 0) {
+        json_decref (o);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+void list_attrs_cb (flux_t *h,
+                    flux_msg_handler_t *mh,
+                    const flux_msg_t *msg,
+                    void *arg)
+{
+    struct list_ctx *ctx = arg;
+    const char **attrs;
+    json_t *a = NULL;
+    int i;
+
+    if (!ctx->jsctx->initialized) {
+        if (flux_msglist_append (ctx->deferred_requests, msg) < 0)
+            goto error;
+        return;
+    }
+    if (!(a = json_array ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+
+    attrs = job_attrs ();
+    assert (attrs);
+
+    for (i = 0; attrs[i] != NULL; i++) {
+        if (list_attrs_append (a, attrs[i]) < 0)
+            goto error;
+    }
+
+    if (list_attrs_append (a, "all") < 0)
+        goto error;
+
+    if (flux_respond_pack (h, msg, "{s:O}", "attrs", a) < 0)
+        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+
+    json_decref (a);
+    return;
+
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    json_decref (a);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/list.h b/src/modules/job-list/list.h
new file mode 100644
index 000000000000..d052c22237e0
--- /dev/null
+++ b/src/modules/job-list/list.h
@@ -0,0 +1,29 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_LIST_LIST_H
+#define _FLUX_JOB_LIST_LIST_H
+
+#include <flux/core.h>
+
+void list_cb (flux_t *h, flux_msg_handler_t *mh,
+              const flux_msg_t *msg, void *arg);
+
+void list_id_cb (flux_t *h, flux_msg_handler_t *mh,
+                 const flux_msg_t *msg, void *arg);
+
+void list_attrs_cb (flux_t *h, flux_msg_handler_t *mh,
+                    const flux_msg_t *msg, void *arg);
+
+#endif /* ! _FLUX_JOB_LIST_LIST_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-list/match.c b/src/modules/job-list/match.c
new file mode 100644
index 000000000000..c7697c3c92d6
--- /dev/null
+++ b/src/modules/job-list/match.c
@@ -0,0 +1,963 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <jansson.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "match.h"
+#include "match_util.h"
+#include "job_util.h"
+
+typedef int (*match_f) (struct list_constraint *,
+                        const struct job *,
+                        unsigned int *,
+                        flux_error_t *);
+
+struct list_constraint {
+    struct match_ctx *mctx;
+    zlistx_t *values;
+    match_f match;
+    unsigned int comparisons;   /* total across multiple calls to job_match() */
+};
+
+typedef enum {
+    MATCH_T_SUBMIT = 1,
+    MATCH_T_DEPEND = 2,
+    MATCH_T_RUN = 3,
+    MATCH_T_CLEANUP = 4,
+    MATCH_T_INACTIVE = 5,
+} match_timestamp_type_t;
+
+typedef enum {
+    MATCH_GREATER_THAN_EQUAL = 1,
+    MATCH_LESS_THAN_EQUAL = 2,
+    MATCH_GREATER_THAN = 3,
+    MATCH_LESS_THAN = 4,
+} match_comparison_t;
+
+#define MIN_MATCH_HOSTLIST 1024
+
+struct timestamp_value {
+    double t_value;
+    match_timestamp_type_t t_type;
+    match_comparison_t t_comp;
+};
+
+#define CONSTRAINT_COMPARISON_MAX 1000000
+
+static inline int inc_check_comparison (struct match_ctx *mctx,
+                                        unsigned int *comparisons,
+                                        flux_error_t *errp)
+{
+    if (mctx->max_comparisons
+        && (++(*comparisons)) > mctx->max_comparisons) {
+        errprintf (errp,
+                   "Excessive comparisons made, "
+                   "limit search via states or since");
+        return -1;
+    }
+    return 0;
+}
+
+static void timestamp_value_destroy (void *data)
+{
+    if (data) {
+        int save_errno = errno;
+        free (data);
+        errno = save_errno;
+    }
+}
+
+/* zlistx_set_destructor */
+static void wrap_timestamp_value_destroy (void **item)
+{
+    if (item) {
+        struct timestamp_value *tv = *item;
+        timestamp_value_destroy (tv);
+        (*item) = NULL;
+    }
+}
+
+static struct timestamp_value *timestamp_value_create (
+    double t_value,
+    const char *type,
+    match_comparison_t comp)
+{
+    struct timestamp_value *tv;
+
+    if (!(tv = calloc (1, sizeof (*tv))))
+        return NULL;
+    tv->t_value = t_value;
+
+    if (streq (type, "t_submit"))
+        tv->t_type = MATCH_T_SUBMIT;
+    else if (streq (type, "t_depend"))
+        tv->t_type = MATCH_T_DEPEND;
+    else if (streq (type, "t_run"))
+        tv->t_type = MATCH_T_RUN;
+    else if (streq (type, "t_cleanup"))
+        tv->t_type = MATCH_T_CLEANUP;
+    else if (streq (type, "t_inactive"))
+        tv->t_type = MATCH_T_INACTIVE;
+    else
+        goto cleanup;
+
+    tv->t_comp = comp;
+    return tv;
+
+cleanup:
+    timestamp_value_destroy (tv);
+    return NULL;
+}
+
+static struct timestamp_value *timestamp_value_create_str (
+    const char *t_value,
+    const char *type,
+    match_comparison_t comp,
+    flux_error_t *errp)
+{
+    double t;
+    char *endptr;
+    errno = 0;
+    t = strtod (t_value, &endptr);
+    if (errno != 0 || *endptr != '\0') {
+        errprintf (errp, "Invalid timestamp value specified");
+        return NULL;
+    }
+    if (t < 0.0) {
+        errprintf (errp, "timestamp value must be >= 0.0");
+        return NULL;
+    }
+    return timestamp_value_create (t, type, comp);
+}
+
+static int match_true (struct list_constraint *c,
+                       const struct job *job,
+                       unsigned int *comparisons,
+                       flux_error_t *errp)
+{
+    if (inc_check_comparison (c->mctx, comparisons, errp) < 0)
+        return -1;
+    return 1;
+}
+
+static struct list_constraint *list_constraint_new (struct match_ctx *mctx,
+                                                    match_f match_cb,
+                                                    destructor_f destructor_cb,
+                                                    flux_error_t *errp)
+{
+    struct list_constraint *c;
+    if (!(c = calloc (1, sizeof (*c)))
+        || !(c->values = zlistx_new ())) {
+        list_constraint_destroy (c);
+        errprintf (errp, "Out of memory");
+        return NULL;
+    }
+    c->mctx = mctx;
+    c->match = match_cb;
+    if (destructor_cb)
+        zlistx_set_destructor (c->values, destructor_cb);
+    return c;
+}
+
+static void list_constraint_destructor (void **item)
+{
+    if (item) {
+        list_constraint_destroy (*item);
+        *item = NULL;
+    }
+}
+
+/* zlistx_set_destructor */
+static void wrap_free (void **item)
+{
+    if (item) {
+        free (*item);
+        (*item) = NULL;
+    }
+}
+
+static int match_userid (struct list_constraint *c,
+                         const struct job *job,
+                         unsigned int *comparisons,
+                         flux_error_t *errp)
+{
+    uint32_t *userid = zlistx_first (c->values);
+    while (userid) {
+        if (inc_check_comparison (c->mctx, comparisons, errp) < 0)
+            return -1;
+        if ((*userid) == FLUX_USERID_UNKNOWN)
+            return 1;
+        if ((*userid) == job->userid)
+            return 1;
+        userid = zlistx_next (c->values);
+    }
+    return 0;
+}
+
+static struct list_constraint *create_userid_constraint (struct match_ctx *mctx,
+                                                         json_t *values,
+                                                         flux_error_t *errp)
+{
+    struct list_constraint *c;
+    json_t *entry;
+    size_t index;
+
+    if (!(c = list_constraint_new (mctx, match_userid, wrap_free, errp)))
+        return NULL;
+    json_array_foreach (values, index, entry) {
+        uint32_t *userid;
+        if (!json_is_integer (entry)) {
+            errprintf (errp, "userid value must be an integer");
+            goto error;
+        }
+        if (!(userid = malloc (sizeof (*userid))))
+            goto error;
+        (*userid) = json_integer_value (entry);
+        if (!zlistx_add_end (c->values, userid)) {
+            free (userid);
+            goto error;
+        }
+    }
+    return c;
+ error:
+    list_constraint_destroy (c);
+    return NULL;
+}
+
+static struct list_constraint *create_string_constraint (struct match_ctx *mctx,
+                                                         const char *op,
+                                                         json_t *values,
+                                                         match_f match_cb,
+                                                         flux_error_t *errp)
+{
+    struct list_constraint *c;
+    json_t *entry;
+    size_t index;
+
+    if (!(c = list_constraint_new (mctx, match_cb, wrap_free, errp)))
+        return NULL;
+    json_array_foreach (values, index, entry) {
+        char *s = NULL;
+        if (!json_is_string (entry)) {
+            errprintf (errp, "%s value must be a string", op);
+            goto error;
+        }
+        if (!(s = strdup (json_string_value (entry))))
+            return NULL;
+        if (!zlistx_add_end (c->values, s)) {
+            free (s);
+            goto error;
+        }
+    }
+    return c;
+ error:
+    list_constraint_destroy (c);
+    return NULL;
+}
+
+static int match_name (struct list_constraint *c,
+                       const struct job *job,
+                       unsigned int *comparisons,
+                       flux_error_t *errp)
+{
+    const char *name = zlistx_first (c->values);
+    while (name) {
+        if (inc_check_comparison (c->mctx, comparisons, errp) < 0)
+            return -1;
+        if (job->name && streq (name, job->name))
+            return 1;
+        name = zlistx_next (c->values);
+    }
+    return 0;
+}
+
+static struct list_constraint *create_name_constraint (struct match_ctx *mctx,
+                                                       json_t *values,
+                                                       flux_error_t *errp)
+{
+    return create_string_constraint (mctx, "name", values, match_name, errp);
+}
+
+static int match_queue (struct list_constraint *c,
+                        const struct job *job,
+                        unsigned int *comparisons,
+                        flux_error_t *errp)
+{
+    const char *queue = zlistx_first (c->values);
+    while (queue) {
+        if (inc_check_comparison (c->mctx, comparisons, errp) < 0)
+            return -1;
+        if (job->queue && streq (queue, job->queue))
+            return 1;
+        queue = zlistx_next (c->values);
+    }
+    return 0;
+}
+
+static struct list_constraint *create_queue_constraint (struct match_ctx *mctx,
+                                                        json_t *values,
+                                                        flux_error_t *errp)
+{
+    return create_string_constraint (mctx, "queue", values, match_queue, errp);
+}
+
+static struct list_constraint *create_bitmask_constraint (
+    struct match_ctx *mctx,
+    const char *op,
+    json_t *values,
+    match_f match_cb,
+    array_to_bitmask_f array_to_bitmask_cb,
+    flux_error_t *errp)
+{
+    struct list_constraint *c;
+    int *bitmask = NULL;
+    int tmp;
+    if ((tmp = array_to_bitmask_cb (values, errp)) < 0)
+        return NULL;
+    if (!(bitmask = malloc (sizeof (*bitmask))))
+        return NULL;
+    (*bitmask) = tmp;
+    if (!(c = list_constraint_new (mctx, match_cb, wrap_free, errp))
+        || !zlistx_add_end (c->values, bitmask)) {
+        list_constraint_destroy (c);
+        free (bitmask);
+        return NULL;
+    }
+    return c;
+}
+
+static int match_states (struct list_constraint *c,
+                         const struct job *job,
+                         unsigned int *comparisons,
+                         flux_error_t *errp)
+{
+    int *states = zlistx_first (c->values);
+    if (inc_check_comparison (c->mctx, comparisons, errp) < 0)
+        return -1;
+    return ((*states) & job->state) ? 1 : 0;
+}
+
+static struct list_constraint *create_states_constraint (struct match_ctx *mctx,
+                                                         json_t *values,
+                                                         flux_error_t *errp)
+{
+    return create_bitmask_constraint (mctx,
+                                      "states",
+                                      values,
+                                      match_states,
+                                      array_to_states_bitmask,
+                                      errp);
+}
+
+static int match_results (struct list_constraint *c,
+                          const struct job *job,
+                          unsigned int *comparisons,
+                          flux_error_t *errp)
+{
+    int *results = zlistx_first (c->values);
+    if (inc_check_comparison (c->mctx, comparisons, errp) < 0)
+        return -1;
+    if (job->state != FLUX_JOB_STATE_INACTIVE)
+        return 0;
+    return ((*results) & job->result) ? 1 : 0;
+}
+
+static int array_to_results_bitmask (json_t *values, flux_error_t *errp)
+{
+    int results = 0;
+    json_t *entry;
+    size_t index;
+    int valid_results = (FLUX_JOB_RESULT_COMPLETED
+                         | FLUX_JOB_RESULT_FAILED
+                         | FLUX_JOB_RESULT_CANCELED
+                         | FLUX_JOB_RESULT_TIMEOUT);
+
+    json_array_foreach (values, index, entry) {
+        flux_job_result_t result;
+        if (json_is_string (entry)) {
+            const char *resultstr = json_string_value (entry);
+            if (flux_job_strtoresult (resultstr, &result) < 0) {
+                errprintf (errp,
+                           "invalid results value '%s' specified",
+                           resultstr);
+                return -1;
+            }
+        }
+        else if (json_is_integer (entry)) {
+            result = json_integer_value (entry);
+            if (result & ~valid_results) {
+                errprintf (errp,
+                           "invalid results value '%Xh' specified",
+                           result);
+                return -1;
+            }
+        }
+        else {
+            errprintf (errp, "results value invalid type");
+            return -1;
+        }
+        results |= result;
+    }
+    return results;
+}
+
+static struct list_constraint *create_results_constraint (struct match_ctx *mctx,
+                                                          json_t *values,
+                                                          flux_error_t *errp)
+{
+    return create_bitmask_constraint (mctx,
+                                      "results",
+                                      values,
+                                      match_results,
+                                      array_to_results_bitmask,
+                                      errp);
+}
+
+static int match_hostlist (struct list_constraint *c,
+                           const struct job *job,
+                           unsigned int *comparisons,
+                           flux_error_t *errp)
+{
+    struct hostlist *hl = zlistx_first (c->values);
+    const char *host;
+
+    /* nodelist may not exist if job never ran */
+    if (!job->nodelist)
+        return 0;
+    if (!job->nodelist_hl) {
+        /* hack to remove const */
+        struct job *jobtmp = (struct job *)job;
+        if (!(jobtmp->nodelist_hl = hostlist_decode (job->nodelist)))
+            return 0;
+    }
+    host = hostlist_first (hl);
+    while (host) {
+        if (inc_check_comparison (c->mctx, comparisons, errp) < 0)
+            return -1;
+        if (hostlist_find (job->nodelist_hl, host) >= 0)
+            return 1;
+        host = hostlist_next (hl);
+    }
+    return 0;
+}
+
+/* zlistx_set_destructor */
+static void wrap_hostlist_destroy (void **item)
+{
+    if (item) {
+        struct hostlist *hl = *item;
+        hostlist_destroy (hl);
+        (*item) = NULL;
+    }
+}
+
+static struct list_constraint *create_hostlist_constraint (
+    struct match_ctx *mctx,
+    json_t *values,
+    flux_error_t *errp)
+{
+    struct list_constraint *c;
+    struct hostlist *hl = NULL;
+    json_t *entry;
+    size_t index;
+
+    if (!(c = list_constraint_new (mctx,
+                                   match_hostlist,
+                                   wrap_hostlist_destroy,
+                                   errp)))
+        return NULL;
+    /* Create a single hostlist if user specifies multiple nodes or
+     * RFC29 hostlist range */
+    if (!(hl = hostlist_create ())) {
+        errprintf (errp, "failed to create hostlist structure");
+        goto error;
+    }
+    json_array_foreach (values, index, entry) {
+        if (!json_is_string (entry)) {
+            errprintf (errp, "host value must be a string");
+            goto error;
+        }
+        if (hostlist_append (hl, json_string_value (entry)) <= 0) {
+            errprintf (errp, "host value not in valid Hostlist format");
+            goto error;
+        }
+    }
+    if (hostlist_count (hl) > mctx->max_hostlist) {
+        errprintf (errp, "too many hosts specified");
+        goto error;
+    }
+    if (!zlistx_add_end (c->values, hl)) {
+        errprintf (errp, "failed to append hostlist structure");
+        goto error;
+    }
+    return c;
+ error:
+    hostlist_destroy (hl);
+    list_constraint_destroy (c);
+    return NULL;
+}
+
+static int match_ranks (struct list_constraint *c,
+                        const struct job *job,
+                        unsigned int *comparisons,
+                        flux_error_t *errp)
+{
+    struct idset *idset = zlistx_first (c->values);
+    size_t n, m;
+
+    /* ranks may not exist if job never ran */
+    if (!job->ranks)
+        return 0;
+    if (!job->ranks_idset) {
+        /* hack to remove const */
+        struct job *jobtmp = (struct job *)job;
+        if (!(jobtmp->ranks_idset = idset_decode (job->ranks)))
+            return 0;
+    }
+    /* Account for all ranks being compared before calling
+     * inc_check_comparison. This is the smallest of the job or
+     * comparison idset
+     */
+    m = idset_count (job->ranks_idset);
+    n = idset_count (idset);
+    *comparisons += (m < n ? m : n) - 1;
+    if (inc_check_comparison (c->mctx, comparisons, errp) < 0)
+        return -1;
+    return idset_has_intersection (job->ranks_idset, idset);
+}
+
+
+/* zlistx_set_destructor */
+static void wrap_idset_destroy (void **item)
+{
+    if (item) {
+        struct idset *idset = *item;
+        idset_destroy (idset);
+        (*item) = NULL;
+    }
+}
+
+static struct list_constraint *create_ranks_constraint (
+    struct match_ctx *mctx,
+    json_t *values,
+    flux_error_t *errp)
+{
+    struct list_constraint *c;
+    struct idset *idset = NULL;
+    json_t *entry;
+    size_t index;
+
+    if (!(c = list_constraint_new (mctx,
+                                   match_ranks,
+                                   wrap_idset_destroy,
+                                   errp)))
+        return NULL;
+
+    if (!(idset = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+        errprintf (errp, "failed to create idset structure");
+        goto error;
+    }
+    json_array_foreach (values, index, entry) {
+        const char *ids;
+        idset_error_t error;
+        if (!json_is_string (entry)
+            || !(ids = json_string_value (entry))) {
+            errprintf (errp, "ranks value must be a string");
+            goto error;
+        }
+        if (idset_decode_add (idset, ids, -1, &error) < 0) {
+            errprintf (errp, "ranks value '%s': %s", ids, error.text);
+            goto error;
+        }
+    }
+    if (idset_count (idset) > mctx->max_hostlist) {
+        errprintf (errp, "too many ranks specified");
+        goto error;
+    }
+    if (!zlistx_add_end (c->values, idset)) {
+        errprintf (errp, "failed to append idset structure");
+        goto error;
+    }
+    return c;
+ error:
+    idset_destroy (idset);
+    list_constraint_destroy (c);
+    return NULL;
+}
+
+
+static int match_timestamp (struct list_constraint *c,
+                            const struct job *job,
+                            unsigned int *comparisons,
+                            flux_error_t *errp)
+{
+    struct timestamp_value *tv = zlistx_first (c->values);
+    double t;
+
+    if (inc_check_comparison (c->mctx, comparisons, errp) < 0)
+        return -1;
+
+    if (tv->t_type == MATCH_T_SUBMIT)
+        t = job->t_submit;
+    else if (tv->t_type == MATCH_T_DEPEND) {
+        /* if submit_version < 1, it means it was not set.  This is
+         * before the introduction of event `validate` after 0.41.1.
+         * Before the introduction of this event, t_submit and
+         * t_depend are the same.
+         */
+        if (job->submit_version < 1)
+            t = job->t_submit;
+        else if (job->states_mask & FLUX_JOB_STATE_DEPEND)
+            t = job->t_depend;
+        else
+            return 0;
+    }
+    else if (tv->t_type == MATCH_T_RUN
+             && (job->states_mask & FLUX_JOB_STATE_RUN))
+        t = job->t_run;
+    else if (tv->t_type == MATCH_T_CLEANUP
+             && (job->states_mask & FLUX_JOB_STATE_CLEANUP))
+        t = job->t_cleanup;
+    else if (tv->t_type == MATCH_T_INACTIVE
+             && (job->states_mask & FLUX_JOB_STATE_INACTIVE))
+        t = job->t_inactive;
+    else
+        return 0;
+
+    if (tv->t_comp == MATCH_GREATER_THAN_EQUAL)
+        return t >= tv->t_value;
+    else if (tv->t_comp == MATCH_LESS_THAN_EQUAL)
+        return t <= tv->t_value;
+    else if (tv->t_comp == MATCH_GREATER_THAN)
+        return t > tv->t_value;
+    else /* tv->t_comp == MATCH_LESS_THAN */
+        return t < tv->t_value;
+}
+
+static struct list_constraint *create_timestamp_constraint (struct match_ctx *mctx,
+                                                            const char *type,
+                                                            json_t *values,
+                                                            flux_error_t *errp)
+{
+    struct timestamp_value *tv = NULL;
+    struct list_constraint *c;
+    const char *str;
+    json_t *v = json_array_get (values, 0);
+
+    if (!v) {
+        errprintf (errp, "timestamp value not specified");
+        return NULL;
+    }
+    if (!json_is_string (v)) {
+        errprintf (errp, "%s value must be a string", type);
+        return NULL;
+    }
+    str = json_string_value (v);
+    if (strstarts (str, ">="))
+        tv = timestamp_value_create_str (str + 2,
+                                         type,
+                                         MATCH_GREATER_THAN_EQUAL,
+                                         errp);
+    else if (strstarts (str, "<="))
+        tv = timestamp_value_create_str (str + 2,
+                                         type,
+                                         MATCH_LESS_THAN_EQUAL,
+                                         errp);
+    else if (strstarts (str, ">"))
+        tv = timestamp_value_create_str (str + 1,
+                                         type,
+                                         MATCH_GREATER_THAN,
+                                         errp);
+    else if (strstarts (str, "<"))
+        tv = timestamp_value_create_str (str + 1,
+                                         type,
+                                         MATCH_LESS_THAN,
+                                         errp);
+    else
+        errprintf (errp, "timestamp comparison operator not specified");
+
+    if (!tv)
+        return NULL;
+
+    if (!(c = list_constraint_new (mctx,
+                                   match_timestamp,
+                                   wrap_timestamp_value_destroy,
+                                   errp))
+        || !zlistx_add_end (c->values, tv)) {
+        list_constraint_destroy (c);
+        timestamp_value_destroy (tv);
+        return NULL;
+    }
+    return c;
+}
+
+static int match_and (struct list_constraint *c,
+                      const struct job *job,
+                      unsigned int *comparisons,
+                      flux_error_t *errp)
+{
+    struct list_constraint *cp = zlistx_first (c->values);
+    while (cp) {
+        int ret = cp->match (cp, job, comparisons, errp);
+        /* i.e. return immediately if false or error */
+        if (ret != 1)
+            return ret;
+        cp = zlistx_next (c->values);
+    }
+    return 1;
+}
+
+static int match_or (struct list_constraint *c,
+                     const struct job *job,
+                     unsigned int *comparisons,
+                     flux_error_t *errp)
+{
+    struct list_constraint *cp = zlistx_first (c->values);
+    /* no values in "or" defined as true per RFC31 */
+    if (!cp)
+        return 1;
+    while (cp) {
+        int ret = cp->match (cp, job, comparisons, errp);
+        /* i.e. return immediately if true or error */
+        if (ret != 0)
+            return ret;
+        cp = zlistx_next (c->values);
+    }
+    return 0;
+}
+
+static int match_not (struct list_constraint *c,
+                      const struct job *job,
+                      unsigned int *comparisons,
+                      flux_error_t *errp)
+{
+    int ret;
+    if ((ret = match_and (c, job, comparisons, errp)) < 0)
+        return -1;
+    return ret ? 0 : 1;
+}
+
+static struct list_constraint *conditional_constraint (struct match_ctx *mctx,
+                                                       const char *type,
+                                                       json_t *values,
+                                                       flux_error_t *errp)
+{
+    json_t *entry;
+    size_t index;
+    struct list_constraint *c;
+    match_f match_cb;
+
+    if (streq (type, "and"))
+        match_cb = match_and;
+    else if (streq (type, "or"))
+        match_cb = match_or;
+    else /* streq (type, "not") */
+        match_cb = match_not;
+
+    if (!(c = list_constraint_new (mctx,
+                                   match_cb,
+                                   list_constraint_destructor,
+                                   errp)))
+        return NULL;
+
+    json_array_foreach (values, index, entry) {
+        struct list_constraint *cp = list_constraint_create (mctx, entry, errp);
+        if (!cp)
+            goto error;
+        if (!zlistx_add_end (c->values, cp)) {
+            errprintf (errp, "Out of memory");
+            list_constraint_destroy (cp);
+            goto error;
+        }
+    }
+    return c;
+
+ error:
+    list_constraint_destroy (c);
+    return NULL;
+}
+
+void list_constraint_destroy (struct list_constraint *constraint)
+{
+    if (constraint) {
+        int saved_errno = errno;
+        zlistx_destroy (&constraint->values);
+        free (constraint);
+        errno = saved_errno;
+    }
+}
+
+struct list_constraint *list_constraint_create (struct match_ctx *mctx,
+                                                json_t *constraint,
+                                                flux_error_t *errp)
+{
+    const char *op;
+    json_t *values;
+
+    if (!mctx) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (constraint) {
+        if (!json_is_object (constraint)) {
+            errprintf (errp, "constraint must be JSON object");
+            return NULL;
+        }
+        if (json_object_size (constraint) > 1) {
+            errprintf (errp, "constraint must only contain 1 element");
+            return NULL;
+        }
+        json_object_foreach (constraint, op, values) {
+            if (!json_is_array (values)) {
+                errprintf (errp, "operator %s values not an array", op);
+                return NULL;
+            }
+            if (streq (op, "userid"))
+                return create_userid_constraint (mctx, values, errp);
+            else if (streq (op, "name"))
+                return create_name_constraint (mctx, values, errp);
+            else if (streq (op, "queue"))
+                return create_queue_constraint (mctx, values, errp);
+            else if (streq (op, "states"))
+                return create_states_constraint (mctx, values, errp);
+            else if (streq (op, "results"))
+                return create_results_constraint (mctx, values, errp);
+            else if (streq (op, "hostlist"))
+                return create_hostlist_constraint (mctx, values, errp);
+            else if (streq (op, "ranks"))
+                return create_ranks_constraint (mctx, values, errp);
+            else if (streq (op, "t_submit")
+                     || streq (op, "t_depend")
+                     || streq (op, "t_run")
+                     || streq (op, "t_cleanup")
+                     || streq (op, "t_inactive"))
+                return create_timestamp_constraint (mctx, op, values, errp);
+            else if (streq (op, "or") || streq (op, "and") || streq (op, "not"))
+                return conditional_constraint (mctx, op, values, errp);
+            else {
+                errprintf (errp, "unknown constraint operator: %s", op);
+                return NULL;
+            }
+        }
+    }
+    return list_constraint_new (mctx, match_true, NULL, errp);
+}
+
+int job_match (const struct job *job,
+               struct list_constraint *constraint,
+               flux_error_t *errp)
+{
+    if (!job || !constraint) {
+        errno = EINVAL;
+        return -1;
+    }
+    return constraint->match (constraint, job, &constraint->comparisons, errp);
+}
+
+static int config_parse_max_comparisons (struct match_ctx *mctx,
+                                         const flux_conf_t *conf,
+                                         flux_error_t *errp)
+{
+    int64_t max_comparisons = CONSTRAINT_COMPARISON_MAX;
+    flux_error_t error;
+
+    if (flux_conf_unpack (conf,
+                          &error,
+                          "{s?{s?I}}",
+                          "job-list",
+                          "max_comparisons", &max_comparisons) < 0) {
+        errprintf (errp,
+                   "error reading config for job-list: %s",
+                   error.text);
+        return -1;
+    }
+
+    if (max_comparisons < 0) {
+        errprintf (errp, "job-list.max_comparisons must be >= 0");
+        return -1;
+    }
+
+    mctx->max_comparisons = max_comparisons;
+    return 0;
+}
+
+int job_match_config_reload (struct match_ctx *mctx,
+                             const flux_conf_t *conf,
+                             flux_error_t *errp)
+{
+    return config_parse_max_comparisons (mctx, conf, errp);
+}
+
+struct match_ctx *match_ctx_create (flux_t *h)
+{
+    struct match_ctx *mctx = NULL;
+    flux_error_t error;
+
+    if (!(mctx = calloc (1, sizeof (*mctx))))
+        return NULL;
+    mctx->h = h;
+
+    if (config_parse_max_comparisons (mctx,
+                                      flux_get_conf (mctx->h),
+                                      &error) < 0) {
+        flux_log (mctx->h, LOG_ERR, "%s", error.text);
+        goto error;
+    }
+
+    if (flux_get_size (mctx->h, &mctx->max_hostlist) < 0) {
+        flux_log_error (h, "failed to get instance size");
+        goto error;
+    }
+
+    /* Notes:
+     *
+     * We do not want a hostlist constraint match to DoS this module.
+     * So we want to configure a "max" amount of hosts that can exist
+     * within a hostlist constraint.
+     *
+     * Under normal operating conditions, the number of brokers should
+     * represent the most likely maximum.  But there are some corner
+     * cases.  For example, the instance gets reconfigured to be
+     * smaller, which is not an uncommon thing to do towards a
+     * cluster's end of life and hardware is beginning to die.
+     *
+     * So we configure the following compromise.  If the number of
+     * brokers is below our defined minimum MIN_MATCH_HOSTLIST, we'll
+     * allow max_hostlist to be increased to this number.
+     */
+    if (mctx->max_hostlist < MIN_MATCH_HOSTLIST)
+        mctx->max_hostlist = MIN_MATCH_HOSTLIST;
+
+    return mctx;
+
+error:
+    match_ctx_destroy (mctx);
+    return NULL;
+}
+
+void match_ctx_destroy (struct match_ctx *mctx)
+{
+    if (mctx)
+        free (mctx);
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/job-list/match.h b/src/modules/job-list/match.h
new file mode 100644
index 000000000000..1d22bf63aaba
--- /dev/null
+++ b/src/modules/job-list/match.h
@@ -0,0 +1,57 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_JOB_LIST_MATCH_H
+#define HAVE_JOB_LIST_MATCH_H 1
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h> /* flux_error_t */
+#include <jansson.h>
+
+#include "job_data.h"
+
+struct match_ctx {
+    flux_t *h;
+    uint64_t max_comparisons;
+    uint32_t max_hostlist;
+};
+
+struct match_ctx *match_ctx_create (flux_t *h);
+
+void match_ctx_destroy (struct match_ctx *mctx);
+
+/*  Load and validate RFC 31 constraint spec 'constraint'.  'constraint'
+ *  can be NULL to indicate a constraint that matches everything.
+ *
+ *  Returns a list constraint object if constraint is valid spec,
+ *  Returns NULL with error in errp if errp != NULL.
+ */
+struct list_constraint *list_constraint_create (struct match_ctx *mctx,
+                                                json_t *constraint,
+                                                flux_error_t *errp);
+
+void list_constraint_destroy (struct list_constraint *constraint);
+
+/*  Return 1 if job matches constraints in RFC 31 constraint
+ *   specification 'constraint'.  Return 0 if not.  Return -1
+ *   on error.
+ */
+int job_match (const struct job *job,
+               struct list_constraint *constraint,
+               flux_error_t *errp);
+
+int job_match_config_reload (struct match_ctx *mctx,
+                             const flux_conf_t *conf,
+                             flux_error_t *errp);
+
+#endif /* !HAVE_JOB_LIST_MATCH_H */
diff --git a/src/modules/job-list/match_util.c b/src/modules/job-list/match_util.c
new file mode 100644
index 000000000000..e7a0a6f36b2d
--- /dev/null
+++ b/src/modules/job-list/match_util.c
@@ -0,0 +1,64 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <jansson.h>
+
+#include "src/common/libutil/errprintf.h"
+
+#include "match_util.h"
+
+int array_to_states_bitmask (json_t *values, flux_error_t *errp)
+{
+    int states = 0;
+    json_t *entry;
+    size_t index;
+    int valid_states = (FLUX_JOB_STATE_NEW
+                        | FLUX_JOB_STATE_PENDING
+                        | FLUX_JOB_STATE_RUNNING
+                        | FLUX_JOB_STATE_INACTIVE);
+
+    json_array_foreach (values, index, entry) {
+        flux_job_state_t state;
+        if (json_is_string (entry)) {
+            const char *statestr = json_string_value (entry);
+            if (flux_job_strtostate (statestr, &state) < 0) {
+                errprintf (errp,
+                           "invalid states value '%s' specified",
+                           statestr);
+                return -1;
+            }
+        }
+        else if (json_is_integer (entry)) {
+            state = json_integer_value (entry);
+            if (state & ~valid_states) {
+                errprintf (errp,
+                           "invalid states value '%Xh' specified",
+                           state);
+                return -1;
+            }
+        }
+        else {
+            errprintf (errp, "states value invalid type");
+            return -1;
+        }
+        states |= state;
+    }
+    return states;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/job-list/match_util.h b/src/modules/job-list/match_util.h
new file mode 100644
index 000000000000..59dbcd9c500c
--- /dev/null
+++ b/src/modules/job-list/match_util.h
@@ -0,0 +1,29 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_JOB_LIST_MATCH_UTIL_H
+#define HAVE_JOB_LIST_MATCH_UTIL_H 1
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h> /* flux_error_t */
+#include <jansson.h>
+
+/* identical to czmq_destructor, czmq.h happens to define a
+ * conflicting symbol we use */
+typedef void (destructor_f) (void **item);
+
+typedef int (*array_to_bitmask_f) (json_t *, flux_error_t *);
+
+int array_to_states_bitmask (json_t *values, flux_error_t *errp);
+
+#endif /* !HAVE_JOB_LIST_MATCH_UTIL_H */
diff --git a/src/modules/job-list/state_match.c b/src/modules/job-list/state_match.c
new file mode 100644
index 000000000000..1f055e40e089
--- /dev/null
+++ b/src/modules/job-list/state_match.c
@@ -0,0 +1,429 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <jansson.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "state_match.h"
+#include "match_util.h"
+
+/* MATCH_ALWAYS - constraint always matches job in state X
+ * MATCH_MAYBE - constraint maybe matches job in state X
+ * MATCH_NEVER - constraint never matches job in state X
+ *
+ * examples:
+ *
+ * states=depend
+ *
+ * This constraint ALWAYS matches a job in state depend and NEVER matches
+ * a job in any other job state.
+ *
+ * userid=42
+ *
+ * This constraint MAYBE matches a job in job state X because
+ * the job state does not matter, it depends on the userid.
+ *
+ * NOT (userid=42)
+ *
+ * This constraint MAYBE matches a job in job state X because again,
+ * it depends on the userid.  The NOT of a MAYBE is still MAYBE.
+ *
+ * (states=depend OR userid=42)
+ *
+ * This constraint ALWAYS matches a job in state depend, but MAYBE matches
+ * a job in any other job state, since it depends on the userid.
+ *
+ * (states=depend AND userid=42)
+ *
+ * This constraint MAYBE matches a job state in state depend, because
+ * it depends on the userid.  It NEVER matches a job in any other
+ * state.
+ *
+ */
+typedef enum {
+    MATCH_NOTSET,
+    MATCH_ALWAYS,
+    MATCH_MAYBE,
+    MATCH_NEVER,
+} state_match_t;
+
+typedef state_match_t (*match_f) (struct state_constraint *,
+                                  flux_job_state_t state);
+
+struct state_constraint {
+    zlistx_t *values;
+    match_f match;
+};
+
+static state_match_t match_always (struct state_constraint *c, flux_job_state_t state)
+{
+    return MATCH_ALWAYS;
+}
+
+static state_match_t match_maybe (struct state_constraint *c, flux_job_state_t state)
+{
+    return MATCH_MAYBE;
+}
+
+static struct state_constraint *state_constraint_new (match_f match_cb,
+                                                      destructor_f destructor_cb,
+                                                      flux_error_t *errp)
+{
+    struct state_constraint *c;
+    if (!(c = calloc (1, sizeof (*c)))
+        || !(c->values = zlistx_new ())) {
+        state_constraint_destroy (c);
+        errprintf (errp, "Out of memory");
+        return NULL;
+    }
+    c->match = match_cb;
+    if (destructor_cb)
+        zlistx_set_destructor (c->values, destructor_cb);
+    return c;
+}
+
+static void state_constraint_destructor (void **item)
+{
+    if (item) {
+        state_constraint_destroy (*item);
+        *item = NULL;
+    }
+}
+
+/* zlistx_set_destructor */
+static void wrap_free (void **item)
+{
+    if (item) {
+        free (*item);
+        (*item) = NULL;
+    }
+}
+
+static state_match_t match_states (struct state_constraint *c,
+                                   flux_job_state_t state)
+{
+    int *states = zlistx_first (c->values);
+    if ((*states) & state)
+        return MATCH_ALWAYS;
+    return MATCH_NEVER;
+}
+
+static struct state_constraint *create_states_constraint (json_t *values,
+                                                          flux_error_t *errp)
+{
+    struct state_constraint *c;
+    int *bitmask = NULL;
+    int tmp;
+    if ((tmp = array_to_states_bitmask (values, errp)) < 0)
+        return NULL;
+    /* if no states specified, returns MATCH_ALWAYS */
+    if (!tmp)
+        return state_constraint_new (match_always, NULL, errp);
+    if (!(bitmask = malloc (sizeof (*bitmask))))
+        return NULL;
+    (*bitmask) = tmp;
+    if (!(c = state_constraint_new (match_states, wrap_free, errp))
+        || !zlistx_add_end (c->values, bitmask)) {
+        state_constraint_destroy (c);
+        free (bitmask);
+        return NULL;
+    }
+    return c;
+}
+
+static state_match_t match_result (struct state_constraint *c, flux_job_state_t state)
+{
+    if (state != FLUX_JOB_STATE_INACTIVE)
+        return MATCH_NEVER;
+    return MATCH_MAYBE;
+}
+
+/* N.B. Not all job states can be reached, e.g. a pending job is
+ * canceled, so it never reaches the RUN state.  That is still handled
+ * here in this logic.  e.g. a constraint on `t_run` can MAYBE pass if
+ * the job state is INACTIVE.  We don't know if `t_run` was ever set,
+ * but since it can MAYBE be set, we must check.
+ */
+static state_match_t match_t_submit (struct state_constraint *c,
+                                     flux_job_state_t state)
+{
+    return MATCH_MAYBE;
+}
+
+static state_match_t match_t_depend (struct state_constraint *c,
+                                     flux_job_state_t state)
+{
+    if (state >= FLUX_JOB_STATE_DEPEND)
+        return MATCH_MAYBE;
+    return MATCH_NEVER;
+}
+
+static state_match_t match_t_run (struct state_constraint *c,
+                                  flux_job_state_t state)
+{
+    if (state >= FLUX_JOB_STATE_RUN)
+        return MATCH_MAYBE;
+    return MATCH_NEVER;
+}
+
+static state_match_t match_t_cleanup (struct state_constraint *c,
+                                      flux_job_state_t state)
+{
+    if (state >= FLUX_JOB_STATE_CLEANUP)
+        return MATCH_MAYBE;
+    return MATCH_NEVER;
+}
+
+static state_match_t match_t_inactive (struct state_constraint *c,
+                                       flux_job_state_t state)
+{
+    if (state == FLUX_JOB_STATE_INACTIVE)
+        return MATCH_MAYBE;
+    return MATCH_NEVER;
+}
+
+static struct state_constraint *create_timestamp_constraint (const char *type,
+                                                             flux_error_t *errp)
+{
+    struct state_constraint *c;
+    match_f cb;
+
+    if (streq (type, "t_submit"))
+        cb = match_t_submit;
+    else if (streq (type, "t_depend"))
+        cb = match_t_depend;
+    else if (streq (type, "t_run"))
+        cb = match_t_run;
+    else if (streq (type, "t_cleanup"))
+        cb = match_t_cleanup;
+    else /* streq (type, "t_inactive") */
+        cb = match_t_inactive;
+
+    if (!(c = state_constraint_new (cb, NULL, errp)))
+        return NULL;
+    return c;
+}
+
+static state_match_t match_and (struct state_constraint *c,
+                                flux_job_state_t state)
+{
+    state_match_t rv = MATCH_NOTSET;
+    struct state_constraint *cp = zlistx_first (c->values);
+    while (cp) {
+        /* This is an and statement, so if it a match is NEVER, we
+         * know that this constraint will return NEVER all the time.
+         *
+         * An ALWAYS can be demoted to a MAYBE and a MAYBE can be
+         * demoted to NEVER, so we keep iterating the match callbacks.
+         */
+        state_match_t m = cp->match (cp, state);
+        if (m == MATCH_NEVER)
+            return MATCH_NEVER;
+        else if (rv == MATCH_NOTSET)
+            rv = m;
+        else if (rv == MATCH_ALWAYS
+                 && m == MATCH_MAYBE) {
+            rv = MATCH_MAYBE;
+        }
+        /* else if rv == MATCH_MAYBE,
+         *  m == MATCH_MAYBE or m == MAYBE_ALWAYS,
+         *  rv stays MATCH_MAYBE
+         */
+        cp = zlistx_next (c->values);
+    }
+    /* empty op return MATCH_ALWAYS */
+    if (rv == MATCH_NOTSET)
+        return MATCH_ALWAYS;
+    return rv;
+}
+
+static state_match_t match_or (struct state_constraint *c,
+                               flux_job_state_t state)
+{
+    state_match_t rv = MATCH_NOTSET;
+    struct state_constraint *cp = zlistx_first (c->values);
+    while (cp) {
+        /* This is an or statement, so if it a match is ALWAYS, we
+         * know that this constraint will return ALWAYS all the time.
+         *
+         * A NEVER can be promoted to to a MAYBE and a MAYBE can be
+         * promoted to an ALWAYS, so we keep on iterating the match
+         * callbacks.
+         */
+        state_match_t m = cp->match (cp, state);
+        if (m == MATCH_ALWAYS)
+            return MATCH_ALWAYS;
+        else if (rv == MATCH_NOTSET)
+            rv = m;
+        else if (rv == MATCH_NEVER
+                 && m == MATCH_MAYBE)
+            rv = MATCH_MAYBE;
+        /* else if rv == MATCH_MAYBE,
+         *  m == MATCH_NEVER or m == MAYBE_MAYBE,
+         *  rv stays MATCH_MAYBE
+         */
+        cp = zlistx_next (c->values);
+    }
+    /* empty op return MATCH_ALWAYS */
+    if (rv == MATCH_NOTSET)
+        return MATCH_ALWAYS;
+    return rv;
+}
+
+static state_match_t match_not (struct state_constraint *c,
+                                flux_job_state_t state)
+{
+    state_match_t m = match_and (c, state);
+    if (m == MATCH_ALWAYS)
+        return MATCH_NEVER;
+    else if (m == MATCH_NEVER)
+        return MATCH_ALWAYS;
+    return MATCH_MAYBE;
+}
+
+static struct state_constraint *conditional_constraint (const char *type,
+                                                        json_t *values,
+                                                        flux_error_t *errp)
+{
+    json_t *entry;
+    size_t index;
+    struct state_constraint *c;
+    match_f match_cb;
+
+    if (streq (type, "and"))
+        match_cb = match_and;
+    else if (streq (type, "or"))
+        match_cb = match_or;
+    else /* streq (type, "not") */
+        match_cb = match_not;
+
+    if (!(c = state_constraint_new (match_cb,
+                                    state_constraint_destructor,
+                                    errp)))
+        return NULL;
+
+    json_array_foreach (values, index, entry) {
+        struct state_constraint *cp = state_constraint_create (entry, errp);
+        if (!cp)
+            goto error;
+        if (!zlistx_add_end (c->values, cp)) {
+            errprintf (errp, "Out of memory");
+            state_constraint_destroy (cp);
+            goto error;
+        }
+    }
+    return c;
+
+ error:
+    state_constraint_destroy (c);
+    return NULL;
+}
+
+void state_constraint_destroy (struct state_constraint *constraint)
+{
+    if (constraint) {
+        int saved_errno = errno;
+        zlistx_destroy (&constraint->values);
+        free (constraint);
+        errno = saved_errno;
+    }
+}
+
+struct state_constraint *state_constraint_create (json_t *constraint, flux_error_t *errp)
+{
+    const char *op;
+    json_t *values;
+
+    if (constraint) {
+        if (!json_is_object (constraint)) {
+            errprintf (errp, "constraint must be JSON object");
+            return NULL;
+        }
+        if (json_object_size (constraint) > 1) {
+            errprintf (errp, "constraint must only contain 1 element");
+            return NULL;
+        }
+        json_object_foreach (constraint, op, values) {
+            if (!json_is_array (values)) {
+                errprintf (errp, "operator %s values not an array", op);
+                return NULL;
+            }
+            if (streq (op, "userid")
+                || streq (op, "name")
+                || streq (op, "queue")
+                || streq (op, "hostlist")
+                || streq (op, "ranks"))
+                return state_constraint_new (match_maybe, NULL, errp);
+            else if (streq (op, "results"))
+                return state_constraint_new (match_result, NULL, errp);
+            else if (streq (op, "states"))
+                return create_states_constraint (values, errp);
+            else if (streq (op, "t_submit")
+                     || streq (op, "t_depend")
+                     || streq (op, "t_run")
+                     || streq (op, "t_cleanup")
+                     || streq (op, "t_inactive"))
+                return create_timestamp_constraint (op, errp);
+            else if (streq (op, "or") || streq (op, "and") || streq (op, "not"))
+                return conditional_constraint (op, values, errp);
+            else {
+                errprintf (errp, "unknown constraint operator: %s", op);
+                return NULL;
+            }
+        }
+    }
+    return state_constraint_new (match_always, NULL, errp);
+}
+
+bool state_match (int state, struct state_constraint *constraint)
+{
+    int valid_states = (FLUX_JOB_STATE_ACTIVE | FLUX_JOB_STATE_INACTIVE);
+
+    if (!state
+        || (state & ~valid_states)
+        || ((state & (state - 1)) != 0 /* classic is more than 1 bit set trick */
+            && state != FLUX_JOB_STATE_PENDING
+            && state != FLUX_JOB_STATE_RUNNING
+            && state != FLUX_JOB_STATE_ACTIVE)
+        || !constraint)
+        return false;
+
+    if ((state & (state - 1)) != 0) {
+        if (state == FLUX_JOB_STATE_PENDING)
+            return (state_match (FLUX_JOB_STATE_DEPEND, constraint)
+                    || state_match (FLUX_JOB_STATE_PRIORITY, constraint)
+                    || state_match (FLUX_JOB_STATE_SCHED, constraint));
+        else if (state == FLUX_JOB_STATE_RUNNING)
+            return (state_match (FLUX_JOB_STATE_RUN, constraint)
+                    || state_match (FLUX_JOB_STATE_CLEANUP, constraint));
+        else /* state == FLUX_JOB_STATE_ACTIVE */
+            return (state_match (FLUX_JOB_STATE_PENDING, constraint)
+                    || state_match (FLUX_JOB_STATE_RUNNING, constraint));
+    }
+    else {
+        state_match_t m;
+        m = constraint->match (constraint, state);
+        if (m == MATCH_ALWAYS || m == MATCH_MAYBE)
+            return true;
+        return false;
+    }
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/job-list/state_match.h b/src/modules/job-list/state_match.h
new file mode 100644
index 000000000000..56422011ae13
--- /dev/null
+++ b/src/modules/job-list/state_match.h
@@ -0,0 +1,37 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef HAVE_JOB_LIST_STATE_MATCH_H
+#define HAVE_JOB_LIST_STATE_MATCH_H 1
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h> /* flux_error_t */
+#include <jansson.h>
+
+#include "job_data.h"
+
+/*  Similar to list_constraint_create() but only cares about
+ *  "states" operation and the potential for a consraint to
+ *  return true given a job state..
+ */
+struct state_constraint *state_constraint_create (json_t *constraint,
+                                                  flux_error_t *errp);
+
+void state_constraint_destroy (struct state_constraint *constraint);
+
+/* determines if a job in 'state' could potentially return true with
+ * the given constraint.  'state' can be job state or virtual job state.
+ */
+bool state_match (int state, struct state_constraint *constraint);
+
+#endif /* !HAVE_JOB_LIST_STATE_MATCH_H */
diff --git a/src/modules/job-list/stats.c b/src/modules/job-list/stats.c
new file mode 100644
index 000000000000..cf2f8fa12151
--- /dev/null
+++ b/src/modules/job-list/stats.c
@@ -0,0 +1,517 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <assert.h>
+#include <ctype.h>
+#include <flux/core.h>
+
+#include "ccan/str/str.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/errno_safe.h"
+
+#include "stats.h"
+#include "job_data.h"
+
+#define BATCH_DELAY 0.2
+
+struct job_stats_ctx {
+    flux_t *h;
+    struct job_stats all;
+    zhashx_t *queue_stats;
+    flux_msg_handler_t **handlers;
+    struct flux_msglist *watchers;
+    flux_watcher_t *timer;
+    bool timer_running;
+};
+
+static void arm_timer (struct job_stats_ctx *statsctx);
+
+static void free_wrapper (void **item)
+{
+    if (item) {
+        free (*item);
+        (*item) = NULL;
+    }
+}
+
+static struct job_stats *queue_stats_lookup (struct job_stats_ctx *statsctx,
+                                             const char *name,
+                                             bool create_if_missing)
+{
+    struct job_stats *stats = NULL;
+
+    if (!name)
+        return NULL;
+
+    stats = zhashx_lookup (statsctx->queue_stats, name);
+    if (!stats && create_if_missing) {
+        if (!(stats = calloc (1, sizeof (*stats))))
+            return NULL;
+        (void)zhashx_insert (statsctx->queue_stats, name, stats);
+    }
+    return stats;
+}
+
+/*  Return the index into stats->state_count[] array for the
+ *   job state 'state'
+ */
+static inline int state_index (flux_job_state_t state)
+{
+    int i = 0;
+    while (!(state & (1<<i)))
+        i++;
+    assert (i < FLUX_JOB_NR_STATES);
+    return i;
+}
+
+/*  Return a lowercase state name for the state at 'index' the
+ *   stats->state_count[] array.
+ */
+static const char *state_index_name (int index)
+{
+    return flux_job_statetostr ((1<<index), "l");
+}
+
+static void stats_add (struct job_stats *stats,
+                       struct job *job,
+                       flux_job_state_t state)
+{
+    if (state == FLUX_JOB_STATE_NEW)
+        return;
+
+    stats->state_count[state_index (state)]++;
+
+    if (state == FLUX_JOB_STATE_INACTIVE) {
+        if (!job->success) {
+            if (job->exception_occurred) {
+                if (streq (job->exception_type, "cancel"))
+                    stats->canceled++;
+                else if (streq (job->exception_type, "timeout"))
+                    stats->timeout++;
+                else
+                    stats->failed++;
+            }
+            else
+                stats->failed++;
+        }
+        else
+            stats->successful++;
+    }
+}
+
+static void stats_update (struct job_stats *stats,
+                          struct job *job,
+                          flux_job_state_t newstate)
+{
+    /*  Stats for NEW are not tracked */
+    if (job->state != FLUX_JOB_STATE_NEW)
+        stats->state_count[state_index (job->state)]--;
+
+    stats_add (stats, job, newstate);
+}
+
+void job_stats_update (struct job_stats_ctx *statsctx,
+                       struct job *job,
+                       flux_job_state_t newstate)
+{
+    struct job_stats *stats;
+
+    stats_update (&statsctx->all, job, newstate);
+
+    if ((stats = queue_stats_lookup (statsctx, job->queue, true)))
+        stats_update (stats, job, newstate);
+
+    arm_timer (statsctx);
+}
+
+void job_stats_add_queue (struct job_stats_ctx *statsctx,
+                          struct job *job)
+{
+    struct job_stats *stats;
+
+    if ((stats = queue_stats_lookup (statsctx, job->queue, true)))
+        stats_add (stats, job, job->state);
+
+    arm_timer (statsctx);
+}
+
+static void stats_remove (struct job_stats *stats,
+                          struct job *job)
+{
+    /*  Stats for NEW are not tracked */
+    if (job->state != FLUX_JOB_STATE_NEW)
+        stats->state_count[state_index (job->state)]--;
+
+    if (job->state == FLUX_JOB_STATE_INACTIVE) {
+        if (!job->success) {
+            if (job->exception_occurred) {
+                if (streq (job->exception_type, "cancel"))
+                    stats->canceled--;
+                else if (streq (job->exception_type, "timeout"))
+                    stats->timeout--;
+                else
+                    stats->failed--;
+            }
+            else
+                stats->failed--;
+        }
+        else
+            stats->successful--;
+    }
+}
+
+void job_stats_remove_queue (struct job_stats_ctx *statsctx,
+                             struct job *job)
+{
+    struct job_stats *stats;
+
+    if (!(stats = queue_stats_lookup (statsctx, job->queue, false))) {
+        if (job->queue)
+            flux_log (statsctx->h,
+                      LOG_DEBUG,
+                      "no queue stats for %s",
+                      job->queue);
+        return;
+    }
+
+    stats_remove (stats, job);
+    arm_timer (statsctx);
+}
+
+static void stats_purge (struct job_stats *stats, struct job *job)
+{
+    stats->state_count[state_index (job->state)]--;
+
+    if (!job->success) {
+        if (job->exception_occurred) {
+            if (streq (job->exception_type, "cancel"))
+                stats->canceled--;
+            else if (streq (job->exception_type, "timeout"))
+                stats->timeout--;
+            else
+                stats->failed--;
+        }
+        else
+            stats->failed--;
+    }
+    else
+        stats->successful--;
+    stats->inactive_purged++;
+}
+
+/* An inactive job is being purged, so statistics must be updated.
+ */
+void job_stats_purge (struct job_stats_ctx *statsctx, struct job *job)
+{
+    struct job_stats *stats;
+
+    assert (job->state == FLUX_JOB_STATE_INACTIVE);
+
+    stats_purge (&statsctx->all, job);
+
+    if (!(stats = queue_stats_lookup (statsctx, job->queue, false))) {
+        if (job->queue)
+            flux_log (statsctx->h,
+                      LOG_DEBUG,
+                      "no queue stats for %s",
+                      job->queue);
+        return;
+    }
+
+    stats_purge (stats, job);
+    arm_timer (statsctx);
+}
+
+static int object_set_integer (json_t *o,
+                               const char *key,
+                               unsigned int n)
+{
+    json_t *val = json_integer (n);
+    if (!val || json_object_set_new (o, key, val) < 0) {
+        json_decref (val);
+        return -1;
+    }
+    return 0;
+}
+
+static json_t *job_states_encode (struct job_stats *stats)
+{
+    unsigned int total = 0;
+    json_t *o = json_object ();
+    if (!o)
+        return NULL;
+    for (int i = 1; i < FLUX_JOB_NR_STATES; i++) {
+        if (object_set_integer (o,
+                                state_index_name (i),
+                                stats->state_count[i]) < 0)
+            goto error;
+        total += stats->state_count[i];
+    }
+    if (object_set_integer (o, "total", total) < 0)
+        goto error;
+    return o;
+error:
+    json_decref (o);
+    return NULL;
+}
+
+static json_t *stats_encode (struct job_stats *stats, const char *name)
+{
+    json_t *o;
+    json_t *states;
+
+    if (!(states = job_states_encode (stats))
+        || !(o = json_pack ("{ s:O s:i s:i s:i s:i s:i }",
+                            "job_states", states,
+                            "successful", stats->successful,
+                            "failed", stats->failed,
+                            "canceled", stats->canceled,
+                            "timeout", stats->timeout,
+                            "inactive_purged", stats->inactive_purged))) {
+        json_decref (states);
+        errno = ENOMEM;
+        return NULL;
+    }
+    json_decref (states);
+
+    if (name) {
+        json_t *no = json_string (name);
+        if (!no || json_object_set_new (o, "name", no) < 0) {
+            json_decref (no);
+            json_decref (o);
+            errno = ENOMEM;
+            return NULL;
+        }
+    }
+    return o;
+}
+
+static json_t *queue_stats_encode (struct job_stats_ctx *statsctx)
+{
+    struct job_stats *stats;
+    json_t *queues;
+
+    if (!(queues = json_array ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    stats = zhashx_first (statsctx->queue_stats);
+    while (stats) {
+        const char *name = zhashx_cursor (statsctx->queue_stats);
+        json_t *qo = stats_encode (stats, name);
+        if (!qo) {
+            int save_errno = errno;
+            json_decref (queues);
+            errno = save_errno;
+            return NULL;
+        }
+        if (json_array_append_new (queues, qo) < 0) {
+            json_decref (qo);
+            json_decref (queues);
+            errno = ENOMEM;
+            return NULL;
+        }
+        stats = zhashx_next (statsctx->queue_stats);
+    }
+
+    return queues;
+}
+
+static json_t *job_stats_encode (struct job_stats_ctx *statsctx)
+{
+    json_t *o = NULL;
+    json_t *queues;
+
+    if (!(o = stats_encode (&statsctx->all, NULL)))
+        return NULL;
+
+    if (!(queues = queue_stats_encode (statsctx))) {
+        int save_errno = errno;
+        json_decref (o);
+        errno = save_errno;
+        return NULL;
+    }
+
+    if (json_object_set_new (o, "queues", queues) < 0) {
+        json_decref (queues);
+        json_decref (o);
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    return o;
+}
+
+static int job_stats_respond (struct job_stats_ctx *statsctx,
+                              const flux_msg_t *msg)
+{
+    json_t *o;
+    int rc;
+
+    if (!(o = job_stats_encode (statsctx)))
+        return -1;
+    rc = flux_respond_pack (statsctx->h, msg, "O", o);
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return rc;
+}
+
+void job_stats_disconnect (struct job_stats_ctx *statsctx,
+                           const flux_msg_t *msg)
+{
+    flux_msglist_disconnect (statsctx->watchers, msg);
+}
+
+int job_stats_watchers (struct job_stats_ctx *statsctx)
+{
+    return flux_msglist_count (statsctx->watchers);
+}
+
+static void timer_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct job_stats_ctx *statsctx = arg;
+    const flux_msg_t *msg;
+
+    msg = flux_msglist_first (statsctx->watchers);
+    while (msg) {
+        if (job_stats_respond (statsctx, msg) < 0)
+            flux_log_error (statsctx->h, "error responding to job-stats");
+        msg = flux_msglist_next (statsctx->watchers);
+    }
+    flux_watcher_stop (w);
+    statsctx->timer_running = false;
+}
+
+static void arm_timer (struct job_stats_ctx *statsctx)
+{
+    if (!statsctx->timer_running) {
+        flux_timer_watcher_reset (statsctx->timer, BATCH_DELAY, 0);
+        flux_watcher_start (statsctx->timer);
+        statsctx->timer_running = true;
+    }
+}
+
+static void job_stats_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
+{
+    struct job_stats_ctx *statsctx = arg;
+
+    if (flux_msg_is_streaming (msg)) {
+        if (flux_msglist_append (statsctx->watchers, msg) < 0)
+            goto error;
+    }
+    if (job_stats_respond (statsctx, msg) < 0)
+        flux_log_error (h, "error responding to job-stats request");
+    return;
+error:
+    if (flux_respond_error (statsctx->h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to job-stats request");
+}
+
+static int config_parse_queues (struct job_stats_ctx *statsctx,
+                                const flux_conf_t *conf,
+                                flux_error_t *errp)
+{
+    json_t *queues;
+
+    if (flux_conf_unpack (conf, NULL, "{s:o}", "queues", &queues) == 0
+        && json_object_size (queues) > 0) {
+        const char *name;
+        json_t *value;
+        json_object_foreach (queues, name, value) {
+            /* setup initial queue stats, so that user gets initial
+             * stats before first job is submitted to the queue */
+            if (!queue_stats_lookup (statsctx, name, true)) {
+                flux_log_error (statsctx->h, "queue_stats_lookup");
+                return -1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+int job_stats_config_reload (struct job_stats_ctx *statsctx,
+                             const flux_conf_t *conf,
+                             flux_error_t *errp)
+{
+    return config_parse_queues (statsctx, conf, errp);
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { .typemask     = FLUX_MSGTYPE_REQUEST,
+      .topic_glob   = "job-list.job-stats",
+      .cb           = job_stats_cb,
+      .rolemask     = FLUX_ROLE_USER
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+struct job_stats_ctx *job_stats_ctx_create (flux_t *h)
+{
+    struct job_stats_ctx *statsctx = NULL;
+    flux_error_t error;
+
+    if (!(statsctx = calloc (1, sizeof (*statsctx))))
+        return NULL;
+    statsctx->h = h;
+
+    if (!(statsctx->queue_stats = zhashx_new ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    zhashx_set_destructor (statsctx->queue_stats, free_wrapper);
+    if (flux_msg_handler_addvec (h, htab, statsctx, &statsctx->handlers) < 0)
+        goto error;
+    if (!(statsctx->watchers = flux_msglist_create ()))
+        goto error;
+    if (!(statsctx->timer = flux_timer_watcher_create (flux_get_reactor (h),
+                                                       BATCH_DELAY,
+                                                       0.,
+                                                       timer_cb,
+                                                       statsctx)))
+        goto error;
+
+    if (config_parse_queues (statsctx,
+                             flux_get_conf (statsctx->h),
+                             &error) < 0) {
+        flux_log (statsctx->h, LOG_ERR, "%s", error.text);
+        goto error;
+    }
+
+    return statsctx;
+
+error:
+    job_stats_ctx_destroy (statsctx);
+    return NULL;
+}
+
+void job_stats_ctx_destroy (struct job_stats_ctx *statsctx)
+{
+    if (statsctx) {
+        int save_errno = errno;
+        flux_msg_handler_delvec (statsctx->handlers);
+        flux_msglist_destroy (statsctx->watchers);
+        flux_watcher_destroy (statsctx->timer);
+        zhashx_destroy (&statsctx->queue_stats);
+        free (statsctx);
+        errno = save_errno;
+    }
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/modules/job-list/stats.h b/src/modules/job-list/stats.h
new file mode 100644
index 000000000000..b97c3ebf8347
--- /dev/null
+++ b/src/modules/job-list/stats.h
@@ -0,0 +1,60 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_LIST_JOB_STATS_H
+#define _FLUX_JOB_LIST_JOB_STATS_H
+
+#include <flux/core.h> /* FLUX_JOB_NR_STATES */
+#include <jansson.h>
+
+#include "job_data.h"
+
+struct job_stats {
+    unsigned int state_count[FLUX_JOB_NR_STATES];
+    unsigned int successful;
+    unsigned int failed;
+    unsigned int timeout;
+    unsigned int canceled;
+    unsigned int inactive_purged;
+};
+
+struct job_stats_ctx *job_stats_ctx_create (flux_t *h);
+
+void job_stats_ctx_destroy (struct job_stats_ctx *statsctx);
+
+void job_stats_update (struct job_stats_ctx *statsctx,
+                       struct job *job,
+                       flux_job_state_t newstate);
+
+void job_stats_add_queue (struct job_stats_ctx *statsctx,
+                          struct job *job);
+
+void job_stats_remove_queue (struct job_stats_ctx *statsctx,
+                             struct job *job);
+
+void job_stats_purge (struct job_stats_ctx *statsctx, struct job *job);
+
+/* A client has disconnected from job-list.
+ * Cancel streaming job-stats request, if any.
+ */
+void job_stats_disconnect (struct job_stats_ctx *statsctx,
+                           const flux_msg_t *msg);
+
+/* Return the number of job-stats streaming clients.
+ */
+int job_stats_watchers (struct job_stats_ctx *statsctx);
+
+int job_stats_config_reload (struct job_stats_ctx *statsctx,
+                             const flux_conf_t *conf,
+                             flux_error_t *errp);
+
+#endif /* ! _FLUX_JOB_LIST_JOB_STATS_H */
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/modules/job-list/test/R/1node_1core.R b/src/modules/job-list/test/R/1node_1core.R
new file mode 100644
index 000000000000..a1aa4da6375e
--- /dev/null
+++ b/src/modules/job-list/test/R/1node_1core.R
@@ -0,0 +1,18 @@
+{
+  "version": 1,
+  "execution": {
+    "R_lite": [
+      {
+        "rank": "0",
+        "children": {
+          "core": "0"
+        }
+      }
+    ],
+    "starttime": 0,
+    "expiration": 0,
+    "nodelist": [
+      "node1"
+    ]
+  }
+}
diff --git a/src/modules/job-list/test/R/1node_4core.R b/src/modules/job-list/test/R/1node_4core.R
new file mode 100644
index 000000000000..b8e961184f7e
--- /dev/null
+++ b/src/modules/job-list/test/R/1node_4core.R
@@ -0,0 +1,18 @@
+{
+  "version": 1,
+  "execution": {
+    "R_lite": [
+      {
+        "rank": "0",
+        "children": {
+          "core": "0-3"
+        }
+      }
+    ],
+    "starttime": 0,
+    "expiration": 0,
+    "nodelist": [
+      "node1"
+    ]
+  }
+}
diff --git a/src/modules/job-list/test/R/4node_1core.R b/src/modules/job-list/test/R/4node_1core.R
new file mode 100644
index 000000000000..62143bcb988c
--- /dev/null
+++ b/src/modules/job-list/test/R/4node_1core.R
@@ -0,0 +1,18 @@
+{
+  "version": 1,
+  "execution": {
+    "R_lite": [
+      {
+        "rank": "0-3",
+        "children": {
+          "core": "0"
+        }
+      }
+    ],
+    "starttime": 0,
+    "expiration": 0,
+    "nodelist": [
+      "node[1-4]"
+    ]
+  }
+}
diff --git a/src/modules/job-list/test/R/4node_4core.R b/src/modules/job-list/test/R/4node_4core.R
new file mode 100644
index 000000000000..e67bdc0c28ba
--- /dev/null
+++ b/src/modules/job-list/test/R/4node_4core.R
@@ -0,0 +1,18 @@
+{
+  "version": 1,
+  "execution": {
+    "R_lite": [
+      {
+        "rank": "0-3",
+        "children": {
+          "core": "0-3"
+        }
+      }
+    ],
+    "starttime": 0,
+    "expiration": 0,
+    "nodelist": [
+      "node[1-4]"
+    ]
+  }
+}
diff --git a/src/modules/job-list/test/R/invalid_R_lite.R b/src/modules/job-list/test/R/invalid_R_lite.R
new file mode 100644
index 000000000000..ea428ba51366
--- /dev/null
+++ b/src/modules/job-list/test/R/invalid_R_lite.R
@@ -0,0 +1,11 @@
+{
+  "version": 1,
+  "execution": {
+    "R_lite": "foo",
+    "starttime": 0,
+    "expiration": 0,
+    "nodelist": [
+      "node1"
+    ]
+  }
+}
diff --git a/src/modules/job-list/test/R/invalid_json.R b/src/modules/job-list/test/R/invalid_json.R
new file mode 100644
index 000000000000..257cc5642cb1
--- /dev/null
+++ b/src/modules/job-list/test/R/invalid_json.R
@@ -0,0 +1 @@
+foo
diff --git a/src/modules/job-list/test/R/invalid_nodelist.R b/src/modules/job-list/test/R/invalid_nodelist.R
new file mode 100644
index 000000000000..f3ea076c3054
--- /dev/null
+++ b/src/modules/job-list/test/R/invalid_nodelist.R
@@ -0,0 +1,16 @@
+{
+  "version": 1,
+  "execution": {
+    "R_lite": [
+      {
+        "rank": "0",
+        "children": {
+          "core": "0"
+        }
+      }
+    ],
+    "starttime": 0,
+    "expiration": 0,
+    "nodelist": "foo"
+  }
+}
diff --git a/src/modules/job-list/test/R/invalid_version.R b/src/modules/job-list/test/R/invalid_version.R
new file mode 100644
index 000000000000..5386172e3163
--- /dev/null
+++ b/src/modules/job-list/test/R/invalid_version.R
@@ -0,0 +1,18 @@
+{
+  "version": 999,
+  "execution": {
+    "R_lite": [
+      {
+        "rank": "0",
+        "children": {
+          "core": "0"
+        }
+      }
+    ],
+    "starttime": 0,
+    "expiration": 0,
+    "nodelist": [
+      "node1"
+    ]
+  }
+}
diff --git a/src/modules/job-list/test/R/missing_R_lite.R b/src/modules/job-list/test/R/missing_R_lite.R
new file mode 100644
index 000000000000..2c7760d1b1ca
--- /dev/null
+++ b/src/modules/job-list/test/R/missing_R_lite.R
@@ -0,0 +1,10 @@
+{
+  "version": 1,
+  "execution": {
+    "starttime": 0,
+    "expiration": 0,
+    "nodelist": [
+      "node1"
+    ]
+  }
+}
diff --git a/src/modules/job-list/test/R/missing_expiration.R b/src/modules/job-list/test/R/missing_expiration.R
new file mode 100644
index 000000000000..755489ae4ee8
--- /dev/null
+++ b/src/modules/job-list/test/R/missing_expiration.R
@@ -0,0 +1,17 @@
+{
+  "version": 1,
+  "execution": {
+    "R_lite": [
+      {
+        "rank": "0",
+        "children": {
+          "core": "0"
+        }
+      }
+    ],
+    "starttime": 0,
+    "nodelist": [
+      "node1"
+    ]
+  }
+}
diff --git a/src/modules/job-list/test/R/missing_nodelist.R b/src/modules/job-list/test/R/missing_nodelist.R
new file mode 100644
index 000000000000..fa2d5daa68ab
--- /dev/null
+++ b/src/modules/job-list/test/R/missing_nodelist.R
@@ -0,0 +1,15 @@
+{
+  "version": 1,
+  "execution": {
+    "R_lite": [
+      {
+        "rank": "0",
+        "children": {
+          "core": "0"
+        }
+      }
+    ],
+    "starttime": 0,
+    "expiration": 0,
+  }
+}
diff --git a/src/modules/job-list/test/R/missing_starttime.R b/src/modules/job-list/test/R/missing_starttime.R
new file mode 100644
index 000000000000..9c632fb41101
--- /dev/null
+++ b/src/modules/job-list/test/R/missing_starttime.R
@@ -0,0 +1,17 @@
+{
+  "version": 1,
+  "execution": {
+    "R_lite": [
+      {
+        "rank": "0",
+        "children": {
+          "core": "0"
+        }
+      }
+    ],
+    "expiration": 0,
+    "nodelist": [
+      "node1"
+    ]
+  }
+}
diff --git a/src/modules/job-list/test/R/missing_version.R b/src/modules/job-list/test/R/missing_version.R
new file mode 100644
index 000000000000..151bc0f44730
--- /dev/null
+++ b/src/modules/job-list/test/R/missing_version.R
@@ -0,0 +1,17 @@
+{
+  "execution": {
+    "R_lite": [
+      {
+        "rank": "0",
+        "children": {
+          "core": "0"
+        }
+      }
+    ],
+    "starttime": 0,
+    "expiration": 0,
+    "nodelist": [
+      "node1"
+    ]
+  }
+}
diff --git a/src/modules/job-list/test/job_data.c b/src/modules/job-list/test/job_data.c
new file mode 100644
index 000000000000..ad65147fbf03
--- /dev/null
+++ b/src/modules/job-list/test/job_data.c
@@ -0,0 +1,893 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/read_all.h"
+#include "src/modules/job-list/job_data.h"
+#include "ccan/str/str.h"
+
+struct test_jobspec_corner_case {
+    const char *filename;
+    int expected;
+} jobspec_corner_case_tests[] = {
+    { TEST_SRCDIR "/jobspec/invalid_json.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/missing_attributes.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_attributes_system_job.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_attributes_system_missing_duration.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/missing_tasks.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_tasks_array.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_tasks_missing_command.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_command_array.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_command_string.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_per_resource_missing_type.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/missing_version.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_version.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/missing_resources.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_resources.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_resources_missing_type.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_resources_invalid_type.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_resources_missing_count.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_resources_invalid_count.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_resources_noslots.jobspec", -1 },
+    { TEST_SRCDIR "/jobspec/invalid_resources_nocores.jobspec", -1 },
+    { NULL, 0 },
+};
+
+struct test_jobspec_job_name {
+    const char *filename;
+    const char *job_name;
+} jobspec_job_name_tests[] = {
+    { TEST_SRCDIR "/jobspec/1slot.jobspec", "hostname" },
+    { TEST_SRCDIR "/jobspec/job_name_alt.jobspec", "altname" },
+    { NULL, 0 },
+};
+
+struct test_jobspec_cwd {
+    const char *filename;
+    const char *cwd;
+} jobspec_cwd_tests[] = {
+    { TEST_SRCDIR "/jobspec/1slot.jobspec", "/tmp/job" },
+    { TEST_SRCDIR "/jobspec/cwd_not_specified.jobspec", NULL },
+    { NULL, 0 },
+};
+
+struct test_jobspec_queue {
+    const char *filename;
+    const char *queue;
+} jobspec_queue_tests[] = {
+    { TEST_SRCDIR "/jobspec/1slot.jobspec", NULL },
+    { TEST_SRCDIR "/jobspec/queue_specified.jobspec", "batch" },
+    { NULL, 0 },
+};
+
+struct test_jobspec_project_bank {
+    const char *filename;
+    const char *project;
+    const char *bank;
+} jobspec_project_bank_tests[] = {
+    {
+        TEST_SRCDIR "/jobspec/1slot.jobspec",
+        NULL,
+        NULL,
+    },
+    {
+        TEST_SRCDIR "/jobspec/1slot_project_bank.jobspec",
+        "myproject",
+        "mybank",
+    },
+    { NULL, NULL, NULL},
+};
+
+struct test_jobspec_duration {
+    const char *filename;
+    double duration;
+} jobspec_duration_tests[] = {
+    { TEST_SRCDIR "/jobspec/1slot.jobspec", 0.0 },
+    { TEST_SRCDIR "/jobspec/duration_alt.jobspec", 100.0 },
+    { NULL, 0 },
+};
+
+struct test_R_corner_case {
+    const char *filename;
+    int expected;
+} R_corner_case_tests[] = {
+    { TEST_SRCDIR "/R/missing_starttime.R", 0 },
+    { TEST_SRCDIR "/R/missing_expiration.R", 0 },
+    { TEST_SRCDIR "/R/invalid_json.R", -1 },
+    { TEST_SRCDIR "/R/missing_version.R", -1 },
+    { TEST_SRCDIR "/R/invalid_version.R", -1 },
+    { TEST_SRCDIR "/R/invalid_R_lite.R", -1 },
+    { TEST_SRCDIR "/R/missing_nodelist.R", -1 },
+    { TEST_SRCDIR "/R/invalid_nodelist.R", -1 },
+    { NULL, 0 },
+};
+
+struct test_R_ranks {
+    const char *filename;
+    const char *ranks;
+} R_ranks_tests[] = {
+    { TEST_SRCDIR "/R/1node_4core.R", "0" },
+    { TEST_SRCDIR "/R/4node_4core.R", "[0-3]" },
+    { NULL, 0 },
+};
+
+struct test_R_nodelist {
+    const char *filename;
+    const char *nodelist;
+} R_nodelist_tests[] = {
+    { TEST_SRCDIR "/R/1node_4core.R", "node1" },
+    { TEST_SRCDIR "/R/4node_4core.R", "node[1-4]" },
+    { NULL, 0 },
+};
+
+struct test_nnodes {
+    const char *jobspec_filename;
+    const char *R_filename;
+    int nnodes_after_jobspec;
+    int nnodes_after_R;
+} nnodes_tests[] = {
+    {
+        TEST_SRCDIR "/jobspec/1slot.jobspec",
+        TEST_SRCDIR "/R/1node_1core.R",
+        -1,
+        1,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4slot.jobspec",
+        TEST_SRCDIR "/R/4node_4core.R",
+        -1,
+        4,
+    },
+    {
+        TEST_SRCDIR "/jobspec/1node.jobspec",
+        TEST_SRCDIR "/R/1node_4core.R",
+        1,
+        1,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4node.jobspec",
+        TEST_SRCDIR "/R/4node_4core.R",
+        4,
+        4,
+    },
+    { NULL, NULL, 0, 0 },
+};
+
+struct test_ntasks {
+    const char *jobspec_filename;
+    const char *R_filename;
+    int ntasks_after_jobspec;
+    int ntasks_after_R;
+} ntasks_tests[] = {
+    {
+        TEST_SRCDIR "/jobspec/1slot.jobspec",
+        TEST_SRCDIR "/R/1node_1core.R",
+        1,
+        1,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4slot.jobspec",
+        TEST_SRCDIR "/R/1node_4core.R",
+        4,
+        4,
+    },
+    {
+        TEST_SRCDIR "/jobspec/1node.jobspec",
+        TEST_SRCDIR "/R/1node_4core.R",
+        1,
+        1,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4node.jobspec",
+        TEST_SRCDIR "/R/4node_4core.R",
+        4,
+        4,
+    },
+    {
+        TEST_SRCDIR "/jobspec/1node_perresourcenode4.jobspec",
+        TEST_SRCDIR "/R/1node_4core.R",
+        4,
+        4,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4node_perresourcenode4.jobspec",
+        TEST_SRCDIR "/R/4node_4core.R",
+        16,
+        16,
+    },
+    {
+        TEST_SRCDIR "/jobspec/1slot_perresourcecore4.jobspec",
+        TEST_SRCDIR "/R/1node_4core.R",
+        4,
+        4,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4slot_perresourcecore4.jobspec",
+        TEST_SRCDIR "/R/1node_4core.R",
+        16,
+        16,
+    },
+    {
+        TEST_SRCDIR "/jobspec/1node_perresourcecore4.jobspec",
+        TEST_SRCDIR "/R/1node_4core.R",
+        -1,
+        16,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4node_perresourcecore4.jobspec",
+        TEST_SRCDIR "/R/4node_4core.R",
+        -1,
+        64,
+    },
+    { NULL, NULL, 0, 0 },
+};
+
+struct test_ncores {
+    const char *jobspec_filename;
+    const char *R_filename;
+    int ncores_after_jobspec;
+    int ncores_after_R;
+} ncores_tests[] = {
+    {
+        TEST_SRCDIR "/jobspec/1slot.jobspec",
+        TEST_SRCDIR "/R/1node_1core.R",
+        1,
+        1,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4slot.jobspec",
+        TEST_SRCDIR "/R/1node_4core.R",
+        4,
+        4,
+    },
+    {
+        TEST_SRCDIR "/jobspec/1slot_4core.jobspec",
+        TEST_SRCDIR "/R/1node_4core.R",
+        4,
+        4,
+    },
+    {
+        TEST_SRCDIR "/jobspec/1node.jobspec",
+        TEST_SRCDIR "/R/1node_4core.R",
+        -1,
+        4,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4node.jobspec",
+        TEST_SRCDIR "/R/4node_4core.R",
+        -1,
+        16,
+    },
+    {
+        TEST_SRCDIR "/jobspec/1node_1slot_nonexclusive.jobspec",
+        TEST_SRCDIR "/R/1node_1core.R",
+        1,
+        1,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4node_1slot_nonexclusive.jobspec",
+        TEST_SRCDIR "/R/4node_1core.R",
+        4,
+        4,
+    },
+    {
+        TEST_SRCDIR "/jobspec/4node_4slot_nonexclusive.jobspec",
+        TEST_SRCDIR "/R/4node_4core.R",
+        16,
+        16,
+    },
+    { NULL, NULL, 0, 0 },
+};
+
+static void read_file (const char *filename, void **datap)
+{
+    int fd;
+    ssize_t size;
+
+    if ((fd = open (filename, O_RDONLY)) < 0)
+        BAIL_OUT ("failed to open %s", filename);
+    /* N.B. read_all() NUL terminates buffer */
+    if ((size = read_all (fd, datap)) < 0)
+        BAIL_OUT ("failed to read data %s", filename);
+    close (fd);
+}
+
+static int parse_jobspec (struct job *job, const char *filename)
+{
+    char *data;
+    int ret;
+
+    read_file (filename, (void **)&data);
+
+    ret = job_parse_jobspec_fatal (job, data, NULL);
+
+    free (data);
+    return ret;
+}
+
+static int parse_R (struct job *job, const char *filename)
+{
+    char *data;
+    int ret;
+
+    read_file (filename, (void **)&data);
+
+    ret = job_parse_R_fatal (job, data, NULL);
+
+    free (data);
+    return ret;
+}
+
+static void test_jobspec_corner_case (void)
+{
+    struct test_jobspec_corner_case *test;
+
+    test = jobspec_corner_case_tests;
+    while (test->filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *filename = test->filename;
+        int expected = test->expected;
+        int ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        ret = parse_jobspec (job, filename);
+        ok (ret == expected, "job_parse_jobspec passes on %s", filename);
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_jobspec_job_name (void)
+{
+    struct test_jobspec_job_name *test;
+
+    test = jobspec_job_name_tests;
+    while (test->filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *filename = test->filename;
+        const char *job_name = test->job_name;
+        int ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        ret = parse_jobspec (job, filename);
+        ok (ret == 0, "job_parse_jobspec parsed %s", filename);
+        ok (streq (job_name, job->name),
+            "job_parse_jobspec correctly parsed job name %s=%s",
+            job_name, job->name);
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_jobspec_cwd (void)
+{
+    struct test_jobspec_cwd *test;
+
+    test = jobspec_cwd_tests;
+    while (test->filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *filename = test->filename;
+        const char *cwd = test->cwd;
+        int ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        ret = parse_jobspec (job, filename);
+        ok (ret == 0, "job_parse_jobspec parsed %s", filename);
+        if (cwd) {
+            ok (streq (cwd, job->cwd),
+                "job_parse_jobspec correctly parsed job cwd %s=%s",
+                cwd, job->cwd);
+        }
+        else {
+            ok (job->cwd == NULL,
+                "job_parse_jobspec correctly parsed no job cwd");
+        }
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_jobspec_queue (void)
+{
+    struct test_jobspec_queue *test;
+
+    test = jobspec_queue_tests;
+    while (test->filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *filename = test->filename;
+        const char *queue = test->queue;
+        int ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        ret = parse_jobspec (job, filename);
+        ok (ret == 0, "job_parse_jobspec parsed %s", filename);
+        if (queue) {
+            ok (streq (queue, job->queue),
+                "job_parse_jobspec correctly parsed job queue %s=%s",
+                queue, job->queue);
+        }
+        else {
+            ok (job->queue == NULL,
+                "job_parse_jobspec correctly parsed no job queue");
+        }
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_jobspec_project_bank (void)
+{
+    struct test_jobspec_project_bank *test;
+
+    test = jobspec_project_bank_tests;
+    while (test->filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *filename = test->filename;
+        const char *project = test->project;
+        const char *bank = test->bank;
+        int ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        ret = parse_jobspec (job, filename);
+        ok (ret == 0, "job_parse_jobspec parsed %s", filename);
+        if (project) {
+            ok (streq (project, job->project),
+                "job_parse_jobspec correctly parsed job project %s=%s",
+                project, job->project);
+        }
+        else {
+            ok (job->project == NULL,
+                "job_parse_jobspec correctly parsed no job project");
+        }
+        if (bank) {
+            ok (streq (bank, job->bank),
+                "job_parse_jobspec correctly parsed job bank %s=%s",
+                bank, job->bank);
+        }
+        else {
+            ok (job->bank == NULL,
+                "job_parse_jobspec correctly parsed no job bank");
+        }
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_jobspec_duration (void)
+{
+    struct test_jobspec_duration *test;
+
+    test = jobspec_duration_tests;
+    while (test->filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *filename = test->filename;
+        double duration = test->duration;
+        int ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        ret = parse_jobspec (job, filename);
+        ok (ret == 0, "job_parse_jobspec parsed %s", filename);
+        ok (duration == job->duration,
+            "job_parse_jobspec correctly parsed duration %f=%f",
+            duration, job->duration);
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_R_corner_case (void)
+{
+    struct test_R_corner_case *test;
+
+    test = R_corner_case_tests;
+    while (test->filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *filename = test->filename;
+        int expected = test->expected;
+        int ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        ret = parse_R (job, filename);
+        ok (ret == expected, "job_parse_R passes on %s", filename);
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_R_ranks (void)
+{
+    struct test_R_ranks *test;
+
+    test = R_ranks_tests;
+    while (test->filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *filename = test->filename;
+        const char *ranks = test->ranks;
+        int ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        ret = parse_R (job, filename);
+        ok (ret == 0, "job_parse_R parsed %s", filename);
+        ok (streq (ranks, job->ranks),
+            "job_parse_jobspec correctly parsed job ranks %s=%s",
+            ranks, job->ranks);
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_R_nodelist (void)
+{
+    struct test_R_nodelist *test;
+
+    test = R_nodelist_tests;
+    while (test->filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *filename = test->filename;
+        const char *nodelist = test->nodelist;
+        int ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        ret = parse_R (job, filename);
+        ok (ret == 0, "job_parse_R parsed %s", filename);
+        ok (streq (nodelist, job->nodelist),
+            "job_parse_jobspec correctly parsed job nodelist %s=%s",
+            nodelist, job->nodelist);
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_nnodes (void)
+{
+    struct test_nnodes *test;
+
+    test = nnodes_tests;
+    while (test->jobspec_filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *jobspec_filename = test->jobspec_filename;
+        const char *R_filename = test->R_filename;
+        int nnodes_after_jobspec = test->nnodes_after_jobspec;
+        int nnodes_after_R = test->nnodes_after_R;
+        int jobspec_ret, R_ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        jobspec_ret = parse_jobspec (job, jobspec_filename);
+        ok (jobspec_ret == 0, "job_parse_jobspec parsed %s", jobspec_filename);
+
+        ok (nnodes_after_jobspec == job->nnodes,
+            "job_parse_jobspec correctly parsed nnodes %d=%d",
+            nnodes_after_jobspec, job->nnodes);
+
+        R_ret = parse_R (job, R_filename);
+        ok (R_ret == 0, "job_parse_R parsed %s", R_filename);
+
+        ok (nnodes_after_R == job->nnodes,
+            "job_parse_R correctly parsed nnodes %d=%d",
+            nnodes_after_R, job->nnodes);
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_ntasks (void)
+{
+    struct test_ntasks *test;
+
+    test = ntasks_tests;
+    while (test->jobspec_filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *jobspec_filename = test->jobspec_filename;
+        const char *R_filename = test->R_filename;
+        int ntasks_after_jobspec = test->ntasks_after_jobspec;
+        int ntasks_after_R = test->ntasks_after_R;
+        int jobspec_ret, R_ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        jobspec_ret = parse_jobspec (job, jobspec_filename);
+        ok (jobspec_ret == 0, "job_parse_jobspec parsed %s", jobspec_filename);
+
+        ok (ntasks_after_jobspec == job->ntasks,
+            "job_parse_jobspec correctly parsed ntasks %d=%d",
+            ntasks_after_jobspec, job->ntasks);
+
+        R_ret = parse_R (job, R_filename);
+        ok (R_ret == 0, "job_parse_R parsed %s", R_filename);
+
+        ok (ntasks_after_R == job->ntasks,
+            "job_parse_R correctly parsed ntasks %d=%d",
+            ntasks_after_R, job->ntasks);
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_ncores (void)
+{
+    struct test_ncores *test;
+
+    test = ncores_tests;
+    while (test->jobspec_filename) {
+        struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+        const char *jobspec_filename = test->jobspec_filename;
+        const char *R_filename = test->R_filename;
+        int ncores_after_jobspec = test->ncores_after_jobspec;
+        int ncores_after_R = test->ncores_after_R;
+        int jobspec_ret, R_ret;
+
+        if (!job)
+            BAIL_OUT ("job_create failed");
+
+        jobspec_ret = parse_jobspec (job, jobspec_filename);
+        ok (jobspec_ret == 0, "job_parse_jobspec parsed %s", jobspec_filename);
+
+        ok (ncores_after_jobspec == job->ncores,
+            "job_parse_jobspec correctly parsed ncores %d=%d",
+            ncores_after_jobspec, job->ncores);
+
+        R_ret = parse_R (job, R_filename);
+        ok (R_ret == 0, "job_parse_R parsed %s", R_filename);
+
+        ok (ncores_after_R == job->ncores,
+            "job_parse_R correctly parsed ncores %d=%d",
+            ncores_after_R, job->ncores);
+
+        job_destroy (job);
+        test++;
+    }
+}
+
+static void test_jobspec_update (void)
+{
+    struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+    const char *filename = TEST_SRCDIR "/jobspec/1slot.jobspec";
+    char *data;
+    int ret;
+    const char *command = NULL;
+    const char *name = NULL;
+    const char *queue = NULL;
+    const char *tmp = NULL;
+    double duration;
+    json_t *o;
+
+    if (!job)
+        BAIL_OUT ("job_create failed");
+
+    read_file (filename, (void **)&data);
+
+    if (!(o = json_pack ("{s:[{s:[s] s:s s:{s:i}}] s:s s:s s:f s:s}",
+                         "tasks",
+                           "command", "ls",
+                           "slot", "task",
+                           "count",
+                             "per_slot", 1,
+                         "attributes.system.job.name", "foo",
+                         "attributes.system.queue", "bar",
+                         "attributes.system.duration", 42.0,
+                         "dummy", "dummy")))
+        BAIL_OUT ("json_pack failed");
+
+    if (job_parse_jobspec_fatal (job, data, o) < 0)
+        BAIL_OUT ("cannot load basic jobspec");
+
+    json_decref (o);
+
+    ret = json_unpack (job->jobspec,
+                       "{s:[{s:[s]}]}",
+                       "tasks",
+                         "command", &command);
+    ok (ret == 0, "parsed initial jobspec command");
+
+    ret = json_unpack (job->jobspec,
+                       "{s:{s:{s?{s?s}}}}",
+                       "attributes",
+                       "system",
+                       "job",
+                       "name", &name);
+    ok (ret == 0, "parsed initial jobspec name");
+
+    ret = json_unpack (job->jobspec,
+                       "{s:{s?{s?s s:F}}}",
+                       "attributes",
+                       "system",
+                       "queue", &queue,
+                       "duration", &duration);
+    ok (ret == 0, "parsed initial jobspec queue, duration");
+
+    ok (command && streq (command, "ls"), "initial jobspec command == ls");
+    ok (name && streq (name, "foo"), "initial jobspec name == foo");
+    ok (queue && streq (queue, "bar"), "initial jobspec queue == bar");
+    ok (duration == 42.0, "initial jobspec duration == 42.0");
+
+    ok (job->name && streq (job->name, "foo"), "initial job->name == foo");
+    ok (job->queue && streq (job->queue, "bar"), "initial job->queue == foo");
+    ok (job->duration == 42.0, "initial job->duration == 42.0");
+
+    ret = json_unpack (job->jobspec, "{s:s}", "dummy", &tmp);
+    ok (ret == -1, "job_parse_jobspec does not set non jobspec field");
+
+    ret = job_jobspec_update (job, NULL);
+    ok (ret == 0, "job_jobspec_update success with no update");
+
+    if (!(o = json_pack ("{s:[{s:[s] s:s s:{s:i}}] s:s s:s s:f}",
+                         "tasks",
+                           "command", "uptime",
+                           "slot", "task",
+                           "count",
+                             "per_slot", 1,
+                         "attributes.system.job.name", "monkey",
+                         "attributes.system.queue", "gorilla",
+                         "attributes.system.duration", 100.0)))
+        BAIL_OUT ("json_pack failed");
+    ret = job_jobspec_update (job, o);
+    ok (ret == 0, "job_jobspec_update");
+    json_decref (o);
+
+    ret = json_unpack (job->jobspec,
+                       "{s:[{s:[s]}]}",
+                       "tasks",
+                         "command", &command);
+    ok (ret == 0, "parsed updated jobspec command");
+
+    ret = json_unpack (job->jobspec,
+                       "{s:{s?{s?{s?s}}}}",
+                       "attributes",
+                       "system",
+                       "job",
+                       "name", &name);
+    ok (ret == 0, "parsed updated jobspec name");
+
+    ret = json_unpack (job->jobspec,
+                       "{s:{s?{s?s s:F}}}",
+                       "attributes",
+                       "system",
+                       "queue", &queue,
+                       "duration", &duration);
+    ok (ret == 0, "parsed updated jobspec queue, duration");
+
+    ok (command != NULL && streq (command, "uptime"), "jobspec command == uptime");
+    ok (name != NULL && streq (name, "monkey"), "jobspec name == monkey");
+    ok (queue != NULL && streq (queue, "gorilla"), "jobspec queue == gorilla");
+    ok (duration == 100.0, "jobspec duration == 100.0");
+
+    ok (job->name && streq (job->name, "monkey"), "job->name == monkey");
+    ok (job->queue && streq (job->queue, "gorilla"), "job->queue == gorilla");
+    ok (job->duration == 100.0, "job->duration == 100.0");
+
+    free (data);
+}
+
+static void test_R_update (void)
+{
+    struct job *job = job_create (NULL, FLUX_JOBID_ANY);
+    const char *filename = TEST_SRCDIR "/R/1node_1core.R";
+    char *data;
+    int ret;
+    double expiration;
+    const char *tmp = NULL;
+    json_t *o;
+
+    if (!job)
+        BAIL_OUT ("job_create failed");
+
+    read_file (filename, (void **)&data);
+
+    if (!(o = json_pack ("{s:f}", "expiration", 100.0)))
+        BAIL_OUT ("json_pack failed");
+
+    if (job_parse_R (job, data, o) < 0)
+        BAIL_OUT ("cannot load basic R");
+
+    json_decref (o);
+
+    ret = json_unpack (job->R,
+                       "{s:{s:F}}",
+                       "execution",
+                       "expiration", &expiration);
+    ok (ret == 0, "parsed initial R expiration");
+
+    ok (expiration == 100.0, "initial R expiration == 100.0");
+    ok (job->expiration == 100.0, "initial job->expiration == 100.0");
+
+    ret = job_R_update (job, NULL);
+    ok (ret == 0, "job_R_update success with no update");
+
+    if (!(o = json_pack ("{s:f s:s}",
+                         "expiration", 200.0,
+                         "dummy", "dummy")))
+        BAIL_OUT ("json_pack failed");
+    ret = job_R_update (job, o);
+    ok (ret == 0, "job_R_update");
+    json_decref (o);
+
+    ret = json_unpack (job->R,
+                       "{s:{s:F}}",
+                       "execution",
+                       "expiration", &expiration);
+    ok (ret == 0, "parsed updated R expiration");
+
+    ok (expiration == 200.0, "R expiration == 200.0");
+    ok (job->expiration == 200.0, "job->expiration == 200.0");
+
+    ret = json_unpack (job->R, "{s?s}", "dummy", &tmp);
+    ok (ret == 0, "parsed updated R dummy");
+
+    ok (tmp == NULL, "R not updated with illegal update key");
+
+    free (data);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_jobspec_corner_case ();
+    test_jobspec_job_name ();
+    test_jobspec_cwd ();
+    test_jobspec_queue ();
+    test_jobspec_project_bank ();
+    test_jobspec_duration ();
+    test_R_corner_case ();
+    test_R_ranks ();
+    test_R_nodelist ();
+    test_nnodes ();
+    test_ntasks ();
+    test_ncores ();
+    test_jobspec_update ();
+    test_R_update ();
+
+    done_testing ();
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/job-list/test/jobspec/1node.jobspec b/src/modules/job-list/test/jobspec/1node.jobspec
new file mode 100644
index 000000000000..ab96c7bb7076
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/1node.jobspec
@@ -0,0 +1,40 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 1,
+      "exclusive": true,
+      "with": [
+        {
+          "type": "slot",
+          "count": 1,
+          "with": [
+            {
+              "type": "core",
+              "count": 1
+            }
+          ],
+          "label": "task"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/1node_1slot_nonexclusive.jobspec b/src/modules/job-list/test/jobspec/1node_1slot_nonexclusive.jobspec
new file mode 100644
index 000000000000..2af55a5c8221
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/1node_1slot_nonexclusive.jobspec
@@ -0,0 +1,39 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 1,
+      "with": [
+        {
+          "type": "slot",
+          "count": 1,
+          "with": [
+            {
+              "type": "core",
+              "count": 1
+            }
+          ],
+          "label": "task"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/1node_perresourcecore4.jobspec b/src/modules/job-list/test/jobspec/1node_perresourcecore4.jobspec
new file mode 100644
index 000000000000..06207973a496
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/1node_perresourcecore4.jobspec
@@ -0,0 +1,48 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 1,
+      "exclusive": true,
+      "with": [
+        {
+          "type": "slot",
+          "count": 1,
+          "with": [
+            {
+              "type": "core",
+              "count": 1
+            }
+          ],
+          "label": "task"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "shell": {
+        "options": {
+          "per-resource": {
+            "type": "core",
+            "count": 4
+          }
+        }
+      },
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/1node_perresourcenode4.jobspec b/src/modules/job-list/test/jobspec/1node_perresourcenode4.jobspec
new file mode 100644
index 000000000000..9f7a2959c8b7
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/1node_perresourcenode4.jobspec
@@ -0,0 +1,48 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 1,
+      "exclusive": true,
+      "with": [
+        {
+          "type": "slot",
+          "count": 1,
+          "with": [
+            {
+              "type": "core",
+              "count": 1
+            }
+          ],
+          "label": "task"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "shell": {
+        "options": {
+          "per-resource": {
+            "type": "node",
+            "count": 4
+          }
+        }
+      },
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/1slot.jobspec b/src/modules/job-list/test/jobspec/1slot.jobspec
new file mode 100644
index 000000000000..e5bbcc7aa77e
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/1slot.jobspec
@@ -0,0 +1,33 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/1slot_4core.jobspec b/src/modules/job-list/test/jobspec/1slot_4core.jobspec
new file mode 100644
index 000000000000..6219ea3ad0ed
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/1slot_4core.jobspec
@@ -0,0 +1,33 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 4
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/1slot_perresourcecore4.jobspec b/src/modules/job-list/test/jobspec/1slot_perresourcecore4.jobspec
new file mode 100644
index 000000000000..9e1bc6939827
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/1slot_perresourcecore4.jobspec
@@ -0,0 +1,41 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "shell": {
+        "options": {
+          "per-resource": {
+            "type": "core",
+            "count": 4
+          }
+        }
+      },
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/1slot_project_bank.jobspec b/src/modules/job-list/test/jobspec/1slot_project_bank.jobspec
new file mode 100644
index 000000000000..9a7bcf1d22b5
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/1slot_project_bank.jobspec
@@ -0,0 +1,35 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job",
+      "project": "myproject",
+      "bank": "mybank"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/4node.jobspec b/src/modules/job-list/test/jobspec/4node.jobspec
new file mode 100644
index 000000000000..5119a015f761
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/4node.jobspec
@@ -0,0 +1,40 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 4,
+      "exclusive": true,
+      "with": [
+        {
+          "type": "slot",
+          "count": 1,
+          "with": [
+            {
+              "type": "core",
+              "count": 1
+            }
+          ],
+          "label": "task"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/4node_1slot_nonexclusive.jobspec b/src/modules/job-list/test/jobspec/4node_1slot_nonexclusive.jobspec
new file mode 100644
index 000000000000..8e6f5ead6b06
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/4node_1slot_nonexclusive.jobspec
@@ -0,0 +1,39 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 4,
+      "with": [
+        {
+          "type": "slot",
+          "count": 1,
+          "with": [
+            {
+              "type": "core",
+              "count": 1
+            }
+          ],
+          "label": "task"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/4node_4slot_nonexclusive.jobspec b/src/modules/job-list/test/jobspec/4node_4slot_nonexclusive.jobspec
new file mode 100644
index 000000000000..b4c404bea42b
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/4node_4slot_nonexclusive.jobspec
@@ -0,0 +1,39 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 4,
+      "with": [
+        {
+          "type": "slot",
+          "count": 4,
+          "with": [
+            {
+              "type": "core",
+              "count": 1
+            }
+          ],
+          "label": "task"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/4node_perresourcecore4.jobspec b/src/modules/job-list/test/jobspec/4node_perresourcecore4.jobspec
new file mode 100644
index 000000000000..809b361dfc0a
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/4node_perresourcecore4.jobspec
@@ -0,0 +1,48 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 4,
+      "exclusive": true,
+      "with": [
+        {
+          "type": "slot",
+          "count": 1,
+          "with": [
+            {
+              "type": "core",
+              "count": 1
+            }
+          ],
+          "label": "task"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "shell": {
+        "options": {
+          "per-resource": {
+            "type": "core",
+            "count": 4
+          }
+        }
+      },
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/4node_perresourcenode4.jobspec b/src/modules/job-list/test/jobspec/4node_perresourcenode4.jobspec
new file mode 100644
index 000000000000..5e1ecfa68eae
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/4node_perresourcenode4.jobspec
@@ -0,0 +1,48 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 4,
+      "exclusive": true,
+      "with": [
+        {
+          "type": "slot",
+          "count": 1,
+          "with": [
+            {
+              "type": "core",
+              "count": 1
+            }
+          ],
+          "label": "task"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "shell": {
+        "options": {
+          "per-resource": {
+            "type": "node",
+            "count": 4
+          }
+        }
+      },
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/4slot.jobspec b/src/modules/job-list/test/jobspec/4slot.jobspec
new file mode 100644
index 000000000000..3c7f063820b8
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/4slot.jobspec
@@ -0,0 +1,33 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 4,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/4slot_perresourcecore4.jobspec b/src/modules/job-list/test/jobspec/4slot_perresourcecore4.jobspec
new file mode 100644
index 000000000000..9fd681d0948b
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/4slot_perresourcecore4.jobspec
@@ -0,0 +1,41 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 4,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "shell": {
+        "options": {
+          "per-resource": {
+            "type": "core",
+            "count": 4
+          }
+        }
+      },
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/cwd_not_specified.jobspec b/src/modules/job-list/test/jobspec/cwd_not_specified.jobspec
new file mode 100644
index 000000000000..ca00d6ee7794
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/cwd_not_specified.jobspec
@@ -0,0 +1,32 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/duration_alt.jobspec b/src/modules/job-list/test/jobspec/duration_alt.jobspec
new file mode 100644
index 000000000000..e09758a9f355
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/duration_alt.jobspec
@@ -0,0 +1,33 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 100,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_attributes_system_job.jobspec b/src/modules/job-list/test/jobspec/invalid_attributes_system_job.jobspec
new file mode 100644
index 000000000000..0d0d281b282b
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_attributes_system_job.jobspec
@@ -0,0 +1,34 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job",
+      "job": "notanobject"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_attributes_system_missing_duration.jobspec b/src/modules/job-list/test/jobspec/invalid_attributes_system_missing_duration.jobspec
new file mode 100644
index 000000000000..9114e66bb508
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_attributes_system_missing_duration.jobspec
@@ -0,0 +1,39 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 1,
+      "exclusive": true,
+      "with": [
+        {
+          "type": "slot",
+          "count": 1,
+          "with": [
+            {
+              "type": "core",
+              "count": 1
+            }
+          ],
+          "label": "task"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_command_array.jobspec b/src/modules/job-list/test/jobspec/invalid_command_array.jobspec
new file mode 100644
index 000000000000..dbc2084e76b3
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_command_array.jobspec
@@ -0,0 +1,31 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": "foo",
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_command_string.jobspec b/src/modules/job-list/test/jobspec/invalid_command_string.jobspec
new file mode 100644
index 000000000000..d8d28876be33
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_command_string.jobspec
@@ -0,0 +1,33 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        0
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_json.jobspec b/src/modules/job-list/test/jobspec/invalid_json.jobspec
new file mode 100644
index 000000000000..257cc5642cb1
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_json.jobspec
@@ -0,0 +1 @@
+foo
diff --git a/src/modules/job-list/test/jobspec/invalid_per_resource_missing_type.jobspec b/src/modules/job-list/test/jobspec/invalid_per_resource_missing_type.jobspec
new file mode 100644
index 000000000000..1edcad1f2f6f
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_per_resource_missing_type.jobspec
@@ -0,0 +1,40 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "shell": {
+        "options": {
+          "per-resource": {
+            "count": 8
+          },
+        }
+      },
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_resources.jobspec b/src/modules/job-list/test/jobspec/invalid_resources.jobspec
new file mode 100644
index 000000000000..2951ff852b0b
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_resources.jobspec
@@ -0,0 +1,21 @@
+{
+  "resources": "foo",
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_resources_invalid_count.jobspec b/src/modules/job-list/test/jobspec/invalid_resources_invalid_count.jobspec
new file mode 100644
index 000000000000..ca0461bc0324
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_resources_invalid_count.jobspec
@@ -0,0 +1,33 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 0,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_resources_invalid_type.jobspec b/src/modules/job-list/test/jobspec/invalid_resources_invalid_type.jobspec
new file mode 100644
index 000000000000..1b9df2e0ffb3
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_resources_invalid_type.jobspec
@@ -0,0 +1,33 @@
+{
+  "resources": [
+    {
+      "type": "foo",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_resources_missing_count.jobspec b/src/modules/job-list/test/jobspec/invalid_resources_missing_count.jobspec
new file mode 100644
index 000000000000..84031868e3a3
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_resources_missing_count.jobspec
@@ -0,0 +1,32 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_resources_missing_type.jobspec b/src/modules/job-list/test/jobspec/invalid_resources_missing_type.jobspec
new file mode 100644
index 000000000000..1f5dfbee5189
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_resources_missing_type.jobspec
@@ -0,0 +1,32 @@
+{
+  "resources": [
+    {
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_resources_nocores.jobspec b/src/modules/job-list/test/jobspec/invalid_resources_nocores.jobspec
new file mode 100644
index 000000000000..3e902404bc21
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_resources_nocores.jobspec
@@ -0,0 +1,27 @@
+{
+  "resources": [
+    {
+      "type": "node",
+      "count": 1,
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_resources_noslots.jobspec b/src/modules/job-list/test/jobspec/invalid_resources_noslots.jobspec
new file mode 100644
index 000000000000..9aa04c5af5f5
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_resources_noslots.jobspec
@@ -0,0 +1,27 @@
+{
+  "resources": [
+    {
+      "type": "gpu",
+      "count": 1,
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_tasks_array.jobspec b/src/modules/job-list/test/jobspec/invalid_tasks_array.jobspec
new file mode 100644
index 000000000000..623049bd2be4
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_tasks_array.jobspec
@@ -0,0 +1,23 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": "foo",
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_tasks_missing_command.jobspec b/src/modules/job-list/test/jobspec/invalid_tasks_missing_command.jobspec
new file mode 100644
index 000000000000..77428ec431b8
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_tasks_missing_command.jobspec
@@ -0,0 +1,30 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/invalid_version.jobspec b/src/modules/job-list/test/jobspec/invalid_version.jobspec
new file mode 100644
index 000000000000..0d6fdebd4ccb
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/invalid_version.jobspec
@@ -0,0 +1,33 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 999
+}
diff --git a/src/modules/job-list/test/jobspec/job_name_alt.jobspec b/src/modules/job-list/test/jobspec/job_name_alt.jobspec
new file mode 100644
index 000000000000..1d39fa744a7c
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/job_name_alt.jobspec
@@ -0,0 +1,36 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job",
+      "job": {
+        "name": "altname"
+      }
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/missing_attributes.jobspec b/src/modules/job-list/test/jobspec/missing_attributes.jobspec
new file mode 100644
index 000000000000..89ceb54b89c5
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/missing_attributes.jobspec
@@ -0,0 +1,27 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/missing_resources.jobspec b/src/modules/job-list/test/jobspec/missing_resources.jobspec
new file mode 100644
index 000000000000..e41aaf9ac011
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/missing_resources.jobspec
@@ -0,0 +1,20 @@
+{
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/missing_tasks.jobspec b/src/modules/job-list/test/jobspec/missing_tasks.jobspec
new file mode 100644
index 000000000000..5897dae5987a
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/missing_tasks.jobspec
@@ -0,0 +1,22 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/jobspec/missing_version.jobspec b/src/modules/job-list/test/jobspec/missing_version.jobspec
new file mode 100644
index 000000000000..7ffbdeebd272
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/missing_version.jobspec
@@ -0,0 +1,32 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job"
+    }
+  },
+}
diff --git a/src/modules/job-list/test/jobspec/queue_specified.jobspec b/src/modules/job-list/test/jobspec/queue_specified.jobspec
new file mode 100644
index 000000000000..a7122c0554ed
--- /dev/null
+++ b/src/modules/job-list/test/jobspec/queue_specified.jobspec
@@ -0,0 +1,34 @@
+{
+  "resources": [
+    {
+      "type": "slot",
+      "count": 1,
+      "with": [
+        {
+          "type": "core",
+          "count": 1
+        }
+      ],
+      "label": "task"
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "slot": "task",
+      "count": {
+        "per_slot": 1
+      }
+    }
+  ],
+  "attributes": {
+    "system": {
+      "duration": 0,
+      "cwd": "/tmp/job",
+      "queue": "batch"
+    }
+  },
+  "version": 1
+}
diff --git a/src/modules/job-list/test/match.c b/src/modules/job-list/test/match.c
new file mode 100644
index 000000000000..205953bdc710
--- /dev/null
+++ b/src/modules/job-list/test/match.c
@@ -0,0 +1,2118 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/modules/job-list/job_data.h"
+#include "src/modules/job-list/match.h"
+#include "ccan/str/str.h"
+
+/* normally created by job-list "main code" and passed to job_match().
+ * we create a global one here and initialize it manually.
+ */
+struct match_ctx mctx = { .h = NULL,
+                          .max_hostlist = 1024,
+                          .max_comparisons = 0 };
+
+static void list_constraint_create_corner_case (const char *str,
+                                                const char *msg)
+{
+    struct list_constraint *c;
+    flux_error_t error;
+    json_error_t jerror;
+    json_t *jc;
+
+    if (!(jc = json_loads (str, 0, &jerror)))
+        BAIL_OUT ("json constraint invalid: %s", jerror.text);
+
+    c = list_constraint_create (&mctx, jc, &error);
+
+    ok (c == NULL, "list_constraint_create fails on %s", msg);
+    diag ("error: %s", error.text);
+    json_decref (jc);
+}
+
+static void test_corner_case (void)
+{
+    ok (job_match (NULL, NULL, NULL) < 0
+        && errno == EINVAL,
+        "job_match returns EINVAL on NULL inputs");
+
+    ok (list_constraint_create (NULL, NULL, NULL) == NULL
+        && errno == EINVAL,
+        "list_constraint_create fails on all NULL inputs");
+
+    list_constraint_create_corner_case ("{\"userid\":[1], \"name\":[\"foo\"] }",
+                                        "object with too many keys");
+    list_constraint_create_corner_case ("{\"userid\":1}",
+                                        "object with values not array");
+    list_constraint_create_corner_case ("{\"foo\":[1]}",
+                                        "object with invalid operation");
+    list_constraint_create_corner_case ("{\"userid\":[\"foo\"]}",
+                                        "userid value not integer");
+    list_constraint_create_corner_case ("{\"name\":[1]}",
+                                        "name value not string");
+    list_constraint_create_corner_case ("{\"queue\":[1]}",
+                                        "queue value not string");
+    list_constraint_create_corner_case ("{\"states\":[0.0]}",
+                                        "states value not integer or string");
+    list_constraint_create_corner_case ("{\"states\":[\"foo\"]}",
+                                        "states value not valid string");
+    list_constraint_create_corner_case ("{\"states\":[8192]}",
+                                        "states value not valid integer");
+    list_constraint_create_corner_case ("{\"results\":[0.0]}",
+                                        "results value not integer or string");
+    list_constraint_create_corner_case ("{\"results\":[\"foo\"]}",
+                                        "results value not valid string");
+    list_constraint_create_corner_case ("{\"results\":[8192]}",
+                                        "results value not valid integer");
+    list_constraint_create_corner_case ("{\"t_depend\":[]}",
+                                        "t_depend value not specified");
+    list_constraint_create_corner_case ("{\"t_depend\":[1.0]}",
+                                        "t_depend value in invalid format (int)");
+    list_constraint_create_corner_case ("{\"t_depend\":[\"0.0\"]}",
+                                        "t_depend no comparison operator");
+    list_constraint_create_corner_case ("{\"t_depend\":[\">=foof\"]}",
+                                        "t_depend value invalid (str)");
+    list_constraint_create_corner_case ("{\"t_depend\":[\">=-1.0\"]}",
+                                        "t_depend value < 0.0 (str)");
+    list_constraint_create_corner_case ("{\"not\":[1]}",
+                                        "sub constraint not a constraint");
+}
+
+static struct job *setup_job (uint32_t userid,
+                              const char *name,
+                              const char *queue,
+                              const char *nodelist,
+                              const char *ranks,
+                              flux_job_state_t state,
+                              flux_job_result_t result,
+                              double t_submit,
+                              double t_depend,
+                              double t_run,
+                              double t_cleanup,
+                              double t_inactive)
+{
+    struct job *job;
+    int bitmask = 0x1;
+    if (!(job = job_create (NULL, FLUX_JOBID_ANY)))
+        BAIL_OUT ("failed to create job");
+    job->userid = userid;
+    if (name)
+        job->name = name;
+    if (queue)
+        job->queue = queue;
+    if (nodelist) {
+        /* N.B. internally is not const, so strdup it */
+        if (!(job->nodelist = strdup (nodelist)))
+            BAIL_OUT ("failed to strdup nodelist");
+    }
+   if (ranks) {
+        /* N.B. internally is not const, so strdup it */
+        if (!(job->ranks = strdup (ranks)))
+            BAIL_OUT ("failed to strdup ranks");
+    }
+    job->state = state;
+    if (state) {
+        /* Assume all jobs run, we don't skip any states, so add bitmask
+         * for all states lower than configured one
+         */
+        job->states_mask = job->state;
+        while (!(job->states_mask & bitmask)) {
+            job->states_mask |= bitmask;
+            bitmask <<= 1;
+        }
+    }
+    job->result = result;
+    job->t_submit = t_submit;
+    job->t_depend = t_depend;
+    job->t_run = t_run;
+    job->t_cleanup = t_cleanup;
+    job->t_inactive = t_inactive;
+    /* assume for all tests */
+    job->submit_version = 1;
+    return job;
+}
+
+static struct list_constraint *create_list_constraint (const char *constraint)
+{
+    struct list_constraint *c;
+    flux_error_t error;
+    json_error_t jerror;
+    json_t *jc = NULL;
+
+    if (constraint) {
+        if (!(jc = json_loads (constraint, 0, &jerror)))
+            BAIL_OUT ("json constraint invalid: %s", jerror.text);
+    }
+
+    if (!(c = list_constraint_create (&mctx, jc, &error)))
+        BAIL_OUT ("list constraint create fail: %s", error.text);
+
+    json_decref (jc);
+    return c;
+}
+
+static void test_basic_special_cases (void)
+{
+    struct job *job = setup_job (0,
+                                 NULL,
+                                 NULL,
+                                 NULL,
+                                 NULL,
+                                 0,
+                                 0,
+                                 0.0,
+                                 0.0,
+                                 0.0,
+                                 0.0,
+                                 0.0);
+    struct list_constraint *c;
+    flux_error_t error;
+    int rv;
+
+    c = create_list_constraint ("{}");
+    rv = job_match (job, c, &error);
+    ok (rv == true, "empty object works as expected");
+    list_constraint_destroy (c);
+
+    c = create_list_constraint (NULL);
+    rv = job_match (job, c, &error);
+    ok (rv == true, "NULL constraint works as expected");
+    list_constraint_destroy (c);
+
+    job_destroy (job);
+}
+
+struct basic_userid_test {
+    uint32_t userid;
+    int expected;
+};
+
+struct basic_userid_constraint_test {
+    const char *constraint;
+    struct basic_userid_test tests[4];
+} basic_userid_tests[] = {
+    {
+        "{ \"userid\": [ ] }",
+        {
+            { 42, false, },
+            {  0, false, },
+        },
+    },
+    {
+        "{ \"userid\": [ 42 ] }",
+        {
+            { 42, true,  },
+            { 43, false, },
+            {  0, false, },
+        },
+    },
+    {
+        "{ \"userid\": [ 42, 43 ] }",
+        {
+            { 42, true,  },
+            { 43, true,  },
+            { 44, false, },
+            {  0, false, },
+        },
+    },
+    /* FLUX_USERID_UNKNOWN = 0xFFFFFFFF */
+    {
+        "{ \"userid\": [ -1 ] }",
+        {
+            { 42, true,  },
+            { 43, true,  },
+            {  0, false, },
+        },
+    },
+    {
+        NULL,
+        {
+            { 0, false, },
+        },
+    },
+};
+
+static void test_basic_userid (void)
+{
+    struct basic_userid_constraint_test *ctests = basic_userid_tests;
+    int index = 0;
+
+    while (ctests->constraint) {
+        struct basic_userid_test *tests = ctests->tests;
+        struct list_constraint *c;
+        int index2 = 0;
+
+        c = create_list_constraint (ctests->constraint);
+        while (tests->userid) {
+            struct job *job;
+            flux_error_t error;
+            int rv;
+            job = setup_job (tests->userid,
+                             NULL,
+                             NULL,
+                             NULL,
+                             NULL,
+                             0,
+                             0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0);
+            rv = job_match (job, c, &error);
+            ok (rv == tests->expected,
+                "basic userid job match test #%d/#%d",
+                index, index2);
+            job_destroy (job);
+            index2++;
+            tests++;
+        }
+
+        index++;
+        list_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+struct basic_name_test {
+    const char *name;
+    int expected;
+    bool end;                   /* name can be NULL */
+};
+
+struct basic_name_constraint_test {
+    const char *constraint;
+    struct basic_name_test tests[5];
+} basic_name_tests[] = {
+    {
+        "{ \"name\": [ ] }",
+        {
+            /* N.B. name can potentially be NULL */
+            {  NULL, false, false, },
+            {  NULL, false,  true, },
+        },
+    },
+    {
+        "{ \"name\": [ \"foo\" ] }",
+        {
+            /* N.B. name can potentially be NULL */
+            {  NULL, false, false, },
+            { "foo",  true, false, },
+            { "bar", false, false, },
+            {  NULL, false,  true, },
+        },
+    },
+    {
+        "{ \"name\": [ \"foo\", \"bar\" ] }",
+        {
+            /* N.B. name can potentially be NULL */
+            {  NULL, false, false, },
+            { "foo",  true, false, },
+            { "bar",  true, false, },
+            { "baz", false, false, },
+            {  NULL, false,  true, },
+        },
+    },
+    {
+        NULL,
+        {
+            { NULL, false,  true, },
+        },
+    },
+};
+
+static void test_basic_name (void)
+{
+    struct basic_name_constraint_test *ctests = basic_name_tests;
+    int index = 0;
+
+    while (ctests->constraint) {
+        struct basic_name_test *tests = ctests->tests;
+        struct list_constraint *c;
+        int index2 = 0;
+
+        c = create_list_constraint (ctests->constraint);
+        while (!tests->end) {
+            struct job *job;
+            flux_error_t error;
+            int rv;
+            job = setup_job (0,
+                             tests->name,
+                             NULL,
+                             NULL,
+                             NULL,
+                             0,
+                             0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0);
+            rv = job_match (job, c, &error);
+            ok (rv == tests->expected,
+                "basic name job match test #%d/#%d",
+                index, index2);
+            job_destroy (job);
+            index2++;
+            tests++;
+        }
+
+        index++;
+        list_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+struct basic_queue_test {
+    const char *queue;
+    int expected;
+    bool end;                   /* queue can be NULL */
+};
+
+struct basic_queue_constraint_test {
+    const char *constraint;
+    struct basic_queue_test tests[5];
+} basic_queue_tests[] = {
+    {
+        "{ \"queue\": [ ] }",
+        {
+            /* N.B. queue can potentially be NULL */
+            {  NULL, false, false, },
+            {  NULL, false,  true, },
+        },
+    },
+    {
+        "{ \"queue\": [ \"foo\" ] }",
+        {
+            /* N.B. queue can potentially be NULL */
+            {  NULL, false, false, },
+            { "foo",  true, false, },
+            { "bar", false, false, },
+            {  NULL, false,  true, },
+        },
+    },
+    {
+        "{ \"queue\": [ \"foo\", \"bar\" ] }",
+        {
+            /* N.B. queue can potentially be NULL */
+            {  NULL, false, false, },
+            { "foo",  true, false, },
+            { "bar",  true, false, },
+            { "baz", false, false, },
+            {  NULL, false,  true, },
+        },
+    },
+    {
+        NULL,
+        {
+            { NULL, false,  true, },
+        },
+    },
+};
+
+static void test_basic_queue (void)
+{
+    struct basic_queue_constraint_test *ctests = basic_queue_tests;
+    int index = 0;
+
+    while (ctests->constraint) {
+        struct basic_queue_test *tests = ctests->tests;
+        struct list_constraint *c;
+        int index2 = 0;
+
+        c = create_list_constraint (ctests->constraint);
+        while (!tests->end) {
+            struct job *job;
+            flux_error_t error;
+            int rv;
+            job = setup_job (0,
+                             NULL,
+                             tests->queue,
+                             NULL,
+                             NULL,
+                             0,
+                             0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0);
+            rv = job_match (job, c, &error);
+            ok (rv == tests->expected,
+                "basic queue job match test #%d/#%d",
+                index, index2);
+            job_destroy (job);
+            index2++;
+            tests++;
+        }
+
+        index++;
+        list_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+struct basic_states_test {
+    flux_job_state_t state;
+    int expected;
+};
+
+struct basic_states_constraint_test {
+    const char *constraint;
+    struct basic_states_test tests[4];
+} basic_states_tests[] = {
+    {
+        "{ \"states\": [ ] }",
+        {
+            { FLUX_JOB_STATE_NEW, false,  },
+            {                  0, false, },
+        },
+    },
+    {
+        /* sanity check integer inputs work, we assume FLUX_JOB_STATE_NEW
+         * will always be 1, use strings everywhere else
+         */
+        "{ \"states\": [ 1 ] }",
+        {
+            { FLUX_JOB_STATE_NEW, true,  },
+            {                  0, false, },
+        },
+    },
+    {
+        "{ \"states\": [ \"sched\" ] }",
+        {
+            { FLUX_JOB_STATE_SCHED,  true, },
+            {   FLUX_JOB_STATE_RUN, false, },
+            {                    0, false, },
+        },
+    },
+    {
+        "{ \"states\": [ \"sched\", \"RUN\" ] }",
+        {
+            {    FLUX_JOB_STATE_SCHED, true,  },
+            {      FLUX_JOB_STATE_RUN, true,  },
+            { FLUX_JOB_STATE_INACTIVE, false, },
+            {                       0, false, },
+        },
+    },
+    {
+        NULL,
+        {
+            { 0, false, },
+        },
+    },
+};
+
+static void test_basic_states (void)
+{
+    struct basic_states_constraint_test *ctests = basic_states_tests;
+    int index = 0;
+
+    while (ctests->constraint) {
+        struct basic_states_test *tests = ctests->tests;
+        struct list_constraint *c;
+        int index2 = 0;
+
+        c = create_list_constraint (ctests->constraint);
+        while (tests->state) {
+            struct job *job;
+            flux_error_t error;
+            int rv;
+            job = setup_job (0,
+                             NULL,
+                             NULL,
+                             NULL,
+                             NULL,
+                             tests->state,
+                             0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0);
+            rv = job_match (job, c, &error);
+            ok (rv == tests->expected,
+                "basic states job match test #%d/#%d",
+                index, index2);
+            job_destroy (job);
+            index2++;
+            tests++;
+        }
+
+        index++;
+        list_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+struct basic_results_test {
+    flux_job_state_t state;
+    flux_job_result_t result;
+    int expected;
+};
+
+struct basic_results_constraint_test {
+    const char *constraint;
+    struct basic_results_test tests[4];
+} basic_results_tests[] = {
+    {
+        "{ \"results\": [ ] }",
+        {
+            { FLUX_JOB_STATE_NEW, FLUX_JOB_RESULT_COMPLETED, false, },
+            {                  0,                         0, false, },
+        },
+    },
+    {
+        /* sanity check integer inputs work, we assume
+         * FLUX_JOB_RESULT_COMPLETED will always be 1, use strings
+         * everywhere else
+         */
+        "{ \"results\": [ 1 ] }",
+        {
+            { FLUX_JOB_STATE_INACTIVE, FLUX_JOB_RESULT_COMPLETED,  true, },
+            {                       0,                         0, false, },
+        },
+    },
+    {
+        "{ \"results\": [ \"completed\" ] }",
+        {
+            { FLUX_JOB_STATE_RUN,                              0, false, },
+            { FLUX_JOB_STATE_INACTIVE, FLUX_JOB_RESULT_COMPLETED,  true, },
+            { FLUX_JOB_STATE_INACTIVE,    FLUX_JOB_RESULT_FAILED, false, },
+            {                       0,                         0, false, },
+        },
+    },
+    {
+        "{ \"results\": [ \"completed\", \"FAILED\" ] }",
+        {
+            { FLUX_JOB_STATE_INACTIVE, FLUX_JOB_RESULT_COMPLETED,  true, },
+            { FLUX_JOB_STATE_INACTIVE,    FLUX_JOB_RESULT_FAILED,  true, },
+            { FLUX_JOB_STATE_INACTIVE,  FLUX_JOB_RESULT_CANCELED, false, },
+            {                       0,                         0, false, },
+        },
+    },
+    {
+        NULL,
+        {
+            { 0, 0, false, },
+        },
+    },
+};
+
+static void test_basic_results (void)
+{
+    struct basic_results_constraint_test *ctests = basic_results_tests;
+    int index = 0;
+
+    while (ctests->constraint) {
+        struct basic_results_test *tests = ctests->tests;
+        struct list_constraint *c;
+        int index2 = 0;
+
+        c = create_list_constraint (ctests->constraint);
+        while (tests->state) {  /* result can be 0, iterate on state > 0 */
+            struct job *job;
+            flux_error_t error;
+            int rv;
+            job = setup_job (0,
+                             NULL,
+                             NULL,
+                             NULL,
+                             NULL,
+                             tests->state,
+                             tests->result,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0);
+            rv = job_match (job, c, &error);
+            ok (rv == tests->expected,
+                "basic results job match test #%d/#%d",
+                index, index2);
+            job_destroy (job);
+            index2++;
+            tests++;
+        }
+
+        index++;
+        list_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+static void test_corner_case_hostlist (void)
+{
+    struct list_constraint *c;
+    flux_error_t error;
+    json_error_t jerror;
+    json_t *jc = NULL;
+
+    /* hostrange exceeds maximum allowed */
+
+    if (!(jc = json_loads ("{ \"hostlist\": [ \"foo[1-5000]\" ] }", 0, &jerror)))
+        BAIL_OUT ("json constraint invalid: %s", jerror.text);
+
+    c = list_constraint_create (&mctx, jc, &error);
+    ok (c == NULL,
+        "list_constraint_create fail hostlist with excess hosts: %s",
+        error.text);
+
+    json_decref (jc);
+}
+
+struct basic_hostlist_test {
+    const char *nodelist;
+    bool expected;
+    bool end;                   /* nodelist can be NULL */
+};
+
+struct basic_hostlist_constraint_test {
+    const char *constraint;
+    struct basic_hostlist_test tests[9];
+} basic_hostlist_tests[] = {
+    {
+        "{ \"hostlist\": [ ] }",
+        {
+            /* N.B. nodelist can potentially be NULL */
+            {  NULL, false, false, },
+            {  NULL, false,  true, },
+        },
+    },
+    {
+        "{ \"hostlist\": [ \"foo1\" ] }",
+        {
+            /* N.B. host can potentially be NULL */
+            {  NULL,      false, false, },
+            { "foo1",      true, false, },
+            { "foo2",     false, false, },
+            { "foo[1-2]",  true, false, },
+            { "foo[2-3]", false, false, },
+            {  NULL,      false,  true, },
+        },
+    },
+    {
+        "{ \"hostlist\": [ \"foo[1-2]\" ] }",
+        {
+            /* N.B. host can potentially be NULL */
+            {  NULL,      false, false, },
+            { "foo1",      true, false, },
+            { "foo2",      true, false, },
+            { "foo[1-2]",  true, false, },
+            { "foo[2-3]",  true, false, },
+            { "foo[3-4]", false, false, },
+            {  NULL,      false,  true, },
+        },
+    },
+    {
+        "{ \"hostlist\": [ \"foo1\", \"foo2\", \"foo3\" ] }",
+        {
+            /* N.B. host can potentially be NULL */
+            {  NULL,      false, false, },
+            { "foo1",      true, false, },
+            { "foo2",      true, false, },
+            { "foo[1-2]",  true, false, },
+            { "foo[2-3]",  true, false, },
+            { "foo[3-4]",  true, false, },
+            { "foo4",     false, false, },
+            { "foo[4-5]", false, false, },
+            {  NULL,      false,  true, },
+        },
+    },
+    {
+        NULL,
+        {
+            { NULL, false,  true, },
+        },
+    },
+};
+
+static void test_basic_hostlist (void)
+{
+    struct basic_hostlist_constraint_test *ctests = basic_hostlist_tests;
+    int index = 0;
+
+    while (ctests->constraint) {
+        struct basic_hostlist_test *tests = ctests->tests;
+        struct list_constraint *c;
+        int index2 = 0;
+
+        c = create_list_constraint (ctests->constraint);
+        while (!tests->end) {
+            struct job *job;
+            flux_error_t error;
+            int rv;
+            job = setup_job (0,
+                             NULL,
+                             NULL,
+                             tests->nodelist,
+                             NULL,
+                             0,
+                             0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0);
+            rv = job_match (job, c, &error);
+            ok (rv == tests->expected,
+                "basic host job match test #%d/#%d",
+                index, index2);
+            job_destroy (job);
+            index2++;
+            tests++;
+        }
+
+        index++;
+        list_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+
+struct basic_ranks_test {
+    const char *ranks;
+    bool expected;
+    bool end;                   /* ranks can be NULL */
+};
+
+struct basic_ranks_constraint_test {
+    const char *constraint;
+    struct basic_ranks_test tests[9];
+} basic_ranks_tests[] = {
+    {
+        "{ \"ranks\": [ ] }",
+        {
+            /* N.B. ranks can potentially be NULL */
+            {  NULL, false, false, },
+            {  NULL, false,  true, },
+        },
+    },
+    {
+        "{ \"ranks\": [ \"1\" ] }",
+        {
+            /* N.B. ranks can potentially be NULL */
+            {  NULL, false, false, },
+            { "1",    true, false, },
+            { "2",   false, false, },
+            { "1-2",  true, false, },
+            { "2-3", false, false, },
+            {  NULL, false, true, },
+        },
+    },
+    {
+        "{ \"ranks\": [ \"1-2\" ] }",
+        {
+            /* N.B. ranks can potentially be NULL */
+            {  NULL, false, false, },
+            { "1",    true, false, },
+            { "2",    true, false, },
+            { "1-2",  true, false, },
+            { "2-3",  true, false, },
+            { "3-4", false, false, },
+            {  NULL, false,  true, },
+        },
+    },
+    {
+        "{ \"ranks\": [ \"1\", \"2\", \"3\" ] }",
+        {
+            /* N.B. ranks can potentially be NULL */
+            {  NULL, false, false, },
+            { "1",    true, false, },
+            { "2",    true, false, },
+            { "1-2",  true, false, },
+            { "2-3",  true, false, },
+            { "3-4",  true, false, },
+            { "4",   false, false, },
+            { "4-5", false, false, },
+            {  NULL, false,  true, },
+        },
+    },
+    {
+        NULL,
+        {
+            { NULL, false,  true, },
+        },
+    },
+};
+
+static void test_basic_ranks (void)
+{
+    struct basic_ranks_constraint_test *ctests = basic_ranks_tests;
+    int index = 0;
+
+    while (ctests->constraint) {
+        struct basic_ranks_test *tests = ctests->tests;
+        struct list_constraint *c;
+        int index2 = 0;
+
+        c = create_list_constraint (ctests->constraint);
+        while (!tests->end) {
+            struct job *job;
+            flux_error_t error;
+            int rv;
+            job = setup_job (0,
+                             NULL,
+                             NULL,
+                             NULL,
+                             tests->ranks,
+                             0,
+                             0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0);
+            rv = job_match (job, c, &error);
+            ok (rv == tests->expected,
+                "basic rank job match test #%d/#%d job->ranks = %s (expected %d got %d)",
+                index, index2, tests->ranks, tests->expected, rv);
+            job_destroy (job);
+            index2++;
+            tests++;
+        }
+
+        index++;
+        list_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+struct basic_timestamp_test {
+    flux_job_state_t state;
+    int submit_version;
+    double t_submit;
+    double t_depend;
+    double t_run;
+    double t_cleanup;
+    double t_inactive;
+    int expected;
+    bool end;                   /* timestamps can be 0 */
+};
+
+struct basic_timestamp_constraint_test {
+    const char *constraint;
+    struct basic_timestamp_test tests[7];
+} basic_timestamp_tests[] = {
+    {
+        "{ \"t_submit\": [ \">=0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, true, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, true, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, true, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, true, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, true, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_depend\": [ \">=0.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, true,  false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, true,  false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, true,  false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, true,  false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, true,  false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true,  false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    /* N.B. t_run >= 0 is false if state RUN not yet reached */
+    {
+        "{ \"t_run\": [ \">=0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, true,  false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, true,  false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true,  false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    /* N.B. t_cleanup >= 0 is false if state CLEANUP not yet reached */
+    {
+        "{ \"t_cleanup\": [ \">=0.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, true,  false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true,  false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    /* N.B. t_inactive >= 0 is false if state INACTIVE not yet reached */
+    {
+        "{ \"t_inactive\": [ \">=0.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true,  false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \"<100.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0,  true, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \"<=100.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0,  true, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \"<50.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \"<=50.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0,  true, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \"<25.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \"<=25.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \">100.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \">=100.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \">50.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \">=50.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0,  true, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \">25.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0,  true, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_inactive\": [ \">=25.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0,  true, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    /*
+     * Need to test special legacy case, submit_version == 0 where
+     * `t_depend` means `t_submit`.  So all tests fail for <15.0 when
+     * submit version == 1, but should all pass for submit version == 0.
+     */
+    {
+        "{ \"t_depend\": [ \"<15.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_SCHED,    1, 10.0, 20.0,  0.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_RUN,      1, 10.0, 20.0, 30.0,  0.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_CLEANUP,  1, 10.0, 20.0, 30.0, 40.0,  0.0, false, false, },
+            { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        "{ \"t_depend\": [ \"<15.0\" ] }",
+        {
+            { FLUX_JOB_STATE_DEPEND,   0, 10.0, 20.0,  0.0,  0.0,  0.0,  true, false, },
+            { FLUX_JOB_STATE_PRIORITY, 0, 10.0, 20.0,  0.0,  0.0,  0.0,  true, false, },
+            { FLUX_JOB_STATE_SCHED,    0, 10.0, 20.0,  0.0,  0.0,  0.0,  true, false, },
+            { FLUX_JOB_STATE_RUN,      0, 10.0, 20.0, 30.0,  0.0,  0.0,  true, false, },
+            { FLUX_JOB_STATE_CLEANUP,  0, 10.0, 20.0, 30.0, 40.0,  0.0,  true, false, },
+            { FLUX_JOB_STATE_INACTIVE, 0, 10.0, 20.0, 30.0, 40.0, 50.0,  true, false, },
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+    {
+        NULL,
+        {
+            { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false,  true, },
+        },
+    },
+};
+
+static void test_basic_timestamp (void)
+{
+    struct basic_timestamp_constraint_test *ctests = basic_timestamp_tests;
+    int index = 0;
+
+    while (ctests->constraint) {
+        struct basic_timestamp_test *tests = ctests->tests;
+        struct list_constraint *c;
+        int index2 = 0;
+
+        c = create_list_constraint (ctests->constraint);
+        while (!tests->end) {
+            struct job *job;
+            flux_error_t error;
+            int rv;
+            job = setup_job (0,
+                             NULL,
+                             NULL,
+                             NULL,
+                             NULL,
+                             tests->state,
+                             0,
+                             tests->t_submit,
+                             tests->t_depend,
+                             tests->t_run,
+                             tests->t_cleanup,
+                             tests->t_inactive);
+            /* special for legacy corner case */
+            job->submit_version = tests->submit_version;
+            rv = job_match (job, c, &error);
+            ok (rv == tests->expected,
+                "basic timestamp job match test #%d/#%d",
+                index, index2);
+            job_destroy (job);
+            index2++;
+            tests++;
+        }
+
+        index++;
+        list_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+struct basic_conditionals_test {
+    uint32_t userid;
+    const char *name;
+    int expected;
+};
+
+struct basic_conditionals_constraint_test {
+    const char *constraint;
+    struct basic_conditionals_test tests[5];
+} basic_conditionals_tests[] = {
+    {
+        "{ \"or\": [] }",
+        {
+            { 42, "foo",  true, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        "{ \"and\": [] }",
+        {
+            { 42, "foo",  true, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        "{ \"not\": [] }",
+        {
+            { 42, "foo", false, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        "{ \"not\": [ { \"userid\": [ 42 ] } ] }",
+        {
+            { 42, "foo", false, },
+            { 43, "foo",  true, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        "{ \"or\": \
+           [ \
+             { \"userid\": [ 42 ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            { 43, "bar", false, },
+            { 42, "bar",  true, },
+            { 43, "foo",  true, },
+            { 42, "foo",  true, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        "{ \"or\": \
+           [ \
+             { \"not\": [ { \"userid\": [ 42 ] } ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            { 43, "bar",  true, },
+            { 42, "bar", false, },
+            { 43, "foo",  true, },
+            { 42, "foo",  true, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        "{ \"not\": \
+           [ \
+             { \"or\": \
+               [ \
+                 { \"userid\": [ 42 ] }, \
+                 { \"name\": [ \"foo\" ] } \
+               ] \
+             } \
+           ] \
+        }",
+        {
+            { 43, "bar",  true, },
+            { 42, "bar", false, },
+            { 43, "foo", false, },
+            { 42, "foo", false, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"userid\": [ 42 ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            { 43, "bar", false, },
+            { 42, "bar", false, },
+            { 43, "foo", false, },
+            { 42, "foo",  true, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"not\": [ { \"userid\": [ 42 ] } ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            { 43, "bar", false, },
+            { 42, "bar", false, },
+            { 43, "foo",  true, },
+            { 42, "foo", false, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        "{ \"not\": \
+           [ \
+             { \"and\": \
+               [ \
+                 { \"userid\": [ 42 ] }, \
+                 { \"name\": [ \"foo\" ] } \
+               ] \
+             } \
+           ] \
+        }",
+        {
+            { 43, "bar",  true, },
+            { 42, "bar",  true, },
+            { 43, "foo",  true, },
+            { 42, "foo", false, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"or\": \
+               [ \
+                 { \"userid\": [ 42 ] }, \
+                 { \"userid\": [ 43 ] } \
+               ] \
+             }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            { 43, "bar", false, },
+            { 42, "bar", false, },
+            { 43, "foo",  true, },
+            { 42, "foo",  true, },
+            {  0,  NULL, false, },
+        },
+    },
+    {
+        NULL,
+        {
+            {  0,  NULL, false, },
+        },
+    },
+};
+
+static void test_basic_conditionals (void)
+{
+    struct basic_conditionals_constraint_test *ctests = basic_conditionals_tests;
+    int index = 0;
+
+    while (ctests->constraint) {
+        struct basic_conditionals_test *tests = ctests->tests;
+        struct list_constraint *c;
+        int index2 = 0;
+
+        c = create_list_constraint (ctests->constraint);
+        while (tests->userid) {
+            struct job *job;
+            flux_error_t error;
+            int rv;
+            job = setup_job (tests->userid,
+                             tests->name,
+                             NULL,
+                             NULL,
+                             NULL,
+                             0,
+                             0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0,
+                             0.0);
+            rv = job_match (job, c, &error);
+            ok (rv == tests->expected,
+                "basic conditionals job match test #%d/#%d",
+                index, index2);
+            job_destroy (job);
+            index2++;
+            tests++;
+        }
+
+        index++;
+        list_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+/* following tests emulate some "realworld"-ish matching */
+struct realworld_test {
+    uint32_t userid;
+    const char *name;
+    const char *queue;
+    const char *nodelist;
+    const char *ranks;
+    flux_job_state_t state;
+    flux_job_result_t result;
+    double t_run;
+    double t_inactive;
+    int expected;
+};
+
+struct realworld_constraint_test {
+    const char *constraint;
+    struct realworld_test tests[8];
+} realworld_tests[] = {
+    {
+        /* all the jobs in all states for a specific user */
+        "{ \"and\": \
+           [ \
+             { \"userid\": [ 42 ] }, \
+             { \"states\": [ \"pending\", \"running\", \"inactive\" ] } \
+           ] \
+        }",
+        {
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_DEPEND,
+                0,
+                0.0,
+                0.0,
+                true,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_RUN,
+                0,
+                0.0,
+                0.0,
+                true,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                0.0,
+                2000.0,
+                true,
+            },
+            {
+                43,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                0.0,
+                2000.0,
+                false,
+            },
+            {
+                0,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+                0,
+                0,
+                0.0,
+                0.0,
+                false
+            },
+        },
+    },
+    {
+        /* all the unsuccessful jobs for a specific user */
+        "{ \"and\": \
+           [ \
+             { \"userid\": [ 42 ] }, \
+             { \"results\": [ \"failed\", \"canceled\", \"timeout\" ] } \
+           ] \
+        }",
+        {
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_FAILED,
+                0.0,
+                2000.0,
+                true,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_CANCELED,
+                0.0,
+                2000.0,
+                true,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_TIMEOUT,
+                0.0,
+                2000.0,
+                true,
+            },
+            {
+                43,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_FAILED,
+                0.0,
+                2000.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_DEPEND,
+                0,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_RUN,
+                0,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                0,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+                0,
+                0,
+                0.0,
+                0.0,
+                false
+            },
+        },
+    },
+    {
+        /* all the pending and running jobs for a user, in two specific queues */
+        "{ \"and\": \
+           [ \
+             { \"userid\": [ 42 ] }, \
+             { \"states\" : [ \"pending\", \"running\" ] }, \
+             { \"queue\": [ \"batch\", \"debug\" ] } \
+           ] \
+        }",
+        {
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_DEPEND,
+                0,
+                0.0,
+                0.0,
+                true,
+            },
+            {
+                42,
+                "foo",
+                "debug",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_DEPEND,
+                0,
+                0.0,
+                0.0,
+                true,
+            },
+            {
+                42,
+                "foo",
+                "debug",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_RUN,
+                0,
+                0.0,
+                0.0,
+                true,
+            },
+            {
+                43,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_DEPEND,
+                0,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                0.0,
+                2000.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "gpu",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_DEPEND,
+                0,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                0,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+                0,
+                0,
+                0.0,
+                0.0,
+                false
+            },
+        },
+    },
+    {
+        /* jobs for a user, in queue batch, with specific job name, are running */
+        "{ \"and\": \
+           [ \
+             { \"userid\": [ 42 ] }, \
+             { \"queue\": [ \"batch\" ] }, \
+             { \"name\": [ \"foo\" ] }, \
+             { \"states\": [ \"running\" ] } \
+           ] \
+        }",
+        {
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_RUN,
+                0,
+                0.0,
+                0.0,
+                true,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_CLEANUP,
+                0,
+                0.0,
+                0.0,
+                true,
+            },
+            {
+                43,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_RUN,
+                0,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "debug",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_RUN,
+                0,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "bar",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_RUN,
+                0,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                0.0,
+                2000.0,
+                false,
+            },
+            {
+                0,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+                0,
+                0,
+                0.0,
+                0.0,
+                false
+            },
+        },
+    },
+    {
+        /* all the inactive jobs since a specific time (via t_inactve) */
+        "{ \"and\": \
+           [ \
+             { \"states\": [ \"inactive\" ] }, \
+             { \"t_inactive\": [ \">=500.0\" ] } \
+           ] \
+        }",
+        {
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_SCHED,
+                0,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_RUN,
+                0,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                0.0,
+                100.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                NULL,
+                NULL,
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                0.0,
+                1000.0,
+                true,
+            },
+            {
+                0,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+                0,
+                0,
+                0.0,
+                0.0,
+                false
+            },
+        },
+    },
+    {
+        /* jobs for a user that ran on specific hostlist */
+        "{ \"and\": \
+           [ \
+             { \"userid\": [ 42 ] }, \
+             { \"hostlist\": [ \"node1\", \"node2\" ] } \
+           ] \
+        }",
+        {
+            {
+                42,
+                "foo",
+                "batch",
+                "node[1-3]",
+                "0-2",
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                0.0,
+                0.0,
+                true,
+            },
+            {
+                43,
+                "foo",
+                "batch",
+                "node[1-3]",
+                "0-2",
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                "node[2-4]",
+                "1-3",
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                0.0,
+                0.0,
+                true,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                "node[3-4]",
+                "2-3",
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                0.0,
+                0.0,
+                false,
+            },
+            {
+                0,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+                0,
+                0,
+                0.0,
+                0.0,
+                false
+            },
+        },
+    },
+    {
+        /* jobs that ran on specific hostlist during a time period
+         */
+        "{ \"and\": \
+           [ \
+             { \"hostlist\": [ \"node1\", \"node2\" ] }, \
+             { \"t_run\": [ \">=500.0\" ] }, \
+             { \"t_inactive\": [ \"<=5000.0\" ] } \
+           ] \
+        }",
+        {
+            {
+                42,
+                "foo",
+                "batch",
+                "node[1-3]",
+                "0-2",
+                FLUX_JOB_STATE_RUN,
+                0,
+                1000.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                "node[1-3]",
+                "0-2",
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                1000.0,
+                2000.0,
+                true,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                "node[2-3]",
+                "1-2",
+                FLUX_JOB_STATE_RUN,
+                0,
+                1000.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                "node[2-3]",
+                "1-2",
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                1000.0,
+                2000.0,
+                true,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                "node[2-3]",
+                "1-2",
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                1000.0,
+                6000.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                "node[3-4]",
+                "2-3",
+                FLUX_JOB_STATE_RUN,
+                0,
+                1000.0,
+                0.0,
+                false,
+            },
+            {
+                42,
+                "foo",
+                "batch",
+                "node[3-4]",
+                "2-3",
+                FLUX_JOB_STATE_INACTIVE,
+                FLUX_JOB_RESULT_COMPLETED,
+                1000.0,
+                2000.0,
+                false,
+            },
+            {
+                0,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+                0,
+                0,
+                0.0,
+                0.0,
+                false
+            },
+        },
+    },
+    {
+        NULL,
+        {
+            {
+                0,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+                0,
+                0,
+                0.0,
+                0.0,
+                false
+            },
+        },
+    },
+};
+
+static void test_realworld (void)
+{
+    struct realworld_constraint_test *ctests = realworld_tests;
+    int index = 0;
+
+    while (ctests->constraint) {
+        struct realworld_test *tests = ctests->tests;
+        struct list_constraint *c;
+        int index2 = 0;
+
+        c = create_list_constraint (ctests->constraint);
+        while (tests->userid) {
+            struct job *job;
+            flux_error_t error;
+            int rv;
+            job = setup_job (tests->userid,
+                             tests->name,
+                             tests->queue,
+                             tests->nodelist,
+                             tests->ranks,
+                             tests->state,
+                             tests->result,
+                             0.0,
+                             0.0,
+                             tests->t_run,
+                             0.0,
+                             tests->t_inactive);
+            rv = job_match (job, c, &error);
+            ok (rv == tests->expected,
+                "realworld job match test #%d/#%d",
+                index, index2);
+            job_destroy (job);
+            index2++;
+            tests++;
+        }
+
+        index++;
+        list_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_corner_case ();
+    test_basic_special_cases ();
+    test_basic_userid ();
+    test_basic_name ();
+    test_basic_queue ();
+    test_basic_states ();
+    test_basic_results ();
+    test_corner_case_hostlist ();
+    test_basic_hostlist ();
+    test_basic_ranks ();
+    test_basic_timestamp ();
+    test_basic_conditionals ();
+    test_realworld ();
+
+    done_testing ();
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/job-list/test/state_match.c b/src/modules/job-list/test/state_match.c
new file mode 100644
index 000000000000..2a47ed18d97b
--- /dev/null
+++ b/src/modules/job-list/test/state_match.c
@@ -0,0 +1,1228 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/modules/job-list/job_data.h"
+#include "src/modules/job-list/state_match.h"
+#include "ccan/str/str.h"
+
+static void state_constraint_create_corner_case (const char *str,
+                                                 const char *msg)
+{
+    struct state_constraint *c;
+    flux_error_t error;
+    json_error_t jerror;
+    json_t *jc;
+
+    if (!(jc = json_loads (str, 0, &jerror)))
+        BAIL_OUT ("json constraint invalid: %s", jerror.text);
+
+    c = state_constraint_create (jc, &error);
+    ok (c == NULL, "state_constraint_create fails on %s", msg);
+    diag ("error: %s", error.text);
+    json_decref (jc);
+}
+
+static void test_corner_case (void)
+{
+    ok (state_match (0, NULL) == false,
+        "state_match returns false on NULL inputs");
+
+    state_constraint_create_corner_case ("{\"userid\":[1], \"name\":[\"foo\"] }",
+                                         "object with too many keys");
+    state_constraint_create_corner_case ("{\"userid\":1}",
+                                         "object with values not array");
+    state_constraint_create_corner_case ("{\"foo\":[1]}",
+                                         "object with invalid operation");
+    state_constraint_create_corner_case ("{\"not\":[1]}",
+                                         "sub constraint not a constraint");
+}
+
+/* expected array - expected values for
+ * FLUX_JOB_STATE_DEPEND
+ * FLUX_JOB_STATE_PRIORITY
+ * FLUX_JOB_STATE_SCHED
+ * FLUX_JOB_STATE_RUN
+ * FLUX_JOB_STATE_CLEANUP
+ * FLUX_JOB_STATE_INACTIVE
+ * FLUX_JOB_STATE_PENDING
+ * FLUX_JOB_STATE_RUNNING
+ * FLUX_JOB_STATE_ACTIVE
+ */
+struct state_match_constraint_test {
+    const char *constraint;
+    bool expected[9];
+} state_match_tests[] = {
+    /*
+     * Empty values tests
+     */
+    {
+        "{ \"states\": [ ] }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    {
+        "{ \"and\": [ ] }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    {
+        "{ \"or\": [ ] }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    {
+        "{ \"not\": [ ] }",
+        {
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+        }
+    },
+    /*
+     * Simple states tests
+     */
+    {
+        "{ \"states\": [ \"pending\" ] }",
+        {
+            true,
+            true,
+            true,
+            false,
+            false,
+            false,
+            true,
+            false,
+            true,
+        }
+    },
+    {
+        "{ \"and\": [ { \"states\": [ \"pending\" ] } ] }",
+        {
+            true,
+            true,
+            true,
+            false,
+            false,
+            false,
+            true,
+            false,
+            true,
+        }
+    },
+    {
+        "{ \"or\": [ { \"states\": [ \"pending\" ] } ] }",
+        {
+            true,
+            true,
+            true,
+            false,
+            false,
+            false,
+            true,
+            false,
+            true,
+        }
+    },
+    {
+        "{ \"not\": [ { \"states\": [ \"pending\" ] } ] }",
+        {
+            false,
+            false,
+            false,
+            true,
+            true,
+            true,
+            false,
+            true,
+            true,
+        }
+    },
+    /*
+     * Simple results tests
+     */
+    /* N.B. "results" assumes job state == INACTIVE */
+    {
+        "{ \"results\": [ \"completed\" ] }",
+        {
+            false,
+            false,
+            false,
+            false,
+            false,
+            true,
+            false,
+            false,
+            false,
+        }
+    },
+    /* N.B. Returning 'true' for FLUX_JOB_STATE_INACTIVE may be
+     * surprising here.  If the job state is FLUX_JOB_STATE_INACTIVE,
+     * the result of "results=COMPLETED" is "maybe true", b/c it
+     * depends on the actual result.  So the "not" of a "maybe true"
+     * is still "maybe true".
+     */
+    {
+        "{ \"not\": [ { \"results\": [ \"completed\" ] } ] }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    /*
+     * Simple timestamp tests
+     */
+    {
+        "{ \"t_submit\": [ 100.0 ] }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    {
+        "{ \"t_run\": [ \">100.0\" ] }",
+        {
+            false,
+            false,
+            false,
+            true,
+            true,
+            true,
+            false,
+            true,
+            true,
+        }
+    },
+    /* N.B. For state depend, priority, sched, is always false, so not
+     * makes it always true.  For states run, cleanup, and inactive is
+     * maybe true, so not maybe true = true.  So all would return
+     * true.
+     */
+    {
+        "{ \"not\": [ { \"t_run\": [ \"<=500\" ] } ] }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    /*
+     * AND tests w/ states
+     */
+    {
+        "{ \"and\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"states\": [ \"priority\" ] } \
+           ] \
+        }",
+        {
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+        }
+    },
+    {
+        "{ \"not\": \
+           [ \
+             { \"and\": \
+               [ \
+                 { \"states\": [ \"depend\" ] }, \
+                 { \"states\": [ \"priority\" ] } \
+               ] \
+             } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"not\": [ { \"states\": [ \"depend\" ] } ] }, \
+             { \"states\": [ \"priority\" ] } \
+           ] \
+        }",
+        {
+            false,
+            true,
+            false,
+            false,
+            false,
+            false,
+            true,
+            false,
+            true,
+        }
+    },
+    /*
+     * OR tests w/ states
+     */
+    {
+        "{ \"or\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"states\": [ \"priority\" ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            false,
+            false,
+            false,
+            false,
+            true,
+            false,
+            true,
+        }
+    },
+    {
+        "{ \"not\": \
+           [ \
+             { \"or\": \
+               [ \
+                 { \"states\": [ \"depend\" ] }, \
+                 { \"states\": [ \"priority\" ] } \
+               ] \
+             } \
+           ] \
+        }",
+        {
+            false,
+            false,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    {
+        "{ \"or\": \
+           [ \
+             { \"not\": [ { \"states\": [ \"depend\" ] } ] }, \
+             { \"states\": [ \"priority\" ] } \
+           ] \
+        }",
+        {
+            false,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    /*
+     * AND tests w/ states & results
+     */
+    {
+        "{ \"and\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"results\": [ \"completed\" ] } \
+           ] \
+        }",
+        {
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+        }
+    },
+    {
+        "{ \"not\": \
+           [ \
+             { \"and\": \
+               [ \
+                 { \"states\": [ \"depend\" ] }, \
+                 { \"results\": [ \"completed\" ] } \
+               ] \
+             } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"not\": [ { \"results\": [ \"completed\" ] } ] } \
+           ] \
+        }",
+        {
+            true,
+            false,
+            false,
+            false,
+            false,
+            false,
+            true,
+            false,
+            true,
+        }
+    },
+    /*
+     * OR tests w/ states & results
+     */
+    {
+        "{ \"or\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"results\": [ \"completed\" ] } \
+           ] \
+        }",
+        {
+            true,
+            false,
+            false,
+            false,
+            false,
+            true,
+            true,
+            false,
+            true,
+        }
+    },
+    {
+        "{ \"not\": \
+           [ \
+             { \"or\": \
+               [ \
+                 { \"states\": [ \"depend\" ] }, \
+                 { \"results\": [ \"completed\" ] } \
+               ] \
+             } \
+           ] \
+        }",
+        {
+            false,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    {
+        "{ \"or\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"not\": [ { \"results\": [ \"completed\" ] } ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    /*
+     * AND tests w/ states & t_inactive
+     */
+    {
+        "{ \"and\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"t_inactive\": [ \">=100.0\" ] } \
+           ] \
+        }",
+        {
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+        }
+    },
+    {
+        "{ \"not\": \
+           [ \
+             { \"and\": \
+               [ \
+                 { \"states\": [ \"depend\" ] }, \
+                 { \"t_inactive\": [ \">=100.0\" ] } \
+               ] \
+             } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"not\": [ { \"t_inactive\": [ \">=100.0\" ] } ] } \
+           ] \
+        }",
+        {
+            true,
+            false,
+            false,
+            false,
+            false,
+            false,
+            true,
+            false,
+            true,
+        }
+    },
+    /*
+     * OR tests w/ states & t_inactive
+     */
+    {
+        "{ \"or\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"t_inactive\": [ \">=100.0\" ] } \
+           ] \
+        }",
+        {
+            true,
+            false,
+            false,
+            false,
+            false,
+            true,
+            true,
+            false,
+            true,
+        }
+    },
+    {
+        "{ \"not\": \
+           [ \
+             { \"or\": \
+               [ \
+                 { \"states\": [ \"depend\" ] }, \
+                 { \"t_inactive\": [ \">=100.0\" ] } \
+               ] \
+             } \
+           ] \
+        }",
+        {
+            false,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    {
+        "{ \"or\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"not\": [ { \"t_inactive\": [ \">=100.0\" ] } ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        }
+    },
+    /*
+     * Simple non-states tests
+     */
+    {
+        "{ \"userid\": [ 42 ] }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    {
+        "{ \"not\": [ { \"userid\": [ 42 ] } ] }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    /*
+     * non-states AND tests
+     */
+    {
+        "{ \"and\": \
+           [ \
+             { \"userid\": [ 42 ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"not\": [ { \"userid\": [ 42 ] } ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    /*
+     * non-states OR tests
+     */
+    {
+        "{ \"or\": \
+           [ \
+             { \"userid\": [ 42 ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    {
+        "{ \"or\": \
+           [ \
+             { \"not\": [ { \"userid\": [ 42 ] } ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    /*
+     * states and non-states AND tests
+     */
+    {
+        "{ \"and\": \
+           [ \
+             { \"states\": [ \"running\" ] }, \
+             { \"userid\": [ 42 ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            false,
+            false,
+            false,
+            true,
+            true,
+            false,
+            false,
+            true,
+            true,
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"not\": [ { \"states\": [ \"running\" ] } ] }, \
+             { \"userid\": [ 42 ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            false,
+            false,
+            true,
+            true,
+            false,
+            true,
+        },
+    },
+    /* N.B. All returning true may be difficult to understand here.
+     * The states check is effectively irrelevant.  The userid or name
+     * could could be false, leading to the "and" constraint
+     * potentially being false for any job state.  So the full
+     * constraint could be true for any job state.
+     */
+    {
+        "{ \"not\": \
+           [ \
+             { \"and\": \
+               [ \
+                 { \"states\": [ \"running\" ] }, \
+                 { \"userid\": [ 42 ] }, \
+                 { \"name\": [ \"foo\" ] } \
+               ] \
+             } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    /*
+     * states and non-states OR tests
+     */
+    /* N.B. All states return true here, b/c the states check is sort
+     * of irrelevant, the userid or name checks could always return
+     * true, leading to the or statement to be true that any state
+     * could be matched with this constraint.
+     */
+    {
+        "{ \"or\": \
+           [ \
+             { \"states\": [ \"running\" ] }, \
+             { \"userid\": [ 42 ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    {
+        "{ \"or\": \
+           [ \
+             { \"not\": [ { \"states\": [ \"running\" ] } ] }, \
+             { \"userid\": [ 42 ] }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    {
+        "{ \"not\": \
+           [ \
+             { \"or\": \
+               [ \
+                 { \"states\": [ \"running\" ] }, \
+                 { \"userid\": [ 42 ] }, \
+                 { \"name\": [ \"foo\" ] } \
+               ] \
+             } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            false,
+            false,
+            true,
+            true,
+            false,
+            true,
+        },
+    },
+    /*
+     * complex tests, conditionals inside conditionals
+     */
+    {
+        "{ \"and\": \
+           [ \
+             { \"and\": \
+               [ \
+                 { \"states\": [ \"priority\" ] }, \
+                 { \"userid\": [ 42 ] } \
+               ] \
+             }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            false,
+            true,
+            false,
+            false,
+            false,
+            false,
+            true,
+            false,
+            true,
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"or\": \
+               [ \
+                 { \"states\": [ \"priority\" ] }, \
+                 { \"userid\": [ 42 ] } \
+               ] \
+             }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"and\": \
+               [ \
+                 { \"results\": [ \"completed\" ] }, \
+                 { \"userid\": [ 42 ] } \
+               ] \
+             }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            false,
+            false,
+            false,
+            false,
+            false,
+            true,
+            false,
+            false,
+            false,
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"or\": \
+               [ \
+                 { \"results\": [ \"completed\" ] }, \
+                 { \"userid\": [ 42 ] } \
+               ] \
+             }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"or\": \
+               [ \
+                 { \"states\": [ \"priority\" ] }, \
+                 { \"userid\": [ 42 ] } \
+               ] \
+             }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            false,
+            false,
+            false,
+            false,
+            false,
+            true,
+            false,
+            true,
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"not\": [ { \"states\": [ \"depend\" ] } ] }, \
+             { \"or\": \
+               [ \
+                 { \"states\": [ \"priority\" ] }, \
+                 { \"userid\": [ 42 ] } \
+               ] \
+             }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            false,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+            true,
+        },
+    },
+    {
+        "{ \"and\": \
+           [ \
+             { \"states\": [ \"depend\" ] }, \
+             { \"not\": \
+               [ \
+                 { \"or\": \
+                   [ \
+                     { \"states\": [ \"priority\" ] }, \
+                     { \"userid\": [ 42 ] } \
+                   ] \
+                 } \
+               ] \
+             }, \
+             { \"name\": [ \"foo\" ] } \
+           ] \
+        }",
+        {
+            true,
+            false,
+            false,
+            false,
+            false,
+            false,
+            true,
+            false,
+            true,
+        },
+    },
+    /* cover every constraint operator
+     * - every test here should fail as we AND several impossible things
+     */
+    {
+        "{ \"and\": \
+           [ \
+             { \"userid\": [ 42 ] }, \
+             { \"name\": [ \"foo\" ] }, \
+             { \"queue\": [ \"foo\" ] }, \
+             { \"hostlist\": [ \"bar\" ] }, \
+             { \"states\": [ \"running\" ] }, \
+             { \"results\": [ \"completed\" ] }, \
+             { \"t_submit\": [ \">=500.0\" ] }, \
+             { \"t_depend\": [ \">=100.0\" ] }, \
+             { \"t_run\": [ \"<=100.0\" ] }, \
+             { \"t_cleanup\": [ \">=100.0\" ] }, \
+             { \"t_inactive\": [ \"<=100.0\" ] } \
+           ] \
+        }",
+        {
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+        },
+    },
+    {
+        NULL,
+        {
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+            false,
+        },
+    },
+};
+
+static struct state_constraint *create_state_constraint (const char *constraint)
+{
+    struct state_constraint *c;
+    flux_error_t error;
+    json_error_t jerror;
+    json_t *jc = NULL;
+
+    if (constraint) {
+        if (!(jc = json_loads (constraint, 0, &jerror)))
+            BAIL_OUT ("json constraint invalid: %s", jerror.text);
+    }
+
+    if (!(c = state_constraint_create (jc, &error)))
+        BAIL_OUT ("constraint create fail: %s", error.text);
+
+    json_decref (jc);
+    return c;
+}
+
+static void test_state_match (void)
+{
+    struct state_match_constraint_test *ctests = state_match_tests;
+    struct state_constraint *c;
+    bool rv;
+    int index = 0;
+
+    /* First test special case */
+    c = create_state_constraint (NULL);
+    rv = state_match (FLUX_JOB_STATE_DEPEND, c);
+    ok (rv == ctests->expected[0], "state match test NULL DEPEND");
+    rv = state_match (FLUX_JOB_STATE_PRIORITY, c);
+    ok (rv == ctests->expected[1], "state match test NULL PRIORITY");
+    rv = state_match (FLUX_JOB_STATE_SCHED, c);
+    ok (rv == ctests->expected[2], "state match test NULL SCHED");
+    rv = state_match (FLUX_JOB_STATE_RUN, c);
+    ok (rv == ctests->expected[3], "state match test NULL RUN");
+    rv = state_match (FLUX_JOB_STATE_CLEANUP, c);
+    ok (rv == ctests->expected[4], "state match test NULL CLEANUP");
+    rv = state_match (FLUX_JOB_STATE_INACTIVE, c);
+    ok (rv == ctests->expected[5], "state match test NULL INACTIVE");
+    rv = state_match (FLUX_JOB_STATE_PENDING, c);
+    ok (rv == ctests->expected[6], "state match test NULL PENDING");
+    rv = state_match (FLUX_JOB_STATE_RUNNING, c);
+    ok (rv == ctests->expected[7], "state match test NULL RUNNING");
+    rv = state_match (FLUX_JOB_STATE_ACTIVE, c);
+    ok (rv == ctests->expected[8], "state match test NULL ACTIVE");
+
+    while (ctests->constraint) {
+        c = create_state_constraint (ctests->constraint);
+        rv = state_match (FLUX_JOB_STATE_DEPEND, c);
+        ok (rv == ctests->expected[0], "state match test #%d DEPEND", index);
+        rv = state_match (FLUX_JOB_STATE_PRIORITY, c);
+        ok (rv == ctests->expected[1], "state match test #%d PRIORITY", index);
+        rv = state_match (FLUX_JOB_STATE_SCHED, c);
+        ok (rv == ctests->expected[2], "state match test #%d SCHED", index);
+        rv = state_match (FLUX_JOB_STATE_RUN, c);
+        ok (rv == ctests->expected[3], "state match test #%d RUN", index);
+        rv = state_match (FLUX_JOB_STATE_CLEANUP, c);
+        ok (rv == ctests->expected[4], "state match test #%d CLEANUP", index);
+        rv = state_match (FLUX_JOB_STATE_INACTIVE, c);
+        ok (rv == ctests->expected[5], "state match test #%d INACTIVE", index);
+        rv = state_match (FLUX_JOB_STATE_PENDING, c);
+        ok (rv == ctests->expected[6], "state match test #%d PENDING", index);
+        rv = state_match (FLUX_JOB_STATE_RUNNING, c);
+        ok (rv == ctests->expected[7], "state match test #%d RUNNING", index);
+        rv = state_match (FLUX_JOB_STATE_ACTIVE, c);
+        ok (rv == ctests->expected[8], "state match test #%d ACTIVE", index);
+
+        index++;
+        state_constraint_destroy (c);
+        ctests++;
+    }
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_corner_case ();
+    test_state_match ();
+
+    done_testing ();
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/job-manager/Makefile.am b/src/modules/job-manager/Makefile.am
index 99f493010f60..cf7d8923ccc7 100644
--- a/src/modules/job-manager/Makefile.am
+++ b/src/modules/job-manager/Makefile.am
@@ -6,18 +6,29 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) $(JANSSON_CFLAGS)
+	-I$(top_srcdir)/src/common/libccan \
+	$(JANSSON_CFLAGS)
 
-fluxmod_LTLIBRARIES = job-manager.la
+noinst_LTLIBRARIES = \
+	libjob-manager.la
 
-job_manager_la_SOURCES = \
+jobtap_plugin_LTLIBRARIES = \
+	plugins/submit-hold.la \
+	plugins/alloc-bypass.la \
+	plugins/alloc-check.la \
+	plugins/perilog.la
+
+libjob_manager_la_SOURCES = \
 	job-manager.c \
 	job-manager.h \
 	job.c \
 	job.h \
+	conf.c \
+	conf.h \
 	submit.c \
 	submit.h \
 	drain.c \
@@ -34,20 +45,79 @@ job_manager_la_SOURCES = \
 	kill.c \
 	alloc.h \
 	alloc.c \
+	housekeeping.h \
+	housekeeping.c \
 	start.h \
 	start.c \
 	list.h \
 	list.c \
-	priority.h \
-	priority.c
+	purge.h \
+	purge.c \
+	urgency.h \
+	urgency.c \
+	annotate.h \
+	annotate.c \
+	journal.h \
+	journal.c \
+	getattr.h \
+	getattr.c \
+	prioritize.h \
+	prioritize.c \
+	queue.h \
+	queue.c \
+	jobtap-internal.h \
+	jobtap.h \
+	jobtap.c \
+	update.h \
+	update.c \
+	plugins/priority-default.c \
+	plugins/limit-job-size.c \
+	plugins/limit-duration.c \
+	plugins/dependency-after.c \
+	plugins/begin-time.c \
+	plugins/update-duration.c \
+	plugins/validate-duration.c \
+	plugins/history.c \
+	plugins/post-event.c
+
+fluxinclude_HEADERS = \
+	jobtap.h
+
+plugins_submit_hold_la_SOURCES = \
+	plugins/submit-hold.c
+plugins_submit_hold_la_LDFLAGS = \
+	$(fluxplugin_ldflags) \
+	-module
+
+plugins_alloc_bypass_la_SOURCES = \
+	plugins/alloc-bypass.c
+plugins_alloc_bypass_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/librlist/librlist.la
+plugins_alloc_bypass_la_LDFLAGS = \
+	$(fluxplugin_ldflags) \
+	-module
+
+plugins_alloc_check_la_SOURCES = \
+	plugins/alloc-check.c
+plugins_alloc_check_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/librlist/librlist.la
+plugins_alloc_check_la_LDFLAGS = \
+	$(fluxplugin_ldflags) \
+	-module
+
+plugins_perilog_la_SOURCES = \
+	plugins/perilog.c
+plugins_perilog_la_LIBADD = \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/librlist/librlist.la \
+	$(top_builddir)/src/common/libjob/libjob.la
+plugins_perilog_la_LDFLAGS = \
+	$(fluxplugin_ldflags) \
+	-module
 
-job_manager_la_LDFLAGS = $(fluxmod_ldflags) -module
-job_manager_la_LIBADD = $(fluxmod_libadd) \
-		    $(top_builddir)/src/common/libjob/libjob.la \
-		    $(top_builddir)/src/common/libflux-internal.la \
-		    $(top_builddir)/src/common/libflux-core.la \
-		    $(top_builddir)/src/common/libflux-optparse.la \
-		    $(ZMQ_LIBS)
 
 TESTS = \
 	test_job.t \
@@ -55,25 +125,25 @@ TESTS = \
 	test_raise.t \
 	test_kill.t \
 	test_restart.t \
-	test_submit.t
+	test_annotate.t
 
 test_ldadd = \
-        $(top_builddir)/src/modules/job-manager/event.o \
-        $(top_builddir)/src/modules/job-manager/job.o \
-        $(top_builddir)/src/modules/job-manager/alloc.o \
-        $(top_builddir)/src/modules/job-manager/start.o \
-        $(top_builddir)/src/modules/job-manager/drain.o \
-        $(top_builddir)/src/modules/job-manager/submit.o \
-        $(top_builddir)/src/modules/job-manager/wait.o \
+	libjob-manager.la \
 	$(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(top_builddir)/src/common/librlist/librlist.la \
 	$(top_builddir)/src/common/libjob/libjob.la \
-	$(top_builddir)/src/common/libflux-internal.la \
 	$(top_builddir)/src/common/libflux-core.la \
-	$(ZMQ_LIBS) $(LIBPTHREAD) $(JANSSON_LIBS)
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(LIBPTHREAD) \
+	$(JANSSON_LIBS)
 
 test_cppflags = \
 	$(AM_CPPFLAGS)
 
+test_ldflags = \
+	-no-install
+
 check_PROGRAMS = $(TESTS)
 
 TEST_EXTENSIONS = .t
@@ -84,32 +154,44 @@ test_job_t_SOURCES = test/job.c
 test_job_t_CPPFLAGS = $(test_cppflags)
 test_job_t_LDADD = \
         $(test_ldadd)
+test_job_t_LDFLAGS = \
+        $(test_ldflags)
 
 test_list_t_SOURCES = test/list.c
 test_list_t_CPPFLAGS = $(test_cppflags)
 test_list_t_LDADD = \
         $(top_builddir)/src/modules/job-manager/list.o \
-        $(test_ldadd)
+	$(test_ldadd)
+test_list_t_LDFLAGS = \
+        $(test_ldflags)
 
 test_raise_t_SOURCES = test/raise.c
 test_raise_t_CPPFLAGS = $(test_cppflags)
 test_raise_t_LDADD = \
         $(top_builddir)/src/modules/job-manager/raise.o \
         $(test_ldadd)
+test_raise_t_LDFLAGS = \
+        $(test_ldflags)
 
 test_kill_t_SOURCES = test/kill.c
 test_kill_t_CPPFLAGS = $(test_cppflags)
 test_kill_t_LDADD = \
         $(top_builddir)/src/modules/job-manager/kill.o \
         $(test_ldadd)
+test_kill_t_LDFLAGS = \
+        $(test_ldflags)
 
 test_restart_t_SOURCES = test/restart.c
 test_restart_t_CPPFLAGS = $(test_cppflags)
 test_restart_t_LDADD = \
         $(top_builddir)/src/modules/job-manager/restart.o \
         $(test_ldadd)
+test_restart_t_LDFLAGS = \
+        $(test_ldflags)
 
-test_submit_t_SOURCES = test/submit.c
-test_submit_t_CPPFLAGS = $(test_cppflags)
-test_submit_t_LDADD = \
+test_annotate_t_SOURCES = test/annotate.c
+test_annotate_t_CPPFLAGS = $(test_cppflags)
+test_annotate_t_LDADD = \
         $(test_ldadd)
+test_annotate_t_LDFLAGS = \
+        $(test_ldflags)
diff --git a/src/modules/job-manager/alloc.c b/src/modules/job-manager/alloc.c
index dfcb68bf34fa..d5e9e68716e2 100644
--- a/src/modules/job-manager/alloc.c
+++ b/src/modules/job-manager/alloc.c
@@ -10,119 +10,70 @@
 
 /* alloc.c - scheduler interface
  *
- * STARTUP:
- *
- * 1) Scheduler sends job-manager.sched-hello request:
- *   <empty payload>
- * Job manager responds with array of job objects that have allocated resources:
- *   {"alloc":[{"id":I, "priority":i, "userid":i, "t_submit":f},{},{},...]}
- * Scheduler should read those jobs' R from KVS and mark resources allocated.
- *
- * 2) Scheduler sends job-manager.sched-ready request:
- *   {"mode":s}
- * mode is scheduler's preference for throttling alloc requests:
- *   "single"    - limit of one pending alloc request (e.g. for FCFS)
- *   "unlimited" - no limit on number of pending alloc requests
- * Job manager responds with alloc queue depth (sched may ignore this)
- *   {"count":i}
- *
- * ALLOCATION (see notes 3-4 below):
- *
- * Job manager sends sched.alloc request:
- *   {"id":I, "priority":i, "userid":i, "t_submit":f}
- * Scheduler responds with:
- *   {"id":I, "type":i, "note"?:s}
- * Where type is one of:
- * 0 - resources allocated (sched commits R to KVS before responding)
- * 1 - annotation (just updates note, see below)
- * 2 - job cannot run (note is set to error string)
- * 3 - alloc was canceled
- * Type 0, 2, 3 are taken as final responses, type 1 is not.
- *
- * CANCELLATION
- *
- * Job manager sends sched.cancel request:
- *   {"id":I}
- * Scheduler does not respond to the cancellation, but does respond to
- * the alloc request for 'id' if still pending (see ALLOCATION above).
- *
- * DE-ALLOCATION (see notes 3-4 below):
- *
- * Job manager sends sched.free request:
- *   {"id":I}
- * Scheduler reads R from KVS and marks resources free and responds with
- *   {"id":I}
- *
- * EXCEPTION:
- *
- * Job manager sends a job-exception event:
- *   {"id":I, "type":s, "severity":i}
- * If severity=0 exception is received for a job with a pending alloc request,
- * scheduler responds with with type=2 (error).
- *
- * TEARDOWN:
- *
- * Scheduler sends each internally queued alloc request a regular ENOSYS
- * RPC error response.
- *
- * The receipt of a regular RPC error of any type from the scheduler
- * causes the job manager to assume the scheduler is unloading and to
- * stop sending requests.
- *
- * Notes:
- * 1) scheduler can be loaded after jobs have begun to be submitted
- * 2) scheduler can be unloaded without affecting workload
- * 3) alloc/free requests and responses are matched using jobid, not
- *    the normal matchtag, for scalability.
- * 4) a normal RPC error response to alloc/free triggers teardown
- * 5) 'note' in alloc response is intended to be human readable note displayed
- *    with job info (could be estimated start time, or error detail)
+ * Please refer to RFC27 for scheduler protocol
  *
  * TODO:
- * - handle type=1 annotation for queue listing (currently ignored)
  * - implement flow control (credit based?) interface mode
- * - handle post alloc request job priority change
  */
 
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
 #include <jansson.h>
-#include <czmq.h>
 #include <flux/core.h>
-#include <assert.h>
+#include <flux/schedutil.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/librlist/rlist.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
 
 #include "job.h"
 #include "alloc.h"
 #include "event.h"
 #include "drain.h"
-
-typedef enum {
-    SCHED_SINGLE,       // only allow one outstanding sched.alloc request
-    SCHED_UNLIMITED,    // send all sched.alloc requests immediately
-} sched_interface_t;
+#include "annotate.h"
+#include "raise.h"
+#include "queue.h"
+#include "housekeeping.h"
 
 struct alloc {
     struct job_manager *ctx;
     flux_msg_handler_t **handlers;
     zlistx_t *queue;
-    sched_interface_t mode;
-    bool ready;
-    bool disable;
-    char *disable_reason;
+    zlistx_t *sent;             // track jobs w/ alloc reqs, mode=limited only
+    bool scheduler_is_online;
     flux_watcher_t *prep;
     flux_watcher_t *check;
     flux_watcher_t *idle;
-    unsigned int alloc_pending_count; // for mode=single, max of 1
-    unsigned int free_pending_count;
+    unsigned int alloc_limit;   // will have a value of 0 in mode=unlimited
+    char *sched_sender;         // scheduler uuid for disconnect processing
 };
 
+static void requeue_pending (struct alloc *alloc, struct job *job)
+{
+    struct job_manager *ctx = alloc->ctx;
+
+    if (!job->alloc_pending)
+        return;
+    if (job_priority_queue_delete (alloc->sent, job) < 0)
+        flux_log (ctx->h, LOG_ERR, "failed to dequeue pending job");
+    job->alloc_pending = 0;
+    if (queue_started (alloc->ctx->queue, job)) {
+        if (job_priority_queue_insert (alloc->queue, job) < 0)
+            flux_log (ctx->h, LOG_ERR, "failed to enqueue job for scheduling");
+        job->alloc_queued = 1;
+    }
+    annotations_clear_and_publish (ctx, job, "sched");
+}
+
 /* Initiate teardown.  Clear any alloc/free requests, and clear
- * the alloc->ready flag to stop prep/check from allocating.
+ * the alloc->scheduler_is_online flag to stop prep/check from allocating.
  */
 static void interface_teardown (struct alloc *alloc, char *s, int errnum)
 {
-    if (alloc->ready) {
+    if (alloc->scheduler_is_online) {
         struct job *job;
         struct job_manager *ctx = alloc->ctx;
 
@@ -134,73 +85,33 @@ static void interface_teardown (struct alloc *alloc, char *s, int errnum)
             /* jobs with alloc request pending need to go back in the queue
              * so they will automatically send alloc again.
              */
-            if (job->alloc_pending) {
-                bool fwd = job->priority > FLUX_JOB_PRIORITY_DEFAULT ? true
-                                                                     : false;
-
-                assert (job->handle == NULL);
-                if (!(job->handle = zlistx_insert (alloc->queue, job, fwd)))
-                    flux_log_error (ctx->h, "%s: queue_insert", __FUNCTION__);
-                job->alloc_pending = 0;
-                job->alloc_queued = 1;
-            }
-            /* jobs with free request pending (much smaller window for this
-             * to be true) need to be picked up again after 'ready'.
-             */
-            job->free_pending = 0;
+            if (job->alloc_pending)
+                requeue_pending (alloc, job);
             job = zhashx_next (ctx->active_jobs);
         }
-        alloc->ready = false;
-        alloc->alloc_pending_count = 0;
-        alloc->free_pending_count = 0;
+        alloc->scheduler_is_online = false;
+        free (alloc->sched_sender);
+        alloc->sched_sender = NULL;
         drain_check (alloc->ctx->drain);
     }
 }
 
-/* Handle a sched.free response.
- */
-static void free_response_cb (flux_t *h, flux_msg_handler_t *mh,
-                              const flux_msg_t *msg, void *arg)
-{
-    struct job_manager *ctx = arg;
-    flux_jobid_t id = 0;
-    struct job *job;
-
-    if (flux_response_decode (msg, NULL, NULL) < 0)
-        goto teardown;
-    if (flux_msg_unpack (msg, "{s:I}", "id", &id) < 0)
-        goto teardown;
-    if (!(job = zhashx_lookup (ctx->active_jobs, &id))) {
-        flux_log (h, LOG_ERR, "sched.free-response: id=%ju not active",
-                  (uintmax_t)id);
-        errno = EINVAL;
-        goto teardown;
-    }
-    if (!job->has_resources) {
-        flux_log (h, LOG_ERR, "sched.free-response: id=%ju not allocated",
-                  (uintmax_t)id);
-        errno = EINVAL;
-        goto teardown;
-    }
-    job->free_pending = 0;
-    ctx->alloc->free_pending_count--;
-    if (event_job_post_pack (ctx->event, job, "free", NULL) < 0)
-        goto teardown;
-    return;
-teardown:
-    interface_teardown (ctx->alloc, "free response error", errno);
-}
-
-/* Send sched.free request for job.
- * Update flags.
+/* Send sched.free request.
  */
-int free_request (struct alloc *alloc, struct job *job)
+int free_request (struct alloc *alloc,
+                  flux_jobid_t id,
+                  json_t *R,
+                  bool final)
 {
     flux_msg_t *msg;
 
     if (!(msg = flux_request_encode ("sched.free", NULL)))
         return -1;
-    if (flux_msg_pack (msg, "{s:I}", "id", job->id) < 0)
+    if (flux_msg_pack (msg,
+                       "{s:I s:O s:b}",
+                       "id", id,
+                       "R", R,
+                       "final", final) < 0)
         goto error;
     if (flux_send (alloc->ctx->h, msg, 0) < 0)
         goto error;
@@ -223,9 +134,8 @@ int cancel_request (struct alloc *alloc, struct job *job)
                              FLUX_NODEID_ANY,
                              FLUX_RPC_NORESPONSE,
                              "{s:I}",
-                             "id",
-                             job->id))) {
-        flux_log_error (h, "sending sched.cancel id=%ju", (uintmax_t)job->id);
+                             "id", job->id))) {
+        flux_log_error (h, "sending sched.cancel id=%s", idf58 (job->id));
         return -1;
     }
     flux_future_destroy (f);
@@ -235,73 +145,122 @@ int cancel_request (struct alloc *alloc, struct job *job)
 /* Handle a sched.alloc response.
  * Update flags.
  */
-static void alloc_response_cb (flux_t *h, flux_msg_handler_t *mh,
-                               const flux_msg_t *msg, void *arg)
+static void alloc_response_cb (flux_t *h,
+                               flux_msg_handler_t *mh,
+                               const flux_msg_t *msg,
+                               void *arg)
 {
     struct job_manager *ctx = arg;
     struct alloc *alloc = ctx->alloc;
     flux_jobid_t id;
     int type;
-    const char *note = NULL;
+    char *note = NULL;
+    json_t *annotations = NULL;
+    json_t *R = NULL;
     struct job *job;
 
     if (flux_response_decode (msg, NULL, NULL) < 0)
         goto teardown; // ENOSYS here if scheduler not loaded/shutting down
-    if (flux_msg_unpack (msg, "{s:I s:i s?:s}",
-                              "id", &id,
-                              "type", &type,
-                              "note", &note) < 0)
-        goto teardown;
-    if (!(job = zhashx_lookup (ctx->active_jobs, &id))) {
-        flux_log (h, LOG_ERR, "sched.alloc-response: id=%ju not active",
-                  (uintmax_t)id);
-        errno = EINVAL;
+    if (flux_msg_unpack (msg,
+                         "{s:I s:i s?s s?o s?o}",
+                         "id", &id,
+                         "type", &type,
+                         "note", &note,
+                         "annotations", &annotations,
+                         "R", &R) < 0)
         goto teardown;
-    }
-    if (!job->alloc_pending) {
-        flux_log (h, LOG_ERR, "sched.alloc-response: id=%ju not requested",
-                  (uintmax_t)id);
-        errno = EINVAL;
-        goto teardown;
-    }
+
+    job = zhashx_lookup (ctx->active_jobs, &id);
+    if (job && !job->alloc_pending)
+        job = NULL;
+
     switch (type) {
-    case 0: // success
-        alloc->alloc_pending_count--;
-        job->alloc_pending = 0;
-        if (job->has_resources) {
+    case FLUX_SCHED_ALLOC_SUCCESS:
+        if (!R) {
+            flux_log (h, LOG_ERR, "sched.alloc-response: protocol error");
+            errno = EPROTO;
+            goto teardown;
+        }
+        (void)json_object_del (R, "scheduling");
+
+        if (!job) {
+            (void)free_request (alloc, id, R, true);
+            break;
+        }
+        if (job_priority_queue_delete (alloc->sent, job) < 0)
+            flux_log (ctx->h, LOG_ERR, "failed to dequeue pending job");
+        if (job->has_resources || job->R_redacted) {
             flux_log (h,
                       LOG_ERR,
-                      "sched.alloc-response: id=%ju already allocated",
-                      (uintmax_t)id);
+                      "sched.alloc-response: id=%s already allocated",
+                      idf58 (id));
             errno = EEXIST;
             goto teardown;
         }
-        if (event_job_post_pack (ctx->event, job, "alloc",
-                                 "{ s:s }",
-                                 "note", note ? note : "") < 0)
-            goto teardown;
+        job->R_redacted = json_incref (R);
+        if (annotations_update_and_publish (ctx, job, annotations) < 0)
+            flux_log_error (h, "annotations_update: id=%s", idf58 (id));
+
+        /*  Only modify job state after annotation event is published
+         */
+        job->alloc_pending = 0;
+        if (job->annotations) {
+            if (event_job_post_pack (ctx->event,
+                                     job,
+                                     "alloc",
+                                     0,
+                                     "{s:O}",
+                                     "annotations", job->annotations) < 0)
+                goto teardown;
+        }
+        else {
+            if (event_job_post_pack (ctx->event, job, "alloc", 0, NULL) < 0)
+                goto teardown;
+        }
         break;
-    case 1: // annotation FIXME
+    case FLUX_SCHED_ALLOC_ANNOTATE: // annotation
+        if (!annotations) {
+            errno = EPROTO;
+            goto teardown;
+        }
+        if (!job)
+            break;
+        if (annotations_update_and_publish (ctx, job, annotations) < 0)
+            flux_log_error (h, "annotations_update: id=%s", idf58 (id));
         break;
-    case 2: // error
-        alloc->alloc_pending_count--;
+    case FLUX_SCHED_ALLOC_DENY: // error
+        if (!job)
+            break;
         job->alloc_pending = 0;
-        if (event_job_post_pack (ctx->event, job, "exception",
-                                 "{ s:s s:i s:i s:s }",
-                                 "type", "alloc",
-                                 "severity", 0,
-                                 "userid", FLUX_USERID_UNKNOWN,
-                                 "note", note ? note : "") < 0)
+        if (job_priority_queue_delete (alloc->sent, job) < 0)
+            flux_log (ctx->h, LOG_ERR, "failed to dequeue pending job");
+        annotations_clear_and_publish (ctx, job, NULL);
+        if (raise_job_exception (ctx,
+                                 job,
+                                 "alloc",
+                                 0,
+                                 ctx->owner,
+                                 note) < 0)
             goto teardown;
         break;
-    case 3: // canceled
-        alloc->alloc_pending_count--;
+    case FLUX_SCHED_ALLOC_CANCEL:
+        if (!job)
+            break;
+        if (job->state == FLUX_JOB_STATE_SCHED)
+            requeue_pending (alloc, job);
+        else {
+            if (job_priority_queue_delete (alloc->sent, job) < 0)
+                flux_log (ctx->h, LOG_ERR, "failed to dequeue pending job");
+            annotations_clear_and_publish (ctx, job, NULL);
+        }
         job->alloc_pending = 0;
-        if (event_job_action (ctx->event, job) < 0) {
-            flux_log_error (h,
-                            "event_job_action id=%ju on alloc cancel",
-                            (uintmax_t)id);
-            goto teardown;
+        if (queue_started (alloc->ctx->queue, job)) {
+            if (event_job_action (ctx->event, job) < 0) {
+                flux_log_error (h,
+                                "event_job_action id=%s on alloc cancel",
+                                idf58 (id));
+                goto teardown;
+            }
         }
         drain_check (alloc->ctx->drain);
         break;
@@ -323,11 +282,13 @@ int alloc_request (struct alloc *alloc, struct job *job)
 
     if (!(msg = flux_request_encode ("sched.alloc", NULL)))
         return -1;
-    if (flux_msg_pack (msg, "{s:I s:i s:i s:f}",
-                            "id", job->id,
-                            "priority", job->priority,
-                            "userid", job->userid,
-                            "t_submit", job->t_submit) < 0)
+    if (flux_msg_pack (msg,
+                       "{s:I s:I s:I s:f s:O}",
+                       "id", job->id,
+                       "priority", (json_int_t)job->priority,
+                       "userid", (json_int_t) job->userid,
+                       "t_submit", job->t_submit,
+                       "jobspec", job->jobspec_redacted) < 0)
         goto error;
     if (flux_send (alloc->ctx->h, msg, 0) < 0)
         goto error;
@@ -339,47 +300,48 @@ int alloc_request (struct alloc *alloc, struct job *job)
 }
 
 /* sched-hello:
- * Scheduler obtains a list of jobs that have resources allocated.
+ * Scheduler obtains jobs that have resources allocated.
  */
-static void hello_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+static void hello_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     struct job_manager *ctx = arg;
     struct job *job;
-    json_t *o = NULL;
-    json_t *entry;
 
+    /* N.B. no "state" is set in struct alloc after a hello msg, so do
+     * not set ctx->alloc->sched_sender in here.  Do so only in the
+     * ready callback */
     if (flux_request_decode (msg, NULL, NULL) < 0)
         goto error;
+    if (!flux_msg_is_streaming (msg)) {
+        errno = EPROTO;
+        goto error;
+    }
     flux_log (h, LOG_DEBUG, "scheduler: hello");
-    if (!(o = json_array ()))
-        goto nomem;
     job = zhashx_first (ctx->active_jobs);
     while (job) {
-        if (job->has_resources) {
-            if (!(entry = json_pack ("{s:I s:i s:i s:f}",
-                                     "id", job->id,
-                                     "priority", job->priority,
-                                     "userid", job->userid,
-                                     "t_submit", job->t_submit)))
-                goto nomem;
-            if (json_array_append_new (o, entry) < 0) {
-                json_decref (entry);
-                goto nomem;
-            }
+        if (job->has_resources && !job->alloc_bypass) {
+            if (flux_respond_pack (h,
+                                   msg,
+                                   "{s:I s:I s:I s:f}",
+                                   "id", job->id,
+                                   "priority", job->priority,
+                                   "userid", (json_int_t) job->userid,
+                                   "t_submit", job->t_submit) < 0)
+                goto error;
         }
         job = zhashx_next (ctx->active_jobs);
     }
-    if (flux_respond_pack (h, msg, "{s:O}", "alloc", o) < 0)
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-    json_decref (o);
+    if (housekeeping_hello_respond (ctx->housekeeping, msg) < 0)
+        goto error;
+    if (flux_respond_error (h, msg, ENODATA, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
     return;
-nomem:
-    errno = ENOMEM;
 error:
     if (flux_respond_error (h, msg, errno, NULL) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    json_decref (o);
 }
 
 /* sched-ready:
@@ -387,25 +349,47 @@ static void hello_cb (flux_t *h, flux_msg_handler_t *mh,
  * and tells job-manager to start allocations.  job-manager tells
  * scheduler how many jobs are in the queue.
  */
-static void ready_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+static void ready_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     struct job_manager *ctx = arg;
     const char *mode;
+    int limit = 0;
     int count;
     struct job *job;
+    const char *sender;
 
-    if (flux_request_unpack (msg, NULL, "{s:s}", "mode", &mode) < 0)
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s?i}",
+                             "mode", &mode,
+                             "limit", &limit) < 0)
         goto error;
-    if (!strcmp (mode, "single"))
-        ctx->alloc->mode = SCHED_SINGLE;
-    else if (!strcmp (mode, "unlimited"))
-        ctx->alloc->mode = SCHED_UNLIMITED;
+    if (streq (mode, "limited")) {
+        if (limit <= 0) {
+            errno = EPROTO;
+            goto error;
+        }
+        ctx->alloc->alloc_limit = limit;
+    }
+    else if (streq (mode, "unlimited"))
+        ctx->alloc->alloc_limit = 0;
     else {
         errno = EPROTO;
         goto error;
     }
-    ctx->alloc->ready = true;
+    if (!(sender = flux_msg_route_first (msg))) {
+        flux_log (h, LOG_ERR, "%s: flux_msg_get_route_first: sender is NULL",
+                  __FUNCTION__);
+        goto error;
+    }
+    if (sender) {
+        if (!(ctx->alloc->sched_sender = strdup (sender)))
+            goto error;
+    }
+    ctx->alloc->scheduler_is_online = true;
     flux_log (h, LOG_DEBUG, "scheduler: ready %s", mode);
     count = zlistx_size (ctx->alloc->queue);
     if (flux_respond_pack (h, msg, "{s:i}", "count", count) < 0)
@@ -416,9 +400,9 @@ static void ready_cb (flux_t *h, flux_msg_handler_t *mh,
     job = zhashx_first (ctx->active_jobs);
     while (job) {
         /* N.B. first/next are NOT deletion safe but event_job_action()
-         * won't call zhashx_delete() for jobs in FLUX_JOB_CLEANUP state.
+         * won't call zhashx_delete() for jobs in FLUX_JOB_STATE_CLEANUP state.
          */
-        if (job->state == FLUX_JOB_CLEANUP && job->has_resources) {
+        if (job->state == FLUX_JOB_STATE_CLEANUP && job->has_resources) {
             if (event_job_action (ctx->event, job) < 0)
                 flux_log_error (h, "%s: event_job_action", __FUNCTION__);
         }
@@ -430,104 +414,150 @@ static void ready_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
 }
 
+
+static bool alloc_work_available (struct job_manager *ctx)
+{
+    struct job *job;
+
+    if (!ctx->alloc->scheduler_is_online) // scheduler is not ready for alloc
+        return false;
+    if (!(job = zlistx_first (ctx->alloc->queue))) // queue is empty
+        return false;
+    if (ctx->alloc->alloc_limit > 0 // alloc limit reached
+        && zlistx_size (ctx->alloc->sent) >= ctx->alloc->alloc_limit)
+        return false;
+    /* The alloc->queue is sorted from highest to lowest priority, so if the
+     * first job has priority=MIN (held), all other jobs must have the same
+     * priority, and no alloc requests can be sent.
+     */
+    if (job->priority == FLUX_JOB_PRIORITY_MIN)
+        return false;
+    return true;
+}
+
 /* prep:
  * Runs right before reactor calls poll(2).
  * If a job can be scheduled, start idle watcher.
  */
-static void prep_cb (flux_reactor_t *r, flux_watcher_t *w,
-                     int revents, void *arg)
+static void prep_cb (flux_reactor_t *r,
+                     flux_watcher_t *w,
+                     int revents,
+                     void *arg)
 {
     struct job_manager *ctx = arg;
-    struct alloc *alloc = ctx->alloc;
 
-    if (!alloc->ready || alloc->disable)
-        return;
-    if (alloc->mode == SCHED_SINGLE && alloc->alloc_pending_count > 0)
-        return;
-    if (zlistx_first (alloc->queue))
-        flux_watcher_start (alloc->idle);
+    if (alloc_work_available (ctx))
+        flux_watcher_start (ctx->alloc->idle);
 }
 
 /* check:
  * Runs right after reactor calls poll(2).
  * Stop idle watcher, and send next alloc request, if available.
  */
-static void check_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
+static void check_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
 {
     struct job_manager *ctx = arg;
     struct alloc *alloc = ctx->alloc;
     struct job *job;
 
     flux_watcher_stop (alloc->idle);
-    if (!alloc->ready || alloc->disable)
-        return;
-    if (alloc->mode == SCHED_SINGLE && alloc->alloc_pending_count > 0)
+
+    if (!alloc_work_available (ctx))
         return;
-    if ((job = zlistx_first (alloc->queue))) {
-        if (alloc_request (alloc, job) < 0) {
-            flux_log_error (ctx->h, "alloc_request fatal error");
-            flux_reactor_stop_error (flux_get_reactor (ctx->h));
-            return;
-        }
-        zlistx_delete (alloc->queue, job->handle);
-        job->handle = NULL;
-        job->alloc_pending = 1;
-        job->alloc_queued = 0;
-        alloc->alloc_pending_count++;
-        if ((job->flags & FLUX_JOB_DEBUG))
-            (void)event_job_post_pack (ctx->event, job,
-                                       "debug.alloc-request", NULL);
 
+    job = zlistx_first (alloc->queue);
+
+    if (alloc_request (alloc, job) < 0) {
+        flux_log_error (ctx->h, "alloc_request fatal error");
+        flux_reactor_stop_error (flux_get_reactor (ctx->h));
+        return;
     }
+    job_priority_queue_delete (alloc->queue, job);
+    job->alloc_pending = 1;
+    job->alloc_queued = 0;
+    if (job_priority_queue_insert (alloc->sent, job) < 0)
+        flux_log (ctx->h, LOG_ERR, "failed to enqueue pending job");
+    /* Post event for debugging if job was submitted FLUX_JOB_DEBUG flag.
+     */
+    if ((job->flags & FLUX_JOB_DEBUG))
+        (void)event_job_post_pack (ctx->event,
+                                   job,
+                                   "debug.alloc-request",
+                                   0,
+                                   NULL);
 }
 
-/* called from event_job_action() FLUX_JOB_CLEANUP */
-int alloc_send_free_request (struct alloc *alloc, struct job *job)
+int alloc_send_free_request (struct alloc *alloc,
+                             json_t *R,
+                             flux_jobid_t id,
+                             bool final)
 {
-    assert (job->state == FLUX_JOB_CLEANUP);
-    if (!job->free_pending && alloc->ready) {
-        if (free_request (alloc, job) < 0)
+    if (alloc->scheduler_is_online) {
+        if (free_request (alloc, id, R, final) < 0)
             return -1;
-        job->free_pending = 1;
-        if ((job->flags & FLUX_JOB_DEBUG))
-            (void)event_job_post_pack (alloc->ctx->event, job,
-                                       "debug.free-request", NULL);
-        alloc->free_pending_count++;
     }
     return 0;
 }
 
-/* called from event_job_action() FLUX_JOB_SCHED */
+/* called from event_job_action() FLUX_JOB_STATE_SCHED */
 int alloc_enqueue_alloc_request (struct alloc *alloc, struct job *job)
 {
-    assert (job->state == FLUX_JOB_SCHED);
-    if (!job->alloc_queued && !job->alloc_pending) {
-        bool fwd = job->priority > FLUX_JOB_PRIORITY_DEFAULT ? true : false;
-        assert (job->handle == NULL);
-        if (!(job->handle = zlistx_insert (alloc->queue, job, fwd)))
+    if (job->state != FLUX_JOB_STATE_SCHED)
+        return -1;
+    if (!job->alloc_bypass
+        && !job->alloc_queued
+        && !job->alloc_pending
+        && job->priority != FLUX_JOB_PRIORITY_MIN
+        && queue_started (alloc->ctx->queue, job)) {
+        if (job_priority_queue_insert (alloc->queue, job) < 0)
             return -1;
         job->alloc_queued = 1;
     }
     return 0;
 }
 
-/* called from event_job_action() FLUX_JOB_CLEANUP */
+/* called from event_job_action() FLUX_JOB_STATE_CLEANUP
+ * or transition from FLUX_JOB_STATE_SCHED back to FLUX_JOB_STATE_PRIORITY.
+ */
 void alloc_dequeue_alloc_request (struct alloc *alloc, struct job *job)
 {
     if (job->alloc_queued) {
-        zlistx_delete (alloc->queue, job->handle);
-        job->handle = NULL;
+        job_priority_queue_delete (alloc->queue, job);
         job->alloc_queued = 0;
     }
 }
 
-/* called from event_job_action() FLUX_JOB_CLEANUP */
-int alloc_cancel_alloc_request (struct alloc *alloc, struct job *job)
+/* Send a sched.cancel request for job.  This RPC receives no direct response.
+ * Instead, the sched.alloc request receives a FLUX_SCHED_ALLOC_CANCEL or a
+ * FLUX_SCHED_ALLOC_SUCCESS response.
+ *
+ * As described in RFC 27, sched.alloc requests are canceled when:
+ * 1) a job in SCHED state is canceled
+ * 2) a queue is administratively disabled
+ * 3) when repriortizing jobs in limited mode
+ *
+ * The finalize flag is for the first case.  It allows the job to continue
+ * through CLEANUP without waiting for the scheduler to respond to the cancel.
+ * The sched.alloc response handler must handle the case where the job is
+ * no longer active or its alloc_pending flag is clear.  Essentially 'finalize'
+ * causes the job related finalization stuff to happen here rather than
+ * in the sched.alloc response handler.
+ */
+int alloc_cancel_alloc_request (struct alloc *alloc,
+                                struct job *job,
+                                bool finalize)
 {
     if (job->alloc_pending) {
         if (cancel_request (alloc, job) < 0)
             return -1;
+        if (finalize) {
+            job->alloc_pending = 0;
+            job_priority_queue_delete (alloc->sent, job);
+            annotations_clear_and_publish (alloc->ctx, job, NULL);
+        }
     }
     return 0;
 }
@@ -543,115 +573,155 @@ struct job *alloc_queue_next (struct alloc *alloc)
     return zlistx_next (alloc->queue);
 }
 
-/* called from priority_handle_request() */
+/* called from reprioritize_job() */
 void alloc_queue_reorder (struct alloc *alloc, struct job *job)
 {
-    bool fwd = job->priority > FLUX_JOB_PRIORITY_DEFAULT ? true : false;
-
-    zlistx_reorder (alloc->queue, job->handle, fwd);
+    job_priority_queue_reorder (alloc->queue, job);
 }
 
-int alloc_pending_count (struct alloc *alloc)
+void alloc_pending_reorder (struct alloc *alloc, struct job *job)
 {
-    return alloc->alloc_pending_count;
+    job_priority_queue_reorder (alloc->sent, job);
 }
 
-/* Cancel all pending alloc requests in preparation for disabling
- * resource allocation.
- */
-static void cancel_all_pending (struct alloc *alloc)
+int alloc_queue_reprioritize (struct alloc *alloc)
 {
-    if (alloc->alloc_pending_count > 0) {
-        struct job *job;
+    job_priority_queue_sort (alloc->queue);
+    job_priority_queue_sort (alloc->sent);
 
-        job = zhashx_first (alloc->ctx->active_jobs);
-        while (job) {
-            if (job->alloc_pending)
-                cancel_request (alloc, job);
-            job = zhashx_next (alloc->ctx->active_jobs);
+    if (alloc->alloc_limit)
+        return alloc_queue_recalc_pending (alloc);
+    return 0;
+}
+
+/* called if highest priority job may have changed */
+int alloc_queue_recalc_pending (struct alloc *alloc)
+{
+    struct job *head = zlistx_first (alloc->queue);
+    struct job *tail = zlistx_last (alloc->sent);
+    while (alloc->alloc_limit
+           && head
+           && tail) {
+        if (job_priority_comparator (head, tail) < 0) {
+            if (alloc_cancel_alloc_request (alloc, tail, false) < 0) {
+                flux_log_error (alloc->ctx->h,
+                                "%s: alloc_cancel_alloc_request",
+                                __FUNCTION__);
+                return -1;
+            }
         }
+        else
+            break;
+        head = zlistx_next (alloc->queue);
+        tail = zlistx_prev (alloc->sent);
     }
+    return 0;
 }
 
-/* Control resource allocation (query/start/stop).
- * If 'query_only' is true, report allocaction status without altering it.
- * Otherwise update the alloc->disable flag, and for disable only,
- * optionally set alloc->disable_reason.
- *
- * What it means to be administratively disabled:
- * While allocation is disabled, the scheduler can remain loaded and handle
- * requests, but the job manager won't send any more allocation requests.
- * Pending alloc requests are canceled (jobs remain in SCHED state and
- * return to alloc->queue).  The job manager continues to send free requests
- * to the scheduler as jobs relinquish resources.
- *
- * If allocation is adminstratively enabled, but the scheduler is not loaded,
- * the current state is reported as disabled with reason "Scheduler is offline".
- */
-static void alloc_admin_cb (flux_t *h,
+int alloc_queue_count (struct alloc *alloc)
+{
+    return zlistx_size (alloc->queue);
+}
+
+int alloc_pending_count (struct alloc *alloc)
+{
+    return zlistx_size (alloc->sent);
+}
+
+bool alloc_sched_ready (struct alloc *alloc)
+{
+    return alloc->scheduler_is_online;
+}
+
+static void alloc_query_cb (flux_t *h,
                             flux_msg_handler_t *mh,
                             const flux_msg_t *msg,
                             void *arg)
 {
     struct job_manager *ctx = arg;
     struct alloc *alloc = ctx->alloc;
-    int query_only;
-    int enable;
-    const char *reason = NULL;
 
-    if (flux_request_unpack (msg,
-                             NULL,
-                             "{s:b s:b s?:s}",
-                             "query_only",
-                             &query_only,
-                             "enable",
-                             &enable,
-                             "reason",
-                             &reason) < 0)
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:i s:i s:i}",
+                           "queue_length", zlistx_size (alloc->queue),
+                           "alloc_pending", zlistx_size (alloc->sent),
+                           "running", alloc->ctx->running_jobs) < 0)
+        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
+    return;
+}
+
+static void resource_status_cb (flux_t *h,
+                                flux_msg_handler_t *mh,
+                                const flux_msg_t *msg,
+                                void *arg)
+{
+    struct job_manager *ctx = arg;
+    struct alloc *alloc = ctx->alloc;
+    struct rlist *rl;
+    json_t *R = NULL;
+    flux_error_t error;
+    struct job *job;
+
+    if (!(rl = rlist_create ())) {
+        errprintf (&error, "error creating rlist object");
         goto error;
-    if (!query_only) {
-        if (!enable) {
-            char *cpy = NULL;
-            if (reason && strlen (reason) > 0 && !(cpy = strdup (reason)))
+    }
+    job = zhashx_first (alloc->ctx->active_jobs);
+    while (job) {
+        if ((job->has_resources && !job->free_posted)
+            && job->R_redacted && !job->alloc_bypass) {
+            struct rlist *rl2;
+            json_error_t jerror;
+
+            if (!(rl2 = rlist_from_json (job->R_redacted, &jerror))) {
+                errprintf (&error,
+                           "%s: error converting JSON to rlist: %s",
+                           idf58 (job->id),
+                           jerror.text);
+                goto error;
+            }
+            if (rlist_append (rl, rl2) < 0) {
+                errprintf (&error, "%s: duplicate allocation", idf58 (job->id));
+                rlist_destroy (rl2);
                 goto error;
-            free (alloc->disable_reason);
-            alloc->disable_reason = cpy;
-            cancel_all_pending (alloc);
+            }
+            rlist_destroy (rl2);
         }
-        alloc->disable = enable ? false : true;
-    }
-    if (alloc->disable) { // administratively disabled
-        enable = 0;
-        reason  = alloc->disable_reason;
-    }
-    else if (!alloc->ready) { // scheduler not loaded (waiting for hello)
-        enable = 0;
-        reason = "Scheduler is offline";
+        job = zhashx_next (alloc->ctx->active_jobs);
     }
-    else { // condtion normal
-        enable = 1;
-        reason = NULL;
+    if (housekeeping_stat_append (ctx->housekeeping, rl, &error) < 0)
+        goto error;
+    if (!(R = rlist_to_R (rl))) {
+        errprintf (&error, "error converting rlist to JSON");
+        goto error;
     }
-    if (flux_respond_pack (h,
-                           msg,
-                           "{s:b s:s s:i s:i s:i s:i}",
-                           "enable",
-                           enable,
-                           "reason",
-                           reason ? reason : "",
-                           "queue_length",
-                           zlistx_size (alloc->queue),
-                           "alloc_pending",
-                           alloc->alloc_pending_count,
-                           "free_pending",
-                           alloc->free_pending_count,
-                           "running",
-                           alloc->ctx->running_jobs) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
+    if (flux_respond_pack (h, msg, "{s:O}", "allocated", R) < 0)
+        flux_log_error (h, "error responding to resource-status request");
+    json_decref (R);
+    rlist_destroy (rl);
     return;
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    if (flux_respond_error (h, msg, EINVAL, error.text) < 0)
+        flux_log_error (h, "error responding to resource-status request");
+    json_decref (R);
+    rlist_destroy (rl);
+}
+
+void alloc_disconnect_rpc (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct job_manager *ctx = arg;
+    struct alloc *alloc = ctx->alloc;
+
+    if (alloc->sched_sender) {
+        const char *sender;
+        if ((sender = flux_msg_route_first (msg))
+            && streq (sender, alloc->sched_sender))
+            interface_teardown (ctx->alloc, "disconnect", 0);
+    }
 }
 
 void alloc_ctx_destroy (struct alloc *alloc)
@@ -663,18 +733,39 @@ void alloc_ctx_destroy (struct alloc *alloc)
         flux_watcher_destroy (alloc->check);
         flux_watcher_destroy (alloc->idle);
         zlistx_destroy (&alloc->queue);
-        free (alloc->disable_reason);
+        zlistx_destroy (&alloc->sent);
+        free (alloc->sched_sender);
         free (alloc);
         errno = saved_errno;
     }
 }
 
 static const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_REQUEST,  "job-manager.sched-hello", hello_cb, 0},
-    { FLUX_MSGTYPE_REQUEST,  "job-manager.sched-ready", ready_cb, 0},
-    { FLUX_MSGTYPE_REQUEST,  "job-manager.alloc-admin", alloc_admin_cb, 0},
-    { FLUX_MSGTYPE_RESPONSE, "sched.alloc", alloc_response_cb, 0},
-    { FLUX_MSGTYPE_RESPONSE, "sched.free", free_response_cb, 0},
+    {   FLUX_MSGTYPE_REQUEST,
+        "job-manager.sched-hello",
+        hello_cb,
+        0
+    },
+    {   FLUX_MSGTYPE_REQUEST,
+        "job-manager.sched-ready",
+        ready_cb,
+        0
+    },
+    {   FLUX_MSGTYPE_REQUEST,
+        "job-manager.alloc-query",
+        alloc_query_cb,
+        FLUX_ROLE_USER,
+    },
+    {   FLUX_MSGTYPE_REQUEST,
+        "job-manager.resource-status",
+        resource_status_cb,
+        0
+    },
+    {   FLUX_MSGTYPE_RESPONSE,
+        "sched.alloc",
+        alloc_response_cb,
+        0
+    },
     FLUX_MSGHANDLER_TABLE_END,
 };
 
@@ -686,12 +777,9 @@ struct alloc *alloc_ctx_create (struct job_manager *ctx)
     if (!(alloc = calloc (1, sizeof (*alloc))))
         return NULL;
     alloc->ctx = ctx;
-    if (!(alloc->queue = zlistx_new()))
+    if (!(alloc->queue = job_priority_queue_create ())
+        || !(alloc->sent = job_priority_queue_create ()))
         goto error;
-    zlistx_set_destructor (alloc->queue, job_destructor);
-    zlistx_set_comparator (alloc->queue, job_comparator);
-    zlistx_set_duplicator (alloc->queue, job_duplicator);
-
     if (flux_msg_handler_addvec (ctx->h, htab, ctx, &alloc->handlers) < 0)
         goto error;
     alloc->prep = flux_prepare_watcher_create (r, prep_cb, ctx);
diff --git a/src/modules/job-manager/alloc.h b/src/modules/job-manager/alloc.h
index 459805e5aa5f..f5fcd82f169e 100644
--- a/src/modules/job-manager/alloc.h
+++ b/src/modules/job-manager/alloc.h
@@ -11,6 +11,7 @@
 #ifndef _FLUX_JOB_MANAGER_ALLOC_H
 #define _FLUX_JOB_MANAGER_ALLOC_H
 
+#include <jansson.h>
 #include <flux/core.h>
 
 #include "job.h"
@@ -31,27 +32,57 @@ void alloc_dequeue_alloc_request (struct alloc *alloc, struct job *job);
 
 /* Send a request to cancel pending alloc request.
  * This function is a no-op if job->alloc_pending is not set.
+ * If finalize is true, update the job as though the cancelation
+ * request has already been handled, so the job can progress through
+ * CLEANUP without waiting for the scheduler response.
  */
-int alloc_cancel_alloc_request (struct alloc *alloc, struct job *job);
+int alloc_cancel_alloc_request (struct alloc *alloc,
+                                struct job *job,
+                                bool finalize);
+
+/* Accessor for the count of queued alloc requests.
+ */
+int alloc_queue_count (struct alloc *alloc);
 
 /* Accessor for the count of pending alloc requests.
  */
 int alloc_pending_count (struct alloc *alloc);
 
-/* Call from CLEANUP state to release resources.
- * This function is a no-op if job->free_pending is set.
+/* Release resources back to the scheduler.
  */
-int alloc_send_free_request (struct alloc *alloc, struct job *job);
+int alloc_send_free_request (struct alloc *alloc,
+                             json_t *R,
+                             flux_jobid_t id,
+                             bool final);
 
 /* List pending jobs
  */
 struct job *alloc_queue_first (struct alloc *alloc);
 struct job *alloc_queue_next (struct alloc *alloc);
 
-/* Reorder job in scheduler queue, e.g. after priority change.
+/* Reorder job in scheduler queue, e.g. after urgency change.
  */
 void alloc_queue_reorder (struct alloc *alloc, struct job *job);
 
+/* Reorder job in pending jobs queue, e.g. after urgency change.
+ */
+void alloc_pending_reorder (struct alloc *alloc, struct job *job);
+
+/* Re-sort alloc queue and pending jobs.
+ * Recalculate pending jobs if necessary
+ */
+int alloc_queue_reprioritize (struct alloc *alloc);
+
+/* Recalculate pending job, e.g. after urgency change */
+int alloc_queue_recalc_pending (struct alloc *alloc);
+
+void alloc_disconnect_rpc (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg);
+
+bool alloc_sched_ready (struct alloc *alloc);
+
 #endif /* ! _FLUX_JOB_MANAGER_ALLOC_H */
 
 /*
diff --git a/src/modules/job-manager/annotate.c b/src/modules/job-manager/annotate.c
new file mode 100644
index 000000000000..22327014c18b
--- /dev/null
+++ b/src/modules/job-manager/annotate.c
@@ -0,0 +1,232 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* annotate - user requests to annotate a job
+ *
+ * Purpose:
+ *   Handle job-manager.annotate RPC
+ *
+ * Input:
+ * - job id, annotations
+ *
+ * Action:
+ *  -update annotations
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <ctype.h>
+#include <assert.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/jpath.h"
+#include "src/common/libjob/idf58.h"
+
+#include "job.h"
+#include "event.h"
+#include "annotate.h"
+#include "job-manager.h"
+
+struct annotate {
+    struct job_manager *ctx;
+    flux_msg_handler_t **handlers;
+};
+
+static void annotations_clear (struct job *job)
+{
+    if (job->annotations) {
+        json_decref (job->annotations);
+        job->annotations = NULL;
+    }
+}
+
+int update_annotation_recursive (json_t *orig, const char *path, json_t *new)
+{
+    if (jpath_update (orig, path, new) < 0
+        || jpath_clear_null (orig) < 0)
+        return -1;
+    return 0;
+}
+
+int annotations_update (struct job *job, const char *path, json_t *annotations)
+{
+    if (!json_is_object (annotations)) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (annotations) {
+        if (!job->annotations) {
+            if (!(job->annotations = json_object ())) {
+                errno = ENOMEM;
+                return -1;
+            }
+        }
+        if (update_annotation_recursive (job->annotations,
+                                         path,
+                                         annotations) < 0)
+            return -1;
+        /* Special case: if user cleared all entries, assume we no
+         * longer need annotations object.  If cleared, caller
+         * will handle advertisement of the clear.
+         */
+        if (!json_object_size (job->annotations))
+            annotations_clear (job);
+    }
+    return 0;
+}
+
+void annotations_clear_and_publish (struct job_manager *ctx,
+                                    struct job *job,
+                                    const char *key)
+{
+    if (job->annotations) {
+        if (key)
+            (void)json_object_del (job->annotations, key);
+        else
+            (void)json_object_clear (job->annotations);
+        if (json_object_size (job->annotations) == 0) {
+            annotations_clear (job);
+            if (event_job_post_pack (ctx->event,
+                                     job,
+                                     "annotations",
+                                     EVENT_NO_COMMIT,
+                                     "{s:n}",
+                                     "annotations") < 0) {
+                flux_log_error (ctx->h,
+                                "error posting null annotations event for %s",
+                                idf58 (job->id));
+            }
+        }
+    }
+}
+
+int annotations_update_and_publish (struct job_manager *ctx,
+                                    struct job *job,
+                                    json_t *annotations)
+{
+    int rc = -1;
+    json_t *tmp = NULL;
+
+    if (annotations_update (job, ".", annotations) < 0)
+        return -1;
+    if (job->annotations) {
+        /* deep copy necessary for journal history, as
+         * job->annotations can be modified in future */
+        if (!(tmp = json_deep_copy (job->annotations))) {
+            errno = ENOMEM;
+            return -1;
+        }
+    }
+    if (event_job_post_pack (ctx->event,
+                             job,
+                             "annotations",
+                             EVENT_NO_COMMIT,
+                             "{s:O?}",
+                             "annotations", tmp) < 0)
+        goto error;
+    rc = 0;
+error:
+    json_decref (tmp);
+    return rc;
+}
+
+void annotate_memo_request (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct job_manager *ctx = arg;
+    struct flux_msg_cred cred;
+    flux_jobid_t id;
+    json_t *memo = NULL;
+    struct job *job;
+    const char *errstr = NULL;
+    int no_commit = 0;
+    json_t *tmp = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s?b s:o}",
+                             "id", &id,
+                             "volatile", &no_commit,
+                             "memo", &memo) < 0
+        || flux_msg_get_cred (msg, &cred) < 0)
+        goto error;
+    if (!(job = zhashx_lookup (ctx->active_jobs, &id))) {
+        if (!(job = zhashx_lookup (ctx->inactive_jobs, &id)))
+            errstr = "unknown job id";
+        else
+            errstr = "job is inactive";
+        errno = ENOENT;
+        goto error;
+    }
+    if (flux_msg_cred_authorize (cred, job->userid) < 0) {
+        errstr = "guests can only add a memo to their own jobs";
+        goto error;
+    }
+    if (event_job_post_pack (ctx->event,
+                             job,
+                             "memo",
+                             no_commit ? EVENT_NO_COMMIT : 0,
+                             "O",
+                             memo) < 0) {
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
+    json_decref (tmp);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    json_decref (tmp);
+}
+
+
+void annotate_ctx_destroy (struct annotate *annotate)
+{
+    if (annotate) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (annotate->handlers);
+        free (annotate);
+        errno = saved_errno;
+    }
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.memo",
+        annotate_memo_request,
+        FLUX_ROLE_USER
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+struct annotate *annotate_ctx_create (struct job_manager *ctx)
+{
+    struct annotate *annotate;
+
+    if (!(annotate = calloc (1, sizeof (*annotate))))
+        return NULL;
+    annotate->ctx = ctx;
+    if (flux_msg_handler_addvec (ctx->h, htab, ctx, &annotate->handlers) < 0)
+        goto error;
+    return annotate;
+error:
+    annotate_ctx_destroy (annotate);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-manager/annotate.h b/src/modules/job-manager/annotate.h
new file mode 100644
index 000000000000..8e17e74abb3c
--- /dev/null
+++ b/src/modules/job-manager/annotate.h
@@ -0,0 +1,42 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_ANNOTATE_H
+#define _FLUX_JOB_MANAGER_ANNOTATE_H
+
+#include <stdint.h>
+
+#include "job.h"
+#include "job-manager.h"
+
+int annotations_update (struct job *job, const char *path, json_t *annotations);
+
+struct annotate *annotate_ctx_create (struct job_manager *ctx);
+void annotate_ctx_destroy (struct annotate *annotate);
+
+/* exposed for unit testing only */
+int update_annotation_recursive (json_t *orig, const char *path, json_t *new);
+
+int annotations_update_and_publish (struct job_manager *ctx,
+                                    struct job *job,
+                                    json_t *annotations);
+
+/* clear key from annotations, or clear all annotations if key == NULL.
+ * If that transitioned the annotations object from non-empty to empty,
+ * post an annotations event with the context of {"annotations":null}.
+ */
+void annotations_clear_and_publish (struct job_manager *ctx,
+                                    struct job *job,
+                                    const char *key);
+
+#endif /* ! _FLUX_JOB_MANAGER_ANNOTATE_H */
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-manager/conf.c b/src/modules/job-manager/conf.c
new file mode 100644
index 000000000000..b9e9842dc7f0
--- /dev/null
+++ b/src/modules/job-manager/conf.c
@@ -0,0 +1,186 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* conf.c - handle job manager configuration
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libfluxutil/policy.h"
+
+#include "job-manager.h"
+#include "journal.h"
+#include "conf.h"
+
+struct conf_callback {
+    conf_update_f cb;
+    void *arg;
+};
+
+struct conf {
+    zlistx_t *callbacks;
+    flux_msg_handler_t **handlers;
+    struct job_manager *ctx;
+};
+
+static void conf_callback_destroy (struct conf_callback *ccb)
+{
+    if (ccb) {
+        int saved_errno = errno;
+        free (ccb);
+        errno = saved_errno;
+    }
+}
+
+// zlistx_destructor_fn signature
+static void conf_callback_destructor (void **item)
+{
+    if (item) {
+        conf_callback_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static struct conf_callback *conf_callback_create (conf_update_f cb, void *arg)
+{
+    struct conf_callback *ccb;
+
+    if (!(ccb = calloc (1, sizeof (*ccb))))
+        return NULL;
+    ccb->cb = cb;
+    ccb->arg = arg;
+    return ccb;
+}
+
+void conf_unregister_callback (struct conf *conf, conf_update_f cb)
+{
+    struct conf_callback *ccb;
+
+    ccb = zlistx_first (conf->callbacks);
+    while (ccb) {
+        if (ccb->cb == cb) {
+            zlistx_delete (conf->callbacks, zlistx_cursor (conf->callbacks));
+            break;
+        }
+        ccb = zlistx_next (conf->callbacks);
+    }
+}
+
+int conf_register_callback (struct conf *conf,
+                            flux_error_t *error,
+                            conf_update_f cb,
+                            void *arg)
+{
+    struct conf_callback *ccb;
+    int rc;
+
+    rc = cb (flux_get_conf (conf->ctx->h), error, arg);
+
+    if (rc < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (rc == 1) {
+        if (!(ccb = conf_callback_create (cb, arg))
+            || zlistx_add_end (conf->callbacks, ccb) == NULL) {
+            conf_callback_destroy (ccb);
+            errprintf (error, "out of memory adding config callback");
+            errno = ENOMEM;
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static void config_reload_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct conf *conf = arg;
+    const flux_conf_t *instance_conf;
+    struct conf_callback *ccb;
+    flux_error_t error;
+    const char *errstr = NULL;
+
+    if (flux_conf_reload_decode (msg, &instance_conf) < 0)
+        goto error;
+    if (policy_validate (instance_conf, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    ccb = zlistx_first (conf->callbacks);
+    while (ccb) {
+        if (ccb->cb (instance_conf, &error, ccb->arg) < 0) {
+            errstr = error.text;
+            errno = EINVAL;
+            goto error;
+        }
+        ccb = zlistx_next (conf->callbacks);
+    }
+    if (flux_set_conf (h, flux_conf_incref (instance_conf)) < 0) {
+        errstr = "error updating cached configuration";
+        flux_conf_decref (instance_conf);
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST, "job-manager.config-reload", config_reload_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void conf_destroy (struct conf *conf)
+{
+    if (conf) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (conf->handlers);
+        zlistx_destroy (&conf->callbacks);
+        free (conf);
+        errno = saved_errno;
+    }
+}
+
+struct conf *conf_create (struct job_manager *ctx, flux_error_t *error)
+{
+    struct conf *conf;
+
+    if (!(conf = calloc (1, sizeof (*conf))))
+        goto error;
+    conf->ctx = ctx;
+    if (policy_validate (flux_get_conf (ctx->h), error) < 0)
+        goto error_nofill;
+    if (!(conf->callbacks = zlistx_new ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    zlistx_set_destructor (conf->callbacks, conf_callback_destructor);
+    if (flux_msg_handler_addvec (ctx->h, htab, conf, &conf->handlers) < 0)
+        goto error;
+    return conf;
+error:
+    errprintf (error, "%s", strerror (errno));
+error_nofill:
+    conf_destroy (conf);
+    return NULL;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/conf.h b/src/modules/job-manager/conf.h
new file mode 100644
index 000000000000..529d3ba1664c
--- /dev/null
+++ b/src/modules/job-manager/conf.h
@@ -0,0 +1,42 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_CONF_H
+#define _FLUX_JOB_MANAGER_CONF_H
+
+#include <stdbool.h>
+#include <flux/core.h>
+
+#include "job-manager.h"
+
+/* Return value:
+ *   0=success, one-shot
+ *  -1=failure (set 'error' but not errno)
+ *   1=success, continue to invoke callback on config updates
+ */
+typedef int (*conf_update_f)(const flux_conf_t *conf,
+                             flux_error_t *error,
+                             void *arg);
+
+struct conf *conf_create (struct job_manager *ctx, flux_error_t *error);
+void conf_destroy (struct conf *conf);
+
+/* Immediately call 'cb' on current config object, and then on config updates
+ * as indicated by initial callback's return value (see above).
+ */
+int conf_register_callback (struct conf *conf,
+                            flux_error_t *error,
+                            conf_update_f cb,
+                            void *arg);
+void conf_unregister_callback (struct conf *conf, conf_update_f cb);
+
+#endif /* ! _FLUX_JOB_MANAGER_CONF_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/drain.c b/src/modules/job-manager/drain.c
index d2a8d33c3811..d9d1de8c4990 100644
--- a/src/modules/job-manager/drain.c
+++ b/src/modules/job-manager/drain.c
@@ -15,10 +15,10 @@
 #include "config.h"
 #endif
 #include <stdbool.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "drain.h"
 #include "submit.h"
@@ -50,7 +50,7 @@ void drain_check (struct drain *drain)
     if (zhashx_size (drain->ctx->active_jobs) == 0) {
         while ((msg = zlist_pop (drain->drain_requests))) {
             if (!(rsp = flux_response_derive (msg, 0))
-                        || event_batch_respond (drain->ctx->event, rsp) < 0)
+                || event_batch_respond (drain->ctx->event, rsp) < 0)
                 flux_log_error (drain->ctx->h,
                                 "error handing drain request off");
             flux_msg_decref (rsp);
@@ -61,13 +61,13 @@ void drain_check (struct drain *drain)
     /* Idle - no jobs in RUN or CLEANUP state, and no pending alloc requests.
      */
     if (alloc_pending_count (drain->ctx->alloc) == 0
-                                    && drain->ctx->running_jobs == 0) {
+        && drain->ctx->running_jobs == 0) {
         int pending = zhashx_size (drain->ctx->active_jobs)
                                  - drain->ctx->running_jobs;
         while ((msg = zlist_pop (drain->idle_requests))) {
             if (!(rsp = flux_response_derive (msg, 0))
-                        || flux_msg_pack (rsp, "{s:i}", "pending", pending) < 0
-                        || event_batch_respond (drain->ctx->event, rsp) < 0)
+                || flux_msg_pack (rsp, "{s:i}", "pending", pending) < 0
+                || event_batch_respond (drain->ctx->event, rsp) < 0)
                 flux_log_error (drain->ctx->h,
                                 "error handing idle request off");
             flux_msg_decref (rsp);
@@ -160,7 +160,7 @@ struct drain *drain_ctx_create (struct job_manager *ctx)
         return NULL;
     drain->ctx = ctx;
     if (!(drain->drain_requests = zlist_new ())
-            || !(drain->idle_requests = zlist_new ())) {
+        || !(drain->idle_requests = zlist_new ())) {
         errno = ENOMEM;
         goto error;
     }
diff --git a/src/modules/job-manager/event.c b/src/modules/job-manager/event.c
index b8eefcf9fc3c..84b466a649e9 100644
--- a/src/modules/job-manager/event.c
+++ b/src/modules/job-manager/event.c
@@ -36,27 +36,38 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
+#include <time.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libjob/idf58.h"
+#include "ccan/ptrint/ptrint.h"
+#include "ccan/str/str.h"
 
 #include "alloc.h"
+#include "housekeeping.h"
 #include "start.h"
 #include "drain.h"
+#include "journal.h"
 #include "wait.h"
+#include "prioritize.h"
+#include "annotate.h"
+#include "purge.h"
+#include "jobtap-internal.h"
 
 #include "event.h"
 
-#include "src/common/libeventlog/eventlog.h"
-
-const double batch_timeout = 0.01;
-
 struct event {
     struct job_manager *ctx;
+    flux_msg_handler_t **handlers;
+    double batch_timeout;
     struct event_batch *batch;
     flux_watcher_t *timer;
     zlist_t *pending;
-    zlist_t *pub_futures;
+    zhashx_t *evindex;
 };
 
 struct event_batch {
@@ -65,16 +76,18 @@ struct event_batch {
     flux_future_t *f;
     json_t *state_trans;
     zlist_t *responses; // responses deferred until batch complete
+    zlist_t *jobs;      // jobs held until batch complete
 };
 
-struct event_batch *event_batch_create (struct event *event);
-void event_batch_destroy (struct event_batch *batch);
+static struct event_batch *event_batch_create (struct event *event);
+static void event_batch_destroy (struct event_batch *batch);
+static int event_job_post_deferred (struct event *event, struct job *job);
 
 /* Batch commit has completed.
  * If there was a commit error, log it and stop the reactor.
  * Destroy 'batch'.
  */
-void commit_continuation (flux_future_t *f, void *arg)
+static void commit_continuation (flux_future_t *f, void *arg)
 {
     struct event_batch *batch = arg;
     struct event *event = batch->event;
@@ -88,26 +101,9 @@ void commit_continuation (flux_future_t *f, void *arg)
     event_batch_destroy (batch);
 }
 
-/* job-state event publish has completed.
- * If there was a publish error, log it and stop the reactor.
- * Destroy 'f'.
- */
-void publish_continuation (flux_future_t *f, void *arg)
-{
-    struct event *event = arg;
-    struct job_manager *ctx = event->ctx;
-
-    if (flux_future_get (f, NULL) < 0) {
-        flux_log_error (ctx->h, "%s: event publish failed", __FUNCTION__);
-        flux_reactor_stop_error (flux_get_reactor (ctx->h));
-    }
-    zlist_remove (event->pub_futures, f);
-    flux_future_destroy (f);
-}
-
 /* Close the current batch, if any, and commit it.
  */
-void event_batch_commit (struct event *event)
+static void event_batch_commit (struct event *event)
 {
     struct event_batch *batch = event->batch;
     struct job_manager *ctx = event->ctx;
@@ -122,7 +118,7 @@ void event_batch_commit (struct event *event)
             if (zlist_append (event->pending, batch) < 0)
                 goto nomem;
         }
-        else { // just publish events and be done
+        else { // just send responses and be done
             event_batch_destroy (batch);
         }
     }
@@ -135,46 +131,18 @@ void event_batch_commit (struct event *event)
     event_batch_destroy (batch);
 }
 
-void timer_cb (flux_reactor_t *r, flux_watcher_t *w, int revents, void *arg)
+static void timer_cb (flux_reactor_t *r, flux_watcher_t *w, int revents, void *arg)
 {
     struct job_manager *ctx = arg;
     event_batch_commit (ctx->event);
 }
 
-void event_publish_state (struct event *event, json_t *state_trans)
-{
-    struct job_manager *ctx = event->ctx;
-    flux_future_t *f;
-
-    if (!(f = flux_event_publish_pack (ctx->h,
-                                       "job-state",
-                                       0,
-                                       "{s:O}",
-                                       "transitions",
-                                       state_trans))) {
-        flux_log_error (ctx->h, "%s: flux_event_publish_pack", __FUNCTION__);
-        goto error;
-    }
-    if (flux_future_then (f, -1., publish_continuation, event) < 0) {
-        flux_future_destroy (f);
-        flux_log_error (ctx->h, "%s: flux_future_then", __FUNCTION__);
-        goto error;
-    }
-    if (zlist_append (event->pub_futures, f) < 0) {
-        flux_future_destroy (f);
-        flux_log_error (ctx->h, "%s: zlist_append", __FUNCTION__);
-        goto error;
-    }
-    return;
-error:
-    flux_reactor_stop_error (flux_get_reactor (ctx->h));
-}
-
 /* Besides cleaning up, this function has the following side effects:
- * - publish state transition event (if any)
+ * - send listener responses (only under error scenarios, should be
+ *   sent in event_batch_commit()).
  * - respond to deferred responses (if any)
  */
-void event_batch_destroy (struct event_batch *batch)
+static void event_batch_destroy (struct event_batch *batch)
 {
     if (batch) {
         int saved_errno = errno;
@@ -182,9 +150,16 @@ void event_batch_destroy (struct event_batch *batch)
         flux_kvs_txn_destroy (batch->txn);
         if (batch->f)
             (void)flux_future_wait_for (batch->f, -1);
-        if (batch->state_trans) {
-            event_publish_state (batch->event, batch->state_trans);
-            json_decref (batch->state_trans);
+        if (batch->jobs) {
+            struct job *job;
+            while ((job = zlist_pop (batch->jobs))) {
+                job->hold_events = 0;
+                if (event_job_post_deferred (batch->event, job) < 0)
+                    flux_log_error (batch->event->ctx->h,
+                                    "%s: error posting deferred events",
+                                    idf58 (job->id));
+            }
+            zlist_destroy (&batch->jobs);
         }
         if (batch->responses) {
             flux_msg_t *msg;
@@ -202,31 +177,25 @@ void event_batch_destroy (struct event_batch *batch)
     }
 }
 
-struct event_batch *event_batch_create (struct event *event)
+static struct event_batch *event_batch_create (struct event *event)
 {
     struct event_batch *batch;
 
     if (!(batch = calloc (1, sizeof (*batch))))
         return NULL;
-    if (!(batch->state_trans = json_array ()))
-        goto nomem;
     batch->event = event;
     return batch;
-nomem:
-    errno = ENOMEM;
-    event_batch_destroy (batch);
-    return NULL;
 }
 
 /* Create a new "batch" if there is none.
  * No-op if batch already started.
  */
-int event_batch_start (struct event *event)
+static int event_batch_start (struct event *event)
 {
     if (!event->batch) {
         if (!(event->batch = event_batch_create (event)))
             return -1;
-        flux_timer_watcher_reset (event->timer, batch_timeout, 0.);
+        flux_timer_watcher_reset (event->timer, event->batch_timeout, 0.);
         flux_watcher_start (event->timer);
     }
     return 0;
@@ -258,26 +227,20 @@ static int event_batch_commit_event (struct event *event,
     return 0;
 }
 
-int event_batch_pub_state (struct event *event, struct job *job,
-                           double timestamp)
+int event_batch_add_job (struct event *event, struct job *job)
 {
-    json_t *o;
-
     if (event_batch_start (event) < 0)
-        goto error;
-    if (!(o = json_pack ("[I,s,f]",
-                         job->id,
-                         flux_job_statetostr (job->state, false),
-                         timestamp)))
-        goto nomem;
-    if (json_array_append_new (event->batch->state_trans, o)) {
-        json_decref (o);
-        goto nomem;
+        return -1;
+    if (!(event->batch->jobs)) {
+        if (!(event->batch->jobs = zlist_new ()))
+            goto nomem;
     }
+    if (zlist_append (event->batch->jobs, job) < 0)
+        goto nomem;
+    job->hold_events = 1;
     return 0;
 nomem:
     errno = ENOMEM;
-error:
     return -1;
 }
 
@@ -305,50 +268,113 @@ int event_job_action (struct event *event, struct job *job)
     struct job_manager *ctx = event->ctx;
 
     switch (job->state) {
-        case FLUX_JOB_NEW:
+        case FLUX_JOB_STATE_NEW:
             break;
-        case FLUX_JOB_DEPEND:
-            if (event_job_post_pack (event, job, "depend", NULL) < 0)
-                return -1;
+        case FLUX_JOB_STATE_DEPEND:
+            /*  Post the "depend" event when the job has no more dependency
+             *   references outstanding and a depend event hasn't already
+             *   been posted.
+             *
+             *  The job->depend_posted flag is required in the case that
+             *   events are being queued and handled asynchronously, and
+             *   therefore the post of the "depend" event does not immediately
+             *   transition the job to the PRIORITY state.
+             */
+            if (job_dependency_count (job) == 0
+                && !job_event_is_queued (job, "dependency-add")
+                && !job->depend_posted) {
+                if (event_job_post_pack (event, job, "depend", 0, NULL) < 0)
+                    return -1;
+                job->depend_posted = 1;
+            }
             break;
-        case FLUX_JOB_SCHED:
+        case FLUX_JOB_STATE_PRIORITY:
+            /*
+             * In the event we have re-entered this state from the
+             * SCHED state, dequeue the job first.
+             */
+            if (job->alloc_pending)
+                alloc_cancel_alloc_request (ctx->alloc, job, false);
+            if (job->alloc_queued)
+                alloc_dequeue_alloc_request (ctx->alloc, job);
+            break;
+        case FLUX_JOB_STATE_SCHED:
             if (alloc_enqueue_alloc_request (ctx->alloc, job) < 0)
                 return -1;
+            if (alloc_queue_recalc_pending (ctx->alloc) < 0)
+                return -1;
             break;
-        case FLUX_JOB_RUN:
-            if (start_send_request (ctx->start, job) < 0)
+        case FLUX_JOB_STATE_RUN:
+            /* Send the start request only if prolog is not running/pending.
+             */
+            if (!job->perilog_active
+                && !job_event_is_queued (job, "prolog-start")
+                && start_send_request (ctx->start, job) < 0)
                 return -1;
             break;
-        case FLUX_JOB_CLEANUP:
+        case FLUX_JOB_STATE_CLEANUP:
             if (job->alloc_pending)
-                alloc_cancel_alloc_request (ctx->alloc, job);
+                alloc_cancel_alloc_request (ctx->alloc, job, true);
             if (job->alloc_queued)
                 alloc_dequeue_alloc_request (ctx->alloc, job);
 
             /* N.B. start_pending indicates that the start request is still
              * expecting responses.  The final response is the 'release'
              * response with final=true.  Thus once the flag is clear,
-             * it is safe to release all resources to the scheduler.
+             * it is safe for the job to release its resources to housekeeping.
              */
-            if (job->has_resources && !job->start_pending
-                                   && !job->free_pending) {
-                if (alloc_send_free_request (ctx->alloc, job) < 0)
+            if (job->has_resources
+                && !job_event_is_queued (job, "epilog-start")
+                && !job->perilog_active
+                && !job->start_pending
+                && !job->free_posted) {
+
+                /* Release resources to housekeeping, but only if they were
+                 * allocated by the scheduler.
+                 */
+                if (!job->alloc_bypass
+                    && housekeeping_start (ctx->housekeeping,
+                                           job->R_redacted,
+                                           job->id,
+                                           job->userid) < 0)
                     return -1;
+                if (event_job_post_pack (ctx->event, job, "free", 0, NULL) < 0)
+                    return -1;
+                job->free_posted = 1;
             }
+
             /* Post cleanup event when cleanup is complete.
              */
-            if (!job->alloc_queued && !job->alloc_pending
-                                   && !job->free_pending
-                                   && !job->start_pending
-                                   && !job->has_resources) {
+            if (!job->alloc_queued
+                && !job->alloc_pending
+                && !job->start_pending
+                && !job->has_resources
+                && !job_event_is_queued (job, "epilog-start")
+                && !job->perilog_active) {
 
-                if (event_job_post_pack (event, job, "clean", NULL) < 0)
+                if (event_job_post_pack (event, job, "clean", 0, NULL) < 0)
                     return -1;
             }
             break;
-        case FLUX_JOB_INACTIVE:
+        case FLUX_JOB_STATE_INACTIVE:
+            job->eventlog_readonly = 1;
             if ((job->flags & FLUX_JOB_WAITABLE))
                 wait_notify_inactive (ctx->wait, job);
+            /* Reminder: event_job_action() may be called more than once
+             * for a job + state, therefore zhashx_insert() may fail here and
+             * not be indicative of a problem.
+             */
+            if (zhashx_insert (ctx->inactive_jobs, &job->id, job) == 0) {
+                (void)jobtap_call (ctx->jobtap, job, "job.inactive-add", NULL);
+                if (purge_enqueue_job (ctx->purge, job) < 0) {
+                    flux_log (event->ctx->h,
+                              LOG_ERR,
+                              "%s: error adding inactive job to purge queue",
+                               idf58 (job->id));
+                }
+            }
+            (void) jobtap_call (ctx->jobtap, job, "job.destroy", NULL);
+            job_aux_destroy (job);
             zhashx_delete (ctx->active_jobs, &job->id);
             drain_check (ctx->drain);
             break;
@@ -356,13 +382,14 @@ int event_job_action (struct event *event, struct job *job)
     return 0;
 }
 
-int event_submit_context_decode (json_t *context,
-                                 int *priority,
-                                 uint32_t *userid,
-                                 int *flags)
+static int event_submit_context_decode (json_t *context,
+                                        int *urgency,
+                                        uint32_t *userid,
+                                        int *flags)
 {
-    if (json_unpack (context, "{ s:i s:i s:i }",
-                     "priority", priority,
+    if (json_unpack (context,
+                     "{s:i s:i s:i}",
+                     "urgency", urgency,
                      "userid", userid,
                      "flags", flags) < 0) {
         errno = EPROTO;
@@ -372,21 +399,20 @@ int event_submit_context_decode (json_t *context,
     return 0;
 }
 
-int event_priority_context_decode (json_t *context,
-                                   int *priority)
+static int event_priority_context_decode (json_t *context,
+                                          int64_t *priority)
 {
-    if (json_unpack (context, "{ s:i }", "priority", priority) < 0) {
+    if (json_unpack (context, "{ s:I }", "priority", priority) < 0) {
         errno = EPROTO;
         return -1;
     }
-
     return 0;
 }
 
-int event_exception_context_decode (json_t *context,
-                                    int *severity)
+static int event_urgency_context_decode (json_t *context,
+                                         int *urgency)
 {
-    if (json_unpack (context, "{ s:i }", "severity", severity) < 0) {
+    if (json_unpack (context, "{ s:i }", "urgency", urgency) < 0) {
         errno = EPROTO;
         return -1;
     }
@@ -394,8 +420,26 @@ int event_exception_context_decode (json_t *context,
     return 0;
 }
 
-int event_release_context_decode (json_t *context,
-                                  int *final)
+static int event_exception_context_decode (json_t *context,
+                                           int *severity,
+                                           const char **typep)
+{
+    const char *type;
+
+    if (json_unpack (context,
+                     "{s:i s:s}",
+                     "severity", severity,
+                     "type", &type) < 0) {
+        errno = EPROTO;
+        return -1;
+    }
+    if (typep)
+        *typep = type;
+    return 0;
+}
+
+static int event_release_context_decode (json_t *context,
+                                         int *final)
 {
     *final = 0;
 
@@ -407,9 +451,118 @@ int event_release_context_decode (json_t *context,
     return 0;
 }
 
+static int event_handle_dependency (struct job *job,
+                                    const char *cmd,
+                                    json_t *context)
+{
+    const char *desc;
+
+    if (json_unpack (context, "{s:s}", "description", &desc) < 0) {
+        errno = EPROTO;
+        return -1;
+    }
+    if (streq (cmd, "add"))
+        return job_dependency_add (job, desc);
+    else if (streq (cmd, "remove"))
+        return job_dependency_remove (job, desc);
+    else {
+        errno = EINVAL;
+        return -1;
+    }
+    return 0;
+}
+
+/* Apply updates to the jobspec copy held in memory by the job manager.  The
+ * context object is a dictionary where the keys are period-delimited paths.
+ * For example, "attributes.system.duration":3600.
+ */
+static int event_handle_jobspec_update (struct job *job, json_t *context)
+{
+    if (!job->jobspec_redacted
+        || job_apply_jobspec_updates (job, context) < 0)
+        return -1;
+    return 0;
+}
+
+static int event_handle_set_flags (struct job *job,
+                                   json_t *context)
+{
+    json_t *o = NULL;
+    size_t index;
+    json_t *value;
+
+    if (json_unpack (context, "{s:o}", "flags", &o) < 0) {
+        errno = EPROTO;
+        return -1;
+    }
+    json_array_foreach (o, index, value) {
+        if (job_flag_set (job, json_string_value (value)) < 0) {
+            errno = EPROTO;
+            return -1;
+        }
+    }
+    return 0;
+}
+
+/*  Handle an prolog-* or epilog-* event
+ */
+static int event_handle_perilog (struct job *job,
+                                 const char *cmd,
+                                 json_t *context)
+{
+    if (streq (cmd, "start")) {
+        if (job->perilog_active == UINT8_MAX) {
+            errno = EOVERFLOW;
+            return -1;
+        }
+        job->perilog_active++;
+    }
+    else if (streq (cmd, "finish")) {
+        if (job->perilog_active > 0)
+            job->perilog_active--;
+    }
+    else {
+        errno = EPROTO;
+        return -1;
+    }
+    return 0;
+}
+
+static int event_handle_memo (struct job *job, json_t *o)
+{
+    return annotations_update (job, "user", o);
+}
+
+/*  Return a callback topic string for the current job state
+ *
+ *   NOTE: 'job.state.new' and 'job.state.depend' are not currently used
+ *    since jobs do not transition through these states in
+ *    event_job_post_pack().
+ */
+static const char *state_topic (struct job *job)
+{
+    switch (job->state) {
+        case FLUX_JOB_STATE_NEW:
+            return "job.state.new";
+        case FLUX_JOB_STATE_DEPEND:
+            return "job.state.depend";
+        case FLUX_JOB_STATE_PRIORITY:
+            return "job.state.priority";
+        case FLUX_JOB_STATE_SCHED:
+            return "job.state.sched";
+        case FLUX_JOB_STATE_RUN:
+            return "job.state.run";
+        case FLUX_JOB_STATE_CLEANUP:
+            return "job.state.cleanup";
+        case FLUX_JOB_STATE_INACTIVE:
+            return "job.state.inactive";
+    }
+    /* NOTREACHED */
+    return "job.state.none";
+}
+
 /* This function implements state transitions per RFC 21.
- * If FLUX_JOB_WAITABLE flag is set, then on a fatal exception or
- * cleanup event, capture the event in job->end_event for flux_job_wait().
+ * On a fatal exception or cleanup event, capture the event in job->end_event.
  */
 int event_job_update (struct job *job, json_t *event)
 {
@@ -420,74 +573,142 @@ int event_job_update (struct job *job, json_t *event)
     if (eventlog_entry_parse (event, &timestamp, &name, &context) < 0)
         goto error;
 
-    if (!strcmp (name, "submit")) {
-        if (job->state != FLUX_JOB_NEW)
+    if (streq (name, "submit")) { // invariant: submit is always first event
+        if (job->state != FLUX_JOB_STATE_NEW)
             goto inval;
         job->t_submit = timestamp;
         if (event_submit_context_decode (context,
-                                         &job->priority,
+                                         &job->urgency,
                                          &job->userid,
                                          &job->flags) < 0)
             goto error;
-        job->state = FLUX_JOB_DEPEND;
     }
-    if (!strcmp (name, "depend")) {
-        if (job->state != FLUX_JOB_DEPEND)
+    else if (streq (name, "invalidate")) {
+        job->eventlog_readonly = 1;
+    }
+    else if (streq (name, "validate")) {
+        job->state = FLUX_JOB_STATE_DEPEND;
+    }
+    else if (streq (name, "jobspec-update")) {
+        if (event_handle_jobspec_update (job, context) < 0)
+            goto inval;
+        /*  Transition a job in SCHED state back to PRIORITY to trigger
+         *  possible recalculation of job priority, update scheduler with
+         *  new jobspec, etc. Job will transition back to SCHED after a
+         *  priority is assigned.
+         */
+        if (job->state == FLUX_JOB_STATE_SCHED)
+            job->state = FLUX_JOB_STATE_PRIORITY;
+    }
+    else if (streq (name, "resource-update")) {
+        if (job_apply_resource_updates (job, context) < 0)
             goto inval;
-        job->state = FLUX_JOB_SCHED;
     }
-    else if (!strcmp (name, "priority")) {
-        if (event_priority_context_decode (context, &job->priority) < 0)
+    else if (strstarts (name, "dependency-")) {
+        if (job->state == FLUX_JOB_STATE_DEPEND
+            || job->state == FLUX_JOB_STATE_NEW) {
+            if (event_handle_dependency (job, name+11, context) < 0)
+                goto error;
+        }
+    }
+    else if (streq (name, "set-flags")) {
+        if (event_handle_set_flags (job, context) < 0)
             goto error;
     }
-    else if (!strcmp (name, "exception")) {
-        int severity;
-        if (job->state == FLUX_JOB_NEW || job->state == FLUX_JOB_INACTIVE)
-            goto inval;
-        if (event_exception_context_decode (context, &severity) < 0)
+    else if (streq (name, "memo")) {
+        if (event_handle_memo (job, context) < 0)
             goto error;
-        if (severity == 0) {
-            if ((job->flags & FLUX_JOB_WAITABLE) && !job->end_event)
-                job->end_event = json_incref (event);
-
-            job->state = FLUX_JOB_CLEANUP;
+    }
+    else if (streq (name, "depend")) {
+        if (job->state == FLUX_JOB_STATE_DEPEND)
+            job->state = FLUX_JOB_STATE_PRIORITY;
+    }
+    else if (streq (name, "priority")) {
+        if (job->state == FLUX_JOB_STATE_PRIORITY
+            || job->state == FLUX_JOB_STATE_SCHED) {
+            if (event_priority_context_decode (context, &job->priority) < 0)
+                goto error;
         }
+        if (job->state == FLUX_JOB_STATE_PRIORITY)
+            job->state = FLUX_JOB_STATE_SCHED;
     }
-    else if (!strcmp (name, "alloc")) {
-        if (job->state != FLUX_JOB_SCHED && job->state != FLUX_JOB_CLEANUP)
-            goto inval;
+    else if (streq (name, "urgency")) {
+        /* Update urgency value.  If in SCHED state, transition back to
+         * PRIORITY state to trigger jobtap plugin to recompute priority.
+         * N.B. event_job_action() takes the job out of the alloc queue,
+         * if applicable.  The job will be requeued when it transitions back
+         * to SCHED with a new priority value.
+         */
+        if (event_urgency_context_decode (context, &job->urgency) < 0)
+            goto error;
+        if (job->state == FLUX_JOB_STATE_SCHED)
+            job->state = FLUX_JOB_STATE_PRIORITY;
+    }
+    else if (streq (name, "exception")) {
+        int severity;
+        const char *type;
+        if (job->state != FLUX_JOB_STATE_INACTIVE
+            && job->state != FLUX_JOB_STATE_NEW) {
+            if (event_exception_context_decode (context, &severity, &type) < 0)
+                goto error;
+            if (severity == 0) {
+                // resource allocation could not be reinstiated
+                if (streq (type, "scheduler-restart")) {
+                    if (job->has_resources)
+                        job->has_resources = 0;
+                }
+                if (!job->end_event)
+                    job->end_event = json_incref (event);
+                job->state = FLUX_JOB_STATE_CLEANUP;
+            }
+        }
+    }
+    else if (streq (name, "alloc")) {
         job->has_resources = 1;
-        if (job->state == FLUX_JOB_SCHED)
-            job->state = FLUX_JOB_RUN;
+        if (job->state == FLUX_JOB_STATE_SCHED)
+            job->state = FLUX_JOB_STATE_RUN;
     }
-    else if (!strcmp (name, "free")) {
-        if (job->state != FLUX_JOB_CLEANUP)
-            goto inval;
+    else if (streq (name, "free")) {
         job->has_resources = 0;
     }
-    else if (!strcmp (name, "finish")) {
-        if (job->state != FLUX_JOB_RUN && job->state != FLUX_JOB_CLEANUP)
-            goto inval;
-        if (job->state == FLUX_JOB_RUN) {
-            if ((job->flags & FLUX_JOB_WAITABLE) && !job->end_event)
+    else if (streq (name, "finish")) {
+        if (job->state == FLUX_JOB_STATE_RUN) {
+            if (!job->end_event)
                 job->end_event = json_incref (event);
-
-            job->state = FLUX_JOB_CLEANUP;
+            job->state = FLUX_JOB_STATE_CLEANUP;
         }
     }
-    else if (!strcmp (name, "release")) {
+    else if (streq (name, "release")) {
         int final;
-        if (job->state != FLUX_JOB_RUN && job->state != FLUX_JOB_CLEANUP)
-            goto inval;
         if (event_release_context_decode (context, &final) < 0)
             goto error;
-        if (final && job->state == FLUX_JOB_RUN)
-            goto inval;
     }
-    else if (!strcmp (name, "clean")) {
-        if (job->state != FLUX_JOB_CLEANUP)
-            goto inval;
-        job->state = FLUX_JOB_INACTIVE;
+    else if (streq (name, "clean")) {
+        if (job->state == FLUX_JOB_STATE_CLEANUP) {
+            job->state = FLUX_JOB_STATE_INACTIVE;
+            job->t_clean = timestamp;
+        }
+    }
+    else if (strstarts (name, "prolog-")) {
+        if (!job->start_pending) {
+            if (event_handle_perilog (job, name+7, context) < 0)
+                goto error;
+        }
+    }
+    else if (strstarts (name, "epilog-")) {
+        if (job->state == FLUX_JOB_STATE_CLEANUP) {
+            if (event_handle_perilog (job, name+7, context) < 0)
+                goto error;
+        }
+    }
+    else if (streq (name, "flux-restart")) {
+        /* The flux-restart event is currently only posted to jobs in
+         * SCHED state since that is the only state transition defined
+         * for the event in RFC21.  In the future, other transitions
+         * may be defined.
+         */
+        if (job->state == FLUX_JOB_STATE_SCHED)
+            job->state = FLUX_JOB_STATE_PRIORITY;
     }
     return 0;
 inval:
@@ -496,61 +717,230 @@ int event_job_update (struct job *job, json_t *event)
     return -1;
 }
 
-static int get_timestamp_now (double *timestamp)
+/*  Call jobtap plugin for event if necessary.
+ *  Currently jobtap plugins are called only on state transitions or
+ *   update of job urgency via "urgency" event.
+ */
+static int event_jobtap_call (struct event *event,
+                              struct job *job,
+                              const char *name,
+                              json_t *entry,
+                              flux_job_state_t old_state)
 {
-    struct timespec ts;
-    if (clock_gettime (CLOCK_REALTIME, &ts) < 0)
-        return -1;
-    *timestamp = (1E-9 * ts.tv_nsec) + ts.tv_sec;
+
+    /*  Notify any subscribers of all events, separately from
+     *   special cases for state change and urgency events below.
+     */
+    if (jobtap_notify_subscribers (event->ctx->jobtap,
+                                   job,
+                                   name,
+                                   "{s:O}",
+                                   "entry", entry) < 0)
+            flux_log (event->ctx->h, LOG_ERR,
+                      "jobtap: event.%s callback failed for job %s",
+                      name,
+                      idf58 (job->id));
+
+    /*
+     *  Notify plugins not subscribed to all events of a jobspec update
+     *  since this is a more common case.
+     *
+     *  This callback should occur before the state transition callback
+     *  below, since jobspec-update will transition a job in SCHED back to
+     *  PRIORITY, and plugins should be notified of the jobspec changes
+     *  *before* the `job.state.priority` callback to allow for adjustment
+     *  of internal state normally established before the first time the
+     *  job.state.priority topic is called.
+     */
+    if (streq (name, "jobspec-update")) {
+        json_t *updates;
+        if (json_unpack (entry, "{s:o}", "context", &updates) < 0) {
+            flux_log (event->ctx->h,
+                      LOG_ERR,
+                      "unable to unpack jobspec-update context for %s",
+                      idf58 (job->id));
+            return -1;
+        }
+        (void) jobtap_call (event->ctx->jobtap,
+                            job,
+                            "job.update",
+                            "{s:O}",
+                            "updates", updates);
+    }
+
+    if (job->state != old_state) {
+        /*
+         *  Call plugin callback on state change
+         */
+        return jobtap_call (event->ctx->jobtap,
+                            job,
+                            state_topic (job),
+                            "{s:O s:i}",
+                            "entry", entry,
+                            "prev_state", old_state);
+    }
     return 0;
 }
 
-int event_job_post_pack (struct event *event,
-                         struct job *job,
-                         const char *name,
-                         const char *context_fmt,
-                         ...)
+static int event_job_cache (struct event *event,
+                            struct job *job,
+                            const char *name)
+{
+    int id;
+    /*  Get a unique event id for event 'name' and stash it with the job */
+    if ((id = event_index (event, name)) < 0)
+        return -1;
+    return job_event_id_set (job, id);
+}
+
+int event_job_process_entry (struct event *event,
+                             struct job *job,
+                             int flags,
+                             json_t *entry)
 {
-    va_list ap;
-    json_t *entry = NULL;
-    int saved_errno;
-    double timestamp;
     flux_job_state_t old_state = job->state;
+    const char *name;
+    json_t *context;
 
-    va_start (ap, context_fmt);
-    if (get_timestamp_now (&timestamp) < 0)
-        goto error;
-    if (!(entry = eventlog_entry_vpack (timestamp, name, context_fmt, ap)))
+    if (eventlog_entry_parse (entry, NULL, &name, &context) < 0)
         return -1;
-    if (event_job_update (job, entry) < 0) // modifies job->state
-        goto error;
-    if (event_batch_commit_event (event, job, entry) < 0)
-        goto error;
-    if (job->state != old_state) {
-        if (event_batch_pub_state (event, job, timestamp) < 0)
-            goto error;
+
+    /*  Forbid fatal exceptions in NEW state.
+     */
+    if (job->state == FLUX_JOB_STATE_NEW && streq (name, "exception")) {
+        int severity;
+        if (event_exception_context_decode (context, &severity, NULL))
+            return -1;
+        if (severity == 0) {
+            flux_log (event->ctx->h,
+                      LOG_ERR,
+                      "fatal job exception was posted in NEW state");
+            return -1;
+        }
+    }
+
+    if (!(flags & EVENT_NO_COMMIT)) {
+        if (json_array_append (job->eventlog, entry) < 0) {
+            errno = ENOMEM;
+            return -1;
+        }
     }
 
+    if (journal_process_event (event->ctx->journal,
+                               job->id,
+                               name,
+                               entry) < 0)
+        return -1;
+    if (event_job_update (job, entry) < 0) // modifies job->state
+        return -1;
+    if (event_job_cache (event, job, name) < 0)
+        return -1;
+    if (!(flags & EVENT_NO_COMMIT)
+        && event_batch_commit_event (event, job, entry) < 0)
+        return -1;
+
     /* Keep track of running job count.
      * If queue reaches idle state, event_job_action() triggers any waiters.
      */
-    if ((job->state & FLUX_JOB_RUNNING) && !(old_state & FLUX_JOB_RUNNING))
+    if ((job->state & FLUX_JOB_STATE_RUNNING)
+        && !(old_state & FLUX_JOB_STATE_RUNNING))
         event->ctx->running_jobs++;
-    else if (!(job->state & FLUX_JOB_RUNNING) && (old_state & FLUX_JOB_RUNNING))
+    else if (!(job->state & FLUX_JOB_STATE_RUNNING)
+        && (old_state & FLUX_JOB_STATE_RUNNING))
         event->ctx->running_jobs--;
 
-    if (event_job_action (event, job) < 0)
-        goto error;
+    /*  Note: Failure from the jobtap call is currently ignored, but will
+     *   be logged in jobtap_call(). The goal is to do something with the
+     *   errors at some point (perhaps raise a job exception).
+     */
+    (void) event_jobtap_call (event, job, name, entry, old_state);
 
-    json_decref (entry);
-    va_end (ap);
+    /*  After processing a resource-update event, send the updated
+     *  expiration to the job execution service.
+     */
+    if (streq (name, "resource-update")
+        && start_send_expiration_update (event->ctx->start, job, context) < 0)
+        flux_log (event->ctx->h,
+                  LOG_ERR,
+                  "%s: failed to send expiration update to exec service",
+                  idf58 (job->id));
+
+    return event_job_action (event, job);
+}
+
+static int event_job_post_deferred (struct event *event, struct job *job)
+{
+    int flags;
+    json_t *entry;
+
+    job_incref (job); // in case event_job_process_entry() decrefs job
+    while (job_event_peek (job, &flags, &entry) == 0
+        && !job->eventlog_readonly) {
+        if (event_job_process_entry (event, job, flags, entry) < 0) {
+            int saved_errno = errno;
+            while (job_event_dequeue (job, NULL, NULL) == 0)
+                ;
+            errno = saved_errno;
+            job_decref (job);
+            return -1;
+        }
+        job_event_dequeue (job, NULL, NULL);
+    }
+    job_decref (job);
     return 0;
-error:
+}
+
+/* Since event_job_process_entry() might call event_job_post_*() to post
+ * new events, use job->event_queue to ensure events are processed in order
+ * and unnecessary recursion is avoided.
+ */
+int event_job_post_entry (struct event *event,
+                          struct job *job,
+                          int flags,
+                          json_t *entry)
+{
+    if (job_event_enqueue (job, flags, entry) < 0)
+        return -1;
+    if (json_array_size (job->event_queue) > 1 || job->hold_events)
+        return 0; // break recursion
+    return event_job_post_deferred (event, job);
+}
+
+int event_job_post_vpack (struct event *event,
+                          struct job *job,
+                          const char *name,
+                          int flags,
+                          const char *context_fmt,
+                          va_list ap)
+{
+    json_t *entry = NULL;
+    int saved_errno;
+    int rc;
+
+    if (!(entry = eventlog_entry_vpack (0., name, context_fmt, ap)))
+        return -1;
+    rc = event_job_post_entry (event, job, flags, entry);
+
     saved_errno = errno;
     json_decref (entry);
-    va_end (ap);
     errno = saved_errno;
-    return -1;
+    return rc;
+}
+
+int event_job_post_pack (struct event *event,
+                         struct job *job,
+                         const char *name,
+                         int flags,
+                         const char *context_fmt,
+                         ...)
+{
+    int rc;
+    va_list ap;
+
+    va_start (ap, context_fmt);
+    rc = event_job_post_vpack (event, job, name, flags, context_fmt, ap);
+    va_end (ap);
+    return rc;
 }
 
 /* Finalizes in-flight batch KVS commits and event pubs (synchronously).
@@ -560,28 +950,49 @@ void event_ctx_destroy (struct event *event)
     if (event) {
         int saved_errno = errno;
         flux_watcher_destroy (event->timer);
+        flux_msg_handler_delvec (event->handlers);
         event_batch_commit (event);
         if (event->pending) {
             struct event_batch *batch;
             while ((batch = zlist_pop (event->pending)))
-                event_batch_destroy (batch); // N.B. can append to pub_futures
+                event_batch_destroy (batch);
         }
         zlist_destroy (&event->pending);
-        if (event->pub_futures) {
-            flux_future_t *f;
-            while ((f = zlist_pop (event->pub_futures))) {
-                if (flux_future_get (f, NULL) < 0)
-                    flux_log_error (event->ctx->h,
-                                    "error publishing job-state event");
-                flux_future_destroy (f);
-            }
-        }
-        zlist_destroy (&event->pub_futures);
+        zhashx_destroy (&event->evindex);
         free (event);
         errno = saved_errno;
     }
 }
 
+static void set_timeout_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
+{
+    struct event *event = arg;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:F}",
+                             "timeout", &event->batch_timeout) < 0)
+        goto error;
+    if (flux_respond (h, msg, NULL) < 0)
+        goto error;
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "flux_msg_respond_error");
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,
+      "job-manager.set-batch-timeout",
+      set_timeout_cb,
+      0
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
 struct event *event_ctx_create (struct job_manager *ctx)
 {
     struct event *event;
@@ -589,6 +1000,7 @@ struct event *event_ctx_create (struct job_manager *ctx)
     if (!(event = calloc (1, sizeof (*event))))
         return NULL;
     event->ctx = ctx;
+    event->batch_timeout = 0.01;
     if (!(event->timer = flux_timer_watcher_create (flux_get_reactor (ctx->h),
                                                     0.,
                                                     0.,
@@ -597,8 +1009,11 @@ struct event *event_ctx_create (struct job_manager *ctx)
         goto error;
     if (!(event->pending = zlist_new ()))
         goto nomem;
-    if (!(event->pub_futures = zlist_new ()))
+    if (!(event->evindex = zhashx_new ()))
         goto nomem;
+    if (flux_msg_handler_addvec (ctx->h, htab, event, &event->handlers) < 0)
+        goto error;
+
     return event;
 nomem:
     errno = ENOMEM;
@@ -607,6 +1022,16 @@ struct event *event_ctx_create (struct job_manager *ctx)
     return NULL;
 }
 
+int event_index (struct event *event, const char *name)
+{
+    void *entry = zhashx_lookup (event->evindex, name);
+    if (!entry) {
+        entry = int2ptr (((int) zhashx_size (event->evindex) + 1));
+        (void)zhashx_insert (event->evindex, name, entry);
+    }
+    return ptr2int (entry);
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/job-manager/event.h b/src/modules/job-manager/event.h
index c62c87813a19..32d1d4283db2 100644
--- a/src/modules/job-manager/event.h
+++ b/src/modules/job-manager/event.h
@@ -18,6 +18,14 @@
 #include "job.h"
 #include "job-manager.h"
 
+enum job_manager_event_flags {
+
+    /*  EVENT_NO_COMMIT events are the same as any other event, except
+     *   that the event is not posted to the job eventlog in the KVS.
+     */
+    EVENT_NO_COMMIT = 1,
+};
+
 /* Take any action for 'job' currently needed based on its internal state.
  * Returns 0 on success, -1 on failure with errno set.
  * This function is idempotent.
@@ -39,21 +47,46 @@ int event_batch_pub_state (struct event *event, struct job *job,
  */
 int event_batch_respond (struct event *event, const flux_msg_t *msg);
 
+/* Add job to batch, job event handling will be paused until batch completion.
+ */
+int event_batch_add_job (struct event *event, struct job *job);
+
 /* Post event 'name' and optionally 'context' to 'job'.
  * Internally, calls event_job_update(), then event_job_action(), then commits
  * the event to job KVS eventlog.  The KVS commit completes asynchronously.
- * The future passed in as an argument should not be destroyed.
  * Returns 0 on success, -1 on failure with errno set.
  */
 int event_job_post_pack (struct event *event,
                          struct job *job,
                          const char *name,
+                         int flags,
                          const char *context_fmt,
                          ...);
 
+int event_job_post_vpack (struct event *event,
+                          struct job *job,
+                          const char *name,
+                          int flags,
+                          const char *context_fmt,
+                          va_list ap);
+
+int event_job_post_entry (struct event *event,
+                          struct job *job,
+                          int flags,
+                          json_t *entry);
+
 void event_ctx_destroy (struct event *event);
 struct event *event_ctx_create (struct job_manager *ctx);
 
+void event_listeners_disconnect_rpc (flux_t *h,
+                                     flux_msg_handler_t *mh,
+                                     const flux_msg_t *msg,
+                                     void *arg);
+
+
+/*  Return globally unique index for event name */
+int event_index (struct event *event, const char *name);
+
 #endif /* _FLUX_JOB_MANAGER_EVENT_H */
 
 /*
diff --git a/src/modules/job-manager/getattr.c b/src/modules/job-manager/getattr.c
new file mode 100644
index 000000000000..4bcb5df8f9ca
--- /dev/null
+++ b/src/modules/job-manager/getattr.c
@@ -0,0 +1,141 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* getattr - fetch job information about one job
+ *
+ * Purpose:
+ *   Expose job manager internals for testing.
+ *
+ * Input:
+ * - List of attributes
+ *
+ * Output:
+ * - Dictionary of attributes and values.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
+#include "job.h"
+#include "job-manager.h"
+
+#include "getattr.h"
+
+static json_t *make_dict (struct job *job,
+                          json_t *attrs,
+                          flux_error_t *errp)
+{
+    size_t index;
+    json_t *val;
+    json_t *dict;
+
+    if (!(dict = json_object ()))
+        goto nomem;
+    json_array_foreach (attrs, index, val) {
+        const char *key = json_string_value (val);
+        if (!key) {
+            errprintf (errp, "attribute list contains non-string");
+            errno = EPROTO;
+            goto error;
+        }
+        if (streq (key, "jobspec")) {
+            if (!job->jobspec_redacted) {
+                errprintf (errp, "jobspec is NULL");
+                errno = ENOENT;
+                goto error;
+            }
+            if (json_object_set (dict, key, job->jobspec_redacted) < 0)
+                goto nomem;
+        }
+        else if (streq (key, "R")) {
+            if (!job->R_redacted) {
+                errprintf (errp, "R is NULL");
+                errno = ENOENT;
+                goto error;
+            }
+            if (json_object_set (dict, key, job->R_redacted) < 0)
+                goto nomem;
+        }
+        else if (streq (key, "eventlog")) {
+            if (json_object_set (dict, key, job->eventlog) < 0)
+                goto nomem;
+        }
+        else {
+            errprintf (errp, "unknown attr %s", key);
+            errno = ENOENT;
+            goto error;
+        }
+    }
+    return dict;
+nomem:
+    errprintf (errp, "out of memory");
+    errno = ENOMEM;
+error:
+    ERRNO_SAFE_WRAP (json_decref, dict);
+    return NULL;
+}
+
+void getattr_handle_request (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
+{
+    struct job_manager *ctx = arg;
+    struct flux_msg_cred cred;
+    flux_jobid_t id;
+    struct job *job;
+    json_t *attrs;
+    const char *errstr = NULL;
+    flux_error_t error;
+    json_t *dict = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:o}",
+                             "id", &id,
+                             "attrs", &attrs) < 0
+        || flux_msg_get_cred (msg, &cred) < 0)
+        goto error;
+    if (!(job = zhashx_lookup (ctx->active_jobs, &id))
+        && !(job = zhashx_lookup (ctx->inactive_jobs, &id))) {
+        errstr = "unknown job";
+        errno = EINVAL;
+        goto error;
+    }
+    /* Security: guests can only access their own jobs
+     */
+    if (flux_msg_cred_authorize (cred, job->userid) < 0) {
+        errstr = "guests can only reprioritize their own jobs";
+        goto error;
+    }
+    if (!(dict = make_dict (job, attrs, &error))) {
+        errstr = error.text;
+        goto error;
+    }
+    if (flux_respond_pack (h, msg, "O", dict) < 0)
+        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+    json_decref (dict);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    json_decref (dict);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-manager/getattr.h b/src/modules/job-manager/getattr.h
new file mode 100644
index 000000000000..a9c940dec9fe
--- /dev/null
+++ b/src/modules/job-manager/getattr.h
@@ -0,0 +1,29 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_GETATTR_H_
+#define _FLUX_JOB_MANAGER_GETATTR_H_
+
+#include <flux/core.h>
+#include "job-manager.h"
+
+/* Handle a 'getattr' request
+ */
+void getattr_handle_request (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg);
+
+
+#endif /* ! _FLUX_JOB_MANAGER_GETATTR_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-manager/housekeeping.c b/src/modules/job-manager/housekeeping.c
new file mode 100644
index 000000000000..50e16f21a717
--- /dev/null
+++ b/src/modules/job-manager/housekeeping.c
@@ -0,0 +1,852 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* housekeeping - clean resources prior to release to the scheduler
+ *
+ * Purpose:
+ *   Resources are released by jobs to housekeeping.  Housekeeping runs
+ *   an epilog-like script, then releases resources to the scheduler.
+ *   Unlike the job manager epilog, housekeeping runs after the job, which
+ *   is allowed to exit CLEANUP when resources are handed over to housekeeping.
+ *   The scheduler still thinks resources are allocated to the job.
+ *
+ * Configuration:
+ *   [job-manager.housekeeping]
+ *   #command = ["command", "arg1", "arg2", ...]
+ *   release-after = "FSD"
+ *
+ * Partial release:
+ *   The 'release-after' config key enables partial release of resources.
+ *   - If unset, resources for a given job are not released until all exec
+ *     targets have completed housekeeping.
+ *   - If set to "0", resources are released as each exec target completes.
+ *   - If set to a nozero duration, a timer starts when the first exec target
+ *     for a given job completes.  When the timer expires, resources for all
+ *     the completed exec targets are released.  Following that, resources
+ *     are released as each target completes.
+ *
+ * Script credentials:
+ *   The housekeeping script runs as the instance owner (e.g. "flux").
+ *   On a real system, "command" is configured to "imp run housekeeping",
+ *   and the IMP is configured to launch the flux-housekeeping systemd
+ *   service as root.
+ *
+ * Script environment:
+ *   FLUX_JOB_ID - the job whose resources are running housekeeping
+ *   FLUX_JOB_USERID - the UID of the job's owner
+ *   FLUX_URI - the URI of the local flux broker
+ *   The IMP must be configured to explicitly allow FLUX_* to pass through.
+ *
+ * Script error handling:
+ *   If housekeeping fails on a node or set of nodes, this is logged to
+ *   the flux circular buffer at LOG_ERR.
+ *   Stdout is logged at LOG_INFO and stderr at LOG_ERR.
+ *
+ * Error handling under systemd:
+ *   When using systemd, any output is captured by the systemd journal on
+ *   the remote node, accessed with 'journalctl -u flux-housekeeping@*'.
+ *
+ *   If the housekeeping script fails, the systemd unit file automatically
+ *   drains the node.
+ *
+ * Core scheduled instances:
+ *   Note that housekeeping runs after every job even if the job did not
+ *   allocate the whole node.
+ *
+ * Job manager module stats:
+ *   'flux module stats job-manager | jq .housekeeping' returns the following:
+ *     {"running":o}
+ *   "running" is a dictionary of jobids (f58) for jobs currently
+ *   running housekeeping.  Each job object consists of:
+ *     {"pending":s "allocated":s, "t_start":f}
+ *   where
+ *     pending: set of ranks on which housekeeping is needed/active
+ *     allocated: set of ranks still allocated by housekeeping
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <flux/idset.h>
+#include <unistd.h>
+#include <signal.h>
+#ifdef HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/librlist/rlist.h"
+#include "src/common/libhostlist/hostlist.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libsubprocess/bulk-exec.h"
+#include "src/common/libsubprocess/command.h"
+#include "ccan/str/str.h"
+
+#include "job.h"
+#include "alloc.h"
+#include "job-manager.h"
+#include "conf.h"
+
+#include "housekeeping.h"
+
+extern char **environ;
+
+// -1 = never, 0 = immediate, >0 = time in seconds
+static const double default_release_after = -1;
+
+struct allocation {
+    flux_jobid_t id;
+    struct rlist *rl;       // R, diminished each time a subset is released
+    struct idset *pending;  // ranks in need of housekeeping
+    struct housekeeping *hk;
+    flux_watcher_t *timer;
+    bool timer_armed;
+    bool timer_expired;
+    int free_count;         // number of releases
+    double t_start;
+    struct bulk_exec *bulk_exec;
+    void *list_handle;
+};
+
+struct housekeeping {
+    struct job_manager *ctx;
+    flux_cmd_t *cmd; // NULL if not configured
+    double release_after;
+    char *imp_path;
+    zlistx_t *allocations;
+    flux_msg_handler_t **handlers;
+};
+
+static struct bulk_exec_ops bulk_ops;
+
+static void allocation_timeout (flux_reactor_t *r,
+                                flux_watcher_t *w,
+                                int revents,
+                                void *arg);
+
+static void allocation_destroy (struct allocation *a)
+{
+    if (a) {
+        int saved_errno = errno;
+        rlist_destroy (a->rl);
+        idset_destroy (a->pending);
+        flux_watcher_destroy (a->timer);
+        bulk_exec_destroy (a->bulk_exec);
+        free (a);
+        errno = saved_errno;
+    }
+}
+
+// zlistx_destructor_fn footprint
+static void allocation_destructor (void **item)
+{
+    if (item) {
+        allocation_destroy (*item);
+        *item = NULL;
+    }
+
+}
+
+static int update_cmd_env (flux_cmd_t *cmd, flux_jobid_t id, uint32_t userid)
+{
+    if (flux_cmd_setenvf (cmd, 1, "FLUX_JOB_ID", "%ju", (uintmax_t)id) < 0
+        || flux_cmd_setenvf (cmd, 1, "FLUX_JOB_USERID", "%u", userid) < 0)
+        return -1;
+    return 0;
+}
+
+static struct allocation *allocation_create (struct housekeeping *hk,
+                                             json_t *R,
+                                             flux_jobid_t id,
+                                             uint32_t userid)
+{
+    struct allocation *a;
+    flux_reactor_t *r = flux_get_reactor (hk->ctx->h);
+
+    if (!(a = calloc (1, sizeof (*a))))
+        return NULL;
+    a->hk = hk;
+    a->id = id;
+    a->t_start = flux_reactor_now (flux_get_reactor (hk->ctx->h));
+    if (!(a->rl = rlist_from_json (R, NULL))
+        || !(a->pending = rlist_ranks (a->rl))
+        || !(a->timer = flux_timer_watcher_create (r,
+                                                   0,
+                                                   0.,
+                                                   allocation_timeout,
+                                                   a))
+        || !(a->bulk_exec = bulk_exec_create (&bulk_ops,
+                                              "rexec",
+                                              id,
+                                              "housekeeping",
+                                               a))
+        || update_cmd_env (hk->cmd, id, userid) < 0
+        || bulk_exec_push_cmd (a->bulk_exec, a->pending, hk->cmd, 0) < 0) {
+        allocation_destroy (a);
+        return NULL;
+    }
+    return a;
+}
+
+/*  Return the set of ranks in the remaining resource set (a->rl) which are
+ *  not still pending housekeeping (a->pending). That is:
+ *
+ *   ranks (a->rl) -= a->pending
+ *
+ */
+static struct idset *get_housekept_ranks (struct allocation *a)
+{
+    struct idset *ranks;
+
+    if (!(ranks = rlist_ranks (a->rl)))
+        goto error;
+    if (idset_subtract (ranks, a->pending) < 0)
+        goto error;
+    return ranks;
+error:
+    idset_destroy (ranks);
+    return NULL;
+}
+
+/* Release any resources in a->rl associated with ranks that are no longer
+ * pending for housekeeping.  Then remove them from a->rl.
+ */
+static void allocation_release (struct allocation *a)
+{
+    struct job_manager *ctx = a->hk->ctx;
+    struct idset *ranks = NULL;
+    struct rlist *rl = NULL;
+    json_t *R = NULL;
+    bool final = false;
+
+    if ((ranks = get_housekept_ranks (a)) && idset_count (ranks) == 0) {
+        idset_destroy (ranks);
+        return; // nothing to do
+    }
+    if (idset_empty (a->pending))
+        final = true;
+
+    if (!ranks
+        || !(rl = rlist_copy_ranks (a->rl, ranks))
+        || !(R = rlist_to_R (rl))
+        || alloc_send_free_request (ctx->alloc, R, a->id, final) < 0
+        || rlist_remove_ranks (a->rl, ranks) < 0) {
+        char *s = idset_encode (ranks, IDSET_FLAG_RANGE);
+        flux_log (ctx->h,
+                  LOG_ERR,
+                  "housekeeping error releasing resources for job %s ranks %s",
+                  idf58 (a->id),
+                  s ? s : "NULL");
+        free (s);
+    }
+    else
+        a->free_count++;
+    json_decref (R);
+    rlist_destroy (rl);
+    idset_destroy (ranks);
+}
+
+static void allocation_remove (struct allocation *a)
+{
+    if (!a->list_handle
+        || zlistx_delete (a->hk->allocations, a->list_handle) < 0) {
+        flux_log (a->hk->ctx->h,
+                  LOG_CRIT,
+                  "housekeeping: internal error removing allocation for %s",
+                  idf58 (a->id));
+    }
+}
+
+static void allocation_timeout (flux_reactor_t *r,
+                                flux_watcher_t *w,
+                                int revents,
+                                void *arg)
+{
+    struct allocation *a = arg;
+
+    a->timer_expired = true;
+
+    // release the ranks that have completed housekeeping so far
+    allocation_release (a);
+
+    /* Note: All resources will never be released under the timeout
+     * because completion of housekeeping on the final rank will
+     * always release all resources immediately instead of waiting
+     * for the timer. Therefore, there is no need to check if
+     * rlist_rnodes (a->rl) is zero here (it never will be).
+     */
+}
+
+/* 'rank' has completed housekeeping.
+ */
+static bool housekeeping_finish_one (struct allocation *a, int rank)
+{
+    if (!idset_test (a->pending, rank))
+        return false;
+    idset_clear (a->pending, rank);
+
+    if (idset_count (a->pending) == 0
+        || a->hk->release_after == 0
+        || a->timer_expired) {
+        allocation_release (a);
+    }
+    if (!a->timer_armed && a->hk->release_after > 0) {
+        flux_timer_watcher_reset (a->timer, a->hk->release_after, 0.);
+        flux_watcher_start (a->timer);
+        a->timer_armed = true;
+    }
+    return true;
+}
+
+static void set_failed_reason (const char **s, const char *reason)
+{
+    if (!*s)
+        *s = reason;
+    else if (!streq (*s, reason))
+        *s = "multiple failure modes";
+}
+
+static void bulk_start (struct bulk_exec *bulk_exec, void *arg)
+{
+    struct allocation *a = arg;
+    flux_t *h = a->hk->ctx->h;
+
+    flux_log (h, LOG_DEBUG, "housekeeping: %s started", idf58 (a->id));
+}
+
+static void bulk_exit (struct bulk_exec *bulk_exec,
+                       void *arg,
+                       const struct idset *ids)
+{
+    struct allocation *a = arg;
+    flux_t *h = a->hk->ctx->h;
+    unsigned int rank;
+    struct idset *failed_ranks = NULL;
+    char *failed_ranks_str = NULL;
+    char *failed_hosts = NULL;
+    const char *failed_reason = NULL;
+
+    rank = idset_first (ids);
+    while (rank != IDSET_INVALID_ID) {
+        if (housekeeping_finish_one (a, rank)) {
+            flux_subprocess_t *p = bulk_exec_get_subprocess (bulk_exec, rank);
+            bool fail = false;
+            int n;
+            if ((n = flux_subprocess_signaled (p)) > 0) {
+                fail = true;
+                set_failed_reason (&failed_reason, strsignal (n));
+            }
+            else {
+                n = flux_subprocess_exit_code (p);
+                if (n != 0) {
+                    fail = true;
+                    set_failed_reason (&failed_reason, "nonzero exit code");
+                }
+            }
+            if (fail) {
+                if (!failed_ranks)
+                    failed_ranks = idset_create (0, IDSET_FLAG_AUTOGROW);
+                idset_set (failed_ranks, rank);
+            }
+        }
+        rank = idset_next (ids, rank);
+    }
+    // log a consolidated error message for potentially multiple ranks
+    if (failed_ranks
+        && (failed_ranks_str = idset_encode (failed_ranks, IDSET_FLAG_RANGE))
+        && (failed_hosts = flux_hostmap_lookup (h, failed_ranks_str, NULL))
+        && failed_reason) {
+        flux_log (h,
+                  LOG_ERR,
+                  "housekeeping: %s (rank %s) %s: %s",
+                  failed_hosts,
+                  failed_ranks_str,
+                  idf58 (a->id),
+                  failed_reason);
+
+    }
+    idset_destroy (failed_ranks);
+    free (failed_ranks_str);
+    free (failed_hosts);
+}
+
+static void bulk_complete (struct bulk_exec *bulk_exec, void *arg)
+{
+    struct allocation *a = arg;
+    flux_t *h = a->hk->ctx->h;
+
+    flux_log (h, LOG_DEBUG, "housekeeping: %s complete", idf58 (a->id));
+    allocation_remove (a);
+}
+
+static void bulk_output (struct bulk_exec *bulk_exec,
+                         flux_subprocess_t *p,
+                         const char *stream,
+                         const char *data,
+                         int data_len,
+                         void *arg)
+{
+    struct allocation *a = arg;
+    flux_t *h = a->hk->ctx->h;
+    int rank = flux_subprocess_rank (p);
+
+    flux_log (h,
+              streq (stream, "stderr") ? LOG_ERR : LOG_INFO,
+              "housekeeping: %s (rank %d) %s: %.*s",
+              flux_get_hostbyrank (h, rank),
+              rank,
+              idf58 (a->id),
+              data_len,
+              data);
+}
+
+static void bulk_error (struct bulk_exec *bulk_exec,
+                        flux_subprocess_t *p,
+                        void *arg)
+{
+    struct allocation *a = arg;
+    flux_t *h = a->hk->ctx->h;
+    int rank = flux_subprocess_rank (p);
+    const char *hostname = flux_get_hostbyrank (h, rank);
+    const char *error = flux_subprocess_fail_error (p);
+
+    flux_log (h,
+              LOG_ERR,
+              "housekeeping: %s (rank %d) %s: %s",
+              hostname,
+              rank,
+              idf58 (a->id),
+              error);
+
+    housekeeping_finish_one (a, rank);
+}
+
+int housekeeping_start (struct housekeeping *hk,
+                        json_t *R,
+                        flux_jobid_t id,
+                        uint32_t userid)
+{
+    flux_t *h = hk->ctx->h;
+    struct allocation *a;
+
+    /* Housekeeping is not configured
+     */
+    if (!hk->cmd)
+        goto skip;
+
+    /* Create and start the 'allocation' and put it in our list.
+     * N.B. bulk_exec_start() starts watchers but does not send RPCs.
+     */
+    if (!(a = allocation_create (hk, R, id, userid))
+        || bulk_exec_start (h, a->bulk_exec) < 0
+        || !(a->list_handle = zlistx_insert (hk->allocations, a, false))) {
+        flux_log (h,
+                  LOG_ERR,
+                  "housekeeping: %s error creating alloc object"
+                  " - returning resources to the scheduler",
+                  idf58 (id));
+        allocation_destroy (a);
+        goto skip;
+    }
+    return 0;
+skip:
+    return alloc_send_free_request (hk->ctx->alloc, R, id, true);
+}
+
+static int housekeeping_hello_respond_one (struct housekeeping *hk,
+                                           const flux_msg_t *msg,
+                                           struct allocation *a,
+                                           flux_error_t *error)
+{
+    struct job *job;
+
+    if (a->free_count > 0) {
+        errprintf (error, "partial release is not supported by RFC 27 hello");
+        goto error;
+    }
+    if (!(job = zhashx_lookup (hk->ctx->inactive_jobs, &a->id))
+        && !(job = zhashx_lookup (hk->ctx->active_jobs, &a->id))) {
+        errprintf (error, "the job could not be looked up during RFC 27 hello");
+        goto error;
+    }
+    if (flux_respond_pack (hk->ctx->h,
+                           msg,
+                           "{s:I s:I s:I s:f}",
+                           "id", job->id,
+                           "priority", job->priority,
+                           "userid", (json_int_t)job->userid,
+                           "t_submit", job->t_submit) < 0) {
+        errprintf (error,
+                   "the RFC 27 hello response could not be sent: %s",
+                   strerror (errno));
+        goto error;
+    }
+    return 0;
+error:
+    return -1;
+}
+
+static void kill_continuation (flux_future_t *f, void *arg)
+{
+    struct housekeeping *hk = arg;
+
+    if (flux_future_get (f, NULL) < 0)
+        flux_log (hk->ctx->h, LOG_ERR, "kill: %s", future_strerror (f, errno));
+    flux_future_destroy (f);
+}
+
+/* Participate in the scheduler hello protocol, where the scheduler is informed
+ * of resources that are already allocated.  Since partial release is not yet
+ * supported in the hello protocol, for now, we must let go of any partial
+ * allocations.  Send remaining housekeeping tasks a SIGTERM, log an error,
+ * and delete the allocation.
+ */
+int housekeeping_hello_respond (struct housekeeping *hk, const flux_msg_t *msg)
+{
+    struct allocation *a;
+    flux_error_t error;
+
+    a = zlistx_first (hk->allocations);
+    while (a) {
+        if (housekeeping_hello_respond_one (hk, msg, a, &error) < 0) {
+            char *ranks;
+            char *hosts = NULL;
+            flux_future_t *f;
+
+            if ((ranks = idset_encode (a->pending, IDSET_FLAG_RANGE)))
+                hosts = flux_hostmap_lookup (hk->ctx->h, ranks, NULL);
+            flux_log (hk->ctx->h,
+                      LOG_ERR,
+                      "housekeeping: %s (rank %s) from %s will be terminated"
+                      " because %s",
+                      hosts ? hosts : "?",
+                      ranks ? ranks : "?",
+                      idf58 (a->id),
+                      error.text);
+            free (hosts);
+            free (ranks);
+
+            f = bulk_exec_kill (a->bulk_exec, NULL, SIGTERM);
+            if (flux_future_then (f, -1, kill_continuation, hk) < 0)
+                flux_future_destroy (f);
+
+            // delete the allocation to avoid sending frees later
+            allocation_remove (a);
+        }
+        a = zlistx_next (hk->allocations);
+    }
+    return 0;
+}
+
+static json_t *housekeeping_get_stats_job (struct allocation *a)
+{
+    struct idset *ranks;
+    char *s = NULL;
+    char *p = NULL;
+    json_t *job = NULL;
+
+    if (!(ranks = rlist_ranks (a->rl))
+        || !(p = idset_encode (ranks, IDSET_FLAG_RANGE))
+        || !(s = idset_encode (a->pending, IDSET_FLAG_RANGE)))
+        goto out;
+    job = json_pack ("{s:f s:s s:s}",
+                     "t_start", a->t_start,
+                     "pending", s,
+                     "allocated", p);
+out:
+    idset_destroy (ranks);
+    free (s);
+    free (p);
+    return job;
+}
+
+/* Support adding a housekeeping object to the the 'job-manager.stats-get'
+ * response in job-manager.c.
+ */
+json_t *housekeeping_get_stats (struct housekeeping *hk)
+{
+    json_t *running;
+    json_t *stats = NULL;
+    struct allocation *a;
+
+    if (!(running = json_object ()))
+        goto nomem;
+    a = zlistx_first (hk->allocations);
+    while (a) {
+        json_t *job;
+        if (!(job = housekeeping_get_stats_job (a))
+            || json_object_set_new (running, idf58 (a->id), job) < 0) {
+            json_decref (job);
+            goto nomem;
+        }
+        a = zlistx_next (hk->allocations);
+    }
+    if (!(stats = json_pack ("{s:O}", "running", running)))
+        goto nomem;
+    json_decref (running);
+    return stats;
+nomem:
+    json_decref (running);
+    errno = ENOMEM;
+    return NULL;
+}
+
+/* Support accounting for resources stuck in housekeeping when preparing the
+ * 'job-manager.resource-status' response in alloc.c.
+ */
+int housekeeping_stat_append (struct housekeeping *hk,
+                              struct rlist *rl,
+                              flux_error_t *error)
+{
+    struct allocation *a;
+    a = zlistx_first (hk->allocations);
+    while (a) {
+        if (rlist_append (rl, a->rl) < 0) {
+            errprintf (error,
+                       "%s: duplicate housekeeping allocation",
+                       idf58 (a->id));
+            return -1;
+        }
+        a = zlistx_next (hk->allocations);
+    }
+    return 0;
+}
+
+static void housekeeping_kill_cb (flux_t *h,
+                                  flux_msg_handler_t *mh,
+                                  const flux_msg_t *msg,
+                                  void *arg)
+{
+    struct housekeeping *hk = arg;
+    int signum;
+    flux_jobid_t jobid = FLUX_JOBID_ANY;
+    const char *ranks = NULL;
+    struct idset *ids = NULL;
+    idset_error_t error;
+    const char *errmsg = NULL;
+    struct allocation *a;
+    flux_future_t *f;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s?I s?s}",
+                             "signum", &signum,
+                             "jobid", &jobid,
+                             "ranks", &ranks) < 0)
+        goto error;
+    if (ranks) {
+        if (!(ids = idset_decode_ex (ranks, -1, -1, 0, &error))) {
+            errmsg = error.text;
+            goto error;
+        }
+    }
+    a = zlistx_first (hk->allocations);
+    while (a) {
+        if (a->id == jobid || jobid == FLUX_JOBID_ANY) {
+            if (a->bulk_exec) {
+                f = bulk_exec_kill (a->bulk_exec, ids, signum);
+                if (flux_future_then (f, -1, kill_continuation, hk) < 0)
+                    flux_future_destroy (f);
+            }
+        }
+        a = zlistx_next (hk->allocations);
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to housekeeping-kill");
+    idset_destroy (ids);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to housekeeping-kill");
+    idset_destroy (ids);
+}
+
+static flux_cmd_t *create_cmd (json_t *cmdline)
+{
+    size_t index;
+    json_t *value;
+    char *argz = NULL;
+    size_t argz_len = 0;
+    int argc;
+    char **argv = NULL;
+    flux_cmd_t *cmd = NULL;
+
+    json_array_foreach (cmdline, index, value) {
+        if (!json_is_string (value)
+            || argz_add (&argz, &argz_len, json_string_value (value)) != 0)
+            goto done;
+    }
+    if ((argc = argz_count (argz, argz_len)) == 0
+        || !(argv = calloc (argc + 1, sizeof (argv[0]))))
+        goto done;
+    argz_extract (argz, argz_len, argv);
+    if (!(cmd = flux_cmd_create (argc, argv, environ)))
+        goto done;
+done:
+    free (argz);
+    free (argv);
+    return cmd;
+}
+
+static int housekeeping_parse_config (const flux_conf_t *conf,
+                                      flux_error_t *error,
+                                      void *arg)
+{
+    struct housekeeping *hk = arg;
+    json_t *housekeeping = NULL;
+    flux_error_t e;
+    json_error_t jerror;
+    json_t *cmdline = NULL;
+    const char *release_after = NULL;
+    flux_cmd_t *cmd = NULL;
+    const char *imp_path = NULL;
+    char *imp_path_cpy = NULL;
+    int use_systemd_unit = 0;
+
+    if (flux_conf_unpack (conf,
+                          &e,
+                          "{s?{s?o}}",
+                          "job-manager",
+                            "housekeeping", &housekeeping) < 0)
+        return errprintf (error, "job-manager.housekeeping: %s", e.text);
+
+    // if the housekeeping table is not present, housekeeping is not configured
+    if (!housekeeping)
+        goto done;
+
+    if (json_unpack_ex (housekeeping,
+                        &jerror,
+                        0,
+                        "{s?o s?s s?b !}",
+                        "command", &cmdline,
+                        "release-after", &release_after,
+                        "use-systemd-unit", &use_systemd_unit) < 0)
+        return errprintf (error, "job-manager.housekeeping: %s", jerror.text);
+
+    if (use_systemd_unit) {
+        flux_log (hk->ctx->h,
+                  LOG_ERR,
+                  "job-manager.housekeeping.use-systemd-unit is deprecated"
+                  " - ignoring");
+    }
+
+    // let job-exec handle exec errors
+    (void)flux_conf_unpack (conf, NULL, "{s?{s?s}}", "exec", "imp", &imp_path);
+
+    if (release_after) {
+        if (fsd_parse_duration (release_after, &hk->release_after) < 0)
+            return errprintf (error,
+                              "job-manager.housekeeping.release-after"
+                              " FSD parse error");
+    }
+
+    if (cmdline) {
+        if (!(cmd = create_cmd (cmdline)))
+            return errprintf (error, "error creating housekeeping command");
+    }
+
+    // if no command line was defined, assume "imp exec housekeeping"
+    else {
+        if (!imp_path) {
+            return errprintf (error,
+                              "job-manager.housekeeping implies IMP"
+                              " but exec.imp is undefined");
+        }
+        json_t *o;
+        if ((o = json_pack ("[sss]", imp_path, "run", "housekeeping")))
+            cmd = create_cmd (o);
+        json_decref (o);
+        if (!cmd)
+            return errprintf (error, "error creating housekeeping command");
+        if (!(imp_path_cpy = strdup (imp_path))) {
+            flux_cmd_destroy (cmd);
+            return errprintf (error, "error duplicating IMP path");
+        }
+    }
+done:
+    flux_cmd_destroy (hk->cmd);
+    hk->cmd = cmd;
+    free (hk->imp_path);
+    hk->imp_path = imp_path_cpy;
+    flux_log (hk->ctx->h,
+              LOG_DEBUG,
+              "housekeeping is %sconfigured%s",
+              hk->cmd ? "" : "not ",
+              hk->imp_path ? " with IMP" : "");
+    return 1; // allow dynamic changes
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        .typemask = FLUX_MSGTYPE_REQUEST,
+        .topic_glob = "job-manager.housekeeping-kill",
+        .cb = housekeeping_kill_cb,
+        .rolemask = 0
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void housekeeping_ctx_destroy (struct housekeeping *hk)
+{
+    if (hk) {
+        int saved_errno = errno;
+        conf_unregister_callback (hk->ctx->conf, housekeeping_parse_config);
+        flux_cmd_destroy (hk->cmd);
+        zlistx_destroy (&hk->allocations);
+        flux_msg_handler_delvec (hk->handlers);
+        free (hk->imp_path);
+        free (hk);
+        errno = saved_errno;
+    }
+}
+
+struct housekeeping *housekeeping_ctx_create (struct job_manager *ctx)
+{
+    struct housekeeping *hk;
+    flux_error_t error;
+
+    if (!(hk = calloc (1, sizeof (*hk))))
+        return NULL;
+    hk->ctx = ctx;
+    hk->release_after = default_release_after;
+    if (!(hk->allocations = zlistx_new ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    zlistx_set_destructor (hk->allocations, allocation_destructor);
+    if (conf_register_callback (ctx->conf,
+                                &error,
+                                housekeeping_parse_config,
+                                hk) < 0) {
+        flux_log (ctx->h, LOG_ERR, "%s", error.text);
+        goto error;
+    }
+    if (flux_msg_handler_addvec (ctx->h, htab, hk, &hk->handlers) < 0)
+        goto error;
+    return hk;
+error:
+    housekeeping_ctx_destroy (hk);
+    return NULL;
+}
+
+static struct bulk_exec_ops bulk_ops = {
+    .on_start = bulk_start,
+    .on_exit = bulk_exit,
+    .on_complete = bulk_complete,
+    .on_output = bulk_output,
+    .on_error = bulk_error,
+};
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/housekeeping.h b/src/modules/job-manager/housekeeping.h
new file mode 100644
index 000000000000..a468d3967632
--- /dev/null
+++ b/src/modules/job-manager/housekeeping.h
@@ -0,0 +1,45 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_HOUSEKEEPING_H
+#define _FLUX_JOB_MANAGER_HOUSEKEEPING_H
+
+#include <flux/core.h>
+#include "src/common/librlist/rlist.h"
+#include "job-manager.h"
+
+struct housekeeping *housekeeping_ctx_create (struct job_manager *ctx);
+void housekeeping_ctx_destroy (struct housekeeping *hk);
+
+/* Call this to transfer a job's R to the housekeeping subsystem.  The job
+ * may treat R as freed, but R will remain allocated from the scheduler's
+ * perspective until the housekeeping script is run on each execution target.
+ */
+int housekeeping_start (struct housekeeping *hk,
+                        json_t *R,
+                        flux_jobid_t id,
+                        uint32_t userid);
+
+/* Call this to add responses to the scheduler's hello request at startup.
+ * It should inform the scheduler about resources that are still allocated,
+ * but no longer directly held by jobs.
+ */
+int housekeeping_hello_respond (struct housekeeping *hk, const flux_msg_t *msg);
+
+json_t *housekeeping_get_stats (struct housekeeping *hk);
+
+int housekeeping_stat_append (struct housekeeping *hk,
+                              struct rlist *rl,
+                              flux_error_t *error);
+
+
+#endif /* ! _FLUX_JOB_MANAGER_HOUSEKEEPING_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/job-manager.c b/src/modules/job-manager/job-manager.c
index 675067900380..f90102b608fd 100644
--- a/src/modules/job-manager/job-manager.c
+++ b/src/modules/job-manager/job-manager.c
@@ -11,25 +11,99 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <unistd.h>
+#include <sys/types.h>
 #include <flux/core.h>
 
 #include "src/common/libjob/job_hash.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "job.h"
+#include "conf.h"
 #include "submit.h"
 #include "restart.h"
 #include "raise.h"
 #include "kill.h"
 #include "list.h"
-#include "priority.h"
+#include "urgency.h"
 #include "alloc.h"
+#include "housekeeping.h"
 #include "start.h"
 #include "event.h"
 #include "drain.h"
 #include "wait.h"
+#include "purge.h"
+#include "queue.h"
+#include "annotate.h"
+#include "journal.h"
+#include "getattr.h"
+#include "update.h"
+#include "jobtap-internal.h"
 
 #include "job-manager.h"
 
+void getinfo_handle_request (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
+{
+    struct job_manager *ctx = arg;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:I}",
+                           "max_jobid", ctx->max_jobid) < 0)
+        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+void disconnect_rpc (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    /* disconnects occur once per client, there is no way to know
+     * which services a client used, so we must check all services for
+     * cleanup */
+    alloc_disconnect_rpc (h, mh, msg, arg);
+    wait_disconnect_rpc (h, mh, msg, arg);
+    journal_listeners_disconnect_rpc (h, mh, msg, arg);
+}
+
+static void stats_cb (flux_t *h, flux_msg_handler_t *mh,
+                      const flux_msg_t *msg, void *arg)
+{
+    struct job_manager *ctx = arg;
+    json_t *journal = journal_get_stats (ctx->journal);
+    json_t *housekeeping = housekeeping_get_stats (ctx->housekeeping);
+    if (!housekeeping || !journal)
+        goto error;
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:O s:i s:i s:I s:O}",
+                           "journal", journal,
+                           "active_jobs", zhashx_size (ctx->active_jobs),
+                           "inactive_jobs", zhashx_size (ctx->inactive_jobs),
+                           "max_jobid", ctx->max_jobid,
+                           "housekeeping", housekeeping) < 0) {
+        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+        goto error;
+    }
+    json_decref (housekeeping);
+    json_decref (journal);
+    return;
+ error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    json_decref (housekeeping);
+    json_decref (journal);
+}
+
 static const struct flux_msg_handler_spec htab[] = {
     {
         FLUX_MSGTYPE_REQUEST,
@@ -39,10 +113,47 @@ static const struct flux_msg_handler_spec htab[] = {
     },
     {
         FLUX_MSGTYPE_REQUEST,
-        "job-manager.priority",
-        priority_handle_request,
+        "job-manager.urgency",
+        urgency_handle_request,
         FLUX_ROLE_USER
     },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.getattr",
+        getattr_handle_request,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.getinfo",
+        getinfo_handle_request,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.jobtap",
+        jobtap_handler,
+        FLUX_ROLE_OWNER,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.jobtap-query",
+        jobtap_query_handler,
+        FLUX_ROLE_OWNER,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.disconnect",
+        disconnect_rpc,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.stats-get",
+        stats_cb,
+        FLUX_ROLE_USER,
+    },
+
     FLUX_MSGHANDLER_TABLE_END,
 };
 
@@ -51,16 +162,37 @@ int mod_main (flux_t *h, int argc, char **argv)
     flux_reactor_t *r = flux_get_reactor (h);
     int rc = -1;
     struct job_manager ctx;
+    flux_error_t error;
 
     memset (&ctx, 0, sizeof (ctx));
     ctx.h = h;
+    ctx.owner = getuid ();
 
-    if (!(ctx.active_jobs = job_hash_create ())) {
-        flux_log_error (h, "error creating active_jobs hash");
+    if (!(ctx.active_jobs = job_hash_create ())
+        || !(ctx.inactive_jobs = job_hash_create ())) {
+        flux_log_error (h, "error creating jobs hash");
         goto done;
     }
     zhashx_set_destructor (ctx.active_jobs, job_destructor);
     zhashx_set_duplicator (ctx.active_jobs, job_duplicator);
+    zhashx_set_destructor (ctx.inactive_jobs, job_destructor);
+    zhashx_set_duplicator (ctx.inactive_jobs, job_duplicator);
+    if (!(ctx.conf = conf_create (&ctx, &error))) {
+        flux_log (h, LOG_ERR, "config: %s", error.text);
+        goto done;
+    }
+    if (!(ctx.jobtap = jobtap_create (&ctx))) {
+        flux_log (h, LOG_ERR, "error creating jobtap interface");
+        goto done;
+    }
+    if (!(ctx.purge = purge_create (&ctx))) {
+        flux_log_error (h, "error creating purge context");
+        goto done;
+    }
+    if (!(ctx.queue = queue_ctx_create (&ctx))) {
+        flux_log_error (h, "error creating queue context");
+        goto done;
+    }
     if (!(ctx.event = event_ctx_create (&ctx))) {
         flux_log_error (h, "error creating event batcher");
         goto done;
@@ -73,6 +205,10 @@ int mod_main (flux_t *h, int argc, char **argv)
         flux_log_error (h, "error creating scheduler interface");
         goto done;
     }
+    if (!(ctx.housekeeping = housekeeping_ctx_create (&ctx))) {
+        flux_log_error (h, "error creating resource housekeeping interface");
+        goto done;
+    }
     if (!(ctx.start = start_ctx_create (&ctx))) {
         flux_log_error (h, "error creating exec interface");
         goto done;
@@ -93,35 +229,61 @@ int mod_main (flux_t *h, int argc, char **argv)
         flux_log_error (h, "error creating kill interface");
         goto done;
     }
+    if (!(ctx.annotate = annotate_ctx_create (&ctx))) {
+        flux_log_error (h, "error creating annotate interface");
+        goto done;
+    }
+    if (!(ctx.journal = journal_ctx_create (&ctx))) {
+        flux_log_error (h, "error creating journal interface");
+        goto done;
+    }
+    if (!(ctx.update = update_ctx_create (&ctx))) {
+        flux_log_error (h, "error creating job update interface");
+        goto done;
+    }
     if (flux_msg_handler_addvec (h, htab, &ctx, &ctx.handlers) < 0) {
         flux_log_error (h, "flux_msghandler_add");
         goto done;
     }
-    if (restart_from_kvs (&ctx) < 0) {
-        flux_log_error (h, "restart_from_kvs");
+    if (restart_from_kvs (&ctx) < 0) // logs its own error messages
         goto done;
-    }
     if (flux_reactor_run (r, 0) < 0) {
         flux_log_error (h, "flux_reactor_run");
         goto done;
     }
+    if (restart_save_state (&ctx) < 0) {
+        flux_log_error (h, "error saving job manager state to KVS");
+        goto done;
+    }
     rc = 0;
 done:
     flux_msg_handler_delvec (ctx.handlers);
+    queue_ctx_destroy (ctx.queue);
+    purge_destroy (ctx.purge);
+    journal_ctx_destroy (ctx.journal);
+    annotate_ctx_destroy (ctx.annotate);
     kill_ctx_destroy (ctx.kill);
     raise_ctx_destroy (ctx.raise);
     wait_ctx_destroy (ctx.wait);
     drain_ctx_destroy (ctx.drain);
     start_ctx_destroy (ctx.start);
+    housekeeping_ctx_destroy (ctx.housekeeping);
     alloc_ctx_destroy (ctx.alloc);
     submit_ctx_destroy (ctx.submit);
     event_ctx_destroy (ctx.event);
+    update_ctx_destroy (ctx.update);
+    /* job aux containers may call destructors in jobtap plugins, so destroy
+     * jobs before unloading plugins; but don't destroy job hashes until after.
+     */
+    zhashx_purge (ctx.active_jobs);
+    zhashx_purge (ctx.inactive_jobs);
+    jobtap_destroy (ctx.jobtap);
+    conf_destroy (ctx.conf);
     zhashx_destroy (&ctx.active_jobs);
+    zhashx_destroy (&ctx.inactive_jobs);
     return rc;
 }
 
-MOD_NAME ("job-manager");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/job-manager/job-manager.h b/src/modules/job-manager/job-manager.h
index 9196f03ab481..87236d1e73ba 100644
--- a/src/modules/job-manager/job-manager.h
+++ b/src/modules/job-manager/job-manager.h
@@ -11,19 +11,32 @@
 #ifndef _FLUX_JOB_MANAGER_H
 #define _FLUX_JOB_MANAGER_H
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
 struct job_manager {
     flux_t *h;
     flux_msg_handler_t **handlers;
     zhashx_t *active_jobs;
+    zhashx_t *inactive_jobs;
     int running_jobs; // count of jobs in RUN | CLEANUP state
+    flux_jobid_t max_jobid; // largest jobid allocated thus far
+    uid_t owner;
+    struct conf *conf;
     struct start *start;
     struct alloc *alloc;
+    struct housekeeping *housekeeping;
     struct event *event;
     struct submit *submit;
     struct drain *drain;
     struct waitjob *wait;
     struct raise *raise;
     struct kill *kill;
+    struct annotate *annotate;
+    struct journal *journal;
+    struct purge *purge;
+    struct queue_ctx *queue;
+    struct update *update;
+    struct jobtap *jobtap;
 };
 
 #endif /* !_FLUX_JOB_MANAGER_H */
diff --git a/src/modules/job-manager/job.c b/src/modules/job-manager/job.c
index 4a5026dd046e..da2e15582278 100644
--- a/src/modules/job-manager/job.c
+++ b/src/modules/job-manager/job.c
@@ -14,12 +14,22 @@
 #include <stdlib.h>
 #include <flux/core.h>
 #include <jansson.h>
-#include <czmq.h>
+#include <assert.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libutil/grudgeset.h"
+#include "src/common/libutil/jpath.h"
+#include "src/common/libutil/aux.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
 
 #include "job.h"
 #include "event.h"
 
-#include "src/common/libeventlog/eventlog.h"
+#define EVENTS_BITMAP_SIZE 64
+
+static void subscribers_destroy (struct job *job);
 
 void job_decref (struct job *job)
 {
@@ -27,6 +37,15 @@ void job_decref (struct job *job)
         int saved_errno = errno;
         json_decref (job->end_event);
         flux_msg_decref (job->waiter);
+        json_decref (job->jobspec_redacted);
+        json_decref (job->R_redacted);
+        json_decref (job->eventlog);
+        json_decref (job->annotations);
+        grudgeset_destroy (job->dependencies);
+        subscribers_destroy (job);
+        free (job->events);
+        aux_destroy (&job->aux);
+        json_decref (job->event_queue);
         free (job);
         errno = saved_errno;
     }
@@ -40,51 +59,256 @@ struct job *job_incref (struct job *job)
     return job;
 }
 
-struct job *job_create (void)
+static struct job *job_alloc (void)
 {
     struct job *job;
 
     if (!(job = calloc (1, sizeof (*job))))
         return NULL;
+    if (!(job->events = bitmap_alloc0 (EVENTS_BITMAP_SIZE)))
+        goto error;
     job->refcount = 1;
     job->userid = FLUX_USERID_UNKNOWN;
-    job->priority = FLUX_JOB_PRIORITY_DEFAULT;
-    job->state = FLUX_JOB_NEW;
+    job->urgency = FLUX_JOB_URGENCY_DEFAULT;
+    job->priority = -1;
+    job->state = FLUX_JOB_STATE_NEW;
+    if (!(job->event_queue = json_array ())) {
+        errno = ENOMEM;
+        goto error;
+    }
     return job;
+error:
+    job_decref (job);
+    return NULL;
 }
 
-struct job *job_create_from_eventlog (flux_jobid_t id, const char *s)
+struct job *job_create (void)
+{
+    struct job *job;
+
+    if (!(job = job_alloc ()))
+        return NULL;
+    if (!(job->eventlog = json_array ())) {
+        errno = ENOMEM;
+        job_decref (job);
+        return NULL;
+    }
+    return job;
+}
+
+int job_dependency_count (struct job *job)
+{
+    return grudgeset_size (job->dependencies);
+}
+
+int job_dependency_add (struct job *job, const char *description)
+{
+    assert (job->state == FLUX_JOB_STATE_NEW
+            || job->state == FLUX_JOB_STATE_DEPEND);
+    if (grudgeset_add (&job->dependencies, description) < 0
+        && errno != EEXIST)
+        return -1;
+    return job_dependency_count (job);
+}
+
+int job_dependency_remove (struct job *job, const char *description)
+{
+    return grudgeset_remove (job->dependencies, description);
+}
+
+static int job_flag_set_internal (struct job *job,
+                                  const char *flag,
+                                  bool dry_run)
+{
+   if (streq (flag, "alloc-bypass")) {
+        if (!dry_run)
+            job->alloc_bypass = 1;
+    }
+    else if (streq (flag, "debug")) {
+        if (!dry_run)
+            job->flags |= FLUX_JOB_DEBUG;
+    }
+    else if (streq (flag, "immutable")) {
+        if (!dry_run)
+            job->immutable = 1;
+    }
+    else {
+        errno = EINVAL;
+        return -1;
+    }
+    return 0;
+}
+
+int job_flag_set (struct job *job, const char *flag)
+{
+    return job_flag_set_internal (job, flag, false);
+}
+
+bool job_flag_valid (struct job *job, const char *flag)
+{
+    if (job_flag_set_internal (job, flag, true) < 0)
+        return false;
+    return true;
+}
+
+int job_aux_set (struct job *job,
+                 const char *name,
+                 void *val,
+                 flux_free_f destroy)
+{
+    return aux_set (&job->aux, name, val, destroy);
+}
+
+void *job_aux_get (struct job *job, const char *name)
+{
+    return aux_get (job->aux, name);
+}
+
+void job_aux_delete (struct job *job, const void *val)
+{
+    aux_delete (&job->aux, val);
+}
+
+void job_aux_destroy (struct job *job)
+{
+    aux_destroy (&job->aux);
+}
+
+static int jobspec_redacted_parse_queue (struct job *job)
+{
+    if (job->jobspec_redacted) {
+        /* unit tests assume empty jobspec legal, so all fields
+         * optional
+         */
+        if (json_unpack (job->jobspec_redacted,
+                         "{s?{s?{s?s}}}",
+                         "attributes",
+                           "system",
+                             "queue", &job->queue) < 0) {
+            errno = EINVAL;
+            return -1;
+        }
+    }
+    return 0;
+}
+
+struct job *job_create_from_eventlog (flux_jobid_t id,
+                                      const char *eventlog,
+                                      const char *jobspec,
+                                      const char *R,
+                                      flux_error_t *error)
 {
     struct job *job;
-    json_t *a = NULL;
     size_t index;
     json_t *event;
+    int version = -1; // invalid
 
-    if (!(job = job_create ()))
+    if (!(job = job_alloc()))
         return NULL;
     job->id = id;
 
-    if (!(a = eventlog_decode (s)))
+    if (!(job->jobspec_redacted = json_loads (jobspec, 0, NULL))) {
+        errprintf (error, "failed to decode jobspec");
+        goto inval;
+    }
+    jpath_del (job->jobspec_redacted, "attributes.system.environment");
+
+    if (jobspec_redacted_parse_queue (job) < 0) {
+        errprintf (error, "failed to decode jobspec queue");
+        goto inval;
+    }
+
+    if (R) {
+        if (!(job->R_redacted = json_loads (R, 0, NULL))) {
+            errprintf (error, "failed to decode R");
+            goto inval;
+        }
+        (void)json_object_del (job->R_redacted, "scheduling");
+    }
+
+    if (!(job->eventlog = eventlog_decode (eventlog))) {
+        errprintf (error, "failed to decode eventlog");
         goto error;
+    }
+
+    json_array_foreach (job->eventlog, index, event) {
+        const char *name = "unknown";
+        json_t *context;
 
-    json_array_foreach (a, index, event) {
-        if (event_job_update (job, event) < 0)
+        if (index == 0) {
+            if (eventlog_entry_parse (event, NULL, &name, &context) < 0) {
+                errprintf (error, "eventlog parse error on line %zu", index);
+                goto error;
+            }
+            if (!streq (name, "submit")) {
+                errprintf (error, "first event is %s not submit", name);
+                goto inval;
+            }
+            /* For now, support flux-core versions prior to 0.41.1 that don't
+             * have a submit version attr, as is now required by RFC 21.
+             * Allow version=-1 to pass through for work around below.
+             */
+            (void)json_unpack (context, "{s?i}", "version", &version);
+            if (version != -1 && version != 1) {
+                errprintf (error, "eventlog v%d is unsupported", version);
+                goto inval;
+            }
+        }
+
+        if (event_job_update (job, event) < 0) {
+            errprintf (error, "could not apply %s", name);
             goto error;
+        }
+
+        /* Work around flux-framework/flux-core#4398.
+         * "submit" used to transition NEW->DEPEND in unversioned eventlog,
+         * but as of version 1, "validate" is required to transition.  Allow
+         * old jobs to be ingested from the KVS by flux-core 0.41.1+.
+         */
+        if (index == 0 && version == -1)
+            job->state = FLUX_JOB_STATE_DEPEND;
     }
 
-    if (job->state == FLUX_JOB_NEW)
+    if (job->state == FLUX_JOB_STATE_NEW) {
+        errprintf (error,
+                   "job state (%s) is invalid after replay",
+                   flux_job_statetostr (job->state, "L"));
         goto inval;
+    }
 
-    json_decref (a);
     return job;
 inval:
     errno = EINVAL;
 error:
     job_decref (job);
-    json_decref (a);
     return NULL;
 }
 
+struct job *job_create_from_json (json_t *o)
+{
+    struct job *job;
+
+    if (!(job = job_create ()))
+        return NULL;
+    if (json_unpack (o,
+                     "{s:I s:i s:i s:f s:i s:O}",
+                     "id", &job->id,
+                     "urgency", &job->urgency,
+                     "userid", &job->userid,
+                     "t_submit", &job->t_submit,
+                     "flags", &job->flags,
+                     "jobspec", &job->jobspec_redacted) < 0) {
+        errno = EPROTO;
+        job_decref (job);
+        return NULL;
+    }
+    if (jobspec_redacted_parse_queue (job) < 0) {
+        job_decref (job);
+        return NULL;
+    }
+    return job;
+}
+
 #define NUMCMP(a,b) ((a)==(b)?0:((a)<(b)?-1:1))
 
 /* Decref a job.
@@ -106,20 +330,436 @@ void *job_duplicator (const void *item)
     return job_incref ((struct job *)item);
 }
 
-/* Compare jobs, ordering by (1) priority, (2) t_submit.
+/* Compare jobs, ordering by (1) priority, (2) job id.
  * N.B. zlistx_comparator_fn signature
  */
-int job_comparator (const void *a1, const void *a2)
+int job_priority_comparator (const void *a1, const void *a2)
 {
     const struct job *j1 = a1;
     const struct job *j2 = a2;
     int rc;
 
     if ((rc = (-1)*NUMCMP (j1->priority, j2->priority)) == 0)
-        rc = NUMCMP (j1->t_submit, j2->t_submit);
+        rc = NUMCMP (j1->id, j2->id);
     return rc;
 }
 
+/* Compare inactive jobs, ordering by the time they became inactive.
+ * N.B. zlistx_comparator_fn signature
+ */
+int job_age_comparator (const void *a1, const void *a2)
+{
+    const struct job *j1 = a1;
+    const struct job *j2 = a2;
+
+    return NUMCMP (j1->t_clean, j2->t_clean);
+}
+
+/*  This structure is stashed in a plugin which has subscribed to
+ *   job events. The reference to the plugin itself is required so
+ *   that the aux_item destructor can remove the plugin itself from
+ *   the subscribers list of any active jobs to which it had an
+ *   active subscription. (See plugin_job_subscriptions_destroy()).
+ */
+struct plugin_job_subscriptions {
+    flux_plugin_t *p;
+    zlistx_t *jobs;
+};
+
+static struct plugin_job_subscriptions *
+plugin_job_subscriptions_create (flux_plugin_t *p)
+{
+    struct plugin_job_subscriptions *ps = malloc (sizeof (*ps));
+    if (!ps || !(ps->jobs = zlistx_new ())) {
+        free (ps);
+        return NULL;
+    }
+    ps->p = p;
+    return ps;
+}
+
+static void plugin_job_subscriptions_destroy (void *arg)
+{
+    struct plugin_job_subscriptions *ps = arg;
+    if (ps) {
+        struct job *job = zlistx_first (ps->jobs);
+        while (job) {
+            job_events_unsubscribe (job, ps->p);
+            job = zlistx_next (ps->jobs);
+        }
+        zlistx_destroy (&ps->jobs);
+        free (ps);
+    }
+}
+
+/*  Clear the subscription of a plugin from a job
+ */
+static void plugin_clear_subscription (flux_plugin_t *p, struct job *job)
+{
+    struct plugin_job_subscriptions *ps;
+    if ((ps = flux_plugin_aux_get (p, "flux::job-subscriptions"))) {
+        void *handle = zlistx_find (ps->jobs, job);
+        if (handle)
+            zlistx_delete (ps->jobs, handle);
+    }
+}
+
+/*  Unsubscribe plugin p from job events.
+ *
+ *  Clear plugin p from this job's list of subscribers as well as
+ *   the job from the plugin's list of subscriptions.
+ */
+void job_events_unsubscribe (struct job *job, flux_plugin_t *p)
+{
+    void *handle;
+    if (job->subscribers
+        && (handle = zlistx_find (job->subscribers, p))) {
+
+        /*  Remove plugin from job */
+        zlistx_delete (job->subscribers, handle);
+
+        /*  Remove job from plugin */
+        plugin_clear_subscription (p, job);
+    }
+}
+
+/*  Destroy this job's plugin subscribers list.
+ *
+ *  This can't be done with a zlist destructor because we need
+ *   a reference to the struct job during destruction in order to remove
+ *   the job from all subscribed plugins subscription lists.
+ *
+ */
+static void subscribers_destroy (struct job *job)
+{
+    if (job->subscribers) {
+        flux_plugin_t *p = zlistx_first (job->subscribers);
+        while (p) {
+            plugin_clear_subscription (p, job);
+            p = zlistx_next (job->subscribers);
+        }
+    }
+    zlistx_destroy (&job->subscribers);
+}
+
+/*  Add a plugin to a job's subscribers list.
+ */
+int job_events_subscribe (struct job *job, flux_plugin_t *p)
+{
+    struct plugin_job_subscriptions *ps;
+
+    /*  Create a subscribers list for the job if it does not already exist
+     */
+    if (!job->subscribers && !(job->subscribers = zlistx_new())) {
+        errno = ENOMEM;
+        return -1;
+    }
+
+    /*  Create a subscriptions list for the plugin if it does not already
+     *   exist. Ensure that when the plugin is unloaded it is unsubscribed
+     *   from all current job events subscriptions.
+     */
+    if (!(ps = flux_plugin_aux_get (p, "flux::job-subscriptions"))) {
+        if (!(ps = plugin_job_subscriptions_create (p))
+            || flux_plugin_aux_set (p,
+                                    "flux::job-subscriptions",
+                                    ps,
+                                    plugin_job_subscriptions_destroy) < 0) {
+            plugin_job_subscriptions_destroy (ps);
+            errno = ENOMEM;
+            return -1;
+        }
+    }
+
+    /*  Add the job to the plugin subscriptions list, and the plugin to the
+     *   the job's subscribers list. If either fails, attempt to clean up.
+     *   This may leave empty lists on the job and plugin, but those will
+     *   be ultimately destroyed when the job is inactive or theplugin is
+     *   unloaded.
+     */
+    if (!zlistx_add_end (ps->jobs, job)
+        || !zlistx_add_end (job->subscribers, p)) {
+        plugin_clear_subscription (p, job);
+        errno = ENOMEM;
+        return -1;
+    }
+
+    return 0;
+}
+
+int job_event_id_set (struct job *job, int id)
+{
+    if (id < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (id >= EVENTS_BITMAP_SIZE) {
+        errno = ENOSPC;
+        return -1;
+    }
+    bitmap_set_bit (job->events, id);
+    return 0;
+}
+
+int job_event_id_test (struct job *job, int id)
+{
+    if (id < 0 || id >= EVENTS_BITMAP_SIZE) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (bitmap_test_bit (job->events, id))
+        return 1;
+    return 0;
+}
+
+int job_event_enqueue (struct job *job, int flags, json_t *entry)
+{
+    json_t *wrap;
+
+    if (job->eventlog_readonly) {
+        errno = EROFS;
+        return -1;
+    }
+    if (!(wrap = json_pack ("{s:i s:O}",
+                            "flags", flags,
+                            "entry", entry))
+        || json_array_append_new (job->event_queue, wrap) < 0) {
+        json_decref (wrap);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+int job_event_peek (struct job *job, int *flagsp, json_t **entryp)
+{
+    json_t *wrap;
+    json_t *entry;
+    int flags;
+
+    if (!(wrap = json_array_get (job->event_queue, 0))) {
+        errno = ENOENT;
+        return -1; // queue empty
+    }
+    if (json_unpack (wrap,
+                     "{s:i s:o}",
+                     "flags", &flags,
+                     "entry", &entry) < 0) {
+        errno = EPROTO;
+        return -1;
+    }
+    if (entryp)
+        *entryp = entry;
+    if (flagsp)
+        *flagsp = flags;
+    return 0;
+}
+
+int job_event_dequeue (struct job *job, int *flagsp, json_t **entryp)
+{
+    json_t *entry;
+    int flags;
+
+    if (job_event_peek (job, &flags, &entry) < 0)
+        return -1;
+    if (entryp)
+        *entryp = json_incref (entry);
+    if (flagsp)
+        *flagsp = flags;
+    json_array_remove (job->event_queue, 0);
+    return 0;
+}
+
+bool job_event_is_queued (struct job *job, const char *name)
+{
+    size_t index;
+    json_t *wrap;
+    json_t *entry;
+    const char *entry_name;
+
+    json_array_foreach (job->event_queue, index, wrap) {
+        if (json_unpack (wrap, "{s:o}", "entry", &entry) == 0
+            && eventlog_entry_parse (entry, NULL, &entry_name, NULL) == 0
+            && streq (entry_name, name))
+            return true;
+    }
+    return false;
+}
+
+const char *job_event_queue_print (struct job *job, char *buf, int size)
+{
+    size_t index;
+    json_t *wrap;
+    json_t *entry;
+    const char *name;
+    size_t used = 0;
+    size_t n;
+
+    buf[0] = '\0';
+    json_array_foreach (job->event_queue, index, wrap) {
+        if (json_unpack (wrap, "{s:o}", "entry", &entry) < 0
+            || eventlog_entry_parse (entry, NULL, &name, NULL) < 0)
+            name = "unknown";
+        n = snprintf (buf + used,
+                      size - used,
+                      "%s%s",
+                      index > 0 ? "/" : "",
+                      name);
+        if (n >= size - used)
+            break;
+        used += n;
+    }
+    return buf;
+}
+
+bool validate_jobspec_updates (json_t *updates)
+{
+    const char *key;
+    json_t *entry;
+    json_object_foreach (updates, key, entry) {
+        if (!streq (key, "attributes")
+            && !strstarts (key, "attributes.")
+            && !streq (key, "resources")
+            && !strstarts (key, "resources.")
+            && !streq (key, "tasks")
+            && !strstarts (key, "tasks."))
+            return false;
+    }
+    return true;
+}
+
+static int jobspec_apply_updates (json_t *jobspec, json_t *updates)
+{
+    const char *path;
+    json_t *val;
+
+    if (!jobspec) {
+        errno = EINVAL;
+        return -1;
+    }
+    json_object_foreach (updates, path, val) {
+        if (jpath_set (jobspec, path, val) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+int job_apply_jobspec_updates (struct job *job, json_t *updates)
+{
+    if (jobspec_apply_updates (job->jobspec_redacted, updates) < 0
+        || jobspec_redacted_parse_queue (job) < 0)
+        return -1;
+    return 0;
+}
+
+json_t *job_jobspec_with_updates (struct job *job, json_t *updates)
+{
+    json_t *jobspec;
+
+    if (!job->jobspec_redacted) {
+        errno = EAGAIN;
+        return NULL;
+    }
+    if (!(jobspec = json_deep_copy (job->jobspec_redacted))) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    if (jobspec_apply_updates (jobspec, updates) < 0) {
+        int saved_errno = errno;
+        json_decref (jobspec);
+        errno = saved_errno;
+        return NULL;
+    }
+    return jobspec;
+}
+
+int job_apply_resource_updates (struct job *job, json_t *updates)
+{
+    json_t *val;
+
+    if (!job->R_redacted) {
+        errno = EAGAIN;
+        return -1;
+    }
+    /* Currently only an expiration key is allowed in a resource-update
+     * event. Return an error with errno=EINVAL if there is more than one
+     * key in the updates object or the existing key is not 'expiration':
+     */
+    if (json_object_size (updates) != 1
+        || !(val = json_object_get (updates, "expiration"))
+        || !json_is_number (val)
+        || json_number_value (val) < 0.) {
+        errno = EINVAL;
+        return -1;
+    }
+    /*  Update redacted copy of R in place.
+     */
+    return jpath_set (job->R_redacted, "execution.expiration", val);
+}
+
+zlistx_t *job_priority_queue_create (void)
+{
+    zlistx_t *l;
+
+    if (!(l = zlistx_new())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    zlistx_set_destructor (l, job_destructor);
+    zlistx_set_comparator (l, job_priority_comparator);
+    zlistx_set_duplicator (l, job_duplicator);
+    return l;
+}
+
+int job_priority_queue_insert (zlistx_t *l, struct job *job)
+{
+    bool fwd = job->priority > (FLUX_JOB_PRIORITY_MAX / 2);
+    if (job->handle) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(job->handle = zlistx_insert (l, job, fwd))) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+int job_priority_queue_delete (zlistx_t *l, struct job *job)
+{
+    if (!job->handle) {
+        errno = EINVAL;
+        return -1;
+    }
+    (void)zlistx_delete (l, job->handle);
+    job->handle = NULL;
+    return 0;
+}
+
+void job_priority_queue_reorder (zlistx_t *l, struct job *job)
+{
+    if (job->handle) {
+        bool fwd = job->priority > (FLUX_JOB_PRIORITY_MAX / 2);
+        zlistx_reorder (l, job->handle, fwd);
+    }
+}
+
+/*  N.B.: zlistx_sort() invalidates all list handles since
+ *   the sort swaps contents of nodes, not the nodes themselves.
+ *   Therefore, job handles into the list must be re-acquired here.
+ */
+void job_priority_queue_sort (zlistx_t *l)
+{
+    struct job *job;
+
+    zlistx_sort (l);
+    job = zlistx_first (l);
+    while (job) {
+        job->handle = zlistx_cursor (l);
+        job = zlistx_next (l);
+    }
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/job-manager/job.h b/src/modules/job-manager/job.h
index 04b87357c416..60b2f17b606a 100644
--- a/src/modules/job-manager/job.h
+++ b/src/modules/job-manager/job.h
@@ -12,28 +12,57 @@
 #define _FLUX_JOB_MANAGER_JOB_H
 
 #include <stdint.h>
-#include <czmq.h>
 #include <jansson.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libjob/job.h"
+#include "src/common/libutil/grudgeset.h"
+#include "src/common/libflux/plugin.h"
+#include "ccan/bitmap/bitmap.h"
 
 struct job {
     flux_jobid_t id;
     uint32_t userid;
-    int priority;
+    int urgency;
+    int64_t priority;
     double t_submit;
+    const char *queue;
     int flags;
+    json_t *jobspec_redacted;
+    json_t *R_redacted;
+    json_t *eventlog;
     flux_job_state_t state;
+    json_t *event_queue;
     json_t *end_event;      // event that caused transition to CLEANUP state
     const flux_msg_t *waiter; // flux_job_wait() request
+    double t_clean;
 
+    uint8_t depend_posted:1;// depend event already posted
     uint8_t alloc_queued:1; // queued for alloc, but alloc request not sent
     uint8_t alloc_pending:1;// alloc request sent to sched
-    uint8_t free_pending:1; // free request sent to sched
+    uint8_t alloc_bypass:1; // alloc bypass enabled
+    uint8_t free_posted:1;  // free event already posted
     uint8_t has_resources:1;
     uint8_t start_pending:1;// start request sent to job-exec
+    uint8_t reattach:1;
+    uint8_t eventlog_readonly:1;// job is inactive or invalid
+    uint8_t hold_events:1;  // queue events instead of posting immediately
+    uint8_t immutable:1;    // user job updates are disabled
+
+    uint8_t perilog_active; // if nonzero, prolog/epilog active
+
+    json_t *annotations;
+
+    struct grudgeset *dependencies;
+
+    zlistx_t *subscribers;  // list of plugins subscribed to all job events
+
+    struct bitmap *events;  // set of events by id posted to this job
 
     void *handle;           // zlistx_t handle
     int refcount;           // private to job.c
+
+    struct aux_item *aux;
 };
 
 void job_decref (struct job *job);
@@ -41,14 +70,89 @@ struct job *job_incref (struct job *job);
 
 struct job *job_create (void);
 
-struct job *job_create_from_eventlog (flux_jobid_t id, const char *eventlog);
+struct job *job_create_from_eventlog (flux_jobid_t id,
+                                      const char *eventlog,
+                                      const char *jobspec,
+                                      const char *R,
+                                      flux_error_t *error);
+struct job *job_create_from_json (json_t *o);
+
+/* N.B. aux items are destroyed when job transitions to inactive.
+ */
+int job_aux_set (struct job *job,
+                 const char *name,
+                 void *val,
+                 flux_free_f destroy);
+void *job_aux_get (struct job *job, const char *name);
+void job_aux_delete (struct job *job, const void *val);
+void job_aux_destroy (struct job *job);
 
 /* Helpers for maintaining czmq containers of 'struct job'.
- * The comparator sorts by (1) priority, then (2) t_submit.
+ * job_priority_comparator sorts by (1) priority, then (2) jobid.
+ * job_age_comparator sorts by the time the job became inactive.
  */
 void job_destructor (void **item);
 void *job_duplicator (const void *item);
-int job_comparator (const void *a1, const void *a2);
+int job_priority_comparator (const void *a1, const void *a2);
+int job_age_comparator (const void *a1, const void *a2);
+
+/*  Add and remove job dependencies
+ */
+int job_dependency_add (struct job *job, const char *description);
+int job_dependency_remove (struct job *job, const char *description);
+int job_dependency_count (struct job *job);
+
+/*  Set a limited set of flags by name on job
+ */
+int job_flag_set (struct job *job, const char *flag);
+
+/*  Test if flag name 'flag' is a valid job flag
+ */
+bool job_flag_valid (struct job *job, const char *flag);
+
+/*  Allow a flux_plugin_t to subscribe to all job events
+ */
+int job_events_subscribe (struct job *job, flux_plugin_t *p);
+
+void job_events_unsubscribe (struct job *job, flux_plugin_t *p);
+
+/*  Add and test for events posted to jobs by a global event id.
+ *  (Event names are translated to id by the event class)
+ */
+int job_event_id_set (struct job *job, int id);
+int job_event_id_test (struct job *job, int id);
+
+/*  Enqeue/dequeue event from job's event queue.
+ */
+int job_event_enqueue (struct job *job, int flags, json_t *entry);
+int job_event_dequeue (struct job *job, int *flagsp, json_t **entryp);
+int job_event_peek (struct job *job, int *flagsp, json_t **entryp);
+bool job_event_is_queued (struct job *job, const char *name);
+const char *job_event_queue_print (struct job *job, char *buf, int size);
+
+/*  Validate updates as valid RFC 21 jobspec-update event context:
+ */
+bool validate_jobspec_updates (json_t *updates);
+
+/*  Apply updates to jobspec
+ */
+int job_apply_jobspec_updates (struct job *job, json_t *updates);
+
+/*  Return a copy of the jobspec for 'job' with 'updates' applied.
+ */
+json_t *job_jobspec_with_updates (struct job *job, json_t *updates);
+
+/* Apply resource updates to redacted copy of R
+ */
+int job_apply_resource_updates (struct job *job, json_t *updates);
+
+/* job->handle tracks list position.
+ */
+zlistx_t *job_priority_queue_create (void);
+int job_priority_queue_insert (zlistx_t *l, struct job *job);
+int job_priority_queue_delete (zlistx_t *l, struct job *job);
+void job_priority_queue_reorder (zlistx_t *l, struct job *job);
+void job_priority_queue_sort (zlistx_t *l);
 
 #endif /* _FLUX_JOB_MANAGER_JOB_H */
 
diff --git a/src/modules/job-manager/jobtap-internal.h b/src/modules/job-manager/jobtap-internal.h
new file mode 100644
index 000000000000..529ba231d489
--- /dev/null
+++ b/src/modules/job-manager/jobtap-internal.h
@@ -0,0 +1,147 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_JOBTAP_H
+#define _FLUX_JOB_MANAGER_JOBTAP_H
+
+#include "src/common/libflux/types.h" /* flux_error_t */
+
+#include "job.h"
+#include "job-manager.h"
+
+struct jobtap * jobtap_create (struct job_manager *ctx);
+
+void jobtap_destroy (struct jobtap *jobtap);
+
+/*  Call the jobtap plugin with topic string `topic`.
+ *   `fmt` is jansson-style pack arguments to add the plugin args
+ */
+int jobtap_call (struct jobtap *jobtap,
+                 struct job *job,
+                 const char *topic,
+                 const char *fmt,
+                 ...);
+
+/*  Jobtap call specific for getting a new priority from the jobtap
+ *   plugin if available. The priority will be returned in `pprio` if
+ *   it was set.
+ */
+int jobtap_get_priority (struct jobtap *jobtap,
+                         struct job *job,
+                         int64_t *pprio);
+
+
+/*  Jobtap call specific to validating a job during submission. If the
+ *   plugin returns failure from this callback the job will be rejected
+ *   with an optional error message passed back in `errp`.
+ */
+int jobtap_validate (struct jobtap *jobtap,
+                     struct job *job,
+                     char **errp);
+
+int jobtap_call_create (struct jobtap *jobtap,
+                        struct job *job,
+                        char **errp);
+
+/*  Jobtap call to iterate attributes.system.dependencies dictionary
+ *   and call job.dependency.<schema> for each entry.
+ *
+ *  If there is no plugin registered to handle a given scheme, then
+ *   if raise_exception is true a nonfatal job exception is raised,
+ *   otherwise an error is returned. A plugin which handles a given schema
+ *   may also reject the job if the dependency stanza has errors.
+ */
+int jobtap_check_dependencies (struct jobtap *jobtap,
+                               struct job *job,
+                               bool raise_exception,
+                               char **errp);
+
+/*  Call `job.update.<key>` callback to verify that a jobspec update of
+ *  'key' to 'value' is allowed. The flux_msg_cred parameter should be set
+ *  to the credentials of the original requestor.
+ *
+ *  Returns an error with 'errp' (Caller must free) set if no plugin is
+ *  registered to handle updates of 'key', or if the callback returned an
+ *  error.
+ *
+ *  If the update needs further validation via `job.validate`, then
+ *  needs_validation will be set nonzero. The caller should be sure to pass
+ *  the updated jobspec to `job.validate` before posting updates to the job
+ *  eventlog.
+ *
+ *  If the update requires a feasibility check with the scheduler, then
+ *  require_feasibility will be set nonzero. The caller should attempt to
+ *  request a feasibility check before applying updates.
+ */
+int jobtap_job_update (struct jobtap *jobtap,
+                       struct flux_msg_cred cred,
+                       struct job *job,
+                       const char *key,
+                       json_t *value,
+                       int *needs_validation,
+                       int *needs_feasibility,
+                       json_t **updates,
+                       char **errp);
+
+/*  Call the `job.validate` plugin stack, but using an updated jobspec by
+ *  applying 'updates' to 'job'.
+ *
+ *  If validation fails, then this function will return -1 with the error
+ *  set in 'errp' (Caller must free).
+ */
+int jobtap_validate_updates (struct jobtap *jobtap,
+                             struct job *job,
+                             json_t *updates,
+                             char **errp);
+
+/*  Load a new jobtap from `path`. Path may start with `builtin.` to
+ *   attempt to load one of the builtin jobtap plugins.
+ */
+flux_plugin_t * jobtap_load (struct jobtap *jobtap,
+                             const char *path,
+                             json_t *conf,
+                             flux_error_t *errp);
+
+typedef int (*jobtap_builtin_f) (flux_plugin_t *p, void *arg);
+
+/*  Add a new jobtap builtin plugin.
+ *  Allows builtins to be created externally to the jobtap module.
+ */
+int jobtap_register_builtin (struct jobtap *jobtap,
+                             const char *name,
+                             jobtap_builtin_f init_cb,
+                             void *arg);
+
+/*  Job manager RPC handler for loading new jobtap plugins.
+ */
+void jobtap_handler (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg);
+
+/*  Job manager RPC handler for querying jobtap plugin data.
+ */
+void jobtap_query_handler (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg);
+
+int jobtap_notify_subscribers (struct jobtap *jobtap,
+                               struct job *job,
+                               const char *event_name,
+                               const char *fmt,
+                               ...);
+
+#endif /* _FLUX_JOB_MANAGER_JOBTAP_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/modules/job-manager/jobtap.c b/src/modules/job-manager/jobtap.c
new file mode 100644
index 000000000000..d690374ee427
--- /dev/null
+++ b/src/modules/job-manager/jobtap.c
@@ -0,0 +1,2679 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* jobtap.c - a job manager plugin interface
+ *
+ *  Maintains a list of one or more job manager plugins which
+ *   "tap" into job state transitions and/or events.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <unistd.h>
+#include <stdarg.h>
+#include <fnmatch.h>
+
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/iterators.h"
+#include "src/common/libutil/basename.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/aux.h"
+#include "src/common/libjob/idf58.h"
+#include "ccan/str/str.h"
+
+#include "annotate.h"
+#include "prioritize.h"
+#include "conf.h"
+#include "event.h"
+#include "raise.h"
+#include "jobtap.h"
+#include "jobtap-internal.h"
+
+#define FLUX_JOBTAP_PRIORITY_UNAVAIL INT64_C(-2)
+
+extern int priority_default_plugin_init (flux_plugin_t *p);
+extern int limit_job_size_plugin_init (flux_plugin_t *p);
+extern int limit_duration_plugin_init (flux_plugin_t *p);
+extern int after_plugin_init (flux_plugin_t *p);
+extern int begin_time_plugin_init (flux_plugin_t *p);
+extern int validate_duration_plugin_init (flux_plugin_t *p);
+extern int update_duration_plugin_init (flux_plugin_t *p);
+extern int history_plugin_init (flux_plugin_t *p);
+extern int post_event_init (flux_plugin_t *p);
+
+struct jobtap_builtin {
+    const char *name;
+    flux_plugin_init_f init;
+};
+
+struct jobtap_builtin_ex {
+    char *name;
+    jobtap_builtin_f init_cb;
+    void *arg;
+};
+
+static struct jobtap_builtin jobtap_builtins [] = {
+    { ".priority-default", priority_default_plugin_init },
+    { ".limit-job-size", limit_job_size_plugin_init },
+    { ".limit-duration", limit_duration_plugin_init },
+    { ".dependency-after", after_plugin_init },
+    { ".begin-time", &begin_time_plugin_init },
+    { ".validate-duration", &validate_duration_plugin_init },
+    { ".update-duration", &update_duration_plugin_init },
+    { ".history", &history_plugin_init },
+    { ".post-event", &post_event_init },
+    { 0 },
+};
+
+struct jobtap {
+    struct job_manager *ctx;
+    char *searchpath;
+    zlistx_t *builtins_ex;
+    zlistx_t *plugins;
+    zhashx_t *plugins_byuuid;
+    zlistx_t *jobstack;
+    json_t *jobspec_update;
+    bool configured;
+};
+
+struct dependency {
+    bool add;
+    char *description;
+};
+
+static int jobtap_job_raise (struct jobtap *jobtap,
+                             struct job *job,
+                             const char *type,
+                             int severity,
+                             const char *fmt, ...);
+
+static int dependencies_unpack (struct jobtap * jobtap,
+                                struct job * job,
+                                char **errp,
+                                json_t **resultp);
+
+static int jobtap_check_dependency (struct jobtap *jobtap,
+                                    flux_plugin_t *p,
+                                    struct job *job,
+                                    flux_plugin_arg_t *args,
+                                    int index,
+                                    json_t *entry,
+                                    char **errp);
+
+static struct aux_wrap *aux_wrap_get (flux_plugin_t *p,
+                                      struct job *job,
+                                      bool create);
+
+/*  zlistx_t plugin destructor */
+static void plugin_destroy (void **item)
+{
+    if (item) {
+        flux_plugin_t *p = *item;
+        flux_plugin_destroy (p);
+        *item = NULL;
+    }
+}
+
+static void jobtap_builtin_ex_destroy (struct jobtap_builtin_ex *ex)
+{
+    if (ex) {
+        int saved_errno = errno;
+        free (ex->name);
+        free (ex);
+        errno = saved_errno;
+    }
+}
+
+/*  zlistx_t jobtap_builtin_ex destructor */
+static void builtin_ex_destructor (void **item)
+{
+    if (item) {
+        struct jobtap_builtin_ex *ex = *item;
+        jobtap_builtin_ex_destroy (ex);
+        *item = NULL;
+    }
+}
+
+struct jobtap_builtin_ex * jobtap_builtin_ex_create (const char *name,
+                                                     jobtap_builtin_f init_cb,
+                                                     void *arg)
+{
+    struct jobtap_builtin_ex *ex = calloc (1, sizeof (*ex));
+    if (!ex || !(ex->name = strdup (name)))
+        goto error;
+    ex->init_cb = init_cb;
+    ex->arg = arg;
+    return ex;
+error:
+    jobtap_builtin_ex_destroy (ex);
+    return NULL;
+}
+
+static const char *jobtap_plugin_name (flux_plugin_t *p)
+{
+    const char *name;
+    if (!p)
+        return "none";
+    if ((name = flux_plugin_aux_get (p, "jobtap::basename"))
+        || (name = flux_plugin_get_name (p)))
+        return name;
+    return "unknown";
+}
+
+static flux_plugin_arg_t *jobtap_args_create (struct jobtap *jobtap,
+                                              struct job *job)
+{
+    flux_plugin_arg_t *args = flux_plugin_arg_create ();
+    if (!args)
+        return NULL;
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_IN,
+                              "{s:O s:I s:I s:i s:i s:I s:f}",
+                              "jobspec", job->jobspec_redacted,
+                              "id", job->id,
+                              "userid", (json_int_t) job->userid,
+                              "urgency", job->urgency,
+                              "state", job->state,
+                              "priority", job->priority,
+                              "t_submit", job->t_submit) < 0)
+        goto error;
+    if (job->R_redacted) {
+        if (flux_plugin_arg_pack (args,
+                                  FLUX_PLUGIN_ARG_IN,
+                                  "{s:O}",
+                                  "R", job->R_redacted) < 0)
+            goto error;
+    }
+    /*
+     *  Always start with empty OUT args. This allows unpack of OUT
+     *   args to work without error, even if plugin does not set any
+     *   OUT args.
+     */
+    if (flux_plugin_arg_set (args, FLUX_PLUGIN_ARG_OUT, "{}") < 0)
+        goto error;
+
+    return args;
+error:
+    flux_plugin_arg_destroy (args);
+    return NULL;
+}
+
+static flux_plugin_arg_t *jobtap_args_vcreate (struct jobtap *jobtap,
+                                               struct job *job,
+                                               const char *fmt,
+                                               va_list ap)
+{
+    flux_plugin_arg_t *args = jobtap_args_create (jobtap, job);
+    if (!args)
+        return NULL;
+
+    if (fmt
+        && flux_plugin_arg_vpack (args,
+                                  FLUX_PLUGIN_ARG_IN,
+                                  fmt, ap) < 0)
+        goto error;
+    return args;
+error:
+    flux_plugin_arg_destroy (args);
+    return NULL;
+}
+
+
+static int plugin_check_dependencies (struct jobtap *jobtap,
+                                      flux_plugin_t *p,
+                                      struct job *job,
+                                      flux_plugin_arg_t *args)
+{
+    json_t *dependencies = NULL;
+    json_t *entry = NULL;
+    size_t index;
+    char *error;
+
+    if (dependencies_unpack (jobtap, job, &error, &dependencies) < 0) {
+        flux_log (jobtap->ctx->h,
+                  LOG_ERR,
+                  "id=%s: plugin_register_dependencies: %s",
+                  idf58 (job->id),
+                  error);
+        free (error);
+        return -1;
+    }
+
+    if (dependencies == NULL)
+        return 0;
+
+    json_array_foreach (dependencies, index, entry) {
+        char *error;
+        if (jobtap_check_dependency (jobtap,
+                                     p,
+                                     job,
+                                     args,
+                                     index,
+                                     entry,
+                                     &error) < 0) {
+            flux_log (jobtap->ctx->h,
+                      LOG_ERR,
+                      "plugin_check_dependencies: %s", error);
+        }
+    }
+    return 0;
+}
+
+static struct job * current_job (struct jobtap *jobtap)
+{
+    return zlistx_head (jobtap->jobstack);
+}
+
+static int current_job_push (struct jobtap *jobtap, struct job *job)
+{
+    if (!zlistx_add_start (jobtap->jobstack, job)) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+static int current_job_pop (struct jobtap *jobtap)
+{
+    return zlistx_delete (jobtap->jobstack, NULL);
+}
+
+static flux_plugin_t * jobtap_load_plugin (struct jobtap *jobtap,
+                                           const char *path,
+                                           json_t *conf,
+                                           flux_error_t *errp)
+{
+    struct job_manager *ctx = jobtap->ctx;
+    flux_plugin_t *p = NULL;
+    flux_plugin_arg_t *args;
+    zlistx_t *jobs;
+    struct job *job;
+
+    if (!(p = jobtap_load (jobtap, path, conf, errp)))
+        goto error;
+
+    /*  Make plugin aware of all active jobs.
+     */
+    if (!(jobs = zhashx_values (ctx->active_jobs))) {
+        errprintf (errp, "zhashx_values() failed");
+        goto error;
+    }
+    job = zlistx_first (jobs);
+    while (job) {
+        if (current_job_push (jobtap, job) < 0) {
+            errprintf (errp, "Out of memory adding to jobtap jobstack");
+            goto error;
+        }
+        if (!(args = jobtap_args_create (jobtap, job))) {
+            errprintf (errp, "Failed to create args for job");
+            goto error;
+        }
+
+        /*  Notify this plugin of all jobs via `job.create` and `job.new`
+         *   callbacks.
+         */
+        (void) flux_plugin_call (p, "job.create", args);
+        (void) flux_plugin_call (p, "job.new", args);
+
+        /*  If job is in DEPEND state then there may be pending dependencies.
+         *  Notify plugin of the DEPEND state assuming it needs to create
+         *   some state in order to resolve the dependency.
+         */
+        if (job->state == FLUX_JOB_STATE_DEPEND) {
+            if (plugin_check_dependencies (jobtap, p, job, args) < 0)
+                errprintf (errp,
+                           "failed to check dependencies for job %s",
+                           idf58 (job->id));
+            (void) flux_plugin_call (p, "job.state.depend", args);
+        }
+
+        flux_plugin_arg_destroy (args);
+        if (current_job_pop (jobtap)) {
+            errprintf (errp, "Error popping current job off jobtap stack");
+            goto error;
+        }
+        job = zlistx_next (jobs);
+    }
+    zlistx_destroy (&jobs);
+
+    /* Now schedule reprioritize of all jobs
+     */
+    if (reprioritize_all (ctx) < 0) {
+        errprintf (errp,
+                   "%s loaded but unable to reprioritize jobs",
+                   jobtap_plugin_name (p));
+    }
+    return p;
+error:
+    flux_plugin_destroy (p);
+    return NULL;
+}
+
+static bool isa_glob (const char *s)
+{
+    if (strchr (s, '*') || strchr (s, '?') || strchr (s, '['))
+        return true;
+    return false;
+}
+
+static void jobtap_finalize (struct jobtap *jobtap, flux_plugin_t *p)
+{
+    zlistx_t *jobs;
+
+    if ((jobs = zhashx_values (jobtap->ctx->active_jobs))) {
+        struct job *job;
+
+        job = zlistx_first (jobs);
+        while (job) {
+            struct aux_wrap *wrap;
+
+            if ((wrap = aux_wrap_get (p, job, false)))
+                job_aux_delete (job, wrap);
+
+            job = zlistx_next (jobs);
+        }
+        zlistx_destroy (&jobs);
+    }
+}
+
+static int jobtap_remove (struct jobtap *jobtap,
+                          const char *arg,
+                          flux_error_t *errp)
+{
+    int count = 0;
+    bool isglob = isa_glob (arg);
+    bool all = streq (arg, "all");
+
+    flux_plugin_t *p = zlistx_first (jobtap->plugins);
+    while (p) {
+        const char *name = jobtap_plugin_name (p);
+        if ((all && name[0] != '.')
+            || (isglob && fnmatch (arg, name, FNM_PERIOD) == 0)
+            || streq (arg, name)) {
+            jobtap_finalize (jobtap, p);
+            zhashx_delete (jobtap->plugins_byuuid, flux_plugin_get_uuid (p));
+            zlistx_detach_cur (jobtap->plugins);
+            flux_plugin_destroy (p);
+            count++;
+        }
+        p = zlistx_next (jobtap->plugins);
+    }
+    if (count == 0 && !all) {
+        errno = ENOENT;
+        return errprintf (errp, "Failed to find plugin to remove");
+    }
+    return count;
+}
+
+static int jobtap_conf_entry (struct jobtap *jobtap,
+                              int index,
+                              json_t *entry,
+                              flux_error_t *errp)
+{
+    json_error_t json_err;
+    flux_error_t jobtap_err;
+    const char *load = NULL;
+    const char *remove = NULL;
+    json_t *conf = NULL;
+
+    if (json_unpack_ex (entry,
+                        &json_err,
+                        0,
+                        "{s?s s?o s?s}",
+                        "load", &load,
+                        "conf", &conf,
+                        "remove", &remove) < 0) {
+        return errprintf (errp,
+                          "[job-manager.plugins][%d]: %s",
+                          index,
+                          json_err.text);
+    }
+    if (remove && jobtap_remove (jobtap, remove, &jobtap_err) < 0) {
+        return errprintf (errp,
+                          "[job-manager.plugins][%d]: remove %s: %s",
+                          index,
+                          remove,
+                          jobtap_err.text);
+    }
+    if (load && !jobtap_load_plugin (jobtap, load, conf, &jobtap_err)) {
+        return errprintf (errp,
+                          "[job-manager.plugins][%d]: load: %s",
+                          index,
+                          jobtap_err.text);
+    }
+    return 0;
+}
+
+static int jobtap_call_conf_update (flux_plugin_t *p,
+                                    const flux_conf_t *conf,
+                                    flux_error_t *errp)
+{
+    const char *name = flux_plugin_get_name (p);
+    flux_plugin_arg_t *args;
+    json_t *o;
+
+    if (flux_conf_unpack (conf, errp, "o", &o) < 0)
+        return -1;
+    if (!(args = flux_plugin_arg_create ())
+        || flux_plugin_arg_pack (args,
+                                 FLUX_PLUGIN_ARG_IN,
+                                 "{s:O}",
+                                 "conf", o) < 0) {
+        errprintf (errp, "error preparing args for %s jobtap plugin", name);
+        goto error;
+    }
+    if (flux_plugin_call (p, "conf.update", args) < 0) {
+        const char *errmsg;
+        if (flux_plugin_arg_unpack (args,
+                                    FLUX_PLUGIN_ARG_OUT,
+                                    "{s:s}",
+                                    "errmsg", &errmsg) < 0)
+            errprintf (errp, "config rejected by %s jobtap plugin", name);
+        else
+            errprintf (errp, "%s", errmsg);
+        errno = EINVAL;
+        goto error;
+    }
+    flux_plugin_arg_destroy (args);
+    return 0;
+error:
+    flux_plugin_arg_destroy (args);
+    return -1;
+}
+
+static int jobtap_stack_call_conf_update (struct jobtap *jobtap,
+                                          const flux_conf_t *conf,
+                                          flux_error_t *errp)
+{
+    flux_plugin_t *p;
+
+    p = zlistx_first (jobtap->plugins);
+    while (p) {
+        if (jobtap_call_conf_update (p, conf, errp) < 0)
+            return -1;
+        p = zlistx_next (jobtap->plugins);
+    }
+    return 0;
+}
+
+static int jobtap_parse_config (const flux_conf_t *conf,
+                                flux_error_t *errp,
+                                void *arg)
+{
+    struct jobtap *jobtap = arg;
+    json_t *plugins = NULL;
+    flux_error_t error;
+    json_t *entry;
+    int i;
+
+    if (!conf)
+        return errprintf (errp, "conf object can't be NULL");
+
+    /* Changes to [job-manager.plugins] are currently ignored.
+     */
+    if (!jobtap->configured) {
+        if (flux_conf_unpack (conf,
+                              &error,
+                              "{s?{s?o}}",
+                              "job-manager",
+                                "plugins", &plugins) < 0) {
+            return errprintf (errp,
+                              "[job-manager.plugins]: unpack error: %s",
+                              error.text);
+        }
+        if (plugins) {
+            if (!json_is_array (plugins)) {
+                return errprintf (errp,
+                             "[job-manager.plugins] config must be an array");
+            }
+            json_array_foreach (plugins, i, entry) {
+                if (jobtap_conf_entry (jobtap, i, entry, errp) < 0)
+                    return -1;
+            }
+        }
+        jobtap->configured = true;
+    }
+
+    /* Process plugins that want 'conf.update' notifications.
+     * In this case the 'conf' object is the entire instance config
+     * rather than [job-manager.plugins.<name>.conf].
+     */
+    if (jobtap_stack_call_conf_update (jobtap, conf, errp) < 0)
+        return -1;
+
+    return 1; // indicates to conf.c that callback wants updates
+}
+
+static int plugin_byname (const void *item1, const void *item2)
+{
+    const char *name1 = jobtap_plugin_name ((flux_plugin_t *) item1);
+    const char *name2 = item2;
+    if (!name1 || !name2)
+        return -1;
+    return strcmp (name1, name2);
+}
+
+static int load_builtins (struct jobtap *jobtap)
+{
+    struct jobtap_builtin *builtin = jobtap_builtins;
+    flux_error_t error;
+
+    while (builtin && builtin->name) {
+        /*  Yes, this will require re-scanning the builtin plugin list
+         *   in order to lookup the plugin init function by name for
+         *   each loaded plugin. However, this keeps code duplication
+         *   down since jobtap_load() does a lot of work. Plus, this
+         *   is only called once at job-manager startup.
+         *
+         *  If the size of the builtins list gets large this should be
+         *   revisited.
+         */
+        if (!jobtap_load (jobtap, builtin->name, NULL, &error)) {
+            flux_log (jobtap->ctx->h,
+                      LOG_ERR,
+                      "jobtap: %s: %s",
+                      builtin->name,
+                      error.text);
+            return -1;
+        }
+        builtin++;
+    }
+    return 0;
+}
+
+struct jobtap *jobtap_create (struct job_manager *ctx)
+{
+    const char *path;
+    flux_error_t error;
+    struct jobtap *jobtap = calloc (1, sizeof (*jobtap));
+    if (!jobtap)
+        return NULL;
+    jobtap->ctx = ctx;
+    if ((path = flux_conf_builtin_get ("jobtap_pluginpath", FLUX_CONF_AUTO))
+        && !(jobtap->searchpath = strdup (path)))
+        goto error;
+    if (!(jobtap->plugins = zlistx_new ())
+        || !(jobtap->plugins_byuuid = zhashx_new ())
+        || !(jobtap->jobstack = zlistx_new ())
+        || !(jobtap->builtins_ex = zlistx_new ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    zlistx_set_destructor (jobtap->plugins, plugin_destroy);
+    zlistx_set_comparator (jobtap->plugins, plugin_byname);
+    zhashx_set_key_duplicator (jobtap->plugins_byuuid, NULL);
+    zhashx_set_key_destructor (jobtap->plugins_byuuid, NULL);
+    zlistx_set_destructor (jobtap->jobstack, job_destructor);
+    zlistx_set_duplicator (jobtap->jobstack, job_duplicator);
+    zlistx_set_destructor (jobtap->builtins_ex, builtin_ex_destructor);
+
+
+    if (load_builtins (jobtap) < 0) {
+        flux_log (ctx->h, LOG_ERR, "jobtap: failed to init builtins");
+        goto error;
+    }
+
+    if (conf_register_callback (ctx->conf,
+                                &error,
+                                jobtap_parse_config,
+                                jobtap) < 0) {
+        flux_log (ctx->h, LOG_ERR, "%s", error.text);
+        goto error;
+    }
+
+    return jobtap;
+error:
+    jobtap_destroy (jobtap);
+    return NULL;
+}
+
+void jobtap_destroy (struct jobtap *jobtap)
+{
+    if (jobtap) {
+        int saved_errno = errno;
+        conf_unregister_callback (jobtap->ctx->conf, jobtap_parse_config);
+        zlistx_destroy (&jobtap->plugins);
+        zhashx_destroy (&jobtap->plugins_byuuid);
+        zlistx_destroy (&jobtap->jobstack);
+        zlistx_destroy (&jobtap->builtins_ex);
+        jobtap->ctx = NULL;
+        free (jobtap->searchpath);
+        free (jobtap);
+        errno = saved_errno;
+    }
+}
+
+static int jobtap_topic_match_count (struct jobtap *jobtap,
+                                     const char *topic)
+{
+    int count = 0;
+    flux_plugin_t *p = zlistx_first (jobtap->plugins);
+    while (p) {
+        if (flux_plugin_match_handler (p, topic))
+            count++;
+        p = zlistx_next (jobtap->plugins);
+    }
+    return count;
+}
+
+static int jobtap_post_jobspec_updates (struct jobtap *jobtap,
+                                        struct job *job)
+{
+    int rc;
+    json_t *update = jobtap->jobspec_update;
+
+    if (!update)
+        return 0;
+    /*
+     * Note: Ensure this function is reentrant by nullifying
+     * jobtap->jobspec_update before posting the jobspec-update event.
+     * The jobspec-update event may trigger other jobtap callbacks, most
+     * notably via a call to "job.update".
+     */
+    jobtap->jobspec_update = NULL;
+    rc = event_job_post_pack (jobtap->ctx->event,
+                              job,
+                              "jobspec-update",
+                              0,
+                              "O",
+                              update);
+    ERRNO_SAFE_WRAP (json_decref, update);
+    return rc;
+}
+
+static int jobtap_stack_call (struct jobtap *jobtap,
+                              zlistx_t *plugins,
+                              struct job *job,
+                              const char *topic,
+                              flux_plugin_arg_t *args)
+{
+    int retcode = 0;
+    flux_plugin_t *p = NULL;
+
+    /* Duplicate list to make jobtap_stack_call reentrant */
+    zlistx_t *l = zlistx_dup (plugins);
+    if (!l)
+        return -1;
+    zlistx_set_destructor (l, NULL);
+
+    if (current_job_push (jobtap, job) < 0)
+        return -1;
+    p = zlistx_first (l);
+    while (p) {
+        int rc = flux_plugin_call (p, topic, args);
+        if (rc < 0)  {
+            flux_log (jobtap->ctx->h,
+                      LOG_DEBUG,
+                      "jobtap: %s: %s: rc=%d",
+                      jobtap_plugin_name (p),
+                      topic,
+                      rc);
+            retcode = -1;
+            break;
+        }
+        /*  Post any pending jobspec updates now. This is done after
+         *  the callback returns to avoid rewriting jobspec during a
+         *  plugin callback that modifies it.
+         */
+        if (jobtap_post_jobspec_updates (jobtap, job) < 0) {
+            flux_log_error (jobtap->ctx->h,
+                            "jobtap: %s: %s: failed to apply jobspec updates",
+                            jobtap_plugin_name (p),
+                            topic);
+            retcode = -1;
+            break;
+        }
+        retcode += rc;
+        p = zlistx_next (l);
+    }
+    zlistx_destroy (&l);
+    if (current_job_pop (jobtap) < 0)
+        return -1;
+    return retcode;
+}
+
+int jobtap_get_priority (struct jobtap *jobtap,
+                         struct job *job,
+                         int64_t *pprio)
+{
+    int rc = -1;
+    flux_plugin_arg_t *args;
+    int64_t priority = FLUX_JOBTAP_PRIORITY_UNAVAIL;
+
+    if (!jobtap || !job || !pprio) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!(args = jobtap_args_create (jobtap, job)))
+        return -1;
+
+    rc = jobtap_stack_call (jobtap,
+                            jobtap->plugins,
+                            job,
+                            "job.priority.get",
+                            args);
+
+    if (rc >= 1) {
+        /*
+         *  A priority.get callback was run. Try to unpack a new priority
+         */
+        if (flux_plugin_arg_unpack (args,
+                                    FLUX_PLUGIN_ARG_OUT,
+                                    "{s?I}",
+                                    "priority", &priority) < 0) {
+            flux_log (jobtap->ctx->h,
+                      LOG_ERR,
+                      "jobtap: job.priority.get: arg_unpack: %s",
+                      flux_plugin_arg_strerror (args));
+            /*  Note failure, but keep current priority */
+            priority = job->priority;
+            rc = -1;
+        }
+        if (priority == FLUX_JOBTAP_PRIORITY_UNAVAIL) {
+            /*
+             *  Plugin cannot determine priority at this time. Set
+             *   priority to the current job->priority so that a priority
+             *   event is not generated.
+             */
+            priority = job->priority;
+            /*
+             *  A plugin cannot return an "unavailable" priority from the
+             *   priority.get callback for jobs in SCHED state. Log an error
+             *   in this case and make no change to priority.
+             */
+            if (job->state == FLUX_JOB_STATE_SCHED)
+                flux_log (jobtap->ctx->h,
+                          LOG_ERR,
+                          "jobtap: %s: BUG: plugin didn't return priority",
+                          idf58 (job->id));
+        }
+        /*
+         *   O/w, plugin provided a new priority.
+         */
+    }
+    else if (rc < 0) {
+        /*
+         *  priority.get callback was run and failed. Log the error
+         *   and return the current priority.
+         */
+        flux_log (jobtap->ctx->h,
+                  LOG_ERR,
+                  "jobtap: job.priority.get: callback failed");
+        priority = job->priority;
+    }
+
+    flux_plugin_arg_destroy (args);
+    *pprio = priority;
+    return rc;
+}
+
+static void error_asprintf (struct jobtap *jobtap,
+                            struct job *job,
+                            char **errp,
+                            const char *fmt, ...)
+{
+    va_list ap;
+    int saved_errno = errno;
+    va_start (ap, fmt);
+    if (vasprintf (errp, fmt, ap) < 0)
+        flux_log_error (jobtap->ctx->h,
+                        "id=%s: failed to create error string: fmt=%s",
+                        idf58 (job->id), fmt);
+    va_end (ap);
+    errno = saved_errno;
+}
+
+/* Common function for job.create and job.validate.
+ * Both can reject a job with textual error for the submit RPC.
+ */
+static int jobtap_call_early (struct jobtap *jobtap,
+                              struct job *job,
+                              const char *topic,
+                              char **errp)
+{
+    int rc;
+    flux_plugin_arg_t *args;
+    const char *errmsg = NULL;
+
+    if (jobtap_topic_match_count (jobtap, topic) == 0)
+        return 0;
+    if (!(args = jobtap_args_create (jobtap, job)))
+        return -1;
+
+    rc = jobtap_stack_call (jobtap,
+                            jobtap->plugins,
+                            job,
+                            topic,
+                            args);
+
+    if (rc < 0) {
+        /*
+         *  Plugin callback failed, check for errmsg for this job
+         *   If plugin did not provide an error message, then construct
+         *   a generic error "rejected by plugin".
+         */
+        if (flux_plugin_arg_unpack (args,
+                                    FLUX_PLUGIN_ARG_OUT,
+                                    "{s:s}",
+                                    "errmsg", &errmsg) < 0)
+                errmsg = "rejected by job-manager plugin";
+        if ((*errp = strdup (errmsg)) == NULL)
+            flux_log (jobtap->ctx->h, LOG_ERR,
+                      "jobtap: validate failed to capture errmsg");
+    }
+    flux_plugin_arg_destroy (args);
+    return rc;
+}
+
+int jobtap_validate (struct jobtap *jobtap, struct job *job, char **errp)
+{
+    return jobtap_call_early (jobtap, job, "job.validate", errp);
+}
+
+int jobtap_call_create (struct jobtap *jobtap, struct job *job, char **errp)
+{
+    return jobtap_call_early (jobtap, job, "job.create", errp);
+}
+
+static int make_dependency_topic (struct jobtap *jobtap,
+                                  struct job *job,
+                                  int index,
+                                  json_t *entry,
+                                  const char **schemep,
+                                  char *topic,
+                                  int topiclen,
+                                  char **errp)
+{
+    *schemep = NULL;
+    if (json_unpack (entry, "{s:s}", "scheme", schemep) < 0
+        || *schemep == NULL) {
+        error_asprintf (jobtap,
+                        job,
+                        errp,
+                        "dependency[%d] missing string scheme",
+                        index);
+        return -1;
+    }
+
+    if (snprintf (topic,
+                  topiclen,
+                  "job.dependency.%s",
+                  *schemep) > topiclen) {
+        error_asprintf (jobtap, job, errp,
+                        "rejecting absurdly long dependency scheme: %s",
+                        *schemep);
+        return -1;
+    }
+
+    return 0;
+}
+
+static int jobtap_check_dependency (struct jobtap *jobtap,
+                                    flux_plugin_t *p,
+                                    struct job *job,
+                                    flux_plugin_arg_t *args,
+                                    int index,
+                                    json_t *entry,
+                                    char **errp)
+{
+    int rc = -1;
+    char topic [128];
+    const char *scheme = NULL;
+
+    if (make_dependency_topic (jobtap,
+                               job,
+                               index,
+                               entry,
+                               &scheme,
+                               topic,
+                               sizeof (topic),
+                               errp) < 0)
+        return -1;
+
+    /*  If we're only calling this topic for a single plugin, and there
+     *   is no matching handler, return without error immediately
+     */
+    if (p && !flux_plugin_match_handler (p, topic))
+        return 0;
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_IN,
+                              "{s:O}",
+                              "dependency", entry) < 0
+        || flux_plugin_arg_set (args, FLUX_PLUGIN_ARG_OUT, "{}") < 0) {
+        flux_log_error (jobtap->ctx->h,
+                        "jobtap_check_dependency: failed to prepare args");
+        return -1;
+    }
+
+    if (p)
+        rc = flux_plugin_call (p, topic, args);
+    else
+        rc = jobtap_stack_call (jobtap, jobtap->plugins, job, topic, args);
+
+    if (rc == 0) {
+        /*  No handler for job.dependency.<scheme>. return an error.
+         */
+        error_asprintf (jobtap,
+                        job,
+                        errp,
+                        "dependency scheme \"%s\" not supported",
+                        scheme);
+        rc = -1;
+    }
+    else if (rc < 0) {
+        /*
+         *  Plugin callback failed, check for errmsg for this job
+         *   If plugin did not provide an error message, then construct
+         *   a generic error "rejected by plugin".
+         */
+        const char *errmsg;
+        if (flux_plugin_arg_unpack (args,
+                                    FLUX_PLUGIN_ARG_OUT,
+                                    "{s:s}",
+                                    "errmsg", &errmsg) < 0) {
+                errmsg = "rejected by job-manager dependency plugin";
+        }
+        error_asprintf (jobtap, job, errp, "%s", errmsg);
+    }
+    return rc;
+}
+
+static int dependencies_unpack (struct jobtap * jobtap,
+                                struct job * job,
+                                char **errp,
+                                json_t **resultp)
+{
+    json_t *dependencies = NULL;
+    json_error_t error;
+
+    if (json_unpack_ex (job->jobspec_redacted,
+                        &error,
+                        0,
+                        "{s:{s?{s?o}}}",
+                        "attributes",
+                          "system",
+                            "dependencies", &dependencies) < 0) {
+        error_asprintf (jobtap,
+                        job,
+                        errp,
+                        "unable to unpack dependencies: %s",
+                        error.text);
+        return -1;
+    }
+
+    if (!dependencies)
+        return 0;
+
+    if (!json_is_array (dependencies)) {
+        error_asprintf (jobtap,
+                        job,
+                        errp,
+                        "dependencies object must be an array");
+        return -1;
+    }
+
+    if (json_array_size (dependencies) == 0)
+        return 0;
+
+    *resultp = dependencies;
+    return 0;
+}
+
+int jobtap_check_dependencies (struct jobtap *jobtap,
+                               struct job *job,
+                               bool raise_exception,
+                               char **errp)
+{
+    int rc = -1;
+    flux_plugin_arg_t *args = NULL;
+    json_t *dependencies = NULL;
+    json_t *entry;
+    size_t index;
+
+    if ((rc = dependencies_unpack (jobtap, job, errp, &dependencies)) < 0
+        || dependencies == NULL)
+        return rc;
+
+    if (!(args = jobtap_args_create (jobtap, job))) {
+        error_asprintf (jobtap,
+                        job,
+                        errp,
+                        "jobtap_check_dependencies: failed to create args");
+        return -1;
+    }
+
+    json_array_foreach (dependencies, index, entry) {
+        rc = jobtap_check_dependency (jobtap,
+                                      NULL,
+                                      job,
+                                      args,
+                                      index,
+                                      entry,
+                                      errp);
+        if (rc < 0) {
+            if (!raise_exception)
+                goto out;
+            if (jobtap_job_raise (jobtap,
+                                  job,
+                                  "dependency",
+                                  4, /* LOG_WARNING */
+                                  "%s (job may be stuck in DEPEND state)",
+                                  *errp) < 0)
+                flux_log_error (jobtap->ctx->h,
+                                "id=%s: failed to raise dependency exception",
+                                idf58 (job->id));
+            free (*errp);
+            *errp = NULL;
+        }
+    }
+    rc = 0;
+out:
+    flux_plugin_arg_destroy (args);
+    return rc;
+}
+
+int jobtap_notify_subscribers (struct jobtap *jobtap,
+                               struct job *job,
+                               const char *name,
+                               const char *fmt,
+                               ...)
+{
+    flux_plugin_arg_t *args;
+    char topic [64];
+    int topiclen = 64;
+    va_list ap;
+    int rc;
+
+    if (!job->subscribers)
+        return 0;
+
+    if (snprintf (topic, topiclen, "job.event.%s", name) >= topiclen) {
+        flux_log (jobtap->ctx->h,
+                  LOG_ERR,
+                  "jobtap: %s: %s: event topic name too long",
+                  name,
+                  idf58 (job->id));
+        return -1;
+    }
+
+    va_start (ap, fmt);
+    args = jobtap_args_vcreate (jobtap, job, fmt, ap);
+    va_end (ap);
+    if (!args) {
+        flux_log (jobtap->ctx->h,
+                  LOG_ERR,
+                  "jobtap: %s: %s: failed to create plugin args",
+                  topic,
+                  idf58 (job->id));
+        return -1;
+    }
+
+    rc = jobtap_stack_call (jobtap, job->subscribers, job, topic, args);
+    flux_plugin_arg_destroy (args);
+    return rc;
+}
+
+int jobtap_call (struct jobtap *jobtap,
+                 struct job *job,
+                 const char *topic,
+                 const char *fmt,
+                 ...)
+{
+    int rc = -1;
+    json_t *note = NULL;
+    json_t *R = NULL;
+    flux_plugin_arg_t *args;
+    int64_t priority = FLUX_JOBTAP_PRIORITY_UNAVAIL;
+    va_list ap;
+
+    if (jobtap_topic_match_count (jobtap, topic) == 0)
+        return 0;
+
+    va_start (ap, fmt);
+    if (!(args = jobtap_args_vcreate (jobtap, job, fmt, ap))) {
+        flux_log (jobtap->ctx->h,
+                  LOG_ERR,
+                  "jobtap: %s: %s: failed to create plugin args",
+                  topic,
+                  idf58 (job->id));
+    }
+    va_end (ap);
+
+    if (!args)
+        return -1;
+
+    rc = jobtap_stack_call (jobtap, jobtap->plugins, job, topic, args);
+    if (rc < 0) {
+        flux_log (jobtap->ctx->h,
+                  LOG_ERR,
+                  "jobtap: %s: callback returned error",
+                  topic);
+    }
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_OUT,
+                                "{s?I s?o s?o}",
+                                "priority", &priority,
+                                "annotations", &note,
+                                "R", &R) < 0) {
+        if (jobtap_job_raise (jobtap,
+                              job,
+                              topic, 4,
+                              "arg_unpack: %s%s",
+                              flux_plugin_arg_strerror (args),
+                              job->state == FLUX_JOB_STATE_PRIORITY ?
+                              " (job may be stuck in PRIORITY state)" : "") < 0)
+            flux_log (jobtap->ctx->h,
+                      LOG_ERR,
+                      "%s: jobtap_job_raise: %s",
+                       topic,
+                       strerror (errno));
+        rc = -1;
+    }
+    if (R != NULL) {
+        if (!streq (topic, "job.state.sched")) {
+            flux_log (jobtap->ctx->h,
+                      LOG_ERR,
+                      "jobtap: %s: %s: R may only be set in SCHED state",
+                      topic,
+                      idf58 (job->id));
+            rc = -1;
+        }
+        else if (job->R_redacted) {
+            flux_log (jobtap->ctx->h,
+                      LOG_ERR,
+                      "jobtap: %s: %s: R is already set",
+                      topic,
+                      idf58 (job->id));
+            rc = -1;
+        }
+        else
+            job->R_redacted = json_incref (R);
+    }
+    if (note != NULL) {
+        /*
+         *  Allow plugins to update annotations. (A failure here will be
+         *   logged but not considered a fatal error)
+         *
+         *  In job.new callback annotations are not published because an
+         *   annotation event published to the journal before the first
+         *   job state event may confuse consumers (i.e. job-info).
+         */
+        int ret;
+        if (streq (topic, "job.new"))
+            ret = annotations_update (job, ".", note);
+        else
+            ret = annotations_update_and_publish (jobtap->ctx, job, note);
+        if (ret < 0)
+            flux_log_error (jobtap->ctx->h,
+                            "jobtap: %s: %s: annotations_update",
+                            topic,
+                            idf58 (job->id));
+    }
+    if (priority >= FLUX_JOB_PRIORITY_MIN) {
+        /*
+         *  Reprioritize job if plugin returned a priority.
+         *   Note: reprioritize_job() is a no-op if job is not in
+         *         PRIORITY or SCHED state)
+         */
+        if (reprioritize_job (jobtap->ctx, job, priority) < 0)
+            flux_log_error (jobtap->ctx->h, "jobtap: reprioritize_job");
+    }
+    /*  else: FLUX_JOBTAP_PRIORITY_UNAVAIL, job cannot yet be assigned a
+     *   priority. This is a fall-through condiition. A job in PRIORITY
+     *   state will stay there until the plugin actively calls
+     *   flux_jobtap_reprioritize_job()
+     */
+    flux_plugin_arg_destroy (args);
+    return rc;
+}
+
+static int jobtap_load_builtin (flux_plugin_t *p,
+                                const char *name)
+{
+    struct jobtap_builtin *builtin = jobtap_builtins;
+
+    while (builtin && builtin->name) {
+        if (streq (name, builtin->name)) {
+            if (flux_plugin_set_name (p, builtin->name) < 0)
+                return -1;
+            return (*builtin->init) (p);
+        }
+        builtin++;
+    }
+
+    errno = ENOENT;
+    return -1;
+}
+
+static int jobtap_load_builtin_ex (struct jobtap *jobtap,
+                                   flux_plugin_t *p,
+                                   const char *name)
+{
+    struct jobtap_builtin_ex *ex = zlistx_first (jobtap->builtins_ex);
+    while (ex) {
+        if (streq (name, ex->name)) {
+            if (flux_plugin_set_name (p, ex->name) < 0)
+                return -1;
+            return (*ex->init_cb) (p, ex->arg);
+        }
+        ex = zlistx_next (jobtap->builtins_ex);
+    }
+    errno = ENOENT;
+    return -1;
+}
+
+int jobtap_register_builtin (struct jobtap *jobtap,
+                             const char *name,
+                             jobtap_builtin_f init_cb,
+                             void *arg)
+{
+    struct jobtap_builtin_ex *ex;
+
+    if (!jobtap || !name || !init_cb || !strstarts (name, ".")) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(ex = jobtap_builtin_ex_create (name, init_cb, arg)))
+        return -1;
+    if (!(zlistx_add_end (jobtap->builtins_ex, ex))) {
+        jobtap_builtin_ex_destroy (ex);
+        errno = ENOMEM;
+    }
+    return 0;
+}
+
+/*  Return 1 if either searchpath is NULL, or path starts with '/' or './'.
+ */
+static int no_searchpath (const char *searchpath, const char *path)
+{
+    return (!searchpath
+            || path[0] == '/'
+            || (path[0] == '.' && path[1] == '/'));
+}
+
+static void item_free (void **item )
+{
+    if (*item) {
+        free (*item);
+        *item = NULL;
+    }
+}
+
+static zlistx_t *path_list (const char *searchpath, const char *path)
+{
+    char *copy;
+    char *str;
+    char *dir;
+    char *s;
+    char *sp = NULL;
+    zlistx_t *l = zlistx_new ();
+
+    if (!l || !(copy = strdup (searchpath)))
+        return NULL;
+    str = copy;
+
+    zlistx_set_destructor (l, item_free);
+
+    while ((dir = strtok_r (str, ":", &sp))) {
+        if (asprintf (&s, "%s/%s", dir, path) < 0)
+            goto error;
+        if (!zlistx_add_end (l, s))
+            goto error;
+        str = NULL;
+    }
+    free (copy);
+    return l;
+error:
+    ERRNO_SAFE_WRAP (free, copy);
+    ERRNO_SAFE_WRAP (zlistx_destroy, &l);
+    return NULL;
+}
+
+static int plugin_set_name (flux_plugin_t *p,
+                            const char *basename)
+{
+    int rc = -1;
+    char *q;
+    char *copy = NULL;
+    const char *name = flux_plugin_get_name (p);
+
+    /*  It is ok to have a custom name, but that name may
+     *   not contain '/' or '.'
+     */
+    if (name && !strchr (name, '/') && !strchr (name, '.'))
+        return 0;
+    if (!(copy = strdup (basename)))
+        return -1;
+    if ((q = strchr (copy, '.')))
+        *q = '\0';
+    rc = flux_plugin_set_name (p, copy);
+    ERRNO_SAFE_WRAP (free, copy);
+    return rc;
+}
+
+static int plugin_try_load (struct jobtap *jobtap,
+                            flux_plugin_t *p,
+                            const char *fullpath,
+                            flux_error_t *errp)
+{
+    char *name = NULL;
+
+    if (flux_plugin_load_dso (p, fullpath) < 0)
+        return errprintf (errp,
+                          "%s",
+                          flux_plugin_strerror (p));
+    if (!(name = strdup (basename_simple (fullpath)))
+        || flux_plugin_aux_set (p, "jobtap::basename", name, free) < 0) {
+        ERRNO_SAFE_WRAP (free, name);
+        return errprintf (errp,
+                          "%s: failed to create plugin basename",
+                          fullpath);
+    }
+    if (plugin_set_name (p, name) < 0)
+        return errprintf (errp,
+                          "%s: unable to set a plugin name",
+                           fullpath);
+    if (zlistx_find (jobtap->plugins, (void *) jobtap_plugin_name (p))) {
+        errno = EEXIST;
+        return errprintf (errp,
+                          "%s already loaded",
+                          jobtap_plugin_name (p));
+    }
+    return 0;
+}
+
+int jobtap_plugin_load_first (struct jobtap *jobtap,
+                              flux_plugin_t *p,
+                              const char *path,
+                              flux_error_t *errp)
+{
+    bool found = false;
+    zlistx_t *l;
+    char *fullpath;
+
+    if (no_searchpath (jobtap->searchpath, path))
+        return plugin_try_load (jobtap, p, path, errp);
+
+    if (!(l = path_list (jobtap->searchpath, path)))
+        return -1;
+
+    fullpath = zlistx_first (l);
+    while (fullpath) {
+        int rc = plugin_try_load (jobtap, p, fullpath, errp);
+        if (rc < 0 && errno != ENOENT) {
+            ERRNO_SAFE_WRAP (zlistx_destroy , &l);
+            return -1;
+        }
+        if (rc == 0) {
+            found = true;
+            break;
+        }
+        fullpath = zlistx_next (l);
+    }
+    zlistx_destroy (&l);
+    if (!found) {
+        errno = ENOENT;
+        return errprintf (errp, "%s: No such plugin found", path);
+    }
+    return 0;
+}
+
+static bool is_builtin (const char *path)
+{
+    /*  A builtin plugin starts with '.' and does not contain a slash
+     */
+    return (path[0] == '.' && !strchr (path, '/'));
+}
+
+flux_plugin_t * jobtap_load (struct jobtap *jobtap,
+                             const char *path,
+                             json_t *conf,
+                             flux_error_t *errp)
+{
+    flux_plugin_t *p = NULL;
+    char *conf_str = NULL;
+
+    err_init (errp);
+
+    if (conf && !json_is_null (conf)) {
+        if (!json_is_object (conf)) {
+            errno = EINVAL;
+            errprintf (errp, "jobptap: plugin conf must be a JSON object");
+            goto error;
+        }
+        if (!(conf_str = json_dumps (conf, 0))) {
+            errno = ENOMEM;
+            errprintf (errp, "%s: %s",
+                      "jobtap: json_dumps(conf) failed",
+                      strerror (errno));
+            goto error;
+        }
+    }
+
+    if (!(p = flux_plugin_create ())
+        || flux_plugin_aux_set (p, "flux::jobtap", jobtap, NULL) < 0)
+        goto error;
+    if (conf_str) {
+        int rc = flux_plugin_set_conf (p, conf_str);
+        free (conf_str);
+        if (rc < 0)
+            goto error;
+    }
+    if (is_builtin (path)) {
+        if (jobtap_load_builtin (p, path) < 0
+            && jobtap_load_builtin_ex (jobtap, p, path) < 0)
+            goto error;
+    }
+    else {
+        flux_plugin_set_flags (p, FLUX_PLUGIN_RTLD_NOW);
+        if (jobtap_plugin_load_first (jobtap, p, path, errp) < 0)
+            goto error;
+    }
+    /* Call conf.update here for two reasons
+     * - fail the plugin load if config is invalid
+     * - make sure plugin has config before job.* callbacks begin
+     */
+    if (jobtap_call_conf_update (p, flux_get_conf (jobtap->ctx->h), errp) < 0)
+        goto error;
+
+    char *uuid = (char *)flux_plugin_get_uuid (p);
+    if (zhashx_insert (jobtap->plugins_byuuid, uuid, p) < 0) {
+        errprintf (errp, "Error adding plugin to list");
+        errno = EEXIST;
+        goto error;
+    }
+    if (!zlistx_add_end (jobtap->plugins, p)) {
+        zhashx_delete (jobtap->plugins_byuuid, uuid);
+        errprintf (errp, "Out of memory adding plugin to list");
+        errno = ENOMEM;
+        goto error;
+    }
+    return p;
+error:
+    if (errp && errp->text[0] == '\0')
+        strncpy (errp->text,
+                 flux_plugin_strerror (p),
+                 sizeof (errp->text) - 1);
+    flux_plugin_destroy (p);
+    return NULL;
+}
+
+static int jobtap_handle_remove_req (struct job_manager *ctx,
+                                     const flux_msg_t *msg,
+                                     const char *arg)
+{
+    flux_error_t error;
+    if (jobtap_remove (ctx->jobtap, arg, &error) < 0) {
+        if (flux_respond_error (ctx->h,
+                                msg,
+                                errno ? errno : EINVAL,
+                                error.text) < 0)
+            flux_log_error (ctx->h,
+                            "jobtap_handle_remove_req: flux_respond_error");
+        return -1;
+    }
+    return 0;
+}
+
+static int jobtap_handle_load_req (struct job_manager *ctx,
+                                   const flux_msg_t *msg,
+                                   const char *path,
+                                   json_t *conf)
+{
+    flux_error_t error;
+    flux_plugin_t *p = NULL;
+
+    if (!(p = jobtap_load_plugin (ctx->jobtap, path, conf, &error))) {
+        if (flux_respond_error (ctx->h,
+                                msg,
+                                errno ? errno : EINVAL,
+                                error.text[0] ? error.text : NULL) < 0)
+            flux_log_error (ctx->h, "jobtap_handler: flux_respond_error");
+        return -1;
+    }
+    return 0;
+}
+
+static json_t *jobtap_plugin_list (struct jobtap *jobtap)
+{
+    flux_plugin_t *p;
+    json_t *result = json_array ();
+    if (result == NULL)
+        return NULL;
+    p = zlistx_first (jobtap->plugins);
+    while (p) {
+        json_t *o = json_string (jobtap_plugin_name (p));
+        if (o == NULL)
+            goto error;
+        if (json_array_append_new (result, o) < 0) {
+            json_decref (o);
+            goto error;
+        }
+        p = zlistx_next (jobtap->plugins);
+    }
+    return result;
+error:
+    json_decref (result);
+    return NULL;
+}
+
+static void jobtap_handle_list_req (flux_t *h,
+                                    struct jobtap *jobtap,
+                                    const flux_msg_t *msg)
+{
+    json_t *o = jobtap_plugin_list (jobtap);
+    if (o == NULL)
+        flux_respond_error (h, msg, ENOMEM, "Failed to create plugin list");
+    else if (flux_respond_pack (h,
+                                msg,
+                                "{s:o}",
+                                "plugins", o) < 0)
+        flux_log_error (h, "jobtap_handle_list: flux_respond");
+}
+
+void jobtap_handler (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    struct job_manager *ctx = arg;
+    const char *path = NULL;
+    const char *remove = NULL;
+    int query_only = 0;
+    json_t *conf = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s?s s?o s?s s?b}",
+                             "load", &path,
+                             "conf", &conf,
+                             "remove", &remove,
+                             "query_only", &query_only) < 0) {
+        if (flux_respond_error (h, msg, EPROTO, NULL) < 0)
+            flux_log_error (h, "jobtap_handler: flux_respond_error");
+        return;
+    }
+    if (query_only) {
+        jobtap_handle_list_req (h, ctx->jobtap, msg);
+        return;
+    }
+    if (remove && jobtap_handle_remove_req (ctx, msg, remove) < 0)
+        return;
+    if (path && jobtap_handle_load_req (ctx, msg, path, conf) < 0)
+        return;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "jobtap_handler: flux_respond");
+}
+
+static int jobtap_query_plugin (flux_plugin_t *p,
+                                char **json_str,
+                                flux_error_t *errp)
+{
+    int rc = -1;
+    flux_plugin_arg_t *args;
+    const char *path = flux_plugin_get_path (p);
+    const char *name = jobtap_plugin_name (p);
+
+    if (path == NULL)
+        path = "builtin";
+
+    if (!(args = flux_plugin_arg_create ()))
+        return errprintf (errp,
+                          "flux_plugin_arg_create: %s",
+                          strerror (errno));
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                             "{s:s s:s}",
+                             "name", name,
+                             "path", path) < 0) {
+        errprintf (errp, "%s", flux_plugin_arg_strerror (args));
+        goto out;
+    }
+
+    if (flux_plugin_call (p, "plugin.query", args) < 0) {
+        errprintf (errp, "plugin.query failed");
+        goto out;
+    }
+
+    if (flux_plugin_arg_get (args, FLUX_PLUGIN_ARG_OUT, json_str) < 0
+        && errno != ENOENT) {
+        errprintf (errp,
+                   "failed to get plugin.query out args: %s",
+                   strerror (errno));
+        goto out;
+    }
+    rc = 0;
+out:
+    flux_plugin_arg_destroy (args);
+    return rc;
+}
+
+void jobtap_query_handler (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct job_manager *ctx = arg;
+    const char *name = NULL;
+    char *result = NULL;
+    flux_plugin_t *p;
+    flux_error_t error;
+    bool found = false;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "name", &name) < 0) {
+        errprintf (&error, "Protocol error");
+        goto error;
+    }
+
+    p = zlistx_first (ctx->jobtap->plugins);
+    while (p) {
+        if (streq (name, jobtap_plugin_name (p))) {
+            found = true;
+            if (jobtap_query_plugin (p, &result, &error) < 0)
+                goto error;
+            break;
+        }
+        p = zlistx_next (ctx->jobtap->plugins);
+    }
+    if (!found) {
+        errprintf (&error, "%s: plugin not found", name);
+        goto error;
+    }
+    if (flux_respond (h, msg, result) < 0)
+        flux_log_error (h, "jobtap_query_handler: flux_respond");
+    free (result);
+    return;
+error:
+    if (flux_respond_error (h, msg, 0, error.text) < 0)
+        flux_log_error (h, "jobtap_query_handler: flux_respond_error");
+}
+
+flux_t *flux_jobtap_get_flux (flux_plugin_t *p)
+{
+    struct jobtap *jobtap = NULL;
+
+    if (p == NULL
+        || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))
+        || !jobtap->ctx) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return jobtap->ctx->h;
+}
+
+static int build_jobtap_topic (flux_plugin_t *p,
+                               const char *method,
+                               char *buf,
+                               int len)
+{
+    /*  N.B. use plugin provided or sanitized name (trailing .so removed)
+     *   in topic string. This name is stored as the main plugin name.
+     */
+    const char *name = flux_plugin_get_name (p);
+
+    /*
+     *  Detect improperly initialized plugin name before continuing:
+     */
+    if (name == NULL || strchr (name, '/')) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (*name == '.') // skip conventional "." prefix used in hidden plugins
+        name++;
+    if (snprintf (buf,
+                  len,
+                  "job-manager.%s%s%s",
+                  name,
+                  method ? "." : "",
+                  method ? method : "") >= len) {
+        errno = EINVAL;
+        return -1;
+    }
+    return 0;
+}
+
+int flux_jobtap_service_register_ex (flux_plugin_t *p,
+                                     const char *method,
+                                     uint32_t rolemask,
+                                     flux_msg_handler_f cb,
+                                     void *arg)
+{
+    struct flux_match match = FLUX_MATCH_REQUEST;
+    flux_msg_handler_t *mh;
+    char topic[1024];
+    flux_t *h;
+
+    if (!(h = flux_jobtap_get_flux (p))
+        || build_jobtap_topic (p, method, topic, sizeof (topic)) < 0)
+        return -1;
+
+    match.topic_glob = topic;
+    if (!(mh = flux_msg_handler_create (h, match, cb, arg)))
+        return -1;
+
+    if (flux_plugin_aux_set (p,
+                             NULL,
+                             mh,
+                             (flux_free_f) flux_msg_handler_destroy) < 0) {
+        flux_msg_handler_destroy (mh);
+        return -1;
+    }
+    flux_msg_handler_allow_rolemask (mh, rolemask);
+    flux_msg_handler_start (mh);
+    flux_log (h, LOG_DEBUG, "jobtap plugin %s registered method %s",
+              jobtap_plugin_name (p),
+              topic);
+    return 0;
+}
+
+int flux_jobtap_service_register (flux_plugin_t *p,
+                                  const char *method,
+                                  flux_msg_handler_f cb,
+                                  void *arg)
+{
+    return flux_jobtap_service_register_ex (p, method, 0, cb, arg);
+}
+
+int flux_jobtap_reprioritize_all (flux_plugin_t *p)
+{
+    struct jobtap *jobtap = flux_plugin_aux_get (p, "flux::jobtap");
+    if (!jobtap) {
+        errno = EINVAL;
+        return -1;
+    }
+    return reprioritize_all (jobtap->ctx);
+}
+
+int flux_jobtap_reprioritize_job (flux_plugin_t *p,
+                                  flux_jobid_t id,
+                                  unsigned int priority)
+{
+    struct jobtap *jobtap = flux_plugin_aux_get (p, "flux::jobtap");
+    if (!jobtap) {
+        errno = EINVAL;
+        return -1;
+    }
+    return reprioritize_id (jobtap->ctx, id, priority);
+}
+
+int flux_jobtap_priority_unavail (flux_plugin_t *p, flux_plugin_arg_t *args)
+{
+    struct jobtap *jobtap = flux_plugin_aux_get (p, "flux::jobtap");
+    if (!jobtap) {
+        errno = EINVAL;
+        return -1;
+    }
+    /* Still todo: check valid state, etc.
+     */
+    return flux_plugin_arg_pack (args,
+                                 FLUX_PLUGIN_ARG_OUT,
+                                 "{s:I}",
+                                 "priority", FLUX_JOBTAP_PRIORITY_UNAVAIL);
+}
+
+static void jobtap_verror (flux_plugin_t *p,
+                           flux_plugin_arg_t *args,
+                           const char *fmt,
+                           va_list ap)
+{
+    flux_error_t error;
+
+    verrprintf (&error, fmt, ap);
+
+    if (error.text[0] != '\0') {
+        if (flux_plugin_arg_pack (args,
+                                  FLUX_PLUGIN_ARG_OUT,
+                                  "{s:s}",
+                                  "errmsg", error.text) < 0) {
+            flux_log_error (flux_jobtap_get_flux (p),
+                            "flux_jobtap_reject_job: failed to pack error");
+        }
+    }
+}
+
+int flux_jobtap_error (flux_plugin_t *p,
+                       flux_plugin_arg_t *args,
+                       const char *fmt,
+                       ...)
+{
+    va_list ap;
+    va_start (ap, fmt);
+    jobtap_verror (p, args, fmt, ap);
+    va_end (ap);
+    return -1;
+}
+
+
+int flux_jobtap_reject_job (flux_plugin_t *p,
+                            flux_plugin_arg_t *args,
+                            const char *fmt,
+                            ...)
+{
+    if (fmt) {
+        va_list ap;
+        va_start (ap, fmt);
+        jobtap_verror (p, args, fmt, ap);
+        va_end (ap);
+    }
+    else {
+        flux_jobtap_error (p,
+                           args,
+                           "rejected by job-manager plugin '%s'",
+                           jobtap_plugin_name (p));
+    }
+    return -1;
+}
+
+static struct job *lookup_active_job (struct job_manager *ctx,
+                                      flux_jobid_t id)
+{
+    struct job *job = zhashx_lookup (ctx->active_jobs, &id);
+    if (!job)
+        errno = ENOENT;
+    return job;
+}
+
+static struct job *lookup_job (struct job_manager *ctx, flux_jobid_t id)
+{
+    struct job *job;
+    if (!(job = lookup_active_job (ctx, id))
+        && !(job = zhashx_lookup (ctx->inactive_jobs, &id)))
+        errno = ENOENT;
+    return job;
+}
+
+static int jobtap_emit_dependency_event (struct jobtap *jobtap,
+                                         struct job *job,
+                                         bool add,
+                                         const char *description)
+{
+    int flags = 0;
+    const char *event = add ? "dependency-add" : "dependency-remove";
+
+    if (job->state != FLUX_JOB_STATE_DEPEND
+        && job->state != FLUX_JOB_STATE_NEW) {
+        errno = EINVAL;
+        return -1;
+    }
+    return event_job_post_pack (jobtap->ctx->event,
+                                job,
+                                event,
+                                flags,
+                                "{s:s}",
+                                "description", description);
+}
+
+static int emit_dependency_event (flux_plugin_t *p,
+                                  flux_jobid_t id,
+                                  bool add,
+                                  const char *description)
+{
+    struct job *job;
+    struct jobtap *jobtap = flux_plugin_aux_get (p, "flux::jobtap");
+    if (!jobtap) {
+        errno = EINVAL;
+        return -1;
+    }
+    job = current_job (jobtap);
+    if (!job || id != job->id) {
+        if (!(job = lookup_active_job (jobtap->ctx, id)))
+            return -1;
+    }
+    return jobtap_emit_dependency_event (jobtap, job, add, description);
+}
+
+int flux_jobtap_dependency_add (flux_plugin_t *p,
+                                flux_jobid_t id,
+                                const char *description)
+{
+    return emit_dependency_event (p, id, true, description);
+}
+
+int flux_jobtap_dependency_remove (flux_plugin_t *p,
+                                   flux_jobid_t id,
+                                   const char *description)
+{
+    return emit_dependency_event (p, id, false, description);
+}
+
+static struct job * jobtap_lookup_jobid (flux_plugin_t *p, flux_jobid_t id)
+{
+    struct jobtap *jobtap;
+    struct job *job;
+    if (p == NULL
+        || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))) {
+        errno = EINVAL;
+        return NULL;
+    }
+    job = current_job (jobtap);
+    if (id == FLUX_JOBTAP_CURRENT_JOB || (job && id == job->id)) {
+        errno = EINVAL;
+        return job;
+    }
+    return lookup_job (jobtap->ctx, id);
+}
+
+static struct job * jobtap_lookup_active_jobid (flux_plugin_t *p,
+                                                flux_jobid_t id)
+{
+    struct job * job = jobtap_lookup_jobid (p, id);
+    if (!job || job->state == FLUX_JOB_STATE_INACTIVE) {
+        errno = ENOENT;
+        return NULL;
+    }
+    return job;
+}
+
+/* Job aux items are not stored in the job aux container directly, to avoid
+ * segfaults that might result from registering a destructor resident in a
+ * plugin that could be unloaded before the item is destroyed.
+ *
+ * Instead, each plugin stores one item named "jobtap::<uuid>" which contains
+ * an aux container, and the actual items are stored in the inner container.
+ * As plugin is being unloaded, the outer container is destroyed before
+ * the plugin is destroyed, causing the inner container and its items to be
+ * destroyed also.
+ */
+
+struct aux_wrap {
+    struct aux_item *aux;
+    struct jobtap *jobtap;
+    char *uuid;
+};
+
+static struct aux_wrap *aux_wrap_create (flux_plugin_t *p)
+{
+    const char *uuid = flux_plugin_get_uuid (p);
+    struct aux_wrap *wrap;
+
+    if (!(wrap = calloc (1, sizeof (*wrap) + strlen (uuid) + 1)))
+        return NULL;
+    wrap->jobtap = flux_plugin_aux_get (p, "flux::jobtap");
+    wrap->uuid = (char *)(wrap + 1);
+    strcpy (wrap->uuid, uuid);
+    return wrap;
+}
+
+// flux_free_f signature
+static void aux_wrap_destructor (void *item)
+{
+    struct aux_wrap *wrap = item;
+    if (wrap) {
+        int saved_errno = errno;
+        if (zhashx_lookup (wrap->jobtap->plugins_byuuid, wrap->uuid))
+            aux_destroy (&wrap->aux);
+        else {
+            flux_log (wrap->jobtap->ctx->h,
+                      LOG_ERR,
+                      "leaking job aux item(s) abandoned by unloaded plugin");
+        }
+        free (wrap);
+        errno = saved_errno;
+    }
+}
+
+static struct aux_wrap *aux_wrap_get (flux_plugin_t *p,
+                                      struct job *job,
+                                      bool create)
+{
+    char wname[64];
+    struct aux_wrap *wrap;
+
+    snprintf (wname, sizeof (wname), "jobtap::%s", flux_plugin_get_uuid (p));
+    if (!(wrap = job_aux_get (job, wname))) {
+        if (!create)
+            return NULL;
+        if (!(wrap = aux_wrap_create (p))
+            || job_aux_set (job, wname, wrap, aux_wrap_destructor) < 0) {
+            aux_wrap_destructor (wrap);
+            return NULL;
+        }
+    }
+    return wrap;
+}
+
+int flux_jobtap_job_aux_set (flux_plugin_t *p,
+                             flux_jobid_t id,
+                             const char *name,
+                             void *val,
+                             flux_free_f free_fn)
+{
+    struct job *job;
+    struct aux_wrap *wrap;
+
+    if (!(job = jobtap_lookup_jobid (p, id))
+        || !(wrap = aux_wrap_get (p, job, true)))
+        return -1;
+    return aux_set (&wrap->aux, name, val, free_fn);
+}
+
+void * flux_jobtap_job_aux_get (flux_plugin_t *p,
+                                flux_jobid_t id,
+                                const char *name)
+{
+    struct job *job;
+    struct aux_wrap *wrap;
+
+    if (!(job = jobtap_lookup_jobid (p, id))
+        || !(wrap = aux_wrap_get (p, job, false)))
+        return NULL;
+    return aux_get (wrap->aux, name);
+}
+
+int flux_jobtap_job_aux_delete (flux_plugin_t *p,
+                                flux_jobid_t id,
+                                void *val)
+{
+    struct job *job;
+    struct aux_wrap *wrap;
+
+    if (!(job = jobtap_lookup_jobid (p, id)))
+        return -1;
+    if ((wrap = aux_wrap_get (p, job, false)))
+        aux_delete (&wrap->aux, val);
+    return 0;
+}
+
+int flux_jobtap_job_set_flag (flux_plugin_t *p,
+                              flux_jobid_t id,
+                              const char *flag)
+{
+    struct jobtap *jobtap;
+    struct job *job;
+    if (!p || !flag || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(job = jobtap_lookup_active_jobid (p, id))) {
+        errno = ENOENT;
+        return -1;
+    }
+    if (!job_flag_valid (job, flag))
+        return -1;
+    return event_job_post_pack (jobtap->ctx->event,
+                                job,
+                                "set-flags",
+                                0,
+                                "{s:[s]}",
+                                "flags", flag);
+}
+
+static int jobtap_job_vraise (struct jobtap *jobtap,
+                              struct job *job,
+                              const char *type,
+                              int severity,
+                              const char *fmt,
+                              va_list ap)
+{
+    char note [1024];
+    if (vsnprintf (note, sizeof (note), fmt, ap) >= sizeof (note))
+        note[sizeof(note) - 2] = '+';
+    return raise_job_exception (jobtap->ctx,
+                                job,
+                                type,
+                                severity,
+                                jobtap->ctx->owner,
+                                note);
+}
+
+static int jobtap_job_raise (struct jobtap *jobtap,
+                             struct job *job,
+                             const char *type,
+                             int severity,
+                             const char *fmt, ...)
+{
+    int rc;
+    va_list ap;
+    va_start (ap, fmt);
+    rc = jobtap_job_vraise (jobtap, job, type, severity, fmt, ap);
+    va_end (ap);
+    return rc;
+}
+
+int flux_jobtap_raise_exception (flux_plugin_t *p,
+                                 flux_jobid_t id,
+                                 const char *type,
+                                 int severity,
+                                 const char *fmt, ...)
+{
+    struct jobtap *jobtap;
+    struct job *job;
+    int rc;
+    va_list ap;
+
+    if (!p || !type || !fmt
+        || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(job = jobtap_lookup_active_jobid (p, id)))
+        return -1;
+    va_start (ap, fmt);
+    rc = jobtap_job_vraise (jobtap, job, type, severity, fmt, ap);
+    va_end (ap);
+    return rc;
+}
+
+flux_plugin_arg_t * flux_jobtap_job_lookup (flux_plugin_t *p,
+                                            flux_jobid_t id)
+{
+    struct jobtap *jobtap;
+    struct job *job;
+    if (!p || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(job = jobtap_lookup_jobid (p, id))) {
+        errno = ENOENT;
+        return NULL;
+    }
+    return jobtap_args_create (jobtap, job);
+}
+
+int flux_jobtap_get_job_result (flux_plugin_t *p,
+                                flux_jobid_t id,
+                                flux_job_result_t *rp)
+{
+    struct jobtap *jobtap;
+    struct job *job;
+    json_error_t error;
+    const char *name = NULL;
+    int waitstatus = -1;
+    int exception_severity = -1;
+    const char *exception_type = NULL;
+    flux_job_result_t result = FLUX_JOB_RESULT_FAILED;
+
+    if (!p || !rp || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(job = jobtap_lookup_jobid (p, id))) {
+        errno = ENOENT;
+        return -1;
+    }
+    if (job->state != FLUX_JOB_STATE_CLEANUP
+        && job->state != FLUX_JOB_STATE_INACTIVE) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (json_unpack_ex (job->end_event,
+                        &error,
+                        0,
+                        "{s:s s:{s?i s?s s?i}}",
+                        "name", &name,
+                        "context",
+                        "status", &waitstatus,
+                        "type", &exception_type,
+                        "severity", &exception_severity) < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (streq (name, "finish") && waitstatus == 0)
+        result = FLUX_JOB_RESULT_COMPLETED;
+    else if (streq (name, "exception")) {
+        if (exception_type != NULL) {
+            if (streq (exception_type, "cancel"))
+                result = FLUX_JOB_RESULT_CANCELED;
+            else if (streq (exception_type, "timeout"))
+                result = FLUX_JOB_RESULT_TIMEOUT;
+        }
+    }
+    *rp = result;
+    return 0;
+}
+
+int flux_jobtap_event_post_pack (flux_plugin_t *p,
+                                 flux_jobid_t id,
+                                 const char *name,
+                                 const char *fmt,
+                                 ...)
+{
+    int rc;
+    va_list ap;
+    struct jobtap *jobtap;
+    struct job *job;
+
+    if (!p || !name
+        || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(job = jobtap_lookup_active_jobid (p, id)))
+        return -1;
+    va_start (ap, fmt);
+    rc = event_job_post_vpack (jobtap->ctx->event, job, name, 0, fmt, ap);
+    va_end (ap);
+    return rc;
+}
+
+int flux_jobtap_jobspec_update_id_pack (flux_plugin_t *p,
+                                        flux_jobid_t id,
+                                        const char *fmt,
+                                        ...)
+{
+    int rc = -1;
+    va_list ap;
+    struct jobtap *jobtap;
+    struct job *job;
+    json_error_t error;
+    json_t *update = NULL;
+
+    if (!p
+        || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))
+        || !(job = jobtap_lookup_active_jobid (p, id))
+        || job->state == FLUX_JOB_STATE_RUN
+        || job->state == FLUX_JOB_STATE_CLEANUP
+        || job->eventlog_readonly) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    /* This interface is only appropriate from outside a jobtap callback,
+     * i.e. called asynchronously to update a job. If 'job' is equivalent
+     * to the current job at the top of the jobtap stack, return an error.
+     */
+    if (job == current_job (jobtap)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    va_start (ap, fmt);
+    update = json_vpack_ex (&error, 0, fmt, ap);
+    va_end (ap);
+    if (!update) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!validate_jobspec_updates (update)) {
+        errno = EINVAL;
+        goto out;
+    }
+    /* XXX: should job.validate be called on these updates before posting?
+     */
+    rc = event_job_post_pack (jobtap->ctx->event,
+                              job,
+                              "jobspec-update",
+                              0,
+                              "O",
+                              update);
+out:
+    ERRNO_SAFE_WRAP (json_decref, update);
+    return rc;
+}
+
+int flux_jobtap_jobspec_update_pack (flux_plugin_t *p, const char *fmt, ...)
+{
+    int rc = -1;
+    int saved_errno;
+    va_list ap;
+    struct jobtap *jobtap;
+    struct job * job;
+    json_t *o = NULL;
+    json_error_t error;
+
+    if (!p
+        || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))
+        || !(job = current_job (jobtap))
+        || job->state == FLUX_JOB_STATE_RUN
+        || job->state == FLUX_JOB_STATE_CLEANUP
+        || job->eventlog_readonly) {
+        errno = EINVAL;
+        return -1;
+    }
+    va_start (ap, fmt);
+    o = json_vpack_ex (&error, 0, fmt, ap);
+    va_end (ap);
+    if (!o) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!validate_jobspec_updates (o)) {
+        errno = EINVAL;
+        goto out;
+    }
+    if (!jobtap->jobspec_update)
+        jobtap->jobspec_update = json_incref (o);
+    else if (json_object_update (jobtap->jobspec_update, o) < 0) {
+        errno = EINVAL;
+        goto out;
+    }
+    rc = 0;
+out:
+    saved_errno = errno;
+    json_decref (o);
+    errno = saved_errno;
+    return rc;
+}
+
+int flux_jobtap_job_subscribe (flux_plugin_t *p, flux_jobid_t id)
+{
+    struct job *job;
+    if (!(job = jobtap_lookup_active_jobid (p, id)))
+        return -1;
+    return job_events_subscribe (job, p);
+}
+
+void flux_jobtap_job_unsubscribe (flux_plugin_t *p, flux_jobid_t id)
+{
+    struct job *job;
+    if ((job = jobtap_lookup_active_jobid (p, id)))
+        job_events_unsubscribe (job, p);
+}
+
+int flux_jobtap_job_event_posted (flux_plugin_t *p,
+                                  flux_jobid_t id,
+                                  const char *name)
+{
+    int index;
+    struct job * job;
+    struct jobtap *jobtap;
+
+    if (!p
+        || !name
+        || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(job = jobtap_lookup_jobid (p, id))
+        || (index = event_index (jobtap->ctx->event, name)) < 0)
+        return -1;
+    if (job_event_id_test (job, index))
+        return 1;
+    return 0;
+}
+
+static int jobtap_emit_perilog_event (struct jobtap *jobtap,
+                                      struct job *job,
+                                      bool prolog,
+                                      bool start,
+                                      const char *description,
+                                      int status)
+{
+    int flags = 0;
+    const char *event = prolog ? start ? "prolog-start" : "prolog-finish" :
+                                 start ? "epilog-start" : "epilog-finish";
+
+    if (!description) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    /*  prolog events cannot be emitted after a start request is pending.
+     *
+     *  epilog events cannot be emitted outside of CLEANUP state
+     *   and must be emitted before free request is pending.
+     */
+    if ((prolog && job->start_pending)
+        || ((prolog && start) && job->state == FLUX_JOB_STATE_CLEANUP)
+        || (!prolog && job->state != FLUX_JOB_STATE_CLEANUP)) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (start)
+        return event_job_post_pack (jobtap->ctx->event,
+                                    job,
+                                    event,
+                                    flags,
+                                    "{s:s}",
+                                    "description", description);
+    else
+        return event_job_post_pack (jobtap->ctx->event,
+                                    job,
+                                    event,
+                                    flags,
+                                    "{s:s s:i}",
+                                    "description", description,
+                                    "status", status);
+}
+
+int flux_jobtap_prolog_start (flux_plugin_t *p, const char *description)
+{
+    struct job * job;
+    struct jobtap *jobtap;
+
+    if (!p
+        || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))
+        || !(job = current_job (jobtap))) {
+        errno = EINVAL;
+        return -1;
+    }
+    return jobtap_emit_perilog_event (jobtap, job, true, true, description, 0);
+}
+
+int flux_jobtap_prolog_finish (flux_plugin_t *p,
+                               flux_jobid_t id,
+                               const char *description,
+                               int status)
+{
+    struct job * job;
+    struct jobtap *jobtap;
+
+    if (!p || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(job = jobtap_lookup_active_jobid (p, id)))
+        return -1;
+    return jobtap_emit_perilog_event (jobtap,
+                                      job,
+                                      true,
+                                      false,
+                                      description,
+                                      status);
+}
+
+int flux_jobtap_epilog_start (flux_plugin_t *p, const char *description)
+{
+    struct job * job;
+    struct jobtap *jobtap;
+
+    if (!p
+        || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))
+        || !(job = current_job (jobtap))) {
+        errno = EINVAL;
+        return -1;
+    }
+    return jobtap_emit_perilog_event (jobtap, job, false, true, description, 0);
+}
+
+int flux_jobtap_epilog_finish (flux_plugin_t *p,
+                               flux_jobid_t id,
+                               const char *description,
+                               int status)
+{
+    struct job * job;
+    struct jobtap *jobtap;
+
+    if (!p || !(jobtap = flux_plugin_aux_get (p, "flux::jobtap"))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(job = jobtap_lookup_active_jobid (p, id)))
+        return -1;
+    return jobtap_emit_perilog_event (jobtap,
+                                      job,
+                                      false,
+                                      false,
+                                      description,
+                                      status);
+}
+
+int jobtap_job_update (struct jobtap *jobtap,
+                       struct flux_msg_cred cred,
+                       struct job *job,
+                       const char *key,
+                       json_t *value,
+                       int *needs_validation,
+                       int *require_feasibility,
+                       json_t **additional_updates,
+                       char **errp)
+{
+    int rc = -1;
+    char topic[128];
+    int topiclen = sizeof (topic);
+    flux_plugin_arg_t *args = NULL;
+
+    if (snprintf (topic, topiclen, "job.update.%s", key) >= topiclen) {
+        error_asprintf (jobtap, job, errp,
+                        "topic string overflow");
+        return -1;
+    }
+
+    if (!(args = jobtap_args_create (jobtap, job))
+        || flux_plugin_arg_pack (args,
+                                 FLUX_PLUGIN_ARG_IN,
+                                 "{s:{s:I s:I} s:s s:O}",
+                                 "cred",
+                                   "userid", (json_int_t) cred.userid,
+                                   "rolemask", (json_int_t) cred.rolemask,
+                                 "key", key,
+                                 "value", value) < 0
+        || flux_plugin_arg_set (args, FLUX_PLUGIN_ARG_OUT, "{}") < 0) {
+        error_asprintf (jobtap, job, errp,
+                "jobtap_job_update: failed to create args");
+        flux_plugin_arg_destroy (args);
+        return -1;
+    }
+    rc = jobtap_stack_call (jobtap, jobtap->plugins, job, topic, args);
+    if (rc == 0) {
+        /* No plugin handles update of this jobspec key, reject the update.
+         */
+        error_asprintf (jobtap, job, errp, "update of %s not supported", key);
+        rc = -1;
+        errno = EINVAL;
+    }
+    else if (rc < 0) {
+        /* Callback failed, check for provided errmsg */
+        const char *errmsg;
+        if (flux_plugin_arg_unpack (args,
+                                    FLUX_PLUGIN_ARG_OUT,
+                                    "{s:s}",
+                                    "errmsg", &errmsg) < 0) {
+            errmsg = "update rejected by job-manager plugin";
+        }
+        error_asprintf (jobtap, job, errp, "%s", errmsg);
+        errno = EINVAL;
+    }
+    else if (rc > 0) {
+        /*  Default is to require further validation by calling job.validate
+         *  with the updated jobspec. However, a plugin may note that the
+         *  update is already validated or should bypass validation by
+         *  setting "validated" in the plugin OUT arguments to a nonzero
+         *  value.
+         *
+         *  Similarly, a plugin can request feasibility check, but the
+         *  default is no feasibility will be performed on valid updates.
+         */
+        int validated = 0;
+        int feasibility = 0;
+        json_t *updates = NULL;
+        if ((rc = flux_plugin_arg_unpack (args,
+                                          FLUX_PLUGIN_ARG_OUT,
+                                          "{s?i s?i s?o}",
+                                          "validated", &validated,
+                                          "feasibility", &feasibility,
+                                          "updates", &updates) < 0)) {
+            error_asprintf (jobtap,
+                            job,
+                            errp,
+                            "failed to unpack update flags");
+            return -1;
+        }
+        if (needs_validation != NULL)
+            *needs_validation = !validated;
+        if (require_feasibility != NULL)
+            *require_feasibility = feasibility;
+        if (additional_updates && updates) {
+            if (*additional_updates == NULL)
+                *additional_updates = json_incref (updates);
+            else if (json_object_update (*additional_updates, updates) < 0) {
+                error_asprintf (jobtap,
+                                job,
+                                errp,
+                                "failed to apply required extra job updates");
+                return -1;
+            }
+        }
+    }
+    flux_plugin_arg_destroy (args);
+    return rc;
+}
+
+int jobtap_validate_updates (struct jobtap *jobtap,
+                             struct job *job,
+                             json_t *updates,
+                             char **errp)
+{
+    int rc = -1;
+    json_t *jobspec_updated = NULL;
+    flux_plugin_arg_t *args = NULL;
+
+    if (!(jobspec_updated = job_jobspec_with_updates (job, updates))) {
+        error_asprintf (jobtap, job, errp, "update: %s", strerror (errno));
+        goto error;
+    }
+
+    /*  Create plugin args, then override jobspec with updated version
+     */
+    if (!(args = jobtap_args_create (jobtap, job))
+        || flux_plugin_arg_pack (args,
+                                 FLUX_PLUGIN_ARG_IN,
+                                 "{s:O}",
+                                 "jobspec", jobspec_updated) < 0) {
+        error_asprintf (jobtap, job, errp, "update: %s",
+                        flux_plugin_arg_strerror (args));
+        goto error;
+    }
+
+    /*  Call validation stack
+     */
+    rc = jobtap_stack_call (jobtap,
+                            jobtap->plugins,
+                            job,
+                            "job.validate",
+                            args);
+
+    if (rc < 0) {
+        const char *errmsg;
+        /*
+         *  Plugin callback failed, check for errmsg for this job
+         *   If plugin did not provide an error message, then construct
+         *   a generic error "rejected by plugin".
+         */
+        if (flux_plugin_arg_unpack (args,
+                                    FLUX_PLUGIN_ARG_OUT,
+                                    "{s:s}",
+                                    "errmsg", &errmsg) < 0)
+                errmsg = "rejected by job-manager plugin";
+        if ((*errp = strdup (errmsg)) == NULL)
+            flux_log (jobtap->ctx->h, LOG_ERR,
+                      "jobtap: validate failed to capture errmsg");
+        errno = EINVAL;
+    }
+error:
+    ERRNO_SAFE_WRAP (json_decref, jobspec_updated);
+    flux_plugin_arg_destroy (args);
+    return rc;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/modules/job-manager/jobtap.h b/src/modules/job-manager/jobtap.h
new file mode 100644
index 000000000000..1baa3a07f4d2
--- /dev/null
+++ b/src/modules/job-manager/jobtap.h
@@ -0,0 +1,282 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Flux job-manager "jobtap" plugin interface
+ */
+
+#ifndef FLUX_JOBTAP_H
+#define FLUX_JOBTAP_H
+
+#include <flux/core.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define FLUX_JOBTAP_CURRENT_JOB FLUX_JOBID_ANY
+
+/*  Get a copy of the flux_t handle which may then be used to
+ *   set timer, periodic, prep/check/idle watchers, etc.
+ */
+flux_t * flux_jobtap_get_flux (flux_plugin_t *p);
+
+/*  Register a service handler for `method` within the job-manager.
+ *   The service will be 'job-manager.<name>.<method>'
+ */
+int flux_jobtap_service_register (flux_plugin_t *p,
+                                  const char *method,
+                                  flux_msg_handler_f cb,
+                                  void *arg);
+
+/*  Extended version of above with capability of registering a service that
+ *   guests can access.
+ */
+int flux_jobtap_service_register_ex (flux_plugin_t *p,
+                                     const char *method,
+                                     uint32_t rolemask,
+                                     flux_msg_handler_f cb,
+                                     void *arg);
+
+/*  Start a loop to re-prioritize all jobs. The plugin "priority.get"
+ *   callback will be called for each job currently in SCHED or
+ *   PRIORITY states.
+ */
+int flux_jobtap_reprioritize_all (flux_plugin_t *p);
+
+/*  Set the priority of job `id` to `priority`. This does nothing
+ *   if the priority isn't changed from the current value or if the
+ *   job is not in the PRIORITY or SCHED states. O/w, a priority event
+ *   is generated, job->priority is updated, and the job may move from
+ *   PRIORITY->SCHED state.
+ */
+int flux_jobtap_reprioritize_job (flux_plugin_t *p,
+                                  flux_jobid_t id,
+                                  unsigned int priority);
+
+/*  Convenience function to return unavailable priority in PRIORITY state
+ */
+int flux_jobtap_priority_unavail (flux_plugin_t *p,
+                                  flux_plugin_arg_t *args);
+
+/*  Convenience function to set 'errstr' in return args to formatted message.
+ *   returns -1 to allow an idiom like
+ *
+ *   return flux_jobtap_error (p, args, "error message");
+ */
+int flux_jobtap_error (flux_plugin_t *p,
+                       flux_plugin_arg_t *args,
+                       const char *fmt, ...)
+                       __attribute__ ((format (printf, 3, 4)));
+
+/*  Convenience function to be used in job.validate callback to reject a job.
+ *   Identical to flux_jobtap_error () except if 'fmt' is NULL, a default
+ *   message of the form "rejected by job-manager plugin <name>" is substituted.
+ */
+int flux_jobtap_reject_job (flux_plugin_t *p,
+                            flux_plugin_arg_t *args,
+                            const char *fmt, ...)
+                            __attribute__ ((format (printf, 3, 4)));
+
+
+/*  Add a job dependency to a job with the given description. The dependency
+ *   will keep the job in the DEPEND state until it is removed later via the
+ *   flux_jobtap_dependency_remove() function.
+ *
+ *  This function is only valid from the job.state.depend callback.
+ *
+ *  A job dependency may only be added to a job once.
+ *
+ *  Returns 0 on success, or -1 on error with errno set:
+ *   - ENOENT: job 'id' not found
+ *   - EEXIST: dependency 'description' has already been used
+ *   - EINVAL: invalid argument
+ *
+ */
+int flux_jobtap_dependency_add (flux_plugin_t *p,
+                                flux_jobid_t id,
+                                const char *description);
+
+/*  Remove a currently active job dependency with the given description.
+ *   Once all outstanding job dependencies have been removed, then the job
+ *   will proceed out of the DEPEND state.
+ *
+ *  Returns 0 on success, -1 on failure with errno set:
+ *   ENOENT: job 'id' not found or 'description' is not a current dependency
+ *   EINVAL: invalid argument
+ *
+ */
+int flux_jobtap_dependency_remove (flux_plugin_t *p,
+                                   flux_jobid_t id,
+                                   const char *description);
+
+
+/*  Set plugin-specific data to an individual job by name.
+ *   The optional destructor, free_fn, will be called when `name` is
+ *   overwritten by a new value, or when the job is complete and its
+ *   underlying job object is destroyed.
+ *
+ *  If `id` is FLUX_JOBTAP_CURRENT_JOB, then the aux data will be attached
+ *   to the currently processed job, if any.
+ */
+int flux_jobtap_job_aux_set (flux_plugin_t *p,
+                             flux_jobid_t id,
+                             const char *name,
+                             void *val,
+                             flux_free_f free_fn);
+
+/*  Get plugin-specific data from a job.
+ *
+ *  If `id` is FLUX_JOBTAP_CURRENT_JOB, then the current job will be used.
+ */
+void * flux_jobtap_job_aux_get (flux_plugin_t *p,
+                                flux_jobid_t id,
+                                const char *name);
+
+
+/*  Delete plugin specific data `val` from a job.
+ *
+ *  If `id` is FLUX_JOBTAP_CURRENT_JOB then the current job will be used.
+ */
+int flux_jobtap_job_aux_delete (flux_plugin_t *p,
+                                flux_jobid_t id,
+                                void *val);
+
+/*  Set a named flag on job `id`.
+ */
+int flux_jobtap_job_set_flag (flux_plugin_t *p,
+                              flux_jobid_t id,
+                              const char *flag);
+
+
+/*  Raise an exception for job 'id' or current job if FLUX_JOBTAP_CURRENT_JOB
+ */
+int flux_jobtap_raise_exception (flux_plugin_t *p,
+                                 flux_jobid_t id,
+                                 const char *type,
+                                 int severity,
+                                 const char *fmt,
+                                 ...);
+
+/*  Post event 'name' to job `id` with optional context defined by
+ *   args `fmt, ...`.
+ *
+ *  If `id` is FLUX_JOBTAP_CURRENT_JOB then the event will be posted to
+ *   the current job.
+ */
+int flux_jobtap_event_post_pack (flux_plugin_t *p,
+                                 flux_jobid_t id,
+                                 const char *name,
+                                 const char *fmt,
+                                 ...);
+
+/*  Post a request to update jobspec for the current job. Accumulated updates
+ *  will be applied once the current callback returns so that the jobspec is
+ *  not modified while the plugin may have references to it.
+ *
+ *  The update is defined by one or more period-delimited keys and values
+ *  specified by `fmt`, ..., e.g.
+ *
+ *      flux_jobtap_jobspec_update_pack (p,
+ *                                       "{s:i s:s}",
+ *                                       "attributes.system.duration", 3600,
+ *                                       "attributes.system.queue", "batch");
+ *
+ *  Returns -1 with errno set to EINVAL for invalid arguments, if there is
+ *  no current job, or if the current job is in RUN, CLEANUP, or INACTIVE
+ *  states.
+ */
+int flux_jobtap_jobspec_update_pack (flux_plugin_t *p, const char *fmt, ...);
+
+/*  Similar to flux_jobtap_jobspec_update_pack(), but asynchronously update
+ *  a specific jobid. This version assumes the job is quiescent, so the event
+ *  is applied immediately.
+ *
+ *  Returns -1 with errno set to EINVAL for invalid arguments, if the job
+ *  does not exist, if the target job is in RUN, CLEANUP or INACTIVE states,
+ *  or if the function is called from a jobtap callback for the target job.
+ */
+int flux_jobtap_jobspec_update_id_pack (flux_plugin_t *p,
+                                        flux_jobid_t id,
+                                        const char *fmt,
+                                        ...);
+
+/*  Return a flux_plugin_arg_t object for a job.
+ *
+ *  The result can then be unpacked with flux_plugin_arg_unpack(3) to get
+ *   active job information such as userid, state, etc.
+ *
+ *  If `id` is not an active job or is not found in the job manager's
+ *   inactive job cache then NULL is returned with ENOENT set.
+ *
+ *  Caller must free with flux_plugin_arg_destroy(3).
+ */
+flux_plugin_arg_t * flux_jobtap_job_lookup (flux_plugin_t *p,
+                                            flux_jobid_t id);
+
+
+int flux_jobtap_get_job_result (flux_plugin_t *p,
+                                flux_jobid_t id,
+                                flux_job_result_t *resultp);
+
+/*  Return 1 if event 'name' has been posted to job 'id', 0 if not.
+ */
+int flux_jobtap_job_event_posted (flux_plugin_t *p,
+                                  flux_jobid_t id,
+                                  const char *name);
+
+/*
+ *  This function subscribes plugin 'p' to extra callbacks for job 'id'
+ *   such as job.event.<name> for each job event. The plugin must have a
+ *   handler registered that matches one or more of these callbacks.
+ */
+int flux_jobtap_job_subscribe (flux_plugin_t *p, flux_jobid_t id);
+
+
+/*  Unsubscribe plugin 'p' from extra callbacks for job 'id'
+ */
+void flux_jobtap_job_unsubscribe (flux_plugin_t *p, flux_jobid_t id);
+
+
+/*  Post an event to the current job eventlog indicating that a prolog
+ *   action has started. This will block the start request to the
+ *   execution system until `flux_jobtap_prolog_finish()` is called.
+ */
+int flux_jobtap_prolog_start (flux_plugin_t *p, const char *description);
+
+/*  Post an event to the eventlog for job id indicating that a prolog
+ *   action has finished. The description should match the description
+ *   of an outstanding prolog start event. `status` is informational
+ *   and should be 0 to indicate success, non-zero for failure.
+ */
+int flux_jobtap_prolog_finish (flux_plugin_t *p,
+                               flux_jobid_t id,
+                               const char *description,
+                               int status);
+
+/*  Post an event to the current job eventlog indicating that an epilog
+ *   action has started. This will block the free request to the
+ *   scheduler until `flux_jobtap_epilog_finish()` is called.
+ */
+int flux_jobtap_epilog_start (flux_plugin_t *p, const char *description);
+
+/*  Post an event to the eventlog for job id indicating that an epilog
+ *   action has finished. The description should match the description
+ *   of an outstanding epilog start event. `status` is informational
+ *   and should be 0 to indicate success, non-zero for failure.
+ */
+int flux_jobtap_epilog_finish (flux_plugin_t *p,
+                               flux_jobid_t id,
+                               const char *description,
+                               int status);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !FLUX_JOBTAP_H */
diff --git a/src/modules/job-manager/journal.c b/src/modules/job-manager/journal.c
new file mode 100644
index 000000000000..0fa83f9c8046
--- /dev/null
+++ b/src/modules/job-manager/journal.c
@@ -0,0 +1,403 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* journal.c - stream job events
+ *
+ * This allows another service to track detailed information about
+ * all jobs.  The journal consumer makes a job-manager.events-journal
+ * request with optional allow/deny filter and boolean 'full' flag:
+ *   {"full"?b, "allow"?{"name":1, ...}, "deny"?{"name:1, ...}}
+ *
+ * If "full" is true, the journal begins with all the inactive jobs.
+ * If "full" is false, the journal begins with all the active jobs.
+ * If "full" is unspecified, it is assumed to be false.
+ * If allow/deny rules are specified, they filter the job events by name.
+ *
+ * The journal consumer receives a stream of responses until the job
+ * manager is unloaded or the request is canceled.  Each response consists of
+ * an object containing a jobid, an array of events, and optional data:
+ *   {"id":I, "events":[], "jobspec"?s, "R"?s}
+ *
+ * During processing of the initial backlog, the events array will contain
+ * all the events posted so far for each job, plus R and jobspec if available.
+ * The jobs are returned in hash traversal order.  Once backlog processing
+ * is complete, a sentinel response is transmitted with id of FLUX_JOBID_ANY
+ * and an empty events array:
+ *   {"id":-1, "events":[]}
+ *
+ * The sentinel informs the consumer that it is now caught up and that future
+ * responses will be for events that are are posted in real time.
+ *
+ * Additional responses contain at most one event.  The redacted jobspec is
+ * included with the "submit" event.  The redacted R object is included
+ * with the "alloc" event.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libjob/idf58.h"
+#include "ccan/str/str.h"
+
+#include "conf.h"
+#include "job.h"
+#include "journal.h"
+
+struct journal {
+    struct job_manager *ctx;
+    flux_msg_handler_t **handlers;
+    struct flux_msglist *listeners;
+    int event_count;
+};
+
+struct journal_filter { // stored as aux item in request message
+    json_t *allow;      // allow, deny are owned by message
+    json_t *deny;
+};
+
+static bool allow_deny_check (const flux_msg_t *msg, const char *name)
+{
+    bool add_entry = true;
+    struct journal_filter *filter = flux_msg_aux_get (msg, "filter");
+
+    if (filter->allow) {
+        add_entry = false;
+        if (json_object_get (filter->allow, name))
+            add_entry = true;
+    }
+
+    if (add_entry && filter->deny) {
+        if (json_object_get (filter->deny, name))
+            add_entry = false;
+    }
+
+    return add_entry;
+}
+
+static bool allow_all (const flux_msg_t *msg)
+{
+    struct journal_filter *filter = flux_msg_aux_get (msg, "filter");
+    if (filter->allow || filter->deny)
+        return false;
+    return true;
+}
+
+int journal_process_event (struct journal *journal,
+                           flux_jobid_t id,
+                           const char *name,
+                           json_t *entry)
+{
+    struct job_manager *ctx = journal->ctx;
+    const flux_msg_t *msg;
+    json_t *o;
+
+    if (!(o = json_pack ("{s:I s:[O]}",
+                         "id", id,
+                         "events", entry)))
+        goto error;
+    if (streq (name, "submit")) {
+        struct job *job;
+        if (!(job = zhashx_lookup (ctx->active_jobs, &id))
+            || !job->jobspec_redacted
+            || json_object_set (o, "jobspec", job->jobspec_redacted) < 0)
+            goto error;
+    }
+    else if (streq (name, "alloc")) {
+        struct job *job;
+        if (!(job = zhashx_lookup (ctx->active_jobs, &id))
+            || !job->R_redacted
+            || json_object_set (o, "R", job->R_redacted) < 0)
+            goto error;
+    }
+    journal->event_count++;
+    msg = flux_msglist_first (journal->listeners);
+    while (msg) {
+        if (allow_deny_check (msg, name)
+            && flux_respond_pack (ctx->h, msg, "O", o) < 0) {
+            flux_log_error (ctx->h,
+                            "error responding to"
+                            " job-manager.events-journal request");
+        }
+        msg = flux_msglist_next (journal->listeners);
+    }
+    json_decref (o);
+    return 0;
+error:
+    flux_log_error (ctx->h,
+                    "error preparing journal response for %s %s",
+                    idf58 (id),
+                    name);
+    json_decref (o);
+    return 0;
+}
+
+static void filter_destroy (struct journal_filter *filter)
+{
+    ERRNO_SAFE_WRAP (free, filter);
+}
+
+static int send_job_events (struct job_manager *ctx,
+                            const flux_msg_t *msg,
+                            struct job *job)
+{
+    json_t *eventlog;
+    json_t *o = NULL;
+
+    if (allow_all (msg)) {
+        eventlog = json_incref (job->eventlog);
+    }
+    else {
+        size_t index;
+        json_t *entry;
+        const char *name;
+
+        if (!(eventlog = json_array ()))
+            goto nomem;
+        json_array_foreach (job->eventlog, index, entry) {
+            if (eventlog_entry_parse (entry, NULL, &name, NULL) < 0)
+                goto error;
+            if (!allow_deny_check (msg, name))
+                continue;
+            if (json_array_append (eventlog, entry) < 0)
+                goto nomem;
+        }
+    }
+    if (!(o = json_pack ("{s:I s:O}",
+                         "id", job->id,
+                         "events", eventlog)))
+        goto nomem;
+    if (job->jobspec_redacted) {
+        if (json_object_set (o, "jobspec", job->jobspec_redacted) < 0)
+            goto nomem;
+    }
+    if (job->R_redacted) {
+        if (json_object_set (o, "R", job->R_redacted) < 0)
+            goto nomem;
+    }
+    if (flux_respond_pack (ctx->h, msg, "O", o) < 0)
+        goto error;
+    json_decref (o);
+    json_decref (eventlog);
+    return 0;
+ nomem:
+    errno = ENOMEM;
+ error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    ERRNO_SAFE_WRAP (json_decref, eventlog);
+    return -1;
+}
+
+/* The entire backlog must be sent to a journal consumer before
+ * any new events can be generated, event if it's large.
+ */
+static int send_backlog (struct job_manager *ctx,
+                         const flux_msg_t *msg,
+                         bool full)
+{
+    struct job *job;
+    int job_count = zhashx_size (ctx->active_jobs);
+
+    if (full)
+        job_count += zhashx_size (ctx->inactive_jobs);
+
+    if (job_count > 0) {
+        flux_log (ctx->h,
+                  LOG_DEBUG,
+                  "begin sending journal backlog: %d jobs",
+                  job_count);
+    }
+
+    if (full) {
+        job = zhashx_first (ctx->inactive_jobs);
+        while (job) {
+            if (send_job_events (ctx, msg, job) < 0)
+                return -1;
+            job = zhashx_next (ctx->inactive_jobs);
+        }
+    }
+    job = zhashx_first (ctx->active_jobs);
+    while (job) {
+        if (send_job_events (ctx, msg, job) < 0)
+            return -1;
+        job = zhashx_next (ctx->active_jobs);
+    }
+
+    if (job_count > 0) {
+        flux_log (ctx->h,
+                  LOG_DEBUG,
+                  "finished sending journal backlog");
+    }
+    /* Send a special response with id = FLUX_JOB_ANY to demarcate the
+     * backlog from ongoing events.  The consumer may ignore this message.
+     */
+    if (flux_respond_pack (ctx->h,
+                           msg,
+                           "{s:I s:[]}",
+                           "id", FLUX_JOBID_ANY,
+                           "events") < 0)
+        return -1;
+    return 0;
+}
+
+static void journal_handle_request (flux_t *h,
+                                    flux_msg_handler_t *mh,
+                                    const flux_msg_t *msg,
+                                    void *arg)
+{
+    struct job_manager *ctx = arg;
+    const char *topic = "unknown";
+    struct journal *journal = ctx->journal;
+    struct journal_filter *filter;
+    int full = 0;
+    const char *errstr = NULL;
+
+    if (!(filter = calloc (1, sizeof (*filter))))
+        goto error;
+    if (flux_request_unpack (msg,
+                             &topic,
+                             "{s?o s?o s?b}",
+                             "allow", &filter->allow,
+                             "deny", &filter->deny,
+                             "full", &full) < 0
+        || flux_msg_aux_set (msg, "filter", filter,
+                             (flux_free_f)filter_destroy) < 0) {
+        filter_destroy (filter);
+        goto error;
+    }
+    if (!flux_msg_is_streaming (msg)) {
+        errno = EPROTO;
+        errstr = "job-manager.events requires streaming RPC flag";
+        goto error;
+    }
+
+    if (filter->allow && !json_is_object (filter->allow)) {
+        errno = EPROTO;
+        errstr = "job-manager.events allow should be an object";
+        goto error;
+    }
+
+    if (filter->deny && !json_is_object (filter->deny)) {
+        errno = EPROTO;
+        errstr = "job-manager.events deny should be an object";
+        goto error;
+    }
+
+    if (send_backlog (ctx, msg, full) < 0) {
+        flux_log_error (h, "error responding to %s", topic);
+        return;
+    }
+    if (flux_msglist_append (journal->listeners, msg) < 0)
+        goto error;
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to %s", topic);
+}
+
+json_t *journal_get_stats (struct journal *journal)
+{
+    json_t *o;
+
+    o = json_pack ("{s:i s:i}",
+                   "listeners", flux_msglist_count (journal->listeners),
+                   "events", journal->event_count);
+
+    return o;
+}
+
+static void journal_cancel_request (flux_t *h, flux_msg_handler_t *mh,
+                                    const flux_msg_t *msg, void *arg)
+{
+    struct job_manager *ctx = arg;
+
+    if (flux_msglist_cancel (h, ctx->journal->listeners, msg) < 0)
+        flux_log_error (h, "error handling job-manager.events-journal-cancel");
+}
+
+void journal_listeners_disconnect_rpc (flux_t *h,
+                                       flux_msg_handler_t *mh,
+                                       const flux_msg_t *msg,
+                                       void *arg)
+{
+    struct job_manager *ctx = arg;
+
+    if (flux_msglist_disconnect (ctx->journal->listeners, msg) < 0)
+        flux_log_error (h, "error handling job-manager.disconnect (journal)");
+}
+
+void journal_ctx_destroy (struct journal *journal)
+{
+    if (journal) {
+        int saved_errno = errno;
+        flux_t *h = journal->ctx->h;
+
+        flux_msg_handler_delvec (journal->handlers);
+        if (journal->listeners) {
+            const flux_msg_t *msg;
+
+            msg = flux_msglist_first (journal->listeners);
+            while (msg) {
+                if (flux_respond_error (h, msg, ENODATA, NULL) < 0)
+                    flux_log_error (h, "error responding to journal request");
+                flux_msglist_delete (journal->listeners);
+                msg = flux_msglist_next (journal->listeners);
+            }
+            flux_msglist_destroy (journal->listeners);
+        }
+        free (journal);
+        errno = saved_errno;
+    }
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.events-journal",
+        journal_handle_request,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.events-journal-cancel",
+        journal_cancel_request,
+        0
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+struct journal *journal_ctx_create (struct job_manager *ctx)
+{
+    struct journal *journal;
+
+    if (!(journal = calloc (1, sizeof (*journal))))
+        return NULL;
+    journal->ctx = ctx;
+    if (flux_msg_handler_addvec (ctx->h, htab, ctx, &journal->handlers) < 0)
+        goto error;
+    if (!(journal->listeners = flux_msglist_create ()))
+        goto error;
+    return journal;
+error:
+    journal_ctx_destroy (journal);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/modules/job-manager/journal.h b/src/modules/job-manager/journal.h
new file mode 100644
index 000000000000..1c3dde25b88e
--- /dev/null
+++ b/src/modules/job-manager/journal.h
@@ -0,0 +1,44 @@
+
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_JOURNAL_H
+#define _FLUX_JOB_MANAGER_JOURNAL_H
+
+#include <stdarg.h>
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "job-manager.h"
+
+/* Process the event by sending to any listeners that request the
+ * event and append to the journal history.
+ */
+int journal_process_event (struct journal *journal,
+                           flux_jobid_t id,
+                           const char *name,
+                           json_t *entry);
+
+void journal_ctx_destroy (struct journal *journal);
+struct journal *journal_ctx_create (struct job_manager *ctx);
+
+void journal_listeners_disconnect_rpc (flux_t *h,
+                                       flux_msg_handler_t *mh,
+                                       const flux_msg_t *msg,
+                                       void *arg);
+
+json_t *journal_get_stats (struct journal *journal);
+
+#endif /* _FLUX_JOB_MANAGER_JOURNAL_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/modules/job-manager/kill.c b/src/modules/job-manager/kill.c
index a9ac40d4626f..513c4f7846a4 100644
--- a/src/modules/job-manager/kill.c
+++ b/src/modules/job-manager/kill.c
@@ -30,10 +30,12 @@
 #include <signal.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
 #include "job.h"
 #include "event.h"
 #include "kill.h"
-#include <job-manager.h>
+#include "job-manager.h"
 
 #ifndef SIGRTMAX
 #  define SIGRTMAX 64
@@ -72,9 +74,11 @@ void kill_handle_request (flux_t *h,
     char topic [64];
     const char *errstr = NULL;
 
-    if (flux_request_unpack (msg, NULL, "{s:I s:i}",
-                                        "id", &id,
-                                        "signum", &sig) < 0)
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:i}",
+                             "id", &id,
+                             "signum", &sig) < 0)
         goto error;
     if (kill_check_signal (sig) < 0) {
         errstr = "Invalid signal number";
@@ -82,7 +86,10 @@ void kill_handle_request (flux_t *h,
         goto error;
     }
     if (!(job = zhashx_lookup (ctx->active_jobs, &id))) {
-        errstr = "unknown job id";
+        if (!(job = zhashx_lookup (ctx->inactive_jobs, &id)))
+            errstr = "unknown job id";
+        else
+            errstr = "job is inactive";
         errno = EINVAL;
         goto error;
     }
@@ -90,7 +97,7 @@ void kill_handle_request (flux_t *h,
         errstr = "guests may only send signals to their own jobs";
         goto error;
     }
-    if (job->state != FLUX_JOB_RUN) {
+    if (!(job->state & FLUX_JOB_STATE_RUNNING)) {
         errstr = "job is not running";
         errno = EINVAL;
         goto error;
@@ -132,12 +139,9 @@ void killall_handle_request (flux_t *h,
     if (flux_request_unpack (msg,
                              NULL,
                              "{s:b s:i s:i}",
-                             "dry_run",
-                             &dry_run,
-                             "userid",
-                             &userid,
-                             "signum",
-                             &signum) < 0) {
+                             "dry_run", &dry_run,
+                             "userid", &userid,
+                             "signum", &signum) < 0) {
         errstr = "error decoding request";
         goto error;
     }
@@ -155,7 +159,7 @@ void killall_handle_request (flux_t *h,
     }
     job = zhashx_first (ctx->active_jobs);
     while (job) {
-        if (job->state != FLUX_JOB_RUN)
+        if (!(job->state & FLUX_JOB_STATE_RUNNING))
             goto next;
         if (userid != FLUX_USERID_UNKNOWN && userid != job->userid)
             goto next;
@@ -169,8 +173,7 @@ void killall_handle_request (flux_t *h,
                                                topic,
                                                0,
                                                "{s:i}",
-                                               "signum",
-                                               signum))) {
+                                               "signum", signum))) {
                 error_count++;
                 goto next;
             }
@@ -182,10 +185,8 @@ void killall_handle_request (flux_t *h,
     if (flux_respond_pack (h,
                            msg,
                            "{s:i s:i}",
-                           "count",
-                           count,
-                           "errors",
-                           error_count) < 0)
+                           "count", count,
+                           "errors", error_count) < 0)
         flux_log_error (h, "%s: flux_respond", __FUNCTION__);
     return;
 error:
diff --git a/src/modules/job-manager/list.c b/src/modules/job-manager/list.c
index 784ae5c562c7..0288f8847169 100644
--- a/src/modules/job-manager/list.c
+++ b/src/modules/job-manager/list.c
@@ -27,6 +27,7 @@
 #include <flux/core.h>
 
 #include "src/common/libjob/job.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "job.h"
 #include "list.h"
@@ -39,20 +40,33 @@ int list_append_job (json_t *jobs, struct job *job)
 {
     json_t *o;
 
-    if (!(o = json_pack ("{s:I s:i s:i s:f s:i}",
-                         "id",
-                         job->id,
-                         "userid",
-                         job->userid,
-                         "priority",
-                         job->priority,
-                         "t_submit",
-                         job->t_submit,
-                         "state",
-                         job->state))) {
+    /* We pack priority with I instead of i to avoid issue of
+     * signed vs unsigned int */
+    if (!(o = json_pack ("{s:I s:I s:i s:I s:f s:i}",
+                         "id", job->id,
+                         "userid", (json_int_t) job->userid,
+                         "urgency", job->urgency,
+                         "priority", job->priority,
+                         "t_submit", job->t_submit,
+                         "state", job->state))) {
         errno = ENOMEM;
         return -1;
     }
+    if (job->annotations) {
+        if (json_object_set (o, "annotations", job->annotations) < 0) {
+            json_decref (o);
+            errno = ENOMEM;
+            return -1;
+        }
+    }
+    if (job->dependencies) {
+        json_t *deps = grudgeset_tojson (job->dependencies);
+        if (json_object_set (o, "dependencies", deps) < 0) {
+            json_decref (o);
+            errno = ENOMEM;
+            return -1;
+        }
+    }
     if (json_array_append_new (jobs, o) < 0) {
         json_decref (o);
         errno = ENOMEM;
@@ -74,8 +88,7 @@ void list_handle_request (flux_t *h,
     if (flux_request_unpack (msg,
                              NULL,
                              "{s:i}",
-                             "max_entries",
-                             &max_entries) < 0)
+                             "max_entries", &max_entries) < 0)
         goto error;
     if (max_entries < 0) {
         errno = EPROTO;
@@ -86,7 +99,7 @@ void list_handle_request (flux_t *h,
         goto error;
     }
     /* First list jobs in SCHED (S) state
-     * (priority, then t_submit order).
+     * (urgency, then job id order).
      */
     job = alloc_queue_first (ctx->alloc);
     while (job && (max_entries == 0 || json_array_size (jobs) < max_entries)) {
diff --git a/src/modules/job-manager/plugins/alloc-bypass.c b/src/modules/job-manager/plugins/alloc-bypass.c
new file mode 100644
index 000000000000..95363d6fa11f
--- /dev/null
+++ b/src/modules/job-manager/plugins/alloc-bypass.c
@@ -0,0 +1,230 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* alloc-bypass.c - If attributes.system.R exists in jobspec, then
+ *  bypass scheduler alloc protocol and use R directly (for instance
+ *  owner use only)
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <sys/types.h>
+
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "src/common/librlist/rlist.h"
+#include "src/common/libjob/idf58.h"
+
+static void alloc_continuation (flux_future_t *f, void *arg)
+{
+    flux_plugin_t *p = arg;
+    flux_jobid_t *idptr = flux_future_aux_get (f, "jobid");
+
+    if (flux_future_get (f, NULL) < 0) {
+        flux_jobtap_raise_exception (p,
+                                     *idptr,
+                                     "alloc", 0,
+                                     "failed to commit R to kvs: %s",
+                                      strerror (errno));
+        goto done;
+    }
+    if (flux_jobtap_event_post_pack (p,
+                                     *idptr,
+                                     "alloc",
+                                     "{s:b}",
+                                     "bypass", true) < 0) {
+        flux_jobtap_raise_exception (p,
+                                     *idptr,
+                                     "alloc", 0,
+                                     "failed to post alloc event: %s",
+                                     strerror (errno));
+        goto done;
+    }
+done:
+    flux_future_destroy (f);
+}
+
+static flux_future_t *commit_R (flux_plugin_t *p,
+                                flux_jobid_t id,
+                                json_t *R)
+{
+    flux_future_t *f = NULL;
+    flux_kvs_txn_t *txn = NULL;
+    flux_t *h = flux_jobtap_get_flux (p);
+    char key[64];
+
+    if (!h
+        || flux_job_kvs_key (key, sizeof (key), id, "R") < 0
+        || !(txn = flux_kvs_txn_create ()))
+        return NULL;
+
+    if (flux_kvs_txn_pack (txn, 0, key, "O", R) < 0)
+        goto out;
+
+    f = flux_kvs_commit (h, NULL, 0, txn);
+out:
+    flux_kvs_txn_destroy (txn);
+    return f;
+}
+
+static int alloc_start (flux_plugin_t *p,
+                        flux_jobid_t id,
+                        json_t *R)
+{
+    int saved_errno;
+    flux_future_t *f = NULL;
+    flux_jobid_t *idptr = NULL;
+
+    if (!(f = commit_R (p, id, R))
+        || flux_future_then (f, -1, alloc_continuation, p) < 0)
+        goto error;
+
+    if (!(idptr = malloc (sizeof (*idptr)))
+        || flux_future_aux_set (f, "jobid", idptr, free) < 0)
+        goto error;
+
+    *idptr = id;
+    return 0;
+error:
+    saved_errno = errno;
+    flux_future_destroy (f);
+    free (idptr);
+    errno = saved_errno;
+    return -1;
+}
+
+static int sched_cb (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *arg)
+{
+    json_t *R;
+    flux_jobid_t id;
+
+    /*  If alloc-bypass::R set on this job then commit R to KVS
+     *   and set alloc-bypass flag
+     */
+    if (!(R = flux_jobtap_job_aux_get (p,
+                                       FLUX_JOBTAP_CURRENT_JOB,
+                                       "alloc-bypass::R")))
+        return 0;
+
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I}",
+                                "id", &id) < 0) {
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "alloc", 0,
+                                     "alloc-bypass: %s: unpack: %s",
+                                     topic,
+                                     flux_plugin_arg_strerror (args));
+        return -1;
+    }
+
+    if (alloc_start (p, id, R) < 0)
+        flux_jobtap_raise_exception (p, id, "alloc", 0,
+                                     "failed to commit R to kvs");
+    if (flux_plugin_arg_pack (args, FLUX_PLUGIN_ARG_OUT, "{s:O}", "R", R) < 0)
+        return -1;
+
+    return 0;
+}
+
+static int validate_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *arg)
+{
+    json_t *R = NULL;
+    struct rlist *rl;
+    json_error_t error;
+    uint32_t userid = (uint32_t) -1;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:i s:{s:{s?{s?{s?o}}}}}",
+                                "userid", &userid,
+                                "jobspec",
+                                 "attributes",
+                                  "system",
+                                   "alloc-bypass",
+                                    "R", &R) < 0) {
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "invalid system.alloc-bypass.R: %s",
+                                       flux_plugin_arg_strerror (args));
+    }
+
+    /*  Nothing to do if no R provided
+     */
+    if (R == NULL)
+        return 0;
+
+    if (userid != getuid ())
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "Guest user cannot use alloc bypass");
+
+    /*  Sanity check R for validity
+     */
+    if (!(rl = rlist_from_json (R, &error)))
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "alloc-bypass: invalid R: %s",
+                                       error.text);
+    rlist_destroy (rl);
+
+    /*  Store R in job structure to avoid re-fetching from plugin args
+     *   in job.state.sched callback.
+     */
+    if (flux_jobtap_job_aux_set (p,
+                                 FLUX_JOBTAP_CURRENT_JOB,
+                                 "alloc-bypass::R",
+                                 json_incref (R),
+                                 (flux_free_f)json_decref) < 0) {
+        int saved_errno = errno;
+        json_decref (R);
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "failed to capture alloc-bypass R: %s",
+                                       strerror (saved_errno));
+    }
+
+    if (flux_jobtap_job_set_flag (p,
+                                  FLUX_JOBTAP_CURRENT_JOB,
+                                  "alloc-bypass") < 0) {
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "alloc", 0,
+                                     "Failed to set alloc-bypass: %s",
+                                     strerror (errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.state.sched",   sched_cb,    NULL },
+    { "job.validate",      validate_cb, NULL },
+    { 0 }
+};
+
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_register (p, "alloc-bypass", tab);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/plugins/alloc-check.c b/src/modules/job-manager/plugins/alloc-check.c
new file mode 100644
index 000000000000..9c3689726808
--- /dev/null
+++ b/src/modules/job-manager/plugins/alloc-check.c
@@ -0,0 +1,205 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* alloc-check.c - plugin to ensure resources are never double booked
+ *
+ * A fatal exception is raised on jobs that are granted resources already
+ * granted to another.
+ *
+ * N.B.  This plugin does not account for any jobs that might already have
+ * allocations when the plugin is loaded.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "ccan/str/str.h"
+#include "src/common/librlist/rlist.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libeventlog/eventlog.h"
+
+#define PLUGIN_NAME    "alloc-check"
+static const char *auxname = PLUGIN_NAME "::resdb";
+
+/* Start out with empty resource set.  Add resources on job.event.alloc
+ * (scheduler has allocated resources to job).  Subtract resources on
+ * job.event.free (job manager has returned resources to the scheduler).
+ */
+struct resdb {
+    struct rlist *allocated;
+};
+
+static void resdb_destroy (struct resdb *resdb)
+{
+    if (resdb) {
+        int saved_errno = errno;
+        rlist_destroy (resdb->allocated);
+        free (resdb);
+        errno = saved_errno;
+    }
+}
+
+static struct resdb *resdb_create (void)
+{
+    struct resdb *resdb;
+
+    if (!(resdb = calloc (1, sizeof (*resdb))))
+        return NULL;
+    if (!(resdb->allocated = rlist_create())) {
+        free (resdb);
+        errno = ENOMEM;
+        return NULL;
+    }
+    return resdb;
+}
+
+/* When a job is presented to the scheduler via the RFC 27 'hello' handshake
+ * upon scheduler reload, the scheduler raises a fatal scheduler-restart
+ * exception if it cannot re-allocate the job's resources and the job manager
+ * marks resources free without posting a free event.  This plugin must
+ * account for those resources.  See flux-framework/flux-core#5889
+ */
+static bool is_hello_failure (json_t *entry)
+{
+    const char *type;
+    int severity;
+    json_t *context;
+
+    if (eventlog_entry_parse (entry, NULL, NULL, &context) == 0
+        && json_unpack (context,
+                        "{s:i s:s}",
+                        "severity", &severity,
+                        "type", &type) == 0
+        && severity == 0
+        && streq (type, "scheduler-restart"))
+        return true;
+    return false;
+}
+
+static int jobtap_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *arg)
+{
+    struct resdb *resdb = flux_plugin_aux_get (p, auxname);
+    flux_t *h = flux_jobtap_get_flux (p);
+    flux_jobid_t id;
+    json_t *entry = NULL;
+    json_t *R = NULL;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s?o s?o}",
+                                "id", &id,
+                                "entry", &entry,
+                                "R", &R) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "%s %s: unpack: %s",
+                  PLUGIN_NAME,
+                  topic,
+                  flux_plugin_arg_strerror (args));
+        return -1;
+    }
+    /* job.event.* callbacks are not received unless subscribed on a per-job
+     * basis, so subscribe to them in the job.new callback.
+     */
+    if (streq (topic, "job.new")) {
+        if (flux_jobtap_job_subscribe (p, id) < 0) {
+            flux_log_error (h,
+                            "%s(%s) %s: subscribe",
+                            PLUGIN_NAME,
+                            idf58 (id),
+                            topic);
+        }
+    }
+    /* Attach R that was just allocated to the job to the job aux container
+     * so we don't have to parse it again on free.  Call rlist_append() to add
+     * the resources to resdb->allocated.  If that fails, some resources are
+     * already allocated so raise a fatal exception on the job.
+     */
+    else if (streq (topic, "job.event.alloc")) {
+        struct rlist *rl = NULL;
+        if (!R
+            || !(rl = rlist_from_json (R, NULL))
+            || flux_jobtap_job_aux_set (p,
+                                        id,
+                                        PLUGIN_NAME "::R",
+                                        rl,
+                                        (flux_free_f)rlist_destroy) < 0) {
+            flux_log_error (h,
+                            "%s(%s) %s: failed to parse or cache R",
+                            PLUGIN_NAME,
+                            idf58 (id),
+                            topic);
+            rlist_destroy (rl);
+            return -1;
+        }
+        if (rlist_append (resdb->allocated, rl) < 0) {
+            flux_jobtap_raise_exception (p,
+                                         id,
+                                         "alloc-check",
+                                         0,
+                                         "resources already allocated");
+        }
+    }
+    /* Get R that was just freed from the job's aux container and remove it
+     * from resdb->allocated.  Any jobs that had allocations before the module
+     * will not have the R aux item, so silently return success in that case.
+     */
+    else if (streq (topic, "job.event.free")
+        || (streq (topic, "job.event.exception") && is_hello_failure (entry))) {
+        struct rlist *rl = flux_jobtap_job_aux_get (p, id, PLUGIN_NAME "::R");
+        if (rl) {
+            struct rlist *diff;
+            if (!(diff = rlist_diff (resdb->allocated, rl))) {
+                flux_log_error (h,
+                                "%s(%s) %s: rlist_diff",
+                                PLUGIN_NAME,
+                                idf58 (id),
+                                topic);
+                return -1;
+            }
+            rlist_destroy (resdb->allocated);
+            resdb->allocated = diff;
+        }
+    }
+    return 0;
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.event.alloc",   jobtap_cb,    NULL },
+    { "job.event.free",    jobtap_cb,    NULL },
+    { "job.event.exception", jobtap_cb,  NULL },
+    { "job.new",           jobtap_cb,    NULL },
+    { 0 }
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    struct resdb *resdb;
+
+    if (!(resdb = resdb_create ())
+        || flux_plugin_aux_set (p,
+                                auxname,
+                                resdb,
+                                (flux_free_f)resdb_destroy) < 0) {
+        resdb_destroy (resdb);
+        return -1;
+    }
+    return flux_plugin_register (p, "alloc-check", tab);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/plugins/begin-time.c b/src/modules/job-manager/plugins/begin-time.c
new file mode 100644
index 000000000000..6a2fe7ea3463
--- /dev/null
+++ b/src/modules/job-manager/plugins/begin-time.c
@@ -0,0 +1,190 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/*  begin-time: Builtin job-manager begin-time dependency plugin */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <time.h>
+#include <math.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "src/common/libjob/idf58.h"
+
+struct begin_time_arg {
+    flux_plugin_t *p;
+    flux_watcher_t *w;
+    flux_jobid_t id;
+    double begin_time;
+    char desc [128];
+};
+
+static void begin_time_arg_destroy (struct begin_time_arg *b)
+{
+    if (b) {
+        flux_watcher_destroy (b->w);
+        free (b);
+    }
+}
+
+static struct begin_time_arg * begin_time_arg_create (flux_plugin_t *p,
+                                                      flux_jobid_t id,
+                                                      double begin_time)
+{
+    struct begin_time_arg *b = NULL;
+    int len = sizeof (b->desc);
+
+    if (!(b = calloc (1, sizeof (*b)))
+        || snprintf (b->desc, len, "begin-time=%.3f", begin_time) >= len)
+        goto err;
+    b->p = p;
+    b->id = id;
+    b->begin_time = begin_time;
+    return b;
+err:
+    begin_time_arg_destroy (b);
+    return NULL;
+}
+
+static void begin_time_cb (flux_reactor_t *r,
+                           flux_watcher_t *w,
+                           int revents,
+                           void *arg)
+{
+    struct begin_time_arg *b = arg;
+    flux_t *h = flux_jobtap_get_flux (b->p);
+    if (flux_jobtap_dependency_remove (b->p, b->id, b->desc) < 0)
+        flux_log_error (h, "begin-time: flux_jobtap_dependency_remove");
+    if (flux_jobtap_job_aux_delete (b->p, b->id, b) < 0)
+        flux_log_error (h, "begin-time: flux_jobtap_job_aux_delete");
+}
+
+
+static int add_begin_time (flux_plugin_t *p,
+                           flux_t *h,
+                           flux_reactor_t *r,
+                           flux_jobid_t id,
+                           double begin_time)
+{
+    struct begin_time_arg *arg = NULL;
+
+    if (!(arg = begin_time_arg_create (p, id, begin_time))) {
+        flux_log (h, LOG_ERR, "failed to create begin-time args");
+        goto error;
+    }
+
+    if (!(arg->w = flux_periodic_watcher_create (r,
+                                                 begin_time,
+                                                 0.,
+                                                 NULL,
+                                                 begin_time_cb,
+                                                 arg))) {
+        flux_log_error (h, "flux_periodic_watcher_create");
+        goto error;
+    }
+    flux_watcher_start (arg->w);
+
+    if (flux_jobtap_dependency_add (p, id, arg->desc) < 0) {
+        flux_log_error (h, "%s: flux_jobtap_dependency_add", idf58 (id));
+        goto error;
+    }
+
+    /*  In case job is destroyed before begin_time, tie destruction
+     *   of this watcher to the current job.
+     */
+    if (flux_jobtap_job_aux_set (p,
+                                 FLUX_JOBTAP_CURRENT_JOB,
+                                 "flux::begin-time",
+                                 arg,
+                                 (flux_free_f) begin_time_arg_destroy) < 0) {
+        flux_log_error (h, "flux_jobtap_job_aux_set");
+        goto error;
+    }
+    return 0;
+error:
+    begin_time_arg_destroy (arg);
+    return -1;
+}
+
+/*  Parse string 's' to a floating-point timestamp,
+ *   ensuring validity of the result.
+ */
+static int parse_timestamp (const char *s, double *dp)
+{
+    double d;
+    char *p;
+    if (s == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    d = strtod (s, &p);
+
+    /*  Ensure d is a valid timestamp
+     */
+    if (d < 0. || isnan(d) || isinf(d) || *p != '\0') {
+        errno = EINVAL;
+        return -1;
+    }
+    *dp = d;
+    return 0;
+}
+
+/*  Handle job.dependency.begin-time requests
+ */
+static int depend_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *arg)
+{
+    flux_jobid_t id;
+    const char *s;
+    double begin_time = 0.;
+    flux_reactor_t *r;
+    flux_t *h = flux_jobtap_get_flux (p);
+
+    if (!h || !(r = flux_get_reactor (h)))
+        return -1;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:{s:s}}",
+                                "id", &id,
+                                "dependency",
+                                  "value", &s) < 0)
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "error processing begin-time: %s",
+                                       flux_plugin_arg_strerror (args));
+    if (parse_timestamp (s, &begin_time) < 0)
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "Invalid begin-time=%s",
+                                       s);
+
+    if (add_begin_time (p, h, r, id, begin_time) < 0)
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "Unable to initialize begin-time");
+    return 0;
+}
+
+int begin_time_plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_add_handler (p,
+                                    "job.dependency.begin-time",
+                                    depend_cb,
+                                    NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/plugins/dependency-after.c b/src/modules/job-manager/plugins/dependency-after.c
new file mode 100644
index 000000000000..ce47d71c2f10
--- /dev/null
+++ b/src/modules/job-manager/plugins/dependency-after.c
@@ -0,0 +1,719 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* dependency-simple.c - don't start a job until after another starts,
+ *   completes, or fails.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "src/common/libutil/iterators.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
+static zlistx_t *global_reflist = NULL;
+
+/* Types of "after*" dependencies:
+ */
+enum after_type {
+    AFTER_START =   0x1,
+    AFTER_FINISH =  0x2,
+    AFTER_SUCCESS = 0x4,
+    AFTER_FAILURE = 0x8
+};
+
+struct after_info {
+    enum after_type type;
+    flux_jobid_t depid;
+    char *description;
+};
+
+/*  Reference to an after_info object on another job's dependency list
+ */
+struct after_ref {
+    flux_jobid_t id;
+    zlistx_t *list;
+    struct after_info *info;
+};
+
+static const char * after_typestr (enum after_type type)
+{
+    switch (type) {
+        case AFTER_START:
+            return "after-start";
+        case AFTER_FINISH:
+            return "after-finish";
+        case AFTER_SUCCESS:
+            return "after-success";
+        case AFTER_FAILURE:
+            return "after-failure";
+    }
+    return "";
+}
+
+static int after_type_parse (const char *s, enum after_type *tp)
+{
+    if (streq (s, "after"))
+        *tp = AFTER_START;
+    else if (streq (s, "afterany"))
+        *tp = AFTER_FINISH;
+    else if (streq (s, "afterok"))
+        *tp = AFTER_SUCCESS;
+    else if (streq (s, "afternotok"))
+        *tp = AFTER_FAILURE;
+    else
+        return -1;
+    return 0;
+}
+
+static void after_info_destroy (struct after_info *after)
+{
+    if (after) {
+        free (after->description);
+        free (after);
+    }
+}
+
+/*  zlistx_destructor_fn for after_info objects
+ */
+static void after_info_destructor (void **item)
+{
+    if (*item) {
+        after_info_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static struct after_info *after_info_create (flux_jobid_t id,
+                                             enum after_type type,
+                                             const char *desc)
+{
+    struct after_info *after = calloc (1, sizeof (*after));
+    if (!after
+        || asprintf (&after->description,
+                     "%s=%s",
+                     after_typestr (type),
+                     desc) < 0)
+        goto error;
+    after->type = type;
+    after->depid = id;
+    return after;
+error:
+    after_info_destroy (after);
+    return NULL;
+}
+
+static void after_ref_destroy (struct after_ref *ref)
+{
+    free (ref);
+}
+
+/*  zlistx_destructor_fn for after_ref objects
+ */
+static void after_ref_destructor (void **item)
+{
+    if (*item) {
+        void *handle = zlistx_find (global_reflist, *item);
+        if (handle)
+            zlistx_delete (global_reflist, handle);
+        after_ref_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static struct after_ref * after_ref_create (flux_jobid_t id,
+                                            zlistx_t *l,
+                                            struct after_info *after)
+{
+    struct after_ref *ref = calloc (1, sizeof (*ref));
+    if (!ref)
+        return NULL;
+    ref->id = id;
+    ref->list = l;
+    ref->info = after;
+    zlistx_add_end (global_reflist, ref);
+    return ref;
+}
+
+/*  flux_free_f destructor for zlistx_t
+ */
+static void list_destructor (void *arg)
+{
+    zlistx_t *l = arg;
+    zlistx_destroy (&l);
+}
+
+/*  Get or create a list embedded in jobid id
+ */
+static zlistx_t * embedded_list_get (flux_plugin_t *p,
+                                     flux_jobid_t id,
+                                     const char *name,
+                                     zlistx_destructor_fn destructor)
+{
+    zlistx_t *l = flux_jobtap_job_aux_get (p, id, name);
+    if (!l) {
+        if (!(l = zlistx_new ())) {
+            errno = ENOMEM;
+            return NULL;
+        }
+        if (flux_jobtap_job_aux_set (p, id, name, l, list_destructor) < 0) {
+            zlistx_destroy (&l);
+            errno = ENOENT;
+            return NULL;
+        }
+        if (destructor)
+            zlistx_set_destructor (l, destructor);
+    }
+    return l;
+}
+
+static zlistx_t * after_list_get (flux_plugin_t *p, flux_jobid_t id)
+{
+    return embedded_list_get (p, id,
+                              "flux::after_list",
+                              (zlistx_destructor_fn *) after_info_destructor);
+}
+
+static zlistx_t * after_refs_get (flux_plugin_t *p, flux_jobid_t id)
+{
+    return embedded_list_get (p, id,
+                              "flux::after_refs",
+                              (zlistx_destructor_fn *) after_ref_destructor);
+}
+
+static zlistx_t *after_list_check (flux_plugin_t *p, flux_jobid_t id)
+{
+    return flux_jobtap_job_aux_get (p,
+                                    id > 0 ? id : FLUX_JOBTAP_CURRENT_JOB,
+                                    "flux::after_list");
+}
+
+static zlistx_t *after_refs_check (flux_plugin_t *p)
+{
+    return flux_jobtap_job_aux_get (p,
+                                    FLUX_JOBTAP_CURRENT_JOB,
+                                    "flux::after_refs");
+}
+
+/*  Lookup a job and return its userid and state information.
+ *
+ *   `id` may  be FLUX_JOBTAP_CURRENT_JOB to return information for
+ *   the current jobtap jobid.
+ */
+static int lookup_job_uid_state (flux_plugin_t *p,
+                                 flux_jobid_t id,
+                                 uint32_t *puid,
+                                 flux_job_state_t *pstate)
+{
+    int rc = 0;
+    flux_plugin_arg_t *args = flux_jobtap_job_lookup (p, id);
+    if (!args
+        || flux_plugin_arg_unpack (args,
+                                   FLUX_PLUGIN_ARG_IN,
+                                   "{s:i s:i}",
+                                   "userid", puid,
+                                   "state", pstate) < 0)
+        rc = -1;
+    flux_plugin_arg_destroy (args);
+    return rc;
+}
+
+/*  Handle a job in INACTIVE state.
+ *
+ *  Get the job result and check for various error states (e.g. afterok
+ *   and job already completed with failure, after and job had an exception
+ *   before it was started, etc.)
+ */
+static int dependency_handle_inactive (flux_plugin_t *p,
+                                       flux_plugin_arg_t *args,
+                                       struct after_info *after,
+                                       flux_jobid_t afterid,
+                                       const char *jobid)
+{
+    int rc = -1;
+    flux_job_result_t result;
+    enum after_type type = after->type;
+
+    if (flux_jobtap_get_job_result (p, afterid, &result) < 0)
+        return flux_jobtap_reject_job (p, args,
+                                       "dependency: failed to get %ss result",
+                                       jobid);
+
+    if (type == AFTER_START
+        && !flux_jobtap_job_event_posted (p, afterid, "start"))
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "dependency: after: %s never started",
+                                       jobid);
+    if (type == AFTER_SUCCESS
+        && result != FLUX_JOB_RESULT_COMPLETED)
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "dependency: afterok: "
+                                       "job %s failed or was canceled",
+                                       jobid);
+    if (type == AFTER_FAILURE
+        && result == FLUX_JOB_RESULT_COMPLETED)
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "dependency: afternotok:"
+                                       " job %s succeeded",
+                                        jobid);
+    rc = flux_jobtap_dependency_remove (p, after->depid, after->description);
+    if (rc < 0)
+        flux_log_error (flux_jobtap_get_flux (p),
+                        "flux_jobtap_dependency_remove");
+    return rc;
+}
+
+/*  Handler for job.dependency.after*
+ *
+ */
+static int dependency_after_cb (flux_plugin_t *p,
+                                const char *topic,
+                                flux_plugin_arg_t *args,
+                                void *data)
+{
+    const char *scheme = NULL;
+    const char *jobid = NULL;
+    enum after_type type;
+    flux_jobid_t afterid;
+    flux_jobid_t id;
+    uint32_t uid;
+    uint32_t target_uid;
+    flux_job_state_t target_state;
+    struct after_info *after;
+    struct after_ref *ref;
+    zlistx_t *l;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:i s:{s:s s:s}}",
+                                "id", &id,
+                                "userid", &uid,
+                                "dependency",
+                                "scheme", &scheme,
+                                "value", &jobid) < 0)
+        return flux_jobtap_reject_job (p, args,
+                                       "dependency: after: %s",
+                                       flux_plugin_arg_strerror (args));
+
+    /*  Parse the type of dependency being requested from the scheme:
+     */
+    if (after_type_parse (scheme, &type) < 0)
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "invalid dependency scheme: %s",
+                                       scheme);
+
+    /*  Parse the value argument, which must be a valid jobid
+     *  Do not allow FLUX_JOBID_ANY/FLUX_JOBTAP_CURRENT_JOBID to be specified
+     */
+    if (flux_job_id_parse (jobid, &afterid) < 0
+        || afterid == FLUX_JOBTAP_CURRENT_JOB)
+        return flux_jobtap_reject_job (p, args,
+                                       "%s: %s: \"%s\" is not a valid jobid",
+                                       "dependency",
+                                       scheme,
+                                       jobid);
+
+    /*  Lookup userid and state of target job `afterid`
+     */
+    if (lookup_job_uid_state (p, afterid, &target_uid, &target_state) < 0) {
+        return flux_jobtap_reject_job (p, args,
+                                       "%s: %s: id %s: %s",
+                                       "dependency",
+                                       scheme,
+                                       jobid,
+                                       errno == ENOENT ?
+                                       "job not found" :
+                                       strerror (errno));
+    }
+
+    /*  Requesting userid must match target job uid
+     */
+    if (uid != target_uid)
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "%s: Permission denied for job %s",
+                                       scheme,
+                                       jobid);
+
+    if (!(after = after_info_create (id, type, jobid)))
+        return flux_jobtap_reject_job (p, args,
+                                       "failed to establish job dependency");
+
+    /*  Emit the dependency
+     */
+    if (flux_jobtap_dependency_add (p, id, after->description) < 0) {
+        after_info_destroy (after);
+        return flux_jobtap_reject_job (p, args, "Unable to add job dependency");
+    }
+
+    /*  If the job is already INACTIVE, then the dependency can be resolved
+     *   immediately, or if the dependency cannot be resolved then the job
+     *   is rejected.
+     */
+    if (target_state == FLUX_JOB_STATE_INACTIVE) {
+        int rc = dependency_handle_inactive (p, args, after, afterid, jobid);
+        after_info_destroy (after);
+        return rc;
+    }
+
+    /*  Corner case, requisite job may have already started. Check for that
+     *   here and immediately satisfy dependency before adding to various
+     *   lists below
+     */
+    if (type == AFTER_START
+        && flux_jobtap_job_event_posted (p, afterid, "start") == 1) {
+        if (flux_jobtap_dependency_remove (p, id, after->description) < 0)
+            flux_log_error (flux_jobtap_get_flux (p),
+                            "flux_jobtap_dependency_remove");
+        after_info_destroy (after);
+        return 0;
+    }
+
+    /*  Append this dependency to the deplist in the target jobid:
+     */
+    if (!(l = after_list_get (p, afterid))
+        || !zlistx_add_end (l, after)) {
+        after_info_destroy (after);
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "failed to append to list");
+    }
+
+    /*  Create a reference in the current job to the depednency, so it can
+     *   be removed if this job terminates before PRIORITY state.
+     */
+    if (!(ref = after_ref_create (afterid, l, after))
+        || !(l = after_refs_get (p, id))
+        || !zlistx_add_end (l, ref)) {
+        after_info_destroy (after);
+        after_ref_destroy (ref);
+        return flux_jobtap_reject_job (p, args, "failed to create ref");
+    }
+
+    /*  If the target is AFTER_START, subscribe to events for the target
+     *   jobid so this plugin gets the job.event.start callback for the
+     *   target job.
+     */
+    if (type == AFTER_START
+        && flux_jobtap_job_subscribe (p, afterid) < 0) {
+        after_info_destroy (after);
+        after_ref_destroy (ref);
+        return flux_jobtap_reject_job (p, args, "failed to subscribe to %s",
+                                       idf58 (id));
+    }
+
+    return 0;
+}
+
+
+/*  Attempt to remove the job dependency described by `after`. If this
+ *   fails, try to raise a fatal job exception on the current job.
+ */
+static void remove_jobid_dependency (flux_plugin_t *p,
+                                     struct after_info *after)
+{
+    if (flux_jobtap_dependency_remove (p,
+                                       after->depid,
+                                       after->description) < 0) {
+        if (flux_jobtap_raise_exception (p,
+                                         after->depid,
+                                         "dependency",
+                                         0,
+                                         "Failed to remove dependency %s",
+                                         after->description) < 0) {
+            flux_log_error (flux_jobtap_get_flux (p),
+                            "flux_jobtap_raise_exception: id=%s",
+                            idf58 (after->depid));
+        }
+    }
+}
+
+
+/*  Release all dependent jobs in the dependency list `l` with types
+ *   in the mask `typemask`.
+ */
+static void release_all (flux_plugin_t *p, zlistx_t *l, int typemask)
+{
+    if (l) {
+        struct after_info *after = zlistx_first (l);
+        while (after) {
+            if (after->type & typemask) {
+                /*  Remove dependency (possibly moving dependent job
+                 *   out of the DEPEND state.
+                 */
+                remove_jobid_dependency (p, after);
+
+                /*  Delete this entry since it has been resolved.
+                 */
+                if (zlistx_delete (l, zlistx_cursor (l)) < 0)
+                    flux_log (flux_jobtap_get_flux (p),
+                              LOG_ERR,
+                              "release_all: zlistx_delete");
+            }
+            after = zlistx_next (l);
+        }
+    }
+}
+
+/*
+ *  Raise exceptions for all unhandled depednencies in list `l`.
+ */
+static void raise_exceptions (flux_plugin_t *p, zlistx_t *l)
+{
+    if (l) {
+        struct after_info *after;
+        FOREACH_ZLISTX (l, after) {
+            if (flux_jobtap_raise_exception (p,
+                                             after->depid,
+                                             "dependency",
+                                             0,
+                                             "%s %s can never be satisfied",
+                                             "dependency",
+                                             after->description) < 0)
+                flux_log_error (flux_jobtap_get_flux (p),
+                                "id=%s: unable to raise exception for %s",
+                                idf58 (after->depid),
+                                after->description);
+        }
+        /*  N.B. = entry will be deleted at list destruction */
+    }
+}
+
+/*  If this job has any outstanding after-dependency references, then the
+ *   job has transitioned from dependency state directly to cleanup
+ *   (e.g. due to cancelation) and the refs must be cleaned up. This avoids
+ *   prerequisite jobs from emitting erroneous dependencies for completed
+ *   jobs.
+ */
+static void release_dependency_references (flux_plugin_t *p)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+    zlistx_t *l;
+    if ((l = after_refs_check (p))) {
+        struct after_ref *ref = zlistx_first (l);
+        while (ref) {
+            /*  For each after_ref entry, check to ensure the job and its
+             *   after dependencies list still exists. If so, remove this
+             *   job's entry from the list.
+             */
+            zlistx_t *after_list = after_list_check (p, ref->id);
+            if (after_list == ref->list) {
+                void *handle = zlistx_find (after_list, ref->info);
+                if (handle && zlistx_delete (ref->list, handle) < 0) {
+                    flux_log_error (h, "%s: %s: zlistx_delete",
+                                    "dependency-after",
+                                    "release_references");
+                }
+            }
+            ref = zlistx_next (l);
+        }
+    }
+
+    /*  Destroy this job's dependency reference list.
+     */
+    if (flux_jobtap_job_aux_delete (p, FLUX_JOBTAP_CURRENT_JOB, l) < 0)
+        flux_log_error (h, "release_references: flux_jobtap_job_aux_delete");
+}
+
+static int release_dependent_jobs (flux_plugin_t *p, zlistx_t *l)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+    flux_job_result_t result;
+
+    if (l == NULL)
+        return 0;
+
+    if (flux_jobtap_get_job_result (p,
+                                    FLUX_JOBTAP_CURRENT_JOB,
+                                    &result) < 0) {
+        flux_log_error (h, "dependency-after: flux_jobtap_get_result");
+        return -1;
+    }
+
+    /*  Release dependent jobs based on requisite job result.
+     *   Entries will be removed from the list as they are processed.
+     */
+    if (result != FLUX_JOB_RESULT_COMPLETED)
+        release_all (p, l, AFTER_FINISH | AFTER_FAILURE);
+    else
+        release_all (p, l, AFTER_FINISH | AFTER_SUCCESS);
+
+    /*  Any remaining dependencies can't now be satisfied.
+     *   Raise exceptions on any remaining members of list `l`
+     */
+    raise_exceptions (p, l);
+
+    return 0;
+}
+
+/*  In job.state.priority, delete any dependency references in a job
+ *   that was dependent on other jobs. This prevents the references
+ *   from being released at job completion. See release_references()
+ *   call in inactive_cb() above.
+ */
+static int priority_cb (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    zlistx_t *l = after_refs_check (p);
+    if (l) {
+        /*  Job has proceeded out of DEPEND state.
+         *  Delete any dependency reference list since it will no longer
+         *  be used, and we don't need to attempt deref of dependencies
+         *  (see bottom of inactive_cb()).
+         */
+        if (flux_jobtap_job_aux_delete (p, FLUX_JOBTAP_CURRENT_JOB, l) < 0) {
+            flux_log_error (flux_jobtap_get_flux (p),
+                            "dependency-after: flux_jobtap_job_aux_delete");
+        }
+    }
+    return 0;
+}
+
+/*  On start event, release all AFTER_START dependencies
+ */
+static int start_cb (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *data)
+{
+    release_all (p, after_list_check (p, 0), AFTER_START);
+
+    /*  This is the only job event we care about, unsubscribe from
+     *   future job events
+     */
+    flux_jobtap_job_unsubscribe (p, FLUX_JOBTAP_CURRENT_JOB);
+
+    return 0;
+}
+
+/*  In INACTIVE state, release remaining dependent jobs.
+ */
+static int inactive_cb (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    /*  Only need to check for dependent jobs if this job has an
+     *   embedded dependency list
+     */
+    release_dependent_jobs (p, after_list_check (p, 0));
+
+    /*  "Release" any references this job had to any dependencies
+     *  (references should still exist only if job skipped PRIORITY state.)
+     */
+    release_dependency_references (p);
+
+    return 0;
+}
+
+static json_t *deps_to_json (flux_plugin_t *p)
+{
+    json_t *o = NULL;
+    zlistx_t *l;
+
+    if (!(o = json_array ()))
+        return NULL;
+
+    if ((l = global_reflist)) {
+        struct after_ref *ref = zlistx_first (l);
+        while (ref) {
+            struct after_info *info = ref->info;
+            json_t *entry = NULL;
+
+            if (!(entry = json_pack ("{s:I s:I s:s s:s}",
+                                     "id", ref->id,
+                                     "depid", info->depid,
+                                     "type", after_typestr (info->type),
+                                     "description", info->description))
+                || json_array_append_new (o, entry) < 0) {
+                json_decref (entry);
+                goto error;
+            }
+            ref = zlistx_next (l);
+        }
+    }
+    return o;
+error:
+    json_decref (o);
+    return NULL;
+}
+
+static int query_cb (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *data)
+{
+    json_t *o = deps_to_json (p);
+
+    if (!o) {
+        flux_log (flux_jobtap_get_flux (p),
+                  LOG_ERR,
+                  "dependency-after: deps_to_json failed");
+        return -1;
+    }
+
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:O}",
+                              "dependencies", o) < 0)
+        flux_log_error (flux_jobtap_get_flux (p),
+                        "dependency-after: query_cb: flux_plugin_arg_pack: %s",
+                        flux_plugin_arg_strerror (args));
+    json_decref (o);
+    return 0;
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.dependency.after",      dependency_after_cb, NULL },
+    { "job.dependency.afterok",    dependency_after_cb, NULL },
+    { "job.dependency.afterany",   dependency_after_cb, NULL },
+    { "job.dependency.afternotok", dependency_after_cb, NULL },
+    { "job.state.priority",        priority_cb,         NULL },
+    { "job.state.inactive",        inactive_cb,         NULL },
+    { "job.event.start",           start_cb,            NULL },
+    { "plugin.query",              query_cb,            NULL },
+    { 0 }
+};
+
+static void reflist_destroy (zlistx_t *l)
+{
+    zlistx_destroy (&l);
+    global_reflist = NULL;
+}
+
+int after_plugin_init (flux_plugin_t *p)
+{
+    if (!(global_reflist = zlistx_new ())
+        || flux_plugin_aux_set (p,
+                                NULL,
+                                global_reflist,
+                                (flux_free_f)reflist_destroy) < 0) {
+        reflist_destroy (global_reflist);
+        return -1;
+    }
+    return flux_plugin_register (p, ".dependency-after", tab);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/modules/job-manager/plugins/history.c b/src/modules/job-manager/plugins/history.c
new file mode 100644
index 000000000000..1fc929a049f0
--- /dev/null
+++ b/src/modules/job-manager/plugins/history.c
@@ -0,0 +1,332 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* history.c - track jobs in t_submit order per user
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "ccan/ptrint/ptrint.h"
+#include "ccan/str/str.h"
+#include "src/common/libutil/hola.h"
+#include "src/common/libutil/slice.h"
+#include "src/common/libutil/errprintf.h"
+
+struct job_entry {
+    flux_jobid_t id;
+    double t_submit;
+};
+
+struct history {
+    flux_plugin_t *p;
+    struct hola *users; // userid => job list
+};
+
+#define COMPARE_NUM_REVERSE(a,b) ((a)>(b)?-1:(a)<(b)?1:0)
+
+/* Keys in history->users are calculated from int2ptr(uid), but that won't
+ * work for root.  Substitute (uid_t)-1 in that case as it's reserved per
+ * POSIX.  See also: flux-framework/flux-core#5475.
+ */
+static const void *userid2key (int userid)
+{
+    if (userid == 0)
+        return int2ptr ((uid_t)-1);
+    return int2ptr (userid);
+}
+
+static int key2userid (const void *key)
+{
+    if (key == int2ptr ((uid_t)-1))
+        return 0;
+    return ptr2int (key);
+}
+
+static void job_entry_destroy (struct job_entry *entry)
+{
+    if (entry) {
+        int saved_errno = errno;
+        free (entry);
+        errno = saved_errno;
+    }
+}
+
+static struct job_entry *job_entry_create (void)
+{
+    struct job_entry *entry;
+
+    if (!(entry = calloc (1, sizeof (*entry))))
+        return NULL;
+    return entry;
+}
+
+// zlistx_destructor_fn footprint
+static void job_entry_destructor (void **item)
+{
+    if (item) {
+        job_entry_destroy (*item);
+        *item = NULL;
+    }
+}
+
+// zlistx_comparator_fn footprint
+static int job_entry_comparator (const void *item1, const void *item2)
+{
+    const struct job_entry *a = item1;
+    const struct job_entry *b = item2;
+
+    return COMPARE_NUM_REVERSE (a->t_submit, b->t_submit);
+}
+
+// zhashx_hash_fn footprint
+static size_t userid_hasher (const void *key)
+{
+    return key2userid (key);
+}
+
+// zhashx_comparator_fn footprint
+static int userid_comparator (const void *item1, const void *item2)
+{
+    return COMPARE_NUM_REVERSE (key2userid (item1), key2userid (item2));
+}
+
+static void history_destroy (struct history *hist)
+{
+    if (hist) {
+        int saved_errno = errno;
+        hola_destroy (hist->users);
+        free (hist);
+        errno = saved_errno;
+    };
+}
+
+static struct history *history_create (flux_plugin_t *p)
+{
+    struct history *hist;
+
+    if (!(hist = calloc (1, sizeof (*hist))))
+        return NULL;
+    hist->p = p;
+    if (!(hist->users = hola_create (HOLA_AUTOCREATE)))
+        goto error;
+    hola_set_hash_key_destructor (hist->users, NULL);
+    hola_set_hash_key_duplicator (hist->users, NULL);
+    hola_set_hash_key_comparator (hist->users, userid_comparator);
+    hola_set_hash_key_hasher (hist->users, userid_hasher);
+    hola_set_list_destructor (hist->users, job_entry_destructor);
+    hola_set_list_comparator (hist->users, job_entry_comparator);
+    return hist;
+error:
+    history_destroy (hist);
+    return NULL;
+}
+
+static int jobtap_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *arg)
+{
+    struct history *hist = arg;
+    struct job_entry *entry;
+    int userid;
+    const void *key;
+
+    if (!(entry = job_entry_create ()))
+        return -1;
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:f s:i}",
+                                "id", &entry->id,
+                                "t_submit", &entry->t_submit,
+                                "userid", &userid) < 0)
+        return -1;
+    key = userid2key (userid);
+    if (streq (topic, "job.inactive-remove")) {
+        void *handle;
+        if ((handle = hola_list_find (hist->users, key, entry)))
+            hola_list_delete (hist->users, key, handle);
+        job_entry_destroy (entry);
+    }
+    else if (streq (topic, "job.inactive-add")) {
+        if (hola_list_find (hist->users, key, entry)) {
+            job_entry_destroy (entry);
+            return 0;
+        }
+        if (!hola_list_insert (hist->users, key, entry, true)) {
+            job_entry_destroy (entry);
+            return -1;
+        }
+    }
+    else if (streq (topic, "job.new")) {
+        if (!hola_list_insert (hist->users, key, entry, true)) {
+            job_entry_destroy (entry);
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static int append_int (json_t *a, json_int_t i)
+{
+    json_t *o;
+    if (!(o = json_integer (i))
+        || json_array_append_new (a, o) < 0) {
+        json_decref (o);
+        return -1;
+    }
+    return 0;
+}
+
+static int list_slice_reverse (zlistx_t *l, struct slice *sl, json_t *a)
+{
+    if (l) {
+        struct job_entry *entry = zlistx_last (l);
+        size_t list_index = zlistx_size (l) - 1;
+        int slice_index = slice_first (sl);
+
+        while (entry && slice_index != -1) {
+            if (list_index == slice_index) {
+                if (append_int (a, entry->id) < 0)
+                    return -1;
+                slice_index = slice_next (sl);
+            }
+            list_index--;
+            entry = zlistx_prev (l);
+        }
+    }
+    return 0;
+}
+
+static int list_slice_forward (zlistx_t *l, struct slice *sl, json_t *a)
+{
+    if (l) {
+        struct job_entry *entry = zlistx_first (l);
+        size_t list_index = 0;
+        int slice_index = slice_first (sl);
+
+        while (entry && slice_index != -1) {
+            if (list_index == slice_index) {
+                if (append_int (a, entry->id) < 0)
+                    return -1;
+                slice_index = slice_next (sl);
+            }
+            list_index++;
+            entry = zlistx_next (l);
+        }
+    }
+    return 0;
+}
+
+static json_t *history_slice (struct history *hist,
+                              int userid,
+                              const char *slice,
+                              flux_error_t *error)
+{
+    json_t *a;
+    zlistx_t *l;
+    struct slice sl;
+    size_t list_size = 0;
+    int rc;
+
+    if ((l = hola_hash_lookup (hist->users, userid2key (userid))))
+        list_size = zlistx_size (l);
+    if (slice_parse (&sl, slice, list_size) < 0) {
+        errprintf (error, "could not parse python-style slice expression");
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(a = json_array ()))
+        goto oom;
+    if (sl.step > 0)
+        rc = list_slice_forward (l, &sl, a);
+    else
+        rc = list_slice_reverse (l, &sl, a);
+    if (rc < 0)
+        goto oom;
+    return a;
+oom:
+    json_decref (a);
+    errprintf (error, "out of memory");
+    errno = ENOMEM;
+    return NULL;
+}
+
+static void history_get_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
+{
+    struct history *hist = arg;
+    const char *slice;
+    struct flux_msg_cred cred;
+    json_t *jobs;
+    flux_error_t error;
+    const char *errmsg = NULL;
+
+    if (flux_request_unpack (msg, NULL, "{s:s}", "slice", &slice) < 0
+        || flux_msg_get_cred (msg, &cred) < 0)
+        goto error;
+    if (!(jobs = history_slice (hist, cred.userid, slice, &error))) {
+        errmsg = error.text;
+        errno = EINVAL;
+        goto error;
+    }
+    if (flux_respond_pack (h, msg, "{s:O}", "jobs", jobs) < 0)
+        flux_log_error (h, "error responding to job-manager.history.get");
+    json_decref (jobs);
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to job-manager.history.get");
+}
+
+
+int history_plugin_init (flux_plugin_t *p)
+{
+    struct history *hist;
+
+    if (!(hist = history_create (p))
+        || flux_plugin_aux_set (p,
+                                NULL,
+                                hist,
+                                (flux_free_f)history_destroy) < 0) {
+        history_destroy (hist);
+        return -1;
+    }
+    if (flux_jobtap_service_register_ex (p,
+                                         "get",
+                                         FLUX_ROLE_USER,
+                                         history_get_cb,
+                                         hist) < 0)
+        return -1;
+
+    if (flux_plugin_add_handler (p,
+                                 "job.new",
+                                 jobtap_cb,
+                                 hist) < 0
+        || flux_plugin_add_handler (p,
+                                    "job.inactive-add",
+                                    jobtap_cb,
+                                    hist) < 0
+        || flux_plugin_add_handler (p,
+                                    "job.inactive-remove",
+                                    jobtap_cb,
+                                    hist) < 0)
+        return -1;
+
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/plugins/limit-duration.c b/src/modules/job-manager/plugins/limit-duration.c
new file mode 100644
index 000000000000..7735d4a79905
--- /dev/null
+++ b/src/modules/job-manager/plugins/limit-duration.c
@@ -0,0 +1,317 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* limit-duration.c - validate job requests against configured duration limits
+ *
+ * This plugin uses the job.validate callback to accept or reject job
+ * requests.  Any default jobspec values would have been applied earlier
+ * (where applicable) at ingest.
+ *
+ * General limit:
+ *  policy.limits.duration
+ * Queue-specific limit:
+ *  queues.<name>.policy.limits.duration
+ *
+ * N.B. a queue limit may override the general limit with a higher or lower
+ * limit, or "0" for unlimited.
+ *
+ * See also:
+ *  RFC 33/Flux Job Queues
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
+struct limit_duration {
+    double general_limit;   // general duration limit (seconds)
+    zhashx_t *queues;       // queue name => duration limit (double * seconds)
+    flux_t *h;
+};
+
+#define DURATION_INVALID      (-1)
+#define DURATION_UNLIMITED    (0)
+
+static const char *auxkey = "limit-duration";
+
+// zhashx_destructor_fn footprint
+static void duration_destroy (void **item)
+{
+    if (item) {
+        free (*item);
+        *item = NULL;
+    }
+}
+
+// zhashx_duplicator_fn footprint
+static void *duration_duplicate (const void *item)
+{
+    double *cpy;
+    if (!(cpy = calloc (1, sizeof (*cpy))))
+        return NULL;
+    *cpy = *(double *)item;
+    return cpy;
+}
+
+static zhashx_t *queues_create (void)
+{
+    zhashx_t *queues;
+
+    if (!(queues = zhashx_new ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    zhashx_set_destructor (queues, duration_destroy);
+    zhashx_set_duplicator (queues, duration_duplicate);
+    return queues;
+}
+
+static double queues_lookup (zhashx_t *queues, const char *name)
+{
+    double *dp;
+
+    if (name && (dp = zhashx_lookup (queues, name)))
+        return *dp;
+    return DURATION_INVALID;
+}
+
+static void queues_insert (zhashx_t *queues, const char *name, double duration)
+{
+    (void)zhashx_insert (queues, name, &duration); // dups duration
+}
+
+static void limit_duration_destroy (struct limit_duration *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        zhashx_destroy (&ctx->queues);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static struct limit_duration *limit_duration_create (flux_t *h)
+{
+    struct limit_duration *ctx;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+    if (!(ctx->queues = queues_create ()))
+        goto error;
+    ctx->h = h;
+    ctx->general_limit = DURATION_INVALID;
+    return ctx;
+error:
+    limit_duration_destroy (ctx);
+    return NULL;
+}
+
+static int duration_parse (double *duration,
+                           json_t *conf,
+                           flux_error_t *error)
+{
+    double d = DURATION_INVALID;
+    const char *ds = NULL;
+    json_error_t jerror;
+    const char *name = "policy.limits.duration";
+
+    if (json_unpack_ex (conf,
+                        &jerror,
+                        0,
+                        "{s?{s?{s?s}}}",
+                        "policy",
+                          "limits",
+                            "duration", &ds) < 0) {
+        errprintf (error, "%s: %s", name, jerror.text);
+        goto inval;
+    }
+    if (ds) {
+        if (fsd_parse_duration (ds, &d) < 0) {
+            errprintf (error, "%s: FSD value is malformed", name);
+            return -1;
+        }
+    }
+    *duration = d;
+    return 0;
+inval:
+    errno = EINVAL;
+    return -1;
+}
+
+static int queues_parse (zhashx_t **zhp,
+                         json_t *conf,
+                         flux_error_t *error)
+{
+    json_t *queues;
+    zhashx_t *zh;
+
+    if (!(zh = queues_create ())) {
+        errprintf (error, "out of memory parsing [queues]");
+        goto error;
+    }
+    if ((queues = json_object_get (conf, "queues"))) {
+        const char *name;
+        json_t *entry;
+        double duration;
+        flux_error_t e;
+
+        json_object_foreach (queues, name, entry) {
+            if (duration_parse (&duration, entry, &e) < 0) {
+                errprintf (error, "queues.%s.%s", name, e.text);
+                goto error;
+            }
+            queues_insert (zh, name, duration);
+        }
+    }
+    *zhp = zh;
+    return 0;
+error:
+    ERRNO_SAFE_WRAP (zhashx_destroy, &zh);
+    return -1;
+}
+
+static int check_limit (struct limit_duration *ctx,
+                        double duration,
+                        const char *queue,
+                        flux_error_t *error)
+{
+    double limit = ctx->general_limit;
+    double qlimit = queues_lookup (ctx->queues, queue);
+
+    if (qlimit != DURATION_INVALID)
+        limit = qlimit;
+    if (limit != DURATION_INVALID
+        && limit != DURATION_UNLIMITED
+        && (duration > limit || duration == DURATION_UNLIMITED)) {
+        char fsd[64];
+        fsd_format_duration_ex (fsd, sizeof (fsd), limit, 2);
+        return errprintf (error,
+                          "requested duration exceeds policy limit of %s",
+                          fsd);
+    }
+    return 0;
+}
+
+static int validate_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *arg)
+{
+    struct limit_duration *ctx = flux_plugin_aux_get (p, auxkey);
+    flux_job_state_t state;
+    double duration = DURATION_UNLIMITED;
+    const char *queue = NULL;
+    flux_error_t error;
+
+    /* If no limits are configured, return immediately.  This is the common
+     * case for a non-system instance and since this plugin is always loaded,
+     * don't waste time.
+     */
+    if ((ctx->general_limit == DURATION_INVALID
+        || ctx->general_limit == DURATION_UNLIMITED)
+        && zhashx_size (ctx->queues) == 0)
+        return 0;
+
+    /* Parse jobspec attributes:
+     * - attributes.system.queue (NULL if unspecified)
+     * - attributes.system.duration (DURATION_UNLIMITED if unspecified)
+     */
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:i s?{s?{s?{s?F s?s}}}}",
+                                "state", &state,
+                                "jobspec",
+                                  "attributes",
+                                    "system",
+                                      "duration", &duration,
+                                      "queue", &queue) < 0) {
+        errprintf (&error,
+                   "limit-duration: error unpacking job.validate arguments: %s",
+                   flux_plugin_arg_strerror (args));
+        goto error;
+    }
+
+    if (check_limit (ctx, duration, queue, &error) < 0)
+        goto error;
+
+    return 0;
+error:
+    flux_jobtap_reject_job (p, args, "%s", error.text);
+    return -1;
+}
+
+/* conf.update callback - called on plugin load, and when config is updated
+ * This function has two purposes:
+ * - Validate proposed 'conf' and return human readable errors if rejected
+ * - Pre-parse and cache the config in 'ctx' to streamline job validation
+ */
+static int conf_update_cb (flux_plugin_t *p,
+                           const char *topic,
+                           flux_plugin_arg_t *args,
+                           void *arg)
+{
+    struct limit_duration *ctx = flux_plugin_aux_get (p, auxkey);
+    flux_error_t error;
+    json_t *conf;
+    double duration;
+    zhashx_t *queues;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:o}",
+                                "conf", &conf) < 0) {
+        errprintf (&error,
+                   "limit-duration: error unpacking conf.update arguments: %s",
+                   flux_plugin_arg_strerror (args));
+        goto error;
+    }
+    if (duration_parse (&duration, conf, &error) < 0
+        || queues_parse (&queues, conf, &error) < 0)
+        goto error;
+    ctx->general_limit = duration;
+    zhashx_destroy (&ctx->queues);
+    ctx->queues = queues;
+    return 0;
+error:
+    return flux_jobtap_error (p, args, "%s", error.text);
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.validate", validate_cb, NULL },
+    { "conf.update", conf_update_cb, NULL },
+    { 0 }
+};
+
+int limit_duration_plugin_init (flux_plugin_t *p)
+{
+    struct limit_duration *ctx;
+
+    if (!(ctx = limit_duration_create (flux_jobtap_get_flux (p)))
+        || flux_plugin_aux_set (p,
+                                auxkey,
+                                ctx,
+                                (flux_free_f)limit_duration_destroy) < 0) {
+        limit_duration_destroy (ctx);
+        return -1;
+    }
+
+    return flux_plugin_register (p, ".limit-duration", tab);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/plugins/limit-job-size.c b/src/modules/job-manager/plugins/limit-job-size.c
new file mode 100644
index 000000000000..a27cc49161dc
--- /dev/null
+++ b/src/modules/job-manager/plugins/limit-job-size.c
@@ -0,0 +1,468 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* limit-job-size.c - validate job requests against configured job size limits
+ *
+ * This plugin uses the job.validate callback to accept or reject job
+ * requests.  Any default jobspec values would have been applied earlier
+ * (where applicable) in the job.create callback.
+ *
+ * General limit:
+ *  [policy.limits.job-size]
+ * Queue-specific limit:
+ *  [queues.<name>.policy.limits.job-size]
+ *
+ * N.B. a queue limit may override the general limit with a higher or lower
+ * limit, even "unlimited".  Since 0 may be a valid size limit, -1 is reserved
+ * to mean unlimited in this situation.
+ *
+ * See also:
+ *  RFC 33/Flux Job Queues
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "src/common/libjob/jj.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
+#define SIZE_INVALID   (-2)
+#define SIZE_UNLIMITED (-1)
+
+#define SIZE_IS_OUT_OF_BOUNDS(n) \
+    ((n) < 0 && (n) != SIZE_INVALID && (n) != SIZE_UNLIMITED)
+
+#define LIMIT_OVER(limit,val) \
+    ((limit) != SIZE_INVALID && (limit) != SIZE_UNLIMITED && (val) > (limit))
+#define LIMIT_UNDER(limit,val) \
+    ((limit) != SIZE_INVALID && (limit) != SIZE_UNLIMITED && (val) < (limit))
+
+struct job_size {
+    int nnodes;
+    int ncores;
+    int ngpus;
+};
+
+struct limits {
+    struct job_size max;
+    struct job_size min;
+};
+
+struct limit_job_size {
+    struct limits general_limits;
+    zhashx_t *queues; // queue name => struct limits
+    flux_t *h;
+};
+
+const char *auxkey = "limit-job-size";
+
+static void job_size_clear (struct job_size *js)
+{
+    js->nnodes = SIZE_INVALID;
+    js->ncores = SIZE_INVALID;
+    js->ngpus = SIZE_INVALID;
+}
+
+static bool job_size_isset (struct job_size *js)
+{
+    if (js) {
+        if (js->nnodes != SIZE_INVALID
+            || js->ncores != SIZE_INVALID
+            || js->ngpus != SIZE_INVALID)
+            return true;
+    }
+    return false;
+}
+
+static void job_size_override (struct job_size *js1,
+                               struct job_size *js2)
+{
+    if (js1 && js2) {
+        if (js2->nnodes != SIZE_INVALID)
+            js1->nnodes = js2->nnodes;
+        if (js2->ncores != SIZE_INVALID)
+            js1->ncores = js2->ncores;
+        if (js2->ngpus != SIZE_INVALID)
+            js1->ngpus = js2->ngpus;
+    }
+}
+
+static void limits_clear (struct limits *l)
+{
+    job_size_clear (&l->max);
+    job_size_clear (&l->min);
+}
+
+static bool limits_isset (struct limits *l)
+{
+    if (l) {
+        if (job_size_isset (&l->max)
+            || job_size_isset (&l->min))
+            return true;
+    }
+    return false;
+}
+
+static void limits_override (struct limits *l1,
+                             struct limits *l2)
+{
+    if (l1 && l2) {
+        job_size_override (&l1->max, &l2->max);
+        job_size_override (&l1->min, &l2->min);
+    }
+}
+
+// zhashx_destructor_fn footprint
+static void limits_destroy (void **item)
+{
+    if (item) {
+        free (*item);
+        *item = NULL;
+    }
+}
+
+// zhashx_duplicator_fn footprint
+static void *limits_duplicate (const void *item)
+{
+    struct limits *limits;
+
+    if (!(limits = calloc (1, sizeof (*limits))))
+        return NULL;
+    *limits = *(struct limits *)item;
+    return limits;
+}
+
+static zhashx_t *queues_create (void)
+{
+    zhashx_t *queues;
+
+    if (!(queues = zhashx_new ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    zhashx_set_destructor (queues, limits_destroy);
+    zhashx_set_duplicator (queues, limits_duplicate);
+    return queues;
+}
+
+static void queues_insert (zhashx_t *queues,
+                           const char *name,
+                           struct limits *limits)
+{
+    (void)zhashx_insert (queues, name, limits); // dups limits
+}
+
+static struct limits *queues_lookup (zhashx_t *queues, const char *name)
+{
+    return queues ? zhashx_lookup (queues, name) : NULL;
+}
+
+static void limit_job_size_destroy (struct limit_job_size *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        zhashx_destroy (&ctx->queues);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static struct limit_job_size *limit_job_size_create (flux_t *h)
+{
+    struct limit_job_size *ctx;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+    if (!(ctx->queues = queues_create ()))
+        goto error;
+    ctx->h = h;
+    limits_clear (&ctx->general_limits);
+    return ctx;
+error:
+    limit_job_size_destroy (ctx);
+    return NULL;
+}
+
+static int job_size_parse (struct job_size *jsp,
+                           json_t *o,
+                           flux_error_t *error)
+{
+    struct job_size js;
+
+    job_size_clear (&js);
+    if (o) {
+        json_error_t jerror;
+
+        if (json_unpack_ex (o,
+                            &jerror,
+                            0,
+                            "{s?i s?i s?i}",
+                            "nnodes", &js.nnodes,
+                            "ncores", &js.ncores,
+                            "ngpus", &js.ngpus) < 0) {
+            errprintf (error, "%s", jerror.text);
+            errno = EINVAL;
+            return -1;
+        }
+        if (SIZE_IS_OUT_OF_BOUNDS (js.nnodes)
+            || SIZE_IS_OUT_OF_BOUNDS (js.ncores)
+            || SIZE_IS_OUT_OF_BOUNDS (js.ngpus)) {
+            errprintf (error, "size must be -1 (unlimited), or >= 0");
+            errno = EINVAL;
+            return -1;
+        }
+    }
+    *jsp = js;
+    return 0;
+}
+
+static int limits_parse (struct limits *limitsp,
+                         json_t *conf,
+                         flux_error_t *error)
+{
+    struct limits limits;
+    json_t *min = NULL;
+    json_t *max = NULL;
+    json_error_t jerror;
+    flux_error_t e;
+
+    if (json_unpack_ex (conf,
+                        &jerror,
+                        0,
+                        "{s?{s?{s?{s?o s?o}}}}",
+                        "policy",
+                          "limits",
+                            "job-size",
+                              "max", &max,
+                              "min", &min) < 0) {
+        errprintf (error, "policy.limits.job-size: %s", jerror.text);
+        return -1;
+    }
+    if (job_size_parse (&limits.max, max, &e) < 0) {
+        errprintf (error, "policy.limits.job-size.max: %s", e.text);
+        return -1;
+    }
+    if (job_size_parse (&limits.min, min, &e) < 0) {
+        errprintf (error, "policy.limits.job-size.min: %s", e.text);
+        return -1;
+    }
+    *limitsp = limits;
+    return 0;
+}
+
+static int queues_parse (zhashx_t **zhp,
+                         json_t *conf,
+                         flux_error_t *error)
+{
+    json_t *queues;
+    zhashx_t *zh;
+
+    if (!(zh = queues_create ())) {
+        errprintf (error, "out of memory parsing [queues]");
+        goto error;
+    }
+    if ((queues = json_object_get (conf, "queues"))) {
+        const char *name;
+        json_t *entry;
+        struct limits limits;
+        flux_error_t e;
+
+        json_object_foreach (queues, name, entry) {
+            if (limits_parse (&limits, entry, &e) < 0) {
+                errprintf (error, "queues.%s.%s", name, e.text);
+                goto error;
+            }
+            queues_insert (zh, name, &limits);
+        }
+    }
+    *zhp = zh;
+    return 0;
+error:
+    ERRNO_SAFE_WRAP (zhashx_destroy, &zh);
+    return -1;
+}
+
+static int check_limits (struct limit_job_size *ctx,
+                         struct jj_counts *counts,
+                         const char *queue,
+                         flux_error_t *error)
+{
+    struct limits limits;
+    struct limits *queue_limits;
+
+    limits = ctx->general_limits;
+    if (queue && (queue_limits = queues_lookup (ctx->queues, queue)))
+        limits_override (&limits, queue_limits);
+
+    if (LIMIT_OVER (limits.max.nnodes, counts->nnodes)) {
+        return errprintf (error,
+                          "requested nnodes exceeds policy limit of %d",
+                          limits.max.nnodes);
+    }
+    if (LIMIT_OVER (limits.max.ncores, counts->nslots * counts->slot_size)) {
+        return errprintf (error,
+                          "requested ncores exceeds policy limit of %d",
+                          limits.max.ncores);
+    }
+    if (LIMIT_OVER (limits.max.ngpus, counts->nslots * counts->slot_gpus)) {
+        return errprintf (error,
+                          "requested ngpus exceeds policy limit of %d",
+                          limits.max.ngpus);
+    }
+    if (LIMIT_UNDER (limits.min.nnodes, counts->nnodes)) {
+        return errprintf (error,
+                          "requested nnodes is under policy limit of %d",
+                          limits.min.nnodes);
+    }
+    if (LIMIT_UNDER (limits.min.ncores, counts->nslots * counts->slot_size)) {
+        return errprintf (error,
+                          "requested ncores is under policy limit of %d",
+                          limits.min.ncores);
+    }
+    if (LIMIT_UNDER (limits.min.ngpus, counts->nslots * counts->slot_gpus)) {
+        return errprintf (error,
+                          "requested ngpus is under policy limit of %d",
+                          limits.min.ngpus);
+    }
+
+    return 0;
+}
+
+static int validate_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *arg)
+{
+    struct limit_job_size *ctx = flux_plugin_aux_get (p, auxkey);
+    flux_job_state_t state;
+    json_t *jobspec = NULL;
+    struct jj_counts counts;
+    const char *queue = NULL;
+    flux_error_t error;
+    json_error_t jerror;
+
+    /* If no limits are configured, return immediately.  This is the common
+     * case for a non-system instance and since this plugin is always loaded,
+     * don't waste time.
+     */
+    if (!limits_isset (&ctx->general_limits)
+        && zhashx_size (ctx->queues) == 0)
+        return 0;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:i s:o}",
+                                "state", &state,
+                                "jobspec", &jobspec) < 0) {
+        errprintf (&error,
+                   "limit-job-size: error unpacking job.validate arguments: %s",
+                   flux_plugin_arg_strerror (args));
+        goto error;
+    }
+
+    if (jj_get_counts_json (jobspec, &counts) < 0) {
+        errprintf (&error, "%s", counts.error);
+        goto error;
+    }
+
+    /* Parse (optional) jobspec attributes.system.queue.
+     * Leave queue NULL if unspecified.
+     * Throw an error if it's the wrong type or related.
+     */
+    if (json_unpack_ex (jobspec,
+                        &jerror,
+                        0,
+                        "{s:{s?{s?s}}}",
+                        "attributes",
+                          "system",
+                            "queue", &queue) < 0) {
+        errprintf (&error,
+                   "Error parsing jobspec attributes.system.queue: %s",
+                   jerror.text);
+        goto error;
+    }
+
+    if (check_limits (ctx, &counts, queue, &error) < 0)
+        goto error;
+
+    return 0;
+error:
+    flux_jobtap_reject_job (p, args, "%s", error.text);
+    return -1;
+}
+
+/* conf.update callback - called on plugin load, and when config is updated
+ * This function has two purposes:
+ * - Validate proposed 'conf' and return human readable errors if rejected
+ * - Pre-parse and cache the config in 'ctx' to streamline job validation
+ */
+static int conf_update_cb (flux_plugin_t *p,
+                           const char *topic,
+                           flux_plugin_arg_t *args,
+                           void *arg)
+{
+    struct limit_job_size *ctx = flux_plugin_aux_get (p, auxkey);
+    flux_error_t error;
+    json_t *conf;
+    struct limits limits;
+    zhashx_t *queues;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:o}",
+                                "conf", &conf) < 0) {
+        errprintf (&error,
+                   "limit-job-size: error unpacking conf.update arguments: %s",
+                   flux_plugin_arg_strerror (args));
+        goto error;
+    }
+    if (limits_parse (&limits, conf, &error) < 0)
+        goto error;
+    if (queues_parse (&queues, conf, &error) < 0)
+        goto error;
+    ctx->general_limits = limits;
+    zhashx_destroy (&ctx->queues);
+    ctx->queues = queues;
+    return 0;
+error:
+    return flux_jobtap_error (p, args, "%s", error.text);
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.validate", validate_cb, NULL },
+    { "conf.update", conf_update_cb, NULL },
+    { 0 }
+};
+
+int limit_job_size_plugin_init (flux_plugin_t *p)
+{
+    struct limit_job_size *ctx;
+
+    if (!(ctx = limit_job_size_create (flux_jobtap_get_flux (p)))
+        || flux_plugin_aux_set (p,
+                                auxkey,
+                                ctx,
+                                (flux_free_f)limit_job_size_destroy) < 0) {
+        limit_job_size_destroy (ctx);
+        return -1;
+    }
+
+    return flux_plugin_register (p, ".limit-job-size", tab);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/plugins/perilog.c b/src/modules/job-manager/plugins/perilog.c
new file mode 100644
index 000000000000..291b725d807a
--- /dev/null
+++ b/src/modules/job-manager/plugins/perilog.c
@@ -0,0 +1,1575 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/*  perilog.c : execute a job manager prolog/epilog for jobs
+ *
+ *  Run prolog and/or epilog commands on rank 0 before jobs
+ *   have been allocated or freed resources.
+ *
+ *  Notes:
+ *
+ *  - The job manager prolog is started at the RUN state.
+ *
+ *  - If a job gets a fatal exception while the prolog is
+ *    running, the prolog is canceled and a SIGTERM signal
+ *    is sent. After a configurable timeout, ranks on which
+ *    the prolog is still active are drained.
+ *
+ *  - The epilog is started as a result of a "finish" event or
+ *    when the prolog completes if a fatal job exception has been
+ *    raised. Therefore the job manager epilog is always run if
+ *    a prolog has run.
+ *
+ *  - Requires that a prolog and/or epilog command be configured
+ *    in the [job-manager.prolog] and [job-manager.epilog]
+ *    tables, e.g.
+ *
+ *     [job-manager.prolog]
+ *     command = [ "command", "arg1", "arg2" ]
+ *     timeout = "30m"
+ *
+ *  - The queue should be idle before unloading/reloading this
+ *     plugin. Otherwise jobs may become stuck because a prolog
+ *     or epilog in progress will result in a missing -finish
+ *     event in the job's eventlog.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <limits.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <regex.h>
+#include <math.h>
+#include "src/common/libmissing/macros.h"
+#define EXIT_CODE(x) __W_EXITCODE(x,0)
+
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+#include <flux/idset.h>
+
+#include "src/common/libjob/job_hash.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/fsd.h"
+#include "ccan/str/str.h"
+#include "src/broker/state_machine.h" // for STATE_CLEANUP
+#include "src/common/libsubprocess/bulk-exec.h"
+#include "src/common/librlist/rlist.h"
+
+extern char **environ;
+
+/*
+ *  Configuration for a single perilog process
+ */
+struct perilog_procdesc {
+    flux_cmd_t *cmd;
+    bool uses_imp;
+    bool prolog;
+    bool per_rank;
+    bool cancel_on_exception;
+    double timeout;
+    double kill_timeout;
+};
+
+/*  Global prolog/epilog configuration
+ */
+static struct perilog_conf {
+    bool initialized;
+
+    char *imp_path;
+    struct perilog_procdesc *prolog;
+    struct perilog_procdesc *epilog;
+
+    zhashx_t *processes;       /*  List of outstanding perilog_proc objects */
+    zlistx_t *log_ignore;      /*  List of regex patterns to ignore in logs */
+    flux_future_t *watch_f;    /*  Watch for broker entering CLEANUP state */
+    bool shutting_down;        /*  True when broker has entered CLEANUP */
+} perilog_config;
+
+
+/*  Data for a prolog/epilog process
+ */
+struct perilog_proc {
+    flux_plugin_t *p;
+    flux_jobid_t id;
+    uint32_t userid;
+    json_t *R;
+    bool per_rank;
+    bool prolog;
+    bool cancel_on_exception;
+    bool canceled;
+    bool timedout;
+    bool cancel_timeout;
+    double kill_timeout;
+    flux_future_t *kill_f;
+    flux_future_t *drain_f;
+    flux_watcher_t *timer;
+    flux_watcher_t *kill_timer;
+    struct bulk_exec *bulk_exec;
+    struct idset *ranks;
+    char *failed_ranks;
+};
+
+/* The default time (sec) to wait for the prolog to terminate after SIGTERM.
+ */
+static double default_kill_timeout = 60.;
+
+static struct perilog_proc *procdesc_run (flux_t *h,
+                                          flux_plugin_t *p,
+                                          struct perilog_procdesc *pd,
+                                          flux_jobid_t id,
+                                          uint32_t userid,
+                                          json_t *R);
+
+static void timeout_cb (flux_reactor_t *r,
+                        flux_watcher_t *w,
+                        int revents,
+                        void *arg);
+
+static void perilog_procdesc_destroy (struct perilog_procdesc *pd)
+{
+    if (pd) {
+        int saved_errno = errno;
+        flux_cmd_destroy (pd->cmd);
+        free (pd);
+        errno = saved_errno;
+    }
+}
+
+static flux_cmd_t *cmd_from_json (json_t *o)
+{
+    size_t index;
+    json_t *value;
+    flux_cmd_t *cmd;
+
+    if (!json_is_array (o))
+        return NULL;
+
+    if (!(cmd = flux_cmd_create (0, NULL, environ)))
+        return NULL;
+
+    json_array_foreach (o, index, value) {
+        const char *arg = json_string_value (value);
+        if (!value
+            || flux_cmd_argv_append (cmd, arg) < 0)
+            goto fail;
+    }
+    return cmd;
+fail:
+    flux_cmd_destroy (cmd);
+    return NULL;
+}
+
+
+static struct perilog_procdesc *perilog_procdesc_create (json_t *o,
+                                                         bool prolog,
+                                                         flux_error_t *errp)
+{
+    struct perilog_procdesc *pd = NULL;
+    int per_rank = 0;
+    int cancel_on_exception = -1;
+    const char *timeout;
+    double kill_timeout = -1.;
+    flux_cmd_t *cmd = NULL;
+    json_t *command = NULL;
+    bool uses_imp = false;
+    json_error_t error;
+
+    const char *name = prolog ? "prolog" : "epilog";
+
+    /*  Set default timeout for prolog to 30m, unlimited for epilog
+     */
+    timeout = prolog ? "30m" : "0";
+
+    if (json_unpack_ex (o,
+                        &error,
+                        0,
+                        "{s?o s?s s?F s?b s?b !}",
+                        "command", &command,
+                        "timeout", &timeout,
+                        "kill-timeout", &kill_timeout,
+                        "per-rank", &per_rank,
+                        "cancel-on-exception", &cancel_on_exception) < 0) {
+        errprintf (errp, "%s", error.text);
+        return NULL;
+    }
+    if (command && !json_is_array (command)) {
+        errprintf (errp, "command must be an array");
+        return NULL;
+    }
+    if (kill_timeout > 0.) {
+        if (!prolog) {
+            errprintf (errp, "kill-timeout not allowed for epilog");
+            return NULL;
+        }
+    }
+    /*  If no command is set but exec.imp is non-NULL then set command to
+     *  [ "$imp_path", "run", "$name" ]
+     */
+    if (!command && perilog_config.imp_path) {
+        json_t *imp_cmd;
+        if ((imp_cmd = json_pack ("[sss]",
+                                  perilog_config.imp_path,
+                                  "run",
+                                  name)))
+            cmd = cmd_from_json (imp_cmd);
+        json_decref (imp_cmd);
+        if (!cmd) {
+            errprintf (errp, "error creating %s command", name);
+            return NULL;
+        }
+        uses_imp = true;
+    }
+    if (!(pd = calloc (1, sizeof (*pd)))) {
+        errprintf (errp, "Out of memory");
+        return NULL;
+    }
+    if (command && !(cmd = cmd_from_json (command))) {
+        errprintf (errp, "malformed %s command", prolog ? "prolog" : "epilog");
+        goto error;
+    }
+    if (timeout && fsd_parse_duration (timeout, &pd->timeout) < 0) {
+        errprintf (errp, "invalid %s timeout", prolog ? "prolog" : "epilog");
+        goto error;
+    }
+    /* Special case: INFINITY disables timeout so set timeout = 0.0:
+     */
+    if (pd->timeout == INFINITY)
+        pd->timeout = 0.;
+    if (!cmd) {
+        errprintf (errp, "no command specified and exec.imp not defined");
+        goto error;
+    }
+
+    pd->cmd = cmd;
+    pd->kill_timeout = kill_timeout > 0. ? kill_timeout : default_kill_timeout;
+    pd->per_rank = per_rank;
+    pd->prolog = prolog;
+    pd->uses_imp = uses_imp;
+
+    /* If cancel_on_exception unset, default to prolog=true, epilog=false
+     * Otherwise, use set value:
+     */
+    if (cancel_on_exception < 0)
+        pd->cancel_on_exception = prolog;
+    else
+        pd->cancel_on_exception = cancel_on_exception;
+
+    return pd;
+error:
+    flux_cmd_destroy (cmd);
+    perilog_procdesc_destroy (pd);
+    return NULL;
+}
+
+static struct perilog_proc * perilog_proc_create (flux_plugin_t *p,
+                                                  flux_jobid_t id,
+                                                  uint32_t userid,
+                                                  bool prolog)
+{
+    struct perilog_proc *proc = calloc (1, sizeof (*proc));
+    if (proc == NULL)
+        return NULL;
+    proc->p = p;
+    proc->id = id;
+    proc->userid = userid;
+    proc->prolog = prolog;
+    if (zhashx_insert (perilog_config.processes, &proc->id, proc) < 0) {
+        free (proc);
+        errno = EEXIST;
+        return NULL;
+    }
+    return proc;
+}
+
+static void perilog_proc_destroy (struct perilog_proc *proc)
+{
+    if (proc) {
+        int saved_errno = errno;
+        idset_destroy (proc->ranks);
+        free (proc->failed_ranks);
+        json_decref (proc->R);
+        bulk_exec_destroy (proc->bulk_exec);
+        flux_future_destroy (proc->kill_f);
+        flux_future_destroy (proc->drain_f);
+        flux_watcher_destroy (proc->timer);
+        flux_watcher_destroy (proc->kill_timer);
+        free (proc);
+        errno = saved_errno;
+    }
+}
+
+static const char *perilog_proc_name (struct perilog_proc *proc)
+{
+    return proc->prolog ? "prolog" : "epilog";
+}
+
+static double perilog_proc_timeout (struct perilog_proc *proc)
+{
+    if (proc->prolog)
+        return perilog_config.prolog->timeout;
+    return perilog_config.epilog->timeout;
+}
+
+/*  zhashx_destructor_fn prototype
+ */
+static void perilog_proc_destructor (void **item)
+{
+    if (item) {
+        struct perilog_proc *proc = *item;
+        /*  Delete this perilog_proc entry from job hash first,
+         *  since job-exception handler detects if a perilog is currently
+         *  executing by checking for the perilog_proc aux_item:
+         */
+        flux_jobtap_job_aux_set (proc->p,
+                                 proc->id,
+                                 "perilog_proc",
+                                 NULL,
+                                 NULL);
+        perilog_proc_destroy (proc);
+        *item = NULL;
+    }
+}
+
+/*  delete process from global hash - calls perilog_proc_destructor()
+ */
+static void perilog_proc_delete (struct perilog_proc *proc)
+{
+    if (proc) {
+        zhashx_delete (perilog_config.processes, &proc->id);
+    }
+}
+
+static void emit_finish_event (struct perilog_proc *proc,
+                               struct bulk_exec *bulk_exec)
+{
+    int status = bulk_exec_rc (bulk_exec);
+    if (proc->prolog) {
+        int rc;
+
+        /*
+         *  If prolog failed, raise job exception before prolog-finish
+         *   event is emitted to ensure job isn't halfway started before
+         *   the exception is raised:
+         */
+        if ((status != 0 && !proc->canceled) || proc->cancel_timeout) {
+            flux_t *h = flux_jobtap_get_flux (proc->p);
+            int code = WIFEXITED (status) ? WEXITSTATUS (status) : -1;
+            int sig;
+            char *errmsg;
+            char *hosts = NULL;
+
+            if (!(hosts = flux_hostmap_lookup (h, proc->failed_ranks, NULL)))
+                hosts = strdup ("unknown");
+
+            if (proc->cancel_timeout) {
+                rc = asprintf (&errmsg,
+                               "prolog canceled then timed out on %s (rank %s)",
+                               hosts,
+                               proc->failed_ranks);
+                status = 1;
+            }
+            else if (proc->timedout) {
+                rc = asprintf (&errmsg,
+                               "prolog timed out on %s (rank %s)",
+                               hosts,
+                               proc->failed_ranks);
+            }
+            /*  Report that prolog was signaled if WIFSIGNALED() is true, or
+             *  exit code > 128 (where standard exit code is 127+signo from
+             *  most shells)
+             */
+            else if (WIFSIGNALED (status) || code > 128) {
+                sig = WIFSIGNALED (status) ? WTERMSIG (status) : code - 128;
+                rc = asprintf (&errmsg,
+                               "prolog killed by signal %d on %s (rank %s)",
+                               sig,
+                               hosts ? hosts : "unknown",
+                               proc->failed_ranks);
+            }
+            else
+                rc = asprintf (&errmsg,
+                               "prolog exited with code=%d on %s (rank %s)",
+                               code,
+                               hosts ? hosts : "unknown",
+                               proc->failed_ranks);
+
+            free (hosts);
+            if (rc < 0)
+                errmsg = NULL;
+            if (flux_jobtap_raise_exception (proc->p,
+                                             proc->id,
+                                             "prolog",
+                                             0,
+                                             "%s",
+                                             errmsg ?
+                                             errmsg :
+                                             "job prolog failed") < 0)
+                    flux_log_error (flux_jobtap_get_flux (proc->p),
+                                    "prolog-finish: jobtap_raise_exception");
+            free (errmsg);
+        }
+        if (flux_jobtap_prolog_finish (proc->p,
+                                       proc->id,
+                                       "job-manager.prolog",
+                                       status) < 0)
+            flux_log_error (flux_jobtap_get_flux (proc->p),
+                            "flux_jobtap_prolog_finish: id=%s status=%d",
+                            idf58 (proc->id),
+                            status);
+    }
+    else {
+        /*
+         *  Epilog complete: unsubscribe this plugin from the
+         *   finished job and post an epilog-finish event.
+         *
+         *  No job exception is raised since the job is already exiting,
+         *   and it is expected that the actual epilog script will
+         *   drain nodes or take other action on failure if necessary.
+         */
+        flux_jobtap_job_unsubscribe (proc->p, proc->id);
+        if (flux_jobtap_epilog_finish (proc->p,
+                                       proc->id,
+                                       "job-manager.epilog",
+                                       status) < 0)
+            flux_log_error (flux_jobtap_get_flux (proc->p),
+                            "flux_jobtap_epilog_finish");
+    }
+}
+
+static bool subprocess_failed (flux_subprocess_t *p)
+{
+    if (flux_subprocess_state (p) == FLUX_SUBPROCESS_FAILED
+        || flux_subprocess_status (p) != 0)
+        return true;
+    return false;
+}
+
+/* Drain ranks that failed, are still active or both. */
+static flux_future_t *proc_drain_ranks (struct perilog_proc *proc,
+                                        bool drain_failed,
+                                        bool drain_active)
+{
+    struct idset *failed = NULL;
+    flux_future_t *f = NULL;
+    unsigned long rank;
+    const char *msg;
+    char reason[256];
+    flux_t *h = flux_jobtap_get_flux (proc->p);
+
+    if (!(failed = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+        flux_log_error (h, "drain_failed_ranks: idset_create");
+        goto out;
+    }
+
+    rank = idset_first (proc->ranks);
+    while (rank != IDSET_INVALID_ID) {
+        flux_subprocess_t *p;
+        if ((p = bulk_exec_get_subprocess (proc->bulk_exec, rank))) {
+            if ((drain_failed && subprocess_failed (p))
+                || (drain_active && flux_subprocess_active (p))) {
+                if (idset_set (failed, rank) < 0){
+                    flux_log_error (h,
+                                    "failed to add rank=%lu to drain set",
+                                    rank);
+                }
+            }
+        }
+        rank = idset_next (proc->ranks, rank);
+    }
+    if (!(proc->failed_ranks = idset_encode (failed, IDSET_FLAG_RANGE))) {
+        flux_log_error (h,
+                        "%s: error encoding %s failed ranks",
+                        idf58 (proc->id),
+                        perilog_proc_name (proc));
+        goto out;
+    }
+
+    if (proc->canceled)
+        msg = "canceled then timed out";
+    else if (proc->timedout)
+        msg = "timed out";
+    else
+        msg = "failed";
+
+    (void) snprintf (reason,
+                     sizeof (reason),
+                     "%s %s for job %s",
+                     perilog_proc_name (proc),
+                     msg,
+                     idf58 (proc->id));
+
+    if (!(f = flux_rpc_pack (h,
+                             "resource.drain",
+                             0,
+                             0,
+                             "{s:s s:s s:s}",
+                             "targets", proc->failed_ranks,
+                             "reason", reason,
+                             "mode", "update"))) {
+        flux_log (h,
+                  LOG_ERR,
+                  "%s: %s: failed to send drain RPC for ranks %s",
+                  idf58 (proc->id),
+                  perilog_proc_name (proc),
+                  proc->failed_ranks);
+        goto out;
+    }
+out:
+    idset_destroy (failed);
+    return f;
+}
+
+static bool perilog_proc_failed (struct perilog_proc *proc)
+{
+    if (proc->canceled
+        || proc->timedout
+        || bulk_exec_rc (proc->bulk_exec) > 0)
+        return true;
+    return false;
+}
+
+static void perilog_proc_finish (struct perilog_proc *proc)
+{
+    flux_t *h = flux_jobtap_get_flux (proc->p);
+    flux_plugin_t *p;
+    uint32_t userid;
+    flux_jobid_t id;
+    json_t *R;
+    struct perilog_procdesc *pd;
+    bool run_epilog = false;
+
+
+    /*  If a prolog was completing, and it failed in some way, then there
+     *  will be no finish event to trigger the epilog. However, an epilog
+     *  should still be run in case it is required to clean up or revert
+     *  something done by the prolog. So do that here.
+     */
+    if (proc->prolog
+        && perilog_proc_failed (proc)
+        && (pd = perilog_config.epilog)) {
+        /* epilog process can't be started until prolog perilog_proc is
+         * deleted, so capture necessary info here and set a boolean to
+         * create the epilog before leaving this function.
+         */
+        run_epilog = true;
+        p = proc->p;
+        id = proc->id;
+        userid = proc->userid;
+        R = proc->R;
+
+        /* The epilog-start event must be posted before the prolog-finish
+         * event to avoid the job potentially going straight to INACTIVE
+         * after the prolog-finish event is posted below
+         */
+        if (flux_jobtap_event_post_pack (p,
+                                         id,
+                                         "epilog-start",
+                                         "{s:s}",
+                                         "description",
+                                         "job-manager.epilog") < 0) {
+            flux_log_error (h,
+                            "%s: failed to post epilog-start on prolog-finish",
+                            idf58 (proc->id));
+            run_epilog = false;
+        }
+    }
+    emit_finish_event (proc, proc->bulk_exec);
+    perilog_proc_delete (proc);
+
+    if (run_epilog) {
+        struct perilog_proc *epilog;
+
+        if (!(epilog = procdesc_run (h, p, pd, id, userid, R))
+            || flux_jobtap_job_aux_set (p,
+                                        id,
+                                        "perilog_proc",
+                                        epilog,
+                                        NULL) < 0) {
+            flux_log_error (h,
+                            "%s: failed to start epilog on prolog-finish",
+                            idf58 (proc->id));
+
+            /* Since epilog-start event was emitted above, we must emit an
+             * epilog-finish event to avoid hanging the job
+             */
+            if (flux_jobtap_epilog_finish (p, id,"job-manager.epilog", 1) < 0) {
+                flux_log_error (h,
+                                "%s: failed to post epilog-finish event",
+                                idf58 (proc->id));
+            }
+            perilog_proc_delete (epilog);
+        }
+    }
+}
+
+static void drain_failed_cb (flux_future_t *f, void *arg)
+{
+    struct perilog_proc *proc = arg;
+    flux_t *h = flux_jobtap_get_flux (proc->p);
+
+    if (flux_future_get (f, NULL) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "Failed to drain ranks with failed %s for %s: %s",
+                  perilog_proc_name (proc),
+                  idf58 (proc->id),
+                  future_strerror (f, errno));
+    }
+    /* future destroyed by perilog_proc_delete()
+     */
+    perilog_proc_finish (proc);
+}
+
+static void proc_drain_and_finish (struct perilog_proc *proc,
+                                   bool drain_failed,
+                                   bool drain_active)
+{
+
+    if (drain_failed || drain_active) {
+        /* Drain the set of ranks that failed the prolog/epilog. If the
+         * drain RPC is successful, then wait for the response before
+         * emitting the "prolog/epilog-finish" event. O/w, resources could
+         * be freed and handed out to new jobs before they are drained.
+         */
+        if ((proc->drain_f = proc_drain_ranks (proc,
+                                               drain_failed,
+                                               drain_active))
+            && flux_future_then (proc->drain_f,
+                                 -1.,
+                                 drain_failed_cb,
+                                 proc) == 0)
+            return;
+
+        /* O/w, drain RPC failed, fall through so finish event is still
+         * emitted.
+         */
+    }
+    perilog_proc_finish (proc);
+}
+
+static void completion_cb (struct bulk_exec *bulk_exec, void *arg)
+{
+    struct perilog_proc *proc = bulk_exec_aux_get (bulk_exec, "perilog_proc");
+    if (proc) {
+        bool drain_failed = false;
+
+        if (proc->per_rank
+            && !proc->canceled
+            && bulk_exec_rc (bulk_exec) != 0)
+            drain_failed = true;
+
+        proc_drain_and_finish (proc, drain_failed, false);
+    }
+}
+
+static void error_cb (struct bulk_exec *bulk_exec,
+                      flux_subprocess_t *p,
+                      void *arg)
+{
+    struct perilog_proc *proc = bulk_exec_aux_get (bulk_exec, "perilog_proc");
+    flux_t *h = flux_jobtap_get_flux (proc->p);
+    int rank = flux_subprocess_rank (p);
+    const char *hostname = flux_get_hostbyrank (h, rank);
+    const char *error = flux_subprocess_fail_error (p);
+
+    if (!proc)
+        return;
+
+    flux_log (h,
+              LOG_ERR,
+              "%s: %s: %s (rank %d): %s",
+              idf58 (proc->id),
+              perilog_proc_name (proc),
+              hostname,
+              rank,
+              error);
+}
+
+static bool perilog_log_ignore (struct perilog_conf *conf, const char *s)
+{
+    if (conf->log_ignore) {
+        const regex_t *reg = zlistx_first (conf->log_ignore);
+        while (reg) {
+            if (regexec (reg, s, 0, NULL, 0) == 0)
+                return true;
+            reg = zlistx_next (conf->log_ignore);
+        }
+    }
+    return false;
+}
+
+static void io_cb (struct bulk_exec *bulk_exec,
+                   flux_subprocess_t *sp,
+                   const char *stream,
+                   const char *data,
+                   int len,
+                   void *arg)
+{
+    flux_t *h;
+    struct perilog_proc *proc;
+    char buf[len+1]; /* bulk_exec output is not NUL terminated */
+
+    if (len <= 0
+        || !(proc = bulk_exec_aux_get (bulk_exec, "perilog_proc"))
+        || !(h = flux_jobtap_get_flux (proc->p)))
+        return;
+
+    /* Copy data to NUL terminated buffer */
+    memcpy (buf, data, len);
+    buf [len] = '\0';
+
+    if (!perilog_log_ignore (&perilog_config, buf)) {
+        int level = LOG_INFO;
+        int rank = flux_subprocess_rank (sp);
+        const char *hostname = flux_get_hostbyrank (h, rank);
+
+        if (streq (stream, "stderr"))
+            level = LOG_ERR;
+        flux_log (h,
+                  level,
+                  "%s: %s: %s (rank %d): %s: %s",
+                  idf58 (proc->id),
+                  perilog_proc_name (proc),
+                  hostname,
+                  rank,
+                  stream,
+                  buf);
+    }
+}
+
+
+static void start_cb (struct bulk_exec *exec, void *arg)
+{
+    struct perilog_proc *proc = bulk_exec_aux_get (exec, "perilog_proc");
+    /*  Start timeout timer when processes have started
+     */
+    if (proc && proc->timer)
+        flux_watcher_start (proc->timer);
+}
+
+static struct bulk_exec_ops ops = {
+        .on_start = start_cb,
+        .on_exit = NULL,
+        .on_complete = completion_cb,
+        .on_error = error_cb,
+        .on_output = io_cb,
+};
+
+static struct idset *ranks_from_R (json_t *R)
+{
+    struct idset *ranks;
+    struct rlist *rl = rlist_from_json (R, NULL);
+    if (!rl)
+        return NULL;
+    ranks = rlist_ranks (rl);
+    rlist_destroy (rl);
+    return ranks;
+}
+
+static struct perilog_proc *procdesc_run (flux_t *h,
+                                          flux_plugin_t *p,
+                                          struct perilog_procdesc *pd,
+                                          flux_jobid_t id,
+                                          uint32_t userid,
+                                          json_t *R)
+{
+    struct perilog_proc *proc = NULL;
+    struct idset *ranks = NULL;
+    struct bulk_exec *bulk_exec = NULL;
+    double timeout;
+
+    if (!(proc = perilog_proc_create (p, id, userid, pd->prolog))) {
+        flux_log_error (h,
+                        "%s: proc_create",
+                        pd->prolog ? "prolog" : "epilog");
+        goto error;
+    }
+    if (flux_cmd_setenvf (pd->cmd, 1, "FLUX_JOB_ID", "%s", idf58 (id)) < 0
+        || flux_cmd_setenvf (pd->cmd, 1, "FLUX_JOB_USERID", "%u", userid) < 0) {
+        flux_log_error (h,
+                        "%s: flux_cmd_create",
+                        perilog_proc_name (proc));
+        goto error;
+    }
+    if (pd->per_rank) {
+        if (!(ranks = ranks_from_R (R))) {
+            flux_log (h,
+                      LOG_ERR,
+                      "%s: %s: failed to decode ranks from R",
+                      idf58 (id),
+                      perilog_proc_name (proc));
+            goto error;
+        }
+    }
+    else if (!(ranks = idset_decode ("0"))) {
+        flux_log_error (h, "%s: idset_decode", perilog_proc_name (proc));
+        goto error;
+    }
+
+    if (!(bulk_exec = bulk_exec_create (&ops,
+                                        "rexec",
+                                        id,
+                                        perilog_proc_name (proc),
+                                        NULL))
+        || bulk_exec_push_cmd (bulk_exec, ranks, pd->cmd, 0) < 0) {
+        flux_log_error (h,
+                        "failed to create %s bulk exec cmd for %s",
+                        perilog_proc_name (proc),
+                        idf58 (id));
+        goto error;
+    }
+    if (bulk_exec_start (h, bulk_exec) < 0) {
+        flux_log_error (h, "%s: bulk_exec_start", perilog_proc_name (proc));
+        goto error;
+    }
+    if (bulk_exec_aux_set (bulk_exec, "perilog_proc", proc, NULL) < 0) {
+        flux_log_error (h,
+                        "%s: bulk_exec_aux_set",
+                        perilog_proc_name (proc));
+        goto error;
+    }
+    timeout = perilog_proc_timeout (proc);
+    if (timeout > 0.0) {
+        flux_watcher_t *w;
+        if (!(w = flux_timer_watcher_create (flux_get_reactor (h),
+                                             timeout,
+                                             0.,
+                                             timeout_cb,
+                                             proc))) {
+            flux_log_error (h,
+                            "%s: failed to create timeout timer",
+                            perilog_proc_name (proc));
+            goto error;
+        }
+        proc->timer = w;
+        /* Note: watcher will be started in bulk-exec start callback
+         */
+    }
+    proc->R = json_incref (R);
+    proc->bulk_exec = bulk_exec;
+    proc->ranks = ranks;
+    proc->per_rank = pd->per_rank;
+    proc->cancel_on_exception = pd->cancel_on_exception;
+    proc->kill_timeout = pd->kill_timeout;
+
+    /* proc now has ownership of bulk_exec, ranks
+     */
+    return proc;
+error:
+    idset_destroy (ranks);
+    bulk_exec_destroy (bulk_exec);
+    perilog_proc_destroy (proc);
+    return NULL;
+}
+
+static int run_command (flux_plugin_t *p,
+                        flux_plugin_arg_t *args,
+                        struct perilog_procdesc *pd)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+    struct perilog_proc *proc = NULL;
+    flux_jobid_t id;
+    uint32_t userid;
+    json_t *R;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:i s:o}",
+                                "id", &id,
+                                "userid", &userid,
+                                "R", &R) < 0) {
+        flux_log_error (h, "flux_plugin_arg_unpack");
+        return -1;
+    }
+
+    if (!(proc = procdesc_run (h, p, pd, id, userid, R)))
+        return -1;
+
+    if (flux_jobtap_job_aux_set (p,
+                                 FLUX_JOBTAP_CURRENT_JOB,
+                                 "perilog_proc",
+                                 proc,
+                                 NULL) < 0) {
+        flux_log_error (h,
+                        "%s: flux_jobtap_job_aux_set",
+                        perilog_proc_name (proc));
+        goto error;
+    }
+    return 0;
+error:
+    perilog_proc_destroy (proc);
+    return 01;
+}
+
+static int run_cb (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *arg)
+{
+
+    /*
+     *  Subscribe to job events if an epilog or prolog command is
+     *   registered. This is needed to allow this plugin to subscribe
+     *   to the finish event for the epilog, and any exception events
+     *   for the prolog (so it can be canceled).
+     */
+    if (perilog_config.epilog || perilog_config.prolog) {
+        if (flux_jobtap_job_subscribe (p, FLUX_JOBTAP_CURRENT_JOB) < 0) {
+            flux_jobtap_raise_exception (p,
+                                         FLUX_JOBTAP_CURRENT_JOB,
+                                         "prolog",
+                                         0,
+                                         "failed to subscribe to job events");
+            return -1;
+        }
+    }
+
+    if (perilog_config.prolog == NULL)
+        return 0;
+
+    if (run_command (p, args, perilog_config.prolog) < 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "prolog",
+                                     0,
+                                     "failed to start job prolog");
+        return -1;
+    }
+    return flux_jobtap_prolog_start (p, "job-manager.prolog");
+}
+
+static int job_finish_cb (flux_plugin_t *p,
+                          const char *topic,
+                          flux_plugin_arg_t *args,
+                          void *arg)
+{
+    if (perilog_config.epilog == NULL)
+        return 0;
+
+    /*  Don't start new epilog processes if the broker is shutting down.
+     *   Flux currently cancels running jobs as part of shutdown.  If the
+     *   broker takes longer than the systemd TimeoutStopSec (e.g. 90s) to
+     *   stop, it may be killed and data may be lost.  Since epilog scripts
+     *   are site-defined and may take an arbitrarily long time to run,
+     *   simply skip them during shutdown.  This may be relaxed once Flux
+     *   is capable of restarting with running jobs.
+     */
+    if (perilog_config.shutting_down)
+        return 0;
+
+    if (run_command (p, args, perilog_config.epilog) < 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "epilog",
+                                     0,
+                                     "failed to start job epilog");
+        return -1;
+    }
+    return flux_jobtap_epilog_start (p, "job-manager.epilog");
+}
+
+static void proc_kill_cb (flux_future_t *f, void *arg)
+{
+    struct perilog_proc *proc = arg;
+    flux_t *h = flux_future_get_flux (f);
+
+    if (flux_future_get (f, NULL) < 0) {
+        flux_log_error (h,
+                        "%s: Failed to signal job %s",
+                        idf58 (proc->id),
+                        perilog_proc_name (proc));
+    }
+}
+
+static int proc_kill (struct perilog_proc *proc)
+{
+    flux_t *h = flux_jobtap_get_flux (proc->p);
+
+    if (proc->kill_timer)
+        return 0;
+
+    if (!(proc->kill_f = bulk_exec_kill (proc->bulk_exec, NULL, SIGTERM)))
+        return -1;
+
+    if (flux_future_then (proc->kill_f, -1., proc_kill_cb, proc) < 0) {
+        flux_log_error (h, "proc_kill: flux_future_then");
+        flux_future_destroy (proc->kill_f);
+        proc->kill_f = NULL;
+        return -1;
+    }
+
+    return 0;
+}
+
+static void proc_kill_timeout_cb (flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg)
+{
+    struct perilog_proc *proc = arg;
+    flux_t *h = flux_jobtap_get_flux (proc->p);
+    flux_log (h,
+              LOG_ERR,
+              "%s: timed out waiting for SIGTERM to terminate %s",
+              idf58 (proc->id),
+              perilog_proc_name (proc));
+    /*  Drain active ranks and post finish event
+     */
+    proc->cancel_timeout = true;
+    proc_drain_and_finish (proc, false, true);
+}
+
+static int proc_kill_timer_start (struct perilog_proc *proc, double timeout)
+{
+    if (proc->kill_timer == NULL) {
+        flux_t *h = flux_jobtap_get_flux (proc->p);
+        flux_reactor_t *r = flux_get_reactor (h);
+        proc->kill_timer = flux_timer_watcher_create (r,
+                                                      timeout,
+                                                      0.,
+                                                      proc_kill_timeout_cb,
+                                                      proc);
+        if (!proc->kill_timer) {
+            flux_log_error (h,
+                            "%s: failed to start %s kill timer",
+                            idf58 (proc->id),
+                            perilog_proc_name (proc));
+            /* Since timer cb won't be run, drain and send finish event now
+             */
+            proc_drain_and_finish (proc, false, true);
+            return -1;
+        }
+        flux_watcher_start (proc->kill_timer);
+    }
+    return 0;
+}
+
+static void timeout_cb (flux_reactor_t *r,
+                        flux_watcher_t *w,
+                        int revents,
+                        void *arg)
+{
+    struct perilog_proc *proc = arg;
+    proc->timedout = true;
+    if (proc_kill (proc) < 0)
+        flux_log_error (flux_jobtap_get_flux (proc->p),
+                        "failed to kill %s for %s",
+                        perilog_proc_name (proc),
+                        idf58 (proc->id));
+    (void) proc_kill_timer_start (proc, proc->kill_timeout);
+}
+
+static int exception_cb (flux_plugin_t *p,
+                         const char *topic,
+                         flux_plugin_arg_t *args,
+                         void *arg)
+{
+    /*  On exception, kill any prolog running for this job:
+     *   Follow up with SIGKILL after 10s.
+     */
+    struct perilog_proc *proc;
+    int severity;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:{s:{s:i}}}",
+                                "entry",
+                                "context",
+                                "severity", &severity) < 0)
+        return -1;
+
+    if (severity == 0
+        && (proc = flux_jobtap_job_aux_get (p,
+                                           FLUX_JOBTAP_CURRENT_JOB,
+                                           "perilog_proc"))
+        && proc->cancel_on_exception
+        && !proc->canceled
+        && bulk_exec_active_count (proc->bulk_exec) > 0) {
+
+        /* Set canceled flag to disable draining of failed prolog nodes
+         */
+        proc->canceled = true;
+        if (proc_kill (proc) < 0
+            || proc_kill_timer_start (proc, proc->kill_timeout) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static regex_t *regexp_create (const char *pattern)
+{
+    regex_t *reg = calloc (1, sizeof (*reg));
+    if (!reg)
+        return NULL;
+    if (regcomp (reg, pattern, REG_EXTENDED | REG_NOSUB) != 0) {
+        free (reg);
+        return NULL;
+    }
+    return reg;
+}
+
+static void regexp_destroy (regex_t *reg)
+{
+    if (reg) {
+        int saved_errno = errno;
+        regfree (reg);
+        free (reg);
+        errno = saved_errno;
+    }
+}
+
+static void regexp_free (void **item)
+{
+    if (item) {
+        regex_t *reg = *item;
+        regexp_destroy (reg);
+        reg = NULL;
+    }
+}
+
+static int regexp_list_append (zlistx_t *l,
+                               const char *pattern,
+                               flux_error_t *errp);
+
+static zlistx_t *regexp_list_create ()
+{
+    zlistx_t *l = NULL;
+    if (!(l = zlistx_new ()))
+        return NULL;
+    zlistx_set_destructor (l, regexp_free);
+
+    /*  Always ignore empty lines
+     */
+    if (regexp_list_append (l, "^\\s*$", NULL) < 0) {
+        zlistx_destroy (&l);
+        return NULL;
+    }
+    return l;
+}
+
+static int regexp_list_append (zlistx_t *l,
+                               const char *pattern,
+                               flux_error_t *errp)
+{
+    regex_t *reg = NULL;
+    if (!(reg = regexp_create (pattern))) {
+        errprintf (errp, "Failed to compile regex: %s", pattern);
+        return -1;
+    }
+    if (!zlistx_add_end (l, reg)) {
+        regexp_destroy (reg);
+        errprintf (errp, "Out of memory adding regex pattern");
+        return -1;
+    }
+    return 0;
+}
+
+static int regexp_list_append_array (zlistx_t *l,
+                                     json_t *array,
+                                     flux_error_t *errp)
+{
+    size_t index;
+    json_t *entry;
+
+    if (!json_is_array (array)) {
+        errprintf (errp, "not an array");
+        return -1;
+    }
+
+    json_array_foreach (array, index, entry) {
+        const char *pattern = json_string_value (entry);
+        if (pattern == NULL) {
+            errprintf (errp, "all entries must be a string value");
+            return -1;
+        }
+        if (regexp_list_append (l, pattern, errp) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static void monitor_continuation (flux_future_t *f, void *arg)
+{
+    struct perilog_conf *conf = arg;
+    flux_t *h = flux_future_get_flux (f);
+    int state = -1;
+
+    if (flux_rpc_get_unpack (f, "{s:i}", "state", &state) < 0) {
+        if (errno != ENODATA) {
+            flux_log (h,
+                      LOG_ERR,
+                      "error watching broker state: %s",
+                      future_strerror (f, errno));
+        }
+        return;
+    }
+    if (state == STATE_CLEANUP) // the broker state, not a job state!
+        conf->shutting_down = true;
+    flux_future_reset (f);
+}
+
+static void free_config (struct perilog_conf *conf)
+{
+    flux_future_destroy (conf->watch_f);
+    perilog_procdesc_destroy (conf->prolog);
+    perilog_procdesc_destroy (conf->epilog);
+    zhashx_destroy (&conf->processes);
+    zlistx_destroy (&conf->log_ignore);
+}
+
+/*   Initialize a perilog_config object
+ */
+static int conf_init (flux_plugin_t *p, struct perilog_conf *conf)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+
+    memset (conf, 0, sizeof (*conf));
+    conf->initialized = true;
+
+    if (!(conf->processes = job_hash_create ())) {
+        flux_log (h, LOG_ERR, "perilog: failed to create job hash");
+        return -1;
+    }
+    zhashx_set_destructor (conf->processes,
+                           perilog_proc_destructor);
+
+    /* Watch for broker transition to CLEANUP.
+     */
+    if (!(conf->watch_f = flux_rpc_pack (h,
+                                         "state-machine.monitor",
+                                          0,
+                                          FLUX_RPC_STREAMING,
+                                          "{s:i}",
+                                          "final", STATE_CLEANUP))
+        || flux_future_then (conf->watch_f,
+                             -1,
+                             monitor_continuation,
+                             conf) < 0) {
+        flux_log_error (h, "perilog: error watching broker state");
+        goto error;
+    }
+
+    /* Free config at plugin destruction:
+     */
+    if (flux_plugin_aux_set (p, NULL, conf, (flux_free_f) free_config) < 0)
+        goto error;
+
+    return 0;
+error:
+    free_config (conf);
+    return -1;
+}
+
+static int watcher_remaining_time (flux_watcher_t *w)
+{
+    double next_wakeup = flux_watcher_next_wakeup (w);
+    return (int) round (next_wakeup - flux_reactor_time ());
+}
+
+static json_t *proc_to_json (struct perilog_proc *proc)
+{
+    const char *state;
+    struct idset *active_ranks;
+    char *ranks = NULL;
+    int remaining_time = -1.;
+    json_t *o;
+    int total = bulk_exec_total (proc->bulk_exec);
+    int active = total - bulk_exec_complete (proc->bulk_exec);
+
+    if (proc->canceled)
+        state = "canceled";
+    else if (proc->timedout)
+        state = "timeout";
+    else
+        state = "running";
+
+    if ((active_ranks = bulk_exec_active_ranks (proc->bulk_exec)))
+        ranks = idset_encode (active_ranks, IDSET_FLAG_RANGE);
+
+    if (proc->kill_timer)
+        remaining_time = watcher_remaining_time (proc->kill_timer);
+    else if (proc->timer)
+        remaining_time = watcher_remaining_time (proc->timer);
+
+    o = json_pack ("{s:s s:s s:i s:i s:s s:i}",
+                   "name", perilog_proc_name (proc),
+                   "state", state,
+                   "total", total,
+                   "active", active,
+                   "active_ranks", ranks ? ranks : "",
+                   "remaining_time", remaining_time);
+    free (ranks);
+    idset_destroy (active_ranks);
+    return o;
+}
+
+static json_t *cmdline_tojson (flux_cmd_t *cmd)
+{
+    int argc;
+    json_t *o = json_array ();
+
+    if (!o)
+        return NULL;
+
+    argc = flux_cmd_argc (cmd);
+    for (int i = 0; i < argc; i++) {
+        json_t *arg;
+        if ((!(arg = json_string (flux_cmd_arg (cmd, i))))
+            || json_array_append_new (o, arg) < 0) {
+            json_decref (arg);
+            goto error;
+        }
+    }
+    return o;
+error:
+    json_decref (o);
+    return NULL;
+}
+
+static json_t *procdesc_to_json (struct perilog_procdesc *pd)
+{
+    json_t *cmd = NULL;
+    json_t *o;
+
+    if (!pd || !pd->cmd)
+        return json_object ();
+
+    if (!(cmd = cmdline_tojson (pd->cmd)))
+        return NULL;
+
+    o = json_pack ("{s:O s:b s:b s:f s:f}",
+                   "command", cmd,
+                   "per_rank", pd->per_rank,
+                   "cancel_on_exception", pd->cancel_on_exception,
+                   "timeout", pd->timeout,
+                   "kill-timeout", pd->kill_timeout);
+
+    json_decref (cmd);
+    return o;
+}
+
+static json_t *conf_to_json (struct perilog_conf *conf)
+{
+    json_t *o = NULL;
+    json_t *prolog = procdesc_to_json (perilog_config.prolog);
+    json_t *epilog = procdesc_to_json (perilog_config.epilog);
+
+
+    if (!prolog || !epilog)
+        goto out;
+    o = json_pack ("{s:O s:O}",
+                   "prolog", prolog,
+                   "epilog", epilog);
+out:
+    json_decref (prolog);
+    json_decref (epilog);
+    return o;
+}
+
+static json_t *procs_to_json (zhashx_t *processes)
+{
+    struct perilog_proc *proc;
+    json_t *o = NULL;
+
+    if (!(o = json_object ()))
+        return NULL;
+
+    proc = zhashx_first (processes);
+    while (proc) {
+        json_t *entry;
+        if (!(entry = proc_to_json (proc))
+            || json_object_set_new (o, idf58 (proc->id), entry) < 0) {
+            json_decref (entry);
+            goto error;
+        }
+        proc = zhashx_next (processes);
+    }
+    return o;
+error:
+    json_decref (o);
+    return NULL;
+}
+
+static int query_cb (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *arg)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+    json_t *conf = NULL;
+    json_t *procs = NULL;
+    int rc = -1;
+
+    if (!(conf = conf_to_json (&perilog_config))
+        || !(procs = procs_to_json (perilog_config.processes))) {
+        flux_log (h,
+                  LOG_ERR,
+                  "perilog: failed to create query_cb json results");
+        goto out;
+    }
+
+    if ((rc = flux_plugin_arg_pack (args,
+                                    FLUX_PLUGIN_ARG_OUT,
+                                    "{s:O s:O}",
+                                    "conf", conf,
+                                    "procs", procs)) < 0)
+        flux_log_error (h,
+                        "perilog: query_cb: flux_plugin_arg_pack: %s",
+                        flux_plugin_arg_strerror (args));
+out:
+    json_decref (conf);
+    json_decref (procs);
+    return rc;
+}
+
+static int conf_update_cb (flux_plugin_t *p,
+                           const char *topic,
+                           flux_plugin_arg_t *args,
+                           void *arg)
+{
+    json_t *conf;
+    flux_error_t error;
+    json_error_t jerror;
+    const char *imp_path = NULL;
+    json_t *prolog_config = NULL;
+    json_t *epilog_config = NULL;
+    struct perilog_procdesc *prolog = NULL;
+    struct perilog_procdesc *epilog = NULL;
+    json_t *log_ignore_config = NULL;
+    zlistx_t *log_ignore = NULL;
+
+    /* Perform one-time initialization of config if necessary
+     */
+    if (!perilog_config.initialized
+        && conf_init (p, &perilog_config) < 0) {
+        errprintf (&error, "failed to initialize perilog config");
+        goto error;
+    }
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:o}",
+                                "conf", &conf) < 0) {
+        errprintf (&error,
+                   "perilog: error unpackage conf.update arguments: %s",
+                   flux_plugin_arg_strerror (args));
+        goto error;
+    }
+
+    if (json_unpack_ex (conf,
+                        &jerror,
+                        0,
+                        "{s?{s?s} s?{s?o s?o s?{s?o}}}",
+                        "exec",
+                          "imp", &imp_path,
+                        "job-manager",
+                          "prolog", &prolog_config,
+                          "epilog", &epilog_config,
+                          "perilog",
+                            "log-ignore", &log_ignore_config) < 0) {
+        errprintf (&error,
+                   "perilog: error unpacking config: %s",
+                   jerror.text);
+        goto error;
+    }
+
+    /*  Capture IMP path before first call to perilog_procdesc_create()
+     */
+    free (perilog_config.imp_path);
+    perilog_config.imp_path = NULL;
+    if (imp_path) {
+        if (!(perilog_config.imp_path = strdup (imp_path))) {
+            errprintf (&error, "failed to duplicate imp_path");
+            goto error;
+        }
+    }
+
+    if (prolog_config) {
+        flux_error_t perr;
+        if (!(prolog = perilog_procdesc_create (prolog_config,
+                                                true,
+                                                &perr))) {
+            errprintf (&error,
+                       "[job-manager.prolog]: %s",
+                       perr.text);
+            goto error;
+        }
+    }
+    if (epilog_config) {
+        flux_error_t perr;
+        if (!(epilog = perilog_procdesc_create (epilog_config,
+                                                false,
+                                                &perr))) {
+            errprintf (&error,
+                       "[job-manager.epilog]: %s",
+                       perr.text);
+            goto error;
+        }
+    }
+
+    /* Always start with default log_ignore list (ignores empty lines)
+     */
+    if (!(log_ignore = regexp_list_create ())) {
+        errprintf (&error, "Out of memory creating log_ignore list");
+        goto error;
+    }
+    if (log_ignore_config) {
+        flux_error_t perr;
+        if (regexp_list_append_array (log_ignore,
+                                      log_ignore_config,
+                                      &perr) < 0) {
+            errprintf (&error,
+                       "[job-manager.perilog]: error parsing log-ignore: %s",
+                       perr.text);
+            goto error;
+        }
+    }
+
+    /*  Swap config:
+     */
+    perilog_procdesc_destroy (perilog_config.prolog);
+    perilog_config.prolog = prolog;
+
+    perilog_procdesc_destroy (perilog_config.epilog);
+    perilog_config.epilog = epilog;
+
+    zlistx_destroy (&perilog_config.log_ignore);
+    perilog_config.log_ignore = log_ignore;
+
+    return 0;
+error:
+    perilog_procdesc_destroy (epilog);
+    perilog_procdesc_destroy (prolog);
+    zlistx_destroy (&log_ignore);
+    return flux_jobtap_error (p, args, "%s", error.text);
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.state.run",       run_cb,       NULL },
+    { "job.event.finish",    job_finish_cb,NULL },
+    { "job.event.exception", exception_cb, NULL },
+    { "conf.update",  conf_update_cb, NULL },
+    { "plugin.query", query_cb, NULL },
+    { 0 }
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    perilog_config.initialized = false;
+    return flux_plugin_register (p, "perilog", tab);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/plugins/post-event.c b/src/modules/job-manager/plugins/post-event.c
new file mode 100644
index 000000000000..6e54a3513eca
--- /dev/null
+++ b/src/modules/job-manager/plugins/post-event.c
@@ -0,0 +1,59 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* post-event.c - post manual events to job eventlog
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+static void post_event_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    flux_plugin_t *p = arg;
+    flux_jobid_t id;
+    const char *name;
+    json_t *context = NULL;
+
+    if (flux_msg_unpack (msg,
+                         "{s:I s:s s?o}",
+                         "id", &id,
+                         "name", &name,
+                         "context", &context) < 0)
+        goto error;
+    if (context) {
+        if (flux_jobtap_event_post_pack (p, id, name, "O", context) < 0)
+            goto error;
+    }
+    else if (flux_jobtap_event_post_pack (p, id, name, NULL) < 0)
+            goto error;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to job-manager.post-event");
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to job-manager.post-event");
+}
+
+
+int post_event_init (flux_plugin_t *p)
+{
+    if (flux_jobtap_service_register_ex (p, "post", 0, post_event_cb, p) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/plugins/priority-default.c b/src/modules/job-manager/plugins/priority-default.c
new file mode 100644
index 000000000000..8b8347f310ba
--- /dev/null
+++ b/src/modules/job-manager/plugins/priority-default.c
@@ -0,0 +1,72 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* priority-default.c - builtin default priority plugin.
+ *
+ * Simply sets priority to current urgency.
+ *
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+/*  The current implementation of priority.get just copies
+ *   the urgency to the priority.
+ */
+static int priority_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    int urgency = -1;
+    flux_t *h = flux_jobtap_get_flux (p);
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:i}",
+                                "urgency", &urgency) < 0) {
+        flux_log (h, LOG_ERR,
+                 "flux_plugin_arg_unpack: %s",
+                 flux_plugin_arg_strerror (args));
+        return -1;
+    }
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                             "{s:i}",
+                             "priority", urgency) < 0) {
+        flux_log (h, LOG_ERR,
+                 "flux_plugin_arg_pack: %s",
+                 flux_plugin_arg_strerror (args));
+        return -1;
+    }
+    return 0;
+}
+
+int priority_default_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_add_handler (p,
+                                 "job.state.priority",
+                                 priority_cb,
+                                 NULL) < 0
+        || flux_plugin_add_handler (p,
+                                    "job.priority.get",
+                                    priority_cb,
+                                    NULL) < 0) {
+        return -1;
+    }
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
diff --git a/src/modules/job-manager/plugins/submit-hold.c b/src/modules/job-manager/plugins/submit-hold.c
new file mode 100644
index 000000000000..507e523fb14c
--- /dev/null
+++ b/src/modules/job-manager/plugins/submit-hold.c
@@ -0,0 +1,55 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* submit-hold.c - jobtap plugin that holds all submitted jobs
+ *
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <sys/types.h>
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+/* Set all jobs urgency to FLUX_JOB_URGENCY_HOLD.
+ */
+static int depend_cb (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *arg)
+{
+    int urgency;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:i}",
+                                "urgency", &urgency) < 0)
+        return -1;
+
+    if (urgency != FLUX_JOB_URGENCY_HOLD) {
+        return flux_jobtap_event_post_pack (p,
+                                            FLUX_JOBTAP_CURRENT_JOB,
+                                            "urgency",
+                                            "{s:i s:i}",
+                                            "userid", getuid (),
+                                            "urgency", FLUX_JOB_URGENCY_HOLD);
+    }
+    return 0;
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_add_handler (p, "job.state.depend", depend_cb, NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/plugins/update-duration.c b/src/modules/job-manager/plugins/update-duration.c
new file mode 100644
index 000000000000..42dcc2d82561
--- /dev/null
+++ b/src/modules/job-manager/plugins/update-duration.c
@@ -0,0 +1,94 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* update-duration.c - allow updates of attributes.system.duration for jobs
+ *
+ * This plugin implements a 'job.update.attributes.system.duration'
+ * callback to enable duration updates for pending jobs.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+/*
+ *  Allow instance owner to update duration to any value, even if it
+ *  exceeds a configured duration limit. By default, this is true, to
+ *  disable this behavior, reload the `.update-duration` plugin with
+ *  owner-allow-any=0.
+ */
+static int owner_allow_any = 1;
+
+static int duration_update_cb (flux_plugin_t *p,
+                               const char *topic,
+                               flux_plugin_arg_t *args,
+                               void *arg)
+{
+    struct flux_msg_cred cred;
+    flux_job_state_t state;
+    double duration;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:F s:i s:{s:i s:i}}",
+                                "value", &duration,
+                                "state", &state,
+                                "cred",
+                                  "userid", &cred.userid,
+                                  "rolemask", &cred.rolemask) < 0) {
+        flux_jobtap_error (p, args, "plugin args unpack failed");
+        return -1;
+    }
+    if (duration < 0.) {
+        flux_jobtap_error (p, args, "duration must not be negative");
+        return -1;
+    }
+    if (state == FLUX_JOB_STATE_RUN
+        || state == FLUX_JOB_STATE_CLEANUP) {
+        /*  Allow update of duration of running job for instance owner
+         *  only:
+         */
+        if (!(cred.rolemask & FLUX_ROLE_OWNER) || !owner_allow_any) {
+            flux_jobtap_error (p,
+                               args,
+                               "duration update of running job requires"
+                               " instance owner privileges");
+            return -1;
+        }
+    }
+    if ((cred.rolemask & FLUX_ROLE_OWNER) && owner_allow_any) {
+        /* If owner is allowed to make any duration adjustment, then
+         * report that value is validated via out arguments:
+         */
+        flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:i}",
+                              "validated", 1);
+    }
+    return 0;
+}
+
+int update_duration_plugin_init (flux_plugin_t *p)
+{
+    flux_plugin_conf_unpack (p,
+                             "{s:i}",
+                             "owner-allow-any",
+                             &owner_allow_any);
+    return flux_plugin_add_handler (p,
+                                    "job.update.attributes.system.duration",
+                                    duration_update_cb,
+                                    NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/plugins/validate-duration.c b/src/modules/job-manager/plugins/validate-duration.c
new file mode 100644
index 000000000000..81e4ec21e3d2
--- /dev/null
+++ b/src/modules/job-manager/plugins/validate-duration.c
@@ -0,0 +1,125 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/*  validate-duration.c - Ensure a job's duration doesn't exceed
+ *   the current resource expiration (at the moment of submission)
+ */
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <math.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "src/common/libutil/fsd.h"
+
+static double expiration = 0.;
+
+static int job_duration_check (flux_plugin_t *p,
+                               flux_plugin_arg_t *args,
+                               double duration)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+
+    if (h && duration > 0. && expiration > 0.) {
+        flux_reactor_t *r = flux_get_reactor (h);
+        double now = r ? flux_reactor_now (r) : flux_reactor_time ();
+        double rem = expiration - now;
+
+        if (duration > rem) {
+            char dfsd [64];
+            char rfsd [64];
+
+            if (fsd_format_duration_ex (dfsd, sizeof (dfsd), duration, 2) < 0
+                || fsd_format_duration_ex (rfsd, sizeof (rfsd), rem, 2) < 0) {
+                const char *msg = "duration exceeds instance lifetime";
+                return flux_jobtap_reject_job (p, args, "%s", msg);
+            }
+            return flux_jobtap_reject_job (p,
+                                           args,
+                                           "job duration (%s) exceeds "
+                                           "remaining instance lifetime (%s)",
+                                           dfsd,
+                                           rfsd);
+        }
+    }
+    return 0;
+}
+
+static int validate_duration (flux_plugin_t *p,
+                              const char *topic,
+                              flux_plugin_arg_t *args,
+                              void *arg)
+{
+    double duration = -1.;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:{s:{s?{s?F}}}}",
+                                "jobspec",
+                                 "attributes",
+                                  "system",
+                                   "duration", &duration) < 0) {
+        return flux_jobtap_reject_job (p,
+                                       args,
+                                       "failed to unpack duration: %s",
+                                       flux_plugin_arg_strerror (args));
+    }
+    return job_duration_check (p, args, duration);
+}
+
+static void kvs_lookup_cb (flux_future_t *f, void *arg)
+{
+    flux_t *h = flux_future_get_flux (f);
+    double val;
+    if (flux_kvs_lookup_get_unpack (f,
+                                    "{s:{s:F}}",
+                                    "execution",
+                                      "expiration", &val) < 0) {
+        flux_log_error (h, "flux_kvs_lookup_unpack");
+    }
+    flux_future_reset (f);
+    if (fabs (val - expiration) < 1.e-5)
+        return;
+    expiration = val;
+    flux_log (h,
+              LOG_DEBUG,
+              "duration-validator: updated expiration to %.2f",
+              expiration);
+}
+
+int validate_duration_plugin_init (flux_plugin_t *p)
+{
+    flux_t *h;
+    flux_future_t *f;
+
+    flux_plugin_set_name (p, ".validate-duration");
+
+    h = flux_jobtap_get_flux (p);
+
+    if (!(f = flux_kvs_lookup (h,
+                               NULL,
+                               FLUX_KVS_WATCH | FLUX_KVS_WAITCREATE,
+                               "resource.R"))) {
+        flux_log_error (h, "flux_kvs_lookup");
+        return -1;
+    }
+    flux_plugin_aux_set (p, NULL, f, (flux_free_f) flux_future_destroy);
+    if (flux_future_then (f, -1., kvs_lookup_cb, NULL) < 0) {
+        flux_log_error (h, "flux_kvs_lookup_get_unpack");
+        return -1;
+    }
+    return flux_plugin_add_handler (p,
+                                    "job.validate",
+                                    validate_duration,
+                                    NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/prioritize.c b/src/modules/job-manager/prioritize.c
new file mode 100644
index 000000000000..8d0e32410d60
--- /dev/null
+++ b/src/modules/job-manager/prioritize.c
@@ -0,0 +1,270 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* prioritize - job priority related functions
+ *
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "src/common/libjob/idf58.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "job.h"
+#include "event.h"
+#include "alloc.h"
+#include "jobtap-internal.h"
+#include "job-manager.h"
+
+#include "prioritize.h"
+
+static int sched_prioritize (flux_t *h, json_t *priorities)
+{
+    flux_future_t *f;
+    if (json_array_size (priorities) == 0) {
+        json_decref (priorities);
+        return 0;
+    }
+    if (!(f = flux_rpc_pack (h,
+                             "sched.prioritize",
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_NORESPONSE,
+                             "{s:o}",
+                             "jobs", priorities)))
+        return -1;
+    flux_future_destroy (f);
+    return 0;
+}
+
+static int sched_prioritize_one (struct job_manager *ctx, struct job *job)
+{
+    json_t * priorities = json_pack ("[[II]]", job->id, job->priority);
+    if (!priorities) {
+        flux_log (ctx->h, LOG_ERR, "sched_prioritize: json_pack failed");
+        return -1;
+    }
+    if (sched_prioritize (ctx->h, priorities) < 0) {
+        flux_log_error (ctx->h,
+                        "rpc: sched.priority: id=%s",
+                        idf58 (job->id));
+        json_decref (priorities);
+        return -1;
+    }
+    return 0;
+}
+
+static int reprioritize_one (struct job_manager *ctx,
+                             struct job *job,
+                             int64_t priority,
+                             bool oneshot)
+{
+    int flags = 0;
+
+    /*  Urgency values that specify "hold" and "expedite" override
+     *   requested priority:
+     */
+    if (job->urgency == FLUX_JOB_URGENCY_HOLD)
+        priority = FLUX_JOB_PRIORITY_MIN;
+    else if (job->urgency == FLUX_JOB_URGENCY_EXPEDITE)
+        priority = FLUX_JOB_PRIORITY_MAX;
+
+    /*
+     *  If priority did not change, _and_ the job is in SCHED state,
+     *   then do not post a priority event, since this would be useless
+     *   noise in the eventlog. However, be sure to post a priority event
+     *   in PRIORITY state, since this is what transitions a job to the
+     *   SCHED state.
+     */
+    if (priority == job->priority && job->state == FLUX_JOB_STATE_SCHED)
+        return 0;
+
+    /*  Priority event is only posted to job eventlog in
+     *   PRIORITY state, or transition of priority away from
+     *   MIN (held) or MAX (expedited).
+     */
+    /*  XXX: Disable for now, tests assume all priority updates
+     *       go to KVS eventlog
+    if (!(job->flags & FLUX_JOB_DEBUG)
+        && job->state != FLUX_JOB_STATE_PRIORITY
+        && job->priority != FLUX_JOB_PRIORITY_MIN
+        && job->priority != FLUX_JOB_PRIORITY_MAX)
+        flags = EVENT_NO_COMMIT;
+     */
+
+    /*  Post 'priority' event.
+     *
+     *  This call will result in job->priority being set, and,
+     *   if the job is in the PRIORITY state, will transition to
+     *   the SCHED state, invoke plugin callbacks, etc.
+     */
+    if (event_job_post_pack (ctx->event, job,
+                             "priority",
+                             flags,
+                             "{s:I}",
+                             "priority", priority) < 0)
+        return -1;
+
+    /*  Posting the priority event used to immediately set job->priority,
+     *   but after flux-framework/flux-core#4351, the event may be deferred.
+     *   Since the code below for sched.prioritize and queue reordering needs
+     *   the new job->priority value, set it here *also*.
+     *   See also: flux-framework/flux-core#6062
+     */
+    job->priority = priority;
+
+    /*  Update alloc queues, cancel outstanding alloc requests for
+     *   newly "held" jobs, and if in "oneshot" mode, notify scheduler
+     *   of priority change
+     */
+    if (job->alloc_queued && oneshot) {
+        alloc_queue_reorder (ctx->alloc, job);
+        if (alloc_queue_recalc_pending (ctx->alloc) < 0)
+            return -1;
+    }
+    else if (job->alloc_pending) {
+        if (job->priority == FLUX_JOB_PRIORITY_MIN) {
+            if (alloc_cancel_alloc_request (ctx->alloc, job, false) < 0)
+                return -1;
+        }
+        else if (oneshot) {
+            if (sched_prioritize_one (ctx, job) < 0)
+                return -1;
+            alloc_pending_reorder (ctx->alloc, job);
+            if (alloc_queue_recalc_pending (ctx->alloc) < 0)
+                return -1;
+        }
+    }
+    return 0;
+}
+
+int reprioritize_job (struct job_manager *ctx,
+                      struct job *job,
+                      int64_t priority)
+{
+    if (!job) {
+        errno = EINVAL;
+        return -1;
+    }
+    /*  For convenience, do not return an error if a job is not in
+     *   a "prioritizable" state (PRIORITY || SCHED). Just do nothing.
+     */
+    if (job->state != FLUX_JOB_STATE_PRIORITY
+        && job->state != FLUX_JOB_STATE_SCHED)
+        return 0;
+    return reprioritize_one (ctx, job, priority, true);
+}
+
+int reprioritize_id (struct job_manager *ctx,
+                     flux_jobid_t id,
+                     int64_t priority)
+{
+    struct job *job = zhashx_lookup (ctx->active_jobs, &id);
+    if (!job) {
+        errno = ENOENT;
+        return -1;
+    }
+    return reprioritize_job (ctx, job, priority);
+}
+
+/*  Request reprioritization of all jobs
+ */
+int reprioritize_all (struct job_manager *ctx)
+{
+    int64_t priority;
+    flux_t *h = ctx->h;
+    struct job *job = zhashx_first (ctx->active_jobs);
+    json_t *priorities = json_array ();
+
+    if (!priorities)
+        return -1;
+
+    for (job = zhashx_first (ctx->active_jobs); job;
+         job = zhashx_next (ctx->active_jobs)) {
+        /*
+         *  Only process jobs between PRIORITY and SCHED states:
+         */
+        if (job->state != FLUX_JOB_STATE_PRIORITY
+            && job->state != FLUX_JOB_STATE_SCHED)
+            continue;
+
+        /*  Call plugin to get immediate priority calculation
+         */
+        if (jobtap_get_priority (ctx->jobtap, job, &priority) < 0) {
+            flux_log_error (h, "jobtap_get_priority: %s",
+                            idf58 (job->id));
+            continue;
+        }
+
+        /*  Only do any work if job priority was set and differs
+         *   from current job priority
+         */
+        if (priority > -1 && job->priority != priority) {
+
+            /*  Re-prioritize job. This will update job->priority and
+             *   post a priority event if the priority changes
+             */
+            if (reprioritize_one (ctx, job, priority, false) < 0) {
+                flux_log_error (h, "reprioritize_one: %s",
+                                idf58 (job->id));
+                goto error;
+            }
+
+            /*  The rest of the work here is only for jobs with
+             *   outstanding alloc requests
+             */
+            if (!job->alloc_pending)
+                continue;
+
+            /*  Collect changed priorities which are > 0 in a priorities
+             *   array for use with sched.prioritize RPC.
+             *
+             *   priority == 0 or held jobs have already been handled
+             *   by the call to `reprioritize_one()` above.
+             */
+            if (job->priority > FLUX_JOB_PRIORITY_MIN) {
+                json_t *entry = json_pack ("[II]", job->id, job->priority);
+                if (!entry || json_array_append_new (priorities, entry) < 0) {
+                    json_decref (entry);
+                    flux_log (h, LOG_ERR,
+                              "reprioritize: json_pack/append failed");
+                        goto error;
+                }
+            }
+        }
+    }
+
+    /*  Reorder alloc queue and pending jobs. Canceled alloc requests
+     *   will be reinserted into the queue as the scheduler responds
+     *   to them. Note: ctx->alloc may not be initialized if this function
+     *   is called during jobtap initialization.
+     */
+    if (ctx->alloc)
+        alloc_queue_reprioritize (ctx->alloc);
+
+    /*  Update scheduler with any changed priorities */
+    if (sched_prioritize (ctx->h, priorities) < 0) {
+        flux_log_error (ctx->h,
+                        "reprioritize: sched.priority: failed for %zu jobs",
+                        json_array_size (priorities));
+        goto error;
+    }
+
+    return 0;
+error:
+    json_decref (priorities);
+    return -1;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-manager/prioritize.h b/src/modules/job-manager/prioritize.h
new file mode 100644
index 000000000000..5d7b2ef13d04
--- /dev/null
+++ b/src/modules/job-manager/prioritize.h
@@ -0,0 +1,38 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_PRIORITIZE_H
+#define _FLUX_JOB_MANAGER_PRIORITIZE_H
+
+#include <flux/core.h>
+#include "job-manager.h"
+
+/*  Request that all jobs be reprioritized. This involves calling the
+ *   job.priority.get plugin callback for all jobs, and sending the
+ *   sched.prioritize RPC to update the scheduler with any job
+ *   priorities which have changed.
+ */
+int reprioritize_all (struct job_manager *ctx);
+
+/*  Request reprioritization of a single job
+ */
+int reprioritize_job (struct job_manager *ctx,
+                      struct job *job,
+                      int64_t priority);
+
+int reprioritize_id (struct job_manager *ctx,
+                     flux_jobid_t id,
+                     int64_t priority);
+
+#endif /* ! _FLUX_JOB_MANAGER_PRIORITIZE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-manager/priority.c b/src/modules/job-manager/priority.c
deleted file mode 100644
index d2f048308ed5..000000000000
--- a/src/modules/job-manager/priority.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* priority - adjust job priority
- *
- * Purpose:
- *   Support flux job priority command for adjusting job priority
- *   after submission.  Guests can reduce their jobs' priority, or increase
- *   up to the default priority.
- *
- * Input:
- * - job id
- * - new priority
- *
- * Output:
- * - n/a
- *
- * Caveats:
- * - Need to handle case where job has already made request for resources.
- */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <flux/core.h>
-
-#include "job.h"
-#include "event.h"
-#include "alloc.h"
-#include "job-manager.h"
-
-#include "priority.h"
-
-#define MAXOF(a,b)   ((a)>(b)?(a):(b))
-
-void priority_handle_request (flux_t *h,
-                              flux_msg_handler_t *mh,
-                              const flux_msg_t *msg,
-                              void *arg)
-{
-    struct job_manager *ctx = arg;
-    struct flux_msg_cred cred;
-    flux_jobid_t id;
-    struct job *job;
-    int priority;
-    const char *errstr = NULL;
-
-    if (flux_request_unpack (msg, NULL, "{s:I s:i}",
-                                        "id", &id,
-                                        "priority", &priority) < 0
-                    || flux_msg_get_cred (msg, &cred) < 0)
-        goto error;
-    if (priority < FLUX_JOB_PRIORITY_MIN || priority > FLUX_JOB_PRIORITY_MAX) {
-        errstr = "priority value is out of range";
-        errno = EINVAL;
-        goto error;
-    }
-    if (!(job = zhashx_lookup (ctx->active_jobs, &id))) {
-        errstr = "unknown job";
-        errno = EINVAL;
-        goto error;
-    }
-    /* Security: guests can only adjust jobs that they submitted.
-     */
-    if (flux_msg_cred_authorize (cred, job->userid) < 0) {
-        errstr = "guests can only reprioritize their own jobs";
-        goto error;
-    }
-    /* Security: guests can only reduce priority, or increase up to default.
-     */
-    if (!(cred.rolemask & FLUX_ROLE_OWNER)
-            && priority > MAXOF (FLUX_JOB_PRIORITY_DEFAULT, job->priority)) {
-        errstr = "guests can only adjust priority <= default";
-        errno = EPERM;
-        goto error;
-    }
-    /* Post event, change job's queue position, and respond.
-     */
-    if (event_job_post_pack (ctx->event, job,
-                             "priority",
-                             "{ s:i s:i }",
-                             "userid", cred.userid,
-                             "priority", priority) < 0)
-        goto error;
-    alloc_queue_reorder (ctx->alloc, job);
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, errstr) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-manager/priority.h b/src/modules/job-manager/priority.h
deleted file mode 100644
index 053451539bef..000000000000
--- a/src/modules/job-manager/priority.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_JOB_MANAGER_PRIORITY_H
-#define _FLUX_JOB_MANAGER_PRIORITY_H
-
-#include <flux/core.h>
-#include "job-manager.h"
-
-/* Handle a 'priority' request - job priority adjustment
- */
-void priority_handle_request (flux_t *h,
-                              flux_msg_handler_t *mh,
-                              const flux_msg_t *msg,
-                              void *arg);
-
-
-#endif /* ! _FLUX_JOB_MANAGER_PRIORITY_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/job-manager/purge.c b/src/modules/job-manager/purge.c
new file mode 100644
index 000000000000..3c6dcccc651c
--- /dev/null
+++ b/src/modules/job-manager/purge.c
@@ -0,0 +1,629 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* purge.c - remove old inactive jobs
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <assert.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libccan/ccan/ptrint/ptrint.h"
+
+#include "job-manager.h"
+#include "job.h"
+#include "purge.h"
+#include "conf.h"
+#include "jobtap-internal.h"
+#include "restart.h"
+
+#define INACTIVE_NUM_UNLIMITED  (-1)
+#define INACTIVE_AGE_UNLIMITED  (-1.)
+
+struct purge {
+    struct job_manager *ctx;
+    double age_limit;
+    int num_limit;
+    zlistx_t *queue;
+    flux_future_t *f_sync;
+    flux_future_t *f_purge;
+
+    flux_msg_handler_t **handlers;
+    struct flux_msglist *requests;
+};
+
+static const int purge_batch_max = 100; // max KVS ops per txn
+
+/* Add an inactive job to the "purge queue".
+ * The queue is ordered by the time the job became inactive, so
+ * the first job in the queue is the oldest.
+ */
+int purge_enqueue_job (struct purge *purge, struct job *job)
+{
+    assert (job->handle == NULL);
+    if (!(job->handle = zlistx_insert (purge->queue, job, false))) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+static int purge_publish (struct purge *purge, json_t *jobs)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_event_publish_pack (purge->ctx->h,
+                                       "job-purge-inactive",
+                                       0,
+                                       "{s:O}",
+                                       "jobs", jobs)))
+        return -1;
+    flux_future_destroy (f);
+    return 0;
+}
+
+/* Return true if candidate job is eligible for purging based on
+ * provided limits.  A job is purgeable if either limit is exceeded.
+ */
+static bool purge_eligible (double age,
+                            double age_limit,
+                            int num,
+                            int num_limit)
+{
+    if (age_limit != INACTIVE_AGE_UNLIMITED && age > age_limit)
+        return true;
+    if (num_limit != INACTIVE_NUM_UNLIMITED && num > num_limit)
+        return true;
+    return false;
+}
+
+static int purge_eligible_count (struct purge *purge,
+                                 double age_limit,
+                                 int num_limit)
+{
+    double now = flux_reactor_now (flux_get_reactor (purge->ctx->h));
+    struct job *job;
+    int count = 0;
+
+    job = zlistx_first (purge->queue);
+    while (job) {
+        if (!purge_eligible (now - job->t_clean,
+                             age_limit,
+                             zlistx_size (purge->queue) - count,
+                             num_limit))
+            break;
+        count++;
+        job = zlistx_next (purge->queue);
+    }
+    return count;
+}
+
+/* handle common purging work
+ * - unlink job in kvs txn
+ * - check if max_jobid needs to be updated
+ * - add job id to jobs array for later publishing
+ * - jobtap call "job.inactive-remove"
+ * - delete job from purge->queue and purge->ctx->inactive_jobs
+ */
+static int process_job_purge (struct purge *purge,
+                              struct job *job,
+                              flux_kvs_txn_t *txn,
+                              json_t *jobs)
+{
+    char key[64];
+
+    if (flux_job_kvs_key (key, sizeof (key), job->id, NULL) < 0
+        || flux_kvs_txn_unlink (txn, 0, key) < 0)
+        return -1;
+
+    /* Update max_jobid kvs entry if we are purging it.
+     * See also flux-framework/flux-core#4300.
+     */
+    if (job->id == purge->ctx->max_jobid) {
+        if (restart_save_state_to_txn (purge->ctx, txn) < 0)
+            flux_log_error (purge->ctx->h,
+                            "Error adding job-manager state to purge transaction");
+    }
+
+    json_t *o = json_integer (job->id);
+    if (!o || json_array_append_new (jobs, o)) {
+        json_decref (o);
+        errno = ENOMEM;
+        return -1;
+    }
+
+    (void)jobtap_call (purge->ctx->jobtap,
+                       job,
+                       "job.inactive-remove",
+                       NULL);
+
+    (void)zlistx_delete (purge->queue, job->handle);
+    job->handle = NULL;
+    zhashx_delete (purge->ctx->inactive_jobs, &job->id);
+    return 0;
+}
+
+static struct job *find_purge_candidate (struct purge *purge,
+                                         flux_jobid_t id,
+                                         const char **errmsg)
+{
+    struct job *job = zlistx_first (purge->queue);
+    while (job) {
+        if (job->id == id)
+            break;
+        job = zlistx_next (purge->queue);
+    }
+    if (!job) {
+        if (!(job = zhashx_lookup (purge->ctx->active_jobs, &id))) {
+            (*errmsg) = "id not found";
+            errno = ENOENT;
+        }
+        else {
+            (*errmsg) = "cannot purge active job";
+            errno = EINVAL;
+        }
+        return NULL;
+    }
+    return job;
+}
+
+static int purge_job_id (struct purge *purge,
+                         flux_jobid_t id,
+                         flux_kvs_txn_t *txn,
+                         json_t *jobs,
+                         const char **errmsg)
+{
+    struct job *job = find_purge_candidate (purge, id, errmsg);
+    if (!job)
+        return -1;
+    if (process_job_purge (purge, job, txn, jobs) < 0)
+        return -1;
+    return 0;
+}
+
+static int purge_jobs (struct purge *purge,
+                       double age_limit,
+                       int num_limit,
+                       int max_purge_count,
+                       flux_kvs_txn_t *txn,
+                       json_t *jobs,
+                       int *countp)
+{
+    double now = flux_reactor_now (flux_get_reactor (purge->ctx->h));
+    struct job *job;
+    int count = 0;
+
+    while ((job = zlistx_first (purge->queue)) && count < max_purge_count) {
+        if (!purge_eligible (now - job->t_clean,
+                             age_limit,
+                             zlistx_size (purge->queue),
+                             num_limit))
+            break;
+        if (process_job_purge (purge, job, txn, jobs) < 0)
+            return -1;
+        count++;
+    }
+    if (count == 0) {
+        errno = ENODATA;
+        return -1;
+    }
+    (*countp) = count;
+    return 0;
+}
+
+/* Send a KVS commit containing unlinks for one or more inactive jobs.
+ * (only one if jobid specified).
+ * Return future if successful, with 'count' added to aux hash.
+ * Return NULL on failure with errno set.
+ * N.B. Failure with errno=ENODATA just means there were no eligible jobs.
+ */
+static flux_future_t *purge_inactive_jobs (struct purge *purge,
+                                           double age_limit,
+                                           int num_limit,
+                                           int max_purge_count,
+                                           flux_jobid_t id,
+                                           const char **errmsg)
+{
+    flux_kvs_txn_t *txn;
+    json_t *jobs = NULL;
+    flux_future_t *f = NULL;
+    int count = 0;
+
+    if (!(txn = flux_kvs_txn_create ()))
+        return NULL;
+    if (!(jobs = json_array ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    if (!id) {
+        if (purge_jobs (purge,
+                        age_limit,
+                        num_limit,
+                        max_purge_count,
+                        txn,
+                        jobs,
+                        &count) < 0)
+            goto error;
+    }
+    else {
+        if (purge_job_id (purge, id, txn, jobs, errmsg) < 0)
+            goto error;
+        count = 1;
+    }
+    if (!(f = flux_kvs_commit (purge->ctx->h, NULL, 0, txn))
+        || flux_future_aux_set (f, "count", int2ptr (count), NULL) < 0
+        || purge_publish (purge, jobs) < 0)
+        goto error;
+    flux_kvs_txn_destroy (txn);
+    json_decref (jobs);
+    /* N.B. if kvs commit fails, jobs are still removed from the hash/list.
+     * It doesn't seem worth the effort to structure the code to avoid this
+     * due to high complexity of solution, low probability of error, and
+     * minor consequences.
+     */
+    return f;
+error:
+    flux_kvs_txn_destroy (txn);
+    json_decref (jobs);
+    flux_future_destroy (f);
+    return NULL;
+}
+
+/* Complete periodic purge.
+ */
+static void purge_continuation (flux_future_t *f, void *arg)
+{
+    flux_t *h = flux_future_get_flux (f);
+    struct purge *purge = arg;
+    int count = ptr2int (flux_future_aux_get (f, "count"));
+
+    if (flux_rpc_get (f, NULL) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "error committing purge KVS transaction: %s",
+                  future_strerror (f, errno));
+        goto done;
+    }
+    flux_log (h, LOG_DEBUG, "purged %d inactive jobs", count);
+done:
+    if (purge->f_purge == f)
+        purge->f_purge = NULL;
+    flux_future_destroy (f);
+}
+
+/* Periodically check for inactive jobs that meet purge criteria, if
+ * criteria are configured.  If not configured, this callback is not enabled.
+ */
+static void sync_cb (flux_future_t *f_sync, void *arg)
+{
+    flux_t *h = flux_future_get_flux (f_sync);
+    struct purge *purge = arg;
+
+    if (flux_future_get (f_sync, NULL) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "purge synchronization error: %s",
+                  future_strerror (f_sync, errno));
+    }
+    if (!(purge->f_purge)) {
+        flux_future_t *f;
+        if (!(f = purge_inactive_jobs (purge,
+                                       purge->age_limit,
+                                       purge->num_limit,
+                                       purge_batch_max,
+                                       0,
+                                       NULL)) /* 0 == do not purge single job id */
+            || flux_future_then (f, -1, purge_continuation, purge) < 0) {
+            flux_future_destroy (f);
+            if (errno != ENODATA)
+                flux_log_error (h, "error creating purge KVS transaction");
+            goto done;
+        }
+        purge->f_purge = f;
+    }
+done:
+    flux_future_reset (f_sync);
+}
+
+/* Start or stop heartbeat driven sync callback after configuration change.
+ */
+int purge_sync_update (struct purge *purge)
+{
+    if (purge->age_limit != INACTIVE_AGE_UNLIMITED
+        || purge->num_limit != INACTIVE_NUM_UNLIMITED) {
+        if (!purge->f_sync) {
+            if (!(purge->f_sync = flux_sync_create (purge->ctx->h, 0.))
+                || flux_future_then (purge->f_sync, -1, sync_cb, purge) < 0) {
+                flux_future_destroy (purge->f_sync);
+                purge->f_sync = NULL;
+                return -1;
+            }
+        }
+    }
+    else {
+        if (purge->f_sync) {
+            flux_future_destroy (purge->f_sync);
+            purge->f_sync = NULL;
+        }
+    }
+    return 0;
+}
+
+/* Locate the message containing 'f'.
+ * Side effect: cursor is parked on this message, so flux_msglist_delete()
+ * can be used to delete it later.
+ */
+static const flux_msg_t *find_request (struct flux_msglist *l, flux_future_t *f)
+{
+    const flux_msg_t *msg;
+
+    msg = flux_msglist_first (l);
+    while (msg) {
+        if (flux_msg_aux_get (msg, "future") == f)
+            return msg;
+        msg = flux_msglist_next (l);
+    }
+    return NULL;
+}
+
+static void purge_request_continuation (flux_future_t *f, void *arg)
+{
+    flux_t *h = flux_future_get_flux (f);
+    struct purge *purge = arg;
+    int count = ptr2int (flux_future_aux_get (f, "count"));
+    const flux_msg_t *msg = find_request (purge->requests, f);
+
+    assert (msg != NULL);
+    if (flux_rpc_get (f, NULL) < 0) {
+        if (flux_respond_error (h, msg, errno, future_strerror (f, errno)) < 0)
+            flux_log_error (h, "error responding to purge request");
+        goto done;
+    }
+    if (flux_respond_pack (h, msg, "{s:i}", "count", count) < 0)
+        flux_log_error (h, "error responding to purge request");
+done:
+    // assumes cursor still positioned from find_request(), and destroys f
+    flux_msglist_delete (purge->requests);
+}
+
+static void purge_request_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct purge *purge = arg;
+    double age_limit;
+    int num_limit;
+    int batch;
+    int force;
+    flux_future_t *f;
+    const char *errmsg = NULL;
+    flux_error_t error;
+    int count = 0;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:f s:i s:i s:b}",
+                             "age_limit", &age_limit,
+                             "num_limit", &num_limit,
+                             "batch", &batch,
+                             "force", &force) < 0)
+        goto error;
+    if (age_limit != INACTIVE_AGE_UNLIMITED && age_limit < 0) {
+        errmsg = "if set, age limit must be >= 0";
+        errno = EINVAL;
+        goto error;
+    }
+    if (num_limit != INACTIVE_NUM_UNLIMITED && num_limit < 0) {
+        errmsg = "if set, num limit must be >= 0";
+        errno = EINVAL;
+        goto error;
+    }
+    if (batch < 1 || batch > purge_batch_max) {
+        errprintf (&error, "batch must be >= 1 and <= %d", purge_batch_max);
+        errmsg = error.text;
+        errno = EINVAL;
+        goto error;
+    }
+    if (!force) { // just return count
+        count = purge_eligible_count (purge, age_limit, num_limit);
+        goto done;
+    }
+    if (!(f = purge_inactive_jobs (purge,
+                                   age_limit,
+                                   num_limit,
+                                   batch,
+                                   0,
+                                   NULL)) /* 0 == do not purge single job id */
+        || flux_future_then (f, -1, purge_request_continuation, purge) < 0) {
+        flux_future_destroy (f);
+        if (errno == ENODATA) // ENODATA means zero jobs can be purged
+            goto done;
+        goto error;
+    }
+    if (flux_msg_aux_set (msg,
+                          "future",
+                          f,
+                          (flux_free_f)flux_future_destroy) < 0) {
+        flux_future_destroy (f);
+        goto error;
+    }
+    if (flux_msglist_append (purge->requests, msg) < 0)
+        goto error; // future destroyed with msg by dispatcher
+    return;
+done:
+    if (flux_respond_pack (h, msg, "{s:i}", "count", count) < 0)
+        flux_log_error (h, "error responding to purge request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to purge request");
+}
+
+static void purge_id_request_cb (flux_t *h,
+                                 flux_msg_handler_t *mh,
+                                 const flux_msg_t *msg,
+                                 void *arg)
+{
+    struct purge *purge = arg;
+    flux_jobid_t id;
+    int force;
+    flux_future_t *f;
+    const char *errmsg = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:i}",
+                             "id", &id,
+                             "force", &force) < 0)
+        goto error;
+
+    if (id == FLUX_JOBID_ANY) {
+        errno = EINVAL;
+        goto error;
+    }
+
+    if (!force) { // just return if job eligible to be purged
+        struct job *job = find_purge_candidate (purge, id, &errmsg);
+        if (!job)
+            goto error;
+        goto done;
+    }
+
+    if (!(f = purge_inactive_jobs (purge,
+                                   0,
+                                   0,
+                                   0,
+                                   id,
+                                   &errmsg))
+        || flux_future_then (f, -1, purge_request_continuation, purge) < 0) {
+        flux_future_destroy (f);
+        goto error;
+    }
+    if (flux_msg_aux_set (msg,
+                          "future",
+                          f,
+                          (flux_free_f)flux_future_destroy) < 0) {
+        flux_future_destroy (f);
+        goto error;
+    }
+    if (flux_msglist_append (purge->requests, msg) < 0)
+        goto error; // future destroyed with msg by dispatcher
+    return;
+done:
+    if (flux_respond_pack (h, msg, "{s:i}", "count", 1) < 0)
+        flux_log_error (h, "error responding to purge request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to purge id request");
+}
+
+static int purge_parse_config (const flux_conf_t *conf,
+                               flux_error_t *error,
+                               void *arg)
+{
+    struct purge *purge = arg;
+    flux_error_t e;
+    const char *fsd = NULL;
+    double age_limit = INACTIVE_AGE_UNLIMITED;
+    int num_limit = INACTIVE_NUM_UNLIMITED;
+
+    if (flux_conf_unpack (conf,
+                          &e,
+                          "{s?{s?s s?i}}",
+                          "job-manager",
+                            "inactive-age-limit", &fsd,
+                            "inactive-num-limit", &num_limit) < 0)
+        return errprintf (error, "job-manager.max-inactive-*: %s", e.text);
+    if (fsd) {
+        double t;
+        if (fsd_parse_duration (fsd, &t) < 0)
+            return errprintf (error,
+                              "job-manager.inactive-age-limit: invalid FSD");
+        age_limit = t;
+    }
+    if (num_limit != INACTIVE_NUM_UNLIMITED) {
+        if (num_limit < 0)
+            return errprintf (error,
+                              "job-manager.inactive-num-limit: must be >= 0");
+    }
+    purge->age_limit = age_limit;
+    purge->num_limit = num_limit;
+
+    if (purge_sync_update (purge) < 0)
+        flux_log_error (purge->ctx->h, "could not start purge sync callbacks");
+    return 1; // indicates to conf.c that callback wants updates
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST, "job-manager.purge", purge_request_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "job-manager.purge-id", purge_id_request_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void purge_destroy (struct purge *purge)
+{
+    if (purge) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (purge->handlers);
+        zlistx_destroy (&purge->queue);
+        flux_msglist_destroy (purge->requests);
+        conf_unregister_callback (purge->ctx->conf, purge_parse_config);
+        flux_future_destroy (purge->f_sync);
+        flux_future_destroy (purge->f_purge);
+        free (purge);
+        errno = saved_errno;
+    }
+}
+
+struct purge *purge_create (struct job_manager *ctx)
+{
+    struct purge *purge;
+    flux_error_t error;
+
+    if (!(purge = calloc (1, sizeof (*purge))))
+        return NULL;
+    purge->ctx = ctx;
+    purge->age_limit = INACTIVE_AGE_UNLIMITED;
+    purge->num_limit = INACTIVE_NUM_UNLIMITED;
+
+    if (!(purge->queue = zlistx_new()))
+        goto error;
+    zlistx_set_destructor (purge->queue, job_destructor);
+    zlistx_set_comparator (purge->queue, job_age_comparator);
+    zlistx_set_duplicator (purge->queue, job_duplicator);
+
+    if (conf_register_callback (ctx->conf,
+                                &error,
+                                purge_parse_config,
+                                purge) < 0) {
+        flux_log (ctx->h,
+                  LOG_ERR,
+                  "error parsing job-manager config: %s",
+                  error.text);
+        goto error;
+    }
+    if (flux_msg_handler_addvec (ctx->h, htab, purge, &purge->handlers) < 0)
+        goto error;
+    if (!(purge->requests = flux_msglist_create ()))
+        goto error;
+    return purge;
+error:
+    purge_destroy (purge);
+    return NULL;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/purge.h b/src/modules/job-manager/purge.h
new file mode 100644
index 000000000000..554326e8a891
--- /dev/null
+++ b/src/modules/job-manager/purge.h
@@ -0,0 +1,26 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_PURGE_H
+#define _FLUX_JOB_MANAGER_PURGE_H
+
+#include <stdbool.h>
+#include <flux/core.h>
+
+#include "job-manager.h"
+
+struct purge *purge_create (struct job_manager *ctx);
+void purge_destroy (struct purge *purge);
+
+int purge_enqueue_job (struct purge *purge, struct job *job);
+
+#endif /* ! _FLUX_JOB_MANAGER_PURGE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/queue.c b/src/modules/job-manager/queue.c
new file mode 100644
index 000000000000..e184beff0aa2
--- /dev/null
+++ b/src/modules/job-manager/queue.c
@@ -0,0 +1,1048 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* queue.c - job queues
+ *
+ * The job manager currently has only one actual queue in alloc.c,
+ * a vestigal design from before named queues.  Therefore, 'struct queue'
+ * below is currently a container for queue state, not for jobs as one
+ * might reasonably expect.
+ *
+ * Notes:
+ * - By default, only a single anonymous queue is defined.  If any named queues
+ *   are defined, the anonymous queue is removed.
+ *
+ * - A job requests to be in a particular queue by requiring the resource
+ *   property associated with the nodes in the queue.  If it requires nothing,
+ *   the anonymous queue is assumed.  the 'default' frobnicator plugin may be
+ *   configured to add a default queue name when one is unspecified.
+ *
+ * - When a queue is enabled, jobs submitted for that queue are accepted.
+ *   When it is disabled, the job submission program fails immediately.
+ *
+ * - When a queue is started, alloc requests for jobs in SCHED state are
+ *   presented to the scheduler.  When it is stopped, those alloc requests
+ *   are canceled.
+ *
+ * - After a queue is stopped, the job manager continues to send free
+ *   requests to the scheduler for the queue as resources are released.
+ *   Jobs/housekeeping are not canceled when a queue is stopped.
+ *
+ * - When a queue is enabled and stopped, job submissions to the queue are
+ *   accepted, but the jobs will not run until the queue is started.
+ *
+ * See also:
+ * RFC 33/Flux Job Queues
+ * RFC 27/Resource Allocation Protocol Version 1
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/jpath.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
+#include "alloc.h"
+#include "job-manager.h"
+#include "jobtap-internal.h"
+#include "jobtap.h"
+#include "conf.h"
+#include "restart.h"
+#include "queue.h"
+
+struct queue {
+    char *name;
+    bool is_enabled;    // jobs may be submitted to this queue
+    char *disable_reason;   // reason if disabled
+    bool is_started;    // current queue state
+    bool is_started_sticky; // tracks is_started unless --nocheckpoint
+    char *stop_reason; // reason if stopped (optionally set)
+    json_t *requires;  // required properties array
+};
+
+struct queue_ctx {
+    struct job_manager *ctx;
+    flux_msg_handler_t **handlers;
+    union {
+        struct queue *anon;
+        zhashx_t *named;
+    };
+    bool have_named_queues;
+};
+
+static void queue_destroy (struct queue *q)
+{
+    if (q) {
+        int saved_errno = errno;
+        json_decref (q->requires);
+        free (q->name);
+        free (q->disable_reason);
+        free (q);
+        errno = saved_errno;
+    }
+}
+
+// zhashx_destructor_fn signature
+static void queue_destructor (void **item)
+{
+    if (item) {
+        queue_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static struct queue *queue_create (const char *name, json_t *config)
+{
+    struct queue *q;
+
+    if (!(q = calloc (1, sizeof (*q))))
+        return NULL;
+    if (name && !(q->name = strdup (name)))
+        goto error;
+    q->is_enabled = true;
+
+    if (config && json_unpack (config, "{s?O}", "requires", &q->requires) < 0)
+        goto error;
+
+    /* The anonymous queue begins life started, while named queues do not.
+     */
+    if (name)
+        q->is_started_sticky = q->is_started = false;
+    else
+        q->is_started_sticky = q->is_started = true;
+    return q;
+error:
+    queue_destroy (q);
+    return NULL;
+}
+
+static struct queue *queue_first (struct queue_ctx *qctx)
+{
+    if (qctx->have_named_queues)
+        return zhashx_first (qctx->named);
+    return qctx->anon;
+}
+
+static struct queue *queue_next (struct queue_ctx *qctx)
+{
+    if (qctx->have_named_queues)
+        return zhashx_next (qctx->named);
+    return NULL;
+}
+
+static int queue_enable (struct queue *q)
+{
+    q->is_enabled = true;
+    free (q->disable_reason);
+    q->disable_reason = NULL;
+    return 0;
+}
+
+static int queue_disable (struct queue *q, const char *reason)
+{
+    char *cpy;
+    if (!(cpy = strdup (reason)))
+        return -1;
+    free (q->disable_reason);
+    q->disable_reason = cpy;
+    q->is_enabled = false;
+    return 0;
+}
+
+static int queue_start (struct queue *q, bool nocheckpoint)
+{
+    q->is_started = true;
+    if (!nocheckpoint)
+        q->is_started_sticky = q->is_started;
+    free (q->stop_reason);
+    q->stop_reason = NULL;
+    return 0;
+}
+static int queue_stop (struct queue *q, const char *reason, bool nocheckpoint)
+{
+    char *cpy = NULL;
+    if (reason) {
+        if (!(cpy = strdup (reason)))
+            return -1;
+    }
+    free (q->stop_reason);
+    q->stop_reason = cpy;
+    q->is_started = false;
+    if (!nocheckpoint)
+        q->is_started_sticky = q->is_started;
+    return 0;
+}
+
+static int queue_enable_all (struct queue_ctx *qctx)
+{
+    struct queue *q = queue_first (qctx);
+    while (q) {
+        if (queue_enable (q) < 0)
+            return -1;
+        q = queue_next (qctx);
+    }
+    return 0;
+}
+
+static int queue_disable_all (struct queue_ctx *qctx, const char *reason)
+{
+    struct queue *q = queue_first (qctx);
+    while (q) {
+        if (queue_disable (q, reason) < 0)
+            return -1;
+        q = queue_next (qctx);
+    }
+    return 0;
+}
+
+static int queue_start_all (struct queue_ctx *qctx, bool nocheckpoint)
+{
+    struct queue *q = queue_first (qctx);
+    while (q) {
+        if (queue_start (q, nocheckpoint) < 0)
+            return -1;
+        q = queue_next (qctx);
+    }
+    return 0;
+}
+
+static int queue_stop_all (struct queue_ctx *qctx,
+                           const char *reason,
+                           bool nocheckpoint)
+{
+    struct queue *q = queue_first (qctx);
+    while (q) {
+        if (queue_stop (q, reason, nocheckpoint) < 0)
+            return -1;
+        q = queue_next (qctx);
+    }
+    return 0;
+}
+
+struct queue *queue_lookup (struct queue_ctx *qctx,
+                            const char *name,
+                            flux_error_t *error)
+{
+    if (name) {
+        struct queue *q;
+
+        if (!qctx->have_named_queues
+            || !(q = zhashx_lookup (qctx->named, name))) {
+            errprintf (error, "'%s' is not a valid queue", name);
+            return NULL;
+        }
+        return q;
+    }
+    else {
+        if (qctx->have_named_queues) {
+            errprintf (error, "a named queue is required");
+            return NULL;
+        }
+        return qctx->anon;
+    }
+}
+
+static int set_string (json_t *o, const char *key, const char *val)
+{
+    json_t *s = json_string (val);
+    if (!s || json_object_set_new (o, key, s) < 0) {
+        json_decref (s);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+static int queue_ctx_save_one (json_t *a, struct queue *q)
+{
+    json_t *entry;
+
+    if (!(entry = json_pack ("{s:b s:b}",
+                             "enable", q->is_enabled,
+                             "start", q->is_started_sticky)))
+        goto nomem;
+    if (q->name) {
+        if (set_string (entry, "name", q->name) < 0)
+            goto error;
+    }
+    if (!entry)
+        goto nomem;
+    if (!q->is_enabled) {
+        if (set_string (entry, "disable_reason", q->disable_reason) < 0)
+            goto error;
+    }
+    if (!q->is_started_sticky && q->stop_reason) {
+        if (set_string (entry, "stop_reason", q->stop_reason) < 0)
+            goto error;
+    }
+    if (json_array_append_new (a, entry) < 0)
+        goto nomem;
+    return 0;
+nomem:
+    errno = ENOMEM;
+error:
+    ERRNO_SAFE_WRAP (json_decref, entry);
+    return -1;
+}
+
+json_t *queue_ctx_save (struct queue_ctx *qctx)
+{
+    json_t *a;
+    struct queue *q;
+
+    if (!(a = json_array ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    q = queue_first (qctx);
+    while (q) {
+        if (queue_ctx_save_one (a, q) < 0)
+            goto error;
+        q = queue_next (qctx);
+    }
+    return a;
+error:
+    ERRNO_SAFE_WRAP (json_decref, a);
+    return NULL;
+}
+
+static int restore_state_v0 (struct queue_ctx *qctx, json_t *entry)
+{
+    const char *name = NULL;
+    const char *reason = NULL;
+    const char *disable_reason = NULL;
+    int enable;
+    struct queue *q = NULL;
+
+    if (json_unpack (entry,
+                     "{s?s s:b s?s s?s}",
+                     "name", &name,
+                     "enable", &enable,
+                     "reason", &reason,
+                     "disable_reason", &disable_reason) < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    /* "reason" is backwards compatible field name for "disable_reason" */
+    if (!disable_reason && reason)
+        disable_reason = reason;
+
+    if ((q = queue_lookup (qctx, name, NULL))) {
+        if (enable) {
+            if (queue_enable (q) < 0)
+                return -1;
+        }
+        else {
+            if (queue_disable (q, disable_reason) < 0)
+                return -1;
+        }
+    }
+    return 0;
+}
+
+static int restore_state_v1 (struct queue_ctx *qctx, json_t *entry)
+{
+    const char *name = NULL;
+    const char *disable_reason = NULL;
+    const char *stop_reason = NULL;
+    int enable;
+    int start;
+    struct queue *q = NULL;
+
+    if (json_unpack (entry,
+                     "{s?s s:b s?s s:b s?s}",
+                     "name", &name,
+                     "enable", &enable,
+                     "disable_reason", &disable_reason,
+                     "start", &start,
+                     "stop_reason", &stop_reason) < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (name && qctx->have_named_queues)
+        q = zhashx_lookup (qctx->named, name);
+    else if (!name && !qctx->have_named_queues)
+        q = qctx->anon;
+    if (q) {
+        if (enable) {
+            if (queue_enable (q) < 0)
+                return -1;
+        }
+        else {
+            if (queue_disable (q, disable_reason) < 0)
+                return -1;
+        }
+        if (start) {
+            if (queue_start (q, false) < 0)
+                return -1;
+        }
+        else {
+            if (queue_stop (q, stop_reason, false) < 0)
+                return -1;
+        }
+    }
+    return 0;
+}
+
+int queue_ctx_restore (struct queue_ctx *qctx, int version, json_t *o)
+{
+    size_t index;
+    json_t *entry;
+
+    if ((version != 0 && version != 1)
+        || !o
+        || !json_is_array (o)) {
+        errno = EINVAL;
+        return -1;
+    }
+    json_array_foreach (o, index, entry) {
+        if (version == 0) {
+            if (restore_state_v0 (qctx, entry) < 0)
+                return -1;
+        }
+        else { /* version == 1 */
+            if (restore_state_v1 (qctx, entry) < 0)
+                return -1;
+        }
+    }
+    return 0;
+}
+
+int queue_submit_check (struct queue_ctx *qctx,
+                        json_t *jobspec,
+                        flux_error_t *error)
+{
+    struct queue *q;
+    json_t *o;
+    const char *name = NULL;
+
+    if ((o = jpath_get (jobspec, "attributes.system.queue")))
+        name = json_string_value (o);
+
+    if (!(q = queue_lookup (qctx, name, error))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!q->is_enabled) {
+        errprintf (error, "job submission%s%s is disabled: %s",
+                   name ? " to " : "",
+                   name ? name : "",
+                   q->disable_reason);
+        errno = EINVAL;
+        return -1;
+    }
+    return 0;
+}
+
+bool queue_started (struct queue_ctx *qctx, struct job *job)
+{
+    if (qctx->have_named_queues) {
+        struct queue *q;
+        if (!job->queue)
+            return false;
+        if (!(q = zhashx_lookup (qctx->named, job->queue))) {
+            flux_log (qctx->ctx->h, LOG_ERR,
+                      "%s: job %s invalid queue: %s",
+                      __FUNCTION__, idf58 (job->id), job->queue);
+            return false;
+        }
+        return q->is_started;
+    }
+
+    return qctx->anon->is_started;
+}
+
+/* N.B. the basic queue configuration should have already been validated by
+ * policy_validate() so we shouldn't need to produce detailed configuration
+ * errors for users here.
+ */
+static int queue_configure (const flux_conf_t *conf,
+                            flux_error_t *error,
+                            void *arg)
+{
+    struct queue_ctx *qctx = arg;
+    json_t *queues;
+
+    if (flux_conf_unpack (conf, NULL, "{s:o}", "queues", &queues) == 0
+        && json_object_size (queues) > 0) {
+        const char *name;
+        json_t *value;
+        struct queue *q;
+        zlistx_t *keys;
+
+        /* destroy anon queue and create hash if necessary
+         */
+        if (!qctx->have_named_queues) {
+            qctx->have_named_queues = true;
+            queue_destroy (qctx->anon);
+            if (!(qctx->named = zhashx_new ()))
+                goto nomem;
+            zhashx_set_destructor (qctx->named, queue_destructor);
+        }
+        /* remove any queues that disappeared from config
+         */
+        if (!(keys = zhashx_keys (qctx->named)))
+            goto nomem;
+        name = zlistx_first (keys);
+        while (name) {
+            if (!json_object_get (queues, name))
+                zhashx_delete (qctx->named, name);
+            name = zlistx_next (keys);
+        }
+        zlistx_destroy (&keys);
+        /* add any new queues that appeared in config.  Note that
+         * named queues default to being enabled/stopped.  On initial
+         * module load, job-manager may change that state based on
+         * prior checkpointed information.
+         */
+        json_object_foreach (queues, name, value) {
+            if (!zhashx_lookup (qctx->named, name)) {
+                if (!(q = queue_create (name, value)))
+                    goto nomem;
+                (void)zhashx_insert (qctx->named, name, q);
+            }
+        }
+    }
+    else {
+        if (qctx->have_named_queues) {
+            qctx->have_named_queues = false;
+            zhashx_destroy (&qctx->named);
+            if (!(qctx->anon = queue_create (NULL, NULL)))
+                goto nomem;
+        }
+    }
+    return 1;
+nomem:
+    errprintf (error, "out of memory while processing queue configuration");
+    errno = ENOMEM;
+    return -1;
+}
+
+static void queue_list_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct queue_ctx *qctx = arg;
+    struct queue *q;
+    json_t *a = NULL;;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (!(a = json_array ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    if (qctx->have_named_queues) {
+        q = zhashx_first (qctx->named);
+        while (q) {
+            json_t *o;
+            if (!(o = json_string (q->name))
+                || json_array_append_new (a, o) < 0) {
+                json_decref (o);
+                errno = ENOMEM;
+                goto error;
+            }
+            q = zhashx_next (qctx->named);
+        }
+    }
+    if (flux_respond_pack (h, msg, "{s:O}", "queues", a) < 0)
+        flux_log_error (h, "error responding to job-manager.queue-list");
+    json_decref (a);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to job-manager.queue-list");
+    json_decref (a);
+}
+
+static void queue_status_cb (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
+{
+    struct queue_ctx *qctx = arg;
+    flux_error_t error;
+    const char *errmsg = NULL;
+    const char *name = NULL;
+    struct queue *q;
+    json_t *o = NULL;
+    bool start;
+    const char *stop_reason = NULL;
+
+    if (flux_request_unpack (msg, NULL, "{s?s}", "name", &name) < 0)
+        goto error;
+    if (!(q = queue_lookup (qctx, name, &error))) {
+        errmsg = error.text;
+        errno = EINVAL;
+        goto error;
+    }
+    /* If the scheduler is not loaded the queue is considered stopped
+     * with special reason "Scheduler is offline".
+     */
+    if (!alloc_sched_ready (qctx->ctx->alloc)) {
+        start = false;
+        stop_reason = "Scheduler is offline";
+    }
+    else {
+        start = q->is_started;
+        stop_reason = q->stop_reason;
+    }
+    if (!(o = json_pack ("{s:b s:b}",
+                         "enable", q->is_enabled,
+                         "start", start))) {
+        errno = ENOMEM;
+        goto error;
+    }
+    if (!q->is_enabled) {
+        if (set_string (o, "disable_reason", q->disable_reason) < 0)
+            goto error;
+    }
+    if (!start && stop_reason) {
+        if (set_string (o, "stop_reason", stop_reason) < 0)
+            goto error;
+    }
+    if (flux_respond_pack (h, msg, "O", o) < 0)
+        flux_log_error (h, "error responding to job-manager.queue-status");
+    json_decref (o);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to job-manager.queue-status");
+    json_decref (o);
+}
+
+static void queue_enable_cb (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
+{
+    struct queue_ctx *qctx = arg;
+    flux_error_t error;
+    const char *errmsg = NULL;
+    const char *name = NULL;
+    int enable;
+    const char *disable_reason = NULL;
+    int all;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s?s s:b s?s s:b}",
+                             "name", &name,
+                             "enable", &enable,
+                             "reason", &disable_reason,
+                             "all", &all) < 0)
+        goto error;
+    if (!enable && !disable_reason) {
+        errmsg = "reason is required for disable";
+        errno = EINVAL;
+        goto error;
+    }
+    if (!name) {
+        if (qctx->have_named_queues && !all) {
+            errmsg = "Use --all to apply this command to all queues";
+            errno = EINVAL;
+            goto error;
+        }
+        if (enable) {
+            if (queue_enable_all (qctx))
+                goto error;
+        }
+        else {
+            if (queue_disable_all (qctx, disable_reason))
+                goto error;
+        }
+    }
+    else {
+        struct queue *q;
+        if (!(q = queue_lookup (qctx, name, &error))) {
+            errmsg = error.text;
+            errno = EINVAL;
+            goto error;
+        }
+        if (enable) {
+            if (queue_enable (q) < 0)
+                goto error;
+        }
+        else {
+            if (queue_disable (q, disable_reason) < 0)
+                goto error;
+        }
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to job-manager.queue-enable");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to job-manager.queue-enable");
+}
+
+static int enqueue_jobs (struct queue_ctx *qctx, const char *name)
+{
+    struct job *job = zhashx_first (qctx->ctx->active_jobs);
+    while (job) {
+        if (!name || (job->queue && streq (job->queue, name))) {
+            if (!job->alloc_queued
+                && !job->alloc_pending
+                && job->state == FLUX_JOB_STATE_SCHED) {
+                if (alloc_enqueue_alloc_request (qctx->ctx->alloc, job) < 0)
+                    return -1;
+                if (alloc_queue_recalc_pending (qctx->ctx->alloc) < 0)
+                    return -1;
+            }
+        }
+        job = zhashx_next (qctx->ctx->active_jobs);
+    }
+    return 0;
+}
+
+static void dequeue_jobs (struct queue_ctx *qctx, const char *name)
+{
+    if (alloc_queue_count (qctx->ctx->alloc) > 0
+        || alloc_pending_count (qctx->ctx->alloc) > 0) {
+        struct job *job = zhashx_first (qctx->ctx->active_jobs);
+        while (job) {
+            if (!name || (job->queue && streq (job->queue, name))) {
+                if (job->alloc_queued)
+                    alloc_dequeue_alloc_request (qctx->ctx->alloc, job);
+                else if (job->alloc_pending)
+                    alloc_cancel_alloc_request (qctx->ctx->alloc, job, false);
+            }
+            job = zhashx_next (qctx->ctx->active_jobs);
+        }
+    }
+}
+
+static void queue_start_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
+{
+    struct queue_ctx *qctx = arg;
+    flux_error_t error;
+    const char *errmsg = NULL;
+    const char *name = NULL;
+    int start;
+    const char *stop_reason = NULL;
+    int all;
+    int nocheckpoint = 0;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s?s s:b s?s s:b s?b}",
+                             "name", &name,
+                             "start", &start,
+                             "reason", &stop_reason,
+                             "all", &all,
+                             "nocheckpoint", &nocheckpoint) < 0)
+        goto error;
+    if (!name) {
+        if (qctx->have_named_queues && !all) {
+            errmsg = "Use --all to apply this command to all queues";
+            errno = EINVAL;
+            goto error;
+        }
+        if (start) {
+            if (queue_start_all (qctx, nocheckpoint))
+                goto error;
+            if (enqueue_jobs (qctx, NULL) < 0)
+                goto error;
+        }
+        else {
+            if (queue_stop_all (qctx, stop_reason, nocheckpoint))
+                goto error;
+            dequeue_jobs (qctx, NULL);
+        }
+    }
+    else {
+        struct queue *q;
+        if (!(q = queue_lookup (qctx, name, &error))) {
+            errmsg = error.text;
+            errno = EINVAL;
+            goto error;
+        }
+        if (start) {
+            if (queue_start (q, nocheckpoint) < 0)
+                goto error;
+            if (enqueue_jobs (qctx, name) < 0)
+                goto error;
+        }
+        else {
+            if (queue_stop (q, stop_reason, nocheckpoint) < 0)
+                goto error;
+            dequeue_jobs (qctx, name);
+        }
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to job-manager.queue-start");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to job-manager.queue-start");
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.queue-list",
+        queue_list_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.queue-status",
+        queue_status_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.queue-enable",
+        queue_enable_cb,
+        0,
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.queue-start",
+        queue_start_cb,
+        0,
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void queue_ctx_destroy (struct queue_ctx *qctx)
+{
+    if (qctx) {
+        int saved_errno = errno;
+        conf_unregister_callback (qctx->ctx->conf, queue_configure);
+        flux_msg_handler_delvec (qctx->handlers);
+        if (qctx->have_named_queues)
+            zhashx_destroy (&qctx->named);
+        else
+            queue_destroy (qctx->anon);
+        free (qctx);
+        errno = saved_errno;
+    }
+}
+
+/*  Test equality of two constraint objects.
+ *  For now, two constraints are equivalent if:
+ *
+ *  - both are either NULL or empty objects (i.e. size == 0)
+ *    (Note: json_object_size (NULL) == 0)
+ *
+ *  - json_equal(a, b) returns true
+ */
+static bool constraints_equal (json_t *c1, json_t *c2)
+{
+    if ((json_object_size (c1) == 0 && json_object_size (c2) == 0)
+        || json_equal (c1, c2))
+        return true;
+    return false;
+}
+
+static int constraints_match_check (struct queue_ctx *qctx,
+                                    const char *name,
+                                    json_t *constraints,
+                                    flux_error_t *errp)
+{
+    int rc = -1;
+    json_t *expected = NULL;
+    struct queue *q;
+
+    /*  Return an error if the job's current queue doesn't exist since we
+     *  can't validate current constraints (This should not happen in normal
+     *  situations).
+     */
+    if (!(q = queue_lookup (qctx, name, errp)))
+        return -1;
+
+    /*  If current queue has constraints, then create a constraint object
+     *  for equivalence test below:
+     */
+    if (q->requires
+        && !(expected = json_pack ("{s:O}", "properties", q->requires))) {
+        errprintf (errp, "failed to get constraints for current queue");
+        goto out;
+    }
+
+    /*  Constraints of current job and queue must match exactly or queue
+     *  update will be rejected. This is because the entire constraints
+     *  object will be overwritten on queue update, and we do not want to
+     *  replace any extra constraints provided on the submission commandline
+     *  (and these likely wouldn't make sense in the new queue anyway)
+     */
+    if (!constraints_equal (constraints, expected)) {
+        errprintf (errp,
+                   "job appears to have non-queue constraints, "
+                   "unable to update queue to %s",
+                   name);
+        goto out;
+    }
+    rc = 0;
+out:
+    json_decref (expected);
+    return rc;
+}
+
+static int queue_update_cb (flux_plugin_t *p,
+                            const char *topic,
+                            flux_plugin_arg_t *args,
+                            void *arg)
+{
+    int rc;
+    struct queue_ctx *qctx = arg;
+    flux_job_state_t state;
+    const char *name;
+    const char *current_queue = NULL;
+    json_t *constraints = NULL;
+    flux_error_t error;
+    struct queue *newq;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s s:i s:{s:{s:{s?s s?o}}}}",
+                                "value", &name,
+                                "state", &state,
+                                "jobspec",
+                                 "attributes",
+                                  "system",
+                                   "queue", &current_queue,
+                                   "constraints", &constraints) < 0) {
+        flux_jobtap_error (p, args, "plugin args unpack failed");
+        return -1;
+    }
+    if (state == FLUX_JOB_STATE_RUN
+        || state == FLUX_JOB_STATE_CLEANUP) {
+        flux_jobtap_error (p,
+                           args,
+                           "update of queue for running job not supported");
+        return -1;
+    }
+    if (current_queue && streq (current_queue, name)) {
+        flux_jobtap_error (p,
+                           args,
+                           "job queue is already set to %s",
+                           name);
+        return -1;
+    }
+    if (!(newq = queue_lookup (qctx, name, &error))) {
+        flux_jobtap_error (p, args, "%s", error.text);
+        return -1;
+    }
+    if (!newq->is_enabled) {
+        flux_jobtap_error (p,
+                           args,
+                           "queue %s is currently disabled",
+                           name);
+        return -1;
+    }
+    /*  Constraints must match current queue exactly since they will be
+     *  overwritten with new queue constraints after queue is updated:
+     */
+    if (constraints_match_check (qctx, current_queue, constraints, &error)) {
+        flux_jobtap_error (p, args, "%s", error.text);
+        return -1;
+    }
+    /*  Request the update service do a feasibility check for this update
+     *  and append an additional update of the job constraints.
+     *
+     *  This is done via two different calls below dependent on whether the
+     *  new queue has any constraints.
+     */
+    if (newq->requires) {
+        /*  Replace current constraints with those of the new queue
+         */
+        rc = flux_plugin_arg_pack (args,
+                                   FLUX_PLUGIN_ARG_OUT,
+                                   "{s:i s:{s:{s:O}}}",
+                                   "feasibility", 1,
+                                   "updates",
+                                    "attributes.system.constraints",
+                                     "properties", newq->requires);
+    }
+    else {
+        /*  New queue has no requirements. Set constraints to empty object.
+         */
+        rc = flux_plugin_arg_pack (args,
+                                   FLUX_PLUGIN_ARG_OUT,
+                                   "{s:i s:{s:{}}}",
+                                   "feasibility", 1,
+                                   "updates",
+                                    "attributes.system.constraints");
+    }
+    /*  If either of the above packs failed then return an error:
+     */
+    if (rc < 0) {
+        flux_jobtap_error (p,
+                           args,
+                           "unable to create jobtap out arguments");
+        return -1;
+    }
+    return 0;
+}
+
+static int update_queue_plugin_init (flux_plugin_t *p, void *arg)
+{
+    return flux_plugin_add_handler (p,
+                                    "job.update.attributes.system.queue",
+                                    queue_update_cb,
+                                    arg);
+}
+
+struct queue_ctx *queue_ctx_create (struct job_manager *ctx)
+{
+    struct queue_ctx *qctx;
+    flux_error_t error;
+
+    if (!(qctx = calloc (1, sizeof (*qctx))))
+        return NULL;
+    qctx->ctx = ctx;
+    if (!(qctx->anon = queue_create (NULL, NULL)))
+        goto error;
+    if (flux_msg_handler_addvec (ctx->h,
+                                 htab,
+                                 qctx,
+                                 &qctx->handlers) < 0)
+        goto error;
+    if (conf_register_callback (ctx->conf,
+                                &error,
+                                queue_configure,
+                                qctx) < 0) {
+        flux_log (ctx->h,
+                  LOG_ERR,
+                  "error parsing queue config: %s",
+                  error.text);
+        goto error;
+    }
+    if (jobtap_register_builtin (ctx->jobtap,
+                                 ".update-queue",
+                                 update_queue_plugin_init,
+                                 qctx) < 0
+        || !jobtap_load (ctx->jobtap, ".update-queue", NULL, NULL)) {
+        flux_log (ctx->h,
+                  LOG_ERR,
+                  "Failed to register and load update-queue plugin");
+        goto error;
+    }
+    return qctx;
+error:
+    queue_ctx_destroy (qctx);
+    return NULL;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/queue.h b/src/modules/job-manager/queue.h
new file mode 100644
index 000000000000..17e1aba9a3f7
--- /dev/null
+++ b/src/modules/job-manager/queue.h
@@ -0,0 +1,32 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_QUEUE_H
+#define _FLUX_JOB_MANAGER_QUEUE_H
+
+#include <stdbool.h>
+
+#include "job-manager.h"
+
+struct queue_ctx *queue_ctx_create (struct job_manager *ctx);
+void queue_ctx_destroy (struct queue_ctx *qctx);
+
+json_t *queue_ctx_save (struct queue_ctx *qctx);
+int queue_ctx_restore (struct queue_ctx *qctx, int version, json_t *o);
+
+int queue_submit_check (struct queue_ctx *qctx,
+                        json_t *jobspec,
+                        flux_error_t *error);
+
+bool queue_started (struct queue_ctx *qctx, struct job *job);
+
+#endif /* ! _FLUX_JOB_MANAGER_QUEUE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/job-manager/raise.c b/src/modules/job-manager/raise.c
index 66342fc4f505..8ed86eb66dea 100644
--- a/src/modules/job-manager/raise.c
+++ b/src/modules/job-manager/raise.c
@@ -28,9 +28,14 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <unistd.h>
 #include <ctype.h>
 #include <flux/core.h>
 
+#include "src/common/libjob/idf58.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/errno_safe.h"
+
 #include "job.h"
 #include "event.h"
 #include "raise.h"
@@ -60,30 +65,39 @@ int raise_check_severity (int severity)
     return 0;
 }
 
-/* NB: job object may be destroyed in event_job_post_pack().
- * Do not reference the object after calling this function.
- * Do not call this function and continue to iterate on the job hash
- * with zhash_next().
- */
-int raise_job (struct job_manager *ctx,
-               const char *type,
-               int severity,
-               uint32_t userid,
-               const char *note,
-               struct job *job)
+int raise_job_exception (struct job_manager *ctx,
+                         struct job *job,
+                         const char *type,
+                         int severity,
+                         uint32_t userid,
+                         const char *note)
 {
     flux_jobid_t id = job->id;
     flux_future_t *f;
+    json_t *evctx;
 
-    if (event_job_post_pack (ctx->event,
-                             job,
-                             "exception",
-                             "{ s:s s:i s:i s:s }",
+    // if no note specified, set to empty string per RFC21
+    if (!note)
+        note = "";
+    // create exception event context with the required keys per RFC 21
+    if (!(evctx = json_pack ("{s:s s:i s:s}",
                              "type", type,
                              "severity", severity,
-                             "userid", userid,
-                             "note", note ? note : "") < 0)
-        return -1;
+                             "note", note)))
+        goto nomem;
+    // add optional userid key
+    if (userid != FLUX_USERID_UNKNOWN) {
+        json_t *val;
+        if (!(val = json_integer (userid))
+            || json_object_set_new (evctx, "userid", val) < 0) {
+            json_decref (val);
+            goto nomem;
+        }
+    }
+    // post exception to job eventlog
+    if (event_job_post_pack (ctx->event, job, "exception", 0, "O", evctx) < 0)
+        goto error;
+    // publish job-exception event
     if (!(f = flux_event_publish_pack (ctx->h,
                                        "job-exception",
                                        FLUX_MSGFLAG_PRIVATE,
@@ -91,9 +105,15 @@ int raise_job (struct job_manager *ctx,
                                        "id", id,
                                        "type", type,
                                        "severity", severity)))
-        return -1;
+        goto error;
     flux_future_destroy (f);
+    json_decref (evctx);
     return 0;
+nomem:
+    errno = ENOMEM;
+error:
+    ERRNO_SAFE_WRAP (json_decref, evctx);
+    return -1;
 }
 
 void raise_handle_request (flux_t *h,
@@ -110,12 +130,14 @@ void raise_handle_request (flux_t *h,
     const char *note = NULL;
     const char *errstr = NULL;
 
-    if (flux_request_unpack (msg, NULL, "{s:I s:i s:s s?:s}",
-                                        "id", &id,
-                                        "severity", &severity,
-                                        "type", &type,
-                                        "note", &note) < 0
-                    || flux_msg_get_cred (msg, &cred) < 0)
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:i s:s s?s}",
+                             "id", &id,
+                             "severity", &severity,
+                             "type", &type,
+                             "note", &note) < 0
+        || flux_msg_get_cred (msg, &cred) < 0)
         goto error;
     if (raise_check_severity (severity)) {
         errstr = "invalid exception severity";
@@ -128,15 +150,18 @@ void raise_handle_request (flux_t *h,
         goto error;
     }
     if (!(job = zhashx_lookup (ctx->active_jobs, &id))) {
-        errstr = "unknown job id";
-        errno = EINVAL;
+        if (!(job = zhashx_lookup (ctx->inactive_jobs, &id)))
+            errstr = "unknown job id";
+        else
+            errstr = "job is inactive";
+        errno = ENOENT;
         goto error;
     }
     if (flux_msg_cred_authorize (cred, job->userid) < 0) {
         errstr = "guests can only raise exceptions on their own jobs";
         goto error;
     }
-    if (raise_job (ctx, type, severity, cred.userid, note, job) < 0)
+    if (raise_job_exception (ctx, job, type, severity, cred.userid, note) < 0)
         goto error;
     /* NB: job object may be destroyed in event_job_post_pack().
      * Do not reference the object after this point:
@@ -208,19 +233,13 @@ void raiseall_handle_request (flux_t *h,
 
     if (flux_request_unpack (msg,
                              NULL,
-                             "{s:b s:i s:i s:i s:s s?:s}",
-                             "dry_run",
-                             &dry_run,
-                             "userid",
-                             &userid,
-                             "states",
-                             &state_mask,
-                             "severity",
-                             &severity,
-                             "type",
-                             &type,
-                             "note",
-                             &note) < 0)
+                             "{s:b s:i s:i s:i s:s s?s}",
+                             "dry_run", &dry_run,
+                             "userid", &userid,
+                             "states", &state_mask,
+                             "severity", &severity,
+                             "type", &type,
+                             "note", &note) < 0)
         goto error;
     if (flux_msg_get_cred (msg, &cred) < 0)
         goto error;
@@ -246,10 +265,15 @@ void raiseall_handle_request (flux_t *h,
     if (!dry_run) {
         job = zlistx_first (target_jobs);
         while (job) {
-            if (raise_job (ctx, type, severity, cred.userid, note, job) < 0) {
+            if (raise_job_exception (ctx,
+                                     job,
+                                     type,
+                                     severity,
+                                     cred.userid,
+                                     note) < 0) {
                 flux_log_error (h,
-                                "error raising exception on id=%ju",
-                                (uintmax_t)job->id);
+                                "error raising exception on id=%s",
+                                idf58 (job->id));
                 error_count++;
             }
             job = zlistx_next (target_jobs);
@@ -258,10 +282,8 @@ void raiseall_handle_request (flux_t *h,
     if (flux_respond_pack (h,
                            msg,
                            "{s:i s:i}",
-                           "count",
-                           zlistx_size (target_jobs),
-                           "errors",
-                           error_count) < 0)
+                           "count", zlistx_size (target_jobs),
+                           "errors", error_count) < 0)
         flux_log_error (h, "%s: flux_respond", __FUNCTION__);
     zlistx_destroy (&target_jobs);
     return;
diff --git a/src/modules/job-manager/raise.h b/src/modules/job-manager/raise.h
index 392916a8780d..84ef9af72dbd 100644
--- a/src/modules/job-manager/raise.h
+++ b/src/modules/job-manager/raise.h
@@ -22,6 +22,22 @@ void raise_ctx_destroy (struct raise *raise);
 int raise_check_type (const char *type);
 int raise_check_severity (int severity);
 
+/* Raise a job exception: post to job eventlog and publish job-exception
+ * message.
+ *
+ * N.B. job object may be destroyed in event_job_post_pack().
+ * Do not reference the object after calling this function.
+ * Do not call this function and continue to iterate on the job hash
+ * with zhash_next().
+ */
+int raise_job_exception (struct job_manager *ctx,
+                         struct job *job,
+                         const char *type,
+                         int severity,
+                         uint32_t userid,   // skip if FLUX_USERID_NONE
+                         const char *note); // skip if NULL
+
+
 #endif /* ! _FLUX_JOB_MANAGER_RAISE_H */
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/modules/job-manager/restart.c b/src/modules/job-manager/restart.c
index 579e879ca82a..049bdf47319a 100644
--- a/src/modules/job-manager/restart.c
+++ b/src/modules/job-manager/restart.c
@@ -14,21 +14,28 @@
 #include "config.h"
 #endif
 #include <stdlib.h>
-#include <argz.h>
-#include <envz.h>
 #include <flux/core.h>
 
+#include "src/common/libjob/idf58.h"
 #include "src/common/libutil/fluid.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "job.h"
 #include "restart.h"
 #include "event.h"
 #include "wait.h"
+#include "queue.h"
+#include "jobtap-internal.h"
 
 /* restart_map callback should return -1 on error to stop map with error,
  * or 0 on success.  'job' is only valid for the duration of the callback.
  */
-typedef int (*restart_map_f)(struct job *job, void *arg);
+typedef int (*restart_map_f)(struct job *job, void *arg, flux_error_t *error);
+
+const char *checkpoint_key = "checkpoint.job-manager";
+
+#define CHECKPOINT_VERSION 1
 
 int restart_count_char (const char *s, char c)
 {
@@ -40,43 +47,168 @@ int restart_count_char (const char *s, char c)
     return count;
 }
 
-static int depthfirst_map_one (flux_t *h, const char *key, int dirskip,
-                               restart_map_f cb, void *arg)
+static flux_future_t *lookup_job_data (flux_t *h,
+                                       flux_jobid_t id,
+                                       const char *key)
 {
-    flux_jobid_t id;
+    char path[64];
     flux_future_t *f;
+
+    if (flux_job_kvs_key (path, sizeof (path), id, key) < 0
+        || !(f = flux_kvs_lookup (h, NULL, 0, path)))
+        return NULL;
+    return f;
+}
+
+static const char *lookup_job_data_get (flux_future_t *f,
+                                        flux_error_t *error)
+{
+    const char *result;
+
+    if (flux_kvs_lookup_get (f, &result) < 0) {
+        errprintf (error,
+                   "lookup %s: %s",
+                   flux_kvs_lookup_get_key (f),
+                   strerror (errno));
+        return NULL;
+    }
+    return result;
+}
+
+static struct job *lookup_job (flux_t *h,
+                               flux_jobid_t id,
+                               flux_error_t *error,
+                               bool *fatal)
+{
+    flux_future_t *f1 = NULL;
+    flux_future_t *f2 = NULL;
+    flux_future_t *f3 = NULL;
     const char *eventlog;
+    const char *jobspec;
+    const char *R;
+    struct job *job = NULL;
+    flux_error_t e;
+
+    if (!(f1 = lookup_job_data (h, id, "eventlog"))
+        || !(f2 = lookup_job_data (h, id, "jobspec"))
+        || !(f3 = lookup_job_data (h, id, "R"))) {
+        errprintf (error,
+                   "cannot send lookup requests for job %s: %s",
+                   idf58 (id),
+                   strerror (errno));
+        *fatal = true;
+        goto done;
+    }
+    if (!(eventlog = lookup_job_data_get (f1, error))) {
+        *fatal = false;
+        goto done;
+    }
+    if (!(jobspec = lookup_job_data_get (f2, error))) {
+        *fatal = false;
+        goto done;
+    }
+    /* Ignore error if this returns NULL, since R is only available
+     * after resources have been allocated.
+     */
+    R = lookup_job_data_get (f3, NULL);
+
+    /* Treat these errors as non-fatal to avoid a nuisance on restart.
+     * See also: flux-framework/flux-core#6123
+     */
+    if (!(job = job_create_from_eventlog (id,
+                                          eventlog,
+                                          jobspec,
+                                          R,
+                                          &e))) {
+        errprintf (error,
+                   "replay %s: %s",
+                   flux_kvs_lookup_get_key (f1),
+                   e.text);
+        *fatal = false;
+    }
+done:
+    flux_future_destroy (f1);
+    flux_future_destroy (f2);
+    flux_future_destroy (f3);
+    return job;
+}
+
+/* A job could not be reloaded due to some problem like a truncated eventlog.
+ * Move job data to lost+found for manual cleanup.
+ */
+static void move_to_lost_found (flux_t *h, const char *key, flux_jobid_t id)
+{
+    char nkey[128];
+    flux_future_t *f;
+
+    snprintf (nkey, sizeof (nkey), "lost+found.job.%s", idf58 (id));
+    if (!(f = flux_kvs_move (h, NULL, key, NULL, nkey, 0))
+        || flux_future_get (f, NULL) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "mv %s %s: %s",
+                  key,
+                  nkey,
+                  future_strerror (f, errno));
+    }
+    flux_future_destroy (f);
+}
+
+/* Create a 'struct job' from the KVS, using synchronous KVS RPCs.
+ * Return 1 on success, 0 on non-fatal error, or -1 on a fatal error,
+ * where a fatal error will prevent flux from starting.
+ */
+static int depthfirst_map_one (flux_t *h,
+                               const char *key,
+                               int dirskip,
+                               restart_map_f cb,
+                               void *arg,
+                               flux_error_t *error)
+{
+    flux_jobid_t id;
     struct job *job = NULL;
-    char path[64];
     int rc = -1;
 
     if (strlen (key) <= dirskip) {
+        errprintf (error, "internal error key=%s dirskip=%d", key, dirskip);
         errno = EINVAL;
         return -1;
     }
-    if (fluid_decode (key + dirskip + 1, &id, FLUID_STRING_DOTHEX) < 0)
-        return -1;
-    if (flux_job_kvs_key (path, sizeof (path), id, "eventlog") < 0) {
-        errno = EINVAL;
+    if (fluid_decode (key + dirskip + 1, &id, FLUID_STRING_DOTHEX) < 0) {
+        errprintf (error, "could not decode %s to job ID", key + dirskip + 1);
         return -1;
     }
-    if (!(f = flux_kvs_lookup (h, NULL, 0, path)))
-        goto done;
-    if (flux_kvs_lookup_get (f, &eventlog) < 0)
-        goto done;
-    if (!(job = job_create_from_eventlog (id, eventlog)))
-        goto done;
-    if (cb (job, arg) < 0)
+
+    flux_error_t lookup_error;
+    bool fatal = false;
+    if (!(job = lookup_job (h, id, &lookup_error, &fatal))) {
+        if (fatal) {
+            errprintf (error, "%s", lookup_error.text);
+            return -1;
+        }
+        move_to_lost_found (h, key, id);
+        flux_log (h,
+                  LOG_ERR,
+                  "job %s not replayed: %s",
+                  idf58 (id),
+                  lookup_error.text);
+        return 0;
+    }
+
+    if (cb (job, arg, error) < 0)
         goto done;
     rc = 1;
 done:
-    flux_future_destroy (f);
     job_decref (job);
     return rc;
 }
 
-static int depthfirst_map (flux_t *h, const char *key,
-                           int dirskip, restart_map_f cb, void *arg)
+static int depthfirst_map (flux_t *h,
+                           const char *key,
+                           int dirskip,
+                           restart_map_f cb,
+                           void *arg,
+                           flux_error_t *error)
 {
     flux_future_t *f;
     const flux_kvsdir_t *dir;
@@ -87,26 +219,48 @@ static int depthfirst_map (flux_t *h, const char *key,
     int rc = -1;
 
     path_level = restart_count_char (key + dirskip, '.');
-    if (!(f = flux_kvs_lookup (h, NULL, FLUX_KVS_READDIR, key)))
+    if (!(f = flux_kvs_lookup (h, NULL, FLUX_KVS_READDIR, key))) {
+        errprintf (error,
+                   "cannot send lookup request for %s: %s",
+                   key,
+                   strerror (errno));
         return -1;
+    }
     if (flux_kvs_lookup_get_dir (f, &dir) < 0) {
         if (errno == ENOENT && path_level == 0)
             rc = 0;
+        else {
+            errprintf (error,
+                       "could not look up %s: %s",
+                       key,
+                       strerror (errno));
+        }
         goto done;
     }
-    if (!(itr = flux_kvsitr_create (dir)))
+    if (!(itr = flux_kvsitr_create (dir))) {
+        errprintf (error,
+                   "could not create iterator for %s: %s",
+                   key,
+                   strerror (errno));
         goto done;
+    }
     while ((name = flux_kvsitr_next (itr))) {
         char *nkey;
         int n;
         if (!flux_kvsdir_isdir (dir, name))
             continue;
-        if (!(nkey = flux_kvsdir_key_at (dir, name)))
+        if (!(nkey = flux_kvsdir_key_at (dir, name))) {
+            errprintf (error,
+                       "could not build key for %s in %s: %s",
+                       name,
+                       key,
+                       strerror (errno));
             goto done_destroyitr;
+        }
         if (path_level == 3) // orig 'key' = .A.B.C, thus 'nkey' is complete
-            n = depthfirst_map_one (h, nkey, dirskip, cb, arg);
+            n = depthfirst_map_one (h, nkey, dirskip, cb, arg, error);
         else
-            n = depthfirst_map (h, nkey, dirskip, cb, arg);
+            n = depthfirst_map (h, nkey, dirskip, cb, arg, error);
         if (n < 0) {
             int saved_errno = errno;
             free (nkey);
@@ -128,42 +282,216 @@ static int depthfirst_map (flux_t *h, const char *key,
  * The job state/flags has been recreated by replaying the job's eventlog.
  * Enqueue the job and kick off actions appropriate for job's current state.
  */
-static int restart_map_cb (struct job *job, void *arg)
+static int restart_map_cb (struct job *job, void *arg, flux_error_t *error)
 {
     struct job_manager *ctx = arg;
+    flux_job_state_t state = job->state;
 
-    if (zhashx_insert (ctx->active_jobs, &job->id, job) < 0)
+    if (zhashx_insert (ctx->active_jobs, &job->id, job) < 0) {
+        errprintf (error,
+                   "could not insert job %s into active job hash",
+                   idf58 (job->id));
         return -1;
+    }
+    if (ctx->max_jobid < job->id)
+        ctx->max_jobid = job->id;
     if ((job->flags & FLUX_JOB_WAITABLE))
         wait_notify_active (ctx->wait, job);
     if (event_job_action (ctx->event, job) < 0) {
-        flux_log_error (ctx->h, "%s: event_job_action id=%ju",
-                        __FUNCTION__, (uintmax_t)job->id);
+        flux_log_error (ctx->h,
+                        "replay warning: %s->%s action failed on job %s",
+                        flux_job_statetostr (state, "L"),
+                        flux_job_statetostr (job->state, "L"),
+                        idf58 (job->id));
     }
     return 0;
 }
 
-/* Load any active jobs present in the KVS at startup.
- */
+int restart_save_state_to_txn (struct job_manager *ctx, flux_kvs_txn_t *txn)
+{
+    json_t *queue;
+
+    if (!(queue = queue_ctx_save (ctx->queue)))
+        return -1;
+    if (flux_kvs_txn_pack (txn,
+                           0,
+                           checkpoint_key,
+                           "{s:i s:I s:O}",
+                           "version", CHECKPOINT_VERSION,
+                           "max_jobid", ctx->max_jobid,
+                           "queue", queue) < 0) {
+        json_decref (queue);
+        return -1;
+    }
+    json_decref (queue);
+    return 0;
+}
+
+int restart_save_state (struct job_manager *ctx)
+{
+    flux_future_t *f = NULL;
+    flux_kvs_txn_t *txn;
+    int rc = -1;
+
+    if (!(txn = flux_kvs_txn_create ())
+        || restart_save_state_to_txn (ctx, txn) < 0
+        || !(f = flux_kvs_commit (ctx->h, NULL, 0, txn))
+        || flux_future_get (f, NULL) < 0)
+        goto done;
+    rc = 0;
+done:
+    flux_future_destroy (f);
+    flux_kvs_txn_destroy (txn);
+    return rc;
+}
+
+static int restart_restore_state (struct job_manager *ctx)
+{
+    flux_future_t *f;
+    flux_jobid_t id;
+    json_t *queue = NULL;
+    int version = 0;
+
+    if (!(f = flux_kvs_lookup (ctx->h, NULL, 0, checkpoint_key)))
+        return -1;
+
+    if (flux_kvs_lookup_get_unpack (f,
+                                    "{s?i s:I s?o}",
+                                    "version", &version,
+                                    "max_jobid", &id,
+                                    "queue", &queue) < 0)
+        goto error;
+    if (version > 1) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (ctx->max_jobid < id)
+        ctx->max_jobid = id;
+    if (queue) {
+        if (queue_ctx_restore (ctx->queue, version, queue) < 0)
+            goto error;
+    }
+    flux_future_destroy (f);
+    return 0;
+error:
+    flux_future_destroy (f);
+    return -1;
+}
+
 int restart_from_kvs (struct job_manager *ctx)
 {
     const char *dirname = "job";
     int dirskip = strlen (dirname);
     int count;
     struct job *job;
+    flux_error_t error;
 
-    count = depthfirst_map (ctx->h, dirname, dirskip, restart_map_cb, ctx);
-    if (count < 0)
+    /* Load any active jobs present in the KVS at startup.
+     */
+    count = depthfirst_map (ctx->h,
+                            dirname,
+                            dirskip,
+                            restart_map_cb,
+                            ctx,
+                            &error);
+    if (count < 0) {
+        flux_log (ctx->h, LOG_ERR, "restart failed: %s", error.text);
         return -1;
-    /* Initialize the count of "running" jobs
+    }
+    flux_log (ctx->h, LOG_INFO, "restart: %d jobs", count);
+    /* Post flux-restart to any jobs in SCHED state, so they may
+     * transition back to PRIORITY and re-obtain the priority.
+     *
+     * Initialize the count of "running" jobs
      */
     job = zhashx_first (ctx->active_jobs);
     while (job) {
-        if ((job->state & FLUX_JOB_RUNNING) != 0)
+        if (job->state == FLUX_JOB_STATE_NEW
+            || job->state == FLUX_JOB_STATE_DEPEND) {
+            char *errmsg = NULL;
+            if (jobtap_check_dependencies (ctx->jobtap,
+                                           job,
+                                           true,
+                                           &errmsg) < 0) {
+                flux_log (ctx->h, LOG_ERR,
+                          "restart: id=%s: dependency check failed: %s",
+                          idf58 (job->id), errmsg);
+            }
+            free (errmsg);
+        }
+        /*
+         *  On restart, call 'job.create' and 'job.new' plugin callbacks
+         *   since this is the first time this instance of the job-manager
+         *   has seen this job. Be sure to call these before posting any
+         *   other events below, since these should always be the first
+         *   callbacks for a job.
+         *
+         *  Jobs in SCHED state may also immediately transition back to
+         *   PRIORITY, potentially generating two other plugin callbacks
+         *   after this one. (job.priority, job.sched...)
+         */
+        if (jobtap_call (ctx->jobtap, job, "job.create", NULL) < 0)
+            flux_log_error (ctx->h, "jobtap_call (id=%s, create)",
+                            idf58 (job->id));
+        if (jobtap_call (ctx->jobtap, job, "job.new", NULL) < 0)
+            flux_log_error (ctx->h, "jobtap_call (id=%s, new)",
+                            idf58 (job->id));
+
+        if (job->state == FLUX_JOB_STATE_SCHED) {
+            /*
+             * This is confusing. In order to update priority on transition
+             *  back to PRIORITY state, the priority must be reset to "-1",
+             *  even though the last priority value was reconstructed from
+             *  the eventlog. This is because the transitioning "priority"
+             *  event is only posted when the priority changes.
+             */
+            job->priority = -1;
+            if (event_job_post_pack (ctx->event,
+                                     job,
+                                     "flux-restart",
+                                     0,
+                                     NULL) < 0) {
+                flux_log_error (ctx->h, "%s: event_job_post_pack id=%s",
+                                __FUNCTION__, idf58 (job->id));
+            }
+        }
+        else if ((job->state & FLUX_JOB_STATE_RUNNING) != 0) {
             ctx->running_jobs++;
+            job->reattach = 1;
+            if ((job->flags & FLUX_JOB_DEBUG)) {
+                if (event_job_post_pack (ctx->event,
+                                         job,
+                                         "debug.exec-reattach-start",
+                                         0,
+                                         "{s:I}",
+                                         "id", idf58 (job->id)) < 0)
+                    flux_log_error (ctx->h, "%s: event_job_post_pack id=%s",
+                                    __FUNCTION__, idf58 (job->id));
+            }
+        }
+        job = zhashx_next (ctx->active_jobs);
+    }
+    flux_log (ctx->h, LOG_INFO, "restart: %d running jobs", ctx->running_jobs);
+
+    job = zhashx_first (ctx->inactive_jobs);
+    while (job) {
+        (void)jobtap_call (ctx->jobtap, job, "job.inactive-add", NULL);
         job = zhashx_next (ctx->active_jobs);
     }
-    flux_log (ctx->h, LOG_DEBUG, "%s: added %d jobs", __FUNCTION__, count);
+
+    /* Restore misc state.
+     */
+    if (restart_restore_state (ctx) < 0) {
+        if (errno != ENOENT) {
+            flux_log_error (ctx->h, "restart: %s", checkpoint_key);
+            return -1;
+        }
+        flux_log (ctx->h, LOG_INFO, "restart: %s not found", checkpoint_key);
+    }
+    flux_log (ctx->h,
+              LOG_DEBUG,
+              "restart: max_jobid=%s",
+              idf58 (ctx->max_jobid));
     return 0;
 }
 
diff --git a/src/modules/job-manager/restart.h b/src/modules/job-manager/restart.h
index ae77c0ab2fff..d8b0d78d0b3a 100644
--- a/src/modules/job-manager/restart.h
+++ b/src/modules/job-manager/restart.h
@@ -20,6 +20,11 @@ int restart_from_kvs (struct job_manager *ctx);
 /* exposed for unit testing only */
 int restart_count_char (const char *s, char c);
 
+int restart_save_state (struct job_manager *ctx);
+
+int restart_save_state_to_txn (struct job_manager *ctx, flux_kvs_txn_t *txn);
+
+
 #endif /* _FLUX_JOB_MANAGER_RESTART_H */
 
 /*
diff --git a/src/modules/job-manager/start.c b/src/modules/job-manager/start.c
index ad311c3bde8f..7b02568ac287 100644
--- a/src/modules/job-manager/start.c
+++ b/src/modules/job-manager/start.c
@@ -43,7 +43,7 @@
  *             data: {"ranks":s "final":b}
  *
  * "exception" - raise an exception (0 is fatal)
- *               data: {"severity":i "type":s "note"?:s}
+ *               data: {"severity":i "type":s "note":s}
  *
  * "finish" - data: {"status":i}
  *
@@ -85,8 +85,13 @@
 #include <flux/core.h>
 #include <assert.h>
 
+#include "src/common/libjob/idf58.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
+
 #include "job.h"
 #include "event.h"
+#include "raise.h"
 
 #include "start.h"
 
@@ -94,10 +99,13 @@ struct start {
     struct job_manager *ctx;
     flux_msg_handler_t **handlers;
     char *topic;
+    char *update_topic;
 };
 
-static void hello_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+static void hello_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     struct job_manager *ctx = arg;
     struct start *start = ctx->start;
@@ -119,9 +127,12 @@ static void hello_cb (flux_t *h, flux_msg_handler_t *mh,
             job = zhashx_next (ctx->active_jobs);
         }
         free (start->topic);
+        free (start->update_topic);
         start->topic = NULL;
+        start->update_topic = NULL;
     }
-    if (asprintf (&start->topic, "%s.start", service_name) < 0)
+    if (asprintf (&start->topic, "%s.start", service_name) < 0
+        || asprintf (&start->update_topic, "%s.expiration", service_name) < 0)
         goto error;
     if (flux_respond (h, msg, NULL) < 0)
         flux_log_error (h, "%s: flux_respond", __FUNCTION__);
@@ -129,10 +140,12 @@ static void hello_cb (flux_t *h, flux_msg_handler_t *mh,
      */
     job = zhashx_first (ctx->active_jobs);
     while (job) {
-        if (job->state == FLUX_JOB_RUN) {
+        if (job->state == FLUX_JOB_STATE_RUN) {
             if (event_job_action (ctx->event, job) < 0)
-                flux_log_error (h, "%s: event_job_action id=%ju", __FUNCTION__,
-                                (uintmax_t)job->id);
+                flux_log_error (h,
+                                "%s: event_job_action id=%s",
+                                __FUNCTION__,
+                                idf58 (job->id));
         }
         job = zhashx_next (ctx->active_jobs);
     }
@@ -148,8 +161,11 @@ static void interface_teardown (struct start *start, char *s, int errnum)
         struct job_manager *ctx = start->ctx;
         struct job *job;
 
-        flux_log (ctx->h, LOG_DEBUG, "start: stop due to %s: %s",
-                  s, flux_strerror (errnum));
+        flux_log (ctx->h,
+                  LOG_DEBUG,
+                  "start: stop due to %s: %s",
+                  s,
+                  flux_strerror (errnum));
 
         free (start->topic);
         start->topic = NULL;
@@ -158,9 +174,12 @@ static void interface_teardown (struct start *start, char *s, int errnum)
         while (job) {
             if (job->start_pending) {
                 if ((job->flags & FLUX_JOB_DEBUG))
-                    (void)event_job_post_pack (ctx->event, job,
+                    (void)event_job_post_pack (ctx->event,
+                                               job,
                                                "debug.start-lost",
-                                               "{ s:s }", "note", s);
+                                               0,
+                                               "{s:s}",
+                                               "note", s);
                 job->start_pending = 0;
             }
             job = zhashx_next (ctx->active_jobs);
@@ -168,8 +187,10 @@ static void interface_teardown (struct start *start, char *s, int errnum)
     }
 }
 
-static void start_response_cb (flux_t *h, flux_msg_handler_t *mh,
-                               const flux_msg_t *msg, void *arg)
+static void start_response_cb (flux_t *h,
+                               flux_msg_handler_t *mh,
+                               const flux_msg_t *msg,
+                               void *arg)
 {
     struct job_manager *ctx = arg;
     struct start *start = ctx->start;
@@ -181,71 +202,106 @@ static void start_response_cb (flux_t *h, flux_msg_handler_t *mh,
 
     if (flux_response_decode (msg, &topic, NULL) < 0)
         goto teardown; // e.g. ENOSYS
-    if (!start->topic || strcmp (start->topic, topic) != 0) {
+    if (!start->topic || !streq (start->topic, topic)) {
         flux_log_error (h, "start: topic=%s not registered", topic);
         goto error;
     }
-    if (flux_msg_unpack (msg, "{s:I s:s s:o}", "id", &id,
-                                               "type", &type,
-                                               "data", &data) < 0) {
+    if (flux_msg_unpack (msg,
+                         "{s:I s:s s:o}",
+                         "id", &id,
+                         "type", &type,
+                         "data", &data) < 0) {
         flux_log_error (h, "start response payload");
         goto error;
     }
     if (!(job = zhashx_lookup (ctx->active_jobs, &id))) {
-        flux_log (h, LOG_ERR, "start response: id=%ju not active",
-                  (uintmax_t)id);
+        flux_log (h,
+                  LOG_ERR,
+                  "start response: id=%s not active",
+                  idf58 (id));
         errno = EINVAL;
         goto error;
     }
-    if (!strcmp (type, "start")) {
-        if (event_job_post_pack (ctx->event, job, "start", NULL) < 0)
-            goto error_post;
+    if (streq (type, "start")) {
+        if (job->reattach)
+            flux_log (h,
+                      LOG_ERR,
+                      "start response: id=%s should not get start event",
+                      idf58 (id));
+        else {
+            if (event_job_post_pack (ctx->event, job, "start", 0, NULL) < 0)
+                goto error_post;
+        }
+    }
+    else if (streq (type, "reattached")) {
+        if ((job->flags & FLUX_JOB_DEBUG)) {
+            if (event_job_post_pack (ctx->event,
+                                     job,
+                                     "debug.exec-reattach-finish",
+                                     0,
+                                     NULL) < 0)
+                goto error_post;
+        }
     }
-    else if (!strcmp (type, "release")) {
+    else if (streq (type, "release")) {
         const char *idset;
         int final;
-        if (json_unpack (data, "{s:s s:b}", "ranks", &idset,
-                                            "final", &final) < 0) {
+        if (json_unpack (data,
+                         "{s:s s:b}",
+                         "ranks", &idset,
+                         "final", &final) < 0) {
             errno = EPROTO;
             flux_log_error (h, "start: release response: malformed data");
             goto error;
         }
         if (final) // final release is end-of-stream
             job->start_pending = 0;
-        if (event_job_post_pack (ctx->event, job, "release",
-                                 "{ s:s s:b }",
+        if (event_job_post_pack (ctx->event,
+                                 job,
+                                 "release",
+                                 0,
+                                 "{s:s s:b}",
                                  "ranks", idset,
                                  "final", final) < 0)
             goto error_post;
     }
-    else if (!strcmp (type, "exception")) {
+    else if (streq (type, "exception")) {
         int xseverity;
         const char *xtype;
         const char *xnote = NULL;
-        if (json_unpack (data, "{s:i s:s s?:s}", "severity", &xseverity,
-                                                 "type", &xtype,
-                                                 "note", &xnote) < 0) {
+        if (json_unpack (data,
+                         "{s:i s:s s?s}",
+                         "severity", &xseverity,
+                         "type", &xtype,
+                         "note", &xnote) < 0) {
             errno = EPROTO;
             flux_log_error (h, "start: exception response: malformed data");
             goto error;
         }
-        if (event_job_post_pack (ctx->event, job, "exception",
-                                 "{ s:s s:i s:i s:s }",
+        if (event_job_post_pack (ctx->event,
+                                 job,
+                                 "exception",
+                                 0,
+                                 "{s:s s:i s:I s:s}",
                                  "type", xtype,
                                  "severity", xseverity,
-                                 "userid", FLUX_USERID_UNKNOWN,
-                                 "note", xnote ? xnote : "")  < 0)
+                                 "userid", (json_int_t)ctx->owner,
+                                 "note", xnote)  < 0)
             goto error_post;
     }
-    else if (!strcmp (type, "finish")) {
+    else if (streq (type, "finish")) {
         int status;
         if (json_unpack (data, "{s:i}", "status", &status) < 0) {
             errno = EPROTO;
             flux_log_error (h, "start: finish response: malformed data");
             goto error;
         }
-        if (event_job_post_pack (ctx->event, job, "finish",
-                                 "{ s:i }", "status", status) < 0)
+        if (event_job_post_pack (ctx->event,
+                                 job,
+                                 "finish",
+                                 0,
+                                 "{s:i}",
+                                 "status", status) < 0)
             goto error_post;
     }
     else {
@@ -269,21 +325,28 @@ int start_send_request (struct start *start, struct job *job)
     struct job_manager *ctx = start->ctx;
     flux_msg_t *msg;
 
-    assert (job->state == FLUX_JOB_RUN);
+    assert (job->state == FLUX_JOB_STATE_RUN);
     if (!job->start_pending && start->topic != NULL) {
         if (!(msg = flux_request_encode (start->topic, NULL)))
             return -1;
-        if (flux_msg_pack (msg, "{s:I s:i}",
-                                "id", job->id,
-                                "userid", job->userid) < 0)
+        if (flux_msg_pack (msg,
+                           "{s:I s:I s:O s:b s:O}",
+                           "id", job->id,
+                           "userid", (json_int_t) job->userid,
+                           "jobspec", job->jobspec_redacted,
+                           "reattach", job->reattach,
+                           "R", job->R_redacted) < 0)
             goto error;
         if (flux_send (ctx->h, msg, 0) < 0)
             goto error;
         flux_msg_destroy (msg);
         job->start_pending = 1;
         if ((job->flags & FLUX_JOB_DEBUG))
-            (void)event_job_post_pack (ctx->event, job,
-                                       "debug.start-request", NULL);
+            (void)event_job_post_pack (ctx->event,
+                                       job,
+                                       "debug.start-request",
+                                       0,
+                                       NULL);
     }
     return 0;
 error:
@@ -291,12 +354,65 @@ int start_send_request (struct start *start, struct job *job)
     return -1;
 }
 
+static void expiration_update_cb (flux_future_t *f, void *arg)
+{
+    struct job *job = arg;
+    if (flux_future_get (f, NULL) < 0) {
+        struct job_manager *ctx = flux_future_aux_get (f, "job-manager::ctx");
+        const char *note = "failed to send expiration update to exec system: "
+                           "job termination may not coincide with expiration";
+        if (ctx != NULL
+            && raise_job_exception (ctx,
+                                    job,
+                                    "exec",
+                                    1,
+                                    FLUX_USERID_UNKNOWN,
+                                    note) < 0)
+            flux_log_error (ctx->h, "expiration_update: raise_job_exception");
+    }
+    job_aux_delete (job, "job-manager::R-update");
+}
+
+/* Send <exec_service>.expiration request to adjust job expiration
+ */
+int start_send_expiration_update (struct start *start,
+                                  struct job *job,
+                                  json_t *context)
+{
+    struct job_manager *ctx = start->ctx;
+    flux_future_t *f = NULL;
+    double expiration;
+
+    if (json_unpack (context, "{s:F}", "expiration", &expiration) < 0
+        || !(f = flux_rpc_pack (start->ctx->h,
+                                start->update_topic,
+                                0,
+                                0,
+                                "{s:I s:f}",
+                                "id", job->id,
+                                "expiration", expiration))
+        || job_aux_set (job,
+                        "job-manager::R-update",
+                        f,
+                        (flux_free_f) flux_future_destroy) < 0
+        || flux_future_then (f, -1., expiration_update_cb, job) < 0
+        || flux_future_aux_set (f, "job-manager::ctx", ctx, NULL) < 0) {
+        int saved_errno = errno;
+        (void) job_aux_delete (job, "job-manager::R-update");
+        flux_future_destroy (f);
+        errno = saved_errno;
+        return -1;
+    }
+    return 0;
+}
+
 void start_ctx_destroy (struct start *start)
 {
     if (start) {
         int saved_errno = errno;;
         flux_msg_handler_delvec (start->handlers);
         free (start->topic);
+        free (start->update_topic);
         free (start);
         errno = saved_errno;
     }
diff --git a/src/modules/job-manager/start.h b/src/modules/job-manager/start.h
index 6c13cf09bbba..f9dbe3378ade 100644
--- a/src/modules/job-manager/start.h
+++ b/src/modules/job-manager/start.h
@@ -20,6 +20,10 @@ void start_ctx_destroy (struct start *start);
 
 int start_send_request (struct start *start, struct job *job);
 
+int start_send_expiration_update (struct start *start,
+                                  struct job *job,
+                                  json_t *context);
+
 #endif /* ! _FLUX_JOB_MANAGER_START_H */
 
 /*
diff --git a/src/modules/job-manager/submit.c b/src/modules/job-manager/submit.c
index bb3611eb4bd5..475816eb5816 100644
--- a/src/modules/job-manager/submit.c
+++ b/src/modules/job-manager/submit.c
@@ -13,134 +13,146 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libutil/errno_safe.h"
+
 #include "job.h"
 #include "alloc.h"
 #include "event.h"
 #include "wait.h"
+#include "queue.h"
+#include "jobtap-internal.h"
 
 #include "submit.h"
 
-#include "src/common/libeventlog/eventlog.h"
-
 struct submit {
     struct job_manager *ctx;
-    bool submit_disable;
-    char *disable_errmsg;
     flux_msg_handler_t **handlers;
 };
 
-/* Decode 'o' into a struct job, then add it to the active_job hash.
- * Also record the job in 'newjobs'.
+/* Submit event requires special handling to use job->t_submit (from
+ * job-ingest) as the event timestamp.
  */
-int submit_add_one_job (zhashx_t *active_jobs, zlist_t *newjobs, json_t *o)
+static int submit_post_event (struct job_manager *ctx, struct job *job)
 {
-    struct job *job;
+    json_t *entry = NULL;
+    int rv = -1;
 
-    if (!(job = job_create ()))
-        return -1;
-    if (json_unpack (o, "{s:I s:i s:i s:f s:i}",
-                        "id", &job->id,
-                        "priority", &job->priority,
-                        "userid", &job->userid,
-                        "t_submit", &job->t_submit,
-                        "flags", &job->flags) < 0) {
-        errno = EPROTO;
-        job_decref (job);
-        return -1;
-    }
-    if (zhashx_insert (active_jobs, &job->id, job) < 0) {
-        job_decref (job);
-        /* zhashx_insert() fails if hash item already exists.
-         * This is not an error - there is a window for restart_from_kvs()
-         * to pick up a job that also has a submit request in flight.
-         */
-        return 0;
-    }
-    if (zlist_push (newjobs, job) < 0) {
-        zhashx_delete (active_jobs, &job->id);
-        job_decref (job);
-        errno = ENOMEM;
+    entry = eventlog_entry_pack (job->t_submit,
+                                 "submit",
+                                 "{s:I s:i s:i s:i}",
+                                 "userid", (json_int_t) job->userid,
+                                 "urgency", job->urgency,
+                                 "flags", job->flags,
+                                 "version", 1);
+    if (!entry)
         return -1;
-    }
-    return 0;
+
+    rv = event_job_post_entry (ctx->event, job, 0, entry);
+    ERRNO_SAFE_WRAP (json_decref, entry);
+    return rv;
 }
 
-/* The submit request has failed.  Dequeue jobs recorded in 'newjobs',
- * then destroy the newjobs list.
- */
-void submit_add_jobs_cleanup (zhashx_t *active_jobs, zlist_t *newjobs)
+static void set_errorf (json_t *errors,
+                        flux_jobid_t id,
+                        const char *fmt, ...)
 {
-    if (newjobs) {
-        int saved_errno = errno;
-        struct job *job;
-        while ((job = zlist_pop (newjobs))) {
-            zhashx_delete (active_jobs, &job->id);
-            job_decref (job);
-        }
-        zlist_destroy (&newjobs);
-        errno = saved_errno;
+    va_list ap;
+    char buf[256];
+    json_t *o;
+
+    va_start (ap, fmt);
+    vsnprintf (buf, sizeof (buf), fmt, ap);
+    va_end (ap);
+    if (!(o = json_pack ("[Is]", id, buf))
+        || json_array_append_new (errors, o)) {
+        json_decref (o);
+        return;
     }
 }
 
-/* Add jobs from 'jobs' array to 'active_jobs' hash.
- * On success, return a list of struct job's.
- * On failure, return NULL with errno set (no jobs added).
- */
-zlist_t *submit_add_jobs (zhashx_t *active_jobs, json_t *jobs)
+static int submit_job (struct job_manager *ctx,
+                       struct job *job,
+                       json_t *errors)
 {
-    size_t index;
-    json_t *el;
-    zlist_t *newjobs;
+    flux_error_t e;
+    char *error = NULL;
 
-    if (!(newjobs = zlist_new ())) {
-        errno = ENOMEM;
-        return NULL;
+    if (queue_submit_check (ctx->queue, job->jobspec_redacted, &e) < 0) {
+        set_errorf (errors, job->id, "%s", e.text);
+        return -1;
     }
-    json_array_foreach (jobs, index, el) {
-        if (submit_add_one_job (active_jobs, newjobs, el) < 0)
-            goto error;
+    if (zhashx_insert (ctx->active_jobs, &job->id, job) < 0) {
+        set_errorf (errors, job->id, "hash insert failed");
+        return -1;
     }
-    return newjobs;
-error:
-    submit_add_jobs_cleanup (active_jobs, newjobs);
-    return NULL;
-}
-
-/* Submit event requires special handling.  It cannot go through
- * event_job_post_pack() because job-ingest already logged it.
- * However, we want to let the state machine choose the next state and action,
- * We instead re-create the event and run it directly through
- * event_job_update() and event_job_action().
- */
-int submit_post_event (struct event *event, struct job *job)
-{
-    json_t *entry = NULL;
-    int rv = -1;
-
-    entry = eventlog_entry_pack (job->t_submit,
-                                 "submit",
-                                 "{ s:i s:i s:i }",
-                                 "userid", job->userid,
-                                 "priority", job->priority,
-                                 "flags", job->flags);
-    if (!entry)
-        goto error;
-    if (event_job_update (job, entry) < 0) /* NEW -> DEPEND */
-        goto error;
-    if (event_batch_pub_state (event, job, job->t_submit) < 0)
+    /* Post the submit event.
+     */
+    if (submit_post_event (ctx, job) < 0) {
+        set_errorf (errors, job->id, "error posting submit event");
         goto error;
-    if (event_job_action (event, job) < 0)
+    }
+    /* Post the job.create callback.  Since the plugin might post events,
+     * it is called _after_ the submit event is posted, since submit SHALL
+     * be the first event per RFC 21.
+     */
+    /* Call job.validate callback.
+     */
+    if (jobtap_call_create (ctx->jobtap, job, &error) < 0
+        || jobtap_validate (ctx->jobtap, job, &error) < 0
+        || jobtap_check_dependencies (ctx->jobtap, job, false, &error) < 0) {
+        set_errorf (errors,
+                    job->id,
+                    "%s",
+                    error ? error : "rejected by plugin");
+        free (error);
+        goto error_post_invalid;
+    }
+    /* Call job.new callback now that job is accepted, so plugins
+     * may account for this job.
+     */
+    (void) jobtap_call (ctx->jobtap, job, "job.new", NULL);
+
+    /* Add this job to current commit batch. This pauses event processing
+     *  for the job until the current batch is committed to the KVS. This
+     *  ensures that the job eventlog is available in the KVS before further
+     *  state transitions for the job are made (i.e. before the job is
+     *  allocated resources and is started by the job exec system.)
+     */
+    if (event_batch_add_job (ctx->event, job) < 0)
         goto error;
-    rv = 0;
- error:
-    json_decref (entry);
-    return rv;
+
+    /* Post the validate event.
+     */
+    if (event_job_post_pack (ctx->event, job, "validate", 0, NULL) < 0) {
+        set_errorf (errors, job->id, "error posting validate event");
+        goto error_post_invalid;
+    }
+    if ((job->flags & FLUX_JOB_WAITABLE))
+        wait_notify_active (ctx->wait, job);
+    if (ctx->max_jobid < job->id)
+        ctx->max_jobid = job->id;
+    return 0;
+error_post_invalid:
+    /* Let journal consumers know this job did not pass validation and
+     * all data concerning it should be expunged.
+     */
+    (void)event_job_post_pack (ctx->event,
+                               job,
+                               "invalidate",
+                               EVENT_NO_COMMIT,
+                               NULL);
+    (void) jobtap_call (ctx->jobtap, job, "job.destroy", NULL);
+error:
+    zhashx_delete (ctx->active_jobs, &job->id);
+    return -1;
 }
 
+
 /* handle submit request (from job-ingest module)
  * This is a batched request for one or more jobs already validated
  * by the ingest module, and already instantiated in the KVS.
@@ -151,90 +163,47 @@ static void submit_cb (flux_t *h, flux_msg_handler_t *mh,
 {
     struct job_manager *ctx = arg;
     json_t *jobs;
-    zlist_t *newjobs;
-    struct job *job;
+    size_t index;
+    json_t *o;
+    json_t *errors = NULL;
+    flux_msg_t *response = NULL;
     const char *errmsg = NULL;
 
     if (flux_request_unpack (msg, NULL, "{s:o}", "jobs", &jobs) < 0) {
         flux_log_error (h, "%s", __FUNCTION__);
         goto error;
     }
-    if (ctx->submit->submit_disable) {
-        errno = EINVAL;
-        errmsg = ctx->submit->disable_errmsg;
-        goto error;
-    }
-    if (!(newjobs = submit_add_jobs (ctx->active_jobs, jobs))) {
-        flux_log_error (h, "%s: error enqueuing batch", __FUNCTION__);
+    if (!(errors = json_array ())) {
+        errno = ENOMEM;
         goto error;
     }
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    flux_log (h, LOG_DEBUG, "%s: added %d jobs", __FUNCTION__,
-                            (int)zlist_size (newjobs));
-
-    /* Submitting user is being responded to with jobid's.
-     * Now walk the list of new jobs and advance their state.
+    /* Process each job sequentially, noting any failures (such as rejection
+     * by a validator plugin) in 'errors' which becomes the response payload.
      */
-    while ((job = zlist_pop (newjobs))) {
-        if (submit_post_event (ctx->event, job) < 0)
-            flux_log_error (h, "%s: submit_post_event id=%ju",
-                            __FUNCTION__, (uintmax_t)job->id);
-
-        if ((job->flags & FLUX_JOB_WAITABLE))
-            wait_notify_active (ctx->wait, job);
+    json_array_foreach (jobs, index, o) {
+        struct job *job;
+        if (!(job = job_create_from_json (o)))
+            goto error; // fail all on unlikely EPROTO/ENOMEM
+        submit_job (ctx, job, errors);
         job_decref (job);
     }
-    zlist_destroy (&newjobs);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, errmsg) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-static void submit_admin_cb (flux_t *h, flux_msg_handler_t *mh,
-                             const flux_msg_t *msg, void *arg)
-{
-    struct job_manager *ctx = arg;
-    const char *error_prefix = "job submission is disabled: ";
-    int enable;
-    int query_only;
-    const char *reason;
-
-    if (flux_request_unpack (msg,
-                             NULL,
-                             "{s:b s:b s:s}",
-                             "query_only",
-                             &query_only,
-                             "enable",
-                             &enable,
-                             "reason",
-                             &reason) < 0)
+    /* Attach response to commit batch, to maintain the invariant that the
+     * job ID is only returned to the user after the submit event is committed.
+     */
+    if (!(response = flux_response_derive (msg, 0))
+        || flux_msg_pack (response, "{s:O}", "errors", errors) < 0
+        || event_batch_respond (ctx->event, response) < 0) {
+        flux_log_error (h, "error enqueuing response to submit request");
         goto error;
-    if (!query_only) {
-        if (!enable) {
-            char *errmsg;
-            if (asprintf (&errmsg, "%s%s", error_prefix, reason) < 0)
-                goto error;
-            free (ctx->submit->disable_errmsg);
-            ctx->submit->disable_errmsg = errmsg;
-        }
-        ctx->submit->submit_disable = enable ? false : true;
     }
-    if (ctx->submit->submit_disable)
-        reason = ctx->submit->disable_errmsg + strlen (error_prefix);
-    if (flux_respond_pack (h,
-                           msg,
-                           "{s:b s:s}",
-                           "enable",
-                           ctx->submit->submit_disable ? 0 : 1,
-                           "reason",
-                           ctx->submit->submit_disable ? reason : "") < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
+    flux_msg_decref (response);
+    json_decref (errors);
     return;
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    flux_msg_decref (response);
+    json_decref (errors);
 }
 
 void submit_ctx_destroy (struct submit *submit)
@@ -242,15 +211,17 @@ void submit_ctx_destroy (struct submit *submit)
     if (submit) {
         int saved_errno = errno;
         flux_msg_handler_delvec (submit->handlers);
-        free (submit->disable_errmsg);
         free (submit);
         errno = saved_errno;
     }
 }
 
 static const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_REQUEST, "job-manager.submit", submit_cb, 0},
-    { FLUX_MSGTYPE_REQUEST, "job-manager.submit-admin", submit_admin_cb, 0},
+    {   FLUX_MSGTYPE_REQUEST,
+        "job-manager.submit",
+        submit_cb,
+        0
+    },
     FLUX_MSGHANDLER_TABLE_END,
 };
 
diff --git a/src/modules/job-manager/submit.h b/src/modules/job-manager/submit.h
index b9e3bfbe0ece..2e528f1b233f 100644
--- a/src/modules/job-manager/submit.h
+++ b/src/modules/job-manager/submit.h
@@ -12,10 +12,11 @@
 #define _FLUX_JOB_MANAGER_SUBMIT_H
 
 #include <stdbool.h>
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
 #include "job.h"
 #include "job-manager.h"
 
@@ -23,10 +24,9 @@ struct submit *submit_ctx_create (struct job_manager *ctx);
 void submit_ctx_destroy (struct submit *submit);
 
 /* exposed for unit testing only */
-int submit_add_one_job (zhashx_t *active_jobs, zlist_t *newjobs, json_t *o);
-void submit_add_jobs_cleanup (zhashx_t *active_jobs, zlist_t *newjobs);
-zlist_t *submit_add_jobs (zhashx_t *active_jobs, json_t *jobs);
-int submit_post_event (struct event *event, struct job *job);
+void submit_add_jobs_cleanup (zhashx_t *active_jobs, zlistx_t *newjobs);
+zlistx_t *submit_jobs_to_list (json_t *jobs);
+int submit_hash_jobs (zhashx_t *active_jobs, zlistx_t *newjobs);
 
 #endif /* ! _FLUX_JOB_MANAGER_SUBMIT_H */
 
diff --git a/src/modules/job-manager/test/annotate.c b/src/modules/job-manager/test/annotate.c
new file mode 100644
index 000000000000..0173692a5e66
--- /dev/null
+++ b/src/modules/job-manager/test/annotate.c
@@ -0,0 +1,260 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/modules/job-manager/annotate.h"
+
+void basic (void)
+{
+    json_t *orig;
+    json_t *new;
+    json_t *cmp;
+    int rc;
+
+    orig = json_object ();
+    new = json_object ();
+    cmp = json_object ();
+    if (!orig || !new || !cmp)
+        BAIL_OUT ("json_object() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive does nothing on empty dictionary");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack ("{s:n}", "blah");
+    cmp = json_object ();
+    if (!new || !cmp)
+        BAIL_OUT ("json_object() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive does nothing removing non-existent key");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack("{s:s s:i}", "str", "foo", "num", 1);
+    cmp = json_pack("{s:s s:i}", "str", "foo", "num", 1);
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive updates orig appropriately");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack("{s:s}", "str", "bar");
+    cmp = json_pack("{s:s s:i}", "str", "bar", "num", 1);
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive overwrites existing key");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack("{s:n}", "num");
+    cmp = json_pack("{s:s}", "str", "bar");
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive removes value on json null setting");
+
+    json_decref (new);
+    json_decref (cmp);
+    json_decref (orig);
+}
+
+void recursive (void)
+{
+    json_t *orig;
+    json_t *new;
+    json_t *cmp;
+    int rc;
+
+    orig = json_object ();
+    new = json_pack ("{s:{}}", "obj", "str", "foo");
+    cmp = json_pack ("{}");
+    if (!orig || !new || !cmp)
+        BAIL_OUT ("json_object/pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive recursively does nothing on "
+        "empty dictionary");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack ("{s:{s:s}}", "obj", "str", "foo");
+    cmp = json_pack ("{s:{s:s}}", "obj", "str", "foo");
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive sets dictionary");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack ("{s:{s:n}}", "obj", "blah");
+    cmp = json_pack ("{s:{s:s}}", "obj", "str", "foo");
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive recursively does nothing "
+        "removing non-existent key");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack("{s:{s:i}}", "obj", "num", 1);
+    cmp = json_pack("{s:{s:s s:i}}", "obj", "str", "foo", "num", 1);
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive recursively updates orig appropriately");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack("{s:{s:s}}", "obj", "str", "bar");
+    cmp = json_pack("{s:{s:s s:i}}", "obj", "str", "bar", "num", 1);
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive recursively overwrites existing key");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack("{s:{s:n}}", "obj", "num");
+    cmp = json_pack("{s:{s:s}}", "obj", "str", "bar");
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive recursively removes value "
+        "on json null setting");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack("{s:{s:n}}", "obj", "str");
+    cmp = json_pack("{}");
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive recursively removes empty "
+        "sub-dictionaries");
+
+    json_decref (new);
+    json_decref (cmp);
+    json_decref (orig);
+}
+
+void overwrite (void)
+{
+    json_t *orig;
+    json_t *new;
+    json_t *cmp;
+    int rc;
+
+    orig = json_object ();
+    new = json_pack ("{s:{s:s}}", "obj", "str", "foo");
+    cmp = json_pack ("{s:{s:s}}", "obj", "str", "foo");
+    if (!orig || !new || !cmp)
+        BAIL_OUT ("json_object/pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive sets dictionary");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack("{s:s}", "obj", "foo");
+    cmp = json_pack("{s:s}", "obj", "foo");
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive overwrites object with non-object");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack("{s:{s:s}}", "obj", "str", "bar");
+    cmp = json_pack("{s:{s:s}}", "obj", "str", "bar");
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive overwrites non-object with object");
+
+    json_decref (new);
+    json_decref (cmp);
+
+    new = json_pack("{s:n}", "obj");
+    cmp = json_pack("{}");
+    if (!new || !cmp)
+        BAIL_OUT ("json_pack() failed");
+
+    rc = update_annotation_recursive (orig, ".", new);
+    ok (!rc && json_equal (orig, cmp) > 0,
+        "update_annotation_recursive removes whole dict on json null setting");
+
+    json_decref (new);
+    json_decref (cmp);
+    json_decref (orig);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    basic ();
+    recursive ();
+    overwrite ();
+
+    done_testing ();
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/job-manager/test/job.c b/src/modules/job-manager/test/job.c
index f789fc477747..2230f41fbde8 100644
--- a/src/modules/job-manager/test/job.c
+++ b/src/modules/job-manager/test/job.c
@@ -15,7 +15,9 @@
 #include <flux/core.h>
 
 #include "src/common/libtap/tap.h"
+#include "src/common/libeventlog/eventlog.h"
 #include "src/modules/job-manager/job.h"
+#include "ccan/str/str.h"
 
 void test_create (void)
 {
@@ -27,14 +29,13 @@ void test_create (void)
     ok (job->refcount == 1,
         "job_create set refcount to 1");
     ok (job->id == 0
-        && job->priority == FLUX_JOB_PRIORITY_DEFAULT
-        && job->state == FLUX_JOB_NEW
+        && job->urgency == FLUX_JOB_URGENCY_DEFAULT
+        && job->state == FLUX_JOB_STATE_NEW
         && job->userid == FLUX_USERID_UNKNOWN
         && job->t_submit == 0
         && job->flags == 0,
-        "job_create set id, priority, userid, and t_submit to expected values");
+        "job_create set id, urgency, userid, and t_submit to expected values");
     ok (!job->alloc_pending
-        && !job->free_pending
         && !job->has_resources,
         "job_create set no internal flags");
     ok (job->handle == NULL,
@@ -58,153 +59,805 @@ void test_create (void)
 const char *test_input[] = {
     /* 0 */
     "{\"timestamp\":42.2,\"name\":\"submit\","
-     "\"context\":{\"userid\":66,\"priority\":16,\"flags\":42}}\n",
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":1}}\n"
+    "{\"timestamp\":42.3,\"name\":\"validate\"}\n",
+
 
     /* 1 */
     "{\"timestamp\":42.2,\"name\":\"submit\","
-     "\"context\":{\"userid\":66,\"priority\":16,\"flags\":42}}\n"
-    "{\"timestamp\":42.3,\"name\":\"priority\","
-     "\"context\":{\"userid\":42,\"priority\":1}}\n",
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":1}}\n"
+    "{\"timestamp\":42.25,\"name\":\"validate\"}\n"
+    "{\"timestamp\":42.3,\"name\":\"urgency\","
+     "\"context\":{\"userid\":42,\"urgency\":1}}\n",
 
     /* 2 */
     "{\"timestamp\":42.2,\"name\":\"submit\","
-     "\"context\":{\"userid\":66,\"priority\":16,\"flags\":42}}\n"
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":1}}\n"
+    "{\"timestamp\":42.25,\"name\":\"validate\"}\n"
+    "{\"timestamp\":42.3,\"name\":\"depend\"}\n"
+    "{\"timestamp\":42.4,\"name\":\"priority\","
+     "\"context\":{\"priority\":1}}\n",
+
+    /* 3 */
+    "{\"timestamp\":42.2,\"name\":\"submit\","
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":1}}\n"
+    "{\"timestamp\":42.25,\"name\":\"validate\"}\n"
     "{\"timestamp\":42.3,\"name\":\"exception\","
      "\"context\":{\"type\":\"cancel\",\"severity\":0,\"userid\":42}}\n",
 
-    /* 3 */
+    /* 4 */
     "{\"timestamp\":42.2,\"name\":\"submit\","
-     "\"context\":{\"userid\":66,\"priority\":16,\"flags\":42}}\n"
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":1}}\n"
+    "{\"timestamp\":42.25,\"name\":\"validate\"}\n"
     "{\"timestamp\":42.3,\"name\":\"exception\","
      "\"context\":{\"type\":\"meep\",\"severity\":1,\"userid\":42}}\n",
 
-    /* 4 */
+    /* 5 */
     "{\"timestamp\":42.2,\"name\":\"submit\","
-     "\"context\":{\"userid\":66,\"priority\":16,\"flags\":42}}\n"
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":1}}\n"
+    "{\"timestamp\":42.25,\"name\":\"validate\"}\n"
     "{\"timestamp\":42.3,\"name\":\"depend\"}\n"
-    "{\"timestamp\":42.4,\"name\":\"alloc\"}\n",
+    "{\"timestamp\":42.4,\"name\":\"priority\","
+     "\"context\":{\"priority\":100}}\n"
+    "{\"timestamp\":42.5,\"name\":\"alloc\"}\n",
 
-    /* 5 */
+    /* 6 */
     "{\"timestamp\":42.3,\"name\":\"alloc\"}\n",
 
-    /* 6 */
+    /* 7 */
     "{\"timestamp\":42.2,\"name\":\"submit\","
-     "\"context\":{\"userid\":66,\"priority\":16,\"flags\":42}}\n"
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":1}}\n"
+    "{\"timestamp\":42.25,\"name\":\"validate\"}\n"
     "{\"timestamp\":42.3,\"name\":\"depend\"}\n"
-    "{\"timestamp\":42.3,\"name\":\"alloc\"}\n"
-    "{\"timestamp\":42.4,\"name\":\"exception\","
+    "{\"timestamp\":42.4,\"name\":\"priority\","
+     "\"context\":{\"priority\":100}}\n"
+    "{\"timestamp\":42.4,\"name\":\"alloc\"}\n"
+    "{\"timestamp\":42.5,\"name\":\"exception\","
      "\"context\":{\"type\":\"gasp\",\"severity\":0,\"userid\":42}}\n"
-    "{\"timestamp\":42.5,\"name\":\"free\"}\n",
+    "{\"timestamp\":42.6,\"name\":\"free\"}\n",
+
+    /* 8 - no version attribute */
+    "{\"timestamp\":42.2,\"name\":\"submit\","
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42}}\n"
+    "{\"timestamp\":42.3,\"name\":\"depend\"}\n",
+
+    /* 9 - version=0 (invalid) */
+    "{\"timestamp\":42.2,\"name\":\"submit\","
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":0}}\n",
+
+    /* 10 - submit+validate+submit should cause event_job_update to fail */
+    "{\"timestamp\":42.2,\"name\":\"submit\","
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":1}}\n"
+    "{\"timestamp\":42.25,\"name\":\"validate\"}\n"
+    "{\"timestamp\":42.2,\"name\":\"submit\","
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":1}}\n",
+
+    /* 11 - submit leaves state NEW which is invalid */
+    "{\"timestamp\":42.2,\"name\":\"submit\","
+     "\"context\":{\"userid\":66,\"urgency\":16,\"flags\":42,\"version\":1}}\n",
 };
 
 void test_create_from_eventlog (void)
 {
     struct job *job;
+    flux_error_t error;
+
+    errno = 0;
+    error.text[0] = '\0';
+    job = job_create_from_eventlog (2, "xyz", "{}", NULL, &error);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_eventlog on bad eventlog fails with EINVAL");
+    like (error.text, "failed to decode eventlog",
+          "and error.text is set");
+
+    errno = 0;
+    error.text[0] = '\0';
+    job = job_create_from_eventlog (2,
+                                    test_input[0],
+                                    "}badjson}",
+                                    NULL,
+                                    &error);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_eventlog on bad jobspec fails with EINVAL");
+    like (error.text, "failed to decode jobspec",
+          "and error.text is set");
+
+    errno = 0;
+    error.text[0] = '\0';
+    job = job_create_from_eventlog (2,
+                                    test_input[0],
+                                    "{}",
+                                    "}badjson}",
+                                    &error);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_eventlog on bad R fails with EINVAL");
+    like (error.text, "failed to decode R",
+          "and error.text is set");
 
     /* 0 - submit only */
-    job = job_create_from_eventlog (2, test_input[0]);
-    if (job == NULL)
-        BAIL_OUT ("job_create_from_eventlog log=(submit) failed");
+    job = job_create_from_eventlog (2,
+                                    test_input[0],
+                                    "{}",
+                                    NULL,
+                                    &error);
+    if (job == NULL) {
+        BAIL_OUT ("job_create_from_eventlog log=(submit) failed: %s",
+                  error.text);
+    }
     ok (job->refcount == 1,
         "job_create_from_eventlog log=(submit) set refcount to 1");
     ok (job->id == 2,
         "job_create_from_eventlog log=(submit) set id from param");
     ok (!job->alloc_pending
-        && !job->free_pending
         && !job->has_resources,
         "job_create_from_eventlog log=(submit)  set no internal flags");
     ok (job->userid == 66,
         "job_create_from_eventlog log=(submit) set userid from submit");
     ok (job->flags == 42,
         "job_create_from_eventlog log=(submit) set flags from submit");
-    ok (job->priority == 16,
-        "job_create_from_eventlog log=(submit) set priority from submit");
+    ok (job->urgency == 16,
+        "job_create_from_eventlog log=(submit) set urgency from submit");
     ok (job->t_submit == 42.2,
         "job_create_from_eventlog log=(submit) set t_submit from submit");
-    ok (job->state == FLUX_JOB_DEPEND,
+    ok (job->state == FLUX_JOB_STATE_DEPEND,
         "job_create_from_eventlog log=(submit) set state=DEPEND");
     job_decref (job);
 
-    /* 1 - submit + priority */
-    job = job_create_from_eventlog (3, test_input[1]);
-    if (job == NULL)
-        BAIL_OUT ("job_create_from_eventlog log=(submit+pri) failed");
+    /* 1 - submit + urgency */
+    job = job_create_from_eventlog (3,
+                                    test_input[1],
+                                    "{}",
+                                    NULL,
+                                    &error);
+    if (job == NULL) {
+        BAIL_OUT ("job_create_from_eventlog log=(submit+urgency) failed: %s",
+                  error.text);
+    }
+    ok (job->id == 3,
+        "job_create_from_eventlog log=(submit+urgency) set id from param");
+    ok (job->userid == 66,
+        "job_create_from_eventlog log=(submit+urgency) set userid from submit");
+    ok (job->urgency == 1,
+        "job_create_from_eventlog log=(submit+urgency) set urgency from urgency");
+    ok (job->t_submit == 42.2,
+        "job_create_from_eventlog log=(submit+urgency) set t_submit from submit");
+    ok (!job->alloc_pending
+        && !job->has_resources,
+        "job_create_from_eventlog log=(submit+urgency) set no internal flags");
+    ok (job->state == FLUX_JOB_STATE_DEPEND,
+        "job_create_from_eventlog log=(submit+urgency) set state=DEPEND");
+    job_decref (job);
+
+    /* 2 - submit + depend + priority */
+    job = job_create_from_eventlog (3,
+                                    test_input[2],
+                                    "{}",
+                                    NULL,
+                                    &error);
+    if (job == NULL) {
+        BAIL_OUT ("job_create_from_eventlog log=(submit+depend+priority) failed: %s",
+                  error.text);
+    }
     ok (job->id == 3,
-        "job_create_from_eventlog log=(submit+pri) set id from param");
+        "job_create_from_eventlog log=(submit+depend+priority) set id from param");
     ok (job->userid == 66,
-        "job_create_from_eventlog log=(submit+pri) set userid from submit");
+        "job_create_from_eventlog log=(submit+depend+priority) set userid from submit");
+    ok (job->urgency == 16,
+        "job_create_from_eventlog log=(submit+depend+priority) set urgency from submit");
     ok (job->priority == 1,
-        "job_create_from_eventlog log=(submit+pri) set priority from priority");
+        "job_create_from_eventlog log=(submit+depend+priority) set priority from priority");
     ok (job->t_submit == 42.2,
-        "job_create_from_eventlog log=(submit+pri) set t_submit from submit");
+        "job_create_from_eventlog log=(submit+depend+priority) set t_submit from submit");
     ok (!job->alloc_pending
-        && !job->free_pending
         && !job->has_resources,
-        "job_create_from_eventlog log=(submit+pri) set no internal flags");
-    ok (job->state == FLUX_JOB_DEPEND,
-        "job_create_from_eventlog log=(submit+pri) set state=DEPEND");
+        "job_create_from_eventlog log=(submit+depend+priority) set no internal flags");
+    ok (job->state == FLUX_JOB_STATE_SCHED,
+        "job_create_from_eventlog log=(submit+depend+priority) set state=SCHED");
     job_decref (job);
 
-    /* 2 - submit + exception severity 0 */
-    job = job_create_from_eventlog (3, test_input[2]);
-    if (job == NULL)
-        BAIL_OUT ("job_create_from_eventlog log=(submit+ex0) failed");
+    /* 3 - submit + exception severity 0 */
+    job = job_create_from_eventlog (3,
+                                    test_input[3],
+                                    "{}",
+                                    NULL,
+                                    &error);
+    if (job == NULL) {
+        BAIL_OUT ("job_create_from_eventlog log=(submit+ex0) failed: %s",
+                  error.text);
+    }
     ok (job->userid == 66,
         "job_create_from_eventlog log=(submit+ex0) set userid from submit");
-    ok (job->priority == 16,
-        "job_create_from_eventlog log=(submit+ex0) set priority from submit");
+    ok (job->urgency == 16,
+        "job_create_from_eventlog log=(submit+ex0) set urgency from submit");
     ok (job->t_submit == 42.2,
         "job_create_from_eventlog log=(submit+ex0) set t_submit from submit");
     ok (!job->alloc_pending
-        && !job->free_pending
         && !job->has_resources,
         "job_create_from_eventlog log=(submit+ex0) set no internal flags");
-    ok (job->state == FLUX_JOB_CLEANUP,
+    ok (job->state == FLUX_JOB_STATE_CLEANUP,
         "job_create_from_eventlog log=(submit+ex0) set state=CLEANUP");
     job_decref (job);
 
-    /* 3 - submit + exception severity 1 */
-    job = job_create_from_eventlog (3, test_input[3]);
-    if (job == NULL)
-        BAIL_OUT ("job_create_from_eventlog log=(submit+ex1) failed");
-    ok (job->state == FLUX_JOB_DEPEND,
+    /* 4 - submit + exception severity 1 */
+    job = job_create_from_eventlog (3,
+                                    test_input[4],
+                                    "{}",
+                                    NULL,
+                                    &error);
+    if (job == NULL) {
+        BAIL_OUT ("job_create_from_eventlog log=(submit+ex1) failed: %s",
+                  error.text);
+    }
+    ok (job->state == FLUX_JOB_STATE_DEPEND,
         "job_create_from_eventlog log=(submit+ex1) set state=DEPEND");
     ok (!job->alloc_pending
-        && !job->free_pending
         && !job->has_resources,
         "job_create_from_eventlog log=(submit+ex1) set no internal flags");
     job_decref (job);
 
-    /* 4 - submit + depend + alloc */
-    job = job_create_from_eventlog (3, test_input[4]);
-    if (job == NULL)
-        BAIL_OUT ("job_create_from_eventlog log=(submit+depend+alloc) failed");
+    /* 5 - submit + depend + priority + alloc */
+    job = job_create_from_eventlog (3,
+                                    test_input[5],
+                                    "{}",
+                                    "{}",
+                                    &error);
+    if (job == NULL) {
+        BAIL_OUT ("job_create_from_eventlog log=(submit+depend+priority+alloc) failed: %s",
+                  error.text);
+    }
     ok (!job->alloc_pending
-        && !job->free_pending
         && job->has_resources,
-        "job_create_from_eventlog log=(submit+depend+alloc) set has_resources flag");
-    ok (job->state == FLUX_JOB_RUN,
-        "job_create_from_eventlog log=(submit+depend+alloc) set state=RUN");
+        "job_create_from_eventlog log=(submit+depend+priority+alloc) set has_resources flag");
+    ok (job->R_redacted != NULL,
+        "and R is set");
+    ok (job->state == FLUX_JOB_STATE_RUN,
+        "job_create_from_eventlog log=(submit+depend+priority+alloc) set state=RUN");
     job_decref (job);
 
-    /* 5 - missing submit */
+    /* 6 - missing submit */
     errno = 0;
-    job = job_create_from_eventlog (3, test_input[5]);
+    error.text[0] = '\0';
+    job = job_create_from_eventlog (3,
+                                    test_input[6],
+                                    "{}",
+                                    "{}",
+                                    &error);
     ok (job == NULL && errno == EINVAL,
         "job_create_from_eventlog log=(alloc) fails with EINVAL");
+    ok (strlen (error.text) > 0,
+        "and error.text is set");
 
-    /* 6 - submit + depend + alloc + ex0 + free */
-    job = job_create_from_eventlog (3, test_input[6]);
-    if (job == NULL)
-        BAIL_OUT ("job_create_from_eventlog log=(submit+depend+alloc+ex0+free) failed");
+    /* 7 - submit + depend + priority + alloc + ex0 + free */
+    job = job_create_from_eventlog (3,
+                                    test_input[7],
+                                    "{}",
+                                    "{}",
+                                    &error);
+    if (job == NULL) {
+        BAIL_OUT ("job_create_from_eventlog log=(submit+depend+priority+alloc+ex0+free) failed: %s",
+                  error.text);
+    }
     ok (!job->alloc_pending
-        && !job->free_pending
         && !job->has_resources,
-        "job_create_from_eventlog log=(submit+depend+alloc+ex0+free) set no internal flags");
-    ok (job->state == FLUX_JOB_CLEANUP,
-        "job_create_from_eventlog log=(submit+depend+alloc+ex0+free) set state=CLEANUP");
+        "job_create_from_eventlog log=(submit+depend+priority+alloc+ex0+free) set no internal flags");
+    ok (job->state == FLUX_JOB_STATE_CLEANUP,
+        "job_create_from_eventlog log=(submit+depend+priority+alloc+ex0+free) set state=CLEANUP");
+    job_decref (job);
+
+    /* 8 - no version (has no validate event) */
+    job = job_create_from_eventlog (3,
+                                    test_input[8],
+                                    "{}",
+                                    NULL,
+                                    &error);
+    ok (job != NULL,
+        "job_create_from_eventlog version log=(submit.v0+depend) works");
+    ok (job->state == FLUX_JOB_STATE_PRIORITY,
+        "job_create_from_eventlog version log=(submit.v0+depend) state=PRIORITY");
+    job_decref (job);
+
+    /* 9 - invalid version */
+    errno = 0;
+    error.text[0] = '\0';
+    job = job_create_from_eventlog (3,
+                                    test_input[9],
+                                    "{}",
+                                    NULL,
+                                    &error);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_eventlog log=(submit.v0) fails with EINVAL");
+    like (error.text, "eventlog v.* is unsupported",
+          "and error.text is set");
+
+    /* 10 - two submits */
+    errno = 0;
+    error.text[0] = '\0';
+    job = job_create_from_eventlog (3,
+                                    test_input[10],
+                                    "{}",
+                                    NULL,
+                                    &error);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_eventlog log=(submit,validate,submit) fails with EINVAL");
+    like (error.text, "could not apply",
+          "and error.text is set");
+
+    /* 11 - one submit */
+    errno = 0;
+    error.text[0] = '\0';
+    job = job_create_from_eventlog (3,
+                                    test_input[11],
+                                    "{}",
+                                    NULL,
+                                    &error);
+    ok (job == NULL && errno == EINVAL,
+        "job_create_from_eventlog log=(submit) fails with EINVAL");
+    like (error.text, "job state .* is invalid after replay",
+          "and error.text is set");
+}
+
+void test_create_from_json (void)
+{
+    json_t *o;
+    struct job *job;
+
+    errno = 0;
+    ok ((job = job_create_from_json (json_null ())) == NULL && errno == EPROTO,
+        "job_create_from_json on malformed object fails with EPROTO");
+
+    if (!(o = json_pack ("{s:I s:i s:i s:f s:i s:{}}",
+                         "id", 1LL,
+                         "urgency", 10,
+                         "userid", 42,
+                         "t_submit", 1.0,
+                         "flags", 0,
+                         "jobspec")))
+        BAIL_OUT ("json_pack failed");
+    ok ((job = job_create_from_json (o)) != NULL,
+        "job_create_from_json works");
+    ok (job->id == 1
+        && job->urgency == 10
+        && job->userid == 42
+        && job->t_submit == 1.0
+        && job->queue == NULL
+        && job->flags == 0,
+        "job json object was properly decoded");
+    json_decref (o);
+    job_decref (job);
+
+    if (!(o = json_pack ("{s:I s:i s:i s:f s:i s:{s:{s:{s:s}}}}",
+                         "id", 1LL,
+                         "urgency", 10,
+                         "userid", 42,
+                         "t_submit", 1.0,
+                         "flags", 0,
+                         "jobspec",
+                         "attributes", "system", "queue", "foo")))
+        BAIL_OUT ("json_pack failed");
+    ok ((job = job_create_from_json (o)) != NULL,
+        "job_create_from_json works");
+    ok (job->id == 1
+        && job->urgency == 10
+        && job->userid == 42
+        && job->t_submit == 1.0
+        && job->queue && streq (job->queue, "foo")
+        && job->flags == 0,
+        "job json object was properly decoded w/ queue");
+    json_decref (o);
+    job_decref (job);
+}
+
+static void test_subscribe (void)
+{
+    flux_plugin_t *p = flux_plugin_create ();
+    flux_plugin_t *p2 = flux_plugin_create ();
+    struct job * job = job_create ();
+    struct job * job2 = job_create ();
+    if (!job || !job2 || !p || !p2)
+        BAIL_OUT ("failed to create jobs and/or plugins");
+
+    ok (job->subscribers == NULL && job2->subscribers == NULL,
+        "job->subscribers is NULL with no subscribers");
+    ok (job_events_subscribe (job, p) == 0,
+        "job_events_subscribe works");
+    ok (job->subscribers && zlistx_size (job->subscribers) == 1,
+        "job now has one subscription");
+    ok (zlistx_head (job->subscribers) == p,
+        "plugin is first subscriber on list");
+
+    ok (job_events_subscribe (job, p2) == 0,
+        "2nd job_events_subscribe works");
+    ok (job->subscribers && zlistx_size (job->subscribers) == 2,
+        "job now has two subscribers");
+
+    ok (job_events_subscribe (job2, p2) == 0,
+        "subscribe plugin 2 to a second job");
+    ok (job2->subscribers && zlistx_size (job2->subscribers) == 1,
+        "job2 now has one subscriber");
+    ok (zlistx_head (job2->subscribers) == p2,
+        "plugin 2 is first subscriber on job2 subscriber list");
+
+    flux_plugin_destroy (p);
+    pass ("destroy first plugin");
+
+    ok (job->subscribers && zlistx_size (job->subscribers) == 1,
+        "after plugin destruction, job has 1 subscriber");
+    ok (zlistx_head (job->subscribers) == p2,
+        "plugin 2 is now first subscriber on list");
+
+    /*  Now destroy job before plugin
+     */
+    job_decref (job);
+    pass ("destroy job before plugin");
+    job_decref (job2);
+    pass ("destroy job2 before plugin");
+    flux_plugin_destroy (p2);
+    pass ("destroy 2nd plugin after all jobs");
+}
+
+static void test_event_id_cache (void)
+{
+    struct job *job = job_create ();
+    ok (job_event_id_set (job, 1024) < 0 && errno == ENOSPC,
+        "job_event_id_set 1024 returns ENOSPC");
+    ok (job_event_id_set (job, -1) < 0 && errno == EINVAL,
+        "job_event_id_set -1 returns EINVAL");
+
+    ok (job_event_id_test (job, 1024) < 0 && errno == EINVAL,
+        "job_event_id_test 1024 returns EINVAL");
+    ok (job_event_id_test (job, -1) < 0 && errno == EINVAL,
+        "job_event_id_test -1 returns EINVAL");
+
+    ok (job_event_id_test (job, 0) == 0,
+        "job_event_id_test 0 returns 0");
+    ok (job_event_id_test (job, 63) == 0,
+        "job_event_id_test 63 returns 0");
+
+    ok (job_event_id_set (job, 0) == 0,
+        "job_event_id_set works");
+    ok (job_event_id_test (job, 0) == 1,
+        "job_event_id_test 0 now returns 1");
+    ok (job_event_id_set (job, 3) == 0,
+        "job_event_id_set works");
+    ok (job_event_id_test (job, 3) == 1,
+        "job_event_id_test 3 now returns 1");
+    ok (job_event_id_set (job, 63) == 0,
+        "job_event_id_set works");
+    ok (job_event_id_test (job, 63) == 1,
+        "job_event_id_test 63 now returns 1");
+
+    ok (job_event_id_set (job, 3) == 0,
+        "job_event_id_set of the same event works");
+    ok (job_event_id_test (job, 3) == 1,
+        "job_event_id_test of multiple set event works");
+
+    job_decref (job);
+}
+
+static void test_event_queue (void)
+{
+    struct job *job = job_create ();
+    int flags;
+    json_t *entry;
+    const char *name;
+    json_t *context;
+    int i;
+    char dbuf[128];
+
+    if (!job)
+        BAIL_OUT ("job_create failed");
+
+    if (json_array_append (job->event_queue, json_null ()) < 0)
+        BAIL_OUT ("could not enqueue bad event entry");
+    errno = 0;
+    ok (job_event_peek (job, &flags, &entry) < 0 && errno == EPROTO,
+        "job_event_peek fails with EPROTO on badly wrapped eventlog entry");
+    errno = 0;
+    ok (job_event_dequeue (job, &flags, &entry) < 0 && errno == EPROTO,
+        "job_event_dequeue fails with EPROTO on badly wrapped eventlog entry");
+    json_array_remove (job->event_queue, 0);
+
+    errno = 0;
+    ok (job_event_peek (job, &flags, &entry) < 0 && errno == ENOENT,
+        "job_event_peek fails with ENOENT when there are no events");
+    ok (job_event_is_queued (job, "foo") == false,
+        "job_event_is_queued foo returns false");
+    /* Post two test events
+     */
+    entry = eventlog_entry_pack (0.,
+                                 "foo",
+                                 "{s:i}",
+                                 "bar", 42);
+    if (!entry)
+        BAIL_OUT ("eventlog_entry_pack failed");
+    ok (job_event_enqueue (job, 42, entry) == 0,
+        "job_event_enqueue works");
+    json_decref (entry);
+    diag ("queue: %s", job_event_queue_print (job, dbuf, sizeof (dbuf)));
+    ok (json_array_size (job->event_queue) == 1,
+        "queue size is 1");
+    ok (job_event_is_queued (job, "foo") == true,
+        "job_event_is_queued foo returns true");
+    entry = eventlog_entry_pack (0.,
+                                 "bar",
+                                 "{s:i}",
+                                 "baz", 43);
+    if (!entry)
+        BAIL_OUT ("eventlog_entry_pack failed");
+    ok (job_event_enqueue (job, 43, entry) == 0,
+        "job_event_enqueue works");
+    json_decref (entry);
+    diag ("queue: %s", job_event_queue_print (job, dbuf, sizeof (dbuf)));
+    ok (json_array_size (job->event_queue) == 2,
+        "queue size is 2");
+    ok (job_event_is_queued (job, "bar") == true,
+        "job_event_is_queued bar returns true");
+    /* Check the first event
+     */
+    flags = 0;
+    entry = NULL;
+    ok (job_event_peek (job, &flags, &entry) == 0,
+        "job_event_peek works");
+    ok (eventlog_entry_parse (entry, NULL, &name, &context) == 0
+        && streq (name, "foo")
+        && json_unpack (context, "{s:i}", "bar", &i) == 0
+        && i == 42,
+        "eventlog entry is correct");
+    ok (flags == 42,
+        "flags are correct");
+    ok (json_array_size (job->event_queue) == 2,
+        "queue size is still 2");
+    flags = 0;
+    entry = NULL;
+    ok (job_event_dequeue (job, &flags, &entry) == 0,
+        "job_event_dequeue with NULL args works");
+    diag ("queue: %s", job_event_queue_print (job, dbuf, sizeof (dbuf)));
+    ok (json_array_size (job->event_queue) == 1,
+        "queue size is now 1");
+    ok (eventlog_entry_parse (entry, NULL, &name, &context) == 0
+        && streq (name, "foo")
+        && json_unpack (context, "{s:i}", "bar", &i) == 0
+        && i == 42,
+        "eventlog entry is correct");
+    ok (flags == 42,
+        "flags are correct");
+    json_decref (entry);
+    ok (json_array_size (job->event_queue) == 1,
+        "queue size is now 1");
+    /* Check the second event
+     */
+    flags = 0;
+    entry = NULL;
+    ok (job_event_peek (job, &flags, &entry) == 0,
+        "job_event_peek works");
+    ok (eventlog_entry_parse (entry, NULL, &name, &context) == 0
+        && streq (name, "bar")
+        && json_unpack (context, "{s:i}", "baz", &i) == 0
+        && i == 43,
+        "eventlog entry is correct");
+    ok (flags == 43,
+        "flags are correct");
+    ok (json_array_size (job->event_queue) == 1,
+        "queue size is still 1");
+    ok (job_event_dequeue (job, NULL, NULL) == 0,
+        "job_event_dequeue with NULL args works");
+    ok (json_array_size (job->event_queue) == 0,
+        "queue size is now 0");
+
+    job_decref (job);
+}
+
+static void test_jobspec_update (void)
+{
+    struct job *job;
+    flux_error_t error;
+    const char *command;
+    const char *queue;
+    json_t *o;
+    json_t *cpy;
+    int ret;
+
+    /* corner cases */
+
+    if (!(o = json_pack ("{s:s}", "dummy", "dummy")))
+        BAIL_OUT ("failed to create update");
+
+    ok (validate_jobspec_updates (o) == false,
+        "validate_jobspec_updates fails on bad update keys");
+
+    json_decref (o);
+
+    if (!(job = job_create()))
+        BAIL_OUT ("failed to create empty job");
+
+    ret = job_apply_jobspec_updates (job, NULL);
+    ok (ret == -1 && errno == EINVAL,
+        "job_apply_jobspec_updates fail on job with no jobspec");
+
+    cpy = job_jobspec_with_updates (job, NULL);
+    ok (cpy == NULL && errno == EAGAIN,
+        "job_jobspec_with_updates fail on job with no jobspec");
+
+    job_decref (job);
+
+    /* functional tests */
+
+    if (!(job = job_create_from_eventlog (1234,
+                                          test_input[0],
+                                          "{}",
+                                          NULL,
+                                          &error)))
+        BAIL_OUT ("failed to create job w/ empty jobspec");
+
+    if (!(o = json_pack ("{s:[{s:[s]}] s:s}",
+                         "tasks",
+                           "command", "hostname",
+                         "attributes.system.queue", "foo")))
+        BAIL_OUT ("failed to create update");
+
+    ok (validate_jobspec_updates (o) == true,
+        "validate_jobspec_updates success update keys");
+
+    ok (job->queue == NULL, "job->queue NULL before update");
+
+    ret = job_apply_jobspec_updates (job, o);
+    ok (ret == 0, "job_apply_jobspec_updates success");
+    json_decref (o);
+
+    ret = json_unpack (job->jobspec_redacted,
+                       "{s:[{s:[s]}]}",
+                       "tasks",
+                         "command", &command);
+    ok (ret == 0, "parsed jobspec command");
+    ok (command && streq (command, "hostname"),
+        "jobspec command updated correctly");
+
+    ret = json_unpack (job->jobspec_redacted,
+                       "{s:{s:{s:s}}}",
+                       "attributes",
+                         "system",
+                           "queue", &queue);
+    ok (ret == 0, "parsed jobspec queue");
+    ok (queue && streq (queue, "foo"),
+        "jobspec queue updated correctly");
+
+    ok (job->queue && streq (job->queue, "foo"), "job->queue=foo after update");
+
+    if (!(o = json_pack ("{s:s}", "attributes.system.queue", "bar")))
+        BAIL_OUT ("failed to create update");
+
+    cpy = job_jobspec_with_updates (job, o);
+    ok (cpy != NULL, "job_jobspec_with_updates success");
+    json_decref (o);
+
+    ret = json_unpack (cpy,
+                       "{s:{s:{s:s}}}",
+                       "attributes",
+                         "system",
+                           "queue", &queue);
+    ok (ret == 0, "parsed jobspec queue in cpy");
+    ok (queue && streq (queue, "bar"), "jobspec cpy has updated queue");
+    json_decref (cpy);
+
+    ret = json_unpack (job->jobspec_redacted,
+                       "{s:{s:{s:s}}}",
+                       "attributes",
+                         "system",
+                           "queue", &queue);
+    ok (ret == 0, "parsed jobspec queue in original");
+    ok (queue && streq (queue, "foo"), "job jobspec not modified");
+
     job_decref (job);
+}
+
+static void test_resource_update ()
+{
+    struct job *job;
+    double expiration;
+    json_t *update;
+    int rc;
+
 
+    if (!(job = job_create()))
+        BAIL_OUT ("failed to create empty job");
+    if (!(update = json_pack ("{s:f}", "expiration", 100.)))
+        BAIL_OUT ("failed to create update");
+    ok (update != NULL, "create valid resource-update context");
+
+    rc = job_apply_resource_updates (job, update);
+    ok (rc == -1 && errno == EAGAIN,
+        "job_apply_resource_updates fails on job without R_redacted");
+    json_decref (update);
+
+    job->R_redacted = json_pack ("{s:i s:{s:f s:f}}",
+                                 "version", 1,
+                                 "execution",
+                                 "starttime", 1.,
+                                 "expiration", 2.);
+    if (!job->R_redacted)
+        BAIL_OUT ("Failed to create fake R_redacted");
+    ok (job->R_redacted != NULL,
+        "Create fake job->R_redacted");
+
+    if (!(update = json_pack ("{s:f s:s}",
+                              "expiration", 100.,
+                              "dummy", "test")))
+        BAIL_OUT ("failed to create update");
+    ok (update != NULL,
+        "create resource-update context w/ multiple updates");
+
+    rc = job_apply_resource_updates (job, update);
+    ok (rc == -1 && errno == EINVAL,
+        "job_apply_resource_updates fails with multiple updates");
+    json_decref (update);
+
+    if (!(update = json_pack ("{s:s}", "dummy", "test")))
+        BAIL_OUT ("failed to create update");
+    ok (update != NULL,
+        "create resource-update context w/ invalid update key");
+
+    rc = job_apply_resource_updates (job, update);
+    ok (rc == -1 && errno == EINVAL,
+        "job_apply_resource_updates fails with invalid update key");
+    json_decref (update);
+
+    if (!(update = json_pack ("{s:s}", "expiration", "test")))
+        BAIL_OUT ("failed to create update");
+    ok (update != NULL,
+        "create resource-update context w/ invalid value");
+
+    rc = job_apply_resource_updates (job, update);
+    ok (rc == -1 && errno == EINVAL,
+        "job_apply_resource_updates fails with invalid update value");
+    json_decref (update);
+
+    if (!(update = json_pack ("{s:f}", "expiration", -1.0)))
+        BAIL_OUT ("failed to create update");
+    ok (update != NULL,
+        "create resource-update context w/ negative expiration");
+
+    rc = job_apply_resource_updates (job, update);
+    ok (rc == -1 && errno == EINVAL,
+        "job_apply_resource_updates fails with negative expiration");
+    json_decref (update);
+
+    if (!(update = json_pack ("{s:f}", "expiration", 100.0)))
+        BAIL_OUT ("failed to create update");
+    ok (update != NULL,
+        "create resource-update context w/ valid expiration");
+
+    rc = job_apply_resource_updates (job, update);
+    ok (rc ==0,
+        "job_apply_resource_updates() works with valid expiration");
+    json_decref (update);
+
+    if (json_unpack (job->R_redacted,
+                     "{s:{s:F}}",
+                     "execution",
+                      "expiration", &expiration) < 0)
+        BAIL_OUT ("Failed to unpack new expiration from R_redacted");
+    ok (expiration == 100.,
+        "expiration was updated in R_redacted");
+
+    if (!(update = json_pack ("{s:f}", "expiration", 0.0)))
+        BAIL_OUT ("failed to create update");
+    ok (update != NULL,
+        "create resource-update context w/ 0.0 expiration");
+
+    rc = job_apply_resource_updates (job, update);
+    ok (rc ==0,
+        "job_apply_resource_updates() works with 0.0 expiration");
+    json_decref (update);
+
+    if (json_unpack (job->R_redacted,
+                     "{s:{s:F}}",
+                     "execution",
+                      "expiration", &expiration) < 0)
+        BAIL_OUT ("Failed to unpack new expiration from R_redacted");
+    ok (expiration == 0.0,
+        "expiration was updated in R_redacted");
+
+    job_decref (job);
 }
 
 int main (int argc, char *argv[])
@@ -213,6 +866,12 @@ int main (int argc, char *argv[])
 
     test_create ();
     test_create_from_eventlog ();
+    test_create_from_json ();
+    test_subscribe ();
+    test_event_id_cache ();
+    test_event_queue ();
+    test_jobspec_update ();
+    test_resource_update ();
 
     done_testing ();
 }
diff --git a/src/modules/job-manager/test/kill.c b/src/modules/job-manager/test/kill.c
index abee95822f21..5b7256c35346 100644
--- a/src/modules/job-manager/test/kill.c
+++ b/src/modules/job-manager/test/kill.c
@@ -26,8 +26,8 @@ int main (int argc, char **argv)
         "kill_check_signal signum=SIGKILL works");
     ok (kill_check_signal (-1) < 0,
         "kill_check_signal signum=-1 fails");
-    ok (kill_check_signal (SIGRTMAX + 1) < 0,
-        "kill_check_signal signum=SIGRTMAX+1 fails");
+    ok (kill_check_signal (NSIG) < 0,
+        "kill_check_signal signum=NSIG fails");
 
     done_testing ();
 
diff --git a/src/modules/job-manager/test/list.c b/src/modules/job-manager/test/list.c
index 3a7384425044..a215dceaa9b7 100644
--- a/src/modules/job-manager/test/list.c
+++ b/src/modules/job-manager/test/list.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <jansson.h>
 
 #include "src/common/libtap/tap.h"
@@ -46,8 +49,8 @@ int main (int argc, char *argv[])
         "array[0] id=1");
     if (id != 1)
         diag ("id=%d", (int)id);
-    ok (json_object_size (el) == 5,
-        "array[1] size=5");
+    ok (json_object_size (el) == 6,
+        "array[1] size=6");
 
     json_decref (jobs);
     job_decref (job);
diff --git a/src/modules/job-manager/test/restart.c b/src/modules/job-manager/test/restart.c
index ad13c3104e8e..95ac117f3ef5 100644
--- a/src/modules/job-manager/test/restart.c
+++ b/src/modules/job-manager/test/restart.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <jansson.h>
 
 #include "src/common/libtap/tap.h"
diff --git a/src/modules/job-manager/test/submit.c b/src/modules/job-manager/test/submit.c
deleted file mode 100644
index f5e7fa61d442..000000000000
--- a/src/modules/job-manager/test/submit.c
+++ /dev/null
@@ -1,128 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <jansson.h>
-#include <czmq.h>
-#include <flux/core.h>
-#include "src/common/libjob/job_hash.h"
-#include "src/common/libtap/tap.h"
-
-#include "src/modules/job-manager/submit.h"
-
-void single_job_check (zhashx_t *active_jobs)
-{
-    zlist_t *newjobs;
-    json_t *job1;
-    json_t *job2;
-    struct job *job;
-
-    ok (zhashx_size (active_jobs) == 0,
-        "hash is initially empty");
-
-    if (!(newjobs = zlist_new ()))
-        BAIL_OUT ("zlist_new() failed");
-
-    /* good job */
-    if (!(job1 = json_pack ("{s:I s:i s:i s:f s:i}",
-                            "id", 1,
-                            "priority", 10,
-                            "userid", 42,
-                            "t_submit", 1.0,
-                            "flags", 0)))
-        BAIL_OUT ("json_pack() failed");
-    ok (submit_add_one_job (active_jobs, newjobs, job1) == 0,
-        "submit_add_one_job works");
-    ok (zhashx_size (active_jobs) == 1,
-        "hash contains one job");
-    ok ((job = zlist_head (newjobs)) != NULL,
-        "newjobs contains one job");
-    ok (job->id == 1 && job->priority == 10 && job->userid == 42
-        && job->t_submit == 1.0 && job->flags == 0,
-        "struct job was properly decoded");
-
-    /* malformed job */
-    if (!(job2 = json_pack ("{s:I}", "id", 2)))
-        BAIL_OUT ("json_pack() failed");
-    errno = 0;
-    ok (submit_add_one_job (active_jobs, newjobs, job2) < 0 && errno == EPROTO,
-        "submit_add_one job o=(malformed) fails with EPROTO");
-
-    /* resubmit orig job */
-    ok (submit_add_one_job (active_jobs, newjobs, job1) == 0,
-        "submit_add_one_job o=(dup id) works");
-    ok (zhashx_size (active_jobs) == 1,
-        "but hash contains one job");
-    ok (zlist_size (newjobs) == 1,
-        "and newjobs still contains one job");
-
-    /* clean up (batch submit error path) */
-    submit_add_jobs_cleanup (active_jobs, newjobs); // destroys newjobs
-    ok (zhashx_size (active_jobs) == 0,
-        "submit_add_jobs_cleanup removed orig hash entry");
-
-    json_decref (job2);
-    json_decref (job1);
-}
-
-void multi_job_check (zhashx_t *active_jobs)
-{
-
-    zlist_t *newjobs;
-    json_t *jobs;
-
-    ok (zhashx_size (active_jobs) == 0,
-        "hash is initially empty");
-    if (!(jobs = json_pack ("[{s:I s:i s:i s:f s:i},"
-                             "{s:I s:i s:i s:f s:i}]",
-                            "id", 1,
-                            "priority", 10,
-                            "userid", 42,
-                            "t_submit", 1.0,
-                            "flags", 0,
-                            "id", 2,
-                            "priority", 11,
-                            "userid", 43,
-                            "t_submit", 1.1,
-                            "flags", 1)))
-        BAIL_OUT ("json_pack() failed");
-
-    newjobs = submit_add_jobs (active_jobs, jobs);
-    ok (newjobs != NULL,
-        "submit_add_jobs works");
-    ok (zhashx_size (active_jobs) == 2,
-        "hash contains 2 jobs");
-    ok (zlist_size (newjobs) == 2,
-        "newjobs contains 2 jobs");
-    submit_add_jobs_cleanup (active_jobs, newjobs);
-    ok (zhashx_size (active_jobs) == 0,
-        "submit_add_jobs_cleanup removed hash entries");
-
-    json_decref (jobs);
-}
-
-int main (int argc, char *argv[])
-{
-    zhashx_t *active_jobs;
-
-    plan (NO_PLAN);
-
-    if (!(active_jobs = job_hash_create ()))
-        BAIL_OUT ("job_hash_create() failed");
-
-    single_job_check (active_jobs);
-    multi_job_check (active_jobs);
-
-    zhashx_destroy (&active_jobs);
-    done_testing ();
-}
-
-/*
- * vi:ts=4 sw=4 expandtab
- */
diff --git a/src/modules/job-manager/update.c b/src/modules/job-manager/update.c
new file mode 100644
index 000000000000..5f7a15b34f67
--- /dev/null
+++ b/src/modules/job-manager/update.c
@@ -0,0 +1,794 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* update - handle job update requests
+ *
+ * UPDATE REQUEST
+ *
+ * An update request payload consists of a jobid and dictionary of
+ * period-delimited keys to update in that job, e.g.
+ *
+ * { "id": 123456, "updates": {"attributes.system.duration", 3600.}}
+ *
+ * OPERATION
+ *
+ * For each update key, a jobtap callback "job.update.KEY" is executed.
+ * Currently at least one plugin MUST validate the update, therefore
+ * update keys are only supported if there is a plugin that explicitly
+ * allows the update by returning 0 from the 'job.update.*' callback.
+ *
+ * Note: in the future, some keys MAY be explicitly allowed in an allow
+ * list directly within this module.
+ *
+ * If any update in a request fails to be validated, then the request
+ * fails immediately. That is, either all updates are applied or none
+ * are.
+ *
+ * Once all updates are validated by callbacks, then updates as applied
+ * to jobspec are validated by passing an updated jobspec to the
+ * `job.validate` jobtap plugin stack.
+ *
+ * Plugins may request that this validation step be skipped by setting
+ * the 'validated' flag to 1 in the FLUX_PLUGIN_OUT_ARGS of the
+ * `job.update.*` callback. The `job.validate` step will only be skipped
+ * all keys in an update have the validated flag set.
+ *
+ * Plugins may also request a job feasibility check by setting a
+ * 'feasibility' flag to 1 in the FLUX_PLUGIN_OUT_ARGS. If any plugin
+ * requests a feasibility check, then feasibility is run for the proposed
+ * jobspec as a whole.
+ *
+ * A plugin may request additional updates by setting an 'updates' key in
+ * in the plugin out arguments. The updates key follows the same format as
+ * the RFC 21 jobspec-update event and the update request defined here.
+ *
+ * As a special case, if a job is running and a duration update is
+ * being applied, the update service will send a sched.expiration RPC to
+ * the scheduler to ensure the expiration can be adjusted. If this RPC
+ * fails with an error other than ENOSYS, then the update is rejected.
+ *
+ * If all steps above are successful, then a `jobspec-update`event is
+ * posted for the job and a success response sent to the caller.
+ *
+ * If a job is running, and the update results in a change in `R`, then
+ * a resource-update event MAY also be emitted for the job. Currently,
+ * only an update of the expiration in R is supported.
+ *
+ * FUTURE WORK
+ *
+ * - Plugins should also somehow be able to initiate asynchronous work
+ *   before validating an update. There is no support for async plugin
+ *   callbacks in jobtap at this time, though.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <math.h>       /* INFINITY */
+#include <flux/core.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libjob/idf58.h"
+
+#include "update.h"
+#include "job-manager.h"
+#include "jobtap-internal.h"
+#include "event.h"
+
+struct update {
+    struct job_manager *ctx;
+    flux_msg_handler_t **handlers;
+    zlistx_t *pending_requests;
+
+    flux_future_t *kvs_watch_f;
+    double instance_expiration;
+};
+
+struct update_request {
+    void *handle;                 /* zlistx_t handle                       */
+    flux_future_t *feasibility_f; /* feasibility request future            */
+    flux_future_t *expiration_f;  /* sched.expiration request future       */
+    struct update *update;        /* pointer back to update struct         */
+    const flux_msg_t *msg;        /* original update request msg           */
+    struct flux_msg_cred cred;    /* update request credentials            */
+    struct job *job;              /* target job                            */
+    json_t *updates;              /* requested updates object              */
+    unsigned int validate:1;      /* 1: validate updates, 0: no validation */
+};
+
+static void update_request_destroy (struct update_request *req)
+{
+    if (req) {
+        int saved_errno = errno;
+        flux_future_destroy (req->feasibility_f);
+        flux_future_destroy (req->expiration_f);
+        flux_msg_decref (req->msg);
+        job_decref (req->job);
+        free (req);
+        errno = saved_errno;
+    }
+}
+
+/* zlistx_t destructor_fn */
+static void update_request_destructor (void **item)
+{
+    if (item) {
+        struct update_request *req = *item;
+        update_request_destroy (req);
+        *item = NULL;
+    }
+}
+
+static struct update_request *
+update_request_create (struct update *update,
+                       const flux_msg_t *msg,
+                       struct flux_msg_cred cred,
+                       struct job *job,
+                       json_t *updates)
+{
+    struct update_request *req;
+
+    if (!(req = calloc (1, sizeof (*req))))
+        return NULL;
+    req->update = update;
+    req->msg = flux_msg_incref (msg);
+    req->job = job_incref (job);
+    req->cred = cred;
+    req->updates = updates;
+    return req;
+}
+
+static int expiration_from_duration (struct job *job,
+                                     flux_error_t *errp,
+                                     double duration,
+                                     double *expiration)
+{
+    if (duration == 0.)
+        *expiration = 0.;
+    else {
+        /*  Decode starttime of job's current R and add updated duration:
+        */
+        double starttime = -1.0;
+        if (!job->R_redacted
+            || json_unpack (job->R_redacted,
+                            "{s:{s?F}}",
+                            "execution",
+                            "starttime", &starttime) < 0
+            || starttime <= 0.) {
+            errprintf (errp, "unable to get job starttime");
+            return -1;
+        }
+        *expiration = starttime + duration;
+        if (*expiration <= flux_reactor_time ()) {
+            errprintf (errp,
+                       "requested duration places job expiration in the past");
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static int post_resource_updates (struct job_manager *ctx,
+                                  struct job *job,
+                                  json_t *updates,
+                                  flux_error_t *errp)
+{
+    double duration, expiration;
+
+    /*  Updates for a running job may require a corresponding
+     *  resource-update event. Currently this only applies to
+     *  a duration update for a running job.
+     */
+    if (json_unpack (updates,
+                     "{s:F}",
+                     "attributes.system.duration", &duration) < 0)
+        return 0;
+
+    if (expiration_from_duration (job, errp, duration, &expiration) < 0) {
+        return -1;
+    }
+    /*  Post resource-update event to modify expiration:
+     */
+    if (event_job_post_pack (ctx->event,
+                             job,
+                             "resource-update",
+                             0,
+                             "{s:f}",
+                             "expiration", expiration) < 0) {
+        errprintf (errp, "failed to pack resource-update event");
+        return -1;
+    }
+    return 0;
+}
+
+static void post_job_updates (struct job_manager *ctx,
+                              const flux_msg_t *msg,
+                              struct flux_msg_cred cred,
+                              struct job *job,
+                              json_t *updates,
+                              int validate)
+{
+    flux_error_t error;
+
+    /*  If this update was requested by the instance owner, and the
+     *  job owner is not the instance owner, and job validation was
+     *  bypassed (validate != true), then disable future job updates
+     *  as not permitted by marking the job immutable.
+     *
+     *  The reasons for doing this are two-fold:
+     *
+     *  - A future update of an unrelated attribute could fail validation
+     *    due to this attribute update. This could result in a confusing
+     *    error message.
+     *
+     *  - Bypassing validation for individual, previously updated attributes
+     *    could be complex and might open the update process to unintended
+     *    vulnerabilities (e.g. a user update after an instance owner update
+     *    could allow a job access to resources, time limits, etc that are
+     *    not intended for normal users.)
+     */
+    if (!validate
+        && (cred.rolemask & FLUX_ROLE_OWNER)
+        && cred.userid != job->userid) {
+        if (event_job_post_pack (ctx->event,
+                                job,
+                                "set-flags",
+                                0,
+                                "{s:[s]}",
+                                "flags", "immutable") < 0) {
+            errprintf (&error, "failed to set job immutable flag");
+            goto error;
+        }
+
+    }
+
+    /*  All updates have been allowed by plugins and validated as a unit,
+     *  so now emit jobspec-update event.
+     */
+    if (event_job_post_pack (ctx->event,
+                             job,
+                             "jobspec-update",
+                             0,
+                             "O",
+                             updates) < 0) {
+        errprintf (&error, "failed to pack jobspec-update event");
+        goto error;
+    }
+
+    /*  If job is running, then post any necessary resource-update events:
+     */
+    if (job->state & FLUX_JOB_STATE_RUNNING
+        && post_resource_updates (ctx, job, updates, &error) < 0) {
+        goto error;
+    }
+
+    if (flux_respond (ctx->h, msg, NULL) < 0)
+        flux_log_error (ctx->h, "%s: flux_respond", __FUNCTION__);
+    return;
+error:
+    if (flux_respond_error (ctx->h, msg, errno, error.text) < 0)
+        flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+static void feasibility_cb (flux_future_t *f, void *arg)
+{
+    struct update_request *req = arg;
+    if (flux_future_get (f, NULL) < 0) {
+        if (flux_respond_error (req->update->ctx->h,
+                                req->msg,
+                                errno,
+                                future_strerror (f, errno)) < 0)
+            flux_log_error (req->update->ctx->h,
+                            "%s: flux_respond_error",
+                            __FUNCTION__);
+    }
+    else {
+        post_job_updates (req->update->ctx,
+                          req->msg,
+                          req->cred,
+                          req->job,
+                          req->updates,
+                          req->validate);
+    }
+    zlistx_delete (req->update->pending_requests, req->handle);
+}
+
+static void sched_expiration_cb (flux_future_t *f, void *arg)
+{
+    struct update_request *req = arg;
+    if (flux_future_get (f, NULL) < 0 && errno != ENOSYS) {
+        if (flux_respond_error (req->update->ctx->h,
+                                req->msg,
+                                errno,
+                                "scheduler refused expiration update") < 0)
+            flux_log_error (req->update->ctx->h,
+                            "%s: flux_respond_error",
+                            __FUNCTION__);
+    }
+    else {
+        post_job_updates (req->update->ctx,
+                          req->msg,
+                          req->cred,
+                          req->job,
+                          req->updates,
+                          req->validate);
+    }
+    zlistx_delete (req->update->pending_requests, req->handle);
+}
+
+static struct update_request *
+pending_request_create (struct update *update,
+                        const flux_msg_t *msg,
+                        struct flux_msg_cred cred,
+                        struct job *job,
+                        json_t *updates,
+                        int validate)
+{
+    struct update_request *req = NULL;
+    if (!(req = update_request_create (update, msg, cred, job, updates))
+        || !(req->handle = zlistx_add_end (update->pending_requests, req)))
+        goto error;
+    req->validate = validate;
+    return req;
+error:
+    update_request_destroy (req);
+    errno = ENOMEM;
+    return NULL;
+}
+
+static int update_feasibility_check (struct update *update,
+                                     const flux_msg_t *msg,
+                                     struct flux_msg_cred cred,
+                                     struct job *job,
+                                     json_t *updates,
+                                     int validate)
+{
+    json_t *jobspec = NULL;
+    struct update_request *req = NULL;
+    flux_future_t *f = NULL;
+
+    if (!(jobspec = job_jobspec_with_updates (job, updates))
+        || !(req = pending_request_create (update,
+                                           msg,
+                                           cred,
+                                           job,
+                                           updates,
+                                           validate))
+        || !(f = flux_rpc_pack (update->ctx->h,
+                                "feasibility.check",
+                                0,
+                                0,
+                                "{s:O}",
+                                "jobspec", jobspec))
+        || flux_future_then (f, -1., feasibility_cb, req) < 0)
+        goto error;
+    req->feasibility_f = f;
+    json_decref (jobspec);
+    return 0;
+error:
+    json_decref (jobspec);
+    update_request_destroy (req);
+    flux_future_destroy (f);
+    return -1;
+
+}
+
+static int sched_expiration_check (struct update *update,
+                                   flux_error_t *errp,
+                                   const flux_msg_t *msg,
+                                   struct flux_msg_cred cred,
+                                   struct job *job,
+                                   json_t *updates,
+                                   int validate)
+{
+    struct update_request *req = NULL;
+    flux_future_t *f = NULL;
+    double expiration, duration;
+
+    if (json_unpack (updates,
+                     "{s:F}",
+                     "attributes.system.duration", &duration) < 0) {
+        errprintf (errp, "failed to unpack attributes.system.duration");
+        return -1;
+    }
+    if (expiration_from_duration (job, errp, duration, &expiration) < 0)
+        return -1;
+
+    if (!(req = pending_request_create (update,
+                                        msg,
+                                        cred,
+                                        job,
+                                        updates,
+                                        validate))
+        || !(f = flux_rpc_pack (update->ctx->h,
+                                "sched.expiration",
+                                0,
+                                0,
+                                "{s:I s:f}",
+                                "id", job->id,
+                                "expiration", expiration))
+        || flux_future_then (f, -1., sched_expiration_cb, req) < 0) {
+        errprintf (errp,
+                   "failed to send sched.expiration rpc: %s",
+                   strerror (errno));
+        goto error;
+    }
+    req->expiration_f = f;
+    return 0;
+error:
+    update_request_destroy (req);
+    flux_future_destroy (f);
+    return -1;
+}
+
+
+static void update_job (struct update *update,
+                        const flux_msg_t *msg,
+                        struct flux_msg_cred cred,
+                        struct job *job,
+                        json_t *updates)
+{
+    char *error = NULL;
+    const char *key;
+    json_t *value;
+    int validate = 0; /* validation of result necessary */
+    int feasibility = 0; /* feasibilty check necessary */
+    json_t *additional_updates = NULL;
+    struct job_manager *ctx = update->ctx;
+
+    /*  Loop through one or more proposed updates in `updates` object
+     *  and call `job.update.<key>` job plugin(s) to validate each
+     *  update.
+     */
+    json_object_foreach (updates, key, value) {
+        int needs_validation = 1;
+        int require_feasibility = 0;
+        if (jobtap_job_update (ctx->jobtap,
+                               cred,
+                               job,
+                               key,
+                               value,
+                               &needs_validation,
+                               &require_feasibility,
+                               &additional_updates,
+                               &error) < 0)
+            goto error;
+        /*  If any jobspec key needs further validation, then all
+         *  keys will be validated at the same time. This means a key
+         *  that might not need further validation when updated alone
+         *  may need to be validated when paired with other keys in a
+         *  single update:
+         */
+        if (needs_validation)
+            validate = 1;
+        /*  Similarly, if any key requires a feasibility check, then
+         *  request feasibilty on the update as a whole.
+         */
+        if (require_feasibility)
+            feasibility = 1;
+    }
+    if (additional_updates
+        && json_object_update (updates, additional_updates) < 0) {
+        error = strdup ("unable to apply additional required updates");
+        goto error;
+    }
+    if (validate
+        && jobtap_validate_updates (ctx->jobtap,
+                                    job,
+                                    updates,
+                                    &error) < 0)
+        goto error;
+
+    if (feasibility)
+        update_feasibility_check (update, msg, cred, job, updates, validate);
+    else if (job->state & FLUX_JOB_STATE_RUNNING
+             && json_object_get (updates, "attributes.system.duration")) {
+        flux_error_t ferror;
+        if (sched_expiration_check (update,
+                                    &ferror,
+                                    msg,
+                                    cred,
+                                    job,
+                                    updates,
+                                    validate) < 0) {
+            error = strdup (ferror.text);
+            goto error;
+        }
+    }
+    else
+        post_job_updates (ctx, msg, cred, job, updates, validate);
+
+    json_decref (additional_updates);
+    return;
+error:
+    if (flux_respond_error (ctx->h, msg, EINVAL, error) < 0)
+        flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+    json_decref (additional_updates);
+    free (error);
+}
+
+static void update_handle_request (flux_t *h,
+                                   flux_msg_handler_t *mh,
+                                   const flux_msg_t *msg,
+                                   void *arg)
+{
+    struct update *update = arg;
+    struct job_manager *ctx = update->ctx;
+    flux_jobid_t id;
+    struct job *job;
+    json_t *updates;
+    struct flux_msg_cred cred;
+    const char *errstr = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:o}",
+                             "id", &id,
+                             "updates", &updates) < 0)
+        goto error;
+
+    /*  Validate updates object, currently all updates MUST
+     *  start with `attributes.`:
+     */
+    if (!validate_jobspec_updates (updates)) {
+        errstr = "one or more jobspec updates are invalid";
+        errno = EINVAL;
+        goto error;
+    }
+    /*  Verify jobid exists and is not inactive
+     */
+    if (!(job = zhashx_lookup (ctx->active_jobs, &id))) {
+        if (!(job = zhashx_lookup (ctx->inactive_jobs, &id))) {
+            errstr = "unknown job id";
+            errno = ENOENT;
+        }
+        else {
+            errstr = "job is inactive";
+            errno = EINVAL;
+        }
+        goto error;
+    }
+    /*  Fetch the credential from this message and ensure the user
+     *  has authorization to update this job.
+     */
+    if (flux_msg_get_cred (msg, &cred) < 0
+        || flux_msg_cred_authorize (cred, job->userid) < 0) {
+        errstr = "guests may only update their own jobs";
+        goto error;
+    }
+    if (job->immutable && !(cred.rolemask & FLUX_ROLE_OWNER)) {
+        errstr = "job is immutable due to previous instance owner update";
+        errno = EPERM;
+        goto error;
+    }
+    /*  Process the update request. Response will be handled in update_job().
+     */
+    update_job (update, msg, cred, job, updates);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+static void send_error_responses (struct update *update)
+{
+    struct update_request *req = zlistx_first (update->pending_requests);
+    while (req) {
+        if (flux_respond_error (update->ctx->h,
+                                req->msg,
+                                EAGAIN,
+                                "job manager is shutting down") < 0)
+            flux_log_error (update->ctx->h,
+                            "%s: error responding to "
+                            "job-manager.update request",
+                            __FUNCTION__);
+        req = zlistx_next (update->pending_requests);
+    }
+}
+
+static void update_expiration_from_lookup_response (struct update *update,
+                                                    flux_future_t *f)
+{
+    flux_t *h = update->ctx->h;
+    const char *R;
+    json_t *o = NULL;
+    json_error_t error;
+
+    error.text[0] = '\0';
+
+    if (flux_kvs_lookup_get (f, &R) < 0
+        || !(o = json_loads (R, 0, NULL))
+        || json_unpack_ex (o, &error, 0,
+                           "{s:{s:F}}",
+                           "execution",
+                            "expiration", &update->instance_expiration) < 0)
+        flux_log (h,
+                  LOG_ERR,
+                  "failed to unpack current instance expiration: %s",
+                  error.text);
+    json_decref (o);
+}
+
+static inline double expiration_diff (double old, double new)
+{
+    /*  If the old expiration was 0. (unlimited) then return -inf since
+     *  this best represents the reduction of expiration from unlimited.
+     *  If new expiration is unlimited, then return +inf.
+     *  O/w, return difference between new and old.
+     */
+    if (old == 0.)
+        return -INFINITY;
+    if (new == 0.)
+        return INFINITY;
+    return new - old;
+}
+
+/*  An update to resource.R has occurred. Adjust expiration of all running
+ *  jobs where no duration is set in jobspec, but the job currently has a set
+ *  expiration. This implies the expiration was set automatically by the
+ *  scheduler and needs an update.
+ *
+ *  The motivating case here is an administrative extension of a batch or
+ *  alloc job time limit. This code extends that expiration update to all
+ *  running jobs, which otherwise may have their expiration set to the
+ *  previous instance time limit.
+ */
+
+static void resource_update_cb (flux_future_t *f, void *arg)
+{
+    struct update *update = arg;
+    flux_t *h = update->ctx->h;
+    struct job *job;
+    double old_expiration = update->instance_expiration;
+
+    update_expiration_from_lookup_response (update, f);
+    flux_future_reset (f);
+
+    /*  If this is the first successful update, or there are no
+     *  running jobs, or the expiration was not updated, then there is
+     *  nothing left to do.
+     */
+    if (old_expiration == -1.
+        || fabs (update->instance_expiration - old_expiration) < 1.e-5
+        || update->ctx->running_jobs == 0)
+        return;
+
+    flux_log (h,
+              LOG_INFO,
+              "resource expiration updated from %.2f to %.2f (%+.6g)",
+              old_expiration,
+              update->instance_expiration,
+              expiration_diff (old_expiration, update->instance_expiration));
+
+
+    /*  Otherwise, check each running job to determine if an adjustment
+     *  of its expiration is required:
+     */
+    job = zhashx_first (update->ctx->active_jobs);
+    while (job) {
+        if (job->state == FLUX_JOB_STATE_RUN) {
+            double expiration = -1.;
+            double duration = 0.;
+            /*
+             *  Get current job expiration (if set) and jobspec duration.
+             *  Assume the expiration job of the job needs to be updated
+             *  only if and expiration was set for the job _and_ the job
+             *  duration was unset or 0. This indicates that the expiration
+             *  was likely automatically set by the scheduler based on
+             *  the instance expiration (which is now being updated).
+             *
+             */
+            if (json_unpack (job->R_redacted,
+                            "{s:{s?F}}",
+                            "execution",
+                             "expiration", &expiration) < 0
+                || json_unpack (job->jobspec_redacted,
+                                "{s:{s:{s?F}}}",
+                                "attributes",
+                                 "system",
+                                  "duration", &duration) < 0) {
+                flux_log (h,
+                          LOG_ERR,
+                          "failed to unpack job %s data for expiration update",
+                          idf58 (job->id));
+            }
+            /*  Job needs an update if no or unlimited duration specified
+             *  in jobspec (duration == 0.) but an expiration was set in R
+             *  (expiration >= 0.):
+             */
+            if (expiration >= 0 && duration == 0.) {
+                flux_log (h,
+                          LOG_INFO,
+                          "updated expiration of %s from %.2f to %.2f (%+.6g)",
+                          idf58 (job->id),
+                          expiration,
+                          update->instance_expiration,
+                          expiration_diff (expiration,
+                                           update->instance_expiration));
+                if (event_job_post_pack (update->ctx->event,
+                                         job,
+                                         "resource-update",
+                                         0,
+                                         "{s:f}",
+                                         "expiration",
+                                         update->instance_expiration) < 0)
+                    flux_log (h,
+                              LOG_ERR,
+                              "failed to pack resource-update event");
+            }
+        }
+        job = zhashx_next (update->ctx->active_jobs);
+    }
+}
+
+void update_ctx_destroy (struct update *update)
+{
+    if (update) {
+        int saved_errno = errno;
+        send_error_responses (update);
+        flux_kvs_lookup_cancel (update->kvs_watch_f);
+        flux_future_destroy (update->kvs_watch_f);
+        flux_msg_handler_delvec (update->handlers);
+        zlistx_destroy (&update->pending_requests);
+        free (update);
+        errno = saved_errno;
+    }
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "job-manager.update",
+        update_handle_request,
+        FLUX_ROLE_USER
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+struct update *update_ctx_create (struct job_manager *ctx)
+{
+    struct update *update;
+
+    if (!(update = calloc (1, sizeof (*update))))
+        return NULL;
+    update->ctx = ctx;
+    if (flux_msg_handler_addvec (ctx->h, htab, update, &update->handlers) < 0)
+        goto error;
+    if (!(update->pending_requests = zlistx_new ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    zlistx_set_destructor (update->pending_requests,
+                           update_request_destructor);
+
+    /*  Watch resource.R in KVS for updates
+     */
+    update->kvs_watch_f = flux_kvs_lookup (ctx->h,
+                                           NULL,
+                                           FLUX_KVS_WATCH | FLUX_KVS_WAITCREATE,
+                                           "resource.R");
+    if (!update->kvs_watch_f
+        || flux_future_then (update->kvs_watch_f,
+                             -1.,
+                             resource_update_cb,
+                             update) < 0) {
+        flux_log_error (ctx->h, "failed to setup watch on resource.R");
+        goto error;
+    }
+    update->instance_expiration = -1.;
+    return update;
+error:
+    update_ctx_destroy (update);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-manager/update.h b/src/modules/job-manager/update.h
new file mode 100644
index 000000000000..8ef4545ba960
--- /dev/null
+++ b/src/modules/job-manager/update.h
@@ -0,0 +1,22 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_UPDATE_H
+#define _FLUX_JOB_MANAGER_UPDATE_H
+
+#include "job-manager.h"
+
+struct update *update_ctx_create (struct job_manager *ctx);
+void update_ctx_destroy (struct update *update);
+
+#endif /* ! _FLUX_JOB_MANAGER_UPDATE_H */
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-manager/urgency.c b/src/modules/job-manager/urgency.c
new file mode 100644
index 000000000000..dc36b034edfc
--- /dev/null
+++ b/src/modules/job-manager/urgency.c
@@ -0,0 +1,119 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* urgency - adjust job urgency
+ *
+ * Purpose:
+ *   Support flux job urgency command for adjusting job urgency
+ *   after submission.  Guests can reduce their jobs' urgency, or increase
+ *   up to the default urgency.
+ *
+ * Input:
+ * - job id
+ * - new urgency
+ *
+ * Output:
+ * - old urgency
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "job.h"
+#include "event.h"
+#include "alloc.h"
+#include "job-manager.h"
+
+#include "urgency.h"
+
+#define MAXOF(a,b)   ((a)>(b)?(a):(b))
+
+void urgency_handle_request (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
+{
+    struct job_manager *ctx = arg;
+    struct flux_msg_cred cred;
+    flux_jobid_t id;
+    struct job *job;
+    int urgency, orig_urgency;
+    const char *errstr = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:i}",
+                             "id", &id,
+                             "urgency", &urgency) < 0
+        || flux_msg_get_cred (msg, &cred) < 0)
+        goto error;
+    if (urgency < FLUX_JOB_URGENCY_MIN
+        || urgency > FLUX_JOB_URGENCY_MAX) {
+        errstr = "urgency value is out of range";
+        errno = EINVAL;
+        goto error;
+    }
+    if (!(job = zhashx_lookup (ctx->active_jobs, &id))) {
+        if (!(job = zhashx_lookup (ctx->inactive_jobs, &id)))
+            errstr = "unknown job";
+        else
+            errstr = "job is inactive";
+        errno = EINVAL;
+        goto error;
+    }
+    /* Security: guests can only adjust jobs that they submitted.
+     */
+    if (flux_msg_cred_authorize (cred, job->userid) < 0) {
+        errstr = "guests can only reprioritize their own jobs";
+        goto error;
+    }
+    /* Security: guests can only reduce urgency, or increase up to default.
+     */
+    if (!(cred.rolemask & FLUX_ROLE_OWNER)
+        && urgency > MAXOF (FLUX_JOB_URGENCY_DEFAULT, job->urgency)) {
+        errstr = "guests can only adjust urgency <= default";
+        errno = EPERM;
+        goto error;
+    }
+    if (job->has_resources) {
+        errstr = "urgency cannot be changed once resources are allocated";
+        errno = EINVAL;
+        goto error;
+    }
+    /* Post event: this will update job->urgency, which will then result
+     *  in a call to recalculate priority, which reprioritize job if there
+     *  was a priority change.
+     */
+    orig_urgency = job->urgency;
+    if (event_job_post_pack (ctx->event,
+                             job,
+                             "urgency",
+                             0,
+                             "{s:I s:i}",
+                             "userid", (json_int_t) cred.userid,
+                             "urgency", urgency) < 0)
+        goto error;
+    if (flux_respond_pack (h, msg, "{s:i}", "old_urgency", orig_urgency) < 0) {
+        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+        goto error;
+    }
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-manager/urgency.h b/src/modules/job-manager/urgency.h
new file mode 100644
index 000000000000..c9f59a08789f
--- /dev/null
+++ b/src/modules/job-manager/urgency.h
@@ -0,0 +1,29 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_JOB_MANAGER_URGENCY_H
+#define _FLUX_JOB_MANAGER_URGENCY_H
+
+#include <flux/core.h>
+#include "job-manager.h"
+
+/* Handle a 'urgency' request - job urgency adjustment
+ */
+void urgency_handle_request (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg);
+
+
+#endif /* ! _FLUX_JOB_MANAGER_URGENCY_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/job-manager/wait.c b/src/modules/job-manager/wait.c
index a80ebf7a49f8..aae6c73079c8 100644
--- a/src/modules/job-manager/wait.c
+++ b/src/modules/job-manager/wait.c
@@ -25,7 +25,7 @@
  *
  * If the target job is active when the wait request is received,
  * the request is tacked onto the 'struct job' and processed upon
- * transtion to INACTIVE state.  If the target waitable job has already
+ * transition to INACTIVE state.  If the target waitable job has already
  * transitioned to INACTIVE, it is found in the wait->zombies hash
  * and the request is processed immediately.
  *
@@ -49,12 +49,16 @@
 #include "config.h"
 #endif
 #include <stdbool.h>
-#include <czmq.h>
+#include <assert.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
 #include "src/common/libeventlog/eventlog.h"
 #include "src/common/libjob/job_hash.h"
+#include "src/common/libjob/idf58.h"
+#include "ccan/str/str.h"
 
 #include "drain.h"
 #include "submit.h"
@@ -66,13 +70,12 @@ struct waitjob {
     zhashx_t *zombies;
     int waiters; // count of waiters blocked on specific active jobs
     int waitables; // count of active waitable jobs
-    zlistx_t *requests; // requests to wait in FLUX_JOBID_ANY
+    struct flux_msglist *requests; // requests to wait in FLUX_JOBID_ANY
 };
 
 static int decode_job_result (struct job *job,
                               bool *success,
-                              char *errbuf,
-                              int errbufsz)
+                              flux_error_t *errp)
 {
     const char *name;
     json_t *context;
@@ -84,53 +87,32 @@ static int decode_job_result (struct job *job,
 
     /* Exception - set errbuf=description, set success=false
      */
-    if (!strcmp (name, "exception")) {
+    if (streq (name, "exception")) {
         const char *type;
-        const char *note = NULL;
+        const char *note;
 
         if (json_unpack (context,
-                         "{s:s s?:s}",
-                         "type",
-                         &type,
-                         "note",
-                         &note) < 0)
+                         "{s:s s:s}",
+                         "type", &type,
+                         "note", &note) < 0)
             return -1;
-        (void)snprintf (errbuf,
-                        errbufsz,
-                        "Fatal exception type=%s %s",
-                        type,
-                        note ? note : "");
+        errprintf (errp,
+                   "Fatal exception type=%s %s",
+                   type,
+                   note);
         *success = false;
     }
     /* Shells exited - set errbuf=decoded status byte,
      * set success=true if all shells exited with 0, otherwise false.
      */
-    else if (!strcmp (name, "finish")) {
+    else if (streq (name, "finish")) {
         int status;
-
         if (json_unpack (context, "{s:i}", "status", &status) < 0)
             return -1;
-        if (WIFSIGNALED (status)) {
-            (void)snprintf (errbuf,
-                            errbufsz,
-                            "task(s) %s",
-                            strsignal (WTERMSIG (status)));
-            *success = false;
-        }
-        else if (WIFEXITED (status)) {
-            (void)snprintf (errbuf,
-                            errbufsz,
-                            "task(s) exited with exit code %d",
-                            WEXITSTATUS (status));
-            *success = WEXITSTATUS (status) == 0 ? true : false;
-        }
-        else {
-            (void)snprintf (errbuf,
-                            errbufsz,
-                            "unexpected wait(2) status %d",
-                            status);
+        if (flux_job_waitstatus_to_exitcode (status, errp) != 0)
             *success = false;
-        }
+        else
+            *success = true;
     }
     else
         return -1;
@@ -144,30 +126,27 @@ static void wait_respond (struct waitjob *wait,
                           struct job *job)
 {
     flux_t *h = wait->ctx->h;
-    char errbuf[1024];
+    flux_error_t error;
     bool success;
 
-    if (decode_job_result (job, &success, errbuf, sizeof (errbuf)) < 0) {
+    if (decode_job_result (job, &success, &error) < 0) {
         flux_log (h,
                   LOG_ERR,
-                  "wait_respond id=%ju: result decode failure",
-                  (uintmax_t)job->id);
+                  "wait_respond id=%s: result decode failure",
+                  idf58 (job->id));
         goto error;
     }
     if (flux_respond_pack (h,
                            msg,
                            "{s:I s:b s:s}",
-                           "id",
-                           job->id,
-                           "success",
-                           success ? 1 : 0,
-                           "errstr",
-                           errbuf) < 0)
-        flux_log_error (h, "wait_respond id=%ju", (uintmax_t)job->id);
+                           "id", job->id,
+                           "success", success ? 1 : 0,
+                           "errstr", error.text) < 0)
+        flux_log_error (h, "wait_respond id=%s", idf58 (job->id));
     return;
 error:
     if (flux_respond_error (h, msg, errno, "Flux job wait internal error") < 0)
-        flux_log_error (h, "wait_respond id=%ju", (uintmax_t)job->id);
+        flux_log_error (h, "wait_respond id=%s", idf58 (job->id));
 }
 
 /* Callback from event_job_action().  The 'job' has entered INACTIVE state.
@@ -187,9 +166,9 @@ void wait_notify_inactive (struct waitjob *wait, struct job *job)
         job->waiter = NULL;
         wait->waiters--;
     }
-    else if ((req = zlistx_detach (wait->requests, NULL))) {
+    else if ((req = flux_msglist_first (wait->requests))) {
         wait_respond (wait, req, job);
-        flux_msg_decref (req);
+        flux_msglist_delete (wait->requests);
     }
     else {
         if (zhashx_insert (wait->zombies, &job->id, job) < 0) // increfs job
@@ -232,12 +211,8 @@ static void wait_rpc (flux_t *h,
         /* Enqueue request until a waitable job transitions to inactive.
          */
         else {
-            if (zlistx_add_end (wait->requests,
-                                (void *)flux_msg_incref (msg)) < 0) {
-                flux_msg_decref (msg);
-                errno = ENOMEM;
+            if (flux_msglist_append (wait->requests, msg) < 0)
                 goto error;
-            }
         }
     }
     else {
@@ -275,8 +250,8 @@ static void wait_rpc (flux_t *h,
      * (1) wait on specific ID increased wait->waiters, or
      * (2) wait on FLUX_JOBID_ANY increased wait->requests.
      */
-    if (zlistx_size (wait->requests) + wait->waiters > wait->waitables) {
-        const flux_msg_t *req = zlistx_last (wait->requests);
+    if (flux_msglist_count (wait->requests) + wait->waiters > wait->waitables) {
+        const flux_msg_t *req = flux_msglist_last (wait->requests);
 
         if (req) {
             if (flux_respond_error (h,
@@ -284,8 +259,7 @@ static void wait_rpc (flux_t *h,
                                     ECHILD,
                                     "there are no more waitable jobs") < 0)
                 flux_log_error (h, "%s: flux_respond_error", __func__);
-            zlistx_detach_cur (wait->requests);
-            flux_msg_decref (req);
+            flux_msglist_delete (wait->requests);
         }
     }
     return;
@@ -299,46 +273,28 @@ static void wait_rpc (flux_t *h,
 
 /* A client has disconnected.  Destroy any waiters registered by that client.
  */
-static void disconnect_rpc (flux_t *h,
-                            flux_msg_handler_t *mh,
-                            const flux_msg_t *msg,
-                            void *arg)
+void wait_disconnect_rpc (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
 {
     struct job_manager *ctx = arg;
     struct waitjob *wait = ctx->wait;
-    char *sender;
-    char *w_sender;
     struct job *job;
-    const flux_msg_t *req;
 
-    if (flux_msg_get_route_first (msg, &sender) < 0)
-        return;
     job = zhashx_first (ctx->active_jobs);
     while (job && wait->waiters > 0) {
         if (job->waiter) {
-            if (flux_msg_get_route_first (job->waiter, &w_sender) == 0) {
-                if (!strcmp (sender, w_sender)) {
-                    flux_msg_decref (job->waiter);
-                    job->waiter = NULL;
-                    wait->waiters--;
-                }
+            if (flux_msg_route_match_first (job->waiter, msg)) {
+                flux_msg_decref (job->waiter);
+                job->waiter = NULL;
+                wait->waiters--;
             }
-            free (w_sender);
         }
         job = zhashx_next (ctx->active_jobs);
     }
-    req = zlistx_first (wait->requests);
-    while (req) {
-        if (flux_msg_get_route_first (req, &w_sender) == 0) {
-            if (!strcmp (sender, w_sender)) {
-                zlistx_detach_cur (wait->requests);
-                flux_msg_decref (req);
-            }
-            free (w_sender);
-        }
-        req = zlistx_next (wait->requests);
-    }
-    free (sender);
+
+    flux_msglist_disconnect (wait->requests, msg);
 }
 
 struct job *wait_zombie_first (struct waitjob *wait)
@@ -368,7 +324,7 @@ void wait_ctx_destroy (struct waitjob *wait)
 
         /* Iterate through active jobs, sending ENOSYS response to
          * any pending wait requests, indicating that the module is unloading.
-         * Use wait->waiters count to avoid unncessary scanning.
+         * Use wait->waiters count to avoid unnecessary scanning.
          */
         job = zhashx_first (wait->ctx->active_jobs);
         while (job && wait->waiters > 0) {
@@ -387,11 +343,12 @@ void wait_ctx_destroy (struct waitjob *wait)
         if (wait->requests) {
             const flux_msg_t *msg;
 
-            while ((msg = zlistx_detach (wait->requests, NULL))) {
+            while ((msg = flux_msglist_first (wait->requests))) {
                 respond_unloading (h, msg);
-                flux_msg_decref (msg);
+                flux_msglist_delete (wait->requests);
+                msg = flux_msglist_next (wait->requests);
             }
-            zlistx_destroy (&wait->requests);
+            flux_msglist_destroy (wait->requests);
         }
 
         zhashx_destroy (&wait->zombies);
@@ -407,12 +364,6 @@ static const struct flux_msg_handler_spec htab[] = {
         .cb = wait_rpc,
         .rolemask = 0
     },
-    {
-        .typemask = FLUX_MSGTYPE_REQUEST,
-        .topic_glob = "job-manager.disconnect",
-        .cb = disconnect_rpc,
-        .rolemask = 0
-    },
     FLUX_MSGHANDLER_TABLE_END,
 };
 
@@ -429,7 +380,7 @@ struct waitjob *wait_ctx_create (struct job_manager *ctx)
     zhashx_set_destructor (wait->zombies, job_destructor);
     zhashx_set_duplicator (wait->zombies, job_duplicator);
 
-    if (!(wait->requests = zlistx_new ()))
+    if (!(wait->requests = flux_msglist_create ()))
         goto error;
 
     if (flux_msg_handler_addvec (ctx->h, htab, ctx, &wait->handlers) < 0)
diff --git a/src/modules/job-manager/wait.h b/src/modules/job-manager/wait.h
index 772590109d14..229378d2b9f4 100644
--- a/src/modules/job-manager/wait.h
+++ b/src/modules/job-manager/wait.h
@@ -22,6 +22,11 @@ void wait_notify_active (struct waitjob *wait, struct job *job);
 struct waitjob *wait_ctx_create (struct job_manager *ctx);
 void wait_ctx_destroy (struct waitjob *wait);
 
+void wait_disconnect_rpc (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg);
+
 struct job *wait_zombie_first (struct waitjob *wait);
 struct job *wait_zombie_next (struct waitjob *wait);
 
diff --git a/src/modules/kvs-watch/Makefile.am b/src/modules/kvs-watch/Makefile.am
deleted file mode 100644
index 350dc09a6a26..000000000000
--- a/src/modules/kvs-watch/Makefile.am
+++ /dev/null
@@ -1,26 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) $(FLUX_SECURITY_CFLAGS) $(YAMLCPP_CFLAGS)
-
-fluxmod_LTLIBRARIES = kvs-watch.la
-
-kvs_watch_la_SOURCES = \
-	kvs-watch.c
-
-
-kvs_watch_la_LDFLAGS = $(fluxmod_ldflags) -module
-kvs_watch_la_LIBADD = $(fluxmod_libadd) \
-	$(top_builddir)/src/common/libkvs/libkvs.la \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(top_builddir)/src/common/libflux-optparse.la \
-	$(ZMQ_LIBS)
diff --git a/src/modules/kvs-watch/kvs-watch.c b/src/modules/kvs-watch/kvs-watch.c
index 293298c1c19d..5f3e0609d576 100644
--- a/src/modules/kvs-watch/kvs-watch.c
+++ b/src/modules/kvs-watch/kvs-watch.c
@@ -13,20 +13,23 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
 #include <jansson.h>
+#include <assert.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libkvs/treeobj.h"
 #include "src/common/libkvs/kvs_util_private.h"
 #include "src/common/libutil/blobref.h"
+#include "src/common/libcontent/content.h"
+#include "src/common/libutil/errprintf.h"
 
 /* State for one watcher */
 struct watcher {
     const flux_msg_t *request;  // request message
     struct flux_msg_cred cred;  // request cred
     int rootseq;                // last root sequence number sent
-    bool cancelled;             // true if watcher has been cancelled
+    bool canceled;              // true if watcher has been canceled
     bool mute;                  // true if response should be suppressed
     bool responded;             // true if watcher has responded atleast once
     bool initial_rpc_sent;      // flag is initial watch rpc sent
@@ -36,10 +39,14 @@ struct watcher {
     char *key;                  // lookup key
     int flags;                  // kvs_lookup flags
     zlist_t *lookups;           // list of futures, in commit order
+    zlist_t *loads;             // list of futures, content loads in ref order
 
     struct ns_monitor *nsm;     // back pointer for removal
     json_t *prev;               // previous watch value for KVS_WATCH_FULL/UNIQ
-    int append_offset;          // offset for KVS_WATCH_APPEND
+    bool index_valid;           // flag if prev_start_index/prev_end_index set
+    int prev_start_index;       // previous start index loaded
+    int prev_end_index;         // previous end index loaded
+    void *handle;               // zlistx_t handle
 };
 
 /* Current KVS root.
@@ -62,7 +69,7 @@ struct ns_monitor {
     int fatal_errnum;           // non-skippable error pending for all watchers
     int errnum;                 // if non-zero, error pending for all watchers
     struct watch_ctx *ctx;      // back-pointer to watch_ctx
-    zlist_t *watchers;          // list of watchers of this namespace
+    zlistx_t *watchers;         // list of watchers of this namespace
     char *topic;                // topic string for subscription
     bool subscribed;            // subscription active
     flux_future_t *getrootf;    // initial getroot future
@@ -88,13 +95,20 @@ static void watcher_destroy (struct watcher *w)
                 flux_future_destroy (f);
             zlist_destroy (&w->lookups);
         }
+        if (w->loads) {
+            flux_future_t *f;
+            while ((f = zlist_pop (w->loads)))
+                flux_future_destroy (f);
+            zlist_destroy (&w->loads);
+        }
         json_decref (w->prev);
         free (w);
         errno = saved_errno;
     }
 }
 
-static struct watcher *watcher_create (const flux_msg_t *msg, const char *key,
+static struct watcher *watcher_create (const flux_msg_t *msg,
+                                       const char *key,
                                        int flags)
 {
     struct watcher *w;
@@ -108,6 +122,8 @@ static struct watcher *watcher_create (const flux_msg_t *msg, const char *key,
         goto error;
     if (!(w->lookups = zlist_new ()))
         goto error_nomem;
+    if (!(w->loads = zlist_new ()))
+        goto error_nomem;
     w->flags = flags;
     w->rootseq = -1;
     return w;
@@ -130,7 +146,8 @@ static void commit_destroy (struct commit *commit)
     }
 }
 
-static struct commit *commit_create (const char *rootref, int rootseq,
+static struct commit *commit_create (const char *rootref,
+                                     int rootseq,
                                      json_t *keys)
 {
     struct commit *commit = calloc (1, sizeof (*commit));
@@ -151,12 +168,7 @@ static void namespace_destroy (struct ns_monitor *nsm)
     if (nsm) {
         int saved_errno = errno;
         commit_destroy (nsm->commit);
-        if (nsm->watchers) {
-            struct watcher *w;
-            while ((w = zlist_pop (nsm->watchers)))
-                watcher_destroy (w);
-            zlist_destroy (&nsm->watchers);
-        }
+        zlistx_destroy (&nsm->watchers);
         if (nsm->subscribed)
             (void)flux_event_unsubscribe (nsm->ctx->h, nsm->topic);
         free (nsm->topic);
@@ -167,14 +179,24 @@ static void namespace_destroy (struct ns_monitor *nsm)
     }
 }
 
+static void watcher_destructor (void **item)
+{
+    if (item) {
+        struct watcher *w = *item;
+        watcher_destroy (w);
+        *item = NULL;
+    }
+}
+
 static struct ns_monitor *namespace_create (struct watch_ctx *ctx,
                                             const char *ns)
 {
     struct ns_monitor *nsm = calloc (1, sizeof (*nsm));
     if (!nsm)
         return NULL;
-    if (!(nsm->watchers = zlist_new ()))
+    if (!(nsm->watchers = zlistx_new ()))
         goto error;
+    zlistx_set_destructor (nsm->watchers, watcher_destructor);
     if (!(nsm->ns_name = strdup (ns)))
         goto error;
     /* We are subscribing to the kvs.namespace-<NS> substring.
@@ -208,40 +230,135 @@ static struct ns_monitor *namespace_create (struct watch_ctx *ctx,
     return NULL;
 }
 
-/* Helper for watcher_respond - is key a member of array?
- * N.B. array 'a' can be NULL
+/* Helper for watcher_respond - is key a member of object?
+ * N.B. object 'o' can be NULL
  */
-static bool array_match (json_t *a, const char *key)
+static bool key_match (json_t *o, const char *key)
 {
-    size_t index;
-    json_t *value;
-
-    json_array_foreach (a, index, value) {
-        const char *s = json_string_value (value);
-        if (s && !strcmp (s, key))
-            return true;
-    }
+    if (o && json_object_get (o, key))
+        return true;
     return false;
 }
 
 static void watcher_cleanup (struct ns_monitor *nsm, struct watcher *w)
 {
     /* wait for all in flight lookups to complete before destroying watcher */
-    if (zlist_size (w->lookups) == 0) {
-        zlist_remove (nsm->watchers, w);
-        watcher_destroy (w);
-    }
+    if (zlist_size (w->lookups) == 0 && zlist_size (w->loads) == 0)
+        zlistx_delete (nsm->watchers, w->handle);
     /* if nsm->getrootf, destroy when getroot_continuation completes */
-    if (zlist_size (nsm->watchers) == 0
+    if (zlistx_size (nsm->watchers) == 0
         && !nsm->getrootf)
         zhash_delete (nsm->ctx->namespaces, nsm->ns_name);
 }
 
+static void handle_load_response (flux_future_t *f, struct watcher *w)
+{
+    flux_t *h = flux_future_get_flux (f);
+    const void *data;
+    size_t size;
+    flux_error_t err;
+
+    if (content_load_get (f, &data, &size) < 0) {
+        errprintf (&err, "failed to load content data");
+        goto error_respond;
+    }
+
+    if (!w->mute) {
+        json_t *val = treeobj_create_val (data, size);
+        if (!val) {
+            errprintf (&err, "failed to create treeobj value");
+            goto error_respond;
+        }
+        if (flux_respond_pack (h, w->request, "{ s:o }", "val", val) < 0) {
+            flux_log_error (h,
+                            "%s: failed to respond to kvs-watch.lookup",
+                            __FUNCTION__);
+            json_decref (val);
+            goto finished;
+        }
+        w->responded = true;
+    }
+
+    return;
+error_respond:
+    if (!w->mute) {
+        if (flux_respond_error (h, w->request, errno, err.text) < 0)
+            flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    }
+finished:
+    w->finished = true;
+}
+
+static void load_continuation (flux_future_t *f, void *arg)
+{
+    struct watcher *w = arg;
+    struct ns_monitor *nsm = w->nsm;
+
+    while ((f = zlist_first (w->loads)) && flux_future_is_ready (f)) {
+        f = zlist_pop (w->loads);
+        if (!w->finished)
+            handle_load_response (f, w);
+        flux_future_destroy (f);
+        /* if WAITCREATE and !WATCH, then we only care about sending
+         * one response and being done.  We can use the responded flag
+         * to indicate that condition.
+         */
+        if (w->responded
+            && (w->flags & FLUX_KVS_WAITCREATE)
+            && !(w->flags & FLUX_KVS_WATCH))
+            w->finished = true;
+    }
+    if (w->finished)
+        watcher_cleanup (nsm, w);
+}
+
+static flux_future_t *load_ref (flux_t *h, struct watcher *w, const char *ref)
+{
+    flux_future_t *f = NULL;
+
+    if (!(f = content_load_byblobref (h, ref, 0))
+        || flux_future_then (f, -1., load_continuation, w) < 0)
+        goto error;
+    if (zlist_append (w->loads, f) < 0) {
+        errno = ENOMEM;
+        goto error;
+    }
+
+    return f;
+
+error:
+    flux_future_destroy (f);
+    return NULL;
+}
+
+static int load_range (flux_t *h,
+                       struct watcher *w,
+                       int start_index,
+                       int end_index,
+                       json_t *val)
+{
+    int i;
+
+    for (i = start_index; i <= end_index; i++) {
+        flux_future_t *f;
+        const char *ref = treeobj_get_blobref (val, i);
+        if (!ref)
+            return -1;
+        if (!(f = load_ref (h, w, ref)))
+            return -1;
+    }
+    return 0;
+}
+
 static int handle_initial_response (flux_t *h,
                                     struct watcher *w,
                                     json_t *val,
-                                    int root_seq)
+                                    const char *root_ref,
+                                    int root_seq,
+                                    const char *namespace)
 {
+    flux_error_t err;
+
     /* this is the first response case, store the first response
      * val */
     if ((w->flags & FLUX_KVS_WATCH_FULL)
@@ -249,22 +366,63 @@ static int handle_initial_response (flux_t *h,
         w->prev = json_incref (val);
 
     if ((w->flags & FLUX_KVS_WATCH_APPEND)) {
-        if (treeobj_decode_val (val,
-                                NULL,
-                                &w->append_offset) < 0) {
-            flux_log_error (h, "%s: treeobj_decode_val", __FUNCTION__);
-            return -1;
+        /* The very first response may be a 'val' treeobj instead of
+         * 'valref', if there have been no appends yet.
+         */
+        if (treeobj_is_val (val)) {
+            w->index_valid = true;
+            w->prev_start_index = 0;
+            w->prev_end_index = 0;
+            /* since this is a val object, we can just return it */
+            goto out;
+        }
+        else if (treeobj_is_valref (val)) {
+            w->index_valid = true;
+            w->prev_start_index = 0;
+            w->prev_end_index = treeobj_get_count (val) - 1;
+        }
+        else {
+            errprintf (&err,
+                       "%s cannot be watched with WATCH_APPEND",
+                       treeobj_type_name (val));
+            errno = EINVAL;
+            goto error_respond;
         }
+
+        if (load_range (h,
+                        w,
+                        w->prev_start_index,
+                        w->prev_end_index,
+                        val) < 0) {
+            errprintf (&err,
+                       "error sending request for content blobs [%d:%d]",
+                       w->prev_start_index,
+                       w->prev_end_index);
+            goto error_respond;
+        }
+
+        w->initial_rootseq = root_seq;
+        return 0;
     }
 
+out:
     if (flux_respond_pack (h, w->request, "{ s:O }", "val", val) < 0) {
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+        flux_log_error (h,
+                        "%s: failed to respond to kvs-watch.lookup",
+                        __FUNCTION__);
         return -1;
     }
 
     w->initial_rootseq = root_seq;
     w->responded = true;
     return 0;
+
+error_respond:
+    if (!w->mute) {
+        if (flux_respond_error (h, w->request, errno, err.text) < 0)
+            flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    }
+    return -1;
 }
 
 static int handle_compare_response (flux_t *h,
@@ -278,7 +436,9 @@ static int handle_compare_response (flux_t *h,
         w->prev = json_incref (val);
 
         if (flux_respond_pack (h, w->request, "{ s:O }", "val", val) < 0) {
-            flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+            flux_log_error (h,
+                            "%s: failed to respond to kvs-watch.lookup",
+                            __FUNCTION__);
             return -1;
         }
 
@@ -294,7 +454,9 @@ static int handle_compare_response (flux_t *h,
         w->prev = json_incref (val);
 
         if (flux_respond_pack (h, w->request, "{ s:O }", "val", val) < 0) {
-            flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+            flux_log_error (h,
+                            "%s: failed to respond to kvs-watch.lookup",
+                            __FUNCTION__);
             return -1;
         }
     }
@@ -303,69 +465,129 @@ static int handle_compare_response (flux_t *h,
 }
 
 static int handle_append_response (flux_t *h,
-                                    struct watcher *w,
-                                    json_t *val)
+                                   struct watcher *w,
+                                   json_t *val,
+                                   const char *root_ref,
+                                   int root_seq,
+                                   const char *namespace)
 {
+    flux_error_t err;
+
     if (!w->responded) {
         /* this is the first response case, store the first response
          * info.  This is here b/c initial response could have been
-         * ENOENT case */
-        if (treeobj_decode_val (val,
-                                NULL,
-                                &w->append_offset) < 0) {
-            flux_log_error (h, "%s: treeobj_decode_val", __FUNCTION__);
-            return -1;
+         * ENOENT case.
+         *
+         * The very first response may be a 'val' treeobj instead of
+         * 'valref', if there have been no appends yet.
+         */
+        if (treeobj_is_val (val)) {
+            w->index_valid = true;
+            w->prev_start_index = 0;
+            w->prev_end_index = 0;
+            /* since this is a val object, we can just return it */
+            if (flux_respond_pack (h, w->request, "{ s:O }", "val", val) < 0) {
+                flux_log_error (h,
+                                "%s: failed to respond to kvs-watch.lookup",
+                                __FUNCTION__);
+                goto error_out;
+            }
+            w->responded = true;
         }
+        else if (treeobj_is_valref (val)) {
+            /* N.B. It may not be obvious why we have to check
+             * w->index_valid if we have not yet responded.  It is
+             * possible we have received a setroot response and an
+             * updated valref before loads from the content store have
+             * returned to the caller.
+             */
+            if (w->index_valid) {
+                int new_end_index = treeobj_get_count (val) - 1;
+                if (new_end_index > w->prev_end_index) {
+                    w->prev_start_index = w->prev_end_index + 1;
+                    w->prev_end_index = new_end_index;
+                }
+                else if (new_end_index < w->prev_end_index) {
+                    errprintf (&err, "key watched with WATCH_APPEND truncated");
+                    errno = EINVAL;
+                    goto error_respond;
+                }
+                else
+                    goto out;
+            }
+            else {
+                w->index_valid = true;
+                w->prev_start_index = 0;
+                w->prev_end_index = treeobj_get_count (val) - 1;
+            }
 
-        if (flux_respond_pack (h, w->request, "{ s:O }", "val", val) < 0) {
-            flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-            return -1;
+            if (load_range (h,
+                            w,
+                            w->prev_start_index,
+                            w->prev_end_index,
+                            val) < 0) {
+                errprintf (&err,
+                           "error sending request for content blobs [%d:%d]",
+                           w->prev_start_index,
+                           w->prev_end_index);
+                goto error_respond;
+            }
+        }
+        else {
+            errprintf (&err,
+                       "%s cannot be watched with WATCH_APPEND",
+                       treeobj_type_name (val));
+            errno = EINVAL;
+            goto error_respond;
         }
-
-        w->responded = true;
     }
     else {
-        json_t *new_val = NULL;
-        void *new_data = NULL;
-        int new_offset;
-
-        if (treeobj_decode_val (val,
-                                &new_data,
-                                &new_offset) < 0) {
-            flux_log_error (h, "%s: treeobj_decode_val", __FUNCTION__);
-            return -1;
+        if (treeobj_is_valref (val)) {
+            int new_end_index;
+            if (!w->index_valid) {
+                errno = EPROTO;
+                goto error_respond;
+            }
+            new_end_index = treeobj_get_count (val) - 1;
+            if (new_end_index > w->prev_end_index) {
+                w->prev_start_index = w->prev_end_index + 1;
+                w->prev_end_index = new_end_index;
+            }
+            else if (new_end_index < w->prev_end_index) {
+                errprintf (&err, "key watched with WATCH_APPEND shortened");
+                errno = EINVAL;
+                goto error_respond;
+            }
+            else
+                goto out;
+
+            if (load_range (h,
+                            w,
+                            w->prev_start_index,
+                            w->prev_end_index,
+                            val) < 0) {
+                errprintf (&err, "error loading reference");
+                goto error_respond;
+            }
         }
-
-        /* check length to determine if append actually happened, note
-         * that zero length append is legal
-         *
-         * Note that this check does not ensure that the key was not
-         * "fake" appended to.  i.e. the key overwritten with data
-         * longer than the original.
-         */
-        if (new_offset < w->append_offset) {
-            free (new_data);
+        else {
+            errprintf (&err,
+                       "value of key watched with WATCH_APPEND overwritten");
             errno = EINVAL;
-            return -1;
-        }
-
-        if (!(new_val = treeobj_create_val (new_data + w->append_offset,
-                                            new_offset - w->append_offset))) {
-            free (new_data);
-            return -1;
-        }
-
-        free (new_data);
-        w->append_offset = new_offset;
-
-        if (flux_respond_pack (h, w->request, "{ s:o }", "val", new_val) < 0) {
-            json_decref (new_val);
-            flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
-            return -1;
+            goto error_respond;
         }
     }
 
+out:
     return 0;
+
+error_respond:
+    if (!w->mute) {
+        if (flux_respond_error (h, w->request, errno, err.text) < 0)
+            flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    }
+error_out:
+    return -1;
 }
 
 static int handle_normal_response (flux_t *h,
@@ -373,7 +595,9 @@ static int handle_normal_response (flux_t *h,
                                    json_t *val)
 {
     if (flux_respond_pack (h, w->request, "{ s:O }", "val", val) < 0) {
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+        flux_log_error (h,
+                        "%s: failed to respond to kvs-watch.lookup",
+                        __FUNCTION__);
         return -1;
     }
 
@@ -393,6 +617,7 @@ static void handle_lookup_response (flux_future_t *f,
 {
     flux_t *h = flux_future_get_flux (f);
     int errnum;
+    const char *root_ref;
     int root_seq;
     json_t *val;
 
@@ -401,7 +626,8 @@ static void handle_lookup_response (flux_future_t *f,
         w->initial_rpc_received = true;
 
         /* First check for ENOENT */
-        if (!flux_rpc_get_unpack (f, "{ s:i s:i }",
+        if (!flux_rpc_get_unpack (f,
+                                  "{ s:i s:i }",
                                   "errno", &errnum,
                                   "rootseq", &root_seq)) {
             assert (errnum == ENOENT);
@@ -414,8 +640,10 @@ static void handle_lookup_response (flux_future_t *f,
             goto error;
         }
 
-        if (flux_rpc_get_unpack (f, "{ s:o s:i }",
+        if (flux_rpc_get_unpack (f,
+                                 "{ s:o s:s s:i }",
                                  "val", &val,
+                                 "rootref", &root_ref,
                                  "rootseq", &root_seq) < 0) {
             /* It is worth mentioning ENOTSUP error conditions here.
              *
@@ -437,21 +665,26 @@ static void handle_lookup_response (flux_future_t *f,
             goto error;
         }
 
-        if (handle_initial_response (h, w, val, root_seq) < 0)
-            goto error;
+        if (handle_initial_response (h,
+                                     w,
+                                     val,
+                                     root_ref,
+                                     root_seq,
+                                     w->nsm->ns_name) < 0)
+            goto finished;
     }
     else {
         /* First check for ENOENT */
-        if (!flux_rpc_get_unpack (f, "{ s:i s:i }",
-                                  "errno", &errnum,
-                                  "rootseq", &root_seq)) {
+        if (!flux_rpc_get_unpack (f, "{ s:i }", "errno", &errnum)) {
             assert (errnum == ENOENT);
             errno = errnum;
             goto error;
         }
 
-        if (flux_rpc_get_unpack (f, "{ s:o s:i }",
+        if (flux_rpc_get_unpack (f,
+                                 "{ s:o s:s s:i }",
                                  "val", &val,
+                                 "rootref", &root_ref,
                                  "rootseq", &root_seq) < 0)
             goto error;
 
@@ -464,15 +697,20 @@ static void handle_lookup_response (flux_future_t *f,
             if ((w->flags & FLUX_KVS_WATCH_FULL)
                 || (w->flags & FLUX_KVS_WATCH_UNIQ)) {
                 if (handle_compare_response (h, w, val) < 0)
-                    goto error;
+                    goto finished;
             }
             else if (w->flags & FLUX_KVS_WATCH_APPEND) {
-                if (handle_append_response (h, w, val) < 0)
-                    goto error;
+                if (handle_append_response (h,
+                                            w,
+                                            val,
+                                            root_ref,
+                                            root_seq,
+                                            w->nsm->ns_name) < 0)
+                    goto finished;
             }
             else {
                 if (handle_normal_response (h, w, val) < 0)
-                    goto error;
+                    goto finished;
             }
         }
     }
@@ -482,6 +720,7 @@ static void handle_lookup_response (flux_future_t *f,
         if (flux_respond_error (h, w->request, errno, NULL) < 0)
             flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
     }
+finished:
     w->finished = true;
 }
 
@@ -530,22 +769,27 @@ static flux_future_t *lookupat (flux_t *h,
     json_t *o = NULL;
     flux_future_t *f;
     int saved_errno;
+    int flags = w->flags;
 
     if (!(msg = flux_request_encode ("kvs.lookup-plus", NULL)))
         return NULL;
+    if (flags & FLUX_KVS_WATCH_APPEND)
+        flags |= FLUX_KVS_TREEOBJ;
     if (!w->initial_rpc_sent) {
-        if (flux_msg_pack (msg, "{s:s s:s s:i}",
+        if (flux_msg_pack (msg,
+                           "{s:s s:s s:i}",
                            "key", w->key,
                            "namespace", ns,
-                           "flags", w->flags) < 0)
+                           "flags", flags) < 0)
             goto error;
     }
     else {
         if (!(o = treeobj_create_dirref (blobref)))
             goto error;
-        if (flux_msg_pack (msg, "{s:s s:i s:i s:O}",
+        if (flux_msg_pack (msg,
+                           "{s:s s:i s:i s:O}",
                            "key", w->key,
-                           "flags", w->flags,
+                           "flags", flags,
                            "rootseq", root_seq,
                            "rootdir", o) < 0)
             goto error;
@@ -615,7 +859,7 @@ static void watcher_respond (struct ns_monitor *nsm, struct watcher *w)
      */
     if (w->finished)
         goto finished;
-    if (w->cancelled) {
+    if (w->canceled) {
         errno = ENODATA;
         goto error_respond;
     }
@@ -648,24 +892,28 @@ static void watcher_respond (struct ns_monitor *nsm, struct watcher *w)
     }
     /* flux_kvs_lookup (FLUX_KVS_WATCH)
      *
-     * Ordering note: KVS lookups can be returned out of order.  KVS lookup
-     * futures are added to the w->lookups zlist in commit order here, and
-     * in lookup_continuation(), fulfilled futures are popped off the head
-     * of w->lookups until an unfulfilled future is encountered, so that
-     * responses are always returned to the watcher in commit order.
+     * Ordering note: KVS lookups can be returned out of order because
+     * they are processed asynchronously.  For example, some values
+     * may be cached within the KVS while others are not.
      *
-     * Security note: although the requestor has already been authenticated
+     * KVS lookup futures are added to the w->lookups zlist in commit
+     * order here, and in lookup_continuation(), fulfilled futures are
+     * popped off the head of w->lookups until an unfulfilled future
+     * is encountered, so that responses are always returned to the
+     * watcher in commit order.
+     *
+     * Security note: although the requester has already been authenticated
      * to access the namespace by check_authorization() above, we make the
-     * kvs.lookupat request with the requestor's creds, in case the key lookup
+     * kvs.lookupat request with the requester's creds, in case the key lookup
      * traverses to a new namespace.  Leave it up to the KVS module to ensure
-     * the requestor is permitted to access *that* namespace.
+     * the requester is permitted to access *that* namespace.
      *
      * Note on FLUX_KVS_WATCH_FULL: A lookup / comparison is done on every
      * change.
      */
     if (w->rootseq == -1
         || (w->flags & FLUX_KVS_WATCH_FULL)
-        || array_match (nsm->commit->keys, w->key)) {
+        || key_match (nsm->commit->keys, w->key)) {
         if (process_lookup_response (nsm, w) < 0)
             goto error_respond;
     }
@@ -681,89 +929,103 @@ static void watcher_respond (struct ns_monitor *nsm, struct watcher *w)
 }
 
 /* Respond to all ready watchers.
- * N.B. watcher_respond() may call zlist_remove() on nsm->watchers.
- * Since zlist_t is not deletion-safe for traversal, a temporary duplicate
- * must be created here.
+ * N.B. watcher_respond() may call zlistx_delete() on nsm->watchers.
  */
 static void watcher_respond_ns (struct ns_monitor *nsm)
 {
-    zlist_t *l;
     struct watcher *w;
 
-    if ((l = zlist_dup (nsm->watchers))) {
-        w = zlist_first (l);
-        while (w) {
-            watcher_respond (nsm, w);
-            w = zlist_next (l);
-        }
-        zlist_destroy (&l);
+    w = zlistx_first (nsm->watchers);
+    while (w) {
+        /* Note: get next watcher before calling watcher_respond() since
+         * `nsm` may be destroyed when next == NULL:
+         */
+        struct watcher *next = zlistx_next (nsm->watchers);
+        watcher_respond (nsm, w);
+
+        /* Note: No use-after-free possible since `nsm` may only be destroyed
+         * if nsm->watchers is empty and and thus next == NULL
+         */
+        w = next;
     }
-    else
-        flux_log_error (nsm->ctx->h, "%s: zlist_dup", __FUNCTION__);
 }
 
-/* Cancel watcher 'w' if it matches (sender, matchtag).
- * matchtag=FLUX_MATCHTAG_NONE matches any matchtag.
- * If 'mute' is true, suppress response (e.g. for disconnect handling).
+/* Cancel watcher 'w' if it matches:
+ * - credentials and matchtag if cancel true
+ * - credentials if cancel false
+ * Suppress response if cancel is false (disconnect)
  */
-static void watcher_cancel (struct ns_monitor *nsm, struct watcher *w,
-                            const char *sender, uint32_t matchtag,
-                            bool mute)
+static void watcher_cancel (struct ns_monitor *nsm,
+                            struct watcher *w,
+                            const flux_msg_t *msg,
+                            uint32_t matchtag,
+                            bool cancel)
 {
-    uint32_t t;
-    char *s;
-
-    if (matchtag != FLUX_MATCHTAG_NONE
-            && (flux_msg_get_matchtag (w->request, &t) < 0 || matchtag != t))
-        return;
-    if (flux_msg_get_route_first (w->request, &s) < 0)
+    if (!flux_disconnect_match (msg, w->request))
         return;
-    if (!strcmp (sender, s)) {
-        w->cancelled = true;
-        w->mute = mute;
-        watcher_respond (nsm, w);
+    if (cancel) {
+        uint32_t tag;
+        if (flux_msg_get_matchtag (w->request, &tag) < 0
+            || tag != matchtag)
+            return;
     }
-    free (s);
+    w->canceled = true;
+    w->mute = !cancel;
+    watcher_respond (nsm, w);
 }
 
-/* Cancel all namespace watchers that match (sender, matchtag).
- * If 'mute' is true, suppress response.
+/* Cancel all namespace watchers that match:
+ * - credentials and matchtag if cancel true
+ * - credentials if cancel false
+ * Suppress response if cancel is false
  */
 static void watcher_cancel_ns (struct ns_monitor *nsm,
-                               const char *sender, uint32_t matchtag,
-                               bool mute)
+                               const flux_msg_t *msg,
+                               uint32_t matchtag,
+                               bool cancel)
 {
-    zlist_t *l;
     struct watcher *w;
 
-    if ((l = zlist_dup (nsm->watchers))) {
-        w = zlist_first (l);
-        while (w) {
-            watcher_cancel (nsm, w, sender, matchtag, mute);
-            w = zlist_next (l);
-        }
-        zlist_destroy (&l);
+    w = zlistx_first (nsm->watchers);
+    while (w) {
+        /* Note: get next watcher before calling watcher_respond() since
+         * `nsm` may be destroyed when next == NULL:
+         */
+        struct watcher *next = zlistx_next (nsm->watchers);
+        watcher_cancel (nsm, w, msg, matchtag, cancel);
+
+        /* Note: No use-after-free possible since `nsm` may only be destroyed
+         * if nsm->watchers is empty and and thus next == NULL
+         */
+        w = next;
     }
-    else
-        flux_log_error (nsm->ctx->h, "%s: zlist_dup", __FUNCTION__);
 }
 
-/* Cancel all watchers that match (sender, matchtag).
- * If 'mute' is true, suppress response.
+/* Cancel all watchers that match:
+ * - credentials and matchtag if cancel true
+ * - credentials if cancel false
+ * Suppress response if cancel is false
  */
 static void watcher_cancel_all (struct watch_ctx *ctx,
-                                const char *sender, uint32_t matchtag,
-                                bool mute)
+                                const flux_msg_t *msg,
+                                bool cancel)
 {
     zlist_t *l;
     char *name;
     struct ns_monitor *nsm;
+    uint32_t matchtag = FLUX_MATCHTAG_NONE;
+
+    if (cancel
+        && flux_msg_unpack (msg, "{s:i}", "matchtag", &matchtag) < 0) {
+        flux_log_error (ctx->h, "failed to get matchtag from cancel request");
+        return;
+    }
 
     if ((l = zhash_keys (ctx->namespaces))) {
         name = zlist_first (l);
         while (name) {
             nsm = zhash_lookup (ctx->namespaces, name);
-            watcher_cancel_ns (nsm, sender, matchtag, mute);
+            watcher_cancel_ns (nsm, msg, matchtag, cancel);
             name = zlist_next (l);
         }
         zlist_destroy (&l);
@@ -775,8 +1037,10 @@ static void watcher_cancel_all (struct watch_ctx *ctx,
 /* kvs.namespace-removed-* event
  * A namespace has been removed.  All watchers should receive ENOTSUP.
  */
-static void removed_cb (flux_t *h, flux_msg_handler_t *mh,
-                        const flux_msg_t *msg, void *arg)
+static void removed_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
 {
     struct watch_ctx *ctx = arg;
     const char *ns;
@@ -796,8 +1060,10 @@ static void removed_cb (flux_t *h, flux_msg_handler_t *mh,
  * Update namespace with new namespace info.
  * N.B. commit->keys is empty in this case, in contrast setroot_cb().
  */
-static void namespace_created_cb (flux_t *h, flux_msg_handler_t *mh,
-                                  const flux_msg_t *msg, void *arg)
+static void namespace_created_cb (flux_t *h,
+                                  flux_msg_handler_t *mh,
+                                  const flux_msg_t *msg,
+                                  void *arg)
 {
     struct watch_ctx *ctx = arg;
     struct ns_monitor *nsm;
@@ -807,7 +1073,9 @@ static void namespace_created_cb (flux_t *h, flux_msg_handler_t *mh,
     int owner;
     struct commit *commit;
 
-    if (flux_event_unpack (msg, NULL, "{s:s s:i s:s s:i}",
+    if (flux_event_unpack (msg,
+                           NULL,
+                           "{s:s s:i s:s s:i}",
                            "namespace", &ns,
                            "rootseq", &rootseq,
                            "rootref", &rootref,
@@ -834,8 +1102,10 @@ static void namespace_created_cb (flux_t *h, flux_msg_handler_t *mh,
  * Update namespace with new commit info.
  * Subscribe/unsubscribe is tied to 'struct ns_monitor' create/destroy.
  */
-static void setroot_cb (flux_t *h, flux_msg_handler_t *mh,
-                        const flux_msg_t *msg, void *arg)
+static void setroot_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
 {
     struct watch_ctx *ctx = arg;
     struct ns_monitor *nsm;
@@ -846,7 +1116,9 @@ static void setroot_cb (flux_t *h, flux_msg_handler_t *mh,
     json_t *keys;
     struct commit *commit;
 
-    if (flux_event_unpack (msg, NULL, "{s:s s:i s:s s:i s:o}",
+    if (flux_event_unpack (msg,
+                           NULL,
+                           "{s:s s:i s:s s:i s:o}",
                            "namespace", &ns,
                            "rootseq", &rootseq,
                            "rootref", &rootref,
@@ -883,8 +1155,8 @@ static void namespace_getroot_continuation (flux_future_t *f, void *arg)
     uint32_t owner;
     struct commit *commit;
 
-    /* small racy chance watcher cancelled before getroot completes */
-    if (zlist_size (nsm->watchers) == 0) {
+    /* small racy chance watcher canceled before getroot completes */
+    if (zlistx_size (nsm->watchers) == 0) {
         zhash_delete (nsm->ctx->namespaces, nsm->ns_name);
         return;
     }
@@ -929,10 +1201,7 @@ struct ns_monitor *namespace_monitor (struct watch_ctx *ctx,
     if (!(nsm = zhash_lookup (ctx->namespaces, ns))) {
         if (!(nsm = namespace_create (ctx, ns)))
             return NULL;
-        if (zhash_insert (ctx->namespaces, ns, nsm) < 0) {
-            namespace_destroy (nsm);
-            return NULL;
-        }
+        (void)zhash_insert (ctx->namespaces, ns, nsm);
         zhash_freefn (ctx->namespaces, ns,
                       (zhash_free_fn *)namespace_destroy);
         /* store future in namespace, so namespace can be destroyed
@@ -950,8 +1219,10 @@ struct ns_monitor *namespace_monitor (struct watch_ctx *ctx,
     return nsm;
 }
 
-static void lookup_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
+static void lookup_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
 {
     struct watch_ctx *ctx = arg;
     const char *ns;
@@ -961,7 +1232,9 @@ static void lookup_cb (flux_t *h, flux_msg_handler_t *mh,
     struct watcher *w;
     const char *errmsg = NULL;
 
-    if (flux_request_unpack (msg, NULL, "{s:s s:s s:i}",
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s:s s:i}",
                              "namespace", &ns,
                              "key", &key,
                              "flags", &flags) < 0)
@@ -982,7 +1255,7 @@ static void lookup_cb (flux_t *h, flux_msg_handler_t *mh,
     if (!(w = watcher_create (msg, key, flags)))
         goto error;
     w->nsm = nsm;
-    if (zlist_append (nsm->watchers, w) < 0) {
+    if (!(w->handle = zlistx_add_end (nsm->watchers, w))) {
         watcher_destroy (w);
         errno = ENOMEM;
         goto error;
@@ -997,55 +1270,36 @@ static void lookup_cb (flux_t *h, flux_msg_handler_t *mh,
 
 /* kvs-watch.cancel request
  * The user called flux_kvs_lookup_cancel() which expects no response.
- * The enclosed matchtag and the cancel sender are used to find the
- * watcher that is to be cancelled.  The watcher will receive an ENODATA
- * response message.
+ * The watcher will receive an ENODATA response message.
  */
-static void cancel_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
+static void cancel_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
 {
     struct watch_ctx *ctx = arg;
-    uint32_t matchtag;
-    char *sender;
-
-    if (flux_request_unpack (msg, NULL, "{s:i}", "matchtag", &matchtag) < 0) {
-        flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
-        return;
-    }
-    if (flux_msg_get_route_first (msg, &sender) < 0) {
-        flux_log_error (h, "%s: flux_msg_get_route_first", __FUNCTION__);
-        return;
-    }
-    watcher_cancel_all (ctx, sender, matchtag, false);
-    free (sender);
+    watcher_cancel_all (ctx, msg, true);
 }
 
 /* kvs-watch.disconnect request
  * This is sent automatically upon local connector disconnect.
- * The disconnect sender is used to find any watchers to be cancelled.
+ * The disconnect sender is used to find any watchers to be canceled.
  */
-static void disconnect_cb (flux_t *h, flux_msg_handler_t *mh,
-                           const flux_msg_t *msg, void *arg)
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
 {
     struct watch_ctx *ctx = arg;
-    char *sender;
-
-    if (flux_request_decode (msg, NULL, NULL) < 0) {
-        flux_log_error (h, "%s: flux_request_decode", __FUNCTION__);
-        return;
-    }
-    if (flux_msg_get_route_first (msg, &sender) < 0) {
-        flux_log_error (h, "%s: flux_msg_get_route_first", __FUNCTION__);
-        return;
-    }
-    watcher_cancel_all (ctx, sender, FLUX_MATCHTAG_NONE, true);
-    free (sender);
+    watcher_cancel_all (ctx, msg, false);
 }
 
-/* kvs-watch.stats.get request
+/* kvs-watch.stats-get request
  */
-static void stats_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+static void stats_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     struct watch_ctx *ctx = arg;
     struct ns_monitor *nsm;
@@ -1062,21 +1316,25 @@ static void stats_cb (flux_t *h, flux_msg_handler_t *mh,
                                                       : -1,
                                "rootref", nsm->commit ? nsm->commit->rootref
                                                       : "(null)",
-                               "watchers", (int)zlist_size (nsm->watchers));
+                               "watchers", (int)zlistx_size (nsm->watchers));
         if (!o)
             goto nomem;
         if (json_object_set_new (stats, nsm->ns_name, o) < 0) {
             json_decref (o);
             goto nomem;
         }
-        watchers += zlist_size (nsm->watchers);
+        watchers += zlistx_size (nsm->watchers);
         nsm = zhash_next (ctx->namespaces);
     }
-    if (flux_respond_pack (h, msg, "{s:i s:i s:O}",
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:i s:i s:O}",
                            "watchers", watchers,
                            "namespace-count", (int)zhash_size (ctx->namespaces),
                            "namespaces", stats) < 0)
-        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+        flux_log_error (h,
+                        "%s: failed to respond to kvs-watch.stats-get",
+                        __FUNCTION__);
     json_decref (stats);
     return;
 nomem:
@@ -1104,9 +1362,9 @@ static const struct flux_msg_handler_spec htab[] = {
       .rolemask     = 0
     },
     { .typemask     = FLUX_MSGTYPE_REQUEST,
-      .topic_glob   = "kvs-watch.stats.get",
+      .topic_glob   = "kvs-watch.stats-get",
       .cb           = stats_cb,
-      .rolemask     = 0
+      .rolemask     = FLUX_ROLE_USER
     },
     { .typemask     = FLUX_MSGTYPE_REQUEST,
       .topic_glob   = "kvs-watch.lookup",
@@ -1170,8 +1428,6 @@ int mod_main (flux_t *h, int argc, char **argv)
     return rc;
 }
 
-MOD_NAME ("kvs-watch");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/kvs/Makefile.am b/src/modules/kvs/Makefile.am
index 1e163a943aa9..7777d6b912a1 100644
--- a/src/modules/kvs/Makefile.am
+++ b/src/modules/kvs/Makefile.am
@@ -6,35 +6,34 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/librouter \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
+	$(JANSSON_CFLAGS)
 
-fluxmod_LTLIBRARIES = kvs.la
+noinst_LTLIBRARIES = libkvs.la
 
-kvs_la_SOURCES = \
+libkvs_la_SOURCES = \
 	kvs.c \
 	cache.c \
 	cache.h \
 	waitqueue.c \
 	waitqueue.h \
-	lookup.h \
 	lookup.c \
-	treq.h \
+	lookup.h \
 	treq.c \
-	kvstxn.h \
+	treq.h \
 	kvstxn.c \
-	kvsroot.h \
+	kvstxn.h \
 	kvsroot.c \
-	kvssync.h \
-	kvssync.c
-
-kvs_la_LDFLAGS = $(fluxmod_ldflags) -module
-kvs_la_LIBADD = $(top_builddir)/src/common/libkvs/libkvs.la \
-		$(top_builddir)/src/common/libflux-internal.la \
-		$(top_builddir)/src/common/libflux-core.la \
-		$(ZMQ_LIBS)
+	kvsroot.h \
+	kvs_wait_version.c \
+	kvs_wait_version.h \
+	kvs_checkpoint.c \
+	kvs_checkpoint.h
 
 TESTS = \
 	test_waitqueue.t \
@@ -43,18 +42,24 @@ TESTS = \
 	test_treq.t \
 	test_kvstxn.t \
 	test_kvsroot.t \
-	test_kvssync.t
+	test_kvs_wait_version.t
 
 test_ldadd = \
+	$(builddir)/libkvs.la \
 	$(top_builddir)/src/common/libkvs/libkvs.la \
-	$(top_builddir)/src/common/libflux-internal.la \
 	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-internal.la \
 	$(top_builddir)/src/common/libtap/libtap.la \
-        $(ZMQ_LIBS) $(LIBPTHREAD)
+	$(JANSSON_LIBS) \
+        $(LIBPTHREAD)
+
+test_ldflags = \
+	-no-install
 
 test_cppflags = \
         $(AM_CPPFLAGS) \
-        -I$(top_srcdir)/src/common/libtap
+        -I$(top_srcdir)/src/common/libtap \
+	$(JANSSON_CFLAGS)
 
 check_PROGRAMS = $(TESTS)
 
@@ -67,6 +72,8 @@ test_waitqueue_t_CPPFLAGS = $(test_cppflags)
 test_waitqueue_t_LDADD = \
 	$(top_builddir)/src/modules/kvs/waitqueue.o \
 	$(test_ldadd)
+test_waitqueue_t_LDFLAGS = \
+	$(test_ldflags)
 
 test_cache_t_SOURCES = test/cache.c
 test_cache_t_CPPFLAGS = $(test_cppflags)
@@ -74,6 +81,8 @@ test_cache_t_LDADD = \
 	$(top_builddir)/src/modules/kvs/cache.o \
 	$(top_builddir)/src/modules/kvs/waitqueue.o \
 	$(test_ldadd)
+test_cache_t_LDFLAGS = \
+	$(test_ldflags)
 
 test_lookup_t_SOURCES = test/lookup.c
 test_lookup_t_CPPFLAGS = $(test_cppflags)
@@ -85,12 +94,16 @@ test_lookup_t_LDADD = \
 	$(top_builddir)/src/modules/kvs/kvstxn.o \
 	$(top_builddir)/src/modules/kvs/treq.o \
 	$(test_ldadd)
+test_lookup_t_LDFLAGS = \
+	$(test_ldflags)
 
 test_treq_t_SOURCES = test/treq.c
 test_treq_t_CPPFLAGS = $(test_cppflags)
 test_treq_t_LDADD = \
 	$(top_builddir)/src/modules/kvs/treq.o \
 	$(test_ldadd)
+test_treq_t_LDFLAGS = \
+	$(test_ldflags)
 
 test_kvstxn_t_SOURCES = test/kvstxn.c
 test_kvstxn_t_CPPFLAGS = $(test_cppflags)
@@ -102,6 +115,8 @@ test_kvstxn_t_LDADD = \
 	$(top_builddir)/src/modules/kvs/treq.o \
 	$(top_builddir)/src/modules/kvs/waitqueue.o \
 	$(test_ldadd)
+test_kvstxn_t_LDFLAGS = \
+	$(test_ldflags)
 
 test_kvsroot_t_SOURCES = test/kvsroot.c
 test_kvsroot_t_CPPFLAGS = $(test_cppflags)
@@ -112,14 +127,20 @@ test_kvsroot_t_LDADD = \
 	$(top_builddir)/src/modules/kvs/cache.o \
 	$(top_builddir)/src/modules/kvs/treq.o \
 	$(test_ldadd)
+test_kvsroot_t_LDFLAGS = \
+	$(test_ldflags)
 
-test_kvssync_t_SOURCES = test/kvssync.c
-test_kvssync_t_CPPFLAGS = $(test_cppflags)
-test_kvssync_t_LDADD = \
-	$(top_builddir)/src/modules/kvs/kvssync.o \
+test_kvs_wait_version_t_SOURCES = test/kvs_wait_version.c
+test_kvs_wait_version_t_CPPFLAGS = $(test_cppflags)
+test_kvs_wait_version_t_LDADD = \
+	$(top_builddir)/src/modules/kvs/kvs_wait_version.o \
 	$(top_builddir)/src/modules/kvs/waitqueue.o \
 	$(top_builddir)/src/modules/kvs/kvsroot.o \
 	$(top_builddir)/src/modules/kvs/kvstxn.o \
 	$(top_builddir)/src/modules/kvs/cache.o \
 	$(top_builddir)/src/modules/kvs/treq.o \
 	$(test_ldadd)
+test_kvs_wait_version_t_LDFLAGS = \
+	$(test_ldflags)
+
+EXTRA_DIST = README.md
diff --git a/src/modules/kvs/README.md b/src/modules/kvs/README.md
new file mode 100644
index 000000000000..ed776d9fa583
--- /dev/null
+++ b/src/modules/kvs/README.md
@@ -0,0 +1,6 @@
+## Design notes
+
+Assorted design notes are available in
+[Resources for Flux Developers](https://flux-framework.readthedocs.io/projects/flux-core/en/latest/guide/internals.html)
+including for the
+[KVS](https://flux-framework.readthedocs.io/projects/flux-core/en/latest/guide/kvs.html).
diff --git a/src/modules/kvs/cache.c b/src/modules/kvs/cache.c
index 3b3e249a8398..6f25e14bfe6b 100644
--- a/src/modules/kvs/cache.c
+++ b/src/modules/kvs/cache.c
@@ -22,10 +22,15 @@
 #include <stdbool.h>
 #include <ctype.h>
 #include <sys/time.h>
-#include <czmq.h>
 #include <flux/core.h>
 #include <jansson.h>
 
+#ifndef EBADE
+#define EBADE EINVAL
+#endif
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libccan/ccan/list/list.h"
 #include "src/common/libkvs/treeobj.h"
 #include "src/common/libutil/blobref.h"
 #include "src/common/libutil/tstat.h"
@@ -42,7 +47,7 @@ struct cache_entry {
     void *data;             /* value raw data */
     int len;
     json_t *o;              /* value treeobj object */
-    int lastuse_epoch;      /* time of last use for cache expiry */
+    double lastuse_time;    /* time of last use for cache expiry */
     bool valid;             /* flag indicating if raw data or treeobj
                              * set, don't use data == NULL as test, as
                              * zero length data can be valid */
@@ -50,12 +55,36 @@ struct cache_entry {
     int errnum;
     char *blobref;
     int refcount;
+    struct list_node entries_node;
+    struct list_head *notdirty_list;
+    struct list_node notdirty_node;
+    struct list_head *valid_list;
+    struct list_node valid_node;
 };
 
 struct cache {
+    flux_reactor_t *r;
+    double fake_time;       /* -1. for invalid */
     zhashx_t *zhx;
+    /* entries_list is for fast iteration through entries, faster than
+     * using zhashx iterators or zhashx_keys() */
+    struct list_head entries_list;
+    /* list of entries with notdirty & valid waitqueue's with messages
+     * on them.  These lists are used to avoid excess iteration
+     * through zhx */
+    struct list_head notdirty_list;
+    struct list_head valid_list;
 };
 
+static double cache_now (struct cache *cache)
+{
+    if (cache->fake_time >= 0.)
+        return cache->fake_time;
+    if (cache->r)
+        return flux_reactor_now (cache->r);
+    return 0.;
+}
+
 struct cache_entry *cache_entry_create (const char *ref)
 {
     struct cache_entry *entry;
@@ -65,17 +94,17 @@ struct cache_entry *cache_entry_create (const char *ref)
         return NULL;
     }
 
-    if (!(entry = calloc (1, sizeof (*entry)))) {
-        errno = ENOMEM;
+    if (!(entry = calloc (1, sizeof (*entry))))
         return NULL;
-    }
 
     if (!(entry->blobref = strdup (ref))) {
         cache_entry_destroy (entry);
-        errno = ENOMEM;
         return NULL;
     }
 
+    list_node_init (&entry->entries_node);
+    list_node_init (&entry->notdirty_node);
+    list_node_init (&entry->valid_node);
     return entry;
 }
 
@@ -104,6 +133,8 @@ int cache_entry_set_dirty (struct cache_entry *entry, bool val)
                     entry->dirty = true;
                     return -1;
                 }
+                if (!wait_queue_msgs_count (entry->waitlist_notdirty))
+                    list_del_init (&entry->notdirty_node);
             }
         }
         return 0;
@@ -129,6 +160,7 @@ int cache_entry_force_clear_dirty (struct cache_entry *entry)
         if (entry->dirty) {
             if (entry->waitlist_notdirty) {
                 wait_queue_destroy (entry->waitlist_notdirty);
+                list_del_init (&entry->notdirty_node);
                 entry->waitlist_notdirty = NULL;
             }
             entry->dirty = false;
@@ -150,7 +182,8 @@ void cache_entry_decref (struct cache_entry *entry)
         entry->refcount--;
 }
 
-int cache_entry_get_raw (struct cache_entry *entry, const void **data,
+int cache_entry_get_raw (struct cache_entry *entry,
+                         const void **data,
                          int *len)
 {
     if (!entry || !entry->valid)
@@ -191,6 +224,8 @@ int cache_entry_set_raw (struct cache_entry *entry, const void *data, int len)
     if (entry->waitlist_valid) {
         if (wait_runqueue (entry->waitlist_valid) < 0)
             goto reset_invalid;
+        if (!wait_queue_msgs_count (entry->waitlist_valid))
+            list_del_init (&entry->valid_node);
     }
     return 0;
 reset_invalid:
@@ -222,6 +257,8 @@ int cache_entry_set_errnum_on_valid (struct cache_entry *entry, int errnum)
             return -1;
         if (wait_runqueue (entry->waitlist_valid) < 0)
             return -1;
+        if (!wait_queue_msgs_count (entry->waitlist_valid))
+            list_del_init (&entry->valid_node);
     }
 
     return 0;
@@ -242,6 +279,8 @@ int cache_entry_set_errnum_on_notdirty (struct cache_entry *entry, int errnum)
             return -1;
         if (wait_runqueue (entry->waitlist_notdirty) < 0)
             return -1;
+        if (!wait_queue_msgs_count (entry->waitlist_notdirty))
+            list_del_init (&entry->notdirty_node);
     }
 
     return 0;
@@ -262,17 +301,30 @@ void cache_entry_destroy (void *arg)
 {
     struct cache_entry *entry = arg;
     if (entry) {
+        int saved_errno = errno;
         free (entry->data);
         json_decref (entry->o);
-        if (entry->waitlist_notdirty)
+        if (entry->waitlist_notdirty) {
             wait_queue_destroy (entry->waitlist_notdirty);
-        if (entry->waitlist_valid)
+            list_del (&entry->notdirty_node);
+        }
+        if (entry->waitlist_valid) {
             wait_queue_destroy (entry->waitlist_valid);
+            list_del (&entry->valid_node);
+        }
         free (entry->blobref);
         free (entry);
+        errno = saved_errno;
     }
 }
 
+/* ccan list doesn't appear to have a macro to check if a node exists
+ * on a list */
+static inline bool on_list (struct list_node *n)
+{
+    return !(n->next == n->prev && n->next == n);
+}
+
 int cache_entry_wait_notdirty (struct cache_entry *entry, wait_t *wait)
 {
     if (wait) {
@@ -282,6 +334,10 @@ int cache_entry_wait_notdirty (struct cache_entry *entry, wait_t *wait)
         }
         if (wait_addqueue (entry->waitlist_notdirty, wait) < 0)
             return -1;
+        if (wait_queue_msgs_count (entry->waitlist_notdirty) > 0
+            && entry->notdirty_list
+            && !on_list (&entry->notdirty_node))
+            list_add (entry->notdirty_list, &entry->notdirty_node);
     }
     return 0;
 }
@@ -295,25 +351,38 @@ int cache_entry_wait_valid (struct cache_entry *entry, wait_t *wait)
         }
         if (wait_addqueue (entry->waitlist_valid, wait) < 0)
             return -1;
+        if (wait_queue_msgs_count (entry->waitlist_valid) > 0
+            && entry->valid_list
+            && !on_list (&entry->valid_node))
+            list_add (entry->valid_list, &entry->valid_node);
     }
     return 0;
 }
 
-struct cache_entry *cache_lookup (struct cache *cache, const char *ref,
-                                  int current_epoch)
+struct cache_entry *cache_lookup (struct cache *cache, const char *ref)
 {
     struct cache_entry *entry = zhashx_lookup (cache->zhx, ref);
-    if (entry && current_epoch > entry->lastuse_epoch)
-        entry->lastuse_epoch = current_epoch;
+    double current_time = cache_now (cache);
+    if (entry && current_time > entry->lastuse_time)
+        entry->lastuse_time = current_time;
     return entry;
 }
 
 int cache_insert (struct cache *cache, struct cache_entry *entry)
 {
-    int rc;
+    __attribute__((unused)) int rc;
 
     if (cache && entry) {
         rc = zhashx_insert (cache->zhx, entry->blobref, entry);
+        list_add (&cache->entries_list, &entry->entries_node);
+        entry->notdirty_list = &cache->notdirty_list;
+        entry->valid_list = &cache->valid_list;
+        if (entry->waitlist_notdirty
+            && wait_queue_msgs_count (entry->waitlist_notdirty) > 0)
+            list_add (entry->notdirty_list, &entry->notdirty_node);
+        if (entry->waitlist_valid
+            && wait_queue_msgs_count (entry->waitlist_valid) > 0)
+            list_add (entry->valid_list, &entry->valid_node);
         assert (rc == 0);
     }
     return 0;
@@ -329,6 +398,7 @@ int cache_remove_entry (struct cache *cache, const char *ref)
             || !wait_queue_length (entry->waitlist_notdirty))
         && (!entry->waitlist_valid
             || !wait_queue_length (entry->waitlist_valid))) {
+        list_del (&entry->entries_node);
         zhashx_delete (cache->zhx, ref);
         return 1;
     }
@@ -340,47 +410,40 @@ int cache_count_entries (struct cache *cache)
     return zhashx_size (cache->zhx);
 }
 
-static int cache_entry_age (struct cache_entry *entry, int current_epoch)
+static int cache_entry_age (struct cache_entry *entry, struct cache *cache)
 {
+    double current_time = cache_now (cache);
     if (!entry)
         return -1;
-    if (entry->lastuse_epoch == 0)
-        entry->lastuse_epoch = current_epoch;
-    return current_epoch - entry->lastuse_epoch;
+    if (entry->lastuse_time == 0.)
+        entry->lastuse_time = current_time;
+    return current_time - entry->lastuse_time;
 }
 
-int cache_expire_entries (struct cache *cache, int current_epoch, int thresh)
+int cache_expire_entries (struct cache *cache, double thresh)
 {
-    zlistx_t *keys;
-    char *ref;
-    struct cache_entry *entry;
+    struct cache_entry *entry = NULL;
+    struct cache_entry *next = NULL;
     int count = 0;
 
-    /* Do not use zhashx_first()/zhashx_next() or FOREACH_ZHASHX, as
-     * zhashx_delete() call below modifies hash */
-    if (!(keys = zhashx_keys (cache->zhx))) {
-        errno = ENOMEM;
-        return -1;
-    }
-    ref = zlistx_first (keys);
-    while (ref) {
-        if ((entry = zhashx_lookup (cache->zhx, ref))
-            && !cache_entry_get_dirty (entry)
+    list_for_each_safe (&cache->entries_list, entry, next, entries_node) {
+        if (!cache_entry_get_dirty (entry)
             && cache_entry_get_valid (entry)
             && !entry->refcount
-            && (thresh == 0
-                || cache_entry_age (entry, current_epoch) > thresh)) {
-                zhashx_delete (cache->zhx, ref);
+            && (thresh == 0. || cache_entry_age (entry, cache) > thresh)) {
+                list_del (&entry->entries_node);
+                zhashx_delete (cache->zhx, entry->blobref);
                 count++;
         }
-        ref = zlistx_next (keys);
     }
-    zlistx_destroy (&keys);
     return count;
 }
 
-int cache_get_stats (struct cache *cache, tstat_t *ts, int *sizep,
-                     int *incompletep, int *dirtyp)
+int cache_get_stats (struct cache *cache,
+                     tstat_t *ts,
+                     int *sizep,
+                     int *incompletep,
+                     int *dirtyp)
 {
     struct cache_entry *entry;
     const char *key;
@@ -413,22 +476,21 @@ int cache_get_stats (struct cache *cache, tstat_t *ts, int *sizep,
 
 int cache_wait_destroy_msg (struct cache *cache, wait_test_msg_f cb, void *arg)
 {
-    const char *key;
-    struct cache_entry *entry;
+    struct cache_entry *entry = NULL;
     int n, count = 0;
     int rc = -1;
 
-    FOREACH_ZHASHX (cache->zhx, key, entry) {
-        if (entry->waitlist_valid) {
-            if ((n = wait_destroy_msg (entry->waitlist_valid, cb, arg)) < 0)
-                goto done;
-            count += n;
-        }
-        if (entry->waitlist_notdirty) {
-            if ((n = wait_destroy_msg (entry->waitlist_notdirty, cb, arg)) < 0)
-                goto done;
-            count += n;
-        }
+    list_for_each (&cache->notdirty_list, entry, notdirty_node) {
+        assert (entry->waitlist_notdirty);
+        if ((n = wait_destroy_msg (entry->waitlist_notdirty, cb, arg)) < 0)
+            goto done;
+        count += n;
+    }
+    list_for_each (&cache->valid_list, entry, valid_node) {
+        assert (entry->waitlist_valid);
+        if ((n = wait_destroy_msg (entry->waitlist_valid, cb, arg)) < 0)
+            goto done;
+        count += n;
     }
     rc = count;
 done:
@@ -442,27 +504,31 @@ const char *cache_entry_get_blobref (struct cache_entry *entry)
 
 static void cache_entry_destroy_wrapper (void **arg)
 {
-    struct cache_entry **entry = (struct cache_entry **)arg;
-    if (entry)
-        cache_entry_destroy (*entry);
+    if (arg) {
+        cache_entry_destroy (*arg);
+        *arg = NULL;
+    }
 }
 
-struct cache *cache_create (void)
+struct cache *cache_create (flux_reactor_t *r)
 {
     struct cache *cache = calloc (1, sizeof (*cache));
-    if (!cache) {
-        errno = ENOMEM;
+    if (!cache)
         return NULL;
-    }
     if (!(cache->zhx = zhashx_new ())) {
         free (cache);
         errno = ENOMEM;
         return NULL;
     }
+    cache->r = r;
+    cache->fake_time = -1.;
     /* do not duplicate hash keys, use blobrefs stored in cache entry */
     zhashx_set_key_destructor (cache->zhx, NULL);
     zhashx_set_key_duplicator (cache->zhx, NULL);
     zhashx_set_destructor (cache->zhx, cache_entry_destroy_wrapper);
+    list_head_init (&cache->entries_list);
+    list_head_init (&cache->notdirty_list);
+    list_head_init (&cache->valid_list);
     return cache;
 }
 
@@ -474,6 +540,20 @@ void cache_destroy (struct cache *cache)
     }
 }
 
+/* for testing */
+void cache_entry_set_fake_time (struct cache_entry *entry, double time)
+{
+    if (entry)
+        entry->lastuse_time = time;
+}
+
+/* for testing */
+void cache_set_fake_time (struct cache *cache, double time)
+{
+    if (cache)
+        cache->fake_time = time;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/kvs/cache.h b/src/modules/kvs/cache.h
index 9ffb868009b0..d368f9a01f08 100644
--- a/src/modules/kvs/cache.h
+++ b/src/modules/kvs/cache.h
@@ -86,7 +86,8 @@ void cache_entry_decref (struct cache_entry *entry);
  * cache_entry_set_raw() & cache_entry_clear_data()
  * return -1 on error, 0 on success
  */
-int cache_entry_get_raw (struct cache_entry *entry, const void **data,
+int cache_entry_get_raw (struct cache_entry *entry,
+                         const void **data,
                          int *len);
 int cache_entry_set_raw (struct cache_entry *entry, const void *data, int len);
 
@@ -110,16 +111,16 @@ int cache_entry_wait_valid (struct cache_entry *entry, wait_t *wait);
 const char *cache_entry_get_blobref (struct cache_entry *entry);
 
 /* Create/destroy the cache container and its contents.
+ * 'r' is used as a source of relative current time for cache aging.
+ * If NULL, the cache never ages.
  */
-struct cache *cache_create (void);
+struct cache *cache_create (flux_reactor_t *r);
 void cache_destroy (struct cache *cache);
 
 /* Look up a cache entry.
- * Update the entry's "last used" time to 'current_epoch',
- * taking care not to not run backwards.
+ * Update the cache entry's "last used" time.
  */
-struct cache_entry *cache_lookup (struct cache *cache,
-                                  const char *ref, int current_epoch);
+struct cache_entry *cache_lookup (struct cache *cache, const char *ref);
 
 /* Insert entry in the cache.  Reference for entry created during
  * cache_entry_create() time.  Ownership of the cache entry is
@@ -127,8 +128,9 @@ struct cache_entry *cache_lookup (struct cache *cache,
  */
 int cache_insert (struct cache *cache, struct cache_entry *entry);
 
-/* Remove a cache_entry from the cache.  Will not be removed if dirty
- * or there are any waiters of any sort.
+/* Remove a cache_entry from the cache.  Will not be removed if dirty,
+ * if there are any waiters of any sort, or if there are any references
+ * taken on the entry (i.e. with cache_entry_incref()).
  * Returns 1 on removed, 0 if not
  */
 int cache_remove_entry (struct cache *cache, const char *ref);
@@ -138,22 +140,34 @@ int cache_remove_entry (struct cache *cache, const char *ref);
 int cache_count_entries (struct cache *cache);
 
 /* Expire cache entries that are not dirty, not incomplete, and last
- * used more than 'thresh' epoch's ago.
+ * used more than 'max_age' seconds ago.  If max_age == 0, expire all
+ * entries that are not dirty/incomplete.
  * Returns -1 on error, expired count on success.
  */
-int cache_expire_entries (struct cache *cache, int current_epoch, int thresh);
+int cache_expire_entries (struct cache *cache, double max_age);
 
 /* Obtain statistics on the cache.
  * Returns -1 on error, 0 on success
  */
-int cache_get_stats (struct cache *cache, tstat_t *ts, int *size,
-                     int *incomplete, int *dirty);
+int cache_get_stats (struct cache *cache,
+                     tstat_t *ts,
+                     int *size,
+                     int *incomplete,
+                     int *dirty);
 
 /* Destroy wait_t's on the waitqueue_t of any cache entry
  * if they meet match criteria.
  */
 int cache_wait_destroy_msg (struct cache *cache, wait_test_msg_f cb, void *arg);
 
+/* for testing */
+void cache_entry_set_fake_time (struct cache_entry *entry, double time);
+void cache_set_fake_time (struct cache *cache, double time);
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
+
 #endif /* !_FLUX_KVS_CACHE_H */
 
 /*
diff --git a/src/modules/kvs/kvs.c b/src/modules/kvs/kvs.c
index 72cbf0d0b689..d36229b24fe2 100644
--- a/src/modules/kvs/kvs.c
+++ b/src/modules/kvs/kvs.c
@@ -20,16 +20,23 @@
 #include <unistd.h>
 #include <stdbool.h>
 #include <sys/time.h>
-#include <czmq.h>
 #include <flux/core.h>
 #include <jansson.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libccan/ccan/list/list.h"
 #include "src/common/libutil/blobref.h"
 #include "src/common/libutil/monotime.h"
 #include "src/common/libutil/tstat.h"
+#include "src/common/libutil/timestamp.h"
+#include "src/common/libutil/errprintf.h"
 #include "src/common/libkvs/treeobj.h"
+#include "src/common/libkvs/kvs_checkpoint.h"
 #include "src/common/libkvs/kvs_txn_private.h"
 #include "src/common/libkvs/kvs_util_private.h"
+#include "src/common/libcontent/content.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/librouter/msg_hash.h"
 
 #include "waitqueue.h"
 #include "cache.h"
@@ -38,133 +45,148 @@
 #include "treq.h"
 #include "kvstxn.h"
 #include "kvsroot.h"
-#include "kvssync.h"
+#include "kvs_wait_version.h"
+#include "kvs_checkpoint.h"
 
-/* Expire cache_entry after 'max_lastuse_age' heartbeats.
+/* heartbeat_sync_cb() is called periodically to manage cached content
+ * and namespaces.  Synchronize with the system heartbeat if possible,
+ * but keep the time between checks bounded by 'heartbeat_sync_min'
+ * and 'heartbeat_sync_max' seconds.
  */
-const int max_lastuse_age = 5;
+const double heartbeat_sync_min = 1.;
+const double heartbeat_sync_max = 30.;
 
-/* Expire namespaces after 'max_namespace_age' heartbeats.
- *
- * If heartbeats are the default of 2 seconds, 1000 heartbeats is
- * about half an hour.
+/* Expire cache_entry after 'max_lastuse_age' seconds.
  */
-const int max_namespace_age = 1000;
+const double max_lastuse_age = 10.;
 
-/* Include root directory in kvs.namespace-<NS>-setroot event.
+/* Expire namespaces after 'max_namespace_age' seconds.
  */
-const bool event_includes_rootdir = true;
+const double max_namespace_age = 3600.;
 
-typedef struct {
+struct kvs_ctx {
     struct cache *cache;    /* blobref => cache_entry */
     kvsroot_mgr_t *krm;
-    int faults;                 /* for kvs.stats.get, etc. */
+    int faults;                 /* for kvs.stats-get, etc. */
     flux_t *h;
     uint32_t rank;
-    int epoch;              /* tracks current heartbeat epoch */
     flux_watcher_t *prep_w;
     flux_watcher_t *idle_w;
     flux_watcher_t *check_w;
     int transaction_merge;
     bool events_init;            /* flag */
-    const char *hash_name;
+    char *hash_name;
     unsigned int seq;           /* for commit transactions */
-} kvs_ctx_t;
+    kvs_checkpoint_t *kcp;
+    zhashx_t *requests;         /* track unfinished requests */
+    struct list_head work_queue;
+};
 
 struct kvs_cb_data {
-    kvs_ctx_t *ctx;
+    struct kvs_ctx *ctx;
     struct kvsroot *root;
     wait_t *wait;
     int errnum;
-    bool ready;
-    char *sender;
+    const flux_msg_t *msg;
 };
 
-static void transaction_prep_cb (flux_reactor_t *r, flux_watcher_t *w,
-                                 int revents, void *arg);
-static void transaction_check_cb (flux_reactor_t *r, flux_watcher_t *w,
-                                  int revents, void *arg);
-static void start_root_remove (kvs_ctx_t *ctx, const char *ns);
+static void transaction_prep_cb (flux_reactor_t *r,
+                                 flux_watcher_t *w,
+                                 int revents,
+                                 void *arg);
+static void transaction_check_cb (flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg);
+static void start_root_remove (struct kvs_ctx *ctx, const char *ns);
+static void work_queue_check_append (struct kvs_ctx *ctx,
+                                     struct kvsroot *root);
+static void kvstxn_apply (kvstxn_t *kt);
 
 /*
- * kvs_ctx_t functions
+ * kvs_ctx functions
  */
-static void freectx (void *arg)
+static void kvs_ctx_destroy (struct kvs_ctx *ctx)
 {
-    kvs_ctx_t *ctx = arg;
     if (ctx) {
+        int saved_errno = errno;
         cache_destroy (ctx->cache);
         kvsroot_mgr_destroy (ctx->krm);
         flux_watcher_destroy (ctx->prep_w);
         flux_watcher_destroy (ctx->check_w);
         flux_watcher_destroy (ctx->idle_w);
+        kvs_checkpoint_destroy (ctx->kcp);
+        free (ctx->hash_name);
+        zhashx_destroy (&ctx->requests);
         free (ctx);
+        errno = saved_errno;
     }
 }
 
-static kvs_ctx_t *getctx (flux_t *h)
+static void request_tracking_add (struct kvs_ctx *ctx, const flux_msg_t *msg)
 {
-    kvs_ctx_t *ctx = (kvs_ctx_t *)flux_aux_get (h, "kvssrv");
-    flux_reactor_t *r;
-    int saved_errno;
+    /* ignore if item already tracked */
+    zhashx_insert (ctx->requests, msg, (flux_msg_t *)msg);
+}
 
-    if (!ctx) {
-        if (!(ctx = calloc (1, sizeof (*ctx)))) {
-            saved_errno = ENOMEM;
-            goto error;
-        }
-        if (!(r = flux_get_reactor (h))) {
-            saved_errno = errno;
-            goto error;
-        }
-        if (!(ctx->hash_name = flux_attr_get (h, "content.hash"))) {
-            saved_errno = errno;
-            flux_log_error (h, "content.hash");
-            goto error;
-        }
-        ctx->cache = cache_create ();
-        if (!ctx->cache) {
-            saved_errno = ENOMEM;
+static void request_tracking_remove (struct kvs_ctx *ctx, const flux_msg_t *msg)
+{
+    zhashx_delete (ctx->requests, msg);
+}
+
+static void work_queue_check_append_wrapper (struct kvsroot *root, void *arg)
+{
+    struct kvs_ctx *ctx = arg;
+    work_queue_check_append (ctx, root);
+}
+
+static struct kvs_ctx *kvs_ctx_create (flux_t *h)
+{
+    flux_reactor_t *r = flux_get_reactor (h);
+    struct kvs_ctx *ctx;
+    const char *s;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+    ctx->h = h;
+    if (!(s = flux_attr_get (h, "content.hash"))
+        || !(ctx->hash_name = strdup (s))) {
+        flux_log_error (h, "getattr content.hash");
+        goto error;
+    }
+    if (!(ctx->cache = cache_create (r)))
+        goto error;
+    if (!(ctx->krm = kvsroot_mgr_create (ctx->h, ctx)))
+        goto error;
+    if (flux_get_rank (ctx->h, &ctx->rank) < 0)
+        goto error;
+    if (ctx->rank == 0) {
+        ctx->prep_w = flux_prepare_watcher_create (r, transaction_prep_cb, ctx);
+        if (!ctx->prep_w)
             goto error;
-        }
-        if (!(ctx->krm = kvsroot_mgr_create (ctx->h, ctx))) {
-            saved_errno = ENOMEM;
+        ctx->check_w = flux_check_watcher_create (r, transaction_check_cb, ctx);
+        if (!ctx->check_w)
             goto error;
-        }
-        ctx->h = h;
-        if (flux_get_rank (h, &ctx->rank) < 0) {
-            saved_errno = errno;
+        ctx->idle_w = flux_idle_watcher_create (r, NULL, NULL);
+        if (!ctx->idle_w)
             goto error;
-        }
-        if (ctx->rank == 0) {
-            ctx->prep_w = flux_prepare_watcher_create (r, transaction_prep_cb, ctx);
-            if (!ctx->prep_w) {
-                saved_errno = errno;
-                goto error;
-            }
-            ctx->check_w = flux_check_watcher_create (r, transaction_check_cb, ctx);
-            if (!ctx->check_w) {
-                saved_errno = errno;
-                goto error;
-            }
-            ctx->idle_w = flux_idle_watcher_create (r, NULL, NULL);
-            if (!ctx->idle_w) {
-                saved_errno = errno;
-                goto error;
-            }
-            flux_watcher_start (ctx->prep_w);
-            flux_watcher_start (ctx->check_w);
-        }
-        ctx->transaction_merge = 1;
-        if (flux_aux_set (h, "kvssrv", ctx, freectx) < 0) {
-            saved_errno = errno;
+        flux_watcher_start (ctx->prep_w);
+        flux_watcher_start (ctx->check_w);
+        ctx->kcp = kvs_checkpoint_create (h,
+                                          NULL, /* set later */
+                                          0.0,  /* default 0.0, set later */
+                                          work_queue_check_append_wrapper,
+                                          ctx);
+        if (!ctx->kcp)
             goto error;
-        }
     }
+    ctx->transaction_merge = 1;
+    if (!(ctx->requests = msg_hash_create (MSG_HASH_TYPE_UUID_MATCHTAG)))
+        goto error;
+    list_head_init (&ctx->work_queue);
     return ctx;
 error:
-    freectx (ctx);
-    errno = saved_errno;
+    kvs_ctx_destroy (ctx);
     return NULL;
 }
 
@@ -172,7 +194,7 @@ static kvs_ctx_t *getctx (flux_t *h)
  * event subscribe/unsubscribe
  */
 
-static int event_subscribe (kvs_ctx_t *ctx, const char *ns)
+static int event_subscribe (struct kvs_ctx *ctx, const char *ns)
 {
     char *topic = NULL;
     int rc = -1;
@@ -207,8 +229,7 @@ static int event_subscribe (kvs_ctx_t *ctx, const char *ns)
         /* These belong to all namespaces, subscribe once the first
          * time we init a namespace */
 
-        if (flux_event_subscribe (ctx->h, "hb") < 0
-            || flux_event_subscribe (ctx->h, "kvs.stats.clear") < 0
+        if (flux_event_subscribe (ctx->h, "kvs.stats.clear") < 0
             || flux_event_subscribe (ctx->h, "kvs.dropcache") < 0) {
             flux_log_error (ctx->h, "flux_event_subscribe");
             goto cleanup;
@@ -243,7 +264,7 @@ static int event_subscribe (kvs_ctx_t *ctx, const char *ns)
     return rc;
 }
 
-static int event_unsubscribe (kvs_ctx_t *ctx, const char *ns)
+static int event_unsubscribe (struct kvs_ctx *ctx, const char *ns)
 {
     char *topic = NULL;
     int rc = -1;
@@ -268,7 +289,8 @@ static int event_unsubscribe (kvs_ctx_t *ctx, const char *ns)
  * security
  */
 
-static int check_user (kvs_ctx_t *ctx, struct kvsroot *root,
+static int check_user (struct kvs_ctx *ctx,
+                       struct kvsroot *root,
                        const flux_msg_t *msg)
 {
     struct flux_msg_cred cred;
@@ -285,19 +307,21 @@ static int check_user (kvs_ctx_t *ctx, struct kvsroot *root,
  * set/get root
  */
 
-static void setroot (kvs_ctx_t *ctx, struct kvsroot *root,
-                     const char *rootref, int rootseq)
+static void setroot (struct kvs_ctx *ctx,
+                     struct kvsroot *root,
+                     const char *rootref,
+                     int rootseq)
 {
     if (rootseq == 0 || rootseq > root->seq) {
         kvsroot_setroot (ctx->krm, root, rootref, rootseq);
-        kvssync_process (root, false);
-        root->last_update_epoch = ctx->epoch;
+        kvs_wait_version_process (root, false);
+        root->last_update_time = flux_reactor_now (flux_get_reactor (ctx->h));
     }
 }
 
 static void getroot_completion (flux_future_t *f, void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     flux_msg_t *msg = NULL;
     const char *ns;
     int rootseq, flags;
@@ -309,18 +333,14 @@ static void getroot_completion (flux_future_t *f, void *arg)
     msg = flux_future_aux_get (f, "msg");
     assert (msg);
 
-    if (flux_request_unpack (msg, NULL, "{ s:s }",
-                             "namespace", &ns) < 0) {
-        flux_log_error (ctx->h, "%s: flux_request_unpack", __FUNCTION__);
-        goto error;
-    }
-
     /* N.B. owner read into uint32_t */
-    if (flux_rpc_get_unpack (f, "{ s:i s:i s:s s:i }",
+    if (flux_rpc_get_unpack (f,
+                             "{ s:i s:i s:s s:i s:s }",
                              "owner", &owner,
                              "rootseq", &rootseq,
                              "rootref", &ref,
-                             "flags", &flags) < 0) {
+                             "flags", &flags,
+                             "namespace", &ns) < 0) {
         if (errno != ENOTSUP)
             flux_log_error (ctx->h, "%s: flux_rpc_get_unpack", __FUNCTION__);
         goto error;
@@ -355,34 +375,43 @@ static void getroot_completion (flux_future_t *f, void *arg)
     if (!root->remove)
         setroot (ctx, root, ref, rootseq);
 
-    /* flux_requeue_nocopy takes ownership of 'msg', no need to destroy */
-    if (flux_requeue_nocopy (ctx->h, msg, FLUX_RQ_HEAD) < 0) {
-        flux_log_error (ctx->h, "%s: flux_requeue_nocopy", __FUNCTION__);
+    if (flux_requeue (ctx->h, msg, FLUX_RQ_HEAD) < 0) {
+        flux_log_error (ctx->h, "%s: flux_requeue", __FUNCTION__);
         goto error;
     }
 
+    flux_msg_destroy (msg);
     flux_future_destroy (f);
     return;
 
 error:
     if (flux_respond_error (ctx->h, msg, errno, NULL) < 0)
         flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+    /* N.B. getroot request from other requests (e.g. lookup, commit)
+     * may stall and be tracked.  So we need to remove tracking of the
+     * request if there is an error.  We do not remove tracking on
+     * getroot success, as the original request (e.g. lookup, commit)
+     * will deal with the success case.
+     */
+    request_tracking_remove (ctx, msg);
     flux_msg_destroy (msg);
     flux_future_destroy (f);
 }
 
-static int getroot_request_send (kvs_ctx_t *ctx,
+static int getroot_request_send (struct kvs_ctx *ctx,
                                  const char *ns,
                                  flux_msg_handler_t *mh,
                                  const flux_msg_t *msg,
-                                 lookup_t *lh,
-                                 flux_msg_handler_f cb)
+                                 lookup_t *lh)
 {
     flux_future_t *f = NULL;
     flux_msg_t *msgcpy = NULL;
     int saved_errno;
 
-    if (!(f = flux_rpc_pack (ctx->h, "kvs.getroot", FLUX_NODEID_UPSTREAM, 0,
+    if (!(f = flux_rpc_pack (ctx->h,
+                             "kvs.getroot",
+                             FLUX_NODEID_UPSTREAM,
+                             0,
                              "{ s:s }",
                              "namespace", ns)))
         goto error;
@@ -416,11 +445,11 @@ static int getroot_request_send (kvs_ctx_t *ctx,
     return -1;
 }
 
-static struct kvsroot *getroot (kvs_ctx_t *ctx, const char *ns,
+static struct kvsroot *getroot (struct kvs_ctx *ctx,
+                                const char *ns,
                                 flux_msg_handler_t *mh,
                                 const flux_msg_t *msg,
                                 lookup_t *lh,
-                                flux_msg_handler_f cb,
                                 bool *stall)
 {
     struct kvsroot *root;
@@ -433,7 +462,7 @@ static struct kvsroot *getroot (kvs_ctx_t *ctx, const char *ns,
             return NULL;
         }
         else {
-            if (getroot_request_send (ctx, ns, mh, msg, lh, cb) < 0) {
+            if (getroot_request_send (ctx, ns, mh, msg, lh) < 0) {
                 flux_log_error (ctx->h, "getroot_request_send");
                 return NULL;
             }
@@ -452,7 +481,7 @@ static struct kvsroot *getroot (kvs_ctx_t *ctx, const char *ns,
  * load
  */
 
-static void content_load_cache_entry_error (kvs_ctx_t *ctx,
+static void content_load_cache_entry_error (struct kvs_ctx *ctx,
                                             struct cache_entry *entry,
                                             int errnum,
                                             const char *blobref)
@@ -470,9 +499,9 @@ static void content_load_cache_entry_error (kvs_ctx_t *ctx,
 
 static void content_load_completion (flux_future_t *f, void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     const void *data;
-    int size;
+    size_t size;
     const char *blobref;
     struct cache_entry *entry;
 
@@ -483,13 +512,13 @@ static void content_load_completion (flux_future_t *f, void *arg)
      * b/c it is not yet valid.  But check and log incase there is
      * logic error dealng with error paths using cache_remove_entry().
      */
-    if (!(entry = cache_lookup (ctx->cache, blobref, ctx->epoch))) {
+    if (!(entry = cache_lookup (ctx->cache, blobref))) {
         flux_log (ctx->h, LOG_ERR, "%s: cache_lookup", __FUNCTION__);
         goto done;
     }
 
-    if (flux_content_load_get (f, &data, &size) < 0) {
-        flux_log_error (ctx->h, "%s: flux_content_load_get", __FUNCTION__);
+    if (content_load_get (f, &data, &size) < 0) {
+        flux_log_error (ctx->h, "%s: content_load_get", __FUNCTION__);
         content_load_cache_entry_error (ctx, entry, errno, blobref);
         goto done;
     }
@@ -508,22 +537,20 @@ static void content_load_completion (flux_future_t *f, void *arg)
     flux_future_destroy (f);
 }
 
-/* Send content load request and setup contination to handle response.
+/* Send content load request and setup continuation to handle response.
  */
-static int content_load_request_send (kvs_ctx_t *ctx, const char *ref)
+static int content_load_request_send (struct kvs_ctx *ctx, const char *ref)
 {
     flux_future_t *f = NULL;
     char *refcpy;
     int saved_errno;
 
-    if (!(f = flux_content_load (ctx->h, ref, 0))) {
-        flux_log_error (ctx->h, "%s: flux_content_load", __FUNCTION__);
+    if (!(f = content_load_byblobref (ctx->h, ref, 0))) {
+        flux_log_error (ctx->h, "%s: content_load_byblobref", __FUNCTION__);
         goto error;
     }
-    if (!(refcpy = strdup (ref))) {
-        errno = ENOMEM;
+    if (!(refcpy = strdup (ref)))
         goto error;
-    }
     if (flux_future_aux_set (f, "ref", refcpy, free) < 0) {
         flux_log_error (ctx->h, "%s: flux_future_aux_set", __FUNCTION__);
         free (refcpy);
@@ -543,10 +570,14 @@ static int content_load_request_send (kvs_ctx_t *ctx, const char *ref)
 
 /* Return 0 on success, -1 on error.  Set stall variable appropriately
  */
-static int load (kvs_ctx_t *ctx, const char *ref, wait_t *wait, bool *stall)
+static int load (struct kvs_ctx *ctx,
+                 const char *ref,
+                 wait_t *wait,
+                 bool *stall)
 {
-    struct cache_entry *entry = cache_lookup (ctx->cache, ref, ctx->epoch);
-    int saved_errno, ret;
+    struct cache_entry *entry = cache_lookup (ctx->cache, ref);
+    int saved_errno;
+    __attribute__((unused)) int ret;
 
     assert (wait != NULL);
 
@@ -554,19 +585,18 @@ static int load (kvs_ctx_t *ctx, const char *ref, wait_t *wait, bool *stall)
      */
     if (!entry) {
         if (!(entry = cache_entry_create (ref))) {
-            flux_log_error (ctx->h, "%s: cache_entry_create",
-                            __FUNCTION__);
+            flux_log_error (ctx->h, "%s: cache_entry_create", __FUNCTION__);
             return -1;
         }
         if (cache_insert (ctx->cache, entry) < 0) {
-            flux_log_error (ctx->h, "%s: cache_insert",
-                            __FUNCTION__);
+            flux_log_error (ctx->h, "%s: cache_insert", __FUNCTION__);
             cache_entry_destroy (entry);
             return -1;
         }
         if (content_load_request_send (ctx, ref) < 0) {
             saved_errno = errno;
-            flux_log_error (ctx->h, "%s: content_load_request_send",
+            flux_log_error (ctx->h,
+                            "%s: content_load_request_send",
                             __FUNCTION__);
             /* cache entry just created, should always work */
             ret = cache_remove_entry (ctx->cache, ref);
@@ -584,7 +614,7 @@ static int load (kvs_ctx_t *ctx, const char *ref, wait_t *wait, bool *stall)
          * multiple times from the same kvstxn and on the same
          * reference, we're effectively adding identical waiters onto
          * this cache entry.  This is far better than sending multiple
-         * RPCs (the cache entry chck above protects against this),
+         * RPCs (the cache entry check above protects against this),
          * but could be improved later.  See Issue #1751.
          */
         if (cache_entry_wait_valid (entry, wait) < 0) {
@@ -611,16 +641,16 @@ static int load (kvs_ctx_t *ctx, const char *ref, wait_t *wait, bool *stall)
 
 static void content_store_completion (flux_future_t *f, void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     struct cache_entry *entry;
     const char *cache_blobref, *blobref;
-    int ret;
+    __attribute__((unused)) int ret;
 
     cache_blobref = flux_future_aux_get (f, "cache_blobref");
     assert (cache_blobref);
 
-    if (flux_content_store_get (f, &blobref) < 0) {
-        flux_log_error (ctx->h, "%s: flux_content_store_get", __FUNCTION__);
+    if (content_store_get_blobref (f, ctx->hash_name, &blobref) < 0) {
+        flux_log_error (ctx->h, "%s: content_store_get_blobref", __FUNCTION__);
         goto error;
     }
 
@@ -628,8 +658,9 @@ static void content_store_completion (flux_future_t *f, void *arg)
      * location we calculated.
      * N.B. perhaps this check is excessive and could be removed
      */
-    if (strcmp (blobref, cache_blobref)) {
-        flux_log (ctx->h, LOG_ERR, "%s: inconsistent blobref returned",
+    if (!streq (blobref, cache_blobref)) {
+        flux_log (ctx->h,
+                  LOG_ERR, "%s: inconsistent blobref returned",
                   __FUNCTION__);
         errno = EPROTO;
         goto error;
@@ -641,7 +672,7 @@ static void content_store_completion (flux_future_t *f, void *arg)
      * b/c it was dirty.  But check and log incase there is logic
      * error dealng with error paths using cache_remove_entry().
      */
-    if (!(entry = cache_lookup (ctx->cache, blobref, ctx->epoch))) {
+    if (!(entry = cache_lookup (ctx->cache, blobref))) {
         flux_log (ctx->h, LOG_ERR, "%s: cache_lookup", __FUNCTION__);
         goto error;
     }
@@ -654,8 +685,7 @@ static void content_store_completion (flux_future_t *f, void *arg)
      * work as well.
      */
     if (cache_entry_set_dirty (entry, false) < 0) {
-        flux_log_error (ctx->h, "%s: cache_entry_set_dirty",
-                        __FUNCTION__);
+        flux_log_error (ctx->h, "%s: cache_entry_set_dirty", __FUNCTION__);
         goto error;
     }
 
@@ -673,7 +703,7 @@ static void content_store_completion (flux_future_t *f, void *arg)
      */
 
     /* we can't do anything if this cache_lookup fails */
-    if (!(entry = cache_lookup (ctx->cache, cache_blobref, ctx->epoch))) {
+    if (!(entry = cache_lookup (ctx->cache, cache_blobref))) {
         flux_log (ctx->h, LOG_ERR, "%s: cache_lookup", __FUNCTION__);
         return;
     }
@@ -685,7 +715,9 @@ static void content_store_completion (flux_future_t *f, void *arg)
      * must call cache_entry_force_clear_dirty().  flushed.
      */
     if (cache_entry_set_errnum_on_notdirty (entry, errno) < 0) {
-        flux_log (ctx->h, LOG_ERR, "%s: cache_entry_set_errnum_on_notdirty",
+        flux_log (ctx->h,
+                  LOG_ERR,
+                  "%s: cache_entry_set_errnum_on_notdirty",
                   __FUNCTION__);
         ret = cache_entry_force_clear_dirty (entry);
         assert (ret == 0);
@@ -700,13 +732,15 @@ static void content_store_completion (flux_future_t *f, void *arg)
         flux_log (ctx->h, LOG_ERR, "%s: cache_remove_entry", __FUNCTION__);
 }
 
-static int content_store_request_send (kvs_ctx_t *ctx, const char *blobref,
-                                       const void *data, int len)
+static int content_store_request_send (struct kvs_ctx *ctx,
+                                       const char *blobref,
+                                       const void *data,
+                                       int len)
 {
     flux_future_t *f;
     int saved_errno, rc = -1;
 
-    if (!(f = flux_content_store (ctx->h, data, len, 0)))
+    if (!(f = content_store (ctx->h, data, len, 0)))
         goto error;
     if (flux_future_aux_set (f, "cache_blobref", (void *)blobref, NULL) < 0) {
         saved_errno = errno;
@@ -754,7 +788,8 @@ static int kvstxn_cache_cb (kvstxn_t *kt, struct cache_entry *entry, void *data)
     assert (cache_entry_get_dirty (entry));
 
     if (cache_entry_get_raw (entry, &storedata, &storedatalen) < 0) {
-        flux_log_error (cbd->ctx->h, "%s: cache_entry_get_raw",
+        flux_log_error (cbd->ctx->h,
+                        "%s: cache_entry_get_raw",
                         __FUNCTION__);
         kvstxn_cleanup_dirty_cache_entry (kt, entry);
         return -1;
@@ -769,7 +804,8 @@ static int kvstxn_cache_cb (kvstxn_t *kt, struct cache_entry *entry, void *data)
                                     storedata,
                                     storedatalen) < 0) {
         cbd->errnum = errno;
-        flux_log_error (cbd->ctx->h, "%s: content_store_request_send",
+        flux_log_error (cbd->ctx->h,
+                        "%s: content_store_request_send",
                         __FUNCTION__);
         kvstxn_cleanup_dirty_cache_entry (kt, entry);
         return -1;
@@ -783,52 +819,31 @@ static int kvstxn_cache_cb (kvstxn_t *kt, struct cache_entry *entry, void *data)
     return 0;
 }
 
-static void flux_msg_destroy_wrapper (void *arg)
-{
-    flux_msg_t *msg = arg;
-    flux_msg_destroy (msg);
-}
-
-static int setroot_event_send (kvs_ctx_t *ctx, struct kvsroot *root,
-                               json_t *names, json_t *keys)
+static int setroot_event_send (struct kvs_ctx *ctx,
+                               struct kvsroot *root,
+                               json_t *names,
+                               json_t *keys)
 {
-    const json_t *root_dir = NULL;
-    json_t *nullobj = NULL;
     flux_msg_t *msg = NULL;
     char *setroot_topic = NULL;
     int saved_errno, rc = -1;
 
     assert (ctx->rank == 0);
 
-    if (event_includes_rootdir) {
-        struct cache_entry *entry;
-
-        if ((entry = cache_lookup (ctx->cache, root->ref, ctx->epoch)))
-            root_dir = cache_entry_get_treeobj (entry);
-        assert (root_dir != NULL); // root entry is always in cache on rank 0
-    }
-    else {
-        if (!(nullobj = json_null ())) {
-            saved_errno = errno;
-            flux_log_error (ctx->h, "%s: json_null", __FUNCTION__);
-            goto done;
-        }
-        root_dir = nullobj;
-    }
-
-    if (asprintf (&setroot_topic, "kvs.namespace-%s-setroot", root->ns_name) < 0) {
-        saved_errno = ENOMEM;
+    if (asprintf (&setroot_topic,
+                  "kvs.namespace-%s-setroot",
+                  root->ns_name) < 0) {
+        saved_errno = errno;
         flux_log_error (ctx->h, "%s: asprintf", __FUNCTION__);
         goto done;
     }
 
     if (!(msg = flux_event_pack (setroot_topic,
-                                 "{ s:s s:i s:s s:O s:O s:O s:i}",
+                                 "{ s:s s:i s:s s:O s:O s:i}",
                                  "namespace", root->ns_name,
                                  "rootseq", root->seq,
                                  "rootref", root->ref,
                                  "names", names,
-                                 "rootdir", root_dir,
                                  "keys", keys,
                                  "owner", root->owner))) {
         saved_errno = errno;
@@ -847,26 +862,28 @@ static int setroot_event_send (kvs_ctx_t *ctx, struct kvsroot *root,
 done:
     free (setroot_topic);
     flux_msg_destroy (msg);
-    json_decref (nullobj);
     if (rc < 0)
         errno = saved_errno;
     return rc;
 }
 
-static int error_event_send (kvs_ctx_t *ctx, const char *ns,
-                             json_t *names, int errnum)
+static int error_event_send (struct kvs_ctx *ctx,
+                             const char *ns,
+                             json_t *names,
+                             int errnum)
 {
     flux_msg_t *msg = NULL;
     char *error_topic = NULL;
     int saved_errno, rc = -1;
 
     if (asprintf (&error_topic, "kvs.namespace-%s-error", ns) < 0) {
-        saved_errno = ENOMEM;
+        saved_errno = errno;
         flux_log_error (ctx->h, "%s: asprintf", __FUNCTION__);
         goto done;
     }
 
-    if (!(msg = flux_event_pack (error_topic, "{ s:s s:O s:i }",
+    if (!(msg = flux_event_pack (error_topic,
+                                 "{ s:s s:O s:i }",
                                  "namespace", ns,
                                  "names", names,
                                  "errnum", errnum))) {
@@ -891,15 +908,17 @@ static int error_event_send (kvs_ctx_t *ctx, const char *ns,
     return rc;
 }
 
-static int error_event_send_to_name (kvs_ctx_t *ctx, const char *ns,
-                                     const char *name, int errnum)
+static int error_event_send_to_name (struct kvs_ctx *ctx,
+                                     const char *ns,
+                                     const char *name,
+                                     int errnum)
 {
     json_t *names = NULL;
     int rc = -1;
 
     if (!(names = json_pack ("[ s ]", name))) {
-        flux_log_error (ctx->h, "%s: json_pack", __FUNCTION__);
         errno = ENOMEM;
+        flux_log_error (ctx->h, "%s: json_pack", __FUNCTION__);
         goto done;
     }
 
@@ -915,6 +934,36 @@ static void kvstxn_wait_error_cb (wait_t *w, int errnum, void *arg)
     kvstxn_set_aux_errnum (kt, errnum);
 }
 
+/* ccan list doesn't appear to have a macro to check if a node exists
+ * on a list */
+static inline bool root_on_work_queue (struct list_node *n)
+{
+    return !(n->next == n->prev && n->next == n);
+}
+
+static void work_queue_append (struct kvs_ctx *ctx, struct kvsroot *root)
+{
+    if (!root_on_work_queue (&root->work_queue_node))
+        list_add_tail (&ctx->work_queue, &root->work_queue_node);
+}
+
+static void work_queue_remove (struct kvsroot *root)
+{
+    list_del_init (&root->work_queue_node);
+}
+
+static void work_queue_check_append (struct kvs_ctx *ctx, struct kvsroot *root)
+{
+    if (kvstxn_mgr_transaction_ready (root->ktm))
+        work_queue_append (ctx, root);
+}
+
+static void kvstxn_apply_cb (flux_future_t *f, void *arg)
+{
+    kvstxn_t *kt = arg;
+    kvstxn_apply (kt);
+}
+
 /* Write all the ops for a particular commit/fence request (rank 0
  * only).  The setroot event will cause responses to be sent to the
  * transaction requests and clean up the treq_t state.  This
@@ -922,7 +971,7 @@ static void kvstxn_wait_error_cb (wait_t *w, int errnum, void *arg)
  */
 static void kvstxn_apply (kvstxn_t *kt)
 {
-    kvs_ctx_t *ctx = kvstxn_get_aux (kt);
+    struct kvs_ctx *ctx = kvstxn_get_aux (kt);
     const char *ns;
     struct kvsroot *root = NULL;
     wait_t *wait = NULL;
@@ -945,8 +994,6 @@ static void kvstxn_apply (kvstxn_t *kt)
     assert (root);
 
     if (root->remove) {
-        flux_log (ctx->h, LOG_DEBUG, "%s: namespace %s removed", __FUNCTION__,
-                  ns);
         errnum = ENOTSUP;
         goto done;
     }
@@ -955,8 +1002,8 @@ static void kvstxn_apply (kvstxn_t *kt)
         goto done;
 
     if ((ret = kvstxn_process (kt,
-                               ctx->epoch,
-                               root->ref)) == KVSTXN_PROCESS_ERROR) {
+                               root->ref,
+                               root->seq)) == KVSTXN_PROCESS_ERROR) {
         errnum = kvstxn_get_errnum (kt);
         goto done;
     }
@@ -1021,6 +1068,34 @@ static void kvstxn_apply (kvstxn_t *kt)
         assert (wait_get_usecount (wait) > 0);
         goto stall;
     }
+    else if (ret == KVSTXN_PROCESS_SYNC_CONTENT_FLUSH) {
+        /* N.B. futre is managed by kvstxn, should not call
+         * flux_future_destroy() on it */
+        flux_future_t *f = kvstxn_sync_content_flush (kt);
+        if (!f) {
+            errnum = errno;
+            goto done;
+        }
+        if (flux_future_then (f, -1., kvstxn_apply_cb, kt) < 0) {
+            errnum = errno;
+            goto done;
+        }
+        goto stall;
+    }
+    else if (ret == KVSTXN_PROCESS_SYNC_CHECKPOINT) {
+        /* N.B. futre is managed by kvstxn, should not call
+         * flux_future_destroy() on it */
+        flux_future_t *f = kvstxn_sync_checkpoint (kt);
+        if (!f) {
+            errnum = errno;
+            goto done;
+        }
+        if (flux_future_then (f, -1., kvstxn_apply_cb, kt) < 0) {
+            errnum = errno;
+            goto done;
+        }
+        goto stall;
+    }
     /* else ret == KVSTXN_PROCESS_FINISHED */
 
     /* This finalizes the transaction by replacing root->ref with
@@ -1030,16 +1105,23 @@ static void kvstxn_apply (kvstxn_t *kt)
 done:
     if (errnum == 0) {
         json_t *names = kvstxn_get_names (kt);
+        int internal_flags = kvstxn_get_internal_flags (kt);
         int count;
         if ((count = json_array_size (names)) > 1) {
             int opcount = 0;
             opcount = json_array_size (kvstxn_get_ops (kt));
-            flux_log (ctx->h, LOG_DEBUG, "aggregated %d transactions (%d ops)",
-                      count, opcount);
+            flux_log (ctx->h,
+                      LOG_DEBUG,
+                      "aggregated %d transactions (%d ops)",
+                      count,
+                      opcount);
+        }
+        if (!(internal_flags & KVSTXN_INTERNAL_FLAG_NO_PUBLISH)) {
+            setroot (ctx, root, kvstxn_get_newroot_ref (kt), root->seq + 1);
+            setroot_event_send (ctx, root, names, kvstxn_get_keys (kt));
         }
-        setroot (ctx, root, kvstxn_get_newroot_ref (kt), root->seq + 1);
-        setroot_event_send (ctx, root, names, kvstxn_get_keys (kt));
-    } else {
+    }
+    else {
         fallback = kvstxn_fallback_mergeable (kt);
 
         /* if merged transaction is fallbackable, ignore the fallback option
@@ -1048,9 +1130,12 @@ static void kvstxn_apply (kvstxn_t *kt)
         if (errnum == ENOMEM || errnum == ENOTSUP)
             fallback = false;
 
-        if (!fallback)
-            error_event_send (ctx, root->ns_name, kvstxn_get_names (kt),
+        if (!fallback) {
+            error_event_send (ctx,
+                              root->ns_name,
+                              kvstxn_get_names (kt),
                               errnum);
+        }
     }
     wait_destroy (wait);
 
@@ -1058,9 +1143,12 @@ static void kvstxn_apply (kvstxn_t *kt)
      * N.B. treq_t remains in the treq_mgr_t hash until event is received.
      */
     kvstxn_mgr_remove_transaction (root->ktm, kt, fallback);
-    return;
 
 stall:
+    if (kvstxn_mgr_transaction_ready (root->ktm))
+        work_queue_append (ctx, root);
+    else
+        work_queue_remove (root);
     return;
 }
 
@@ -1068,42 +1156,26 @@ static void kvstxn_apply (kvstxn_t *kt)
  * pre/check event callbacks
  */
 
-static int kvstxn_prep_root_cb (struct kvsroot *root, void *arg)
-{
-    struct kvs_cb_data *cbd = arg;
-
-    if (kvstxn_mgr_transaction_ready (root->ktm)) {
-        cbd->ready = true;
-        return 1;
-    }
-
-    return 0;
-}
-
-static void transaction_prep_cb (flux_reactor_t *r, flux_watcher_t *w,
-                                 int revents, void *arg)
+static void transaction_prep_cb (flux_reactor_t *r,
+                                 flux_watcher_t *w,
+                                 int revents,
+                                 void *arg)
 {
-    kvs_ctx_t *ctx = arg;
-    struct kvs_cb_data cbd = { .ctx = ctx, .ready = false };
-
-    if (kvsroot_mgr_iter_roots (ctx->krm, kvstxn_prep_root_cb, &cbd) < 0) {
-        flux_log_error (ctx->h, "%s: kvsroot_mgr_iter_roots", __FUNCTION__);
-        return;
-    }
+    struct kvs_ctx *ctx = arg;
 
-    if (cbd.ready)
+    if (!list_empty (&ctx->work_queue))
         flux_watcher_start (ctx->idle_w);
 }
 
-static int kvstxn_check_root_cb (struct kvsroot *root, void *arg)
+static void kvstxn_check_root_cb (struct kvsroot *root, void *arg)
 {
-    struct kvs_cb_data *cbd = arg;
+    struct kvs_ctx *ctx = arg;
     kvstxn_t *kt;
 
     if ((kt = kvstxn_mgr_get_ready_transaction (root->ktm))) {
-        if (cbd->ctx->transaction_merge) {
-            /* if merge fails, set errnum in txn_t, let
-             * txn_apply() handle error handling.
+        if (ctx->transaction_merge) {
+            /* if merge fails, set errnum in kvstxn_t, let
+             * kvstxn_apply() handle error handling.
              */
             if (kvstxn_mgr_merge_ready_transactions (root->ktm) < 0)
                 kvstxn_set_aux_errnum (kt, errno);
@@ -1121,22 +1193,21 @@ static int kvstxn_check_root_cb (struct kvsroot *root, void *arg)
          */
         kvstxn_apply (kt);
     }
-
-    return 0;
 }
 
-static void transaction_check_cb (flux_reactor_t *r, flux_watcher_t *w,
-                                  int revents, void *arg)
+static void transaction_check_cb (flux_reactor_t *r,
+                                  flux_watcher_t *w,
+                                  int revents,
+                                  void *arg)
 {
-    kvs_ctx_t *ctx = arg;
-    struct kvs_cb_data cbd = { .ctx = ctx, .ready = false };
+    struct kvs_ctx *ctx = arg;
+    struct kvsroot *root = NULL;
+    struct kvsroot *next = NULL;
 
     flux_watcher_stop (ctx->idle_w);
 
-    if (kvsroot_mgr_iter_roots (ctx->krm, kvstxn_check_root_cb, &cbd) < 0) {
-        flux_log_error (ctx->h, "%s: kvsroot_mgr_iter_roots", __FUNCTION__);
-        return;
-    }
+    list_for_each_safe (&ctx->work_queue, root, next, work_queue_node)
+        kvstxn_check_root_cb (root, ctx);
 }
 
 /*
@@ -1146,7 +1217,7 @@ static void transaction_check_cb (flux_reactor_t *r, flux_watcher_t *w,
 static void dropcache_request_cb (flux_t *h, flux_msg_handler_t *mh,
                                   const flux_msg_t *msg, void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     int size, expcount = 0;
 
     /* irrelevant if root not initialized, drop cache entries */
@@ -1154,13 +1225,17 @@ static void dropcache_request_cb (flux_t *h, flux_msg_handler_t *mh,
     if (flux_request_decode (msg, NULL, NULL) < 0)
         goto error;
     size = cache_count_entries (ctx->cache);
-    if ((expcount = cache_expire_entries (ctx->cache, ctx->epoch, 0)) < 0) {
+    if ((expcount = cache_expire_entries (ctx->cache, 0)) < 0) {
         flux_log_error (ctx->h, "%s: cache_expire_entries", __FUNCTION__);
         goto error;
     }
-    else
-        flux_log (h, LOG_ALERT, "dropped %d of %d cache entries",
-                  expcount, size);
+    else {
+        flux_log (h,
+                  LOG_ALERT,
+                  "dropped %d of %d cache entries",
+                  expcount,
+                  size);
+    }
     if (flux_respond (h, msg, NULL) < 0)
         flux_log_error (h, "%s: flux_respond", __FUNCTION__);
     return;
@@ -1169,10 +1244,12 @@ static void dropcache_request_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
 }
 
-static void dropcache_event_cb (flux_t *h, flux_msg_handler_t *mh,
-                                const flux_msg_t *msg, void *arg)
+static void dropcache_event_cb (flux_t *h,
+                                flux_msg_handler_t *mh,
+                                const flux_msg_t *msg,
+                                void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     int size, expcount = 0;
 
     /* irrelevant if root not initialized, drop cache entries */
@@ -1182,38 +1259,44 @@ static void dropcache_event_cb (flux_t *h, flux_msg_handler_t *mh,
         return;
     }
     size = cache_count_entries (ctx->cache);
-    if ((expcount = cache_expire_entries (ctx->cache, ctx->epoch, 0)) < 0)
+    if ((expcount = cache_expire_entries (ctx->cache, 0)) < 0)
         flux_log_error (ctx->h, "%s: cache_expire_entries", __FUNCTION__);
-    else
-        flux_log (h, LOG_ALERT, "dropped %d of %d cache entries",
-                  expcount, size);
+    else {
+        flux_log (h,
+                  LOG_ALERT,
+                  "dropped %d of %d cache entries",
+                  expcount,
+                  size);
+    }
 }
 
 static int heartbeat_root_cb (struct kvsroot *root, void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
+    double now = flux_reactor_now (flux_get_reactor (ctx->h));
 
     if (root->remove) {
-        if (!zlist_size (root->synclist)
+        if (!zlist_size (root->wait_version_list)
             && !treq_mgr_transactions_count (root->trm)
             && !kvstxn_mgr_ready_transaction_count (root->ktm)) {
 
             if (event_unsubscribe (ctx, root->ns_name) < 0)
-                flux_log_error (ctx->h, "%s: event_unsubscribe",
-                                __FUNCTION__);
+                flux_log_error (ctx->h, "%s: event_unsubscribe", __FUNCTION__);
 
-            if (kvsroot_mgr_remove_root (ctx->krm, root->ns_name) < 0)
-                flux_log_error (ctx->h, "%s: kvsroot_mgr_remove_root",
+            if (kvsroot_mgr_remove_root (ctx->krm, root->ns_name) < 0) {
+                flux_log_error (ctx->h,
+                                "%s: kvsroot_mgr_remove_root",
                                 __FUNCTION__);
+            }
         }
     }
     else if (ctx->rank != 0
-             && !root->remove
-             && strcasecmp (root->ns_name, KVS_PRIMARY_NAMESPACE)
-             && (ctx->epoch - root->last_update_epoch) > max_namespace_age
-             && !zlist_size (root->synclist)
-             && !treq_mgr_transactions_count (root->trm)
-             && !kvstxn_mgr_ready_transaction_count (root->ktm)) {
+        && !root->remove
+        && !root->is_primary
+        && (now - root->last_update_time) > max_namespace_age
+        && !zlist_size (root->wait_version_list)
+        && !treq_mgr_transactions_count (root->trm)
+        && !kvstxn_mgr_ready_transaction_count (root->ktm)) {
         /* remove a root if it not the primary one, has timed out
          * on a follower node, and it does not have any watchers,
          * and no one is trying to write/change something.
@@ -1221,27 +1304,23 @@ static int heartbeat_root_cb (struct kvsroot *root, void *arg)
         start_root_remove (ctx, root->ns_name);
     }
     else /* "touch" root */
-        (void)cache_lookup (ctx->cache, root->ref, ctx->epoch);
+        (void)cache_lookup (ctx->cache, root->ref);
 
     return 0;
 }
 
-static void heartbeat_cb (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg)
+static void heartbeat_sync_cb (flux_future_t *f, void *arg)
 {
-    kvs_ctx_t *ctx = arg;
-
-    if (flux_heartbeat_decode (msg, &ctx->epoch) < 0) {
-        flux_log_error (ctx->h, "%s: flux_heartbeat_decode", __FUNCTION__);
-        return;
-    }
+    struct kvs_ctx *ctx = arg;
 
     /* don't error return, fallthrough to deal with rest as necessary */
     if (kvsroot_mgr_iter_roots (ctx->krm, heartbeat_root_cb, ctx) < 0)
         flux_log_error (ctx->h, "%s: kvsroot_mgr_iter_roots", __FUNCTION__);
 
-    if (cache_expire_entries (ctx->cache, ctx->epoch, max_lastuse_age) < 0)
+    if (cache_expire_entries (ctx->cache, max_lastuse_age) < 0)
         flux_log_error (ctx->h, "%s: cache_expire_entries", __FUNCTION__);
+
+    flux_future_reset (f);
 }
 
 static int lookup_load_cb (lookup_t *lh, const char *ref, void *data)
@@ -1265,12 +1344,13 @@ static void lookup_wait_error_cb (wait_t *w, int errnum, void *arg)
     lookup_set_aux_errnum (lh, errnum);
 }
 
-static lookup_t *lookup_common (flux_t *h, flux_msg_handler_t *mh,
-                                const flux_msg_t *msg, void *arg,
+static lookup_t *lookup_common (flux_t *h,
+                                flux_msg_handler_t *mh,
+                                const flux_msg_t *msg,
+                                struct kvs_ctx *ctx,
                                 flux_msg_handler_f replay_cb,
                                 bool *stall)
 {
-    kvs_ctx_t *ctx = arg;
     int flags;
     const char *ns = NULL;
     const char *key;
@@ -1281,7 +1361,6 @@ static lookup_t *lookup_common (flux_t *h, flux_msg_handler_t *mh,
     wait_t *wait = NULL;
     lookup_process_t lret;
     int rc = -1;
-    int ret;
 
     /* if lookup_handle exists in msg as aux data, is a replay */
     lh = flux_msg_aux_get (msg, "lookup_handle");
@@ -1289,25 +1368,19 @@ static lookup_t *lookup_common (flux_t *h, flux_msg_handler_t *mh,
         struct flux_msg_cred cred;
         int root_seq = -1;
 
-        if (flux_request_unpack (msg, NULL, "{ s:s s:i }",
+        /* namespace, rootdir, and rootseq optional */
+        if (flux_request_unpack (msg,
+                                 NULL,
+                                 "{ s:s s:i s?s s?o s?i}",
                                  "key", &key,
-                                 "flags", &flags) < 0) {
+                                 "flags", &flags,
+                                 "namespace", &ns,
+                                 "rootdir", &root_dirent,
+                                 "rootseq", &root_seq) < 0) {
             flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
             goto done;
         }
 
-        /* namespace is optional */
-        (void)flux_request_unpack (msg, NULL, "{ s:s }",
-                                   "namespace", &ns);
-
-        /* rootdir is optional */
-        (void)flux_request_unpack (msg, NULL, "{ s:o }",
-                                   "rootdir", &root_dirent);
-
-        /* rootseq is optional */
-        (void)flux_request_unpack (msg, NULL, "{ s:i }",
-                                   "rootseq", &root_seq);
-
         /* either namespace or rootdir must be specified */
         if (!ns && !root_dirent) {
             errno = EPROTO;
@@ -1333,7 +1406,6 @@ static lookup_t *lookup_common (flux_t *h, flux_msg_handler_t *mh,
 
         if (!(lh = lookup_create (ctx->cache,
                                   ctx->krm,
-                                  ctx->epoch,
                                   ns,
                                   root_ref,
                                   root_seq,
@@ -1351,9 +1423,6 @@ static lookup_t *lookup_common (flux_t *h, flux_msg_handler_t *mh,
             errno = err;
             goto done;
         }
-
-        ret = lookup_set_current_epoch (lh, ctx->epoch);
-        assert (ret == 0);
     }
 
     lret = lookup (lh);
@@ -1364,12 +1433,12 @@ static lookup_t *lookup_common (flux_t *h, flux_msg_handler_t *mh,
     }
     else if (lret == LOOKUP_PROCESS_LOAD_MISSING_NAMESPACE) {
         bool stall = false;
-        struct kvsroot *root;
+        __attribute__((unused)) struct kvsroot *root;
 
         ns = lookup_missing_namespace (lh);
         assert (ns);
 
-        root = getroot (ctx, ns, mh, msg, lh, replay_cb, &stall);
+        root = getroot (ctx, ns, mh, msg, lh, &stall);
         assert (!root);
 
         if (stall)
@@ -1379,8 +1448,7 @@ static lookup_t *lookup_common (flux_t *h, flux_msg_handler_t *mh,
     else if (lret == LOOKUP_PROCESS_LOAD_MISSING_REFS) {
         struct kvs_cb_data cbd;
 
-        if (!(wait = wait_create_msg_handler (h, mh, msg, ctx,
-                                              replay_cb)))
+        if (!(wait = wait_create_msg_handler (h, mh, msg, ctx, replay_cb)))
             goto done;
 
         if (wait_set_error_cb (wait, lookup_wait_error_cb, lh) < 0)
@@ -1426,17 +1494,21 @@ static lookup_t *lookup_common (flux_t *h, flux_msg_handler_t *mh,
     return NULL;
 }
 
-static void lookup_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                               const flux_msg_t *msg, void *arg)
+static void lookup_request_cb (flux_t *h,
+                               flux_msg_handler_t *mh,
+                               const flux_msg_t *msg,
+                               void *arg)
 {
+    struct kvs_ctx *ctx = arg;
     lookup_t *lh;
     json_t *val;
     bool stall = false;
 
-    if (!(lh = lookup_common (h, mh, msg, arg, lookup_request_cb,
-                              &stall))) {
-        if (stall)
+    if (!(lh = lookup_common (h, mh, msg, ctx, lookup_request_cb, &stall))) {
+        if (stall) {
+            request_tracking_add (ctx, msg);
             return;
+        }
         goto error;
     }
 
@@ -1448,10 +1520,12 @@ static void lookup_request_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
     lookup_destroy (lh);
     json_decref (val);
+    request_tracking_remove (ctx, msg);
     return;
 error:
     if (flux_respond_error (h, msg, errno, NULL) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    request_tracking_remove (ctx, msg);
     lookup_destroy (lh);
 }
 
@@ -1462,19 +1536,28 @@ static void lookup_request_cb (flux_t *h, flux_msg_handler_t *mh,
  * on lookups (including ENOENT failed lookups) to determine what
  * lookups can be considered to be read-your-writes consistency safe.
  */
-static void lookup_plus_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                    const flux_msg_t *msg, void *arg)
+static void lookup_plus_request_cb (flux_t *h,
+                                    flux_msg_handler_t *mh,
+                                    const flux_msg_t *msg,
+                                    void *arg)
 {
+    struct kvs_ctx *ctx = arg;
     lookup_t *lh;
     json_t *val = NULL;
     const char *root_ref;
     int root_seq;
     bool stall = false;
 
-    if (!(lh = lookup_common (h, mh, msg, arg, lookup_plus_request_cb,
+    if (!(lh = lookup_common (h,
+                              mh,
+                              msg,
+                              ctx,
+                              lookup_plus_request_cb,
                               &stall))) {
-        if (stall)
+        if (stall) {
+            request_tracking_add (ctx, msg);
             return;
+        }
         goto error;
     }
 
@@ -1484,14 +1567,18 @@ static void lookup_plus_request_cb (flux_t *h, flux_msg_handler_t *mh,
     assert (root_seq >= 0);
 
     if (!(val = lookup_get_value (lh))) {
-        if (flux_respond_pack (h, msg, "{ s:i s:i s:s }",
+        if (flux_respond_pack (h,
+                               msg,
+                               "{ s:i s:i s:s }",
                                "errno", ENOENT,
                                "rootseq", root_seq,
                                "rootref", root_ref) < 0)
             flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
     }
     else {
-        if (flux_respond_pack (h, msg, "{ s:O s:i s:s }",
+        if (flux_respond_pack (h,
+                               msg,
+                               "{ s:O s:i s:s }",
                                "val", val,
                                "rootseq", root_seq,
                                "rootref", root_ref) < 0)
@@ -1499,10 +1586,12 @@ static void lookup_plus_request_cb (flux_t *h, flux_msg_handler_t *mh,
     }
     lookup_destroy (lh);
     json_decref (val);
+    request_tracking_remove (ctx, msg);
     return;
 error:
     if (flux_respond_error (h, msg, errno, NULL) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    request_tracking_remove (ctx, msg);
 }
 
 
@@ -1513,20 +1602,27 @@ static int finalize_transaction_req (treq_t *tr,
     struct kvs_cb_data *cbd = data;
 
     if (cbd->errnum) {
-        if (flux_respond_error (cbd->ctx->h, req, cbd->errnum, NULL) < 0)
-            flux_log_error (cbd->ctx->h, "%s: flux_respond_error", __FUNCTION__);
+        if (flux_respond_error (cbd->ctx->h, req, cbd->errnum, NULL) < 0) {
+            flux_log_error (cbd->ctx->h,
+                            "%s: flux_respond_error",
+                            __FUNCTION__);
+        }
     }
     else {
-        if (flux_respond_pack (cbd->ctx->h, req, "{ s:s s:i }",
+        if (flux_respond_pack (cbd->ctx->h,
+                               req,
+                               "{ s:s s:i }",
                                "rootref", cbd->root->ref,
                                "rootseq", cbd->root->seq) < 0)
             flux_log_error (cbd->ctx->h, "%s: flux_respond_pack", __FUNCTION__);
     }
 
+    request_tracking_remove (cbd->ctx, req);
     return 0;
 }
 
-static void finalize_transaction_bynames (kvs_ctx_t *ctx, struct kvsroot *root,
+static void finalize_transaction_bynames (struct kvs_ctx *ctx,
+                                          struct kvsroot *root,
                                           json_t *names, int errnum)
 {
     int i, len;
@@ -1547,26 +1643,65 @@ static void finalize_transaction_bynames (kvs_ctx_t *ctx, struct kvsroot *root,
         nameval = json_string_value (name);
         if ((tr = treq_mgr_lookup_transaction (root->trm, nameval))) {
             treq_iter_request_copies (tr, finalize_transaction_req, &cbd);
-            if (treq_mgr_remove_transaction (root->trm, nameval) < 0)
-                flux_log_error (ctx->h, "%s: treq_mgr_remove_transaction",
+            if (treq_mgr_remove_transaction (root->trm, nameval) < 0) {
+                flux_log_error (ctx->h,
+                                "%s: treq_mgr_remove_transaction",
                                 __FUNCTION__);
+            }
         }
     }
 }
 
+static int guest_treeobj_authorize (json_t *treeobj, flux_error_t *error)
+{
+    if (json_is_null (treeobj)
+        || treeobj_is_val (treeobj)
+        || (treeobj_is_dir (treeobj) && treeobj_get_count (treeobj) == 0))
+        return 0;
+
+    const char *type = treeobj_get_type (treeobj);
+    errprintf (error,
+               "guests may not commit %s%s objects",
+               treeobj_is_dir (treeobj) ? "non-empty " : "",
+               type ? type : "???");
+    errno = EPERM;
+    return -1;
+}
+
+static int guest_commit_authorize (json_t *ops, flux_error_t *error)
+{
+    size_t index;
+    json_t *op;
+
+    json_array_foreach (ops, index, op) {
+        json_t *treeobj;
+        if (txn_decode_op (op, NULL, NULL, &treeobj) < 0) {
+            errprintf (error, "could not decode commit operation");
+            return -1;
+        }
+        if (guest_treeobj_authorize (treeobj, error) < 0)
+            return -1;
+    }
+    return 0;
+}
+
 /* kvs.relaycommit (rank 0 only, no response).
  */
-static void relaycommit_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                    const flux_msg_t *msg, void *arg)
+static void relaycommit_request_cb (flux_t *h,
+                                    flux_msg_handler_t *mh,
+                                    const flux_msg_t *msg,
+                                    void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     struct kvsroot *root;
     const char *ns;
     const char *name;
     int flags;
     json_t *ops = NULL;
 
-    if (flux_request_unpack (msg, NULL, "{ s:o s:s s:s s:i }",
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{ s:o s:s s:s s:i }",
                              "ops", &ops,
                              "name", &name,
                              "namespace", &ns,
@@ -1577,18 +1712,19 @@ static void relaycommit_request_cb (flux_t *h, flux_msg_handler_t *mh,
 
     /* namespace must exist given we are on rank 0 */
     if (!(root = kvsroot_mgr_lookup_root_safe (ctx->krm, ns))) {
-        flux_log (h, LOG_ERR, "%s: namespace %s not available",
-                  __FUNCTION__, ns);
         errno = ENOTSUP;
         goto error;
     }
 
-    if (kvstxn_mgr_add_transaction (root->ktm, name, ops, flags) < 0) {
-        flux_log_error (h, "%s: kvstxn_mgr_add_transaction",
-                        __FUNCTION__);
+    if (kvstxn_mgr_add_transaction (root->ktm, name, ops, flags, 0) < 0) {
+        flux_log_error (h, "%s: kvstxn_mgr_add_transaction", __FUNCTION__);
         goto error;
     }
 
+    /* N.B. no request tracking for relay.  The relay does not get a
+     * response, only the original via finalize_transaction_bynames().
+     */
+    work_queue_check_append (ctx, root);
     return;
 
 error:
@@ -1603,34 +1739,41 @@ static void relaycommit_request_cb (flux_t *h, flux_msg_handler_t *mh,
 /* kvs.commit
  * Sent from users to local kvs module.
  */
-static void commit_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                               const flux_msg_t *msg, void *arg)
+static void commit_request_cb (flux_t *h,
+                               flux_msg_handler_t *mh,
+                               const flux_msg_t *msg,
+                               void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     struct kvsroot *root;
     const char *ns;
     int saved_errno, flags;
     bool stall = false;
     json_t *ops = NULL;
     treq_t *tr;
+    flux_error_t error;
+    const char *errmsg = NULL;
 
-    if (flux_request_unpack (msg, NULL, "{ s:o s:s s:i }",
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{ s:o s:s s:i }",
                              "ops", &ops,
                              "namespace", &ns,
                              "flags", &flags) < 0) {
         flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
         goto error;
     }
+    if (flux_msg_authorize (msg, FLUX_USERID_UNKNOWN) < 0
+        && guest_commit_authorize (ops, &error) < 0) {
+        errmsg = error.text;
+        goto error;
+    }
 
-    if (!(root = getroot (ctx,
-                          ns,
-                          mh,
-                          msg,
-                          NULL,
-                          commit_request_cb,
-                          &stall))) {
-        if (stall)
+    if (!(root = getroot (ctx, ns, mh, msg, NULL, &stall))) {
+        if (stall) {
+            request_tracking_add (ctx, msg);
             return;
+        }
         goto error;
     }
 
@@ -1663,17 +1806,23 @@ static void commit_request_cb (flux_t *h, flux_msg_handler_t *mh,
         if (kvstxn_mgr_add_transaction (root->ktm,
                                         treq_get_name (tr),
                                         ops,
-                                        flags) < 0) {
+                                        flags,
+                                        0) < 0) {
             flux_log_error (h, "%s: kvstxn_mgr_add_transaction",
                             __FUNCTION__);
             goto error;
         }
+
+        work_queue_check_append (ctx, root);
     }
     else {
         flux_future_t *f;
 
         /* route to rank 0 as instance owner */
-        if (!(f = flux_rpc_pack (h, "kvs.relaycommit", 0, FLUX_RPC_NORESPONSE,
+        if (!(f = flux_rpc_pack (h,
+                                 "kvs.relaycommit",
+                                 0,
+                                 FLUX_RPC_NORESPONSE,
                                  "{ s:O s:s s:s s:i }",
                                  "ops", ops,
                                  "name", treq_get_name (tr),
@@ -1684,20 +1833,24 @@ static void commit_request_cb (flux_t *h, flux_msg_handler_t *mh,
         }
         flux_future_destroy (f);
     }
+    request_tracking_add (ctx, msg);
     return;
 
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    request_tracking_remove (ctx, msg);
 }
 
 
 /* kvs.relayfence (rank 0 only, no response).
  */
-static void relayfence_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                   const flux_msg_t *msg, void *arg)
+static void relayfence_request_cb (flux_t *h,
+                                   flux_msg_handler_t *mh,
+                                   const flux_msg_t *msg,
+                                   void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     struct kvsroot *root;
     const char *ns;
     const char *name;
@@ -1705,7 +1858,9 @@ static void relayfence_request_cb (flux_t *h, flux_msg_handler_t *mh,
     json_t *ops = NULL;
     treq_t *tr;
 
-    if (flux_request_unpack (msg, NULL, "{ s:o s:s s:s s:i s:i }",
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{ s:o s:s s:s s:i s:i }",
                              "ops", &ops,
                              "name", &name,
                              "namespace", &ns,
@@ -1717,8 +1872,6 @@ static void relayfence_request_cb (flux_t *h, flux_msg_handler_t *mh,
 
     /* namespace must exist given we are on rank 0 */
     if (!(root = kvsroot_mgr_lookup_root_safe (ctx->krm, ns))) {
-        flux_log (h, LOG_ERR, "%s: namespace %s not available",
-                  __FUNCTION__, ns);
         errno = ENOTSUP;
         goto error;
     }
@@ -1761,13 +1914,18 @@ static void relayfence_request_cb (flux_t *h, flux_msg_handler_t *mh,
         if (kvstxn_mgr_add_transaction (root->ktm,
                                         treq_get_name (tr),
                                         treq_get_ops (tr),
-                                        treq_get_flags (tr)) < 0) {
-            flux_log_error (h, "%s: kvstxn_mgr_add_transaction",
-                            __FUNCTION__);
+                                        treq_get_flags (tr),
+                                        0) < 0) {
+            flux_log_error (h, "%s: kvstxn_mgr_add_transaction", __FUNCTION__);
             goto error;
         }
+
+        work_queue_check_append (ctx, root);
     }
 
+    /* N.B. no request tracking for relay.  The relay does not get a
+     * response, only the original via finalize_transaction_bynames().
+     */
     return;
 
 error:
@@ -1782,10 +1940,12 @@ static void relayfence_request_cb (flux_t *h, flux_msg_handler_t *mh,
 /* kvs.fence
  * Sent from users to local kvs module.
  */
-static void fence_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                              const flux_msg_t *msg, void *arg)
+static void fence_request_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     struct kvsroot *root;
     const char *ns;
     const char *name;
@@ -1793,8 +1953,12 @@ static void fence_request_cb (flux_t *h, flux_msg_handler_t *mh,
     bool stall = false;
     json_t *ops = NULL;
     treq_t *tr;
+    flux_error_t error;
+    const char *errmsg = NULL;
 
-    if (flux_request_unpack (msg, NULL, "{ s:o s:s s:s s:i s:i }",
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{ s:o s:s s:s s:i s:i }",
                              "ops", &ops,
                              "name", &name,
                              "namespace", &ns,
@@ -1803,16 +1967,18 @@ static void fence_request_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
         goto error;
     }
+    if (flux_msg_authorize (msg, FLUX_USERID_UNKNOWN) < 0
+        && guest_commit_authorize (ops, &error) < 0) {
+        errno = EPERM;
+        errmsg = error.text;
+        goto error;
+    }
 
-    if (!(root = getroot (ctx,
-                          ns,
-                          mh,
-                          msg,
-                          NULL,
-                          fence_request_cb,
-                          &stall))) {
-        if (stall)
+    if (!(root = getroot (ctx, ns, mh, msg, NULL, &stall))) {
+        if (stall) {
+            request_tracking_add (ctx, msg);
             goto stall;
+        }
         goto error;
     }
 
@@ -1866,18 +2032,25 @@ static void fence_request_cb (flux_t *h, flux_msg_handler_t *mh,
             if (kvstxn_mgr_add_transaction (root->ktm,
                                             treq_get_name (tr),
                                             treq_get_ops (tr),
-                                            treq_get_flags (tr)) < 0) {
-                flux_log_error (h, "%s: kvstxn_mgr_add_transaction",
+                                            treq_get_flags (tr),
+                                            0) < 0) {
+                flux_log_error (h,
+                                "%s: kvstxn_mgr_add_transaction",
                                 __FUNCTION__);
                 goto error;
             }
+
+            work_queue_check_append (ctx, root);
         }
     }
     else {
         flux_future_t *f;
 
         /* route to rank 0 as instance owner */
-        if (!(f = flux_rpc_pack (h, "kvs.relayfence", 0, FLUX_RPC_NORESPONSE,
+        if (!(f = flux_rpc_pack (h,
+                                 "kvs.relayfence",
+                                 0,
+                                 FLUX_RPC_NORESPONSE,
                                  "{ s:O s:s s:s s:i s:i }",
                                  "ops", ops,
                                  "name", name,
@@ -1889,69 +2062,84 @@ static void fence_request_cb (flux_t *h, flux_msg_handler_t *mh,
         }
         flux_future_destroy (f);
     }
+    request_tracking_add (ctx, msg);
     return;
 
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    request_tracking_remove (ctx, msg);
 stall:
     return;
 }
 
-/* For wait_version().
- */
-static void sync_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                             const flux_msg_t *msg, void *arg)
+static void wait_version_request_cb (flux_t *h,
+                                     flux_msg_handler_t *mh,
+                                     const flux_msg_t *msg,
+                                     void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     const char *ns;
     struct kvsroot *root;
     int rootseq;
     bool stall = false;
 
-    if (flux_request_unpack (msg, NULL, "{ s:i s:s }",
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{ s:i s:s }",
                              "rootseq", &rootseq,
                              "namespace", &ns) < 0) {
         flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
         goto error;
     }
 
-    if (!(root = getroot (ctx, ns, mh, msg, NULL, sync_request_cb,
-                          &stall))) {
-        if (stall)
+    if (!(root = getroot (ctx, ns, mh, msg, NULL, &stall))) {
+        if (stall) {
+            request_tracking_add (ctx, msg);
             return;
+        }
         goto error;
     }
 
     if (root->seq < rootseq) {
-        if (kvssync_add (root, sync_request_cb, h, mh, msg, ctx, rootseq) < 0) {
-            flux_log_error (h, "%s: kvssync_add", __FUNCTION__);
+        if (kvs_wait_version_add (root,
+                                  wait_version_request_cb,
+                                  h,
+                                  mh,
+                                  msg,
+                                  ctx,
+                                  rootseq) < 0) {
+            flux_log_error (h, "%s: kvs_wait_version_add", __FUNCTION__);
             goto error;
         }
+        request_tracking_add (ctx, msg);
         return; /* stall */
     }
 
-    if (flux_respond_pack (h, msg, "{ s:i s:s }",
+    if (flux_respond_pack (h,
+                           msg,
+                           "{ s:i s:s }",
                            "rootseq", root->seq,
                            "rootref", root->ref) < 0)
         flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
 
+    request_tracking_remove (ctx, msg);
     return;
 
 error:
     if (flux_respond_error (h, msg, errno, NULL) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    request_tracking_remove (ctx, msg);
 }
 
 static void getroot_request_cb (flux_t *h, flux_msg_handler_t *mh,
                                 const flux_msg_t *msg, void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     const char *ns;
     struct kvsroot *root;
 
-    if (flux_request_unpack (msg, NULL, "{ s:s }",
-                             "namespace", &ns) < 0) {
+    if (flux_request_unpack (msg, NULL, "{ s:s }", "namespace", &ns) < 0) {
         flux_log_error (ctx->h, "%s: flux_request_unpack", __FUNCTION__);
         goto error;
     }
@@ -1959,7 +2147,6 @@ static void getroot_request_cb (flux_t *h, flux_msg_handler_t *mh,
     if (ctx->rank == 0) {
         /* namespace must exist given we are on rank 0 */
         if (!(root = kvsroot_mgr_lookup_root_safe (ctx->krm, ns))) {
-            flux_log (h, LOG_DEBUG, "namespace %s not available", ns);
             errno = ENOTSUP;
             goto error;
         }
@@ -1968,41 +2155,51 @@ static void getroot_request_cb (flux_t *h, flux_msg_handler_t *mh,
             goto error;
     }
     else {
-        /* If root is not initialized, we have to intialize ourselves
+        /* If root is not initialized, we have to initialize ourselves
          * first.
          */
         bool stall = false;
-        if (!(root = getroot (ctx, ns, mh, msg, NULL,
-                              getroot_request_cb, &stall))) {
-            if (stall)
+        if (!(root = getroot (ctx, ns, mh, msg, NULL, &stall))) {
+            if (stall) {
+                request_tracking_add (ctx, msg);
                 return;
+            }
             goto error;
         }
     }
 
     /* N.B. owner cast into int */
-    if (flux_respond_pack (h, msg, "{ s:i s:i s:s s:i }",
+    if (flux_respond_pack (h,
+                           msg,
+                           "{ s:i s:i s:s s:i s:s }",
                            "owner", root->owner,
                            "rootseq", root->seq,
                            "rootref", root->ref,
-                           "flags", root->flags) < 0)
+                           "flags", root->flags,
+                           "namespace", root->ns_name) < 0)
         flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+    request_tracking_remove (ctx, msg);
     return;
 error:
     if (flux_respond_error (h, msg, errno, NULL) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    request_tracking_remove (ctx, msg);
 }
 
-static void error_event_cb (flux_t *h, flux_msg_handler_t *mh,
-                              const flux_msg_t *msg, void *arg)
+static void error_event_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     struct kvsroot *root;
     const char *ns;
     json_t *names = NULL;
     int errnum;
 
-    if (flux_event_unpack (msg, NULL, "{ s:s s:o s:i }",
+    if (flux_event_unpack (msg,
+                           NULL,
+                           "{ s:s s:o s:i }",
                            "namespace", &ns,
                            "names", &names,
                            "errnum", &errnum) < 0) {
@@ -2011,67 +2208,26 @@ static void error_event_cb (flux_t *h, flux_msg_handler_t *mh,
     }
 
     /* if root not initialized, nothing to do
-     * - it is ok that the namespace be marked for removal, we may be
-     *   cleaning up lingering transactions.
+     * - note that it is possible the namespace has been marked for
+     *   removal, we may be cleaning up lingering transactions and
+     *   need to report to those callers that namespace not available
+     *   via finalize_transaction_bynames() below.
+     * - i.e. we're calling kvsroot_mgr_lookup_root() not
+     *   kvsroot_mgr_lookup_root_safe().
      */
-    if (!(root = kvsroot_mgr_lookup_root (ctx->krm, ns))) {
-        flux_log (ctx->h, LOG_ERR, "%s: received unknown namespace %s",
-                  __FUNCTION__, ns);
+    if (!(root = kvsroot_mgr_lookup_root (ctx->krm, ns)))
         return;
-    }
 
     finalize_transaction_bynames (ctx, root, names, errnum);
 }
 
-/* Optimization: the current rootdir object is optionally included
- * in the kvs.namespace-<NS>-setroot event.  Prime the local cache with it.
- * If there are complications, just skip it.  Not critical.
- */
-static void prime_cache_with_rootdir (kvs_ctx_t *ctx, json_t *rootdir)
-{
-    struct cache_entry *entry;
-    char ref[BLOBREF_MAX_STRING_SIZE];
-    void *data = NULL;
-    int len;
-
-    if (treeobj_validate (rootdir) < 0 || !treeobj_is_dir (rootdir)) {
-        flux_log (ctx->h, LOG_ERR, "%s: invalid rootdir", __FUNCTION__);
-        goto done;
-    }
-    if (!(data = treeobj_encode (rootdir))) {
-        flux_log_error (ctx->h, "%s: treeobj_encode", __FUNCTION__);
-        goto done;
-    }
-    len = strlen (data);
-    if (blobref_hash (ctx->hash_name, data, len, ref, sizeof (ref)) < 0) {
-        flux_log_error (ctx->h, "%s: blobref_hash", __FUNCTION__);
-        goto done;
-    }
-    if ((entry = cache_lookup (ctx->cache, ref, ctx->epoch)))
-        goto done; // already in cache, possibly dirty/invalid - we don't care
-    if (!(entry = cache_entry_create (ref))) {
-        flux_log_error (ctx->h, "%s: cache_entry_create", __FUNCTION__);
-        goto done;
-    }
-    if (cache_entry_set_raw (entry, data, len) < 0) {
-        flux_log_error (ctx->h, "%s: cache_entry_set_raw", __FUNCTION__);
-        cache_entry_destroy (entry);
-        goto done;
-    }
-    if (cache_insert (ctx->cache, entry) < 0) {
-        flux_log_error (ctx->h, "%s: cache_insert", __FUNCTION__);
-        cache_entry_destroy (entry);
-        goto done;
-    }
-done:
-    free (data);
-}
-
 /* Alter the (rootref, rootseq) in response to a setroot event.
  */
-static void setroot_event_process (kvs_ctx_t *ctx, struct kvsroot *root,
-                                   json_t *names, json_t *rootdir,
-                                   const char *rootref, int rootseq)
+static void setroot_event_process (struct kvs_ctx *ctx,
+                                   struct kvsroot *root,
+                                   json_t *names,
+                                   const char *rootref,
+                                   int rootseq)
 {
     int errnum = 0;
 
@@ -2086,33 +2242,28 @@ static void setroot_event_process (kvs_ctx_t *ctx, struct kvsroot *root,
     if (errnum)
         return;
 
-    /* Optimization: prime local cache with directory object, if provided
-     * in event message.  Ignore failure here - object will be fetched on
-     * demand from content cache if not in local cache.
-     */
-    if (!json_is_null (rootdir))
-        prime_cache_with_rootdir (ctx, rootdir);
-
     setroot (ctx, root, rootref, rootseq);
 }
 
-static void setroot_event_cb (flux_t *h, flux_msg_handler_t *mh,
-                              const flux_msg_t *msg, void *arg)
+static void setroot_event_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     struct kvsroot *root;
     const char *ns;
     int rootseq;
     const char *rootref;
-    json_t *rootdir = NULL;
     json_t *names = NULL;
 
-    if (flux_event_unpack (msg, NULL, "{ s:s s:i s:s s:o s:o }",
+    if (flux_event_unpack (msg,
+                           NULL,
+                           "{ s:s s:i s:s s:o }",
                            "namespace", &ns,
                            "rootseq", &rootseq,
                            "rootref", &rootref,
-                           "names", &names,
-                           "rootdir", &rootdir) < 0) {
+                           "names", &names) < 0) {
         flux_log_error (ctx->h, "%s: flux_event_unpack", __FUNCTION__);
         return;
     }
@@ -2123,48 +2274,25 @@ static void setroot_event_cb (flux_t *h, flux_msg_handler_t *mh,
      *   order (commit/fence completes before namespace removed, but
      *   namespace remove event received before setroot).
      */
-    if (!(root = kvsroot_mgr_lookup_root (ctx->krm, ns))) {
-        flux_log (ctx->h, LOG_ERR, "%s: received unknown namespace %s",
-                  __FUNCTION__, ns);
+    if (!(root = kvsroot_mgr_lookup_root (ctx->krm, ns)))
         return;
-    }
 
     if (root->setroot_pause) {
-        flux_msg_t *msgcpy;
-
         assert (root->setroot_queue);
-
-        if (!(msgcpy = flux_msg_copy (msg, true))) {
-            flux_log_error (ctx->h, "%s: flux_msg_copy", __FUNCTION__);
+        if (flux_msglist_append (root->setroot_queue, msg) < 0) {
+            flux_log_error (ctx->h, "%s: flux_msglist_append", __FUNCTION__);
             return;
         }
-
-        if (zlist_append (root->setroot_queue, msgcpy) < 0) {
-            flux_log_error (ctx->h, "%s: zlist_append", __FUNCTION__);
-            return;
-        }
-
-        zlist_freefn (root->setroot_queue,
-                      msgcpy,
-                      flux_msg_destroy_wrapper,
-                      true);
         return;
     }
 
-    setroot_event_process (ctx, root, names, rootdir, rootref, rootseq);
+    setroot_event_process (ctx, root, names, rootref, rootseq);
 }
 
 static bool disconnect_cmp (const flux_msg_t *msg, void *arg)
 {
-    char *sender = arg;
-    char *s = NULL;
-    bool match = false;
-
-    if (flux_msg_get_route_first (msg, &s) == 0 && !strcmp (s, sender))
-        match = true;
-    if (s)
-        free (s);
-    return match;
+    flux_msg_t *msgreq = arg;
+    return flux_msg_route_match_first (msgreq, msg);
 }
 
 static int disconnect_request_root_cb (struct kvsroot *root, void *arg)
@@ -2173,31 +2301,29 @@ static int disconnect_request_root_cb (struct kvsroot *root, void *arg)
 
     /* Log error, but don't return -1, can continue to iterate
      * remaining roots */
-    if (kvssync_remove_msg (root, disconnect_cmp, cbd->sender) < 0)
-        flux_log_error (cbd->ctx->h, "%s: kvssync_remove_msg", __FUNCTION__);
-
+    if (kvs_wait_version_remove_msg (root,
+                                     disconnect_cmp,
+                                     (void *)cbd->msg) < 0) {
+        flux_log_error (cbd->ctx->h,
+                        "%s: kvs_wait_version_remove_msg",
+                        __FUNCTION__);
+    }
     return 0;
 }
 
 static void disconnect_request_cb (flux_t *h, flux_msg_handler_t *mh,
                                    const flux_msg_t *msg, void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     struct kvs_cb_data cbd;
-    char *sender = NULL;
 
-    if (flux_request_decode (msg, NULL, NULL) < 0)
-        return;
-    if (flux_msg_get_route_first (msg, &sender) < 0)
-        return;
     cbd.ctx = ctx;
-    cbd.sender = sender;
+    cbd.msg = msg;
     if (kvsroot_mgr_iter_roots (ctx->krm, disconnect_request_root_cb, &cbd) < 0)
         flux_log_error (h, "%s: kvsroot_mgr_iter_roots", __FUNCTION__);
 
-    if (cache_wait_destroy_msg (ctx->cache, disconnect_cmp, sender) < 0)
+    if (cache_wait_destroy_msg (ctx->cache, disconnect_cmp, (void *)msg) < 0)
         flux_log_error (h, "%s: wait_destroy_msg", __FUNCTION__);
-    free (sender);
 }
 
 static int stats_get_root_cb (struct kvsroot *root, void *arg)
@@ -2206,8 +2332,8 @@ static int stats_get_root_cb (struct kvsroot *root, void *arg)
     json_t *s;
 
     if (!(s = json_pack ("{ s:i s:i s:i s:i s:i }",
-                         "#syncers",
-                         zlist_size (root->synclist),
+                         "#versionwaiters",
+                         zlist_size (root->wait_version_list),
                          "#no-op stores",
                          kvstxn_mgr_get_noop_stores (root->ktm),
                          "#transactions",
@@ -2223,15 +2349,16 @@ static int stats_get_root_cb (struct kvsroot *root, void *arg)
     return 0;
 }
 
-static void stats_get_cb (flux_t *h, flux_msg_handler_t *mh,
-                          const flux_msg_t *msg, void *arg)
+static void stats_get_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     json_t *tstats = NULL;
     json_t *cstats = NULL;
     json_t *nsstats = NULL;
-    tstat_t ts = { .min = 0.0, .max = 0.0, .M = 0.0, .S = 0.0, .newM = 0.0,
-                   .newS = 0.0, .n = 0 };
+    tstat_t ts = { 0 };
     int size = 0, incomplete = 0, dirty = 0;
     double scale = 1E-3;
 
@@ -2286,10 +2413,12 @@ static void stats_get_cb (flux_t *h, flux_msg_handler_t *mh,
         }
     }
 
-    if (flux_respond_pack (h, msg,
-                           "{ s:O s:O }",
+    if (flux_respond_pack (h,
+                           msg,
+                           "{ s:O s:O s:i }",
                            "cache", cstats,
-                           "namespace", nsstats) < 0)
+                           "namespace", nsstats,
+                           "pending_requests", zhashx_size (ctx->requests)) < 0)
         flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
     json_decref (tstats);
     json_decref (cstats);
@@ -2311,7 +2440,7 @@ static int stats_clear_root_cb (struct kvsroot *root, void *arg)
     return 0;
 }
 
-static void stats_clear (kvs_ctx_t *ctx)
+static void stats_clear (struct kvs_ctx *ctx)
 {
     ctx->faults = 0;
 
@@ -2319,18 +2448,22 @@ static void stats_clear (kvs_ctx_t *ctx)
         flux_log_error (ctx->h, "%s: kvsroot_mgr_iter_roots", __FUNCTION__);
 }
 
-static void stats_clear_event_cb (flux_t *h, flux_msg_handler_t *mh,
-                                  const flux_msg_t *msg, void *arg)
+static void stats_clear_event_cb (flux_t *h,
+                                  flux_msg_handler_t *mh,
+                                  const flux_msg_t *msg,
+                                  void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
 
     stats_clear (ctx);
 }
 
-static void stats_clear_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                    const flux_msg_t *msg, void *arg)
+static void stats_clear_request_cb (flux_t *h,
+                                    flux_msg_handler_t *mh,
+                                    const flux_msg_t *msg,
+                                    void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
 
     stats_clear (ctx);
 
@@ -2338,21 +2471,24 @@ static void stats_clear_request_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_log_error (h, "%s: flux_respond", __FUNCTION__);
 }
 
-static int namespace_create (kvs_ctx_t *ctx, const char *ns,
-                             uint32_t owner, int flags)
+static int namespace_create (struct kvs_ctx *ctx,
+                             const char *ns,
+                             const char *rootref,
+                             uint32_t owner,
+                             int flags,
+                             const char **errmsg)
 {
     struct kvsroot *root;
-    json_t *rootdir = NULL;
-    char ref[BLOBREF_MAX_STRING_SIZE];
-    void *data = NULL;
     flux_msg_t *msg = NULL;
     char *topic = NULL;
-    int len;
     int rv = -1;
 
     /* If namespace already exists, return EEXIST.  Doesn't matter if
      * namespace is in process of being removed */
-    if (kvsroot_mgr_lookup_root (ctx->krm, ns)) {
+    if ((root = kvsroot_mgr_lookup_root (ctx->krm, ns))) {
+        if (root->remove)
+            (*errmsg) = "namespace with identical name in process "
+                "of being removed. Try again later";
         errno = EEXIST;
         return -1;
     }
@@ -2367,23 +2503,7 @@ static int namespace_create (kvs_ctx_t *ctx, const char *ns,
         return -1;
     }
 
-    if (!(rootdir = treeobj_create_dir ())) {
-        flux_log_error (ctx->h, "%s: treeobj_create_dir", __FUNCTION__);
-        goto cleanup;
-    }
-
-    if (!(data = treeobj_encode (rootdir))) {
-        flux_log_error (ctx->h, "%s: treeobj_encode", __FUNCTION__);
-        goto cleanup;
-    }
-    len = strlen (data);
-
-    if (blobref_hash (ctx->hash_name, data, len, ref, sizeof (ref)) < 0) {
-        flux_log_error (ctx->h, "%s: blobref_hash", __FUNCTION__);
-        goto cleanup;
-    }
-
-    setroot (ctx, root, ref, 0);
+    setroot (ctx, root, rootref, 0);
 
     if (event_subscribe (ctx, ns) < 0) {
         flux_log_error (ctx->h, "%s: event_subscribe", __FUNCTION__);
@@ -2417,26 +2537,31 @@ static int namespace_create (kvs_ctx_t *ctx, const char *ns,
 cleanup:
     if (rv < 0)
         kvsroot_mgr_remove_root (ctx->krm, ns);
-    free (data);
-    json_decref (rootdir);
     free (topic);
     flux_msg_destroy (msg);
     return rv;
 }
 
-static void namespace_create_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                         const flux_msg_t *msg, void *arg)
+static void namespace_create_request_cb (flux_t *h,
+                                         flux_msg_handler_t *mh,
+                                         const flux_msg_t *msg,
+                                         void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
+    const char *errmsg = NULL;
     const char *ns;
+    const char *rootref;
     uint32_t owner;
     int flags;
 
     assert (ctx->rank == 0);
 
     /* N.B. owner read into uint32_t */
-    if (flux_request_unpack (msg, NULL, "{ s:s s:i s:i }",
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{ s:s s:s s:i s:i }",
                              "namespace", &ns,
+                             "rootref", &rootref,
                              "owner", &owner,
                              "flags", &flags) < 0) {
         flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
@@ -2444,16 +2569,16 @@ static void namespace_create_request_cb (flux_t *h, flux_msg_handler_t *mh,
     }
 
     if (owner == FLUX_USERID_UNKNOWN)
-        owner = geteuid ();
+        owner = getuid ();
 
-    if (namespace_create (ctx, ns, owner, flags) < 0)
+    if (namespace_create (ctx, ns, rootref, owner, flags, &errmsg) < 0)
         goto error;
 
     if (flux_respond (h, msg, NULL) < 0)
         flux_log_error (h, "%s: flux_respond", __FUNCTION__);
     return;
 error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
 }
 
@@ -2469,8 +2594,8 @@ static int root_remove_process_transactions (treq_t *tr, void *data)
         json_t *names = NULL;
 
         if (!(names = json_pack ("[ s ]", treq_get_name (tr)))) {
-            flux_log_error (cbd->ctx->h, "%s: json_pack", __FUNCTION__);
             errno = ENOMEM;
+            flux_log_error (cbd->ctx->h, "%s: json_pack", __FUNCTION__);
             return -1;
         }
 
@@ -2480,7 +2605,7 @@ static int root_remove_process_transactions (treq_t *tr, void *data)
     return 0;
 }
 
-static void start_root_remove (kvs_ctx_t *ctx, const char *ns)
+static void start_root_remove (struct kvs_ctx *ctx, const char *ns)
 {
     struct kvsroot *root;
 
@@ -2490,11 +2615,14 @@ static void start_root_remove (kvs_ctx_t *ctx, const char *ns)
 
         root->remove = true;
 
-        /* Now that root has been marked for removal from roothash, run through
-         * the whole synclist.  requests will notice root removed, return
-         * ENOTSUP to all those trying to sync.
+        work_queue_remove (root);
+
+        /* Now that root has been marked for removal from roothash,
+         * run through the whole wait_version_list.  requests will
+         * notice root removed, return ENOTSUP to all those trying to
+         * sync.
          */
-        kvssync_process (root, true);
+        kvs_wait_version_process (root, true);
 
         /* Ready transactions will be processed and errors returned to
          * callers via the code path in kvstxn_apply().  But not ready
@@ -2507,14 +2635,14 @@ static void start_root_remove (kvs_ctx_t *ctx, const char *ns)
          */
 
         if (treq_mgr_iter_transactions (root->trm,
-                                          root_remove_process_transactions,
-                                          &cbd) < 0)
+                                        root_remove_process_transactions,
+                                        &cbd) < 0)
             flux_log_error (ctx->h, "%s: treq_mgr_iter_transactions",
                             __FUNCTION__);
     }
 }
 
-static int namespace_remove (kvs_ctx_t *ctx, const char *ns)
+static int namespace_remove (struct kvs_ctx *ctx, const char *ns)
 {
     flux_msg_t *msg = NULL;
     int saved_errno, rc = -1;
@@ -2528,10 +2656,9 @@ static int namespace_remove (kvs_ctx_t *ctx, const char *ns)
     }
 
     if (asprintf (&topic, "kvs.namespace-%s-removed", ns) < 0) {
-        saved_errno = ENOMEM;
+        saved_errno = errno;
         goto cleanup;
     }
-
     if (!(msg = flux_event_pack (topic, "{ s:s }", "namespace", ns))) {
         saved_errno = errno;
         flux_log_error (ctx->h, "%s: flux_event_pack", __FUNCTION__);
@@ -2557,16 +2684,17 @@ static int namespace_remove (kvs_ctx_t *ctx, const char *ns)
     return rc;
 }
 
-static void namespace_remove_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                         const flux_msg_t *msg, void *arg)
+static void namespace_remove_request_cb (flux_t *h,
+                                         flux_msg_handler_t *mh,
+                                         const flux_msg_t *msg,
+                                         void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     const char *ns;
 
     assert (ctx->rank == 0);
 
-    if (flux_request_unpack (msg, NULL, "{ s:s }",
-                             "namespace", &ns) < 0) {
+    if (flux_request_unpack (msg, NULL, "{ s:s }", "namespace", &ns) < 0) {
         flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
         goto error;
     }
@@ -2589,14 +2717,15 @@ static void namespace_remove_request_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
 }
 
-static void namespace_removed_event_cb (flux_t *h, flux_msg_handler_t *mh,
-                                        const flux_msg_t *msg, void *arg)
+static void namespace_removed_event_cb (flux_t *h,
+                                        flux_msg_handler_t *mh,
+                                        const flux_msg_t *msg,
+                                        void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     const char *ns;
 
-    if (flux_event_unpack (msg, NULL, "{ s:s }",
-                           "namespace", &ns) < 0) {
+    if (flux_event_unpack (msg, NULL, "{ s:s }", "namespace", &ns) < 0) {
         flux_log_error (ctx->h, "%s: flux_event_unpack", __FUNCTION__);
         return;
     }
@@ -2627,24 +2756,23 @@ static int namespace_list_cb (struct kvsroot *root, void *arg)
     return 0;
 }
 
-static void namespace_list_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                       const flux_msg_t *msg, void *arg)
+static void namespace_list_request_cb (flux_t *h,
+                                       flux_msg_handler_t *mh,
+                                       const flux_msg_t *msg,
+                                       void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     json_t *namespaces = NULL;
 
     if (!(namespaces = json_array ()))
         goto nomem;
 
-    if (kvsroot_mgr_iter_roots (ctx->krm, namespace_list_cb,
-                                namespaces) < 0) {
+    if (kvsroot_mgr_iter_roots (ctx->krm, namespace_list_cb, namespaces) < 0) {
         flux_log_error (h, "%s: kvsroot_mgr_iter_roots", __FUNCTION__);
         goto error;
     }
 
-    if (flux_respond_pack (h, msg, "{ s:O }",
-                           "namespaces",
-                           namespaces) < 0)
+    if (flux_respond_pack (h, msg, "{ s:O }", "namespaces", namespaces) < 0)
         flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
     json_decref (namespaces);
     return;
@@ -2663,27 +2791,22 @@ static void namespace_list_request_cb (flux_t *h, flux_msg_handler_t *mh,
  * to the KVS.  This can be used for testing purposes, such as testing
  * if read-your-writes consistency is working.
  */
-static void setroot_pause_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                      const flux_msg_t *msg, void *arg)
+static void setroot_pause_request_cb (flux_t *h,
+                                      flux_msg_handler_t *mh,
+                                      const flux_msg_t *msg,
+                                      void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     const char *ns = NULL;
     struct kvsroot *root;
     bool stall = false;
 
-    if (flux_request_unpack (msg, NULL, "{ s:s }",
-                             "namespace", &ns) < 0) {
+    if (flux_request_unpack (msg, NULL, "{ s:s }", "namespace", &ns) < 0) {
         flux_log_error (ctx->h, "%s: flux_request_unpack", __FUNCTION__);
         goto error;
     }
 
-    if (!(root = getroot (ctx,
-                          ns,
-                          mh,
-                          msg,
-                          NULL,
-                          setroot_pause_request_cb,
-                          &stall))) {
+    if (!(root = getroot (ctx, ns, mh, msg, NULL, &stall))) {
         if (stall)
             return;
         goto error;
@@ -2692,10 +2815,8 @@ static void setroot_pause_request_cb (flux_t *h, flux_msg_handler_t *mh,
     root->setroot_pause = true;
 
     if (!root->setroot_queue) {
-        if (!(root->setroot_queue = zlist_new ())) {
-            errno = ENOMEM;
+        if (!(root->setroot_queue = flux_msglist_create ()))
             goto error;
-        }
     }
 
     if (flux_respond (h, msg, NULL) < 0)
@@ -2706,26 +2827,27 @@ static void setroot_pause_request_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
 }
 
-static void setroot_unpause_process_msg (kvs_ctx_t *ctx, struct kvsroot *root,
-                                         flux_msg_t *msg)
+static void setroot_unpause_process_msg (struct kvs_ctx *ctx,
+                                         struct kvsroot *root,
+                                         const flux_msg_t *msg)
 {
     const char *ns;
     int rootseq;
     const char *rootref;
-    json_t *rootdir = NULL;
     json_t *names = NULL;
 
-    if (flux_event_unpack (msg, NULL, "{ s:s s:i s:s s:o s:o }",
+    if (flux_event_unpack (msg,
+                           NULL,
+                           "{ s:s s:i s:s s:o }",
                            "namespace", &ns,
                            "rootseq", &rootseq,
                            "rootref", &rootref,
-                           "names", &names,
-                           "rootdir", &rootdir) < 0) {
+                           "names", &names) < 0) {
         flux_log_error (ctx->h, "%s: flux_event_unpack", __FUNCTION__);
         return;
     }
 
-    setroot_event_process (ctx, root, names, rootdir, rootref, rootseq);
+    setroot_event_process (ctx, root, names, rootref, rootseq);
     return;
 }
 
@@ -2734,28 +2856,22 @@ static void setroot_unpause_process_msg (kvs_ctx_t *ctx, struct kvsroot *root,
  * events that were received during a pause will be processed in the
  * order they were received.
  */
-static void setroot_unpause_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                                        const flux_msg_t *msg, void *arg)
+static void setroot_unpause_request_cb (flux_t *h,
+                                        flux_msg_handler_t *mh,
+                                        const flux_msg_t *msg,
+                                        void *arg)
 {
-    kvs_ctx_t *ctx = arg;
+    struct kvs_ctx *ctx = arg;
     const char *ns = NULL;
     struct kvsroot *root;
-    flux_msg_t *m;
     bool stall = false;
 
-    if (flux_request_unpack (msg, NULL, "{ s:s }",
-                             "namespace", &ns) < 0) {
+    if (flux_request_unpack (msg, NULL, "{ s:s }", "namespace", &ns) < 0) {
         flux_log_error (ctx->h, "%s: flux_request_unpack", __FUNCTION__);
         goto error;
     }
 
-    if (!(root = getroot (ctx,
-                          ns,
-                          mh,
-                          msg,
-                          NULL,
-                          setroot_unpause_request_cb,
-                          &stall))) {
+    if (!(root = getroot (ctx, ns, mh, msg, NULL, &stall))) {
         if (stall)
             return;
         goto error;
@@ -2765,9 +2881,10 @@ static void setroot_unpause_request_cb (flux_t *h, flux_msg_handler_t *mh,
 
     /* user never called pause if !root->setroot_queue*/
     if (root->setroot_queue) {
-        while ((m = zlist_pop (root->setroot_queue))) {
+        const flux_msg_t *m;
+        while ((m = flux_msglist_pop (root->setroot_queue))) {
             setroot_unpause_process_msg (ctx, root, m);
-            flux_msg_destroy (m);
+            flux_msg_decref (m);
         }
     }
     if (flux_respond (h, msg, NULL) < 0)
@@ -2778,122 +2895,277 @@ static void setroot_unpause_request_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
 }
 
+static void config_reload_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct kvs_ctx *ctx = arg;
+    const flux_conf_t *conf;
+    const char *errstr = NULL;
+    flux_error_t error;
+
+    if (flux_conf_reload_decode (msg, &conf) < 0)
+        goto error;
+    if (kvs_checkpoint_reload (ctx->kcp, conf, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+    return;
+ error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+}
+
 /* see comments above in event_subscribe() regarding event
  * subscriptions to kvs.namespace */
 static const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_REQUEST, "kvs.stats.get",  stats_get_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.stats.clear",stats_clear_request_cb, 0 },
-    { FLUX_MSGTYPE_EVENT,   "kvs.stats.clear",stats_clear_event_cb, 0 },
-    { FLUX_MSGTYPE_EVENT,   "kvs.namespace-*-setroot",  setroot_event_cb, 0 },
-    { FLUX_MSGTYPE_EVENT,   "kvs.namespace-*-error",    error_event_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.getroot",
-                            getroot_request_cb, FLUX_ROLE_USER },
-    { FLUX_MSGTYPE_REQUEST, "kvs.dropcache",  dropcache_request_cb, 0 },
-    { FLUX_MSGTYPE_EVENT,   "kvs.dropcache",  dropcache_event_cb, 0 },
-    { FLUX_MSGTYPE_EVENT,   "hb",             heartbeat_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.disconnect", disconnect_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.sync",
-                            sync_request_cb, FLUX_ROLE_USER },
-    { FLUX_MSGTYPE_REQUEST, "kvs.lookup",
-                            lookup_request_cb, FLUX_ROLE_USER },
-    { FLUX_MSGTYPE_REQUEST, "kvs.lookup-plus",
-                            lookup_plus_request_cb, FLUX_ROLE_USER },
-    { FLUX_MSGTYPE_REQUEST, "kvs.commit",
-                            commit_request_cb, FLUX_ROLE_USER },
-    { FLUX_MSGTYPE_REQUEST, "kvs.relaycommit", relaycommit_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.fence",
-                            fence_request_cb, FLUX_ROLE_USER },
-    { FLUX_MSGTYPE_REQUEST, "kvs.relayfence", relayfence_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.namespace-create",
-                            namespace_create_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.namespace-remove",
-                            namespace_remove_request_cb, 0 },
-    { FLUX_MSGTYPE_EVENT,   "kvs.namespace-*-removed",
-                            namespace_removed_event_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.namespace-list",
-                            namespace_list_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.setroot-pause",
-                            setroot_pause_request_cb, FLUX_ROLE_USER },
-    { FLUX_MSGTYPE_REQUEST, "kvs.setroot-unpause",
-                            setroot_unpause_request_cb, FLUX_ROLE_USER },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.stats-get",
+        stats_get_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.stats-clear",
+        stats_clear_request_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_EVENT,
+        "kvs.stats-clear",
+        stats_clear_event_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_EVENT,
+        "kvs.namespace-*-setroot",
+        setroot_event_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_EVENT,
+        "kvs.namespace-*-error",
+        error_event_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.getroot",
+        getroot_request_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.dropcache",
+        dropcache_request_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_EVENT,
+        "kvs.dropcache",
+        dropcache_event_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.disconnect",
+        disconnect_request_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.wait-version",
+        wait_version_request_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.lookup",
+        lookup_request_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.lookup-plus",
+        lookup_plus_request_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.commit",
+        commit_request_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.relaycommit",
+        relaycommit_request_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.fence",
+        fence_request_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.relayfence",
+        relayfence_request_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.namespace-create",
+        namespace_create_request_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.namespace-remove",
+        namespace_remove_request_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_EVENT,
+        "kvs.namespace-*-removed",
+        namespace_removed_event_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.namespace-list",
+        namespace_list_request_cb,
+        0
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.setroot-pause",
+        setroot_pause_request_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.setroot-unpause",
+        setroot_unpause_request_cb,
+        FLUX_ROLE_USER
+    },
+    {
+        FLUX_MSGTYPE_REQUEST,
+        "kvs.config-reload",
+        config_reload_cb,
+        0
+    },
     FLUX_MSGHANDLER_TABLE_END,
 };
 
-static void process_args (kvs_ctx_t *ctx, int ac, char **av)
+static int process_config (struct kvs_ctx *ctx)
+{
+    flux_error_t error;
+    if (kvs_checkpoint_config_parse (ctx->kcp,
+                                     flux_get_conf (ctx->h),
+                                     &error) < 0) {
+        flux_log (ctx->h, LOG_ERR, "%s", error.text);
+        return -1;
+    }
+    return 0;
+}
+
+static int process_args (struct kvs_ctx *ctx, int ac, char **av)
 {
     int i;
 
     for (i = 0; i < ac; i++) {
-        if (strncmp (av[i], "transaction-merge=", 13) == 0)
-            ctx->transaction_merge = strtoul (av[i]+13, NULL, 10);
-        else
+        if (strstarts (av[i], "transaction-merge=")) {
+            char *endptr;
+            errno = 0;
+            ctx->transaction_merge = strtoul (av[i]+18, &endptr, 10);
+            if (errno != 0 || *endptr != '\0') {
+                errno = EINVAL;
+                return -1;
+            }
+        }
+        else {
             flux_log (ctx->h, LOG_ERR, "Unknown option `%s'", av[i]);
+            errno = EINVAL;
+            return -1;
+        }
     }
+    return 0;
 }
 
-/* Synchronously get string value by key from checkpoint service.
- * Copy value to buf with '\0' termination.
+/* Synchronously get checkpoint data by key from checkpoint service.
+ * Copy rootref buf with '\0' termination.
  * Return 0 on success, -1 on failure,
  */
-static int checkpoint_get (flux_t *h, const char *key, char *buf, size_t len)
+static int checkpoint_get (flux_t *h, char *buf, size_t len, int *seq)
 {
-    flux_future_t *f;
-    const char *value;
+    flux_future_t *f = NULL;
+    const char *rootref;
+    double timestamp = 0;
+    char datestr[128] = "N/A";
+    int rv = -1;
 
-    if (!(f = flux_rpc_pack (h,
-                             "kvs-checkpoint.get",
-                             0,
-                             0,
-                             "{s:s}",
-                             "key",
-                             key)))
+    if (!(f = kvs_checkpoint_lookup (h, NULL, 0)))
         return -1;
-    if (flux_rpc_get_unpack (f, "{s:s}", "value", &value) < 0)
+
+    if (kvs_checkpoint_lookup_get_rootref (f, &rootref) < 0)
         goto error;
-    if (strlen (value) >= len) {
+
+    if (strlen (rootref) >= len) {
         errno = EINVAL;
         goto error;
     }
-    strcpy (buf, value);
-    flux_future_destroy (f);
-    return 0;
+    strcpy (buf, rootref);
+
+    (void)kvs_checkpoint_lookup_get_sequence (f, seq);
+
+    (void)kvs_checkpoint_lookup_get_timestamp (f, &timestamp);
+    if (timestamp > 0)
+        timestamp_tostr (timestamp, datestr, sizeof (datestr));
+
+    flux_log (h, LOG_INFO,
+              "restored KVS from checkpoint on %s", datestr);
+
+    rv = 0;
 error:
     flux_future_destroy (f);
-    return -1;
+    return rv;
 }
 
-/* Synchronously store key-value pair to checkpoint service.
+/* Synchronously store checkpoint to checkpoint service.
  * Returns 0 on success, -1 on failure.
  */
-static int checkpoint_put (flux_t *h, const char *key, const char *value)
+static int checkpoint_put (flux_t *h, const char *rootref, int rootseq)
 {
-    flux_future_t *f;
+    flux_future_t *f = NULL;
+    int rv = -1;
 
-    if (!(f = flux_rpc_pack (h,
-                             "kvs-checkpoint.put",
-                             0,
-                             0,
-                             "{s:s s:s}",
-                             "key",
-                             key,
-                             "value",
-                             value)))
-        return -1;
-    if (flux_rpc_get (f, NULL) < 0) {
-        flux_future_destroy (f);
-        return -1;
-    }
+    if (!(f = kvs_checkpoint_commit (h, NULL, rootref, rootseq, 0, 0))
+        || flux_rpc_get (f, NULL) < 0)
+        goto error;
+    rv = 0;
+error:
     flux_future_destroy (f);
-    return 0;
+    return rv;
 }
 
 /* Store initial root in local cache, and flush to content cache
  * synchronously.  The corresponding blobref is written into 'ref'.
+ *
+ * N.B. The code for creating a new / empty kvs namespace assumes that
+ * an empty RFC 11 dir object was already created / stored, but
+ * offline garbage collection could remove it.
  */
-static int store_initial_rootdir (kvs_ctx_t *ctx, char *ref, int ref_len)
+static int store_initial_rootdir (struct kvs_ctx *ctx, char *ref, int ref_len)
 {
     struct cache_entry *entry;
-    int saved_errno, ret;
+    int saved_errno;
+    __attribute__((unused)) int ret;
     void *data = NULL;
     int len;
     flux_future_t *f = NULL;
@@ -2911,7 +3183,7 @@ static int store_initial_rootdir (kvs_ctx_t *ctx, char *ref, int ref_len)
         flux_log_error (ctx->h, "%s: blobref_hash", __FUNCTION__);
         goto error;
     }
-    if (!(entry = cache_lookup (ctx->cache, ref, ctx->epoch))) {
+    if (!(entry = cache_lookup (ctx->cache, ref))) {
         if (!(entry = cache_entry_create (ref))) {
             flux_log_error (ctx->h, "%s: cache_entry_create", __FUNCTION__);
             goto error;
@@ -2927,15 +3199,15 @@ static int store_initial_rootdir (kvs_ctx_t *ctx, char *ref, int ref_len)
             flux_log_error (ctx->h, "%s: cache_entry_set_raw", __FUNCTION__);
             goto error_uncache;
         }
-        if (!(f = flux_content_store (ctx->h, data, len, 0))
-                || flux_content_store_get (f, &newref) < 0) {
-            flux_log_error (ctx->h, "%s: flux_content_store", __FUNCTION__);
+        if (!(f = content_store (ctx->h, data, len, 0))
+                || content_store_get_blobref (f, ctx->hash_name, &newref) < 0) {
+            flux_log_error (ctx->h, "%s: content_store", __FUNCTION__);
             goto error_uncache;
         }
         /* Sanity check that content cache is using the same hash alg as KVS.
          * It should suffice to do this once at startup.
          */
-        if (strcmp (newref, ref) != 0) {
+        if (!streq (newref, ref)) {
             errno = EPROTO;
             flux_log_error (ctx->h, "%s: hash mismatch kvs=%s content=%s",
                             __FUNCTION__, ref, newref);
@@ -2961,31 +3233,39 @@ static int store_initial_rootdir (kvs_ctx_t *ctx, char *ref, int ref_len)
 
 int mod_main (flux_t *h, int argc, char **argv)
 {
-    kvs_ctx_t *ctx = getctx (h);
+    struct kvs_ctx *ctx;
     flux_msg_handler_t **handlers = NULL;
+    flux_future_t *f_heartbeat_sync = NULL;
     int rc = -1;
 
-    if (!ctx) {
+    if (!(ctx = kvs_ctx_create (h))) {
         flux_log_error (h, "error creating KVS context");
         goto done;
     }
-    process_args (ctx, argc, argv);
+    if (process_config (ctx) < 0)
+        goto done;
+    if (process_args (ctx, argc, argv) < 0)
+        goto done;
     if (ctx->rank == 0) {
         struct kvsroot *root;
+        char empty_dir_rootref[BLOBREF_MAX_STRING_SIZE];
         char rootref[BLOBREF_MAX_STRING_SIZE];
-        uint32_t owner = geteuid ();
+        int seq = 0;
+        uint32_t owner = getuid ();
+
+        if (store_initial_rootdir (ctx,
+                                   empty_dir_rootref,
+                                   sizeof (empty_dir_rootref)) < 0) {
+            flux_log_error (h, "store_initial_rootdir");
+            goto done;
+        }
 
         /* Look for a checkpoint and use it if found.
-         * Otherwise start the primary root namespace with an empty directory.
+         * Otherwise start the primary root namespace with an empty directory
+         * and seq = 0.
          */
-        if (checkpoint_get (h, "kvs-primary", rootref, sizeof (rootref)) == 0)
-            flux_log (h, LOG_INFO, "restored kvs-primary from checkpoint");
-        else {
-            if (store_initial_rootdir (ctx, rootref, sizeof (rootref)) < 0) {
-                flux_log_error (h, "storing initial root object");
-                goto done;
-            }
-        }
+        if (checkpoint_get (h, rootref, sizeof (rootref), &seq) < 0)
+            memcpy (rootref, empty_dir_rootref, sizeof (empty_dir_rootref));
 
         /* primary namespace must always be there and not marked
          * for removal
@@ -3004,21 +3284,45 @@ int mod_main (flux_t *h, int argc, char **argv)
             }
         }
 
-        setroot (ctx, root, rootref, 0);
+        setroot (ctx, root, rootref, seq);
 
         if (event_subscribe (ctx, KVS_PRIMARY_NAMESPACE) < 0) {
             flux_log_error (h, "event_subscribe");
             goto done;
         }
+
+        kvs_checkpoint_update_root_primary (ctx->kcp, root);
     }
     if (flux_msg_handler_addvec (h, htab, ctx, &handlers) < 0) {
         flux_log_error (h, "flux_msg_handler_addvec");
         goto done;
     }
+    if (!(f_heartbeat_sync = flux_sync_create (h, heartbeat_sync_min))
+            || flux_future_then (f_heartbeat_sync,
+                                 heartbeat_sync_max,
+                                 heartbeat_sync_cb,
+                                 ctx) < 0) {
+        flux_log_error (h, "error starting heartbeat synchronization");
+        goto done;
+    }
+    kvs_checkpoint_start (ctx->kcp);
     if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
         flux_log_error (h, "flux_reactor_run");
         goto done;
     }
+    if (zhashx_size (ctx->requests) > 0) {
+        /* anything that has not yet completed gets an ENOSYS */
+        const flux_msg_t *msg = zhashx_first (ctx->requests);
+        while (msg) {
+            const char *topic = "unknown";
+            if (flux_msg_get_topic (msg, &topic) < 0)
+                flux_log_error (ctx->h, "%s: flux_msg_get_topic", __FUNCTION__);
+            if (flux_respond_error (ctx->h, msg, ENOSYS, NULL) < 0)
+                flux_log_error (ctx->h, "%s: flux_respond_error", __FUNCTION__);
+            flux_log (ctx->h, LOG_ERR, "failing pending '%s' request", topic);
+            msg = zhashx_next (ctx->requests);
+        }
+    }
     /* Checkpoint the KVS root to the content backing store.
      * If backing store is not loaded, silently proceed without checkpoint.
      */
@@ -3030,7 +3334,7 @@ int mod_main (flux_t *h, int argc, char **argv)
             flux_log_error (h, "error looking up primary root");
             goto done;
         }
-        if (checkpoint_put (ctx->h, "kvs-primary", root->ref) < 0) {
+        if (checkpoint_put (ctx->h, root->ref, root->seq) < 0) {
             if (errno != ENOSYS) { // service not loaded is not an error
                 flux_log_error (h, "error saving primary KVS checkpoint");
                 goto done;
@@ -3039,12 +3343,12 @@ int mod_main (flux_t *h, int argc, char **argv)
     }
     rc = 0;
 done:
+    flux_future_destroy (f_heartbeat_sync);
     flux_msg_handler_delvec (handlers);
+    kvs_ctx_destroy (ctx);
     return rc;
 }
 
-MOD_NAME ("kvs");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/kvs/kvs_checkpoint.c b/src/modules/kvs/kvs_checkpoint.c
new file mode 100644
index 000000000000..6e610b57928b
--- /dev/null
+++ b/src/modules/kvs/kvs_checkpoint.c
@@ -0,0 +1,228 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <ctype.h>
+#include <errno.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/fsd.h"
+
+#include "kvs_checkpoint.h"
+#include "kvsroot.h"
+
+struct kvs_checkpoint {
+    flux_t *h;
+    struct kvsroot *root_primary;
+    double checkpoint_period;   /* in seconds */
+    flux_watcher_t *checkpoint_w;
+    kvs_checkpoint_txn_cb txn_cb;
+    void *txn_cb_arg;
+    int last_checkpoint_seq;
+};
+
+static int checkpoint_period_parse (const flux_conf_t *conf,
+                                    flux_error_t *errp,
+                                    double *checkpoint_period)
+{
+    flux_error_t error;
+    const char *str = NULL;
+
+    if (flux_conf_unpack (conf,
+                          &error,
+                          "{s?{s?s}}",
+                          "kvs",
+                          "checkpoint-period", &str) < 0) {
+        errprintf (errp, "error reading config for kvs: %s", error.text);
+        return -1;
+    }
+
+    if (str) {
+        if (fsd_parse_duration (str, checkpoint_period) < 0) {
+            errprintf (errp, "invalid checkpoint-period config: %s", str);
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+int kvs_checkpoint_config_parse (kvs_checkpoint_t *kcp,
+                                 const flux_conf_t *conf,
+                                 flux_error_t *errp)
+{
+    if (kcp) {
+        double checkpoint_period = kcp->checkpoint_period;
+        if (checkpoint_period_parse (conf, errp, &checkpoint_period) < 0)
+            return -1;
+        kcp->checkpoint_period = checkpoint_period;
+    }
+    return 0;
+}
+
+int kvs_checkpoint_reload (kvs_checkpoint_t *kcp,
+                           const flux_conf_t *conf,
+                           flux_error_t *errp)
+{
+    if (kcp) {
+        double checkpoint_period = kcp->checkpoint_period;
+        if (checkpoint_period_parse (conf,
+                                     errp,
+                                     &checkpoint_period) < 0)
+            return -1;
+
+        if (checkpoint_period != kcp->checkpoint_period) {
+            kcp->checkpoint_period = checkpoint_period;
+            flux_watcher_stop (kcp->checkpoint_w);
+
+            if (kcp->root_primary
+                && kcp->checkpoint_period > 0.0) {
+                flux_timer_watcher_reset (kcp->checkpoint_w,
+                                          kcp->checkpoint_period,
+                                          kcp->checkpoint_period);
+                flux_watcher_start (kcp->checkpoint_w);
+            }
+        }
+    }
+    return 0;
+}
+
+
+static void checkpoint_cb (flux_reactor_t *r,
+                           flux_watcher_t *w,
+                           int revents,
+                           void *arg)
+{
+    kvs_checkpoint_t *kcp = arg;
+    char name[64];
+    json_t *ops = NULL;
+
+    /* if no changes to root since last checkpoint-period, do
+     * nothing */
+    if (kcp->last_checkpoint_seq == kcp->root_primary->seq)
+        return;
+
+    snprintf (name,
+              sizeof (name),
+              "checkpoint-period.%u",
+              kcp->root_primary->seq);
+
+    if (!(ops = json_array ())) {
+        errno = ENOMEM;
+        flux_log_error (kcp->h, "checkpoint-period setup failure");
+        goto done;
+    }
+
+    /* Set FLUX_KVS_SYNC, to perform the checkpoint.
+     *
+     * Set KVSTXN_INTERNAL_FLAG_NO_PUBLISH, this is an internal KVS
+     * module transaction to checkpoint.  It has no operations so the
+     * KVS data will not change.  Therefore no setroot() needs to be
+     * called after this is done.
+     */
+    if (kvstxn_mgr_add_transaction (kcp->root_primary->ktm,
+                                    name,
+                                    ops,
+                                    FLUX_KVS_SYNC,
+                                    KVSTXN_INTERNAL_FLAG_NO_PUBLISH) < 0) {
+        flux_log_error (kcp->h, "%s: kvstxn_mgr_add_transaction", __FUNCTION__);
+        goto done;
+    }
+
+    if (kcp->txn_cb)
+        kcp->txn_cb (kcp->root_primary, kcp->txn_cb_arg);
+
+    /* N.B. "last_checkpoint_seq" protects against unnecessary
+     * checkpointing when there is no activity in the primary KVS.
+     */
+    kcp->last_checkpoint_seq = kcp->root_primary->seq;
+
+done:
+    json_decref (ops);
+}
+
+kvs_checkpoint_t *kvs_checkpoint_create (flux_t *h,
+                                         struct kvsroot *root_primary,
+                                         double checkpoint_period,
+                                         kvs_checkpoint_txn_cb txn_cb,
+                                         void *txn_cb_arg)
+{
+    kvs_checkpoint_t *kcp = NULL;
+
+    if (!(kcp = calloc (1, sizeof (*kcp))))
+        goto error;
+
+    kcp->h = h;
+    kcp->root_primary = root_primary; /* can be NULL initially */
+    kcp->checkpoint_period = checkpoint_period;
+    kcp->txn_cb = txn_cb;
+    kcp->txn_cb_arg = txn_cb_arg;
+
+    /* create regardless of checkpoint-period value, in case user
+     * reconfigures later.
+     */
+    if (!(kcp->checkpoint_w =
+          flux_timer_watcher_create (flux_get_reactor (h),
+                                     kcp->checkpoint_period,
+                                     kcp->checkpoint_period,
+                                     checkpoint_cb,
+                                     kcp))) {
+        flux_log_error (kcp->h, "flux_timer_watcher_create");
+        goto error;
+
+    }
+
+    return kcp;
+
+ error:
+    kvs_checkpoint_destroy (kcp);
+    return NULL;
+}
+
+void kvs_checkpoint_update_root_primary (kvs_checkpoint_t *kcp,
+                                         struct kvsroot *root_primary)
+{
+    if (kcp && root_primary)
+        kcp->root_primary = root_primary;
+}
+
+void kvs_checkpoint_start (kvs_checkpoint_t *kcp)
+{
+    if (kcp
+        && kcp->root_primary
+        && kcp->checkpoint_period > 0.0) {
+        flux_watcher_stop (kcp->checkpoint_w);
+        flux_timer_watcher_reset (kcp->checkpoint_w,
+                                  kcp->checkpoint_period,
+                                  kcp->checkpoint_period);
+        flux_watcher_start (kcp->checkpoint_w);
+    }
+}
+
+void kvs_checkpoint_destroy (kvs_checkpoint_t *kcp)
+{
+    if (kcp) {
+        int save_errno = errno;
+        flux_watcher_destroy (kcp->checkpoint_w);
+        free (kcp);
+        errno = save_errno;
+    }
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/kvs/kvs_checkpoint.h b/src/modules/kvs/kvs_checkpoint.h
new file mode 100644
index 000000000000..ad16ef695471
--- /dev/null
+++ b/src/modules/kvs/kvs_checkpoint.h
@@ -0,0 +1,74 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_KVS_CHECKPOINT_H
+#define _FLUX_KVS_CHECKPOINT_H
+
+#include <flux/core.h>
+
+#include "kvsroot.h"
+
+/* kvs_checkpoint will handle checkpointing for the checkpoint-period
+ * configuration under the [kvs] table.  Internally the checkpoint-period
+ * value and a timer are managed.
+ *
+ * To avoid excess comparisons for `rank == 0` throughout KVS code,
+ * most functions below are no-ops if the `kvs_checkpoint_t` argument
+ * is NULL.
+ */
+
+typedef struct kvs_checkpoint kvs_checkpoint_t;
+
+/* callback after sync/checkpoint transaction submitted */
+typedef void (*kvs_checkpoint_txn_cb)(struct kvsroot *root, void *arg);
+
+/* root_primary - root of primary namespace, will be passed to txn_cb
+ *              - can be NULL if not available at creation time, use
+ *                kvs_checkpoint_update_root_primary() to set later.
+ * checkpoint_period - timer will trigger a checkpoint every X seconds,
+ *                   - no timer will be done if <= 0.0.
+ * txn_cb - callback after each checkpoint transaction submitted
+ * txn_cb_arg - passed to txn_cb
+ */
+kvs_checkpoint_t *kvs_checkpoint_create (flux_t *h,
+                                         struct kvsroot *root_primary,
+                                         double checkpoint_period,
+                                         kvs_checkpoint_txn_cb txn_cb,
+                                         void *txn_cb_arg);
+
+/* update internal checkpoint_period setting as needed */
+int kvs_checkpoint_config_parse (kvs_checkpoint_t *kcp,
+                                 const flux_conf_t *conf,
+                                 flux_error_t *errp);
+
+/* update internal checkpoint_period setting as needed and restart
+ * internal timers if needed
+ */
+int kvs_checkpoint_reload (kvs_checkpoint_t *kcp,
+                           const flux_conf_t *conf,
+                           flux_error_t *errp);
+
+/* update kvsroot used internally */
+void kvs_checkpoint_update_root_primary (kvs_checkpoint_t *kcp,
+                                         struct kvsroot *root_primary);
+
+/* start / restart checkpoint timer.  If root_primary not yet set or
+ * checkpoint_period <= 0.0, will do nothing.
+ */
+void kvs_checkpoint_start (kvs_checkpoint_t *kcp);
+
+void kvs_checkpoint_destroy (kvs_checkpoint_t *kcp);
+
+
+#endif /* !_FLUX_KVS_CHECKPOINT_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/kvs/kvs_wait_version.c b/src/modules/kvs/kvs_wait_version.c
new file mode 100644
index 000000000000..b62943cd0ed6
--- /dev/null
+++ b/src/modules/kvs/kvs_wait_version.c
@@ -0,0 +1,165 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <stdbool.h>
+#include <ctype.h>
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "kvs_wait_version.h"
+
+struct kvs_wait_version {
+    flux_msg_handler_f cb;
+    flux_t *h;
+    flux_msg_handler_t *mh;
+    const flux_msg_t *msg;
+    void *arg;
+    int seq;
+};
+
+static int kvs_wait_version_cmp (void *item1, void *item2)
+{
+    struct kvs_wait_version *ks1 = item1;
+    struct kvs_wait_version *ks2 = item2;
+
+    if (ks1->seq < ks2->seq)
+        return -1;
+    if (ks1->seq > ks2->seq)
+        return 1;
+    return 0;
+}
+
+static void kvs_wait_version_destroy (void *data)
+{
+    struct kvs_wait_version *ks = data;
+    if (ks) {
+        flux_msg_decref (ks->msg);
+        free (ks);
+    }
+}
+
+int kvs_wait_version_add (struct kvsroot *root,
+                          flux_msg_handler_f cb,
+                          flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg,
+                          int seq)
+{
+    struct kvs_wait_version *kwv = NULL;
+
+    if (!root || !msg || root->seq >= seq) {
+        errno = EINVAL;
+        goto error;
+    }
+
+    if (!(kwv = calloc (1, sizeof (*kwv))))
+        goto error;
+
+    kwv->msg = flux_msg_incref (msg);
+    kwv->cb = cb;
+    kwv->h = h;
+    kwv->mh = mh;
+    kwv->arg = arg;
+    kwv->seq = seq;
+
+    if (zlist_push (root->wait_version_list, kwv) < 0) {
+        errno = ENOMEM;
+        goto error;
+    }
+    zlist_freefn (root->wait_version_list,
+                  kwv,
+                  kvs_wait_version_destroy,
+                  false);
+
+    zlist_sort (root->wait_version_list, kvs_wait_version_cmp);
+
+    return 0;
+
+ error:
+    kvs_wait_version_destroy (kwv);
+    return -1;
+}
+
+void kvs_wait_version_process (struct kvsroot *root, bool all)
+{
+    struct kvs_wait_version *kwv;
+
+    if (!root)
+        return;
+
+    /* notify sync waiters that version has been reached */
+
+    kwv = zlist_first (root->wait_version_list);
+    while (kwv && (all || root->seq >= kwv->seq)) {
+        kwv = zlist_pop (root->wait_version_list);
+        kwv->cb (kwv->h, kwv->mh, kwv->msg, kwv->arg);
+        kvs_wait_version_destroy (kwv);
+        kwv = zlist_first (root->wait_version_list);
+    }
+}
+
+int kvs_wait_version_remove_msg (struct kvsroot *root,
+                                 kvs_wait_version_test_msg_f cmp,
+                                 void *arg)
+{
+    zlist_t *tmp = NULL;
+    struct kvs_wait_version *kwv;
+    int rc = -1;
+    int saved_errno;
+
+    if (!root || !cmp) {
+        saved_errno = EINVAL;
+        goto error;
+    }
+
+    kwv = zlist_first (root->wait_version_list);
+    while (kwv) {
+        if (cmp (kwv->msg, arg)) {
+            if (!tmp && !(tmp = zlist_new ())) {
+                saved_errno = ENOMEM;
+                goto error;
+            }
+            if (zlist_append (tmp, kwv) < 0) {
+                saved_errno = ENOMEM;
+                goto error;
+            }
+        }
+        kwv = zlist_next (root->wait_version_list);
+    }
+    if (tmp) {
+        while ((kwv = zlist_pop (tmp)))
+            zlist_remove (root->wait_version_list, kwv);
+    }
+    rc = 0;
+ error:
+    /* if an error occurs above in zlist_new() or zlist_append(),
+     * simply destroy the tmp list.  Nothing has been removed off of
+     * the original queue yet.  Allow user to handle error as they see
+     * fit.
+     */
+    zlist_destroy (&tmp);
+    if (rc < 0)
+        errno = saved_errno;
+    return rc;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/kvs/kvs_wait_version.h b/src/modules/kvs/kvs_wait_version.h
new file mode 100644
index 000000000000..586a929e951b
--- /dev/null
+++ b/src/modules/kvs/kvs_wait_version.h
@@ -0,0 +1,45 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_KVS_WAIT_VERSION_H
+#define _FLUX_KVS_WAIT_VERSION_H
+
+#include <stdbool.h>
+#include <flux/core.h>
+
+#include "kvsroot.h"
+
+typedef bool (*kvs_wait_version_test_msg_f)(const flux_msg_t *msg, void *arg);
+
+/* add a kvs_wait_version structure to the kvsroot synclist */
+int kvs_wait_version_add (struct kvsroot *root,
+                          flux_msg_handler_f cb,
+                          flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg,
+                          int seq);
+
+/* if a root sequence number has gone past a sequence number, call the
+ * callback.  If 'all' is true, run callback on all wait_version_list
+ * regardless.
+ */
+void kvs_wait_version_process (struct kvsroot *root, bool all);
+
+/* remove message on wait_version_list that meet 'cmp' conditions */
+int kvs_wait_version_remove_msg (struct kvsroot *root,
+                                 kvs_wait_version_test_msg_f cmp,
+                                 void *arg);
+
+#endif /* !_FLUX_KVS_WAIT_VERSION_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/kvs/kvsroot.c b/src/modules/kvs/kvsroot.c
index 6a0208200e00..36015cb2736d 100644
--- a/src/modules/kvs/kvsroot.c
+++ b/src/modules/kvs/kvsroot.c
@@ -18,9 +18,12 @@
 #include <unistd.h>
 #include <stdbool.h>
 #include <ctype.h>
-#include <czmq.h>
 #include <flux/core.h>
 #include <jansson.h>
+#include <assert.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
 
 #include "kvsroot.h"
 
@@ -86,10 +89,10 @@ static void kvsroot_destroy (void *data)
             kvstxn_mgr_destroy (root->ktm);
         if (root->trm)
             treq_mgr_destroy (root->trm);
-        if (root->synclist)
-            zlist_destroy (&root->synclist);
+        if (root->wait_version_list)
+            zlist_destroy (&root->wait_version_list);
         if (root->setroot_queue)
-            zlist_destroy (&root->setroot_queue);
+            flux_msglist_destroy (root->setroot_queue);
         free (data);
     }
 }
@@ -120,6 +123,9 @@ struct kvsroot *kvsroot_mgr_create_root (kvsroot_mgr_t *krm,
         goto error;
     }
 
+    if (streq (root->ns_name, KVS_PRIMARY_NAMESPACE))
+        root->is_primary = true;
+
     if (!(root->ktm = kvstxn_mgr_create (cache,
                                          root->ns_name,
                                          hash_name,
@@ -134,7 +140,7 @@ struct kvsroot *kvsroot_mgr_create_root (kvsroot_mgr_t *krm,
         goto error;
     }
 
-    if (!(root->synclist = zlist_new ())) {
+    if (!(root->wait_version_list = zlist_new ())) {
         flux_log_error (krm->h, "zlist_new");
         goto error;
     }
@@ -144,6 +150,7 @@ struct kvsroot *kvsroot_mgr_create_root (kvsroot_mgr_t *krm,
     root->remove = false;
 
     if (zhash_insert (krm->roothash, ns, root) < 0) {
+        errno = EEXIST;
         flux_log_error (krm->h, "zhash_insert");
         goto error;
     }
@@ -156,6 +163,7 @@ struct kvsroot *kvsroot_mgr_create_root (kvsroot_mgr_t *krm,
         goto error;
     }
 
+    list_node_init (&root->work_queue_node);
     return root;
 
  error:
@@ -245,8 +253,10 @@ int kvsroot_mgr_iter_roots (kvsroot_mgr_t *krm, kvsroot_root_f cb, void *arg)
 /* Convenience functions on struct kvsroot
  */
 
-void kvsroot_setroot (kvsroot_mgr_t *krm, struct kvsroot *root,
-                      const char *root_ref, int root_seq)
+void kvsroot_setroot (kvsroot_mgr_t *krm,
+                      struct kvsroot *root,
+                      const char *root_ref,
+                      int root_seq)
 {
     if (!root || !root_ref)
         return;
@@ -257,7 +267,8 @@ void kvsroot_setroot (kvsroot_mgr_t *krm, struct kvsroot *root,
     root->seq = root_seq;
 }
 
-int kvsroot_check_user (kvsroot_mgr_t *krm, struct kvsroot *root,
+int kvsroot_check_user (kvsroot_mgr_t *krm,
+                        struct kvsroot *root,
                         struct flux_msg_cred cred)
 {
     if (!root) {
diff --git a/src/modules/kvs/kvsroot.h b/src/modules/kvs/kvsroot.h
index a425519ac8d7..e223bcf68bbb 100644
--- a/src/modules/kvs/kvsroot.h
+++ b/src/modules/kvs/kvsroot.h
@@ -19,22 +19,26 @@
 #include "treq.h"
 #include "waitqueue.h"
 #include "src/common/libutil/blobref.h"
+#include "src/common/libccan/ccan/list/list.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 typedef struct kvsroot_mgr kvsroot_mgr_t;
 
 struct kvsroot {
     char *ns_name;
+    bool is_primary;
     uint32_t owner;
     int seq;
     char ref[BLOBREF_MAX_STRING_SIZE];
     kvstxn_mgr_t *ktm;
     treq_mgr_t *trm;
-    zlist_t *synclist;
-    int last_update_epoch;
+    zlist_t *wait_version_list;
+    double last_update_time;
     int flags;
     bool remove;
     bool setroot_pause;
-    zlist_t *setroot_queue;
+    struct flux_msglist *setroot_queue;
+    struct list_node work_queue_node;
 };
 
 /* return -1 on error, 0 on success, 1 on success & to stop iterating */
@@ -71,10 +75,13 @@ int kvsroot_mgr_iter_roots (kvsroot_mgr_t *krm, kvsroot_root_f cb, void *arg);
 /* Convenience functions on struct kvsroot
  */
 
-void kvsroot_setroot (kvsroot_mgr_t *krm, struct kvsroot *root,
-                      const char *root_ref, int root_seq);
+void kvsroot_setroot (kvsroot_mgr_t *krm,
+                      struct kvsroot *root,
+                      const char *root_ref,
+                      int root_seq);
 
-int kvsroot_check_user (kvsroot_mgr_t *krm,struct kvsroot *root,
+int kvsroot_check_user (kvsroot_mgr_t *krm,
+                        struct kvsroot *root,
                         struct flux_msg_cred cred);
 
 #endif /* !_FLUX_KVS_KVSROOT_H */
diff --git a/src/modules/kvs/kvssync.c b/src/modules/kvs/kvssync.c
deleted file mode 100644
index 31e0883512a2..000000000000
--- a/src/modules/kvs/kvssync.c
+++ /dev/null
@@ -1,159 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <stdbool.h>
-#include <ctype.h>
-#include <czmq.h>
-#include <flux/core.h>
-
-#include "kvssync.h"
-
-struct kvssync {
-    flux_msg_handler_f cb;
-    flux_t *h;
-    flux_msg_handler_t *mh;
-    const flux_msg_t *msg;
-    void *arg;
-    int seq;
-};
-
-static int kvssync_cmp (void *item1, void *item2)
-{
-    struct kvssync *ks1 = item1;
-    struct kvssync *ks2 = item2;
-
-    if (ks1->seq < ks2->seq)
-        return -1;
-    if (ks1->seq > ks2->seq)
-        return 1;
-    return 0;
-}
-
-static void kvssync_destroy (void *data)
-{
-    struct kvssync *ks = data;
-    if (ks) {
-        flux_msg_decref (ks->msg);
-        free (ks);
-    }
-}
-
-int kvssync_add (struct kvsroot *root, flux_msg_handler_f cb, flux_t *h,
-                 flux_msg_handler_t *mh, const flux_msg_t *msg, void *arg,
-                 int seq)
-{
-    struct kvssync *ks = NULL;
-
-    if (!root || !msg || root->seq >= seq) {
-        errno = EINVAL;
-        goto error;
-    }
-
-    if (!(ks = calloc (1, sizeof (*ks)))) {
-        errno = ENOMEM;
-        goto error;
-    }
-
-    ks->msg = flux_msg_incref (msg);
-    ks->cb = cb;
-    ks->h = h;
-    ks->mh = mh;
-    ks->arg = arg;
-    ks->seq = seq;
-
-    if (zlist_push (root->synclist, ks) < 0) {
-        errno = ENOMEM;
-        goto error;
-    }
-    zlist_freefn (root->synclist, ks, kvssync_destroy, false);
-
-    zlist_sort (root->synclist, kvssync_cmp);
-
-    return 0;
-
-error:
-    kvssync_destroy (ks);
-    return -1;
-}
-
-void kvssync_process (struct kvsroot *root, bool all)
-{
-    struct kvssync *ks;
-
-    if (!root)
-        return;
-
-    /* notify sync waiters that version has been reached */
-
-    ks = zlist_first (root->synclist);
-    while (ks && (all || root->seq >= ks->seq)) {
-        ks = zlist_pop (root->synclist);
-        ks->cb (ks->h, ks->mh, ks->msg, ks->arg);
-        kvssync_destroy (ks);
-        ks = zlist_first (root->synclist);
-    }
-}
-
-int kvssync_remove_msg (struct kvsroot *root,
-                        kvssync_test_msg_f cmp,
-                        void *arg)
-{
-    zlist_t *tmp = NULL;
-    struct kvssync *ks;
-    int rc = -1;
-    int saved_errno;
-
-    if (!root || !cmp) {
-        saved_errno = EINVAL;
-        goto error;
-    }
-
-    ks = zlist_first (root->synclist);
-    while (ks) {
-        if (cmp (ks->msg, arg)) {
-            if (!tmp && !(tmp = zlist_new ())) {
-                saved_errno = ENOMEM;
-                goto error;
-            }
-            if (zlist_append (tmp, ks) < 0) {
-                saved_errno = ENOMEM;
-                goto error;
-            }
-        }
-        ks = zlist_next (root->synclist);
-    }
-    if (tmp) {
-        while ((ks = zlist_pop (tmp)))
-            zlist_remove (root->synclist, ks);
-    }
-    rc = 0;
-error:
-    /* if an error occurs above in zlist_new() or zlist_append(),
-     * simply destroy the tmp list.  Nothing has been removed off of
-     * the original queue yet.  Allow user to handle error as they see
-     * fit.
-     */
-    zlist_destroy (&tmp);
-    if (rc < 0)
-        errno = saved_errno;
-    return rc;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/kvs/kvssync.h b/src/modules/kvs/kvssync.h
deleted file mode 100644
index 6d1e00b09711..000000000000
--- a/src/modules/kvs/kvssync.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef _FLUX_KVS_KVSSYNC_H
-#define _FLUX_KVS_KVSSYNC_H
-
-#include <stdbool.h>
-#include <flux/core.h>
-
-#include "kvsroot.h"
-
-typedef bool (*kvssync_test_msg_f)(const flux_msg_t *msg, void *arg);
-
-/* add a kvssync structure to the kvsroot synclist */
-int kvssync_add (struct kvsroot *root, flux_msg_handler_f cb, flux_t *h,
-                 flux_msg_handler_t *mh, const flux_msg_t *msg, void *arg,
-                 int seq);
-
-/* if a root sequence number has gone past a sync sequence number,
- * call the callback.  If 'all' is true, run callback on all synclist
- * regardless.
- */
-void kvssync_process (struct kvsroot *root, bool all);
-
-/* remove message on synclist that meet 'cmp' conditions */
-int kvssync_remove_msg (struct kvsroot *root,
-                        kvssync_test_msg_f cmp,
-                        void *arg);
-
-#endif /* !_FLUX_KVS_KVSROOT_H */
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/kvs/kvstxn.c b/src/modules/kvs/kvstxn.c
index 6e852fad6240..1a7e1eea99c4 100644
--- a/src/modules/kvs/kvstxn.c
+++ b/src/modules/kvs/kvstxn.c
@@ -18,28 +18,28 @@
 #include <unistd.h>
 #include <stdbool.h>
 #include <ctype.h>
-#include <czmq.h>
 #include <flux/core.h>
 #include <jansson.h>
-#include <sodium.h>
+#include <assert.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libccan/ccan/base64/base64.h"
 #include "src/common/libutil/macros.h"
 #include "src/common/libutil/blobref.h"
 #include "src/common/libkvs/treeobj.h"
+#include "src/common/libkvs/kvs_checkpoint.h"
+#include "src/common/libkvs/kvs_commit.h"
 #include "src/common/libkvs/kvs_txn_private.h"
 #include "src/common/libkvs/kvs_util_private.h"
+#include "ccan/str/str.h"
 
 #include "kvstxn.h"
 
-#define KVSTXN_PROCESSING      0x01
-#define KVSTXN_MERGED          0x02 /* kvstxn is a merger of transactions */
-#define KVSTXN_MERGE_COMPONENT 0x04 /* kvstxn is member of a merger */
-
 struct kvstxn_mgr {
     struct cache *cache;
     const char *ns_name;
     const char *hash_name;
-    int noop_stores;            /* for kvs.stats.get, etc.*/
+    int noop_stores;            /* for kvs.stats-get, etc.*/
     zlist_t *ready;
     flux_t *h;
     void *aux;
@@ -52,22 +52,54 @@ struct kvstxn {
     json_t *ops;
     json_t *keys;
     json_t *names;
-    int flags;
+    int flags;                  /* kvs flags from request caller */
+    int internal_flags;         /* special kvstxn api internal flags */
     json_t *rootcpy;   /* working copy of root dir */
     const json_t *rootdir;      /* source of rootcpy above */
     struct cache_entry *entry;  /* for reference counting rootdir above */
+    struct cache_entry *newroot_entry;  /* for reference counting new root */
     char newroot[BLOBREF_MAX_STRING_SIZE];
     zlist_t *missing_refs_list;
     zlist_t *dirty_cache_entries_list;
-    int internal_flags;
+    flux_future_t *f_sync_content_flush;
+    flux_future_t *f_sync_checkpoint;
+    bool processing;            /* kvstxn is being processed */
+    bool merged;                /* kvstxn is a merger of transactions */
+    bool merge_component;       /* kvstxn is member of a merger */
     kvstxn_mgr_t *ktm;
+    /* State transitions
+     *
+     * INIT - perform initializations / checks
+     * LOAD_ROOT - load KVS root
+     *           - if needed, report missing refs to caller and stall
+     * APPLY_OPS - apply changes to KVS
+     *           - if needed, report missing refs to caller and stall
+     * STORE - generate dirty entries for caller to store
+     * GENERATE_KEYS - stall until stores complete
+     *               - generate keys modified in txn
+     * SYNC_CONTENT_FLUSH - call content.flush (for FLUX_KVS_SYNC)
+     * SYNC_CHECKPOINT - call kvs_checkpoint_commit (for FLUX_KVS_SYNC)
+     * FINISHED - end state
+     *
+     * INIT -> LOAD_ROOT
+     * LOAD_ROOT -> APPLY_OPS
+     * LOAD_ROOT -> GENERATE_KEYS (if no ops)
+     * APPLY_OPS -> STORE
+     * STORE -> GENERATE_KEYS
+     * GENERATE_KEYS -> FINISHED
+     * GENERATE_KEYS -> SYNC_CONTENT_FLUSH
+     * SYNC_CONTENT_FLUSH -> SYNC_CHECKPOINT
+     * SYNC_CHECKPOINT -> FINISHED
+     */
     enum {
         KVSTXN_STATE_INIT = 1,
         KVSTXN_STATE_LOAD_ROOT = 2,
         KVSTXN_STATE_APPLY_OPS = 3,
         KVSTXN_STATE_STORE = 4,
-        KVSTXN_STATE_PRE_FINISHED = 5,
-        KVSTXN_STATE_FINISHED = 6,
+        KVSTXN_STATE_GENERATE_KEYS = 5,
+        KVSTXN_STATE_SYNC_CONTENT_FLUSH = 6,
+        KVSTXN_STATE_SYNC_CHECKPOINT = 7,
+        KVSTXN_STATE_FINISHED = 8,
     } state;
 };
 
@@ -79,10 +111,13 @@ static void kvstxn_destroy (kvstxn_t *kt)
         json_decref (kt->names);
         json_decref (kt->rootcpy);
         cache_entry_decref (kt->entry);
+        cache_entry_decref (kt->newroot_entry);
         if (kt->missing_refs_list)
             zlist_destroy (&kt->missing_refs_list);
         if (kt->dirty_cache_entries_list)
             zlist_destroy (&kt->dirty_cache_entries_list);
+        flux_future_destroy (kt->f_sync_content_flush);
+        flux_future_destroy (kt->f_sync_checkpoint);
         free (kt);
     }
 }
@@ -90,7 +125,8 @@ static void kvstxn_destroy (kvstxn_t *kt)
 static kvstxn_t *kvstxn_create (kvstxn_mgr_t *ktm,
                                 const char *name,
                                 json_t *ops,
-                                int flags)
+                                int flags,
+                                int internal_flags)
 {
     kvstxn_t *kt;
 
@@ -116,8 +152,10 @@ static kvstxn_t *kvstxn_create (kvstxn_mgr_t *ktm,
         }
     }
     kt->flags = flags;
+    kt->internal_flags = internal_flags;
     if (!(kt->missing_refs_list = zlist_new ()))
         goto error_enomem;
+    zlist_autofree (kt->missing_refs_list);
     if (!(kt->dirty_cache_entries_list = zlist_new ()))
         goto error_enomem;
     kt->ktm = ktm;
@@ -147,7 +185,7 @@ int kvstxn_set_aux_errnum (kvstxn_t *kt, int errnum)
 
 bool kvstxn_fallback_mergeable (kvstxn_t *kt)
 {
-    if (kt->internal_flags & KVSTXN_MERGED)
+    if (kt->merged)
         return true;
     return false;
 }
@@ -167,6 +205,11 @@ int kvstxn_get_flags (kvstxn_t *kt)
     return kt->flags;
 }
 
+int kvstxn_get_internal_flags (kvstxn_t *kt)
+{
+    return kt->internal_flags;
+}
+
 const char *kvstxn_get_namespace (kvstxn_t *kt)
 {
     return kt->ktm->ns_name;
@@ -208,12 +251,17 @@ json_t *kvstxn_get_keys (kvstxn_t *kt)
 void kvstxn_cleanup_dirty_cache_entry (kvstxn_t *kt, struct cache_entry *entry)
 {
     if (kt->state == KVSTXN_STATE_STORE
-        || kt->state == KVSTXN_STATE_PRE_FINISHED) {
+        || kt->state == KVSTXN_STATE_GENERATE_KEYS) {
         char ref[BLOBREF_MAX_STRING_SIZE];
         const void *data;
         int len;
-        int ret;
+        __attribute__((unused)) int ret;
 
+        /* special case, must clear */
+        if (kt->newroot_entry == entry)
+            kt->newroot_entry = NULL;
+
+        cache_entry_decref (entry);
         assert (cache_entry_get_valid (entry) == true);
         assert (cache_entry_get_dirty (entry) == true);
         ret = cache_entry_clear_dirty (entry);
@@ -238,6 +286,18 @@ static void cleanup_dirty_cache_list (kvstxn_t *kt)
         kvstxn_cleanup_dirty_cache_entry (kt, entry);
 }
 
+static int kvstxn_add_dirty_cache_entry (kvstxn_t *kt, struct cache_entry *entry)
+{
+    cache_entry_incref (entry);
+    if (zlist_push (kt->dirty_cache_entries_list, entry) < 0) {
+        /* cache_entry_decref() called in kvstxn_cleanup_dirty_cache_entry() */
+        kvstxn_cleanup_dirty_cache_entry (kt, entry);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
 /* Store object 'o' under key 'ref' in local cache.
  * Object reference is still owned by the caller.
  * 'is_raw' indicates this data is a json string w/ base64 value and
@@ -246,7 +306,7 @@ static void cleanup_dirty_cache_list (kvstxn_t *kt)
  * Returns -1 on error, 0 on success entry already there, 1 on success
  * entry needs to be flushed to content store
  */
-static int store_cache (kvstxn_t *kt, int current_epoch, json_t *o,
+static int store_cache (kvstxn_t *kt, json_t *o,
                         bool is_raw, char *ref, int ref_len,
                         struct cache_entry **entryp)
 {
@@ -254,20 +314,19 @@ static int store_cache (kvstxn_t *kt, int current_epoch, json_t *o,
     int saved_errno, rc;
     const char *xdata;
     char *data = NULL;
-    size_t xlen, len;
+    size_t xlen, databuflen;
+    ssize_t datalen = 0;
 
     if (is_raw) {
         xdata = json_string_value (o);
         xlen = strlen (xdata);
-        len = BASE64_DECODE_SIZE (xlen);
-        if (len > 0) {
-            if (!(data = malloc (len))) {
+        databuflen = base64_decoded_length (xlen);
+        if (databuflen > 0) {
+            if (!(data = malloc (databuflen))) {
                 flux_log_error (kt->ktm->h, "malloc");
                 goto error;
             }
-            if (sodium_base642bin ((unsigned char *)data, len, xdata, xlen,
-                                   NULL, &len, NULL,
-                                   sodium_base64_VARIANT_ORIGINAL) < 0) {
+            if ((datalen = base64_decode (data, databuflen, xdata, xlen)) < 0) {
                 errno = EPROTO;
                 goto error;
             }
@@ -278,13 +337,13 @@ static int store_cache (kvstxn_t *kt, int current_epoch, json_t *o,
             flux_log_error (kt->ktm->h, "%s: treeobj_encode", __FUNCTION__);
             goto error;
         }
-        len = strlen (data);
+        datalen = strlen (data);
     }
-    if (blobref_hash (kt->ktm->hash_name, data, len, ref, ref_len) < 0) {
+    if (blobref_hash (kt->ktm->hash_name, data, datalen, ref, ref_len) < 0) {
         flux_log_error (kt->ktm->h, "%s: blobref_hash", __FUNCTION__);
         goto error;
     }
-    if (!(entry = cache_lookup (kt->ktm->cache, ref, current_epoch))) {
+    if (!(entry = cache_lookup (kt->ktm->cache, ref))) {
         if (!(entry = cache_entry_create (ref))) {
             flux_log_error (kt->ktm->h, "%s: cache_entry_create", __FUNCTION__);
             goto error;
@@ -300,15 +359,15 @@ static int store_cache (kvstxn_t *kt, int current_epoch, json_t *o,
         rc = 0;
     }
     else {
-        if (cache_entry_set_raw (entry, data, len) < 0) {
-            int ret;
+        if (cache_entry_set_raw (entry, data, datalen) < 0) {
+            __attribute__((unused)) int ret;
             ret = cache_remove_entry (kt->ktm->cache, ref);
             assert (ret == 1);
             goto error;
         }
         if (cache_entry_set_dirty (entry, true) < 0) {
             flux_log_error (kt->ktm->h, "%s: cache_entry_set_dirty",__FUNCTION__);
-            int ret;
+            __attribute__((unused)) int ret;
             ret = cache_remove_entry (kt->ktm->cache, ref);
             assert (ret == 1);
             goto error;
@@ -330,7 +389,7 @@ static int store_cache (kvstxn_t *kt, int current_epoch, json_t *o,
  * Store (large) FILEVAL objects, converting them to FILEREFs.
  * Return 0 on success, -1 on error
  */
-static int kvstxn_unroll (kvstxn_t *kt, int current_epoch, json_t *dir)
+static int kvstxn_unroll (kvstxn_t *kt, json_t *dir)
 {
     json_t *dir_entry;
     json_t *dir_data;
@@ -353,17 +412,18 @@ static int kvstxn_unroll (kvstxn_t *kt, int current_epoch, json_t *dir)
     while (iter) {
         dir_entry = json_object_iter_value (iter);
         if (treeobj_is_dir (dir_entry)) {
-            if (kvstxn_unroll (kt, current_epoch, dir_entry) < 0) /* depth first */
+            if (kvstxn_unroll (kt, dir_entry) < 0) /* depth first */
                 return -1;
-            if ((ret = store_cache (kt, current_epoch, dir_entry,
-                                    false, ref, sizeof (ref), &entry)) < 0)
+            if ((ret = store_cache (kt,
+                                    dir_entry,
+                                    false,
+                                    ref,
+                                    sizeof (ref),
+                                    &entry)) < 0)
                 return -1;
             if (ret) {
-                if (zlist_push (kt->dirty_cache_entries_list, entry) < 0) {
-                    kvstxn_cleanup_dirty_cache_entry (kt, entry);
-                    errno = ENOMEM;
+                if (kvstxn_add_dirty_cache_entry (kt, entry) < 0)
                     return -1;
-                }
             }
             if (!(ktmp = treeobj_create_dirref (ref)))
                 return -1;
@@ -375,23 +435,20 @@ static int kvstxn_unroll (kvstxn_t *kt, int current_epoch, json_t *dir)
         }
         else if (treeobj_is_val (dir_entry)) {
             json_t *val_data;
-            const char *str;
 
             if (!(val_data = treeobj_get_data (dir_entry)))
                 return -1;
-            /* jansson >= 2.7 could use json_string_length() instead */
-            str = json_string_value (val_data);
-            assert (str);
-            if (strlen (str) > BLOBREF_MAX_STRING_SIZE) {
-                if ((ret = store_cache (kt, current_epoch, val_data,
-                                        true, ref, sizeof (ref), &entry)) < 0)
+            if (json_string_length (val_data) > BLOBREF_MAX_STRING_SIZE) {
+                if ((ret = store_cache (kt,
+                                        val_data,
+                                        true,
+                                        ref,
+                                        sizeof (ref),
+                                        &entry)) < 0)
                     return -1;
                 if (ret) {
-                    if (zlist_push (kt->dirty_cache_entries_list, entry) < 0) {
-                        kvstxn_cleanup_dirty_cache_entry (kt, entry);
-                        errno = ENOMEM;
+                    if (kvstxn_add_dirty_cache_entry (kt, entry) < 0)
                         return -1;
-                    }
                 }
                 if (!(ktmp = treeobj_create_valref (ref)))
                     return -1;
@@ -408,8 +465,10 @@ static int kvstxn_unroll (kvstxn_t *kt, int current_epoch, json_t *dir)
     return 0;
 }
 
-static int kvstxn_val_data_to_cache (kvstxn_t *kt, int current_epoch,
-                                     json_t *val, char *ref, int ref_len)
+static int kvstxn_val_data_to_cache (kvstxn_t *kt,
+                                     json_t *val,
+                                     char *ref,
+                                     int ref_len)
 {
     struct cache_entry *entry;
     json_t *val_data;
@@ -418,23 +477,27 @@ static int kvstxn_val_data_to_cache (kvstxn_t *kt, int current_epoch,
     if (!(val_data = treeobj_get_data (val)))
         return -1;
 
-    if ((ret = store_cache (kt, current_epoch, val_data,
-                            true, ref, ref_len, &entry)) < 0)
+    if ((ret = store_cache (kt,
+                            val_data,
+                            true,
+                            ref,
+                            ref_len,
+                            &entry)) < 0)
         return -1;
 
     if (ret) {
-        if (zlist_push (kt->dirty_cache_entries_list, entry) < 0) {
-            kvstxn_cleanup_dirty_cache_entry (kt, entry);
-            errno = ENOMEM;
+        if (kvstxn_add_dirty_cache_entry (kt, entry) < 0)
             return -1;
-        }
     }
 
     return 0;
 }
 
-static int kvstxn_append (kvstxn_t *kt, int current_epoch, json_t *dirent,
-                          json_t *dir, const char *final_name, bool *append)
+static int kvstxn_append (kvstxn_t *kt,
+                          json_t *dirent,
+                          json_t *dir,
+                          const char *final_name,
+                          bool *append)
 {
     json_t *entry;
 
@@ -449,6 +512,10 @@ static int kvstxn_append (kvstxn_t *kt, int current_epoch, json_t *dirent,
         /* entry not found, treat like normal insertion */
         if (treeobj_insert_entry (dir, final_name, dirent) < 0)
             return -1;
+        /* N.B. although this is an "insert", we still treat this as
+         * an "append".  If we don't, the "append" could be issued
+         * twice, leading to duplicated data.  See issue #6207. */
+        (*append) = true;
     }
     else if (treeobj_is_valref (entry)) {
         char ref[BLOBREF_MAX_STRING_SIZE];
@@ -465,8 +532,7 @@ static int kvstxn_append (kvstxn_t *kt, int current_epoch, json_t *dirent,
          * sitting in the KVS cache.
          */
 
-        if (kvstxn_val_data_to_cache (kt, current_epoch, dirent, ref,
-                                      sizeof (ref)) < 0)
+        if (kvstxn_val_data_to_cache (kt, dirent, ref, sizeof (ref)) < 0)
             return -1;
 
         if (!(cpy = treeobj_deep_copy (entry)))
@@ -504,12 +570,10 @@ static int kvstxn_append (kvstxn_t *kt, int current_epoch, json_t *dirent,
          * same as the treeobj valref case above.
          */
 
-        if (kvstxn_val_data_to_cache (kt, current_epoch, entry, ref1,
-                                      sizeof (ref1)) < 0)
+        if (kvstxn_val_data_to_cache (kt, entry, ref1, sizeof (ref1)) < 0)
             return -1;
 
-        if (kvstxn_val_data_to_cache (kt, current_epoch, dirent, ref2,
-                                      sizeof (ref2)) < 0)
+        if (kvstxn_val_data_to_cache (kt, dirent, ref2, sizeof (ref2)) < 0)
             return -1;
 
         if (!(ktmp = treeobj_create_valref (ref1)))
@@ -553,9 +617,11 @@ static int kvstxn_append (kvstxn_t *kt, int current_epoch, json_t *dirent,
 
 /* link (key, dirent) into directory 'dir'.
  */
-static int kvstxn_link_dirent (kvstxn_t *kt, int current_epoch,
-                               json_t *rootdir, const char *key,
-                               json_t *dirent, int flags,
+static int kvstxn_link_dirent (kvstxn_t *kt,
+                               json_t *rootdir,
+                               const char *key,
+                               json_t *dirent,
+                               int flags,
                                const char **missing_ref,
                                bool *append)
 {
@@ -574,7 +640,7 @@ static int kvstxn_link_dirent (kvstxn_t *kt, int current_epoch,
 
     /* Special case root
      */
-    if (strcmp (name, ".") == 0) {
+    if (streq (name, ".")) {
         saved_errno = EINVAL;
         goto done;
     }
@@ -592,21 +658,24 @@ static int kvstxn_link_dirent (kvstxn_t *kt, int current_epoch,
         }
 
         if (!(dir_entry = treeobj_get_entry (dir, name))) {
-            if (json_is_null (dirent)) /* key deletion - it doesn't exist so return */
-                goto success;
+            if (json_is_null (dirent))
+                goto success; /* key deletion - it doesn't exist so return */
             if (!(subdir = treeobj_create_dir ())) {
                 saved_errno = errno;
                 goto done;
             }
-            if (treeobj_insert_entry (dir, name, subdir) < 0) {
+            /* subdir just created above, no need to validate */
+            if (treeobj_insert_entry_novalidate (dir, name, subdir) < 0) {
                 saved_errno = errno;
                 json_decref (subdir);
                 goto done;
             }
             json_decref (subdir);
-        } else if (treeobj_is_dir (dir_entry)) {
+        }
+        else if (treeobj_is_dir (dir_entry)) {
             subdir = dir_entry;
-        } else if (treeobj_is_dirref (dir_entry)) {
+        }
+        else if (treeobj_is_dirref (dir_entry)) {
             struct cache_entry *entry;
             const char *ref;
             const json_t *subdirktmp;
@@ -618,7 +687,9 @@ static int kvstxn_link_dirent (kvstxn_t *kt, int current_epoch,
             }
 
             if (refcount != 1) {
-                flux_log (kt->ktm->h, LOG_ERR, "invalid dirref count: %d",
+                flux_log (kt->ktm->h,
+                          LOG_ERR,
+                          "invalid dirref count: %d",
                           refcount);
                 saved_errno = ENOTRECOVERABLE;
                 goto done;
@@ -629,7 +700,7 @@ static int kvstxn_link_dirent (kvstxn_t *kt, int current_epoch,
                 goto done;
             }
 
-            if (!(entry = cache_lookup (kt->ktm->cache, ref, current_epoch))
+            if (!(entry = cache_lookup (kt->ktm->cache, ref))
                 || !cache_entry_get_valid (entry)) {
                 *missing_ref = ref;
                 goto success; /* stall */
@@ -646,13 +717,15 @@ static int kvstxn_link_dirent (kvstxn_t *kt, int current_epoch,
                 goto done;
             }
 
-            if (treeobj_insert_entry (dir, name, subdir) < 0) {
+            /* copy from entry already in cache, assume novalidate ok */
+            if (treeobj_insert_entry_novalidate (dir, name, subdir) < 0) {
                 saved_errno = errno;
                 json_decref (subdir);
                 goto done;
             }
             json_decref (subdir);
-        } else if (treeobj_is_symlink (dir_entry)) {
+        }
+        else if (treeobj_is_symlink (dir_entry)) {
             const char *ns = NULL;
             const char *target = NULL;
             char *nkey = NULL;
@@ -664,17 +737,16 @@ static int kvstxn_link_dirent (kvstxn_t *kt, int current_epoch,
             assert (target);
 
             /* can't cross into a new namespace */
-            if (ns && strcmp (ns, kt->ktm->ns_name)) {
+            if (ns && !streq (ns, kt->ktm->ns_name)) {
                 saved_errno = EINVAL;
                 goto done;
             }
 
             if (asprintf (&nkey, "%s.%s", target, next) < 0) {
-                saved_errno = ENOMEM;
+                saved_errno = errno;
                 goto done;
             }
             if (kvstxn_link_dirent (kt,
-                                    current_epoch,
                                     rootdir,
                                     nkey,
                                     dirent,
@@ -687,14 +759,16 @@ static int kvstxn_link_dirent (kvstxn_t *kt, int current_epoch,
             }
             free (nkey);
             goto success;
-        } else {
-            if (json_is_null (dirent)) /* key deletion - it doesn't exist so return */
-                goto success;
+        }
+        else {
+            if (json_is_null (dirent))
+                goto success; /* key deletion - it doesn't exist so return */
             if (!(subdir = treeobj_create_dir ())) {
                 saved_errno = errno;
                 goto done;
             }
-            if (treeobj_insert_entry (dir, name, subdir) < 0) {
+            /* subdir just created above, no need to validate */
+            if (treeobj_insert_entry_novalidate (dir, name, subdir) < 0) {
                 saved_errno = errno;
                 json_decref (subdir);
                 goto done;
@@ -709,18 +783,18 @@ static int kvstxn_link_dirent (kvstxn_t *kt, int current_epoch,
      */
     if (!json_is_null (dirent)) {
         if (flags & FLUX_KVS_APPEND) {
-            if (kvstxn_append (kt,
-                               current_epoch,
-                               dirent,
-                               dir,
-                               name,
-                               append) < 0) {
+            if (kvstxn_append (kt, dirent, dir, name, append) < 0) {
                 saved_errno = errno;
                 goto done;
             }
         }
         else {
-            /* if not append, it's a normal insertion */
+            /* if not append, it's a normal insertion
+             *
+             * N.B. this is the primary insertion and what is being
+             * inserted must be checked.  So we cannot use the
+             * novalidate alternative function.
+             */
             if (treeobj_insert_entry (dir, name, dirent) < 0) {
                 saved_errno = errno;
                 goto done;
@@ -747,65 +821,34 @@ static int kvstxn_link_dirent (kvstxn_t *kt, int current_epoch,
 
 static int add_missing_ref (kvstxn_t *kt, const char *ref)
 {
-    char *refcpy = NULL;
-
-    if (!(refcpy = strdup (ref))) {
-        errno = ENOMEM;
-        goto err;
-    }
-
-    if (zlist_push (kt->missing_refs_list, (void *)refcpy) < 0) {
+    if (zlist_push (kt->missing_refs_list, (void *)ref) < 0) {
         errno = ENOMEM;
-        goto err;
+        return -1;
     }
-
-    if (! zlist_freefn (kt->missing_refs_list, (void *)refcpy,
-                        free, false))
-        goto err;
-
     return 0;
-
-err:
-    free (refcpy);
-    return -1;
 }
 
-/* normalize key for setroot, and add it to keys array, if unique */
-static int normalize_and_append_unique (json_t *keys, const char *key)
+/* normalize key for setroot, and add it to keys dict, if unique */
+static int normalize_and_add_unique (json_t *keys, const char *key)
 {
     char *key_norm;
-    size_t index;
-    json_t *value;
-    bool unique = true;
+    int rc = -1;
 
     if ((key_norm = kvs_util_normalize_key (key, NULL)) == NULL)
         return -1;
-    json_array_foreach (keys, index, value) {
-        const char *s = json_string_value (value);
-        if (s && !strcmp (s, key_norm)) {
-            unique = false;
-            break;
-        }
-    }
-    if (unique) {
-        json_t *o;
-        if (!(o = json_string (key_norm)))
-            goto error;
-        if (json_array_append_new (keys, o) < 0) {
-            json_decref (o);
-            goto error;
-        }
-    }
-    free (key_norm);
-    return 0;
+    /* we don't need the value, just use a json null */
+    if (json_object_set_new (keys, key_norm, json_null ()) < 0)
+        goto error;
+    rc = 0;
 error:
     free (key_norm);
-    return -1;
+    return rc;
 }
 
-/* Create array of keys (strings) from array of operations ({ "key":s ... })
- * The keys array is for inclusion in the kvs.setroot event, so we can
- * notify watchers of keys that their key may have changed.
+/* Create dict of keys (strings) from array of operations ({ "key":s ... })
+ * The keys are for inclusion in the kvs.setroot event, so we can
+ * notify watchers of keys that their key may have changed.  The value in
+ * the dict is unneeded, so we set it to a json null.
  */
 static json_t *keys_from_ops (json_t *ops)
 {
@@ -813,13 +856,13 @@ static json_t *keys_from_ops (json_t *ops)
     size_t index;
     json_t *op;
 
-    if (!(keys = json_array ()))
+    if (!(keys = json_object ()))
         return NULL;
     json_array_foreach (ops, index, op) {
         const char *key;
         if (json_unpack (op, "{s:s}", "key", &key) < 0)
             goto error;
-        if (normalize_and_append_unique (keys, key) < 0)
+        if (normalize_and_add_unique (keys, key) < 0)
             goto error;
     }
     return keys;
@@ -829,218 +872,314 @@ static json_t *keys_from_ops (json_t *ops)
 }
 
 kvstxn_process_t kvstxn_process (kvstxn_t *kt,
-                                 int current_epoch,
-                                 const char *rootdir_ref)
+                                 const char *root_ref,
+                                 int root_seq)
 {
     /* Incase user calls kvstxn_process() again */
     if (kt->errnum)
         return KVSTXN_PROCESS_ERROR;
 
-    if (!(kt->internal_flags & KVSTXN_PROCESSING)) {
+    if (!kt->processing) {
         kt->errnum = EINVAL;
         return KVSTXN_PROCESS_ERROR;
     }
 
-    switch (kt->state) {
-    case KVSTXN_STATE_INIT:
-    case KVSTXN_STATE_LOAD_ROOT:
-    {
-        /* Make a copy of the root directory.
-         */
-        struct cache_entry *entry;
+    /* Only exit the loop by returning from the function */
+    while (1) {
+        if (kt->state == KVSTXN_STATE_INIT) {
+            /* Do some initial checks */
+            if (kt->flags & FLUX_KVS_SYNC
+                && !streq (kt->ktm->ns_name, KVS_PRIMARY_NAMESPACE)) {
+                kt->errnum = EINVAL;
+                return KVSTXN_PROCESS_ERROR;
+            }
+            kt->state = KVSTXN_STATE_LOAD_ROOT;
+        }
+        else if (kt->state == KVSTXN_STATE_LOAD_ROOT) {
+            /* Make a copy of the root directory.
+             */
+            struct cache_entry *entry;
 
-        /* Caller didn't call kvstxn_iter_missing_refs() */
-        if (zlist_first (kt->missing_refs_list))
-            goto stall_load;
+            /* Caller didn't call kvstxn_iter_missing_refs() */
+            if (zlist_first (kt->missing_refs_list)) {
+                kt->blocked = 1;
+                return KVSTXN_PROCESS_LOAD_MISSING_REFS;
+            }
 
-        kt->state = KVSTXN_STATE_LOAD_ROOT;
+            kt->state = KVSTXN_STATE_LOAD_ROOT;
 
-        if (!(entry = cache_lookup (kt->ktm->cache,
-                                    rootdir_ref,
-                                    current_epoch))
-            || !cache_entry_get_valid (entry)) {
+            if (!(entry = cache_lookup (kt->ktm->cache, root_ref))
+                || !cache_entry_get_valid (entry)) {
 
-            if (add_missing_ref (kt, rootdir_ref) < 0) {
-                kt->errnum = errno;
+                if (add_missing_ref (kt, root_ref) < 0) {
+                    kt->errnum = errno;
+                    return KVSTXN_PROCESS_ERROR;
+                }
+                kt->blocked = 1;
+                return KVSTXN_PROCESS_LOAD_MISSING_REFS;
+            }
+
+            if (!(kt->rootdir = cache_entry_get_treeobj (entry))) {
+                kt->errnum = ENOTRECOVERABLE;
                 return KVSTXN_PROCESS_ERROR;
             }
-            goto stall_load;
-        }
 
-        if (!(kt->rootdir = cache_entry_get_treeobj (entry))) {
-            kt->errnum = ENOTRECOVERABLE;
-            return KVSTXN_PROCESS_ERROR;
-        }
+            /* Special optimization, continue to APPLY_OPS state only
+             * if there are operations to process, otherwise we can
+             * skip to the GENERATE_KEYS state.  Sometimes operations
+             * can be zero length when using FLUX_KVS_SYNC or other
+             * flags.
+             */
+            if (json_array_size (kt->ops)) {
+                /* take reference because we're storing rootdir */
+                cache_entry_incref (entry);
+                kt->entry = entry;
 
-        /* take reference because we're storing rootdir */
-        cache_entry_incref (entry);
-        kt->entry = entry;
+                if (!(kt->rootcpy = treeobj_deep_copy (kt->rootdir))) {
+                    kt->errnum = errno;
+                    return KVSTXN_PROCESS_ERROR;
+                }
 
-        if (!(kt->rootcpy = treeobj_deep_copy (kt->rootdir))) {
-            kt->errnum = errno;
-            return KVSTXN_PROCESS_ERROR;
+                kt->state = KVSTXN_STATE_APPLY_OPS;
+            }
+            else {
+                /* place current rootref into newroot, it won't change */
+                strcpy (kt->newroot, root_ref);
+                kt->state = KVSTXN_STATE_GENERATE_KEYS;
+            }
         }
+        else if (kt->state == KVSTXN_STATE_APPLY_OPS) {
+            /* Apply each op (e.g. key = val) in sequence to the root
+             * copy.  A side effect of walking key paths is to convert
+             * dirref objects to dir objects in the copy.  This allows
+             * the transaction to be self-contained in the rootcpy
+             * until it is unrolled later on.
+             *
+             * Note that it is possible for multiple identical missing
+             * references to be added to the missing_refs_list list.
+             * Callers must deal with this appropriately.
+             */
+            json_t *op, *dirent;
+            const char *missing_ref = NULL;
+            int i, len = json_array_size (kt->ops);
+            const char *key;
+            int flags;
+            bool append = false;
+
+            /* Caller didn't call kvstxn_iter_missing_refs() */
+            if (zlist_first (kt->missing_refs_list)) {
+                kt->blocked = 1;
+                return KVSTXN_PROCESS_LOAD_MISSING_REFS;
+            }
 
-        kt->state = KVSTXN_STATE_APPLY_OPS;
-        /* fallthrough */
-    }
-    case KVSTXN_STATE_APPLY_OPS:
-    {
-        /* Apply each op (e.g. key = val) in sequence to the root
-         * copy.  A side effect of walking key paths is to convert
-         * dirref objects to dir objects in the copy.  This allows
-         * the transaction to be self-contained in the rootcpy
-         * until it is unrolled later on.
-         *
-         * Note that it is possible for multiple identical missing
-         * references to be added to the missing_refs_list list.
-         * Callers must deal with this appropriately.
-         */
-        json_t *op, *dirent;
-        const char *missing_ref = NULL;
-        int i, len = json_array_size (kt->ops);
-        const char *key;
-        int flags;
-        bool append = false;
+            for (i = 0; i < len; i++) {
+                missing_ref = NULL;
+                op = json_array_get (kt->ops, i);
+                assert (op != NULL);
+                if (txn_decode_op (op, &key, &flags, &dirent) < 0) {
+                    kt->errnum = errno;
+                    break;
+                }
+                if (kvstxn_link_dirent (kt,
+                                        kt->rootcpy,
+                                        key,
+                                        dirent,
+                                        flags,
+                                        &missing_ref,
+                                        &append) < 0) {
+                    kt->errnum = errno;
+                    break;
+                }
+                if (missing_ref) {
+                    if (add_missing_ref (kt, missing_ref) < 0) {
+                        kt->errnum = errno;
+                        break;
+                    }
+                }
+            }
 
-        /* Caller didn't call kvstxn_iter_missing_refs() */
-        if (zlist_first (kt->missing_refs_list))
-            goto stall_load;
+            if (kt->errnum != 0) {
+                /* empty missing_refs_list to prevent mistakes later */
+                zlist_purge (kt->missing_refs_list);
+                return KVSTXN_PROCESS_ERROR;
+            }
 
-        for (i = 0; i < len; i++) {
-            missing_ref = NULL;
-            op = json_array_get (kt->ops, i);
-            assert (op != NULL);
-            if (txn_decode_op (op, &key, &flags, &dirent) < 0) {
-                kt->errnum = errno;
-                break;
+            if (zlist_first (kt->missing_refs_list)) {
+                /* if we are stalling and an append has been done on the
+                 * rootcpy, we cannot re-apply the operations on the
+                 * replay of this transaction.  It would result in
+                 * duplicate appends on a key.  We'll start over with a
+                 * fresh rootcpy on the replay. */
+                if (append) {
+                    json_decref (kt->rootcpy);
+                    if (!(kt->rootcpy = treeobj_deep_copy (kt->rootdir))) {
+                        kt->errnum = errno;
+                        return KVSTXN_PROCESS_ERROR;
+                    }
+                }
+                kt->blocked = 1;
+                return KVSTXN_PROCESS_LOAD_MISSING_REFS;
             }
-            if (kvstxn_link_dirent (kt,
-                                    current_epoch,
-                                    kt->rootcpy,
-                                    key,
-                                    dirent,
-                                    flags,
-                                    &missing_ref,
-                                    &append) < 0) {
+
+            kt->state = KVSTXN_STATE_STORE;
+        }
+        else if (kt->state == KVSTXN_STATE_STORE) {
+            /* Unroll the root copy.
+             * When a dir is found, store an object and replace it
+             * with a dirref.  Finally, store the unrolled root copy
+             * as an object and keep its reference in kt->newroot.
+             * Flushes to content cache are asynchronous but we don't
+             * proceed until they are completed.
+             */
+            struct cache_entry *entry;
+            int sret;
+
+            if (kvstxn_unroll (kt, kt->rootcpy) < 0)
                 kt->errnum = errno;
-                break;
-            }
-            if (missing_ref) {
-                if (add_missing_ref (kt, missing_ref) < 0) {
+            else if ((sret = store_cache (kt,
+                                          kt->rootcpy,
+                                          false,
+                                          kt->newroot,
+                                          sizeof (kt->newroot),
+                                          &entry)) < 0)
+                kt->errnum = errno;
+            else if (sret) {
+                if (kvstxn_add_dirty_cache_entry (kt, entry) < 0)
                     kt->errnum = errno;
-                    break;
-                }
             }
-        }
 
-        if (kt->errnum != 0) {
-            char *ref;
-            /* empty missing_refs_list to prevent mistakes later */
-            while ((ref = zlist_pop (kt->missing_refs_list)))
-                free (ref);
-            return KVSTXN_PROCESS_ERROR;
+            if (kt->errnum) {
+                cleanup_dirty_cache_list (kt);
+                return KVSTXN_PROCESS_ERROR;
+            }
+
+            /* cache now has ownership of rootcpy, we don't need our
+             * rootcpy anymore.  But we may still need to stall user.
+             */
+            kt->state = KVSTXN_STATE_GENERATE_KEYS;
+            json_decref (kt->rootcpy);
+            kt->rootcpy = NULL;
+
+            /* the cache entry for the new root has the chance to expire
+             * in between the processing of dirty cache entries and the
+             * user being done with the transaction.  Therefore a call
+             * later to kvstxn_get_newroot_ref() could result in a
+             * reference that no longer has its cache entry valid.  We'll
+             * take an additional reference on the cache entry to ensure
+             * it can't expire until the transaction is completed.
+             */
+            kt->newroot_entry = entry;
+            cache_entry_incref (kt->newroot_entry);
         }
+        else if (kt->state == KVSTXN_STATE_GENERATE_KEYS) {
+            /* Caller didn't call kvstxn_iter_dirty_cache_entries() */
+            if (zlist_first (kt->dirty_cache_entries_list)) {
+                kt->blocked = 1;
+                return KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES;
+            }
 
-        if (zlist_first (kt->missing_refs_list)) {
-            /* if we are stalling and an append has been done on the
-             * rootcpy, we cannot re-apply the operations on the
-             * replay of this transaction.  It would result in
-             * duplicate appends on a key.  We'll start over with a
-             * fresh rootcpy on the replay. */
-            if (append) {
-                json_decref (kt->rootcpy);
-                if (!(kt->rootcpy = treeobj_deep_copy (kt->rootdir))) {
+            /* now generate keys for setroot */
+            if (!(kt->keys = keys_from_ops (kt->ops))) {
+                kt->errnum = ENOMEM;
+                return KVSTXN_PROCESS_ERROR;
+            }
+
+            if (kt->flags & FLUX_KVS_SYNC)
+                kt->state = KVSTXN_STATE_SYNC_CONTENT_FLUSH;
+            else
+                kt->state = KVSTXN_STATE_FINISHED;
+        }
+        else if (kt->state == KVSTXN_STATE_SYNC_CONTENT_FLUSH) {
+            if (!(kt->f_sync_content_flush)) {
+                kt->f_sync_content_flush = flux_rpc (kt->ktm->h,
+                                                     "content.flush",
+                                                     NULL,
+                                                     0,
+                                                     0);
+                if (!kt->f_sync_content_flush) {
                     kt->errnum = errno;
                     return KVSTXN_PROCESS_ERROR;
                 }
+                kt->blocked = 1;
+                return KVSTXN_PROCESS_SYNC_CONTENT_FLUSH;
             }
-            goto stall_load;
-        }
 
-        kt->state = KVSTXN_STATE_STORE;
-        /* fallthrough */
-    }
-    case KVSTXN_STATE_STORE:
-    {
-        /* Unroll the root copy.
-         * When a dir is found, store an object and replace it
-         * with a dirref.  Finally, store the unrolled root copy
-         * as an object and keep its reference in kt->newroot.
-         * Flushes to content cache are asynchronous but we don't
-         * proceed until they are completed.
-         */
-        struct cache_entry *entry;
-        int sret;
-
-        if (kvstxn_unroll (kt, current_epoch, kt->rootcpy) < 0)
-            kt->errnum = errno;
-        else if ((sret = store_cache (kt,
-                                      current_epoch,
-                                      kt->rootcpy,
-                                      false,
-                                      kt->newroot,
-                                      sizeof (kt->newroot),
-                                      &entry)) < 0)
-            kt->errnum = errno;
-        else if (sret
-                 && zlist_push (kt->dirty_cache_entries_list, entry) < 0) {
-            kvstxn_cleanup_dirty_cache_entry (kt, entry);
-            kt->errnum = ENOMEM;
-        }
+            /* user did not wait for future to complex */
+            if (!flux_future_is_ready (kt->f_sync_content_flush)) {
+                kt->blocked = 1;
+                return KVSTXN_PROCESS_SYNC_CONTENT_FLUSH;
+            }
 
-        if (kt->errnum) {
-            cleanup_dirty_cache_list (kt);
-            return KVSTXN_PROCESS_ERROR;
+            if (flux_rpc_get (kt->f_sync_content_flush, NULL) < 0) {
+                kt->errnum = errno;
+                return KVSTXN_PROCESS_ERROR;
+            }
+
+            kt->state = KVSTXN_STATE_SYNC_CHECKPOINT;
         }
+        else if (kt->state == KVSTXN_STATE_SYNC_CHECKPOINT) {
+
+            if (!(kt->f_sync_checkpoint)) {
+                int newseq = root_seq;
+
+                /* if we're publishing, seq will be the seq after
+                 * the current one.
+                 */
+                if (!(kt->internal_flags & KVSTXN_INTERNAL_FLAG_NO_PUBLISH))
+                    newseq++;
+
+                kt->f_sync_checkpoint = kvs_checkpoint_commit (kt->ktm->h,
+                                                               NULL,
+                                                               kt->newroot,
+                                                               newseq,
+                                                               0,
+                                                               0);
+                if (!kt->f_sync_checkpoint) {
+                    kt->errnum = errno;
+                    return KVSTXN_PROCESS_ERROR;
+                }
+                kt->blocked = 1;
+                return KVSTXN_PROCESS_SYNC_CHECKPOINT;
+            }
 
-        /* cache now has ownership of rootcpy, we don't need our
-         * rootcpy anymore.  But we may still need to stall user.
-         */
-        kt->state = KVSTXN_STATE_PRE_FINISHED;
-        json_decref (kt->rootcpy);
-        kt->rootcpy = NULL;
+            /* user did not wait for future to complex */
+            if (!flux_future_is_ready (kt->f_sync_checkpoint)) {
+                kt->blocked = 1;
+                return KVSTXN_PROCESS_SYNC_CHECKPOINT;
+            }
 
-        /* fallthrough */
-    }
-    case KVSTXN_STATE_PRE_FINISHED:
-        /* If we did not fall through to here, caller didn't call
-         * kvstxn_iter_dirty_cache_entries()
-         */
-        if (zlist_first (kt->dirty_cache_entries_list))
-            goto stall_store;
+            if (flux_rpc_get (kt->f_sync_checkpoint, NULL) < 0) {
+                kt->errnum = errno;
+                return KVSTXN_PROCESS_ERROR;
+            }
 
-        /* now generate keys for setroot */
-        if (!(kt->keys = keys_from_ops (kt->ops))) {
-            kt->errnum = ENOMEM;
+            /* N.B. After confirmation that a checkpoint is
+             * successful, immediately goto the FINISHED state so the
+             * kvs can transition to the new root reference.  We
+             * cannot do anything else that can lead to an error.  If
+             * an error would occur, we would have checkpointed a root
+             * reference that has never been the actual root reference
+             * of the KVS.
+             */
+            kt->state = KVSTXN_STATE_FINISHED;
+        }
+        else if (kt->state == KVSTXN_STATE_FINISHED) {
+            return KVSTXN_PROCESS_FINISHED;
+        }
+        else {
+            flux_log (kt->ktm->h, LOG_ERR, "invalid kvstxn state: %d", kt->state);
+            kt->errnum = ENOTRECOVERABLE;
             return KVSTXN_PROCESS_ERROR;
         }
-
-        kt->state = KVSTXN_STATE_FINISHED;
-        /* fallthrough */
-    case KVSTXN_STATE_FINISHED:
-        break;
-    default:
-        flux_log (kt->ktm->h, LOG_ERR, "invalid kvstxn state: %d", kt->state);
-        kt->errnum = ENOTRECOVERABLE;
-        return KVSTXN_PROCESS_ERROR;
     }
 
-    return KVSTXN_PROCESS_FINISHED;
-
- stall_load:
-    kt->blocked = 1;
-    return KVSTXN_PROCESS_LOAD_MISSING_REFS;
-
- stall_store:
-    kt->blocked = 1;
-    return KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES;
+    /* UNREACHABLE */
+    return KVSTXN_PROCESS_ERROR;
 }
 
 int kvstxn_iter_missing_refs (kvstxn_t *kt, kvstxn_ref_f cb, void *data)
 {
     char *ref;
-    int saved_errno, rc = 0;
 
     if (kt->state != KVSTXN_STATE_LOAD_ROOT
         && kt->state != KVSTXN_STATE_APPLY_OPS) {
@@ -1050,21 +1189,15 @@ int kvstxn_iter_missing_refs (kvstxn_t *kt, kvstxn_ref_f cb, void *data)
 
     while ((ref = zlist_pop (kt->missing_refs_list))) {
         if (cb (kt, ref, data) < 0) {
+            int saved_errno = errno;
             free (ref);
-            saved_errno = errno;
-            rc = -1;
-            break;
+            zlist_purge (kt->missing_refs_list);
+            errno = saved_errno;
+            return -1;
         }
         free (ref);
     }
-
-    if (rc < 0) {
-        while ((ref = zlist_pop (kt->missing_refs_list)))
-            free (ref);
-        errno = saved_errno;
-    }
-
-    return rc;
+    return 0;
 }
 
 int kvstxn_iter_dirty_cache_entries (kvstxn_t *kt,
@@ -1072,27 +1205,42 @@ int kvstxn_iter_dirty_cache_entries (kvstxn_t *kt,
                                      void *data)
 {
     struct cache_entry *entry;
-    int saved_errno, rc = 0;
 
-    if (kt->state != KVSTXN_STATE_PRE_FINISHED) {
+    if (kt->state != KVSTXN_STATE_GENERATE_KEYS) {
         errno = EINVAL;
         return -1;
     }
 
     while ((entry = zlist_pop (kt->dirty_cache_entries_list))) {
+        cache_entry_decref (entry);
         if (cb (kt, entry, data) < 0) {
-            saved_errno = errno;
-            rc = -1;
-            break;
+            int saved_errno = errno;
+            cleanup_dirty_cache_list (kt);
+            errno = saved_errno;
+            return -1;
         }
     }
+    return 0;
+}
 
-    if (rc < 0) {
-        cleanup_dirty_cache_list (kt);
-        errno = saved_errno;
+flux_future_t *kvstxn_sync_content_flush (kvstxn_t *kt)
+{
+    if (kt->state != KVSTXN_STATE_SYNC_CONTENT_FLUSH) {
+        errno = EINVAL;
+        return NULL;
     }
 
-    return rc;
+    return kt->f_sync_content_flush;
+}
+
+flux_future_t *kvstxn_sync_checkpoint (kvstxn_t *kt)
+{
+    if (kt->state != KVSTXN_STATE_SYNC_CHECKPOINT) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    return kt->f_sync_checkpoint;
 }
 
 kvstxn_mgr_t *kvstxn_mgr_create (struct cache *cache,
@@ -1110,7 +1258,7 @@ kvstxn_mgr_t *kvstxn_mgr_create (struct cache *cache,
     }
 
     if (!(ktm = calloc (1, sizeof (*ktm)))) {
-        saved_errno = ENOMEM;
+        saved_errno = errno;
         goto error;
     }
     ktm->cache = cache;
@@ -1142,11 +1290,15 @@ void kvstxn_mgr_destroy (kvstxn_mgr_t *ktm)
 int kvstxn_mgr_add_transaction (kvstxn_mgr_t *ktm,
                                 const char *name,
                                 json_t *ops,
-                                int flags)
+                                int flags,
+                                int internal_flags)
 {
     kvstxn_t *kt;
+    int valid_internal_flags = KVSTXN_INTERNAL_FLAG_NO_PUBLISH;
 
-    if (!name || !ops) {
+    if (!name
+        || !ops
+        || (internal_flags & ~valid_internal_flags)) {
         errno = EINVAL;
         return -1;
     }
@@ -1154,7 +1306,8 @@ int kvstxn_mgr_add_transaction (kvstxn_mgr_t *ktm,
     if (!(kt = kvstxn_create (ktm,
                               name,
                               ops,
-                              flags)))
+                              flags,
+                              internal_flags)))
         return -1;
 
     if (zlist_append (ktm->ready, kt) < 0) {
@@ -1180,28 +1333,29 @@ kvstxn_t *kvstxn_mgr_get_ready_transaction (kvstxn_mgr_t *ktm)
 {
     if (kvstxn_mgr_transaction_ready (ktm)) {
         kvstxn_t *kt = zlist_first (ktm->ready);
-        kt->internal_flags |= KVSTXN_PROCESSING;
+        kt->processing = true;
         return kt;
     }
     return NULL;
 }
 
-void kvstxn_mgr_remove_transaction (kvstxn_mgr_t *ktm, kvstxn_t *kt,
+void kvstxn_mgr_remove_transaction (kvstxn_mgr_t *ktm,
+                                    kvstxn_t *kt,
                                     bool fallback)
 {
-    if (kt->internal_flags & KVSTXN_PROCESSING) {
+    if (kt->processing) {
         bool kvstxn_is_merged = false;
 
-        if (kt->internal_flags & KVSTXN_MERGED)
+        if (kt->merged)
             kvstxn_is_merged = true;
 
         zlist_remove (ktm->ready, kt);
 
         if (kvstxn_is_merged) {
             kvstxn_t *kt_tmp = zlist_first (ktm->ready);
-            while (kt_tmp && (kt_tmp->internal_flags & KVSTXN_MERGE_COMPONENT)) {
+            while (kt_tmp && kt_tmp->merge_component) {
                 if (fallback) {
-                    kt_tmp->internal_flags &= ~KVSTXN_MERGE_COMPONENT;
+                    kt_tmp->merge_component = false;
                     kt_tmp->flags |= FLUX_KVS_NO_MERGE;
                 }
                 else
@@ -1228,11 +1382,23 @@ int kvstxn_mgr_ready_transaction_count (kvstxn_mgr_t *ktm)
     return zlist_size (ktm->ready);
 }
 
+/* N.B. FLUX_KVS_SYNC implies FLUX_KVS_NO_MERGE, as we checkpoint
+ * after the specific commit completes.  So FLUX_KVS_SYNC is
+ * treated identically to FLUX_KVS_NO_MERGE in merge logic.
+ */
+static bool kvstxn_no_merge (kvstxn_t *kt)
+{
+    if ((kt->flags & FLUX_KVS_NO_MERGE)
+        || (kt->flags & FLUX_KVS_SYNC))
+        return true;
+    return false;
+}
+
 static int kvstxn_merge (kvstxn_t *dest, kvstxn_t *src)
 {
     int i, len;
 
-    if (src->flags & FLUX_KVS_NO_MERGE
+    if (kvstxn_no_merge (src)
         || dest->flags != src->flags)
         return 0;
 
@@ -1258,17 +1424,6 @@ static int kvstxn_merge (kvstxn_t *dest, kvstxn_t *src)
             }
         }
     }
-    if ((len = json_array_size (src->keys))) {
-        for (i = 0; i < len; i++) {
-            json_t *key;
-            if ((key = json_array_get (src->keys, i))) {
-                if (json_array_append (dest->keys, key) < 0) {
-                    errno = ENOMEM;
-                    return -1;
-                }
-            }
-        }
-    }
 
     return 1;
 }
@@ -1282,7 +1437,7 @@ static int kvstxn_merge (kvstxn_t *dest, kvstxn_t *src)
  *
  * Break when an unmergeable transaction is discovered.  We do not
  * wish to merge non-adjacent transactions, as it can create
- * undesireable out of order scenarios.  e.g.
+ * undesirable out of order scenarios.  e.g.
  *
  * transaction #1 is mergeable:     set A=1
  * transaction #2 is non-mergeable: set A=2
@@ -1299,25 +1454,30 @@ int kvstxn_mgr_merge_ready_transactions (kvstxn_mgr_t *ktm)
     int count = 0;
 
     /* transaction must still be in state where merged in ops can be
-     * applied */
+     * applied. */
     first = zlist_first (ktm->ready);
     if (!first
         || first->errnum != 0
         || first->aux_errnum != 0
         || first->state > KVSTXN_STATE_APPLY_OPS
-        || (first->flags & FLUX_KVS_NO_MERGE)
-        || first->internal_flags & KVSTXN_MERGED)
+        || kvstxn_no_merge (first)
+        || first->merged)
         return 0;
 
     second = zlist_next (ktm->ready);
     if (!second
-        || (second->flags & FLUX_KVS_NO_MERGE)
-        || (first->flags != second->flags))
+        || kvstxn_no_merge (second)
+        || (first->flags != second->flags)
+        || (first->internal_flags != second->internal_flags))
         return 0;
 
-    if (!(new = kvstxn_create (ktm, NULL, NULL, first->flags)))
+    if (!(new = kvstxn_create (ktm,
+                               NULL,
+                               NULL,
+                               first->flags,
+                               first->internal_flags)))
         return -1;
-    new->internal_flags |= KVSTXN_MERGED;
+    new->merged = true;
 
     nextkt = zlist_first (ktm->ready);
     do {
@@ -1349,11 +1509,10 @@ int kvstxn_mgr_merge_ready_transactions (kvstxn_mgr_t *ktm)
     nextkt = zlist_first (ktm->ready);
     nextkt = zlist_next (ktm->ready);
     do {
-        /* Wipe out KVSTXN_PROCESSING flag if user previously got
-         * the kvstxn_t
+        /* reset processing flag if user previously got the kvstxn_t
          */
-        nextkt->internal_flags &= ~KVSTXN_PROCESSING;
-        nextkt->internal_flags |= KVSTXN_MERGE_COMPONENT;
+        nextkt->processing = false;
+        nextkt->merge_component = true;
     } while (--count && (nextkt = zlist_next (ktm->ready)));
 
     return 0;
diff --git a/src/modules/kvs/kvstxn.h b/src/modules/kvs/kvstxn.h
index a62d89c70284..1e715cf35b12 100644
--- a/src/modules/kvs/kvstxn.h
+++ b/src/modules/kvs/kvstxn.h
@@ -12,7 +12,6 @@
 #define _FLUX_KVS_KVSTXN_H
 
 #include <flux/core.h>
-#include <czmq.h>
 
 #include "cache.h"
 
@@ -23,9 +22,21 @@ typedef enum {
     KVSTXN_PROCESS_ERROR = 1,
     KVSTXN_PROCESS_LOAD_MISSING_REFS = 2,
     KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES = 3,
-    KVSTXN_PROCESS_FINISHED = 4,
+    KVSTXN_PROCESS_SYNC_CONTENT_FLUSH = 4,
+    KVSTXN_PROCESS_SYNC_CHECKPOINT = 5,
+    KVSTXN_PROCESS_FINISHED = 6,
 } kvstxn_process_t;
 
+/* api flags, to be used with kvstxn_mgr_add_transaction()
+ *
+ * KVSTXN_INTERNAL_FLAG_NO_PUBLISH - Indicate that this transaction
+ * should not publish its change after the transaction completes.
+ * Note that kvstxn does not use this flag internally, users can check
+ * that it has been set via kvstxn_get_internal_flags().
+ */
+
+#define KVSTXN_INTERNAL_FLAG_NO_PUBLISH 0x01
+
 /*
  * kvstxn_t API
  */
@@ -59,6 +70,7 @@ bool kvstxn_fallback_mergeable (kvstxn_t *kt);
 json_t *kvstxn_get_ops (kvstxn_t *kt);
 json_t *kvstxn_get_names (kvstxn_t *kt);
 int kvstxn_get_flags (kvstxn_t *kt);
+int kvstxn_get_internal_flags (kvstxn_t *kt);
 
 /* returns namespace passed into kvstxn_mgr_create() */
 const char *kvstxn_get_namespace (kvstxn_t *kt);
@@ -83,6 +95,8 @@ json_t *kvstxn_get_keys (kvstxn_t *kt);
  * KVSTXN_PROCESS_LOAD_MISSING_REFS stall & load,
  * KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES stall & process dirty cache
  * entries,
+ * KVSTXN_PROCESS_SYNC_CONTENT_FLUSH stall & wait for future to fulfill
+ * KVSTXN_PROCESS_SYNC_CHECKPOINT stall & wait for future to fulfill
  * KVSTXN_PROCESS_FINISHED all done
  *
  * on error, call kvstxn_get_errnum() to get error number
@@ -92,12 +106,16 @@ json_t *kvstxn_get_keys (kvstxn_t *kt);
  * on stall & process dirty cache entries, call
  * kvstxn_iter_dirty_cache_entries() to process entries.
  *
+ * on stall & content-flush, call kvstxn_sync_content_flush() to get future.
+ *
+ * on stall & checkpoint, call kvstxn_sync_checkpoint() to get future.
+ *
  * on completion, call kvstxn_get_newroot_ref() to get reference to
  * new root to be stored.
  */
 kvstxn_process_t kvstxn_process (kvstxn_t *kt,
-                                 int current_epoch,
-                                 const char *rootdir_ref);
+                                 const char *root_ref,
+                                 int root_seq);
 
 /* on stall, iterate through all missing refs that the caller should
  * load into the cache
@@ -122,12 +140,18 @@ int kvstxn_iter_dirty_cache_entries (kvstxn_t *kt,
  */
 void kvstxn_cleanup_dirty_cache_entry (kvstxn_t *kt, struct cache_entry *entry);
 
+/* on stall, get confent.flush future to wait for fulfillment on */
+flux_future_t *kvstxn_sync_content_flush (kvstxn_t *kt);
+
+/* on stall, get checkpoint future to wait for fulfillment on */
+flux_future_t *kvstxn_sync_checkpoint (kvstxn_t *kt);
+
 /*
  * kvstxn_mgr_t API
  */
 
 /* flux_t is optional, if NULL logging will go to stderr */
-kvstxn_mgr_t *kvstxn_mgr_create (struct cache *ktache,
+kvstxn_mgr_t *kvstxn_mgr_create (struct cache *cache,
                                  const char *ns,
                                  const char *hash_name,
                                  flux_t *h,
@@ -144,7 +168,8 @@ void kvstxn_mgr_destroy (kvstxn_mgr_t *ktm);
 int kvstxn_mgr_add_transaction (kvstxn_mgr_t *ktm,
                                 const char *name,
                                 json_t *ops,
-                                int flags);
+                                int flags,
+                                int internal_flags);
 
 /* returns true if there is a transaction ready for processing and is
  * not blocked, false if not.
@@ -172,7 +197,8 @@ kvstxn_t *kvstxn_mgr_get_ready_transaction (kvstxn_mgr_t *ktm);
  * an error (i.e. you don't use kvstxn_get_newroot_ref to get a new
  * root).
  */
-void kvstxn_mgr_remove_transaction (kvstxn_mgr_t *ktm, kvstxn_t *kt,
+void kvstxn_mgr_remove_transaction (kvstxn_mgr_t *ktm,
+                                    kvstxn_t *kt,
                                     bool fallback);
 
 int kvstxn_mgr_get_noop_stores (kvstxn_mgr_t *ktm);
diff --git a/src/modules/kvs/lookup.c b/src/modules/kvs/lookup.c
index abd04b1f1477..e95488494b19 100644
--- a/src/modules/kvs/lookup.c
+++ b/src/modules/kvs/lookup.c
@@ -11,6 +11,7 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -18,13 +19,15 @@
 #include <unistd.h>
 #include <stdbool.h>
 #include <ctype.h>
-#include <czmq.h>
 #include <flux/core.h>
 #include <jansson.h>
+#include <assert.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/blobref.h"
 #include "src/common/libkvs/treeobj.h"
 #include "src/common/libkvs/kvs_util_private.h"
+#include "ccan/str/str.h"
 
 #include "cache.h"
 #include "kvsroot.h"
@@ -54,7 +57,6 @@ struct lookup {
     /* inputs from user */
     struct cache *cache;
     kvsroot_mgr_t *krm;
-    int current_epoch;
 
     char *ns_name;
     char *root_ref;
@@ -181,16 +183,16 @@ static walk_level_t *walk_level_create (const char *root_ref,
     int saved_errno;
 
     if (!wl) {
-        saved_errno = ENOMEM;
+        saved_errno = errno;
         goto error;
     }
     if (!(wl->path_copy = strdup (path))) {
-        saved_errno = ENOMEM;
+        saved_errno = errno;
         goto error;
     }
     wl->depth = depth;
     if (!(wl->root_ref = strdup (root_ref))) {
-        saved_errno = ENOMEM;
+        saved_errno = errno;
         goto error;
     }
     if (!(wl->root_dirent = treeobj_create_dirref (root_ref))) {
@@ -243,7 +245,7 @@ static lookup_process_t symlink_check_namespace (lookup_t *lh,
     if (!root) {
         free (lh->missing_namespace);
         if (!(lh->missing_namespace = strdup (ns))) {
-            lh->errnum = ENOMEM;
+            lh->errnum = errno;
             goto done;
         }
         ret = LOOKUP_PROCESS_LOAD_MISSING_NAMESPACE;
@@ -312,7 +314,7 @@ static lookup_process_t walk_symlink (lookup_t *lh,
         /* if symlink target is root, no need to recurse, just get
          * root_dirent and continue on.
          */
-        if (!strcmp (target, ".")) {
+        if (streq (target, ".")) {
             if (root) {
                 json_decref (wl->tmp_dirent);
                 wl->tmp_dirent = treeobj_create_dirref (root->ref);
@@ -393,7 +395,7 @@ static lookup_process_t walk (lookup_t *lh)
                 goto error;
             }
 
-            if (!(entry = cache_lookup (lh->cache, refstr, lh->current_epoch))
+            if (!(entry = cache_lookup (lh->cache, refstr))
                 || !cache_entry_get_valid (entry)) {
                 lh->missing_ref = refstr;
                 return LOOKUP_PROCESS_LOAD_MISSING_REFS;
@@ -429,11 +431,16 @@ static lookup_process_t walk (lookup_t *lh)
             }
             else {
                 char *s = json_dumps (wl->dirent, JSON_ENCODE_ANY);
-                flux_log (lh->h, LOG_ERR,
+                flux_log (lh->h,
+                          LOG_ERR,
                           "%s: unknown/unexpected dirent type: "
                           "lh->path=%s pathcomp=%s wl->dirent(ptr)=%p "
                           "wl->dirent(str)=%s",
-                          __FUNCTION__, lh->path, pathcomp, wl->dirent, s);
+                          __FUNCTION__,
+                          lh->path,
+                          pathcomp,
+                          wl->dirent,
+                          s);
                 free (s);
                 lh->errnum = ENOTRECOVERABLE;
                 goto error;
@@ -512,7 +519,6 @@ static lookup_process_t walk (lookup_t *lh)
 
 lookup_t *lookup_create (struct cache *cache,
                          kvsroot_mgr_t *krm,
-                         int current_epoch,
                          const char *ns,
                          const char *root_ref,
                          int root_seq,
@@ -536,18 +542,17 @@ lookup_t *lookup_create (struct cache *cache,
     }
 
     if (!(lh = calloc (1, sizeof (*lh)))) {
-        saved_errno = ENOMEM;
+        saved_errno = errno;
         goto cleanup;
     }
 
     lh->cache = cache;
     lh->krm = krm;
-    lh->current_epoch = current_epoch;
 
     if (ns) {
         /* must duplicate strings, user may not keep pointer alive */
         if (!(lh->ns_name = strdup (ns))) {
-            saved_errno = ENOMEM;
+            saved_errno = errno;
             goto cleanup;
         }
     }
@@ -559,7 +564,7 @@ lookup_t *lookup_create (struct cache *cache,
 
     if (root_ref) {
         if (!(lh->root_ref = strdup (root_ref))) {
-            saved_errno = ENOMEM;
+            saved_errno = errno;
             goto cleanup;
         }
         lh->root_seq = root_seq;
@@ -668,7 +673,7 @@ int lookup_iter_missing_refs (lookup_t *lh, lookup_ref_f cb, void *data)
                 if (!(ref = treeobj_get_blobref (lh->valref_missing_refs, i)))
                     return -1;
 
-                if (!(entry = cache_lookup (lh->cache, ref, lh->current_epoch))
+                if (!(entry = cache_lookup (lh->cache, ref))
                     || !cache_entry_get_valid (entry)) {
 
                     /* valref points to raw data, raw_data flag is always
@@ -699,13 +704,6 @@ const char *lookup_missing_namespace (lookup_t *lh)
     return NULL;
 }
 
-int lookup_get_current_epoch (lookup_t *lh)
-{
-    if (lh)
-        return lh->current_epoch;
-    return -1;
-}
-
 const char *lookup_get_namespace (lookup_t *lh)
 {
     if (lh)
@@ -727,15 +725,6 @@ int lookup_get_root_seq (lookup_t *lh)
     return -1;
 }
 
-int lookup_set_current_epoch (lookup_t *lh, int epoch)
-{
-    if (lh) {
-        lh->current_epoch = epoch;
-        return 0;
-    }
-    return -1;
-}
-
 static int namespace_still_valid (lookup_t *lh)
 {
     struct kvsroot *root;
@@ -773,7 +762,7 @@ static int get_single_blobref_valref_value (lookup_t *lh, bool *stall)
         lh->errnum = errno;
         return -1;
     }
-    if (!(entry = cache_lookup (lh->cache, reftmp, lh->current_epoch))
+    if (!(entry = cache_lookup (lh->cache, reftmp))
         || !cache_entry_get_valid (entry)) {
         lh->valref_missing_refs = lh->wdirent;
         (*stall) = true;
@@ -792,8 +781,10 @@ static int get_single_blobref_valref_value (lookup_t *lh, bool *stall)
     return 0;
 }
 
-static int get_multi_blobref_valref_length (lookup_t *lh, int refcount,
-                                            int *total_len, bool *stall)
+static int get_multi_blobref_valref_length (lookup_t *lh,
+                                            int refcount,
+                                            int *total_len,
+                                            bool *stall)
 {
     struct cache_entry *entry;
     const char *reftmp;
@@ -806,7 +797,7 @@ static int get_multi_blobref_valref_length (lookup_t *lh, int refcount,
             lh->errnum = errno;
             return -1;
         }
-        if (!(entry = cache_lookup (lh->cache, reftmp, lh->current_epoch))
+        if (!(entry = cache_lookup (lh->cache, reftmp))
             || !cache_entry_get_valid (entry)) {
             lh->valref_missing_refs = lh->wdirent;
             (*stall) = true;
@@ -832,7 +823,8 @@ static int get_multi_blobref_valref_length (lookup_t *lh, int refcount,
     return 0;
 }
 
-static char *get_multi_blobref_valref_data (lookup_t *lh, int refcount,
+static char *get_multi_blobref_valref_data (lookup_t *lh,
+                                            int refcount,
                                             int total_len)
 {
     struct cache_entry *entry;
@@ -849,7 +841,7 @@ static char *get_multi_blobref_valref_data (lookup_t *lh, int refcount,
     }
 
     for (i = 0; i < refcount; i++) {
-        int ret;
+        __attribute__((unused)) int ret;
 
         /* this function should only be called if all cache entries
          * known to be valid & raw, thus assert checks below */
@@ -857,7 +849,7 @@ static char *get_multi_blobref_valref_data (lookup_t *lh, int refcount,
         reftmp = treeobj_get_blobref (lh->wdirent, i);
         assert (reftmp);
 
-        entry = cache_lookup (lh->cache, reftmp, lh->current_epoch);
+        entry = cache_lookup (lh->cache, reftmp);
         assert (entry);
         assert (cache_entry_get_valid (entry));
 
@@ -874,7 +866,8 @@ static char *get_multi_blobref_valref_data (lookup_t *lh, int refcount,
 
 /* return 0 on success, -1 on failure.  On success, stall should be
  * check */
-static int get_multi_blobref_valref_value (lookup_t *lh, int refcount,
+static int get_multi_blobref_valref_value (lookup_t *lh,
+                                           int refcount,
                                            bool *stall)
 {
     char *valbuf = NULL;
@@ -940,7 +933,7 @@ lookup_process_t lookup (lookup_t *lh)
                 if (!root) {
                     free (lh->missing_namespace);
                     if (!(lh->missing_namespace = strdup (lh->ns_name))) {
-                        lh->errnum = ENOMEM;
+                        lh->errnum = errno;
                         goto error;
                     }
                     return LOOKUP_PROCESS_LOAD_MISSING_NAMESPACE;
@@ -955,7 +948,7 @@ lookup_process_t lookup (lookup_t *lh)
                  * namespace could timeout or be removed when
                  * stalling */
                 if (!(lh->root_ref = strdup (root->ref))) {
-                    lh->errnum = ENOMEM;
+                    lh->errnum = errno;
                     goto error;
                 }
                 lh->root_seq = root->seq;
@@ -971,7 +964,7 @@ lookup_process_t lookup (lookup_t *lh)
             }
 
             /* special case root */
-            if (!strcmp (lh->path, ".")) {
+            if (streq (lh->path, ".")) {
                 if ((lh->flags & FLUX_KVS_TREEOBJ)) {
                     if (!(lh->val = treeobj_create_dirref (lh->root_ref))) {
                         lh->errnum = errno;
@@ -982,9 +975,7 @@ lookup_process_t lookup (lookup_t *lh)
                         lh->errnum = EISDIR;
                         goto error;
                     }
-                    if (!(entry = cache_lookup (lh->cache,
-                                                lh->root_ref,
-                                                lh->current_epoch))
+                    if (!(entry = cache_lookup (lh->cache, lh->root_ref))
                         || !cache_entry_get_valid (entry)) {
                         lh->missing_ref = lh->root_ref;
                         return LOOKUP_PROCESS_LOAD_MISSING_REFS;
@@ -1082,8 +1073,7 @@ lookup_process_t lookup (lookup_t *lh)
                     lh->errnum = errno;
                     goto error;
                 }
-                if (!(entry = cache_lookup (lh->cache, reftmp,
-                                            lh->current_epoch))
+                if (!(entry = cache_lookup (lh->cache, reftmp))
                     || !cache_entry_get_valid (entry)) {
                     lh->missing_ref = reftmp;
                     return LOOKUP_PROCESS_LOAD_MISSING_REFS;
@@ -1102,7 +1092,8 @@ lookup_process_t lookup (lookup_t *lh)
                     lh->errnum = errno;
                     goto error;
                 }
-            } else if (treeobj_is_valref (lh->wdirent)) {
+            }
+            else if (treeobj_is_valref (lh->wdirent)) {
                 bool stall;
 
                 if ((lh->flags & FLUX_KVS_READLINK)) {
@@ -1118,7 +1109,9 @@ lookup_process_t lookup (lookup_t *lh)
                     goto error;
                 }
                 if (!refcount) {
-                    flux_log (lh->h, LOG_ERR, "invalid valref count: %d",
+                    flux_log (lh->h,
+                              LOG_ERR,
+                              "invalid valref count: %d",
                               refcount);
                     lh->errnum = ENOTRECOVERABLE;
                     goto error;
@@ -1137,7 +1130,8 @@ lookup_process_t lookup (lookup_t *lh)
                     if (stall)
                         return LOOKUP_PROCESS_LOAD_MISSING_REFS;
                 }
-            } else if (treeobj_is_dir (lh->wdirent)) {
+            }
+            else if (treeobj_is_dir (lh->wdirent)) {
                 if ((lh->flags & FLUX_KVS_READLINK)) {
                     lh->errnum = EINVAL;
                     goto error;
@@ -1150,7 +1144,8 @@ lookup_process_t lookup (lookup_t *lh)
                     lh->errnum = errno;
                     goto error;
                 }
-            } else if (treeobj_is_val (lh->wdirent)) {
+            }
+            else if (treeobj_is_val (lh->wdirent)) {
                 if ((lh->flags & FLUX_KVS_READLINK)) {
                     lh->errnum = EINVAL;
                     goto error;
@@ -1163,7 +1158,8 @@ lookup_process_t lookup (lookup_t *lh)
                     lh->errnum = errno;
                     goto error;
                 }
-            } else if (treeobj_is_symlink (lh->wdirent)) {
+            }
+            else if (treeobj_is_symlink (lh->wdirent)) {
                 /* this should be "impossible" */
                 if (!(lh->flags & FLUX_KVS_READLINK)) {
                     lh->errnum = EPROTO;
@@ -1177,10 +1173,15 @@ lookup_process_t lookup (lookup_t *lh)
                     lh->errnum = errno;
                     goto error;
                 }
-            } else {
+            }
+            else {
                 char *s = json_dumps (lh->wdirent, JSON_ENCODE_ANY);
-                flux_log (lh->h, LOG_ERR, "%s: corrupt dirent: %p, %s",
-                          __FUNCTION__, lh->wdirent, s);
+                flux_log (lh->h,
+                          LOG_ERR,
+                          "%s: corrupt dirent: %p, %s",
+                          __FUNCTION__,
+                          lh->wdirent,
+                          s);
                 free (s);
                 lh->errnum = ENOTRECOVERABLE;
                 goto error;
@@ -1190,8 +1191,11 @@ lookup_process_t lookup (lookup_t *lh)
         case LOOKUP_STATE_FINISHED:
             break;
         default:
-            flux_log (lh->h, LOG_ERR, "%s: invalid state %d",
-                      __FUNCTION__, lh->state);
+            flux_log (lh->h,
+                      LOG_ERR,
+                      "%s: invalid state %d",
+                      __FUNCTION__,
+                      lh->state);
             lh->errnum = ENOTRECOVERABLE;
             goto error;
     }
diff --git a/src/modules/kvs/lookup.h b/src/modules/kvs/lookup.h
index 4dc5eeaad999..ba2ffe67202e 100644
--- a/src/modules/kvs/lookup.h
+++ b/src/modules/kvs/lookup.h
@@ -40,7 +40,6 @@ typedef int (*lookup_ref_f)(lookup_t *c,
  */
 lookup_t *lookup_create (struct cache *cache,
                          kvsroot_mgr_t *krm,
-                         int current_epoch,
                          const char *ns,
                          const char *root_ref,
                          int root_seq,
@@ -97,7 +96,7 @@ const char *lookup_get_namespace (lookup_t *lh);
  * root_ref will be the root_ref passed in via lookup_create() or the
  * root_ref used from the namespace.  The root_seq is only if the
  * root_ref was from a namespace.  Note that the values are not valid
- * unless the lookup completes (LOOKUP_PROCESS_FINISED).
+ * unless the lookup completes (LOOKUP_PROCESS_FINISHED).
  */
 const char *lookup_get_root_ref (lookup_t *lh);
 int lookup_get_root_seq (lookup_t *lh);
diff --git a/src/modules/kvs/test/cache.c b/src/modules/kvs/test/cache.c
index 83d15d117051..3669792f14f3 100644
--- a/src/modules/kvs/test/cache.c
+++ b/src/modules/kvs/test/cache.c
@@ -14,11 +14,16 @@
 #include <stdbool.h>
 #include <jansson.h>
 
+#ifndef EBADE
+#define EBADE EINVAL
+#endif
+
 #include "src/common/libkvs/treeobj.h"
 #include "src/common/libutil/tstat.h"
 #include "src/common/libtap/tap.h"
 #include "src/modules/kvs/waitqueue.h"
 #include "src/modules/kvs/cache.h"
+#include "ccan/str/str.h"
 
 static int cache_entry_set_treeobj (struct cache_entry *entry, const json_t *o)
 {
@@ -73,11 +78,15 @@ void cache_tests (void)
     struct cache *cache;
     tstat_t ts;
     int size, incomplete, dirty;
+    flux_reactor_t *r;
+
+    if (!(r = flux_reactor_create (0)))
+        BAIL_OUT ("flux_reactor_create failed");
 
     cache_destroy (NULL);
     diag ("cache_destroy accept NULL arg");
 
-    ok ((cache = cache_create ()) != NULL,
+    ok ((cache = cache_create (r)) != NULL,
         "cache_create works");
     ok (cache_count_entries (cache) == 0,
         "cache contains 0 entries");
@@ -89,6 +98,7 @@ void cache_tests (void)
     ok (incomplete == 0, "empty cache, incomplete == 0");
     ok (dirty == 0, "empty cache, dirty == 0");
     cache_destroy (cache);
+    flux_reactor_destroy (r);
 }
 
 void cache_entry_basic_tests (void)
@@ -190,7 +200,7 @@ void cache_entry_raw_tests (void)
 
     ok (cache_entry_get_raw (e, (const void **)&datatmp, &len) == 0,
         "raw data retrieved from cache entry");
-    ok (datatmp && strcmp (datatmp, data) == 0,
+    ok (datatmp && streq (datatmp, data),
         "raw data matches expected string");
     ok (datatmp && (len == strlen (data) + 1),
         "raw data length matches expected length");
@@ -198,21 +208,21 @@ void cache_entry_raw_tests (void)
     ok (cache_entry_set_dirty (e, true) == 0,
         "cache_entry_set_dirty success");
     ok (cache_entry_get_dirty (e) == true,
-        "cache entry succcessfully set dirty");
+        "cache entry successfully set dirty");
 
     ok (cache_entry_clear_dirty (e) == 0,
         "cache_entry_clear_dirty success");
     ok (cache_entry_get_dirty (e) == false,
-        "cache entry succcessfully now not dirty, b/c no waiters");
+        "cache entry successfully now not dirty, b/c no waiters");
 
     ok (cache_entry_set_dirty (e, true) == 0,
         "cache_entry_set_dirty success");
     ok (cache_entry_get_dirty (e) == true,
-        "cache entry succcessfully set dirty");
+        "cache entry successfully set dirty");
     ok (cache_entry_force_clear_dirty (e) == 0,
         "cache_entry_force_clear_dirty success");
     ok (cache_entry_get_dirty (e) == false,
-        "cache entry succcessfully now not dirty");
+        "cache entry successfully now not dirty");
 
     cache_entry_destroy (e); /* destroys data */
     free (data);
@@ -458,7 +468,7 @@ void cache_blobref_tests (void)
     struct cache_entry *e;
     const char *ref;
 
-    ok ((cache = cache_create ()) != NULL,
+    ok ((cache = cache_create (NULL)) != NULL,
         "cache_create works");
     ok ((e = cache_entry_create ("abcd")) != NULL,
         "cache_entry_create works");
@@ -466,7 +476,7 @@ void cache_blobref_tests (void)
         "cache_insert works");
     ok ((ref = cache_entry_get_blobref (e)) != NULL,
         "cache_entry_get_blobref success");
-    ok (!strcmp (ref, "abcd"),
+    ok (streq (ref, "abcd"),
         "cache_entry_get_blobref returned correct ref");
 
     cache_destroy (cache);
@@ -480,20 +490,20 @@ void cache_remove_entry_tests (void)
     wait_t *w;
     int count;
 
-    ok ((cache = cache_create ()) != NULL,
+    ok ((cache = cache_create (NULL)) != NULL,
         "cache_create works");
 
     ok ((e = cache_entry_create ("remove-ref")) != NULL,
         "cache_entry_create works");
     ok (cache_insert (cache, e) == 0,
         "cache_insert works");
-    ok (cache_lookup (cache, "remove-ref", 0) != NULL,
+    ok (cache_lookup (cache, "remove-ref") != NULL,
         "cache_lookup verify entry exists");
     ok (cache_remove_entry (cache, "blalalala") == 0,
         "cache_remove_entry failed on bad reference");
     ok (cache_remove_entry (cache, "remove-ref") == 1,
         "cache_remove_entry removed cache entry w/o object");
-    ok (cache_lookup (cache, "remove-ref", 0) == NULL,
+    ok (cache_lookup (cache, "remove-ref") == NULL,
         "cache_lookup verify entry gone");
 
     count = 0;
@@ -503,7 +513,7 @@ void cache_remove_entry_tests (void)
         "cache_entry_create created empty object");
     ok (cache_insert (cache, e) == 0,
         "cache_insert works");
-    ok (cache_lookup (cache, "remove-ref", 0) != NULL,
+    ok (cache_lookup (cache, "remove-ref") != NULL,
         "cache_lookup verify entry exists");
     ok (cache_entry_get_valid (e) == false,
         "cache entry invalid, adding waiter");
@@ -521,7 +531,7 @@ void cache_remove_entry_tests (void)
         "waiter callback ran");
     ok (cache_remove_entry (cache, "remove-ref") == 1,
         "cache_remove_entry removed cache entry after valid waiter gone");
-    ok (cache_lookup (cache, "remove-ref", 0) == NULL,
+    ok (cache_lookup (cache, "remove-ref") == NULL,
         "cache_lookup verify entry gone");
 
     count = 0;
@@ -535,7 +545,7 @@ void cache_remove_entry_tests (void)
     json_decref (o);
     ok (cache_insert (cache, e) == 0,
         "cache_insert works");
-    ok (cache_lookup (cache, "remove-ref", 0) != NULL,
+    ok (cache_lookup (cache, "remove-ref") != NULL,
         "cache_lookup verify entry exists");
     ok (cache_entry_set_dirty (e, true) == 0,
         "cache_entry_set_dirty success");
@@ -551,7 +561,7 @@ void cache_remove_entry_tests (void)
         "waiter callback ran");
     ok (cache_remove_entry (cache, "remove-ref") == 1,
         "cache_remove_entry removed cache entry after notdirty waiter gone");
-    ok (cache_lookup (cache, "remove-ref", 0) == NULL,
+    ok (cache_lookup (cache, "remove-ref") == NULL,
         "cache_lookup verify entry gone");
 
     cache_destroy (cache);
@@ -571,7 +581,7 @@ void cache_expiration_tests (void)
 
     /* Put entry in cache and test lookup, expire
      */
-    ok ((cache = cache_create ()) != NULL,
+    ok ((cache = cache_create (NULL)) != NULL,
         "cache_create works");
     ok (cache_count_entries (cache) == 0,
         "cache contains 0 entries");
@@ -583,10 +593,11 @@ void cache_expiration_tests (void)
         "cache_insert works");
     ok (cache_count_entries (cache) == 1,
         "cache contains 1 entry after insert");
-    ok (cache_lookup (cache, "yyy1", 0) == NULL,
+    ok (cache_lookup (cache, "yyy1") == NULL,
         "cache_lookup of wrong hash fails");
-    ok ((e2 = cache_lookup (cache, "xxx1", 42)) != NULL,
+    ok ((e2 = cache_lookup (cache, "xxx1")) != NULL,
         "cache_lookup of correct hash works (last use=42)");
+    cache_entry_set_fake_time (e2, 42);
     ok (cache_entry_get_treeobj (e2) == NULL,
         "no treeobj object found");
     ok (cache_count_entries (cache) == 1,
@@ -598,11 +609,13 @@ void cache_expiration_tests (void)
     ok (size == 0, "cache w/ entry w/o data, size == 0");
     ok (incomplete == 1, "cache w/ entry w/o data, incomplete == 1");
     ok (dirty == 0, "cache w/ entry w/o data, dirty == 0");
-    ok (cache_expire_entries (cache, 43, 1) == 0,
+    cache_set_fake_time (cache, 43);
+    ok (cache_expire_entries (cache, 1) == 0,
         "cache_expire_entries now=43 thresh=1 expired 0 b/c entry invalid");
     ok (cache_count_entries (cache) == 1,
         "cache contains 1 entry");
-    ok (cache_expire_entries (cache, 44, 1) == 0,
+    cache_set_fake_time (cache, 44);
+    ok (cache_expire_entries (cache, 1) == 0,
         "cache_expire_entries now=44 thresh=1 expired 0");
     ok (cache_count_entries (cache) == 1,
         "cache contains 1 entry");
@@ -618,10 +631,11 @@ void cache_expiration_tests (void)
         "cache_insert works");
     ok (cache_count_entries (cache) == 2,
         "cache contains 2 entries after insert");
-    ok (cache_lookup (cache, "yyy2", 0) == NULL,
+    ok (cache_lookup (cache, "yyy2") == NULL,
         "cache_lookup of wrong hash fails");
-    ok ((e4 = cache_lookup (cache, "xxx2", 42)) != NULL,
+    ok ((e4 = cache_lookup (cache, "xxx2")) != NULL,
         "cache_lookup of correct hash works (last use=42)");
+    cache_entry_set_fake_time (e4, 42);
     ok ((otmp = cache_entry_get_treeobj (e4)) != NULL,
         "cache_entry_get_treeobj found entry");
     otest = treeobj_create_val ("foo", 3);
@@ -654,11 +668,13 @@ void cache_expiration_tests (void)
     ok (cache_entry_set_dirty (e4, false) == 0,
         "cache_entry_set_dirty success");
 
-    ok (cache_expire_entries (cache, 43, 1) == 0,
+    cache_set_fake_time (cache, 43);
+    ok (cache_expire_entries (cache, 1) == 0,
         "cache_expire_entries now=43 thresh=1 expired 0");
     ok (cache_count_entries (cache) == 2,
         "cache contains 2 entries");
-    ok (cache_expire_entries (cache, 44, 1) == 1,
+    cache_set_fake_time (cache, 44);
+    ok (cache_expire_entries (cache, 1) == 1,
         "cache_expire_entries now=44 thresh=1 expired 1");
     ok (cache_count_entries (cache) == 1,
         "cache contains 1 entry");
@@ -673,10 +689,11 @@ void cache_expiration_tests (void)
         "cache_insert works");
     ok (cache_count_entries (cache) == 2,
         "cache contains 2 entries after insert");
-    ok (cache_lookup (cache, "yyy3", 0) == NULL,
+    ok (cache_lookup (cache, "yyy3") == NULL,
         "cache_lookup of wrong hash fails");
-    ok ((e6 = cache_lookup (cache, "xxx3", 42)) != NULL,
+    ok ((e6 = cache_lookup (cache, "xxx3")) != NULL,
         "cache_lookup of correct hash works (last use=42)");
+    cache_entry_set_fake_time (e6, 42);
     ok (cache_entry_get_raw (e6, &data, &len) == 0,
         "cache_entry_get_raw found entry");
     ok (len == 6
@@ -685,14 +702,16 @@ void cache_expiration_tests (void)
 
     cache_entry_incref (e6);
 
-    ok (cache_expire_entries (cache, 44, 1) == 0,
+    cache_set_fake_time (cache, 44);
+    ok (cache_expire_entries (cache, 1) == 0,
         "cache_expire_entries now=44 thresh=1 expired 0");
     ok (cache_count_entries (cache) == 2,
         "cache contains 2 entries");
 
     cache_entry_decref (e6);
 
-    ok (cache_expire_entries (cache, 44, 1) == 1,
+    cache_set_fake_time (cache, 44);
+    ok (cache_expire_entries (cache, 1) == 1,
         "cache_expire_entries now=44 thresh=1 expired 1");
     ok (cache_count_entries (cache) == 1,
         "cache contains 1 entry");
diff --git a/src/modules/kvs/test/kvs_wait_version.c b/src/modules/kvs/test/kvs_wait_version.c
new file mode 100644
index 000000000000..6865e4008c71
--- /dev/null
+++ b/src/modules/kvs/test/kvs_wait_version.c
@@ -0,0 +1,268 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdbool.h>
+#include <unistd.h>
+#include <errno.h>
+#include <jansson.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libkvs/kvs.h"
+#include "src/modules/kvs/kvsroot.h"
+#include "src/modules/kvs/kvs_wait_version.h"
+#include "src/modules/kvs/cache.h"
+#include "ccan/str/str.h"
+
+const char *root_ref = "1234";  /* random string, doesn't matter for tests */
+int count = 0;
+
+void basic_corner_case_tests (void)
+{
+    ok (kvs_wait_version_add (NULL, NULL, NULL, NULL, NULL, NULL, 0) < 0
+        && errno == EINVAL,
+        "kvs_wait_version_add fails with EINVAL on bad input");
+
+    ok (kvs_wait_version_remove_msg (NULL, NULL, NULL) < 0
+        && errno == EINVAL,
+        "kvs_wait_version_remove_msg fails with EINVAL on bad input");
+
+    /* doesn't segfault on NULL */
+    kvs_wait_version_process (NULL, false);
+}
+
+void cb (flux_t *h, flux_msg_handler_t *mh, const flux_msg_t *msg, void *arg)
+{
+    count++;
+}
+
+void basic_api_tests (void)
+{
+    kvsroot_mgr_t *krm;
+    struct cache *cache;
+    struct kvsroot *root;
+    flux_msg_t *msg;
+
+    cache = cache_create (NULL);
+
+    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
+        "kvsroot_mgr_create works");
+
+    ok (kvsroot_mgr_root_count (krm) == 0,
+        "kvsroot_mgr_root_count returns correct count of roots");
+
+    ok ((root = kvsroot_mgr_create_root (krm,
+                                         cache,
+                                         "sha1",
+                                         KVS_PRIMARY_NAMESPACE,
+                                         1234,
+                                         0)) != NULL,
+         "kvsroot_mgr_create_root works");
+
+    msg = flux_msg_create (FLUX_MSGTYPE_REQUEST);
+
+    ok (!kvs_wait_version_add (root, cb, NULL, NULL, msg, NULL, 2),
+        "kvs_wait_version_add w/ seq = 2 works");
+    ok (!kvs_wait_version_add (root, cb, NULL, NULL, msg, NULL, 3),
+        "kvs_wait_version_add w/ seq = 3 works");
+    ok (!kvs_wait_version_add (root, cb, NULL, NULL, msg, NULL, 4),
+        "kvs_wait_version_add w/ seq = 4 works");
+
+    ok (zlist_size (root->wait_version_list) == 3,
+        "wait_version_list is length 3");
+
+    kvsroot_setroot (krm, root, root_ref, 1);
+
+    count = 0;
+
+    kvs_wait_version_process (root, false);
+
+    ok (count == 0,
+        "kvs_wait_version_process did not call cb on seq = 1");
+
+    ok (zlist_size (root->wait_version_list) == 3,
+        "wait_version_list is length 3");
+
+    kvsroot_setroot (krm, root, root_ref, 2);
+
+    count = 0;
+
+    kvs_wait_version_process (root, false);
+
+    ok (count == 1,
+        "kvs_wait_version_process called callback once on seq = 2");
+
+    ok (zlist_size (root->wait_version_list) == 2,
+        "wait_version_list is length 2");
+
+    kvsroot_setroot (krm, root, root_ref, 4);
+
+    count = 0;
+
+    kvs_wait_version_process (root, false);
+
+    ok (count == 2,
+        "kvs_wait_version_process called callback twice on seq = 4");
+
+    ok (zlist_size (root->wait_version_list) == 0,
+        "wait_version_list is length 0");
+
+    ok (!kvs_wait_version_add (root, cb, NULL, NULL, msg, NULL, 5),
+        "kvs_wait_version_add w/ seq = 5 works");
+    ok (!kvs_wait_version_add (root, cb, NULL, NULL, msg, NULL, 6),
+        "kvs_wait_version_add w/ seq = 6 works");
+    ok (!kvs_wait_version_add (root, cb, NULL, NULL, msg, NULL, 7),
+        "kvs_wait_version_add w/ seq = 7 works");
+
+    ok (zlist_size (root->wait_version_list) == 3,
+        "wait_version_list is length 3");
+
+    count = 0;
+
+    kvs_wait_version_process (root, true);
+
+    ok (count == 3,
+        "kvs_wait_version_process called callback thrice on all flag = true");
+
+    ok (zlist_size (root->wait_version_list) == 0,
+        "wait_version_list is length 0");
+
+    /* cover some alternate insertion pattern, descending and
+     * duplicate numbers */
+
+    ok (!kvs_wait_version_add (root, cb, NULL, NULL, msg, NULL, 9),
+        "kvs_wait_version_add w/ seq = 9 works");
+    ok (!kvs_wait_version_add (root, cb, NULL, NULL, msg, NULL, 8),
+        "kvs_wait_version_add w/ seq = 8 works");
+    ok (!kvs_wait_version_add (root, cb, NULL, NULL, msg, NULL, 8),
+        "kvs_wait_version_add w/ seq = 8 works");
+
+    ok (zlist_size (root->wait_version_list) == 3,
+        "wait_version_list is length 3");
+
+    count = 0;
+
+    kvs_wait_version_process (root, true);
+
+    ok (count == 3,
+        "kvs_wait_version_process called callback thrice on all flag = true");
+
+    flux_msg_destroy (msg);
+
+    kvsroot_mgr_destroy (krm);
+
+    cache_destroy (cache);
+}
+
+bool msgcmp (const flux_msg_t *msg, void *arg)
+{
+    const char *id;
+    bool match = false;
+    if ((id = flux_msg_route_first (msg))
+        && (streq (id, "1")
+            || streq (id, "2")
+            || streq (id, "3")
+            || streq (id, "4")
+            || streq (id, "5")))
+        match = true;
+    return match;
+}
+
+bool msgcmp_true (const flux_msg_t *msg, void *arg)
+{
+    return true;
+}
+
+void basic_remove_tests (void)
+{
+    kvsroot_mgr_t *krm;
+    struct cache *cache;
+    struct kvsroot *root;
+    int i;
+
+    cache = cache_create (NULL);
+
+    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
+        "kvsroot_mgr_create works");
+
+    ok (kvsroot_mgr_root_count (krm) == 0,
+        "kvsroot_mgr_root_count returns correct count of roots");
+
+    ok ((root = kvsroot_mgr_create_root (krm,
+                                         cache,
+                                         "sha1",
+                                         KVS_PRIMARY_NAMESPACE,
+                                         1234,
+                                         0)) != NULL,
+         "kvsroot_mgr_create_root works");
+
+
+    /* Add 10 syncs to queue, selectively destroy */
+    for (i = 1; i <= 10; i++) {
+        flux_msg_t *msg;
+        char s[16];
+        snprintf (s, sizeof (s), "%d", i);
+        if (!(msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
+            break;
+        flux_msg_route_enable (msg);
+        if (flux_msg_route_push (msg, s) < 0)
+            break;
+        ok (!kvs_wait_version_add (root, cb, NULL, NULL, msg, NULL, i),
+            "kvs_wait_version_add w/ seq = %d works", i);
+        flux_msg_destroy (msg);
+    }
+
+    ok (zlist_size (root->wait_version_list) == 10,
+        "wait_version_list is length 10");
+
+    count = 0;
+
+    ok (!kvs_wait_version_remove_msg (root, msgcmp, NULL),
+        "kvs_wait_version_remove_msg works");
+
+    ok (zlist_size (root->wait_version_list) == 5,
+        "wait_version_list is length 5");
+
+    ok (!kvs_wait_version_remove_msg (root, msgcmp, NULL),
+        "kvs_wait_version_remove_msg works");
+
+    ok (zlist_size (root->wait_version_list) == 5,
+        "wait_version_list is still length 5");
+
+    ok (!kvs_wait_version_remove_msg (root, msgcmp_true, NULL),
+        "kvs_wait_version_remove_msg works");
+
+    ok (zlist_size (root->wait_version_list) == 0,
+        "wait_version_list is length 0");
+
+    kvsroot_mgr_destroy (krm);
+
+    cache_destroy (cache);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    basic_corner_case_tests ();
+    basic_api_tests ();
+    basic_remove_tests ();
+
+    done_testing ();
+    return (0);
+}
+
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/kvs/test/kvsroot.c b/src/modules/kvs/test/kvsroot.c
index 876e0ea2094b..791dce7af6c0 100644
--- a/src/modules/kvs/test/kvsroot.c
+++ b/src/modules/kvs/test/kvsroot.c
@@ -16,9 +16,11 @@
 #include <jansson.h>
 
 #include "src/common/libtap/tap.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libkvs/kvs.h"
 #include "src/modules/kvs/kvsroot.h"
 #include "src/modules/kvs/treq.h"
+#include "ccan/str/str.h"
 
 int global = 0;
 
@@ -30,7 +32,7 @@ void basic_api_tests (void)
     struct kvsroot *tmproot;
     struct flux_msg_cred cred;
 
-    cache = cache_create ();
+    cache = cache_create (NULL);
 
     ok ((krm = kvsroot_mgr_create (NULL, &global)) != NULL,
         "kvsroot_mgr_create works");
@@ -46,6 +48,9 @@ void basic_api_tests (void)
                                          0)) != NULL,
          "kvsroot_mgr_create_root works");
 
+    ok (root->is_primary == true,
+        "root is primary namespace");
+
     ok (kvsroot_mgr_root_count (krm) == 1,
         "kvsroot_mgr_root_count returns correct count of roots");
 
@@ -76,7 +81,7 @@ void basic_api_tests (void)
 
     kvsroot_setroot (krm, root, "foobar", 18);
 
-    ok (!strcmp (root->ref, "foobar"),
+    ok (streq (root->ref, "foobar"),
         "kvsroot_setroot set ref correctly");
 
     ok (root->seq == 18,
@@ -129,6 +134,33 @@ void basic_api_tests (void)
     cache_destroy (cache);
 }
 
+void basic_api_tests_non_primary (void)
+{
+    kvsroot_mgr_t *krm;
+    struct cache *cache;
+    struct kvsroot *root;
+
+    cache = cache_create (NULL);
+
+    ok ((krm = kvsroot_mgr_create (NULL, &global)) != NULL,
+        "kvsroot_mgr_create works");
+
+    ok ((root = kvsroot_mgr_create_root (krm,
+                                         cache,
+                                         "sha1",
+                                         "foobar",
+                                         1234,
+                                         0)) != NULL,
+         "kvsroot_mgr_create_root works");
+
+    ok (root->is_primary == false,
+        "root is not primary namespace");
+
+    kvsroot_mgr_destroy (krm);
+
+    cache_destroy (cache);
+}
+
 int count_roots_cb (struct kvsroot *root, void *arg)
 {
     int *count = arg;
@@ -163,7 +195,7 @@ void basic_iter_tests (void)
     struct kvsroot *root;
     int count;
 
-    cache = cache_create ();
+    cache = cache_create (NULL);
 
     ok ((krm = kvsroot_mgr_create (NULL, &global)) != NULL,
         "kvsroot_mgr_create works");
@@ -172,7 +204,7 @@ void basic_iter_tests (void)
                                          cache,
                                          "sha1",
                                          "foo",
-                                         geteuid (),
+                                         getuid (),
                                          0)) != NULL,
          "kvsroot_mgr_create_root works");
 
@@ -180,7 +212,7 @@ void basic_iter_tests (void)
                                          cache,
                                          "sha1",
                                          "bar",
-                                         geteuid (),
+                                         getuid (),
                                          0)) != NULL,
          "kvsroot_mgr_create_root works");
 
@@ -196,7 +228,7 @@ void basic_iter_tests (void)
 
     count = 0;
     ok (kvsroot_mgr_iter_roots (krm, count_roots_early_exit_cb, &count) == 0,
-        "kvsroot_mgr_iter_roots works if exitting midway");
+        "kvsroot_mgr_iter_roots works if exiting midway");
 
     ok (count == 1,
         "kvsroot_mgr_iter_roots called callback correct number of times");
@@ -223,7 +255,7 @@ void basic_kvstxn_mgr_tests (void)
     json_t *ops = NULL;
     void *tmpaux;
 
-    cache = cache_create ();
+    cache = cache_create (NULL);
 
     ok ((krm = kvsroot_mgr_create (NULL, &global)) != NULL,
         "kvsroot_mgr_create works");
@@ -232,7 +264,7 @@ void basic_kvstxn_mgr_tests (void)
                                          cache,
                                          "sha1",
                                          KVS_PRIMARY_NAMESPACE,
-                                         geteuid (),
+                                         getuid (),
                                          0)) != NULL,
          "kvsroot_mgr_create_root works");
 
@@ -244,6 +276,7 @@ void basic_kvstxn_mgr_tests (void)
     ok (kvstxn_mgr_add_transaction (root->ktm,
                                     "foo",
                                     ops,
+                                    0,
                                     0) == 0,
         "kvstxn_mgr_add_transaction works");
 
@@ -267,6 +300,7 @@ int main (int argc, char *argv[])
     plan (NO_PLAN);
 
     basic_api_tests ();
+    basic_api_tests_non_primary ();
     basic_iter_tests ();
     basic_kvstxn_mgr_tests ();
 
diff --git a/src/modules/kvs/test/kvssync.c b/src/modules/kvs/test/kvssync.c
deleted file mode 100644
index ee629cd049ff..000000000000
--- a/src/modules/kvs/test/kvssync.c
+++ /dev/null
@@ -1,267 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdbool.h>
-#include <unistd.h>
-#include <errno.h>
-#include <jansson.h>
-
-#include "src/common/libtap/tap.h"
-#include "src/common/libkvs/kvs.h"
-#include "src/modules/kvs/kvsroot.h"
-#include "src/modules/kvs/kvssync.h"
-#include "src/modules/kvs/cache.h"
-
-const char *root_ref = "1234";  /* random string, doesn't matter for tests */
-int count = 0;
-
-void basic_corner_case_tests (void)
-{
-    ok (kvssync_add (NULL, NULL, NULL, NULL, NULL, NULL, 0) < 0
-        && errno == EINVAL,
-        "kvssync_add fails with EINVAL on bad input");
-
-    ok (kvssync_remove_msg (NULL, NULL, NULL) < 0
-        && errno == EINVAL,
-        "kvssync_remove_msg fails with EINVAL on bad input");
-
-    /* doesn't segfault on NULL */
-    kvssync_process (NULL, false);
-}
-
-void cb (flux_t *h, flux_msg_handler_t *mh, const flux_msg_t *msg, void *arg)
-{
-    count++;
-}
-
-void basic_api_tests (void)
-{
-    kvsroot_mgr_t *krm;
-    struct cache *cache;
-    struct kvsroot *root;
-    flux_msg_t *msg;
-
-    cache = cache_create ();
-
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
-
-    ok (kvsroot_mgr_root_count (krm) == 0,
-        "kvsroot_mgr_root_count returns correct count of roots");
-
-    ok ((root = kvsroot_mgr_create_root (krm,
-                                         cache,
-                                         "sha1",
-                                         KVS_PRIMARY_NAMESPACE,
-                                         1234,
-                                         0)) != NULL,
-         "kvsroot_mgr_create_root works");
-
-    msg = flux_msg_create (FLUX_MSGTYPE_REQUEST);
-
-    ok (!kvssync_add (root, cb, NULL, NULL, msg, NULL, 2),
-        "kvssync_add w/ seq = 2 works");
-    ok (!kvssync_add (root, cb, NULL, NULL, msg, NULL, 3),
-        "kvssync_add w/ seq = 3 works");
-    ok (!kvssync_add (root, cb, NULL, NULL, msg, NULL, 4),
-        "kvssync_add w/ seq = 4 works");
-
-    ok (zlist_size (root->synclist) == 3,
-        "synclist is length 3");
-
-    kvsroot_setroot (krm, root, root_ref, 1);
-
-    count = 0;
-
-    kvssync_process (root, false);
-
-    ok (count == 0,
-        "kvssync_process did not call cb on seq = 1");
-
-    ok (zlist_size (root->synclist) == 3,
-        "synclist is length 3");
-
-    kvsroot_setroot (krm, root, root_ref, 2);
-
-    count = 0;
-
-    kvssync_process (root, false);
-
-    ok (count == 1,
-        "kvssync_process called callback once on seq = 2");
-
-    ok (zlist_size (root->synclist) == 2,
-        "synclist is length 2");
-
-    kvsroot_setroot (krm, root, root_ref, 4);
-
-    count = 0;
-
-    kvssync_process (root, false);
-
-    ok (count == 2,
-        "kvssync_process called callback twice on seq = 4");
-
-    ok (zlist_size (root->synclist) == 0,
-        "synclist is length 0");
-
-    ok (!kvssync_add (root, cb, NULL, NULL, msg, NULL, 5),
-        "kvssync_add w/ seq = 5 works");
-    ok (!kvssync_add (root, cb, NULL, NULL, msg, NULL, 6),
-        "kvssync_add w/ seq = 6 works");
-    ok (!kvssync_add (root, cb, NULL, NULL, msg, NULL, 7),
-        "kvssync_add w/ seq = 7 works");
-
-    ok (zlist_size (root->synclist) == 3,
-        "synclist is length 3");
-
-    count = 0;
-
-    kvssync_process (root, true);
-
-    ok (count == 3,
-        "kvssync_process called callback thrice on all flag = true");
-
-    ok (zlist_size (root->synclist) == 0,
-        "synclist is length 0");
-
-    /* cover some alternate insertion pattern, descending and
-     * duplicate numbers */
-
-    ok (!kvssync_add (root, cb, NULL, NULL, msg, NULL, 9),
-        "kvssync_add w/ seq = 9 works");
-    ok (!kvssync_add (root, cb, NULL, NULL, msg, NULL, 8),
-        "kvssync_add w/ seq = 8 works");
-    ok (!kvssync_add (root, cb, NULL, NULL, msg, NULL, 8),
-        "kvssync_add w/ seq = 8 works");
-
-    ok (zlist_size (root->synclist) == 3,
-        "synclist is length 3");
-
-    count = 0;
-
-    kvssync_process (root, true);
-
-    ok (count == 3,
-        "kvssync_process called callback thrice on all flag = true");
-
-    flux_msg_destroy (msg);
-
-    kvsroot_mgr_destroy (krm);
-
-    cache_destroy (cache);
-}
-
-bool msgcmp (const flux_msg_t *msg, void *arg)
-{
-    char *id = NULL;
-    bool match = false;
-    if (flux_msg_get_route_first (msg, &id) == 0
-        && (!strcmp (id, "1")
-            || !strcmp (id, "2")
-            || !strcmp (id, "3")
-            || !strcmp (id, "4")
-            || !strcmp (id, "5")))
-        match = true;
-    if (id)
-        free (id);
-    return match;
-}
-
-bool msgcmp_true (const flux_msg_t *msg, void *arg)
-{
-    return true;
-}
-
-void basic_remove_tests (void)
-{
-    kvsroot_mgr_t *krm;
-    struct cache *cache;
-    struct kvsroot *root;
-    int i;
-
-    cache = cache_create ();
-
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
-
-    ok (kvsroot_mgr_root_count (krm) == 0,
-        "kvsroot_mgr_root_count returns correct count of roots");
-
-    ok ((root = kvsroot_mgr_create_root (krm,
-                                         cache,
-                                         "sha1",
-                                         KVS_PRIMARY_NAMESPACE,
-                                         1234,
-                                         0)) != NULL,
-         "kvsroot_mgr_create_root works");
-
-
-    /* Add 10 syncs to queue, selectively destroy */
-    for (i = 1; i <= 10; i++) {
-        flux_msg_t *msg;
-        char s[16];
-        snprintf (s, sizeof (s), "%d", i);
-        if (!(msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
-            break;
-        if (flux_msg_enable_route (msg) < 0 || flux_msg_push_route (msg, s) < 0)
-            break;
-        ok (!kvssync_add (root, cb, NULL, NULL, msg, NULL, i),
-            "kvssync_add w/ seq = %d works", i);
-        flux_msg_destroy (msg);
-    }
-
-    ok (zlist_size (root->synclist) == 10,
-        "synclist is length 10");
-
-    count = 0;
-
-    ok (!kvssync_remove_msg (root, msgcmp, NULL),
-        "kvssync_remove_msg works");
-
-    ok (zlist_size (root->synclist) == 5,
-        "synclist is length 5");
-
-    ok (!kvssync_remove_msg (root, msgcmp, NULL),
-        "kvssync_remove_msg works");
-
-    ok (zlist_size (root->synclist) == 5,
-        "synclist is still length 5");
-
-    ok (!kvssync_remove_msg (root, msgcmp_true, NULL),
-        "kvssync_remove_msg works");
-
-    ok (zlist_size (root->synclist) == 0,
-        "synclist is length 0");
-
-    kvsroot_mgr_destroy (krm);
-
-    cache_destroy (cache);
-}
-
-int main (int argc, char *argv[])
-{
-    plan (NO_PLAN);
-
-    basic_corner_case_tests ();
-    basic_api_tests ();
-    basic_remove_tests ();
-
-    done_testing ();
-    return (0);
-}
-
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/kvs/test/kvstxn.c b/src/modules/kvs/test/kvstxn.c
index a4f717bb3fbd..301925aaa337 100644
--- a/src/modules/kvs/test/kvstxn.c
+++ b/src/modules/kvs/test/kvstxn.c
@@ -16,6 +16,7 @@
 #include <assert.h>
 
 #include "src/common/libtap/tap.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/blobref.h"
 #include "src/common/libkvs/kvs.h"
 #include "src/common/libkvs/treeobj.h"
@@ -25,12 +26,27 @@
 #include "src/modules/kvs/kvstxn.h"
 #include "src/modules/kvs/kvsroot.h"
 #include "src/modules/kvs/lookup.h"
+#include "ccan/str/str.h"
 
 static int test_global = 5;
 
 /* Use when we do not yet have a root_ref. */
 static const char *ref_dummy = "sha1-508259c0f7fd50e47716b50ad1f0fc6ed46017f9";
 
+static void ktest_finalize (struct cache *cache, kvsroot_mgr_t *krm)
+{
+    cache_destroy (cache);
+    kvsroot_mgr_destroy (krm);
+}
+
+static void ktest_init (struct cache **cache, kvsroot_mgr_t **krm)
+{
+    if (!(*cache = cache_create (NULL)))
+        BAIL_OUT ("cache_create failed");
+    if (!(*krm = kvsroot_mgr_create (NULL, NULL)))
+        BAIL_OUT ("kvsroot_mgr_create failed");
+};
+
 static int treeobj_hash (const char *hash_name, json_t *obj,
                          char *blobref, int blobref_len)
 {
@@ -88,7 +104,7 @@ static struct cache_entry *create_cache_entry_raw (const char *ref,
                                                    int len)
 {
     struct cache_entry *entry;
-    int ret;
+    __attribute__((unused)) int ret;
 
     assert (data);
     assert (len);
@@ -105,7 +121,7 @@ static struct cache_entry *create_cache_entry_treeobj (const char *ref,
                                                        json_t *o)
 {
     struct cache_entry *entry;
-    int ret;
+    __attribute__((unused)) int ret;
 
     assert (o);
 
@@ -145,7 +161,7 @@ struct cache *create_cache_with_empty_rootdir (char *ref, int ref_len)
 
     rootdir = treeobj_create_dir ();
 
-    ok ((cache = cache_create ()) != NULL,
+    ok ((cache = cache_create (NULL)) != NULL,
         "cache_create works");
     ok (treeobj_hash ("sha1", rootdir, ref, ref_len) == 0,
         "treeobj_hash worked");
@@ -225,7 +241,7 @@ void kvstxn_mgr_basic_tests (void)
     ok (kvstxn_mgr_get_ready_transaction (ktm) == NULL,
         "kvstxn_mgr_get_ready_transaction initially returns NULL for no ready transactions");
 
-    ok (kvstxn_mgr_add_transaction (ktm, NULL, NULL, 0) < 0
+    ok (kvstxn_mgr_add_transaction (ktm, NULL, NULL, 0, 0) < 0
         && errno == EINVAL,
         "kvstxn_mgr_add_transaction fails with EINVAL on bad input");
 
@@ -235,6 +251,7 @@ void kvstxn_mgr_basic_tests (void)
     ok (kvstxn_mgr_add_transaction (ktm,
                                     "transaction1",
                                     ops,
+                                    0,
                                     0) == 0,
         "kvstxn_mgr_add_transaction works");
 
@@ -261,22 +278,26 @@ void kvstxn_mgr_basic_tests (void)
     cache_destroy (cache);
 }
 
-void create_ready_kvstxn (kvstxn_mgr_t *ktm,
-                          const char *name,
-                          const char *key,
-                          const char *val,
-                          int op_flags,
-                          int transaction_flags)
+static void create_ready_kvstxn_wrapper (kvstxn_mgr_t *ktm,
+                                         const char *name,
+                                         const char *key,
+                                         const char *val,
+                                         int op_flags,
+                                         int transaction_flags,
+                                         int internal_flags)
 {
     json_t *ops = NULL;
 
     ops = json_array ();
-    ops_append (ops, key, val, op_flags);
+    /* val can be NULL for a deletion */
+    if (key)
+        ops_append (ops, key, val, op_flags);
 
     ok (kvstxn_mgr_add_transaction (ktm,
                                     name,
                                     ops,
-                                    transaction_flags) == 0,
+                                    transaction_flags,
+                                    internal_flags) == 0,
         "kvstxn_mgr_add_transaction works");
 
     json_decref (ops);
@@ -285,6 +306,39 @@ void create_ready_kvstxn (kvstxn_mgr_t *ktm,
         "kvstxn_mgr_transaction_ready says a kvstxn is ready");
 }
 
+void create_ready_kvstxn (kvstxn_mgr_t *ktm,
+                          const char *name,
+                          const char *key,
+                          const char *val,
+                          int op_flags,
+                          int transaction_flags)
+{
+    create_ready_kvstxn_wrapper (ktm,
+                                 name,
+                                 key,
+                                 val,
+                                 op_flags,
+                                 transaction_flags,
+                                 0);
+}
+
+void create_ready_kvstxn_internal_flags (kvstxn_mgr_t *ktm,
+                                         const char *name,
+                                         const char *key,
+                                         const char *val,
+                                         int op_flags,
+                                         int transaction_flags,
+                                         int internal_flags)
+{
+    create_ready_kvstxn_wrapper (ktm,
+                                 name,
+                                 key,
+                                 val,
+                                 op_flags,
+                                 transaction_flags,
+                                 internal_flags);
+}
+
 /* Return true if 'key' is referenced an 'ops' array entry.
  */
 bool is_op_key (json_t *ops, const char *key)
@@ -297,7 +351,7 @@ bool is_op_key (json_t *ops, const char *key)
     json_array_foreach (ops, index, entry) {
         if ((o = json_object_get (entry, "key"))
                     && (k = json_string_value (o))
-                    && !strcmp (key, k))
+                    && streq (key, k))
             return true;
     }
     return false;
@@ -307,36 +361,26 @@ bool is_op_key (json_t *ops, const char *key)
  */
 bool keys_match_ops (json_t *ops, json_t *keys)
 {
-    size_t index;
-    json_t *key;
-    const char *k;
-
-    json_array_foreach (keys, index, key) {
-        k = json_string_value (key);
-        if (k == NULL || !is_op_key (ops, k))
+    const char *key;
+    json_t *value;
+    json_object_foreach (keys, key, value) {
+        if (!is_op_key (ops, key))
             return false;
     }
     return true;
 }
 
 
-/* Return true if 'key' is a member of 'keys' array
+/* Return true if 'key' is a member of 'keys' dict
  */
 bool is_key (json_t *keys, const char *key)
 {
-    size_t index;
-    json_t *o;
-    const char *k;
-
-    json_array_foreach (keys, index, o) {
-        if ((k = json_string_value (o))
-                    && !strcmp (key, k))
-            return true;
-    }
+    if (json_object_get (keys, key))
+        return true;
     return false;
 }
 
-/* Return true if all ops have a key array entry
+/* Return true if all ops have a key dict entry
  */
 bool ops_match_keys (json_t *keys, json_t *ops)
 {
@@ -372,6 +416,7 @@ void verify_ready_kvstxn (kvstxn_mgr_t *ktm,
                           json_t *names,
                           json_t *ops,
                           int flags,
+                          int internal_flags,
                           const char *extramsg)
 {
     json_t *o;
@@ -395,6 +440,9 @@ void verify_ready_kvstxn (kvstxn_mgr_t *ktm,
     ok (kvstxn_get_flags (kt) == flags,
         "flags do not match");
 
+    ok (kvstxn_get_internal_flags (kt) == internal_flags,
+        "internal_flags do not match");
+
     ok (kvstxn_get_newroot_ref (kt) == NULL,
         "kvstxn_get_newroot returns NULL on non-processed transaction");
 
@@ -442,7 +490,7 @@ void kvstxn_mgr_merge_tests (void)
     ops_append (ops, "key1", "1", 0);
     ops_append (ops, "key2", "2", 0);
 
-    verify_ready_kvstxn (ktm, names, ops, 0, "merged transaction");
+    verify_ready_kvstxn (ktm, names, ops, 0, 0, "merged transaction");
 
     json_decref (names);
     json_decref (ops);
@@ -450,7 +498,7 @@ void kvstxn_mgr_merge_tests (void)
 
     clear_ready_kvstxns (ktm);
 
-    /* test unsuccessful merge */
+    /* test unsuccessful merge (FLUX_KVS_NO_MERGE) */
 
     create_ready_kvstxn (ktm, "transaction1", "key1", "1", 0, FLUX_KVS_NO_MERGE);
     create_ready_kvstxn (ktm, "transaction2", "key2", "2", 0, 0);
@@ -464,7 +512,12 @@ void kvstxn_mgr_merge_tests (void)
     ops = json_array ();
     ops_append (ops, "key1", "1", 0);
 
-    verify_ready_kvstxn (ktm, names, ops, FLUX_KVS_NO_MERGE, "unmerged transaction");
+    verify_ready_kvstxn (ktm,
+                         names,
+                         ops,
+                         FLUX_KVS_NO_MERGE,
+                         0,
+                         "unmerged transaction");
 
     json_decref (names);
     json_decref (ops);
@@ -472,7 +525,7 @@ void kvstxn_mgr_merge_tests (void)
 
     clear_ready_kvstxns (ktm);
 
-    /* test unsuccessful merge */
+    /* test unsuccessful merge (FLUX_KVS_NO_MERGE) */
 
     create_ready_kvstxn (ktm, "transaction1", "key1", "1", 0, 0);
     create_ready_kvstxn (ktm, "transaction2", "key2", "2", 0, FLUX_KVS_NO_MERGE);
@@ -486,7 +539,56 @@ void kvstxn_mgr_merge_tests (void)
     ops = json_array ();
     ops_append (ops, "key1", "1", 0);
 
-    verify_ready_kvstxn (ktm, names, ops, 0, "unmerged transaction");
+    verify_ready_kvstxn (ktm, names, ops, 0, 0, "unmerged transaction");
+
+    json_decref (names);
+    json_decref (ops);
+    ops = NULL;
+
+    clear_ready_kvstxns (ktm);
+
+    /* test unsuccessful merge (FLUX_KVS_SYNC) */
+
+    create_ready_kvstxn (ktm, "transaction1", "key1", "1", 0, FLUX_KVS_SYNC);
+    create_ready_kvstxn (ktm, "transaction2", "key2", "2", 0, 0);
+
+    ok (kvstxn_mgr_merge_ready_transactions (ktm) == 0,
+        "kvstxn_mgr_merge_ready_transactions success");
+
+    names = json_array ();
+    json_array_append_new (names, json_string ("transaction1"));
+
+    ops = json_array ();
+    ops_append (ops, "key1", "1", 0);
+
+    verify_ready_kvstxn (ktm,
+                         names,
+                         ops,
+                         FLUX_KVS_SYNC,
+                         0,
+                         "unmerged transaction");
+
+    json_decref (names);
+    json_decref (ops);
+    ops = NULL;
+
+    clear_ready_kvstxns (ktm);
+
+    /* test unsuccessful merge (FLUX_KVS_SYNC) */
+
+    create_ready_kvstxn (ktm, "transaction1", "key1", "1", 0, 0);
+    create_ready_kvstxn (ktm, "transaction2", "key2", "2", 0, FLUX_KVS_SYNC);
+
+    ok (kvstxn_mgr_merge_ready_transactions (ktm) == 0,
+        "kvstxn_mgr_merge_ready_transactions success");
+
+    names = json_array ();
+    json_array_append_new (names, json_string ("transaction1"));
+
+    ops = json_array ();
+    ops_append (ops, "key1", "1", 0);
+
+    verify_ready_kvstxn (ktm, names, ops, 0, 0, "unmerged transaction");
 
     json_decref (names);
     json_decref (ops);
@@ -508,7 +610,7 @@ void kvstxn_mgr_merge_tests (void)
     ops = json_array ();
     ops_append (ops, "key1", "1", 0);
 
-    verify_ready_kvstxn (ktm, names, ops, 0, "unmerged fence");
+    verify_ready_kvstxn (ktm, names, ops, 0, 0, "unmerged fence");
 
     json_decref (names);
     json_decref (ops);
@@ -556,7 +658,7 @@ void kvstxn_basic_tests (void)
     ops = json_array ();
     ops_append (ops, "key1", "1", 0);
 
-    verify_ready_kvstxn (ktm, names, ops, 0x44, "basic test");
+    verify_ready_kvstxn (ktm, names, ops, 0x44, 0, "basic test");
 
     json_decref (names);
     json_decref (ops);
@@ -583,7 +685,7 @@ void kvstxn_basic_tests (void)
     ok ((ns = kvstxn_get_namespace (kt)) != NULL,
         "kvstxn_get_namespace returns non-NULL");
 
-    ok (!strcmp (ns, KVS_PRIMARY_NAMESPACE),
+    ok (streq (ns, KVS_PRIMARY_NAMESPACE),
         "kvstxn_get_namespace returns correct string");
 
     ok (kvstxn_get_aux (kt) == &test_global,
@@ -598,6 +700,44 @@ void kvstxn_basic_tests (void)
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) < 0,
         "kvstxn_iter_dirty_cache_entries returns < 0 for call on invalid state");
 
+    ok (kvstxn_sync_content_flush (kt) == NULL,
+        "kvstxn_sync_content_flush returns NULL for call on invalid state");
+
+    ok (kvstxn_sync_checkpoint (kt) == NULL,
+        "kvstxn_sync_checkpoint returns NULL for call on invalid state");
+
+    kvstxn_mgr_destroy (ktm);
+    cache_destroy (cache);
+}
+
+void kvstxn_corner_case_tests (void)
+{
+    struct cache *cache;
+    kvstxn_mgr_t *ktm;
+    kvstxn_t *kt;
+    char rootref[BLOBREF_MAX_STRING_SIZE];
+
+    cache = create_cache_with_empty_rootdir (rootref, sizeof (rootref));
+
+    /* Test non-default namespace doesn't work with FLUX_KVS_SYNC */
+
+    ok ((ktm = kvstxn_mgr_create (cache,
+                                  "foobar",
+                                  "sha1",
+                                  NULL,
+                                  &test_global)) != NULL,
+        "kvstxn_mgr_create works");
+
+    create_ready_kvstxn (ktm, "transactionA", "keyA", "A", 0, FLUX_KVS_SYNC);
+
+    ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
+        "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
+
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_ERROR
+        && kvstxn_get_errnum (kt) == EINVAL,
+        "kvstxn_sync_checkpoint returns EINVAL on FLUX_KVS_SYNC "
+        "with non-default namespace");
+
     kvstxn_mgr_destroy (ktm);
     cache_destroy (cache);
 }
@@ -643,7 +783,6 @@ void verify_value (struct cache *cache,
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              ns,
                              root_ref,
                              0,
@@ -701,7 +840,7 @@ void kvstxn_basic_kvstxn_process_test (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_count_dirty_cb, &count) == 0,
@@ -710,7 +849,7 @@ void kvstxn_basic_kvstxn_process_test (void)
     ok (count == 1,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -730,6 +869,115 @@ void kvstxn_basic_kvstxn_process_test (void)
     cache_destroy (cache);
 }
 
+void kvstxn_basic_kvstxn_process_test_empty_ops (void)
+{
+    struct cache *cache;
+    kvsroot_mgr_t *krm;
+    kvstxn_mgr_t *ktm;
+    kvstxn_t *kt;
+    char rootref[BLOBREF_MAX_STRING_SIZE];
+    const char *newroot;
+
+    cache = create_cache_with_empty_rootdir (rootref, sizeof (rootref));
+
+    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
+        "kvsroot_mgr_create works");
+
+    setup_kvsroot (krm, KVS_PRIMARY_NAMESPACE, cache, ref_dummy);
+
+    ok ((ktm = kvstxn_mgr_create (cache,
+                                  KVS_PRIMARY_NAMESPACE,
+                                  "sha1",
+                                  NULL,
+                                  &test_global)) != NULL,
+        "kvstxn_mgr_create works");
+
+    create_ready_kvstxn (ktm, "transaction1", NULL, NULL, 0, 0);
+
+    ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
+        "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
+
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
+        "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
+
+    ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
+        "kvstxn_get_newroot_ref returns != NULL when processing complete");
+
+    ok (streq (newroot, rootref),
+        "root stays identical when no ops in transaction");
+
+    verify_keys_and_ops_standard (kt);
+
+    kvstxn_mgr_remove_transaction (ktm, kt, false);
+
+    ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) == NULL,
+        "kvstxn_mgr_get_ready_transaction returns NULL, no more kvstxns");
+
+    kvstxn_mgr_destroy (ktm);
+    kvsroot_mgr_destroy (krm);
+    cache_destroy (cache);
+}
+
+void kvstxn_basic_kvstxn_process_test_internal_flags (void)
+{
+    struct cache *cache;
+    kvsroot_mgr_t *krm;
+    kvstxn_mgr_t *ktm;
+    kvstxn_t *kt;
+    char rootref[BLOBREF_MAX_STRING_SIZE];
+    const char *newroot;
+    int flags;
+
+    cache = create_cache_with_empty_rootdir (rootref, sizeof (rootref));
+
+    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
+        "kvsroot_mgr_create works");
+
+    setup_kvsroot (krm, KVS_PRIMARY_NAMESPACE, cache, ref_dummy);
+
+    ok ((ktm = kvstxn_mgr_create (cache,
+                                  KVS_PRIMARY_NAMESPACE,
+                                  "sha1",
+                                  NULL,
+                                  &test_global)) != NULL,
+        "kvstxn_mgr_create works");
+
+    create_ready_kvstxn_internal_flags (ktm,
+                                        "transaction1",
+                                        NULL,
+                                        NULL,
+                                        0,
+                                        0,
+                                        KVSTXN_INTERNAL_FLAG_NO_PUBLISH);
+
+    ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
+        "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
+
+    flags = kvstxn_get_internal_flags (kt);
+    ok (flags == KVSTXN_INTERNAL_FLAG_NO_PUBLISH,
+        "kvstxn_get_internal_flags returns correct flags");
+
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
+        "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
+
+    ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
+        "kvstxn_get_newroot_ref returns != NULL when processing complete");
+
+    ok (streq (newroot, rootref),
+        "root stays identical when no ops in transaction");
+
+    verify_keys_and_ops_standard (kt);
+
+    kvstxn_mgr_remove_transaction (ktm, kt, false);
+
+    ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) == NULL,
+        "kvstxn_mgr_get_ready_transaction returns NULL, no more kvstxns");
+
+    kvstxn_mgr_destroy (ktm);
+    kvsroot_mgr_destroy (krm);
+    cache_destroy (cache);
+}
+
 void kvstxn_basic_kvstxn_process_test_normalization (void)
 {
     struct cache *cache;
@@ -760,7 +1008,7 @@ void kvstxn_basic_kvstxn_process_test_normalization (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_count_dirty_cb, &count) == 0,
@@ -769,7 +1017,7 @@ void kvstxn_basic_kvstxn_process_test_normalization (void)
     ok (count == 2,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -825,7 +1073,7 @@ void kvstxn_basic_kvstxn_process_test_multiple_transactions (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_count_dirty_cb, &count) == 0,
@@ -834,7 +1082,7 @@ void kvstxn_basic_kvstxn_process_test_multiple_transactions (void)
     ok (count == 1,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -850,7 +1098,7 @@ void kvstxn_basic_kvstxn_process_test_multiple_transactions (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     count = 0;
@@ -862,7 +1110,7 @@ void kvstxn_basic_kvstxn_process_test_multiple_transactions (void)
     ok (count == 2,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -927,7 +1175,7 @@ void kvstxn_basic_kvstxn_process_test_multiple_transactions_merge (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_count_dirty_cb, &count) == 0,
@@ -940,7 +1188,7 @@ void kvstxn_basic_kvstxn_process_test_multiple_transactions_merge (void)
     ok (count == 3,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -960,13 +1208,13 @@ void kvstxn_basic_kvstxn_process_test_multiple_transactions_merge (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns NULL, no more kvstxns");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_count_dirty_cb, &count) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -1018,7 +1266,7 @@ void kvstxn_basic_kvstxn_process_test_invalid_transaction (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready transaction");
 
-    ok (kvstxn_process (ktbad, 1, rootref) == KVSTXN_PROCESS_ERROR
+    ok (kvstxn_process (ktbad, rootref, 0) == KVSTXN_PROCESS_ERROR
         && kvstxn_get_errnum (ktbad) == EINVAL,
         "kvstxn_process fails on bad kvstxn");
 
@@ -1036,10 +1284,7 @@ void kvstxn_basic_root_not_dir (void)
     json_t *root;
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* make a non-dir root */
     root = treeobj_create_val ("abcd", 4);
@@ -1061,19 +1306,18 @@ void kvstxn_basic_root_not_dir (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR");
 
     /* error is caught continuously */
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR again");
 
     ok (kvstxn_get_errnum (kt) == EINVAL,
         "kvstxn_get_errnum return EINVAL");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (root);
 }
 
@@ -1088,7 +1332,7 @@ int rootref_cb (kvstxn_t *kt, const char *ref, void *data)
     json_t *rootdir;
     struct cache_entry *entry;
 
-    ok (strcmp (ref, rd->rootref) == 0,
+    ok (streq (ref, rd->rootref),
         "missing root reference is what we expect it to be");
 
     ok ((rootdir = treeobj_create_dir ()) != NULL,
@@ -1115,10 +1359,8 @@ void kvstxn_process_root_missing (void)
     json_t *rootdir;
     const char *newroot;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
+
     ok ((rootdir = treeobj_create_dir ()) != NULL,
         "treeobj_create_dir works");
 
@@ -1141,11 +1383,11 @@ void kvstxn_process_root_missing (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
         "kvstxn_process returns KVSTXN_PROCESS_LOAD_MISSING_REFS");
 
     /* user forgot to call kvstxn_iter_missing_refs() test */
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
         "kvstxn_process returns KVSTXN_PROCESS_LOAD_MISSING_REFS again");
 
     rd.cache = cache;
@@ -1154,17 +1396,17 @@ void kvstxn_process_root_missing (void)
     ok (kvstxn_iter_missing_refs (kt, rootref_cb, &rd) == 0,
         "kvstxn_iter_missing_refs works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     /* user forgot to call kvstxn_iter_dirty_cache_entries() test */
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES again");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -1175,8 +1417,7 @@ void kvstxn_process_root_missing (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "key1", "1");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
 }
 
 int missingref_count_cb (kvstxn_t *kt, const char *ref, void *data)
@@ -1201,10 +1442,7 @@ void kvstxn_process_missing_ref (void)
     const char *newroot;
     int count = 0;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -1246,11 +1484,11 @@ void kvstxn_process_missing_ref (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
         "kvstxn_process returns KVSTXN_PROCESS_LOAD_MISSING_REFS");
 
     /* user forgot to call kvstxn_iter_missing_refs() test */
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
         "kvstxn_process returns KVSTXN_PROCESS_LOAD_MISSING_REFS again");
 
     ok (kvstxn_iter_missing_refs (kt, missingref_count_cb, &count) == 0,
@@ -1266,17 +1504,17 @@ void kvstxn_process_missing_ref (void)
 
     (void)cache_insert (cache, entry);
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     /* user forgot to call kvstxn_iter_dirty_cache_entries() test */
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES again");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -1287,8 +1525,7 @@ void kvstxn_process_missing_ref (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "dir.val", "52");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -1312,10 +1549,7 @@ void kvstxn_process_multiple_missing_ref (void)
     json_t *ops = NULL;
     int count = 0;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -1381,6 +1615,7 @@ void kvstxn_process_multiple_missing_ref (void)
     ok (kvstxn_mgr_add_transaction (ktm,
                                     "transaction1",
                                     ops,
+                                    0,
                                     0) == 0,
         "kvstxn_mgr_add_transaction works");
 
@@ -1392,7 +1627,7 @@ void kvstxn_process_multiple_missing_ref (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
         "kvstxn_process returns KVSTXN_PROCESS_LOAD_MISSING_REFS");
 
     ok (kvstxn_iter_missing_refs (kt, missingref_count_cb, &count) == 0,
@@ -1418,13 +1653,13 @@ void kvstxn_process_multiple_missing_ref (void)
 
     (void)cache_insert (cache, entry);
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -1437,8 +1672,7 @@ void kvstxn_process_multiple_missing_ref (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "dir3.c", "72");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir1);
     json_decref (dir2);
     json_decref (dir3);
@@ -1460,10 +1694,7 @@ void kvstxn_process_multiple_identical_missing_ref (void)
     json_t *ops = NULL;
     int count = 0;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -1508,6 +1739,7 @@ void kvstxn_process_multiple_identical_missing_ref (void)
     ok (kvstxn_mgr_add_transaction (ktm,
                                     "transaction1",
                                     ops,
+                                    0,
                                     0) == 0,
         "kvstxn_mgr_add_transaction works");
 
@@ -1519,7 +1751,7 @@ void kvstxn_process_multiple_identical_missing_ref (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
         "kvstxn_process returns KVSTXN_PROCESS_LOAD_MISSING_REFS");
 
     ok (kvstxn_iter_missing_refs (kt, missingref_count_cb, &count) == 0,
@@ -1535,13 +1767,13 @@ void kvstxn_process_multiple_identical_missing_ref (void)
 
     (void)cache_insert (cache, entry);
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -1554,8 +1786,7 @@ void kvstxn_process_multiple_identical_missing_ref (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "dir.c", "72");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -1575,10 +1806,7 @@ void kvstxn_process_missing_ref_removed (void)
     json_t *ops = NULL;
     int count = 0;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -1624,6 +1852,7 @@ void kvstxn_process_missing_ref_removed (void)
     ok (kvstxn_mgr_add_transaction (ktm,
                                     "transaction1",
                                     ops,
+                                    0,
                                     0) == 0,
         "kvstxn_mgr_add_transaction works");
 
@@ -1632,7 +1861,7 @@ void kvstxn_process_missing_ref_removed (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
         "kvstxn_process returns KVSTXN_PROCESS_LOAD_MISSING_REFS");
 
     ok (kvstxn_iter_missing_refs (kt, missingref_count_cb, &count) == 0,
@@ -1648,13 +1877,13 @@ void kvstxn_process_missing_ref_removed (void)
 
     (void)cache_insert (cache, entry);
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -1667,8 +1896,7 @@ void kvstxn_process_missing_ref_removed (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "dir", NULL);
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -1700,10 +1928,7 @@ void kvstxn_process_error_callbacks (void)
     char root_ref[BLOBREF_MAX_STRING_SIZE];
     char dir_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -1743,7 +1968,7 @@ void kvstxn_process_error_callbacks (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
         "kvstxn_process returns KVSTXN_PROCESS_LOAD_MISSING_REFS");
 
     errno = 0;
@@ -1755,7 +1980,7 @@ void kvstxn_process_error_callbacks (void)
      * kvstxn_process call */
     (void)cache_insert (cache, create_cache_entry_treeobj (dir_ref, dir));
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     errno = 0;
@@ -1764,8 +1989,7 @@ void kvstxn_process_error_callbacks (void)
         "kvstxn_iter_dirty_cache_entries errors on callback error & returns correct errno");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -1799,10 +2023,7 @@ void kvstxn_process_error_callbacks_partway (void)
     char root_ref[BLOBREF_MAX_STRING_SIZE];
     char dir_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -1847,7 +2068,7 @@ void kvstxn_process_error_callbacks_partway (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     errno = 0;
@@ -1861,8 +2082,7 @@ void kvstxn_process_error_callbacks_partway (void)
         "correct number of successful returns from dirty cache callback");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -1876,10 +2096,7 @@ void kvstxn_process_invalid_operation (void)
     json_t *root;
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is an empty root */
     root = treeobj_create_dir ();
@@ -1901,19 +2118,18 @@ void kvstxn_process_invalid_operation (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR");
 
     /* error is caught continuously */
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR again");
 
     ok (kvstxn_get_errnum (kt) == EINVAL,
         "kvstxn_get_errnum return EINVAL");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (root);
 }
 
@@ -1948,6 +2164,7 @@ void kvstxn_process_malformed_operation (void)
     ok (kvstxn_mgr_add_transaction (ktm,
                                     "malformed",
                                     ops,
+                                    0,
                                     0) == 0,
         "kvstxn_mgr_add_transaction works");
 
@@ -1955,7 +2172,7 @@ void kvstxn_process_malformed_operation (void)
      */
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR
         && kvstxn_get_errnum (kt) == EPROTO,
         "kvstxn_process encountered EPROTO error");
 
@@ -1974,10 +2191,7 @@ void kvstxn_process_invalid_hash (void)
     json_t *root;
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is an empty root */
     root = treeobj_create_dir ();
@@ -1999,19 +2213,18 @@ void kvstxn_process_invalid_hash (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR");
 
     /* verify kvstxn_process() does not continue processing */
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR on second call");
 
     ok (kvstxn_get_errnum (kt) == EINVAL,
         "kvstxn_get_errnum return EINVAL %d", kvstxn_get_errnum (kt));
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (root);
 }
 
@@ -2027,10 +2240,7 @@ void kvstxn_process_follow_link_no_namespace (void)
     char dir_ref[BLOBREF_MAX_STRING_SIZE];
     const char *newroot;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -2074,13 +2284,13 @@ void kvstxn_process_follow_link_no_namespace (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2091,8 +2301,7 @@ void kvstxn_process_follow_link_no_namespace (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "symlink.val", "52");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -2107,10 +2316,7 @@ void kvstxn_process_follow_link_namespace (void)
     char root_ref[BLOBREF_MAX_STRING_SIZE];
     const char *newroot;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -2146,13 +2352,13 @@ void kvstxn_process_follow_link_namespace (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2182,7 +2388,7 @@ void kvstxn_process_follow_link_namespace (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR");
 
     ok (kvstxn_get_errnum (kt) == EINVAL,
@@ -2191,8 +2397,7 @@ void kvstxn_process_follow_link_namespace (void)
     kvstxn_mgr_remove_transaction (ktm, kt, false);
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (root);
 }
 
@@ -2207,10 +2412,7 @@ void kvstxn_process_dirval_test (void)
     char root_ref[BLOBREF_MAX_STRING_SIZE];
     const char *newroot;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -2244,13 +2446,13 @@ void kvstxn_process_dirval_test (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2261,8 +2463,7 @@ void kvstxn_process_dirval_test (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "dir.val", "52");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -2279,10 +2480,7 @@ void kvstxn_process_delete_test (void)
     char dir_ref[BLOBREF_MAX_STRING_SIZE];
     const char *newroot;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -2325,13 +2523,13 @@ void kvstxn_process_delete_test (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2342,8 +2540,7 @@ void kvstxn_process_delete_test (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "dir.val", NULL);
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -2358,10 +2555,7 @@ void kvstxn_process_delete_nosubdir_test (void)
     char root_ref[BLOBREF_MAX_STRING_SIZE];
     const char *newroot;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is an empty root */
     root = treeobj_create_dir ();
@@ -2387,7 +2581,7 @@ void kvstxn_process_delete_nosubdir_test (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2398,8 +2592,7 @@ void kvstxn_process_delete_nosubdir_test (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "noexistdir.val", NULL);
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (root);
 }
 
@@ -2415,10 +2608,7 @@ void kvstxn_process_delete_filevalinpath_test (void)
     char dir_ref[BLOBREF_MAX_STRING_SIZE];
     const char *newroot;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -2462,7 +2652,7 @@ void kvstxn_process_delete_filevalinpath_test (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2473,8 +2663,7 @@ void kvstxn_process_delete_filevalinpath_test (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "dir.val.valbaz", NULL);
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -2491,10 +2680,7 @@ void kvstxn_process_bad_dirrefs (void)
     char root_ref[BLOBREF_MAX_STRING_SIZE];
     char dir_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -2537,19 +2723,18 @@ void kvstxn_process_bad_dirrefs (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR");
 
     /* error is caught continuously */
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR again");
 
     ok (kvstxn_get_errnum (kt) == ENOTRECOVERABLE,
         "kvstxn_get_errnum return ENOTRECOVERABLE");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (dirref);
     json_decref (root);
@@ -2588,10 +2773,7 @@ void kvstxn_process_big_fileval (void)
     struct cache_count cache_count;
     int i;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -2624,7 +2806,7 @@ void kvstxn_process_big_fileval (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     cache_count.treeobj_count = 0;
@@ -2639,7 +2821,7 @@ void kvstxn_process_big_fileval (void)
     ok (cache_count.total_count == 1,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2663,7 +2845,7 @@ void kvstxn_process_big_fileval (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     cache_count.treeobj_count = 0;
@@ -2681,7 +2863,7 @@ void kvstxn_process_big_fileval (void)
     ok (cache_count.total_count == 2,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2692,8 +2874,7 @@ void kvstxn_process_big_fileval (void)
     verify_value (cache, krm, KVS_PRIMARY_NAMESPACE, newroot, "val", bigstr);
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (root);
 }
 
@@ -2712,10 +2893,7 @@ void kvstxn_process_giant_dir (void)
     char dir_ref[BLOBREF_MAX_STRING_SIZE];
     const char *newroot;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is.
      *
@@ -2798,13 +2976,13 @@ void kvstxn_process_giant_dir (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_noop_cb, NULL) == 0,
         "kvstxn_iter_dirty_cache_entries works for dirty cache entries");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2822,8 +3000,7 @@ void kvstxn_process_giant_dir (void)
         "kvstxn_mgr_get_ready_transaction returns NULL, no more kvstxns");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -2840,10 +3017,7 @@ void kvstxn_process_append (void)
     char root_ref[BLOBREF_MAX_STRING_SIZE];
     const char *newroot;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -2885,7 +3059,7 @@ void kvstxn_process_append (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     count = 0;
@@ -2897,7 +3071,7 @@ void kvstxn_process_append (void)
     ok (count == 3,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2918,7 +3092,7 @@ void kvstxn_process_append (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     count = 0;
@@ -2930,7 +3104,7 @@ void kvstxn_process_append (void)
     ok (count == 2,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2951,7 +3125,7 @@ void kvstxn_process_append (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     count = 0;
@@ -2962,7 +3136,7 @@ void kvstxn_process_append (void)
     ok (count == 1,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -2975,8 +3149,7 @@ void kvstxn_process_append (void)
     kvstxn_mgr_remove_transaction (ktm, kt, false);
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (root);
 }
 
@@ -2990,10 +3163,7 @@ void kvstxn_process_append_errors (void)
     json_t *dir;
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -3030,7 +3200,7 @@ void kvstxn_process_append_errors (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR");
 
     ok (kvstxn_get_errnum (kt) == EISDIR,
@@ -3047,7 +3217,7 @@ void kvstxn_process_append_errors (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR");
 
     ok (kvstxn_get_errnum (kt) == EOPNOTSUPP,
@@ -3064,7 +3234,7 @@ void kvstxn_process_append_errors (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR");
 
     ok (kvstxn_get_errnum (kt) == EOPNOTSUPP,
@@ -3073,8 +3243,7 @@ void kvstxn_process_append_errors (void)
     kvstxn_mgr_remove_transaction (ktm, kt, false);
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -3097,10 +3266,7 @@ void kvstxn_process_append_no_duplicate (void)
     const char *newroot;
     json_t *ops = NULL;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ktest_init (&cache, &krm);
 
     /* This root is
      *
@@ -3148,6 +3314,7 @@ void kvstxn_process_append_no_duplicate (void)
     ok (kvstxn_mgr_add_transaction (ktm,
                                     "transaction1",
                                     ops,
+                                    0,
                                     0) == 0,
         "kvstxn_mgr_add_transaction works");
 
@@ -3156,7 +3323,7 @@ void kvstxn_process_append_no_duplicate (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready kvstxn");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_LOAD_MISSING_REFS,
         "kvstxn_process returns KVSTXN_PROCESS_LOAD_MISSING_REFS");
 
     ok (kvstxn_iter_missing_refs (kt, missingref_count_cb, &count) == 0,
@@ -3172,7 +3339,7 @@ void kvstxn_process_append_no_duplicate (void)
 
     (void)cache_insert (cache, entry);
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     count = 0;
@@ -3185,7 +3352,7 @@ void kvstxn_process_append_no_duplicate (void)
     ok (count == 4,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, root_ref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, root_ref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -3199,8 +3366,7 @@ void kvstxn_process_append_no_duplicate (void)
     kvstxn_mgr_remove_transaction (ktm, kt, false);
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
     json_decref (dir);
     json_decref (root);
 }
@@ -3251,7 +3417,7 @@ void kvstxn_process_fallback_merge (void)
     ok (kvstxn_fallback_mergeable (kt) == true,
         "kvstxn_fallback_mergeable returns true on merged transaction");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     ok (kvstxn_iter_dirty_cache_entries (kt, cache_count_dirty_cb, &count) == 0,
@@ -3260,7 +3426,7 @@ void kvstxn_process_fallback_merge (void)
     ok (count == 1,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -3291,7 +3457,7 @@ void kvstxn_process_fallback_merge (void)
     ok ((kt = kvstxn_mgr_get_ready_transaction (ktm)) != NULL,
         "kvstxn_mgr_get_ready_transaction returns ready transaction");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR");
 
     ok (kvstxn_get_errnum (kt) == EINVAL,
@@ -3314,7 +3480,7 @@ void kvstxn_process_fallback_merge (void)
     ok (kvstxn_fallback_mergeable (kt) == false,
         "kvstxn_fallback_mergeable returns false on unmerged transaction");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES,
         "kvstxn_process returns KVSTXN_PROCESS_DIRTY_CACHE_ENTRIES");
 
     count = 0;
@@ -3324,7 +3490,7 @@ void kvstxn_process_fallback_merge (void)
     ok (count == 1,
         "correct number of cache entries were dirty");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_FINISHED,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_FINISHED,
         "kvstxn_process returns KVSTXN_PROCESS_FINISHED");
 
     ok ((newroot = kvstxn_get_newroot_ref (kt)) != NULL,
@@ -3346,7 +3512,7 @@ void kvstxn_process_fallback_merge (void)
     ok (kvstxn_fallback_mergeable (kt) == false,
         "kvstxn_fallback_mergeable returns false on unmerged transaction");
 
-    ok (kvstxn_process (kt, 1, rootref) == KVSTXN_PROCESS_ERROR,
+    ok (kvstxn_process (kt, rootref, 0) == KVSTXN_PROCESS_ERROR,
         "kvstxn_process returns KVSTXN_PROCESS_ERROR");
 
     ok (kvstxn_get_errnum (kt) == EINVAL,
@@ -3360,8 +3526,7 @@ void kvstxn_process_fallback_merge (void)
         "kvstxn_mgr_get_ready_transaction returns NULL, no more transactions");
 
     kvstxn_mgr_destroy (ktm);
-    kvsroot_mgr_destroy (krm);
-    cache_destroy (cache);
+    ktest_finalize (cache, krm);
 }
 
 int main (int argc, char *argv[])
@@ -3371,7 +3536,10 @@ int main (int argc, char *argv[])
     kvstxn_mgr_basic_tests ();
     kvstxn_mgr_merge_tests ();
     kvstxn_basic_tests ();
+    kvstxn_corner_case_tests ();
     kvstxn_basic_kvstxn_process_test ();
+    kvstxn_basic_kvstxn_process_test_empty_ops ();
+    kvstxn_basic_kvstxn_process_test_internal_flags ();
     kvstxn_basic_kvstxn_process_test_normalization ();
     kvstxn_basic_kvstxn_process_test_multiple_transactions ();
     kvstxn_basic_kvstxn_process_test_multiple_transactions_merge ();
diff --git a/src/modules/kvs/test/lookup.c b/src/modules/kvs/test/lookup.c
index e0b61a6c6bd9..ceb1bae5332c 100644
--- a/src/modules/kvs/test/lookup.c
+++ b/src/modules/kvs/test/lookup.c
@@ -18,12 +18,14 @@
 #include <sys/types.h>
 #include <assert.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libtap/tap.h"
 #include "src/common/libkvs/treeobj.h"
 #include "src/common/libkvs/kvs_util_private.h"
 #include "src/modules/kvs/cache.h"
 #include "src/modules/kvs/lookup.h"
 #include "src/common/libutil/blobref.h"
+#include "ccan/str/str.h"
 
 struct flux_msg_cred owner_cred = { .userid = 0, .rolemask = FLUX_ROLE_OWNER };
 struct flux_msg_cred user_cred = { .userid = 0, .rolemask = FLUX_ROLE_USER };
@@ -34,6 +36,20 @@ struct lookup_ref_data
     int count;
 };
 
+static void ltest_finalize (struct cache *cache, kvsroot_mgr_t *krm)
+{
+    cache_destroy (cache);
+    kvsroot_mgr_destroy (krm);
+}
+
+static void ltest_init (struct cache **cache, kvsroot_mgr_t **krm)
+{
+    if (!(*cache = cache_create (NULL)))
+        BAIL_OUT ("cache_create failed");
+    if (!(*krm = kvsroot_mgr_create (NULL, NULL)))
+        BAIL_OUT ("kvsroot_mgr_create failed");
+};
+
 static int treeobj_hash (const char *hash_name, json_t *obj,
                          char *blobref, int blobref_len)
 {
@@ -91,7 +107,7 @@ static struct cache_entry *create_cache_entry_raw (const char *ref,
                                                    int len)
 {
     struct cache_entry *entry;
-    int ret;
+    __attribute__((unused)) int ret;
 
     assert (data);
     assert (len);
@@ -108,7 +124,7 @@ static struct cache_entry *create_cache_entry_treeobj (const char *ref,
                                                        json_t *o)
 {
     struct cache_entry *entry;
-    int ret;
+    __attribute__((unused)) int ret;
 
     assert (o);
 
@@ -210,15 +226,12 @@ void basic_api (void)
     lookup_t *lh;
     const char *tmp;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
+
     setup_kvsroot (krm, KVS_PRIMARY_NAMESPACE, cache, "root.ref.foo", 0);
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             42,
                              KVS_PRIMARY_NAMESPACE,
                              "root.ref.foo",
                              0,
@@ -227,18 +240,12 @@ void basic_api (void)
                              FLUX_KVS_READLINK | FLUX_KVS_TREEOBJ,
                              NULL)) != NULL,
         "lookup_create works");
-    ok (lookup_get_current_epoch (lh) == 42,
-        "lookup_get_current_epoch works");
     ok ((tmp = lookup_get_namespace (lh)) != NULL,
         "lookup_get_namespace works");
-    ok (!strcmp (tmp, KVS_PRIMARY_NAMESPACE),
+    ok (streq (tmp, KVS_PRIMARY_NAMESPACE),
         "lookup_get_namespace returns correct string");
     ok (lookup_missing_namespace (lh) == NULL,
         "lookup_missing_namespace returned NULL, no missing namespace yet");
-    ok (lookup_set_current_epoch (lh, 43) == 0,
-        "lookup_set_current_epoch works");
-    ok (lookup_get_current_epoch (lh) == 43,
-        "lookup_get_current_epoch works");
     ok (lookup_get_aux_errnum (lh) == 0,
         "lookup_get_aux_errnum returns no error");
     ok (lookup_set_aux_errnum (lh, EINVAL) == EINVAL,
@@ -252,8 +259,7 @@ void basic_api (void)
 
     lookup_destroy (lh);
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
 }
 
 void basic_api_errors (void)
@@ -264,7 +270,6 @@ void basic_api_errors (void)
 
     ok (lookup_create (NULL,
                        NULL,
-                       0,
                        NULL,
                        NULL,
                        0,
@@ -274,14 +279,10 @@ void basic_api_errors (void)
                        NULL) == NULL,
         "lookup_create fails on bad input");
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             42,
                              NULL,
                              NULL,
                              0,
@@ -295,7 +296,6 @@ void basic_api_errors (void)
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             42,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -322,23 +322,18 @@ void basic_api_errors (void)
         "lookup_iter_missing_refs fails on NULL pointer");
     ok (lookup_missing_namespace (NULL) == NULL,
         "lookup_missing_namespace fails on NULL pointer");
-    ok (lookup_get_current_epoch (NULL) < 0,
-        "lookup_get_current_epoch fails on NULL pointer");
     ok (lookup_get_namespace (NULL) == NULL,
         "lookup_get_namespace fails on NULL pointer");
     ok (lookup_get_root_ref (NULL) == NULL,
         "lookup_get_root_ref fails on NULL pointer");
     ok (lookup_get_root_seq (NULL) < 0,
         "lookup_get_root_seq fails on NULL pointer");
-    ok (lookup_set_current_epoch (NULL, 42) < 0,
-        "lookup_set_current_epoch fails on NULL pointer");
     /* lookup_destroy ok on NULL pointer */
     lookup_destroy (NULL);
 
     lookup_destroy (lh);
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
 }
 
 /* basic lookup to test a few situations that we don't want to
@@ -352,10 +347,7 @@ void basic_lookup (void) {
     char root_ref[BLOBREF_MAX_STRING_SIZE];
     const char *tmp;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -374,7 +366,6 @@ void basic_lookup (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -387,7 +378,7 @@ void basic_lookup (void) {
         "lookup process finished");
     ok ((tmp = lookup_get_root_ref (lh)) != NULL,
         "lookup_get_root_ref returns non-NULL");
-    ok (!strcmp (tmp, root_ref),
+    ok (streq (tmp, root_ref),
         "lookup_get_root_ref returned correct root_ref");
     ok (lookup_get_root_seq (lh) >= 0,
         "lookup_get_root_seq returned valid root_seq");
@@ -396,7 +387,6 @@ void basic_lookup (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              root_ref,
                              18,
@@ -409,12 +399,15 @@ void basic_lookup (void) {
         "lookup process finished");
     ok ((tmp = lookup_get_root_ref (lh)) != NULL,
         "lookup_get_root_ref returns non-NULL");
-    ok (!strcmp (tmp, root_ref),
+    ok (streq (tmp, root_ref),
         "lookup_get_root_ref returned correct root_ref");
     ok (lookup_get_root_seq (lh) == 18,
         "lookup_get_root_seq returned correct root_seq");
 
     lookup_destroy (lh);
+
+    ltest_finalize (cache, krm);
+    json_decref (root);
 }
 
 void check_common (lookup_t *lh,
@@ -474,7 +467,7 @@ void check_common (lookup_t *lh,
                 "%s: missing ref returned one missing refs", msg);
 
             if (ld.ref) {
-                ok (strcmp (ld.ref, missing_ref_result) == 0,
+                ok (streq (ld.ref, missing_ref_result),
                     "%s: missing ref returned matched expectation", msg);
             }
             else {
@@ -573,10 +566,7 @@ void lookup_root (void) {
     char valref_ref[BLOBREF_MAX_STRING_SIZE];
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -599,7 +589,6 @@ void lookup_root (void) {
     /* flags = 0, should error EISDIR */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -613,7 +602,6 @@ void lookup_root (void) {
     /* flags = FLUX_KVS_READDIR, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -627,7 +615,6 @@ void lookup_root (void) {
     /* flags = FLUX_KVS_TREEOBJ, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -643,7 +630,6 @@ void lookup_root (void) {
     /* flags = FLUX_KVS_READDIR, bad root_ref, should error EINVAL */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              valref_ref,
                              0,
@@ -654,8 +640,7 @@ void lookup_root (void) {
         "lookup_create on root w/ flag = FLUX_KVS_READDIR, bad root_ref, should EINVAL");
     check_error (lh, EINVAL, "root w/ FLUX_KVS_READDIR, bad root_ref, should EINVAL");
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (root);
 }
 
@@ -677,10 +662,7 @@ void lookup_basic (void) {
     char dirref_test_ref[BLOBREF_MAX_STRING_SIZE];
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -754,7 +736,6 @@ void lookup_basic (void) {
     /* lookup dir via dirref */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -768,7 +749,6 @@ void lookup_basic (void) {
     /* lookup value via valref */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -788,7 +768,6 @@ void lookup_basic (void) {
      */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -802,7 +781,6 @@ void lookup_basic (void) {
     /* Lookup value via valref with multiple blobrefs */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -822,7 +800,6 @@ void lookup_basic (void) {
      */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -836,7 +813,6 @@ void lookup_basic (void) {
     /* lookup value via val */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -852,7 +828,6 @@ void lookup_basic (void) {
     /* lookup dir via dir */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -866,7 +841,6 @@ void lookup_basic (void) {
     /* lookup symlink */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -882,7 +856,6 @@ void lookup_basic (void) {
     /* lookup symlinkNS */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -898,7 +871,6 @@ void lookup_basic (void) {
     /* lookup dirref treeobj */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -914,7 +886,6 @@ void lookup_basic (void) {
     /* lookup valref treeobj */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -930,7 +901,6 @@ void lookup_basic (void) {
     /* lookup val treeobj */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -946,7 +916,6 @@ void lookup_basic (void) {
     /* lookup dir treeobj */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -960,7 +929,6 @@ void lookup_basic (void) {
     /* lookup symlink treeobj */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -976,7 +944,6 @@ void lookup_basic (void) {
     /* lookup symlinkNS treeobj */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -989,8 +956,7 @@ void lookup_basic (void) {
     check_value (lh, test, "lookup dirref.symlinkNS treeobj");
     json_decref (test);
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (dirref_test);
     json_decref (dir);
     json_decref (dirref);
@@ -1012,10 +978,7 @@ void lookup_errors (void) {
     char valref_ref[BLOBREF_MAX_STRING_SIZE];
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -1082,7 +1045,6 @@ void lookup_errors (void) {
      * decides what to do with entry not found */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1097,7 +1059,6 @@ void lookup_errors (void) {
      * decides what to do with entry not found */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1112,7 +1073,6 @@ void lookup_errors (void) {
      * decides what to do with entry not found */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1126,7 +1086,6 @@ void lookup_errors (void) {
     /* Lookup path w/ dir in middle, should get ENOTRECOVERABLE */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1140,7 +1099,6 @@ void lookup_errors (void) {
     /* Lookup path w/ infinite link loop, should get ELOOP */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1154,7 +1112,6 @@ void lookup_errors (void) {
     /* Lookup path w/ infinite symlinkNS loop, should get ELOOP */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1168,7 +1125,6 @@ void lookup_errors (void) {
     /* Lookup path w/ infinite symlink w/ & w/o namespace loop, should get ELOOP */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1182,7 +1138,6 @@ void lookup_errors (void) {
     /* Lookup a dirref, but expecting a link, should get EINVAL. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1196,7 +1151,6 @@ void lookup_errors (void) {
     /* Lookup a dir, but expecting a link, should get EINVAL. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1210,7 +1164,6 @@ void lookup_errors (void) {
     /* Lookup a valref, but expecting a link, should get EINVAL. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1224,7 +1177,6 @@ void lookup_errors (void) {
     /* Lookup a val, but expecting a link, should get EINVAL. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1238,7 +1190,6 @@ void lookup_errors (void) {
     /* Lookup a dirref, but don't expect a dir, should get EISDIR. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1252,7 +1203,6 @@ void lookup_errors (void) {
     /* Lookup a dir, but don't expect a dir, should get EISDIR. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1266,7 +1216,6 @@ void lookup_errors (void) {
     /* Lookup a valref, but expecting a dir, should get ENOTDIR. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1280,7 +1229,6 @@ void lookup_errors (void) {
     /* Lookup a val, but expecting a dir, should get ENOTDIR. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1294,7 +1242,6 @@ void lookup_errors (void) {
     /* Lookup a symlink, but expecting a dir, should get ENOTDIR. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1308,7 +1255,6 @@ void lookup_errors (void) {
     /* Lookup a symlinkNS, but expecting a dir, should get ENOTDIR. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1322,7 +1268,6 @@ void lookup_errors (void) {
     /* Lookup a dirref that doesn't point to a dir, should get ENOTRECOVERABLE. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1337,7 +1282,6 @@ void lookup_errors (void) {
      * should get ENOTRECOVERABLE. */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1351,7 +1295,6 @@ void lookup_errors (void) {
     /* Lookup with an invalid root_ref, should get EINVAL */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              valref_ref,
                              0,
@@ -1365,7 +1308,6 @@ void lookup_errors (void) {
     /* Lookup dirref with multiple blobrefs, should get ENOTRECOVERABLE */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1380,7 +1322,6 @@ void lookup_errors (void) {
      * get ENOTRECOVERABLE */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1398,7 +1339,6 @@ void lookup_errors (void) {
     /* Lookup with an invalid root_ref, should get EINVAL */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              valref_ref,
                              0,
@@ -1413,8 +1353,7 @@ void lookup_errors (void) {
         "lookup still returns LOOKUP_PROCESS_ERROR on second call");
     lookup_destroy (lh);
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (dirref);
     json_decref (dir);
     json_decref (root);
@@ -1434,10 +1373,7 @@ void lookup_security (void) {
     struct flux_msg_cred owner_7 = { .userid = 7, .rolemask = FLUX_ROLE_OWNER};
     struct flux_msg_cred user_7 = { .userid = 7, .rolemask = FLUX_ROLE_USER};
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -1457,7 +1393,6 @@ void lookup_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1472,7 +1407,6 @@ void lookup_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1487,7 +1421,6 @@ void lookup_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1501,7 +1434,6 @@ void lookup_security (void) {
     /* if root_ref is set, namespace checks won't occur */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              root_ref,
                              0,
@@ -1516,7 +1448,6 @@ void lookup_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "altnamespace",
                              NULL,
                              0,
@@ -1531,7 +1462,6 @@ void lookup_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "altnamespace",
                              NULL,
                              0,
@@ -1546,7 +1476,6 @@ void lookup_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "altnamespace",
                              NULL,
                              0,
@@ -1561,7 +1490,6 @@ void lookup_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "altnamespace",
                              NULL,
                              0,
@@ -1572,8 +1500,7 @@ void lookup_security (void) {
         "lookup_create on val on namespace altnamespace with rolemask user and invalid owner");
     check_error (lh, EPERM, "lookup_create on val on namespace altnamespace with rolemask user and invalid owner");
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (root);
 }
 
@@ -1594,10 +1521,7 @@ void lookup_links (void) {
     char dirref1_ref[BLOBREF_MAX_STRING_SIZE];
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -1666,7 +1590,6 @@ void lookup_links (void) {
     /* lookup val, follow two links */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1682,7 +1605,6 @@ void lookup_links (void) {
     /* lookup val, link is middle of path */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1698,7 +1620,6 @@ void lookup_links (void) {
     /* lookup valref, link is middle of path */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1714,7 +1635,6 @@ void lookup_links (void) {
     /* lookup dir, link is middle of path */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1728,7 +1648,6 @@ void lookup_links (void) {
     /* lookup dirref, link is middle of path */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1742,7 +1661,6 @@ void lookup_links (void) {
     /* lookup symlink, link is middle of path */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1758,7 +1676,6 @@ void lookup_links (void) {
     /* lookup val, link is last part in path */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1774,7 +1691,6 @@ void lookup_links (void) {
     /* lookup valref, link is last part in path */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1790,7 +1706,6 @@ void lookup_links (void) {
     /* lookup dir, link is last part in path */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1804,7 +1719,6 @@ void lookup_links (void) {
     /* lookup dirref, link is last part in path */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1818,7 +1732,6 @@ void lookup_links (void) {
     /* lookup symlink, link is last part in path */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -1831,8 +1744,7 @@ void lookup_links (void) {
     check_value (lh, test, "dirref1.link2symlink");
     json_decref (test);
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (dirref3);
     json_decref (dir);
     json_decref (dirref2);
@@ -1853,10 +1765,7 @@ void lookup_alt_root (void) {
     char dirref2_ref[BLOBREF_MAX_STRING_SIZE];
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -1892,7 +1801,6 @@ void lookup_alt_root (void) {
     /* lookup val, alt root-ref dirref1_ref */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              dirref1_ref,
                              0,
@@ -1908,7 +1816,6 @@ void lookup_alt_root (void) {
     /* lookup val, alt root-ref dirref2_ref */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              dirref2_ref,
                              0,
@@ -1924,7 +1831,6 @@ void lookup_alt_root (void) {
     /* lookup val, alt root-ref dirref1_ref */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              dirref1_ref,
                              0,
@@ -1940,7 +1846,6 @@ void lookup_alt_root (void) {
     /* lookup val, alt root-ref dirref2_ref */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              dirref2_ref,
                              0,
@@ -1953,8 +1858,7 @@ void lookup_alt_root (void) {
     check_value (lh, test, "alt root val");
     json_decref (test);
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (dirref1);
     json_decref (dirref2);
     json_decref (root);
@@ -1972,10 +1876,7 @@ void lookup_root_symlink (void) {
     char valref_ref[BLOBREF_MAX_STRING_SIZE];
     char dirref_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -2011,7 +1912,6 @@ void lookup_root_symlink (void) {
     /* flags = 0, should error EISDIR */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2025,7 +1925,6 @@ void lookup_root_symlink (void) {
     /* flags = FLUX_KVS_READDIR, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2039,7 +1938,6 @@ void lookup_root_symlink (void) {
     /* flags = FLUX_KVS_READDIR, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2054,7 +1952,6 @@ void lookup_root_symlink (void) {
     /* flags = FLUX_KVS_TREEOBJ, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2069,7 +1966,6 @@ void lookup_root_symlink (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2085,7 +1981,6 @@ void lookup_root_symlink (void) {
     /* flags = FLUX_KVS_READDIR, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              dirref_ref,
                              0,
@@ -2099,7 +1994,6 @@ void lookup_root_symlink (void) {
     /* flags = FLUX_KVS_READDIR, bad root_ref, should error EINVAL */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              valref_ref,
                              0,
@@ -2110,8 +2004,7 @@ void lookup_root_symlink (void) {
         "lookup_create on symlinkroot w/ flag = FLUX_KVS_READDIR, bad root_ref, should EINVAL");
     check_error (lh, EINVAL, "symlinkroot w/ FLUX_KVS_READDIR, bad root_ref, should EINVAL");
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (dirref);
     json_decref (root);
 }
@@ -2127,10 +2020,7 @@ void lookup_symlinkNS (void) {
     char root_refA[BLOBREF_MAX_STRING_SIZE];
     char root_refB[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -2170,7 +2060,6 @@ void lookup_symlinkNS (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2183,7 +2072,6 @@ void lookup_symlinkNS (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2196,7 +2084,6 @@ void lookup_symlinkNS (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2211,7 +2098,6 @@ void lookup_symlinkNS (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2226,7 +2112,6 @@ void lookup_symlinkNS (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2241,7 +2126,6 @@ void lookup_symlinkNS (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2256,7 +2140,6 @@ void lookup_symlinkNS (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2269,7 +2152,6 @@ void lookup_symlinkNS (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2280,8 +2162,7 @@ void lookup_symlinkNS (void) {
         "lookup_create symlinkNS2B on namespace A, readdir");
     check_value (lh, rootB, "symlinkNS2B on namespace A, readdir");
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (rootA);
     json_decref (rootB);
 }
@@ -2302,10 +2183,7 @@ void lookup_symlinkNS_security (void) {
     struct flux_msg_cred user_1000 = { .rolemask = FLUX_ROLE_USER,
                                        .userid = 1000 };
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -2347,7 +2225,6 @@ void lookup_symlinkNS_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2362,7 +2239,6 @@ void lookup_symlinkNS_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2377,7 +2253,6 @@ void lookup_symlinkNS_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2392,7 +2267,6 @@ void lookup_symlinkNS_security (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "A",
                              NULL,
                              0,
@@ -2403,8 +2277,7 @@ void lookup_symlinkNS_security (void) {
         "lookup_create on symlinkNS2C.val with rolemask user and invalid owner");
     check_error (lh, EPERM, "lookup_create on symlinkNS2C.val with rolemask user and invalid owner");
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (rootA);
     json_decref (rootB);
     json_decref (rootC);
@@ -2422,10 +2295,7 @@ void lookup_stall_namespace (void) {
     char root_ref2[BLOBREF_MAX_STRING_SIZE];
     const char *tmp;
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -2453,7 +2323,6 @@ void lookup_stall_namespace (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2466,7 +2335,7 @@ void lookup_stall_namespace (void) {
         "lookup stalled on missing namespace");
     ok ((tmp = lookup_missing_namespace (lh)) != NULL,
         "lookup_missing_namespace returned non-NULL");
-    ok (!strcmp (tmp, KVS_PRIMARY_NAMESPACE),
+    ok (streq (tmp, KVS_PRIMARY_NAMESPACE),
         "lookup_missing_namespace returned correct namespace");
 
     setup_kvsroot (krm, KVS_PRIMARY_NAMESPACE, cache, root_ref1, 0);
@@ -2479,7 +2348,6 @@ void lookup_stall_namespace (void) {
     /* lookup "val" should succeed cleanly */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2496,7 +2364,6 @@ void lookup_stall_namespace (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "foo",
                              NULL,
                              0,
@@ -2509,7 +2376,7 @@ void lookup_stall_namespace (void) {
         "lookup stalled on missing namespace");
     ok ((tmp = lookup_missing_namespace (lh)) != NULL,
         "lookup_missing_namespace returned non-NULL");
-    ok (!strcmp (tmp, "foo"),
+    ok (streq (tmp, "foo"),
         "lookup_missing_namespace returned correct namespace");
 
     setup_kvsroot (krm, "foo", cache, root_ref2, 0);
@@ -2522,7 +2389,6 @@ void lookup_stall_namespace (void) {
     /* lookup val on namespace foo should succeed cleanly */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "foo",
                              NULL,
                              0,
@@ -2539,7 +2405,6 @@ void lookup_stall_namespace (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              "roottest",
                              NULL,
                              0,
@@ -2552,7 +2417,7 @@ void lookup_stall_namespace (void) {
         "lookup stalled on missing namespace");
     ok ((tmp = lookup_missing_namespace (lh)) != NULL,
         "lookup_missing_namespace returned non-NULL");
-    ok (!strcmp (tmp, "roottest"),
+    ok (streq (tmp, "roottest"),
         "lookup_missing_namespace returned correct namespace");
 
     setup_kvsroot (krm, "roottest", cache, root_ref1, 0);
@@ -2562,8 +2427,7 @@ void lookup_stall_namespace (void) {
     check_value (lh, test, ".");
     json_decref (test);
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (root1);
     json_decref (root2);
 }
@@ -2576,10 +2440,7 @@ void lookup_stall_ref_root (void) {
     lookup_t *lh;
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -2598,7 +2459,6 @@ void lookup_stall_ref_root (void) {
     /* lookup root ".", should stall on root */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2617,7 +2477,6 @@ void lookup_stall_ref_root (void) {
     /* lookup root ".", now fully cached, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2628,8 +2487,7 @@ void lookup_stall_ref_root (void) {
         "lookup_create stalltest \".\"");
     check_value (lh, root, "root \".\" #2");
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (root);
 }
 
@@ -2655,10 +2513,7 @@ void lookup_stall_ref (void) {
     char dirref2_ref[BLOBREF_MAX_STRING_SIZE];
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -2738,7 +2593,6 @@ void lookup_stall_ref (void) {
     /* lookup dirref1.val, should stall on root */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2764,7 +2618,6 @@ void lookup_stall_ref (void) {
     /* lookup dirref1.val, now fully cached, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2780,7 +2633,6 @@ void lookup_stall_ref (void) {
     /* lookup symlink.val, should stall */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              root_ref,
                              0,
@@ -2801,7 +2653,6 @@ void lookup_stall_ref (void) {
     /* lookup symlink.val, now fully cached, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2817,7 +2668,6 @@ void lookup_stall_ref (void) {
     /* lookup dirref1.valref, should stall */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2838,7 +2688,6 @@ void lookup_stall_ref (void) {
     /* lookup dirref1.valref, now fully cached, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2854,7 +2703,6 @@ void lookup_stall_ref (void) {
     /* lookup dirref1.valref_multi, should stall */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2877,7 +2725,6 @@ void lookup_stall_ref (void) {
     /* lookup dirref1.valref_multi, now fully cached, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2893,7 +2740,6 @@ void lookup_stall_ref (void) {
     /* lookup dirref1.valref_multi2, should stall */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2916,7 +2762,6 @@ void lookup_stall_ref (void) {
     /* lookup dirref1.valref_multi2, now fully cached, should succeed */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2932,7 +2777,6 @@ void lookup_stall_ref (void) {
     /* lookup dirref1.valrefmisc, should stall */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2954,7 +2798,6 @@ void lookup_stall_ref (void) {
     /* lookup dirref1.valrefmisc_multi, should stall */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -2973,8 +2816,7 @@ void lookup_stall_ref (void) {
         "dirref1.valrefmisc_multi: error & errno properly returned from callback error");
     lookup_destroy (lh);
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (dirref1);
     json_decref (valref_tmp1);
     json_decref (valref_tmp2);
@@ -2995,10 +2837,7 @@ void lookup_stall_namespace_removed (void) {
     char dirref_ref[BLOBREF_MAX_STRING_SIZE];
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -3037,7 +2876,6 @@ void lookup_stall_namespace_removed (void) {
     /* lookup dirref.valref, should stall on root */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -3065,7 +2903,6 @@ void lookup_stall_namespace_removed (void) {
     /* lookup dirref.valref, should stall on dirref */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -3091,7 +2928,6 @@ void lookup_stall_namespace_removed (void) {
     /* lookup dirref.valref, should stall on valref */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -3125,7 +2961,6 @@ void lookup_stall_namespace_removed (void) {
     /* lookup dirref.valref, should stall on root */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -3156,7 +2991,6 @@ void lookup_stall_namespace_removed (void) {
     /* lookup dirref.valref, should stall on dirref */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -3186,7 +3020,6 @@ void lookup_stall_namespace_removed (void) {
     /* lookup dirref.valref, should stall on valref */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -3223,7 +3056,6 @@ void lookup_stall_namespace_removed (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              root_ref,
                              0,
@@ -3264,7 +3096,6 @@ void lookup_stall_namespace_removed (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              root_ref,
                              0,
@@ -3306,8 +3137,7 @@ void lookup_stall_namespace_removed (void) {
     cache_remove_entry (cache, valref_ref);
     setup_kvsroot (krm, KVS_PRIMARY_NAMESPACE, cache, root_ref, 0);
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (dirref);
     json_decref (valref);
     json_decref (root);
@@ -3327,10 +3157,7 @@ void lookup_stall_ref_expire_cache_entries (void) {
     char dirref2_ref[BLOBREF_MAX_STRING_SIZE];
     char root_ref[BLOBREF_MAX_STRING_SIZE];
 
-    ok ((cache = cache_create ()) != NULL,
-        "cache_create works");
-    ok ((krm = kvsroot_mgr_create (NULL, NULL)) != NULL,
-        "kvsroot_mgr_create works");
+    ltest_init (&cache, &krm);
 
     /* This cache is
      *
@@ -3374,7 +3201,6 @@ void lookup_stall_ref_expire_cache_entries (void) {
     /* lookup dirref1.val, should stall on root */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -3393,7 +3219,7 @@ void lookup_stall_ref_expire_cache_entries (void) {
     ok (cache_count_entries (cache) == 1,
         "cache_count_entries returns 1");
 
-    ok (cache_expire_entries (cache, 10, 1) == 0,
+    ok (cache_expire_entries (cache, 0) == 0,
         "cache_expire_entries expires 0 entries, b/c references appropriately taken");
 
     (void)cache_insert (cache, create_cache_entry_treeobj (dirref1_ref, dirref1));
@@ -3405,7 +3231,7 @@ void lookup_stall_ref_expire_cache_entries (void) {
 
     /* clear cache */
 
-    ok (cache_expire_entries (cache, 10, 1) == 2,
+    ok (cache_expire_entries (cache, 0) == 2,
         "cache_expire_entries expires 2 entries");
 
     ok (cache_count_entries (cache) == 0,
@@ -3413,7 +3239,6 @@ void lookup_stall_ref_expire_cache_entries (void) {
 
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              NULL,
                              root_ref,
                              0,
@@ -3431,7 +3256,7 @@ void lookup_stall_ref_expire_cache_entries (void) {
     ok (cache_count_entries (cache) == 1,
         "cache_count_entries returns 1");
 
-    ok (cache_expire_entries (cache, 10, 1) == 0,
+    ok (cache_expire_entries (cache, 0) == 0,
         "cache_expire_entries expires 0 entries, b/c references appropriately taken");
 
     (void)cache_insert (cache, create_cache_entry_treeobj (dirref2_ref, dirref2));
@@ -3443,7 +3268,7 @@ void lookup_stall_ref_expire_cache_entries (void) {
 
     /* clear cache */
 
-    ok (cache_expire_entries (cache, 10, 1) == 2,
+    ok (cache_expire_entries (cache, 0) == 2,
         "cache_expire_entries expires 2 entries");
 
     ok (cache_count_entries (cache) == 0,
@@ -3452,7 +3277,6 @@ void lookup_stall_ref_expire_cache_entries (void) {
     /* lookup dirref1.valref, should stall */
     ok ((lh = lookup_create (cache,
                              krm,
-                             1,
                              KVS_PRIMARY_NAMESPACE,
                              NULL,
                              0,
@@ -3474,7 +3298,7 @@ void lookup_stall_ref_expire_cache_entries (void) {
     ok (cache_count_entries (cache) == 2,
         "cache_count_entries returns 2");
 
-    ok (cache_expire_entries (cache, 10, 1) == 1,
+    ok (cache_expire_entries (cache, 0) == 1,
         "cache_expire_entries expires 1 entry, only 1 entry has reference on it");
 
     (void)cache_insert (cache, create_cache_entry_raw (valref_ref, "abcd", 4));
@@ -3486,14 +3310,13 @@ void lookup_stall_ref_expire_cache_entries (void) {
 
     /* clear cache */
 
-    ok (cache_expire_entries (cache, 10, 1) == 2,
+    ok (cache_expire_entries (cache, 0) == 2,
         "cache_expire_entries expires 2 entries");
 
     ok (cache_count_entries (cache) == 0,
         "cache_count_entries returns 0");
 
-    cache_destroy (cache);
-    kvsroot_mgr_destroy (krm);
+    ltest_finalize (cache, krm);
     json_decref (dirref1);
     json_decref (dirref2);
     json_decref (root);
diff --git a/src/modules/kvs/test/treq.c b/src/modules/kvs/test/treq.c
index 92c1262780d9..76c6cbe2bd1f 100644
--- a/src/modules/kvs/test/treq.c
+++ b/src/modules/kvs/test/treq.c
@@ -19,6 +19,7 @@
 #include "src/common/libflux/message.h"
 #include "src/common/libflux/request.h"
 #include "src/modules/kvs/treq.h"
+#include "ccan/str/str.h"
 
 int msg_cb (treq_t *tr, const flux_msg_t *req, void *data)
 {
@@ -26,7 +27,7 @@ int msg_cb (treq_t *tr, const flux_msg_t *req, void *data)
     const char *topic;
 
     if (!flux_msg_get_topic (req, &topic)
-        && !strcmp (topic, "mytopic"))
+        && streq (topic, "mytopic"))
         (*count)++;
 
     return 0;
@@ -58,7 +59,7 @@ void treq_basic_tests (void)
     ok ((name = treq_get_name (tr)) != NULL,
         "treq_get_name works");
 
-    ok (strcmp (name, "foo") == 0,
+    ok (streq (name, "foo"),
         "treq_get_name returns the correct name");
 
     ok (treq_get_nprocs (tr) == 1,
diff --git a/src/modules/kvs/test/waitqueue.c b/src/modules/kvs/test/waitqueue.c
index 73fe021e69d7..f258c49b167c 100644
--- a/src/modules/kvs/test/waitqueue.c
+++ b/src/modules/kvs/test/waitqueue.c
@@ -8,9 +8,14 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
 #include "src/modules/kvs/waitqueue.h"
 #include "src/common/libflux/message.h"
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 void wait_cb (void *arg)
 {
@@ -43,13 +48,11 @@ void msghand (flux_t *h, flux_msg_handler_t *mh,
 
 bool msgcmp (const flux_msg_t *msg, void *arg)
 {
-    char *id = NULL;
+    const char *id;
     bool match = false;
-    if (flux_msg_get_route_first (msg, &id) == 0
-        && (!strcmp (id, "19") || !strcmp (id, "18") || !strcmp (id, "17")))
+    if ((id = flux_msg_route_first (msg))
+        && (streq (id, "19") || streq (id, "18") || streq (id, "17")))
         match = true;
-    if (id)
-        free (id);
     return match;
 }
 
@@ -105,7 +108,7 @@ int main (int argc, char *argv[])
     ok (wait_msg_aux_set (w, "aux", "val", NULL) == 0,
         "wait_msg_aux_set works");
     str = wait_msg_aux_get (w, "aux");
-    ok (str && !strcmp (str, "val"),
+    ok (str && streq (str, "val"),
         "wait_msg_aux_get works and returns correct value");
     flux_msg_destroy (msg);
     wait_destroy (w);
@@ -169,8 +172,16 @@ int main (int argc, char *argv[])
         "wait_create works");
     ok ((q = wait_queue_create ()) != NULL,
         "wait_queue_create works");
+    ok (wait_queue_length (q) == 0,
+        "wait_queue_length 0 on new queue");
+    ok (wait_queue_msgs_count (q) == 0,
+        "wait_queue_msgs_count 0 on new queue");
     ok (wait_addqueue (q, w) == 0,
         "wait_addqueue works");
+    ok (wait_queue_length (q) == 1,
+        "wait_queue_length 1 after addqueue");
+    ok (wait_queue_msgs_count (q) == 0,
+        "wait_queue_msgs_count 0 after addqueue w/o msg");
     ok (wait_get_usecount (w) == 1,
         "wait_get_usecount 1 after wait_addqueue");
     ok (count == 0,
@@ -191,6 +202,8 @@ int main (int argc, char *argv[])
         "wait_queue_create works");
     ok (wait_queue_length (q) == 0 && wait_queue_length (q2) == 0,
         "wait_queue_length 0 on new queue");
+    ok (wait_queue_msgs_count (q) == 0 && wait_queue_msgs_count (q2) == 0,
+        "wait_queue_msgs_count 0 on new queue");
 
     /* Create wait_t for msg
      * Add to two queues, run queues, wait_t called once
@@ -203,7 +216,6 @@ int main (int argc, char *argv[])
     ok (w != NULL,
         "wait_create_msg_handler with non-NULL msg works");
     flux_msg_destroy (msg);
-
     ok (wait_get_usecount (w) == 0,
         "wait_usecount 0 initially");
     ok (wait_addqueue (q, w) == 0,
@@ -216,11 +228,15 @@ int main (int argc, char *argv[])
         "wait_usecount 2 after adding to second queue");
     ok (wait_queue_length (q) == 1 && wait_queue_length (q2) == 1,
         "wait_queue_length of each queue is 1");
+    ok (wait_queue_msgs_count (q) == 1 && wait_queue_msgs_count (q2) == 1,
+        "wait_queue_msgs_count of each queue is 1");
 
     ok (wait_runqueue (q) == 0,
         "wait_runqueue success");
     ok (wait_queue_length (q) == 0 && wait_queue_length (q2) == 1,
         "wait_runqueue dequeued wait_t from first queue");
+    ok (wait_queue_msgs_count (q) == 0 && wait_queue_msgs_count (q2) == 1,
+        "wait_runqueue dequeued wait_t from first queue, msgs count valid");
     ok (wait_get_usecount (w) == 1,
         "wait_usecount 1 after one run");
     ok (count == 0,
@@ -230,6 +246,8 @@ int main (int argc, char *argv[])
         "wait_runqueue success");
     ok (wait_queue_length (q) == 0 && wait_queue_length (q2) == 0,
         "wait_runqueue dequeued wait_t from second queue");
+    ok (wait_queue_msgs_count (q) == 0 && wait_queue_msgs_count (q2) == 0,
+        "wait_runqueue dequeued wait_t from second queue, msgs count valid");
     ok (count == 1,
         "wait_t callback has run");
 
@@ -242,7 +260,8 @@ int main (int argc, char *argv[])
         snprintf (s, sizeof (s), "%d", i);
         if (!(msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)))
             break;
-        if (flux_msg_enable_route (msg) < 0 || flux_msg_push_route (msg, s) < 0)
+        flux_msg_route_enable (msg);
+        if (flux_msg_route_push (msg, s) < 0)
             break;
         if (!(w = wait_create_msg_handler (NULL, NULL, msg, &count, msghand)))
             break;
@@ -259,6 +278,8 @@ int main (int argc, char *argv[])
         "wait_destroy_msg found 3 matches");
     ok (wait_queue_length (q) == 17,
         "wait_queue_length 17 after 3 deletions");
+    ok (wait_queue_msgs_count (q) == 17,
+        "wait_queue_msgs_count 17 after 3 deletions");
     ok (count == 0,
         "wait_t callback has not run");
 
@@ -266,6 +287,8 @@ int main (int argc, char *argv[])
         "wait_destroy_msg found 17 matches");
     ok (wait_queue_length (q) == 0,
         "wait_queue_length 0 after 17 deletions");
+    ok (wait_queue_msgs_count (q) == 0,
+        "wait_queue_msgs_count 0 after 17 deletions");
     ok (count == 0,
         "wait_t callback has not run");
 
diff --git a/src/modules/kvs/treq.c b/src/modules/kvs/treq.c
index 732e8bea2b53..1f6532e390f3 100644
--- a/src/modules/kvs/treq.c
+++ b/src/modules/kvs/treq.c
@@ -17,11 +17,11 @@
 #include <errno.h>
 #include <unistd.h>
 #include <stdbool.h>
-#include <ctype.h>
-#include <czmq.h>
 #include <flux/core.h>
 #include <jansson.h>
+#include <assert.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/errno_safe.h"
 
 #include "treq.h"
@@ -52,7 +52,7 @@ treq_mgr_t *treq_mgr_create (void)
     int saved_errno;
 
     if (!(trm = calloc (1, sizeof (*trm)))) {
-        saved_errno = ENOMEM;
+        saved_errno = errno;
         goto error;
     }
     if (!(trm->transactions = zhash_new ())) {
@@ -143,15 +143,13 @@ int treq_mgr_iter_transactions (treq_mgr_t *trm, treq_itr_f cb, void *data)
 int treq_mgr_remove_transaction (treq_mgr_t *trm, const char *name)
 {
     /* it's dangerous to remove if we're in the middle of an
-     * interation, so save name for removal later.
+     * iteration, so save name for removal later.
      */
     if (trm->iterating_transactions) {
         char *str = strdup (name);
 
-        if (!str) {
-            errno = ENOMEM;
+        if (!str)
             return -1;
-        }
 
         if (zlist_append (trm->removelist, str) < 0) {
             free (str);
@@ -192,8 +190,11 @@ static treq_t *treq_create_common (int nprocs, int flags)
         saved_errno = EINVAL;
         goto error;
     }
-    if (!(tr = calloc (1, sizeof (*tr)))
-        || !(tr->ops = json_array ())
+    if (!(tr = calloc (1, sizeof (*tr)))) {
+        saved_errno = errno;
+        goto error;
+    }
+    if (!(tr->ops = json_array ())
         || !(tr->requests = zlist_new ())) {
         saved_errno = ENOMEM;
         goto error;
@@ -225,7 +226,7 @@ treq_t *treq_create (const char *name, int nprocs, int flags)
     }
 
     if (!(tr->name = strdup (name))) {
-        saved_errno = ENOMEM;
+        saved_errno = errno;
         goto error;
     }
 
@@ -236,7 +237,10 @@ treq_t *treq_create (const char *name, int nprocs, int flags)
     return NULL;
 }
 
-treq_t *treq_create_rank (uint32_t rank, unsigned int seq, int nprocs, int flags)
+treq_t *treq_create_rank (uint32_t rank,
+                          unsigned int seq,
+                          int nprocs,
+                          int flags)
 {
     treq_t *tr = NULL;
     int saved_errno;
@@ -247,7 +251,7 @@ treq_t *treq_create_rank (uint32_t rank, unsigned int seq, int nprocs, int flags
     }
 
     if (asprintf (&(tr->name), "treq.%u.%u", rank, seq) < 0) {
-        saved_errno = ENOMEM;
+        saved_errno = errno;
         goto error;
     }
 
diff --git a/src/modules/kvs/treq.h b/src/modules/kvs/treq.h
index 1fb879c4da9c..2cdccf4bcdcc 100644
--- a/src/modules/kvs/treq.h
+++ b/src/modules/kvs/treq.h
@@ -11,7 +11,6 @@
 #ifndef _FLUX_KVS_TREQ_H
 #define _FLUX_KVS_TREQ_H
 
-#include <czmq.h>
 #include <jansson.h>
 
 typedef struct treq_mgr treq_mgr_t;
@@ -55,7 +54,10 @@ int treq_mgr_transactions_count (treq_mgr_t *trm);
 treq_t *treq_create (const char *name, int nprocs, int flags);
 
 /* treq_create_rank - internally will create name based on rank & seq */
-treq_t *treq_create_rank (uint32_t rank, unsigned int seq, int nprocs, int flags);
+treq_t *treq_create_rank (uint32_t rank,
+                          unsigned int seq,
+                          int nprocs,
+                          int flags);
 
 void treq_destroy (treq_t *tr);
 
diff --git a/src/modules/kvs/waitqueue.c b/src/modules/kvs/waitqueue.c
index 9b0f45ebda2f..72162a32d52a 100644
--- a/src/modules/kvs/waitqueue.c
+++ b/src/modules/kvs/waitqueue.c
@@ -11,8 +11,10 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
 #include <flux/core.h>
+#include <assert.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "waitqueue.h"
 
@@ -40,6 +42,9 @@ struct wait_struct {
 struct waitqueue_struct {
     int magic;
     zlist_t *q;
+    /* special counter, count entries on 'q' that have messages,
+     * i.e. w->hand.msg */
+    int msgs_on_queue;
 };
 
 int wait_get_usecount (wait_t *w)
@@ -50,18 +55,18 @@ int wait_get_usecount (wait_t *w)
 wait_t *wait_create (wait_cb_f cb, void *arg)
 {
     wait_t *w = calloc (1, sizeof (*w));
-    if (!w) {
-        errno = ENOMEM;
+    if (!w)
         return NULL;
-    }
     w->magic = WAIT_MAGIC;
     w->cb = cb;
     w->cb_arg = arg;
     return w;
 }
 
-wait_t *wait_create_msg_handler (flux_t *h, flux_msg_handler_t *mh,
-                                 const flux_msg_t *msg, void *arg,
+wait_t *wait_create_msg_handler (flux_t *h,
+                                 flux_msg_handler_t *mh,
+                                 const flux_msg_t *msg,
+                                 void *arg,
                                  flux_msg_handler_f cb)
 {
     wait_t *w = wait_create (NULL, NULL);
@@ -86,7 +91,9 @@ void wait_destroy (wait_t *w)
     }
 }
 
-int wait_msg_aux_set (wait_t *w, const char *name, void *aux,
+int wait_msg_aux_set (wait_t *w,
+                      const char *name,
+                      void *aux,
                       flux_free_f destroy)
 {
     if (w && w->hand.msg)
@@ -104,10 +111,8 @@ void *wait_msg_aux_get (wait_t *w, const char *name)
 waitqueue_t *wait_queue_create (void)
 {
     waitqueue_t *q = calloc (1, sizeof (*q));
-    if (!q) {
-        errno = ENOMEM;
+    if (!q)
         return NULL;
-    }
     if (!(q->q = zlist_new ())) {
         free (q);
         errno = ENOMEM;
@@ -138,6 +143,12 @@ int wait_queue_length (waitqueue_t *q)
     return zlist_size (q->q);
 }
 
+int wait_queue_msgs_count (waitqueue_t *q)
+{
+    assert (q->magic == WAITQUEUE_MAGIC);
+    return q->msgs_on_queue;
+}
+
 int wait_queue_iter (waitqueue_t *q, wait_iter_cb_f cb, void *arg)
 {
     wait_t *w;
@@ -161,6 +172,8 @@ int wait_addqueue (waitqueue_t *q, wait_t *w)
         return -1;
     }
     w->usecount++;
+    if (w->hand.msg)
+        q->msgs_on_queue++;
     return 0;
 }
 
@@ -197,8 +210,11 @@ int wait_runqueue (waitqueue_t *q)
             return -1;
         }
         zlist_purge (q->q);
-        while ((w = zlist_pop (cpy)))
+        while ((w = zlist_pop (cpy))) {
+            if (w->hand.msg)
+                q->msgs_on_queue--;
             wait_runone (w);
+        }
         zlist_destroy (&cpy);
     }
     return 0;
@@ -263,6 +279,7 @@ int wait_destroy_msg (waitqueue_t *q, wait_test_msg_f cb, void *arg)
     if (tmp) {
         while ((w = zlist_pop (tmp))) {
             zlist_remove (q->q, w);
+            q->msgs_on_queue--;
             if (--w->usecount == 0)
                 wait_destroy (w);
         }
diff --git a/src/modules/kvs/waitqueue.h b/src/modules/kvs/waitqueue.h
index 1ee1707f146e..684dc5783642 100644
--- a/src/modules/kvs/waitqueue.h
+++ b/src/modules/kvs/waitqueue.h
@@ -46,6 +46,11 @@ int wait_get_usecount (wait_t *wait);
 waitqueue_t *wait_queue_create (void);
 void wait_queue_destroy (waitqueue_t *q);
 int wait_queue_length (waitqueue_t *q);
+/* Special counter indicating queue entries with msgs created with
+ * wait_create_msg_handler().  Useful if user wishes to avoid excess
+ * iteration calls on wait_destroy_msg().
+ */
+int wait_queue_msgs_count (waitqueue_t *q);
 
 /* Add a wait_t to a queue.
  * Returns -1 on error, 0 on success
@@ -68,14 +73,19 @@ int wait_runqueue (waitqueue_t *q);
  * The message handler will be reinvoked once the wait_t usecount reaches zero.
  * Message will be copied and destroyed with the wait_t.
  */
-wait_t *wait_create_msg_handler (flux_t *h, flux_msg_handler_t *mh,
-                                 const flux_msg_t *msg, void *arg,
+wait_t *wait_create_msg_handler (flux_t *h,
+                                 flux_msg_handler_t *mh,
+                                 const flux_msg_t *msg,
+                                 void *arg,
                                  flux_msg_handler_f cb);
 
 /* Set/get auxiliary data to the flux message stored in a wait_t */
-int wait_msg_aux_set (wait_t *w, const char *name, void *aux,
+int wait_msg_aux_set (wait_t *w,
+                      const char *name,
+                      void *aux,
                       flux_free_f destroy);
-void *wait_msg_aux_get (wait_t *w, const char *name);
+void *wait_msg_aux_get (wait_t *w,
+                        const char *name);
 
 /* Get/set an aux errnum on a wait that can be retrieved later.
  * In addition, a callback can be set which can be triggered
diff --git a/src/modules/pymod/Makefile.am b/src/modules/pymod/Makefile.am
deleted file mode 100644
index 1b6ad5fe0c81..000000000000
--- a/src/modules/pymod/Makefile.am
+++ /dev/null
@@ -1,35 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	-DPYTHON_LIBRARY=\"$(PYTHON_LIBRARY)\" \
-	$(ZMQ_CFLAGS) \
-	$(PYTHON_CPPFLAGS) \
-	"-DFLUX_PYTHON_PATH=\"${pyexecdir}\""
-
-#
-# Comms module
-#
-fluxmod_LTLIBRARIES = pymod.la
-
-pymod_la_SOURCES = py_mod.c
-pymod_la_LDFLAGS = $(fluxmod_ldflags) -module
-
-# allow pymod to find the configured libpython at runtime
-pymod_la_LDFLAGS += -Wl,-rpath -Wl,$(PYTHON_PREFIX)/lib
-
-pymod_la_LIBADD = $(top_builddir)/src/common/libflux-core.la \
-		  $(top_builddir)/src/common/libflux-internal.la \
-		  $(top_builddir)/src/common/libflux-optparse.la \
-		  $(ZMQ_LIBS) \
-		  $(PYTHON_LDFLAGS)
-
-fluxpymod_PYTHON = echo.py __init__.py
-
diff --git a/src/modules/pymod/echo.py b/src/modules/pymod/echo.py
deleted file mode 100644
index f0be35545485..000000000000
--- a/src/modules/pymod/echo.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import syslog
-import flux
-
-
-def echo_cb(h, typemask, message, arg):
-    h.log(syslog.LOG_INFO, "in cb, args:{}".format((typemask, message, arg)))
-    h.respond(message, message.payload_str)
-    return 0
-
-
-def mod_main(h, *args):
-    with h.msg_watcher_create(echo_cb, topic_glob="echo.*") as mw:
-        if h.reactor_run(h.get_reactor(), 0) < 0:
-            h.fatal_error("reactor start failed!")
-        h.log(syslog.LOG_INFO, "echo unloading")
diff --git a/src/modules/pymod/py_mod.c b/src/modules/pymod/py_mod.c
deleted file mode 100644
index 5a578d369f34..000000000000
--- a/src/modules/pymod/py_mod.c
+++ /dev/null
@@ -1,204 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <Python.h>
-#include <stdio.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <string.h>
-#include <wchar.h>
-#include <errno.h>
-#include <dlfcn.h>
-#include <flux/core.h>
-#include <czmq.h>
-
-#include "src/common/libutil/log.h"
-#include "src/common/liboptparse/optparse.h"
-#include "src/common/libutil/xzmalloc.h"
-
-#ifndef Py_PYTHON_H
-typedef void PyObject;
-#endif
-
-PyObject * py_unicode_or_string(const char *str) {
-#if PY_MAJOR_VERSION >= 3
-    return PyUnicode_FromString(str);
-#else
-    return PyString_FromString(str);
-#endif
-}
-
-void add_if_not_present(PyObject *list, const char* path){
-    if(path){
-        PyObject *pymod_path = py_unicode_or_string(path);
-        if (!PySequence_Contains(list, pymod_path)){
-            PyList_Append(list, pymod_path);
-        }else{
-            Py_DECREF(pymod_path);
-        }
-    }
-}
-
-void print_usage(){
-    printf("pymod usage: flux module load pymod --module=<modname> [--path=<module path>] [--verbose] [--help]]\n");
-}
-
-zhash_t *zhash_fromargv (int argc, char **argv)
-{
-    zhash_t *args = zhash_new ();
-    int i;
-
-    if (args) {
-        for (i = 0; i < argc; i++) {
-            char *key = xstrdup (argv[i]);
-            char *val = strchr (key, '=');
-            if (val) {
-                *val++ = '\0';
-                zhash_update (args, key, xstrdup (val));
-                zhash_freefn (args, key, free);
-            }
-            free (key);
-        }
-    }
-    return args;
-}
-
-const char *usage_msg = "[OPTIONS] MODULE_NAME";
-static struct optparse_option opts[] = {
-    { .name = "verbose",    .key = 'v', .has_arg = 0,
-      .usage = "Be loud", },
-    { .name = "path",     .key = 'p', .has_arg = 1, .arginfo = "PATH",
-      .usage = "Director{y,ies} to add to PYTHONPATH before finding your module", },
-    OPTPARSE_TABLE_END,
-};
-
-static int register_pymod_service_name (flux_t *h, const char *name)
-{
-    flux_future_t *f;
-    int saved_errno = 0;
-    int rc = -1;
-
-    /* Register a service name based on the name of the loaded script
-     */
-    if (!(f = flux_service_register (h, name))) {
-        saved_errno = errno;
-        flux_log_error (h, "service.add: flux_rpc_pack");
-        goto done;
-    }
-    if ((rc = flux_future_get (f, NULL)) < 0) {
-        saved_errno = errno;
-        flux_log_error (h, "service.add: %s", name);
-        goto done;
-    }
-done:
-    flux_future_destroy (f);
-    errno = saved_errno;
-    return rc;
-}
-
-// Based on code from https://bugs.python.org/issue17870
-PyObject* PyLong_FromUintptr_t(uintptr_t value)
-{
-    if (sizeof(uintptr_t) == sizeof(long)) {
-        return PyLong_FromLong(value);
-    } else if (sizeof(uintptr_t) <= sizeof(PY_LONG_LONG)) {
-        return PyLong_FromLongLong((PY_LONG_LONG)value);
-    } else {
-        return NULL;
-    }
-}
-
-int mod_main (flux_t *h, int argc, char **argv)
-{
-    optparse_t *p = optparse_create ("pymod");
-    if (optparse_add_option_table (p, opts) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_add_option_table");
-    if (optparse_set (p, OPTPARSE_USAGE, usage_msg) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_set usage");
-    int option_index = optparse_parse_args (p, argc, argv);
-
-    if (option_index <= 0 || optparse_hasopt(p, "help") || option_index >= argc){
-        optparse_print_usage(p);
-        return (option_index < 0);
-    }
-    const char * module_name = argv[option_index];
-
-#if PY_MAJOR_VERSION >= 3
-    wchar_t *program = L"pymod";
-#else
-    char *program = "pymod";
-#endif
-    Py_SetProgramName(program);
-    Py_Initialize();
-
-    PyObject *search_path = PySys_GetObject("path");
-    // Add installation search paths
-    add_if_not_present(search_path, optparse_get_str(p, "path", ""));
-    add_if_not_present(search_path, FLUX_PYTHON_PATH);
-
-    PySys_SetObject("path", search_path);
-    if(optparse_hasopt(p, "verbose")){
-        PyObject_Print(search_path, stderr, 0);
-    }
-
-    flux_log(h, LOG_INFO, "loading python module named: %s", module_name);
-    if (!dlopen (PYTHON_LIBRARY, RTLD_LAZY|RTLD_GLOBAL))
-        flux_log_error (h, "Unable to dlopen libpython");
-
-    PyObject *module = PyImport_ImportModule("flux.core.trampoline");
-    if(!module){
-        PyErr_Print();
-        return EINVAL;
-    }
-    if (register_pymod_service_name (h, module_name) < 0)
-        return -1;
-
-    PyObject *mod_main = PyObject_GetAttrString(module, "mod_main_trampoline");
-    if(mod_main && PyCallable_Check(mod_main)){
-        //maybe unpack args directly? probably easier to use a dict
-        PyObject *py_args = PyTuple_New(3);
-        PyObject *pystr_mod_name = py_unicode_or_string(module_name);
-        PyTuple_SetItem(py_args, 0, pystr_mod_name);
-        PyObject *py_flux_handle = PyLong_FromUintptr_t((uintptr_t)h);
-        if (py_flux_handle == NULL)
-          return -1;
-        PyTuple_SetItem(py_args, 1, py_flux_handle);
-
-        //Convert zhash to native python dict, should preserve mods
-        //through switch to argc-style arguments
-        PyObject *arg_list = PyList_New(0);
-        char ** it = argv + option_index;
-        int i;
-        for (i=0; *it; i++, it++){
-            PyList_Append(arg_list, py_unicode_or_string(*it));
-        }
-
-        PyTuple_SetItem(py_args, 2, arg_list);
-        // Call into trampoline
-        PyObject_CallObject(mod_main, py_args);
-        if(PyErr_Occurred()){
-            PyErr_Print();
-        }
-        Py_DECREF(py_args);
-        Py_DECREF(arg_list);
-    }
-    Py_Finalize();
-    return 0;
-}
-
-MOD_NAME ("pymod");
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/modules/resource/Makefile.am b/src/modules/resource/Makefile.am
new file mode 100644
index 000000000000..526a8e7a4cc6
--- /dev/null
+++ b/src/modules/resource/Makefile.am
@@ -0,0 +1,80 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
+	-I$(top_builddir)/src/common/libflux \
+	$(JANSSON_CFLAGS) \
+	$(HWLOC_CFLAGS)
+
+noinst_LTLIBRARIES = libresource.la
+
+libresource_la_SOURCES = \
+	resource.c \
+	resource.h \
+	monitor.c \
+	monitor.h \
+	topo.c \
+	topo.h \
+	drain.c \
+	drain.h \
+	exclude.c \
+	exclude.h \
+	reslog.c \
+	reslog.h \
+	acquire.c \
+	acquire.h \
+	inventory.c \
+	inventory.h \
+	rutil.c \
+	rutil.h \
+	status.c \
+	status.h \
+	drainset.h \
+	drainset.c \
+	upgrade.h \
+	upgrade.c
+
+TESTS = \
+	test_rutil.t \
+	test_drainset.t
+
+test_ldadd = \
+	$(builddir)/libresource.la \
+	$(top_builddir)/src/common/librlist/librlist.la \
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(HWLOC_LIBS) \
+	$(JANSSON_LIBS) \
+	$(LIBPTHREAD)
+
+test_ldflags = \
+        -no-install
+
+test_cppflags = \
+        $(AM_CPPFLAGS) \
+	$(JANSSON_CFLAGS)
+
+check_PROGRAMS = $(TESTS)
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+       $(top_srcdir)/config/tap-driver.sh
+
+test_rutil_t_SOURCES = test/rutil.c
+test_rutil_t_CPPFLAGS = $(test_cppflags)
+test_rutil_t_LDADD = $(test_ldadd)
+test_rutil_t_LDFLAGS = $(test_ldflags)
+
+test_drainset_t_SOURCES = test/drainset.c
+test_drainset_t_CPPFLAGS = $(test_cppflags)
+test_drainset_t_LDADD = $(test_ldadd)
+test_drainset_t_LDFLAGS = $(test_ldflags)
diff --git a/src/modules/resource/acquire.c b/src/modules/resource/acquire.c
new file mode 100644
index 000000000000..a23261ada800
--- /dev/null
+++ b/src/modules/resource/acquire.c
@@ -0,0 +1,510 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* acquire.c - let schedulers acquire resources and monitor their availability
+ *
+ * PROTOCOL
+ *
+ * Scheduler makes resource.acquire RPC.  Streaming responses are of the form:
+ *
+ * First response:
+ *   {resources:resource_object up:idset}
+ * Subsequent responses:
+ *   {up?:idset down?:idset}
+ *
+ * Where:
+ * - resource_object maps execution target ids to resources
+ *   (see RESOURCE OBJECT) below
+ * - idset is a set of execution target ids, encoded as a string.
+ *
+ * Execution targets that are excluded by configuration are omitted from
+ * resource_object in the initial response.  Targets should be considered
+ * "down" until they appear as a member of an "up" idset.
+ *
+ * As execution targets from the resource_object go online or are undrained,
+ * they are marked "up".  As they go offline or are drained, they are marked
+ * "down".
+ *
+ * If the exclusion configuration changes, any newly excluded execution
+ * targets from the resource_object are marked "down".  On the next
+ * scheduler reload, the resource set will omit those targets.
+ *
+ * RESOURCE OBJECT
+ *
+ * The Rv1 format described in RFC 20 is used.
+ *
+ * LIMITATIONS
+ *
+ * Currently, only a single resource.acquire RPC is allowed to be pending
+ * at a time.  Upon scheduler unload, the automatically generated disconnect
+ * request frees up this slot.  If a scheduler wishes to terminate the RPC
+ * sooner, it may send a resource.acquire-cancel RPC containing the matchtag
+ * of the resource.acquire RPC.  Per RFC 6, the former does not receive a
+ * response, and the latter receives a (terminating) ECANCELED response.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libidset/idset.h"
+#include "src/common/librlist/rlist.h"
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
+
+#include "resource.h"
+#include "reslog.h"
+#include "inventory.h"
+#include "exclude.h"
+#include "drain.h"
+#include "acquire.h"
+#include "monitor.h"
+#include "rutil.h"
+
+/* Stored as aux item in request message.
+ */
+struct acquire_request {
+    int response_count;                 // count of response messages sent
+    json_t *resources;                  // resource object
+    struct idset *valid;                // valid targets
+    struct idset *up;                   // available targets
+};
+
+struct acquire {
+    struct resource_ctx *ctx;
+    flux_msg_handler_t **handlers;
+    struct flux_msglist *requests;      // N.B. there can be only one currently
+    bool mute;                          // suspend responses during shutdown
+};
+
+
+static void acquire_request_destroy (struct acquire_request *ar)
+{
+    if (ar) {
+        int saved_errno = errno;
+        json_decref (ar->resources);
+        idset_destroy (ar->valid);
+        idset_destroy (ar->up);
+        free (ar);
+        errno = saved_errno;
+    }
+}
+
+/* Initialize request context once resource object is available.
+ * This may be called from acquire_cb() or reslog_cb().
+ */
+static int acquire_request_init (struct acquire_request *ar,
+                                 struct acquire *acquire,
+                                 json_t *resobj)
+{
+    struct resource_ctx *ctx = acquire->ctx;
+    const struct idset *exclude = exclude_get (ctx->exclude);
+    json_error_t e;
+    struct rlist *rl;
+    struct idset *drain = NULL;
+
+    if (resobj == NULL || !(rl = rlist_from_json (resobj, &e))) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (exclude && idset_count (exclude) > 0) {
+        (void)rlist_remove_ranks (rl, (struct idset *)exclude);
+        if (!(ar->resources = rlist_to_R (rl))) {
+            errno = ENOMEM;
+            goto error;
+        }
+    }
+    else {
+        if (!(ar->resources = json_copy (resobj)))
+            goto nomem;
+    }
+    if (!(ar->valid = rlist_ranks (rl))) // excluded ranks are not valid
+        goto nomem;
+    if (!(ar->up = idset_copy (ar->valid))) // and up omits excluded ranks
+        goto error;
+    if (!(drain = drain_get (ctx->drain)))
+        goto error;
+    if (idset_subtract (ar->up, drain) < 0)
+        goto error;
+    if (idset_subtract (ar->up, monitor_get_down (ctx->monitor)) < 0)
+        goto error;
+    if (idset_subtract (ar->up, monitor_get_torpid (ctx->monitor)) < 0)
+        goto error;
+    rlist_destroy (rl);
+    idset_destroy (drain);
+    return 0;
+nomem:
+    errno = ENOMEM;
+error:
+    rlist_destroy (rl);
+    idset_destroy (drain);
+    return -1;
+}
+
+/* reslog_cb() says 'name' event occurred.
+ * If anything changed with respect to target availability, populate
+ * up and/or down idsets with the changes.
+ * Replace ar->up with new set of available targets.
+ */
+static int acquire_request_update (struct acquire_request *ar,
+                                   struct acquire *acquire,
+                                   const char *name,
+                                   struct idset **up,
+                                   struct idset **dn)
+{
+    struct resource_ctx *ctx = acquire->ctx;
+    struct idset *new_up;
+    struct idset *drain = NULL;
+
+    if (!(new_up = idset_copy (ar->valid)))
+        return -1;
+    if (!(drain = drain_get (ctx->drain)))
+        goto error;
+    if (idset_subtract (new_up, drain) < 0)
+        goto error;
+    if (idset_subtract (new_up, monitor_get_down (ctx->monitor)) < 0)
+        goto error;
+    if (idset_subtract (new_up, monitor_get_torpid (ctx->monitor)) < 0)
+        goto error;
+    if (idset_subtract (new_up, exclude_get (ctx->exclude)) < 0)
+        goto error;
+    if (rutil_idset_diff (ar->up, new_up, up, dn) < 0)
+        goto error;
+    idset_destroy (ar->up);
+    ar->up = new_up;
+    idset_destroy (drain);
+    return 0;
+error:
+    idset_destroy (new_up);
+    idset_destroy (drain);
+    return -1;
+}
+
+/* Send the first response to resource.acquire request.  This presumes
+ * that acquire_request_init() has already prepared ar->resources and ar->up.
+ */
+static int acquire_respond_first (flux_t *h, const flux_msg_t *msg)
+{
+    struct acquire_request *ar = flux_msg_aux_get (msg, "acquire");
+    json_t *o = NULL;
+
+    if (!(o = json_object()))
+        goto nomem;
+    if (json_object_set (o, "resources", ar->resources) < 0)
+        goto nomem;
+    if (rutil_set_json_idset (o, "up", ar->up) < 0)
+        goto error;
+    if (flux_respond_pack (h, msg, "O", o) < 0)
+        goto error;
+    json_decref (o);
+    ar->response_count++;
+    return 0;
+nomem:
+    errno = ENOMEM;
+error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return -1;
+}
+
+/* Send a subsequent response to resource.acquire request, driven by
+ * reslog_cb().
+ */
+static int acquire_respond_next (flux_t *h,
+                                 const flux_msg_t *msg,
+                                 struct idset *up,
+                                 struct idset *down)
+{
+    struct acquire_request *ar = flux_msg_aux_get (msg, "acquire");
+    json_t *o;
+
+    if (!(o = json_object()))
+        goto nomem;
+    if (up && rutil_set_json_idset (o, "up", up) < 0)
+        goto error;
+    if (down && rutil_set_json_idset (o, "down", down) < 0)
+        goto error;
+    if (flux_respond_pack (h, msg, "O", o) < 0)
+        goto error;
+    json_decref (o);
+    ar->response_count++;
+    return 0;
+nomem:
+    errno = ENOMEM;
+error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return -1;
+}
+
+/* Handle a resource.acquire request.
+ * Currently there is only one request slot.
+ * The response is deferred until resources are available.
+ */
+static void acquire_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
+{
+    struct acquire *acquire = arg;
+    struct acquire_request *ar;
+    json_t *resobj;
+
+    if (!(ar = calloc (1, sizeof (*ar))))
+        goto error;
+    if (flux_request_decode (msg, NULL, NULL) < 0
+        || flux_msg_aux_set (msg,
+                             "acquire",
+                             ar,
+                             (flux_free_f)acquire_request_destroy) < 0) {
+        acquire_request_destroy (ar);
+        goto error;
+    }
+    if (flux_msglist_count (acquire->requests) == 1) {
+        errno = EBUSY;
+        goto error;
+    }
+    if (flux_msglist_append (acquire->requests, msg) < 0)
+        goto error;
+    if (!(resobj = inventory_get (acquire->ctx->inventory)))
+        return; // defer response until resource-define event
+
+    if (acquire_request_init (ar, acquire, resobj) < 0)
+        goto error;
+    if (acquire_respond_first (h, msg) < 0)
+        flux_log_error (h, "error responding to acquire request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to acquire request");
+}
+
+/* Handle resource.acquire-cancel request.
+ */
+static void cancel_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    struct acquire *acquire = arg;
+    int count;
+
+    if ((count = flux_msglist_cancel (h, acquire->requests, msg)) < 0)
+        flux_log_error (h, "error handling discnonect request");
+    if (count > 0)
+        flux_log (h, LOG_DEBUG, "canceled %d resource.acquire", count);
+}
+
+/* Suspend resource.acquire responses during shutdown
+ */
+static void mute_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    struct acquire *acquire = arg;
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    acquire->mute = true;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to acquire-mute request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to acquire-mute request");
+}
+
+/* Handle resource.disconnect message.
+ */
+void acquire_disconnect (struct acquire *acquire, const flux_msg_t *msg)
+{
+    if (acquire) { // acquire is NULL on rank > 0
+        flux_t *h = acquire->ctx->h;
+        int count;
+
+        if ((count = flux_msglist_disconnect (acquire->requests, msg)) < 0)
+            flux_log_error (h, "error handling discnonect request");
+        if (count > 0)
+            flux_log (h, LOG_DEBUG, "aborted %d resource.acquire(s)", count);
+    }
+}
+
+/* An event was committed to resource.eventlog.
+ * Generate response to acquire requests as appropriate.
+ * FWIW, this function is not called until after the eventlog KVS
+ * commit completes.
+ */
+static void reslog_cb (struct reslog *reslog,
+                       const char *name,
+                       json_t *context,
+                       void *arg)
+{
+    struct acquire *acquire = arg;
+    struct resource_ctx *ctx = acquire->ctx;
+    flux_t *h = ctx->h;
+    const char *errmsg = NULL;
+    json_t *resobj;
+    const flux_msg_t *msg;
+
+    if (acquire->mute)
+        return;
+
+    msg = flux_msglist_first (acquire->requests);
+    while (msg) {
+        struct acquire_request *ar = flux_msg_aux_get (msg, "acquire");
+
+        if (streq (name, "resource-define")) {
+            if (ar->response_count == 0) {
+                if (!(resobj = inventory_get (ctx->inventory))) {
+                    errmsg = "resource discovery failed or interrupted";
+                    errno = ENOENT;
+                    goto error;
+                }
+                if (acquire_request_init (ar, acquire, resobj) < 0) {
+                    errmsg = "error preparing first resource.acquire response";
+                    goto error;
+
+                }
+                if (acquire_respond_first (h, msg) < 0) {
+                    flux_log_error (h,
+                                    "error responding to resource.acquire (%s)",
+                                    name);
+                }
+            }
+        }
+        else if (streq (name, "resource-update")) {
+            double expiration = -1.;
+
+            /*  Handle resource-update event. Currently the only supported
+             *  context of such an event is an expiration update
+             */
+            if (json_unpack (context,
+                             "{s?F}",
+                             "expiration", &expiration) < 0) {
+                errmsg = "error preparing resource.acquire update response";
+                goto error;
+            }
+            if (expiration >= 0.
+                && flux_respond_pack (h,
+                                      msg,
+                                      "{s:f}",
+                                      "expiration", expiration) < 0) {
+                    flux_log_error (h,
+                                    "error responding to resource.acquire (%s)",
+                                    name);
+                    goto error;
+            }
+        }
+        else if (streq (name, "online")
+                 || streq (name, "offline")
+                 || streq (name, "drain")
+                 || streq (name, "undrain")
+                 || streq (name, "torpid")
+                 || streq (name, "lively")) {
+            if (ar->response_count > 0) {
+                struct idset *up, *dn;
+                if (acquire_request_update (ar,
+                                            acquire,
+                                            name,
+                                            &up,
+                                            &dn) < 0) {
+                    errmsg = "error preparing resource.acquire update response";
+                    goto error;
+                }
+                if (up || dn) {
+                    if (acquire_respond_next (h,
+                                              msg,
+                                              up,
+                                              dn) < 0) {
+                        flux_log_error (h,
+                                    "error responding to resource.acquire (%s)",
+                                    name);
+                    }
+                }
+                idset_destroy (up);
+                idset_destroy (dn);
+            }
+        }
+        msg = flux_msglist_next (acquire->requests);
+    }
+
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to acquire request");
+    flux_msglist_delete (acquire->requests);
+}
+
+int acquire_clients (struct acquire *acquire)
+{
+    return flux_msglist_count (acquire->requests);
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,  "resource.acquire", acquire_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST,  "resource.acquire-cancel", cancel_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST,  "resource.acquire-mute", mute_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void acquire_destroy (struct acquire *acquire)
+{
+    if (acquire) {
+        int saved_errno = errno;
+
+        flux_msg_handler_delvec (acquire->handlers);
+        reslog_remove_callback (acquire->ctx->reslog, reslog_cb, acquire);
+
+        if (acquire->requests) {
+            const flux_msg_t *msg;
+            flux_t *h = acquire->ctx->h;
+
+            msg = flux_msglist_first (acquire->requests);
+            while (msg) {
+                if (flux_respond_error (h,
+                                        msg,
+                                        ECANCELED,
+                                        "the resource module was unloaded") < 0)
+                    flux_log_error (h, "error responding to acquire request");
+                flux_msglist_delete (acquire->requests);
+                msg = flux_msglist_next (acquire->requests);
+            }
+            flux_msglist_destroy (acquire->requests);
+        }
+        free (acquire);
+        errno = saved_errno;
+    }
+}
+
+struct acquire *acquire_create (struct resource_ctx *ctx)
+{
+    struct acquire *acquire;
+
+    if (!(acquire = calloc (1, sizeof (*acquire))))
+        return NULL;
+    acquire->ctx = ctx;
+    if (!(acquire->requests = flux_msglist_create ()))
+        goto error;
+    if (flux_msg_handler_addvec (ctx->h,
+                                 htab,
+                                 acquire,
+                                 &acquire->handlers) < 0)
+        goto error;
+    if (reslog_add_callback (ctx->reslog, reslog_cb, acquire) < 0)
+        goto error;
+    return acquire;
+error:
+    acquire_destroy (acquire);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/acquire.h b/src/modules/resource/acquire.h
new file mode 100644
index 000000000000..ffb66059e328
--- /dev/null
+++ b/src/modules/resource/acquire.h
@@ -0,0 +1,25 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_ACQUIRE_H
+#define _FLUX_RESOURCE_ACQUIRE_H
+
+struct acquire *acquire_create (struct resource_ctx *ctx);
+void acquire_destroy (struct acquire *acquire);
+
+void acquire_disconnect (struct acquire *acquire, const flux_msg_t *msg);
+int acquire_clients (struct acquire *acquire);
+
+#endif /* !_FLUX_RESOURCE_ACQUIRE_H */
+
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/drain.c b/src/modules/resource/drain.c
new file mode 100644
index 000000000000..ca96c4495618
--- /dev/null
+++ b/src/modules/resource/drain.c
@@ -0,0 +1,807 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* drain.c - handle drain/undrain requests
+ *
+ * Drained execution targets should be temporarily excluded from scheduling,
+ * but may be used for determining job request satisfiability.
+ *
+ * Handle RPCs from front-end commands.
+ * - if a node in undrain target is not drained, request fails
+ * - if a node in undrain target is excluded, request fails
+ * - if a node in drain target is already drained, request status depends
+ *   on setting of optional 'mode' member:
+ *    - If mode is not set, request fails
+ *    - If mode=overwrite, request succeeds and reason is updated
+ *    - If mode=force-overwrite, request succeeds and timestamp and reason
+ *      are updated
+ *    - If mode=update, request succeeds and reason is updated only for
+ *      those target that are not drained or do not have reason set.
+ *
+ * Post events for each drain/undrain action.  Drain state is sticky
+ * across module reload / instance restart.  The state is reacquired
+ * by replaying the eventlog.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <time.h>
+#include <math.h>
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libidset/idset.h"
+#include "src/common/libhostlist/hostlist.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "ccan/str/str.h"
+
+#include "resource.h"
+#include "reslog.h"
+#include "exclude.h"
+#include "drain.h"
+#include "rutil.h"
+#include "inventory.h"
+#include "drainset.h"
+
+struct draininfo {
+    bool drained;
+    double timestamp;
+    char *reason;
+};
+
+struct drain {
+    struct resource_ctx *ctx;
+    struct draininfo *info; // rank-indexed array [0:size-1]
+    flux_msg_handler_t **handlers;
+};
+
+struct drain_init_args {
+    struct drain *drain;
+    const struct idset *exclude;
+};
+
+static int get_timestamp_now (double *timestamp)
+{
+    struct timespec ts;
+    if (clock_gettime (CLOCK_REALTIME, &ts) < 0)
+        return -1;
+    *timestamp = (1E-9 * ts.tv_nsec) + ts.tv_sec;
+    return 0;
+}
+
+static int update_draininfo_rank (struct drain *drain,
+                                  unsigned int rank,
+                                  bool drained,
+                                  double timestamp,
+                                  const char *reason,
+                                  int overwrite)
+{
+    char *cpy = NULL;
+
+    if (rank >= drain->ctx->size) {
+        errno = EINVAL;
+        return -1;
+    }
+    /*  Skip rank if it is already drained with an existing reason
+     *   and the overwrite flag is not set.
+     */
+    if (!overwrite
+        && drain->info[rank].drained
+        && drain->info[rank].reason)
+        return 0;
+
+    if (reason && !(cpy = strdup (reason)))
+        return -1;
+
+    free (drain->info[rank].reason);
+    drain->info[rank].reason = cpy;
+    if (drain->info[rank].drained != drained || overwrite == 2) {
+        drain->info[rank].drained = drained;
+        drain->info[rank].timestamp = timestamp;
+    }
+    return 0;
+}
+
+static int update_draininfo_idset (struct drain *drain,
+                                   struct idset *idset,
+                                   bool drained,
+                                   double timestamp,
+                                   const char *reason,
+                                   int overwrite)
+{
+    unsigned int rank;
+
+    rank = idset_first (idset);
+    while (rank != IDSET_INVALID_ID) {
+        if (update_draininfo_rank (drain,
+                                   rank,
+                                   drained,
+                                   timestamp,
+                                   reason,
+                                   overwrite) < 0)
+            return -1;
+        rank = idset_next (idset, rank);
+    }
+    return 0;
+}
+
+/*  Check if all targets in idset are either not drained, or do
+ *   not have a reason currently set. If one or more ranks do
+ *   not meet this criteria then the function returns -1 and
+ *   calls out the ranks in the returned errstr.
+ */
+static int check_draininfo_idset (struct drain *drain,
+                                  struct idset *idset,
+                                  flux_error_t *errp)
+{
+    int rc = 0;
+    unsigned int rank;
+    bool was_excluded = false;
+    bool was_drained = false;
+    const struct idset *exclude;
+    struct idset *errids = idset_create (0, IDSET_FLAG_AUTOGROW);
+
+    errp->text[0] = '\0';
+
+    if (!errids)
+        return -1;
+    exclude = exclude_get (drain->ctx->exclude);
+
+    rank = idset_first (idset);
+    while (rank != IDSET_INVALID_ID) {
+        bool is_error = false;
+        if (idset_test (exclude, rank)) {
+            was_excluded = true;
+            is_error = true;
+        }
+        if (drain->info[rank].drained && drain->info[rank].reason) {
+            was_drained = true;
+            is_error = true;
+        }
+        if (is_error) {
+            rc = -1;
+            if (idset_set (errids, rank) < 0)
+                flux_log_error (drain->ctx->h,
+                                "check_draininfo_idset: idset_set(%d)",
+                                rank);
+        }
+        rank = idset_next (idset, rank);
+    }
+    if (rc < 0) {
+        char *s;
+        int n = idset_count (errids);
+
+        if (!(s = idset_encode (errids, IDSET_FLAG_RANGE)))
+            flux_log_error (drain->ctx->h,
+                            "check_draininfo_idset: idset_encode");
+        errprintf (errp,
+                   "rank%s %s %s%s%s",
+                   n > 1 ? "s" : "",
+                   s ? s : "(unknown)",
+                   was_drained ? "already drained" : "",
+                   was_drained && was_excluded ? " or " : "",
+                   was_excluded ? "excluded" : "");
+        free (s);
+
+        /*  If any node was drained, then return EEXIST as a hint of this
+         *  fact. Otherwise, an attempt to drain an excluded node was made,
+         *  and that is invalid, so return EINVAL.
+         */
+        errno = was_drained ? EEXIST : EINVAL;
+    }
+    idset_destroy (errids);
+    return rc;
+}
+
+json_t *drain_get_info (struct drain *drain)
+{
+    json_t *o = NULL;
+    struct drainset *ds = drainset_create ();
+    if (!ds)
+        goto error;
+    for (unsigned int rank = 0; rank < drain->ctx->size; rank++) {
+        if (drain->info[rank].drained) {
+            if (drainset_drain_rank (ds,
+                                     rank,
+                                     round(drain->info[rank].timestamp),
+                                     drain->info[rank].reason) < 0)
+                goto error;
+        }
+    }
+    o = drainset_to_json (ds);
+error:
+    drainset_destroy (ds);
+    return o;
+}
+
+struct idset *drain_get (struct drain *drain)
+{
+    unsigned int rank;
+    struct idset *ids;
+
+    if (!(ids = idset_create (drain->ctx->size, 0)))
+        return NULL;
+    for (rank = 0; rank < drain->ctx->size; rank++) {
+        if (drain->info[rank].drained) {
+            if (idset_set (ids, rank) < 0) {
+                idset_destroy (ids);
+                return NULL;
+            }
+        }
+    }
+    return ids;
+}
+
+/* Decode string-encoded idset from drain/undrain request.
+ * Catch various errors common to both requests.
+ * On success, return idset object (caller must free).
+ * On error, capture human readable error in 'errbuf', set errno, return NULL.
+ */
+static struct idset *drain_idset_decode (struct drain *drain,
+                                         const char *ranks,
+                                         flux_error_t *errp)
+{
+    struct idset *idset;
+
+    if (!(idset = inventory_targets_to_ranks (drain->ctx->inventory,
+                                              ranks, errp)))
+        return NULL;
+    if (idset_count (idset) == 0) {
+        errprintf (errp, "idset is empty");
+        errno = EINVAL;
+        goto error;
+    }
+    if (idset_last (idset) >= drain->ctx->size) {
+        errprintf (errp, "idset is out of range");
+        errno = EINVAL;
+        goto error;
+    }
+    return idset;
+error:
+    idset_destroy (idset);
+    return NULL;
+}
+
+/* Drain a set of ranked execution targets.
+ */
+static void drain_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
+{
+    int rc;
+    struct drain *drain = arg;
+    const char *s;
+    const char *mode = NULL;
+    const char *reason = NULL;
+    struct idset *idset = NULL;
+    const char *errstr = NULL;
+    char *idstr = NULL;
+    char *nodelist = NULL;
+    flux_error_t error;
+    double timestamp;
+    int overwrite = 0;
+    int update_only = 0;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s?s s?s}",
+                             "targets", &s,
+                             "reason", &reason,
+                             "mode", &mode) < 0)
+        goto error;
+    if (!(idset = drain_idset_decode (drain, s, &error))) {
+        errstr = error.text;
+        goto error;
+    }
+    if (get_timestamp_now (&timestamp) < 0)
+        goto error;
+
+    if (mode) {
+        errstr = "Invalid mode specified";
+        if (streq (mode, "update"))
+            update_only = 1;
+        else if (streq (mode, "overwrite"))
+            overwrite = 1;
+        else if (streq (mode, "force-overwrite"))
+            overwrite = 2;
+        else {
+            errno = EINVAL;
+            goto error;
+        }
+    }
+
+    /*  If neither overwrite or update_only are set, then return error unless
+     *   none of the target ranks are already drained.
+     */
+    if (!overwrite &&
+        !update_only &&
+        check_draininfo_idset (drain, idset, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (update_draininfo_idset (drain,
+                                idset,
+                                true,
+                                timestamp,
+                                reason,
+                                overwrite) < 0)
+        goto error;
+    if (!(idstr = idset_encode (idset, IDSET_FLAG_RANGE))
+        || !(nodelist = flux_hostmap_lookup (h, idstr, NULL)))
+        goto error;
+
+    /*  If draining with no reason, do not encode 'reason' in the
+     *   eventlog so that it can be replayed as reason=NULL.
+     */
+    if (reason)
+        rc = reslog_post_pack (drain->ctx->reslog,
+                               msg,
+                               timestamp,
+                               "drain",
+                               0,
+                               "{s:s s:s s:s s:i}",
+                               "idset", idstr,
+                               "nodelist", nodelist,
+                               "reason", reason,
+                               "overwrite", overwrite);
+    else
+        rc = reslog_post_pack (drain->ctx->reslog,
+                               msg,
+                               timestamp,
+                               "drain",
+                               0,
+                               "{s:s s:s s:i}",
+                               "idset", idstr,
+                               "nodelist", nodelist,
+                               "overwrite", overwrite);
+    if (rc < 0)
+        goto error;
+    free (nodelist);
+    free (idstr);
+    idset_destroy (idset);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to drain request");
+    free (nodelist);
+    free (idstr);
+    idset_destroy (idset);
+}
+
+int drain_rank (struct drain *drain, uint32_t rank, const char *reason)
+{
+    char rankstr[16];
+    double timestamp;
+    char *nodelist = NULL;
+    int rc = -1;
+
+    if (rank >= drain->ctx->size || !reason) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (get_timestamp_now (&timestamp) < 0)
+        return -1;
+    if (update_draininfo_rank (drain, rank, true, timestamp, reason, 0) < 0)
+        return -1;
+    snprintf (rankstr, sizeof (rankstr), "%ju", (uintmax_t)rank);
+    if (!(nodelist = flux_hostmap_lookup (drain->ctx->h, rankstr, NULL)))
+        return -1;
+    if (reslog_post_pack (drain->ctx->reslog,
+                          NULL,
+                          timestamp,
+                          "drain",
+                          0,
+                          "{s:s s:s s:s}",
+                          "idset", rankstr,
+                          "nodelist", nodelist,
+                          "reason", reason) < 0
+        || reslog_sync (drain->ctx->reslog) < 0)
+        goto done;
+    rc = 0;
+done:
+    ERRNO_SAFE_WRAP (free, nodelist);
+    return rc;
+}
+
+static int undrain_rank_idset (struct drain *drain,
+                               const flux_msg_t *msg,
+                               struct idset *idset)
+{
+    char *idstr;
+    char *nodelist = NULL;
+    int rc = -1;
+
+    if (idset_count (idset) == 0)
+        return 0;
+    if (update_draininfo_idset (drain, idset, false, 0., NULL, 1) < 0)
+        return -1;
+    if (!(idstr = idset_encode (idset, IDSET_FLAG_RANGE))
+        || !(nodelist = flux_hostmap_lookup (drain->ctx->h, idstr, NULL)))
+        goto done;
+    rc = reslog_post_pack (drain->ctx->reslog,
+                           msg,
+                           0.,
+                           "undrain",
+                           0,
+                           "{s:s s:s}",
+                           "idset", idstr,
+                           "nodelist", nodelist);
+done:
+    ERRNO_SAFE_WRAP (free, nodelist);
+    ERRNO_SAFE_WRAP (free, idstr);
+    return rc;
+}
+
+/* Un-drain a set of ranked execution targets.
+ * If any of the ranks are not drained, fail the whole request.
+ */
+static void undrain_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
+{
+    struct drain *drain = arg;
+    const char *s;
+    const char *mode = NULL;
+    struct idset *idset = NULL;
+    struct idset *undrained = NULL;
+    unsigned int id;
+    const char *errstr = NULL;
+    flux_error_t error;
+    bool force = false;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s?s}",
+                             "targets", &s,
+                             "mode", &mode) < 0)
+        goto error;
+    if (!(idset = drain_idset_decode (drain, s, &error))) {
+        errstr = error.text;
+        goto error;
+    }
+    if (mode) {
+        if (streq (mode, "force"))
+            force = true;
+        else {
+            errprintf (&error, "invalid undrain mode '%s' specified", mode);
+            errno = EINVAL;
+            errstr = error.text;
+            goto error;
+        }
+    }
+    if (!force && !(undrained = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+        errprintf (&error,
+                   "failed to create idset for undrained ranks: %s",
+                   strerror (errno));
+        errstr = error.text;
+        goto error;
+    }
+    id = idset_first (idset);
+    while (id != IDSET_INVALID_ID) {
+        if (!drain->info[id].drained) {
+            int rc;
+            /* This rank is already undrained, remove it from targets
+             * if mode=force. Otherwise, add rank to undrained idset.
+             */
+            rc = force ? idset_clear (idset, id) : idset_set (undrained, id);
+            if (rc < 0) {
+                errprintf (&error, "failed to update undrain target idset");
+                errstr = error.text;
+                goto error;
+            }
+        }
+        id = idset_next (idset, id);
+    }
+    if (!force && idset_count (undrained) > 0) {
+        char *nodelist = NULL;
+        char *ranks = idset_encode (undrained, IDSET_FLAG_RANGE);
+        if (ranks)
+            nodelist = flux_hostmap_lookup (h, ranks, NULL);
+        errprintf (&error,
+                   "%s (rank%s %s) not drained",
+                   nodelist ? nodelist : "unknown",
+                   idset_count (undrained) > 1 ? "s" : "",
+                   ranks ? ranks : "unknown");
+        free (ranks);
+        free (nodelist);
+        errstr = error.text;
+        errno = EINVAL;
+        goto error;
+    }
+    if (idset_count (idset) == 0) {
+        /* If idset is now empty then no targets are drained and
+         * mode=force was used. Therefore, immediately return success:
+         */
+        if (flux_respond (h, msg, NULL) < 0)
+            flux_log_error (h, "error responding to undrain request");
+    }
+    else if (undrain_rank_idset (drain, msg, idset) < 0)
+        goto error;
+    idset_destroy (idset);
+    idset_destroy (undrained);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to undrain request");
+    idset_destroy (idset);
+    idset_destroy (undrained);
+}
+
+/* Add rank to ids, adjusting rank if the rank:host mapping has changed.
+ * Don't add the rank if the host no longer exists, or if it exceeds
+ * the instance size.
+ *
+ * N.B. When running multiple brokers per node, flux_get_rankbyhost()
+ * returns the first rank on 'host', so its result cannot be directly
+ * used as the new rank.  Instead, first check that flux_get_hostbyrank()
+ * differs from 'host'.
+ */
+static void add_target (struct idset *ids,
+                        unsigned int rank,
+                        const char *host,
+                        flux_t *h)
+{
+    if (host) {
+        const char *nhost = flux_get_hostbyrank (h, rank);
+        int nrank;
+
+        if (!streq (host, nhost)) { // nhost could be "(null)" on bad rank
+            if ((nrank = flux_get_rankbyhost (h, host)) < 0)
+                return;
+            rank = nrank;
+        }
+    }
+    (void)idset_set (ids, rank); // no-op if rank exceeds fixed set size
+}
+
+/* Return an idset containing decoded 'ranks', possibly adjusted based on
+ * 'nodelist' and the instance size.  Any ranks that are invalid are simply
+ * not added (not treated as an error).
+ */
+static struct idset *decode_targets (struct drain *drain,
+                                     const char *ranks,
+                                     const char *nodelist)
+{
+    struct idset *ids;
+    struct hostlist *nl = NULL;
+    struct idset *newids = NULL;
+    unsigned int rank;
+    int index;
+
+    if (!(ids = idset_decode (ranks))
+        || !(nl = hostlist_decode (nodelist))
+        || !(newids = idset_create (drain->ctx->size, 0)))
+        goto done;
+
+    index = 0;
+    rank = idset_first (ids);
+    while (rank != IDSET_INVALID_ID) {
+        const char *host = hostlist_nth (nl, index++);
+
+        add_target (newids, rank, host, drain->ctx->h);
+        rank = idset_next (ids, rank);
+    }
+done:
+    hostlist_destroy (nl);
+    idset_destroy (ids);
+    return newids;
+}
+
+/* Recover drained idset from eventlog.
+ */
+static int replay_eventlog (struct drain *drain,
+                            const json_t *eventlog,
+                            flux_error_t *error)
+{
+    size_t index;
+    json_t *entry;
+
+    if (eventlog) {
+        json_array_foreach (eventlog, index, entry) {
+            double timestamp;
+            const char *name;
+            json_t *context;
+            const char *s;
+            const char *reason = NULL;
+            struct idset *idset;
+
+            if (eventlog_entry_parse (entry, &timestamp, &name, &context) < 0) {
+                errprintf (error, "line %zu: event parse error", index + 1);
+                return -1;
+            }
+            if (streq (name, "drain")) {
+                int overwrite = 1;
+                const char *nodelist;
+                if (json_unpack (context,
+                                 "{s:s s:s s?s s?i}",
+                                 "idset", &s,
+                                 "nodelist", &nodelist,
+                                 "reason", &reason,
+                                 "overwrite", &overwrite) < 0) {
+                    errprintf (error, "line %zu: drain parse error", index + 1);
+                    errno = EPROTO;
+                    return -1;
+                }
+                if (!(idset = decode_targets (drain, s, nodelist))) {
+                    errprintf (error,
+                               "line %zu: drain target decode error",
+                               index + 1);
+                    return -1;
+                }
+                if (update_draininfo_idset (drain,
+                                            idset,
+                                            true,
+                                            timestamp,
+                                            reason,
+                                            overwrite) < 0) {
+                    errprintf (error,
+                               "line %zu: drain update error",
+                               index + 1);
+                    idset_destroy (idset);
+                    return -1;
+                }
+                idset_destroy (idset);
+            }
+            else if (streq (name, "undrain")) {
+                const char *nodelist = NULL;
+                if (json_unpack (context,
+                                 "{s:s s:s}",
+                                 "idset", &s,
+                                 "nodelist", &nodelist) < 0) {
+                    errprintf (error,
+                               "line %zu: undrain parse error",
+                               index + 1);
+                    errno = EPROTO;
+                    return -1;
+                }
+                if (!(idset = decode_targets (drain, s, nodelist))) {
+                    errprintf (error,
+                               "line %zu: undrain target decode error",
+                               index + 1);
+                    return -1;
+                }
+                if (update_draininfo_idset (drain,
+                                            idset,
+                                            false,
+                                            timestamp,
+                                            NULL,
+                                            1) < 0) {
+                    errprintf (error,
+                               "line %zu: undrain update error",
+                               index + 1);
+                    idset_destroy (idset);
+                    return -1;
+                }
+                idset_destroy (idset);
+            }
+        }
+    }
+    return 0;
+}
+
+/* Excluded targets may not be drained.  If, after replaying the eventlog,
+ * any excluded nodes are drained, undrain them.  Besides updating the current
+ * drain state, an undrain event must be posted to resource.eventlog so that
+ * if the target is unexcluded later on, it starts out undrained.
+ */
+static int reconcile_excluded (struct drain *drain,
+                               const struct idset *exclude,
+                               flux_error_t *error)
+{
+    struct idset *drained;
+    struct idset *undrain_ranks = NULL;
+    char *s = NULL;
+    char *nodelist = NULL;
+    int rc = -1;
+
+    if (!exclude)
+        return 0;
+    if (!(drained = drain_get (drain))
+        || !(undrain_ranks = idset_intersect (drained, exclude))) {
+        errprintf (error,
+                   "error calculating drained ∩ excluded: %s",
+                   strerror (errno));
+        goto done;
+    }
+    if (idset_count (undrain_ranks) > 0) {
+        double timestamp;
+        if (get_timestamp_now (&timestamp) < 0
+            || update_draininfo_idset (drain,
+                                       undrain_ranks,
+                                       false,
+                                       timestamp,
+                                       NULL,
+                                       1) < 0) {
+            errprintf (error,
+                       "error draining excluded nodes: %s",
+                       strerror (errno));
+            goto done;
+        }
+        if (!(s = idset_encode (undrain_ranks, IDSET_FLAG_RANGE))
+            || !(nodelist = flux_hostmap_lookup (drain->ctx->h, s, NULL))
+            || reslog_post_pack (drain->ctx->reslog,
+                                 NULL,
+                                 timestamp,
+                                 "undrain",
+                                 0,
+                                 "{s:s s:s}",
+                                 "idset", s,
+                                 "nodelist", nodelist) < 0) {
+            errprintf (error,
+                       "error posting drain event for excluded nodes: %s",
+                       strerror (errno));
+            goto done;
+        }
+    }
+    rc = 0;
+done:
+    ERRNO_SAFE_WRAP (free, nodelist);
+    ERRNO_SAFE_WRAP (free, s);
+    idset_destroy (undrain_ranks);
+    idset_destroy (drained);
+    return rc;
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,  "resource.drain", drain_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST,  "resource.undrain", undrain_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void drain_destroy (struct drain *drain)
+{
+    if (drain) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (drain->handlers);
+        if (drain->info) {
+            unsigned int rank;
+            for (rank = 0; rank < drain->ctx->size; rank++)
+                free (drain->info[rank].reason);
+            free (drain->info);
+        }
+        free (drain);
+        errno = saved_errno;
+    }
+}
+
+struct drain *drain_create (struct resource_ctx *ctx, const json_t *eventlog)
+{
+    struct drain *drain;
+    flux_error_t error;
+
+    if (!(drain = calloc (1, sizeof (*drain))))
+        return NULL;
+    drain->ctx = ctx;
+    if (!(drain->info = calloc (ctx->size, sizeof (drain->info[0]))))
+        goto error;
+    if (replay_eventlog (drain, eventlog, &error) < 0) {
+        flux_log (ctx->h, LOG_ERR, "%s: %s", RESLOG_KEY, error.text);
+        goto error;
+    }
+    if (reconcile_excluded (drain, exclude_get (ctx->exclude), &error) < 0) {
+        flux_log (ctx->h, LOG_ERR, "%s", error.text);
+        goto error;
+    }
+    if (flux_msg_handler_addvec (ctx->h, htab, drain, &drain->handlers) < 0)
+        goto error;
+    return drain;
+error:
+    drain_destroy (drain);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/drain.h b/src/modules/resource/drain.h
new file mode 100644
index 000000000000..c5f10c368c85
--- /dev/null
+++ b/src/modules/resource/drain.h
@@ -0,0 +1,34 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_DRAIN_H
+#define _FLUX_RESOURCE_DRAIN_H
+
+struct drain *drain_create (struct resource_ctx *ctx, const json_t *eventlog);
+void drain_destroy (struct drain *drain);
+
+struct idset *drain_get (struct drain *drain);
+
+/* Get object containing summary of drained nodes, for use in restart event.
+ * Keys are idsets, values are object { "timestamp":f, "reason":s}.
+ * Caller is given a reference on JSON object - free with json_decref().
+ */
+json_t *drain_get_info  (struct drain *drain);
+
+/* Drain 'rank' for 'reason'.  Call this on rank 0 only, otherwise use
+ * resource.drain RPC.
+ */
+int drain_rank (struct drain *drain, uint32_t rank, const char *reason);
+
+#endif /* !_FLUX_RESOURCE_DRAIN_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/drainset.c b/src/modules/resource/drainset.c
new file mode 100644
index 000000000000..beabe15a6f87
--- /dev/null
+++ b/src/modules/resource/drainset.c
@@ -0,0 +1,212 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/*  drainset.c - a set of drained ranks with timestamp and reason
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <time.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "drainset.h"
+
+struct draininfo {
+    struct idset *ranks;
+    double timestamp;
+    char *reason;
+};
+
+struct drainset {
+    zhashx_t *map;
+};
+
+static void draininfo_destroy (struct draininfo *d)
+{
+    if (d) {
+        int saved_errno = errno;
+        idset_destroy (d->ranks);
+        free (d->reason);
+        free (d);
+        errno = saved_errno;
+    }
+}
+
+static void draininfo_free (void **item)
+{
+    if (item) {
+        draininfo_destroy (*item);
+        *item = NULL;
+    }
+}
+
+static struct draininfo *draininfo_create_rank (unsigned int rank,
+                                                const char *reason,
+                                                double timestamp)
+{
+    struct draininfo *d;
+    if (!(d = calloc (1, sizeof (*d)))
+        || !(d->ranks = idset_create (0, IDSET_FLAG_AUTOGROW))
+        || idset_set (d->ranks, rank) < 0
+        || (reason && !(d->reason = strdup (reason))))
+        goto error;
+    d->timestamp = timestamp;
+    return d;
+error:
+    draininfo_destroy (d);
+    return NULL;
+}
+
+/* Use "modified Bernstein hash" as employed by zhashx internally, but input
+ * is draininfo reason+timestamp instead of a simple NULL-terminated string.
+ * Copied from: msg_hash_uuid_matchtag_hasher()
+ */
+static size_t draininfo_hasher (const void *key)
+{
+    const struct draininfo *d = key;
+    size_t key_hash = 0;
+    const char *cp;
+
+    cp = d->reason ? d->reason : "";
+    while (*cp)
+        key_hash = 33 * key_hash ^ *cp++;
+    cp = (const char *) &d->timestamp;
+    for (int i = 0; i < sizeof (d->timestamp); i++)
+        key_hash = 33 * key_hash ^ *cp++;
+    return key_hash;
+}
+
+static int drainmap_key_cmp (const void *key1, const void *key2)
+{
+    const struct draininfo *d1 = key1;
+    const struct draininfo *d2 = key2;
+    if (d1->timestamp == d2->timestamp) {
+        const char *s1 = d1->reason;
+        const char *s2 = d2->reason;
+        return strcmp (s1 ? s1 : "", s2 ? s2 : "");
+    }
+    return d1->timestamp < d2->timestamp ? -1 : 1;
+}
+
+static zhashx_t *drainmap_create ()
+{
+    zhashx_t *map;
+
+    if (!(map = zhashx_new ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    zhashx_set_key_hasher (map, draininfo_hasher);
+    zhashx_set_key_comparator (map, drainmap_key_cmp);
+    zhashx_set_key_destructor (map, draininfo_free);
+    zhashx_set_key_duplicator (map, NULL);
+    return map;
+}
+
+void drainset_destroy (struct drainset *ds)
+{
+    if (ds) {
+        int saved_errno = errno;
+        zhashx_destroy (&ds->map);
+        free (ds);
+        errno = saved_errno;
+    }
+}
+
+struct drainset *drainset_create (void)
+{
+    struct drainset *ds;
+
+    if (!(ds = malloc (sizeof (*ds)))
+        || !(ds->map = drainmap_create ()))
+        goto error;
+    return ds;
+error:
+    drainset_destroy (ds);
+    return NULL;
+}
+
+static struct draininfo *drainset_find (struct drainset *ds,
+                                        double timestamp,
+                                        const char *reason)
+{
+    struct draininfo tmp = {.timestamp = timestamp, .reason = (char *)reason};
+    return zhashx_lookup (ds->map, &tmp);
+}
+
+int drainset_drain_rank (struct drainset *ds,
+                         unsigned int rank,
+                         double timestamp,
+                         const char *reason)
+{
+    int rc = -1;
+    struct draininfo *match;
+    struct draininfo *new = NULL;
+    if (!ds) {
+        errno = EINVAL;
+        return -1;
+    }
+    if ((match = drainset_find (ds, timestamp, reason))) {
+        if (idset_set (match->ranks, rank) < 0)
+            return -1;
+        return 0;
+    }
+    if (!(new = draininfo_create_rank (rank, reason, timestamp))
+        || zhashx_insert (ds->map, new, new) < 0) {
+        draininfo_destroy (new);
+        goto out;
+    }
+    rc = 0;
+out:
+    return rc;
+}
+
+json_t *drainset_to_json (struct drainset *ds)
+{
+    json_t *o;
+    struct draininfo *d;
+
+    if (!(o = json_object ()))
+        goto nomem;
+    d = zhashx_first (ds->map);
+    while (d) {
+        json_t *val;
+        char *s;
+        if (!(val = json_pack ("{s:f s:s}",
+                               "timestamp", d->timestamp,
+                               "reason", d->reason ? d->reason : ""))
+            || !(s = idset_encode (d->ranks, IDSET_FLAG_RANGE))) {
+            json_decref (val);
+            goto nomem;
+        }
+        if (json_object_set_new (o, s, val) < 0) {
+            ERRNO_SAFE_WRAP (json_decref, val);
+            ERRNO_SAFE_WRAP (free, s);
+            goto error;
+        }
+        free (s);
+        d = zhashx_next (ds->map);
+    }
+    return o;
+nomem:
+    errno = EPROTO;
+error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/drainset.h b/src/modules/resource/drainset.h
new file mode 100644
index 000000000000..02aea03db35e
--- /dev/null
+++ b/src/modules/resource/drainset.h
@@ -0,0 +1,34 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_DRAINSET_H
+#define _FLUX_RESOURCE_DRAINSET_H
+
+#include <stdbool.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/idset.h>
+
+struct drainset * drainset_create (void);
+void drainset_destroy (struct drainset *dset);
+
+int drainset_drain_rank (struct drainset *dset,
+                         unsigned int rank,
+                         double timestamp,
+                         const char *reason);
+
+json_t *drainset_to_json (struct drainset *dset);
+
+#endif /* ! _FLUX_RESOURCE_DRAINSET_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/exclude.c b/src/modules/resource/exclude.c
new file mode 100644
index 000000000000..67803104937b
--- /dev/null
+++ b/src/modules/resource/exclude.c
@@ -0,0 +1,91 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* exclude.c - get static list of exec targets excluded from scheduling
+ *
+ * Caveats:
+ * - There is no way to exclude at a finer granularity than execution target
+ *   (e.g. by core would be useful).
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libidset/idset.h"
+#include "src/common/libutil/errprintf.h"
+
+#include "resource.h"
+#include "reslog.h"
+#include "exclude.h"
+#include "rutil.h"
+#include "inventory.h"
+#include "drain.h"
+
+struct exclude {
+    struct resource_ctx *ctx;
+    struct idset *idset;
+};
+
+const struct idset *exclude_get (struct exclude *exclude)
+{
+    return exclude->idset;
+}
+
+void exclude_destroy (struct exclude *exclude)
+{
+    if (exclude) {
+        int saved_errno = errno;
+        idset_destroy (exclude->idset);
+        free (exclude);
+        errno = saved_errno;
+    }
+}
+
+struct exclude *exclude_create (struct resource_ctx *ctx,
+                                const char *exclude_idset)
+{
+    struct exclude *exclude;
+
+    if (!(exclude = calloc (1, sizeof (*exclude))))
+        return NULL;
+    exclude->ctx = ctx;
+    if (exclude_idset) {
+        flux_error_t error;
+        if (!(exclude->idset = inventory_targets_to_ranks (ctx->inventory,
+                                                           exclude_idset,
+                                                           &error))) {
+            flux_log (ctx->h,
+                      LOG_ERR,
+                      "error decoding exclude set %s: %s",
+                      exclude_idset,
+                      error.text);
+            goto error;
+        }
+        if (idset_count (exclude->idset) > 0
+            && idset_last (exclude->idset) >= exclude->ctx->size) {
+            flux_log_error (ctx->h,
+                            "exclude set %s is out of range",
+                            exclude_idset);
+            goto error;
+        }
+    }
+    return exclude;
+error:
+    exclude_destroy (exclude);
+    return NULL;
+}
+
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/exclude.h b/src/modules/resource/exclude.h
new file mode 100644
index 000000000000..57660007b8d5
--- /dev/null
+++ b/src/modules/resource/exclude.h
@@ -0,0 +1,23 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_EXCLUDE_H
+#define _FLUX_RESOURCE_EXCLUDE_H
+
+struct exclude *exclude_create (struct resource_ctx *ctx, const char *idset);
+void exclude_destroy (struct exclude *exclude);
+
+const struct idset *exclude_get (struct exclude *exclude);
+
+#endif /* !_FLUX_RESOURCE_EXCLUDE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/inventory.c b/src/modules/resource/inventory.c
new file mode 100644
index 000000000000..ee00bad322ae
--- /dev/null
+++ b/src/modules/resource/inventory.c
@@ -0,0 +1,940 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* inventory.c - container for instance resources
+ *
+ * Instance resources (R) are initialized here.
+ *
+ * Three main sources of R were described in flux-framework/flux-core#3238:
+ * 1. configured resources (e.g. system instance)
+ * 2. resources assigned to instance by enclosing instance
+ * 3. dynamic discovery
+ *
+ * This module captures R internally, commits it to 'resource.R' in the KVS,
+ * and posts the resource-define event to resource.eventlog.
+ *
+ * The three cases above work as follows:
+ *
+ * Case 1 (method=configuration)
+ * -----------------------------
+ * TOML config specifies [resource] path, pointing to R.  R is parsed
+ * and is "re-ranked" if the 'hostlist' broker attribute defines a
+ * mapping of ranks to hostnames AND there exists a [bootstrap] config.
+ * (Sys admins are not required to regenerate R when they reassign broker
+ * ranks via [bootstrap]).
+ *
+ * R is configured on all ranks during resource module load.  On rank 0,
+ * resource.R is committed to the KVS, and the resource-define event is
+ * posted to resource.eventlog.
+ *
+ * topo.c ensures that configured resources match the hwloc topology on
+ * all ranks.  If there are missing resources, offending ranks are drained.
+ *
+ * Case 2 (method=job-info)
+ * ------------------------
+ * On the rank 0 broker, if the 'parent-uri' broker attribute is defined,
+ * a connection is made to the parent broker, and R is read from the
+ * job-info module.  This R was assigned to the instance by the enclosing
+ * instance scheduler, and includes ranks representing brokers in the
+ * enclosing instance.
+ *
+ * If the same number of ranks are defined in R as there are brokers in this
+ * instance, then the ranks are renumbered to be contiguous starting from
+ * zero.  If a different number of ranks are defined (e.g. launching multiple
+ * brokers per node), we bail out of case 2 and fall through to case 3.
+ *
+ * On rank 0, resource.R is committed to the KVS, and the resource-define
+ * event is posted to resource.eventlog.  The other ranks request R from
+ * their TBON parent using the 'resource.get' RPC, synchronously, so R
+ * is defined on all ranks after module load completes.
+ *
+ * Case 3 (method=dynamic-discovery)
+ * ---------------------------------
+ * If inventory_create() returns without defining R, topo.c initiates
+ * resource discovery.  Module load may complete before R is defined.
+ *
+ * Once the topology has been reduced to R on rank 0, resource.R is committed
+ * to the KVS, and the resource-define event is posted to resource.eventlog.
+ * This event serves as synchronization to indicate that R is now available.
+ * acquire.c watches for this event.
+ *
+ * Test Features
+ * -------------
+ * When the module is reloaded on rank 0, if resource.R is found in the KVS,
+ * it is reused.  This allows the rank 0 resource module to be reloaded in test
+ * without the need to go through resource discovery (case 3) or interacting
+ * with enclosing instance (case 2) again.  An existing resource.R is ignored
+ * if resources are set by configuration (case 1).
+ *
+ * Tests that require fake resources may set them with
+ * 'flux resource reload PATH', where PATH points to a file containing R.
+ * Alternatively, use 'flux resource reload -x DIR' to load <rank>.xml files
+ * and use them to generate R.
+ *
+ * It's also possible to fake resources by placing them in resource.R and
+ * then (re-) loading the resource module.  This is how the sharness 'job'
+ * personality fakes resources.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/librlist/rlist.h"
+#include "src/common/librlist/rhwloc.h"
+#include "src/common/libidset/idset.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/jpath.h"
+
+#include "rutil.h"
+#include "resource.h"
+#include "reslog.h"
+#include "acquire.h"
+#include "inventory.h"
+
+
+struct inventory {
+    struct resource_ctx *ctx;
+
+    json_t *R;
+    char *method;
+
+    flux_future_t *put_f;          /* inventory put future */
+
+    flux_t *parent_h;              /* handle to parent instance */
+    flux_future_t *R_watch_f;      /* job-info.update-watch future */
+
+    flux_msg_handler_t **handlers;
+};
+
+static int rank_from_key (const char *key);
+
+static int inventory_put_finalize (struct inventory *inv)
+{
+    const char *method = flux_future_aux_get (inv->put_f, "method");
+    int rc = -1;
+
+    if (flux_future_get (inv->put_f, NULL) < 0) {
+        flux_log_error (inv->ctx->h, "error committing R to KVS");
+        goto done;
+    }
+    if (reslog_post_pack (inv->ctx->reslog,
+                          NULL,
+                          0.,
+                          "resource-define",
+                          0,
+                          "{s:s}",
+                          "method",
+                          method) < 0) {
+        flux_log_error (inv->ctx->h, "error posting resource-define event");
+        goto done;
+    }
+    rc = 0;
+done:
+    flux_future_destroy (inv->put_f);
+    inv->put_f = NULL;
+    return rc;
+}
+
+static void inventory_put_continuation (flux_future_t *f, void *arg)
+{
+    struct inventory *inv = arg;
+
+    (void)inventory_put_finalize (inv);
+}
+
+static flux_future_t *inventory_put_R (struct inventory *inv)
+{
+    flux_kvs_txn_t *txn = NULL;
+    flux_future_t *f = NULL;
+
+    if (!(txn = flux_kvs_txn_create ())
+        || flux_kvs_txn_pack (txn, 0, "resource.R", "O", inv->R) < 0)
+        goto error;
+    f = flux_kvs_commit (inv->ctx->h, NULL, 0, txn);
+error:
+    flux_kvs_txn_destroy (txn);
+    return f;
+}
+
+/* (rank 0) Commit resource.R to the KVS, then upon completion,
+ * post resource-define event to resource.eventlog.
+ */
+int inventory_put (struct inventory *inv, json_t *R, const char *method)
+
+{
+    char *cpy;
+
+    if (inv->ctx->rank != 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (inv->R) {
+        errno = EEXIST;
+        return -1;
+    }
+    inv->R = json_incref (R);
+    if (!(inv->put_f = inventory_put_R (inv)))
+        goto error;
+    if (flux_future_then (inv->put_f, -1, inventory_put_continuation, inv) < 0)
+        goto error;
+    if (flux_future_aux_set (inv->put_f, "method", (void *)method, NULL) < 0)
+        goto error;
+    if (!(cpy = strdup (method)))
+        goto error;
+    inv->method = cpy;
+    return 0;
+error:
+    flux_future_destroy (inv->put_f);
+    inv->put_f = NULL;
+    return -1;
+}
+
+json_t *inventory_get (struct inventory *inv)
+{
+    if (!inv->R) {
+        errno = ENOENT;
+        return NULL;
+    }
+    return inv->R;
+}
+
+const char *inventory_get_method (struct inventory *inv)
+{
+    if (!inv->method) {
+        errno = ENOENT;
+        return NULL;
+    }
+    return inv->method;
+}
+
+struct idset *inventory_targets_to_ranks (struct inventory *inv,
+                                          const char *targets,
+                                          flux_error_t *errp)
+{
+    struct idset *ids = NULL;
+
+    if (!(ids = idset_decode (targets))) {
+        /*  Not a valid idset, maybe an RFC29 Hostlist
+         */
+        flux_error_t err;
+        struct rlist *rl;
+        if (!inv->R || !(rl = rlist_from_json (inv->R, NULL))) {
+            errprintf (errp, "R is unavailable for mapping hostnames to ranks");
+            errno = EINVAL;
+            return NULL;
+        }
+        if (!(ids = rlist_hosts_to_ranks (rl, targets, &err))) {
+            errprintf (errp, "invalid targets: %s", err.text);
+            rlist_destroy (rl);
+            errno = EINVAL;
+            return NULL;
+        }
+        rlist_destroy (rl);
+    }
+    return ids;
+}
+
+/* Test if [bootstrap] table exists in the configuration.
+ * If it does then we can assume that the 'hostlist' attribute was
+ * derived from the TOML config, and may be used to re-rank a configured R.
+ */
+static bool conf_has_bootstrap (flux_t *h)
+{
+    json_t *o;
+
+    if (flux_conf_unpack (flux_get_conf (h),
+                          NULL,
+                          "{s:o}",
+                          "bootstrap", &o) < 0)
+        return false;
+    return true;
+}
+
+static int convert_R_conf (flux_t *h, json_t *conf_R, json_t **Rp)
+{
+    json_error_t e;
+    struct rlist *rl;
+    flux_error_t err;
+    json_t *R;
+    const char *hosts;
+
+    if (!(rl = rlist_from_json (conf_R, &e))) {
+        flux_log (h, LOG_ERR, "error parsing R: %s", e.text);
+        errno = EINVAL;
+        return -1;
+    }
+    if (conf_has_bootstrap (h)) {
+        if (!(hosts = flux_attr_get (h, "hostlist"))) {
+            flux_log_error (h, "Unable to get hostlist attribute");
+            goto error;
+        }
+        err.text[0] = '\0';
+        if (rlist_rerank (rl, hosts, &err) < 0) {
+            flux_log (h, LOG_ERR, "error reranking R: %s", err.text);
+            /*
+             *  rlist_rerank() repurposes errno like EOVERFLOW and ENOSPC,
+             *   but this may cause confusion when logging an error
+             *   return from this function. Since the specific error has
+             *   already been printed, reset errno to EINVAL so that
+             *   the calling function simply prints "Invalid argument".
+             */
+            errno = EINVAL;
+            goto error;
+        }
+    }
+    if (!(R = rlist_to_R (rl))) {
+        errno = ENOMEM;
+        goto error;
+    }
+    rlist_destroy (rl);
+    *Rp = R;
+    return 0;
+error:
+    rlist_destroy (rl);
+    return -1;
+}
+
+static bool no_duplicates (const char *hosts)
+{
+    struct hostlist *hl = hostlist_decode (hosts);
+    bool result = false;
+
+    if (hl) {
+        int count = hostlist_count (hl);
+        hostlist_uniq (hl);
+        if (hostlist_count (hl) == count)
+            result = true;
+    }
+
+    hostlist_destroy (hl);
+
+    return result;
+}
+
+/* Derive resource object from R, normalizing broker ranks to origin.
+ * Return success (0) but leave *Rp alone if conversion cannot be performed,
+ * thus *Rp should be set to NULL before calling this function.
+ * On failure return -1 (errno is not set).
+ */
+static int convert_R (flux_t *h, json_t *R, int size, json_t **Rp)
+{
+    struct rlist *rl;
+    struct idset *ranks;
+    const char *hosts;
+    int count;
+    int rc = -1;
+    flux_error_t err;
+
+    if (!(rl = rlist_from_json (R, NULL)))
+        return -1;
+    if (!(ranks = rlist_ranks (rl)))
+        goto error;
+    count = idset_count (ranks);
+    if (count != size) {
+        flux_log (h,
+                  LOG_DEBUG,
+                  "cannot map %d ranks of R to %d brokers, "
+                  "falling back to discovery",
+                  count,
+                  size);
+        goto noconvert;
+    }
+    /*  If we have an assigned hostlist and there is no more than
+     *   one broker per rank (i.e. no duplicates), then rerank R
+     *   based on the assigned hostlist.
+     */
+    if ((hosts = flux_attr_get (h, "hostlist"))
+        && no_duplicates (hosts)) {
+
+        /*  Allow rlist_rerank() to fail here. This could be due to a fake
+         *   R used in testing, or other conditions where it won't make
+         *   sense to apply the re-ranking anyway. Just issue a warning
+         *   and continue on failure.
+         */
+        err.text[0] = '\0';
+        if (rlist_rerank (rl, hosts, &err) < 0)
+            flux_log (h, LOG_DEBUG,
+                      "Warning: rerank of R failed: %s", err.text);
+    }
+    /*  Also always remap ids to zero origin
+     */
+    if (rlist_remap (rl) < 0)
+        goto error;
+    if (!(*Rp = rlist_to_R (rl)))
+        goto error;
+noconvert:
+    rc = 0;
+error:
+    idset_destroy (ranks);
+    rlist_destroy (rl);
+    return rc;
+}
+
+/*  Cancel and destroy R job-info.update-watch future
+ */
+static void R_watch_destroy (struct inventory *inv)
+{
+    flux_future_t *f;
+
+    if (!inv->R_watch_f)
+        return;
+
+    f = flux_rpc_pack (inv->ctx->h,
+                       "job-info.update-watch-cancel",
+                       FLUX_NODEID_ANY,
+                       FLUX_RPC_NORESPONSE,
+                       "{s:i}",
+                       "matchtag", flux_rpc_get_matchtag (inv->R_watch_f));
+    if (!f)
+        flux_log_error (inv->ctx->h,
+                        "job-info.update-watch-cancel failed");
+    flux_future_destroy (f);
+    flux_future_destroy (inv->R_watch_f);
+    inv->R_watch_f = NULL;
+}
+
+static void inventory_put_update_cb (flux_future_t *f, void *arg)
+{
+    struct inventory *inv = arg;
+    double expiration = -1.;
+
+    if (flux_future_get (f, NULL) < 0)
+        flux_log_error (inv->ctx->h, "failed to commit updated R to kvs");
+
+    if (json_unpack (inv->R,
+                     "{s:{s:F}}",
+                     "execution",
+                      "expiration", &expiration) < 0)
+        flux_log_error (inv->ctx->h,
+                        "failed to get updated expiration from R");
+
+    if (reslog_post_pack (inv->ctx->reslog,
+                          NULL,
+                          0.,
+                          "resource-update",
+                          0,
+                          "{s:f}",
+                          "expiration",
+                          expiration) < 0) {
+        flux_log_error (inv->ctx->h, "error posting resource-update event");
+    }
+    flux_future_destroy (f);
+    inv->put_f = NULL;
+}
+
+/*  Handle updates to R from parent instance. Currently, the only supported
+ *  update is an adjustment to expiration.
+ */
+static void R_update_cb (flux_future_t *f, void *arg)
+{
+    struct inventory *inv = arg;
+    flux_t *h = inv->ctx->h;
+    double expiration = -1.;
+    json_t *o = NULL;
+
+    if (flux_rpc_get_unpack (f,
+                             "{s:{s:{s:F}}}",
+                             "R",
+                              "execution",
+                               "expiration", &expiration) < 0) {
+        flux_log_error (h, "failed to unpack updated R expiration");
+        goto out;
+    }
+    /*  Update local inventory and send expiration update to scheduler
+     */
+    if (!(o = json_real (expiration))
+        || jpath_set (inv->R, "execution.expiration", o) < 0) {
+        flux_log (h, LOG_ERR, "failed to update expiration in inventory R");
+        goto out;
+    }
+    /*  Update R in KVS, post resource-update event when commit is complete.
+     */
+    if (!(inv->put_f = inventory_put_R (inv))
+        || flux_future_then (inv->put_f,
+                             -1.,
+                             inventory_put_update_cb,
+                             inv) < 0) {
+        flux_future_destroy (inv->put_f);
+        inv->put_f = NULL;
+        goto out;
+    }
+out:
+    json_decref (o);
+    flux_future_reset (f);
+}
+
+static int lookup_R_fallback (struct inventory *inv, flux_jobid_t id)
+{
+    flux_t *h = inv->ctx->h;
+    int rc = -1;
+    flux_future_t *f;
+    char *s;
+    json_t *job_R = NULL;
+    json_t *R = NULL;
+
+    if (!(f = flux_rpc_pack (inv->parent_h,
+                             "job-info.lookup",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:I s:[s] s:i}",
+                             "id", id,
+                             "keys", "R",
+                             "flags", 0))
+        || flux_rpc_get_unpack (f, "{s:s}", "R", &s) < 0
+        || !(job_R = json_loads (s, 0, NULL))) {
+        flux_log_error (h, "lookup R from enclosing instance (fallback)");
+        goto done;
+    }
+    if (convert_R (h, job_R, inv->ctx->size, &R) < 0) {
+        flux_log (h, LOG_ERR, "fatal error while normalizing R");
+        errno = EINVAL;
+        goto done;
+    }
+    /*  Only call inventory_put() if conversion of R was successful,
+     *  i.e. R != NULL. (if conversion failed, fall-through to dynamic
+     *  discovery will call inventory_put() later).
+     */
+    if (R && inventory_put (inv, R, "job-info") < 0)
+        goto done;
+    rc = 0;
+done:
+    /*  Parent handle is not used again in fallback case:
+     */
+    resource_parent_handle_close (inv->ctx);
+    inv->parent_h = NULL;
+    json_decref (R);
+    json_decref (job_R);
+    flux_future_destroy (f);
+    return rc;
+}
+
+static int start_resource_watch (struct inventory *inv, bool no_resource_watch)
+{
+    flux_t *h = inv->ctx->h;
+    const char *jobid;
+    json_t *job_R;
+    flux_jobid_t id;
+    flux_future_t *f = NULL;
+    json_t *R = NULL;
+    int rc = -1;
+    const char *service = "job-info.update-watch";
+
+    /*  Testing-only: send update-watch request to wrong service name to
+     *  simulate start under an older instance that does not support this
+     *  RPC.
+     */
+    if (no_resource_watch)
+        service = "job-info.update-watch-fake";
+
+    if (!(jobid = flux_attr_get (h, "jobid")))
+        return 0;
+    if (flux_job_id_parse (jobid, &id) < 0) {
+        flux_log_error (h, "error decoding jobid %s", jobid);
+        return -1;
+    }
+    if (!(inv->parent_h = resource_parent_handle_open (inv->ctx)))
+        goto done;
+
+    /*  Associate the main flux_t handle reactor with the parent handle
+     *  reactor so that events from both can be handled with the single
+     *  reactor instance:
+     */
+    if (flux_set_reactor (inv->parent_h, flux_get_reactor (h)) < 0) {
+        flux_log_error (h, "flux_set_reactor");
+        goto done;
+    }
+    if (!(f = flux_rpc_pack (inv->parent_h,
+                             service,
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_STREAMING,
+                             "{s:I s:s s:i}",
+                             "id", id,
+                             "key", "R",
+                             "flags", 0))) {
+        flux_log_error (h, "error sending request to enclosing instance");
+        goto done;
+    }
+
+    /* Get first response synchronously
+     */
+    if (flux_rpc_get_unpack (f, "{s:o}", "R", &job_R) < 0) {
+        if (errno == ENOSYS) {
+            /*  Parent instance doesn't support job-info.update-watch.
+             *  Note: job-info.update-watch was added in v0.56.0.
+             *  Fall back to job-info.lookup and return:
+             */
+            flux_future_destroy (f);
+            flux_log (inv->ctx->h,
+                      LOG_DEBUG,
+                      "no support for %s in parent, falling back to %s",
+                      service,
+                      "job-info.lookup");
+            return lookup_R_fallback (inv, id);
+        }
+        else {
+            flux_log_error (h, "lookup R from enclosing instance KVS");
+            goto done;
+        }
+    }
+    if (convert_R (h, job_R, inv->ctx->size, &R) < 0) {
+        flux_log (h, LOG_ERR, "fatal error while normalizing R");
+        errno = EINVAL;
+        goto done;
+    }
+    flux_future_reset (f);
+    inv->R_watch_f = f;
+    if (R) { // R = NULL if no conversion possible (fall through to discovery)
+        if (inventory_put (inv, R, "job-info") < 0)
+            goto done;
+        if (flux_future_then (f,
+                              -1.,
+                              R_update_cb,
+                              inv) < 0)
+            flux_log (h, LOG_ERR, "Failed to register callback for R updates");
+    }
+    rc = 0;
+done:
+    /* Cancel and destroy R watch future if no R obtained through this means
+     */
+    if (!R) {
+        R_watch_destroy (inv);
+        resource_parent_handle_close (inv->ctx);
+        inv->parent_h = NULL;
+    }
+    json_decref (R);
+    return rc;
+}
+
+static int get_from_kvs (struct inventory *inv, const char *key)
+{
+    flux_future_t *f;
+    json_t *o;
+    int rc = -1;
+
+    if (!(f = flux_kvs_lookup (inv->ctx->h, NULL, 0, key)))
+        return -1;
+    if (flux_kvs_lookup_get_unpack (f, "o", &o) < 0) {
+        if (errno == ENOENT)
+            rc = 0;
+        goto done;
+    }
+    if (inventory_put (inv, o, "kvs") < 0)
+        goto done;
+    rc = 0;
+done:
+    flux_future_destroy (f);
+    return rc;
+}
+
+static void resource_get (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
+{
+    struct inventory *inv = arg;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (!inv->R) {
+        errno = ENOENT;
+        goto error;
+    }
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:O s:s}",
+                           "R",
+                           inv->R,
+                           "method",
+                           inv->method) < 0)
+        flux_log_error (h, "error responding to resource.get request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to resource.get request");
+}
+
+static int get_from_upstream (struct inventory *inv)
+{
+    flux_t *h = inv->ctx->h;
+    flux_future_t *f;
+    json_t *R;
+    const char *method;
+    int rc = -1;
+
+    if (!(f = flux_rpc (h,"resource.get", NULL, FLUX_NODEID_UPSTREAM, 0)))
+        return -1;
+    if (flux_rpc_get_unpack (f,
+                             "{s:o s:s}",
+                             "R",
+                             &R,
+                             "method",
+                             &method) < 0) {
+        if (errno == ENOENT)
+            rc = 0;
+        goto done;
+    }
+    else {
+        if (!(inv->method = strdup (method)))
+            goto done;
+        inv->R = json_incref (R);
+    }
+    rc = 0;
+done:
+    flux_future_destroy (f);
+    return rc;
+}
+
+static int rank_from_key (const char *key)
+{
+    char *endptr;
+    int rank;
+
+    errno = 0;
+    rank = strtoul (key, &endptr, 10);
+    if (errno != 0 || *endptr != '\0')
+        return -1;
+    return rank;
+}
+
+static json_t *resobj_from_xml (json_t *xml)
+{
+    const char *key;
+    json_t *value;
+    const char *s;
+    int rank;
+    struct rlist *rl = NULL;
+    struct rlist *rl2;
+    json_t *R;
+
+    json_object_foreach (xml, key, value) {
+        if ((rank = rank_from_key (key)) < 0)
+            goto error;
+        if (!(s = json_string_value (value)) || strlen (s) == 0)
+            goto error;
+        if (!(rl2 = rlist_from_hwloc (rank, s)))
+            goto error;
+        if (rl) {
+            if (rlist_append (rl, rl2) < 0) {
+                rlist_destroy (rl2);;
+                goto error;
+            }
+            rlist_destroy (rl2);
+        }
+        else
+            rl = rl2;
+    }
+    if (!(R = rlist_to_R (rl)))
+        goto error;
+    rlist_destroy (rl);
+    return R;
+error:
+    errno = EINVAL;
+    rlist_destroy (rl);
+    return NULL;
+}
+
+static int resobj_check_ranks (json_t *resobj, int size)
+{
+    json_error_t e;
+    struct rlist *rl;
+    struct idset *ids = NULL;
+    unsigned long last;
+    int rc = -1;
+
+    if (!(rl = rlist_from_json (resobj, &e)))
+        goto done;
+    if (!(ids = rlist_ranks (rl)))
+        goto done;
+    last = idset_last (ids);
+    if (last != IDSET_INVALID_ID && last >= size)
+        goto done;
+    rc = 0;
+done:
+    idset_destroy (ids);
+    rlist_destroy (rl);
+    return rc;
+}
+
+int inventory_get_size (struct inventory *inv)
+{
+    struct rlist *rl = NULL;
+    struct idset *ids = NULL;
+    int count = 0;
+
+    if (inv != NULL
+        && inv->R != NULL
+        && (rl = rlist_from_json (inv->R, NULL))
+        && (ids = rlist_ranks (rl))) {
+        count = idset_count (ids);
+    }
+    idset_destroy (ids);
+    rlist_destroy (rl);
+    return count;
+}
+
+static void resource_reload (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
+{
+    struct inventory *inv = arg;
+    flux_error_t error;
+    const char *errstr = NULL;
+    const char *path;
+    int xml_flag;
+    int force_flag;
+    json_t *resobj = NULL;
+    json_t *xml = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s:b s:b}",
+                             "path",
+                             &path,
+                             "xml",
+                             &xml_flag,
+                             "force",
+                             &force_flag) < 0)
+        goto error;
+    if (inv->ctx->rank != 0) {
+        errno = ENOSYS;
+        errstr = "resource.reload is only available on rank 0";
+        goto error;
+    }
+    if (xml_flag) {
+        if (!(xml = rutil_load_xml_dir (path, &error))) {
+            errstr = error.text;
+            goto error;
+        }
+
+        if (!(resobj = resobj_from_xml (xml))) {
+            errprintf (&error,
+                       "error building R from hwloc XML: %s",
+                       strerror (errno));
+            errstr = error.text;
+            goto error;
+        }
+    }
+    else {
+        if (!(resobj = rutil_load_file (path, &error))) {
+            errstr = error.text;
+            goto error;
+        }
+    }
+    if (resobj_check_ranks (resobj, inv->ctx->size) < 0) {
+        if (force_flag) {
+            flux_log (inv->ctx->h,
+                      LOG_ERR,
+                      "WARN: resource object contains ranks exceeding size=%d",
+                      (int)inv->ctx->size);
+        }
+        else {
+            errprintf (&error,
+                       "resource object contains ranks execeeding size=%d %s",
+                       (int)inv->ctx->size,
+                       "(override with -f))");
+            errstr = error.text;
+            errno = EINVAL;
+            goto error;
+        }
+    }
+    if (acquire_clients (inv->ctx->acquire) > 0) {
+        errno = EBUSY;
+        errstr = "resources are busy (unload scheduler?)";
+        goto error;
+    }
+    if (inv->R) {
+        json_decref (inv->R);
+        inv->R = NULL;
+        free (inv->method);
+        inv->method = NULL;
+    }
+    if (inventory_put (inv, resobj, "reload") < 0)
+        goto error;
+    if (inventory_put_finalize (inv) < 0)
+        goto error;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to resource.reload request");
+    json_decref (resobj);
+    json_decref (xml);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to resource.reload request");
+    json_decref (resobj);
+    json_decref (xml);
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST, "resource.reload",  resource_reload, 0 },
+    { FLUX_MSGTYPE_REQUEST, "resource.get",  resource_get, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void inventory_destroy (struct inventory *inv)
+{
+    if (inv) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (inv->handlers);
+        json_decref (inv->R);
+        free (inv->method);
+        flux_future_destroy (inv->put_f);
+        R_watch_destroy (inv);
+        resource_parent_handle_close (inv->ctx);
+        free (inv);
+        errno = saved_errno;
+    }
+}
+
+struct inventory *inventory_create (struct resource_ctx *ctx,
+                                    json_t *conf_R,
+                                    bool no_update_watch)
+{
+    struct inventory *inv;
+    json_t *R = NULL;
+
+    if (!(inv = calloc (1, sizeof (*inv))))
+        return NULL;
+    inv->ctx = ctx;
+    if (flux_msg_handler_addvec (ctx->h, htab, inv, &inv->handlers) < 0)
+        goto error;
+    if (conf_R && convert_R_conf (ctx->h, conf_R, &R) < 0)
+        goto error;
+    if (ctx->rank == 0) {
+        if (R && inventory_put (inv, R, "configuration") < 0)
+            goto error;
+        if (!inv->R && get_from_kvs (inv, "resource.R") < 0)
+            goto error;
+        if (!inv->R && start_resource_watch (inv, no_update_watch) < 0)
+            goto error;
+    }
+    else {
+        if (R)
+            inv->R = json_incref (R);
+        if (!inv->R && get_from_upstream (inv) < 0)
+            goto error;
+    }
+    /* If inv->R is NULL after all that, dynamic discovery occurs.
+     */
+    json_decref (R);
+    return inv;
+error:
+    ERRNO_SAFE_WRAP (json_decref, R);
+    inventory_destroy (inv);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/inventory.h b/src/modules/resource/inventory.h
new file mode 100644
index 000000000000..db226cb22eed
--- /dev/null
+++ b/src/modules/resource/inventory.h
@@ -0,0 +1,61 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_INVENTORY_H
+#define _FLUX_RESOURCE_INVENTORY_H
+
+/* Create resource inventory.
+ * R is configured resource object, if any (ref taken).
+ * R is obtained from enclosing Flux instance or probed dynamically otherwise.
+ */
+struct inventory *inventory_create (struct resource_ctx *ctx,
+                                    json_t *R,
+                                    bool no_update_watch);
+void inventory_destroy (struct inventory *inv);
+
+/* Get resource object.
+ * Returned resource object shall not be modified or freed by the caller.
+ */
+json_t *inventory_get (struct inventory *inv);
+
+/* Get the method used to construct the resource object.
+ * (NULL if resource object is unavailable, errno set)
+ */
+const char *inventory_get_method (struct inventory *inv);
+
+/* Set resource object from internal discovery.
+ * Takes a reference on 'R' but does not copy.
+ * Caller shall not modify R after calling this function.
+ * This triggers writing of resource.R to the KVS, and posting of
+ * 'resource-define' to resource.eventlog.  The KVS commits are asynchronous.
+ */
+int inventory_put (struct inventory *inv, json_t *R, const char *method);
+
+/* Return a set of ranks for a string of "targets". The 'targets' argument
+ * may be an RFC22 encoded idset or RFC29 hostlist. If an idset, the
+ * decoded idset is returned, if a hostlist, then the set of ranks
+ * corresponding to the hostnames in 'targets' is returned.
+ *
+ * On error, a textual error string will be returned in errbuf
+ */
+struct idset *inventory_targets_to_ranks (struct inventory *inv,
+                                          const char *targets,
+                                          flux_error_t *errp);
+
+/* Get the number of execution targets in R
+ */
+int inventory_get_size (struct inventory *inv);
+
+#endif /* !_FLUX_RESOURCE_INVENTORY_H */
+
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/monitor.c b/src/modules/resource/monitor.c
new file mode 100644
index 000000000000..221e6db4dff2
--- /dev/null
+++ b/src/modules/resource/monitor.c
@@ -0,0 +1,374 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* monitor.c - track execution targets joining/leaving the instance
+ *
+ * Watches the broker.online group and posts online/offline events as
+ * the broker.online set changes.
+ *
+ * The initial online set used in the restart event will be empty as
+ * the initial response to the request to watch broker.online cannot
+ * be processed until the reactor runs.
+ *
+ * Some synchronization notes:
+ * - rc1 completes on rank 0 before any other ranks can join broker.online,
+ *   therefore the scheduler must allow flux module load to complete with
+ *   potentially all node resources offline, or deadlock will result.
+ * - it is racy to read broker.online and assume that online events have
+ *   been posted for those ranks, as the resource module needs time to
+ *   receive notification from the broker and process it.
+ * - the initial program starts once broker.online reaches the configured
+ *   quorum (all ranks unless configured otherwise, e.g. system instance).
+ *   It is racy to assume that online events have been posted for the quorum
+ *   ranks in the initial program for the same reason as above.
+ * - the 'resource.monitor-waitup' RPC allows a test to wait for some number
+ *   of ranks to be up, where "up" is defined as having had an online event
+ *   posted.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libidset/idset.h"
+#include "src/common/libutil/errno_safe.h"
+
+#include "resource.h"
+#include "reslog.h"
+#include "monitor.h"
+#include "rutil.h"
+
+struct monitor {
+    struct resource_ctx *ctx;
+    flux_future_t *f_online;
+    flux_future_t *f_torpid;
+    struct idset *torpid;
+    struct idset *up;
+    struct idset *down; // cached result of monitor_get_down()
+    flux_msg_handler_t **handlers;
+    struct flux_msglist *waitup_requests;
+    int size;
+};
+
+static void notify_waitup (struct monitor *monitor);
+
+const struct idset *monitor_get_up (struct monitor *monitor)
+{
+    return monitor->up;
+}
+
+const struct idset *monitor_get_torpid (struct monitor *monitor)
+{
+    return monitor->torpid;
+}
+
+const struct idset *monitor_get_down (struct monitor *monitor)
+{
+    unsigned int id;
+
+    if (!monitor->down) {
+        if (!(monitor->down = idset_create (monitor->size, 0)))
+            return NULL;
+    }
+    for (id = 0; id < monitor->size; id++) {
+        if (idset_test (monitor->up, id))
+            (void)idset_clear (monitor->down, id);
+        else
+            (void)idset_set (monitor->down, id);
+    }
+    return monitor->down;
+}
+
+/* Send a streaming groups.get RPC for broker group 'name'.
+ */
+static flux_future_t *group_monitor (flux_t *h, const char *name)
+{
+    return flux_rpc_pack (h,
+                          "groups.get",
+                          FLUX_NODEID_ANY,
+                          FLUX_RPC_STREAMING,
+                          "{s:s}",
+                          "name", name);
+}
+
+/* Handle a response to the group monitor request, parsing the
+ * encoded idset in the payload.
+ */
+static struct idset *group_get (flux_future_t *f)
+{
+    const char *members;
+    if (flux_rpc_get_unpack (f, "{s:s}", "members", &members) < 0)
+        return NULL;
+    return idset_decode (members);
+}
+
+/* Post event 'name' with a context containing idset:s, where 's' is
+ * the string encoding of 'ids'.  The event is not propagated to the KVS.
+ */
+static int post_event (struct monitor *monitor,
+                       const char *name,
+                       struct idset *ids)
+{
+    char *s = NULL;
+
+    if (idset_count (ids) == 0)
+        return 0;
+    if (!(s = idset_encode (ids, IDSET_FLAG_RANGE))
+        || reslog_post_pack (monitor->ctx->reslog,
+                             NULL,
+                             0.,
+                             name,
+                             EVENT_NO_COMMIT,
+                             "{s:s}",
+                             "idset", s) < 0) {
+        ERRNO_SAFE_WRAP (free, s);
+        return -1;
+    }
+    free (s);
+    return 0;
+}
+
+/* Post 'join_event' and/or 'leave_event' to record ids added or removed
+ * in 'newset' relative to 'oldset'.
+ */
+static int post_join_leave (struct monitor *monitor,
+                            const struct idset *oldset,
+                            const struct idset *newset,
+                            const char *join_event,
+                            const char *leave_event)
+{
+    struct idset *join;
+    struct idset *leave = NULL;
+    int rc = -1;
+
+    if (!(join = idset_difference (newset, oldset))
+        || !(leave = idset_difference (oldset, newset))
+        || post_event (monitor, join_event, join) < 0
+        || post_event (monitor, leave_event, leave) < 0)
+        goto error;
+    rc = 0;
+error:
+    idset_destroy (join);
+    idset_destroy (leave);
+    return rc;
+}
+
+/* Leader: set of online brokers has changed.
+ * Update monitor->up and post online/offline events to resource.eventlog.
+ * Avoid posting events if nothing changed.
+ */
+static void broker_online_cb (flux_future_t *f, void *arg)
+{
+    struct monitor *monitor = arg;
+    flux_t *h = monitor->ctx->h;
+    struct idset *up = NULL;
+
+    if (!(up = group_get (f))) {
+        flux_log (h,
+                  LOG_ERR,
+                  "monitor: broker.online: %s",
+                  future_strerror (f, errno));
+        return;
+    }
+    if (post_join_leave (monitor, monitor->up, up, "online", "offline") < 0) {
+        flux_log_error (h, "monitor: error posting online/offline event");
+        idset_destroy (up);
+        flux_future_reset (f);
+        return;
+    }
+
+    idset_destroy (monitor->up);
+    monitor->up = up;
+
+    notify_waitup (monitor);
+
+    flux_future_reset (f);
+}
+
+static void broker_torpid_cb (flux_future_t *f, void *arg)
+{
+    struct monitor *monitor = arg;
+    flux_t *h = monitor->ctx->h;
+    struct idset *torpid = NULL;
+
+    if (!(torpid = group_get (f))) {
+        flux_log (h,
+                  LOG_ERR,
+                  "monitor: broker.torpid: %s",
+                  future_strerror (f, errno));
+        return;
+    }
+    if (post_join_leave (monitor,
+                         monitor->torpid,
+                         torpid,
+                         "torpid",
+                         "lively") < 0) {
+        flux_log_error (h, "monitor: error posting torpid/lively event");
+        idset_destroy (torpid);
+        flux_future_reset (f);
+        return;
+    }
+
+    idset_destroy (monitor->torpid);
+    monitor->torpid = torpid;
+
+    flux_future_reset (f);
+}
+
+static void notify_waitup (struct monitor *monitor)
+{
+    const flux_msg_t *msg;
+    int upcount = idset_count (monitor->up);
+
+    msg = flux_msglist_first (monitor->waitup_requests);
+    while (msg) {
+        int upwant;
+        int rc;
+        if (flux_request_unpack (msg, NULL, "{s:i}", "up", &upwant) < 0)
+            rc = flux_respond_error (monitor->ctx->h, msg, errno, NULL);
+        else if (upwant == upcount)
+            rc = flux_respond (monitor->ctx->h, msg, NULL);
+        else
+            goto next;
+        if (rc < 0)
+            flux_log_error (monitor->ctx->h,
+                            "error responding to monitor-waitup request");
+        flux_msglist_delete (monitor->waitup_requests);
+next:
+        msg = flux_msglist_next (monitor->waitup_requests);
+    }
+}
+
+static void waitup_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    struct monitor *monitor = arg;
+    const char *errstr = NULL;
+    int up;
+
+    if (flux_request_unpack (msg, NULL, "{s:i}", "up", &up) < 0)
+        goto error;
+    if (monitor->ctx->rank != 0) {
+        errno = EPROTO;
+        errstr = "this RPC only works on rank 0";
+        goto error;
+    }
+    if (up > monitor->size || up < 0) {
+        errno = EPROTO;
+        errstr = "up value is out of range";
+        goto error;
+    }
+    if (idset_count (monitor->up) != up) {
+        if (flux_msglist_append (monitor->waitup_requests, msg) < 0)
+            goto error;
+        return; // response deferred
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to monitor-waitup request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to monitor-waitup request");
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,  "resource.monitor-waitup", waitup_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void monitor_destroy (struct monitor *monitor)
+{
+    if (monitor) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (monitor->handlers);
+        idset_destroy (monitor->up);
+        idset_destroy (monitor->down);
+        idset_destroy (monitor->torpid);
+        flux_future_destroy (monitor->f_online);
+        flux_future_destroy (monitor->f_torpid);
+        flux_msglist_destroy (monitor->waitup_requests);
+        free (monitor);
+        errno = saved_errno;
+    }
+}
+
+struct monitor *monitor_create (struct resource_ctx *ctx,
+                                int inventory_size,
+                                bool monitor_force_up)
+{
+    struct monitor *monitor;
+
+    if (!(monitor = calloc (1, sizeof (*monitor))))
+        return NULL;
+    monitor->ctx = ctx;
+    /* In recovery mode, if the instance was started by PMI, the size of
+     * the recovery instance will be 1 but the resource inventory size may be
+     * larger.  Up/down sets should be built with the inventory size in this
+     * case.  However, we cannot unconditionally use the inventory size, since
+     * it will be zero at this point if resources are being dynamically
+     * discovered, e.g. when Flux is launched by a foreign resource manager.
+     */
+    monitor->size = ctx->size;
+    if (monitor->size < inventory_size)
+        monitor->size = inventory_size;
+
+    if (flux_msg_handler_addvec (ctx->h, htab, monitor, &monitor->handlers) < 0)
+        goto error;
+
+    /* Monitor currently doesn't do anything on follower ranks,
+     * except respond to RPCs with a human readable error.
+     */
+    if (ctx->rank > 0)
+        goto done;
+
+    if (!(monitor->waitup_requests = flux_msglist_create ()))
+        goto error;
+
+    /* Initialize up to the empty set unless 'monitor_force_up' is true.
+     * N.B. Initial up value will appear in 'restart' event posted
+     * to resource.eventlog.
+     */
+    if (!(monitor->up = idset_create (monitor->size, 0))
+        || !(monitor->torpid = idset_create (monitor->size, 0)))
+        goto error;
+    if (monitor_force_up) {
+        if (idset_range_set (monitor->up, 0, monitor->size - 1) < 0)
+            goto error;
+    }
+    else if (!flux_attr_get (ctx->h, "broker.recovery-mode")) {
+        if (!(monitor->f_online = group_monitor (ctx->h, "broker.online"))
+            || flux_future_then (monitor->f_online,
+                                 -1,
+                                 broker_online_cb,
+                                 monitor) < 0)
+            goto error;
+        if (!(monitor->f_torpid = group_monitor (ctx->h, "broker.torpid"))
+            || flux_future_then (monitor->f_torpid,
+                                 -1,
+                                 broker_torpid_cb,
+                                 monitor) < 0)
+            goto error;
+    }
+done:
+    return monitor;
+error:
+    monitor_destroy (monitor);
+    return NULL;
+}
+
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/monitor.h b/src/modules/resource/monitor.h
new file mode 100644
index 000000000000..869513d2fb97
--- /dev/null
+++ b/src/modules/resource/monitor.h
@@ -0,0 +1,28 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_MONITOR_H
+#define _FLUX_RESOURCE_MONITOR_H
+
+struct monitor *monitor_create (struct resource_ctx *ctx,
+                                int inventory_size,
+                                bool monitor_force_up);
+void monitor_destroy (struct monitor *monitor);
+
+const struct idset *monitor_get_down (struct monitor *monitor);
+const struct idset *monitor_get_up (struct monitor *monitor);
+
+const struct idset *monitor_get_torpid (struct monitor *monitor);
+
+#endif /* !_FLUX_RESOURCE_MONITOR_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/reslog.c b/src/modules/resource/reslog.c
new file mode 100644
index 000000000000..649453ff7a34
--- /dev/null
+++ b/src/modules/resource/reslog.c
@@ -0,0 +1,274 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libeventlog/eventlog.h"
+
+#include "reslog.h"
+
+struct reslog_watcher {
+    reslog_cb_f cb;
+    void *arg;
+};
+
+struct event_info {
+    json_t *event;          // JSON form of event
+    const flux_msg_t *msg;  // optional request to be answered on commit
+};
+
+struct reslog {
+    flux_t *h;
+    zlist_t *pending;       // list of pending futures
+    zlist_t *watchers;
+};
+
+static const char *auxkey = "flux::event_info";
+
+/* zlist_compare_fn() footprint */
+static int watcher_compare (void *item1, void *item2)
+{
+    struct reslog_watcher *w1 = item1;
+    struct reslog_watcher *w2 = item2;
+    if (w1 && w2 && w1->cb == w2->cb)
+        return 0;
+    return -1;
+}
+
+/* Call registered callbacks, if any, with the event name that just completed.
+ */
+static void notify_callbacks (struct reslog *reslog, json_t *event)
+{
+    const char *name;
+    json_t *context;
+    struct reslog_watcher *w;
+
+    if (json_unpack (event,
+                     "{s:s s:o}",
+                     "name", &name,
+                     "context", &context) < 0) {
+        flux_log (reslog->h, LOG_ERR, "error unpacking event for callback");
+        return;
+    }
+    w = zlist_first (reslog->watchers);
+    while (w) {
+        if (w->cb)
+            w->cb (reslog, name, context, w->arg);
+        w = zlist_next (reslog->watchers);
+    }
+}
+
+static void event_info_destroy (struct event_info *info)
+{
+    if (info) {
+        json_decref (info->event);
+        flux_msg_decref (info->msg);
+        ERRNO_SAFE_WRAP (free, info);
+    }
+}
+
+static struct event_info *event_info_create (json_t *event,
+                                             const flux_msg_t *request)
+{
+    struct event_info *info;
+
+    if (!(info = calloc (1, sizeof (*info))))
+        return NULL;
+    info->event = json_incref (event);
+    info->msg = flux_msg_incref (request);
+    return info;
+}
+
+int post_handler (struct reslog *reslog, flux_future_t *f)
+{
+    struct event_info *info = flux_future_aux_get (f, auxkey);
+    int rc;
+
+    if ((rc = flux_future_get (f, NULL)) < 0) {
+        flux_log_error (reslog->h, "committing to %s", RESLOG_KEY);
+        if (info->msg) {
+            if (flux_respond_error (reslog->h, info->msg, errno, NULL) < 0)
+                flux_log_error (reslog->h, "responding to request after post");
+        }
+        goto done;
+    }
+    else {
+        if (info->msg) {
+            if (flux_respond (reslog->h, info->msg, NULL) < 0)
+                flux_log_error (reslog->h, "responding to request after post");
+        }
+    }
+    notify_callbacks (reslog, info->event);
+done:
+    zlist_remove (reslog->pending, f);
+    flux_future_destroy (f);
+
+    if ((f = zlist_first (reslog->pending))
+        && (info = flux_future_aux_get (f, auxkey))
+        && info->msg == NULL)
+        flux_future_fulfill (f, NULL, NULL);
+
+    return rc;
+}
+
+static void post_continuation (flux_future_t *f, void *arg)
+{
+    struct reslog *reslog = arg;
+
+    (void)post_handler (reslog, f);
+}
+
+int reslog_sync (struct reslog *reslog)
+{
+    flux_future_t *f;
+    while ((f = zlist_pop (reslog->pending))) {
+        if (post_handler (reslog, f) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+int reslog_post_pack (struct reslog *reslog,
+                      const flux_msg_t *request,
+                      double timestamp,
+                      const char *name,
+                      int flags,
+                      const char *fmt,
+                      ...)
+{
+    va_list ap;
+    json_t *event;
+    char *val = NULL;
+    flux_kvs_txn_t *txn = NULL;
+    flux_future_t *f = NULL;
+    struct event_info *info;
+
+    va_start (ap, fmt);
+    event = eventlog_entry_vpack (timestamp, name, fmt, ap);
+    va_end (ap);
+
+    if (!event)
+        return -1;
+    if ((flags & EVENT_NO_COMMIT)) {
+        if (!(f = flux_future_create (NULL, NULL)))
+            goto error;
+        flux_future_set_flux (f, reslog->h);
+        if (zlist_size (reslog->pending) == 0)
+            flux_future_fulfill (f, NULL, NULL);
+    }
+    else {
+        if (!(val = eventlog_entry_encode (event)))
+            goto error;
+        if (!(txn = flux_kvs_txn_create ()))
+            goto error;
+        if (flux_kvs_txn_put (txn, FLUX_KVS_APPEND, RESLOG_KEY, val) < 0)
+            goto error;
+        if (!(f = flux_kvs_commit (reslog->h, NULL, 0, txn)))
+            goto error;
+    }
+    if (!(info = event_info_create (event, request)))
+        goto error;
+    if (flux_future_aux_set (f,
+                             auxkey,
+                             info,
+                             (flux_free_f)event_info_destroy) < 0) {
+        event_info_destroy (info);
+        goto error;
+    }
+    if (flux_future_then (f, -1, post_continuation, reslog) < 0)
+        goto error;
+    if (zlist_append (reslog->pending, f) < 0)
+        goto nomem;
+    free (val);
+    flux_kvs_txn_destroy (txn);
+    json_decref (event);
+    return 0;
+nomem:
+    errno = ENOMEM;
+error:
+    flux_future_destroy (f);
+    flux_kvs_txn_destroy (txn);
+    ERRNO_SAFE_WRAP (free, val);
+    ERRNO_SAFE_WRAP (json_decref, event);
+    return -1;
+}
+
+void reslog_remove_callback (struct reslog *reslog, reslog_cb_f cb, void *arg)
+{
+    if (reslog) {
+        struct reslog_watcher w = { .cb = cb, .arg = arg };
+        zlist_remove (reslog->watchers, &w);
+    }
+}
+
+int reslog_add_callback (struct reslog *reslog, reslog_cb_f cb, void *arg)
+{
+    struct reslog_watcher *w;
+
+    if (!reslog) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(w = calloc (1, sizeof (*w))))
+        return -1;
+    w->cb = cb;
+    w->arg = arg;
+    if (zlist_append (reslog->watchers, w) < 0) {
+        free (w);
+        errno = ENOMEM;
+        return -1;
+    }
+    zlist_freefn (reslog->watchers, w, free, true);
+    return 0;
+}
+
+void reslog_destroy (struct reslog *reslog)
+{
+    if (reslog) {
+        int saved_errno = errno;
+        if (reslog->pending) {
+            flux_future_t *f;
+            while ((f = zlist_pop (reslog->pending)))
+                (void)post_handler (reslog, f);
+            zlist_destroy (&reslog->pending);
+        }
+        zlist_destroy (&reslog->watchers);
+        free (reslog);
+        errno = saved_errno;
+    }
+}
+
+struct reslog *reslog_create (flux_t *h)
+{
+    struct reslog *reslog;
+
+    if (!(reslog = calloc (1, sizeof (*reslog))))
+        return NULL;
+    reslog->h = h;
+    if (!(reslog->pending = zlist_new ())
+        || !(reslog->watchers = zlist_new ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    zlist_comparefn (reslog->watchers, watcher_compare);
+    return reslog;
+error:
+    reslog_destroy (reslog);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/reslog.h b/src/modules/resource/reslog.h
new file mode 100644
index 000000000000..beb6818c832b
--- /dev/null
+++ b/src/modules/resource/reslog.h
@@ -0,0 +1,56 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_RESLOG_H
+#define _FLUX_RESOURCE_RESLOG_H
+
+struct reslog;
+
+enum reslog_flags {
+    EVENT_NO_COMMIT = 1,
+};
+
+typedef void (*reslog_cb_f)(struct reslog *reslog,
+                            const char *name,
+                            json_t *context,
+                            void *arg);
+
+struct reslog *reslog_create (flux_t *h);
+void reslog_destroy (struct reslog *reslog);
+
+/* Post an event to the eventlog.  This function returns immediately,
+ * and the commit to the eventlog completes asynchronously.
+ * If 'request' is non-NULL, a success/fail response is sent upon commit
+ * completion.
+ */
+int reslog_post_pack (struct reslog *reslog,
+                      const flux_msg_t *request,
+                      double timestamp,
+                      const char *name,
+                      int flags,
+                      const char *fmt,
+                      ...);
+
+/* Force all pending commits to the eventlog to complete.
+ */
+int reslog_sync (struct reslog *reslog);
+
+/* Get a callback for each event.
+ */
+int reslog_add_callback (struct reslog *reslog, reslog_cb_f cb, void *arg);
+void reslog_remove_callback (struct reslog *reslog, reslog_cb_f cb, void *arg);
+
+#define RESLOG_KEY "resource.eventlog"
+
+#endif /* !_FLUX_RESOURCE_RESLOG_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/resource.c b/src/modules/resource/resource.c
new file mode 100644
index 000000000000..1553cc199819
--- /dev/null
+++ b/src/modules/resource/resource.c
@@ -0,0 +1,439 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* resource.c - resource discovery and monitoring service
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libidset/idset.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/librlist/rlist.h"
+#include "ccan/str/str.h"
+
+#include "resource.h"
+#include "inventory.h"
+#include "reslog.h"
+#include "topo.h"
+#include "monitor.h"
+#include "drain.h"
+#include "exclude.h"
+#include "acquire.h"
+#include "rutil.h"
+#include "status.h"
+#include "upgrade.h"
+
+/* Parse [resource] table.
+ *
+ * exclude = "targets"
+ *   Exclude specified broker rank(s) or hosts from scheduling
+ *
+ * [[resource.confg]]
+ *   Resource configuration array
+ *
+ * path = "/path"
+ *   Set path to resource object (if no resource.config array)
+ *
+ * noverify = true
+ *   Skip verification that configured resources match local hwloc
+ *
+ * norestrict = false
+ *   When generating hwloc topology XML, do not restrict to current cpumask
+ *
+ * no-update-watch = false
+ *   For testing purposes, simulate missing job-info.update-watch service
+ *   in parent instance by sending to an invalid service name.
+ */
+static int parse_config (struct resource_ctx *ctx,
+                         const flux_conf_t *conf,
+                         const char **excludep,
+                         json_t **R,
+                         bool *noverifyp,
+                         bool *norestrictp,
+                         bool *no_update_watchp,
+                         flux_error_t *errp)
+{
+    flux_error_t error;
+    const char *exclude  = NULL;
+    const char *path = NULL;
+    const char *scheduling_path = NULL;
+    int noverify = 0;
+    int norestrict = 0;
+    int no_update_watch = 0;
+    json_t *o = NULL;
+    json_t *config = NULL;
+
+    if (flux_conf_unpack (conf,
+                          &error,
+                          "{s?{s?s s?s s?o s?s s?b s?b s?b !}}",
+                          "resource",
+                            "path", &path,
+                            "scheduling", &scheduling_path,
+                            "config", &config,
+                            "exclude", &exclude,
+                            "norestrict", &norestrict,
+                            "noverify", &noverify,
+                            "no-update-watch", &no_update_watch) < 0) {
+        errprintf (errp,
+                   "error parsing [resource] configuration: %s",
+                   error.text);
+        return -1;
+    }
+    if (config) {
+        struct rlist *rl = rlist_from_config (config, &error);
+        if (!rl) {
+            errprintf (errp,
+                       "error parsing [resource.config] array: %s",
+                       error.text);
+            return -1;
+        }
+        if (!(o = rlist_to_R (rl)))
+            return errprintf (errp, "rlist_to_R: %s", strerror (errno));
+        rlist_destroy (rl);
+    }
+    else if (path) {
+        json_error_t e;
+        if (!(o = json_load_file (path, 0, &e))) {
+            errprintf (errp,
+                       "%s: %s on line %d",
+                       e.source,
+                       e.text,
+                       e.line);
+            return -1;
+        }
+    }
+    /* resource.scheduling key, if configured, is only required on rank 0,
+     * since by definition it is used only by the scheduler.
+     */
+    if (scheduling_path && ctx->rank == 0) {
+        json_t *scheduling;
+        json_error_t e;
+        if (!o) {
+            errprintf (errp,
+                       "resource.scheduling requires "
+                       "resource.path or [resource.config]");
+            return -1;
+        }
+        if (!(scheduling = json_load_file (scheduling_path, 0, &e))) {
+            errprintf (errp,
+                       "error loading resource.scheduling: %s on line %d",
+                       e.text,
+                       e.line);
+            json_decref (o);
+            return -1;
+        }
+        if (json_object_set_new (o, "scheduling", scheduling) < 0) {
+            errprintf (errp, "failed to set scheduling key in R");
+            json_decref (o);
+            json_decref (scheduling);
+            return -1;
+        }
+    }
+    if (excludep)
+        *excludep = exclude;
+    if (noverifyp)
+        *noverifyp = noverify ? true : false;
+    if (norestrictp)
+        *norestrictp = norestrict ? true : false;
+    if (no_update_watchp)
+        *no_update_watchp = no_update_watch ? true : false;
+    if (R)
+        *R = o;
+    else
+        json_decref (o);
+    return 0;
+}
+
+/* Broker is sending us a new config object because 'flux config reload'
+ * was run.  Parse it and respond with human readable errors.
+ * At the moment this doesn't do much - just cache the new config.
+ */
+static void config_reload_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct resource_ctx *ctx = arg;
+    const flux_conf_t *conf;
+    flux_error_t error;
+    const char *errstr = NULL;
+
+    if (flux_conf_reload_decode (msg, &conf) < 0)
+        goto error;
+    if (parse_config (ctx, conf, NULL, NULL, NULL, NULL, NULL, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (flux_set_conf (h, flux_conf_incref (conf)) < 0) {
+        errstr = "error updating cached configuration";
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+}
+
+/* Handle client disconnect.
+ */
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct resource_ctx *ctx = arg;
+
+    if (ctx->acquire)
+        acquire_disconnect (ctx->acquire, msg);
+    if (ctx->status)
+        status_disconnect (ctx->status, msg);
+}
+
+flux_t *resource_parent_handle_open (struct resource_ctx *ctx)
+{
+    if (!ctx->parent_h) {
+        const char *uri = flux_attr_get (ctx->h, "parent-uri");
+        if (!uri || !flux_attr_get (ctx->h, "jobid")) {
+            errno = ENOENT;
+            return NULL;
+        }
+        if (!(ctx->parent_h = flux_open (uri, 0)))
+            flux_log_error (ctx->h, "error opening %s", uri);
+    }
+    ctx->parent_refcount++;
+    return ctx->parent_h;
+}
+
+void resource_parent_handle_close (struct resource_ctx *ctx)
+{
+    if (ctx && --ctx->parent_refcount == 0) {
+        flux_close (ctx->parent_h);
+        ctx->parent_h = NULL;
+    }
+}
+
+static void resource_ctx_destroy (struct resource_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        status_destroy (ctx->status);
+        acquire_destroy (ctx->acquire);
+        drain_destroy (ctx->drain);
+        topo_destroy (ctx->topology);
+        monitor_destroy (ctx->monitor);
+        exclude_destroy (ctx->exclude);
+        reslog_destroy (ctx->reslog);
+        inventory_destroy (ctx->inventory);
+        flux_msg_handler_delvec (ctx->handlers);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static struct resource_ctx *resource_ctx_create (flux_t *h)
+{
+    struct resource_ctx *ctx;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+    ctx->h = h;
+    return ctx;
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        .typemask = FLUX_MSGTYPE_REQUEST,
+        .topic_glob = "resource.config-reload",
+        .cb = config_reload_cb,
+        .rolemask = 0
+    },
+    {
+        .typemask = FLUX_MSGTYPE_REQUEST,
+        .topic_glob = "resource.disconnect",
+        .cb = disconnect_cb,
+        .rolemask = FLUX_ROLE_USER
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+/* Synchronously read resource.eventlog, and parse into
+ * a JSON array for replay by the various subsystems.
+ * 'eventlog' is set to NULL if it doesn't exist (no error).
+ */
+static int reload_eventlog (flux_t *h, json_t **eventlog)
+{
+    flux_future_t *f;
+    const char *s;
+    json_t *o;
+
+    if (!(f = flux_kvs_lookup (h, NULL, 0, RESLOG_KEY)))
+        return -1;
+    if (flux_kvs_lookup_get (f, &s) < 0) {
+        if (errno != ENOENT) {
+            flux_log_error (h, "%s: lookup error", RESLOG_KEY);
+            goto error;
+        }
+        o = NULL;
+    }
+    else {
+        if (!(o = eventlog_decode (s))) {
+            flux_log (h, LOG_ERR, "%s: decode error", RESLOG_KEY);
+            goto error;
+        }
+    }
+    *eventlog = o;
+    flux_future_destroy (f);
+    return 0;
+error:
+    flux_future_destroy (f);
+    return -1;
+}
+
+int parse_args (flux_t *h,
+                int argc,
+                char **argv,
+                bool *monitor_force_up,
+                bool *noverify,
+                bool *no_update_watch)
+{
+    int i;
+    for (i = 0; i < argc; i++) {
+        /* Test option to force all ranks to be marked online in the initial
+         * 'restart' event posted to resource.eventlog.
+         */
+        if (streq (argv[i], "monitor-force-up"))
+            *monitor_force_up = true;
+        else if (streq (argv[i], "noverify"))
+            *noverify = true;
+        else  {
+            flux_log (h, LOG_ERR, "unknown option: %s", argv[i]);
+            errno = EINVAL;
+            return -1;
+        }
+    }
+    return 0;
+}
+
+
+int mod_main (flux_t *h, int argc, char **argv)
+{
+    struct resource_ctx *ctx;
+    flux_error_t error;
+    const char *exclude_idset;
+    json_t *eventlog = NULL;
+    bool monitor_force_up = false;
+    bool noverify = false;
+    bool norestrict = false;
+    bool no_update_watch = false;
+    json_t *R_from_config;
+
+    if (!(ctx = resource_ctx_create (h)))
+        goto error;
+    if (flux_get_size (h, &ctx->size) < 0)
+        goto error;
+    if (flux_get_rank (h, &ctx->rank) < 0)
+        goto error;
+    if (parse_config (ctx,
+                      flux_get_conf (h),
+                      &exclude_idset,
+                      &R_from_config,
+                      &noverify,
+                      &norestrict,
+                      &no_update_watch,
+                      &error) < 0) {
+        flux_log (h, LOG_ERR, "%s", error.text);
+        goto error;
+    }
+    if (parse_args (h,
+                    argc,
+                    argv,
+                    &monitor_force_up,
+                    &noverify,
+                    &no_update_watch) < 0)
+        goto error;
+    if (flux_attr_get (ctx->h, "broker.recovery-mode"))
+        noverify = true;
+
+    /*  Note: Order of creation of resource subsystems is important.
+     *  Create inventory on all ranks first, since it is required by
+     *  the exclude and drain subsystems on rank 0.
+     */
+    if (!(ctx->inventory = inventory_create (ctx,
+                                             R_from_config,
+                                             no_update_watch)))
+        goto error;
+    /*  Done with R_from_config now, so free it.
+     */
+    json_decref (R_from_config);
+    if (ctx->rank == 0) {
+        /*  Create reslog and reload eventlog before initializing
+         *  acquire, exclude, and drain subsystems, since these
+         *  are required by acquire and exclude.
+         */
+        if (!(ctx->reslog = reslog_create (h)))
+            goto error;
+        if (reload_eventlog (h, &eventlog) < 0)
+            goto error;
+        /* One time only: purge the eventlog (including KVS) of
+         * pre-0.62.0 events, upgrading drain events with hostnames.
+         * See flux-framework/flux-core#5931.
+         */
+        if (upgrade_eventlog (h, &eventlog) < 0)
+            goto error;
+        if (!(ctx->acquire = acquire_create (ctx)))
+            goto error;
+
+        /*  Initialize exclude subsystem before drain since drain uses
+         *  the exclude idset to ensure drained ranks that are now
+         *  excluded are ignored.
+         */
+        if (!(ctx->exclude = exclude_create (ctx, exclude_idset)))
+            goto error;
+        if (!(ctx->drain = drain_create (ctx, eventlog)))
+            goto error;
+    }
+    /*  topology is initialized after exclude/drain etc since this
+     *  rank may attempt to drain itself due to a topology mismatch.
+     */
+    if (!(ctx->topology = topo_create (ctx, noverify, norestrict)))
+        goto error;
+    if (!(ctx->monitor = monitor_create (ctx,
+                                         inventory_get_size (ctx->inventory),
+                                         monitor_force_up)))
+        goto error;
+    if (!(ctx->status = status_create (ctx)))
+        goto error;
+    if (flux_msg_handler_addvec (h, htab, ctx, &ctx->handlers) < 0)
+        goto error;
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
+        flux_log_error (h, "flux_reactor_run");
+        goto error;
+    }
+    resource_ctx_destroy (ctx);
+    json_decref (eventlog);
+    return 0;
+error:
+    resource_ctx_destroy (ctx);
+    ERRNO_SAFE_WRAP (json_decref, eventlog);
+    return -1;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/resource.h b/src/modules/resource/resource.h
new file mode 100644
index 000000000000..1766dcc6d127
--- /dev/null
+++ b/src/modules/resource/resource.h
@@ -0,0 +1,47 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_H
+#define _FLUX_RESOURCE_H
+
+struct resource_ctx {
+    flux_t *h;
+    flux_msg_handler_t **handlers;
+    struct inventory *inventory;
+    struct monitor *monitor;
+    struct topo *topology;
+    struct drain *drain;
+    struct exclude *exclude;
+    struct acquire *acquire;
+    struct reslog *reslog;
+    struct status *status;
+
+    flux_t *parent_h;
+    int parent_refcount;
+
+    uint32_t rank;
+    uint32_t size;
+};
+
+/*  Get a shared handle to he parent instance if the parent-uri attribute
+ *  is set. Adds a reference to the shared parent handle. Caller must call
+ *  resource_parent_handle_close().
+ *
+ *  Returns NULL on error with errno set to ENOENT if there is no parent-uri,
+ *  or error from flux_open(3).
+ */
+flux_t *resource_parent_handle_open (struct resource_ctx *ctx);
+void resource_parent_handle_close (struct resource_ctx *ctx);
+
+#endif /* !_FLUX_RESOURCE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/rutil.c b/src/modules/resource/rutil.c
new file mode 100644
index 000000000000..88bdd369887f
--- /dev/null
+++ b/src/modules/resource/rutil.c
@@ -0,0 +1,272 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* rutil.c - random standalone helper functions */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libidset/idset.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/dirwalk.h"
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "rutil.h"
+
+int rutil_idset_diff (const struct idset *ids1,
+                      const struct idset *ids2,
+                      struct idset **addp,
+                      struct idset **subp)
+{
+    struct idset *add = NULL;
+    struct idset *sub = NULL;
+    unsigned int id;
+
+    if (!addp || !subp) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (ids1) { // find ids in ids1 but not in ids2, and add to 'sub'
+        id = idset_first (ids1);
+        while (id != IDSET_INVALID_ID) {
+            if (!ids2 || !idset_test (ids2, id)) {
+                if (!sub && !(sub = idset_create (0, IDSET_FLAG_AUTOGROW)))
+                    goto error;
+                if (idset_set (sub, id) < 0)
+                    goto error;
+            }
+            id = idset_next (ids1, id);
+        }
+    }
+    if (ids2) { // find ids in ids2 but not in ids1, and add to 'add'
+        id = idset_first (ids2);
+        while (id != IDSET_INVALID_ID) {
+            if (!ids1 || !idset_test (ids1, id)) {
+                if (!add && !(add = idset_create (0, IDSET_FLAG_AUTOGROW)))
+                    goto error;
+                if (idset_set (add, id) < 0)
+                    goto error;
+            }
+            id = idset_next (ids2, id);
+        }
+    }
+    *addp = add;
+    *subp = sub;
+    return 0;
+error:
+    idset_destroy (add);
+    idset_destroy (sub);
+    return -1;
+}
+
+int rutil_set_json_idset (json_t *o, const char *key, const struct idset *ids)
+{
+    json_t *val = NULL;
+    char *s = NULL;
+
+    if (o == NULL || key == NULL || strlen (key) == 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (ids && !(s = idset_encode (ids, IDSET_FLAG_RANGE)))
+        return -1;
+    if (!(val = json_string (s ? s : "")))
+        goto nomem;
+    if (json_object_set_new (o, key, val) < 0)
+        goto nomem;
+    free (s);
+    return 0;
+nomem:
+    errno = ENOMEM;
+    ERRNO_SAFE_WRAP (free, s);
+    ERRNO_SAFE_WRAP (json_decref, val);
+    return -1;
+}
+
+char *rutil_read_file (const char *path, flux_error_t *errp)
+{
+    int fd;
+    char *buf;
+
+    if ((fd = open (path, O_RDONLY)) < 0 || read_all (fd, (void **)&buf) < 0) {
+        errprintf (errp, "%s: %s", path, strerror (errno));
+        if (fd >= 0)
+            ERRNO_SAFE_WRAP (close, fd);
+        return NULL;
+    }
+    close (fd);
+    return buf;
+}
+
+json_t *rutil_load_file (const char *path, flux_error_t *errp)
+{
+    json_t *o;
+    json_error_t e;
+
+    if (!(o = json_load_file (path, 0, &e))) {
+        errprintf (errp, "%s:%d %s", e.source, e.line, e.text);
+        errno = EPROTO;
+        if (access (path, R_OK) < 0)
+            errno = ENOENT;
+        return NULL;
+    }
+    return o;
+}
+
+static int set_string (json_t *o, const char *key, const char *val)
+{
+    json_t *oval;
+
+    if (!(oval = json_string (val)))
+        goto nomem;
+    if (json_object_set_new (o, key, oval) < 0) {
+        json_decref (oval);
+        goto nomem;
+    }
+    return 0;
+nomem:
+    errno = ENOMEM;
+    return -1;
+}
+
+static const char *get_error (json_t *o)
+{
+    int saved_errno;
+    const char *s;
+    int rc;
+
+    saved_errno = errno;
+    rc = json_unpack (o, "{s:s}", "errstr", &s);
+    (void)json_object_del (o, "errstr");
+    errno = saved_errno;
+    if (rc < 0)
+        return NULL;
+    return s;
+}
+
+static void set_error (json_t *o, const char *errstr)
+{
+    (void)set_string (o, "errstr", errstr);
+}
+
+static int load_xml_file (dirwalk_t *d, void *arg)
+{
+    json_t *o = arg;
+    const char *name = dirwalk_name (d);
+    char *endptr;
+    int rank;
+    char *s;
+    char key[32];
+    flux_error_t error;
+
+    /* Only pay attention to files ending in "<rank<.xml"
+     */
+    if (dirwalk_isdir (d))
+        return 0;
+    errno = 0;
+    rank = strtol (name, &endptr, 10);
+    if (errno > 0 || !streq (endptr, ".xml"))
+        return 0;
+
+    /* Read the file and encode as JSON string, storing under rank key.
+     * On error, store human readable error string in object and stop iteration.
+     */
+    if (!(s = rutil_read_file (dirwalk_path (d), &error))) {
+        dirwalk_stop (d, errno);
+        set_error (o, error.text);
+        return 0;
+    }
+    snprintf (key, sizeof (key), "%d", rank);
+    if (set_string (o, key, s) < 0) {
+        dirwalk_stop (d, errno);
+        free (s);
+        return 0;
+    }
+    free (s);
+    return 0;
+}
+
+json_t *rutil_load_xml_dir (const char *path, flux_error_t *errp)
+{
+    json_t *o;
+
+    if (!(o = json_object ())) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    if (dirwalk (path, 0, load_xml_file, o) < 0) {
+        const char *errstr = get_error (o);
+        errprintf (errp,
+                   "%s: %s",
+                   path,
+                   errstr ? errstr : strerror (errno));
+        ERRNO_SAFE_WRAP (json_decref, o);
+        return NULL;
+    }
+    if (json_object_size (o) == 0) {
+        errprintf (errp,
+                   "%s: invalid directory: no XML input files found",
+                   path);
+        json_decref (o);
+        errno = EINVAL;
+        return NULL;
+    }
+    return o;
+}
+
+int rutil_idkey_map (json_t *obj, rutil_idkey_map_f map, void *arg)
+{
+    const char *key;
+    json_t *val;
+    struct idset *idset;
+    unsigned int id;
+
+    json_object_foreach (obj, key, val) {
+        if (!(idset = idset_decode (key)))
+            return -1;
+        id = idset_first (idset);
+        while (id != IDSET_INVALID_ID) {
+            if (map (id, val, arg) < 0) {
+                idset_destroy (idset);
+                return -1;
+            }
+            id = idset_next (idset, id);
+        }
+        idset_destroy (idset);
+    }
+    return 0;
+}
+
+static int idkey_count_map (unsigned int id, json_t *val, void *arg)
+{
+    int *count = arg;
+    (*count)++;
+    return 0;
+}
+
+int rutil_idkey_count (json_t *obj)
+{
+    int count = 0;
+    (void)rutil_idkey_map (obj, idkey_count_map, &count);
+    return count;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/rutil.h b/src/modules/resource/rutil.h
new file mode 100644
index 000000000000..7da8ca12aad2
--- /dev/null
+++ b/src/modules/resource/rutil.h
@@ -0,0 +1,56 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_RUTIL_H
+#define _FLUX_RESOURCE_RUTIL_H
+
+/* Compare 'old_set' to 'new_set'.
+ * Create '*add' for ids in new_set but not in old_set (sets NULL if n/a).
+ * Create '*sub' for ids in old_set but not in new_set (sets NULL if n/a).
+ */
+int rutil_idset_diff (const struct idset *old_set,
+                      const struct idset *new_set,
+                      struct idset **add,
+                      struct idset **sub);
+
+/* Set key=val in a json object, where val is the string
+ * representation of 'ids', or the empty string if 'ids' is NULL.
+ */
+int rutil_set_json_idset (json_t *o,
+                          const char *key,
+                          const struct idset *ids);
+
+/* Load data by path:
+ * - rutil_read_file() returns data as a NULL-terminated string.
+ * - rutil_load_file() parses data as a JSON object and returns it.
+ * - rutil_load_xml_dir() parses <rank>.xml files in path, and returns
+ *   a JSON object with ranks as keys and XML strings as values.
+ * On error put human readable error in errbuf and return NULL
+ */
+char *rutil_read_file (const char *path, flux_error_t *errp);
+json_t *rutil_load_file (const char *path, flux_error_t *errp);
+json_t *rutil_load_xml_dir (const char *path, flux_error_t *errp);
+
+/* Map over object with idset keys, calling 'map' for each id.
+ * map function returns 0 on success, -1 with errno set to abort.
+ */
+typedef int (*rutil_idkey_map_f)(unsigned int id, json_t *val, void *arg);
+int rutil_idkey_map (json_t *obj, rutil_idkey_map_f map, void *arg);
+
+/* Count ranks represented in idkey object.
+ */
+int rutil_idkey_count (json_t *obj);
+
+#endif /* !_FLUX_RESOURCE_RUTIL_H */
+
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/status.c b/src/modules/resource/status.c
new file mode 100644
index 000000000000..bdce54185f9e
--- /dev/null
+++ b/src/modules/resource/status.c
@@ -0,0 +1,550 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+ \************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/idset.h>
+
+#include "resource.h"
+#include "inventory.h"
+#include "drain.h"
+#include "rutil.h"
+#include "monitor.h"
+#include "exclude.h"
+#include "status.h"
+#include "reslog.h"
+
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/librlist/rlist.h"
+#include "ccan/str/str.h"
+
+struct status_cache {
+    struct rlist *rl;   // exclusions removed
+    json_t *R_all;
+    json_t *R_down;
+};
+
+struct status {
+    struct resource_ctx *ctx;
+    flux_msg_handler_t **handlers;
+    struct flux_msglist *requests;
+    struct status_cache cache;
+    json_t *R_empty;
+};
+
+static void invalidate_cache (struct status_cache *cache, bool all)
+{
+    if (all) {
+        rlist_destroy (cache->rl);
+        cache->rl = NULL;
+        json_decref (cache->R_all);
+        cache->R_all = NULL;
+    }
+    json_decref (cache->R_down);
+    cache->R_down = NULL;
+}
+
+static json_t *prepare_status_payload (struct status *status)
+{
+    struct resource_ctx *ctx = status->ctx;
+    const struct idset *down = monitor_get_down (ctx->monitor);
+    const struct idset *torpid = monitor_get_torpid (ctx->monitor);
+    const struct idset *exclude = exclude_get (ctx->exclude);
+    const json_t *R;
+    json_t *o = NULL;
+    json_t *drain_info = NULL;
+
+    if (!(R = inventory_get (ctx->inventory))
+        || !(drain_info = drain_get_info (ctx->drain)))
+        goto error;
+    if (!(o = json_pack ("{s:O s:O}", "R", R, "drain", drain_info))) {
+        errno = ENOMEM;
+        goto error;
+    }
+    if (rutil_set_json_idset (o, "online", monitor_get_up (ctx->monitor)) < 0
+        || rutil_set_json_idset (o, "offline", down) < 0
+        || rutil_set_json_idset (o, "exclude", exclude) < 0
+        || rutil_set_json_idset (o, "torpid", torpid) < 0)
+        goto error;
+    json_decref (drain_info);
+    return o;
+error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    ERRNO_SAFE_WRAP (json_decref, drain_info);
+    return NULL;
+}
+
+static void status_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    struct status *status = arg;
+    json_t *o = NULL;
+    flux_error_t error;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0) {
+        errprintf (&error, "error decoding request: %s", strerror (errno));
+        goto error;
+    }
+    if (status->ctx->rank != 0) {
+        errprintf (&error, "this RPC only works on rank 0");
+        errno = EPROTO;
+        goto error;
+    }
+    if (!(o = prepare_status_payload (status))) {
+        errprintf (&error, "error preparing response: %s", strerror (errno));
+        goto error;
+    }
+    if (flux_respond_pack (h, msg, "O", o) < 0)
+        flux_log_error (h, "error responding to resource.status request");
+    json_decref (o);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, error.text) < 0)
+        flux_log_error (h, "error responding to resource.status request");
+    json_decref (o);
+}
+
+/* Mark the ranks in 'ids' DOWN in the resource set 'rl'.
+ */
+static int mark_down (struct rlist *rl, const struct idset *ids)
+{
+    if (ids) {
+        char *s;
+
+        if (!(s = idset_encode (ids, IDSET_FLAG_RANGE)))
+            return -1;
+        if (rlist_mark_down (rl, s) < 0) {
+            free (s);
+            errno = EINVAL;
+            return -1;
+        }
+        free (s);
+    }
+    return 0;
+}
+
+/* Get an Rv1 resource object that includes all resources.
+ */
+static const json_t *get_all (struct status *status, const struct rlist *rl)
+{
+
+    if (!status->cache.R_all)
+        status->cache.R_all = rlist_to_R (rl);
+    return status->cache.R_all;
+}
+
+/* Get an Rv1 resource object that includes only DOWN resources.
+ * This modifies 'rl' but only to mark nodes up/down for rlist_copy_down().
+ * The up/down state is not used by other users of cache->rl.
+ */
+static json_t *get_down (struct status *status, struct rlist *rl)
+{
+    if (!status->cache.R_down) {
+        struct rlist *rdown = NULL;
+        struct idset *drain = NULL;
+        const struct idset *down = monitor_get_down (status->ctx->monitor);
+        const struct idset *torpid = monitor_get_torpid (status->ctx->monitor);
+
+        if ((drain = drain_get (status->ctx->drain))
+            && rlist_mark_up (rl, "all") >= 0
+            && mark_down (rl, down) == 0
+            && mark_down (rl, torpid) == 0
+            && mark_down (rl, drain) == 0
+            && (rdown = rlist_copy_down (rl)))
+            status->cache.R_down = rlist_to_R (rdown);
+        idset_destroy (drain);
+        rlist_destroy (rdown);
+    }
+    return status->cache.R_down;
+}
+
+/* Create an empty but valid Rv1 object.
+ */
+static json_t *get_empty_set (void)
+{
+    struct rlist *rl;
+    json_t *o;
+
+    if (!(rl = rlist_create ()))
+        return NULL;
+    o = rlist_to_R (rl);
+    rlist_destroy (rl);
+    return o;
+}
+
+/* Update property 'name' in 'alloc' resource set.
+ * Take the intersection of the alloc ranks vs the property ranks,
+ * and if non-empty, add properties to 'alloc' for those ranks.
+ */
+static int update_one_property (struct rlist *alloc,
+                                struct idset *alloc_ranks,
+                                struct idset *prop_ranks,
+                                const char *name)
+{
+    struct idset *ids;
+    char *targets = NULL;
+    int rc = -1;
+
+    if (!(ids = idset_intersect (alloc_ranks, prop_ranks))
+        || idset_count (ids) == 0) {
+        rc = 0;
+        goto done;
+    }
+    if (!(targets = idset_encode (ids, IDSET_FLAG_RANGE)))
+        goto done;
+    if (rlist_add_property (alloc, NULL, name, targets) < 0)
+        goto done;
+    rc = 0;
+done:
+    free (targets);
+    idset_destroy (ids);
+    return rc;
+}
+
+/* Fetch properties from a resource set in JSON form.
+ */
+static json_t *get_properties (const struct rlist *rl)
+{
+    char *s;
+    json_t *o = NULL;
+
+    if ((s = rlist_properties_encode (rl)))
+        o = json_loads (s, 0, NULL);
+    free (s);
+    return o;
+}
+
+/* Given a resource set 'all' with properties, assign any to 'alloc'
+ * that have matching ranks.
+ */
+static int update_properties (struct rlist *alloc, const struct rlist *all)
+{
+    struct idset *alloc_ranks;
+    json_t *props;
+    const char *name;
+    json_t *val;
+
+    if (!(alloc_ranks = rlist_ranks (alloc)))
+        return -1;
+    if (!(props = get_properties (all))
+        || json_object_size (props) == 0) {
+        json_decref (props);
+        idset_destroy (alloc_ranks);
+        return 0;
+    }
+    json_object_foreach (props, name, val) {
+        struct idset *prop_ranks;
+
+        if (!(prop_ranks = idset_decode (json_string_value (val))))
+            continue;
+        if (update_one_property (alloc, alloc_ranks, prop_ranks, name) < 0) {
+            idset_destroy (prop_ranks);
+            goto error;
+        }
+        idset_destroy (prop_ranks);
+    }
+    idset_destroy (alloc_ranks);
+    json_decref (props);
+    return 0;
+error:
+    idset_destroy (alloc_ranks);
+    json_decref (props);
+    return -1;
+}
+
+static json_t *update_properties_json (json_t *R, const struct rlist *all)
+{
+    struct rlist *alloc;
+    json_t *R2 = NULL;
+
+    if (!(alloc = rlist_from_json (R, NULL)))
+        return NULL;
+    if (update_properties (alloc, all) < 0)
+        goto done;
+    R2 = rlist_to_R (alloc);
+done:
+    rlist_destroy (alloc);
+    return R2;
+}
+
+/* Create an rlist object from R.  Omit the scheduling key.  Then:
+ * - exclude the ranks in 'exclude' (if non-NULL)
+ */
+static struct rlist *create_rlist (const json_t *R,
+                                   const struct idset *exclude)
+{
+    json_t *cpy;
+    struct rlist *rl;
+
+    if (!(cpy = json_copy ((json_t *)R))) { // thin copy - to del top level key
+        errno = ENOMEM;
+        return NULL;
+    }
+    (void)json_object_del (cpy, "scheduling");
+
+    if (!(rl = rlist_from_json (cpy, NULL)))
+        goto error;
+
+    if (exclude) {
+        if (rlist_remove_ranks (rl, (struct idset *)exclude) < 0)
+            goto error;
+    }
+    json_decref (cpy);
+    return rl;
+error:
+    json_decref (cpy);
+    rlist_destroy (rl);
+    errno = EINVAL;
+    return NULL;
+}
+
+static struct rlist *get_resource_list (struct status *status)
+{
+    if (!status->cache.rl) {
+        const json_t *R;
+        const struct idset *exclude = exclude_get (status->ctx->exclude);
+
+        if ((R = inventory_get (status->ctx->inventory)))
+            status->cache.rl = create_rlist (R, exclude);
+    }
+    return status->cache.rl;
+}
+
+/* See issue #5776 for an example of what the sched.resource-status
+ * RPC returns.  This payload intended to be identical, except 'allocated'
+ * is the calculated set provided by the job manager rather than the actual
+ * one from the scheduler itself (for performance reasons).
+ */
+static json_t *prepare_sched_status_payload (struct status *status,
+                                             json_t *allocated)
+{
+    struct rlist *rl;
+    const json_t *all;
+    const json_t *down;
+    json_t *o;
+    json_t *result = NULL;
+
+    if (!(rl = get_resource_list (status))
+        || !(all = get_all (status, rl))
+        || !(down = get_down (status, rl)))
+        goto error;
+    if (!(result = json_pack ("{s:O s:O}",
+                              "all", all,
+                              "down", down))) {
+        errno = ENOMEM;
+        goto error;
+    }
+    if (allocated)
+        o = update_properties_json (allocated, rl);
+    else
+        o = json_incref (status->R_empty);
+    if (!o || json_object_set_new (result, "allocated", o) < 0) {
+        json_decref (o);
+        goto error;
+    }
+    return result;
+error:
+    ERRNO_SAFE_WRAP (json_decref, result);
+    return NULL;
+}
+
+static void remove_request (struct flux_msglist *ml, const flux_msg_t *msg)
+{
+    const flux_msg_t *m;
+
+    m = flux_msglist_first (ml);
+    while (m) {
+        if (m == msg) {
+            flux_msglist_delete (ml); // delete @cursor
+            break;
+        }
+        m = flux_msglist_next (ml);
+    }
+}
+
+/* The job-manager.resource-status RPC has completed.
+ * Finish handling resource.sched-status.  Notes:
+ * - Treat ENOSYS from job-manager.resource-status as the empty set.  This
+ *   could happen IRL because the resource module loads before job-manager.
+ * - Both the future and the message are unreferenced/destroyed
+ *   when msg is removed from the status->requests list.
+ */
+static void sched_status_continuation (flux_future_t *f, void *arg)
+{
+    const flux_msg_t *msg = flux_future_aux_get (f, "flux::request");
+    struct status *status = arg;
+    flux_t *h = status->ctx->h;
+    flux_error_t error;
+    json_t *allocated = NULL;
+    json_t *o = NULL;
+
+    if (flux_rpc_get_unpack (f, "{s:o}", "allocated", &allocated) < 0
+        && errno != ENOSYS) {
+        errprintf (&error,
+                   "job-manager.resource-status request failed: %s",
+                   future_strerror (f, errno));
+        goto error;
+    }
+    if (!(o = prepare_sched_status_payload (status, allocated))) {
+        errprintf (&error, "error preparing response: %s", strerror (errno));
+        goto error;
+    }
+    if (flux_respond_pack (h, msg, "O", o) < 0)
+        flux_log_error (h, "error responding to resource.sched-status");
+    json_decref (o);
+    remove_request (status->requests, msg);
+    return;
+error:
+    if (flux_respond_error (h, msg, EINVAL, error.text) < 0)
+        flux_log_error (h, "error responding to resource.sched-status");
+    json_decref (o);
+    remove_request (status->requests, msg);
+}
+
+/* To answer this query, an RPC must be sent to the job manager to get
+ * the set of allocated resources.  Get that started, then place the request
+ * on status->requests and continue answering in the RPC continuation.
+ * The rest of the information required is local.
+ */
+static void sched_status_cb (flux_t *h,
+                             flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
+{
+    struct status *status = arg;
+    flux_future_t *f;
+    flux_error_t error;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0) {
+        errprintf (&error, "error decoding request: %s", strerror (errno));
+        goto error;
+    }
+    if (status->ctx->rank != 0) {
+        errprintf (&error, "this RPC only works on rank 0");
+        errno = EPROTO;
+        goto error;
+    }
+    if (!(f = flux_rpc (h, "job-manager.resource-status", NULL, 0, 0))
+        || flux_future_then (f, -1, sched_status_continuation, status) < 0
+        || flux_future_aux_set (f, "flux::request", (void *)msg, NULL) < 0
+        || flux_msg_aux_set (msg,
+                             NULL,
+                             f,
+                             (flux_free_f)flux_future_destroy) < 0) {
+        errprintf (&error,
+                   "error sending job-manager.resource-status request: %s",
+                   strerror (errno));
+        flux_future_destroy (f);
+        goto error;
+    }
+    if (flux_msglist_append (status->requests, msg) < 0) {
+        errprintf (&error, "error saving request mesg: %s", strerror (errno));
+        goto error;
+    }
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, error.text) < 0)
+        flux_log_error (h, "error responding to resource.sched-status");
+}
+
+/* Watch for resource eventlog events that might invalidate cached data.
+ * resource-define
+ *   could be called in test from 'flux resource reload'
+ * resource-update
+ *   expiration only at this time - ignore
+ * online, offline, drain, undrain
+ *   invalidate R_down only
+ */
+static void reslog_cb (struct reslog *reslog,
+                       const char *name,
+                       json_t *context,
+                       void *arg)
+{
+    struct status *status = arg;
+
+    if (streq (name, "resource-define"))
+        invalidate_cache (&status->cache, true);
+    else if (streq (name, "online")
+        || streq (name, "offline")
+        || streq (name, "drain")
+        || streq (name, "undrain")
+        || streq (name, "torpid")
+        || streq (name, "lively"))
+        invalidate_cache (&status->cache, false);
+}
+
+/* Disconnect hook called from resource module's main disconnect
+ * message handler.
+ */
+void status_disconnect (struct status *status, const flux_msg_t *msg)
+{
+    (void)flux_msglist_disconnect (status->requests, msg);
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    {
+        .typemask = FLUX_MSGTYPE_REQUEST,
+        .topic_glob = "resource.status",
+        .cb = status_cb,
+        .rolemask = FLUX_ROLE_USER,
+    },
+    {
+        .typemask = FLUX_MSGTYPE_REQUEST,
+        .topic_glob = "resource.sched-status",
+        .cb = sched_status_cb,
+        .rolemask = FLUX_ROLE_USER,
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+void status_destroy (struct status *status)
+{
+    if (status) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (status->handlers);
+        flux_msglist_destroy (status->requests);
+        reslog_remove_callback (status->ctx->reslog, reslog_cb, status);
+        invalidate_cache (&status->cache, true);
+        json_decref (status->R_empty);
+        free (status);
+        errno = saved_errno;
+    }
+}
+
+struct status *status_create (struct resource_ctx *ctx)
+{
+    struct status *status;
+
+    if (!(status = calloc (1, sizeof (*status))))
+        return NULL;
+    status->ctx = ctx;
+    if (!(status->requests = flux_msglist_create ()))
+        goto error;
+    if (flux_msg_handler_addvec (ctx->h, htab, status, &status->handlers) < 0)
+        goto error;
+    if (ctx->rank == 0) {
+        if (reslog_add_callback (ctx->reslog, reslog_cb, status) < 0)
+            goto error;
+        if (!(status->R_empty = get_empty_set ()))
+            goto error;
+    }
+    return status;
+error:
+    status_destroy (status);
+    return NULL;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/resource/status.h b/src/modules/resource/status.h
new file mode 100644
index 000000000000..b1aecd6427e5
--- /dev/null
+++ b/src/modules/resource/status.h
@@ -0,0 +1,20 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+ \************************************************************/
+
+#ifndef _RESOURCE_STATUS_H
+#define _RESOURCE_STATUS_H
+
+struct status *status_create (struct resource_ctx *ctx);
+void status_destroy (struct status *status);
+void status_disconnect (struct status *status, const flux_msg_t *msg);
+
+#endif /* ! _RESOURCE_STATUS_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/resource/test/drainset.c b/src/modules/resource/test/drainset.c
new file mode 100644
index 000000000000..4c90352837d0
--- /dev/null
+++ b/src/modules/resource/test/drainset.c
@@ -0,0 +1,111 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <errno.h>
+#include <string.h>
+
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+
+#include "src/modules/resource/drainset.h"
+
+static void check_drainset (struct drainset *ds,
+                           const char *json_str)
+{
+    char *s;
+    json_t *o = drainset_to_json (ds);
+    json_t *expected = json_loads (json_str, 0, NULL);
+    if (!o || !expected)
+        BAIL_OUT ("drainset_to_json failed");
+    if (!(s = json_dumps (o, JSON_COMPACT)))
+        BAIL_OUT ("json_dumps failed");
+    diag ("drainset_to_json = %s", s);
+    diag ("expected =         %s", json_str);
+    ok (json_equal (expected, o),
+        "drainset_to_json got expected result");
+    json_decref (expected);
+    json_decref (o);
+    free (s);
+}
+
+static void test_empty ()
+{
+    struct drainset *ds = drainset_create ();
+    if (!ds)
+        BAIL_OUT ("drainset_create failed");
+    diag ("empty drainset should return empty JSON object");
+    check_drainset (ds, "{}");
+    drainset_destroy (ds);
+}
+
+static void test_basic ()
+{
+    struct drainset *ds = drainset_create ();
+    if (!ds)
+        BAIL_OUT ("drainset_create failed");
+
+    ok (drainset_drain_rank (NULL, 0, 1234.0, NULL) < 0 && errno == EINVAL,
+        "drainset_drain_rank (NULL, ...) returns EINVAL");
+
+    for (unsigned int i = 0; i < 8; i++) {
+        ok (drainset_drain_rank (ds, i, 1234.0, "test") == 0,
+            "drainset_drain_rank: rank=%u", i);
+    }
+    check_drainset (ds,
+                    "{\"0-7\":{\"timestamp\":1234.0,\"reason\":\"test\"}}");
+    drainset_destroy (ds);
+}
+
+static void test_multiple ()
+{
+    struct drainset *ds = drainset_create ();
+
+    if (!ds)
+        BAIL_OUT ("drainset_create failed");
+
+    ok (drainset_drain_rank (ds, 0, 1234.0, "test") == 0,
+        "drainset_drain_rank: rank=0");
+    ok (drainset_drain_rank (ds, 1, 2345.0, "test") == 0,
+        "drainset_drain_rank: rank=1");
+    ok (drainset_drain_rank (ds, 2, 1234.0, "test1") == 0,
+        "drainset_drain_rank: rank=1");
+    ok (drainset_drain_rank (ds, 3, 1234.0, "test") == 0,
+        "drainset_drain_rank: rank=0");
+    ok (drainset_drain_rank (ds, 4, 1234.0, NULL) == 0,
+        "drainset_drain_rank: rank=1");
+
+    check_drainset (ds,
+                    "{\"0,3\":{\"timestamp\":1234.0,\"reason\":\"test\"},"
+                    "\"1\":{\"timestamp\":2345.0,\"reason\":\"test\"},"
+                    "\"2\":{\"timestamp\":1234.0,\"reason\":\"test1\"},"
+                    "\"4\":{\"timestamp\":1234.0,\"reason\":\"\"}}");
+    drainset_destroy (ds);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+    test_empty ();
+    test_basic ();
+    test_multiple ();
+    done_testing ();
+    return (0);
+}
+
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/resource/test/rutil.c b/src/modules/resource/test/rutil.c
new file mode 100644
index 000000000000..2dea5ad218ef
--- /dev/null
+++ b/src/modules/resource/test/rutil.c
@@ -0,0 +1,368 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+#include <stdlib.h>
+#include <fcntl.h>
+
+#include "src/common/libtap/tap.h"
+#include "src/common/libutil/cleanup.h"
+#include "src/common/libutil/read_all.h"
+#include "src/common/libidset/idset.h"
+#include "ccan/str/str.h"
+
+#include "src/modules/resource/rutil.h"
+
+void test_idset_diff (void)
+{
+    struct idset *ids1;
+    struct idset *ids2;
+    struct idset *add;
+    struct idset *sub;
+
+    if (!(ids1 = idset_create (1024, 0)))
+        BAIL_OUT ("idset_create failed");
+    if (!(ids2 = idset_create (1024, 0)))
+        BAIL_OUT ("idset_create failed");
+
+    ok (rutil_idset_diff (NULL, ids2, &add, &sub) == 0
+        && add == NULL
+        && sub == NULL,
+        "rutil_idset_diff ids1=NULL works");
+    idset_destroy (add);
+    idset_destroy (sub);
+
+    ok (rutil_idset_diff (ids1, NULL, &add, &sub) == 0
+        && add == NULL
+        && sub == NULL,
+        "rutil_idset_diff ids2=NULL works");
+    idset_destroy (add);
+    idset_destroy (sub);
+
+    errno = 0;
+    ok (rutil_idset_diff (ids1, ids2, NULL, &sub) < 0 && errno == EINVAL,
+        "rutil_idset_diff add=NULL fails with EINVAL");
+    errno = 0;
+    ok (rutil_idset_diff (ids1, ids2, &add, NULL) < 0 && errno == EINVAL,
+        "rutil_idset_diff sub=NULL fails with EINVAL");
+
+    if (idset_set (ids1, 1) < 0 || idset_set (ids2, 2) < 0)
+        BAIL_OUT ("idset_set failed");
+    add = sub = NULL;
+    ok (rutil_idset_diff (ids1, ids2, &add, &sub) == 0
+        && add != NULL && idset_count (add) == 1 && idset_test (add, 2)
+        && sub != NULL && idset_count (sub) == 1 && idset_test (sub, 1),
+        "rutil_idset_diff [1] [2] sets add=[2] sub=[1]");
+    idset_destroy (add);
+    idset_destroy (sub);
+
+    add = sub = NULL;
+    ok (rutil_idset_diff (ids2, ids1, &add, &sub) == 0
+        && add != NULL && idset_count (add) == 1 && idset_test (add, 1)
+        && sub != NULL && idset_count (sub) == 1 && idset_test (sub, 2),
+        "rutil_idset_diff [2] [1] sets add=[1] sub=[2]");
+    idset_destroy (add);
+    idset_destroy (sub);
+
+    if (idset_set (ids1, 2) < 0)
+        BAIL_OUT ("idset_set failed");
+    add = sub = NULL;
+    ok (rutil_idset_diff (ids1, ids2, &add, &sub) == 0
+        && add == NULL
+        && sub != NULL && idset_count (sub) == 1 && idset_test (sub, 1),
+        "rutil_idset_diff [1-2] [2] sets add=NULL sub=[1]");
+    idset_destroy (add);
+    idset_destroy (sub);
+
+    add = sub = NULL;
+    ok (rutil_idset_diff (ids2, ids1, &add, &sub) == 0
+        && add != NULL && idset_count (add) == 1 && idset_test (add, 1)
+        && sub == NULL,
+        "rutil_idset_diff [2] [1-2] sets add=[1] sub=NULL");
+    idset_destroy (add);
+    idset_destroy (sub);
+
+    if (idset_set (ids2, 1) < 0)
+        BAIL_OUT ("idset_set failed");
+    add = sub = NULL;
+    ok (rutil_idset_diff (ids1, ids2, &add, &sub) == 0
+        && add == NULL
+        && sub == NULL,
+        "rutil_idset_diff [1-2] [1-2] sets add=NULL sub=NULL");
+    idset_destroy (add);
+    idset_destroy (sub);
+
+    idset_destroy (ids1);
+    idset_destroy (ids2);
+}
+
+void test_set_json_idset (void)
+{
+    json_t *o;
+    json_t *o2;
+    const char *s;
+    struct idset *ids;
+
+    if (!(ids= idset_create (1024, 0)))
+        BAIL_OUT ("idset_create failed");
+    if (idset_set (ids, 42) < 0)
+        BAIL_OUT ("idset_set failed");
+
+    if (!(o = json_object ()))
+        BAIL_OUT ("json_object failed");
+
+    errno = 0;
+    ok (rutil_set_json_idset (NULL, "foo", NULL) < 0 && errno == EINVAL,
+        "rutil_set_json_idset obj=NULL fails with EINVAL");
+    errno = 0;
+    ok (rutil_set_json_idset (o, NULL, NULL) < 0 && errno == EINVAL,
+        "rutil_set_json_idset key=NULL fails with EINVAL");
+    errno = 0;
+    ok (rutil_set_json_idset (o, "", NULL) < 0 && errno == EINVAL,
+        "rutil_set_json_idset key=(empty) fails with EINVAL");
+
+    ok (rutil_set_json_idset (o, "foo", NULL) == 0
+            && (o2 = json_object_get (o, "foo"))
+            && (s = json_string_value (o2))
+            && streq (s, ""),
+        "rutil_set_json_idset ids=NULL sets empty string value");
+    ok (rutil_set_json_idset (o, "bar", ids) == 0
+            && (o2 = json_object_get (o, "bar"))
+            && (s = json_string_value (o2))
+            && streq (s, "42"),
+        "rutil_set_json_idset ids=[42] sets encoded value");
+
+    json_decref (o);
+    idset_destroy (ids);
+}
+
+static char *create_tmp_file (const char *content)
+{
+    char *path;
+    char *tmpdir = getenv ("TMPDIR");
+    int fd = -1;
+
+    if (!tmpdir)
+        tmpdir = "/tmp";
+    if (asprintf (&path, "%s/rutil-test.XXXXXX", tmpdir) < 0)
+        BAIL_OUT ("error allocating buffer");
+    if ((fd = mkostemp (path, O_WRONLY)) < 0)
+        BAIL_OUT ("error creating temp file");
+
+    cleanup_push_string (cleanup_file, path);
+
+    if (write_all (fd, content, strlen (content)) < 0)
+        BAIL_OUT ("writing to creating temp file");
+    close (fd);
+    return path;
+}
+
+void test_read_file (void)
+{
+    flux_error_t error;
+    char *s;
+    char *tmp = create_tmp_file ("XXX");
+
+    errno = 0;
+    error.text[0] = '\0';
+    s = rutil_read_file ("/noexist", &error);
+    ok (s == NULL && errno == ENOENT && strlen (error.text) > 0,
+         "rutil_read_file path=/noexist fails with ENOENT and human error");
+    diag ("%s", error.text);
+
+    s = rutil_read_file (tmp, &error);
+    ok (s != NULL && streq (s, "XXX"),
+        "rutil_read_file works");
+    free (s);
+
+    free (tmp);
+}
+
+void test_load_file (void)
+{
+    flux_error_t error;
+    json_t *o;
+    char *good = create_tmp_file ("{\"foo\":42}");
+    char *bad = create_tmp_file ("XXX");
+
+    errno = 0;
+    error.text[0] = '\0';
+    o = rutil_load_file ("/noexist", &error);
+    ok (o == NULL && errno == ENOENT && strlen (error.text) > 0,
+         "rutil_load_file path=/noexist fails with ENOENT and human error");
+    diag ("%s", error.text);
+
+    errno = 0;
+    error.text[0] = '\0';
+    o = rutil_load_file (bad, &error);
+    ok (o == NULL && errno != 0 && strlen (error.text) > 0,
+         "rutil_load_file with errno and human error on bad JSON");
+    diag ("%s", error.text);
+
+    o = rutil_load_file (good, &error);
+    ok (o != NULL && json_object_get (o, "foo"),
+         "rutil_load_file with good JSON works");
+    json_decref (o);
+
+    free (good);
+    free (bad);
+}
+
+static char *create_tmp_xml_dir (int size)
+{
+    char *path;
+    char *tmpdir = getenv ("TMPDIR");
+    int i;
+
+    if (!tmpdir)
+        tmpdir = "/tmp";
+    if (asprintf (&path, "%s/rutil-test.XXXXXX", tmpdir) < 0)
+        BAIL_OUT ("error allocating buffer");
+    if (!mkdtemp (path))
+        BAIL_OUT ("failed to create tmp xmldir");
+
+    cleanup_push_string (cleanup_directory_recursive, path);
+
+    for (i = 0; i < size; i++) {
+        char fpath[1024];
+        int ffd;
+        snprintf (fpath, sizeof (fpath), "%s/%d.xml", path, i);
+        ffd = open (fpath, O_WRONLY | O_CREAT, 0644);
+        if (ffd < 0)
+            BAIL_OUT ("failed to create %s", fpath);
+        if (write_all (ffd, "\"foo\"",  5) < 0)
+            BAIL_OUT ("failed to write %s", fpath);
+        close (ffd);
+    }
+
+    return path;
+}
+
+void test_load_xml_dir (void)
+{
+    const int count = 8;
+    char *path = create_tmp_xml_dir (count);
+    flux_error_t error;
+    json_t *o;
+
+    errno = 0;
+    error.text[0] = '\0';
+    o = rutil_load_xml_dir ("/noexist", &error);
+    ok (o == NULL && errno == ENOENT && strlen (error.text) > 0,
+         "rutil_load_xml_dir path=/noexist fails with ENOENT and human error");
+    diag ("%s", error.text);
+
+    o = rutil_load_xml_dir (path, &error);
+    ok (o != NULL,
+        "rutil_load_xml_dir works");
+    if (o) {
+        char *tmp = json_dumps (o, JSON_COMPACT);
+        diag ("%s", tmp);
+        free (tmp);
+    }
+    ok (json_object_size (o) == count,
+        "and contains the expected number of keys");
+    json_decref (o);
+
+    free (path);
+}
+
+void diag_obj (const char *prefix, json_t *obj)
+{
+    char *s;
+    s = json_dumps (obj, JSON_COMPACT);
+    diag ("%s: %s", prefix, s ? s : "fail");
+    free (s);
+}
+
+static int map_exit_iter;
+int mapit (unsigned int id, json_t *val, void *arg)
+{
+    int *map_count = arg;
+    if (map_exit_iter != -1 && *map_count == map_exit_iter) {
+        errno = 123456;
+        return -1;
+    }
+    (*map_count)++;
+    return 0;
+}
+
+void test_idkey_map (void)
+{
+    json_t *obj;
+    int map_count;
+    json_error_t error;
+
+    /*  Recreate test object for map here.
+     *  This was created step-wise with rutil_idkey_insert_id before,
+     *   but that function has since been removed.
+     */
+    if (!(obj = json_pack_ex (&error, 0,
+                              "{s:{s:s s:i} s:{s:s s:i} s:{s:s s:i}}",
+                              "0",
+                               "foo", "ZZZ",
+                               "bar", 42,
+                              "2",
+                               "foo", "xyz",
+                               "bar", 42,
+                              "1,3",
+                               "foo", "xyz",
+                               "bar", 43)))
+        BAIL_OUT ("json_pack failed: %s", error.text);
+
+    diag_obj ("obj", obj);
+
+    ok (json_object_size (obj) == 3,
+        "object size 3 keys");
+
+    map_count = 0;
+    map_exit_iter = -1;
+    ok (rutil_idkey_map (obj, mapit, &map_count) == 0 && map_count == 4,
+        "rutil_idkey_map called once per id (there are %d)", map_count);
+
+    ok (rutil_idkey_count (obj) == 4,
+        "rutil_idkey_count agrees");
+
+    map_count = 0;
+    map_exit_iter = 1;
+    errno = 0;
+    ok (rutil_idkey_map (obj, mapit, &map_count) < 0 && map_count == 1
+        && errno == 123456,
+        "rutil_idkey_map fails when map function returns -1 with errno set");
+    map_exit_iter = -1;
+
+    json_decref (obj);
+}
+
+int main (int argc, char *argv[])
+{
+    plan (NO_PLAN);
+
+    test_idset_diff ();
+    test_set_json_idset ();
+
+    test_read_file ();
+    test_load_file ();
+    test_load_xml_dir ();
+
+    test_idkey_map ();
+
+    done_testing ();
+    return (0);
+}
+
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/modules/resource/topo.c b/src/modules/resource/topo.c
new file mode 100644
index 000000000000..93ca0577ed76
--- /dev/null
+++ b/src/modules/resource/topo.c
@@ -0,0 +1,367 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* topo.c - load and verify the local rank's hwloc topology
+ *
+ * If resources are known at module load time, verify the topology against
+ * this rank's portion of the resource object (unless noverify is set).
+ *
+ * Reduce r_local from each rank, leaving the result in topo->reduce->rl
+ * on rank 0.  If resources are not known, then this R is set in inventory.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <flux/core.h>
+
+#include "src/common/libidset/idset.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/librlist/rhwloc.h"
+#include "src/common/librlist/rlist.h"
+#include "ccan/str/str.h"
+
+#include "resource.h"
+#include "inventory.h"
+#include "reslog.h"
+#include "drain.h"
+#include "rutil.h"
+#include "topo.h"
+
+struct reduction {
+    int count;          // number of ranks represented
+    int descendants;    // number of TBON descendants
+    struct rlist *rl;   // resources: self + descendants
+};
+
+struct topo {
+    struct resource_ctx *ctx;
+    flux_msg_handler_t **handlers;
+    char *xml;
+    struct rlist *r_local;
+
+    struct reduction reduce;
+};
+
+static int drain_self (struct topo *topo, const char *reason)
+{
+    flux_log (topo->ctx->h, LOG_ERR, "draining: %s", reason);
+
+    if (topo->ctx->rank == 0) {
+        if (drain_rank (topo->ctx->drain, topo->ctx->rank, reason) < 0)
+            return -1;
+    }
+    else {
+        char rankstr[16];
+        flux_future_t *f;
+
+        snprintf (rankstr, sizeof (rankstr), "%ju", (uintmax_t)topo->ctx->rank);
+        if (!(f = flux_rpc_pack (topo->ctx->h,
+                                 "resource.drain",
+                                 0,
+                                 0,
+                                 "{s:s s:s s:s}",
+                                 "targets", rankstr,
+                                 "reason", reason,
+                                 "mode", "update")))
+            return -1;
+        if (flux_rpc_get (f, NULL) < 0) {
+            flux_future_destroy (f);
+            return -1;
+        }
+        flux_future_destroy (f);
+    }
+    return 0;
+}
+
+static int topo_verify (struct topo *topo, json_t *R, bool nodrain)
+{
+    json_error_t e;
+    struct rlist *rl = NULL;
+    struct rlist *rl_cores = NULL;
+    struct rlist *r_local_cores = NULL;
+    flux_error_t error;
+    int rc = -1;
+
+    if (!(rl = rlist_from_json (R, &e))) {
+        flux_log (topo->ctx->h, LOG_ERR, "R: %s", e.text);
+        errno = EINVAL;
+        return -1;
+    }
+
+    /*  Only verify cores (and rank hostname) for now.
+     *
+     *  This is to allow GPUs to be configured or set in a job's allocated
+     *  R even when the system installed libhwloc fails to detect GPUs
+     *  due to lack of appropriately configured backend or other reason.
+     *  (See flux-core issue #4181 for more details)
+     */
+    if (!(r_local_cores = rlist_copy_cores (topo->r_local))
+        || !(rl_cores = rlist_copy_cores (rl))) {
+        flux_log_error (topo->ctx->h, "rlist_copy_cores");
+        goto out;
+    }
+    rc = rlist_verify (&error, rl_cores, r_local_cores);
+    if (rc < 0 && !nodrain) {
+        if (drain_self (topo, error.text) < 0)
+            goto out;
+    }
+    else if (rc != 0)
+        flux_log (topo->ctx->h, LOG_ERR, "verify: %s", error.text);
+    rc = 0;
+out:
+    rlist_destroy (rl_cores);
+    rlist_destroy (r_local_cores);
+    rlist_destroy (rl);
+    return rc;
+}
+
+/* Call this on any rank when there are no more descendants reporting.
+ * On rank 0, this finalizes the reduction.
+ * On other ranks, the reduction is sent upstream.
+ */
+static int topo_reduce_finalize (struct topo *topo)
+{
+    json_t *resobj = NULL;
+
+    if (!(resobj = rlist_to_R (topo->reduce.rl))) {
+        flux_log (topo->ctx->h, LOG_ERR, "error converting reduced rlist");
+        errno = EINVAL;
+        return -1;
+    }
+    if (topo->ctx->rank == 0) {
+        if (!inventory_get (topo->ctx->inventory)) {
+            if (inventory_put (topo->ctx->inventory,
+                               resobj,
+                               "dynamic-discovery") < 0) {
+                flux_log_error (topo->ctx->h,
+                                "error setting reduced resource object");
+                goto error;
+            }
+        }
+    }
+    else {
+        flux_future_t *f;
+
+        if (!(f = flux_rpc_pack (topo->ctx->h,
+                                 "resource.topo-reduce",
+                                 FLUX_NODEID_UPSTREAM,
+                                 FLUX_RPC_NORESPONSE,
+                                 "{s:i s:O}",
+                                 "count", topo->reduce.count,
+                                 "resource", resobj))) {
+            flux_log_error (topo->ctx->h,
+                            "resource.topo-reduce: error sending request");
+            goto error;
+        }
+        flux_future_destroy (f);
+    }
+    json_decref (resobj);
+    return 0;
+error:
+    ERRNO_SAFE_WRAP (json_decref, resobj);
+    return -1;
+}
+
+/* Accept reduction input from downstream ranks.
+ */
+static void topo_reduce_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
+{
+    struct topo *topo = arg;
+    json_t *resobj;
+    struct rlist *rl = NULL;
+    int count;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s:o}",
+                             "count", &count,
+                             "resource", &resobj) < 0
+        || !(rl = rlist_from_json (resobj, NULL))) {
+        flux_log (h, LOG_ERR, "error decoding topo-reduce request");
+        return;
+    }
+    if (rlist_append (topo->reduce.rl, rl) < 0) {
+        /* N.B. log nothing in this case as this error will occur naturally
+         * when the resource module is reloaded and resource object is a dup.
+         */
+        goto done;
+    }
+    topo->reduce.count += count;
+    if (topo->reduce.count == topo->reduce.descendants + 1) {
+        if (topo_reduce_finalize (topo) < 0) // logs its own errors
+            goto done;
+    }
+done:
+    rlist_destroy (rl);
+}
+
+/* Set up for reduction of distributed topo->r_local to inventory.
+ * Ranks with descendants wait for all of them to report in, then roll
+ * up their own and their descendants' contributions into one object and
+ * report that.  N.B. This is not a "timed batch" style reduction since the
+ * final result cannot be obtained without the participation of all ranks.
+ */
+static int topo_reduce (struct topo *topo)
+{
+    const char *val;
+
+    if (!(val = flux_attr_get (topo->ctx->h, "tbon.descendants")))
+        return -1;
+    errno = 0;
+    topo->reduce.descendants = strtoul (val, NULL, 10);
+    if (errno > 0)
+        return -1;
+
+    topo->reduce.count = 1;
+    if (!(topo->reduce.rl = rlist_copy_empty (topo->r_local)))
+        goto nomem;
+
+    if (topo->reduce.descendants == 0) {
+        if (topo_reduce_finalize (topo) < 0)
+            return -1;
+    }
+    return 0;
+nomem:
+    errno = ENOMEM;
+    return -1;
+}
+
+static void topo_get_cb (flux_t *h,
+                         flux_msg_handler_t *mh,
+                         const flux_msg_t *msg,
+                         void *arg)
+{
+    struct topo *topo = arg;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (flux_respond (h, msg, topo->xml) < 0)
+        flux_log_error (h, "error responding to topo-get request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to topo-get request");
+}
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST, "resource.topo-reduce",  topo_reduce_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "resource.topo-get", topo_get_cb, FLUX_ROLE_USER },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+
+void topo_destroy (struct topo *topo)
+{
+    if (topo) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (topo->handlers);
+        free (topo->xml);
+        rlist_destroy (topo->reduce.rl);
+        rlist_destroy (topo->r_local);
+        free (topo);
+        errno = saved_errno;
+    }
+}
+
+static char *topo_get_local_xml (struct resource_ctx *ctx, bool no_restrict)
+{
+    flux_t *parent_h;
+    flux_future_t *f = NULL;
+    char *result = NULL;
+    const char *xml;
+
+    errno = 0;
+    if (!(parent_h = resource_parent_handle_open (ctx))
+        || !(f = flux_rpc (parent_h,
+                           "resource.topo-get",
+                           NULL,
+                           FLUX_NODEID_ANY,
+                           0))
+        || flux_rpc_get (f, &xml) < 0) {
+        rhwloc_flags_t flags = no_restrict ? RHWLOC_NO_RESTRICT : 0;
+        /*  ENOENT just means there is no parent instance.
+         *  No need for an error.
+         */
+        if (errno && errno != ENOENT)
+            flux_log (ctx->h,
+                      LOG_DEBUG,
+                      "resource.topo-get to parent failed: %s",
+                      strerror (errno));
+        result = rhwloc_local_topology_xml (flags);
+        goto out;
+    }
+    flux_log (ctx->h,
+              LOG_INFO,
+              "retrieved local hwloc XML from parent (norestrict=%s)",
+              no_restrict ? "true" : "false");
+    if (no_restrict) {
+        result = strdup (xml);
+        goto out;
+    }
+    /*  restrict topology to current CPU binding
+     */
+    result = rhwloc_topology_xml_restrict (xml);
+out:
+    flux_future_destroy (f);
+    resource_parent_handle_close (ctx);
+    return result;
+}
+
+struct topo *topo_create (struct resource_ctx *ctx,
+                          bool no_verify,
+                          bool no_restrict)
+{
+    struct topo *topo;
+    json_t *R;
+
+    if (!(topo = calloc (1, sizeof (*topo))))
+        return NULL;
+    topo->ctx = ctx;
+    if (!(topo->xml = topo_get_local_xml (ctx, no_restrict))) {
+        flux_log (ctx->h, LOG_ERR, "error loading hwloc topology");
+        goto error;
+    }
+    if (!(topo->r_local = rlist_from_hwloc (ctx->rank, topo->xml))) {
+        flux_log_error (ctx->h, "error creating local resource object");
+        goto error;
+    }
+    /* If global resource object is known now, use it to verify topo.
+     */
+    if ((R = inventory_get (ctx->inventory))) {
+        const char *method = inventory_get_method (ctx->inventory);
+        bool nodrain = false;
+
+        if (method && streq (method, "job-info"))
+            nodrain = true;
+        if (!no_verify && topo_verify (topo, R, nodrain) < 0)
+            goto error;
+    }
+    /* Reduce topo to rank 0 unconditionally in case it is needed.
+     */
+    if (topo_reduce (topo) < 0) {
+        flux_log_error (ctx->h, "error setting up topo reduction");
+        goto error;
+    }
+    if (flux_msg_handler_addvec (ctx->h, htab, topo, &topo->handlers) < 0)
+        goto error;
+    return topo;
+error:
+    topo_destroy (topo);
+    return NULL;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/topo.h b/src/modules/resource/topo.h
new file mode 100644
index 000000000000..df28e448289a
--- /dev/null
+++ b/src/modules/resource/topo.h
@@ -0,0 +1,25 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_TOPO_H
+#define _FLUX_RESOURCE_TOPO_H
+
+struct topo *topo_create (struct resource_ctx *ctx,
+                          bool no_verify,
+                          bool no_restrict);
+void topo_destroy (struct topo *topo);
+
+
+#endif /* !_FLUX_RESOURCE_TOPO_H */
+
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/modules/resource/upgrade.c b/src/modules/resource/upgrade.c
new file mode 100644
index 000000000000..6786bb667074
--- /dev/null
+++ b/src/modules/resource/upgrade.c
@@ -0,0 +1,294 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* upgrade.c - update resource.eventlog with new format
+ *
+ * The resource.eventlog format changed in 0.62.0.  If a 'resource-init'
+ * event is found, it is the older format and can be upgraded:
+ * - drop all events prior to the last resource-init
+ * - convert drain summary of the last resource-init into discrete drain events
+ * - remove all remaining events that are no longer valid
+ * - add a nodelist to drain/undrain events, if missing
+ *
+ * If an upgrade occurred, rewrite the kvs resource.eventlog.  This eliminates
+ * the risk of drain events referring to the wrong hosts if the rank:host
+ * mapping changes in the future.  This rewrite will only occur once as the
+ * upgrade code does nothing if a resource-init event is not found and they
+ * are no longer produced as of 0.62.0.
+ *
+ * N.B. the new format consisting only of drain/undrain/resource-define
+ * events can be parsed by old flux-core releases so a flux-core downgrade
+ * is possible.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/timestamp.h"
+#include "src/common/libidset/idset.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "ccan/str/str.h"
+
+#include "reslog.h"
+#include "upgrade.h"
+
+static int rewrite_eventlog (flux_t *h, json_t *newlog)
+{
+    char *s;
+    flux_kvs_txn_t *txn = NULL;
+    flux_future_t *f = NULL;
+    int rc = -1;
+
+    if (!(s = eventlog_encode (newlog))
+        || !(txn = flux_kvs_txn_create ())
+        || flux_kvs_txn_put (txn, 0, RESLOG_KEY, s) < 0
+        || !(f = flux_kvs_commit (h, NULL, 0, txn))
+        || flux_rpc_get (f, NULL) < 0)
+        goto done;
+    rc = 0;
+done:
+    flux_future_destroy (f);
+    flux_kvs_txn_destroy (txn);
+    ERRNO_SAFE_WRAP (free, s);
+    return rc;
+}
+
+/* Add nodelist to (un)drain context.
+ * If any ranks are invalid, the entire event is thrown out (and logged).
+ * See also flux-framework/flux-core#4791.
+ */
+static int upgrade_drain_context (const char *name,
+                                  double ts,
+                                  json_t *context,
+                                  flux_t *h)
+{
+    const char *idset;
+    const char *nodelist = NULL;
+    const char *reason = NULL;
+
+    if (json_unpack (context,
+                     "{s:s s?s s?s}",
+                     "idset", &idset,
+                     "nodelist", &nodelist,
+                     "reason", &reason) < 0) {
+        flux_log (h,
+                  LOG_WARNING,
+                  "dropping old %s event with invalid context",
+                  name);
+        return -1;
+    }
+    if (!nodelist) {
+        char *nl;
+        json_t *o = NULL;
+
+        if (!(nl = flux_hostmap_lookup (h, idset, NULL))
+            || !(o = json_string (nl))
+            || json_object_set_new (context, "nodelist", o) < 0) {
+            char timebuf[64] = { 0 };
+            timestamp_tostr ((time_t)ts, timebuf, sizeof (timebuf));
+            flux_log (h,
+                      LOG_WARNING,
+                      "dropping old %s event with invalid ranks"
+                      " (ranks=%s timestamp=%s UTC reason=%s)",
+                      name,
+                      idset,
+                      timebuf,
+                      reason ? reason : "");
+            json_decref (o);
+            free (nl);
+            return -1;
+        }
+        free (nl);
+    }
+    return 0;
+}
+
+static ssize_t upgrade_insert_index (json_t *eventlog, double timestamp)
+{
+    size_t index;
+    json_t *entry;
+    json_array_foreach (eventlog, index, entry) {
+        double ts;
+        if (eventlog_entry_parse (entry, &ts, NULL, NULL) < 0)
+            return -1;
+        if (ts >= timestamp)
+            return index;
+    }
+    return 0;
+}
+
+static int upgrade_insert_drain_event (json_t *eventlog,
+                                       double timestamp,
+                                       const char *idset,
+                                       const char *nodelist,
+                                       const char *reason)
+{
+    ssize_t index;
+    json_t *entry;
+
+    if ((index = upgrade_insert_index (eventlog, timestamp)) < 0)
+        return -1;
+    if (reason) {
+        entry = eventlog_entry_pack (timestamp,
+                                     "drain",
+                                     "{s:s s:s s:s}",
+                                     "idset", idset,
+                                     "nodelist", nodelist,
+                                     "reason", reason);
+        }
+    else {
+        entry = eventlog_entry_pack (timestamp,
+                                     "drain",
+                                     "{s:s s:s}",
+                                     "idset", idset,
+                                     "nodelist", nodelist);
+    }
+    if (!entry)
+        return -1;
+    if (json_array_insert_new (eventlog, index, entry) < 0) {
+        json_decref (entry);
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
+
+/* Add drain events to the eventlog that are reconstructed from the drain
+ * summary object of a legacy 'resource-init' event.
+ */
+static int upgrade_resource_init (json_t *context, json_t *eventlog, flux_t *h)
+{
+    const char *idset;
+    json_t *drain;
+    json_t *o;
+
+    if (json_unpack (context, "{s:o}", "drain", &drain) < 0) {
+        errno = EPROTO;
+        return -1;
+    }
+    json_object_foreach (drain, idset, o) {
+        double ts;
+        char *reason = NULL;
+        char *nl;
+
+        if (json_unpack (o,
+                         "{s:f s?s}",
+                         "timestamp", &ts,
+                         "reason", &reason) < 0) {
+            errno = EPROTO;
+            return -1;
+        }
+        if (!(nl = flux_hostmap_lookup (h, idset, NULL))) {
+            char timebuf[64] = { 0 };
+            timestamp_tostr ((time_t)ts, timebuf, sizeof (timebuf));
+            flux_log (h,
+                      LOG_WARNING,
+                      "dropping old drain data with invalid ranks"
+                      " (ranks=%s timestamp=%s UTC reason=%s)",
+                      idset,
+                      timebuf,
+                      reason ? reason : NULL);
+            continue;
+        }
+        if (upgrade_insert_drain_event (eventlog, ts, idset, nl, reason) < 0) {
+            ERRNO_SAFE_WRAP (free, nl);
+            return -1;
+        }
+        free (nl);
+    }
+    return 0;
+}
+
+int upgrade_eventlog (flux_t *h, json_t **eventlog)
+{
+    json_t *newlog = NULL;
+    ssize_t index;
+    json_t *entry;
+    double ts;
+    const char *name;
+    json_t *context;
+
+    if (!*eventlog)
+        return 0;
+    /* Scan backwards for resource-init.  If not found, nothing to do.
+     */
+    for (index = json_array_size (*eventlog) - 1; index >= 0; index--) {
+        if (!(entry = json_array_get (*eventlog, index))
+            || eventlog_entry_parse (entry, NULL, &name, &context) < 0)
+            goto parse_error;
+        if (streq (name, "resource-init"))
+            break;
+    }
+    if (index < 0)
+        return 0;
+
+    /* Create new eventlog containing only expanded drain summary from last
+     * resource-init.  Ignore events prior to that one.
+     */
+    if (!(newlog = json_array ()))
+        goto nomem;
+    if (upgrade_resource_init (context, newlog, h) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "%s: fatal error processing resource-init on line %zu",
+                  RESLOG_KEY,
+                  index + 1);
+        goto error;
+    }
+
+    /* Append more valid events, augmenting drain/undrain with nodelist
+     * as needed.
+     */
+    for (index = index + 1; index < json_array_size (*eventlog); index++) {
+        if (!(entry = json_array_get (*eventlog, index))
+            || eventlog_entry_parse (entry, &ts, &name, &context) < 0)
+            goto parse_error;
+        if (streq (name, "drain") || streq (name, "undrain")) {
+            // logs any drain/undrain events that could not be upgraded
+            if (upgrade_drain_context (name, ts, context, h) < 0)
+                continue;
+        }
+        else if (!streq (name, "resource-define"))
+            continue;
+        if (json_array_append (newlog, entry) < 0)
+            goto nomem;
+    }
+    if (rewrite_eventlog (h, newlog) < 0)
+        goto error;
+    size_t oldsize = json_array_size (*eventlog);
+    size_t newsize = json_array_size (newlog);
+    flux_log (h,
+              LOG_INFO,
+              "%s: reduced from %zu to %zu entries",
+              RESLOG_KEY,
+              oldsize,
+              newsize);
+    json_decref (*eventlog);
+    *eventlog = newlog;
+    return 0;
+nomem:
+    errno = ENOMEM;
+error:
+    ERRNO_SAFE_WRAP (json_decref, newlog);
+    return -1;
+parse_error:
+    flux_log (h, LOG_ERR, "%s: parse error on line %zu", RESLOG_KEY, index + 1);
+    json_decref (newlog);
+    errno = EINVAL;
+    return -1;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/resource/upgrade.h b/src/modules/resource/upgrade.h
new file mode 100644
index 000000000000..59456d4124f6
--- /dev/null
+++ b/src/modules/resource/upgrade.h
@@ -0,0 +1,21 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _FLUX_RESOURCE_UPGRADE_H
+#define _FLUX_RESOURCE_UPGRADE_H
+
+#include <jansson.h>
+#include <flux/core.h>
+
+int upgrade_eventlog (flux_t *h, json_t **eventlog);
+
+#endif /* !_FLUX_RESOURCE_UPGRADE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sched-simple/Makefile.am b/src/modules/sched-simple/Makefile.am
deleted file mode 100644
index 01211812dda5..000000000000
--- a/src/modules/sched-simple/Makefile.am
+++ /dev/null
@@ -1,95 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) $(JANSSON_CFLAGS)
-
-fluxmod_LTLIBRARIES = \
-	sched-simple.la
-
-noinst_LTLIBRARIES = \
-	libjj.la
-
-noinst_PROGRAMS = \
-	rlist-query
-
-libjj_la_SOURCES = \
-	libjj.h \
-	libjj.c
-
-sched_simple_la_SOURCES = \
-	sched.c \
-	rnode.c \
-	rnode.h \
-	rlist.c \
-	rlist.h
-
-sched_simple_la_LDFLAGS = \
-	$(fluxmod_ldflags) \
-	-module
-
-sched_simple_la_LIBADD = \
-	$(fluxmod_libadd) \
-	libjj.la \
-	$(top_builddir)/src/common/libschedutil/libschedutil.la \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(top_builddir)/src/common/libflux-optparse.la \
-	$(ZMQ_LIBS)
-
-test_ldadd = \
-	$(top_builddir)/src/common/libtap/libtap.la \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(ZMQ_LIBS) $(LIBPTHREAD) $(JANSSON_LIBS)
-
-test_cppflags = \
-	$(AM_CPPFLAGS)
-
-TESTS = \
-	test_rnode.t \
-	test_rlist.t
-
-check_PROGRAMS = \
-	$(TESTS) \
-	rlist-query
-
-test_rnode_t_SOURCES = \
-	rnode.c \
-	rnode.h \
-	test/rnode.c
-test_rnode_t_CPPFLAGS = \
-	$(test_cppflags)
-test_rnode_t_LDADD = \
-	$(test_ldadd)
-
-test_rlist_t_SOURCES = \
-	rnode.c \
-	rnode.h \
-	rlist.c \
-	rlist.h \
-	test/rlist.c
-test_rlist_t_CPPFLAGS = \
-	$(test_cppflags)
-test_rlist_t_LDADD = \
-	$(test_ldadd)
-
-rlist_query_SOURCES = \
-	rnode.c \
-	rnode.h \
-	rlist.c \
-	rlist.h \
-	test/rlist-query.c
-rlist_query_CPPFLAGS = \
-	$(test_cppflags)
-rlist_query_LDADD = \
-	$(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(ZMQ_LIBS) $(LIBPTHREAD) $(JANSSON_LIBS)
diff --git a/src/modules/sched-simple/libjj.c b/src/modules/sched-simple/libjj.c
deleted file mode 100644
index 4402f7335198..000000000000
--- a/src/modules/sched-simple/libjj.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <errno.h>
-#include <string.h>
-#include <jansson.h>
-
-#include "libjj.h"
-
-static int jj_read_level (json_t *o, int level, struct jj_counts *jj);
-
-static int jj_read_vertex (json_t *o, int level, struct jj_counts *jj)
-{
-    int count;
-    const char *type = NULL;
-    json_t *with = NULL;
-    json_error_t error;
-
-    if (json_unpack_ex (o, &error, 0, "{ s:s s:i s?o }",
-                       "type", &type,
-                       "count", &count,
-                       "with", &with) < 0) {
-        snprintf (jj->error, sizeof (jj->error) - 1,
-                  "level %d: %s", level, error.text);
-        errno = EINVAL;
-        return -1;
-    }
-    if (count <= 0) {
-        sprintf (jj->error, "Invalid count %d for type '%s'",
-                            count, type);
-        errno = EINVAL;
-        return -1;
-    }
-    if (strcmp (type, "node") == 0)
-        jj->nnodes = count;
-    else if (strcmp (type, "slot") == 0)
-        jj->nslots = count;
-    else if (strcmp (type, "core") == 0)
-        jj->slot_size = count;
-    else {
-        sprintf (jj->error, "Unsupported resource type '%s'", type);
-        errno = EINVAL;
-        return -1;
-    }
-    if (with)
-        return jj_read_level (with, level+1, jj);
-    return 0;
-
-}
-
-static int jj_read_level (json_t *o, int level, struct jj_counts *jj)
-{
-    int i;
-    json_t *v = NULL;
-
-    if (!json_is_array (o)) {
-        snprintf (jj->error, sizeof (jj->error) - 1,
-                  "level %d: must be an array", level);
-        errno = EINVAL;
-        return -1;
-    }
-    json_array_foreach (o, i, v) {
-        if (jj_read_vertex (v, level, jj) < 0)
-            return -1;
-    }
-    return 0;
-}
-
-int libjj_get_counts (const char *spec, struct jj_counts *jj)
-{
-    int saved_errno;
-    int rc = -1;
-    int version;
-    json_t *resources = NULL;
-    json_t *o = NULL;
-    json_error_t error;
-
-    if (!jj) {
-        errno = EINVAL;
-        return -1;
-    }
-    memset (jj, 0, sizeof (*jj));
-
-    if ((o = json_loads (spec, 0, &error)) == NULL) {
-        snprintf (jj->error, sizeof (jj->error) - 1,
-                  "JSON load: %s", error.text);
-        errno = EINVAL;
-        return -1;
-    }
-
-    if (json_unpack_ex (o, &error, 0, "{s:i,s:o}",
-                        "version", &version,
-                        "resources", &resources) < 0) {
-        snprintf (jj->error, sizeof (jj->error) - 1,
-                  "at top level: %s", error.text);
-        errno = EINVAL;
-        goto err;
-    }
-    if (version != 1) {
-        snprintf (jj->error, sizeof (jj->error) - 1,
-                 "Invalid version: expected 1, got %d", version);
-        errno = EINVAL;
-        goto err;
-    }
-    if (jj_read_level (resources, 0, jj) < 0)
-        goto err;
-
-    if (jj->nslots <= 0) {
-        snprintf (jj->error, sizeof (jj->error) - 1,
-                 "Unable to determine slot count");
-        errno = EINVAL;
-        goto err;
-    }
-    if (jj->slot_size <= 0) {
-        snprintf (jj->error, sizeof (jj->error) - 1,
-                 "Unable to determine slot size");
-        errno = EINVAL;
-        goto err;
-    }
-    if (jj->nnodes)
-        jj->nslots *= jj->nnodes;
-    rc = 0;
-err:
-    saved_errno = errno;
-    json_decref (o);
-    errno = saved_errno;
-    return rc;
-}
-                       
-
-/* vi: ts=4 sw=4 expandtab
- */
diff --git a/src/modules/sched-simple/libjj.h b/src/modules/sched-simple/libjj.h
deleted file mode 100644
index bd845ae7a592..000000000000
--- a/src/modules/sched-simple/libjj.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef HAVE_SCHED_LIBJJ_H
-#define HAVE_SCHED_LIBJJ_H 1
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#define JJ_ERROR_TEXT_LENGTH 256
-
-struct jj_counts {
-    int nnodes;    /* total number of nodes requested */
-    int nslots;    /* total number of slots requested */
-    int slot_size; /* number of cores per slot        */
-
-    char error[JJ_ERROR_TEXT_LENGTH]; /* On error, contains error description */
-};
-
-/*  Parse jobspec from json string `spec`, return resource request summary
- *   in `counts` on success.
- *  Returns 0 on success and -1 on failure with errno set and jj->error[]
- *   with an error message string.
- */
-int libjj_get_counts (const char *spec, struct jj_counts *counts);
-
-#endif /* !HAVE_SCHED_LIBJJ_H */
diff --git a/src/modules/sched-simple/rlist.c b/src/modules/sched-simple/rlist.c
deleted file mode 100644
index 8b0ec03ba525..000000000000
--- a/src/modules/sched-simple/rlist.c
+++ /dev/null
@@ -1,895 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdlib.h>
-#include <assert.h>
-#include <errno.h>
-#include <string.h>
-#include <czmq.h>
-#include <jansson.h>
-
-#include "src/common/libidset/idset.h"
-#include "rnode.h"
-#include "rlist.h"
-#include "libjj.h"
-
-void rlist_destroy (struct rlist *rl)
-{
-    if (rl) {
-        zlistx_destroy (&rl->nodes);
-        free (rl);
-    }
-}
-
-static void rn_free_fn (void **x)
-{
-    rnode_destroy (*(struct rnode **)x);
-    *x = NULL;
-}
-
-struct rlist *rlist_create (void)
-{
-    struct rlist *rl = calloc (1, sizeof (*rl));
-    if (!(rl->nodes = zlistx_new ()))
-        goto err;
-    zlistx_set_destructor (rl->nodes, rn_free_fn);
-    return (rl);
-err:
-    rlist_destroy (rl);
-    return (NULL);
-}
-
-struct rlist *rlist_copy_empty (const struct rlist *orig)
-{
-    struct rnode *n;
-    struct rlist *rl = rlist_create ();
-    if (!rl)
-        return NULL;
-    n = zlistx_first (orig->nodes);
-    while (n) {
-        n = rnode_create_idset (n->rank, n->ids);
-        if (!n || !zlistx_add_end (rl->nodes, n))
-            goto fail;
-        rl->total += rnode_count (n);
-        n = zlistx_next (orig->nodes);
-    }
-    rl->avail = rl->total;
-    return rl;
-fail:
-    rlist_destroy (rl);
-    return NULL;
-}
-
-
-static struct rnode *rlist_find_rank (struct rlist *rl, uint32_t rank)
-{
-    struct rnode *n = zlistx_first (rl->nodes);
-    while (n) {
-        if (n->rank == rank)
-            return (n);
-        n = zlistx_next (rl->nodes);
-    }
-    return NULL;
-}
-
-/*  Compare two values from idset_first()/idset_next():
- *  Returns:
- *    0   : if x == y
- *    > 0 : if x > y
- *    < 0 : if x < y
- *
- *  Where IDSET_INVALID_ID is considered to come before all numbers.
- */
-static int idset_val_cmp (unsigned int x, unsigned int y)
-{
-    if (x == y)
-        return 0;
-    else if (x == IDSET_INVALID_ID)
-        return -1;
-    else if (y == IDSET_INVALID_ID)
-        return  1;
-    else
-        return (x - y);
-}
-
-static int idset_cmp (struct idset *set1, struct idset *set2)
-{
-    int rv = 0;
-    if (!idset_equal (set1, set2)) {
-        /*  Sort on the first non-equal integer (see idset_val_cmp())
-         */
-        unsigned int a = idset_first (set1);
-        unsigned int b = idset_first (set2);
-        while ((rv = idset_val_cmp (a, b)) == 0) {
-            a = idset_next (set1, a);
-            b = idset_next (set2, b);
-        }
-    }
-    return rv;
-}
-
-static int idset_add_set (struct idset *set, struct idset *new)
-{
-    unsigned int i = idset_first (new);
-    while (i != IDSET_INVALID_ID) {
-        if (idset_test (set, i)) {
-            errno = EEXIST;
-            return -1;
-        }
-        if (idset_set (set, i) < 0)
-            return -1;
-        i = idset_next (new, i);
-    }
-    return 0;
-}
-
-static int idset_remove_set (struct idset *set, struct idset *remove)
-{
-    unsigned int i = idset_first (remove);
-    while (i != IDSET_INVALID_ID) {
-        if (!idset_test (set, i)) {
-            errno = ENOENT;
-            return -1;
-        }
-        if (idset_clear (set, i) < 0)
-            return -1;
-        i = idset_next (remove, i);
-    }
-    return 0;
-}
-
-static int rlist_add_rnode (struct rlist *rl, struct rnode *n)
-{
-    struct rnode *found = rlist_find_rank (rl, n->rank);
-    if (found) {
-        if (idset_add_set (found->ids, n->ids) < 0)
-            return (-1);
-        if (idset_add_set (found->avail, n->avail) < 0) {
-            idset_remove_set (found->ids, n->ids);
-            return (-1);
-        }
-    }
-    else if (!zlistx_add_end (rl->nodes, n))
-        return -1;
-    rl->total += rnode_count (n);
-    rl->avail += rnode_avail (n);
-    if (found)
-        rnode_destroy (n);
-    return 0;
-}
-
-static int rlist_append (struct rlist *rl, const char *ranks, json_t *e)
-{
-    int rc = -1;
-    unsigned int n;
-    unsigned int i;
-    const char *corelist = NULL;
-    struct rnode *node;
-    struct idset *ids = idset_decode (ranks);
-    json_error_t err;
-
-    if (!ids || json_unpack_ex (e, &err, 0, "{s:i,s?s}",
-                                "Core", &n,
-                                "cpuset", &corelist) < 0)
-        goto out;
-    i = idset_first (ids);
-    while (i != IDSET_INVALID_ID) {
-        if (corelist)
-            node = rnode_create (i, corelist);
-        else
-            node = rnode_create_count (i, n);
-        if (!node || rlist_add_rnode (rl, node) < 0) {
-            rnode_destroy (node);
-            goto out;
-        }
-        i = idset_next (ids, i);
-    }
-    rc = 0;
-out:
-    idset_destroy (ids);
-    return rc;
-}
-
-struct rlist *rlist_from_hwloc_by_rank (const char *by_rank)
-{
-    struct rlist *rl = NULL;
-    const char *key = NULL;
-    json_t *entry = NULL;
-
-    json_t *o = json_loads (by_rank, 0, NULL);
-    if (o == NULL)
-        return NULL;
-    if (!(rl = rlist_create ()))
-        goto err;
-
-    json_object_foreach (o, key, entry) {
-        if (rlist_append (rl, key, entry) < 0)
-            goto err;
-    }
-    json_decref (o);
-
-    return (rl);
-err:
-    json_decref (o);
-    rlist_destroy (rl);
-    return NULL;
-}
-
-int rlist_append_rank (struct rlist *rl, unsigned int rank, const char *ids)
-{
-    struct rnode *n = rnode_create (rank, ids);
-    if (!n || rlist_add_rnode (rl, n) < 0) {
-        rnode_destroy (n);
-        return -1;
-    }
-    return 0;
-}
-
-int rlist_append_ranks (struct rlist *rl, const char *rank, const char *ids)
-{
-    int rc = -1;
-    unsigned int i;
-    struct idset * ranks = idset_decode (rank);
-    if (!ranks)
-        return -1;
-    i = idset_first (ranks);
-    while (i != IDSET_INVALID_ID) {
-        if (rlist_append_rank (rl, i, ids) < 0)
-            goto err;
-        i = idset_next (ranks, i);
-    }
-    rc = 0;
-err:
-    idset_destroy (ranks);
-    return rc;
-}
-
-int rlist_append_idset (struct rlist *rl, int rank, struct idset *idset)
-{
-    struct rnode *n = rnode_create_idset (rank, idset);
-    if (!n || rlist_add_rnode (rl, n) < 0) {
-        rnode_destroy (n);
-        return -1;
-    }
-    return 0;
-}
-
-static int rlist_append_rank_entry (struct rlist *rl, json_t *entry,
-                                    json_error_t *ep)
-{
-    const char *ranks;
-    const char *cores;
-    if (json_unpack_ex (entry, ep, 0,
-                        "{s:s,s:{s:s}}",
-                        "rank", &ranks,
-                        "children", "core", &cores) < 0) {
-        return -1;
-    }
-    return rlist_append_ranks (rl, ranks, cores);
-}
-
-struct rlist *rlist_from_R (const char *s)
-{
-    int i, version;
-    struct rlist *rl = NULL;
-    json_t *entry = NULL;
-    json_t *R_lite = NULL;
-    json_error_t error;
-
-    json_t *o = json_loads (s, 0, NULL);
-    if (o == NULL)
-        return NULL;
-    if (!o || json_unpack_ex (o, &error, 0,
-                              "{s:i,s:{s:o}}",
-                              "version", &version,
-                              "execution", "R_lite", &R_lite) < 0) {
-        goto err;
-    }
-    if (version != 1)
-        goto err;
-    if (!(rl = rlist_create ()))
-        goto err;
-    json_array_foreach (R_lite, i, entry) {
-        if (rlist_append_rank_entry (rl, entry, &error) < 0)
-            goto err;
-    }
-    json_decref (o);
-    return (rl);
-err:
-    rlist_destroy (rl);
-    json_decref (o);
-    return (NULL);
-}
-
-/* Helper for rlist_compressed */
-struct multi_rnode {
-    struct idset *ids;
-    const struct rnode *rnode;
-};
-
-static int multi_rnode_cmp (struct multi_rnode *x, struct idset *ids)
-{
-    return (idset_cmp (x->rnode->avail, ids));
-}
-
-static void multi_rnode_destroy (struct multi_rnode **mrn)
-{
-    if (mrn && *mrn) {
-        (*mrn)->rnode = NULL;
-        idset_destroy ((*mrn)->ids);
-        free (*mrn);
-        *mrn = NULL;
-    }
-}
-
-struct multi_rnode * multi_rnode_create (struct rnode *rnode)
-{
-    struct multi_rnode *mrn = calloc (1, sizeof (*mrn));
-    if (mrn == NULL)
-        return NULL;
-    if (!(mrn->ids = idset_create (0, IDSET_FLAG_AUTOGROW))
-        || (idset_set (mrn->ids, rnode->rank) < 0))
-        goto fail;
-    mrn->rnode = rnode;
-    return (mrn);
-fail:
-    multi_rnode_destroy (&mrn);
-    return NULL;
-}
-
-json_t *multi_rnode_tojson (struct multi_rnode *mrn)
-{
-    json_t *o = NULL;
-    char *ids = idset_encode (mrn->rnode->avail, IDSET_FLAG_RANGE);
-    char *ranks = idset_encode (mrn->ids, IDSET_FLAG_RANGE);
-
-    if (!ids || !ranks)
-        goto done;
-    o = json_pack ("{s:s,s:{s:s}}", "rank", ranks, "children", "core", ids);
-done:
-    free (ids);
-    free (ranks);
-    return (o);
-}
-
-static zlistx_t * rlist_mrlist (struct rlist *rl)
-{
-    struct rnode *n = NULL;
-    struct multi_rnode *mrn = NULL;
-    zlistx_t *l = zlistx_new ();
-
-    zlistx_set_comparator (l, (czmq_comparator *) multi_rnode_cmp);
-    zlistx_set_destructor (l, (czmq_destructor *) multi_rnode_destroy);
-
-    n = zlistx_first (rl->nodes);
-    while (n) {
-        if (zlistx_find (l, n->avail)) {
-            if (!(mrn = zlistx_handle_item (zlistx_cursor (l)))
-              || idset_set (mrn->ids, n->rank) < 0) {
-                goto fail;
-            }
-        }
-        else if (rnode_avail (n) > 0) {
-            if (!(mrn = multi_rnode_create (n))
-                || !zlistx_add_end (l, mrn)) {
-                goto fail;
-            }
-        }
-        n = zlistx_next (rl->nodes);
-    }
-    return (l);
-fail:
-    zlistx_destroy (&l);
-    return NULL;
-}
-
-static json_t * rlist_compressed (struct rlist *rl)
-{
-    struct multi_rnode *mrn = NULL;
-    json_t *o = json_array ();
-    zlistx_t *l = rlist_mrlist (rl);
-
-    if (!l)
-        return NULL;
-    mrn = zlistx_first (l);
-    while (mrn) {
-        json_t *entry = multi_rnode_tojson (mrn);
-        if (!entry || json_array_append_new (o, entry) != 0) {
-            json_decref (entry);
-            goto fail;
-        }
-        mrn = zlistx_next (l);
-    }
-    zlistx_destroy (&l);
-    return (o);
-fail:
-    zlistx_destroy (&l);
-    json_decref (o);
-    return NULL;
-}
-
-static int
-sprintfcat (char **s, size_t *sz, size_t *lenp, const char *fmt, ...)
-{
-    int done = false;
-    va_list ap;
-    int n = 0;
-    while (!done) {
-        int nleft = *sz-*lenp;
-        va_start (ap, fmt);
-        n = vsnprintf ((*s)+*lenp, nleft, fmt, ap);
-        if (n < 0 || n >= nleft) {
-            char *p;
-            *sz += 128;
-            if (!(p = realloc (*s, *sz)))
-                return -1;
-            *s = p;
-        }
-        else
-            done = true;
-        va_end (ap);
-    }
-    *lenp += n;
-    return (n);
-}
-
-char * rlist_dumps (struct rlist *rl)
-{
-    int flags = IDSET_FLAG_RANGE | IDSET_FLAG_BRACKETS;
-    char * result = NULL;
-    size_t len = 0;
-    size_t size = 64;
-    struct multi_rnode *mrn = NULL;
-    zlistx_t *l = NULL;
-
-    if (rl == NULL) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    if (!(l = rlist_mrlist (rl))
-        || !(result = calloc (size, sizeof (char))))
-        goto fail;
-
-    mrn = zlistx_first (l);
-    while (mrn) {
-        char *ranks = idset_encode (mrn->ids, flags);
-        char *cores = idset_encode (mrn->rnode->avail, flags);
-        if (sprintfcat (&result, &size, &len , "%srank%s/core%s",
-                         result[0] != '\0' ? " ": "",
-                         ranks, cores) < 0)
-            goto fail;
-        free (ranks);
-        free (cores);
-        mrn = zlistx_next (l);
-    }
-    zlistx_destroy (&l);
-    return (result);
-fail:
-    free (result);
-    zlistx_destroy (&l);
-    return NULL;
-}
-
-json_t *rlist_to_R (struct rlist *rl)
-{
-    json_t *R = NULL;
-    json_t *R_lite = rlist_compressed (rl);
-    if (!R_lite)
-        return NULL;
-    if (!(R = json_pack ("{s:i, s:{s:o}}",
-                         "version", 1,
-                         "execution",
-                         "R_lite", R_lite)))
-        json_decref (R_lite);
-    return (R);
-}
-
-static int by_rank (const void *item1, const void *item2)
-{
-    const struct rnode *x = item1;
-    const struct rnode *y = item2;
-    return (x->rank - y->rank);
-}
-
-static int by_avail (const void *item1, const void *item2)
-{
-    int n;
-    const struct rnode *x = item1;
-    const struct rnode *y = item2;
-    if ((n = rnode_avail (x) - rnode_avail (y)) == 0)
-        n = by_rank (x, y);
-    return n;
-}
-
-static int by_used (const void *item1, const void *item2)
-{
-    int n;
-    const struct rnode *x = item1;
-    const struct rnode *y = item2;
-    if ((n = rnode_avail (y) - rnode_avail (x)) == 0)
-        n = by_rank (x, y);
-    return n;
-}
-
-static int rlist_rnode_alloc (struct rlist *rl, struct rnode *n,
-                              int count, struct idset **idsetp)
-{
-    if (!n || rnode_alloc (n, count, idsetp) < 0)
-        return -1;
-    rl->avail -= idset_count (*idsetp);
-    return 0;
-}
-
-#if 0
-static uint32_t rlist_rnode_rank (struct rlist *rl)
-{
-    struct rnode *n = zlistx_item (rl->nodes);
-    if (n)
-        return n->rank;
-    else
-        return (uint32_t)-1;
-}
-#endif
-
-static struct rnode *rlist_first (struct rlist *rl)
-{
-    return zlistx_first (rl->nodes);
-}
-
-static struct rnode *rlist_next (struct rlist *rl)
-{
-    return zlistx_next (rl->nodes);
-}
-
-/*
- *  Allocate the first available N slots of size cores_per_slot from
- *   resource list rl after sorting the nodes with the current sort strategy.
- */
-static struct rlist * rlist_alloc_first_fit (struct rlist *rl,
-                                             int cores_per_slot,
-                                             int slots)
-{
-    int rc;
-    struct idset *ids = NULL;
-    struct rnode *n = NULL;
-    struct rlist *result = NULL;
-
-    zlistx_sort (rl->nodes);
-
-    if (!(n = rlist_first (rl)))
-        return NULL;
-
-    if (!(result = rlist_create ()))
-        return NULL;
-
-    /* 2. assign slots to first nodes where they fit
-     */
-    while (n && slots) {
-        /*  Try to allocate a slot on this node. If we fail with ENOSPC,
-         *   then advance to the next node and try again.
-         */
-        if ((rc = rlist_rnode_alloc (rl, n, cores_per_slot, &ids)) < 0) {
-            if (errno != ENOSPC)
-                goto unwind;
-            n = rlist_next (rl);
-            continue;
-        }
-        /*  Append the allocated cores to the result set and continue
-         *   if needed
-         */
-        rc = rlist_append_idset (result, n->rank, ids);
-        idset_destroy (ids);
-        if (rc < 0)
-            goto unwind;
-        slots--;
-    }
-    if (slots != 0) {
-unwind:
-        rlist_free (rl, result);
-        rlist_destroy (result);
-        errno = ENOSPC;
-        return NULL;
-    }
-    return result;
-}
-
-/*
- *  Allocate `slots` of size cores_per_slot from rlist `rl` and return
- *   the result. Sorts the node list by smallest available first, so that
- *   we get something like "best fit". (minimize nodes used)
- */
-static struct rlist * rlist_alloc_best_fit (struct rlist *rl,
-                                            int cores_per_slot,
-                                            int slots)
-{
-    zlistx_set_comparator (rl->nodes, by_avail);
-    return rlist_alloc_first_fit (rl, cores_per_slot, slots);
-}
-
-/*
- *  Allocate `slots` of size cores_per_slot from rlist `rl` and return
- *   the result. Sorts the node list by least utilized first, so that
- *   we get something like "worst fit". (Spread jobs across nodes)
- */
-static struct rlist * rlist_alloc_worst_fit (struct rlist *rl,
-                                             int cores_per_slot,
-                                             int slots)
-{
-    zlistx_set_comparator (rl->nodes, by_used);
-    return rlist_alloc_first_fit (rl, cores_per_slot, slots);
-}
-
-
-static zlistx_t *rlist_get_nnodes (struct rlist *rl, int nnodes)
-{
-    struct rnode *n;
-    zlistx_t *l = zlistx_new ();
-    if (!l)
-        return NULL;
-    n = zlistx_first (rl->nodes);
-    while (nnodes > 0) {
-        if (zlistx_add_end (l, n) < 0)
-            goto err;
-        n = zlistx_next (rl->nodes);
-        nnodes--;
-    }
-    return (l);
-err:
-    zlistx_destroy (&l);
-    return NULL;
-}
-
-/*  Allocate 'slots' of size 'cores_per_slot' across exactly `nnodes`.
- *  Works by getting the first N least utilized nodes and spreading
- *  the nslots evenly across the result.
- */
-static struct rlist *rlist_alloc_nnodes (struct rlist *rl, int nnodes,
-                                         int cores_per_slot, int slots)
-{
-    struct rlist *result = NULL;
-    struct rnode *n = NULL;
-    zlistx_t *cl = NULL;
-
-    if (rlist_nnodes (rl) < nnodes) {
-        errno = ENOSPC;
-        return NULL;
-    }
-    if (slots < nnodes) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if (!(result = rlist_create ()))
-        return NULL;
-
-    /* 1. sort rank list by used cores ascending:
-     */
-    zlistx_set_comparator (rl->nodes, by_used);
-    zlistx_sort (rl->nodes);
-
-    /* 2. get a list of the first n nodes
-     */
-    if (!(cl = rlist_get_nnodes (rl, nnodes)))
-        return NULL;
-
-    /* We will sort candidate list by used cores on each iteration to
-     *  ensure even spread of slots across nodes
-     */
-    zlistx_set_comparator (cl, by_used);
-
-    /*
-     * 3. divide slots across all nodes, placing each slot
-     *    on most empty node first
-     */
-    while (slots > 0) {
-        int rc;
-        struct idset *ids = NULL;
-        n = zlistx_first (cl);
-        /*
-         * if we can't allocate on this node, give up. Since it is the
-         *  least loaded node from the least loaded nodelist, we know
-         *  we don't have enough resources to satisfy request.
-         */
-        if (rlist_rnode_alloc (rl, n, cores_per_slot, &ids) < 0)
-            goto unwind;
-        rc = rlist_append_idset (result, n->rank, ids);
-        idset_destroy (ids);
-        if (rc < 0)
-            goto unwind;
-
-        /*  If a node is empty, remove it from consideration.
-         *  O/w, force it to the back of the list to ensure all N
-         *   nodes are considered at least once.
-         */
-        zlistx_reorder (cl, zlistx_cursor (cl), false);
-        if (rnode_avail (n) == 0)
-            zlistx_detach (cl, zlistx_cursor (cl));
-        else
-            zlistx_move_end (cl, zlistx_cursor (cl));
-        slots--;
-    }
-    zlistx_destroy (&cl);
-    return result;
-unwind:
-    zlistx_destroy (&cl);
-    rlist_free (rl, result);
-    rlist_destroy (result);
-    errno = ENOSPC;
-    return NULL;
-}
-
-static struct rlist *rlist_try_alloc (struct rlist *rl, const char *mode,
-                                      int nnodes, int slots, int cores_per_slot)
-{
-    struct rlist *result = NULL;
-
-    if (!rl) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    /*  Reset default sort to order nodes by "rank" */
-    zlistx_set_comparator (rl->nodes, by_rank);
-
-    if (nnodes > 0)
-        result = rlist_alloc_nnodes (rl, nnodes, cores_per_slot, slots);
-    else if (mode == NULL || strcmp (mode, "worst-fit") == 0)
-        result = rlist_alloc_worst_fit (rl, cores_per_slot, slots);
-    else if (mode && strcmp (mode, "best-fit") == 0)
-        result = rlist_alloc_best_fit (rl, cores_per_slot, slots);
-    else if (mode && strcmp (mode, "first-fit") == 0)
-        result = rlist_alloc_first_fit (rl, cores_per_slot, slots);
-    else
-        errno = EINVAL;
-    return result;
-}
-
-/*  Determine if allocation request is feasible for rlist `rl`.
- */
-static bool rlist_alloc_feasible (const struct rlist *rl, const char *mode,
-                                  int nnodes, int slots, int slotsz)
-{
-    bool rc = false;
-    struct rlist *result = NULL;
-    struct rlist *all = rlist_copy_empty (rl);
-    if (all && (result = rlist_try_alloc (all, mode, nnodes, slots, slotsz)))
-        rc = true;
-    rlist_destroy (all);
-    rlist_destroy (result);
-    return rc;
-}
-
-struct rlist *rlist_alloc (struct rlist *rl, const char *mode,
-                          int nnodes, int slots, int slotsz)
-{
-    int total = slots * slotsz;
-    struct rlist *result = NULL;
-
-    if (slots <= 0 || slotsz <= 0 || nnodes < 0) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if (total > rl->total) {
-        errno = EOVERFLOW;
-        return NULL;
-    }
-    if (total > rl->avail) {
-        if (rlist_alloc_feasible (rl, mode, nnodes, slots, slotsz))
-            errno = ENOSPC;
-        else
-            errno = EOVERFLOW;
-        return NULL;
-    }
-
-    /*
-     *   Try allocation. If it fails with not enough resources (ENOSPC),
-     *    then try again on an empty copy of rlist to see the request could
-     *    *ever* be satisfied. Adjust errno to EOVERFLOW if not.
-     */
-    result = rlist_try_alloc (rl, mode, nnodes, slots, slotsz);
-    if (!result && (errno == ENOSPC)) {
-        if (rlist_alloc_feasible (rl, mode, nnodes, slots, slotsz))
-            errno = ENOSPC;
-        else
-            errno = EOVERFLOW;
-    }
-    return (result);
-}
-
-static int rlist_free_rnode (struct rlist *rl, struct rnode *n)
-{
-    struct rnode *rnode = rlist_find_rank (rl, n->rank);
-    if (!rnode) {
-        errno = ENOENT;
-        return -1;
-    }
-    if (rnode_free_idset (rnode, n->ids) < 0)
-        return -1;
-    rl->avail += idset_count (n->ids);
-    return 0;
-}
-
-static int rlist_remove_rnode (struct rlist *rl, struct rnode *n)
-{
-    struct rnode *rnode = rlist_find_rank (rl, n->rank);
-    if (!rnode) {
-        errno = ENOENT;
-        return -1;
-    }
-    if (rnode_alloc_idset (rnode, n->avail) < 0)
-        return -1;
-    rl->avail -= idset_count (n->avail);
-    return 0;
-}
-
-int rlist_free (struct rlist *rl, struct rlist *alloc)
-{
-    zlistx_t *freed = NULL;
-    struct rnode *n = NULL;
-
-    if (!(freed = zlistx_new ()))
-        return -1;
-
-    n = zlistx_first (alloc->nodes);
-    while (n) {
-        if (rlist_free_rnode (rl, n) < 0)
-            goto cleanup;
-        zlistx_add_end (freed, n);
-        n = zlistx_next (alloc->nodes);
-    }
-    zlistx_destroy (&freed);
-    return (0);
-cleanup:
-    /* re-allocate all freed items */
-    n = zlistx_first (freed);
-    while (n) {
-        rlist_remove_rnode (rl, n);
-        n = zlistx_next (freed);
-    }
-    zlistx_destroy (&freed);
-    return (-1);
-}
-
-int rlist_remove (struct rlist *rl, struct rlist *alloc)
-{
-    zlistx_t *allocd = NULL;
-    struct rnode *n = NULL;
-    if (!alloc || !(allocd = zlistx_new ()))
-        return -1;
-    n = zlistx_first (alloc->nodes);
-    while (n) {
-        if (rlist_remove_rnode (rl, n) < 0)
-            goto cleanup;
-        zlistx_add_end (allocd, n);
-        n = zlistx_next (alloc->nodes);
-    }
-    zlistx_destroy (&allocd);
-    return 0;
-cleanup:
-    n = zlistx_first (allocd);
-    while (n) {
-        rlist_free_rnode (rl, n);
-        n = zlistx_next (allocd);
-    }
-    zlistx_destroy (&allocd);
-    return -1;
-}
-
-size_t rlist_nnodes (struct rlist *rl)
-{
-    return zlistx_size (rl->nodes);
-}
-
-/* vi: ts=4 sw=4 expandtab
- */
diff --git a/src/modules/sched-simple/rlist.h b/src/modules/sched-simple/rlist.h
deleted file mode 100644
index 8f1538316b5d..000000000000
--- a/src/modules/sched-simple/rlist.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef HAVE_SCHED_RLIST_H
-#define HAVE_SCHED_RLIST_H 1
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <jansson.h>
-#include <czmq.h>
-#include <flux/idset.h>
-
-/* A list of resource nodes */
-struct rlist {
-    int total;
-    int avail;
-    zlistx_t *nodes;
-};
-
-/*  Create an empty rlist object */
-struct rlist *rlist_create (void);
-
-/*  Create a copy of rlist rl with all cores available */
-struct rlist *rlist_copy_empty (const struct rlist *rl);
-
-/*  Create an rlist object from resource.hwloc.by_rank JSON input
- */
-struct rlist *rlist_from_hwloc_by_rank (const char *by_rank);
-
-/*  Destroy an rlist object */
-void rlist_destroy (struct rlist *rl);
-
-/*  Append a new resource node with rank==rank and idset string `ids`
- */
-int rlist_append_rank (struct rlist *rl, unsigned int rank, const char *ids);
-
-/*  Same as rlist_append_rank(), but `ids` is a struct idset
- */
-int rlist_append_idset (struct rlist *rl, int rank, struct idset *ids);
-
-/*  Return number of resource nodes in resource list `rl`
- */
-size_t rlist_nnodes (struct rlist *rl);
-
-/*
- *  Serialize a resource list into v1 "R" format. This encodes only the
- *   "available" ids in each resource node into execution.R_lite
- */
-json_t * rlist_to_R (struct rlist *rl);
-
-/*
- *  Dump short form description of rlist `rl` as a single line string.
- *    Caller must free returned string.
- */
-char *rlist_dumps (struct rlist *rl);
-
-/*
- *  De-serialize a v1 "R" format string into a new resource list object.
- *  Returns a new resource list object on success, NULL on failure.
- */
-struct rlist *rlist_from_R (const char *R);
-
-/*  Attempt to allocate nslots of slot_size across optional nnodes
- *   from the resource list `rl` using algorithm `mode`.
- *
- *  Valid modes (nnodes == 0 only):
- *   NULL or "worst-fit" - allocate from least-used nodes first
- *   "best-fit"          - allocate from most-used nodes first
- *   "first-fit"         - allocate first free slots found in rank order
- *
- *  Returns a new rlist representing the allocation on success,
- *   NULL on failure with errno set:
- *
- *   ENOSPC - unable to fulfill allocation.
- *   EINVAL - An argument was invalid.
- */
-struct rlist * rlist_alloc (struct rlist *rl, const char *mode,
-                            int nnodes, int slot_size, int nslots);
-
-
-/*  Remove rlist "alloc" from rlist "rl".
- */
-int rlist_remove (struct rlist *rl, struct rlist *alloc);
-
-/*  Free resource list `to_free` from resource list `rl`
- */
-int rlist_free (struct rlist *rl, struct rlist *to_free);
-
-#endif /* !HAVE_SCHED_RLIST_H */
diff --git a/src/modules/sched-simple/rnode.c b/src/modules/sched-simple/rnode.c
deleted file mode 100644
index 95e217b4c0b8..000000000000
--- a/src/modules/sched-simple/rnode.c
+++ /dev/null
@@ -1,198 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdlib.h>
-#include <assert.h>
-#include <errno.h>
-#include <jansson.h>
-#include <flux/idset.h>
-
-#include "rnode.h"
-
-void rnode_destroy (struct rnode *n)
-{
-    if (n) {
-        idset_destroy (n->avail);
-        idset_destroy (n->ids);
-        free (n);
-    }
-}
-
-struct rnode *rnode_create (uint32_t rank, const char *ids)
-{
-    struct rnode *n = calloc (1, sizeof (*n));
-    if (n == NULL)
-        return NULL;
-    n->rank = rank;
-    if (!(n->ids = idset_decode (ids))
-        || !(n->avail = idset_copy (n->ids)))
-        goto fail;
-    return (n);
-fail:
-    rnode_destroy (n);
-    return NULL;
-}
-
-struct rnode *rnode_create_idset (uint32_t rank, struct idset *ids)
-{
-    struct rnode *n = calloc (1, sizeof (*n));
-    if (n == NULL)
-        return NULL;
-    n->rank = rank;
-    if (!(n->ids = idset_copy (ids))
-        || !(n->avail = idset_copy (ids)))
-        goto fail;
-    return (n);
-fail:
-    rnode_destroy (n);
-    return NULL;
-}
-
-struct rnode *rnode_create_count (uint32_t rank, int count)
-{
-    struct rnode *n = calloc (1, sizeof (*n));
-    if (n == NULL)
-        return NULL;
-    n->rank = rank;
-    if (!(n->ids = idset_create (0, IDSET_FLAG_AUTOGROW))
-        || (idset_range_set (n->ids, 0, count-1) < 0)
-        || !(n->avail = idset_copy (n->ids)))
-        goto fail;
-    return (n);
-fail:
-    rnode_destroy (n);
-    return NULL;
-}
-
-int rnode_alloc (struct rnode *n, int count, struct idset **setp)
-{
-    struct idset *ids = NULL;
-    unsigned int i;
-    if (idset_count (n->avail) < count) {
-        errno = ENOSPC;
-        return -1;
-    }
-    if (!(ids = idset_create (0, IDSET_FLAG_AUTOGROW)))
-        return -1;
-    i = idset_first (n->avail);
-    while (count--) {
-        idset_set (ids, i);
-        idset_clear (n->avail, i);
-        i = idset_next (n->avail, i);
-    }
-    if (setp != NULL)
-        *setp = ids;
-    return (0);
-}
-
-/*
- *  Test if idset `ids` is a valid set of ids to allocate from the rnode `n`
- *  Return true if the idset is valid, false otherwise.
- */
-static bool alloc_ids_valid (struct rnode *n, struct idset *ids)
-{
-    unsigned int i = idset_first (ids);
-    while (i != IDSET_INVALID_ID) {
-        if (!idset_test (n->ids, i)) {
-            errno = ENOENT;
-            return false;
-        }
-        if (!idset_test (n->avail, i)) {
-            errno = EEXIST;
-            return false;
-        }
-        i = idset_next (ids, i);
-    }
-    return (true);
-}
-
-int rnode_alloc_idset (struct rnode *n, struct idset *ids)
-{
-    unsigned int i;
-    if (!ids) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (!alloc_ids_valid (n, ids))
-        return -1;
-    i = idset_first (ids);
-    while (i != IDSET_INVALID_ID) {
-        idset_clear (n->avail, i);
-        i = idset_next (ids, i);
-    }
-    return 0;
-}
-
-/*
- *  Test if idset `ids` is a valid set of ids to free from the rnode `n`
- *  Return true if the idset is valid, false otherwise.
- */
-static bool free_ids_valid (struct rnode *n, struct idset *ids)
-{
-    unsigned int i = idset_first (ids);
-    while (i != IDSET_INVALID_ID) {
-        if (!idset_test (n->ids, i)) {
-            errno = ENOENT;
-            return false;
-        }
-        if (idset_test (n->avail, i)) {
-            errno = EEXIST;
-            return false;
-        }
-        i = idset_next (ids, i);
-    }
-    return (true);
-}
-
-int rnode_free_idset (struct rnode *n, struct idset *ids)
-{
-    unsigned int i;
-    if (!ids) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (!free_ids_valid (n, ids))
-        return -1;
-    i = idset_first (ids);
-    while (i != IDSET_INVALID_ID) {
-        idset_set (n->avail, i);
-        i = idset_next (ids, i);
-    }
-    return 0;
-}
-
-int rnode_free (struct rnode *n, const char *s)
-{
-    int saved_errno;
-    struct idset *ids = idset_decode (s);
-    int rc = rnode_free_idset (n, ids);
-    saved_errno = errno;
-    idset_destroy (ids);
-    errno = saved_errno;
-    return (rc);
-}
-
-size_t rnode_avail (const struct rnode *n)
-{
-    return (idset_count (n->avail));
-}
-
-size_t rnode_count (const struct rnode *n)
-{
-    return (idset_count (n->ids));
-}
-
-
-/* vi: ts=4 sw=4 expandtab
- */
diff --git a/src/modules/sched-simple/rnode.h b/src/modules/sched-simple/rnode.h
deleted file mode 100644
index ce7db0d36a6f..000000000000
--- a/src/modules/sched-simple/rnode.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef HAVE_SCHED_RNODE_H
-#define HAVE_SCHED_RNODE_H 1
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <inttypes.h>
-#include <jansson.h>
-#include <flux/idset.h>
-
-/* Simple resource node object */
-struct rnode {
-    uint32_t rank;
-    struct idset * ids;
-    struct idset * avail;
-};
-
-/*  Create a resource node object from an existing idset `set`
- */
-struct rnode *rnode_create_idset (uint32_t rank, struct idset *ids);
-
-/*  Create a resource node from a string representation of an idset.
- */
-struct rnode *rnode_create (uint32_t rank, const char *ids);
-
-/*  Create a resource node with `count` ids, starting at 0, i.e.
- *   same as rnode_create (rank, "0-"..count-1).
- */
-struct rnode *rnode_create_count (uint32_t rank, int count);
-
-/*  Destroy rnode object
- */
-void rnode_destroy (struct rnode *n);
-
-/*  Allocate `count` ids from rnode object `n`.
- *   On success, return 0 and allocated ids in `setp`.
- *   On failure, return -1 with errno set:
- *   ENOSPC - there are not `count` ids available in `n`
- *   EINVAL - Invalid arguments
- */
-int rnode_alloc (struct rnode *n, int count, struct idset **setp);
-
-/*  Free the idset `ids` from resource node `n`.
- *  Returns 0 on success, -1 if one or more ids is not allocated.
- */
-int rnode_free (struct rnode *n, const char *ids);
-
-/*  As above, but free a struct idset instead of string.
- */
-int rnode_free_idset (struct rnode *n, struct idset *ids);
-
-/*  Allocate specific idset `ids` from a resource node
- */
-int rnode_alloc_idset (struct rnode *n, struct idset *ids);
-
-/*  Return the number of ids available in resource node `n`.
- */
-size_t rnode_avail (const struct rnode *n);
-
-/*  Return the total number of ids in resource node `n`.
- */
-size_t rnode_count (const struct rnode *n);
-
-#endif /* !HAVE_SCHED_RNODE_H */
diff --git a/src/modules/sched-simple/sched.c b/src/modules/sched-simple/sched.c
index c3e144db86a1..e6dcd84da714 100644
--- a/src/modules/sched-simple/sched.c
+++ b/src/modules/sched-simple/sched.c
@@ -11,31 +11,46 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <czmq.h>
 #include <flux/core.h>
-#include <flux/idset.h>
 #include <flux/schedutil.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
 #include "src/common/libjob/job.h"
-#include "libjj.h"
-#include "rlist.h"
+#include "src/common/libjob/jj.h"
+#include "src/common/libjob/idf58.h"
+#include "src/common/librlist/rlist.h"
+#include "ccan/str/str.h"
+
+// e.g. flux module debug --setbit 0x1 sched-simple
+// e.g. flux module debug --clearbit 0x1 sched-simple
+enum module_debug_flags {
+    DEBUG_FAIL_ALLOC = 1, // while set, alloc requests fail
+    DEBUG_ANNOTATE_REASON_PENDING = 2, // add reason_pending annotation
+    DEBUG_EXPIRATION_UPDATE_DENY = 4,  // deny sched.expiration RPCs
+};
 
 struct jobreq {
     void *handle;
     const flux_msg_t *msg;
     uint32_t uid;
-    int priority;
+    unsigned int priority;
     double t_submit;
     flux_jobid_t id;
     struct jj_counts jj;
+    json_t *constraints;
     int errnum;
 };
 
 struct simple_sched {
     flux_t *h;
-    char *mode;             /* allocation mode */
-    bool single;
+    flux_future_t *acquire_f; /* resource.acquire future */
+
+    char *alloc_mode;             /* allocation mode */
+    char *mode;             /* concurrency mode */
+    unsigned int alloc_limit; /* 0 = unlimited */
+    int schedutil_flags;
     struct rlist *rlist;    /* list of resources */
     zlistx_t *queue;        /* job queue */
     schedutil_t *util_ctx;
@@ -49,13 +64,17 @@ static void jobreq_destroy (struct jobreq *job)
 {
     if (job) {
         flux_msg_decref (job->msg);
+        json_decref (job->constraints);
         ERRNO_SAFE_WRAP (free, job);
     }
 }
 
 static void jobreq_destructor (void **x)
 {
-    jobreq_destroy (*x);
+    if (x) {
+        jobreq_destroy (*x);
+        *x = NULL;
+    }
 }
 
 #define NUMCMP(a,b) ((a)==(b)?0:((a)<(b)?-1:1))
@@ -68,7 +87,7 @@ static int jobreq_cmp (const void *x, const void *y)
     int rc;
 
     if ((rc = (-1)*NUMCMP (j1->priority, j2->priority)) == 0)
-        rc = NUMCMP (j1->t_submit, j2->t_submit);
+        rc = NUMCMP (j1->id, j2->id);
     return rc;
 }
 
@@ -86,21 +105,41 @@ jobreq_find (struct simple_sched *ss, flux_jobid_t id)
 }
 
 static struct jobreq *
-jobreq_create (const flux_msg_t *msg, const char *jobspec)
+jobreq_create (const flux_msg_t *msg)
 {
     struct jobreq *job = calloc (1, sizeof (*job));
+    json_t *jobspec;
 
     if (job == NULL)
         return NULL;
-    if (schedutil_alloc_request_decode (msg,
-                                        &job->id,
-                                        &job->priority,
-                                        &job->uid,
-                                        &job->t_submit) < 0)
+
+    if (flux_msg_unpack (msg,
+                         "{s:I s:i s:i s:f s:o}",
+                         "id", &job->id,
+                         "priority", &job->priority,
+                         "userid", &job->uid,
+                         "t_submit", &job->t_submit,
+                         "jobspec", &jobspec) < 0)
         goto err;
     job->msg = flux_msg_incref (msg);
-    if (libjj_get_counts (jobspec, &job->jj) < 0)
+    if (jj_get_counts_json (jobspec, &job->jj) < 0)
+        job->errnum = errno;
+    else if (job->jj.slot_gpus > 0) {
+        snprintf (job->jj.error,
+                  sizeof (job->jj.error),
+                  "sched-simple does not support resource type 'gpu'");
+        errno = EINVAL;
         job->errnum = errno;
+    }
+    if (json_unpack (jobspec,
+                     "{s:{s?{s?O}}}",
+                     "attributes",
+                       "system",
+                         "constraints", &job->constraints) < 0) {
+        job->errnum = errno;
+        goto err;
+    }
+
     return job;
 err:
     jobreq_destroy (job);
@@ -109,19 +148,30 @@ jobreq_create (const flux_msg_t *msg, const char *jobspec)
 
 static void simple_sched_destroy (flux_t *h, struct simple_sched *ss)
 {
-    struct jobreq *job = zlistx_first (ss->queue);
-    while (job) {
-        flux_respond_error (h, job->msg, ENOSYS, "simple sched exiting");
-        job = zlistx_next (ss->queue);
+    if (ss) {
+        int saved_errno = errno;
+        if (ss->queue) {
+            struct jobreq *job = zlistx_first (ss->queue);
+            while (job) {
+                flux_respond_error (h,
+                                    job->msg,
+                                    ENOSYS,
+                                    "simple sched exiting");
+                job = zlistx_next (ss->queue);
+            }
+            zlistx_destroy (&ss->queue);
+        }
+        flux_future_destroy (ss->acquire_f);
+        flux_watcher_destroy (ss->prep);
+        flux_watcher_destroy (ss->check);
+        flux_watcher_destroy (ss->idle);
+        schedutil_destroy (ss->util_ctx);
+        rlist_destroy (ss->rlist);
+        free (ss->alloc_mode);
+        free (ss->mode);
+        free (ss);
+        errno = saved_errno;
     }
-    zlistx_destroy (&ss->queue);
-    flux_watcher_destroy (ss->prep);
-    flux_watcher_destroy (ss->check);
-    flux_watcher_destroy (ss->idle);
-    schedutil_destroy (ss->util_ctx);
-    rlist_destroy (ss->rlist);
-    free (ss->mode);
-    free (ss);
 }
 
 static struct simple_sched * simple_sched_create (void)
@@ -130,20 +180,48 @@ static struct simple_sched * simple_sched_create (void)
     if (ss == NULL)
         return NULL;
 
-    /* Single alloc request mode is default */
-    ss->single = true;
+    /* default limit to 8, testing shows quite good throughput without
+     * concurrency being excessively large.
+     */
+    ss->alloc_limit = 8;
     return ss;
 }
 
-static char *Rstring_create (struct rlist *l)
+static char *Rstring_create (struct simple_sched *ss,
+                             struct rlist *l,
+                             double now,
+                             double timelimit)
 {
     char *s = NULL;
-    json_t *R = rlist_to_R (l);
-    if (R) {
+    json_t *R = NULL;
+    l->starttime = now;
+    l->expiration = 0.;
+    if (timelimit > 0.) {
+        l->expiration = now + timelimit;
+    }
+    else if (ss->rlist->expiration > 0.) {
+        l->expiration = ss->rlist->expiration;
+    }
+    if ((R = rlist_to_R (l))) {
         s = json_dumps (R, JSON_COMPACT);
         json_decref (R);
     }
-    return (s);
+    return s;
+}
+
+static struct rlist *sched_alloc (struct simple_sched *ss,
+                                  struct jobreq *job,
+                                  flux_error_t *errp)
+{
+    struct rlist_alloc_info ai = {
+        .mode = ss->alloc_mode,
+        .nnodes = job->jj.nnodes,
+        .nslots = job->jj.nslots,
+        .slot_size = job->jj.slot_size,
+        .exclusive = job->jj.exclusive,
+        .constraints = job->constraints
+    };
+    return rlist_alloc (ss->rlist, &ai, errp);
 }
 
 static int try_alloc (flux_t *h, struct simple_sched *ss)
@@ -154,31 +232,52 @@ static int try_alloc (flux_t *h, struct simple_sched *ss)
     struct jj_counts *jj = NULL;
     char *R = NULL;
     struct jobreq *job = zlistx_first (ss->queue);
+    double now = flux_reactor_now (flux_get_reactor (h));
+    bool fail_alloc = flux_module_debug_test (h, DEBUG_FAIL_ALLOC, false);
+    flux_error_t error;
+
     if (!job)
         return -1;
+
     jj = &job->jj;
-    alloc = rlist_alloc (ss->rlist, ss->mode,
-                         jj->nnodes, jj->nslots, jj->slot_size);
-    if (!alloc) {
+    if (!fail_alloc) {
+        errno = 0;
+        alloc = sched_alloc (ss, job, &error);
+    }
+    if (!alloc || !(R = Rstring_create (ss, alloc, now, jj->duration))) {
         const char *note = "unable to allocate provided jobspec";
-        if (errno == ENOSPC)
+        if (alloc != NULL) {
+            /*  unlikely: allocation succeeded but Rstring_create failed */
+            note = "internal scheduler error generating R";
+            flux_log (ss->h, LOG_ERR, "%s", note);
+            if (rlist_free (ss->rlist, alloc) < 0)
+                flux_log_error (h, "try_alloc: rlist_free");
+            rlist_destroy (alloc);
+            alloc = NULL;
+        }
+        else if (errno == ENOSPC)
             return rc;
-        if (errno == EOVERFLOW)
+        else if (errno == EOVERFLOW)
             note = "unsatisfiable request";
-        if (schedutil_alloc_respond_denied (ss->util_ctx,
-                                            job->msg,
-                                            note) < 0)
-            flux_log_error (h, "schedutil_alloc_respond_denied");
+        else if (fail_alloc)
+            note = "DEBUG_FAIL_ALLOC";
+        if (schedutil_alloc_respond_deny (ss->util_ctx, job->msg, note) < 0)
+            flux_log_error (h, "schedutil_alloc_respond_deny");
         goto out;
     }
     s = rlist_dumps (alloc);
-    if (!(R = Rstring_create (alloc)))
-        flux_log_error (h, "Rstring_create");
 
-    if (R && schedutil_alloc_respond_R (ss->util_ctx, job->msg, R, s) < 0)
-        flux_log_error (h, "schedutil_alloc_respond_R");
+    if (schedutil_alloc_respond_success_pack (ss->util_ctx,
+                                              job->msg,
+                                              R,
+                                              "{ s:{s:s s:n s:n} }",
+                                              "sched",
+                                                "resource_summary", s,
+                                                "reason_pending",
+                                                "jobs_ahead") < 0)
+        flux_log_error (h, "schedutil_alloc_respond_success_pack");
 
-    flux_log (h, LOG_DEBUG, "alloc: %ju: %s", (uintmax_t) job->id, s);
+    flux_log (h, LOG_DEBUG, "alloc: %s: %s", idf58 (job->id), s);
     rc = 0;
 
 out:
@@ -189,8 +288,32 @@ static int try_alloc (flux_t *h, struct simple_sched *ss)
     return rc;
 }
 
-static void prep_cb (flux_reactor_t *r, flux_watcher_t *w,
-                     int revents, void *arg)
+static void annotate_reason_pending (struct simple_sched *ss)
+{
+    int jobs_ahead = 0;
+
+    if (!flux_module_debug_test (ss->h, DEBUG_ANNOTATE_REASON_PENDING, false))
+        return;
+
+    struct jobreq *job = zlistx_first (ss->queue);
+    while (job) {
+        if (schedutil_alloc_respond_annotate_pack (ss->util_ctx,
+                                                   job->msg,
+                                                   "{ s:{s:s s:i} }",
+                                                   "sched",
+                                                   "reason_pending",
+                                                     "insufficient resources",
+                                                   "jobs_ahead",
+                                                     jobs_ahead++) < 0)
+            flux_log_error (ss->h, "schedutil_alloc_respond_annotate_pack");
+        job = zlistx_next (ss->queue);
+    }
+}
+
+static void prep_cb (flux_reactor_t *r,
+                     flux_watcher_t *w,
+                     int revents,
+                     void *arg)
 {
     struct simple_sched *ss = arg;
     /* if there is at least one job to schedule, start check and idle */
@@ -201,8 +324,10 @@ static void prep_cb (flux_reactor_t *r, flux_watcher_t *w,
     }
 }
 
-static void check_cb (flux_reactor_t *r, flux_watcher_t *w,
-                      int revents, void *arg)
+static void check_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
 {
     struct simple_sched *ss = arg;
     flux_watcher_stop (ss->idle);
@@ -212,39 +337,71 @@ static void check_cb (flux_reactor_t *r, flux_watcher_t *w,
      *  watcher, i.e. block. O/w, retry on next loop.
      */
     if (try_alloc (ss->h, ss) < 0 && errno == ENOSPC) {
+        annotate_reason_pending (ss);
         flux_watcher_stop (ss->prep);
         flux_watcher_stop (ss->check);
     }
 }
 
-static int try_free (flux_t *h, struct simple_sched *ss, const char *R)
+static int try_free (flux_t *h,
+                     struct simple_sched *ss,
+                     flux_jobid_t id,
+                     json_t *R,
+                     bool final)
 {
     int rc = -1;
     char *r = NULL;
-    struct rlist *alloc = rlist_from_R (R);
+    json_error_t error;
+    struct rlist *alloc = rlist_from_json (R, &error);
     if (!alloc) {
-        flux_log_error (h, "hello: unable to parse R=%s", R);
+        char *s = json_dumps (R, JSON_COMPACT);
+        flux_log_error (h, "free: unable to parse R=%s: %s", s, error.text);
+        ERRNO_SAFE_WRAP (free, s);
         return -1;
     }
     r = rlist_dumps (alloc);
     if ((rc = rlist_free (ss->rlist, alloc)) < 0)
         flux_log_error (h, "free: %s", r);
-    else
-        flux_log (h, LOG_DEBUG, "free: %s", r);
+    else {
+        flux_log (h,
+                  LOG_DEBUG,
+                  "free: %s %s%s",
+                  r,
+                  idf58 (id),
+                  final ? " (final)" : "");
+    }
     free (r);
     rlist_destroy (alloc);
     return rc;
 }
 
-void free_cb (flux_t *h, const flux_msg_t *msg, const char *R, void *arg)
+void free_cb (flux_t *h, const flux_msg_t *msg, const char *R_str, void *arg)
 {
     struct simple_sched *ss = arg;
+    json_t *R;
+    flux_jobid_t id;
+    int final = 0;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:o s?b}",
+                             "id", &id,
+                             "R", &R,
+                             "final", &final) < 0) {
+        flux_log (h, LOG_ERR, "free: error unpacking sched.free request");
+        return;
+    }
 
-    if (try_free (h, ss, R) < 0) {
-        if (flux_respond_error (h, msg, errno, NULL) < 0)
-            flux_log_error (h, "free_cb: flux_respond_error");
+    if (try_free (h, ss, id, R, final) < 0) {
+        flux_log_error (h, "free: could not free R. Stopping scheduler.");
+        /* Make this error fatal to the scheduler so that tests will fail.
+         */
+        flux_reactor_stop_error (flux_get_reactor (h));
         return;
     }
+    /* This is a no-op now that sched.free requires no response
+     * but we still call it to get test coverage.
+     */
     if (schedutil_free_respond (ss->util_ctx, msg) < 0)
         flux_log_error (h, "free_cb: schedutil_free_respond");
 
@@ -252,35 +409,45 @@ void free_cb (flux_t *h, const flux_msg_t *msg, const char *R, void *arg)
     flux_watcher_start (ss->prep);
 }
 
-static void alloc_cb (flux_t *h, const flux_msg_t *msg,
-                      const char *jobspec, void *arg)
+static void alloc_cb (flux_t *h, const flux_msg_t *msg, void *arg)
 {
     struct simple_sched *ss = arg;
     struct jobreq *job;
+    bool search_dir;
 
-    if (ss->single && zlistx_size (ss->queue) > 0) {
-        flux_log (h, LOG_ERR, "alloc received before previous one handled");
+    if (ss->alloc_limit
+        && zlistx_size (ss->queue) >= ss->alloc_limit) {
+        flux_log (h,
+                  LOG_ERR,
+                  "alloc received above max concurrency: %d",
+                  ss->alloc_limit);
         errno = EINVAL;
         goto err;
     }
-    if (!(job = jobreq_create (msg, jobspec))) {
+    if (!(job = jobreq_create (msg))) {
         flux_log_error (h, "alloc: jobreq_create");
         goto err;
     }
     if (job->errnum != 0) {
-        if (schedutil_alloc_respond_denied (ss->util_ctx,
-                                            msg,
-                                            job->jj.error) < 0)
-            flux_log_error (h, "alloc_respond_denied");
+        if (schedutil_alloc_respond_deny (ss->util_ctx,
+                                          msg,
+                                          job->jj.error) < 0)
+            flux_log_error (h, "alloc_respond_deny");
         jobreq_destroy (job);
         return;
     }
-    flux_log (h, LOG_DEBUG, "req: %ju: spec={%d,%d,%d}",
-                            (uintmax_t) job->id, job->jj.nnodes,
-                            job->jj.nslots, job->jj.slot_size);
-    job->handle = zlistx_insert (ss->queue,
-                                 job,
-                                 job->priority > FLUX_JOB_PRIORITY_DEFAULT);
+
+    flux_log (h,
+              LOG_DEBUG,
+              "req: %s: spec={%d,%d,%d} duration=%.1f",
+              idf58 (job->id),
+              job->jj.nnodes,
+              job->jj.nslots,
+              job->jj.slot_size,
+              job->jj.duration);
+
+    search_dir = job->priority > FLUX_JOB_URGENCY_DEFAULT;
+    job->handle = zlistx_insert (ss->queue, job, search_dir);
     flux_watcher_start (ss->prep);
     return;
 err:
@@ -292,110 +459,442 @@ static void alloc_cb (flux_t *h, const flux_msg_t *msg,
  * If a matching job found in queue, respond to the alloc request
  * and "dequeue" it.
  */
-static void cancel_cb (flux_t *h,
-                       flux_jobid_t id,
-                       const char *unused_arg1,
-                       int unused_arg2,
-                       void *arg)
+static void cancel_cb (flux_t *h, const flux_msg_t *msg, void *arg)
 {
     struct simple_sched *ss = arg;
-    struct jobreq *job = jobreq_find (ss, id);
+    flux_jobid_t id;
+    struct jobreq *job;
 
-    if (job) {
+    if (flux_msg_unpack (msg, "{s:I}", "id", &id) < 0) {
+        flux_log_error (h, "invalid sched.cancel request");
+        return;
+    }
+
+    if ((job = jobreq_find (ss, id))) {
         if (schedutil_alloc_respond_cancel (ss->util_ctx, job->msg) < 0) {
             flux_log_error (h, "alloc_respond_cancel");
             return;
         }
         zlistx_delete (ss->queue, job->handle);
+        annotate_reason_pending (ss);
+    }
+}
+
+/* Job manager indicates there is a priority change to a job.  If a
+ * matching job found in queue, update the priority and reorder queue
+ * as necessary.
+ */
+static void prioritize_cb (flux_t *h, const flux_msg_t *msg, void *arg)
+{
+    static int min_sort_size = 4;
+    struct simple_sched *ss = arg;
+    struct jobreq *job;
+    json_t *jobs;
+    size_t index;
+    json_t *arr;
+    size_t count;
+
+    if (flux_request_unpack (msg, NULL, "{s:o}", "jobs", &jobs) < 0)
+        goto proto_error;
+
+    count = json_array_size (jobs);
+    json_array_foreach (jobs, index, arr) {
+        flux_jobid_t id;
+        int64_t priority;
+
+        if (json_unpack (arr, "[I,I]", &id, &priority) < 0)
+            goto proto_error;
+
+        if ((job = jobreq_find (ss, id))) {
+            job->priority = priority;
+            if (count < min_sort_size)
+                zlistx_reorder (ss->queue, job->handle, true);
+        }
     }
+    if (count >= min_sort_size) {
+        zlistx_sort (ss->queue);
+
+        /*  zlistx handles are invalidated after a zlistx_sort(),
+         *   so reacquire them now
+         */
+        job = zlistx_first (ss->queue);
+        while (job) {
+            job->handle = zlistx_cursor (ss->queue);
+            job = zlistx_next (ss->queue);
+        }
+    }
+    annotate_reason_pending (ss);
+    return;
+
+proto_error:
+    flux_log (h, LOG_ERR, "malformed sched.reprioritize request");
+    return;
 }
 
 static int hello_cb (flux_t *h,
-                     flux_jobid_t id,
-                     int priority,
-                     uint32_t userid,
-                     double t_submit,
+                     const flux_msg_t *msg,
                      const char *R,
                      void *arg)
 {
     char *s;
     int rc = -1;
     struct simple_sched *ss = arg;
+    struct rlist *alloc;
+    flux_jobid_t id;
+    unsigned int priority;
+    uint32_t userid;
+    double t_submit;
+
+    if (flux_msg_unpack (msg,
+                         "{s:I s:i s:i s:f}",
+                         "id", &id,
+                         "priority", &priority,
+                         "userid", &userid,
+                         "t_submit", &t_submit) < 0) {
+        flux_log_error (h, "hello: invalid hello payload");
+        return -1;
+    }
+
+    flux_log (h,
+              LOG_DEBUG,
+              "hello: id=%s priority=%u userid=%u t_submit=%0.1f",
+              idf58 (id),
+              priority,
+              (unsigned int)userid,
+              t_submit);
 
-    struct rlist *alloc = rlist_from_R (R);
+    alloc = rlist_from_R (R);
     if (!alloc) {
         flux_log_error (h, "hello: R=%s", R);
         return -1;
     }
     s = rlist_dumps (alloc);
-    if ((rc = rlist_remove (ss->rlist, alloc)) < 0)
+    if ((rc = rlist_set_allocated (ss->rlist, alloc)) < 0)
         flux_log_error (h, "hello: rlist_remove (%s)", s);
     else
         flux_log (h, LOG_DEBUG, "hello: alloc %s", s);
     free (s);
     rlist_destroy (alloc);
-    return 0;
+    return rc;
 }
 
 static void status_cb (flux_t *h, flux_msg_handler_t *mh,
                        const flux_msg_t *msg, void *arg)
 {
     struct simple_sched *ss = arg;
-    json_t *o = NULL;
+    struct rlist *rl = NULL;
+    json_t *all = NULL;
+    json_t *alloc = NULL;
+    json_t *down = NULL;
 
-    if (ss->rlist == NULL) {
-        flux_respond_error (h, msg, EAGAIN, "sched-simple not initialized");
-        return;
+    /* N.B. no need to check if ss->rlist is set.  The reactor is not
+     * run until after synchronous initialization in
+     * simple_sched_init() is complete. */
+
+    /*  Create list of all resources
+     */
+    if (!(rl = rlist_copy_empty (ss->rlist))
+        || rlist_mark_up (rl, "all") < 0
+        || !(all = rlist_to_R (rl))) {
+        flux_log_error (h, "failed to create list of all resources");
+        goto err;
+    }
+    rlist_destroy (rl);
+    rl = NULL;
+
+    /*  Create list of down resources
+     */
+    if (!(rl = rlist_copy_down (ss->rlist))
+        || !(down = rlist_to_R (rl))) {
+        flux_log_error (h, "failed to create list of down resources");
+        goto err;
     }
-    if (!(o = rlist_to_R (ss->rlist))) {
-        flux_log_error (h, "rlist_to_R_compressed");
+    rlist_destroy (rl);
+    rl = NULL;
+
+    /*  Create list of allocated resources
+     */
+    if (!(rl = rlist_copy_allocated (ss->rlist))
+        || !(alloc = rlist_to_R (rl))) {
+        flux_log_error (h, "failed to create list of allocated resources");
         goto err;
     }
-    if (flux_respond_pack (h, msg, "o", o) < 0)
+    rlist_destroy (rl);
+
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:o s:o s:o}",
+                           "all", all,
+                           "allocated", alloc,
+                           "down", down) < 0)
         flux_log_error (h, "flux_respond_pack");
     return;
 err:
-    json_decref (o);
+    rlist_destroy (rl);
+    json_decref (all);
+    json_decref (alloc);
+    json_decref (down);
     if (flux_respond_error (h, msg, errno, NULL) < 0)
         flux_log_error (h, "flux_respond_error");
 }
 
-static int simple_sched_init (flux_t *h, struct simple_sched *ss)
+static void feasibility_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
+{
+    struct simple_sched *ss = arg;
+    struct jj_counts jj;
+    json_t *jobspec;
+    json_t *constraints = NULL;
+    struct rlist *alloc = NULL;
+    const char *errmsg = NULL;
+    flux_error_t error;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:o}",
+                            "jobspec", &jobspec) < 0)
+        goto err;
+    if (json_unpack (jobspec,
+                     "{s:{s?{s?o}}}",
+                     "attributes",
+                       "system",
+                         "constraints", &constraints) < 0)
+        goto err;
+
+    if (jj_get_counts_json (jobspec, &jj) < 0) {
+        errmsg = jj.error;
+        goto err;
+    }
+    if (jj.slot_gpus > 0) {
+        errno = EINVAL;
+        errmsg = "Unsupported resource type 'gpu'";
+        goto err;
+    }
+
+    struct rlist_alloc_info ai = {
+        .mode = ss->alloc_mode,
+        .nnodes = jj.nnodes,
+        .nslots = jj.nslots,
+        .slot_size = jj.slot_size,
+        .constraints = constraints
+    };
+    if (!(alloc = rlist_alloc (ss->rlist, &ai, &error))) {
+        if (errno != ENOSPC) {
+            errmsg = error.text;
+            goto err;
+        }
+        /* Fall-through: if ENOSPC then job is satisfiable */
+    }
+    if (alloc && rlist_free (ss->rlist, alloc) < 0) {
+        /*  If rlist_free() fails we're in trouble because
+         *  ss->rlist will have an invalid allocation. This should
+         *  be rare if not impossible, so just exit the reactor.
+         *
+         *  The sched module can then be reloaded without loss of jobs.
+         */
+        flux_log_error (h, "feasibility_cb: failed to free fake alloc");
+        flux_reactor_stop_error (flux_get_reactor (h));
+        errmsg = "Internal scheduler error";
+        goto err;
+    }
+    rlist_destroy (alloc);
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "feasibility_cb: flux_respond_pack");
+    return;
+err:
+    rlist_destroy (alloc);
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "feasibility_cb: flux_respond_error");
+}
+
+/* For testing purposes, support the sched.expiration RPC even though
+ * sched-simple is not a planning scheduler.
+ */
+static void expiration_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct simple_sched *ss = arg;
+    flux_jobid_t id;
+    double expiration;
+    const char *errmsg = NULL;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:I s:F}",
+                             "id", &id,
+                             "expiration", &expiration) < 0)
+        goto err;
+    if (expiration < 0.) {
+        errno = EINVAL;
+        goto err;
+    }
+    if (flux_module_debug_test (ss->h, DEBUG_EXPIRATION_UPDATE_DENY, false)) {
+        errmsg = "Rejecting expiration update for testing";
+        goto err;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "feasibility_cb: flux_respond_pack");
+    return;
+err:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "expiration_cb: flux_respond_error");
+}
+
+static int ss_resource_update (struct simple_sched *ss, flux_future_t *f)
+{
+    const char *up = NULL;
+    const char *down = NULL;
+    double expiration = -1.;
+    const char *s;
+
+    int rc = flux_rpc_get_unpack (f,
+                                  "{s?s s?s s?F}",
+                                  "up", &up,
+                                  "down", &down,
+                                  "expiration", &expiration);
+    if (rc < 0) {
+        flux_log (ss->h, LOG_ERR, "unpacking acquire response failed");
+        goto err;
+    }
+
+    flux_rpc_get (f, &s);
+    flux_log (ss->h, LOG_DEBUG, "resource update: %s", s);
+
+    /* Update resource states:
+     */
+    if ((up && rlist_mark_up (ss->rlist, up) < 0)
+        || (down && rlist_mark_down (ss->rlist, down) < 0)) {
+        flux_log_error (ss->h, "failed to update resource state");
+        goto err;
+    }
+
+    if (expiration >= 0. && ss->rlist->expiration != expiration) {
+        flux_log (ss->h,
+                  LOG_INFO,
+                  "resource expiration updated to %.2f",
+                  expiration);
+        ss->rlist->expiration = expiration;
+    }
+
+    rc = 0;
+err:
+    flux_future_reset (f);
+    return rc;
+}
+
+static void acquire_continuation (flux_future_t *f, void *arg)
+{
+    struct simple_sched *ss = arg;
+    if (flux_future_get (f, NULL) < 0) {
+        flux_log (ss->h,
+                  LOG_ERR,
+                  "exiting due to resource update failure: %s",
+                  future_strerror (f, errno));
+        flux_reactor_stop (flux_get_reactor (ss->h));
+        return;
+    }
+    if (ss_resource_update (ss, f) == 0)
+        try_alloc (ss->h, ss);
+}
+
+/*  Synchronously acquire resources from resource module.
+ *  Configure internal resource state based on initial acquire response.
+ */
+static int ss_acquire_resources (flux_t *h, struct simple_sched *ss)
 {
     int rc = -1;
-    char *s = NULL;
     flux_future_t *f = NULL;
-    const char *by_rank = NULL;
+    json_t *R;
+    json_error_t e;
+
+    if (!(f = flux_rpc (h,
+                        "resource.acquire",
+                        NULL,
+                        FLUX_NODEID_ANY,
+                        FLUX_RPC_STREAMING))) {
+        flux_log_error (h, "rpc: resources.acquire");
+        goto out;
+    }
+    ss->acquire_f = f;
+    if (flux_rpc_get_unpack (f, "{s:o}", "resources", &R) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "resource.acquire failed: %s",
+                  future_strerror (f, errno));
+        goto out;
+    }
+    if (!(ss->rlist = rlist_from_json (R, &e))) {
+        flux_log_error (h, "rlist_from_json: %s", e.text);
+        goto out;
+    }
+
+    /* Update resource states:
+     * - All resources down by default on first response
+     */
+    if (rlist_mark_down (ss->rlist, "all") < 0) {
+        flux_log_error (h, "failed to set all discovered resources down");
+        goto out;
+    }
 
-    /* synchronously lookup by_rank for initialization */
-    if (!(f = flux_kvs_lookup (h, NULL, FLUX_KVS_WAITCREATE,
-                                  "resource.hwloc.by_rank"))) {
-        flux_log_error (h, "lookup resource.hwloc.by_rank");
+    if (ss_resource_update (ss, f) < 0) {
+        flux_log_error (h, "failed to set initial resource state");
         goto out;
     }
-    if (flux_kvs_lookup_get (f, &by_rank) < 0) {
-        flux_log_error (h, "kvs_lookup_get (resource.hwloc.by_rank)");
+
+    /* Add callback for multi-response acquire RPC
+     */
+    if (flux_future_then (f, -1., acquire_continuation, ss) < 0) {
+        flux_log_error (h, "flux_future_then");
         goto out;
     }
-    if (!(ss->rlist = rlist_from_hwloc_by_rank (by_rank))) {
-        flux_log_error (h, "rank_list_create");
+    rc = 0;
+out:
+    return rc;
+}
+
+static int simple_sched_init (flux_t *h, struct simple_sched *ss)
+{
+    int rc = -1;
+    char *s = NULL;
+    flux_future_t *f = NULL;
+
+    /*  Per RFC 27 register 'feasibility' service for feasibility.check RPC
+     */
+    if (!(f = flux_service_register (h, "feasibility"))
+        || flux_future_get (f, NULL) < 0) {
+        flux_log_error (h, "Failed to register feasibility service");
         goto out;
     }
+
+    /*  Acquire resources from resource module and set initial
+     *   resource state.
+     */
+    if (ss_acquire_resources (h, ss) < 0)
+        goto out;
+
     /*  Complete synchronous hello protocol:
      */
-    if (schedutil_hello (ss->util_ctx, hello_cb, ss) < 0) {
+    if (schedutil_hello (ss->util_ctx) < 0) {
         flux_log_error (h, "schedutil_hello");
         goto out;
     }
     if (schedutil_ready (ss->util_ctx,
-                         ss->single ? "single": "unlimited",
+                         ss->mode ? ss->mode : "limited=8",
                          NULL) < 0) {
         flux_log_error (h, "schedutil_ready");
         goto out;
     }
     s = rlist_dumps (ss->rlist);
-    flux_log (h, LOG_DEBUG, "ready: %d of %d cores: %s",
-                            ss->rlist->avail, ss->rlist->total, s);
+    flux_log (h,
+              LOG_DEBUG,
+              "ready: %d of %d cores: %s",
+              ss->rlist->avail,
+              ss->rlist->total,
+              s);
     free (s);
     rc = 0;
 out:
@@ -403,38 +902,81 @@ static int simple_sched_init (flux_t *h, struct simple_sched *ss)
     return rc;
 }
 
-static char * get_alloc_mode (flux_t *h, const char *mode)
+static char * get_alloc_mode (flux_t *h, const char *alloc_mode)
 {
-    if (strcmp (mode, "worst-fit") == 0
-       || strcmp (mode, "first-fit") == 0
-       || strcmp (mode, "best-fit") == 0)
-        return strdup (mode);
-    flux_log_error (h, "unknown allocation mode: %s\n", mode);
+    if (streq (alloc_mode, "worst-fit")
+        || streq (alloc_mode, "first-fit")
+        || streq (alloc_mode, "best-fit"))
+        return strdup (alloc_mode);
+    flux_log (h, LOG_ERR, "unknown allocation mode: %s", alloc_mode);
     return NULL;
 }
 
+static void set_mode (struct simple_sched *ss, const char *mode)
+{
+    if (strstarts (mode, "limited=")) {
+        char *endptr;
+        int n = strtol (mode+8, &endptr, 0);
+        if (*endptr != '\0' || n <= 0) {
+            flux_log (ss->h, LOG_ERR, "invalid limited value: %s\n", mode);
+            return;
+        }
+        ss->alloc_limit = n;
+    }
+    else if (strcasecmp (mode, "unlimited") == 0) {
+        ss->alloc_limit = 0;
+    }
+    else {
+        flux_log (ss->h, LOG_ERR, "unknown mode: %s", mode);
+        return;
+    }
+    free (ss->mode);
+    if (!(ss->mode = strdup (mode)))
+        flux_log_error (ss->h, "error setting mode: %s", mode);
+}
+
+static struct schedutil_ops ops = {
+    .hello = hello_cb,
+    .alloc = alloc_cb,
+    .free = free_cb,
+    .cancel = cancel_cb,
+    .prioritize = prioritize_cb,
+};
+
 static int process_args (flux_t *h, struct simple_sched *ss,
                          int argc, char *argv[])
 {
     int i;
     for (i = 0; i < argc; i++) {
-        if (strncmp ("mode=", argv[i], 5) == 0) {
-            free (ss->mode);
-            ss->mode = get_alloc_mode (h, argv[i]+5);
+        if (strstarts (argv[i], "alloc-mode=")) {
+            free (ss->alloc_mode);
+            ss->alloc_mode = get_alloc_mode (h, argv[i]+11);
         }
-        else if (strcmp ("unlimited", argv[i]) == 0) {
-            ss->single = false;
+        else if (strstarts (argv[i], "mode=")) {
+            set_mode (ss, argv[i]+5);
+        }
+        else if (streq (argv[i], "test-free-nolookup")) {
+            ss->schedutil_flags |= SCHEDUTIL_FREE_NOLOOKUP;
         }
         else {
             flux_log_error (h, "Unknown module option: '%s'", argv[i]);
+            errno = EINVAL;
             return -1;
         }
     }
     return 0;
 }
 
+
+
 static const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_REQUEST, "sched-simple.status", status_cb, FLUX_ROLE_USER },
+    { FLUX_MSGTYPE_REQUEST, "*.resource-status", status_cb, FLUX_ROLE_USER },
+    { FLUX_MSGTYPE_REQUEST, "*.expiration", expiration_cb, FLUX_ROLE_OWNER },
+    { FLUX_MSGTYPE_REQUEST,
+      "feasibility.check",
+      feasibility_cb,
+      FLUX_ROLE_USER
+    },
     FLUX_MSGHANDLER_TABLE_END,
 };
 
@@ -453,7 +995,7 @@ int mod_main (flux_t *h, int argc, char **argv)
     if (process_args (h, ss, argc, argv) < 0)
         return -1;
 
-    ss->util_ctx = schedutil_create (h, alloc_cb, free_cb, cancel_cb, ss);
+    ss->util_ctx = schedutil_create (h, ss->schedutil_flags, &ops, ss);
     if (ss->util_ctx == NULL) {
         flux_log_error (h, "schedutil_create");
         goto done;
@@ -473,16 +1015,23 @@ int mod_main (flux_t *h, int argc, char **argv)
     zlistx_set_comparator (ss->queue, jobreq_cmp);
     zlistx_set_destructor (ss->queue, jobreq_destructor);
 
+    /* Let `flux module load simple-sched` return before synchronous
+     * initialization with resource and job-manager modules.
+     */
+    if (flux_module_set_running (h) < 0)
+        goto done;
+
     if (simple_sched_init (h, ss) < 0)
         goto done;
+    /* N.B. simple_sched_init() calls schedutil_create(),
+     * which registers the "sched" service  name
+     */
     if (flux_msg_handler_addvec (h, htab, ss, &handlers) < 0) {
         flux_log_error (h, "flux_msg_handler_add");
         goto done;
     }
-    if (flux_reactor_run (r, 0) < 0) {
-        flux_log_error (h, "flux_reactor_run");
+    if (flux_reactor_run (r, 0) < 0)
         goto done;
-    }
     rc = 0;
 done:
     simple_sched_destroy (h, ss);
@@ -490,8 +1039,6 @@ int mod_main (flux_t *h, int argc, char **argv)
     return rc;
 }
 
-MOD_NAME ("sched-simple");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/modules/sched-simple/test/rlist-query.c b/src/modules/sched-simple/test/rlist-query.c
deleted file mode 100644
index 0a340fab76a6..000000000000
--- a/src/modules/sched-simple/test/rlist-query.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-
-#include <flux/core.h>
-#include "src/modules/sched-simple/rlist.h"
-
-int main (int ac, char *av[])
-{
-    const char *s = NULL;
-    char *result = NULL;
-    struct rlist *rl = NULL;
-    flux_future_t *f = NULL;
-    flux_t *h = flux_open (NULL, 0);
-
-    if (h == NULL) {
-        fprintf (stderr, "flux_open: %s\n", strerror (errno));
-        exit (1);
-    }
-    if (!(f = flux_rpc (h, "sched-simple.status", "{}", 0, 0))) {
-        fprintf (stderr, "flux_rpc: %s\n", strerror (errno));
-        exit (1);
-    }
-    if (flux_rpc_get (f, &s) < 0) {
-        fprintf (stderr, "sched-simple.status: %s", strerror (errno));
-        exit (1);
-    }
-    if (!(rl = rlist_from_R (s))) {
-        fprintf (stderr, "unable to read R: %s", strerror (errno));
-        exit (1);
-    }
-    flux_future_destroy (f);
-    result = rlist_dumps (rl);
-    printf ("%s\n", result);
-    free (result);
-    flux_close (h);
-    return 0;
-}
-
-/* vi: ts=4 sw=4 expandtab
- */
diff --git a/src/modules/sched-simple/test/rlist.c b/src/modules/sched-simple/test/rlist.c
deleted file mode 100644
index 69f90c40fb6c..000000000000
--- a/src/modules/sched-simple/test/rlist.c
+++ /dev/null
@@ -1,484 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <unistd.h>
-#include <errno.h>
-#include <jansson.h>
-
-#include "src/common/libtap/tap.h"
-#include "rlist.h"
-
-struct testalloc {
-    int nnodes;
-    int nslots;
-    int slot_size;
-};
-
-struct rlist_test_entry {
-    const char *description;
-    const char *mode;
-    struct testalloc alloc;
-    const char *result;
-    int expected_errno;
-    bool free;
-};
-
-#define RLIST_TEST_END { NULL, NULL, { 0, 0, 0 }, NULL, 0, false }
-
-struct rlist_test_entry test_2n_4c[] = {
-    { "too large of slot returns EOVERFLOW", NULL,
-      { 0, 1, 5 }, NULL, EOVERFLOW, false },
-    { "too many slots returns error", NULL,
-      { 0, 9, 1 }, NULL, EOVERFLOW, false },
-    { "invalid number of nodes returns error", NULL,
-      { -1, 1, 1 }, NULL, EINVAL, false },
-    { "invalid number of slots return error", NULL,
-      { 0, 0, 1 }, NULL, EINVAL, false },
-    { "invalid slot size returns error", NULL,
-      { 0, 1, -1}, NULL, EINVAL, false },
-    { "allocating a single core gets expected result", NULL,
-      { 0, 1, 1 }, "rank0/core0", 0, false },
-    { "allocating another core gets expected result", NULL,
-      { 0, 1, 1 }, "rank1/core0", 0, false },
-    { "allocating another core gets expected result", NULL,
-      { 0, 1, 1 }, "rank0/core1", 0, false },
-    { "allocate 1 slot of size 3 lands on correct node", NULL,
-      { 0, 1, 3 }, "rank1/core[1-3]", 0, false },
-    { "allocate 4 slots of 1 core now returns ENOSPC", NULL,
-      { 0, 4, 1 }, NULL, ENOSPC, false },
-    { "allocate remaining 2 cores", NULL,
-      { 0, 1, 2 }, "rank0/core[2-3]", 0, false },
-    RLIST_TEST_END,
-};
-
-struct rlist_test_entry test_6n_4c[] = {
-    { "best-fit: alloc 1 core", "best-fit",
-      { 0, 1, 1 }, "rank0/core0", 0, false },
-    { "best-fit: alloc 1 slot/size 3 fits on rank0", "best-fit",
-      { 0, 1, 3 }, "rank0/core[1-3]", 0, false },
-    { "best-fit: alloc 2 slots/size 2 fits on rank1","best-fit",
-      { 0, 2, 2 }, "rank1/core[0-3]", 0, false },
-    { "best-fit: alloc 3 slot of size 1",            "best-fit",
-      { 0, 3, 1 }, "rank2/core[0-2]", 0, false },
-    { "best-fit alloc 3 slots of 1 core",            "best-fit",
-      { 0, 3, 1 }, "rank2/core3 rank3/core[0-1]", 0, false },
-    RLIST_TEST_END,
-};
-
-struct rlist_test_entry test_1024n_4c[] = {
-    { "large: 512 nodes with 2 cores", NULL,
-      { 512, 512, 2 }, "rank[0-511]/core[0-1]", 0, false },
-    { "large: 512 slots of 4 cores", NULL,
-      { 0, 512, 4 }, "rank[512-1023]/core[0-3]", 0, true },
-    { "large: 1 core on 10 nodes", NULL,
-      { 10, 10, 1 },  "rank[512-521]/core0", 0, false },
-    { "large: alloc 2 cores on 128 nodes with free", NULL,
-      { 128, 256, 1 }, "rank[522-649]/core[0-1]", 0, true },
-    RLIST_TEST_END,
-};
-
-char *R_create (int ranks, int cores)
-{
-    char *retval;
-    char corelist[64];
-    char ranklist[64];
-    json_t *o = NULL;
-    json_t *R_lite = NULL;
-
-    if ((snprintf (corelist, sizeof (corelist)-1, "0-%d", cores-1) < 0)
-        || (snprintf (ranklist, sizeof (ranklist)-1, "0-%d", ranks -1) < 0))
-        goto err;
-
-    if (!(R_lite = json_pack ("{s:s,s:{s:s}}",
-                    "rank", ranklist,
-                    "children", "core", corelist)))
-        goto err;
-    if (!(o = json_pack ("{s:i, s:{s:[O]}}",
-                   "version", 1,
-                   "execution", "R_lite", R_lite)))
-        goto err;
-    retval = json_dumps (o, JSON_COMPACT);
-    json_decref (o);
-    json_decref (R_lite);
-    return (retval);
-err:
-    json_decref (o);
-    json_decref (R_lite);
-    return NULL;
-}
-
-static struct rlist * rlist_testalloc (struct rlist *rl,
-                                       struct rlist_test_entry *e)
-{
-    return rlist_alloc (rl, e->mode,
-                        e->alloc.nnodes,
-                        e->alloc.nslots,
-                        e->alloc.slot_size);
-}
-
-void run_test_entries (struct rlist_test_entry tests[], int ranks, int cores)
-{
-    struct rlist *rl = NULL;
-    struct rlist *alloc = NULL;
-    struct rlist_test_entry *e = NULL;
-    char *R = R_create (ranks, cores);
-    if (R == NULL)
-        BAIL_OUT ("R_create (ranks=%d, cores=%d) failed", ranks, cores);
-    if (!(rl = rlist_from_R (R)))
-        BAIL_OUT ("rlist_from_R (%s)", R);
-    free (R);
-
-    e = &tests[0];
-    while (e && e->description) {
-        int avail_start = rl->avail;
-
-        alloc = rlist_testalloc (rl, e);
-        if (e->result == NULL) {  // Test for expected failure
-            ok (alloc == NULL && errno == e->expected_errno,
-                "%s: errno=%d", e->description, errno);
-        }
-        else {
-            if (alloc) {
-                char *result = rlist_dumps (alloc);
-                is (result, e->result, "%s: %s", e->description, result);
-                if (e->free) {
-                    ok (rlist_free (rl, alloc) == 0, "rlist_free (%s)", result);
-                    ok (avail_start == rl->avail, "freed all cores");
-                }
-                free (result);
-                rlist_destroy (alloc);
-            }
-            else {
-                fail ("%s: %s", e->description, strerror (errno));
-            }
-        }
-        char *s = rlist_dumps (rl);
-        // diag ("avail=%s", s);
-        free (s);
-        e++;
-    }
-    rlist_destroy (rl);
-}
-
-static void test_simple (void)
-{
-    struct rlist *rl = NULL;
-    struct rlist *alloc = NULL;
-    struct rlist *copy = NULL;
-
-    if (!(rl = rlist_create ()))
-        BAIL_OUT ("Failed to create rlist");
-
-    ok (rl->total == 0 && rl->avail == 0,
-        "rlist_create creates empty list");
-    ok (rlist_append_rank (rl, 0, "0-3") == 0,
-        "rlist_append_rank 0, 0-3");
-    ok (rl->total == 4 && rl->avail == 4,
-        "rlist: avail and total == 4");
-    ok (rlist_append_rank (rl, 1, "0-3") == 0,
-        "rlist_append_rank 1, 0-3");
-    ok (rl->total == 8 && rl->avail == 8,
-        "rlist: avail and total == 4");
-    ok ((alloc = rlist_alloc (rl, NULL, 0, 8, 1)) != NULL,
-        "rlist: alloc all cores works");
-    ok (alloc->total == 8 && alloc->avail == 8,
-        "rlist: alloc: avail = 8, total == 8");
-    ok (rl->total == 8 && rl->avail == 0,
-        "rlist: avail == 0, total == 8");
-    ok ((copy = rlist_copy_empty (rl)) != NULL,
-        "rlist: rlist_copy_empty");
-    ok (copy->total == 8 && copy->avail == 8,
-        "rlist: copy: total = %d, avail = %d", copy->total, copy->avail);
-
-    rlist_destroy (rl);
-    rlist_destroy (alloc);
-    rlist_destroy (copy);
-}
-
-const char by_rank_issue2202[] = "{\
-  \"0\": {\
-    \"Package\": 1,\
-    \"Core\": 1,\
-    \"PU\": 1,\
-    \"cpuset\": \"0\"\
-  },\
-  \"1\": {\
-    \"Package\": 1,\
-    \"Core\": 1,\
-    \"PU\": 1,\
-    \"cpuset\": \"1\"\
-  },\
-  \"2\": {\
-    \"Package\": 1,\
-    \"Core\": 1,\
-    \"PU\": 1,\
-    \"cpuset\": \"2\"\
-  },\
-  \"3\": {\
-    \"Package\": 1,\
-    \"Core\": 1,\
-    \"PU\": 1,\
-    \"cpuset\": \"3\"\
-  }\
-}";
-
-const char by_rank_issue2202b[] = "{\
-\"0\": {\
-    \"Package\": 1,\
-    \"Core\": 2,\
-    \"PU\": 2,\
-    \"cpuset\": \"0-1\"\
-  },\
-  \"1\": {\
-    \"Package\": 1,\
-    \"Core\": 2,\
-    \"PU\": 2,\
-    \"cpuset\": \"0,2\"\
-  },\
-  \"2\": {\
-    \"Package\": 1,\
-    \"Core\": 2,\
-    \"PU\": 2,\
-    \"cpuset\": \"0,3\"\
-  },\
-  \"3\": {\
-    \"Package\": 1,\
-    \"Core\": 2,\
-    \"PU\": 2,\
-    \"cpuset\": \"3-4\"\
-  }\
-}";
-
-
-static void test_issue2202 (void)
-{
-    char *result = NULL;
-    struct rlist *a = NULL;
-
-    struct rlist *rl = rlist_from_hwloc_by_rank (by_rank_issue2202);
-    ok (rl != NULL, "issue2202: rlist_from_by_rank");
-    if (!rl)
-        BAIL_OUT ("unable to create rlist from by_rank_issue2202");
-
-    result = rlist_dumps (rl);
-    is (result,
-        "rank0/core0 rank1/core1 rank2/core2 rank3/core3",
-        "issue2202: rlist_dumps works");
-    free (result);
-
-    a = rlist_alloc (rl, "best-fit", 1, 1, 1);
-    ok (a != NULL,
-        "issue2202: rlist_alloc worked");
-    if (a) {
-        result = rlist_dumps (a);
-        is (result, "rank0/core0", "issue2202: allocated %s", result);
-        free (result);
-        result = rlist_dumps (rl);
-        is (result,
-            "rank1/core1 rank2/core2 rank3/core3",
-            "issue2202: remaining: %s", result);
-        free (result);
-        ok (rlist_free (rl, a) == 0,
-            "issue2202: rlist_free worked: %s", strerror (errno));
-        result = rlist_dumps (rl);
-        is (result,
-            "rank0/core0 rank1/core1 rank2/core2 rank3/core3",
-            "issue2202: rlist now has all cores again");
-        free (result);
-        rlist_destroy (a);
-    }
-    rlist_destroy (rl);
-
-
-    /*  Part B:  test with multiple cores per rank, same cpuset size
-     */
-    rl = rlist_from_hwloc_by_rank (by_rank_issue2202b);
-    ok (rl != NULL, "issue2202: rlist_from_hwloc_by_rank");
-    if (!rl)
-        BAIL_OUT ("unable to create rlist from by_rank_issue2202b");
-
-    result = rlist_dumps (rl);
-    is (result,
-        "rank0/core[0-1] rank1/core[0,2] rank2/core[0,3] rank3/core[3-4]",
-        "issue2202b: rlist_dumps works");
-    free (result);
-
-    a = rlist_alloc (rl, "best-fit", 1, 1, 1);
-    ok (a != NULL,
-        "issue2202b: rlist_alloc worked");
-    if (a) {
-        result = rlist_dumps (a);
-        is (result, "rank0/core0", "issue2202b: allocated %s", result);
-        free (result);
-        result = rlist_dumps (rl);
-        is (result,
-            "rank0/core1 rank1/core[0,2] rank2/core[0,3] rank3/core[3-4]",
-            "issue2202b: remaining: %s", result);
-        free (result);
-        ok (rlist_free (rl, a) == 0,
-            "issue2202b: rlist_free worked: %s", strerror (errno));
-        result = rlist_dumps (rl);
-        is (result,
-            "rank0/core[0-1] rank1/core[0,2] rank2/core[0,3] rank3/core[3-4]",
-            "issue2202b: rlist now has all cores again");
-        free (result);
-        rlist_destroy (a);
-    }
-    rlist_destroy (rl);
-}
-
-const char by_rank_issue2473[] = "{\
-\"0\": {\
-    \"Package\": 1,\
-    \"Core\": 4,\
-    \"PU\": 4,\
-    \"cpuset\": \"0-3\"\
-  },\
-\"1-2\": {\
-    \"Package\": 1,\
-    \"Core\": 2,\
-    \"PU\": 2,\
-    \"cpuset\": \"0-1\"\
-  }\
-}";
-
-static void test_issue2473 (void)
-{
-    char *result;
-    struct rlist *rl;
-    struct rlist *a, *a2;
-
-    rl = rlist_from_hwloc_by_rank (by_rank_issue2473);
-    ok (rl != NULL, "issue2473: add_hwloc_by_rank");
-    if (rl == NULL)
-        BAIL_OUT ("unable to create rlist from by_rank_issue2473");
-
-    ok (rlist_nnodes (rl) == 3,
-        "issue2473: created rlist with 3 nodes");
-    result = rlist_dumps (rl);
-    is (result,
-        "rank0/core[0-3] rank[1-2]/core[0-1]",
-        "issue2473: rlist_dumps works");
-    free (result);
-
-    /* problem: allocated 3 cores on one node */
-    a = rlist_alloc (rl, "worst-fit", 3, 3, 1);
-    ok (a != NULL,
-        "issue2473: rlist_alloc nnodes=3 slots=3 slotsz=1 worked");
-    if (!a)
-        BAIL_OUT ("rlist_alloc failed");
-    ok (rlist_nnodes (a) == 3,
-        "issue2473: allocation has 3 nodes");
-
-    result = rlist_dumps (a);
-    is (result,
-        "rank[0-2]/core0",
-        "issue2473: rlist_dumps shows one core per node");
-    free (result);
-    rlist_free (rl, a);
-
-    /* problem: unsatisfiable */
-    a = rlist_alloc (rl, "worst-fit", 3, 8, 1);
-    ok (a != NULL,
-        "issue2473: rlist_alloc nnodes=3 slots=8 slotsz=1 worked");
-    if (a) {
-        rlist_free (rl, a);
-        rlist_destroy (a);
-    }
-
-    /* not a problem but verify slightly counter-intuitive case discussed
-     * in the issue:
-     * - alloc 1 core on rank0
-     * - ask for 2 cores spread across 2 nodes
-     * - we should get cores on rank[0-1] not rank[1-2]
-     */
-    a = rlist_alloc (rl, "worst-fit", 1, 1, 1);
-    ok (a != NULL,
-        "issue2473: rlist_alloc nnodes=1 slots=1 slotsz=1 worked");
-    if (!a)
-        BAIL_OUT ("rlist_alloc failed");
-
-    result = rlist_dumps (rl);
-    is (result,
-        "rank0/core[1-3] rank[1-2]/core[0-1]",
-        "issue2473: one core was allocated from rank0");
-    free (result);
-
-    a2 = rlist_alloc (rl, "worst-fit", 2, 2, 1);
-    ok (a2 != NULL,
-        "issue2473: rlist_alloc nnodes=2 slots=2 slotsz=1 worked");
-    result = rlist_dumps (a2);
-    is (result,
-        "rank0/core1 rank1/core0",
-        "issue2473: allocated a core from used node, not starting new bin");
-    free (result);
-    rlist_free (rl, a);
-    rlist_destroy (a);
-    rlist_free (rl, a2);
-    rlist_destroy (a2);
-
-    rlist_destroy (rl);
-}
-
-static void test_dumps (void)
-{
-    char *result = NULL;
-    struct rlist *rl = NULL;
-
-    if (!(rl = rlist_create ()))
-        BAIL_OUT ("rlist_dumps: failed to create rlist");
-
-    ok (rlist_dumps (NULL) == NULL,
-        "rlist_dumps (NULL) == NULL");
-
-    result = rlist_dumps (rl);
-    is (result, "",
-        "rlist_dumps: empty list returns empty string");
-    free (result);
-
-    rlist_append_rank (rl, 0, "0-3");
-    result = rlist_dumps (rl);
-    is (result, "rank0/core[0-3]",
-        "rlist_dumps with one rank 4 cores gets expected result");
-    free (result);
-
-    rlist_append_rank (rl, 1, "0-7");
-    result = rlist_dumps (rl);
-    is (result, "rank0/core[0-3] rank1/core[0-7]",
-        "rlist_dumps with two ranks gets expected result");
-    free (result);
-
-    rlist_append_rank (rl, 1234567, "0-12345");
-    rlist_append_rank (rl, 1234568, "0-12346");
-    result = rlist_dumps (rl);
-    is (result, "rank0/core[0-3] rank1/core[0-7] "
-                "rank1234567/core[0-12345] rank1234568/core[0-12346]",
-        "rlist_dumps with long reuslt");
-    free (result);
-    rlist_destroy (rl);
-}
-
-int main (int ac, char *av[])
-{
-    plan (NO_PLAN);
-
-    test_simple ();
-    test_dumps ();
-    run_test_entries (test_2n_4c,       2, 4);
-    run_test_entries (test_6n_4c,       6, 4);
-    run_test_entries (test_1024n_4c, 1024, 4);
-    test_issue2202 ();
-    test_issue2473 ();
-
-    done_testing ();
-}
-
-/* vi: ts=4 sw=4 expandtab
- */
diff --git a/src/modules/sched-simple/test/rnode.c b/src/modules/sched-simple/test/rnode.c
deleted file mode 100644
index e064a54f2342..000000000000
--- a/src/modules/sched-simple/test/rnode.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <errno.h>
-
-#include "src/common/libtap/tap.h"
-#include "rnode.h"
-
-static void rnode_alloc_and_check (struct rnode *n, int count, char *expected)
-{
-    struct idset *ids = NULL;
-    char *result = NULL;
-    int avail = rnode_avail (n);
-    ok (rnode_alloc (n, count, &ids) == 0,
-        "rnode_alloc: count=%d", count);
-    ok (ids != NULL,
-        "rnode_alloc: returns non-null idset");
-    ok (idset_count (ids) == count,
-        "rnode_alloc: returned idset with expected count (%d)",
-        idset_count (ids));
-    if (!(result = idset_encode (ids, IDSET_FLAG_RANGE)))
-        BAIL_OUT ("failed to encode idset result");
-    is (result, expected,
-        "rnode_alloc: count=%d: returned expected result %s", count, result);
-    ok (rnode_avail (n) == avail - count,
-        "rnode_alloc: rnode_avail now %d, expected %d",
-        rnode_avail (n), avail - count);
-    idset_destroy (ids);
-    free (result);
-}
-
-static void rnode_avail_check (struct rnode *n, const char *expected)
-{
-    char *avail = idset_encode (n->avail, IDSET_FLAG_RANGE);
-    if (avail == NULL)
-        BAIL_OUT ("failed to encode n->avail");
-    is (avail, expected,
-        "rnode->avail is expected: %s", avail);
-    free (avail);
-}
-
-int main (int ac, char *av[])
-{
-    struct idset *ids = NULL;
-    struct rnode *n = NULL;
-
-    plan (NO_PLAN);
-
-    if (!(n = rnode_create (0, "0-3")))
-        BAIL_OUT ("could not create an rnode object");
-    ok (rnode_avail (n) == 4,
-        "rnode_avail == 4");
-
-    ok (rnode_alloc (n, 5, &ids) < 0 && errno == ENOSPC,
-        "rnode_alloc too many cores returns errno ENOSPC");
-
-    rnode_alloc_and_check (n, 1, "0");
-    ok (rnode_avail (n) == 3,
-        "rnode_avail == 3");
-    rnode_avail_check (n, "1-3");
-
-    rnode_alloc_and_check (n, 1, "1");
-    ok (rnode_avail (n) == 2,
-        "rnode_avail == 2");
-    rnode_avail_check (n, "2-3");
-
-    rnode_alloc_and_check (n, 2, "2-3");
-    ok (rnode_avail (n) == 0,
-        "rnode_avail == 0");
-    rnode_avail_check (n, "");
-
-    ok (rnode_alloc (n, 1, &ids) < 0 && errno == ENOSPC && ids == NULL,
-        "rnode_alloc on empty rnode fails with ENOSPC");
-
-    ok (rnode_free (n, "3-4") < 0 && errno == ENOENT,
-        "rnode_free with invalid ids fails");
-    ok (rnode_avail (n) == 0,
-        "rnode_avail still is 0");
-    rnode_avail_check (n, "");
-
-    ok (rnode_free (n, "0-1") == 0,
-        "rnode_free (0-1) works");
-    ok (rnode_avail (n) == 2,
-        "rnode_avail now is 2");
-    rnode_avail_check (n, "0-1");
-    ok (rnode_free (n, "0") < 0 && errno == EEXIST,
-        "rnode_free of already available id fails");
-    ok (rnode_avail (n) == 2,
-        "rnode_avail is still 2");
-    ok (rnode_free (n, "3") == 0,
-        "rnode_free '3' works");
-    rnode_avail_check (n, "0-1,3");
-
-    rnode_alloc_and_check (n, 3, "0-1,3");
-
-    rnode_destroy (n);
-
-    n = rnode_create_count (1, 8);
-    if (n == NULL)
-        BAIL_OUT ("rnode_create_count failed");
-    ok (n->rank == 1, "rnode rank set correctly");
-    ok (n != NULL, "rnode_create_count");
-    rnode_avail_check (n, "0-7");
-    rnode_destroy (n);
-
-    struct idset *idset = idset_decode ("0-3");
-    n = rnode_create_idset (3, idset);
-    idset_destroy (idset);
-    if (n == NULL)
-        BAIL_OUT ("rnode_create_idset failed");
-    ok (n != NULL, "rnode_create_idset");
-    ok (n->rank == 3, "rnode rank set correctly");
-    rnode_avail_check (n, "0-3");
-
-    struct idset *alloc = idset_decode ("1,3");
-    ok (rnode_alloc_idset (n, alloc) == 0,
-        "rnode_alloc_idset (1,3)");
-    rnode_avail_check (n, "0,2");
-    ok (rnode_alloc_idset (n, alloc) < 0 && errno == EEXIST,
-        "rnode_alloc_idset with idset already allocated returns EEXIST");
-
-    ok (rnode_free_idset (n, alloc) == 0,
-        "rnode_free_idset (1,3)");
-    rnode_avail_check (n, "0-3");
-
-    ok (rnode_free_idset (n, alloc) < 0 && errno == EEXIST,
-        "rnode_free_idset with idset already available returns EEXIST");
-
-    idset_destroy (alloc);
-    alloc = idset_decode ("4-7");
-    ok (rnode_alloc_idset (n, alloc) < 0 && errno == ENOENT,
-        "rnode_alloc_idset with invalid ids return ENOENT");
-    ok (rnode_free_idset (n, alloc) < 0 && errno == ENOENT,
-        "rnode_free_idset with invalid ids return ENOENT");
-
-    idset_destroy (alloc);
-    rnode_destroy (n);
-    done_testing ();
-}
-
-/* vi: ts=4 sw=4 expandtab
- */
diff --git a/src/modules/sdbus/Makefile.am b/src/modules/sdbus/Makefile.am
new file mode 100644
index 000000000000..5029b927b321
--- /dev/null
+++ b/src/modules/sdbus/Makefile.am
@@ -0,0 +1,81 @@
+AM_CFLAGS = \
+	$(WARNING_CFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	$(CODE_COVERAGE_LIBS)
+
+AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
+	-I$(top_srcdir) \
+	-I$(top_srcdir)/src/include \
+	-I$(top_builddir)/src/common/libflux \
+	-I$(top_srcdir)/src/common/libccan \
+	$(JANSSON_CFLAGS)
+if HAVE_LIBSYSTEMD
+AM_CPPFLAGS += \
+	$(LIBSYSTEMD_CFLAGS)
+endif
+
+noinst_LTLIBRARIES = libsdbus.la
+
+libsdbus_la_SOURCES = \
+	main.c
+if HAVE_LIBSYSTEMD
+libsdbus_la_SOURCES += \
+	sdbus.c \
+	sdbus.h \
+	objpath.c \
+	objpath.h \
+	message.c \
+	message.h \
+	interface.c \
+	interface.h \
+	watcher.c \
+	watcher.h \
+	subscribe.c \
+	subscribe.h \
+	connect.c \
+	connect.h
+endif
+
+test_ldadd = \
+	$(builddir)/libsdbus.la \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libflux-core.la \
+        $(top_builddir)/src/common/libflux-internal.la \
+        $(JANSSON_LIBS)
+if HAVE_LIBSYSTEMD
+test_ldadd += \
+	$(LIBSYSTEMD_LIBS)
+endif
+
+test_cppflags = \
+        $(AM_CPPFLAGS)
+
+test_ldflags = \
+        -no-install
+
+TESTS =
+
+if HAVE_LIBSYSTEMD
+TESTS += \
+	 test_objpath.t \
+	 test_message.t
+endif
+
+check_PROGRAMS = $(TESTS)
+
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+	$(top_srcdir)/config/tap-driver.sh
+
+test_objpath_t_SOURCES = test/objpath.c
+test_objpath_t_CPPFLAGS = $(test_cppflags)
+test_objpath_t_LDADD = $(test_ldadd)
+test_objpath_t_LDFLAGS = $(test_ldflags)
+
+test_message_t_SOURCES = test/message.c
+test_message_t_CPPFLAGS = $(test_cppflags)
+test_message_t_LDADD = $(test_ldadd)
+test_message_t_LDFLAGS = $(test_ldflags)
diff --git a/src/modules/sdbus/README.md b/src/modules/sdbus/README.md
new file mode 100644
index 000000000000..7efa842d73c1
--- /dev/null
+++ b/src/modules/sdbus/README.md
@@ -0,0 +1,45 @@
+## flux systemd dbus bridge
+
+The Flux `sdbus` module connects to `$DBUS_SESSION_BUS_ADDRESS`, which the
+Flux systemd unit file sets to the address of the user level systemd instance
+of the `flux` user.  It then makes D-Bus method calls on behalf of Flux users
+that communicate with `sdbus` using Flux RPCs.  It also allows D-Bus signals
+to be subscribed to via a Flux streaming RPC.
+
+Useful references:
+- [D-Bus specification](https://dbus.freedesktop.org/doc/dbus-specification.html)
+- [The new sd-bus API of systemd](https://0pointer.net/blog/the-new-sd-bus-api-of-systemd.html)
+
+### Moving messages
+
+`sdbus` communicates with D-Bus at a fairly low level so it can operate
+reactively, that is, without busy-polling or blocking.  The main libsystemd
+interfaces used for this are:
+
+- [sd_bus_open_user(3)](https://man7.org/linux/man-pages/man3/sd_bus_open_user.3.html), [sd_bus_close(3)](https://man7.org/linux/man-pages/man3/sd_bus_close.3.html) and friends.
+- [sd_bus_process(3)](https://man7.org/linux/man-pages/man3/sd_bus_process.3.html), [sd_bus_get_fd(3)](https://man7.org/linux/man-pages/man3/sd_bus_get_fd.3.html) and friends
+- [sd_bus_message_send(3)](https://man7.org/linux/man-pages/man3/sd_bus_message_send.3.html)
+
+The Flux `sdbus` module takes care of matching method-reply and method-error
+messages to method-call messages.  The higher level libsystemd functions that
+handle that in the systemd code are not used.
+
+### Message translation
+
+Flux message payloads in JSON are translated to and from D-Bus messages.
+Currently `sdbus` translates a subset of possible D-Bus messages, however
+it does so in a way that leaves the door open for a future translator that
+can handle any message and uses D-Bus introspection to obtain message
+signatures on the fly.
+
+Supported (interface, member) tuples, and their signatures are listed in
+interface.c.  The low-level message translation occurs in message.c.
+Both may be amended as needed to support new method calls.
+
+### Systemd
+
+The main use case of `sdbus` is communicating with systemd.  Some useful
+references are:
+
+- [The D-Bus API of systemd/PID 1](https://www.freedesktop.org/wiki/Software/systemd/dbus/)
+- [systemd.unit(5)](https://man7.org/linux/man-pages/man5/systemd.unit.5.html)
diff --git a/src/modules/sdbus/connect.c b/src/modules/sdbus/connect.c
new file mode 100644
index 000000000000..6665551c1bf2
--- /dev/null
+++ b/src/modules/sdbus/connect.c
@@ -0,0 +1,169 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* connect.c - connect to sd-bus with retries
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h>
+#include <systemd/sd-bus.h>
+
+#include "connect.h"
+
+struct sdconnect {
+    flux_t *h;
+    int attempt;
+    double retry_min;
+    double retry_max;
+    bool first_time;
+};
+
+static void sdconnect_destroy (struct sdconnect *sdc)
+{
+    if (sdc) {
+        int saved_errno = errno;
+        free (sdc);
+        errno = saved_errno;
+    }
+}
+
+static struct sdconnect *sdconnect_create (void)
+{
+    struct sdconnect *sdc;
+
+    if (!(sdc = calloc (1, sizeof (*sdc))))
+        return NULL;
+    return sdc;
+}
+
+
+static void bus_destroy (sd_bus *bus)
+{
+    if (bus) {
+        int saved_errno = errno;
+        sd_bus_flush (bus);
+        sd_bus_close (bus);
+        sd_bus_unref (bus);
+        errno = saved_errno;
+    }
+}
+
+/* The timer callback calls sd_bus_open_user().  If it succeeds, the future
+ * is fulfilled.  If it fails, the timer is re-armed for a calculated timeout.
+ * Retries proceed forever.  If they need to be capped, this can be done by
+ * specifying a flux_future_then() timeout.
+ */
+static void timer_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    flux_future_t *f = arg;
+    struct sdconnect *sdc = flux_future_aux_get (f, "flux::sdc");
+    sd_bus *bus;
+    int e;
+    double timeout;
+
+    sdc->attempt++;
+    timeout = sdc->retry_min * sdc->attempt;
+    if (timeout > sdc->retry_max)
+        timeout = sdc->retry_max;
+
+    if ((e = sd_bus_open_user (&bus)) < 0) {
+        char buf[1024];
+        const char *path = getenv ("DBUS_SESSION_BUS_ADDRESS");
+        if (!path) {
+            if ((path = getenv ("XDG_RUNTIME_DIR"))) {
+                snprintf (buf, sizeof (buf), "unix:path:%s/bus", path);
+                path = buf;
+            }
+        }
+        if (!path)
+            path = "sd_bus_open_user";
+        flux_log (sdc->h,
+                  LOG_INFO,
+                  "%s: %s (retrying in %.0fs)",
+                  path,
+                  strerror (-e),
+                  timeout);
+        goto retry;
+    }
+    flux_log (sdc->h, LOG_INFO, "connected");
+    flux_future_fulfill (f, bus, (flux_free_f)bus_destroy);
+    sdc->attempt = 0;
+    return;
+retry:
+    flux_timer_watcher_reset (w, timeout, 0.);
+    flux_watcher_start (w);
+    return;
+}
+
+/* This function is called when a future returned by sdbus_connect() is
+ * passed to flux_future_get() or flux_future_then().  It starts the connect
+ * timer, which fires immediately if first_time=true; otherwise in retry_min
+ * seconds.
+ */
+static void initialize_cb (flux_future_t *f, void *arg)
+{
+    struct sdconnect *sdc = flux_future_aux_get (f, "flux::sdc");
+    flux_reactor_t *r = flux_future_get_reactor (f);
+    flux_watcher_t *w;
+    double timeout = sdc->first_time ? 0. : sdc->retry_min;
+
+    if (!(w = flux_timer_watcher_create (r, timeout, 0., timer_cb, f))
+        || flux_future_aux_set (f,
+                                NULL,
+                                w,
+                                (flux_free_f)flux_watcher_destroy) < 0) {
+        flux_watcher_destroy (w);
+        goto error;
+    }
+    flux_watcher_start (w);
+    return;
+error:
+    flux_future_fulfill_error (f, errno, NULL);
+}
+
+/* sdbus_connect() returns to the caller without interacting with sd-bus.
+ * The action begins when the user calls flux_future_then() et al on
+ * the returned future.
+ */
+flux_future_t *sdbus_connect (flux_t *h,
+                              bool first_time,
+                              double retry_min,
+                              double retry_max)
+{
+    flux_future_t *f;
+    struct sdconnect *sdc = NULL;
+
+    if (!(f = flux_future_create (initialize_cb, NULL))
+        || !(sdc = sdconnect_create ())
+        || flux_future_aux_set (f,
+                                "flux::sdc",
+                                sdc,
+                                (flux_free_f)sdconnect_destroy) < 0) {
+        sdconnect_destroy (sdc);
+        goto error;
+    }
+    sdc->h = h;
+    sdc->retry_min = retry_min;
+    sdc->retry_max = retry_max;
+    sdc->first_time = first_time;
+    flux_future_set_flux (f, h);
+    return f;
+error:
+    flux_future_destroy (f);
+    return NULL;
+}
+
+// vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/modules/sdbus/connect.h b/src/modules/sdbus/connect.h
new file mode 100644
index 000000000000..faa9a00fcf60
--- /dev/null
+++ b/src/modules/sdbus/connect.h
@@ -0,0 +1,34 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SDBUS_CONNECT_H
+#define _SDBUS_CONNECT_H
+
+#include <flux/core.h>
+
+/* Connect the sd-bus with retries.  When the connect is successful, the
+ * future is fulfilled with an sd_bus object.  When the future is destroyed,
+ * the sd_bus object is flushed, closed, and unreferenced.
+ *
+ * If first_time=true, connect immediately; otherwise, wait retry_min secs.
+ * If the initial connect is unsuccessful, retry in retry_min secs.  If that
+ * is unsuccessful, back off exponentially, leveling off at retry_max secs
+ * between attempts.
+ *
+ * Connect attempt successes and failures are logged at LOG_INFO level.
+ */
+flux_future_t *sdbus_connect (flux_t *h,
+                              bool first_time,
+                              double retry_min,
+                              double retry_max);
+
+#endif /* !_SDBUS_CONNECT_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/interface.c b/src/modules/sdbus/interface.c
new file mode 100644
index 000000000000..b0f27846dc58
--- /dev/null
+++ b/src/modules/sdbus/interface.c
@@ -0,0 +1,368 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* interface.c - D-Bus message translation to/from JSON
+ *
+ * This unfortunately falls short of a generic implementation, so each
+ * D-Bus (interface, member) that we need in Flux requires translation
+ * callbacks here for now.
+ *
+ * To list systemd Manager methods and signatures:
+ *   busctl --user introspect \
+ *      org.freedesktop.systemd1 \
+ *      /org/freedesktop/systemd1 \
+ *      org.freedesktop.systemd1.Manager
+ *
+ * dbus-monitor(1) is a useful debugging tool.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <errno.h>
+#include <systemd/sd-bus.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+#include "objpath.h"
+#include "message.h"
+#include "interface.h"
+
+typedef int (*fromjson_f)(sd_bus_message *m, const char *sig, json_t *param);
+typedef int (*tojson_f)(sd_bus_message *m, const char *sig, json_t *param);
+
+struct xtab {
+    const char *member;
+    const char *fromjson_sig;
+    fromjson_f fromjson;
+    const char *tojson_sig;
+    tojson_f tojson;
+};
+
+static int generic_fromjson (sd_bus_message *m,
+                             const char *sig,
+                             json_t *param)
+{
+    return sdmsg_write (m, sig, param);
+}
+
+static int generic_tojson (sd_bus_message *m,
+                           const char *sig,
+                           json_t *params)
+{
+    return sdmsg_read (m, sig, params);
+}
+
+static int list_units_tojson (sd_bus_message *m,
+                              const char *sig,
+                              json_t *params)
+{
+    int e;
+    json_t *a;
+
+    if (!(a = json_array ()))
+        return -ENOMEM;
+    if ((e = sd_bus_message_enter_container (m, 'a', "(ssssssouso)")) <= 0)
+        goto out;
+    while ((e = sd_bus_message_enter_container (m, 'r', "ssssssouso")) > 0) {
+        json_t *entry;
+        if (!(entry = json_array ())) {
+            e = -ENOMEM;
+            goto out;
+        }
+        if ((e = sdmsg_read (m, "ssssssouso", entry)) <= 0) {
+            if (e == 0)
+                e = -EPROTO;
+            json_decref (entry);
+            goto out;
+        }
+        if (json_array_append_new (a, entry) < 0) {
+            json_decref (entry);
+            e = -ENOMEM;
+            goto out;
+        }
+        if ((e = sd_bus_message_exit_container (m)) < 0)
+            goto out;
+    }
+    if (e < 0 || (e = sd_bus_message_exit_container (m)) < 0)
+        goto out;
+    if (json_array_append_new (params, a) < 0) {
+        e = -ENOMEM;
+        goto out;
+    }
+    return 1;
+out:
+    json_decref (a);
+    return e;
+}
+
+/* This is currently unused in flux so aux is required to be an empty array.
+ */
+static int add_aux_units (sd_bus_message *m, json_t *aux)
+{
+    if (!json_is_array (aux) || json_array_size (aux) > 0)
+        return -EPROTO;
+    return sd_bus_message_append (m, "a(sa(sv))", 0);
+}
+
+// s s a(sv) a(sa(sv))
+static int start_transient_unit_fromjson (sd_bus_message *m,
+                                          const char *sig,
+                                          json_t *params)
+{
+    const char *name;
+    const char *mode;
+    json_t *props;
+    json_t *aux;
+    int e;
+
+    if (json_unpack (params, "[ssoo]", &name, &mode, &props, &aux) < 0)
+        return -EPROTO;
+    if ((e = sd_bus_message_append (m, "s", name)) < 0
+        || (e = sd_bus_message_append (m, "s", mode)) < 0
+        || (e = sdmsg_put (m, "a(sv)", props)) < 0
+        || (e = add_aux_units (m, aux)) < 0)
+        return e;
+    return 0;
+}
+
+/* Manager methods
+ */
+static const struct xtab managertab[] = {
+    { "Subscribe",
+      "",       NULL,
+      "",       NULL,
+    },
+    { "Unsubscribe",
+      "",       NULL,
+      "",       NULL,
+    },
+    { "ListUnitsByPatterns",
+      "asas",           generic_fromjson,
+      "a(ssssssouso)",  list_units_tojson,
+    },
+    { "KillUnit",
+      "ssi",    generic_fromjson,
+      "",       NULL,
+    },
+    { "StopUnit",
+      "ss",     generic_fromjson,
+      "o",      generic_tojson,
+    },
+    { "ResetFailedUnit",
+      "s",      generic_fromjson,
+      "",       NULL,
+    },
+    { "StartTransientUnit",
+      "ssa(sv)a(sa(sv))",   start_transient_unit_fromjson,
+      "o",                  generic_tojson,
+    },
+};
+
+static const struct xtab dbustab[] = {
+    { "AddMatch",
+      "s",      generic_fromjson,
+      "",       NULL
+    },
+    { "RemoveMatch",
+      "s",      generic_fromjson,
+      "",       NULL
+    },
+};
+
+static const struct xtab proptab[] = {
+    { "GetAll",
+      "s",      generic_fromjson,
+      "a{sv}",  generic_tojson
+    },
+    { "Get",
+      "ss",     generic_fromjson,
+      "v",      generic_tojson,
+    },
+    // signal
+    { "PropertiesChanged",
+      "",       NULL,
+      "sa{sv}as",generic_tojson
+    },
+};
+
+
+static const struct xtab *xtab_lookup (const char *interface,
+                                       const char *member,
+                                       flux_error_t *error)
+{
+    const struct xtab *tab = NULL;
+    size_t size = 0;
+
+    if (interface) {
+        if (streq (interface, "org.freedesktop.systemd1.Manager")) {
+            tab = managertab;
+            size = ARRAY_SIZE (managertab);
+        }
+        else if (streq (interface, "org.freedesktop.DBus")) {
+            tab = dbustab;
+            size = ARRAY_SIZE (dbustab);
+        }
+        else if (streq (interface, "org.freedesktop.DBus.Properties")) {
+            tab = proptab;
+            size = ARRAY_SIZE (proptab);
+        }
+    }
+    if (!tab) {
+        errprintf (error, "unknown interface %s", interface);
+        return NULL;
+    }
+    if (member) {
+        for (int i = 0; i < size; i++) {
+            if (streq (tab[i].member, member))
+                return &tab[i];
+        }
+    }
+    errprintf (error, "unknown member %s of interface %s", member, interface);
+    return NULL;
+}
+
+sd_bus_message *interface_request_fromjson (sd_bus *bus,
+                                            json_t *obj,
+                                            flux_error_t *error)
+{
+    json_t *params;
+    const char *destination = "org.freedesktop.systemd1";
+    const char *xpath = "/org/freedesktop/systemd1";
+    const char *interface = "org.freedesktop.systemd1.Manager";
+    const char *member;
+    char *path = NULL;
+    const struct xtab *x;
+    sd_bus_message *m;
+    int e;
+
+    if (json_unpack (obj,
+                     "{s?s s?s s?s s:s s:o}",
+                     "destination", &destination,
+                     "path", &xpath,
+                     "interface", &interface,
+                     "member", &member,
+                     "params", &params) < 0
+        || !json_is_array (params)) {
+        errprintf (error, "malformed request");
+        return NULL;
+    }
+    if (!(x = xtab_lookup (interface, member, error)))
+        return NULL;
+    if (!(path = objpath_encode (xpath))) {
+        errprintf (error, "error encoding object path %s", xpath);
+        return NULL;
+    }
+    if ((e = sd_bus_message_new_method_call (bus,
+                                             &m,
+                                             destination,
+                                             path,
+                                             interface,
+                                             member)) < 0) {
+        errprintf (error, "error creating sd-bus message: %s", strerror (-e));
+        free (path);
+        return NULL;
+    }
+    if (x->fromjson) {
+        if ((e = x->fromjson (m, x->fromjson_sig, params)) < 0) {
+            errprintf (error,
+                       "error translating JSON to %s method-call: %s",
+                       x->member,
+                       strerror (-e));
+            sd_bus_message_unref (m);
+            free (path);
+            return NULL;
+        }
+    }
+    free (path);
+    return m;
+}
+
+json_t *interface_reply_tojson (sd_bus_message *m,
+                                const char *interface,
+                                const char *member,
+                                flux_error_t *error)
+{
+    const struct xtab *x;
+    json_t *o;
+    json_t *params;
+    int e;
+
+    if (!(x = xtab_lookup (interface, member, error)))
+        return NULL;
+    if (!(o = json_pack ("{s:[]}", "params"))) {
+        errprintf (error, "error creating output parameter object");
+        return NULL;
+    }
+    params = json_object_get (o, "params");
+    if (x->tojson) {
+        if ((e = x->tojson (m, x->tojson_sig, params)) <= 0) {
+            if (e == 0)
+                e = -EPROTO;
+            errprintf (error,
+                       "error translating %s method-return to JSON: %s",
+                       x->member,
+                       strerror (-e));
+            json_decref (o);
+            return NULL;
+        }
+    }
+    return o;
+}
+
+json_t *interface_signal_tojson (sd_bus_message *m, flux_error_t *error)
+{
+    const char *iface = sd_bus_message_get_interface (m);
+    const char *member = sd_bus_message_get_member (m);
+    const char *path = sd_bus_message_get_path (m);
+    char *xpath;
+    const struct xtab *x;
+    json_t *o = NULL;
+    json_t *params;
+    int e;
+
+    if (!(x = xtab_lookup (iface, member, error)))
+        return NULL;
+    if (!(xpath = objpath_decode (path))) {
+        errprintf (error, "error decoding object path %s", path);
+        return NULL;
+    }
+    if (!(o = json_pack ("{s:s s:s s:s s:[]}",
+                         "path", xpath,
+                         "interface", iface,
+                         "member", member,
+                         "params"))) {
+        errprintf (error, "error creating output parameter object");
+        free (xpath);
+        return NULL;
+    }
+    params = json_object_get (o, "params");
+    if (x->tojson) {
+        if ((e = x->tojson (m, x->tojson_sig, params)) <= 0) {
+            if (e == 0)
+                e = -EPROTO;
+            errprintf (error,
+                       "error translating %s signal to JSON: %s",
+                       x->member,
+                       strerror (-e));
+            free (o);
+            free (xpath);
+            return NULL;
+        }
+    }
+    free (xpath);
+    return o;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/interface.h b/src/modules/sdbus/interface.h
new file mode 100644
index 000000000000..819bb887e421
--- /dev/null
+++ b/src/modules/sdbus/interface.h
@@ -0,0 +1,31 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SDBUS_INTERFACE_H
+#define _SDBUS_INTERFACE_H
+
+#include <systemd/sd-bus.h>
+#include <jansson.h>
+#include <flux/core.h>
+
+sd_bus_message *interface_request_fromjson (sd_bus *bus,
+                                            json_t *req,
+                                            flux_error_t *error);
+
+json_t *interface_reply_tojson (sd_bus_message *m,
+                                const char *interface,
+                                const char *member,
+                                flux_error_t *error);
+
+json_t *interface_signal_tojson (sd_bus_message *m, flux_error_t *error);
+
+#endif /* !_SDBUS_INTERFACE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/main.c b/src/modules/sdbus/main.c
new file mode 100644
index 000000000000..9c51acb35286
--- /dev/null
+++ b/src/modules/sdbus/main.c
@@ -0,0 +1,57 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* main.c - module main() for sd-bus bridge module
+ *
+ * The sdbus module is built when systemd support is not compiled in
+ * so that attempts to enable it get helpful error messages instead
+ * a generic "not found" error.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+#include "src/common/libutil/errprintf.h"
+
+#if HAVE_LIBSYSTEMD
+#include "sdbus.h"
+#endif
+
+int mod_main (flux_t *h, int argc, char **argv)
+{
+#if HAVE_LIBSYSTEMD
+    struct sdbus_ctx *ctx = NULL;
+    flux_error_t error;
+    int rc = -1;
+
+    if (!(ctx = sdbus_ctx_create (h, &error))) {
+        flux_log (h, LOG_ERR, "%s", error.text);
+        goto error;
+    }
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
+        flux_log_error (h, "reactor exited abnormally");
+        goto error;
+    }
+    rc = 0;
+error:
+    sdbus_ctx_destroy (ctx);
+    return rc;
+#else
+    flux_log (h, LOG_ERR, "flux was not built with systemd support");
+    errno = ENOSYS;
+    return -1;
+#endif
+}
+
+MOD_NAME ("sdbus");
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/message.c b/src/modules/sdbus/message.c
new file mode 100644
index 000000000000..99e36b219bb4
--- /dev/null
+++ b/src/modules/sdbus/message.c
@@ -0,0 +1,486 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* message.c - D-Bus message payload/JSON conversion helpers
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <jansson.h>
+#include <errno.h>
+#include <systemd/sd-bus.h>
+#include <assert.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+#include "objpath.h"
+#include "message.h"
+
+typedef union {
+    uint8_t u8;
+    int16_t i16;
+    uint16_t u16;
+    int32_t i32;
+    uint32_t u32;
+    int64_t i64;
+    uint64_t u64;
+    int i;
+    double f;
+} number_t;
+
+struct tab {
+    uint8_t type;
+    const char *desc;
+};
+
+static struct tab typetab[] = {
+    { SD_BUS_MESSAGE_METHOD_CALL, "method-call" },
+    { SD_BUS_MESSAGE_METHOD_RETURN, "method-return" },
+    { SD_BUS_MESSAGE_METHOD_ERROR, "method-error" },
+    { SD_BUS_MESSAGE_SIGNAL , "signal" },
+};
+
+const char *sdmsg_typestr (sd_bus_message *m)
+{
+    uint8_t type;
+
+    if (sd_bus_message_get_type (m, &type) >= 0) {
+        for (int i = 0; i < ARRAY_SIZE (typetab); i++)
+            if (typetab[i].type == type)
+                return typetab[i].desc;
+    }
+    return "unknown";
+}
+
+static int sdmsg_put_array (sd_bus_message *m, const char *fmt, json_t *o)
+{
+    int e;
+    size_t index;
+    json_t *entry;
+
+    if ((e = sd_bus_message_open_container (m, 'a', fmt)) < 0)
+        return e;
+    json_array_foreach (o, index, entry) {
+        if ((e = sdmsg_put (m, fmt, entry)) < 0)
+            return e;
+    }
+    if ((e = sd_bus_message_close_container (m)) < 0)
+        return e;
+
+    return e;
+}
+
+static int sdmsg_put_string (sd_bus_message *m, char type, json_t *o)
+{
+    int e;
+    switch (type) {
+        case 'g':
+        case 's': {
+            const char *val = json_string_value (o);
+            if (!val)
+                return -EPROTO;
+            if ((e = sd_bus_message_append_basic (m, type, val)) < 0)
+                return e;
+            break;
+        }
+        case 'o': {
+            char *val = objpath_encode (json_string_value (o));
+            if (!val)
+                return -errno;
+            e = sd_bus_message_append_basic (m, type, val);
+            free (val);
+            if (e < 0)
+                return e;
+            break;
+        }
+        default:
+            return -EPROTO;
+    }
+    return 0;
+}
+
+static int sdmsg_put_basic (sd_bus_message *m, char type, json_t *o)
+{
+    number_t n;
+
+    if (!m || !o)
+        return -EPROTO;
+    if (type == 's' || type == 'g' || type == 'o')
+        return sdmsg_put_string (m, type, o);
+    switch (type) {
+        case 'y':
+            n.u8 = json_integer_value (o);
+            break;
+        case 'b':
+            n.i = json_is_true (o) ? 1 : 0;
+            break;
+        case 'n':
+            n.i16 = json_integer_value (o);
+            break;
+        case 'q':
+            n.u16 = json_integer_value (o);
+            break;
+        case 'i':
+            n.i32 = json_integer_value (o);
+            break;
+        case 'u':
+            n.u32 = json_integer_value (o);
+            break;
+        case 'x':
+            n.i64 = json_integer_value (o);
+            break;
+        case 't':
+            n.u64 = json_integer_value (o);
+            break;
+        case 'h':
+            n.i = json_integer_value (o);
+            break;
+        case 'd':
+            n.f = json_real_value (o);
+            break;
+        default:
+            return -EPROTO;
+    }
+    return sd_bus_message_append_basic (m, type, &n);
+}
+
+static int sdmsg_put_variant (sd_bus_message *m, json_t *o)
+{
+    const char *type;
+    json_t *val;
+    int e;
+
+    if (json_unpack (o, "[so]", &type, &val) < 0)
+        return -EPROTO;
+    if ((e = sd_bus_message_open_container (m, 'v', type)) < 0)
+        return e;
+    if ((e = sdmsg_put (m, type, val)) < 0)
+        return e;
+    if ((e = sd_bus_message_close_container (m)) < 0)
+        return e;
+    return 0;
+}
+
+static int sdmsg_put_struct (sd_bus_message *m, const char *fmt, json_t *o)
+{
+    int e;
+
+    if ((e = sd_bus_message_open_container (m, 'r', fmt)) < 0)
+        return e;
+    if ((e = sdmsg_write (m, fmt, o)) < 0)
+        return e;
+    if ((e = sd_bus_message_close_container (m)) < 0)
+        return e;
+    return 0;
+}
+
+int sdmsg_put (sd_bus_message *m, const char *fmt, json_t *o)
+{
+    int e;
+
+    if (streq (fmt, "a(sv)"))
+        e = sdmsg_put_array (m, "(sv)", o);
+    else if (streq (fmt, "a(sasb)"))
+        e = sdmsg_put_array (m, "(sasb)", o);
+    else if (fmt[0] == 'a' && strlen (fmt) == 2)
+        e = sdmsg_put_array (m, &fmt[1], o);
+    else if (streq (fmt, "(sv)"))
+        e = sdmsg_put_struct (m, "sv", o);
+    else if (streq (fmt, "(sasb)"))
+        e = sdmsg_put_struct (m, "sasb", o);
+    else if (streq (fmt, "v"))
+        e = sdmsg_put_variant (m, o);
+    else if (strlen (fmt) == 1)
+        e = sdmsg_put_basic (m, fmt[0], o);
+    else
+        e = -EPROTO;
+    return e;
+}
+
+int sdmsg_write (sd_bus_message *m, const char *fmt, json_t *o)
+{
+    int e;
+
+    if (!json_is_array (o))
+        return -EPROTO;
+
+    int cursor = 0;
+    for (int i = 0; fmt[i] != '\0';) {
+        json_t *entry;
+        char *efmt = NULL;
+
+        if (!(entry = json_array_get (o, cursor++)))
+            return -EPROTO;
+        if (strstarts (&fmt[i], "a(sasb)"))
+            efmt = strndup (&fmt[i], 7);
+        else if (strstarts (&fmt[i], "a(sv)"))
+            efmt = strndup (&fmt[i], 5);
+        else if (fmt[i] == 'a' && strlen (&fmt[i]) > 1)
+            efmt = strndup (&fmt[i], 2);
+        else
+            efmt = strndup (&fmt[i], 1);
+        if (!efmt)
+            return -ENOMEM;
+        if ((e = sdmsg_put (m, efmt, entry)) < 0) {
+            free (efmt);
+            return e;
+        }
+        i += strlen (efmt);
+        free (efmt);
+    }
+    return 0;
+}
+
+static int sdmsg_get_string (sd_bus_message *m, char type, json_t **op)
+{
+    int e;
+    const char *val = NULL;
+    json_t *o = NULL;
+
+    if ((e = sd_bus_message_read_basic (m, type, &val)) <= 0)
+        return e;
+    switch (type) {
+        case 'g':
+        case 's':
+            o = json_string (val);
+            break;
+        case 'o': {
+            char *tmp;
+            if (!(tmp = objpath_decode (val)))
+                return -EPROTO;
+            o = json_string (tmp);
+            free (tmp);
+            break;
+        }
+        default:
+            return -EPROTO;
+    }
+    if (!o)
+        return -ENOMEM;
+    *op = o;
+    return 1;
+}
+
+static int sdmsg_get_basic (sd_bus_message *m, char type, json_t **op)
+{
+    char peek_type;
+    int e;
+    json_t *o;
+    number_t n;
+
+    if ((e = sd_bus_message_peek_type (m, &peek_type, NULL)) <= 0)
+        return e;
+    if (type == 0)
+        type = peek_type;
+    if (type != peek_type)
+        return -EPROTO;
+    if (type == 'g' || type == 's' || type == 'o')
+        return sdmsg_get_string (m, type, op);
+    if ((e = sd_bus_message_read_basic (m, type, &n)) <= 0)
+        return e;
+    switch (type) {
+        case 'y':
+            o = json_integer (n.u8);
+            break;
+        case 'n':
+            o = json_integer (n.i16);
+            break;
+        case 'q':
+            o = json_integer (n.u16);
+            break;
+        case 'i':
+            o = json_integer (n.i32);
+            break;
+        case 'u':
+            o = json_integer (n.u32);
+            break;
+        case 'x':
+            o = json_integer (n.i64);
+            break;
+        case 't':
+            o = json_integer (n.u64);
+            break;
+        case 'b':
+            o = n.i ? json_true () : json_false ();
+            break;
+        case 'h':
+            o = json_integer (n.i);
+            break;
+        case 'd':
+            o = json_real (n.f);
+            break;
+        default:
+            return -EPROTO;
+    }
+    if (!o)
+        return -ENOMEM;
+    *op = o;
+    return 1;
+}
+
+static int sdmsg_get_array (sd_bus_message *m, const char *fmt, json_t **op)
+{
+    json_t *a;
+    int e;
+
+    if (!(a = json_array ()))
+        return -ENOMEM;
+    if ((e = sd_bus_message_enter_container (m, 'a', fmt)) <= 0) {
+        json_decref (a);
+        return e;
+    }
+    while ((e = sdmsg_read (m, fmt, a)) > 0)
+        ;
+    if (e < 0) {
+        json_decref (a);
+        return e;
+    }
+    if ((e = sd_bus_message_exit_container (m)) < 0) {
+        json_decref (a);
+        return e;
+    }
+    *op = a;
+    return 1;
+}
+
+static int sdmsg_get_unknown (sd_bus_message *m, const char *fmt, json_t **op)
+{
+    int e;
+    if ((e = sd_bus_message_skip (m, fmt)) <= 0)
+        return e;
+    *op = json_null ();
+    return e;
+}
+
+static int sdmsg_get_variant (sd_bus_message *m, json_t **op)
+{
+    const char *contents;
+    char type;
+    json_t *val = NULL;
+    json_t *o = NULL;
+    int e;
+
+    if ((e = sd_bus_message_peek_type (m, &type, &contents)) <= 0)
+        return e;
+    if (type != 'v')
+        return -EPROTO;
+    if ((e = sd_bus_message_enter_container (m, 'v', contents)) <= 0)
+        return e;
+    if (strlen (contents) == 1)
+        e = sdmsg_get_basic (m, contents[0], &val);
+    else if (strlen (contents) == 2 && contents[0] == 'a')
+        e = sdmsg_get_array (m, contents + 1, &val);
+    else
+        e = sdmsg_get_unknown (m, contents, &val);
+    if (e <= 0)
+        return e;
+    if (!(o = json_pack ("[sO]", contents, val))) {
+        json_decref (val);
+        return -ENOMEM;
+    }
+    if ((e = sd_bus_message_exit_container (m)) < 0) {
+        json_decref (val);
+        json_decref (o);
+        return e;
+    }
+    json_decref (val);
+    *op = o;
+    return 1;
+}
+
+static int sdmsg_get_property_dict (sd_bus_message *m, json_t **op)
+{
+    json_t *dict;
+    int e;
+
+    if (!(dict = json_object ()))
+        return -ENOMEM;
+    if ((e = sd_bus_message_enter_container (m, 'a', "{sv}")) <= 0)
+        goto out;
+    while ((e = sd_bus_message_enter_container (m, 'e', "sv")) > 0) {
+        const char *key;
+        json_t *val;
+        int e;
+        if ((e = sd_bus_message_read (m, "s", &key)) <= 0
+            || (e = sdmsg_get_variant (m, &val)) <= 0)
+            goto out;
+        if (json_object_set_new (dict, key, val) < 0) {
+            json_decref (val);
+            goto nomem;
+        }
+        if ((e = sd_bus_message_exit_container (m)) < 0)
+            goto out;
+    }
+    if (e < 0 || (e = sd_bus_message_exit_container (m)) < 0)
+        goto out;
+    *op = dict;
+    return 1;
+nomem:
+    e = -ENOMEM;
+out:
+    json_decref (dict);
+    return e;
+}
+
+int sdmsg_get (sd_bus_message *m, const char *fmt, json_t **op)
+{
+    int e;
+
+    if (streq (fmt, "a{sv}"))
+        e = sdmsg_get_property_dict (m, op);
+    else if (fmt[0] == 'a' && strlen (fmt) > 1)
+        e = sdmsg_get_array (m, fmt + 1, op);
+    else if (streq (fmt, "v"))
+        e = sdmsg_get_variant (m, op);
+    else if (strlen (fmt) == 1)
+        e = sdmsg_get_basic (m, fmt[0], op);
+    else
+        e = -EPROTO;
+    return e;
+}
+
+int sdmsg_read (sd_bus_message *m, const char *fmt, json_t *o)
+{
+    for (int i = 0; fmt[i] != '\0';) {
+        json_t *entry = NULL;
+        char *efmt = NULL;
+        int e;
+
+        if (strstarts (&fmt[i], "a{sv}"))
+            efmt = strndup (&fmt[i], 5);
+        else if (fmt[i] == 'a' && strlen (&fmt[i]) > 1)
+            efmt = strndup (&fmt[i], 2);
+        else
+            efmt = strndup (&fmt[i], 1);
+        if (!efmt)
+            return -ENOMEM;
+
+        if ((e = sdmsg_get (m, efmt, &entry)) <= 0) {
+            free (efmt);
+            if (e == 0 && i > 0)
+                e = -EPROTO;
+            return e;
+        }
+        if (json_array_append_new (o, entry) < 0) {
+            free (efmt);
+            json_decref (entry);
+            return -ENOMEM;
+        }
+        i += strlen (efmt);
+        free (efmt);
+    }
+    return 1;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/message.h b/src/modules/sdbus/message.h
new file mode 100644
index 000000000000..9a6a8d436b5d
--- /dev/null
+++ b/src/modules/sdbus/message.h
@@ -0,0 +1,44 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SDBUS_MESSAGE_H
+#define _SDBUS_MESSAGE_H
+
+#include <systemd/sd-bus.h>
+#include <jansson.h>
+
+const char *sdmsg_typestr (sd_bus_message *m);
+
+/* Put one value (or container) specified by 'fmt' from json object 'o' to
+ * current cursor position of message 'm'.  Return 0 on success, or -errno
+ * on failure.
+ */
+int sdmsg_put (sd_bus_message *m, const char *fmt, json_t *o);
+
+/* Put list of values specified by 'fmt' from json array 'o' to current cursor
+ * position of message 'm'.  Return 0 on success or -errno on failure.
+ */
+int sdmsg_write (sd_bus_message *m, const char *fmt, json_t *o);
+
+/* Get one value (or container) specified by 'fmt' from message 'm' at the
+ * current cursor position and return in a new json object assigned to 'op'.
+ * Return 1 on success, or -errno on failure.
+ */
+int sdmsg_get (sd_bus_message *m, const char *fmt, json_t **op);
+
+/* Get list of values specified by 'fmt' from message 'm' at the current cursor
+ * position and append them to the json array 'o'.
+ * Return 1 on success, or -errno on failure.
+ */
+int sdmsg_read (sd_bus_message *m, const char *fmt, json_t *o);
+
+#endif /* !_SDBUS_MESSAGE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/objpath.c b/src/modules/sdbus/objpath.c
new file mode 100644
index 000000000000..19f586cd71f1
--- /dev/null
+++ b/src/modules/sdbus/objpath.c
@@ -0,0 +1,89 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* objpath.c - helpers for dealing with D-Bus object paths
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <errno.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <fnmatch.h>
+#include <stdio.h>
+#include <systemd/sd-bus.h>
+
+#include "src/common/libutil/errno_safe.h"
+
+#include "objpath.h"
+
+static bool path_is_tooshort (const char *s)
+{
+    return fnmatch ("/*", s, FNM_PATHNAME) == 0 ? true : false;
+}
+
+static char *objpath_split (const char *s, const char **suffix)
+{
+    char *cpy;
+    char *cp;
+
+    if (!(cpy = strdup (s)))
+        return NULL;
+    if ((cp = strrchr (cpy, '/')))
+        *cp++ = '\0';
+    if (suffix)
+        *suffix = cp;
+    return cpy;
+}
+
+char *objpath_decode (const char *s)
+{
+    char *prefix;
+    char *tmp = NULL;
+    char *res = NULL;
+    int e;
+
+    if (path_is_tooshort (s))
+        return strdup (s);
+    if (!(prefix = objpath_split (s, NULL)))
+        return NULL;
+    e = sd_bus_path_decode (s, prefix, &tmp);
+    if (e <= 0)
+        goto out;
+    if (asprintf (&res, "%s/%s", prefix, tmp) < 0)
+        goto out;
+out:
+    free (tmp);
+    free (prefix);
+    return res;
+}
+
+char *objpath_encode (const char *s)
+{
+    char *prefix;
+    const char *suffix;
+    char *res = NULL;
+    int e;
+
+    if (path_is_tooshort (s))
+        return strdup (s);
+    if (!(prefix = objpath_split (s, &suffix)))
+        return NULL;
+    if ((e = sd_bus_path_encode (prefix, suffix, &res)) < 0) {
+        errno = -e;
+        goto out;
+    }
+out:
+    free (prefix);
+    return res;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/objpath.h b/src/modules/sdbus/objpath.h
new file mode 100644
index 000000000000..6f65e4801943
--- /dev/null
+++ b/src/modules/sdbus/objpath.h
@@ -0,0 +1,19 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SDBUS_OBJPATH_H
+#define _SDBUS_OBJPATH_H
+
+char *objpath_encode (const char *s);
+char *objpath_decode (const char *s);
+
+#endif /* !_SDBUS_OBJPATH_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/sdbus.c b/src/modules/sdbus/sdbus.c
new file mode 100644
index 000000000000..6fd678c4718e
--- /dev/null
+++ b/src/modules/sdbus/sdbus.c
@@ -0,0 +1,782 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* sdbus.c - sd-bus bridge for user-mode systemd
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <systemd/sd-bus.h>
+#include <fnmatch.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
+
+#include "message.h"
+#include "interface.h"
+#include "watcher.h"
+#include "subscribe.h"
+#include "connect.h"
+#include "objpath.h"
+#include "sdbus.h"
+
+struct sdbus_ctx {
+    flux_future_t *f_conn; // owns ctx->bus
+    sd_bus *bus;
+    flux_watcher_t *bus_w;
+    flux_msg_handler_t **handlers;
+    struct flux_msglist *requests;
+    struct flux_msglist *subscribers;
+    flux_t *h;
+
+    flux_future_t *f_subscribe;
+    uint32_t rank;
+};
+
+struct call_info {
+    uint64_t cookie;
+    char *interface;
+    char *member;
+};
+
+static void sdbus_recover (struct sdbus_ctx *ctx, const char *reason);
+
+/* Connect retry interval (seconds).
+ */
+static const double retry_min = 2;
+static const double retry_max = 60;
+
+static bool sdbus_debug = false;
+
+static __attribute__ ((format (printf, 2, 3)))
+void sdbus_log_debug (flux_t *h, const char *fmt, ...)
+{
+    if (sdbus_debug) {
+        va_list ap;
+
+        va_start (ap, fmt);
+        flux_vlog (h, LOG_DEBUG, fmt, ap);
+        va_end (ap);
+    }
+}
+
+static int authorize_request (const flux_msg_t *msg,
+                              uint32_t rank,
+                              flux_error_t *error)
+{
+    if (rank != 0 || flux_msg_is_local (msg))
+        return 0;
+    errprintf (error, "Remote sdbus requests are not allowed on rank 0");
+    errno = EPERM;
+    return -1;
+}
+
+static void bulk_respond_error (flux_t *h,
+                                struct flux_msglist *msglist,
+                                int errnum,
+                                const char *errmsg)
+{
+    const flux_msg_t *msg;
+
+    while ((msg = flux_msglist_pop (msglist))) {
+        if (flux_respond_error (h, msg, errnum, errmsg) < 0) {
+            const char *topic = "unknown";
+            (void)flux_msg_get_topic (msg, &topic);
+            flux_log_error (h, "error responding to %s request", topic);
+        }
+        flux_msg_decref (msg);
+    }
+}
+
+static bool match_subscription (const flux_msg_t *msg, sd_bus_message *m)
+{
+    const char *path_glob = NULL;
+    const char *member = NULL;
+    const char *interface = NULL;
+
+    (void)flux_request_unpack (msg,
+                               NULL,
+                               "{s?s s?s s?s}",
+                               "path", &path_glob,
+                               "interface", &interface,
+                               "member", &member);
+    if (interface && !streq (interface, sd_bus_message_get_interface (m)))
+        return false;
+    if (member && !streq (member, sd_bus_message_get_member (m)))
+        return false;
+    if (path_glob) {
+        char *m_path = objpath_decode (sd_bus_message_get_path (m));
+        bool match = (m_path && fnmatch (path_glob, m_path, FNM_PATHNAME) == 0);
+        free (m_path);
+        if (!match)
+            return false;
+    }
+    return true;
+}
+
+static bool bulk_respond_match (flux_t *h,
+                                struct flux_msglist *msglist,
+                                sd_bus_message *m)
+{
+    json_t *payload = NULL; // decode deferred until match for performance
+    const flux_msg_t *msg;
+    bool match = false;
+
+    msg = flux_msglist_first (msglist);
+    while (msg) {
+        if (match_subscription (msg, m)) {
+            if (!payload) {
+                if (!(payload = interface_signal_tojson (m, NULL)))
+                    return false;
+            }
+            if (flux_respond_pack (h, msg, "O", payload) < 0)
+                flux_log_error (h, "error responding to subscribe request");
+            else
+                match = true;
+        }
+        msg = flux_msglist_next (msglist);
+    }
+    json_decref (payload);
+    return match;
+}
+
+/* Locate a pending sdbus.call request that matches a cookie from a
+ * bus method-reply or method-error message.  If infop is non-NULL, assign
+ * the message's info struct.
+ */
+static const flux_msg_t *find_request_by_cookie (struct sdbus_ctx *ctx,
+                                                 uint64_t cookie,
+                                                 struct call_info **infop)
+{
+    const flux_msg_t *msg;
+
+    msg = flux_msglist_first (ctx->requests);
+    while (msg) {
+        struct call_info *info;
+        if ((info = flux_msg_aux_get (msg, "info"))
+            && cookie == info->cookie) {
+            if (infop)
+                *infop = info;
+            return msg;
+        }
+        msg = flux_msglist_next (ctx->requests);
+    }
+    return NULL;
+}
+
+/* Log a signal message.
+ * If path refers to a systemd unit, make it pretty for the logs.
+ */
+static void log_msg_signal (flux_t *h,
+                            sd_bus_message *m,
+                            const char *disposition)
+{
+    const char *prefix = "/org/freedesktop/systemd1/unit";
+    const char *path = sd_bus_message_get_path (m);
+    char *s = NULL;
+
+    if (path)
+        (void)sd_bus_path_decode (path, prefix, &s);
+    sdbus_log_debug (h,
+                     "bus %s %s %s %s",
+                     disposition,
+                     sdmsg_typestr (m),
+                     s ? s : path,
+                     sd_bus_message_get_member (m));
+    free (s);
+}
+
+/* Log a method-reply or method-error.
+ */
+static void log_msg_method_reply (flux_t *h,
+                                  sd_bus_message *m,
+                                  struct call_info *info)
+{
+    sdbus_log_debug (h,
+                     "bus recv %s cookie=%ju %s",
+                     sdmsg_typestr (m),
+                     (uintmax_t)info->cookie,
+                     info->member);
+}
+
+static void sdbus_recv (struct sdbus_ctx *ctx, sd_bus_message *m)
+{
+    if (sd_bus_message_is_signal (m, NULL, NULL)) {
+        const char *path = sd_bus_message_get_path (m);
+        const char *iface = sd_bus_message_get_interface (m);
+        const char *member = sd_bus_message_get_member (m);
+
+        /* Apparently sd-bus, when it shuts down nicely, gives us a polite
+         * note informing us that it can no longer abide our company.
+         */
+        if (streq (path, "/org/freedesktop/DBus/Local")
+            && streq (iface, "org.freedesktop.DBus.Local")
+            && streq (member, "Disconnected")) {
+            log_msg_signal (ctx->h, m, "recv");
+            sdbus_recover (ctx, "received Disconnected signal from bus");
+            goto out;
+        }
+        /* Dispatch handled signals to subscribers here.
+         * Log signals with no subscribers as a drop.
+         */
+        if (bulk_respond_match (ctx->h, ctx->subscribers, m))
+            log_msg_signal (ctx->h, m, "recv");
+        else
+            log_msg_signal (ctx->h, m, "drop");
+    }
+    else if (sd_bus_message_is_method_call (m, NULL, NULL)) {
+        /* Log any method calls (for example requesting introspection) as
+         * as "drop".  Flux is purely an sd-bus client and has no methods.
+         */
+        goto log_drop;
+    }
+    else if (sd_bus_message_is_method_error (m, NULL)) {
+        int errnum = sd_bus_message_get_errno (m);
+        const sd_bus_error *error = sd_bus_message_get_error (m);
+        uint64_t cookie;
+        const flux_msg_t *msg;
+        struct call_info *info;
+
+        /* method-error messages that cannot be matched to a pending
+         * sdbus.call request are logged as a "drop".  Perhaps the client
+         * disconnected without waiting for a reply.
+         */
+        if (sd_bus_message_get_reply_cookie (m, &cookie) < 0
+            || !(msg = find_request_by_cookie (ctx, cookie, &info)))
+            goto log_drop;
+        /* method-errors that can be matched are logged and dispatched here.
+         */
+        log_msg_method_reply (ctx->h, m, info);
+        if (errnum == 0)
+            errnum = EINVAL;
+        if (flux_respond_error (ctx->h, msg, errnum, error->message) < 0)
+            flux_log_error (ctx->h, "error responding to sdbus.call");
+        flux_msglist_delete (ctx->requests); // cursor is on completed message
+    }
+    else { // method-reply
+        uint64_t cookie;
+        const flux_msg_t *msg;
+        struct call_info *info;
+        json_t *rep = NULL;
+        flux_error_t error;
+        int rc;
+
+        /* method-reply messages that cannot be matched to a pending
+         * sdbus.call request are logged as a "drop".  Perhaps the client
+         * disconnected without waiting for a reply.
+         */
+        if (sd_bus_message_get_reply_cookie (m, &cookie) < 0
+            || !(msg = find_request_by_cookie (ctx, cookie, &info)))
+            goto log_drop;
+        /* method-replies that can be matched are logged, translated to json,
+         * and dispatched here.  If there's a translation failure, we try to
+         * give the requestor a human readable error. This is helpful when
+         * developing support for new methods, if nothing else.
+         */
+        log_msg_method_reply (ctx->h, m, info);
+        if ((rep = interface_reply_tojson (m,
+                                           info->interface,
+                                           info->member,
+                                           &error)))
+            rc = flux_respond_pack (ctx->h, msg, "O", rep);
+        else
+            rc = flux_respond_error (ctx->h, msg, EINVAL, error.text);
+        if (rc < 0)
+            flux_log_error (ctx->h, "error responding to sdbus.call");
+        json_decref (rep);
+        flux_msglist_delete (ctx->requests); // cursor is on completed message
+    }
+out:
+    return;
+log_drop:
+    sdbus_log_debug (ctx->h, "bus drop %s", sdmsg_typestr (m));
+}
+
+static void call_info_destroy (struct call_info *info)
+{
+    if (info) {
+        int saved_errno = errno;
+        free (info->interface);
+        free (info->member);
+        free (info);
+        errno = saved_errno;
+    }
+}
+
+/* Extract some info from method-call message that will be required when
+ * processing method-reply and method-error messages.  The call_info struct
+ * will be placed in the aux container of the pending sdbus.call request.
+ */
+static struct call_info *call_info_create (sd_bus_message *m, uint64_t cookie)
+{
+    struct call_info *info;
+    const char *interface = sd_bus_message_get_interface (m);
+    const char *member = sd_bus_message_get_member (m);
+
+    if (!(info = calloc (1, sizeof (*info))))
+        return NULL;
+    info->cookie = cookie;
+    if ((interface && !(info->interface = strdup (interface)))
+        || (member && !(info->member = strdup (member))))
+        goto error;
+    return info;
+error:
+    call_info_destroy (info);
+    return NULL;
+}
+
+/* Translate a sdbus.call request to an sd-bus method-call message and send
+ * it.  This function is invoked directly by the sdbus.call request handler
+ * when the bus is active.  When the bus is inactive, it is called by
+ * handle_call_request_backlog() after the bus is reconnected.
+ */
+static int handle_call_request (struct sdbus_ctx *ctx,
+                                const flux_msg_t *msg,
+                                flux_error_t *error)
+{
+    sd_bus_message *m;
+    json_t *req;
+    int e;
+    uint64_t cookie;
+    struct call_info *info;
+
+    if (flux_request_unpack (msg, NULL, "o", &req) < 0) {
+        errprintf (error, "unable to decode call request");
+        return -1;
+    }
+    if (!(m = interface_request_fromjson (ctx->bus, req, error))) {
+        errno = EINVAL;
+        goto error;
+    }
+    if ((e = sd_bus_send (NULL, m, &cookie)) < 0) {
+        errno = -e;
+        errprintf (error, "error sending sdbus request: %s", strerror (errno));
+        goto error;
+    }
+
+    sdbus_log_debug (ctx->h,
+                     "bus send %s cookie=%ju %s",
+                     sdmsg_typestr (m),
+                     (uintmax_t)cookie,
+                     sd_bus_message_get_member (m));
+
+    if (!(info = call_info_create (m, cookie))
+        || flux_msg_aux_set (msg,
+                             "info",
+                             info,
+                             (flux_free_f)call_info_destroy) < 0) {
+        call_info_destroy (info);
+        errprintf (error, "error saving call request state");
+        goto error;
+    }
+    sd_bus_message_unref (m);
+    return 0;
+error:
+    ERRNO_SAFE_WRAP (sd_bus_message_unref, m);
+    return -1;
+}
+
+/* Handle an sdbus.call request.
+ */
+static void call_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    struct sdbus_ctx *ctx = arg;
+    flux_error_t error;
+    const char *errmsg = NULL;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (authorize_request (msg, ctx->rank, &error) < 0) {
+        errmsg = error.text;
+        goto error;
+    }
+    if (ctx->bus) { // defer request if bus is not yet connected
+        if (handle_call_request (ctx, msg, &error) < 0) {
+            errmsg = error.text;
+            goto error;
+        }
+    }
+    if (flux_msglist_append (ctx->requests, msg) < 0)
+        goto error;
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to call request");
+}
+
+/* Handle an sdbus.subscribe request.
+ */
+static void subscribe_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
+{
+    struct sdbus_ctx *ctx = arg;
+    flux_error_t error;
+    const char *errmsg = NULL;
+    const char *s1, *s2, *s3; // not used
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s?s s?s s?s}",
+                             "path", &s1,
+                             "interface", &s2,
+                             "member", &s3) < 0)
+        goto error;
+    if (authorize_request (msg, ctx->rank, &error) < 0) {
+        errmsg = error.text;
+        goto error;
+    }
+    if (!flux_msg_is_streaming (msg)) {
+        errno = EPROTO;
+        goto error;
+    }
+    if (flux_msglist_append (ctx->subscribers, msg) < 0)
+        goto error;
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to sdbus.subscribe request");
+}
+
+/* Handle cancellation of an sdbus.subscribe request as described in RFC 6.
+ */
+static void subscribe_cancel_cb (flux_t *h,
+                                 flux_msg_handler_t *mh,
+                                 const flux_msg_t *msg,
+                                 void *arg)
+{
+    struct sdbus_ctx *ctx = arg;
+
+    if (authorize_request (msg, ctx->rank, NULL) == 0)
+        flux_msglist_cancel (h, ctx->subscribers, msg);
+}
+
+/* Handle disconnection of a client as described in RFC 6.
+ */
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct sdbus_ctx *ctx = arg;
+
+    if (authorize_request (msg, ctx->rank, NULL) == 0) {
+        (void)flux_msglist_disconnect (ctx->requests, msg);
+        (void)flux_msglist_disconnect (ctx->subscribers, msg);
+    }
+}
+
+/* Handle a request to force bus disconnection and recovery for testing.
+ */
+static void reconnect_cb (flux_t *h,
+                          flux_msg_handler_t *mh,
+                          const flux_msg_t *msg,
+                          void *arg)
+{
+    struct sdbus_ctx *ctx = arg;
+    flux_error_t error;
+    const char *errmsg = NULL;
+
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (authorize_request (msg, ctx->rank, &error) < 0) {
+        errmsg = error.text;
+        goto error;
+    }
+    if (!ctx->bus) {
+        errmsg = "bus is not connected";
+        errno = EINVAL;
+        goto error;
+    }
+    sdbus_recover (ctx, "user requested bus reconnect");
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to sdbus.reconnect request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errmsg) < 0)
+        flux_log_error (h, "error responding to sdbus.reconnect request");
+}
+
+static int sdbus_configure (struct sdbus_ctx *ctx,
+                            const flux_conf_t *conf,
+                            flux_error_t *error)
+{
+    flux_error_t conf_error;
+    int debug = 0;
+
+    if (flux_conf_unpack (conf,
+                          &conf_error,
+                          "{s?{s?b}}",
+                          "systemd",
+                            "sdbus-debug", &debug) < 0) {
+        errprintf (error,
+                   "error reading [systemd] config table: %s",
+                   conf_error.text);
+        return -1;
+    }
+    sdbus_debug = (debug ? true : false);
+    return 0;
+}
+
+static void reload_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    struct sdbus_ctx *ctx = arg;
+    const flux_conf_t *conf;
+    flux_error_t error;
+    const char *errstr = NULL;
+
+    if (flux_conf_reload_decode (msg, &conf) < 0) {
+        errstr = "Failed to parse config-reload request";
+        goto error;
+    }
+    if (sdbus_configure (ctx, conf, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+}
+
+static struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,
+      "sdbus.disconnect",
+      disconnect_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "sdbus.call",
+      call_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "sdbus.subscribe",
+      subscribe_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "sdbus.subscribe-cancel",
+      subscribe_cancel_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "sdbus.reconnect",
+      reconnect_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "sdbus.config-reload",
+      reload_cb,
+      0
+    },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+/* The bus watcher callback runs sd_bus_process().  Apparently this is an
+ * edge triggered notification so we need to handle all events now, which
+ * means calling sd_bus_process() in a loop until it returns 0.
+ */
+static void sdbus_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct sdbus_ctx *ctx = arg;
+    int e;
+
+    do {
+        sd_bus_message *m = NULL;
+        if ((e = sd_bus_process (ctx->bus, &m)) < 0) {
+            sdbus_recover (ctx, "error processing sd-bus events");
+            return;
+        }
+        if (m) {
+            // sdbus_recv() may call sdbus_recover() which sets ctx->bus = NULL
+            sdbus_recv (ctx, m);
+            sd_bus_message_unref (m);
+        }
+    } while (e > 0 && ctx->bus != NULL);
+}
+
+/* sdbus.call requests that arrive while the bus connect is in progress
+ * are added to ctx->requests without further processing.  Revisit them now
+ * and begin processing.  Since recovery fails any pending requests, all
+ * requests in ctx->requests are eligible.
+ */
+static void handle_call_request_backlog (struct sdbus_ctx *ctx)
+{
+    const flux_msg_t *msg;
+    flux_error_t error;
+    msg = flux_msglist_first (ctx->requests);
+    while (msg)  {
+        if (handle_call_request (ctx, msg, &error) < 0) {
+            if (flux_respond_error (ctx->h, msg, errno, error.text) < 0)
+                flux_log_error (ctx->h, "error responding to call request");
+        }
+        msg = flux_msglist_next (ctx->requests);
+    }
+}
+
+/* Bus subscribe completed.  Henceforth, sd-bus signals will be forwarded
+ * to subscribers.  Service pending sdbus.call requests.
+ * N.B. handle_call_request_backlog is called here rather than when the connect
+ * is finalized so that a user may asynchronously subscribe to signals,
+ * then initiate an action and expect the subscription to capture all signals
+ * triggered by the action.
+ */
+static void bus_subscribe_continuation (flux_future_t *f, void *arg)
+{
+    struct sdbus_ctx *ctx = arg;
+    flux_error_t error;
+
+    if (flux_rpc_get (f, NULL) < 0) {
+        errprintf (&error, "subscribe error: %s", future_strerror (f, errno));
+        goto error;
+    }
+    handle_call_request_backlog (ctx);
+    return;
+error:
+    sdbus_recover (ctx, error.text);
+}
+
+/* Connect completed. Initiate asynchronous bus subscribe.
+ */
+static void connect_continuation (flux_future_t *f, void *arg)
+{
+    struct sdbus_ctx *ctx = arg;
+    flux_error_t error;
+
+    if (flux_future_get (f, (const void **)&ctx->bus) < 0) {
+        errprintf (&error, "sdbus_connect: %s", future_strerror (f, errno));
+        goto error;
+    }
+    if (!(ctx->bus_w = sdbus_watcher_create (flux_get_reactor (ctx->h),
+                                             ctx->bus,
+                                             sdbus_cb,
+                                             ctx))) {
+        errprintf (&error, "error creating bus watcher: %s", strerror (errno));
+        goto error;
+    }
+    flux_watcher_start (ctx->bus_w);
+
+    if (!(ctx->f_subscribe = sdbus_subscribe (ctx->h))
+        || flux_future_then (ctx->f_subscribe,
+                             -1,
+                             bus_subscribe_continuation,
+                             ctx) < 0) {
+        errprintf (&error, "subscribe error: %s", strerror (errno));
+        goto error;
+    }
+    return;
+error:
+    sdbus_recover (ctx, error.text);
+}
+
+static void sdbus_recover (struct sdbus_ctx *ctx, const char *reason)
+{
+
+    flux_log (ctx->h, LOG_INFO, "disconnect: %s", reason);
+
+    /* Send any pending requests an error.
+     */
+    bulk_respond_error (ctx->h, ctx->subscribers, EAGAIN, reason);
+    bulk_respond_error (ctx->h, ctx->requests, EAGAIN, reason);
+
+    /* Destroy subscribe future.
+     */
+    flux_future_destroy (ctx->f_subscribe);
+    ctx->f_subscribe = NULL;
+
+    /* Destroy the (now defunct) bus connection and its watcher.
+     */
+    flux_watcher_destroy (ctx->bus_w);
+    ctx->bus_w = NULL;
+    flux_future_destroy (ctx->f_conn);
+    ctx->f_conn = NULL;
+    ctx->bus = NULL;
+
+    /* Begin asynchronous reconnect.
+     * Any requests that arrive while this is in progress are deferred.
+     * N.B. setting first_time=false ensures a retry_min second delay before
+     * the connect attempt.  Some small delay seems to be necessary to avoid
+     * libsystemd complaining about unexpected internal states(?) and the
+     * occasional segfault.
+     */
+    if (!(ctx->f_conn = sdbus_connect (ctx->h, false, retry_min, retry_max))
+        || flux_future_then (ctx->f_conn, -1, connect_continuation, ctx) < 0) {
+        flux_log_error (ctx->h, "error starting bus connect");
+        flux_reactor_stop_error (flux_get_reactor (ctx->h));
+    }
+}
+
+void sdbus_ctx_destroy (struct sdbus_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        const char *errmsg = "module is unloading";
+        if (ctx->subscribers) {
+            bulk_respond_error (ctx->h, ctx->subscribers, ENOSYS, errmsg);
+            flux_msglist_destroy (ctx->subscribers);
+        }
+        if (ctx->requests) {
+            bulk_respond_error (ctx->h, ctx->requests, ENOSYS, errmsg);
+            flux_msglist_destroy (ctx->requests);
+        }
+        flux_msg_handler_delvec (ctx->handlers);
+        flux_watcher_destroy (ctx->bus_w);
+        flux_future_destroy (ctx->f_subscribe);
+        if (ctx->bus) {
+            sd_bus_flush (ctx->bus);
+            sd_bus_close (ctx->bus);
+        }
+        flux_future_destroy (ctx->f_conn); // destroys ctx->bus
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+struct sdbus_ctx *sdbus_ctx_create (flux_t *h, flux_error_t *error)
+{
+    struct sdbus_ctx *ctx;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        goto error_create;
+    if (sdbus_configure (ctx, flux_get_conf (h), error) < 0)
+        goto error;
+    if (!(ctx->f_conn = sdbus_connect (h, true, retry_min, retry_max))
+        || flux_future_then (ctx->f_conn, -1, connect_continuation, ctx) < 0
+        || flux_msg_handler_addvec (h, htab, ctx, &ctx->handlers) < 0
+        || !(ctx->requests = flux_msglist_create ())
+        || !(ctx->subscribers = flux_msglist_create ())
+        || flux_get_rank (h, &ctx->rank) < 0)
+        goto error_create;
+    ctx->h = h;
+    return ctx;
+error_create:
+    errprintf (error, "error creating sdbus context: %s", strerror (errno));
+error:
+    sdbus_ctx_destroy (ctx);
+    return NULL;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/sdbus.h b/src/modules/sdbus/sdbus.h
new file mode 100644
index 000000000000..3252b4908006
--- /dev/null
+++ b/src/modules/sdbus/sdbus.h
@@ -0,0 +1,21 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SDBUS_SDBUS_H
+#define _SDBUS_SDBUS_H
+
+#include <flux/core.h>
+
+struct sdbus_ctx *sdbus_ctx_create (flux_t *h, flux_error_t *error);
+void sdbus_ctx_destroy (struct sdbus_ctx *ctx);
+
+#endif /* !_SDBUS_SDBUS_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/subscribe.c b/src/modules/sdbus/subscribe.c
new file mode 100644
index 000000000000..f8e8698b4c5a
--- /dev/null
+++ b/src/modules/sdbus/subscribe.c
@@ -0,0 +1,76 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* subscribe.c - composite RPC for Subscribe and AddMatch
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <flux/core.h>
+
+#include "subscribe.h"
+
+static const char *match_signal_all = "type=signal";
+
+
+static void subscribe_continuation (flux_future_t *f1, void *arg)
+{
+    flux_t *h = flux_future_get_flux (f1);
+    const char *errmsg = NULL;
+    flux_future_t *f2;
+
+    if (flux_rpc_get (f1, NULL) < 0) {
+        errmsg = future_strerror (f1, errno);
+        goto error;
+    }
+    if (!(f2 = flux_rpc_pack (h,
+                              "sdbus.call",
+                              FLUX_NODEID_ANY,
+                              0,
+                              "{s:s s:s s:s s:s s:[s]}",
+                              "destination", "org.freedesktop.DBus",
+                              "path", "/org/freedesktop/DBus",
+                              "interface", "org.freedesktop.DBus",
+                              "member", "AddMatch",
+                              "params", match_signal_all))
+        || flux_future_continue (f1, f2) < 0) {
+        errmsg = "error continuing subscribe request";
+        flux_future_destroy (f2);
+        goto error;
+    }
+    goto done;
+error:
+    flux_future_continue_error (f1, errno, errmsg);
+done:
+    flux_future_destroy (f1);
+}
+
+flux_future_t *sdbus_subscribe (flux_t *h)
+{
+    flux_future_t *f1;
+    flux_future_t *fc;
+
+    if (!(f1 = flux_rpc_pack (h,
+                              "sdbus.call",
+                              FLUX_NODEID_ANY,
+                              0,
+                              "{s:s s:[]}",
+                              "member", "Subscribe",
+                              "params"))
+        || !(fc = flux_future_and_then (f1, subscribe_continuation, NULL))) {
+        flux_future_destroy (f1);
+        return NULL;
+    }
+    return fc;
+}
+
+// vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/modules/sdbus/subscribe.h b/src/modules/sdbus/subscribe.h
new file mode 100644
index 000000000000..b4e603bab3f4
--- /dev/null
+++ b/src/modules/sdbus/subscribe.h
@@ -0,0 +1,27 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SDBUS_SUBSCRIBE_H
+#define _SDBUS_SUBSCRIBE_H
+
+#include <flux/core.h>
+
+/* sdbus RPC for Subscribe and AddMatch method-calls.
+ * The calls are made sequentially and the future is fulfilled when
+ * both complete.
+ * N.B. these are not direct method-calls.  They are RPCs to sdbus.call,
+ * so when made from sdbus itself, they rely on the fact that RPCs to self
+ * do work in broker modules.
+ */
+flux_future_t *sdbus_subscribe (flux_t *h);
+
+#endif /* !_SDBUS_SUBSCRIBE_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/test/message.c b/src/modules/sdbus/test/message.c
new file mode 100644
index 000000000000..63fda871f88e
--- /dev/null
+++ b/src/modules/sdbus/test/message.c
@@ -0,0 +1,492 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <jansson.h>
+#include <systemd/sd-bus.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+#include "message.h"
+
+/* Message diag is not proper TAP output so set to 0 except during development.
+ */
+#define ENABLE_MESSAGE_DIAG 0
+
+void diagjson (json_t *o)
+{
+    char *s = json_dumps (o, JSON_COMPACT);
+    diag ("%s", s ? s : "(null)");
+    free (s);
+}
+
+void diagmsg (sd_bus_message *m)
+{
+#if defined (HAVE_SD_BUS_MESSAGE_DUMP) && ENABLE_MESSAGE_DIAG
+    (void)sd_bus_message_rewind (m, true);
+    (void)sd_bus_message_dump (m, stderr, 0);
+    (void)sd_bus_message_rewind (m, true);
+#endif
+}
+
+void msgtype_is (sd_bus_message *m, const char *fmt)
+{
+    char type[65] = "";
+
+    if (m) {
+        (void)sd_bus_message_rewind (m, true);
+        for (int i = 0; i < sizeof (type) - 1; i++) {
+            if (sd_bus_message_peek_type (m, &type[i], NULL) < 1
+                || sd_bus_message_skip (m, &type[i]) < 0)
+                break;
+        }
+        (void)sd_bus_message_rewind (m, true);
+    }
+    bool match = streq (type, fmt);
+    ok (match, "message type has %s signature", fmt);
+    if (!match)
+        diag ("message type %s != %s signature", type, fmt);
+}
+
+void test_typestr (sd_bus *bus)
+{
+    const char *s;
+    sd_bus_message *m;
+
+    s = sdmsg_typestr (NULL);
+    ok (s && streq (s, "unknown"),
+        "sdmsg_typestr m=NULL returns 'unknown'");
+
+    if (sd_bus_message_new (bus, &m, SD_BUS_MESSAGE_METHOD_CALL) < 0)
+        BAIL_OUT ("could not create method call message");
+    s = sdmsg_typestr (m);
+    ok (s && streq (s, "method-call"),
+        "sdmsg_typestr m=method call returns 'method-call'");
+    sd_bus_message_unref (m);
+
+    if (sd_bus_message_new (bus, &m, SD_BUS_MESSAGE_METHOD_RETURN) < 0)
+        BAIL_OUT ("could not create method return message");
+    s = sdmsg_typestr (m);
+    ok (s && streq (s, "method-return"),
+        "sdmsg_typestr m=method return returns 'method-return'");
+    sd_bus_message_unref (m);
+
+    if (sd_bus_message_new (bus, &m, SD_BUS_MESSAGE_METHOD_ERROR) < 0)
+        BAIL_OUT ("could not create method errr message");
+    s = sdmsg_typestr (m);
+    ok (s && streq (s, "method-error"),
+        "sdmsg_typestr m=method return returns 'method-error'");
+    sd_bus_message_unref (m);
+
+    if (sd_bus_message_new (bus, &m, SD_BUS_MESSAGE_SIGNAL) < 0)
+        BAIL_OUT ("could not create signal message");
+    s = sdmsg_typestr (m);
+    ok (s && streq (s, "signal"),
+        "sdmsg_typestr m=signal returns 'signal'");
+    sd_bus_message_unref (m);
+}
+
+/* Check that an object containing all of the basic D-Bus types can be
+ * converted from json->dbus->json.  The input and output json objects
+ * are compared for equality.
+ */
+void test_basic (sd_bus *bus)
+{
+    json_t *o;
+    json_t *o2;
+    int rc;
+    sd_bus_message *m;
+
+    if (!(o = json_pack ("[ibiiiiiifsss]",
+                         42,
+                         true,
+                         -30000,
+                         48000,
+                         -100000,
+                         100000,
+                         -10,
+                         10,
+                         3.5,
+                         "string",
+                         "",
+                         "/object/path/string.suffix")))
+        BAIL_OUT ("could not pack json array");
+    diagjson (o);
+    if (sd_bus_message_new (bus, &m, SD_BUS_MESSAGE_METHOD_CALL) < 0)
+        BAIL_OUT ("could not create method call message");
+
+    const char *fmt = "ybnqiuxtdsso";
+    rc = sdmsg_write (m, fmt, o);
+    ok (rc == 0,
+        "sdmsg_write works");
+
+    if (sd_bus_message_seal (m, 42, 0) < 0
+        || sd_bus_message_rewind (m, true) < 0)
+        BAIL_OUT ("could not finalize message");
+
+    msgtype_is (m, fmt);
+
+    diagmsg (m);
+
+    if (!(o2 = json_array ()))
+        BAIL_OUT ("could not create json array");
+    rc = sdmsg_read (m, fmt, o2);
+    diag ("sdmsg_read returned %d", rc);
+    ok (rc == 1,
+        "sdmsg_read works");
+    diagjson (o2);
+    ok (sd_bus_message_at_end (m, true),
+        "all message contents were read");
+    ok (json_equal (o, o2),
+        "json in/out are the same");
+
+    json_decref (o2);
+    sd_bus_message_unref (m);
+    json_decref (o);
+}
+
+/* Check that a struct containing string, array-of-string, and boolean "(sasb)"
+ * can be converted from json->dbus.  dbus->json is not supported yet so
+ * use sd_bus_message accessors to check that dbus content is correct.
+ * N.B. An array of (sasb) is required in the StartTransientUnit request.
+ */
+void test_struct_sasb (sd_bus *bus)
+{
+    json_t *o;
+    const char *fmt = "(sasb)";
+    sd_bus_message *m;
+    const char *s;
+    int b;
+
+    if (!(o = json_pack ("[s[ss]b]", "foo", "a1", "a2", 1)))
+        BAIL_OUT ("could not pack json array for (sasb)");
+    diagjson (o);
+
+    if (sd_bus_message_new (bus, &m, SD_BUS_MESSAGE_METHOD_CALL) < 0)
+        BAIL_OUT ("could not create method call message");
+
+    ok (sdmsg_put (m, fmt, o) == 0,
+        "sdmsg_put works with struct (sasb)");
+
+    if (sd_bus_message_seal (m, 42, 0) < 0
+        || sd_bus_message_rewind (m, true) < 0)
+        BAIL_OUT ("could not finalize message");
+
+    diagmsg (m);
+
+    if (sd_bus_message_enter_container (m, 'r', "sasb") <= 0)
+        BAIL_OUT ("could not enter struct container");
+
+    ok (sd_bus_message_read (m, "s", &s) > 0
+        && streq (s, "foo"),
+        "successfully read back first (string) element");
+
+    ok (sd_bus_message_enter_container (m, 'a', "s") > 0
+        && sd_bus_message_read (m, "s", &s) > 0
+        && streq (s, "a1")
+        && sd_bus_message_read (m, "s", &s) > 0
+        && streq (s, "a2")
+        && sd_bus_message_exit_container (m) > 0,
+        "successfully read back second (array) element");
+
+    ok (sd_bus_message_read (m, "b", &b) > 0
+        && b == 1,
+        "successfully read back third (boolean) element");
+
+    if (sd_bus_message_exit_container (m) <= 0)
+        BAIL_OUT ("error exiting struct container");
+
+    sd_bus_message_unref (m);
+    json_decref (o);
+}
+
+/* Convert three variants (integer, string, float) from json->dbus->json.
+ * The input and output json objects are compared for equality.
+ */
+void test_variant (sd_bus *bus)
+{
+    json_t *o;
+    json_t *o2;
+    sd_bus_message *m;
+    int rc;
+
+    if (!(o = json_pack ("[[si][ss][sf]]",
+                         "i", 42,
+                         "s", "fubar",
+                         "d", -1.5)))
+        BAIL_OUT ("could not pack json array");
+    diagjson (o);
+
+    if (sd_bus_message_new (bus, &m, SD_BUS_MESSAGE_METHOD_CALL) < 0)
+        BAIL_OUT ("could not create method call message");
+
+    const char *fmt = "vvv";
+    rc = sdmsg_write (m, fmt, o);
+    ok (rc == 0,
+        "sdmsg_write works with variants");
+
+    if (sd_bus_message_seal (m, 42, 0) < 0
+        || sd_bus_message_rewind (m, true) < 0)
+        BAIL_OUT ("could not finalize message");
+
+    msgtype_is (m, "vvv");
+
+    diagmsg (m);
+
+    if (!(o2 = json_array ()))
+        BAIL_OUT ("could not create json array");
+    rc = sdmsg_read (m, fmt, o2);
+    diag ("sdmsg_read returned %d", rc);
+    ok (rc == 1,
+        "sdmsg_read works");
+    diagjson (o2);
+    ok (sd_bus_message_at_end (m, true),
+        "all message contents were read");
+    ok (json_equal (o, o2),
+        "json in/out are the same");
+
+    json_decref (o2);
+    sd_bus_message_unref (m);
+    json_decref (o);
+}
+
+/* Convert an array-of-string from json->dbus->json.
+ * The input and output json objects are compared for equality.
+ */
+void test_variant_as (sd_bus *bus)
+{
+    sd_bus_message *m;
+    json_t *in;
+    json_t *out;
+    const char *fmt = "v";
+    int rc;
+
+    if (!(in = json_pack ("[[s[sss]]]", "as", "foo", "bar", "baz")))
+        BAIL_OUT ("could not create json object");
+    if (sd_bus_message_new (bus, &m, SD_BUS_MESSAGE_METHOD_RETURN) < 0)
+        BAIL_OUT ("could not create message");
+    ok (sdmsg_write (m, fmt, in) == 0,
+        "sdmsg_write of variant string array works");
+
+    if (sd_bus_message_seal (m, 42, 0) < 0
+        || sd_bus_message_rewind (m, true) < 0)
+        BAIL_OUT ("could not finalize message");
+    diagmsg (m);
+    msgtype_is (m, fmt);
+
+    if (!(out = json_array ()))
+        BAIL_OUT ("could not create json array");
+
+    rc = sdmsg_read (m, fmt, out);
+    diag ("sdmsg_read returned %d", rc);
+    ok (rc == 1,
+        "sdmsg_read works on message containing string array variant");
+
+    diagjson (out);
+
+    ok (json_equal (in, out),
+        "json in/out are the same");
+
+    json_decref (out);
+    json_decref (in);
+    sd_bus_message_unref (m);
+}
+
+/* In property dicts (e.g. GetAll) we don't know how to decode all values yet.
+ * It seems most sane to decode keys with a JSON null value rather than omit
+ * those keys.  Create an sdbus message containing complex variants, then
+ * convert dbus->json.  Verify that values that can't be decoded are null.
+ */
+void test_variant_unknown (sd_bus *bus)
+{
+    sd_bus_message *m;
+    json_t *o;
+    const char *fmt = "svs";
+    uint8_t y[2] = { 99, 100 };
+    int rc;
+
+    if (sd_bus_message_new (bus, &m, SD_BUS_MESSAGE_METHOD_CALL) < 0
+        || sd_bus_message_append (m, "s", "eek") < 0
+        || sd_bus_message_open_container (m, 'v', "a(yy)") < 0
+        || sd_bus_message_open_container (m, 'a', "(yy)") < 0
+        || sd_bus_message_open_container (m, 'r', "yy") < 0
+        || sd_bus_message_append (m, "yy", y[0], y[1]) < 0
+        || sd_bus_message_close_container (m) < 0
+        || sd_bus_message_close_container (m) < 0
+        || sd_bus_message_close_container (m) < 0
+        || sd_bus_message_append (m, "s", "ook") < 0
+        || sd_bus_message_seal (m, 42, 0) < 0
+        || sd_bus_message_rewind (m, true) < 0)
+        BAIL_OUT ("could not create message containing complex variant");
+    diagmsg (m);
+    msgtype_is (m, fmt);
+    if (!(o = json_array ()))
+        BAIL_OUT ("could not create json array");
+
+    rc = sdmsg_read (m, fmt, o);
+    diag ("sdmsg_read returned %d", rc);
+    ok (rc == 1,
+        "sdmsg_read works on message containing complex variant");
+
+    diagjson (o);
+
+    const char *s1, *s2, *type;
+    ok (json_unpack (o, "[s[sn]s]", &s1, &type, &s2) == 0
+        && streq (s1, "eek")
+        && streq (type, "a(yy)")
+        && streq (s2, "ook"),
+        "complex variant was translated to json null");
+
+    json_decref (o);
+    sd_bus_message_unref (m);
+}
+
+/* StartTransientUnit wants a property array rather than the D-bus std dict.
+ * Create one and convert json->dbus.  Then since we don't require the reverse
+ * encoding, use sd_bus_message accessors to verify the result.
+ */
+void test_property_array (sd_bus *bus)
+{
+    json_t *o;
+    json_error_t error;
+    sd_bus_message *m;
+    const char *fmt = "a(sv)";
+    const char *key;
+    const char *s;
+    const char *s2;
+    int b;
+
+    if (!(o = json_pack_ex (&error,
+                            0,
+                            "["
+                            "[s[ss]]"
+                            "[s[sb]]"
+                            "[s[s[ss]]]"
+                            "[s[s[[s[ss]b]]]]"
+                            "]",
+                            "key1", "s", "val1",
+                            "key2", "b", 1,
+                            "key3", "as", "a1", "a2",
+                            "key4", "a(sasb)", "foo", "a1", "a2", 0)))
+        BAIL_OUT ("error creating properties object: %s", error.text);
+    diagjson (o);
+
+    if (sd_bus_message_new (bus, &m, SD_BUS_MESSAGE_METHOD_CALL) < 0)
+        BAIL_OUT ("could not create message");
+    ok (sdmsg_put (m, fmt, o) == 0,
+        "sdmsg_put of property array works");
+    if (sd_bus_message_seal (m, 42, 0) < 0
+        || sd_bus_message_rewind (m, true) < 0)
+        BAIL_OUT ("could not finalize message");
+    diagmsg (m);
+
+    if (sd_bus_message_enter_container (m, 'a', "(sv)") <= 0)
+        BAIL_OUT ("could not enter property array container");
+
+    ok (sd_bus_message_enter_container (m, 'r', "sv") > 0
+        && sd_bus_message_read (m, "s", &key) > 0
+        && sd_bus_message_enter_container (m, 'v', "s") > 0
+        && sd_bus_message_read (m, "s", &s) > 0
+        && sd_bus_message_exit_container (m) > 0
+        && sd_bus_message_exit_container (m) > 0
+        && streq (key, "key1")
+        && streq (s, "val1"),
+        "successfully read back first property");
+
+    ok (sd_bus_message_enter_container (m, 'r', "sv") > 0
+        && sd_bus_message_read (m, "s", &key) > 0
+        && sd_bus_message_enter_container (m, 'v', "b") > 0
+        && sd_bus_message_read (m, "b", &b) > 0
+        && sd_bus_message_exit_container (m) > 0
+        && sd_bus_message_exit_container (m) > 0
+        && streq (key, "key2")
+        && b == 1,
+        "successfully read back second property");
+
+    ok (sd_bus_message_enter_container (m, 'r', "sv") > 0
+        && sd_bus_message_read (m, "s", &key) > 0
+        && sd_bus_message_enter_container (m, 'v', "as") > 0
+        && sd_bus_message_enter_container (m, 'a', "s") > 0
+        && sd_bus_message_read (m, "s", &s) > 0
+        && sd_bus_message_read (m, "s", &s2) > 0
+        && sd_bus_message_exit_container (m) > 0
+        && sd_bus_message_exit_container (m) > 0
+        && sd_bus_message_exit_container (m) > 0
+        && streq (key, "key3")
+        && streq (s, "a1")
+        && streq (s2, "a2"),
+        "successfully read back third property");
+
+    ok (sd_bus_message_enter_container (m, 'r', "sv") > 0
+        && sd_bus_message_read (m, "s", &key) > 0
+        && streq (key, "key4")
+        && sd_bus_message_enter_container (m, 'v', "a(sasb)") > 0
+        && sd_bus_message_enter_container (m, 'a', "(sasb)") > 0
+        && sd_bus_message_enter_container (m, 'r', "sasb") > 0
+        && sd_bus_message_read (m, "s", &s) > 0
+        && streq (s, "foo")
+        && sd_bus_message_enter_container (m, 'a', "s") > 0
+        && sd_bus_message_read (m, "s", &s) > 0
+        && streq (s, "a1")
+        && sd_bus_message_read (m, "s", &s) > 0
+        && streq (s, "a2")
+        && sd_bus_message_exit_container (m) > 0
+        && sd_bus_message_read (m, "b", &b) > 0
+        && b == 0
+        && sd_bus_message_exit_container (m) > 0
+        && sd_bus_message_exit_container (m) > 0
+        && sd_bus_message_exit_container (m) > 0,
+        "successfully read back fourth property");
+
+    if (sd_bus_message_exit_container (m) <= 0)
+        BAIL_OUT ("error exiting property array container");
+
+    json_decref (o);
+    sd_bus_message_unref (m);
+}
+
+int main (int argc, char **argv)
+{
+    sd_bus *bus;
+    int e;
+
+    plan (NO_PLAN);
+
+    if ((e = sd_bus_open_user (&bus)) < 0) {
+        diag ("could not open sdbus: %s", strerror (e));
+        if (!getenv ("DBUS_SESSION_BUS_ADDRESS"))
+            diag ("Hint: DBUS_SESSION_BUS_ADDRESS is not set");
+        if (!getenv ("XDG_RUNTIME_DIR"))
+            diag ("Hint: XDG_RUNTIME_DIR is not set");
+        plan (SKIP_ALL);
+        done_testing ();
+    }
+
+    test_typestr (bus);
+    test_basic (bus);
+    test_struct_sasb (bus);
+    test_variant (bus);
+    test_variant_as (bus);
+    test_variant_unknown (bus);
+    test_property_array (bus);
+
+    sd_bus_flush (bus);
+    sd_bus_close (bus);
+    sd_bus_unref (bus);
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/test/objpath.c b/src/modules/sdbus/test/objpath.c
new file mode 100644
index 000000000000..3dda9d7221ac
--- /dev/null
+++ b/src/modules/sdbus/test/objpath.c
@@ -0,0 +1,83 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <systemd/sd-bus.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+#include "ccan/array_size/array_size.h"
+
+#include "objpath.h"
+
+struct testvec {
+    const char *xpath;
+    const char *path;
+};
+
+static struct testvec opvec[] = {
+    { "/object/path/foo.suffix",
+      "/object/path/foo_2esuffix" },
+    { "/org/freedesktop/systemd1/unit/flux-foo.service",
+      "/org/freedesktop/systemd1/unit/flux_2dfoo_2eservice" },
+    { "/foo/flea-bag",
+      "/foo/flea_2dbag" },
+    { "/foo",
+      "/foo" },
+    { "/",
+      "/" },
+};
+
+void test_decode (sd_bus *bus)
+{
+    for (int i = 0; i < ARRAY_SIZE (opvec); i++) {
+        char *p;
+        p = objpath_encode (opvec[i].xpath);
+        diag ("%s", p);
+        ok (p && streq (p, opvec[i].path),
+            "objpath_encode %s works", opvec[i].xpath);
+        free (p);
+        p = objpath_decode (opvec[i].path);
+        ok (p && streq (p, opvec[i].xpath),
+            "objpath_decode %s works", opvec[i].path);
+        free (p);
+    }
+}
+
+int main (int argc, char **argv)
+{
+    sd_bus *bus;
+    int e;
+
+    plan (NO_PLAN);
+
+    if ((e = sd_bus_open_user (&bus)) < 0) {
+        diag ("could not open sdbus: %s", strerror (e));
+        if (!getenv ("DBUS_SESSION_BUS_ADDRESS"))
+            diag ("Hint: DBUS_SESSION_BUS_ADDRESS is not set");
+        if (!getenv ("XDG_RUNTIME_DIR"))
+            diag ("Hint: XDG_RUNTIME_DIR is not set");
+        plan (SKIP_ALL);
+        done_testing ();
+    }
+
+    test_decode (bus);
+
+    sd_bus_flush (bus);
+    sd_bus_close (bus);
+    sd_bus_unref (bus);
+
+    done_testing ();
+}
+
+// vi: ts=4 sw=4 expandtab
diff --git a/src/modules/sdbus/watcher.c b/src/modules/sdbus/watcher.c
new file mode 100644
index 000000000000..6c771833c65a
--- /dev/null
+++ b/src/modules/sdbus/watcher.c
@@ -0,0 +1,175 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* watcher.c - a flux watcher that becomes ready when sd-bus needs service
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <poll.h>
+#include <errno.h>
+#include <time.h>
+#include <math.h>
+#include <systemd/sd-bus.h>
+#include <flux/core.h>
+
+#include "src/common/libflux/reactor_private.h"
+
+#include "watcher.h"
+
+struct sdbus_watcher {
+    sd_bus *bus;
+    flux_watcher_t *in;
+    flux_watcher_t *out;
+    flux_watcher_t *tmout;
+    flux_watcher_t *prep;
+
+    flux_watcher_t *w;
+
+    flux_watcher_f cb;
+    void *cb_arg;
+};
+
+/* The event loop is about to (possibly) block.  The job of this function
+ * is to ensure that the appropriate watchers are enabled so the event loop
+ * unblocks when sd-bus requires service.
+ * N.B. in practice, it seems that sd_bus_get_events always returns() at
+ * least POLLIN, which makes sense given that the D-Bus spec allows the bus
+ * to send unsolicited signals like 'NameAcquired'.
+ */
+static void prep_cb (flux_reactor_t *r,
+                     flux_watcher_t *w,
+                     int revents,
+                     void *arg)
+{
+    struct sdbus_watcher *sdw = arg;
+    int events;
+    uint64_t usec;
+    struct timespec ts;
+
+    flux_watcher_stop (sdw->in);
+    flux_watcher_stop (sdw->out);
+    flux_watcher_stop (sdw->tmout);
+
+    if ((events = sd_bus_get_events (sdw->bus)) >= 0) {
+        if ((events & POLLIN))
+            flux_watcher_start (sdw->in);
+        if ((events & POLLOUT))
+            flux_watcher_start (sdw->out);
+    }
+    /* sd_bus_get_timeout(3) sets 'usec' to the absolute time when the bus
+     * wants service, or UINT64_MAX for "no timeout".  Convert that to a time
+     * relative to now wanted by the flux timer watcher.
+     * N.B.  floor() rounds 'now' down so that when it is subtracted from
+     * 'usec', the result is rounded up per sd_bus_get_timeout(3)
+     * recommendation.
+     */
+    if (sd_bus_get_timeout (sdw->bus, &usec) >= 0
+        && usec != UINT64_MAX
+        && clock_gettime (CLOCK_MONOTONIC, &ts) == 0) {
+        double now = floor (1E6*ts.tv_sec + 1E-3*ts.tv_nsec);
+        double timeout = 1E-6*(usec - now);
+
+        if (timeout >= 0.) {
+            flux_timer_watcher_reset (sdw->tmout, timeout, 0.);
+            flux_watcher_start (sdw->tmout);
+        }
+    }
+}
+
+/* The timer and/or fd watchers are ready.  Call the bus watcher callback
+ * so it can call sd_bus_process(3).
+ */
+static void bus_cb (flux_reactor_t *r,
+                    flux_watcher_t *w,
+                    int revents,
+                    void *arg)
+{
+    struct sdbus_watcher *sdw = arg;
+
+    if (sdw->cb)
+        sdw->cb (r, sdw->w, revents, sdw->cb_arg);
+}
+
+static void op_start (flux_watcher_t *w)
+{
+    struct sdbus_watcher *sdw = watcher_get_data (w);
+    flux_watcher_start (sdw->prep);
+}
+
+static void op_stop (flux_watcher_t *w)
+{
+    struct sdbus_watcher *sdw = watcher_get_data (w);
+    flux_watcher_stop (sdw->prep);
+    flux_watcher_stop (sdw->in);
+    flux_watcher_stop (sdw->out);
+    flux_watcher_stop (sdw->tmout);
+}
+
+static bool op_is_active (flux_watcher_t *w)
+{
+    struct sdbus_watcher *sdw = watcher_get_data (w);
+    return ev_is_active (sdw->prep);
+}
+
+static void op_destroy (flux_watcher_t *w)
+{
+    struct sdbus_watcher *sdw = watcher_get_data (w);
+    flux_watcher_destroy (sdw->prep);
+    flux_watcher_destroy (sdw->in);
+    flux_watcher_destroy (sdw->out);
+    flux_watcher_destroy (sdw->tmout);
+}
+
+static struct flux_watcher_ops sdbus_watcher_ops = {
+    .start = op_start,
+    .stop = op_stop,
+    .destroy = op_destroy,
+    .is_active = op_is_active,
+};
+
+flux_watcher_t *sdbus_watcher_create (flux_reactor_t *r,
+                                      sd_bus *bus,
+                                      flux_watcher_f cb,
+                                      void *arg)
+{
+    flux_watcher_t *w;
+    struct sdbus_watcher *sdw;
+    int fd;
+
+    if ((fd = sd_bus_get_fd (bus)) < 0) {
+        errno = -fd;
+        return NULL;
+    }
+    if (!(w = watcher_create (r,
+                              sizeof (*sdw),
+                              &sdbus_watcher_ops,
+                              cb,
+                              arg)))
+        return NULL;
+    sdw = watcher_get_data (w);
+    sdw->bus = bus;
+    sdw->w = w;
+    sdw->cb = cb;
+    sdw->cb_arg = arg;
+    if (!(sdw->out = flux_fd_watcher_create (r, fd, FLUX_POLLOUT, bus_cb, sdw))
+        || !(sdw->in = flux_fd_watcher_create (r, fd, FLUX_POLLIN, bus_cb, sdw))
+        || !(sdw->tmout = flux_timer_watcher_create (r, 0., 0., bus_cb, sdw))
+        || !(sdw->prep = flux_prepare_watcher_create (r, prep_cb, sdw)))
+        goto error;
+    return w;
+error:
+    flux_watcher_destroy (w);
+    return NULL;
+}
+
+
+// vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/modules/sdbus/watcher.h b/src/modules/sdbus/watcher.h
new file mode 100644
index 000000000000..4d738b622bef
--- /dev/null
+++ b/src/modules/sdbus/watcher.h
@@ -0,0 +1,28 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SDBUS_WATCHER_H
+#define _SDBUS_WATCHER_H
+
+#include <systemd/sd-bus.h>
+#include <flux/core.h>
+
+/* This watcher is called each time the sd-bus may require service.
+ * The callback should call sd_bus_process(3) to give libsystemd the
+ * opportunity to make progress.
+ */
+flux_watcher_t *sdbus_watcher_create (flux_reactor_t *r,
+                                      sd_bus *bus,
+                                      flux_watcher_f cb,
+                                      void *arg);
+
+#endif /* !_SDBUS_WATCHER_H */
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/sdexec/sdexec.c b/src/modules/sdexec/sdexec.c
new file mode 100644
index 000000000000..28c17a9600fb
--- /dev/null
+++ b/src/modules/sdexec/sdexec.c
@@ -0,0 +1,1235 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* sdexec.c - run subprocesses under systemd as transient units
+ *
+ * Configuration:
+ *  [systemd]
+ *  sdexec-debug = true   # enables debug logging
+ *  enable = true         # enables auto loading by rc script
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <jansson.h>
+#include <uuid.h>
+#ifndef UUID_STR_LEN
+#define UUID_STR_LEN 37     // defined in later libuuid headers
+#endif
+#include <flux/core.h>
+
+
+#include "src/common/libsubprocess/client.h"
+#include "src/common/libioencode/ioencode.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/fdutils.h"
+#include "src/common/libutil/jpath.h"
+#include "src/common/libutil/parse_size.h"
+#include "ccan/str/str.h"
+
+#include "src/common/libsdexec/stop.h"
+#include "src/common/libsdexec/start.h"
+#include "src/common/libsdexec/channel.h"
+#include "src/common/libsdexec/unit.h"
+#include "src/common/libsdexec/property.h"
+
+#define MODULE_NAME "sdexec"
+
+struct sdexec_ctx {
+    flux_t *h;
+    uint32_t rank;
+    char *local_uri;
+    flux_msg_handler_t **handlers;
+    struct flux_msglist *requests; // each exec request "owns" an sdproc
+    struct flux_msglist *kills;
+};
+
+struct sdproc {
+    const flux_msg_t *msg;
+    json_t *cmd;
+    int flags;
+    flux_future_t *f_watch;
+    flux_future_t *f_start;
+    flux_future_t *f_stop;
+    struct unit *unit;
+    struct flux_msglist *write_requests;
+    struct channel *in;
+    struct channel *out;
+    struct channel *err;
+    uint8_t started_response_sent:1;
+    uint8_t finished_response_sent:1;
+    uint8_t out_eof_sent:1;
+    uint8_t err_eof_sent:1;
+
+    int errnum;
+    const char *errstr;
+    flux_error_t error;
+
+    struct sdexec_ctx *ctx;
+};
+
+static bool sdexec_debug;
+
+static __attribute__ ((format (printf, 2, 3)))
+void sdexec_log_debug (flux_t *h, const char *fmt, ...)
+{
+    if (sdexec_debug) {
+        va_list ap;
+
+        va_start (ap, fmt);
+        flux_vlog (h, LOG_DEBUG, fmt, ap);
+        va_end (ap);
+    }
+}
+
+static void delete_message (struct flux_msglist *msglist,
+                            const flux_msg_t *msg)
+{
+    const flux_msg_t *m;
+
+    m = flux_msglist_first (msglist);
+    while (m) {
+        if (msg == m) {
+            flux_msglist_delete (msglist);
+            return;
+        }
+        m = flux_msglist_next (msglist);
+    }
+}
+
+static const flux_msg_t *lookup_message_bypid (struct flux_msglist *msglist,
+                                               pid_t pid)
+{
+    const flux_msg_t *m;
+
+    m = flux_msglist_first (msglist);
+    while (m) {
+        struct sdproc *proc = flux_msg_aux_get (m, "sdproc");
+        if (sdexec_unit_pid (proc->unit) == pid)
+            return m;
+        m = flux_msglist_next (msglist);
+    }
+    return NULL;
+}
+
+static const flux_msg_t *lookup_message_byaux (struct flux_msglist *msglist,
+                                               const char *name,
+                                               void *value)
+{
+    const flux_msg_t *m;
+
+    m = flux_msglist_first (msglist);
+    while (m) {
+        if (flux_msg_aux_get (m, name) == value)
+            return m;
+        m = flux_msglist_next (msglist);
+    }
+    return NULL;
+}
+
+/* Find an sdexec.exec message with the same sender as msg and matchtag as
+ * specified in the msg matchtag field.
+ * N.B. flux_cancel_match() happens to be helpful because RFC 42 subprocess
+ * write works like RFC 6 cancel.
+ */
+static const flux_msg_t *lookup_message_byclient (struct flux_msglist *msglist,
+                                                  const flux_msg_t *msg)
+{
+    const flux_msg_t *m;
+
+    m = flux_msglist_first (msglist);
+    while (m) {
+        if (flux_cancel_match (msg, m))
+            return m;
+        m = flux_msglist_next (msglist);
+    }
+    return NULL;
+}
+
+static void exec_respond_error (struct sdproc *proc,
+                                int errnum,
+                                const char *errstr)
+{
+    if (flux_respond_error (proc->ctx->h, proc->msg, errnum, errstr) < 0)
+        flux_log_error (proc->ctx->h, "error responding to exec request");
+    delete_message (proc->ctx->requests, proc->msg); // destroys proc too
+}
+
+/* Send the streaming response IFF unit cleanup is complete and EOFs have
+ * been sent.  Channel EOF and cleanup might(?) complete out of order so
+ * call this from unit and channel callbacks.
+ */
+static void finalize_exec_request_if_done (struct sdproc *proc)
+{
+    if (sdexec_unit_state (proc->unit) == STATE_INACTIVE
+        && sdexec_unit_substate (proc->unit) == SUBSTATE_DEAD
+        && (!proc->out || proc->out_eof_sent)
+        && (!proc->err || proc->err_eof_sent)) {
+
+        /* If there was an exec error, fail with ENOENT.
+         * N.B. we have no way of discerning which exec(2) error occurred,
+         * so guess ENOENT.  It could actually be EPERM, for example.
+         */
+        if (sdexec_unit_has_failed (proc->unit)) {
+            flux_error_t error;
+            errprintf (&error,
+                       "unit process could not be started (systemd error %d)",
+                       sdexec_unit_systemd_error (proc->unit));
+            exec_respond_error (proc, ENOENT, error.text);
+        }
+        else if (!proc->started_response_sent) {
+            exec_respond_error (proc,
+                                EINVAL,
+                                "Internal error: unfailed inactive.dead unit"
+                                " never received ExecMainPID property");
+        }
+        else if (!proc->finished_response_sent) {
+            exec_respond_error (proc,
+                                EINVAL,
+                                "Internal error: unfailed inactive.dead unit"
+                                " never received ExecMainCode and"
+                                " ExecMainStatus properties.");
+        }
+        else
+            exec_respond_error (proc, ENODATA, NULL);
+    }
+}
+
+static void stop_continuation (flux_future_t *f, void *arg)
+{
+    struct sdproc *proc = arg;
+
+    if (flux_rpc_get (f, NULL) < 0) {
+        flux_log (proc->ctx->h,
+                  LOG_ERR,
+                  "stop %s: %s",
+                  sdexec_unit_name (proc->unit),
+                  future_strerror (f, errno));
+    }
+}
+
+static void reset_continuation (flux_future_t *f, void *arg)
+{
+    struct sdproc *proc = arg;
+
+    if (flux_rpc_get (f, NULL) < 0) {
+        flux_log (proc->ctx->h,
+                  LOG_ERR,
+                  "reset-failed %s: %s",
+                  sdexec_unit_name (proc->unit),
+                  future_strerror (f, errno));
+    }
+}
+
+/* sdbus.subscribe sent a PropertiesChanged response for a particular unit.
+ * Advance the proc->unit state accordingly and send exec responses as needed.
+ * call finalize_exec_request_if_done() in case this update is the last thing
+ * the exec request was waiting for.
+ */
+static void property_changed_continuation (flux_future_t *f, void *arg)
+{
+    flux_t *h = flux_future_get_flux (f);
+    struct sdproc *proc = arg;
+    json_t *properties;
+
+    if (!(properties = sdexec_property_changed_dict (f))) {
+        exec_respond_error (proc, errno, future_strerror (f, errno));
+        return;
+    }
+    if (!sdexec_unit_update (proc->unit, properties)) {
+        flux_future_reset (f);
+        return;
+    }
+
+    sdexec_log_debug (h,
+                      "%s: %s.%s",
+                      sdexec_unit_name (proc->unit),
+                      sdexec_statetostr (sdexec_unit_state (proc->unit)),
+                      sdexec_substatetostr (sdexec_unit_substate (proc->unit)));
+
+    /* The started response must be the first response to an exec request,
+     * so channel output may begin after this.
+     * If there is an exec error, "started" should not be sent.
+     */
+    if (!proc->started_response_sent) {
+        if (sdexec_unit_has_started (proc->unit)) {
+            if (flux_respond_pack (h,
+                                   proc->msg,
+                                   "{s:s s:I}",
+                                   "type", "started",
+                                   "pid", sdexec_unit_pid (proc->unit)) < 0)
+                flux_log_error (h, "error responding to exec request");
+            proc->started_response_sent = 1;
+            sdexec_channel_start_output (proc->out);
+            sdexec_channel_start_output (proc->err);
+        }
+    }
+    /* The finished response is sent when wait status is available.
+     * If there was an exec error, "finished" should not be sent.
+     */
+    if (!proc->finished_response_sent) {
+        if (sdexec_unit_has_finished (proc->unit)) {
+            if (flux_respond_pack (h,
+                                   proc->msg,
+                                   "{s:s s:i}",
+                                   "type", "finished",
+                                   "status",
+                                   sdexec_unit_wait_status (proc->unit)) < 0)
+                flux_log_error (h, "error responding to exec request");
+            proc->finished_response_sent = 1;
+        }
+    }
+    /* If the unit reaches active.exited call StopUnit to cause stdout
+     * and stderr to reach eof, and the unit to transition to inactive.dead.
+     */
+    if (sdexec_unit_state (proc->unit) == STATE_ACTIVE
+        && sdexec_unit_substate (proc->unit) == SUBSTATE_EXITED
+        && proc->finished_response_sent) {
+
+        if (!proc->f_stop) {
+            flux_future_t *f2;
+            sdexec_log_debug (h, "stop %s", sdexec_unit_name (proc->unit));
+            if (!(f2 = sdexec_stop_unit (h,
+                                         proc->ctx->rank,
+                                         sdexec_unit_name (proc->unit),
+                                         "fail"))
+                || flux_future_then (f2, -1, stop_continuation, proc) < 0) {
+                flux_log_error (h, "error initiating unit stop");
+                flux_future_destroy (f2);
+                f2 = NULL;
+            }
+            proc->f_stop = f2;
+        }
+    }
+    /* If the unit reaches failed.failed call ResetFailedUnit to cause stdout
+     * and stderr to reach eof, and the unit to transition to inactive.dead.
+     * We can land here for both a child failure and an exec failure.
+     * Start channel output here in case of the latter so it can be finalized.
+     */
+    if (sdexec_unit_state (proc->unit) == STATE_FAILED
+        && sdexec_unit_substate (proc->unit) == SUBSTATE_FAILED) {
+
+        sdexec_channel_start_output (proc->out);
+        sdexec_channel_start_output (proc->err);
+
+        if (!proc->f_stop) {
+            flux_future_t *f2;
+            sdexec_log_debug (h,
+                              "reset-failed %s",
+                              sdexec_unit_name (proc->unit));
+            if (!(f2 = sdexec_reset_failed_unit (h,
+                                                 proc->ctx->rank,
+                                                 sdexec_unit_name (proc->unit)))
+                || flux_future_then (f2, -1, reset_continuation, proc) < 0) {
+                flux_log_error (h, "error initiating unit reset");
+                flux_future_destroy (f2);
+                f2 = NULL;
+            }
+            proc->f_stop = f2;
+        }
+    }
+    flux_future_reset (f);
+    /* Conditionally send the final RPC response.
+     */
+    finalize_exec_request_if_done (proc);
+}
+
+/* StartTransientUnit reply does not normally generate a sdexec.exec response,
+ * unless it fails.  Streaming responses continue as property change updates
+ * are received from sdbus.
+ */
+static void start_continuation (flux_future_t *f, void *arg)
+{
+    const flux_msg_t *msg = flux_future_aux_get (f, "request");
+    struct sdproc *proc = flux_msg_aux_get (msg, "sdproc");
+    struct sdexec_ctx *ctx = arg;
+
+    if (sdexec_start_transient_unit_get (f, NULL) < 0)
+        goto error;
+    /* Now that systemd has acknowledged the StartTransientUnit request, close
+     * the systemd end of any channel(s).  The assumption is that systemd has
+     * received its fd and has already called dup(2) on it.
+     */
+    sdexec_channel_close_fd (proc->in);
+    sdexec_channel_close_fd (proc->out);
+    sdexec_channel_close_fd (proc->err);
+    /* Now that stdin is ready, re-queue any messages write_cb() left in
+     * proc->write_requests.  Push these messages to the front of the flux_t
+     * queue so that they come before unprocessed writes, if any.
+     */
+    if (proc->write_requests) {
+        const flux_msg_t *request;
+        while ((request = flux_msglist_pop (proc->write_requests))) {
+            int rc = flux_requeue (ctx->h, request, FLUX_RQ_HEAD);
+            flux_msg_decref (request);
+            if (rc < 0) {
+                flux_log_error (ctx->h, "error requeuing early sdexec.write");
+                break;
+            }
+        }
+    }
+    return;
+error:
+    if (flux_respond_error (ctx->h, msg, errno, future_strerror (f, errno)))
+        flux_log_error (ctx->h, "error responding to exec request");
+    delete_message (ctx->requests, msg);
+}
+
+/* Log an error receiving data from unit stdout or stderr.  channel_cb will
+ * be called with an EOF after this callback returns.
+ */
+static void cherror_cb (struct channel *ch, flux_error_t *error, void *arg)
+{
+    struct sdproc *proc = arg;
+    flux_t *h = proc->ctx->h;
+
+    flux_log (h, LOG_ERR, "%s: %s", sdexec_channel_get_name (ch), error->text);
+}
+
+/* Receive some data from unit stdout or stderr and forward it as an
+ * exec response.  In case this was the last thing the exec request was
+ * waiting to receive (e.g. a final EOF), call finalize_exec_request_if_done()
+ * to take care of that if needed.
+ */
+static void channel_cb (struct channel *ch, json_t *io, void *arg)
+{
+    struct sdproc *proc = arg;
+    flux_t *h = proc->ctx->h;
+
+    if (flux_respond_pack (h,
+                           proc->msg,
+                           "{s:s s:i s:O}",
+                           "type", "output",
+                           "pid", sdexec_unit_pid (proc->unit),
+                           "io", io) < 0)
+        flux_log_error (h, "error responding to exec request");
+
+    const char *stream;
+    bool eof;
+    if (iodecode (io, &stream, NULL, NULL, NULL, &eof) == 0 && eof == true) {
+        if (streq (stream, "stdout"))
+            proc->out_eof_sent = true;
+        else if (streq (stream, "stderr"))
+            proc->err_eof_sent = true;
+    }
+    finalize_exec_request_if_done (proc);
+}
+
+/* Since an sdproc is attached to each exec message's aux container, this
+ * destructor is typically called when an exec request is destroyed, e.g.
+ * after unit reaping is complete and the exec client has been sent ENODATA
+ * or another error response.  This ends the sdbus.subscribe request for
+ * property updates on this unit.  The subscribe future is destroyed here;
+ * we do not wait for the ENODATA response.
+ */
+static void sdproc_destroy (struct sdproc *proc)
+{
+    if (proc) {
+        int saved_errno = errno;
+        sdexec_channel_destroy (proc->in);
+        sdexec_channel_destroy (proc->out);
+        sdexec_channel_destroy (proc->err);
+        if (proc->f_watch) {
+            flux_future_t *f;
+            sdexec_log_debug (proc->ctx->h,
+                              "unwatch %s",
+                              sdexec_unit_name (proc->unit));
+            f = flux_rpc_pack (proc->ctx->h,
+                               "sdbus.subscribe-cancel",
+                               proc->ctx->rank,
+                               FLUX_RPC_NORESPONSE,
+                               "{s:i}",
+                               "matchtag",
+                               flux_rpc_get_matchtag (proc->f_watch));
+            flux_future_destroy (f);
+            flux_future_destroy (proc->f_watch);
+        }
+        flux_future_destroy (proc->f_start);
+        flux_future_destroy (proc->f_stop);
+        sdexec_unit_destroy (proc->unit);
+        json_decref (proc->cmd);
+        flux_msglist_destroy (proc->write_requests);
+        free (proc);
+        errno = saved_errno;
+    }
+}
+
+/* Set a key 'k', value 'v' pair in the dictionary named 'name'.
+ * The dictionary is created if it does not exist.
+ * If key is already set, the previous value is overwritten.
+ */
+static int set_dict (json_t *o,
+                     const char *name,
+                     const char *k,
+                     const char *v)
+{
+    json_t *dict;
+    json_t *vo;
+
+    if (!(dict = json_object_get (o, name))) {
+        if (!(dict = json_object ())
+            || json_object_set_new (o, name, dict) < 0) {
+            json_decref (dict);
+            goto nomem;
+        }
+    }
+    if (!(vo = json_string (v))
+        || json_object_set_new (dict, k, vo) < 0) {
+        json_decref (vo);
+        goto nomem;
+    }
+    return 0;
+nomem:
+    errno = ENOMEM;
+    return -1;
+}
+
+/* Look up key 'k' in dictionary named 'name' and assign value to 'vp'.
+ */
+static int get_dict (json_t *o,
+                     const char *name,
+                     const char *k,
+                     const char **vp)
+{
+    const char *v;
+    if (json_unpack (o, "{s:{s:s}}", name, k, &v) < 0) {
+        errno = ENOENT;
+        return -1;
+    }
+    *vp = v;
+    return 0;
+}
+
+static int get_stream_bufsize (json_t *cmd,
+                               const char *stream,
+                               size_t *vp)
+{
+    char key[64];
+    const char *val;
+    uint64_t result;
+
+    snprintf (key, sizeof (key), "%s_BUFSIZE", stream);
+    if (get_dict (cmd, "opts", key, &val) < 0) {
+        *vp = 0;
+        return 0;
+    }
+    if (parse_size (val, &result) < 0 || result > SIZE_MAX)
+        return -1;
+    *vp = result;
+    return 0;
+}
+
+static int get_stream_line_buffer (json_t *cmd,
+                                   const char *stream,
+                                   bool *vp,
+                                   bool default_value)
+{
+    char key[64];
+    const char *val;
+
+    snprintf (key, sizeof (key), "%s_LINE_BUFFER", stream);
+    if (get_dict (cmd, "opts", key, &val) < 0) {
+        *vp = default_value;
+        return 0;
+    }
+    if (!strcasecmp (val, "false"))
+        *vp = false;
+    else if (!strcasecmp (val, "true"))
+        *vp = true;
+    else
+        return -1;
+    return 0;
+}
+
+static struct channel *create_out_channel (flux_t *h,
+                                           json_t *cmd,
+                                           const char *stream,
+                                           void *arg)
+{
+    bool linebuf;
+    size_t bufsize;
+
+    if (get_stream_line_buffer (cmd, stream, &linebuf, true) < 0
+        || get_stream_bufsize (cmd, stream, &bufsize) < 0)
+        return NULL;
+    return sdexec_channel_create_output (h,
+                                         stream,
+                                         bufsize,
+                                         linebuf ? CHANNEL_LINEBUF : 0,
+                                         channel_cb,
+                                         cherror_cb,
+                                         arg);
+}
+
+static struct sdproc *sdproc_create (struct sdexec_ctx *ctx,
+                                     json_t *cmd,
+                                     int flags)
+{
+    struct sdproc *proc;
+    const int valid_flags = SUBPROCESS_REXEC_STDOUT
+        | SUBPROCESS_REXEC_STDERR
+        | SUBPROCESS_REXEC_CHANNEL;
+    const char *name;
+    char *tmp = NULL;
+
+    if ((flags & ~valid_flags) != 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(proc = calloc (1, sizeof (*proc))))
+        return NULL;
+    proc->ctx = ctx;
+    proc->flags = flags;
+    if (!(proc->cmd = json_deep_copy (cmd))) {
+        errno = ENOMEM;
+        goto error;
+    }
+    /* Set SDEXEC_NAME for sdexec_start_transient_unit().
+     * If unset, use a truncated uuid as the name.
+     */
+    if (get_dict (proc->cmd, "opts", "SDEXEC_NAME", &name) < 0) {
+        uuid_t uuid;
+        char uuid_str[UUID_STR_LEN];
+
+        uuid_generate (uuid);
+        uuid_unparse (uuid, uuid_str);
+        uuid_str[13] = '\0'; // plenty of uniqueness
+        if (asprintf (&tmp, "%s.service", uuid_str) < 0
+            || set_dict (proc->cmd, "opts", "SDEXEC_NAME", tmp) < 0)
+            goto error;
+        name = tmp;
+    }
+    if (!(proc->unit = sdexec_unit_create (name)))
+        goto error;
+    /* Ensure that FLUX_URI refers to the local broker.
+     */
+    if (set_dict (proc->cmd, "env", "FLUX_URI", ctx->local_uri) < 0)
+        goto error;
+    /* Create channels for stdio as required by flags.
+     */
+    if (!(proc->in = sdexec_channel_create_input (ctx->h, "stdin")))
+        goto error;
+    if ((flags & SUBPROCESS_REXEC_STDOUT)) {
+        if (!(proc->out = create_out_channel (ctx->h,
+                                              proc->cmd,
+                                              "stdout",
+                                              proc)))
+            goto error;
+    }
+    if ((flags & SUBPROCESS_REXEC_STDERR)) {
+        if (!(proc->err = create_out_channel (ctx->h,
+                                              proc->cmd,
+                                              "stderr",
+                                              proc)))
+            goto error;
+    }
+    free (tmp);
+    return proc;
+error:
+    ERRNO_SAFE_WRAP (free, tmp);
+    sdproc_destroy (proc);
+    return NULL;
+}
+
+static int authorize_request (const flux_msg_t *msg,
+                              uint32_t rank,
+                              flux_error_t *error)
+{
+    if (rank != 0 || flux_msg_is_local (msg))
+        return 0;
+    errprintf (error, "Remote sdexec requests are not allowed on rank 0");
+    errno = EPERM;
+    return -1;
+}
+
+/* Start a process as a systemd transient unit.  This is a streaming request.
+ * It triggers two sdbus RPCs:
+ * 1) sdbus.subscribe (streaming) for updates to this unit's properties
+ * 2) sdbus.call StartTransientUnit to launch the transient unit.
+ * Responses to those are handled in property_changed_continuation()
+ * and start_continuation().
+ */
+static void exec_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    struct sdexec_ctx *ctx = arg;
+    json_t *cmd;
+    int flags;
+    flux_error_t error;
+    const char *errstr = NULL;
+    struct sdproc *proc;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:o s:i}",
+                             "cmd", &cmd,
+                             "flags", &flags) < 0)
+        goto error;
+    if (!flux_msg_is_streaming (msg)) {
+        errstr = "exec request is missing STREAMING flag";
+        errno = EPROTO;
+        goto error;
+    }
+    if (authorize_request (msg, ctx->rank, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if ((flags & SUBPROCESS_REXEC_CHANNEL)) {
+        errstr = "subprocess auxiliary channels are not supported yet";
+        errno = EINVAL;
+        goto error;
+    }
+    if (!(proc = sdproc_create (ctx, cmd, flags))
+        || flux_msg_aux_set (msg,
+                             "sdproc",
+                             proc,
+                             (flux_free_f)sdproc_destroy) < 0) {
+        sdproc_destroy (proc);
+        goto error;
+    }
+    proc->msg = msg;
+    sdexec_log_debug (h, "watch %s", sdexec_unit_name (proc->unit));
+    if (!(proc->f_watch = sdexec_property_changed (h,
+                                               ctx->rank,
+                                               sdexec_unit_path (proc->unit)))
+        || flux_future_then (proc->f_watch,
+                             -1,
+                             property_changed_continuation,
+                             proc) < 0)
+        goto error;
+    sdexec_log_debug (h, "start %s", sdexec_unit_name (proc->unit));
+    if (!(proc->f_start = sdexec_start_transient_unit (h,
+                                             ctx->rank,
+                                             "fail", // mode
+                                             "exec", // type
+                                             proc->cmd,
+                                             sdexec_channel_get_fd (proc->in),
+                                             sdexec_channel_get_fd (proc->out),
+                                             sdexec_channel_get_fd (proc->err),
+                                             &error))) {
+        errstr = error.text;
+        goto error;
+    }
+    if (flux_future_then (proc->f_start, -1, start_continuation, ctx) < 0
+        || flux_future_aux_set (proc->f_start,
+                                "request",
+                                (void *)msg,
+                                NULL) < 0)
+        goto error;
+    if (flux_msglist_append (ctx->requests, msg) < 0)
+        goto error;
+    return; // response occurs later
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to exec request");
+}
+
+/* Send some data to stdin of a unit started with sdexec.exec.
+ * The unit is looked up by pid. This request is "fire and forget"
+ * (no response) per libsubprocess protocol.
+ */
+static void write_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
+{
+    struct sdexec_ctx *ctx = arg;
+    int matchtag;
+    json_t *io;
+    const flux_msg_t *exec_request;
+    flux_error_t error;
+    struct sdproc *proc;
+    const char *stream;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s:o}",
+                             "matchtag", &matchtag,
+                             "io", &io) < 0) {
+        flux_log_error (h, "error decoding write request");
+        return;
+    }
+    if (!flux_msg_is_noresponse (msg)) {
+        flux_log (h, LOG_ERR, "write request is missing NORESPONSE flag");
+        return;
+    }
+    if (authorize_request (msg, ctx->rank, &error) < 0) {
+        flux_log_error (h, "%s", error.text);
+        return;
+    }
+    if (!(exec_request = lookup_message_byclient (ctx->requests, msg))
+        || !(proc = flux_msg_aux_get (exec_request, "sdproc"))) {
+        flux_log (h, LOG_ERR, "sdexec.write: subprocess no longer exists");
+        return;
+    }
+    /* If the systemd unit has not started yet, enqueue the write request for
+     * later processing in start_continuation().  We can tell that it hasn't
+     * started if start_continuation() has not yet handed the stdin channel
+     * file descriptor over to systemd by calling the close function.
+     */
+    if (sdexec_channel_get_fd (proc->in) != -1) { // not yet claimed by systemd
+        if (!proc->write_requests) {
+            if (!(proc->write_requests = flux_msglist_create ())) {
+                flux_log_error (h, "sdexec.write: error creating write queue");
+                return;
+            }
+        }
+        if (flux_msglist_push (proc->write_requests, msg) < 0)
+            flux_log_error (h, "sdexec.write: error enqueueing write request");
+        return;
+    }
+    if (iodecode (io, &stream, NULL, NULL, NULL, NULL) == 0
+        && !streq (stream, "stdin")) {
+        flux_log (h, LOG_ERR, "sdexec.write: %s is an invalid stream", stream);
+        return;
+    }
+    if (sdexec_channel_write (proc->in, io) < 0) {
+        flux_log_error (h, "sdexec.write %s", stream);
+        return;
+    }
+}
+
+static void kill_continuation (flux_future_t *f, void *arg)
+{
+    struct sdexec_ctx *ctx = arg;
+    flux_t *h = ctx->h;
+    const flux_msg_t *msg = lookup_message_byaux (ctx->kills, "kill", f);
+    int rc;
+
+    if (flux_rpc_get (f, NULL) < 0)
+        rc = flux_respond_error (h, msg, errno, future_strerror (f, errno));
+    else
+        rc = flux_respond (h, msg, NULL) ;
+    if (rc < 0)
+        flux_log_error (h, "error responding to kill request");
+    flux_msglist_delete (ctx->kills); // destroys msg and f
+}
+
+/* Handle a kill by pid request.  This does not work on arbitrary pids,
+ * only the pids of units started with sdexec.exec since the sdexec module
+ * was loaded.  Since this sends an sdbus RPC, the response is handled in
+ * kill_continuation() when the sdbus response is received.
+ */
+static void kill_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    struct sdexec_ctx *ctx = arg;
+    pid_t pid;
+    int signum;
+    const flux_msg_t *exec_request;
+    struct sdproc *proc;
+    flux_error_t error;
+    const char *errstr = NULL;
+    flux_future_t *f;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s:i}",
+                             "pid", &pid,
+                             "signum", &signum) < 0)
+        goto error;
+    if (authorize_request (msg, ctx->rank, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (!(exec_request = lookup_message_bypid (ctx->requests, pid))
+        || !(proc = flux_msg_aux_get (exec_request, "sdproc"))) {
+        errprintf (&error, "kill pid=%d not found", pid);
+        errstr = error.text;
+        errno = ESRCH;
+        goto error;
+    }
+    sdexec_log_debug (h,
+                      "kill main %s (signal %d)",
+                      sdexec_unit_name (proc->unit),
+                      signum);
+    if (!(f = sdexec_kill_unit (h,
+                                ctx->rank,
+                                sdexec_unit_name (proc->unit),
+                                "main",
+                                signum))
+        || flux_future_then (f, -1, kill_continuation, ctx) < 0
+        || flux_msg_aux_set (msg,
+                             "kill",
+                             f,
+                             (flux_free_f)flux_future_destroy) < 0) {
+        flux_future_destroy (f);
+        errstr = "error sending KillUnit request";
+        goto error;
+    }
+    // kill_continuation will respond
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to kill request");
+}
+
+/* Handle an sdexec.list request.
+ * At this time, this RPC is only used in test and the returned data
+ * is sparse.  It could be expanded later if needed.
+ */
+static void list_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
+{
+    struct sdexec_ctx *ctx = arg;
+    flux_error_t error;
+    const char *errstr = NULL;
+    json_t *procs = NULL;
+    const flux_msg_t *req;
+
+    if (authorize_request (msg, ctx->rank, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (!(procs = json_array ()))
+        goto nomem;
+    req = flux_msglist_first (ctx->requests);
+    while (req) {
+        struct sdproc *proc;
+        const char *arg0;
+        json_t *o;
+        if ((proc = flux_msg_aux_get (req, "sdproc"))
+            && json_unpack (proc->cmd, "{s:[s]}", "cmdline", &arg0) == 0
+            && (o = json_pack ("{s:i s:s}",
+                               "pid", sdexec_unit_pid (proc->unit),
+                               "cmd", arg0))) {
+            if (json_array_append_new (procs, o) < 0) {
+                json_decref (o);
+                goto nomem;
+            }
+        }
+        req = flux_msglist_next (ctx->requests);
+    }
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:i s:O}",
+                           "rank", ctx->rank,
+                           "procs", procs) < 0)
+        flux_log_error (h, "error responding to list request");
+    json_decref (procs);
+    return;
+nomem:
+    errno = ENOMEM;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to list request");
+    json_decref (procs);
+}
+
+/* Make a string like "inactive.dead" in buf[size]
+ */
+static int get_statestr (char *buf, size_t size, struct unit *unit)
+{
+    const char *s = sdexec_statetostr (sdexec_unit_state (unit));
+    const char *ss = sdexec_substatetostr (sdexec_unit_substate (unit));
+    if (snprintf (buf, size, "%s.%s", s, ss) >= size) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    return 0;
+}
+
+static json_t *get_proc_stats (struct sdproc *proc)
+{
+    json_t *o;
+    char statebuf[64];
+    json_t *in_stats;
+    json_t *out_stats;
+    json_t *err_stats;
+
+    if (!proc->unit
+        || get_statestr (statebuf, sizeof (statebuf), proc->unit) < 0)
+        return NULL;
+    in_stats = sdexec_channel_get_stats (proc->in);
+    out_stats = sdexec_channel_get_stats (proc->out);
+    err_stats = sdexec_channel_get_stats (proc->err);
+    o = json_pack ("{s:s s:i s:O s:O s:O}",
+                   "state", statebuf,
+                   "pid", sdexec_unit_pid (proc->unit),
+                   "in", in_stats ? in_stats : json_null (),
+                   "out", out_stats ? out_stats : json_null (),
+                   "err", err_stats ? err_stats : json_null ());
+    json_decref (in_stats);
+    json_decref (out_stats);
+    json_decref (err_stats);
+    return o;
+}
+
+static void stats_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
+{
+    struct sdexec_ctx *ctx = arg;
+    json_t *procs;
+    const flux_msg_t *m;
+
+    if (!(procs = json_object ()))
+        goto nomem;
+    m = flux_msglist_first (ctx->requests);
+    while (m) {
+        struct sdproc *proc;
+        json_t *entry = NULL;
+
+        if (!(proc = flux_msg_aux_get (m, "sdproc"))
+            || !(entry = get_proc_stats (proc))
+            || json_object_set_new (procs,
+                                    sdexec_unit_name (proc->unit),
+                                    entry) < 0) {
+            json_decref (entry);
+            goto nomem;
+        }
+        m = flux_msglist_next (ctx->requests);
+    }
+    if (flux_respond_pack (h, msg, "{s:O}", "procs", procs) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+    json_decref (procs);
+    return;
+nomem:
+    errno = ENOMEM;
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "error responding to stats-get request");
+    json_decref (procs);
+}
+
+/* When a client (like flux-exec or job-exec) disconnects, send any running
+ * units that were started by that UUID a SIGKILL to begin cleanup.  Leave
+ * the request in ctx->requests so the unit can be "reaped".  Let normal
+ * cleanup of the request (including generating a response which shouldn't
+ * hurt) to occur when that happens.
+ */
+static void disconnect_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    struct sdexec_ctx *ctx = arg;
+    const flux_msg_t *request;
+
+    request = flux_msglist_first (ctx->requests);
+    while (request) {
+        if (flux_disconnect_match (msg, request)) {
+            struct sdproc *proc = flux_msg_aux_get (request, "sdproc");
+            if (proc) {
+                flux_future_t *f;
+                f = sdexec_kill_unit (h,
+                                      ctx->rank,
+                                      sdexec_unit_name (proc->unit),
+                                      "main",
+                                       SIGKILL);
+                flux_future_destroy (f);
+            }
+        }
+        request = flux_msglist_next (ctx->requests);
+    }
+}
+
+/* N.B. systemd.enable is checked in rc1 and ignored here since
+ * it should be OK to load the module manually for testing.
+ */
+static int sdexec_configure (struct sdexec_ctx *ctx,
+                             const flux_conf_t *conf,
+                             flux_error_t *error)
+{
+    flux_error_t conf_error;
+    int debug = 0;
+
+    if (flux_conf_unpack (conf,
+                          &conf_error,
+                          "{s?{s?b}}",
+                          "systemd",
+                            "sdexec-debug", &debug) < 0) {
+        errprintf (error,
+                   "error reading [systemd] config table: %s",
+                   conf_error.text);
+        return -1;
+    }
+    sdexec_debug = (debug ? true : false);
+    return 0;
+}
+
+static void config_reload_cb (flux_t *h,
+                              flux_msg_handler_t *mh,
+                              const flux_msg_t *msg,
+                              void *arg)
+{
+    struct sdexec_ctx *ctx = arg;
+    const flux_conf_t *conf;
+    flux_error_t error;
+    const char *errstr = NULL;
+
+    if (flux_conf_reload_decode (msg, &conf) < 0) {
+        errstr = "Failed to parse config-reload request";
+        goto error;
+    }
+    if (sdexec_configure (ctx, conf, &error) < 0) {
+        errstr = error.text;
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        flux_log_error (h, "error responding to config-reload request");
+}
+
+static struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,
+      "disconnect",
+      disconnect_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "exec",
+      exec_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "write",
+      write_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "kill",
+      kill_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "list",
+      list_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "stats-get",
+      stats_cb,
+      0
+    },
+    { FLUX_MSGTYPE_REQUEST,
+      "config-reload",
+      config_reload_cb,
+      0
+    },
+    FLUX_MSGHANDLER_TABLE_END
+};
+
+static void sdexec_ctx_destroy (struct sdexec_ctx *ctx)
+{
+    if (ctx) {
+        int saved_errno = errno;
+        flux_msg_handler_delvec (ctx->handlers);
+        if (ctx->requests) {
+            const flux_msg_t *msg;
+            msg = flux_msglist_first (ctx->requests);
+            while (msg) {
+                const char *errstr = "sdexec module is unloading";
+                if (flux_respond_error (ctx->h, msg, ENOSYS, errstr) < 0)
+                    flux_log_error (ctx->h, "error responding to exec request");
+                msg = flux_msglist_next (ctx->requests);
+            }
+            flux_msglist_destroy (ctx->requests);
+        }
+        flux_msglist_destroy (ctx->kills);
+        free (ctx->local_uri);
+        free (ctx);
+        errno = saved_errno;
+    }
+}
+
+static struct sdexec_ctx *sdexec_ctx_create (flux_t *h)
+{
+    struct sdexec_ctx *ctx;
+    const char *s;
+
+    if (!(ctx = calloc (1, sizeof (*ctx))))
+        return NULL;
+    ctx->h = h;
+    if (flux_get_rank (h, &ctx->rank) < 0)
+        goto error;
+    if (!(s = flux_attr_get (h, "local-uri"))
+        || !(ctx->local_uri = strdup (s)))
+        goto error;
+    if (!(ctx->requests = flux_msglist_create ())
+        || !(ctx->kills = flux_msglist_create ()))
+        goto error;
+    return ctx;
+error:
+    sdexec_ctx_destroy (ctx);
+    return NULL;
+}
+
+/* Check if the sdbus module is loaded on the local rank by pinging its
+ * stats-get method.  N.B. sdbus handles its D-bus connect asynchronously
+ * so stats-get should be responsive even if D-Bus is not.
+ */
+static int sdbus_is_loaded (flux_t *h, uint32_t rank, flux_error_t *error)
+{
+    flux_future_t *f;
+
+    if (!(f = flux_rpc (h, "sdbus.stats-get", NULL, rank, 0))
+        || flux_rpc_get (f, NULL) < 0) {
+        if (errno == ENOSYS)
+            errprintf (error, "sdbus module is not loaded");
+        else
+            errprintf (error, "sdbus: %s", future_strerror (f, errno));
+        flux_future_destroy (f);
+        return -1;
+    }
+    flux_future_destroy (f);
+    return 0;
+}
+
+int mod_main (flux_t *h, int argc, char **argv)
+{
+    struct sdexec_ctx *ctx;
+    flux_error_t error;
+    int rc = -1;
+
+    if (!(ctx = sdexec_ctx_create (h)))
+        goto error;
+    if (sdexec_configure (ctx, flux_get_conf (h), &error) < 0) {
+        flux_log (h, LOG_ERR, "%s", error.text);
+        goto error;
+    }
+    if (flux_msg_handler_addvec_ex (h,
+                                    MODULE_NAME,
+                                    htab,
+                                    ctx,
+                                    &ctx->handlers) < 0)
+        goto error;
+    if (sdbus_is_loaded (h, ctx->rank, &error) < 0) {
+        flux_log (h, LOG_ERR, "%s", error.text);
+        goto error;
+    }
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
+        flux_log_error (h, "reactor exited abnormally");
+        goto error;
+    }
+    rc = 0;
+error:
+    sdexec_ctx_destroy (ctx);
+    return rc;
+}
+
+MOD_NAME (MODULE_NAME);
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/modules/userdb/Makefile.am b/src/modules/userdb/Makefile.am
deleted file mode 100644
index cd0b39c3afeb..000000000000
--- a/src/modules/userdb/Makefile.am
+++ /dev/null
@@ -1,22 +0,0 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
-
-fluxmod_LTLIBRARIES = userdb.la
-
-userdb_la_SOURCES = userdb.c
-userdb_la_LDFLAGS = $(fluxmod_ldflags) -module
-userdb_la_LIBADD = $(fluxmod_libadd) \
-		    $(top_builddir)/src/common/libflux-internal.la \
-		    $(top_builddir)/src/common/libflux-core.la \
-		    $(top_builddir)/src/common/libflux-optparse.la \
-		    $(ZMQ_LIBS)
diff --git a/src/modules/userdb/userdb.c b/src/modules/userdb/userdb.c
deleted file mode 100644
index 7e4b3d2fed56..000000000000
--- a/src/modules/userdb/userdb.c
+++ /dev/null
@@ -1,381 +0,0 @@
-/************************************************************\
- * Copyright 2017 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* userdb.c - map userid to rolemask
- *
- * The instance owner is automatically added with the FLUX_ROLE_OWNER role.
- *
- * If the module is loaded with --default-rolemask=ROLE[,ROLE,...]
- * then new userids are automatically added upon lookup, with the
- * specified roles.
- */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdio.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <czmq.h>
-#include <flux/core.h>
-
-#include "src/common/liboptparse/optparse.h"
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/oom.h"
-#include "src/common/libutil/xzmalloc.h"
-
-#define USERDB_CTX_MAGIC 0x2134aaaa
-typedef struct {
-    int magic;
-    optparse_t *opt;
-    uint32_t default_rolemask;
-    zhash_t *db;
-    zhash_t *iterators;
-} userdb_ctx_t;
-
-struct user {
-    uint32_t userid;
-    uint32_t rolemask;
-};
-
-static struct optparse_option opts[] = {
-    { .name = "default-rolemask",
-      .has_arg = 1,
-      .flags = OPTPARSE_OPT_AUTOSPLIT,
-      .arginfo = "ROLE[,ROLE,...]",
-      .usage = "Assign specified roles to all users",
-    },
-    OPTPARSE_TABLE_END,
-};
-
-static void freelist (zlist_t *l)
-{
-    zlist_destroy (&l);
-}
-
-static void freectx (void *arg)
-{
-    userdb_ctx_t *ctx = arg;
-    if (ctx) {
-        ctx->magic = ~USERDB_CTX_MAGIC;
-        optparse_destroy (ctx->opt);
-        zhash_destroy (&ctx->db);
-        zhash_destroy (&ctx->iterators);
-        free (ctx);
-    }
-}
-
-static userdb_ctx_t *getctx (flux_t *h, int argc, char **argv)
-{
-    userdb_ctx_t *ctx = (userdb_ctx_t *)flux_aux_get (h, "flux::userdb");
-    const char *arg;
-    optparse_err_t e;
-
-    if (!ctx) {
-        if (!(ctx = calloc (1, sizeof (*ctx)))) {
-            errno = ENOMEM;
-            goto error;
-        }
-        ctx->magic = USERDB_CTX_MAGIC;
-        ctx->default_rolemask = FLUX_ROLE_NONE;
-        if (!(ctx->opt = optparse_create ("userdb"))) {
-            errno = ENOMEM;
-            goto error;
-        }
-        e = optparse_add_option_table (ctx->opt, opts);
-        if (e != OPTPARSE_SUCCESS) {
-            if (e == OPTPARSE_NOMEM)
-                errno = ENOMEM;
-            else
-                errno = EINVAL;
-            goto error;
-        }
-        if (optparse_parse_args (ctx->opt, argc + 1,
-                                           argv - 1) < 0) {
-            errno = EINVAL;
-            goto error;
-        }
-        optparse_getopt_iterator_reset (ctx->opt, "default-rolemask");
-        while ((arg = optparse_getopt_next (ctx->opt, "default-rolemask"))) {
-            if (!strcmp (arg, "user"))
-                ctx->default_rolemask |= FLUX_ROLE_USER;
-            else if (!strcmp (arg, "owner"))
-                ctx->default_rolemask |= FLUX_ROLE_OWNER;
-            else {
-                flux_log (h, LOG_ERR, "unknown role: %s", arg);
-                errno = EINVAL;
-                goto error;
-            }
-        }
-        if (optparse_hasopt (ctx->opt, "default-rolemask"))
-            flux_log (h, LOG_INFO, "default rolemask override=0x%" PRIx32,
-                      ctx->default_rolemask);
-        if (!(ctx->db = zhash_new ()) || !(ctx->iterators = zhash_new ())) {
-            errno = ENOMEM;
-            goto error;
-        }
-        if (flux_aux_set (h, "flux::userdb", ctx, freectx) < 0)
-            goto error;
-    }
-    return ctx;
-error:
-    freectx (ctx);
-    return NULL;
-}
-
-struct user *user_create (uint32_t userid, uint32_t rolemask)
-{
-    struct user *up;
-
-    if (!(up = calloc (1, sizeof (*up)))) {
-        errno = ENOMEM;
-        goto error;
-    }
-    up->userid = userid;
-    up->rolemask = rolemask;
-    return up;
-error:
-    free (up);
-    return NULL;
-}
-
-static struct user *user_add (userdb_ctx_t *ctx, uint32_t userid,
-                                                 uint32_t rolemask)
-{
-    struct user *up = NULL;
-    char key[16];
-
-    snprintf (key, sizeof (key), "%" PRIu32, userid);
-    if (!(up = user_create (userid, rolemask))) {
-        errno = ENOMEM;
-        goto error;
-    }
-    if (zhash_insert (ctx->db, key, up) < 0) {
-        errno = EEXIST;
-        goto error;
-    }
-    zhash_freefn (ctx->db, key, (zhash_free_fn *)free);
-    return up;
-error:
-    free (up);
-    return NULL;
-}
-
-static struct user *user_lookup (userdb_ctx_t *ctx, uint32_t userid)
-{
-    struct user *up = NULL;
-    char key[16];
-
-    snprintf (key, sizeof (key), "%" PRIu32, userid);
-    if (!(up = zhash_lookup (ctx->db, key))) {
-        errno = ENOENT;
-        goto error;
-    }
-    return up;
-error:
-    free (up);
-    return NULL;
-}
-
-static void user_delete (userdb_ctx_t *ctx, uint32_t userid)
-{
-    char key[16];
-
-    snprintf (key, sizeof (key), "%" PRIu32, userid);
-    zhash_delete (ctx->db, key);
-}
-
-static void lookup (flux_t *h, flux_msg_handler_t *mh,
-                    const flux_msg_t *msg, void *arg)
-{
-    userdb_ctx_t *ctx = arg;
-    uint32_t userid;
-    struct user *up;
-
-    if (flux_request_unpack (msg, NULL, "{s:i}", "userid", &userid) < 0)
-        goto error;
-    if (!(up = user_lookup (ctx, userid))) {
-        if (ctx->default_rolemask != FLUX_ROLE_NONE) {
-            if (!(up = user_add (ctx, userid, ctx->default_rolemask)))
-                goto error;
-        } else
-            goto error;
-    }
-    if (flux_respond_pack (h, msg, "{s:i s:i}", "userid", up->userid,
-                                                "rolemask", up->rolemask) < 0)
-        flux_log_error (h, "%s", __FUNCTION__);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s", __FUNCTION__);
-}
-
-static void addrole (flux_t *h, flux_msg_handler_t *mh,
-                     const flux_msg_t *msg, void *arg)
-{
-    userdb_ctx_t *ctx = arg;
-    uint32_t userid, rolemask;
-    struct user *up;
-
-    if (flux_request_unpack (msg, NULL, "{s:i s:i}",
-                             "userid", &userid,
-                             "rolemask", &rolemask) < 0)
-        goto error;
-    if (!(up = user_lookup (ctx, userid))) {
-        if (rolemask == FLUX_ROLE_NONE)
-            rolemask = ctx->default_rolemask;
-        if (rolemask == FLUX_ROLE_NONE) {
-            errno = EINVAL;
-            goto error;
-        }
-        if (!(up = user_add (ctx, userid, rolemask)))
-            goto error;
-    } else
-        up->rolemask |= rolemask;
-    if (flux_respond_pack (h, msg, "{s:i s:i}", "userid", up->userid,
-                                                "rolemask", up->rolemask) < 0)
-        flux_log_error (h, "%s", __FUNCTION__);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s", __FUNCTION__);
-}
-
-static void delrole (flux_t *h, flux_msg_handler_t *mh,
-                     const flux_msg_t *msg, void *arg)
-{
-    userdb_ctx_t *ctx = arg;
-    uint32_t userid, rolemask;
-    struct user *up;
-
-    if (flux_request_unpack (msg, NULL, "{s:i s:i}",
-                             "userid", &userid,
-                             "rolemask", &rolemask) < 0)
-        goto error;
-    if (!(up = user_lookup (ctx, userid)))
-        goto error;
-    up->rolemask &= ~rolemask;
-    if (flux_respond_pack (h, msg, "{s:i s:i}", "userid", up->userid,
-                                                "rolemask", up->rolemask) < 0)
-        flux_log_error (h, "%s", __FUNCTION__);
-    if (up->rolemask == FLUX_ROLE_NONE)
-        user_delete (ctx, userid);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s", __FUNCTION__);
-}
-
-static int compare_keys (const char *s1, const char *s2)
-{
-    uint32_t u1 = strtoul (s1, NULL, 10);
-    uint32_t u2 = strtoul (s2, NULL, 10);
-    if (u1 < u2)
-        return -1;
-    if (u1 > u2)
-        return 1;
-    return 0;
-}
-
-static void getnext (flux_t *h, flux_msg_handler_t *mh,
-                     const flux_msg_t *msg, void *arg)
-{
-    userdb_ctx_t *ctx = arg;
-    char *key;
-    struct user *up = NULL;
-    char *uuid = NULL;
-    zlist_t *itr;
-
-    if (flux_msg_get_route_first (msg, &uuid) < 0)
-        goto error;
-    if (!(itr = zhash_lookup (ctx->iterators, uuid))) {
-        if (!(itr = zhash_keys (ctx->db))) {
-            errno = ENOMEM;
-            goto error;
-        }
-        zlist_sort (itr, (zlist_compare_fn *)compare_keys);
-        zhash_update (ctx->iterators, uuid, itr);
-        zhash_freefn (ctx->iterators, uuid, (zhash_free_fn *)freelist);
-        key = zlist_first (itr);
-    } else {
-        key = zlist_next (itr);
-    }
-    if (!key || !(up = zhash_lookup (ctx->db, key))) {
-        zhash_delete (ctx->iterators, uuid);
-        errno = ENOENT;
-        goto error;
-    }
-
-    if (flux_respond_pack (h, msg, "{s:i s:i}", "userid", up->userid,
-                                                "rolemask", up->rolemask) < 0)
-        flux_log_error (h, "%s", __FUNCTION__);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s", __FUNCTION__);
-    free (uuid);
-}
-
-static void disconnect (flux_t *h, flux_msg_handler_t *mh,
-                        const flux_msg_t *msg, void *arg)
-{
-    userdb_ctx_t *ctx = arg;
-    char *uuid;
-    if (flux_msg_get_route_first (msg, &uuid) == 0) {
-        zhash_delete (ctx->iterators, uuid);
-        free (uuid);
-    }
-}
-
-static const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_REQUEST,  "userdb.lookup", lookup, 0 },
-    { FLUX_MSGTYPE_REQUEST,  "userdb.addrole", addrole, 0 },
-    { FLUX_MSGTYPE_REQUEST,  "userdb.delrole", delrole, 0 },
-    { FLUX_MSGTYPE_REQUEST,  "userdb.getnext", getnext, 0 },
-    { FLUX_MSGTYPE_REQUEST,  "userdb.disconnect", disconnect, 0 },
-    FLUX_MSGHANDLER_TABLE_END,
-};
-
-int mod_main (flux_t *h, int argc, char **argv)
-{
-    int rc = -1;
-    userdb_ctx_t *ctx;
-    flux_msg_handler_t **handlers = NULL;
-    struct user *up;
-
-    if (!(ctx = getctx (h, argc, argv))) {
-        goto done;
-    }
-    if (!(up = user_add (ctx, geteuid (), FLUX_ROLE_OWNER))) {
-        flux_log_error (h, "failed to add owner to userdb");
-        goto done;
-    }
-    if (flux_msg_handler_addvec (h, htab, ctx, &handlers) < 0) {
-        flux_log_error (h, "flux_msghandler_add");
-        goto done;
-    }
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
-        flux_log_error (h, "flux_reactor_run");
-        goto done;
-    }
-    rc = 0;
-done:
-    flux_msg_handler_delvec (handlers);
-    return rc;
-}
-
-MOD_NAME ("userdb");
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/shell/Makefile.am b/src/shell/Makefile.am
index 2ddcb25a9189..57c51d26cbd6 100644
--- a/src/shell/Makefile.am
+++ b/src/shell/Makefile.am
@@ -5,26 +5,24 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
+	$(CODE_COVERAGE_CPPFLAGS) \
 	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS) $(VALGRIND_CFLAGS) \
+	$(VALGRIND_CFLAGS) \
 	$(LUA_INCLUDE) \
-	$(HWLOC_INCLUDE)
+	$(HWLOC_CFLAGS) \
+	$(JANSSON_CFLAGS) \
+	$(LIBARCHIVE_CFLAGS)
 
 shellrcdir = \
-	$(fluxrcdir)/shell
-shellluadir = \
-	$(shellrcdir)/lua.d
+	$(fluxconfdir)/shell
 
-dist_shellrc_SCRIPTS = \
-	initrc.lua
-
-dist_shelllua_SCRIPTS = \
-	lua.d/mvapich.lua \
-	lua.d/intel_mpi.lua \
+nobase_dist_shellrc_SCRIPTS = \
+	initrc.lua \
 	lua.d/openmpi.lua \
-	lua.d/spectrum.lua
+	lua.d/mpi/spectrum.lua
 
 noinst_LTLIBRARIES = \
 	libshell.la \
@@ -35,8 +33,6 @@ libshell_la_SOURCES = \
 	plugstack.h \
 	jobspec.c \
 	jobspec.h \
-	eventlogger.c \
-	eventlogger.h \
 	rcalc.c \
 	rcalc.h
 
@@ -70,40 +66,77 @@ flux_shell_SOURCES = \
 	log.h \
 	events.c \
 	events.h \
-	pmi.c \
-	input.c \
+	pmi/pmi.c \
+	pmi/pmi_exchange.c \
+	pmi/pmi_exchange.h \
+	input/util.h \
+	input/util.c \
+	input/service.c \
+	input/file.c \
+	input/kvs.c \
 	output.c \
 	svc.c \
 	svc.h \
 	kill.c \
 	signals.c \
 	affinity.c \
+	affinity.h \
 	gpubind.c \
 	evlog.c \
+	pty.c \
+	batch.c \
+	tmpdir.c \
+	stage-in.c \
 	mpir/mpir.c \
-	mpir/ptrace.c
+	mpir/ptrace.c \
+	mustache.h \
+	mustache.c \
+	doom.c \
+	exception.c \
+	rlimit.c \
+	taskmap/cyclic.c \
+	taskmap/hostfile.c \
+	signal.c \
+	files.c \
+	hwloc.c \
+	rexec.c
+
+if HAVE_INOTIFY
+flux_shell_SOURCES += oom.c
+endif
 
 flux_shell_LDADD = \
 	$(builddir)/libshell.la \
 	$(builddir)/libmpir.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
+	$(top_builddir)/src/common/librlist/librlist-hwloc.la \
+	$(top_builddir)/src/common/librlist/librlist.la \
 	$(top_builddir)/src/bindings/lua/libfluxlua.la \
 	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-taskmap.la \
+	$(top_builddir)/src/common/libflux-idset.la \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
 	$(top_builddir)/src/common/libpmi/libpmi_server.la \
-	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libpmi/libpmi_common.la \
+	$(top_builddir)/src/common/libczmqcontainers/libczmqcontainers.la \
 	$(top_builddir)/src/common/libflux-optparse.la \
+	$(top_builddir)/src/common/libterminus/libterminus.la \
+	$(top_builddir)/src/common/libutil/libutil.la \
+	$(top_builddir)/src/common/libfilemap/libfilemap.la \
+	$(top_builddir)/src/common/libflux-internal.la \
 	$(LUA_LIB) \
-	$(HWLOC_LIBS)
+	$(HWLOC_LIBS) \
+	$(JANSSON_LIBS) \
+	$(LIBARCHIVE_LIBS)
 
 flux_shell_LDFLAGS = \
 	-export-dynamic \
-	-Wl,--version-script=$(srcdir)/flux-shell.map
-
-EXTRA_DIST = \
-	flux-shell.map
+	-export-symbols-regex "(flux_shell_.*|flux_plugin_get_shell|__asan.*)"
 
 TESTS = \
 	test_jobspec.t \
 	test_plugstack.t \
+	test_mustache.t \
 	mpir/test_rangelist.t \
 	mpir/test_nodelist.t \
 	mpir/test_proctable.t
@@ -114,6 +147,9 @@ test_ldadd = \
 	$(top_builddir)/src/common/libflux-internal.la \
 	$(top_builddir)/src/common/libtap/libtap.la
 
+test_ldflags = \
+	-no-install
+
 test_cppflags = \
         -I$(top_srcdir)/src/common/libtap \
         $(AM_CPPFLAGS)
@@ -135,39 +171,89 @@ test_jobspec_t_CPPFLAGS = $(test_cppflags)
 test_jobspec_t_LDADD = \
 	$(builddir)/libshell.la \
 	$(test_ldadd)
+test_jobspec_t_LDFLAGS = \
+	$(test_ldflags)
 
 test_plugstack_t_SOURCES = plugstack.c test/plugstack.c
 test_plugstack_t_CPPFLAGS = $(test_cppflags) -DPLUGSTACK_STANDALONE
 test_plugstack_t_LDADD = \
 	$(builddir)/libshell.la \
 	$(test_ldadd)
+test_plugstack_t_LDFLAGS = \
+	$(test_ldflags)
 
 test_a_plugin_la_SOURCES = test/plugin_test.c
 test_a_plugin_la_CPPFLAGS = $(test_cppflags) -DTEST_PLUGIN_RESULT=\"A\"
-test_a_plugin_la_LDFLAGS = -module -rpath /nowhere
+test_a_plugin_la_LDFLAGS = -module -rpath /nowhere $(test_ldflags)
 
 test_b_plugin_la_SOURCES = test/plugin_test.c
 test_b_plugin_la_CPPFLAGS = $(test_cppflags) -DTEST_PLUGIN_RESULT=\"B\"
-test_b_plugin_la_LDFLAGS = -module -rpath /nowhere
+test_b_plugin_la_LDFLAGS = -module -rpath /nowhere $(test_ldflags)
 
 test_c_plugin_la_SOURCES = test/plugin_test.c
 test_c_plugin_la_CPPFLAGS = $(test_cppflags) -DTEST_PLUGIN_RESULT=\"C\"
-test_c_plugin_la_LDFLAGS = -module -rpath /nowhere
+test_c_plugin_la_LDFLAGS = -module -rpath /nowhere $(test_ldflags)
 
 mpir_test_rangelist_t_SOURCES = mpir/test/rangelist.c
 mpir_test_rangelist_t_CPPFLAGS = $(test_cppflags)
 mpir_test_rangelist_t_LDADD = \
 	$(builddir)/libmpir.la \
 	$(test_ldadd)
+mpir_test_rangelist_t_LDFLAGS = \
+	$(test_ldflags)
 
 mpir_test_nodelist_t_SOURCES = mpir/test/nodelist.c
 mpir_test_nodelist_t_CPPFLAGS = $(test_cppflags)
 mpir_test_nodelist_t_LDADD = \
 	$(builddir)/libmpir.la \
 	$(test_ldadd)
+mpir_test_nodelist_t_LDFLAGS = \
+	$(test_ldflags)
 
 mpir_test_proctable_t_SOURCES = mpir/test/proctable.c
 mpir_test_proctable_t_CPPFLAGS = $(test_cppflags)
 mpir_test_proctable_t_LDADD = \
 	$(builddir)/libmpir.la \
 	$(test_ldadd)
+mpir_test_proctable_t_LDFLAGS = \
+	$(test_ldflags)
+
+test_mustache_t_SOURCES = \
+	mustache.c \
+	mustache.h \
+	test/mustache.c
+test_mustache_t_CPPFLAGS = \
+	$(test_cppflags)
+test_mustache_t_LDADD = \
+	$(builddir)/libshell.la \
+	$(test_ldadd)
+test_mustache_t_LDFLAGS = \
+	$(test_ldflags)
+
+
+.PHONY: link-shell-plugins clean-shell-plugins
+
+link-shell-plugins: $(shell_plugin_LTLIBRARIES)
+	@for f in $^; do \
+	 soname=`$(GREP) "^dlname=" $$f | $(SED) -e "s|^dlname='\(.*\)'|\1|"`; \
+	 dirname=`dirname $(abs_builddir)/$$f `; \
+	 target=$$dirname/.libs/$$soname; link=$$dirname/$$soname; \
+	 shortdir=`echo $$f | $(SED) -e 's|[^/]*.la||'`; \
+	 shorttarget="$${shortdir}.libs/$$soname"; \
+	 echo "  LN       $$shortdir$$soname -> $$shorttarget"; \
+	 rm -f  $$link; \
+	 $(LN_S) $$target $$link; \
+	done
+
+clean-shell-plugin-links:
+	@for f in $(shell_plugin_LTLIBRARIES); do \
+	 soname=`$(GREP) "^dlname=" $$f | $(SED) -e "s|^dlname='\(.*\)'|\1|"`; \
+	 dirname=`echo $(abs_builddir)/$$f | $(SED) -e 's|/[^/]*.la||'`; \
+	 target=$$dirname/.libs/$$soname; link=$$dirname/$$soname; \
+	 echo "  RM       $$link"; \
+	 rm -f $$link; \
+	done
+
+all-local:: link-shell-plugins
+
+clean-local:: clean-shell-plugin-links
diff --git a/src/shell/affinity.c b/src/shell/affinity.c
index 9f3b86f66668..dc6a074fa53a 100644
--- a/src/shell/affinity.c
+++ b/src/shell/affinity.c
@@ -10,6 +10,8 @@
 
 /* builtin cpu-affinity processing
  */
+#define FLUX_SHELL_PLUGIN_NAME "cpu-affinity"
+
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
@@ -18,6 +20,8 @@
 #include <flux/core.h>
 #include <flux/shell.h>
 
+#include "ccan/str/str.h"
+
 #include "builtins.h"
 
 struct shell_affinity {
@@ -28,32 +32,108 @@ struct shell_affinity {
     hwloc_cpuset_t *pertask;
 };
 
+void cpuset_array_destroy (hwloc_cpuset_t *set, int size)
+{
+    if (set) {
+        for (int i = 0; i < size; i++) {
+            if (set[i] != NULL)
+                hwloc_bitmap_free (set[i]);
+        }
+        free (set);
+    }
+}
+
+hwloc_cpuset_t *cpuset_array_create (int size)
+{
+    hwloc_cpuset_t *set = calloc (size, sizeof (hwloc_cpuset_t));
+    if (!set)
+        return NULL;
+    for (int i = 0; i < size; i++)
+        if (!(set[i] = hwloc_bitmap_alloc ()))
+            goto error;
+    return set;
+error:
+    cpuset_array_destroy (set, size);
+    return NULL;
+}
+
 /*  Run hwloc_topology_restrict() with common flags for this module.
  */
 static int topology_restrict (hwloc_topology_t topo, hwloc_cpuset_t set)
 {
-    int flags = HWLOC_RESTRICT_FLAG_ADAPT_DISTANCES |
-                HWLOC_RESTRICT_FLAG_ADAPT_MISC |
-                HWLOC_RESTRICT_FLAG_ADAPT_IO;
-    flags = 0;
-    if (hwloc_topology_restrict (topo, set, flags) < 0)
+    if (hwloc_topology_restrict (topo, set, 0) < 0)
         return (-1);
     return (0);
 }
 
-/*  Restrict hwloc topology object to current processes binding.
+
+/*  Parse a list of hwloc bitmap strings in list, bitmask, or taskset
+ *  form and return an allocated hwloc_cpuset_t array of size ntasks,
+ *  filled with the resulting bitmasks. If ntasks is greater than the number
+ *  of provided cpusets, then cpusets are reused as necessary.
  */
-static int topology_restrict_current (hwloc_topology_t topo)
+hwloc_cpuset_t *parse_cpuset_list (const char *setlist,
+                                   int ntasks)
 {
-    int rc = -1;
-    hwloc_bitmap_t rset = hwloc_bitmap_alloc ();
-    if (!rset || hwloc_get_cpubind (topo, rset, HWLOC_CPUBIND_PROCESS) < 0)
-        goto out;
-    rc = topology_restrict (topo, rset);
-out:
-    if (rset)
-        hwloc_bitmap_free (rset);
-    return (rc);
+    char *copy = NULL;
+    char *s, *arg, *sptr = NULL;
+    int index, i = 0;
+    hwloc_cpuset_t *cpusets = NULL;
+
+    if (!(cpusets = cpuset_array_create (ntasks))
+        || !(copy = strdup (setlist))) {
+        shell_log_errno ("out of memory");
+        goto err;
+    }
+
+    s = copy;
+    while ((arg = strtok_r (s, ";", &sptr)) && i < ntasks) {
+        int rc;
+        if (strstarts (arg, "0x")) {
+            /*  If string starts with 0x then parse as a bitmask. If a
+             *  comma is present in the string, then this is likely a
+             *  hwloc-style bitmap string, otherwise, try the taskset
+             *  style bitmaps, which are simpler.
+             */
+            if (strchr (arg, ','))
+                rc = hwloc_bitmap_sscanf (cpusets[i], arg);
+            else
+                rc = hwloc_bitmap_taskset_sscanf (cpusets[i], arg);
+        }
+        else {
+            /*  O/w, attempt parse string as a hwloc list-style bitmap:
+             */
+            rc = hwloc_bitmap_list_sscanf (cpusets[i], arg);
+        }
+
+        if (rc < 0 || hwloc_bitmap_weight (cpusets[i]) <= 0) {
+            shell_log_error ("cpuset %s contains no cores or is invalid",
+                             arg);
+            goto err;
+        }
+        s = NULL;
+        i++;
+    }
+    if (i == 0) {
+        shell_log_error ("no cpusets found in affinity list %s", setlist);
+        goto err;
+    }
+
+    /*  If not all tasks were assigned cpusets, then continue, reusing
+     *  cpusets as necessary.
+     */
+    index = 0;
+    for (; i < ntasks; i++) {
+        (void) hwloc_bitmap_copy (cpusets[i], cpusets[index]);
+        if (++index > ntasks)
+            index = 0;
+    }
+    free (copy);
+    return cpusets;
+err:
+    cpuset_array_destroy (cpusets, ntasks);
+    free (copy);
+    return NULL;
 }
 
 /*  Distribute ntasks over the topology 'topo', restricted to the
@@ -65,22 +145,39 @@ static hwloc_cpuset_t *distribute_tasks (hwloc_topology_t topo,
                                          hwloc_cpuset_t cset,
                                          int ntasks)
 {
-    hwloc_obj_t obj[1];
+    hwloc_obj_t *roots;
     hwloc_cpuset_t *cpusetp = NULL;
+    int cores;
+    int depth;
 
     /* restrict topology to current cpuset */
-    if (cset && topology_restrict (topo, cset) < 0)
+    if (cset && topology_restrict (topo, cset) < 0) {
+        shell_log_errno ("topology_restrict failed");
         return NULL;
+    }
     /* create cpuset array for ntasks */
     if (!(cpusetp = calloc (ntasks, sizeof (hwloc_cpuset_t))))
         return NULL;
-    /* Distribute starting at root over remaining objects */
-    obj[0] = hwloc_get_root_obj (topo);
+
+    depth = hwloc_get_type_depth (topo, HWLOC_OBJ_CORE);
+    cores = hwloc_get_nbobjs_by_depth (topo, depth);
+    if (cores <= 0 || !(roots = calloc (cores, sizeof (*roots)))) {
+        shell_log_error ("failed to allocate %d roots for hwloc distrib",
+                         cores);
+        return NULL;
+    }
+
+    for (int i = 0; i < cores; i++)
+        roots[i] = hwloc_get_obj_by_depth (topo, depth, i);
+
+    shell_trace ("distributing %d tasks across %d cores", ntasks, cores);
 
     /* NB: hwloc_distrib() will alloc ntasks cpusets in cpusetp, which
      *     later need to be destroyed with hwloc_bitmap_free().
      */
-    hwloc_distrib (topo, obj, 1, cpusetp, ntasks, HWLOC_OBJ_PU, 0);
+    hwloc_distrib (topo, roots, cores, cpusetp, ntasks, depth, 0);
+
+    free (roots);
     return (cpusetp);
 }
 
@@ -124,6 +221,10 @@ static hwloc_cpuset_t shell_affinity_get_cpuset (struct shell_affinity *sa,
             shell_log_error ("affinity: core%d not in topology", i);
             goto err;
         }
+        if (!core->cpuset) {
+            shell_log_error ("affinity: core%d cpuset is null", i);
+            goto err;
+        }
         hwloc_bitmap_or (resultset, resultset, core->cpuset);
         i = hwloc_bitmap_next (coreset, i);
     }
@@ -144,26 +245,38 @@ static void shell_affinity_destroy (void *arg)
         hwloc_topology_destroy (sa->topo);
     if (sa->cpuset)
         hwloc_bitmap_free (sa->cpuset);
-    if (sa->pertask) {
-        for (int i = 0; i < sa->ntasks; i++) {
-            if (sa->pertask[i] != NULL)
-                hwloc_bitmap_free (sa->pertask[i]);
-        }
-        free (sa->pertask);
-    }
+    cpuset_array_destroy (sa->pertask, sa->ntasks);
     free (sa);
 }
 
 /*  Initialize topology object for affinity processing.
  */
-static int shell_affinity_topology_init (struct shell_affinity *sa)
+static int shell_affinity_topology_init (flux_shell_t *shell,
+                                         struct shell_affinity *sa)
 {
+    const char *xml;
+
+    /*  Fetch hwloc XML cached in job shell to avoid heavyweight
+     *   hwloc topology load (Issue #4365)
+     */
+    if (flux_shell_get_hwloc_xml (shell, &xml) < 0)
+        return shell_log_errno ("failed to unpack hwloc object");
+
     if (hwloc_topology_init (&sa->topo) < 0)
         return shell_log_errno ("hwloc_topology_init");
+
+    if (hwloc_topology_set_xmlbuffer (sa->topo, xml, strlen (xml)) < 0)
+        return shell_log_errno ("hwloc_topology_set_xmlbuffer");
+
+    /*  Tell hwloc that our XML loaded topology is from this system,
+     *   O/w hwloc CPU binding will not work.
+     */
+    if (hwloc_topology_set_flags (sa->topo,
+                                  HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM) < 0)
+        return shell_log_errno ("hwloc_topology_set_flags");
+
     if (hwloc_topology_load (sa->topo) < 0)
         return shell_log_errno ("hwloc_topology_load");
-    if (topology_restrict_current (sa->topo) < 0)
-        return shell_log_errno ("topology_restrict_current");
     return 0;
 }
 
@@ -176,7 +289,7 @@ static struct shell_affinity * shell_affinity_create (flux_shell_t *shell)
     struct shell_affinity *sa = calloc (1, sizeof (*sa));
     if (!sa)
         return NULL;
-    if (shell_affinity_topology_init (sa) < 0)
+    if (shell_affinity_topology_init (shell, sa) < 0)
         goto err;
     if (flux_shell_rank_info_unpack (shell,
                                      -1,
@@ -210,7 +323,7 @@ static bool affinity_getopt (flux_shell_t *shell, const char **resultp)
         shell_warn ("cpu-affinity: invalid option: %s", *resultp);
         return true;
     }
-    else if (strcmp (*resultp, "off") == 0)
+    else if (streq (*resultp, "off"))
         return false;
     return true;
 }
@@ -240,10 +353,6 @@ static int get_taskid (flux_plugin_t *p)
     return flux_shell_task_getid (task);
 }
 
-#if CODE_COVERAGE_ENABLED
-void __gcov_flush (void);
-#endif
-
 static int task_affinity (flux_plugin_t *p,
                           const char *topic,
                           flux_plugin_arg_t *args,
@@ -254,9 +363,6 @@ static int task_affinity (flux_plugin_t *p,
     if (sa->pertask)
         hwloc_set_cpubind (sa->topo, sa->pertask[i], 0);
     shell_affinity_destroy (sa);
-#if CODE_COVERAGE_ENABLED
-    __gcov_flush ();
-#endif
     return 0;
 }
 
@@ -271,8 +377,10 @@ static int affinity_init (flux_plugin_t *p,
 
     if (!shell)
         return shell_log_errno ("flux_plugin_get_shell");
-    if (!affinity_getopt (shell, &option))
+    if (!affinity_getopt (shell, &option)) {
+        shell_debug ("disabling affinity due to cpu-affinity=off");
         return 0;
+    }
     if (!(sa = shell_affinity_create (shell)))
         return shell_log_errno ("shell_affinity_create");
 
@@ -297,22 +405,27 @@ static int affinity_init (flux_plugin_t *p,
      *   resources to which the shell is now bound (from above)
      *  Set a 'task.exec' callback to actually make the per-task binding.
      */
-    if (strcmp (option, "per-task") == 0) {
+    if (streq (option, "per-task")) {
         if (!(sa->pertask = distribute_tasks (sa->topo,
                                               sa->cpuset,
                                               sa->ntasks)))
             shell_log_errno ("distribute_tasks failed");
-        if (flux_plugin_add_handler (p, "task.exec",
-                                     task_affinity,
-                                     sa) < 0)
-            shell_log_errno ("failed to add task.exec handler");
     }
+    else if (strstarts (option, "map:")) {
+        if (!(sa->pertask = parse_cpuset_list (option+4, sa->ntasks)))
+            return -1;
+    }
+    if (sa->pertask
+        && flux_plugin_add_handler (p, "task.exec",
+                                    task_affinity,
+                                    sa) < 0)
+            shell_log_errno ("failed to add task.exec handler");
 
     return 0;
 }
 
 struct shell_builtin builtin_affinity = {
-    .name = "affinity",
+    .name = FLUX_SHELL_PLUGIN_NAME,
     .init = affinity_init,
 };
 
diff --git a/src/shell/affinity.h b/src/shell/affinity.h
new file mode 100644
index 000000000000..5b1b44ecc564
--- /dev/null
+++ b/src/shell/affinity.h
@@ -0,0 +1,37 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SHELL_AFFINITY_H
+#define _SHELL_AFFINITY_H
+
+#include <hwloc.h>
+
+/*  Parse a list of hwloc bitmap strings from `setlist` in list, bitmask,
+ *  or taskset form and return an allocated hwloc_cpuset_t array of size
+ *  `ntasks` filled with the resulting bitmasks. If `ntasks` is greater
+ *  than the number of provided cpusets, then cpusets are reused as
+ *  necessary.
+ */
+hwloc_cpuset_t *parse_cpuset_list (const char *setlist, int ntasks);
+
+/*  Create an empty hwloc_cpuset_t array of size elements
+ */
+hwloc_cpuset_t *cpuset_array_create (int size);
+
+/*  Free memory for hwloc_cpuset_t array returned from cpuset_array_create()
+ *  or parse_cpuset_list().
+ */
+void cpuset_array_destroy (hwloc_cpuset_t *set, int size);
+
+#endif /* !_SHELL_AFFINITY_H */
+
+/* vi: ts=4 sw=4 expandtab
+ */
+
diff --git a/src/shell/batch.c b/src/shell/batch.c
new file mode 100644
index 000000000000..7d496955b651
--- /dev/null
+++ b/src/shell/batch.c
@@ -0,0 +1,245 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* batch script handler
+ */
+#define FLUX_SHELL_PLUGIN_NAME "batch"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/shell.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/basename.h"
+#include "ccan/str/str.h"
+
+#include "builtins.h"
+
+struct batch_info {
+    flux_jobid_t id; /* This jobid */
+    int shell_rank;  /* This shell rank */
+    char *script;    /* Path to locally created job script */
+    json_t *options; /* Extra broker options */
+};
+
+static void batch_info_destroy (void *arg)
+{
+    struct batch_info *b = arg;
+    if (b) {
+        if (b->shell_rank == 0 && b->script)
+            unlink (b->script);
+        free (b->script);
+        free (b);
+    }
+}
+
+static struct batch_info *
+batch_info_create (flux_shell_t *shell, json_t *batch)
+{
+    struct batch_info *b = calloc (1, sizeof (*b));
+    json_error_t err;
+    const char *data;
+    size_t len;
+
+    if (flux_shell_info_unpack (shell,
+                               "{s:i s:I}",
+                               "rank", &b->shell_rank,
+                               "jobid", &b->id) < 0) {
+        shell_log_errno ("failed to unpack shell info");
+        goto error;
+    }
+
+    if (json_unpack_ex (batch, &err, 0,
+                        "{s?o s:s%}",
+                        "broker-opts", &b->options,
+                        "script", &data, &len) < 0) {
+        shell_log_error ("failed to unpack batch info: %s", err.text);
+        goto error;
+    }
+
+    if (b->options && !json_is_array (b->options)) {
+        shell_log_error ("batch.broker-opts attribute must be an array");
+        goto error;
+    }
+
+    if (data && b->shell_rank == 0) {
+        int fd = -1;
+        const char *tmpdir = flux_shell_getenv (shell, "FLUX_JOB_TMPDIR");
+
+        if (!tmpdir) {
+            shell_log_error ("FLUX_JOB_TMPDIR not set");
+            goto error;
+        }
+
+        if (asprintf (&b->script, "%s/script", tmpdir) < 0) {
+            shell_log_error ("asprintf script templated failed");
+            goto error;
+        }
+        if ((fd = open (b->script, O_CREAT|O_EXCL|O_WRONLY, 0700)) < 0) {
+            shell_log_errno ("%s", b->script);
+            goto error;
+        }
+        shell_debug ("Copying batch script size=%zu for job to %s",
+                     len,
+                     b->script);
+        if (write_all (fd, data, len) < 0 || close (fd) < 0) {
+            shell_log_error ("failed to write batch script");
+            goto error;
+        }
+    }
+    return b;
+error:
+    batch_info_destroy (b);
+    return NULL;
+}
+
+static int cmd_append_broker_options (flux_cmd_t *cmd,
+                                      struct batch_info *b)
+{
+    int index;
+    if (b->options == 0)
+        return 0;
+    index = json_array_size (b->options) - 1;
+    while (index >= 0) {
+        json_t *val = json_array_get (b->options, index);
+        const char *s = json_string_value (val);
+        if (s && flux_cmd_argv_insert (cmd, 0, s) < 0)
+            return shell_log_errno ("Failed to prepend broker opt %s", s);
+        index--;
+    }
+    return 0;
+}
+
+static void log_task_commandline (flux_cmd_t *cmd, int taskid)
+{
+    char *s = flux_cmd_stringify (cmd);
+    shell_debug ("task%d: re-writing command to %s", taskid, s);
+    free (s);
+}
+
+static bool is_batch_command (flux_cmd_t *cmd)
+{
+    const char *argv0 = flux_cmd_arg (cmd, 0);
+    const char *argv1 = flux_cmd_arg (cmd, 1);
+
+    return  (argv0
+             && streq (basename_simple (argv0), "flux")
+             && (streq (argv1, "broker") || streq (argv1, "start")));
+}
+
+static int task_batchify (flux_plugin_t *p,
+                          const char *topic,
+                          flux_plugin_arg_t *args,
+                          void *data)
+{
+    struct batch_info *b = data;
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    flux_shell_task_t *task = NULL;
+    flux_cmd_t *cmd = NULL;
+    int taskid = -1;
+
+    if (!shell
+        || !(task = flux_shell_current_task (shell))
+        || !(cmd = flux_shell_task_cmd (task)))
+        return shell_log_errno ("failed to get task cmd");
+
+    /* Nothing to do if task is already constructed as a batch command
+     */
+    if (is_batch_command (cmd))
+        return 0;
+
+    if (flux_shell_task_info_unpack (task, "{s:i}", "rank", &taskid) < 0)
+        return shell_log_errno ("failed to unpack task rank");
+
+    if (taskid == 0) {
+        /* For rank 0 broker, delete argv0 and replace
+         *  with path to our script
+         */
+        if (flux_cmd_argv_delete (cmd, 0) < 0
+            || flux_cmd_argv_insert (cmd, 0, b->script) < 0) {
+            return shell_log_errno ("failed to replace command"
+                                    " with batch script");
+        }
+    }
+    else {
+        /* Other ranks, delete all args, they are unused */
+        while (flux_cmd_argv_delete (cmd, 0) == 0)
+           ;
+    }
+
+    /*  All broker ranks, add broker options */
+    if (cmd_append_broker_options (cmd, b) < 0)
+        return -1;
+
+    /*  All broker ranks: prepend 'flux broker' */
+    if (flux_cmd_argv_insert (cmd, 0, "broker") < 0
+        || flux_cmd_argv_insert (cmd, 0, "flux") < 0)
+        return shell_log_errno ("failed to prepend command with flux broker");
+
+    log_task_commandline (cmd, taskid);
+
+    return 0;
+}
+
+
+static int batch_init (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    json_t *jobspec = NULL;
+    json_t *batch = NULL;
+
+    if (flux_shell_info_unpack (shell,
+                               "{s:o}",
+                               "jobspec", &jobspec) < 0)
+        return shell_log_errno ("failed to unpack jobspec");
+    json_error_t err;
+    if (json_unpack_ex (jobspec, &err, 0,
+                        "{s:{s?{s?o}}}",
+                        "attributes",
+                          "system",
+                            "batch", &batch) < 0) {
+        shell_log_error ("failed to unpack batch object from jobspec: %s",
+                        err.text);
+        return -1;
+    }
+
+    if (batch) {
+        struct batch_info *b = batch_info_create (shell, batch);
+
+        if (!b || flux_plugin_aux_set (p, "batch", b, batch_info_destroy) < 0) {
+            batch_info_destroy (b);
+            return -1;
+        }
+        if (flux_plugin_add_handler (p, "task.init", task_batchify, b) < 0)
+            return shell_log_errno ("failed to add task.init handler");
+    }
+    return 0;
+}
+
+struct shell_builtin builtin_batch = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = batch_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/builtins.c b/src/shell/builtins.c
index f31c49dbf4f4..38baa5ecb969 100644
--- a/src/shell/builtins.c
+++ b/src/shell/builtins.c
@@ -9,6 +9,7 @@
 \************************************************************/
 
 /* job shell builtin plugin loader */
+#define FLUX_SHELL_PLUGIN_NAME NULL
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -29,9 +30,14 @@ static struct shell_builtin builtin_list_end = { 0 };
  *  Then, the name should be added to the 'builtins' list
  *  to get the builtin automatically loaded at shell startup.
  */
+extern struct shell_builtin builtin_tmpdir;
+extern struct shell_builtin builtin_files;
+extern struct shell_builtin builtin_stage_in;
 extern struct shell_builtin builtin_log_eventlog;
 extern struct shell_builtin builtin_pmi;
-extern struct shell_builtin builtin_input;
+extern struct shell_builtin builtin_input_service;
+extern struct shell_builtin builtin_file_input;
+extern struct shell_builtin builtin_kvs_input;
 extern struct shell_builtin builtin_output;
 extern struct shell_builtin builtin_kill;
 extern struct shell_builtin builtin_signals;
@@ -39,11 +45,27 @@ extern struct shell_builtin builtin_affinity;
 extern struct shell_builtin builtin_gpubind;
 extern struct shell_builtin builtin_mpir;
 extern struct shell_builtin builtin_ptrace;
+extern struct shell_builtin builtin_pty;
+extern struct shell_builtin builtin_batch;
+extern struct shell_builtin builtin_doom;
+extern struct shell_builtin builtin_exception;
+extern struct shell_builtin builtin_rlimit;
+extern struct shell_builtin builtin_cyclic;
+extern struct shell_builtin builtin_hostfile;
+extern struct shell_builtin builtin_signal;
+extern struct shell_builtin builtin_oom;
+extern struct shell_builtin builtin_hwloc;
+extern struct shell_builtin builtin_rexec;
 
 static struct shell_builtin * builtins [] = {
+    &builtin_tmpdir,
+    &builtin_files,
+    &builtin_stage_in,
     &builtin_log_eventlog,
     &builtin_pmi,
-    &builtin_input,
+    &builtin_input_service,
+    &builtin_file_input,
+    &builtin_kvs_input,
     &builtin_output,
     &builtin_kill,
     &builtin_signals,
@@ -51,6 +73,19 @@ static struct shell_builtin * builtins [] = {
     &builtin_gpubind,
     &builtin_mpir,
     &builtin_ptrace,
+    &builtin_pty,
+    &builtin_batch,
+    &builtin_doom,
+    &builtin_exception,
+    &builtin_rlimit,
+    &builtin_cyclic,
+    &builtin_hostfile,
+    &builtin_signal,
+#if HAVE_INOTIFY_INIT1
+    &builtin_oom,
+#endif
+    &builtin_hwloc,
+    &builtin_rexec,
     &builtin_list_end,
 };
 
@@ -65,8 +100,15 @@ static int shell_load_builtin (flux_shell_t *shell,
         || flux_plugin_set_name (p, sb->name) < 0
         || flux_plugin_add_handler (p, "shell.validate", sb->validate, NULL) < 0
         || flux_plugin_add_handler (p, "shell.connect",  sb->connect, NULL) < 0
+        || flux_plugin_add_handler (p, "shell.reconnect",
+                                    sb->reconnect, NULL) < 0
         || flux_plugin_add_handler (p, "shell.init", sb->init, NULL) < 0
+        || flux_plugin_add_handler (p,
+                                    "shell.post-init",
+                                    sb->post_init,
+                                    NULL) < 0
         || flux_plugin_add_handler (p, "shell.exit", sb->exit, NULL) < 0
+        || flux_plugin_add_handler (p, "shell.start",sb->start, NULL) < 0
         || flux_plugin_add_handler (p, "task.init",  sb->task_init, NULL) < 0
         || flux_plugin_add_handler (p, "task.fork",  sb->task_fork, NULL) < 0
         || flux_plugin_add_handler (p, "task.exec",  sb->task_exec, NULL) < 0
@@ -74,6 +116,8 @@ static int shell_load_builtin (flux_shell_t *shell,
         return -1;
 
     shell_debug ("loading builtin plugin \"%s\"", sb->name);
+    if (sb->plugin_init && (*sb->plugin_init) (p) < 0)
+        return -1;
     if (plugstack_push (shell->plugstack, p) < 0)
         return -1;
     return (0);
diff --git a/src/shell/builtins.h b/src/shell/builtins.h
index 8bd0284c81cb..1f5b653582da 100644
--- a/src/shell/builtins.h
+++ b/src/shell/builtins.h
@@ -16,12 +16,16 @@
 
 struct shell_builtin {
     const char *name;
+    int (*plugin_init) (flux_plugin_t *p);
     flux_plugin_f validate;
     flux_plugin_f connect;
+    flux_plugin_f reconnect;
     flux_plugin_f init;
+    flux_plugin_f post_init;
     flux_plugin_f task_init;
     flux_plugin_f task_exec;
     flux_plugin_f task_fork;
+    flux_plugin_f start;
     flux_plugin_f task_exit;
     flux_plugin_f exit;
 };
diff --git a/src/shell/doom.c b/src/shell/doom.c
new file mode 100644
index 000000000000..acec67abfbd1
--- /dev/null
+++ b/src/shell/doom.c
@@ -0,0 +1,373 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* doom.c - log first task exit
+ *
+ * Each shell sends a message to shell-0 when its first task exits.
+ * Shell-0 posts an event to the exec eventlog for the first one received.
+ *
+ * Shell-0 sets a timer and posts a fatal exception when the timer fires.
+ *
+ * Shell options to modify the default behavior:
+ *
+ * exit-timeout
+ *   Change the timeout value (FSD), or disable the timer with value "none".
+ *
+ * exit-on-error
+ *   Raise the fatal exception immediately if the first task fails,
+ *   e.g. calls exit with a nonzero value or is terminated by signal.
+ */
+#define FLUX_SHELL_PLUGIN_NAME "doom"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <flux/taskmap.h>
+#include <flux/hostlist.h>
+
+#include <jansson.h>
+#include <assert.h>
+#include <libgen.h>
+
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/basename.h"
+
+#include "builtins.h"
+#include "internal.h"
+#include "task.h"
+
+#define TIMEOUT_NONE (-1.)
+
+static const double default_timeout = 30.;
+
+struct shell_doom {
+    flux_shell_t *shell;
+    const struct taskmap *map;
+    struct hostlist *hl;
+    bool done; // event already posted (shell rank 0) or message sent (> 0)
+    flux_watcher_t *timer;
+    double timeout;
+    bool exit_on_error;
+    int exit_rc;
+    int exit_rank;
+    bool lost_shell;
+};
+
+static int get_exit_code (json_t *task_info)
+{
+    int status;
+
+    if (json_unpack (task_info, "{s:i}", "wait_status", &status) < 0)
+        goto error;
+    if (WIFEXITED (status))
+        return WEXITSTATUS (status);
+    if (WIFSIGNALED (status))
+        return 128 + WTERMSIG (status);
+error:
+    shell_log_error ("error decoding task wait status");
+    return 1;
+}
+
+static int get_exit_rank (json_t *task_info)
+{
+    int rank = -1;
+
+    if (json_unpack (task_info, "{s:i}", "rank", &rank) < 0)
+        shell_log_error ("error decoding task rank");
+    return rank;
+}
+
+static const char *doom_exit_host (struct shell_doom *doom)
+{
+    int nth;
+    if (!doom->map || !doom->hl)
+        return "unknown";
+    if (doom->lost_shell)
+        nth = doom->exit_rank;
+    else
+        nth = taskmap_nodeid (doom->map, doom->exit_rank);
+    return hostlist_nth (doom->hl, nth);
+}
+
+static const char *get_jobspec_command_arg0 (struct shell_doom *doom)
+{
+    json_t *s = json_array_get (doom->shell->info->jobspec->command, 0);
+    return basename_simple (json_string_value (s));
+}
+
+static void doom_check (struct shell_doom *doom,
+                        int rank,
+                        int exitcode,
+                        bool lost_shell)
+{
+    doom->exit_rank = rank;
+    doom->exit_rc = exitcode;
+    doom->lost_shell = lost_shell;
+
+    /* Get copy of shell taskmap and hostlist to include hostnames in
+     * generated errors. Failures here are ignored and result in an
+     * "unknown" hostname generated in errors.
+     */
+    doom->map = flux_shell_get_taskmap (doom->shell);
+
+    /* Note: copy hostlist here because functions like hostlist_find(3)
+     * modify the hostlist cursor, therefore we need a non-const.
+     */
+    doom->hl = hostlist_copy (flux_shell_get_hostlist (doom->shell));
+
+    if (doom->exit_on_error && doom->exit_rc != 0) {
+        shell_die (doom->exit_rc,
+                   "%s: %srank %d on host %s failed and exit-on-error is set",
+                   get_jobspec_command_arg0 (doom),
+                   doom->lost_shell ? "shell " : "",
+                   doom->exit_rank,
+                   doom_exit_host (doom));
+    }
+    else if (doom->timeout != TIMEOUT_NONE)
+        flux_watcher_start (doom->timer);
+}
+
+static void doom_post (struct shell_doom *doom, json_t *task_info)
+{
+    flux_kvs_txn_t *txn;
+    json_t *entry = NULL;
+    char *entrystr = NULL;
+    flux_future_t *f = NULL;
+
+    assert (doom->shell->info->shell_rank == 0);
+
+    if (!(txn = flux_kvs_txn_create ())
+        || !(entry = eventlog_entry_pack (0.,
+                                          "shell.task-exit",
+                                          "O",
+                                          task_info))
+        || !(entrystr = eventlog_entry_encode (entry))
+        || flux_kvs_txn_put (txn,
+                             FLUX_KVS_APPEND,
+                             "exec.eventlog",
+                             entrystr) < 0
+        || !(f = flux_kvs_commit (doom->shell->h, NULL, 0, txn)))
+        shell_log_errno ("error posting task-exit eventlog entry");
+
+    doom_check (doom,
+                get_exit_rank (task_info),
+                get_exit_code (task_info),
+                false);
+
+    flux_future_destroy (f); // fire and forget
+    free (entrystr);
+    json_decref (entry);
+    flux_kvs_txn_destroy (txn);
+}
+
+static void doom_notify_cb (flux_t *h,
+                            flux_msg_handler_t *mh,
+                            const flux_msg_t *msg,
+                            void *arg)
+{
+    struct shell_doom *doom = arg;
+    json_t *task_info;
+
+    assert (doom->shell->info->shell_rank == 0);
+
+    if (doom->done)
+        return;
+    if (flux_request_unpack (msg, NULL, "o", &task_info) < 0) {
+        shell_log_errno ("error parsing first task exit notification");
+        return;
+    }
+    doom_post (doom, task_info);
+    doom->done = true;
+}
+
+static void doom_notify (struct shell_doom *doom, json_t *task_info)
+{
+    flux_future_t *f;
+
+    assert (doom->shell->info->shell_rank > 0);
+
+    if (!(f = flux_shell_rpc_pack (doom->shell,
+                                   "doom",
+                                   0,
+                                   FLUX_RPC_NORESPONSE,
+                                   "O",
+                                   task_info)))
+        shell_log_errno ("error notifying rank 0 of first task exit");
+    flux_future_destroy (f);
+}
+
+static void doom_timeout (flux_reactor_t *r,
+                          flux_watcher_t *w,
+                          int revents,
+                          void *arg)
+{
+    struct shell_doom *doom = arg;
+    char fsd[64];
+
+    fsd_format_duration (fsd, sizeof (fsd), doom->timeout);
+    shell_die (doom->exit_rc,
+               "%s: %srank %d on host %s exited and exit-timeout=%s has expired",
+               get_jobspec_command_arg0 (doom),
+               doom->lost_shell ? "shell " : "",
+               doom->exit_rank,
+               doom_exit_host (doom),
+               fsd);
+}
+
+static int doom_task_exit (flux_plugin_t *p,
+                           const char *topic,
+                           flux_plugin_arg_t *args,
+                           void *arg)
+{
+    flux_shell_t *shell;
+    struct shell_doom *doom;
+    flux_shell_task_t *task;
+
+    if (!(shell = flux_plugin_get_shell (p))
+        || !(doom = flux_plugin_aux_get (p, "doom"))
+        || !(task = flux_shell_current_task (shell)))
+        return -1;
+    if (!doom->done) {
+        json_t *task_info;
+
+        if (flux_shell_task_info_unpack (task, "o", &task_info) < 0)
+            return -1;
+        if (shell->info->shell_rank == 0)
+            doom_post (doom, task_info);
+        else
+            doom_notify (doom, task_info);
+        doom->done = true;
+    }
+    return 0;
+}
+
+static int doom_shell_lost (flux_plugin_t *p,
+                            const char *topic,
+                            flux_plugin_arg_t *args,
+                            void *arg)
+{
+    struct shell_doom *doom = arg;
+    int shell_rank;
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:i}",
+                                "shell_rank", &shell_rank) < 0)
+        return shell_log_errno ("shell.lost: unpack of shell_rank failed");
+    doom_check (doom, shell_rank, 1, true);
+    return 0;
+}
+
+static void doom_destroy (struct shell_doom *doom)
+{
+    if (doom) {
+        int saved_errno = errno;
+        flux_watcher_destroy (doom->timer);
+        hostlist_destroy (doom->hl);
+        free (doom);
+        errno = saved_errno;
+    }
+}
+
+static int parse_args (flux_shell_t *shell,
+                       double *timeout,
+                       bool *exit_on_error)
+{
+    json_t *val = NULL;
+
+    if (flux_shell_getopt_unpack (shell, "exit-timeout", "o", &val) < 0)
+        return -1;
+    if (val) {
+        if (json_is_string (val)) {
+            double n;
+            if (fsd_parse_duration (json_string_value (val), &n) < 0) {
+                if (!strcasecmp (json_string_value (val), "none"))
+                    n = TIMEOUT_NONE;
+                else
+                    goto error;
+            }
+            *timeout = n;
+        }
+        else if (json_is_number (val)) {
+            if (json_number_value (val) < 0)
+                goto error;
+            *timeout = json_number_value (val);
+        }
+        else
+            goto error;
+    }
+    *exit_on_error = (flux_shell_getopt (shell,
+                                         "exit-on-error",
+                                         NULL) == 1) ? true : false;
+    return 0;
+error:
+    shell_log_error ("exit-timeout is not a valid Flux Standard Duration");
+    return -1;
+}
+
+static struct shell_doom *doom_create (flux_shell_t *shell)
+{
+    struct shell_doom *doom;
+
+    if (!(doom = calloc (1, sizeof (*doom))))
+        return NULL;
+    doom->shell = shell;
+    doom->timeout = default_timeout;
+    if (parse_args (shell, &doom->timeout, &doom->exit_on_error) < 0)
+        goto error;
+    if (shell->info->shell_rank == 0) {
+        if (flux_shell_service_register (shell,
+                                         "doom",
+                                         doom_notify_cb,
+                                         doom) < 0)
+            goto error;
+        if (doom->timeout != TIMEOUT_NONE) {
+            if (!(doom->timer = flux_timer_watcher_create (shell->r,
+                                                           doom->timeout,
+                                                           0.,
+                                                           doom_timeout,
+                                                           doom)))
+                goto error;
+        }
+    }
+    return doom;
+error:
+    doom_destroy (doom);
+    return NULL;
+}
+
+static int doom_init (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *arg,
+                      void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    struct shell_doom *doom;
+    if (!shell || !(doom = doom_create (shell)))
+        return -1;
+    if (flux_plugin_aux_set (p, "doom", doom, (flux_free_f) doom_destroy) < 0) {
+        doom_destroy (doom);
+        return -1;
+    }
+    if (flux_plugin_add_handler (p, "shell.lost", doom_shell_lost, doom) < 0)
+        return shell_log_errno ("failed to add shell.lost handler");
+    return 0;
+}
+
+struct shell_builtin builtin_doom = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = doom_init,
+    .task_exit = doom_task_exit,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/eventlogger.c b/src/shell/eventlogger.c
deleted file mode 100644
index cfd6a2f95ab4..000000000000
--- a/src/shell/eventlogger.c
+++ /dev/null
@@ -1,291 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <jansson.h>
-#include <czmq.h>
-#include <flux/core.h>
-
-#include "src/common/libeventlog/eventlog.h"
-#include "eventlogger.h"
-
-struct eventlog_batch {
-    zlist_t *entries;
-    flux_kvs_txn_t *txn;
-    flux_watcher_t *timer;
-    struct eventlogger *ev;
-};
-
-struct eventlogger {
-    int refcount;
-    flux_t *h;
-    double batch_timeout;
-    double commit_timeout;
-    zlist_t *pending;
-    struct eventlog_batch *current;
-    struct eventlogger_ops ops;
-    void *arg;
-};
-
-int eventlogger_set_commit_timeout (struct eventlogger *ev, double timeout)
-{
-    if (!ev || (timeout < 0. && timeout != -1.)) {
-        errno = EINVAL;
-        return -1;
-    }
-    ev->commit_timeout = timeout;
-    return 0;
-}
-
-static void eventlog_batch_destroy (struct eventlog_batch *batch)
-{
-    if (batch) {
-        if (batch->entries)
-            zlist_destroy (&batch->entries);
-        flux_kvs_txn_destroy (batch->txn);
-        flux_watcher_destroy (batch->timer);
-        free (batch);
-    }
-}
-
-static void eventlogger_batch_complete (struct eventlogger *ev,
-                                        struct eventlog_batch *batch)
-{
-    zlist_remove (ev->pending, batch);
-    if (--ev->refcount == 0 && ev->ops.idle)
-        (*ev->ops.idle) (ev, ev->arg);
-}
-
-static int eventlogger_batch_start (struct eventlogger *ev,
-                                    struct eventlog_batch *batch)
-{
-    if (zlist_append (ev->pending, batch) < 0)
-        return -1;
-    zlist_freefn (ev->pending,
-                  batch,
-                  (zlist_free_fn *) eventlog_batch_destroy,
-                  true);
-
-    /*  If refcount just increased to 1, notify that eventlogger is busy */
-    if (++ev->refcount == 1 && ev->ops.busy)
-        (*ev->ops.busy) (ev, ev->arg);
-    return 0;
-}
-
-static void eventlog_batch_error (struct eventlog_batch *batch, int errnum)
-{
-    struct eventlogger *ev = batch->ev;
-    json_t *entry;
-    if (!ev->ops.err)
-        return;
-    entry = zlist_first (batch->entries);
-    while (entry) {
-        (*ev->ops.err) (ev, errnum, entry);
-        entry = zlist_next (batch->entries);
-    }
-}
-
-static void commit_cb (flux_future_t *f, void *arg)
-{
-    struct eventlog_batch *batch = arg;
-    if (flux_future_get (f, NULL) < 0)
-        eventlog_batch_error (batch, errno);
-    eventlogger_batch_complete (batch->ev, batch);
-    flux_future_destroy (f);
-}
-
-static void
-timer_cb (flux_reactor_t *r, flux_watcher_t *w, int revents, void *arg)
-{
-    struct eventlog_batch *batch = arg;
-    struct eventlogger *ev = batch->ev;
-    flux_t *h = ev->h;
-    double timeout = ev->commit_timeout;
-    flux_future_t *f = NULL;
-    int flags = FLUX_KVS_TXN_COMPACT;
-
-    if (!(f = flux_kvs_commit (h, NULL, flags, batch->txn))
-        || flux_future_then (f, timeout, commit_cb, batch) < 0) {
-        eventlog_batch_error (batch, errno);
-        return;
-    }
-    batch->ev->current = NULL;
-}
-
-static struct eventlog_batch * eventlog_batch_create (struct eventlogger *ev)
-{
-    struct eventlog_batch *batch = calloc (1, sizeof (*batch));
-    if (!batch)
-        return NULL;
-    flux_reactor_t *r = flux_get_reactor (ev->h);
-    batch->ev = ev;
-    batch->entries = zlist_new ();
-    batch->txn = flux_kvs_txn_create ();
-    batch->timer = flux_timer_watcher_create (r,
-                                              ev->batch_timeout, 0.,
-                                              timer_cb,
-                                              batch);
-    if (!batch->entries || !batch->txn || !batch->timer) {
-        eventlog_batch_destroy (batch);
-        return NULL;
-    }
-    flux_watcher_start (batch->timer);
-    return batch;
-}
-
-void eventlogger_destroy (struct eventlogger *ev)
-{
-    if (ev) {
-        if (ev->pending)
-            zlist_destroy (&ev->pending);
-        free (ev);
-    }
-}
-
-struct eventlogger *eventlogger_create (flux_t *h,
-                                        double timeout,
-                                        struct eventlogger_ops *ops,
-                                        void *arg)
-{
-    struct eventlogger *ev = calloc (1, sizeof (*ev));
-    if (ev) {
-        ev->pending = zlist_new ();
-        if (!ev->pending) {
-            eventlogger_destroy (ev);
-            return NULL;
-        }
-        ev->h = h;
-        ev->batch_timeout = timeout;
-        ev->commit_timeout = -1.;
-        ev->current = NULL;
-        ev->ops = *ops;
-        ev->arg = arg;
-    }
-    return ev;
-}
-
-static struct eventlog_batch * eventlog_batch_get (struct eventlogger *ev)
-{
-    struct eventlog_batch *batch = ev->current;
-    if (!batch) {
-        if (!(ev->current = eventlog_batch_create (ev)))
-            return NULL;
-        batch = ev->current;
-        eventlogger_batch_start (ev, batch);
-    }
-    return batch;
-}
-
-static int append_wait (struct eventlogger *ev,
-                        const char *path,
-                        const char *entrystr)
-{
-    /*  append_wait also appends all pending transactions synchronously  */
-    struct eventlog_batch *batch = eventlog_batch_get (ev);
-    if (!batch)
-        return -1;
-
-    if (flux_kvs_txn_put (ev->current->txn,
-                          FLUX_KVS_APPEND,
-                          path, entrystr) < 0)
-        return -1;
-
-    return eventlogger_flush (ev);
-}
-
-static int append_async (struct eventlogger *ev,
-                         const char *path,
-                         json_t *entry,
-                         const char *entrystr)
-{
-    struct eventlog_batch *batch = eventlog_batch_get (ev);
-
-    if (!batch)
-        return -1;
-    if (flux_kvs_txn_put (ev->current->txn,
-                          FLUX_KVS_APPEND,
-                          path,
-                          entrystr) < 0)
-            return -1;
-
-    if (zlist_append (batch->entries, entry) < 0)
-        return -1;
-    json_incref (entry);
-    zlist_freefn (batch->entries,
-                  entry,
-                  (zlist_free_fn *) json_decref,
-                  true);
-    return 0;
-}
-
-int eventlogger_append_entry (struct eventlogger *ev,
-                              int flags,
-                              const char *path,
-                              json_t *entry)
-{
-    char *entrystr = NULL;
-    int rc = -1;
-
-    if (!(entrystr = eventlog_entry_encode (entry)))
-        return -1;
-
-    if (flags & EVENTLOGGER_FLAG_WAIT)
-        rc = append_wait (ev, path, entrystr);
-    else
-        rc = append_async (ev, path, entry, entrystr);
-    free (entrystr);
-    return rc;
-}
-
-int eventlogger_append (struct eventlogger *ev,
-                        int flags,
-                        const char *path,
-                        const char *name,
-                        const char *context)
-{
-    int rc = -1;
-    json_t *entry = NULL;
-
-    if (!(entry = eventlog_entry_create (0., name, context)))
-        goto out;
-    rc = eventlogger_append_entry (ev, flags, path, entry);
-out:
-    json_decref (entry);
-    return rc;
-}
-
-int eventlogger_flush (struct eventlogger *ev)
-{
-    int rc = -1;
-    flux_future_t *f = NULL;
-    struct eventlog_batch *batch;
-    int flags = FLUX_KVS_TXN_COMPACT;
-
-    if (!(batch = eventlog_batch_get (ev)))
-        return -1;
-
-    if (!(f = flux_kvs_commit (ev->h, NULL, flags, ev->current->txn))
-        || flux_future_wait_for (f, ev->commit_timeout) < 0)
-        goto out;
-    if ((rc = flux_future_get (f, NULL)) < 0)
-        eventlog_batch_error (batch, errno);
-
-    eventlogger_batch_complete (ev, ev->current);
-    ev->current = NULL;
-out:
-    flux_future_destroy (f);
-    return rc;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/shell/eventlogger.h b/src/shell/eventlogger.h
deleted file mode 100644
index f664531ca798..000000000000
--- a/src/shell/eventlogger.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#ifndef FLUX_SHELL_EVENTLOGGER_H
-#define FLUX_SHELL_EVENTLOGGER_H
-
-#include <flux/core.h>
-#include <jansson.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct eventlogger;
-
-typedef void (*eventlogger_state_f) (struct eventlogger *ev, void *arg);
-typedef void (*eventlogger_err_f) (struct eventlogger *ev,
-                                   int err,
-                                   json_t *entry);
-
-struct eventlogger_ops {
-    eventlogger_state_f busy; /* Called after idle when starting a batch    */
-    eventlogger_state_f idle; /* Called when no more batches pending        */
-    eventlogger_err_f  err;   /* Called on error, once per failed entry     */
-};
-
-enum {
-    EVENTLOGGER_FLAG_ASYNC = 0,  /* Append entry to eventlog asynchronously */
-    EVENTLOGGER_FLAG_WAIT =  1,  /* Append entry to eventlog synchronously  */
-};
-
-/*  Create an eventlogger with batched eventlog appends at interval
- *   `timeout`. Eventlogger will process user callbacks in `ops` as
- *   defined above.
- */
-struct eventlogger *eventlogger_create (flux_t *h,
-                                        double timeout,
-                                        struct eventlogger_ops *ops,
-                                        void *arg);
-
-void eventlogger_destroy (struct eventlogger *ev);
-
-int eventlogger_append (struct eventlogger *ev,
-                        int flags,
-                        const char *path,
-                        const char *name,
-                        const char *context);
-
-int eventlogger_append_entry (struct eventlogger *ev,
-                              int flags,
-                              const char *path,
-                              json_t *entry);
-
-int eventlogger_set_commit_timeout (struct eventlogger *ev, double timeout);
-
-int eventlogger_flush (struct eventlogger *ev);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* !FLUX_SHELL_EVENTLOGGER_H */
diff --git a/src/shell/events.c b/src/shell/events.c
index 402881102c79..f010d766358d 100644
--- a/src/shell/events.c
+++ b/src/shell/events.c
@@ -11,24 +11,36 @@
 /*  Shell exec.eventlog event emitter
  *  Allows context for shell events to be added from multiple sources.
  */
+#define FLUX_SHELL_PLUGIN_NAME NULL
 
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
 
-#include <czmq.h>
 #include <jansson.h>
+#include <assert.h>
 
 #include <flux/shell.h>
 
-#include "eventlogger.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libeventlog/eventlogger.h"
+
+#include "events.h"
 
 struct shell_eventlogger {
     flux_shell_t *shell;
     zhashx_t *contexts;
+    zlistx_t *emitted_events;
     struct eventlogger *ev;
 };
 
+struct emitted_event {
+    char *event;
+    void *handle;
+    bool confirmed_logged;
+};
+
 static void json_free (void **item)
 {
     if (item) {
@@ -38,6 +50,49 @@ static void json_free (void **item)
     }
 }
 
+static void emitted_event_destroy (void *data)
+{
+    if (data) {
+        struct emitted_event *e = data;
+        free (e->event);
+        free (e);
+    }
+}
+
+static void emitted_event_destroy_wrapper (void **data)
+{
+    if (data) {
+        emitted_event_destroy (*data);
+        *data = NULL;
+    }
+}
+
+static struct emitted_event *emitted_event_create (const char *event)
+{
+    struct emitted_event *e = calloc (1, sizeof (*e));
+    if (!e)
+        return NULL;
+    if (!(e->event = strdup (event)))
+        goto error;
+    return e;
+
+error:
+    emitted_event_destroy (e);
+    return NULL;
+}
+
+static int emitted_event_append (struct shell_eventlogger *shev, const char *event)
+{
+    struct emitted_event *e = emitted_event_create (event);
+    if (!e)
+        return -1;
+    if (!(e->handle = zlistx_add_end (shev->emitted_events, e))) {
+        emitted_event_destroy (e);
+        return -1;
+    }
+    return 0;
+}
+
 static void shell_eventlogger_ref (struct eventlogger *ev, void *arg)
 {
     struct shell_eventlogger *shev = arg;
@@ -50,10 +105,94 @@ static void shell_eventlogger_unref (struct eventlogger *ev, void *arg)
     flux_shell_remove_completion_ref (shev->shell, "shell_eventlogger");
 }
 
+static int emit_event (struct shell_eventlogger *shev,
+                       const char *event,
+                       bool save_to_emitted_events)
+{
+    int rc = -1;
+    char *context = NULL;
+    json_t *o = zhashx_lookup (shev->contexts, event);
+    if (o != NULL)
+        context = json_dumps (o, JSON_COMPACT);
+    if (eventlogger_append (shev->ev,
+                            EVENTLOGGER_FLAG_WAIT,
+                            "exec.eventlog",
+                            event,
+                            context) < 0)
+        goto error;
+    if (save_to_emitted_events) {
+        if (emitted_event_append (shev, event) < 0)
+            goto error;
+    }
+    rc = 0;
+error:
+    free (context);
+    return rc;
+}
+
+static int shell_eventlogger_compare_eventlog (struct shell_eventlogger *shev)
+{
+    flux_future_t *f = NULL;
+    struct emitted_event *e;
+    const char *s = NULL;
+    int rv = -1;
+
+    if (!(f = flux_kvs_lookup (flux_shell_get_flux (shev->shell),
+                               NULL,
+                               0,
+                               "exec.eventlog")))
+        return -1;
+
+    /* do this synchronously, since we are reconnecting */
+    if (flux_kvs_lookup_get (f, &s) < 0)
+        goto error;
+
+    e = zlistx_first (shev->emitted_events);
+    while (e) {
+        if (!e->confirmed_logged) {
+            int ret;
+            if ((ret = eventlog_contains_event (s, e->event)) < 0)
+                goto error;
+            if (ret == 1) {
+                if (emit_event (shev, e->event, false) < 0)
+                    goto error;
+                e->confirmed_logged = true;
+            }
+        }
+        e = zlistx_next (shev->emitted_events);
+    }
+
+    rv = 0;
+error:
+    flux_future_destroy (f);
+    return rv;
+}
+
+int shell_eventlogger_reconnect (struct shell_eventlogger *shev)
+{
+    /* during a reconnect, response to event logging may not occur,
+     * thus shell_eventlogger_unref() may not be called.  Clear all
+     * completion references to inflight transactions.
+     */
+
+    while (flux_shell_remove_completion_ref (shev->shell,
+                                             "shell_eventlogger") == 0);
+
+    /* exec.eventlog events are often critical to correct function, so
+     * if any were lost during a reconnect, we need to make sure it
+     * was logged, otherwise try to write it out again.
+     */
+    if (shell_eventlogger_compare_eventlog (shev) < 0)
+        return -1;
+
+    return 0;
+}
+
 void shell_eventlogger_destroy (struct shell_eventlogger *shev)
 {
     if (shev) {
         zhashx_destroy (&shev->contexts);
+        zlistx_destroy (&shev->emitted_events);
         eventlogger_destroy (shev->ev);
         free (shev);
     }
@@ -70,30 +209,21 @@ struct shell_eventlogger *shell_eventlogger_create (flux_shell_t *shell)
 
     if (!(h = flux_shell_get_flux (shell))
         || !(shev->ev = eventlogger_create (h, 0.01, &ops, shev))
-        || !(shev->contexts = zhashx_new ())) {
+        || !(shev->contexts = zhashx_new ())
+        || !(shev->emitted_events = zlistx_new ())) {
         shell_eventlogger_destroy (shev);
         return NULL;
     }
+    shev->shell = shell;
     zhashx_set_destructor (shev->contexts, json_free);
+    zlistx_set_destructor (shev->emitted_events, emitted_event_destroy_wrapper);
     return shev;
 }
 
 int shell_eventlogger_emit_event (struct shell_eventlogger *shev,
-                                  int flags,
                                   const char *event)
 {
-    int rc;
-    char *context = NULL;
-    json_t *o = zhashx_lookup (shev->contexts, event);
-    if (o != NULL)
-        context = json_dumps (o, JSON_COMPACT);
-    rc = eventlogger_append (shev->ev,
-                             EVENTLOGGER_FLAG_WAIT,
-                             "exec.eventlog",
-                             event,
-                             context);
-    free (context);
-    return rc;
+    return emit_event (shev, event, true);
 }
 
 static int context_set (struct shell_eventlogger *shev,
diff --git a/src/shell/events.h b/src/shell/events.h
index 68529111e73c..87d81abeeccb 100644
--- a/src/shell/events.h
+++ b/src/shell/events.h
@@ -17,7 +17,6 @@ void shell_eventlogger_destroy (struct shell_eventlogger *shev);
 struct shell_eventlogger *shell_eventlogger_create (flux_shell_t *shell);
 
 int shell_eventlogger_emit_event (struct shell_eventlogger *shev,
-                                  int flags,
                                   const char *event);
 
 int shell_eventlogger_context_vpack (struct shell_eventlogger *shev,
@@ -25,6 +24,9 @@ int shell_eventlogger_context_vpack (struct shell_eventlogger *shev,
                                      int flags,
                                      const char *fmt,
                                      va_list ap);
+
+int shell_eventlogger_reconnect (struct shell_eventlogger *shev);
+
 #endif /* !_SHELL_EVENTS_H */
 
 /* vi: ts=4 sw=4 expandtab
diff --git a/src/shell/evlog.c b/src/shell/evlog.c
index fa7c643ad0de..38404790a1c6 100644
--- a/src/shell/evlog.c
+++ b/src/shell/evlog.c
@@ -28,19 +28,21 @@
  *   plugins to be set independently of the main shell log
  *   facility level.
  */
+#define FLUX_SHELL_PLUGIN_NAME "evlog"
 
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
 #include <jansson.h>
-#include <czmq.h>
 #include <flux/core.h>
 #include <flux/shell.h>
 
+#include "src/common/libeventlog/eventlogger.h"
+#include "ccan/str/str.h"
+
 #include "info.h"
 #include "internal.h"
 #include "builtins.h"
-#include "eventlogger.h"
 
 struct evlog {
     int sync_mode;
@@ -78,6 +80,10 @@ static int log_eventlog (flux_plugin_t *p,
 
 static void evlog_destroy (struct evlog *evlog)
 {
+    /*  Redirect future logging to stderr */
+    flux_shell_log_setlevel (evlog->level, "stderr");
+
+    eventlogger_flush (evlog->ev);
     eventlogger_destroy (evlog->ev);
     free (evlog);
 }
@@ -94,7 +100,27 @@ static void evlog_unref (struct eventlogger *ev, void *arg)
     flux_shell_remove_completion_ref (evlog->shell, "eventlogger.txn");
 }
 
-static void evlog_error (struct eventlogger *ev, int errnum, json_t *entry)
+static int log_eventlog_reconnect (flux_plugin_t *p,
+                                   const char *topic,
+                                   flux_plugin_arg_t *args,
+                                   void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+
+    /* during a reconnect, response to event logging may not occur,
+     * thus evlog_unref() may not be called.  Clear all completion
+     * references to inflight transactions.
+     */
+
+    while (flux_shell_remove_completion_ref (shell, "eventlogger.txn") == 0);
+
+    return 0;
+}
+
+static void evlog_error (struct eventlogger *ev,
+                         void *arg,
+                         int errnum,
+                         json_t *entry)
 {
     const char *msg;
     if (json_unpack (entry, "{s:{s:s}}",
@@ -103,7 +129,7 @@ static void evlog_error (struct eventlogger *ev, int errnum, json_t *entry)
         fprintf (stderr, "evlog_error: failed to unpack message\n");
         return;
     }
-    fprintf (stderr, "flux-shell: evlog failure: %s: msg=%s\n",
+    fprintf (stderr, "evlog: %s: msg=%s\n",
                      strerror (errnum), msg);
 }
 
@@ -152,7 +178,7 @@ static int log_eventlog_setlevel (flux_plugin_t *p,
                          flux_plugin_arg_strerror (args));
         return -1;
     }
-    if (strcmp (name, "any") == 0 || strcmp (name, "eventlog") == 0)
+    if (streq (name, "any") || streq (name, "eventlog"))
         evlog->level = level;
     return 0;
 }
@@ -171,7 +197,7 @@ static int evlog_shell_exit (flux_plugin_t *p,
     return 0;
 }
 
-/*  Start the evenlog-based logger during shell.connect, just after the
+/*  Start the eventlog-based logger during shell.connect, just after the
  *   shell has obtained a flux_t handle. This allows more early log
  *   messages to make it into the eventlog, but some data (such as
  *   the current shell_rank) is not available at this time.
@@ -184,10 +210,6 @@ static int log_eventlog_start (flux_plugin_t *p,
     flux_shell_t *shell = flux_plugin_get_shell (p);
     struct evlog *evlog = NULL;
 
-    /*  Do not activate eventlogger in standlone mode */
-    if (shell->standalone)
-        return 0;
-
     if (!(evlog = evlog_create (shell)))
         return -1;
     if (flux_plugin_aux_set (p, "evlog", evlog,
@@ -202,7 +224,8 @@ static int log_eventlog_start (flux_plugin_t *p,
                                    evlog) < 0)
         goto err;
 
-    flux_shell_log_setlevel (FLUX_SHELL_ERROR, "stderr");
+    /*  Disable stderr logging */
+    flux_shell_log_setlevel (FLUX_SHELL_QUIET, "stderr");
     return 0;
 err:
     evlog_destroy (evlog);
@@ -210,8 +233,9 @@ static int log_eventlog_start (flux_plugin_t *p,
 }
 
 struct shell_builtin builtin_log_eventlog = {
-    .name = "evlog",
+    .name = FLUX_SHELL_PLUGIN_NAME,
     .connect = log_eventlog_start,
+    .reconnect = log_eventlog_reconnect,
 };
 
 /*
diff --git a/src/shell/exception.c b/src/shell/exception.c
new file mode 100644
index 000000000000..7d5cf78245fd
--- /dev/null
+++ b/src/shell/exception.c
@@ -0,0 +1,111 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* shell plugin handling direct notification of job exceptions
+ *
+ */
+#define FLUX_SHELL_PLUGIN_NAME "exception"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <flux/shell.h>
+
+#include "ccan/str/str.h"
+#include "src/common/libeventlog/eventlog.h"
+
+#include "internal.h"
+#include "builtins.h"
+
+static void exception_handler (flux_t *h,
+                               flux_msg_handler_t *mh,
+                               const flux_msg_t *msg,
+                               void *arg)
+{
+    flux_shell_t *shell = arg;
+    const char *type;
+    int severity = -1;
+    int shell_rank = -1;
+    const char *message = "";
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s:i s:i s?s}",
+                             "type", &type,
+                             "severity", &severity,
+                             "shell_rank", &shell_rank,
+                             "message", &message) < 0)
+        goto error;
+
+    if (strlen (message) > 0)
+        shell_warn ("%s", message);
+
+    if (streq (type, "lost-shell")) {
+        flux_plugin_arg_t *args = flux_plugin_arg_create ();
+        if (!args
+            || flux_plugin_arg_pack (args,
+                                     FLUX_PLUGIN_ARG_IN,
+                                     "{s:i s:i}",
+                                     "shell_rank", shell_rank,
+                                     "severity", severity) < 0) {
+            flux_plugin_arg_destroy (args);
+            goto error;
+        }
+        flux_shell_plugstack_call (shell, "shell.lost", args);
+        flux_plugin_arg_destroy (args);
+    }
+
+    if (flux_respond (h, msg, NULL) < 0)
+        shell_log_errno ("flux_respond");
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        shell_log_errno ("flux_respond_error");
+
+}
+
+static int exception_init (flux_plugin_t *p,
+                           const char *topic,
+                           flux_plugin_arg_t *args,
+                           void *data)
+{
+    flux_t *h;
+    flux_jobid_t id;
+    int shell_rank;
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    if (!shell)
+        return -1;
+    if (!(h = flux_shell_get_flux (shell)))
+        return -1;
+    if (flux_shell_info_unpack (shell,
+                                "{s:I s:i}",
+                                "jobid", &id,
+                                "rank", &shell_rank) < 0)
+        return -1;
+    if (shell_rank != 0)
+        return 0;
+
+    if (flux_shell_service_register (shell,
+                                     "exception",
+                                     exception_handler,
+                                     shell) < 0)
+        return -1;
+    return 0;
+}
+
+struct shell_builtin builtin_exception = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = exception_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/files.c b/src/shell/files.c
new file mode 100644
index 000000000000..87448721b7aa
--- /dev/null
+++ b/src/shell/files.c
@@ -0,0 +1,103 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/*  jobspec "files" attribute handler
+ */
+#define FLUX_SHELL_PLUGIN_NAME "files"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <unistd.h>
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/shell.h>
+
+#include "src/common/libfilemap/filemap.h"
+
+#include "builtins.h"
+
+static void trace (void *arg,
+                   json_t *fileref,
+                   const char *path,
+                   int mode,
+                   int64_t size,
+                   int64_t mtime,
+                   int64_t ctime,
+                   const char *encoding)
+{
+    shell_trace ("extracting file %s size=%ju mode=%04o",
+                 path,
+                 (uintmax_t) size,
+                 mode);
+}
+
+static int extract_job_files (flux_t *h,
+                              const char *dir,
+                              json_t *files)
+{
+    int rc = -1;
+    char *orig_dir;
+    flux_error_t error;
+
+    if (!(orig_dir = getcwd (NULL, 0)))
+        return shell_log_errno ("getcwd");
+    if (chdir (dir) < 0) {
+        shell_log_errno ("chdir %s", dir);
+        goto out;
+    }
+    if (filemap_extract (h, files, 0, &error, trace, NULL) < 0) {
+        shell_log_error ("%s", error.text);
+        goto out;
+    }
+    rc = 0;
+out:
+    if (chdir (orig_dir) < 0)
+        shell_die_errno (1, "failed to chdir back to %s", orig_dir);
+    free (orig_dir);
+    return rc;
+}
+
+static int files_init (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    flux_t *h;
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    json_t *files = NULL;
+    const char *tmpdir;
+
+    if (!shell || !(h = flux_shell_get_flux (shell)))
+        return shell_log_errno ("unable to get shell or flux handle");
+
+    if (!(tmpdir = flux_shell_getenv (shell, "FLUX_JOB_TMPDIR")))
+        return shell_log_errno ("flux_shell_getenv");
+
+    if (flux_shell_info_unpack (shell,
+                               "{s:{s:{s?{s?o}}}}",
+                               "jobspec",
+                                 "attributes",
+                                   "system",
+                                     "files", &files) < 0)
+        return shell_log_errno ("failed to unpack jobspec");
+
+    return extract_job_files (h, tmpdir, files);
+}
+
+struct shell_builtin builtin_files = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = files_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/flux-shell.map b/src/shell/flux-shell.map
deleted file mode 100644
index d3ec526e7fb4..000000000000
--- a/src/shell/flux-shell.map
+++ /dev/null
@@ -1,6 +0,0 @@
-{ global:
-    flux_shell_*;
-    flux_plugin_get_shell;
-    __asan*;
-  local: *;
-};
diff --git a/src/shell/gpubind.c b/src/shell/gpubind.c
index 2c07463d647f..8c2649d437ef 100644
--- a/src/shell/gpubind.c
+++ b/src/shell/gpubind.c
@@ -13,6 +13,7 @@
  * Builtin GPU binding for flux-shell. Spread CUDA_VISIBLE_DEVICES
  *  across tasks depending on number in slot.
  */
+#define FLUX_SHELL_PLUGIN_NAME "gpu-affinity"
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -23,35 +24,52 @@
 #include <flux/shell.h>
 #include <flux/idset.h>
 
+#include "ccan/str/str.h"
+
 #include "builtins.h"
+#include "affinity.h"
 
-int ngpus_per_task = -1;
+struct gpu_affinity {
+    int ntasks;
+    int ngpus;
+    struct idset *gpus;
+    hwloc_cpuset_t *gpusets;
+};
 
-int get_shell_gpus (flux_shell_t *shell,
-                    int *ntasks,
-                    struct idset **ids)
+
+static void gpu_affinity_destroy (struct gpu_affinity *ctx)
 {
-    int rc = -1;
-    const char *gpu_list = NULL;
-    struct idset *gpus = NULL;
+    if (ctx) {
+        idset_destroy (ctx->gpus);
+        cpuset_array_destroy (ctx->gpusets, ctx->ntasks);
+        free (ctx);
+    }
+}
 
+static struct gpu_affinity *gpu_affinity_create (flux_shell_t *shell)
+{
+    const char *gpu_list = NULL;
+    struct gpu_affinity *ctx = calloc (1, sizeof (*ctx));
+    if (!ctx)
+        return NULL;
     if (flux_shell_rank_info_unpack (shell,
                                      -1,
-                                     "{s:i s:{s?:s}}",
-                                     "ntasks", ntasks,
+                                     "{s:i s:{s?s}}",
+                                     "ntasks", &ctx->ntasks,
                                      "resources",
                                        "gpus", &gpu_list) < 0) {
         shell_log_errno ("flux_shell_rank_info_unpack");
-        goto out;
+        goto error;
     }
-    if (!(gpus = idset_decode (gpu_list ? gpu_list : ""))) {
+    if (!(ctx->gpus = idset_decode (gpu_list ? gpu_list : ""))) {
         shell_log_errno ("idset_encode (%s)", gpu_list);
-        goto out;
+        goto error;
     }
-    rc = 0;
-out:
-    *ids = gpus;
-    return rc;
+    ctx->ngpus = idset_count (ctx->gpus);
+    return ctx;
+error:
+    gpu_affinity_destroy (ctx);
+    return NULL;
 }
 
 static int plugin_task_setenv (flux_plugin_t *p,
@@ -66,35 +84,98 @@ static int plugin_task_setenv (flux_plugin_t *p,
     return 0;
 }
 
+static int plugin_task_id (flux_plugin_t *p)
+{
+    int taskid = -1;
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    flux_shell_task_t *task = flux_shell_current_task (shell);
+    if (flux_shell_task_info_unpack (task,
+                                     "{s:i}",
+                                     "localid", &taskid) < 0)
+        return shell_log_errno ("failed to unpack task local id");
+    return taskid;
+}
+
+static struct idset *cpuset_to_idset (hwloc_cpuset_t set)
+{
+    int i;
+    struct idset *idset;
+    if (!(idset = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+        shell_log_errno ("failed to create idset");
+        return NULL;
+    }
+    i = -1;
+    while ((i = hwloc_bitmap_next (set, i)) != -1) {
+        if (idset_set (idset, i) < 0) {
+            shell_log_errno ("failed to set %d in idset", i);
+            idset_destroy (idset);
+            return NULL;
+        }
+    }
+    return idset;
+}
+
 static int gpubind_task_init (flux_plugin_t *p,
                               const char *topic,
                               flux_plugin_arg_t *args,
                               void *data)
 {
     char *s;
-    struct idset *gpus = data;
-    struct idset *ids = idset_create (0, IDSET_FLAG_AUTOGROW);
+    struct idset *ids;
+    int taskid;
+    struct gpu_affinity *ctx = data;
+
+    if (!ctx->gpusets)
+        return 0;
+
+    if ((taskid = plugin_task_id (p)) < 0)
+        return -1;
 
-    for (int i = 0; i < ngpus_per_task; i++) {
-        unsigned id = idset_first (gpus);
-        idset_set (ids, id);
-        idset_clear (gpus, id);
+    /* Need to convert hwloc_cpuset_t to idset since there's no function
+     * to convert a hwloc_cpuset_t to a strict comma-separated list of ids:
+     */
+    if (!(ids = cpuset_to_idset (ctx->gpusets[taskid]))
+        || !(s = idset_encode (ids, 0))) {
+        shell_log_error ("failed to get idset from gpu set for task %d",
+                         taskid);
+        idset_destroy (ids);
+        return -1;
     }
-    s = idset_encode (ids, IDSET_FLAG_RANGE);
     plugin_task_setenv (p, "CUDA_VISIBLE_DEVICES", s);
     free (s);
     idset_destroy (ids);
     return 0;
 }
 
+static hwloc_cpuset_t *distribute_gpus (struct gpu_affinity *ctx)
+{
+    int ngpus_per_task = ctx->ngpus / ctx->ntasks;
+    hwloc_cpuset_t *gpusets = cpuset_array_create (ctx->ntasks);
+    for (int i = 0; i < ctx->ntasks; i++) {
+        for (int j = 0; j < ngpus_per_task; j++) {
+            unsigned id = idset_first (ctx->gpus);
+            if (id == IDSET_INVALID_ID
+                || idset_clear (ctx->gpus, id) < 0) {
+                shell_log_errno ("Failed to get GPU id for task %d", id);
+                goto error;
+            }
+            hwloc_bitmap_set (gpusets[i], id);
+        }
+    }
+    return gpusets;
+error:
+    cpuset_array_destroy (gpusets, ctx->ntasks);
+    return NULL;
+}
+
 static int gpubind_init (flux_plugin_t *p,
                          const char *topic,
                          flux_plugin_arg_t *args,
                          void *data)
 {
-    int rc, ngpus, ntasks;
-    struct idset *gpus;
+    int rc;
     char *opt;
+    struct gpu_affinity *ctx;
     flux_shell_t *shell = flux_plugin_get_shell (p);
 
     if (!shell)
@@ -109,32 +190,47 @@ static int gpubind_init (flux_plugin_t *p,
         /* gpu-affinity defaults to "on" */
         opt = "on";
     }
-    if (strcmp (opt, "off") == 0)
+    if (streq (opt, "off")) {
+        shell_debug ("disabling affinity due to gpu-affinity=off");
         return 0;
-    if (get_shell_gpus (shell, &ntasks, &gpus) < 0)
+    }
+
+    /*  Set default CUDA_VISIBLE_DEVICES to an invalid id, -1, so that
+     *  jobs which are not assigned any GPUs do not use GPUs which
+     *  happen to be available on the current node.
+     */
+    flux_shell_setenvf (shell, 1, "CUDA_VISIBLE_DEVICES", "%d", -1);
+
+    if (!(ctx = gpu_affinity_create (shell)))
         return -1;
-    if (flux_plugin_aux_set (p, NULL, gpus, (flux_free_f)idset_destroy) < 0) {
+    if (flux_plugin_aux_set (p,
+                             NULL,
+                             ctx,
+                             (flux_free_f) gpu_affinity_destroy) < 0) {
         shell_log_errno ("flux_plugin_aux_set");
-        idset_destroy (gpus);
+        gpu_affinity_destroy (ctx);
         return -1;
     }
-    if ((ngpus = idset_count (gpus)) <= 0)
-        return 0;
+
+    if (flux_plugin_add_handler (p,
+                                 "task.init",
+                                 gpubind_task_init,
+                                 ctx) < 0)
+        return shell_log_errno ("gpubind: flux_plugin_add_handler");
 
     flux_shell_setenvf (shell, 0, "CUDA_DEVICE_ORDER", "PCI_BUS_ID");
 
-    if (strcmp (opt, "per-task") == 0) {
-        /*  Set global ngpus_per_task to use in task.init callback:
-         */
-        ngpus_per_task = ngpus / ntasks;
-        if (flux_plugin_add_handler (p,
-                                     "task.init",
-                                     gpubind_task_init,
-                                     gpus) < 0)
-            return shell_log_errno ("gpubind: flux_plugin_add_handler");
+    if (streq (opt, "per-task")) {
+        if (!(ctx->gpusets = distribute_gpus (ctx)))
+            return shell_log_errno ("failed to distribute %d gpus",
+                                    ctx->ngpus);
+    }
+    else if (strstarts (opt, "map:")) {
+        if (!(ctx->gpusets = parse_cpuset_list (opt+4, ctx->ntasks)))
+            return shell_log_errno ("failed to parse gpu map %s", opt+4);
     }
-    else {
-        char *ids = idset_encode (gpus, IDSET_FLAG_RANGE);
+    else if (ctx->ngpus > 0) {
+        char *ids = idset_encode (ctx->gpus, 0);
         flux_shell_setenvf (shell, 1, "CUDA_VISIBLE_DEVICES", "%s", ids);
         free (ids);
     }
@@ -142,7 +238,7 @@ static int gpubind_init (flux_plugin_t *p,
 }
 
 struct shell_builtin builtin_gpubind = {
-    .name = "gpu-affinity",
+    .name = FLUX_SHELL_PLUGIN_NAME,
     .init = gpubind_init
 };
 
diff --git a/src/shell/hwloc.c b/src/shell/hwloc.c
new file mode 100644
index 000000000000..e94ef087274b
--- /dev/null
+++ b/src/shell/hwloc.c
@@ -0,0 +1,119 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* hwloc options handler
+ */
+#define FLUX_SHELL_PLUGIN_NAME "hwloc"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <flux/core.h>
+#include <flux/shell.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/librlist/rhwloc.h"
+#include "ccan/str/str.h"
+
+#include "builtins.h"
+
+static int create_xmlfile (flux_shell_t *shell, int do_restrict)
+{
+    int fd = -1;
+    char *xmlfile = NULL;
+    char *restricted_xml = NULL;
+    const char *hwloc_xml;
+    const char *tmpdir;
+
+    if (!(tmpdir = flux_shell_getenv (shell, "FLUX_JOB_TMPDIR")))
+        tmpdir = "/tmp";
+
+    if (flux_shell_get_hwloc_xml (shell, &hwloc_xml) < 0) {
+        shell_log_error ("failed to get shell hwloc xml");
+        goto error;
+    }
+    if (do_restrict) {
+        if (!(restricted_xml = rhwloc_topology_xml_restrict (hwloc_xml))) {
+            shell_log_errno ("failed to restrict topology xml");
+            goto error;
+        }
+        hwloc_xml = restricted_xml;
+    }
+    if (asprintf (&xmlfile, "%s/hwloc.xml", tmpdir) < 0) {
+        shell_log_error ("asprintf HWLOC_XMLFILE failed");
+        goto error;
+    }
+    if ((fd = open (xmlfile, O_CREAT|O_EXCL|O_WRONLY, 0640)) < 0) {
+        shell_log_errno ("%s", xmlfile);
+        goto error;
+    }
+    shell_debug ("Writing %ld bytes to HWLOC_XMLFILE=%s\n",
+                 (long int) strlen (hwloc_xml),
+                 xmlfile);
+    if (write_all (fd, hwloc_xml, strlen (hwloc_xml)) < 0 || close (fd) < 0) {
+        shell_log_errno ("failed to write HWLOC_XMLFILE");
+        goto error;
+    }
+    if (flux_shell_setenvf (shell, 0, "HWLOC_XMLFILE", "%s", xmlfile) < 0) {
+        shell_log_errno ("failed to set HWLOC_XMLFILE in job environment");
+        goto error;
+    }
+    /*  Note: HWLOC_XMLFILE will be ignored if HWLOC_COMPONENTS is also set
+     *  in the environment, so unset that variable here.
+     */
+    if (flux_shell_unsetenv (shell, "HWLOC_COMPONENTS") < 0
+        && errno != ENOENT) {
+        shell_log_errno ("failed to unset HWLOC_COMPONENTS");
+        goto error;
+    }
+    return 0;
+error:
+    if (fd >= 0)
+        close (fd);
+    free (xmlfile);
+    free (restricted_xml);
+    return -1;
+}
+
+static int hwloc_post_init (flux_plugin_t *p,
+                            const char *topic,
+                            flux_plugin_arg_t *args,
+                            void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    int xmlfile = 0;
+    int do_restrict = 0;
+
+    if (flux_shell_getopt_unpack (shell,
+                                  "hwloc",
+                                  "{s?i s?i}",
+                                    "xmlfile", &xmlfile,
+                                    "restrict", &do_restrict) < 0)
+        return shell_log_errno ("failed to unpack hwloc options");
+
+    if (xmlfile && create_xmlfile (shell, do_restrict) < 0)
+        return shell_log_errno ("failed to write HWLOC_XMLFILE");
+    return 0;
+}
+
+struct shell_builtin builtin_hwloc = {
+    .name      = FLUX_SHELL_PLUGIN_NAME,
+    .post_init = hwloc_post_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/info.c b/src/shell/info.c
index b6c6be056f81..bc65282f8006 100644
--- a/src/shell/info.c
+++ b/src/shell/info.c
@@ -9,163 +9,270 @@
 \************************************************************/
 
 /* job shell info */
+#define FLUX_SHELL_PLUGIN_NAME NULL
 
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 #include <flux/core.h>
 #include <jansson.h>
 
 #include "src/common/libutil/read_all.h"
+#include "src/common/librlist/rhwloc.h"
+#include "ccan/str/str.h"
 
 #include "internal.h"
 #include "info.h"
 #include "jobspec.h"
 
-/* Append string 's' to JSON array 'array'.
- * Return 0 on success, -1 on failure.
- */
-static int array_append_string (json_t *array, const char *s)
-{
-    json_t *o;
-
-    if (!(o = json_string (s)) || json_array_append_new (array, o) < 0) {
-        json_decref (o);
-        return -1;
-    }
-    return 0;
-}
-
-/* If either *jobspec or *R is NULL, fetch it from future and assign.
+/* Get jobspec from job-info.lookup future and assign.
  * Return 0 on success, -1 on failure (and log error).
  * N.B. assigned values remain valid until future is destroyed.
  */
-static int lookup_job_info_get (flux_future_t *f,
-                                const char **jobspec,
-                                const char **R)
+static int lookup_jobspec_get (flux_future_t *f, char **jobspec)
 {
-    if (!*jobspec && flux_rpc_get_unpack (f, "{s:s}", "jobspec", jobspec) < 0)
-        goto error;
-    if (!*R && flux_rpc_get_unpack (f, "{s:s}", "R", R) < 0)
+    flux_error_t error;
+    const char *J;
+    if (flux_rpc_get_unpack (f, "{s:s}", "J", &J) < 0)
         goto error;
+    if (!(*jobspec = flux_unwrap_string (J, true, NULL, &error))) {
+        shell_log_error ("failed to unwrap J: %s", error.text);
+        return -1;
+    }
     return 0;
 error:
     shell_log_error ("job-info: %s", future_strerror (f, errno));
     return -1;
 }
 
-/* If either jobspec or R is NULL, fetch it from the job-info service.
+/* Fetch J from the job-info service.
  * Return future on success or NULL on failure (and log error).
  */
-static flux_future_t *lookup_job_info (flux_t *h,
-                                       flux_jobid_t jobid,
-                                       const char *jobspec,
-                                       const char *R)
+static flux_future_t *lookup_jobspec (flux_t *h, flux_jobid_t jobid)
 {
-    json_t *keys;
     flux_future_t *f;
-
-    if (!(keys = json_array ())
-            || (!R && array_append_string (keys, "R") < 0)
-            || (!jobspec && array_append_string (keys, "jobspec") < 0)) {
-        shell_log_error ("error building json array");
-        return NULL;
-    }
     f = flux_rpc_pack (h,
                        "job-info.lookup",
                        FLUX_NODEID_ANY,
                        0,
-                       "{s:I s:O s:i}",
+                       "{s:I s:[s] s:i}",
                        "id", jobid,
-                       "keys", keys,
+                       "keys", "J",
                        "flags", 0);
     if (!f)
         shell_log_error ("error sending job-info request");
-    json_decref (keys);
     return f;
 }
 
-/* Read content of file 'optarg' and return it or NULL on failure (log error).
- * Caller must free returned result.
+/*  Unpack R from a job-info.update-watch response and update the
+ *  shell's internal info->R and info->rcalc. If a response can't be
+ *  unpacked or rcalc_create_json() fails, just ignore this response
+ *  and let caller decide if the error is fatal.
  */
-static char *parse_arg_file (const char *optarg)
+static int resource_watch_update (struct shell_info *info)
 {
-    int fd;
-    ssize_t size;
-    void *buf = NULL;
-
-    if (!strcmp (optarg, "-"))
-        fd = STDIN_FILENO;
-    else {
-        if ((fd = open (optarg, O_RDONLY)) < 0) {
-            shell_log_errno ("error opening %s", optarg);
-            return NULL;
-        }
+    int rc = -1;
+    flux_future_t *f = info->R_watch_future;
+    json_t *R = NULL;
+    rcalc_t *rcalc = NULL;
+
+    if (flux_rpc_get_unpack (f, "{s:o}", "R", &R) < 0) {
+        shell_log_errno ("error getting R from job-info watch response");
+        goto out;
     }
-    if ((size = read_all (fd, &buf)) < 0)
-        shell_log_errno ("error reading %s", optarg);
-    if (fd != STDIN_FILENO)
-        (void)close (fd);
-    return buf;
+    if (!(rcalc = rcalc_create_json (R))) {
+        shell_log_error ("error decoding R");
+        goto out;
+    }
+    /*  Swap previous and updated R, rcalc:
+     */
+    json_decref (info->R);
+    info->R = json_incref (R);
+    rcalc_destroy (info->rcalc);
+    info->rcalc = rcalc;
+    rc = 0;
+out:
+    flux_future_reset (f);
+    return rc;
 }
 
-/* If option 'name' exists, read it as a file and exit on failure.
- * O/w, return NULL.
- */
-static char *optparse_check_and_loadfile (optparse_t *p, const char *name)
+static void R_update_cb (flux_future_t *f, void *arg)
 {
-    char *result = NULL;
-    const char *path = optparse_get_str (p, name, NULL);
-    if (path) {
-        if (!(result = parse_arg_file (path)))
-            exit (1);
-        return result;
-    }
-    return NULL;
+    flux_shell_t *shell = arg;
+
+    if (resource_watch_update (shell->info) < 0)
+        return;
+
+    /*  Destroy cached shell "info" JSON object otherwise plugins will
+     *  not see the updated R
+     */
+    (void) flux_shell_aux_set (shell, "shell::info", NULL, NULL);
+
+    /* Notify plugins that resources have been updated.
+     * (Assume plugins will emit appropriate error messages, so ignore
+     *  error from flux_shell_plugstack_call()).
+     */
+    (void) flux_shell_plugstack_call (shell, "shell.resource-update", NULL);
 }
 
 /*  Fetch jobinfo (jobspec, R) from job-info service if not provided on
  *   command line, and parse.
  */
-static int shell_init_jobinfo (flux_shell_t *shell,
-                               struct shell_info *info,
-                               const char *jobspec,
-                               const char *R)
+static int shell_init_jobinfo (flux_shell_t *shell, struct shell_info *info)
 {
     int rc = -1;
-    flux_future_t *f = NULL;
+    flux_future_t *f_info = NULL;
+    flux_future_t *f_hwloc = NULL;
+    const char *xml;
+    char *jobspec = NULL;
     json_error_t error;
 
-    if (!R || !jobspec) {
-        /* Fetch missing jobinfo from broker job-info service */
-        if (shell->standalone) {
-            shell_log_error ("Invalid arguments: standalone and R/jobspec are unset");
-            return -1;
-        }
-        if (!(f = lookup_job_info (shell->h, shell->jobid, jobspec, R))
-                || lookup_job_info_get (f, &jobspec, &R) < 0)
+    /*  fetch hwloc topology from resource module to avoid having to
+     *  load from scratch here. The topology XML is then cached for
+     *  future shell plugin use.
+     */
+    if (!(f_hwloc = flux_rpc (shell->h,
+                              "resource.topo-get",
+                              NULL,
+                              FLUX_NODEID_ANY,
+                              0)))
+        goto out;
+
+    /*  fetch R from job-info service
+     */
+    if (!(info->R_watch_future = flux_rpc_pack (shell->h,
+                                                "job-info.update-watch",
+                                                FLUX_NODEID_ANY,
+                                                FLUX_RPC_STREAMING,
+                                                "{s:I s:s s:i}",
+                                                "id", shell->jobid,
+                                                "key", "R",
+                                                "flags", 0)))
+        goto out;
+
+    /*  fetch jobspec (via J) for this job
+     */
+    if (!(f_info = lookup_jobspec (shell->h, shell->jobid)))
+        goto out;
+
+    if (flux_rpc_get (f_hwloc, &xml) < 0
+        || !(info->hwloc_xml = strdup (xml))) {
+        shell_log_error ("error fetching local hwloc xml");
+        if (!(info->hwloc_xml = rhwloc_local_topology_xml (0))) {
+            shell_log_error ("error loading local hwloc xml");
             goto out;
+        }
+    }
+    if (lookup_jobspec_get (f_info, &jobspec) < 0) {
+        shell_log_error ("error fetching jobspec");
+        goto out;
     }
     if (!(info->jobspec = jobspec_parse (jobspec, &error))) {
         shell_log_error ("error parsing jobspec: %s", error.text);
         goto out;
     }
-    if (!(info->rcalc = rcalc_create (R))) {
-        shell_log_error ("error decoding R");
+
+    /*  Synchronously get initial version of R from first job-info
+     *  watch response:
+     */
+    if (resource_watch_update (info) < 0)
+        goto out;
+
+    /*  Register callback for future R updates:
+     */
+    if (flux_future_then (info->R_watch_future,
+                          -1.,
+                          R_update_cb,
+                          shell) < 0) {
+        shell_log_errno ("error registering R watch callback");
         goto out;
     }
     rc = 0;
 out:
-    flux_future_destroy (f);
+    free (jobspec);
+    flux_future_destroy (f_hwloc);
+    flux_future_destroy (f_info);
     return rc;
 }
 
+static int get_per_resource_option (struct jobspec *jobspec,
+                                    const char **typep,
+                                    int *countp)
+{
+    json_error_t err;
+    json_t *o = NULL;
+
+    if (!(o = json_object_get (jobspec->options, "per-resource")))
+        return 0;
+    *countp = 1;
+    if (json_unpack_ex (o, &err, 0,
+                        "{s:s s?i}",
+                        "type", typep,
+                        "count", countp) < 0)
+        return shell_log_errn (0, "invalid per-resource spec: %s", err.text);
+    return 0;
+}
+
+struct taskmap *create_taskmap (struct shell_info *info)
+{
+    struct taskmap *map = taskmap_create ();
+    if (!map)
+        return NULL;
+    for (int i = 0; i < info->shell_size; i++) {
+        struct rcalc_rankinfo ri;
+        if (rcalc_get_nth (info->rcalc, i, &ri) < 0
+            || taskmap_append (map, i, 1, ri.ntasks) < 0) {
+            shell_log_errno ("taskmap: failed to process rank=%d", i);
+            goto error;
+        }
+    }
+    return map;
+error:
+    taskmap_destroy (map);
+    return NULL;
+}
+
+int shell_info_set_taskmap (struct shell_info *info,
+                            struct taskmap *map)
+{
+    flux_error_t error;
+    const struct idset *taskids;
+    struct idset *copy;
+
+    if (!info || !map) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (taskmap_unknown (map)) {
+        shell_log_error ("invalid taskmap: mapping is unknown");
+        return -1;
+    }
+    if (info->taskmap
+        && taskmap_check (info->taskmap, map, &error) < 0) {
+        shell_log_error ("invalid taskmap: %s", error.text);
+        return -1;
+    }
+    if (!(taskids = taskmap_taskids (map, info->shell_rank))
+        || !(copy = idset_copy (taskids)))
+        return -1;
+    idset_destroy (info->taskids);
+    info->taskids = copy;
+    taskmap_destroy (info->taskmap);
+    info->taskmap = map;
+    return 0;
+}
+
 struct shell_info *shell_info_create (flux_shell_t *shell)
 {
     struct shell_info *info;
-    char *R = NULL;
-    char *jobspec = NULL;
+    const char *per_resource = NULL;
+    int per_resource_count = -1;
     int broker_rank = shell->broker_rank;
+    struct taskmap *map = NULL;
 
     if (!(info = calloc (1, sizeof (*info)))) {
         shell_log_errno ("shell_info_create");
@@ -173,19 +280,26 @@ struct shell_info *shell_info_create (flux_shell_t *shell)
     }
     info->jobid = shell->jobid;
 
-    /*  Check for jobspec and/or R on cmdline:
-     */
-    jobspec = optparse_check_and_loadfile (shell->p, "jobspec");
-    R = optparse_check_and_loadfile (shell->p, "resources");
-
-    if (shell_init_jobinfo (shell, info, jobspec, R) < 0)
+    if (shell_init_jobinfo (shell, info) < 0)
         goto error;
 
-    /* Done with potentially allocated jobspec, R strings */
-    free (jobspec);
-    free (R);
+    if (get_per_resource_option (info->jobspec,
+                                 &per_resource,
+                                 &per_resource_count) < 0)
+        goto error;
 
-   if (rcalc_distribute (info->rcalc, info->jobspec->task_count) < 0) {
+    if (per_resource != NULL) {
+        if (rcalc_distribute_per_resource (info->rcalc,
+                                           per_resource,
+                                           per_resource_count) < 0) {
+            shell_log_error ("error distributing %d tasks per-%s over R",
+                             per_resource_count, per_resource);
+            goto error;
+        }
+    }
+    else if (rcalc_distribute (info->rcalc,
+                               info->jobspec->task_count,
+                               info->jobspec->cores_per_slot) < 0) {
         shell_log_error ("error distributing %d tasks over R",
                          info->jobspec->task_count);
         goto error;
@@ -196,6 +310,14 @@ struct shell_info *shell_info_create (flux_shell_t *shell)
     }
     info->shell_size = rcalc_total_nodes (info->rcalc);
     info->shell_rank = info->rankinfo.nodeid;
+    info->total_ntasks = rcalc_total_ntasks (info->rcalc);
+
+    if (!(map = create_taskmap (info))
+        || shell_info_set_taskmap (info, map) < 0) {
+        taskmap_destroy (map);
+        shell_log_error ("error creating taskmap");
+        goto error;
+    }
     return info;
 error:
     shell_info_destroy (info);
@@ -206,8 +328,13 @@ void shell_info_destroy (struct shell_info *info)
 {
     if (info) {
         int saved_errno = errno;
+        flux_future_destroy (info->R_watch_future);
+        json_decref (info->R);
         jobspec_destroy (info->jobspec);
         rcalc_destroy (info->rcalc);
+        taskmap_destroy (info->taskmap);
+        idset_destroy (info->taskids);
+        free (info->hwloc_xml);
         free (info);
         errno = saved_errno;
     }
diff --git a/src/shell/info.h b/src/shell/info.h
index 755e262ac3e0..a9b804a7c29c 100644
--- a/src/shell/info.h
+++ b/src/shell/info.h
@@ -13,6 +13,8 @@
 
 #include <flux/core.h>
 #include <flux/shell.h>
+#include <flux/taskmap.h>
+#include <flux/hostlist.h>
 
 #include <jansson.h>
 #include <stdbool.h>
@@ -25,9 +27,16 @@ struct shell_info {
     flux_jobid_t jobid;
     int shell_rank;
     int shell_size;
+    int total_ntasks;
+    json_t *R;
     struct jobspec *jobspec;
     rcalc_t *rcalc;
     struct rcalc_rankinfo rankinfo;
+    struct taskmap *taskmap;
+    struct idset *taskids;
+    struct hostlist *hostlist;
+    char *hwloc_xml;
+    flux_future_t *R_watch_future;
 };
 
 /* Create shell_info.
@@ -38,6 +47,11 @@ struct shell_info *shell_info_create (flux_shell_t *shell);
 
 void shell_info_destroy (struct shell_info *info);
 
+/*  Set or replace current shell taskmap and taskids idset
+ *  Reference to `map` is stolen on success.
+ */
+int shell_info_set_taskmap (struct shell_info *info, struct taskmap *map);
+
 #endif /* !_SHELL_INFO_H */
 
 /*
diff --git a/src/shell/initrc.lua b/src/shell/initrc.lua
index 88c85c8367dc..6e808d19c834 100644
--- a/src/shell/initrc.lua
+++ b/src/shell/initrc.lua
@@ -1,11 +1,14 @@
--- Skip initrc if running in standalone mode
-if shell.info.options.standalone then return end
-
 -- Load all *.so plugins from plugin.searchpath
 plugin.load { file = "*.so", conf = {} }
 
 -- Source all rc files under shell.rcpath/lua.d/*.lua of shell rcpath:
-source (shell.rcpath .. "/lua.d/*.lua")
+shell.source_rcpath ("*.lua")
+
+-- Attempt to load an mpi version specific rc from rcpath:
+shell.source_rcpath_option ("mpi")
+
+-- If userrc is set in shell options, then load the user supplied initrc here:
+if shell.options.userrc then source (shell.options.userrc) end
 
 -- Uncomment to source user initrc.lua if it exists
 -- source_if_exists "~/.flux/shell/initrc.lua"
diff --git a/src/shell/input.c b/src/shell/input.c
deleted file mode 100644
index fe88cc4206ce..000000000000
--- a/src/shell/input.c
+++ /dev/null
@@ -1,606 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* std input handling
- *
- * Depending on inputs from user, a service is started to receive
- * stdin from front-end command or file is read for redirected
- * standard input.
- */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdio.h>
-#include <string.h>
-#include <flux/core.h>
-
-#include "src/common/libidset/idset.h"
-#include "src/common/libeventlog/eventlog.h"
-#include "src/common/libioencode/ioencode.h"
-
-#include "task.h"
-#include "svc.h"
-#include "internal.h"
-#include "builtins.h"
-
-struct shell_input;
-
-/* input type configured by user for input to the shell */
-enum {
-    FLUX_INPUT_TYPE_SERVICE = 1, /* default */
-    FLUX_INPUT_TYPE_FILE = 2,
-};
-
-/* how input will reach each task */
-enum {
-    FLUX_TASK_INPUT_KVS = 1,
-};
-
-struct shell_task_input_kvs {
-    flux_future_t *input_f;
-    bool input_header_parsed;
-};
-
-struct shell_task_input {
-    struct shell_input *in;
-    struct shell_task *task;
-    int type;
-    struct shell_task_input_kvs input_kvs;
-};
-
-struct shell_input_type_file {
-    const char *path;
-    int fd;
-    flux_watcher_t *w;
-    char *rankstr;
-};
-
-struct shell_input {
-    flux_shell_t *shell;
-    int stdin_type;
-    struct shell_task_input *task_inputs;
-    int ntasks;
-    struct shell_input_type_file stdin_file;
-};
-
-static void shell_task_input_kvs_cleanup (struct shell_task_input_kvs *kp)
-{
-    flux_future_destroy (kp->input_f);
-    kp->input_f = NULL;
-}
-
-static void shell_task_input_cleanup (struct shell_task_input *tp)
-{
-    shell_task_input_kvs_cleanup (&(tp->input_kvs));
-}
-
-static void shell_input_type_file_cleanup (struct shell_input_type_file *fp)
-{
-    close (fp->fd);
-    flux_watcher_destroy (fp->w);
-    free (fp->rankstr);
-}
-
-void shell_input_destroy (struct shell_input *in)
-{
-    if (in) {
-        int saved_errno = errno;
-        int i;
-        shell_input_type_file_cleanup (&(in->stdin_file));
-        for (i = 0; i < in->ntasks; i++)
-            shell_task_input_cleanup (&(in->task_inputs[i]));
-        free (in->task_inputs);
-        free (in);
-        errno = saved_errno;
-    }
-}
-
-static void shell_input_put_kvs_completion (flux_future_t *f, void *arg)
-{
-    struct shell_input *in = arg;
-
-    if (flux_future_get (f, NULL) < 0)
-        /* failng to write stdin to input is a fatal error */
-        shell_die (1, "shell_input_put_kvs: %s", strerror (errno));
-    flux_future_destroy (f);
-
-    if (flux_shell_remove_completion_ref (in->shell, "input.kvs") < 0)
-        shell_log_errno ("flux_shell_remove_completion_ref");
-}
-
-static int shell_input_put_kvs (struct shell_input *in, json_t *context)
-{
-    flux_kvs_txn_t *txn = NULL;
-    flux_future_t *f = NULL;
-    json_t *entry = NULL;
-    char *entrystr = NULL;
-    int saved_errno;
-    int rc = -1;
-
-    if (!(entry = eventlog_entry_pack (0.0, "data", "O", context)))
-        goto error;
-    if (!(entrystr = eventlog_entry_encode (entry)))
-        goto error;
-    if (!(txn = flux_kvs_txn_create ()))
-        goto error;
-    if (flux_kvs_txn_put (txn, FLUX_KVS_APPEND, "input", entrystr) < 0)
-        goto error;
-    if (!(f = flux_kvs_commit (in->shell->h, NULL, 0, txn)))
-        goto error;
-    if (flux_future_then (f, -1, shell_input_put_kvs_completion, in) < 0)
-        goto error;
-    if (flux_shell_add_completion_ref (in->shell, "input.kvs") < 0) {
-        shell_log_errno ("flux_shell_remove_completion_ref");
-        goto error;
-    }
-    /* f memory responsibility of shell_input_put_kvs_completion()
-     * callback */
-    f = NULL;
-    rc = 0;
- error:
-    saved_errno = errno;
-    flux_kvs_txn_destroy (txn);
-    free (entrystr);
-    json_decref (entry);
-    flux_future_destroy (f);
-    errno = saved_errno;
-    return rc;
-}
-
-/* Convert 'iodecode' object to an valid RFC 24 data event.
- * N.B. the iodecode object is a valid "context" for the event.
- */
-static void shell_input_stdin_cb (flux_t *h,
-                                  flux_msg_handler_t *mh,
-                                  const flux_msg_t *msg,
-                                  void *arg)
-{
-    struct shell_input *in = arg;
-    bool eof = false;
-    json_t *o;
-
-    if (shell_svc_allowed (in->shell->svc, msg) < 0)
-        goto error;
-    if (flux_request_unpack (msg, NULL, "o", &o) < 0)
-        goto error;
-    if (iodecode (o, NULL, NULL, NULL, NULL, &eof) < 0)
-        goto error;
-    if (shell_input_put_kvs (in, o) < 0)
-        goto error;
-    if (eof)
-        flux_msg_handler_stop (mh);
-    if (flux_respond (in->shell->h, msg, NULL) < 0)
-        shell_log_errno ("flux_respond");
-    return;
-error:
-    if (flux_respond_error (in->shell->h, msg, errno, NULL) < 0)
-        shell_log_errno ("flux_respond");
-}
-
-static void shell_input_type_file_init (struct shell_input *in)
-{
-    struct shell_input_type_file *fp = &(in->stdin_file);
-    fp->fd = -1;
-}
-
-static int shell_input_parse_type (struct shell_input *in)
-{
-    const char *typestr = NULL;
-    int ret;
-
-    if ((ret = flux_shell_getopt_unpack (in->shell, "input",
-                                         "{s?:{s?:s}}",
-                                         "stdin", "type", &typestr)) < 0)
-        return -1;
-
-    if (!ret || !typestr)
-        return 0;
-
-    if (!strcmp (typestr, "service"))
-        in->stdin_type = FLUX_INPUT_TYPE_SERVICE;
-    else if (!strcmp (typestr, "file")) {
-        struct shell_input_type_file *fp = &(in->stdin_file);
-
-        in->stdin_type = FLUX_INPUT_TYPE_FILE;
-
-        if (flux_shell_getopt_unpack (in->shell, "input",
-                                      "{s:{s?:s}}",
-                                      "stdin", "path", &(fp->path)) < 0)
-            return -1;
-
-        if (fp->path == NULL)
-            return shell_log_errn (0,
-                                   "path for stdin file input not specified");
-    }
-    else
-        return shell_log_errn (0, "invalid input type specified '%s'", typestr);
-
-    return 0;
-}
-
-static int shell_input_kvs_init (struct shell_input *in, json_t *header)
-{
-    flux_kvs_txn_t *txn = NULL;
-    flux_future_t *f = NULL;
-    char *headerstr = NULL;
-    int saved_errno;
-    int rc = -1;
-
-    if (!(headerstr = eventlog_entry_encode (header)))
-        goto error;
-    if (!(txn = flux_kvs_txn_create ()))
-        goto error;
-    if (flux_kvs_txn_put (txn, FLUX_KVS_APPEND, "input", headerstr) < 0)
-        goto error;
-    if (!(f = flux_kvs_commit (in->shell->h, NULL, 0, txn)))
-        goto error;
-    /* Synchronously wait for kvs commit to complete to ensure
-     * guest.input exists before passing shell initialization barrier.
-     * This is required because tasks will immediately try to watch
-     * input eventlog on starting.
-     */
-    if (flux_future_get (f, NULL) < 0)
-        shell_die_errno (1, "failed to create input eventlog");
-    rc = 0;
- error:
-    saved_errno = errno;
-    flux_kvs_txn_destroy (txn);
-    free (headerstr);
-    flux_future_destroy (f);
-    errno = saved_errno;
-    return rc;
-}
-
-static int shell_input_header (struct shell_input *in)
-{
-    json_t *o = NULL;
-    int rc = -1;
-
-    o = eventlog_entry_pack (0, "header",
-                             "{s:i s:{s:s} s:{s:i} s:{}}",
-                             "version", 1,
-                             "encoding",
-                             "stdin", "base64",
-                             "count",
-                             "stdin", 1,
-                             "options");
-    if (!o) {
-        errno = ENOMEM;
-        goto error;
-    }
-    if (shell_input_kvs_init (in, o) < 0) {
-        shell_log_errno ("shell_input_kvs_init");
-        goto error;
-    }
-    rc = 0;
- error:
-    json_decref (o);
-    return rc;
-}
-
-static int shell_input_put_kvs_raw (struct shell_input *in,
-                                    void *buf,
-                                    int len,
-                                    bool eof)
-{
-    json_t *context = NULL;
-    int saved_errno;
-    int rc = -1;
-
-    if (!(context = ioencode ("stdin", in->stdin_file.rankstr, buf, len, eof)))
-        goto error;
-    if (shell_input_put_kvs (in, context) < 0)
-        goto error;
-    rc = 0;
- error:
-    saved_errno = errno;
-    json_decref (context);
-    errno = saved_errno;
-    return rc;
-}
-
-static void shell_input_type_file_cb (flux_reactor_t *r, flux_watcher_t *w,
-                                      int revents, void *arg)
-{
-    struct shell_input *in = arg;
-    struct shell_input_type_file *fp = &(in->stdin_file);
-    long ps = sysconf (_SC_PAGESIZE);
-    char buf[ps];
-    ssize_t n;
-
-    assert (ps > 0);
-
-    /* Failure to read stdin in a fatal error.  Should be cleaner in
-     * future.  Issue #2378 */
-
-    while ((n = read (fp->fd, buf, ps)) > 0) {
-        if (shell_input_put_kvs_raw (in, buf, n, false) < 0)
-            shell_die_errno (1, "shell_input_put_kvs_raw");
-    }
-
-    if (n < 0)
-        shell_die_errno (1, "shell_input_put_kvs_raw");
-
-    if (shell_input_put_kvs_raw (in, NULL, 0, true) < 0)
-        shell_die_errno (1, "shell_input_put_kvs_raw");
-
-    flux_watcher_stop (w);
-}
-
-static int shell_input_type_file_setup (struct shell_input *in)
-{
-    struct shell_input_type_file *fp = &(in->stdin_file);
-
-    if ((fp->fd = open (fp->path, O_RDONLY)) < 0)
-        return shell_log_errno ("error opening input file '%s'", fp->path);
-
-    if (!(fp->w = flux_fd_watcher_create (in->shell->r, fp->fd,
-                                          FLUX_POLLIN,
-                                          shell_input_type_file_cb,
-                                          in)))
-        return shell_log_errno ("flux_fd_watcher_create");
-
-    if (in->shell->info->jobspec->task_count > 1) {
-        if (asprintf (&fp->rankstr, "[0-%d]",
-                      in->shell->info->jobspec->task_count) < 0)
-            return shell_log_errno ("asprintf");
-    }
-    else {
-        if (!(fp->rankstr = strdup ("0")))
-            return shell_log_errno ("asprintf");
-    }
-
-    return 0;
-}
-
-struct shell_input *shell_input_create (flux_shell_t *shell)
-{
-    struct shell_input *in;
-    size_t task_inputs_size;
-    int i;
-
-    if (!(in = calloc (1, sizeof (*in))))
-        return NULL;
-    in->shell = shell;
-    in->stdin_type = FLUX_INPUT_TYPE_SERVICE;
-    in->ntasks = shell->info->rankinfo.ntasks;
-
-    task_inputs_size = sizeof (struct shell_task_input) * in->ntasks;
-    if (!(in->task_inputs = calloc (1, task_inputs_size)))
-        goto error;
-
-    for (i = 0; i < in->ntasks; i++)
-        in->task_inputs[i].type = FLUX_TASK_INPUT_KVS;
-
-    shell_input_type_file_init (in);
-
-    /* Check if user specified shell input */
-    if (shell_input_parse_type (in) < 0)
-        goto error;
-
-    if (shell->info->shell_rank == 0) {
-        /* can't use stdin in standalone, no kvs to write to */
-        if (!in->shell->standalone) {
-            if (in->stdin_type == FLUX_INPUT_TYPE_SERVICE) {
-                if (flux_shell_service_register (in->shell,
-                                                 "stdin",
-                                                 shell_input_stdin_cb,
-                                                 in) < 0)
-                    shell_die_errno (1, "flux_shell_service_register");
-
-                /* Do not add a completion reference for the stdin service, we
-                 * don't care if the user ever sends stdin */
-            }
-
-            if (shell_input_header (in) < 0)
-                goto error;
-
-            if (in->stdin_type == FLUX_INPUT_TYPE_FILE) {
-                if (shell_input_type_file_setup (in) < 0)
-                    goto error;
-                /* Ok to start fd watcher now since shell_input_header()
-                 *  synchronously write guest.input header.
-                 */
-                flux_watcher_start (in->stdin_file.w);
-            }
-        }
-    }
-
-    return in;
-error:
-    shell_input_destroy (in);
-    return NULL;
-}
-
-static int shell_input_init (flux_plugin_t *p,
-                             const char *topic,
-                             flux_plugin_arg_t *args,
-                             void *data)
-{
-    flux_shell_t *shell = flux_plugin_get_shell (p);
-    struct shell_input *in = shell_input_create (shell);
-    if (!in)
-        return -1;
-    if (flux_plugin_aux_set (p, "builtin.input", in,
-                            (flux_free_f) shell_input_destroy) < 0) {
-        shell_input_destroy (in);
-        return -1;
-    }
-    return 0;
-}
-
-/*  Return 1 if idset string 'set' contains the integer id.
- *  O/w, return 0, or -1 on failure to decode 'set'.
- */
-static int idset_string_contains (const char *set, uint32_t id)
-{
-    int rc;
-    struct idset *idset;
-    if (strcmp (set, "all") == 0)
-        return 1;
-    if (!(idset = idset_decode (set)))
-        return shell_log_errno ("idset_decode (%s)", set);
-    rc = idset_test (idset, id);
-    idset_destroy (idset);
-    return rc;
-}
-
-static void shell_task_input_kvs_input_cb (flux_future_t *f, void *arg)
-{
-    struct shell_task_input *task_input = arg;
-    struct shell_task_input_kvs *kp = &(task_input->input_kvs);
-    const char *entry;
-    json_t *o;
-    const char *name;
-    json_t *context;
-
-    if (flux_job_event_watch_get (f, &entry) < 0) {
-        if (errno == ENODATA)
-            goto done;
-        shell_die (1, "flux_job_event_watch_get: %s",
-                   future_strerror (f, errno));
-    }
-    if (!(o = eventlog_entry_decode (entry)))
-        shell_die_errno (1, "eventlog_entry_decode");
-    if (eventlog_entry_parse (o, NULL, &name, &context) < 0)
-        shell_die_errno (1, "eventlog_entry_parse");
-
-    if (!strcmp (name, "header")) {
-        /* Future: per-stream encoding */
-        kp->input_header_parsed = true;
-    }
-    else if (!strcmp (name, "data")) {
-        flux_shell_task_t *task = task_input->task;
-        const char *rank = NULL;
-        if (!kp->input_header_parsed)
-            shell_die (1, "stream data read before header");
-        if (iodecode (context, NULL, &rank, NULL, NULL, NULL) < 0)
-            shell_die (1, "malformed event context");
-        if (idset_string_contains (rank, task->rank) == 1) {
-            const char *stream;
-            char *data = NULL;
-            int len;
-            bool eof;
-            if (iodecode (context, &stream, NULL, &data, &len, &eof) < 0)
-                shell_die (1, "malformed event context");
-            if (len > 0) {
-                if (flux_subprocess_write (task->proc,
-                                           stream,
-                                           data,
-                                           len) < 0) {
-                    if (errno != EPIPE)
-                        shell_die_errno (1, "flux_subprocess_write");
-                    else
-                        eof = true; /* Pretend that we got eof */
-                }
-            }
-            if (eof) {
-                if (flux_subprocess_close (task->proc, stream) < 0)
-                    shell_die_errno (1, "flux_subprocess_close");
-                if (flux_job_event_watch_cancel (f) < 0)
-                    shell_die_errno (1, "flux_job_event_watch_cancel");
-            }
-            free (data);
-        }
-    }
-    json_decref (o);
-    flux_future_reset (f);
-    return;
-done:
-    shell_task_input_kvs_cleanup (kp);
-}
-
-static int shell_task_input_kvs_start (struct shell_task_input *ti)
-{
-    struct shell_task_input_kvs *kp = &(ti->input_kvs);
-    flux_future_t *f = NULL;
-    /*  Start watching kvs guest.input eventlog.
-     *  Since this function is called after shell initialization
-     *   barrier, we are guaranteed that input eventlog exists.
-     */
-    if (!(f = flux_job_event_watch (ti->in->shell->h,
-                                    ti->in->shell->info->jobid,
-                                    "guest.input",
-                                    0)))
-            shell_die_errno (1, "flux_job_event_watch");
-
-    if (flux_future_then (f, -1., shell_task_input_kvs_input_cb, ti) < 0) {
-        flux_future_destroy (f);
-        shell_die_errno (1, "flux_future_then");
-    }
-    kp->input_f = f;
-    return 0;
-}
-
-static struct shell_task_input *get_task_input (struct shell_input *in,
-                                                flux_shell_task_t *task)
-{
-    return &in->task_inputs[task->index];
-}
-
-static int shell_input_task_init (flux_plugin_t *p,
-                                  const char *topic,
-                                  flux_plugin_arg_t *args,
-                                  void *data)
-{
-    flux_shell_t *shell = flux_plugin_get_shell (p);
-    struct shell_input *in = flux_plugin_aux_get (p, "builtin.input");
-    struct shell_task_input *task_input;
-    flux_shell_task_t *task;
-
-    if (!shell || !in || !(task = flux_shell_current_task (shell)))
-        return -1;
-
-    task_input = get_task_input (in, task);
-    task_input->in = in;
-    task_input->task = task;
-
-    if (task_input->type == FLUX_TASK_INPUT_KVS) {
-        /* can't read stdin in standalone mode, no KVS to read from */
-        if (!task_input->in->shell->standalone
-            && shell_task_input_kvs_start (task_input) < 0)
-            shell_die_errno (1, "shell_input_start_task_watch");
-    }
-    return 0;
-}
-
-static int shell_input_task_exit (flux_plugin_t *p,
-                                  const char *topic,
-                                  flux_plugin_arg_t *args,
-                                  void *data)
-{
-    flux_shell_t *shell = flux_plugin_get_shell (p);
-    flux_shell_task_t *task = flux_shell_current_task (shell);
-    struct shell_input *in = flux_plugin_aux_get (p, "builtin.input");
-    struct shell_task_input *task_input;
-
-    if (!shell || !in || !task)
-        return -1;
-
-    task_input = get_task_input (in, task);
-    if (task_input->type == FLUX_TASK_INPUT_KVS
-        && task_input->input_kvs.input_f) {
-        if (flux_job_event_watch_cancel (task_input->input_kvs.input_f) < 0)
-            shell_log_errno ("flux_job_event_watch_cancel");
-    }
-    return 0;
-}
-
-struct shell_builtin builtin_input = {
-    .name = "input",
-    .init = shell_input_init,
-    .task_init = shell_input_task_init,
-    .task_exit = shell_input_task_exit
-};
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/shell/input/file.c b/src/shell/input/file.c
new file mode 100644
index 000000000000..c8df590e6c20
--- /dev/null
+++ b/src/shell/input/file.c
@@ -0,0 +1,185 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* file input handling
+ *
+ * Redirect stdin of tasks to a file.
+ *
+ */
+#define FLUX_SHELL_PLUGIN_NAME "input.file"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+#include <flux/core.h>
+
+#include "src/common/libioencode/ioencode.h"
+#include "ccan/str/str.h"
+
+#include "task.h"
+#include "internal.h"
+#include "builtins.h"
+#include "input/util.h"
+
+struct file_input {
+    flux_shell_t *shell;
+    const char *path;
+};
+
+static void file_input_destroy (struct file_input *fp)
+{
+    if (fp) {
+        int saved_errno = errno;
+        free (fp);
+        errno = saved_errno;
+    }
+}
+
+static struct file_input *file_input_create (flux_shell_t *shell,
+                                             const char *path)
+{
+    struct file_input *fp;
+
+    if (!(fp = calloc (1, sizeof (*fp))))
+        return NULL;
+
+    fp->shell = shell;
+    fp->path = path;
+
+    /*  Path will be opened separately in each task.
+     *  Ensure access here though so users get a single error message
+     *  before launching tasks.
+     */
+    if (access (fp->path, R_OK) < 0) {
+        shell_die_errno (1, "error opening input file '%s'", fp->path);
+        goto error;
+    }
+    return fp;
+error:
+    file_input_destroy (fp);
+    return NULL;
+}
+
+static int file_input_task_exec (flux_plugin_t *p,
+                                 const char *topic,
+                                 flux_plugin_arg_t *args,
+                                 void *data)
+{
+    struct file_input *fp = data;
+    int fd;
+
+    if ((fd = open (fp->path, O_RDONLY)) < 0) {
+        fprintf (stderr,
+                 "error opening input file '%s': %s",
+                 fp->path,
+                 strerror (errno));
+        exit (1);
+    }
+    if (dup2 (fd, STDIN_FILENO) < 0) {
+        fprintf (stderr, "dup2: %s", strerror (errno));
+        exit (1);
+    }
+    return 0;
+}
+
+/*  Init eventlog and add redirect event(s)
+ */
+static void input_eventlog_put_redirect (struct file_input *fp)
+{
+    json_t *context = NULL;
+
+    if (input_eventlog_init (fp->shell) < 0) {
+        shell_log_errno ("failed to initialize input eventlog");
+        return;
+    }
+    if (!(context = json_pack ("{s:s s:s s:s}",
+                               "stream", "stdin",
+                               "rank", "all",
+                               "path", fp->path))) {
+        shell_log_error ("failed to pack redirect eventlog entry");
+        goto error;
+    }
+    if (input_eventlog_put_event (fp->shell, "redirect", context) < 0) {
+        shell_log_errno ("failed to add redirect event to input eventlog");
+        goto error;
+    }
+error:
+    json_decref (context);
+    return;
+}
+
+static int file_input_init (flux_plugin_t *p,
+                            const char *topic,
+                            flux_plugin_arg_t *args,
+                            void *data)
+{
+    const char *type = NULL;
+    const char *path = NULL;
+    struct file_input *fp;
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+
+    if (!shell)
+        return -1;
+    if (flux_shell_getopt_unpack (shell,
+                                  "input",
+                                  "{s?{s?s s?s}}",
+                                  "stdin",
+                                   "type", &type,
+                                   "path", &path) < 0)
+        return -1;
+
+    if (!type || !streq (type, "file"))
+        return 0;
+
+    if (path == NULL) {
+        shell_log_error ("path for stdin file input not specified");
+        return -1;
+    }
+    if (!(fp = file_input_create (shell, path))
+        || flux_plugin_aux_set (p,
+                                NULL,
+                                fp,
+                                (flux_free_f) file_input_destroy) < 0) {
+        shell_log_error ("file input creation failed");
+        file_input_destroy (fp);
+        return -1;
+    }
+
+    if (flux_plugin_add_handler (p,
+                                 "task.exec",
+                                 file_input_task_exec,
+                                 fp) < 0)
+        return -1;
+
+    /* On shell rank 0, post a redirect event for all tasks.
+     * Any errors will be emitted in input_eventlog_put_redirect(),
+     *  and they are not fatal since this is just informational.
+     */
+    if (shell->info->shell_rank == 0)
+        input_eventlog_put_redirect (fp);
+
+    return 0;
+}
+
+struct shell_builtin builtin_file_input = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = file_input_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/input/kvs.c b/src/shell/input/kvs.c
new file mode 100644
index 000000000000..def01099c1ad
--- /dev/null
+++ b/src/shell/input/kvs.c
@@ -0,0 +1,208 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/*  If stdin type is "kvs", watch guest.input eventlog and send
+ *  input data to all local tasks.
+ */
+#define FLUX_SHELL_PLUGIN_NAME "input.kvs"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <unistd.h>
+#include <flux/core.h>
+
+#include "src/common/libidset/idset.h"
+#include "src/common/libeventlog/eventlog.h"
+#include "src/common/libioencode/ioencode.h"
+#include "ccan/str/str.h"
+
+#include "task.h"
+#include "svc.h"
+#include "internal.h"
+#include "builtins.h"
+
+struct task_input_kvs {
+    flux_shell_t *shell;
+    bool header_parsed;
+    flux_future_t *input_f;
+};
+
+static void task_input_kvs_destroy (struct task_input_kvs *kp)
+{
+    if (kp) {
+        int saved_errno = errno;
+        flux_future_destroy (kp->input_f);
+        free (kp);
+        errno = saved_errno;
+    }
+}
+
+static struct task_input_kvs *task_input_kvs_create (flux_shell_t *shell)
+{
+    struct task_input_kvs *kp;
+    if (!(kp = calloc (1, sizeof (*kp))))
+        return NULL;
+    kp->shell = shell;
+    return kp;
+}
+
+/*  Return 1 if idset string 'set' contains the integer id.
+ *  O/w, return 0, or -1 on failure to decode 'set'.
+ */
+static int idset_string_contains (const char *set, uint32_t id)
+{
+    int rc;
+    struct idset *idset;
+    if (streq (set, "all"))
+        return 1;
+    if (!(idset = idset_decode (set)))
+        return shell_log_errno ("idset_decode (%s)", set);
+    rc = idset_test (idset, id);
+    idset_destroy (idset);
+    return rc;
+}
+
+static void input_eventlog_cb (flux_future_t *f, void *arg)
+{
+    struct task_input_kvs *kp = arg;
+    const char *entry;
+    json_t *o;
+    const char *name;
+    json_t *context;
+
+    if (flux_job_event_watch_get (f, &entry) < 0) {
+        if (errno == ENODATA)
+            return;
+        shell_die (1,
+                   "flux_job_event_watch_get: %s",
+                   future_strerror (f, errno));
+    }
+    if (!(o = eventlog_entry_decode (entry)))
+        shell_die_errno (1, "eventlog_entry_decode");
+    if (eventlog_entry_parse (o, NULL, &name, &context) < 0)
+        shell_die_errno (1, "eventlog_entry_parse");
+
+    if (streq (name, "header")) {
+        /* Future: per-stream encoding */
+        kp->header_parsed = true;
+    }
+    else if (streq (name, "data")) {
+        flux_shell_task_t *task;
+        char *data = NULL;
+        const char *rank = NULL;
+        const char *stream = NULL;
+        int len;
+        bool eof;
+
+        if (!kp->header_parsed)
+            shell_die (1, "stream data read before header");
+
+        if (iodecode (context, &stream, &rank, &data, &len, &eof) < 0)
+            shell_die (1, "malformed input event context");
+
+        /*  broadcast input to all matching tasks
+         */
+        task = flux_shell_task_first (kp->shell);
+        while (task != NULL) {
+            if (idset_string_contains (rank, task->rank) == 1) {
+                if (len > 0) {
+                    if (flux_subprocess_write (task->proc,
+                                               stream,
+                                               data,
+                                               len) < 0) {
+                        if (errno != EPIPE)
+                            shell_die_errno (1, "flux_subprocess_write");
+                        else
+                            eof = true; /* Pretend that we got eof */
+                    }
+                }
+                if (eof) {
+                    if (flux_subprocess_close (task->proc, stream) < 0)
+                        shell_die_errno (1, "flux_subprocess_close");
+                }
+            }
+            task = flux_shell_task_next (kp->shell);
+        }
+        free (data);
+    }
+    json_decref (o);
+    flux_future_reset (f);
+    return;
+}
+
+static int task_input_kvs_start (struct task_input_kvs *kp)
+{
+    flux_future_t *f = NULL;
+
+    /*  Start watching kvs guest.input eventlog.
+     *  Since this function is called after shell initialization
+     *   barrier, we are guaranteed that input eventlog exists.
+     */
+    if (!(f = flux_job_event_watch (kp->shell->h,
+                                    kp->shell->info->jobid,
+                                    "guest.input",
+                                    0)))
+            shell_die_errno (1, "flux_job_event_watch");
+
+    if (flux_future_then (f, -1., input_eventlog_cb, kp) < 0) {
+        flux_future_destroy (f);
+        shell_die_errno (1, "flux_future_then");
+    }
+    kp->input_f = f;
+    return 0;
+}
+
+
+static int input_kvs_start (flux_plugin_t *p,
+                            const char *topic,
+                            flux_plugin_arg_t *args,
+                            void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    struct task_input_kvs *kp;
+    const char *type = "service";
+
+    /*  No need to watch kvs input eventlog if input mode is not "kvs"
+     *  or unset.
+     */
+    if (flux_shell_getopt_unpack (shell,
+                                  "input",
+                                  "{s?{s?s}}",
+                                  "stdin",
+                                   "type", &type) < 0)
+        return -1;
+
+    if (!streq (type, "service"))
+        return 0;
+
+
+    if (!(kp = task_input_kvs_create (shell))
+        || flux_plugin_aux_set (p,
+                                NULL,
+                                kp,
+                                (flux_free_f) task_input_kvs_destroy) < 0) {
+        task_input_kvs_destroy (kp);
+        return -1;
+    }
+    if (task_input_kvs_start (kp) < 0)
+        return -1;
+    return 0;
+}
+
+struct shell_builtin builtin_kvs_input = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .start = input_kvs_start,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/input/service.c b/src/shell/input/service.c
new file mode 100644
index 000000000000..0f7b3b440d53
--- /dev/null
+++ b/src/shell/input/service.c
@@ -0,0 +1,200 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/*  Start a standard input service on leader shell for shuttling input
+ *  data to the KVS guest.input eventlog.
+ */
+#define FLUX_SHELL_PLUGIN_NAME "input.service"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <unistd.h>
+#include <flux/core.h>
+
+#include "src/common/libidset/idset.h"
+#include "src/common/libioencode/ioencode.h"
+#include "ccan/str/str.h"
+
+#include "util.h"
+
+#include "task.h"
+#include "svc.h"
+#include "internal.h"
+#include "builtins.h"
+
+struct input_service {
+    flux_shell_t *shell;
+    struct idset *open_tasks;
+};
+
+void input_service_destroy (struct input_service *in)
+{
+    int saved_errno = errno;
+    idset_destroy (in->open_tasks);
+    free (in);
+    errno = saved_errno;
+}
+
+/*  Return true if idset b is a strict subset of a
+ */
+static bool is_subset (const struct idset *a, const struct idset *b)
+{
+    struct idset *isect = idset_intersect (a, b);
+    if (isect) {
+        bool result = idset_equal (isect, b);
+        idset_destroy (isect);
+        return result;
+    }
+    return false;
+}
+
+/*  Subtract idset 'b' from 'a', unless 'ranks' is all then clear 'a'.
+ */
+static int subtract_idset (struct idset *a,
+                           const char *ranks,
+                           struct idset *b)
+{
+    /*  Remove all tasks with EOF from open_tasks idset
+     */
+    if (streq (ranks, "all"))
+        return idset_clear_all (a);
+    else
+        return idset_subtract (a, b);
+}
+
+/* Convert 'iodecode' object to an valid RFC 24 data event.
+ * N.B. the iodecode object is a valid "context" for the event.
+ */
+static void input_service_stdin_cb (flux_t *h,
+                                    flux_msg_handler_t *mh,
+                                    const flux_msg_t *msg,
+                                    void *arg)
+{
+    struct input_service *in = arg;
+    bool eof = false;
+    const char *ranks;
+    struct idset *ids = NULL;
+    json_t *o;
+
+    if (flux_request_unpack (msg, NULL, "o", &o) < 0)
+        goto error;
+    if (idset_count (in->open_tasks) == 0) {
+        errno = EPIPE;
+        goto error;
+    }
+    if (iodecode (o, NULL, &ranks, NULL, NULL, &eof) < 0)
+        goto error;
+    if (!streq (ranks, "all")) {
+        /* Ensure that targeted tasks are still open.
+         * ("all" is treated as "all open")
+         */
+        if (!(ids = idset_decode (ranks)))
+            goto error;
+        if (!is_subset (in->open_tasks, ids)) {
+            errno = EPIPE;
+            goto error;
+        }
+    }
+    if (input_eventlog_put_event (in->shell, "data", o) < 0)
+        goto error;
+    if (eof && subtract_idset (in->open_tasks, ranks, ids) < 0)
+        shell_log_errno ("failed to remove '%s' from open tasks", ranks);
+    if (flux_respond (h, msg, NULL) < 0)
+        shell_log_errno ("flux_respond");
+    idset_destroy (ids);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        shell_log_errno ("flux_respond");
+    idset_destroy (ids);
+}
+
+struct input_service *input_service_create (flux_shell_t *shell)
+{
+    struct input_service *in;
+
+    if (!(in = calloc (1, sizeof (*in))))
+        return NULL;
+    in->shell = shell;
+    if (!(in->open_tasks = idset_create (0, IDSET_FLAG_AUTOGROW))
+        || idset_range_set (in->open_tasks,
+                            0,
+                            shell->info->total_ntasks - 1))
+        goto error;
+    if (flux_shell_service_register (in->shell,
+                                     "stdin",
+                                     input_service_stdin_cb,
+                                     in) < 0)
+        shell_die_errno (1, "flux_shell_service_register");
+
+    /* Do not add a completion reference for the stdin service, we
+     * don't care if the user ever sends stdin */
+
+    if (input_eventlog_init (shell) < 0)
+        goto error;
+
+    return in;
+error:
+    input_service_destroy (in);
+    return NULL;
+}
+
+static int input_service_init (flux_plugin_t *p,
+                               const char *topic,
+                               flux_plugin_arg_t *args,
+                               void *data)
+{
+    struct input_service *in;
+    const char *type = "service";
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+
+    /* Only active on shell rank 0
+     */
+    if (shell->info->shell_rank != 0)
+        return 0;
+
+    if (flux_shell_getopt_unpack (shell,
+                                  "input",
+                                  "{s?{s?s}}",
+                                  "stdin",
+                                   "type", &type) < 0)
+        return -1;
+
+    /* Check validity of input.stdin.type here. Only valid types currently
+     * are "service" and "file":
+     */
+    if (!streq (type, "service") && !streq (type, "file"))
+        return shell_log_errn (0, "input.stdin.type=%s invalid", type);
+
+    if (!streq (type, "service"))
+        return 0;
+
+    if (!(in = input_service_create (shell)))
+        return -1;
+    if (flux_plugin_aux_set (p,
+                             "builtin.input-service",
+                             in,
+                             (flux_free_f) input_service_destroy) < 0) {
+        input_service_destroy (in);
+        return -1;
+    }
+    return 0;
+}
+
+struct shell_builtin builtin_input_service = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = input_service_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/input/util.c b/src/shell/input/util.c
new file mode 100644
index 000000000000..244fb2e33d35
--- /dev/null
+++ b/src/shell/input/util.c
@@ -0,0 +1,143 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define FLUX_SHELL_PLUGIN_NAME "input.util"
+
+#include <flux/core.h>
+#include <flux/shell.h>
+
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libeventlog/eventlog.h"
+
+#include "util.h"
+#include "internal.h"
+
+static void input_put_kvs_completion (flux_future_t *f, void *arg)
+{
+    flux_shell_t *shell = arg;
+
+    if (flux_future_get (f, NULL) < 0)
+        /* failing to write stdin to input is a fatal error */
+        shell_die (1, "input_service_put_kvs: %s", strerror (errno));
+    flux_future_destroy (f);
+
+    if (flux_shell_remove_completion_ref (shell, "input.kvs") < 0)
+        shell_log_errno ("flux_shell_remove_completion_ref");
+}
+
+int input_eventlog_put_event (flux_shell_t *shell,
+                              const char *name,
+                              json_t *context)
+{
+    flux_t *h;
+    flux_kvs_txn_t *txn = NULL;
+    flux_future_t *f = NULL;
+    json_t *entry = NULL;
+    char *entrystr = NULL;
+    int saved_errno;
+    int rc = -1;
+
+    if (!(h = flux_shell_get_flux (shell)))
+        goto error;
+    if (!(entry = eventlog_entry_pack (0.0, name, "O", context)))
+        goto error;
+    if (!(entrystr = eventlog_entry_encode (entry)))
+        goto error;
+    if (!(txn = flux_kvs_txn_create ()))
+        goto error;
+    if (flux_kvs_txn_put (txn, FLUX_KVS_APPEND, "input", entrystr) < 0)
+        goto error;
+    if (!(f = flux_kvs_commit (h, NULL, 0, txn)))
+        goto error;
+    if (flux_future_then (f, -1, input_put_kvs_completion, shell) < 0)
+        goto error;
+    if (flux_shell_add_completion_ref (shell, "input.kvs") < 0) {
+        shell_log_errno ("flux_shell_remove_completion_ref");
+        goto error;
+    }
+    /* f memory responsibility of input_service_put_kvs_completion()
+     * callback */
+    f = NULL;
+    rc = 0;
+ error:
+    saved_errno = errno;
+    flux_kvs_txn_destroy (txn);
+    free (entrystr);
+    json_decref (entry);
+    flux_future_destroy (f);
+    errno = saved_errno;
+    return rc;
+}
+
+static int input_kvs_eventlog_init (flux_shell_t *shell, json_t *header)
+{
+    flux_kvs_txn_t *txn = NULL;
+    flux_future_t *f = NULL;
+    char *headerstr = NULL;
+    int saved_errno;
+    int rc = -1;
+
+    if (!(headerstr = eventlog_entry_encode (header)))
+        goto error;
+    if (!(txn = flux_kvs_txn_create ()))
+        goto error;
+    if (flux_kvs_txn_put (txn, FLUX_KVS_APPEND, "input", headerstr) < 0)
+        goto error;
+    if (!(f = flux_kvs_commit (shell->h, NULL, 0, txn)))
+        goto error;
+    /* Synchronously wait for kvs commit to complete to ensure
+     * guest.input exists before passing shell initialization barrier.
+     * This is required because tasks will immediately try to watch
+     * input eventlog on starting.
+     */
+    if (flux_future_get (f, NULL) < 0)
+        shell_die_errno (1, "failed to create input eventlog");
+    rc = 0;
+ error:
+    saved_errno = errno;
+    flux_kvs_txn_destroy (txn);
+    free (headerstr);
+    flux_future_destroy (f);
+    errno = saved_errno;
+    return rc;
+}
+
+int input_eventlog_init (flux_shell_t *shell)
+{
+    json_t *o = NULL;
+    int rc = -1;
+
+    if (!(o = eventlog_entry_pack (0,
+                                   "header",
+                                   "{s:i s:{s:s} s:{s:i} s:{}}",
+                                   "version", 1,
+                                   "encoding",
+                                    "stdin", "UTF-8",
+                                   "count",
+                                    "stdin", 1,
+                                   "options")))
+        goto error;
+    if (input_kvs_eventlog_init (shell, o) < 0) {
+        shell_log_errno ("input_service_kvs_init");
+        goto error;
+    }
+    rc = 0;
+ error:
+    ERRNO_SAFE_WRAP (json_decref, o);
+    return rc;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/input/util.h b/src/shell/input/util.h
new file mode 100644
index 000000000000..e17dfbdac573
--- /dev/null
+++ b/src/shell/input/util.h
@@ -0,0 +1,32 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/*  Shared internal functions for shell input plugins
+ */
+#ifndef SHELL_INPUT_INTERNAL_H
+#define SHELL_INPUT_INTERNAL_H
+
+#include <jansson.h>
+#include <flux/shell.h>
+
+/*  Initialize the KVS input eventlog. This is done synchronously so that
+ *  the eventlog is ready to use after a successful return of this call.
+ */
+int input_eventlog_init (flux_shell_t *shell);
+
+/*  Put an input eventlog entry `name` defined in `context` to the KVS input
+ *  eventlog.
+ */
+int input_eventlog_put_event (flux_shell_t *shell,
+                              const char *name,
+                              json_t *context);
+
+#endif /* !SHELL_INPUT_INTERNAL_H */
+
diff --git a/src/shell/internal.h b/src/shell/internal.h
index cb2d594aba46..2ac7356e1781 100644
--- a/src/shell/internal.h
+++ b/src/shell/internal.h
@@ -11,18 +11,24 @@
 #ifndef _SHELL_INTERNAL_H
 #define _SHELL_INTERNAL_H
 
-#include <czmq.h>
 #include <flux/core.h>
 #include <flux/optparse.h>
 #include <flux/shell.h>
+#include <limits.h>
 
 #include "src/common/libutil/aux.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
 #include "plugstack.h"
 #include "events.h"
+#include "mustache.h"
 
 struct flux_shell {
     flux_jobid_t jobid;
     int broker_rank;
+    uid_t broker_owner;
+    char hostname [_POSIX_HOST_NAME_MAX + 1];
+    int protocol_fd[2];
 
     optparse_t *p;
     flux_t *h;
@@ -32,6 +38,7 @@ struct flux_shell {
     struct shell_svc *svc;
     zlist_t *tasks;
     flux_shell_task_t *current_task;
+    struct mustache_renderer *mr;
 
     struct plugstack *plugstack;
     struct shell_eventlogger *ev;
@@ -41,7 +48,7 @@ struct flux_shell {
     int rc;
 
     int verbose;
-    bool standalone;
+    int nosetpgrp;
 
     struct aux_item *aux;
 };
diff --git a/src/shell/jobspec.c b/src/shell/jobspec.c
index a7e568228faa..5e39b6783a25 100644
--- a/src/shell/jobspec.c
+++ b/src/shell/jobspec.c
@@ -13,6 +13,7 @@
 #endif
 #include <flux/core.h>
 #include <jansson.h>
+#include "ccan/str/str.h"
 
 #include "jobspec.h"
 
@@ -49,7 +50,7 @@ static int parse_res_level (json_t *o,
     /* For jobspec version 1, expect exactly one array element per level.
      */
     if (json_unpack_ex (o, &loc_error, 0,
-                        "[{s:s s:i s?o}]",
+                        "{s:s s:i s?o}",
                         "type", &res.type,
                         "count", &res.count,
                         "with", &res.with) < 0) {
@@ -63,18 +64,158 @@ static int parse_res_level (json_t *o,
 void jobspec_destroy (struct jobspec *job)
 {
     if (job) {
+        /*  refcounts were incremented on environment, options */
+        json_decref (job->environment);
+        json_decref (job->options);
         json_decref (job->jobspec);
         free (job);
     }
 }
 
+static int recursive_parse_helper (struct jobspec *job,
+                                  json_t *curr_resource,
+                                  json_error_t *error,
+                                  int level,
+                                  int with_multiplier)
+{
+    size_t index;
+    json_t *value;
+    size_t size = json_array_size (curr_resource);
+    struct res_level res;
+    int curr_multiplier;
+
+    if (size == 0) {
+        set_error (error, "Malformed jobspec: resource entry is not a list");
+        return -1;
+    }
+
+    json_array_foreach (curr_resource, index, value) {
+        if (parse_res_level (value, level, &res, error) < 0) {
+            return -1;
+        }
+
+        curr_multiplier = with_multiplier * res.count;
+
+        if (streq (res.type, "node")) {
+            if (job->slot_count > 0) {
+                set_error (error, "node resource encountered after slot resource");
+                return -1;
+            }
+            if (job->cores_per_slot > 0) {
+                set_error (error, "node resource encountered after core resource");
+                return -1;
+            }
+            if (job->node_count > 0) {
+                set_error (error, "node resource encountered after node resource");
+                return -1;
+            }
+
+            job->node_count = curr_multiplier;
+        } else if (streq (res.type, "slot")) {
+            if (job->cores_per_slot > 0) {
+                set_error (error, "slot resource encountered after core resource");
+                return -1;
+            }
+            if (job->slot_count > 0) {
+                set_error (error, "slot resource encountered after slot resource");
+                return -1;
+            }
+
+            job->slot_count = curr_multiplier;
+
+            // Reset the multiplier since we are now looking
+            // to calculate the cores_per_slot value
+            curr_multiplier = 1;
+
+            // Check if we already encountered the `node` resource
+            if (job->node_count > 0) {
+                // N.B.: with a strictly enforced ordering of node then slot
+                // (with arbitrary non-core resources in between)
+                // the slots_per_node will always be a perfectly round integer
+                // (i.e., job->slot_count % job->node_count == 0)
+                job->slots_per_node = job->slot_count / job->node_count;
+            }
+        } else if (streq (res.type, "core")) {
+            if (job->slot_count < 1) {
+                set_error (error, "core resource encountered before slot resource");
+                return -1;
+            }
+            if (job->cores_per_slot > 0) {
+                set_error (error, "core resource encountered after core resource");
+                return -1;
+            }
+
+            job->cores_per_slot = curr_multiplier;
+            // N.B.: despite having found everything we were looking for (i.e.,
+            // node, slot, and core resources), we have to keep recursing to
+            // make sure their aren't additional erroneous node/slot/core
+            // resources in the jobspec
+        }
+
+        if (res.with != NULL) {
+            if (recursive_parse_helper (job,
+                                        res.with,
+                                        error,
+                                        level+1,
+                                        curr_multiplier)
+                < 0) {
+                return -1;
+            }
+        }
+
+        if (streq (res.type, "node")) {
+            if ((job->slot_count <= 0) || (job->cores_per_slot <= 0)) {
+                set_error (error,
+                           "node encountered without slot&core below it");
+                return -1;
+            }
+        } else if (streq (res.type, "slot")) {
+            if (job->cores_per_slot <= 0) {
+                set_error (error, "slot encountered without core below it");
+                return -1;
+            }
+        }
+    }
+    return 0;
+}
+
+/* This function requires that the jobspec resource ordering is the same as the
+ * ordering specified in V1, but it allows additional resources before and in
+ * between the V1 resources (i.e., node, slot, and core).  In shorthand, it
+ * requires that the jobspec follows the form ...->[node]->...->slot->...->core.
+ * Where `node` is optional, and `...` represents any non-V1
+ * resources. Additionally, this function also allows multiple resources at any
+ * level, as long as there is only a single node, slot, and core within the
+ * entire jobspec.
+ */
+static int recursive_parse_jobspec_resources (struct jobspec *job,
+                                              json_t *curr_resource,
+                                              json_error_t *error)
+{
+    if (curr_resource == NULL) {
+        set_error (error, "jobspec top-level resources empty");
+        return -1;
+    }
+
+    // Set node-related values to -1 ahead of time, if the recursive descent
+    // encounters node in the jobspec, it will overwrite these values
+    job->slots_per_node = -1;
+    job->node_count = -1;
+
+    int rc = recursive_parse_helper (job, curr_resource, error, 0, 1);
+
+    if ((rc == 0) && (job->cores_per_slot < 1)) {
+        set_error (error, "Missing core resource");
+        return -1;
+    }
+    return rc;
+}
+
 struct jobspec *jobspec_parse (const char *jobspec, json_error_t *error)
 {
     struct jobspec *job;
-    int version;
     json_t *tasks;
     json_t *resources;
-    struct res_level res[3];
 
     if (!(job = calloc (1, sizeof (*job)))) {
         set_error (error, "Out of memory");
@@ -82,9 +223,20 @@ struct jobspec *jobspec_parse (const char *jobspec, json_error_t *error)
     }
     if (!(job->jobspec = json_loads (jobspec, 0, error)))
         goto error;
+
+    /* N.B.: members of jobspec like environment and shell.options may
+     *  be modified with json_object_update_new() via the shell API
+     *  calls flux_shell_setenvf(3), flux_shell_unsetenv(3), and
+     *  flux_shell_setopt(3). Therefore, the refcount of these objects
+     *  is incremented during unpack (via the "O" specifier), so that
+     *  the objects have json_decref() called directly on them to
+     *  avoid potential leaks (the json_decref() of the outer jobspec
+     *  object itself doesn't seem to catch the changes to these inner
+     *  json_t * objects)
+     */
     if (json_unpack_ex (job->jobspec, error, 0,
-                        "{s:i s:o s:o s:{s:{s?:s s?:o s?:{s?:o}}}}",
-                        "version", &version,
+                        "{s:i s:o s:o s:{s?{s?s s?O s?{s?O}}}}",
+                        "version", &job->version,
                         "resources", &resources,
                         "tasks", &tasks,
                         "attributes",
@@ -94,50 +246,29 @@ struct jobspec *jobspec_parse (const char *jobspec, json_error_t *error)
                                 "shell", "options", &job->options) < 0) {
         goto error;
     }
-    if (version != 1) {
+    if (job->version != 1) {
         set_error (error, "Invalid jobspec version: expected 1 got %d",
-                   version);
+                   job->version);
         goto error;
     }
     if (job->environment && !json_is_object (job->environment)) {
         set_error (error, "attributes.system.environment is not object type");
         goto error;
     }
-    /* For jobspec version 1, expect either:
-     * - node->slot->core->NIL
-     * - slot->core->NIL
-     * Set job->slot_count and job->cores_per_slot.
+    /* Ensure that shell options and environment are never NULL, so a shell
+     * component or plugin may set a new option or environment var.
      */
-    memset (res, 0, sizeof (res));
-    if (parse_res_level (resources, 0, &res[0], error) < 0)
+    if ((!job->options && !(job->options = json_object ()))
+        || (!job->environment && !(job->environment = json_object ()))) {
+        set_error (error, "unable to create empty jobspec options/environment");
         goto error;
-    if (res[0].with && parse_res_level (res[0].with, 1, &res[1], error) < 0)
-        goto error;
-    if (res[1].with && parse_res_level (res[1].with, 2, &res[2], error) < 0)
-        goto error;
-    if (res[0].type != NULL && !strcmp (res[0].type, "slot")
-            && res[1].type != NULL && !strcmp (res[1].type, "core")
-            && res[1].with == NULL) {
-        job->slot_count = res[0].count;
-        job->cores_per_slot = res[1].count;
-        job->slots_per_node = -1; // unspecified
-    }
-    else if (res[0].type != NULL && !strcmp (res[0].type, "node")
-            && res[1].type != NULL && !strcmp (res[1].type, "slot")
-            && res[2].type != NULL && !strcmp (res[2].type, "core")
-            && res[2].with == NULL) {
-        job->slot_count = res[0].count * res[1].count;
-        job->cores_per_slot = res[2].count;
-        job->slots_per_node = res[1].count;
-    }
-    else {
-        set_error (error, "Unexpected resource hierarchy: %s->%s->%s%s",
-                   res[0].type ? res[0].type : "NULL",
-                   res[1].type ? res[1].type : "NULL",
-                   res[2].type ? res[2].type : "NULL",
-                   res[2].with ? "->..." : NULL);
+    }
+
+    if (recursive_parse_jobspec_resources (job, resources, error) < 0) {
+        // recursive_parse_jobspec_resources calls set_error
         goto error;
     }
+
     /* Set job->task_count
      */
     if (json_unpack_ex (tasks, NULL, 0,
diff --git a/src/shell/jobspec.h b/src/shell/jobspec.h
index 19652b32212f..9c4cb9e6b0ce 100644
--- a/src/shell/jobspec.h
+++ b/src/shell/jobspec.h
@@ -16,10 +16,12 @@
 
 struct jobspec {
     json_t *jobspec;
+    int version;                // jobspec version
     int task_count;             // number of tasks in job
     int slot_count;             // number of task slots
     int cores_per_slot;         // number of cores per task slot
     int slots_per_node;         // number of slots per node (-1=unspecified)
+    int node_count;             // number of nodes (-1=unspecified)
     json_t *command;
     const char *cwd;
     json_t *environment;
diff --git a/src/shell/kill.c b/src/shell/kill.c
index 52fbb57db22e..e9f123d797c2 100644
--- a/src/shell/kill.c
+++ b/src/shell/kill.c
@@ -12,6 +12,7 @@
  *
  * Handle 'shell-<id>.kill' events by forwarding signal to local tasks
  */
+#define FLUX_SHELL_PLUGIN_NAME "kill_event_handler"
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -49,7 +50,7 @@ static int kill_event_init (flux_plugin_t *p,
 }
 
 struct shell_builtin builtin_kill = {
-    .name = "kill_event_handler",
+    .name = FLUX_SHELL_PLUGIN_NAME,
     .init = kill_event_init,
 };
 
diff --git a/src/shell/log.c b/src/shell/log.c
index 6317fd89f3cb..5e5b7495bc52 100644
--- a/src/shell/log.c
+++ b/src/shell/log.c
@@ -20,13 +20,18 @@
  * in the shell at runtime.
  *
  */
+#define FLUX_SHELL_PLUGIN_NAME NULL
+
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <signal.h>
 #include <jansson.h>
 #include <flux/core.h>
 #include <flux/shell.h>
 
+#include "ccan/str/str.h"
+
 #include "info.h"
 #include "internal.h"
 #include "log.h"
@@ -77,7 +82,7 @@ static flux_plugin_arg_t *log_msg_args (int level,
                                         const char *msg)
 {
     int rc = -1;
-    int flags = FLUX_PLUGIN_ARG_IN | FLUX_PLUGIN_ARG_UPDATE;
+    int flags = FLUX_PLUGIN_ARG_IN;
     flux_plugin_arg_t *args = flux_plugin_arg_create ();
 
     if (!args)
@@ -138,9 +143,12 @@ static int log_event (int level,
     return rc;
 }
 
-static void send_logmsg (const char *buf, int level, const char *file, int line)
+static void send_logmsg (const char *buf,
+                         int level,
+                         const char *component,
+                         const char *file,
+                         int line)
 {
-    const char *component = plugstack_current_name (logger.shell->plugstack);
     if (logger.rank < 0 && logger.shell->info)
         logger.rank = logger.shell->info->shell_rank;
     if (log_event (level, logger.rank, component, file, line, buf) < 0)
@@ -162,32 +170,27 @@ static int errorcat (int errnum, int start, char *buf, size_t len)
     return n;
 }
 
-/*  Format message, printing an error to stderr on failure
+/*  Format message, appending a '+' if the buffer isn't large enough.
  *  If errnum > 0, then append result of strerror (errnum).
  */
-static int msgfmt (char *buf,
-                   size_t len,
-                   int errnum,
-                   const char *fmt,
-                   va_list ap)
+static void msgfmt (char *buf,
+                    size_t len,
+                    int errnum,
+                    const char *fmt,
+                    va_list ap)
 {
-    int rc = vsnprintf (buf, len, fmt, ap);
-    if ((rc < 0 || rc >= len) || errorcat (errnum, rc, buf, len) < 0) {
-        fprintf (stderr,
-                 "%s: unable to format log msg (%s): %s\n",
-                 logger.prog,
-                 fmt,
-                 strerror (errno));
-        return -1;
-    }
+    int rc;
+    if ((rc = vsnprintf (buf, len, fmt, ap)) >= len
+        || errorcat (errnum, rc, buf, len) < 0)
+        buf[len - 2] = '+';
     /*  Clean up trailing newline, pointless here
      */
-    if (buf[rc-1] == '\n')
-        buf[rc-1] = '\0';
-    return 0;
+    else if (rc > 0 && buf[rc - 1] == '\n')
+        buf[rc - 1] = '\0';
 }
 
-void flux_shell_log (int level,
+void flux_shell_log (const char *component,
+                     int level,
                      const char *file,
                      int line,
                      const char *fmt, ...)
@@ -195,70 +198,102 @@ void flux_shell_log (int level,
     char buf [4096];
     va_list ap;
     va_start (ap, fmt);
-    if (msgfmt (buf, sizeof (buf), 0, fmt, ap) == 0)
-        send_logmsg (buf, level, file, line);
+    msgfmt (buf, sizeof (buf), 0, fmt, ap);
+    send_logmsg (buf, level, component, file, line);
     va_end (ap);
 }
 
-int flux_shell_err (const char *file,
-                   int line,
-                   int errnum,
-                   const char *fmt, ...)
+/* llog compatible wrapper for flux_shell_log
+ */
+void shell_llog (void *arg,
+                 const char *file,
+                 int line,
+                 const char *func,
+                 const char *subsys,
+                 int level,
+                 const char *fmt,
+                 va_list ap)
+{
+    char buf [4096];
+    int buflen = sizeof (buf);
+    int n = vsnprintf (buf, buflen, fmt, ap);
+    if (n >= buflen) {
+        buf[buflen-1] = '\0';
+        buf[buflen-2] = '+';
+    }
+    flux_shell_log (subsys, level, file, line, "%s", buf);
+}
+
+int flux_shell_err (const char *component,
+                    const char *file,
+                    int line,
+                    int errnum,
+                    const char *fmt, ...)
 {
     char buf [4096];
     va_list ap;
     va_start (ap, fmt);
-    if (msgfmt (buf, sizeof (buf), errnum, fmt, ap) == 0)
-        send_logmsg (buf, FLUX_SHELL_ERROR, file, line);
+    msgfmt (buf, sizeof (buf), errnum, fmt, ap);
+    send_logmsg (buf, FLUX_SHELL_ERROR, component, file, line);
     va_end (ap);
     errno = errnum;
     return -1;
 }
 
-void flux_shell_fatal (const char *file,
+void flux_shell_raise (const char *type,
+                      int severity,
+                      const char *fmt, ...)
+{
+    flux_shell_t *shell = logger.shell;
+    flux_future_t *f;
+    char buf [4096];
+    va_list ap;
+
+    if (!shell || !shell->h || logger.exception_logged)
+        return;
+
+    va_start (ap, fmt);
+    msgfmt (buf, sizeof (buf), 0, fmt, ap);
+    va_end (ap);
+
+    if (!(f = flux_job_raise (shell->h,
+                              shell->info->jobid,
+                              "exec",
+                              0,
+                              buf))
+       || flux_future_get (f, NULL) < 0) {
+        fprintf (stderr,
+                 "flux-shell: failed to raise job exception: %s\n",
+                 flux_future_error_string (f));
+    }
+    else
+        shell_log_set_exception_logged ();
+    flux_future_destroy (f);
+}
+
+void flux_shell_fatal (const char *component,
+                       const char *file,
                        int line,
                        int errnum,
                        int exit_code,
                        const char *fmt, ...)
 {
     flux_shell_t *shell = logger.shell;
-    flux_future_t *f = NULL;
     char buf [4096];
     va_list ap;
 
     va_start (ap, fmt);
-    if (msgfmt (buf, sizeof (buf), errnum, fmt, ap) < 0)
-        sprintf (buf, "flux-shell: fatal error");
-    else
-        send_logmsg (buf, FLUX_SHELL_FATAL, file, line);
+    msgfmt (buf, sizeof (buf), errnum, fmt, ap);
+    send_logmsg (buf, FLUX_SHELL_FATAL, component, file, line);
     va_end (ap);
 
     /*  Attempt to kill any running tasks
      */
     flux_shell_killall (shell, SIGKILL);
 
-    /*
-     *  Only need to generate an exception if we have a broker connection
-     *   and are not in standalone mode. O/w, exits immediately below.
-     */
-    if (shell->h && !shell->standalone && !logger.exception_logged) {
-        /*  Raise an exec exception.
-         *  Wait synchronously for response to ensure exception is
-         *   received by job-manager before shell potentially exits.
-         */
-        if (!(f = flux_job_raise (shell->h,
-                                  shell->info->jobid,
-                                  "exec",
-                                  0,
-                                  buf))
-           || flux_future_get (f, NULL) < 0) {
-            fprintf (stderr,
-                     "flux-shell: failed to raise job exception: %s\n",
-                     flux_future_error_string (f));
-        }
-        else
-            shell_log_set_exception_logged ();
-    }
+    if (shell)
+        flux_shell_raise ("exec", 0, "%s", buf);
+
     exit (exit_code);
 }
 
@@ -295,18 +330,15 @@ static int log_setlevel (flux_shell_t *shell, const char *dest, int level)
 
 int flux_shell_log_setlevel (int level, const char *dest)
 {
-    if (level < FLUX_SHELL_FATAL || level > FLUX_SHELL_TRACE) {
+    if (level < FLUX_SHELL_QUIET || level > FLUX_SHELL_TRACE) {
         errno = EINVAL;
         return -1;
     }
 
     /*  Always set internal dispatch level.
-     *  In standalone mode, also set stderr level by default
      */
     if (level > logger.level)
         logger.level = level;
-    if (logger.shell->standalone)
-        logger.fp_level = level;
 
     if (dest == NULL)
         return 0;
@@ -315,7 +347,7 @@ int flux_shell_log_setlevel (int level, const char *dest)
      *   the severity level change.
      */
     if (dest != NULL) {
-        if (strcmp (dest, "stderr") == 0)
+        if (streq (dest, "stderr"))
             logger.fp_level = level;
         else
             return log_setlevel (logger.shell, dest, level);
@@ -329,6 +361,7 @@ int shell_log_init (flux_shell_t *shell, const char *progname)
     logger.level = FLUX_SHELL_NOTICE;
     logger.fp_level = FLUX_SHELL_NOTICE;
     logger.active = 0;
+    logger.exception_logged = 0;
     logger.fp = stderr;
     logger.rank = -1;
     if (progname && !(logger.prog = strdup (progname)))
@@ -354,6 +387,7 @@ void shell_log_fini (void)
     fclose (logger.fp);
 }
 
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/shell/log.h b/src/shell/log.h
index 8d5183a35572..a3215ce3dec5 100644
--- a/src/shell/log.h
+++ b/src/shell/log.h
@@ -32,6 +32,17 @@ void shell_log_set_level (int level);
  */
 void shell_log_set_exception_logged (void);
 
+/*  Shell log function compatible with libutil llog interface
+ */
+void shell_llog (void *arg,
+                 const char *file,
+                 int line,
+                 const char *func,
+                 const char *subsys,
+                 int level,
+                 const char *fmt,
+                 va_list ap);
+
 #endif /* !_SHELL_RC_H */
 
 /* vi: ts=4 sw=4 expandtab
diff --git a/src/shell/lua.d/intel_mpi.lua b/src/shell/lua.d/intel_mpi.lua
deleted file mode 100644
index d5a3fae58fb9..000000000000
--- a/src/shell/lua.d/intel_mpi.lua
+++ /dev/null
@@ -1,28 +0,0 @@
--------------------------------------------------------------
--- Copyright 2020 Lawrence Livermore National Security, LLC
--- (c.f. AUTHORS, NOTICE.LLNS, COPYING)
---
--- This file is part of the Flux resource manager framework.
--- For details, see https://github.com/flux-framework.
---
--- SPDX-License-Identifier: LGPL-3.0
--------------------------------------------------------------
-
--- Set environment specific to Intel MPI
---
--- Intel PMI is an MPICH derivative that bootstraps with the PMI-1 wire
--- protocol, or if I_MPI_PMI_LIBRARY is set, a PMI library.
---
--- If the library is set, override it so it points to Flux's.
--- If the library is unset, do nothing.
---
--- (N.B. We could just unconditionally unset it, but that would prevent the
--- user from setting it in order to enable client side PMI tracing in the
--- Flux PMI library, enabled by setting FLUX_PMI_DEBUG=1)
-
-if shell.getenv ('I_MPI_PMI_LIBRARY') then
-    local f = require 'flux'.new ()
-    local libpmi = f:getattr ('conf.pmi_library_path')
-    shell.setenv ('I_MPI_PMI_LIBRARY', libpmi)
-end
-
diff --git a/src/shell/lua.d/mpi/spectrum.lua b/src/shell/lua.d/mpi/spectrum.lua
new file mode 100644
index 000000000000..341b09f90848
--- /dev/null
+++ b/src/shell/lua.d/mpi/spectrum.lua
@@ -0,0 +1,49 @@
+-------------------------------------------------------------
+-- Copyright 2020 Lawrence Livermore National Security, LLC
+-- (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+--
+-- This file is part of the Flux resource manager framework.
+-- For details, see https://github.com/flux-framework.
+--
+-- SPDX-License-Identifier: LGPL-3.0
+-------------------------------------------------------------
+
+-- Set environment specific to spectrum_mpi (derived from openmpi)
+--
+
+local posix = require 'posix'
+
+-- Clear all existing PMIX_ and OMPI_ values before setting our own
+shell.env_strip ("^PMIX_", "^OMPI_")
+
+-- Assumes the installation paths of Spectrum MPI on LLNL's Sierra
+shell.setenv ('OMPI_MCA_osc', "pt2pt")
+shell.setenv ('OMPI_MCA_pml', "yalla")
+shell.setenv ('OMPI_MCA_btl', "self")
+shell.setenv ('OMPI_MCA_coll_hcoll_enable', '0')
+
+-- Help find libcollectives.so
+shell.prepend_path ('LD_LIBRARY_PATH',
+                    '/opt/ibm/spectrum_mpi/lib/pami_port')
+shell.prepend_path ('LD_PRELOAD',
+                    '/opt/ibm/spectrum_mpi/lib/libpami_cudahook.so')
+
+plugin.register {
+    name = "spectrum",
+    handlers = {
+        {
+         topic = "task.init",
+         fn = function ()
+            local setrlimit = require 'posix'.setrlimit
+            -- Approximately `ulimit -Ss 10240`
+            -- Used to silence IBM MCM warnings
+            setrlimit ("stack", 10485760)
+            local rank = task.info.rank
+            task.setenv ("OMPI_COMM_WORLD_RANK", rank)
+            end
+        }
+    }
+}
+
+-- vi: ts=4 sw=4 expandtab
+
diff --git a/src/shell/lua.d/mvapich.lua b/src/shell/lua.d/mvapich.lua
deleted file mode 100644
index 461379393c14..000000000000
--- a/src/shell/lua.d/mvapich.lua
+++ /dev/null
@@ -1,49 +0,0 @@
--------------------------------------------------------------
--- Copyright 2020 Lawrence Livermore National Security, LLC
--- (c.f. AUTHORS, NOTICE.LLNS, COPYING)
---
--- This file is part of the Flux resource manager framework.
--- For details, see https://github.com/flux-framework.
---
--- SPDX-License-Identifier: LGPL-3.0
--------------------------------------------------------------
-
-local f, err = require 'flux'.new ()
-if not f then error (err) end
-
--- Lua implementation of dirname(3) to avoid pulling in posix module
-local function dirname (d)
-    if not d:match ("/") then return "." end
-    return d:match ("^(.*[^/])/.-$")
-end
-
-local function setenv_prepend (var, val)
-    local path = shell.getenv (var)
-    -- If path not already set, then set it to val
-    if not path then
-        shell.setenv (var, val)
-    -- O/w, if val is not already set in path, prepend it
-    elseif path:match ("^[^:]+") ~= val then
-        shell.setenv (var, val .. ':' .. path)
-    end
-    -- O/w, val already first in path. Do nothing
-end
-
-local libpmi = f:getattr ('conf.pmi_library_path')
-
-setenv_prepend ("LD_LIBRARY_PATH", dirname (libpmi))
-shell.setenv ("MPIRUN_NTASKS", shell.info.ntasks)
-shell.setenv ("MPIRUN_RSH_LAUNCH", 1)
-
-plugin.register {
-    name = "mvapich",
-    handlers = {
-        {
-         topic = "task.init",
-         fn = function ()
-            local rank = task.info.rank
-            task.setenv ("MPIRUN_RANK", rank)
-         end
-        }
-    }
-}
diff --git a/src/shell/lua.d/openmpi.lua b/src/shell/lua.d/openmpi.lua
index ff833e41100c..f63e7544a0ef 100644
--- a/src/shell/lua.d/openmpi.lua
+++ b/src/shell/lua.d/openmpi.lua
@@ -8,6 +8,28 @@
 -- SPDX-License-Identifier: LGPL-3.0
 -------------------------------------------------------------
 
-local f = require 'flux'.new ()
-local rundir  = f:getattr ('broker.rundir')
-shell.setenv ("OMPI_MCA_orte_tmpdir_base", rundir)
+if shell.options.mpi == "none" then return end
+
+-- OpenMPI needs a job-unique directory for vader shmem paths, otherwise
+-- multiple jobs per node may conflict (see flux-framework/flux-core#3649).
+
+-- Note: this plugin changes the path for files that openmpi usually shares
+-- using mmap (MAP_SHARED) from /dev/shm (tmpfs) to /tmp (maybe tmpfs).
+-- Performance may be affected if /tmp is provided by a disk-backed file system.
+
+plugin.register {
+    name = "openmpi",
+    handlers = {
+        {
+            topic = "shell.init",
+            fn = function ()
+                local tmpdir = shell.getenv ("FLUX_JOB_TMPDIR")
+                if tmpdir then
+                    shell.setenv ("OMPI_MCA_btl_vader_backing_directory", tmpdir)
+                end
+            end
+        }
+    }
+}
+
+-- vi:ts=4 sw=4 expandtab
diff --git a/src/shell/lua.d/spectrum.lua b/src/shell/lua.d/spectrum.lua
deleted file mode 100644
index a9885cad208b..000000000000
--- a/src/shell/lua.d/spectrum.lua
+++ /dev/null
@@ -1,87 +0,0 @@
--------------------------------------------------------------
--- Copyright 2020 Lawrence Livermore National Security, LLC
--- (c.f. AUTHORS, NOTICE.LLNS, COPYING)
---
--- This file is part of the Flux resource manager framework.
--- For details, see https://github.com/flux-framework.
---
--- SPDX-License-Identifier: LGPL-3.0
--------------------------------------------------------------
-
--- Set environment specific to spectrum_mpi (derived from openmpi)
---
-if shell.options.mpi ~= "spectrum" then return end
-
-local posix = require 'posix'
-
-function prepend_path (env_var, path)
-    local val = shell.getenv (env_var)
-
-    -- If path is already in env_var, do nothing. We stick ":" on both
-    -- ends of the existing value so we can easily match exact paths
-    -- instead of possibly matching substrings of paths when trying
-    -- to match "zero or more" colons.
-    --
-    if val and ((":"..val..":"):match (":"..path..":")) then return end
-
-    if val == nil then
-       suffix = ''
-    else
-       suffix = ':'..val
-    end
-    shell.setenv (env_var, path..suffix)
-end
-
-local function strip_env_by_prefix (env, prefix)
-    --
-    -- Have to call env:get() to translate env object to Lua table
-    --  in order to use pairs() to iterate environment keys:
-    --
-    for k,v in pairs (env) do
-        if k:match("^"..prefix) then
-            shell.unsetenv (k)
-        end
-    end
-end
-
-local f = require 'flux'.new()
-local rundir = f:getattr ('broker.rundir')
-
-local env = shell.getenv()
-
--- Clear all existing PMIX_ and OMPI_ values before setting our own
-strip_env_by_prefix (env, "PMIX_")
-strip_env_by_prefix (env, "OMPI_")
-
--- Avoid shared memory segment name collisions
--- when flux instance runs >1 broker per node.
-shell.setenv ('OMPI_MCA_orte_tmpdir_base', rundir)
-
--- Assumes the installation paths of Spectrum MPI on LLNL's Sierra
-shell.setenv ('OMPI_MCA_osc', "pt2pt")
-shell.setenv ('OMPI_MCA_pml', "yalla")
-shell.setenv ('OMPI_MCA_btl', "self")
-shell.setenv ('OMPI_MCA_coll_hcoll_enable', '0')
-
--- Help find libcollectives.so
-prepend_path ('LD_LIBRARY_PATH', '/opt/ibm/spectrum_mpi/lib/pami_port')
-prepend_path ('LD_PRELOAD', '/opt/ibm/spectrum_mpi/lib/libpami_cudahook.so')
-
-plugin.register {
-    name = "spectrum",
-    handlers = {
-        {
-         topic = "task.init",
-         fn = function ()
-            local setrlimit = require 'posix'.setrlimit
-            -- Approximately `ulimit -Ss 10240`
-            -- Used to silence IBM MCM warnings
-            setrlimit ("stack", 10485760)
-            end
-        }
-    }
-}
-
-
--- vi: ts=4 sw=4 expandtab
-
diff --git a/src/shell/mpir/mpir.c b/src/shell/mpir/mpir.c
index 3812d3447fe7..d2ffb68d41e5 100644
--- a/src/shell/mpir/mpir.c
+++ b/src/shell/mpir/mpir.c
@@ -11,21 +11,24 @@
 /* MPIR_proctable service for job shell
  *
  */
+#define FLUX_SHELL_PLUGIN_NAME "mpir"
 
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+
 #include <unistd.h>
 #include <stdio.h>
 #include <string.h>
 #include <sys/types.h>
 
-#include <czmq.h>
 #include <jansson.h>
 
 #include <flux/core.h>
 #include <flux/shell.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
 #include "builtins.h"
 #include "mpir/proctable.h"
 
@@ -70,6 +73,7 @@ static int shell_rank (flux_shell_t *shell)
 }
 
 static int proctable_add_task (struct proctable *p,
+                               int broker_rank,
                                flux_shell_task_t *task)
 {
     flux_subprocess_t *proc;
@@ -90,6 +94,7 @@ static int proctable_add_task (struct proctable *p,
     }
 
     if (proctable_append_task (p,
+                               broker_rank,
                                hostname,
                                flux_cmd_arg (cmd, 0),
                                rank,
@@ -103,14 +108,23 @@ static int proctable_add_task (struct proctable *p,
 static struct proctable * local_proctable_create (flux_shell_t *shell)
 {
     flux_shell_task_t *task;
-    struct proctable *p = proctable_create ();
-    if (!p)
+    int broker_rank;
+    struct proctable *p;
+
+    if (flux_shell_rank_info_unpack (shell,
+                                     -1,
+                                     "{s:i}",
+                                     "broker_rank", &broker_rank) < 0) {
+        shell_log_errno ("failed to get broker rank of current shell");
         return NULL;
+    }
 
+    if (!(p = proctable_create ()))
+        return NULL;
     if (!(task = flux_shell_task_first (shell)))
         shell_log_errno ("No tasks?!");
     while (task) {
-        if (proctable_add_task (p, task) < 0)
+        if (proctable_add_task (p, broker_rank, task) < 0)
             goto err;
         task = flux_shell_task_next (shell);
     }
@@ -260,7 +274,7 @@ static int proctable_gather_insert (struct proctable_gather *pg,
                                     struct proctable *p)
 {
     /*  Insert one proctable into sorted position on the proctables list.
-     *   Then, check to determine if all proctables have been receieved
+     *   Then, check to determine if all proctables have been received
      *   and send response.
      */
     if (!zlistx_insert (pg->proctables, p, false))
@@ -385,7 +399,7 @@ static int mpir_service_init (flux_plugin_t *p,
 }
 
 struct shell_builtin builtin_mpir = {
-    .name = "mpir",
+    .name = FLUX_SHELL_PLUGIN_NAME,
     .init = mpir_service_init,
 };
 
diff --git a/src/shell/mpir/nodelist.c b/src/shell/mpir/nodelist.c
index 4f29a49e11af..6d35ae37d2a5 100644
--- a/src/shell/mpir/nodelist.c
+++ b/src/shell/mpir/nodelist.c
@@ -8,6 +8,30 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+/* nodelist.c - compressed encoding of a list of hostnames
+ *
+ * A nodelist is a pure JSON representation of a list of possibly
+ * repeating hostnames. The implementation takes advantage of the
+ * tendency to place a numeric suffix on hostanmes of large HPC
+ * clusters and uses the rangelist implementation to encode the
+ * suffixes of a common hostname prefix.
+ *
+ * A JSON nodelist is an array of entries (entries are called a
+ * "prefix list" in the code below), where each entry represents
+ *  one or more hosts. Entries can have the following form:
+ *
+ * - a single string represents one hostname
+ * - an array entry has 2 elements:
+ *   1. a common hostname prefix
+ *   2. a rangelist representing the set of suffixes
+ *      (see rangelist.c).
+ *      An empty string (no suffix) is represented as -1.
+ *
+ * For each prefix list the common prefix is combined with the
+ * rangelist-encoded suffixes to form the list of hosts.
+ *
+ */
+
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
@@ -15,7 +39,11 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include <stdlib.h>
-#include <czmq.h>
+#include <errno.h>
+#include <ctype.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "ccan/str/str.h"
 
 #include "nodelist.h"
 #include "rangelist.h"
@@ -81,7 +109,7 @@ static char *make_hostname (const char *prefix, int64_t n)
 {
     char *result = NULL;
     if (n >= 0) {
-        if (asprintf (&result, "%s%jd", prefix, n) < 0)
+        if (asprintf (&result, "%s%jd", prefix, (intmax_t)n) < 0)
             return NULL;
     }
     else if (n != RANGELIST_END)
@@ -131,6 +159,12 @@ static int hostname_split (char *name, int *suffix)
     int n = len - 1;
     while (n >= 0 && isdigit (name[n]))
         n--;
+    /*  Now advance past leading zeros (not including a final zero)
+     *  These will not be part of the suffix since they can't be represented
+     *  as an integer.
+     */
+    while (name[n+1] == '0' && name[n+2] != '\0')
+        n++;
     if (++n == len)
         return 0;
     *suffix = (int) strtol (name+n, NULL, 10);
@@ -150,17 +184,17 @@ static int nodelist_append_prefix_list (struct nodelist *nl,
 int nodelist_append (struct nodelist *nl, const char *host)
 {
     int suffix = -1;
-    char name [MAXHOSTNAMELEN+1];
+    char name [4096];
     struct prefix_list *pl = zlist_tail (nl->list);
 
-    if (strlen (host) > MAXHOSTNAMELEN) {
+    if (strlen (host) > sizeof(name) - 1) {
         errno = E2BIG;
         return -1;
     }
     strcpy (name, host);
     hostname_split (name, &suffix);
 
-    if (pl && strcmp (name, pl->prefix) == 0)
+    if (pl && streq (name, pl->prefix))
         return rangelist_append (pl->suffixes, suffix);
 
     if (!(pl = prefix_list_create (name, suffix))
@@ -175,7 +209,7 @@ int nodelist_append_list_destroy (struct nodelist *nl1, struct nodelist *nl2)
 {
     struct prefix_list *pl1 = zlist_tail (nl1->list);
     struct prefix_list *pl2 = zlist_pop (nl2->list);
-    if (strcmp (pl1->prefix, pl2->prefix) == 0) {
+    if (streq (pl1->prefix, pl2->prefix)) {
         rangelist_append_list (pl1->suffixes, pl2->suffixes);
         prefix_list_destroy (pl2);
         pl2 = zlist_pop (nl2->list);
@@ -225,9 +259,9 @@ static struct prefix_list *prefix_list_from_json (json_t *o)
     /*  Special case, string is a single host
      */
     if (json_is_string (o)) {
-        char name [MAXHOSTNAMELEN+1];
+        char name [4096];
         int suffix = -1;
-        strcpy (name, json_string_value (o));
+        strncpy (name, json_string_value (o), sizeof (name) - 1);
         hostname_split (name, &suffix);
         return prefix_list_create (name, suffix);
     }
diff --git a/src/shell/mpir/proctable.c b/src/shell/mpir/proctable.c
index 965bdcb70a47..4b5f660b4b43 100644
--- a/src/shell/mpir/proctable.c
+++ b/src/shell/mpir/proctable.c
@@ -8,12 +8,32 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+/* proctable.c - compressed, JSON encoded MPIR_proctable
+ *
+ * An MPIR proctable is an array of MPIR_PROCDESC entries, including
+ * a taskid, hostname, executable name, and PID for every task in
+ * parallel job. In order to reduce the transfer of data back to a
+ * frontend command, the MPIR proctable is encoded by the job shell
+ * using the compression techniques in rangelist.c and nodelist.c.
+ *
+ * The shell simply encodes the proctable as 4 separate lists, each
+ * encoded in the same order:
+ *
+ *  - nodes: the list of hostnames in 'nodelist' form
+ *  - executables: the list of executables in 'nodelist' form
+ *  - taskids: the list of task ids in 'rangelist' form
+ *  - pids: the list of process ids in 'rangelist' form
+ *  - ranks: (not used in MPIR_proctable) list of broker ranks
+ *
+ */
+
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
 
 #include <errno.h>
-#include <czmq.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "mpir/rangelist.h"
 #include "mpir/nodelist.h"
@@ -28,6 +48,7 @@ struct proctable {
     struct nodelist  *executables;
     struct rangelist *taskids;
     struct rangelist *pids;
+    struct rangelist *ranks;
 };
 
 void proctable_destroy (struct proctable *p)
@@ -37,6 +58,7 @@ void proctable_destroy (struct proctable *p)
         nodelist_destroy (p->executables);
         rangelist_destroy (p->taskids);
         rangelist_destroy (p->pids);
+        rangelist_destroy (p->ranks);
         free (p->mpir_proctable);
         if (p->strings)
             zhashx_destroy (&p->strings);
@@ -71,7 +93,8 @@ struct proctable * proctable_create (void)
         || !(p->nodes = nodelist_create ())
         || !(p->executables = nodelist_create ())
         || !(p->taskids = rangelist_create ())
-        || !(p->pids = rangelist_create ())) {
+        || !(p->pids = rangelist_create ())
+        || !(p->ranks = rangelist_create ())) {
         proctable_destroy (p);
         return NULL;
     }
@@ -79,6 +102,7 @@ struct proctable * proctable_create (void)
 }
 
 int proctable_append_task (struct proctable *p,
+                           int broker_rank,
                            const char *hostname,
                            const char *executable,
                            int taskid,
@@ -87,7 +111,8 @@ int proctable_append_task (struct proctable *p,
     if (nodelist_append (p->nodes, hostname) < 0
         || nodelist_append (p->executables, executable) < 0
         || rangelist_append (p->taskids, taskid) < 0
-        || rangelist_append (p->pids, pid) < 0)
+        || rangelist_append (p->pids, pid) < 0
+        || rangelist_append (p->ranks, broker_rank) < 0)
         return -1;
     return 0;
 }
@@ -100,6 +125,10 @@ int proctable_append_proctable_destroy (struct proctable *p1,
         || rangelist_append_list (p1->taskids, p2->taskids) < 0
         || rangelist_append_list (p1->pids, p2->pids) < 0)
         return -1;
+    if (p1->ranks
+        && p2->ranks
+        && rangelist_append_list (p1->ranks, p2->ranks) < 0)
+        return -1;
     p2->nodes = NULL;
     p2->executables = NULL;
     proctable_destroy (p2);
@@ -112,20 +141,23 @@ struct proctable * proctable_from_json (json_t *o)
     json_t *exe;
     json_t *taskids;
     json_t *pids;
+    json_t *ranks;
     struct proctable *p;
 
-    if (json_unpack (o, "{so so so so}",
+    if (json_unpack (o, "{so so so so s?o}",
                         "ids", &taskids,
                         "executables", &exe,
                         "hosts", &nodes,
-                        "pids", &pids) < 0)
+                        "pids", &pids,
+                        "ranks", &ranks) < 0)
         return NULL;
     if (!(p = proctable_alloc ()))
         return NULL;
     if (!(p->nodes = nodelist_from_json (nodes))
         || !(p->executables = nodelist_from_json (exe))
         || !(p->taskids = rangelist_from_json (taskids))
-        || !(p->pids = rangelist_from_json (pids))) {
+        || !(p->pids = rangelist_from_json (pids))
+        || (ranks && !(p->ranks = rangelist_from_json (ranks)))) {
         proctable_destroy (p);
         return NULL;
     }
@@ -156,7 +188,10 @@ json_t *proctable_to_json (struct proctable *p)
         || !(x = rangelist_to_json (p->taskids))
         || json_object_set_new (o, "ids", x) < 0
         || !(x = rangelist_to_json (p->pids))
-        || json_object_set_new (o, "pids", x) < 0) {
+        || json_object_set_new (o, "pids", x) < 0
+        || !(x = rangelist_to_json (p->ranks))
+        || json_object_set_new (o, "ranks", x) < 0) {
+        json_decref (x);
         json_decref (o);
         return NULL;
     }
@@ -244,6 +279,49 @@ int proctable_get_size (struct proctable *p)
     return rangelist_size (p->taskids);
 }
 
+struct idset *proctable_get_ranks (struct proctable *p,
+                                   const struct idset *taskids)
+{
+    struct idset *result;
+    int taskid = 0;
+    int64_t rank;
+    if (!p->ranks) {
+        errno = ENOSYS;
+        return NULL;
+    }
+    if (!(result = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        return NULL;
+    rank = rangelist_first (p->ranks);
+    while (rank != RANGELIST_END) {
+        if (!taskids || idset_test (taskids, taskid))
+            if (idset_set (result, (unsigned int) rank) < 0)
+                goto error;
+        rank = rangelist_next (p->ranks);
+        taskid++;
+    }
+    return result;
+error:
+    idset_destroy (result);
+    return NULL;
+}
+
+int proctable_get_broker_rank (struct proctable *p, int taskid)
+{
+    int id = 0;
+    int64_t rank;
+    if (!p->ranks) {
+        errno = ENOSYS;
+        return -1;
+    }
+    rank = rangelist_first (p->ranks);
+    while (rank != RANGELIST_END) {
+        if (id == taskid)
+            return (int) rank;
+        rank = rangelist_next (p->ranks);
+        id++;
+    }
+    return -1;
+}
 
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
diff --git a/src/shell/mpir/proctable.h b/src/shell/mpir/proctable.h
index 77138800154e..4cb25e27e62c 100644
--- a/src/shell/mpir/proctable.h
+++ b/src/shell/mpir/proctable.h
@@ -9,10 +9,16 @@
 \************************************************************/
 
 #include <jansson.h>
+#include <flux/idset.h>
 
 #ifndef HAVE_PROCTABLE_H
 #define HAVE_PROCTABLE_H
 
+/* MPIR_PROCDESC is defined in the MPIR Process Acquisition Interface
+ * Version 1:
+ *
+ * See https://www.mpi-forum.org/docs/mpir-specification-03-01-2018.pdf
+ */
 typedef struct {
     char *host_name;
     char *executable_name;
@@ -34,6 +40,7 @@ json_t * proctable_to_json (struct proctable *p);
 /*  Append information for one task to the proctable `p`.
  */
 int proctable_append_task (struct proctable *p,
+                           int broker_rank,
                            const char *hostname,
                            const char *executable,
                            int taskid,
@@ -59,6 +66,14 @@ int proctable_append_proctable_destroy (struct proctable *p1,
 MPIR_PROCDESC *proctable_get_mpir_proctable (struct proctable *p,
                                              int *sizep);
 
+/*  Return the idset of broker ranks which hold the taskids in the
+ *   provided idset. If taskids == NULL, then return all broker ranks.
+ */
+struct idset *proctable_get_ranks (struct proctable *p,
+                                   const struct idset *taskids);
+
+int proctable_get_broker_rank (struct proctable *p, int taskid);
+
 #endif
 
 /*
diff --git a/src/shell/mpir/ptrace.c b/src/shell/mpir/ptrace.c
index 743b672ce297..aeb0bc690968 100644
--- a/src/shell/mpir/ptrace.c
+++ b/src/shell/mpir/ptrace.c
@@ -18,6 +18,8 @@
  *   3. Add "sync=true" to the emitted `shell.start` event
  *       to indicate all tasks are now stopped in exec().
  */
+#define FLUX_SHELL_PLUGIN_NAME "ptrace"
+
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
@@ -26,12 +28,21 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/ptrace.h>
+#include <signal.h>
 
 #include <flux/core.h>
 #include <flux/shell.h>
 
 #include <jansson.h>
 
+// Try BSD request names if linux ones are not defined
+#ifndef PTRACE_TRACEME
+#define PTRACE_TRACEME PT_TRACE_ME
+#endif
+#ifndef PTRACE_DETACH
+#define PTRACE_DETACH PT_DETACH
+#endif
+
 #include "builtins.h"
 
 static int ptrace_traceme (flux_plugin_t *p,
@@ -46,7 +57,7 @@ static int ptrace_traceme (flux_plugin_t *p,
 
 int current_task_pid (flux_shell_t *shell)
 {
-    long pid = -1;
+    json_int_t pid = -1;
     if (flux_shell_task_info_unpack (flux_shell_current_task (shell),
                                      "{s:I}",
                                      "pid", &pid) < 0)
@@ -140,7 +151,7 @@ static int ptrace_init (flux_plugin_t *p,
 }
 
 struct shell_builtin builtin_ptrace = {
-    .name =      "ptrace",
+    .name =      FLUX_SHELL_PLUGIN_NAME,
     .init =      ptrace_init,
 };
 
diff --git a/src/shell/mpir/rangelist.c b/src/shell/mpir/rangelist.c
index 459f412f5692..a54dcc7bcefd 100644
--- a/src/shell/mpir/rangelist.c
+++ b/src/shell/mpir/rangelist.c
@@ -8,6 +8,39 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+/* rangelist.c - compressed encoding of a list of integers
+ *
+ * The rangelist encoding uses a combination of run-length, range
+ * and delta encoding to compress a possibly large set of integers
+ * into a somewhat compact JSON array.
+ *
+ * This implementation is meant for encoding data for the MPIR
+ * process table in a space efficient manner, and takes advantage
+ * of the fact that multiple PIDs and hostnames with numeric
+ * suffixes are usually adjacent (or repeated in the case of hostnames)
+ *
+ * A rangelist is an array of entries that follow these rules:
+ *
+ * - If an entry is a single integer then it represents a single number
+ *   which is delta encoded from the previous entry (or 0 if this is the
+ *   first entry in the rangelist).
+ *
+ * - If an entry is an array, it will have two or three elements. The first
+ *   element is delta encoded from the previous entry (or 0 if no previous
+ *   entry) and represents the start value for a set of integers.
+ *
+ *   - if the second element is > 0, then it represents a run-length
+ *     encoded set of integers beginning at start. E.g. [1234,4] represents
+ *     1234, 1235, 1236, 1237.
+ *
+ *   - if the second element is < 0, then it represents a number of
+ *     repeats of the same value, e.g. [18, -3] represents [18, 18, 18].
+ *
+ *   - if there is a third element, this indicates that the first two
+ *     elements are repeated N times, e.g. [1, -1], [1 -1], [1, -1] is
+ *     equivalent to [1, -1, 2].
+ */
+
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
@@ -15,7 +48,9 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include <stdlib.h>
-#include <czmq.h>
+#include <errno.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
 
 #include "rangelist.h"
 
@@ -223,6 +258,54 @@ json_t *range_to_json (struct range *r, int64_t base)
     return o;
 }
 
+static json_int_t range_json_val (json_t *array, size_t index)
+{
+    return json_integer_value (json_array_get (array, index));
+}
+
+static bool range_json_equal (json_t *a, json_t *b)
+{
+    if (range_json_val (a, 0) == range_json_val (b, 0)
+        && range_json_val (a, 1) == range_json_val (b, 1))
+        return true;
+    return false;
+}
+
+static int increment_range_repeat (json_t *range)
+{
+    json_t *val = NULL;
+    if (json_array_size (range) == 3) {
+        json_int_t repeat = range_json_val (range, 2);
+        if (!(val = json_integer (repeat + 1))
+            || json_array_set_new (range, 2, val) < 0)
+            goto error;
+        return 0;
+    }
+    if (!(val = json_integer (1))
+        || json_array_append_new (range, val) < 0)
+        goto error;
+    return 0;
+error:
+    json_decref (val);
+    return -1;
+}
+
+static bool check_previous_repeat (json_t *array, json_t *range)
+{
+    json_t *prev;
+    size_t size = json_array_size (array);
+    bool result = false;
+
+    if ((size = json_array_size (array)) == 0
+        || !(prev = json_array_get (array, size - 1))
+        || json_array_size (prev) < 2)
+        return false;
+    if ((result = range_json_equal (prev, range))
+        && increment_range_repeat (prev) < 0)
+        return false;
+    return result;
+}
+
 json_t *rangelist_to_json (struct rangelist *rl)
 {
     struct range *r;
@@ -233,7 +316,11 @@ json_t *rangelist_to_json (struct rangelist *rl)
     r = zlist_first (rl->ranges);
     while (r) {
         json_t *range = range_to_json (r, base);
-        if (!range || json_array_append_new (result, range) < 0) {
+        if (!range)
+            goto error;
+        if (check_previous_repeat (result, range))
+            json_decref (range);
+        else if (json_array_append_new (result, range) < 0) {
             json_decref (range);
             goto error;
         }
@@ -246,18 +333,24 @@ json_t *rangelist_to_json (struct rangelist *rl)
     return NULL;
 }
 
-static struct range *range_from_json (json_t *o, int64_t base)
+static struct range *range_from_json (json_t *o, int64_t base, int *repeat)
 {
     int64_t min;
     int64_t range = 0;
 
+    if (repeat)
+        *repeat = 0;
+
     if (json_is_array (o)) {
-        if (json_array_size (o) != 2) {
+        size_t size = json_array_size (o);
+        if (size < 2 || size > 3) {
             errno = EINVAL;
             return NULL;
         }
         min = json_integer_value (json_array_get (o, 0));
         range = json_integer_value (json_array_get (o, 1));
+        if (size == 3 && repeat != NULL)
+            *repeat = json_integer_value (json_array_get (o, 2));
     }
     else {
         if (!json_is_integer (o)) {
@@ -292,12 +385,24 @@ struct rangelist *rangelist_from_json (json_t *o)
     if (!(rl = rangelist_create ()))
         return NULL;
     json_array_foreach (o, i, val) {
-        struct range *r = range_from_json (val, base);
+        int repeat;
+        struct range *r = range_from_json (val, base, &repeat);
+
         if (!r || rangelist_append_range (rl, r) < 0) {
             range_destroy (r);
             goto err;
         }
         base = range_max (r);
+
+        /* Handle repeating ranges */
+        for (int j = 0; j < repeat; j++) {
+            if (!(r = range_from_json (val, base, NULL))
+                || rangelist_append_range (rl, r) < 0) {
+                range_destroy (r);
+                goto err;
+            }
+            base = range_max (r);
+        }
     }
     return rl;
 err:
diff --git a/src/shell/mpir/test/nodelist.c b/src/shell/mpir/test/nodelist.c
index 93022abd0d01..76d5fe420f10 100644
--- a/src/shell/mpir/test/nodelist.c
+++ b/src/shell/mpir/test/nodelist.c
@@ -43,6 +43,32 @@ static const char *test1 [] = {
     NULL,
 };
 
+static const char *test2 [] = {
+    "test01",
+    "test02",
+    "test09",
+    "test0201",
+    "test0202",
+    "test0203",
+    "test1200",
+    "test1201",
+    "test1202",
+    NULL
+};
+
+static const char *test3 [] = {
+    "foo008",
+    "foo008",
+    "foo009",
+    "foo009",
+    "foo010",
+    "foo010",
+    "foo011",
+    "foo011",
+    NULL
+};
+
+
 struct nodelist * nodelist_from_array (const char *names[])
 {
     struct nodelist *nl = nodelist_create ();
@@ -108,6 +134,8 @@ int main (int argc, char **argv)
     plan (NO_PLAN);
     do_test (test0);
     do_test (test1);
+    do_test (test2);
+    do_test (test3);
     test_append ();
     done_testing ();
     return 0;
diff --git a/src/shell/mpir/test/proctable.c b/src/shell/mpir/test/proctable.c
index 16c0674683d2..e5a1f7266707 100644
--- a/src/shell/mpir/test/proctable.c
+++ b/src/shell/mpir/test/proctable.c
@@ -17,37 +17,65 @@
 
 struct entry {
     const char *host;
+    int broker_rank;
     const char *executable;
     int taskid;
     int pid;
 };
 
 struct entry basic [] = {
-    { "foo0", "myapp", 0, 1234 },
-    { "foo0", "myapp", 1, 1235 },
-    { "foo0", "myapp", 2, 1236 },
-    { "foo0", "myapp", 3, 1237 },
-    { "foo1", "myapp", 4, 4589 },
-    { "foo1", "myapp", 5, 4590 },
-    { "foo1", "myapp", 6, 4591 },
-    { "foo1", "myapp", 7, 4592 },
-    { NULL, NULL, 0, 0 }
+    { "foo0", 0, "myapp", 0, 1234 },
+    { "foo0", 0, "myapp", 1, 1235 },
+    { "foo0", 0, "myapp", 2, 1236 },
+    { "foo0", 0, "myapp", 3, 1237 },
+    { "foo1", 1, "myapp", 4, 4589 },
+    { "foo1", 1, "myapp", 5, 4590 },
+    { "foo1", 1, "myapp", 6, 4591 },
+    { "foo1", 1, "myapp", 7, 4592 },
+    { NULL, 0, NULL, 0, 0 }
 };
 
+struct entry leadingzeros [] = {
+    { "foo00", 13, "myapp", 0, 1234 },
+    { "foo00", 13, "myapp", 1, 1235 },
+    { "foo00", 13, "myapp", 2, 1236 },
+    { "foo00", 13, "myapp", 3, 1237 },
+    { "foo01", 15, "myapp", 4, 4589 },
+    { "foo01", 15, "myapp", 5, 4590 },
+    { "foo01", 15, "myapp", 6, 4591 },
+    { "foo01", 15, "myapp", 7, 4592 },
+    { NULL, 0, NULL, 0, 0 }
+};
+
+struct entry moreleadingzeros [] = {
+    { "foo008", 8, "myapp", 0, 1234 },
+    { "foo008", 8, "myapp", 1, 1235 },
+    { "foo009", 9, "myapp", 2, 2689 },
+    { "foo009", 9, "myapp", 3, 2690 },
+    { "foo010", 10, "myapp", 4, 1236 },
+    { "foo010", 10, "myapp", 5, 1237 },
+    { "foo011", 11, "myapp", 6, 4589 },
+    { "foo011", 11, "myapp", 7, 4590 },
+    { "foo012", 12, "myapp", 8, 8591 },
+    { "foo012", 12, "myapp", 9, 8592 },
+    { NULL, 0, NULL, 0, 0 }
+};
+
+
 struct entry append1 [] = {
-    { "foo0", "myapp", 0, 1234 },
-    { "foo0", "myapp", 1, 1235 },
-    { "foo0", "myapp", 2, 1236 },
-    { "foo0", "myapp", 3, 1237 },
-    { NULL, NULL, 0, 0 },
+    { "foo0", 0, "myapp", 0, 1234 },
+    { "foo0", 0, "myapp", 1, 1235 },
+    { "foo0", 0, "myapp", 2, 1236 },
+    { "foo0", 0, "myapp", 3, 1237 },
+    { NULL, 0, NULL, 0, 0 },
 };
 
 struct entry append2 [] = {
-    { "foo1", "myapp", 4, 4589 },
-    { "foo1", "myapp", 5, 4590 },
-    { "foo1", "myapp", 6, 4591 },
-    { "foo1", "myapp", 7, 4592 },
-    { NULL, NULL, 0, 0 },
+    { "foo1", 1, "myapp", 4, 4589 },
+    { "foo1", 1, "myapp", 5, 4590 },
+    { "foo1", 1, "myapp", 6, 4591 },
+    { "foo1", 1, "myapp", 7, 4592 },
+    { NULL, 0, NULL, 0, 0 },
 };
 
 static struct proctable * proctable_test_create (struct entry e[])
@@ -57,12 +85,14 @@ static struct proctable * proctable_test_create (struct entry e[])
         BAIL_OUT ("proctable_create failed");
     while (e->host) {
         ok (proctable_append_task (p,
+                                   e->broker_rank,
                                    e->host,
                                    e->executable,
                                    e->taskid,
                                    e->pid) == 0,
-            "proctable_append [%s,%s,%d,%d]",
+            "proctable_append [%s,%d,%s,%d,%d]",
                                    e->host,
+                                   e->broker_rank,
                                    e->executable,
                                    e->taskid,
                                    e->pid);
@@ -71,28 +101,44 @@ static struct proctable * proctable_test_create (struct entry e[])
     return p;
 }
 
-static void proctable_check (struct proctable *p, struct entry e[])
+static int entry_count (const struct entry *e)
+{
+    int count = 0;
+    while (e->host) {
+        count++;
+        ++e;
+    }
+    return count;
+}
+
+static void proctable_check (struct proctable *p, const struct entry e[])
 {
     MPIR_PROCDESC *table = NULL;
     int size = 0;
+    int expected_size = entry_count (e);
 
     ok ((table = proctable_get_mpir_proctable (p, &size)) != NULL,
         "proctable_get_mpir_proctable");
-    ok (size == 8,
-        "proctable_get_mpir_proctable returned expected size");
-    ok ((size = proctable_get_size (p)) == 8,
+    ok (size == expected_size,
+        "proctable_get_mpir_proctable returned expected size=%d", size);
+    ok ((size = proctable_get_size (p)) == expected_size,
         "proctable is of expected size");
     ok (proctable_first_task (p) == e[0].taskid,
         "proctable_first_task works");
 
     for (int i = 0; i < size; i++) {
+        int broker_rank;
         is (table[i].host_name, e[i].host,
             "task%d: host is %s", i, table[i].host_name);
         is (table[i].executable_name, e[i].executable,
             "task%d: executable is %s", i, table[i].executable_name);
         ok (table[i].pid == e[i].pid,
             "task%d: pid is %d", i, table[i].pid);
+        broker_rank = proctable_get_broker_rank (p, i);
+        ok (broker_rank == e[i].broker_rank,
+            "task%d: broker rank is %d", i, broker_rank);
     }
+
 }
 
 static void dump_json (json_t *o)
@@ -102,11 +148,11 @@ static void dump_json (json_t *o)
     free (s);
 }
 
-static void test_basic (void)
+static void test_proctable (struct entry entries[])
 {
     json_t *o = NULL;
     struct proctable *p2 = NULL;
-    struct proctable *p = proctable_test_create (basic);
+    struct proctable *p = proctable_test_create (entries);
     if (!p)
         BAIL_OUT ("proctable_test_create failed");
     if (!(o = proctable_to_json (p)))
@@ -115,9 +161,11 @@ static void test_basic (void)
     if (!(p2 = proctable_from_json (o)))
         BAIL_OUT ("proctable_from_json failed");
 
-    proctable_check (p, basic);
-    proctable_check (p2, basic);
+    proctable_check (p, entries);
+    proctable_check (p2, entries);
 
+    proctable_destroy (p);
+    proctable_destroy (p2);
     json_decref (o);
 }
 
@@ -137,7 +185,9 @@ static void test_append (void)
 int main (int argc, char **argv)
 {
     plan (NO_PLAN);
-    test_basic ();
+    test_proctable (basic);
+    test_proctable (leadingzeros);
+    test_proctable (moreleadingzeros);
     test_append ();
     done_testing ();
     return 0;
diff --git a/src/shell/mpir/test/rangelist.c b/src/shell/mpir/test/rangelist.c
index 258c636985f8..3a42760ebc96 100644
--- a/src/shell/mpir/test/rangelist.c
+++ b/src/shell/mpir/test/rangelist.c
@@ -71,6 +71,47 @@ void test_rangelist_append_dups (void)
     rangelist_destroy (rl2);
 }
 
+void test_rangelist_append_range_dups (void)
+{
+    struct rangelist *rl = rangelist_create ();
+    ok (rangelist_append (rl, 1) == 0
+        && rangelist_append (rl, 1) == 0
+        && rangelist_append (rl, 1) == 0
+        && rangelist_append (rl, 1) == 0,
+        "rangelist_append 4 1s");
+    ok (rangelist_append (rl, 2) == 0
+        && rangelist_append (rl, 2) == 0
+        && rangelist_append (rl, 2) == 0
+        && rangelist_append (rl, 2) == 0,
+        "rangelist_append 4 2s");
+    ok (rangelist_append (rl, 3) == 0
+        && rangelist_append (rl, 3) == 0
+        && rangelist_append (rl, 3) == 0
+        && rangelist_append (rl, 3) == 0,
+        "rangelist_append 4 3s");
+    ok (rangelist_append (rl, 4) == 0
+        && rangelist_append (rl, 4) == 0
+        && rangelist_append (rl, 4) == 0
+        && rangelist_append (rl, 4) == 0,
+        "rangelist_append 4 4s");
+    rangelist_diag (rl);
+
+    json_t *o;
+    ok ((o = rangelist_to_json (rl)) != NULL,
+        "rangelist_to_json");
+
+    struct rangelist *rl2 = rangelist_from_json (o);
+    ok (rl2 != NULL,
+        "rangelist_from_json works size=%d",
+        rangelist_size (rl2));
+    ok (rangelist_size (rl2) == rangelist_size (rl),
+        "rangelist_size matches");
+
+    json_decref (o);
+    rangelist_destroy (rl);
+    rangelist_destroy (rl2);
+}
+
 void test_rangelist_basic (void)
 {
     struct rangelist *rl2;
@@ -158,6 +199,7 @@ int main (int argc, char **argv)
     test_rangelist_basic ();
     test_rangelist_append ();
     test_rangelist_append_dups ();
+    test_rangelist_append_range_dups ();
     done_testing ();
     return 0;
 }
diff --git a/src/shell/mustache.c b/src/shell/mustache.c
new file mode 100644
index 000000000000..37946a020a91
--- /dev/null
+++ b/src/shell/mustache.c
@@ -0,0 +1,146 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <string.h>
+
+#include "mustache.h"
+#include "src/common/libutil/llog.h"
+
+struct mustache_renderer {
+    mustache_tag_f tag_f;
+    void *tag_arg;
+
+    mustache_log_f llog;
+    void *llog_data;
+};
+
+void mustache_renderer_destroy (struct mustache_renderer *mr)
+{
+    free (mr);
+}
+
+
+struct mustache_renderer * mustache_renderer_create (mustache_tag_f tagfn,
+                                                     void *arg)
+{
+    struct mustache_renderer *mr = NULL;
+
+    if (!tagfn) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(mr = calloc (1, sizeof (*mr))))
+        return NULL;
+    mr->tag_f = tagfn;
+    mr->tag_arg = arg;
+    return (mr);
+}
+
+void mustache_renderer_set_log (struct mustache_renderer *mr,
+                                mustache_log_f log_f,
+                                void *log_arg)
+{
+    if (mr) {
+        mr->llog = log_f;
+        mr->llog_data = log_arg;
+    }
+}
+
+char *mustache_render (struct mustache_renderer *mr, const char *template)
+{
+    char name [1024];
+    size_t size;
+    char *result = NULL;
+    const char *pos = template;
+    FILE *fp = NULL;
+
+    if (mr == NULL || template == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(fp = open_memstream (&result, &size))) {
+        llog_error (mr, "open_memstream");
+        goto fail;
+    }
+    for (;;) {
+        int len;
+        char *end;
+
+        /*  Look for opening "{{"
+         */
+        char *start = strstr (pos, "{{");
+        if (start == NULL) {
+            /*  No more mustache tags, put rest of string and finish
+             */
+            if (fputs (pos, fp) < 0) {
+                llog_error (mr, "fputs(%s): %s", pos, strerror (errno));
+                goto fail;
+            }
+            break;
+        }
+        /*  Write any part of template from current position to next tag
+         */
+        if (start > pos && fwrite (pos, start - pos, 1, fp) == 0) {
+            llog_error (mr, "fwrite: %s", strerror (errno));
+            goto fail;
+        }
+        /*  Advance past opening braces
+         */
+        start += 2;
+
+        /*  Find end of template tag
+         */
+        end = strstr (start, "}}");
+        if (end == NULL) {
+            llog_error (mr, "mustache template error at pos=%d",
+                        (int)(start-template));
+            /*  Copy rest of template and exit
+             */
+            pos = start - 2;
+            if (fputs (pos, fp) < 0) {
+                llog_error (mr, "fputs(%s): %s", pos, strerror (errno));
+                goto fail;
+            }
+            break;
+        }
+        /*  Copy mustache tag into 'name' and substitute with callback
+         */
+        len = end - start;
+        memcpy (name, start, len);
+        name[len] = '\0';
+        if ((*mr->tag_f) (fp, name, mr->tag_arg) < 0) {
+            /*  If callback fails, just fail to expand the current mustache tag.
+             */
+            fprintf (fp, "{{%s}}", name);
+        }
+
+        /*  Advance past closing braces '}}' and continue processing
+         */
+        pos = end + 2;
+    }
+    fclose (fp);
+    return result;
+fail:
+    if (fp) {
+        fclose (fp);
+        free (result);
+    }
+    return strdup (template);
+}
+
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/mustache.h b/src/shell/mustache.h
new file mode 100644
index 000000000000..65a9dbe484c8
--- /dev/null
+++ b/src/shell/mustache.h
@@ -0,0 +1,60 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SHELL_MUSTACHE_H
+#define _SHELL_MUSTACHE_H
+
+#include <stdio.h>
+
+typedef void (*mustache_log_f) (void *arg,
+                                const char *file,
+                                int line,
+                                const char *func,
+                                const char *subsys,
+                                int level,
+                                const char *fmt,
+                                va_list args);
+
+/*  Mustache tag callback function prototype. This function is called
+ *   for any mustache tag 'name' found in the template by the mustache
+ *   template renderer. E.g. if {{foo}} then the tag callback will be
+ *   called with `tag == "foo"`. The function should just write the value
+ *   of `tag` to the FILE stream `fp` if found.
+ *
+ * This function should return < 0 on error, in which case the template
+ *   renderer will copy the unsubstituted mustache tag into the result.
+ */
+typedef int (*mustache_tag_f) (FILE *fp, const char *tag, void *arg);
+
+struct mustache_renderer;
+
+/*  Create a mustache renderer, with mustache tags expanded by tag
+ *   callback `cb`. (See callback definition above).
+ */
+struct mustache_renderer *mustache_renderer_create (mustache_tag_f cb,
+                                                    void *arg);
+
+void mustache_renderer_destroy (struct mustache_renderer *mr);
+
+/*  Set a custom logger for mustache renderer 'mr'.
+ */
+void mustache_renderer_set_log (struct mustache_renderer *mr,
+                                mustache_log_f log,
+                                void *log_data);
+                                
+/*  Render the mustache template 'template' with renderer 'mr'.
+ */
+char * mustache_render (struct mustache_renderer *mr, const char *template);
+
+#endif /* !_SHELL_MUSTACHE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/oom.c b/src/shell/oom.c
new file mode 100644
index 000000000000..2c4deb37918e
--- /dev/null
+++ b/src/shell/oom.c
@@ -0,0 +1,309 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* oom.c - log kernel oom kill events
+ *
+ * This is an no-op if the cgroup v2 memory controller is not set up.
+ */
+
+#define FLUX_SHELL_PLUGIN_NAME "oom"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/inotify.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <unistd.h>
+#ifdef HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <flux/core.h>
+#include <jansson.h>
+#include <assert.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/strstrip.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/errprintf.h"
+#include "src/common/libutil/parse_size.h"
+#include "ccan/str/str.h"
+
+#include "builtins.h"
+#include "internal.h"
+
+struct shell_oom {
+    flux_shell_t *shell;
+    char *memory_events_path;
+    int inotify_fd;
+    int watch_id;
+    flux_watcher_t *w;
+    unsigned long oom_kill;
+};
+
+/* Read the contents of 'path' into NULL terminated buffer.
+ * Caller must free.
+ */
+static char *read_file (const char *path)
+{
+    int fd;
+    char *buf;
+    if ((fd = open (path, O_RDONLY)) < 0)
+        return NULL;
+    if (read_all (fd, (void **)&buf) < 0) {
+        ERRNO_SAFE_WRAP (close, fd);
+        return NULL;
+    }
+    close (fd);
+    return buf;
+}
+
+/* Determine the cgroup v2 path to 'name' for for pid.
+ * Check access(2) to the file according to 'mode' mask.
+ * Caller must free.
+ */
+static char *get_cgroup_path (pid_t pid, const char *name, int mode)
+{
+    char tmp[1024];
+    char *cg;
+    char *path;
+
+    snprintf (tmp, sizeof (tmp), "/proc/%d/cgroup", (int)pid);
+    if (!(cg = read_file (tmp)))
+        return NULL;
+    if (!strstarts (cg, "0::")) // v2 always begins with 0::
+        goto eproto;
+    snprintf (tmp,
+              sizeof (tmp),
+              "/sys/fs/cgroup/%s/%s",
+              strstrip (cg + 3),
+              name);
+    if (!(path = realpath (tmp, NULL)))
+        goto error;
+    if (access (path, mode) < 0) {
+        ERRNO_SAFE_WRAP (free, path);
+        goto error;
+    }
+    free (cg);
+    return path;
+eproto:
+    errno = EPROTO;
+error:
+    ERRNO_SAFE_WRAP (free, cg);
+    return NULL;
+}
+
+static int get_cgroup_value (const char *name, char *buf, size_t len)
+{
+    char *path;
+    char *s = NULL;
+    int rc = -1;
+
+    if (!(path = get_cgroup_path (getpid (), name, R_OK))
+        || !(s = read_file (path)))
+        goto out;
+    if (snprintf (buf, len, "%s", strstrip (s)) >= len)
+        goto out;
+    rc = 0;
+out:
+    free (s);
+    free (path);
+    return rc;
+}
+
+static const char *get_cgroup_size (const char *name)
+{
+    uint64_t size;
+    char rawbuf[32];
+
+    if (get_cgroup_value (name, rawbuf, sizeof (rawbuf)) < 0
+        || parse_size (rawbuf, &size) < 0)
+        return "unknown";
+    return encode_size (size);
+}
+
+/* Parse 'name' from memory.events file.  Example content:
+ *   low 0
+ *   high 0
+ *   max 0
+ *   oom 0
+ *   oom_kill 0
+ *   oom_group_kill 0
+ */
+static int parse_memory_events (const char *s,
+                                const char *name,
+                                unsigned long *valp)
+{
+    char *argz = NULL;
+    size_t argz_len = 0;
+    int e;
+
+    if ((e = argz_create_sep (s, '\n', &argz, &argz_len)) != 0) {
+        errno = e;
+        return -1;
+    }
+    char *entry = NULL;
+    while ((entry = argz_next (argz, argz_len, entry))) {
+        if (strstarts (entry, name) && isblank (entry[strlen (name)])) {
+            unsigned long val;
+            char *endptr;
+            errno = 0;
+            val = strtoul (&entry[strlen (name) + 1], &endptr, 10);
+            if (errno == 0 && *endptr == '\0') {
+                *valp = val;
+                free (argz);
+                return 0;
+            }
+        }
+    }
+    free (argz);
+    errno = EPROTO;
+    return -1;
+}
+
+static void watch_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents,
+                      void *arg)
+{
+    struct shell_oom *oom = arg;
+    char evbuf[sizeof (struct inotify_event) + NAME_MAX + 1];
+    char *me;
+    unsigned long count;
+
+    /* Consume an event from inotify file descriptor.
+     * Ignore event contents since there is only one watch registered.
+     */
+    if (read (oom->inotify_fd, &evbuf, sizeof (evbuf)) < 0) {
+        shell_log_error ("error reading from inotify fd");
+        return;
+    }
+    /* Read memory.events.
+     */
+    if (!(me = read_file (oom->memory_events_path))
+        || parse_memory_events (me, "oom_kill", &count) < 0) {
+        shell_log_error ("error reading %s", oom->memory_events_path);
+        goto out;
+    }
+    /* If any new oom events have been recorded, log them.
+     */
+    if (oom->oom_kill < count) {
+        shell_log_error ("Memory cgroup out of memory: "
+                         "killed %lu task%s on %s.",
+                         count - oom->oom_kill,
+                         count - oom->oom_kill > 1 ? "s" : "",
+                         oom->shell->hostname);
+
+        shell_log_error ("memory.peak = %s", get_cgroup_size ("memory.peak"));
+
+        oom->oom_kill = count;
+    }
+out:
+    free (me);
+}
+
+static void oom_destroy (struct shell_oom *oom)
+{
+    if (oom) {
+        int saved_errno = errno;
+        flux_watcher_destroy (oom->w);
+        if (oom->watch_id >= 0)
+            inotify_rm_watch (oom->inotify_fd, oom->watch_id);
+        if (oom->inotify_fd >= 0)
+            close (oom->inotify_fd);
+        free (oom->memory_events_path);
+        free (oom);
+        errno = saved_errno;
+    }
+}
+
+static struct shell_oom *oom_create (flux_shell_t *shell,
+                                     char *path,
+                                     flux_error_t *errp)
+{
+    struct shell_oom *oom;
+
+    if (!shell) {
+        errprintf (errp, "plugin not initialized with shell");
+        return NULL;
+    }
+    if (!(oom = calloc (1, sizeof (*oom)))) {
+        errprintf (errp, "%s", strerror (errno));
+        return NULL;
+    }
+    oom->inotify_fd = -1;
+    oom->watch_id = -1;
+    oom->shell = shell;
+    if ((oom->inotify_fd = inotify_init1 (IN_NONBLOCK | IN_CLOEXEC)) < 0
+        || (oom->watch_id = inotify_add_watch (oom->inotify_fd,
+                                               path,
+                                               IN_MODIFY)) < 0) {
+        if (errno == EMFILE)
+            errprintf (errp,
+                       "max number of user inotify instances has been reached");
+        else
+            errprintf (errp, "error setting up inotify: %s", strerror (errno));
+        goto error;
+    }
+    if (!(oom->w = flux_fd_watcher_create (shell->r,
+                                           oom->inotify_fd,
+                                           FLUX_POLLIN,
+                                           watch_cb,
+                                           oom))) {
+        errprintf (errp,
+                   "error setting up inotify watcher: %s",
+                   strerror (errno));
+        goto error;
+    }
+    flux_watcher_start (oom->w);
+    oom->memory_events_path = path; // takes ownership
+    return oom;
+error:
+    oom_destroy (oom);
+    return NULL;
+}
+
+static int oom_init (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *arg,
+                     void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    struct shell_oom *oom;
+    flux_error_t error;
+    char *path;
+
+    // assume cgroup is not configured or not v2
+    if (!(path = get_cgroup_path (getpid (), "memory.events", R_OK)))
+        return 0;
+    if (!(oom = oom_create (shell, path, &error))) {
+        ERRNO_SAFE_WRAP (free, path);
+        shell_warn ("disabling oom detection: %s", error.text);
+        return 0;
+    }
+    if (flux_plugin_aux_set (p, "oom", oom, (flux_free_f)oom_destroy) < 0) {
+        oom_destroy (oom);
+        return -1;
+    }
+    shell_debug ("monitoring %s", path);
+    return 0;
+}
+
+struct shell_builtin builtin_oom = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = oom_init,
+};
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/shell/output.c b/src/shell/output.c
index c0e9594333e4..d439d3151c4c 100644
--- a/src/shell/output.c
+++ b/src/shell/output.c
@@ -27,18 +27,17 @@
  *   task sends an EOF for both stdout and stderr.
  * - completion reference also taken for each KVS commit, to ensure
  *   commits complete before shell exits
- * - all shells (even the leader) send I/O to the service with RPC
+ * - follower shells send I/O to the service with RPC
  * - Any errors getting I/O to the leader are logged by RPC completion
  *   callbacks.
  * - Any outstanding RPCs at shell_output_destroy() are synchronously waited for
  *   there (checked for error, then destroyed).
  * - Any outstanding file writes at shell_output_destroy() are
  *   synchronously waited for to complete.
- * - In standalone mode, the loop:// connector enables RPCs to work
- * - In standalone mode, output is written to the shell's stdout/stderr not KVS
  * - The number of in-flight write requests on each shell is limited to
  *   shell_output_hwm, to avoid matchtag exhaustion, etc. for chatty tasks.
  */
+#define FLUX_SHELL_PLUGIN_NAME "output"
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -46,17 +45,28 @@
 #include <stdio.h>
 #include <string.h>
 #include <jansson.h>
+#include <unistd.h>
+#include <fcntl.h>
 #include <flux/core.h>
 
 #include "src/common/libidset/idset.h"
 #include "src/common/libeventlog/eventlog.h"
+#include "src/common/libeventlog/eventlogger.h"
 #include "src/common/libioencode/ioencode.h"
+#include "src/common/libutil/parse_size.h"
+#include "ccan/str/str.h"
 
 #include "task.h"
 #include "svc.h"
 #include "internal.h"
 #include "builtins.h"
-#include "eventlogger.h"
+#include "log.h"
+
+#define SINGLEUSER_OUTPUT_LIMIT "1G"
+#define MULTIUSER_OUTPUT_LIMIT  "10M"
+#define OUTPUT_LIMIT_MAX        1073741824
+/* 104857600 = 100M */
+#define OUTPUT_LIMIT_WARNING    104857600
 
 enum {
     FLUX_OUTPUT_TYPE_TERM = 1,
@@ -72,43 +82,34 @@ struct shell_output_type_file {
     struct shell_output_fd *fdp;
     char *path;
     int label;
+    int flags;
 };
 
 struct shell_output {
     flux_shell_t *shell;
+    const char *kvs_limit_string;
+    size_t kvs_limit_bytes;
     struct eventlogger *ev;
     double batch_timeout;
     int refcount;
-    int eof_pending;
+    struct idset *active_shells;
     zlist_t *pending_writes;
     json_t *output;
     bool stopped;
     int stdout_type;
     int stderr_type;
+    size_t stdout_bytes;
+    size_t stderr_bytes;
     struct shell_output_type_file stdout_file;
     struct shell_output_type_file stderr_file;
     zhash_t *fds;
+    const char *stdout_buffer_type;
+    const char *stderr_buffer_type;
 };
 
 static const int shell_output_lwm = 100;
 static const int shell_output_hwm = 1000;
 
-/* Pause/resume output on 'stream' of 'task'.
- */
-static void shell_output_control_task (struct shell_task *task,
-                                       const char *stream,
-                                       bool stop)
-{
-    if (stop) {
-        if (flux_subprocess_stream_stop (task->proc, stream) < 0)
-            shell_log_errno ("flux_subprocess_stream_stop %d:%s", task->rank, stream);
-    }
-    else {
-        if (flux_subprocess_stream_start (task->proc, stream) < 0)
-            shell_log_errno ("flux_subprocess_stream_start %d:%s", task->rank, stream);
-    }
-}
-
 /* Pause/resume output for all tasks.
  */
 static void shell_output_control (struct shell_output *out, bool stop)
@@ -118,8 +119,14 @@ static void shell_output_control (struct shell_output *out, bool stop)
     if (out->stopped != stop) {
         task = zlist_first (out->shell->tasks);
         while (task) {
-            shell_output_control_task (task, "stdout", stop);
-            shell_output_control_task (task, "stderr", stop);
+            if (stop) {
+                flux_subprocess_stream_stop (task->proc, "stdout");
+                flux_subprocess_stream_stop (task->proc, "stderr");
+            }
+            else {
+                flux_subprocess_stream_start (task->proc, "stdout");
+                flux_subprocess_stream_start (task->proc, "stderr");
+            }
             task = zlist_next (out->shell->tasks);
         }
         out->stopped = stop;
@@ -144,7 +151,7 @@ static int shell_output_term (struct shell_output *out)
             shell_log_errno ("eventlog_entry_parse");
             return -1;
         }
-        if (!strcmp (name, "data")) {
+        if (streq (name, "data")) {
             int output_type;
             FILE *f;
             const char *stream = NULL;
@@ -155,7 +162,7 @@ static int shell_output_term (struct shell_output *out)
                 shell_log_errno ("iodecode");
                 return -1;
             }
-            if (!strcmp (stream, "stdout")) {
+            if (streq (stream, "stdout")) {
                 output_type = out->stdout_type;
                 f = stdout;
             }
@@ -178,7 +185,7 @@ static int shell_output_redirect_stream (struct shell_output *out,
                                          const char *stream,
                                          const char *path)
 {
-    struct idset *idset;
+    struct idset *idset = NULL;
     json_t *entry = NULL;
     char *entrystr = NULL;
     int saved_errno, rc = -1;
@@ -212,7 +219,7 @@ static int shell_output_redirect_stream (struct shell_output *out,
                                        "stream", stream,
                                        "rank", rankptr,
                                        "path", path))) {
-        shell_log_errno ("eventlog_entry_create");
+        shell_log_errno ("eventlog_entry_pack");
         goto error;
     }
     if (!(entrystr = eventlog_entry_encode (entry))) {
@@ -230,6 +237,7 @@ static int shell_output_redirect_stream (struct shell_output *out,
     json_decref (entry);
     free (entrystr);
     free (rankptr);
+    idset_destroy (idset);
     errno = saved_errno;
     return rc;
 }
@@ -288,21 +296,30 @@ static int shell_output_kvs_init (struct shell_output *out, json_t *header)
     return rc;
 }
 
-static int entry_output_is_kvs (struct shell_output *out, json_t *entry)
+/*  Return true if entry is a kvs destination, false otherwise.
+ *  If true, then then stream and len will be set to the stream and
+ *   length of data in this entry.
+ */
+static bool entry_output_is_kvs (struct shell_output *out,
+                                 json_t *entry,
+                                 bool *stdoutp,
+                                 int *lenp,
+                                 bool *eofp)
 {
     json_t *context;
     const char *name;
     const char *stream;
+
     if (eventlog_entry_parse (entry, NULL, &name, &context) < 0) {
         shell_log_errno ("eventlog_entry_parse");
         return 0;
     }
-    if (!strcmp (name, "data")) {
-        if (iodecode (context, &stream, NULL, NULL, NULL, NULL) < 0) {
+    if (streq (name, "data")) {
+        if (iodecode (context, &stream, NULL, NULL, lenp, eofp) < 0) {
             shell_log_errno ("iodecode");
             return 0;
         }
-        if (!strcmp (stream, "stdout"))
+        if ((*stdoutp = streq (stream, "stdout")))
             return (out->stdout_type == FLUX_OUTPUT_TYPE_KVS);
         else
             return (out->stderr_type == FLUX_OUTPUT_TYPE_KVS);
@@ -310,14 +327,53 @@ static int entry_output_is_kvs (struct shell_output *out, json_t *entry)
     return 0;
 }
 
+static bool check_kvs_output_limit (struct shell_output *out,
+                                    bool is_stdout,
+                                    int len)
+{
+    const char *stream;
+    size_t *bytesp;
+    size_t prev;
+
+    if (is_stdout) {
+        stream = "stdout";
+        bytesp = &out->stdout_bytes;
+    }
+    else {
+        stream = "stderr";
+        bytesp = &out->stderr_bytes;
+    }
+
+    prev = *bytesp;
+    *bytesp += len;
+
+    if (*bytesp > out->kvs_limit_bytes) {
+        /*  Only log an error when the threshold is reached.
+        */
+        if (prev <= out->kvs_limit_bytes)
+            shell_warn ("%s will be truncated, %s limit exceeded",
+                        stream,
+                        out->kvs_limit_string);
+        return true;
+    }
+    return false;
+}
+
 static int shell_output_kvs (struct shell_output *out)
 {
     json_t *entry;
     size_t index;
+    bool is_stdout;
+    int len;
+    bool eof;
+
     json_array_foreach (out->output, index, entry) {
-        if (entry_output_is_kvs (out, entry) &&
-            eventlogger_append_entry (out->ev, 0, "output", entry) < 0) {
-            return shell_log_errno ("eventlogger_append");
+        if (entry_output_is_kvs (out, entry, &is_stdout, &len, &eof)) {
+            bool truncate = check_kvs_output_limit (out, is_stdout, len);
+            if (!truncate || eof) {
+                if (eventlogger_append_entry (out->ev, 0, "output", entry) < 0)
+                    return shell_log_errno ("eventlogger_append");
+            }
         }
     }
     return 0;
@@ -338,6 +394,92 @@ static int shell_output_write_fd (int fd, const void *buf, size_t len)
     return n;
 }
 
+static int shell_output_label (struct shell_output_type_file *ofp,
+                               const char *rank)
+{
+    if (shell_output_write_fd (ofp->fdp->fd,  rank, strlen (rank)) < 0
+        || shell_output_write_fd (ofp->fdp->fd, ": ", 2) < 0)
+        return -1;
+    return 0;
+}
+
+static int shell_output_data (struct shell_output *out, json_t *context)
+{
+    struct shell_output_type_file *ofp;
+    int output_type;
+    const char *stream = NULL;
+    const char *rank = NULL;
+    char *data = NULL;
+    int len = 0;
+    int rc = -1;
+
+    if (iodecode (context, &stream, &rank, &data, &len, NULL) < 0) {
+        shell_log_errno ("iodecode");
+        return -1;
+    }
+    if (streq (stream, "stdout")) {
+        output_type = out->stdout_type;
+        ofp = &out->stdout_file;
+    }
+    else {
+        output_type = out->stderr_type;
+        ofp = &out->stderr_file;
+    }
+    if ((output_type == FLUX_OUTPUT_TYPE_FILE) && len > 0) {
+        if (ofp->label
+            && shell_output_label (ofp, rank) < 0)
+            goto out;
+        if (shell_output_write_fd (ofp->fdp->fd, data, len) < 0)
+            goto out;
+    }
+    rc = 0;
+out:
+    free (data);
+    return rc;
+}
+
+/*  Level prefix strings. Nominally, output log event 'level' integers
+ *   are Internet RFC 5424 severity levels. In the context of flux-shell,
+ *   the first 3 levels are equivalently "fatal" errors.
+ */
+static const char *levelstr[] = {
+    "FATAL", "FATAL", "FATAL", "ERROR", " WARN", NULL, "DEBUG", "TRACE"
+};
+
+static void shell_output_log (struct shell_output *out, json_t *context)
+{
+    const char *msg = NULL;
+    const char *file = NULL;
+    const char *component = NULL;
+    int rank = -1;
+    int line = -1;
+    int level = -1;
+    int fd = out->stderr_file.fdp->fd;
+    json_error_t error;
+
+    if (json_unpack_ex (context, &error, 0,
+                     "{ s?i s:i s:s s?s s?s s?i }",
+                     "rank", &rank,
+                     "level", &level,
+                     "message", &msg,
+                     "component", &component,
+                     "file", &file,
+                     "line", &line) < 0) {
+        /*  Ignore log messages that cannot be unpacked so we don't
+         *   log an error while logging.
+         */
+        return;
+    }
+    dprintf (fd, "flux-shell");
+    if (rank >= 0)
+        dprintf (fd, "[%d]", rank);
+    if (level >= 0 && level <= FLUX_SHELL_TRACE)
+        dprintf (fd, ": %s", levelstr [level]);
+    if (component)
+        dprintf (fd, ": %s", component);
+    dprintf (fd, ": %s\n", msg);
+}
+
 static int shell_output_file (struct shell_output *out)
 {
     json_t *entry;
@@ -350,66 +492,100 @@ static int shell_output_file (struct shell_output *out)
             shell_log_errno ("eventlog_entry_parse");
             return -1;
         }
-        if (!strcmp (name, "data")) {
-            struct shell_output_type_file *ofp;
-            int output_type;
-            const char *stream = NULL;
-            const char *rank = NULL;
-            char *data = NULL;
-            int len = 0;
-            if (iodecode (context, &stream, &rank, &data, &len, NULL) < 0) {
-                shell_log_errno ("iodecode");
+        if (streq (name, "data")) {
+            if (shell_output_data (out, context) < 0) {
+                shell_log_errno ("shell_output_data");
                 return -1;
             }
-            if (!strcmp (stream, "stdout")) {
-                output_type = out->stdout_type;
-                ofp = &out->stdout_file;
-            }
-            else {
-                output_type = out->stderr_type;
-                ofp = &out->stderr_file;
-            }
-            if ((output_type == FLUX_OUTPUT_TYPE_FILE) && len > 0) {
-                if (ofp->label) {
-                    char *buf = NULL;
-                    int buflen;
-                    if ((buflen = asprintf (&buf, "%s: ", rank)) < 0)
-                        return -1;
-                    if (shell_output_write_fd (ofp->fdp->fd, buf, buflen) < 0) {
-                        free (buf);
-                        return -1;
-                    }
-                    free (buf);
-                }
-                if (shell_output_write_fd (ofp->fdp->fd, data, len) < 0)
-                    return -1;
-            }
-            free (data);
         }
-    }
+        else if (streq (name, "log"))
+            shell_output_log (out, context);
+   }
     return 0;
 }
 
-/* Convert 'iodecode' object to an valid RFC 24 data event.
- * N.B. the iodecode object is a valid "context" for the event.
- */
-static void shell_output_write_cb (flux_t *h,
-                                   flux_msg_handler_t *mh,
-                                   const flux_msg_t *msg,
-                                   void *arg)
+static void output_truncation_warning (struct shell_output *out)
+{
+    bool warned = false;
+    if (out->stderr_type == FLUX_OUTPUT_TYPE_KVS
+        && out->stderr_bytes > out->kvs_limit_bytes) {
+        shell_warn ("stderr: %zu of %zu bytes truncated",
+                    out->stderr_bytes - out->kvs_limit_bytes,
+                    out->stderr_bytes);
+        warned = true;
+    }
+    if (out->stdout_type == FLUX_OUTPUT_TYPE_KVS &&
+        out->stdout_bytes > out->kvs_limit_bytes) {
+        shell_warn ("stdout: %zu of %zu bytes truncated",
+                    out->stdout_bytes - out->kvs_limit_bytes,
+                    out->stdout_bytes);
+        warned = true;
+    }
+    if (out->stderr_type == FLUX_OUTPUT_TYPE_KVS
+        && (out->stderr_bytes > OUTPUT_LIMIT_WARNING
+            && out->stderr_bytes <= OUTPUT_LIMIT_MAX)) {
+        shell_warn ("high stderr volume (%s), "
+                    "consider redirecting to a file next time "
+                    "(e.g. use --output=FILE)",
+                    encode_size (out->stderr_bytes));
+        warned = true;
+    }
+    if (out->stdout_type == FLUX_OUTPUT_TYPE_KVS
+        && (out->stdout_bytes > OUTPUT_LIMIT_WARNING
+            && out->stdout_bytes <= OUTPUT_LIMIT_MAX)) {
+        shell_warn ("high stdout volume (%s), "
+                    "consider redirecting to a file next time "
+                    "(e.g. use --output=FILE)",
+                    encode_size (out->stdout_bytes));
+        warned = true;
+    }
+    /* Ensure KVS output is flushed to eventlogger if a warning was issued:
+     */
+    if (warned)
+        shell_output_kvs (out);
+}
+
+static void shell_output_decref (struct shell_output *out,
+                                 flux_msg_handler_t *mh)
+{
+    if (--out->refcount == 0) {
+        output_truncation_warning (out);
+        if (mh)
+            flux_msg_handler_stop (mh);
+        if (flux_shell_remove_completion_ref (out->shell, "output.write") < 0)
+            shell_log_errno ("flux_shell_remove_completion_ref");
+
+        /* no more output is coming, flush the last batch of output */
+        if ((out->stdout_type == FLUX_OUTPUT_TYPE_KVS
+            || (out->stderr_type == FLUX_OUTPUT_TYPE_KVS))) {
+            if (eventlogger_flush (out->ev) < 0)
+                shell_log_errno ("eventlogger_flush");
+        }
+    }
+}
+
+static void shell_output_decref_shell_rank (struct shell_output *out,
+                                            int shell_rank,
+                                            flux_msg_handler_t *mh)
+{
+    if (idset_test (out->active_shells, shell_rank)
+        && idset_clear (out->active_shells, shell_rank) == 0)
+        shell_output_decref (out, mh);
+}
+
+static int shell_output_write_leader (struct shell_output *out,
+                                      const char *type,
+                                      int shell_rank,
+                                      json_t *o,
+                                      flux_msg_handler_t *mh) // may be NULL
 {
-    struct shell_output *out = arg;
-    bool eof = false;
-    json_t *o;
     json_t *entry;
 
-    if (shell_svc_allowed (out->shell->svc, msg) < 0)
-        goto error;
-    if (flux_request_unpack (msg, NULL, "o", &o) < 0)
-        goto error;
-    if (iodecode (o, NULL, NULL, NULL, NULL, &eof) < 0)
-        goto error;
-    if (!(entry = eventlog_entry_pack (0., "data", "O", o))) // increfs 'o'
+    if (streq (type, "eof")) {
+        shell_output_decref_shell_rank (out, shell_rank, mh);
+        return 0;
+    }
+    if (!(entry = eventlog_entry_pack (0., type, "O", o))) // increfs 'o'
         goto error;
     if (json_array_append_new (out->output, entry) < 0) {
         json_decref (entry);
@@ -437,20 +613,33 @@ static void shell_output_write_cb (flux_t *h,
         shell_log_error ("json_array_clear failed");
         goto error;
     }
-    if (eof) {
-        if (--out->eof_pending == 0) {
-            flux_msg_handler_stop (mh);
-            if (flux_shell_remove_completion_ref (out->shell, "output.write") < 0)
-                shell_log_errno ("flux_shell_remove_completion_ref");
-            /* no more output is coming, flush the last batch of
-             * output */
-            if ((out->stdout_type == FLUX_OUTPUT_TYPE_KVS
-                 || (out->stderr_type == FLUX_OUTPUT_TYPE_KVS))) {
-                if (eventlogger_flush (out->ev) < 0)
-                    shell_log_errno ("eventlogger_flush");
-            }
-        }
-    }
+    return 0;
+error:
+    return -1;
+}
+
+/* Convert 'iodecode' object to an valid RFC 24 data event.
+ * N.B. the iodecode object is a valid "context" for the event.
+ */
+static void shell_output_write_cb (flux_t *h,
+                                   flux_msg_handler_t *mh,
+                                   const flux_msg_t *msg,
+                                   void *arg)
+{
+    struct shell_output *out = arg;
+    int shell_rank;
+    json_t *o;
+    const char *type;
+
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:s s:i s:o}",
+                             "name", &type,
+                             "shell_rank", &shell_rank,
+                             "context", &o) < 0)
+        goto error;
+    if (shell_output_write_leader (out, type, shell_rank, o, mh) < 0)
+        goto error;
     if (flux_respond (out->shell->h, msg, NULL) < 0)
         shell_log_errno ("flux_respond");
     return;
@@ -463,7 +652,7 @@ static void shell_output_write_completion (flux_future_t *f, void *arg)
 {
     struct shell_output *out = arg;
 
-    if (flux_future_get (f, NULL) < 0)
+    if (flux_future_get (f, NULL) < 0 && errno != ENOSYS)
         shell_log_errno ("shell_output_write");
     zlist_remove (out->pending_writes, f);
     flux_future_destroy (f);
@@ -472,6 +661,40 @@ static void shell_output_write_completion (flux_future_t *f, void *arg)
         shell_output_control (out, false);
 }
 
+static int shell_output_write_type (struct shell_output *out,
+                                    char *type,
+                                    json_t *context)
+{
+    flux_future_t *f = NULL;
+    int shell_rank = out->shell->info->shell_rank;
+
+    if (shell_rank == 0) {
+        if (shell_output_write_leader (out, type, 0, context, NULL) < 0)
+            shell_log_errno ("shell_output_write_leader");
+    }
+    else {
+        if (!(f = flux_shell_rpc_pack (out->shell,
+                                       "write",
+                                        0,
+                                        0,
+                                        "{s:s s:i s:O}",
+                                        "name", type,
+                                        "shell_rank", shell_rank,
+                                        "context", context)))
+            goto error;
+        if (flux_future_then (f, -1, shell_output_write_completion, out) < 0)
+            goto error;
+        if (zlist_append (out->pending_writes, f) < 0)
+            shell_log_error ("zlist_append failed");
+        if (zlist_size (out->pending_writes) >= shell_output_hwm)
+            shell_output_control (out, true);
+    }
+    return 0;
+error:
+    flux_future_destroy (f);
+    return -1;
+}
+
 static int shell_output_write (struct shell_output *out,
                                int rank,
                                const char *stream,
@@ -479,32 +702,42 @@ static int shell_output_write (struct shell_output *out,
                                int len,
                                bool eof)
 {
-    flux_future_t *f = NULL;
+    int rc;
     json_t *o = NULL;
-    char rankstr[64];
+    char rankstr[13];
 
-    snprintf (rankstr, sizeof (rankstr), "%d", rank);
+    /* integer %d guaranteed to fit in 13 bytes
+     */
+    (void) snprintf (rankstr, sizeof (rankstr), "%d", rank);
     if (!(o = ioencode (stream, rankstr, data, len, eof))) {
         shell_log_errno ("ioencode");
         return -1;
     }
-
-    if (!(f = flux_shell_rpc_pack (out->shell, "write", 0, 0, "O", o)))
-        goto error;
-    if (flux_future_then (f, -1, shell_output_write_completion, out) < 0)
-        goto error;
-    if (zlist_append (out->pending_writes, f) < 0)
-        shell_log_error ("zlist_append failed");
+    rc = shell_output_write_type (out, "data", o);
     json_decref (o);
+    return rc;
+}
 
-    if (zlist_size (out->pending_writes) >= shell_output_hwm)
-        shell_output_control (out, true);
-    return 0;
-
-error:
-    flux_future_destroy (f);
-    json_decref (o);
-    return -1;
+static int shell_output_handler (flux_plugin_t *p,
+                                 const char *topic,
+                                 flux_plugin_arg_t *args,
+                                 void *arg)
+{
+    struct shell_output *out = arg;
+    int rank;
+    const char *stream;
+    const void *data;
+    size_t len;
+
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                                "{s:s s:i s:s%}",
+                                "stream", &stream,
+                                "rank", &rank,
+                                "data", &data, &len) < 0) {
+        shell_log_errno ("shell.output: flux_plugin_arg_unpack");
+        return -1;
+    }
+    return shell_output_write (out, rank, stream, data, len, len == 0);
 }
 
 static void shell_output_type_file_cleanup (struct shell_output_type_file *ofp)
@@ -516,12 +749,31 @@ static void shell_output_type_file_cleanup (struct shell_output_type_file *ofp)
 void shell_output_destroy (struct shell_output *out)
 {
     if (out) {
+        int shell_rank = out->shell->info->shell_rank;
         int saved_errno = errno;
+        flux_future_t *f = NULL;
+
+        if (shell_rank != 0) {
+            /* Nonzero shell rank: send EOF to leader shell to notify
+             *  that no more messages will be sent to shell.write
+             */
+            if (!(f = flux_shell_rpc_pack (out->shell,
+                                           "write",
+                                            0,
+                                            0,
+                                            "{s:s s:i s:{}}",
+                                            "name", "eof",
+                                            "shell_rank", shell_rank,
+                                            "context")))
+                shell_log_errno ("shell.write: eof");
+            flux_future_destroy (f);
+        }
+
         if (out->pending_writes) {
             flux_future_t *f;
 
-            while ((f = zlist_pop (out->pending_writes))) { // leader+follower
-                if (flux_future_get (f, NULL) < 0)
+            while ((f = zlist_pop (out->pending_writes))) { // follower only
+                if (flux_future_get (f, NULL) < 0 && errno != ENOSYS)
                     shell_log_errno ("shell_output_write");
                 flux_future_destroy (f);
             }
@@ -556,6 +808,7 @@ void shell_output_destroy (struct shell_output *out)
             zhash_destroy (&out->fds);
         }
         eventlogger_destroy (out->ev);
+        idset_destroy (out->active_shells);
         free (out);
         errno = saved_errno;
     }
@@ -575,11 +828,9 @@ static int shell_output_parse_type (struct shell_output *out,
                                     const char *typestr,
                                     int *typep)
 {
-    if (out->shell->standalone && !strcmp (typestr, "term"))
-        (*typep) = FLUX_OUTPUT_TYPE_TERM;
-    else if (!strcmp (typestr, "kvs"))
+    if (streq (typestr, "kvs"))
         (*typep) = FLUX_OUTPUT_TYPE_KVS;
-    else if (!strcmp (typestr, "file"))
+    else if (streq (typestr, "file"))
         (*typep) = FLUX_OUTPUT_TYPE_FILE;
     else
         return shell_log_errn (EINVAL,
@@ -588,39 +839,6 @@ static int shell_output_parse_type (struct shell_output *out,
     return 0;
 }
 
-/* handle mustache templates and the similar special cases */
-static char *shell_output_get_path (struct shell_output *out, const char *path)
-{
-    char *rv = NULL;
-    char *ptr;
-
-    /* replace {{id}} with jobid */
-    if ((ptr = strstr (path, "{{id}}"))) {
-        char buf[32];
-        int len, buflen, total_len;
-
-        len = strlen (path);
-        buflen = snprintf (buf, sizeof (buf), "%llu",
-                           (unsigned long long)out->shell->info->jobid);
-        /* -6 for {{id}}, +1 for NUL */
-        total_len = len - 6 + buflen + 1;
-        if (total_len > (PATH_MAX + 1)) {
-            errno = EOVERFLOW;
-            return NULL;
-        }
-        if (!(rv = calloc (1, total_len)))
-            return NULL;
-        memcpy (rv, path, ptr - path);
-        memcpy (rv + (ptr - path), buf, buflen);
-        memcpy (rv + (ptr - path) + buflen, ptr + 6, len - (ptr - path) - 6);
-    }
-    else {
-        if (!(rv = strdup (path)))
-            return NULL;
-    }
-    return rv;
-}
-
 static int
 shell_output_setup_type_file (struct shell_output *out,
                               const char *stream,
@@ -628,29 +846,38 @@ shell_output_setup_type_file (struct shell_output *out,
                               struct shell_output_type_file *ofp_copy)
 {
     const char *path = NULL;
+    const char *mode = "truncate";
 
-    if (flux_shell_getopt_unpack (out->shell, "output",
-                                  "{s:{s?:s}}",
-                                  stream, "path", &path) < 0)
+    if (flux_shell_getopt_unpack (out->shell,
+                                  "output",
+                                  "{s?s s:{s?s s?b}}",
+                                  "mode", &mode,
+                                  stream,
+                                    "path", &path,
+                                    "label", &ofp->label) < 0)
         return -1;
 
+    ofp->flags = O_CREAT | O_WRONLY;
+    if (streq (mode, "append"))
+        ofp->flags |= O_APPEND;
+    else if (streq (mode, "truncate"))
+        ofp->flags |= O_TRUNC;
+    else
+        shell_warn ("ignoring invalid output.mode=%s", mode);
+
     if (path == NULL) {
         shell_log_error ("path for %s file output not specified", stream);
         return -1;
     }
 
-    if (!(ofp->path = shell_output_get_path (out, path)))
-        return -1;
-
-    if (flux_shell_getopt_unpack (out->shell, "output",
-                                  "{s:{s?:b}}",
-                                  stream, "label", &(ofp->label)) < 0)
+    if (!(ofp->path = flux_shell_mustache_render (out->shell, path)))
         return -1;
 
     if (ofp_copy) {
         if (!(ofp_copy->path = strdup (ofp->path)))
             return -1;
         ofp_copy->label = ofp->label;
+        ofp_copy->flags = ofp->flags;
     }
 
     return 0;
@@ -665,7 +892,7 @@ static int shell_output_setup_type (struct shell_output *out,
         struct shell_output_type_file *ofp = NULL;
         struct shell_output_type_file *ofp_copy = NULL;
 
-        if (!strcmp (stream, "stdout"))
+        if (streq (stream, "stdout"))
             ofp = &(out->stdout_file);
         else
             ofp = &(out->stderr_file);
@@ -685,12 +912,12 @@ static int shell_output_check_alternate_output (struct shell_output *out)
     const char *stderr_typestr = NULL;
 
     if (flux_shell_getopt_unpack (out->shell, "output",
-                                  "{s?:{s?:s}}",
+                                  "{s?{s?s}}",
                                   "stdout", "type", &stdout_typestr) < 0)
         return -1;
 
     if (flux_shell_getopt_unpack (out->shell, "output",
-                                  "{s?:{s?:s}}",
+                                  "{s?{s?s}}",
                                   "stderr", "type", &stderr_typestr) < 0)
         return -1;
 
@@ -740,6 +967,44 @@ static int shell_output_check_alternate_output (struct shell_output *out)
     return 0;
 }
 
+static int parse_alternate_buffer_type (struct shell_output *out,
+                                        const char *stream,
+                                        const char **buffer_type_ptr)
+{
+    const char *buffer_type = NULL;
+
+    if (flux_shell_getopt_unpack (out->shell, "output",
+                                  "{s?{s?{s?s}}}",
+                                  stream,
+                                  "buffer",
+                                  "type", &buffer_type) < 0)
+        return -1;
+
+    if (buffer_type) {
+        if (strcasecmp (buffer_type, "none")
+            && strcasecmp (buffer_type, "line"))
+            shell_log_error ("invalid buffer type specified: %s",
+                             buffer_type);
+        else
+            (*buffer_type_ptr) = buffer_type;
+    }
+
+    return 0;
+}
+
+static int shell_output_check_alternate_buffer_type (struct shell_output *out)
+{
+    if (parse_alternate_buffer_type (out,
+                                     "stdout",
+                                     &out->stdout_buffer_type) < 0)
+        return -1;
+    if (parse_alternate_buffer_type (out,
+                                     "stderr",
+                                     &out->stderr_buffer_type) < 0)
+        return -1;
+    return 0;
+}
+
 static struct shell_output_fd *shell_output_fd_create (int fd)
 {
     struct shell_output_fd *fdp = calloc (1, sizeof (*fdp));
@@ -762,7 +1027,6 @@ static int shell_output_type_file_setup (struct shell_output *out,
                                          struct shell_output_type_file *ofp)
 {
     mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
-    int open_flags = O_CREAT | O_TRUNC | O_WRONLY;
     struct shell_output_fd *fdp = NULL;
     int saved_errno, fd = -1;
 
@@ -772,7 +1036,7 @@ static int shell_output_type_file_setup (struct shell_output *out,
         return 0;
     }
 
-    if ((fd = open (ofp->path, open_flags, mode)) < 0) {
+    if ((fd = open (ofp->path, ofp->flags, mode)) < 0) {
         shell_log_errno ("error opening output file '%s'", ofp->path);
         goto error;
     }
@@ -781,7 +1045,7 @@ static int shell_output_type_file_setup (struct shell_output *out,
         goto error;
 
     if (zhash_insert (out->fds, ofp->path, fdp) < 0) {
-        errno = ENOMEM;
+        errno = EEXIST;
         goto error;
     }
     zhash_freefn (out->fds, ofp->path, shell_output_fd_destroy);
@@ -796,7 +1060,7 @@ static int shell_output_type_file_setup (struct shell_output *out,
 }
 
 /* Write RFC 24 header event to KVS.  Assume:
- * - fixed base64 encoding for stdout, stderr
+ * - fixed utf-8 encoding for stdout, stderr
  * - no options
  * - no stdlog
  */
@@ -809,11 +1073,11 @@ static int shell_output_header (struct shell_output *out)
                              "{s:i s:{s:s s:s} s:{s:i s:i} s:{}}",
                              "version", 1,
                              "encoding",
-                               "stdout", "base64",
-                               "stderr", "base64",
+                               "stdout", "UTF-8",
+                               "stderr", "UTF-8",
                              "count",
-                               "stdout", out->shell->info->jobspec->task_count,
-                               "stderr", out->shell->info->jobspec->task_count,
+                               "stdout", out->shell->info->total_ntasks,
+                               "stderr", out->shell->info->total_ntasks,
                              "options");
     if (!o) {
         errno = ENOMEM;
@@ -825,13 +1089,10 @@ static int shell_output_header (struct shell_output *out)
             shell_log_errno ("shell_output_term_init");
     }
     /* emit initial output events.
-     * Call this as long as we're not standalone.
      */
-    if (!out->shell->standalone) {
-        if (shell_output_kvs_init (out, o) < 0) {
-            shell_log_errno ("shell_output_kvs_init");
-            goto error;
-        }
+    if (shell_output_kvs_init (out, o) < 0) {
+        shell_log_errno ("shell_output_kvs_init");
+        goto error;
     }
     rc = 0;
 error:
@@ -852,6 +1113,22 @@ static void output_unref (struct eventlogger *ev, void *arg)
     flux_shell_remove_completion_ref (out->shell, "output.txn");
 }
 
+static int output_eventlogger_reconnect (flux_plugin_t *p,
+                                         const char *topic,
+                                         flux_plugin_arg_t *args,
+                                         void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+
+    /* during a reconnect, response to event logging may not occur,
+     * thus output_unref() may not be called.  Clear all completion
+     * references to inflight transactions.
+     */
+
+    while (flux_shell_remove_completion_ref (shell, "output.txn") == 0);
+    return 0;
+}
+
 static int output_eventlogger_start (struct shell_output *out)
 {
     flux_t *h = flux_shell_get_flux (out->shell);
@@ -876,6 +1153,102 @@ static int output_eventlogger_start (struct shell_output *out)
     return 0;
 }
 
+static int log_output (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    struct shell_output *out = data;
+    int rc = 0;
+    int level = -1;
+    json_t *context = NULL;
+
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                                "{s:i}", "level", &level) < 0)
+        return -1;
+    if (level > FLUX_SHELL_NOTICE + out->shell->verbose)
+        return 0;
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN, "o", &context) < 0
+        || shell_output_write_type (out, "log", context) < 0) {
+        rc = -1;
+    }
+    return rc;
+}
+
+static int shell_lost (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    struct shell_output *out = data;
+    int shell_rank;
+
+    /*  A shell has been lost. We need to decref the output refcount by 1
+     *  since we'll never hear from that shell to avoid rank 0 shell from
+     *  hanging.
+     */
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:i}",
+                                "shell_rank", &shell_rank) < 0)
+        return shell_log_errno ("shell.lost: unpack of shell_rank failed");
+    shell_output_decref_shell_rank (out, shell_rank, NULL);
+    shell_debug ("lost shell rank %d", shell_rank);
+    return 0;
+}
+
+static int get_output_limit (struct shell_output *out)
+{
+    json_t *val = NULL;
+    uint64_t size;
+
+    /*  For single-user instances, cap at reasonable size limit.
+     *  O/w use the default multiuser output limit:
+     */
+    if (out->shell->broker_owner == getuid())
+        out->kvs_limit_string = SINGLEUSER_OUTPUT_LIMIT;
+    else
+        out->kvs_limit_string = MULTIUSER_OUTPUT_LIMIT;
+
+    if (flux_shell_getopt_unpack (out->shell,
+                                  "output",
+                                  "{s?o}",
+                                  "limit", &val) < 0) {
+        shell_log_error ("Unable to unpack shell output.limit");
+        return -1;
+    }
+    if (val != NULL) {
+        if (json_is_integer (val)) {
+            json_int_t limit = json_integer_value (val);
+            if (limit <= 0 || limit > OUTPUT_LIMIT_MAX) {
+                shell_log ("Invalid KVS output.limit=%ld", (long) limit);
+                return -1;
+            }
+            out->kvs_limit_bytes = (size_t) limit;
+            /*  Need a string representation of limit for errors
+             */
+            char *s = strdup (encode_size (out->kvs_limit_bytes));
+            if (s && flux_shell_aux_set (out->shell, NULL, s, free) < 0)
+                free (s);
+            else
+                out->kvs_limit_string = s;
+            return 0;
+        }
+        if (!(out->kvs_limit_string = json_string_value (val))) {
+            shell_log_error ("Unable to convert output.limit to string");
+            return -1;
+        }
+    }
+    if (parse_size (out->kvs_limit_string, &size) < 0
+        || size == 0
+        || size > OUTPUT_LIMIT_MAX) {
+        shell_log ("Invalid KVS output.limit=%s", out->kvs_limit_string);
+        return -1;
+    }
+    out->kvs_limit_bytes = (size_t) size;
+    return 0;
+}
+
 struct shell_output *shell_output_create (flux_shell_t *shell)
 {
     struct shell_output *out;
@@ -883,21 +1256,22 @@ struct shell_output *shell_output_create (flux_shell_t *shell)
     if (!(out = calloc (1, sizeof (*out))))
         return NULL;
     out->shell = shell;
-    if (out->shell->standalone) {
-        out->stdout_type = FLUX_OUTPUT_TYPE_TERM;
-        out->stderr_type = FLUX_OUTPUT_TYPE_TERM;
-    }
-    else {
-        out->stdout_type = FLUX_OUTPUT_TYPE_KVS;
-        out->stderr_type = FLUX_OUTPUT_TYPE_KVS;
-    }
+    out->stdout_type = FLUX_OUTPUT_TYPE_KVS;
+    out->stderr_type = FLUX_OUTPUT_TYPE_KVS;
+    out->stdout_buffer_type = "line";
+    out->stderr_buffer_type = "none";
 
+    if (get_output_limit (out) < 0)
+        goto error;
     if (shell_output_check_alternate_output (out) < 0)
         goto error;
+    if (shell_output_check_alternate_buffer_type (out) < 0)
+        goto error;
 
     if (!(out->pending_writes = zlist_new ()))
         goto error;
     if (shell->info->shell_rank == 0) {
+        int ntasks = out->shell->info->rankinfo.ntasks;
         if (output_type_requires_service (out->stdout_type)
             || output_type_requires_service (out->stderr_type)) {
             if (flux_shell_service_register (shell,
@@ -905,10 +1279,26 @@ struct shell_output *shell_output_create (flux_shell_t *shell)
                                              shell_output_write_cb,
                                              out) < 0)
                 goto error;
-            if (output_type_requires_service (out->stdout_type))
-                out->eof_pending += shell->info->jobspec->task_count;
-            if (output_type_requires_service (out->stderr_type))
-                out->eof_pending += shell->info->jobspec->task_count;
+
+            /*  The shell.output.write service needs to wait for all
+             *   remote shells and local tasks before the output destination
+             *   can be closed. Therefore, set a reference counter for
+             *   the number of remote shells (shell_size - 1), plus the
+             *   number of tasks on the leader shell.
+             *
+             *  Remote shells and local tasks will cause the refcount
+             *   to be decremented as they send EOF or exit.
+             */
+            out->refcount = (shell->info->shell_size - 1 + ntasks);
+
+            /*  Account for active shells to avoid double-decrement of
+             *  refcount when a shell exits prematurely
+             */
+            if (!(out->active_shells = idset_create (0, IDSET_FLAG_AUTOGROW))
+                || idset_range_set (out->active_shells,
+                                    0,
+                                    shell->info->shell_size - 1) < 0)
+                goto error;
             if (flux_shell_add_completion_ref (shell, "output.write") < 0)
                 goto error;
             if (!(out->output = json_array ())) {
@@ -923,11 +1313,13 @@ struct shell_output *shell_output_create (flux_shell_t *shell)
                 goto error;
             }
             if (out->stdout_type == FLUX_OUTPUT_TYPE_FILE) {
-                if (shell_output_type_file_setup (out, &(out->stdout_file)) < 0)
+                if (shell_output_type_file_setup (out,
+                                                  &(out->stdout_file)) < 0)
                     goto error;
             }
             if (out->stderr_type == FLUX_OUTPUT_TYPE_FILE) {
-                if (shell_output_type_file_setup (out, &(out->stderr_file)) < 0)
+                if (shell_output_type_file_setup (out,
+                                                  &(out->stderr_file)) < 0)
                     goto error;
             }
         }
@@ -942,24 +1334,91 @@ struct shell_output *shell_output_create (flux_shell_t *shell)
     return NULL;
 }
 
-static void task_output_cb (struct shell_task *task,
-                            const char *stream,
-                            void *arg)
+static int task_setup_buffering (struct shell_task *task,
+                                 const char *stream,
+                                 const char *buffer_type)
+{
+    /* libsubprocess defaults to line buffering, so we only need to
+     * handle != line case */
+    if (!strcasecmp (buffer_type, "none")) {
+        char buf[64];
+        snprintf (buf, sizeof (buf), "%s_LINE_BUFFER", stream);
+        if (flux_cmd_setopt (task->cmd, buf, "false") < 0) {
+            shell_log_errno ("flux_cmd_setopt");
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+static void task_line_output_cb (struct shell_task *task,
+                                 const char *stream,
+                                 void *arg)
 {
     struct shell_output *out = arg;
     const char *data;
     int len;
 
-    data = flux_subprocess_getline (task->proc, stream, &len);
+    len = flux_subprocess_getline (task->proc, stream, &data);
     if (len < 0) {
         shell_log_errno ("read %s task %d", stream, task->rank);
     }
     else if (len > 0) {
-        if (shell_output_write (out, task->rank, stream, data, len, false) < 0)
+        if (shell_output_write (out,
+                                task->rank,
+                                stream,
+                                data,
+                                len,
+                                false) < 0)
             shell_log_errno ("write %s task %d", stream, task->rank);
     }
     else if (flux_subprocess_read_stream_closed (task->proc, stream)) {
-        if (shell_output_write (out, task->rank, stream, NULL, 0, true) < 0)
+        if (shell_output_write (out,
+                                task->rank,
+                                stream,
+                                NULL,
+                                0,
+                                true) < 0)
+            shell_log_errno ("write eof %s task %d", stream, task->rank);
+    }
+}
+
+static void task_none_output_cb (struct shell_task *task,
+                                 const char *stream,
+                                 void *arg)
+{
+    struct shell_output *out = arg;
+    const char *data;
+    int len;
+
+    len = flux_subprocess_read_line (task->proc, stream, &data);
+    if (len < 0) {
+        shell_log_errno ("read line %s task %d", stream, task->rank);
+    }
+    else if (!len) {
+        /* stderr is unbuffered */
+        if ((len = flux_subprocess_read (task->proc, stream, &data)) < 0) {
+            shell_log_errno ("read %s task %d", stream, task->rank);
+            return;
+        }
+    }
+    if (len > 0) {
+        if (shell_output_write (out,
+                                task->rank,
+                                stream,
+                                data,
+                                len,
+                                false) < 0)
+            shell_log_errno ("write %s task %d", stream, task->rank);
+    }
+    else if (flux_subprocess_read_stream_closed (task->proc, stream)) {
+        if (shell_output_write (out,
+                                task->rank,
+                                stream,
+                                NULL,
+                                0,
+                                true) < 0)
             shell_log_errno ("write eof %s task %d", stream, task->rank);
     }
 }
@@ -972,24 +1431,53 @@ static int shell_output_task_init (flux_plugin_t *p,
     flux_shell_t *shell = flux_plugin_get_shell (p);
     struct shell_output *out = flux_plugin_aux_get (p, "builtin.output");
     flux_shell_task_t *task;
+    void (*output_cb)(struct shell_task *, const char *, void *);
 
     if (!shell || !out || !(task = flux_shell_current_task (shell)))
         return -1;
 
+    if (task_setup_buffering (task, "stdout", out->stdout_buffer_type) < 0)
+        return -1;
+    if (task_setup_buffering (task, "stderr", out->stderr_buffer_type) < 0)
+        return -1;
+
     if (output_type_requires_service (out->stdout_type)) {
+        if (!strcasecmp (out->stdout_buffer_type, "line"))
+            output_cb = task_line_output_cb;
+        else
+            output_cb = task_none_output_cb;
         if (flux_shell_task_channel_subscribe (task, "stdout",
-                                               task_output_cb, out) < 0)
+                                               output_cb, out) < 0)
             return -1;
     }
     if (output_type_requires_service (out->stderr_type)) {
+        if (!strcasecmp (out->stderr_buffer_type, "line"))
+            output_cb = task_line_output_cb;
+        else
+            output_cb = task_none_output_cb;
         if (flux_shell_task_channel_subscribe (task, "stderr",
-                                               task_output_cb, out) < 0)
+                                               output_cb, out) < 0)
             return -1;
     }
 
     return 0;
 }
 
+static int shell_output_task_exit (flux_plugin_t *p,
+                                   const char *topic,
+                                   flux_plugin_arg_t *args,
+                                   void *data)
+{
+    struct shell_output *out = flux_plugin_aux_get (p, "builtin.output");
+
+    /*  Leader shell: decrement output.write refcount for each exiting
+     *   task (in lieu of counting EOFs separately from stderr/out)
+     */
+    if (out->shell->info->shell_rank == 0)
+        shell_output_decref (out, NULL);
+    return 0;
+}
+
 static int shell_output_init (flux_plugin_t *p,
                               const char *topic,
                               flux_plugin_arg_t *args,
@@ -1004,13 +1492,35 @@ static int shell_output_init (flux_plugin_t *p,
         shell_output_destroy (out);
         return -1;
     }
+    if (flux_plugin_add_handler (p,
+                                 "shell.output",
+                                 shell_output_handler,
+                                 out) < 0) {
+        shell_output_destroy (out);
+        return -1;
+    }
+
+    /*  If stderr is redirected to file, be sure to also copy log messages
+     *   there as soon as file is opened
+     */
+    if (out->stderr_type == FLUX_OUTPUT_TYPE_FILE) {
+        shell_debug ("redirecting log messages to job output file");
+        if (flux_plugin_add_handler (p, "shell.log", log_output, out) < 0)
+            return shell_log_errno ("failed to add shell.log handler");
+        flux_shell_log_setlevel (FLUX_SHELL_QUIET, "eventlog");
+    }
+    if (flux_plugin_add_handler (p, "shell.lost", shell_lost, out) < 0)
+        return shell_log_errno ("failed to add shell.log handler");
+
     return 0;
 }
 
 struct shell_builtin builtin_output = {
-    .name = "output",
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .reconnect = output_eventlogger_reconnect,
     .init = shell_output_init,
-    .task_init = shell_output_task_init
+    .task_init = shell_output_task_init,
+    .task_exit = shell_output_task_exit,
 };
 
 /*
diff --git a/src/shell/plugstack.c b/src/shell/plugstack.c
index 337eaeff9505..dc37e53d139e 100644
--- a/src/shell/plugstack.c
+++ b/src/shell/plugstack.c
@@ -7,6 +7,7 @@
  *
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
+#define FLUX_SHELL_PLUGIN_NAME NULL
 
 #if HAVE_CONFIG_H
 #  include "config.h"
@@ -16,11 +17,11 @@
 #include <string.h>
 #include <glob.h>
 #include <dlfcn.h>
-#include <czmq.h>
 #include <stdarg.h>
 #include <flux/core.h>
 #include <flux/shell.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/iterators.h"
 
 #include "plugstack.h"
@@ -37,7 +38,6 @@ struct plugstack {
     zhashx_t *aux;      /* aux items to propagate to loaded plugins        */
     zlistx_t *plugins;  /* Ordered list of loaded plugins                  */
     zhashx_t *names;    /* Hash for lookup of plugins by name              */
-    zlistx_t *current;  /* stack holding current plugin in plugstack_call  */
 };
 
 void plugstack_unload_name (struct plugstack *st, const char *name)
@@ -108,7 +108,6 @@ void plugstack_destroy (struct plugstack *st)
     if (st) {
         int saved_errno = errno;
         zlistx_destroy (&st->plugins);
-        zlistx_destroy (&st->current);
         zhashx_destroy (&st->names);
         zhashx_destroy (&st->aux);
         free (st->searchpath);
@@ -119,8 +118,10 @@ void plugstack_destroy (struct plugstack *st)
 
 static void plugin_destroy (flux_plugin_t **pp)
 {
-    flux_plugin_destroy (*pp);
-    *pp = NULL;
+    if (pp) {
+        flux_plugin_destroy (*pp);
+        *pp = NULL;
+    }
 }
 
 struct plugstack * plugstack_create (void)
@@ -128,27 +129,15 @@ struct plugstack * plugstack_create (void)
     struct plugstack *st = calloc (1, sizeof (*st));
     if (!st
         || !(st->plugins = zlistx_new ())
-        || !(st->current = zlistx_new ())
         || !(st->names = zhashx_new ())
         || !(st->aux = zhashx_new ())) {
         plugstack_destroy (st);
         return NULL;
     }
-    zlistx_set_destructor (st->plugins, (czmq_destructor *) plugin_destroy);
+    zlistx_set_destructor (st->plugins, (zlistx_destructor_fn *) plugin_destroy);
     return (st);
 }
 
-const char *plugstack_current_name (struct plugstack *st)
-{
-    if (!st) {
-        errno = EINVAL;
-        return NULL;
-    }
-    if (!st->current)
-        return NULL;
-    return flux_plugin_get_name (zlistx_first (st->current));
-}
-
 /*  Copy the plugin list, unsetting the destructor so plugins aren't
  *   destroyed on destruction of the list copy.
  */
@@ -175,16 +164,12 @@ int plugstack_call (struct plugstack *st,
 
     p = zlistx_first (l);
     while (p) {
-        /*  Push plugin onto the current plugin stack */
-        void * item = zlistx_add_start (st->current, p);
         if (flux_plugin_call (p, name, args) < 0) {
             shell_log_error ("plugin '%s': %s failed",
-                             plugstack_current_name (st),
+                             flux_plugin_get_name (p),
                              name);
             rc = -1;
         }
-        /* Pop plugin from the current plugin stack */
-        zlistx_detach (st->current, item);
         p = zlistx_next (l);
     }
     zlistx_destroy (&l);
@@ -248,8 +233,13 @@ static int plugstack_glob (struct plugstack *st,
 {
     glob_t gl;
     int rc = -1;
+    int flags = 0;
+
+#ifdef GLOB_TILDE_CHECK
+    flags |= GLOB_TILDE_CHECK;
+#endif
 
-    rc = glob (pattern, GLOB_TILDE_CHECK, NULL, &gl);
+    rc = glob (pattern, flags, NULL, &gl);
     switch (rc) {
         case 0:
             rc = load_from_glob (st, &gl, conf);
diff --git a/src/shell/plugstack.h b/src/shell/plugstack.h
index fc6ba43f9fca..20284db5d565 100644
--- a/src/shell/plugstack.h
+++ b/src/shell/plugstack.h
@@ -61,10 +61,6 @@ int plugstack_call (struct plugstack *st,
                     const char *name,
                     flux_plugin_arg_t *args);
 
-/*  Return currently active plugin name, or NULL if not in plugstack
- */
-const char * plugstack_current_name (struct plugstack *st);
-
 #endif /* !_SHELL_PLUGSTACK_H */
 
 /* vi: ts=4 sw=4 expandtab
diff --git a/src/shell/pmi.c b/src/shell/pmi.c
deleted file mode 100644
index 78ba39ff2ffc..000000000000
--- a/src/shell/pmi.c
+++ /dev/null
@@ -1,525 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* builtin PMI-1 plugin for jobs
- *
- * Provide PMI-1 service so that an MPI or Flux job can bootstrap.
- * Much of the work is done by the PMI-1 wire protocol engine in
- * libpmi/simple_server.c and libsubprocess socketpair channels.
- *
- * At startup this module is registered as a builtin shell plugin under
- * the name "pmi"via an entry in builtins.c builtins array.
- *
- * At shell "init", the plugin intiailizes a PMI object including the
- * pmi simple server and empty local kvs cache.
- *
- * During each task's "task init" callback, the pmi plugin sets up the
- * subprocess channel, sets the PMI_FD, PMI_RANK, and PMI_SIZE environment
- * variables, and subscribes to the newly created PMI_FD channel in order
- * to read PMI requests.
- *
- * The output callback pmi_fd_read_cb() reads the request from the PMI_FD
- * channel and pushes it into the PMI-1 protocol engine.  If the request
- * can be immediately answered, the shell_pmi_response_send() callback
- * registered with the engine is invoked, which writes the response to
- * the subprocess channel.
- *
- * Other requests have callbacks from the engine to provide data,
- * which is fed back to the engine, which then calls shell_pmi_response_send().
- * These are kvs_get, kvs_put, and barrier.  Although the task
- * is effectively blocked while these callbacks are handled, they are
- * implemented with asynchronous continuation callbacks so that other tasks
- * and the shell's reactor remain live while the task awaits an answer.
- *
- * The PMI KVS supports a put / barrier / get pattern.  The barrier
- * distributes KVS data that was "put" so that it is available to "get".
- * A local hash captures key-value pairs as they are put.  If the entire
- * job runs under one shell, the barrier is a no-op, and the gets are
- * serviced only from the cache.  Otherwise, the barrier dumps the hash
- * into a Flux KVS txn and commits it with a flux_kvs_fence(), using
- * the number of shells as "nprocs".  Gets are serviced from the cache,
- * with fall-through to a flux_kvs_lookup().
- *
- * If shell->verbose is true (shell --verbose flag was provided), the
- * protocol engine emits client and server telemetry to stderr, and
- * shell_pmi_task_ready() logs read errors, EOF, and finalization to stderr
- * in a compatible format.
- *
- * Caveats:
- * - PMI kvsname parameter is ignored
- * - 64-bit Flux job id's are assigned to integer-typed PMI appnum
- * - PMI publish, unpublish, lookup, spawn are not implemented
- * - Although multiple cycles of put / barrier / get are supported, the
- *   the barrier rewrites data from previous cycles to the Flux KVS.
- * - PMI_Abort() is implemented as log message + exit in the client code.
- *   It does not reach this module.
- * - Teardown of the subprocess channel is deferred until task completion,
- *   although client closes its end after PMI_Finalize().
- */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <unistd.h>
-#include <stdlib.h>
-#include <czmq.h>
-#include <assert.h>
-#include <flux/core.h>
-
-#include "src/common/libpmi/simple_server.h"
-#include "src/common/libpmi/clique.h"
-
-#include "builtins.h"
-#include "internal.h"
-#include "task.h"
-
-#define FQ_KVS_KEY_MAX (SIMPLE_KVS_KEY_MAX + 128)
-
-struct shell_pmi {
-    flux_shell_t *shell;
-    struct pmi_simple_server *server;
-    zhashx_t *kvs;
-    zhashx_t *locals;
-    int cycle;      // count cycles of put / barrier / get
-};
-
-static void shell_pmi_abort (void *arg,
-                             void *client,
-                             int exit_code,
-                             const char *msg)
-{
-    /* Generate job exception (exit_code ignored for now) */
-    shell_die (exit_code,
-               "MPI_Abort%s%s",
-               msg ? ": " : "",
-               msg ? msg : "");
-}
-
-static int shell_pmi_kvs_put (void *arg,
-                              const char *kvsname,
-                              const char *key,
-                              const char *val)
-{
-    struct shell_pmi *pmi = arg;
-
-    zhashx_update (pmi->kvs, key, (char *)val);
-    return 0;
-}
-
-static void pmi_kvs_put_local (struct shell_pmi *pmi,
-                               const char *key,
-                               const char *val)
-{
-    zhashx_update (pmi->kvs, key, (char *)val);
-    zhashx_update (pmi->locals, key, (void *) 0x1);
-}
-
-/* Handle kvs lookup response.
- */
-static void kvs_lookup_continuation (flux_future_t *f, void *arg)
-{
-    struct shell_pmi *pmi = arg;
-    void *cli = flux_future_aux_get (f, "flux::shell_pmi");
-    const char *val = NULL;
-
-    flux_kvs_lookup_get (f, &val); // val remains NULL on failure
-    pmi_simple_server_kvs_get_complete (pmi->server, cli, val);
-    flux_future_destroy (f);
-}
-
-/* Construct a PMI key in job's guest namespace.
- * Put it in a subdir named "pmi".
- */
-static int shell_pmi_kvs_key (char *buf,
-                              int bufsz,
-                              flux_jobid_t id,
-                              const char *key)
-{
-    char tmp[FQ_KVS_KEY_MAX];
-
-    if (snprintf (tmp, sizeof (tmp), "pmi.%s", key) >= sizeof (tmp))
-        return -1;
-    return flux_job_kvs_guest_key (buf, bufsz, id, tmp);
-}
-
-/* Lookup a key: first try the local hash.   If that fails and the
- * job spans multiple shells, do a KVS lookup in the job's private
- * KVS namespace and handle the response in kvs_lookup_continuation().
- */
-static int shell_pmi_kvs_get (void *arg,
-                              void *cli,
-                              const char *kvsname,
-                              const char *key)
-{
-    struct shell_pmi *pmi = arg;
-    flux_t *h = pmi->shell->h;
-    const char *val = NULL;
-
-    if ((val = zhashx_lookup (pmi->kvs, key))) {
-        pmi_simple_server_kvs_get_complete (pmi->server, cli, val);
-        return 0;
-    }
-    if (pmi->shell->info->shell_size > 1) {
-        char nkey[FQ_KVS_KEY_MAX];
-        flux_future_t *f = NULL;
-
-        if (shell_pmi_kvs_key (nkey,
-                               sizeof (nkey),
-                               pmi->shell->jobid,
-                               key) < 0) {
-            shell_log_errno ("shell_pmi_kvs_key");
-            goto out;
-        }
-        if (!(f = flux_kvs_lookup (h, NULL, 0, nkey))) {
-            shell_log_errno ("flux_kvs_lookup");
-            goto out;
-        }
-        if (flux_future_aux_set (f, "flux::shell_pmi", cli, NULL) < 0) {
-            shell_log_errno ("flux_future_aux_set");
-            flux_future_destroy (f);
-            goto out;
-        }
-        if (flux_future_then (f, -1., kvs_lookup_continuation, pmi) < 0) {
-            shell_log_errno ("flux_future_then");
-            flux_future_destroy (f);
-            goto out;
-        }
-        return 0; // response deferred
-    }
-out:
-    return -1; // cause PMI_KVS_Get() to fail with INVALID_KEY
-}
-
-static void kvs_fence_continuation (flux_future_t *f, void *arg)
-{
-    struct shell_pmi *pmi = arg;
-    int rc;
-
-    rc = flux_future_get (f, NULL);
-    pmi_simple_server_barrier_complete (pmi->server, rc);
-    flux_future_destroy (f);
-}
-
-static int shell_pmi_barrier_enter (void *arg)
-{
-    struct shell_pmi *pmi = arg;
-    flux_kvs_txn_t *txn = NULL;
-    const char *key;
-    const char *val;
-    char name[64];
-    int nprocs = pmi->shell->info->shell_size;
-    flux_future_t *f;
-    char nkey[FQ_KVS_KEY_MAX];
-
-    if (nprocs == 1) { // all local: no further sync needed
-        pmi_simple_server_barrier_complete (pmi->server, 0);
-        return 0;
-    }
-    snprintf (name, sizeof (name), "pmi.%ju.%d",
-             (uintmax_t)pmi->shell->jobid,
-             pmi->cycle++);
-    if (!(txn = flux_kvs_txn_create ())) {
-        shell_log_errno ("flux_kvs_txn_create");
-        goto error;
-    }
-    val = zhashx_first (pmi->kvs);
-    while (val) {
-        key = zhashx_cursor (pmi->kvs);
-        /* Special case:
-         * Keys in pmi->locals are not added to the KVS transaction
-         * because they were locally generated and need not be
-         * shared with the other shells.
-         */
-        if (zhashx_lookup (pmi->locals, key)) {
-            val = zhashx_next (pmi->kvs);
-            continue;
-        }
-        if (shell_pmi_kvs_key (nkey,
-                               sizeof (nkey),
-                               pmi->shell->jobid,
-                               key) < 0) {
-            shell_log_errno ("key buffer overflow");
-            goto error;
-        }
-        if (flux_kvs_txn_put (txn, 0, nkey, val) < 0) {
-            shell_log_errno ("flux_kvs_txn_put");
-            goto error;
-        }
-        val = zhashx_next (pmi->kvs);
-    }
-    if (!(f = flux_kvs_fence (pmi->shell->h, NULL, 0, name, nprocs, txn))) {
-        shell_log_errno ("flux_kvs_fence");
-        goto error;
-    }
-    if (flux_future_then (f, -1., kvs_fence_continuation, pmi) < 0) {
-        shell_log_errno ("flux_future_then");
-        flux_future_destroy (f);
-        goto error;
-    }
-    flux_kvs_txn_destroy (txn);
-    return 0;
-error:
-    flux_kvs_txn_destroy (txn);
-    return -1; // cause PMI_Barrier() to fail
-}
-
-static int shell_pmi_response_send (void *client, const char *buf)
-{
-    struct shell_task *task = client;
-
-    return flux_subprocess_write (task->proc, "PMI_FD", buf, strlen (buf));
-}
-
-static void shell_pmi_debug_trace (void *client, const char *line)
-{
-    struct shell_task *task = client;
-
-    shell_trace ("%d: %s", task->rank, line);
-}
-
-static void pmi_fd_cb (flux_shell_task_t *task,
-                       const char *stream,
-                       void *arg)
-{
-    struct shell_pmi *pmi = arg;
-    int len;
-    const char *line;
-    int rc;
-
-    line = flux_subprocess_read_line (task->proc, "PMI_FD", &len);
-    if (len < 0) {
-        shell_trace ("%d: C: pmi read error: %s",
-                     task->rank, flux_strerror (errno));
-        return;
-    }
-    if (len == 0) {
-        shell_trace ("%d: C: pmi EOF", task->rank);
-        return;
-    }
-    rc = pmi_simple_server_request (pmi->server, line, task, task->rank);
-    if (rc < 0) {
-        shell_trace ("%d: S: pmi request error", task->rank);
-        return;
-    }
-    if (rc == 1) {
-        shell_trace ("%d: S: pmi finalized", task->rank);
-    }
-}
-
-/* Generate 'PMI_process_mapping' key (see RFC 13) for MPI clique computation.
- *
- * Create an array of pmi_map_block structures, sized for worst case mapping
- * (no compression possible).  Walk through the rcalc info for each shell rank.
- * If shell's mapping looks identical to previous one, increment block->nodes;
- * otherwise consume another array slot.  Finally, encode to string, put it
- * in the local KVS hash, and free array.
- */
-static int init_clique (struct shell_pmi *pmi)
-{
-    struct pmi_map_block *blocks;
-    int nblocks;
-    int i;
-    char val[SIMPLE_KVS_VAL_MAX];
-
-    if (!(blocks = calloc (pmi->shell->info->shell_size, sizeof (*blocks))))
-        return -1;
-    nblocks = 0;
-
-    for (i = 0; i < pmi->shell->info->shell_size; i++) {
-        struct rcalc_rankinfo ri;
-
-        if (rcalc_get_nth (pmi->shell->info->rcalc, i, &ri) < 0)
-            goto error;
-        if (nblocks == 0 || blocks[nblocks - 1].procs != ri.ntasks) {
-            blocks[nblocks].nodeid = i;
-            blocks[nblocks].procs = ri.ntasks;
-            blocks[nblocks].nodes = 1;
-            nblocks++;
-        }
-        else
-            blocks[nblocks - 1].nodes++;
-    }
-    /* If value exceeds SIMPLE_KVS_VAL_MAX, skip setting the key
-     * without generating an error.  The client side will not treat
-     * a missing key as an error.  It should be unusual though so log it.
-     */
-    if (pmi_process_mapping_encode (blocks, nblocks, val, sizeof (val)) < 0) {
-        shell_log_errno ("pmi_process_mapping_encode");
-        goto out;
-    }
-    pmi_kvs_put_local (pmi, "PMI_process_mapping", val);
-out:
-    free (blocks);
-    return 0;
-error:
-    free (blocks);
-    errno = EINVAL;
-    return -1;
-}
-
-static int set_flux_instance_level (struct shell_pmi *pmi)
-{
-    char *p;
-    long l;
-    int n;
-    int rc = -1;
-    char val [SIMPLE_KVS_VAL_MAX];
-    const char *level = flux_attr_get (pmi->shell->h, "instance-level");
-
-    if (!level)
-        return 0;
-
-    errno = 0;
-    l = strtol (level, &p, 10);
-    if (errno != 0 || *p != '\0' || l < 0) {
-        shell_log_error ("set_flux_instance_level level=%s invalid", level);
-        goto out;
-    }
-    n = snprintf (val, sizeof (val), "%lu", l+1);
-    if (n >= sizeof (val)) {
-        shell_log_errno ("set_flux_instance_level: snprintf");
-        goto out;
-    }
-    pmi_kvs_put_local (pmi, "flux.instance-level", val);
-    rc = 0;
-out:
-    return rc;
-}
-
-static void pmi_destroy (struct shell_pmi *pmi)
-{
-    if (pmi) {
-        int saved_errno = errno;
-        pmi_simple_server_destroy (pmi->server);
-        zhashx_destroy (&pmi->kvs);
-        zhashx_destroy (&pmi->locals);
-        free (pmi);
-        errno = saved_errno;
-    }
-}
-
-// zhashx_duplicator_fn footprint
-static void *kvs_value_duplicator (const void *item)
-{
-    void *cpy = NULL;
-    if (item)
-        cpy = strdup (item);
-    return cpy;
-}
-
-// zhashx_destructor_fn footprint
-static void kvs_value_destructor (void **item)
-{
-    if (*item) {
-        free (*item);
-        *item = NULL;
-    }
-}
-
-static struct pmi_simple_ops shell_pmi_ops = {
-    .kvs_put        = shell_pmi_kvs_put,
-    .kvs_get        = shell_pmi_kvs_get,
-    .barrier_enter  = shell_pmi_barrier_enter,
-    .response_send  = shell_pmi_response_send,
-    .debug_trace    = shell_pmi_debug_trace,
-    .abort          = shell_pmi_abort,
-};
-
-
-static struct shell_pmi *pmi_create (flux_shell_t *shell)
-{
-    struct shell_pmi *pmi;
-    struct shell_info *info = shell->info;
-    int flags = shell->verbose ? PMI_SIMPLE_SERVER_TRACE : 0;
-    char kvsname[32];
-
-    if (!(pmi = calloc (1, sizeof (*pmi))))
-        return NULL;
-    pmi->shell = shell;
-    snprintf (kvsname, sizeof (kvsname), "%ju", (uintmax_t)shell->jobid);
-    if (!(pmi->server = pmi_simple_server_create (shell_pmi_ops,
-                                                  0, // appnum
-                                                  info->jobspec->task_count,
-                                                  info->rankinfo.ntasks,
-                                                  kvsname,
-                                                  flags,
-                                                  pmi)))
-        goto error;
-    if (!(pmi->kvs = zhashx_new ())
-        || !(pmi->locals = zhashx_new ())) {
-        errno = ENOMEM;
-        goto error;
-    }
-    zhashx_set_destructor (pmi->kvs, kvs_value_destructor);
-    zhashx_set_duplicator (pmi->kvs, kvs_value_duplicator);
-    if (init_clique (pmi) < 0)
-        goto error;
-    if (!shell->standalone) {
-        if (set_flux_instance_level (pmi) < 0)
-            goto error;
-    }
-    return pmi;
-error:
-    pmi_destroy (pmi);
-    return NULL;
-}
-
-static int shell_pmi_init (flux_plugin_t *p,
-                           const char *topic,
-                           flux_plugin_arg_t *arg,
-                           void *data)
-{
-    flux_shell_t *shell = flux_plugin_get_shell (p);
-    struct shell_pmi *pmi;
-    if (!shell || !(pmi = pmi_create (shell)))
-        return -1;
-    if (flux_plugin_aux_set (p, "pmi", pmi, (flux_free_f) pmi_destroy) < 0) {
-        pmi_destroy (pmi);
-        return -1;
-    }
-    return 0;
-}
-
-static int shell_pmi_task_init (flux_plugin_t *p,
-                                const char *topic,
-                                flux_plugin_arg_t *args,
-                                void *arg)
-{
-    flux_shell_t *shell;
-    struct shell_pmi *pmi;
-    flux_shell_task_t *task;
-    flux_cmd_t *cmd;
-
-    if (!(shell = flux_plugin_get_shell (p))
-        || !(pmi = flux_plugin_aux_get (p, "pmi"))
-        || !(task = flux_shell_current_task (shell))
-        || !(cmd = flux_shell_task_cmd (task)))
-        return -1;
-
-    if (flux_cmd_add_channel (cmd, "PMI_FD") < 0)
-        return -1;
-    if (flux_cmd_setenvf (cmd, 1, "PMI_RANK", "%d", task->rank) < 0)
-        return -1;
-    if (flux_cmd_setenvf (cmd, 1, "PMI_SIZE", "%d", task->size) < 0)
-        return -1;
-    if (flux_shell_task_channel_subscribe (task, "PMI_FD", pmi_fd_cb, pmi) < 0)
-        return -1;
-    return 0;
-}
-
-struct shell_builtin builtin_pmi = {
-    .name = "pmi",
-    .init = shell_pmi_init,
-    .task_init = shell_pmi_task_init,
-};
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/src/shell/pmi/pmi.c b/src/shell/pmi/pmi.c
new file mode 100644
index 000000000000..044746ecd48f
--- /dev/null
+++ b/src/shell/pmi/pmi.c
@@ -0,0 +1,747 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* builtin PMI-1 plugin for jobs
+ *
+ * Provide PMI-1 service so that an MPI or Flux job can bootstrap.
+ * Much of the work is done by the PMI-1 wire protocol engine in
+ * libpmi/simple_server.c and libsubprocess socketpair channels.
+ *
+ * At startup this module is registered as a builtin shell plugin under
+ * the name "pmi" via an entry in builtins.c builtins array.
+ *
+ * At shell "init", the plugin initializes a PMI object including the
+ * pmi simple server and empty local kvs cache.
+ *
+ * During each task's "task init" callback, the pmi plugin sets up the
+ * subprocess channel, sets the PMI_FD, PMI_RANK, and PMI_SIZE environment
+ * variables, and subscribes to the newly created PMI_FD channel in order
+ * to read PMI requests.
+ *
+ * The output callback pmi_fd_read_cb() reads the request from the PMI_FD
+ * channel and pushes it into the PMI-1 protocol engine.  If the request
+ * can be immediately answered, the shell_pmi_response_send() callback
+ * registered with the engine is invoked, which writes the response to
+ * the subprocess channel.
+ *
+ * Other requests have callbacks from the engine to provide data,
+ * which is fed back to the engine, which then calls shell_pmi_response_send().
+ * These are kvs_get, kvs_put, and barrier.  Although the task
+ * is effectively blocked while these callbacks are handled, they are
+ * implemented with asynchronous continuation callbacks so that other tasks
+ * and the shell's reactor remain live while the task awaits an answer.
+ *
+ * If shell->verbose is true (shell --verbose flag was provided), the
+ * protocol engine emits client and server telemetry to stderr, and
+ * shell_pmi_task_ready() logs read errors, EOF, and finalization to stderr
+ * in a compatible format.
+ *
+ * Caveats:
+ * - PMI kvsname parameter is ignored
+ * - 64-bit Flux job id's are assigned to integer-typed PMI appnum
+ * - PMI publish, unpublish, lookup, spawn are not implemented
+ * - Teardown of the subprocess channel is deferred until task completion,
+ *   although client closes its end after PMI_Finalize().
+ */
+#define FLUX_SHELL_PLUGIN_NAME "pmi-simple"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <libgen.h>
+#ifdef HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libpmi/simple_server.h"
+#include "src/common/libutil/errno_safe.h"
+#include "ccan/str/str.h"
+
+#include "builtins.h"
+#include "internal.h"
+#include "task.h"
+#include "pmi_exchange.h"
+
+struct shell_pmi {
+    flux_shell_t *shell;
+    struct pmi_simple_server *server;
+    json_t *global; // already exchanged
+    json_t *pending;// pending to be exchanged
+    json_t *locals;  // never exchanged
+    struct pmi_exchange *exchange;
+};
+
+/* pmi_simple_ops->warn() signature */
+static void shell_pmi_warn (void *client, const char *msg)
+{
+    shell_warn ("%s", msg);
+}
+
+/* pmi_simple_ops->abort() signature */
+static void shell_pmi_abort (void *arg,
+                             void *client,
+                             int exit_code,
+                             const char *msg)
+{
+    /*  Attempt to raise job exception and return to the shell's reactor.
+     *   This allows the shell to continue to process events and stdio
+     *   until the exec system terminates the job due to the exception.
+     */
+    flux_shell_raise ("exec",
+                      0,
+                      "PMI_Abort%s%s",
+                      msg ? ": " : "",
+                      msg ? msg : "");
+}
+
+static int put_dict (json_t *dict, const char *key, const char *val)
+{
+    json_t *o;
+
+    if (!(o = json_string (val)))
+        goto nomem;
+    if (json_object_set_new (dict, key, o) < 0) {
+        json_decref (o);
+        goto nomem;
+    }
+    return 0;
+nomem:
+    errno = ENOMEM;
+    return -1;
+}
+
+/**
+ ** ops for using native Flux KVS for PMI KVS
+ ** This is used if pmi.kvs=native option is provided.
+ **/
+
+static void native_lookup_continuation (flux_future_t *f, void *arg)
+{
+    struct shell_pmi *pmi = arg;
+    void *cli = flux_future_aux_get (f, "pmi_cli");
+    const char *val = NULL;
+
+    (void)flux_kvs_lookup_get (f, &val); // leave val=NULL on failure
+    pmi_simple_server_kvs_get_complete (pmi->server, cli, val);
+    flux_future_destroy (f);
+}
+
+static int native_lookup (struct shell_pmi *pmi, const char *key, void *cli)
+{
+    char *nkey;
+    flux_future_t *f;
+
+    if (asprintf (&nkey, "pmi.%s", key) < 0)
+        return -1;
+    if (!(f = flux_kvs_lookup (pmi->shell->h, NULL, 0, nkey)))
+        return -1;
+    if (flux_future_aux_set (f, "pmi_cli", cli, NULL) < 0)
+        goto error;
+    if (flux_future_then (f, -1, native_lookup_continuation, pmi) < 0)
+        goto error;
+    free (nkey);
+    return 0;
+error:
+    ERRNO_SAFE_WRAP (free, nkey);
+    flux_future_destroy (f);
+    return -1;
+}
+
+static void native_fence_continuation (flux_future_t *f, void *arg)
+{
+    struct shell_pmi *pmi = arg;
+    int rc = flux_future_get (f, NULL);
+    pmi_simple_server_barrier_complete (pmi->server, rc);
+
+    flux_future_destroy (f);
+    json_object_clear (pmi->pending);
+}
+
+static int native_fence (struct shell_pmi *pmi)
+{
+    flux_kvs_txn_t *txn;
+    const char *key;
+    json_t *val;
+    char *nkey;
+    int rc;
+    char name[64];
+    static int seq = 0;
+    uintmax_t id = (uintmax_t)pmi->shell->jobid;
+    int size = pmi->shell->info->shell_size;
+    flux_future_t *f = NULL;
+
+    if (!(txn = flux_kvs_txn_create ()))
+        return -1;
+    json_object_foreach (pmi->pending, key, val) {
+        if (asprintf (&nkey, "pmi.%s", key) < 0)
+            goto error;
+        rc = flux_kvs_txn_put (txn, 0, nkey, json_string_value (val));
+        ERRNO_SAFE_WRAP (free, nkey);
+        if (rc < 0)
+            goto error;
+    }
+    (void)snprintf (name, sizeof (name), "%juPMI%d", id, seq++);
+    if (!(f = flux_kvs_fence (pmi->shell->h, NULL, 0, name, size, txn)))
+        goto error;
+    if (flux_future_then (f, -1, native_fence_continuation, pmi) < 0)
+        goto error;
+    flux_kvs_txn_destroy (txn);
+    return 0;
+error:
+    flux_future_destroy (f);
+    flux_kvs_txn_destroy (txn);
+    return -1;
+}
+
+/* pmi_simple_ops->kvs_put() signature */
+static int native_kvs_put (void *arg,
+                           const char *kvsname,
+                           const char *key,
+                           const char *val)
+{
+    struct shell_pmi *pmi = arg;
+
+    /* Hack to support "node scope" for partial PMI2 impl needed for Cray.
+     */
+    if (strstarts (key, "local::"))
+        return put_dict (pmi->locals, key, val);
+    return put_dict (pmi->pending, key, val);
+}
+
+/* pmi_simple_ops->barrier_enter() signature */
+static int native_barrier_enter (void *arg)
+{
+    struct shell_pmi *pmi = arg;
+
+    if (pmi->shell->info->shell_size == 1) {
+        pmi_simple_server_barrier_complete (pmi->server, 0);
+        return 0;
+    }
+    if (native_fence (pmi) < 0)
+        return -1; // PMI_FAIL
+    return 0;
+}
+
+/* pmi_simple_ops->kvs_get() signature */
+static int native_kvs_get (void *arg,
+                           void *cli,
+                           const char *kvsname,
+                           const char *key)
+{
+    struct shell_pmi *pmi = arg;
+    json_t *o;
+    const char *val = NULL;
+
+    if ((o = json_object_get (pmi->locals, key))
+            || (o = json_object_get (pmi->pending, key))) {
+        val = json_string_value (o);
+        pmi_simple_server_kvs_get_complete (pmi->server, cli, val);
+        return 0;
+    }
+    if (pmi->shell->info->shell_size > 1) {
+        if (native_lookup (pmi, key, cli) == 0)
+            return 0; // response deferred
+    }
+    return -1; // PMI_ERR_INVALID_KEY
+}
+
+/**
+ ** ops for using purpose-built dict exchange for PMI KVS
+ ** This is used if pmi.kvs=exchange option is provided.
+ **/
+
+static void exchange_cb (struct pmi_exchange *pex, void *arg)
+{
+    struct shell_pmi *pmi = arg;
+    int rc = -1;
+
+    if (pmi_exchange_has_error (pex)) {
+        shell_warn ("exchange failed");
+        goto done;
+    }
+    if (json_object_update (pmi->global, pmi_exchange_get_dict (pex)) < 0) {
+        shell_warn ("failed to update dict after successful exchange");
+        goto done;
+    }
+    json_object_clear (pmi->pending);
+    rc = 0;
+done:
+    pmi_simple_server_barrier_complete (pmi->server, rc);
+}
+
+/* pmi_simple_ops->kvs_get() signature */
+static int exchange_kvs_get (void *arg,
+                             void *cli,
+                             const char *kvsname,
+                             const char *key)
+{
+    struct shell_pmi *pmi = arg;
+    json_t *o;
+    const char *val = NULL;
+
+    if ((o = json_object_get (pmi->locals, key))
+        || (o = json_object_get (pmi->pending, key))
+        || (o = json_object_get (pmi->global, key))) {
+        val = json_string_value (o);
+        pmi_simple_server_kvs_get_complete (pmi->server, cli, val);
+        return 0;
+    }
+    return -1; // PMI_ERR_INVALID_KEY
+}
+
+/* pmi_simple_ops->barrier_enter() signature */
+static int exchange_barrier_enter (void *arg)
+{
+    struct shell_pmi *pmi = arg;
+
+    if (pmi->shell->info->shell_size == 1) {
+        pmi_simple_server_barrier_complete (pmi->server, 0);
+        return 0;
+    }
+    if (pmi_exchange (pmi->exchange,
+                      pmi->pending,
+                      exchange_cb,
+                      pmi) < 0) {
+        shell_warn ("pmi_exchange %s", flux_strerror (errno));
+        return -1; // PMI_FAIL
+    }
+    return 0;
+}
+
+/* pmi_simple_ops->kvs_put() signature */
+static int exchange_kvs_put (void *arg,
+                             const char *kvsname,
+                             const char *key,
+                             const char *val)
+{
+    struct shell_pmi *pmi = arg;
+
+    /* Hack to support "node scope" for partial PMI2 impl needed for Cray.
+     */
+    if (strstarts (key, "local::"))
+        return put_dict (pmi->locals, key, val);
+    return put_dict (pmi->pending, key, val);
+}
+
+/**
+ ** end of KVS implementations
+ **/
+
+/* pmi_simple_ops->response_send() signature */
+static int shell_pmi_response_send (void *client, const char *buf)
+{
+    struct shell_task *task = client;
+
+    return flux_subprocess_write (task->proc, "PMI_FD", buf, strlen (buf));
+}
+
+/* pmi_simple_ops->debug_trace() signature */
+static void shell_pmi_debug_trace (void *client, const char *line)
+{
+    struct shell_task *task = client;
+
+    shell_trace ("%d: %s", task->rank, line);
+}
+
+static void pmi_fd_cb (flux_shell_task_t *task,
+                       const char *stream,
+                       void *arg)
+{
+    struct shell_pmi *pmi = arg;
+    int len;
+    const char *line;
+    int rc;
+
+    len = flux_subprocess_read_line (task->proc, "PMI_FD", &line);
+    if (len < 0) {
+        shell_trace ("%d: C: pmi read error: %s",
+                     task->rank,
+                     flux_strerror (errno));
+        return;
+    }
+    if (len == 0) {
+        shell_trace ("%d: C: pmi EOF", task->rank);
+        return;
+    }
+    rc = pmi_simple_server_request (pmi->server, line, task, task->rank);
+    if (rc < 0) {
+        shell_trace ("%d: S: pmi request error", task->rank);
+        shell_die (1, "PMI-1 wire protocol error");
+
+    }
+    if (rc == 1) {
+        shell_trace ("%d: S: pmi finalized", task->rank);
+    }
+}
+
+/* Generate 'PMI_process_mapping' key (see RFC 13) for MPI clique computation.
+ *
+ * PMI_process_mapping originated with MPICH, which uses it to determine
+ * whether it can short circult the comms path between local ranks with shmem.
+ * MPICH allows the key to be missing or its value to be empty, and in those
+ * cases just skips the optimization.  However, note the following:
+ *
+ * - MVAPICH2 fails with an "Invalid tag" error in MPI_Init() if the key
+ *   does not exist (flux-framework/flux-core#3592) and an even more obscure
+ *   error if it exists but is empty
+ *
+ * - OpenMPI might select conflicting shmem names if the mapping indicates
+ *   that ranks are not co-located when they really are
+ *   (flux-framework/flux-core#3551)
+ */
+static int init_clique (struct shell_pmi *pmi)
+{
+    char *s = NULL;
+    if (!(s = taskmap_encode (pmi->shell->info->taskmap, TASKMAP_ENCODE_PMI))
+        || strlen (s) > SIMPLE_KVS_VAL_MAX) {
+        /* If value exceeds SIMPLE_KVS_VAL_MAX, skip setting the key
+         * without generating an error. The client side will not treat
+         * a missing key as an error. It should be unusual though so log it.
+         */
+        if (pmi->shell->info->shell_rank == 0)
+            shell_warn ("PMI_process_mapping overflows PMI max value.");
+        goto out;
+    }
+    put_dict (pmi->locals, "PMI_process_mapping", s);
+out:
+    free (s);
+    return 0;
+}
+
+static int set_flux_tbon_interface_hint (struct shell_pmi *pmi)
+{
+    const char *hint;
+
+    if (!(hint = flux_attr_get (pmi->shell->h, "tbon.interface-hint")))
+        return 0;
+    put_dict (pmi->locals, "flux.tbon-interface-hint", hint);
+    return 0;
+}
+
+static int set_flux_instance_level (struct shell_pmi *pmi)
+{
+    char *p;
+    long l;
+    int n;
+    int rc = -1;
+    char val [SIMPLE_KVS_VAL_MAX];
+    const char *level = flux_attr_get (pmi->shell->h, "instance-level");
+
+    if (!level)
+        return 0;
+
+    errno = 0;
+    l = strtol (level, &p, 10);
+    if (errno != 0 || *p != '\0' || l < 0) {
+        shell_log_error ("set_flux_instance_level level=%s invalid", level);
+        goto out;
+    }
+    n = snprintf (val, sizeof (val), "%lu", l+1);
+    if (n >= sizeof (val)) {
+        shell_log_errno ("set_flux_instance_level: snprintf");
+        goto out;
+    }
+    put_dict (pmi->locals, "flux.instance-level", val);
+    rc = 0;
+out:
+    return rc;
+}
+
+static int set_flux_taskmap (struct shell_pmi *pmi)
+{
+    struct taskmap *map = pmi->shell->info->taskmap;
+    char *val = NULL;
+    int rc = -1;
+
+    if (!(val = taskmap_encode (map, TASKMAP_ENCODE_WRAPPED))
+        || strlen (val) > SIMPLE_KVS_VAL_MAX)
+        goto out;
+    put_dict (pmi->locals, "flux.taskmap", val);
+    rc = 0;
+out:
+    free (val);
+    return rc;
+}
+
+static void pmi_destroy (struct shell_pmi *pmi)
+{
+    if (pmi) {
+        int saved_errno = errno;
+        pmi_simple_server_destroy (pmi->server);
+        pmi_exchange_destroy (pmi->exchange);
+        json_decref (pmi->global);
+        json_decref (pmi->pending);
+        json_decref (pmi->locals);
+        free (pmi);
+        errno = saved_errno;
+    }
+}
+
+static struct pmi_simple_ops shell_pmi_ops = {
+    .response_send  = shell_pmi_response_send,
+    .debug_trace    = shell_pmi_debug_trace,
+    .abort          = shell_pmi_abort,
+    .warn           = shell_pmi_warn,
+};
+
+static int parse_args (json_t *config,
+                       int *exchange_k,
+                       const char **kvs,
+                       int *nomap)
+{
+    json_error_t error;
+
+    if (config) {
+        if (json_unpack_ex (config,
+                            &error,
+                            0,
+                            "{s?s s?{s?i !} s?i !}",
+                            "kvs", kvs,
+                            "exchange",
+                              "k", exchange_k,
+                            "nomap", nomap) < 0) {
+            shell_log_error ("option error: %s", error.text);
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static struct shell_pmi *pmi_create (flux_shell_t *shell, json_t *config)
+{
+    struct shell_pmi *pmi;
+    struct shell_info *info = shell->info;
+    int flags = shell->verbose ? PMI_SIMPLE_SERVER_TRACE : 0;
+    char kvsname[32];
+    const char *kvs = "exchange";
+    int exchange_k = 0; // 0=use default tree fanout
+    int nomap = 0;      // avoid generation of PMI_process_mapping
+
+    if (!(pmi = calloc (1, sizeof (*pmi))))
+        return NULL;
+    pmi->shell = shell;
+
+    if (parse_args (config, &exchange_k, &kvs, &nomap) < 0)
+        goto error;
+    if (streq (kvs, "native")) {
+        shell_pmi_ops.kvs_put = native_kvs_put;
+        shell_pmi_ops.kvs_get = native_kvs_get;
+        shell_pmi_ops.barrier_enter = native_barrier_enter;
+        if (shell->info->shell_rank == 0)
+            shell_warn ("using native Flux kvs implementation");
+    }
+    else if (streq (kvs, "exchange")) {
+        shell_pmi_ops.kvs_put = exchange_kvs_put;
+        shell_pmi_ops.kvs_get = exchange_kvs_get;
+        shell_pmi_ops.barrier_enter = exchange_barrier_enter;
+        if (!(pmi->exchange = pmi_exchange_create (shell, exchange_k)))
+            goto error;
+    }
+    else {
+        shell_log_error ("Unknown kvs implementation %s", kvs);
+        errno = EINVAL;
+        goto error;
+    }
+
+    /* Use F58 representation of jobid for "kvsname", since the broker
+     * will pull the kvsname and use it as the broker 'jobid' attribute.
+     * This allows the broker attribute to be in the "common" user-facing
+     * jobid representation.
+     */
+    if (flux_job_id_encode (shell->jobid,
+                            "f58",
+                            kvsname,
+                            sizeof (kvsname)) < 0)
+        goto error;
+    if (!(pmi->server = pmi_simple_server_create (shell_pmi_ops,
+                                                  0, // appnum
+                                                  info->total_ntasks,
+                                                  info->rankinfo.ntasks,
+                                                  kvsname,
+                                                  flags,
+                                                  pmi)))
+        goto error;
+    if (!(pmi->global = json_object ())
+        || !(pmi->pending = json_object ())
+        || !(pmi->locals = json_object ())) {
+        errno = ENOMEM;
+        goto error;
+    }
+    if (!nomap && init_clique (pmi) < 0)
+        goto error;
+    if (set_flux_instance_level (pmi) < 0
+        || set_flux_tbon_interface_hint (pmi) < 0
+        || (!nomap && set_flux_taskmap (pmi) < 0))
+        goto error;
+    return pmi;
+error:
+    pmi_destroy (pmi);
+    return NULL;
+}
+
+static bool member_of_csv (const char *list, const char *name)
+{
+    char *argz = NULL;
+    size_t argz_len;
+
+    if (argz_create_sep (list, ',', &argz, &argz_len) == 0) {
+        const char *entry = NULL;
+
+        while ((entry = argz_next (argz, argz_len, entry))) {
+            if (streq (entry, name)) {
+                free (argz);
+                return true;
+            }
+        }
+        free (argz);
+    }
+    return false;
+}
+
+static int shell_pmi_init (flux_plugin_t *p,
+                           const char *topic,
+                           flux_plugin_arg_t *arg,
+                           void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    struct shell_pmi *pmi;
+    json_t *config = NULL;
+    const char *pmi_opt = NULL;
+    bool enable = false;
+
+    if (flux_shell_getopt_unpack (shell, "pmi", "s", &pmi_opt) < 0) {
+        shell_log_error ("pmi shell option must be a string");
+        return -1;
+    }
+    if (flux_shell_getopt_unpack (shell, "pmi-simple", "o", &config) < 0) {
+        shell_log_error ("error parsing pmi-simple shell option");
+        return -1;
+    }
+    /* This plugin is disabled _only_ if '-opmi=LIST' was specified without
+     * "simple" in LIST.  "pmi1" and "pmi2" are considered aliases for
+     * "simple" - see flux-framework/flux-core#5226.
+     */
+    if (pmi_opt) {
+        if (member_of_csv (pmi_opt, "simple"))
+            enable = true;
+        else if (member_of_csv (pmi_opt, "pmi2")) {
+            shell_debug ("pmi2 is interpreted as an alias for simple");
+            enable = true;
+        }
+        else if (member_of_csv (pmi_opt, "pmi1")) {
+            shell_debug ("pmi1 is interpreted as an alias for simple");
+            enable = true;
+        }
+    }
+    else
+        enable = true;
+    if (!enable)
+        return 0; // plugin disabled
+    shell_debug ("simple wire protocol is enabled");
+
+    if (!(pmi = pmi_create (shell, config)))
+        return -1;
+    if (flux_plugin_aux_set (p, "pmi", pmi, (flux_free_f) pmi_destroy) < 0) {
+        pmi_destroy (pmi);
+        return -1;
+    }
+    return 0;
+}
+
+/* Prepend 'path' to the environment variable 'name' which is assumed to
+ * be a colon-separated list. If 'name' isn't already set, set it to 'path'.
+ * Return 0 on success, -1 on failure.
+ */
+static int prepend_path_to_cmd_env (flux_cmd_t *cmd,
+                                    const char *name,
+                                    const char *path)
+{
+    const char *searchpath = flux_cmd_getenv (cmd, name);
+
+    return flux_cmd_setenvf (cmd,
+                             1,
+                             name,
+                             "%s%s%s",
+                             path,
+                             searchpath ? ":" : "",
+                             searchpath ? searchpath : "");
+}
+
+static int shell_pmi_task_init (flux_plugin_t *p,
+                                const char *topic,
+                                flux_plugin_arg_t *args,
+                                void *arg)
+{
+    flux_shell_t *shell;
+    struct shell_pmi *pmi;
+    flux_shell_task_t *task;
+    flux_cmd_t *cmd;
+
+    if (!(pmi = flux_plugin_aux_get (p, "pmi")))
+        return 0; // plugin disabled
+    if (!(shell = flux_plugin_get_shell (p))
+        || !(task = flux_shell_current_task (shell))
+        || !(cmd = flux_shell_task_cmd (task)))
+        return -1;
+
+    if (flux_cmd_add_channel (cmd, "PMI_FD") < 0)
+        return -1;
+    if (flux_cmd_setenvf (cmd, 1, "PMI_RANK", "%d", task->rank) < 0)
+        return -1;
+    if (flux_cmd_setenvf (cmd, 1, "PMI_SIZE", "%d", task->size) < 0)
+        return -1;
+    if (flux_shell_task_channel_subscribe (task, "PMI_FD", pmi_fd_cb, pmi) < 0)
+        return -1;
+    const char *pmipath;
+    if (!(pmipath = flux_conf_builtin_get ("pmi_library_path", FLUX_CONF_AUTO)))
+        return -1;
+    /* Flux libpmi.so and libpmi2.so are installed to a directory outside
+     * of the default ld.so search path.  Add this directory to LD_LIBRARY_PATH
+     * so Flux jobs find Flux PMI libs before Slurm's PMI libs which are in
+     * the system path (a tripping hazard).
+     * N.B. The cray-pals plugin in flux-coral2 will need to undo this
+     * so Cray MPICH finds the Cray libpmi2.so first which uses libpals.
+     * See also flux-framework/flux-core#5714.
+     */
+    char *cpy;
+    char *pmidir;
+    if (!(cpy = strdup (pmipath))
+        || !(pmidir = dirname (cpy))
+        || prepend_path_to_cmd_env (cmd, "LD_LIBRARY_PATH", pmidir) < 0) {
+        free (cpy);
+        return -1;
+    }
+    free (cpy);
+    /* N.B. The pre-v5 OpenMPI flux MCA plugin dlopens the library pointed to
+     * by FLUX_PMI_LIBRARY_PATH.  Since the library only works when this shell
+     * plugin is active, set it here.
+     */
+    if (flux_cmd_setenvf (cmd, 1, "FLUX_PMI_LIBRARY_PATH", "%s", pmipath) < 0)
+        return -1;
+    return 0;
+}
+
+struct shell_builtin builtin_pmi = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = shell_pmi_init,
+    .task_init = shell_pmi_task_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/pmi/pmi_exchange.c b/src/shell/pmi/pmi_exchange.c
new file mode 100644
index 000000000000..90b0ce840e37
--- /dev/null
+++ b/src/shell/pmi/pmi_exchange.c
@@ -0,0 +1,327 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* pmi_exchange.c - sync local dict across shells
+ *
+ * Gather key-value dict from each shell to shell 0, then broadcast
+ * the aggregate dict to all shells.
+ *
+ * Each shell calls pmi_exchange() with a json_t dictionary and
+ * a callback.  Upon completion of the exchange, the callback is invoked.
+ * The callback may access an updated json_t dictionary.
+ *
+ * A binary tree is computed across all shell ranks.
+ * Gather aggregates hashes at each tree level, reducing the number
+ * of messages that have to be handled by shell 0.
+ * Broadcast fans out at each tree level, reducing the number of messages
+ * that have to be sent by rank 0.
+ *
+ * N.B. This binary tree is created from thin air for algorithmic purposes.
+ * Nodes that are peers in the ersatz tree may actually be multiple hops
+ * apart on the Flux tree based overlay network at the broker level.
+ */
+#define FLUX_SHELL_PLUGIN_NAME "pmi-simple"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/shell.h>
+
+#include "src/common/libutil/kary.h"
+
+#include "info.h"
+#include "internal.h"
+
+#include "pmi_exchange.h"
+
+#define DEFAULT_TREE_K 2
+
+struct session {
+    json_t *dict;               // container for gathered dictionary
+    pmi_exchange_f cb;          // callback for exchange completion
+    void *cb_arg;
+
+    zlist_t *requests;          // pending requests from children
+    flux_future_t *f;           // pending request to parent
+
+    struct pmi_exchange *pex;
+    unsigned int local:1;       // pmi_exchange() was called on this shell
+    unsigned int has_error:1;   // an error occurred
+};
+
+struct pmi_exchange {
+    flux_shell_t *shell;
+    int size;
+    int rank;
+    uint32_t parent_rank;
+    int child_count;
+
+    struct session *session;
+};
+
+static void exchange_response_completion (flux_future_t *f, void *arg);
+
+static void session_destroy (struct session *ses)
+{
+    if (ses) {
+        int saved_errno = errno;
+        if (ses->requests) {
+            const flux_msg_t *msg;
+            while ((msg = zlist_pop (ses->requests)))
+                flux_msg_decref (msg);
+            zlist_destroy (&ses->requests);
+        }
+        flux_future_destroy (ses->f);
+        json_decref (ses->dict);
+        free (ses);
+        errno = saved_errno;
+    }
+}
+
+static struct session *session_create (struct pmi_exchange *pex)
+{
+    struct session *ses;
+
+    if (!(ses = calloc (1, sizeof (*ses))))
+        return NULL;
+    ses->pex = pex;
+    if (!(ses->requests = zlist_new ()))
+        goto nomem;
+    if (!(ses->dict = json_object ()))
+        goto nomem;
+    return ses;
+nomem:
+    errno = ENOMEM;
+    session_destroy (ses);
+    return NULL;
+}
+
+static void session_process (struct session *ses)
+{
+    struct pmi_exchange *pex = ses->pex;
+    flux_t *h = ses->pex->shell->h;
+    const flux_msg_t *msg;
+
+    if (ses->has_error)
+        goto done;
+
+    /* Awaiting self or child input?
+     */
+    if (!ses->local || zlist_size (ses->requests) < pex->child_count)
+        return;
+
+    /* Send exchange request, if needed.
+     */
+    if (pex->rank > 0 && !ses->f) {
+        flux_future_t *f;
+
+        if (!(f = flux_shell_rpc_pack (pex->shell,
+                                       "pmi-exchange",
+                                       pex->parent_rank,
+                                       0,
+                                       "O",
+                                       ses->dict))
+                || flux_future_then (f,
+                                     -1,
+                                     exchange_response_completion,
+                                     pex) < 0) {
+            flux_future_destroy (f);
+            shell_warn ("error sending pmi-exchange request");
+            ses->has_error = 1;
+            goto done;
+        }
+        ses->f = f;
+    }
+
+    /* Awaiting parent response?
+     */
+    if (ses->f && !flux_future_is_ready (ses->f))
+        return;
+
+    /* Send exchange response(s), if needed.
+     */
+    while ((msg = zlist_pop (ses->requests))) {
+        if (flux_respond_pack (h, msg, "O", ses->dict) < 0) {
+            shell_warn ("error responding to pmi-exchange request");
+            flux_msg_decref (msg);
+            ses->has_error = 1;
+            goto done;
+        }
+        flux_msg_decref (msg);
+    }
+done:
+    ses->cb (pex, ses->cb_arg);
+    session_destroy (ses);
+    pex->session = NULL;
+}
+
+/* PMI implementation on parent has responded to pmi-exchange request.
+ */
+static void exchange_response_completion (flux_future_t *f, void *arg)
+{
+    struct pmi_exchange *pex = arg;
+    json_t *dict;
+
+    if (flux_rpc_get_unpack (f, "o", &dict) < 0) {
+        shell_warn ("pmi-exchange request: %s", future_strerror (f, errno));
+        pex->session->has_error = 1;
+        goto done;
+    }
+    if (json_object_update (pex->session->dict, dict) < 0) {
+        shell_warn ("pmi-exchange response handling failed to update dict");
+        pex->session->has_error = 1;
+        goto done;
+    }
+done:
+    session_process (pex->session);
+}
+
+/* PMI implementation on child sent a pmi-exchange request
+ */
+static void exchange_request_cb (flux_t *h,
+                                 flux_msg_handler_t *mh,
+                                 const flux_msg_t *msg,
+                                 void *arg)
+{
+    struct pmi_exchange *pex = arg;
+    json_t *dict;
+    const char *errstr = NULL;
+
+    if (flux_request_unpack (msg, NULL, "o", &dict) < 0)
+        goto error;
+    if (!pex->session) {
+        if (!(pex->session = session_create (pex)))
+            goto error;
+    }
+    if (zlist_size (pex->session->requests) == pex->child_count) {
+        errstr = "pmi-exchange received too many child requests";
+        errno = EINPROGRESS;
+        goto error;
+    }
+    if (json_object_update (pex->session->dict, dict) < 0) {
+        errstr = "pmi-exchange request failed to update dict";
+        goto nomem;
+    }
+    if (zlist_append (pex->session->requests,
+                      (void *)flux_msg_incref (msg)) < 0) {
+        flux_msg_decref (msg);
+        errstr = "pmi-exchange request failed to save pending request";
+        goto nomem;
+    }
+    session_process (pex->session);
+    return;
+nomem:
+    errno = ENOMEM;
+error:
+    if (flux_respond_error (h, msg, errno, errstr) < 0)
+        shell_warn ("error responding to pmi-exchange request: %s",
+                    flux_strerror (errno));
+}
+
+/* PMI implementation on _this_ shell is ready to exchange.
+ */
+int pmi_exchange (struct pmi_exchange *pex,
+                  json_t *dict,
+                  pmi_exchange_f cb,
+                  void *arg)
+{
+    if (!pex->session) {
+        if (!(pex->session = session_create (pex)))
+            return -1;
+    }
+    if (pex->session->local) {
+        errno = EINPROGRESS;
+        return -1;
+    }
+    pex->session->cb = cb;
+    pex->session->cb_arg = arg;
+    pex->session->local = 1;
+    if (json_object_update (pex->session->dict, dict) < 0) {
+        errno = ENOMEM;
+        return -1;
+    }
+    session_process (pex->session);
+    return 0;
+}
+
+/* Helper for pmi_exchange_create() - calculate the number of children of
+ * 'rank' in a 'size' tree of degree 'k'.
+ */
+static int child_count (int k, int rank, int size)
+{
+    int i;
+    int count = 0;
+
+    for (i = 0; i < k; i++) {
+        if (kary_childof (k, size, rank, i) != KARY_NONE)
+            count++;
+    }
+    return count;
+}
+
+struct pmi_exchange *pmi_exchange_create (flux_shell_t *shell, int k)
+{
+    struct pmi_exchange *pex;
+
+    if (!(pex = calloc (1, sizeof (*pex))))
+        return NULL;
+    if (k <= 0)
+        k = DEFAULT_TREE_K;
+    else if (k > shell->info->shell_size) {
+        k = shell->info->shell_size;
+        if (shell->info->shell_rank == 0)
+            shell_warn ("requested exchange fanout too large, using k=%d", k);
+    }
+    else {
+        if (shell->info->shell_rank == 0)
+            shell_warn ("using k=%d", k);
+    }
+    pex->shell = shell;
+    pex->size = shell->info->shell_size;
+    pex->rank = shell->info->shell_rank;
+    pex->parent_rank = kary_parentof (k, pex->rank);
+    pex->child_count = child_count (k, pex->rank, pex->size);
+
+    if (flux_shell_service_register (shell,
+                                     "pmi-exchange",
+                                     exchange_request_cb,
+                                     pex) < 0)
+        goto error;
+    return pex;
+error:
+    pmi_exchange_destroy (pex);
+    return NULL;
+}
+
+void pmi_exchange_destroy (struct pmi_exchange *pex)
+{
+    if (pex) {
+        int saved_errno = errno;
+        session_destroy (pex->session);
+        free (pex);
+        errno = saved_errno;
+    }
+}
+
+bool pmi_exchange_has_error (struct pmi_exchange *pex)
+{
+    return pex->session->has_error ? true : false;
+}
+
+json_t *pmi_exchange_get_dict (struct pmi_exchange *pex)
+{
+    return pex->session->dict;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/shell/pmi/pmi_exchange.h b/src/shell/pmi/pmi_exchange.h
new file mode 100644
index 000000000000..c54604b84c70
--- /dev/null
+++ b/src/shell/pmi/pmi_exchange.h
@@ -0,0 +1,42 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef SHELL_PMI_EXCHANGE_H
+#define SHELL_PMI_EXCHANGE_H
+
+/* Create handle for performing multiple sequential exchanges.
+ * 'k' is the tree fanout (k=0 selects internal default).
+ */
+struct pmi_exchange *pmi_exchange_create (flux_shell_t *shell, int k);
+void pmi_exchange_destroy (struct pmi_exchange *pex);
+
+typedef void (*pmi_exchange_f)(struct pmi_exchange *pex, void *arg);
+
+/* Perform one exchange across all shell ranks.
+ * 'dict' is the input from this  shell.  Once the the result of the exchange
+ * is available, 'cb' is invoked.
+ */
+int pmi_exchange (struct pmi_exchange *pex,
+                  json_t *dict,
+                  pmi_exchange_f cb,
+                  void *arg);
+
+/* Accessors may be called only from pmi_exchange_f callback.
+ * pmi_exchange_get_dict() returns a json object that is invalidated when
+ * the callback returns.
+ */
+bool pmi_exchange_has_error (struct pmi_exchange *pex);
+json_t *pmi_exchange_get_dict (struct pmi_exchange *pex);
+
+#endif /* !SHELL_PMI_EXCHANGE_H */
+
+/* vi: ts=4 sw=4 expandtab
+ */
+
diff --git a/src/shell/pty.c b/src/shell/pty.c
new file mode 100644
index 000000000000..2a9c43510c84
--- /dev/null
+++ b/src/shell/pty.c
@@ -0,0 +1,511 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#define FLUX_SHELL_PLUGIN_NAME "pty"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <unistd.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <termios.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/shell.h>
+#include <flux/idset.h>
+
+#include "src/common/libterminus/pty.h"
+#include "src/common/libterminus/terminus.h"
+#include "ccan/ptrint/ptrint.h"
+#include "ccan/str/str.h"
+#include "builtins.h"
+#include "log.h"
+
+static struct flux_terminus_server *
+shell_terminus_server_start (flux_shell_t *shell, const char *shell_service)
+{
+    char service[128];
+    struct flux_terminus_server *t;
+
+    if (snprintf (service,
+                  sizeof (service),
+                  "%s.terminus",
+                  shell_service) >= sizeof (service)) {
+        shell_log_errno ("Failed to build terminus service name");
+        return NULL;
+    }
+
+    /*  Create a terminus server in this shell. 1 per shell */
+    t = flux_terminus_server_create (flux_shell_get_flux (shell),
+                                     service);
+    if (!t) {
+        shell_log_errno ("flux_terminus_server_create");
+        return NULL;
+    }
+    if (flux_shell_aux_set (shell,
+                            "builtin::terminus",
+                            t,
+                            (flux_free_f) flux_terminus_server_destroy) < 0)
+        return NULL;
+    flux_terminus_server_set_log (t, shell_llog, NULL);
+
+    /* Ensure process knows it is a terminus session */
+    flux_shell_setenvf (shell, 1, "FLUX_TERMINUS_SESSION", "0");
+
+    return t;
+}
+
+static void pty_monitor (struct flux_pty *pty, void *data, int len)
+{
+    flux_plugin_arg_t *args;
+    int rank;
+
+    /*  len == 0 indicates pty is closed. If there's a reference on
+     *  stdout, release it here
+     */
+    if (len == 0) {
+        flux_subprocess_t *p;
+        if ((p = flux_pty_aux_get (pty, "subprocess")))
+            flux_subprocess_channel_decref (p, "stdout");
+        return;
+    }
+
+    rank = ptr2int (flux_pty_aux_get (pty, "rank"));
+    if (!(args = flux_plugin_arg_create ())
+        || flux_plugin_arg_pack (args, FLUX_PLUGIN_ARG_IN,
+                                 "{s:s s:i s:s#}",
+                                 "stream", "stdout",
+                                 "rank", rank,
+                                 "data", data, len) < 0) {
+        shell_log_errno ("monitor: packing %d bytes of shell.output: %s",
+                         len,
+                         flux_plugin_arg_strerror (args));
+        return;
+    }
+    flux_shell_plugstack_call (flux_pty_aux_get (pty, "shell"),
+                               "shell.output", args);
+    flux_plugin_arg_destroy (args);
+}
+
+/*  Return an idset of ids that intersect the local taskids on shell rank
+ *   given the idset encoded in `ids` ("all" will intersect with all ids).
+ */
+static struct idset *shell_taskids_intersect (flux_shell_t *shell,
+                                              int rank,
+                                              const char *ids)
+{
+    const char *taskids;
+    struct idset *localids;
+    struct idset *idset;
+    struct idset *result = NULL;
+
+    if (flux_shell_rank_info_unpack (shell,
+                                     rank,
+                                     "{s:s}",
+                                     "taskids", &taskids) < 0)
+        return NULL;
+    if (!(localids = idset_decode (taskids)))
+        return NULL;
+    if (streq (ids, "all"))
+        return localids;
+    if (!(idset = idset_decode (ids)))
+        goto out;
+    result = idset_intersect (localids, idset);
+out:
+    idset_destroy (localids);
+    idset_destroy (idset);
+    return result;
+}
+
+
+/*  Parse any shell 'pty' option.
+ *
+ *  The shell pty option has the form:
+ *
+ *  {
+ *     rasks:s or i   # rank or rank on which to open a pty
+ *     capture:i      # if nonzero, capture pty output to the same
+ *                    #  destination as task output
+ *     interactive:i  # if nonzero, note pty endpoint in shell.init
+ *                    #  for interactive attach from client
+ *  }
+ *
+ *  The default if none of the above are set is pty.ranks = "all".
+ *  If pty.interactive is nonzero, the default is pty.ranks = "0".
+ *
+ *  Return 0 if the option was not present,
+ *         1 if the option was present and parsed without error,
+ *    and -1 if the option was present and had a parse error.
+ */
+static int pty_getopt (flux_shell_t *shell,
+                       int shell_rank,
+                       struct idset **targets,
+                       int *capture,
+                       int *interactive)
+{
+    char *s;
+    const char *ranks;
+    char rbuf [21];
+    json_t *o;
+
+    /*  Only create a session for rank 0 if the pty option was specified
+     */
+    if (flux_shell_getopt (shell, "pty", &s) != 1)
+        return 0;
+
+    /*  Default: pty on all ranks with "non-interactive" attach
+     *   and pty output is copied to stdout location.
+     */
+    ranks = "all";
+    *interactive = 0;
+    *capture = -1;
+
+    if (!(o = json_loads (s, JSON_DECODE_ANY, NULL))) {
+        shell_log_error ("Unable to parse pty shell option: %s", s);
+        return -1;
+    }
+    if (json_is_object (o)) {
+        json_error_t error;
+        json_t *ranks_obj = NULL;
+
+        if (json_unpack_ex (o,
+                            &error,
+                            JSON_STRICT,
+                            "{s?o s?i s?i}",
+                            "ranks", &ranks_obj,
+                            "capture", capture,
+                            "interactive", interactive) < 0) {
+            shell_die (1, "invalid shell pty option: %s", error.text);
+            return -1;
+        }
+
+        if (*interactive) {
+            /*  If pty.interactive is set and pty.ranks is not, then
+             *   default pty.ranks to "0"
+             */
+            if (ranks_obj == NULL)
+                ranks = "0";
+
+            /*  If pty.interactive is set and capture was not set
+             *   then disable capture.
+             */
+            if (*capture == -1)
+                *capture = 0;
+        }
+
+        /*  Allow ranks to be encoded as a string (for RFC 22 IDSet)
+         *   or as an integer for a single rank (e.g. 0).
+         */
+        if (json_is_string (ranks_obj))
+            ranks = json_string_value (ranks_obj);
+        else if (json_is_integer (ranks_obj)) {
+            /*  32bit unsigned guaranteed to fit in 21 bytes */
+            sprintf (rbuf, "%u", (uint32_t) json_integer_value (ranks_obj));
+            ranks = rbuf;
+        }
+
+        /*  Default for capture if not set is 1/true
+         */
+        if (*capture == -1)
+            *capture = 1;
+    }
+    if (!(*targets = shell_taskids_intersect (shell, shell_rank, ranks))) {
+        shell_log_error ("pty: shell_taskids_intersect");
+        return -1;
+    }
+
+    /*  If interactive, then always ensure rank 0 is in the set of targets
+     *  (interactive attach to non-rank 0 task is not yet supported)
+     */
+    if (*interactive
+        && shell_rank == 0
+        && !idset_test (*targets, 0)) {
+        shell_warn ("pty: adding pty to rank 0 for interactive support");
+        idset_set (*targets, 0);
+    }
+    return 1;
+}
+
+static void server_empty (struct flux_terminus_server *ts, void *arg)
+{
+    flux_shell_t *shell = arg;
+    if (flux_shell_remove_completion_ref (shell, "terminus.server") < 0)
+        shell_log_errno ("failed to remove completion ref for terminus.server");
+}
+
+
+static int pty_init (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *arg)
+{
+    const char *shell_service;
+    int shell_rank = -1;
+    flux_shell_t *shell;
+    struct flux_terminus_server *t;
+    int interactive = 0;
+    struct idset *targets;
+    int rank;
+    int capture;
+    int rc;
+
+    if (!(shell = flux_plugin_get_shell (p)))
+        return shell_log_errno ("flux_plugin_get_shell");
+
+    if (flux_shell_info_unpack (shell,
+                                "{s:i s:s}",
+                                "rank", &shell_rank,
+                                "service", &shell_service) < 0)
+        return shell_log_errno ("flux_shell_info_unpack: service");
+
+    /*  Start terminus server for all shells
+     */
+    if (!(t = shell_terminus_server_start (shell, shell_service))) {
+        shell_log_errno ("pty_init: error setting up terminal server");
+        return -1;
+    }
+
+    if ((rc = pty_getopt (shell,
+                          shell_rank,
+                          &targets,
+                          &capture,
+                          &interactive)) != 1)
+        return rc;
+
+    if (idset_count (targets) > 0) {
+        /*
+         *   If there is at least one pty active on this shell rank,
+         *    ensure shell doesn't exit until the terminus server is complete,
+         *    even if all tasks have exited. This is required to support
+         *    an interactive attach from a pty client, which may come after
+         *    the task has exited.
+         */
+        if (flux_shell_add_completion_ref (shell, "terminus.server") < 0
+            || flux_terminus_server_notify_empty (t,
+                                                  server_empty,
+                                                  shell) < 0) {
+            shell_log_errno ("failed to enable pty server notification");
+            return -1;
+        }
+    }
+
+
+    /*  Create a pty session for each local target
+     */
+    rank = idset_first (targets);
+    while (rank != IDSET_INVALID_ID) {
+        struct flux_pty *pty;
+        char name [26];
+        char key [35];
+
+        /*  task<int> guaranteed to fit in 25 characters */
+        sprintf (name, "task%d", rank);
+        /*  builtin::pty.<int> guaranteed to fit in 34 characters */
+        sprintf (key, "builtin::pty.%d", rank);
+
+        /*  Open a new terminal session for this rank
+         */
+        if (!(pty = flux_terminus_server_session_open (t, rank, name)))
+            return shell_log_errno ("terminus_session_open");
+
+        if (flux_shell_aux_set (shell, key, pty, NULL) < 0)
+            goto error;
+
+        /*  Always wait for the pty to be "closed" so that we ensure
+         *   all data is read before the pty exits
+         */
+        flux_pty_wait_on_close (pty);
+
+        /*  For an interactive pty, add the endpoint in the shell.init
+         *   event context. This lets `flux job attach` or other entities
+         *   know that the pty is ready for attach, and also lets them
+         *   key off the presence of this value to know that an interactive
+         *   pty was requested.
+         */
+        if (interactive && rank == 0) {
+            if (flux_shell_add_event_context (shell,
+                                              "shell.init",
+                                              0,
+                                              "{s:s}",
+                                              "pty", "terminus.0") < 0) {
+                shell_log_errno ("flux_shell_add_event_context (pty)");
+                goto error;
+            }
+            if (capture) {
+                /*
+                 * If also capturing the pty output for an interactive
+                 *  pty, note this in the shell.init event context. This
+                 *  will hint to the pty reader that the terminal output
+                 *  is duplicated for rank 0.
+                 */
+                if (flux_shell_add_event_context (shell,
+                                                  "shell.init",
+                                                  0,
+                                                  "{s:i}",
+                                                  "capture", 1) < 0) {
+                    shell_log_errno ("flux_shell_add_event_context (capture)");
+                }
+            }
+            /*  Ensure that rank 0 pty waits for client to attach
+             *   in pty.interactive mode, even if pty.capture is also
+             *   specified.
+             */
+            flux_pty_wait_for_client (pty);
+        }
+
+        /*  Enable capture of pty output to stdout if capture flag is set.
+         *
+         *  Always enable capture on nonzero ranks though, otherwise
+         *   reading from the pty will never be started since nonozero
+         *   ranks do not support interactive attach.
+         */
+        if (capture || rank != 0) {
+            if (flux_pty_aux_set (pty, "shell", shell, NULL) < 0
+                || flux_pty_aux_set (pty, "rank", int2ptr (rank), NULL) < 0
+                || flux_pty_aux_set (pty,
+                                     "capture",
+                                     int2ptr (capture),
+                                     NULL) < 0) {
+                shell_log_errno ("flux_pty_aux_set");
+                goto error;
+            }
+            flux_pty_monitor (pty, pty_monitor);
+        }
+        rank = idset_next (targets, rank);
+    }
+    idset_destroy (targets);
+    return 0;
+error:
+    idset_destroy (targets);
+    flux_terminus_server_destroy (t);
+    return -1;
+}
+
+static struct flux_pty *pty_lookup (flux_shell_t *shell, int rank)
+{
+    char key [35];
+    sprintf (key, "builtin::pty.%d", rank);
+    return flux_shell_aux_get (shell, key);
+}
+
+static int pty_task_exec (flux_plugin_t *p,
+                          const char *topic,
+                          flux_plugin_arg_t *args,
+                          void *arg)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    flux_shell_task_t *task;
+    struct flux_pty *pty;
+    int rank;
+
+    if (!shell)
+        return shell_log_errno ("failed to get shell object");
+
+    if (flux_shell_getopt (shell, "pty", NULL) != 1)
+        return 0;
+
+    if (!(task = flux_shell_current_task (shell))
+        || flux_shell_task_info_unpack (task, "{s:i}", "rank", &rank) < 0)
+        return shell_log_errno ("unable to get task rank");
+
+    if ((pty = pty_lookup (shell, rank))) {
+        /*  Redirect stdio to 'pty'
+         */
+        if (pty && flux_pty_attach (pty) < 0)
+            return shell_log_errno ("pty attach failed");
+
+        /*  Set environment variable so process knows it is running
+         *   under a terminus server.
+         */
+        flux_shell_setenvf (shell, 1, "FLUX_TERMINUS_SESSION", "%d", rank);
+    }
+    return (0);
+}
+
+static int pty_task_fork (flux_plugin_t *p,
+                          const char *topic,
+                          flux_plugin_arg_t *args,
+                          void *arg)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    flux_shell_task_t *task;
+    struct flux_pty *pty;
+    int rank;
+
+    if (!shell)
+        return shell_log_errno ("failed to get shell object");
+
+    if (flux_shell_getopt (shell, "pty", NULL) != 1)
+        return 0;
+
+    if (!(task = flux_shell_current_task (shell))
+        || flux_shell_task_info_unpack (task, "{s:i}", "rank", &rank) < 0)
+        return shell_log_errno ("unable to get task rank");
+
+    /*  If pty is in capture mode, then take a reference on subprocess
+     *  stdout so that EOF is not read until pty exits.
+     */
+   if ((pty = pty_lookup (shell, rank))
+       && ptr2int (flux_pty_aux_get (pty, "capture"))) {
+       flux_subprocess_t *sp = flux_shell_task_subprocess (task);
+       flux_subprocess_channel_incref (sp, "stdout");
+       flux_pty_aux_set (pty, "subprocess", sp, NULL);
+   }
+   return (0);
+}
+
+static int pty_task_exit (flux_plugin_t *p,
+                          const char *topic,
+                          flux_plugin_arg_t *args,
+                          void *arg)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    flux_shell_task_t *task;
+    struct flux_pty *pty;
+    int rank;
+
+    if (!shell)
+        return shell_log_errno ("failed to get shell object");
+
+    if (flux_shell_getopt (shell, "pty", NULL) != 1)
+        return 0;
+
+    if (!(task = flux_shell_current_task (shell))
+        || flux_shell_task_info_unpack (task, "{s:i}", "rank", &rank) < 0)
+        return shell_log_errno ("unable to get task rank");
+
+    if ((pty = pty_lookup (shell, rank))) {
+        struct flux_terminus_server *t = NULL;
+        int status = flux_subprocess_status (flux_shell_task_subprocess (task));
+
+        if (!(t = flux_shell_aux_get (shell, "builtin::terminus")))
+            return shell_log_errno ("failed to get terminus and pty objects");
+
+        shell_debug ("close pty session rank=%d status=%d", rank, status);
+        if (flux_terminus_server_session_close (t, pty, status) < 0)
+            shell_die_errno (1, "pty attach failed");
+    }
+    return (0);
+}
+
+struct shell_builtin builtin_pty = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = pty_init,
+    .task_exec = pty_task_exec,
+    .task_fork = pty_task_fork,
+    .task_exit = pty_task_exit,
+};
+
+/* vi: ts=4 sw=4 expandtab
+ */
diff --git a/src/shell/rc.c b/src/shell/rc.c
index 656727a18df1..f4eebf129803 100644
--- a/src/shell/rc.c
+++ b/src/shell/rc.c
@@ -7,6 +7,7 @@
  *
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
+#define FLUX_SHELL_PLUGIN_NAME NULL
 
 /* Load and run shell rc script
  *
@@ -17,15 +18,16 @@
 #endif
 #include <libgen.h>
 #include <glob.h>
-#include <czmq.h>
 #include <lua.h>
 #include <lauxlib.h>
 #include <lualib.h>
 #include <flux/core.h>
 #include <flux/shell.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/bindings/lua/jansson-lua.h"
 #include "src/bindings/lua/lutil.h"
+#include "ccan/str/str.h"
 #include "internal.h"
 #include "info.h"
 
@@ -164,6 +166,7 @@ static int lua_plugin_cb (flux_plugin_t *p,
         if (!success)
             return -1;
     }
+    lua_settop (L, 0);
     return 0;
 }
 
@@ -257,6 +260,13 @@ static int l_plugin_register (lua_State *L)
         lua_rawgeti (L, -1, ++i);
     }
 
+    /*  If no handlers were specified, assume this was a mistake in
+     *   the plugin.register() call and throw an error.
+     */
+    if (i == 1)
+        return luaL_error (L, "plugin.register: handlers table exists "
+                              "but has no entries. (not an array?)");
+
     /*  Finally, add plugin to shell plugin  stack
      */
     if (plugstack_push (rc_shell->plugstack, p) < 0)
@@ -283,15 +293,18 @@ static int plugin_load (lua_State *L)
     int t = lua_gettop (L);
     int rc = -1;
 
-    if (!lua_istable (L, t))
-        return -1;
-
-    lua_getfield (L, t, "file");
+    if (lua_isstring (L, t))
+        pattern = lua_tostring (L, -1);
+    else if (lua_istable (L, t)) {
+        lua_getfield (L, t, "file");
 
-    pattern = lua_tostring (L, -1);
-    lua_getfield (L, t, "conf");
-    if (lua_istable (L, -1))
-        lua_value_to_json_string (L, -1, &conf);
+        pattern = lua_tostring (L, -1);
+        lua_getfield (L, t, "conf");
+        if (lua_istable (L, -1))
+            lua_value_to_json_string (L, -1, &conf);
+    }
+    else
+        return luaL_error (L, "plugin.load: invalid argument");
 
     if ((rc = plugstack_load (rc_shell->plugstack, pattern, conf)) < 0)
         luaL_error (L, "plugin.load: %s: %s", pattern, strerror (errno));
@@ -315,8 +328,12 @@ static int shell_run_rcfile (flux_shell_t *shell,
                              const char *rcfile)
 {
     struct stat sb;
+
     if (!shell || !L || !rcfile)
         return -1;
+
+    shell_trace ("trying to load %s", rcfile);
+
     if (stat (rcfile, &sb) < 0)
         return -1;
     file_stack_push (rcfile);
@@ -332,6 +349,7 @@ static int shell_run_rcfile (flux_shell_t *shell,
         return -1;
     }
     file_stack_pop ();
+    lua_settop (L, 0);
     return 0;
 }
 
@@ -343,10 +361,12 @@ static int l_source_rcfiles (lua_State *L)
     int rc;
     glob_t gl;
     const char *pattern = lua_tostring (L, -1);
+    int glob_flags = 0;
 
-    /* XXX: using GLOB_TILDE GNU extension for now
-     */
-    if ((rc = glob (pattern, GLOB_TILDE_CHECK, NULL, &gl)) != 0) {
+#ifdef GLOB_TILDE_CHECK
+    glob_flags |= GLOB_TILDE_CHECK;
+#endif
+    if ((rc = glob (pattern, glob_flags, NULL, &gl)) != 0) {
         globfree (&gl);
         if (rc == GLOB_NOMATCH) {
             if (!isa_pattern (pattern))
@@ -484,7 +504,7 @@ static int l_shell_setenv (lua_State *L)
         overwrite = lua_tointeger (L, 3);
     int rc = flux_shell_setenvf (rc_shell, overwrite, name, "%s", val);
     if (rc < 0)
-        return lua_pusherror (L, strerror (errno));
+        return lua_pusherror (L, "%s", strerror (errno));
     lua_pushboolean (L, 1);
     return 1;
 }
@@ -501,7 +521,7 @@ static int l_shell_rankinfo (lua_State *L)
         shell_rank = lua_tointeger (L, -1);
 
     if (flux_shell_get_rank_info (rc_shell, shell_rank, &json_str) < 0)
-        return lua_pusherror (L, strerror (errno));
+        return lua_pusherror (L, "%s", strerror (errno));
     if (json_object_string_to_lua (L, json_str) < 0)
         rc = lua_pusherror (L, "json_object_to_lua: %s", strerror (errno));
     free (json_str);
@@ -537,7 +557,7 @@ static int call_shell_log (int level, lua_State *L)
     const char *s = lua_tostring (L, 1);
 
     get_lua_sourceinfo (L, &ar, &file, &line);
-    flux_shell_log (level, file, line, "%s", s);
+    flux_shell_log (NULL, level, file, line, "%s", s);
     return 0;
 }
 
@@ -572,7 +592,7 @@ static int l_shell_die (lua_State *L)
     const char *s = lua_tostring (L, 1);
 
     get_lua_sourceinfo (L, &ar, &file, &line);
-    flux_shell_fatal (file, line, 0, 1, "%s", s);
+    flux_shell_fatal (NULL, file, line, 0, 1, "%s", s);
     return 0;
 }
 
@@ -583,15 +603,15 @@ static int l_plugin_index (lua_State *L)
     if (key == NULL)
         return luaL_error (L, "plugin: invalid key");
 
-    if (strcmp (key, "load") == 0) {
+    if (streq (key, "load")) {
         lua_pushcfunction (L, l_plugin_load);
         return 1;
     }
-   if (strcmp (key, "register") == 0) {
+   if (streq (key, "register")) {
         lua_pushcfunction (L, l_plugin_register);
         return 1;
     }
-    else if (strcmp (key, "searchpath") == 0) {
+    else if (streq (key, "searchpath")) {
         lua_pushstring (L, plugstack_get_searchpath (rc_shell->plugstack));
         return 1;
     } else {
@@ -604,12 +624,12 @@ static int l_plugin_index (lua_State *L)
 static int l_plugin_newindex (lua_State *L)
 {
     const char *key = lua_tostring (L, 2);
-    if (strcmp (key, "searchpath") == 0) {
+    if (streq (key, "searchpath")) {
         const char *path = lua_tostring (L, 3);
         plugstack_set_searchpath (rc_shell->plugstack, path);
         return 0;
     }
-    return luaL_error (L, "invald plugin method %s called", key);
+    return luaL_error (L, "invalid plugin method %s called", key);
 }
 
 static int l_shell_index (lua_State *L)
@@ -619,43 +639,43 @@ static int l_shell_index (lua_State *L)
     if (key == NULL)
         return luaL_error (L, "shell: invalid key");
 
-    if (strcmp (key, "info") == 0)
+    if (streq (key, "info"))
         return l_shell_info (L);
-    else if (strcmp (key, "getenv") == 0) {
+    else if (streq (key, "getenv")) {
         lua_pushcfunction (L, l_shell_getenv);
         return 1;
     }
-    else if (strcmp (key, "setenv") == 0) {
+    else if (streq (key, "setenv")) {
         lua_pushcfunction (L, l_shell_setenv);
         return 1;
     }
-    else if (strcmp (key, "unsetenv") == 0) {
+    else if (streq (key, "unsetenv")) {
         lua_pushcfunction (L, l_shell_unsetenv);
         return 1;
     }
-    else if (strcmp (key, "get_rankinfo") == 0) {
+    else if (streq (key, "get_rankinfo")) {
         lua_pushcfunction (L, l_shell_rankinfo);
         return 1;
     }
-    else if (strcmp (key, "rankinfo") == 0)
+    else if (streq (key, "rankinfo"))
         return l_shell_rankinfo (L);
-    else if (strcmp (key, "verbose") == 0) {
+    else if (streq (key, "verbose")) {
         lua_pushboolean (L, rc_shell->verbose);
         return 1;
     }
-    else if (strcmp (key, "log") == 0) {
+    else if (streq (key, "log")) {
         lua_pushcfunction (L, l_shell_log);
         return 1;
     }
-    else if (strcmp (key, "debug") == 0) {
+    else if (streq (key, "debug")) {
         lua_pushcfunction (L, l_shell_debug);
         return 1;
     }
-    else if (strcmp (key, "log_error") == 0) {
+    else if (streq (key, "log_error")) {
         lua_pushcfunction (L, l_shell_log_error);
         return 1;
     }
-    else if (strcmp (key, "die") == 0) {
+    else if (streq (key, "die")) {
         lua_pushcfunction (L, l_shell_die);
         return 1;
     }
@@ -675,7 +695,7 @@ static int is_shell_method (const char *name)
 {
     const char **sp = &shell_fields [0];
     while (*sp != NULL) {
-        if (strcmp (name, *sp) == 0)
+        if (streq (name, *sp))
             return 1;
         sp++;
     }
@@ -696,7 +716,7 @@ static int l_shell_newindex (lua_State *L)
                                   key);
         }
 
-        if (strcmp (key, "verbose") == 0) {
+        if (streq (key, "verbose")) {
             int level;
             /*  Handle Lua's baffling choice for lua_tonumber(true) == nil
              *  allows shell.verbose = 1 or true to work.
@@ -764,7 +784,7 @@ static int l_task_setenv (lua_State *L)
     if (lua_gettop (L) == 3)
         overwrite = lua_tointeger (L, 3);
     if (flux_cmd_setenvf (cmd, overwrite, key, "%s", val) < 0)
-        return lua_pusherror (L, strerror (errno));
+        return lua_pusherror (L, "%s", strerror (errno));
     lua_pushboolean (L, 1);
     return 1;
 }
@@ -779,18 +799,18 @@ static int l_task_index (lua_State *L)
 
     if (key == NULL)
         return lua_pusherror (L, "invalid key %s", key);
-    else if (strcmp (key, "info") == 0) {
+    else if (streq (key, "info")) {
         return l_task_info (L, task);
     }
-    else if (strcmp (key, "getenv") == 0) {
+    else if (streq (key, "getenv")) {
         lua_pushcfunction (L, l_task_getenv);
         return 1;
     }
-    else if (strcmp (key, "setenv") == 0) {
+    else if (streq (key, "setenv")) {
         lua_pushcfunction (L, l_task_setenv);
         return 1;
     }
-    else if (strcmp (key, "unsetenv") == 0) {
+    else if (streq (key, "unsetenv")) {
         lua_pushcfunction (L, l_task_unsetenv);
         return 1;
     }
@@ -872,6 +892,17 @@ int shell_rc (flux_shell_t *shell, const char *rcfile)
     /* Save shell global */
     rc_shell = shell;
     free (copy);
+
+    /* Load any flux.shell Lua library */
+    lua_getglobal (L, "require");
+    lua_pushstring (L, "flux.shell");
+    if (lua_pcall (L, 1, LUA_MULTRET, 0) != 0) {
+        shell_debug ("Error loading flux.shell module: %s",
+                     lua_tostring (L, -1));
+    }
+    else
+        shell_trace ("Successfully loaded flux.shell module");
+    lua_settop (L, 0);
     return shell_run_rcfile (shell, L, rcfile);
 }
 
diff --git a/src/shell/rcalc.c b/src/shell/rcalc.c
index 1ead5f3b0b0e..c626dcb136a6 100644
--- a/src/shell/rcalc.c
+++ b/src/shell/rcalc.c
@@ -16,9 +16,11 @@
 #include <unistd.h>
 #include <errno.h>
 #include <jansson.h>
-#include <czmq.h>   /* zlist_t */
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libidset/idset.h"
+#include "src/common/libutil/errprintf.h"
+#include "ccan/str/str.h"
 
 #include "rcalc.h"
 
@@ -29,17 +31,19 @@ struct rankinfo {
     int ngpus;
     const char *cores;
     const char *gpus;
-    cpu_set_t cpuset;
+    struct idset *cpuset;
+    struct idset *gpuset;
 };
 
 struct allocinfo {
     int ncores_avail;
     int ntasks;
-    int basis;
 };
 
 struct rcalc {
     json_t *json;
+    json_t *R_lite;
+    struct idset *orig_ranks;
     int nranks;
     int ncores;
     int ngpus;
@@ -48,219 +52,151 @@ struct rcalc {
     struct allocinfo *alloc;
 };
 
-
-static const char * nexttoken (const char *p, int sep)
+void rcalc_destroy (rcalc_t *r)
 {
-    if (p)
-        p = strchr (p, sep);
-    if (p)
-        p++;
-    return (p);
+    if (r == NULL)
+        return;
+    json_decref (r->json);
+    idset_destroy (r->orig_ranks);
+    for (int i = 0; i < r->nranks; i++) {
+        idset_destroy (r->ranks[i].cpuset);
+        idset_destroy (r->ranks[i].gpuset);
+    }
+    free (r->ranks);
+    free (r->alloc);
+    memset (r, 0, sizeof (*r));
+    free (r);
 }
 
-/*
- *  Temporarily copied from src/bindings/lua/lua-affinity
- */
-static int cstr_to_cpuset(cpu_set_t *mask, const char* str)
+static struct idset * rcalc_ranks (rcalc_t *r, flux_error_t *errp)
 {
-    const char *p, *q;
-    char *endptr;
-    q = str;
-    CPU_ZERO(mask);
-
-    if (strlen (str) == 0)
-        return 0;
-
-    while (p = q, q = nexttoken(q, ','), p) {
-        unsigned long a; /* beginning of range */
-        unsigned long b; /* end of range */
-        unsigned long s; /* stride */
-        const char *c1, *c2;
-
-        a = strtoul(p, &endptr, 10);
-        if (endptr == p)
-            return EINVAL;
-        if (a >= CPU_SETSIZE)
-            return E2BIG;
-        /*
-         *  Leading zeros are an error:
-         */
-        if ((a != 0 && *p == '0') || (a == 0 && memcmp (p, "00", 2L) == 0))
-            return 1;
-
-        b = a;
-        s = 1;
-
-        c1 = nexttoken(p, '-');
-        c2 = nexttoken(p, ',');
-        if (c1 != NULL && (c2 == NULL || c1 < c2)) {
-
-            /*
-             *  Previous conversion should have used up all characters
-             *     up to next '-'
-             */
-            if (endptr != (c1-1)) {
-                return 1;
-            }
+    json_t *entry;
+    size_t index;
+    json_error_t error;
+    struct idset *ranks;
 
-            b = strtoul (c1, &endptr, 10);
-            if (endptr == c1)
-                return EINVAL;
-            if (b >= CPU_SETSIZE)
-                return E2BIG;
-
-            c1 = nexttoken(c1, ':');
-            if (c1 != NULL && (c2 == NULL || c1 < c2)) {
-                s = strtoul (c1, &endptr, 10);
-                if (endptr == c1)
-                    return EINVAL;
-                if (b >= CPU_SETSIZE)
-                    return E2BIG;
-            }
-        }
+    if (!(ranks = idset_create (0, IDSET_FLAG_AUTOGROW)))
+        return NULL;
 
-        if (!(a <= b))
-            return EINVAL;
-        while (a <= b) {
-            CPU_SET(a, mask);
-            a += s;
+    json_array_foreach (r->R_lite, index, entry) {
+        struct idset *ids;
+        const char *rank;
+        if (json_unpack_ex (entry, &error, 0,
+                            "{s:s}",
+                            "rank", &rank) < 0) {
+            errprintf (errp, "%s", error.text);
+            goto err;
+        }
+        if (!(ids = idset_decode (rank))) {
+            errprintf (errp, "invalid idset %s", rank);
+            goto err;
         }
+        if (idset_add (ranks, ids) < 0) {
+            idset_destroy (ids);
+            errprintf (errp, "idset_add (%s): %s", rank, strerror (errno));
+            goto err;
+        }
+        idset_destroy (ids);
     }
-
-    /*  Error if there are left over characters */
-    if (endptr && *endptr != '\0')
-        return EINVAL;
-
-    return 0;
-}
-
-static int cstr_count (const char *str)
-{
-    cpu_set_t set;
-    if (str == NULL)
-        return 0;
-    if (cstr_to_cpuset (&set, str))
-        return -1;
-    return CPU_COUNT (&set);
+    return ranks;
+err:
+    idset_destroy (ranks);
+    return NULL;
 }
 
-static int rankinfo_get (json_t *o, struct rankinfo *ri)
+static int rankinfo_get_children (struct rankinfo *ri,
+                                  json_t *children,
+                                  flux_error_t *errp)
 {
     json_error_t error;
-    int rc = json_unpack_ex (o, &error, 0, "{s:i, s:{s:s,s?:s}}",
-                "rank", &ri->rank,
-                "children",
-                "core", &ri->cores,
-                "gpu",  &ri->gpus);
-    if (rc < 0) {
-        fprintf (stderr, "json_unpack: %s\n", error.text);
-        return -1;
-    }
 
-    if (!ri->cores || cstr_to_cpuset (&ri->cpuset, ri->cores))
-        return -1;
+    if (json_unpack_ex (children, &error, 0,
+                        "{s:s s?s}",
+                        "core", &ri->cores,
+                        "gpu", &ri->gpus) < 0)
+        return errprintf (errp, "%s", error.text);
 
-    ri->ncores = CPU_COUNT (&ri->cpuset);
-    ri->ngpus = cstr_count (ri->gpus);
-    return (0);
-}
+    if (!(ri->cpuset = idset_decode (ri->cores))
+        || !(ri->gpuset = idset_decode (ri->gpus ? ri->gpus : "")))
+        return errprintf (errp, "Failed to decode cpu or gpu sets");
+
+    ri->ncores = idset_count (ri->cpuset);
+    ri->ngpus = idset_count (ri->gpuset);
 
-static int expand_rank_ranges_one (json_t *out, json_t *entry)
-{
-    const char *rank;
-    json_t *children;
-    struct idset *ids;
-    unsigned int id;
-    json_t *n;
-
-    if (json_unpack_ex (entry, NULL, 0,
-                        "{s:s s:o}",
-                        "rank", &rank,
-                        "children", &children) < 0)
-        return -1;
-    if (!(ids = idset_decode (rank)))
-        return -1;
-    id = idset_first (ids);
-    while (id != IDSET_INVALID_ID) {
-        if (!(n = json_pack ("{s:i s:O}",
-                             "rank", id,
-                             "children", children)))
-            return -1;
-        if (json_array_append_new (out, n) < 0) {
-            json_decref (n);
-            return -1;
-        }
-        id = idset_next (ids, id);
-    }
-    idset_destroy (ids);
     return 0;
 }
 
-/* Convert from R version 1 internal R_lite object to the earlier wreck R_lite
- * object understood by this module.  They are the same except Rv1 specifies
- * ranks as an idset rather than integer, allowing for compact representation.
- */
-static json_t *expand_rank_ranges (json_t *R_lite)
+static int rcalc_process_all_ranks (rcalc_t *r, flux_error_t *errp)
 {
-    json_t *out;
     json_t *entry;
     size_t index;
-
-    if (!(out = json_array ())) // accumulate new array
-        return NULL;
-    json_array_foreach (R_lite, index, entry) {
-        if (expand_rank_ranges_one (out, entry) < 0)
-            goto error;
+    json_error_t error;
+    int n = 0;
+
+    json_array_foreach (r->R_lite, index, entry) {
+        const char *rank;
+        json_t *children;
+        unsigned int i;
+        struct idset *ids;
+
+        if (json_unpack_ex (entry, &error, 0,
+                            "{s:s s:o}",
+                            "rank", &rank,
+                            "children", &children) < 0)
+            return errprintf (errp, "%s", error.text);
+
+        if (!(ids = idset_decode (rank)))
+            return errprintf (errp,
+                              "idset_decode (%s): %s",
+                              rank,
+                              strerror (errno));
+
+        i = idset_first (ids);
+        while (i != IDSET_INVALID_ID) {
+            struct rankinfo *ri = &r->ranks[n];
+            ri->id = n;
+            ri->rank = i;
+            if (rankinfo_get_children (ri, children, errp) < 0) {
+                idset_destroy (ids);
+                return -1;
+            }
+            r->ncores += ri->ncores;
+            r->ngpus += ri->ngpus;
+            n++;
+            i = idset_next (ids, i);
+        }
+        idset_destroy (ids);
     }
-    return out;
-error:
-    json_decref (out);
-    return NULL;
-}
-
-void rcalc_destroy (rcalc_t *r)
-{
-    if (r == NULL)
-        return;
-    json_decref (r->json);
-    free (r->ranks);
-    free (r->alloc);
-    memset (r, 0, sizeof (*r));
-    free (r);
+    return 0;
 }
 
-static rcalc_t * rcalc_create_json (json_t *o)
+rcalc_t * rcalc_create_json (json_t *o)
 {
-    int i;
     int version;
-    json_t *R_lite;
+    flux_error_t error;
     rcalc_t *r = calloc (1, sizeof (*r));
     if (!r)
         return (NULL);
+    r->json = json_incref (o);
     if (json_unpack_ex (o, NULL, 0,
                         "{s:i s:{s:o}}",
                         "version", &version,
                         "execution",
-                        "R_lite", &R_lite) < 0)
+                        "R_lite", &r->R_lite) < 0)
         goto fail;
     if (version != 1) {
         errno = EINVAL;
         goto fail;
     }
-    if (!(r->json = expand_rank_ranges (R_lite))) {
-        errno = EINVAL;
+    if (!(r->orig_ranks = rcalc_ranks (r, &error)))
         goto fail;
-    }
-    r->nranks = json_array_size (r->json);
+    r->nranks = idset_count (r->orig_ranks);
     r->ranks = calloc (r->nranks, sizeof (struct rankinfo));
     r->alloc = calloc (r->nranks, sizeof (struct allocinfo));
-    for (i = 0; i < r->nranks; i++) {
-        r->ranks[i].id = i;
-        if (rankinfo_get (json_array_get (r->json, i), &r->ranks[i]) < 0)
-            goto fail;
-        r->ncores += r->ranks[i].ncores;
-        r->ngpus += r->ranks[i].ngpus;
-    }
+
+    if (rcalc_process_all_ranks (r, &error) < 0)
+        goto fail;
+
     return (r);
 fail:
     rcalc_destroy (r);
@@ -304,6 +240,11 @@ int rcalc_total_gpus (rcalc_t *r)
     return r->ngpus;
 }
 
+int rcalc_total_ntasks (rcalc_t *r)
+{
+    return r->ntasks;
+}
+
 int rcalc_total_nodes_used (rcalc_t *r)
 {
     int i;
@@ -355,32 +296,23 @@ static bool allocinfo_add_task (struct allocinfo *ai, int size)
     return (false);
 }
 
-static void rcalc_compute_taskids (rcalc_t *r)
-{
-    int i;
-    int taskid = 0;
-    for (i = 0; i < r->nranks; i++) {
-        r->alloc[i].basis = taskid;
-        taskid += r->alloc[i].ntasks;
-    }
-}
-
 /*
  *  Distribute ntasks over the ranks in `r` "evenly" by a heuristic
  *   that first assigns a number of cores per task, then distributes
  *   over largest nodes first.
  */
-int rcalc_distribute (rcalc_t *r, int ntasks)
+int rcalc_distribute (rcalc_t *r, int ntasks, int cores_per_task)
 {
     struct allocinfo *ai;
     int assigned = 0;
-    int cores_per_task = 0;
     zlist_t *l = NULL;
 
-    /* Punt for now if there are more tasks than cores */
-    if ((cores_per_task = r->ncores/ntasks) == 0) {
-        errno = EINVAL;
-        return -1;
+    if (cores_per_task <= 0) {
+        /* Punt for now if there are more tasks than cores */
+        if ((cores_per_task = r->ncores/ntasks) == 0) {
+            errno = EINVAL;
+            return -1;
+        }
     }
 
     r->ntasks = ntasks;
@@ -400,19 +332,55 @@ int rcalc_distribute (rcalc_t *r, int ntasks)
      *  and leaving "full" ranks off the list.
      */
     while (assigned < ntasks) {
-        ai = zlist_pop (l);
+        if (!(ai = zlist_pop (l))) {
+            zlist_destroy (&l);
+            errno = ENOSPC;
+            return -1;
+        }
         if (allocinfo_add_task (ai, cores_per_task)) {
             zlist_append (l, ai);
             assigned++;
         }
     }
     zlist_destroy (&l);
-
-    /*  Assign taskid basis to each rank in block allocation order */
-    rcalc_compute_taskids (r);
     return (0);
 }
 
+/*  Distribute tasks over resources in `r` by resource type. Assigns
+ *   ntasks tasks to each resource of type name.
+ */
+int rcalc_distribute_per_resource (rcalc_t *r, const char *name, int ntasks)
+{
+    bool by_core = false;
+    bool by_node = false;
+
+    if (streq (name, "core"))
+        by_core = true;
+    else if (streq (name, "node"))
+        by_node = true;
+    else {
+        errno = EINVAL;
+        return  -1;
+    }
+
+    allocinfo_clear (r);
+    r->ntasks = 0;
+    for (int i = 0; i < r->nranks; i++) {
+        if (by_node) {
+            r->alloc[i].ntasks = ntasks;
+            r->alloc[i].ncores_avail = 0;
+            r->ntasks += ntasks;
+        }
+        else if (by_core) {
+            int n = r->alloc[i].ncores_avail * ntasks;
+            r->alloc[i].ntasks = n;
+            r->alloc[i].ncores_avail = 0;
+            r->ntasks += n;
+        }
+    }
+    return 0;
+}
+
 static struct rankinfo *rcalc_rankinfo_find (rcalc_t *r, int rank)
 {
     int i;
@@ -447,8 +415,6 @@ static void rcalc_rankinfo_set (rcalc_t *r, int id,
     rli->rank =   ri->rank;
     rli->ncores = ri->ncores;
     rli->ntasks = ai->ntasks;
-    rli->global_basis =  ai->basis;
-    memcpy (&rli->cpuset, &ri->cpuset, sizeof (cpu_set_t));
     /*  Copy cores string to rli, in the very unlikely event that
      *   we get a huge cores string, indicate truncation.
      */
diff --git a/src/shell/rcalc.h b/src/shell/rcalc.h
index 8dd614cb1bdd..9be3ff6bfdd2 100644
--- a/src/shell/rcalc.h
+++ b/src/shell/rcalc.h
@@ -12,6 +12,7 @@
 #define SHELL_RCALC_H
 
 #include <sched.h>
+#include <jansson.h>
 #include <flux/core.h>
 
 typedef struct rcalc rcalc_t;
@@ -20,20 +21,24 @@ struct rcalc_rankinfo {
     int nodeid;               /* This rank's nodeid within the job       */
     int rank;                 /* The current broker rank                 */
     int ntasks;               /* Number of tasks assigned to this rank   */
-    int global_basis;         /* Task id of the first task on this rank  */
     int ncores;               /* Number of cores allocated on this rank  */
-    cpu_set_t cpuset;         /* cpu_set_t representation of cores list  */
     char cores [128];         /* String core list (directly from R_lite) */
     char gpus [128];          /* String gpu list (directly from R)       */
 };
 
 /* Create resource calc object from JSON string in "Rlite" format */
 rcalc_t *rcalc_create (const char *json_in);
+
+/* As above, but from Jansson json_t object */
+rcalc_t *rcalc_create_json (json_t *R);
+
 /* Same as above, but read JSON input from file */
 rcalc_t *rcalc_createf (FILE *);
 
 void rcalc_destroy (rcalc_t *r);
 
+/*  Return # of total tasks contained in rcalc object */
+int rcalc_total_ntasks (rcalc_t *r);
 /*  Return # of total cores asssigned to rcalc object */
 int rcalc_total_cores (rcalc_t *r);
 /*  Return # of total gpus asssigned to rcalc object */
@@ -46,7 +51,9 @@ int rcalc_total_nodes_used (rcalc_t *r);
 int rcalc_has_rank (rcalc_t *r, int rank);
 
 /*  Distribute ntasks across cores in r */
-int rcalc_distribute (rcalc_t *r, int ntasks);
+int rcalc_distribute (rcalc_t *r, int ntasks, int cores_per_task);
+/*  Distribute ntasks *per-resource* of type `name` in `r` */
+int rcalc_distribute_per_resource (rcalc_t *r, const char *name, int ntasks);
 
 /*  Fill in rcalc_rankinfo for rank */
 int rcalc_get_rankinfo (rcalc_t *r, int rank, struct rcalc_rankinfo *ri);
diff --git a/src/shell/rexec.c b/src/shell/rexec.c
new file mode 100644
index 000000000000..986e3754b8cd
--- /dev/null
+++ b/src/shell/rexec.c
@@ -0,0 +1,143 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* rexec.c - shell subprocess server
+ */
+
+#define FLUX_SHELL_PLUGIN_NAME "rexec"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <sys/types.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libsubprocess/server.h"
+#include "src/common/libutil/errprintf.h"
+
+#include "builtins.h"
+#include "internal.h"
+#include "svc.h"
+#include "log.h"
+
+struct shell_rexec {
+    flux_shell_t *shell;
+    subprocess_server_t *server;
+    char *name;
+    bool parent_is_trusted;
+};
+
+static void rexec_destroy (struct shell_rexec *rexec)
+{
+    if (rexec) {
+        int saved_errno = errno;
+        subprocess_server_destroy (rexec->server);
+        free (rexec->name);
+        free (rexec);
+        errno = saved_errno;
+    }
+}
+
+/* The embedded subprocess server restricts access based on FLUX_ROLE_OWNER,
+ * but this shell cannot trust message credentials if they are passing through
+ * a Flux instance running as a different user (e.g. the "flux" user in a
+ * system instance).  If that user were compromised, they could run arbitrary
+ * commands as any user that currently has a job running.  Therefore, this
+ * additional check ensures that we only trust an instance running as the same
+ * user.
+ *
+ * For good measure, check that the shell userid matches the credential
+ * userid. After the above check, this could only fail in test where the
+ * owner can be mocked.
+ */
+static int rexec_auth_cb (const flux_msg_t *msg,
+                          void *arg,
+                          flux_error_t *errp)
+{
+    struct shell_rexec *rexec = arg;
+    uint32_t userid;
+
+    if (!rexec->parent_is_trusted
+        || flux_msg_get_userid (msg, &userid) < 0
+        || userid != getuid ()) {
+        errno = EPERM;
+        return errprintf (errp, "Access denied");
+    }
+    return 0;
+}
+
+static struct shell_rexec *rexec_create (flux_shell_t *shell)
+{
+    struct shell_rexec *rexec;
+
+    if (!(rexec = calloc (1, sizeof (*rexec))))
+        return NULL;
+    rexec->shell = shell;
+
+    /* Determine if this shell is running as the instance owner, without
+     * trusting the instance owner to tell us.  Since the parent of a guest
+     * shell is flux-imp(1), kill(2) of the parent pid should fail for guests.
+     */
+    pid_t ppid = getppid (); // 0 =  parent is in a different pid namespace
+    if (ppid > 0 && kill (getppid (), 0) == 0)
+        rexec->parent_is_trusted = true;
+
+    /* N.B. subprocess_server_create() registers the methods: exec, write,
+     * kill, list, and disconnect.  Give the server its own namespace.  The
+     * full topic strings will be like "5588-shell-381933322240.rexec.kill".
+     */
+    if (asprintf (&rexec->name, "%s.rexec", shell_svc_name (shell->svc)) < 0)
+        goto error;
+    if (!(rexec->server = subprocess_server_create (flux_shell_get_flux (shell),
+                                                    rexec->name,
+                                                    getenv ("FLUX_URI"),
+                                                    shell_llog,
+                                                    NULL)))
+        goto error;
+    subprocess_server_set_auth_cb (rexec->server,
+                                   rexec_auth_cb,
+                                   rexec);
+    shell_debug ("registered rexec service as %s", rexec->name);
+    return rexec;
+error:
+    rexec_destroy (rexec);
+    return NULL;
+}
+
+static int rexec_init (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *arg,
+                      void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    struct shell_rexec *rexec;
+
+    if (!(rexec = rexec_create (shell)))
+        return -1;
+    if (flux_plugin_aux_set (p,
+                             "rexec",
+                             rexec,
+                             (flux_free_f)rexec_destroy) < 0) {
+        rexec_destroy (rexec);
+        return -1;
+    }
+    return 0;
+}
+
+struct shell_builtin builtin_rexec = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = rexec_init,
+};
+
+// vi:ts=4 sw=4 expandtab
diff --git a/src/shell/rlimit.c b/src/shell/rlimit.c
new file mode 100644
index 000000000000..d6278d0f23d8
--- /dev/null
+++ b/src/shell/rlimit.c
@@ -0,0 +1,139 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* shell rlimit propagations
+ *
+ * Call setrlimit(2) for any resource limits defined in
+ * attributes.system.shell.options.rlimit
+ */
+#define FLUX_SHELL_PLUGIN_NAME "rlimit"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <signal.h>
+#include <sys/resource.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/shell.h>
+
+#include "ccan/str/str.h"
+
+#include "internal.h"
+#include "builtins.h"
+
+static int rlimit_name_to_string (const char *name)
+{
+    if (streq (name, "cpu"))
+        return RLIMIT_CPU;
+    if (streq (name, "fsize"))
+        return RLIMIT_FSIZE;
+    if (streq (name, "data"))
+        return RLIMIT_DATA;
+    if (streq (name, "stack"))
+        return RLIMIT_STACK;
+    if (streq (name, "core"))
+        return RLIMIT_CORE;
+    if (streq (name, "nofile") || streq (name, "ofile"))
+        return RLIMIT_NOFILE;
+    if (streq (name, "as"))
+        return RLIMIT_AS;
+    if (streq (name, "rss"))
+        return RLIMIT_RSS;
+    if (streq (name, "nproc"))
+        return RLIMIT_NPROC;
+    if (streq (name, "memlock"))
+        return RLIMIT_MEMLOCK;
+#ifdef RLIMIT_MSGQUEUE
+    if (streq (name, "msgqueue"))
+        return RLIMIT_MSGQUEUE;
+#endif
+#ifdef RLIMIT_NICE
+    if (streq (name, "nice"))
+        return RLIMIT_NICE;
+#endif
+#ifdef RLIMIT_RTPRIO
+    if (streq (name, "rtprio"))
+        return RLIMIT_RTPRIO;
+#endif
+#ifdef RLIMIT_RTTIME
+    if (streq (name, "rttime"))
+        return RLIMIT_RTTIME;
+#endif
+#ifdef RLIMIT_SIGPENDING
+    if (streq (name, "sigpending"))
+        return RLIMIT_SIGPENDING;
+#endif
+    return -1;
+}
+
+static int rlimit_init (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    int rc = 0;
+    json_t *value;
+    const char *key;
+    json_t *o = NULL;
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+
+    if (!shell)
+        return -1;
+
+    switch (flux_shell_getopt_unpack (shell, "rlimit", "o", &o)) {
+        case -1:
+            return shell_log_errno ("failed to parse rlimit shell option");
+        case 0:
+            return 0;
+        case 1:
+            break;
+    }
+    json_object_foreach (o, key, value) {
+        int resource;
+        struct rlimit rlim;
+        if (!json_is_integer (value)
+            || (resource = rlimit_name_to_string (key)) < 0) {
+            char *s = json_dumps (value, JSON_ENCODE_ANY);
+            shell_log_error ("invalid shell option rlimit.%s%s%s",
+                             key,
+                             s ? "=" : "",
+                             s ? s : "");
+            free (s);
+            rc = -1;
+            continue;
+        }
+        if (getrlimit (resource, &rlim) < 0) {
+            shell_log_errno ("getrlimit %s", key);
+            continue;
+        }
+        rlim.rlim_cur = json_integer_value (value);
+        if (rlim.rlim_max != RLIM_INFINITY
+            && (rlim.rlim_max < rlim.rlim_cur
+                || rlim.rlim_cur == RLIM_INFINITY)) {
+            shell_warn ("%s exceeds current max, raising value to hard limit",
+                        key);
+            rlim.rlim_cur = rlim.rlim_max;
+        }
+        if (setrlimit (resource, &rlim) < 0)
+            shell_log_errno ("setrlimit %s", key);
+    }
+    return rc;
+}
+
+struct shell_builtin builtin_rlimit = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = rlimit_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/shell.c b/src/shell/shell.c
index aad738db9d77..887a732b2e0d 100644
--- a/src/shell/shell.c
+++ b/src/shell/shell.c
@@ -9,6 +9,7 @@
 \************************************************************/
 
 /* job shell mainline */
+#define FLUX_SHELL_PLUGIN_NAME NULL
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -18,14 +19,21 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <stdarg.h>
+#include <locale.h>
 #include <jansson.h>
-#include <czmq.h>
 #include <flux/core.h>
 #include <flux/shell.h>
+#include <flux/idset.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/liboptparse/optparse.h"
 #include "src/common/libeventlog/eventlog.h"
 #include "src/common/libutil/log.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/fdutils.h"
+#include "src/common/libutil/basename.h"
+#include "src/common/libtaskmap/taskmap_private.h"
+#include "ccan/str/str.h"
 
 #include "internal.h"
 #include "builtins.h"
@@ -34,26 +42,19 @@
 #include "task.h"
 #include "rc.h"
 #include "log.h"
+#include "mustache.h"
 
 static char *shell_name = "flux-shell";
 static const char *shell_usage = "[OPTIONS] JOBID";
 
 static struct optparse_option shell_opts[] =  {
-    { .name = "jobspec", .key = 'j', .has_arg = 1, .arginfo = "FILE",
-      .usage = "Get jobspec from FILE, not job-info service", },
-    { .name = "resources", .key = 'R', .has_arg = 1, .arginfo = "FILE",
-      .usage = "Get R from FILE, not job-info service", },
-    { .name = "broker-rank", .key = 'r', .has_arg = 1, .arginfo = "RANK",
-      .usage = "Set broker rank, rather than asking broker", },
-    { .name = "verbose", .key = 'v', .has_arg = 0,
-      .usage = "Log actions to stderr", },
-    { .name = "standalone", .key = 's', .has_arg = 0,
-      .usage = "Run local program without Flux instance", },
-    { .name = "initrc", .has_arg = 1, .arginfo = "FILE",
-      .usage = "Load shell initrc from FILE instead of the system default" },
+    { .name = "reconnect", .has_arg = 0,
+      .usage = "Attempt to reconnect if broker connection is lost" },
     OPTPARSE_TABLE_END
 };
 
+static void shell_events_subscribe (flux_shell_t *shell);
+
 /* Parse optarg as a jobid rank and assign to 'jobid'.
  * Return 0 on success or -1 on failure (log error).
  */
@@ -94,12 +95,6 @@ int flux_shell_setopt (flux_shell_t *shell,
 {
     json_error_t err;
     json_t *o;
-    if (!shell->info->jobspec->options) {
-        if (!(shell->info->jobspec->options = json_object ())) {
-            errno = ENOMEM;
-            return -1;
-        }
-    }
     /* If flux_shell_setopt (shell, name, NULL), delete option:
      */
     if (!json_str)
@@ -142,8 +137,14 @@ int flux_shell_getopt (flux_shell_t *shell, const char *name, char **json_str)
      *   just a fragment of the shell.options object, and these options
      *   themselves do not have a requirement of being JSON objects.
      */
-    if (json_str)
-        *json_str = json_dumps (o, JSON_COMPACT|JSON_ENCODE_ANY);
+    if (json_str) {
+        char *s = json_dumps (o, JSON_COMPACT|JSON_ENCODE_ANY);
+        if (!s) {
+            errno = ENOMEM;
+            return -1;
+        }
+        *json_str = s;
+    }
     return 1;
 }
 
@@ -199,40 +200,102 @@ static void shell_parse_cmdline (flux_shell_t *shell, int argc, char *argv[])
     if (parse_jobid (argv[optindex++], &shell->jobid) < 0)
         exit (1);
 
-    /* In standalone mode, jobspec, resources and broker-rank must be
-     *  set on command line:
-     */
-    if ((shell->standalone = optparse_hasopt (p, "standalone"))) {
-        if (  !optparse_hasopt (p, "jobspec")
-           || !optparse_hasopt (p, "resources")
-           || !optparse_hasopt (p, "broker-rank"))
-            shell_die (1, "standalone mode requires --jobspec, "
-                       "--resources and --broker-rank");
+    shell->p = p;
+}
+
+static int try_reconnect (flux_t *h)
+{
+    int rc = -1;
+    flux_future_t *f;
+
+    if (flux_reconnect (h) < 0) {
+        if (errno == ENOSYS)
+            shell_die (1, "reconnect not implemented by connector");
+        return -1;
     }
 
-    if ((shell->verbose = optparse_getopt (p, "verbose", NULL)))
-        shell_set_verbose (shell->verbose);
-    shell->broker_rank = optparse_get_int (p, "broker-rank", -1);
-    shell->p = p;
+    /*  Wait for broker to enter RUN state.
+     *
+     *  RPC may fail if broker is still shutting down. In that case, return
+     *   error so that we reconnect again and retry.
+     */
+    if (!(f = flux_rpc (h, "state-machine.wait", NULL, FLUX_NODEID_ANY, 0)))
+        shell_die (1, "could not send state-machine.wait RPC");
+    rc = flux_rpc_get (f, NULL);
+
+    flux_future_destroy (f);
+    return rc;
+}
+
+static int reconnect (flux_t *h, void *arg)
+{
+    flux_future_t *f;
+    flux_shell_t *shell = arg;
+
+    shell_log_errno ("broker");
+    while (try_reconnect (h) < 0)
+        sleep (2);
+
+    shell_events_subscribe (shell);
+
+    if (!(f = flux_service_register (h, shell_svc_name (shell->svc))))
+        shell_die (1, "could not re-register shell service name");
+    if (flux_rpc_get (f, NULL) < 0)
+        shell_die (1, "flux_service_register: %s", future_strerror (f, errno));
+    flux_future_destroy (f);
+
+    if (plugstack_call (shell->plugstack, "shell.reconnect", NULL) < 0)
+        shell_log_errno ("shell.reconnect");
+
+    if (shell_eventlogger_reconnect (shell->ev) < 0)
+        shell_log_errno ("shell_eventlogger_reconnect");
+
+    shell_log ("broker: reconnected");
+    return 0;
+}
+
+static uid_t get_instance_owner (flux_t *h)
+{
+    const char *s;
+    char *endptr;
+    int id;
+
+    if (!(s = flux_attr_get (h, "security.owner"))) {
+        shell_log_errno ("error fetching security.owner attribute");
+        return (uid_t) 0;
+    }
+    errno = 0;
+    id = strtoul (s, &endptr, 10);
+    if (errno != 0 || *endptr != '\0') {
+        shell_log_error ("error parsing security.owner=%s", s);
+        return (uid_t) 0;
+    }
+    return (uid_t) id;
 }
 
 static void shell_connect_flux (flux_shell_t *shell)
 {
-    if (!(shell->h = flux_open (shell->standalone ? "loop://" : NULL, 0)))
+    uint32_t rank;
+    int flags = optparse_hasopt (shell->p, "reconnect") ? FLUX_O_RPCTRACK : 0;
+
+    if (!(shell->h = flux_open (NULL, flags)))
         shell_die_errno (1, "flux_open");
 
+    if (optparse_hasopt (shell->p, "reconnect"))
+        flux_comms_error_set (shell->h, reconnect, shell);
+
     /*  Set reactor for flux handle to our custom created reactor.
      */
     flux_set_reactor (shell->h, shell->r);
 
-    /*  Fetch local rank if not already set
+    /*  Fetch local rank
      */
-    if (shell->broker_rank < 0) {
-        uint32_t rank;
-        if (flux_get_rank (shell->h, &rank) < 0)
-            shell_log_errno ("error fetching broker rank");
-        shell->broker_rank = rank;
-    }
+    if (flux_get_rank (shell->h, &rank) < 0)
+        shell_log_errno ("error fetching broker rank");
+    shell->broker_rank = rank;
+
+    shell->broker_owner = get_instance_owner (shell->h);
+
     if (plugstack_call (shell->plugstack, "shell.connect", NULL) < 0)
         shell_log_errno ("shell.connect");
 }
@@ -283,12 +346,37 @@ const char * flux_shell_getenv (flux_shell_t *shell, const char *name)
 
 int flux_shell_get_environ (flux_shell_t *shell, char **json_str)
 {
+    char *s;
     if (!shell || !json_str) {
         errno = EINVAL;
         return -1;
     }
-    *json_str = json_dumps (shell->info->jobspec->environment, JSON_COMPACT);
+    if (!(s = json_dumps (shell->info->jobspec->environment, JSON_COMPACT))) {
+        errno = ENOMEM;
+        return -1;
+    }
+    *json_str = s;
+    return 0;
+}
+
+static int object_set_string (json_t *dict, const char *name, const char *val)
+{
+    json_t *o;
+
+    if (!dict || !name || !val) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(o = json_string (val)))
+        goto nomem;
+    if (json_object_set_new (dict, name, o) < 0) {
+        json_decref (o);
+        goto nomem;
+    }
     return 0;
+nomem:
+    errno = ENOMEM;
+    return -1;
 }
 
 int flux_shell_setenvf (flux_shell_t *shell, int overwrite,
@@ -305,30 +393,93 @@ int flux_shell_setenvf (flux_shell_t *shell, int overwrite,
     }
 
     env = shell->info->jobspec->environment;
-    if (!overwrite && json_object_get (env, name)) {
-        errno = EEXIST;
-        return -1;
-    }
+    if (!overwrite && json_object_get (env, name))
+        return 0;
 
     va_start (ap, fmt);
     rc = vasprintf (&val, fmt, ap);
     va_end (ap);
     if (rc >= 0) {
-        json_t *o = json_string (val);
-        if (o)
-            rc = json_object_set_new (env, name, o);
-        free (val);
+        rc = object_set_string (env, name, val);
+        ERRNO_SAFE_WRAP (free, val);
     }
     return rc;
 }
 
 int flux_shell_unsetenv (flux_shell_t *shell, const char *name)
 {
+    int rc;
     if (!shell || !name) {
         errno = EINVAL;
         return -1;
     }
-    return json_object_del (shell->info->jobspec->environment, name);
+    if ((rc = json_object_del (shell->info->jobspec->environment, name)) < 0)
+        errno = ENOENT;
+    return rc;
+}
+
+int flux_shell_get_hwloc_xml (flux_shell_t *shell, const char **xmlp)
+{
+    if (!shell || !shell->info || !xmlp) {
+        errno = EINVAL;
+        return -1;
+    }
+    *xmlp = shell->info->hwloc_xml;
+    return 0;
+}
+
+const struct taskmap *flux_shell_get_taskmap (flux_shell_t *shell)
+{
+    if (!shell || !shell->info) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return shell->info->taskmap;
+}
+
+static struct hostlist *hostlist_from_R (flux_shell_t *shell)
+{
+    size_t i;
+    json_t *nodelist;
+    json_t *val;
+    struct hostlist *hl = NULL;
+
+    if (flux_shell_info_unpack (shell,
+                                "{s:{s:{s:o}}}",
+                                "R",
+                                 "execution",
+                                  "nodelist", &nodelist) < 0) {
+        shell_log_errno ("unable to get job nodelist");
+        return NULL;
+    }
+    if (!(hl = hostlist_create ())) {
+        shell_log_errno ("hostlist_create");
+        return NULL;
+    }
+    json_array_foreach (nodelist, i, val) {
+        const char *host = json_string_value (val);
+        if (!host)
+            goto error;
+        if (hostlist_append (hl, host) < 0) {
+            shell_log_errno ("hostlist_append %s", host);
+            goto error;
+        }
+    }
+    return hl;
+error:
+    hostlist_destroy (hl);
+    return NULL;
+}
+
+const struct hostlist *flux_shell_get_hostlist (flux_shell_t *shell)
+{
+    if (!shell || !shell->info) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!shell->info->hostlist)
+        shell->info->hostlist = hostlist_from_R (shell);
+    return shell->info->hostlist;
 }
 
 static json_t *flux_shell_get_info_object (flux_shell_t *shell)
@@ -343,15 +494,17 @@ static json_t *flux_shell_get_info_object (flux_shell_t *shell)
         return o;
 
     if (!(o = json_pack_ex (&err, 0,
-                            "{ s:I s:i s:i s:i s:O s:{ s:i s:b }}",
+                            "{ s:I s:i s:i s:i s:i s:s s:O s:O s:{ s:i }}",
                             "jobid", shell->info->jobid,
                             "rank",  shell->info->shell_rank,
+                            "instance_owner", (int) shell->broker_owner,
                             "size",  shell->info->shell_size,
-                            "ntasks", shell->info->rankinfo.ntasks,
+                            "ntasks", shell->info->total_ntasks,
+                            "service", shell_svc_name (shell->svc),
                             "jobspec", shell->info->jobspec->jobspec,
+                            "R", shell->info->R,
                             "options",
-                               "verbose", shell->verbose,
-                               "standalone", shell->standalone)))
+                               "verbose", shell->verbose)))
         return NULL;
     if (flux_shell_aux_set (shell,
                             "shell::info",
@@ -366,14 +519,20 @@ static json_t *flux_shell_get_info_object (flux_shell_t *shell)
 int flux_shell_get_info (flux_shell_t *shell, char **json_str)
 {
     json_t *o;
+    char *s;
+
     if (!shell || !json_str) {
         errno = EINVAL;
         return -1;
     }
     if (!(o = flux_shell_get_info_object (shell)))
         return -1;
-    *json_str = json_dumps (o, JSON_COMPACT);
-    return (*json_str ? 0 : -1);
+    if (!(s = json_dumps (o, JSON_COMPACT))) {
+        errno = ENOMEM;
+        return -1;
+    }
+    *json_str = s;
+    return 0;
 }
 
 int flux_shell_info_vunpack (flux_shell_t *shell, const char *fmt, va_list ap)
@@ -400,32 +559,58 @@ int flux_shell_info_unpack (flux_shell_t *shell, const char *fmt, ...)
     return rc;
 }
 
+static char *get_rank_task_idset (struct taskmap *map, int nodeid)
+{
+    const struct idset *ids;
+    if (!(ids = taskmap_taskids (map, nodeid))) {
+        shell_log_errno ("unable to get taskids set for rank %d", nodeid);
+        return NULL;
+    }
+    return idset_encode (ids, IDSET_FLAG_RANGE);
+}
+
 static json_t *flux_shell_get_rank_info_object (flux_shell_t *shell, int rank)
 {
     json_t *o;
+    json_error_t error;
     char key [128];
-    struct rcalc_rankinfo ri;
+    char *taskids = NULL;
+    struct taskmap *map;
+    struct rcalc_rankinfo rankinfo;
 
     if (!shell->info)
         return NULL;
 
     if (rank == -1)
         rank = shell->info->shell_rank;
-    if (rcalc_get_nth (shell->info->rcalc, rank, &ri) < 0)
-        return NULL;
 
+    if (rank < 0 || rank > shell->info->shell_size - 1) {
+        errno = EINVAL;
+        return NULL;
+    }
     if (snprintf (key, sizeof (key), "shell::rinfo%d", rank) >= sizeof (key))
         return NULL;
 
     if ((o = flux_shell_aux_get (shell, key)))
         return o;
 
-    if (!(o = json_pack ("{ s:i s:i s:{s:s s:s?}}",
-                         "broker_rank", ri.rank,
-                         "ntasks", ri.ntasks,
-                         "resources",
-                           "cores", shell->info->rankinfo.cores,
-                           "gpus",  shell->info->rankinfo.gpus)))
+    map = shell->info->taskmap;
+    if (!(taskids = get_rank_task_idset (map, rank)))
+        return NULL;
+
+    if (rcalc_get_nth (shell->info->rcalc, rank, &rankinfo) < 0)
+        return NULL;
+
+    o = json_pack_ex (&error, 0, "{ s:i s:i s:s s:{s:s s:s?}}",
+                   "broker_rank", rankinfo.rank,
+                   "ntasks", taskmap_ntasks (map, rank),
+                   "taskids", taskids,
+                   "resources",
+                     "cores", rankinfo.cores,
+                     "gpus",  rankinfo.gpus);
+    free (taskids);
+
+    if (o == NULL)
         return NULL;
 
     if (flux_shell_aux_set (shell, key, o, (flux_free_f) json_decref) < 0) {
@@ -441,6 +626,7 @@ int flux_shell_get_rank_info (flux_shell_t *shell,
                               char **json_str)
 {
     json_t *o = NULL;
+    char *s;
 
     if (!shell || !json_str || shell_rank < -1) {
         errno = EINVAL;
@@ -448,8 +634,12 @@ int flux_shell_get_rank_info (flux_shell_t *shell,
     }
     if (!(o = flux_shell_get_rank_info_object (shell, shell_rank)))
         return -1;
-    *json_str = json_dumps (o, JSON_COMPACT);
-    return (*json_str ? 0 : -1);
+    if (!(s = json_dumps (o, JSON_COMPACT))) {
+        errno = ENOMEM;
+        return -1;
+    }
+    *json_str = s;
+    return 0;
 }
 
 int flux_shell_rank_info_vunpack (flux_shell_t *shell,
@@ -481,6 +671,90 @@ int flux_shell_rank_info_unpack (flux_shell_t *shell,
     return rc;
 }
 
+static json_t *flux_shell_get_jobspec_info_object (flux_shell_t *shell)
+{
+    json_t *o = NULL;
+    struct jobspec *jobspec;
+    if (!(jobspec = shell->info->jobspec))
+        return NULL;
+
+    if ((o = flux_shell_aux_get (shell, "shell::jobspec_info")))
+        return o;
+
+    /*  Only v1 supported for now:
+     */
+    if (jobspec->version == 1) {
+        o = json_pack ("{s:i s:i s:i s:i s:i s:i}",
+                       "version", jobspec->version,
+                       "ntasks", jobspec->task_count,
+                       "nslots", jobspec->slot_count,
+                       "cores_per_slot", jobspec->cores_per_slot,
+                       "nnodes", jobspec->node_count,
+                       "slots_per_node", jobspec->slots_per_node);
+    }
+    else
+        o = json_pack ("{s:i}", "version", jobspec->version);
+
+    if (o == NULL)
+        return NULL;
+
+    if (flux_shell_aux_set (shell,
+                            "shell::jobspec_info",
+                            o,
+                            (flux_free_f) json_decref) < 0) {
+        json_decref (o);
+        return NULL;
+    }
+    return o;
+}
+
+int flux_shell_get_jobspec_info (flux_shell_t *shell, char **json_str)
+{
+    json_t *o;
+    char *s;
+
+    if (!shell || !json_str) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(o = flux_shell_get_jobspec_info_object (shell)))
+        return -1;
+    if (!(s = json_dumps (o, JSON_COMPACT))) {
+        errno = ENOMEM;
+        return -1;
+    }
+    *json_str = s;
+    return 0;
+}
+
+int flux_shell_jobspec_info_vunpack (flux_shell_t *shell,
+                                     const char *fmt,
+                                     va_list ap)
+{
+    json_t *o;
+    json_error_t err;
+    if (!shell || !fmt) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(o = flux_shell_get_jobspec_info_object (shell)))
+        return -1;
+    return json_vunpack_ex (o, &err, 0, fmt, ap);
+}
+
+
+int flux_shell_jobspec_info_unpack (flux_shell_t *shell,
+                                    const char *fmt, ...)
+{
+    int rc;
+    va_list ap;
+
+    va_start (ap, fmt);
+    rc = flux_shell_jobspec_info_vunpack (shell, fmt, ap);
+    va_end (ap);
+    return rc;
+}
+
 
 int flux_shell_add_event_handler (flux_shell_t *shell,
                                   const char *subtopic,
@@ -516,16 +790,66 @@ int flux_shell_add_event_handler (flux_shell_t *shell,
     return 0;
 }
 
+struct service_wrap_arg
+{
+    flux_shell_t *shell;
+    flux_msg_handler_f cb;
+    void *arg;
+};
+
+static void shell_service_wrap (flux_t *h,
+                                flux_msg_handler_t *mh,
+                                const flux_msg_t *msg,
+                                void *arg)
+{
+    struct service_wrap_arg *sarg = arg;
+
+    if (shell_svc_allowed (sarg->shell->svc, msg) < 0)
+        goto error;
+    (*sarg->cb) (h, mh, msg, sarg->arg);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        shell_log_errno ("flux_respond");
+}
+
+static struct service_wrap_arg *
+service_wrap_arg_create (flux_shell_t *shell,
+                        flux_msg_handler_f cb,
+                        void *arg)
+{
+    struct service_wrap_arg *sarg = calloc (1, sizeof (*sarg));
+    if (!sarg)
+        return NULL;
+    sarg->shell = shell;
+    sarg->cb = cb;
+    sarg->arg = arg;
+    return sarg;
+}
+
 int flux_shell_service_register (flux_shell_t *shell,
                                  const char *method,
                                  flux_msg_handler_f cb,
                                  void *arg)
 {
+    struct service_wrap_arg *sarg = NULL;
+
     if (!shell || !method || !cb) {
         errno = EINVAL;
         return -1;
     }
-    return shell_svc_register (shell->svc, method, cb, arg);
+    if (!(sarg = service_wrap_arg_create (shell, cb, arg)))
+        return -1;
+
+    if (flux_shell_aux_set (shell, NULL, sarg, free) < 0) {
+        free (sarg);
+        return -1;
+    }
+
+    return shell_svc_register (shell->svc,
+                               method,
+                               shell_service_wrap,
+                               sarg);
 }
 
 flux_future_t *flux_shell_rpc_pack (flux_shell_t *shell,
@@ -648,6 +972,7 @@ static void shell_finalize (flux_shell_t *shell)
     shell->plugstack = NULL;
     plugstack_destroy (plugstack);
 
+    mustache_renderer_destroy (shell->mr);
     shell_eventlogger_destroy (shell->ev);
     shell_svc_destroy (shell->svc);
     shell_info_destroy (shell->info);
@@ -673,15 +998,173 @@ static const char *shell_conf_get (const char *name)
     return flux_conf_builtin_get (name, FLUX_CONF_AUTO);
 }
 
+static int get_protocol_fd (int *pfd)
+{
+    const char *s;
+
+    if ((s = getenv ("FLUX_EXEC_PROTOCOL_FD"))) {
+        char *endptr;
+        int fd;
+
+        errno = 0;
+        fd = strtol (s, &endptr, 10);
+        if (errno != 0 || *endptr != '\0') {
+            errno = EINVAL;
+            return -1;
+        }
+        if (fd_set_cloexec (fd) < 0)
+            return -1;
+        pfd[0] = fd;
+        pfd[1] = fd;
+        return 0;
+    }
+    pfd[0] = STDIN_FILENO;
+    pfd[1] = STDOUT_FILENO;
+    return 0;
+}
+
+char *flux_shell_mustache_render (flux_shell_t *shell, const char *fmt)
+{
+    if (!shell) {
+        /* Note: shell->mr and fmt checked in mustache_render */
+        errno = EINVAL;
+        return NULL;
+    }
+    return mustache_render (shell->mr, fmt);
+}
+
+static int mustache_render_name (flux_shell_t *shell,
+                                 const char *name,
+                                 FILE *fp)
+{
+    const char *jobname = NULL;
+    json_error_t error;
+    if (json_unpack_ex (shell->info->jobspec->jobspec, &error, 0,
+                        "{s:{s:{s?{s?s}}}}",
+                        "attributes",
+                         "system",
+                          "job",
+                           "name", &jobname) < 0) {
+        shell_log_error ("render_name: %s", error.text);
+        jobname = NULL;
+    }
+    if (!jobname) {
+        json_t *cmd = json_array_get (shell->info->jobspec->command, 0);
+        if (!cmd
+            || !(jobname = json_string_value (cmd))
+            || !(jobname = basename_simple (jobname)))
+            jobname = "unknown";
+    }
+    if (fputs (jobname, fp) == EOF) {
+        shell_log_error ("memstream write failed for %s: %s",
+                         name,
+                         strerror (errno));
+    }
+    return 0;
+}
+
+static int mustache_render_jobid (flux_shell_t *shell,
+                                  const char *name,
+                                  FILE *fp)
+{
+    char value[128];
+    const char *type = "f58";
+
+    if (strlen (name) > 2) {
+        if (name[2] != '.') {
+            shell_log_error ("Unknown mustache tag '%s'", name);
+            return -1;
+        }
+        type = name+3;
+    }
+    if (flux_job_id_encode (shell->info->jobid,
+                            type,
+                            value,
+                            sizeof (value)) < 0) {
+        if (errno == EPROTO)
+            shell_log_error ("Invalid jobid encoding '%s' specified", name);
+        else
+            shell_log_errno ("flux_job_id_encode failed for %s", name);
+        return -1;
+    }
+    if (fputs (value, fp) < 0) {
+        shell_log_error ("memstream write failed for %s: %s",
+                         name,
+                         strerror (errno));
+    }
+    return 0;
+}
+
+static int mustache_cb (FILE *fp, const char *name, void *arg)
+{
+    int rc = -1;
+    flux_plugin_arg_t *args;
+    flux_shell_t *shell = arg;
+    const char *result = NULL;
+    char topic[128];
+
+    /*  "jobid" is a synonym for "id" */
+    if (strstarts (name, "jobid"))
+        name += 3;
+    if (strstarts (name, "id"))
+        return mustache_render_jobid (shell, name, fp);
+    if (streq (name, "name"))
+        return mustache_render_name (shell, name, fp);
+
+    if (snprintf (topic,
+                  sizeof (topic),
+                  "mustache.render.%s",
+                  name) >= sizeof (topic)) {
+        shell_log_error ("mustache template name '%s' too long", name);
+        return -1;
+    }
+    if (!(args = flux_plugin_arg_create ())) {
+        shell_log_error ("mustache_cb: failed to create plugin args");
+        return -1;
+    }
+    if (plugstack_call (shell->plugstack, topic, args) < 0) {
+        shell_log_errno ("%s", topic);
+        goto out;
+    }
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_OUT,
+                                "{s:s}",
+                                "result", &result) < 0
+        || result == NULL) {
+        shell_log_error ("Unknown mustache tag '%s'", name);
+        goto out;
+    }
+    if (fputs (result, fp) < 0) {
+        shell_log_error ("memstream write failed for %s: %s",
+                         name,
+                         strerror (errno));
+    }
+    rc = 0;
+out:
+    flux_plugin_arg_destroy (args);
+    return rc;
+}
+
 static void shell_initialize (flux_shell_t *shell)
 {
     const char *pluginpath = shell_conf_get ("shell_pluginpath");
 
     memset (shell, 0, sizeof (struct flux_shell));
+
+    if (gethostname (shell->hostname, sizeof (shell->hostname)) < 0)
+        shell_die_errno (1, "gethostname");
+
+    if (get_protocol_fd (shell->protocol_fd) < 0)
+        shell_die_errno (1, "Failed to parse FLUX_EXEC_PROTOCOL_FD");
+
     if (!(shell->completion_refs = zhashx_new ()))
         shell_die_errno (1, "zhashx_new");
     zhashx_set_destructor (shell->completion_refs, item_free);
 
+    if (!(shell->mr = mustache_renderer_create (mustache_cb, shell)))
+        shell_die_errno (1, "mustache_renderer_create");
+    mustache_renderer_set_log (shell->mr, shell_llog, NULL);
+
     if (!(shell->plugstack = plugstack_create ()))
         shell_die_errno (1, "plugstack_create");
 
@@ -792,151 +1275,86 @@ int flux_shell_add_event_context (flux_shell_t *shell,
     return rc;
 }
 
-static void eventlog_cb (flux_future_t *f, void *arg)
-{
-    json_t *o = NULL;
-    json_t *context = NULL;
-    const char *name;
-    const char *entry;
-
-    if (flux_job_event_watch_get (f, &entry) < 0) {
-        if (errno == ENODATA)
-            return;
-        shell_log_errno ("flux_job_event_watch_get");
-        return;
-    }
-    if (!(o = eventlog_entry_decode (entry))) {
-        shell_log_errno ("eventlog_entry_decode: %s", entry);
-        return;
-    }
-    if (eventlog_entry_parse (o, NULL, &name, &context) < 0) {
-        shell_log_errno ("eventlog_entry_parse: %s", entry);
-        return;
-    }
-    if (strcmp (name, "exception") == 0) {
-        const char *type;
-        int severity;
-        const char *note = NULL;
-        if (json_unpack (context, "{s:s s:i s?s}",
-                         "type", &type,
-                         "severity", &severity,
-                         "note", &note) < 0) {
-            shell_log_errno ("exception event unpack");
-            return;
-        }
-        shell_log_set_exception_logged ();
-        shell_die (1, "job.exception during init barrier, aborting");
-    }
-    json_decref (o);
-    flux_future_reset (f);
-}
-
-static void barrier_cb (flux_future_t *f, void *arg)
-{
-    flux_reactor_t *r = flux_future_get_reactor (f);
-    if (flux_future_get (f, NULL) < 0) {
-        shell_log_errno ("flux_future_get: shell_barrier");
-        flux_reactor_stop_error (r);
-    }
-    else {
-        shell_trace ("shell barrier complete");
-        flux_reactor_stop (r);
-    }
-}
-
 static int shell_barrier (flux_shell_t *shell, const char *name)
 {
-    flux_future_t *log_f = NULL;
-    flux_future_t *f = NULL;
-    flux_t *h = NULL;
-    flux_jobid_t id;
-    char fqname[128];
-    int rc = -1;
+    char buf [8];
 
-    if (shell->standalone || shell->info->shell_size == 1)
+    if (shell->info->shell_size == 1)
         return 0; // NO-OP
-    id = shell->info->jobid;
-    if (snprintf (fqname,
-                  sizeof (fqname),
-                  "shell-%ju-%s",
-                  (uintmax_t) id,
-                   name) >= sizeof (fqname)) {
-        errno = EINVAL;
-        return -1;
-    }
-    /*  Clone shell flux handle so that only barrier and eventlog watch
-     *   messages are dispatched in the temporary reactor call here.
-     *   This allows messages from other shell services to be requeued
-     *   for the real reactor in main().
-     */
-    if (!(h = flux_clone (shell->h)))
-        shell_die_errno (1, "flux_handle_clone");
 
-    if (!(f = flux_barrier (h, fqname, shell->info->shell_size))) {
-        shell_log_errno ("flux_barrier");
-        goto out;
-    }
-    if (!(log_f = flux_job_event_watch (h, id, "eventlog", 0))) {
-        shell_log_errno ("flux_job_event_watch");
-        goto out;
-    }
-    if (flux_future_then (log_f, -1., eventlog_cb, NULL) < 0
-        ||  flux_future_then (f, -1., barrier_cb, NULL) < 0) {
-        shell_log_errno ("flux_future_then");
-        goto out;
-    }
-    if (flux_future_then (f, -1., barrier_cb, shell) < 0) {
-        shell_log_errno ("flux_future_then");
-        goto out;
-    }
-    if (flux_reactor_run (flux_get_reactor (h), 0) >= 0)
-        rc = 0;
-    shell_trace ("exited barrier with rc = %d", rc);
-out:
-    flux_job_event_watch_cancel (log_f);
-    flux_future_destroy (log_f);
-    flux_future_destroy (f);
+    if (shell->protocol_fd[1] < 0)
+        shell_die (1, "required FLUX_EXEC_PROTOCOL_FD not set");
 
-    /*  Close the cloned handle */
-    flux_close (h);
-    return rc;
+    if (dprintf (shell->protocol_fd[1], "enter\n") != 6)
+        shell_die_errno (1, "shell_barrier: dprintf");
+
+    /*  Note: The only expected values currently are "exit=0\n"
+     *   for success and "exit=1\n" for failure. Therefore, if
+     *   read(2) fails, or we don't receive exactly "exit=0\n",
+     *   then this barrier has failed. We exit immediately since
+     *   the reason for the failed barrier has likely been logged
+     *   elsewhere.
+     */
+    memset (buf, 0, sizeof (buf));
+    if (read (shell->protocol_fd[0], buf, 7) < 0)
+        shell_die_errno (1, "shell_barrier: read");
+    if (!streq (buf, "exit=0\n"))
+        exit (1);
+    return 0;
 }
 
-static int shell_init (flux_shell_t *shell)
+static int load_initrc (flux_shell_t *shell, const char *default_rcfile)
 {
     bool required = false;
     const char *rcfile = NULL;
 
-    /* If initrc is set on commmand line or in jobspec, then
+    /* If initrc is set on command line or in jobspec, then
      *  it is required, O/w initrc is treated as empty file.
      */
-    if (optparse_getopt (shell->p, "initrc", &rcfile) > 0
-        || flux_shell_getopt_unpack (shell, "initrc", "s", &rcfile) > 0)
+    if (flux_shell_getopt_unpack (shell, "initrc", "s", &rcfile) > 0)
         required = true;
     else
-        rcfile = shell_conf_get ("shell_initrc");
+        rcfile = default_rcfile;
 
-    /*  Only try loading initrc if the file is readable or required:
+    /* Skip loading initrc file if it is not required or the file isn't
+     * readable.
      */
-    if (access (rcfile, R_OK) == 0 || required) {
-        shell_debug ("Loading %s", rcfile);
-
-        if (shell_rc (shell, rcfile) < 0) {
-            shell_die (1, "loading rc file %s%s%s",
-                       rcfile,
-                       errno ? ": " : "",
-                       errno ? strerror (errno) : "");
-        }
+    if (!required && access (rcfile, R_OK) < 0)
+        return 0;
+
+    shell_debug ("Loading %s", rcfile);
+
+    if (shell_rc (shell, rcfile) < 0) {
+        shell_die (1, "loading rc file %s%s%s",
+                   rcfile,
+                   errno ? ": " : "",
+                   errno ? strerror (errno) : "");
+        return -1;
     }
 
+    return 0;
+}
+
+static int shell_initrc (flux_shell_t *shell)
+{
+    const char *default_rcfile = shell_conf_get ("shell_initrc");
+    const char *result;
+
+    if ((result = flux_attr_get (shell->h, "conf.shell_pluginpath")))
+        plugstack_set_searchpath (shell->plugstack, result);
+    if ((result = flux_attr_get (shell->h, "conf.shell_initrc")))
+        default_rcfile = result;
+
     /* Change current working directory once before all tasks are
      * created, so that each task does not need to chdir().
      */
     if (shell->info->jobspec->cwd) {
         if (chdir (shell->info->jobspec->cwd) < 0) {
-            shell_log_error ("Could not change dir to %s: %s. "
+            shell_log_error ("host %s: Could not change dir to %s: %s. "
                              "Going to /tmp instead",
-                             shell->info->jobspec->cwd, strerror (errno));
+                             shell->hostname,
+                             shell->info->jobspec->cwd,
+                             strerror (errno));
             if (chdir ("/tmp") < 0) {
                 shell_log_errno ("Could not change dir to /tmp");
                 return -1;
@@ -944,20 +1362,141 @@ static int shell_init (flux_shell_t *shell)
         }
     }
 
+    /*  Load initrc file if necessary
+     */
+    return load_initrc (shell, default_rcfile);
+}
+
+static int shell_taskmap (flux_shell_t *shell)
+{
+    int rc;
+    const char *scheme;
+    const char *value = "";
+    char *topic = NULL;
+    char *map = NULL;
+    char *newmap = NULL;
+    flux_plugin_arg_t *args = NULL;
+    struct taskmap *taskmap;
+    flux_error_t error;
+
+    if ((rc = flux_shell_getopt_unpack (shell,
+                                       "taskmap",
+                                       "{s:s s?s}",
+                                       "scheme", &scheme,
+                                       "value", &value)) < 0) {
+        shell_log_error ("failed to parse taskmap shell option");
+        return -1;
+    }
+    if (rc == 0)
+        return 0;
+    if (streq (scheme, "block")) {
+        /*  A value is not allowed for the block taskmap scheme:
+         */
+        if (strlen (value) > 0)
+            shell_die (1,
+                       "block taskmap does not accept a value (got %s)",
+                       value);
+        return 0;
+    }
+
+    shell_trace ("remapping tasks with scheme=%s value=%s",
+                 scheme,
+                 value);
+
+    if (streq (scheme, "manual")) {
+        if (!(taskmap = taskmap_decode (value, &error)))
+            shell_die (1, "taskmap=%s: %s", value, error.text);
+        if (shell_info_set_taskmap (shell->info, taskmap) < 0)
+            shell_die (1, "failed to set new shell taskmap");
+        return 0;
+    }
+
+    rc = -1;
+    if (!(map = taskmap_encode (shell->info->taskmap,
+                                TASKMAP_ENCODE_WRAPPED))) {
+        shell_log_errno ("taskmap.%s: taskmap_encode", scheme);
+        return -1;
+    }
+    if (!(args = flux_plugin_arg_create ())
+        || flux_plugin_arg_pack (args,
+                                 FLUX_PLUGIN_ARG_IN,
+                                 "{s:s s:s s:s}",
+                                 "taskmap", map,
+                                 "scheme", scheme,
+                                 "value", value) < 0) {
+        shell_log_error ("taskmap.%s: failed to create plugin args: %s",
+                         scheme,
+                         flux_plugin_arg_strerror (args));
+        goto out;
+    }
+    if (asprintf (&topic, "taskmap.%s", scheme) < 0
+        || plugstack_call (shell->plugstack, topic, args) < 0) {
+        shell_log_errno ("%s failed", topic);
+        goto out;
+    }
+    /*  Unpack arguments to get new taskmap  */
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_OUT,
+                                "{s:s}",
+                                "taskmap", &newmap) < 0
+        || newmap == NULL) {
+        shell_die (1, "failed to map tasks with scheme=%s", scheme);
+    }
+    if (!(taskmap = taskmap_decode (newmap, &error))) {
+        shell_log_error ("taskmap.%s returned invalid map: %s",
+                          scheme,
+                          error.text);
+        goto out;
+    }
+    if (shell_info_set_taskmap (shell->info, taskmap) < 0) {
+        shell_log_errno ("unable to update taskmap");
+        goto out;
+    }
+    if (shell->info->shell_rank == 0)
+        shell_debug ("taskmap uptdated to %s", newmap);
+    rc = 0;
+out:
+    flux_plugin_arg_destroy (args);
+    free (topic);
+    free (map);
+    return rc;
+}
+
+static int shell_init (flux_shell_t *shell)
+{
     return plugstack_call (shell->plugstack, "shell.init", NULL);
 }
 
+static int shell_post_init (flux_shell_t *shell)
+{
+    return plugstack_call (shell->plugstack, "shell.post-init", NULL);
+}
+
 static int shell_task_init (flux_shell_t *shell)
 {
     return plugstack_call (shell->plugstack, "task.init", NULL);
 }
 
+#if CODE_COVERAGE_ENABLED
+extern void __gcov_dump ();
+extern void __gcov_reset ();
+#endif
 static void shell_task_exec (flux_shell_task_t *task, void *arg)
 {
     flux_shell_t *shell = arg;
     shell->current_task->in_pre_exec = true;
+
+    /*  Set stdout to unbuffered so that any output from task.exec plugins
+     *   is not lost at exec(2).
+     */
+    (void) setvbuf (stdout, NULL, _IONBF, 0);
+
     if (plugstack_call (shell->plugstack, "task.exec", NULL) < 0)
         shell_log_errno ("task.exec plugin(s) failed");
+#if CODE_COVERAGE_ENABLED
+    __gcov_dump ();
+    __gcov_reset ();
+#endif
 }
 
 static int shell_task_forked (flux_shell_t *shell)
@@ -981,24 +1520,22 @@ static void shell_log_info (flux_shell_t *shell)
 {
     if (shell->verbose) {
         struct shell_info *info = shell->info;
+        char *taskids = idset_encode (info->taskids,
+                                      IDSET_FLAG_RANGE | IDSET_FLAG_BRACKETS);
+
         if (info->shell_rank == 0)
             shell_debug ("0: task_count=%d slot_count=%d "
                          "cores_per_slot=%d slots_per_node=%d",
-                         info->jobspec->task_count,
+                         info->total_ntasks,
                          info->jobspec->slot_count,
                          info->jobspec->cores_per_slot,
                          info->jobspec->slots_per_node);
-        if (info->rankinfo.ntasks > 1)
-            shell_debug ("%d: tasks [%d-%d] on cores %s",
-                         info->shell_rank,
-                         info->rankinfo.global_basis,
-                         info->rankinfo.global_basis+info->rankinfo.ntasks - 1,
-                         info->rankinfo.cores);
-        else
-            shell_debug ("%d: tasks [%d] on cores %s",
-                         info->shell_rank,
-                         info->rankinfo.global_basis,
-                         info->rankinfo.cores);
+        shell_debug ("%d: task%s %s on cores %s",
+                     info->shell_rank,
+                     idset_count (info->taskids) > 1 ? "s" : "",
+                     taskids ? taskids : "[unknown]",
+                     info->rankinfo.cores);
+        free (taskids);
     }
 }
 
@@ -1007,20 +1544,65 @@ static void shell_log_info (flux_shell_t *shell)
  */
 static int shell_register_event_context (flux_shell_t *shell)
 {
-    if (shell->standalone || shell->info->shell_rank != 0)
+    int rc = -1;
+    json_t *o = NULL;
+    if (shell->info->shell_rank != 0)
         return 0;
-    if (flux_shell_add_event_context (shell, "shell.init", 0,
-                                      "{s:i s:i}",
-                                      "leader-rank",
-                                      shell->info->rankinfo.rank,
-                                      "size",
-                                      shell->info->shell_size) < 0)
-        return -1;
-    if (flux_shell_add_event_context (shell, "shell.start", 0,
-                                      "{s:i}",
-                                      "task-count",
-                                      shell->info->jobspec->task_count) < 0)
-        return -1;
+    o = taskmap_encode_json (shell->info->taskmap, TASKMAP_ENCODE_WRAPPED);
+    if (o == NULL
+        || flux_shell_add_event_context (shell, "shell.init", 0,
+                                         "{s:i s:i}",
+                                         "leader-rank",
+                                         shell->info->rankinfo.rank,
+                                         "size",
+                                         shell->info->shell_size) < 0
+        || flux_shell_add_event_context (shell, "shell.start", 0,
+                                         "{s:O}",
+                                         "taskmap", o) < 0)
+        goto out;
+    rc = 0;
+out:
+    json_decref (o);
+    return rc;
+}
+
+/*  Export a static list of environment variables from the job environment
+ *   to the current shell environment. This is important for variables like
+ *   FLUX_F58_FORCE_ASCII which should influence some shell behavior.
+ */
+static int shell_export_environment_from_job (flux_shell_t *shell)
+{
+    const char *vars[] = {
+        "FLUX_F58_FORCE_ASCII",
+        NULL,
+    };
+
+    for (int i = 0; vars[i] != NULL; i++) {
+        const char *val = flux_shell_getenv (shell, vars[i]);
+        if (val && setenv (vars[i], val, 1) < 0)
+            return shell_log_errno ("setenv (%s)", vars[i]);
+    }
+    return 0;
+}
+
+/*  Render any mustache templates that appear in command arguments.
+ */
+static int frob_command (flux_shell_t *shell, flux_cmd_t *cmd)
+{
+    for (int i = 0; i < flux_cmd_argc (cmd); i++) {
+        if (strstr (flux_cmd_arg (cmd, i), "{{")) { // possibly mustachioed
+            char *narg;
+            if (!(narg = flux_shell_mustache_render (shell,
+                                                     flux_cmd_arg (cmd, i)))
+                || flux_cmd_argv_insert (cmd, i, narg) < 0) {
+                free (narg);
+                return -1;
+            }
+            free (narg);
+            if (flux_cmd_argv_delete (cmd, i + 1) < 0)
+                return -1;
+        }
+    }
     return 0;
 }
 
@@ -1028,6 +1610,11 @@ int main (int argc, char *argv[])
 {
     flux_shell_t shell;
     int i;
+    unsigned int taskid;
+
+    /* Initialize locale from environment
+     */
+    setlocale (LC_ALL, "");
 
     shell_log_init (&shell, shell_name);
 
@@ -1040,7 +1627,7 @@ int main (int argc, char *argv[])
     if (!(shell.r = flux_reactor_create (FLUX_REACTOR_SIGCHLD)))
         shell_die_errno (1, "flux_reactor_create");
 
-    /* Connect to broker, or if standalone, open loopback connector.
+    /* Connect to broker:
      */
     shell_connect_flux (&shell);
 
@@ -1057,51 +1644,81 @@ int main (int argc, char *argv[])
     if (!(shell.info = shell_info_create (&shell)))
         exit (1);
 
-    if (shell_register_event_context (&shell) < 0)
-        shell_die (1, "failed to add standard shell event context");
+    if (shell_export_environment_from_job (&shell) < 0)
+        exit (1);
 
     /* Set verbose flag if set in attributes.system.shell.verbose */
     if (flux_shell_getopt_unpack (&shell, "verbose", "i", &shell.verbose) < 0)
         shell_die (1, "failed to parse attributes.system.shell.verbose");
 
+    /* Set no_process_group if nosetpgrp option is set */
+    if (flux_shell_getopt_unpack (&shell,
+                                  "nosetpgrp", "i",
+                                  &shell.nosetpgrp) < 0)
+        shell_die (1, "failed to parse attributes.system.shell.nosetpgrp");
+
     /* Reinitialize log facility with new verbosity/shell.info */
     if (shell_log_reinit (&shell) < 0)
         shell_die_errno (1, "shell_log_reinit");
 
-    /* Now that verbosity may have changed, log shell startup info */
-    shell_log_info (&shell);
-
     /* Register service on the leader shell.
      */
     if (!(shell.svc = shell_svc_create (&shell)))
         shell_die (1, "shell_svc_create");
 
-    /* Call shell initialization routines and "shell_init" plugins.
+    /* Change working directory and Load shell initrc
+     */
+    if (shell_initrc (&shell) < 0)
+        shell_die_errno (1, "shell_initrc");
+
+    if (shell_taskmap (&shell) < 0)
+        shell_die (1, "shell_taskmap");
+
+    /* Register the default components of the shell.init eventlog event
+     * context. This includes the current taskmap, which may have been
+     * altered by a plugin during shell_initrc(), so this must be done
+     * after that call completes.
+     */
+    if (shell_register_event_context (&shell) < 0)
+        shell_die (1, "failed to add standard shell event context");
+
+    /* Call "shell_init" plugins.
      */
     if (shell_init (&shell) < 0)
         shell_die_errno (1, "shell_init");
 
+    /* Now that verbosity, task mapping, etc. may have changed, log
+     * basic shell info.
+     */
+    shell_log_info (&shell);
+
     /* Barrier to ensure initialization has completed across all shells.
      */
     if (shell_barrier (&shell, "init") < 0)
         shell_die_errno (1, "shell_barrier");
 
-    /*  Emit an event after barrier completion from rank 0 if not in
-     *   standalone mode.
+    /*  Emit an event after barrier completion from rank 0
      */
     if (shell.info->shell_rank == 0
-        && !shell.standalone
-        && shell_eventlogger_emit_event (shell.ev, 0, "shell.init") < 0)
+        && shell_eventlogger_emit_event (shell.ev, "shell.init") < 0)
             shell_die_errno (1, "failed to emit event shell.init");
 
+    /* Call shell.post-init plugins.
+     */
+    if (shell_post_init (&shell) < 0)
+        shell_die_errno (1, "shell_post_init");
+
     /* Create tasks
      */
     if (!(shell.tasks = zlist_new ()))
         shell_die (1, "zlist_new failed");
-    for (i = 0; i < shell.info->rankinfo.ntasks; i++) {
+
+    i = 0;
+    taskid = idset_first (shell.info->taskids);
+    while (taskid != IDSET_INVALID_ID) {
         struct shell_task *task;
 
-        if (!(task = shell_task_create (shell.info, i)))
+        if (!(task = shell_task_create (&shell, i, taskid)))
             shell_die (1, "shell_task_create index=%d", i);
 
         task->pre_exec_cb = shell_task_exec;
@@ -1113,7 +1730,12 @@ int main (int argc, char *argv[])
         if (shell_task_init (&shell) < 0)
             shell_die (1, "failed to initialize taskid=%d", i);
 
-        if (shell_task_start (task, shell.r, task_completion_cb, &shell) < 0) {
+        /*  Render any mustache templates in command args
+         */
+        if (frob_command (&shell, task->cmd))
+            shell_die (1, "failed rendering of mustachioed command args");
+
+        if (shell_task_start (&shell, task, task_completion_cb, &shell) < 0) {
             int ec = 1;
             /* bash standard, 126 for permission/access denied, 127
              * for command not found.  Note that shell only launches
@@ -1124,8 +1746,11 @@ int main (int argc, char *argv[])
                 ec = 126;
             else if (errno == ENOENT)
                 ec = 127;
-            shell_die (ec, "task %d: start failed: %s: %s",
-                       i, flux_cmd_arg (task->cmd, 0), strerror (errno));
+            shell_die (ec, "task %d (host %s): start failed: %s: %s",
+                       task->rank,
+                       shell.hostname,
+                       flux_cmd_arg (task->cmd, 0),
+                       strerror (errno));
         }
 
         if (zlist_append (shell.tasks, task) < 0)
@@ -1138,6 +1763,9 @@ int main (int argc, char *argv[])
          */
         if (shell_task_forked (&shell) < 0)
             shell_die (1, "shell_task_forked");
+
+        i++;
+        taskid = idset_next (shell.info->taskids, taskid);
     }
     /*  Reset current task since we've left task-specific context:
      */
@@ -1149,12 +1777,10 @@ int main (int argc, char *argv[])
     if (shell_barrier (&shell, "start") < 0)
         shell_die_errno (1, "shell_barrier");
 
-    /*  Emit an event after barrier completion from rank 0 if not in
-     *   standalone mode.
+    /*  Emit an event after barrier completion from rank 0
      */
     if (shell.info->shell_rank == 0
-        && !shell.standalone
-        && shell_eventlogger_emit_event (shell.ev, 0, "shell.start") < 0)
+        && shell_eventlogger_emit_event (shell.ev, "shell.start") < 0)
             shell_die_errno (1, "failed to emit event shell.start");
 
     /* Main reactor loop
diff --git a/src/shell/shell.h b/src/shell/shell.h
index ed4636f5b4df..3d4f909bf58e 100644
--- a/src/shell/shell.h
+++ b/src/shell/shell.h
@@ -12,6 +12,8 @@
 #define FLUX_SHELL_H
 
 #include <flux/core.h>
+#include <flux/taskmap.h>
+#include <flux/hostlist.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -93,14 +95,27 @@ int flux_shell_setenvf (flux_shell_t *shell, int overwrite,
  */
 int flux_shell_unsetenv (flux_shell_t *shell, const char *name);
 
+/*  Return the job shell's cached copy of hwloc XML.
+ */
+int flux_shell_get_hwloc_xml (flux_shell_t *shell, const char **xmlp);
+
+/*  Return the current shell taskmap
+ */
+const struct taskmap *flux_shell_get_taskmap (flux_shell_t *shell);
+
+/*  Return the list of hosts assigned to this job as a hostlist
+ */
+const struct hostlist *flux_shell_get_hostlist (flux_shell_t *shell);
 
 /*  Return shell info as a JSON string.
  *  {
  *   "jobid":I,
+ *   "instance_owner":i,
  *   "rank":i,
  *   "size":i,
  *   "ntasks";i,
- *   "options": { "verbose":b, "standalone":b },
+ *   "service":s,
+ *   "options": { "verbose":b },
  *   "jobspec":o,
  *   "R":o
  *  }
@@ -117,6 +132,7 @@ int flux_shell_info_unpack (flux_shell_t *shell,
  *  {
  *   "broker_rank":i,
  *   "ntasks":i
+ *   "taskids": s // task id list for this rank in RFC 22 idset form.
  *   "resources": { "cores":s, ... }
  *  }
  */
@@ -130,6 +146,33 @@ int flux_shell_rank_info_unpack (flux_shell_t *shell,
                                  int shell_rank,
                                  const char *fmt, ...);
 
+/*  Return summary information about jobspec
+ *
+ *  The only required member of the JSON object is a version number,
+ *   indicating the version of the jobspec for the current job:
+ *
+ *  {
+ *   "version":i         # jobspec version number
+ *  }
+ *
+ *  For jobspec version 1, the following keys are also provided:
+ *  {
+ *   "ntasks":i,         # number of tasks requested
+ *   "nslots":i,         # number of task slots
+ *   "cores_per_slot":i  # number of cores per task slot
+ *   "nnodes":i          # number of nodes requested, -1 if unset
+ *   "slots_per_node":i  # number of slots per node, -1 if unavailable
+ *  }
+ *
+ */
+int flux_shell_get_jobspec_info (flux_shell_t *shell, char **json_str);
+
+
+/*  Access jobspec info object with Jansson-style unpack args
+ */
+int flux_shell_jobspec_info_unpack (flux_shell_t *shell,
+                                    const char *fmt, ...);
+
 /*
  *  Take a "completion reference" on the shell object `shell`.
  *  This function takes a named reference on the shell so that it will
@@ -260,6 +303,7 @@ int flux_shell_task_channel_subscribe (flux_shell_task_t *task,
  *  in the context of shell logging.
  */
 enum {
+    FLUX_SHELL_QUIET  = -1,
     FLUX_SHELL_FATAL  = 0,  /* LOG_EMERG   */
     /* Level 1 Reserved */  /* LOG_ALERT   */
     /* Level 2 Reserved */  /* LOG_CRIT    */
@@ -270,32 +314,47 @@ enum {
     FLUX_SHELL_TRACE  = 7,  /* LOG_DEBUG   */
 };
 
+#ifndef FLUX_SHELL_PLUGIN_NAME
+# error "FLUX_SHELL_PLUGIN_NAME must be defined"
+#endif
+
 #define shell_trace(...) \
-    flux_shell_log (FLUX_SHELL_TRACE, __FILE__, __LINE__, __VA_ARGS__)
+    flux_shell_log (FLUX_SHELL_PLUGIN_NAME, \
+                    FLUX_SHELL_TRACE, __FILE__, __LINE__, __VA_ARGS__)
 
 #define shell_debug(...) \
-    flux_shell_log (FLUX_SHELL_DEBUG, __FILE__, __LINE__, __VA_ARGS__)
+    flux_shell_log (FLUX_SHELL_PLUGIN_NAME, \
+                    FLUX_SHELL_DEBUG, __FILE__, __LINE__, __VA_ARGS__)
 
 #define shell_log(...) \
-    flux_shell_log (FLUX_SHELL_NOTICE, __FILE__, __LINE__, __VA_ARGS__)
+    flux_shell_log (FLUX_SHELL_PLUGIN_NAME, \
+                    FLUX_SHELL_NOTICE, __FILE__, __LINE__, __VA_ARGS__)
 
 #define shell_warn(...) \
-    flux_shell_log (FLUX_SHELL_WARN, __FILE__, __LINE__, __VA_ARGS__)
+    flux_shell_log (FLUX_SHELL_PLUGIN_NAME, \
+                    FLUX_SHELL_WARN, __FILE__, __LINE__, __VA_ARGS__)
 
 #define shell_log_error(...) \
-    flux_shell_log (FLUX_SHELL_ERROR, __FILE__, __LINE__, __VA_ARGS__)
+    flux_shell_log (FLUX_SHELL_PLUGIN_NAME, \
+                    FLUX_SHELL_ERROR, __FILE__, __LINE__, __VA_ARGS__)
 
 #define shell_log_errn(errn, ...) \
-    flux_shell_err (__FILE__, __LINE__, errn, __VA_ARGS__)
+    flux_shell_err (FLUX_SHELL_PLUGIN_NAME, \
+                    __FILE__, __LINE__, errn, __VA_ARGS__)
 
 #define shell_log_errno(...) \
-    flux_shell_err (__FILE__, __LINE__, errno, __VA_ARGS__)
+    flux_shell_err (FLUX_SHELL_PLUGIN_NAME, \
+                    __FILE__, __LINE__, errno, __VA_ARGS__)
 
 #define shell_die(code,...) \
-    flux_shell_fatal (__FILE__, __LINE__, 0, code, __VA_ARGS__)
+    flux_shell_fatal (FLUX_SHELL_PLUGIN_NAME, \
+                      __FILE__, __LINE__, \
+                      0, code, __VA_ARGS__)
 
 #define shell_die_errno(code,...) \
-    flux_shell_fatal (__FILE__, __LINE__, errno, code, __VA_ARGS__)
+    flux_shell_fatal (FLUX_SHELL_PLUGIN_NAME, \
+                      __FILE__, __LINE__, \
+                      errno, code, __VA_ARGS__)
 
 #define shell_set_verbose(n) \
     flux_shell_log_setlevel(FLUX_SHELL_NOTICE+n, NULL)
@@ -305,11 +364,12 @@ enum {
 
 /*  Log a message at level to all registered loggers at level or above
  */
-void flux_shell_log (int level,
+void flux_shell_log (const char *component,
+                     int level,
                      const char *file,
                      int line,
                      const char *fmt, ...)
-                     __attribute__ ((format (printf, 4, 5)));
+                     __attribute__ ((format (printf, 5, 6)));
 
 /*  Log a message at FLUX_SHELL_ERROR level, additionally appending the
  *   result of strerror (errnum) for convenience.
@@ -317,30 +377,41 @@ void flux_shell_log (int level,
  *  Returns -1 with errno = errnum, so that the function can be used as
  *   return flux_shell_err (...);
  */
-int flux_shell_err (const char *file,
+int flux_shell_err (const char *component,
+                    const char *file,
                     int line,
                     int errnum,
                     const char *fmt, ...)
-                    __attribute__ ((format (printf, 4, 5)));
+                    __attribute__ ((format (printf, 5, 6)));
 
 /*  Log a message at FLUX_SHELL_FATAL level and schedule termination of
  *   the job shell. May generate an exception if tasks are already
  *   running. Exits with exit_code.
  */
-void flux_shell_fatal (const char *file,
-                      int line,
-                      int errnum,
-                      int exit_code,
-                      const char *fmt, ...)
-                      __attribute__ ((format (printf, 5, 6)));
+void flux_shell_fatal (const char *component,
+                       const char *file,
+                       int line,
+                       int errnum,
+                       int exit_code,
+                       const char *fmt, ...)
+                       __attribute__ ((format (printf, 6, 7)));
+
+void flux_shell_raise (const char *type, int severity, const char *fmt, ...);
 
 /*  Set default severity of logging destination 'dest' to level.
  *   If dest == NULL then set the internal log dispatch level --
  *   (i.e. no messages above severity level will be logged to any
  *    log destination)
+ *
+ *  If 'level' is FLUX_SHELL_QUIET, then logging to 'dest' is disabled.
  */
 int flux_shell_log_setlevel (int level, const char *dest);
 
+/*  Expand mustache template.  Caller must free the result.
+ */
+char *flux_shell_mustache_render (flux_shell_t *shell, const char *fmt);
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/shell/signal.c b/src/shell/signal.c
new file mode 100644
index 000000000000..1f50c6a564b3
--- /dev/null
+++ b/src/shell/signal.c
@@ -0,0 +1,275 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Send a signal to a job at a predefined value for timeleft
+ *
+ * Set via attributes.system.shell.options.signal, e.g.:
+ *  { "timeleft": 123, "signal": 10 }
+ *
+ * with defaults:
+ *  timeleft = 60.
+ *  signal = 10
+ *
+ * If shell.options.signal is not set in jobspec, then no warning
+ * signal is sent.
+ */
+#define FLUX_SHELL_PLUGIN_NAME "signal"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <signal.h>
+#include <string.h>
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/shell.h>
+
+#include "src/common/libutil/fsd.h"
+#include "src/common/libutil/sigutil.h"
+#include "internal.h"
+#include "builtins.h"
+
+struct shell_signal {
+    flux_shell_t *shell;
+    flux_jobid_t id;
+    flux_watcher_t *watcher;
+    double timeleft;
+    int signum;
+};
+
+static void shell_signal_destroy (struct shell_signal *sig)
+{
+    if (sig) {
+        int saved_errno = errno;
+        flux_watcher_destroy (sig->watcher);
+        free (sig);
+        errno = saved_errno;
+    }
+}
+
+static void kill_cb (flux_future_t *f, void *arg)
+{
+    if (flux_future_get (f, NULL) < 0)
+        shell_log_error ("flux_job_kill");
+    flux_future_destroy (f);
+}
+
+static void shell_signal_cb (flux_reactor_t *r,
+                             flux_watcher_t *w,
+                             int revents,
+                             void *arg)
+{
+    struct shell_signal *sig = arg;
+    flux_future_t *f;
+
+    shell_log ("job will expire in %.1fs, sending %s to job",
+               sig->timeleft,
+               sigutil_signame (sig->signum));
+
+    if (!(f = flux_job_kill (sig->shell->h, sig->id, sig->signum))
+        || flux_future_then (f, -1., kill_cb, sig) < 0)
+        shell_log_error ("failed to send %s to job",
+                         sigutil_signame (sig->signum));
+}
+
+static int set_timeleft_watcher (struct shell_signal *sig)
+{
+    double expiration;
+    double wakeup;
+    double remaining;
+
+    if (flux_shell_info_unpack (sig->shell,
+                                "{s:{s:{s:f}}}",
+                                "R",
+                                "execution",
+                                "expiration", &expiration) < 0) {
+        shell_log_errno ("unable to get job expiration");
+        return -1;
+    }
+
+    /* Destroy any current watcher in case this function is called due
+     * to an update in a job's expiration.
+     */
+    flux_watcher_destroy (sig->watcher);
+    sig->watcher = NULL;
+
+    if (expiration == 0.) {
+        shell_log ("job has no expiration, %s will not be sent",
+                   sigutil_signame (sig->signum));
+        return 0;
+    }
+
+    /* Note: the exec system adjusts expiration by the difference between
+     * the starttime (the time at which R was created), and when the 'start'
+     * event is posted, (after all job shells have started). This shell
+     * plugin does *not* do a similar adjustment, so the signal may be
+     * delivered early. However, this difference should be at most a few
+     * seconds, which should not be critical in real-world scenarios.
+     */
+
+    /* wakeup is the absolute time to signal job, remaining time is
+     * for informational purposes only. Note that if wakeup < current time,
+     * the callback should fire immediately as expected.
+     */
+    wakeup = expiration - sig->timeleft;
+    remaining = wakeup - flux_reactor_time ();
+    shell_debug ("Will send %s to job in %.2fs",
+                 sigutil_signame (sig->signum),
+                 remaining > 0 ? remaining : 0.);
+
+    if (!(sig->watcher = flux_periodic_watcher_create (sig->shell->r,
+                                                       wakeup,
+                                                       0.,
+                                                       NULL,
+                                                       shell_signal_cb,
+                                                       sig))) {
+        shell_log_errno ("flux_periodic_watcher_create");
+        return -1;
+    }
+    flux_watcher_start (sig->watcher);
+    return 0;
+}
+
+static int parse_timeleft_value (json_t *val, double *dp)
+{
+    if (json_is_string (val)) {
+        if (fsd_parse_duration (json_string_value (val), dp) < 0)
+            return -1;
+    }
+    else if (json_is_number (val)) {
+        if ((*dp = json_number_value (val)) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+struct shell_signal *shell_signal_create (flux_shell_t *shell)
+{
+    int rc;
+    json_t *val = NULL;
+    json_t *opt = NULL;
+    json_error_t error;
+
+    struct shell_signal *sig = calloc (1, sizeof (*sig));
+    if (!sig)
+        return NULL;
+    sig->shell = shell;
+
+    /* Default is to send SIGUSR1 with 60s time left
+     */
+    sig->signum = SIGUSR1;
+    sig->timeleft = 60.;
+
+    if (flux_shell_info_unpack (sig->shell,
+                                "{s:I}",
+                                "jobid", &sig->id) < 0) {
+        shell_log_errno ("failed to get jobid");
+        goto error;
+    }
+
+    if ((rc = flux_shell_getopt_unpack (shell, "signal", "o", &opt)) < 0) {
+        shell_log_errno ("unable to get shell `signal' option");
+        goto error;
+    }
+
+    /*  If no signal option provided or signal=0, then return and do nothing:
+     */
+    if (rc == 0 || (json_is_integer (opt) && json_integer_value (opt) == 0)) {
+        sig->timeleft = -1.;
+        return sig;
+    }
+
+    /*  O/w, if `opt` is not a json integer, try to unpack optional args
+     *  Note: this will fail with an error if `opt` is not a number or
+     *   object, or for invalid types in the object (as expected):
+     */
+    if (!json_is_integer (opt)
+        && json_unpack_ex (opt, &error, 0,
+                           "{s?i s?o}",
+                           "signum", &sig->signum,
+                           "timeleft", &val) < 0) {
+        shell_log_error ("error in shell `signal' option: %s", error.text);
+        goto error;
+    }
+
+    if (val && parse_timeleft_value (val, &sig->timeleft) < 0) {
+        shell_log_error ("signal.timeleft=%s is invalid",
+                         json_string_value (val));
+        goto error;
+    }
+
+    return sig;
+error:
+    shell_signal_destroy (sig);
+    return NULL;
+}
+
+static int resource_update_cb (flux_plugin_t *p,
+                               const char *topic,
+                               flux_plugin_arg_t *args,
+                               void *arg)
+{
+    struct shell_signal *sig = arg;
+    return set_timeleft_watcher (sig);
+}
+
+static int signal_init (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    struct shell_signal *sig;
+    int rank;
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+
+    if (!shell
+        || flux_shell_info_unpack (shell, "{s:i}", "rank", &rank) < 0) {
+        shell_log_error ("flux_shell_info_unpack");
+        return -1;
+    }
+
+    /* signal plugin only operates on rank 0
+     */
+    if (rank != 0)
+        return 0;
+
+    if (!(sig = shell_signal_create (shell)))
+        return -1;
+    if (flux_plugin_aux_set (p,
+                             "signal",
+                             sig,
+                             (flux_free_f) shell_signal_destroy) < 0) {
+        shell_signal_destroy (sig);
+        return -1;
+    }
+    if (sig->timeleft <= 0)
+        return 0;
+
+    if (set_timeleft_watcher (sig) < 0)
+        return -1;
+
+    if (flux_plugin_add_handler (p,
+                                 "shell.resource-update",
+                                 resource_update_cb,
+                                 sig) < 0)
+        shell_log_errno ("unable to subscribe to shell resource updates");
+
+    return 0;
+}
+
+struct shell_builtin builtin_signal = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = signal_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/signals.c b/src/shell/signals.c
index 8b3784eb623f..45ab971e48de 100644
--- a/src/shell/signals.c
+++ b/src/shell/signals.c
@@ -15,8 +15,30 @@
  *
  * SIGINT  - forward to all local tasks
  * SIGTERM - forward
+ * SIGALRM - forward
+ * SIGPIPE - ignore
  *
+ * Notes:
+ *
+ * By setting up the signal watchers during "shell.init", there is the
+ * potential for inconsistent exit codes if a signal is received before all
+ * tasks have started.  For example, this could be seen with something
+ * like:
+ *
+ * jobid=`flux submit -n1000 foo.sh`
+ * flux job raise --type=foo --severity=0 $jobid
+ *
+ * i.e. raise sends SIGTERM to job/shell immediately after starting,
+ * but due to the large task count of 1000, the signal is received
+ * before tasks are all setup.  Some tasks could receive SIGTERM while
+ * some (to be created ones) do not.
+ *
+ * Note that the shell should always return an error, but the error
+ * may not be consistent.  This situation is extremely rare and only
+ * seen is testing situations such as the above.  So we elect to not
+ * fix this race.
  */
+#define FLUX_SHELL_PLUGIN_NAME "signals"
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -56,14 +78,19 @@ static int signals_init (flux_plugin_t *p,
     flux_shell_t *shell = flux_plugin_get_shell (p);
     if (!shell)
         return -1;
-    /* forward local SIGINT, SIGTERM to tasks */
-    if (trap_signal (shell, SIGINT) < 0 || trap_signal (shell, SIGTERM) < 0)
+    /* forward local SIGINT, SIGTERM, SIGALRM to tasks */
+    if (trap_signal (shell, SIGINT) < 0
+        || trap_signal (shell, SIGTERM) < 0
+        || trap_signal (shell, SIGALRM) < 0)
         shell_log_errno ("failed to set up signal watchers");
+
+    /* ignore SIGPIPE */
+    signal (SIGPIPE, SIG_IGN);
     return 0;
 }
 
 struct shell_builtin builtin_signals = {
-    .name = "sighandler",
+    .name = FLUX_SHELL_PLUGIN_NAME,
     .init = signals_init,
 };
 
diff --git a/src/shell/stage-in.c b/src/shell/stage-in.c
new file mode 100644
index 000000000000..67c7476cce78
--- /dev/null
+++ b/src/shell/stage-in.c
@@ -0,0 +1,284 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* stage-in.c - copy previously archived files for job */
+
+#define FLUX_SHELL_PLUGIN_NAME "stage-in"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <fnmatch.h>
+#include <unistd.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <libgen.h>
+#include <jansson.h>
+#ifdef HAVE_ARGZ_ADD
+#include <argz.h>
+#else
+#include "src/common/libmissing/argz.h"
+#endif
+#include <archive.h>
+#include <archive_entry.h>
+#include <flux/core.h>
+
+#include "ccan/base64/base64.h"
+#include "ccan/str/str.h"
+#include "src/common/libcontent/content.h"
+#include "src/common/libfilemap/filemap.h"
+#include "src/common/libfilemap/fileref.h"
+#include "src/common/libutil/errno_safe.h"
+#include "src/common/libutil/monotime.h"
+
+#include "builtins.h"
+#include "internal.h"
+#include "info.h"
+
+struct stage_in {
+    json_t *names;
+    const char *pattern;
+    const char *destdir;
+    flux_t *h;
+    int count;
+    size_t total_size;
+};
+
+json_t *parse_names (const char *s, const char *default_value)
+{
+    char *argz = NULL;
+    size_t argz_len;
+    json_t *a;
+    json_t *o;
+    const char *entry;
+
+    if (!(a = json_array ()))
+        return NULL;
+    if (s) {
+        if (argz_create_sep (s, ',', &argz, &argz_len) != 0)
+            goto error;
+        entry = NULL;
+        while ((entry = argz_next (argz, argz_len, entry))) {
+            if (!(o = json_string (entry))
+                || json_array_append_new (a, o) < 0) {
+                json_decref (o);
+                goto error;
+            }
+        }
+    }
+    if (json_array_size (a) == 0 && default_value) {
+        if (!(o = json_string (default_value))
+            || json_array_append_new (a, o) < 0) {
+            json_decref (o);
+            goto error;
+        }
+    }
+    free (argz);
+    return a;
+error:
+    free (argz);
+    json_decref (a);
+    return NULL;
+}
+
+static void trace_cb (void *arg,
+                      json_t *fileref,
+                      const char *path,
+                      int mode,
+                      int64_t size,
+                      int64_t mtime,
+                      int64_t ctime,
+                      const char *encoding)
+{
+    struct stage_in *ctx = arg;
+    char buf[1024];
+    ctx->count++;
+    if (size != -1)
+        ctx->total_size += size;
+    fileref_pretty_print (fileref, NULL, true, buf, sizeof (buf));
+    shell_trace ("%s", buf);
+}
+
+static int extract (struct stage_in *ctx)
+{
+    size_t i;
+    json_t *nameobj;
+
+    json_array_foreach (ctx->names, i, nameobj) {
+        char *key = NULL;
+        flux_future_t *f = NULL;
+        json_t *archive;
+        flux_error_t error;
+
+        if (asprintf (&key, "archive.%s", json_string_value (nameobj)) < 0
+            || !(f = flux_kvs_lookup (ctx->h, "primary", 0, key))
+            || flux_kvs_lookup_get_unpack (f, "o", &archive) < 0) {
+            shell_log_error ("could not lookup %s in primary KVS namespace: %s",
+                             key,
+                             future_strerror (f, errno));
+            flux_future_destroy (f);
+            return -1;
+        }
+        if (ctx->pattern) {
+            size_t index = 0;
+            while (index < json_array_size (archive)) {
+                json_t *entry;
+                const char *path;
+
+                if (!(entry = json_array_get (archive, index))
+                    || json_unpack (entry, "{s:s}", "path", &path) < 0
+                    || fnmatch (ctx->pattern, path, 0) != 0) {
+                    json_array_remove (archive, index);
+                    continue;
+                }
+                index++;
+            }
+        }
+        if (filemap_extract (ctx->h,
+                             archive,
+                             0,
+                             &error,
+                             trace_cb,
+                             ctx) < 0) {
+            shell_log_error ("%s", error.text);
+            flux_future_destroy (f);
+            return -1;
+        }
+        flux_future_destroy (f);
+    }
+    return 0;
+}
+
+static int extract_files (struct stage_in *ctx)
+{
+    char *orig_dir;
+    struct archive *archive = NULL;
+    struct timespec t;
+    int rc = -1;
+
+    if (!(orig_dir = getcwd (NULL, 0))) {
+        shell_log_error ("getcwd: %s", strerror (errno));
+        return -1;
+    }
+    if (chdir (ctx->destdir) < 0) {
+        shell_log_error ("chdir %s: %s", ctx->destdir, strerror (errno));
+        goto done;
+    }
+    shell_debug ("=> %s", ctx->destdir);
+    monotime (&t);
+    if (extract (ctx) == 0) {
+        double elapsed = monotime_since (t) / 1000;
+        shell_debug ("%d files %.1fMB/s",
+                     ctx->count,
+                     1E-6 * ctx->total_size / elapsed);
+        rc = 0;
+    }
+done:
+    if (chdir (orig_dir) < 0) {
+        shell_die (1,
+                   "could not chdir back to original directory %s: %s",
+                   orig_dir,
+                   strerror (errno));
+    }
+    if (archive)
+        archive_write_free (archive);
+    free (orig_dir);
+    return rc;
+}
+
+static int stage_in (flux_shell_t *shell, json_t *config)
+{
+    struct stage_in ctx;
+    const char *names = NULL;
+    const char *tags = NULL;
+    const char *destination = NULL;
+    bool leader_only = false;
+
+    memset (&ctx, 0, sizeof (ctx));
+    ctx.h = shell->h;
+
+    if (json_is_object (config)) {
+        if (json_unpack (config,
+                         "{s?s s?s s?s s?s !}",
+                         "names", &names,
+                         "tags", &tags,
+                         "pattern", &ctx.pattern,
+                         "destination", &destination)) {
+            shell_log_error ("Error parsing stage_in shell option");
+            goto error;
+        }
+    }
+    if (tags) {
+        if (shell->info->shell_rank == 0) {
+            shell_warn ("Setting stage-in.names to the value of deprecated"
+                        " option stage-in.tags.");
+        }
+        names = tags;
+    }
+    if (!(ctx.names = parse_names (names, "main"))) {
+        shell_log_error ("Error parsing stage_in.names shell option");
+        goto error;
+    }
+    if (destination) {
+        if (strstarts (destination, "local:"))
+            ctx.destdir = destination + 6;
+        else if (strstarts (destination, "global:")) {
+            ctx.destdir = destination + 7;
+            leader_only = true;
+        }
+        else if (strchr (destination, ':') == NULL)
+            ctx.destdir = destination;
+        else {
+            shell_log_error ("destination prefix must be local: or global:");
+            goto error;
+        }
+    }
+    if (!ctx.destdir) {
+        ctx.destdir = flux_shell_getenv (shell, "FLUX_JOB_TMPDIR");
+        if (!ctx.destdir) {
+            shell_log_error ("FLUX_JOB_TMPDIR is not set");
+            goto error;
+        }
+    }
+    if (shell->info->shell_rank == 0 || leader_only == false) {
+        if (extract_files (&ctx) < 0)
+            goto error;
+    }
+
+    json_decref (ctx.names);
+    return 0;
+error:
+    json_decref (ctx.names);
+    return -1;
+}
+
+static int stage_in_init (flux_plugin_t *p,
+                          const char *topic,
+                          flux_plugin_arg_t *args,
+                          void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    json_t *config = NULL;
+
+    if (flux_shell_getopt_unpack (shell, "stage-in", "o", &config) < 0)
+        return -1;
+    if (!config)
+        return 0;
+    return stage_in (shell, config);
+}
+
+struct shell_builtin builtin_stage_in = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = stage_in_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/svc.c b/src/shell/svc.c
index 8fa31d39d126..dc74ed8d9de2 100644
--- a/src/shell/svc.c
+++ b/src/shell/svc.c
@@ -25,14 +25,18 @@
  * - Services should not be used until after the shells exit the init barrier,
  *   to ensure service registration has completed.
  */
+#define FLUX_SHELL_PLUGIN_NAME NULL
 
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
 #include <stdio.h>
 #include <string.h>
+#include <unistd.h>
 #include <flux/core.h>
 
+#include "src/common/libjob/idf58.h"
+
 #include "internal.h"
 #include "task.h"
 #include "svc.h"
@@ -43,7 +47,7 @@ struct shell_svc {
     flux_shell_t *shell;
     char name[TOPIC_STRING_SIZE];
     int registered;
-    uid_t uid;      // effective uid of shell
+    uid_t uid;      // uid of shell
     int *rank_table;// map shell rank to broker rank
 };
 
@@ -57,16 +61,25 @@ static int lookup_rank (struct shell_svc *svc, int shell_rank, int *rank)
     return 0;
 }
 
+/* Avoid locale-specific encoding for the shell service name.
+ * See flux-framework/flux-core#5257.
+ */
 static int build_topic (struct shell_svc *svc,
                         const char *method,
                         char *buf,
                         int len)
 {
+    char idbuf[21];
+    if (flux_job_id_encode (svc->shell->info->jobid,
+                            "f58plain",
+                            idbuf,
+                            sizeof (idbuf)) < 0)
+        return -1;
     if (snprintf (buf,
                   len,
-                  "%d-shell-%ju%s%s",
-                  svc->uid,
-                  (uintmax_t)svc->shell->info->jobid,
+                  "%ju-shell-%s%s%s",
+                  (uintmax_t)svc->uid,
+                  idbuf,
                   method ? "." : "",
                   method ? method : "") >= len) {
         errno = EINVAL;
@@ -93,33 +106,9 @@ flux_future_t *shell_svc_vpack (struct shell_svc *svc,
     return flux_rpc_vpack (svc->shell->h, topic, rank, flags, fmt, ap);
 }
 
-flux_future_t *shell_svc_pack (struct shell_svc *svc,
-                               const char *method,
-                               int shell_rank,
-                               int flags,
-                               const char *fmt, ...)
-{
-    flux_future_t *f;
-    va_list ap;
-    va_start (ap, fmt);
-    f = shell_svc_vpack (svc, method, shell_rank, flags, fmt, ap);
-    va_end (ap);
-    return f;
-}
-
 int shell_svc_allowed (struct shell_svc *svc, const flux_msg_t *msg)
 {
-    uint32_t rolemask;
-    uint32_t userid;
-
-    if (flux_msg_get_rolemask (msg, &rolemask) < 0
-            || flux_msg_get_userid (msg, &userid) < 0)
-        return -1;
-    if (!(rolemask & FLUX_ROLE_OWNER) && userid != svc->uid) {
-        errno = EPERM;
-        return -1;
-    }
-    return 0;
+    return flux_msg_authorize (msg, svc->uid);
 }
 
 int shell_svc_register (struct shell_svc *svc,
@@ -168,12 +157,13 @@ struct shell_svc *shell_svc_create (flux_shell_t *shell)
     struct shell_svc *svc;
     struct rcalc_rankinfo ri;
     int shell_size = shell->info->shell_size;
+    flux_future_t *f;
     int i;
 
     if (!(svc = calloc (1, sizeof (*svc))))
         return NULL;
     svc->shell = shell;
-    svc->uid = geteuid ();
+    svc->uid = getuid ();
     if (!(svc->rank_table = calloc (shell_size, sizeof (*svc->rank_table))))
         goto error;
     for (i = 0; i < shell_size; i++) {
@@ -181,32 +171,34 @@ struct shell_svc *shell_svc_create (flux_shell_t *shell)
             goto error;
         svc->rank_table[i] = ri.rank;
     }
-    if (!shell->standalone) {
-        flux_future_t *f;
-        if (build_topic (svc, NULL, svc->name, sizeof (svc->name)) < 0)
-            goto error;
-        if (!(f = flux_service_register (shell->h, svc->name)))
-            goto error;
-        if (flux_future_get (f, NULL) < 0) {
-            flux_future_destroy (f);
-            goto error;
-        }
+    if (build_topic (svc, NULL, svc->name, sizeof (svc->name)) < 0)
+        goto error;
+    if (!(f = flux_service_register (shell->h, svc->name)))
+        goto error;
+    if (flux_future_get (f, NULL) < 0) {
         flux_future_destroy (f);
-        if (flux_shell_add_event_context (shell,
-                                          "shell.init",
-                                          0,
-                                          "{s:s}",
-                                          "service",
-                                          svc->name) < 0)
-            goto error;
-        svc->registered = 1;
+        goto error;
     }
+    flux_future_destroy (f);
+    if (flux_shell_add_event_context (shell,
+                                      "shell.init",
+                                      0,
+                                      "{s:s}",
+                                      "service",
+                                      svc->name) < 0)
+        goto error;
+    svc->registered = 1;
     return svc;
 error:
     shell_svc_destroy (svc);
     return NULL;
 }
 
+const char *shell_svc_name (struct shell_svc *svc)
+{
+    return svc->name;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/src/shell/svc.h b/src/shell/svc.h
index c303afe8786a..8ffdf4bf8940 100644
--- a/src/shell/svc.h
+++ b/src/shell/svc.h
@@ -26,15 +26,6 @@ struct shell_svc *shell_svc_create (flux_shell_t *shell);
 
 /* Send an RPC to a shell 'method' by shell rank.
  */
-flux_future_t *shell_svc_pack (struct shell_svc *svc,
-                               const char *method,
-                               int shell_rank,
-                               int flags,
-                               const char *fmt, ...);
-
-
-/* Same as above, but called with a va_list instead of varargs
- */
 flux_future_t *shell_svc_vpack (struct shell_svc *svc,
                                 const char *method,
                                 int shell_rank,
@@ -55,6 +46,10 @@ int shell_svc_register (struct shell_svc *svc,
  */
 int shell_svc_allowed (struct shell_svc *svc, const flux_msg_t *msg);
 
+/* Return the service name under which shell svc methods are registered:
+ */
+const char *shell_svc_name (struct shell_svc *svc);
+
 #endif /* !SHELL_SVC_H */
 
 /*
diff --git a/src/shell/task.c b/src/shell/task.c
index 243467e985fb..6624972f749f 100644
--- a/src/shell/task.c
+++ b/src/shell/task.c
@@ -21,7 +21,8 @@
  *    . FLUX_JOB_SIZE
  *    . FLUX_JOB_NNODES
  *    . FLUX_JOB_ID
- *    . FLUX_URI (if not running standalone)
+ *    . FLUX_URI
+ *    . correct HOSTNAME if set in job environment
  *
  * Current working directory
  *    Ignore - shell should already be in it.
@@ -32,6 +33,7 @@
  * Each running task adds reactor handlers that are removed on
  * completion.
  */
+#define FLUX_SHELL_PLUGIN_NAME NULL
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -41,6 +43,10 @@
 #include <flux/core.h>
 #include <flux/shell.h>
 
+#include "src/common/libjob/idf58.h"
+#include "src/common/libczmqcontainers/czmq_containers.h"
+
+#include "internal.h"
 #include "task.h"
 #include "info.h"
 
@@ -86,9 +92,11 @@ struct shell_task *shell_task_new (void)
     return NULL;
 }
 
-struct shell_task *shell_task_create (struct shell_info *info,
-                                      int index)
+struct shell_task *shell_task_create (flux_shell_t *shell,
+                                      int index,
+                                      int taskid)
 {
+    struct shell_info *info = shell->info;
     struct shell_task *task;
     const char *key;
     json_t *entry;
@@ -98,26 +106,21 @@ struct shell_task *shell_task_create (struct shell_info *info,
         return NULL;
 
     task->index = index;
-    task->rank = info->rankinfo.global_basis + index;
-    task->size = info->jobspec->task_count;
-    if (!(task->cmd = flux_cmd_create (0,
-                                       NULL,
-                                       info->jobspec->environment ? NULL
-                                                                  : environ)))
+    task->rank = taskid;
+    task->size = info->total_ntasks;
+    if (!(task->cmd = flux_cmd_create (0, NULL, NULL)))
         goto error;
     json_array_foreach (info->jobspec->command, i, entry) {
         if (flux_cmd_argv_append (task->cmd, json_string_value (entry)) < 0)
             goto error;
     }
-    if (info->jobspec->environment) {
-        json_object_foreach (info->jobspec->environment, key, entry) {
-            if (flux_cmd_setenvf (task->cmd,
-                                  1,
-                                  key,
-                                  "%s",
-                                  json_string_value (entry)) < 0)
-                goto error;
-        }
+    json_object_foreach (info->jobspec->environment, key, entry) {
+        if (flux_cmd_setenvf (task->cmd,
+                              1,
+                              key,
+                              "%s",
+                              json_string_value (entry)) < 0)
+            goto error;
     }
     if (flux_cmd_setenvf (task->cmd, 1, "FLUX_TASK_LOCAL_ID", "%d", index) < 0)
         goto error;
@@ -125,24 +128,53 @@ struct shell_task *shell_task_create (struct shell_info *info,
         goto error;
     if (flux_cmd_setenvf (task->cmd, 1, "FLUX_JOB_SIZE", "%d", task->size) < 0)
         goto error;
-    if (flux_cmd_setenvf (task->cmd, 1, "FLUX_JOB_NNODES", "%d",
+    if (flux_cmd_setenvf (task->cmd,
+                          1,
+                          "FLUX_JOB_NNODES",
+                          "%d",
                           info->shell_size) < 0)
         goto error;
-    if (flux_cmd_setenvf (task->cmd, 1, "FLUX_JOB_ID", "%ju",
-                          (uintmax_t)info->jobid) < 0)
+
+    if (flux_cmd_setenvf (task->cmd,
+                          1,
+                          "FLUX_JOB_ID",
+                          "%s",
+                          idf58 (info->jobid)) < 0)
         goto error;
+
+    /* Always unset FLUX_PROXY_REMOTE since this never makes sense
+     * in the environment of a job task.
+     */
+    flux_cmd_unsetenv (task->cmd, "FLUX_PROXY_REMOTE");
     flux_cmd_unsetenv (task->cmd, "FLUX_URI");
     if (getenv ("FLUX_URI")) {
-        if (flux_cmd_setenvf (task->cmd, 1, "FLUX_URI", "%s",
+        if (flux_cmd_setenvf (task->cmd,
+                              1,
+                              "FLUX_URI",
+                              "%s",
                               getenv ("FLUX_URI")) < 0)
             goto error;
     }
     flux_cmd_unsetenv (task->cmd, "FLUX_KVS_NAMESPACE");
     if (getenv ("FLUX_KVS_NAMESPACE")) {
-        if (flux_cmd_setenvf (task->cmd, 1, "FLUX_KVS_NAMESPACE", "%s",
+        if (flux_cmd_setenvf (task->cmd,
+                              1,
+                              "FLUX_KVS_NAMESPACE",
+                              "%s",
                               getenv ("FLUX_KVS_NAMESPACE")) < 0)
             goto error;
     }
+
+    /* If HOSTNAME is set in job environment it is almost certain to be
+     * incorrect. Overwrite with the correct hostname.
+     */
+    if (flux_cmd_getenv (task->cmd, "HOSTNAME")
+        && flux_cmd_setenvf (task->cmd,
+                             1,
+                             "HOSTNAME",
+                             "%s",
+                             shell->hostname) < 0)
+        goto error;
     return task;
 error:
     shell_task_destroy (task);
@@ -185,18 +217,28 @@ static void subproc_preexec_hook (flux_subprocess_t *p, void *arg)
         (*task->pre_exec_cb) (task, task->pre_exec_arg);
 }
 
-int shell_task_start (struct shell_task *task,
-                      flux_reactor_t *r,
+int shell_task_start (struct flux_shell *shell,
+                      struct shell_task *task,
                       shell_task_completion_f cb,
                       void *arg)
 {
     int flags = 0;
+    flux_reactor_t *r = shell->r;
     flux_subprocess_hooks_t hooks = {
         .pre_exec = subproc_preexec_hook,
         .pre_exec_arg = task,
     };
 
-    task->proc = flux_local_exec (r, flags, task->cmd, &subproc_ops, &hooks);
+    if (shell->nosetpgrp)
+        flags |= FLUX_SUBPROCESS_FLAGS_NO_SETPGRP;
+
+    task->proc = flux_local_exec_ex (r,
+                                     flags,
+                                     task->cmd,
+                                     &subproc_ops,
+                                     &hooks,
+                                     NULL,
+                                     NULL);
     if (!task->proc)
         return -1;
     if (flux_subprocess_aux_set (task->proc, "flux::task", task, NULL) < 0) {
diff --git a/src/shell/task.h b/src/shell/task.h
index 4e80367e89fa..b4153b53dcf9 100644
--- a/src/shell/task.h
+++ b/src/shell/task.h
@@ -12,9 +12,10 @@
 #define SHELL_TASK_H
 
 #include <flux/core.h>
-#include <czmq.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/aux.h"
+
 #include "info.h"
 
 
@@ -53,10 +54,12 @@ struct shell_task {
 
 void shell_task_destroy (struct shell_task *task);
 
-struct shell_task *shell_task_create (struct shell_info *info, int index);
+struct shell_task *shell_task_create (flux_shell_t *shell,
+                                      int index,
+                                      int taskid);
 
-int shell_task_start (struct shell_task *task,
-                      flux_reactor_t *r,
+int shell_task_start (struct flux_shell *shell,
+                      struct shell_task *task,
                       shell_task_completion_f cb,
                       void *arg);
 
diff --git a/src/shell/taskmap/cyclic.c b/src/shell/taskmap/cyclic.c
new file mode 100644
index 000000000000..c7f844d37668
--- /dev/null
+++ b/src/shell/taskmap/cyclic.c
@@ -0,0 +1,111 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* shell taskmap.cyclic plugin
+ */
+#define FLUX_SHELL_PLUGIN_NAME "taskmap.cyclic"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <flux/taskmap.h>
+
+#include "builtins.h"
+
+char *taskmap_cyclic (const struct taskmap *orig, int stride)
+{
+    struct taskmap *map = NULL;
+    char *result = NULL;
+    int ntasks;
+    int nnodes;
+
+    if (!(map = taskmap_create ()))
+        goto error;
+
+    ntasks = taskmap_total_ntasks (orig);
+    nnodes = taskmap_nnodes (orig);
+    while (ntasks > 0) {
+        for (int i = 0; i < nnodes; i++) {
+            int ppn = stride;
+            int avail = taskmap_ntasks (orig, i) - taskmap_ntasks (map, i);
+            if (avail == 0)
+                continue;
+            if (ppn > avail)
+                ppn = avail;
+            if (taskmap_append (map, i, 1, ppn) < 0)
+                goto error;
+            ntasks -= ppn;
+        }
+    }
+    result = taskmap_encode (map, TASKMAP_ENCODE_WRAPPED);
+error:
+    taskmap_destroy (map);
+    return result;
+}
+
+static int map_cyclic (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *data)
+{
+    flux_shell_t *shell;
+    char *cyclic;
+    const char *value = NULL;
+    int stride = 1;
+    int rc = -1;
+
+    if (!(shell = flux_plugin_get_shell (p)))
+        return -1;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s?s}",
+                                "value", &value) < 0) {
+        shell_log_error ("unpack: %s", flux_plugin_arg_strerror (args));
+        return -1;
+    }
+    if (value && *value != '\0') {
+        char *endptr;
+        errno = 0;
+        stride = strtol (value, &endptr, 10);
+        if (errno != 0 || *endptr != '\0' || stride <= 0) {
+            shell_log_error ("invalid cyclic stride: %s", value);
+            return -1;
+        }
+    }
+    if (!(cyclic = taskmap_cyclic (flux_shell_get_taskmap (shell), stride))) {
+        shell_log_error ("failed to map tasks with cyclic:%d", stride);
+        goto out;
+    }
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:s}",
+                              "taskmap", cyclic) < 0)
+        goto out;
+    rc = 0;
+out:
+    free (cyclic);
+    return rc;
+}
+
+static int plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_add_handler (p, "taskmap.cyclic", map_cyclic, NULL);
+}
+
+struct shell_builtin builtin_cyclic = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .plugin_init = plugin_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/taskmap/hostfile.c b/src/shell/taskmap/hostfile.c
new file mode 100644
index 000000000000..cf3a9d408f67
--- /dev/null
+++ b/src/shell/taskmap/hostfile.c
@@ -0,0 +1,174 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* shell taskmap.hostfile plugin
+ *
+ * Read a list of hosts from a file, and assign tasks to hosts in order
+ * they are listed.
+ */
+#define FLUX_SHELL_PLUGIN_NAME "taskmap.hostfile"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/taskmap.h>
+#include <flux/hostlist.h>
+
+#include "src/common/libutil/errprintf.h"
+
+#include "builtins.h"
+
+
+/*  Create a taskmap that represents 'ntasks' tasks mapped across a set
+ *  of hosts in 'nodelist', ordered by hostlist 'hl'.
+ */
+char *taskmap_hostlist (int ntasks,
+                        struct hostlist *nodelist,
+                        struct hostlist *hl,
+                        flux_error_t *errp)
+{
+    struct taskmap *map = NULL;
+    char *result = NULL;
+    const char *host = NULL;
+
+    if (!(map = taskmap_create ()))
+        goto error;
+
+    /* Loop through hostlist hl until all tasks have been assigned to hosts
+     */
+    while (ntasks > 0) {
+        int rank;
+        if (host == NULL)
+            host = hostlist_first (hl);
+        if ((rank = hostlist_find (nodelist, host)) < 0) {
+            errprintf (errp, "host %s not found in job nodelist", host);
+            goto error;
+        }
+        if (taskmap_append (map, rank, 1, 1) < 0) {
+            errprintf (errp,
+                       "failed to append task to taskmap: %s",
+                       strerror (errno));
+            goto error;
+        }
+        host = hostlist_next (hl);
+        ntasks--;
+    }
+    result = taskmap_encode (map, TASKMAP_ENCODE_WRAPPED);
+error:
+    taskmap_destroy (map);
+    return result;
+}
+
+static struct hostlist *hostlist_from_file (const char *path)
+{
+    ssize_t n;
+    size_t size;
+    struct hostlist *hl = NULL;
+    FILE *fp = NULL;
+    char *line = NULL;
+
+    if (!(fp = fopen (path, "r"))) {
+        shell_log_errno ("failed to open hostfile: %s", path);
+        goto error;
+    }
+    if (!(hl = hostlist_create ())) {
+        shell_log_errno ("failed to create hostlist");
+        goto error;
+    }
+    while ((n = getline (&line, &size, fp)) != -1) {
+        int len = strlen (line);
+        if (line[len-1] == '\n')
+            line[len-1] = '\0';
+        if (strlen (line) > 0 && hostlist_append (hl, line) < 0) {
+            shell_log_errno ("hostlist_append: %s", line);
+        }
+    }
+error:
+    if (fp)
+        fclose (fp);
+    free (line);
+    return hl;
+}
+
+static int map_hostfile (flux_plugin_t *p,
+                         const char *topic,
+                         flux_plugin_arg_t *args,
+                         void *data)
+{
+    flux_shell_t *shell;
+    int rc = -1;
+    const char *value = NULL;
+    struct hostlist *hl = NULL;
+    struct hostlist *nodelist = NULL;
+    char *map = NULL;
+    int ntasks;
+    flux_error_t error;
+
+    if (!(shell = flux_plugin_get_shell (p)))
+        return -1;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s?s}",
+                                "value", &value) < 0) {
+        shell_log_error ("unpack: %s", flux_plugin_arg_strerror (args));
+        return -1;
+    }
+
+    /*  Note: the result of flux_shell_get_hostlist(3) is copied because
+     *  functions such as hostlist_find(3) modify the hostlist cursor, so
+     *  a `const struct hostlist` can't be used here.
+     */
+    if (!(hl = hostlist_from_file (value))
+        || !(nodelist = hostlist_copy (flux_shell_get_hostlist (shell)))) {
+        shell_log_error ("failed to get hostlists from file and R");
+        goto out;
+    }
+    if ((ntasks = taskmap_total_ntasks (flux_shell_get_taskmap (shell))) < 0)
+        shell_log_error ("failed to get ntasks from current shell taskmap");
+
+    if (!(map = taskmap_hostlist (ntasks, nodelist, hl, &error))) {
+        shell_log_error ("failed to map tasks with hostfile:%s: %s",
+                         value,
+                         error.text);
+        goto out;
+    }
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:s}",
+                              "taskmap", map) < 0) {
+        shell_log_error ("failed to set new taskmap in plugin output args");
+        goto out;
+    }
+    rc = 0;
+out:
+    free (map);
+    hostlist_destroy (hl);
+    hostlist_destroy (nodelist);
+    return rc;
+}
+
+static int plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_add_handler (p, "taskmap.hostfile", map_hostfile, NULL);
+}
+
+struct shell_builtin builtin_hostfile = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .plugin_init = plugin_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/shell/test/jobspec.c b/src/shell/test/jobspec.c
index 429c5fc347bd..c976b1bc1196 100644
--- a/src/shell/test/jobspec.c
+++ b/src/shell/test/jobspec.c
@@ -22,13 +22,70 @@ struct input {
     const char *s;
 };
 
+struct output {
+    int task_count;
+    int slot_count;
+    int cores_per_slot;
+    int slots_per_node;
+};
+
 struct input good_input[] = {
     {
-        "flux jobspec srun hostname",
+        "slot->core",
         "{\"tasks\": [{\"slot\": \"task\", \"count\": {\"per_slot\": 1}, \"command\": [\"hostname\"], \"attributes\": {}}], \"attributes\": {\"system\": {\"cwd\": \"/home/garlick/proj/flux-core/src/cmd\"}}, \"version\": 1, \"resources\": [{\"count\": 1, \"with\": [{\"count\": 1, \"type\": \"core\"}], \"type\": \"slot\", \"label\": \"task\"}]}",
     },
+    {
+        "node->socket->slot->core",
+        "{\"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"socket\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1}], \"label\": \"task\"}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "node[2]->socket[3]->slot[5]->core[3]",
+        "{\"resources\": [{\"type\": \"node\", \"count\": 2, \"with\": [{\"type\": \"socket\", \"count\": 3, \"with\": [{\"type\": \"slot\", \"count\": 5, \"with\": [{\"type\": \"core\", \"count\": 3}], \"label\": \"task\"}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "slot[5]->socket[2]->core[3]",
+        "{\"resources\": [{\"type\": \"slot\", \"count\": 5, \"label\": \"task\", \"with\": [{\"type\": \"socket\", \"count\": 2, \"with\": [{\"type\": \"core\", \"count\": 3}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "node->socket->slot->(core[2],gpu)",
+        "{\"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"socket\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"task\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 2}, {\"type\": \"gpu\", \"count\": 1}]}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "node->socket->slot->(gpu,core)",
+        "{\"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"socket\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"task\", \"count\": 1, \"with\": [{\"type\": \"gpu\", \"count\": 1}, {\"type\": \"core\", \"count\": 1}]}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "node->socket->slot->(core[2]->PU,gpu)",
+        "{\"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"socket\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"task\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 2, \"with\": [{\"type\": \"PU\", \"count\": 1}]}, {\"type\": \"gpu\", \"count\": 1}]}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "node[2]->(storage,slot[3]->core[5])",
+        "{\"resources\": [{\"type\": \"node\", \"count\": 2, \"with\": [{\"type\": \"storage\", \"count\": 1}, {\"type\": \"slot\", \"label\": \"task\", \"count\": 3, \"with\": [{\"type\": \"core\", \"count\": 5}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "(storage,node->slot->core)",
+        "{\"version\": 1, \"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"default\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1}]}]}, {\"type\": \"storage\", \"count\": 1562, \"exclusive\": true}], \"attributes\": {\"system\": {\"duration\": 57600}}, \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"default\", \"count\": {\"per_slot\": 1}}]}",
+    },
+    {
+        "cluster->(storage,node->slot->core)",
+        "{\"version\": 1, \"resources\": [{\"type\": \"cluster\", \"count\": 1, \"with\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"default\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1}]}]}, {\"type\": \"storage\", \"count\": 1562, \"exclusive\": true}]}], \"attributes\": {\"system\": {\"duration\": 57600}}, \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"default\", \"count\": {\"per_slot\": 1}}]}",
+    },
     { NULL, NULL },
 };
+struct output good_output[] =
+    {
+     {1, 1, 1, -1},
+     {1, 1, 1, 1},
+     {30, 30, 3, 15},
+     {5, 5, 6, -1},
+     {1, 1, 2, 1},
+     {1, 1, 1, 1},
+     {1, 1, 2, 1},
+     {6, 6, 5, 3},
+     {1, 1, 1, 1},
+     {1, 1, 1, 1},
+     {0, 0, 0, 0},
+};
 struct input bad_input[] = {
     {
         "empty object",
@@ -70,6 +127,50 @@ struct input bad_input[] = {
         "missing command",
         "{\"tasks\": [{\"slot\": \"task\", \"count\": {\"per_slot\": 1}, \"attributes\": {}}], \"attributes\": {\"system\": {\"cwd\": \"/home/garlick/proj/flux-core/src/cmd\"}}, \"version\": 1, \"resources\": [{\"count\": 1, \"with\": [{\"count\": 1, \"type\": \"core\"}], \"type\": \"slot\", \"label\": \"task\"}]}",
     },
+    {
+        "slot->node->core",
+        "{\"resources\": [{\"type\": \"slot\", \"label\": \"task\", \"count\": 1, \"with\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "node->core->slot",
+        "{\"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"task\", \"count\": 1}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "node->(storage,slot->PU)",
+        "{\"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"storage\", \"count\": 1}, {\"type\": \"slot\", \"label\": \"task\", \"count\": 1, \"with\": [{\"type\": \"PU\", \"count\": 1}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "node->slot->(PU,gpu)",
+        "{\"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"task\", \"count\": 1, \"with\": [{\"type\": \"PU\", \"count\": 1}, {\"type\": \"gpu\", \"count\": 1}]}]}], \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"task\", \"count\": {\"per_slot\": 1}}], \"attributes\": {\"system\": {\"duration\": 0, \"cwd\": \"/usr/libexec/flux\", \"environment\": {}}}, \"version\": 1}",
+    },
+    {
+        "node->(storage->core,slot->PU)",
+        "{\"version\": 1, \"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"storage\", \"count\": 1562, \"exclusive\": true, \"with\": [{\"type\": \"core\", \"count\": 1}]}, {\"type\": \"slot\", \"label\": \"default\", \"count\": 1, \"with\": [{\"type\": \"PU\", \"count\": 1}]}]}], \"attributes\": {\"system\": {\"duration\": 57600}}, \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"default\", \"count\": {\"per_slot\": 1}}]}",
+    },
+    {
+        "node->(slot,core)",
+        "{\"version\": 1, \"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"default\", \"count\": 1}, {\"type\": \"core\", \"count\": 1}]}], \"attributes\": {\"system\": {\"duration\": 57600}}, \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"default\", \"count\": {\"per_slot\": 1}}]}",
+    },
+    {
+        "(node,slot->core)",
+        "{\"version\": 1, \"resources\": [{\"type\": \"node\", \"count\": 1}, {\"type\": \"slot\", \"label\": \"default\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1}]}], \"attributes\": {\"system\": {\"duration\": 57600}}, \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"default\", \"count\": {\"per_slot\": 1}}]}",
+    },
+    {
+        "cluster->(node->slot->core,node)",
+        "{\"version\": 1, \"resources\": [{\"type\": \"cluster\", \"count\": 1, \"with\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"default\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1}]}]}, {\"type\": \"node\", \"count\": 1}]}], \"attributes\": {\"system\": {\"duration\": 57600}}, \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"default\", \"count\": {\"per_slot\": 1}}]}",
+    },
+    {
+        "cluster->(slot->core,node)",
+        "{\"version\": 1, \"resources\": [{\"type\": \"cluster\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"default\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1}]}, {\"type\": \"node\", \"count\": 1}]}], \"attributes\": {\"system\": {\"duration\": 57600}}, \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"default\", \"count\": {\"per_slot\": 1}}]}",
+    },
+    {
+        "cluster->(slot->core,slot->core)",
+        "{\"version\": 1, \"resources\": [{\"type\": \"cluster\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"default\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1}]}, {\"type\": \"slot\", \"label\": \"default\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1}]}]}], \"attributes\": {\"system\": {\"duration\": 57600}}, \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"default\", \"count\": {\"per_slot\": 1}}]}",
+    },
+    {
+        "node->(slot->core,storage->core)",
+        "{\"version\": 1, \"resources\": [{\"type\": \"node\", \"count\": 1, \"with\": [{\"type\": \"slot\", \"label\": \"default\", \"count\": 1, \"with\": [{\"type\": \"core\", \"count\": 1}]}, {\"type\": \"storage\", \"count\": 1562, \"exclusive\": true, \"with\": [{\"type\": \"core\", \"count\": 1}]}]}], \"attributes\": {\"system\": {\"duration\": 57600}}, \"tasks\": [{\"command\": [\"hostname\"], \"slot\": \"default\", \"count\": {\"per_slot\": 1}}]}",
+    },
     { NULL, NULL },
 };
 
@@ -77,16 +178,43 @@ int main (int argc, char **argv)
 {
     plan (NO_PLAN);
     struct jobspec *js;
+    struct output *expect;
     json_error_t error;
     int i;
 
     for (i = 0; good_input[i].desc; i++) {
         js = jobspec_parse (good_input[i].s, &error);
         ok (js != NULL, "good.%d (%s) works", i, good_input[i].desc);
-        if (!js)
+        if (!js) {
             diag ("%s", error.text);
-        else
+        } else {
+            expect = &good_output[i];
+            ok (js->task_count == expect->task_count,
+                "good.%d (%s) task count (%d) == %d",
+                i,
+                good_input[i].desc,
+                js->task_count,
+                expect->task_count);
+            ok (js->slot_count == expect->slot_count,
+                "good.%d (%s) slot count (%d) == %d",
+                i,
+                good_input[i].desc,
+                js->slot_count,
+                expect->slot_count);
+            ok (js->cores_per_slot == expect->cores_per_slot,
+                "good.%d (%s) cores per slot (%d) == %d",
+                i,
+                good_input[i].desc,
+                js->cores_per_slot,
+                expect->cores_per_slot);
+            ok (js->slots_per_node == expect->slots_per_node,
+                "good.%d (%s) slots per node (%d) == %d",
+                i,
+                good_input[i].desc,
+                js->slots_per_node,
+                expect->slots_per_node);
             jobspec_destroy (js);
+        }
     }
 
     for (i = 0; bad_input[i].desc; i++) {
diff --git a/src/shell/test/mustache.c b/src/shell/test/mustache.c
new file mode 100644
index 000000000000..ea1b4aaccd93
--- /dev/null
+++ b/src/shell/test/mustache.c
@@ -0,0 +1,109 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <string.h>
+#include <flux/core.h>
+
+#include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
+
+#include "mustache.h"
+
+struct mustache_test {
+    const char *template;
+    const char *expected;
+    int errnum;
+};
+
+struct mustache_test tests[] = {
+    { "", "", 0 },
+    { "notemplate", "notemplate", 0 },
+    { "{{", "{{", 0 },
+    { "foo-{{", "foo-{{", 0 },
+    { "}}", "}}", 0 },
+    { "foo-}}", "foo-}}", 0 },
+    { "{{boop}}", "{{boop}}", 0 },
+    { "test-{{name}}", "test-foo", 0 },
+    { "test-{{name}}.out", "test-foo.out", 0 },
+    { "test-{{name}}.out", "test-foo.out", 0 },
+    { "{{number}}", "42", 0 },
+    { "{{name}}-{{number}}.out", "foo-42.out", 0 },
+    { "{{name}}-{{number}}.out", "foo-42.out", 0 },
+    { NULL, NULL, 0 },
+};
+
+
+int cb (FILE *fp, const char *tag, void *arg)
+{
+    ok (fp != NULL,
+        "cb passed valid FILE *");
+    ok (tag != NULL,
+        "cb passed valid tag");
+    ok (arg != NULL,
+        "cb passed valid arg");
+    if (streq (tag, "name"))
+        return fputs ("foo", fp);
+    if (streq (tag, "number"))
+        return fputs ("42", fp);
+    errno = ENOENT;
+    return -1;
+}
+
+int main (int argc, char **argv)
+{
+    struct mustache_test *mp = tests;
+    struct mustache_renderer *mr = NULL;
+
+    plan (NO_PLAN);
+
+    mr = mustache_renderer_create (NULL, NULL);
+    ok (mr == NULL && errno == EINVAL,
+        "mustache_renderer_create fails with invalid callback");
+
+    /*  Pass tests as tag_f argument so we have something non-NULL to test
+     *   in the callback
+     */
+    mr = mustache_renderer_create (cb, tests);
+    ok (mr != NULL,
+        "mustache_renderer_create");
+
+    ok (mustache_render (mr, NULL) == NULL && errno == EINVAL,
+        "mustache_render (mr, NULL) returns EINVAL");
+
+    while (mp->template != NULL) {
+        char * result = mustache_render (mr, mp->template);
+        if (mp->expected == NULL)
+            ok (result == NULL && errno == mp->errnum,
+                "mustache_render '%s' failed with errno = %d",
+                mp->template,
+                errno);
+        else
+            is (result, mp->expected,
+                "mustache_render '%s' returned '%s'",
+                mp->template,
+                result);
+        free (result);
+        mp++;
+    }
+
+    mustache_renderer_destroy (mr);
+    done_testing ();
+
+    return 0;
+}
+
+/*
+ * vi:ts=4 sw=4 expandtab
+ */
diff --git a/src/shell/test/plugstack.c b/src/shell/test/plugstack.c
index 25627cd0300b..11ba76df5acb 100644
--- a/src/shell/test/plugstack.c
+++ b/src/shell/test/plugstack.c
@@ -39,31 +39,6 @@ static int bar (flux_plugin_t *p, const char *s,
                                  "{s:s}", "result", "called bar");
 }
 
-static int next_level (flux_plugin_t *p, const char *s,
-                       flux_plugin_arg_t *args, void *arg)
-{
-    struct plugstack *st = arg;
-    return flux_plugin_arg_pack (args,
-                                 FLUX_PLUGIN_ARG_OUT|FLUX_PLUGIN_ARG_UPDATE,
-                                 "{s:s}",
-                                 "next_name", plugstack_current_name (st));
-}
-
-static int check_name (flux_plugin_t *p, const char *s,
-                       flux_plugin_arg_t *args, void *arg)
-{
-    struct plugstack *st = arg;
-    int rc = flux_plugin_arg_pack (args,
-                                   FLUX_PLUGIN_ARG_OUT,
-                                   "{s:s}",
-                                   "name", plugstack_current_name (st));
-    ok (rc == 0,
-        "in check_name: flux_plugin_arg_pack worked");
-
-    /* Check a recursive call to plugstack_call () */
-    return plugstack_call (st, "next.level", args);
-}
-
 void test_invalid_args (struct plugstack *st, flux_plugin_t *p)
 {
     ok (plugstack_push (NULL, p) < 0 && errno == EINVAL,
@@ -85,8 +60,6 @@ void test_invalid_args (struct plugstack *st, flux_plugin_t *p)
         "plugstack_plugin_aux_set (NULL, ...) returns EINVAL");
     ok (plugstack_plugin_aux_set (st, NULL, NULL) < 0 && errno == EINVAL,
         "plugstack_plugin_aux_set (NULL, ...) returns EINVAL");
-    ok (plugstack_current_name (NULL) == NULL && errno == EINVAL,
-        "plugstack_current_name (NULL) returns EINVAL");
 }
 
 void test_load (void)
@@ -184,14 +157,10 @@ int main (int argc, char **argv)
 
     ok (flux_plugin_add_handler (p1, "callback", foo, NULL) == 0,
         "flux_plugin_add_handler (p1, 'callback', &foo)");
-    ok (flux_plugin_add_handler (p1, "check.name", check_name, st) == 0,
-        "flux_plugin_add_handler (p1, 'check.name', &check_name)");
     ok (flux_plugin_add_handler (p2, "callback", bar, NULL) == 0,
         "flux_plugin_add_handler (p2, 'callback', &bar)");
     ok (flux_plugin_add_handler (p3, "callback", bar, NULL) == 0,
         "flux_plugin_add_handler (p3, 'callback', &bar)");
-    ok (flux_plugin_add_handler (p3, "next.level", next_level, st) == 0,
-        "flux_plugin_add_handler (p3, 'next.level', &next_level)");
 
     if (!(args = flux_plugin_arg_create ()))
         BAIL_OUT ("flux_plugin_args_create");
@@ -222,26 +191,6 @@ int main (int argc, char **argv)
     is (result, "called bar",
         "plugstack_call called bar() last");
 
-    /*  Check plugin_current_name() and recursive plugstack_call()
-     *   between two plugins.
-     */
-    ok (plugstack_call (st, "check.name", args) == 0,
-        "plugstack_call (st, 'check.name')");
-    ok (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_OUT,
-                                "{s:s}", "name", &result) == 0,
-        "flux_plugin_arg_unpack");
-    is (result, "mikey",
-        "plugstack_current_name() worked");
-
-    ok (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_OUT,
-                                "{s:s}", "next_name", &result) == 0,
-        "flux_plugin_arg_unpack");
-    is (result, "joey",
-        "plugstack_current_name() worked");
-
-    ok (plugstack_current_name (st) == NULL,
-        "plugstack_current_name() outside of plugstack_call returns NULL");
-
     called_foo = 0;
     called_bar = 0;
     ok (plugstack_push (st, p2) == 0,
diff --git a/src/shell/tmpdir.c b/src/shell/tmpdir.c
new file mode 100644
index 000000000000..8b21abf1af07
--- /dev/null
+++ b/src/shell/tmpdir.c
@@ -0,0 +1,139 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#define FLUX_SHELL_PLUGIN_NAME "tmpdir"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "src/common/libutil/cleanup.h"
+#include "src/common/libjob/idf58.h"
+
+#include "builtins.h"
+#include "internal.h"
+#include "info.h"
+
+static int mkdir_exist_ok (const char *path, bool quiet)
+{
+    if (mkdir (path, 0700) < 0) {
+        if (errno != EEXIST) {
+            if (!quiet)
+                shell_log_errno ("mkdir %s", path);
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static int make_job_path (flux_shell_t *shell,
+                          const char *parent,
+                          char *buf,
+                          size_t size)
+{
+    size_t n;
+
+    n = snprintf (buf,
+                  size,
+                  "%s/jobtmp-%d-%s",
+                  parent,
+                  shell->info->shell_rank,
+                  idf58 (shell->jobid));
+    if (n >= size) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    return 0;
+}
+
+static int mkjobtmp_rundir (flux_shell_t *shell, char *buf, size_t size)
+{
+    const char *rundir;
+
+    if (!(rundir = flux_attr_get (shell->h, "rundir"))
+        || make_job_path (shell, rundir, buf, size) < 0
+        || mkdir_exist_ok (buf, true) < 0)
+        return -1;
+    return 0;
+}
+
+static int mkjobtmp_tmpdir (flux_shell_t *shell,
+                            const char *tmpdir,
+                            char *buf,
+                            size_t size)
+{
+    if (make_job_path (shell, tmpdir ? tmpdir : "/tmp", buf, size) < 0
+        || mkdir_exist_ok (buf, false) < 0)
+        return -1;
+    return 0;
+}
+
+static int mustache_render_tmpdir (flux_plugin_t *p,
+                                   const char *topic,
+                                   flux_plugin_arg_t *args,
+                                   void *data)
+{
+    flux_shell_t *shell = data;
+    const char *jobtmp = flux_shell_getenv (shell, "FLUX_JOB_TMPDIR");
+    return flux_plugin_arg_pack (args,
+                                 FLUX_PLUGIN_ARG_OUT,
+                                "{s:s}",
+                                "result", jobtmp);
+}
+
+static int tmpdir_init (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    const char *tmpdir = flux_shell_getenv (shell, "TMPDIR");
+    char jobtmp[1024];
+
+    /*  Attempt to create TMPDIR if set. If this fails, fallback to /tmp.
+     */
+    if (tmpdir && mkdir_exist_ok (tmpdir, true) < 0) {
+        shell_warn ("Unable to create TMPDIR=%s, resetting TMPDIR=/tmp",
+                    tmpdir);
+        tmpdir = "/tmp";
+        if (flux_shell_setenvf (shell, 1, "TMPDIR", "%s", tmpdir) < 0)
+            shell_die_errno (1, "Unable to set TMPDIR=/tmp");
+    }
+
+    /* Try to create jobtmp in broker rundir.
+     * Fall back to ${TMPDIR:-/tmp} if that fails (e.g. guest user).
+     */
+    if (mkjobtmp_rundir (shell, jobtmp, sizeof (jobtmp)) < 0
+        && mkjobtmp_tmpdir (shell, tmpdir, jobtmp, sizeof (jobtmp)) < 0)
+        shell_die_errno (1, "error creating FLUX_JOB_TMPDIR");
+    cleanup_push_string (cleanup_directory_recursive, jobtmp);
+
+    /* Set/change FLUX_JOB_TMPDIR to jobtmp.
+     */
+    if (flux_shell_setenvf (shell, 1, "FLUX_JOB_TMPDIR", "%s", jobtmp) < 0)
+        shell_die_errno (1, "error updating job environment");
+
+    if (flux_plugin_add_handler (p,
+                                 "mustache.render.tmpdir",
+                                 mustache_render_tmpdir,
+                                 shell) < 0)
+        shell_die_errno (1, "unable to register mustache template callback");
+
+    return 0;
+}
+
+struct shell_builtin builtin_tmpdir = {
+    .name = FLUX_SHELL_PLUGIN_NAME,
+    .init = tmpdir_init,
+};
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/src/test/Makefile.am b/src/test/Makefile.am
index 15d3edf8279b..0cacd0db1b2e 100644
--- a/src/test/Makefile.am
+++ b/src/test/Makefile.am
@@ -1,22 +1,11 @@
-AM_CFLAGS = \
-	$(WARNING_CFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	$(CODE_COVERAGE_LIBS)
-
-AM_CPPFLAGS = \
-	-I$(top_srcdir) \
-	-I$(top_srcdir)/src/include \
-	-I$(top_builddir)/src/common/libflux \
-	$(ZMQ_CFLAGS)
-
 noinst_SCRIPTS = \
 	relnotes.sh \
-	sched-bench.sh
-
-LDADD = $(top_builddir)/src/common/libflux-internal.la \
-	$(top_builddir)/src/common/libflux-core.la \
-	$(ZMQ_LIBS) $(LIBPTHREAD) $(LIBDL)
+	backtrace-all.sh \
+	checks-annotate.sh \
+	checks-lib.sh \
+	checks_run.sh \
+	cppcheck.sh \
+	docker-deploy.sh \
+	generate-matrix.py
 
 EXTRA_DIST = $(noinst_SCRIPTS)
diff --git a/src/test/checks-annotate.sh b/src/test/checks-annotate.sh
new file mode 100755
index 000000000000..39858af88e46
--- /dev/null
+++ b/src/test/checks-annotate.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+#
+#  Post-process testsuite logs and outputs after a failure
+#
+#  Uses GH Workflow commands for GH Actions
+#
+error() {
+    printf "::error::$@\n"
+}
+catfile() {
+    if test -f $1; then
+        printf "::group::$1\n"
+        cat $1
+        printf "::endgroup::\n"
+    fi
+}
+catfile_error() {
+    error "Found $1"
+    catfile $1
+}
+annotate_test_log() {
+    #
+    #  Look through test logfiles for various failure indicators and
+    #   emit an annotation '::error::' to the logfile if found:
+    #
+    local test=$1
+
+    #  Emit an annotation for each failed test ('not ok')
+    grep 'not ok' ${test}.log | while read line; do
+        printf "::error file=${test}.t::%s\n" "${line}"
+    done
+
+    #  Emit an annotation for TAP ERROR lines:
+    grep '^ERROR: ' ${test}.log | while read line; do
+        printf "::error file=${test}.t::%s\n" "${line}"
+    done
+
+    #  Emit an annotation for chain-lint errors:
+    grep '^error: bug in the test script' ${test}.log | while read line; do
+        printf "::error file=${test}.t::%s\n" "${line}"
+    done
+
+    #  Emit an annotation for anything that looks like an ASan error:
+    sed -n 's/==[0-9][0-9]*==ERROR: //p' ${test}.log | while read line; do
+        printf "::error file=${test}.t::%s\n" "${line}"
+    done
+}
+
+#
+#  Check all testsuite *.trs files and check for results that
+#   were not 'SKIP' or 'PASS':
+#
+logfile=/tmp/check-errors.$$
+cat /dev/null >$logfile
+
+errors=0
+total=0
+for trs in $(find . -name *.trs); do
+    : $((total++))
+    result=$(sed -n 's/^.*global-test-result: *//p' ${trs})
+    if test "$result" != "PASS" -a "$result" != "SKIP"; then
+        testbase=${trs//.trs}
+        annotate_test_log $testbase >> $logfile
+        catfile ${testbase}.output >> $logfile
+        catfile ${testbase}.log >> $logfile
+        : $((errors++))
+    fi
+done
+if test $errors -gt 0; then
+    printf "::warning::"
+fi
+printf "Found ${errors} errors from ${total} tests in testsuite\n"
+cat $logfile
+rm $logfile
+
+#
+#  Find and emit all *.asan.* files from test:
+#
+export -f catfile_error
+export -f catfile
+export -f error
+find . -name *.asan.* | xargs -i bash -c 'catfile_error {}'
+
+
+#
+#  Check for any expected tests that were not run:
+#
+ls -1 t/*.t | sort >/tmp/expected
+ls -1 t/*.trs | sed 's/rs$//' | sort >/tmp/actual
+comm -23 /tmp/expected /tmp/actual > missing
+if test -s missing; then
+    error "Detected $(wc -l missing) missing tests:"
+    for f in $(cat missing); do
+        printf "$f\n"
+        file=${f//.t}
+        test -f ${file}.log && catfile ${file}.log
+        test -f ${file}.output && catfile ${file}.output
+    done
+else
+    printf "No missing test runs detected\n"
+fi
+
diff --git a/src/test/checks-lib.sh b/src/test/checks-lib.sh
new file mode 100644
index 000000000000..bdbf9016e880
--- /dev/null
+++ b/src/test/checks-lib.sh
@@ -0,0 +1,48 @@
+#
+# Helper functions for GitHub actions, See:
+#
+# https://docs.github.com/actions/reference/workflow-commands-for-github-actions
+#
+#
+# Only emit group/endgroup when running under CI, see:
+#
+# https://docs.github.com/en/actions/reference/environment-variables
+#
+#
+if test "$CI" = "true"; then
+  checks_group_start() {
+    printf "::group::%s\n" "$1"
+  }
+  checks_group_end() {
+    printf "::endgroup::\n"
+  }
+else
+  checks_group_start() { echo "$@"; }
+  checks_group_end()   { echo "$@"; }
+fi
+
+#
+#  Usage: checks_group DESC COMMANDS...
+#
+checks_group() {
+    local DESC="$1"
+    shift 1
+    checks_group_start "$DESC"
+    eval "$@"
+    rc=$?
+    checks_group_end
+    return $rc
+}
+
+#
+#  Usage: checks_die MESSAGE COMMANDS...
+#
+checks_die() {
+    local MSG="$1"
+    shift 1
+    printf "::error::$MSG\n"
+    if $# -gt 0; then
+        eval "$@"
+    fi
+    exit 1
+}
diff --git a/src/test/checks_run.sh b/src/test/checks_run.sh
new file mode 100755
index 000000000000..c41e0abdb1bb
--- /dev/null
+++ b/src/test/checks_run.sh
@@ -0,0 +1,253 @@
+#!/bin/bash
+#
+#  Test runner script meant to be executed inside of a docker container
+#
+#  Usage: checks_run.sh [OPTIONS...]
+#
+#  Where OPTIONS are passed directly to ./configure
+#
+#  The script is otherwise influenced by the following environment variables:
+#
+#  JOBS=N        Argument for make's -j option, default=2
+#  PROJECT       Flux project, e.g. flux-core
+#  COVERAGE      Run with --enable-code-coverage, `make check-code-coverage`
+#  TEST_INSTALL  Run `make check` against installed flux-core
+#  CPPCHECK      Run cppcheck if set to "t"
+#  DISTCHECK     Run `make distcheck` if set
+#  RECHECK       Run `make recheck` if `make check` fails the first time
+#  UNIT_TEST_ONLY Only run `make check` under ./src
+#  QUICK_CHECK   Run only `make check-prep` and a simple test
+#  PRELOAD       Set as LD_PRELOAD for make and tests
+#  POISON        Install poison libflux and flux(1) in image
+#  INCEPTION     Run tests under a flux instance
+#  chain_lint    Run sharness with --chain-lint if chain_lint=t
+#  SYSTEM        Run only the system sharness tests
+#
+#  And, obviously, some crucial variables that configure itself cares about:
+#
+#  CC, CXX, LDFLAGS, CFLAGS, etc.
+#
+
+
+#  Ensure uname -m reports 32bit architecture if platform was specified
+#   as 386:
+#
+case $PLATFORM in *386)
+  unset PLATFORM
+  echo "Rexecuting under linux32 personality"
+  exec setarch i386 $0 "$@"
+  ;;
+esac
+
+# if make is old, and scl is here, and devtoolset is available and not turned
+# on, re-exec ourself with it active to get a newer make
+if make --version | grep 'GNU Make 4' 2>&1 > /dev/null ; then
+  MAKE="make --output-sync=target --no-print-directory"
+else
+  MAKE="make" #use this if all else fails
+  if test "X$X_SCLS" = "X" ; then
+    if scl -l | grep devtoolset-7 2>&1 >/dev/null ; then
+      echo  bash "$0" "$@" | scl enable devtoolset-7 -
+      exit
+    fi
+  fi
+fi
+
+# source check_group and check_time functions:
+. src/test/checks-lib.sh
+
+ARGS="$@"
+JOBS=${JOBS:-2}
+MAKECMDS="${MAKE} -j ${JOBS}"
+CHECKCMDS="${MAKE} -j ${JOBS} ${DISTCHECK:+dist}check"
+
+# Add non-standard path for libfaketime to LD_LIBRARY_PATH:
+export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/faketime"
+
+# Force git to update the shallow clone and include tags so git-describe works
+checks_group "git fetch tags" "git fetch --unshallow --tags" \
+ git fetch --unshallow --tags || true
+
+checks_group_start "build setup"
+ulimit -c unlimited
+
+# Manually update ccache symlinks (XXX: Is this really necessary?)
+test -x /usr/sbin/update-ccache-symlinks && \
+    sudo /usr/sbin/update-ccache-symlinks
+export PATH=/usr/lib/ccache:$PATH
+
+# Ensure ccache dir exists
+mkdir -p $HOME/.ccache
+
+# clang+ccache requires second cpp pass:
+if echo "$CC" | grep -q "clang"; then
+    CCACHE_CPP=1
+fi
+
+if test "$PROJECT" = "flux-core"; then
+  # Ensure ci builds libev such that libfaketime will work:
+  # (force libev to *not* use syscall interface for clock_gettime())
+  export CPPFLAGS="$CPPFLAGS -DEV_USE_CLOCK_SYSCALL=0 -DEV_USE_MONOTONIC=1"
+
+  # Ensure we always use internal <flux/core.h> by placing a dummy file
+  #  in the same path as ZMQ_FLAGS:
+  sudo sh -c "mkdir -p /usr/include/flux \
+    && echo '#error Non-build-tree flux/core.h!' > /usr/include/flux/core.h"
+fi
+
+POSTCHECKCMDS=":"
+# Enable coverage for $CC-coverage build
+# We can't use distcheck here, it doesn't play well with coverage testing:
+if test "$COVERAGE" = "t"; then
+	export PATH=~/.local/bin/:$PATH
+
+	# install coverage via pip if necessaary
+	coverage -h >/dev/null 2>&1 || python3 -m pip install coverage
+
+	# usercustomize.py must go under USER_SITE, so determine that path:
+	USER_SITE=$(python3 -c 'import site; print(site.USER_SITE)')
+	mkdir -p ${USER_SITE}
+
+	# Setup environment for Python coverage
+	# This file will be loaded by all python scripts run by the
+	# current user, but only activate coverage if COVERAGE_PROCESS_START
+	# is set in the environment.
+	#
+	cat <<-EOF >${USER_SITE}/usercustomize.py
+	try:
+	    import coverage
+	    coverage.process_startup()
+	except ImportError:
+	    pass
+	EOF
+
+	# Add Python coverage config:
+	cat <<-EOF >coverage.rc
+	[run]
+	data_file = $(pwd)/.coverage
+	include = $(pwd)/src/*
+	parallel = True
+	relative_files = True
+	omit = src/bindings/python/flux/utils/*
+	[report]
+	omit = src/bindings/python/flux/utils/*
+	EOF
+
+	rm -f .coverage .coverage*
+
+	#  Tests to run during system testing have "ci=system" in test file
+	SYSTEM_TESTS=$(cd t && grep -l ci=system *.t)
+
+	ARGS="$ARGS --enable-code-coverage"
+
+	CHECKCMDS="\
+	export ENABLE_USER_SITE=1 && \
+	export COVERAGE_PROCESS_START=$(pwd)/coverage.rc && \
+	${MAKE} -j $JOBS check-prep && \
+	(cd t && ${MAKE} -j $JOBS check ${SYSTEM:+TESTS=\"$SYSTEM_TESTS\"})"
+	POSTCHECKCMDS="\
+	${MAKE} code-coverage-capture &&
+	lcov -l flux*-coverage.info && \
+	rm -f coverage.xml && \
+	coverage combine .coverage* && \
+	coverage html && \
+	coverage xml && \
+	chmod 444 coverage.xml && \
+	(coverage report || :)"
+
+# Use make install for T_INSTALL:
+elif test "$TEST_INSTALL" = "t"; then
+    ARGS="$ARGS --prefix=/usr --sysconfdir=/etc"
+    CHECKCMDS="sudo make install && \
+              FLUX_TEST_INSTALLED_PATH=/usr/bin ${MAKE} -j $JOBS check"
+
+# Run checks as Flux jobs:
+elif test "$INCEPTION" = "t"; then
+	CHECKCMDS="make -j ${JOBS} check-prep && \
+	           cd t && ../src/cmd/flux start -s1 ./test-inception.sh && \
+	           cd .."
+fi
+
+if test -n "$PRELOAD" ; then
+  CHECKCMDS="/usr/bin/env 'LD_PRELOAD=$PRELOAD' ${CHECKCMDS}"
+fi
+
+if test -n "$UNIT_TEST_ONLY"; then
+  CHECKCMDS="(cd src && $CHECKCMDS)"
+fi
+
+if test -n "$QUICK_CHECK"; then
+  CHECKCMDS="make -j ${JOBS} check-prep && \
+	     src/cmd/flux start -s 2 flux submit --cc=1-5 --watch -vvv hostname"
+fi
+
+# CI has limited resources, even though number of processors might
+#  might appear to be large. Limit session size for testing to 5 to avoid
+#  spurious timeouts.
+export FLUX_TEST_SIZE_MAX=5
+
+# Invoke MPI tests
+# CentOS 7: mpich only available via environment-module:
+if test -f /usr/share/Modules/init/bash; then
+    . /usr/share/Modules/init/bash && module load mpi
+fi
+export FLUX_TEST_MPI=t
+
+# Generate logfiles from sharness tests for extra information:
+export FLUX_TESTS_LOGFILE=t
+export DISTCHECK_CONFIGURE_FLAGS="${ARGS}"
+
+# Force enable valgrind test
+export FLUX_ENABLE_VALGRIND_TEST=t
+
+if test "$CPPCHECK" = "t"; then
+    sh -x src/test/cppcheck.sh
+fi
+
+echo "Starting MUNGE"
+sudo /sbin/runuser -u munge /usr/sbin/munged
+
+checks_group_end # Setup
+
+checks_group "autogen.sh" ./autogen.sh || checks_die "autogen failed"
+
+WORKDIR=$(pwd)
+if test -n "$BUILD_DIR" ; then
+  mkdir -p "$BUILD_DIR"
+  cd "$BUILD_DIR"
+fi
+
+checks_group "configure ${ARGS}"  ${WORKDIR}/configure ${ARGS} \
+	|| checks_die "configure failed" cat config.log
+checks_group "make clean..." make clean
+
+if test "$POISON" = "t" -a "$PROJECT" = "flux-core"; then
+  checks_group "Installing poison libflux and commands..." \
+    sudo bash src/test/docker/poison-libflux.sh /tmp/poison-cmds
+  export FLUX_EXEC_PATH=/tmp/poison-cmds
+fi
+
+if test "$DISTCHECK" != "t"; then
+  checks_group "${MAKECMDS}" "${MAKECMDS}" \
+	|| checks_die "${MAKECMDS} failed"
+fi
+checks_group "${CHECKCMDS}" "${CHECKCMDS}" && \
+	checks_group "${POSTCHECKCMDS}" "${POSTCHECKCMDS}"
+RC=$?
+
+if test "$RECHECK" = "t" -a $RC -ne 0; then
+  #
+  # `make recheck` is not recursive, only perform it if at least some tests
+  #   under ./t were run (and presumably failed)
+  #
+  if test -s t/t0000-sharness.trs; then
+    printf "::warning::make check failed, trying recheck in ./t\n"
+		(cd t ; checks_group "make recheck" ${MAKE} -j ${JOBS} recheck) && \
+			checks_group "${POSTCHECKCMDS}" "${POSTCHECKCMDS}"
+    RC=$?
+   else
+      printf "::warning::recheck requested but no tests in ./t were run\n"
+   fi
+fi
+
+exit $RC
diff --git a/src/test/cppcheck.sh b/src/test/cppcheck.sh
index c9bccc58994d..4c8996c722d9 100755
--- a/src/test/cppcheck.sh
+++ b/src/test/cppcheck.sh
@@ -6,7 +6,6 @@ cppcheck --force --inline-suppr -j 2 --std=c99 --quiet \
     -i src/common/libtap \
     -i src/common/libminilzo \
     -i src/bindings/python \
-    -i src/common/libutil/sds.c \
     -i src/modules/kvs/test \
     -i src/broker/test \
     -i src/common/libtomlc99/test \
diff --git a/src/test/create-kvs-dumpfile.sh b/src/test/create-kvs-dumpfile.sh
new file mode 100755
index 000000000000..3dad3c8db826
--- /dev/null
+++ b/src/test/create-kvs-dumpfile.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+#
+#  Run a workload in the docker container from a given flux-core tag
+#   (previous tag by default), and preserve a dumpfile.
+#
+OUTPUTDIR="$(pwd)/output"
+TAG=$(git describe --abbrev=0)
+WORKLOAD="\
+#!/bin/sh
+#
+#  Basic workload used to create a set of varied eventlogs in KVS
+#
+flux bulksubmit --quiet --cc=1-4 {} ::: true false nocommand &&
+id=\$(flux submit --urgency=hold hostname) &&
+flux cancel \$id &&
+id=\$(flux submit --wait-event=start sleep 30) &&
+flux submit --quiet --dependency=afternotok:\$id true &&
+flux cancel \$id &&
+flux queue drain &&
+flux jobs -a
+"
+declare -r prog=${0##*/}
+die() { echo -e "$prog: $@"; exit 1; }
+
+declare -r long_opts="help,tag:,workload:,statedir:"
+declare -r short_opts="ht:w:d:"
+declare -r usage="
+Usage: $prog [OPTIONS]\n\
+Run a workload under the latest flux-core tag docker image and save\n\
+the content file to a state directory.\n\
+\n\
+Options:\n\
+ -h, --help             Display this message\n\
+ -t, --tag=NAME         Run against flux-core tag NAME instead of latest tag\n\
+ -w, --workload=SCRIPT  Run workload SCRIPT instead of a default\n\
+ -d, --output=DIR       Save dumpfile to DIR (default=$OUTPUTDIR)
+"
+
+# check if running in OSX
+if [[ "$(uname)" == "Darwin" ]]; then
+    # BSD getopt
+    GETOPTS=`/usr/bin/getopt $short_opts -- $*`
+else
+    # GNU getopt
+    GETOPTS=`/usr/bin/getopt -u -o $short_opts -l $long_opts -n $prog -- $@`
+    if [[ $? != 0 ]]; then
+        die "$usage"
+    fi
+    eval set -- "$GETOPTS"
+fi
+
+while true; do
+    case "$1" in
+      -h|--help)     echo -ne "$usage";          exit 0  ;;
+      -t|--tag)      TAG="$2";                   shift 2 ;;
+      -w|--workload) WORKLOAD_PATH="$2";         shift 2 ;;
+      -d|--output)   OUTPUTDIR="$2";             shift 2 ;;
+      --)            shift; break;                       ;;
+      *)             die "Invalid option '$1'\n$usage"   ;;
+    esac
+done
+
+if test "${TAG}" = "$(git describe)"; then
+    printf "Doing nothing by default since this commit is a tag\n"
+    printf "To force: use --tag=TAG option\n"
+    exit 0
+fi
+
+mkdir "${OUTPUTDIR}" || die "Failed to create $OUTPUTDIR"
+if test -n "$WORKLOAD_PATH"; then
+    cp $WORKLOAD_PATH ${OUTPUTDIR}/workload.sh
+else
+    printf "$WORKLOAD" > ${OUTPUTDIR}/workload.sh
+fi
+
+chmod +x ${OUTPUTDIR}/workload.sh
+
+DUMPFILE=flux-${TAG}.tar.bz2
+printf "Creating dumpfile in ${OUTPUTDIR}/${DUMPFILE}\n"
+docker run -i --rm -u $(id -u) \
+     --mount type=bind,source=${OUTPUTDIR},target=/data \
+     fluxrm/flux-core:bookworm-${TAG} \
+     flux start sh -c "/data/workload.sh; flux dump /data/${DUMPFILE}"
+
+rm ${OUTPUTDIR}/workload.sh
+
diff --git a/src/test/docker-deploy.sh b/src/test/docker-deploy.sh
new file mode 100755
index 000000000000..7b510b69a719
--- /dev/null
+++ b/src/test/docker-deploy.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+#  Tag flux-core and fluxorama docker images
+
+log() { echo "docker-deploy: $@" >&2; }
+die() { log "$@"; exit 1; }
+
+if test "$GITHUB_REPOSITORY" != "flux-framework/flux-core"; then
+    log "not in flux-framework/flux-core repo, exiting..."
+    exit 0
+fi
+
+test -n "$DOCKER_REPO" ||     die "DOCKER_REPO not set"
+test -n "$DOCKER_PASSWORD" || die "DOCKER_PASSWORD not set"
+test -n "$DOCKER_USERNAME" || die "DOCKER_USERNAME not set"
+test -n "$DOCKER_TAG" ||      die "DOCKER_TAG not set"
+
+echo $DOCKER_PASSWORD | docker login -u "$DOCKER_USERNAME" --password-stdin
+
+log "docker push ${DOCKER_TAG}"
+docker push ${DOCKER_TAG}
+
+#  If this is the bookworm build, then also tag without image name:
+if echo "$DOCKER_TAG" | grep "bookworm" | grep -qv "386"; then
+    t="${DOCKER_REPO}:${GITHUB_TAG:-latest}"
+    log "docker push ${t}"
+    docker tag "$DOCKER_TAG" ${t} && docker push ${t}
+fi
+
+#  If this is the el8 build, then build fluxorama image
+if echo "$DOCKER_TAG" | grep -q "el8"; then
+    FLUXORAMA="fluxrm/fluxorama"
+    docker build -t ${FLUXORAMA} src/test/docker/fluxorama
+    docker push ${FLUXORAMA}
+    if test -n "$GITHUB_TAG"; then
+        t=${FLUXORAMA}:${GITHUB_TAG}
+        log "docker push ${t}"
+        docker tag ${FLUXORAMA} ${t} && docker push ${t}
+    fi
+fi
diff --git a/src/test/docker/README.md b/src/test/docker/README.md
index dae98d12b11c..49b50ad89423 100644
--- a/src/test/docker/README.md
+++ b/src/test/docker/README.md
@@ -2,9 +2,9 @@
 
 The Dockerfiles, resulting docker images, and `docker-run-checks.sh`
 script contained herein are used as part of the strategy for CI testing
-of Flux Framework projects under [Travis CI](https://travis-ci.org).
+of Flux Framework projects.
 
-Docker is used under Travis to speed up deployment of an
+Docker is used under CI to speed up deployment of an
 environment with correct build dependencies and to keep a docker
 image deployed at `fluxrm/flux-core` DockerHub with latest master build
 (`fluxrm/flux-core:latest`) and tagged builds (`fluxrm/flux-core:v<tag>`),
@@ -13,21 +13,22 @@ or a tagged version of flux-core.
 
 #### fluxrm/testenv Docker images
 
-The Dockerfiles under `bionic/Dockerfile` and
-`centos7/Dockerfile` describe the images built under the
-`fluxrm/testenv:bionic` and `fluxrm/testenv:centos7`
-respectively, and include the base dependencies required to build
-flux-core. These images are updated manually by flux-core maintainers, but
-the Dockerfiles should be kept up to date for a single point of management.
+The Dockerfiles `jammy/Dockerfile`, `focal/Dockerfile`,
+`el7/Dockerfile`, and `el8/Dockerfile` describe the images built
+under the `fluxrm/testenv:jammy`, `fluxrm/testenv:focal`,
+`fluxrm/testenv:el7`, and `fluxrm/testenv:el8` respectively, and
+include the base dependencies required to build flux-core. These images
+are updated manually by flux-core maintainers, but the Dockerfiles should
+be kept up to date for a single point of management.
 
-#### The travis build Dockerfile
+#### The "checks" build Dockerfile
 
-A secondary Dockerfile exists under `./travis/Dockerfile` which is used
+A secondary Dockerfile exists under `./checks/Dockerfile` which is used
 to customize the `fluxrm/testenv` before building. Without this secondary
 `docker build` stage, there would be no way for PRs on GitHub to add
 new dependencies for users that are not core maintainers (or the "base"
 images would need to be completely rebuilt on each CI run). For now,
-`flux-security` is also built manually within the `travis/Dockerfile`
+`flux-security` is also built manually within the `checks/Dockerfile`
 because it is assumed that package will be rapidly changing, and it
 would not make sense to be constantly updating the base `fluxrm/testenv`
 Docker images.
@@ -35,23 +36,53 @@ Docker images.
 #### Adding a new dependency
 
 When constructing a PR that adds new dependency, the dependency should
-be added (for both CentOS and Ubuntu) in `travis/Dockerfile`. This will
+be added (for both rh/el and Ubuntu) in `checks/Dockerfile`. This will
 result in a temporary docker image being created during testing of the
 PR with the dependency installed.
 
 Later, a flux-core maintainer can move the dependency into the `testenv`
-Docker images `bionic/Dockerfile` and `centos7/Dockerfile`.
+Docker images `jammy/Dockerfile` and `el7/Dockerfile`.
 These docker images should then be built by hand and manually
-pushed to DockerHub at `fluxrm/testenv:bionic` and
-`fluxrm/testenv:centos7`. Be sure to test that the `docker-run-test.sh`
+pushed to DockerHub at `fluxrm/testenv:jammy` and
+`fluxrm/testenv:el7`. Be sure to test that the `docker-run-test.sh`
 script still runs against the new `testenv` images, e.g.:
 
 ```
-$ for i in bionic centos7; do
+$ for i in focal el7 el8 fedora33 fedora34 fedora35 fedora38; do
     make clean &&
     docker build --no-cache -t fluxrm/testenv:$i src/test/docker/$i &&
     src/test/docker/docker-run-checks.sh -j 4 --image=$i &&
     docker push fluxrm/testenv:$i
-   done
+  done
 ```
 
+#### Bookworm and Jammy multiarch images
+
+Building the images for linux/amd64, linux/arm64 and linux/386 requires the
+Docker buildx extensions, see
+
+ https://www.docker.com/blog/multi-arch-build-and-images-the-simple-way/
+
+and run
+```
+$  docker buildx build --push --platform=linux/arm64,linux/amd64 --tag fluxrm/testenv:jammy src/test/docker/jammy
+$  docker buildx build --push --platform=linux/386,linux/amd64,linux/arm64 --tag fluxrm/testenv:bookworm src/test/docker/bookworm
+```
+
+to build and push images to docker hub.
+
+#### Local Testing
+
+Developers can test the docker images themselves. If new dependencies are needed,
+they can update the `$image` Dockerfiles manually (where `$image` is one of jammy, el7, el8, or focal).
+To create a local Docker image, run the command:
+
+```
+docker build -t fluxrm/testenv:$image src/test/docker/$image
+```
+
+To test the locally created image, run:
+
+```
+src/test/docker/docker-run-checks.sh -i $image [options] -- [arguments]
+```
diff --git a/src/test/docker/alpine/Dockerfile b/src/test/docker/alpine/Dockerfile
new file mode 100644
index 000000000000..ed88fd8928d3
--- /dev/null
+++ b/src/test/docker/alpine/Dockerfile
@@ -0,0 +1,62 @@
+FROM alpine:3.18
+
+LABEL maintainer="Mark A. Grondona <mgrondona@llnl.gov>"
+
+# Update pkg caches, install latest pkg utils and basics:
+RUN apk add \
+    coreutils \
+    procps \
+    shadow \
+    bash \
+    sudo \
+    gcc \
+    g++ \
+    autoconf \
+    automake \
+    libtool \
+    make \
+    cmake \
+    musl-dev \
+    musl-dbg \
+    python3-dev \
+    python3 \
+    py3-cffi \
+    py3-yaml \
+    py3-jsonschema \
+    py3-ply \
+    py3-setuptools \
+    py3-sphinx \
+    lua5.1-dev \
+    lua5.1-posix \
+    czmq-dev \
+    jansson-dev \
+    hwloc-dev \
+    lz4-dev \
+    sqlite-dev \
+    ncurses-dev \
+    libarchive-dev \
+    libsodium-dev \
+    linux-pam-dev \
+    git \
+    jq \
+    aspell \
+    aspell-en \
+    valgrind
+
+RUN wget https://github.com/dun/munge/releases/download/munge-0.5.15/munge-0.5.15.tar.xz \
+ && tar -xf munge-0.5.15.tar.xz \
+ && cd munge-0.5.15 \
+ && ./configure --prefix=/usr --sysconfdir=/etc || cat config.log \
+ && make -j 4 \
+ && make install \
+ && cd .. \
+ && rm -rf munge-*
+
+RUN groupadd -r munge \
+ && useradd -d /etc/munge -g munge -s /sbin/nologin -r munge
+
+ENV LANG=C.UTF-8
+
+# Install catch by hand for now:
+COPY scripts/fetch-and-build-catch.sh /fetch-and-build-catch.sh
+RUN /fetch-and-build-catch.sh
diff --git a/src/test/docker/bionic/Dockerfile b/src/test/docker/bionic/Dockerfile
deleted file mode 100644
index 73a877283ef4..000000000000
--- a/src/test/docker/bionic/Dockerfile
+++ /dev/null
@@ -1,97 +0,0 @@
-FROM ubuntu:bionic
-
-LABEL maintainer="Tom Scogland <scogland1@llnl.gov>"
-
-# Update pkg caches, install latest pkg utils:
-RUN apt-get update \
- && apt-get -qq install -y --no-install-recommends \
-	apt-utils
-
-# Utilities
-RUN apt-get -qq install -y --no-install-recommends \
-        ca-certificates \
-        wget \
-        man \
-        git \
-        sudo \
-        vim \
-        luarocks \
-        ruby \
-        munge \
-        lcov \
-        ccache \
-        lua5.2 \
-        mpich \
-        valgrind \
-	jq
-
-# Compilers, autotools
-RUN apt-get -qq install -y --no-install-recommends \
-        build-essential \
-        pkg-config \
-        autotools-dev \
-        libtool \
-        autoconf \
-	automake \
-        make \
-        cmake \
-        clang-6.0 \
-        clang-tidy \
-        gcc-8 \
-        g++-8
-
-# Python
-RUN apt-get -qq install -y --no-install-recommends \
-        python-dev \
-        python-cffi \
-        python-six \
-	python-yaml \
-	python-jsonschema \
-        python3-dev \
-        python3-cffi \
-        python3-six \
-	python3-yaml \
-	python3-jsonschema
-
-# Other deps
-RUN apt-get -qq install -y --no-install-recommends \
-        libsodium-dev \
-        libzmq3-dev \
-        libczmq-dev \
-        libjansson-dev \
-        libmunge-dev \
-        liblua5.2-dev \
-        liblz4-dev \
-        libsqlite3-dev \
-        uuid-dev \
-        libhwloc-dev \
-        libmpich-dev
-
-# Testing utils and libs
-RUN apt-get -qq install -y --no-install-recommends \
-        faketime \
-        libfaketime \
-        pylint \
-        cppcheck \
-        aspell \
-        aspell-en
-
-RUN rm -rf /var/lib/apt/lists/*
-
-# NOTE: luaposix installed by rocks due to Ubuntu bug: #1752082 https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
-RUN luarocks install luaposix
-
-# NOTE: we need asciidoctor 1.5.7 to handle manpages, install with gem install
-RUN /usr/bin/gem install asciidoctor
-
-# Install caliper by hand for now:
-RUN mkdir caliper \
- && cd caliper \
- && wget -O - https://github.com/LLNL/Caliper/archive/v1.7.0.tar.gz | tar xvz --strip-components 1 \
- && mkdir build \
- && cd build \
- && CC=gcc CXX=g++ cmake .. -DCMAKE_INSTALL_PREFIX=/usr \
- && make -j 4 \
- && make install \
- && cd ../.. \
- && rm -rf caliper
diff --git a/src/test/docker/bookworm/Dockerfile b/src/test/docker/bookworm/Dockerfile
new file mode 100644
index 000000000000..c9c00eb2d164
--- /dev/null
+++ b/src/test/docker/bookworm/Dockerfile
@@ -0,0 +1,120 @@
+FROM debian:bookworm
+
+LABEL maintainer="Tom Scogland <scogland1@llnl.gov>"
+
+# Update pkg caches, install latest pkg utils and basics:
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        apt-utils \
+        wget \
+# Utilities
+ && apt-get -qq install -y --no-install-recommends \
+        locales \
+        ca-certificates \
+        wget \
+        man \
+        git \
+        flex \
+        ssh \
+        sudo \
+        vim \
+        luarocks \
+        munge \
+        lcov \
+        ccache \
+        lua5.2 \
+        mpich \
+        valgrind \
+        jq \
+# Compilers, autotools
+        build-essential \
+        pkg-config \
+        autotools-dev \
+        libtool \
+        autoconf \
+        automake \
+        make \
+        cmake \
+        clang-15 \
+        clang-tools-15 \
+        gcc-12 \
+        g++-12 \
+# Python
+        libffi-dev \
+        python3.11-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-wheel \
+# Other deps
+        libsodium-dev \
+        libzmq3-dev \
+        libjansson-dev \
+        libmunge-dev \
+        libncursesw5-dev \
+        liblua5.2-dev \
+        liblz4-dev \
+        libsqlite3-dev \
+        uuid-dev \
+        libhwloc-dev \
+        libmpich-dev \
+        libs3-dev \
+        libevent-dev \
+        libarchive-dev \
+        libpam-dev \
+# Testing utils and libs
+        faketime \
+        libfaketime \
+        pylint \
+        cppcheck \
+        enchant-2 \
+        aspell \
+        aspell-en \
+        time \
+ && rm -rf /var/lib/apt/lists/* \
+# NOTE: sudo pip install is necessary to get differentiated installations of
+# python binary components for multiple python3 variants, --ignore-installed
+# makes it ignore local versions of the packages if your home directory is
+# mapped into the container and contains the same libraries
+ ;  for PY in python3.11 ; do \
+        sudo $PY -m pip install --upgrade --ignore-installed \
+        --break-system-packages \
+	    "markupsafe==2.0.0" \
+            coverage cffi ply six pyyaml "jsonschema>=2.6,<4.0" \
+            sphinx sphinx-rtd-theme sphinxcontrib-spelling; \
+	sudo mkdir -p /usr/lib/${PY}/dist-packages; \
+	echo ../site-packages >/tmp/site-packages.pth; \
+	sudo mv /tmp/site-packages.pth /usr/lib/${PY}/dist-packages; \
+    done ; \
+    apt-get -qq purge -y python3-pip \
+ && apt-get -qq autoremove -y
+
+RUN locale-gen en_US.UTF-8
+
+# NOTE: luaposix installed by rocks due to Ubuntu bug: #1752082 https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
+RUN luarocks install luaposix
+
+# Install catch by hand for now:
+COPY scripts/fetch-and-build-catch.sh /fetch-and-build-catch.sh
+RUN /fetch-and-build-catch.sh
+
+# Install openpmix, prrte
+RUN mkdir prrte \
+ && cd prrte \
+ && git clone https://github.com/openpmix/openpmix.git \
+ && git clone https://github.com/openpmix/prrte.git \
+ && ls -l \
+ && set -x \
+ && cd openpmix \
+ && git checkout fefaed568f33bf86f28afb6e45237f1ec5e4de93 \
+ && ./autogen.pl \
+ && ./configure --prefix=/usr --disable-static && make -j 4 install \
+ && ldconfig \
+ && cd .. \
+ && cd prrte \
+ && git checkout 477894f4720d822b15cab56eee7665107832921c \
+ && ./autogen.pl \
+ && ./configure --prefix=/usr && make -j 4 install \
+ && cd ../.. \
+ && rm -rf prrte
+
+ENV LANG=C.UTF-8
diff --git a/src/test/docker/centos7/Dockerfile b/src/test/docker/centos7/Dockerfile
deleted file mode 100644
index 69d6a09708f7..000000000000
--- a/src/test/docker/centos7/Dockerfile
+++ /dev/null
@@ -1,90 +0,0 @@
-FROM centos:7
-
-LABEL maintainer="Tom Scogland <scogland1@llnl.gov>"
-
-# add EPEL so we don't have to build everything by hand
-# add scl and devtoolset-7 so we can use next-rhel tools, just make 4 for now
-# so we can avoid make sync issues on travis
-RUN yum -y update \
- && yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \
- && yum -y install centos-release-scl-rh \
- && yum -y update \
- && yum -y install \
-      which \
-      sudo \
-      git \
-      wget \
-      vim-minimal \
-      autoconf \
-      automake \
-      libtool \
-      gcc \
-      gcc-c++ \
-      file \
-      make \
-      munge \
-      munge-devel \
-      coreutils \
-      ccache \
-      cppcheck \
-      czmq-devel \
-      hwloc \
-      hwloc-devel \
-      jansson-devel \
-      sqlite-devel \
-      uuid-devel \
-      libuuid-devel \
-      libfaketime \
-      libsodium-devel \
-      lua \
-      lua-devel \
-      lua-posix \
-      mpich-devel \
-      pkgconfig \
-      python-devel \
-      python-cffi \
-      python-six \
-      python-yaml \
-      python-jsonschema \
-      python36-devel \
-      python36-cffi \
-      python36-six \
-      python36-yaml \
-      python36-jsonschema \
-      ruby \
-      sqlite \
-      valgrind \
-      valgrind-devel \
-      man-db \
-      aspell \
-      aspell-en \
-      devtoolset-7-make \
-      lz4-devel \
-      jq \
- && yum clean all
-
-# The cmake from yum is incredibly ancient, download a less ancient one
-RUN wget -q --no-check-certificate https://cmake.org/files/v3.10/cmake-3.10.1-Linux-x86_64.tar.gz\
- && tar -xzf cmake-3.10.1-Linux-x86_64.tar.gz\
- && cp -fR cmake-3.10.1-Linux-x86_64/* /usr\
- && rm -rf cmake-3.10.1-Linux-x86_64\
- && rm cmake-3.10.1-Linux-x86_64.tar.gz
-
-RUN /usr/bin/gem install asciidoctor
-
-# Install caliper by hand for now:
-RUN mkdir caliper \
- && cd caliper \
- && wget -O - https://github.com/LLNL/Caliper/archive/v1.7.0.tar.gz | tar xvz --strip-components 1 \
- && mkdir build \
- && cd build \
- && cmake .. -DCMAKE_INSTALL_PREFIX=/usr \
- && make -j 4 \
- && make install \
- && cd ../.. \
- && rm -rf caliper
-
-COPY config.site /usr/share/config.site
-
-# Create /tmp -> /var/tmp link to ensure Flux tests work in this configuration
-RUN rm -rf /tmp && ln -sf /var/tmp /tmp
diff --git a/src/test/docker/centos8/Dockerfile b/src/test/docker/centos8/Dockerfile
deleted file mode 100644
index 337c80f66c37..000000000000
--- a/src/test/docker/centos8/Dockerfile
+++ /dev/null
@@ -1,98 +0,0 @@
-FROM centos:8
-
-LABEL maintainer="Mark Grondona <mgrondona@llnl.gov>"
-
-#  Enable PowerTools for development packages
-RUN yum -y update \
- && dnf -y install 'dnf-command(config-manager)' \
- && yum config-manager --set-enabled PowerTools \
- && yum -y update
-
-#  Enable EPEL
-RUN yum -y install epel-release
-
-#  Utilities
-RUN yum -y install \
-	wget \
-	man-db \
-	git \
-	sudo \
-	ruby \
-	munge \
-	ccache \
-	lua \
-	mpich \
-	valgrind \
-	jq \
-	which \
-	file \
-	vim
-
-#  Compilers, autotools
-RUN yum -y install \
-	pkgconfig \
-	libtool \
-	autoconf \
-	automake \
-	gcc \
-	gcc-c++ \
-	make \
-	cmake
-	
-#  Python
-RUN yum -y install \
-	python36 \
-	python3-devel \
-	python3-cffi \
-	python3-six \
-	python3-yaml \
-	python3-jsonschema
-
-#  Development dependencies
-RUN yum -y install \
-	libsodium-devel \
-        zeromq-devel \
-	czmq-devel \
-	jansson-devel \
-	munge-devel \
-	lz4-devel \
-	sqlite-devel \
-	libuuid-devel \
-	hwloc-devel \
-	mpich-devel \
-	lua-devel \
-	valgrind-devel
-
-#  Other deps
-RUN yum -y install \
-	perl-Time-HiRes \
-	lua-posix \
-	libfaketime \
-	cppcheck \
-	aspell \
-	aspell-en
-
-#  Clean up
-RUN yum clean all
-
-#  Set default /usr/bin/python to python3
-RUN alternatives --set python /usr/bin/python3
-
-#  Add /usr/bin/mpicc link so MPI tests are built
-RUN alternatives --install /usr/bin/mpicc mpicc /usr/lib64/mpich/bin/mpicc 100
-
-RUN /usr/bin/gem install asciidoctor
-
-# Install caliper by hand for now:
-RUN mkdir caliper \
- && cd caliper \
- && wget -O - https://github.com/LLNL/Caliper/archive/v1.7.0.tar.gz | tar xvz --strip-components 1 \
- && mkdir build \
- && cd build \
- && cmake .. -DCMAKE_INSTALL_PREFIX=/usr \
- && make -j 4 \
- && make install \
- && cd ../.. \
- && rm -rf caliper
-
-COPY config.site /usr/share/config.site
diff --git a/src/test/docker/checks/Dockerfile b/src/test/docker/checks/Dockerfile
new file mode 100644
index 000000000000..f5d51508faad
--- /dev/null
+++ b/src/test/docker/checks/Dockerfile
@@ -0,0 +1,77 @@
+ARG IMAGESRC
+
+FROM $IMAGESRC
+
+# Allow flux-security version, username, UID, and GID to be overridden on
+#  docker build command line:
+#
+ARG USER=fluxuser
+ARG UID=1000
+ARG GID=1000
+ARG FLUX_SECURITY_VERSION
+ARG BASE_IMAGE
+
+# Install flux-security by hand for now:
+#
+RUN CCACHE_DISABLE=1 \
+ && V=$FLUX_SECURITY_VERSION \
+ && PKG=flux-security-$V \
+ && URL=https://github.com/flux-framework/flux-security/releases/download \
+ && wget ${URL}/v${V}/${PKG}.tar.gz \
+ && tar xvfz ${PKG}.tar.gz \
+ && cd ${PKG} \
+ && ./configure --prefix=/usr --sysconfdir=/etc || cat config.log \
+ && make -j 4 \
+ && make install \
+ && cd .. \
+ && rm -rf flux-security-*
+
+
+# Add configured user to image with sudo access:
+#
+RUN set -x && groupadd -g $UID $USER \
+ && useradd -g $USER -u $UID -d /home/$USER -m $USER \
+ && printf "$USER ALL= NOPASSWD: ALL\\n" >> /etc/sudoers
+
+# Also add "flux" user to image with sudo access:
+#
+RUN set -x && groupadd fluxuser \
+ && useradd -g fluxuser -d /home/fluxuser -m fluxuser -s /bin/bash \
+ && printf "fluxuser ALL= NOPASSWD: ALL\\n" >> /etc/sudoers
+
+# Make sure user in appropriate group for sudo on different platforms
+RUN case $BASE_IMAGE in \
+     jammy*) adduser $USER sudo && adduser fluxuser sudo ;; \
+     focal*)  adduser $USER sudo && adduser fluxuser sudo ;; \
+     el*|fedora*) usermod -G wheel $USER && usermod -G wheel fluxuser ;; \
+     *) (>&2 echo "Unknown BASE_IMAGE") ;; \
+    esac
+
+# Install extra dependencies if necessary here.
+#
+# Do not forget to run `apt update` on Ubuntu/jammy.
+# Do NOT run `yum upgrade` on RPM systems (this will unnecessarily upgrade
+#  existing packages)
+#
+RUN case $BASE_IMAGE in \
+     jammy*) ;; \
+     bookworm*) ;; \
+     focal*) ;; \
+     el*|fedora*) ;; \
+     *) (>&2 echo "Unknown BASE_IMAGE") ;; \
+    esac
+
+# Setup MUNGE directories & key
+RUN mkdir -p /var/run/munge \
+ && dd if=/dev/urandom bs=1 count=1024 > /etc/munge/munge.key \
+ && chown -R munge /etc/munge/munge.key /var/run/munge \
+ && chmod 600 /etc/munge/munge.key
+
+COPY entrypoint.sh /usr/local/sbin/
+COPY bashrc /tmp
+RUN cat /tmp/bashrc >> ~fluxuser/.bashrc \
+ && rm /tmp/bashrc
+
+ENV BASE_IMAGE=$BASE_IMAGE
+USER $USER
+WORKDIR /home/$USER
diff --git a/src/test/docker/checks/bashrc b/src/test/docker/checks/bashrc
new file mode 100644
index 000000000000..f1ce06674cb4
--- /dev/null
+++ b/src/test/docker/checks/bashrc
@@ -0,0 +1,4 @@
+if ! echo "$PS1" | grep -q FLUX; then
+  PS1=$'${FLUX_URI+\u0192(s=$(flux getattr size),d=$(flux getattr instance-level)$(which flux|grep -q src/cmd && echo ,builddir))} '${PS1}
+fi
+
diff --git a/src/test/docker/checks/entrypoint.sh b/src/test/docker/checks/entrypoint.sh
new file mode 100755
index 000000000000..eaeefbd70eee
--- /dev/null
+++ b/src/test/docker/checks/entrypoint.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+sudo runuser -u munge /usr/sbin/munged
+exec "$@"
diff --git a/src/test/docker/docker-run-checks.sh b/src/test/docker/docker-run-checks.sh
index fa995e331793..13177f026b12 100755
--- a/src/test/docker/docker-run-checks.sh
+++ b/src/test/docker/docker-run-checks.sh
@@ -1,27 +1,35 @@
 #!/bin/bash
 #
-#  Build flux-core "travis" docker image and run tests, exporting
+#  Build flux "checks" docker image and run tests, exporting
 #   important environment variables to the docker environment.
 #
 #  Arguments here are passed directly to ./configure
 #
 #
 # option Defaults:
-IMAGE=bionic
-FLUX_SECURITY_VERSION=0.2.0
+PROJECT=flux-core
+BASE_DOCKER_REPO=fluxrm/testenv
+
+WORKDIR=/usr/src
+IMAGE=bookworm
 JOBS=2
-MOUNT_HOME_ARGS="--volume=$HOME:/home/$USER -e HOME"
+MOUNT_HOME_ARGS="--volume=$HOME:$HOME -e HOME"
+
+if test "$PROJECT" = "flux-core"; then
+  FLUX_SECURITY_VERSION=0.13.0
+  POISON=t
+fi
 
 #
 declare -r prog=${0##*/}
 die() { echo -e "$prog: $@"; exit 1; }
 
 #
-declare -r long_opts="help,quiet,interactive,image:,flux-security-version:,jobs:,no-cache,no-home,distcheck,tag:,build-directory:,install-only"
-declare -r short_opts="hqIdi:S:j:t:D:"
-declare -r usage="
+declare -r long_opts="help,quiet,interactive,image:,flux-security-version:,jobs:,no-cache,no-home,distcheck,tag:,build-directory:,install-only,no-poison,recheck,unit-test-only,quick-check,inception,platform:,workdir:,system"
+declare -r short_opts="hqIdi:S:j:t:D:Prup:"
+declare usage="
 Usage: $prog [OPTIONS] -- [CONFIGURE_ARGS...]\n\
-Build docker image for travis builds, then run tests inside the new\n\
+Build docker image for CI builds, then run tests inside the new\n\
 container as the current user and group.\n\
 \n\
 Uses the current git repo for the build.\n\
@@ -31,24 +39,36 @@ Options:\n\
      --no-cache                Disable docker caching\n\
      --no-home                 Skip mounting the host home directory\n\
      --install-only            Skip make check, only make install\n\
+     --inception               Run tests as flux jobs\n\
+     --system                  Run under system instance\n\
  -q, --quiet                   Add --quiet to docker-build\n\
  -t, --tag=TAG                 If checks succeed, tag image as NAME\n\
  -i, --image=NAME              Use base docker image NAME (default=$IMAGE)\n\
+ -p, --platform=NAME           Run on alternate platform (if supported)\n\
  -S, --flux-security-version=N Install flux-security vers N (default=$FLUX_SECURITY_VERSION)\n
  -j, --jobs=N                  Value for make -j (default=$JOBS)\n
  -d, --distcheck               Run 'make distcheck' instead of 'make check'\n\
+ -r, --recheck                 Run 'make recheck' after failure\n\
+ -u, --unit-test-only          Only run unit tests\n\
+     --quick-check             Only run check-prep and one basic test\n\
+ -P, --no-poison               Do not install poison libflux and flux(1)\n\
  -D, --build-directory=DIRNAME Name of a subdir to build in, will be made\n\
- -I, --interactive             Instead of running travis build, run docker\n\
+     --workdir=PATH            Use PATH as working directory for build\n\
+ -I, --interactive             Instead of running ci build, run docker\n\
                                 image with interactive shell.\n\
 "
 
 # check if running in OSX
-if [[ "$(uname)" == "Darwin" ]]; then
+if [[ "$(uname)" == "Darwin" ]] && [[ $FORCE_GNU_GETOPT != 1 ]]; then
     # BSD getopt
-    GETOPTS=`/usr/bin/getopt $short_opts -- $*`
+    GETOPTS=`getopt $short_opts -- $*`
+    usage=${usage}"\n\
+    You are using BSD getopt on macOS. BSD getopt does not recognize '='\n\
+    between options. Use a space instead. If gnu-getopt is first in your\n\
+    PATH, force the script to use that by setting FORCE_GNU_GETOPT=1.\n"
 else
     # GNU getopt
-    GETOPTS=`/usr/bin/getopt -u -o $short_opts -l $long_opts -n $prog -- $@`
+    GETOPTS=`getopt -u -o $short_opts -l $long_opts -n $prog -- $@`
     if [[ $? != 0 ]]; then
         die "$usage"
     fi
@@ -59,70 +79,135 @@ while true; do
       -h|--help)                   echo -ne "$usage";          exit 0  ;;
       -q|--quiet)                  QUIET="--quiet";            shift   ;;
       -i|--image)                  IMAGE="$2";                 shift 2 ;;
+      -p|--platform)               PLATFORM="--platform=$2";   shift 2 ;;
       -S|--flux-security-version)  FLUX_SECURITY_VERSION="$2"; shift 2 ;;
       -j|--jobs)                   JOBS="$2";                  shift 2 ;;
       -I|--interactive)            INTERACTIVE="/bin/bash";    shift   ;;
       -d|--distcheck)              DISTCHECK=t;                shift   ;;
+      -r|--recheck)                RECHECK=t;                  shift   ;;
+      -u|--unit-test-only)         UNIT_TEST_ONLY=t;           shift   ;;
+      --quick-check)               QUICK_CHECK=t;              shift   ;;
       -D|--build-directory)        BUILD_DIR="$2";             shift 2 ;;
+      --build-arg)                 BUILD_ARG=" --build-arg $2" shift 2 ;;
+      --workdir)                   WORKDIR="$2";               shift 2 ;;
       --no-cache)                  NO_CACHE="--no-cache";      shift   ;;
       --no-home)                   MOUNT_HOME_ARGS="";         shift   ;;
       --install-only)              INSTALL_ONLY=t;             shift   ;;
+      --inception)                 INCEPTION=t;                shift   ;;
+      --system)                    SYSTEM=t;                   shift   ;;
+      -P|--no-poison)              POISON=0;                   shift   ;;
       -t|--tag)                    TAG="$2";                   shift 2 ;;
       --)                          shift; break;                       ;;
       *)                           die "Invalid option '$1'\n$usage"   ;;
     esac
 done
 
-
 TOP=$(git rev-parse --show-toplevel 2>&1) \
-    || die "not inside flux-core git repository!"
+    || die "not inside $PROJECT git repository!"
 which docker >/dev/null \
     || die "unable to find a docker binary"
+if docker buildx >/dev/null 2>&1; then
+    DOCKER_BUILD="docker buildx build --load"
+else
+    DOCKER_BUILD="docker build"
+fi
+DOCKER_BUILD="docker build"
+
+# distcheck incompatible with some configure args
+if test "$DISTCHECK" = "t"; then
+    test "$RECHECK" = "t" && die "--recheck not allowed with --distcheck"
+    test "$SYSTEM" = "t" && die "--system not allowed with --distcheck"
+    for arg in "$@"; do
+        case $arg in
+          --sysconfdir=*|systemdsystemunitdir=*)
+            die "distcheck incompatible with configure arg $arg"
+        esac
+    done
+fi
+
+if test "$SYSTEM" = "t"; then
+    if test "$IMAGE" != "el8"; then
+        echo >&2 "Setting image to el8 for system checks build"
+    fi
+    IMAGE=el8
+    TAG="checks-builder:el8"
+    INSTALL_ONLY=t
+    NO_CACHE="--no-cache"
+    POISON=0
+fi
 
 CONFIGURE_ARGS="$@"
 
-. ${TOP}/src/test/travis-lib.sh
+. ${TOP}/src/test/checks-lib.sh
+
+#  NOTE: BASE_IMAGE, IMAGESRC, FLUX_SECURITY_VERSION are ignored
+#   unless in flux-core repo
+#
+BUILD_IMAGE=checks-builder:${IMAGE}
+if test "$PROJECT" = "flux-core"; then
+    DOCKERFILE=$TOP/src/test/docker/checks
+else
+    DOCKERFILE=$TOP/src/test/docker/$IMAGE
+fi
 
-travis_fold "docker_build" \
-  "Building image $IMAGE for user $USER $(id -u) group=$(id -g)" \
-  docker build \
+checks_group "Building image $IMAGE for user $USER $(id -u) group=$(id -g)" \
+  ${DOCKER_BUILD} \
+    ${PLATFORM} \
     ${NO_CACHE} \
     ${QUIET} \
     --build-arg BASE_IMAGE=$IMAGE \
-    --build-arg IMAGESRC="fluxrm/testenv:$IMAGE" \
+    --build-arg IMAGESRC="$BASE_DOCKER_REPO:$IMAGE" \
     --build-arg USER=$USER \
     --build-arg UID=$(id -u) \
     --build-arg GID=$(id -g) \
     --build-arg FLUX_SECURITY_VERSION=$FLUX_SECURITY_VERSION \
-    -t travis-builder:${IMAGE} \
-    $TOP/src/test/docker/travis \
+    ${BUILD_ARG:- } \
+    -t ${BUILD_IMAGE} \
+    ${DOCKERFILE} \
     || die "docker build failed"
 
 if [[ -n "$MOUNT_HOME_ARGS" ]]; then
-    echo "mounting $HOME as /home/$USER"
+    echo "mounting $HOME as $HOME"
 fi
-echo "mounting $TOP as /usr/src"
+echo "mounting $TOP as $WORKDIR"
 
+export PLATFORM
+export PROJECT
+export POISON
+export INCEPTION
 export JOBS
 export DISTCHECK
+export RECHECK
+export UNIT_TEST_ONLY
+export QUICK_CHECK
 export BUILD_DIR
+export COVERAGE
 export chain_lint
 
 if [[ "$INSTALL_ONLY" == "t" ]]; then
     docker run --rm \
-	--workdir=/usr/src \
-        --volume=$TOP:/usr/src \
-	travis-builder:${IMAGE} \
-	sh -c "./autogen.sh &&
-               ./configure --prefix=/usr &&
-               make clean && 
-               make -j${JOBS}" \
-	|| (docker rm tmp.$$; die "docker run of 'make install' failed")
+        --workdir=$WORKDIR \
+        --volume=$TOP:$WORKDIR \
+        ${PLATFORM} \
+        ${BUILD_IMAGE} \
+        sh -c "./autogen.sh &&
+               ./configure --prefix=/usr --sysconfdir=/etc \
+                --with-systemdsystemunitdir=/etc/systemd/system \
+                --localstatedir=/var \
+                --with-flux-security &&
+               make clean &&
+               make -j${JOBS}"
+    RC=$?
+    docker rm tmp.$$
+    test $RC -ne 0 &&  die "docker run of 'make install' failed"
 else
     docker run --rm \
-        --workdir=/usr/src \
-        --volume=$TOP:/usr/src \
+        --workdir=$WORKDIR \
+        --volume=$TOP:$WORKDIR \
+        --mount type=tmpfs,destination=/test/tmpfs-1m,tmpfs-size=1048576 \
+        ${PLATFORM} \
         $MOUNT_HOME_ARGS \
+        -e PLATFORM \
         -e CC \
         -e CXX \
         -e LDFLAGS \
@@ -135,20 +220,36 @@ else
         -e TEST_INSTALL \
         -e CPPCHECK \
         -e DISTCHECK \
+        -e RECHECK \
+        -e UNIT_TEST_ONLY \
+        -e QUICK_CHECK \
         -e chain_lint \
         -e JOBS \
         -e USER \
-        -e TRAVIS \
+	-e PROJECT \
+        -e CI \
         -e TAP_DRIVER_QUIET \
+        -e FLUX_TEST_TIMEOUT \
+        -e FLUX_TEST_SIZE_MAX \
+        -e PYTHON \
         -e PYTHON_VERSION \
         -e PRELOAD \
+        -e POISON \
+        -e INCEPTION \
         -e ASAN_OPTIONS \
         -e BUILD_DIR \
+        -e S3_ACCESS_KEY_ID \
+        -e S3_SECRET_ACCESS_KEY \
+        -e S3_HOSTNAME \
+        -e S3_BUCKET \
+	-e PSM3_HAL \
+	-e PSM3_DEVICES \
         --cap-add SYS_PTRACE \
         --tty \
         ${INTERACTIVE:+--interactive} \
-        travis-builder:${IMAGE} \
-        ${INTERACTIVE:-./src/test/travis_run.sh ${CONFIGURE_ARGS}} \
+        --network=host \
+        ${BUILD_IMAGE} \
+        ${INTERACTIVE:-./src/test/checks_run.sh ${CONFIGURE_ARGS}} \
     || die "docker run failed"
 fi
 
@@ -156,20 +257,29 @@ if test -n "$TAG"; then
     # Re-run 'make install' in fresh image, otherwise we get all
     # the context from the build above
     docker run --name=tmp.$$ \
-	--workdir=/usr/src \
-        --volume=$TOP:/usr/src \
+        --workdir=${WORKDIR}/${BUILD_DIR} \
+        --volume=$TOP:${WORKDIR} \
         --user="root" \
-	travis-builder:${IMAGE} \
+        ${PLATFORM} \
+	${BUILD_IMAGE} \
 	sh -c "make install && \
-               su -c 'flux keygen' flux && \
                userdel $USER" \
 	|| (docker rm tmp.$$; die "docker run of 'make install' failed")
     docker commit \
-	--change 'CMD "/usr/bin/flux"' \
-	--change 'USER flux' \
-	--change 'WORKDIR /home/flux' \
+	--change 'ENTRYPOINT [ "/usr/local/sbin/entrypoint.sh" ]' \
+	--change 'CMD [ "/usr/bin/flux",  "start", "/bin/bash" ]' \
+	--change 'USER fluxuser' \
+	--change 'WORKDIR /home/fluxuser' \
 	tmp.$$ $TAG \
 	|| die "docker commit failed"
     docker rm tmp.$$
     echo "Tagged image $TAG"
 fi
+
+if test -n "$SYSTEM"; then
+    ${TOP}/src/test/docker/docker-run-systest.sh \
+	--image=${TAG} \
+        --jobs=${JOBS} \
+        -- src/test/checks_run.sh ${CONFIGURE_ARGS} \
+    || die "docker-run-systest.sh failed"
+fi
diff --git a/src/test/docker/docker-run-systest.sh b/src/test/docker/docker-run-systest.sh
new file mode 100755
index 000000000000..71bf38c985e3
--- /dev/null
+++ b/src/test/docker/docker-run-systest.sh
@@ -0,0 +1,198 @@
+#!/bin/bash
+#
+#  Build fluxorama image with builddir mounted as /usr/src for testing.
+#
+
+PROJECT=flux-core
+WORKDIR=/usr/src
+MOUNT_HOME_ARGS="--volume=$HOME:/home/$USER -e HOME"
+JOBS=2
+IMAGE="fluxrm/flux-core:el8"
+
+declare -r prog=${0##*/}
+die() { echo -e "$prog: $@"; exit 1; }
+
+declare -r long_opts="help,no-home,no-cache,rebuild,jobs:,image:"
+declare -r short_opts="hrj:i:"
+declare -r usage="
+Usage: $prog [OPTIONS]\n\
+Build fluxorama system test docker image for CI builds\n\
+\n\
+Options:\n\
+ -h, --help                    Display this message\n\
+ -j, --jobs=N                  Value for make -j (default=$JOBS)\n\
+ -i, --image=NAME              Base image (default=$IMAGE)\n\
+     --rebuild                 Rebuild base fluxorama image from source\n\
+     --no-home                 Skip mounting the host home directory\n\
+     --no-cache                Run docker build with --no-cache option\n\
+"
+
+# check if running in OSX
+if [[ "$(uname)" == "Darwin" ]]; then
+    # BSD getopt
+    GETOPTS=`/usr/bin/getopt $short_opts -- $*`
+else
+    # GNU getopt
+    GETOPTS=`/usr/bin/getopt -u -o $short_opts -l $long_opts -n $prog -- $@`
+    if [[ $? != 0 ]]; then
+        die "$usage"
+    fi
+    eval set -- "$GETOPTS"
+fi
+
+while true; do
+    case "$1" in
+      -h|--help)     echo -ne "$usage";          exit 0  ;;
+      -j|--jobs)     JOBS="$2";                  shift 2 ;;
+      -i|--image)    IMAGE="$2";                 shift 2 ;;
+      --rebuild)     REBUILD_BASE_IMAGE=t;       shift   ;;
+      --no-home)     MOUNT_HOME_ARGS="";         shift   ;;
+      --no-cache)    NOCACHE="--no-cache";       shift   ;;
+      --)            shift; break;                       ;;
+      *)             die "Invalid option '$1'\n$usage"   ;;
+    esac
+done
+
+if test $# -eq 0; then
+    set "bash"
+fi
+
+TOP=$(git rev-parse --show-toplevel 2>&1) \
+    || die "not inside $PROJECT git repository!"
+which podman >/dev/null \
+    || die "unable to find podman binary!"
+which docker >/dev/null \
+    || die "unable to find docker binary!"
+
+. ${TOP}/src/test/checks-lib.sh
+
+if test "$REBUILD_BASE_IMAGE" = "t"; then
+    checks_group "Rebuilding fluxrm/flux-core:el8 from source" \
+      $TOP/src/test/docker/docker-run-checks.sh \
+        -j $JOBS \
+        -i el8 \
+        -t $IMAGE \
+        --install-only
+fi
+
+#  Note: podman cannot pull from local docker images in GitHub actions, so
+#  we have to use docker save -> podman load to make the image available
+#  to podman. This is done in steps to avoid a potential issue with
+#  podman reporting:
+#
+#  Error: payload does not match any of the supported image formats:
+#
+#  Saving to a file (avoiding a colon in the name) seems to work around
+#  these issues.
+#
+checks_group "Moving $IMAGE from docker to podman" \
+  docker save -o /tmp/systest-$$.tar $IMAGE \
+  && ls -lh /tmp/systest-$$.tar \
+  && (podman load -i /tmp/systest-$$.tar || die "podman load failed") \
+  && rm -f /tmp/systest-$$.tar
+
+#  Note: There's a bug in podman < 4 which saves an image loaded from
+#  docker save as the wrong name, so we use the image digest instead:
+IMAGE=$(docker images --format {{.ID}} $IMAGE)
+
+checks_group "Building system image for user $USER $(id -u) group=$(id -g)" \
+  podman build \
+    ${NOCACHE} \
+    --build-arg IMAGE=$IMAGE \
+    --build-arg USER=$USER \
+    --build-arg UID=$(id -u) \
+    --build-arg GID=$(id -g) \
+    -t fluxorama:systest \
+    --target=systest \
+    ${TOP}/src/test/docker/fluxorama \
+    || die "docker build failed"
+
+NAME=flux-system-test-$$
+checks_group "Launching system instance container $NAME" \
+  podman run -d --rm \
+    --hostname=fluxorama \
+    --workdir=$WORKDIR \
+    $MOUNT_HOME_ARGS \
+    --volume=$TOP:$WORKDIR \
+    --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \
+    --tmpfs=/run \
+    --mount=type=tmpfs,destination=/test/tmpfs-1m,tmpfs-size=1048576 \
+    --cap-add SYS_PTRACE \
+    --name=flux-system-test-$$ \
+    --network=host \
+    --systemd=always \
+    --userns=keep-id \
+    --security-opt unmask=/sys/fs/cgroup \
+    fluxorama:systest \
+    || die "docker run of fluxorama test container failed"
+
+until podman exec -u $USER:$GID \
+    flux-system-test-$$ flux run hostname 2>/dev/null; do
+    echo "Waiting for flux-system-test-$$ to be ready"
+    sleep 1
+done
+
+#  Start user@uid.service service for unit tests:
+#
+checks_group "Starting user service user@$(id -u).service" \
+  podman exec flux-system-test-$$ \
+    systemctl start user@$(id -u).service \
+    || die "podman start user@$(id -u).service failed"
+
+if test -t 0; then
+    INTERACTIVE="-ti"
+fi
+
+checks_group "Executing tests under system instance container" \
+  podman exec \
+    "${INTERACTIVE}" \
+    -u $USER:$GID \
+    ${CC+-e CC=$CC} \
+    ${CXX+-e CXX=$CXX} \
+    ${LDFLAGS+-e LDFLAGS=$LDFLAGS} \
+    ${CFLAGS+-e CFLAGS=$CFLAGS} \
+    ${CPPFLAGS+-e CPPFLAGS=$CPPFLAGS} \
+    -e PS1 \
+    -e GCOV \
+    -e CCACHE_CPP2 \
+    -e CCACHE_READONLY \
+    -e COVERAGE \
+    -e TEST_INSTALL \
+    -e CPPCHECK \
+    -e DISTCHECK \
+    -e RECHECK \
+    -e UNIT_TEST_ONLY \
+    -e chain_lint \
+    -e JOBS \
+    -e USER \
+    -e PROJECT \
+    -e CI \
+    -e TAP_DRIVER_QUIET \
+    -e FLUX_TEST_TIMEOUT \
+    -e FLUX_TEST_SIZE_MAX \
+    -e FLUX_ENABLE_SYSTEM_TESTS=t \
+    -e PYTHON_VERSION \
+    -e PRELOAD \
+    -e POISON \
+    -e INCEPTION \
+    -e ASAN_OPTIONS \
+    -e BUILD_DIR \
+    -e S3_ACCESS_KEY_ID \
+    -e S3_SECRET_ACCESS_KEY \
+    -e S3_HOSTNAME \
+    -e S3_BUCKET \
+    -e HOME=/home/$USER \
+    -e XDG_RUNTIME_DIR=/run/user/$(id -u) \
+    -e DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u)/bus \
+    -e SYSTEM=t \
+    -w $WORKDIR \
+    flux-system-test-$$ "$@"
+RC=$?
+
+podman exec -ti flux-system-test-$$ shutdown -r now
+
+if test $RC -ne 0; then
+    die "system tests failed with rc=$RC"
+fi
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/test/docker/el7/Dockerfile b/src/test/docker/el7/Dockerfile
new file mode 100644
index 000000000000..eb386fdf3160
--- /dev/null
+++ b/src/test/docker/el7/Dockerfile
@@ -0,0 +1,84 @@
+FROM centos:7
+
+LABEL maintainer="Tom Scogland <scogland1@llnl.gov>"
+
+# add EPEL so we don't have to build everything by hand
+# add scl and devtoolset-7 so we can use next-rhel tools, just make 4 for now
+# so we can avoid make sync issues on travis
+RUN yum -y update \
+ && yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \
+ && yum -y install centos-release-scl-rh \
+ && yum -y update \
+ && yum -y install \
+      which \
+      sudo \
+      git \
+      wget \
+      vim-minimal \
+      autoconf \
+      automake \
+      libtool \
+      gcc \
+      gcc-c++ \
+      file \
+      make \
+      munge \
+      munge-devel \
+      ncurses-devel \
+      coreutils \
+      ccache \
+      cppcheck \
+      hwloc \
+      hwloc-devel \
+      jansson-devel \
+      sqlite-devel \
+      uuid-devel \
+      libuuid-devel \
+      libfaketime \
+      libsodium-devel \
+      lua \
+      lua-devel \
+      lua-posix \
+      mpich-devel \
+      pkgconfig \
+      python-devel \
+      python-cffi \
+      python-six \
+      python-yaml \
+      python-jsonschema \
+      python36-devel \
+      python36-cffi \
+      python36-six \
+      python36-yaml \
+      python36-jsonschema \
+      sqlite \
+      valgrind \
+      valgrind-devel \
+      man-db \
+      enchant \
+      aspell \
+      aspell-en \
+      devtoolset-7-make \
+      lz4-devel \
+      jq \
+      libs3-devel \
+      libarchive-devel \
+      pam-devel \
+ && yum clean all
+
+# Sphinx packages for docs
+RUN python3 -m pip install sphinx sphinx-rtd-theme sphinxcontrib-spelling
+
+# The cmake from yum is incredibly ancient, download a less ancient one
+RUN wget -q --no-check-certificate https://cmake.org/files/v3.10/cmake-3.10.1-Linux-x86_64.tar.gz\
+ && tar -xzf cmake-3.10.1-Linux-x86_64.tar.gz\
+ && cp -fR cmake-3.10.1-Linux-x86_64/* /usr\
+ && rm -rf cmake-3.10.1-Linux-x86_64\
+ && rm cmake-3.10.1-Linux-x86_64.tar.gz
+
+COPY config.site /usr/share/config.site
+
+ENV LANG=en_US.UTF-8
+
+# Create /tmp -> /var/tmp link to ensure Flux tests work in this configuration
+RUN rm -rf /tmp && ln -sf /var/tmp /tmp
diff --git a/src/test/docker/centos7/config.site b/src/test/docker/el7/config.site
similarity index 100%
rename from src/test/docker/centos7/config.site
rename to src/test/docker/el7/config.site
diff --git a/src/test/docker/el8/Dockerfile b/src/test/docker/el8/Dockerfile
new file mode 100644
index 000000000000..265fd8cdafae
--- /dev/null
+++ b/src/test/docker/el8/Dockerfile
@@ -0,0 +1,105 @@
+FROM rockylinux:8
+
+LABEL maintainer="Mark Grondona <mgrondona@llnl.gov>"
+
+#  Enable PowerTools for development packages
+RUN yum -y update \
+ && dnf -y install 'dnf-command(config-manager)' \
+ && yum config-manager --set-enabled powertools \
+ && yum -y update \
+#  Enable EPEL
+ && yum -y install epel-release \
+#  Utilities
+ && yum -y install \
+	wget \
+	man-db \
+	less \
+	git \
+	sudo \
+	munge \
+	ccache \
+	lua \
+	valgrind \
+	jq \
+	which \
+	file \
+	vim \
+	patch \
+	diffutils \
+#  Compilers, autotools
+	pkgconfig \
+	libtool \
+	autoconf \
+	automake \
+	gcc \
+	gcc-c++ \
+	clang \
+	clang-tools-extra \
+	make \
+	ninja-build \
+	cmake \
+	bison \
+	flex \
+#  Python
+	python36 \
+	python3-devel \
+	python3-cffi \
+	python3-six \
+	python3-yaml \
+	python3-jsonschema \
+	python3-sphinx \
+#  Development dependencies
+	libsodium-devel \
+	zeromq-devel \
+	jansson-devel \
+	munge-devel \
+	ncurses-devel \
+	lz4-devel \
+	sqlite-devel \
+	libuuid-devel \
+	hwloc-devel \
+	lua-devel \
+	valgrind-devel \
+	libs3-devel \
+	systemd-devel \
+	libarchive-devel \
+	pam-devel \
+#  Other deps
+	perl-Time-HiRes \
+	lua-posix \
+	libfaketime \
+	cppcheck \
+	enchant \
+	aspell \
+	aspell-en \
+	glibc-langpack-en \
+	hwloc \
+ && yum clean all
+
+#  Set default /usr/bin/python to python3
+RUN alternatives --set python /usr/bin/python3
+
+# Install catch by hand for now:
+COPY scripts/fetch-and-build-catch.sh /fetch-and-build-catch.sh
+RUN /fetch-and-build-catch.sh
+
+# Install mvapich2
+RUN mkdir mvapich2 \
+ && cd mvapich2 \
+ && wget -O - http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.3.6.tar.gz | tar xvz --strip-components 1 \
+ && ./configure --with-device=ch3:sock --disable-fortran --prefix=/usr \
+ && make -j4 \
+ && make install \
+ && cd .. \
+ && rm -rf mvapich2
+
+# Install lcov
+RUN rpm --nodeps -i http://downloads.sourceforge.net/ltp/lcov-1.14-1.noarch.rpm
+
+# Install Python 3 coverage
+RUN pip3 install coverage
+
+ENV LANG=C.UTF-8
+RUN printf "LANG=C.UTF-8" > /etc/locale.conf
+
+COPY src/test/docker/el9/config.site /usr/share/config.site
diff --git a/src/test/docker/centos8/config.site b/src/test/docker/el8/config.site
similarity index 100%
rename from src/test/docker/centos8/config.site
rename to src/test/docker/el8/config.site
diff --git a/src/test/docker/el9/Dockerfile b/src/test/docker/el9/Dockerfile
new file mode 100644
index 000000000000..dc39e521965c
--- /dev/null
+++ b/src/test/docker/el9/Dockerfile
@@ -0,0 +1,98 @@
+FROM rockylinux:9
+
+LABEL maintainer="Mark Grondona <mgrondona@llnl.gov>"
+
+#  Enable PowerTools for development packages
+RUN dnf -y install 'dnf-command(config-manager)' \
+ && dnf config-manager --enable crb \
+#  Enable EPEL
+ && dnf -y install epel-release \
+#  Utilities
+ && dnf -y install \
+	wget \
+	man-db \
+	less \
+	git \
+	sudo \
+	munge \
+	ccache \
+	lua \
+	valgrind \
+	jq \
+	which \
+	file \
+	vim \
+	patch \
+	diffutils \
+#  Compilers, autotools
+	pkgconfig \
+	libtool \
+	autoconf \
+	automake \
+	gcc \
+	gcc-c++ \
+	clang \
+	clang-tools-extra \
+	make \
+	ninja-build \
+	cmake \
+	bison \
+	flex \
+#  Python
+	python3-devel \
+	python3-cffi \
+	python3-six \
+	python3-yaml \
+	python3-jsonschema \
+	python3-sphinx \
+	python3-coverage \
+#  Development dependencies
+	libsodium-devel \
+	zeromq-devel \
+	jansson-devel \
+	munge-devel \
+	ncurses-devel \
+	lz4-devel \
+	sqlite-devel \
+	libuuid-devel \
+	hwloc-devel \
+	lua-devel \
+	valgrind-devel \
+	libs3-devel \
+	systemd-devel \
+	libarchive-devel \
+	pam-devel \
+	mpich-devel \
+#  Other deps
+	perl-Time-HiRes \
+	lua-posix \
+	libfaketime \
+	cppcheck \
+	enchant \
+	aspell \
+	aspell-en \
+	glibc-langpack-en \
+	hwloc \
+	lcov \
+ && dnf clean all
+
+#  Set default /usr/bin/python to python3
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
+# Install catch by hand for now:
+COPY scripts/fetch-and-build-catch.sh /fetch-and-build-catch.sh
+RUN /fetch-and-build-catch.sh
+
+
+ENV LANG=C.UTF-8
+RUN printf "LANG=C.UTF-8" > /etc/locale.conf
+
+COPY src/test/docker/el9/config.site /usr/share/config.site
+
+# the psm3 connector added to libfabrics in ~1.12 causes errors when allowed to
+# use non-local connectors on a system with virtual NICs, since we're in a
+# docker container, prevent this
+ENV PSM3_HAL=loopback
+# hwloc tries to look for opengl devices  by connecting to a port that might
+# sometimes be an x11 port, but more often for us is munge, turn it off
+ENV HWLOC_COMPONENTS=-gl
diff --git a/src/test/docker/el9/config.site b/src/test/docker/el9/config.site
new file mode 100644
index 000000000000..5f3ec6260736
--- /dev/null
+++ b/src/test/docker/el9/config.site
@@ -0,0 +1,6 @@
+# Force libdir to /usr/lib64 if prefix=/usr on this platform
+if test "$prefix" = "/usr" ; then
+    test $sysconfdir = "${prefix}/etc" && sysconfdir=/etc
+    libdir=/usr/lib64
+fi
+:
diff --git a/src/test/docker/fedora33/Dockerfile b/src/test/docker/fedora33/Dockerfile
new file mode 100644
index 000000000000..617479e563c1
--- /dev/null
+++ b/src/test/docker/fedora33/Dockerfile
@@ -0,0 +1,78 @@
+FROM fedora:33
+
+LABEL maintainer="Mark Grondona <mgrondona@llnl.gov>"
+
+#  Enable PowerTools for development packages
+RUN yum -y update \
+ && yum -y update \
+#  Utilities
+ && yum -y install \
+	wget \
+	man-db \
+	less \
+	git \
+	sudo \
+	munge \
+	ccache \
+	lua \
+	mpich \
+	valgrind \
+	jq \
+	which \
+	file \
+	vim \
+	patch \
+	diffutils \
+	hostname \
+#  Compilers, autotools
+	pkgconfig \
+	libtool \
+	autoconf \
+	automake \
+	gcc \
+	gcc-c++ \
+	libasan \
+	make \
+	cmake \
+#  Python
+	python36 \
+	python3-devel \
+	python3-cffi \
+	python3-six \
+	python3-yaml \
+	python3-jsonschema \
+	python3-sphinx \
+#  Development dependencies
+	libsodium-devel \
+	zeromq-devel \
+	jansson-devel \
+	munge-devel \
+	ncurses-devel \
+	lz4-devel \
+	sqlite-devel \
+	libuuid-devel \
+	hwloc-devel \
+	mpich-devel \
+	lua-devel \
+	valgrind-devel \
+	libs3-devel \
+	libarchive-devel \
+	pam-devel \
+#  Other deps
+	perl-Time-HiRes \
+	lua-posix \
+	libfaketime \
+	cppcheck \
+	enchant \
+	aspell \
+	aspell-en \
+	glibc-langpack-en \
+ && yum clean all
+
+#  Add /usr/bin/mpicc link so MPI tests are built
+RUN alternatives --install /usr/bin/mpicc mpicc /usr/lib64/mpich/bin/mpicc 100
+
+ENV LANG=C.UTF-8
+RUN printf "LANG=C.UTF-8" > /etc/locale.conf
+
+COPY config.site /usr/share/config.site
diff --git a/src/test/docker/fedora33/config.site b/src/test/docker/fedora33/config.site
new file mode 100644
index 000000000000..5f3ec6260736
--- /dev/null
+++ b/src/test/docker/fedora33/config.site
@@ -0,0 +1,6 @@
+# Force libdir to /usr/lib64 if prefix=/usr on this platform
+if test "$prefix" = "/usr" ; then
+    test $sysconfdir = "${prefix}/etc" && sysconfdir=/etc
+    libdir=/usr/lib64
+fi
+:
diff --git a/src/test/docker/fedora34/Dockerfile b/src/test/docker/fedora34/Dockerfile
new file mode 100644
index 000000000000..2f6d4f24ebaa
--- /dev/null
+++ b/src/test/docker/fedora34/Dockerfile
@@ -0,0 +1,78 @@
+FROM fedora:34
+
+LABEL maintainer="Mark Grondona <mgrondona@llnl.gov>"
+
+#  Enable PowerTools for development packages
+RUN yum -y update \
+ && yum -y update \
+#  Utilities
+ && yum -y install \
+	wget \
+	man-db \
+	less \
+	git \
+	sudo \
+	munge \
+	ccache \
+	lua \
+	mpich \
+	valgrind \
+	jq \
+	which \
+	file \
+	vim \
+	patch \
+	diffutils \
+	hostname \
+#  Compilers, autotools
+	pkgconfig \
+	libtool \
+	autoconf \
+	automake \
+	gcc \
+	gcc-c++ \
+	libasan \
+	make \
+	cmake \
+#  Python
+	python36 \
+	python3-devel \
+	python3-cffi \
+	python3-six \
+	python3-yaml \
+	python3-jsonschema \
+	python3-sphinx \
+#  Development dependencies
+	libsodium-devel \
+	zeromq-devel \
+	jansson-devel \
+	munge-devel \
+	ncurses-devel \
+	lz4-devel \
+	sqlite-devel \
+	libuuid-devel \
+	hwloc-devel \
+	mpich-devel \
+	lua-devel \
+	valgrind-devel \
+	libs3-devel \
+	libarchive-devel \
+	pam-devel \
+#  Other deps
+	perl-Time-HiRes \
+	lua-posix \
+	libfaketime \
+	cppcheck \
+	enchant \
+	aspell \
+	aspell-en \
+	glibc-langpack-en \
+ && yum clean all
+
+#  Add /usr/bin/mpicc link so MPI tests are built
+RUN alternatives --install /usr/bin/mpicc mpicc /usr/lib64/mpich/bin/mpicc 100
+
+ENV LANG=C.UTF-8
+RUN printf "LANG=C.UTF-8" > /etc/locale.conf
+
+COPY config.site /usr/share/config.site
diff --git a/src/test/docker/fedora34/config.site b/src/test/docker/fedora34/config.site
new file mode 100644
index 000000000000..5f3ec6260736
--- /dev/null
+++ b/src/test/docker/fedora34/config.site
@@ -0,0 +1,6 @@
+# Force libdir to /usr/lib64 if prefix=/usr on this platform
+if test "$prefix" = "/usr" ; then
+    test $sysconfdir = "${prefix}/etc" && sysconfdir=/etc
+    libdir=/usr/lib64
+fi
+:
diff --git a/src/test/docker/fedora35/Dockerfile b/src/test/docker/fedora35/Dockerfile
new file mode 100644
index 000000000000..17303279fb9e
--- /dev/null
+++ b/src/test/docker/fedora35/Dockerfile
@@ -0,0 +1,102 @@
+FROM fedora:35
+
+LABEL maintainer="Mark Grondona <mgrondona@llnl.gov>"
+
+#  Enable PowerTools for development packages
+RUN yum -y update \
+ && yum -y update \
+#  Utilities
+ && yum -y install \
+	wget \
+	man-db \
+	less \
+	git \
+	sudo \
+	munge \
+	ccache \
+	lua \
+	mpich \
+	valgrind \
+	jq \
+	which \
+	file \
+	vim \
+	patch \
+	diffutils \
+	hostname \
+	flex \
+#  Compilers, autotools
+	pkgconfig \
+	libtool \
+	autoconf \
+	automake \
+	gcc \
+	gcc-c++ \
+	libasan \
+	make \
+	cmake \
+#  Python
+	python36 \
+	python3-devel \
+	python3-cffi \
+	python3-six \
+	python3-yaml \
+	python3-jsonschema \
+	python3-sphinx \
+#  Development dependencies
+	libsodium-devel \
+	zeromq-devel \
+	jansson-devel \
+	munge-devel \
+	ncurses-devel \
+	lz4-devel \
+	sqlite-devel \
+	libuuid-devel \
+	hwloc-devel \
+	mpich-devel \
+	lua-devel \
+	valgrind-devel \
+	libs3-devel \
+	libarchive-devel \
+	pam-devel \
+	libevent-devel \
+#  Other deps
+	perl-Time-HiRes \
+	lua-posix \
+	libfaketime \
+	cppcheck \
+	enchant \
+	aspell \
+	aspell-en \
+	glibc-langpack-en \
+	lcov \
+	hwloc \
+ && yum clean all
+
+#  Add /usr/bin/mpicc link so MPI tests are built
+RUN alternatives --install /usr/bin/mpicc mpicc /usr/lib64/mpich/bin/mpicc 100
+
+# Install openpmix, prrte
+RUN mkdir prrte \
+ && cd prrte \
+ && git clone https://github.com/openpmix/openpmix.git \
+ && git clone https://github.com/openpmix/prrte.git \
+ && ls -l \
+ && set -x \
+ && cd openpmix \
+ && git checkout fefaed568f33bf86f28afb6e45237f1ec5e4de93 \
+ && ./autogen.pl \
+ && ./configure --prefix=/usr --disable-static && make -j 4 install \
+ && ldconfig \
+ && cd .. \
+ && cd prrte \
+ && git checkout 477894f4720d822b15cab56eee7665107832921c \
+ && ./autogen.pl \
+ && ./configure --prefix=/usr && make -j 4 install \
+ && cd ../.. \
+ && rm -rf prrte
+
+ENV LANG=C.UTF-8
+RUN printf "LANG=C.UTF-8" > /etc/locale.conf
+
+COPY config.site /usr/share/config.site
diff --git a/src/test/docker/fedora35/config.site b/src/test/docker/fedora35/config.site
new file mode 100644
index 000000000000..5f3ec6260736
--- /dev/null
+++ b/src/test/docker/fedora35/config.site
@@ -0,0 +1,6 @@
+# Force libdir to /usr/lib64 if prefix=/usr on this platform
+if test "$prefix" = "/usr" ; then
+    test $sysconfdir = "${prefix}/etc" && sysconfdir=/etc
+    libdir=/usr/lib64
+fi
+:
diff --git a/src/test/docker/fedora38/Dockerfile b/src/test/docker/fedora38/Dockerfile
new file mode 100644
index 000000000000..16cda14c360d
--- /dev/null
+++ b/src/test/docker/fedora38/Dockerfile
@@ -0,0 +1,77 @@
+FROM fedora:38
+
+LABEL maintainer="Mark Grondona <mgrondona@llnl.gov>"
+
+#  Enable PowerTools for development packages
+RUN yum -y update \
+ && yum -y update \
+#  Utilities
+ && yum -y install \
+	wget \
+	man-db \
+	less \
+	git \
+	sudo \
+	munge \
+	ccache \
+	lua \
+	mpich \
+	valgrind \
+	jq \
+	which \
+	file \
+	vim \
+	patch \
+	diffutils \
+	hostname \
+#  Compilers, autotools
+	pkgconfig \
+	libtool \
+	autoconf \
+	automake \
+	gcc \
+	gcc-c++ \
+	libasan \
+	make \
+	cmake \
+#  Python
+	python3-devel \
+	python3-cffi \
+	python3-six \
+	python3-yaml \
+	python3-jsonschema \
+	python3-sphinx \
+#  Development dependencies
+	libsodium-devel \
+	zeromq-devel \
+	jansson-devel \
+	munge-devel \
+	ncurses-devel \
+	lz4-devel \
+	sqlite-devel \
+	libuuid-devel \
+	hwloc-devel \
+	mpich-devel \
+	lua-devel \
+	valgrind-devel \
+	libs3-devel \
+	libarchive-devel \
+	pam-devel \
+#  Other deps
+	perl-Time-HiRes \
+	lua-posix \
+	libfaketime \
+	cppcheck \
+	enchant \
+	aspell \
+	aspell-en \
+	glibc-langpack-en \
+ && yum clean all
+
+#  Add /usr/bin/mpicc link so MPI tests are built
+RUN alternatives --install /usr/bin/mpicc mpicc /usr/lib64/mpich/bin/mpicc 100
+
+ENV LANG=C.UTF-8
+RUN printf "LANG=C.UTF-8" > /etc/locale.conf
+
+COPY config.site /usr/share/config.site
diff --git a/src/test/docker/fedora38/config.site b/src/test/docker/fedora38/config.site
new file mode 100644
index 000000000000..5f3ec6260736
--- /dev/null
+++ b/src/test/docker/fedora38/config.site
@@ -0,0 +1,6 @@
+# Force libdir to /usr/lib64 if prefix=/usr on this platform
+if test "$prefix" = "/usr" ; then
+    test $sysconfdir = "${prefix}/etc" && sysconfdir=/etc
+    libdir=/usr/lib64
+fi
+:
diff --git a/src/test/docker/fedora39/Dockerfile b/src/test/docker/fedora39/Dockerfile
new file mode 100644
index 000000000000..cad59e577d3c
--- /dev/null
+++ b/src/test/docker/fedora39/Dockerfile
@@ -0,0 +1,78 @@
+FROM fedora:39
+
+LABEL maintainer="Mark Grondona <mgrondona@llnl.gov>"
+
+#  Enable PowerTools for development packages
+RUN yum -y update \
+ && yum -y update \
+#  Utilities
+ && yum -y install \
+	wget \
+	man-db \
+	less \
+	git \
+	sudo \
+	munge \
+	ccache \
+	lua \
+	mpich \
+	valgrind \
+	jq \
+	which \
+	file \
+	vim \
+	patch \
+	diffutils \
+	hostname \
+#  Compilers, autotools
+	pkgconfig \
+	libtool \
+	autoconf \
+	automake \
+	gcc \
+	gcc-c++ \
+	libasan \
+	make \
+	cmake \
+#  Python
+	python3-devel \
+	python3-cffi \
+	python3-six \
+	python3-yaml \
+	python3-jsonschema \
+	python3-sphinx \
+#  Development dependencies
+	libsodium-devel \
+	zeromq-devel \
+	jansson-devel \
+	munge-devel \
+	ncurses-devel \
+	lz4-devel \
+	sqlite-devel \
+	libuuid-devel \
+	hwloc-devel \
+	mpich-devel \
+	lua-devel \
+	valgrind-devel \
+	libs3-devel \
+	libarchive-devel \
+	pam-devel \
+#  Other deps
+	perl-Time-HiRes \
+	lua-posix \
+	libfaketime \
+	cppcheck \
+	enchant \
+	aspell \
+	aspell-en \
+	time \
+	glibc-langpack-en \
+ && yum clean all
+
+#  Add /usr/bin/mpicc link so MPI tests are built
+RUN alternatives --install /usr/bin/mpicc mpicc /usr/lib64/mpich/bin/mpicc 100
+
+ENV LANG=C.UTF-8
+RUN printf "LANG=C.UTF-8" > /etc/locale.conf
+
+COPY config.site /usr/share/config.site
diff --git a/src/test/docker/fedora39/config.site b/src/test/docker/fedora39/config.site
new file mode 100644
index 000000000000..5f3ec6260736
--- /dev/null
+++ b/src/test/docker/fedora39/config.site
@@ -0,0 +1,6 @@
+# Force libdir to /usr/lib64 if prefix=/usr on this platform
+if test "$prefix" = "/usr" ; then
+    test $sysconfdir = "${prefix}/etc" && sysconfdir=/etc
+    libdir=/usr/lib64
+fi
+:
diff --git a/src/test/docker/fedora40/Dockerfile b/src/test/docker/fedora40/Dockerfile
new file mode 100644
index 000000000000..067d07ff5e20
--- /dev/null
+++ b/src/test/docker/fedora40/Dockerfile
@@ -0,0 +1,91 @@
+FROM fedora:40
+
+LABEL maintainer="Mark Grondona <mgrondona@llnl.gov>"
+
+#  Enable PowerTools for development packages
+#  Utilities
+RUN dnf -y install \
+	wget \
+	man-db \
+	less \
+	git \
+	sudo \
+	munge \
+	ccache \
+	lua \
+	mpich \
+	valgrind \
+	jq \
+	which \
+	file \
+	vim \
+	patch \
+	diffutils \
+	hostname \
+#  Compilers, autotools
+	pkgconfig \
+	libtool \
+	autoconf \
+	automake \
+	gcc \
+	gcc-c++ \
+	clang \
+	clang-tools-extra \
+	libasan \
+	make \
+	ninja-build \
+	cmake \
+#  Python
+	python3-devel \
+	python3-cffi \
+	python3-six \
+	python3-yaml \
+	python3-jsonschema \
+	python3-sphinx \
+	python3-setuptools \
+#  Development dependencies
+	libsodium-devel \
+	zeromq-devel \
+	jansson-devel \
+	munge-devel \
+	ncurses-devel \
+	lz4-devel \
+	sqlite-devel \
+	libuuid-devel \
+	hwloc-devel \
+	mpich-devel \
+	lua-devel \
+	valgrind-devel \
+	libs3-devel \
+	libarchive-devel \
+	pam-devel \
+	pmix-devel \
+	catch-devel \
+#  Other deps
+	perl-Time-HiRes \
+	lua-posix \
+	libfaketime \
+	cppcheck \
+	enchant \
+	aspell \
+	aspell-en \
+	time \
+	glibc-langpack-en \
+	lcov \
+ && dnf clean all
+
+#  Add /usr/bin/mpicc link so MPI tests are built
+RUN alternatives --install /usr/bin/mpicc mpicc /usr/lib64/mpich/bin/mpicc 100
+
+ENV LANG=C.UTF-8
+RUN printf "LANG=C.UTF-8" > /etc/locale.conf
+
+COPY src/test/docker/fedora40/config.site /usr/share/config.site
+
+# the psm3 connector added to libfabrics in ~1.12 causes errors when allowed to
+# use non-local connectors on a system with virtual NICs, since we're in a
+# docker container, prevent this
+ENV PSM3_HAL=loopback
+# hwloc tries to look for opengl devices  by connecting to a port that might
+# sometimes be an x11 port, but more often for us is munge, turn it off
+ENV HWLOC_COMPONENTS=-gl
diff --git a/src/test/docker/fedora40/config.site b/src/test/docker/fedora40/config.site
new file mode 100644
index 000000000000..5f3ec6260736
--- /dev/null
+++ b/src/test/docker/fedora40/config.site
@@ -0,0 +1,6 @@
+# Force libdir to /usr/lib64 if prefix=/usr on this platform
+if test "$prefix" = "/usr" ; then
+    test $sysconfdir = "${prefix}/etc" && sysconfdir=/etc
+    libdir=/usr/lib64
+fi
+:
diff --git a/src/test/docker/fluxorama/Dockerfile b/src/test/docker/fluxorama/Dockerfile
new file mode 100644
index 000000000000..a178715ec979
--- /dev/null
+++ b/src/test/docker/fluxorama/Dockerfile
@@ -0,0 +1,97 @@
+ARG IMAGE=fluxrm/flux-core:el8
+FROM $IMAGE AS systest
+
+# Default container user.
+# "fluxuser" should match fluxrm/flux-core image
+ARG USER=fluxuser
+ARG UID=2100
+ARG GID=2100
+
+ENV container docker
+
+STOPSIGNAL SIGRTMIN+3
+
+USER root
+
+VOLUME [ "/sys/fs/cgroup" ]
+
+# See: https://hub.docker.com/r/centos/systemd/dockerfile
+RUN (cd /lib/systemd/system/sysinit.target.wants/; \
+     for i in *; \
+         do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; \
+      done); \
+    rm -f /lib/systemd/system/multi-user.target.wants/*;\
+    rm -f /etc/systemd/system/*.wants/*;\
+    rm -f /lib/systemd/system/local-fs.target.wants/*; \
+    rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+    rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+    rm -f /lib/systemd/system/basic.target.wants/*;\
+    rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+RUN id $USER \
+ || ( groupadd -g $UID $USER \
+   && useradd -g $USER -u $UID -d /home/$USER -m $USER \
+   && printf "$USER ALL=(root,flux) NOPASSWD: ALL\\n" >> /etc/sudoers \
+   && usermod -G wheel $USER \
+ )
+
+#
+# Add flux user and group
+#
+RUN useradd --user-group --system -d /home/flux -m flux
+RUN touch /home/flux/.bashrc
+RUN echo export XDG_RUNTIME_DIR=/run/user/$(id -u flux) >> /home/flux/.bashrc
+RUN echo export DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u flux)/bus >> /home/flux/.bashrc
+
+#
+#  Add users besides fluxuser for mult-user testing
+RUN for i in $(seq 1 5); do \
+      user="user${i}"; \
+      uid=$((${UID} + 100 + $i)); \
+      printf "Adding ${user}\n"; \
+      groupadd -g $uid $user; \
+      useradd -g $user -u $uid -d /home/$user -m $user; \
+    done
+
+#  Copy in configuration
+COPY imp.toml /etc/flux/imp/conf.d/imp.toml
+COPY job-exec.toml /etc/flux/system/conf.d/exec.toml
+COPY access.toml /etc/flux/system/conf.d/access.toml
+
+RUN chmod 4755 /usr/libexec/flux/flux-imp \
+ && chmod 0644 /etc/flux/imp/conf.d/imp.toml \
+ && chmod 0644 /etc/flux/system/conf.d/exec.toml \
+ && chmod 0644 /etc/flux/system/conf.d/access.toml \
+ && systemctl enable flux.service \
+ && systemctl enable munge.service
+
+WORKDIR /home/$USER
+EXPOSE 7681
+CMD [ "init" ]
+
+FROM systest AS fluxorama
+
+RUN dnf install -y psmisc tmux \
+ && dnf clean all
+
+#
+#  Install ttyd
+RUN wget https://github.com/tsl0922/ttyd/releases/download/1.6.0/ttyd_linux.x86_64
+RUN mv ttyd_linux.x86_64 /usr/bin/ttyd && chmod 755 /usr/bin/ttyd
+
+#
+#  Register ttyd service
+RUN user=$USER && uid=$(id -u $user) && gid=$(id -g $user) \
+ && printf "#!/bin/sh\n"       >/bin/ttyd.sh \
+ && printf "rm -f /var/run/nologin\n" >>/bin/ttyd.sh \
+ && printf "/usr/bin/ttyd -o -p 7681 /bin/login $user\n" >>/bin/ttyd.sh \
+ && printf "systemctl halt --no-block\n" >>/bin/ttyd.sh \
+ && chmod 755 /bin/ttyd.sh \
+ && printf "[Unit]\n" >/etc/systemd/system/ttyd.service \
+ && printf "Description=ttyd service\n\n" >>/etc/systemd/system/ttyd.service \
+ && printf "[Service]\n" >>/etc/systemd/system/ttyd.service \
+ && printf "Type=simple\n" >>/etc/systemd/system/ttyd.service \
+ && printf "ExecStart=/bin/ttyd.sh\n" >>/etc/systemd/system/ttyd.service \
+ && printf "User=root\nGroup=root\n" >>/etc/systemd/system/ttyd.service \
+ && printf "[Install]\nWantedBy=multi-user.target\n" >>/etc/systemd/system/ttyd.service \
+ && systemctl enable ttyd.service
diff --git a/src/test/docker/fluxorama/access.toml b/src/test/docker/fluxorama/access.toml
new file mode 100644
index 000000000000..02bfc478c78b
--- /dev/null
+++ b/src/test/docker/fluxorama/access.toml
@@ -0,0 +1,3 @@
+[access]
+allow-guest-user = true
+allow-root-owner = true
diff --git a/src/test/docker/fluxorama/entrypoint.sh b/src/test/docker/fluxorama/entrypoint.sh
new file mode 100755
index 000000000000..584b6891a909
--- /dev/null
+++ b/src/test/docker/fluxorama/entrypoint.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+if systemctl is-enabled ttyd; then
+  #  Set default password for fluxuser
+  if test -z "$FLUXUSER_PASSWORD"; then
+     printf >&2 "ERROR: No password set for fluxuser\n"
+     printf >&2 "Please set via the FLUXUSER_PASSWORD environment variable\n"
+     printf >&2 \
+         "e.g. FLUXUSER_PASSWORD=xxzzyy docker run -e FLUXUSER_PASSWORD ..\n"
+   exit 1
+  fi
+  printf >&2 "Setting requested password for fluxuser..\n"
+  echo fluxuser:${FLUXUSER_PASSWORD} | chpasswd
+fi
+exec "$@"
diff --git a/src/test/docker/fluxorama/f.txt b/src/test/docker/fluxorama/f.txt
new file mode 100644
index 000000000000..b6a717bdd1ac
--- /dev/null
+++ b/src/test/docker/fluxorama/f.txt
@@ -0,0 +1,35 @@
+                                                                 
+                                       .'                        
+                                       o,    .........           
+                                      .k...','....,''''.         
+                .......               ll''''.     ,'''',         
+           .,:::;,''',,:::;.         ';'','       ..'''.         
+        .:o:.      .....  .:o;      .''''.                       
+       ,o,     ,:cc::;;::c:'.,o,   .'''''                        
+     .ll    .ll'.         .;o:.oc .'''',.  .,:::::::c:,.         
+    .d,   .l:.    ':::::::c;.,oco:''''',;cc;,.....   ..;cc'      
+   'x'   ;d.   .cl,       .,lc;lc,''''',;'',;,,;;:cc:'   .:o,    
+  .d'   ,d.   ,d'        ''''',,'''''',,,:ccccc;.   .;l:   .co.  
+  o:    o.   ;d.              .;'''''':lc;'....;cl,    ,o.   ,d. 
+ cO.   ::   'k.                '''''',c.          ;o.   .d'   ;d 
+;kk:  cO;  .xl                .'''''';.            .o:   .d.   d;
+ o:  .lk: .xOx.               '''''',.              .k.   cc   ;d
+ d:   ;d   :k,.              .'''''',                o:   :l   'x
+ oc   ,x   'k.               '''''',.                l:   :c   .x
+ :d   .k.   o:              .,''''',                .x'   o:   ,o
+ .k;   ;l   .d:            .'''''',.               .ok.  ,O,   o;
+  ;k.   c:    co'        .;l,''''':.              .xOk. ;kO'  'k'
+   :x'   :o.   .;cc:;;;:cld;''''''o:              ,k:.  cxo, ckO;
+    ,x:   .cl;.     ...,:ol,''''':cd:           .cx,   .o.   ;k;.
+      ;l;.   .,::::::::;:c;''''':od:,c:'.   ..,cl,    'o.   ,d.  
+        .:c:,'......',::,.''''',,x:cl. .;:cc:,.     'l:    co.   
+            .',;;;,,'.   .'',,,. .d:.:l;.       .'cl;.   'd:     
+                         '''''.    ;o' .,::ccccc:'.    .ll.      
+             ...        .'',,        ,::,.         .':c:.        
+           .'''''.     .'';l.           .;c::cccccc;'.           
+           .'''''.  ..''..ll                                     
+             ....'....  .lkl.                                    
+                        .kk:                                     
+                         c'                                      
+
+Welcome to Flux-o-rama! A Flux system instance test environment and playground.
diff --git a/src/test/docker/fluxorama/imp.toml b/src/test/docker/fluxorama/imp.toml
new file mode 100644
index 000000000000..194248cc451e
--- /dev/null
+++ b/src/test/docker/fluxorama/imp.toml
@@ -0,0 +1,3 @@
+[exec]
+allowed-users = [ "flux" ]
+allowed-shells = [ "/usr/libexec/flux/flux-shell" ]
diff --git a/src/test/docker/fluxorama/job-exec.toml b/src/test/docker/fluxorama/job-exec.toml
new file mode 100644
index 000000000000..093b451753b1
--- /dev/null
+++ b/src/test/docker/fluxorama/job-exec.toml
@@ -0,0 +1,2 @@
+[exec]
+imp = "/usr/libexec/flux/flux-imp"
diff --git a/src/test/docker/focal/Dockerfile b/src/test/docker/focal/Dockerfile
new file mode 100644
index 000000000000..31b884999518
--- /dev/null
+++ b/src/test/docker/focal/Dockerfile
@@ -0,0 +1,111 @@
+FROM ubuntu:focal
+
+LABEL maintainer="Stephen Herbein <herbein1@llnl.gov>"
+
+# avoid debconf from asking for input
+ENV DEBIAN_FRONTEND noninteractive
+
+# Update pkg caches, install latest pkg utils:
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        apt-utils \
+ && rm -rf /var/lib/apt/lists/*
+
+# Ubuntu's minimal image installs a bonus `man` binary, breaking some of our
+# tests (i.e., `flux help foo`).  Remove their wrapper and install the real
+# thing, without installing every man page under the sun (which the `unminimize`
+# command would do)
+RUN rm -f /usr/bin/man && dpkg-divert --quiet --remove --rename /usr/bin/man \
+    && apt-get update \
+    && apt-get -qq install -y --no-install-recommends man-db \
+    && rm -rf /var/lib/apt/lists/*
+
+# Utilities
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        locales \
+        ca-certificates \
+        wget \
+        man \
+        git \
+        sudo \
+        vim \
+        munge \
+        lcov \
+        ccache \
+        lua5.2 \
+        lua-posix \
+        valgrind \
+        jq \
+ && rm -rf /var/lib/apt/lists/*
+
+# Compilers, autotools
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        build-essential \
+        pkg-config \
+        autotools-dev \
+        libtool \
+        autoconf \
+        automake \
+        make \
+        cmake \
+        clang \
+        clang-tidy \
+ && rm -rf /var/lib/apt/lists/*
+
+# Python
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        python3-dev \
+        python3.8-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-wheel \
+        python3-cffi \
+        python3-six \
+        python3-yaml \
+        python3-jsonschema \
+ && rm -rf /var/lib/apt/lists/*
+
+# Sphinx packages for docs
+RUN python3 -m pip install sphinx sphinx-rtd-theme sphinxcontrib-spelling
+
+RUN mkdir -p /usr/lib/python3.8/dist-packages \
+ && echo ../site-packages >/usr/lib/python3.8/dist-packages/site-packages.pth
+
+# Other deps
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        libsodium-dev \
+        libzmq3-dev \
+        libjansson-dev \
+        libmunge-dev \
+        libncurses-dev \
+        liblua5.2-dev \
+        lua-posix-dev \
+        liblz4-dev \
+        libsqlite3-dev \
+        uuid-dev \
+        libhwloc-dev \
+        libopenmpi-dev \
+        libs3-dev \
+        libarchive-dev \
+        libpam-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+# Testing utils and libs
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        faketime \
+        libfaketime \
+        pylint \
+        cppcheck \
+        enchant \
+        aspell \
+        aspell-en \
+        time \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN locale-gen en_US.UTF-8
+ENV LANG=C.UTF-8
diff --git a/src/test/docker/jammy/Dockerfile b/src/test/docker/jammy/Dockerfile
new file mode 100644
index 000000000000..ef703a30b99d
--- /dev/null
+++ b/src/test/docker/jammy/Dockerfile
@@ -0,0 +1,146 @@
+FROM ubuntu:jammy
+
+LABEL maintainer="Tom Scogland <scogland1@llnl.gov>"
+
+# Update pkg caches, install latest pkg utils:
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        apt-utils \
+ && rm -rf /var/lib/apt/lists/*
+
+# Utilities
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        locales \
+        ca-certificates \
+        wget \
+        man \
+        git \
+        flex \
+        ssh \
+        sudo \
+        vim \
+        luarocks \
+        munge \
+        lcov \
+        ccache \
+        lua5.2 \
+        mpich \
+        valgrind \
+        jq \
+ && rm -rf /var/lib/apt/lists/*
+
+# Ubuntu containers now "minimize" themselves, so manpages aren't installed.
+# To warn people about this, /usr/bin/man is a shell script that doesn't bother
+# to look for the man page, but just prints a warning.  Link /usr/bin/man.REAL
+# to /usr/bin/man to make this behave
+RUN ln -sf /usr/bin/man.REAL /usr/bin/man
+
+# Compilers, autotools
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        build-essential \
+        pkg-config \
+        autotools-dev \
+        libtool \
+        autoconf \
+        automake \
+        make \
+        cmake \
+        clang-15 \
+        clang-tools-15 \
+        gcc-12 \
+        g++-12 \
+ && rm -rf /var/lib/apt/lists/*
+
+# Python
+# NOTE: sudo pip install is necessary to get differentiated installations of
+# python binary components for multiple python3 variants, --ignore-installed
+# makes it ignore local versions of the packages if your home directory is
+# mapped into the container and contains the same libraries
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+	libffi-dev \
+        python3-dev \
+        python3.11-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-wheel \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN for PY in python3.10 python3.11 ; do \
+        sudo $PY -m pip install --upgrade --ignore-installed \
+	    "markupsafe==2.0.0" \
+            coverage cffi ply six pyyaml "jsonschema>=2.6,<4.0" \
+            sphinx sphinx-rtd-theme sphinxcontrib-spelling; \
+	sudo mkdir -p /usr/lib/${PY}/dist-packages; \
+	echo ../site-packages >/tmp/site-packages.pth; \
+	sudo mv /tmp/site-packages.pth /usr/lib/${PY}/dist-packages; \
+    done ; \
+    apt-get -qq purge -y python3-pip \
+ && apt-get -qq autoremove -y
+
+# Other deps
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        libsodium-dev \
+        libzmq3-dev \
+        libjansson-dev \
+        libmunge-dev \
+        libncursesw5-dev \
+        liblua5.2-dev \
+        liblz4-dev \
+        libsqlite3-dev \
+        uuid-dev \
+        libhwloc-dev \
+        libmpich-dev \
+        libs3-dev \
+        libevent-dev \
+        libarchive-dev \
+        libpam-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+# Testing utils and libs
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        faketime \
+        libfaketime \
+        pylint \
+        cppcheck \
+        enchant-2 \
+        aspell \
+        aspell-en \
+        time \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN locale-gen en_US.UTF-8
+
+# NOTE: luaposix installed by rocks due to Ubuntu bug: #1752082 https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
+RUN luarocks install luaposix
+
+# Install catch by hand for now:
+COPY scripts/fetch-and-build-catch.sh /fetch-and-build-catch.sh
+RUN /fetch-and-build-catch.sh
+
+
+# Install openpmix, prrte
+RUN mkdir prrte \
+ && cd prrte \
+ && git clone https://github.com/openpmix/openpmix.git \
+ && git clone https://github.com/openpmix/prrte.git \
+ && ls -l \
+ && set -x \
+ && cd openpmix \
+ && git checkout fefaed568f33bf86f28afb6e45237f1ec5e4de93 \
+ && ./autogen.pl \
+ && ./configure --prefix=/usr --disable-static && make -j 4 install \
+ && ldconfig \
+ && cd .. \
+ && cd prrte \
+ && git checkout 477894f4720d822b15cab56eee7665107832921c \
+ && ./autogen.pl \
+ && ./configure --prefix=/usr && make -j 4 install \
+ && cd ../.. \
+ && rm -rf prrte
+
+ENV LANG=C.UTF-8
diff --git a/src/test/docker/noble/Dockerfile b/src/test/docker/noble/Dockerfile
new file mode 100644
index 000000000000..dcf616fac866
--- /dev/null
+++ b/src/test/docker/noble/Dockerfile
@@ -0,0 +1,121 @@
+FROM ubuntu:noble
+
+LABEL maintainer="Tom Scogland <scogland1@llnl.gov>"
+
+# copy in requirements files
+COPY scripts/requirements-ci.txt /requirements-ci.txt
+COPY scripts/requirements-doc.txt /requirements-doc.txt
+
+# Utilities
+RUN apt-get update \
+# install latest pkg utils:
+ && apt-get -qq install -y --no-install-recommends \
+        apt-utils \
+ && apt-get -qq install -y --no-install-recommends \
+        locales \
+        ca-certificates \
+        wget \
+        man \
+        git \
+        flex \
+        ssh \
+        sudo \
+        vim \
+        luarocks \
+        munge \
+        lcov \
+        ccache \
+        lua5.2 \
+        lua5.2-posix-dev \
+        valgrind \
+        jq \
+# Compilers, autotools
+        build-essential \
+        pkg-config \
+        autotools-dev \
+        libtool \
+        autoconf \
+        automake \
+        make \
+        cmake \
+        ninja-build \
+        clang-18 \
+        clang-tools-18 \
+# default version
+        gcc-13 \
+        g++-13 \
+# newest
+        gcc-14 \
+        g++-14 \
+# Python
+        libffi-dev \
+## python 3.12
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-wheel \
+# Other deps
+        libsodium-dev \
+        libzmq3-dev \
+        libjansson-dev \
+        libmunge-dev \
+        libncursesw5-dev \
+        liblua5.2-dev \
+        liblz4-dev \
+        libsqlite3-dev \
+        uuid-dev \
+        libhwloc-dev \
+        libmpich-dev \
+        libs3-dev \
+        libevent-dev \
+        libarchive-dev \
+        libpam-dev \
+        libpmix-dev \
+# testing utils and libs
+        faketime \
+        libfaketime \
+        pylint \
+        cppcheck \
+        enchant-2 \
+        catch2 \
+        aspell \
+        aspell-en \
+        time \
+# Testing utils and libs
+# NOTE: sudo pip install is necessary to get differentiated installations of
+# python binary components for multiple python3 variants, --ignore-installed
+# makes it ignore local versions of the packages if your home directory is
+# mapped into the container and contains the same libraries
+ && (for PY in python3.12 ; do \
+        sudo $PY -m pip install --upgrade --ignore-installed --break-system-packages -r /requirements-ci.txt ; \
+        sudo mkdir -p /usr/lib/${PY}/dist-packages; \
+        echo ../site-packages >/tmp/site-packages.pth; \
+        sudo mv /tmp/site-packages.pth /usr/lib/${PY}/dist-packages; \
+    done) \
+ && apt-get -qq purge -y python3-pip \
+ && apt-get -qq autoremove -y \
+ && rm -rf /var/lib/apt/lists/*
+
+# Ubuntu containers now "minimize" themselves, so manpages aren't installed.
+# To warn people about this, /usr/bin/man is a shell script that doesn't bother
+# to look for the man page, but just prints a warning.  Link /usr/bin/man.REAL
+# to /usr/bin/man to make this behave
+RUN ln -sf /usr/bin/man.REAL /usr/bin/man
+
+ENV LANG=C.UTF-8
+
+# the image has a UID 1000 named ubuntu now for some reason, fix it
+RUN userdel ubuntu
+# noble defaults to an invalid TZ /UTC
+ENV TZ="America/Los_Angeles"
+
+# noble currently packages an MPICH linked with PMIX, so we can't bootstrap it
+# natively, nor can its own mpiexec, see here:
+# https://bugs.launchpad.net/ubuntu/+source/mpich/+bug/2072338
+# so we build our own copy of current stable
+COPY scripts/fetch-and-build-mpich.sh /fetch-and-build-mpich.sh
+RUN /fetch-and-build-mpich.sh
+
+# hwloc tries to look for opengl devices  by connecting to a port that might
+# sometimes be an x11 port, but more often for us is munge, turn it off
+ENV HWLOC_COMPONENTS=-gl
diff --git a/src/test/docker/poison-libflux.sh.in b/src/test/docker/poison-libflux.sh.in
new file mode 100644
index 000000000000..08466acb8195
--- /dev/null
+++ b/src/test/docker/poison-libflux.sh.in
@@ -0,0 +1,107 @@
+#!/bin/bash
+#
+#  Build a "poison" libflux-core.so to install in system path.
+#
+#  Also put some poison commands in a fake FLUX_EXEC_PATH given on
+#   commandline.
+#
+#  This will hopefully ensure no flux-core internal tests try to
+#   load the system libflux-core.so
+#
+
+CMDDIR=${1:-/tmp/poison-cmds}
+WORKDIR=$(mktemp -d)
+printf " Changing working directory to $WORKDIR\n"
+cd $WORKDIR || exit 1
+
+cleanup() {
+    rm -rf $WORKDIR
+    printf " Cleaning up...\n"
+}
+trap cleanup EXIT
+
+LIBDIR=@X_LIBDIR@
+BINDIR=@X_BINDIR@
+
+#  Install poison flux(1) executable in PATH
+#
+printf " Installing poison flux binary to ${BINDIR}/flux\n"
+cat >flux.sh <<EOF
+#!/bin/sh
+echo "Error: system flux executable run during test!" >&2
+EOF
+install -m 0755 flux.sh ${BINDIR}/flux
+
+
+#  Create poison dso:
+cat >poison.c <<EOF
+#include <stdio.h>
+#include <stdlib.h>
+
+void __attribute__((constructor)) poison (void)
+{
+    fprintf (stderr, "error: program loaded system %s\n", LIBNAME);
+    exit (1);
+}
+EOF
+
+CORE_VERSION=@LIBFLUX_CORE_VERSION_INFO@
+IDSET_VERSION=@LIBFLUX_IDSET_VERSION_INFO@
+HOSTLIST_VERSION=@LIBFLUX_HOSTLIST_VERSION_INFO@
+SCHEDUTIL_VERSION=@LIBFLUX_SCHEDUTIL_VERSION_INFO@
+OPTPARSE_VERSION=@LIBFLUX_OPTPARSE_VERSION_INFO@
+
+get_current() {
+    name=${1^^}_VERSION
+    echo ${!name} | cut -d: -f1
+}
+get_revision() {
+    name=${1^^}_VERSION
+    echo ${!name} | cut -d: -f2
+}
+get_age() {
+    name=${1^^}_VERSION
+    echo ${!name} | cut -d: -f3
+}
+
+printf " Installing poison flux libs in ${LIBDIR}\n"
+for lib in core idset optparse schedutil hostlist; do
+
+    #  Compile:
+    printf " BUILD    poison libflux-${lib}.so\n"
+    cc -D LIBNAME=\"libflux-${lib}\" -fPIC -shared -o libflux-${lib}.so poison.c
+
+    #  Install:
+    printf " INSTALL  poison libflux-${lib}.so to ${LIBDIR}\n"
+    install -m 0755 libflux-${lib}.so ${LIBDIR}
+
+    #  Create version links
+    #  N.B. This is very linux specific -- we may need a different way to
+    #   do this in the future.
+    #
+    current=$(get_current $lib)
+    revision=$(get_revision $lib)
+    age=$(get_age $lib)
+    major=$((current - age))
+    suffix=$major.$age.$revision
+
+    printf " LINK     ${LIBDIR}/libflux-${lib}.so.${major}\n"
+    ln -sf ${LIBDIR}/libflux-${lib}.so ${LIBDIR}/libflux-${lib}.so.${suffix}
+    ln -sf ${LIBDIR}/libflux-${lib}.so ${LIBDIR}/libflux-${lib}.so.${major}
+
+done
+
+printf " Installing poison flux commands...\n"
+mkdir -p -m 0755 $CMDDIR
+for cmd in \
+	terminus ping keygen logger event module kvs start job queue exec \
+	cron mini jobs resource admin jobtap job-validator \
+	job-exec-override uri pstree; do
+	cat <<-EOF >${CMDDIR}/flux-${cmd}
+	#!/bin/sh
+	printf "Error: running poison command $0\n"
+	exit 1
+	EOF
+	chmod +x ${CMDDIR}/flux-${cmd}
+done
+
diff --git a/src/test/docker/travis/Dockerfile b/src/test/docker/travis/Dockerfile
deleted file mode 100644
index a92e2d0402eb..000000000000
--- a/src/test/docker/travis/Dockerfile
+++ /dev/null
@@ -1,70 +0,0 @@
-ARG IMAGESRC
-
-FROM $IMAGESRC
-
-# Allow flux-security version, username, UID, and GID to be overidden on
-#  docker build command line:
-#
-ARG USER=flux
-ARG UID=1000
-ARG GID=1000
-ARG FLUX_SECURITY_VERSION
-ARG BASE_IMAGE
-
-# Install flux-security by hand for now:
-#
-RUN CCACHE_DISABLE=1 \
- && V=$FLUX_SECURITY_VERSION \
- && PKG=flux-security-$V \
- && URL=https://github.com/flux-framework/flux-security/releases/download \
- && wget ${URL}/v${V}/${PKG}.tar.gz \
- && tar xvfz ${PKG}.tar.gz \
- && cd ${PKG} \
- && ./configure --prefix=/usr || cat config.log \
- && make -j 4 \
- && make install \
- && cd .. \
- && rm -rf flux-security-*
-
-
-# Add configured user to image with sudo access:
-#
-RUN set -x && groupadd -g $UID $USER \
- && useradd -g $USER -u $UID -d /home/$USER -m $USER \
- && printf "$USER ALL= NOPASSWD: ALL\\n" >> /etc/sudoers
-
-# Also add "flux" user to image with sudo access:
-#
-RUN set -x && groupadd flux \
- && useradd -g flux -d /home/flux -m flux \
- && printf "flux ALL= NOPASSWD: ALL\\n" >> /etc/sudoers
-
-# Make sure user in appropriate group for sudo on different platorms
-RUN case $BASE_IMAGE in \
-     bionic*) adduser $USER sudo && adduser flux sudo ;; \
-     centos*) usermod -G wheel $USER && usermod -G wheel flux ;; \
-     *) (>&2 echo "Unknown BASE_IMAGE") ;; \
-    esac
-
-# Install extra dependencies if necessary here.
-#
-# Do not forget to run `apt update` on Ubuntu/bionic.
-# Do NOT run `yum upgrade` on CentOS (this will unnecessarily upgrade
-#  existing packages)
-#
-RUN case $BASE_IMAGE in \
-     bionic*) ;; \
-     centos*) ;; \
-     *) (>&2 echo "Unknown BASE_IMAGE") ;; \
-    esac
-
-# Setup MUNGE directories & key
-RUN mkdir -p /var/run/munge \
- && dd if=/dev/urandom bs=1 count=1024 > /etc/munge/munge.key \
- && chown -R munge /etc/munge/munge.key /var/run/munge \
- && chmod 600 /etc/munge/munge.key
-
-
-ENV BASE_IMAGE=$BASE_IMAGE
-USER $USER
-WORKDIR /home/$USER
diff --git a/src/test/generate-matrix.py b/src/test/generate-matrix.py
new file mode 100755
index 000000000000..8cd084219643
--- /dev/null
+++ b/src/test/generate-matrix.py
@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+#
+#  Generate a build matrix for use with github workflows
+#
+
+from copy import deepcopy
+import json
+import os
+import re
+
+docker_run_checks = "src/test/docker/docker-run-checks.sh"
+
+default_args = (
+    "--prefix=/usr"
+    " --sysconfdir=/etc"
+    " --with-systemdsystemunitdir=/etc/systemd/system"
+    " --localstatedir=/var"
+    " --with-flux-security"
+)
+
+DOCKER_REPO = "fluxrm/flux-core"
+
+
+def on_master_or_tag(matrix):
+    return matrix.branch == "master" or matrix.tag
+
+
+DEFAULT_MULTIARCH_PLATFORMS = {
+    "linux/arm64": {
+        "when": on_master_or_tag,
+        "suffix": " - arm64",
+        "command_args": "--install-only ",
+        "timeout_minutes": 90,
+    },
+    "linux/amd64": {"when": lambda _: True},
+}
+
+
+class BuildMatrix:
+    def __init__(self):
+        self.matrix = []
+        self.branch = None
+        self.tag = None
+
+        #  Set self.branch or self.tag based on GITHUB_REF
+        if "GITHUB_REF" in os.environ:
+            self.ref = os.environ["GITHUB_REF"]
+            match = re.search("^refs/heads/(.*)", self.ref)
+            if match:
+                self.branch = match.group(1)
+            match = re.search("^refs/tags/(.*)", self.ref)
+            if match:
+                self.tag = match.group(1)
+
+    def create_docker_tag(self, image, env, command, platform):
+        """Create docker tag string if this is master branch or a tag"""
+        if self.branch == "master" or self.tag:
+            tag = f"{DOCKER_REPO}:{image}"
+            if self.tag:
+                tag += f"-{self.tag}"
+            if platform is not None:
+                tag += "-" + platform.split("/")[1]
+            env["DOCKER_TAG"] = tag
+            command += f" --tag={tag}"
+            return True, command
+
+        return False, command
+
+    def env_add_s3(self, args, env):
+        """Add necessary environment and args to test content-s3 module"""
+        env.update(
+            dict(
+                S3_ACCESS_KEY_ID="minioadmin",
+                S3_SECRET_ACCESS_KEY="minioadmin",
+                S3_HOSTNAME="127.0.0.1:9000",
+                S3_BUCKET="flux-minio",
+            )
+        )
+        args += " --enable-content-s3"
+        return args
+
+    def add_build(
+        self,
+        name=None,
+        image=None,
+        args=default_args,
+        jobs=6,
+        env=None,
+        docker_tag=False,
+        test_s3=False,
+        coverage=False,
+        coverage_flags=None,
+        recheck=True,
+        platform=None,
+        command_args="",
+        timeout_minutes=60,
+    ):
+        """Add a build to the matrix.include array"""
+
+        # Extra environment to add to this command:
+        # NOTE: ensure we copy the dict rather than modify, re-used dicts can cause
+        #       overwriting
+        env = dict(env) if env is not None else {}
+
+        # hwloc tries to look for opengl devices  by connecting to a port that might
+        # sometimes be an x11 port, but more often for us is munge, turn it off
+        env["HWLOC_COMPONENTS"] = "-gl"
+        # the psm3 connector added to libfabrics in ~1.12 causes errors when allowed to
+        # use non-local connectors on a system with virtual NICs, since we're in a
+        # docker container, prevent this
+        env["PSM3_HAL"] = "loopback"
+
+        needs_buildx = False
+        if platform:
+            command_args += f"--platform={platform}"
+            needs_buildx = True
+
+        # The command to run:
+        command = f"{docker_run_checks} -j{jobs} --image={image} {command_args}"
+
+        # Add --recheck option if requested
+        if recheck and "DISTCHECK" not in env:
+            command += " --recheck"
+
+        if docker_tag:
+            #  Only export docker_tag if this is main branch or a tag:
+            docker_tag, command = self.create_docker_tag(image, env, command, platform)
+
+        if test_s3:
+            args = self.env_add_s3(args, env)
+
+        if coverage:
+            env["COVERAGE"] = "t"
+
+        create_release = False
+        if self.tag and "DISTCHECK" in env:
+            create_release = True
+
+        command += f" -- --enable-docs {args}"
+
+        self.matrix.append(
+            {
+                "name": name,
+                "env": env,
+                "command": command,
+                "image": image,
+                "tag": self.tag,
+                "branch": self.branch,
+                "coverage": coverage,
+                "coverage_flags": coverage_flags,
+                "test_s3": test_s3,
+                "docker_tag": docker_tag,
+                "needs_buildx": needs_buildx,
+                "create_release": create_release,
+                "timeout_minutes": timeout_minutes,
+            }
+        )
+
+    def add_multiarch_build(
+        self,
+        name: str,
+        platforms=DEFAULT_MULTIARCH_PLATFORMS,
+        default_suffix="",
+        image=None,
+        docker_tag=True,
+        **kwargs,
+    ):
+        for p, args in platforms.items():
+            if args["when"](self):
+                suffix = args.get("suffix", default_suffix)
+                self.add_build(
+                    name + suffix,
+                    platform=p,
+                    docker_tag=docker_tag,
+                    image=image if image is not None else name,
+                    command_args=args.get("command_args", ""),
+                    timeout_minutes=args.get("timeout_minutes", 30),
+                    **kwargs,
+                )
+
+    def __str__(self):
+        """Return compact JSON representation of matrix"""
+        return json.dumps(
+            {"include": self.matrix}, skipkeys=True, separators=(",", ":")
+        )
+
+
+matrix = BuildMatrix()
+
+# Multi-arch builds, arm only builds on
+bookworm_platforms = deepcopy(DEFAULT_MULTIARCH_PLATFORMS)
+bookworm_platforms["linux/386"] = {"when": lambda _: True, "suffix": " - 32 bit"}
+common_args = (
+    "--prefix=/usr"
+    " --sysconfdir=/etc"
+    " --with-systemdsystemunitdir=/etc/systemd/system"
+    " --localstatedir=/var"
+    " --with-flux-security"
+)
+matrix.add_multiarch_build(
+    name="bookworm",
+    default_suffix=" - test-install",
+    platforms=bookworm_platforms,
+    args=common_args,
+    env=dict(
+        TEST_INSTALL="t",
+    ),
+)
+
+matrix.add_multiarch_build(
+    name="noble",
+    default_suffix=" - test-install",
+    args=common_args,
+    env=dict(
+        TEST_INSTALL="t",
+    ),
+)
+matrix.add_multiarch_build(
+    name="el9",
+    default_suffix=" - test-install",
+    args=common_args,
+    env=dict(
+        TEST_INSTALL="t",
+    ),
+)
+matrix.add_multiarch_build(
+    name="alpine",
+    default_suffix=" - test-install",
+    args=common_args,
+    env=dict(
+        TEST_INSTALL="t",
+    ),
+)
+# single arch builds that still produce a container
+matrix.add_build(
+    name="fedora40 - test-install",
+    image="fedora40",
+    args=common_args,
+    env=dict(
+        TEST_INSTALL="t",
+    ),
+    docker_tag=True,
+)
+
+# Ubuntu: TEST_INSTALL
+matrix.add_build(
+    name="jammy - test-install",
+    image="jammy",
+    env=dict(
+        TEST_INSTALL="t",
+    ),
+    args="--with-flux-security",
+    docker_tag=True,
+)
+
+# Ubuntu 20.04: py3.8, deprecated
+matrix.add_build(
+    name="focal - py3.8",
+    image="focal",
+    env=dict(PYTHON_VERSION="3.8"),
+    docker_tag=True,
+)
+
+
+# Debian: gcc-12, content-s3, distcheck
+matrix.add_build(
+    name="bookworm - gcc-12,content-s3,distcheck",
+    image="bookworm",
+    env=dict(
+        CC="gcc-12",
+        CXX="g++12",
+        DISTCHECK="t",
+    ),
+    args="--with-flux-security",
+    test_s3=True,
+)
+
+# fedora40: clang-18
+matrix.add_build(
+    name="fedora40 - clang-18",
+    image="fedora40",
+    env=dict(
+        CC="clang-18",
+        CXX="clang++-18",
+        CFLAGS="-O2 -gdwarf-4",
+        chain_lint="t",
+    ),
+    args="--with-flux-security",
+    command_args="--workdir=/usr/src/" + "workdir/" * 15,
+)
+
+# coverage
+matrix.add_build(
+    name="coverage",
+    image="bookworm",
+    coverage_flags="ci-basic",
+    coverage=True,
+    args="--with-flux-security",
+)
+
+# RHEL8 clone
+matrix.add_build(
+    name="el8 - ascii",
+    image="el8",
+    env=dict(PYTHON_VERSION="3.6", LDFLAGS="-Wl,-z,relro  -Wl,-z,now"),
+    args="--enable-broken-locale-mode",
+)
+
+# el8 - test install
+matrix.add_build(
+    name="el8 - test-install",
+    image="el8",
+    env=dict(
+        TEST_INSTALL="t",
+    ),
+    args="--with-flux-security",
+    docker_tag=True,
+)
+
+# RHEL8 clone, system, coverage
+matrix.add_build(
+    name="el8 - system,coverage",
+    coverage_flags="ci-system",
+    image="el8",
+    coverage=True,
+    command_args="--system",
+    args="--with-flux-security",
+)
+
+# inception
+matrix.add_build(
+    name="inception",
+    image="fedora40",
+    command_args="--inception",
+)
+
+print(matrix)
diff --git a/src/test/scaling/instancebench.py b/src/test/scaling/instancebench.py
new file mode 100755
index 000000000000..6ad1dedeaba7
--- /dev/null
+++ b/src/test/scaling/instancebench.py
@@ -0,0 +1,360 @@
+#!/usr/bin/env python3
+##############################################################
+# Copyright 2024 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import argparse
+import itertools
+import math
+import os
+import sys
+import time
+
+import flux
+import flux.uri
+from flux.job import cancel_async
+from flux.resource import resource_list
+
+
+class InstanceBench:
+    """Class representing a single Flux instance bootstrap benchmark"""
+
+    def __init__(
+        self,
+        flux_handle,
+        nnodes,
+        brokers_per_node=1,
+        topo="kary:2",
+        conf=None,
+        progress=None,
+        exclusive=True,
+    ):
+
+        self.flux_handle = flux_handle
+        self.nnodes = nnodes
+        self.brokers_per_node = brokers_per_node
+        self.topo = topo
+        self.id = None
+        self.t0 = None
+        self.t_submit = None
+        self.t_start = None
+        self.t_uri = None
+        self.t_shell_init = None
+        self.t_ready = None
+        self.t_finish = None
+        self.child_handle = None
+        self.then_cb = None
+        self.then_args = []
+        self.then_kw_args = {}
+        self.progress = progress
+        self.size = nnodes * brokers_per_node
+        self.topo = topo
+        self.name = f"[N:{nnodes:<4d} SZ:{self.size:<4d} {self.topo:<8}]"
+
+        broker_opts = ["-Sbroker.rc2_none=1"]
+        if topo is not None:
+            broker_opts.append(f"-Stbon.topo={topo}")
+        if conf is not None:
+            broker_opts.append("-c{{tmpdir}}/conf.json")
+
+        jobspec = flux.job.JobspecV1.from_command(
+            command=["flux", "broker", *broker_opts],
+            exclusive=exclusive,
+            num_nodes=nnodes,
+            num_tasks=nnodes * brokers_per_node,
+        )
+        jobspec.setattr_shell_option("mpi", "none")
+        if conf is not None:
+            jobspec.add_file("conf.json", conf)
+        self.jobspec = jobspec
+
+    def log(self, msg):
+        try:
+            ts = self.ts or (time.time() - self.t0)
+        except AttributeError:
+            ts = 0.0
+        print(f"{self.name}: {ts:6.3f}s: {msg}", file=sys.stderr, flush=True)
+        self.ts = None
+
+    def then(self, cb, *args, **kw_args):
+        self.then_cb = cb
+        self.then_args = args
+        self.then_kw_args = kw_args
+
+    def submit(self):
+        self.t0 = time.time()
+        flux.job.submit_async(self.flux_handle, self.jobspec).then(self.submit_cb)
+        return self
+
+    def submit_cb(self, future):
+        try:
+            self.id = future.get_id()
+        except OSError as exc:
+            print(exc, file=sys.stderr)
+            return
+        if self.progress:
+            job = flux.job.JobInfo(
+                {
+                    "id": self.id,
+                    "state": flux.constants.FLUX_JOB_STATE_SCHED,
+                    "t_submit": time.time(),
+                }
+            )
+            self.progress.add_job(job)
+        flux.job.event_watch_async(self.flux_handle, self.id).then(self.bg_wait_cb)
+
+    def child_ready_cb(self, future):
+        future.get()
+        self.t_ready = time.time()
+        self.log("ready")
+
+        self.size = self.child_handle.attr_get("size")
+        self.topo = self.child_handle.attr_get("tbon.topo")
+
+        #  Shutdown and report timing:
+        self.log("requesting shutdown")
+        self.child_handle.rpc("shutdown.start", {"loglevel": 1})
+
+    def bg_wait_cb(self, future):
+        event = future.get_event()
+        if self.progress:
+            self.progress.process_event(self.id, event)
+        if not event:
+            #  The job has unexpectedly exited since we're at the end
+            #   of the eventlog. Run `flux job attach` since this will dump
+            #   any errors or output, then raise an exception.
+            os.system(f"flux job attach {self.id} >&2")
+            raise OSError(f"{self.id}: unexpectedly exited")
+
+        self.ts = event.timestamp - self.t0
+        # self.log(f"{event.name}")
+        if event.name == "submit":
+            self.t_submit = event.timestamp
+        elif event.name == "alloc":
+            self.t_alloc = event.timestamp
+        elif event.name == "start":
+            self.t_start = event.timestamp
+            flux.job.event_watch_async(
+                self.flux_handle, self.id, eventlog="guest.exec.eventlog"
+            ).then(self.shell_init_wait_cb)
+        elif event.name == "memo" and "uri" in event.context:
+            self.t_uri = event.timestamp
+            uri = str(flux.uri.JobURI(event.context["uri"]))
+            self.log(f"opening handle to {self.id}")
+            self.child_handle = flux.Flux(uri)
+
+            #  Set main handle reactor as reactor for his child handle so
+            #  events can be processed:
+            self.child_handle.flux_set_reactor(self.flux_handle.get_reactor())
+
+            self.log("connected to child job")
+
+            #  Wait for child instance to be ready:
+            self.child_handle.rpc("state-machine.wait").then(self.child_ready_cb)
+
+        elif event.name == "finish":
+            self.t_finish = event.timestamp
+            future.cancel(stop=True)
+            if self.then_cb is not None:
+                self.then_cb(self, *self.then_args, **self.then_kw_wargs)
+            if self.progress:
+                # Notify ProgressBar that this job is done via a None event
+                self.progress.process_event(self.id, None)
+
+    def shell_init_wait_cb(self, future):
+        event = future.get_event()
+        if not event:
+            return
+        self.ts = event.timestamp - self.t0
+        self.log(f"exec.{event.name}")
+        if event.name == "shell.init":
+            self.t_shell_init = event.timestamp
+            future.cancel(stop=True)
+
+    def timing_header(self, file=sys.stdout):
+        print(
+            "%5s %5s %8s %8s %8s %8s %8s %8s %8s"
+            % (
+                "NODES",
+                "SIZE",
+                "TOPO",
+                "T_START",
+                "T_URI",
+                "T_INIT",
+                "T_READY",
+                "(TOTAL)",
+                "T_SHUTDN",
+            ),
+            file=file,
+        )
+
+    def report_timing(self, file=sys.stdout):
+        # Only report instances that got to the ready state
+        if not self.t_ready:
+            return
+
+        # Avoid error if t_shutdown is None
+        if not self.t_finish:
+            t_shutdown = "       -"
+        else:
+            t_shutdown = f"{self.t_finish - self.t_ready:8.3f}"
+
+        print(
+            "%5s %5s %8s %8.3f %8.3f %8.3f %8.3f %8.3f %s"
+            % (
+                self.nnodes,
+                self.size,
+                self.topo,
+                self.t_start - self.t_alloc,
+                self.t_uri - self.t_start,
+                self.t_shell_init - self.t_alloc,
+                self.t_ready - self.t_shell_init,
+                self.t_ready - self.t_alloc,
+                t_shutdown,
+            ),
+            file=file,
+        )
+
+
+def generate_values(end):
+    """
+    Generate a list of powers of 2 (including `1` by default), up to and
+    including `end`. If `end` is not a power of 2 insert it as the last
+    element in list to ensure it is present.
+    The list is returned in reverse order (largest values first)
+    """
+    stop = int(math.log2(end)) + 1
+    values = [1 << i for i in range(stop)]
+    if end not in values:
+        values.append(end)
+    values.reverse()
+    return values
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        prog="instance-timing", formatter_class=flux.util.help_formatter()
+    )
+    parser.add_argument(
+        "-N",
+        "--max-nodes",
+        metavar="N",
+        type=int,
+        default=None,
+        help="Scale up to N nodes by powers of two",
+    )
+    parser.add_argument(
+        "-B",
+        "--max-brokers-per-node",
+        type=int,
+        metavar="N",
+        default=1,
+        help="Run powers of 2 brokers-per-node up to N",
+    )
+    parser.add_argument(
+        "--topo",
+        metavar="TOPO,...",
+        type=str,
+        default="kary:2",
+        help="add one or more tbon.topo values to test",
+    )
+    parser.add_argument(
+        "--non-exclusive",
+        action="store_true",
+        help="Do not set exclusive flag on submitted jobs",
+    )
+    parser.add_argument(
+        "-L",
+        "--log-file",
+        metavar="FILE",
+        help="log results to FILE in addition to stdout",
+    )
+    return parser.parse_args()
+
+
+def get_max_nnodes(flux_handle):
+    """
+    Get the maximum nodes available in the default queue or anonymous
+    queue if there are no queues configured.
+    """
+    resources = resource_list(flux.Flux()).get()
+    try:
+        config = flux_handle.rpc("config.get").get()
+        defaultq = config["policy"]["jobspec"]["defaults"]["system"]["queue"]
+        constraint = config["queues"][defaultq]["requires"]
+        avail = resources["up"].copy_constraint({"properties": constraint})
+    except KeyError:
+        avail = resources["up"]
+    return avail.nnodes
+
+
+def print_results(instances, ofile=sys.stdout):
+    instances[0].timing_header(ofile)
+    for ib in instances:
+        ib.report_timing(ofile)
+
+
+def try_cancel_all(handle, instances):
+    n = 0
+    for ib in instances:
+        if not ib.t_finish:
+            n += 1
+            ib.log("canceling job")
+            cancel_async(handle, ib.id)
+    print(f"canceled {n} jobs", file=sys.stderr)
+
+
+def main():
+    args = parse_args()
+    args.topo = args.topo.split(",")
+    exclusive = not args.non_exclusive
+
+    h = flux.Flux()
+    if not args.max_nodes:
+        args.max_nodes = get_max_nnodes(h)
+
+    nnodes = generate_values(args.max_nodes)
+    bpn = generate_values(args.max_brokers_per_node)
+
+    inputs = list(itertools.product(nnodes, bpn, args.topo))
+    inputs.sort(key=lambda x: x[0] * x[1])
+    inputs.reverse()
+
+    progress = flux.job.watcher.JobProgressBar(h)
+    progress.start()
+    instances = []
+    for i in inputs:
+        instances.append(
+            InstanceBench(
+                h,
+                i[0],
+                brokers_per_node=i[1],
+                topo=i[2],
+                progress=progress,
+                exclusive=exclusive,
+            ).submit()
+        )
+
+    try:
+        h.reactor_run()
+    except (KeyboardInterrupt, Exception):
+        #  Cancel all remaining jobs and print available results instead
+        #  of just exiting on exception
+        try_cancel_all(h, instances)
+
+    print_results(instances)
+    if args.log_file:
+        with open(args.log_file, "w") as ofile:
+            print_results(instances, ofile=ofile)
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/test/scaling/mpi-scale.sh b/src/test/scaling/mpi-scale.sh
new file mode 100755
index 000000000000..3ea95c3ecaf6
--- /dev/null
+++ b/src/test/scaling/mpi-scale.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+##############################################################
+# Copyright 2024 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+NNODES=$(flux resource list -no {nnodes})
+NCORES=$(flux resource list -no {ncores})
+CPN=$((${NCORES}/${NNODES}))
+
+printf "MPI scale testing on ${LCSCHEDCLUSTER:-$(hostname)}\n"
+printf "TIME: $(date -Is)\n"
+printf "INFO: $(flux resource info)\n"
+printf "TOPO: $(flux getattr tbon.topo)\n"
+printf "\n"
+printf "%6s %8s %14s %14s %14s %14s\n" NODES NTASKS INIT BARRIER FINALIZE TOTAL
+
+seq2()
+{
+    local start=$1
+    local end=$2
+    local printend=1
+
+    while [[ $start -lt $end ]]; do
+        printf "$start\n"
+        [[ $start = $end ]] && printend=0
+        ((start*=2))
+    done
+    [[ $printend = 1 ]] && printf "$end\n"
+}
+
+flux bulksubmit --watch --progress --quiet --nodes={0} --tasks-per-node={1} \
+    --exclusive \
+    --env=FLUX_MPI_TEST_TIMING=t \
+    ./t/mpi/hello \
+    ::: $(seq2 1 ${NNODES}) \
+    ::: $(seq2 1 ${CPN})
+
+# vi: ts=4 sw=4 expandtab
diff --git a/src/test/sched-bench.sh b/src/test/sched-bench.sh
deleted file mode 100755
index 49e60cc343d3..000000000000
--- a/src/test/sched-bench.sh
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/bin/bash
-#
-# Run a set of jobs (with or without exec subsystem loaded) and
-#  print some timing stats gathered from job eventlogs.
-#
-declare prog=$(basename $0)
-
-declare NNODES=8
-declare CPN=32
-
-declare -r long_opts="help,nnodes:,cores-per-node:,jobs:,noexec,sched-opts:verbose"
-declare -r short_opts="hvN:c:j:o:"
-declare -r usage="\
-\n\
-Usage: $prog [OPTIONS]\n\
-Simple Flux sched/exec test benchmark.\n\
-\n\
-Options:\n\
- -h, --help              display this messages\n\
- -v, --verbose           run with extra verbose output\n\
- -N, --nnodes=NNODES     set simulated number of nodes (default=${NNODES})\n\
- -c, --cores-per-node=N  set simulated cores per node (default=${CPN})\n\
- -j, --jobs=NJOBS        set number of jobs to run (default nnodes*cpn)\n\
- -o, --sched-opts=OPTS   set scheduler module load options\n\
-     --noexec            do not simulate execution, just scheduling\n"
-
-
-log() { local fmt=$1; shift; printf >&2 "$prog: $fmt" "$@"; }
-die() { log "$@" && exit 1; }
-
-verbose() { test $VERBOSE = "t" && log "$@"; }
-
-log_timing_msg() {
-    local name=$1
-    local start=$2
-    local end=$3
-    local elapsed=$(echo "$end - $start" | bc -l)
-    local jps=$(echo "$NJOBS/$elapsed" | bc -l)
-    log "$name $NJOBS jobs in %.3fs (%.2f job/s)\n" $elapsed $jps
-}
-
-GETOPTS=$(/usr/bin/getopt -u -o $short_opts -l $long_opts -n $prog -- $@)
-if test $? != 0; then
-    echo  "$usage"
-    exit 1
-fi
-
-eval set -- "$GETOPTS"
-while true; do
-    case "$1" in
-      -v|--verbose)         VERBOSE=t;  shift   ;;
-      -N|--nnodes)          NNODES=$2;  shift 2 ;;
-      -c|--cores-per-node)  CPN=$2;     shift 2 ;;
-      -j|--jobs)            NJOBS=$2;   shift 2 ;;
-      -o|--sched-opts)      OPTS="$2";  shift 2 ;;
-      --noexec)             NOEXEC=t;   shift   ;;
-      --)                   shift ; break ;     ;;
-      -h|--help)            echo -e "$usage" ; exit 0           ;;
-      *)                    die "Invalid option '$1'\n$usage"   ;;
-    esac
-done
-
-# If not set, set number of jobs to nnodes * cores-per-node:
-NJOBS=${NJOBS:-$((${NNODES}*${CPN}))}
-
-log "On branch $(git rev-parse --abbrev-ref HEAD): $(git describe)\n"
-log "starting with $NJOBS jobs across ${NNODES} nodes with ${CPN} cores/node.\n"
-log "broker.pid=$(flux getattr broker.pid)\n"
-
-#  Reload scheduler so we can insert a fake resource set:
-flux module remove sched-simple
-flux kvs put \
-    resource.hwloc.by_rank="{\"[0-$(($NNODES-1))]\":{\"Core\":$CPN}}"
-flux mini run --dry-run --setattr=system.exec.test.run_duration=.001s hostname \
-    > job.json
-
-log "Loading sched-simple: ${OPTS}\n"
-flux module load sched-simple ${OPTS} || die "Failed to load sched-simple"
-
-#  If not testing exec system, remove the job-exec module
-test "$NOEXEC" = "t" && flux module remove job-exec
-
-t_start=$(date +%s.%N)
-t/ingest/submitbench -f 1024 -r $NJOBS job.json > job.list
-t_ingest=$(date +%s.%N)
-log_timing_msg ingested $t_start $t_ingest
-
-test "$VERBOSE" = "t" && flux queue status -v
-
-last=$(tail -1 job.list)
-first=$(head -1 job.list)
-
-starttime=$(flux job eventlog $first | awk '$2 == "submit" {print $1}')
-alloctime=$(flux job wait-event $last alloc | awk '$2 == "alloc" {print $1}')
-log_timing_msg allocated $starttime $alloctime
-
-test "$VERBOSE" = "t" && flux queue status -v
-
-if test -z "$NOEXEC"; then
-    runtime=$(flux job wait-event $last clean | awk '{print $1}')
-    log_timing_msg ran $starttime $runtime
-fi
-
-flux job cancelall -f
-flux queue idle --quiet
-
-test "$VERBOSE" = "t" && flux queue status -v
-
-t_done=$(date +%s.%N)
-log_timing_msg "total walltime for" $t_start $t_done
-
-#  If not testing exec system, reinstall job-exec to avoid error from rc3
-# test "$NOEXEC" = "t" && flux module load job-exec
-# XXX: Not a good idea, starts running jobs just before script exit
-
-
-# vi: ts=4 sw=4 expandtab
diff --git a/src/test/throughput.py b/src/test/throughput.py
new file mode 100755
index 000000000000..b2eec392d023
--- /dev/null
+++ b/src/test/throughput.py
@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+##############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import sys
+import time
+import argparse
+import json
+
+import flux
+from flux import job
+from flux.job import JobspecV1
+from flux.progress import Bottombar
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Run job throughput test")
+    parser.add_argument(
+        "-n",
+        "--njobs",
+        type=int,
+        metavar="N",
+        help="Set the total number of jobs to run",
+        default=100,
+    )
+    parser.add_argument(
+        "-t",
+        "--runtime",
+        help="When simulating execution, runtime of each job (default=1ms)",
+        default="0.001s",
+    )
+    parser.add_argument(
+        "-x",
+        "--exec",
+        help="Do not simulate execution, actually run jobs",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-o",
+        "--setopt",
+        action="append",
+        help="Set shell option OPT or OPT=VAL (multiple use OK)",
+        metavar="OPT",
+    )
+    parser.add_argument(
+        "--setattr",
+        action="append",
+        help="Set job attribute ATTR to VAL (multiple use OK)",
+        metavar="ATTR=VAL",
+    )
+    parser.add_argument(
+        "-s",
+        "--status",
+        help="Add a status bar to bottom of terminal",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Log job events",
+        action="store_true",
+    )
+    parser.add_argument("command", nargs=argparse.REMAINDER, default=["true"])
+
+    return parser.parse_args()
+
+
+def create_test_jobspec(args):
+
+    #  Create a test jobspec
+    if not args.command:
+        args.command = ["true"]
+    jobspec = JobspecV1.from_command(args.command)
+
+    #  Set any requested shell options
+    if args.setopt is not None:
+        for keyval in args.setopt:
+            # Split into key, val with a default for 1 if no val given:
+            key, val = (keyval.split("=", 1) + [1])[:2]
+            try:
+                val = json.loads(val)
+            except (json.JSONDecodeError, TypeError):
+                pass
+            jobspec.setattr_shell_option(key, val)
+
+    #  Set any requested Jobspec attributes
+    if args.setattr is not None:
+        for keyval in args.setattr:
+            tmp = keyval.split("=", 1)
+            if len(tmp) != 2:
+                raise ValueError("--setattr: Missing value for attr " + keyval)
+            key = tmp[0]
+            try:
+                val = json.loads(tmp[1])
+            except (json.JSONDecodeError, TypeError):
+                val = tmp[1]
+            jobspec.setattr(key, val)
+
+    if not args.exec:
+        jobspec.setattr("system.exec.test.run_duration", args.runtime)
+
+    return jobspec
+
+
+class BulkRun:
+
+    # pylint: disable=too-many-instance-attributes
+
+    def __init__(self, handle, total, jobspec):
+        self.handle = handle
+        self.total = total
+        self.jobspec = jobspec
+        self.jobs = {}
+        self.submitted = 0
+        self.running = 0
+        self.complete = 0
+        self.bbar = None
+
+    def statusline(self, _bbar, _width):
+        return (
+            f"{self.total:>6} jobs: {self.submitted:>6} submitted, "
+            f"{self.running:>6} running, {self.complete} completed"
+        )
+
+    def event_cb(self, future, args, jobid):
+        event = future.get_event()
+        if event is not None:
+            if args.verbose:
+                print(f"{jobid}: {event.name}")
+            if event.name == "submit":
+                self.submitted += 1
+                self.jobs[jobid][event.name] = event
+            elif event.name == "start":
+                self.running += 1
+            elif event.name == "finish":
+                self.running -= 1
+                self.complete += 1
+            elif event.name == "clean":
+                self.jobs[jobid][event.name] = event
+            if self.bbar:
+                self.bbar.update()
+
+    def handle_submit(self, args, jobid):
+        self.jobs[jobid] = {"t_submit": time.time()}
+        fut = job.event_watch_async(self.handle, jobid)
+        fut.then(self.event_cb, args, jobid)
+
+    def submit_cb(self, future, args):
+        # pylint: disable=broad-except
+        try:
+            self.handle_submit(args, future.get_id())
+        except Exception as exc:
+            print(f"Submission failed: {exc}", file=sys.stderr)
+
+    def submit_async(self, args):
+        spec = self.jobspec.dumps()
+        for _ in range(args.njobs):
+            job.submit_async(self.handle, spec).then(self.submit_cb, args)
+
+    def run(self, args):
+        if args.status:
+            self.bbar = Bottombar(self.statusline).start()
+
+        self.submit_async(args)
+
+        self.handle.reactor_run()
+
+        if self.bbar:
+            self.bbar.stop()
+
+        return self
+
+
+def main():
+
+    args = parse_args()
+
+    time0 = time.time()
+
+    jobspec = create_test_jobspec(args)
+
+    bulk = BulkRun(flux.Flux(), args.njobs, jobspec).run(args)
+
+    jobs = bulk.jobs
+
+    #  Get the job with the earliest 'submit' event:
+    first = jobs[min(jobs.keys(), key=lambda x: jobs[x]["submit"].timestamp)]
+
+    #  Get the job with the latest 'clean' event:
+    last = jobs[max(jobs.keys(), key=lambda x: jobs[x]["clean"].timestamp)]
+
+    #  Get the job with the latest 't_submit' time:
+    lastsubmit = jobs[max(jobs.keys(), key=lambda x: jobs[x]["t_submit"])]
+    submit_time = lastsubmit["t_submit"] - time0
+    sjps = args.njobs / submit_time
+
+    script_runtime = time.time() - time0
+    job_runtime = last["clean"].timestamp - first["submit"].timestamp
+    jps = args.njobs / job_runtime
+    jpsb = args.njobs / script_runtime
+
+    print(f"number of jobs: {args.njobs}")
+    print(f"submit time:    {submit_time:<6.3f}s ({sjps:5.1f} job/s)")
+    print(f"script runtime: {script_runtime:<6.3f}s")
+    print(f"job runtime:    {job_runtime:<6.3f}s")
+    print(f"throughput:     {jps:<.1f} job/s (script: {jpsb:5.1f} job/s)")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/test/travis-lib.sh b/src/test/travis-lib.sh
deleted file mode 100644
index 74b5c12c3b99..000000000000
--- a/src/test/travis-lib.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Fold commands in travis-ci output window with timing
-# https://github.com/travis-ci/travis-ci/issues/2285
-# and
-# https://github.com/travis-ci/travis-build/tree/master/lib/travis/build/bash
-#
-
-
-# Globals for travis_time_start/end:
-TRAVIS_TIME=
-TRAVIS_TIME_ID=
-
-#  Only emit travis_start/end and travis_time blocks if we're actually
-#   running under Travis-CI
-#
-if test "$TRAVIS" = "true"; then
-  travis_time_start() {
-    TRAVIS_TIME=$(date +%s%N)
-    TRAVIS_TIME_ID="$(printf %08x $((RANDOM * RANDOM)))"
-    printf 'travis_time:start:%s\r\033[0K' $TRAVIS_TIME_ID
-  }
-  travis_time_end() {
-    local finish=$(date +%s%N)
-    printf 'travis_time:end:%s:start=%s,finish=%s,duration=%s\r\033[0K' \
-           $TRAVIS_TIME_ID $TRAVIS_TIME $finish $((finish - TRAVIS_TIME))
-  }
-  travis_fold_start() {
-    printf 'travis_fold:start:%s\r\033[0K\033[33;1m%s\033[0m' "$1" "$2"
-  }
-  travis_fold_end() {
-    printf 'travis_fold:end:%s\r\033[0K' "$1"
-  }
-else
-  travis_time_start() { return 0;}
-  travis_time_end()   { return 0;}
-  travis_fold_start() { return 0;}
-  travis_fold_end()   { return 0;}
-fi
-
-travis_fold() {
-    local NAME=$1
-    local DESC=$2
-    shift 2
-    travis_fold_start "$NAME" "$DESC"
-    travis_time_start
-    echo
-    eval "$@"
-    rc=$?
-    travis_time_end
-    travis_fold_end "$NAME"
-    return $rc
-}
diff --git a/src/test/travis_run.sh b/src/test/travis_run.sh
deleted file mode 100755
index f5f09c02d713..000000000000
--- a/src/test/travis_run.sh
+++ /dev/null
@@ -1,132 +0,0 @@
-#!/bin/bash
-#
-#  Test runner script meant to be executed inside of a docker container
-#
-#  Usage: travis_run.sh [OPTIONS...]
-#
-#  Where OPTIONS are passed directly to ./configure
-#
-#  The script is otherwise influenced by the following environment variables:
-#
-#  JOBS=N        Argument for make's -j option, default=2
-#  COVERAGE      Run with --enable-code-coverage, `make check-code-coverage`
-#  TEST_INSTALL  Run `make check` against installed flux-core
-#  CPPCHECK      Run cppcheck if set to "t"
-#  DISTCHECK     Run `make distcheck` if set
-#  PRELOAD       Set as LD_PRELOAD for make and tests
-#  chain_lint    Run sharness with --chain-lint if chain_lint=t
-#
-#  And, obviously, some crucial variables that configure itself cares about:
-#
-#  CC, CXX, LDFLAGS, CFLAGS, etc.
-#
-
-# if make is old, and scl is here, and devtoolset is available and not turned
-# on, re-exec ourself with it active to get a newer make
-if make --version | grep 'GNU Make 4' 2>&1 > /dev/null ; then
-  MAKE="make --output-sync=target --no-print-directory"
-else
-  MAKE="make" #use this if all else fails
-  if test "X$X_SCLS" = "X" ; then
-    if scl -l | grep devtoolset-7 2>&1 >/dev/null ; then
-      echo  bash "$0" "$@" | scl enable devtoolset-7 -
-      exit
-    fi
-  fi
-fi
-
-# source travis_fold and travis_time functions:
-. src/test/travis-lib.sh
-
-ARGS="$@"
-JOBS=${JOBS:-2}
-MAKECMDS="${MAKE} -j ${JOBS}"
-CHECKCMDS="${MAKE} -k -j ${JOBS} ${DISTCHECK:+dist}check"
-
-# Add non-standard path for libfaketime to LD_LIBRARY_PATH:
-export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/faketime"
-
-# Force git to update the shallow clone and include tags so git-describe works
-travis_fold "git_fetch_tags" "git fetch --unshallow --tags" \
- git fetch --unshallow --tags || true
-ulimit -c unlimited
-
-# Manually update ccache symlinks (XXX: Is this really necessary?)
-test -x /usr/sbin/update-ccache-symlinks && \
-    sudo /usr/sbin/update-ccache-symlinks
-export PATH=/usr/lib/ccache:$PATH
-
-# Ensure ccache dir exists
-mkdir -p $HOME/.ccache
-
-# clang+ccache requries second cpp pass:
-if echo "$CC" | grep -q "clang"; then
-    CCACHE_CPP=1
-fi
-
-# Ensure travis builds libev such that libfaketime will work:
-# (force libev to *not* use syscall interface for clock_gettime())
-export CPPFLAGS="$CPPFLAGS -DEV_USE_CLOCK_SYSCALL=0 -DEV_USE_MONOTONIC=1"
-
-# Ensure we always use internal <flux/core.h> by placing a dummy file
-#  in the same path as ZMQ_FLAGS:
-sudo sh -c "mkdir -p /usr/include/flux \
-    && echo '#error Non-build-tree flux/core.h!' > /usr/include/flux/core.h"
-
-# Enable coverage for $CC-coverage build
-# We can't use distcheck here, it doesn't play well with coverage testing:
-if test "$COVERAGE" = "t"; then
-    ARGS="$ARGS --enable-code-coverage"
-    CHECKCMDS="${MAKE} -j $JOBS check-code-coverage && \
-              lcov -l flux*-coverage.info"
-
-# Use make install for T_INSTALL:
-elif test "$TEST_INSTALL" = "t"; then
-    ARGS="$ARGS --prefix=/usr --sysconfdir=/etc"
-    CHECKCMDS="sudo make install && \
-              /usr/bin/flux keygen --force && \
-              FLUX_TEST_INSTALLED_PATH=/usr/bin ${MAKE} -j $JOBS check"
-fi
-
-if test -n "$PRELOAD" ; then
-  CHECKCMDS="/usr/bin/env 'LD_PRELOAD=$PRELOAD' ${CHECKCMDS}"
-fi
-
-# Travis has limited resources, even though number of processors might
-#  might appear to be large. Limit session size for testing to 5 to avoid
-#  spurious timeouts.
-export FLUX_TEST_SIZE_MAX=5
-
-# Invoke MPI tests
-# CentOS 7: mpich only available via environment-module:
-if test -f /usr/share/Modules/init/bash; then
-    . /usr/share/Modules/init/bash && module load mpi
-fi
-export FLUX_TEST_MPI=t
-
-# Generate logfiles from sharness tests for extra information:
-export FLUX_TESTS_LOGFILE=t
-export DISTCHECK_CONFIGURE_FLAGS="${ARGS}"
-
-
-if test "$CPPCHECK" = "t"; then
-    sh -x src/test/cppcheck.sh
-fi
-
-echo "Starting MUNGE"
-sudo /sbin/runuser -u munge /usr/sbin/munged
-
-travis_fold "autogen.sh" "./autogen.sh..." ./autogen.sh
-
-if test -n "$BUILD_DIR" ; then
-  mkdir -p "$BUILD_DIR"
-  cd "$BUILD_DIR"
-fi
-
-travis_fold "configure"  "/usr/src/configure ${ARGS}..." /usr/src/configure ${ARGS}
-travis_fold "make_clean" "make clean..." make clean
-
-echo running: ${MAKECMDS}
-travis_fold "build" "${MAKECMDS}" eval ${MAKECMDS}
-echo running: ${CHECKCMDS}
-travis_fold "check" "${CHECKCMDS}" eval ${CHECKCMDS}
diff --git a/t/Makefile.am b/t/Makefile.am
index 6e749b68ed64..2c05706d82e1 100644
--- a/t/Makefile.am
+++ b/t/Makefile.am
@@ -6,10 +6,11 @@ AM_LDFLAGS = \
 	$(CODE_COVERAGE_LIBS)
 
 AM_CPPFLAGS = \
-        -I$(top_srcdir) \
+	-I$(top_srcdir) \
 	-I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libccan \
 	-I$(top_builddir)/src/common/libflux \
-        $(ZMQ_CFLAGS)
+	$(JANSSON_CFLAGS)
 
 #  Always set LUA_PATH such that builddir/?.lua is first so that the
 #   build version of fluxometer.lua is found.
@@ -22,15 +23,16 @@ AM_TESTS_ENVIRONMENT = \
 	export LUA_PATH="$(builddir)/?.lua;$$LUA_PATH;;;";\
 	test -n "$$FLUX_TEST_INSTALLED_PATH" && \
 	export LUA_CPATH="$(abs_top_builddir)/src/bindings/lua/.libs/?.so;$$LUA_CPATH;;"; \
-	export PYTHONPATH="$(abs_top_builddir)/src/bindings/python:$(abs_top_srcdir)/src/bindings/python:$(top_srcdir)/t/python/tap:$(PYTHON_SITE_PKG):$$PYTHONPATH";
+	export PYTHONPATH="$(abs_top_builddir)/src/bindings/python:$(abs_top_srcdir)/src/bindings/python:$(top_srcdir)/t/python/tap:$(PYTHON_SITE_PKG):$$PYTHONPATH";\
+	export PYTHON="${PYTHON}";
 
 TEST_EXTENSIONS = .t .py
 T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
-        $(top_srcdir)/config/tap-driver.sh
+	$(top_srcdir)/config/tap-driver.sh
 PY_LOG_DRIVER = $(PYTHON) $(top_srcdir)/config/tap-driver.py
 
 lua_SCRIPTS = \
-        fluxometer.lua
+	fluxometer.lua
 
 install-data-local:
 	$(INSTALL) -m644 fluxometer/conf.lua.installed \
@@ -45,8 +47,17 @@ clean-local:
 #   in TESTS so that `make check` runs them first, hopefully resulting
 #   in a reduced makespan overall.
 LONGTESTSCRIPTS = \
+	t9000-system.t \
 	t5000-valgrind.t \
-	t3100-flux-in-flux.t
+	t3100-flux-in-flux.t \
+	t3200-instance-restart.t \
+	t3202-instance-restart-testexec.t \
+	t3203-instance-recovery.t \
+	t4000-issues-test-driver.t \
+	t2801-top-cmd.t \
+	t2808-shutdown-cmd.t \
+	t2712-python-cli-alloc.t \
+	t2714-python-cli-batch.t
 
 # This list is included in both TESTS and dist_check_SCRIPTS.
 TESTSCRIPTS = \
@@ -58,20 +69,35 @@ TESTSCRIPTS = \
 	t0004-event.t \
 	t0005-exec.t \
 	t0005-rexec.t \
+	t0005-exec-jobid.t \
 	t0007-ping.t \
 	t0008-attr.t \
 	t0009-dmesg.t \
 	t0010-generic-utils.t \
 	t0011-content-cache.t \
 	t0012-content-sqlite.t \
+	t0024-content-s3.t \
+	t0028-content-backing-none.t \
+	t0029-archive-mmap.t \
+	t0030-marshall.t \
+	t0031-constraint-parser.t \
+	t0032-directives-parser.t \
+	t0033-filemap-cmd.t \
+	t0025-broker-state-machine.t \
+	t0027-broker-groups.t \
 	t0013-config-file.t \
 	t0014-runlevel.t \
 	t0015-cron.t \
 	t0016-cron-faketime.t \
 	t0017-security.t \
-	t0019-jobspec-schema.t \
-	t0021-flux-jobspec.t \
+	t0018-content-files.t \
+	t0019-tbon-config.t \
+	t0020-terminus.t \
+	t0021-archive-cmd.t \
 	t0022-jj-reader.t \
+	t0023-jobspec1-validate.t \
+	t0026-flux-R.t \
+	t0090-content-enospc.t \
 	t1000-kvs.t \
 	t1001-kvs-internals.t \
 	t1003-kvs-stress.t \
@@ -80,52 +106,156 @@ TESTSCRIPTS = \
 	t1007-kvs-lookup-watch.t \
 	t1008-kvs-eventlog.t \
 	t1009-kvs-copy.t \
+	t1010-kvs-commit-sync.t \
+	t1011-kvs-checkpoint-period.t \
 	t1101-barrier-basic.t \
 	t1102-cmddriver.t \
 	t1103-apidisconnect.t \
 	t1105-proxy.t \
 	t1106-ssh-connector.t \
+	t1107-heartbeat.t \
+	t1200-stats-basic.t \
 	t2004-hydra.t \
-	t2005-hwloc-basic.t \
-	t2007-caliper.t \
 	t2008-althash.t \
 	t2010-kvs-snapshot-restore.t \
-	t2100-aggregate.t \
-	t2200-job-ingest.t \
+	t2100-job-ingest.t \
+	t2110-job-ingest-validator.t \
+	t2111-job-ingest-config.t \
+	t2112-job-ingest-frobnicator.t \
+	t2113-job-ingest-pipeline.t \
 	t2201-job-cmd.t \
 	t2202-job-manager.t \
-	t2203-job-manager-dummysched.t \
-	t2204-job-info.t \
-	t2205-job-info-security.t \
-	t2206-job-manager-bulk-state.t \
-	t2207-job-manager-wait.t \
-	t2208-queue-cmd.t \
-	t2210-job-manager-bugs.t \
+	t2203-job-manager-single.t \
+	t2204-job-manager-limited.t \
+	t2205-job-manager-unlimited.t \
+	t2206-job-manager-annotate.t \
+	t2208-job-manager-wait.t \
+	t2209-job-manager-bugs.t \
+	t2210-job-manager-events-journal.t \
+	t2211-job-manager-jobspec.t \
+	t2212-job-manager-plugins.t \
+	t2213-job-manager-hold-single.t \
+	t2214-job-manager-hold-limited.t \
+	t2215-job-manager-hold-unlimited.t \
+	t2216-job-manager-priority-order-single.t \
+	t2217-job-manager-priority-order-limited.t \
+	t2218-job-manager-priority-order-unlimited.t \
+	t2219-job-manager-restart.t \
+	t2220-job-manager-R.t \
+	t2221-job-manager-limit-duration.t \
+	t2222-job-manager-limit-job-size.t \
+	t2223-job-manager-queue-priority-order-limited.t \
+	t2224-job-manager-queue-priority-order-unlimited.t \
+	t2226-housekeeping.t \
+	t2230-job-info-lookup.t \
+	t2231-job-info-eventlog-watch.t \
+	t2232-job-info-security.t \
+	t2233-job-info-update.t \
+	t2240-queue-cmd.t \
+	t2241-queue-cmd-list.t \
+	t2245-policy-config.t \
+	t2260-job-list.t \
+	t2261-job-list-update.t \
+	t2262-job-list-stats.t \
+	t2270-job-dependencies.t \
+	t2271-job-dependency-after.t \
+	t2272-job-begin-time.t \
+	t2273-job-alloc-bypass.t \
+	t2274-manager-perilog-per-rank.t \
+	t2275-job-duration-validator.t \
+	t2276-job-requires.t \
+	t2280-job-memo.t \
+	t2290-job-update.t \
+	t2291-job-update-queue.t \
+	t2292-job-update-running.t \
 	t2300-sched-simple.t \
-	t2301-schedutil-outstanding-requests.t \
+	t2302-sched-simple-up-down.t \
+	t2303-sched-hello.t \
+	t2304-sched-simple-alloc-check.t \
+	t2305-sched-slow.t \
+	t2310-resource-module.t \
+	t2311-resource-drain.t \
+	t2312-resource-exclude.t \
+	t2313-resource-acquire.t \
+	t2314-resource-monitor.t \
+	t2315-resource-system.t \
+	t2350-resource-list.t \
+	t2351-resource-status-input.t \
+	t2352-resource-cmd-config.t \
+	t2353-resource-eventlog.t \
+	t2354-resource-status.t \
 	t2400-job-exec-test.t \
 	t2401-job-exec-hello.t \
 	t2402-job-exec-dummy.t \
 	t2403-job-exec-conf.t \
+	t2404-job-exec-multiuser.t \
+	t2406-job-exec-cleanup.t \
+	t2407-sdbus.t \
+	t2408-sdbus-recovery.t \
+	t2409-sdexec.t \
+	t2410-sdexec-memlimit.t \
+	t2411-sdexec-job.t \
 	t2500-job-attach.t \
 	t2501-job-status.t \
 	t2600-job-shell-rcalc.t \
-	t2601-job-shell-standalone.t \
 	t2602-job-shell.t \
 	t2603-job-shell-initrc.t \
 	t2604-job-shell-affinity.t \
-	t2605-job-shell-output-redirection-standalone.t \
 	t2606-job-shell-output-redirection.t \
 	t2607-job-shell-input.t \
 	t2608-job-shell-log.t \
 	t2609-job-shell-events.t \
 	t2610-job-shell-mpir.t \
-	t2700-mini-cmd.t \
+	t2611-debug-emulate.t \
+	t2612-job-shell-pty.t \
+	t2613-job-shell-batch.t \
+	t2614-job-shell-doom.t \
+	t2615-job-shell-rlimit.t \
+	t2616-job-shell-taskmap.t \
+	t2616-job-shell-taskmap-hostfile.t \
+	t2617-job-shell-stage-in.t \
+	t2618-job-shell-signal.t \
+	t2619-job-shell-hwloc.t \
+	t2710-python-cli-submit.t \
+	t2711-python-cli-run.t \
+	t2713-python-cli-bulksubmit.t \
+	t2715-python-cli-cancel.t \
+	t2716-python-cli-batch-conf.t \
 	t2800-jobs-cmd.t \
+	t2800-jobs-recursive.t \
+	t2800-jobs-instance-info.t \
+	t2800-jobs-config.t \
+	t2802-uri-cmd.t \
+	t2803-flux-pstree.t \
+	t2804-uptime-cmd.t \
+	t2805-startlog-cmd.t \
+	t2806-config-cmd.t \
+	t2807-dump-cmd.t \
+	t2809-job-purge.t \
+	t2810-kvs-garbage-collect.t \
+	t2811-flux-pgrep.t \
+	t2812-flux-job-last.t \
+	t2813-flux-watch.t \
+	t2814-hostlist-cmd.t \
+	t2815-post-job-event.t \
+	t2900-job-timelimits.t \
 	t3000-mpi-basic.t \
 	t3001-mpi-personalities.t \
-	t4000-issues-test-driver.t \
-	t9001-pymod.t \
+	t3002-pmi.t \
+	t3003-mpi-abort.t \
+	t3201-crontabs.t \
+	t3300-system-basic.t \
+	t3301-system-latestart.t \
+	t3302-system-offline.t \
+	t3303-system-healthcheck.t \
+	t3304-system-rpctrack-down.t \
+	t3305-system-rpctrack-up.t \
+	t3306-system-routercrash.t \
+	t3307-system-leafcrash.t \
+	t3308-system-torpid.t \
+	t3309-system-reconnect.t \
+	t3400-overlay-trace.t \
+	t3401-module-trace.t \
 	lua/t0001-send-recv.t \
 	lua/t0002-rpc.t \
 	lua/t0003-events.t \
@@ -141,8 +271,24 @@ TESTSCRIPTS = \
 	python/t0005-kvs.py \
 	python/t0006-request.py \
 	python/t0007-watchers.py \
+	python/t0008-message.py \
 	python/t0010-job.py \
 	python/t0012-futures.py \
+	python/t0013-job-list.py \
+	python/t0014-job-kvslookup.py \
+	python/t0015-job-output.py \
+	python/t0020-hostlist.py \
+	python/t0021-idset.py \
+	python/t0022-resource-set.py \
+	python/t0023-executor.py \
+	python/t0024-util.py \
+	python/t0025-uri.py \
+	python/t0026-tree.py \
+	python/t0027-constraint-parser.py \
+	python/t0028-compat36.py \
+	python/t0029-fileref.py \
+	python/t0030-journal.py \
+	python/t0031-conf-builtin.py \
 	python/t1000-service-add-remove.py
 
 if HAVE_FLUX_SECURITY
@@ -150,9 +296,7 @@ TESTSCRIPTS += python/t0009-security.py
 endif
 
 # make check runs these TAP tests directly (both scripts and programs)
-TESTS = \
-	shmem/backtoback.t \
-	$(TESTSCRIPTS)
+TESTS = $(TESTSCRIPTS)
 
 EXTRA_DIST= \
 	aggregate-results.sh \
@@ -163,26 +307,68 @@ EXTRA_DIST= \
 	test-under-flux/t_modcheck.t \
 	test-under-flux/test.t \
 	rc/rc1-kvs \
-	rc/rc1-testenv \
 	rc/rc1-job \
 	rc/rc3-kvs \
-	rc/rc3-testenv \
 	rc/rc3-job \
 	shell/input \
 	shell/output \
 	shell/initrc/tests \
 	flux-jobs/tests \
-	jobspec
+	scripts/strerror_symbol \
+	scripts/run_timeout.py \
+	scripts/startctl.py \
+	scripts/groups.py \
+	scripts/rexec.py \
+	jobspec \
+	resource \
+	marshall \
+	batch \
+	job-manager/dumps \
+	flux-resource
 
 dist_check_SCRIPTS = \
 	$(TESTSCRIPTS) \
-        issues/t0441-kvs-put-get.sh \
-        issues/t0505-msg-handler-reg.lua \
-        issues/t0821-kvs-segfault.sh \
+	system/0001-basic.t \
+	system/0002-exec-with-imp.t \
+	system/0004-recovery.t \
+	system/0005-exec.t \
+	issues/t0441-kvs-put-get.sh \
+	issues/t0505-msg-handler-reg.lua \
+	issues/t0821-kvs-segfault.sh \
 	issues/t1760-kvs-use-after-free.sh \
 	issues/t2281-service-add-crash.sh \
 	issues/t2284-initial-program-format-chars.sh \
 	issues/t2686-shell-input-race.sh \
+	issues/t3186-python-future-get-sigint.sh \
+	issues/t3415-job-shell-segfault-on-null.sh \
+	issues/t3432-python-sigint.sh \
+	issues/t3429-python-future-ref.py \
+	issues/t3470-multithread-reactor-run.py \
+	issues/t3503-nofiles-limit.sh \
+	issues/t3775-binary-io.sh \
+	issues/t3920-running-underflow.sh \
+	issues/t3960-job-exec-ehostunreach.sh \
+	issues/t3906-job-exec-exception.sh \
+	issues/t3982-python-message-ref.py \
+	issues/t4182-resource-rerank.sh \
+	issues/t4184-sched-simple-restart.sh \
+	issues/t4222-kvs-assume-empty-dir.sh \
+	issues/t4375-content-flush-hang.sh \
+	issues/t4378-content-flush-force.sh \
+	issues/t4379-dirty-cache-entries-flush.sh \
+	issues/t4413-empty-eventlog.sh \
+	issues/t4465-job-list-use-after-free.sh \
+	issues/t4482-flush-list-corruption.sh \
+	issues/t4583-free-range-test.sh \
+	issues/t4612-eventlog-overwrite-crash.sh \
+	issues/t4711-job-list-purge-inactive.sh \
+	issues/t4852-t_submit-legacy.sh \
+	issues/t5105-signal-propagation.sh \
+	issues/t5308-kvsdir-initial-path.py \
+	issues/t5368-kvs-commit-clear.py \
+	issues/t5657-kvs-getroot-namespace.sh \
+	issues/t5892-shutdown-no-epilog.sh \
+	issues/t2492-shell-lost.sh \
 	python/__init__.py \
 	python/subflux.py \
 	python/tap \
@@ -191,9 +377,24 @@ dist_check_SCRIPTS = \
 	scripts/waitfile.lua \
 	scripts/t0004-event-helper.sh \
 	scripts/tssh \
+	scripts/sign-as.py \
+	scripts/runpty.py \
+	scripts/dmesg-grep.py \
+	scripts/stats-listen.py \
+	scripts/sqlite-query.py \
+	scripts/pipe.py \
 	valgrind/valgrind-workload.sh \
 	valgrind/workload.d/job \
+	valgrind/workload.d/job-info \
+	valgrind/workload.d/job-multinode \
+	valgrind/workload.d/job-wait \
+	valgrind/workload.d/job-cancel \
+	valgrind/workload.d/job-list \
+	valgrind/workload.d/job-sdexec \
+	valgrind/workload.d/resource \
+	content/content-helper.sh \
 	kvs/kvs-helper.sh \
+	kvs/change-checkpoint.py \
 	job-manager/exec-service.lua \
 	job-manager/drain-cancel.py \
 	job-manager/bulk-state.py \
@@ -201,22 +402,25 @@ dist_check_SCRIPTS = \
 	job-manager/submit-waitany.py \
 	job-manager/submit-sliding-window.py \
 	job-manager/wait-interrupted.py \
+	job-manager/sched-helper.sh \
+	job-manager/job-conv.py \
 	job-attach/outputsleep.sh \
 	job-exec/dummy.sh \
-	job-info/list-id.py \
-	schedutil/req_and_unload.py \
-	ingest/fake-validate.sh \
-	ingest/bad-validate.sh
+	job-exec/imp.sh \
+	job-exec/imp-fail.sh \
+	job-list/list-id.py \
+	job-list/list-rpc.py \
+	job-list/job-list-helper.sh \
+	ingest/bad-validate.py
 
 check_PROGRAMS = \
-	shmem/backtoback.t \
 	loop/logstderr \
 	loop/issue2337 \
 	loop/issue2711 \
+	kvs/content-spam \
 	kvs/torture \
 	kvs/dtree \
 	kvs/blobref \
-	kvs/hashtest \
 	kvs/watch_disconnect \
 	kvs/commit \
 	kvs/fence_api \
@@ -229,312 +433,686 @@ check_PROGRAMS = \
 	kvs/issue1876 \
 	kvs/waitcreate_cancel \
 	kvs/setrootevents \
-	kvs/checkpoint \
 	request/treq \
 	request/rpc \
+	request/rpc_stream \
 	barrier/tbarrier \
 	reactor/reactorcat \
 	rexec/rexec \
-	rexec/rexec_ps \
 	rexec/rexec_count_stdout \
 	rexec/rexec_getline \
 	job-manager/list-jobs \
+	job-manager/print-constants \
+	job-manager/events_journal_stream \
+	job-info/info_lookup \
+	job-info/update_lookup \
+	job-info/update_watch_stream \
 	ingest/submitbench \
 	sched-simple/jj-reader \
 	shell/rcalc \
-	shell/lptest \
-	shell/mpir
+	shell/mpir \
+	debug/stall \
+	hwloc/hwloc-convert \
+	hwloc/hwloc-version \
+	util/jobspec1-validate \
+	util/handle \
+	util/marshall
 
 if HAVE_MPI
 check_PROGRAMS += \
-	mpi/hello
+	mpi/hello \
+	mpi/abort \
+	mpi/version \
+	mpi/mpich_basic/self \
+	mpi/mpich_basic/simple \
+	mpi/mpich_basic/sendrecv \
+	mpi/mpich_basic/srtest \
+	mpi/mpich_basic/netpipe \
+	mpi/mpich_basic/patterns \
+	mpi/mpich_basic/adapt
 endif
 
 check_LTLIBRARIES = \
-	module/parent.la \
-	module/child.la \
+	module/testmod.la \
+	module/running.la \
+	module/legacy.la \
 	request/req.la \
-	ingest/job-manager-dummy.la \
-	job-manager/sched-dummy.la \
+	ingest/job-manager.la \
+	disconnect/watcher.la \
 	shell/plugins/dummy.la \
 	shell/plugins/conftest.la \
 	shell/plugins/invalid-args.la \
 	shell/plugins/getopt.la \
 	shell/plugins/log.la \
-	shell/plugins/test-event.la
+	shell/plugins/test-event.la \
+	shell/plugins/jobspec-info.la \
+	shell/plugins/taskmap-reverse.la \
+	job-manager/plugins/priority-wait.la \
+	job-manager/plugins/priority-invert.la \
+	job-manager/plugins/args.la \
+	job-manager/plugins/test.la \
+	job-manager/plugins/job_aux.la \
+	job-manager/plugins/jobtap_api.la \
+	job-manager/plugins/random.la \
+	job-manager/plugins/validate.la \
+	job-manager/plugins/dependency-test.la \
+	job-manager/plugins/subscribe.la \
+	job-manager/plugins/cleanup-event.la \
+	job-manager/plugins/create-event.la \
+	job-manager/plugins/create-reject.la \
+	job-manager/plugins/perilog-test.la \
+	job-manager/plugins/config.la \
+	job-manager/plugins/jobspec-update.la \
+	job-manager/plugins/jobspec-update-job-list.la \
+	job-manager/plugins/resource-update-expiration.la \
+	job-manager/plugins/update-test.la \
+	job-manager/plugins/project-bank-validate.la \
+	job-manager/plugins/offline.la \
+	stats/stats-basic.la \
+	stats/stats-immediate.la
+
+check-prep:
+	$(MAKE) $(check_PROGRAMS) $(check_LTLIBRARIES)
 
 dist_check_DATA = \
 	hwloc-data/sierra2/0.xml \
 	hwloc-data/sierra2/1.xml \
+	hwloc-data/corona/0.xml \
 	hwloc-data/1N/shared/02-brokers/0.xml \
 	hwloc-data/1N/shared/02-brokers/1.xml \
 	hwloc-data/1N/nonoverlapping/02-brokers/0.xml \
 	hwloc-data/1N/nonoverlapping/02-brokers/1.xml \
-	valgrind/valgrind.supp \
-	conf.d/bad-hosts/boot.toml \
-	conf.d/bad-hosts2/boot.toml \
-	conf.d/bad-nobootstrap/boot.toml \
-	conf.d/bad-nomatch/boot.toml \
-	conf.d/bad-toml/boot.toml \
-	conf.d/good-nohosts/boot.toml \
-	conf.d/good-emptyhosts/boot.toml \
-	conf.d/good-ipc2/boot.toml \
-	conf.d/good-tcp4/boot.toml
+	hwloc-data/1N/hwloc-versions/v1.11.11/0.xml \
+	hwloc-data/1N/hwloc-versions/v2.1.0/0.xml \
+	hwloc-data/1N/hwloc-versions/v2to1/0.xml \
+	valgrind/valgrind.supp
 
 test_ldadd = \
-        $(top_builddir)/src/common/libflux-internal.la \
-        $(top_builddir)/src/common/libflux-core.la \
-        $(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libtap/libtap.la \
 	$(top_builddir)/src/common/libflux-optparse.la \
-        $(ZMQ_LIBS) $(LIBPTHREAD)
+	$(top_builddir)/src/common/libflux-core.la \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(LIBPTHREAD)
 
-test_cppflags = \
-        -I$(top_srcdir)/src/common/libtap \
-        $(AM_CPPFLAGS)
+test_ldflags = \
+	-no-install
 
-shmem_backtoback_t_SOURCES = shmem/backtoback.c
-shmem_backtoback_t_CPPFLAGS = $(test_cppflags)
-shmem_backtoback_t_LDADD = $(test_ldadd) $(LIBDL)
+test_cppflags = \
+	-I$(top_srcdir)/src/common/libtap \
+	$(AM_CPPFLAGS)
 
 loop_logstderr_SOURCES = loop/logstderr.c
 loop_logstderr_CPPFLAGS = $(test_cppflags)
-loop_logstderr_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+loop_logstderr_LDADD = $(test_ldadd)
+loop_logstderr_LDFLAGS = $(test_ldflags)
 
 loop_issue2337_SOURCES = loop/issue2337.c
 loop_issue2337_CPPFLAGS = $(test_cppflags)
-loop_issue2337_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+loop_issue2337_LDADD = $(test_ldadd)
+loop_issue2337_LDFLAGS = $(test_ldflags)
 
 loop_issue2711_SOURCES = loop/issue2711.c
 loop_issue2711_CPPFLAGS = $(test_cppflags)
-loop_issue2711_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+loop_issue2711_LDADD = $(test_ldadd)
+loop_issue2711_LDFLAGS = $(test_ldflags)
 
 mpi_hello_SOURCES = mpi/hello.c
-mpi_hello_CPPFLAGS = $(MPI_CFLAGS)
-mpi_hello_LDADD = $(MPI_CLDFLAGS) $(LIBRT)
+mpi_hello_CPPFLAGS = $(MPI_CFLAGS) $(test_cppflags)
+mpi_hello_LDADD = $(MPI_CLDFLAGS) $(test_ldadd)
+mpi_hello_LDFLAGS = $(test_ldflags)
+
+mpi_abort_SOURCES = mpi/abort.c
+mpi_abort_CPPFLAGS = $(MPI_CFLAGS) $(test_cppflags)
+mpi_abort_LDADD = $(MPI_CLDFLAGS) $(test_ldadd)
+mpi_abort_LDFLAGS = $(test_ldflags)
+
+mpi_version_SOURCES = mpi/version.c
+mpi_version_CPPFLAGS = $(MPI_CFLAGS) $(test_cppflags)
+mpi_version_LDADD = $(MPI_CLDFLAGS) $(test_ldadd)
+mpi_version_LDFLAGS = $(test_ldflags)
+
+mpi_mpich_basic_adapt_SOURCES = mpi/mpich_basic/adapt.c mpi/mpich_basic/GetOpt.c mpi/mpich_basic/GetOpt.h
+mpi_mpich_basic_adapt_CPPFLAGS = $(MPI_CFLAGS) $(test_cppflags)
+mpi_mpich_basic_adapt_LDADD = $(test_ldadd)
+mpi_mpich_basic_adapt_LDFLAGS = $(MPI_CLDFLAGS) $(test_ldflags)
+
+mpi_mpich_basic_netpipe_SOURCES = mpi/mpich_basic/netmpi.c mpi/mpich_basic/GetOpt.c mpi/mpich_basic/GetOpt.h
+mpi_mpich_basic_netpipe_CPPFLAGS = $(MPI_CFLAGS) $(test_cppflags)
+mpi_mpich_basic_netpipe_LDADD = $(test_ldadd)
+mpi_mpich_basic_netpipe_LDFLAGS = $(MPI_CLDFLAGS) $(test_ldflags)
+
+mpi_mpich_basic_patterns_SOURCES = mpi/mpich_basic/patterns.c
+mpi_mpich_basic_patterns_CPPFLAGS = $(MPI_CFLAGS) $(test_cppflags)
+mpi_mpich_basic_patterns_LDADD = $(test_ldadd)
+mpi_mpich_basic_patterns_LDFLAGS = $(MPI_CLDFLAGS) $(test_ldflags)
+
+mpi_mpich_basic_srtest_SOURCES = mpi/mpich_basic/srtest.c
+mpi_mpich_basic_srtest_CPPFLAGS = $(MPI_CFLAGS) $(test_cppflags)
+mpi_mpich_basic_srtest_LDADD = $(test_ldadd)
+mpi_mpich_basic_srtest_LDFLAGS = $(MPI_CLDFLAGS) $(test_ldflags)
+
+mpi_mpich_basic_self_SOURCES = mpi/mpich_basic/self.c
+mpi_mpich_basic_self_CPPFLAGS = $(MPI_CFLAGS) $(test_cppflags)
+mpi_mpich_basic_self_LDADD = $(MPI_CLDFLAGS) $(test_ldadd)
+mpi_mpich_basic_self_LDFLAGS = $(MPI_CLDFLAGS) $(test_ldflags)
+
+mpi_mpich_basic_sendrecv_SOURCES = mpi/mpich_basic/sendrecv.c
+mpi_mpich_basic_sendrecv_CPPFLAGS = $(MPI_CFLAGS) $(test_cppflags)
+mpi_mpich_basic_sendrecv_LDADD = $(MPI_CLDFLAGS) $(test_ldadd)
+mpi_mpich_basic_sendrecv_LDFLAGS = $(MPI_CLDFLAGS) $(test_ldflags)
+
+mpi_mpich_basic_simple_SOURCES = mpi/mpich_basic/simple.c
+mpi_mpich_basic_simple_CPPFLAGS = $(MPI_CFLAGS) $(test_cppflags)
+mpi_mpich_basic_simple_LDADD = $(MPI_CLDFLAGS) $(test_ldadd)
+mpi_mpich_basic_simple_LDFLAGS = $(MPI_CLDFLAGS) $(test_ldflags)
+
+kvs_content_spam_SOURCES = kvs/content-spam.c
+kvs_content_spam_CPPFLAGS = $(test_cppflags)
+kvs_content_spam_LDADD = $(test_ldadd)
+kvs_content_spam_LDFLAGS = $(test_ldflags)
 
 kvs_torture_SOURCES = kvs/torture.c
 kvs_torture_CPPFLAGS = $(test_cppflags)
-kvs_torture_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_torture_LDADD = $(test_ldadd)
+kvs_torture_LDFLAGS = $(test_ldflags)
 
 kvs_dtree_SOURCES = kvs/dtree.c
 kvs_dtree_CPPFLAGS = $(test_cppflags)
-kvs_dtree_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_dtree_LDADD = $(test_ldadd)
+kvs_dtree_LDFLAGS = $(test_ldflags)
 
 kvs_blobref_SOURCES = kvs/blobref.c
 kvs_blobref_CPPFLAGS = $(test_cppflags)
-kvs_blobref_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_blobref_LDADD = $(test_ldadd)
+kvs_blobref_LDFLAGS = $(test_ldflags)
 
 kvs_commit_SOURCES = kvs/commit.c
 kvs_commit_CPPFLAGS = $(test_cppflags)
-kvs_commit_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_commit_LDADD = $(test_ldadd)
+kvs_commit_LDFLAGS = $(test_ldflags)
 
 kvs_fence_api_SOURCES = kvs/fence_api.c
 kvs_fence_api_CPPFLAGS = $(test_cppflags)
-kvs_fence_api_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_fence_api_LDADD = $(test_ldadd)
+kvs_fence_api_LDFLAGS = $(test_ldflags)
 
 kvs_transactionmerge_SOURCES = kvs/transactionmerge.c
 kvs_transactionmerge_CPPFLAGS = $(test_cppflags)
-kvs_transactionmerge_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_transactionmerge_LDADD = $(test_ldadd)
+kvs_transactionmerge_LDFLAGS = $(test_ldflags)
 
 kvs_fence_namespace_remove_SOURCES = kvs/fence_namespace_remove.c
 kvs_fence_namespace_remove_CPPFLAGS = $(test_cppflags)
-kvs_fence_namespace_remove_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_fence_namespace_remove_LDADD = $(test_ldadd)
+kvs_fence_namespace_remove_LDFLAGS = $(test_ldflags)
 
 kvs_fence_invalid_SOURCES = kvs/fence_invalid.c
 kvs_fence_invalid_CPPFLAGS = $(test_cppflags)
-kvs_fence_invalid_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_fence_invalid_LDADD = $(test_ldadd)
+kvs_fence_invalid_LDFLAGS = $(test_ldflags)
 
 kvs_lookup_invalid_SOURCES = kvs/lookup_invalid.c
 kvs_lookup_invalid_CPPFLAGS = $(test_cppflags)
-kvs_lookup_invalid_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_lookup_invalid_LDADD = $(test_ldadd)
+kvs_lookup_invalid_LDFLAGS = $(test_ldflags)
 
 kvs_commit_order_SOURCES = kvs/commit_order.c
 kvs_commit_order_CPPFLAGS = $(test_cppflags)
-kvs_commit_order_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_commit_order_LDADD = $(test_ldadd)
+kvs_commit_order_LDFLAGS = $(test_ldflags)
 
 kvs_watch_disconnect_SOURCES = kvs/watch_disconnect.c
 kvs_watch_disconnect_CPPFLAGS = $(test_cppflags)
-kvs_watch_disconnect_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
-
-kvs_hashtest_SOURCES = kvs/hashtest.c
-kvs_hashtest_CPPFLAGS = $(test_cppflags) $(SQLITE_CFLAGS)
-kvs_hashtest_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL) $(LIBJUDY) $(SQLITE_LIBS)
+kvs_watch_disconnect_LDADD = $(test_ldadd)
+kvs_watch_disconnect_LDFLAGS = $(test_ldflags)
 
 kvs_issue1760_SOURCES = kvs/issue1760.c
 kvs_issue1760_CPPFLAGS = $(test_cppflags)
-kvs_issue1760_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_issue1760_LDADD = $(test_ldadd)
+kvs_issue1760_LDFLAGS = $(test_ldflags)
 
 kvs_issue1876_SOURCES = kvs/issue1876.c
 kvs_issue1876_CPPFLAGS = $(test_cppflags)
-kvs_issue1876_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_issue1876_LDADD = $(test_ldadd)
+kvs_issue1876_LDFLAGS = $(test_ldflags)
 
 kvs_waitcreate_cancel_SOURCES = kvs/waitcreate_cancel.c
 kvs_waitcreate_cancel_CPPFLAGS = $(test_cppflags)
-kvs_waitcreate_cancel_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_waitcreate_cancel_LDADD = $(test_ldadd)
+kvs_waitcreate_cancel_LDFLAGS = $(test_ldflags)
 
 kvs_setrootevents_SOURCES = kvs/setrootevents.c
 kvs_setrootevents_CPPFLAGS = $(test_cppflags)
-kvs_setrootevents_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
-
-kvs_checkpoint_SOURCES = kvs/checkpoint.c
-kvs_checkpoint_CPPFLAGS = $(test_cppflags)
-kvs_checkpoint_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+kvs_setrootevents_LDADD = $(test_ldadd)
+kvs_setrootevents_LDFLAGS = $(test_ldflags)
 
 request_treq_SOURCES = request/treq.c
 request_treq_CPPFLAGS = $(test_cppflags)
-request_treq_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+request_treq_LDADD = $(test_ldadd)
+request_treq_LDFLAGS = $(test_ldflags)
 
 request_rpc_SOURCES = request/rpc.c
 request_rpc_CPPFLAGS = $(test_cppflags)
-request_rpc_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+request_rpc_LDADD = $(test_ldadd)
+request_rpc_LDFLAGS = $(test_ldflags)
+
+request_rpc_stream_SOURCES = request/rpc_stream.c
+request_rpc_stream_CPPFLAGS = $(test_cppflags)
+request_rpc_stream_LDADD = $(test_ldadd)
+request_rpc_stream_LDFLAGS = $(test_ldflags)
+
+module_testmod_la_SOURCES = module/testmod.c
+module_testmod_la_CPPFLAGS = $(test_cppflags)
+module_testmod_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowher
+module_testmod_la_LIBADD = $(test_ldadd) $(LIBDL)
 
-module_parent_la_SOURCES = module/parent.c
-module_parent_la_CPPFLAGS = $(test_cppflags)
-module_parent_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowher
-module_parent_la_LIBADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+module_running_la_SOURCES = module/running.c
+module_running_la_CPPFLAGS = $(test_cppflags)
+module_running_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowher
+module_running_la_LIBADD = $(test_ldadd)
 
-module_child_la_SOURCES = module/child.c
-module_child_la_CPPFLAGS = $(test_cppflags)
-module_child_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowher
-module_child_la_LIBADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+module_legacy_la_SOURCES = module/legacy.c
+module_legacy_la_CPPFLAGS = $(test_cppflags)
+module_legacy_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowher
+module_legacy_la_LIBADD = $(test_ldadd) $(LIBDL)
 
 barrier_tbarrier_SOURCES = barrier/tbarrier.c
 barrier_tbarrier_CPPFLAGS = $(test_cppflags)
-barrier_tbarrier_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+barrier_tbarrier_LDADD = $(test_ldadd)
+barrier_tbarrier_LDFLAGS = $(test_ldflags)
 
 request_req_la_SOURCES = request/req.c
 request_req_la_CPPFLAGS = $(test_cppflags)
 request_req_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowher
-request_req_la_LIBADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+request_req_la_LIBADD = $(test_ldadd)
 
 shell_rcalc_SOURCES = shell/rcalc.c
 shell_rcalc_CPPFLAGS = $(test_cppflags)
-shell_rcalc_LDADD = \
-        $(top_builddir)/src/shell/libshell.la \
-        $(test_ldadd) $(LIBDL) $(LIBUTIL)
-
-shell_lptest_SOURCES = shell/lptest.c
-shell_lptest_CPPFLAGS = $(test_cppflags)
-shell_lptest_LDADD = \
-        $(test_ldadd) $(LIBDL) $(LIBUTIL)
+shell_rcalc_LDADD = $(top_builddir)/src/shell/libshell.la \
+	$(test_ldadd)
+shell_rcalc_LDFLAGS = $(test_ldflags)
 
 shell_mpir_SOURCES = shell/mpir.c
 shell_mpir_CPPFLAGS = $(test_cppflags)
 shell_mpir_LDADD = \
 	$(top_builddir)/src/shell/libmpir.la \
-        $(test_ldadd) $(LIBDL) $(LIBUTIL)
+	$(top_builddir)/src/cmd/job/mpir.o \
+	$(top_builddir)/src/common/libdebugged/libdebugged.la \
+	$(test_ldadd)
+shell_mpir_LDFLAGS = $(test_ldflags)
+
+debug_stall_SOURCES = debug/stall.c
+debug_stall_CPPFLAGS = $(test_cppflags)
 
 reactor_reactorcat_SOURCES = reactor/reactorcat.c
 reactor_reactorcat_CPPFLAGS = $(test_cppflags)
-reactor_reactorcat_LDADD = \
-	 $(test_ldadd) $(LIBDL) $(LIBUTIL)
+reactor_reactorcat_LDADD = $(test_ldadd)
+reactor_reactorcat_LDFLAGS = \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(test_ldflags)
 
 rexec_rexec_SOURCES = rexec/rexec.c
 rexec_rexec_CPPFLAGS = $(test_cppflags)
-rexec_rexec_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
-
-rexec_rexec_ps_SOURCES = rexec/rexec_ps.c
-rexec_rexec_ps_CPPFLAGS = $(test_cppflags)
-rexec_rexec_ps_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+rexec_rexec_LDADD = $(test_ldadd)
+rexec_rexec_LDFLAGS = \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(test_ldflags)
 
 rexec_rexec_count_stdout_SOURCES = rexec/rexec_count_stdout.c
 rexec_rexec_count_stdout_CPPFLAGS = $(test_cppflags)
-rexec_rexec_count_stdout_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+rexec_rexec_count_stdout_LDADD = $(test_ldadd)
+rexec_rexec_count_stdout_LDFLAGS = $(test_ldflags)
 
 rexec_rexec_getline_SOURCES = rexec/rexec_getline.c
 rexec_rexec_getline_CPPFLAGS = $(test_cppflags)
-rexec_rexec_getline_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+rexec_rexec_getline_LDADD = $(test_ldadd)
+rexec_rexec_getline_LDFLAGS = \
+	$(top_builddir)/src/common/libsubprocess/libsubprocess.la \
+	$(test_ldflags)
 
-ingest_job_manager_dummy_la_SOURCES = ingest/job-manager-dummy.c
-ingest_job_manager_dummy_la_CPPFLAGS = $(test_cppflags)
-ingest_job_manager_dummy_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowhere
-ingest_job_manager_dummy_la_LIBADD = \
-        $(test_ldadd) $(LIBDL) $(LIBUTIL)
+ingest_job_manager_la_SOURCES = ingest/job-manager.c
+ingest_job_manager_la_CPPFLAGS = $(test_cppflags)
+ingest_job_manager_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowhere
+ingest_job_manager_la_LIBADD = $(test_ldadd)
 
 ingest_submitbench_SOURCES = ingest/submitbench.c
 ingest_submitbench_CPPFLAGS = $(test_cppflags)
-ingest_submitbench_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
+ingest_submitbench_LDADD = $(test_ldadd)
+ingest_submitbench_LDFLAGS = $(test_ldflags)
 
 job_manager_list_jobs_SOURCES = job-manager/list-jobs.c
 job_manager_list_jobs_CPPFLAGS = $(test_cppflags)
-job_manager_list_jobs_LDADD = \
-	$(test_ldadd) $(LIBDL) $(LIBUTIL)
-
-job_manager_sched_dummy_la_SOURCES = job-manager/sched-dummy.c
-job_manager_sched_dummy_la_CPPFLAGS = $(test_cppflags)
-job_manager_sched_dummy_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowhere
-job_manager_sched_dummy_la_LIBADD = \
-        $(top_builddir)/src/common/libschedutil/libschedutil.la \
-        $(test_ldadd) $(LIBDL) $(LIBUTIL)
+job_manager_list_jobs_LDADD = $(test_ldadd)
+job_manager_list_jobs_LDFLAGS = $(test_ldflags)
+
+job_manager_print_constants_SOURCES = job-manager/print-constants.c
+job_manager_print_constants_CPPFLAGS = $(test_cppflags)
+job_manager_print_constants_LDADD = $(test_ldadd)
+job_manager_print_constants_LDFLAGS = $(test_ldflags)
+
+job_manager_events_journal_stream_SOURCES = job-manager/events_journal_stream.c
+job_manager_events_journal_stream_CPPFLAGS = $(test_cppflags)
+job_manager_events_journal_stream_LDADD = $(test_ldadd)
+job_manager_events_journal_stream_LDFLAGS = $(test_ldflags)
+
+job_info_info_lookup_SOURCES = job-info/info_lookup.c
+job_info_info_lookup_CPPFLAGS = $(test_cppflags)
+job_info_info_lookup_LDADD = $(test_ldadd)
+job_info_info_lookup_LDFLAGS = $(test_ldflags)
+
+job_info_update_lookup_SOURCES = job-info/update_lookup.c
+job_info_update_lookup_CPPFLAGS = $(test_cppflags)
+job_info_update_lookup_LDADD = $(test_ldadd)
+job_info_update_lookup_LDFLAGS = $(test_ldflags)
+
+job_info_update_watch_stream_SOURCES = job-info/update_watch_stream.c
+job_info_update_watch_stream_CPPFLAGS = $(test_cppflags)
+job_info_update_watch_stream_LDADD = $(test_ldadd)
+job_info_update_watch_stream_LDFLAGS = $(test_ldflags)
+
+disconnect_watcher_la_SOURCES = disconnect/watcher.c
+disconnect_watcher_la_CPPFLAGS = $(test_cppflags)
+disconnect_watcher_la_LDFLAGS = $(fluxmod_ldflags) -module -rpath /nowhere
+disconnect_watcher_la_LIBADD = $(test_ldadd)
 
 sched_simple_jj_reader_SOURCES = sched-simple/jj-reader.c
 sched_simple_jj_reader_CPPFLAGS = $(test_cppflags)
 sched_simple_jj_reader_LDADD = \
-	$(top_builddir)/src/modules/sched-simple/libjj.la \
+	$(top_builddir)/src/common/libjob/libjob.la \
 	$(test_ldadd)
+sched_simple_jj_reader_LDFLAGS = $(test_ldflags)
 
 shell_plugins_dummy_la_SOURCES = shell/plugins/dummy.c
 shell_plugins_dummy_la_CPPFLAGS = $(test_cppflags)
-shell_plugins_dummy_la_LDFLAGS = -module -rpath /nowhere
+shell_plugins_dummy_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
 
 shell_plugins_conftest_la_SOURCES = shell/plugins/conftest.c
 shell_plugins_conftest_la_CPPFLAGS = $(test_cppflags)
-shell_plugins_conftest_la_LDFLAGS = -module -rpath /nowhere
+shell_plugins_conftest_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
 
 shell_plugins_invalid_args_la_SOURCES = shell/plugins/invalid-args.c
 shell_plugins_invalid_args_la_CPPFLAGS = $(test_cppflags)
-shell_plugins_invalid_args_la_LDFLAGS = -module -rpath /nowhere
+shell_plugins_invalid_args_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
 shell_plugins_invalid_args_la_LIBADD = \
 	$(top_builddir)/src/common/libtap/libtap.la \
-        $(top_builddir)/src/common/libflux-core.la
+	$(top_builddir)/src/common/libflux-core.la
 
 shell_plugins_getopt_la_SOURCES = shell/plugins/getopt.c
 shell_plugins_getopt_la_CPPFLAGS = $(test_cppflags)
-shell_plugins_getopt_la_LDFLAGS = -module -rpath /nowhere
+shell_plugins_getopt_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
 shell_plugins_getopt_la_LIBADD = \
 	$(top_builddir)/src/common/libtap/libtap.la \
-        $(top_builddir)/src/common/libflux-core.la
+	$(top_builddir)/src/common/libflux-core.la
 
 shell_plugins_log_la_SOURCES = shell/plugins/log.c
 shell_plugins_log_la_CPPFLAGS = $(test_cppflags)
-shell_plugins_log_la_LDFLAGS = -module -rpath /nowhere
+shell_plugins_log_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
 shell_plugins_log_la_LIBADD = \
 	$(top_builddir)/src/common/libtap/libtap.la \
-        $(top_builddir)/src/common/libflux-core.la
+	$(top_builddir)/src/common/libflux-core.la
 
 shell_plugins_test_event_la_SOURCES = shell/plugins/test-event.c
 shell_plugins_test_event_la_CPPFLAGS = $(test_cppflags)
-shell_plugins_test_event_la_LDFLAGS = -module -rpath /nowhere
+shell_plugins_test_event_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
 shell_plugins_test_event_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+shell_plugins_jobspec_info_la_SOURCES = shell/plugins/jobspec-info.c
+shell_plugins_jobspec_info_la_CPPFLAGS = $(test_cppflags)
+shell_plugins_jobspec_info_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+shell_plugins_jobspec_info_la_LIBADD = \
+	$(top_builddir)/src/common/libtap/libtap.la \
+	$(top_builddir)/src/common/libflux-core.la
+
+shell_plugins_taskmap_reverse_la_SOURCES = shell/plugins/taskmap-reverse.c
+shell_plugins_taskmap_reverse_la_CPPFLAGS = $(test_cppflags)
+shell_plugins_taskmap_reverse_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+shell_plugins_taskmap_reverse_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_priority_wait_la_SOURCES = \
+	job-manager/plugins/priority-wait.c
+job_manager_plugins_priority_wait_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_priority_wait_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_priority_wait_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_priority_invert_la_SOURCES = \
+	job-manager/plugins/priority-invert.c
+job_manager_plugins_priority_invert_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_priority_invert_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_priority_invert_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_args_la_SOURCES = \
+	job-manager/plugins/args.c
+job_manager_plugins_args_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_args_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_args_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_subscribe_la_SOURCES = \
+	job-manager/plugins/subscribe.c
+job_manager_plugins_subscribe_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_subscribe_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_subscribe_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_test_la_SOURCES = \
+	job-manager/plugins/test.c
+job_manager_plugins_test_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_test_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_test_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_job_aux_la_SOURCES = \
+	job-manager/plugins/job_aux.c
+job_manager_plugins_job_aux_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_job_aux_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_job_aux_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_jobtap_api_la_SOURCES = \
+	job-manager/plugins/jobtap_api.c
+job_manager_plugins_jobtap_api_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_jobtap_api_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_jobtap_api_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_random_la_SOURCES = \
+	job-manager/plugins/random.c
+job_manager_plugins_random_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_random_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_random_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+
+job_manager_plugins_validate_la_SOURCES = \
+	job-manager/plugins/validate.c
+job_manager_plugins_validate_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_validate_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_validate_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_dependency_test_la_SOURCES = \
+	job-manager/plugins/dependency-test.c
+job_manager_plugins_dependency_test_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_dependency_test_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_dependency_test_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_cleanup_event_la_SOURCES = \
+	job-manager/plugins/cleanup-event.c
+job_manager_plugins_cleanup_event_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_cleanup_event_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_cleanup_event_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_create_event_la_SOURCES = \
+	job-manager/plugins/create-event.c
+job_manager_plugins_create_event_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_create_event_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_create_event_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_create_reject_la_SOURCES = \
+	job-manager/plugins/create-reject.c
+job_manager_plugins_create_reject_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_create_reject_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_create_reject_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_config_la_SOURCES = \
+	job-manager/plugins/config.c
+job_manager_plugins_config_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_config_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_config_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_perilog_test_la_SOURCES = \
+	job-manager/plugins/perilog-test.c
+job_manager_plugins_perilog_test_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_perilog_test_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_perilog_test_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_jobspec_update_la_SOURCES = \
+	job-manager/plugins/jobspec-update.c
+job_manager_plugins_jobspec_update_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_jobspec_update_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_jobspec_update_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_jobspec_update_job_list_la_SOURCES = \
+	job-manager/plugins/jobspec-update-job-list.c
+job_manager_plugins_jobspec_update_job_list_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_jobspec_update_job_list_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_jobspec_update_job_list_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_resource_update_expiration_la_SOURCES = \
+	job-manager/plugins/resource-update-expiration.c
+job_manager_plugins_resource_update_expiration_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_resource_update_expiration_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_resource_update_expiration_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_update_test_la_SOURCES = \
+	job-manager/plugins/update-test.c
+job_manager_plugins_update_test_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_update_test_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_update_test_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_project_bank_validate_la_SOURCES = \
+	job-manager/plugins/project-bank-validate.c
+job_manager_plugins_project_bank_validate_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_project_bank_validate_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_project_bank_validate_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+
+job_manager_plugins_offline_la_SOURCES = \
+	job-manager/plugins/offline.c
+job_manager_plugins_offline_la_CPPFLAGS = \
+	$(test_cppflags)
+job_manager_plugins_offline_la_LDFLAGS = \
+        $(fluxplugin_ldflags) -module -rpath /nowhere
+job_manager_plugins_offline_la_LIBADD = \
+        $(top_builddir)/src/common/libflux-internal.la \
         $(top_builddir)/src/common/libflux-core.la
+
+hwloc_hwloc_convert_SOURCES = hwloc/hwloc-convert.c
+hwloc_hwloc_convert_CPPFLAGS = $(HWLOC_CFLAGS) $(test_cppflags)
+hwloc_hwloc_convert_LDADD = $(HWLOC_LIBS) \
+	$(test_ldadd)
+hwloc_hwloc_convert_LDFLAGS = $(test_ldadd)
+
+hwloc_hwloc_version_SOURCES = hwloc/hwloc-version.c
+hwloc_hwloc_version_CPPFLAGS = $(HWLOC_CFLAGS) $(test_cppflags)
+hwloc_hwloc_version_LDADD = $(HWLOC_LIBS) \
+	$(test_ldadd)
+hwloc_hwloc_version_LDFLAGS = $(test_ldadd)
+
+util_jobspec1_validate_SOURCES = util/jobspec1-validate.c
+util_jobspec1_validate_CPPFLAGS = $(test_cppflags)
+util_jobspec1_validate_LDADD = $(test_ldadd)
+util_jobspec1_validate_LDFLAGS = $(test_ldflags)
+
+util_handle_SOURCES = util/handle.c
+util_handle_CPPFLAGS = $(test_cppflags)
+util_handle_LDADD = $(test_ldadd)
+util_handle_LDFLAGS = $(test_ldflags)
+
+util_marshall_SOURCES = util/marshall.c
+util_marshall_CPPFLAGS = $(test_cppflags)
+util_marshall_LDADD = $(test_ldadd)
+util_marshall_LDFLAGS = $(test_ldflags)
+
+stats_stats_basic_la_SOURCES = stats/stats-basic.c
+stats_stats_basic_la_CPPFLAGS = $(test_cppflags)
+stats_stats_basic_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+stats_stats_basic_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
+
+stats_stats_immediate_la_SOURCES = stats/stats-immediate.c
+stats_stats_immediate_la_CPPFLAGS = $(test_cppflags)
+stats_stats_immediate_la_LDFLAGS = \
+	$(fluxplugin_ldflags) -module -rpath /nowhere
+stats_stats_immediate_la_LIBADD = \
+	$(top_builddir)/src/common/libflux-internal.la \
+	$(top_builddir)/src/common/libflux-core.la
diff --git a/t/README.md b/t/README.md
index 2c7a59932807..fd03ed503ab6 100644
--- a/t/README.md
+++ b/t/README.md
@@ -27,11 +27,25 @@ Running tests
 
 Tests may be run in as many as 3 different ways, the easiest
 of which is to issue `make check` from this directory or at
-the top-level flux-core build directory. The tests may also
-all be invoked via the `./runtests.sh` script, which runs
-all tests in turn and aggregates results of all tests at completion.
-Finally, since the tests output TAP, they may be run through a TAP
-harness such as the [prove] command, e.g.
+the top-level flux-core build directory.
+
+Some systems may have poor performance running `make -j N check`
+due to hwloc topology discovery occurring in parallel across many
+tests, each of which may start multiple brokers per test. To
+alleviate this issue, Flux may be directed to read topology from
+an XML file with the `FLUX_HWLOC_XMLFILE` environment variable,
+which avoids most dynamic topology discovery for the entire
+testsuite, e.g.
+
+```
+$ hwloc-ls --of xml >machine.xml
+$ FLUX_HWLOC_XMLFILE=$(pwd)/machine.xml make -j 32 check
+```
+
+The tests may also all be invoked via the `./runtests.sh` script,
+which runs all tests in turn and aggregates results of all tests
+at completion.  Finally, since the tests output TAP, they may be run
+through a TAP harness such as the [prove] command, e.g.
 
 ```
 $ prove --timer ./t*.t
@@ -48,7 +62,7 @@ Result: PASS
 Test scripts may also be run individually, as in:
 
 ```
-$ ./t0001-basic.t 
+$ ./t0001-basic.t
 ok 1 - TEST_NAME is set
 ok 2 - run_timeout works
 ok 3 - we can find a flux binary
@@ -100,15 +114,21 @@ All tests support a standard set of options:
 
 ```
 
+Normally long running tests are not executed.  The environment
+variable `TEST_LONG` may be set to have all long running tests run.
+
 The environment variable `FLUX_TEST_INSTALLED_PATH` may also be set
 to the path to an *installed* version of the `flux(1)` command, for
 running the testsuite against an installed version of flux-core.
 
+The environment variable `FLUX_TEST_VALGRIND` may be set to `t`
+to run tests under valgrind.
+
 
 Skipping Tests
 --------------
 
-The environment variable `SHARNESS_SKIP_TESTS` is a space separated
+The environment variable `SKIP_TESTS` is a space separated
 list of *patterns* that tells which tests to skip, and can either
 match the test number `t[0-]{4}` to skip an entire test script,
 or have an appended `.$number` to skip test `$number` in the
@@ -121,13 +141,7 @@ The test files are by convention named
 
     tNNNN-<name>.sh
 
-where N is a decimal digit. For flux-core tests, only the first digit
-has meaning, where so far the only digits used are:
-
- - 0 - basic tests. Verify testsuite and basic command functionality.
- - 1 - kvs tests
- - 2 - TBD
- - 3 - ...
+where N is a decimal digit.
 
 Writing Tests
 -------------
@@ -140,7 +154,7 @@ After copyright notices, etc, the test script should assign
 to the variable `test_description`, like this:
 
 ```
-test_description='Test basic commands functionality under a comms instance
+test_description='Test basic commands functionality under a Flux instance
 
 Ensure the very basics of flux commands work.
 This suite verifies functionality that may be assumed working by
@@ -182,12 +196,12 @@ The flux-core testsuite extends the sharness [API] with the
 following extra functions:
 
 ```
-  test_under_flux <size> : 
-	Re-invokes the test library under a flux comms instance of
+  test_under_flux <size> :
+	Re-invokes the test library under a flux instance of
 	size <size>. If size is not given a default of 1 is used.
 	This function essentially invokes
-	
-	  exec flux start --size=N /path/to/test/script args...
+
+	  exec flux start --test-size=N /path/to/test/script args...
 
   run_timeout S COMMAND... :
 	Runs COMMAND with timeout of S seconds. run_timeout will
@@ -241,8 +255,8 @@ object methods `say()` to print diagnostics, and `die()` to
 terminate the tests with failure.
 
 --
-[sharness]: https://github.com/mlafeldt/sharness
-[API]: https://github.com/mlafeldt/sharness/blob/master/API.md
+[sharness]: https://github.com/chriscool/sharness
+[API]: https://github.com/chriscool/sharness/blob/master/API.md
 [TAP]: http://testanything.org
 [prove]: http://linux.die.net/man/1/prove
-[lua-TestMore]: http://fperrad.github.io/lua-TestMore/
+[lua-TestMore]: https://fperrad.frama.io/lua-TestMore/
diff --git a/t/barrier/tbarrier.c b/t/barrier/tbarrier.c
index 8a48e70f9f71..cf251329f109 100644
--- a/t/barrier/tbarrier.c
+++ b/t/barrier/tbarrier.c
@@ -14,7 +14,6 @@
 #include <getopt.h>
 #include <assert.h>
 #include <libgen.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "src/common/libutil/log.h"
diff --git a/t/batch/directives/invalid/001-invalid-directive.sh b/t/batch/directives/invalid/001-invalid-directive.sh
new file mode 100644
index 000000000000..f75d98753a18
--- /dev/null
+++ b/t/batch/directives/invalid/001-invalid-directive.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+#
+#FLUX: -N4 -n4
+#FLUX: --job-name=test
+hostname
+#FLUX: --dump
diff --git a/t/batch/directives/invalid/002-sentinel-changed.sh b/t/batch/directives/invalid/002-sentinel-changed.sh
new file mode 100644
index 000000000000..d56957c3856f
--- /dev/null
+++ b/t/batch/directives/invalid/002-sentinel-changed.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+#
+#FLUX: -N4 -n4
+#FLUX: --job-name=test
+#flux: --dump
diff --git a/t/batch/directives/invalid/003-unclosed-multiline.sh b/t/batch/directives/invalid/003-unclosed-multiline.sh
new file mode 100644
index 000000000000..d0f06fb120c6
--- /dev/null
+++ b/t/batch/directives/invalid/003-unclosed-multiline.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+#FLUX: -N4 -n4
+#FLUX: --setattr=user.foo="""
+#FLUX: [table]
+#FLUX: value = "foo"
+#FLUX: ""
+date; hostname
diff --git a/t/batch/directives/invalid/004-unclose-triplequote.sh b/t/batch/directives/invalid/004-unclose-triplequote.sh
new file mode 100644
index 000000000000..114ca41a71bf
--- /dev/null
+++ b/t/batch/directives/invalid/004-unclose-triplequote.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+# flux: --job-name='''It's a "job"
+date; hostname
diff --git a/t/batch/directives/invalid/005-changed-sentinel.sh b/t/batch/directives/invalid/005-changed-sentinel.sh
new file mode 100644
index 000000000000..767fe76d1b6b
--- /dev/null
+++ b/t/batch/directives/invalid/005-changed-sentinel.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+#
+# flux: -N4 --job-name=test
+# flux: --output=flux-{{id}}.out
+#flux: --dump
+#
diff --git a/t/batch/directives/invalid/006-bad-shell-quoting.sh b/t/batch/directives/invalid/006-bad-shell-quoting.sh
new file mode 100644
index 000000000000..ce0abfa2c69b
--- /dev/null
+++ b/t/batch/directives/invalid/006-bad-shell-quoting.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+#
+# flux: --job-name="foo
diff --git a/t/batch/directives/invalid/007-bad-shell-quoting.sh b/t/batch/directives/invalid/007-bad-shell-quoting.sh
new file mode 100644
index 000000000000..0b6100ec7528
--- /dev/null
+++ b/t/batch/directives/invalid/007-bad-shell-quoting.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+#
+# flux: --job-name='foo''
diff --git a/t/batch/directives/invalid/expected/001-invalid-directive.pattern b/t/batch/directives/invalid/expected/001-invalid-directive.pattern
new file mode 100644
index 000000000000..9b2783870dd8
--- /dev/null
+++ b/t/batch/directives/invalid/expected/001-invalid-directive.pattern
@@ -0,0 +1 @@
+ValueError: line 6: orphan 'FLUX:' detected: directives disabled after line 4
diff --git a/t/batch/directives/invalid/expected/002-sentinel-changed.pattern b/t/batch/directives/invalid/expected/002-sentinel-changed.pattern
new file mode 100644
index 000000000000..7351fef06966
--- /dev/null
+++ b/t/batch/directives/invalid/expected/002-sentinel-changed.pattern
@@ -0,0 +1 @@
+ValueError: line 5: sentinel changed from '#FLUX:' to '#flux:'
diff --git a/t/batch/directives/invalid/expected/003-unclosed-multiline.pattern b/t/batch/directives/invalid/expected/003-unclosed-multiline.pattern
new file mode 100644
index 000000000000..575b5a47b34e
--- /dev/null
+++ b/t/batch/directives/invalid/expected/003-unclosed-multiline.pattern
@@ -0,0 +1 @@
+ValueError: line 8: unterminated multi-line quote at line 4: `--setattr=user.foo="""'
diff --git a/t/batch/directives/invalid/expected/004-unclose-triplequote.pattern b/t/batch/directives/invalid/expected/004-unclose-triplequote.pattern
new file mode 100644
index 000000000000..6226ccf92a4f
--- /dev/null
+++ b/t/batch/directives/invalid/expected/004-unclose-triplequote.pattern
@@ -0,0 +1 @@
+ValueError: line 2: unclosed triple quote: --job-name='''It's a "job"
diff --git a/t/batch/directives/invalid/expected/005-changed-sentinel.pattern b/t/batch/directives/invalid/expected/005-changed-sentinel.pattern
new file mode 100644
index 000000000000..5486f3181114
--- /dev/null
+++ b/t/batch/directives/invalid/expected/005-changed-sentinel.pattern
@@ -0,0 +1 @@
+ValueError: line 5: sentinel changed from '# flux:' to '#flux:'
diff --git a/t/batch/directives/invalid/expected/006-bad-shell-quoting.pattern b/t/batch/directives/invalid/expected/006-bad-shell-quoting.pattern
new file mode 100644
index 000000000000..0777dfad03e0
--- /dev/null
+++ b/t/batch/directives/invalid/expected/006-bad-shell-quoting.pattern
@@ -0,0 +1 @@
+ValueError: line 3:  --job-name="foo: No closing quotation
diff --git a/t/batch/directives/invalid/expected/007-bad-shell-quoting.pattern b/t/batch/directives/invalid/expected/007-bad-shell-quoting.pattern
new file mode 100644
index 000000000000..cc802e360be0
--- /dev/null
+++ b/t/batch/directives/invalid/expected/007-bad-shell-quoting.pattern
@@ -0,0 +1 @@
+ValueError: line 3:  --job-name='foo'': No closing quotation
diff --git a/t/batch/directives/valid/001-simple.sh b/t/batch/directives/valid/001-simple.sh
new file mode 100644
index 000000000000..ac506cff03b8
--- /dev/null
+++ b/t/batch/directives/valid/001-simple.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+# simple directives
+#FLUX: -N4                  # Request four nodes
+#FLUX: --queue=batch        # Submit to the batch queue
+#FLUX: --job-name=app001    # Set an explicit job name
+flux run -N4 app
diff --git a/t/batch/directives/valid/002-python.py b/t/batch/directives/valid/002-python.py
new file mode 100644
index 000000000000..f0b736c53fa4
--- /dev/null
+++ b/t/batch/directives/valid/002-python.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+# Directives embedded in python docstring
+def main():
+    """
+    flux: -N4
+    flux: --queue=batch
+    flux: --job-name="my python job"
+
+    # Set some arbitrary user data:
+    flux: --setattr=user.data='''
+    flux: x, y, z
+    flux: a, b, c
+    flux: '''
+    """
+    run()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/t/batch/directives/valid/003-lua.lua b/t/batch/directives/valid/003-lua.lua
new file mode 100644
index 000000000000..5c64ee9b2fd9
--- /dev/null
+++ b/t/batch/directives/valid/003-lua.lua
@@ -0,0 +1,9 @@
+#!/usr/bin/lua
+-- Directives embedded in Lua script:
+--
+-- flux: -N1 --exclusive
+--
+-- flux: --output=job.out # Set output file
+--
+local app = require 'app'
+app.run()
diff --git a/t/batch/directives/valid/004-mixed-comments.sh b/t/batch/directives/valid/004-mixed-comments.sh
new file mode 100644
index 000000000000..4630f65174c2
--- /dev/null
+++ b/t/batch/directives/valid/004-mixed-comments.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+# Set flux directives
+#FLUX: -N1
+# Set job name:
+#FLUX: --job-name=test
+hostname; date
diff --git a/t/batch/directives/valid/005-mixed-comments.py b/t/batch/directives/valid/005-mixed-comments.py
new file mode 100644
index 000000000000..85aa2cb09488
--- /dev/null
+++ b/t/batch/directives/valid/005-mixed-comments.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+"""
+flux: --nodes=4
+
+Set an arbitrary value in jobspec:
+flux: --setattr=user.foo="hello, earth"
+"""
diff --git a/t/batch/directives/valid/006-single-quotes.sh b/t/batch/directives/valid/006-single-quotes.sh
new file mode 100644
index 000000000000..2b8a3410cdc5
--- /dev/null
+++ b/t/batch/directives/valid/006-single-quotes.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+# flux: --setattr=user.data='{"option": "arg"}'
diff --git a/t/batch/directives/valid/007-triple-quotes.sh b/t/batch/directives/valid/007-triple-quotes.sh
new file mode 100644
index 000000000000..f845343c2416
--- /dev/null
+++ b/t/batch/directives/valid/007-triple-quotes.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+# flux: --job-name='''It's a "job"'''
diff --git a/t/batch/directives/valid/008-multiline-with-indent.sh b/t/batch/directives/valid/008-multiline-with-indent.sh
new file mode 100644
index 000000000000..202947c57db2
--- /dev/null
+++ b/t/batch/directives/valid/008-multiline-with-indent.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+# flux: -N4 --exclusive
+# flux: --setattr=user.conf="""
+# flux: [config]
+# flux:   item = "foo"
+# flux: """
+foo
diff --git a/t/batch/directives/valid/009-multiline-comment-char-escaped.sh b/t/batch/directives/valid/009-multiline-comment-char-escaped.sh
new file mode 100644
index 000000000000..4d2e807a93ba
--- /dev/null
+++ b/t/batch/directives/valid/009-multiline-comment-char-escaped.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+# flux: -N4 --exclusive
+# flux: --job-name=foo --setattr=user.conf="""
+# flux: [config]
+# flux:   item = "foo"  # an inline comment
+# flux: # another comment
+# flux: [tab2]
+# flux:   b = 'bar'
+# flux: """
+foo
diff --git a/t/batch/directives/valid/010-noop.sh b/t/batch/directives/valid/010-noop.sh
new file mode 100644
index 000000000000..39429f56a104
--- /dev/null
+++ b/t/batch/directives/valid/010-noop.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+#
+# flux: -n 32
+# flux: # clear environment except for PATH and PYTHONPATH
+# flux: --env=-* --env=PATH --env=PYTHONPATH
diff --git a/t/batch/directives/valid/011-quoted-punctuation.sh b/t/batch/directives/valid/011-quoted-punctuation.sh
new file mode 100644
index 000000000000..8fc7d90b39ad
--- /dev/null
+++ b/t/batch/directives/valid/011-quoted-punctuation.sh
@@ -0,0 +1,6 @@
+#flux: --output='Test_quoted.{{id}}.out'
+#flux: --output=Test_unquoted.{{id}}.out
+#flux: --output=Test_unquoted.-out
+#flux: --output=Test_unquoted.:out
+#flux: --output=Test_unquoted.@out
+#flux: --output=Test[unquoted]
diff --git a/t/batch/directives/valid/expected/001-simple.expected b/t/batch/directives/valid/expected/001-simple.expected
new file mode 100644
index 000000000000..946275cc2f3e
--- /dev/null
+++ b/t/batch/directives/valid/expected/001-simple.expected
@@ -0,0 +1,3 @@
+SETARGS(['-N4'])
+SETARGS(['--queue=batch'])
+SETARGS(['--job-name=app001'])
diff --git a/t/batch/directives/valid/expected/002-python.expected b/t/batch/directives/valid/expected/002-python.expected
new file mode 100644
index 000000000000..25bd3fb9605b
--- /dev/null
+++ b/t/batch/directives/valid/expected/002-python.expected
@@ -0,0 +1,4 @@
+SETARGS(['-N4'])
+SETARGS(['--queue=batch'])
+SETARGS(['--job-name=my python job'])
+SETARGS(['--setattr=user.data=x, y, z\na, b, c\n'])
diff --git a/t/batch/directives/valid/expected/003-lua.expected b/t/batch/directives/valid/expected/003-lua.expected
new file mode 100644
index 000000000000..73ef82f5d764
--- /dev/null
+++ b/t/batch/directives/valid/expected/003-lua.expected
@@ -0,0 +1,2 @@
+SETARGS(['-N1', '--exclusive'])
+SETARGS(['--output=job.out'])
diff --git a/t/batch/directives/valid/expected/004-mixed-comments.expected b/t/batch/directives/valid/expected/004-mixed-comments.expected
new file mode 100644
index 000000000000..6c74e4efad26
--- /dev/null
+++ b/t/batch/directives/valid/expected/004-mixed-comments.expected
@@ -0,0 +1,2 @@
+SETARGS(['-N1'])
+SETARGS(['--job-name=test'])
diff --git a/t/batch/directives/valid/expected/005-mixed-comments.expected b/t/batch/directives/valid/expected/005-mixed-comments.expected
new file mode 100644
index 000000000000..fcd136d71023
--- /dev/null
+++ b/t/batch/directives/valid/expected/005-mixed-comments.expected
@@ -0,0 +1,2 @@
+SETARGS(['--nodes=4'])
+SETARGS(['--setattr=user.foo=hello, earth'])
diff --git a/t/batch/directives/valid/expected/006-single-quotes.expected b/t/batch/directives/valid/expected/006-single-quotes.expected
new file mode 100644
index 000000000000..51d7b1b58723
--- /dev/null
+++ b/t/batch/directives/valid/expected/006-single-quotes.expected
@@ -0,0 +1 @@
+SETARGS(['--setattr=user.data={"option": "arg"}'])
diff --git a/t/batch/directives/valid/expected/007-triple-quotes.expected b/t/batch/directives/valid/expected/007-triple-quotes.expected
new file mode 100644
index 000000000000..021015ce9b09
--- /dev/null
+++ b/t/batch/directives/valid/expected/007-triple-quotes.expected
@@ -0,0 +1 @@
+SETARGS(['--job-name=It\'s a "job"'])
diff --git a/t/batch/directives/valid/expected/008-multiline-with-indent.expected b/t/batch/directives/valid/expected/008-multiline-with-indent.expected
new file mode 100644
index 000000000000..b90e53117fb9
--- /dev/null
+++ b/t/batch/directives/valid/expected/008-multiline-with-indent.expected
@@ -0,0 +1,2 @@
+SETARGS(['-N4', '--exclusive'])
+SETARGS(['--setattr=user.conf=[config]\n  item = "foo"\n'])
diff --git a/t/batch/directives/valid/expected/009-multiline-comment-char-escaped.expected b/t/batch/directives/valid/expected/009-multiline-comment-char-escaped.expected
new file mode 100644
index 000000000000..0986b266dca3
--- /dev/null
+++ b/t/batch/directives/valid/expected/009-multiline-comment-char-escaped.expected
@@ -0,0 +1,2 @@
+SETARGS(['-N4', '--exclusive'])
+SETARGS(['--job-name=foo', '--setattr=user.conf=[config]\n  item = "foo"  # an inline comment\n# another comment\n[tab2]\n  b = \'bar\'\n'])
diff --git a/t/batch/directives/valid/expected/010-noop.expected b/t/batch/directives/valid/expected/010-noop.expected
new file mode 100644
index 000000000000..24c80d28123c
--- /dev/null
+++ b/t/batch/directives/valid/expected/010-noop.expected
@@ -0,0 +1,3 @@
+SETARGS(['-n', '32'])
+NOOP([])
+SETARGS(['--env=-*', '--env=PATH', '--env=PYTHONPATH'])
diff --git a/t/batch/directives/valid/expected/011-quoted-punctuation.expected b/t/batch/directives/valid/expected/011-quoted-punctuation.expected
new file mode 100644
index 000000000000..0e44b1adc80a
--- /dev/null
+++ b/t/batch/directives/valid/expected/011-quoted-punctuation.expected
@@ -0,0 +1,6 @@
+SETARGS(['--output=Test_quoted.{{id}}.out'])
+SETARGS(['--output=Test_unquoted.{{id}}.out'])
+SETARGS(['--output=Test_unquoted.-out'])
+SETARGS(['--output=Test_unquoted.:out'])
+SETARGS(['--output=Test_unquoted.@out'])
+SETARGS(['--output=Test[unquoted]'])
diff --git a/t/batch/jobspec/v0.47.json b/t/batch/jobspec/v0.47.json
new file mode 100644
index 000000000000..1ce988d94741
--- /dev/null
+++ b/t/batch/jobspec/v0.47.json
@@ -0,0 +1,55 @@
+{
+  "attributes": {
+    "system": {
+      "batch": {
+        "broker-opts": [
+          "-Stestattr=foo"
+        ],
+        "script": "#!/bin/sh\ntest $(flux getattr testattr) = \"foo\"\n"
+      },
+      "cwd": "/tmp",
+      "duration": 0,
+      "environment": {
+        "PATH": "/bin:/usr/bin"
+      },
+      "shell": {
+        "options": {
+          "output": {
+            "stdout": {
+              "path": "flux-{{id}}.out",
+              "type": "file"
+            }
+          },
+          "per-resource": {
+            "type": "node"
+          }
+        }
+      }
+    }
+  },
+  "resources": [
+    {
+      "count": 1,
+      "label": "task",
+      "type": "slot",
+      "with": [
+        {
+          "count": 1,
+          "type": "core"
+        }
+      ]
+    }
+  ],
+  "tasks": [
+    {
+      "command": [
+        "hostname"
+      ],
+      "count": {
+        "per_slot": 1
+      },
+      "slot": "task"
+    }
+  ],
+  "version": 1
+}
diff --git a/t/conf.d/bad-hosts/boot.toml b/t/conf.d/bad-hosts/boot.toml
deleted file mode 100644
index c5ca12ff5d6d..000000000000
--- a/t/conf.d/bad-hosts/boot.toml
+++ /dev/null
@@ -1,5 +0,0 @@
-[bootstrap]
-
-hosts = [
-    42
-]
diff --git a/t/conf.d/bad-hosts2/boot.toml b/t/conf.d/bad-hosts2/boot.toml
deleted file mode 100644
index fcb381ff495a..000000000000
--- a/t/conf.d/bad-hosts2/boot.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-[bootstrap]
-
-hosts = 42
diff --git a/t/conf.d/bad-nobootstrap/boot.toml b/t/conf.d/bad-nobootstrap/boot.toml
deleted file mode 100644
index 37fab957c7b2..000000000000
--- a/t/conf.d/bad-nobootstrap/boot.toml
+++ /dev/null
@@ -1,4 +0,0 @@
-
-hosts = [
-    { host = "foo" },
-]
diff --git a/t/conf.d/bad-nomatch/boot.toml b/t/conf.d/bad-nomatch/boot.toml
deleted file mode 100644
index 0c6cf61f3ced..000000000000
--- a/t/conf.d/bad-nomatch/boot.toml
+++ /dev/null
@@ -1,8 +0,0 @@
-[bootstrap]
-
-default_bind = "ipc://@flux-testipc-1-0"
-default_connect = "ipc://@flux-testipc-1-0"
-
-hosts = [
-    { host = "matchnobody" },
-]
diff --git a/t/conf.d/bad-toml/boot.toml b/t/conf.d/bad-toml/boot.toml
deleted file mode 100644
index 40ab7a1c4ba6..000000000000
--- a/t/conf.d/bad-toml/boot.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-[bootstrap]
-
-bad-toml
diff --git a/t/conf.d/good-emptyhosts/boot.toml b/t/conf.d/good-emptyhosts/boot.toml
deleted file mode 100644
index ae6907bd2c25..000000000000
--- a/t/conf.d/good-emptyhosts/boot.toml
+++ /dev/null
@@ -1,4 +0,0 @@
-[bootstrap]
-
-hosts = [
-]
diff --git a/t/conf.d/good-ipc2/boot.toml b/t/conf.d/good-ipc2/boot.toml
deleted file mode 100644
index 0b74052cc8d2..000000000000
--- a/t/conf.d/good-ipc2/boot.toml
+++ /dev/null
@@ -1,6 +0,0 @@
-[bootstrap]
-
-hosts = [
-    { host="fake0", bind="ipc:///tmp/test-ipc2-0", connect="ipc:///tmp/test-ipc2-0" },
-    { host="fake1" }
-]
diff --git a/t/conf.d/good-nohosts/bad-nohosts/boot.toml b/t/conf.d/good-nohosts/bad-nohosts/boot.toml
deleted file mode 100644
index aad8ef162225..000000000000
--- a/t/conf.d/good-nohosts/bad-nohosts/boot.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-[bootstrap]
-
-# missing hosts array
diff --git a/t/conf.d/good-nohosts/boot.toml b/t/conf.d/good-nohosts/boot.toml
deleted file mode 100644
index 0ae814e542c2..000000000000
--- a/t/conf.d/good-nohosts/boot.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-[bootstrap]
-
diff --git a/t/conf.d/good-tcp4/boot.toml b/t/conf.d/good-tcp4/boot.toml
deleted file mode 100644
index 94d6f3e516ed..000000000000
--- a/t/conf.d/good-tcp4/boot.toml
+++ /dev/null
@@ -1,7 +0,0 @@
-[bootstrap]
-
-hosts = [
-    { host="fake0", bind="tcp://127.0.0.1:5080", connect="tcp://127.0.0.1:5080" },
-    { host="fake1", bind="tcp://127.0.0.1:5081", connect="tcp://127.0.0.1:5081" },
-    { host="fake[2-3]" }
-]
diff --git a/t/content/content-helper.sh b/t/content/content-helper.sh
new file mode 100755
index 000000000000..6530a73ec266
--- /dev/null
+++ b/t/content/content-helper.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+
+# content module test helper functions
+
+# Get RPC message contents for checkpoint put
+# Usage: checkpoint_put_msg key rootref
+checkpoint_put_msg() {
+	o="{key:\"$1\",value:{version:1,rootref:\"$2\",timestamp:2.2}}"
+	echo ${o}
+}
+
+# checkpoint rootref at specific key
+# Usage: checkpoint_put key rootref
+checkpoint_put() {
+	o=$(checkpoint_put_msg $1 $2)
+	jq -j -c -n ${o} | $RPC content.checkpoint-put
+}
+
+# Get RPC message contents for checkpoint get
+# Usage: checkpoint_get_msg key
+checkpoint_get_msg() {
+	o="{key:\"$1\"}"
+	echo ${o}
+}
+
+# get checkpoint rootref at key
+# Usage: checkpoint_get key
+checkpoint_get() {
+	o=$(checkpoint_get_msg $1)
+	jq -j -c -n ${o} | $RPC content.checkpoint-get
+}
+
+# Identical to checkpoint_put(), but go directly to backing store
+# Usage: checkpoint_put key rootref
+checkpoint_backing_put() {
+	o=$(checkpoint_put_msg $1 $2)
+	jq -j -c -n ${o} | $RPC content-backing.checkpoint-put
+}
+
+# Identical to checkpoint_get(), but go directly to backing store
+# Usage: checkpoint_get key
+checkpoint_backing_get() {
+	o=$(checkpoint_get_msg $1)
+	jq -j -c -n ${o} | $RPC content-backing.checkpoint-get
+}
diff --git a/t/debug/stall.c b/t/debug/stall.c
new file mode 100644
index 000000000000..8e1454d5067a
--- /dev/null
+++ b/t/debug/stall.c
@@ -0,0 +1,64 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* stall.c - test program for debugger support: stalling until SIGCONT */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <signal.h>
+
+void handle_sigcont (int sig)
+{
+    fprintf (stdout, "Caught SIGCONT\n");
+    exit (0);
+}
+
+int main (int argc, char *argv[])
+{
+    FILE *fptr = NULL;
+    int stall_sec = 0;
+
+    if (argc != 3) {
+        fprintf (stderr, "Usage: stall <filename> <stall_sec>\n");
+        exit (1);
+    }
+
+    signal (SIGCONT, handle_sigcont);
+
+    fprintf (stdout, "Signal handler for SIGCONT installed\n");
+
+    if ( !(fptr = fopen (argv[1], "w"))) {
+        fprintf (stderr, "Error: Can't write to %s\n", argv[1]);
+        exit (1);
+    }
+    fclose (fptr);
+
+    fprintf (stdout, "Sync file created: %s\n", argv[1]);
+
+    if ((stall_sec = atoi (argv[2])) < 0) {
+        fprintf (stderr, "Error: stall time (%d) must be > 0!\n", stall_sec);
+        exit (1);
+    }
+
+    fprintf (stdout, "Will sleep for: %d\n", stall_sec);
+
+    sleep (stall_sec);
+
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/disconnect/watcher.c b/t/disconnect/watcher.c
new file mode 100644
index 000000000000..ec99f376b968
--- /dev/null
+++ b/t/disconnect/watcher.c
@@ -0,0 +1,43 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Try to watch a non-existent key from a module.
+ * Sharness code will verify that watch count goes up, and
+ * then when module unloads, watch count will go down
+ * because broker generated disconnect message.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+int mod_main (flux_t *h, int argc, char *argv[])
+{
+    flux_future_t *f;
+    int rc;
+
+    if (!(f = flux_kvs_lookup (h,
+                               NULL,
+                               FLUX_KVS_WATCH | FLUX_KVS_WAITCREATE,
+                               "noexist"))) {
+        flux_log_error (h, "flux_kvs_lookup");
+        return -1;
+    }
+    if ((rc = flux_reactor_run (flux_get_reactor (h), 0)) < 0)
+        flux_log_error (h, "flux_reactor_run");
+
+    flux_future_destroy (f);
+    return rc;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/flux-jobs/tests/contextual_info/description b/t/flux-jobs/tests/contextual_info/description
new file mode 100644
index 000000000000..6ca87ede4934
--- /dev/null
+++ b/t/flux-jobs/tests/contextual_info/description
@@ -0,0 +1 @@
+check contextual_info field in various scenarios
diff --git a/t/flux-jobs/tests/contextual_info/format b/t/flux-jobs/tests/contextual_info/format
new file mode 100644
index 000000000000..448b86240df6
--- /dev/null
+++ b/t/flux-jobs/tests/contextual_info/format
@@ -0,0 +1 @@
+{contextual_info:h}
diff --git a/t/flux-jobs/tests/contextual_info/input b/t/flux-jobs/tests/contextual_info/input
new file mode 100644
index 000000000000..69a93652f5bd
--- /dev/null
+++ b/t/flux-jobs/tests/contextual_info/input
@@ -0,0 +1,6 @@
+{"id": 49727668224, "userid": 1000, "urgency": 16, "t_submit": 1664571163.9159656, "t_depend": 1664571163.9159656, "state": 4, "name": "hostname", "ntasks": 1, "duration": 0.0}
+{"id": 49727668224, "userid": 1000, "urgency": 16, "priority": 16, "t_submit": 1664571163.9159656, "t_depend": 1664571163.9159656, "state": 8, "name": "hostname", "ntasks": 1, "duration": 0.0, "annotations": {"sched": {"t_estimate": 1664571164.1953933}}}
+{"id": 54391734272, "userid": 1000, "urgency": 16, "t_submit": 1664571164.1953933, "t_depend": 1664571164.1953933, "state": 2, "name": "hostname", "ntasks": 1, "duration": 0.0, "dependencies": ["after-finish=ƒ2ARxLQf"]}
+{"id": 59122909184, "userid": 1000, "urgency": 16, "t_submit": 1664571164.476613, "t_depend": 1664571164.476613, "state": 2, "name": "hostname", "ntasks": 1, "duration": 0.0, "dependencies": ["after-success=ƒ2RsQrcw"]}
+{"id": 44258295808, "userid": 1000, "urgency": 16, "priority": 16, "t_submit": 1664571163.5911362, "t_depend": 1664571163.5911362, "t_run": 1664571163.6055603, "state": 16, "name": "sleep", "ntasks": 16, "duration": 0.0, "nnodes": 4, "ranks": "[0-3]", "nodelist": "asp,asp,asp,asp", "expiration": 0.0, "annotations": {"sched": {"resource_summary": "rank[0-3]/core[0-3]"}}}
+{"id": 35114713088, "userid": 1000, "urgency": 16, "priority": 16, "t_submit": 1664571163.0459371, "t_depend": 1664571163.0459371, "t_run": 1664571163.0629747, "t_cleanup": 1664571163.1124246, "t_inactive": 1664571163.1161304, "state": 64, "name": "hostname", "ntasks": 1, "duration": 0.0, "nnodes": 1, "ranks": "0", "nodelist": "asp", "success": true, "exception_occurred": false, "result": 1, "expiration": 0.0, "annotations": {"sched": {"resource_summary": "rank0/core0"}}, "waitstatus": 0}
diff --git a/t/flux-jobs/tests/contextual_info/output b/t/flux-jobs/tests/contextual_info/output
new file mode 100644
index 000000000000..201163c478ed
--- /dev/null
+++ b/t/flux-jobs/tests/contextual_info/output
@@ -0,0 +1,6 @@
+priority-wait
+eta:now
+depends:after-finish=ƒ2ARxLQf
+depends:after-success=ƒ2RsQrcw
+asp,asp,asp,asp
+asp
diff --git a/t/flux-jobs/tests/issue#2634/format b/t/flux-jobs/tests/issue#2634/format
index 35d473c55427..8bcb9b3aafc6 100644
--- a/t/flux-jobs/tests/issue#2634/format
+++ b/t/flux-jobs/tests/issue#2634/format
@@ -1 +1 @@
-{runtime_fsd_hyphen} {runtime_fsd}
+{runtime!F:h} {runtime!F}
diff --git a/t/flux-jobs/tests/issue#2634/input b/t/flux-jobs/tests/issue#2634/input
index b1dbbd1c8b58..06d8f73bd68d 100644
--- a/t/flux-jobs/tests/issue#2634/input
+++ b/t/flux-jobs/tests/issue#2634/input
@@ -1 +1 @@
-{"id": 406008627200, "userid": 6885, "priority": 16, "t_submit": 1579133377.8522565, "state": 32, "name": "hostname", "ntasks": 128, "t_depend": 1579133377.8522565, "t_sched": 1579133377.8655972, "t_cleanup": 1579133377.8667088, "t_inactive": 1579133377.8667269}
+{"id": 406008627200, "userid": 6885, "urgency": 16, "t_submit": 1579133377.8522565, "state": 32, "name": "hostname", "ntasks": 128, "t_depend": 1579133377.8522565, "t_cleanup": 1579133377.8667088, "t_inactive": 1579133377.8667269}
diff --git a/t/flux-jobs/tests/issue#2658/format b/t/flux-jobs/tests/issue#2658/format
index e2dafba7422a..cb36f26a2036 100644
--- a/t/flux-jobs/tests/issue#2658/format
+++ b/t/flux-jobs/tests/issue#2658/format
@@ -1 +1 @@
-{runtime_fsd}
+{runtime!F}
diff --git a/t/flux-jobs/tests/issue#2658/input b/t/flux-jobs/tests/issue#2658/input
index a40d5fd1ded0..fa512f87b546 100644
--- a/t/flux-jobs/tests/issue#2658/input
+++ b/t/flux-jobs/tests/issue#2658/input
@@ -1,15 +1,15 @@
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579463919.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579463919.405585,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579463920.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579463929.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579463979.3045852,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579463979.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579464520.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579464620.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579467519.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579469920.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579523920.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1579550319.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1580327919.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1679463919.4045851,"t_inactive":1579463919.4712203}
-{"id":207416721408,"userid":1000,"priority":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_sched":1579463919.40202,"t_run":1579463919.4045851,"t_cleanup":1589563919.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579463919.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579463919.405585,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579463920.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579463929.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579463979.3045852,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579463979.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579464520.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579464620.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579467519.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579469920.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579523920.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1579550319.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1580327919.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1679463919.4045851,"t_inactive":1579463919.4712203}
+{"id":207416721408,"userid":1000,"urgency":16,"t_submit":1579463919.3885272,"state":32,"name":"sleep","ntasks":1,"t_depend":1579463919.3885272,"t_run":1579463919.4045851,"t_cleanup":1589563919.4045851,"t_inactive":1579463919.4712203}
diff --git a/t/flux-jobs/tests/issue#5827/description b/t/flux-jobs/tests/issue#5827/description
new file mode 100644
index 000000000000..5ed2ed7cd19d
--- /dev/null
+++ b/t/flux-jobs/tests/issue#5827/description
@@ -0,0 +1 @@
+D conversion flag outputs empty string on epoch timestamp
diff --git a/t/flux-jobs/tests/issue#5827/format b/t/flux-jobs/tests/issue#5827/format
new file mode 100644
index 000000000000..25e735effda4
--- /dev/null
+++ b/t/flux-jobs/tests/issue#5827/format
@@ -0,0 +1 @@
+{t_run},{t_run!D}
diff --git a/t/flux-jobs/tests/issue#5827/input b/t/flux-jobs/tests/issue#5827/input
new file mode 100644
index 000000000000..9a676982336f
--- /dev/null
+++ b/t/flux-jobs/tests/issue#5827/input
@@ -0,0 +1 @@
+{"id": 375843192832, "userid": 1000, "urgency": 16, "priority": 0, "t_submit": 1711580059.6155457, "t_depend": 1711580059.6265068, "state": 8, "name": "sleep", "cwd": "/tmp/foo", "ntasks": 1, "ncores": 1, "duration": 0.0}
diff --git a/t/flux-jobs/tests/issue#5827/output b/t/flux-jobs/tests/issue#5827/output
new file mode 100644
index 000000000000..4ac540fc343c
--- /dev/null
+++ b/t/flux-jobs/tests/issue#5827/output
@@ -0,0 +1 @@
+0.0,
diff --git a/t/flux-jobs/tests/returncode/description b/t/flux-jobs/tests/returncode/description
new file mode 100644
index 000000000000..95ea9a7c2c7f
--- /dev/null
+++ b/t/flux-jobs/tests/returncode/description
@@ -0,0 +1 @@
+check job returncode in various scenarios
diff --git a/t/flux-jobs/tests/returncode/format b/t/flux-jobs/tests/returncode/format
new file mode 100644
index 000000000000..e072de9c6f94
--- /dev/null
+++ b/t/flux-jobs/tests/returncode/format
@@ -0,0 +1 @@
+{returncode:h}
diff --git a/t/flux-jobs/tests/returncode/input b/t/flux-jobs/tests/returncode/input
new file mode 100644
index 000000000000..51ad6d202864
--- /dev/null
+++ b/t/flux-jobs/tests/returncode/input
@@ -0,0 +1,6 @@
+{"id": 106958577205248, "userid": 1000, "urgency": 16, "priority": 16, "t_submit": 1607972908.4363101, "state": 16, "name": "sleep", "ntasks": 16, "nnodes": 4, "ranks": "[0-3]", "nodelist": "asp,asp,asp,asp", "expiration": 0.0, "t_depend": 1607972908.4363101, "t_run": 1607972908.4532418, "annotations": {"sched": {"resource_summary": "rank[0-3]/core[0-3]"}}}
+{"id": 97053292101632, "userid": 1000, "urgency": 16, "priority": 16, "t_submit": 1607972318.0359256, "state": 64, "name": "sleep", "ntasks": 16, "nnodes": 4, "ranks": "[0-3]", "nodelist": "asp,asp,asp,asp", "expiration": 0.0, "success": true, "exception_occurred": false, "result": 1, "waitstatus": 15, "t_depend": 1607972318.0359256, "t_run": 1607972318.0530958, "t_cleanup": 1607972618.1682575, "t_inactive": 1607972618.1734343, "annotations": {"sched": {"resource_summary": "rank[0-3]/core[0-3]"}}}
+{"id": 98083329277952, "userid": 1000, "urgency": 16, "priority": 16, "t_submit": 1607972379.4304638, "state": 64, "name": "sleep", "ntasks": 1, "success": false, "exception_occurred": true, "exception_severity": 0, "exception_type": "cancel", "exception_note": "interrupted by ctrl-C", "result": 4, "t_depend": 1607972379.4304638, "t_cleanup": 1607972380.1219881, "t_inactive": 1607972380.1224892}
+{"id": 96775562067968, "userid": 1000, "urgency": 16, "priority": 16, "t_submit": 1607972301.4822261, "state": 64, "name": "sh", "ntasks": 1, "nnodes": 1, "ranks": "0", "nodelist": "asp", "expiration": 0.0, "success": false, "exception_occurred": false, "result": 2, "waitstatus": 10752, "t_depend": 1607972301.4822261, "t_run": 1607972301.4992967, "t_cleanup": 1607972301.5430801, "t_inactive": 1607972301.5474372, "annotations": {"sched": {"resource_summary": "rank0/core0"}}}
+{"id": 96619970166784, "userid": 1000, "urgency": 16, "priority": 16, "t_submit": 1607972292.2074587, "state": 64, "name": "false", "ntasks": 1, "nnodes": 1, "ranks": "0", "nodelist": "asp", "expiration": 0.0, "success": false, "exception_occurred": false, "result": 2, "waitstatus": 256, "t_depend": 1607972292.2074587, "t_run": 1607972292.2241008, "t_cleanup": 1607972292.2736692, "t_inactive": 1607972292.2778435, "annotations": {"sched": {"resource_summary": "rank0/core0"}}}
+{"id": 96444648259584, "userid": 1000, "urgency": 16, "priority": 16, "t_submit": 1607972281.7579372, "state": 64, "name": "true", "ntasks": 1, "nnodes": 1, "ranks": "0", "nodelist": "asp", "expiration": 0.0, "success": true, "exception_occurred": false, "result": 1, "waitstatus": 0, "t_depend": 1607972281.7579372, "t_run": 1607972281.7752936, "t_cleanup": 1607972281.8277526, "t_inactive": 1607972281.832587, "annotations": {"sched": {"resource_summary": "rank0/core0"}}}
diff --git a/t/flux-jobs/tests/returncode/output b/t/flux-jobs/tests/returncode/output
new file mode 100644
index 000000000000..265e995f70f6
--- /dev/null
+++ b/t/flux-jobs/tests/returncode/output
@@ -0,0 +1,6 @@
+-
+-15
+-128
+42
+1
+0
diff --git a/t/flux-resource/list/fluke-info.expected b/t/flux-resource/list/fluke-info.expected
new file mode 100644
index 000000000000..e9bab4f868f7
--- /dev/null
+++ b/t/flux-resource/list/fluke-info.expected
@@ -0,0 +1 @@
+101 Nodes, 404 Cores, 0 GPUs
diff --git a/t/flux-resource/list/fluke.R b/t/flux-resource/list/fluke.R
new file mode 100644
index 000000000000..b3b8f17e93a7
--- /dev/null
+++ b/t/flux-resource/list/fluke.R
@@ -0,0 +1 @@
+{"version": 1, "execution": {"R_lite": [{"rank": "0-100", "children": {"core": "0-3"}}], "starttime": 0.0, "expiration": 0.0, "nodelist": ["fluke[3-103]"], "properties": {"batch": "0-92", "debug": "93-100"}}}
diff --git a/t/flux-resource/list/fluke.config b/t/flux-resource/list/fluke.config
new file mode 100644
index 000000000000..4f8d2d7b4bfe
--- /dev/null
+++ b/t/flux-resource/list/fluke.config
@@ -0,0 +1,14 @@
+{
+  "queues": {
+    "batch": {
+      "requires": [
+        "batch"
+      ]
+    },
+    "debug": {
+      "requires": [
+        "debug"
+      ]
+    }
+  }
+}
diff --git a/t/flux-resource/list/fluke.expected b/t/flux-resource/list/fluke.expected
new file mode 100644
index 000000000000..a41a87ebd5ae
--- /dev/null
+++ b/t/flux-resource/list/fluke.expected
@@ -0,0 +1,6 @@
+     STATE PROPERTIES NNODES   NCORES    NGPUS
+      free batch          83      332        0
+      free debug           5       20        0
+ allocated                 0        0        0
+      down batch          10       40        0
+      down debug           3       12        0
diff --git a/t/flux-resource/list/fluke.json b/t/flux-resource/list/fluke.json
new file mode 100644
index 000000000000..cdd4e6a6e785
--- /dev/null
+++ b/t/flux-resource/list/fluke.json
@@ -0,0 +1,57 @@
+{
+  "all": {
+    "version": 1,
+    "execution": {
+      "R_lite": [
+        {
+          "rank": "0-100",
+          "children": {
+            "core": "0-3"
+          }
+        }
+      ],
+      "starttime": 0.0,
+      "expiration": 0.0,
+      "nodelist": [
+        "fluke[3-103]"
+      ],
+      "properties": {
+        "debug": "93-100",
+        "batch": "0-92"
+      }
+    }
+  },
+  "down": {
+    "version": 1,
+    "execution": {
+      "R_lite": [
+        {
+          "rank": "14-15,21,40,58,62,73-74,76,89,94-95,100",
+          "children": {
+            "core": "0-3"
+          }
+        }
+      ],
+      "starttime": 0.0,
+      "expiration": 0.0,
+      "nodelist": [
+        "fluke[17-18,24,43,61,65,76-77,79,92,97-98,103]"
+      ],
+      "properties": {
+        "debug": "94-95,100",
+        "batch": "14-15,21,40,58,62,73-74,76,89"
+      }
+    }
+  },
+  "allocated": {
+    "version": 1,
+    "execution": {
+      "R_lite": [],
+      "starttime": 0.0,
+      "expiration": 0.0,
+      "nodelist": [
+        ""
+      ]
+    }
+  }
+}
diff --git a/t/flux-resource/list/fluxion-info.expected b/t/flux-resource/list/fluxion-info.expected
new file mode 100644
index 000000000000..0d3690da2ff8
--- /dev/null
+++ b/t/flux-resource/list/fluxion-info.expected
@@ -0,0 +1 @@
+4 Nodes, 16 Cores, 0 GPUs
diff --git a/t/flux-resource/list/fluxion.R b/t/flux-resource/list/fluxion.R
new file mode 100644
index 000000000000..35770b5f87ce
--- /dev/null
+++ b/t/flux-resource/list/fluxion.R
@@ -0,0 +1 @@
+{"version": 1, "execution": {"R_lite": [{"rank": "0-3", "children": {"core": "0-3"}}], "starttime": 0.0, "expiration": 0.0}}
diff --git a/t/flux-resource/list/fluxion.expected b/t/flux-resource/list/fluxion.expected
new file mode 100644
index 000000000000..f9fd32d2af9d
--- /dev/null
+++ b/t/flux-resource/list/fluxion.expected
@@ -0,0 +1,4 @@
+     STATE PROPERTIES NNODES   NCORES    NGPUS
+      free                 3        8        0
+ allocated                 2        4        0
+      down                 1        4        0
diff --git a/t/flux-resource/list/fluxion.json b/t/flux-resource/list/fluxion.json
new file mode 100644
index 000000000000..3c2fb4ae03a2
--- /dev/null
+++ b/t/flux-resource/list/fluxion.json
@@ -0,0 +1,78 @@
+{
+  "all": {
+    "version": 1,
+    "execution": {
+      "R_lite": [
+        {
+          "rank": "0",
+          "node": "5b12c7ea7263",
+          "children": {
+            "core": "0-3"
+          }
+        },
+        {
+          "rank": "1",
+          "node": "5b12c7ea7263",
+          "children": {
+            "core": "0-3"
+          }
+        },
+        {
+          "rank": "2",
+          "node": "5b12c7ea7263",
+          "children": {
+            "core": "0-3"
+          }
+        },
+        {
+          "rank": "3",
+          "node": "5b12c7ea7263",
+          "children": {
+            "core": "0-3"
+          }
+        }
+      ],
+      "starttime": 0,
+      "expiration": 0
+    }
+  },
+  "down": {
+    "version": 1,
+    "execution": {
+      "R_lite": [
+        {
+          "rank": "3",
+          "node": "5b12c7ea7263",
+          "children": {
+            "core": "0-3"
+          }
+        }
+      ],
+      "starttime": 0,
+      "expiration": 0
+    }
+  },
+  "allocated": {
+    "version": 1,
+    "execution": {
+      "R_lite": [
+        {
+          "rank": "0",
+          "node": "5b12c7ea7263",
+          "children": {
+            "core": "2-3"
+          }
+        },
+        {
+          "rank": "1",
+          "node": "5b12c7ea7263",
+          "children": {
+            "core": "2-3"
+          }
+        }
+      ],
+      "starttime": 0,
+      "expiration": 0
+    }
+  }
+}
diff --git a/t/flux-resource/list/missing-info.expected b/t/flux-resource/list/missing-info.expected
new file mode 100644
index 000000000000..0d3690da2ff8
--- /dev/null
+++ b/t/flux-resource/list/missing-info.expected
@@ -0,0 +1 @@
+4 Nodes, 16 Cores, 0 GPUs
diff --git a/t/flux-resource/list/missing.R b/t/flux-resource/list/missing.R
new file mode 100644
index 000000000000..35770b5f87ce
--- /dev/null
+++ b/t/flux-resource/list/missing.R
@@ -0,0 +1 @@
+{"version": 1, "execution": {"R_lite": [{"rank": "0-3", "children": {"core": "0-3"}}], "starttime": 0.0, "expiration": 0.0}}
diff --git a/t/flux-resource/list/missing.expected b/t/flux-resource/list/missing.expected
new file mode 100644
index 000000000000..e192f109f61a
--- /dev/null
+++ b/t/flux-resource/list/missing.expected
@@ -0,0 +1,4 @@
+     STATE PROPERTIES NNODES   NCORES    NGPUS
+      free                 4       16        0
+ allocated                 0        0        0
+      down                 0        0        0
diff --git a/t/flux-resource/list/missing.json b/t/flux-resource/list/missing.json
new file mode 100644
index 000000000000..b9da2e13a64f
--- /dev/null
+++ b/t/flux-resource/list/missing.json
@@ -0,0 +1,15 @@
+{
+  "all": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-3"
+        }
+      ]
+    },
+    "version": 1
+  }
+}
diff --git a/t/flux-resource/list/normal-input-info.expected b/t/flux-resource/list/normal-input-info.expected
new file mode 100644
index 000000000000..0d3690da2ff8
--- /dev/null
+++ b/t/flux-resource/list/normal-input-info.expected
@@ -0,0 +1 @@
+4 Nodes, 16 Cores, 0 GPUs
diff --git a/t/flux-resource/list/normal-input.R b/t/flux-resource/list/normal-input.R
new file mode 100644
index 000000000000..35770b5f87ce
--- /dev/null
+++ b/t/flux-resource/list/normal-input.R
@@ -0,0 +1 @@
+{"version": 1, "execution": {"R_lite": [{"rank": "0-3", "children": {"core": "0-3"}}], "starttime": 0.0, "expiration": 0.0}}
diff --git a/t/flux-resource/list/normal-input.expected b/t/flux-resource/list/normal-input.expected
new file mode 100644
index 000000000000..e192f109f61a
--- /dev/null
+++ b/t/flux-resource/list/normal-input.expected
@@ -0,0 +1,4 @@
+     STATE PROPERTIES NNODES   NCORES    NGPUS
+      free                 4       16        0
+ allocated                 0        0        0
+      down                 0        0        0
diff --git a/t/flux-resource/list/normal-input.json b/t/flux-resource/list/normal-input.json
new file mode 100644
index 000000000000..63d535be4de3
--- /dev/null
+++ b/t/flux-resource/list/normal-input.json
@@ -0,0 +1,27 @@
+{
+  "all": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-3"
+        }
+      ]
+    },
+    "version": 1
+  },
+  "allocated": {
+    "execution": {
+      "R_lite": []
+    },
+    "version": 1
+  },
+  "down": {
+    "execution": {
+      "R_lite": []
+    },
+    "version": 1
+  }
+}
diff --git a/t/flux-resource/list/normal-new-info.expected b/t/flux-resource/list/normal-new-info.expected
new file mode 100644
index 000000000000..a28cd9233dd3
--- /dev/null
+++ b/t/flux-resource/list/normal-new-info.expected
@@ -0,0 +1 @@
+5 Nodes, 20 Cores, 0 GPUs
diff --git a/t/flux-resource/list/normal-new.R b/t/flux-resource/list/normal-new.R
new file mode 100644
index 000000000000..7a8e53fe0898
--- /dev/null
+++ b/t/flux-resource/list/normal-new.R
@@ -0,0 +1 @@
+{"version": 1, "execution": {"R_lite": [{"rank": "0-4", "children": {"core": "0-3"}}], "starttime": 0.0, "expiration": 0.0, "nodelist": ["pi[3,0-2,4]"], "properties": {"8g": "0-2"}}}
diff --git a/t/flux-resource/list/normal-new.expected b/t/flux-resource/list/normal-new.expected
new file mode 100644
index 000000000000..d3e19bf1536f
--- /dev/null
+++ b/t/flux-resource/list/normal-new.expected
@@ -0,0 +1,5 @@
+     STATE PROPERTIES NNODES   NCORES    NGPUS
+      free 8g              3       12        0
+      free                 2        8        0
+ allocated                 0        0        0
+      down                 0        0        0
diff --git a/t/flux-resource/list/normal-new.json b/t/flux-resource/list/normal-new.json
new file mode 100644
index 000000000000..9a691d0fd6c5
--- /dev/null
+++ b/t/flux-resource/list/normal-new.json
@@ -0,0 +1,25 @@
+{
+  "all": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-4"
+        }
+      ],
+      "expiration": 0,
+      "nodelist": [
+        "pi[3,0-2,4]"
+      ],
+      "properties": {
+        "8g": "0-2"
+      },
+      "starttime": 0
+    },
+    "version": 1
+  },
+  "allocated": null,
+  "down": null
+}
diff --git a/t/flux-resource/list/null-info.expected b/t/flux-resource/list/null-info.expected
new file mode 100644
index 000000000000..0d3690da2ff8
--- /dev/null
+++ b/t/flux-resource/list/null-info.expected
@@ -0,0 +1 @@
+4 Nodes, 16 Cores, 0 GPUs
diff --git a/t/flux-resource/list/null.R b/t/flux-resource/list/null.R
new file mode 100644
index 000000000000..35770b5f87ce
--- /dev/null
+++ b/t/flux-resource/list/null.R
@@ -0,0 +1 @@
+{"version": 1, "execution": {"R_lite": [{"rank": "0-3", "children": {"core": "0-3"}}], "starttime": 0.0, "expiration": 0.0}}
diff --git a/t/flux-resource/list/null.expected b/t/flux-resource/list/null.expected
new file mode 100644
index 000000000000..e192f109f61a
--- /dev/null
+++ b/t/flux-resource/list/null.expected
@@ -0,0 +1,4 @@
+     STATE PROPERTIES NNODES   NCORES    NGPUS
+      free                 4       16        0
+ allocated                 0        0        0
+      down                 0        0        0
diff --git a/t/flux-resource/list/null.json b/t/flux-resource/list/null.json
new file mode 100644
index 000000000000..4bac8bc036da
--- /dev/null
+++ b/t/flux-resource/list/null.json
@@ -0,0 +1,17 @@
+{
+  "all": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-3"
+        }
+      ]
+    },
+    "version": 1
+  },
+  "allocated": null,
+  "down": null
+}
diff --git a/t/flux-resource/list/properties-info.expected b/t/flux-resource/list/properties-info.expected
new file mode 100644
index 000000000000..0d3690da2ff8
--- /dev/null
+++ b/t/flux-resource/list/properties-info.expected
@@ -0,0 +1 @@
+4 Nodes, 16 Cores, 0 GPUs
diff --git a/t/flux-resource/status/drain.expected b/t/flux-resource/status/drain.expected
new file mode 100644
index 000000000000..01969e13920f
--- /dev/null
+++ b/t/flux-resource/status/drain.expected
@@ -0,0 +1,3 @@
+     STATE NNODES RANKS           NODELIST
+     avail      2 2-3             foo[3-4]
+   drained      2 0-1             foo[1-2]
diff --git a/t/flux-resource/status/drain.json b/t/flux-resource/status/drain.json
new file mode 100644
index 000000000000..7300c9f84799
--- /dev/null
+++ b/t/flux-resource/status/drain.json
@@ -0,0 +1,34 @@
+{
+  "R": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-3"
+        }
+      ],
+      "expiration": 0,
+      "nodelist": [
+        "foo[1-4]"
+      ],
+      "starttime": 0
+    },
+    "version": 1
+  },
+  "drain": {
+    "0": {
+      "reason": "bad node",
+      "timestamp": 1607544031.1368911
+    },
+    "1": {
+      "reason": "testing drain",
+      "timestamp": 1607544037.2190773
+    }
+  },
+  "exclude": "",
+  "torpid": "",
+  "offline": "",
+  "online": "0-3"
+}
diff --git a/t/flux-resource/status/example.expected b/t/flux-resource/status/example.expected
new file mode 100644
index 000000000000..1e8363cf56d4
--- /dev/null
+++ b/t/flux-resource/status/example.expected
@@ -0,0 +1,5 @@
+     STATE NNODES RANKS           NODELIST
+     avail      2 2-3             foo[3-4]
+   exclude      1 0               foo1
+   drained      1 0               foo1
+  drained*      1 1               foo2
diff --git a/t/flux-resource/status/example.json b/t/flux-resource/status/example.json
new file mode 100644
index 000000000000..0870469204a5
--- /dev/null
+++ b/t/flux-resource/status/example.json
@@ -0,0 +1,34 @@
+{
+  "R": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-3"
+        }
+      ],
+      "expiration": 0,
+      "nodelist": [
+        "foo[1-4]"
+      ],
+      "starttime": 0
+    },
+    "version": 1
+  },
+  "drain": {
+    "0": {
+      "reason": "bad node",
+      "timestamp": 1607544031.1368911
+    },
+    "1": {
+      "reason": "testing drain",
+      "timestamp": 1607544037.2190773
+    }
+  },
+  "exclude": "0",
+  "torpid": "",
+  "offline": "1",
+  "online": "0,2-3"
+}
diff --git a/t/flux-resource/status/exclude.expected b/t/flux-resource/status/exclude.expected
new file mode 100644
index 000000000000..8f4f87ca02d6
--- /dev/null
+++ b/t/flux-resource/status/exclude.expected
@@ -0,0 +1,4 @@
+     STATE NNODES RANKS           NODELIST
+     avail      2 2-3             foo[3-4]
+    avail*      1 1               foo2
+   exclude      1 0               foo1
diff --git a/t/flux-resource/status/exclude.json b/t/flux-resource/status/exclude.json
new file mode 100644
index 000000000000..e111b8ce49f3
--- /dev/null
+++ b/t/flux-resource/status/exclude.json
@@ -0,0 +1,25 @@
+{
+  "R": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-3"
+        }
+      ],
+      "expiration": 0,
+      "nodelist": [
+        "foo[1-4]"
+      ],
+      "starttime": 0
+    },
+    "version": 1
+  },
+  "drain": {},
+  "exclude": "0",
+  "torpid": "",
+  "offline": "1",
+  "online": "0,2-3"
+}
diff --git a/t/flux-resource/status/fluke.config b/t/flux-resource/status/fluke.config
new file mode 100644
index 000000000000..4f8d2d7b4bfe
--- /dev/null
+++ b/t/flux-resource/status/fluke.config
@@ -0,0 +1,14 @@
+{
+  "queues": {
+    "batch": {
+      "requires": [
+        "batch"
+      ]
+    },
+    "debug": {
+      "requires": [
+        "debug"
+      ]
+    }
+  }
+}
diff --git a/t/flux-resource/status/fluke.expected b/t/flux-resource/status/fluke.expected
new file mode 100644
index 000000000000..cf87c6f26f30
--- /dev/null
+++ b/t/flux-resource/status/fluke.expected
@@ -0,0 +1,5 @@
+     STATE NNODES RANKS           NODELIST
+     avail     85 3-13,16-20,22-39,41-57,59-61,63-72,75,77-88,90-93,96-99 fluke[6-16,19-23,25-42,44-60,62-64,66-75,78,80-91,93-96,99-102]
+    avail*     11 14,21,58,62,73-74,76,89,94-95,100 fluke[17,24,61,65,76-77,79,92,97-98,103]
+   exclude      3 0-2             fluke[1,3,108]
+   drained      2 15,40           fluke[18,43]
diff --git a/t/flux-resource/status/fluke.json b/t/flux-resource/status/fluke.json
new file mode 100644
index 000000000000..e9db29e0967b
--- /dev/null
+++ b/t/flux-resource/status/fluke.json
@@ -0,0 +1,39 @@
+{
+  "R": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-100"
+        }
+      ],
+      "expiration": 0.0,
+      "nodelist": [
+        "fluke[1,3,108,6-103]"
+      ],
+      "properties": {
+        "batch": "3-92",
+        "debug": "93-100",
+        "testprop": "99-100"
+      },
+      "starttime": 0.0
+    },
+    "version": 1
+  },
+  "drain": {
+    "15": {
+      "reason": "broker was unresponsive",
+      "timestamp": 1709673393.7776122
+    },
+    "40": {
+      "reason": "broker was unresponsive",
+      "timestamp": 1711663831.1545894
+    }
+  },
+  "exclude": "0-2",
+  "torpid": "",
+  "offline": "14,21,58,62,73-74,76,89,94-95,100",
+  "online": "0-13,15-20,22-57,59-61,63-72,75,77-88,90-93,96-99"
+}
diff --git a/t/flux-resource/status/offline.expected b/t/flux-resource/status/offline.expected
new file mode 100644
index 000000000000..c74e8e43010a
--- /dev/null
+++ b/t/flux-resource/status/offline.expected
@@ -0,0 +1,3 @@
+     STATE NNODES RANKS           NODELIST
+     avail      3 0,2-3           foo[1,3-4]
+    avail*      1 1               foo2
diff --git a/t/flux-resource/status/offline.json b/t/flux-resource/status/offline.json
new file mode 100644
index 000000000000..1d2e3d91b18f
--- /dev/null
+++ b/t/flux-resource/status/offline.json
@@ -0,0 +1,25 @@
+{
+  "R": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-3"
+        }
+      ],
+      "expiration": 0,
+      "nodelist": [
+        "foo[1-4]"
+      ],
+      "starttime": 0
+    },
+    "version": 1
+  },
+  "drain": {},
+  "exclude": "",
+  "torpid": "",
+  "offline": "1",
+  "online": "0,2-3"
+}
diff --git a/t/flux-resource/status/simple.expected b/t/flux-resource/status/simple.expected
new file mode 100644
index 000000000000..e3e9299d63f8
--- /dev/null
+++ b/t/flux-resource/status/simple.expected
@@ -0,0 +1,2 @@
+     STATE NNODES RANKS           NODELIST
+     avail      4 0-3             foo[1-4]
diff --git a/t/flux-resource/status/simple.json b/t/flux-resource/status/simple.json
new file mode 100644
index 000000000000..6496c7ca1f58
--- /dev/null
+++ b/t/flux-resource/status/simple.json
@@ -0,0 +1,25 @@
+{
+  "R": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-3"
+        }
+      ],
+      "expiration": 0,
+      "nodelist": [
+        "foo[1-4]"
+      ],
+      "starttime": 0
+    },
+    "version": 1
+  },
+  "drain": {},
+  "exclude": "",
+  "torpid": "",
+  "offline": "",
+  "online": "0-3"
+}
diff --git a/t/flux-resource/status/torpid.expected b/t/flux-resource/status/torpid.expected
new file mode 100644
index 000000000000..fd107216558c
--- /dev/null
+++ b/t/flux-resource/status/torpid.expected
@@ -0,0 +1,4 @@
+     STATE NNODES RANKS           NODELIST
+     avail      1 2               foo3
+    torpid      1 3               foo4
+   drained      2 0-1             foo[1-2]
diff --git a/t/flux-resource/status/torpid.json b/t/flux-resource/status/torpid.json
new file mode 100644
index 000000000000..1d5bdad13b2c
--- /dev/null
+++ b/t/flux-resource/status/torpid.json
@@ -0,0 +1,34 @@
+{
+  "R": {
+    "execution": {
+      "R_lite": [
+        {
+          "children": {
+            "core": "0-3"
+          },
+          "rank": "0-3"
+        }
+      ],
+      "expiration": 0,
+      "nodelist": [
+        "foo[1-4]"
+      ],
+      "starttime": 0
+    },
+    "version": 1
+  },
+  "drain": {
+    "0": {
+      "reason": "bad node",
+      "timestamp": 1607544031.1368911
+    },
+    "1": {
+      "reason": "testing drain",
+      "timestamp": 1607544037.2190773
+    }
+  },
+  "exclude": "",
+  "torpid": "1,3",
+  "offline": "",
+  "online": "0-3"
+}
diff --git a/t/fluxometer.lua b/t/fluxometer.lua
index 9ee661e656d8..73b68b152942 100644
--- a/t/fluxometer.lua
+++ b/t/fluxometer.lua
@@ -33,6 +33,7 @@
 --  If FLUXOMETER_LUA_PATH is set, place this path at the front of
 --  package.path so we load the same fluxometer.conf as before.
 --
+local unpack = table.unpack or unpack
 local fpath = os.getenv ("FLUXOMETER_LUA_PATH")
 if fpath then
     package.path = fpath .. ';' .. package.path
@@ -45,12 +46,6 @@ fluxTest.__index = fluxTest
 local getopt = require 'flux.alt_getopt'.get_opts
 local posix = require 'flux.posix'
 
---  Options:
-local cmdline_opts = {
-    help =     { char = 'h' },
-    verbose =  { char = 'v' }
-}
-
 ---
 --  Append path p to PATH environment variable:
 --
@@ -70,7 +65,7 @@ function fluxTest:start_session (t)
     local extra_args = t.args or {}
     local cmd = { self.flux_path, "start",
                   unpack (self.start_args),
-                  string.format ("--size=%d", size) }
+                  string.format ("--test-size=%d", size) }
 
     if t.args then
         for _,v in pairs (t.args) do
@@ -80,6 +75,10 @@ function fluxTest:start_session (t)
 
     table.insert (cmd, self.arg0)
 
+    if (self.opts.r) then
+        table.insert(cmd, "--root="..self.opts.r)
+    end
+
     -- Set FLUXOMETER_LUA_PATH to ensure we load the same fluxometer.conf
     --  after `flux start ...`
     posix.setenv ("FLUXOMETER_LUA_PATH", self.src_dir..'/?.lua')
@@ -116,6 +115,13 @@ function fluxTest.init (...)
     test.arg0 = debug.getinfo (2).source:sub (2)
     test.prog = test.arg0:match ("/*([^/]+)%.t")
 
+    -- Support the same common arguments as sharness:
+    test.opts, optind = getopt (arg, "dvr:",
+                                 { debug = 'd',
+                                   verbose = 'v',
+                                   root = 'r'
+                                 })
+
     local cwd, err = posix.getcwd ()
     if not cwd then error (err) end
     test.src_dir = cwd
@@ -127,7 +133,7 @@ function fluxTest.init (...)
     end
 
     test.log_file = "lua-"..test.prog..".broker.log"
-    test.start_args = { "-o,-Slog-filename=" .. test.log_file }
+    test.start_args = { "-Slog-filename=" .. test.log_file }
 
     local path = fluxTest.fluxbindir .. "/flux"
     local mode = posix.stat (path, 'mode')
@@ -139,9 +145,12 @@ function fluxTest.init (...)
     end
 
     test.trash_dir = "trash-directory.lua-"..test.prog
+    if test.opts.r then
+        test.trash_dir = test.opts.r .. "/" .. test.trash_dir
+    end
     if test.flux_active then
         os.execute ("rm -rf "..test.trash_dir)
-        posix.mkdir (test.trash_dir)
+        os.execute ("mkdir -p "..test.trash_dir)
         posix.chdir (test.trash_dir)
         cleanup (function ()
                      posix.chdir (test.src_dir)
diff --git a/t/hwloc-data/1N/hwloc-versions/v1.11.11/0.xml b/t/hwloc-data/1N/hwloc-versions/v1.11.11/0.xml
new file mode 100644
index 000000000000..5e367c7ffc7b
--- /dev/null
+++ b/t/hwloc-data/1N/hwloc-versions/v1.11.11/0.xml
@@ -0,0 +1,987 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE topology SYSTEM "hwloc.dtd">
+<topology>
+  <object type="Machine" os_index="0" cpuset="0xffffffff,0xffffffff,0xffffffff" complete_cpuset="0xffffffff,0xffffffff,0xffffffff" online_cpuset="0xffffffff,0xffffffff,0xffffffff" allowed_cpuset="0xffffffff,0xffffffff,0xffffffff" nodeset="0x000000ff" complete_nodeset="0x000000ff" allowed_nodeset="0x000000ff">
+    <page_type size="4096" count="0"/>
+    <page_type size="2097152" count="0"/>
+    <page_type size="1073741824" count="0"/>
+    <info name="DMIProductName" value="Altus XO1114GT"/>
+    <info name="DMIProductVersion" value="0100"/>
+    <info name="DMIBoardVendor" value="GIGABYTE"/>
+    <info name="DMIBoardName" value="MZ71-OP0-ZB"/>
+    <info name="DMIBoardVersion" value="01000100"/>
+    <info name="DMIBoardAssetTag" value="01234567890123456789AB"/>
+    <info name="DMIChassisVendor" value="Penguin Computing"/>
+    <info name="DMIChassisType" value="23"/>
+    <info name="DMIChassisVersion" value="01234567"/>
+    <info name="DMIChassisAssetTag" value="01234567890123456789AB"/>
+    <info name="DMIBIOSVendor" value="GIGABYTE"/>
+    <info name="DMIBIOSVersion" value="F06c"/>
+    <info name="DMIBIOSDate" value="12/14/2018"/>
+    <info name="DMISysVendor" value="Penguin Computing"/>
+    <info name="Backend" value="Linux"/>
+    <info name="LinuxCgroup" value="/"/>
+    <info name="OSName" value="Linux"/>
+    <info name="OSRelease" value="3.10.0-1127.0.0.1chaos.ch6.x86_64"/>
+    <info name="OSVersion" value="#1 SMP Fri Apr 3 08:56:52 PDT 2020"/>
+    <info name="HostName" value="corona40"/>
+    <info name="Architecture" value="x86_64"/>
+    <info name="hwlocVersion" value="1.11.11"/>
+    <info name="ProcessName" value="lstopo"/>
+    <distances nbobjs="8" relative_depth="2" latency_base="10.000000">
+      <latency value="1.000000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="1.600000"/>
+      <latency value="1.000000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="1.000000"/>
+      <latency value="1.600000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="1.000000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="1.000000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="1.600000"/>
+      <latency value="1.000000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="1.000000"/>
+      <latency value="1.600000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="3.200000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="1.600000"/>
+      <latency value="1.000000"/>
+    </distances>
+    <object type="Package" os_index="0" cpuset="0x000000ff,0xffff0000,0x00ffffff" complete_cpuset="0x000000ff,0xffff0000,0x00ffffff" online_cpuset="0x000000ff,0xffff0000,0x00ffffff" allowed_cpuset="0x000000ff,0xffff0000,0x00ffffff" nodeset="0x0000000f" complete_nodeset="0x0000000f" allowed_nodeset="0x0000000f">
+      <info name="CPUVendor" value="AuthenticAMD"/>
+      <info name="CPUFamilyNumber" value="23"/>
+      <info name="CPUModelNumber" value="1"/>
+      <info name="CPUModel" value="AMD EPYC 7401 24-Core Processor"/>
+      <info name="CPUStepping" value="2"/>
+      <object type="NUMANode" os_index="0" cpuset="0x003f0000,0x0000003f" complete_cpuset="0x003f0000,0x0000003f" online_cpuset="0x003f0000,0x0000003f" allowed_cpuset="0x003f0000,0x0000003f" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" local_memory="33299529728">
+        <page_type size="4096" count="8129768"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Cache" cpuset="0x00070000,0x00000007" complete_cpuset="0x00070000,0x00000007" online_cpuset="0x00070000,0x00000007" allowed_cpuset="0x00070000,0x00000007" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" online_cpuset="0x00010000,0x00000001" allowed_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" online_cpuset="0x00010000,0x00000001" allowed_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" online_cpuset="0x00010000,0x00000001" allowed_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="0" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" online_cpuset="0x00010000,0x00000001" allowed_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="0" cpuset="0x00000001" complete_cpuset="0x00000001" online_cpuset="0x00000001" allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="48" cpuset="0x00010000,0x0" complete_cpuset="0x00010000,0x0" online_cpuset="0x00010000,0x0" allowed_cpuset="0x00010000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" online_cpuset="0x00020000,0x00000002" allowed_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" online_cpuset="0x00020000,0x00000002" allowed_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" online_cpuset="0x00020000,0x00000002" allowed_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="1" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" online_cpuset="0x00020000,0x00000002" allowed_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="1" cpuset="0x00000002" complete_cpuset="0x00000002" online_cpuset="0x00000002" allowed_cpuset="0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="49" cpuset="0x00020000,0x0" complete_cpuset="0x00020000,0x0" online_cpuset="0x00020000,0x0" allowed_cpuset="0x00020000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" online_cpuset="0x00040000,0x00000004" allowed_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" online_cpuset="0x00040000,0x00000004" allowed_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" online_cpuset="0x00040000,0x00000004" allowed_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="2" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" online_cpuset="0x00040000,0x00000004" allowed_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="2" cpuset="0x00000004" complete_cpuset="0x00000004" online_cpuset="0x00000004" allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="50" cpuset="0x00040000,0x0" complete_cpuset="0x00040000,0x0" online_cpuset="0x00040000,0x0" allowed_cpuset="0x00040000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" cpuset="0x00380000,0x00000038" complete_cpuset="0x00380000,0x00000038" online_cpuset="0x00380000,0x00000038" allowed_cpuset="0x00380000,0x00000038" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" online_cpuset="0x00080000,0x00000008" allowed_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" online_cpuset="0x00080000,0x00000008" allowed_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" online_cpuset="0x00080000,0x00000008" allowed_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="4" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" online_cpuset="0x00080000,0x00000008" allowed_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="3" cpuset="0x00000008" complete_cpuset="0x00000008" online_cpuset="0x00000008" allowed_cpuset="0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="51" cpuset="0x00080000,0x0" complete_cpuset="0x00080000,0x0" online_cpuset="0x00080000,0x0" allowed_cpuset="0x00080000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" online_cpuset="0x00100000,0x00000010" allowed_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" online_cpuset="0x00100000,0x00000010" allowed_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" online_cpuset="0x00100000,0x00000010" allowed_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="5" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" online_cpuset="0x00100000,0x00000010" allowed_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="4" cpuset="0x00000010" complete_cpuset="0x00000010" online_cpuset="0x00000010" allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="52" cpuset="0x00100000,0x0" complete_cpuset="0x00100000,0x0" online_cpuset="0x00100000,0x0" allowed_cpuset="0x00100000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" online_cpuset="0x00200000,0x00000020" allowed_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" online_cpuset="0x00200000,0x00000020" allowed_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" online_cpuset="0x00200000,0x00000020" allowed_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="6" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" online_cpuset="0x00200000,0x00000020" allowed_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="5" cpuset="0x00000020" complete_cpuset="0x00000020" online_cpuset="0x00000020" allowed_cpuset="0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="53" cpuset="0x00200000,0x0" complete_cpuset="0x00200000,0x0" online_cpuset="0x00200000,0x0" allowed_cpuset="0x00200000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" os_index="0" bridge_type="0-1" depth="0" bridge_pci="0000:[00-07]">
+          <object type="Bridge" os_index="17" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="3.938462">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="PCIDev" os_index="4096" name="HGST, Inc. Ultrastar SN200 Series NVMe SSD" pci_busid="0000:01:00.0" pci_type="0108 [1c58:0023] [1c58:0023] 02" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="HGST, Inc."/>
+              <info name="PCIDevice" value="Ultrastar SN200 Series NVMe SSD"/>
+              <info name="PCISlot" value="1"/>
+              <object type="OSDev" name="nvme0n1" osdev_type="0">
+                <info name="LinuxDeviceID" value="259:0"/>
+                <info name="Model" value="HUSMR7616BDP301"/>
+                <info name="SerialNumber" value="SDM0000107A5"/>
+                <info name="Type" value="Disk"/>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" os_index="19" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[03-03]" pci_busid="0000:00:01.3" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="1.000000">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="PCIDev" os_index="12288" name="Intel Corporation I350 Gigabit Network Connection" pci_busid="0000:03:00.0" pci_type="0200 [8086:1521] [1458:0000] 01" pci_link_speed="1.000000">
+              <info name="PCIVendor" value="Intel Corporation"/>
+              <info name="PCIDevice" value="I350 Gigabit Network Connection"/>
+              <object type="OSDev" name="enp3s0f0" osdev_type="2">
+                <info name="Address" value="e0:d5:5e:ca:b0:22"/>
+              </object>
+            </object>
+            <object type="PCIDev" os_index="12289" name="Intel Corporation I350 Gigabit Network Connection" pci_busid="0000:03:00.1" pci_type="0200 [8086:1521] [1458:0000] 01" pci_link_speed="1.000000">
+              <info name="PCIVendor" value="Intel Corporation"/>
+              <info name="PCIDevice" value="I350 Gigabit Network Connection"/>
+              <object type="OSDev" name="enp3s0f1" osdev_type="2">
+                <info name="Address" value="e0:d5:5e:ca:b0:23"/>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" os_index="22" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[04-05]" pci_busid="0000:00:01.6" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="0.250000">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" os_index="16384" name="ASPEED Technology, Inc. AST1150 PCI-to-PCI Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[05-05]" pci_busid="0000:04:00.0" pci_type="0604 [1a03:1150] [0000:0000] 04" pci_link_speed="0.250000">
+              <info name="PCIVendor" value="ASPEED Technology, Inc."/>
+              <info name="PCIDevice" value="AST1150 PCI-to-PCI Bridge"/>
+              <object type="PCIDev" os_index="20480" name="ASPEED Technology, Inc. ASPEED Graphics Family" pci_busid="0000:05:00.0" pci_type="0300 [1a03:2000] [1458:1000] 41" pci_link_speed="0.000000">
+                <info name="PCIVendor" value="ASPEED Technology, Inc."/>
+                <info name="PCIDevice" value="ASPEED Graphics Family"/>
+                <object type="OSDev" name="card0" osdev_type="1"/>
+                <object type="OSDev" name="controlD64" osdev_type="1"/>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" os_index="129" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) Internal PCIe GPP Bridge 0 to Bus B" bridge_type="1-1" depth="1" bridge_pci="0000:[07-07]" pci_busid="0000:00:08.1" pci_type="0604 [1022:1454] [0000:0000] 00" pci_link_speed="15.753846">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) Internal PCIe GPP Bridge 0 to Bus B"/>
+            <object type="PCIDev" os_index="28674" name="Advanced Micro Devices, Inc. [AMD] FCH SATA Controller [AHCI mode]" pci_busid="0000:07:00.2" pci_type="0106 [1022:7901] [1458:1000] 51" pci_link_speed="15.753846">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="FCH SATA Controller [AHCI mode]"/>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="1" cpuset="0x0fc00000,0x00000fc0" complete_cpuset="0x0fc00000,0x00000fc0" online_cpuset="0x0fc00000,0x00000fc0" allowed_cpuset="0x0fc00000,0x00000fc0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" local_memory="33819570176">
+        <page_type size="4096" count="8256731"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Cache" cpuset="0x01c00000,0x000001c0" complete_cpuset="0x01c00000,0x000001c0" online_cpuset="0x01c00000,0x000001c0" allowed_cpuset="0x01c00000,0x000001c0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" online_cpuset="0x00400000,0x00000040" allowed_cpuset="0x00400000,0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" online_cpuset="0x00400000,0x00000040" allowed_cpuset="0x00400000,0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" online_cpuset="0x00400000,0x00000040" allowed_cpuset="0x00400000,0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="8" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" online_cpuset="0x00400000,0x00000040" allowed_cpuset="0x00400000,0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="6" cpuset="0x00000040" complete_cpuset="0x00000040" online_cpuset="0x00000040" allowed_cpuset="0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="54" cpuset="0x00400000,0x0" complete_cpuset="0x00400000,0x0" online_cpuset="0x00400000,0x0" allowed_cpuset="0x00400000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" online_cpuset="0x00800000,0x00000080" allowed_cpuset="0x00800000,0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" online_cpuset="0x00800000,0x00000080" allowed_cpuset="0x00800000,0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" online_cpuset="0x00800000,0x00000080" allowed_cpuset="0x00800000,0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="9" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" online_cpuset="0x00800000,0x00000080" allowed_cpuset="0x00800000,0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="7" cpuset="0x00000080" complete_cpuset="0x00000080" online_cpuset="0x00000080" allowed_cpuset="0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="55" cpuset="0x00800000,0x0" complete_cpuset="0x00800000,0x0" online_cpuset="0x00800000,0x0" allowed_cpuset="0x00800000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" online_cpuset="0x01000000,0x00000100" allowed_cpuset="0x01000000,0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" online_cpuset="0x01000000,0x00000100" allowed_cpuset="0x01000000,0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" online_cpuset="0x01000000,0x00000100" allowed_cpuset="0x01000000,0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="10" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" online_cpuset="0x01000000,0x00000100" allowed_cpuset="0x01000000,0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="8" cpuset="0x00000100" complete_cpuset="0x00000100" online_cpuset="0x00000100" allowed_cpuset="0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="56" cpuset="0x01000000,0x0" complete_cpuset="0x01000000,0x0" online_cpuset="0x01000000,0x0" allowed_cpuset="0x01000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" cpuset="0x0e000000,0x00000e00" complete_cpuset="0x0e000000,0x00000e00" online_cpuset="0x0e000000,0x00000e00" allowed_cpuset="0x0e000000,0x00000e00" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" online_cpuset="0x02000000,0x00000200" allowed_cpuset="0x02000000,0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" online_cpuset="0x02000000,0x00000200" allowed_cpuset="0x02000000,0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" online_cpuset="0x02000000,0x00000200" allowed_cpuset="0x02000000,0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="12" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" online_cpuset="0x02000000,0x00000200" allowed_cpuset="0x02000000,0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="9" cpuset="0x00000200" complete_cpuset="0x00000200" online_cpuset="0x00000200" allowed_cpuset="0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="57" cpuset="0x02000000,0x0" complete_cpuset="0x02000000,0x0" online_cpuset="0x02000000,0x0" allowed_cpuset="0x02000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" online_cpuset="0x04000000,0x00000400" allowed_cpuset="0x04000000,0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" online_cpuset="0x04000000,0x00000400" allowed_cpuset="0x04000000,0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" online_cpuset="0x04000000,0x00000400" allowed_cpuset="0x04000000,0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="13" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" online_cpuset="0x04000000,0x00000400" allowed_cpuset="0x04000000,0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="10" cpuset="0x00000400" complete_cpuset="0x00000400" online_cpuset="0x00000400" allowed_cpuset="0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="58" cpuset="0x04000000,0x0" complete_cpuset="0x04000000,0x0" online_cpuset="0x04000000,0x0" allowed_cpuset="0x04000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" online_cpuset="0x08000000,0x00000800" allowed_cpuset="0x08000000,0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" online_cpuset="0x08000000,0x00000800" allowed_cpuset="0x08000000,0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" online_cpuset="0x08000000,0x00000800" allowed_cpuset="0x08000000,0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="14" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" online_cpuset="0x08000000,0x00000800" allowed_cpuset="0x08000000,0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="11" cpuset="0x00000800" complete_cpuset="0x00000800" online_cpuset="0x00000800" allowed_cpuset="0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="59" cpuset="0x08000000,0x0" complete_cpuset="0x08000000,0x0" online_cpuset="0x08000000,0x0" allowed_cpuset="0x08000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" os_index="1" bridge_type="0-1" depth="0" bridge_pci="0000:[10-15]">
+          <object type="Bridge" os_index="65553" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[11-13]" pci_busid="0000:10:01.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="3.938462">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" os_index="69632" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="2" bridge_pci="0000:[12-13]" pci_busid="0000:11:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+              <object type="Bridge" os_index="73728" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="3" bridge_pci="0000:[13-13]" pci_busid="0000:12:00.0" pci_type="0604 [1002:14a1] [0000:0000] 00" pci_link_speed="31.507692">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="PCIDev" os_index="77824" name="Advanced Micro Devices, Inc. [AMD/ATI] Vega 20" pci_busid="0000:13:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <info name="PCIDevice" value="Vega 20"/>
+                  <object type="OSDev" name="card1" osdev_type="1"/>
+                  <object type="OSDev" name="renderD128" osdev_type="1"/>
+                  <object type="OSDev" name="controlD65" osdev_type="1"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="2" cpuset="0x00000003,0xf0000000,0x0003f000" complete_cpuset="0x00000003,0xf0000000,0x0003f000" online_cpuset="0x00000003,0xf0000000,0x0003f000" allowed_cpuset="0x00000003,0xf0000000,0x0003f000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" local_memory="33819574272">
+        <page_type size="4096" count="8256732"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Cache" cpuset="0x70000000,0x00007000" complete_cpuset="0x70000000,0x00007000" online_cpuset="0x70000000,0x00007000" allowed_cpuset="0x70000000,0x00007000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" online_cpuset="0x10000000,0x00001000" allowed_cpuset="0x10000000,0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" online_cpuset="0x10000000,0x00001000" allowed_cpuset="0x10000000,0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" online_cpuset="0x10000000,0x00001000" allowed_cpuset="0x10000000,0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="16" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" online_cpuset="0x10000000,0x00001000" allowed_cpuset="0x10000000,0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+                  <object type="PU" os_index="12" cpuset="0x00001000" complete_cpuset="0x00001000" online_cpuset="0x00001000" allowed_cpuset="0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                  <object type="PU" os_index="60" cpuset="0x10000000,0x0" complete_cpuset="0x10000000,0x0" online_cpuset="0x10000000,0x0" allowed_cpuset="0x10000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" online_cpuset="0x20000000,0x00002000" allowed_cpuset="0x20000000,0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" online_cpuset="0x20000000,0x00002000" allowed_cpuset="0x20000000,0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" online_cpuset="0x20000000,0x00002000" allowed_cpuset="0x20000000,0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="17" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" online_cpuset="0x20000000,0x00002000" allowed_cpuset="0x20000000,0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+                  <object type="PU" os_index="13" cpuset="0x00002000" complete_cpuset="0x00002000" online_cpuset="0x00002000" allowed_cpuset="0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                  <object type="PU" os_index="61" cpuset="0x20000000,0x0" complete_cpuset="0x20000000,0x0" online_cpuset="0x20000000,0x0" allowed_cpuset="0x20000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" online_cpuset="0x40000000,0x00004000" allowed_cpuset="0x40000000,0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" online_cpuset="0x40000000,0x00004000" allowed_cpuset="0x40000000,0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" online_cpuset="0x40000000,0x00004000" allowed_cpuset="0x40000000,0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="18" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" online_cpuset="0x40000000,0x00004000" allowed_cpuset="0x40000000,0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+                  <object type="PU" os_index="14" cpuset="0x00004000" complete_cpuset="0x00004000" online_cpuset="0x00004000" allowed_cpuset="0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                  <object type="PU" os_index="62" cpuset="0x40000000,0x0" complete_cpuset="0x40000000,0x0" online_cpuset="0x40000000,0x0" allowed_cpuset="0x40000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" cpuset="0x00000003,0x80000000,0x00038000" complete_cpuset="0x00000003,0x80000000,0x00038000" online_cpuset="0x00000003,0x80000000,0x00038000" allowed_cpuset="0x00000003,0x80000000,0x00038000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" online_cpuset="0x80000000,0x00008000" allowed_cpuset="0x80000000,0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" online_cpuset="0x80000000,0x00008000" allowed_cpuset="0x80000000,0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" online_cpuset="0x80000000,0x00008000" allowed_cpuset="0x80000000,0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="20" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" online_cpuset="0x80000000,0x00008000" allowed_cpuset="0x80000000,0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+                  <object type="PU" os_index="15" cpuset="0x00008000" complete_cpuset="0x00008000" online_cpuset="0x00008000" allowed_cpuset="0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                  <object type="PU" os_index="63" cpuset="0x80000000,0x0" complete_cpuset="0x80000000,0x0" online_cpuset="0x80000000,0x0" allowed_cpuset="0x80000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" online_cpuset="0x00000001,,0x00010000" allowed_cpuset="0x00000001,,0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" online_cpuset="0x00000001,,0x00010000" allowed_cpuset="0x00000001,,0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" online_cpuset="0x00000001,,0x00010000" allowed_cpuset="0x00000001,,0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="21" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" online_cpuset="0x00000001,,0x00010000" allowed_cpuset="0x00000001,,0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+                  <object type="PU" os_index="16" cpuset="0x00010000" complete_cpuset="0x00010000" online_cpuset="0x00010000" allowed_cpuset="0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                  <object type="PU" os_index="64" cpuset="0x00000001,,0x0" complete_cpuset="0x00000001,,0x0" online_cpuset="0x00000001,,0x0" allowed_cpuset="0x00000001,,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" online_cpuset="0x00000002,,0x00020000" allowed_cpuset="0x00000002,,0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" online_cpuset="0x00000002,,0x00020000" allowed_cpuset="0x00000002,,0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" online_cpuset="0x00000002,,0x00020000" allowed_cpuset="0x00000002,,0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="22" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" online_cpuset="0x00000002,,0x00020000" allowed_cpuset="0x00000002,,0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+                  <object type="PU" os_index="17" cpuset="0x00020000" complete_cpuset="0x00020000" online_cpuset="0x00020000" allowed_cpuset="0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                  <object type="PU" os_index="65" cpuset="0x00000002,,0x0" complete_cpuset="0x00000002,,0x0" online_cpuset="0x00000002,,0x0" allowed_cpuset="0x00000002,,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" os_index="2" bridge_type="0-1" depth="0" bridge_pci="0000:[20-25]">
+          <object type="Bridge" os_index="131121" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[21-23]" pci_busid="0000:20:03.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="3.938462">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" os_index="135168" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="2" bridge_pci="0000:[22-23]" pci_busid="0000:21:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+              <object type="Bridge" os_index="139264" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="3" bridge_pci="0000:[23-23]" pci_busid="0000:22:00.0" pci_type="0604 [1002:14a1] [0000:0000] 00" pci_link_speed="31.507692">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="PCIDev" os_index="143360" name="Advanced Micro Devices, Inc. [AMD/ATI] Vega 20" pci_busid="0000:23:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <info name="PCIDevice" value="Vega 20"/>
+                  <object type="OSDev" name="card2" osdev_type="1"/>
+                  <object type="OSDev" name="renderD129" osdev_type="1"/>
+                  <object type="OSDev" name="controlD66" osdev_type="1"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="3" cpuset="0x000000fc,,0x00fc0000" complete_cpuset="0x000000fc,,0x00fc0000" online_cpuset="0x000000fc,,0x00fc0000" allowed_cpuset="0x000000fc,,0x00fc0000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" local_memory="33802997760">
+        <page_type size="4096" count="8252685"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Cache" cpuset="0x0000001c,,0x001c0000" complete_cpuset="0x0000001c,,0x001c0000" online_cpuset="0x0000001c,,0x001c0000" allowed_cpuset="0x0000001c,,0x001c0000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" online_cpuset="0x00000004,,0x00040000" allowed_cpuset="0x00000004,,0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" online_cpuset="0x00000004,,0x00040000" allowed_cpuset="0x00000004,,0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" online_cpuset="0x00000004,,0x00040000" allowed_cpuset="0x00000004,,0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="24" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" online_cpuset="0x00000004,,0x00040000" allowed_cpuset="0x00000004,,0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+                  <object type="PU" os_index="18" cpuset="0x00040000" complete_cpuset="0x00040000" online_cpuset="0x00040000" allowed_cpuset="0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                  <object type="PU" os_index="66" cpuset="0x00000004,,0x0" complete_cpuset="0x00000004,,0x0" online_cpuset="0x00000004,,0x0" allowed_cpuset="0x00000004,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" online_cpuset="0x00000008,,0x00080000" allowed_cpuset="0x00000008,,0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" online_cpuset="0x00000008,,0x00080000" allowed_cpuset="0x00000008,,0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" online_cpuset="0x00000008,,0x00080000" allowed_cpuset="0x00000008,,0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="25" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" online_cpuset="0x00000008,,0x00080000" allowed_cpuset="0x00000008,,0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+                  <object type="PU" os_index="19" cpuset="0x00080000" complete_cpuset="0x00080000" online_cpuset="0x00080000" allowed_cpuset="0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                  <object type="PU" os_index="67" cpuset="0x00000008,,0x0" complete_cpuset="0x00000008,,0x0" online_cpuset="0x00000008,,0x0" allowed_cpuset="0x00000008,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" online_cpuset="0x00000010,,0x00100000" allowed_cpuset="0x00000010,,0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" online_cpuset="0x00000010,,0x00100000" allowed_cpuset="0x00000010,,0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" online_cpuset="0x00000010,,0x00100000" allowed_cpuset="0x00000010,,0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="26" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" online_cpuset="0x00000010,,0x00100000" allowed_cpuset="0x00000010,,0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+                  <object type="PU" os_index="20" cpuset="0x00100000" complete_cpuset="0x00100000" online_cpuset="0x00100000" allowed_cpuset="0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                  <object type="PU" os_index="68" cpuset="0x00000010,,0x0" complete_cpuset="0x00000010,,0x0" online_cpuset="0x00000010,,0x0" allowed_cpuset="0x00000010,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" cpuset="0x000000e0,,0x00e00000" complete_cpuset="0x000000e0,,0x00e00000" online_cpuset="0x000000e0,,0x00e00000" allowed_cpuset="0x000000e0,,0x00e00000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" online_cpuset="0x00000020,,0x00200000" allowed_cpuset="0x00000020,,0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" online_cpuset="0x00000020,,0x00200000" allowed_cpuset="0x00000020,,0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" online_cpuset="0x00000020,,0x00200000" allowed_cpuset="0x00000020,,0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="28" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" online_cpuset="0x00000020,,0x00200000" allowed_cpuset="0x00000020,,0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+                  <object type="PU" os_index="21" cpuset="0x00200000" complete_cpuset="0x00200000" online_cpuset="0x00200000" allowed_cpuset="0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                  <object type="PU" os_index="69" cpuset="0x00000020,,0x0" complete_cpuset="0x00000020,,0x0" online_cpuset="0x00000020,,0x0" allowed_cpuset="0x00000020,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" online_cpuset="0x00000040,,0x00400000" allowed_cpuset="0x00000040,,0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" online_cpuset="0x00000040,,0x00400000" allowed_cpuset="0x00000040,,0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" online_cpuset="0x00000040,,0x00400000" allowed_cpuset="0x00000040,,0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="29" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" online_cpuset="0x00000040,,0x00400000" allowed_cpuset="0x00000040,,0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+                  <object type="PU" os_index="22" cpuset="0x00400000" complete_cpuset="0x00400000" online_cpuset="0x00400000" allowed_cpuset="0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                  <object type="PU" os_index="70" cpuset="0x00000040,,0x0" complete_cpuset="0x00000040,,0x0" online_cpuset="0x00000040,,0x0" allowed_cpuset="0x00000040,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" online_cpuset="0x00000080,,0x00800000" allowed_cpuset="0x00000080,,0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" online_cpuset="0x00000080,,0x00800000" allowed_cpuset="0x00000080,,0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" online_cpuset="0x00000080,,0x00800000" allowed_cpuset="0x00000080,,0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="30" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" online_cpuset="0x00000080,,0x00800000" allowed_cpuset="0x00000080,,0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+                  <object type="PU" os_index="23" cpuset="0x00800000" complete_cpuset="0x00800000" online_cpuset="0x00800000" allowed_cpuset="0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                  <object type="PU" os_index="71" cpuset="0x00000080,,0x0" complete_cpuset="0x00000080,,0x0" online_cpuset="0x00000080,,0x0" allowed_cpuset="0x00000080,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" os_index="3" bridge_type="0-1" depth="0" bridge_pci="0000:[30-36]">
+          <object type="Bridge" os_index="196657" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[31-34]" pci_busid="0000:30:03.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="15.753846">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" os_index="200704" name="PLX Technology, Inc. PEX 8747 48-Lane, 5-Port PCI Express Gen 3 (8.0 GT/s) Switch" bridge_type="1-1" depth="2" bridge_pci="0000:[32-34]" pci_busid="0000:31:00.0" pci_type="0604 [10b5:8747] [0000:0000] ca" pci_link_speed="15.753846">
+              <info name="PCIVendor" value="PLX Technology, Inc."/>
+              <info name="PCIDevice" value="PEX 8747 48-Lane, 5-Port PCI Express Gen 3 (8.0 GT/s) Switch"/>
+              <object type="Bridge" os_index="204928" name="PLX Technology, Inc. PEX 8747 48-Lane, 5-Port PCI Express Gen 3 (8.0 GT/s) Switch" bridge_type="1-1" depth="3" bridge_pci="0000:[33-33]" pci_busid="0000:32:08.0" pci_type="0604 [10b5:8747] [0000:0000] ca" pci_link_speed="15.753846">
+                <info name="PCIVendor" value="PLX Technology, Inc."/>
+                <info name="PCIDevice" value="PEX 8747 48-Lane, 5-Port PCI Express Gen 3 (8.0 GT/s) Switch"/>
+                <object type="PCIDev" os_index="208896" name="Mellanox Technologies MT28908 Family [ConnectX-6]" pci_busid="0000:33:00.0" pci_type="0207 [15b3:101b] [15b3:0006] 00" pci_link_speed="15.753846">
+                  <info name="PCIVendor" value="Mellanox Technologies"/>
+                  <info name="PCIDevice" value="MT28908 Family [ConnectX-6]"/>
+                  <object type="OSDev" name="hsi0" osdev_type="2">
+                    <info name="Address" value="80:00:01:07:fe:80:00:00:00:00:00:00:98:03:9b:03:00:82:9d:b4"/>
+                    <info name="Port" value="1"/>
+                  </object>
+                  <object type="OSDev" name="mlx5_0" osdev_type="3">
+                    <info name="NodeGUID" value="9803:9b03:0082:9db4"/>
+                    <info name="SysImageGUID" value="9803:9b03:0082:9db4"/>
+                    <info name="Port1State" value="4"/>
+                    <info name="Port1LID" value="0x1c"/>
+                    <info name="Port1LMC" value="0"/>
+                    <info name="Port1GID0" value="fe80:0000:0000:0000:9803:9b03:0082:9db4"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+    </object>
+    <object type="Package" os_index="1" cpuset="0xffffff00,0x0000ffff,0xff000000" complete_cpuset="0xffffff00,0x0000ffff,0xff000000" online_cpuset="0xffffff00,0x0000ffff,0xff000000" allowed_cpuset="0xffffff00,0x0000ffff,0xff000000" nodeset="0x000000f0" complete_nodeset="0x000000f0" allowed_nodeset="0x000000f0">
+      <info name="CPUVendor" value="AuthenticAMD"/>
+      <info name="CPUFamilyNumber" value="23"/>
+      <info name="CPUModelNumber" value="1"/>
+      <info name="CPUModel" value="AMD EPYC 7401 24-Core Processor"/>
+      <info name="CPUStepping" value="2"/>
+      <object type="NUMANode" os_index="4" cpuset="0x00003f00,,0x3f000000" complete_cpuset="0x00003f00,,0x3f000000" online_cpuset="0x00003f00,,0x3f000000" allowed_cpuset="0x00003f00,,0x3f000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" local_memory="33819574272">
+        <page_type size="4096" count="8256732"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Cache" cpuset="0x00000700,,0x07000000" complete_cpuset="0x00000700,,0x07000000" online_cpuset="0x00000700,,0x07000000" allowed_cpuset="0x00000700,,0x07000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" online_cpuset="0x00000100,,0x01000000" allowed_cpuset="0x00000100,,0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" online_cpuset="0x00000100,,0x01000000" allowed_cpuset="0x00000100,,0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" online_cpuset="0x00000100,,0x01000000" allowed_cpuset="0x00000100,,0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="0" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" online_cpuset="0x00000100,,0x01000000" allowed_cpuset="0x00000100,,0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+                  <object type="PU" os_index="24" cpuset="0x01000000" complete_cpuset="0x01000000" online_cpuset="0x01000000" allowed_cpuset="0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                  <object type="PU" os_index="72" cpuset="0x00000100,,0x0" complete_cpuset="0x00000100,,0x0" online_cpuset="0x00000100,,0x0" allowed_cpuset="0x00000100,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" online_cpuset="0x00000200,,0x02000000" allowed_cpuset="0x00000200,,0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" online_cpuset="0x00000200,,0x02000000" allowed_cpuset="0x00000200,,0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" online_cpuset="0x00000200,,0x02000000" allowed_cpuset="0x00000200,,0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="1" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" online_cpuset="0x00000200,,0x02000000" allowed_cpuset="0x00000200,,0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+                  <object type="PU" os_index="25" cpuset="0x02000000" complete_cpuset="0x02000000" online_cpuset="0x02000000" allowed_cpuset="0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                  <object type="PU" os_index="73" cpuset="0x00000200,,0x0" complete_cpuset="0x00000200,,0x0" online_cpuset="0x00000200,,0x0" allowed_cpuset="0x00000200,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" online_cpuset="0x00000400,,0x04000000" allowed_cpuset="0x00000400,,0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" online_cpuset="0x00000400,,0x04000000" allowed_cpuset="0x00000400,,0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" online_cpuset="0x00000400,,0x04000000" allowed_cpuset="0x00000400,,0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="2" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" online_cpuset="0x00000400,,0x04000000" allowed_cpuset="0x00000400,,0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+                  <object type="PU" os_index="26" cpuset="0x04000000" complete_cpuset="0x04000000" online_cpuset="0x04000000" allowed_cpuset="0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                  <object type="PU" os_index="74" cpuset="0x00000400,,0x0" complete_cpuset="0x00000400,,0x0" online_cpuset="0x00000400,,0x0" allowed_cpuset="0x00000400,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" cpuset="0x00003800,,0x38000000" complete_cpuset="0x00003800,,0x38000000" online_cpuset="0x00003800,,0x38000000" allowed_cpuset="0x00003800,,0x38000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" online_cpuset="0x00000800,,0x08000000" allowed_cpuset="0x00000800,,0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" online_cpuset="0x00000800,,0x08000000" allowed_cpuset="0x00000800,,0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" online_cpuset="0x00000800,,0x08000000" allowed_cpuset="0x00000800,,0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="4" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" online_cpuset="0x00000800,,0x08000000" allowed_cpuset="0x00000800,,0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+                  <object type="PU" os_index="27" cpuset="0x08000000" complete_cpuset="0x08000000" online_cpuset="0x08000000" allowed_cpuset="0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                  <object type="PU" os_index="75" cpuset="0x00000800,,0x0" complete_cpuset="0x00000800,,0x0" online_cpuset="0x00000800,,0x0" allowed_cpuset="0x00000800,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" online_cpuset="0x00001000,,0x10000000" allowed_cpuset="0x00001000,,0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" online_cpuset="0x00001000,,0x10000000" allowed_cpuset="0x00001000,,0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" online_cpuset="0x00001000,,0x10000000" allowed_cpuset="0x00001000,,0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="5" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" online_cpuset="0x00001000,,0x10000000" allowed_cpuset="0x00001000,,0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+                  <object type="PU" os_index="28" cpuset="0x10000000" complete_cpuset="0x10000000" online_cpuset="0x10000000" allowed_cpuset="0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                  <object type="PU" os_index="76" cpuset="0x00001000,,0x0" complete_cpuset="0x00001000,,0x0" online_cpuset="0x00001000,,0x0" allowed_cpuset="0x00001000,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" online_cpuset="0x00002000,,0x20000000" allowed_cpuset="0x00002000,,0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" online_cpuset="0x00002000,,0x20000000" allowed_cpuset="0x00002000,,0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" online_cpuset="0x00002000,,0x20000000" allowed_cpuset="0x00002000,,0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="6" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" online_cpuset="0x00002000,,0x20000000" allowed_cpuset="0x00002000,,0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+                  <object type="PU" os_index="29" cpuset="0x20000000" complete_cpuset="0x20000000" online_cpuset="0x20000000" allowed_cpuset="0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                  <object type="PU" os_index="77" cpuset="0x00002000,,0x0" complete_cpuset="0x00002000,,0x0" online_cpuset="0x00002000,,0x0" allowed_cpuset="0x00002000,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="5" cpuset="0x000fc000,0x0000000f,0xc0000000" complete_cpuset="0x000fc000,0x0000000f,0xc0000000" online_cpuset="0x000fc000,0x0000000f,0xc0000000" allowed_cpuset="0x000fc000,0x0000000f,0xc0000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" local_memory="33819570176">
+        <page_type size="4096" count="8256731"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Cache" cpuset="0x0001c000,0x00000001,0xc0000000" complete_cpuset="0x0001c000,0x00000001,0xc0000000" online_cpuset="0x0001c000,0x00000001,0xc0000000" allowed_cpuset="0x0001c000,0x00000001,0xc0000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" online_cpuset="0x00004000,,0x40000000" allowed_cpuset="0x00004000,,0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" online_cpuset="0x00004000,,0x40000000" allowed_cpuset="0x00004000,,0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" online_cpuset="0x00004000,,0x40000000" allowed_cpuset="0x00004000,,0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="8" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" online_cpuset="0x00004000,,0x40000000" allowed_cpuset="0x00004000,,0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+                  <object type="PU" os_index="30" cpuset="0x40000000" complete_cpuset="0x40000000" online_cpuset="0x40000000" allowed_cpuset="0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                  <object type="PU" os_index="78" cpuset="0x00004000,,0x0" complete_cpuset="0x00004000,,0x0" online_cpuset="0x00004000,,0x0" allowed_cpuset="0x00004000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" online_cpuset="0x00008000,,0x80000000" allowed_cpuset="0x00008000,,0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" online_cpuset="0x00008000,,0x80000000" allowed_cpuset="0x00008000,,0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" online_cpuset="0x00008000,,0x80000000" allowed_cpuset="0x00008000,,0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="9" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" online_cpuset="0x00008000,,0x80000000" allowed_cpuset="0x00008000,,0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+                  <object type="PU" os_index="31" cpuset="0x80000000" complete_cpuset="0x80000000" online_cpuset="0x80000000" allowed_cpuset="0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                  <object type="PU" os_index="79" cpuset="0x00008000,,0x0" complete_cpuset="0x00008000,,0x0" online_cpuset="0x00008000,,0x0" allowed_cpuset="0x00008000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" online_cpuset="0x00010000,0x00000001,0x0" allowed_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" online_cpuset="0x00010000,0x00000001,0x0" allowed_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" online_cpuset="0x00010000,0x00000001,0x0" allowed_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="10" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" online_cpuset="0x00010000,0x00000001,0x0" allowed_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+                  <object type="PU" os_index="32" cpuset="0x00000001,0x0" complete_cpuset="0x00000001,0x0" online_cpuset="0x00000001,0x0" allowed_cpuset="0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                  <object type="PU" os_index="80" cpuset="0x00010000,,0x0" complete_cpuset="0x00010000,,0x0" online_cpuset="0x00010000,,0x0" allowed_cpuset="0x00010000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" cpuset="0x000e0000,0x0000000e,0x0" complete_cpuset="0x000e0000,0x0000000e,0x0" online_cpuset="0x000e0000,0x0000000e,0x0" allowed_cpuset="0x000e0000,0x0000000e,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" online_cpuset="0x00020000,0x00000002,0x0" allowed_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" online_cpuset="0x00020000,0x00000002,0x0" allowed_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" online_cpuset="0x00020000,0x00000002,0x0" allowed_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="12" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" online_cpuset="0x00020000,0x00000002,0x0" allowed_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+                  <object type="PU" os_index="33" cpuset="0x00000002,0x0" complete_cpuset="0x00000002,0x0" online_cpuset="0x00000002,0x0" allowed_cpuset="0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                  <object type="PU" os_index="81" cpuset="0x00020000,,0x0" complete_cpuset="0x00020000,,0x0" online_cpuset="0x00020000,,0x0" allowed_cpuset="0x00020000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" online_cpuset="0x00040000,0x00000004,0x0" allowed_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" online_cpuset="0x00040000,0x00000004,0x0" allowed_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" online_cpuset="0x00040000,0x00000004,0x0" allowed_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="13" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" online_cpuset="0x00040000,0x00000004,0x0" allowed_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+                  <object type="PU" os_index="34" cpuset="0x00000004,0x0" complete_cpuset="0x00000004,0x0" online_cpuset="0x00000004,0x0" allowed_cpuset="0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                  <object type="PU" os_index="82" cpuset="0x00040000,,0x0" complete_cpuset="0x00040000,,0x0" online_cpuset="0x00040000,,0x0" allowed_cpuset="0x00040000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" online_cpuset="0x00080000,0x00000008,0x0" allowed_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" online_cpuset="0x00080000,0x00000008,0x0" allowed_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" online_cpuset="0x00080000,0x00000008,0x0" allowed_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="14" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" online_cpuset="0x00080000,0x00000008,0x0" allowed_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+                  <object type="PU" os_index="35" cpuset="0x00000008,0x0" complete_cpuset="0x00000008,0x0" online_cpuset="0x00000008,0x0" allowed_cpuset="0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                  <object type="PU" os_index="83" cpuset="0x00080000,,0x0" complete_cpuset="0x00080000,,0x0" online_cpuset="0x00080000,,0x0" allowed_cpuset="0x00080000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" os_index="5" bridge_type="0-1" depth="0" bridge_pci="0000:[50-55]">
+          <object type="Bridge" os_index="327697" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[51-53]" pci_busid="0000:50:01.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="3.938462">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" os_index="331776" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="2" bridge_pci="0000:[52-53]" pci_busid="0000:51:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+              <object type="Bridge" os_index="335872" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="3" bridge_pci="0000:[53-53]" pci_busid="0000:52:00.0" pci_type="0604 [1002:14a1] [0000:0000] 00" pci_link_speed="31.507692">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="PCIDev" os_index="339968" name="Advanced Micro Devices, Inc. [AMD/ATI] Vega 20" pci_busid="0000:53:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <info name="PCIDevice" value="Vega 20"/>
+                  <object type="OSDev" name="card3" osdev_type="1"/>
+                  <object type="OSDev" name="renderD130" osdev_type="1"/>
+                  <object type="OSDev" name="controlD67" osdev_type="1"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="6" cpuset="0x03f00000,0x000003f0,0x0" complete_cpuset="0x03f00000,0x000003f0,0x0" online_cpuset="0x03f00000,0x000003f0,0x0" allowed_cpuset="0x03f00000,0x000003f0,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" local_memory="33819574272">
+        <page_type size="4096" count="8256732"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Cache" cpuset="0x00700000,0x00000070,0x0" complete_cpuset="0x00700000,0x00000070,0x0" online_cpuset="0x00700000,0x00000070,0x0" allowed_cpuset="0x00700000,0x00000070,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" online_cpuset="0x00100000,0x00000010,0x0" allowed_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" online_cpuset="0x00100000,0x00000010,0x0" allowed_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" online_cpuset="0x00100000,0x00000010,0x0" allowed_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="16" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" online_cpuset="0x00100000,0x00000010,0x0" allowed_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+                  <object type="PU" os_index="36" cpuset="0x00000010,0x0" complete_cpuset="0x00000010,0x0" online_cpuset="0x00000010,0x0" allowed_cpuset="0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                  <object type="PU" os_index="84" cpuset="0x00100000,,0x0" complete_cpuset="0x00100000,,0x0" online_cpuset="0x00100000,,0x0" allowed_cpuset="0x00100000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" online_cpuset="0x00200000,0x00000020,0x0" allowed_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" online_cpuset="0x00200000,0x00000020,0x0" allowed_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" online_cpuset="0x00200000,0x00000020,0x0" allowed_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="17" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" online_cpuset="0x00200000,0x00000020,0x0" allowed_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+                  <object type="PU" os_index="37" cpuset="0x00000020,0x0" complete_cpuset="0x00000020,0x0" online_cpuset="0x00000020,0x0" allowed_cpuset="0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                  <object type="PU" os_index="85" cpuset="0x00200000,,0x0" complete_cpuset="0x00200000,,0x0" online_cpuset="0x00200000,,0x0" allowed_cpuset="0x00200000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" online_cpuset="0x00400000,0x00000040,0x0" allowed_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" online_cpuset="0x00400000,0x00000040,0x0" allowed_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" online_cpuset="0x00400000,0x00000040,0x0" allowed_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="18" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" online_cpuset="0x00400000,0x00000040,0x0" allowed_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+                  <object type="PU" os_index="38" cpuset="0x00000040,0x0" complete_cpuset="0x00000040,0x0" online_cpuset="0x00000040,0x0" allowed_cpuset="0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                  <object type="PU" os_index="86" cpuset="0x00400000,,0x0" complete_cpuset="0x00400000,,0x0" online_cpuset="0x00400000,,0x0" allowed_cpuset="0x00400000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" cpuset="0x03800000,0x00000380,0x0" complete_cpuset="0x03800000,0x00000380,0x0" online_cpuset="0x03800000,0x00000380,0x0" allowed_cpuset="0x03800000,0x00000380,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" online_cpuset="0x00800000,0x00000080,0x0" allowed_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" online_cpuset="0x00800000,0x00000080,0x0" allowed_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" online_cpuset="0x00800000,0x00000080,0x0" allowed_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="20" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" online_cpuset="0x00800000,0x00000080,0x0" allowed_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+                  <object type="PU" os_index="39" cpuset="0x00000080,0x0" complete_cpuset="0x00000080,0x0" online_cpuset="0x00000080,0x0" allowed_cpuset="0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                  <object type="PU" os_index="87" cpuset="0x00800000,,0x0" complete_cpuset="0x00800000,,0x0" online_cpuset="0x00800000,,0x0" allowed_cpuset="0x00800000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" online_cpuset="0x01000000,0x00000100,0x0" allowed_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" online_cpuset="0x01000000,0x00000100,0x0" allowed_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" online_cpuset="0x01000000,0x00000100,0x0" allowed_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="21" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" online_cpuset="0x01000000,0x00000100,0x0" allowed_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+                  <object type="PU" os_index="40" cpuset="0x00000100,0x0" complete_cpuset="0x00000100,0x0" online_cpuset="0x00000100,0x0" allowed_cpuset="0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                  <object type="PU" os_index="88" cpuset="0x01000000,,0x0" complete_cpuset="0x01000000,,0x0" online_cpuset="0x01000000,,0x0" allowed_cpuset="0x01000000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" online_cpuset="0x02000000,0x00000200,0x0" allowed_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" online_cpuset="0x02000000,0x00000200,0x0" allowed_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" online_cpuset="0x02000000,0x00000200,0x0" allowed_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="22" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" online_cpuset="0x02000000,0x00000200,0x0" allowed_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+                  <object type="PU" os_index="41" cpuset="0x00000200,0x0" complete_cpuset="0x00000200,0x0" online_cpuset="0x00000200,0x0" allowed_cpuset="0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                  <object type="PU" os_index="89" cpuset="0x02000000,,0x0" complete_cpuset="0x02000000,,0x0" online_cpuset="0x02000000,,0x0" allowed_cpuset="0x02000000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="7" cpuset="0xfc000000,0x0000fc00,0x0" complete_cpuset="0xfc000000,0x0000fc00,0x0" online_cpuset="0xfc000000,0x0000fc00,0x0" allowed_cpuset="0xfc000000,0x0000fc00,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" local_memory="33819033600">
+        <page_type size="4096" count="8256600"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Cache" cpuset="0x1c000000,0x00001c00,0x0" complete_cpuset="0x1c000000,0x00001c00,0x0" online_cpuset="0x1c000000,0x00001c00,0x0" allowed_cpuset="0x1c000000,0x00001c00,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" online_cpuset="0x04000000,0x00000400,0x0" allowed_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" online_cpuset="0x04000000,0x00000400,0x0" allowed_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" online_cpuset="0x04000000,0x00000400,0x0" allowed_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="24" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" online_cpuset="0x04000000,0x00000400,0x0" allowed_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+                  <object type="PU" os_index="42" cpuset="0x00000400,0x0" complete_cpuset="0x00000400,0x0" online_cpuset="0x00000400,0x0" allowed_cpuset="0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                  <object type="PU" os_index="90" cpuset="0x04000000,,0x0" complete_cpuset="0x04000000,,0x0" online_cpuset="0x04000000,,0x0" allowed_cpuset="0x04000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" online_cpuset="0x08000000,0x00000800,0x0" allowed_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" online_cpuset="0x08000000,0x00000800,0x0" allowed_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" online_cpuset="0x08000000,0x00000800,0x0" allowed_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="25" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" online_cpuset="0x08000000,0x00000800,0x0" allowed_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+                  <object type="PU" os_index="43" cpuset="0x00000800,0x0" complete_cpuset="0x00000800,0x0" online_cpuset="0x00000800,0x0" allowed_cpuset="0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                  <object type="PU" os_index="91" cpuset="0x08000000,,0x0" complete_cpuset="0x08000000,,0x0" online_cpuset="0x08000000,,0x0" allowed_cpuset="0x08000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" online_cpuset="0x10000000,0x00001000,0x0" allowed_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" online_cpuset="0x10000000,0x00001000,0x0" allowed_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" online_cpuset="0x10000000,0x00001000,0x0" allowed_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="26" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" online_cpuset="0x10000000,0x00001000,0x0" allowed_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+                  <object type="PU" os_index="44" cpuset="0x00001000,0x0" complete_cpuset="0x00001000,0x0" online_cpuset="0x00001000,0x0" allowed_cpuset="0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                  <object type="PU" os_index="92" cpuset="0x10000000,,0x0" complete_cpuset="0x10000000,,0x0" online_cpuset="0x10000000,,0x0" allowed_cpuset="0x10000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" cpuset="0xe0000000,0x0000e000,0x0" complete_cpuset="0xe0000000,0x0000e000,0x0" online_cpuset="0xe0000000,0x0000e000,0x0" allowed_cpuset="0xe0000000,0x0000e000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" online_cpuset="0x20000000,0x00002000,0x0" allowed_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" online_cpuset="0x20000000,0x00002000,0x0" allowed_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" online_cpuset="0x20000000,0x00002000,0x0" allowed_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="28" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" online_cpuset="0x20000000,0x00002000,0x0" allowed_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+                  <object type="PU" os_index="45" cpuset="0x00002000,0x0" complete_cpuset="0x00002000,0x0" online_cpuset="0x00002000,0x0" allowed_cpuset="0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                  <object type="PU" os_index="93" cpuset="0x20000000,,0x0" complete_cpuset="0x20000000,,0x0" online_cpuset="0x20000000,,0x0" allowed_cpuset="0x20000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" online_cpuset="0x40000000,0x00004000,0x0" allowed_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" online_cpuset="0x40000000,0x00004000,0x0" allowed_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" online_cpuset="0x40000000,0x00004000,0x0" allowed_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="29" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" online_cpuset="0x40000000,0x00004000,0x0" allowed_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+                  <object type="PU" os_index="46" cpuset="0x00004000,0x0" complete_cpuset="0x00004000,0x0" online_cpuset="0x00004000,0x0" allowed_cpuset="0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                  <object type="PU" os_index="94" cpuset="0x40000000,,0x0" complete_cpuset="0x40000000,,0x0" online_cpuset="0x40000000,,0x0" allowed_cpuset="0x40000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" online_cpuset="0x80000000,0x00008000,0x0" allowed_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" online_cpuset="0x80000000,0x00008000,0x0" allowed_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" online_cpuset="0x80000000,0x00008000,0x0" allowed_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="30" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" online_cpuset="0x80000000,0x00008000,0x0" allowed_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+                  <object type="PU" os_index="47" cpuset="0x00008000,0x0" complete_cpuset="0x00008000,0x0" online_cpuset="0x00008000,0x0" allowed_cpuset="0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                  <object type="PU" os_index="95" cpuset="0x80000000,,0x0" complete_cpuset="0x80000000,,0x0" online_cpuset="0x80000000,,0x0" allowed_cpuset="0x80000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" os_index="7" bridge_type="0-1" depth="0" bridge_pci="0000:[70-75]">
+          <object type="Bridge" os_index="458801" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[71-73]" pci_busid="0000:70:03.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="3.938462">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" os_index="462848" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="2" bridge_pci="0000:[72-73]" pci_busid="0000:71:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+              <object type="Bridge" os_index="466944" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="3" bridge_pci="0000:[73-73]" pci_busid="0000:72:00.0" pci_type="0604 [1002:14a1] [0000:0000] 00" pci_link_speed="31.507692">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="PCIDev" os_index="471040" name="Advanced Micro Devices, Inc. [AMD/ATI] Vega 20" pci_busid="0000:73:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <info name="PCIDevice" value="Vega 20"/>
+                  <object type="OSDev" name="card4" osdev_type="1"/>
+                  <object type="OSDev" name="renderD131" osdev_type="1"/>
+                  <object type="OSDev" name="controlD68" osdev_type="1"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+    </object>
+  </object>
+</topology>
diff --git a/t/hwloc-data/1N/hwloc-versions/v2.1.0/0.xml b/t/hwloc-data/1N/hwloc-versions/v2.1.0/0.xml
new file mode 100644
index 000000000000..8ad159806f71
--- /dev/null
+++ b/t/hwloc-data/1N/hwloc-versions/v2.1.0/0.xml
@@ -0,0 +1,940 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE topology SYSTEM "hwloc2.dtd">
+<topology version="2.0">
+  <object type="Machine" os_index="0" cpuset="0xffffffff,0xffffffff,0xffffffff" complete_cpuset="0xffffffff,0xffffffff,0xffffffff" allowed_cpuset="0xffffffff,0xffffffff,0xffffffff" nodeset="0x000000ff" complete_nodeset="0x000000ff" allowed_nodeset="0x000000ff" gp_index="1">
+    <info name="DMIProductName" value="Altus XO1114GT"/>
+    <info name="DMIProductVersion" value="0100"/>
+    <info name="DMIBoardVendor" value="GIGABYTE"/>
+    <info name="DMIBoardName" value="MZ71-OP0-ZB"/>
+    <info name="DMIBoardVersion" value="01000100"/>
+    <info name="DMIBoardAssetTag" value="01234567890123456789AB"/>
+    <info name="DMIChassisVendor" value="Penguin Computing"/>
+    <info name="DMIChassisType" value="23"/>
+    <info name="DMIChassisVersion" value="01234567"/>
+    <info name="DMIChassisAssetTag" value="01234567890123456789AB"/>
+    <info name="DMIBIOSVendor" value="GIGABYTE"/>
+    <info name="DMIBIOSVersion" value="F06c"/>
+    <info name="DMIBIOSDate" value="12/14/2018"/>
+    <info name="DMISysVendor" value="Penguin Computing"/>
+    <info name="Backend" value="Linux"/>
+    <info name="LinuxCgroup" value="/"/>
+    <info name="OSName" value="Linux"/>
+    <info name="OSRelease" value="3.10.0-1127.0.0.1chaos.ch6.x86_64"/>
+    <info name="OSVersion" value="#1 SMP Fri Apr 3 08:56:52 PDT 2020"/>
+    <info name="HostName" value="corona40"/>
+    <info name="Architecture" value="x86_64"/>
+    <info name="hwlocVersion" value="2.1.0"/>
+    <info name="ProcessName" value="lstopo"/>
+    <object type="Package" os_index="0" cpuset="0x000000ff,0xffff0000,0x00ffffff" complete_cpuset="0x000000ff,0xffff0000,0x00ffffff" nodeset="0x0000000f" complete_nodeset="0x0000000f" gp_index="3">
+      <info name="CPUVendor" value="AuthenticAMD"/>
+      <info name="CPUFamilyNumber" value="23"/>
+      <info name="CPUModelNumber" value="1"/>
+      <info name="CPUModel" value="AMD EPYC 7401 24-Core Processor                "/>
+      <info name="CPUStepping" value="2"/>
+      <object type="Group" cpuset="0x003f0000,0x0000003f" complete_cpuset="0x003f0000,0x0000003f" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="316" kind="1001" subkind="0">
+        <object type="NUMANode" os_index="0" cpuset="0x003f0000,0x0000003f" complete_cpuset="0x003f0000,0x0000003f" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="308" local_memory="33299529728">
+          <page_type size="4096" count="8129768"/>
+          <page_type size="2097152" count="0"/>
+          <page_type size="1073741824" count="0"/>
+        </object>
+        <object type="L3Cache" cpuset="0x00070000,0x00000007" complete_cpuset="0x00070000,0x00000007" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="8" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="7" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="5" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="6" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="0" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="2">
+                  <object type="PU" os_index="0" cpuset="0x00000001" complete_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="4"/>
+                  <object type="PU" os_index="48" cpuset="0x00010000,0x0" complete_cpuset="0x00010000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="260"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="13" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="11" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="12" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="1" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="9">
+                  <object type="PU" os_index="1" cpuset="0x00000002" complete_cpuset="0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="10"/>
+                  <object type="PU" os_index="49" cpuset="0x00020000,0x0" complete_cpuset="0x00020000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="261"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="18" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="16" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="17" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="2" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="14">
+                  <object type="PU" os_index="2" cpuset="0x00000004" complete_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="15"/>
+                  <object type="PU" os_index="50" cpuset="0x00040000,0x0" complete_cpuset="0x00040000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="262"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="L3Cache" cpuset="0x00380000,0x00000038" complete_cpuset="0x00380000,0x00000038" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="24" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="23" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="21" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="22" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="4" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="19">
+                  <object type="PU" os_index="3" cpuset="0x00000008" complete_cpuset="0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="20"/>
+                  <object type="PU" os_index="51" cpuset="0x00080000,0x0" complete_cpuset="0x00080000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="263"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="29" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="27" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="28" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="5" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="25">
+                  <object type="PU" os_index="4" cpuset="0x00000010" complete_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="26"/>
+                  <object type="PU" os_index="52" cpuset="0x00100000,0x0" complete_cpuset="0x00100000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="264"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="34" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="32" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="33" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="6" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="30">
+                  <object type="PU" os_index="5" cpuset="0x00000020" complete_cpuset="0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="31"/>
+                  <object type="PU" os_index="53" cpuset="0x00200000,0x0" complete_cpuset="0x00200000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" gp_index="265"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" gp_index="375" bridge_type="0-1" depth="0" bridge_pci="0000:[00-07]">
+          <object type="Bridge" gp_index="328" bridge_type="1-1" depth="1" bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [1022:1453] [1458:1000] 00" pci_link_speed="3.938462">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="PCIDev" gp_index="334" pci_busid="0000:01:00.0" pci_type="0108 [1c58:0023] [1c58:0023] 02" pci_link_speed="3.938462">
+              <info name="PCISlot" value="1"/>
+              <info name="PCIVendor" value="HGST, Inc."/>
+              <info name="PCIDevice" value="Ultrastar SN200 Series NVMe SSD"/>
+              <object type="OSDev" gp_index="384" name="nvme0n1" subtype="Disk" osdev_type="0">
+                <info name="Size" value="1562813784"/>
+                <info name="SectorSize" value="4096"/>
+                <info name="LinuxDeviceID" value="259:0"/>
+                <info name="Model" value="HUSMR7616BDP301"/>
+                <info name="SerialNumber" value="SDM0000107A5"/>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" gp_index="330" bridge_type="1-1" depth="1" bridge_pci="0000:[03-03]" pci_busid="0000:00:01.3" pci_type="0604 [1022:1453] [1458:1000] 00" pci_link_speed="1.000000">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="PCIDev" gp_index="339" pci_busid="0000:03:00.0" pci_type="0200 [8086:1521] [1458:0000] 01" pci_link_speed="1.000000">
+              <info name="PCIVendor" value="Intel Corporation"/>
+              <info name="PCIDevice" value="I350 Gigabit Network Connection"/>
+              <object type="OSDev" gp_index="387" name="enp3s0f0" osdev_type="2">
+                <info name="Address" value="e0:d5:5e:ca:b0:22"/>
+              </object>
+            </object>
+            <object type="PCIDev" gp_index="340" pci_busid="0000:03:00.1" pci_type="0200 [8086:1521] [1458:0000] 01" pci_link_speed="1.000000">
+              <info name="PCIVendor" value="Intel Corporation"/>
+              <info name="PCIDevice" value="I350 Gigabit Network Connection"/>
+              <object type="OSDev" gp_index="386" name="enp3s0f1" osdev_type="2">
+                <info name="Address" value="e0:d5:5e:ca:b0:23"/>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" gp_index="331" bridge_type="1-1" depth="1" bridge_pci="0000:[04-05]" pci_busid="0000:00:01.6" pci_type="0604 [1022:1453] [1458:1000] 00" pci_link_speed="0.250000">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" gp_index="342" bridge_type="1-1" depth="2" bridge_pci="0000:[05-05]" pci_busid="0000:04:00.0" pci_type="0604 [1a03:1150] [1a03:1150] 04" pci_link_speed="0.250000">
+              <info name="PCIVendor" value="ASPEED Technology, Inc."/>
+              <info name="PCIDevice" value="AST1150 PCI-to-PCI Bridge"/>
+              <object type="PCIDev" gp_index="344" pci_busid="0000:05:00.0" pci_type="0300 [1a03:2000] [1458:1000] 41" pci_link_speed="0.000000">
+                <info name="PCIVendor" value="ASPEED Technology, Inc."/>
+                <info name="PCIDevice" value="ASPEED Graphics Family"/>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" gp_index="333" bridge_type="1-1" depth="1" bridge_pci="0000:[07-07]" pci_busid="0000:00:08.1" pci_type="0604 [1022:1454] [1000:1458] 00" pci_link_speed="15.753846">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) Internal PCIe GPP Bridge 0 to Bus B"/>
+            <object type="PCIDev" gp_index="345" pci_busid="0000:07:00.2" pci_type="0106 [1022:7901] [1458:1000] 51" pci_link_speed="15.753846">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="FCH SATA Controller [AHCI mode]"/>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="Group" cpuset="0x0fc00000,0x00000fc0" complete_cpuset="0x0fc00000,0x00000fc0" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="317" kind="1001" subkind="0">
+        <object type="NUMANode" os_index="1" cpuset="0x0fc00000,0x00000fc0" complete_cpuset="0x0fc00000,0x00000fc0" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="309" local_memory="33819570176">
+          <page_type size="4096" count="8256731"/>
+          <page_type size="2097152" count="0"/>
+          <page_type size="1073741824" count="0"/>
+        </object>
+        <object type="L3Cache" cpuset="0x01c00000,0x000001c0" complete_cpuset="0x01c00000,0x000001c0" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="40" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="39" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="37" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="38" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="8" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="35">
+                  <object type="PU" os_index="6" cpuset="0x00000040" complete_cpuset="0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="36"/>
+                  <object type="PU" os_index="54" cpuset="0x00400000,0x0" complete_cpuset="0x00400000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="266"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="45" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="43" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="44" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="9" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="41">
+                  <object type="PU" os_index="7" cpuset="0x00000080" complete_cpuset="0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="42"/>
+                  <object type="PU" os_index="55" cpuset="0x00800000,0x0" complete_cpuset="0x00800000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="267"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="50" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="48" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="49" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="10" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="46">
+                  <object type="PU" os_index="8" cpuset="0x00000100" complete_cpuset="0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="47"/>
+                  <object type="PU" os_index="56" cpuset="0x01000000,0x0" complete_cpuset="0x01000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="268"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="L3Cache" cpuset="0x0e000000,0x00000e00" complete_cpuset="0x0e000000,0x00000e00" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="56" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="55" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="53" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="54" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="12" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="51">
+                  <object type="PU" os_index="9" cpuset="0x00000200" complete_cpuset="0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="52"/>
+                  <object type="PU" os_index="57" cpuset="0x02000000,0x0" complete_cpuset="0x02000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="269"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="61" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="59" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="60" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="13" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="57">
+                  <object type="PU" os_index="10" cpuset="0x00000400" complete_cpuset="0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="58"/>
+                  <object type="PU" os_index="58" cpuset="0x04000000,0x0" complete_cpuset="0x04000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="270"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="66" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="64" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="65" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="14" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="62">
+                  <object type="PU" os_index="11" cpuset="0x00000800" complete_cpuset="0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="63"/>
+                  <object type="PU" os_index="59" cpuset="0x08000000,0x0" complete_cpuset="0x08000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" gp_index="271"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" gp_index="376" bridge_type="0-1" depth="0" bridge_pci="0000:[10-15]">
+          <object type="Bridge" gp_index="346" bridge_type="1-1" depth="1" bridge_pci="0000:[11-13]" pci_busid="0000:10:01.1" pci_type="0604 [1022:1453] [1458:1000] 00" pci_link_speed="3.938462">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" gp_index="349" bridge_type="1-1" depth="2" bridge_pci="0000:[12-13]" pci_busid="0000:11:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+              <object type="Bridge" gp_index="350" bridge_type="1-1" depth="3" bridge_pci="0000:[13-13]" pci_busid="0000:12:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="PCIDev" gp_index="351" pci_busid="0000:13:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <info name="PCIDevice" value="Vega 20"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="Group" cpuset="0x00000003,0xf0000000,0x0003f000" complete_cpuset="0x00000003,0xf0000000,0x0003f000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="318" kind="1001" subkind="0">
+        <object type="NUMANode" os_index="2" cpuset="0x00000003,0xf0000000,0x0003f000" complete_cpuset="0x00000003,0xf0000000,0x0003f000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="310" local_memory="33819574272">
+          <page_type size="4096" count="8256732"/>
+          <page_type size="2097152" count="0"/>
+          <page_type size="1073741824" count="0"/>
+        </object>
+        <object type="L3Cache" cpuset="0x70000000,0x00007000" complete_cpuset="0x70000000,0x00007000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="72" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="71" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="69" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="70" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="16" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="67">
+                  <object type="PU" os_index="12" cpuset="0x00001000" complete_cpuset="0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="68"/>
+                  <object type="PU" os_index="60" cpuset="0x10000000,0x0" complete_cpuset="0x10000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="272"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="77" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="75" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="76" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="17" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="73">
+                  <object type="PU" os_index="13" cpuset="0x00002000" complete_cpuset="0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="74"/>
+                  <object type="PU" os_index="61" cpuset="0x20000000,0x0" complete_cpuset="0x20000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="273"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="82" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="80" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="81" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="18" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="78">
+                  <object type="PU" os_index="14" cpuset="0x00004000" complete_cpuset="0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="79"/>
+                  <object type="PU" os_index="62" cpuset="0x40000000,0x0" complete_cpuset="0x40000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="274"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="L3Cache" cpuset="0x00000003,0x80000000,0x00038000" complete_cpuset="0x00000003,0x80000000,0x00038000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="88" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="87" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="85" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="86" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="20" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="83">
+                  <object type="PU" os_index="15" cpuset="0x00008000" complete_cpuset="0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="84"/>
+                  <object type="PU" os_index="63" cpuset="0x80000000,0x0" complete_cpuset="0x80000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="275"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="93" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="91" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="92" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="21" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="89">
+                  <object type="PU" os_index="16" cpuset="0x00010000" complete_cpuset="0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="90"/>
+                  <object type="PU" os_index="64" cpuset="0x00000001,,0x0" complete_cpuset="0x00000001,,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="276"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="98" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="96" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="97" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="22" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="94">
+                  <object type="PU" os_index="17" cpuset="0x00020000" complete_cpuset="0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="95"/>
+                  <object type="PU" os_index="65" cpuset="0x00000002,,0x0" complete_cpuset="0x00000002,,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" gp_index="277"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" gp_index="377" bridge_type="0-1" depth="0" bridge_pci="0000:[20-25]">
+          <object type="Bridge" gp_index="352" bridge_type="1-1" depth="1" bridge_pci="0000:[21-23]" pci_busid="0000:20:03.1" pci_type="0604 [1022:1453] [1458:1000] 00" pci_link_speed="3.938462">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" gp_index="355" bridge_type="1-1" depth="2" bridge_pci="0000:[22-23]" pci_busid="0000:21:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+              <object type="Bridge" gp_index="356" bridge_type="1-1" depth="3" bridge_pci="0000:[23-23]" pci_busid="0000:22:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="PCIDev" gp_index="357" pci_busid="0000:23:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <info name="PCIDevice" value="Vega 20"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="Group" cpuset="0x000000fc,,0x00fc0000" complete_cpuset="0x000000fc,,0x00fc0000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="319" kind="1001" subkind="0">
+        <object type="NUMANode" os_index="3" cpuset="0x000000fc,,0x00fc0000" complete_cpuset="0x000000fc,,0x00fc0000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="311" local_memory="33802997760">
+          <page_type size="4096" count="8252685"/>
+          <page_type size="2097152" count="0"/>
+          <page_type size="1073741824" count="0"/>
+        </object>
+        <object type="L3Cache" cpuset="0x0000001c,,0x001c0000" complete_cpuset="0x0000001c,,0x001c0000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="104" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="103" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="101" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="102" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="24" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="99">
+                  <object type="PU" os_index="18" cpuset="0x00040000" complete_cpuset="0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="100"/>
+                  <object type="PU" os_index="66" cpuset="0x00000004,,0x0" complete_cpuset="0x00000004,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="278"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="109" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="107" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="108" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="25" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="105">
+                  <object type="PU" os_index="19" cpuset="0x00080000" complete_cpuset="0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="106"/>
+                  <object type="PU" os_index="67" cpuset="0x00000008,,0x0" complete_cpuset="0x00000008,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="279"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="114" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="112" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="113" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="26" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="110">
+                  <object type="PU" os_index="20" cpuset="0x00100000" complete_cpuset="0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="111"/>
+                  <object type="PU" os_index="68" cpuset="0x00000010,,0x0" complete_cpuset="0x00000010,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="280"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="L3Cache" cpuset="0x000000e0,,0x00e00000" complete_cpuset="0x000000e0,,0x00e00000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="120" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="119" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="117" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="118" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="28" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="115">
+                  <object type="PU" os_index="21" cpuset="0x00200000" complete_cpuset="0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="116"/>
+                  <object type="PU" os_index="69" cpuset="0x00000020,,0x0" complete_cpuset="0x00000020,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="281"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="125" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="123" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="124" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="29" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="121">
+                  <object type="PU" os_index="22" cpuset="0x00400000" complete_cpuset="0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="122"/>
+                  <object type="PU" os_index="70" cpuset="0x00000040,,0x0" complete_cpuset="0x00000040,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="282"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="130" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="128" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="129" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="30" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="126">
+                  <object type="PU" os_index="23" cpuset="0x00800000" complete_cpuset="0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="127"/>
+                  <object type="PU" os_index="71" cpuset="0x00000080,,0x0" complete_cpuset="0x00000080,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" gp_index="283"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" gp_index="378" bridge_type="0-1" depth="0" bridge_pci="0000:[30-36]">
+          <object type="Bridge" gp_index="358" bridge_type="1-1" depth="1" bridge_pci="0000:[31-34]" pci_busid="0000:30:03.1" pci_type="0604 [1022:1453] [1458:1000] 00" pci_link_speed="15.753846">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" gp_index="361" bridge_type="1-1" depth="2" bridge_pci="0000:[32-34]" pci_busid="0000:31:00.0" pci_type="0604 [10b5:8747] [10b5:8747] ca" pci_link_speed="15.753846">
+              <info name="PCIVendor" value="PLX Technology, Inc."/>
+              <info name="PCIDevice" value="PEX 8747 48-Lane, 5-Port PCI Express Gen 3 (8.0 GT/s) Switch"/>
+              <object type="Bridge" gp_index="362" bridge_type="1-1" depth="3" bridge_pci="0000:[33-33]" pci_busid="0000:32:08.0" pci_type="0604 [10b5:8747] [10b5:8747] ca" pci_link_speed="15.753846">
+                <info name="PCIVendor" value="PLX Technology, Inc."/>
+                <info name="PCIDevice" value="PEX 8747 48-Lane, 5-Port PCI Express Gen 3 (8.0 GT/s) Switch"/>
+                <object type="PCIDev" gp_index="364" pci_busid="0000:33:00.0" pci_type="0207 [15b3:101b] [15b3:0006] 00" pci_link_speed="15.753846">
+                  <info name="PCIVendor" value="Mellanox Technologies"/>
+                  <info name="PCIDevice" value="MT28908 Family [ConnectX-6]"/>
+                  <object type="OSDev" gp_index="385" name="hsi0" osdev_type="2">
+                    <info name="Address" value="80:00:01:07:fe:80:00:00:00:00:00:00:98:03:9b:03:00:82:9d:b4"/>
+                    <info name="Port" value="1"/>
+                  </object>
+                  <object type="OSDev" gp_index="388" name="mlx5_0" osdev_type="3">
+                    <info name="NodeGUID" value="9803:9b03:0082:9db4"/>
+                    <info name="SysImageGUID" value="9803:9b03:0082:9db4"/>
+                    <info name="Port1State" value="4"/>
+                    <info name="Port1LID" value="0x1c"/>
+                    <info name="Port1LMC" value="0"/>
+                    <info name="Port1GID0" value="fe80:0000:0000:0000:9803:9b03:0082:9db4"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+    </object>
+    <object type="Package" os_index="1" cpuset="0xffffff00,0x0000ffff,0xff000000" complete_cpuset="0xffffff00,0x0000ffff,0xff000000" nodeset="0x000000f0" complete_nodeset="0x000000f0" gp_index="132">
+      <info name="CPUVendor" value="AuthenticAMD"/>
+      <info name="CPUFamilyNumber" value="23"/>
+      <info name="CPUModelNumber" value="1"/>
+      <info name="CPUModel" value="AMD EPYC 7401 24-Core Processor                "/>
+      <info name="CPUStepping" value="2"/>
+      <object type="Group" cpuset="0x00003f00,,0x3f000000" complete_cpuset="0x00003f00,,0x3f000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="320" kind="1001" subkind="0">
+        <object type="NUMANode" os_index="4" cpuset="0x00003f00,,0x3f000000" complete_cpuset="0x00003f00,,0x3f000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="312" local_memory="33819574272">
+          <page_type size="4096" count="8256732"/>
+          <page_type size="2097152" count="0"/>
+          <page_type size="1073741824" count="0"/>
+        </object>
+        <object type="L3Cache" cpuset="0x00000700,,0x07000000" complete_cpuset="0x00000700,,0x07000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="137" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="136" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="134" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="135" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="0" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="131">
+                  <object type="PU" os_index="24" cpuset="0x01000000" complete_cpuset="0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="133"/>
+                  <object type="PU" os_index="72" cpuset="0x00000100,,0x0" complete_cpuset="0x00000100,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="284"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="142" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="140" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="141" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="1" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="138">
+                  <object type="PU" os_index="25" cpuset="0x02000000" complete_cpuset="0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="139"/>
+                  <object type="PU" os_index="73" cpuset="0x00000200,,0x0" complete_cpuset="0x00000200,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="285"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="147" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="145" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="146" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="2" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="143">
+                  <object type="PU" os_index="26" cpuset="0x04000000" complete_cpuset="0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="144"/>
+                  <object type="PU" os_index="74" cpuset="0x00000400,,0x0" complete_cpuset="0x00000400,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="286"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="L3Cache" cpuset="0x00003800,,0x38000000" complete_cpuset="0x00003800,,0x38000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="153" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="152" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="150" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="151" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="4" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="148">
+                  <object type="PU" os_index="27" cpuset="0x08000000" complete_cpuset="0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="149"/>
+                  <object type="PU" os_index="75" cpuset="0x00000800,,0x0" complete_cpuset="0x00000800,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="287"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="158" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="156" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="157" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="5" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="154">
+                  <object type="PU" os_index="28" cpuset="0x10000000" complete_cpuset="0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="155"/>
+                  <object type="PU" os_index="76" cpuset="0x00001000,,0x0" complete_cpuset="0x00001000,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="288"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="163" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="161" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="162" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="6" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="159">
+                  <object type="PU" os_index="29" cpuset="0x20000000" complete_cpuset="0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="160"/>
+                  <object type="PU" os_index="77" cpuset="0x00002000,,0x0" complete_cpuset="0x00002000,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" gp_index="289"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="Group" cpuset="0x000fc000,0x0000000f,0xc0000000" complete_cpuset="0x000fc000,0x0000000f,0xc0000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="321" kind="1001" subkind="0">
+        <object type="NUMANode" os_index="5" cpuset="0x000fc000,0x0000000f,0xc0000000" complete_cpuset="0x000fc000,0x0000000f,0xc0000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="313" local_memory="33819570176">
+          <page_type size="4096" count="8256731"/>
+          <page_type size="2097152" count="0"/>
+          <page_type size="1073741824" count="0"/>
+        </object>
+        <object type="L3Cache" cpuset="0x0001c000,0x00000001,0xc0000000" complete_cpuset="0x0001c000,0x00000001,0xc0000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="169" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="168" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="166" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="167" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="8" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="164">
+                  <object type="PU" os_index="30" cpuset="0x40000000" complete_cpuset="0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="165"/>
+                  <object type="PU" os_index="78" cpuset="0x00004000,,0x0" complete_cpuset="0x00004000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="290"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="174" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="172" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="173" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="9" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="170">
+                  <object type="PU" os_index="31" cpuset="0x80000000" complete_cpuset="0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="171"/>
+                  <object type="PU" os_index="79" cpuset="0x00008000,,0x0" complete_cpuset="0x00008000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="291"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="179" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="177" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="178" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="10" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="175">
+                  <object type="PU" os_index="32" cpuset="0x00000001,0x0" complete_cpuset="0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="176"/>
+                  <object type="PU" os_index="80" cpuset="0x00010000,,0x0" complete_cpuset="0x00010000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="292"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="L3Cache" cpuset="0x000e0000,0x0000000e,0x0" complete_cpuset="0x000e0000,0x0000000e,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="185" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="184" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="182" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="183" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="12" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="180">
+                  <object type="PU" os_index="33" cpuset="0x00000002,0x0" complete_cpuset="0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="181"/>
+                  <object type="PU" os_index="81" cpuset="0x00020000,,0x0" complete_cpuset="0x00020000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="293"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="190" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="188" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="189" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="13" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="186">
+                  <object type="PU" os_index="34" cpuset="0x00000004,0x0" complete_cpuset="0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="187"/>
+                  <object type="PU" os_index="82" cpuset="0x00040000,,0x0" complete_cpuset="0x00040000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="294"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="195" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="193" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="194" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="14" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="191">
+                  <object type="PU" os_index="35" cpuset="0x00000008,0x0" complete_cpuset="0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="192"/>
+                  <object type="PU" os_index="83" cpuset="0x00080000,,0x0" complete_cpuset="0x00080000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" gp_index="295"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" gp_index="380" bridge_type="0-1" depth="0" bridge_pci="0000:[50-55]">
+          <object type="Bridge" gp_index="369" bridge_type="1-1" depth="1" bridge_pci="0000:[51-53]" pci_busid="0000:50:01.1" pci_type="0604 [1022:1453] [1458:1000] 00" pci_link_speed="3.938462">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" gp_index="372" bridge_type="1-1" depth="2" bridge_pci="0000:[52-53]" pci_busid="0000:51:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+              <object type="Bridge" gp_index="373" bridge_type="1-1" depth="3" bridge_pci="0000:[53-53]" pci_busid="0000:52:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="PCIDev" gp_index="374" pci_busid="0000:53:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <info name="PCIDevice" value="Vega 20"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="Group" cpuset="0x03f00000,0x000003f0,0x0" complete_cpuset="0x03f00000,0x000003f0,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="322" kind="1001" subkind="0">
+        <object type="NUMANode" os_index="6" cpuset="0x03f00000,0x000003f0,0x0" complete_cpuset="0x03f00000,0x000003f0,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="314" local_memory="33819574272">
+          <page_type size="4096" count="8256732"/>
+          <page_type size="2097152" count="0"/>
+          <page_type size="1073741824" count="0"/>
+        </object>
+        <object type="L3Cache" cpuset="0x00700000,0x00000070,0x0" complete_cpuset="0x00700000,0x00000070,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="201" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="200" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="198" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="199" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="16" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="196">
+                  <object type="PU" os_index="36" cpuset="0x00000010,0x0" complete_cpuset="0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="197"/>
+                  <object type="PU" os_index="84" cpuset="0x00100000,,0x0" complete_cpuset="0x00100000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="296"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="206" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="204" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="205" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="17" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="202">
+                  <object type="PU" os_index="37" cpuset="0x00000020,0x0" complete_cpuset="0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="203"/>
+                  <object type="PU" os_index="85" cpuset="0x00200000,,0x0" complete_cpuset="0x00200000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="297"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="211" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="209" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="210" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="18" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="207">
+                  <object type="PU" os_index="38" cpuset="0x00000040,0x0" complete_cpuset="0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="208"/>
+                  <object type="PU" os_index="86" cpuset="0x00400000,,0x0" complete_cpuset="0x00400000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="298"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="L3Cache" cpuset="0x03800000,0x00000380,0x0" complete_cpuset="0x03800000,0x00000380,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="217" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="216" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="214" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="215" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="20" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="212">
+                  <object type="PU" os_index="39" cpuset="0x00000080,0x0" complete_cpuset="0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="213"/>
+                  <object type="PU" os_index="87" cpuset="0x00800000,,0x0" complete_cpuset="0x00800000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="299"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="222" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="220" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="221" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="21" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="218">
+                  <object type="PU" os_index="40" cpuset="0x00000100,0x0" complete_cpuset="0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="219"/>
+                  <object type="PU" os_index="88" cpuset="0x01000000,,0x0" complete_cpuset="0x01000000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="300"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="227" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="225" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="226" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="22" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="223">
+                  <object type="PU" os_index="41" cpuset="0x00000200,0x0" complete_cpuset="0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="224"/>
+                  <object type="PU" os_index="89" cpuset="0x02000000,,0x0" complete_cpuset="0x02000000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" gp_index="301"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="Group" cpuset="0xfc000000,0x0000fc00,0x0" complete_cpuset="0xfc000000,0x0000fc00,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="323" kind="1001" subkind="0">
+        <object type="NUMANode" os_index="7" cpuset="0xfc000000,0x0000fc00,0x0" complete_cpuset="0xfc000000,0x0000fc00,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="315" local_memory="33819033600">
+          <page_type size="4096" count="8256600"/>
+          <page_type size="2097152" count="0"/>
+          <page_type size="1073741824" count="0"/>
+        </object>
+        <object type="L3Cache" cpuset="0x1c000000,0x00001c00,0x0" complete_cpuset="0x1c000000,0x00001c00,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="233" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="232" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="230" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="231" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="24" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="228">
+                  <object type="PU" os_index="42" cpuset="0x00000400,0x0" complete_cpuset="0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="229"/>
+                  <object type="PU" os_index="90" cpuset="0x04000000,,0x0" complete_cpuset="0x04000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="302"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="238" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="236" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="237" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="25" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="234">
+                  <object type="PU" os_index="43" cpuset="0x00000800,0x0" complete_cpuset="0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="235"/>
+                  <object type="PU" os_index="91" cpuset="0x08000000,,0x0" complete_cpuset="0x08000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="303"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="243" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="241" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="242" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="26" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="239">
+                  <object type="PU" os_index="44" cpuset="0x00001000,0x0" complete_cpuset="0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="240"/>
+                  <object type="PU" os_index="92" cpuset="0x10000000,,0x0" complete_cpuset="0x10000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="304"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="L3Cache" cpuset="0xe0000000,0x0000e000,0x0" complete_cpuset="0xe0000000,0x0000e000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="249" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="L2Cache" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="248" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="246" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="247" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="28" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="244">
+                  <object type="PU" os_index="45" cpuset="0x00002000,0x0" complete_cpuset="0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="245"/>
+                  <object type="PU" os_index="93" cpuset="0x20000000,,0x0" complete_cpuset="0x20000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="305"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="254" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="252" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="253" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="29" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="250">
+                  <object type="PU" os_index="46" cpuset="0x00004000,0x0" complete_cpuset="0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="251"/>
+                  <object type="PU" os_index="94" cpuset="0x40000000,,0x0" complete_cpuset="0x40000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="306"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="L2Cache" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="259" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="L1Cache" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="257" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="L1iCache" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="258" cache_size="65536" depth="1" cache_linesize="64" cache_associativity="4" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="30" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="255">
+                  <object type="PU" os_index="47" cpuset="0x00008000,0x0" complete_cpuset="0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="256"/>
+                  <object type="PU" os_index="95" cpuset="0x80000000,,0x0" complete_cpuset="0x80000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" gp_index="307"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" gp_index="382" bridge_type="0-1" depth="0" bridge_pci="0000:[70-75]">
+          <object type="Bridge" gp_index="335" bridge_type="1-1" depth="1" bridge_pci="0000:[71-73]" pci_busid="0000:70:03.1" pci_type="0604 [1022:1453] [1458:1000] 00" pci_link_speed="3.938462">
+            <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+            <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+            <object type="Bridge" gp_index="338" bridge_type="1-1" depth="2" bridge_pci="0000:[72-73]" pci_busid="0000:71:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+              <object type="Bridge" gp_index="341" bridge_type="1-1" depth="3" bridge_pci="0000:[73-73]" pci_busid="0000:72:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="PCIDev" gp_index="343" pci_busid="0000:73:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <info name="PCIDevice" value="Vega 20"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+    </object>
+    <object type="OSDev" gp_index="383" name="sda" subtype="Disk" osdev_type="0">
+      <info name="Size" value="26214400"/>
+      <info name="SectorSize" value="512"/>
+      <info name="LinuxDeviceID" value="8:0"/>
+      <info name="Vendor" value="LIO-ORG"/>
+      <info name="Model" value="FILEIO"/>
+      <info name="Revision" value="4.0"/>
+      <info name="SerialNumber" value="60014054e2824b7073e3300794400382"/>
+    </object>
+  </object>
+  <distances2 type="NUMANode" nbobjs="8" kind="5" name="NUMALatency" indexing="os">
+    <indexes length="16">0 1 2 3 4 5 6 7 </indexes>
+    <u64values length="30">10 16 16 16 32 32 32 32 16 10 </u64values>
+    <u64values length="30">16 16 32 32 32 32 16 16 10 16 </u64values>
+    <u64values length="30">32 32 32 32 16 16 16 10 32 32 </u64values>
+    <u64values length="30">32 32 32 32 32 32 10 16 16 16 </u64values>
+    <u64values length="30">32 32 32 32 16 10 16 16 32 32 </u64values>
+    <u64values length="30">32 32 16 16 10 16 32 32 32 32 </u64values>
+    <u64values length="12">16 16 16 10 </u64values>
+  </distances2>
+</topology>
diff --git a/t/hwloc-data/1N/hwloc-versions/v2to1/0.xml b/t/hwloc-data/1N/hwloc-versions/v2to1/0.xml
new file mode 100644
index 000000000000..3ba2ebd8414a
--- /dev/null
+++ b/t/hwloc-data/1N/hwloc-versions/v2to1/0.xml
@@ -0,0 +1,568 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE topology SYSTEM "hwloc.dtd">
+<topology>
+  <object type="Machine" os_index="0" cpuset="0xffffffff,0xffffffff,0xffffffff" complete_cpuset="0xffffffff,0xffffffff,0xffffffff" online_cpuset="0xffffffff,0xffffffff,0xffffffff" allowed_cpuset="0xffffffff,0xffffffff,0xffffffff" nodeset="0x000000ff" complete_nodeset="0x000000ff" allowed_nodeset="0x000000ff">
+    <info name="DMIProductName" value="Altus XO1114GT"/>
+    <info name="DMIProductVersion" value="0100"/>
+    <info name="DMIBoardVendor" value="GIGABYTE"/>
+    <info name="DMIBoardName" value="MZ71-OP0-ZB"/>
+    <info name="DMIBoardVersion" value="01000100"/>
+    <info name="DMIBoardAssetTag" value="01234567890123456789AB"/>
+    <info name="DMIChassisVendor" value="Penguin Computing"/>
+    <info name="DMIChassisType" value="23"/>
+    <info name="DMIChassisVersion" value="01234567"/>
+    <info name="DMIChassisAssetTag" value="01234567890123456789AB"/>
+    <info name="DMIBIOSVendor" value="GIGABYTE"/>
+    <info name="DMIBIOSVersion" value="F06c"/>
+    <info name="DMIBIOSDate" value="12/14/2018"/>
+    <info name="DMISysVendor" value="Penguin Computing"/>
+    <info name="Backend" value="Linux"/>
+    <info name="LinuxCgroup" value="/"/>
+    <info name="OSName" value="Linux"/>
+    <info name="OSRelease" value="3.10.0-1127.0.0.1chaos.ch6.x86_64"/>
+    <info name="OSVersion" value="#1 SMP Fri Apr 3 08:56:52 PDT 2020"/>
+    <info name="HostName" value="corona40"/>
+    <info name="Architecture" value="x86_64"/>
+    <info name="hwlocVersion" value="1.11.11"/>
+    <info name="ProcessName" value="lstopo"/>
+    <distances nbobjs="8" relative_depth="3" latency_base="1.000000">
+      <latency value="10.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="16.000000"/>
+      <latency value="10.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="10.000000"/>
+      <latency value="16.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="10.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="10.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="16.000000"/>
+      <latency value="10.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="10.000000"/>
+      <latency value="16.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="16.000000"/>
+      <latency value="10.000000"/>
+    </distances>
+    <object type="Socket" os_index="0" cpuset="0x000000ff,0xffff0000,0x00ffffff" complete_cpuset="0x000000ff,0xffff0000,0x00ffffff" online_cpuset="0x000000ff,0xffff0000,0x00ffffff" allowed_cpuset="0x000000ff,0xffff0000,0x00ffffff" nodeset="0x0000000f" complete_nodeset="0x0000000f" allowed_nodeset="0x0000000f">
+      <info name="CPUVendor" value="AuthenticAMD"/>
+      <info name="CPUFamilyNumber" value="23"/>
+      <info name="CPUModelNumber" value="1"/>
+      <info name="CPUModel" value="AMD EPYC 7401 24-Core Processor"/>
+      <info name="CPUStepping" value="2"/>
+      <object type="NUMANode" os_index="0" cpuset="0x003f0000,0x0000003f" complete_cpuset="0x003f0000,0x0000003f" online_cpuset="0x003f0000,0x0000003f" allowed_cpuset="0x003f0000,0x0000003f" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" local_memory="33299529728">
+        <page_type size="4096" count="8129768"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Group" cpuset="0x003f0000,0x0000003f" complete_cpuset="0x003f0000,0x0000003f" online_cpuset="0x003f0000,0x0000003f" allowed_cpuset="0x003f0000,0x0000003f" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" depth="0">
+          <object type="Cache" cpuset="0x00070000,0x00000007" complete_cpuset="0x00070000,0x00000007" online_cpuset="0x00070000,0x00000007" allowed_cpuset="0x00070000,0x00000007" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="0" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" online_cpuset="0x00010000,0x00000001" allowed_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+              <object type="PU" os_index="0" cpuset="0x00000001" complete_cpuset="0x00000001" online_cpuset="0x00000001" allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+              <object type="PU" os_index="48" cpuset="0x00010000,0x0" complete_cpuset="0x00010000,0x0" online_cpuset="0x00010000,0x0" allowed_cpuset="0x00010000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+            </object>
+            <object type="Core" os_index="1" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" online_cpuset="0x00020000,0x00000002" allowed_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+              <object type="PU" os_index="1" cpuset="0x00000002" complete_cpuset="0x00000002" online_cpuset="0x00000002" allowed_cpuset="0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+              <object type="PU" os_index="49" cpuset="0x00020000,0x0" complete_cpuset="0x00020000,0x0" online_cpuset="0x00020000,0x0" allowed_cpuset="0x00020000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+            </object>
+            <object type="Core" os_index="2" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" online_cpuset="0x00040000,0x00000004" allowed_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+              <object type="PU" os_index="2" cpuset="0x00000004" complete_cpuset="0x00000004" online_cpuset="0x00000004" allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+              <object type="PU" os_index="50" cpuset="0x00040000,0x0" complete_cpuset="0x00040000,0x0" online_cpuset="0x00040000,0x0" allowed_cpuset="0x00040000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00380000,0x00000038" complete_cpuset="0x00380000,0x00000038" online_cpuset="0x00380000,0x00000038" allowed_cpuset="0x00380000,0x00000038" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="4" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" online_cpuset="0x00080000,0x00000008" allowed_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+              <object type="PU" os_index="3" cpuset="0x00000008" complete_cpuset="0x00000008" online_cpuset="0x00000008" allowed_cpuset="0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+              <object type="PU" os_index="51" cpuset="0x00080000,0x0" complete_cpuset="0x00080000,0x0" online_cpuset="0x00080000,0x0" allowed_cpuset="0x00080000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+            </object>
+            <object type="Core" os_index="5" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" online_cpuset="0x00100000,0x00000010" allowed_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+              <object type="PU" os_index="4" cpuset="0x00000010" complete_cpuset="0x00000010" online_cpuset="0x00000010" allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+              <object type="PU" os_index="52" cpuset="0x00100000,0x0" complete_cpuset="0x00100000,0x0" online_cpuset="0x00100000,0x0" allowed_cpuset="0x00100000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+            </object>
+            <object type="Core" os_index="6" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" online_cpuset="0x00200000,0x00000020" allowed_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+              <object type="PU" os_index="5" cpuset="0x00000020" complete_cpuset="0x00000020" online_cpuset="0x00000020" allowed_cpuset="0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+              <object type="PU" os_index="53" cpuset="0x00200000,0x0" complete_cpuset="0x00200000,0x0" online_cpuset="0x00200000,0x0" allowed_cpuset="0x00200000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+            </object>
+          </object>
+          <object type="Bridge" os_index="0" bridge_type="0-1" depth="0" bridge_pci="0000:[00-07]">
+            <object type="Bridge" os_index="17" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+              <object type="PCIDev" os_index="4096" name="HGST, Inc. Ultrastar SN200 Series NVMe SSD" pci_busid="0000:01:00.0" pci_type="0108 [1c58:0023] [1c58:0023] 02" pci_link_speed="3.938462">
+                <info name="PCIVendor" value="HGST, Inc."/>
+                <info name="PCIDevice" value="Ultrastar SN200 Series NVMe SSD"/>
+                <info name="PCISlot" value="1"/>
+                <object type="OSDev" name="nvme0n1" osdev_type="0">
+                  <info name="LinuxDeviceID" value="259:0"/>
+                  <info name="Model" value="HUSMR7616BDP301"/>
+                  <info name="SerialNumber" value="SDM0000107A5"/>
+                  <info name="Type" value="Disk"/>
+                </object>
+              </object>
+            </object>
+            <object type="Bridge" os_index="19" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[03-03]" pci_busid="0000:00:01.3" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="1.000000">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+              <object type="PCIDev" os_index="12288" name="Intel Corporation I350 Gigabit Network Connection" pci_busid="0000:03:00.0" pci_type="0200 [8086:1521] [1458:0000] 01" pci_link_speed="1.000000">
+                <info name="PCIVendor" value="Intel Corporation"/>
+                <info name="PCIDevice" value="I350 Gigabit Network Connection"/>
+                <object type="OSDev" name="enp3s0f0" osdev_type="2">
+                  <info name="Address" value="e0:d5:5e:ca:b0:22"/>
+                </object>
+              </object>
+              <object type="PCIDev" os_index="12289" name="Intel Corporation I350 Gigabit Network Connection" pci_busid="0000:03:00.1" pci_type="0200 [8086:1521] [1458:0000] 01" pci_link_speed="1.000000">
+                <info name="PCIVendor" value="Intel Corporation"/>
+                <info name="PCIDevice" value="I350 Gigabit Network Connection"/>
+                <object type="OSDev" name="enp3s0f1" osdev_type="2">
+                  <info name="Address" value="e0:d5:5e:ca:b0:23"/>
+                </object>
+              </object>
+            </object>
+            <object type="Bridge" os_index="22" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[04-05]" pci_busid="0000:00:01.6" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="0.250000">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+              <object type="Bridge" os_index="16384" name="ASPEED Technology, Inc. AST1150 PCI-to-PCI Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[05-05]" pci_busid="0000:04:00.0" pci_type="0604 [1a03:1150] [0000:0000] 04" pci_link_speed="0.250000">
+                <info name="PCIVendor" value="ASPEED Technology, Inc."/>
+                <info name="PCIDevice" value="AST1150 PCI-to-PCI Bridge"/>
+                <object type="PCIDev" os_index="20480" name="ASPEED Technology, Inc. ASPEED Graphics Family" pci_busid="0000:05:00.0" pci_type="0300 [1a03:2000] [1458:1000] 41" pci_link_speed="0.000000">
+                  <info name="PCIVendor" value="ASPEED Technology, Inc."/>
+                  <info name="PCIDevice" value="ASPEED Graphics Family"/>
+                  <object type="OSDev" name="card0" osdev_type="1"/>
+                  <object type="OSDev" name="controlD64" osdev_type="1"/>
+                </object>
+              </object>
+            </object>
+            <object type="Bridge" os_index="129" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) Internal PCIe GPP Bridge 0 to Bus B" bridge_type="1-1" depth="1" bridge_pci="0000:[07-07]" pci_busid="0000:00:08.1" pci_type="0604 [1022:1454] [0000:0000] 00" pci_link_speed="15.753846">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="Family 17h (Models 00h-0fh) Internal PCIe GPP Bridge 0 to Bus B"/>
+              <object type="PCIDev" os_index="28674" name="Advanced Micro Devices, Inc. [AMD] FCH SATA Controller [AHCI mode]" pci_busid="0000:07:00.2" pci_type="0106 [1022:7901] [1458:1000] 51" pci_link_speed="15.753846">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+                <info name="PCIDevice" value="FCH SATA Controller [AHCI mode]"/>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="1" cpuset="0x0fc00000,0x00000fc0" complete_cpuset="0x0fc00000,0x00000fc0" online_cpuset="0x0fc00000,0x00000fc0" allowed_cpuset="0x0fc00000,0x00000fc0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" local_memory="33819570176">
+        <page_type size="4096" count="8256731"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Group" cpuset="0x0fc00000,0x00000fc0" complete_cpuset="0x0fc00000,0x00000fc0" online_cpuset="0x0fc00000,0x00000fc0" allowed_cpuset="0x0fc00000,0x00000fc0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" depth="0">
+          <object type="Cache" cpuset="0x01c00000,0x000001c0" complete_cpuset="0x01c00000,0x000001c0" online_cpuset="0x01c00000,0x000001c0" allowed_cpuset="0x01c00000,0x000001c0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="8" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" online_cpuset="0x00400000,0x00000040" allowed_cpuset="0x00400000,0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+              <object type="PU" os_index="6" cpuset="0x00000040" complete_cpuset="0x00000040" online_cpuset="0x00000040" allowed_cpuset="0x00000040" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+              <object type="PU" os_index="54" cpuset="0x00400000,0x0" complete_cpuset="0x00400000,0x0" online_cpuset="0x00400000,0x0" allowed_cpuset="0x00400000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+            </object>
+            <object type="Core" os_index="9" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" online_cpuset="0x00800000,0x00000080" allowed_cpuset="0x00800000,0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+              <object type="PU" os_index="7" cpuset="0x00000080" complete_cpuset="0x00000080" online_cpuset="0x00000080" allowed_cpuset="0x00000080" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+              <object type="PU" os_index="55" cpuset="0x00800000,0x0" complete_cpuset="0x00800000,0x0" online_cpuset="0x00800000,0x0" allowed_cpuset="0x00800000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+            </object>
+            <object type="Core" os_index="10" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" online_cpuset="0x01000000,0x00000100" allowed_cpuset="0x01000000,0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+              <object type="PU" os_index="8" cpuset="0x00000100" complete_cpuset="0x00000100" online_cpuset="0x00000100" allowed_cpuset="0x00000100" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+              <object type="PU" os_index="56" cpuset="0x01000000,0x0" complete_cpuset="0x01000000,0x0" online_cpuset="0x01000000,0x0" allowed_cpuset="0x01000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x0e000000,0x00000e00" complete_cpuset="0x0e000000,0x00000e00" online_cpuset="0x0e000000,0x00000e00" allowed_cpuset="0x0e000000,0x00000e00" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="12" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" online_cpuset="0x02000000,0x00000200" allowed_cpuset="0x02000000,0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+              <object type="PU" os_index="9" cpuset="0x00000200" complete_cpuset="0x00000200" online_cpuset="0x00000200" allowed_cpuset="0x00000200" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+              <object type="PU" os_index="57" cpuset="0x02000000,0x0" complete_cpuset="0x02000000,0x0" online_cpuset="0x02000000,0x0" allowed_cpuset="0x02000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+            </object>
+            <object type="Core" os_index="13" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" online_cpuset="0x04000000,0x00000400" allowed_cpuset="0x04000000,0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+              <object type="PU" os_index="10" cpuset="0x00000400" complete_cpuset="0x00000400" online_cpuset="0x00000400" allowed_cpuset="0x00000400" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+              <object type="PU" os_index="58" cpuset="0x04000000,0x0" complete_cpuset="0x04000000,0x0" online_cpuset="0x04000000,0x0" allowed_cpuset="0x04000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+            </object>
+            <object type="Core" os_index="14" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" online_cpuset="0x08000000,0x00000800" allowed_cpuset="0x08000000,0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+              <object type="PU" os_index="11" cpuset="0x00000800" complete_cpuset="0x00000800" online_cpuset="0x00000800" allowed_cpuset="0x00000800" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+              <object type="PU" os_index="59" cpuset="0x08000000,0x0" complete_cpuset="0x08000000,0x0" online_cpuset="0x08000000,0x0" allowed_cpuset="0x08000000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+            </object>
+          </object>
+          <object type="Bridge" os_index="1" bridge_type="0-1" depth="0" bridge_pci="0000:[10-15]">
+            <object type="Bridge" os_index="65553" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[11-13]" pci_busid="0000:10:01.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+              <object type="Bridge" os_index="69632" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="2" bridge_pci="0000:[12-13]" pci_busid="0000:11:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="Bridge" os_index="73728" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="3" bridge_pci="0000:[13-13]" pci_busid="0000:12:00.0" pci_type="0604 [1002:14a1] [0000:0000] 00" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <object type="PCIDev" os_index="77824" name="Advanced Micro Devices, Inc. [AMD/ATI] Vega 20" pci_busid="0000:13:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                    <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                    <info name="PCIDevice" value="Vega 20"/>
+                    <object type="OSDev" name="card1" osdev_type="1"/>
+                    <object type="OSDev" name="renderD128" osdev_type="1"/>
+                    <object type="OSDev" name="controlD65" osdev_type="1"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="2" cpuset="0x00000003,0xf0000000,0x0003f000" complete_cpuset="0x00000003,0xf0000000,0x0003f000" online_cpuset="0x00000003,0xf0000000,0x0003f000" allowed_cpuset="0x00000003,0xf0000000,0x0003f000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" local_memory="33819574272">
+        <page_type size="4096" count="8256732"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Group" cpuset="0x00000003,0xf0000000,0x0003f000" complete_cpuset="0x00000003,0xf0000000,0x0003f000" online_cpuset="0x00000003,0xf0000000,0x0003f000" allowed_cpuset="0x00000003,0xf0000000,0x0003f000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" depth="0">
+          <object type="Cache" cpuset="0x70000000,0x00007000" complete_cpuset="0x70000000,0x00007000" online_cpuset="0x70000000,0x00007000" allowed_cpuset="0x70000000,0x00007000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="16" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" online_cpuset="0x10000000,0x00001000" allowed_cpuset="0x10000000,0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+              <object type="PU" os_index="12" cpuset="0x00001000" complete_cpuset="0x00001000" online_cpuset="0x00001000" allowed_cpuset="0x00001000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+              <object type="PU" os_index="60" cpuset="0x10000000,0x0" complete_cpuset="0x10000000,0x0" online_cpuset="0x10000000,0x0" allowed_cpuset="0x10000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+            </object>
+            <object type="Core" os_index="17" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" online_cpuset="0x20000000,0x00002000" allowed_cpuset="0x20000000,0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+              <object type="PU" os_index="13" cpuset="0x00002000" complete_cpuset="0x00002000" online_cpuset="0x00002000" allowed_cpuset="0x00002000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+              <object type="PU" os_index="61" cpuset="0x20000000,0x0" complete_cpuset="0x20000000,0x0" online_cpuset="0x20000000,0x0" allowed_cpuset="0x20000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+            </object>
+            <object type="Core" os_index="18" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" online_cpuset="0x40000000,0x00004000" allowed_cpuset="0x40000000,0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+              <object type="PU" os_index="14" cpuset="0x00004000" complete_cpuset="0x00004000" online_cpuset="0x00004000" allowed_cpuset="0x00004000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+              <object type="PU" os_index="62" cpuset="0x40000000,0x0" complete_cpuset="0x40000000,0x0" online_cpuset="0x40000000,0x0" allowed_cpuset="0x40000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00000003,0x80000000,0x00038000" complete_cpuset="0x00000003,0x80000000,0x00038000" online_cpuset="0x00000003,0x80000000,0x00038000" allowed_cpuset="0x00000003,0x80000000,0x00038000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="20" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" online_cpuset="0x80000000,0x00008000" allowed_cpuset="0x80000000,0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+              <object type="PU" os_index="15" cpuset="0x00008000" complete_cpuset="0x00008000" online_cpuset="0x00008000" allowed_cpuset="0x00008000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+              <object type="PU" os_index="63" cpuset="0x80000000,0x0" complete_cpuset="0x80000000,0x0" online_cpuset="0x80000000,0x0" allowed_cpuset="0x80000000,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+            </object>
+            <object type="Core" os_index="21" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" online_cpuset="0x00000001,,0x00010000" allowed_cpuset="0x00000001,,0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+              <object type="PU" os_index="16" cpuset="0x00010000" complete_cpuset="0x00010000" online_cpuset="0x00010000" allowed_cpuset="0x00010000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+              <object type="PU" os_index="64" cpuset="0x00000001,,0x0" complete_cpuset="0x00000001,,0x0" online_cpuset="0x00000001,,0x0" allowed_cpuset="0x00000001,,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+            </object>
+            <object type="Core" os_index="22" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" online_cpuset="0x00000002,,0x00020000" allowed_cpuset="0x00000002,,0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004">
+              <object type="PU" os_index="17" cpuset="0x00020000" complete_cpuset="0x00020000" online_cpuset="0x00020000" allowed_cpuset="0x00020000" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+              <object type="PU" os_index="65" cpuset="0x00000002,,0x0" complete_cpuset="0x00000002,,0x0" online_cpuset="0x00000002,,0x0" allowed_cpuset="0x00000002,,0x0" nodeset="0x00000004" complete_nodeset="0x00000004" allowed_nodeset="0x00000004"/>
+            </object>
+          </object>
+          <object type="Bridge" os_index="2" bridge_type="0-1" depth="0" bridge_pci="0000:[20-25]">
+            <object type="Bridge" os_index="131121" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[21-23]" pci_busid="0000:20:03.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+              <object type="Bridge" os_index="135168" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="2" bridge_pci="0000:[22-23]" pci_busid="0000:21:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="Bridge" os_index="139264" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="3" bridge_pci="0000:[23-23]" pci_busid="0000:22:00.0" pci_type="0604 [1002:14a1] [0000:0000] 00" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <object type="PCIDev" os_index="143360" name="Advanced Micro Devices, Inc. [AMD/ATI] Vega 20" pci_busid="0000:23:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                    <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                    <info name="PCIDevice" value="Vega 20"/>
+                    <object type="OSDev" name="card2" osdev_type="1"/>
+                    <object type="OSDev" name="renderD129" osdev_type="1"/>
+                    <object type="OSDev" name="controlD66" osdev_type="1"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="3" cpuset="0x000000fc,,0x00fc0000" complete_cpuset="0x000000fc,,0x00fc0000" online_cpuset="0x000000fc,,0x00fc0000" allowed_cpuset="0x000000fc,,0x00fc0000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" local_memory="33802997760">
+        <page_type size="4096" count="8252685"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Group" cpuset="0x000000fc,,0x00fc0000" complete_cpuset="0x000000fc,,0x00fc0000" online_cpuset="0x000000fc,,0x00fc0000" allowed_cpuset="0x000000fc,,0x00fc0000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" depth="0">
+          <object type="Cache" cpuset="0x0000001c,,0x001c0000" complete_cpuset="0x0000001c,,0x001c0000" online_cpuset="0x0000001c,,0x001c0000" allowed_cpuset="0x0000001c,,0x001c0000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="24" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" online_cpuset="0x00000004,,0x00040000" allowed_cpuset="0x00000004,,0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+              <object type="PU" os_index="18" cpuset="0x00040000" complete_cpuset="0x00040000" online_cpuset="0x00040000" allowed_cpuset="0x00040000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+              <object type="PU" os_index="66" cpuset="0x00000004,,0x0" complete_cpuset="0x00000004,,0x0" online_cpuset="0x00000004,,0x0" allowed_cpuset="0x00000004,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+            </object>
+            <object type="Core" os_index="25" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" online_cpuset="0x00000008,,0x00080000" allowed_cpuset="0x00000008,,0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+              <object type="PU" os_index="19" cpuset="0x00080000" complete_cpuset="0x00080000" online_cpuset="0x00080000" allowed_cpuset="0x00080000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+              <object type="PU" os_index="67" cpuset="0x00000008,,0x0" complete_cpuset="0x00000008,,0x0" online_cpuset="0x00000008,,0x0" allowed_cpuset="0x00000008,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+            </object>
+            <object type="Core" os_index="26" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" online_cpuset="0x00000010,,0x00100000" allowed_cpuset="0x00000010,,0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+              <object type="PU" os_index="20" cpuset="0x00100000" complete_cpuset="0x00100000" online_cpuset="0x00100000" allowed_cpuset="0x00100000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+              <object type="PU" os_index="68" cpuset="0x00000010,,0x0" complete_cpuset="0x00000010,,0x0" online_cpuset="0x00000010,,0x0" allowed_cpuset="0x00000010,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x000000e0,,0x00e00000" complete_cpuset="0x000000e0,,0x00e00000" online_cpuset="0x000000e0,,0x00e00000" allowed_cpuset="0x000000e0,,0x00e00000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="28" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" online_cpuset="0x00000020,,0x00200000" allowed_cpuset="0x00000020,,0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+              <object type="PU" os_index="21" cpuset="0x00200000" complete_cpuset="0x00200000" online_cpuset="0x00200000" allowed_cpuset="0x00200000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+              <object type="PU" os_index="69" cpuset="0x00000020,,0x0" complete_cpuset="0x00000020,,0x0" online_cpuset="0x00000020,,0x0" allowed_cpuset="0x00000020,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+            </object>
+            <object type="Core" os_index="29" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" online_cpuset="0x00000040,,0x00400000" allowed_cpuset="0x00000040,,0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+              <object type="PU" os_index="22" cpuset="0x00400000" complete_cpuset="0x00400000" online_cpuset="0x00400000" allowed_cpuset="0x00400000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+              <object type="PU" os_index="70" cpuset="0x00000040,,0x0" complete_cpuset="0x00000040,,0x0" online_cpuset="0x00000040,,0x0" allowed_cpuset="0x00000040,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+            </object>
+            <object type="Core" os_index="30" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" online_cpuset="0x00000080,,0x00800000" allowed_cpuset="0x00000080,,0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008">
+              <object type="PU" os_index="23" cpuset="0x00800000" complete_cpuset="0x00800000" online_cpuset="0x00800000" allowed_cpuset="0x00800000" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+              <object type="PU" os_index="71" cpuset="0x00000080,,0x0" complete_cpuset="0x00000080,,0x0" online_cpuset="0x00000080,,0x0" allowed_cpuset="0x00000080,,0x0" nodeset="0x00000008" complete_nodeset="0x00000008" allowed_nodeset="0x00000008"/>
+            </object>
+          </object>
+          <object type="Bridge" os_index="3" bridge_type="0-1" depth="0" bridge_pci="0000:[30-36]">
+            <object type="Bridge" os_index="196657" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[31-34]" pci_busid="0000:30:03.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="15.753846">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+              <object type="Bridge" os_index="200704" name="PLX Technology, Inc. PEX 8747 48-Lane, 5-Port PCI Express Gen 3 (8.0 GT/s) Switch" bridge_type="1-1" depth="2" bridge_pci="0000:[32-34]" pci_busid="0000:31:00.0" pci_type="0604 [10b5:8747] [0000:0000] ca" pci_link_speed="15.753846">
+                <info name="PCIVendor" value="PLX Technology, Inc."/>
+                <info name="PCIDevice" value="PEX 8747 48-Lane, 5-Port PCI Express Gen 3 (8.0 GT/s) Switch"/>
+                <object type="Bridge" os_index="204928" name="PLX Technology, Inc. PEX 8747 48-Lane, 5-Port PCI Express Gen 3 (8.0 GT/s) Switch" bridge_type="1-1" depth="3" bridge_pci="0000:[33-33]" pci_busid="0000:32:08.0" pci_type="0604 [10b5:8747] [0000:0000] ca" pci_link_speed="15.753846">
+                  <info name="PCIVendor" value="PLX Technology, Inc."/>
+                  <info name="PCIDevice" value="PEX 8747 48-Lane, 5-Port PCI Express Gen 3 (8.0 GT/s) Switch"/>
+                  <object type="PCIDev" os_index="208896" name="Mellanox Technologies MT28908 Family [ConnectX-6]" pci_busid="0000:33:00.0" pci_type="0207 [15b3:101b] [15b3:0006] 00" pci_link_speed="15.753846">
+                    <info name="PCIVendor" value="Mellanox Technologies"/>
+                    <info name="PCIDevice" value="MT28908 Family [ConnectX-6]"/>
+                    <object type="OSDev" name="hsi0" osdev_type="2">
+                      <info name="Address" value="80:00:01:07:fe:80:00:00:00:00:00:00:98:03:9b:03:00:82:9d:b4"/>
+                      <info name="Port" value="1"/>
+                    </object>
+                    <object type="OSDev" name="mlx5_0" osdev_type="3">
+                      <info name="NodeGUID" value="9803:9b03:0082:9db4"/>
+                      <info name="SysImageGUID" value="9803:9b03:0082:9db4"/>
+                      <info name="Port1State" value="4"/>
+                      <info name="Port1LID" value="0x1c"/>
+                      <info name="Port1LMC" value="0"/>
+                      <info name="Port1GID0" value="fe80:0000:0000:0000:9803:9b03:0082:9db4"/>
+                    </object>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+    </object>
+    <object type="Socket" os_index="1" cpuset="0xffffff00,0x0000ffff,0xff000000" complete_cpuset="0xffffff00,0x0000ffff,0xff000000" online_cpuset="0xffffff00,0x0000ffff,0xff000000" allowed_cpuset="0xffffff00,0x0000ffff,0xff000000" nodeset="0x000000f0" complete_nodeset="0x000000f0" allowed_nodeset="0x000000f0">
+      <info name="CPUVendor" value="AuthenticAMD"/>
+      <info name="CPUFamilyNumber" value="23"/>
+      <info name="CPUModelNumber" value="1"/>
+      <info name="CPUModel" value="AMD EPYC 7401 24-Core Processor"/>
+      <info name="CPUStepping" value="2"/>
+      <object type="NUMANode" os_index="4" cpuset="0x00003f00,,0x3f000000" complete_cpuset="0x00003f00,,0x3f000000" online_cpuset="0x00003f00,,0x3f000000" allowed_cpuset="0x00003f00,,0x3f000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" local_memory="33819574272">
+        <page_type size="4096" count="8256732"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Group" cpuset="0x00003f00,,0x3f000000" complete_cpuset="0x00003f00,,0x3f000000" online_cpuset="0x00003f00,,0x3f000000" allowed_cpuset="0x00003f00,,0x3f000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" depth="0">
+          <object type="Cache" cpuset="0x00000700,,0x07000000" complete_cpuset="0x00000700,,0x07000000" online_cpuset="0x00000700,,0x07000000" allowed_cpuset="0x00000700,,0x07000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="0" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" online_cpuset="0x00000100,,0x01000000" allowed_cpuset="0x00000100,,0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+              <object type="PU" os_index="24" cpuset="0x01000000" complete_cpuset="0x01000000" online_cpuset="0x01000000" allowed_cpuset="0x01000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+              <object type="PU" os_index="72" cpuset="0x00000100,,0x0" complete_cpuset="0x00000100,,0x0" online_cpuset="0x00000100,,0x0" allowed_cpuset="0x00000100,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+            </object>
+            <object type="Core" os_index="1" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" online_cpuset="0x00000200,,0x02000000" allowed_cpuset="0x00000200,,0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+              <object type="PU" os_index="25" cpuset="0x02000000" complete_cpuset="0x02000000" online_cpuset="0x02000000" allowed_cpuset="0x02000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+              <object type="PU" os_index="73" cpuset="0x00000200,,0x0" complete_cpuset="0x00000200,,0x0" online_cpuset="0x00000200,,0x0" allowed_cpuset="0x00000200,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+            </object>
+            <object type="Core" os_index="2" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" online_cpuset="0x00000400,,0x04000000" allowed_cpuset="0x00000400,,0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+              <object type="PU" os_index="26" cpuset="0x04000000" complete_cpuset="0x04000000" online_cpuset="0x04000000" allowed_cpuset="0x04000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+              <object type="PU" os_index="74" cpuset="0x00000400,,0x0" complete_cpuset="0x00000400,,0x0" online_cpuset="0x00000400,,0x0" allowed_cpuset="0x00000400,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x00003800,,0x38000000" complete_cpuset="0x00003800,,0x38000000" online_cpuset="0x00003800,,0x38000000" allowed_cpuset="0x00003800,,0x38000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="4" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" online_cpuset="0x00000800,,0x08000000" allowed_cpuset="0x00000800,,0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+              <object type="PU" os_index="27" cpuset="0x08000000" complete_cpuset="0x08000000" online_cpuset="0x08000000" allowed_cpuset="0x08000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+              <object type="PU" os_index="75" cpuset="0x00000800,,0x0" complete_cpuset="0x00000800,,0x0" online_cpuset="0x00000800,,0x0" allowed_cpuset="0x00000800,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+            </object>
+            <object type="Core" os_index="5" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" online_cpuset="0x00001000,,0x10000000" allowed_cpuset="0x00001000,,0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+              <object type="PU" os_index="28" cpuset="0x10000000" complete_cpuset="0x10000000" online_cpuset="0x10000000" allowed_cpuset="0x10000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+              <object type="PU" os_index="76" cpuset="0x00001000,,0x0" complete_cpuset="0x00001000,,0x0" online_cpuset="0x00001000,,0x0" allowed_cpuset="0x00001000,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+            </object>
+            <object type="Core" os_index="6" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" online_cpuset="0x00002000,,0x20000000" allowed_cpuset="0x00002000,,0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010">
+              <object type="PU" os_index="29" cpuset="0x20000000" complete_cpuset="0x20000000" online_cpuset="0x20000000" allowed_cpuset="0x20000000" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+              <object type="PU" os_index="77" cpuset="0x00002000,,0x0" complete_cpuset="0x00002000,,0x0" online_cpuset="0x00002000,,0x0" allowed_cpuset="0x00002000,,0x0" nodeset="0x00000010" complete_nodeset="0x00000010" allowed_nodeset="0x00000010"/>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="5" cpuset="0x000fc000,0x0000000f,0xc0000000" complete_cpuset="0x000fc000,0x0000000f,0xc0000000" online_cpuset="0x000fc000,0x0000000f,0xc0000000" allowed_cpuset="0x000fc000,0x0000000f,0xc0000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" local_memory="33819570176">
+        <page_type size="4096" count="8256731"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Group" cpuset="0x000fc000,0x0000000f,0xc0000000" complete_cpuset="0x000fc000,0x0000000f,0xc0000000" online_cpuset="0x000fc000,0x0000000f,0xc0000000" allowed_cpuset="0x000fc000,0x0000000f,0xc0000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" depth="0">
+          <object type="Cache" cpuset="0x0001c000,0x00000001,0xc0000000" complete_cpuset="0x0001c000,0x00000001,0xc0000000" online_cpuset="0x0001c000,0x00000001,0xc0000000" allowed_cpuset="0x0001c000,0x00000001,0xc0000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="8" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" online_cpuset="0x00004000,,0x40000000" allowed_cpuset="0x00004000,,0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+              <object type="PU" os_index="30" cpuset="0x40000000" complete_cpuset="0x40000000" online_cpuset="0x40000000" allowed_cpuset="0x40000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+              <object type="PU" os_index="78" cpuset="0x00004000,,0x0" complete_cpuset="0x00004000,,0x0" online_cpuset="0x00004000,,0x0" allowed_cpuset="0x00004000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+            </object>
+            <object type="Core" os_index="9" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" online_cpuset="0x00008000,,0x80000000" allowed_cpuset="0x00008000,,0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+              <object type="PU" os_index="31" cpuset="0x80000000" complete_cpuset="0x80000000" online_cpuset="0x80000000" allowed_cpuset="0x80000000" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+              <object type="PU" os_index="79" cpuset="0x00008000,,0x0" complete_cpuset="0x00008000,,0x0" online_cpuset="0x00008000,,0x0" allowed_cpuset="0x00008000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+            </object>
+            <object type="Core" os_index="10" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" online_cpuset="0x00010000,0x00000001,0x0" allowed_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+              <object type="PU" os_index="32" cpuset="0x00000001,0x0" complete_cpuset="0x00000001,0x0" online_cpuset="0x00000001,0x0" allowed_cpuset="0x00000001,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+              <object type="PU" os_index="80" cpuset="0x00010000,,0x0" complete_cpuset="0x00010000,,0x0" online_cpuset="0x00010000,,0x0" allowed_cpuset="0x00010000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x000e0000,0x0000000e,0x0" complete_cpuset="0x000e0000,0x0000000e,0x0" online_cpuset="0x000e0000,0x0000000e,0x0" allowed_cpuset="0x000e0000,0x0000000e,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="12" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" online_cpuset="0x00020000,0x00000002,0x0" allowed_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+              <object type="PU" os_index="33" cpuset="0x00000002,0x0" complete_cpuset="0x00000002,0x0" online_cpuset="0x00000002,0x0" allowed_cpuset="0x00000002,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+              <object type="PU" os_index="81" cpuset="0x00020000,,0x0" complete_cpuset="0x00020000,,0x0" online_cpuset="0x00020000,,0x0" allowed_cpuset="0x00020000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+            </object>
+            <object type="Core" os_index="13" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" online_cpuset="0x00040000,0x00000004,0x0" allowed_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+              <object type="PU" os_index="34" cpuset="0x00000004,0x0" complete_cpuset="0x00000004,0x0" online_cpuset="0x00000004,0x0" allowed_cpuset="0x00000004,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+              <object type="PU" os_index="82" cpuset="0x00040000,,0x0" complete_cpuset="0x00040000,,0x0" online_cpuset="0x00040000,,0x0" allowed_cpuset="0x00040000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+            </object>
+            <object type="Core" os_index="14" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" online_cpuset="0x00080000,0x00000008,0x0" allowed_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020">
+              <object type="PU" os_index="35" cpuset="0x00000008,0x0" complete_cpuset="0x00000008,0x0" online_cpuset="0x00000008,0x0" allowed_cpuset="0x00000008,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+              <object type="PU" os_index="83" cpuset="0x00080000,,0x0" complete_cpuset="0x00080000,,0x0" online_cpuset="0x00080000,,0x0" allowed_cpuset="0x00080000,,0x0" nodeset="0x00000020" complete_nodeset="0x00000020" allowed_nodeset="0x00000020"/>
+            </object>
+          </object>
+          <object type="Bridge" os_index="5" bridge_type="0-1" depth="0" bridge_pci="0000:[50-55]">
+            <object type="Bridge" os_index="327697" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[51-53]" pci_busid="0000:50:01.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+              <object type="Bridge" os_index="331776" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="2" bridge_pci="0000:[52-53]" pci_busid="0000:51:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="Bridge" os_index="335872" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="3" bridge_pci="0000:[53-53]" pci_busid="0000:52:00.0" pci_type="0604 [1002:14a1] [0000:0000] 00" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <object type="PCIDev" os_index="339968" name="Advanced Micro Devices, Inc. [AMD/ATI] Vega 20" pci_busid="0000:53:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                    <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                    <info name="PCIDevice" value="Vega 20"/>
+                    <object type="OSDev" name="card3" osdev_type="1"/>
+                    <object type="OSDev" name="renderD130" osdev_type="1"/>
+                    <object type="OSDev" name="controlD67" osdev_type="1"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="6" cpuset="0x03f00000,0x000003f0,0x0" complete_cpuset="0x03f00000,0x000003f0,0x0" online_cpuset="0x03f00000,0x000003f0,0x0" allowed_cpuset="0x03f00000,0x000003f0,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" local_memory="33819574272">
+        <page_type size="4096" count="8256732"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Group" cpuset="0x03f00000,0x000003f0,0x0" complete_cpuset="0x03f00000,0x000003f0,0x0" online_cpuset="0x03f00000,0x000003f0,0x0" allowed_cpuset="0x03f00000,0x000003f0,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" depth="0">
+          <object type="Cache" cpuset="0x00700000,0x00000070,0x0" complete_cpuset="0x00700000,0x00000070,0x0" online_cpuset="0x00700000,0x00000070,0x0" allowed_cpuset="0x00700000,0x00000070,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="16" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" online_cpuset="0x00100000,0x00000010,0x0" allowed_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+              <object type="PU" os_index="36" cpuset="0x00000010,0x0" complete_cpuset="0x00000010,0x0" online_cpuset="0x00000010,0x0" allowed_cpuset="0x00000010,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+              <object type="PU" os_index="84" cpuset="0x00100000,,0x0" complete_cpuset="0x00100000,,0x0" online_cpuset="0x00100000,,0x0" allowed_cpuset="0x00100000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+            </object>
+            <object type="Core" os_index="17" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" online_cpuset="0x00200000,0x00000020,0x0" allowed_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+              <object type="PU" os_index="37" cpuset="0x00000020,0x0" complete_cpuset="0x00000020,0x0" online_cpuset="0x00000020,0x0" allowed_cpuset="0x00000020,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+              <object type="PU" os_index="85" cpuset="0x00200000,,0x0" complete_cpuset="0x00200000,,0x0" online_cpuset="0x00200000,,0x0" allowed_cpuset="0x00200000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+            </object>
+            <object type="Core" os_index="18" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" online_cpuset="0x00400000,0x00000040,0x0" allowed_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+              <object type="PU" os_index="38" cpuset="0x00000040,0x0" complete_cpuset="0x00000040,0x0" online_cpuset="0x00000040,0x0" allowed_cpuset="0x00000040,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+              <object type="PU" os_index="86" cpuset="0x00400000,,0x0" complete_cpuset="0x00400000,,0x0" online_cpuset="0x00400000,,0x0" allowed_cpuset="0x00400000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0x03800000,0x00000380,0x0" complete_cpuset="0x03800000,0x00000380,0x0" online_cpuset="0x03800000,0x00000380,0x0" allowed_cpuset="0x03800000,0x00000380,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="20" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" online_cpuset="0x00800000,0x00000080,0x0" allowed_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+              <object type="PU" os_index="39" cpuset="0x00000080,0x0" complete_cpuset="0x00000080,0x0" online_cpuset="0x00000080,0x0" allowed_cpuset="0x00000080,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+              <object type="PU" os_index="87" cpuset="0x00800000,,0x0" complete_cpuset="0x00800000,,0x0" online_cpuset="0x00800000,,0x0" allowed_cpuset="0x00800000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+            </object>
+            <object type="Core" os_index="21" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" online_cpuset="0x01000000,0x00000100,0x0" allowed_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+              <object type="PU" os_index="40" cpuset="0x00000100,0x0" complete_cpuset="0x00000100,0x0" online_cpuset="0x00000100,0x0" allowed_cpuset="0x00000100,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+              <object type="PU" os_index="88" cpuset="0x01000000,,0x0" complete_cpuset="0x01000000,,0x0" online_cpuset="0x01000000,,0x0" allowed_cpuset="0x01000000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+            </object>
+            <object type="Core" os_index="22" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" online_cpuset="0x02000000,0x00000200,0x0" allowed_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040">
+              <object type="PU" os_index="41" cpuset="0x00000200,0x0" complete_cpuset="0x00000200,0x0" online_cpuset="0x00000200,0x0" allowed_cpuset="0x00000200,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+              <object type="PU" os_index="89" cpuset="0x02000000,,0x0" complete_cpuset="0x02000000,,0x0" online_cpuset="0x02000000,,0x0" allowed_cpuset="0x02000000,,0x0" nodeset="0x00000040" complete_nodeset="0x00000040" allowed_nodeset="0x00000040"/>
+            </object>
+          </object>
+        </object>
+      </object>
+      <object type="NUMANode" os_index="7" cpuset="0xfc000000,0x0000fc00,0x0" complete_cpuset="0xfc000000,0x0000fc00,0x0" online_cpuset="0xfc000000,0x0000fc00,0x0" allowed_cpuset="0xfc000000,0x0000fc00,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" local_memory="33819033600">
+        <page_type size="4096" count="8256600"/>
+        <page_type size="2097152" count="0"/>
+        <page_type size="1073741824" count="0"/>
+        <object type="Group" cpuset="0xfc000000,0x0000fc00,0x0" complete_cpuset="0xfc000000,0x0000fc00,0x0" online_cpuset="0xfc000000,0x0000fc00,0x0" allowed_cpuset="0xfc000000,0x0000fc00,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" depth="0">
+          <object type="Cache" cpuset="0x1c000000,0x00001c00,0x0" complete_cpuset="0x1c000000,0x00001c00,0x0" online_cpuset="0x1c000000,0x00001c00,0x0" allowed_cpuset="0x1c000000,0x00001c00,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="24" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" online_cpuset="0x04000000,0x00000400,0x0" allowed_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+              <object type="PU" os_index="42" cpuset="0x00000400,0x0" complete_cpuset="0x00000400,0x0" online_cpuset="0x00000400,0x0" allowed_cpuset="0x00000400,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+              <object type="PU" os_index="90" cpuset="0x04000000,,0x0" complete_cpuset="0x04000000,,0x0" online_cpuset="0x04000000,,0x0" allowed_cpuset="0x04000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+            </object>
+            <object type="Core" os_index="25" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" online_cpuset="0x08000000,0x00000800,0x0" allowed_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+              <object type="PU" os_index="43" cpuset="0x00000800,0x0" complete_cpuset="0x00000800,0x0" online_cpuset="0x00000800,0x0" allowed_cpuset="0x00000800,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+              <object type="PU" os_index="91" cpuset="0x08000000,,0x0" complete_cpuset="0x08000000,,0x0" online_cpuset="0x08000000,,0x0" allowed_cpuset="0x08000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+            </object>
+            <object type="Core" os_index="26" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" online_cpuset="0x10000000,0x00001000,0x0" allowed_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+              <object type="PU" os_index="44" cpuset="0x00001000,0x0" complete_cpuset="0x00001000,0x0" online_cpuset="0x00001000,0x0" allowed_cpuset="0x00001000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+              <object type="PU" os_index="92" cpuset="0x10000000,,0x0" complete_cpuset="0x10000000,,0x0" online_cpuset="0x10000000,,0x0" allowed_cpuset="0x10000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+            </object>
+          </object>
+          <object type="Cache" cpuset="0xe0000000,0x0000e000,0x0" complete_cpuset="0xe0000000,0x0000e000,0x0" online_cpuset="0xe0000000,0x0000e000,0x0" allowed_cpuset="0xe0000000,0x0000e000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080" cache_size="8388608" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+            <info name="Inclusive" value="0"/>
+            <object type="Core" os_index="28" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" online_cpuset="0x20000000,0x00002000,0x0" allowed_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+              <object type="PU" os_index="45" cpuset="0x00002000,0x0" complete_cpuset="0x00002000,0x0" online_cpuset="0x00002000,0x0" allowed_cpuset="0x00002000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+              <object type="PU" os_index="93" cpuset="0x20000000,,0x0" complete_cpuset="0x20000000,,0x0" online_cpuset="0x20000000,,0x0" allowed_cpuset="0x20000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+            </object>
+            <object type="Core" os_index="29" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" online_cpuset="0x40000000,0x00004000,0x0" allowed_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+              <object type="PU" os_index="46" cpuset="0x00004000,0x0" complete_cpuset="0x00004000,0x0" online_cpuset="0x00004000,0x0" allowed_cpuset="0x00004000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+              <object type="PU" os_index="94" cpuset="0x40000000,,0x0" complete_cpuset="0x40000000,,0x0" online_cpuset="0x40000000,,0x0" allowed_cpuset="0x40000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+            </object>
+            <object type="Core" os_index="30" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" online_cpuset="0x80000000,0x00008000,0x0" allowed_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080">
+              <object type="PU" os_index="47" cpuset="0x00008000,0x0" complete_cpuset="0x00008000,0x0" online_cpuset="0x00008000,0x0" allowed_cpuset="0x00008000,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+              <object type="PU" os_index="95" cpuset="0x80000000,,0x0" complete_cpuset="0x80000000,,0x0" online_cpuset="0x80000000,,0x0" allowed_cpuset="0x80000000,,0x0" nodeset="0x00000080" complete_nodeset="0x00000080" allowed_nodeset="0x00000080"/>
+            </object>
+          </object>
+          <object type="Bridge" os_index="7" bridge_type="0-1" depth="0" bridge_pci="0000:[70-75]">
+            <object type="Bridge" os_index="458801" name="Advanced Micro Devices, Inc. [AMD] Family 17h (Models 00h-0fh) PCIe GPP Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[71-73]" pci_busid="0000:70:03.1" pci_type="0604 [1022:1453] [0000:0000] 00" pci_link_speed="3.938462">
+              <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD]"/>
+              <info name="PCIDevice" value="Family 17h (Models 00h-0fh) PCIe GPP Bridge"/>
+              <object type="Bridge" os_index="462848" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="2" bridge_pci="0000:[72-73]" pci_busid="0000:71:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="3.938462">
+                <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                <object type="Bridge" os_index="466944" name="Advanced Micro Devices, Inc. [AMD/ATI]" bridge_type="1-1" depth="3" bridge_pci="0000:[73-73]" pci_busid="0000:72:00.0" pci_type="0604 [1002:14a1] [0000:0000] 00" pci_link_speed="31.507692">
+                  <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                  <object type="PCIDev" os_index="471040" name="Advanced Micro Devices, Inc. [AMD/ATI] Vega 20" pci_busid="0000:73:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.507692">
+                    <info name="PCIVendor" value="Advanced Micro Devices, Inc. [AMD/ATI]"/>
+                    <info name="PCIDevice" value="Vega 20"/>
+                    <object type="OSDev" name="card4" osdev_type="1"/>
+                    <object type="OSDev" name="renderD131" osdev_type="1"/>
+                    <object type="OSDev" name="controlD68" osdev_type="1"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+    </object>
+  </object>
+</topology>
diff --git a/t/hwloc-data/corona/0.xml b/t/hwloc-data/corona/0.xml
new file mode 100644
index 000000000000..63bf3b80db71
--- /dev/null
+++ b/t/hwloc-data/corona/0.xml
@@ -0,0 +1,1006 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE topology SYSTEM "hwloc.dtd">
+<topology>
+  <object type="Machine" os_index="0" cpuset="0xffffffff,0xffffffff,0xffffffff" complete_cpuset="0xffffffff,0xffffffff,0xffffffff" online_cpuset="0xffffffff,0xffffffff,0xffffffff" allowed_cpuset="0xffffffff,0xffffffff,0xffffffff" nodeset="0x00000003" complete_nodeset="0x00000003" allowed_nodeset="0x00000003">
+    <info name="DMIProductName" value="AS -4124GS-TNR-CGP-MT037"/>
+    <info name="DMIProductVersion" value="0123456789"/>
+    <info name="DMIBoardVendor" value="Supermicro"/>
+    <info name="DMIBoardName" value="H12DSG-O-CPU"/>
+    <info name="DMIBoardVersion" value="1.01A"/>
+    <info name="DMIBoardAssetTag" value="To be filled by O.E.M."/>
+    <info name="DMIChassisVendor" value="Supermicro"/>
+    <info name="DMIChassisType" value="1"/>
+    <info name="DMIChassisVersion" value="0123456789"/>
+    <info name="DMIChassisAssetTag" value="To be filled by O.E.M."/>
+    <info name="DMIBIOSVendor" value="American Megatrends Inc."/>
+    <info name="DMIBIOSVersion" value="2.3"/>
+    <info name="DMIBIOSDate" value="10/21/2021"/>
+    <info name="DMISysVendor" value="Supermicro"/>
+    <info name="Backend" value="Linux"/>
+    <info name="LinuxCgroup" value="/slurm/uid_6885/job_26909/step_interactive"/>
+    <info name="OSName" value="Linux"/>
+    <info name="OSRelease" value="4.18.0-348.12.2.3toss.t4.x86_64"/>
+    <info name="OSVersion" value="#1 SMP Mon Mar 7 13:48:48 PST 2022"/>
+    <info name="HostName" value="corona263"/>
+    <info name="Architecture" value="x86_64"/>
+    <info name="hwlocVersion" value="2.7.0"/>
+    <info name="ProcessName" value="lstopo"/>
+    <distances nbobjs="2" relative_depth="2" latency_base="1.000000">
+      <latency value="10.000000"/>
+      <latency value="32.000000"/>
+      <latency value="32.000000"/>
+      <latency value="10.000000"/>
+    </distances>
+    <object type="NUMANode" os_index="0" cpuset="0x000000ff,0xffff0000,0x00ffffff" complete_cpuset="0x000000ff,0xffff0000,0x00ffffff" online_cpuset="0x000000ff,0xffff0000,0x00ffffff" allowed_cpuset="0x000000ff,0xffff0000,0x00ffffff" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" local_memory="134734184448">
+      <page_type size="4096" count="32894088"/>
+      <page_type size="2097152" count="0"/>
+      <page_type size="1073741824" count="0"/>
+      <object type="Socket" os_index="0" cpuset="0x000000ff,0xffff0000,0x00ffffff" complete_cpuset="0x000000ff,0xffff0000,0x00ffffff" online_cpuset="0x000000ff,0xffff0000,0x00ffffff" allowed_cpuset="0x000000ff,0xffff0000,0x00ffffff" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+        <info name="CPUVendor" value="AuthenticAMD"/>
+        <info name="CPUFamilyNumber" value="23"/>
+        <info name="CPUModelNumber" value="49"/>
+        <info name="CPUModel" value="AMD EPYC 7402 24-Core Processor                "/>
+        <info name="CPUStepping" value="0"/>
+        <object type="Cache" os_index="0" cpuset="0x00070000,0x00000007" complete_cpuset="0x00070000,0x00000007" online_cpuset="0x00070000,0x00000007" allowed_cpuset="0x00070000,0x00000007" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="0" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" online_cpuset="0x00010000,0x00000001" allowed_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="0" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" online_cpuset="0x00010000,0x00000001" allowed_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="0" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" online_cpuset="0x00010000,0x00000001" allowed_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="0" cpuset="0x00010000,0x00000001" complete_cpuset="0x00010000,0x00000001" online_cpuset="0x00010000,0x00000001" allowed_cpuset="0x00010000,0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="0" cpuset="0x00000001" complete_cpuset="0x00000001" online_cpuset="0x00000001" allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="48" cpuset="0x00010000,0x0" complete_cpuset="0x00010000,0x0" online_cpuset="0x00010000,0x0" allowed_cpuset="0x00010000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="1" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" online_cpuset="0x00020000,0x00000002" allowed_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="1" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" online_cpuset="0x00020000,0x00000002" allowed_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="1" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" online_cpuset="0x00020000,0x00000002" allowed_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="1" cpuset="0x00020000,0x00000002" complete_cpuset="0x00020000,0x00000002" online_cpuset="0x00020000,0x00000002" allowed_cpuset="0x00020000,0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="1" cpuset="0x00000002" complete_cpuset="0x00000002" online_cpuset="0x00000002" allowed_cpuset="0x00000002" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="49" cpuset="0x00020000,0x0" complete_cpuset="0x00020000,0x0" online_cpuset="0x00020000,0x0" allowed_cpuset="0x00020000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="2" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" online_cpuset="0x00040000,0x00000004" allowed_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="2" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" online_cpuset="0x00040000,0x00000004" allowed_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="2" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" online_cpuset="0x00040000,0x00000004" allowed_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="2" cpuset="0x00040000,0x00000004" complete_cpuset="0x00040000,0x00000004" online_cpuset="0x00040000,0x00000004" allowed_cpuset="0x00040000,0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="2" cpuset="0x00000004" complete_cpuset="0x00000004" online_cpuset="0x00000004" allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="50" cpuset="0x00040000,0x0" complete_cpuset="0x00040000,0x0" online_cpuset="0x00040000,0x0" allowed_cpuset="0x00040000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="1" cpuset="0x00380000,0x00000038" complete_cpuset="0x00380000,0x00000038" online_cpuset="0x00380000,0x00000038" allowed_cpuset="0x00380000,0x00000038" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="4" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" online_cpuset="0x00080000,0x00000008" allowed_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="4" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" online_cpuset="0x00080000,0x00000008" allowed_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="4" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" online_cpuset="0x00080000,0x00000008" allowed_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="4" cpuset="0x00080000,0x00000008" complete_cpuset="0x00080000,0x00000008" online_cpuset="0x00080000,0x00000008" allowed_cpuset="0x00080000,0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="3" cpuset="0x00000008" complete_cpuset="0x00000008" online_cpuset="0x00000008" allowed_cpuset="0x00000008" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="51" cpuset="0x00080000,0x0" complete_cpuset="0x00080000,0x0" online_cpuset="0x00080000,0x0" allowed_cpuset="0x00080000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="5" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" online_cpuset="0x00100000,0x00000010" allowed_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="5" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" online_cpuset="0x00100000,0x00000010" allowed_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="5" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" online_cpuset="0x00100000,0x00000010" allowed_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="5" cpuset="0x00100000,0x00000010" complete_cpuset="0x00100000,0x00000010" online_cpuset="0x00100000,0x00000010" allowed_cpuset="0x00100000,0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="4" cpuset="0x00000010" complete_cpuset="0x00000010" online_cpuset="0x00000010" allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="52" cpuset="0x00100000,0x0" complete_cpuset="0x00100000,0x0" online_cpuset="0x00100000,0x0" allowed_cpuset="0x00100000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="6" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" online_cpuset="0x00200000,0x00000020" allowed_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="6" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" online_cpuset="0x00200000,0x00000020" allowed_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="6" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" online_cpuset="0x00200000,0x00000020" allowed_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="6" cpuset="0x00200000,0x00000020" complete_cpuset="0x00200000,0x00000020" online_cpuset="0x00200000,0x00000020" allowed_cpuset="0x00200000,0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="5" cpuset="0x00000020" complete_cpuset="0x00000020" online_cpuset="0x00000020" allowed_cpuset="0x00000020" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="53" cpuset="0x00200000,0x0" complete_cpuset="0x00200000,0x0" online_cpuset="0x00200000,0x0" allowed_cpuset="0x00200000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="2" cpuset="0x01c00000,0x000001c0" complete_cpuset="0x01c00000,0x000001c0" online_cpuset="0x01c00000,0x000001c0" allowed_cpuset="0x01c00000,0x000001c0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="8" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" online_cpuset="0x00400000,0x00000040" allowed_cpuset="0x00400000,0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="8" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" online_cpuset="0x00400000,0x00000040" allowed_cpuset="0x00400000,0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="8" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" online_cpuset="0x00400000,0x00000040" allowed_cpuset="0x00400000,0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="8" cpuset="0x00400000,0x00000040" complete_cpuset="0x00400000,0x00000040" online_cpuset="0x00400000,0x00000040" allowed_cpuset="0x00400000,0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="6" cpuset="0x00000040" complete_cpuset="0x00000040" online_cpuset="0x00000040" allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="54" cpuset="0x00400000,0x0" complete_cpuset="0x00400000,0x0" online_cpuset="0x00400000,0x0" allowed_cpuset="0x00400000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="9" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" online_cpuset="0x00800000,0x00000080" allowed_cpuset="0x00800000,0x00000080" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="9" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" online_cpuset="0x00800000,0x00000080" allowed_cpuset="0x00800000,0x00000080" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="9" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" online_cpuset="0x00800000,0x00000080" allowed_cpuset="0x00800000,0x00000080" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="9" cpuset="0x00800000,0x00000080" complete_cpuset="0x00800000,0x00000080" online_cpuset="0x00800000,0x00000080" allowed_cpuset="0x00800000,0x00000080" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="7" cpuset="0x00000080" complete_cpuset="0x00000080" online_cpuset="0x00000080" allowed_cpuset="0x00000080" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="55" cpuset="0x00800000,0x0" complete_cpuset="0x00800000,0x0" online_cpuset="0x00800000,0x0" allowed_cpuset="0x00800000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="10" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" online_cpuset="0x01000000,0x00000100" allowed_cpuset="0x01000000,0x00000100" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="10" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" online_cpuset="0x01000000,0x00000100" allowed_cpuset="0x01000000,0x00000100" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="10" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" online_cpuset="0x01000000,0x00000100" allowed_cpuset="0x01000000,0x00000100" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="10" cpuset="0x01000000,0x00000100" complete_cpuset="0x01000000,0x00000100" online_cpuset="0x01000000,0x00000100" allowed_cpuset="0x01000000,0x00000100" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="8" cpuset="0x00000100" complete_cpuset="0x00000100" online_cpuset="0x00000100" allowed_cpuset="0x00000100" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="56" cpuset="0x01000000,0x0" complete_cpuset="0x01000000,0x0" online_cpuset="0x01000000,0x0" allowed_cpuset="0x01000000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="3" cpuset="0x0e000000,0x00000e00" complete_cpuset="0x0e000000,0x00000e00" online_cpuset="0x0e000000,0x00000e00" allowed_cpuset="0x0e000000,0x00000e00" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="12" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" online_cpuset="0x02000000,0x00000200" allowed_cpuset="0x02000000,0x00000200" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="12" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" online_cpuset="0x02000000,0x00000200" allowed_cpuset="0x02000000,0x00000200" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="12" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" online_cpuset="0x02000000,0x00000200" allowed_cpuset="0x02000000,0x00000200" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="12" cpuset="0x02000000,0x00000200" complete_cpuset="0x02000000,0x00000200" online_cpuset="0x02000000,0x00000200" allowed_cpuset="0x02000000,0x00000200" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="9" cpuset="0x00000200" complete_cpuset="0x00000200" online_cpuset="0x00000200" allowed_cpuset="0x00000200" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="57" cpuset="0x02000000,0x0" complete_cpuset="0x02000000,0x0" online_cpuset="0x02000000,0x0" allowed_cpuset="0x02000000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="13" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" online_cpuset="0x04000000,0x00000400" allowed_cpuset="0x04000000,0x00000400" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="13" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" online_cpuset="0x04000000,0x00000400" allowed_cpuset="0x04000000,0x00000400" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="13" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" online_cpuset="0x04000000,0x00000400" allowed_cpuset="0x04000000,0x00000400" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="13" cpuset="0x04000000,0x00000400" complete_cpuset="0x04000000,0x00000400" online_cpuset="0x04000000,0x00000400" allowed_cpuset="0x04000000,0x00000400" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="10" cpuset="0x00000400" complete_cpuset="0x00000400" online_cpuset="0x00000400" allowed_cpuset="0x00000400" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="58" cpuset="0x04000000,0x0" complete_cpuset="0x04000000,0x0" online_cpuset="0x04000000,0x0" allowed_cpuset="0x04000000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="14" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" online_cpuset="0x08000000,0x00000800" allowed_cpuset="0x08000000,0x00000800" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="14" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" online_cpuset="0x08000000,0x00000800" allowed_cpuset="0x08000000,0x00000800" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="14" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" online_cpuset="0x08000000,0x00000800" allowed_cpuset="0x08000000,0x00000800" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="14" cpuset="0x08000000,0x00000800" complete_cpuset="0x08000000,0x00000800" online_cpuset="0x08000000,0x00000800" allowed_cpuset="0x08000000,0x00000800" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="11" cpuset="0x00000800" complete_cpuset="0x00000800" online_cpuset="0x00000800" allowed_cpuset="0x00000800" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="59" cpuset="0x08000000,0x0" complete_cpuset="0x08000000,0x0" online_cpuset="0x08000000,0x0" allowed_cpuset="0x08000000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="4" cpuset="0x70000000,0x00007000" complete_cpuset="0x70000000,0x00007000" online_cpuset="0x70000000,0x00007000" allowed_cpuset="0x70000000,0x00007000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="16" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" online_cpuset="0x10000000,0x00001000" allowed_cpuset="0x10000000,0x00001000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="16" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" online_cpuset="0x10000000,0x00001000" allowed_cpuset="0x10000000,0x00001000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="16" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" online_cpuset="0x10000000,0x00001000" allowed_cpuset="0x10000000,0x00001000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="16" cpuset="0x10000000,0x00001000" complete_cpuset="0x10000000,0x00001000" online_cpuset="0x10000000,0x00001000" allowed_cpuset="0x10000000,0x00001000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="12" cpuset="0x00001000" complete_cpuset="0x00001000" online_cpuset="0x00001000" allowed_cpuset="0x00001000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="60" cpuset="0x10000000,0x0" complete_cpuset="0x10000000,0x0" online_cpuset="0x10000000,0x0" allowed_cpuset="0x10000000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="17" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" online_cpuset="0x20000000,0x00002000" allowed_cpuset="0x20000000,0x00002000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="17" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" online_cpuset="0x20000000,0x00002000" allowed_cpuset="0x20000000,0x00002000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="17" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" online_cpuset="0x20000000,0x00002000" allowed_cpuset="0x20000000,0x00002000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="17" cpuset="0x20000000,0x00002000" complete_cpuset="0x20000000,0x00002000" online_cpuset="0x20000000,0x00002000" allowed_cpuset="0x20000000,0x00002000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="13" cpuset="0x00002000" complete_cpuset="0x00002000" online_cpuset="0x00002000" allowed_cpuset="0x00002000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="61" cpuset="0x20000000,0x0" complete_cpuset="0x20000000,0x0" online_cpuset="0x20000000,0x0" allowed_cpuset="0x20000000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="18" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" online_cpuset="0x40000000,0x00004000" allowed_cpuset="0x40000000,0x00004000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="18" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" online_cpuset="0x40000000,0x00004000" allowed_cpuset="0x40000000,0x00004000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="18" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" online_cpuset="0x40000000,0x00004000" allowed_cpuset="0x40000000,0x00004000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="18" cpuset="0x40000000,0x00004000" complete_cpuset="0x40000000,0x00004000" online_cpuset="0x40000000,0x00004000" allowed_cpuset="0x40000000,0x00004000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="14" cpuset="0x00004000" complete_cpuset="0x00004000" online_cpuset="0x00004000" allowed_cpuset="0x00004000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="62" cpuset="0x40000000,0x0" complete_cpuset="0x40000000,0x0" online_cpuset="0x40000000,0x0" allowed_cpuset="0x40000000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="5" cpuset="0x00000003,0x80000000,0x00038000" complete_cpuset="0x00000003,0x80000000,0x00038000" online_cpuset="0x00000003,0x80000000,0x00038000" allowed_cpuset="0x00000003,0x80000000,0x00038000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="20" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" online_cpuset="0x80000000,0x00008000" allowed_cpuset="0x80000000,0x00008000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="20" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" online_cpuset="0x80000000,0x00008000" allowed_cpuset="0x80000000,0x00008000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="20" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" online_cpuset="0x80000000,0x00008000" allowed_cpuset="0x80000000,0x00008000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="20" cpuset="0x80000000,0x00008000" complete_cpuset="0x80000000,0x00008000" online_cpuset="0x80000000,0x00008000" allowed_cpuset="0x80000000,0x00008000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="15" cpuset="0x00008000" complete_cpuset="0x00008000" online_cpuset="0x00008000" allowed_cpuset="0x00008000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="63" cpuset="0x80000000,0x0" complete_cpuset="0x80000000,0x0" online_cpuset="0x80000000,0x0" allowed_cpuset="0x80000000,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="21" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" online_cpuset="0x00000001,,0x00010000" allowed_cpuset="0x00000001,,0x00010000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="21" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" online_cpuset="0x00000001,,0x00010000" allowed_cpuset="0x00000001,,0x00010000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="21" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" online_cpuset="0x00000001,,0x00010000" allowed_cpuset="0x00000001,,0x00010000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="21" cpuset="0x00000001,,0x00010000" complete_cpuset="0x00000001,,0x00010000" online_cpuset="0x00000001,,0x00010000" allowed_cpuset="0x00000001,,0x00010000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="16" cpuset="0x00010000" complete_cpuset="0x00010000" online_cpuset="0x00010000" allowed_cpuset="0x00010000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="64" cpuset="0x00000001,,0x0" complete_cpuset="0x00000001,,0x0" online_cpuset="0x00000001,,0x0" allowed_cpuset="0x00000001,,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="22" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" online_cpuset="0x00000002,,0x00020000" allowed_cpuset="0x00000002,,0x00020000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="22" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" online_cpuset="0x00000002,,0x00020000" allowed_cpuset="0x00000002,,0x00020000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="22" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" online_cpuset="0x00000002,,0x00020000" allowed_cpuset="0x00000002,,0x00020000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="22" cpuset="0x00000002,,0x00020000" complete_cpuset="0x00000002,,0x00020000" online_cpuset="0x00000002,,0x00020000" allowed_cpuset="0x00000002,,0x00020000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="17" cpuset="0x00020000" complete_cpuset="0x00020000" online_cpuset="0x00020000" allowed_cpuset="0x00020000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="65" cpuset="0x00000002,,0x0" complete_cpuset="0x00000002,,0x0" online_cpuset="0x00000002,,0x0" allowed_cpuset="0x00000002,,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="6" cpuset="0x0000001c,,0x001c0000" complete_cpuset="0x0000001c,,0x001c0000" online_cpuset="0x0000001c,,0x001c0000" allowed_cpuset="0x0000001c,,0x001c0000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="24" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" online_cpuset="0x00000004,,0x00040000" allowed_cpuset="0x00000004,,0x00040000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="24" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" online_cpuset="0x00000004,,0x00040000" allowed_cpuset="0x00000004,,0x00040000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="24" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" online_cpuset="0x00000004,,0x00040000" allowed_cpuset="0x00000004,,0x00040000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="24" cpuset="0x00000004,,0x00040000" complete_cpuset="0x00000004,,0x00040000" online_cpuset="0x00000004,,0x00040000" allowed_cpuset="0x00000004,,0x00040000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="18" cpuset="0x00040000" complete_cpuset="0x00040000" online_cpuset="0x00040000" allowed_cpuset="0x00040000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="66" cpuset="0x00000004,,0x0" complete_cpuset="0x00000004,,0x0" online_cpuset="0x00000004,,0x0" allowed_cpuset="0x00000004,,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="25" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" online_cpuset="0x00000008,,0x00080000" allowed_cpuset="0x00000008,,0x00080000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="25" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" online_cpuset="0x00000008,,0x00080000" allowed_cpuset="0x00000008,,0x00080000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="25" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" online_cpuset="0x00000008,,0x00080000" allowed_cpuset="0x00000008,,0x00080000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="25" cpuset="0x00000008,,0x00080000" complete_cpuset="0x00000008,,0x00080000" online_cpuset="0x00000008,,0x00080000" allowed_cpuset="0x00000008,,0x00080000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="19" cpuset="0x00080000" complete_cpuset="0x00080000" online_cpuset="0x00080000" allowed_cpuset="0x00080000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="67" cpuset="0x00000008,,0x0" complete_cpuset="0x00000008,,0x0" online_cpuset="0x00000008,,0x0" allowed_cpuset="0x00000008,,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="26" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" online_cpuset="0x00000010,,0x00100000" allowed_cpuset="0x00000010,,0x00100000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="26" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" online_cpuset="0x00000010,,0x00100000" allowed_cpuset="0x00000010,,0x00100000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="26" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" online_cpuset="0x00000010,,0x00100000" allowed_cpuset="0x00000010,,0x00100000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="26" cpuset="0x00000010,,0x00100000" complete_cpuset="0x00000010,,0x00100000" online_cpuset="0x00000010,,0x00100000" allowed_cpuset="0x00000010,,0x00100000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="20" cpuset="0x00100000" complete_cpuset="0x00100000" online_cpuset="0x00100000" allowed_cpuset="0x00100000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="68" cpuset="0x00000010,,0x0" complete_cpuset="0x00000010,,0x0" online_cpuset="0x00000010,,0x0" allowed_cpuset="0x00000010,,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="7" cpuset="0x000000e0,,0x00e00000" complete_cpuset="0x000000e0,,0x00e00000" online_cpuset="0x000000e0,,0x00e00000" allowed_cpuset="0x000000e0,,0x00e00000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="28" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" online_cpuset="0x00000020,,0x00200000" allowed_cpuset="0x00000020,,0x00200000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="28" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" online_cpuset="0x00000020,,0x00200000" allowed_cpuset="0x00000020,,0x00200000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="28" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" online_cpuset="0x00000020,,0x00200000" allowed_cpuset="0x00000020,,0x00200000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="28" cpuset="0x00000020,,0x00200000" complete_cpuset="0x00000020,,0x00200000" online_cpuset="0x00000020,,0x00200000" allowed_cpuset="0x00000020,,0x00200000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="21" cpuset="0x00200000" complete_cpuset="0x00200000" online_cpuset="0x00200000" allowed_cpuset="0x00200000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="69" cpuset="0x00000020,,0x0" complete_cpuset="0x00000020,,0x0" online_cpuset="0x00000020,,0x0" allowed_cpuset="0x00000020,,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="29" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" online_cpuset="0x00000040,,0x00400000" allowed_cpuset="0x00000040,,0x00400000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="29" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" online_cpuset="0x00000040,,0x00400000" allowed_cpuset="0x00000040,,0x00400000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="29" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" online_cpuset="0x00000040,,0x00400000" allowed_cpuset="0x00000040,,0x00400000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="29" cpuset="0x00000040,,0x00400000" complete_cpuset="0x00000040,,0x00400000" online_cpuset="0x00000040,,0x00400000" allowed_cpuset="0x00000040,,0x00400000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="22" cpuset="0x00400000" complete_cpuset="0x00400000" online_cpuset="0x00400000" allowed_cpuset="0x00400000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="70" cpuset="0x00000040,,0x0" complete_cpuset="0x00000040,,0x0" online_cpuset="0x00000040,,0x0" allowed_cpuset="0x00000040,,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="30" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" online_cpuset="0x00000080,,0x00800000" allowed_cpuset="0x00000080,,0x00800000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="30" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" online_cpuset="0x00000080,,0x00800000" allowed_cpuset="0x00000080,,0x00800000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="30" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" online_cpuset="0x00000080,,0x00800000" allowed_cpuset="0x00000080,,0x00800000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="30" cpuset="0x00000080,,0x00800000" complete_cpuset="0x00000080,,0x00800000" online_cpuset="0x00000080,,0x00800000" allowed_cpuset="0x00000080,,0x00800000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001">
+                  <object type="PU" os_index="23" cpuset="0x00800000" complete_cpuset="0x00800000" online_cpuset="0x00800000" allowed_cpuset="0x00800000" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                  <object type="PU" os_index="71" cpuset="0x00000080,,0x0" complete_cpuset="0x00000080,,0x0" online_cpuset="0x00000080,,0x0" allowed_cpuset="0x00000080,,0x0" nodeset="0x00000001" complete_nodeset="0x00000001" allowed_nodeset="0x00000001"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" bridge_type="0-1" depth="0" bridge_pci="0000:[00-05]">
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[01-03]" pci_busid="0000:00:01.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="31.507692">
+            <object type="Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[02-03]" pci_busid="0000:01:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="31.507692">
+              <object type="Bridge" bridge_type="1-1" depth="3" bridge_pci="0000:[03-03]" pci_busid="0000:02:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <object type="PCIDev" pci_busid="0000:03:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.000000">
+                  <object type="OSDev" name="rsmi2" osdev_type="1">
+                    <info name="Backend" value="RSMI"/>
+                    <info name="GPUVendor" value="AMD"/>
+                    <info name="GPUModel" value="deon Instinct MI50 32GB"/>
+                    <info name="AMDSerial" value="20180003580c"/>
+                    <info name="AMDUUID" value="9aa0304172df8896"/>
+                    <info name="XGMIHiveID" value="0"/>
+                    <info name="RSMIVRAMSize" value="33538048"/>
+                    <info name="RSMIVisibleVRAMSize" value="33538048"/>
+                    <info name="RSMIGTTSize" value="263645540"/>
+                    <info name="Type" value="RSMI"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" bridge_type="0-1" depth="0" bridge_pci="0000:[20-2b]">
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[21-21]" pci_busid="0000:20:01.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="3.938462">
+            <object type="PCIDev" pci_busid="0000:21:00.0" pci_type="0108 [144d:a808] [144d:a801] 00" pci_link_speed="3.938462">
+              <info name="PCISlot" value="0"/>
+              <object type="OSDev" name="nvme0n1" osdev_type="0">
+                <info name="Size" value="3750738264"/>
+                <info name="SectorSize" value="512"/>
+                <info name="LinuxDeviceID" value="259:0"/>
+                <info name="Vendor" value="Samsung"/>
+                <info name="Model" value="SAMSUNG MZQLB3T8HALS-00007"/>
+                <info name="SerialNumber" value="S438NA0N407539"/>
+                <info name="Type" value="Disk"/>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[25-27]" pci_busid="0000:20:03.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="31.507692">
+            <object type="Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[26-27]" pci_busid="0000:25:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="31.507692">
+              <object type="Bridge" bridge_type="1-1" depth="3" bridge_pci="0000:[27-27]" pci_busid="0000:26:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <object type="PCIDev" pci_busid="0000:27:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.000000">
+                  <object type="OSDev" name="rsmi3" osdev_type="1">
+                    <info name="Backend" value="RSMI"/>
+                    <info name="GPUVendor" value="AMD"/>
+                    <info name="GPUModel" value="deon Instinct MI50 32GB"/>
+                    <info name="AMDSerial" value="20180009790c"/>
+                    <info name="AMDUUID" value="eb6c214172df888f"/>
+                    <info name="XGMIHiveID" value="0"/>
+                    <info name="RSMIVRAMSize" value="33538048"/>
+                    <info name="RSMIVisibleVRAMSize" value="33538048"/>
+                    <info name="RSMIGTTSize" value="263645540"/>
+                    <info name="Type" value="RSMI"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[2a-2a]" pci_busid="0000:20:08.2" pci_type="0604 [1022:1484] [1022:1484] 00" pci_link_speed="31.507692">
+            <object type="PCIDev" pci_busid="0000:2a:00.0" pci_type="0106 [1022:7901] [15d9:7901] 51" pci_link_speed="31.507692"/>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[2b-2b]" pci_busid="0000:20:08.3" pci_type="0604 [1022:1484] [1022:1484] 00" pci_link_speed="31.507692">
+            <object type="PCIDev" pci_busid="0000:2b:00.0" pci_type="0106 [1022:7901] [15d9:7901] 51" pci_link_speed="31.507692"/>
+          </object>
+        </object>
+        <object type="Bridge" bridge_type="0-1" depth="0" bridge_pci="0000:[40-47]">
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[41-43]" pci_busid="0000:40:01.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="31.507692">
+            <object type="Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[42-43]" pci_busid="0000:41:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="31.507692">
+              <object type="Bridge" bridge_type="1-1" depth="3" bridge_pci="0000:[43-43]" pci_busid="0000:42:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <object type="PCIDev" pci_busid="0000:43:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.000000">
+                  <object type="OSDev" name="rsmi1" osdev_type="1">
+                    <info name="Backend" value="RSMI"/>
+                    <info name="GPUVendor" value="AMD"/>
+                    <info name="GPUModel" value="deon Instinct MI50 32GB"/>
+                    <info name="AMDSerial" value="20170000570c"/>
+                    <info name="AMDUUID" value="cb88208172edb14d"/>
+                    <info name="XGMIHiveID" value="0"/>
+                    <info name="RSMIVRAMSize" value="33538048"/>
+                    <info name="RSMIVisibleVRAMSize" value="33538048"/>
+                    <info name="RSMIGTTSize" value="263645540"/>
+                    <info name="Type" value="RSMI"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[46-46]" pci_busid="0000:40:08.2" pci_type="0604 [1022:1484] [1022:1484] 00" pci_link_speed="31.507692">
+            <object type="PCIDev" pci_busid="0000:46:00.0" pci_type="0106 [1022:7901] [15d9:7901] 51" pci_link_speed="31.507692"/>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[47-47]" pci_busid="0000:40:08.3" pci_type="0604 [1022:1484] [1022:1484] 00" pci_link_speed="31.507692">
+            <object type="PCIDev" pci_busid="0000:47:00.0" pci_type="0106 [1022:7901] [15d9:7901] 51" pci_link_speed="31.507692"/>
+          </object>
+        </object>
+        <object type="Bridge" bridge_type="0-1" depth="0" bridge_pci="0000:[60-67]">
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[61-63]" pci_busid="0000:60:03.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="31.507692">
+            <object type="Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[62-63]" pci_busid="0000:61:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="31.507692">
+              <object type="Bridge" bridge_type="1-1" depth="3" bridge_pci="0000:[63-63]" pci_busid="0000:62:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <object type="PCIDev" pci_busid="0000:63:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.000000">
+                  <object type="OSDev" name="rsmi0" osdev_type="1">
+                    <info name="Backend" value="RSMI"/>
+                    <info name="GPUVendor" value="AMD"/>
+                    <info name="GPUModel" value="deon Instinct MI50 32GB"/>
+                    <info name="AMDSerial" value="20080000770c"/>
+                    <info name="AMDUUID" value="3b72598172e626c4"/>
+                    <info name="XGMIHiveID" value="0"/>
+                    <info name="RSMIVRAMSize" value="33538048"/>
+                    <info name="RSMIVisibleVRAMSize" value="33538048"/>
+                    <info name="RSMIGTTSize" value="263645540"/>
+                    <info name="Type" value="RSMI"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[64-65]" pci_busid="0000:60:05.2" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="0.250000">
+            <object type="Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[65-65]" pci_busid="0000:64:00.0" pci_type="0604 [1a03:1150] [1a03:1150] 04" pci_link_speed="0.250000">
+              <object type="PCIDev" pci_busid="0000:65:00.0" pci_type="0300 [1a03:2000] [15d9:1b55] 41" pci_link_speed="0.000000"/>
+            </object>
+          </object>
+        </object>
+      </object>
+    </object>
+    <object type="NUMANode" os_index="1" cpuset="0xffffff00,0x0000ffff,0xff000000" complete_cpuset="0xffffff00,0x0000ffff,0xff000000" online_cpuset="0xffffff00,0x0000ffff,0xff000000" allowed_cpuset="0xffffff00,0x0000ffff,0xff000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" local_memory="135238848512">
+      <page_type size="4096" count="33017297"/>
+      <page_type size="2097152" count="0"/>
+      <page_type size="1073741824" count="0"/>
+      <object type="Socket" os_index="1" cpuset="0xffffff00,0x0000ffff,0xff000000" complete_cpuset="0xffffff00,0x0000ffff,0xff000000" online_cpuset="0xffffff00,0x0000ffff,0xff000000" allowed_cpuset="0xffffff00,0x0000ffff,0xff000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+        <info name="CPUVendor" value="AuthenticAMD"/>
+        <info name="CPUFamilyNumber" value="23"/>
+        <info name="CPUModelNumber" value="49"/>
+        <info name="CPUModel" value="AMD EPYC 7402 24-Core Processor                "/>
+        <info name="CPUStepping" value="0"/>
+        <object type="Cache" os_index="16" cpuset="0x00000700,,0x07000000" complete_cpuset="0x00000700,,0x07000000" online_cpuset="0x00000700,,0x07000000" allowed_cpuset="0x00000700,,0x07000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="64" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" online_cpuset="0x00000100,,0x01000000" allowed_cpuset="0x00000100,,0x01000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="64" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" online_cpuset="0x00000100,,0x01000000" allowed_cpuset="0x00000100,,0x01000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="64" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" online_cpuset="0x00000100,,0x01000000" allowed_cpuset="0x00000100,,0x01000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="0" cpuset="0x00000100,,0x01000000" complete_cpuset="0x00000100,,0x01000000" online_cpuset="0x00000100,,0x01000000" allowed_cpuset="0x00000100,,0x01000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="24" cpuset="0x01000000" complete_cpuset="0x01000000" online_cpuset="0x01000000" allowed_cpuset="0x01000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="72" cpuset="0x00000100,,0x0" complete_cpuset="0x00000100,,0x0" online_cpuset="0x00000100,,0x0" allowed_cpuset="0x00000100,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="65" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" online_cpuset="0x00000200,,0x02000000" allowed_cpuset="0x00000200,,0x02000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="65" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" online_cpuset="0x00000200,,0x02000000" allowed_cpuset="0x00000200,,0x02000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="65" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" online_cpuset="0x00000200,,0x02000000" allowed_cpuset="0x00000200,,0x02000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="1" cpuset="0x00000200,,0x02000000" complete_cpuset="0x00000200,,0x02000000" online_cpuset="0x00000200,,0x02000000" allowed_cpuset="0x00000200,,0x02000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="25" cpuset="0x02000000" complete_cpuset="0x02000000" online_cpuset="0x02000000" allowed_cpuset="0x02000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="73" cpuset="0x00000200,,0x0" complete_cpuset="0x00000200,,0x0" online_cpuset="0x00000200,,0x0" allowed_cpuset="0x00000200,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="66" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" online_cpuset="0x00000400,,0x04000000" allowed_cpuset="0x00000400,,0x04000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="66" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" online_cpuset="0x00000400,,0x04000000" allowed_cpuset="0x00000400,,0x04000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="66" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" online_cpuset="0x00000400,,0x04000000" allowed_cpuset="0x00000400,,0x04000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="2" cpuset="0x00000400,,0x04000000" complete_cpuset="0x00000400,,0x04000000" online_cpuset="0x00000400,,0x04000000" allowed_cpuset="0x00000400,,0x04000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="26" cpuset="0x04000000" complete_cpuset="0x04000000" online_cpuset="0x04000000" allowed_cpuset="0x04000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="74" cpuset="0x00000400,,0x0" complete_cpuset="0x00000400,,0x0" online_cpuset="0x00000400,,0x0" allowed_cpuset="0x00000400,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="17" cpuset="0x00003800,,0x38000000" complete_cpuset="0x00003800,,0x38000000" online_cpuset="0x00003800,,0x38000000" allowed_cpuset="0x00003800,,0x38000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="68" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" online_cpuset="0x00000800,,0x08000000" allowed_cpuset="0x00000800,,0x08000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="68" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" online_cpuset="0x00000800,,0x08000000" allowed_cpuset="0x00000800,,0x08000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="68" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" online_cpuset="0x00000800,,0x08000000" allowed_cpuset="0x00000800,,0x08000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="4" cpuset="0x00000800,,0x08000000" complete_cpuset="0x00000800,,0x08000000" online_cpuset="0x00000800,,0x08000000" allowed_cpuset="0x00000800,,0x08000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="27" cpuset="0x08000000" complete_cpuset="0x08000000" online_cpuset="0x08000000" allowed_cpuset="0x08000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="75" cpuset="0x00000800,,0x0" complete_cpuset="0x00000800,,0x0" online_cpuset="0x00000800,,0x0" allowed_cpuset="0x00000800,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="69" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" online_cpuset="0x00001000,,0x10000000" allowed_cpuset="0x00001000,,0x10000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="69" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" online_cpuset="0x00001000,,0x10000000" allowed_cpuset="0x00001000,,0x10000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="69" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" online_cpuset="0x00001000,,0x10000000" allowed_cpuset="0x00001000,,0x10000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="5" cpuset="0x00001000,,0x10000000" complete_cpuset="0x00001000,,0x10000000" online_cpuset="0x00001000,,0x10000000" allowed_cpuset="0x00001000,,0x10000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="28" cpuset="0x10000000" complete_cpuset="0x10000000" online_cpuset="0x10000000" allowed_cpuset="0x10000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="76" cpuset="0x00001000,,0x0" complete_cpuset="0x00001000,,0x0" online_cpuset="0x00001000,,0x0" allowed_cpuset="0x00001000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="70" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" online_cpuset="0x00002000,,0x20000000" allowed_cpuset="0x00002000,,0x20000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="70" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" online_cpuset="0x00002000,,0x20000000" allowed_cpuset="0x00002000,,0x20000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="70" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" online_cpuset="0x00002000,,0x20000000" allowed_cpuset="0x00002000,,0x20000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="6" cpuset="0x00002000,,0x20000000" complete_cpuset="0x00002000,,0x20000000" online_cpuset="0x00002000,,0x20000000" allowed_cpuset="0x00002000,,0x20000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="29" cpuset="0x20000000" complete_cpuset="0x20000000" online_cpuset="0x20000000" allowed_cpuset="0x20000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="77" cpuset="0x00002000,,0x0" complete_cpuset="0x00002000,,0x0" online_cpuset="0x00002000,,0x0" allowed_cpuset="0x00002000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="18" cpuset="0x0001c000,0x00000001,0xc0000000" complete_cpuset="0x0001c000,0x00000001,0xc0000000" online_cpuset="0x0001c000,0x00000001,0xc0000000" allowed_cpuset="0x0001c000,0x00000001,0xc0000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="72" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" online_cpuset="0x00004000,,0x40000000" allowed_cpuset="0x00004000,,0x40000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="72" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" online_cpuset="0x00004000,,0x40000000" allowed_cpuset="0x00004000,,0x40000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="72" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" online_cpuset="0x00004000,,0x40000000" allowed_cpuset="0x00004000,,0x40000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="8" cpuset="0x00004000,,0x40000000" complete_cpuset="0x00004000,,0x40000000" online_cpuset="0x00004000,,0x40000000" allowed_cpuset="0x00004000,,0x40000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="30" cpuset="0x40000000" complete_cpuset="0x40000000" online_cpuset="0x40000000" allowed_cpuset="0x40000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="78" cpuset="0x00004000,,0x0" complete_cpuset="0x00004000,,0x0" online_cpuset="0x00004000,,0x0" allowed_cpuset="0x00004000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="73" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" online_cpuset="0x00008000,,0x80000000" allowed_cpuset="0x00008000,,0x80000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="73" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" online_cpuset="0x00008000,,0x80000000" allowed_cpuset="0x00008000,,0x80000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="73" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" online_cpuset="0x00008000,,0x80000000" allowed_cpuset="0x00008000,,0x80000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="9" cpuset="0x00008000,,0x80000000" complete_cpuset="0x00008000,,0x80000000" online_cpuset="0x00008000,,0x80000000" allowed_cpuset="0x00008000,,0x80000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="31" cpuset="0x80000000" complete_cpuset="0x80000000" online_cpuset="0x80000000" allowed_cpuset="0x80000000" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="79" cpuset="0x00008000,,0x0" complete_cpuset="0x00008000,,0x0" online_cpuset="0x00008000,,0x0" allowed_cpuset="0x00008000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="74" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" online_cpuset="0x00010000,0x00000001,0x0" allowed_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="74" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" online_cpuset="0x00010000,0x00000001,0x0" allowed_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="74" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" online_cpuset="0x00010000,0x00000001,0x0" allowed_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="10" cpuset="0x00010000,0x00000001,0x0" complete_cpuset="0x00010000,0x00000001,0x0" online_cpuset="0x00010000,0x00000001,0x0" allowed_cpuset="0x00010000,0x00000001,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="32" cpuset="0x00000001,0x0" complete_cpuset="0x00000001,0x0" online_cpuset="0x00000001,0x0" allowed_cpuset="0x00000001,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="80" cpuset="0x00010000,,0x0" complete_cpuset="0x00010000,,0x0" online_cpuset="0x00010000,,0x0" allowed_cpuset="0x00010000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="19" cpuset="0x000e0000,0x0000000e,0x0" complete_cpuset="0x000e0000,0x0000000e,0x0" online_cpuset="0x000e0000,0x0000000e,0x0" allowed_cpuset="0x000e0000,0x0000000e,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="76" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" online_cpuset="0x00020000,0x00000002,0x0" allowed_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="76" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" online_cpuset="0x00020000,0x00000002,0x0" allowed_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="76" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" online_cpuset="0x00020000,0x00000002,0x0" allowed_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="12" cpuset="0x00020000,0x00000002,0x0" complete_cpuset="0x00020000,0x00000002,0x0" online_cpuset="0x00020000,0x00000002,0x0" allowed_cpuset="0x00020000,0x00000002,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="33" cpuset="0x00000002,0x0" complete_cpuset="0x00000002,0x0" online_cpuset="0x00000002,0x0" allowed_cpuset="0x00000002,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="81" cpuset="0x00020000,,0x0" complete_cpuset="0x00020000,,0x0" online_cpuset="0x00020000,,0x0" allowed_cpuset="0x00020000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="77" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" online_cpuset="0x00040000,0x00000004,0x0" allowed_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="77" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" online_cpuset="0x00040000,0x00000004,0x0" allowed_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="77" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" online_cpuset="0x00040000,0x00000004,0x0" allowed_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="13" cpuset="0x00040000,0x00000004,0x0" complete_cpuset="0x00040000,0x00000004,0x0" online_cpuset="0x00040000,0x00000004,0x0" allowed_cpuset="0x00040000,0x00000004,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="34" cpuset="0x00000004,0x0" complete_cpuset="0x00000004,0x0" online_cpuset="0x00000004,0x0" allowed_cpuset="0x00000004,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="82" cpuset="0x00040000,,0x0" complete_cpuset="0x00040000,,0x0" online_cpuset="0x00040000,,0x0" allowed_cpuset="0x00040000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="78" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" online_cpuset="0x00080000,0x00000008,0x0" allowed_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="78" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" online_cpuset="0x00080000,0x00000008,0x0" allowed_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="78" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" online_cpuset="0x00080000,0x00000008,0x0" allowed_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="14" cpuset="0x00080000,0x00000008,0x0" complete_cpuset="0x00080000,0x00000008,0x0" online_cpuset="0x00080000,0x00000008,0x0" allowed_cpuset="0x00080000,0x00000008,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="35" cpuset="0x00000008,0x0" complete_cpuset="0x00000008,0x0" online_cpuset="0x00000008,0x0" allowed_cpuset="0x00000008,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="83" cpuset="0x00080000,,0x0" complete_cpuset="0x00080000,,0x0" online_cpuset="0x00080000,,0x0" allowed_cpuset="0x00080000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="20" cpuset="0x00700000,0x00000070,0x0" complete_cpuset="0x00700000,0x00000070,0x0" online_cpuset="0x00700000,0x00000070,0x0" allowed_cpuset="0x00700000,0x00000070,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="80" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" online_cpuset="0x00100000,0x00000010,0x0" allowed_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="80" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" online_cpuset="0x00100000,0x00000010,0x0" allowed_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="80" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" online_cpuset="0x00100000,0x00000010,0x0" allowed_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="16" cpuset="0x00100000,0x00000010,0x0" complete_cpuset="0x00100000,0x00000010,0x0" online_cpuset="0x00100000,0x00000010,0x0" allowed_cpuset="0x00100000,0x00000010,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="36" cpuset="0x00000010,0x0" complete_cpuset="0x00000010,0x0" online_cpuset="0x00000010,0x0" allowed_cpuset="0x00000010,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="84" cpuset="0x00100000,,0x0" complete_cpuset="0x00100000,,0x0" online_cpuset="0x00100000,,0x0" allowed_cpuset="0x00100000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="81" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" online_cpuset="0x00200000,0x00000020,0x0" allowed_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="81" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" online_cpuset="0x00200000,0x00000020,0x0" allowed_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="81" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" online_cpuset="0x00200000,0x00000020,0x0" allowed_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="17" cpuset="0x00200000,0x00000020,0x0" complete_cpuset="0x00200000,0x00000020,0x0" online_cpuset="0x00200000,0x00000020,0x0" allowed_cpuset="0x00200000,0x00000020,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="37" cpuset="0x00000020,0x0" complete_cpuset="0x00000020,0x0" online_cpuset="0x00000020,0x0" allowed_cpuset="0x00000020,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="85" cpuset="0x00200000,,0x0" complete_cpuset="0x00200000,,0x0" online_cpuset="0x00200000,,0x0" allowed_cpuset="0x00200000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="82" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" online_cpuset="0x00400000,0x00000040,0x0" allowed_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="82" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" online_cpuset="0x00400000,0x00000040,0x0" allowed_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="82" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" online_cpuset="0x00400000,0x00000040,0x0" allowed_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="18" cpuset="0x00400000,0x00000040,0x0" complete_cpuset="0x00400000,0x00000040,0x0" online_cpuset="0x00400000,0x00000040,0x0" allowed_cpuset="0x00400000,0x00000040,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="38" cpuset="0x00000040,0x0" complete_cpuset="0x00000040,0x0" online_cpuset="0x00000040,0x0" allowed_cpuset="0x00000040,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="86" cpuset="0x00400000,,0x0" complete_cpuset="0x00400000,,0x0" online_cpuset="0x00400000,,0x0" allowed_cpuset="0x00400000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="21" cpuset="0x03800000,0x00000380,0x0" complete_cpuset="0x03800000,0x00000380,0x0" online_cpuset="0x03800000,0x00000380,0x0" allowed_cpuset="0x03800000,0x00000380,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="84" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" online_cpuset="0x00800000,0x00000080,0x0" allowed_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="84" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" online_cpuset="0x00800000,0x00000080,0x0" allowed_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="84" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" online_cpuset="0x00800000,0x00000080,0x0" allowed_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="20" cpuset="0x00800000,0x00000080,0x0" complete_cpuset="0x00800000,0x00000080,0x0" online_cpuset="0x00800000,0x00000080,0x0" allowed_cpuset="0x00800000,0x00000080,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="39" cpuset="0x00000080,0x0" complete_cpuset="0x00000080,0x0" online_cpuset="0x00000080,0x0" allowed_cpuset="0x00000080,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="87" cpuset="0x00800000,,0x0" complete_cpuset="0x00800000,,0x0" online_cpuset="0x00800000,,0x0" allowed_cpuset="0x00800000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="85" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" online_cpuset="0x01000000,0x00000100,0x0" allowed_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="85" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" online_cpuset="0x01000000,0x00000100,0x0" allowed_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="85" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" online_cpuset="0x01000000,0x00000100,0x0" allowed_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="21" cpuset="0x01000000,0x00000100,0x0" complete_cpuset="0x01000000,0x00000100,0x0" online_cpuset="0x01000000,0x00000100,0x0" allowed_cpuset="0x01000000,0x00000100,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="40" cpuset="0x00000100,0x0" complete_cpuset="0x00000100,0x0" online_cpuset="0x00000100,0x0" allowed_cpuset="0x00000100,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="88" cpuset="0x01000000,,0x0" complete_cpuset="0x01000000,,0x0" online_cpuset="0x01000000,,0x0" allowed_cpuset="0x01000000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="86" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" online_cpuset="0x02000000,0x00000200,0x0" allowed_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="86" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" online_cpuset="0x02000000,0x00000200,0x0" allowed_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="86" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" online_cpuset="0x02000000,0x00000200,0x0" allowed_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="22" cpuset="0x02000000,0x00000200,0x0" complete_cpuset="0x02000000,0x00000200,0x0" online_cpuset="0x02000000,0x00000200,0x0" allowed_cpuset="0x02000000,0x00000200,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="41" cpuset="0x00000200,0x0" complete_cpuset="0x00000200,0x0" online_cpuset="0x00000200,0x0" allowed_cpuset="0x00000200,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="89" cpuset="0x02000000,,0x0" complete_cpuset="0x02000000,,0x0" online_cpuset="0x02000000,,0x0" allowed_cpuset="0x02000000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="22" cpuset="0x1c000000,0x00001c00,0x0" complete_cpuset="0x1c000000,0x00001c00,0x0" online_cpuset="0x1c000000,0x00001c00,0x0" allowed_cpuset="0x1c000000,0x00001c00,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="88" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" online_cpuset="0x04000000,0x00000400,0x0" allowed_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="88" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" online_cpuset="0x04000000,0x00000400,0x0" allowed_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="88" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" online_cpuset="0x04000000,0x00000400,0x0" allowed_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="24" cpuset="0x04000000,0x00000400,0x0" complete_cpuset="0x04000000,0x00000400,0x0" online_cpuset="0x04000000,0x00000400,0x0" allowed_cpuset="0x04000000,0x00000400,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="42" cpuset="0x00000400,0x0" complete_cpuset="0x00000400,0x0" online_cpuset="0x00000400,0x0" allowed_cpuset="0x00000400,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="90" cpuset="0x04000000,,0x0" complete_cpuset="0x04000000,,0x0" online_cpuset="0x04000000,,0x0" allowed_cpuset="0x04000000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="89" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" online_cpuset="0x08000000,0x00000800,0x0" allowed_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="89" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" online_cpuset="0x08000000,0x00000800,0x0" allowed_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="89" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" online_cpuset="0x08000000,0x00000800,0x0" allowed_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="25" cpuset="0x08000000,0x00000800,0x0" complete_cpuset="0x08000000,0x00000800,0x0" online_cpuset="0x08000000,0x00000800,0x0" allowed_cpuset="0x08000000,0x00000800,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="43" cpuset="0x00000800,0x0" complete_cpuset="0x00000800,0x0" online_cpuset="0x00000800,0x0" allowed_cpuset="0x00000800,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="91" cpuset="0x08000000,,0x0" complete_cpuset="0x08000000,,0x0" online_cpuset="0x08000000,,0x0" allowed_cpuset="0x08000000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="90" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" online_cpuset="0x10000000,0x00001000,0x0" allowed_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="90" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" online_cpuset="0x10000000,0x00001000,0x0" allowed_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="90" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" online_cpuset="0x10000000,0x00001000,0x0" allowed_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="26" cpuset="0x10000000,0x00001000,0x0" complete_cpuset="0x10000000,0x00001000,0x0" online_cpuset="0x10000000,0x00001000,0x0" allowed_cpuset="0x10000000,0x00001000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="44" cpuset="0x00001000,0x0" complete_cpuset="0x00001000,0x0" online_cpuset="0x00001000,0x0" allowed_cpuset="0x00001000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="92" cpuset="0x10000000,,0x0" complete_cpuset="0x10000000,,0x0" online_cpuset="0x10000000,,0x0" allowed_cpuset="0x10000000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Cache" os_index="23" cpuset="0xe0000000,0x0000e000,0x0" complete_cpuset="0xe0000000,0x0000e000,0x0" online_cpuset="0xe0000000,0x0000e000,0x0" allowed_cpuset="0xe0000000,0x0000e000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="16777216" depth="3" cache_linesize="64" cache_associativity="16" cache_type="0">
+          <info name="Inclusive" value="0"/>
+          <object type="Cache" os_index="92" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" online_cpuset="0x20000000,0x00002000,0x0" allowed_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="92" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" online_cpuset="0x20000000,0x00002000,0x0" allowed_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="92" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" online_cpuset="0x20000000,0x00002000,0x0" allowed_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="28" cpuset="0x20000000,0x00002000,0x0" complete_cpuset="0x20000000,0x00002000,0x0" online_cpuset="0x20000000,0x00002000,0x0" allowed_cpuset="0x20000000,0x00002000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="45" cpuset="0x00002000,0x0" complete_cpuset="0x00002000,0x0" online_cpuset="0x00002000,0x0" allowed_cpuset="0x00002000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="93" cpuset="0x20000000,,0x0" complete_cpuset="0x20000000,,0x0" online_cpuset="0x20000000,,0x0" allowed_cpuset="0x20000000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="93" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" online_cpuset="0x40000000,0x00004000,0x0" allowed_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="93" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" online_cpuset="0x40000000,0x00004000,0x0" allowed_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="93" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" online_cpuset="0x40000000,0x00004000,0x0" allowed_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="29" cpuset="0x40000000,0x00004000,0x0" complete_cpuset="0x40000000,0x00004000,0x0" online_cpuset="0x40000000,0x00004000,0x0" allowed_cpuset="0x40000000,0x00004000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="46" cpuset="0x00004000,0x0" complete_cpuset="0x00004000,0x0" online_cpuset="0x00004000,0x0" allowed_cpuset="0x00004000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="94" cpuset="0x40000000,,0x0" complete_cpuset="0x40000000,,0x0" online_cpuset="0x40000000,,0x0" allowed_cpuset="0x40000000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Cache" os_index="94" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" online_cpuset="0x80000000,0x00008000,0x0" allowed_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="524288" depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
+            <info name="Inclusive" value="1"/>
+            <object type="Cache" os_index="94" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" online_cpuset="0x80000000,0x00008000,0x0" allowed_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="1">
+              <info name="Inclusive" value="0"/>
+              <object type="Cache" os_index="94" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" online_cpuset="0x80000000,0x00008000,0x0" allowed_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" cache_associativity="8" cache_type="2">
+                <info name="Inclusive" value="0"/>
+                <object type="Core" os_index="30" cpuset="0x80000000,0x00008000,0x0" complete_cpuset="0x80000000,0x00008000,0x0" online_cpuset="0x80000000,0x00008000,0x0" allowed_cpuset="0x80000000,0x00008000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002">
+                  <object type="PU" os_index="47" cpuset="0x00008000,0x0" complete_cpuset="0x00008000,0x0" online_cpuset="0x00008000,0x0" allowed_cpuset="0x00008000,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                  <object type="PU" os_index="95" cpuset="0x80000000,,0x0" complete_cpuset="0x80000000,,0x0" online_cpuset="0x80000000,,0x0" allowed_cpuset="0x80000000,,0x0" nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002"/>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" bridge_type="0-1" depth="0" bridge_pci="0000:[80-85]">
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[81-83]" pci_busid="0000:80:01.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="31.507692">
+            <object type="Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[82-83]" pci_busid="0000:81:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="31.507692">
+              <object type="Bridge" bridge_type="1-1" depth="3" bridge_pci="0000:[83-83]" pci_busid="0000:82:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <object type="PCIDev" pci_busid="0000:83:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.000000">
+                  <object type="OSDev" name="rsmi6" osdev_type="1">
+                    <info name="Backend" value="RSMI"/>
+                    <info name="GPUVendor" value="AMD"/>
+                    <info name="GPUModel" value="deon Instinct MI50 32GB"/>
+                    <info name="AMDSerial" value="20080004280c"/>
+                    <info name="AMDUUID" value="6986496172e17d3a"/>
+                    <info name="XGMIHiveID" value="0"/>
+                    <info name="RSMIVRAMSize" value="33538048"/>
+                    <info name="RSMIVisibleVRAMSize" value="33538048"/>
+                    <info name="RSMIGTTSize" value="263645540"/>
+                    <info name="Type" value="RSMI"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+        </object>
+        <object type="Bridge" bridge_type="0-1" depth="0" bridge_pci="0000:[a0-a7]">
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[a1-a3]" pci_busid="0000:a0:03.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="31.507692">
+            <object type="Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[a2-a3]" pci_busid="0000:a1:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="31.507692">
+              <object type="Bridge" bridge_type="1-1" depth="3" bridge_pci="0000:[a3-a3]" pci_busid="0000:a2:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <object type="PCIDev" pci_busid="0000:a3:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.000000">
+                  <object type="OSDev" name="rsmi7" osdev_type="1">
+                    <info name="Backend" value="RSMI"/>
+                    <info name="GPUVendor" value="AMD"/>
+                    <info name="GPUModel" value="deon Instinct MI50 32GB"/>
+                    <info name="AMDSerial" value="20170007010c"/>
+                    <info name="AMDUUID" value="6fe2214172e626c3"/>
+                    <info name="XGMIHiveID" value="0"/>
+                    <info name="RSMIVRAMSize" value="33538048"/>
+                    <info name="RSMIVisibleVRAMSize" value="33538048"/>
+                    <info name="RSMIGTTSize" value="263645540"/>
+                    <info name="Type" value="RSMI"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[a6-a6]" pci_busid="0000:a0:08.2" pci_type="0604 [1022:1484] [1022:1484] 00" pci_link_speed="31.507692">
+            <object type="PCIDev" pci_busid="0000:a6:00.0" pci_type="0106 [1022:7901] [15d9:7901] 51" pci_link_speed="31.507692"/>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[a7-a7]" pci_busid="0000:a0:08.3" pci_type="0604 [1022:1484] [1022:1484] 00" pci_link_speed="31.507692">
+            <object type="PCIDev" pci_busid="0000:a7:00.0" pci_type="0106 [1022:7901] [15d9:7901] 51" pci_link_speed="31.507692"/>
+          </object>
+        </object>
+        <object type="Bridge" bridge_type="0-1" depth="0" bridge_pci="0000:[c0-c8]">
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[c1-c3]" pci_busid="0000:c0:01.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="31.507692">
+            <object type="Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[c2-c3]" pci_busid="0000:c1:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="31.507692">
+              <object type="Bridge" bridge_type="1-1" depth="3" bridge_pci="0000:[c3-c3]" pci_busid="0000:c2:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <object type="PCIDev" pci_busid="0000:c3:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.000000">
+                  <object type="OSDev" name="rsmi5" osdev_type="1">
+                    <info name="Backend" value="RSMI"/>
+                    <info name="GPUVendor" value="AMD"/>
+                    <info name="GPUModel" value="deon Instinct MI50 32GB"/>
+                    <info name="AMDSerial" value="20080001600c"/>
+                    <info name="AMDUUID" value="d06230a172e620ce"/>
+                    <info name="XGMIHiveID" value="0"/>
+                    <info name="RSMIVRAMSize" value="33538048"/>
+                    <info name="RSMIVisibleVRAMSize" value="33538048"/>
+                    <info name="RSMIGTTSize" value="263645540"/>
+                    <info name="Type" value="RSMI"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[c4-c4]" pci_busid="0000:c0:03.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="31.507692">
+            <object type="PCIDev" pci_busid="0000:c4:00.0" pci_type="0207 [15b3:101b] [15b3:0006] 00" pci_link_speed="31.507692">
+              <info name="PCISlot" value="8"/>
+              <object type="OSDev" name="hsi0" osdev_type="2">
+                <info name="Address" value="00:00:01:c6:fe:80:00:00:00:00:00:00:0c:42:a1:03:00:6f:14:ec"/>
+                <info name="Port" value="1"/>
+              </object>
+              <object type="OSDev" name="mlx5_0" osdev_type="3">
+                <info name="NodeGUID" value="0c42:a103:006f:14ec"/>
+                <info name="SysImageGUID" value="0c42:a103:006f:14ec"/>
+                <info name="Port1State" value="4"/>
+                <info name="Port1LID" value="0x5d"/>
+                <info name="Port1LMC" value="0"/>
+                <info name="Port1GID0" value="fe80:0000:0000:0000:0c42:a103:006f:14ec"/>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[c7-c7]" pci_busid="0000:c0:08.2" pci_type="0604 [1022:1484] [1022:1484] 00" pci_link_speed="31.507692">
+            <object type="PCIDev" pci_busid="0000:c7:00.0" pci_type="0106 [1022:7901] [15d9:7901] 51" pci_link_speed="31.507692"/>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[c8-c8]" pci_busid="0000:c0:08.3" pci_type="0604 [1022:1484] [1022:1484] 00" pci_link_speed="31.507692">
+            <object type="PCIDev" pci_busid="0000:c8:00.0" pci_type="0106 [1022:7901] [15d9:7901] 51" pci_link_speed="31.507692"/>
+          </object>
+        </object>
+        <object type="Bridge" bridge_type="0-1" depth="0" bridge_pci="0000:[e0-eb]">
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[e1-e3]" pci_busid="0000:e0:03.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="31.507692">
+            <object type="Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[e2-e3]" pci_busid="0000:e1:00.0" pci_type="0604 [1002:14a0] [0000:0000] 02" pci_link_speed="31.507692">
+              <object type="Bridge" bridge_type="1-1" depth="3" bridge_pci="0000:[e3-e3]" pci_busid="0000:e2:00.0" pci_type="0604 [1002:14a1] [1002:14a1] 00" pci_link_speed="31.507692">
+                <object type="PCIDev" pci_busid="0000:e3:00.0" pci_type="0380 [1002:66a1] [1002:0834] 02" pci_link_speed="31.000000">
+                  <object type="OSDev" name="rsmi4" osdev_type="1">
+                    <info name="Backend" value="RSMI"/>
+                    <info name="GPUVendor" value="AMD"/>
+                    <info name="GPUModel" value="deon Instinct MI50 32GB"/>
+                    <info name="AMDSerial" value="20080009890c"/>
+                    <info name="AMDUUID" value="122c188172e6212f"/>
+                    <info name="XGMIHiveID" value="0"/>
+                    <info name="RSMIVRAMSize" value="33538048"/>
+                    <info name="RSMIVisibleVRAMSize" value="33538048"/>
+                    <info name="RSMIGTTSize" value="263645540"/>
+                    <info name="Type" value="RSMI"/>
+                  </object>
+                </object>
+              </object>
+            </object>
+          </object>
+          <object type="Bridge" bridge_type="1-1" depth="1" bridge_pci="0000:[e4-e9]" pci_busid="0000:e0:05.1" pci_type="0604 [1022:1483] [1022:1453] 00" pci_link_speed="0.615385">
+            <object type="Bridge" bridge_type="1-1" depth="2" bridge_pci="0000:[e5-e9]" pci_busid="0000:e4:00.0" pci_type="0604 [12d8:2404] [0000:0000] 05" pci_link_speed="0.615385">
+              <object type="Bridge" bridge_type="1-1" depth="3" bridge_pci="0000:[e7-e8]" pci_busid="0000:e5:02.0" pci_type="0604 [12d8:2404] [0000:0000] 05" pci_link_speed="0.615385">
+                <object type="PCIDev" pci_busid="0000:e7:00.0" pci_type="0200 [8086:1521] [15d9:088e] 01" pci_link_speed="0.615385">
+                  <object type="OSDev" name="eno1" osdev_type="2">
+                    <info name="Address" value="ac:1f:6b:48:8e:5e"/>
+                  </object>
+                </object>
+                <object type="PCIDev" pci_busid="0000:e7:00.1" pci_type="0200 [8086:1521] [15d9:088e] 01" pci_link_speed="0.615385">
+                  <object type="OSDev" name="eno2" osdev_type="2">
+                    <info name="Address" value="ac:1f:6b:48:8e:5f"/>
+                  </object>
+                </object>
+              </object>
+              <object type="Bridge" bridge_type="1-1" depth="3" bridge_pci="0000:[e9-e9]" pci_busid="0000:e5:03.0" pci_type="0604 [12d8:2404] [0000:0000] 05" pci_link_speed="0.615385">
+                <object type="PCIDev" pci_busid="0000:e9:00.0" pci_type="0106 [1b4b:9230] [1b4b:9230] 11" pci_link_speed="0.615385"/>
+              </object>
+            </object>
+          </object>
+        </object>
+      </object>
+    </object>
+    <object type="OSDev" name="sdb" osdev_type="0">
+      <info name="Size" value="34603008"/>
+      <info name="SectorSize" value="512"/>
+      <info name="LinuxDeviceID" value="8:16"/>
+      <info name="Vendor" value="LIO-ORG"/>
+      <info name="Model" value="FILEIO"/>
+      <info name="Revision" value="4.0"/>
+      <info name="SerialNumber" value="600140573dad0e94d04fc9cebc7bca0d"/>
+      <info name="Type" value="Disk"/>
+    </object>
+    <object type="OSDev" name="sda" osdev_type="0">
+      <info name="Size" value="34603008"/>
+      <info name="SectorSize" value="512"/>
+      <info name="LinuxDeviceID" value="8:0"/>
+      <info name="Vendor" value="LIO-ORG"/>
+      <info name="Model" value="FILEIO"/>
+      <info name="Revision" value="4.0"/>
+      <info name="SerialNumber" value="600140573dad0e94d04fc9cebc7bca0d"/>
+      <info name="Type" value="Disk"/>
+    </object>
+  </object>
+</topology>
diff --git a/t/hwloc/hwloc-convert.c b/t/hwloc/hwloc-convert.c
new file mode 100644
index 000000000000..6328bc315da5
--- /dev/null
+++ b/t/hwloc/hwloc-convert.c
@@ -0,0 +1,74 @@
+#include <errno.h>
+#include <string.h>
+#include <stdio.h>
+
+#include <hwloc.h>
+
+int main(int argc, char *argv[])
+{
+#if HWLOC_API_VERSION < 0x20000
+    fprintf(stderr, "hwloc-convert requires hwloc v2.0+\n");
+    return 1;
+#else
+    const char* usage = "USAGE: hwloc-convert input_xml output_xml";
+    if (argc != 3) {
+        printf("Incorrect arguments supplied.\n");
+        printf("%s\n", usage);
+    }
+
+    const char *inpath = argv[1];
+    int rc = 0, exit_code = 0;
+
+    hwloc_topology_t topology;
+    if (hwloc_topology_init (&topology) < 0) {
+        fprintf(stderr, "Error initializing hwloc topology\n");
+        exit_code = 1;
+        goto ret;
+    }
+    if (hwloc_topology_set_io_types_filter(topology,
+                                           HWLOC_TYPE_FILTER_KEEP_IMPORTANT)
+        < 0) {
+        fprintf(stderr, "hwloc_topology_set_io_types_filter\n");
+        exit_code = errno;
+        goto ret;
+    }
+    if (hwloc_topology_set_cache_types_filter(topology,
+                                              HWLOC_TYPE_FILTER_KEEP_STRUCTURE)
+        < 0) {
+        fprintf(stderr, "hwloc_topology_set_cache_types_filter\n");
+        exit_code = errno;
+        goto ret;
+    }
+    if (hwloc_topology_set_icache_types_filter(topology,
+                                               HWLOC_TYPE_FILTER_KEEP_STRUCTURE)
+        < 0) {
+        fprintf(stderr, "hwloc_topology_set_icache_types_filter\n");
+        exit_code = errno;
+        goto ret;
+    }
+    if ((rc = hwloc_topology_set_xml(topology, inpath)) < 0) {
+        fprintf(stderr, "Error reading XML: %s\n", strerror(errno));
+        exit_code = errno;
+        goto ret;
+    }
+    if ((rc = hwloc_topology_load(topology)) < 0) {
+        fprintf(stderr, "Error loading topology\n");
+        exit_code = 1;
+        goto ret;
+    }
+
+    const char *outpath = argv[2];
+    if ((rc = hwloc_topology_export_xml(topology, outpath,
+                                        HWLOC_TOPOLOGY_EXPORT_XML_FLAG_V1))
+        < 0) {
+        fprintf(stderr, "Error exporting XML\n");
+        exit_code = 1;
+        goto ret;
+
+    }
+
+ret:
+    hwloc_topology_destroy (topology);
+    return exit_code;
+#endif
+}
diff --git a/t/hwloc/hwloc-version.c b/t/hwloc/hwloc-version.c
new file mode 100644
index 000000000000..7de84017d91a
--- /dev/null
+++ b/t/hwloc/hwloc-version.c
@@ -0,0 +1,12 @@
+#include <stdio.h>
+#include <hwloc.h>
+
+int main(void)
+{
+#if HWLOC_API_VERSION >= 0x20000
+    printf("2\n");
+#else
+    printf("1\n");
+#endif
+    return 0;
+}
diff --git a/t/ingest/bad-validate.py b/t/ingest/bad-validate.py
new file mode 100644
index 000000000000..5e1fde33fabd
--- /dev/null
+++ b/t/ingest/bad-validate.py
@@ -0,0 +1,20 @@
+##############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import sys
+from flux.job.validator import ValidatorPlugin
+
+
+class Validator(ValidatorPlugin):
+    """Fake validator that exits on first validate() call"""
+
+    def validate(self, args):
+        print("Exiting with code 1 for testing", file=sys.stderr, flush=True)
+        sys.exit(1)
diff --git a/t/ingest/bad-validate.sh b/t/ingest/bad-validate.sh
deleted file mode 100755
index 9ec18ba93ec4..000000000000
--- a/t/ingest/bad-validate.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-read line
-exit 1
diff --git a/t/ingest/fake-validate.sh b/t/ingest/fake-validate.sh
deleted file mode 100755
index ddadf983b4db..000000000000
--- a/t/ingest/fake-validate.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-while read line
-do
-    echo '{"errnum": 0}'
-done <&0
-
-exit 0
diff --git a/t/ingest/job-manager-dummy.c b/t/ingest/job-manager-dummy.c
deleted file mode 100644
index beb1b956eeb3..000000000000
--- a/t/ingest/job-manager-dummy.c
+++ /dev/null
@@ -1,164 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* dummy job manager for test */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <flux/core.h>
-#include <jansson.h>
-
-#include "src/common/libeventlog/eventlog.h"
-
-const char *eventlog_path = "test.ingest.eventlog";
-
-/* KVS commit completed.
- * Respond to original request which was copied and passed as 'arg'.
- */
-static void commit_continuation (flux_future_t *f, void *arg)
-{
-    flux_t *h = flux_future_get_flux (f);
-    flux_msg_t *msg = arg;
-
-    if (flux_future_get (f, NULL) < 0) {
-        if (flux_respond_error (h, msg, errno, NULL) < 0)
-            flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    }
-    else {
-        if (flux_respond (h, msg, NULL) < 0)
-            flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    }
-    flux_msg_destroy (msg);
-    flux_future_destroy (f);
-}
-
-/* Given a JSON job object, encode a KVS eventlog entry
- * to represent its submission, timestamped now.  Caller must free.
- */
-static char *create_eventlog_entry (json_t *job)
-{
-    int priority;
-    flux_jobid_t id;
-    uint32_t userid;
-    double t_submit;
-    json_t *entry = NULL;
-    char *entrystr;
-    int save_errno;
-
-    if (json_unpack (job, "{s:I s:i s:i s:f}", "id", &id,
-                                               "userid", &userid,
-                                               "priority", &priority,
-                                               "t_submit", &t_submit) < 0)
-        goto error_inval;
-    if (!(entry = eventlog_entry_pack (0., "submit", "{ s:I s:i s:i s:f }",
-                                       "id", id,
-                                       "priority", priority,
-                                       "userid", userid,
-                                       "t_submit", t_submit)))
-        goto error;
-    if (!(entrystr = eventlog_entry_encode (entry)))
-        goto error;
-    json_decref (entry);
-    return entrystr;
-
-error_inval:
-    errno = EINVAL;
-error:
-    save_errno = errno;
-    json_decref (entry);
-    errno = save_errno;
-    return NULL;
-}
-
-/* Given a JSON array of job records, add an eventlog
- * update for each job to a KVS transaction and return it.
- */
-static flux_kvs_txn_t *create_eventlog_txn (json_t *jobs)
-{
-    flux_kvs_txn_t *txn;
-    size_t index;
-    json_t *job;
-
-    if (!(txn = flux_kvs_txn_create ()))
-        return NULL;
-    json_array_foreach (jobs, index, job) {
-        char *event = create_eventlog_entry (job);
-        if (!event)
-            goto error;
-        if (flux_kvs_txn_put (txn, FLUX_KVS_APPEND, eventlog_path, event) < 0) {
-            int saved_errno = errno;
-            free (event);
-            errno = saved_errno;
-            goto error;
-        }
-        free (event);
-    }
-    return txn;
-error:
-    flux_kvs_txn_destroy (txn);
-    return NULL;
-}
-
-static void submit_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
-{
-    json_t *jobs;
-    flux_kvs_txn_t *txn = NULL;
-    flux_future_t *f = NULL;
-    flux_msg_t *cpy = NULL;
-
-    if (flux_request_unpack (msg, NULL, "{s:o}", "jobs", &jobs) < 0)
-        goto error;
-    if (!(cpy = flux_msg_copy (msg, false)))
-        goto error;
-    if (!(txn = create_eventlog_txn (jobs)))
-        goto error;
-    if (!(f = flux_kvs_commit (h, NULL, 0, txn)))
-        goto error;
-    if (flux_future_then (f, -1., commit_continuation, cpy) < 0)
-        goto error;
-    flux_kvs_txn_destroy (txn);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    flux_future_destroy (f);
-    flux_msg_destroy (cpy);
-    flux_kvs_txn_destroy (txn);
-}
-
-static const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_REQUEST,  "job-manager.submit", submit_cb, 0 },
-    FLUX_MSGHANDLER_TABLE_END,
-};
-
-int mod_main (flux_t *h, int argc, char *argv[])
-{
-    flux_msg_handler_t **handlers = NULL;
-    int rc = -1;
-
-    if (flux_msg_handler_addvec (h, htab, NULL, &handlers) < 0) {
-        flux_log_error (h, "flux_msghandler_add");
-        goto done;
-    }
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
-        goto done;
-    rc = 0;
-done:
-    flux_msg_handler_delvec (handlers);
-    return rc;
-}
-
-MOD_NAME ("job-manager");
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/t/ingest/job-manager.c b/t/ingest/job-manager.c
new file mode 100644
index 000000000000..c6bf6e8e2492
--- /dev/null
+++ b/t/ingest/job-manager.c
@@ -0,0 +1,194 @@
+/************************************************************\
+ * Copyright 2018 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* dummy job manager for test */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <jansson.h>
+
+#include "src/common/libeventlog/eventlog.h"
+#include "ccan/str/str.h"
+
+const char *eventlog_path = "test.ingest.eventlog";
+static int force_fail = 0;
+
+/* KVS commit completed.
+ * Respond to original request which was copied and passed as 'arg'.
+ */
+static void commit_continuation (flux_future_t *f, void *arg)
+{
+    flux_t *h = flux_future_get_flux (f);
+    flux_msg_t *msg = arg;
+
+    if (flux_future_get (f, NULL) < 0) {
+        if (flux_respond_error (h, msg, errno, NULL) < 0)
+            flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+    }
+    else {
+        if (flux_respond (h, msg, "{}") < 0)
+            flux_log_error (h, "%s: flux_respond", __FUNCTION__);
+    }
+    flux_msg_destroy (msg);
+    flux_future_destroy (f);
+}
+
+/* Given a JSON job object, encode a KVS eventlog entry
+ * to represent its submission, timestamped now.  Caller must free.
+ */
+static char *create_eventlog_entry (json_t *job)
+{
+    int urgency;
+    flux_jobid_t id;
+    uint32_t userid;
+    double t_submit;
+    json_t *entry = NULL;
+    char *entrystr;
+    int save_errno;
+
+    if (json_unpack (job, "{s:I s:i s:i s:f}", "id", &id,
+                                               "userid", &userid,
+                                               "urgency", &urgency,
+                                               "t_submit", &t_submit) < 0)
+        goto error_inval;
+    if (!(entry = eventlog_entry_pack (0., "submit", "{ s:I s:i s:I s:f }",
+                                       "id", id,
+                                       "urgency", urgency,
+                                       "userid", (json_int_t) userid,
+                                       "t_submit", t_submit)))
+        goto error;
+    if (!(entrystr = eventlog_entry_encode (entry)))
+        goto error;
+    json_decref (entry);
+    return entrystr;
+
+error_inval:
+    errno = EINVAL;
+error:
+    save_errno = errno;
+    json_decref (entry);
+    errno = save_errno;
+    return NULL;
+}
+
+/* Given a JSON array of job records, add an eventlog
+ * update for each job to a KVS transaction and return it.
+ */
+static flux_kvs_txn_t *create_eventlog_txn (json_t *jobs)
+{
+    flux_kvs_txn_t *txn;
+    size_t index;
+    json_t *job;
+
+    if (!(txn = flux_kvs_txn_create ()))
+        return NULL;
+    json_array_foreach (jobs, index, job) {
+        char *event = create_eventlog_entry (job);
+        if (!event)
+            goto error;
+        if (flux_kvs_txn_put (txn, FLUX_KVS_APPEND, eventlog_path, event) < 0) {
+            int saved_errno = errno;
+            free (event);
+            errno = saved_errno;
+            goto error;
+        }
+        free (event);
+    }
+    return txn;
+error:
+    flux_kvs_txn_destroy (txn);
+    return NULL;
+}
+
+static void submit_cb (flux_t *h, flux_msg_handler_t *mh,
+                       const flux_msg_t *msg, void *arg)
+{
+    json_t *jobs;
+    flux_kvs_txn_t *txn = NULL;
+    flux_future_t *f = NULL;
+    flux_msg_t *cpy = NULL;
+
+    if (force_fail) {
+        errno = EAGAIN;
+        goto error;
+    }
+
+    if (flux_request_unpack (msg, NULL, "{s:o}", "jobs", &jobs) < 0)
+        goto error;
+    if (!(cpy = flux_msg_copy (msg, false)))
+        goto error;
+    if (!(txn = create_eventlog_txn (jobs)))
+        goto error;
+    if (!(f = flux_kvs_commit (h, NULL, 0, txn)))
+        goto error;
+    if (flux_future_then (f, -1., commit_continuation, cpy) < 0)
+        goto error;
+    flux_kvs_txn_destroy (txn);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
+    flux_future_destroy (f);
+    flux_msg_destroy (cpy);
+    flux_kvs_txn_destroy (txn);
+}
+
+void getinfo_cb (flux_t *h,
+                 flux_msg_handler_t *mh,
+                 const flux_msg_t *msg,
+                 void *arg)
+{
+    flux_jobid_t id = (1000ULL * 1000) << 24; // fluid with 1000s timestamp
+    if (flux_request_decode (msg, NULL, NULL) < 0)
+        goto error;
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:I}",
+                           "max_jobid",
+                           id) < 0)
+        flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
+    return;
+error:
+    if (flux_respond_error (h, msg, errno, NULL) < 0)
+        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
+}
+
+
+static const struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST,  "job-manager.submit", submit_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST,  "job-manager.getinfo", getinfo_cb, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+int mod_main (flux_t *h, int argc, char *argv[])
+{
+    flux_msg_handler_t **handlers = NULL;
+    int rc = -1;
+
+    if (argc >= 1 && streq (argv[0], "force_fail"))
+        force_fail = 1;
+
+    if (flux_msg_handler_addvec (h, htab, NULL, &handlers) < 0) {
+        flux_log_error (h, "flux_msghandler_add");
+        goto done;
+    }
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+        goto done;
+    rc = 0;
+done:
+    flux_msg_handler_delvec (handlers);
+    return rc;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/ingest/submitbench.c b/t/ingest/submitbench.c
index 7be866807114..8da981a8f33f 100644
--- a/t/ingest/submitbench.c
+++ b/t/ingest/submitbench.c
@@ -29,6 +29,7 @@
 #include "src/common/libutil/fluid.h"
 #include "src/common/libjob/job.h"
 #include "src/common/libutil/read_all.h"
+#include "ccan/str/str.h"
 
 int cmd_submitbench (optparse_t *p, int argc, char **argv);
 
@@ -40,8 +41,8 @@ static struct optparse_option opts[] =  {
     { .name = "fanout", .key = 'f', .has_arg = 1, .arginfo = "N",
       .usage = "Run at most N RPCs in parallel",
     },
-    { .name = "priority", .key = 'p', .has_arg = 1, .arginfo = "N",
-      .usage = "Set job priority (0-31, default=16)",
+    { .name = "urgency", .key = 'u', .has_arg = 1, .arginfo = "N",
+      .usage = "Set job urgency (0-31, default=16)",
     },
     { .name = "flags", .key = 'F', .has_arg = 3,
       .flags = OPTPARSE_OPT_AUTOSPLIT,
@@ -102,7 +103,8 @@ struct submitbench_ctx {
     void *jobspec;
     int jobspecsz;
     const char *J;
-    int priority;
+    int urgency;
+    uint32_t owner;
 };
 
 /* Read entire file 'name' ("-" for stdin).  Exit program on error.
@@ -113,7 +115,7 @@ size_t read_jobspec (const char *name, void **bufp)
     ssize_t size;
     void *buf;
 
-    if (!strcmp (name, "-"))
+    if (streq (name, "-"))
         fd = STDIN_FILENO;
     else {
         if ((fd = open (name, O_RDONLY)) < 0)
@@ -142,7 +144,7 @@ void submitbench_continuation (flux_future_t *f, void *arg)
         else
             log_msg_exit ("submit: %s", future_strerror (f, errno));
     }
-    printf ("%llu\n", (unsigned long long)id);
+    printf ("%ju\n", (uintmax_t)id);
     flux_future_destroy (f);
 
     ctx->rxcount++;
@@ -181,10 +183,14 @@ void submitbench_check (flux_reactor_t *r, flux_watcher_t *w,
         flux_future_t *f;
 #if HAVE_FLUX_SECURITY
         if (ctx->sec) {
+            const char *mech = ctx->sign_type;
+
+            if (mech == NULL && getuid () == ctx->owner)
+                mech = "none";
             if (!ctx->J || !optparse_hasopt (ctx->p, "reuse-signature")) {
                 if (!(ctx->J = flux_sign_wrap (ctx->sec, ctx->jobspec,
                                                ctx->jobspecsz,
-                                               ctx->sign_type, 0)))
+                                               mech, 0)))
                     log_err_exit ("flux_sign_wrap: %s",
                                   flux_security_last_error (ctx->sec));
             }
@@ -192,7 +198,7 @@ void submitbench_check (flux_reactor_t *r, flux_watcher_t *w,
         }
 #endif
         if (!(f = flux_job_submit (ctx->h, ctx->J ? ctx->J : ctx->jobspec,
-                                   ctx->priority, flags)))
+                                   ctx->urgency, flags)))
             log_err_exit ("flux_job_submit");
         if (flux_future_then (f, -1., submitbench_continuation, ctx) < 0)
             log_err_exit ("flux_future_then");
@@ -215,7 +221,7 @@ int cmd_submitbench (optparse_t *p, int argc, char **argv)
     if (optparse_hasopt (p, "flags")) {
         const char *name;
         while ((name = optparse_getopt_next (p, "flags"))) {
-            if (!strcmp (name, "debug"))
+            if (streq (name, "debug"))
                 ctx.flags |= FLUX_JOB_DEBUG;
             else
                 log_msg_exit ("unknown flag: %s", name);
@@ -235,6 +241,7 @@ int cmd_submitbench (optparse_t *p, int argc, char **argv)
             log_err_exit ("security config %s", flux_security_last_error (ctx.sec));
         ctx.sign_type = optparse_get_str (p, "sign-type", NULL);
     }
+
 #endif
     if (!(ctx.h = flux_open (NULL, 0)))
         log_err_exit ("flux_open");
@@ -243,7 +250,15 @@ int cmd_submitbench (optparse_t *p, int argc, char **argv)
     ctx.max_queue_depth = optparse_get_int (p, "fanout", 256);
     ctx.totcount = optparse_get_int (p, "repeat", 1);
     ctx.jobspecsz = read_jobspec (argv[optindex++], &ctx.jobspec);
-    ctx.priority = optparse_get_int (p, "priority", FLUX_JOB_PRIORITY_DEFAULT);
+    ctx.urgency = optparse_get_int (p, "urgency", FLUX_JOB_URGENCY_DEFAULT);
+
+    const char *tmp;
+    if (!(tmp = flux_attr_get (ctx.h, "security.owner")))
+        log_err_exit ("getattr security.owner");
+    errno = 0;
+    ctx.owner = strtoul (tmp, NULL, 10);
+    if (errno != 0)
+        log_err_exit ("getattr security.owner conversion to uint32");
 
     /* Prep/check/idle watchers perform flow control, keeping
      * at most ctx.max_queue_depth RPCs outstanding.
diff --git a/t/issues/t2284-initial-program-format-chars.sh b/t/issues/t2284-initial-program-format-chars.sh
index fc0968efd953..9be984959287 100755
--- a/t/issues/t2284-initial-program-format-chars.sh
+++ b/t/issues/t2284-initial-program-format-chars.sh
@@ -3,7 +3,7 @@
 
 for s in %h %g %%h %f; do
     echo "Running flux broker echo $s"
-    output=$(flux broker --shutdown-grace=0.1 -Sbroker.rc1_path= -Sbroker.rc3_path= /bin/echo $s)
+    output=$(flux broker -Sbroker.rc1_path= -Sbroker.rc3_path= /bin/echo $s)
     test "$output" = "$s"
 done
 exit 0
diff --git a/t/issues/t2492-shell-lost.sh b/t/issues/t2492-shell-lost.sh
new file mode 100755
index 000000000000..966802f4f7e4
--- /dev/null
+++ b/t/issues/t2492-shell-lost.sh
@@ -0,0 +1,73 @@
+#!/bin/sh
+
+log() { printf "t2492: $@\n" >&2; }
+die() { log "$@"; exit 1; }
+
+#  Check if we need to start parent job, if so, reexec under flux-start
+#  using the per-broker for test-pmi-clique so that the instance mapping
+#  simulates multiple nodes.
+#
+export FLUX_URI_RESOLVE_LOCAL=t
+if test "$T2492_ACTIVE" != "t"; then
+    export T2492_ACTIVE=t
+    log "Re-launching test script under flux-start using $(command -v flux)"
+    exec flux start -s 4 $0
+fi
+
+CRITICAL_RANKS=$(cat <<EOF
+import os
+import sys
+import time
+import flux
+
+try:
+    h = flux.Flux()
+    #
+    # Wait on barrier to ensure all tasks have started
+    # (prevents premature termintion by SIGINT while Python is initiating)
+    #
+    size = int(os.environ["FLUX_JOB_SIZE"])
+    taskid = int(os.environ["FLUX_TASK_RANK"])
+    if taskid == 0:
+        print(f"waiting on barrier for {size} tasks", flush=True)
+    h.barrier("test", size).get()
+    if taskid == 0:
+        print(f"exited barrier", flush=True)
+
+    # Kill job shell on broker rank 3
+    broker_rank = h.get_rank()
+    if broker_rank == int(sys.argv[1]):
+        #  kill job shell
+        os.kill(os.getppid(), 9)
+        sys.exit(0)
+
+    # Sleep in all other tasks
+    time.sleep(600)
+except KeyboardInterrupt:
+    sys.exit(0)
+EOF
+)
+for rank in 3 1; do
+    log ""
+    log "Testing handling of lost shell rank $rank:"
+
+    id=$(flux submit -N4 --tasks-per-node=1 \
+         --input=/dev/null \
+         -o exit-timeout=none \
+         --add-file=critical.py="${CRITICAL_RANKS}" \
+         flux python {{tmpdir}}/critical.py $rank)
+
+    log "Sumbmitted job $id. Waiting for shell rank $rank to be lost"
+
+    value="shell rank $rank (on $(hostname -s)): Killed"
+    flux job wait-event -Wt 15 -Hvp output -m message="$value" $id log
+
+    log "Sending SIGINT to $id. Job should now exit"
+    flux job kill --signal=2 $id
+    flux job attach -vEX $id
+    rc=$?
+    log "Job exited with rc=$rc (expecting 137 (128+9))"
+    test $rc -eq 137 || die "Unexpected job exit code $rc"
+done
+
+
diff --git a/t/issues/t2686-shell-input-race.sh b/t/issues/t2686-shell-input-race.sh
index 99f6157013d4..3fb55f4bed10 100755
--- a/t/issues/t2686-shell-input-race.sh
+++ b/t/issues/t2686-shell-input-race.sh
@@ -11,16 +11,16 @@ cat <<EOF >test.sh
 RC=0
 RANKS="[0-\$((\$(flux getattr size)-1))]"
 
-flux kvs put resource.hwloc.by_rank="{\"\$RANKS\": {\"Package\": 1, \"Core\": 16, \"PU\": 16, \"cpuset\": \"0-15\"}}"
-
-flux kvs get resource.hwloc.by_rank
+flux kvs put resource.R="\$(flux R encode -r\$RANKS -c0-15)"
+flux kvs get resource.R
 
 flux module remove sched-simple
+flux module reload resource monitor-force-up noverify
 flux module load sched-simple
 
 flux dmesg | grep 'sched-simple.*ready'  | tail -1
 
-flux mini submit --dry-run -o cpu-affinity=off -N2 -n2 sleep 0 \
+flux submit --dry-run -o cpu-affinity=off -N2 -n2 sleep 0 \
     | ${SUBMITBENCH} -r 24 - >jobs.list
 
 cat jobs.list
diff --git a/t/issues/t3186-python-future-get-sigint.sh b/t/issues/t3186-python-future-get-sigint.sh
new file mode 100755
index 000000000000..8d56722d43f4
--- /dev/null
+++ b/t/issues/t3186-python-future-get-sigint.sh
@@ -0,0 +1,44 @@
+#!/bin/sh
+# future/rpc.get() should be interruptible:
+
+SERVICE="t3186"
+waitfile=${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua
+
+log() { echo "t3186: $@" >&2; }
+die() { log "$@"; exit 1; }
+
+cat <<EOF >get-intr.py || die "Failed to create test script"
+import flux
+
+f = flux.Flux()
+f.service_register("$SERVICE").get()
+print("get-intr.py: Added service $SERVICE", flush=True)
+
+# The following should block until interrupted:
+f.rpc("${SERVICE}.echo").get()
+EOF
+
+log "Created test script get-intr.py"
+
+flux python get-intr.py >t3186.log 2>&1 &
+pid=$!
+
+log "Started PID=$pid"
+
+$waitfile --timeout=10 --pattern="Added service" t3186.log
+
+log "Sending SIGINT to $pid"
+kill -INT $pid || die "Failed to kill PID $pid"
+
+while kill -0 $pid; do
+    log "PID $pid still running, trying again"
+    kill -INT $pid
+    sleep 0.25
+done
+
+log "Waiting for $pid to exit"
+wait $!
+STATUS=$?
+
+test $STATUS -eq 130 || die "process exited with $STATUS expected 130"
+log "Python script exited with status $STATUS"
diff --git a/t/issues/t3415-job-shell-segfault-on-null.sh b/t/issues/t3415-job-shell-segfault-on-null.sh
new file mode 100755
index 000000000000..651b2546ec56
--- /dev/null
+++ b/t/issues/t3415-job-shell-segfault-on-null.sh
@@ -0,0 +1,7 @@
+#!/bin/sh -e
+# submit a job with json-null and ensure it doesn't cause the shell
+#  to segfault
+
+flux job attach -vEX $(flux run --dry-run hostname \
+    | jq .attributes.system.shell.options.foo=null \
+    | flux job submit)
diff --git a/t/issues/t3429-python-future-ref.py b/t/issues/t3429-python-future-ref.py
new file mode 100755
index 000000000000..9bc2e86647b3
--- /dev/null
+++ b/t/issues/t3429-python-future-ref.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+#
+#  Test for issue #3429 - circular future reference occurs when
+#   an exception occurs in a Future continuation_callback. Then
+#   the future is not garbage collected and Python hangs in the
+#   Flux reactor
+#
+import flux
+
+
+def ping_cb(rpc):
+    print(rpc.get_str())
+
+
+h = flux.Flux()
+print("asynchronous: ping kvs.foo")
+h.rpc("kvs.foo", {}).then(ping_cb)
+try:
+    rc = h.reactor_run()
+except OSError as exc:
+    print(f"Got exception: {exc}")
+
+print("Done")
diff --git a/t/issues/t3432-python-sigint.sh b/t/issues/t3432-python-sigint.sh
new file mode 100755
index 000000000000..6c8ff1c2d658
--- /dev/null
+++ b/t/issues/t3432-python-sigint.sh
@@ -0,0 +1,65 @@
+#!/bin/sh
+# reactor_run() is uninterruptible if callback makes synchronous RPC
+
+waitfile=${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua
+
+log() { echo "t3432: $@" >&2; }
+die() { log "$@"; cat t3423.log >&2; exit 1; }
+
+cat <<EOF >reactor-intr.py || die "Failed to create test script"
+import sys
+import flux
+from flux.core.watchers import TimerWatcher
+
+def timeout_cb(handle, watcher, revents, _args):
+    raise OSError("Timed out!")
+
+def startup_cb(handle, watcher, revents, _args):
+    print("sending ping request")
+    h.rpc("kvs.ping", {}).then(ping_cb, do_sync=True)
+
+def ping_cb(rpc, do_sync=False):
+    print("ping_cb")
+    print(rpc.get_str())
+    if do_sync:
+       print("synchronous ping in ping_cb:")
+       print(rpc.get_flux().rpc("kvs.ping", {}).get_str())
+       print("ready.")
+       sys.stdout.flush()
+
+do_sync = len(sys.argv) > 1 and sys.argv[1] == "sync" 
+
+h = flux.Flux()
+
+print("starting timer watchers")
+tw1 = TimerWatcher(h, 0.01, startup_cb)
+tw2 = TimerWatcher(h, 20., timeout_cb)
+
+print("starting reactor_run")
+try:
+    tw1.start()
+    tw2.start()
+    h.reactor_run()
+except KeyboardInterrupt as exc:
+    print("Got KeyboardInterrupt. Exiting...")
+    pass
+EOF
+
+log "Created test script reactor-intr.py"
+
+flux python reactor-intr.py sync >t3432.log 2>&1 &
+pid=$!
+
+log "Started PID=$pid"
+
+$waitfile --timeout=10 --pattern="^ready" t3432.log || die "waitfile failed"
+
+log "Sending SIGINT to $pid"
+kill -INT $pid || die "Failed to kill PID $pid"
+
+log "Waiting for $pid to exit"
+wait $pid
+STATUS=$?
+
+test $STATUS -eq 0 || die "process exited with $STATUS expected 0"
+log "Python script exited with status $STATUS"
diff --git a/t/issues/t3470-multithread-reactor-run.py b/t/issues/t3470-multithread-reactor-run.py
new file mode 100755
index 000000000000..c1fd5b5f2557
--- /dev/null
+++ b/t/issues/t3470-multithread-reactor-run.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+#
+#  Simple test for issue #3470: multiple threads calling reactor_run
+#   causes libev assertion failure.
+#
+from queue import Queue
+import sys
+import threading
+
+import flux
+
+
+def cb(f, watcher, msg, i):
+    print(f"{i}: {msg.payload_str}")
+    watcher.stop()
+
+
+def get_events(i, queue):
+    f = flux.Flux()
+    f.event_subscribe("test-event")
+    queue.put(True)
+    w = f.msg_watcher_create(cb, topic_glob="test-event", args=i)
+    w.start()
+    f.reactor_run()
+    w.destroy()
+
+
+def main():
+    nthreads = 2
+    threads = []
+    queue = Queue()
+    for i in range(0, nthreads):
+        thread = threading.Thread(
+            target=get_events,
+            args=(
+                i,
+                queue,
+            ),
+        )
+        thread.start()
+        threads.append(thread)
+
+    print(f"starting {nthreads} threads", file=sys.stderr)
+
+    # Ensure threads have subscribed to 'test-event'
+    for thread in threads:
+        queue.get()
+        print(f"got response from {thread}", file=sys.stderr)
+
+    print(f"{nthreads} threads started", file=sys.stderr)
+
+    flux.Flux().event_send("test-event", "hello")
+
+    print(f"published test-event", file=sys.stderr)
+
+    for thread in threads:
+        thread.join()
+
+    print("Done", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/t/issues/t3492-kvs-subcommand-help.sh b/t/issues/t3492-kvs-subcommand-help.sh
new file mode 100755
index 000000000000..6b887c8cb450
--- /dev/null
+++ b/t/issues/t3492-kvs-subcommand-help.sh
@@ -0,0 +1,10 @@
+#!/bin/sh -e
+# flux kvs namespace -h
+# and
+# flux kvs eventlog -h
+# don't output appropriate help output
+
+TEST=issue3492
+
+flux kvs namespace -h 2>&1 | grep create && test $? -eq 0
+flux kvs eventlog -h 2>&1 | grep append && test $? -eq 0
diff --git a/t/issues/t3503-nofiles-limit.sh b/t/issues/t3503-nofiles-limit.sh
new file mode 100755
index 000000000000..998dc4eecfa8
--- /dev/null
+++ b/t/issues/t3503-nofiles-limit.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+#  Ensure job-exec doesn't segfault when broker runs over nofile ulimit
+#
+#  Note: the file descriptor limit and number of jobs in the test were
+#   chosen somewhat arbitrarily. The fd limit must be low, but not so
+#   low that `flux start` or other essential services fail to initialize.
+#   The number of jobs should be chosen such that some of them will
+#   definitely cause the broker to run over the artificially lowered
+#   fd limit.
+#
+ulimit -n 115
+ulimit -Hn 115
+flux start \
+    sh -c '
+flux submit --cc=1-12 hostname &&
+flux queue drain &&
+flux jobs -a
+'
diff --git a/t/issues/t3617-flux-shell-depends-zmq.sh b/t/issues/t3617-flux-shell-depends-zmq.sh
new file mode 100755
index 000000000000..68001a1801a3
--- /dev/null
+++ b/t/issues/t3617-flux-shell-depends-zmq.sh
@@ -0,0 +1,9 @@
+#!/bin/sh -e
+# flux-shell does not link to libzmq
+
+FLUX_SHELL="${FLUX_BUILD_DIR}/src/shell/.libs/lt-flux-shell"
+if libtool --mode=execute ldd ${FLUX_SHELL} | grep -q zmq
+then
+    exit 1
+fi
+exit 0
diff --git a/t/issues/t3775-binary-io.sh b/t/issues/t3775-binary-io.sh
new file mode 100755
index 000000000000..cafe3aa1720a
--- /dev/null
+++ b/t/issues/t3775-binary-io.sh
@@ -0,0 +1,9 @@
+#!/bin/sh -e
+# job-shell properly encodes binary data on stdin/out/err
+
+dd if=/dev/urandom bs=1k count=1 > data
+cat data | flux run cat >data2
+cmp data data2
+cat data | flux run sh -c 'cat >&2' 2>data3
+cmp data data3
+exit 0
diff --git a/t/issues/t3906-job-exec-exception.sh b/t/issues/t3906-job-exec-exception.sh
new file mode 100755
index 000000000000..ee0ce254fe2f
--- /dev/null
+++ b/t/issues/t3906-job-exec-exception.sh
@@ -0,0 +1,18 @@
+#!/bin/sh -e
+#
+#  Ensure a job exception is raised when broker rank disconnects
+#
+#  1. Submit job and ensure it has started
+#  2. Kill a leaf broker
+#  3. Ensure job exception is raised and includes appropriate error note
+#
+export startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+SHELL=/bin/sh flux start -s 4 -Stbon.topo=kary:4 --test-exit-mode=leader '\
+   id=$(flux submit -n4 -N4 sleep 300) \
+&& flux job wait-event $id start \
+&& $startctl kill 3 9 \
+&& flux job wait-event $id exception >t3906.output 2>&1'
+
+cat t3906.output
+
+grep 'node failure on .*rank 3' t3906.output
diff --git a/t/issues/t3920-running-underflow.sh b/t/issues/t3920-running-underflow.sh
new file mode 100755
index 000000000000..340dc63a8417
--- /dev/null
+++ b/t/issues/t3920-running-underflow.sh
@@ -0,0 +1,18 @@
+#!/bin/sh -e
+#
+# Event posted in CLEANUP state after an exception in SCHED state
+#  does not cause underflow of the running job count.
+#
+export PLUGIN=${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs/cleanup-event.so
+SHELL=/bin/sh flux start '\
+   flux jobtap load $PLUGIN \
+&& flux queue stop \
+&& jobid=$(flux submit hostname) \
+&& flux job wait-event $jobid depend \
+&& flux cancel $jobid \
+&& flux job attach -vE $jobid \
+; flux queue status -v >t3920.output'
+
+cat t3920.output
+
+grep '0 running jobs' t3920.output
diff --git a/t/issues/t3960-job-exec-ehostunreach.sh b/t/issues/t3960-job-exec-ehostunreach.sh
new file mode 100755
index 000000000000..8f5578e87b8c
--- /dev/null
+++ b/t/issues/t3960-job-exec-ehostunreach.sh
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+#
+#  Trigger an EHOSTUNREACH error when attempting to kill a job.
+#
+#  1. Submit job and ensure it has started
+#  2. Send SIGSTOP to a leaf broker
+#  3. Cancel job and wait for exception to appear in job eventlog
+#  4. The subprocess kill RPC should now be sent to the stopped broker
+#  5. Kill the stopped broker with SIGKILL
+#  6. Wait for job clean event and dump logs
+#  7. Ensure killed rank appears in logs for exec_kill message
+#
+#
+export startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+SHELL=/bin/sh flux start -s 4 -Stbon.topo=kary:4 --test-exit-mode=leader '\
+   id=$(flux submit -n4 -N4 sleep 300) \
+&& flux job wait-event $id start \
+&& $startctl kill 3 19 \
+&& flux cancel $id \
+&& flux job wait-event $id exception \
+&& $startctl kill 3 9 \
+&& flux job attach -vE $id \
+;  flux dmesg >t3960.output 2>&1'
+
+cat t3960.output
+
+grep 'exec_kill.*rank 3' t3960.output
diff --git a/t/issues/t3982-python-message-ref.py b/t/issues/t3982-python-message-ref.py
new file mode 100755
index 000000000000..322c0950f08b
--- /dev/null
+++ b/t/issues/t3982-python-message-ref.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+#
+#  Test for issue #3982 - Ensure python handles message refcount properly
+#
+import sys
+import flux
+from flux.constants import FLUX_MSGTYPE_REQUEST
+
+_SAVED_MESSAGES = []
+
+
+def create_cb(fh, t, msg, timer):
+    _SAVED_MESSAGES.append(msg)
+    print("server: got request, starting timer", file=sys.stderr)
+    timer.start()
+
+
+def timer_cb(fh, *args, **kwargs):
+    try:
+        msg = _SAVED_MESSAGES.pop()
+    except IndexError:
+        pass
+    else:
+        fh.respond(msg, {"success": True})
+        print("server: responded to message", file=sys.stderr)
+
+
+def rpc_cb(future):
+    if future.get()["success"]:
+        print("client: Success", file=sys.stderr)
+        future.get_flux().reactor_stop()
+    else:
+        future.get_flux().reactor_stop_error()
+
+
+fh = flux.Flux()
+fh.service_register("t3982").get()
+
+timer = fh.timer_watcher_create(0.01, timer_cb)
+
+w = fh.msg_watcher_create(create_cb, FLUX_MSGTYPE_REQUEST, "t3982.test", timer)
+w.start()
+
+fh.rpc("t3982.test", {}).then(rpc_cb)
+print("client: Sent request", file=sys.stderr)
+
+fh.reactor_run()
diff --git a/t/issues/t4182-resource-rerank.sh b/t/issues/t4182-resource-rerank.sh
new file mode 100755
index 000000000000..43e8234138bb
--- /dev/null
+++ b/t/issues/t4182-resource-rerank.sh
@@ -0,0 +1,70 @@
+#!/bin/sh -e
+#
+#  Ensure resource module resets ranks of R to match hostlist attribute
+#
+
+#  Do what flux start --test-hosts does, but using FLUX_TASK_RANK
+#   to find fake hostname in provided hostlist
+cat <<EOF2 >t4182-broker-wrapper.sh
+#!/bin/bash
+# Usage: t4182-broker-wrapper.sh hostlist ARGS ...
+
+hostlist=\$1; shift
+FLUX_FAKE_HOSTNAME=\$(flux hostlist --nth=\$FLUX_TASK_RANK "\$hostlist")
+flux broker -Shostlist="\$hostlist" "\$@"
+EOF2
+chmod +x t4182-broker-wrapper.sh
+
+cat <<EOF >t4182-test.sh
+#!/bin/bash -e
+
+R="\$((flux R encode -r 0 -c 0 -H foo0 && \
+     flux R encode -r 1 -c 0-1 -H foo1 && \
+     flux R encode -r 2-3 -c 0-3 -H 'foo[2-3]' \
+    ) | flux R append)"
+
+flux kvs put resource.R="\$R"
+flux kvs get resource.R
+
+flux module remove sched-simple
+flux module reload resource monitor-force-up noverify
+flux module load sched-simple
+
+flux dmesg | grep 'sched-simple.*ready'  | tail -1
+
+flux resource list
+
+#  Disable rlist/hwloc resource verification
+#    --test-hosts set FLUX_FAKE_HOSTNAME but hwloc doesn't know about that
+echo "resource.noverify = true" >t4182-resource.toml
+
+#  ensure R rerank failure is ignored (i.e. job completes successfully)
+flux run -o per-resource.type=node -o cpu-affinity=off -n 11 \
+	flux start --config-path=t4182-resource.toml \
+	flux getattr hostlist
+
+#  ensure R is reranked based on hostlist attribute:
+flux run -o per-resource.type=node -o cpu-affinity=off -n 11 \
+	./t4182-broker-wrapper.sh "foo[3,2,1,0]" \
+	--config-path=t4182-resource.toml \
+	sh -c 'flux kvs get resource.R' >t4182-test.out
+
+EOF
+
+#  Use --test-hosts instead of setting hostlist attr so that overlay
+#   hello check for expected hostname:rank won't fail and abort the test
+flux start --test-hosts="foo[0-3]" --test-size=4 bash ./t4182-test.sh
+
+jq -S . <t4182-test.out
+
+NODELIST="$(jq -r .execution.nodelist[0] <t4182-test.out)"
+EXPECTED="foo[3,2,1,0]"
+
+if test "$NODELIST" = "$EXPECTED"; then
+	exit 0
+else
+	echo >&2 "Got $NODELIST, expected $EXPECTED"
+	exit 1
+fi
+
+
diff --git a/t/issues/t4184-sched-simple-restart.sh b/t/issues/t4184-sched-simple-restart.sh
new file mode 100755
index 000000000000..58192b96cb25
--- /dev/null
+++ b/t/issues/t4184-sched-simple-restart.sh
@@ -0,0 +1,46 @@
+#!/bin/bash -e
+
+cat <<-EOF >t4184.sh
+#!/bin/sh -e
+
+which flux
+
+NCORES=\$(flux resource list -s up -no {ncores})
+jobids=\$(flux submit --cc=0-1 -n \$NCORES sleep 600)
+id=\$(echo \$jobids | cut -d ' ' -f1)
+id2=\$(echo \$jobids | cut -d ' ' -f2)
+flux job wait-event \$id start
+
+flux resource drain 0
+
+flux module reload sched-simple
+
+flux resource list
+flux resource undrain 0
+
+echo "t4184: waiting for resources to be back up:"
+while test \$(flux resource list -s up -no {ncores}) -ne \$NCORES; do
+  sleep 0.25
+done
+
+echo "t4184: canceling \$id"
+flux cancel \${id}
+
+echo "t4184: waiting for \$id to end"
+flux job wait-event \$id clean
+
+echo "t4184: waiting for \$id2 to start..."
+
+flux job wait-event -t 100 \$id2 start
+
+echo "t4184: canceling \$id2..."
+flux cancel \$id2
+
+echo "t4184: waiting for \$id2 to end..."
+flux job wait-event -t 100 \$id2 clean
+
+EOF
+
+chmod +x t4184.sh
+
+flux start -s 1 ./t4184.sh
diff --git a/t/issues/t4222-kvs-assume-empty-dir.sh b/t/issues/t4222-kvs-assume-empty-dir.sh
new file mode 100755
index 000000000000..8fc0767ae476
--- /dev/null
+++ b/t/issues/t4222-kvs-assume-empty-dir.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -e
+
+flux module remove kvs-watch
+flux module remove kvs
+
+flux dump --checkpoint issue4222.tar
+
+flux content flush
+flux content dropcache
+flux module remove content-sqlite
+
+sqlitepath=$(flux getattr rundir)/content.sqlite
+mv $sqlitepath $sqlitepath.bak
+
+flux module load content-sqlite
+
+flux restore --checkpoint issue4222.tar
+
+flux module load kvs
+flux module load kvs-watch
+
+flux kvs namespace create issue4222ns
+flux kvs put --namespace=issue4222ns a=1
diff --git a/t/issues/t4227-test.sh b/t/issues/t4227-test.sh
new file mode 100755
index 000000000000..b5cc66643d66
--- /dev/null
+++ b/t/issues/t4227-test.sh
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+flux run -vvv hostname
+flux submit -vvv --wait hostname
diff --git a/t/issues/t4375-content-flush-hang.sh b/t/issues/t4375-content-flush-hang.sh
new file mode 100755
index 000000000000..4ef1c127510b
--- /dev/null
+++ b/t/issues/t4375-content-flush-hang.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+flux content flush
+
+flux module remove content-sqlite
+
+# we need to have a dirty entry in the content-cache for content-flush
+# to mean anything.
+flux kvs put issue4375=issue4375
+
+! flux content flush
+
+flux module load content-sqlite
diff --git a/t/issues/t4378-content-flush-force.sh b/t/issues/t4378-content-flush-force.sh
new file mode 100755
index 000000000000..264494147d60
--- /dev/null
+++ b/t/issues/t4378-content-flush-force.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+flux content flush
+
+flux module remove content-sqlite
+
+# create a dirty cache entry in the content-cache
+flux kvs put issue4378=issue4378bug
+
+flux module load content-sqlite
+
+# Issue 4378 - without fix, this flux content flush would hang
+flux content flush
diff --git a/t/issues/t4379-dirty-cache-entries-flush.sh b/t/issues/t4379-dirty-cache-entries-flush.sh
new file mode 100755
index 000000000000..154f52976d67
--- /dev/null
+++ b/t/issues/t4379-dirty-cache-entries-flush.sh
@@ -0,0 +1,40 @@
+#!/bin/bash -e
+
+flux content flush
+
+count=`flux module stats --parse dirty content`
+if [ ${count} -ne 0 ]
+then
+    echo "dirty entries not 0 at start of test"
+    return 1
+fi
+
+flux module remove content-sqlite
+
+# create a dirty cache entry in the content-cache
+flux kvs put issue4379=issue4379bug
+
+count=`flux module stats --parse dirty content`
+if [ ${count} -ne 1 ]
+then
+    echo "dirty entries not 1 after a put"
+    return 1
+fi
+
+flux module load content-sqlite
+
+# Issue 4379 - this dirty count would stay at 1
+# Check count in a loop, although highly improbable, technically could
+# be racy
+count=`flux module stats --parse dirty content`
+i=0
+while [ ${count} -ne 0 ] && [ $i -lt 50 ]
+do
+    sleep 0.1
+    i=$((i + 1))
+done
+if [ ${count} -ne 0 ]
+then
+    echo "dirty entries not 0"
+    return 1
+fi
diff --git a/t/issues/t4413-empty-eventlog.sh b/t/issues/t4413-empty-eventlog.sh
new file mode 100755
index 000000000000..09681b0aa1d1
--- /dev/null
+++ b/t/issues/t4413-empty-eventlog.sh
@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+# create a fake job eventlog in the KVS, no normal mechanism will
+# create an empty eventlog
+
+jobpath=`flux job id --to=kvs 123456789`
+flux kvs put "${jobpath}.eventlog"=""
+
+# Issue 4413, previously would return Cannot allocate memory
+flux job info 123456789 eventlog 2>&1 | grep "error parsing eventlog"
+
+
diff --git a/t/issues/t4465-job-list-use-after-free.sh b/t/issues/t4465-job-list-use-after-free.sh
new file mode 100755
index 000000000000..ed910507c830
--- /dev/null
+++ b/t/issues/t4465-job-list-use-after-free.sh
@@ -0,0 +1,36 @@
+#!/bin/sh -e
+
+prog=$(basename $0)
+
+if test -z "$FLUX_ENABLE_VALGRIND_TEST"; then
+    echo "skipping ${prog}: FLUX_ENABLE_VALGRIND_TEST is not set" >&2
+    exit 0
+fi
+VALGRIND=`which valgrind`
+if test -z "${VALGRIND}"; then
+    echo "skipping ${prog}: valgrind executable not found" >&2
+    exit 0
+fi
+VALGRIND_SUPPRESSIONS=${SHARNESS_TEST_SRCDIR}/valgrind/valgrind.supp
+
+STATEDIR=issue4470-statedir
+
+rm -rf ${STATEDIR}
+mkdir ${STATEDIR}
+
+# This test reproduces the failure more often when jobs complete in
+# a different order than submitted, hence number of jobs submitted equals
+# number of broker ranks.
+#
+flux start --test-size=8 -Sstatedir=${STATEDIR} \
+    bash -c "flux submit --cc 1-8 --quiet true && flux queue drain"
+
+flux start --test-size=1 -Sstatedir=${STATEDIR} \
+    --wrap=libtool,e,${VALGRIND} \
+    --wrap=--tool=memcheck \
+    --wrap=--trace-children=no \
+    --wrap=--child-silent-after-fork=yes \
+    --wrap=--num-callers=30 \
+    --wrap=--error-exitcode=1 \
+    --wrap=--suppressions=$VALGRIND_SUPPRESSIONS \
+    bash -c "flux job purge --force --num-limit=4 && flux jobs -a 2>/dev/null"
diff --git a/t/issues/t4482-flush-list-corruption.sh b/t/issues/t4482-flush-list-corruption.sh
new file mode 100755
index 000000000000..2f2da615a86a
--- /dev/null
+++ b/t/issues/t4482-flush-list-corruption.sh
@@ -0,0 +1,43 @@
+#!/bin/sh -e
+
+# How this test works
+#
+# add some unique data to the content cache, we do several stores to
+# build up a decent length internal list of flushable cache entries.
+#
+# write some of the same data again, if error present internal flush
+# list will be messed up and length of flush list < number of dirty
+# entries (acct_dirty).
+#
+# before fix, flux content flush will hang b/c number of dirty entries
+# (acct_dirty) never reaches zero.
+
+cat <<-EOF >t4482.sh
+#!/bin/sh -e
+
+flux module load content
+
+echo "abcde" | flux content store
+echo "fghij" | flux content store
+echo "klmno" | flux content store
+echo "pqrst" | flux content store
+echo "tuvwx" | flux content store
+
+echo "fghij" | flux content store
+echo "klmno" | flux content store
+
+flux module load content-sqlite
+
+flux content flush
+
+flux module remove content-sqlite
+flux module remove content
+
+EOF
+
+chmod +x t4482.sh
+
+flux start -s 1 \
+    --setattr=broker.rc1_path= \
+    --setattr=broker.rc3_path= \
+    ./t4482.sh
diff --git a/t/issues/t4583-free-range-test.sh b/t/issues/t4583-free-range-test.sh
new file mode 100755
index 000000000000..4ec7e5e8ee56
--- /dev/null
+++ b/t/issues/t4583-free-range-test.sh
@@ -0,0 +1,80 @@
+#!/bin/bash -e
+#
+#  Test that batch/alloc jobs are resilient to leaf not failures
+#
+
+log ()
+{
+    printf "free-range-test: $@"
+}
+
+list_descendants ()
+{
+    local children=$(ps -o pid= --ppid "$1")
+
+    for pid in $children; do
+        list_descendants "$pid"
+    done
+
+    echo "$children"
+}
+
+
+#  Check if we need to start parent job, if so, reexec under flux-start
+#  using the per-broker for test-pmi-clique so that the instance mapping
+#  simulates multiple nodes.
+#
+export FLUX_URI_RESOLVE_LOCAL=t
+if test "$FREE_RANGE_TEST_ACTIVE" != "t"; then
+    export FREE_RANGE_TEST_ACTIVE=t
+    log "Re-launching test script under flux-start\n"
+    exec flux start -s 4 \
+        --test-exit-mode=leader \
+        --test-pmi-clique=per-broker \
+        -Stbon.topo=kary:0 $0
+fi
+
+#  Start a job with tbon.topo=kary:0
+log "Starting a child instance with flat topology\n"
+jobid=$(flux alloc -N4 -o exit-timeout=none --bg --broker-opts=-Stbon.topo=kary:0)
+
+log "Started job $jobid\n"
+
+#  Run a job on all ranks of child job
+log "Current overlay status of $jobid:\n"
+flux proxy $jobid flux overlay status
+
+log "Launch a sleep job within $jobid:\n"
+flux proxy $jobid flux submit -N4 sleep inf
+
+flux pstree -x --skip-root=no
+
+#  Now simulate a node failure by killing a broker in parent
+#  instance, along with all its children
+broker_pid=$(flux exec -r 3 flux getattr broker.pid)
+
+log "Killing rank 3 (pid %d) and all children\n" $broker_pid
+kill -9 $(list_descendants $broker_pid) $broker_pid
+
+log "Wait for exception event in $jobid\n"
+flux job wait-event -t 100 $jobid exception
+
+log "But running a 3 node job in $jobid still works:\n"
+flux proxy $jobid flux run -t 100s -N3 hostname
+
+log "Overlay status of $jobid should show rank lost:\n"
+flux proxy $jobid flux overlay status
+
+log "Call flux shutdown on $jobid\n"
+flux shutdown --quiet $jobid
+
+log "job $jobid should exit cleanly (no hang) and a zero exit code:\n"
+flux job wait-event -t 100 $jobid finish
+
+log "dump output from job:\n\n"
+flux job attach $jobid
+rc=$?
+printf "\n"
+log "flux-job attach exited with code=$rc\n"
+
+exit $rc
diff --git a/t/issues/t4612-eventlog-overwrite-crash.sh b/t/issues/t4612-eventlog-overwrite-crash.sh
new file mode 100755
index 000000000000..5502a5bda86a
--- /dev/null
+++ b/t/issues/t4612-eventlog-overwrite-crash.sh
@@ -0,0 +1,28 @@
+#!/bin/bash -e
+
+waitfile=${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua
+
+echo pid=$pid
+flux resource list
+flux jobs -a
+
+jobid=$(flux submit --wait-event=start sh -c "echo foo; sleep 300")
+
+kvsdir=$(flux job id --to=kvs $jobid)
+
+# issue a command that will watch / monitor the job's eventlog
+flux job attach $jobid > t4612.out &
+
+# ensure backgrounded process has started to monitor eventlog
+$waitfile --count=1 --timeout=100 --pattern=foo t4612.out
+
+# now overwrite the eventlog without changing its length
+flux kvs get --raw ${kvsdir}.eventlog \
+	| sed -e s/submit/foobar/ \
+	| flux kvs put --raw ${kvsdir}.eventlog=-
+
+wait
+
+# if flux broker segfaulted, this won't work
+flux cancel $jobid
+flux job status $jobid || true
diff --git a/t/issues/t4711-job-list-purge-inactive.sh b/t/issues/t4711-job-list-purge-inactive.sh
new file mode 100755
index 000000000000..a005041dd08b
--- /dev/null
+++ b/t/issues/t4711-job-list-purge-inactive.sh
@@ -0,0 +1,24 @@
+#!/bin/bash -e
+
+prejob=$(flux job stats | jq .job_states.run)
+
+jobid=$(flux submit --wait-event=start sleep 100 | flux job id)
+
+postjob=$(flux job stats | jq .job_states.run)
+
+if [ ${postjob} -ne $((prejob + 1)) ]
+then
+    echo "stat counts invalid: ${postjob}, ${prejob}"
+    exit 1
+fi
+
+flux event pub job-purge-inactive "{\"jobs\":[${jobid}]}"
+
+# if job-list module asserted this won't work
+
+# this is technically a bit racy, as we won't know for sure if the
+# event has been received / processed by the job-list module yet.  In
+# the rare event the racy bit is hit and our fix regresses, subsequent
+# flux actions should catch a failure.
+
+flux job stats
diff --git a/t/issues/t4852-t_submit-legacy.sh b/t/issues/t4852-t_submit-legacy.sh
new file mode 100755
index 000000000000..f66460923063
--- /dev/null
+++ b/t/issues/t4852-t_submit-legacy.sh
@@ -0,0 +1,49 @@
+#!/bin/bash -e
+
+# ensure t_submit/t_depend is available from job-list in older
+# versions of flux that did not have the validate event.  To test, we
+# remove the validate in an existing job's eventlog.
+
+cat <<-EOF >t4852setup.sh
+#!/bin/sh -e
+
+jobid=\$(flux submit --wait true)
+
+kvspath=\$(flux job id --to=kvs \$jobid)
+
+flux kvs get \$kvspath.eventlog | grep -v validate > job4852.log
+
+# need to remove version field in submit event for legacy eventlog format
+# lazily "remove" it by just renaming it
+head -n 1 job4852.log | sed -e "s/version/foobar/" > job4852.log2
+tail -n +2 job4852.log >> job4852.log2
+
+# head -n -1 to remove trailing newline
+cat job4852.log2 | head -n -1 | flux kvs put -r \${kvspath}.eventlog=-
+
+EOF
+
+cat <<-EOF >t4852test.sh
+#!/bin/sh -e
+
+#assuming only one job in this test
+flux jobs -a -no {id} > job4852.id
+flux job list-ids \$(cat job4852.id) > job4852.out
+grep t_submit job4852.out
+grep t_depend job4852.out
+cat job4852.out | jq -e ".t_submit == .t_depend"
+EOF
+
+chmod +x t4852setup.sh
+chmod +x t4852test.sh
+
+STATEDIR=issue4852-statedir
+mkdir issue4852-statedir
+
+flux start -s 1 \
+    --setattr=statedir=${STATEDIR} \
+    ./t4852setup.sh
+
+flux start -s 1 \
+    --setattr=statedir=${STATEDIR} \
+    ./t4852test.sh
diff --git a/t/issues/t5105-signal-propagation.sh b/t/issues/t5105-signal-propagation.sh
new file mode 100755
index 000000000000..ec68452b969e
--- /dev/null
+++ b/t/issues/t5105-signal-propagation.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+# Run an 3 level nested instance, with a test job at the final level
+
+waitfile=$SHARNESS_TEST_SRCDIR/scripts/waitfile.lua
+
+cat <<EOF >test.py
+import signal
+import time
+import flux
+import os
+
+h = flux.Flux()
+level = h.attr_get("instance-level")
+jobid = os.getenv("FLUX_JOB_ID")
+signal.signal(
+    signal.SIGUSR1,
+    lambda x, y: print(f"job {jobid} in level {level} got SIGUSR1", flush=True),
+)
+open("ready", 'a').close()
+signal.pause()
+EOF
+
+id=$(flux submit --output=log flux start \
+     flux run flux start \
+     flux run flux python ./test.py)
+
+$waitfile -t 100 -v ready
+
+flux job kill -s SIGUSR1 $id
+
+$waitfile -t 100 -v -p "got SIGUSR1" log
+
+flux job status --json -v $id
+
+# vi: ts=4 sw=4 expandtab
+
diff --git a/t/issues/t5308-kvsdir-initial-path.py b/t/issues/t5308-kvsdir-initial-path.py
new file mode 100755
index 000000000000..3b5a139e39aa
--- /dev/null
+++ b/t/issues/t5308-kvsdir-initial-path.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+#
+#  Modifying a job data via object (KVSDir) returned via job_kvs()
+#    works.
+#
+import sys
+import flux
+import flux.job
+from flux.job import JobspecV1
+
+handle = flux.Flux()
+jobspec = JobspecV1.from_command(command=["true"], num_tasks=1, num_nodes=1)
+jobid = flux.job.submit(handle, jobspec, waitable=True)
+flux.job.wait(handle, jobid=jobid)
+jobdir = flux.job.job_kvs(handle, jobid)
+jobdir["foo"] = "bar"
+jobdir.commit()
+
+jobdir2 = flux.job.job_kvs(handle, jobid)
+if jobdir2["foo"] != "bar":
+    sys.exit(1)
diff --git a/t/issues/t5368-kvs-commit-clear.py b/t/issues/t5368-kvs-commit-clear.py
new file mode 100755
index 000000000000..b4c672c31ce1
--- /dev/null
+++ b/t/issues/t5368-kvs-commit-clear.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+#
+#  Internal KVS txn cleared between commits.
+#
+import sys
+import flux
+import flux.kvs
+import os
+import errno
+
+handle = flux.Flux()
+
+flux.kvs.put(handle, "issue5368A", 1)
+flux.kvs.commit(handle)
+val = flux.kvs.get(handle, "issue5368A")
+if val != 1:
+    print("could not write issue5368A")
+    sys.exit(1)
+
+# delete key outside of this test's flux handle
+os.system("flux kvs unlink issue5368A")
+
+flux.kvs.put(handle, "issue5368B", 1)
+flux.kvs.commit(handle)
+
+# If the internal KVS transaction is not cleared, then the key
+# "issue5368A" will be rewritten.  We want ENOENT to occur.
+try:
+    val = flux.kvs.get(handle, "issue5368A")
+    print("key issue5368A retrieved successfully")
+except OSError as e:
+    if e.errno == errno.ENOENT:
+        sys.exit(0)
+    print("unexpected errno " + e.errno)
+
+sys.exit(1)
diff --git a/t/issues/t5657-kvs-getroot-namespace.sh b/t/issues/t5657-kvs-getroot-namespace.sh
new file mode 100755
index 000000000000..4bbfc3d13ff5
--- /dev/null
+++ b/t/issues/t5657-kvs-getroot-namespace.sh
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+
+# internal KVS getroot request does not cache namespace with correct name
+
+TEST=issue5657
+
+cat <<-EOF >t5657test.sh
+#!/bin/sh -e
+
+flux kvs namespace create issue5657ns
+
+flux kvs put --namespace=issue5657ns a=1
+flux kvs link -T issue5657ns a link2a
+
+# double check getting link2a treeobj on rank 1 works
+flux exec -r 1 flux kvs get --treeobj link2a
+
+# before fix, next line would hang because symlink would not cache correct
+# namespace name, leading to internal infinite loop
+
+flux exec -r 1 flux kvs get link2a
+
+EOF
+
+chmod +x t5657test.sh
+
+flux start -s 2 ./t5657test.sh
diff --git a/t/issues/t5892-shutdown-no-epilog.sh b/t/issues/t5892-shutdown-no-epilog.sh
new file mode 100755
index 000000000000..4d718a58842d
--- /dev/null
+++ b/t/issues/t5892-shutdown-no-epilog.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# ensure epilog doesn't run on jobs canceled at shutdown
+
+cat <<-EOF >t5892.toml
+[job-manager]
+plugins = [
+  { load = "perilog.so" },
+]
+
+[job-manager.epilog]
+command = [ "touch", "t5892-epilog-flag" ]
+EOF
+
+flux start --config-path=t5892.toml \
+	flux submit sleep inf
+
+rc=0
+if test -f t5892-epilog-flag; then
+	echo The epilog did run, contrary to expectations.>&2
+	rc=1
+else
+	echo The epilog did not run, as expected. >&2
+fi
+
+rm -f t5892-epilog-flag
+rm -f t5892.toml
+
+exit $rc
diff --git a/t/job-exec/dummy.sh b/t/job-exec/dummy.sh
index b26219583558..87a44d09a051 100755
--- a/t/job-exec/dummy.sh
+++ b/t/job-exec/dummy.sh
@@ -28,7 +28,8 @@
 #  drive tests of expected execution behavior.
 #
 
-die() { echo "dummy-shell: $@" >&2; exit 1; }
+die() { echo "dummy-shell[$BROKER_RANK]: $@" >&2; exit 1; }
+log() { echo "dummy-shell[$BROKER_RANK]: $@" >&2; }
 
 #
 #  Declare globals:
@@ -46,7 +47,7 @@ declare -r JOBID=$1
 #  Fetch read-only job information and verify all values found:
 #
 declare -r BROKER_RANK=$(flux getattr rank)
-declare -r JOBSPEC=$(flux job info $JOBID jobspec)
+declare -r JOBSPEC=$(flux job info --original $JOBID jobspec)
 declare -r R=$(flux job info $JOBID R)
 
 for var in FLUX_KVS_NAMESPACE BROKER_RANK JOBSPEC R; do
@@ -74,9 +75,9 @@ get_job_shell_rank() {
     get_ranklist
     for r in "${RANKLIST[@]}"; do
         if [[ $r == $BROKER_RANK ]]; then
-            JOB_SHELL_RANK=$r
+            JOB_SHELL_RANK=$i
             return 0
-	fi
+        fi
         ((i++))
     done
     die "My rank: $BROKER_RANK, not found in ranklist!"
@@ -93,14 +94,67 @@ get_duration() {
     if test "$DURATION" = "null"; then DURATION=0.01; fi
 }
 
+get_cwd() {
+    CWD=$(json_get "$JOBSPEC" .attributes.system.cwd)
+    if test "$CWD" = "null"; then CWD=.; fi
+}
+
 get_command() {
     COMMAND=($(json_get "$JOBSPEC" '.tasks[0].command[]' | tr '\n' ' '))
 }
 
 get_traps() {
+    trap "log got SIGTERM" 15
     TRAP=$(json_get "$JOBSPEC" '.attributes.system.environment.TRAP')
     if test "x$TRAP" != "xnull" ; then
-        trap "/bin/echo got signal $TRAP" $TRAP
+        trap "log got signal $TRAP" $TRAP
+    fi
+}
+
+test_mock_failure() {
+    FAIL_MODE=$(json_get "$JOBSPEC" '.attributes.system.environment.FAIL_MODE')
+    case "$FAIL_MODE" in
+        before_barrier_entry)
+            #
+            #  Attempt to exit early from shell rank 1 *before* rank 0
+            #   enters the shell barrier. This is best effort since there
+            #   is not a good way to ensure a separate shell process has
+            #   reached the barrier.
+            #
+            log "Got FAIL_MODE=$FAIL_MODE"
+            if test $JOB_SHELL_RANK -eq 0; then
+                sleep 1
+            elif test $JOB_SHELL_RANK -eq 1; then
+                log "before_barrier: exiting early on job shell rank 1"
+                exit 1
+            fi
+            ;;
+        after_barrier_entry)
+            #
+            #  Similer to above, but try to ensure that rank 0 shell has
+            #   entered the barrier before rank 1 unexpectedly exits.
+            #   See caveats about best effort above.
+            #
+            log "after_barrier_entry: rank=$JOB_SHELL_RANK"
+            if test $JOB_SHELL_RANK -eq 1; then
+                log "after_barrier: exiting early on job shell rank 1"
+                sleep 1
+                log "exiting"
+                exit 1
+            fi
+            ;;
+        *)
+            ;;
+    esac
+}
+
+barrier() {
+    if [[ ${NNODES} -gt 1 ]]; then
+        echo enter >&${FLUX_EXEC_PROTOCOL_FD:-1}
+        read -u ${FLUX_EXEC_PROTOCOL_FD:-0} line
+        if [[ ${line##*=} -ne 0 ]]; then
+            exit ${line##*=}
+        fi
     fi
 }
 
@@ -109,12 +163,22 @@ get_traps() {
 #
 get_job_shell_rank
 get_nnodes
+barrier
 get_duration
 get_command
 get_traps
+get_cwd
+
+cd $CWD
+
+test_mock_failure
+barrier
 
 #
 #  Run specified COMMAND:
 #
-echo Running  "${COMMAND[@]}"
+log Running  "${COMMAND[@]}"
+flux kvs eventlog append exec.eventlog shell.start
 eval "${COMMAND[@]}"
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/job-exec/imp-fail.sh b/t/job-exec/imp-fail.sh
new file mode 100755
index 000000000000..c29dcbdfef57
--- /dev/null
+++ b/t/job-exec/imp-fail.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+#  Mock flux-imp that fails on broker rank 1
+#
+cmd=$1
+
+# Test requirement, make sure we change back to sharness trash directory,
+#  multiuser jobs do not cause IMP to chdir to cwd of job:
+cd $SHARNESS_TRASH_DIRECTORY
+
+case "$cmd" in
+    exec)
+        shift;
+        printf "test-imp: Going to fail on rank 1\n" >&2
+        if test $(flux getattr rank) = 1; then exit 0; fi
+        exec "$@" ;;
+    *)
+        printf "test-imp: Fatal: Unknown cmd=$cmd\n" >&2; exit 1 ;;
+esac
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/job-exec/imp.sh b/t/job-exec/imp.sh
new file mode 100755
index 000000000000..d361b5af98d3
--- /dev/null
+++ b/t/job-exec/imp.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+cmd=$1
+
+# Test requirement, make sure we change back to sharness trash directory,
+#  multiuser jobs do not cause IMP to chdir to cwd of job:
+cd $SHARNESS_TRASH_DIRECTORY
+
+case "$cmd" in
+    exec)
+        shift; 
+        #  Copy IMP's input to a file so tests can check result:
+        ${FLUX_IMP_EXEC_HELPER:-cat} >imp-$(flux job id $2).input
+        printf "test-imp: Running $* \n" >&2
+        exec "$@" ;;
+    kill)
+        signal=$2;
+        pid=$3;
+        shift 3;
+        printf "test-imp: Kill pid $pid signal $signal\n" >&2
+        ps -fp $pid >&2
+        kill -$signal -$pid ;;
+    *)
+        printf "test-imp: Fatal: Unknown cmd=$cmd\n" >&2; exit 1 ;;
+esac
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/job-info/info_lookup.c b/t/job-info/info_lookup.c
new file mode 100644
index 000000000000..1d3ff0163951
--- /dev/null
+++ b/t/job-info/info_lookup.c
@@ -0,0 +1,128 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <jansson.h>
+#include <stdio.h>
+#include <getopt.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/log.h"
+#include "ccan/str/str.h"
+
+#define OPTIONS "jc"
+static const struct option longopts[] = {
+    {"json-decode", no_argument, 0, 'j'},
+    {"current", no_argument, 0, 'c'},
+    { 0, 0, 0, 0 },
+};
+
+static void usage (void)
+{
+    fprintf (stderr, "Usage: info_lookup [--json-decode] "
+                     "[--current] <jobid> <key> ...\n");
+    exit (1);
+}
+
+int main (int argc, char *argv[])
+{
+    flux_t *h;
+    flux_future_t *f;
+    flux_jobid_t id;
+    json_t *keys;
+    int i, ch;
+    int flags = 0;
+
+    while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
+        switch (ch) {
+        case 'j': /* --json-decode */
+            flags |= FLUX_JOB_LOOKUP_JSON_DECODE;
+            break;
+        case 'c': /* --current */
+            flags |= FLUX_JOB_LOOKUP_CURRENT;
+            break;
+        default:
+            usage ();
+            break;
+        }
+    }
+    if ((argc - optind) < 2)
+        usage ();
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (flux_job_id_parse (argv[optind++], &id) < 0)
+        log_msg_exit ("error parsing jobid: %s", argv[1]);
+
+    if (!(keys = json_array ()))
+        log_msg_exit ("json_array");
+
+    for (i = optind; i < argc; i++) {
+        json_t *key = json_string (argv[i]);
+        if (!key)
+            log_msg_exit ("json_string_value");
+        if (json_array_append_new (keys, key) < 0)
+            log_msg_exit ("json_array_append_new");
+    }
+
+    if (!(f = flux_rpc_pack (h,
+                             "job-info.lookup",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:I s:O s:i}",
+                             "id", id,
+                             "keys", keys,
+                             "flags", flags)))
+        log_err_exit ("flux_rpc_pack");
+
+    for (i = optind; i < argc; i++) {
+        if (flags & FLUX_JOB_LOOKUP_JSON_DECODE) {
+            json_t *value;
+            char *s;
+            if (flux_rpc_get_unpack (f, "{s:o}", argv[i], &value) < 0)
+                log_msg_exit ("job-info.lookup: %s",
+                              future_strerror (f, errno));
+            if (streq (argv[i], "jobspec") || streq (argv[i], "R")) {
+                if (!json_is_object (value))
+                    log_msg_exit ("job-info.lookup: key %s not an object",
+                                  argv[i]);
+            }
+            else {
+                if (!json_is_string (value))
+                    log_msg_exit ("job-info.lookup: key %s not a string",
+                                  argv[i]);
+            }
+            if (!(s = json_dumps (value, JSON_ENCODE_ANY)))
+                log_msg_exit ("invalid json result");
+            printf ("%s\n", s);
+            free (s);
+        }
+        else {
+            const char *s;
+            if (flux_rpc_get_unpack (f, "{s:s}", argv[i], &s) < 0)
+                log_msg_exit ("job-info.lookup: %s",
+                              future_strerror (f, errno));
+            printf ("%s\n", s);
+        }
+        fflush (stdout);
+    }
+
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/job-info/update_lookup.c b/t/job-info/update_lookup.c
new file mode 100644
index 000000000000..6417dbeedd4c
--- /dev/null
+++ b/t/job-info/update_lookup.c
@@ -0,0 +1,73 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Note that the job-info.update-lookup RPC target is deprecated.
+ * This is to test legacy behavior.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <jansson.h>
+#include <stdio.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/log.h"
+
+int main (int argc, char *argv[])
+{
+    flux_t *h;
+    flux_future_t *f;
+    flux_jobid_t id;
+    const char *key;
+    json_t *value;
+    char *s;
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (argc != 3) {
+        fprintf (stderr, "Usage: update_lookup <jobid> <key>\n");
+        exit (1);
+    }
+
+    if (flux_job_id_parse (argv[1], &id) < 0)
+        log_msg_exit ("error parsing jobid: %s", argv[1]);
+
+    key = argv[2];
+
+    if (!(f = flux_rpc_pack (h,
+                             "job-info.update-lookup",
+                             FLUX_NODEID_ANY,
+                             0,
+                             "{s:I s:s s:i}",
+                             "id", id,
+                             "key", key,
+                             "flags", 0)))
+        log_err_exit ("flux_rpc_pack");
+
+    if (flux_rpc_get_unpack (f, "{s:o}", key, &value) < 0)
+        log_msg_exit ("job-info.update-lookup: %s",
+                      future_strerror (f, errno));
+    if (!(s = json_dumps (value, 0)))
+        log_msg_exit ("invalid json result");
+    printf ("%s\n", s);
+    fflush (stdout);
+    free (s);
+
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/job-info/update_watch_stream.c b/t/job-info/update_watch_stream.c
new file mode 100644
index 000000000000..19674ccf2986
--- /dev/null
+++ b/t/job-info/update_watch_stream.c
@@ -0,0 +1,92 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <jansson.h>
+#include <signal.h>
+#include <stdio.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/log.h"
+
+flux_t *h;
+flux_future_t *f;
+
+void cancel_cb (int sig)
+{
+    flux_future_t *f2;
+    if (!(f2 = flux_rpc_pack (h,
+                              "job-info.update-watch-cancel",
+                              FLUX_NODEID_ANY,
+                              FLUX_RPC_NORESPONSE,
+                              "{s:i}",
+                              "matchtag", (int)flux_rpc_get_matchtag (f))))
+        log_err_exit ("flux_rpc_pack");
+    flux_future_destroy (f2);
+}
+
+int main (int argc, char *argv[])
+{
+    flux_jobid_t id;
+    const char *key;
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (argc != 3) {
+        fprintf (stderr, "Usage: update_watch_stream <jobid> <key>\n");
+        exit (1);
+    }
+
+    if (flux_job_id_parse (argv[1], &id) < 0)
+        log_msg_exit ("error parsing jobid: %s", argv[1]);
+
+    key = argv[2];
+
+    if (!(f = flux_rpc_pack (h,
+                             "job-info.update-watch",
+                             FLUX_NODEID_ANY,
+                             FLUX_RPC_STREAMING,
+                             "{s:I s:s s:i}",
+                             "id", id,
+                             "key", key,
+                             "flags", 0)))
+        log_err_exit ("flux_rpc_pack");
+
+    if (signal (SIGUSR1, cancel_cb) == SIG_ERR)
+        log_err_exit ("signal");
+
+    while (1) {
+        json_t *value;
+        char *s;
+        if (flux_rpc_get_unpack (f, "{s:o}", key, &value) < 0) {
+            if (errno == ENODATA)
+                break;
+            log_msg_exit ("job-info.update-watch: %s",
+                          future_strerror (f, errno));
+        }
+        if (!(s = json_dumps (value, 0)))
+            log_msg_exit ("invalid json result");
+        printf ("%s\n", s);
+        fflush (stdout);
+        free (s);
+        flux_future_reset (f);
+    }
+    flux_future_destroy (f);
+    flux_close (h);
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/job-list/job-list-helper.sh b/t/job-list/job-list-helper.sh
new file mode 100755
index 000000000000..9d81cf6a11ad
--- /dev/null
+++ b/t/job-list/job-list-helper.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+#
+
+# Return the expected jobids list in a given state:
+#   "all", "pending", "running", "inactive", "active",
+#   "completed", "canceled", "failed", "timeout"
+#
+job_list_state_ids() {
+    for f in "$@"; do
+        cat ${f}.ids
+    done
+}
+
+# Return the expected count of jobs in a given state (See above for list)
+#
+job_list_state_count() {
+    job_list_state_ids "$@" | wc -l
+}
diff --git a/t/job-list/jobspec-permissive.jsonschema b/t/job-list/jobspec-permissive.jsonschema
new file mode 100644
index 000000000000..25f44676b11f
--- /dev/null
+++ b/t/job-list/jobspec-permissive.jsonschema
@@ -0,0 +1,7 @@
+{
+  "title": "permissive-jobspec",
+  "description": "Flux permissive jobspec",
+  "type": "object",
+  "properties": {
+  }
+}
diff --git a/t/job-info/list-id.py b/t/job-list/list-id.py
similarity index 86%
rename from t/job-info/list-id.py
rename to t/job-list/list-id.py
index d6bdbe06de26..8d9e33ce0e47 100755
--- a/t/job-info/list-id.py
+++ b/t/job-list/list-id.py
@@ -26,20 +26,20 @@
 jobspec = job.JobspecV1.from_command(["sleep", "0"], num_tasks=1, cores_per_task=1)
 
 
-def list_cb(f, arg):
+def list_cb(f):
     print(f.get())
 
 
-def submit_cb(f, arg):
+def submit_cb(f):
     jobid = job.submit_get_id(f)
-    h.rpc("job-info.list-id", dict(id=jobid, attrs=attrs)).then(list_cb)
+    h.rpc("job-list.list-id", dict(id=jobid, attrs=attrs)).then(list_cb)
 
 
 h = flux.Flux()
 for i in range(njobs):
     job.submit_async(h, jobspec).then(submit_cb)
 
-if h.reactor_run(h.get_reactor(), 0) < 0:
+if h.reactor_run() < 0:
     h.fatal_error("reactor_run failed")
 
 # vim: tabstop=4 shiftwidth=4 expandtab
diff --git a/t/job-list/list-rpc.py b/t/job-list/list-rpc.py
new file mode 100755
index 000000000000..aecae0cec38f
--- /dev/null
+++ b/t/job-list/list-rpc.py
@@ -0,0 +1,31 @@
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+# Usage: flux python list-rpc.py < payload.json
+#
+#  Send constructed payloads to job-info list interfaces and
+#  print errors for benefit of testing
+
+import flux
+import sys
+
+h = flux.Flux()
+payload = sys.stdin.read()
+if len(sys.argv) > 1:
+    name = sys.argv[1]
+else:
+    name = "list"
+try:
+    print(h.rpc(f"job-list.{name}", payload).get())
+except OSError as err:
+    print(f"errno {err.errno}: {err.strerror}")
+
+
+# vim: tabstop=4 shiftwidth=4 expandtab
diff --git a/t/job-manager/bulk-state.py b/t/job-manager/bulk-state.py
index b860e0019e5f..4a8d4100f793 100755
--- a/t/job-manager/bulk-state.py
+++ b/t/job-manager/bulk-state.py
@@ -20,7 +20,8 @@
 from flux.job import JobspecV1
 import sys
 
-expected_states = ["NEW", "DEPEND", "SCHED", "RUN", "CLEANUP", "INACTIVE"]
+expected_states = ["NEW", "DEPEND", "PRIORITY", "SCHED", "RUN", "CLEANUP", "INACTIVE"]
+
 
 # Return True if all jobs in the jobs dictionary have reached 'INACTIVE' state
 def all_inactive(jobs):
diff --git a/t/job-manager/dumps/invalid/dump-baddir.tar.bz2 b/t/job-manager/dumps/invalid/dump-baddir.tar.bz2
new file mode 100644
index 000000000000..460e2ae3f00f
Binary files /dev/null and b/t/job-manager/dumps/invalid/dump-baddir.tar.bz2 differ
diff --git a/t/job-manager/dumps/valid/dump-0-40-0.tar.bz2 b/t/job-manager/dumps/valid/dump-0-40-0.tar.bz2
new file mode 100644
index 000000000000..351e1a2014b2
Binary files /dev/null and b/t/job-manager/dumps/valid/dump-0-40-0.tar.bz2 differ
diff --git a/t/job-manager/dumps/valid/dump-0-41-0.tar.bz2 b/t/job-manager/dumps/valid/dump-0-41-0.tar.bz2
new file mode 100644
index 000000000000..68cebde6673b
Binary files /dev/null and b/t/job-manager/dumps/valid/dump-0-41-0.tar.bz2 differ
diff --git a/t/job-manager/dumps/valid/dump-0-45-0-no-checkpoint.tar.bz2 b/t/job-manager/dumps/valid/dump-0-45-0-no-checkpoint.tar.bz2
new file mode 100644
index 000000000000..07df5b5a2ffa
Binary files /dev/null and b/t/job-manager/dumps/valid/dump-0-45-0-no-checkpoint.tar.bz2 differ
diff --git a/t/job-manager/dumps/valid/dump-0-45-0-queue-anon.tar.bz2 b/t/job-manager/dumps/valid/dump-0-45-0-queue-anon.tar.bz2
new file mode 100644
index 000000000000..60f121090c2b
Binary files /dev/null and b/t/job-manager/dumps/valid/dump-0-45-0-queue-anon.tar.bz2 differ
diff --git a/t/job-manager/dumps/valid/dump-0-45-0-queues-named.tar.bz2 b/t/job-manager/dumps/valid/dump-0-45-0-queues-named.tar.bz2
new file mode 100644
index 000000000000..640a3da7f80c
Binary files /dev/null and b/t/job-manager/dumps/valid/dump-0-45-0-queues-named.tar.bz2 differ
diff --git a/t/job-manager/dumps/valid/dump-0-46-0-queue-anon-v1.tar.bz2 b/t/job-manager/dumps/valid/dump-0-46-0-queue-anon-v1.tar.bz2
new file mode 100644
index 000000000000..d37eca6cfc6c
Binary files /dev/null and b/t/job-manager/dumps/valid/dump-0-46-0-queue-anon-v1.tar.bz2 differ
diff --git a/t/job-manager/dumps/valid/dump-0-46-0-queues-named-v1.tar.bz2 b/t/job-manager/dumps/valid/dump-0-46-0-queues-named-v1.tar.bz2
new file mode 100644
index 000000000000..20f0b827d7c6
Binary files /dev/null and b/t/job-manager/dumps/valid/dump-0-46-0-queues-named-v1.tar.bz2 differ
diff --git a/t/job-manager/dumps/warn/dump-noeventlog.tar.bz2 b/t/job-manager/dumps/warn/dump-noeventlog.tar.bz2
new file mode 100644
index 000000000000..43994adf0271
Binary files /dev/null and b/t/job-manager/dumps/warn/dump-noeventlog.tar.bz2 differ
diff --git a/t/job-manager/dumps/warn/dump-nojobspec.tar.bz2 b/t/job-manager/dumps/warn/dump-nojobspec.tar.bz2
new file mode 100644
index 000000000000..6fe6615577fc
Binary files /dev/null and b/t/job-manager/dumps/warn/dump-nojobspec.tar.bz2 differ
diff --git a/t/job-manager/dumps/warn/dump-nosubmit.tar.bz2 b/t/job-manager/dumps/warn/dump-nosubmit.tar.bz2
new file mode 100644
index 000000000000..22fbe4f12a54
Binary files /dev/null and b/t/job-manager/dumps/warn/dump-nosubmit.tar.bz2 differ
diff --git a/t/job-manager/dumps/warn/dump-shorteventlog.tar.bz2 b/t/job-manager/dumps/warn/dump-shorteventlog.tar.bz2
new file mode 100644
index 000000000000..977a3df3bdbc
Binary files /dev/null and b/t/job-manager/dumps/warn/dump-shorteventlog.tar.bz2 differ
diff --git a/t/job-manager/events_journal_stream.c b/t/job-manager/events_journal_stream.c
new file mode 100644
index 000000000000..6e32af4edaf0
--- /dev/null
+++ b/t/job-manager/events_journal_stream.c
@@ -0,0 +1,110 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <jansson.h>
+#include <signal.h>
+#include <stdio.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/log.h"
+
+flux_t *h;
+flux_future_t *f;
+
+void cancel_cb (int sig)
+{
+    flux_future_t *f2;
+    if (!(f2 = flux_rpc_pack (h,
+                              "job-manager.events-journal-cancel",
+                              FLUX_NODEID_ANY,
+                              FLUX_RPC_NORESPONSE,
+                              "{s:i}",
+                              "matchtag", (int)flux_rpc_get_matchtag (f))))
+        log_err_exit ("flux_rpc_pack");
+    flux_future_destroy (f2);
+}
+
+int main (int argc, char *argv[])
+{
+    ssize_t inlen;
+    void *inbuf;
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (argc != 1) {
+        fprintf (stderr, "Usage: events_journal_stream <payload\n");
+        exit (1);
+    }
+
+    if ((inlen = read_all (STDIN_FILENO, &inbuf)) < 0)
+        log_err_exit ("read from stdin");
+    if (inlen > 0)  // flux stringified JSON payloads are sent with \0-term
+        inlen++;    //  and read_all() ensures inbuf has one, not acct in inlen
+
+    if (!(f = flux_rpc_raw (h,
+                            "job-manager.events-journal",
+                            inbuf,
+                            inlen,
+                            FLUX_NODEID_ANY,
+                            FLUX_RPC_STREAMING)))
+        log_err_exit ("flux_rpc_raw");
+
+    if (signal (SIGUSR1, cancel_cb) == SIG_ERR)
+        log_err_exit ("signal");
+
+    while (1) {
+        flux_jobid_t id;
+        json_t *events;
+        size_t index;
+        json_t *entry;
+        if (flux_rpc_get_unpack (f,
+                                 "{s:I s:o}",
+                                 "id", &id,
+                                 "events", &events) < 0) {
+            if (errno == ENODATA)
+                break;
+            log_msg_exit ("job-manager.events-journal: %s",
+                          future_strerror (f, errno));
+        }
+        json_array_foreach (events, index, entry) {
+            /* For testing, wrap each eventlog entry in an outer object that
+             * includes the jobid.  Not coincidentally, this looks like
+             * the old format for job manager journal entries.
+             */
+            json_t *o;
+            char *s;
+
+            if (!(o = json_pack ("{s:I s:O}",
+                                 "id", id,
+                                 "entry", entry))
+                || !(s = json_dumps (o, 0)))
+                log_msg_exit ("Error creating eventlog envelope");
+            printf ("%s\n", s);
+            fflush (stdout);
+            free (s);
+            json_decref (o);
+        }
+        flux_future_reset (f);
+    }
+    flux_future_destroy (f);
+    free (inbuf);
+    flux_close (h);
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/job-manager/exec-service.lua b/t/job-manager/exec-service.lua
index 286f07984f9f..8ffe769bdbf0 100755
--- a/t/job-manager/exec-service.lua
+++ b/t/job-manager/exec-service.lua
@@ -73,7 +73,7 @@ assert (f:msghandler {
     msgtypes = { flux.MSGTYPE_EVENT },
     handler = function (f, msg, mh)
         local id = msg.data.id
-        printf ("%s: exeception for %d\n", service, id)
+        printf ("%s: exception for %d\n", service, id)
         if jobs[id] then
             job_complete (jobs[id].msg, id, 9)
         end
diff --git a/t/job-manager/job-conv.py b/t/job-manager/job-conv.py
new file mode 100755
index 000000000000..d891befca233
--- /dev/null
+++ b/t/job-manager/job-conv.py
@@ -0,0 +1,121 @@
+##############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import sys
+import argparse
+
+from flux.core.inner import ffi, raw
+
+
+def statetostr(args):
+    fmt = "L"
+    if args.single:
+        fmt = "S"
+    if not args.states:
+        args.states = [line.strip() for line in sys.stdin]
+
+    for state in args.states:
+        print(raw.flux_job_statetostr(int(state), fmt).decode("utf-8"))
+
+
+def strtostate(args):
+    state = ffi.new("flux_job_state_t [1]")
+    if not args.strings:
+        args.strings = [line.strip() for line in sys.stdin]
+
+    for s in args.strings:
+        try:
+            raw.flux_job_strtostate(s, state)
+        except Exception:
+            print(f"invalid string {s}")
+            sys.exit(1)
+        print(int(state[0]))
+
+
+def resulttostr(args):
+    fmt = "L"
+    if args.abbrev:
+        fmt = "S"
+    if not args.results:
+        args.results = [line.strip() for line in sys.stdin]
+
+    for result in args.results:
+        print(raw.flux_job_resulttostr(int(result), fmt).decode("utf-8"))
+
+
+def strtoresult(args):
+    result = ffi.new("flux_job_result_t [1]")
+    if not args.strings:
+        args.strings = [line.strip() for line in sys.stdin]
+
+    for s in args.strings:
+        try:
+            raw.flux_job_strtoresult(s, result)
+        except Exception:
+            print(f"invalid string {s}")
+            sys.exit(1)
+        print(int(result[0]))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog="job-conv")
+    subparsers = parser.add_subparsers(
+        title="subcommands", description="", dest="subcommand"
+    )
+    subparsers.required = True
+
+    statetostr_parser = subparsers.add_parser("statetostr")
+    statetostr_parser.add_argument(
+        "-s",
+        "--single",
+        action="store_true",
+        help="Output single string abbreviation",
+    )
+    statetostr_parser.add_argument(
+        "states",
+        nargs="*",
+        help="List of states to convert",
+    )
+    statetostr_parser.set_defaults(func=statetostr)
+
+    strtostate_parser = subparsers.add_parser("strtostate")
+    strtostate_parser.add_argument(
+        "strings",
+        nargs="*",
+        help="List of strings to convert",
+    )
+    strtostate_parser.set_defaults(func=strtostate)
+
+    resulttostr_parser = subparsers.add_parser("resulttostr")
+    resulttostr_parser.add_argument(
+        "-a",
+        "--abbrev",
+        action="store_true",
+        help="Output abbreviated result string",
+    )
+    resulttostr_parser.add_argument(
+        "results",
+        nargs="*",
+        help="List of results to convert",
+    )
+    resulttostr_parser.set_defaults(func=resulttostr)
+
+    strtoresult_parser = subparsers.add_parser("strtoresult")
+    strtoresult_parser.add_argument(
+        "strings",
+        nargs="*",
+        help="List of strings to convert",
+    )
+    strtoresult_parser.set_defaults(func=strtoresult)
+
+    args = parser.parse_args()
+    args.func(args)
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/job-manager/list-jobs.c b/t/job-manager/list-jobs.c
index 34c05878f585..6d00df6cd8d6 100644
--- a/t/job-manager/list-jobs.c
+++ b/t/job-manager/list-jobs.c
@@ -15,7 +15,6 @@
 #include <string.h>
 #include <ctype.h>
 #include <assert.h>
-#include <czmq.h>
 #include <flux/core.h>
 #include <flux/optparse.h>
 #include <jansson.h>
@@ -31,21 +30,6 @@ static struct optparse_option list_opts[] =  {
     OPTPARSE_TABLE_END
 };
 
-/* convert floating point timestamp (UNIX epoch, UTC) to ISO 8601 string,
- * with second precision
- */
-static int iso_timestr (double timestamp, char *buf, size_t size)
-{
-    time_t sec = timestamp;
-    struct tm tm;
-
-    if (!gmtime_r (&sec, &tm))
-        return -1;
-    if (strftime (buf, size, "%FT%TZ", &tm) == 0)
-        return -1;
-    return 0;
-}
-
 int main (int argc, char *argv[])
 {
     flux_t *h;
@@ -88,28 +72,8 @@ int main (int argc, char *argv[])
         log_err_exit ("flux_rpc_get_unpack");
 
     json_array_foreach (jobs, index, value) {
-        flux_jobid_t id;
-        int priority;
-        uint32_t userid;
-        double t_submit;
-        char timestr[80];
-        flux_job_state_t state;
-
-        if (json_unpack (value, "{s:I s:i s:i s:f s:i}",
-                                "id", &id,
-                                "priority", &priority,
-                                "userid", &userid,
-                                "t_submit", &t_submit,
-                                "state", &state) < 0)
-            log_msg_exit ("error parsing job data");
-        if (iso_timestr (t_submit, timestr, sizeof (timestr)) < 0)
-            log_err_exit ("time conversion error");
-        printf ("%llu\t%s\t%lu\t%d\t%s\n",
-                (unsigned long long)id,
-                flux_job_statetostr (state, true),
-                (unsigned long)userid,
-                priority,
-                timestr);
+        json_dumpf (value, stdout, JSON_COMPACT);
+        printf ("\n");
     }
 
     flux_future_destroy (f);
diff --git a/t/job-manager/plugins/args.c b/t/job-manager/plugins/args.c
new file mode 100644
index 000000000000..f2d952ab27ac
--- /dev/null
+++ b/t/job-manager/plugins/args.c
@@ -0,0 +1,103 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* args.c - test job-manager jobtap plugin callback for expected args
+ */
+
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+#include "ccan/str/str.h"
+
+static int cb (flux_plugin_t *p,
+               const char *topic,
+               flux_plugin_arg_t *args,
+               void *arg)
+{
+    json_t *resources = NULL;
+    json_t *entry = NULL;
+    flux_jobid_t id = FLUX_JOBID_ANY;
+    uint32_t userid = (uint32_t) -1;
+    int urgency = -1;
+    unsigned int priority = 1234;
+    flux_job_state_t state = 4096;
+    flux_job_state_t prev_state = 4096;
+    double t_submit = 0.0;
+    flux_t *h = flux_jobtap_get_flux (p);
+
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                                "{s:{s:o} s?o s:I s:i s:i s:i s:i s?i s:f}",
+                                "jobspec", "resources", &resources,
+                                 "entry", &entry,
+                                "id", &id,
+                                "userid", &userid,
+                                "urgency", &urgency,
+                                "priority", &priority,
+                                "state", &state,
+                                "prev_state", &prev_state,
+                                "t_submit", &t_submit) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "flux_plugin_arg_unpack: %s",
+                  flux_plugin_arg_strerror (args));
+        return -1;
+    }
+
+    if (streq (topic, "job.new")) {
+        /*  Subscribe to events so we get all job.event.* callbacks */
+        if (flux_jobtap_job_subscribe (p, FLUX_JOBTAP_CURRENT_JOB) < 0) {
+            flux_log (h,
+                      LOG_ERR,
+                      "%s: jobtap_job_subscribe: %s",
+                       topic,
+                       strerror (errno));
+        }
+    }
+    if (strstarts (topic, "job.state.")) {
+        if (entry == NULL
+            || state == 4096
+            || prev_state == 4096) {
+            flux_log (h,
+                      LOG_ERR,
+                      "%s: entry=%p state=%d prev_state=%d",
+                       topic, entry, state, prev_state);
+            return -1;
+        }
+    }
+    if (resources == NULL
+        || id == FLUX_JOBID_ANY
+        || userid == (uint32_t) -1
+        || urgency == -1
+        || priority == 1234
+        || t_submit == 0.0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "%s: res=%p id=%ju uid=%d  urg=%d, pri=%d, t_submit=%f",
+                  topic,
+                  resources,
+                  (uintmax_t)id,
+                  userid,
+                  urgency,
+                  priority,
+                  t_submit);
+        return -1;
+    }
+    flux_log (h, LOG_INFO, "args-check: %s: OK", topic);
+    return 0;
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    flux_plugin_set_name (p, "args");
+    return flux_plugin_add_handler (p, "job.*", cb, NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/cleanup-event.c b/t/job-manager/plugins/cleanup-event.c
new file mode 100644
index 000000000000..ac44221a0fb8
--- /dev/null
+++ b/t/job-manager/plugins/cleanup-event.c
@@ -0,0 +1,36 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* cleanup-event.c - emit a test event in CLEANUP state
+ */
+
+#include <flux/jobtap.h>
+
+static int cleanup_cb (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *arg)
+{
+    return flux_jobtap_event_post_pack (p,
+                                        FLUX_JOBTAP_CURRENT_JOB,
+                                       "test-event",
+                                       NULL);
+}
+
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_add_handler (p,
+                                    "job.state.cleanup",
+                                     cleanup_cb,
+                                     NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/config.c b/t/job-manager/plugins/config.c
new file mode 100644
index 000000000000..a4a699a46fc4
--- /dev/null
+++ b/t/job-manager/plugins/config.c
@@ -0,0 +1,43 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* config.c - test conf.update callback
+ */
+
+#include <jansson.h>
+#include <flux/jobtap.h>
+
+static int conf_update_cb (flux_plugin_t *p,
+                           const char *topic,
+                           flux_plugin_arg_t *args,
+                           void *arg)
+{
+    const char *test;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:{s:{s:s}}}",
+                                "conf",
+                                  "testconfig",
+                                    "testkey", &test) < 0)
+        return flux_jobtap_error (p,
+                                  args,
+                                  "Error parsing [testconfig]: %s",
+                                  flux_plugin_arg_strerror (args));
+    return 0;
+}
+
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_add_handler (p, "conf.update", conf_update_cb, NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/create-event.c b/t/job-manager/plugins/create-event.c
new file mode 100644
index 000000000000..eeec4c916570
--- /dev/null
+++ b/t/job-manager/plugins/create-event.c
@@ -0,0 +1,33 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* create-event.c - emit a test event from the job.create callback
+ */
+
+#include <flux/jobtap.h>
+
+static int create_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *arg)
+{
+    return flux_jobtap_event_post_pack (p,
+                                        FLUX_JOBTAP_CURRENT_JOB,
+                                       "test-event",
+                                       NULL);
+}
+
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_add_handler (p, "job.create", create_cb, NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/create-reject.c b/t/job-manager/plugins/create-reject.c
new file mode 100644
index 000000000000..234a98f3efbc
--- /dev/null
+++ b/t/job-manager/plugins/create-reject.c
@@ -0,0 +1,30 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* create-reject.c - reject a job from job.create callback
+ */
+
+#include <flux/jobtap.h>
+
+static int create_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *arg)
+{
+    return flux_jobtap_reject_job (p, args, "nope");
+}
+
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_add_handler (p, "job.create", create_cb, NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/dependency-test.c b/t/job-manager/plugins/dependency-test.c
new file mode 100644
index 000000000000..29814303bd73
--- /dev/null
+++ b/t/job-manager/plugins/dependency-test.c
@@ -0,0 +1,174 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* dependency-test.c - keep jobs in depend state and wait for
+ *  an RPC to release
+ */
+
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+
+static void remove_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
+{
+    flux_jobid_t id;
+    const char *description = NULL;
+    flux_plugin_t *p = arg;
+
+    if (flux_request_unpack (msg, NULL,
+                             "{s:I s:s}",
+                             "id", &id,
+                             "description", &description) < 0) {
+        flux_log_error (h, "failed to unpack dependency-test.remove msg");
+        goto error;
+    }
+    if (flux_jobtap_dependency_remove (p, id, description) < 0)
+        goto error;
+    if (flux_jobtap_job_aux_set (p, id, description, NULL, NULL) < 0)
+        goto error;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "flux_respond");
+    return;
+error:
+    flux_respond_error (h, msg, errno, flux_msg_last_error (msg));
+}
+
+static void check_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
+{
+    flux_jobid_t id;
+    const char *name = NULL;
+    flux_plugin_t *p = arg;
+
+    if (flux_request_unpack (msg, NULL,
+                             "{s:I s:s}",
+                             "id", &id,
+                             "name", &name) < 0) {
+        flux_log_error (h, "failed to unpack dependency-test check msg");
+        goto error;
+    }
+    if (flux_jobtap_job_aux_get (p, id, name) != p) {
+        errno = ENOENT;
+        goto error;
+    }
+    if (flux_respond (h, msg, NULL) < 0) {
+        flux_log_error (h, "flux_respond");
+        goto error;
+    }
+    return;
+error:
+    flux_respond_error (h, msg, errno, NULL);
+}
+
+static int dependency_test_cb (flux_plugin_t *p,
+                               const char *topic,
+                               flux_plugin_arg_t *args,
+                               void *arg)
+{
+    flux_jobid_t id;
+    const char *name = NULL;
+    int remove = 0;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:{s:s s?i}}",
+                                "id", &id,
+                                "dependency",
+                                "value", &name,
+                                "remove", &remove) < 0) {
+        return flux_jobtap_reject_job (p, args,
+                                       "failed to unpack dependency args: %s",
+                                       flux_plugin_arg_strerror (args));
+    }
+
+    /* Associate some plugin state with the job so we can detect
+     *  successful plugin state creation in testing.
+     */
+    if (flux_jobtap_job_aux_set (p, id, name, p, NULL) < 0)
+        return flux_jobtap_reject_job (p, args,
+                                       "flux_jobap_job_aux_set failed: %s",
+                                        strerror (errno));
+
+    if (flux_jobtap_dependency_add (p, id, name) < 0) {
+        flux_log_error (flux_jobtap_get_flux (p),
+                        "flux_jobtap_dependency_add (%s)",
+                        name);
+        return -1;
+    }
+    if (remove) {
+        if (flux_jobtap_dependency_remove (p, id, name) < 0)
+            return flux_jobtap_reject_job (p, args,
+                                           "dependency_remove: %s",
+                                            strerror (errno));
+        if (flux_jobtap_job_aux_set (p, id, name, NULL, NULL) < 0)
+            return flux_jobtap_reject_job (p, args,
+                                           "flux_jobtap_job_aux_set: %s",
+                                            strerror (errno));
+    }
+    return 0;
+}
+
+static int depend_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *data)
+{
+    const char *description = NULL;
+    flux_jobid_t id;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:{s:{s?{s?s}}}}",
+                                "id", &id,
+                                "jobspec",
+                                "attributes",
+                                "system",
+                                "dependency-test", &description) < 0) {
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "dependency-test", 0,
+                                     "failed to unpack dependency-test args");
+        return -1;
+    }
+    if (description) {
+        if (flux_jobtap_dependency_add (p, id, description) < 0) {
+            flux_jobtap_raise_exception (p,
+                                         FLUX_JOBTAP_CURRENT_JOB,
+                                         "dependency-test", 0,
+                                         "dependency_add: %s",
+                                         strerror (errno));
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.dependency.test", dependency_test_cb, NULL },
+    { "job.state.depend",    depend_cb,          NULL },
+    { 0 },
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, "dependency-test", tab) < 0
+        || flux_jobtap_service_register (p, "remove", remove_cb, p) < 0
+        || flux_jobtap_service_register (p, "check", check_cb, p) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/job_aux.c b/t/job-manager/plugins/job_aux.c
new file mode 100644
index 000000000000..dd5e27e1ec4a
--- /dev/null
+++ b/t/job-manager/plugins/job_aux.c
@@ -0,0 +1,139 @@
+#include <string.h>
+#include <errno.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+static void my_cleanup (void *arg)
+{
+    flux_t *h = arg;
+
+    flux_log (h, LOG_INFO, "job_aux test destructor invoked");
+}
+
+static int depend_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *arg)
+{
+    int rc;
+    void *val;
+    flux_jobid_t id;
+    flux_t *h = flux_jobtap_get_flux (p);
+
+    /*  Test job_aux by jobid here since the current job will be active */
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I}",
+                                "id", &id) < 0)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "failed to unpack jobid: %s",
+                                            flux_plugin_arg_strerror (args));
+
+
+    /*  test aux_set with jobid */
+    rc = flux_jobtap_job_aux_set (p, id, "foo", p, NULL);
+    if (rc != 0)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "flux_jobtap_aux_set failed: %s",
+                                            strerror (errno));
+
+    val = flux_jobtap_job_aux_get (p, id, "foo");
+    if (!val || val != p)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "flux_jobtap_aux_get failed: %s",
+                                            strerror (errno));
+
+    rc = flux_jobtap_job_aux_delete (p, id, val);
+    if (rc < 0)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "flux_jobtap_aux_delete failed: %s",
+                                            strerror (errno));
+
+    if (flux_jobtap_job_aux_get (p, id, "foo"))
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "flux_jobtap_aux_get: %s",
+                                            "unexpected success");
+
+    /* Leave an entry for cleanup later */
+    (void)flux_jobtap_job_aux_set (p, id, "foo", h, my_cleanup);
+
+
+    return 0;
+}
+
+static int validate_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *arg)
+{
+    void *val;
+    /*  Test all job_aux() jobtap interfaces */
+    int rc = flux_jobtap_job_aux_set (NULL,
+                                      FLUX_JOBTAP_CURRENT_JOB,
+                                      "foo",
+                                      p,
+                                      NULL);
+    if (rc >= 0)
+        return flux_jobtap_reject_job (p, args,
+                                       "flux_jobtap_aux_set(NULL, ...) >= 0");
+    if (errno != EINVAL)
+        return flux_jobtap_reject_job (p, args,
+                                       "flux_jobtap_aux_set(NULL, ...) "
+                                       "expected errno == EINVAL");
+
+    rc = flux_jobtap_job_aux_set (p, 1234, "foo", p, NULL);
+    if (rc >= 0)
+        return flux_jobtap_reject_job (p, args,
+                                       "flux_jobtap_aux_set(p, 1234,...) >= 0");
+    if (errno != ENOENT)
+        return flux_jobtap_reject_job (p, args,
+                                       "flux_jobtap_aux_set(NULL, ...) "
+                                       "expected errno == EINVAL, got %d",
+                                       errno);
+
+    /*  test aux_set with current job */
+    rc = flux_jobtap_job_aux_set (p,
+                                  FLUX_JOBTAP_CURRENT_JOB,
+                                  "foo",
+                                  p,
+                                  NULL);
+    if (rc != 0)
+        return flux_jobtap_reject_job (p, args,
+                                       "flux_jobtap_aux_set() failed: %s",
+                                       strerror (errno));
+
+    val = flux_jobtap_job_aux_get (p, FLUX_JOBTAP_CURRENT_JOB, "foo");
+    if (!val || val != p)
+        return flux_jobtap_reject_job (p, args,
+                                       "flux_jobtap_aux_get() failed: %s",
+                                       strerror (errno));
+
+    rc = flux_jobtap_job_aux_delete (p, FLUX_JOBTAP_CURRENT_JOB, val);
+    if (rc < 0)
+        return flux_jobtap_reject_job (p, args,
+                                       "flux_jobtap_aux_delete() failed: %s",
+                                       strerror (errno));
+
+    if (flux_jobtap_job_aux_get (p, FLUX_JOBTAP_CURRENT_JOB, "foo"))
+        return flux_jobtap_reject_job (p, args,
+                                       "flux_jobtap_aux_get(): %s",
+                                       "unexpected success");
+
+    return 0;
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_add_handler (p, "job.validate", validate_cb, NULL) < 0
+        || flux_plugin_add_handler (p, "job.state.depend", depend_cb, NULL) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/jobspec-update-job-list.c b/t/job-manager/plugins/jobspec-update-job-list.c
new file mode 100644
index 000000000000..bb8da81920c6
--- /dev/null
+++ b/t/job-manager/plugins/jobspec-update-job-list.c
@@ -0,0 +1,134 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* jobspec-update-job-list.c - test jobspec-update event in job-list
+ * module
+ */
+
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "ccan/str/str.h"
+#include "src/common/libutil/errprintf.h"
+
+static int validate_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    if (flux_jobtap_jobspec_update_pack (p,
+                                         "{s:f}",
+                                         "attributes.system.duration",
+                                         1000.0) , 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "jobspec-update", 0,
+                                     "update failure");
+        return -1;
+    }
+    return 0;
+}
+
+static int depend_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *data)
+{
+    json_t *o = NULL;
+    json_t *jobspec_tasks = NULL;
+    json_t *task;
+    json_t *command;
+    json_t *new_command = NULL;
+    int rc = -1;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:{s:o}}",
+                                "jobspec",
+                                  "tasks", &o) < 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "jobspec-update", 0,
+                                     "cannot read jobspec");
+        goto cleanup;
+    }
+
+    if (!(jobspec_tasks = json_deep_copy (o))
+        || !(task = json_array_get (jobspec_tasks, 0))
+        || !(command = json_object_get (task, "command"))
+        || !(new_command = json_string ("hostname"))
+        || json_array_set (command, 0, new_command) < 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "jobspec-update", 0,
+                                     "cannot update jobspec tasks");
+        goto cleanup;
+    }
+
+    if (flux_jobtap_jobspec_update_pack (p,
+                                         "{s:O}",
+                                         "tasks", jobspec_tasks) < 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "jobspec-update", 0,
+                                     "update failure");
+        goto cleanup;
+    }
+
+    rc = 0;
+cleanup:
+    json_decref (jobspec_tasks);
+    json_decref (new_command);
+    return rc;
+}
+
+static int sched_cb (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *data)
+{
+    static bool updated = false;
+    /* Avoid emitting this jobspec-update event more than once per test.
+     * Note: this means the test plugin will only work for one job without
+     *  reloading the plugin
+     */
+    if (updated)
+        return 0;
+    if (flux_jobtap_jobspec_update_pack (p,
+                                         "{s:s}",
+                                         "attributes.system.queue",
+                                         "updatequeue") < 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "jobspec-update", 0,
+                                     "update failure");
+        return -1;
+    }
+    updated = true;
+    return 0;
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.validate", validate_cb, NULL },
+    { "job.state.depend", depend_cb, NULL },
+    { "job.state.sched", sched_cb, NULL },
+    { 0 },
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, "jobspec-update-job-list", tab) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/jobspec-update.c b/t/job-manager/plugins/jobspec-update.c
new file mode 100644
index 000000000000..b9a139dc0aa4
--- /dev/null
+++ b/t/job-manager/plugins/jobspec-update.c
@@ -0,0 +1,253 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* jobspec-update.c - test flux_jobtop_jobspec_update_pack(3)
+ */
+
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "ccan/str/str.h"
+#include "src/common/libutil/errprintf.h"
+
+static int get_and_update_jobspec_name (flux_error_t *errp,
+                                        flux_plugin_t *p,
+                                        flux_plugin_arg_t *args,
+                                        const char **cur_namep,
+                                        char *name)
+{
+    flux_jobid_t id;
+    const char *current_name = NULL;
+    char *copy = NULL;
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:{s:{s?{s?{s?s}}}}}",
+                                "id", &id,
+                                "jobspec",
+                                "attributes",
+                                "system",
+                                "job",
+                                "name",
+                                &current_name) < 0) {
+        errprintf (errp,
+                   "failed to unpack job name: %s",
+                   flux_plugin_arg_strerror (args));
+        return -1;
+    }
+
+    /* flux_jobtap_jobspec_update_id_pack() should fail here, since this
+     * function is always called in the context of a jobtap callback:
+     */
+    if (flux_jobtap_jobspec_update_id_pack (p,
+                                            id,
+                                            "{s:s}",
+                                            "attributes.system.foo",
+                                            "bar") == 0) {
+        errprintf (errp,
+                   "flux_jobtap_jobspec_update_id_pack() unexpected success");
+        return -1;
+    }
+
+    if (current_name && !(copy = strdup (current_name))) {
+        errprintf (errp, "failed to copy job name");
+        return -1;
+    }
+
+    /*  Update job name in jobspec and ensure it doesn't change in this
+     *  function.
+     */
+    if (flux_jobtap_jobspec_update_pack (p,
+                                         "{s:s}",
+                                         "attributes.system.job.name",
+                                         name) < 0) {
+        errprintf (errp,
+                   "flux_jobtap_jobspec_update_pack: %s",
+                   strerror (errno));
+        goto error;
+    }
+    /*  Ensure name hasn't changed after update is posted.
+     */
+    if (copy && current_name && !(streq (current_name, copy))) {
+        errprintf (errp, "unpacked job name failed to match after update");
+        goto error;
+    }
+    /*  jobspec update with key not starting with attributes. should fail:
+     */
+    if (flux_jobtap_jobspec_update_pack (p, "{s:s}", "foo.bar", "baz") == 0) {
+        errprintf (errp,
+                   "update key not starting with attributes. not rejected");
+        goto error;
+    }
+    /*  Add a second key to update in another call:
+     */
+    if (flux_jobtap_jobspec_update_pack (p,
+                                         "{s:i}",
+                                         "attributes.system.update-test",
+                                         1) < 0) {
+        errprintf (errp,
+                   "flux_jobtap_jobspec_update_pack: %s",
+                   strerror (errno));
+        goto error;
+    }
+    if (cur_namep)
+        *cur_namep = current_name;
+    free (copy);
+    return 0;
+error:
+    free (copy);
+    return -1;
+}
+
+static int update_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *data)
+{
+    json_t *updates = NULL;
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:o}",
+                                "updates", &updates) < 0) {
+        flux_jobtap_reject_job (p,
+                                args,
+                                "job.update: %s",
+                                flux_plugin_arg_strerror (args));
+    }
+    return 0;
+}
+
+static int new_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    flux_error_t error;
+    if (get_and_update_jobspec_name (&error, p, args, NULL, "new") < 0)
+        flux_jobtap_reject_job (p, args, "jobspec-update: %s", error.text);
+    return 0;
+}
+
+static int priority_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    flux_error_t error;
+    if (get_and_update_jobspec_name (&error, p, args, NULL, "priority") < 0)
+        flux_jobtap_reject_job (p, args, "jobspec-update: %s", error.text);
+    return 0;
+}
+
+
+static int validate_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    flux_error_t error;
+    if (get_and_update_jobspec_name (&error, p, args, NULL, "validated") < 0)
+        flux_jobtap_reject_job (p, args, "jobspec-update: %s", error.text);
+    return 0;
+}
+
+static int depend_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    const char *name;
+    flux_error_t error;
+
+    if (get_and_update_jobspec_name (&error, p, args, &name, "depend") < 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "jobspec-update",
+                                     0,
+                                     "get_and_update_name failed: %s",
+                                     error.text);
+        return -1;
+    }
+    /*  Ensure jobspec was updated during validate
+     */
+    if (!name) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "jobspec-update", 0,
+                                     "expected job name was NULL");
+        return -1;
+    }
+    if (!streq (name, "new")) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "jobspec-update", 0,
+                                     "expected job name 'validated' got %s",
+                                     name);
+        return -1;
+    }
+    return 0;
+}
+
+static int run_cb (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *data)
+{
+    /*  Jobspec update after RUN is expected to fail
+     */
+    if (flux_jobtap_jobspec_update_pack (p,
+                                         "{s:i}",
+                                         "attributes.system.run-update",
+                                         1) == 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "jobspec-update", 0,
+                                     "expected update failure, got success");
+    }
+    return 0;
+}
+
+static void update_msg_cb (flux_t *h,
+                           flux_msg_handler_t *mh,
+                           const flux_msg_t *msg,
+                           void *arg)
+{
+    flux_plugin_t *p = arg;
+    flux_jobid_t id;
+    json_t *update;
+
+    if (flux_msg_unpack (msg, "{s:I s:o}", "id", &id, "update", &update) < 0
+        || flux_jobtap_jobspec_update_id_pack (p, id, "O", update) < 0)
+        flux_jobtap_raise_exception (p, id, "test", 0, "update failed");
+    flux_respond (h, msg, NULL);
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.new", new_cb, NULL },
+    { "job.update", update_cb, NULL },
+    { "job.validate", validate_cb, NULL },
+    { "job.state.priority", priority_cb, NULL },
+    { "job.state.depend", depend_cb, NULL },
+    { "job.state.run", run_cb, NULL },
+    { 0 },
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, "jobspec-update", tab) < 0)
+        return -1;
+    if (flux_jobtap_service_register (p, "update", update_msg_cb, p) < 0)
+        flux_log_error (flux_jobtap_get_flux (p),
+                        "flux_jobtap_service_register");
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/jobtap_api.c b/t/job-manager/plugins/jobtap_api.c
new file mode 100644
index 000000000000..acdc66847cce
--- /dev/null
+++ b/t/job-manager/plugins/jobtap_api.c
@@ -0,0 +1,581 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#include <string.h>
+#include <errno.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+#include "ccan/str/str.h"
+
+static int test_prolog_start_finish (flux_plugin_t *p,
+                                     const char *topic,
+                                     flux_plugin_arg_t *args)
+{
+    errno = 0;
+    if (flux_jobtap_prolog_start (NULL, NULL) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s: errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_prolog_start (NULL NULL)",
+                                     errno,
+                                     EINVAL);
+    errno = 0;
+    if (flux_jobtap_prolog_start (p, NULL) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s: errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_prolog_start (p, NULL)",
+                                     errno,
+                                     EINVAL);
+
+    errno = 0;
+    if (streq (topic, "job.state.cleanup")) {
+        if (flux_jobtap_prolog_start (p, "test") == 0
+            || errno != EINVAL)
+            flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                         "test", 0,
+                                         "%s: %s: errno=%d != %d",
+                                         topic,
+                                         "flux_jobtap_prolog_start ",
+                                         "after start request should fail",
+                                         errno,
+                                         EINVAL);
+
+
+    }
+    errno = 0;
+    if (flux_jobtap_prolog_finish (NULL, FLUX_JOBTAP_CURRENT_JOB, NULL, 0) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s: errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_prolog_finish (NULL, ...)",
+                                     errno,
+                                     EINVAL);
+    errno = 0;
+    if (flux_jobtap_prolog_finish (p, FLUX_JOBTAP_CURRENT_JOB, NULL, 0) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s: errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_prolog_finish (p, NULL...)",
+                                     errno,
+                                     EINVAL);
+    errno = 0;
+    if (flux_jobtap_prolog_finish (NULL, FLUX_JOBTAP_CURRENT_JOB, NULL, 0) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s: errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_prolog_finish (p, 1)",
+                                     errno,
+                                     EINVAL);
+    errno = 0;
+    if (flux_jobtap_prolog_finish (p, 1, "test", 0) == 0
+        || errno != ENOENT)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s (%s): errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_prolog_finish",
+                                     "p, 1, \"test\", 0",
+                                     errno,
+                                     EINVAL);
+
+
+    return 0;
+}
+
+
+static int test_epilog_start_finish (flux_plugin_t *p,
+                                     const char *topic,
+                                     flux_plugin_arg_t *args)
+{
+    errno = 0;
+    if (flux_jobtap_epilog_start (NULL, NULL) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s: errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_epilog_start (NULL NULL)",
+                                     errno,
+                                     EINVAL);
+    errno = 0;
+    if (flux_jobtap_epilog_start (p, NULL) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s: errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_epilog_start (p, NULL)",
+                                     errno,
+                                     EINVAL);
+
+    errno = 0;
+    if (streq (topic, "job.state.run")) {
+        if (flux_jobtap_epilog_start (p, "test") == 0
+            || errno != EINVAL)
+            flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                         "test", 0,
+                                         "%s: %s: errno=%d != %d",
+                                         topic,
+                                         "flux_jobtap_epilog_start ",
+                                         "after start request should fail",
+                                         errno,
+                                         EINVAL);
+
+
+    }
+    errno = 0;
+    if (flux_jobtap_epilog_finish (NULL, FLUX_JOBTAP_CURRENT_JOB, NULL, 0) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s: errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_epilog_finish (NULL, ...)",
+                                     errno,
+                                     EINVAL);
+    errno = 0;
+    if (flux_jobtap_epilog_finish (p, FLUX_JOBTAP_CURRENT_JOB, NULL, 0) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s: errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_epilog_finish (p, NULL...)",
+                                     errno,
+                                     EINVAL);
+    errno = 0;
+    if (flux_jobtap_epilog_finish (NULL, FLUX_JOBTAP_CURRENT_JOB, NULL, 0) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s: errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_epilog_finish (p, 1)",
+                                     errno,
+                                     EINVAL);
+    errno = 0;
+    if (flux_jobtap_epilog_finish (p, 1, "test", 0) == 0
+        || errno != ENOENT)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s (%s): errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_epilog_finish",
+                                     "p, 1, \"test\", 0",
+                                     errno,
+                                     EINVAL);
+
+
+    return 0;
+}
+
+
+static int test_event_post_pack (flux_plugin_t *p,
+                                 const char *topic,
+                                 flux_plugin_arg_t *args)
+{
+    const char *event = NULL;
+
+    errno = 0;
+    if (flux_jobtap_event_post_pack (NULL, 0, NULL, NULL) == 0
+        || errno != EINVAL)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s (%s): errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_event_post_pack",
+                                     " (NULL, ...)",
+                                     errno,
+                                     EINVAL);
+    errno = 0;
+    if (flux_jobtap_event_post_pack (p, 0, "foo", NULL) == 0
+        || errno != ENOENT)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s (%s): errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_event_post_pack",
+                                     " (NULL, ...)",
+                                     errno,
+                                     ENOENT);
+
+    const char *state;
+    if (strstarts (topic, "job.state."))
+        state = topic+10;
+    else
+        state = topic+4;
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                                "{s:{s:{s?{s?{s?s}}}}}",
+                                "jobspec",
+                                 "attributes",
+                                  "system",
+                                   state,
+                                    "post-event", &event) < 0)
+        return flux_jobtap_raise_exception (p,
+                                            FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "%s: %s: unpack_args: %s",
+                                            topic,
+                                            "test_event_post",
+                                            flux_plugin_arg_strerror (args));
+    if (event != NULL) {
+        if (flux_jobtap_event_post_pack (p,
+                                         FLUX_JOBTAP_CURRENT_JOB,
+                                         event,
+                                         "{s:s}",
+                                         "test_context", "yes") < 0)
+            flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                         "test", 0,
+                                         "%s: %s (event=%s): %s",
+                                         topic,
+                                         "flux_jobtap_event_post_pack",
+                                         event,
+                                         strerror (errno));
+    }
+
+    return 0;
+}
+
+static void set_flag_expect_error (const char *topic,
+                                   flux_plugin_t *p,
+                                   flux_jobid_t id,
+                                   char *flag,
+                                   char *msg,
+                                   int expected_errno)
+{
+    errno = 0;
+    int rc = flux_jobtap_job_set_flag (p, id, flag);
+    if (rc == 0 || errno != expected_errno)
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "%s: %s (%s): errno=%d != %d",
+                                     topic,
+                                     "flux_jobtap_job_set_flag",
+                                     msg,
+                                     errno,
+                                     expected_errno);
+}
+
+
+static int test_job_flags (flux_plugin_t *p,
+                           const char *topic,
+                           flux_plugin_arg_t *args)
+{
+    const char *flag = NULL;
+
+    set_flag_expect_error (topic, NULL, 0, NULL,    "NULL, 0, NULL", EINVAL);
+    set_flag_expect_error (topic, p,    0, NULL,    "p, 0, NULL",    EINVAL);
+    set_flag_expect_error (topic, p,    0, "debug", "p, 0, debug",   ENOENT);
+
+    set_flag_expect_error (topic, p,
+                           FLUX_JOBTAP_CURRENT_JOB,
+                           "foo",
+                           "p, FLUX_JOBTAP_CURRENT_JOB, foo",
+                           EINVAL);
+    const char *state;
+    if (strstarts (topic, "job.state."))
+        state = topic+10;
+    else
+        state = topic+4;
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                                "{s:{s:{s?{s?{s?s}}}}}",
+                                "jobspec",
+                                "attributes",
+                                "system",
+                                state,
+                                "set_flag", &flag) < 0)
+        return flux_jobtap_raise_exception (p,
+                                            FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "%s: %s: unpack_args: %s",
+                                            topic,
+                                            "test_job_flags",
+                                            flux_plugin_arg_strerror (args));
+    if (flag != NULL) {
+        if (flux_jobtap_job_set_flag (p, FLUX_JOBTAP_CURRENT_JOB, flag) < 0)
+            flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                         "test", 0,
+                                         "%s: %s (flag=%s): %s",
+                                         topic,
+                                         "flux_jobtap_job_set_flag",
+                                         flag,
+                                         strerror (errno));
+    }
+    return 0;
+}
+
+static int test_job_lookup (flux_plugin_t *p,
+                            const char *topic,
+                            flux_plugin_arg_t *args)
+{
+    flux_jobid_t id;
+    flux_jobid_t lookupid = FLUX_JOBID_ANY;
+    flux_plugin_arg_t *oarg;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:{s:{s?{s?I}}}}",
+                                "id", &id,
+                                "jobspec",
+                                  "attributes",
+                                    "system",
+                                      "lookup-id", &lookupid) < 0)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "%s: failed to unpack lookupid: %s",
+                                            topic,
+                                            flux_plugin_arg_strerror (args));
+
+    errno = 0;
+    oarg = flux_jobtap_job_lookup (NULL, FLUX_JOBID_ANY);
+    if (oarg != NULL || errno != EINVAL)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                           "test", 0,
+                                           "%s: %s: expected errno=%d got %d",
+                                           topic,
+                                           "flux_jobtap_job_lookup",
+                                           EINVAL,
+                                           errno);
+
+    errno = 0;
+    oarg = flux_jobtap_job_lookup (p, 1234);
+    if (oarg != NULL || errno != ENOENT)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                           "test", 0,
+                                           "%s: %s: expected errno=%d got %d",
+                                           topic,
+                                           "flux_jobtap_job_lookup",
+                                           ENOENT,
+                                           errno);
+
+    /*  lookup current job works */
+    oarg = flux_jobtap_job_lookup (p, FLUX_JOBTAP_CURRENT_JOB);
+    if (oarg == NULL)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                           "test", 0,
+                                           "%s: %s: on current job failed: %s",
+                                           topic,
+                                           "flux_jobtap_job_lookup",
+                                           strerror (errno));
+    flux_plugin_arg_destroy (oarg);
+
+    /*  Skip final test if lookupid not set in jobspec */
+    if (lookupid == FLUX_JOBID_ANY)
+        return 0;
+
+    /*  lookup other job works */
+    oarg = flux_jobtap_job_lookup (p, lookupid);
+    if (oarg == NULL)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                           "test", 0,
+                                           "%s: %s: on %ju failed: %s",
+                                           topic,
+                                           "flux_jobtap_job_lookup",
+                                           (uintmax_t) lookupid,
+                                           strerror (errno));
+    flux_plugin_arg_destroy (oarg);
+    return 0;
+}
+
+static int test_job_result (flux_plugin_t *p,
+                            const char *topic,
+                            flux_plugin_arg_t *args)
+{
+    int rc;
+    flux_jobid_t id;
+    flux_job_result_t result;
+    flux_job_result_t expected_result = FLUX_JOB_RESULT_COMPLETED;
+    const char *s = NULL;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:{s:{s?{s?s}}}}",
+                                "id", &id,
+                                "jobspec",
+                                  "attributes",
+                                    "system",
+                                      "expected-result", &s) < 0)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "%s: failed to unpack result: %s",
+                                            topic,
+                                            flux_plugin_arg_strerror (args));
+
+    if (s != NULL && flux_job_strtoresult (s, &expected_result) < 0)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "%s: flux_job_strtoresult: %s",
+                                            topic,
+                                            strerror (errno));
+
+    /*  Test flux_jobtap_get_job_result(3) ENOENT */
+    errno = 0;
+    rc = flux_jobtap_get_job_result (p, 1234, &result);
+    if (rc == 0 || errno != ENOENT)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                           "test", 0,
+                                           "%s: %s: expected errno=%d got %d",
+                                           topic,
+                                           "flux_jobtap_get_job_result",
+                                           ENOENT,
+                                           errno);
+
+    /*  Test flux_jobtap_get_job_result(3) EINVAL */
+    errno = 0;
+    rc = flux_jobtap_get_job_result (NULL, 1234, &result);
+    if (rc == 0 || errno != EINVAL)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                           "test", 0,
+                                           "%s: %s: expected errno=%d got %d",
+                                           topic,
+                                           "flux_jobtap_get_job_result",
+                                           EINVAL,
+                                           errno);
+
+
+    rc = flux_jobtap_get_job_result (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     &result);
+    if (rc < 0 || expected_result != result)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                           "test", 0,
+                                           "%s: %s: expected result=%d got %d",
+                                           topic,
+                                           "flux_jobtap_get_job_result",
+                                           expected_result,
+                                           result);
+
+    return 0;
+}
+
+static int inactive_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *arg)
+{
+    return test_job_result (p, topic, args);
+}
+
+static int cleanup_cb (flux_plugin_t *p,
+                       const char *topic,
+                       flux_plugin_arg_t *args,
+                       void *arg)
+{
+    test_event_post_pack (p, topic, args);
+    test_prolog_start_finish (p, topic, args);
+    test_epilog_start_finish (p, topic, args);
+    return test_job_result (p, topic, args);
+}
+
+static int run_cb (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *arg)
+{
+    int rc;
+    flux_job_result_t result;
+
+    test_job_flags (p, topic, args);
+
+    /*  Test flux_jobtap_get_job_result(3) returns EINVAL here */
+    errno = 0;
+    rc = flux_jobtap_get_job_result (p, FLUX_JOBTAP_CURRENT_JOB, &result);
+    if (rc == 0 || errno != EINVAL)
+        return flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                           "test", 0,
+                                           "%s: %s: expected errno=%d got %d",
+                                           topic,
+                                           "flux_jobtap_get_job_result",
+                                           EINVAL,
+                                           errno);
+    test_event_post_pack (p, topic, args);
+    test_prolog_start_finish (p, topic, args);
+    test_epilog_start_finish (p, topic, args);
+    return 0;
+}
+
+static int sched_cb (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *arg)
+{
+    test_job_flags (p, topic, args);
+    test_event_post_pack (p, topic, args);
+    return 0;
+}
+
+static int priority_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *arg)
+{
+    test_job_flags (p, topic, args);
+    test_event_post_pack (p, topic, args);
+    return 0;
+}
+
+static int depend_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *arg)
+{
+    test_job_flags (p, topic, args);
+    test_event_post_pack (p, topic, args);
+    return test_job_lookup (p, topic, args);
+}
+
+
+static int validate_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *arg)
+{
+    test_event_post_pack (p, topic, args);
+    return test_job_lookup (p, topic, args);
+}
+
+static int new_cb (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *arg)
+{
+    test_event_post_pack (p, topic, args);
+    return test_job_flags (p, topic, args);
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.new",            new_cb,      NULL },
+    { "job.validate",       validate_cb, NULL },
+    { "job.state.priority", priority_cb, NULL },
+    { "job.state.depend",   depend_cb,   NULL },
+    { "job.state.sched",    sched_cb,    NULL },
+    { "job.state.run",      run_cb,      NULL },
+    { "job.state.cleanup",  cleanup_cb,  NULL },
+    { "job.state.inactive", inactive_cb, NULL },
+    { 0 }
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_register (p, "api-test", tab);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/offline.c b/t/job-manager/plugins/offline.c
new file mode 100644
index 000000000000..6dcd2a79cbb5
--- /dev/null
+++ b/t/job-manager/plugins/offline.c
@@ -0,0 +1,51 @@
+/************************************************************\
+ * Copyright 2024 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* cleanup-event.c - emit a test event in CLEANUP state
+ */
+
+#include <flux/jobtap.h>
+
+/* Disconnect rank 3 by default */
+static int rank = 3;
+
+static int run_cb (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *arg)
+{
+    /*  Immediately on state RUN, disconnect the configured rank
+     */
+    flux_t *h = flux_jobtap_get_flux (p);
+    flux_future_t *f;
+
+    /*  Assumes parent of rank is rank 0 */
+    if (!(f = flux_rpc_pack (h,
+                             "overlay.disconnect-subtree",
+                             0,
+                             0,
+                             "{s:i}",
+                             "rank", rank))
+        || flux_rpc_get (f, NULL) < 0) {
+        flux_log_error (h, "failed to disconnect rank %d", rank);
+    }
+    flux_future_destroy (f);
+    return 0;
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_add_handler (p,
+                                    "job.state.run",
+                                     run_cb,
+                                     NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/perilog-test.c b/t/job-manager/plugins/perilog-test.c
new file mode 100644
index 000000000000..320b783f106b
--- /dev/null
+++ b/t/job-manager/plugins/perilog-test.c
@@ -0,0 +1,180 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* perilog-test.c - basic tests for job manager prolog/epilog
+ */
+
+#include <errno.h>
+#include <string.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+#include "ccan/str/str.h"
+
+struct perilog_data {
+    flux_plugin_t *p;
+    flux_jobid_t id;
+    char *name;
+    bool prolog;
+    int status;
+};
+
+static int prolog_exception = 0;
+static int prolog_count = 1;
+
+static struct perilog_data *
+perilog_data_create (flux_plugin_t *p,
+                     flux_jobid_t id,
+                     bool prolog,
+                     const char *name,
+                     int status)
+{
+    struct perilog_data *d = malloc (sizeof (*d));
+    if (!d)
+        return NULL;
+    if (!(d->name = strdup (name))) {
+        free (d);
+        return NULL;
+    }
+    d->p = p;
+    d->id = id;
+    d->prolog = prolog;
+    d->status = status;
+    return d;
+}
+
+static void perilog_data_destroy (struct perilog_data *d)
+{
+    if (d) {
+        free (d->name);
+        free (d);
+    }
+}
+
+static void timer_cb (flux_reactor_t *r,
+                      flux_watcher_t *w,
+                      int revents, void *arg)
+{
+    struct perilog_data *d = arg;
+    if (d->prolog) {
+        if (flux_jobtap_prolog_finish (d->p, d->id, d->name, d->status) < 0)
+            flux_jobtap_raise_exception (d->p, FLUX_JOBTAP_CURRENT_JOB,
+                                        "test", 0,
+                                        "flux_jobtap_prolog_finish: %s",
+                                        strerror (errno));
+    }
+    else {
+        if (flux_jobtap_epilog_finish (d->p, d->id, d->name, d->status) < 0)
+            flux_jobtap_raise_exception (d->p, FLUX_JOBTAP_CURRENT_JOB,
+                                        "test", 0,
+                                        "flux_jobtap_epilog_finish: %s",
+                                        strerror (errno));
+    }
+    flux_watcher_destroy (w);
+    perilog_data_destroy (d);
+}
+
+static int cb (flux_plugin_t *p,
+               const char *topic,
+               flux_plugin_arg_t *args,
+               void *arg)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+    flux_watcher_t *tw;
+    flux_jobid_t id;
+    struct perilog_data *d;
+    int rc;
+    int prolog = streq (topic, "job.state.run");
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I}",
+                                "id", &id) < 0) {
+        flux_log_error (h, "flux_plugin_arg_unpack");
+        return -1;
+    }
+
+    if (!(d = perilog_data_create (p, id, prolog, "test", 0))) {
+        flux_log_error (h, "perilog_data_create");
+        return -1;
+    }
+
+    tw = flux_timer_watcher_create (flux_get_reactor (h),
+                                    0.1,
+                                    0.0,
+                                    timer_cb,
+                                    d);
+    if (tw == NULL) {
+        flux_log_error (h, "timer_watcher_create");
+        return -1;
+    }
+
+    flux_watcher_start (tw);
+    if (prolog) {
+        int count = prolog_count;
+        rc = flux_jobtap_prolog_start (p, "test");
+        while (--count) {
+            char name[64];
+            (void) snprintf (name, sizeof (name), "test-%d", prolog_count);
+            if (!(d = perilog_data_create (p, id, prolog, name, 0))) {
+                flux_log_error (h, "perilog_data_create");
+                return -1;
+            }
+
+            tw = flux_timer_watcher_create (flux_get_reactor (h),
+                                            0.1,
+                                            0.0,
+                                            timer_cb,
+                                            d);
+            if (tw == NULL) {
+                flux_log_error (h, "timer_watcher_create");
+                return -1;
+            }
+            flux_watcher_start (tw);
+            rc = flux_jobtap_prolog_start (p, name);
+        }
+    }
+    else
+        rc = flux_jobtap_epilog_start (p, "test");
+    if (rc < 0) {
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "flux_jobtap_%s_start failed: %s",
+                                     prolog ? "prolog" : "epilog",
+                                     strerror (errno));
+    }
+    if (prolog && prolog_exception) {
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "test", 0,
+                                     "prolog test exception");
+        /* Use timer_cb to finish prolog */
+        timer_cb (flux_get_reactor (h), tw, 0, d);
+    }
+    return 0;
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.state.run",     cb, NULL },
+    { "job.state.cleanup", cb, NULL },
+    { 0 },
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, "perilog-test", tab) < 0)
+        return -1;
+    flux_plugin_conf_unpack (p, "{s?i s?i}",
+                             "prolog-exception", &prolog_exception,
+                             "prolog-count", &prolog_count);
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/priority-invert.c b/t/job-manager/plugins/priority-invert.c
new file mode 100644
index 000000000000..54a1812eb755
--- /dev/null
+++ b/t/job-manager/plugins/priority-invert.c
@@ -0,0 +1,90 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* priority-invert.c - invert all priorities from what they currently
+ * are / should be.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+static void trigger_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
+{
+    flux_plugin_t *p = arg;
+
+    if (flux_jobtap_reprioritize_all (p) < 0)
+        goto error;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "flux_respond");
+    return;
+error:
+    flux_respond_error (h, msg, errno, flux_msg_last_error (msg));
+}
+
+static int priority_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *arg)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+    int urgency;
+    int64_t priority;
+
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                                "{s:i s:I}",
+                                "urgency", &urgency,
+                                "priority", &priority) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "flux_plugin_arg_unpack: %s",
+                  flux_plugin_arg_strerror (args));
+        return -1;
+    }
+
+    /* if this is the first time we're initializing priority, let the
+     * job-manager set the default */
+    if (priority < 0)
+        return 0;
+
+    priority = FLUX_JOB_URGENCY_MAX - urgency;
+    if (flux_plugin_arg_pack (args, FLUX_PLUGIN_ARG_OUT,
+                              "{s:I}",
+                              "priority", priority) < 0) {
+        flux_log (h,
+                  LOG_ERR,
+                  "flux_plugin_arg_pack: %s",
+                  flux_plugin_arg_strerror (args));
+        return -1;
+    }
+    return 0;
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.state.priority", priority_cb, NULL },
+    { "job.priority.get", priority_cb, NULL },
+    { 0 },
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, "priority-invert", tab) < 0
+        || flux_jobtap_service_register (p, "trigger", trigger_cb, p) < 0)
+        return -1;
+
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/priority-wait.c b/t/job-manager/plugins/priority-wait.c
new file mode 100644
index 000000000000..f63250864ad3
--- /dev/null
+++ b/t/job-manager/plugins/priority-wait.c
@@ -0,0 +1,71 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* priority-wait.c - keep jobs in priority state and wait for
+ *  an RPC to assign priority.
+ */
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+
+static void release_cb (flux_t *h,
+                        flux_msg_handler_t *mh,
+                        const flux_msg_t *msg,
+                        void *arg)
+{
+    flux_jobid_t id;
+    int64_t priority = -1;
+    flux_plugin_t *p = arg;
+
+    if (flux_request_unpack (msg, NULL,
+                             "{s:I s:I}",
+                             "id", &id,
+                             "priority", &priority) < 0) {
+        flux_log_error (h, "failed to unpack priority-wait.release msg");
+        goto error;
+    }
+    if (priority < FLUX_JOB_PRIORITY_MIN
+        || priority > FLUX_JOB_PRIORITY_MAX) {
+        errno = EINVAL;
+        goto error;
+    }
+    if (flux_jobtap_reprioritize_job (p, id, (unsigned int) priority) < 0)
+        goto error;
+    if (flux_respond (h, msg, NULL) < 0)
+        flux_log_error (h, "flux_respond");
+    return;
+error:
+    flux_respond_error (h, msg, errno, flux_msg_last_error (msg));
+}
+
+static int priority_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *arg)
+{
+    return flux_jobtap_priority_unavail (p, args);
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.state.priority", priority_cb, NULL },
+    { "job.priority.get", priority_cb, NULL },
+    { 0 },
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, "priority-wait", tab) < 0
+        || flux_jobtap_service_register (p, "release", release_cb, p) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/project-bank-validate.c b/t/job-manager/plugins/project-bank-validate.c
new file mode 100644
index 000000000000..70e584a8ee71
--- /dev/null
+++ b/t/job-manager/plugins/project-bank-validate.c
@@ -0,0 +1,58 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* allow updates of attributes.system.{project,bank} for jobs */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+static int project_bank_cb (flux_plugin_t *p,
+                            const char *topic,
+                            flux_plugin_arg_t *args,
+                            void *arg)
+{
+    flux_job_state_t state;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:i}",
+                                "state", &state) < 0) {
+        flux_jobtap_error (p, args, "plugin args unpack failed");
+        return -1;
+    }
+    if (state == FLUX_JOB_STATE_RUN
+        || state == FLUX_JOB_STATE_CLEANUP) {
+        flux_jobtap_error (p,
+                           args,
+                           "update of project or bank for running job not supported");
+        return -1;
+    }
+    return 0;
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.update.attributes.system.project", project_bank_cb, NULL },
+    { "job.update.attributes.system.bank", project_bank_cb, NULL },
+    { 0 },
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, "project-bank-validate", tab) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/random.c b/t/job-manager/plugins/random.c
new file mode 100644
index 000000000000..63faa561fd03
--- /dev/null
+++ b/t/job-manager/plugins/random.c
@@ -0,0 +1,88 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* builtins/random.c - test plugin that randomizes priority every
+ *  second.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdlib.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+static int priority_cb (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    int64_t priority = lrand48 ();
+    if (flux_plugin_arg_pack (args, FLUX_PLUGIN_ARG_OUT,
+                              "{s:I}",
+                              "priority", priority) < 0) {
+        flux_t *h = flux_jobtap_get_flux (p);
+        flux_log (h, LOG_ERR,
+                 "flux_plugin_arg_pack: %s",
+                 flux_plugin_arg_strerror (args));
+        return -1;
+    }
+    return 0;
+}
+
+static void reprioritize (flux_reactor_t *r,
+                          flux_watcher_t *w,
+                          int revents,
+                          void *arg)
+{
+    flux_jobtap_reprioritize_all (arg);
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    flux_reactor_t *r;
+    flux_watcher_t *tw;
+    flux_t *h = flux_jobtap_get_flux (p);
+
+    if (flux_plugin_set_name (p, "random") < 0)
+        return -1;
+
+    if (!h
+        || !(r = flux_get_reactor (h))
+        || !(tw = flux_timer_watcher_create (r, 1., 1., reprioritize, p)))
+        return -1;
+
+    srand48 (getpid());
+    flux_watcher_start (tw);
+
+    /* Auto-destroy timer watcher on plugin exit:
+     */
+    flux_plugin_aux_set (p, NULL, tw, (flux_free_f) flux_watcher_destroy);
+
+    if (flux_plugin_add_handler (p,
+                                 "job.state.priority",
+                                 priority_cb,
+                                 NULL) < 0
+        || flux_plugin_add_handler (p,
+                                    "job.priority.get",
+                                    priority_cb,
+                                    NULL) < 0) {
+        return -1;
+    }
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/job-manager/plugins/resource-update-expiration.c b/t/job-manager/plugins/resource-update-expiration.c
new file mode 100644
index 000000000000..3631da43cfc6
--- /dev/null
+++ b/t/job-manager/plugins/resource-update-expiration.c
@@ -0,0 +1,69 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* jobspec-update-job-list.c - test jobspec-update event in job-list
+ * module
+ */
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "ccan/str/str.h"
+#include "src/common/libutil/errprintf.h"
+
+static int run_cb (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *data)
+{
+    double expiration;
+    flux_jobid_t id;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:{s:{s:F}}}",
+                                "id", &id,
+                                "R",
+                                 "execution",
+                                  "expiration", &expiration) < 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "resource-update", 0,
+                                     "unpack failure");
+        return -1;
+    }
+
+    if (flux_jobtap_event_post_pack (p,
+                                     id,
+                                     "resource-update",
+                                     "{s:f}",
+                                     "expiration", expiration + 3600.) < 0) {
+        flux_jobtap_raise_exception (p,
+                                     FLUX_JOBTAP_CURRENT_JOB,
+                                     "resource-update", 0,
+                                     "update failure");
+        return -1;
+    }
+    return 0;
+}
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.state.run", run_cb, NULL },
+    { 0 },
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, "resource-update-expiration", tab) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/subscribe.c b/t/job-manager/plugins/subscribe.c
new file mode 100644
index 000000000000..506f8491d820
--- /dev/null
+++ b/t/job-manager/plugins/subscribe.c
@@ -0,0 +1,85 @@
+#include <stdio.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+#include "ccan/str/str.h"
+
+static int cb (flux_plugin_t *p,
+               const char *topic,
+               flux_plugin_arg_t *args,
+               void *arg)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+
+    if (streq (topic, "job.event.start")) {
+        /*  Test flux_jobtap_job_event_posted(), then unsusbscribe()
+         */
+        if (flux_jobtap_job_event_posted (NULL, 0, NULL) != -1
+            || flux_jobtap_job_event_posted (p, 0, NULL) != -1)
+            flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                         "subscribe-test",
+                                         0,
+                                         "event_count() invalid args failed");
+        if (flux_jobtap_job_event_posted (p,
+                                         FLUX_JOBTAP_CURRENT_JOB,
+                                         "start") != 1)
+            flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                         "subscribe-test",
+                                         0,
+                                         "event_count 'start' didn't return 1");
+        flux_jobtap_job_unsubscribe (p, FLUX_JOBTAP_CURRENT_JOB);
+    }
+    else if (streq (topic, "job.event.finish")) {
+        flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                     "subscribe-test",
+                                     0,
+                                     "unexpectedly got finish event",
+                                     strerror (errno));
+        return -1;
+    }
+
+    flux_log (h, LOG_INFO, "subscribe-check: %s: OK", topic);
+    if (streq (topic, "job.event.start")) {
+        // Test for nonzero exit from job.event.* callback:
+        return -1;
+    }
+    else
+        return 0;
+}
+
+static int new_cb (flux_plugin_t *p,
+                   const char *topic,
+                   flux_plugin_arg_t *args,
+                   void *arg)
+{
+    /*  Test invalid arguments */
+    flux_jobtap_job_unsubscribe (NULL, 0);
+    flux_jobtap_job_unsubscribe (p, 0);
+    if (flux_jobtap_job_subscribe (NULL, 0) != -1
+        || flux_jobtap_job_subscribe (p, 0) != -1)
+        return flux_jobtap_reject_job (p, args,
+                                       "subscribe-test: "
+                                       "invalid args check failed");
+    if (flux_jobtap_job_subscribe (p, FLUX_JOBTAP_CURRENT_JOB) < 0)
+        return flux_jobtap_reject_job (p, args,
+                                       "subscribe-test: "
+                                       "flux_jobtap_job_subscribe: %s",
+                                       strerror (errno));
+    return 0;
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+    flux_plugin_set_name (p, "subscribe-test");
+
+    if (flux_plugin_add_handler (p, "job.event.*", cb, NULL) < 0
+        || flux_plugin_add_handler (p, "job.validate", new_cb, NULL) < 0) {
+        flux_log_error (h, "flux_plugin_add_handler");
+        return -1;
+    }
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/test.c b/t/job-manager/plugins/test.c
new file mode 100644
index 000000000000..8dd39936a083
--- /dev/null
+++ b/t/job-manager/plugins/test.c
@@ -0,0 +1,150 @@
+#include <stdio.h>
+#include <jansson.h>
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+#include "ccan/str/str.h"
+
+static int cb (flux_plugin_t *p,
+               const char *topic,
+               flux_plugin_arg_t *args,
+               void *arg)
+{
+    flux_jobid_t id;
+    const char *test_mode;
+    flux_t *h = flux_jobtap_get_flux (p);
+
+    /*  Get test-mode argument from jobspec
+     */
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:{s:{s:{s:{s:s}}}}}",
+                                "id", &id,
+                                "jobspec",
+                                 "attributes",
+                                  "system",
+                                   "jobtap",
+                                    "test-mode", &test_mode) < 0) {
+        flux_log (h, LOG_ERR,
+                  "test: flux_plugin_arg_unpack: %s",
+                  flux_plugin_arg_strerror (args));
+        return -1;
+    }
+
+    if (streq (topic, "job.validate")) {
+        if (streq (test_mode, "validate failure"))
+            return flux_jobtap_reject_job (p, args, "rejected for testing");
+        if (streq (test_mode, "validate failure nullmsg"))
+            return flux_jobtap_reject_job (p, args, NULL);
+        if (streq (test_mode, "validate failure nomsg"))
+            return -1;
+        return 0;
+    }
+
+    /*  Update annotations with the test mode:
+     */
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:{s:s}}",
+                              "annotations",
+                              "test", test_mode) < 0)
+        flux_log (h, LOG_ERR,
+                  "arg_pack: %s",
+                  flux_plugin_arg_strerror (args));
+
+    if (streq (topic, "job.state.priority")) {
+        if (streq (test_mode, "priority unset"))
+            return 0;
+        if (streq (test_mode, "callback error"))
+            return -1;
+        if (streq (test_mode, "annotations error")) {
+            if (flux_plugin_arg_pack (args,
+                                      FLUX_PLUGIN_ARG_OUT,
+                                      "{s:s}",
+                                      "annotations", "test") < 0)
+                flux_log (h, LOG_ERR,
+                          "arg_pack: %s",
+                          flux_plugin_arg_strerror (args));
+            return 0;
+        }
+        if (streq (test_mode, "priority type error")) {
+            flux_plugin_arg_pack (args,
+                                  FLUX_PLUGIN_ARG_OUT,
+                                  "{s:s}",
+                                  "priority", "foo");
+        }
+    }
+    else if (streq (topic, "job.state.sched")) {
+        if (streq (test_mode, "sched: priority unavail"))
+            return flux_jobtap_priority_unavail (p, args);
+        if (streq (test_mode, "sched: callback error"))
+            return -1;
+        if (streq (test_mode, "sched: update priority")) {
+            flux_plugin_arg_pack (args,
+                                  FLUX_PLUGIN_ARG_OUT,
+                                  "{s:i}",
+                                  "priority", 42);
+        }
+        if (streq (test_mode, "sched: dependency-add")) {
+            return flux_jobtap_dependency_add (p, id, "foo");
+        }
+        if (streq (test_mode, "sched: exception")) {
+            flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                        "test", 0,
+                                        "sched: test exception");
+        }
+        if (streq (test_mode, "sched: exception error")) {
+            if (flux_jobtap_raise_exception (NULL, 0, "test", 0, "") >= 0
+                || errno != EINVAL)
+                flux_jobtap_raise_exception (p, FLUX_JOBTAP_CURRENT_JOB,
+                                            "test", 0,
+                                            "sched: exception error failed");
+        }
+    }
+    else if (streq (topic, "job.priority.get")) {
+        if (streq (test_mode, "priority.get: fail"))
+            return -1;
+        if (streq (test_mode, "priority.get: unavail"))
+            return flux_jobtap_priority_unavail (p, args);
+        if (streq (test_mode, "priority.get: bad arg")) {
+            flux_plugin_arg_pack (args,
+                                  FLUX_PLUGIN_ARG_OUT,
+                                  "{s:s}",
+                                  "priority", "foo");
+        }
+    }
+    return 0;
+}
+
+static void reprioritize_cb (flux_t *h, flux_msg_handler_t *mh,
+                             const flux_msg_t *msg,
+                             void *arg)
+{
+    flux_plugin_t *p = arg;
+    flux_log (h, LOG_INFO, "jobtap.test: reprioritizing all jobs");
+    if (flux_jobtap_reprioritize_all (p) < 0)
+        flux_log_error (h, "reprioritize");
+    flux_log (h, LOG_INFO, "jobtap.test: reprioritizing all jobs complete");
+    if (flux_respond (h, msg, "{}") < 0)
+        flux_log_error (h, "flux_respond");
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    flux_t *h = flux_jobtap_get_flux (p);
+    flux_plugin_set_name (p, "test");
+
+    /*  Print config if we got one */
+    flux_log (h, LOG_INFO, "jobtap.test: conf=%s", flux_plugin_get_conf (p));
+
+    /*  Allow reprioritization of all jobs via an RPC:
+     */
+    if (flux_jobtap_service_register (p,
+                                      "reprioritize",
+                                      reprioritize_cb, p) < 0) {
+        flux_log_error (h, "jobtap_service_register");
+        return -1;
+    }
+    return flux_plugin_add_handler (p, "job.*", cb, NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/update-test.c b/t/job-manager/plugins/update-test.c
new file mode 100644
index 000000000000..073b4b98d069
--- /dev/null
+++ b/t/job-manager/plugins/update-test.c
@@ -0,0 +1,79 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* update-test.c - test plugin authorization of job update
+ *  allow updates of the 'test' and 'test2' attributes for test
+ *  purposes.
+ */
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+#include "ccan/str/str.h"
+
+static int update_cb (flux_plugin_t *p,
+                      const char *topic,
+                      flux_plugin_arg_t *args,
+                      void *data)
+{
+    struct flux_msg_cred cred;
+    const char *value;
+
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s s:{s:i s:i}}",
+                                "value", &value,
+                                "cred",
+                                  "userid", &cred.userid,
+                                  "rolemask", &cred.rolemask) < 0)
+        return flux_jobtap_error (p, args, "plugin args unpack failed");
+    if (streq (value, "fail-test"))
+        return flux_jobtap_error (p, args, "rejecting update: fail-test");
+    return 0;
+}
+
+static int job_updated (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    const char *value = NULL;
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:{s?s}}",
+                                "updates",
+                                 "attributes.system.test", &value) < 0)
+        return flux_jobtap_error (p, args, "plugin args unpack failed");
+    if (value
+        && flux_jobtap_event_post_pack (p,
+                                        FLUX_JOBTAP_CURRENT_JOB,
+                                        "update-test",
+                                        "{s:s}",
+                                        "value", value) < 0)
+        return flux_jobtap_error (p, args, "flux_job_event_post_pack failed");
+    return 0;
+}
+
+
+static const struct flux_plugin_handler tab[] = {
+    { "job.update", job_updated, NULL },
+    { "job.update.attributes.system.test", update_cb, NULL },
+    { "job.update.attributes.system.test2", update_cb, NULL },
+    { 0 },
+};
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    if (flux_plugin_register (p, "update-test", tab) < 0)
+        return -1;
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/plugins/validate.c b/t/job-manager/plugins/validate.c
new file mode 100644
index 000000000000..db87658757de
--- /dev/null
+++ b/t/job-manager/plugins/validate.c
@@ -0,0 +1,52 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Test plugin which limits job to 4 per user */
+
+#include <stdio.h>
+
+#include <flux/core.h>
+#include <flux/jobtap.h>
+
+static int reject_id = 4;
+
+static int validate (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *arg)
+{
+    int i;
+    flux_jobid_t jobid;
+    /* Failure to unpack not an error, just let jobs without
+     *  validate-test-id through
+     */
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                                "{s:I s:{s:{s:{s:{s:i}}}}}",
+                                "id", &jobid,
+                                "jobspec",
+                                 "attributes", "system", "jobtap",
+                                  "validate-test-id", &i) < 0)
+        return 0;
+    if (i == reject_id)
+        return flux_jobtap_reject_job (p,
+			               args,
+			               "Job had reject_id == %d jobid=%ju",
+			               i,
+				       (uintmax_t)jobid);
+    return 0;
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    flux_plugin_set_name (p, "test-validate");
+    return flux_plugin_add_handler (p, "job.validate", validate, NULL);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/job-manager/print-constants.c b/t/job-manager/print-constants.c
new file mode 100644
index 000000000000..aa9344db4075
--- /dev/null
+++ b/t/job-manager/print-constants.c
@@ -0,0 +1,36 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <flux/core.h>
+#include "ccan/str/str.h"
+
+int main (int argc, char *argv[])
+{
+    if (argc != 2) {
+        fprintf (stderr, "Usage: print-constants NAME\n");
+        return 1;
+    }
+    if (streq (argv[1], "FLUX_JOBID_ANY")) {
+        printf ("%llx\n", (long long unsigned)FLUX_JOBID_ANY);
+    }
+    else {
+        fprintf (stderr, "unknown name\n");
+        return 1;
+    }
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/job-manager/sched-dummy.c b/t/job-manager/sched-dummy.c
deleted file mode 100644
index c3560580df5d..000000000000
--- a/t/job-manager/sched-dummy.c
+++ /dev/null
@@ -1,304 +0,0 @@
-/************************************************************\
- * Copyright 2018 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* Simple scheduler for testing:
- * - presume that each job is requesting one core
- * - track core counts, not specific core id's
- * - mode=single
- *
- * Command line usage:
- *   flux module load sched-dummy [--cores=N]
- * Options
- *   --cores=N      specifies the total number of cores available (default 16)
- */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <flux/core.h>
-#include <flux/schedutil.h>
-#include <jansson.h>
-#include <czmq.h>
-#include "src/common/liboptparse/optparse.h"
-
-// flux module debug --setbit 0x1 sched-dummy
-// flux module debug --clearbit 0x1 sched-dummy
-enum module_debug_flags {
-    DEBUG_FAIL_ALLOC = 1, // while set, alloc requests fail
-};
-
-struct job {
-    flux_msg_t *msg;
-    flux_jobid_t id;
-    int priority;
-    uint32_t userid;
-    double t_submit;
-    char *jobspec;
-};
-
-struct sched_ctx {
-    flux_t *h;
-    schedutil_t *schedutil_ctx;
-    optparse_t *opt;
-    struct job *job; // backlog of 1 alloc request
-    int cores_total;
-    int cores_free;
-    flux_watcher_t *prep;
-};
-
-static void job_destroy (struct job *job)
-{
-    if (job) {
-        int saved_errno = errno;
-        free (job->jobspec);
-        flux_msg_destroy (job->msg);
-        free (job);
-        errno = saved_errno;
-    }
-}
-
-/* Create job struct from sched.alloc request.
- */
-static struct job *job_create (const flux_msg_t *msg, const char *jobspec)
-{
-    struct job *job;
-
-    if (!(job = calloc (1, sizeof (*job))))
-        return NULL;
-    if (schedutil_alloc_request_decode (msg, &job->id, &job->priority,
-                                        &job->userid, &job->t_submit) < 0)
-        goto error;
-    if (!(job->jobspec = strdup (jobspec)))
-        goto error;
-    if (!(job->msg = flux_msg_copy (msg, true)))
-        goto error;
-    return job;
-error:
-    job_destroy (job);
-    return NULL;
-}
-
-void try_alloc (struct sched_ctx *sc)
-{
-    if (sc->job) {
-        if (flux_module_debug_test (sc->h, DEBUG_FAIL_ALLOC, false)) {
-            if (schedutil_alloc_respond_denied (sc->schedutil_ctx,
-                                                sc->job->msg,
-                                                "DEBUG_FAIL_ALLOC") < 0)
-                flux_log_error (sc->h, "schedutil_alloc_respond_denied");
-            goto done;
-        }
-        if (sc->cores_free > 0) {
-            if (schedutil_alloc_respond_R (sc->schedutil_ctx, sc->job->msg,
-                                           "1core", NULL) < 0)
-                flux_log_error (sc->h, "schedutil_alloc_respond_R");
-            sc->cores_free--;
-            goto done;
-        }
-        if (schedutil_alloc_respond_note (sc->schedutil_ctx, sc->job->msg,
-                                          "no cores available") < 0)
-            flux_log_error (sc->h, "schedutil_alloc_respond_note");
-    }
-    return;
-done:
-    job_destroy (sc->job);
-    sc->job = NULL;
-}
-
-void exception_cb (flux_t *h, flux_jobid_t id,
-                   const char *type, int severity, void *arg)
-{
-    struct sched_ctx *sc = arg;
-    char note[80];
-
-    if (severity > 0 || sc->job == NULL || sc->job->id != id)
-        return;
-    (void)snprintf (note, sizeof(note),
-                    "alloc aborted due to exception type=%s", type);
-    if (schedutil_alloc_respond_denied (sc->schedutil_ctx,
-                                        sc->job->msg,
-                                        note) < 0)
-        flux_log_error (h, "%s: alloc_respond_denied", __FUNCTION__);
-    job_destroy (sc->job);
-    sc->job = NULL;
-}
-
-void free_cb (flux_t *h, const flux_msg_t *msg, const char *R, void *arg)
-{
-    struct sched_ctx *sc = arg;
-    flux_jobid_t id;
-
-    if (schedutil_free_request_decode (msg, &id) < 0)
-        goto error;
-    flux_log (h, LOG_DEBUG, "free: id=%llu R=%s",
-              (unsigned long long)id, R);
-    sc->cores_free++;
-    if (schedutil_free_respond (sc->schedutil_ctx, msg) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    try_alloc (sc);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-void alloc_cb (flux_t *h, const flux_msg_t *msg,
-               const char *jobspec, void *arg)
-{
-    struct sched_ctx *sc = arg;
-    struct job *job;
-
-    if (!(job = job_create (msg, jobspec))) {
-        flux_log_error (h, "%s: job_create", __FUNCTION__);
-        goto error;
-    }
-    if (sc->job) {
-        flux_log_error (h, "alloc received before previous one handled");
-        goto error;
-    }
-    sc->job = job;
-    flux_log (h, LOG_DEBUG, "alloc: id=%llu jobspec=%s",
-              (unsigned long long)job->id, job->jobspec);
-    try_alloc (sc);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-int hello_cb (flux_t *h,
-              flux_jobid_t id,
-              int priority,
-              uint32_t userid,
-              double t_submit,
-              const char *R,
-              void *arg)
-{
-    struct sched_ctx *sc = arg;
-
-    flux_log (h, LOG_DEBUG,
-              "%s: id=%ju priority=%d userid=%u t_submit=%0.1f R=%s",
-              __func__,
-              (uintmax_t)id,
-              priority,
-              (unsigned int)userid,
-              t_submit,
-              R);
-    sc->cores_free--;
-    return 0;
-}
-
-static struct optparse_option dummy_opts[] = {
-    {   .name = "cores",
-        .has_arg = 1,
-        .flags = 0,
-        .arginfo = "COUNT",
-        .usage = "Core count (default 16)",
-    },
-    OPTPARSE_TABLE_END,
-};
-
-/* N.B. module argv[0] is first argument, not module name.
- */
-optparse_t *options_parse (int argc, char **argv)
-{
-    optparse_t *opt;
-    if (!(opt = optparse_create ("sched-dummy"))) {
-        errno = ENOMEM;
-        return NULL;
-    }
-    if (optparse_add_option_table (opt, dummy_opts) != OPTPARSE_SUCCESS)
-        goto error;
-    if (optparse_parse_args (opt,  argc + 1, argv - 1) < 0)
-        goto error;
-    return opt;
-error:
-    optparse_destroy (opt);
-    errno = EINVAL;
-    return NULL;
-}
-
-void sched_destroy (struct sched_ctx *sc)
-{
-    if (sc) {
-        int saved_errno = errno;
-        schedutil_destroy (sc->schedutil_ctx);
-        optparse_destroy (sc->opt);
-        if (sc->job) {
-            /* Causes job-manager to pause scheduler interface.
-             */
-            if (flux_respond_error (sc->h, sc->job->msg, ENOSYS,
-                                    "scheduler unloading") < 0)
-                flux_log_error (sc->h, "flux_respond_error");
-            job_destroy (sc->job);
-        }
-        free (sc);
-        errno = saved_errno;
-    }
-}
-
-struct sched_ctx *sched_create (flux_t *h, int argc, char **argv)
-{
-    struct sched_ctx *sc;
-
-    if (!(sc = calloc (1, sizeof (*sc))))
-        return NULL;
-    sc->h = h;
-    sc->schedutil_ctx = schedutil_create (h,
-                                          alloc_cb,
-                                          free_cb,
-                                          exception_cb,
-                                          sc);
-    if (sc->schedutil_ctx == NULL) {
-        flux_log_error (h, "schedutil_create");
-        goto error;
-    }
-    if (!(sc->opt = options_parse (argc, argv))) {
-        errno = EINVAL;
-        goto error;
-    }
-    sc->cores_total = optparse_get_int (sc->opt, "cores", 16);
-    sc->cores_free = sc->cores_total;
-    return sc;
-error:
-    sched_destroy (sc);
-    return NULL;
-}
-
-int mod_main (flux_t *h, int argc, char *argv[])
-{
-    int rc = -1;
-    struct sched_ctx *sc;
-    int count;
-
-    if (!(sc = sched_create (h, argc, argv)))
-        return -1;
-    flux_log (h, LOG_DEBUG, "res pool is %d cores", sc->cores_total);
-    if (schedutil_hello (sc->schedutil_ctx, hello_cb, sc) < 0) {
-        flux_log_error (h, "schedutil_hello");
-        goto done;
-    }
-    if (schedutil_ready (sc->schedutil_ctx, "single", &count) < 0) {
-        flux_log_error (h, "schedutil_ready");
-        goto done;
-    }
-    flux_log (sc->h, LOG_DEBUG, "ready: count=%d", count);
-
-    if ((rc = flux_reactor_run (flux_get_reactor (h), 0)) < 0)
-        flux_log_error (h, "flux_reactor_run");
-done:
-    sched_destroy (sc);
-    return rc;
-}
-MOD_NAME ("sched-dummy");
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/t/job-manager/sched-helper.sh b/t/job-manager/sched-helper.sh
new file mode 100755
index 000000000000..bf7e7d06f26e
--- /dev/null
+++ b/t/job-manager/sched-helper.sh
@@ -0,0 +1,215 @@
+#!/bin/sh
+#
+
+# job-manager sched helper functions
+
+JMGR_JOB_LIST=${FLUX_BUILD_DIR}/t/job-manager/list-jobs
+JOB_CONV="flux python ${FLUX_SOURCE_DIR}/t/job-manager/job-conv.py"
+
+# internal function to get job state via job manager
+#
+# if job is not found by list-jobs, but the clean event exists
+# in the job's eventlog, return state as inactive
+#
+# use job-conf tool to convert state numeric value to string
+# value.
+#
+# arg1 - jobid
+_jmgr_get_state() {
+        local id=$(flux job id $1)
+        local state=$(${JMGR_JOB_LIST} \
+                      | grep ${id} \
+                      | jq .state \
+                      | ${JOB_CONV} statetostr -s)
+        test -z "$state" \
+                && flux job wait-event --timeout=5 ${id} clean >/dev/null \
+                && state=I
+        echo $state
+}
+
+# verify if job is in specific state through job manager
+#
+# function will loop for up to 5 seconds in case state change is slow
+#
+# arg1 - jobid
+# arg2 - single character expected state (e.g. R = running)
+jmgr_check_state() {
+        local id=$1
+        local wantstate=$2
+        for try in $(seq 1 10); do
+                test $(_jmgr_get_state $id) = $wantstate && return 0
+                sleep 0.5
+        done
+        return 1
+}
+
+# internal function to get job annotation key value via job manager
+#
+# arg1 - jobid
+# arg2 - key
+_jmgr_get_annotation() {
+        local id=$(flux job id $1)
+        local key=$2
+        local note="$(${JMGR_JOB_LIST} | grep ${id} | jq .annotations | jq ."${key}")"
+        echo $note
+}
+
+# verify if job contains specific annotation key & value through job manager
+#
+# function will loop for up to 5 seconds in case annotation update
+# arrives slowly
+#
+# arg1 - jobid
+# arg2 - key in annotation
+# arg3 - value of key in annotation
+#
+jmgr_check_annotation() {
+        local id=$1
+        local key=$2
+        local value="$3"
+        for try in $(seq 1 10); do
+                test "$(_jmgr_get_annotation $id $key)" = "${value}" && return 0
+                sleep 0.5
+        done
+        return 1
+}
+
+# verify if job contains specific memo key & value through job manager
+#  (currently memo is a user annotation)
+#
+# function will loop for up to 5 seconds in case annotation update
+# arrives slowly
+#
+# arg1 - jobid
+# arg2 - key in memo
+# arg3 - value of key
+#
+jmgr_check_memo() {
+	jmgr_check_annotation $1 "user.$2" $3
+	return $?
+}
+
+# verify if job contains specific annotation key through job manager
+#
+# arg1 - jobid
+# arg2 - key in annotation
+#
+jmgr_check_annotation_exists() {
+        local id=$(flux job id $1)
+        local key=$2
+        ${JMGR_JOB_LIST} | grep ${id} | jq .annotations | jq -e ."${key}" > /dev/null
+}
+
+#  like jmgr_check_annotation_exists() but for job memo
+#
+jmgr_check_memo_exists() {
+	jmgr_check_annotation_exists $1 "user.$2"
+}
+
+# verify that job contains no annotations through job manager
+#
+# arg1 - jobid
+jmgr_check_no_annotations() {
+        local id=$(flux job id $1)
+        if ${JMGR_JOB_LIST} | grep ${id} > /dev/null
+        then
+            ${JMGR_JOB_LIST} | grep ${id} | jq -e .annotations > /dev/null && return 1
+        fi
+        return 0
+}
+
+# internal function to get job annotation key value via flux job list
+#
+# arg1 - jobid
+# arg2 - key
+_jlist_get_annotation() {
+        local id=$(flux job id $1)
+        local key=$2
+        local note="$(flux job list -A | grep ${id} | jq .annotations | jq ."${key}")"
+        echo $note
+}
+
+# verify if annotation published to job-list
+#
+# function will loop for up to 5 seconds in case annotation update
+# arrives slowly
+#
+# arg1 - jobid
+# arg2 - key in annotation
+# arg3 - value of key in annotation
+#
+jlist_check_annotation() {
+        local id=$1
+        local key=$2
+        local value="$3"
+        for try in $(seq 1 10); do
+                test "$(_jlist_get_annotation $id $key)" = "${value}" && return 0
+                sleep 0.5
+        done
+        return 1
+}
+
+jlist_check_memo() {
+	jlist_check_annotation $1 "user.$2" $3
+	return $?
+}
+
+# verify that job contains no annotations via job-list
+#
+# arg1 - jobid
+#
+jlist_check_no_annotations() {
+        local id=$(flux job id $1)
+        flux job list -A | grep ${id} | jq -e .annotations > /dev/null && return 1
+        return 0
+}
+
+# verify if job contains specific annotation key through job-list
+#
+# arg1 - jobid
+# arg2 - key in annotation
+#
+jlist_check_annotation_exists() {
+        local id=$(flux job id $1)
+        local key=$2
+        flux job list -A | grep ${id} | jq .annotations | jq -e ."${key}" > /dev/null
+}
+
+# verify if annotation seen via flux-jobs
+#
+# function will loop for up to 5 seconds in case annotation update
+# arrives slowly
+#
+# arg1 - jobid
+# arg2 - key in annotation
+# arg3 - value of key in annotation
+fjobs_check_annotation() {
+        local id=$(flux job id $1)
+        local key=$2
+        local value="$3"
+        for try in $(seq 1 10); do
+                test "$(flux jobs -n --format={${key}} ${id})" = "${value}" && return 0
+                sleep 0.5
+        done
+        return 1
+}
+
+# verify that flux-jobs see no annotations
+#
+# arg1 - jobid
+fjobs_check_no_annotations() {
+        local id=$1
+        test -z $(flux jobs -n --format="{annotations}" ${id}) && return 0
+        return 1
+}
+
+# verify flux-jobs sees annotation
+#
+# arg1 - jobid
+# arg2 - key in annotation
+fjobs_check_annotation_exists() {
+        local id=$1
+        local key=$2
+        test -z $(flux jobs -n --format="{${key}}" ${id}) && return 1
+        return 0
+}
diff --git a/t/job-manager/submit-sliding-window.py b/t/job-manager/submit-sliding-window.py
index ffdadc16dc04..0dd468269aab 100755
--- a/t/job-manager/submit-sliding-window.py
+++ b/t/job-manager/submit-sliding-window.py
@@ -29,7 +29,7 @@
 # Open connection to broker
 h = flux.Flux()
 
-jobspec = JobspecV1.from_command(["/bin/true"])
+jobspec = JobspecV1.from_command(["true"])
 done = 0
 running = 0
 
diff --git a/t/job-manager/submit-wait.py b/t/job-manager/submit-wait.py
index 3d8edde97009..5c7b15b2116e 100755
--- a/t/job-manager/submit-wait.py
+++ b/t/job-manager/submit-wait.py
@@ -27,16 +27,16 @@
 h = flux.Flux()
 
 # Submit njobs test jobs (half will fail)
-jobspec = JobspecV1.from_command(["/bin/true"])
-jobspec_fail = JobspecV1.from_command(["/bin/false"])
+jobspec = JobspecV1.from_command(["true"])
+jobspec_fail = JobspecV1.from_command(["false"])
 jobs = []
 for i in range(njobs):
     if i < njobs / 2:
         jobid = job.submit(h, jobspec, waitable=True)
-        print("submit: {} /bin/true".format(jobid))
+        print("submit: {} true".format(jobid))
     else:
         jobid = job.submit(h, jobspec_fail, waitable=True)
-        print("submit: {} /bin/false".format(jobid))
+        print("submit: {} false".format(jobid))
     jobs.append(jobid)
 
 # Wait for each job in turn
diff --git a/t/job-manager/submit-waitany.py b/t/job-manager/submit-waitany.py
index 1225a75da4ce..7fd394bd7bf9 100755
--- a/t/job-manager/submit-waitany.py
+++ b/t/job-manager/submit-waitany.py
@@ -27,15 +27,15 @@
 h = flux.Flux()
 
 # Submit njobs test jobs (half will fail)
-jobspec = JobspecV1.from_command(["/bin/true"])
-jobspec_fail = JobspecV1.from_command(["/bin/false"])
+jobspec = JobspecV1.from_command(["true"])
+jobspec_fail = JobspecV1.from_command(["false"])
 for i in range(njobs):
     if i < njobs / 2:
         jobid = job.submit(h, jobspec, waitable=True)
-        print("submit: {} /bin/true".format(jobid))
+        print("submit: {} true".format(jobid))
     else:
         jobid = job.submit(h, jobspec_fail, waitable=True)
-        print("submit: {} /bin/false".format(jobid))
+        print("submit: {} false".format(jobid))
 
 
 # Wait for njobs jobs
diff --git a/t/job-manager/wait-interrupted.py b/t/job-manager/wait-interrupted.py
index 229b0ea37d44..b8300cb2f4be 100755
--- a/t/job-manager/wait-interrupted.py
+++ b/t/job-manager/wait-interrupted.py
@@ -28,11 +28,11 @@
 h = flux.Flux()
 
 # Submit njobs test jobs
-jobspec = JobspecV1.from_command(["/bin/true"])
+jobspec = JobspecV1.from_command(["true"])
 jobs = []
 for i in range(njobs):
     jobid = job.submit(h, jobspec, waitable=True)
-    print("submit: {} /bin/true".format(jobid))
+    print("submit: {} true".format(jobid))
     jobs.append(jobid)
 
 # Async wait which we immediately abandon
diff --git a/t/jobspec/invalid/attributes_bad_entry.yaml b/t/jobspec/invalid/attributes_bad_entry.yaml
index 51c9329c8285..e7cdd61cac64 100644
--- a/t/jobspec/invalid/attributes_bad_entry.yaml
+++ b/t/jobspec/invalid/attributes_bad_entry.yaml
@@ -4,7 +4,7 @@ resources:
     count: 1
     label: foo
     with:
-      - type: node
+      - type: core
         count: 1
 tasks:
   - command: [ "app" ]
diff --git a/t/jobspec/invalid/attributes_not_mapping.yaml b/t/jobspec/invalid/attributes_not_mapping.yaml
index 3bc65c348bab..143d5d4db409 100644
--- a/t/jobspec/invalid/attributes_not_mapping.yaml
+++ b/t/jobspec/invalid/attributes_not_mapping.yaml
@@ -4,7 +4,7 @@ resources:
     count: 1
     label: foo
     with:
-      - type: node
+      - type: core
         count: 1
 tasks:
   - command: [ "app" ]
diff --git a/t/jobspec/invalid/attributes_null.yaml b/t/jobspec/invalid/attributes_null.yaml
new file mode 100644
index 000000000000..8d9bdf343ab7
--- /dev/null
+++ b/t/jobspec/invalid/attributes_null.yaml
@@ -0,0 +1,14 @@
+version: 1
+resources:
+  - type: slot
+    count: 1
+    label: foo
+    with:
+      - type: core
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
diff --git a/t/jobspec/invalid/constraints_not_object.yaml b/t/jobspec/invalid/constraints_not_object.yaml
new file mode 100644
index 000000000000..5a4aca498f51
--- /dev/null
+++ b/t/jobspec/invalid/constraints_not_object.yaml
@@ -0,0 +1,18 @@
+version: 1
+resources:
+  - type: slot
+    count: 1
+    label: foo
+    with:
+      - type: core
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    constraints:
+      - foo
+      - bar
diff --git a/t/jobspec/invalid/dependencies_not_array.yaml b/t/jobspec/invalid/dependencies_not_array.yaml
new file mode 100644
index 000000000000..dbc7a99462c9
--- /dev/null
+++ b/t/jobspec/invalid/dependencies_not_array.yaml
@@ -0,0 +1,17 @@
+version: 1
+resources:
+  - type: slot
+    count: 1
+    label: foo
+    with:
+      - type: core
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    dependencies:
+      scheme: foo
diff --git a/t/jobspec/invalid/dependency_invalid.yaml b/t/jobspec/invalid/dependency_invalid.yaml
new file mode 100644
index 000000000000..7bf61d0f8b01
--- /dev/null
+++ b/t/jobspec/invalid/dependency_invalid.yaml
@@ -0,0 +1,18 @@
+version: 1
+resources:
+  - type: slot
+    count: 1
+    label: foo
+    with:
+      - type: core
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    dependencies:
+      - scheme: foo
+        value: 1
diff --git a/t/jobspec/invalid/missing_resources.yaml b/t/jobspec/invalid/missing_resources.yaml
index 5d5fd449af5e..a7cbc2663f5a 100644
--- a/t/jobspec/invalid/missing_resources.yaml
+++ b/t/jobspec/invalid/missing_resources.yaml
@@ -5,3 +5,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/missing_tasks.yaml b/t/jobspec/invalid/missing_tasks.yaml
index 9bd63914b1de..cb0e87c7130e 100644
--- a/t/jobspec/invalid/missing_tasks.yaml
+++ b/t/jobspec/invalid/missing_tasks.yaml
@@ -7,3 +7,5 @@ resources:
       - type: node
         count: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/missing_version.yaml b/t/jobspec/invalid/missing_version.yaml
index 9e7f22b3f2c3..0c8818249f1e 100644
--- a/t/jobspec/invalid/missing_version.yaml
+++ b/t/jobspec/invalid/missing_version.yaml
@@ -11,3 +11,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_count_bad_type.yaml b/t/jobspec/invalid/resource_count_bad_type.yaml
index dc436cdfe2b8..f8cee9d1c30f 100644
--- a/t/jobspec/invalid/resource_count_bad_type.yaml
+++ b/t/jobspec/invalid/resource_count_bad_type.yaml
@@ -14,3 +14,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_count_is_zero.yaml b/t/jobspec/invalid/resource_count_is_zero.yaml
new file mode 100644
index 000000000000..897c8ae996f3
--- /dev/null
+++ b/t/jobspec/invalid/resource_count_is_zero.yaml
@@ -0,0 +1,16 @@
+version: 1
+resources:
+  - type: slot
+    count: 0
+    label: foo
+    with:
+      - type: core
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_count_missing_max.yaml b/t/jobspec/invalid/resource_count_missing_max.yaml
index ca3b56932cdb..39e9913df1e6 100644
--- a/t/jobspec/invalid/resource_count_missing_max.yaml
+++ b/t/jobspec/invalid/resource_count_missing_max.yaml
@@ -15,3 +15,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_count_missing_min.yaml b/t/jobspec/invalid/resource_count_missing_min.yaml
index a7e25cd92d1d..b87f6e87bee5 100644
--- a/t/jobspec/invalid/resource_count_missing_min.yaml
+++ b/t/jobspec/invalid/resource_count_missing_min.yaml
@@ -15,3 +15,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_count_missing_operand.yaml b/t/jobspec/invalid/resource_count_missing_operand.yaml
index dfc1ee4f632b..a066730f4e23 100644
--- a/t/jobspec/invalid/resource_count_missing_operand.yaml
+++ b/t/jobspec/invalid/resource_count_missing_operand.yaml
@@ -15,3 +15,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_count_missing_operator.yaml b/t/jobspec/invalid/resource_count_missing_operator.yaml
index 6305b0ffb416..454e8578a85c 100644
--- a/t/jobspec/invalid/resource_count_missing_operator.yaml
+++ b/t/jobspec/invalid/resource_count_missing_operator.yaml
@@ -15,3 +15,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_count_scalar_bad_value.yaml b/t/jobspec/invalid/resource_count_scalar_bad_value.yaml
index 6343b7e35648..edd9cd10ec65 100644
--- a/t/jobspec/invalid/resource_count_scalar_bad_value.yaml
+++ b/t/jobspec/invalid/resource_count_scalar_bad_value.yaml
@@ -4,7 +4,7 @@ resources:
     count: bar
     label: foo
     with:
-      - type: node
+      - type: core
         count: 1
 tasks:
   - command: [ "app" ]
@@ -12,3 +12,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_exclusive_invalid.yaml b/t/jobspec/invalid/resource_exclusive_invalid.yaml
index e7c201f7a56d..c798321a71d5 100644
--- a/t/jobspec/invalid/resource_exclusive_invalid.yaml
+++ b/t/jobspec/invalid/resource_exclusive_invalid.yaml
@@ -13,3 +13,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_exclusive_not_scalar.yaml b/t/jobspec/invalid/resource_exclusive_not_scalar.yaml
index 18f3192284d2..56d03744855f 100644
--- a/t/jobspec/invalid/resource_exclusive_not_scalar.yaml
+++ b/t/jobspec/invalid/resource_exclusive_not_scalar.yaml
@@ -15,3 +15,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_id_not_scalar.yaml b/t/jobspec/invalid/resource_id_not_scalar.yaml
index 74a24856a6ed..508e2ce68860 100644
--- a/t/jobspec/invalid/resource_id_not_scalar.yaml
+++ b/t/jobspec/invalid/resource_id_not_scalar.yaml
@@ -15,3 +15,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_label_not_scalar.yaml b/t/jobspec/invalid/resource_label_not_scalar.yaml
index 16af5ac7b684..bab458687206 100644
--- a/t/jobspec/invalid/resource_label_not_scalar.yaml
+++ b/t/jobspec/invalid/resource_label_not_scalar.yaml
@@ -14,3 +14,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_missing_count.yaml b/t/jobspec/invalid/resource_missing_count.yaml
index 886495768da0..d3d632c277cf 100644
--- a/t/jobspec/invalid/resource_missing_count.yaml
+++ b/t/jobspec/invalid/resource_missing_count.yaml
@@ -11,3 +11,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_missing_type.yaml b/t/jobspec/invalid/resource_missing_type.yaml
index 8c6adbe4e7a5..aa085a1e2779 100644
--- a/t/jobspec/invalid/resource_missing_type.yaml
+++ b/t/jobspec/invalid/resource_missing_type.yaml
@@ -11,3 +11,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_node_with_notarray.yaml b/t/jobspec/invalid/resource_node_with_notarray.yaml
new file mode 100644
index 000000000000..bab7b3d142f0
--- /dev/null
+++ b/t/jobspec/invalid/resource_node_with_notarray.yaml
@@ -0,0 +1,13 @@
+version: 1
+resources:
+  - type: node
+    count: 4
+    with: 42
+tasks:
+  - command: [ "app" ]
+    slot: default
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 3600.
diff --git a/t/jobspec/invalid/resource_not_mapping.yaml b/t/jobspec/invalid/resource_not_mapping.yaml
index eb744126cb43..20451609e6af 100644
--- a/t/jobspec/invalid/resource_not_mapping.yaml
+++ b/t/jobspec/invalid/resource_not_mapping.yaml
@@ -12,3 +12,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_slot_not_labelled.yaml b/t/jobspec/invalid/resource_slot_not_labelled.yaml
index c2910f5ecc9b..f8787b509a7b 100644
--- a/t/jobspec/invalid/resource_slot_not_labelled.yaml
+++ b/t/jobspec/invalid/resource_slot_not_labelled.yaml
@@ -11,3 +11,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_slot_with_notarray.yaml b/t/jobspec/invalid/resource_slot_with_notarray.yaml
new file mode 100644
index 000000000000..5cec2056d607
--- /dev/null
+++ b/t/jobspec/invalid/resource_slot_with_notarray.yaml
@@ -0,0 +1,14 @@
+version: 1
+resources:
+  - type: slot
+    label: foo
+    count: 1
+    with: 42
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_slot_with_zero.yaml b/t/jobspec/invalid/resource_slot_with_zero.yaml
new file mode 100644
index 000000000000..49b46e4f563d
--- /dev/null
+++ b/t/jobspec/invalid/resource_slot_with_zero.yaml
@@ -0,0 +1,16 @@
+version: 1
+resources:
+  - type: slot
+    label: foo
+    count: 0
+    with:
+      - type: core
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resource_unit_not_scalar.yaml b/t/jobspec/invalid/resource_unit_not_scalar.yaml
index 6d3ed7ce4a64..1769b8dfb77c 100644
--- a/t/jobspec/invalid/resource_unit_not_scalar.yaml
+++ b/t/jobspec/invalid/resource_unit_not_scalar.yaml
@@ -15,3 +15,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/resources_not_sequence.yaml b/t/jobspec/invalid/resources_not_sequence.yaml
index f58b28b7434f..387a805afd1b 100644
--- a/t/jobspec/invalid/resources_not_sequence.yaml
+++ b/t/jobspec/invalid/resources_not_sequence.yaml
@@ -12,3 +12,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/task_command_array_notstring.yaml b/t/jobspec/invalid/task_command_array_notstring.yaml
new file mode 100644
index 000000000000..09f36bc88435
--- /dev/null
+++ b/t/jobspec/invalid/task_command_array_notstring.yaml
@@ -0,0 +1,16 @@
+version: 1
+resources:
+  - type: slot
+    count: 1
+    label: foo
+    with:
+      - type: core
+        count: 1
+tasks:
+  - command: [1,2,3]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/task_command_not_array.yaml b/t/jobspec/invalid/task_command_not_array.yaml
index 5aedc085f602..c7382d97d926 100644
--- a/t/jobspec/invalid/task_command_not_array.yaml
+++ b/t/jobspec/invalid/task_command_not_array.yaml
@@ -4,7 +4,7 @@ resources:
     count: 1
     label: foo
     with:
-      - type: node
+      - type: core
         count: 1
 tasks:
   - command: app
@@ -12,3 +12,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/task_command_zero_length_array.yaml b/t/jobspec/invalid/task_command_zero_length_array.yaml
new file mode 100644
index 000000000000..0937265fffe2
--- /dev/null
+++ b/t/jobspec/invalid/task_command_zero_length_array.yaml
@@ -0,0 +1,16 @@
+version: 1
+resources:
+  - type: slot
+    count: 1
+    label: foo
+    with:
+      - type: core
+        count: 1
+tasks:
+  - command: []
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/task_count_not_mapping.yaml b/t/jobspec/invalid/task_count_not_mapping.yaml
index 2f89c5491ee2..55fb67be5ce4 100644
--- a/t/jobspec/invalid/task_count_not_mapping.yaml
+++ b/t/jobspec/invalid/task_count_not_mapping.yaml
@@ -4,10 +4,12 @@ resources:
     count: 1
     label: foo
     with:
-      - type: node
+      - type: core
         count: 1
 tasks:
   - command: [ "app" ]
     slot: foo
     count: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/task_count_per_slot_zero.yaml b/t/jobspec/invalid/task_count_per_slot_zero.yaml
new file mode 100644
index 000000000000..816522df629d
--- /dev/null
+++ b/t/jobspec/invalid/task_count_per_slot_zero.yaml
@@ -0,0 +1,22 @@
+version: 1
+resources:
+  - type: node
+    count: 4
+    with:
+      - type: slot
+        count: 1
+        label: myslot
+        with:
+          - type: core
+            count: 1
+tasks:
+  - command: [ "hostname" ]
+    slot: myslot
+    count:
+      per_slot: 0
+attributes:
+  system:
+    duration: 3600.
+    cwd: "/home/flux"
+    environment:
+      HOME: "/home/flux"
diff --git a/t/jobspec/invalid/task_count_total_zero.yaml b/t/jobspec/invalid/task_count_total_zero.yaml
new file mode 100644
index 000000000000..69947ae25e95
--- /dev/null
+++ b/t/jobspec/invalid/task_count_total_zero.yaml
@@ -0,0 +1,22 @@
+version: 1
+resources:
+  - type: node
+    count: 4
+    with:
+      - type: slot
+        count: 1
+        label: myslot
+        with:
+          - type: core
+            count: 1
+tasks:
+  - command: [ "hostname" ]
+    slot: myslot
+    count:
+      total: 0
+attributes:
+  system:
+    duration: 3600.
+    cwd: "/home/flux"
+    environment:
+      HOME: "/home/flux"
diff --git a/t/jobspec/invalid/task_missing_command.yaml b/t/jobspec/invalid/task_missing_command.yaml
index 334556f83bc7..464f2088286d 100644
--- a/t/jobspec/invalid/task_missing_command.yaml
+++ b/t/jobspec/invalid/task_missing_command.yaml
@@ -4,10 +4,12 @@ resources:
     count: 1
     label: foo
     with:
-      - type: node
+      - type: core
         count: 1
 tasks:
   - slot: foo
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/task_missing_count.yaml b/t/jobspec/invalid/task_missing_count.yaml
index 16480139bcad..557c3d9ef94d 100644
--- a/t/jobspec/invalid/task_missing_count.yaml
+++ b/t/jobspec/invalid/task_missing_count.yaml
@@ -4,9 +4,11 @@ resources:
     count: 1
     label: foo
     with:
-      - type: node
+      - type: core
         count: 1
 tasks:
   - command: [ "app" ]
     slot: foo
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/task_missing_slot.yaml b/t/jobspec/invalid/task_missing_slot.yaml
index 3d946ef5d207..bb96c417108f 100644
--- a/t/jobspec/invalid/task_missing_slot.yaml
+++ b/t/jobspec/invalid/task_missing_slot.yaml
@@ -4,10 +4,12 @@ resources:
     count: 1
     label: foo
     with:
-      - type: node
+      - type: core
         count: 1
 tasks:
   - command: [ "app" ]
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/task_not_mapping.yaml b/t/jobspec/invalid/task_not_mapping.yaml
index 8f4fed305e42..5382ff0826b4 100644
--- a/t/jobspec/invalid/task_not_mapping.yaml
+++ b/t/jobspec/invalid/task_not_mapping.yaml
@@ -4,8 +4,10 @@ resources:
     count: 1
     label: foo
     with:
-      - type: node
+      - type: core
         count: 1
 tasks:
   - foo
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/tasks_not_sequence.yaml b/t/jobspec/invalid/tasks_not_sequence.yaml
index c80e96849bd3..f1af86a1c052 100644
--- a/t/jobspec/invalid/tasks_not_sequence.yaml
+++ b/t/jobspec/invalid/tasks_not_sequence.yaml
@@ -4,7 +4,7 @@ resources:
     count: 1
     label: foo
     with:
-      - type: node
+      - type: core
         count: 1
 tasks:
   command: [ "app" ]
@@ -12,3 +12,5 @@ tasks:
   count:
     per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/version_bad_number.yaml b/t/jobspec/invalid/version_bad_number.yaml
index e6181e4708fb..b786a8cd2952 100644
--- a/t/jobspec/invalid/version_bad_number.yaml
+++ b/t/jobspec/invalid/version_bad_number.yaml
@@ -12,3 +12,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid/version_not_scalar.yaml b/t/jobspec/invalid/version_not_scalar.yaml
index 1d00bb61159b..dca13b813153 100644
--- a/t/jobspec/invalid/version_not_scalar.yaml
+++ b/t/jobspec/invalid/version_not_scalar.yaml
@@ -13,3 +13,5 @@ tasks:
     count:
       per_slot: 1
 attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid_v1/core_count_invalid.yaml b/t/jobspec/invalid_v1/core_count_invalid.yaml
new file mode 100644
index 000000000000..ee156fc7f01c
--- /dev/null
+++ b/t/jobspec/invalid_v1/core_count_invalid.yaml
@@ -0,0 +1,19 @@
+version: 1
+resources:
+  - type: node
+    count: 1
+    with:
+      - type: slot
+        count: 1
+        label: default
+        with:
+          - type: core
+            count: 0
+tasks:
+  - command: [ "app" ]
+    slot: default
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 3600.
diff --git a/t/jobspec/invalid_v1/duration_missing.yaml b/t/jobspec/invalid_v1/duration_missing.yaml
new file mode 100644
index 000000000000..c1a429eeb46f
--- /dev/null
+++ b/t/jobspec/invalid_v1/duration_missing.yaml
@@ -0,0 +1,16 @@
+version: 1
+resources:
+  - type: slot
+    count: 1
+    label: task
+    with:
+      - type: core
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    foo: 1
diff --git a/t/jobspec/invalid_v1/gpu_count_invalid.yaml b/t/jobspec/invalid_v1/gpu_count_invalid.yaml
new file mode 100644
index 000000000000..42c9d1a7ea42
--- /dev/null
+++ b/t/jobspec/invalid_v1/gpu_count_invalid.yaml
@@ -0,0 +1,22 @@
+version: 1
+resources:
+  - type: node
+    count: 1
+    with:
+      - type: slot
+        count: 1
+        label: default
+        with:
+          - type: core
+            count: 1
+          - type: gpu
+            count: -1
+
+tasks:
+  - command: [ "app" ]
+    slot: default
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 3600.
diff --git a/t/jobspec/invalid_v1/node_vertex_missing_count.yaml b/t/jobspec/invalid_v1/node_vertex_missing_count.yaml
new file mode 100644
index 000000000000..352e04fd536c
--- /dev/null
+++ b/t/jobspec/invalid_v1/node_vertex_missing_count.yaml
@@ -0,0 +1,18 @@
+version: 1
+resources:
+  - type: node
+    with:
+      - type: slot
+        count: 1
+        label: default
+        with:
+          - type: core
+            count: 2
+tasks:
+  - command: [ "app" ]
+    slot: default
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 3600.
diff --git a/t/jobspec/invalid_v1/node_vertex_with_notarray.yaml b/t/jobspec/invalid_v1/node_vertex_with_notarray.yaml
new file mode 100644
index 000000000000..8e479386b1f2
--- /dev/null
+++ b/t/jobspec/invalid_v1/node_vertex_with_notarray.yaml
@@ -0,0 +1,17 @@
+version: 1
+resources:
+  - type: node
+    count: 1
+    with:
+      - type: slot
+        count: 1
+        label: default
+        with: 2
+tasks:
+  - command: [ "app" ]
+    slot: default
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 3600.
diff --git a/t/jobspec/invalid_v1/node_vertex_with_toomany.yaml b/t/jobspec/invalid_v1/node_vertex_with_toomany.yaml
new file mode 100644
index 000000000000..8ce6be2e241c
--- /dev/null
+++ b/t/jobspec/invalid_v1/node_vertex_with_toomany.yaml
@@ -0,0 +1,19 @@
+version: 1
+resources:
+  - type: node
+    count: 1
+    with:
+      - type: slot
+        count: 1
+        label: default
+        with:
+      - type: core
+        count: 2
+tasks:
+  - command: [ "app" ]
+    slot: default
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 3600.
diff --git a/t/jobspec/invalid_v1/node_vertex_zero_count.yaml b/t/jobspec/invalid_v1/node_vertex_zero_count.yaml
new file mode 100644
index 000000000000..2661f8e86f1c
--- /dev/null
+++ b/t/jobspec/invalid_v1/node_vertex_zero_count.yaml
@@ -0,0 +1,19 @@
+version: 1
+resources:
+  - type: node
+    count: 0
+    with:
+      - type: slot
+        count: 1
+        label: default
+        with:
+          - type: core
+            count: 2
+tasks:
+  - command: [ "app" ]
+    slot: default
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 3600.
diff --git a/t/jobspec/invalid_v1/res_type_unknown.yaml b/t/jobspec/invalid_v1/res_type_unknown.yaml
new file mode 100644
index 000000000000..e24fc0a1b663
--- /dev/null
+++ b/t/jobspec/invalid_v1/res_type_unknown.yaml
@@ -0,0 +1,22 @@
+version: 1
+resources:
+  - type: smurf
+    count: 4
+    with:
+      - type: slot
+        count: 1
+        label: default
+        with:
+          - type: core
+            count: 2
+tasks:
+  - command: [ "app" ]
+    slot: default
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 3600.
+    cwd: "/home/flux"
+    environment:
+      HOME: "/home/flux"
diff --git a/t/jobspec/invalid_v1/slot_node.yaml b/t/jobspec/invalid_v1/slot_node.yaml
new file mode 100644
index 000000000000..0620acdc9139
--- /dev/null
+++ b/t/jobspec/invalid_v1/slot_node.yaml
@@ -0,0 +1,16 @@
+version: 1
+resources:
+  - type: slot
+    count: 1
+    label: task
+    with:
+      - type: node
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid_v1/version_wrong.yaml b/t/jobspec/invalid_v1/version_wrong.yaml
new file mode 100644
index 000000000000..a3ce87c529b3
--- /dev/null
+++ b/t/jobspec/invalid_v1/version_wrong.yaml
@@ -0,0 +1,16 @@
+version: 2
+resources:
+  - type: slot
+    count: 1
+    label: task
+    with:
+      - type: core
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid_v1/with_toomany.yaml b/t/jobspec/invalid_v1/with_toomany.yaml
new file mode 100644
index 000000000000..17c83e105fa9
--- /dev/null
+++ b/t/jobspec/invalid_v1/with_toomany.yaml
@@ -0,0 +1,20 @@
+version: 1
+resources:
+  - type: slot
+    label: foo
+    count: 1
+    with:
+      - type: core
+        count: 1
+      - type: gpu
+        count: 1
+      - type: smurf
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/invalid_v1/with_whatnow.yaml b/t/jobspec/invalid_v1/with_whatnow.yaml
new file mode 100644
index 000000000000..3e714dc1e97f
--- /dev/null
+++ b/t/jobspec/invalid_v1/with_whatnow.yaml
@@ -0,0 +1,18 @@
+version: 1
+resources:
+  - type: slot
+    label: foo
+    count: 1
+    with:
+      - type: piglets
+        count: 1
+      - type: mermaids
+        count: 1
+tasks:
+  - command: [ "app" ]
+    slot: foo
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 1
diff --git a/t/jobspec/summarize-minimal-jobspec.py b/t/jobspec/summarize-minimal-jobspec.py
index 604a39486b32..6affdfa73a1a 100755
--- a/t/jobspec/summarize-minimal-jobspec.py
+++ b/t/jobspec/summarize-minimal-jobspec.py
@@ -1,7 +1,5 @@
 # Usage: flux python summarize-minimal-jobspec.py -j jobspec >summary.txt
 
-from __future__ import print_function
-
 import sys
 import json
 import argparse
@@ -11,9 +9,10 @@
 
 def load_jobspec(stream, prefix=""):
     try:
-        jobspec = yaml.safe_load(stream)
+        data = stream.read().encode("utf-8")
+        jobspec = yaml.safe_load(data)
         return jobspec
-    except (yaml.YAMLError) as e:
+    except yaml.YAMLError as e:
         print("{}{}".format(prefix, e.problem))
         sys.exit(1)
 
@@ -56,7 +55,7 @@ def main():
 
     if args.jobspec:
         try:
-            with open(args.jobspec, "r") as fd:
+            with open(args.jobspec, "r", encoding="utf-8") as fd:
                 jobspec = yaml.safe_load(fd)
         except (OSError, IOError) as e:
             print("{}{}".format(args.jobspec, e.strerror))
diff --git a/t/jobspec/valid/attributes_system.yaml b/t/jobspec/valid/attributes_system.yaml
index eddf8499059e..0bee12ee4548 100644
--- a/t/jobspec/valid/attributes_system.yaml
+++ b/t/jobspec/valid/attributes_system.yaml
@@ -1,4 +1,4 @@
-version: 1
+version: 999
 resources:
   - type: slot
     count: 1
diff --git a/t/jobspec/valid/attributes_user.yaml b/t/jobspec/valid/attributes_user.yaml
index 4b7ae49abcae..3971f8e87291 100644
--- a/t/jobspec/valid/attributes_user.yaml
+++ b/t/jobspec/valid/attributes_user.yaml
@@ -1,4 +1,4 @@
-version: 1
+version: 999
 resources:
   - type: slot
     count: 1
diff --git a/t/jobspec/valid/basic.yaml b/t/jobspec/valid/basic.yaml
index 2c5a095a6d94..a5a50e48204f 100644
--- a/t/jobspec/valid/basic.yaml
+++ b/t/jobspec/valid/basic.yaml
@@ -1,4 +1,4 @@
-version: 1
+version: 999
 resources:
   - type: slot
     count: 1
@@ -11,4 +11,4 @@ tasks:
     slot: foo
     count:
       per_slot: 1
-attributes:
+attributes: {}
diff --git a/t/jobspec/valid_v1/gpu_count_zero.yaml b/t/jobspec/valid_v1/gpu_count_zero.yaml
new file mode 100644
index 000000000000..592934ca2bd4
--- /dev/null
+++ b/t/jobspec/valid_v1/gpu_count_zero.yaml
@@ -0,0 +1,22 @@
+version: 1
+resources:
+  - type: node
+    count: 1
+    with:
+      - type: slot
+        count: 1
+        label: default
+        with:
+          - type: core
+            count: 1
+          - type: gpu
+            count: 0
+
+tasks:
+  - command: [ "app" ]
+    slot: default
+    count:
+      per_slot: 1
+attributes:
+  system:
+    duration: 3600.
diff --git a/t/jobspec/validate.py b/t/jobspec/validate.py
deleted file mode 100755
index e5bd689ccc65..000000000000
--- a/t/jobspec/validate.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# Usage: flux python validate.py --schema=jobspec.json data.json [data.json ...]
-# Usage: cat data.json | flux python validate.py --schema=jobspec.json
-
-import sys
-import argparse
-import json
-import yaml
-import jsonschema
-
-
-def validate_input(jobspec_stream, schema, label=""):
-    errors = 0
-    try:
-        data = yaml.safe_load(jobspec_stream)
-        jsonschema.validate(data, schema)
-    except (yaml.YAMLError) as e:
-        print("{}: {}".format(label, e.problem))
-        errors = errors + 1
-    except (Exception) as e:
-        print("{}: {}".format(label, str(e)))
-        errors = errors + 1
-    except:
-        print("{}: unknown error".format(label))
-        errors = errors + 1
-    else:
-        print("{}: ok".format(label))
-    return errors
-
-
-def validate_inputfile(infile, schema):
-    errors = 0
-    try:
-        with open(infile) as fd:
-            errors += validate_input(fd, schema, label=infile)
-    except (OSError, IOError) as e:
-        print("{}: {}".format(infile, e.strerror))
-        errors = errors + 1
-    return errors
-
-
-# parse command line args
-parser = argparse.ArgumentParser()
-parser.add_argument("--schema", "-s", type=str, required=True)
-parser.add_argument("jobspecs", nargs="*")
-args = parser.parse_args()
-
-# Parse json-schema file (JSON format)
-try:
-    schema = json.load(open(args.schema))
-except (OSError, IOError) as e:
-    sys.exit("{}: {}".format(args.schema, e.strerror))
-except:
-    sys.exit("{}: unknown error".format(args.schema))
-
-# Validate each file on command line
-errors = 0
-if args.jobspecs:
-    for infile in args.jobspecs:
-        errors += validate_inputfile(infile, schema)
-else:
-    errors += validate_input(sys.stdin, schema, label="stdin")
-
-sys.exit(errors)
diff --git a/t/jobspec/y2j.py b/t/jobspec/y2j.py
index 43df63e3886d..f0ec84664f9b 100755
--- a/t/jobspec/y2j.py
+++ b/t/jobspec/y2j.py
@@ -6,7 +6,7 @@
     obj = yaml.safe_load(sys.stdin)
 except (OSError, IOError) as e:
     sys.exit("y2j: " + e.strerror)
-except (yaml.YAMLError) as e:
+except yaml.YAMLError as e:
     sys.exit("y2j: " + e.problem)
 
 json.dump(obj, sys.stdout, separators=(",", ":"))
diff --git a/t/kvs/change-checkpoint.py b/t/kvs/change-checkpoint.py
new file mode 100755
index 000000000000..a235642e130d
--- /dev/null
+++ b/t/kvs/change-checkpoint.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import sys
+import sqlite3
+
+if len(sys.argv) < 4:
+    print("change-checkpoint.py <file> <key> <value>")
+    sys.exit(1)
+path = sys.argv[1].encode("utf-8", errors="surrogateescape").decode()
+conn = sqlite3.connect(path)
+cursor = conn.cursor()
+s = (
+    'REPLACE INTO checkpt (key,value) values ("'
+    + sys.argv[2]
+    + '", "'
+    + sys.argv[3]
+    + '")'
+)
+cursor.execute(s)
+conn.commit()
+sys.exit(0)
diff --git a/t/kvs/checkpoint.c b/t/kvs/checkpoint.c
deleted file mode 100644
index c868b771e4f7..000000000000
--- a/t/kvs/checkpoint.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/************************************************************\
- * Copyright 2020 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdio.h>
-#include <flux/core.h>
-#include "src/common/libutil/log.h"
-
-void usage (void)
-{
-    fprintf (stderr, "Usage: checkpoint get key\n"
-                     "   or: checkpoint put key value\n");
-    exit (1);
-}
-
-int main (int argc, char **argv)
-{
-    flux_t *h;
-    const char *cmd;
-    const char *key;
-    const char *value;
-    flux_future_t *f;
-
-    if (argc < 2 || argc > 4)
-        usage ();
-    cmd = argv[1];
-    key = argv[2];
-    if (argc == 4)
-        value = argv[3];
-    if (strcmp (cmd, "get") != 0 && strcmp (cmd, "put") != 0)
-        usage ();
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (!strcmp (cmd, "put")) {
-        if (!(f = flux_rpc_pack (h,
-                                 "kvs-checkpoint.put",
-                                 0,
-                                 0,
-                                 "{s:s s:s}",
-                                 "key",
-                                 key,
-                                 "value",
-                                 value)))
-            log_err_exit("flux_rpc");
-        if (flux_rpc_get (f, NULL) < 0)
-            log_err_exit ("%s", key);
-    }
-    else {
-        if (!(f = flux_rpc_pack (h,
-                                 "kvs-checkpoint.get",
-                                 0,
-                                 0,
-                                 "{s:s}",
-                                 "key",
-                                 key)))
-            log_err_exit("flux_rpc");
-        if (flux_rpc_get_unpack (f, "{s:s}", "value", &value) < 0)
-            log_err_exit ("%s", key);
-        printf ("%s\n", value);
-    }
-
-    flux_future_destroy (f);
-    flux_close (h);
-    return 0;
-}
-
-/*
- * vi: ts=4 sw=4 expandtab
- */
diff --git a/t/kvs/commit.c b/t/kvs/commit.c
index 7010c7688ab0..c405ab737482 100644
--- a/t/kvs/commit.c
+++ b/t/kvs/commit.c
@@ -18,10 +18,10 @@
 #include <pthread.h>
 #include <getopt.h>
 #include <inttypes.h>
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/oom.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/xzmalloc.h"
@@ -130,7 +130,7 @@ int main (int argc, char *argv[])
     tstat_t ts;
     struct timespec t0;
 
-    log_init (basename (argv[0]));
+    log_init (argv[0]);
 
     while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
         switch (ch) {
diff --git a/t/kvs/content-spam.c b/t/kvs/content-spam.c
new file mode 100644
index 000000000000..8521e5eb7640
--- /dev/null
+++ b/t/kvs/content-spam.c
@@ -0,0 +1,90 @@
+/************************************************************\
+ * Copyright 2016 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Usage: content-spam N [M]
+ * Store N random entries, keeping M requests in flight (default 1)
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/blobref.h"
+#include "src/common/libutil/log.h"
+#include "src/common/libcontent/content.h"
+
+static int spam_max_inflight;
+static int spam_cur_inflight;
+
+static void store_completion (flux_future_t *f, void *arg)
+{
+    flux_t *h = arg;
+    const char *blobref;
+    const char *hash_type = flux_aux_get (h, "hash_type");
+
+    if (content_store_get_blobref (f, hash_type, &blobref) < 0)
+        log_err_exit ("store");
+    printf ("%s\n", blobref);
+    flux_future_destroy (f);
+    if (--spam_cur_inflight < spam_max_inflight/2)
+        flux_reactor_stop (flux_get_reactor (h));
+}
+
+int main (int ac, char *av[])
+{
+    int i, count;
+    flux_future_t *f;
+    flux_t *h;
+    char data[256];
+    int size = 256;
+    const char *s;
+    char *hash_type;
+
+    if (ac != 2 && ac != 3) {
+        fprintf (stderr, "Usage: content-spam N [M]\n");
+        exit (1);
+    }
+    count = strtoul (av[1], NULL, 10);
+    if (ac == 3)
+        spam_max_inflight = strtoul (av[2], NULL, 10);
+    else
+        spam_max_inflight = 1;
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+    if (!(s = flux_attr_get (h, "content.hash"))
+        || !(hash_type = strdup (s))
+        || flux_aux_set (h, "hash_type", hash_type, (flux_free_f)free) < 0)
+        log_err_exit ("getattr content.hash");
+
+    spam_cur_inflight = 0;
+    i = 0;
+    while (i < count || spam_cur_inflight > 0) {
+        while (i < count && spam_cur_inflight < spam_max_inflight) {
+            snprintf (data, size, "spam-o-matic pid=%d seq=%d", getpid(), i);
+            if (!(f = content_store (h, data, size, 0)))
+                log_err_exit ("content_store(%d)", i);
+            if (flux_future_then (f, -1., store_completion, h) < 0)
+                log_err_exit ("flux_future_then(%d)", i);
+            spam_cur_inflight++;
+            i++;
+        }
+        if (flux_reactor_run (flux_get_reactor (h), 0) < 0)
+            log_err ("flux_reactor_run");
+    }
+    flux_close (h);
+    exit (0);
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/t/kvs/fence_api.c b/t/kvs/fence_api.c
index d40e9a9df9fc..79615bb285fe 100644
--- a/t/kvs/fence_api.c
+++ b/t/kvs/fence_api.c
@@ -18,7 +18,6 @@
 #include <pthread.h>
 #include <getopt.h>
 #include <inttypes.h>
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
 
@@ -27,6 +26,7 @@
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/monotime.h"
 #include "src/common/libutil/tstat.h"
+#include "ccan/str/str.h"
 
 typedef struct {
     pthread_t t;
@@ -34,16 +34,31 @@ typedef struct {
     int n;
     flux_t *h;
     char *treeobj;
+    char *rootref;
     int sequence;
 } thd_t;
 
 static int count = -1;
 static char *prefix = NULL;
 static char *fence_name;
+static bool syncflag = false;
+static bool symlinkflag = false;
+static const char *namespace = NULL;
+
+#define OPTIONS "n:Ss"
+static const struct option longopts[] = {
+    {"namespace", required_argument, 0, 'n'},
+    {"sync",      no_argument,       0, 'S'},
+    {"symlink",   no_argument,       0, 's'},
+    {0, 0, 0, 0},
+};
 
 static void usage (void)
 {
-    fprintf (stderr, "Usage: fence_api count prefix\n");
+    fprintf (stderr,
+             "Usage: fence_api "
+             "[--sync] [--symlink] [--namespace=ns] "
+             "count prefix\n");
     exit (1);
 }
 
@@ -55,7 +70,9 @@ void *thread (void *arg)
     flux_future_t *f;
     flux_kvs_txn_t *txn;
     const char *treeobj;
+    const char *rootref;
     int sequence;
+    int flags = 0;
 
     if (!(t->h = flux_open (NULL, 0))) {
         log_err ("%d: flux_open", t->n);
@@ -74,10 +91,19 @@ void *thread (void *arg)
 
     key = xasprintf ("%s.%"PRIu32".%d", prefix, rank, t->n);
 
-    if (flux_kvs_txn_pack (txn, 0, key, "i", 42) < 0)
-        log_err_exit ("%s", key);
+    if (symlinkflag) {
+        if (flux_kvs_txn_symlink (txn, 0, key, NULL, "a-target") < 0)
+            log_err_exit ("%s", key);
+    }
+    else {
+        if (flux_kvs_txn_pack (txn, 0, key, "i", 42) < 0)
+            log_err_exit ("%s", key);
+    }
 
-    if (!(f = flux_kvs_fence (t->h, NULL, 0, fence_name,
+    if (syncflag)
+        flags |= FLUX_KVS_SYNC;
+
+    if (!(f = flux_kvs_fence (t->h, namespace, flags, fence_name,
                               count, txn))
         || flux_future_get (f, NULL) < 0)
         log_err_exit ("flux_kvs_fence");
@@ -86,10 +112,13 @@ void *thread (void *arg)
 
     if (flux_kvs_commit_get_treeobj (f, &treeobj) < 0)
         log_err_exit ("flux_kvs_commit_get_treeobj");
+    if (flux_kvs_commit_get_rootref (f, &rootref) < 0)
+        log_err_exit ("flux_kvs_commit_get_rootref");
     if (flux_kvs_commit_get_sequence (f, &sequence) < 0)
         log_err_exit ("flux_kvs_commit_get_sequence");
 
     t->treeobj = xstrdup (treeobj);
+    t->rootref = xstrdup (rootref);
     t->sequence = sequence;
 
     flux_future_destroy (f);
@@ -106,17 +135,32 @@ void *thread (void *arg)
 int main (int argc, char *argv[])
 {
     thd_t *thd;
-    int i, num, rc;
-
-    log_init (basename (argv[0]));
-
-    if (argc != 3)
+    int i, ch, num, rc;
+
+    log_init (argv[0]);
+
+    while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
+        switch (ch) {
+        case 'S':
+            syncflag = true;
+            break;
+        case 's':
+            symlinkflag = true;
+            break;
+        case 'n':
+            namespace = optarg;
+            break;
+        default:
+            usage ();
+        }
+    }
+    if ((argc - optind) != 2)
         usage ();
 
-    count = strtoul (argv[1], NULL, 10);
+    count = strtoul (argv[optind], NULL, 10);
     if (count <= 1)
         log_msg_exit ("commit count must be > 1");
-    prefix = argv[2];
+    prefix = argv[optind+1];
 
     /* create a fence name for this test that is random-ish */
     srand (time (NULL));
@@ -142,16 +186,21 @@ int main (int argc, char *argv[])
      * should all be the same
      */
     for (i = 1; i < count; i++) {
-        if (strcmp (thd[0].treeobj, thd[i].treeobj))
+        if (!streq (thd[0].treeobj, thd[i].treeobj))
             log_msg_exit ("treeobj mismatch: %s != %s\n",
                           thd[0].treeobj, thd[i].treeobj);
+        if (!streq (thd[0].rootref, thd[i].rootref))
+            log_msg_exit ("rootref mismatch: %s != %s\n",
+                          thd[0].rootref, thd[i].rootref);
         if (thd[0].sequence != thd[i].sequence)
             log_msg_exit ("sequence mismatch: %d != %d\n",
                           thd[0].sequence, thd[i].sequence);
     }
 
-    for (i = 0; i < count; i++)
+    for (i = 0; i < count; i++) {
         free (thd[i].treeobj);
+        free (thd[i].rootref);
+    }
     free (thd);
 
     log_fini ();
diff --git a/t/kvs/fence_invalid.c b/t/kvs/fence_invalid.c
index 6114e48a12a9..cd9b5d07fb25 100644
--- a/t/kvs/fence_invalid.c
+++ b/t/kvs/fence_invalid.c
@@ -16,7 +16,6 @@
 #include <pthread.h>
 #include <getopt.h>
 #include <inttypes.h>
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
 
@@ -46,7 +45,7 @@ int main (int argc, char *argv[])
     flux_kvs_txn_t *txn1 = NULL;
     flux_kvs_txn_t *txn2 = NULL;
 
-    log_init (basename (argv[0]));
+    log_init (argv[0]);
 
     if (argc != 2)
         usage ();
diff --git a/t/kvs/fence_namespace_remove.c b/t/kvs/fence_namespace_remove.c
index d48a34e7c3e3..7d0a1ea9884a 100644
--- a/t/kvs/fence_namespace_remove.c
+++ b/t/kvs/fence_namespace_remove.c
@@ -19,7 +19,6 @@
 #include <pthread.h>
 #include <getopt.h>
 #include <inttypes.h>
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
 
@@ -47,7 +46,7 @@ int main (int argc, char *argv[])
     flux_future_t *f = NULL;
     flux_kvs_txn_t *txn = NULL;
 
-    log_init (basename (argv[0]));
+    log_init (argv[0]);
 
     if (argc != 3)
         usage ();
diff --git a/t/kvs/hashtest.c b/t/kvs/hashtest.c
deleted file mode 100644
index 4f98413b6d17..000000000000
--- a/t/kvs/hashtest.c
+++ /dev/null
@@ -1,725 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <sys/time.h>
-#include <sys/resource.h>
-#include <assert.h>
-#include <stdarg.h>
-#include <czmq.h>
-#include <stdlib.h>
-#if HAVE_LIBJUDY
-#include <Judy.h>
-#endif
-#if HAVE_SOPHIA
-#include <sophia.h>
-#endif
-#include <sqlite3.h>
-
-#include "src/common/libutil/xzmalloc.h"
-#include "src/common/libutil/monotime.h"
-#include "src/common/libutil/cleanup.h"
-#include "src/common/libutil/log.h"
-#if HAVE_LSD_HASH
-#include "src/common/liblsd/hash.h"
-#endif
-#if HAVE_HATTRIE
-#include "src/common/libhat-trie/hat-trie.h"
-#endif
-
-//const int num_keys = 1024*1024;
-const int num_keys = 1024*1024*10;
-
-struct hash_impl {
-    void (*destroy)(struct hash_impl *h);
-    void (*insert)(struct hash_impl *h, zlist_t *items);
-    void (*lookup)(struct hash_impl *h, zlist_t *items);
-    void *h;
-};
-
-struct item {
-    zdigest_t *zd;
-    char data[16];
-    uint8_t key[20];
-    char skey[41];
-};
-
-void rusage (struct rusage *res)
-{
-    int rc = getrusage (RUSAGE_SELF, res);
-    assert (rc == 0);
-}
-
-long int rusage_maxrss_since (struct rusage *res)
-{
-    struct rusage new;
-    int rc = getrusage (RUSAGE_SELF, &new);
-    assert (rc == 0);
-    return new.ru_maxrss - res->ru_maxrss;
-}
-
-struct item *item_create (int id)
-{
-    struct item *item = xzmalloc (sizeof (*item));
-    assert (item != NULL);
-    snprintf (item->data, sizeof (item->data), "%d", id);
-
-    item->zd = zdigest_new ();
-    assert (item->zd != NULL);
-    zdigest_update (item->zd, (uint8_t *)item->data, sizeof (item->data));
-
-    assert (zdigest_size (item->zd) == sizeof (item->key));
-    memcpy (item->key, zdigest_data (item->zd), zdigest_size (item->zd));
-
-    assert (strlen (zdigest_string (item->zd)) == sizeof (item->skey) - 1);
-    strcpy (item->skey, zdigest_string (item->zd));
-    zdigest_destroy (&item->zd);
-
-    return item;
-}
-
-void item_destroy (void *arg)
-{
-    struct item *item = arg;
-    free (item);
-}
-
-zlist_t *create_items (void)
-{
-    zlist_t *items = zlist_new ();
-    struct item *item;
-    int i, rc;
-
-    assert (items != NULL);
-    for (i = 0; i < num_keys; i++) {
-        item = item_create (i);
-        rc = zlist_append (items, item);
-        assert (rc == 0);
-    }
-    return items;
-}
-
-/* zhash
- */
-
-void insert_zhash (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item;
-    int rc;
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        rc = zhash_insert (impl->h, item->skey, item);
-        assert (rc == 0);
-        item = zlist_next (items);
-    }
-}
-
-void lookup_zhash (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item, *ip;
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        ip = zhash_lookup (impl->h, item->skey);
-        assert (ip != NULL);
-        assert (ip == item);
-        item = zlist_next (items);
-    }
-}
-
-void destroy_zhash (struct hash_impl *impl)
-{
-    zhash_t *zh = impl->h;
-    zhash_destroy (&zh);
-    free (impl);
-}
-
-struct hash_impl *create_zhash (void)
-{
-    struct hash_impl *impl = xzmalloc (sizeof (*impl));
-    impl->h = zhash_new ();
-    assert (impl->h != NULL);
-    impl->insert = insert_zhash;
-    impl->lookup = lookup_zhash;
-    impl->destroy = destroy_zhash;
-    return impl;
-}
-
-/* zhashx
- */
-#if HAVE_ZHASHX_NEW
-void insert_zhashx (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item;
-    int rc;
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        rc = zhashx_insert (impl->h, item->key, item);
-        assert (rc == 0);
-        item = zlist_next (items);
-    }
-}
-
-void lookup_zhashx (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item, *ip;
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        ip = zhashx_lookup (impl->h, item->key);
-        assert (ip != NULL);
-        assert (ip == item);
-        item = zlist_next (items);
-    }
-}
-
-void destroy_zhashx (struct hash_impl *impl)
-{
-    zhashx_t *zh = impl->h;
-    zhashx_destroy (&zh);
-    free (impl);
-}
-
-
-void *duplicator_zhashx (const void *item)
-{
-    return (void *)item;
-}
-
-void destructor_zhashx (void **item)
-{
-    *item = NULL;
-}
-
-int comparator_zhashx (const void *item1, const void *item2)
-{
-    return memcmp (item1, item2, 20);
-}
-
-size_t hasher_zhashx (const void *item)
-{
-    return *(size_t *)item;
-}
-
-struct hash_impl *create_zhashx (void)
-{
-    struct hash_impl *impl = xzmalloc (sizeof (*impl));
-    impl->h = zhashx_new ();
-    assert (impl->h != NULL);
-
-    /* Use 20 byte keys that are not copied.
-     */
-    zhashx_set_key_duplicator (impl->h, duplicator_zhashx);
-    zhashx_set_key_destructor (impl->h, destructor_zhashx);
-    zhashx_set_key_comparator (impl->h, comparator_zhashx);
-    zhashx_set_key_hasher (impl->h, hasher_zhashx);
-
-    impl->insert = insert_zhashx;
-    impl->lookup = lookup_zhashx;
-    impl->destroy = destroy_zhashx;
-    return impl;
-}
-#endif /* HAVE_ZHASHX_NEW */
-
-/* lsd-hash
- */
-
-#if HAVE_LSD_HASH
-void insert_lsd (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item, *ip;
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        ip = hash_insert (impl->h, item->key, item);
-        assert (ip != NULL);
-        item = zlist_next (items);
-    }
-}
-
-void lookup_lsd (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item, *ip;
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        ip = hash_find (impl->h, item->key);
-        assert (ip != NULL);
-        assert (ip == item);
-        item = zlist_next (items);
-    }
-}
-
-void destroy_lsd (struct hash_impl *impl)
-{
-    hash_destroy (impl->h);
-    free (impl);
-}
-
-unsigned int hash_lsd (const void *key)
-{
-    return *(unsigned int *)key; /* 1st 4 bytes of sha1 */
-}
-
-int cmp_lsd (const void *key1, const void *key2)
-{
-    return memcmp (key1, key2, 20);
-}
-
-struct hash_impl *create_lsd (void)
-{
-    struct hash_impl *impl = xzmalloc (sizeof (*impl));
-    impl->h = hash_create (1024*1024*8, hash_lsd, cmp_lsd, NULL);
-    assert (impl->h != NULL);
-
-    impl->insert = insert_lsd;
-    impl->lookup = lookup_lsd;
-    impl->destroy = destroy_lsd;
-
-    return impl;
-}
-#endif
-
-/* judy
- */
-
-#if HAVE_LIBJUDY
-void insert_judy (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item;
-    //Pvoid_t array = NULL;
-    PWord_t valp;
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        JHSI (valp, impl->h, item->key, sizeof (item->key));
-        assert (valp != PJERR);
-        assert (*valp == (uintptr_t)0); /* nonzero indicates dup */
-        *valp = (uintptr_t)item;
-        item = zlist_next (items);
-    }
-}
-
-void lookup_judy (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item;
-    //Pvoid_t array = NULL;
-    PWord_t valp;
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        JHSG (valp, impl->h, item->key, sizeof (item->key));
-        assert (valp != NULL);
-        assert (*valp == (uintptr_t)item);
-        item = zlist_next (items);
-    }
-}
-
-void destroy_judy (struct hash_impl *impl)
-{
-    Word_t bytes;
-    JHSFA (bytes, impl->h);
-    msg ("judy freed %lu Kbytes of memory", bytes / 1024);
-    free (impl);
-}
-
-struct hash_impl *create_judy (void)
-{
-    struct hash_impl *impl = xzmalloc (sizeof (*impl));
-
-    impl->insert = insert_judy;
-    impl->lookup = lookup_judy;
-    impl->destroy = destroy_judy;
-
-    return impl;
-}
-#endif
-
-/* sophia
- */
-
-#if HAVE_SOPHIA
-void log_sophia_error (void *env, const char *fmt, ...)
-{
-    va_list ap;
-    va_start (ap, fmt);
-    char *s = xvasprintf (fmt, ap);
-    va_end (ap);
-
-    int error_size;
-    char *error = NULL;
-    if (env)
-        error = sp_getstring (env, "sophia.error", &error_size);
-    fprintf (stderr, "%s: %s", s, error ? error : "failure");
-    if (error)
-        free (error);
-    free (s);
-}
-
-void insert_sophia (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item;
-    void *db, *o;
-    int rc;
-
-    db = sp_getobject (impl->h, "db.test");
-    if (!db)
-        log_sophia_error (impl->h, "db.test");
-    assert (db != NULL);
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        // existence check slows down by about 23X! */
-        //o = sp_object (db);
-        //assert (o != NULL);
-        //rc = sp_setstring (o, "key", item->key, sizeof (item->key));
-        //assert (rc == 0);
-        //void *result = sp_get (db, o); /* destroys 'o' */
-        //assert (result == NULL);
-
-        o = sp_object (db);
-        assert (o != NULL);
-        rc = sp_setstring (o, "key", item->key, sizeof (item->key));
-        assert (rc == 0);
-        rc = sp_setstring (o, "value", item, sizeof (*item));
-        assert (rc == 0);
-        rc = sp_set (db, o); /* destroys 'o', copies item  */
-        assert (rc == 0);
-        item = zlist_next (items);
-    }
-
-    sp_destroy (db);
-}
-
-void lookup_sophia (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item, *ip;
-    void *db, *o, *result;
-    int rc, item_size;
-    int count = 0;
-
-    db = sp_getobject (impl->h, "db.test");
-    if (!db)
-        log_sophia_error (impl->h, "db.test");
-    assert (db != NULL);
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        o = sp_object (db);
-        assert (o != NULL);
-        rc = sp_setstring (o, "key", item->key, sizeof (item->key));
-        assert (rc == 0);
-        result = sp_get (db, o); /* destroys 'o' */
-        assert (result != NULL);
-        ip = sp_getstring (result, "value", &item_size);
-        assert (ip != NULL);
-        assert (item_size == sizeof (*item));
-        assert (memcmp (ip, item, item_size) == 0);
-        sp_destroy (result);
-        item = zlist_next (items);
-        count++;
-        //if (count % 10000 == 0)
-        //    msg ("lookup: %d of %d", count, num_keys);
-    }
-
-    sp_destroy (db);
-}
-
-void destroy_sophia (struct hash_impl *impl)
-{
-    sp_destroy (impl->h);
-    free (impl);
-}
-
-struct hash_impl *create_sophia (void)
-{
-    struct hash_impl *impl = xzmalloc (sizeof (*impl));
-    char template[] = "/tmp/hashtest-sophia.XXXXXX";
-    char *path;
-    int rc;
-
-    path = mkdtemp (template);
-    assert (path != NULL);
-    cleanup_push_string (cleanup_directory_recursive, path);
-    log_msg ("sophia.path: %s", path);
-    impl->h = sp_env ();
-    assert (impl->h != NULL);
-    rc = sp_setstring (impl->h, "sophia.path", path, 0);
-    assert (rc == 0);
-    // 16m limit increases memory used during insert by about 2X
-    //rc = sp_setint (impl->h, "memory.limit", 1024*1024*16);
-    //assert (rc == 0);
-    rc = sp_setstring (impl->h, "db", "test", 0);
-    assert (rc == 0);
-    rc = sp_setstring (impl->h, "db.test.index.key", "string", 0);
-    assert (rc == 0);
-    // N.B. lz4 slows down lookups by about 4X
-    //rc = sp_setstring (impl->h, "db.test.compression", "lz4", 0);
-    //assert (rc == 0);
-    rc = sp_open (impl->h);
-    assert (rc == 0);
-
-    impl->insert = insert_sophia;
-    impl->lookup = lookup_sophia;
-    impl->destroy = destroy_sophia;
-
-    return impl;
-}
-#endif
-
-#if HAVE_HATTRIE
-void insert_hat (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item;
-    value_t *val;
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        val = hattrie_get (impl->h, (char *)item->key, sizeof (item->key));
-        assert (val != NULL);
-        assert (*val == 0);
-        *(void **)val = item;
-        item = zlist_next (items);
-    }
-}
-
-void lookup_hat (struct hash_impl *impl, zlist_t *items)
-{
-    struct item *item;
-    value_t *val;
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        val = hattrie_tryget (impl->h, (char *)item->key, sizeof (item->key));
-        assert (*(void **)val == item);
-        item = zlist_next (items);
-    }
-}
-
-void destroy_hat (struct hash_impl *impl)
-{
-    hattrie_free (impl->h);
-    free (impl);
-}
-
-struct hash_impl *create_hat (void)
-{
-    struct hash_impl *impl = xzmalloc (sizeof (*impl));
-    impl->h = hattrie_create ();
-    assert (impl->h != NULL);
-    impl->insert = insert_hat;
-    impl->lookup = lookup_hat;
-    impl->destroy = destroy_hat;
-    return impl;
-}
-#endif
-
-void insert_sqlite (struct hash_impl *impl, zlist_t *items)
-{
-    const char *sql = "INSERT INTO objects (hash,object) values (?1, ?2)";
-    sqlite3_stmt *stmt;
-    int rc;
-    struct item *item;
-
-    rc = sqlite3_prepare_v2 (impl->h, sql, -1, &stmt, NULL);
-    assert (rc == SQLITE_OK);
-
-    rc = sqlite3_exec (impl->h, "BEGIN", 0, 0, 0);
-    assert (rc == SQLITE_OK);
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        rc = sqlite3_bind_text (stmt, 1, (char *)item->key,
-                                sizeof (item->key), SQLITE_STATIC);
-        assert (rc == SQLITE_OK);
-        rc = sqlite3_bind_blob (stmt, 2, item, sizeof (*item), SQLITE_STATIC);
-        assert (rc == SQLITE_OK);
-        rc = sqlite3_step (stmt);
-        assert (rc == SQLITE_DONE);
-        rc = sqlite3_reset (stmt);
-        assert (rc == SQLITE_OK);
-
-        item = zlist_next (items);
-    }
-    rc = sqlite3_exec (impl->h, "COMMIT", 0, 0, 0);
-    assert (rc == SQLITE_OK);
-
-    sqlite3_finalize (stmt);
-}
-
-void lookup_sqlite (struct hash_impl *impl, zlist_t *items)
-{
-    const char *sql = "SELECT object FROM objects WHERE hash = ?1 LIMIT 1";
-    sqlite3_stmt *stmt;
-    int rc;
-    struct item *item;
-    const void *val;
-
-    rc = sqlite3_prepare_v2 (impl->h, sql, -1, &stmt, NULL);
-    assert (rc == SQLITE_OK);
-
-    item = zlist_first (items);
-    while (item != NULL) {
-        rc = sqlite3_bind_text (stmt, 1, (char *)item->key,
-                                sizeof (item->key), SQLITE_STATIC);
-        assert (rc == SQLITE_OK);
-        rc = sqlite3_step (stmt);
-        assert (rc == SQLITE_ROW);
-        assert (sqlite3_column_type (stmt, 0) == SQLITE_BLOB);
-        assert (sqlite3_column_bytes (stmt, 0) == sizeof (*item));
-        val = sqlite3_column_blob (stmt, 0);
-        assert (val != NULL);
-        assert (memcmp (val, item, sizeof (*item)) == 0);
-
-        rc = sqlite3_step (stmt);
-        assert (rc == SQLITE_DONE);
-        rc = sqlite3_reset (stmt);
-        assert (rc == SQLITE_OK);
-
-        item = zlist_next (items);
-    }
-
-    sqlite3_finalize (stmt);
-}
-
-void destroy_sqlite (struct hash_impl *impl)
-{
-    sqlite3_close (impl->h);
-}
-
-struct hash_impl *create_sqlite (void)
-{
-    int rc;
-    sqlite3 *db;
-    char template[] = "/tmp/hashtest-sqlite.XXXXXX";
-    char *path;
-    struct hash_impl *impl = xzmalloc (sizeof (*impl));
-    char dbpath[PATH_MAX];
-
-    path = mkdtemp (template);
-    assert (path != NULL);
-    cleanup_push_string (cleanup_directory_recursive, path);
-    log_msg ("sqlite path: %s", path);
-
-    snprintf (dbpath, sizeof (dbpath), "%s/db", path);
-    rc = sqlite3_open (dbpath, &db);
-    assert (rc == SQLITE_OK);
-
-    // avoid creating journal
-    rc = sqlite3_exec (db, "PRAGMA journal_mode=OFF", NULL, NULL, NULL);
-    assert (rc == SQLITE_OK);
-
-    // avoid fsync
-    rc = sqlite3_exec (db, "PRAGMA synchronous=OFF", NULL, NULL, NULL);
-    assert (rc == SQLITE_OK);
-
-    // avoid mutex locking
-    rc = sqlite3_exec (db, "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL, NULL);
-    assert (rc == SQLITE_OK);
-
-    // raise max db pages cached in memory from 2000
-    //rc = sqlite3_exec (db, "PRAGMA cache_size=16000", NULL, NULL, NULL);
-    //assert (rc == SQLITE_OK);
-
-    // raise db page size from default 1024 bytes
-    // N.B. must set before table create
-    //rc = sqlite3_exec (db, "PRAGMA page_size=4096", NULL, NULL, NULL);
-    //assert (rc == SQLITE_OK);
-
-    rc = sqlite3_exec (db, "CREATE TABLE objects("
-            "hash CHAR(20) PRIMARY KEY,"
-            "object BLOB"
-            ");", NULL, NULL, NULL);
-    assert (rc == SQLITE_OK);
-
-    impl->h = db;
-    impl->insert = insert_sqlite;
-    impl->lookup = lookup_sqlite;
-    impl->destroy = destroy_sqlite;
-    return impl;
-}
-
-void usage (void)
-{
-    fprintf (stderr, "Usage: hashtest zhash | zhashx | judy | lsd | hat"
-           " | sophia | sqlite\n");
-    exit (1);
-}
-
-int main (int argc, char *argv[])
-{
-    zlist_t *items;
-    struct hash_impl *impl = NULL;
-    struct timespec t0;
-    struct rusage res;
-
-    if (argc == 1)
-        usage ();
-
-    rusage (&res);
-    monotime (&t0);
-    if (!strcmp (argv[1], "zhash"))
-        impl = create_zhash ();
-#if HAVE_ZHASHX_NEW
-    else if (!strcmp (argv[1], "zhashx"))
-        impl = create_zhashx ();
-#endif
-#if HAVE_LSD_HASH
-    else if (!strcmp (argv[1], "lsd"))
-        impl = create_lsd ();
-#endif
-#if HAVE_LIBJUDY
-    else if (!strcmp (argv[1], "judy"))
-        impl = create_judy ();
-#endif
-#if HAVE_SOPHIA
-    else if (!strcmp (argv[1], "sophia"))
-        impl = create_sophia ();
-#endif
-#if HAVE_HATTRIE
-    else if (!strcmp (argv[1], "hat"))
-        impl = create_hat ();
-#endif
-    else if (!strcmp (argv[1], "sqlite"))
-        impl = create_sqlite ();
-    if (!impl)
-        usage ();
-    log_msg ("create hash: %.2fs (%+ldK)", monotime_since (t0) * 1E-3,
-                                       rusage_maxrss_since (&res));
-
-    rusage (&res);
-    monotime (&t0);
-    items = create_items ();
-    log_msg ("create items: %.2fs (%+ldK)", monotime_since (t0) * 1E-3,
-                                        rusage_maxrss_since (&res));
-
-    rusage (&res);
-    monotime (&t0);
-    impl->insert (impl, items);
-    log_msg ("insert items: %.2fs (%+ldK)", monotime_since (t0) * 1E-3,
-                                        rusage_maxrss_since (&res));
-
-    rusage (&res);
-    monotime (&t0);
-    impl->lookup (impl, items);
-    log_msg ("lookup items: %.2fs (%+ldK)", monotime_since (t0) * 1E-3,
-                                        rusage_maxrss_since (&res));
-
-    impl->destroy (impl);
-    return 0;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/t/kvs/issue1876.c b/t/kvs/issue1876.c
index 090916f913f5..385f77252173 100644
--- a/t/kvs/issue1876.c
+++ b/t/kvs/issue1876.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <flux/core.h>
 #include "src/common/libutil/log.h"
 
@@ -28,7 +31,8 @@ int main (int argc, char *argv[])
     for (i = 0; i < 1000; i++) {
         flux_future_t *f;
 
-        log_msg ("loop=%d", i);
+        if (i % 100 == 0)
+            log_msg ("loop=%d", i);
 
         if (!(f = flux_kvs_lookup (h, NULL, FLUX_KVS_WATCH
                                    | FLUX_KVS_WAITCREATE, key)))
diff --git a/t/kvs/lookup_invalid.c b/t/kvs/lookup_invalid.c
index 78855dda8aec..878bc8191e17 100644
--- a/t/kvs/lookup_invalid.c
+++ b/t/kvs/lookup_invalid.c
@@ -14,9 +14,7 @@
 #include <assert.h>
 #include <libgen.h>
 #include <pthread.h>
-#include <getopt.h>
 #include <inttypes.h>
-#include <czmq.h>
 #include <jansson.h>
 #include <flux/core.h>
 
@@ -36,7 +34,7 @@ int main (int argc, char *argv[])
     char *key = NULL;
     flux_future_t *f = NULL;
 
-    log_init (basename (argv[0]));
+    log_init (argv[0]);
 
     if (argc != 2)
         usage ();
diff --git a/t/kvs/torture.c b/t/kvs/torture.c
index cf0450277ac8..f53f2dbd58d3 100644
--- a/t/kvs/torture.c
+++ b/t/kvs/torture.c
@@ -19,13 +19,13 @@
 #include <stdbool.h>
 #include <stdarg.h>
 #include <inttypes.h>
-#include <czmq.h>
 #include <flux/core.h>
 
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/monotime.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/oom.h"
+#include "ccan/str/str.h"
 
 
 #define OPTIONS "hc:s:p:qv"
@@ -152,7 +152,7 @@ int main (int argc, char *argv[])
             log_err_exit ("flux_kvs_lookup '%s'", key);
         if (verbose)
             log_msg ("%s = %s", key, s);
-        if (strcmp (s, val) != 0)
+        if (!streq (s, val))
             log_msg_exit ("kvs_lookup: key '%s' wrong value '%s'", key, s);
         free (key);
         flux_future_destroy (f);
diff --git a/t/kvs/transactionmerge.c b/t/kvs/transactionmerge.c
index 396d8e596b43..d06c2304de25 100644
--- a/t/kvs/transactionmerge.c
+++ b/t/kvs/transactionmerge.c
@@ -84,10 +84,6 @@ static void watch_count_cb (flux_future_t *f, void *arg)
 
     flux_future_reset (f);
 
-    /* re-call to set timeout */
-    if (flux_future_then (f, WATCH_TIMEOUT, watch_count_cb, arg) < 0)
-        log_err_exit ("flux_future_then");
-
     if (changecount == threadcount) {
         if (flux_kvs_lookup_cancel (f) < 0)
             log_err_exit ("flux_kvs_lookup_cancel");
@@ -126,7 +122,7 @@ int main (int argc, char *argv[])
     flux_reactor_t *r;
     int i, rc, ch;
 
-    log_init (basename (argv[0]));
+    log_init (argv[0]);
 
     while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
         switch (ch) {
diff --git a/t/kvs/waitcreate_cancel.c b/t/kvs/waitcreate_cancel.c
index 175ed54f42be..f610bf1269ad 100644
--- a/t/kvs/waitcreate_cancel.c
+++ b/t/kvs/waitcreate_cancel.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <flux/core.h>
 #include "src/common/libutil/log.h"
 
diff --git a/t/kvs/watch_disconnect.c b/t/kvs/watch_disconnect.c
index 07516d50b7ee..31b4a1cca286 100644
--- a/t/kvs/watch_disconnect.c
+++ b/t/kvs/watch_disconnect.c
@@ -59,10 +59,10 @@ int count_watchers (flux_t *h)
     if (flux_get_size (h, &size) < 0)
         log_err_exit ("flux_get_size");
     for (rank = 0; rank < size; rank++) {
-        if (!(f = flux_rpc (h, "kvs-watch.stats.get", NULL, rank, 0)))
-            log_err_exit ("flux_rpc kvs-watch.stats.get");
+        if (!(f = flux_rpc (h, "kvs-watch.stats-get", NULL, rank, 0)))
+            log_err_exit ("flux_rpc kvs-watch.stats-get");
         if (flux_rpc_get_unpack (f, "{ s:i }", "watchers", &n) < 0)
-            log_err_exit ("kvs-watch.stats.get");
+            log_err_exit ("kvs-watch.stats-get");
         count += n;
         flux_future_destroy (f);
     }
diff --git a/t/lua/t0001-send-recv.t b/t/lua/t0001-send-recv.t
index 03c92f1973bf..4f924cd26315 100755
--- a/t/lua/t0001-send-recv.t
+++ b/t/lua/t0001-send-recv.t
@@ -17,7 +17,7 @@ is (err, nil, "error is nil")
 --
 
 local packet = { seq = "1", pad = "xxxxxx" }
-local matchtag, err = f:send ("cmb.ping", packet)
+local matchtag, err = f:send ("broker.ping", packet)
 isnt (rc, 0, "send: rc is not 0")
 is (err, nil, "send: err is nil")
 
@@ -25,27 +25,27 @@ is (err, nil, "send: err is nil")
 local msg, tag = f:recv ()
 is (msg.seq, "1", "recv: got expected ping sequence")
 is (msg.pad, "xxxxxx", "recv: got expected ping pad")
-is (tag, "cmb.ping", "recv: got expected tag on ping response")
+is (tag, "broker.ping", "recv: got expected tag on ping response")
 
 for k,v in pairs(msg) do note ("msg."..k.."="..v) end
 
 --
 --  Test f:send() with rank argument
 --
-local rc, err = f:send ("cmb.ping", packet, 0)
+local rc, err = f:send ("broker.ping", packet, 0)
 isnt (rc, 0, "send to rank 0: rc is not nil or zero")
 is (err, nil, "send to rank 0: err is nil")
 
 local msg, tag = f:recv ()
 is (msg.seq, "1", "recv: got expected ping sequence")
 is (msg.pad, "xxxxxx", "recv: got expected ping pad")
-is (tag, "cmb.ping", "recv: got expected tag on ping response")
+is (tag, "broker.ping", "recv: got expected tag on ping response")
 
 
 ---
 ---  Test send with recvmsg()
 ---
-local matchtag, err = f:send ("cmb.ping", packet, 1)
+local matchtag, err = f:send ("broker.ping", packet, 1)
 isnt (rc, 0, "send to rank 1: rc is not nil or zero")
 isnt (rc, nil, "send to rank 1: rc is not nil or zero")
 is (err, nil, "send to rank 1: err is nil")
@@ -56,7 +56,7 @@ is (msg.matchtag, matchtag, "recvmsg: matchtag matches (".. matchtag..")")
 is (msg.errnum, 0, "recvmsg: message errnum is 0")
 is (msg.data.seq, "1", "recv: got expected ping sequence")
 is (msg.data.pad, "xxxxxx", "recv: got expected ping pad")
-is (msg.tag, "cmb.ping", "recv: got expected tag on ping response")
+is (msg.tag, "broker.ping", "recv: got expected tag on ping response")
 
 
 f, err = flux.new ("loop://")
diff --git a/t/lua/t0002-rpc.t b/t/lua/t0002-rpc.t
index fddad9590d9a..09fde4a66a3a 100755
--- a/t/lua/t0002-rpc.t
+++ b/t/lua/t0002-rpc.t
@@ -5,7 +5,7 @@
 local test = require 'fluxometer'.init (...)
 test:start_session { size=2 }
 
-plan (19)
+plan (18)
 
 local flux = require_ok ('flux')
 local f, err = flux.new()
@@ -14,13 +14,12 @@ is (err, nil, "error is nil")
 
 is (f.rank, 0, "running on rank 0")
 is (f.size, 2, "session size is 2")
-is (f.arity, 2, "session arity is 2")
 
 --
 --  Use 'ping' packet to test rpc
 --
 local packet = { seq = "1", pad = "xxxxxx" }
-local msg, err = f:rpc ("cmb.ping", packet)
+local msg, err = f:rpc ("broker.ping", packet)
 type_ok (msg, 'table', "rpc: return type is table")
 is (err, nil, "rpc: err is nil")
 is (msg.seq, "1", "recv: got expected ping sequence")
@@ -30,15 +29,15 @@ is (msg.pad, "xxxxxx", "recv: got expected ping pad")
 --  Send invalid 'ping' packet to test response
 --
 local packet = { 1, 2, 3 }  -- Force encoding to be 'array'
-local msg, err = f:rpc ("cmb.ping", packet)
+local msg, err = f:rpc ("broker.ping", packet)
 is (msg, nil, "rpc: invalid packet: nil response indicates error")
 is (err, "Invalid argument", "rpc: invalid packet: err is 'Invalid argument'")
 
 --
--- 'ping' to specfic rank
+-- 'ping' to specific rank
 --
 local packet = { seq = "1", pad = "xxxxxx" }
-local msg, err = f:rpc ("cmb.ping", packet, 1)
+local msg, err = f:rpc ("broker.ping", packet, 1)
 type_ok (msg, 'table', "rpc: return type is table")
 is (err, nil, "rpc: err is nil")
 is (msg.seq, "1", "recv: got expected ping sequence")
@@ -50,8 +49,8 @@ is (msg.errnum, nil, "recv: errnum is zero")
 --
 note ("ping to non-existent rank")
 local packet = { seq = "1", pad = "xxxxxx" }
-local msg, err = f:rpc ("cmb.ping", packet, 99)
+local msg, err = f:rpc ("broker.ping", packet, 99)
 is (msg, nil, "rpc: nil return indicates error")
-is (err, "No route to host", "rpc: err is 'No route to host'")
+ok (err == "No route to host" or err == "Host is unreachable", "rpc: err is 'No route to host' or 'Host is unreachable'")
 
 done_testing ()
diff --git a/t/lua/t0003-events.t b/t/lua/t0003-events.t
index d5b504c784ea..b83190037c6a 100755
--- a/t/lua/t0003-events.t
+++ b/t/lua/t0003-events.t
@@ -55,36 +55,36 @@ type_ok (msg, 'table', "recv_event: got msg as a table")
 is_deeply (msg, {}, "recv_event: got empty payload as expected")
 
 
--- poke at event.pub service
+-- poke at event.publish service
 -- good request, no payload
 local request = { topic = "foo", flags = 0 }
-local response, err = f:rpc ("event.pub", request);
-is (err, nil, "event.pub: works without payload")
+local response, err = f:rpc ("event.publish", request);
+is (err, nil, "event.publish: works without payload")
 
 -- good request, with raw payload
 local request = { topic = "foo", flags = 0, payload = "aGVsbG8gd29ybGQ=" }
-local response, err = f:rpc ("event.pub", request);
-is (err, nil, "event.pub: works with payload")
+local response, err = f:rpc ("event.publish", request);
+is (err, nil, "event.publish: works with payload")
 
 -- good request, with JSON "{}\0"
 local request = { topic = "foo", flags = 0, payload = "e30A" }
-local response, err = f:rpc ("event.pub", request);
-is (err, nil, "event.pub: works with json payload")
+local response, err = f:rpc ("event.publish", request);
+is (err, nil, "event.publish: works with json payload")
 
 -- flags missing from request
 local request = { topic = "foo" }
-local response, err = f:rpc ("event.pub", request);
-is (err, "Protocol error", "event.pub: no flags, fails with EPROTO")
+local response, err = f:rpc ("event.publish", request);
+is (err, "Protocol error", "event.publish: no flags, fails with EPROTO")
 
 -- mangled base64 payload
-local request = { topic = "foo", flags = 0, payload = "aGVsbG8gd29ybGQ" }
-local response, err = f:rpc ("event.pub", request);
-is (err, "Protocol error", "event.pub: bad base64, fails with EPROTO")
+local request = { topic = "foo", flags = 0, payload = "aGVsbG8gd29ybGQ%" }
+local response, err = f:rpc ("event.publish", request);
+is (err, "Protocol error", "event.publish: bad base64, fails with EPROTO")
 
 -- good request, mangled JSON payload "{\0"
 local request = { topic = "foo", flags = 4, payload = "ewA=" }
-local response, err = f:rpc ("event.pub", request);
-is (err, "Protocol error", "event.pub: bad json payload, fails with EPROTO")
+local response, err = f:rpc ("event.publish", request);
+is (err, "Protocol error", "event.publish: bad json payload, fails with EPROTO")
 
 done_testing ()
 
diff --git a/t/lua/t1001-timeouts.t b/t/lua/t1001-timeouts.t
index 5229ee1bdb71..ee092f02c29e 100755
--- a/t/lua/t1001-timeouts.t
+++ b/t/lua/t1001-timeouts.t
@@ -26,7 +26,7 @@ local to, err = f:timer {
 }
 type_ok (to, 'userdata', "created timeout handler")
 is (err, nil, "error from timer create is nil")
-is (to.timeout, 250, 'timeout is 500ms')
+is (to.timeout, 250, 'timeout is 250ms')
 type_ok (to.id, 'number', 'new timeout id is '..to.id)
 
 local r = f:reactor()
diff --git a/t/marshall/invalid/v0.46.1.data b/t/marshall/invalid/v0.46.1.data
new file mode 100644
index 000000000000..ac4f5d12ecd0
Binary files /dev/null and b/t/marshall/invalid/v0.46.1.data differ
diff --git a/t/marshall/valid/v0.47.data b/t/marshall/valid/v0.47.data
new file mode 100644
index 000000000000..38e344f6062c
Binary files /dev/null and b/t/marshall/valid/v0.47.data differ
diff --git a/t/module/child.c b/t/module/child.c
deleted file mode 100644
index dc85f9f6e808..000000000000
--- a/t/module/child.c
+++ /dev/null
@@ -1,33 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <flux/core.h>
-
-/* Validate that hardwired arguments were passed.
- * Return 0 on success, -1 on failure with errno set.
- */
-int mod_main (void *ctx, int argc, char *argv[])
-{
-    if (argc != 2 || strcmp (argv[0], "foo=42") != 0
-                  || strcmp (argv[1], "bar=abcd") != 0)
-        return -1;
-    return 0;
-}
-
-/* This dso extends a comms module named "parent".
- */
-MOD_NAME ("parent.child");
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/t/module/legacy.c b/t/module/legacy.c
new file mode 100644
index 000000000000..6b0e5fd8d1be
--- /dev/null
+++ b/t/module/legacy.c
@@ -0,0 +1,22 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* Do not include project config.h, we are testing public module
+ * interface */
+#include <flux/core.h>
+
+int mod_main (flux_t *h, int argc, char **argv)
+{
+    return flux_reactor_run (flux_get_reactor (h), 0);
+}
+
+MOD_NAME("legacy");
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/module/parent.c b/t/module/parent.c
deleted file mode 100644
index f7e9da561d5f..000000000000
--- a/t/module/parent.c
+++ /dev/null
@@ -1,263 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <libgen.h>
-#include <dlfcn.h>
-#include <argz.h>
-#include <flux/core.h>
-#include <czmq.h>
-#include <jansson.h>
-
-#include "src/common/libutil/xzmalloc.h"
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/oom.h"
-
-typedef struct {
-    char *name;
-    int size;
-    char *digest;
-    int idle;
-    int status;
-    void *dso;
-    mod_main_f *main;
-} module_t;
-
-static zhash_t *modules = NULL;
-static uint32_t rank;
-
-/* Calculate file digest using zfile() class from czmq.
- * Caller must free.
- */
-char *digest (const char *path)
-{
-    zfile_t *zf = zfile_new (NULL, path);
-    char *digest = NULL;
-    if (zf)
-        digest = xstrdup (zfile_digest (zf));
-    zfile_destroy (&zf);
-    return digest;
-}
-
-static void module_destroy (module_t *m)
-{
-    if (m->name)
-        free (m->name);
-    if (m->digest)
-        free (m->digest);
-    if (m->dso)
-        dlclose (m->dso);
-    free (m);
-}
-
-static module_t *module_create (const char *path, char *argz, size_t argz_len)
-{
-    module_t *m = xzmalloc (sizeof (*m));
-    struct stat sb;
-    char **av = NULL;
-
-    if (stat (path, &sb) < 0 || !(m->name = flux_modname (path, NULL, NULL))
-                             || !(m->digest = digest (path))) {
-        module_destroy (m);
-        errno = ESRCH;
-        return NULL;
-    }
-    m->size = sb.st_size;
-    m->dso = dlopen (path, RTLD_NOW | RTLD_LOCAL);
-    if (!m->dso || !(m->main = dlsym (m->dso, "mod_main"))) {
-        module_destroy (m);
-        errno = EINVAL;
-        return NULL;
-    }
-    av = xzmalloc (sizeof (av[0]) * (argz_count (argz, argz_len) + 1));
-    argz_extract (argz, argz_len, av);
-    if (m->main (NULL, argz_count (argz, argz_len), av) < 0) {
-        module_destroy (m);
-        errno = EINVAL;
-        return NULL;
-    }
-    if (zhash_lookup (modules, m->name)) {
-        module_destroy (m);
-        errno = EEXIST;
-        return NULL;
-    }
-    zhash_update (modules, m->name, m);
-    zhash_freefn (modules, m->name, (zhash_free_fn *)module_destroy);
-    if (av)
-        free (av);
-    return m;
-}
-
-/* N.B. services is hardwired to test1,test2,testN, where N is the local
- * broker rank.  This is a specific setup for the flux-module test.  This
- * base component does not perform message routing to its extension modules.
- */
-static json_t *module_list (void)
-{
-    json_t *mods;
-    zlist_t *keys;
-    module_t *m;
-    char *name;
-    char rankstr[16];
-    int n;
-
-    if (!(mods = json_array ()))
-        oom ();
-    if (!(keys = zhash_keys (modules)))
-        oom ();
-    name = zlist_first (keys);
-    n = snprintf (rankstr, sizeof (rankstr), "rank%d", (int)rank);
-    assert (n < sizeof (rankstr));
-    while (name) {
-        json_t *o;
-        m = zhash_lookup (modules, name);
-        if (!(o = json_pack ("{s:s s:i s:s s:i s:i s:[s,s,s]}",
-                             "name", m->name,
-                             "size", m->size,
-                             "digest", m->digest,
-                             "idle", m->idle,
-                             "status", m->status,
-                             "services", "test1", "test2", rankstr)))
-            oom ();
-        if (json_array_append_new (mods, o) < 0)
-            oom ();
-        name = zlist_next (keys);
-    }
-    zlist_destroy (&keys);
-    return mods;
-}
-
-static void insmod_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                               const flux_msg_t *msg, void *arg)
-{
-    const char *path;
-    json_t *args;
-    size_t index;
-    json_t *value;
-    char *argz = NULL;
-    size_t argz_len = 0;
-    module_t *m = NULL;
-    error_t e;
-
-    if (flux_request_unpack (msg, NULL, "{s:s s:o}", "path", &path,
-                                                     "args", &args) < 0)
-        goto error;
-    if (!json_is_array (args))
-        goto proto;
-    json_array_foreach (args, index, value) {
-        if (!json_is_string (value))
-            goto proto;
-        if ((e = argz_add (&argz, &argz_len, json_string_value (value)))) {
-            errno = e;
-            goto error;
-        }
-    }
-    if (!(m = module_create (path, argz, argz_len)))
-        goto error;
-    flux_log (h, LOG_DEBUG, "insmod %s", m->name);
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    free (argz);
-    return;
-proto:
-    errno = EPROTO;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-    free (argz);
-}
-
-static void rmmod_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                              const flux_msg_t *msg, void *arg)
-{
-    const char *name;
-
-    if (flux_request_unpack (msg, NULL, "{s:s}", "name", &name) < 0)
-        goto error;
-    if (!zhash_lookup (modules, name)) {
-        errno = ENOENT;
-        goto error;
-    }
-    zhash_delete (modules, name);
-    flux_log (h, LOG_DEBUG, "rmmod %s", name);
-    if (flux_respond (h, msg, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-static void lsmod_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                              const flux_msg_t *msg, void *arg)
-{
-    json_t *mods = NULL;
-
-    if (flux_request_decode (msg, NULL, NULL) < 0)
-        goto error;
-    mods = module_list ();
-    if (flux_respond_pack (h, msg, "{s:O}", "mods", mods) < 0)
-        flux_log_error (h, "%s: flux_respond", __FUNCTION__);
-    json_decref (mods);
-    return;
-error:
-    if (flux_respond_error (h, msg, errno, NULL) < 0)
-        flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
-}
-
-const struct flux_msg_handler_spec htab[] = {
-    { FLUX_MSGTYPE_REQUEST, "parent.insmod",         insmod_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "parent.rmmod",          rmmod_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "parent.lsmod",          lsmod_request_cb, 0 },
-    FLUX_MSGHANDLER_TABLE_END,
-};
-
-int mod_main (flux_t *h, int argc, char **argv)
-{
-    int saved_errno;
-    flux_msg_handler_t **handlers = NULL;
-
-    if (argc == 1 && !strcmp (argv[0], "--init-failure")) {
-        flux_log (h, LOG_INFO, "aborting during init per test request");
-        errno = EIO;
-        goto error;
-    }
-    if (!(modules = zhash_new ())) {
-        errno = ENOMEM;
-        goto error;
-    }
-    if (flux_get_rank (h, &rank) < 0)
-        goto error;
-    if (flux_msg_handler_addvec (h, htab, NULL, &handlers) < 0)
-        goto error;
-    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
-        flux_log_error (h, "flux_reactor_run");
-        goto error;
-    }
-    zhash_destroy (&modules);
-    return 0;
-error:
-    saved_errno = errno;
-    flux_msg_handler_delvec (handlers);
-    zhash_destroy (&modules);
-    errno = saved_errno;
-    return -1;
-}
-
-MOD_NAME ("parent");
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/t/module/running.c b/t/module/running.c
new file mode 100644
index 000000000000..a291662d3ea5
--- /dev/null
+++ b/t/module/running.c
@@ -0,0 +1,43 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+
+int mod_main (flux_t *h, int argc, char *argv[])
+{
+    flux_msg_t *msg;
+    struct flux_match match;
+
+    if (flux_event_subscribe (h, "running.go") < 0)
+        return -1;
+    if (flux_module_set_running (h) < 0)
+        return -1;
+    match = FLUX_MATCH_EVENT;
+    match.topic_glob = "running.go";
+    if (!(msg = flux_recv (h, match, 0))) {
+        flux_log_error (h, "flux_recv");
+        return -1;
+    }
+    flux_log (h, LOG_DEBUG, "received event");
+    flux_msg_destroy (msg);
+
+    if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
+        flux_log_error (h, "flux_reactor_run");
+        return -1;
+    }
+    return 0;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/module/testmod.c b/t/module/testmod.c
new file mode 100644
index 000000000000..4fc3020c804d
--- /dev/null
+++ b/t/module/testmod.c
@@ -0,0 +1,91 @@
+/************************************************************\
+ * Copyright 2014 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include "ccan/str/str.h"
+
+static void info (flux_t *h,
+                  flux_msg_handler_t *mh,
+                  const flux_msg_t *msg,
+                  void *arg)
+{
+    if (flux_respond (h, msg, flux_aux_get (h, "flux::name")) < 0)
+        flux_log_error (h, "error responding to info request");
+}
+
+static struct flux_msg_handler_spec htab[] = {
+    { FLUX_MSGTYPE_REQUEST, "info", info, 0 },
+    FLUX_MSGHANDLER_TABLE_END,
+};
+
+int mod_main (flux_t *h, int argc, char **argv)
+{
+    flux_msg_handler_t **handlers = NULL;
+    int rc;
+
+    for (int i = 0; i < argc; i++) {
+        /* Dynamically register a service name if requested.
+         */
+        if (strstarts (argv[i], "--service=")) {
+            const char *service = argv[i] + 10;
+            flux_future_t *f;
+            if (!(f = flux_service_register (h, service))
+                || flux_rpc_get (f, NULL) < 0) {
+                flux_log (h, LOG_ERR, "failed to register service %s", service);
+                return -1;
+            }
+            flux_future_destroy (f);
+        }
+        else if (streq (argv[i], "--init-failure")) {
+            flux_log (h, LOG_INFO, "aborting during init per test request");
+            errno = EIO;
+            return -1;
+        }
+        else if (strstarts (argv[i], "--attr-is-cached=")) {
+            const char *attr = argv[i] + 17;
+            const char *name;
+            name = flux_attr_cache_first (h);
+            while (name) {
+                if (streq (attr, name))
+                    break;
+                name = flux_attr_cache_next (h);
+            }
+            if (name == NULL) {
+                flux_log (h, LOG_ERR, "attr %s is not present in cache", attr);
+                errno = ENOENT;
+                return -1;
+            }
+        }
+        else if (streq (argv[i], "--config-is-cached")) {
+            if (!flux_get_conf (h)) {
+                flux_log (h, LOG_ERR, "config object is not cached");
+                errno = ENOENT;
+                return -1;
+            }
+        }
+    }
+    if (flux_msg_handler_addvec_ex (h,
+                                    flux_aux_get (h, "flux::name"),
+                                    htab,
+                                    NULL,
+                                    &handlers) < 0)
+        return -1;
+    if ((rc = flux_reactor_run (flux_get_reactor (h), 0)) < 0)
+        flux_log_error (h, "flux_reactor_run");
+    flux_msg_handler_delvec (handlers);
+    return rc;
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/mpi/abort.c b/t/mpi/abort.c
new file mode 100644
index 000000000000..d0abd9a732e4
--- /dev/null
+++ b/t/mpi/abort.c
@@ -0,0 +1,44 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdlib.h>
+#include <stdio.h>
+#include <mpi.h>
+
+int main (int argc, char *argv[])
+{
+    int id, ntasks;
+    int abort_rank = -1;
+
+    if (argc == 2)
+        abort_rank = strtol (argv[1], NULL, 10);
+
+    MPI_Init (&argc, &argv);
+    MPI_Comm_rank (MPI_COMM_WORLD, &id);
+    MPI_Comm_size (MPI_COMM_WORLD, &ntasks);
+
+    printf ("Hello World from rank %d\n", id);
+
+    if (id == abort_rank) {
+        fprintf (stderr, "Rank %d is going to MPI_Abort now\n", id);
+        MPI_Abort (MPI_COMM_WORLD, 42);
+    }
+    MPI_Barrier (MPI_COMM_WORLD);
+
+    MPI_Finalize ();
+
+    return 0;
+}
+
+// vi: ts=4 sw=4 expandtab
+
diff --git a/t/mpi/hello.c b/t/mpi/hello.c
index dc2dd8d9536e..4ee376479273 100644
--- a/t/mpi/hello.c
+++ b/t/mpi/hello.c
@@ -8,67 +8,96 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdint.h>
 #include <mpi.h>
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <time.h>
 
-static struct timespec diff (struct timespec start, struct timespec end)
+#include "src/common/libutil/monotime.h"
+
+static inline void monotime_diff (struct timespec *t0,
+                                  struct timespec *result)
 {
-    struct timespec temp;
-    if ((end.tv_nsec-start.tv_nsec)<0) {
-        temp.tv_sec = end.tv_sec-start.tv_sec-1;
-        temp.tv_nsec = 1000000000+end.tv_nsec-start.tv_nsec;
-    } else {
-        temp.tv_sec = end.tv_sec-start.tv_sec;
-        temp.tv_nsec = end.tv_nsec-start.tv_nsec;
+    struct timespec now;
+    monotime (&now);
+    result->tv_sec  = now.tv_sec - t0->tv_sec;
+    result->tv_nsec = now.tv_nsec - t0->tv_nsec;
+    if (result->tv_nsec < 0) {
+        --result->tv_sec;
+        result->tv_nsec += 1000000000L;
     }
-    return temp;
 }
 
-static double time_since (struct timespec t0)
+static void die (const char *msg)
 {
-    struct timespec ts, d;
-    clock_gettime (CLOCK_MONOTONIC, &ts);
-
-    d = diff (t0, ts);
-
-    return ((double) d.tv_sec * 1000 + (double) d.tv_nsec / 1000000);
+    fprintf (stderr, "%s\n", msg);
+    exit (1);
 }
 
-int
-main(int argc, char *argv[])
+int main (int argc, char *argv[])
 {
-        int id, ntasks;
-	struct timespec t;
+    int id, ntasks;
+    struct timespec t0, t, times[4];
+    const char *label;
+    bool timing = getenv ("FLUX_MPI_TEST_TIMING");
 
-	clock_gettime (CLOCK_MONOTONIC, &t);
-        MPI_Init(&argc, &argv);
-        MPI_Comm_rank(MPI_COMM_WORLD, &id);
-        MPI_Comm_size(MPI_COMM_WORLD, &ntasks);
-        if (id == 0) {
-                printf("0: completed MPI_Init in %0.3fs.  There are %d tasks\n",
-		   time_since (t)/1000, ntasks);
-                fflush(stdout);
-        }
+    if (!(label = getenv ("FLUX_JOB_CC")))
+        if (!(label = getenv ("FLUX_JOB_ID")))
+            label = "0";
 
-	if (id == 0)
-		clock_gettime (CLOCK_MONOTONIC, &t);
-        MPI_Barrier(MPI_COMM_WORLD);
-        if (id == 0) {
-                printf("0: completed first barrier in %0.3fs\n",
-				time_since (t) / 1000);
-                fflush(stdout);
-        }
+    monotime (&t0);
+    if (MPI_Init (&argc, &argv) != MPI_SUCCESS)
+        die ("MPI_Init failed\n");
+    if (MPI_Comm_rank (MPI_COMM_WORLD, &id) != MPI_SUCCESS
+        || MPI_Comm_size (MPI_COMM_WORLD, &ntasks) != MPI_SUCCESS)
+        die ("MPI_Comm_rank/size failed");
+    monotime_diff (&t0, &times[0]);
+    if (!timing && id == 0) {
+        printf ("%s: completed MPI_Init in %0.3fs. There are %d tasks\n",
+                label,
+                monotime_since (t0) / 1000,
+                ntasks);
+        fflush (stdout);
+    }
 
-	if (id == 0)
-		clock_gettime (CLOCK_MONOTONIC, &t);
-        MPI_Finalize();
-        if (id == 0) {
-                printf("0: completed MPI_Finalize in %0.3fs\n",
-				time_since (t) / 1000);
-                fflush(stdout);
+    monotime (&t);
+    if (MPI_Barrier (MPI_COMM_WORLD) != MPI_SUCCESS)
+        die ("MPI_Barrier failed");
+    monotime_diff (&t, &times[1]);
+    if (!timing && id == 0) {
+        printf ("%s: completed first barrier in %0.3fs\n",
+                label,
+                monotime_since (t) / 1000);
+        fflush (stdout);
+    }
+
+    monotime (&t);
+    MPI_Finalize ();
+    monotime_diff (&t, &times[2]);
+    monotime_diff (&t0, &times[3]);
+
+    if (id == 0) {
+        if (timing) {
+            printf ("%6s %8d", getenv ("FLUX_JOB_NNODES"), ntasks);
+            for (int i = 0; i < 4; i++)
+                printf (" %4ju.%.9ld",
+                        (uintmax_t) times[i].tv_sec,
+                        times[i].tv_nsec);
+            printf ("\n");
         }
-        return 0;
+        else
+            printf ("%s: completed MPI_Finalize in %0.3fs\n",
+                    label,
+                    monotime_since (t) / 1000);
+        fflush (stdout);
+    }
+    return 0;
 }
+
+// vi: ts=4 sw=4 expandtab
+
diff --git a/t/mpi/mpich_basic/GetOpt.c b/t/mpi/mpich_basic/GetOpt.c
new file mode 100644
index 000000000000..34adfc969ce0
--- /dev/null
+++ b/t/mpi/mpich_basic/GetOpt.c
@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#include "GetOpt.h"
+#include <stdio.h>
+
+bool GetOpt(int *argc, LPTSTR ** argv, CLPTSTR flag)
+{
+    int i, j;
+    if (flag == NULL)
+        return false;
+
+    for (i = 0; i < *argc; i++) {
+        if (_tcsicmp((*argv)[i], flag) == 0) {
+            for (j = i; j < *argc; j++) {
+                (*argv)[j] = (*argv)[j + 1];
+            }
+            *argc -= 1;
+            return true;
+        }
+    }
+    return false;
+}
+
+bool GetOptInt(int *argc, LPTSTR ** argv, CLPTSTR flag, int *n)
+{
+    int i, j;
+    if (flag == NULL)
+        return false;
+
+    for (i = 0; i < *argc; i++) {
+        if (_tcsicmp((*argv)[i], flag) == 0) {
+            if (i + 1 == *argc)
+                return false;
+            *n = _ttoi((*argv)[i + 1]);
+            for (j = i; j < *argc - 1; j++) {
+                (*argv)[j] = (*argv)[j + 2];
+            }
+            *argc -= 2;
+            return true;
+        }
+    }
+    return false;
+}
+
+bool GetOptLong(int *argc, LPTSTR ** argv, CLPTSTR flag, long *n)
+{
+    int i;
+    if (GetOptInt(argc, argv, flag, &i)) {
+        *n = (long) i;
+        return true;
+    }
+    return false;
+}
+
+bool GetOptDouble(int *argc, LPTSTR ** argv, CLPTSTR flag, double *d)
+{
+    int i, j;
+
+    if (flag == NULL)
+        return false;
+
+    for (i = 0; i < *argc; i++) {
+        if (_tcsicmp((*argv)[i], flag) == 0) {
+            if (i + 1 == *argc)
+                return false;
+            *d = _tcstod((*argv)[i + 1], NULL);
+            for (j = i; j < *argc - 1; j++) {
+                (*argv)[j] = (*argv)[j + 2];
+            }
+            *argc -= 2;
+            return true;
+        }
+    }
+    return false;
+}
+
+bool GetOptString(int *argc, LPTSTR ** argv, CLPTSTR flag, char *str)
+{
+    int i, j;
+
+    if (flag == NULL)
+        return false;
+
+    for (i = 0; i < *argc; i++) {
+        if (_tcsicmp((*argv)[i], flag) == 0) {
+            if (i + 1 == *argc)
+                return false;
+            strcpy(str, (*argv)[i + 1]);
+            for (j = i; j < *argc - 1; j++) {
+                (*argv)[j] = (*argv)[j + 2];
+            }
+            *argc -= 2;
+            return true;
+        }
+    }
+    return false;
+}
diff --git a/t/mpi/mpich_basic/GetOpt.h b/t/mpi/mpich_basic/GetOpt.h
new file mode 100644
index 000000000000..af959cc24345
--- /dev/null
+++ b/t/mpi/mpich_basic/GetOpt.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#ifndef GETOPT_H_INCLUDED
+#define GETOPT_H_INCLUDED
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#define CLPTSTR const char *
+#define LPTSTR char *
+#define LPCTSTR char *
+#define _tcsicmp strcmp
+#define _ttoi atoi
+#define _tcstod(a,b) atof(a)
+#define stricmp strcmp
+#define bool int
+#define true 1
+#define false 0
+
+bool GetOpt(int *argc, char ***argv, const char *flag);
+bool GetOptInt(int *argc, char ***argv, const char *flag, int *n);
+bool GetOptLong(int *argc, char ***argv, const char *flag, long *n);
+bool GetOptDouble(int *argc, char ***argv, const char *flag, double *d);
+bool GetOptString(int *argc, char ***argv, const char *flag, char *str);
+
+#endif /* GETOPT_H_INCLUDED */
diff --git a/t/mpi/mpich_basic/adapt.c b/t/mpi/mpich_basic/adapt.c
new file mode 100644
index 000000000000..68d3d3b04a33
--- /dev/null
+++ b/t/mpi/mpich_basic/adapt.c
@@ -0,0 +1,1477 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#ifdef HAVE_WINDOWS_H
+#include <winsock2.h>
+#include <windows.h>
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "mpi.h"
+#include "GetOpt.h"
+
+#ifndef BOOL
+typedef int BOOL;
+#endif
+#ifndef TRUE
+#define TRUE 1
+#endif
+#ifndef FALSE
+#define FALSE 0
+#endif
+
+#define MIN(x, y)	(((x) < (y))?(x):(y))
+#define MAX(x, y)	(((x) > (y))?(x):(y))
+
+#ifdef HAVE_WINDOWS_H
+#define POINTER_TO_INT(a)   ((int)(0xffffffff & (__int64) (a)))
+#else
+#define POINTER_TO_INT(a)   ((MPI_Aint)(a))
+#endif
+
+#define CREATE_DIFFERENCE_CURVES
+#undef CREATE_SINGLE_CURVE
+
+#define MAX_NUM_O12_TRIALS 18
+#define TRIALS          7
+#define PERT            3
+#define LONGTIME        1e99
+#define CHARSIZE        8
+#define RUNTM		0.1
+#define MAXINT          2147483647
+#define MAX_LAT_TIME    2
+#define LEFT_PROCESS    0
+#define MIDDLE_PROCESS  1
+#define RIGHT_PROCESS   2
+#define MSG_TAG_01      9901
+#define MSG_TAG_12      9912
+#define MSG_TAG_012     9012
+
+int g_left_rank = -1;
+int g_middle_rank = -1;
+int g_right_rank = -1;
+int g_proc_loc = -1;
+int g_NSAMP = 250;
+double g_STOPTM = 0.1;
+int g_latency012_reps = 1000;
+int g_nIproc = 0;
+int g_nNproc = 0;
+
+typedef struct ArgStruct {
+    char *sbuff;                /* Send buffer      */
+    char *rbuff;                /* Recv buffer      */
+    int bufflen;                /* Length of buffer */
+    int nbor, nbor2;            /* neighbor */
+    int iproc;                  /* rank */
+    int tr;                     /* transmitter/receiver flag */
+    int latency_reps;           /* reps needed to time latency */
+} ArgStruct;
+
+typedef struct Data {
+    double t;
+    double bps;
+    int bits;
+    int repeat;
+} Data;
+
+int Setup(int middle_rank, ArgStruct * p01, ArgStruct * p12, ArgStruct * p012);
+void Sync(ArgStruct * p);
+void Sync012(ArgStruct * p);
+void SendTime(ArgStruct * p, double *t);
+void RecvTime(ArgStruct * p, double *t);
+void SendReps(ArgStruct * p, int *rpt);
+void RecvReps(ArgStruct * p, int *rpt);
+double TestLatency(ArgStruct * p);
+double TestLatency012(ArgStruct * p);
+void PrintOptions(void);
+int DetermineLatencyReps(ArgStruct * p);
+int DetermineLatencyReps012(ArgStruct * p);
+
+void PrintOptions()
+{
+    printf("\n");
+    printf("Usage: adapt flags\n");
+    printf(" flags:\n");
+    printf("       -reps #iterations\n");
+    printf("       -time stop_time\n");
+    printf("       -start initial_msg_size\n");
+    printf("       -end final_msg_size\n");
+    printf("       -out outputfile\n");
+    printf("       -nocache\n");
+    printf("       -pert\n");
+    printf("       -noprint\n");
+    printf("       -middle rank_0_1_or_2\n");
+    printf("Requires exactly three processes\n");
+    printf("\n");
+}
+
+int main(int argc, char *argv[])
+{
+    FILE *out = 0;              /* Output data file                     */
+    char s[255];                /* Generic string                       */
+    char *memtmp;
+    char *memtmp1;
+    MPI_Status status;
+
+    int ii, i, j, k, n, nq,     /* Loop indices                         */
+     bufoffset = 0,             /* Align buffer to this                 */
+        bufalign = 16 * 1024,   /* Boundary to align buffer to          */
+        nrepeat01, nrepeat12,   /* Number of time to do the transmission */
+        nrepeat012, len,        /* Number of bytes to be transmitted    */
+        inc = 1,        /* Increment value                      */
+        pert,   /* Perturbation value                   */
+        ipert,  /* index of the perturbation loop       */
+        start = 0,      /* Starting value for signature curve   */
+        end = MAXINT,   /* Ending value for signature curve     */
+        printopt = 1,   /* Debug print statements flag          */
+        middle_rank = 0,        /* rank 0, 1 or 2 where 2-0-1 or 0-1-2 or 1-2-0 */
+        tint;
+
+    ArgStruct args01, args12, args012;  /* Argumentsfor all the calls   */
+
+    double t, t0, t1,           /* Time variables                       */
+     tlast01, tlast12, tlast012,        /* Time for the last transmission    */
+     latency01, latency12,      /* Network message latency              */
+     latency012, tdouble;       /* Network message latency to go from 0 -> 1 -> 2 */
+#ifdef CREATE_DIFFERENCE_CURVES
+    int itrial, ntrials;
+    double *dtrials;
+#endif
+
+    Data *bwdata01, *bwdata12, *bwdata012;      /* Bandwidth curve data       */
+
+    BOOL bNoCache = FALSE;
+    BOOL bSavePert = FALSE;
+    BOOL bUseMegaBytes = FALSE;
+
+    MPI_Init(&argc, &argv);
+
+    MPI_Comm_size(MPI_COMM_WORLD, &g_nNproc);
+    MPI_Comm_rank(MPI_COMM_WORLD, &g_nIproc);
+
+    if (g_nNproc != 3) {
+        if (g_nIproc == 0)
+            PrintOptions();
+        MPI_Finalize();
+        exit(0);
+    }
+
+    GetOptDouble(&argc, &argv, "-time", &g_STOPTM);
+    GetOptInt(&argc, &argv, "-reps", &g_NSAMP);
+    GetOptInt(&argc, &argv, "-start", &start);
+    GetOptInt(&argc, &argv, "-end", &end);
+    bNoCache = GetOpt(&argc, &argv, "-nocache");
+    bUseMegaBytes = GetOpt(&argc, &argv, "-mb");
+    if (GetOpt(&argc, &argv, "-noprint"))
+        printopt = 0;
+    bSavePert = GetOpt(&argc, &argv, "-pert");
+    GetOptInt(&argc, &argv, "-middle", &middle_rank);
+    if (middle_rank < 0 || middle_rank > 2)
+        middle_rank = 0;
+
+    bwdata01 = malloc((g_NSAMP + 1) * sizeof(Data));
+    bwdata12 = malloc((g_NSAMP + 1) * sizeof(Data));
+    bwdata012 = malloc((g_NSAMP + 1) * sizeof(Data));
+
+    if (g_nIproc == 0)
+        strcpy(s, "adapt.out");
+    GetOptString(&argc, &argv, "-out", s);
+
+    if (start > end) {
+        fprintf(stdout, "Start MUST be LESS than end\n");
+        exit(420132);
+    }
+
+    Setup(middle_rank, &args01, &args12, &args012);
+
+    if (g_nIproc == 0) {
+        if ((out = fopen(s, "w")) == NULL) {
+            fprintf(stdout, "Can't open %s for output\n", s);
+            exit(1);
+        }
+    }
+
+    /* Calculate latency */
+    switch (g_proc_loc) {
+        case LEFT_PROCESS:
+            latency01 = TestLatency(&args01);
+            /*printf("[0] latency01 = %0.9f\n", latency01);fflush(stdout); */
+            RecvTime(&args01, &latency12);
+            /*printf("[0] latency12 = %0.9f\n", latency12);fflush(stdout); */
+            break;
+        case MIDDLE_PROCESS:
+            latency01 = TestLatency(&args01);
+            /*printf("[1] latency01 = %0.9f\n", latency01);fflush(stdout); */
+            SendTime(&args12, &latency01);
+            latency12 = TestLatency(&args12);
+            /*printf("[1] latency12 = %0.9f\n", latency12);fflush(stdout); */
+            SendTime(&args01, &latency12);
+            break;
+        case RIGHT_PROCESS:
+            RecvTime(&args12, &latency01);
+            /*printf("[2] latency01 = %0.9f\n", latency01);fflush(stdout); */
+            latency12 = TestLatency(&args12);
+            /*printf("[2] latency12 = %0.9f\n", latency12);fflush(stdout); */
+            break;
+    }
+
+    latency012 = TestLatency012(&args012);
+
+    if ((g_nIproc == 0) && printopt) {
+        printf("Latency%d%d_ : %0.9f\n", g_left_rank, g_middle_rank, latency01);
+        printf("Latency_%d%d : %0.9f\n", g_middle_rank, g_right_rank, latency12);
+        printf("Latency%d%d%d : %0.9f\n", g_left_rank, g_middle_rank, g_right_rank, latency012);
+        fflush(stdout);
+        printf("Now starting main loop\n");
+        fflush(stdout);
+    }
+    tlast01 = latency01;
+    tlast12 = latency12;
+    tlast012 = latency012;
+    inc = (start > 1) ? start / 2 : inc;
+    args01.bufflen = start;
+    args12.bufflen = start;
+    args012.bufflen = start;
+
+#ifdef CREATE_DIFFERENCE_CURVES
+    /* print the header line of the output file */
+    if (g_nIproc == 0) {
+        fprintf(out, "bytes\tMbits/s\ttime\tMbits/s\ttime");
+        for (ii = 1, itrial = 0; itrial < MAX_NUM_O12_TRIALS; ii <<= 1, itrial++)
+            fprintf(out, "\t%d", ii);
+        fprintf(out, "\n");
+        fflush(out);
+    }
+    ntrials = MAX_NUM_O12_TRIALS;
+    dtrials = malloc(sizeof(double) * ntrials);
+#endif
+
+    /* Main loop of benchmark */
+    for (nq = n = 0, len = start;
+         n < g_NSAMP && tlast012 < g_STOPTM && len <= end; len = len + inc, nq++) {
+        if (nq > 2)
+            inc = (nq % 2) ? inc + inc : inc;
+
+        /* clear the old values */
+        for (itrial = 0; itrial < ntrials; itrial++) {
+            dtrials[itrial] = LONGTIME;
+        }
+
+        /* This is a perturbation loop to test nearby values */
+        for (ipert = 0, pert = (inc > PERT + 1) ? -PERT : 0;
+             pert <= PERT; ipert++, n++, pert += (inc > PERT + 1) ? PERT : PERT + 1) {
+
+
+            /*****************************************************/
+            /*         Run a trial between rank 0 and 1          */
+            /*****************************************************/
+
+            MPI_Barrier(MPI_COMM_WORLD);
+
+
+            if (g_proc_loc == RIGHT_PROCESS)
+                goto skip_01_trial;
+
+            /* Calculate howmany times to repeat the experiment. */
+            if (args01.tr) {
+                if (args01.bufflen == 0)
+                    nrepeat01 = args01.latency_reps;
+                else
+                    nrepeat01 = (int) (MAX((RUNTM / ((double) args01.bufflen /
+                                                     (args01.bufflen - inc + 1.0) * tlast01)),
+                                           TRIALS));
+                SendReps(&args01, &nrepeat01);
+            } else {
+                RecvReps(&args01, &nrepeat01);
+            }
+
+            /* Allocate the buffer */
+            args01.bufflen = len + pert;
+            /* printf("allocating %d bytes\n", args01.bufflen * nrepeat01 + bufalign); */
+            if (bNoCache) {
+                if ((args01.sbuff =
+                     (char *) malloc(args01.bufflen * nrepeat01 + bufalign)) == (char *) NULL) {
+                    fprintf(stdout, "Couldn't allocate memory\n");
+                    fflush(stdout);
+                    break;
+                }
+            } else {
+                if ((args01.sbuff = (char *) malloc(args01.bufflen + bufalign)) == (char *) NULL) {
+                    fprintf(stdout, "Couldn't allocate memory\n");
+                    fflush(stdout);
+                    break;
+                }
+            }
+            /* if ((args01.rbuff = (char *)malloc(args01.bufflen * nrepeat01 + bufalign)) == (char *)NULL) */
+            if ((args01.rbuff = (char *) malloc(args01.bufflen + bufalign)) == (char *) NULL) {
+                fprintf(stdout, "Couldn't allocate memory\n");
+                fflush(stdout);
+                break;
+            }
+
+            /* save the original pointers in case alignment moves them */
+            memtmp = args01.sbuff;
+            memtmp1 = args01.rbuff;
+
+            /* Possibly align the data buffer */
+            if (!bNoCache) {
+                if (bufalign != 0) {
+                    args01.sbuff +=
+                        (bufalign - (POINTER_TO_INT(args01.sbuff) % bufalign) +
+                         bufoffset) % bufalign;
+                    /* args01.rbuff += (bufalign - ((MPI_Aint)args01.rbuff % bufalign) + bufoffset) % bufalign; */
+                }
+            }
+            args01.rbuff +=
+                (bufalign - (POINTER_TO_INT(args01.rbuff) % bufalign) + bufoffset) % bufalign;
+
+            if (args01.tr && printopt) {
+                fprintf(stdout, "%3d: %9d bytes %4d times --> ", n, args01.bufflen, nrepeat01);
+                fflush(stdout);
+            }
+
+            /* Finally, we get to transmit or receive and time */
+            if (args01.tr) {
+                bwdata01[n].t = LONGTIME;
+                t1 = 0;
+                for (i = 0; i < TRIALS; i++) {
+                    if (bNoCache) {
+                        if (bufalign != 0) {
+                            args01.sbuff =
+                                memtmp +
+                                ((bufalign - (POINTER_TO_INT(args01.sbuff) % bufalign) +
+                                  bufoffset) % bufalign);
+                            /* args01.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args01.rbuff % bufalign) + bufoffset) % bufalign); */
+                        } else {
+                            args01.sbuff = memtmp;
+                            /* args01.rbuff = memtmp1; */
+                        }
+                    }
+
+                    Sync(&args01);
+                    t0 = MPI_Wtime();
+                    for (j = 0; j < nrepeat01; j++) {
+                        MPI_Send(args01.sbuff, args01.bufflen, MPI_BYTE, args01.nbor, MSG_TAG_01,
+                                 MPI_COMM_WORLD);
+                        MPI_Recv(args01.rbuff, args01.bufflen, MPI_BYTE, args01.nbor, MSG_TAG_01,
+                                 MPI_COMM_WORLD, &status);
+                        if (bNoCache) {
+                            args01.sbuff += args01.bufflen;
+                            /* args01.rbuff += args01.bufflen; */
+                        }
+                    }
+                    t = (MPI_Wtime() - t0) / (2 * nrepeat01);
+
+                    t1 += t;
+                    bwdata01[n].t = MIN(bwdata01[n].t, t);
+                }
+                SendTime(&args01, &bwdata01[n].t);
+            } else {
+                bwdata01[n].t = LONGTIME;
+                t1 = 0;
+                for (i = 0; i < TRIALS; i++) {
+                    if (bNoCache) {
+                        if (bufalign != 0) {
+                            args01.sbuff =
+                                memtmp +
+                                ((bufalign - (POINTER_TO_INT(args01.sbuff) % bufalign) +
+                                  bufoffset) % bufalign);
+                            /* args01.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args01.rbuff % bufalign) + bufoffset) % bufalign); */
+                        } else {
+                            args01.sbuff = memtmp;
+                            /* args01.rbuff = memtmp1; */
+                        }
+                    }
+
+                    Sync(&args01);
+                    t0 = MPI_Wtime();
+                    for (j = 0; j < nrepeat01; j++) {
+                        MPI_Recv(args01.rbuff, args01.bufflen, MPI_BYTE, args01.nbor, MSG_TAG_01,
+                                 MPI_COMM_WORLD, &status);
+                        MPI_Send(args01.sbuff, args01.bufflen, MPI_BYTE, args01.nbor, MSG_TAG_01,
+                                 MPI_COMM_WORLD);
+                        if (bNoCache) {
+                            args01.sbuff += args01.bufflen;
+                            /* args01.rbuff += args01.bufflen; */
+                        }
+                    }
+                    t = (MPI_Wtime() - t0) / (2 * nrepeat01);
+                }
+                RecvTime(&args01, &bwdata01[n].t);
+            }
+            tlast01 = bwdata01[n].t;
+            bwdata01[n].bits = args01.bufflen * CHARSIZE;
+            bwdata01[n].bps = bwdata01[n].bits / (bwdata01[n].t * 1024 * 1024);
+            bwdata01[n].repeat = nrepeat01;
+
+            if (args01.tr) {
+                if (bSavePert) {
+                    if (args01.iproc == 0) {
+                        if (bUseMegaBytes)
+                            fprintf(out, "%d\t%f\t%0.9f\t", bwdata01[n].bits / 8,
+                                    bwdata01[n].bps / 8, bwdata01[n].t);
+                        else
+                            fprintf(out, "%d\t%f\t%0.9f\t", bwdata01[n].bits / 8, bwdata01[n].bps,
+                                    bwdata01[n].t);
+                        fflush(out);
+                    } else {
+                        MPI_Send(&bwdata01[n].bits, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
+                        MPI_Send(&bwdata01[n].bps, 1, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD);
+                        MPI_Send(&bwdata01[n].t, 1, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD);
+                    }
+                }
+            }
+
+            free(memtmp);
+            free(memtmp1);
+
+            if (args01.tr && printopt) {
+                if (bUseMegaBytes)
+                    printf(" %6.2f MBps in %0.9f sec\n", bwdata01[n].bps / 8, tlast01);
+                else
+                    printf(" %6.2f Mbps in %0.9f sec\n", bwdata01[n].bps, tlast01);
+                fflush(stdout);
+            }
+
+          skip_01_trial:
+            if (g_proc_loc == RIGHT_PROCESS && g_nIproc == 0 && bSavePert) {
+                MPI_Recv(&tint, 1, MPI_INT, g_left_rank, 1, MPI_COMM_WORLD, &status);
+                fprintf(out, "%d\t", tint / 8);
+                MPI_Recv(&tdouble, 1, MPI_DOUBLE, g_left_rank, 1, MPI_COMM_WORLD, &status);
+                if (bUseMegaBytes)
+                    tdouble = tdouble / 8.0;
+                fprintf(out, "%f\t", tdouble);
+                MPI_Recv(&tdouble, 1, MPI_DOUBLE, g_left_rank, 1, MPI_COMM_WORLD, &status);
+                fprintf(out, "%0.9f\t", tdouble);
+                fflush(out);
+            }
+
+
+            /*****************************************************/
+            /*         Run a trial between rank 1 and 2          */
+            /*****************************************************/
+
+            MPI_Barrier(MPI_COMM_WORLD);
+
+
+            if (g_proc_loc == LEFT_PROCESS)
+                goto skip_12_trial;
+
+            /* Calculate howmany times to repeat the experiment. */
+            if (args12.tr) {
+                if (args12.bufflen == 0)
+                    nrepeat12 = args12.latency_reps;
+                else
+                    nrepeat12 = (int) (MAX((RUNTM / ((double) args12.bufflen /
+                                                     (args12.bufflen - inc + 1.0) * tlast12)),
+                                           TRIALS));
+                SendReps(&args12, &nrepeat12);
+            } else {
+                RecvReps(&args12, &nrepeat12);
+            }
+
+            /* Allocate the buffer */
+            args12.bufflen = len + pert;
+            /* printf("allocating %d bytes\n", args12.bufflen * nrepeat12 + bufalign); */
+            if (bNoCache) {
+                if ((args12.sbuff =
+                     (char *) malloc(args12.bufflen * nrepeat12 + bufalign)) == (char *) NULL) {
+                    fprintf(stdout, "Couldn't allocate memory\n");
+                    fflush(stdout);
+                    break;
+                }
+            } else {
+                if ((args12.sbuff = (char *) malloc(args12.bufflen + bufalign)) == (char *) NULL) {
+                    fprintf(stdout, "Couldn't allocate memory\n");
+                    fflush(stdout);
+                    break;
+                }
+            }
+            /* if ((args12.rbuff = (char *)malloc(args12.bufflen * nrepeat12 + bufalign)) == (char *)NULL) */
+            if ((args12.rbuff = (char *) malloc(args12.bufflen + bufalign)) == (char *) NULL) {
+                fprintf(stdout, "Couldn't allocate memory\n");
+                fflush(stdout);
+                break;
+            }
+
+            /* save the original pointers in case alignment moves them */
+            memtmp = args12.sbuff;
+            memtmp1 = args12.rbuff;
+
+            /* Possibly align the data buffer */
+            if (!bNoCache) {
+                if (bufalign != 0) {
+                    args12.sbuff +=
+                        (bufalign - (POINTER_TO_INT(args12.sbuff) % bufalign) +
+                         bufoffset) % bufalign;
+                    /* args12.rbuff += (bufalign - ((MPI_Aint)args12.rbuff % bufalign) + bufoffset) % bufalign; */
+                }
+            }
+            args12.rbuff +=
+                (bufalign - (POINTER_TO_INT(args12.rbuff) % bufalign) + bufoffset) % bufalign;
+
+            if (args12.tr && printopt) {
+                printf("%3d: %9d bytes %4d times --> ", n, args12.bufflen, nrepeat12);
+                fflush(stdout);
+            }
+
+            /* Finally, we get to transmit or receive and time */
+            if (args12.tr) {
+                bwdata12[n].t = LONGTIME;
+                t1 = 0;
+                for (i = 0; i < TRIALS; i++) {
+                    if (bNoCache) {
+                        if (bufalign != 0) {
+                            args12.sbuff =
+                                memtmp +
+                                ((bufalign - (POINTER_TO_INT(args12.sbuff) % bufalign) +
+                                  bufoffset) % bufalign);
+                            /* args12.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args12.rbuff % bufalign) + bufoffset) % bufalign); */
+                        } else {
+                            args12.sbuff = memtmp;
+                            /* args12.rbuff = memtmp1; */
+                        }
+                    }
+
+                    Sync(&args12);
+                    t0 = MPI_Wtime();
+                    for (j = 0; j < nrepeat12; j++) {
+                        MPI_Send(args12.sbuff, args12.bufflen, MPI_BYTE, args12.nbor, MSG_TAG_12,
+                                 MPI_COMM_WORLD);
+                        MPI_Recv(args12.rbuff, args12.bufflen, MPI_BYTE, args12.nbor, MSG_TAG_12,
+                                 MPI_COMM_WORLD, &status);
+                        if (bNoCache) {
+                            args12.sbuff += args12.bufflen;
+                            /* args12.rbuff += args12.bufflen; */
+                        }
+                    }
+                    t = (MPI_Wtime() - t0) / (2 * nrepeat12);
+
+                    t1 += t;
+                    bwdata12[n].t = MIN(bwdata12[n].t, t);
+                }
+                SendTime(&args12, &bwdata12[n].t);
+            } else {
+                bwdata12[n].t = LONGTIME;
+                t1 = 0;
+                for (i = 0; i < TRIALS; i++) {
+                    if (bNoCache) {
+                        if (bufalign != 0) {
+                            args12.sbuff =
+                                memtmp +
+                                ((bufalign - (POINTER_TO_INT(args12.sbuff) % bufalign) +
+                                  bufoffset) % bufalign);
+                            /* args12.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args12.rbuff % bufalign) + bufoffset) % bufalign); */
+                        } else {
+                            args12.sbuff = memtmp;
+                            /* args12.rbuff = memtmp1; */
+                        }
+                    }
+
+                    Sync(&args12);
+                    t0 = MPI_Wtime();
+                    for (j = 0; j < nrepeat12; j++) {
+                        MPI_Recv(args12.rbuff, args12.bufflen, MPI_BYTE, args12.nbor, MSG_TAG_12,
+                                 MPI_COMM_WORLD, &status);
+                        MPI_Send(args12.sbuff, args12.bufflen, MPI_BYTE, args12.nbor, MSG_TAG_12,
+                                 MPI_COMM_WORLD);
+                        if (bNoCache) {
+                            args12.sbuff += args12.bufflen;
+                            /* args12.rbuff += args12.bufflen; */
+                        }
+                    }
+                    t = (MPI_Wtime() - t0) / (2 * nrepeat12);
+                }
+                RecvTime(&args12, &bwdata12[n].t);
+            }
+            tlast12 = bwdata12[n].t;
+            bwdata12[n].bits = args12.bufflen * CHARSIZE;
+            bwdata12[n].bps = bwdata12[n].bits / (bwdata12[n].t * 1024 * 1024);
+            bwdata12[n].repeat = nrepeat12;
+
+            if (args12.tr) {
+                if (bSavePert) {
+                    if (g_nIproc == 0) {
+                        if (bUseMegaBytes)
+                            fprintf(out, "%f\t%0.9f\t", bwdata12[n].bps / 8, bwdata12[n].t);
+                        else
+                            fprintf(out, "%f\t%0.9f\t", bwdata12[n].bps, bwdata12[n].t);
+                        fflush(out);
+                    } else {
+                        MPI_Send(&bwdata12[n].bps, 1, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD);
+                        MPI_Send(&bwdata12[n].t, 1, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD);
+                    }
+                }
+            }
+
+            free(memtmp);
+            free(memtmp1);
+
+            if (args12.tr && printopt) {
+                if (bUseMegaBytes)
+                    printf(" %6.2f MBps in %0.9f sec\n", bwdata12[n].bps / 8, tlast12);
+                else
+                    printf(" %6.2f Mbps in %0.9f sec\n", bwdata12[n].bps, tlast12);
+                fflush(stdout);
+            }
+
+          skip_12_trial:
+            if (g_proc_loc == LEFT_PROCESS && g_nIproc == 0 && bSavePert) {
+                MPI_Recv(&tdouble, 1, MPI_DOUBLE, g_middle_rank, 1, MPI_COMM_WORLD, &status);
+                if (bUseMegaBytes)
+                    tdouble = tdouble / 8.0;
+                fprintf(out, "%f\t", tdouble);
+                MPI_Recv(&tdouble, 1, MPI_DOUBLE, g_middle_rank, 1, MPI_COMM_WORLD, &status);
+                fprintf(out, "%0.9f\t", tdouble);
+                fflush(out);
+            }
+#ifdef CREATE_DIFFERENCE_CURVES
+            /*****************************************************/
+            /*         Run a trial between rank 0, 1 and 2       */
+            /*****************************************************/
+
+            MPI_Barrier(MPI_COMM_WORLD);
+
+
+            /* Calculate howmany times to repeat the experiment. */
+            if (g_nIproc == 0) {
+                if (args012.bufflen == 0)
+                    nrepeat012 = g_latency012_reps;
+                else
+                    nrepeat012 = (int) (MAX((RUNTM / ((double) args012.bufflen /
+                                                      (args012.bufflen - inc + 1.0) * tlast012)),
+                                            TRIALS));
+                MPI_Bcast(&nrepeat012, 1, MPI_INT, 0, MPI_COMM_WORLD);
+            } else {
+                MPI_Bcast(&nrepeat012, 1, MPI_INT, 0, MPI_COMM_WORLD);
+            }
+
+            /* Allocate the buffer */
+            args012.bufflen = len + pert;
+            /* printf("allocating %d bytes\n", args12.bufflen * nrepeat012 + bufalign); */
+            if (bNoCache) {
+                if ((args012.sbuff =
+                     (char *) malloc(args012.bufflen * nrepeat012 + bufalign)) == (char *) NULL) {
+                    fprintf(stdout, "Couldn't allocate memory\n");
+                    fflush(stdout);
+                    break;
+                }
+            } else {
+                if ((args012.sbuff = (char *) malloc(args012.bufflen + bufalign)) == (char *) NULL) {
+                    fprintf(stdout, "Couldn't allocate memory\n");
+                    fflush(stdout);
+                    break;
+                }
+            }
+            /* if ((args012.rbuff = (char *)malloc(args012.bufflen * nrepeat012 + bufalign)) == (char *)NULL) */
+            if ((args012.rbuff = (char *) malloc(args012.bufflen + bufalign)) == (char *) NULL) {
+                fprintf(stdout, "Couldn't allocate memory\n");
+                fflush(stdout);
+                break;
+            }
+
+            /* save the original pointers in case alignment moves them */
+            memtmp = args012.sbuff;
+            memtmp1 = args012.rbuff;
+
+            /* Possibly align the data buffer */
+            if (!bNoCache) {
+                if (bufalign != 0) {
+                    args012.sbuff +=
+                        (bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) +
+                         bufoffset) % bufalign;
+                    /* args12.rbuff += (bufalign - ((MPI_Aint)args12.rbuff % bufalign) + bufoffset) % bufalign; */
+                }
+            }
+            args012.rbuff +=
+                (bufalign - (POINTER_TO_INT(args012.rbuff) % bufalign) + bufoffset) % bufalign;
+
+            if (g_nIproc == 0 && printopt) {
+                printf("%3d: %9d bytes %4d times --> ", n, args012.bufflen, nrepeat012);
+                fflush(stdout);
+            }
+
+            for (itrial = 0, ii = 1; ii <= nrepeat012 && itrial < ntrials; ii <<= 1, itrial++) {
+                /* Finally, we get to transmit or receive and time */
+                switch (g_proc_loc) {
+                    case LEFT_PROCESS:
+                        bwdata012[n].t = LONGTIME;
+                        t1 = 0;
+                        for (i = 0; i < TRIALS; i++) {
+                            if (bNoCache) {
+                                if (bufalign != 0) {
+                                    args012.sbuff =
+                                        memtmp +
+                                        ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) +
+                                          bufoffset) % bufalign);
+                                    /* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
+                                } else {
+                                    args012.sbuff = memtmp;
+                                    /* args012.rbuff = memtmp1; */
+                                }
+                            }
+
+                            Sync012(&args012);
+                            t0 = MPI_Wtime();
+                            for (j = 0; j < nrepeat012; j++) {
+                                MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                         MSG_TAG_012, MPI_COMM_WORLD);
+                                MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                         MSG_TAG_012, MPI_COMM_WORLD, &status);
+                                if (bNoCache) {
+                                    args012.sbuff += args012.bufflen;
+                                    /* args012.rbuff += args012.bufflen; */
+                                }
+                            }
+                            t = (MPI_Wtime() - t0) / (2 * nrepeat012);
+
+                            t1 += t;
+                            bwdata012[n].t = MIN(bwdata012[n].t, t);
+                        }
+                        MPI_Bcast(&bwdata012[n].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
+                        break;
+                    case MIDDLE_PROCESS:
+                        bwdata012[n].t = LONGTIME;
+                        t1 = 0;
+                        for (i = 0; i < TRIALS; i++) {
+                            if (bNoCache) {
+                                if (bufalign != 0) {
+                                    args012.sbuff =
+                                        memtmp +
+                                        ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) +
+                                          bufoffset) % bufalign);
+                                    /* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
+                                } else {
+                                    args012.sbuff = memtmp;
+                                    /* args012.rbuff = memtmp1; */
+                                }
+                            }
+
+                            Sync012(&args012);
+                            t0 = MPI_Wtime();
+
+                        /******* use the ii variable here !!! ******/
+
+                            for (j = 0; j <= nrepeat012 - ii; j += ii) {
+                                for (k = 0; k < ii; k++) {
+                                    MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE,
+                                             args012.nbor2, MSG_TAG_012, MPI_COMM_WORLD);
+                                    MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE,
+                                             args012.nbor2, MSG_TAG_012, MPI_COMM_WORLD, &status);
+                                }
+                                /* do the left process second because it does the timing and needs to include time to send to the right process. */
+                                for (k = 0; k < ii; k++) {
+                                    MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                             MSG_TAG_012, MPI_COMM_WORLD, &status);
+                                    MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                             MSG_TAG_012, MPI_COMM_WORLD);
+                                }
+                                if (bNoCache) {
+                                    args012.sbuff += args012.bufflen;
+                                    /* args012.rbuff += args012.bufflen; */
+                                }
+                            }
+                            j = nrepeat012 % ii;
+                            for (k = 0; k < j; k++) {
+                                MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor2,
+                                         MSG_TAG_012, MPI_COMM_WORLD);
+                                MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor2,
+                                         MSG_TAG_012, MPI_COMM_WORLD, &status);
+                            }
+                            /* do the left process second because it does the timing and needs to include time to send to the right process. */
+                            for (k = 0; k < j; k++) {
+                                MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                         MSG_TAG_012, MPI_COMM_WORLD, &status);
+                                MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                         MSG_TAG_012, MPI_COMM_WORLD);
+                            }
+                            t = (MPI_Wtime() - t0) / (2 * nrepeat012);
+                        }
+                        MPI_Bcast(&bwdata012[n].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
+                        break;
+                    case RIGHT_PROCESS:
+                        bwdata012[n].t = LONGTIME;
+                        t1 = 0;
+                        for (i = 0; i < TRIALS; i++) {
+                            if (bNoCache) {
+                                if (bufalign != 0) {
+                                    args012.sbuff =
+                                        memtmp +
+                                        ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) +
+                                          bufoffset) % bufalign);
+                                    /* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
+                                } else {
+                                    args012.sbuff = memtmp;
+                                    /* args012.rbuff = memtmp1; */
+                                }
+                            }
+
+                            Sync012(&args012);
+                            t0 = MPI_Wtime();
+                            for (j = 0; j < nrepeat012; j++) {
+                                MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                         MSG_TAG_012, MPI_COMM_WORLD, &status);
+                                MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                         MSG_TAG_012, MPI_COMM_WORLD);
+                                if (bNoCache) {
+                                    args012.sbuff += args012.bufflen;
+                                    /* args012.rbuff += args012.bufflen; */
+                                }
+                            }
+                            t = (MPI_Wtime() - t0) / (2 * nrepeat012);
+                        }
+                        MPI_Bcast(&bwdata012[n].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
+                        break;
+                }
+                tlast012 = bwdata012[n].t;
+                bwdata012[n].bits = args012.bufflen * CHARSIZE;
+                bwdata012[n].bps = bwdata012[n].bits / (bwdata012[n].t * 1024 * 1024);
+                bwdata012[n].repeat = nrepeat012;
+                if (itrial < ntrials) {
+                    dtrials[itrial] = MIN(dtrials[itrial], bwdata012[n].t);
+                }
+
+                if (g_nIproc == 0) {
+                    if (bSavePert) {
+                        fprintf(out, "\t%0.9f", bwdata012[n].t);
+                        fflush(out);
+                    }
+                    if (printopt) {
+                        printf(" %0.9f", tlast012);
+                        fflush(stdout);
+                    }
+                }
+            }
+            if (g_nIproc == 0) {
+                if (bSavePert) {
+                    fprintf(out, "\n");
+                    fflush(out);
+                }
+                if (printopt) {
+                    printf("\n");
+                    fflush(stdout);
+                }
+            }
+
+            free(memtmp);
+            free(memtmp1);
+#endif
+
+#ifdef CREATE_SINGLE_CURVE
+            /*****************************************************/
+            /*         Run a trial between rank 0, 1 and 2       */
+            /*****************************************************/
+
+            MPI_Barrier(MPI_COMM_WORLD);
+
+
+            /* Calculate howmany times to repeat the experiment. */
+            if (g_nIproc == 0) {
+                if (args012.bufflen == 0)
+                    nrepeat012 = g_latency012_reps;
+                else
+                    nrepeat012 = (int) (MAX((RUNTM / ((double) args012.bufflen /
+                                                      (args012.bufflen - inc + 1.0) * tlast012)),
+                                            TRIALS));
+                MPI_Bcast(&nrepeat012, 1, MPI_INT, 0, MPI_COMM_WORLD);
+            } else {
+                MPI_Bcast(&nrepeat012, 1, MPI_INT, 0, MPI_COMM_WORLD);
+            }
+
+            /* Allocate the buffer */
+            args012.bufflen = len + pert;
+            /* printf("allocating %d bytes\n", args12.bufflen * nrepeat012 + bufalign); */
+            if (bNoCache) {
+                if ((args012.sbuff =
+                     (char *) malloc(args012.bufflen * nrepeat012 + bufalign)) == (char *) NULL) {
+                    fprintf(stdout, "Couldn't allocate memory\n");
+                    fflush(stdout);
+                    break;
+                }
+            } else {
+                if ((args012.sbuff = (char *) malloc(args012.bufflen + bufalign)) == (char *) NULL) {
+                    fprintf(stdout, "Couldn't allocate memory\n");
+                    fflush(stdout);
+                    break;
+                }
+            }
+            /* if ((args012.rbuff = (char *)malloc(args012.bufflen * nrepeat012 + bufalign)) == (char *)NULL) */
+            if ((args012.rbuff = (char *) malloc(args012.bufflen + bufalign)) == (char *) NULL) {
+                fprintf(stdout, "Couldn't allocate memory\n");
+                fflush(stdout);
+                break;
+            }
+
+            /* save the original pointers in case alignment moves them */
+            memtmp = args012.sbuff;
+            memtmp1 = args012.rbuff;
+
+            /* Possibly align the data buffer */
+            if (!bNoCache) {
+                if (bufalign != 0) {
+                    args012.sbuff +=
+                        (bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) +
+                         bufoffset) % bufalign;
+                    /* args12.rbuff += (bufalign - ((MPI_Aint)args12.rbuff % bufalign) + bufoffset) % bufalign; */
+                }
+            }
+            args012.rbuff +=
+                (bufalign - (POINTER_TO_INT(args012.rbuff) % bufalign) + bufoffset) % bufalign;
+
+            if (g_nIproc == 0 && printopt) {
+                printf("%3d: %9d bytes %4d times --> ", n, args012.bufflen, nrepeat012);
+                fflush(stdout);
+            }
+
+            /* Finally, we get to transmit or receive and time */
+            switch (g_proc_loc) {
+                case LEFT_PROCESS:
+                    bwdata012[n].t = LONGTIME;
+                    t1 = 0;
+                    for (i = 0; i < TRIALS; i++) {
+                        if (bNoCache) {
+                            if (bufalign != 0) {
+                                args012.sbuff =
+                                    memtmp +
+                                    ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) +
+                                      bufoffset) % bufalign);
+                                /* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
+                            } else {
+                                args012.sbuff = memtmp;
+                                /* args012.rbuff = memtmp1; */
+                            }
+                        }
+
+                        Sync012(&args012);
+                        t0 = MPI_Wtime();
+                        for (j = 0; j < nrepeat012; j++) {
+                            MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                     MSG_TAG_012, MPI_COMM_WORLD);
+                            MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                     MSG_TAG_012, MPI_COMM_WORLD, &status);
+                            if (bNoCache) {
+                                args012.sbuff += args012.bufflen;
+                                /* args012.rbuff += args012.bufflen; */
+                            }
+                        }
+                        t = (MPI_Wtime() - t0) / (2 * nrepeat012);
+
+                        t1 += t;
+                        bwdata012[n].t = MIN(bwdata012[n].t, t);
+                    }
+                    MPI_Bcast(&bwdata012[n].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
+                    break;
+                case MIDDLE_PROCESS:
+                    bwdata012[n].t = LONGTIME;
+                    t1 = 0;
+                    for (i = 0; i < TRIALS; i++) {
+                        if (bNoCache) {
+                            if (bufalign != 0) {
+                                args012.sbuff =
+                                    memtmp +
+                                    ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) +
+                                      bufoffset) % bufalign);
+                                /* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
+                            } else {
+                                args012.sbuff = memtmp;
+                                /* args012.rbuff = memtmp1; */
+                            }
+                        }
+
+                        Sync012(&args012);
+                        t0 = MPI_Wtime();
+                        for (j = 0; j < nrepeat012; j++) {
+                            MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                     MSG_TAG_012, MPI_COMM_WORLD, &status);
+                            MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor2,
+                                     MSG_TAG_012, MPI_COMM_WORLD);
+                            MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor2,
+                                     MSG_TAG_012, MPI_COMM_WORLD, &status);
+                            MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                     MSG_TAG_012, MPI_COMM_WORLD);
+                            if (bNoCache) {
+                                args012.sbuff += args012.bufflen;
+                                /* args012.rbuff += args012.bufflen; */
+                            }
+                        }
+                        t = (MPI_Wtime() - t0) / (2 * nrepeat012);
+                    }
+                    MPI_Bcast(&bwdata012[n].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
+                    break;
+                case RIGHT_PROCESS:
+                    bwdata012[n].t = LONGTIME;
+                    t1 = 0;
+                    for (i = 0; i < TRIALS; i++) {
+                        if (bNoCache) {
+                            if (bufalign != 0) {
+                                args012.sbuff =
+                                    memtmp +
+                                    ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) +
+                                      bufoffset) % bufalign);
+                                /* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
+                            } else {
+                                args012.sbuff = memtmp;
+                                /* args012.rbuff = memtmp1; */
+                            }
+                        }
+
+                        Sync012(&args012);
+                        t0 = MPI_Wtime();
+                        for (j = 0; j < nrepeat012; j++) {
+                            MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                     MSG_TAG_012, MPI_COMM_WORLD, &status);
+                            MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor,
+                                     MSG_TAG_012, MPI_COMM_WORLD);
+                            if (bNoCache) {
+                                args012.sbuff += args012.bufflen;
+                                /* args012.rbuff += args012.bufflen; */
+                            }
+                        }
+                        t = (MPI_Wtime() - t0) / (2 * nrepeat012);
+                    }
+                    MPI_Bcast(&bwdata012[n].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
+                    break;
+            }
+            tlast012 = bwdata012[n].t;
+            bwdata012[n].bits = args012.bufflen * CHARSIZE;
+            bwdata012[n].bps = bwdata012[n].bits / (bwdata012[n].t * 1024 * 1024);
+            bwdata012[n].repeat = nrepeat012;
+
+            if (g_nIproc == 0) {
+                if (bSavePert) {
+                    if (bUseMegaBytes)
+                        fprintf(out, "%f\t%0.9f\n", bwdata012[n].bps / 8, bwdata012[n].t);
+                    else
+                        fprintf(out, "%f\t%0.9f\n", bwdata012[n].bps, bwdata012[n].t);
+                    fflush(out);
+                }
+            }
+
+            free(memtmp);
+            free(memtmp1);
+
+            if (g_nIproc == 0 && printopt) {
+                if (bUseMegaBytes)
+                    printf(" %6.2f MBps in %0.9f sec\n", bwdata012[n].bps / 8, tlast012);
+                else
+                    printf(" %6.2f Mbps in %0.9f sec\n", bwdata012[n].bps, tlast012);
+                fflush(stdout);
+            }
+#endif
+
+        }       /* End of perturbation loop */
+
+        if (!bSavePert) {       /* && g_nIproc == 0) */
+            /* if we didn't save all of the perturbation loops, find the max and save it */
+            int index01 = 1, index12 = 1;
+            double dmax01 = bwdata01[n - 1].bps;
+            double dmax12 = bwdata12[n - 1].bps;
+#ifdef CREATE_SINGLE_CURVE
+            int index012 = 1;
+            double dmax012 = bwdata012[n - 1].bps;
+#endif
+            for (; ipert > 1; ipert--) {
+                if (bwdata01[n - ipert].bps > dmax01) {
+                    index01 = ipert;
+                    dmax01 = bwdata01[n - ipert].bps;
+                }
+                if (bwdata12[n - ipert].bps > dmax12) {
+                    index12 = ipert;
+                    dmax12 = bwdata12[n - ipert].bps;
+                }
+#ifdef CREATE_SINGLE_CURVE
+                if (bwdata012[n - ipert].bps > dmax012) {
+                    index012 = ipert;
+                    dmax012 = bwdata012[n - ipert].bps;
+                }
+#endif
+            }
+            /* get the left stuff out */
+            MPI_Bcast(&index01, 1, MPI_INT, g_left_rank, MPI_COMM_WORLD);
+            MPI_Bcast(&bwdata01[n - index01].bits, 1, MPI_INT, g_left_rank, MPI_COMM_WORLD);
+            MPI_Bcast(&bwdata01[n - index01].bps, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
+            MPI_Bcast(&bwdata01[n - index01].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
+            /* get the right stuff out */
+            MPI_Bcast(&index12, 1, MPI_INT, g_middle_rank, MPI_COMM_WORLD);
+            MPI_Bcast(&bwdata12[n - index12].bps, 1, MPI_DOUBLE, g_middle_rank, MPI_COMM_WORLD);
+            MPI_Bcast(&bwdata12[n - index12].t, 1, MPI_DOUBLE, g_middle_rank, MPI_COMM_WORLD);
+            if (g_nIproc == 0) {
+                if (bUseMegaBytes) {
+                    fprintf(out, "%d\t%f\t%0.9f\t", bwdata01[n - index01].bits / 8,
+                            bwdata01[n - index01].bps / 8, bwdata01[n - index01].t);
+                    fprintf(out, "%f\t%0.9f\t", bwdata12[n - index12].bps / 8,
+                            bwdata12[n - index12].t);
+#ifdef CREATE_SINGLE_CURVE
+                    fprintf(out, "%f\t%0.9f\n", bwdata012[n - index012].bps / 8,
+                            bwdata012[n - index012].t);
+#endif
+                } else {
+                    fprintf(out, "%d\t%f\t%0.9f\t", bwdata01[n - index01].bits / 8,
+                            bwdata01[n - index01].bps, bwdata01[n - index01].t);
+                    fprintf(out, "%f\t%0.9f\t", bwdata12[n - index12].bps, bwdata12[n - index12].t);
+#ifdef CREATE_SINGLE_CURVE
+                    fprintf(out, "%f\t%0.9f\n", bwdata012[n - index012].bps,
+                            bwdata012[n - index012].t);
+#endif
+                }
+#ifdef CREATE_DIFFERENCE_CURVES
+                for (itrial = 0; itrial < ntrials && dtrials[itrial] != LONGTIME; itrial++) {
+                    fprintf(out, "%0.9f\t", dtrials[itrial]);
+                }
+                fprintf(out, "\n");
+#endif
+                fflush(out);
+            }
+        }
+    }   /* End of main loop  */
+
+    if (g_nIproc == 0)
+        fclose(out);
+    /* THE_END:          */
+    MPI_Finalize();
+    free(bwdata01);
+    free(bwdata12);
+    free(bwdata012);
+    (void)t1; // avoid "t1 set but not used"
+    return 0;
+}
+
+int Setup(int middle_rank, ArgStruct * p01, ArgStruct * p12, ArgStruct * p012)
+{
+    char s[255];
+    int len = 255;
+
+    p01->iproc = p12->iproc = p012->iproc = g_nIproc;
+
+    MPI_Get_processor_name(s, &len);
+    /*gethostname(s, len); */
+    printf("%d: %s\n", p01->iproc, s);
+    fflush(stdout);
+
+    switch (middle_rank) {
+        case 0:
+            switch (g_nIproc) {
+                case 0:
+                    g_proc_loc = MIDDLE_PROCESS;
+                    p01->nbor = 2;
+                    p01->tr = FALSE;
+                    p12->nbor = 1;
+                    p12->tr = TRUE;
+                    p012->nbor = 2;
+                    p012->nbor2 = 1;
+                    break;
+                case 1:
+                    g_proc_loc = RIGHT_PROCESS;
+                    p01->nbor = -1;
+                    p01->tr = FALSE;
+                    p12->nbor = 0;
+                    p12->tr = FALSE;
+                    p012->nbor = 0;
+                    p012->nbor2 = -1;
+                    break;
+                case 2:
+                    g_proc_loc = LEFT_PROCESS;
+                    p01->nbor = 0;
+                    p01->tr = TRUE;
+                    p12->nbor = -1;
+                    p12->tr = FALSE;
+                    p012->nbor = 0;
+                    p012->nbor2 = -1;
+                    break;
+            }
+            g_left_rank = 2;
+            g_middle_rank = 0;
+            g_right_rank = 1;
+            break;
+        case 1:
+            switch (g_nIproc) {
+                case 0:
+                    g_proc_loc = LEFT_PROCESS;
+                    p01->nbor = 1;
+                    p01->tr = TRUE;
+                    p12->nbor = -1;
+                    p12->tr = FALSE;
+                    p012->nbor = 1;
+                    p012->nbor2 = -1;
+                    break;
+                case 1:
+                    g_proc_loc = MIDDLE_PROCESS;
+                    p01->nbor = 0;
+                    p01->tr = FALSE;
+                    p12->nbor = 2;
+                    p12->tr = TRUE;
+                    p012->nbor = 0;
+                    p012->nbor2 = 2;
+                    break;
+                case 2:
+                    g_proc_loc = RIGHT_PROCESS;
+                    p01->nbor = -1;
+                    p01->tr = FALSE;
+                    p12->nbor = 1;
+                    p12->tr = FALSE;
+                    p012->nbor = 1;
+                    p012->nbor2 = -1;
+                    break;
+            }
+            g_left_rank = 0;
+            g_middle_rank = 1;
+            g_right_rank = 2;
+            break;
+        case 2:
+            switch (g_nIproc) {
+                case 0:
+                    g_proc_loc = RIGHT_PROCESS;
+                    p01->nbor = -1;
+                    p01->tr = FALSE;
+                    p12->nbor = 2;
+                    p12->tr = FALSE;
+                    p012->nbor = 2;
+                    p012->nbor2 = -1;
+                    break;
+                case 1:
+                    g_proc_loc = LEFT_PROCESS;
+                    p01->nbor = 2;
+                    p01->tr = TRUE;
+                    p12->nbor = -1;
+                    p12->tr = FALSE;
+                    p012->nbor = 2;
+                    p012->nbor2 = -1;
+                    break;
+                case 2:
+                    g_proc_loc = MIDDLE_PROCESS;
+                    p01->nbor = 1;
+                    p01->tr = FALSE;
+                    p12->nbor = 0;
+                    p12->tr = TRUE;
+                    p012->nbor = 1;
+                    p012->nbor2 = 0;
+                    break;
+            }
+            g_left_rank = 1;
+            g_middle_rank = 2;
+            g_right_rank = 0;
+            break;
+    }
+
+    return 1;
+}
+
+void Sync(ArgStruct * p)
+{
+    MPI_Status status;
+    if (p->tr) {
+        MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+        MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+        MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+    } else {
+        MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+        MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+        MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+    }
+}
+
+void Sync012(ArgStruct * p)
+{
+    MPI_Status status;
+    switch (g_proc_loc) {
+        case LEFT_PROCESS:
+            MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+            MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+            MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+            break;
+        case MIDDLE_PROCESS:
+            MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+            MPI_Send(NULL, 0, MPI_BYTE, p->nbor2, 1, MPI_COMM_WORLD);
+            MPI_Recv(NULL, 0, MPI_BYTE, p->nbor2, 1, MPI_COMM_WORLD, &status);
+            MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+            MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+            MPI_Send(NULL, 0, MPI_BYTE, p->nbor2, 1, MPI_COMM_WORLD);
+            break;
+        case RIGHT_PROCESS:
+            MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+            MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+            MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+            break;
+    }
+}
+
+int DetermineLatencyReps(ArgStruct * p)
+{
+    MPI_Status status;
+    double t0, duration = 0;
+    int reps = 1, prev_reps = 0;
+    int i;
+
+    /* prime the send/receive pipes */
+    Sync(p);
+    Sync(p);
+    Sync(p);
+
+    /* test how long it takes to send n messages
+     * where n = 1, 2, 4, 8, 16, 32, ...
+     */
+    t0 = MPI_Wtime();
+    t0 = MPI_Wtime();
+    t0 = MPI_Wtime();
+    while ((duration < RUNTM) || (duration < MAX_LAT_TIME && reps < 1000)) {
+        t0 = MPI_Wtime();
+        for (i = 0; i < reps - prev_reps; i++) {
+            Sync(p);
+        }
+        duration += MPI_Wtime() - t0;
+        prev_reps = reps;
+        reps = reps * 2;
+
+        /* use duration from the root only */
+        if (p->tr)
+            MPI_Send(&duration, 1, MPI_DOUBLE, p->nbor, 2, MPI_COMM_WORLD);
+        else
+            MPI_Recv(&duration, 1, MPI_DOUBLE, p->nbor, 2, MPI_COMM_WORLD, &status);
+    }
+
+    return reps;
+}
+
+int DetermineLatencyReps012(ArgStruct * p)
+{
+    double t0, duration = 0;
+    int reps = 1, prev_reps = 0;
+    int i;
+
+    /* prime the send/receive pipes */
+    Sync012(p);
+    Sync012(p);
+    Sync012(p);
+
+    /* test how long it takes to send n messages
+     * where n = 1, 2, 4, 8, 16, 32, ...
+     */
+    t0 = MPI_Wtime();
+    t0 = MPI_Wtime();
+    t0 = MPI_Wtime();
+    while ((duration < RUNTM) || (duration < MAX_LAT_TIME && reps < 1000)) {
+        t0 = MPI_Wtime();
+        for (i = 0; i < reps - prev_reps; i++) {
+            Sync012(p);
+        }
+        duration += MPI_Wtime() - t0;
+        prev_reps = reps;
+        reps = reps * 2;
+
+        /* use duration from the root only */
+        MPI_Bcast(&duration, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
+    }
+
+    return reps;
+}
+
+double TestLatency(ArgStruct * p)
+{
+    double latency, t0, min_latency = LONGTIME;
+    int i, j;
+    MPI_Status status;
+    char str[100];
+
+    /* calculate the latency between rank 0 and rank 1 */
+    p->latency_reps = DetermineLatencyReps(p);
+    if (/*p->latency_reps < 1024 && */ p->tr) {
+        if (g_proc_loc == LEFT_PROCESS) {
+            sprintf(str, "%d <-> %d      ", p->iproc, p->nbor);
+        } else {
+            sprintf(str, "      %d <-> %d", p->iproc, p->nbor);
+        }
+        /*printf("To determine %s latency, using %d reps\n", p->iproc == 0 ? "0 -> 1     " : "     1 -> 2", p->latency_reps); */
+        printf("To determine %s latency, using %d reps.\n", str, p->latency_reps);
+        fflush(stdout);
+    }
+
+    for (j = 0; j < TRIALS; j++) {
+        Sync(p);
+        t0 = MPI_Wtime();
+        t0 = MPI_Wtime();
+        t0 = MPI_Wtime();
+        t0 = MPI_Wtime();
+        for (i = 0; i < p->latency_reps; i++) {
+            if (p->tr) {
+                MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+                MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+            } else {
+                MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+                MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+            }
+        }
+        latency = (MPI_Wtime() - t0) / (2 * p->latency_reps);
+        min_latency = MIN(min_latency, latency);
+    }
+
+    return min_latency;
+}
+
+double TestLatency012(ArgStruct * p)
+{
+    double latency, t0, min_latency = LONGTIME;
+    int i, j;
+    MPI_Status status;
+
+    g_latency012_reps = DetermineLatencyReps012(p);
+    if (g_proc_loc == MIDDLE_PROCESS) {
+        printf("To determine %d <-- %d --> %d latency, using %d reps\n", p->nbor, p->iproc,
+               p->nbor2, g_latency012_reps);
+        fflush(stdout);
+    }
+
+    for (j = 0; j < TRIALS; j++) {
+        Sync012(p);
+        t0 = MPI_Wtime();
+        t0 = MPI_Wtime();
+        t0 = MPI_Wtime();
+        t0 = MPI_Wtime();
+        for (i = 0; i < g_latency012_reps; i++) {
+            switch (g_proc_loc) {
+                case LEFT_PROCESS:
+                    MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+                    MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+                    break;
+                case MIDDLE_PROCESS:
+                    MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+                    MPI_Send(NULL, 0, MPI_BYTE, p->nbor2, 1, MPI_COMM_WORLD);
+                    MPI_Recv(NULL, 0, MPI_BYTE, p->nbor2, 1, MPI_COMM_WORLD, &status);
+                    MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+                    break;
+                case RIGHT_PROCESS:
+                    MPI_Recv(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD, &status);
+                    MPI_Send(NULL, 0, MPI_BYTE, p->nbor, 1, MPI_COMM_WORLD);
+                    break;
+            }
+        }
+        latency = (MPI_Wtime() - t0) / (2 * g_latency012_reps);
+        min_latency = MIN(min_latency, latency);
+    }
+
+    return min_latency;
+}
+
+void SendTime(ArgStruct * p, double *t)
+{
+    MPI_Send(t, 1, MPI_DOUBLE, p->nbor, 2, MPI_COMM_WORLD);
+}
+
+void RecvTime(ArgStruct * p, double *t)
+{
+    MPI_Status status;
+    MPI_Recv(t, 1, MPI_DOUBLE, p->nbor, 2, MPI_COMM_WORLD, &status);
+}
+
+void SendReps(ArgStruct * p, int *rpt)
+{
+    MPI_Send(rpt, 1, MPI_INT, p->nbor, 2, MPI_COMM_WORLD);
+}
+
+void RecvReps(ArgStruct * p, int *rpt)
+{
+    MPI_Status status;
+    MPI_Recv(rpt, 1, MPI_INT, p->nbor, 2, MPI_COMM_WORLD, &status);
+}
diff --git a/t/mpi/mpich_basic/netmpi.c b/t/mpi/mpich_basic/netmpi.c
new file mode 100644
index 000000000000..2df461e162b0
--- /dev/null
+++ b/t/mpi/mpich_basic/netmpi.c
@@ -0,0 +1,667 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#ifdef HAVE_WINDOWS_H
+#include <winsock2.h>
+#include <windows.h>
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include "mpi.h"
+#include "GetOpt.h"
+#include <string.h>
+
+#ifndef BOOL
+typedef int BOOL;
+#endif
+#ifndef TRUE
+#define TRUE 1
+#endif
+#ifndef FALSE
+#define FALSE 0
+#endif
+
+#define  TRIALS 	3
+#define  REPEAT 	1000
+int g_NSAMP = 250;
+#define  PERT		3
+/*#define  LATENCYREPS	1000*/
+int g_LATENCYREPS = 1000;
+#define  LONGTIME	1e99
+#define  CHARSIZE	8
+#define  PATIENCE	50
+#define  RUNTM		0.25
+double g_STOPTM = 0.1;
+#define  MAXINT 	2147483647
+
+#define ABS(x)		(((x) < 0)?(-(x)):(x))
+#define MIN(x, y)	(((x) < (y))?(x):(y))
+#define MAX(x, y)	(((x) > (y))?(x):(y))
+
+int g_nIproc = 0, g_nNproc = 0;
+
+typedef struct protocolstruct ProtocolStruct;
+struct protocolstruct {
+    int nbor, iproc;
+};
+
+typedef struct argstruct ArgStruct;
+struct argstruct {
+    /* This is the common information that is needed for all tests      */
+    char *host;                 /* Name of receiving host                       */
+    char *buff;                 /* Transmitted buffer                           */
+    char *buff1;                /* Transmitted buffer                           */
+    size_t bufflen;             /* Length of transmitted buffer                 */
+    int tr,                     /* Transmit flag                                */
+     nbuff;                     /* Number of buffers to transmit                */
+
+    /* Now we work with a union of information for protocol dependent stuff */
+    ProtocolStruct prot;        /* Structure holding necessary info for TCP */
+};
+
+typedef struct data Data;
+struct data {
+    double t;
+    double bps;
+    double variance;
+    int bits;
+    int repeat;
+};
+
+double When(void);
+int Setup(ArgStruct * p);
+void Sync(ArgStruct * p);
+void SendData(ArgStruct * p);
+void RecvData(ArgStruct * p);
+void SendRecvData(ArgStruct * p);
+void SendTime(ArgStruct * p, double *t, int *rpt);
+void RecvTime(ArgStruct * p, double *t, int *rpt);
+int Establish(ArgStruct * p);
+int CleanUp(ArgStruct * p);
+double TestLatency(ArgStruct * p);
+double TestSyncTime(ArgStruct * p);
+void PrintOptions(void);
+int DetermineLatencyReps(ArgStruct * p);
+
+void PrintOptions()
+{
+    printf("\n");
+    printf("Usage: netpipe flags\n");
+    printf(" flags:\n");
+    printf("       -reps #iterations\n");
+    printf("       -time stop_time\n");
+    printf("       -start initial_msg_size\n");
+    printf("       -end final_msg_size\n");
+    printf("       -out outputfile\n");
+    printf("       -nocache\n");
+    printf("       -headtohead\n");
+    printf("       -pert\n");
+    printf("       -noprint\n");
+    printf("       -onebuffer largest_buffer_size\n");
+    printf("Requires exactly two processes\n");
+    printf("\n");
+}
+
+int main(int argc, char *argv[])
+{
+    FILE *out = 0;              /* Output data file                     */
+    char s[255];                /* Generic string                       */
+    char *memtmp;
+    char *memtmp1;
+
+    int i, j, n, nq,            /* Loop indices                         */
+     bufoffset = 0,             /* Align buffer to this                 */
+        bufalign = 16 * 1024,   /* Boundary to align buffer to          */
+        nrepeat,        /* Number of time to do the transmission */
+        nzero = 0, inc = 1,     /* Increment value                      */
+        detailflag = 0, /* Set to examine the signature curve detail */
+        pert,   /* Perturbation value                   */
+        ipert,  /* index of the perturbation loop       */
+        start = 0,      /* Starting value for signature curve   */
+        end = MAXINT,   /* Ending value for signature curve     */
+        streamopt = 0,  /* Streaming mode flag                  */
+        printopt = 1;   /* Debug print statements flag          */
+    int one_buffer = 0;
+    int onebuffersize = 100 * 1024 * 1024;
+    int quit = 0;
+    size_t len;                 /* Number of bytes to be transmitted    */
+
+    ArgStruct args;             /* Argumentsfor all the calls           */
+
+    double t, t0, t1, t2,       /* Time variables                       */
+     tlast,                     /* Time for the last transmission       */
+     tzero = 0, latency,        /* Network message latency              */
+        synctime;       /* Network synchronization time         */
+
+    Data *bwdata;               /* Bandwidth curve data                 */
+
+    BOOL bNoCache = FALSE;
+    BOOL bHeadToHead = FALSE;
+    BOOL bSavePert = FALSE;
+    BOOL bUseMegaBytes = FALSE;
+
+    MPI_Init(&argc, &argv);
+
+    MPI_Comm_size(MPI_COMM_WORLD, &g_nNproc);
+    MPI_Comm_rank(MPI_COMM_WORLD, &g_nIproc);
+
+    if (g_nNproc != 2) {
+        if (g_nIproc == 0)
+            PrintOptions();
+        MPI_Finalize();
+        exit(0);
+    }
+
+    GetOptDouble(&argc, &argv, "-time", &g_STOPTM);
+    GetOptInt(&argc, &argv, "-reps", &g_NSAMP);
+    GetOptInt(&argc, &argv, "-start", &start);
+    GetOptInt(&argc, &argv, "-end", &end);
+    one_buffer = GetOptInt(&argc, &argv, "-onebuffer", &onebuffersize);
+    if (one_buffer) {
+        if (onebuffersize < 1) {
+            one_buffer = 0;
+        } else {
+            onebuffersize += bufalign;
+        }
+    }
+    bNoCache = GetOpt(&argc, &argv, "-nocache");
+    bHeadToHead = GetOpt(&argc, &argv, "-headtohead");
+    bUseMegaBytes = GetOpt(&argc, &argv, "-mb");
+    if (GetOpt(&argc, &argv, "-noprint"))
+        printopt = 0;
+    bSavePert = GetOpt(&argc, &argv, "-pert");
+
+    bwdata = malloc((g_NSAMP + 1) * sizeof(Data));
+
+    if (g_nIproc == 0)
+        strcpy(s, "Netpipe.out");
+    GetOptString(&argc, &argv, "-out", s);
+
+    if (start > end) {
+        fprintf(stdout, "Start MUST be LESS than end\n");
+        exit(420132);
+    }
+
+    args.nbuff = TRIALS;
+
+    Setup(&args);
+    Establish(&args);
+
+    if (args.tr) {
+        if ((out = fopen(s, "w")) == NULL) {
+            fprintf(stdout, "Can't open %s for output\n", s);
+            exit(1);
+        }
+    }
+
+    latency = TestLatency(&args);
+    synctime = TestSyncTime(&args);
+
+
+    if (args.tr) {
+        SendTime(&args, &latency, &nzero);
+    } else {
+        RecvTime(&args, &latency, &nzero);
+    }
+    if (args.tr && printopt) {
+        printf("Latency: %0.9f\n", latency);
+        fflush(stdout);
+        printf("Sync Time: %0.9f\n", synctime);
+        fflush(stdout);
+        printf("Now starting main loop\n");
+        fflush(stdout);
+    }
+    tlast = latency;
+    inc = (start > 1 && !detailflag) ? start / 2 : inc;
+    args.bufflen = start;
+
+    if (one_buffer) {
+        args.buff = (char *) malloc(onebuffersize);
+        args.buff1 = (char *) malloc(onebuffersize);
+    }
+
+    /* Main loop of benchmark */
+    for (nq = n = 0, len = start;
+         n < g_NSAMP && tlast < g_STOPTM && len <= end && !quit; len = len + inc, nq++) {
+        if (nq > 2 && !detailflag)
+            inc = ((nq % 2)) ? inc + inc : inc;
+
+        /* This is a perturbation loop to test nearby values */
+        for (ipert = 0, pert = (!detailflag && inc > PERT + 1) ? -PERT : 0;
+             pert <= PERT && !quit;
+             ipert++, n++, pert += (!detailflag && inc > PERT + 1) ? PERT : PERT + 1) {
+
+            /* Calculate howmany times to repeat the experiment. */
+            if (args.tr) {
+                if (args.bufflen == 0)
+                    nrepeat = g_LATENCYREPS;
+                else
+                    nrepeat = (int) (MAX((RUNTM / ((double) args.bufflen /
+                                                   (args.bufflen - inc + 1.0) * tlast)), TRIALS));
+                SendTime(&args, &tzero, &nrepeat);
+            } else {
+                nrepeat = 1;    /* Just needs to be greater than zero */
+                RecvTime(&args, &tzero, &nrepeat);
+            }
+
+            /* Allocate the buffer */
+            args.bufflen = len + pert;
+            if (one_buffer) {
+                if (bNoCache) {
+                    if (args.bufflen * nrepeat + bufalign > onebuffersize) {
+                        fprintf(stdout, "Exceeded user specified buffer size\n");
+                        fflush(stdout);
+                        quit = 1;
+                        break;
+                    }
+                } else {
+                    if (args.bufflen + bufalign > onebuffersize) {
+                        fprintf(stdout, "Exceeded user specified buffer size\n");
+                        fflush(stdout);
+                        quit = 1;
+                        break;
+                    }
+                }
+            } else {
+                /* printf("allocating %d bytes\n",
+                 * args.bufflen * nrepeat + bufalign); */
+                if (bNoCache) {
+                    if ((args.buff =
+                         (char *) malloc(args.bufflen * nrepeat + bufalign)) == (char *) NULL) {
+                        fprintf(stdout, "Couldn't allocate memory\n");
+                        fflush(stdout);
+                        break;
+                    }
+                } else {
+                    if ((args.buff = (char *) malloc(args.bufflen + bufalign)) == (char *) NULL) {
+                        fprintf(stdout, "Couldn't allocate memory\n");
+                        fflush(stdout);
+                        break;
+                    }
+                }
+                /* if ((args.buff1 = (char *)malloc(args.bufflen * nrepeat + bufalign)) == (char *)NULL) */
+                if ((args.buff1 = (char *) malloc(args.bufflen + bufalign)) == (char *) NULL) {
+                    fprintf(stdout, "Couldn't allocate memory\n");
+                    fflush(stdout);
+                    break;
+                }
+            }
+            /* Possibly align the data buffer */
+            memtmp = args.buff;
+            memtmp1 = args.buff1;
+
+            if (!bNoCache) {
+                if (bufalign != 0) {
+                    args.buff +=
+                        (bufalign - ((MPI_Aint) args.buff % bufalign) + bufoffset) % bufalign;
+                    /* args.buff1 += (bufalign - ((MPI_Aint)args.buff1 % bufalign) + bufoffset) % bufalign; */
+                }
+            }
+            args.buff1 += (bufalign - ((MPI_Aint) args.buff1 % bufalign) + bufoffset) % bufalign;
+
+            if (args.tr && printopt) {
+                fprintf(stdout, "%3d: %9zu bytes %4d times --> ", n, args.bufflen, nrepeat);
+                fflush(stdout);
+            }
+
+            /* Finally, we get to transmit or receive and time */
+            if (args.tr) {
+                bwdata[n].t = LONGTIME;
+                t2 = t1 = 0;
+                for (i = 0; i < TRIALS; i++) {
+                    if (bNoCache) {
+                        if (bufalign != 0) {
+                            args.buff =
+                                memtmp +
+                                ((bufalign - ((MPI_Aint) args.buff % bufalign) +
+                                  bufoffset) % bufalign);
+                            /* args.buff1 = memtmp1 + ((bufalign - ((MPI_Aint)args.buff1 % bufalign) + bufoffset) % bufalign); */
+                        } else {
+                            args.buff = memtmp;
+                            /* args.buff1 = memtmp1; */
+                        }
+                    }
+
+                    Sync(&args);
+                    t0 = When();
+                    for (j = 0; j < nrepeat; j++) {
+                        if (bHeadToHead)
+                            SendRecvData(&args);
+                        else {
+                            SendData(&args);
+                            if (!streamopt) {
+                                RecvData(&args);
+                            }
+                        }
+                        if (bNoCache) {
+                            args.buff += args.bufflen;
+                            /* args.buff1 += args.bufflen; */
+                        }
+                    }
+                    t = (When() - t0) / ((1 + !streamopt) * nrepeat);
+
+                    if (!streamopt) {
+                        t2 += t * t;
+                        t1 += t;
+                        bwdata[n].t = MIN(bwdata[n].t, t);
+                    }
+                }
+                if (!streamopt)
+                    SendTime(&args, &bwdata[n].t, &nzero);
+                else
+                    RecvTime(&args, &bwdata[n].t, &nzero);
+
+                if (!streamopt)
+                    bwdata[n].variance = t2 / TRIALS - t1 / TRIALS * t1 / TRIALS;
+
+            } else {
+                bwdata[n].t = LONGTIME;
+                t2 = t1 = 0;
+                for (i = 0; i < TRIALS; i++) {
+                    if (bNoCache) {
+                        if (bufalign != 0) {
+                            args.buff =
+                                memtmp +
+                                ((bufalign - ((MPI_Aint) args.buff % bufalign) +
+                                  bufoffset) % bufalign);
+                            /* args.buff1 = memtmp1 + ((bufalign - ((MPI_Aint)args.buff1 % bufalign) + bufoffset) % bufalign); */
+                        } else {
+                            args.buff = memtmp;
+                            /* args.buff1 = memtmp1; */
+                        }
+                    }
+
+                    Sync(&args);
+                    t0 = When();
+                    for (j = 0; j < nrepeat; j++) {
+                        if (bHeadToHead)
+                            SendRecvData(&args);
+                        else {
+                            RecvData(&args);
+                            if (!streamopt)
+                                SendData(&args);
+                        }
+                        if (bNoCache) {
+                            args.buff += args.bufflen;
+                            /* args.buff1 += args.bufflen; */
+                        }
+                    }
+                    t = (When() - t0) / ((1 + !streamopt) * nrepeat);
+
+                    if (streamopt) {
+                        t2 += t * t;
+                        t1 += t;
+                        bwdata[n].t = MIN(bwdata[n].t, t);
+                    }
+                }
+                if (streamopt)
+                    SendTime(&args, &bwdata[n].t, &nzero);
+                else
+                    RecvTime(&args, &bwdata[n].t, &nzero);
+
+                if (streamopt)
+                    bwdata[n].variance = t2 / TRIALS - t1 / TRIALS * t1 / TRIALS;
+            }
+            tlast = bwdata[n].t;
+            bwdata[n].bits = args.bufflen * CHARSIZE;
+            bwdata[n].bps = bwdata[n].bits / (bwdata[n].t * 1024 * 1024);
+            bwdata[n].repeat = nrepeat;
+
+            if (args.tr) {
+                if (bSavePert) {
+                    /* fprintf(out,"%f\t%f\t%d\t%d\t%f\n", bwdata[n].t, bwdata[n].bps,
+                     * bwdata[n].bits, bwdata[n].bits / 8, bwdata[n].variance); */
+                    if (bUseMegaBytes)
+                        fprintf(out, "%d\t%f\t%0.9f\n", bwdata[n].bits / 8, bwdata[n].bps / 8,
+                                bwdata[n].t);
+                    else
+                        fprintf(out, "%d\t%f\t%0.9f\n", bwdata[n].bits / 8, bwdata[n].bps,
+                                bwdata[n].t);
+                    fflush(out);
+                }
+            }
+            if (!one_buffer) {
+                free(memtmp);
+                free(memtmp1);
+            }
+            if (args.tr && printopt) {
+                if (bUseMegaBytes)
+                    fprintf(stdout, " %6.2f MBps in %0.9f sec\n", bwdata[n].bps / 8, tlast);
+                else
+                    fprintf(stdout, " %6.2f Mbps in %0.9f sec\n", bwdata[n].bps, tlast);
+                fflush(stdout);
+            }
+        }       /* End of perturbation loop */
+        if (!bSavePert && args.tr) {
+            /* if we didn't save all of the perturbation loops, find the max and save it */
+            int index = 1;
+            double dmax = bwdata[n - 1].bps;
+            for (; ipert > 1; ipert--) {
+                if (bwdata[n - ipert].bps > dmax) {
+                    index = ipert;
+                    dmax = bwdata[n - ipert].bps;
+                }
+            }
+            if (bUseMegaBytes)
+                fprintf(out, "%d\t%f\t%0.9f\n", bwdata[n - index].bits / 8,
+                        bwdata[n - index].bps / 8, bwdata[n - index].t);
+            else
+                fprintf(out, "%d\t%f\t%0.9f\n", bwdata[n - index].bits / 8, bwdata[n - index].bps,
+                        bwdata[n - index].t);
+            fflush(out);
+        }
+    }   /* End of main loop  */
+
+    if (args.tr)
+        fclose(out);
+    /* THE_END:          */
+    CleanUp(&args);
+    free(bwdata);
+    return 0;
+}
+
+
+/* Return the current time in seconds, using a double precision number. 	 */
+double When()
+{
+    return MPI_Wtime();
+}
+
+int Setup(ArgStruct * p)
+{
+    int nproc;
+    char s[255];
+    int len = 255;
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &p->prot.iproc);
+    MPI_Comm_size(MPI_COMM_WORLD, &nproc);
+
+    MPI_Get_processor_name(s, &len);
+    /*gethostname(s, len); */
+    printf("%d: %s\n", p->prot.iproc, s);
+    fflush(stdout);
+
+    if (p->prot.iproc == 0)
+        p->prot.nbor = 1;
+    else
+        p->prot.nbor = 0;
+
+    if (nproc < 2) {
+        printf("Need two processes\n");
+        printf("nproc: %i\n", nproc);
+        exit(-2);
+    }
+
+    if (p->prot.iproc == 0)
+        p->tr = 1;
+    else
+        p->tr = 0;
+    return 1;
+}
+
+void Sync(ArgStruct * p)
+{
+    char ch = 0;
+    MPI_Status status;
+    if (p->tr) {
+        MPI_Send(&ch, 0, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD);
+        MPI_Recv(&ch, 0, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD, &status);
+        MPI_Send(&ch, 0, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD);
+    } else {
+        MPI_Recv(&ch, 0, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD, &status);
+        MPI_Send(&ch, 0, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD);
+        MPI_Recv(&ch, 0, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD, &status);
+    }
+}
+
+int DetermineLatencyReps(ArgStruct * p)
+{
+    MPI_Status status;
+    double t0, duration = 0;
+    int reps = 1, prev_reps = 0;
+    int i;
+
+    /* prime the send/receive pipes */
+    Sync(p);
+    Sync(p);
+    Sync(p);
+
+    /* test how long it takes to send n messages
+     * where n = 1, 2, 4, 8, 16, 32, ...
+     */
+    t0 = When();
+    t0 = When();
+    t0 = When();
+    while ((duration < 1) || (duration < 3 && reps < 1000)) {
+        t0 = When();
+        for (i = 0; i < reps - prev_reps; i++) {
+            Sync(p);
+        }
+        duration += When() - t0;
+        prev_reps = reps;
+        reps = reps * 2;
+
+        /* use duration from the root only */
+        if (p->prot.iproc == 0)
+            MPI_Send(&duration, 1, MPI_DOUBLE, p->prot.nbor, 2, MPI_COMM_WORLD);
+        else
+            MPI_Recv(&duration, 1, MPI_DOUBLE, p->prot.nbor, 2, MPI_COMM_WORLD, &status);
+    }
+
+    return reps;
+}
+
+double TestLatency(ArgStruct * p)
+{
+    double latency, t0;
+    int i;
+
+    g_LATENCYREPS = DetermineLatencyReps(p);
+    if (g_LATENCYREPS < 1024 && p->prot.iproc == 0) {
+        printf("Using %d reps to determine latency\n", g_LATENCYREPS);
+        fflush(stdout);
+    }
+
+    p->bufflen = 0;
+    p->buff = NULL;     /*(char *)malloc(p->bufflen); */
+    p->buff1 = NULL;    /*(char *)malloc(p->bufflen); */
+    Sync(p);
+    t0 = When();
+    t0 = When();
+    t0 = When();
+    t0 = When();
+    for (i = 0; i < g_LATENCYREPS; i++) {
+        if (p->tr) {
+            SendData(p);
+            RecvData(p);
+        } else {
+            RecvData(p);
+            SendData(p);
+        }
+    }
+    latency = (When() - t0) / (2 * g_LATENCYREPS);
+    /*
+     * free(p->buff);
+     * free(p->buff1);
+     */
+
+    return latency;
+}
+
+double TestSyncTime(ArgStruct * p)
+{
+    double synctime, t0;
+    int i;
+
+    t0 = When();
+    t0 = When();
+    t0 = When();
+    t0 = When();
+    t0 = When();
+    t0 = When();
+    for (i = 0; i < g_LATENCYREPS; i++)
+        Sync(p);
+    synctime = (When() - t0) / g_LATENCYREPS;
+
+    return synctime;
+}
+
+void SendRecvData(ArgStruct * p)
+{
+    MPI_Status status;
+
+    /*MPI_Sendrecv(p->buff, p->bufflen, MPI_BYTE, p->prot.nbor, 1, p->buff1, p->bufflen, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD, &status); */
+
+    MPI_Request request;
+    MPI_Irecv(p->buff1, p->bufflen, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD, &request);
+    MPI_Send(p->buff, p->bufflen, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD);
+    MPI_Wait(&request, &status);
+
+    /*
+     * MPI_Send(p->buff, p->bufflen, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD);
+     * MPI_Recv(p->buff1, p->bufflen, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD, &status);
+     */
+}
+
+void SendData(ArgStruct * p)
+{
+    MPI_Send(p->buff, p->bufflen, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD);
+}
+
+void RecvData(ArgStruct * p)
+{
+    MPI_Status status;
+    MPI_Recv(p->buff1, p->bufflen, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD, &status);
+}
+
+
+void SendTime(ArgStruct * p, double *t, int *rpt)
+{
+    if (*rpt > 0)
+        MPI_Send(rpt, 1, MPI_INT, p->prot.nbor, 2, MPI_COMM_WORLD);
+    else
+        MPI_Send(t, 1, MPI_DOUBLE, p->prot.nbor, 2, MPI_COMM_WORLD);
+}
+
+void RecvTime(ArgStruct * p, double *t, int *rpt)
+{
+    MPI_Status status;
+    if (*rpt > 0)
+        MPI_Recv(rpt, 1, MPI_INT, p->prot.nbor, 2, MPI_COMM_WORLD, &status);
+    else
+        MPI_Recv(t, 1, MPI_DOUBLE, p->prot.nbor, 2, MPI_COMM_WORLD, &status);
+}
+
+int Establish(ArgStruct * p)
+{
+    return 1;
+}
+
+int CleanUp(ArgStruct * p)
+{
+    /*MPI_Barrier(MPI_COMM_WORLD); */
+    MPI_Finalize();
+    return 1;
+}
diff --git a/t/mpi/mpich_basic/patterns.c b/t/mpi/mpich_basic/patterns.c
new file mode 100644
index 000000000000..9cef6c9c83e0
--- /dev/null
+++ b/t/mpi/mpich_basic/patterns.c
@@ -0,0 +1,382 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#include "mpi.h"
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+#ifndef TRUE
+#define TRUE 1
+#endif
+#ifndef FALSE
+#define FALSE 0
+#endif
+
+#define DEFAULT_RNDV_SIZE 24*1024
+/* Prototypes */
+int SendRecvTest(int, int);
+int IsendIrecvTest(int);
+int IsenIrecvTest2(int, int);
+int OutOfOrderTest(int, int);
+int ForceUnexpectedTest(int, int);
+int RndvTest(int, int, int);
+
+int SendRecvTest(int rank, int n)
+{
+    int tag = 1;
+    MPI_Status status;
+    char buffer[100];
+    int i;
+
+    if (rank == 0) {
+        strcpy(buffer, "Hello process one.");
+        for (i = 0; i < n; i++)
+            MPI_Send(buffer, 100, MPI_BYTE, 1, tag, MPI_COMM_WORLD);
+    } else if (rank == 1) {
+        for (i = 0; i < n; i++)
+            MPI_Recv(buffer, 100, MPI_BYTE, 0, tag, MPI_COMM_WORLD, &status);
+        /*printf("Rank 1: received message '%s'\n", buffer);fflush(stdout); */
+    }
+
+    return TRUE;
+}
+
+int IsendIrecvTest(int rank)
+{
+    int tag = 1;
+    MPI_Status status;
+    MPI_Request request;
+    char buffer[100];
+
+    if (rank == 0) {
+        strcpy(buffer, "Hello process one.");
+        MPI_Isend(buffer, 100, MPI_BYTE, 1, tag, MPI_COMM_WORLD, &request);
+        MPI_Wait(&request, &status);
+    } else if (rank == 1) {
+        MPI_Irecv(buffer, 100, MPI_BYTE, 0, tag, MPI_COMM_WORLD, &request);
+        MPI_Wait(&request, &status);
+        /*printf("Rank 1: received message '%s'\n", buffer);fflush(stdout); */
+    }
+
+    return TRUE;
+}
+
+int IsendIrecvTest2(int rank, int buf_size);
+int IsendIrecvTest2(int rank, int buf_size)
+{
+    int tag1 = 1;
+    int tag2 = 2;
+    MPI_Status status;
+    MPI_Request request1, request2;
+    char *buffer;
+
+    buffer = (char *) malloc(buf_size);
+    if (buffer == NULL)
+        return FALSE;
+    if (rank == 0) {
+        strcpy(buffer, "Hello process one.");
+        MPI_Isend(buffer, buf_size, MPI_BYTE, 1, tag1, MPI_COMM_WORLD, &request1);
+        MPI_Isend(buffer, buf_size, MPI_BYTE, 1, tag2, MPI_COMM_WORLD, &request2);
+        MPI_Wait(&request1, &status);
+        MPI_Wait(&request2, &status);
+    } else if (rank == 1) {
+        MPI_Irecv(buffer, buf_size, MPI_BYTE, 0, tag1, MPI_COMM_WORLD, &request1);
+        MPI_Irecv(buffer, buf_size, MPI_BYTE, 0, tag2, MPI_COMM_WORLD, &request2);
+        MPI_Wait(&request1, &status);
+        MPI_Wait(&request2, &status);
+        /*printf("Rank 1: received message '%s'\n", buffer);fflush(stdout); */
+    }
+
+    free(buffer);
+    return TRUE;
+}
+
+int OutOfOrderTest(int rank, int buf_size)
+{
+    int tag1 = 1;
+    int tag2 = 2;
+    MPI_Status status;
+    MPI_Request request1, request2;
+    char *buffer;
+
+    buffer = (char *) malloc(buf_size);
+    if (buffer == NULL)
+        return FALSE;
+    if (rank == 0) {
+        strcpy(buffer, "Hello process one.");
+        MPI_Isend(buffer, buf_size, MPI_BYTE, 1, tag1, MPI_COMM_WORLD, &request1);
+        MPI_Isend(buffer, buf_size, MPI_BYTE, 1, tag2, MPI_COMM_WORLD, &request2);
+        MPI_Wait(&request1, &status);
+        MPI_Wait(&request2, &status);
+    } else if (rank == 1) {
+        MPI_Irecv(buffer, buf_size, MPI_BYTE, 0, tag2, MPI_COMM_WORLD, &request1);
+        MPI_Irecv(buffer, buf_size, MPI_BYTE, 0, tag1, MPI_COMM_WORLD, &request2);
+        MPI_Wait(&request2, &status);
+        MPI_Wait(&request1, &status);
+        /*printf("Rank 1: received message '%s'\n", buffer);fflush(stdout); */
+    }
+
+    free(buffer);
+    return TRUE;
+}
+
+int ForceUnexpectedTest(int rank, int buf_size)
+{
+    int tag1 = 1;
+    int tag2 = 2;
+    MPI_Status status;
+    MPI_Request request1, request2;
+    char *buffer;
+
+    buffer = (char *) malloc(buf_size);
+    if (buffer == NULL)
+        return FALSE;
+
+    if (rank == 0) {
+        strcpy(buffer, "Hello process one.");
+        MPI_Isend(buffer, buf_size, MPI_BYTE, 1, tag1, MPI_COMM_WORLD, &request1);
+        MPI_Isend(buffer, buf_size, MPI_BYTE, 1, tag2, MPI_COMM_WORLD, &request2);
+        MPI_Wait(&request1, &status);
+        MPI_Wait(&request2, &status);
+        MPI_Recv(buffer, buf_size, MPI_BYTE, 1, tag1, MPI_COMM_WORLD, &status);
+    } else if (rank == 1) {
+        MPI_Irecv(buffer, buf_size, MPI_BYTE, 0, tag2, MPI_COMM_WORLD, &request2);
+        MPI_Wait(&request2, &status);
+        MPI_Irecv(buffer, buf_size, MPI_BYTE, 0, tag1, MPI_COMM_WORLD, &request1);
+        MPI_Wait(&request1, &status);
+        /*printf("Rank 1: received message '%s'\n", buffer);fflush(stdout); */
+        MPI_Send(buffer, buf_size, MPI_BYTE, 0, tag1, MPI_COMM_WORLD);
+    }
+
+    free(buffer);
+
+    return TRUE;
+}
+
+int RndvTest(int rank, int size, int reps)
+{
+    int tag = 1;
+    MPI_Status status;
+    char *buffer;
+    int i;
+
+    buffer = (char *) malloc(size);
+    if (buffer == NULL) {
+        printf("malloc failed to allocate %d bytes.\n", size);
+        exit(0);
+    }
+    if (rank == 0) {
+        for (i = 0; i < reps; i++) {
+            if (reps == 1) {
+                printf("0: sending to process 1\n");
+                fflush(stdout);
+            }
+            MPI_Send(buffer, size, MPI_BYTE, 1, tag, MPI_COMM_WORLD);
+            if (reps == 1) {
+                printf("0: receiving from process 1\n");
+                fflush(stdout);
+            }
+            MPI_Recv(buffer, size, MPI_BYTE, 1, tag, MPI_COMM_WORLD, &status);
+            if (reps == 1) {
+                printf("0: done\n");
+                fflush(stdout);
+            }
+        }
+    } else if (rank == 1) {
+        for (i = 0; i < reps; i++) {
+            if (reps == 1) {
+                printf("1: receiving from process 0\n");
+                fflush(stdout);
+            }
+            MPI_Recv(buffer, size, MPI_BYTE, 0, tag, MPI_COMM_WORLD, &status);
+            if (reps == 1) {
+                printf("1: sending to process 0\n");
+                fflush(stdout);
+            }
+            MPI_Send(buffer, size, MPI_BYTE, 0, tag, MPI_COMM_WORLD);
+            if (reps == 1) {
+                printf("1: done\n");
+                fflush(stdout);
+            }
+        }
+    }
+    free(buffer);
+
+    return TRUE;
+}
+
+int main(int argc, char *argv[])
+{
+    int result;
+    int size, rank;
+    int bDoAll = FALSE;
+    int reps;
+    int rndv_size = DEFAULT_RNDV_SIZE;
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+    if (size < 2) {
+        printf("Two processes needed.\n");
+        printf("options:\n");
+        printf(" sr [reps] ............... send/recv\n");
+        printf(" isr ..................... isend/irecv\n");
+        printf(" iisr .................... isend,isend/irecv,irecv wait\n");
+        printf(" oo ...................... out of order isend/irecv\n");
+        printf(" unex .................... force unexpected msg\n");
+        printf(" rndv [size] ............. rndv\n");
+        printf(" rndv_reps [reps] [size] . rndv\n");
+        printf(" rndv_iisr [size] ........ rndv iisr\n");
+        printf(" rndv_oo [size] .......... rndv oo\n");
+        printf(" rndv_unex [size] ........ rndv unex\n");
+        printf("default rndv size = %d bytes\n", DEFAULT_RNDV_SIZE);
+        MPI_Finalize();
+        return 0;
+    }
+
+    if (rank > 1) {
+        printf("Rank %d, I am not participating.\n", rank);
+        fflush(stdout);
+    } else {
+        if (argc < 2)
+            bDoAll = TRUE;
+
+        if (bDoAll || (strcmp(argv[1], "sr") == 0)) {
+            reps = 1;
+            if (argc > 2) {
+                reps = atoi(argv[2]);
+                if (reps < 1)
+                    reps = 1;
+            }
+            if (rank == 0) {
+                printf("Send/recv test: %d reps\n", reps);
+                fflush(stdout);
+            }
+            result = SendRecvTest(rank, reps);
+            printf(result ? "%d:SUCCESS - sr\n" : "%d:FAILURE - sr\n", rank);
+            fflush(stdout);
+        }
+
+        if (bDoAll || (strcmp(argv[1], "isr") == 0)) {
+            if (rank == 0) {
+                printf("Isend/irecv wait test\n");
+                fflush(stdout);
+            }
+            result = IsendIrecvTest(rank);
+            printf(result ? "%d:SUCCESS - isr\n" : "%d:FAILURE - isr\n", rank);
+            fflush(stdout);
+        }
+
+        if (bDoAll || (strcmp(argv[1], "iisr") == 0)) {
+            if (rank == 0) {
+                printf("Isend,isend/irecv,irecv wait wait test\n");
+                fflush(stdout);
+            }
+            result = IsendIrecvTest2(rank, 100);
+            printf(result ? "%d:SUCCESS - iisr\n" : "%d:FAILURE - iisr\n", rank);
+            fflush(stdout);
+        }
+
+        if (bDoAll || (strcmp(argv[1], "oo") == 0)) {
+            if (rank == 0) {
+                printf("Out of order isend/irecv test\n");
+                fflush(stdout);
+            }
+            result = OutOfOrderTest(rank, 100);
+            printf(result ? "%d:SUCCESS - oo\n" : "%d:FAILURE - oo\n", rank);
+            fflush(stdout);
+        }
+
+        if (bDoAll || (strcmp(argv[1], "unex") == 0)) {
+            if (rank == 0) {
+                printf("Force unexpected message test\n");
+                fflush(stdout);
+            }
+            result = ForceUnexpectedTest(rank, 100);
+            printf(result ? "%d:SUCCESS - unex\n" : "%d:FAILURE - unex\n", rank);
+            fflush(stdout);
+        }
+
+        if (bDoAll || (strcmp(argv[1], "rndv") == 0)) {
+            if (argc > 2) {
+                rndv_size = atoi(argv[2]);
+                if (rndv_size < 1024)
+                    rndv_size = 1024;
+            }
+            if (rank == 0) {
+                printf("Rndv test\n");
+                fflush(stdout);
+            }
+            result = RndvTest(rank, rndv_size, 1);
+            printf(result ? "%d:SUCCESS - rndv\n" : "%d:FAILURE - rndv\n", rank);
+            fflush(stdout);
+        }
+
+        if (bDoAll || (strcmp(argv[1], "rndv_reps") == 0)) {
+            reps = 100;
+            if (argc > 2) {
+                reps = atoi(argv[2]);
+                if (reps < 1)
+                    reps = 1;
+            }
+            if (argc > 3) {
+                rndv_size = atoi(argv[3]);
+            }
+            if (rank == 0) {
+                printf("Rndv test: %d reps of size %d\n", reps, rndv_size);
+                fflush(stdout);
+            }
+            result = RndvTest(rank, rndv_size, reps);
+            printf(result ? "%d:SUCCESS - rndv_reps\n" : "%d:FAILURE - rndv_reps\n", rank);
+            fflush(stdout);
+        }
+
+        if (bDoAll || (strcmp(argv[1], "rndv_iisr") == 0)) {
+            if (rank == 0) {
+                printf("Rndv isend,isend/irecv,irecv wait wait test\n");
+                fflush(stdout);
+            }
+            if (argc > 2) {
+                rndv_size = atoi(argv[2]);
+            }
+            result = IsendIrecvTest2(rank, rndv_size);
+            printf(result ? "%d:SUCCESS - rndv_iisr\n" : "%d:FAILURE - rndv_iisr\n", rank);
+            fflush(stdout);
+        }
+
+        if (bDoAll || (strcmp(argv[1], "rndv_oo") == 0)) {
+            if (rank == 0) {
+                printf("Rndv out of order isend/irecv test\n");
+                fflush(stdout);
+            }
+            if (argc > 2) {
+                rndv_size = atoi(argv[2]);
+            }
+            result = OutOfOrderTest(rank, rndv_size);
+            printf(result ? "%d:SUCCESS - rndv_oo\n" : "%d:FAILURE - rndv_oo\n", rank);
+            fflush(stdout);
+        }
+
+        if (bDoAll || (strcmp(argv[1], "rndv_unex") == 0)) {
+            if (argc > 2) {
+                rndv_size = atoi(argv[2]);
+                if (rndv_size < 1024)
+                    rndv_size = 1024;
+            }
+            if (rank == 0) {
+                printf("Force unexpected rndv message test\n");
+                fflush(stdout);
+            }
+            result = ForceUnexpectedTest(rank, rndv_size);
+            printf(result ? "%d:SUCCESS - rndv_unex\n" : "%d:FAILURE - rndv_unex\n", rank);
+            fflush(stdout);
+        }
+    }
+
+    MPI_Finalize();
+    return 0;
+}
diff --git a/t/mpi/mpich_basic/self.c b/t/mpi/mpich_basic/self.c
new file mode 100644
index 000000000000..c999c9dd0196
--- /dev/null
+++ b/t/mpi/mpich_basic/self.c
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#include "mpi.h"
+
+int main(int argc, char *argv[])
+{
+    int i, j;
+    MPI_Status status;
+
+    MPI_Init(&argc, &argv);
+
+    MPI_Sendrecv(&i, 1, MPI_INT, 0, 100, &j, 1, MPI_INT, 0, 100, MPI_COMM_WORLD, &status);
+
+    MPI_Finalize();
+    return (0);
+}
diff --git a/t/mpi/mpich_basic/sendrecv.c b/t/mpi/mpich_basic/sendrecv.c
new file mode 100644
index 000000000000..c7f970031cc1
--- /dev/null
+++ b/t/mpi/mpich_basic/sendrecv.c
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#include "mpi.h"
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+#define LARGE_SIZE  100*1024
+#define RNDV_SIZE   256*1024
+
+int main(int argc, char *argv[])
+{
+    int size, rank;
+    char buffer[100] = "garbage";
+    char big_buffer[RNDV_SIZE] = "big garbage";
+    MPI_Status status;
+    int tag = 1;
+    int reps = 1;
+    int i;
+
+    printf("Simple Send/Recv test.\n");
+    fflush(stdout);
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+    if (size < 2) {
+        printf("Two processes needed.\n");
+        MPI_Finalize();
+        return 0;
+    }
+
+    if (argc > 1) {
+        reps = atoi(argv[1]);
+        if (reps < 1)
+            reps = 1;
+    }
+
+    if (rank == 0) {
+        printf("Rank 0: sending 100 bytes messages to process 1.\n");
+        fflush(stdout);
+        strcpy(buffer, "Hello process one.");
+        for (i = 0; i < reps; i++) {
+            MPI_Send(buffer, 100, MPI_BYTE, 1, tag, MPI_COMM_WORLD);
+            MPI_Recv(buffer, 100, MPI_BYTE, 1, tag, MPI_COMM_WORLD, &status);
+        }
+        strcpy(big_buffer, "Hello again process one.");
+        printf("Rank 0: sending %dk bytes messages to process 1.\n", LARGE_SIZE / 1024);
+        fflush(stdout);
+        for (i = 0; i < reps; i++) {
+            MPI_Send(big_buffer, LARGE_SIZE, MPI_BYTE, 1, tag, MPI_COMM_WORLD);
+            MPI_Recv(big_buffer, LARGE_SIZE, MPI_BYTE, 1, tag, MPI_COMM_WORLD, &status);
+        }
+        strcpy(big_buffer, "Hello yet again process one.");
+        printf("Rank 0: sending %dk bytes messages to process 1.\n", RNDV_SIZE / 1024);
+        fflush(stdout);
+        for (i = 0; i < reps; i++) {
+            MPI_Send(big_buffer, RNDV_SIZE, MPI_BYTE, 1, tag, MPI_COMM_WORLD);
+            MPI_Recv(big_buffer, RNDV_SIZE, MPI_BYTE, 1, tag, MPI_COMM_WORLD, &status);
+        }
+    } else if (rank == 1) {
+        printf("Rank 1: receiving messages from process 0.\n");
+        fflush(stdout);
+        for (i = 0; i < reps; i++) {
+            MPI_Recv(buffer, 100, MPI_BYTE, 0, tag, MPI_COMM_WORLD, &status);
+            MPI_Send(buffer, 100, MPI_BYTE, 0, tag, MPI_COMM_WORLD);
+        }
+        printf("Rank 1: received message '%s'\n", buffer);
+        fflush(stdout);
+        for (i = 0; i < reps; i++) {
+            MPI_Recv(big_buffer, LARGE_SIZE, MPI_BYTE, 0, tag, MPI_COMM_WORLD, &status);
+            MPI_Send(big_buffer, LARGE_SIZE, MPI_BYTE, 0, tag, MPI_COMM_WORLD);
+        }
+        printf("Rank 1: received message '%s'\n", big_buffer);
+        fflush(stdout);
+        for (i = 0; i < reps; i++) {
+            MPI_Recv(big_buffer, RNDV_SIZE, MPI_BYTE, 0, tag, MPI_COMM_WORLD, &status);
+            MPI_Send(big_buffer, RNDV_SIZE, MPI_BYTE, 0, tag, MPI_COMM_WORLD);
+        }
+        printf("Rank 1: received message '%s'\n", big_buffer);
+        fflush(stdout);
+    } else {
+        printf("Rank %d, I am not participating.\n", rank);
+        fflush(stdout);
+    }
+
+    MPI_Finalize();
+    return 0;
+}
diff --git a/t/mpi/mpich_basic/simple.c b/t/mpi/mpich_basic/simple.c
new file mode 100644
index 000000000000..fa45cecca644
--- /dev/null
+++ b/t/mpi/mpich_basic/simple.c
@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#include "mpi.h"
+
+int main(int argc, char *argv[])
+{
+    MPI_Init(&argc, &argv);
+    MPI_Finalize();
+
+    return 0;
+}
diff --git a/t/mpi/mpich_basic/srtest.c b/t/mpi/mpich_basic/srtest.c
new file mode 100644
index 000000000000..6c053a48063a
--- /dev/null
+++ b/t/mpi/mpich_basic/srtest.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#include "mpi.h"
+#include <stdio.h>
+#include <string.h>
+
+#define BUFLEN 512
+
+int main(int argc, char *argv[])
+{
+    int myid, numprocs, next, namelen;
+    char buffer[BUFLEN], processor_name[MPI_MAX_PROCESSOR_NAME];
+    MPI_Status status;
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
+    MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+    MPI_Get_processor_name(processor_name, &namelen);
+    fprintf(stderr, "Process %d of %d is alive on %s\n", myid, numprocs, processor_name);
+
+    strcpy(buffer, "hello there");
+    if (myid == numprocs - 1)
+        next = 0;
+    else
+        next = myid + 1;
+
+    if (myid == 0) {
+        printf("%d sending '%s' \n", myid, buffer);
+        MPI_Send(buffer, (int) strlen(buffer) + 1, MPI_CHAR, next, 99, MPI_COMM_WORLD);
+        printf("%d receiving \n", myid);
+        MPI_Recv(buffer, BUFLEN, MPI_CHAR, MPI_ANY_SOURCE, 99, MPI_COMM_WORLD, &status);
+        printf("%d received '%s' \n", myid, buffer);
+        /* mpdprintf(001,"%d receiving \n",myid); */
+    } else {
+        printf("%d receiving  \n", myid);
+        MPI_Recv(buffer, BUFLEN, MPI_CHAR, MPI_ANY_SOURCE, 99, MPI_COMM_WORLD, &status);
+        printf("%d received '%s' \n", myid, buffer);
+        /* mpdprintf(001,"%d receiving \n",myid); */
+        MPI_Send(buffer, (int) strlen(buffer) + 1, MPI_CHAR, next, 99, MPI_COMM_WORLD);
+        printf("%d sent '%s' \n", myid, buffer);
+    }
+    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Finalize();
+    return (0);
+}
diff --git a/t/mpi/version.c b/t/mpi/version.c
new file mode 100644
index 000000000000..e3de2e5be1b7
--- /dev/null
+++ b/t/mpi/version.c
@@ -0,0 +1,35 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <mpi.h>
+#include <stdio.h>
+
+int main (int argc, char *argv[])
+{
+    char version[MPI_MAX_LIBRARY_VERSION_STRING];
+    int len;
+    int exit_rc = -1;
+
+    MPI_Get_library_version (version, &len);
+    if (len < 0) {
+        fprintf (stderr, "MPI_Get_library_version failed\n");
+        goto done;
+    }
+    printf ("%s\n", version);
+    exit_rc = 0;
+done:
+    return exit_rc;
+}
+
+// vi: ts=4 sw=4 expandtab
+
diff --git a/t/prove-under-flux.sh b/t/prove-under-flux.sh
new file mode 100755
index 000000000000..2cb67a943df8
--- /dev/null
+++ b/t/prove-under-flux.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+prove --merge --timer -j 128 --exec="flux run -n1" ./t*.t ./python/t*.py
diff --git a/t/python/sideflux.py b/t/python/sideflux.py
deleted file mode 100644
index 95fe95d0ea09..000000000000
--- a/t/python/sideflux.py
+++ /dev/null
@@ -1,232 +0,0 @@
-###############################################################
-# Copyright 2014 Lawrence Livermore National Security, LLC
-# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
-#
-# This file is part of the Flux resource manager framework.
-# For details, see https://github.com/flux-framework.
-#
-# SPDX-License-Identifier: LGPL-3.0
-###############################################################
-
-from __future__ import print_function
-import io
-import re
-import os
-import sys
-import json
-import subprocess
-import multiprocessing as mp
-import contextlib
-import errno
-import pprint
-import shutil
-import tempfile
-import time
-from six.moves import queue as Queue
-import pycotap
-
-# pprint.pprint(os.environ)
-flux_exe = ""
-if os.environ.get("CHECK_BUILDDIR", None) is not None:
-    flux_exe = os.path.abspath(os.environ["CHECK_BUILDDIR"] + "/src/cmd/flux")
-else:
-    flux_exe = os.path.abspath(
-        os.path.dirname(os.path.abspath(__file__)) + "/../../../cmd/flux"
-    )
-
-
-@contextlib.contextmanager
-def get_tmpdir():
-    d = tempfile.mkdtemp()
-    try:
-        yield d
-    finally:
-        shutil.rmtree(d)
-
-
-def consume(stream):
-    while True:
-        l = stream.readline()
-        if not l:
-            break
-        sys.stdout.write(l)
-
-
-class SideFlux(object):
-    def __init__(self, size=1):
-        global flux_exe
-        self.size = size
-        self.tmpdir = tempfile.mkdtemp(prefix="flux-sandbox-")
-        self.flux_uri = "local://" + self.tmpdir + "/0"
-        self.cleaned = False
-
-    def start(self):
-        flux_command = [
-            flux_exe,
-            "start",
-            "--bootstrap=selfpmi",
-            "--size={}".format(self.size),
-            "-o",
-            "-Slog-forward-level=7",
-            "--scratchdir=" + self.tmpdir,
-            "bash",
-        ]
-        # print ' '.join(flux_command)
-        FNULL = open(os.devnull, "w+")
-        self.subenv = os.environ.copy()
-        self.subenv.pop("FLUX_URI", None)
-        self.subenv["TMPDIR"] = self.tmpdir
-        self.sub = subprocess.Popen(
-            flux_command,
-            stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            close_fds=True,  # Start a process session to clean up brokers
-            preexec_fn=os.setsid,
-            env=self.subenv,
-        )
-        self.sub_in = io.TextIOWrapper(
-            self.sub.stdin,
-            encoding="utf-8",
-            line_buffering=True,  # send data on newline
-        )
-        self.sub_out = io.TextIOWrapper(self.sub.stdout, encoding="utf-8")
-        self.p = mp.Process(target=consume, args=(self.sub_out,))
-
-        print("echo READY", file=self.sub_in)
-
-        self.env_items = {}
-        self.env_items["FLUX_URI"] = self.flux_uri
-
-        while True:
-            line = self.sub_out.readline()
-            if os.environ.get("SIDEFLUX_DEBUG", False):
-                print(line)
-            if line != "":
-                m = re.match(r"\s*(?P<var>[^= ]+)=(?P<val>.*)", line.rstrip())
-                if m:
-                    if os.environ.get("SIDEFLUX_DEBUG", False):
-                        print("setting", m.group("var"), "to", m.group("val"))
-                    v = m.group("val")
-                    if re.search(r"/\.\./", v):
-                        v = os.path.abspath(v)
-                    self.env_items[m.group("var")] = v
-                if re.search("READY", line):
-                    break
-                else:
-                    if self.sub.poll() is not None:
-                        raise EnvironmentError(self.sub.poll())
-        self.p.start()
-
-    def get_uri():
-        return self.flux_uri
-
-    def apply_environment(self):
-        for k, v in self.env_items.items():
-            os.environ[k] = v
-
-    def destroy(self):
-        if self.cleaned:
-            return
-        self.cleaned = True
-        if os.path.exists(self.tmpdir):
-            shutil.rmtree(self.tmpdir)
-        try:
-            self.sub_in.close()
-        except AttributeError:
-            pass
-        # Kill the process group headed by the subprocess
-        os.killpg(self.sub.pid, 15)
-        if self.p is not None:
-            self.p.terminate()
-            self.p.join()
-
-    def run_flux_cmd(self, command=""):
-        global flux_exe
-        print("{} {}".format(flux_exe, command), file=self.sub_in)
-
-    def run_cmd(self, command=""):
-        global flux_exe
-        print(command, file=self.sub_in)
-
-    def __del__(self):
-        self.destroy()
-
-
-@contextlib.contextmanager
-def run_beside_flux(size=1):
-    f = SideFlux(size)
-    f.start()
-    env = os.environ.copy()
-    f.apply_environment()
-    # print json.dumps(dict(os.environ))
-    try:
-        yield f
-    finally:
-        os.environ.update(env)
-        f.destroy()
-
-
-def apply_wrapper(fun, environment, args, kwargs):
-    for k, v in environment.iteritems():
-        os.environ[k] = v
-    return fun(*args, **kwargs)
-
-
-class AsyncTimeout(Exception):
-    def __init__(self, message):
-        super(AsyncTimeout, self).__init__(message)
-
-
-class SimpleAsyncRunner(object):
-    def __init__(self, fun, args, side):
-        def q_wrapper(q):
-            try:
-                res = fun(*args)
-                q.put(res)
-            except:
-                q.put(None)
-
-        self.q = mp.Queue()
-        self.p = mp.Process(target=q_wrapper, args=(self.q,))
-        self.p.start()
-        self.done = False
-        self.res = None
-        self.side = side
-
-    def get(self, timeout=None):
-        """ Get the result, raises AsyncTimeout on timeout failure """
-        if not self.done:
-            try:
-                self.res = self.q.get(True, timeout)
-            except Queue.Empty:
-                raise AsyncTimeout("The result is not ready, has a test run too long?")
-            self.done = True
-        return self.res
-
-    def ready(self):
-        if not self.q.empty():
-            return True
-        else:
-            return False
-
-    def __del__(self):
-        self.p.join()
-
-
-def apply_under_flux_async(size, fun, args=tuple(), kwargs=dict()):
-    f = SideFlux(size)
-    f.start()
-    result_queue = mp.Queue()
-    res = SimpleAsyncRunner(apply_wrapper, (fun, f.env_items, args, kwargs), f)
-    return res
-
-
-def apply_under_flux(size, fun, args=tuple(), kwargs=dict()):
-    return apply_under_flux_async(size, fun, args, kwargs).get()
-
-
-if __name__ == "__main__":
-    with run_beside_flux(1) as fp:
-        while True:
-            pass
diff --git a/t/python/subflux.py b/t/python/subflux.py
index e6e455e021c2..1a4641f9ec9f 100644
--- a/t/python/subflux.py
+++ b/t/python/subflux.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
 # (c.f. AUTHORS, NOTICE.LLNS, COPYING)
@@ -10,11 +8,10 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-from __future__ import print_function
-
+import argparse
 import os
-import sys
 import subprocess
+import sys
 
 script_dir = os.path.dirname(os.path.abspath(__file__))
 
@@ -27,6 +24,19 @@
 )
 flux_exe = os.path.join(builddir, "src", "cmd", "flux")
 
+sys.path.append(script_dir + "/tap")
+
+
+#  Ignore -v, --verbose and --root options so that python test scripts
+#   can absorb the same options as sharness tests. Later, something could
+#   be done with these options, but for now they are dropped silently.
+parser = argparse.ArgumentParser()
+parser.add_argument("--debug", "-d", action="store_true")
+parser.add_argument("--root", metavar="PATH", type=str)
+args, remainder = parser.parse_known_args()
+
+sys.argv[1:] = remainder
+
 
 def is_exe(fpath):
     return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
@@ -45,15 +55,15 @@ def rerun_under_flux(size=1, personality="full"):
     # ported from sharness.d/flux-sharness.sh
     child_env["FLUX_BUILD_DIR"] = builddir
     child_env["FLUX_SOURCE_DIR"] = srcdir
-    command = [flux_exe, "start", "--bootstrap=selfpmi", "--size", str(size)]
+    command = [flux_exe, "start", "--test-size", str(size)]
     if personality != "full":
         for rc_num in [1, 3]:
             attr = "broker.rc{}_path".format(rc_num)
             if personality == "minimal":
-                command.append("-o,-S{}=".format(attr))
+                command.append("-S{}=".format(attr))
             else:
                 path = "{}/t/rc/rc{}-{}".format(srcdir, rc_num, personality)
-                command.append("-o,-S{}={}".format(attr, path))
+                command.append("-S{}={}".format(attr, path))
                 if not is_exe(path):
                     print("cannot execute {}".format(path), file=sys.stderr)
                     sys.exit(1)
@@ -64,4 +74,8 @@ def rerun_under_flux(size=1, personality="full"):
         command, env=child_env, bufsize=-1, stdout=sys.stdout, stderr=sys.stderr
     )
     p.wait()
+    if p.returncode > 0:
+        sys.exit(p.returncode)
+    elif p.returncode < 0:
+        sys.exit(128 + -p.returncode)
     return False
diff --git a/t/python/t0001-handle.py b/t/python/t0001-handle.py
index 9f363c08b646..7f317e062997 100755
--- a/t/python/t0001-handle.py
+++ b/t/python/t0001-handle.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
@@ -10,11 +10,8 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-from __future__ import print_function
-
-import unittest
 import syslog
-import six
+import unittest
 
 import flux
 from subflux import rerun_under_flux
@@ -36,36 +33,36 @@ def test_create_handle(self):
 
     def test_log(self):
         """Successfully connected to flux"""
-        self.f.log(syslog.LOG_INFO, u"hello")
+        self.f.log(syslog.LOG_INFO, "hello")
         self.f.log(syslog.LOG_INFO, b"world")
 
     def test_rpc_ping(self):
         """Sending a ping"""
         # python 3 json doesn't support bytes, but python 2 will treat them as str (i.e., bytes)
-        r = self.f.rpc(b"cmb.ping", {"seq": 1, "pad": "stuff"}).get()
+        r = self.f.rpc(b"broker.ping", {"seq": 1, "pad": "stuff"}).get()
         self.assertEqual(r["seq"], 1)
-        self.assertEqual(r["pad"], u"stuff")
-        self.assertTrue(isinstance(r["pad"], six.text_type))
+        self.assertEqual(r["pad"], "stuff")
+        self.assertTrue(isinstance(r["pad"], str))
 
     def test_anonymous_handle_rpc_ping(self):
         """Send a ping using an anonymous/unnamed flux handle"""
-        r = flux.Flux().rpc(b"cmb.ping", {"seq": 1, "pad": "stuff"}).get()
+        r = flux.Flux().rpc(b"broker.ping", {"seq": 1, "pad": "stuff"}).get()
         self.assertIsNotNone(r)
         self.assertEqual(r["seq"], 1)
-        self.assertEqual(r["pad"], u"stuff")
+        self.assertEqual(r["pad"], "stuff")
 
     def test_rpc_ping_unicode(self):
         """Sending a ping"""
         r = self.f.rpc(
-            u"cmb.ping", {u"\xa3": u"value", u"key": u"\u32db \u263a \u32e1"}
+            "broker.ping", {"\xa3": "value", "key": "\u32db \u263a \u32e1"}
         ).get()
-        self.assertEqual(r[u"\xa3"], u"value")
-        self.assertEqual(r["key"], u"\u32db \u263a \u32e1")
-        self.assertTrue(isinstance(r["key"], six.text_type))
+        self.assertEqual(r["\xa3"], "value")
+        self.assertEqual(r["key"], "\u32db \u263a \u32e1")
+        self.assertTrue(isinstance(r["key"], str))
 
     def test_rpc_with(self):
         """Sending a ping"""
-        with self.f.rpc("cmb.ping", {"seq": 1, "pad": "stuff"}) as r:
+        with self.f.rpc("broker.ping", {"seq": 1, "pad": "stuff"}) as r:
             j = r.get()
             self.assertEqual(j["seq"], 1)
             self.assertEqual(j["pad"], "stuff")
@@ -82,6 +79,39 @@ def test_get_rank(self):
         rank = self.f.get_rank()
         self.assertEqual(rank, 0)
 
+    def test_attr_get(self):
+        local_uri = self.f.attr_get("local-uri")
+        self.assertTrue(isinstance(local_uri, str))
+        self.assertEqual(local_uri[:6], "local:")
+
+        attr_rank = int(self.f.attr_get("rank"))
+        rank = self.f.get_rank()
+        self.assertEqual(attr_rank, rank)
+
+    def test_conf_get(self):
+        # Works with empty config
+        self.assertEqual(self.f.conf_get(), {})
+
+        # load test config
+        testconf = {"a": {"b": {"c": 42}, "foo": "bar"}}
+        self.f.rpc("config.load", testconf).get()
+
+        # conf_get() still returns old config
+        self.assertEqual(self.f.conf_get(), {})
+
+        # conf_get() with update=True returns new config
+        self.assertEqual(self.f.conf_get(update=True), testconf)
+
+        # conf_get() with key works
+        self.assertEqual(self.f.conf_get("a"), testconf["a"])
+        self.assertEqual(self.f.conf_get("a.b"), testconf["a"]["b"])
+        self.assertEqual(self.f.conf_get("a.b.c"), testconf["a"]["b"]["c"])
+        self.assertEqual(self.f.conf_get("a.foo"), testconf["a"]["foo"])
+
+        # conf_get() works with default
+        self.assertIsNone(self.f.conf_get("a.baz"))
+        self.assertEqual(self.f.conf_get("a.baz", default=42), 42)
+
 
 if __name__ == "__main__":
     if rerun_under_flux(__flux_size()):
diff --git a/t/python/t0002-wrapper.py b/t/python/t0002-wrapper.py
index 4e19861b4240..ffceab23684d 100755
--- a/t/python/t0002-wrapper.py
+++ b/t/python/t0002-wrapper.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
@@ -12,10 +12,9 @@
 
 import unittest
 
-import six
 import flux
-from flux.core.inner import ffi, raw
 import flux.wrapper
+from flux.core.inner import ffi, raw
 from subflux import rerun_under_flux
 
 
@@ -24,10 +23,10 @@ def __flux_size():
 
 
 class TestWrapper(unittest.TestCase):
-    def test_call_non_existant(self):
+    def test_call_non_existent(self):
         f = flux.Flux("loop://")
         with self.assertRaises(flux.wrapper.MissingFunctionError):
-            f.non_existant_function_that_should_die("stuff")
+            f.non_existent_function_that_should_die("stuff")
 
     def test_call_insufficient_arguments(self):
         f = flux.Flux("loop://")
@@ -42,19 +41,19 @@ def test_call_invalid_argument_type(self):
     def test_null_handle_exception(self):
         f = flux.Flux()
         payload = {"seq": 1, "pad": "stuff"}
-        future = f.rpc("cmb.ping", payload)
-        resp = future.get()
+        future = f.rpc("broker.ping", payload)
+        future.get()
         future.pimpl.handle = None
-        with six.assertRaisesRegex(
-            self, ValueError, r"Attempting to call a cached, bound method.*NULL handle"
+        with self.assertRaisesRegex(
+            ValueError, r"Attempting to call a cached, bound method.*NULL handle"
         ):
-            resp = future.get()
+            future.get()
 
     def test_automatic_unwrapping(self):
         flux.core.inner.raw.flux_log(flux.Flux("loop://"), 0, "stuff")
 
     def test_masked_function(self):
-        with six.assertRaisesRegex(self, AttributeError, r".*masks function.*"):
+        with self.assertRaisesRegex(AttributeError, r".*masks function.*"):
             flux.Flux("loop://").rpc("topic").pimpl.flux_request_encode("request", 15)
 
     def test_set_pimpl_handle(self):
@@ -67,7 +66,7 @@ def test_set_pimpl_handle(self):
     def test_set_pimpl_handle_invalid(self):
         f = flux.Flux("loop://")
         r = f.rpc("topic")
-        with six.assertRaisesRegex(self, TypeError, r".*expected a.*"):
+        with self.assertRaisesRegex(TypeError, r".*expected a.*"):
             r.handle = f.rpc("other topic")
 
     def test_read_basic_value(self):
@@ -78,8 +77,7 @@ def test_invalid_constructor_call(self):
         infinite recursion as documented in
         https://github.com/flux-framework/flux-core/issues/2485
         """
-        with six.assertRaisesRegex(
-            self,
+        with self.assertRaisesRegex(
             TypeError,
             (
                 r"(.*takes at least.*arguments.*)"
diff --git a/t/python/t0003-barrier.py b/t/python/t0003-barrier.py
index 2aa9550d3403..68ff63ef1616 100755
--- a/t/python/t0003-barrier.py
+++ b/t/python/t0003-barrier.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
@@ -10,11 +10,8 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-from __future__ import print_function
-import unittest
 import multiprocessing as mp
-
-from six.moves import range as range
+import unittest
 
 import flux
 from subflux import rerun_under_flux
@@ -23,7 +20,7 @@
 def barr_count(x, name, count):
     print(x, name, count)
     f = flux.Flux()
-    f.barrier(name, count)
+    f.barrier(name, count).get()
 
 
 def __flux_size():
@@ -36,17 +33,20 @@ def setUpClass(self):
         self.f = flux.Flux()
 
     def test_single(self):
-        self.f.barrier("testbarrier1", 1)
-        self.f.barrier(u"testbarrier1", 1)
+        self.f.barrier("testbarrier1", 1).get()
+        self.f.barrier("testbarrier1", 1).get()
 
     def test_eight(self):
-        for topic in [b"testbarrier2", u"\xa3", u"\u32db \u263a \u32e1"]:
+        p = mp.Pool(8)
+        for topic in [b"testbarrier2", "\xa3", "\u32db \u263a \u32e1"]:
             for i in range(1, 9):
-                p = mp.Pool(i)
                 reslist = []
                 for j in range(0, i):
                     res = p.apply_async(barr_count, (j, topic, i))
                     reslist.append(res)
+                reslist[0].wait(10)  # timeout in 10 seconds if no success
+                for r in reslist[1:]:  # wait for rest with short timeouts
+                    r.get(0.5)
 
 
 if __name__ == "__main__":
diff --git a/t/python/t0004-event.py b/t/python/t0004-event.py
index 1be32a50a947..5151f5ced36b 100755
--- a/t/python/t0004-event.py
+++ b/t/python/t0004-event.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
@@ -10,9 +10,7 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-from __future__ import print_function
 import unittest
-import six
 
 import flux
 from subflux import rerun_under_flux
@@ -36,19 +34,19 @@ def test_t1_1_unsub(self):
         """Unsubscribe from an event"""
         self.assertGreaterEqual(self.f.event_subscribe("testevent.2"), 0)
         self.assertGreaterEqual(self.f.event_unsubscribe("testevent.2"), 0)
-        with self.assertRaisesRegexp(EnvironmentError, "No such file"):
+        with self.assertRaisesRegex(EnvironmentError, "No such file"):
             self.f.event_unsubscribe("nonexistent.event")
 
     def test_full_event(self):
         """Subscribe send receive and unpack event"""
-        event_names = [b"testevent.3", u"\u32db \u263a \u32e1"]
+        event_names = [b"testevent.3", "\u32db \u263a \u32e1"]
         for event_name in event_names:
             self.assertGreaterEqual(self.f.event_subscribe(event_name), 0)
             self.assertGreaterEqual(self.f.event_send(event_name, {"test": "yay!"}), 0)
             evt = self.f.event_recv()
             self.assertIsNotNone(evt)
 
-            if isinstance(event_name, six.binary_type):
+            if isinstance(event_name, bytes):
                 self.assertEqual(evt.topic, event_name.decode("utf-8"))
             else:
                 self.assertEqual(evt.topic, event_name)
diff --git a/t/python/t0005-kvs.py b/t/python/t0005-kvs.py
index df6b4894e53c..8541d4d94056 100755
--- a/t/python/t0005-kvs.py
+++ b/t/python/t0005-kvs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
@@ -10,13 +10,13 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-from __future__ import print_function
+import ast
+import errno
 import unittest
-import six
 
 import flux
+import flux.constants
 import flux.kvs
-
 from subflux import rerun_under_flux
 
 
@@ -29,112 +29,322 @@ class TestKVS(unittest.TestCase):
     def setUpClass(self):
         self.f = flux.Flux()
 
-    def test_kvs_dir_open(self):
-        with flux.kvs.get_dir(self.f) as d:
-            self.assertIsNotNone(d)
-
-    def test_kvs_dir_open(self):
+    def test_basic_01_kvs_dir_open(self):
         with flux.kvs.get_dir(self.f) as d:
             self.assertIsNotNone(d)
 
-    def set_and_check_context(self, key, value, msg=""):
+    def set_and_check_context(self, key, value, type):
         kd = flux.kvs.KVSDir(self.f)
         kd[key] = value
         kd.commit()
-        nv = kd[key]
-        self.assertEqual(value, nv)
-        self.assertFalse(isinstance(nv, six.binary_type))
-        return kd
 
-    def test_set_int(self):
-        self.set_and_check_context("int", 10)
+        kd2 = flux.kvs.KVSDir(self.f)
+        nv = kd2[key]
+        if isinstance(value, bytes) and type is str:
+            self.assertEqual(value.decode("utf-8"), nv)
+        elif isinstance(value, bytes) and type is dict:
+            # convert value bytes into string, then convert into
+            # Python dict via ast.literal_evl for comparison.
+            self.assertDictEqual(ast.literal_eval(value.decode("utf-8")), nv)
+        else:
+            self.assertEqual(value, nv)
+        if type is not None:
+            self.assertTrue(isinstance(nv, type))
+
+        return kd2
+
+    def test_basic_02_set_int(self):
+        self.set_and_check_context("int", 10, int)
 
-    def test_set_float(self):
-        self.set_and_check_context("float", 10.5)
+    def test_basic_03_set_float(self):
+        self.set_and_check_context("float", 10.5, float)
 
-    def test_set_string(self):
-        self.set_and_check_context("string", "stuff")
+    def test_basic_04_set_string(self):
+        self.set_and_check_context("string", "stuff", str)
 
-    def test_set_unicode(self):
-        self.set_and_check_context(u"unicode", u"\u32db \u263a \u32e1")
+    def test_basic_05_set_none(self):
+        self.set_and_check_context("none", None, None)
 
-    def test_set_list(self):
-        self.set_and_check_context("list", [1, 2, 3, 4])
+    def test_basic_06_set_unicode(self):
+        self.set_and_check_context("unicode", "\u32db \u263a \u32e1", str)
 
-    def test_set_dict(self):
+    def test_basic_07_set_bytes(self):
+        self.set_and_check_context("bytes", bytes.fromhex("deadbeef"), bytes)
+
+    def test_basic_08_set_list(self):
+        self.set_and_check_context("list", [1, 2, 3, 4], list)
+
+    def test_basic_09_set_dict(self):
         self.set_and_check_context(
-            "dict", {"thing": "stuff", "other thing": "more stuff"}
+            "dict", {"thing": "stuff", "other thing": "more stuff"}, dict
         )
 
-    def test_exists_dir(self):
+    def test_basic_10_set_legal_json(self):
+        self.set_and_check_context("badjson", b"{}", dict)
+
+    def test_basic_11_set_not_legal_json(self):
+        self.set_and_check_context("badjson", b"{", str)
+
+    def test_basic_12_set_deep(self):
+        self.set_and_check_context("a.b.c.e.f.j.k", 5, int)
+
+    def test_api_01_exists_dir(self):
         with flux.kvs.get_dir(self.f) as kd:
             kd.mkdir("pytestdir")
         self.assertTrue(flux.kvs.exists(self.f, "pytestdir"))
 
-    def test_exists_true(self):
+    def test_api_02_exists_true(self):
         flux.kvs.put(self.f, "thing", 15)
         flux.kvs.commit(self.f)
         self.assertTrue(flux.kvs.exists(self.f, "thing"))
 
-    def test_exists_false(self):
+    def test_api_03_exists_false(self):
         self.assertFalse(flux.kvs.exists(self.f, "argbah"))
 
-    def test_commit_flags(self):
+    def test_api_04_isdir_true(self):
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("testisdir")
+        self.assertTrue(flux.kvs.isdir(self.f, "testisdir"))
+
+    def test_api_05_isdir_false(self):
+        flux.kvs.put(self.f, "testisdirkey", 15)
+        flux.kvs.commit(self.f)
+        self.assertFalse(flux.kvs.isdir(self.f, "testisdirkey"))
+        self.assertFalse(flux.kvs.isdir(self.f, "not_a_key_i_made"))
+
+    def test_api_06_put_mkdir(self):
+        flux.kvs.put_mkdir(self.f, "txn_mkdir")
+        flux.kvs.commit(self.f)
+        self.assertTrue(flux.kvs.exists(self.f, "txn_mkdir"))
+
+    def test_api_07_put_unlink(self):
+        flux.kvs.put(self.f, "txn_unlink", 1)
+        flux.kvs.commit(self.f)
+        flux.kvs.put_unlink(self.f, "txn_unlink")
+        flux.kvs.commit(self.f)
+        self.assertFalse(flux.kvs.exists(self.f, "txn_unlink"))
+
+    def test_api_08_put_symlink(self):
+        flux.kvs.put_symlink(self.f, "txn_symlink", "txn_target")
+        flux.kvs.commit(self.f)
+        self.assertFalse(flux.kvs.exists(self.f, "txn_symlink"))
+
+    def test_api_09_commit_flags(self):
         flux.kvs.put(self.f, "flagcheck", 42)
         flux.kvs.commit(self.f, 1)
         self.assertTrue(flux.kvs.exists(self.f, "flagcheck"))
 
-    def test_remove(self):
-        kd = self.set_and_check_context("todel", "things to delete")
+    # just testing that passing flags work, these are pitiful KVS
+    # changes and the flags don't mean much
+    def test_api_10_commit_flags(self):
+        flux.kvs.put(self.f, "commitflags", "foo")
+        flux.kvs.commit(self.f, flux.constants.FLUX_KVS_NO_MERGE)
+        flux.kvs.put(self.f, "commitflags", "baz")
+        flux.kvs.commit(self.f, flux.constants.FLUX_KVS_TXN_COMPACT)
+        flux.kvs.put(self.f, "commitflags", "bar")
+        flux.kvs.commit(self.f, flux.constants.FLUX_KVS_SYNC)
+
+    # try to overwrite root dir, will fail on commit
+    def test_api_11_commit_fail(self):
+        with self.assertRaises(OSError) as ctx:
+            flux.kvs.put(self.f, ".", "foof")
+            flux.kvs.commit(self.f)
+        self.assertEqual(ctx.exception.errno, errno.EINVAL)
+
+        # Issue #5333, make sure internal bad transaction cleared and
+        # subsequent commit works
+        flux.kvs.commit(self.f)
+
+    def bad_input(self, func, *args):
+        with self.assertRaises(OSError) as ctx:
+            func(*args)
+        self.assertEqual(ctx.exception.errno, errno.EINVAL)
+
+    def test_bad_input_01_exists(self):
+        self.bad_input(flux.kvs.exists, self.f, "")
+
+    def test_bad_input_02_isdir(self):
+        self.bad_input(flux.kvs.isdir, self.f, "")
+
+    def test_bad_input_03_get(self):
+        self.bad_input(flux.kvs.get, self.f, "")
+
+    def test_bad_input_04_get_dir(self):
+        self.bad_input(flux.kvs.get_dir, self.f, "")
+
+    def test_bad_input_05_put_exception(self):
+        self.bad_input(flux.kvs.put, self.f, "", "")
+
+    def test_bad_input_06_put_mkdir_exception(self):
+        self.bad_input(flux.kvs.put_mkdir, self.f, "")
+
+    def test_bad_input_07_put_unlink(self):
+        self.bad_input(flux.kvs.put_unlink, self.f, "")
+
+    def test_bad_input_08_put_symlink(self):
+        self.bad_input(flux.kvs.put_symlink, self.f, "", "")
+
+    def test_kvsdir_01_bad_init(self):
+        with self.assertRaises(ValueError):
+            flux.kvs.KVSDir()
+
+    def test_kvsdir_02_read_non_existent(self):
+        with self.assertRaises(KeyError):
+            print(
+                flux.kvs.KVSDir(self.f)[
+                    "crazykeythatclearlydoesntexistandneverwillinanyuniverse"
+                ]
+            )
+
+    def test_kvsdir_03_read_non_existent_basedir(self):
+        with self.assertRaisesRegex(EnvironmentError, "No such file"):
+            print(
+                flux.kvs.KVSDir(
+                    self.f, "crazykeythatclearlydoesntexistandneverwillinanyuniverse"
+                )
+            )
+
+    def test_kvsdir_04_remove(self):
+        kd = self.set_and_check_context("todel", "things to delete", str)
         del kd["todel"]
         kd.commit()
         with self.assertRaises(KeyError):
             stuff = kd["todel"]
             print(stuff)
 
-    def test_fill(self):
+    def test_kvsdir_05_fill(self):
         with flux.kvs.get_dir(self.f) as kd:
             kd.fill({"things": 1, "stuff": "strstuff", "dir.other_thing": "dirstuff"})
             kd.commit()
 
-            self.assertEqual(kd["things"], 1)
-            self.assertEqual(kd["stuff"], "strstuff")
-            self.assertEqual(kd["dir"]["other_thing"], "dirstuff")
+        with flux.kvs.get_dir(self.f) as kd2:
+            self.assertEqual(kd2["things"], 1)
+            self.assertEqual(kd2["stuff"], "strstuff")
+            self.assertEqual(kd2["dir"]["other_thing"], "dirstuff")
 
-    def test_set_deep(self):
-        self.set_and_check_context("a.b.c.e.f.j.k", 5)
+    def test_kvsdir_06_mkdir_fill(self):
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir(
+                "mkdirfill",
+                {
+                    "thingies": 1,
+                    "stuffs": "strstuffs",
+                    "dir.other_thingies": "dirstuffs",
+                },
+            )
+            kd.commit()
 
-    def test_bad_init(self):
-        with self.assertRaises(ValueError):
-            flux.kvs.KVSDir()
+        with flux.kvs.get_dir(self.f) as kd2:
+            self.assertEqual(kd2["mkdirfill.thingies"], 1)
+            self.assertEqual(kd2["mkdirfill.stuffs"], "strstuffs")
+            self.assertEqual(kd2["mkdirfill"]["dir"]["other_thingies"], "dirstuffs")
 
-    def test_key_at(self):
+    def test_kvsdir_07_key_at(self):
         with flux.kvs.get_dir(self.f) as kd:
             kd.mkdir("testkeyat")
         with flux.kvs.get_dir(self.f, "testkeyat") as kd:
             self.assertEqual(kd.key_at("meh"), "testkeyat.meh")
 
-    def test_walk_with_no_handle(self):
-        with self.assertRaises(ValueError):
-            flux.kvs.walk("dir").next()
+    def test_kvsdir_08_exists_initial_path(self):
+        with flux.kvs.get_dir(self.f) as kd:
+            kd["exists1"] = 1
+            kd.mkdir("existssubdir")
+            kd["existssubdir.exists2"] = 2
+
+        with flux.kvs.get_dir(self.f) as kd2:
+            self.assertTrue(kd2.exists("exists1"))
+            self.assertTrue(kd2.exists("existssubdir.exists2"))
+
+        with flux.kvs.get_dir(self.f, "existssubdir") as kd3:
+            self.assertFalse(kd3.exists("exists1"))
+            self.assertTrue(kd3.exists("exists2"))
+
+    def test_kvsdir_09_key_initial_path(self):
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("initialpath")
+
+        kd2 = flux.kvs.KVSDir(self.f)
+        kd2["initialpath.a"] = 1
+        kd2["initialpath"]["b"] = 2
+        kd2.commit()
+
+        kd3 = flux.kvs.KVSDir(self.f, "initialpath")
+        kd3["c"] = 3
+        kd3["d.e.f"] = 4
+        kd3.commit()
+
+        kd4 = flux.kvs.KVSDir(self.f)
+        self.assertEqual(kd4["initialpath.a"], 1)
+        self.assertEqual(kd4["initialpath"]["a"], 1)
+        self.assertEqual(kd4["initialpath.b"], 2)
+        self.assertEqual(kd4["initialpath"]["b"], 2)
+        self.assertEqual(kd4["initialpath.c"], 3)
+        self.assertEqual(kd4["initialpath"]["c"], 3)
+        self.assertEqual(kd4["initialpath.d.e.f"], 4)
+        self.assertEqual(kd4["initialpath"]["d.e.f"], 4)
+        self.assertEqual(kd4["initialpath"]["d"]["e"]["f"], 4)
+
+        kd5 = flux.kvs.KVSDir(self.f, "initialpath")
+        self.assertEqual(kd5["a"], 1)
+        self.assertEqual(kd5["b"], 2)
+        self.assertEqual(kd5["c"], 3)
+        self.assertEqual(kd5["d.e.f"], 4)
+        self.assertEqual(kd5["d"]["e"]["f"], 4)
+
+    def test_kvsdir_10_unlink_initial_path(self):
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("unlinkinitialpath")
+            kd["unlinkinitialpath"]["a"] = 1
 
-    def test_read_non_existent(self):
+        kd2 = flux.kvs.KVSDir(self.f)
+        self.assertEqual(kd2["unlinkinitialpath.a"], 1)
+
+        kd3 = flux.kvs.KVSDir(self.f, "unlinkinitialpath")
+        del kd3["a"]
+        kd3.commit()
+
+        kd4 = flux.kvs.KVSDir(self.f)
         with self.assertRaises(KeyError):
-            print(
-                flux.kvs.KVSDir(self.f)[
-                    "crazykeythatclearlydoesntexistandneverwillinanyuniverse"
-                ]
-            )
+            kd4["unlinkinitialpath.a"]
 
-    def test_read_non_existent_basedir(self):
-        with self.assertRaisesRegexp(EnvironmentError, "No such file"):
-            print(
-                flux.kvs.KVSDir(
-                    self.f, "crazykeythatclearlydoesntexistandneverwillinanyuniverse"
-                )
-            )
+    def test_kvsdir_11_fill_initial_path(self):
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("fillinitialpath")
+
+        with flux.kvs.get_dir(self.f, "fillinitialpath") as kd2:
+            kd2.fill({"g": 1, "h": "bar", "i.j.k": "baz"})
+            kd2.commit()
+
+        with flux.kvs.get_dir(self.f) as kd3:
+            self.assertEqual(kd3["fillinitialpath.g"], 1)
+            self.assertEqual(kd3["fillinitialpath.h"], "bar")
+            self.assertEqual(kd3["fillinitialpath.i.j.k"], "baz")
+            self.assertEqual(kd3["fillinitialpath.i"]["j"]["k"], "baz")
 
-    def test_iterator(self):
+        with flux.kvs.get_dir(self.f, "fillinitialpath") as kd4:
+            self.assertEqual(kd4["g"], 1)
+            self.assertEqual(kd4["h"], "bar")
+            self.assertEqual(kd4["i.j.k"], "baz")
+            self.assertEqual(kd4["i"]["j"]["k"], "baz")
+
+    def test_kvsdir_12_mkdir_initial_path(self):
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("mkdirinitialpath", {"l": 1, "m": "bar", "n.o.p": "baz"})
+            kd.commit()
+
+        with flux.kvs.get_dir(self.f) as kd2:
+            self.assertEqual(kd2["mkdirinitialpath.l"], 1)
+            self.assertEqual(kd2["mkdirinitialpath.m"], "bar")
+            self.assertEqual(kd2["mkdirinitialpath.n.o.p"], "baz")
+            self.assertEqual(kd2["mkdirinitialpath.n"]["o"]["p"], "baz")
+
+        with flux.kvs.get_dir(self.f, "mkdirinitialpath") as kd3:
+            self.assertEqual(kd3["l"], 1)
+            self.assertEqual(kd3["m"], "bar")
+            self.assertEqual(kd3["n.o.p"], "baz")
+            self.assertEqual(kd3["n"]["o"]["p"], "baz")
+
+    def test_kvsdir_13_iterator(self):
         keys = ["testdir1a." + str(x) for x in range(1, 15)]
         with flux.kvs.get_dir(self.f) as kd:
             for k in keys:
@@ -146,7 +356,44 @@ def test_iterator(self):
                 self.assertEqual(v, "bar")
                 print("passed {}".format(k))
 
-    def test_walk(self):
+    def test_kvsdir_14_files(self):
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("filestest", {"somefile": 1, "somefile2": 2})
+            kd.mkdir("filestest.subdir")
+            kd.commit()
+
+        with flux.kvs.get_dir(self.f, "filestest") as kd2:
+            files = [x for x in kd2.files()]
+            self.assertEqual(len(files), 2)
+            self.assertIn("somefile", files)
+            self.assertIn("somefile2", files)
+
+    def test_kvsdir_15_directories(self):
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("directoriestest", {"somefile": 1, "somefile2": 2})
+            kd.mkdir("directoriestest.subdir")
+            kd.commit()
+
+        with flux.kvs.get_dir(self.f, "directoriestest") as kd2:
+            directories = [x for x in kd2.directories()]
+            self.assertEqual(len(directories), 1)
+            self.assertIn("subdir", directories)
+
+    def test_kvsdir_16_list_all(self):
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("listalltest", {"somefile": 1, "somefile2": 2})
+            kd.mkdir("listalltest.subdir")
+            kd.commit()
+
+        with flux.kvs.get_dir(self.f, "listalltest") as kd2:
+            (files, directories) = kd2.list_all()
+            self.assertEqual(len(files), 2)
+            self.assertEqual(len(directories), 1)
+            self.assertIn("somefile", files)
+            self.assertIn("somefile2", files)
+            self.assertIn("subdir", directories)
+
+    def test_misc_01_walk(self):
         keys = ["testwalk." + str(x) for x in range(1, 15)]
         with flux.kvs.get_dir(self.f) as kd:
             for k in keys:
@@ -162,22 +409,576 @@ def test_walk(self):
         for r, ds, fs in walk_gen:
             pass
 
-    def test_put_mkdir(self):
-        flux.kvs.put_mkdir(self.f, "txn_mkdir")
-        flux.kvs.commit(self.f)
-        self.assertTrue(flux.kvs.exists(self.f, "txn_mkdir"))
+    def test_misc_02_walk_with_no_handle(self):
+        with self.assertRaises(ValueError):
+            flux.kvs.walk("dir").next()
 
-    def test_put_unlink(self):
-        flux.kvs.put(self.f, "txn_unlink", 1)
-        flux.kvs.commit(self.f)
-        flux.kvs.put_unlink(self.f, "txn_unlink")
-        flux.kvs.commit(self.f)
-        self.assertFalse(flux.kvs.exists(self.f, "txn_unlink"))
+    # N.B. namespace tests may depend on prior tests creating
+    # namespaces and data within those namespaces.  So order matters
+    # in these tests and the numbering should be kept to enforce that
+    # order.
+
+    def test_namespace_01_namespace_list(self):
+        nslist = flux.kvs.namespace_list(self.f)
+        self.assertIn("primary", nslist)
+
+    def test_namespace_02_namespace_create(self):
+        flux.kvs.namespace_create(self.f, "testns1")
+        flux.kvs.namespace_create(self.f, "testns2")
+        flux.kvs.namespace_create(self.f, "testns3")
+        nslist = flux.kvs.namespace_list(self.f)
+        self.assertIn("testns1", nslist)
+        self.assertIn("testns2", nslist)
+        self.assertIn("testns3", nslist)
+
+    def test_namespace_03_namespace_remove(self):
+        flux.kvs.namespace_remove(self.f, "testns3")
+        # namespace removal is eventually consistent, may need to
+        # wait a bit to confirm.
+        removed = False
+        for i in range(30):
+            nslist = flux.kvs.namespace_list(self.f)
+            if "testns3" not in nslist:
+                removed = True
+                break
+        self.assertTrue(removed)
+
+    def test_namespace_04_commit(self):
+        flux.kvs.put_mkdir(self.f, "testdirns1")
+        flux.kvs.put(self.f, "testdirns1.a", 1)
+        flux.kvs.commit(self.f, namespace="testns1")
+        flux.kvs.put_mkdir(self.f, "testdirns2")
+        flux.kvs.put(self.f, "testdirns2.a", 2)
+        flux.kvs.commit(self.f, namespace="testns2")
+
+    def test_namespace_05_get(self):
+        self.assertEqual(flux.kvs.get(self.f, "testdirns1.a", namespace="testns1"), 1)
+        self.assertEqual(flux.kvs.get(self.f, "testdirns2.a", namespace="testns2"), 2)
+        with self.assertRaises(OSError) as cm:
+            flux.kvs.get(self.f, "testdirns1.a", namespace="testns2")
+        self.assertEqual(cm.exception.errno, errno.ENOENT)
+        with self.assertRaises(OSError) as cm:
+            flux.kvs.get(self.f, "testdirns2.a", namespace="testns1")
+        self.assertEqual(cm.exception.errno, errno.ENOENT)
+
+    def test_namespace_06_exists(self):
+        self.assertTrue(flux.kvs.exists(self.f, "testdirns1", namespace="testns1"))
+        self.assertTrue(flux.kvs.exists(self.f, "testdirns1.a", namespace="testns1"))
+        self.assertFalse(flux.kvs.exists(self.f, "testdirns1", namespace="testns2"))
+        self.assertFalse(flux.kvs.exists(self.f, "testdirns1.a", namespace="testns2"))
+
+        self.assertFalse(flux.kvs.exists(self.f, "testdirns2", namespace="testns1"))
+        self.assertFalse(flux.kvs.exists(self.f, "testdirns2.a", namespace="testns1"))
+        self.assertTrue(flux.kvs.exists(self.f, "testdirns2", namespace="testns2"))
+        self.assertTrue(flux.kvs.exists(self.f, "testdirns2.a", namespace="testns2"))
+
+    def test_namespace_06_isdir(self):
+        self.assertTrue(flux.kvs.isdir(self.f, "testdirns1", namespace="testns1"))
+        self.assertFalse(flux.kvs.isdir(self.f, "testdirns1.a", namespace="testns1"))
+        self.assertFalse(flux.kvs.isdir(self.f, "testdirns1", namespace="testns2"))
+        self.assertFalse(flux.kvs.isdir(self.f, "testdirns1.a", namespace="testns2"))
+
+        self.assertFalse(flux.kvs.isdir(self.f, "testdirns2", namespace="testns1"))
+        self.assertFalse(flux.kvs.isdir(self.f, "testdirns2.a", namespace="testns1"))
+        self.assertTrue(flux.kvs.isdir(self.f, "testdirns2", namespace="testns2"))
+        self.assertFalse(flux.kvs.isdir(self.f, "testdirns2.a", namespace="testns2"))
+
+    def test_namespace_07_unlink(self):
+        flux.kvs.namespace_create(self.f, "testnsunlink")
+        flux.kvs.put(self.f, "todelete", 1)
+        flux.kvs.commit(self.f, namespace="testnsunlink")
+        self.assertTrue(flux.kvs.exists(self.f, "todelete", namespace="testnsunlink"))
+
+        flux.kvs.put_unlink(self.f, "todelete")
+        flux.kvs.commit(self.f, namespace="testnsunlink")
+
+        with self.assertRaises(OSError) as cm:
+            flux.kvs.get(self.f, "todelete", namespace="testnsunlink")
+        self.assertEqual(cm.exception.errno, errno.ENOENT)
+
+    def test_namespace_08_KVSDir_fill(self):
+        flux.kvs.namespace_create(self.f, "testnsfill")
+        with flux.kvs.get_dir(self.f, namespace="testnsfill") as kd:
+            kd.fill({"testdirnsfill.a": 1})
+            kd.fill({"testdirnsfill.b": 2})
+            kd.commit()
 
-    def test_put_symlink(self):
-        flux.kvs.put_symlink(self.f, "txn_symlink", "txn_target")
-        flux.kvs.commit(self.f)
-        self.assertFalse(flux.kvs.exists(self.f, "txn_symlink"))
+        self.assertTrue(flux.kvs.isdir(self.f, "testdirnsfill", namespace="testnsfill"))
+        self.assertTrue(
+            flux.kvs.exists(self.f, "testdirnsfill.a", namespace="testnsfill")
+        )
+        self.assertTrue(
+            flux.kvs.exists(self.f, "testdirnsfill.b", namespace="testnsfill")
+        )
+
+    def test_namespace_09_KVSDir_mkdir_fill(self):
+        flux.kvs.namespace_create(self.f, "testnsmkdirfill")
+        with flux.kvs.get_dir(self.f, namespace="testnsmkdirfill") as kd:
+            kd.mkdir("testdirnsmkdirfill", {"a": 1})
+            kd.commit()
+
+        self.assertTrue(
+            flux.kvs.isdir(self.f, "testdirnsmkdirfill", namespace="testnsmkdirfill")
+        )
+        self.assertTrue(
+            flux.kvs.exists(self.f, "testdirnsmkdirfill.a", namespace="testnsmkdirfill")
+        )
+
+    def test_namespace_10_KVSDir_initial_path(self):
+        with flux.kvs.get_dir(self.f, "testdirns1", namespace="testns1") as kd:
+            self.assertTrue(kd.exists("a"))
+            kd["b"] = 2
+
+        with flux.kvs.get_dir(self.f, namespace="testns1") as kd2:
+            self.assertTrue(kd2.exists("testdirns1.a"))
+            self.assertTrue(kd2.exists("testdirns1.b"))
+
+        self.assertFalse(flux.kvs.exists(self.f, "testdirns1.b", namespace="testns2"))
+
+    def test_namespace_11_KVSDir_files(self):
+        with flux.kvs.get_dir(self.f, "testdirns1", namespace="testns1") as kd:
+            files = [x for x in kd.files()]
+            self.assertEqual(len(files), 2)
+            self.assertIn("a", files)
+            self.assertIn("b", files)
+
+        with flux.kvs.get_dir(self.f, "testdirns2", namespace="testns2") as kd2:
+            files = [x for x in kd2.files()]
+            self.assertEqual(len(files), 1)
+            self.assertIn("a", files)
+
+    def test_namespace_12_KVSDir_directories(self):
+        with flux.kvs.get_dir(self.f, namespace="testns1") as kd:
+            directories = [x for x in kd.directories()]
+            self.assertEqual(len(directories), 1)
+            self.assertIn("testdirns1", directories)
+
+        with flux.kvs.get_dir(self.f, namespace="testns2") as kd2:
+            directories = [x for x in kd2.directories()]
+            self.assertEqual(len(directories), 1)
+            self.assertIn("testdirns2", directories)
+
+    def test_namespace_13_KVSDir_list_all(self):
+        with flux.kvs.get_dir(self.f, namespace="testns1") as kd:
+            (files, directories) = kd.list_all()
+            self.assertEqual(len(files), 0)
+            self.assertEqual(len(directories), 1)
+
+        with flux.kvs.get_dir(self.f, namespace="testns2") as kd2:
+            (files, directories) = kd2.list_all()
+            self.assertEqual(len(files), 0)
+            self.assertEqual(len(directories), 1)
+
+    def test_namespace_14_walk(self):
+        walk_gen = flux.kvs.walk(
+            ".", flux_handle=self.f, topdown=True, namespace="testns1"
+        )
+        (r, ds, fs) = next(walk_gen)
+        self.assertEqual(r, "")
+        self.assertEqual(len(list(ds)), 1)
+        self.assertEqual(len(list(fs)), 0)
+
+        walk_gen = flux.kvs.walk(
+            "testdirns1", flux_handle=self.f, topdown=True, namespace="testns1"
+        )
+        (r, ds, fs) = next(walk_gen)
+        self.assertEqual(r, "")
+        self.assertEqual(len(list(ds)), 0)
+        self.assertEqual(len(list(fs)), 2)
+
+    # N.B. some kvstxn tests depend on prior tests creating dirs/keys.
+    # Thus tests are numbered to ensure they are performed in order.
+
+    def test_kvstxn_01_basic(self):
+        txn = flux.kvs.KVSTxn(self.f)
+        txn.mkdir("kvstxntestdir")
+        txn.put("kvstxntestdir.a", 1)
+        txn.put("kvstxntestdir.b", 2)
+        txn.symlink("symlink", "kvstxntestdir.a")
+        txn.commit()
+
+        self.assertTrue(flux.kvs.isdir(self.f, "kvstxntestdir"))
+        self.assertEqual(flux.kvs.get(self.f, "kvstxntestdir.a"), 1)
+        self.assertEqual(flux.kvs.get(self.f, "kvstxntestdir.b"), 2)
+        self.assertEqual(flux.kvs.get(self.f, "symlink"), 1)
+
+        txn.unlink("kvstxntestdir.b")
+        txn.commit()
+
+        self.assertFalse(flux.kvs.exists(self.f, "kvstxntestdir.b"))
+
+    def test_kvstxn_02_initial_path(self):
+        txn = flux.kvs.KVSTxn(self.f, "kvstxntestdir")
+        txn.put("c", 3)
+        txn.commit()
+
+        self.assertEqual(flux.kvs.get(self.f, "kvstxntestdir.c"), 3)
+
+        txn.unlink("c")
+        txn.commit()
+
+        self.assertFalse(flux.kvs.exists(self.f, "kvstxntestdir.c"))
+
+    def test_kvstxn_03_namespace(self):
+        flux.kvs.namespace_create(self.f, "testns")
+
+        txn = flux.kvs.KVSTxn(self.f, namespace="testns")
+        txn.put("nskey", 1)
+        txn.commit()
+
+        self.assertTrue(flux.kvs.exists(self.f, "nskey", namespace="testns"))
+        self.assertFalse(flux.kvs.exists(self.f, "nskey"))
+
+    def test_kvstxn_04_context_manager(self):
+        with flux.kvs.KVSTxn(self.f) as txn:
+            txn.put("kvstxntestdir.d", 4)
+
+        self.assertEqual(flux.kvs.get(self.f, "kvstxntestdir.d"), 4)
+
+        with flux.kvs.KVSTxn(self.f) as txn2:
+            txn2.unlink("kvstxntestdir.d")
+
+        self.assertFalse(flux.kvs.exists(self.f, "kvstxntestdir.d"))
+
+    def test_kvstxn_05_pass_to_api(self):
+        # N.B. generally user should never do this, testing to ensure
+        # logic works.
+        txn = flux.kvs.KVSTxn(self.f)
+        flux.kvs.put_mkdir(self.f, "kvstxntestapi", _kvstxn=txn)
+        flux.kvs.put(self.f, "kvstxntestapi.a", 1, _kvstxn=txn)
+        flux.kvs.put(self.f, "kvstxntestapi.b", 2, _kvstxn=txn)
+        flux.kvs.put_symlink(self.f, "symlink", "kvstxntestapi.a", _kvstxn=txn)
+        flux.kvs.commit(self.f, _kvstxn=txn)
+
+        self.assertTrue(flux.kvs.isdir(self.f, "kvstxntestapi"))
+        self.assertEqual(flux.kvs.get(self.f, "kvstxntestapi.a"), 1)
+        self.assertEqual(flux.kvs.get(self.f, "kvstxntestapi.b"), 2)
+        self.assertEqual(flux.kvs.get(self.f, "symlink"), 1)
+
+        flux.kvs.put_unlink(self.f, "kvstxntestapi.b", _kvstxn=txn)
+        flux.kvs.commit(self.f, _kvstxn=txn)
+
+        self.assertFalse(flux.kvs.exists(self.f, "kvstxntestapi.b"))
+
+    def test_kvstxn_06_multiple(self):
+        with flux.kvs.KVSTxn(self.f) as txn:
+            txn.mkdir("kvstxntestmulti")
+
+        txn1 = flux.kvs.KVSTxn(self.f)
+        txn1.put("kvstxntestmulti.a", 1)
+        txn1.put("kvstxntestmulti.c", 3)
+
+        txn2 = flux.kvs.KVSTxn(self.f)
+        txn2.put("kvstxntestmulti.b", 2)
+        txn2.put("kvstxntestmulti.d", 4)
+
+        txn1.commit()
+
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmulti.a"))
+        self.assertFalse(flux.kvs.exists(self.f, "kvstxntestmulti.b"))
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmulti.c"))
+        self.assertFalse(flux.kvs.exists(self.f, "kvstxntestmulti.d"))
+
+        txn2.commit()
+
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmulti.a"))
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmulti.b"))
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmulti.c"))
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmulti.d"))
+
+    def test_kvstxn_06_multiple_kvsdir(self):
+        with flux.kvs.KVSTxn(self.f) as txn:
+            txn.mkdir("kvstxntestmultikvsdir")
+
+        kvsdir1 = flux.kvs.get_dir(self.f, "kvstxntestmultikvsdir")
+        kvsdir1["a"] = 1
+        kvsdir1["c"] = 3
+
+        kvsdir2 = flux.kvs.get_dir(self.f, "kvstxntestmultikvsdir")
+        kvsdir2["b"] = 2
+        kvsdir2["d"] = 4
+
+        kvsdir1.commit()
+
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmultikvsdir.a"))
+        self.assertFalse(flux.kvs.exists(self.f, "kvstxntestmultikvsdir.b"))
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmultikvsdir.c"))
+        self.assertFalse(flux.kvs.exists(self.f, "kvstxntestmultikvsdir.d"))
+
+        kvsdir2.commit()
+
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmultikvsdir.a"))
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmultikvsdir.b"))
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmultikvsdir.c"))
+        self.assertTrue(flux.kvs.exists(self.f, "kvstxntestmultikvsdir.d"))
+
+    # this test ensures subdirs get the same internal transaction of
+    # the parent KVSDir object, therefore only a single commit() call is
+    # necessary
+    def test_kvstxn_07_kvsdir_subdir(self):
+        with flux.kvs.KVSTxn(self.f) as txn:
+            txn.mkdir("kvstxntestsubdir")
+            txn.mkdir("kvstxntestsubdir.subdir")
+
+        kvsdir = flux.kvs.get_dir(
+            self.f,
+        )
+        subdir = kvsdir["kvstxntestsubdir"]["subdir"]
+        subdir["a"] = 1
+
+        kvsdir.commit()
+
+        self.assertEqual(flux.kvs.get(self.f, "kvstxntestsubdir.subdir.a"), 1)
+
+    # this test ensures commits are not done when an object reference
+    # count goes to 0
+    def test_kvstxn_08_kvsdir_keep_reference(self):
+        with flux.kvs.KVSTxn(self.f) as txn:
+            txn.mkdir("kvstxntestkeepref")
+            txn.mkdir("kvstxntestkeepref.dir1")
+            txn.mkdir("kvstxntestkeepref.dir1.dir2")
+
+        kvsdir = flux.kvs.get_dir(
+            self.f,
+        )
+        kvsdir["kvstxntestkeepref"]["dir1"]["dir2"]["a"] = 1
+        # N.B. flake8 check, F841 = "assigned to but never used"
+        keepref1 = kvsdir["kvstxntestkeepref"]  # noqa: F841
+        keepref2 = kvsdir["kvstxntestkeepref"]["dir1"]  # noqa: F841
+        keepref3 = kvsdir["kvstxntestkeepref"]["dir1"]["dir2"]  # noqa: F841
+
+        kvsdir.commit()
+
+        self.assertEqual(flux.kvs.get(self.f, "kvstxntestkeepref.dir1.dir2.a"), 1)
+
+    def test_commic_async_01_basic(self):
+        flux.kvs.put(self.f, "commicasync1", "foo")
+        f = flux.kvs.commit_async(self.f)
+        f.get()
+        self.assertEqual(flux.kvs.get(self.f, "commicasync1"), "foo")
+
+    def test_commic_async_02_with_kvstxn(self):
+        txn = flux.kvs.KVSTxn(self.f)
+        txn.put("commitasync2", "bar")
+
+        f = flux.kvs.commit_async(self.f, _kvstxn=txn)
+        f.get()
+        self.assertEqual(flux.kvs.get(self.f, "commitasync2"), "bar")
+
+    def test_commic_async_03_kvstxn(self):
+        txn = flux.kvs.KVSTxn(self.f)
+        txn.put("commitasync3", "baz")
+        f = txn.commit_async()
+        f.get()
+        self.assertEqual(flux.kvs.get(self.f, "commitasync3"), "baz")
+
+    def test_commic_async_04_kvsdir(self):
+        kd = flux.kvs.KVSDir(self.f)
+        kd["commitasync4"] = "qux"
+        f = kd.commit_async()
+        f.get()
+        self.assertEqual(flux.kvs.get(self.f, "commitasync4"), "qux")
+
+    def test_kvswatch_01_key_ENOENT(self):
+        with self.assertRaises(OSError) as cm:
+            future = flux.kvs.kvs_watch_async(self.f, "grog")
+            future.get()
+        self.assertEqual(cm.exception.errno, errno.ENOENT)
+
+    def _change_value_kvs_nowait(self, key, val):
+        kvstxn = flux.kvs.KVSTxn(self.f)
+        kvstxn.put(key, val)
+        flux.kvs.commit_async(self.f, _kvstxn=kvstxn)
+
+    def test_kvswatch_02_kvs_watch_async_basic(self):
+        myarg = dict(a=1, b=2)
+        vals = []
+
+        def cb(future, arg):
+            self.assertEqual(arg, myarg)
+            val = future.get()
+            if val is None:
+                return
+            elif val == 1:
+                self._change_value_kvs_nowait("kvswatch1.val", 2)
+            elif val == 2:
+                future.cancel()
+            vals.append(val)
+
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("kvswatch1")
+            kd["kvswatch1.val"] = 1
+
+        future = flux.kvs.kvs_watch_async(self.f, "kvswatch1.val")
+        self.assertIsInstance(future, flux.kvs.KVSWatchFuture)
+        future.then(cb, myarg)
+        rc = self.f.reactor_run()
+        self.assertGreaterEqual(rc, 0)
+        self.assertEqual(len(vals), 2)
+        self.assertEqual(vals[0], 1)
+        self.assertEqual(vals[1], 2)
+
+    def test_kvswatch_03_kvs_watch_async_no_autoreset(self):
+        vals = []
+
+        def cb(future, arg):
+            val = future.get(autoreset=False)
+            if val is None:
+                return
+            elif val == 1:
+                valtmp = future.get()
+                # Value hasn't changed
+                self.assertEqual(valtmp, 1)
+                self._change_value_kvs_nowait("kvswatch2.val", 2)
+            elif val == 2:
+                valtmp = future.get()
+                # Value hasn't changed
+                self.assertEqual(valtmp, 2)
+                future.cancel()
+            vals.append(val)
+
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("kvswatch2")
+            kd["kvswatch2.val"] = 1
+
+        future = flux.kvs.kvs_watch_async(self.f, "kvswatch2.val")
+        self.assertIsInstance(future, flux.kvs.KVSWatchFuture)
+        future.then(cb, None)
+        rc = self.f.reactor_run()
+        self.assertGreaterEqual(rc, 0)
+        self.assertEqual(len(vals), 2)
+        self.assertEqual(vals[0], 1)
+        self.assertEqual(vals[1], 2)
+
+    def test_kvswatch_04_kvs_watch_async_waitcreate(self):
+        # To test waitcreate, we create two KVS watchers, one with
+        # waitcreate and one without.  The one without waitcreate will
+        # create the field we are waiting to be created.
+        results = []
+
+        def cb_ENOENT(future, arg):
+            try:
+                future.get()
+            except OSError as e:
+                self.assertEqual(e.errno, errno.ENOENT)
+                self._change_value_kvs_nowait("kvswatch3.val", 1)
+                results.append("ENOENT")
+
+        def cb_waitcreate(future, arg):
+            val = future.get()
+            if val is None:
+                return
+            elif val == 1:
+                future.cancel()
+            results.append(val)
+
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("kvswatch3")
+
+        future1 = flux.kvs.kvs_watch_async(self.f, "kvswatch3.val")
+        future2 = flux.kvs.kvs_watch_async(self.f, "kvswatch3.val", waitcreate=True)
+        future1.then(cb_ENOENT, None)
+        future2.then(cb_waitcreate, None)
+        rc = self.f.reactor_run()
+        self.assertGreaterEqual(rc, 0)
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results[0], "ENOENT")
+        self.assertEqual(results[1], 1)
+
+    def test_kvswatch_05_kvs_watch_async_uniq(self):
+        # To test uniq, we create two KVS watchers, one with uniq and
+        # one without.  The one with uniq should see fewer changes
+        # than the one without.
+        vals = []
+        uniq_vals = []
+
+        def cb(future, arg):
+            otherfuture = arg
+            val = future.get()
+            if val is None:
+                return
+            elif val == 1:
+                if len(vals) == 0:
+                    self._change_value_kvs_nowait("kvswatch4.val", 1)
+                elif len(vals) == 1:
+                    self._change_value_kvs_nowait("kvswatch4.val", 2)
+            vals.append(val)
+            if len(vals) == 3 and len(uniq_vals) == 2:
+                future.cancel()
+                otherfuture.cancel()
+
+        def cb_uniq(future, arg):
+            otherfuture = arg
+            val = future.get()
+            if val is None:
+                return
+            uniq_vals.append(val)
+            if len(vals) == 3 and len(uniq_vals) == 2:
+                future.cancel()
+                otherfuture.cancel()
+
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("kvswatch4")
+            kd["kvswatch4.val"] = 1
+
+        future1 = flux.kvs.kvs_watch_async(self.f, "kvswatch4.val")
+        future2 = flux.kvs.kvs_watch_async(self.f, "kvswatch4.val", uniq=True)
+        future1.then(cb, future2)
+        future2.then(cb_uniq, future1)
+        rc = self.f.reactor_run()
+        self.assertGreaterEqual(rc, 0)
+        self.assertEqual(len(vals), 3)
+        self.assertEqual(vals[0], 1)
+        self.assertEqual(vals[1], 1)
+        self.assertEqual(vals[2], 2)
+        self.assertEqual(len(uniq_vals), 2)
+        self.assertEqual(uniq_vals[0], 1)
+        self.assertEqual(uniq_vals[1], 2)
+
+    def test_kvswatch_06_kvs_watch_async_full(self):
+        # To test full, we create two KVS watchers, one with full and
+        # one without.  The one with full should see a change if we
+        # delete an upstream directory.  The other watcher should not.
+        vals = []
+        full_vals = []
+
+        def cb(future, arg):
+            val = future.get()
+            if val is None:
+                return
+            vals.append(val)
+
+        def cb_full(future, arg):
+            otherfuture = arg
+            try:
+                val = future.get()
+                if val is None:
+                    return
+                elif val == 1:
+                    # Set to None == unlink/delete/remove
+                    self._change_value_kvs_nowait("kvswatch5", None)
+            except OSError:
+                future.cancel()
+                otherfuture.cancel()
+                full_vals.append("ENOENT")
+                return
+            full_vals.append(val)
+
+        with flux.kvs.get_dir(self.f) as kd:
+            kd.mkdir("kvswatch5")
+            kd["kvswatch5.val"] = 1
+
+        future1 = flux.kvs.kvs_watch_async(self.f, "kvswatch5.val")
+        future2 = flux.kvs.kvs_watch_async(self.f, "kvswatch5.val", full=True)
+        future1.then(cb, None)
+        future2.then(cb_full, future1)
+        rc = self.f.reactor_run()
+        self.assertGreaterEqual(rc, 0)
+        self.assertEqual(len(vals), 1)
+        self.assertEqual(vals[0], 1)
+        self.assertEqual(len(full_vals), 2)
+        self.assertEqual(full_vals[0], 1)
+        self.assertEqual(full_vals[1], "ENOENT")
 
 
 if __name__ == "__main__":
diff --git a/t/python/t0006-request.py b/t/python/t0006-request.py
index 85ebc174626c..8504f8145220 100755
--- a/t/python/t0006-request.py
+++ b/t/python/t0006-request.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
@@ -10,9 +10,9 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
+import errno
 import unittest
 
-import errno
 import flux
 from subflux import rerun_under_flux
 
diff --git a/t/python/t0007-watchers.py b/t/python/t0007-watchers.py
index 6b1bfa428a46..0cd00c2d9ce4 100755
--- a/t/python/t0007-watchers.py
+++ b/t/python/t0007-watchers.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
@@ -10,6 +10,9 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
+import gc
+import os
+import signal
 import unittest
 
 import flux
@@ -53,18 +56,26 @@ def test_timer_with_reactor(self):
 
         def cb(x, y, z, w):
             timer_ran[0] = True
-            x.reactor_stop(self.f.get_reactor())
+            x.reactor_stop()
 
         with self.f.timer_watcher_create(0.1, cb) as timer:
             self.assertIsNotNone(timer, msg="timeout create")
-            ret = self.f.reactor_run(self.f.get_reactor(), 0)
+            ret = self.f.reactor_run()
             self.assertEqual(ret, 0, msg="Reactor exit")
             self.assertTrue(timer_ran[0], msg="Timer did not run successfully")
 
+    def test_timer_callback_exception(self):
+        def cb(x, y, z, w):
+            raise RuntimeError("this is a test")
+
+        with self.f.timer_watcher_create(0.01, cb):
+            with self.assertRaises(RuntimeError):
+                self.f.reactor_run()
+
     def test_msg_watcher_unicode(self):
         with self.f.msg_watcher_create(
             lambda handle, x, y, z: handle.fatal_error("cb should not run"),
-            topic_glob=u"foo.*",
+            topic_glob="foo.*",
         ) as mw:
             self.assertIsNotNone(mw)
 
@@ -75,9 +86,172 @@ def test_msg_watcher_bytes(self):
         ) as mw:
             self.assertIsNotNone(mw)
 
+    def test_msg_watcher_rolemask(self):
+        with self.f.msg_watcher_create(
+            lambda handle, x, y, z: handle.fatal_error("cb should not run"),
+            topic_glob="foo.*",
+            rolemask=flux.constants.FLUX_ROLE_ALL,
+        ) as mw:
+            self.assertIsNotNone(mw)
+
+
+class TestSignal(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.f = flux.Flux()
+
+    def test_signal_watcher_invalid(self):
+        """Add an invalid signal"""
+        with self.assertRaises(EnvironmentError):
+            self.f.signal_watcher_create(
+                -500, lambda x, y: x.fatal_error("signal should not fire")
+            )
+        with self.assertRaises(EnvironmentError):
+            self.f.signal_watcher_create(
+                0, lambda x, y: x.fatal_error("signal should not fire")
+            )
+        with self.assertRaises(EnvironmentError):
+            self.f.signal_watcher_create(
+                500, lambda x, y: x.fatal_error("signal should not fire")
+            )
+
+    def test_s0_signal_watcher_add(self):
+        """Add a signal watcher"""
+        with self.f.signal_watcher_create(
+            2, lambda x, y, z, w: x.fatal_error("signal should not fire")
+        ) as sigw:
+            self.assertIsNotNone(sigw)
+
+    def test_s1_signal_watcher_remove(self):
+        """Remove a timer"""
+        sigw = self.f.signal_watcher_create(
+            2, lambda x, y: x.fatal_error("signal should not fire")
+        )
+        sigw.stop()
+        sigw.destroy()
+
+    def test_signal_watcher(self):
+        cb_called = False
+
+        def cb(handle, watcher, signum, args):
+            nonlocal cb_called
+            cb_called = True
+            handle.reactor_stop()
+
+        def raise_signal(handle, watcher, revents, args):
+            os.kill(os.getpid(), signal.SIGUSR1)
+
+        def stop(h, w, r, y):
+            h.reactor_stop()
+
+        with self.f.signal_watcher_create(signal.SIGUSR1, cb) as watcher:
+            self.assertIsNotNone(watcher, msg="signalWatcher create")
+            to1 = self.f.timer_watcher_create(0.10, raise_signal)
+            to2 = self.f.timer_watcher_create(0.15, stop)
+            to1.start()
+            to2.start()
+            rc = self.f.reactor_run()
+            self.assertTrue(rc >= 0, msg="reactor exit")
+            self.assertTrue(cb_called, "Signal Watcher Called")
+
+    def test_signal_watcher_exception(self):
+        def signal_cb(handle, watcher, signum, args):
+            raise RuntimeError(f"got signal {signum}")
+
+        def raise_signal(handle, watcher, revents, args):
+            os.kill(os.getpid(), signal.SIGUSR2)
+
+        #  Create new Flux handle to avoid potential stale state in
+        #  self.f handle. (makes test unreliable)
+        h = flux.Flux()
+        with h.signal_watcher_create(signal.SIGUSR2, signal_cb):
+            h.timer_watcher_create(0.05, raise_signal).start()
+            with self.assertRaises(RuntimeError):
+                rc = h.reactor_run()
+                self.assertTrue(rc < 0)
+
+
+class TestFdWatcher(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.f = flux.Flux()
+
+    def test_fd_watcher(self):
+        def fd_cb(handle, watcher, fd, revents, _args):
+            reader = os.fdopen(fd)
+            self.assertEqual(reader.read(), "a")
+            handle.reactor_stop()
+
+        def timer_cb(handle, watcher, revents, _args):
+            handle.reactor_stop_error()
+
+        tw = self.f.timer_watcher_create(5, timer_cb)
+        tw.start()
+
+        fdr, fdw = os.pipe()
+        os.set_blocking(fdr, False)
+        writer = os.fdopen(fdw, "w")
+        with self.f.fd_watcher_create(fdr, fd_cb) as fdw:
+            writer.write("a")
+            writer.flush()
+            rc = self.f.reactor_run()
+            self.assertTrue(rc >= 0)
+
+    def test_fd_watcher_exception(self):
+        def fd_cb(handle, watcher, fd, revents, _args):
+            raise RuntimeError
+
+        def timer_cb(handle, watcher, revents, _args):
+            handle.reactor_stop()
+
+        tw = self.f.timer_watcher_create(5, timer_cb)
+        tw.start()
+
+        fdr, fdw = os.pipe()
+        writer = os.fdopen(fdw, "w")
+        with self.f.fd_watcher_create(fdr, fd_cb) as fdw:
+            with self.assertRaises(RuntimeError):
+                writer.write("a")
+                writer.flush()
+                rc = self.f.reactor_run()
+                self.assertTrue(rc < 0)
+
+
+class TestIssue3973(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.f = flux.Flux()
+
+    def test_watcher_gc(self):
+        cb_called = 0
+
+        def cb(handle, watcher, revents, _args):
+            nonlocal cb_called
+            cb_called = cb_called + 1
+            if cb_called == 1:
+                gc.collect()
+            elif cb_called == 5:
+                handle.reactor_stop()
+
+        def cb_abort(handle, watcher, revents, _args):
+            raise RuntimeError("cb_abort should not be called")
+
+        #  Create a watcher with no local reference
+        self.f.timer_watcher_create(0.0, cb, repeat=0.05).start()
+
+        #  Timeout/abort after 5s
+        self.f.timer_watcher_create(5.0, cb_abort).start()
+        self.f.reactor_run()
+
+        #  callback should have been called 5 times, not less
+        self.assertTrue(cb_called == 5)
+
 
 if __name__ == "__main__":
     if rerun_under_flux(__flux_size()):
         from pycotap import TAPTestRunner
 
         unittest.main(testRunner=TAPTestRunner())
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/python/t0008-message.py b/t/python/t0008-message.py
new file mode 100755
index 000000000000..31a525d347a2
--- /dev/null
+++ b/t/python/t0008-message.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+
+###############################################################
+# Copyright 2024 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import json
+import unittest
+
+import flux
+from flux.message import Message
+from subflux import rerun_under_flux
+
+
+def __flux_size():
+    return 2
+
+
+class TestMessage(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.f = flux.Flux()
+        self.f.service_register("test").get()
+
+        self.msg_handler = self.f.msg_watcher_create(
+            lambda f, t, msg, arg: f.respond(msg),
+            flux.constants.FLUX_MSGTYPE_REQUEST,
+            "test.*",
+        )
+        self.msg_handler.start()
+
+    @classmethod
+    def tearDownClass(self):
+        self.f.service_unregister("test").get()
+        self.msg_handler.destroy()
+
+    def test_basic_request(self):
+        payload = {"test": "foo"}
+        m = Message()
+        self.assertIsNotNone(m)
+
+        # default type is request
+        self.assertEqual(m.type, flux.constants.FLUX_MSGTYPE_REQUEST)
+        self.assertEqual(m.type_str, "request")
+
+        m.topic = "test"
+        self.assertEqual(m.topic, "test")
+
+        m.payload_str = "blah"
+        self.assertEqual(m.payload_str, "blah")
+
+        m.payload = payload
+        self.assertEqual(m.payload, payload)
+        self.assertEqual(m.payload_str, json.dumps(payload))
+
+        msgtype, topic, payload_str = m.decode()
+        self.assertEqual(msgtype, flux.constants.FLUX_MSGTYPE_REQUEST)
+        self.assertEqual(topic, "test")
+        self.assertEqual(payload_str, json.dumps(payload))
+
+        # self.type setter works
+        m.type = flux.constants.FLUX_MSGTYPE_RESPONSE
+        self.assertEqual(m.type, flux.constants.FLUX_MSGTYPE_RESPONSE)
+
+    def test_basic_response(self):
+        m = Message(flux.constants.FLUX_MSGTYPE_RESPONSE)
+        self.assertIsNotNone(m)
+        self.assertEqual(m.type, flux.constants.FLUX_MSGTYPE_RESPONSE)
+        self.assertEqual(m.type_str, "response")
+
+        m.topic = "test"
+        self.assertEqual(m.topic, "test")
+
+        msgtype, topic, payload_str = m.decode()
+        self.assertEqual(msgtype, flux.constants.FLUX_MSGTYPE_RESPONSE)
+        self.assertEqual(topic, "test")
+        self.assertIsNone(payload_str)
+
+    def test_baseic_event(self):
+        m = Message(flux.constants.FLUX_MSGTYPE_EVENT)
+        self.assertIsNotNone(m)
+        self.assertEqual(m.type, flux.constants.FLUX_MSGTYPE_EVENT)
+        self.assertEqual(m.type_str, "event")
+
+        m.topic = "event.test"
+        self.assertEqual(m.topic, "event.test")
+
+        msgtype, topic, payload_str = m.decode()
+        self.assertEqual(msgtype, flux.constants.FLUX_MSGTYPE_EVENT)
+        self.assertEqual(topic, "event.test")
+        self.assertIsNone(payload_str)
+
+    def test_send(self):
+        payload = {"test": "foo"}
+        cb_called = [False]
+
+        def cb(handle, watcher, msg, called):
+            cb_called[0] = True
+            msgtype, topic, payload = msg.decode()
+            self.assertEqual(msgtype, flux.constants.FLUX_MSGTYPE_RESPONSE)
+            self.assertEqual(topic, "test.foo")
+            self.assertIsNone(payload)
+            handle.reactor_stop()
+
+        watcher = self.f.msg_watcher_create(
+            cb,
+            flux.constants.FLUX_MSGTYPE_RESPONSE,
+            "test.*",
+        )
+        self.assertIsNotNone(watcher)
+        watcher.start()
+
+        m = Message()
+        self.assertIsNotNone(m)
+
+        # default type is request
+        self.assertEqual(m.type, flux.constants.FLUX_MSGTYPE_REQUEST)
+        self.assertEqual(m.type_str, "request")
+
+        m.topic = "test.foo"
+        self.assertEqual(m.topic, "test.foo")
+        self.payload = payload
+        self.f.send(m)
+
+        rc = self.f.reactor_run()
+        self.assertTrue(rc > 0)
+        self.assertTrue(cb_called[0])
+
+        watcher.destroy()
+
+
+if __name__ == "__main__":
+    if rerun_under_flux(__flux_size()):
+        from pycotap import TAPTestRunner
+
+        unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t0009-security.py b/t/python/t0009-security.py
index be0b2714bc08..e4635206260f 100755
--- a/t/python/t0009-security.py
+++ b/t/python/t0009-security.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
@@ -11,10 +11,15 @@
 ###############################################################
 
 import os
+import sys
 import unittest
 from tempfile import NamedTemporaryFile
 
-from flux.security import SecurityContext
+try:
+    from flux.security import SecurityContext
+except ModuleNotFoundError:
+    print("1..0 # skip flux.security module not available")
+    sys.exit(0)
 
 
 def __flux_size():
@@ -36,7 +41,7 @@ def setUpClass(self):
             self.context = SecurityContext(tmpfile.name)
 
     def test_00_sign_none(self):
-        unsigned_str = u"hello world"
+        unsigned_str = "hello world"
         signed_str = self.context.sign_wrap(unsigned_str, mech_type="none")
         unwrapped_payload, wrapping_user = self.context.sign_unwrap(signed_str)
         unwrapped_str = unwrapped_payload[:].decode("utf-8")
@@ -45,7 +50,7 @@ def test_00_sign_none(self):
         self.assertEqual(wrapping_user, os.getuid())
 
     def test_01_security_error(self):
-        with self.assertRaisesRegexp(EnvironmentError, "sign-unwrap:.*"):
+        with self.assertRaisesRegex(EnvironmentError, "sign-unwrap:.*"):
             unwrapped_payload, wrapping_user = self.context.sign_unwrap(b"foo")
 
 
diff --git a/t/python/t0010-job.py b/t/python/t0010-job.py
index a2f8da5750f6..58cd87806f93 100755
--- a/t/python/t0010-job.py
+++ b/t/python/t0010-job.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
@@ -10,20 +10,24 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-import os
+import datetime
 import errno
-import sys
 import json
+import locale
+import os
+import pathlib
+import signal
+import subprocess
 import unittest
-import datetime
 from glob import glob
 
-import yaml
-import six
-
 import flux
+import flux.constants
+import flux.kvs
+import yaml
 from flux import job
-from flux.job import Jobspec, JobspecV1, ffi
+from flux.job import JobInfo, Jobspec, JobspecV1, ffi
+from flux.job.stats import JobStats
 
 
 def __flux_size():
@@ -31,14 +35,19 @@ def __flux_size():
 
 
 def yaml_to_json(s):
-    obj = yaml.load(s)
+    obj = yaml.safe_load(s)
     return json.dumps(obj, separators=(",", ":"))
 
 
 class TestJob(unittest.TestCase):
     @classmethod
     def setUpClass(self):
+        os.unsetenv("FLUX_F58_FORCE_ASCII")
         self.fh = flux.Flux()
+        self.use_ascii = False
+        build_opts = subprocess.check_output(["flux", "version"]).decode()
+        if locale.getlocale()[1] != "UTF-8" or "ascii-only" in build_opts:
+            self.use_ascii = True
 
         self.jobspec_dir = os.path.abspath(
             os.path.join(os.environ["FLUX_SOURCE_DIR"], "t", "jobspec")
@@ -78,7 +87,7 @@ def test_03_invalid_construction(self):
             ):
                 with self.assertRaises(
                     (ValueError, TypeError, yaml.scanner.ScannerError)
-                ) as cm:
+                ):
                     cls.from_yaml_file(invalid_jobspec_filepath)
 
     def test_04_valid_construction(self):
@@ -133,10 +142,6 @@ def test_10_invalid_duration(self):
                 jobspec.duration = duration
 
     def test_11_cwd_pathlib(self):
-        if six.PY2:
-            return
-        import pathlib
-
         jobspec_path = pathlib.PosixPath(self.jobspec_dir) / "valid" / "basic_v1.yaml"
         jobspec = Jobspec.from_yaml_file(jobspec_path)
         cwd = pathlib.PosixPath("/tmp")
@@ -149,6 +154,659 @@ def test_12_environment(self):
         jobspec.environment = new_env
         self.assertEqual(jobspec.environment, new_env)
 
+    def test_12_0_queue(self):
+        jobspec = Jobspec.from_yaml_stream(self.basic_jobspec)
+        jobspec.queue = "default"
+        self.assertEqual(jobspec.queue, "default")
+
+    def test_12_1_queue_invalid(self):
+        jobspec = Jobspec.from_yaml_stream(self.basic_jobspec)
+        with self.assertRaises(TypeError):
+            jobspec.queue = 12
+
+    def test_13_job_kvs(self):
+        jobid = job.submit(self.fh, self.basic_jobspec, waitable=True)
+        job.wait(self.fh, jobid=jobid)
+        for job_kvs_dir in [
+            job.job_kvs(self.fh, jobid),
+            job.job_kvs_guest(self.fh, jobid),
+        ]:
+            self.assertTrue(isinstance(job_kvs_dir, flux.kvs.KVSDir))
+            self.assertTrue(flux.kvs.exists(self.fh, job_kvs_dir.path))
+            self.assertTrue(flux.kvs.isdir(self.fh, job_kvs_dir.path))
+
+    def test_14_job_cancel_invalid_args(self):
+        with self.assertRaises(ValueError):
+            job.kill(self.fh, "abc")
+        with self.assertRaises(ValueError):
+            job.cancel(self.fh, "abc")
+        with self.assertRaises(OSError):
+            job.kill(self.fh, 123)
+        with self.assertRaises(OSError):
+            job.cancel(self.fh, 123)
+
+    def test_15_job_cancel(self):
+        self.sleep_jobspec = JobspecV1.from_command(["sleep", "1000"])
+        jobid = job.submit(self.fh, self.sleep_jobspec, waitable=True)
+        job.cancel(self.fh, jobid)
+        fut = job.wait_async(self.fh, jobid=jobid).wait_for(5.0)
+        return_id, success, errmsg = fut.get_status()
+        self.assertEqual(return_id, jobid)
+        self.assertFalse(success)
+
+    def test_16_job_kill(self):
+        self.sleep_jobspec = JobspecV1.from_command(["sleep", "1000"])
+        jobid = job.submit(self.fh, self.sleep_jobspec, waitable=True)
+
+        #  Wait for shell to fully start to avoid delay in signal
+        job.event_wait(self.fh, jobid, name="start")
+        job.event_wait(
+            self.fh, jobid, name="shell.start", eventlog="guest.exec.eventlog"
+        )
+        job.kill(self.fh, jobid, signum=signal.SIGKILL)
+        fut = job.wait_async(self.fh, jobid=jobid).wait_for(5.0)
+        return_id, success, errmsg = fut.get_status()
+        self.assertEqual(return_id, jobid)
+        self.assertFalse(success)
+
+    def test_20_000_job_event_functions_invalid_args(self):
+        with self.assertRaises(OSError) as cm:
+            for event in job.event_watch(self.fh, 123):
+                print(event)
+        self.assertEqual(cm.exception.errno, errno.ENOENT)
+        with self.assertRaises(OSError) as cm:
+            job.event_wait(self.fh, 123, "start")
+        self.assertEqual(cm.exception.errno, errno.ENOENT)
+        with self.assertRaises(OSError) as cm:
+            job.event_wait(None, 123, "start")
+        self.assertEqual(cm.exception.errno, errno.EINVAL)
+
+    def test_20_001_job_event_watch_async(self):
+        myarg = dict(a=1, b=2)
+        events = []
+
+        def cb(future, arg):
+            self.assertEqual(arg, myarg)
+            event = future.get_event()
+            if event is None:
+                future.get_flux().reactor_stop()
+                return
+            self.assertIsInstance(event, job.EventLogEvent)
+            events.append(event.name)
+
+        jobid = job.submit(self.fh, JobspecV1.from_command(["sleep", "0"]))
+        self.assertTrue(jobid > 0)
+        future = job.event_watch_async(self.fh, jobid)
+        self.assertIsInstance(future, job.JobEventWatchFuture)
+        future.then(cb, myarg)
+        rc = self.fh.reactor_run()
+        self.assertGreaterEqual(rc, 0)
+        self.assertEqual(len(events), 10)
+        self.assertEqual(events[0], "submit")
+        self.assertEqual(events[-1], "clean")
+
+    def test_20_002_job_event_watch_no_autoreset(self):
+        jobid = job.submit(self.fh, JobspecV1.from_command(["sleep", "0"]))
+        self.assertTrue(jobid > 0)
+        future = job.event_watch_async(self.fh, jobid)
+        self.assertIsInstance(future, job.JobEventWatchFuture)
+
+        # First event should be "submit"
+        event = future.get_event(autoreset=False)
+        self.assertIsInstance(event, job.EventLogEvent)
+        self.assertEqual(event.name, "submit")
+
+        # get_event() again with no reset returns same event:
+        event = future.get_event(autoreset=False)
+        self.assertIsInstance(event, job.EventLogEvent)
+        self.assertEqual(event.name, "submit")
+
+        # reset, then get_event() should get next event
+        future.reset()
+        event = future.get_event(autoreset=False)
+        self.assertIsInstance(event, job.EventLogEvent)
+        self.assertEqual(event.name, "validate")
+
+        future.cancel()
+
+    def test_20_003_job_event_watch_sync(self):
+        jobid = job.submit(self.fh, JobspecV1.from_command(["sleep", "0"]))
+        self.assertTrue(jobid > 0)
+        future = job.event_watch_async(self.fh, jobid)
+        self.assertIsInstance(future, job.JobEventWatchFuture)
+        event = future.get_event()
+        self.assertIsInstance(event, job.EventLogEvent)
+        self.assertEqual(event.name, "submit")
+        future.cancel()
+
+    def test_20_004_job_event_watch(self):
+        jobid = job.submit(self.fh, JobspecV1.from_command(["sleep", "0"]))
+        self.assertTrue(jobid > 0)
+        events = []
+        for event in job.event_watch(self.fh, jobid):
+            self.assertIsInstance(event, job.EventLogEvent)
+            self.assertTrue(hasattr(event, "timestamp"))
+            self.assertTrue(hasattr(event, "name"))
+            self.assertTrue(hasattr(event, "context"))
+            self.assertTrue(type(dict(event)), dict)
+            self.assertIs(type(event.timestamp), float)
+            self.assertIs(type(event.name), str)
+            self.assertIs(type(event.context), dict)
+            events.append(event.name)
+        self.assertEqual(len(events), 10)
+
+    def test_20_005_job_event_watch_with_cancel(self):
+        jobid = job.submit(
+            self.fh, JobspecV1.from_command(["sleep", "3"]), waitable=True
+        )
+        self.assertTrue(jobid > 0)
+        events = []
+        future = job.event_watch_async(self.fh, jobid)
+        while True:
+            event = future.get_event()
+            if event is None:
+                break
+            if event.name == "start":
+                future.cancel()
+            events.append(event.name)
+        self.assertEqual(event, None)
+        # Should have less than the expected number of events due to cancel
+        self.assertLess(len(events), 8)
+        job.cancel(self.fh, jobid)
+        job.wait(self.fh, jobid)
+
+    def test_20_005_1_job_event_watch_with_cancel_stop_true(self):
+        jobid = job.submit(
+            self.fh, JobspecV1.from_command(["sleep", "3"]), waitable=True
+        )
+        self.assertTrue(jobid > 0)
+        events = []
+        future = job.event_watch_async(self.fh, jobid)
+
+        def cb(future, events):
+            event = future.get_event()
+            if event.name == "start":
+                future.cancel(stop=True)
+            events.append(event.name)
+
+        future.then(cb, events)
+        self.fh.reactor_run()
+
+        # Last event should be "start"
+        self.assertEqual(events[-1], "start")
+        job.cancel(self.fh, jobid)
+        job.wait(self.fh, jobid)
+
+    def test_20_006_job_event_wait(self):
+        jobid = job.submit(self.fh, JobspecV1.from_command(["sleep", "0"]))
+        self.assertTrue(jobid > 0)
+        event = job.event_wait(self.fh, jobid, "start")
+        self.assertIsInstance(event, job.EventLogEvent)
+        self.assertEqual(event.name, "start")
+        event = job.event_wait(
+            self.fh, jobid, "shell.init", eventlog="guest.exec.eventlog"
+        )
+        self.assertIsInstance(event, job.EventLogEvent)
+        self.assertEqual(event.name, "shell.init")
+        event = job.event_wait(self.fh, jobid, "clean")
+        self.assertIsInstance(event, job.EventLogEvent)
+        self.assertEqual(event.name, "clean")
+        with self.assertRaises(OSError):
+            job.event_wait(self.fh, jobid, "foo")
+
+    def test_20_007_job_event_wait_exception(self):
+        event = None
+        jobid = job.submit(
+            self.fh, JobspecV1.from_command(["sleep", "0"], num_tasks=128)
+        )
+        self.assertTrue(jobid > 0)
+        try:
+            event = job.event_wait(self.fh, jobid, "start")
+        except job.JobException as err:
+            self.assertEqual(err.severity, 0)
+            self.assertEqual(err.type, "alloc")
+            self.assertGreater(err.timestamp, 0.0)
+        self.assertIs(event, None)
+        try:
+            event = job.event_wait(self.fh, jobid, "start", raiseJobException=False)
+        except OSError as err:
+            self.assertEqual(err.errno, errno.ENODATA)
+        self.assertIs(event, None)
+
+    def test_21_stdio(self):
+        """Test getter/setter methods for stdio properties"""
+        jobspec = Jobspec.from_yaml_stream(self.basic_jobspec)
+        for stream in ("stderr", "stdout", "stdin"):
+            self.assertEqual(getattr(jobspec, stream), None)
+            for path in ("foo.txt", "bar.md", "foo.json"):
+                setattr(jobspec, stream, path)
+                self.assertEqual(getattr(jobspec, stream), path)
+            with self.assertRaises(TypeError):
+                setattr(jobspec, stream, None)
+
+    def test_22_from_batch_command(self):
+        """Test that `from_batch_command` produces a valid jobspec"""
+        jobid = job.submit(
+            self.fh, JobspecV1.from_batch_command("#!/bin/sh\nsleep 0", "nested sleep")
+        )
+        self.assertGreater(jobid, 0)
+        # test that a shebang is required
+        with self.assertRaises(ValueError):
+            job.submit(
+                self.fh,
+                JobspecV1.from_batch_command("sleep 0", "nested sleep with no shebang"),
+            )
+
+    def test_23_from_nest_command(self):
+        """Test that `from_batch_command` produces a valid jobspec"""
+        jobid = job.submit(self.fh, JobspecV1.from_nest_command(["sleep", "0"]))
+        self.assertGreater(jobid, 0)
+
+    def test_24_jobid(self):
+        """Test JobID class"""
+        parse_tests = [
+            {
+                "int": 0,
+                "dec": "0",
+                "hex": "0x0",
+                "dothex": "0000.0000.0000.0000",
+                "kvs": "job.0000.0000.0000.0000",
+                "f58": "ƒ1",
+                "words": "academy-academy-academy--academy-academy-academy",
+                "emoji": "😃",
+            },
+            {
+                "int": 1,
+                "dec": "1",
+                "hex": "0x1",
+                "dothex": "0000.0000.0000.0001",
+                "kvs": "job.0000.0000.0000.0001",
+                "f58": "ƒ2",
+                "words": "acrobat-academy-academy--academy-academy-academy",
+                "emoji": "😄",
+            },
+            {
+                "int": 65535,
+                "dec": "65535",
+                "hex": "0xffff",
+                "dothex": "0000.0000.0000.ffff",
+                "kvs": "job.0000.0000.0000.ffff",
+                "f58": "ƒLUv",
+                "words": "nevada-archive-academy--academy-academy-academy",
+                "emoji": "💁📚",
+            },
+            {
+                "int": 6787342413402046,
+                "dec": "6787342413402046",
+                "hex": "0x181d0d4d850fbe",
+                "dothex": "0018.1d0d.4d85.0fbe",
+                "kvs": "job.0018.1d0d.4d85.0fbe",
+                "f58": "ƒuzzybunny",
+                "words": "cake-plume-nepal--neuron-pencil-academy",
+                "emoji": "👴😱🔚🎮🕙🚩",
+            },
+            {
+                "int": 18446744073709551614,
+                "dec": "18446744073709551614",
+                "hex": "0xfffffffffffffffe",
+                "dothex": "ffff.ffff.ffff.fffe",
+                "kvs": "job.ffff.ffff.ffff.fffe",
+                "f58": "ƒjpXCZedGfVP",
+                "words": "mustang-analyze-verbal--natural-analyze-verbal",
+                "emoji": "🚹💗💧👗😷📷📙",
+            },
+        ]
+        for test in parse_tests:
+            for key in test:
+                if key == "f58" and self.use_ascii:
+                    continue
+                jobid_int = test["int"]
+                jobid = job.JobID(test[key])
+                self.assertEqual(jobid, jobid_int)
+                jobid_repr = repr(jobid)
+                self.assertEqual(jobid_repr, f"JobID({jobid_int})")
+                if key not in ["int", "dec"]:
+                    # Ensure encode back to same type works
+                    self.assertEqual(getattr(jobid, key), test[key])
+
+        # JobID can also take a JobID
+        jobid1 = job.JobID(1234)
+        jobid2 = job.JobID(jobid1)
+        self.assertEqual(jobid1, jobid2)
+        self.assertEqual(jobid2.orig, "1234")
+
+    def test_25_job_list_attrs(self):
+        expected_attrs = [
+            "userid",
+            "urgency",
+            "priority",
+            "t_submit",
+            "t_depend",
+            "t_run",
+            "t_cleanup",
+            "t_inactive",
+            "state",
+            "name",
+            "cwd",
+            "queue",
+            "project",
+            "bank",
+            "ntasks",
+            "ncores",
+            "duration",
+            "nnodes",
+            "ranks",
+            "nodelist",
+            "success",
+            "exception_occurred",
+            "exception_type",
+            "exception_severity",
+            "exception_note",
+            "result",
+            "expiration",
+            "annotations",
+            "waitstatus",
+            "dependencies",
+            "all",
+        ]
+        valid_attrs = self.fh.rpc("job-list.list-attrs", "{}").get()["attrs"]
+        self.assertEqual(set(valid_attrs), set(expected_attrs))
+
+    def test_30_job_stats_sync(self):
+        stats = JobStats(self.fh)
+
+        # stats are uninitialized at first:
+        self.assertEqual(stats.active, -1)
+        self.assertEqual(stats.inactive, -1)
+
+        # synchronous update
+        stats.update_sync()
+        self.assertGreater(stats.inactive, 0)
+
+    def test_31_job_stats_async(self):
+        called = [False]
+
+        def cb(stats, mykw=None):
+            called[0] = True
+            self.assertGreater(stats.inactive, 0)
+            self.assertEqual(mykw, "mykw")
+
+        stats = JobStats(self.fh)
+
+        # stats are uninitialized at first:
+        self.assertEqual(stats.active, -1)
+        self.assertEqual(stats.inactive, -1)
+
+        # asynchronous update, no callback
+        stats.update()
+        self.assertEqual(stats.active, -1)
+        self.assertEqual(stats.inactive, -1)
+
+        self.fh.reactor_run()
+        self.assertGreater(stats.inactive, 0)
+        self.assertFalse(called[0])
+
+        # asynchronous update, with callback
+        stats.update(callback=cb, mykw="mykw")
+        self.fh.reactor_run()
+        self.assertTrue(called[0])
+
+    def assertJobInfoEqual(self, x, y, msg=None):
+
+        self.assertEqual(x.id, y.id)
+        self.assertEqual(x.result, y.result)
+        self.assertEqual(x.returncode, y.returncode)
+        self.assertEqual(x.waitstatus, y.waitstatus)
+        if y.t_run == 0.0:
+            self.assertEqual(x.runtime, y.runtime)
+        else:
+            self.assertGreater(x.runtime, 0.0)
+
+        self.assertEqual(x.exception.occurred, y.exception.occurred)
+
+        if y.exception.occurred:
+            self.assertEqual(x.exception.type, y.exception.type)
+            self.assertEqual(x.exception.severity, y.exception.severity)
+            if y.exception.note:
+                self.assertRegex(x.exception.note, y.exception.note)
+            else:
+                self.assertEqual(x.exception.note, y.exception.note)
+
+    def test_32_job_result(self):
+        result = {}
+        ids = []
+
+        def cb(future, jobid):
+            result[jobid] = future
+
+        ids.append(job.submit(self.fh, JobspecV1.from_command(["true"])))
+        ids.append(job.submit(self.fh, JobspecV1.from_command(["false"])))
+        ids.append(job.submit(self.fh, JobspecV1.from_command(["nosuchprog"])))
+        ids.append(job.submit(self.fh, JobspecV1.from_command(["sleep", "120"])))
+
+        # Submit held job so we can cancel before RUN state
+        ids.append(job.submit(self.fh, JobspecV1.from_command(["true"]), urgency=0))
+        job.cancel(self.fh, ids[4])
+
+        # Submit another held job so we can raise non-cancel exception
+        # before RUN:
+        ids.append(job.submit(self.fh, JobspecV1.from_command(["true"]), urgency=0))
+        self.fh.rpc(
+            "job-manager.raise", {"id": ids[5], "severity": 0, "type": "test"}
+        ).get()
+
+        for jobid in ids:
+            flux.job.result_async(self.fh, jobid).then(cb, jobid)
+
+        def cancel_on_start(future, jobid):
+            event = future.get_event()
+            if event is None:
+                return
+            if event.name == "shell.start":
+                job.cancel(self.fh, jobid)
+                future.cancel()
+
+        job.event_watch_async(self.fh, ids[3], eventlog="guest.exec.eventlog").then(
+            cancel_on_start, ids[3]
+        )
+
+        self.fh.reactor_run()
+        self.assertEqual(len(result.keys()), len(ids))
+
+        self.addTypeEqualityFunc(JobInfo, self.assertJobInfoEqual)
+
+        self.assertEqual(
+            result[ids[0]].get_info(),
+            JobInfo(
+                {
+                    "id": ids[0],
+                    "result": flux.constants.FLUX_JOB_RESULT_COMPLETED,
+                    "t_start": 1.0,
+                    "t_run": 2.0,
+                    "t_cleanup": 3.0,
+                    "waitstatus": 0,
+                    "exception_occurred": False,
+                }
+            ),
+        )
+
+        # t_remaining returns 0 from JobInfo returned from result():
+        self.assertEqual(result[ids[0]].get_info().t_remaining, 0.0)
+
+        # Ensure to_dict() works for a JobInfo returned from result():
+        job_dict = result[ids[0]].get_info().to_dict()
+        self.assertIsInstance(job_dict, dict)
+        self.assertEqual(job_dict["id"], ids[0])
+        self.assertEqual(job_dict["result"], "COMPLETED")
+        self.assertEqual(job_dict["returncode"], 0)
+        self.assertEqual(job_dict["duration"], 0.0)
+
+        self.assertEqual(
+            result[ids[1]].get_info(),
+            JobInfo(
+                {
+                    "id": ids[1],
+                    "result": flux.constants.FLUX_JOB_RESULT_FAILED,
+                    "t_submit": 1.0,
+                    "t_run": 2.0,
+                    "t_cleanup": 3.0,
+                    "waitstatus": 256,
+                    "exception_occurred": False,
+                }
+            ),
+        )
+        # Ensure to_dict() works for a JobInfo returned from result():
+        job_dict = result[ids[1]].get_info().to_dict()
+        self.assertIsInstance(job_dict, dict)
+        self.assertEqual(job_dict["id"], ids[1])
+        self.assertEqual(job_dict["result"], "FAILED")
+        self.assertEqual(job_dict["returncode"], 1)
+        self.assertEqual(job_dict["duration"], 0.0)
+
+        self.assertEqual(
+            result[ids[2]].get_info(),
+            JobInfo(
+                {
+                    "id": ids[2],
+                    "result": flux.constants.FLUX_JOB_RESULT_FAILED,
+                    "t_submit": 1.0,
+                    "t_run": 2.0,
+                    "t_cleanup": 3.0,
+                    "waitstatus": 32512,
+                    "exception_occurred": True,
+                    "exception_type": "exec",
+                    "exception_note": "task 0.*: start failed: nosuchprog: "
+                    "No such file or directory",
+                    "exception_severity": 0,
+                }
+            ),
+        )
+        self.assertEqual(
+            result[ids[3]].get_info(),
+            JobInfo(
+                {
+                    "id": ids[3],
+                    "result": flux.constants.FLUX_JOB_RESULT_CANCELED,
+                    "t_submit": 1.0,
+                    "t_run": 2.0,
+                    "t_cleanup": 3.0,
+                    "waitstatus": 36608,  # 143<<8
+                    "exception_occurred": True,
+                    "exception_type": "cancel",
+                    "exception_note": "",
+                    "exception_severity": 0,
+                }
+            ),
+        )
+        self.assertEqual(
+            result[ids[4]].get_info(),
+            JobInfo(
+                {
+                    "id": ids[4],
+                    "result": flux.constants.FLUX_JOB_RESULT_CANCELED,
+                    "t_submit": 0.0,
+                    "exception_occurred": True,
+                    "exception_type": "cancel",
+                    "exception_note": "",
+                    "exception_severity": 0,
+                }
+            ),
+        )
+        self.assertEqual(
+            result[ids[5]].get_info(),
+            JobInfo(
+                {
+                    "id": ids[5],
+                    "result": flux.constants.FLUX_JOB_RESULT_FAILED,
+                    "t_submit": 0.0,
+                    "exception_occurred": True,
+                    "exception_type": "test",
+                    "exception_note": "",
+                    "exception_severity": 0,
+                }
+            ),
+        )
+        # Explicitly check returncode for job that failed before start
+        # without a cancel exception. This is done since returncode is
+        # a derived attribute and we want to ensure it is explicitly 1:
+        self.assertEqual(result[ids[5]].get_info().returncode, 1)
+        # synchronous job.result() test
+        self.assertEqual(job.result(self.fh, ids[3]), result[ids[3]].get_info())
+
+    def test_33_get_job(self):
+        self.sleep_jobspec = JobspecV1.from_command(["sleep", "5"])
+        jobid = job.submit(self.fh, self.sleep_jobspec)
+        info = job.get_job(self.fh, jobid)
+        self.assertIsInstance(info, dict)
+        for key in [
+            "id",
+            "userid",
+            "urgency",
+            "priority",
+            "t_submit",
+            "t_depend",
+            "state",
+            "name",
+            "cwd",
+            "ntasks",
+            "ncores",
+            "duration",
+            "nnodes",
+            "result",
+            "runtime",
+            "returncode",
+            "waitstatus",
+            "nodelist",
+            "exception",
+        ]:
+            self.assertIn(key, info)
+
+        self.assertEqual(info["id"], jobid)
+        self.assertEqual(info["name"], "sleep")
+        self.assertTrue(info["state"] in ["SCHED", "DEPEND", "RUN"])
+        self.assertEqual(info["ntasks"], 1)
+        self.assertEqual(info["ncores"], 1)
+
+        # Test a job that does not exist
+        info = job.get_job(self.fh, 123456)
+        self.assertIsNone(info)
+
+    def test_34_timeleft(self):
+        spec = JobspecV1.from_command(
+            ["python3", "-c", "import flux; print(flux.job.timeleft())"]
+        )
+        spec.duration = "1m"
+        jobid = job.submit(self.fh, spec, waitable=True)
+        job.wait(self.fh, jobid=jobid)
+        try:
+            job.timeleft()
+            job.timeleft(self.fh)
+        except OSError:
+            pass
+
+    def test_35_setattr_defaults(self):
+        """Test setattr setting defaults"""
+        jobspec = Jobspec.from_yaml_stream(self.basic_jobspec)
+        jobspec.setattr("cow", 1)
+        jobspec.setattr("system.cat", 2)
+        jobspec.setattr("user.dog", 3)
+        jobspec.setattr("attributes.system.chicken", 4)
+        jobspec.setattr("attributes.user.duck", 5)
+        jobspec.setattr("attributes.goat", 6)
+        # N.B. setattr defaults "key" to "attributes.system.key"
+        # but getattr defaults "key" to "attributes.key"
+        self.assertEqual(jobspec.getattr("system.cow"), 1)
+        self.assertEqual(jobspec.getattr("attributes.system.cow"), 1)
+        self.assertEqual(jobspec.getattr("system.cat"), 2)
+        self.assertEqual(jobspec.getattr("attributes.system.cat"), 2)
+        self.assertEqual(jobspec.getattr("user.dog"), 3)
+        self.assertEqual(jobspec.getattr("attributes.user.dog"), 3)
+        self.assertEqual(jobspec.getattr("system.chicken"), 4)
+        self.assertEqual(jobspec.getattr("attributes.system.chicken"), 4)
+        self.assertEqual(jobspec.getattr("user.duck"), 5)
+        self.assertEqual(jobspec.getattr("attributes.user.duck"), 5)
+        self.assertEqual(jobspec.getattr("attributes.goat"), 6)
+
 
 if __name__ == "__main__":
     from subflux import rerun_under_flux
@@ -157,3 +815,6 @@ def test_12_environment(self):
         from pycotap import TAPTestRunner
 
         unittest.main(testRunner=TAPTestRunner())
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/python/t0012-futures.py b/t/python/t0012-futures.py
index 04a182b2016d..fcc399b0e945 100755
--- a/t/python/t0012-futures.py
+++ b/t/python/t0012-futures.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2019 Lawrence Livermore National Security, LLC
@@ -10,17 +10,14 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-from __future__ import print_function
-
-import gc
 import errno
+import gc
 import unittest
 
-import six.moves
 import flux
 import flux.constants
 from flux.core.inner import ffi
-from flux.future import Future
+from flux.future import Future, FutureExt
 from subflux import rerun_under_flux
 
 
@@ -35,21 +32,31 @@ def setUpClass(self):
         self.f = flux.Flux()
         self.ping_payload = {"seq": 1, "pad": "stuff"}
 
+    @staticmethod
+    def is_subset(x, y):
+        """Return true if all keys in x are present and equal to keys in y"""
+        for key, val in x.items():
+            if key not in y:
+                raise ValueError(f"key {key} missing in {y}")
+            if y[key] != val:
+                raise ValueError(f"key {key} is not {val} (got {x[key]})")
+        return True
+
     def test_01_rpc_get(self):
-        future = self.f.rpc("cmb.ping", self.ping_payload)
+        future = self.f.rpc("broker.ping", self.ping_payload)
         resp_payload = future.get()
-        self.assertDictContainsSubset(self.ping_payload, resp_payload)
+        self.assertTrue(self.is_subset(self.ping_payload, resp_payload))
 
     def test_02_get_flux(self):
-        future = self.f.rpc("cmb.ping", self.ping_payload)
+        future = self.f.rpc("broker.ping", self.ping_payload)
         future.get_flux()
         # force a full garbage collection pass to test that the handle is not destructed
         gc.collect(2)
         resp_payload = future.get()
-        self.assertDictContainsSubset(self.ping_payload, resp_payload)
+        self.assertTrue(self.is_subset(self.ping_payload, resp_payload))
 
     def test_02_future_wait_for(self):
-        future = self.f.rpc("cmb.ping", self.ping_payload)
+        future = self.f.rpc("broker.ping", self.ping_payload)
         try:
             future.wait_for(5)
             resp_payload = future.get()
@@ -58,7 +65,7 @@ def test_02_future_wait_for(self):
                 self.fail(msg="future fulfillment timed out")
             else:
                 raise
-        self.assertDictContainsSubset(self.ping_payload, resp_payload)
+        self.assertTrue(self.is_subset(self.ping_payload, resp_payload))
 
     def test_03_future_then(self):
         """Register a 'then' cb and run the reactor to ensure it runs"""
@@ -70,35 +77,111 @@ def then_cb(future, arg):
             try:
                 resp_payload = future.get()
                 cb_ran[0] = True
-                self.assertDictContainsSubset(arg, resp_payload)
+                self.assertTrue(self.is_subset(arg, resp_payload))
             finally:
                 # ensure that reactor is always stopped, avoiding a hung test
                 flux_handle.reactor_stop(reactor)
 
-        self.f.rpc(b"cmb.ping", self.ping_payload).then(then_cb, arg=self.ping_payload)
+        self.f.rpc(b"broker.ping", self.ping_payload).then(
+            then_cb, arg=self.ping_payload
+        )
         # force a full garbage collection pass to test that our anonymous RPC doesn't disappear
         gc.collect(2)
-        ret = self.f.reactor_run(self.f.get_reactor(), 0)
+        ret = self.f.reactor_run()
         self.assertGreaterEqual(ret, 0, msg="Reactor exited with {}".format(ret))
         self.assertTrue(cb_ran[0], msg="Callback did not run successfully")
 
+    def test_03_future_then_exception(self):
+        def then_cb(future):
+            raise RuntimeError("this is a test")
+
+        self.f.rpc("broker.ping", self.ping_payload).then(then_cb)
+        with self.assertRaises(RuntimeError):
+            self.f.reactor_run()
+
+    def test_03_future_then_varargs(self):
+        cb_ran = [False]
+
+        def then_cb(future, one, two, three):
+            cb_ran[0] = True
+            try:
+                self.assertEqual(one, "one")
+                self.assertEqual(two, "two")
+                self.assertEqual(three, "three")
+            finally:
+                future.get_flux().reactor_stop()
+
+        self.f.rpc("broker.ping").then(then_cb, "one", "two", "three")
+        gc.collect(2)
+        ret = self.f.reactor_run()
+        self.assertGreaterEqual(ret, 0, msg="Reactor exited with < 0")
+        self.assertTrue(cb_ran[0], msg="Callback did not run successfully")
+
+    def test_03_future_then_noargs(self):
+        cb_ran = [False]
+
+        def then_cb(future):
+            cb_ran[0] = True
+            future.get_flux().reactor_stop()
+
+        self.f.rpc("broker.ping").then(then_cb)
+        gc.collect(2)
+        ret = self.f.reactor_run()
+        self.assertGreaterEqual(ret, 0, msg="Reactor exited with < 0")
+        self.assertTrue(cb_ran[0], msg="Callback did not run successfully")
+
+    def test_03_future_then_default_args(self):
+        cb_ran = [False]
+
+        def then_cb(future, args=None):
+            cb_ran[0] = True
+            try:
+                self.assertIsNone(args)
+            finally:
+                future.get_flux().reactor_stop()
+
+        self.f.rpc("broker.ping").then(then_cb)
+        gc.collect(2)
+        ret = self.f.reactor_run()
+        self.assertGreaterEqual(ret, 0, msg="Reactor exited with < 0")
+        self.assertTrue(cb_ran[0], msg="Callback did not run successfully")
+
+    def test_03_future_then_kwargs(self):
+        cb_ran = [False]
+
+        def then_cb(future, val1=None, val2=None, val3="default"):
+            cb_ran[0] = True
+            try:
+                self.assertTrue(val1)
+                self.assertTrue(val2)
+                # val3 gets default value
+                self.assertEqual(val3, "default")
+            finally:
+                future.get_flux().reactor_stop()
+
+        self.f.rpc("broker.ping").then(then_cb, val2=True, val1=True)
+        gc.collect(2)
+        ret = self.f.reactor_run()
+        self.assertGreaterEqual(ret, 0, msg="Reactor exited with < 0")
+        self.assertTrue(cb_ran[0], msg="Callback did not run successfully")
+
     def test_04_double_future_then(self):
         """Register two 'then' cbs and ensure it throws an exception"""
         with self.assertRaises(EnvironmentError) as cm:
-            rpc = self.f.rpc(b"cmb.ping")
-            rpc.then(lambda x, y: None)
-            rpc.then(lambda x, y: None)
+            rpc = self.f.rpc(b"broker.ping")
+            rpc.then(lambda x: None)
+            rpc.then(lambda x: None)
         self.assertEqual(cm.exception.errno, errno.EEXIST)
 
     def test_05_future_error_string(self):
         with self.assertRaises(EnvironmentError) as cm:
-            payload = {"J": "", "priority": -1000, "flags": 0}
+            payload = {"J": "", "urgency": -1000, "flags": 0}
             future = self.f.rpc("job-ingest.submit", payload=payload)
             future.get()
         self.assertEqual(cm.exception.errno, errno.EINVAL)
         # Ensure that the result of flux_future_error_string propagated up
         self.assertEqual(cm.exception.strerror, future.error_string())
-        self.assertRegexpMatches(cm.exception.strerror, "priority range is .*")
+        self.assertRegex(cm.exception.strerror, "urgency range is .*")
 
     def test_06_blocking_methods(self):
         future = Future(self.f.future_create(ffi.NULL, ffi.NULL))
@@ -119,15 +202,14 @@ def test_07_streaming_rpcs(self):
         def continuation_cb(future, arg):
             arg["count"] += 1
             if arg["count"] >= arg["target"]:
-                self.f.reactor_stop(self.f.get_reactor())
+                self.f.reactor_stop()
             future.reset()
 
         def service_cb(fh, t, msg, arg):
-            for x in six.moves.range(msg.payload["count"]):
+            for x in range(msg.payload["count"]):
                 fh.respond(msg, {"seq": x})
 
-        fut = self.f.service_register("rpctest")
-        self.f.future_get(fut, ffi.NULL)
+        self.f.service_register("rpctest").get()
         watcher = self.f.msg_watcher_create(
             service_cb, flux.constants.FLUX_MSGTYPE_REQUEST, "rpctest.multi"
         )
@@ -135,10 +217,12 @@ def service_cb(fh, t, msg, arg):
         watcher.start()
 
         arg = {"count": 0, "target": 3}
-        self.f.rpc("rpctest.multi", {"count": arg["target"]}).then(
-            continuation_cb, arg=arg
-        )
-        ret = self.f.reactor_run(self.f.get_reactor(), 0)
+        self.f.rpc(
+            "rpctest.multi",
+            {"count": arg["target"]},
+            flags=flux.constants.FLUX_RPC_STREAMING,
+        ).then(continuation_cb, arg=arg)
+        self.f.reactor_run()
         self.assertEqual(arg["count"], arg["target"])
 
         watcher.stop()
@@ -146,6 +230,101 @@ def service_cb(fh, t, msg, arg):
         fut = self.f.service_unregister("rpctest")
         self.assertEqual(self.f.future_get(fut, ffi.NULL), 0)
 
+    def test_08_future_from_future(self):
+        orig_fut = Future(self.f.future_create(ffi.NULL, ffi.NULL))
+        new_fut = Future(orig_fut)
+        self.assertFalse(new_fut.is_ready())
+        orig_fut.pimpl.fulfill(ffi.NULL, ffi.NULL)
+        self.assertTrue(new_fut.is_ready())
+
+        orig_fut = self.f.rpc("broker.ping", payload=self.ping_payload)
+        new_fut = Future(orig_fut)
+        del orig_fut
+        new_fut.get()
+        # Future's `get` returns `None`, so just test that it is fulfilled
+        self.assertTrue(new_fut.is_ready())
+
+        orig_fut = self.f.rpc("foo.bar")
+        new_fut = Future(orig_fut)
+        del orig_fut
+        with self.assertRaises(EnvironmentError):
+            new_fut.get()
+
+    def test_20_FutureExt(self):
+        cb_ran = [False]
+
+        def init_cb(future):
+            cb_ran[0] = True
+            self.assertIsInstance(future, Future)
+            future.fulfill("foo")
+
+        future = FutureExt(init_cb)
+        self.assertIsInstance(future, FutureExt)
+        self.assertFalse(cb_ran[0])
+        self.assertEqual(future.get(), "foo")
+        self.assertTrue(cb_ran[0])
+
+        future.reset()
+        future.fulfill("bar")
+        self.assertEqual(future.get(), "bar")
+
+        future.reset()
+        future.fulfill({"key": "value"})
+        self.assertDictEqual(future.get(), {"key": "value"})
+
+        # fulfill with None
+        future.reset()
+        future.fulfill()
+        self.assertEqual(future.get(), None)
+
+        future.reset()
+        future.fulfill_error(errno.EINVAL, "test error")
+        with self.assertRaises(OSError):
+            future.get()
+
+    def test_21_FutureExt_complex(self):
+        class Ping5(FutureExt):
+            """Test future that returns after 5 pings"""
+
+            def __init__(self, flux_handle):
+                super().__init__(init_cb=self.init_cb, flux_handle=flux_handle)
+
+            def ping_cb(self, future, original_future):
+                seq = future.get()["seq"]
+                if seq == 5:
+                    try:
+                        original_future.fulfill(future.get())
+                    except Exception as exc:
+                        print(exc)
+                    return
+                h = future.get_flux()
+                h.rpc("broker.ping", {"seq": seq + 1}).then(
+                    self.ping_cb, original_future
+                )
+
+            def init_cb(self, future):
+                h = future.get_flux()
+                h.rpc("broker.ping", {"seq": 1}).then(self.ping_cb, future)
+
+        result = Ping5(self.f).get()
+        self.assertEqual(result["seq"], 5)
+
+        #  Now, try the same in async context:
+        cb_ran = [False]
+
+        def ping5_cb(future, a, b=None):
+            self.assertEqual(future.get()["seq"], 5)
+            self.assertEqual(a, "foo")
+            self.assertEqual(b, "bar")
+            cb_ran[0] = True
+            # XXX: reactor doesn't exit without this for unknown reason
+            # works standalone though...
+            future.get_flux().reactor_stop()
+
+        Ping5(self.f).then(ping5_cb, "foo", b="bar")
+        self.f.reactor_run()
+        self.assertTrue(cb_ran[0])
+
 
 if __name__ == "__main__":
     if rerun_under_flux(__flux_size()):
diff --git a/t/python/t0013-job-list-inactive.py b/t/python/t0013-job-list-inactive.py
deleted file mode 100755
index a5d8743c78fe..000000000000
--- a/t/python/t0013-job-list-inactive.py
+++ /dev/null
@@ -1,241 +0,0 @@
-#!/usr/bin/env python
-
-###############################################################
-# Copyright 2014 Lawrence Livermore National Security, LLC
-# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
-#
-# This file is part of the Flux resource manager framework.
-# For details, see https://github.com/flux-framework.
-#
-# SPDX-License-Identifier: LGPL-3.0
-###############################################################
-
-import os
-import errno
-import sys
-import json
-import unittest
-import datetime
-import time
-from glob import glob
-
-import yaml
-import six
-
-import flux
-from flux import job
-from flux.job import Jobspec, JobspecV1, ffi
-
-
-def __flux_size():
-    return 1
-
-
-class TestJob(unittest.TestCase):
-    @classmethod
-    def setUpClass(self):
-        self.fh = flux.Flux()
-
-    @classmethod
-    def submitJob(self):
-        compute_jobreq = JobspecV1.from_command(
-            command=["sleep", "0"], num_tasks=2, num_nodes=1, cores_per_task=1
-        )
-        compute_jobreq.cwd = os.getcwd()
-        compute_jobreq.environment = dict(os.environ)
-        flux.job.submit(self.fh, compute_jobreq, waitable=True)
-
-    @classmethod
-    def getJobs(self, rpc_handle):
-        try:
-            jobs = rpc_handle.get_jobs()
-            return jobs
-        except EnvironmentError as e:
-            print("{}: {}".format("rpc", e.strerror), file=sys.stderr)
-            sys.exit(1)
-
-    # NOTE: the job-info module has eventual consistency with the jobs stored
-    # in the job-manager's queue. To ensure no raciness in tests, we spin
-    # until all of the inactive jobs have reached INACTIVE state.
-    @classmethod
-    def waitForConsistency(self, jobs_list_length):
-        jobs = []
-        while True:
-            rpc_handle = flux.job.job_list(
-                self.fh, 0, self.attrs, states=flux.constants.FLUX_JOB_INACTIVE
-            )
-            jobs = self.getJobs(rpc_handle)
-            if len(jobs) == jobs_list_length:
-                break
-
-    attrs = [
-        "userid",
-        "state",
-        "name",
-        "ntasks",
-        "t_submit",
-        "t_run",
-        "t_inactive",
-    ]
-
-    # should return an empty list if there are no inactive jobs
-    def test_00_list_inactive_expect_empty_list(self):
-        rpc_handle = flux.job.job_list_inactive(
-            self.fh, time.time() - 3600, 10, self.attrs
-        )
-
-        jobs = self.getJobs(rpc_handle)
-
-        self.assertEqual(len(jobs), 0)
-
-    # make sure one job is read from RPC
-    def test_01_list_inactive_success(self):
-        # submit a sleep 0 job
-        self.submitJob()
-
-        self.waitForConsistency(1)
-
-        rpc_handle = flux.job.job_list_inactive(
-            self.fh, time.time() - 3600, 10, self.attrs
-        )
-
-        jobs = self.getJobs(rpc_handle)
-
-        self.assertEqual(len(jobs), 1)
-
-    # multiple jobs submitted should return a longer list of inactive jobs
-    def test_02_list_multiple_inactive(self):
-        # submit a bundle of sleep 0 jobs
-        for i in range(10):
-            jobid = self.submitJob()
-
-        self.waitForConsistency(10)
-
-        rpc_handle = flux.job.job_list_inactive(
-            self.fh, time.time() - 3600, 20, self.attrs
-        )
-
-        jobs = self.getJobs(rpc_handle)
-
-        self.assertEqual(len(jobs), 10)
-
-    # flux job list-inactive with since = 0.0 should return all inactive jobs
-    def test_03_list_all_inactive(self):
-        rpc_handle = flux.job.job_list_inactive(self.fh, 0.0, 20, self.attrs)
-
-        jobs = self.getJobs(rpc_handle)
-
-        self.assertEqual(len(jobs), 10)
-
-    # flux job list-inactive with max_entries = 5 should only return a subset
-    def test_04_list_subset_of_inactive(self):
-        rpc_handle = flux.job.job_list_inactive(self.fh, 0.0, 5, self.attrs)
-
-        jobs = self.getJobs(rpc_handle)
-
-        self.assertEqual(len(jobs), 5)
-
-    # flux job list-inactive with the most recent timestamp should return len(0)
-    def test_05_most_recent_inactive(self):
-        for i in range(5):
-            jobid = self.submitJob()
-
-        rpc_handle = flux.job.job_list(
-            self.fh, 1, ["t_inactive"], states=flux.constants.FLUX_JOB_INACTIVE
-        )
-
-        jobs = self.getJobs(rpc_handle)
-
-        rpc_handle = flux.job.job_list_inactive(
-            self.fh, jobs[0]["t_inactive"], 1, self.attrs
-        )
-
-        jobs_inactive = self.getJobs(rpc_handle)
-
-        self.assertEqual(len(jobs_inactive), 0)
-
-    # flux job list-inactive with second to most recent timestamp
-    def test_06_second_most_recent_timestamp(self):
-        for i in range(5):
-            jobid = self.submitJob()
-
-        rpc_handle = flux.job.job_list(
-            self.fh, 2, ["t_inactive"], states=flux.constants.FLUX_JOB_INACTIVE
-        )
-
-        jobs = self.getJobs(rpc_handle)
-
-        rpc_handle = flux.job.job_list_inactive(
-            self.fh, jobs[1]["t_inactive"], 1, self.attrs
-        )
-
-        jobs_inactive = self.getJobs(rpc_handle)
-
-        self.assertEqual(len(jobs_inactive), 1)
-        self.assertEqual(jobs_inactive[0]["t_inactive"], jobs[0]["t_inactive"])
-
-    # flux job list-inactive with oldest timestamp
-    def test_07_oldest_timestamp(self):
-        for i in range(5):
-            jobid = self.submitJob()
-
-        rpc_handle = flux.job.job_list(
-            self.fh, 5, ["t_inactive"], states=flux.constants.FLUX_JOB_INACTIVE
-        )
-
-        jobs = self.getJobs(rpc_handle)
-
-        rpc_handle = flux.job.job_list_inactive(
-            self.fh, jobs[4]["t_inactive"], 10, self.attrs
-        )
-
-        jobs_inactive = self.getJobs(rpc_handle)
-
-        self.assertEqual(len(jobs_inactive), 4)
-
-    # flux job list-inactive with middle timestamp #1
-    def test_08_middle_timestamp_1(self):
-        for i in range(11):
-            self.submitJob()
-
-        rpc_handle = flux.job.job_list(
-            self.fh, 20, ["t_inactive"], states=flux.constants.FLUX_JOB_INACTIVE
-        )
-
-        jobs = self.getJobs(rpc_handle)
-
-        rpc_handle = flux.job.job_list_inactive(
-            self.fh, jobs[5]["t_inactive"], 20, self.attrs
-        )
-
-        jobs_inactive = self.getJobs(rpc_handle)
-
-        self.assertEqual(len(jobs_inactive), 5)
-
-    # flux job list-inactive with middle timestamp #2
-    def test_09_middle_timestamp_2(self):
-        for i in range(11):
-            self.submitJob()
-
-        rpc_handle = flux.job.job_list(
-            self.fh, 20, ["t_inactive"], states=flux.constants.FLUX_JOB_INACTIVE
-        )
-
-        jobs = self.getJobs(rpc_handle)
-
-        rpc_handle = flux.job.job_list_inactive(
-            self.fh, jobs[7]["t_inactive"], 20, self.attrs
-        )
-
-        jobs_inactive = self.getJobs(rpc_handle)
-
-        self.assertEqual(len(jobs_inactive), 7)
-
-
-if __name__ == "__main__":
-    from subflux import rerun_under_flux
-
-    if rerun_under_flux(size=__flux_size(), personality="job"):
-        from pycotap import TAPTestRunner
-
-        unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t0013-job-list.py b/t/python/t0013-job-list.py
new file mode 100755
index 000000000000..849ba8d57fb1
--- /dev/null
+++ b/t/python/t0013-job-list.py
@@ -0,0 +1,342 @@
+#!/usr/bin/env python3
+
+###############################################################
+# Copyright 2014 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import os
+import sys
+import time
+import unittest
+
+import flux
+import subflux  # noqa: F401 - for PYTHONPATH
+from flux.job import JobspecV1
+
+
+def __flux_size():
+    return 1
+
+
+class TestJob(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.fh = flux.Flux()
+
+    @classmethod
+    def submitJob(self, command=None):
+        if not command:
+            command = ["sleep", "0"]
+        compute_jobreq = JobspecV1.from_command(
+            command=command, num_tasks=2, num_nodes=1, cores_per_task=1
+        )
+        compute_jobreq.cwd = os.getcwd()
+        compute_jobreq.environment = dict(os.environ)
+        return flux.job.submit(self.fh, compute_jobreq, waitable=True)
+
+    @classmethod
+    def getJobs(self, rpc_handle):
+        try:
+            jobs = rpc_handle.get_jobs()
+            return jobs
+        except EnvironmentError as e:
+            print("{}: {}".format("rpc", e.strerror), file=sys.stderr)
+            sys.exit(1)
+
+    # NOTE: the job-list module has eventual consistency with the jobs stored
+    # in the job-manager's queue. To ensure no raciness in tests, we spin
+    # until all of the inactive jobs have reached INACTIVE state.
+    @classmethod
+    def waitForConsistency(self, jobs_list_length):
+        jobs = []
+        waiting = 0  # number of seconds we have been waiting
+        while True:
+            rpc_handle = flux.job.job_list(
+                self.fh, 0, states=flux.constants.FLUX_JOB_STATE_INACTIVE
+            )
+            jobs = self.getJobs(rpc_handle)
+            if len(jobs) >= jobs_list_length:
+                break
+            time.sleep(1)
+            waiting += 1
+            if waiting > 60:
+                raise TimeoutError()
+
+    # flux job list should return an empty list if there are no jobs
+    def test_00_list_expect_empty_list(self):
+        rpc_handle = flux.job.job_list(self.fh)
+
+        jobs = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs), 0)
+
+    # flux job list-inactive should return an empty list if there are
+    # no inactive jobs
+    def test_01_list_inactive_expect_empty_list(self):
+        rpc_handle = flux.job.job_list_inactive(self.fh, time.time() - 3600, 10)
+
+        jobs = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs), 0)
+
+    core_attrs = {
+        "id",
+        "userid",
+        "urgency",
+        "priority",
+        "state",
+        "t_submit",
+        "t_run",
+        "t_inactive",
+        "name",
+        "ranks",
+    }
+
+    # flux job list make sure one job is read from RPC
+    def test_02_list_success(self):
+        # submit a sleep 0 job
+        self.submitJob()
+
+        self.waitForConsistency(1)
+
+        rpc_handle = flux.job.job_list(self.fh)
+
+        jobs = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs), 1)
+        # number of attributes should be atleast 10
+        self.assertGreater(len(jobs[0]), 10)
+        # make sure the core attributes are there
+        self.assertLessEqual(self.core_attrs, set(jobs[0].keys()))
+
+    # flux job list-inactive make sure one job is read from RPC
+    def test_03_list_inactive_success(self):
+        rpc_handle = flux.job.job_list_inactive(self.fh, time.time() - 3600, 10)
+
+        jobs = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs), 1)
+        # number of attributes should be atleast 10
+        self.assertGreater(len(jobs[0]), 10)
+        # make sure the core attributes are there
+        self.assertLessEqual(self.core_attrs, set(jobs[0].keys()))
+
+    # flux job list - alternate attrs requested
+    def test_04_list_success(self):
+        rpc_handle = flux.job.job_list(self.fh, attrs=["userid"])
+
+        jobs = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs), 1)
+        # returns id, userid only
+        self.assertEqual(len(jobs[0]), 2)
+        self.assertIn("id", jobs[0])
+        self.assertIn("userid", jobs[0])
+
+    # flux job list-inactive - alternate attrs requested
+    def test_05_list_inactive_success(self):
+        rpc_handle = flux.job.job_list_inactive(
+            self.fh, time.time() - 3600, 10, attrs=["userid"]
+        )
+
+        jobs = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs), 1)
+        # returns id, userid only
+        self.assertEqual(len(jobs[0]), 2)
+        self.assertIn("id", jobs[0])
+        self.assertIn("userid", jobs[0])
+
+    # flux job list multiple jobs submitted should return a
+    # longer list of inactive jobs
+    def test_06_list_multiple_inactive(self):
+        # submit a bundle of sleep 0 jobs
+        for i in range(10):
+            self.submitJob()
+
+        # 11 = 10 + 1 in previous tests
+        self.waitForConsistency(11)
+
+        rpc_handle = flux.job.job_list(self.fh)
+
+        jobs = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs), 11)
+
+    # flux job list-inactive multiple jobs submitted should return a
+    # longer list of inactive jobs
+    def test_07_list_inactive_multiple_inactive(self):
+        rpc_handle = flux.job.job_list_inactive(self.fh, time.time() - 3600, 20)
+
+        jobs = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs), 11)
+
+    # flux job list-inactive with since = 0.0 should return all inactive jobs
+    def test_08_list_inactive_all(self):
+        rpc_handle = flux.job.job_list_inactive(self.fh, 0.0, 20)
+
+        jobs = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs), 11)
+
+    # flux job list-inactive with max_entries = 5 should only return a subset
+    def test_09_list_inactive_subset_of_inactive(self):
+        rpc_handle = flux.job.job_list_inactive(self.fh, 0.0, 5)
+
+        jobs = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs), 5)
+
+    # flux job list-inactive with the most recent timestamp should return len(0)
+    def test_10_list_inactive_most_recent_inactive(self):
+        rpc_handle = flux.job.job_list(
+            self.fh, 1, ["t_inactive"], states=flux.constants.FLUX_JOB_STATE_INACTIVE
+        )
+
+        jobs = self.getJobs(rpc_handle)
+
+        rpc_handle = flux.job.job_list_inactive(self.fh, jobs[0]["t_inactive"], 1)
+
+        jobs_inactive = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs_inactive), 0)
+
+    # flux job list-inactive with second to most recent timestamp
+    def test_11_list_inactive_second_most_recent_timestamp(self):
+        rpc_handle = flux.job.job_list(
+            self.fh, 2, ["t_inactive"], states=flux.constants.FLUX_JOB_STATE_INACTIVE
+        )
+
+        jobs = self.getJobs(rpc_handle)
+
+        rpc_handle = flux.job.job_list_inactive(self.fh, jobs[1]["t_inactive"], 1)
+
+        jobs_inactive = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs_inactive), 1)
+        self.assertEqual(jobs_inactive[0]["t_inactive"], jobs[0]["t_inactive"])
+
+    # flux job list-inactive with oldest timestamp
+    def test_12_list_inactive_oldest_timestamp(self):
+        rpc_handle = flux.job.job_list(
+            self.fh, 5, ["t_inactive"], states=flux.constants.FLUX_JOB_STATE_INACTIVE
+        )
+
+        jobs = self.getJobs(rpc_handle)
+
+        rpc_handle = flux.job.job_list_inactive(self.fh, jobs[4]["t_inactive"], 10)
+
+        jobs_inactive = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs_inactive), 4)
+
+    # flux job list-inactive with middle timestamp #1
+    def test_13_list_inactive_middle_timestamp_1(self):
+        rpc_handle = flux.job.job_list(
+            self.fh, 20, ["t_inactive"], states=flux.constants.FLUX_JOB_STATE_INACTIVE
+        )
+
+        jobs = self.getJobs(rpc_handle)
+
+        rpc_handle = flux.job.job_list_inactive(self.fh, jobs[5]["t_inactive"], 20)
+
+        jobs_inactive = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs_inactive), 5)
+
+    # flux job list-inactive with middle timestamp #2
+    def test_14_list_inactive_middle_timestamp_2(self):
+        rpc_handle = flux.job.job_list(
+            self.fh, 20, ["t_inactive"], states=flux.constants.FLUX_JOB_STATE_INACTIVE
+        )
+
+        jobs = self.getJobs(rpc_handle)
+
+        rpc_handle = flux.job.job_list_inactive(self.fh, jobs[7]["t_inactive"], 20)
+
+        jobs_inactive = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs_inactive), 7)
+
+    # flux job list-inactive with name filter
+    def test_15_list_inactive_name_filter(self):
+        # submit a bundle of hostname jobs
+        for i in range(5):
+            self.submitJob(["hostname"])
+
+        # 16 = 5 + 11 in previous tests
+        self.waitForConsistency(16)
+
+        rpc_handle = flux.job.job_list_inactive(self.fh, 0.0, 20, name="sleep")
+
+        jobs_inactive = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs_inactive), 11)
+
+        rpc_handle = flux.job.job_list_inactive(self.fh, 0.0, 20, name="hostname")
+
+        jobs_inactive = self.getJobs(rpc_handle)
+
+        self.assertEqual(len(jobs_inactive), 5)
+
+    # flux job list-id works
+    def test_16_list_id(self):
+        jobid = self.submitJob(["hostname"])
+
+        self.waitForConsistency(17)
+
+        rpc_handle = flux.job.job_list_id(self.fh, jobid)
+
+        job = rpc_handle.get_job()
+        self.assertEqual(job["id"], jobid)
+        # number of attributes should be atleast 10
+        self.assertGreater(len(job), 10)
+
+    # flux job list-id works - alternate attrs requested
+    def test_17_list_id(self):
+        jobid = self.submitJob(["hostname"])
+
+        self.waitForConsistency(18)
+
+        rpc_handle = flux.job.job_list_id(self.fh, jobid, attrs=["userid"])
+
+        job = rpc_handle.get_job()
+        self.assertEqual(job["id"], jobid)
+        # returns id, userid only
+        self.assertEqual(len(job), 2)
+        self.assertIn("id", job)
+        self.assertIn("userid", job)
+
+    # flux job list-id fails on bad id
+    def test_18_list_id_fail(self):
+        rpc_handle = flux.job.job_list_id(self.fh, 123456789)
+        with self.assertRaises(FileNotFoundError):
+            rpc_handle.get_jobinfo()
+
+    def test_19_list_inactive_constraints(self):
+        for job in flux.job.job_list_inactive(
+            self.fh, constraint={"name": ["sleep"]}
+        ).get_jobinfos():
+            self.assertEqual(job.name, "sleep")
+
+    def test_20_list_constraints(self):
+        for job in flux.job.job_list(
+            self.fh, constraint={"name": ["sleep"]}
+        ).get_jobinfos():
+            self.assertEqual(job.name, "sleep")
+
+
+if __name__ == "__main__":
+    from subflux import rerun_under_flux
+
+    if rerun_under_flux(size=__flux_size(), personality="job"):
+        from pycotap import TAPTestRunner
+
+        unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t0014-job-kvslookup.py b/t/python/t0014-job-kvslookup.py
new file mode 100755
index 000000000000..29b7ed83f43c
--- /dev/null
+++ b/t/python/t0014-job-kvslookup.py
@@ -0,0 +1,447 @@
+#!/usr/bin/env python3
+
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import json
+import os
+import unittest
+
+import flux
+import subflux  # noqa: F401 - for PYTHONPATH
+from flux.job import JobspecV1
+
+
+def __flux_size():
+    return 1
+
+
+class TestJob(unittest.TestCase):
+    @classmethod
+    def submitJob(self, command, urgency):
+        compute_jobreq = JobspecV1.from_command(
+            command=command, num_tasks=1, num_nodes=1, cores_per_task=1
+        )
+        testenv = {"FOO": "BAR"}
+        compute_jobreq.environment = testenv
+        return flux.job.submit(self.fh, compute_jobreq, urgency=urgency, waitable=True)
+
+    @classmethod
+    def setUpClass(self):
+        self.fh = flux.Flux()
+
+        # in future use more standard update mechanism instead of
+        # jobtap plugin.  This jobtap plugin will simply increase the
+        # job expiration by 60 minutes.
+        pluginpath = (
+            os.environ["FLUX_BUILD_DIR"]
+            + "/t/job-manager/plugins/.libs/resource-update-expiration.so"
+        )
+        payload = {"load": pluginpath}
+        self.fh.rpc("job-manager.jobtap", payload).get()
+
+        self.jobid1 = self.submitJob(["hostname"], 0)
+        flux.job.event_wait(self.fh, self.jobid1, name="priority")
+        update = {"attributes.system.duration": 100.0}
+        payload = {"id": self.jobid1, "updates": update}
+        self.fh.rpc("job-manager.update", payload).get()
+        payload = {"id": self.jobid1, "urgency": 16}
+        self.fh.rpc("job-manager.urgency", payload).get()
+        flux.job.event_wait(self.fh, self.jobid1, name="clean")
+
+        payload = {"remove": "all"}
+        self.fh.rpc("job-manager.jobtap", payload).get()
+
+        self.jobid2 = self.submitJob(["hostname"], 16)
+        flux.job.event_wait(self.fh, self.jobid2, name="clean")
+
+    def check_jobspec_str(self, data, jobid, duration):
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("jobspec", data)
+        self.assertEqual(type(data["jobspec"]), str)
+        jobspec = json.loads(data["jobspec"])
+        self.assertEqual(jobspec["tasks"][0]["command"][0], "hostname")
+        self.assertEqual(jobspec["attributes"]["system"]["duration"], duration)
+        self.assertNotIn("R", data)
+
+    def check_jobspec_decoded(self, data, jobid, duration):
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("jobspec", data)
+        self.assertEqual(data["jobspec"]["tasks"][0]["command"][0], "hostname")
+        self.assertEqual(data["jobspec"]["attributes"]["system"]["duration"], duration)
+        self.assertNotIn("R", data)
+
+    def check_R_str(self, data, jobid):
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("R", data)
+        self.assertEqual(type(data["R"]), str)
+        R = json.loads(data["R"])
+        self.assertEqual(R["execution"]["R_lite"][0]["rank"], "0")
+
+    def check_R_decoded(self, data, jobid):
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("R", data)
+        self.assertEqual(data["R"]["execution"]["R_lite"][0]["rank"], "0")
+
+    def check_J_str(self, data, jobid):
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("J", data)
+        self.assertEqual(type(data["J"]), str)
+
+    def check_J_decoded(self, data, jobid):
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("J", data)
+        self.assertEqual(type(data["J"]), str)
+
+    def check_jobspec_original_str(self, data, jobid):
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("jobspec", data)
+        self.assertEqual(type(data["jobspec"]), str)
+        jobspec = json.loads(data["jobspec"])
+        self.assertEqual(jobspec["tasks"][0]["command"][0], "hostname")
+        self.assertEqual(jobspec["attributes"]["system"]["duration"], 0)
+        self.assertEqual(jobspec["attributes"]["system"]["environment"]["FOO"], "BAR")
+
+    def check_jobspec_original_decoded(self, data, jobid):
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("jobspec", data)
+        self.assertEqual(data["jobspec"]["tasks"][0]["command"][0], "hostname")
+        self.assertEqual(data["jobspec"]["attributes"]["system"]["duration"], 0)
+        self.assertEqual(
+            data["jobspec"]["attributes"]["system"]["environment"]["FOO"], "BAR"
+        )
+
+    def check_jobspec_base_str(self, data, jobid):
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("jobspec", data)
+        self.assertEqual(type(data["jobspec"]), str)
+        jobspec = json.loads(data["jobspec"])
+        self.assertEqual(jobspec["tasks"][0]["command"][0], "hostname")
+        self.assertEqual(jobspec["attributes"]["system"]["duration"], 0)
+
+    def check_jobspec_base_decoded(self, data, jobid):
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("jobspec", data)
+        self.assertEqual(data["jobspec"]["tasks"][0]["command"][0], "hostname")
+        self.assertEqual(data["jobspec"]["attributes"]["system"]["duration"], 0)
+
+    def check_R_base_str(self, jobid, base, data):
+        self.assertEqual(base["id"], jobid)
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("R", base)
+        self.assertIn("R", data)
+        self.assertEqual(type(base["R"]), str)
+        self.assertEqual(type(data["R"]), str)
+        R_base = json.loads(base["R"])
+        R_data = json.loads(data["R"])
+        base_expiration = R_base["execution"]["expiration"]
+        data_expiration = R_data["execution"]["expiration"]
+        self.assertGreater(data_expiration, base_expiration)
+
+    def check_R_base_decoded(self, jobid, base, data):
+        self.assertEqual(base["id"], jobid)
+        self.assertEqual(data["id"], jobid)
+        self.assertIn("R", base)
+        self.assertIn("R", data)
+        base_expiration = base["R"]["execution"]["expiration"]
+        data_expiration = data["R"]["execution"]["expiration"]
+        self.assertGreater(data_expiration, base_expiration)
+
+    def test_info_00_job_info_lookup(self):
+        rpc = flux.job.job_info_lookup(self.fh, self.jobid1)
+        data = rpc.get()
+        self.check_jobspec_str(data, self.jobid1, 0)
+        data = rpc.get_decode()
+        self.check_jobspec_decoded(data, self.jobid1, 0)
+        self.assertEqual(data["id"], self.jobid1, 0)
+
+    def test_info_01_job_info_lookup_keys(self):
+        rpc = flux.job.job_info_lookup(self.fh, self.jobid1, keys=["R", "J"])
+        data = rpc.get()
+        self.assertNotIn("jobspec", data)
+        self.check_R_str(data, self.jobid1)
+        self.check_J_str(data, self.jobid1)
+        data = rpc.get_decode()
+        self.check_R_decoded(data, self.jobid1)
+        self.check_J_decoded(data, self.jobid1)
+
+    def test_info_02_job_info_lookup_badid(self):
+        rpc = flux.job.job_info_lookup(self.fh, 123456789)
+        with self.assertRaises(FileNotFoundError):
+            rpc.get()
+
+    def test_info_03_job_info_lookup_badkey(self):
+        rpc = flux.job.job_info_lookup(self.fh, self.jobid1, keys=["foo"])
+        with self.assertRaises(FileNotFoundError):
+            rpc.get()
+
+    def test_lookup_01_job_kvs_lookup(self):
+        data = flux.job.job_kvs_lookup(self.fh, self.jobid1)
+        self.check_jobspec_decoded(data, self.jobid1, 100.0)
+
+    def test_lookup_02_job_kvs_lookup_nodecode(self):
+        data = flux.job.job_kvs_lookup(self.fh, self.jobid1, decode=False)
+        self.check_jobspec_str(data, self.jobid1, 100.0)
+
+    def test_lookup_03_job_kvs_lookup_keys(self):
+        data = flux.job.job_kvs_lookup(self.fh, self.jobid1, keys=["R", "J"])
+        self.assertNotIn("jobspec", data)
+        self.check_R_decoded(data, self.jobid1)
+        self.check_J_decoded(data, self.jobid1)
+
+    def test_lookup_04_job_kvs_lookup_keys_nodecode(self):
+        data = flux.job.job_kvs_lookup(
+            self.fh, self.jobid1, keys=["R", "J"], decode=False
+        )
+        self.assertNotIn("jobspec", data)
+        self.check_R_str(data, self.jobid1)
+        self.check_J_str(data, self.jobid1)
+
+    def test_lookup_05_job_kvs_lookup_badid(self):
+        data = flux.job.job_kvs_lookup(self.fh, 123456789)
+        self.assertEqual(data, None)
+
+    def test_lookup_06_job_kvs_lookup_badkey(self):
+        data = flux.job.job_kvs_lookup(self.fh, self.jobid1, keys=["foo"])
+        self.assertEqual(data, None)
+
+    def test_lookup_07_job_kvs_lookup_jobspec_original(self):
+        data = flux.job.job_kvs_lookup(self.fh, self.jobid1, original=True)
+        self.assertNotIn("J", data)
+        self.check_jobspec_original_decoded(data, self.jobid1)
+
+    def test_lookup_08_job_kvs_lookup_jobspec_original_nodecode(self):
+        data = flux.job.job_kvs_lookup(
+            self.fh, self.jobid1, decode=False, original=True
+        )
+        self.assertNotIn("J", data)
+        self.check_jobspec_original_str(data, self.jobid1)
+
+    def test_lookup_09_job_kvs_lookup_jobspec_original_multiple_keys(self):
+        data = flux.job.job_kvs_lookup(
+            self.fh, self.jobid1, keys=["jobspec", "J"], original=True
+        )
+        self.assertIn("J", data)
+        self.check_jobspec_original_decoded(data, self.jobid1)
+
+    def test_lookup_10_job_kvs_lookup_original_no_jobspec(self):
+        data = flux.job.job_kvs_lookup(
+            self.fh, self.jobid1, keys=["R", "J"], original=True
+        )
+        self.assertNotIn("jobspec", data)
+        self.check_R_decoded(data, self.jobid1)
+        self.check_J_decoded(data, self.jobid1)
+
+    def test_lookup_11_job_kvs_lookup_jobspec_base(self):
+        data = flux.job.job_kvs_lookup(self.fh, self.jobid1, base=True)
+        self.assertNotIn("eventlog", data)
+        self.check_jobspec_base_decoded(data, self.jobid1)
+
+    def test_lookup_12_job_kvs_lookup_jobspec_base_nodecode(self):
+        data = flux.job.job_kvs_lookup(self.fh, self.jobid1, decode=False, base=True)
+        self.assertNotIn("eventlog", data)
+        self.check_jobspec_base_str(data, self.jobid1)
+
+    def test_lookup_13_job_kvs_lookup_jobspec_base_multiple_keys(self):
+        data = flux.job.job_kvs_lookup(
+            self.fh, self.jobid1, keys=["jobspec", "eventlog"], base=True
+        )
+        self.assertIn("eventlog", data)
+        self.check_jobspec_base_decoded(data, self.jobid1)
+
+    def test_lookup_14_job_kvs_lookup_R_base(self):
+        base = flux.job.job_kvs_lookup(self.fh, self.jobid1, keys=["R"], base=True)
+        data = flux.job.job_kvs_lookup(self.fh, self.jobid1, keys=["R"])
+        self.assertNotIn("eventlog", base)
+        self.assertNotIn("eventlog", data)
+        self.check_R_base_decoded(self.jobid1, base, data)
+
+    def test_lookup_15_job_kvs_lookup_R_base_nodecode(self):
+        base = flux.job.job_kvs_lookup(
+            self.fh, self.jobid1, keys=["R"], decode=False, base=True
+        )
+        data = flux.job.job_kvs_lookup(self.fh, self.jobid1, keys=["R"], decode=False)
+        self.assertNotIn("eventlog", base)
+        self.assertNotIn("eventlog", data)
+        self.check_R_base_str(self.jobid1, base, data)
+
+    def test_lookup_16_job_kvs_lookup_R_base_multiple_keys(self):
+        base = flux.job.job_kvs_lookup(
+            self.fh, self.jobid1, keys=["R", "eventlog"], decode=False, base=True
+        )
+        data = flux.job.job_kvs_lookup(
+            self.fh, self.jobid1, keys=["R", "eventlog"], decode=False
+        )
+        self.assertIn("eventlog", base)
+        self.assertIn("eventlog", data)
+        self.check_R_base_str(self.jobid1, base, data)
+
+    def test_lookup_17_job_kvs_lookup_base_no_jobspec_R(self):
+        data = flux.job.job_kvs_lookup(self.fh, self.jobid1, keys=["J"], base=True)
+        self.assertNotIn("jobspec", data)
+        self.assertNotIn("R", data)
+        self.check_J_decoded(data, self.jobid1)
+
+    def test_list_00_job_kvs_lookup_list(self):
+        ids = [self.jobid1]
+        data = flux.job.JobKVSLookup(self.fh, ids).data()
+        self.assertEqual(len(data), 1)
+        self.check_jobspec_decoded(data[0], self.jobid1, 100.0)
+
+    def test_list_01_job_kvs_lookup_list_multiple(self):
+        ids = [self.jobid1, self.jobid2]
+        data = flux.job.JobKVSLookup(self.fh, ids).data()
+        self.assertEqual(len(data), 2)
+        self.check_jobspec_decoded(data[0], self.jobid1, 100.0)
+        self.check_jobspec_decoded(data[1], self.jobid2, 0)
+
+    def test_list_02_job_kvs_lookup_list_multiple_nodecode(self):
+        ids = [self.jobid1, self.jobid2]
+        data = flux.job.JobKVSLookup(self.fh, ids, decode=False).data()
+        self.assertEqual(len(data), 2)
+        self.check_jobspec_str(data[0], self.jobid1, 100.0)
+        self.check_jobspec_str(data[1], self.jobid2, 0)
+
+    def test_list_03_job_kvs_lookup_list_multiple_keys(self):
+        ids = [self.jobid1, self.jobid2]
+        data = flux.job.JobKVSLookup(self.fh, ids, keys=["R", "J"]).data()
+        self.assertNotIn("jobspec", data)
+        self.assertEqual(len(data), 2)
+        self.check_R_decoded(data[0], self.jobid1)
+        self.check_J_decoded(data[0], self.jobid1)
+        self.check_R_decoded(data[1], self.jobid2)
+        self.check_J_decoded(data[1], self.jobid2)
+
+    def test_list_04_job_kvs_lookup_list_multiple_keys_nodecode(self):
+        ids = [self.jobid1, self.jobid2]
+        data = flux.job.JobKVSLookup(self.fh, ids, keys=["R", "J"], decode=False).data()
+        self.assertNotIn("jobspec", data)
+        self.assertEqual(len(data), 2)
+        self.check_R_str(data[0], self.jobid1)
+        self.check_J_str(data[0], self.jobid1)
+        self.check_R_str(data[1], self.jobid2)
+        self.check_J_str(data[1], self.jobid2)
+
+    def test_list_05_job_kvs_lookup_list_none(self):
+        data = flux.job.JobKVSLookup(self.fh).data()
+        self.assertEqual(len(data), 0)
+
+    def test_list_06_job_kvs_lookup_list_badid(self):
+        ids = [123456789]
+        datalookup = flux.job.JobKVSLookup(self.fh, ids)
+        data = datalookup.data()
+        self.assertEqual(len(data), 0)
+        self.assertEqual(len(datalookup.errors), 1)
+
+    def test_list_07_job_kvs_lookup_list_badkey(self):
+        ids = [self.jobid1]
+        datalookup = flux.job.JobKVSLookup(self.fh, ids, keys=["foo"])
+        data = datalookup.data()
+        self.assertEqual(len(data), 0)
+        self.assertEqual(len(datalookup.errors), 1)
+
+    def test_list_08_job_kvs_lookup_list_jobspec_original(self):
+        ids = [self.jobid1]
+        data = flux.job.JobKVSLookup(self.fh, ids, original=True).data()
+        self.assertEqual(len(data), 1)
+        self.assertNotIn("J", data[0])
+        self.check_jobspec_original_decoded(data[0], self.jobid1)
+
+    def test_list_09_job_kvs_lookup_list_jobspec_original_nodecode(self):
+        ids = [self.jobid1]
+        data = flux.job.JobKVSLookup(self.fh, ids, decode=False, original=True).data()
+        self.assertEqual(len(data), 1)
+        self.assertNotIn("J", data[0])
+        self.check_jobspec_original_str(data[0], self.jobid1)
+
+    def test_list_10_job_kvs_lookup_list_jobspec_original_multiple_keys(self):
+        ids = [self.jobid1]
+        data = flux.job.JobKVSLookup(
+            self.fh, ids, keys=["jobspec", "J"], original=True
+        ).data()
+        self.assertEqual(len(data), 1)
+        self.assertIn("J", data[0])
+        self.check_jobspec_original_decoded(data[0], self.jobid1)
+
+    def test_list_11_job_kvs_lookup_list_original_no_jobspec(self):
+        ids = [self.jobid1]
+        data = flux.job.JobKVSLookup(
+            self.fh, ids, keys=["R", "J"], original=True
+        ).data()
+        self.assertEqual(len(data), 1)
+        self.assertNotIn("jobspec", data[0])
+        self.check_R_decoded(data[0], self.jobid1)
+        self.check_J_decoded(data[0], self.jobid1)
+
+    def test_list_12_job_kvs_lookup_list_jobspec_base(self):
+        ids = [self.jobid1]
+        data = flux.job.JobKVSLookup(self.fh, ids, base=True).data()
+        self.assertEqual(len(data), 1)
+        self.assertNotIn("J", data[0])
+        self.check_jobspec_base_decoded(data[0], self.jobid1)
+
+    def test_list_13_job_kvs_lookup_list_jobspec_base_nodecode(self):
+        ids = [self.jobid1]
+        data = flux.job.JobKVSLookup(self.fh, ids, decode=False, base=True).data()
+        self.assertEqual(len(data), 1)
+        self.assertNotIn("J", data[0])
+        self.check_jobspec_base_str(data[0], self.jobid1)
+
+    def test_list_14_job_kvs_lookup_list_jobspec_base_multiple_keys(self):
+        ids = [self.jobid1]
+        data = flux.job.JobKVSLookup(
+            self.fh, ids, keys=["jobspec", "J"], base=True
+        ).data()
+        self.assertEqual(len(data), 1)
+        self.assertIn("J", data[0])
+        self.check_jobspec_base_decoded(data[0], self.jobid1)
+
+    def test_list_15_job_kvs_lookup_list_R_base(self):
+        ids = [self.jobid1]
+        base = flux.job.JobKVSLookup(self.fh, ids, keys=["R"], base=True).data()
+        data = flux.job.JobKVSLookup(self.fh, ids, keys=["R"]).data()
+        self.assertEqual(len(base), 1)
+        self.assertEqual(len(data), 1)
+        self.check_R_base_decoded(self.jobid1, base[0], data[0])
+
+    def test_list_16_job_kvs_lookup_list_R_base_nodecode(self):
+        ids = [self.jobid1]
+        base = flux.job.JobKVSLookup(
+            self.fh, ids, keys=["R"], decode=False, base=True
+        ).data()
+        data = flux.job.JobKVSLookup(self.fh, ids, keys=["R"], decode=False).data()
+        self.assertEqual(len(base), 1)
+        self.assertEqual(len(data), 1)
+        self.check_R_base_str(self.jobid1, base[0], data[0])
+
+    def test_list_17_job_kvs_lookup_list_R_base_multiple_keys(self):
+        ids = [self.jobid1]
+        base = flux.job.JobKVSLookup(self.fh, ids, keys=["R", "J"], base=True).data()
+        data = flux.job.JobKVSLookup(self.fh, ids, keys=["R", "J"]).data()
+        self.assertEqual(len(data), 1)
+        self.assertIn("J", data[0])
+        self.check_R_base_decoded(self.jobid1, base[0], data[0])
+
+    def test_list_18_job_kvs_lookup_list_base_no_jobspec_R(self):
+        ids = [self.jobid1]
+        data = flux.job.JobKVSLookup(self.fh, ids, keys=["J"], base=True).data()
+        self.assertEqual(len(data), 1)
+        self.assertNotIn("jobspec", data[0])
+        self.assertNotIn("R", data[0])
+        self.check_J_decoded(data[0], self.jobid1)
+
+
+if __name__ == "__main__":
+    from subflux import rerun_under_flux
+
+    if rerun_under_flux(size=__flux_size(), personality="job"):
+        from pycotap import TAPTestRunner
+
+        unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t0015-job-output.py b/t/python/t0015-job-output.py
new file mode 100755
index 000000000000..c84ec909adb9
--- /dev/null
+++ b/t/python/t0015-job-output.py
@@ -0,0 +1,600 @@
+#!/usr/bin/python3
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import concurrent.futures
+import errno
+import os
+import unittest
+
+import flux
+import subflux  # noqa: F401 - for PYTHONPATH
+from flux.constants import FLUX_JOB_URGENCY_DEFAULT, FLUX_JOB_URGENCY_HOLD
+from flux.job import (
+    JobspecV1,
+    event_wait,
+    job_output,
+    output_event_watch,
+    output_event_watch_async,
+    output_watch,
+    output_watch_async,
+    output_watch_lines,
+    output_watch_lines_async,
+)
+from flux.job.output import LOG_QUIET, LOG_TRACE
+
+
+def __flux_size():
+    return 2
+
+
+class TestJobOutput(unittest.TestCase):
+
+    test_stdout = "line 1\nline 2\nline 3\n"
+    test_stderr = "error 1\nerror 2\n"
+
+    def submit(
+        self,
+        output=None,
+        error=None,
+        hold=False,
+        redirect=None,
+        verbose=False,
+        ntasks=1,
+        cmd=None,
+    ):
+        if output is None:
+            output = self.test_stdout
+        if error is None:
+            error = self.test_stderr
+        if cmd is None:
+            cmd = f"printf '{output}'; printf '{error}' >&2"
+        command = ["bash", "-c", cmd]
+        jobspec = JobspecV1.from_command(
+            command=command, num_tasks=ntasks, num_nodes=1, cores_per_task=1
+        )
+        urgency = FLUX_JOB_URGENCY_DEFAULT
+        jobspec.setattr_shell_option("cpu-affinity", "off")
+        jobspec.environment = dict(os.environ)
+        if hold:
+            urgency = FLUX_JOB_URGENCY_HOLD
+        if verbose:
+            jobspec.setattr_shell_option("verbose", 1)
+        if redirect is not None:
+            jobspec.stdout = redirect
+        return flux.job.submit(self.fh, jobspec, waitable=True, urgency=urgency)
+
+    def release_job(self, jobid):
+        # Release job by setting urgency to default:
+        self.fh.rpc(
+            "job-manager.urgency",
+            {"id": int(jobid), "urgency": FLUX_JOB_URGENCY_DEFAULT},
+        ).get()
+
+    @classmethod
+    def setUpClass(self):
+        self.fh = flux.Flux()
+        self.executor = concurrent.futures.ThreadPoolExecutor(1)
+
+    def test_output_invalid_args(self):
+        with self.assertRaises(FileNotFoundError):
+            job_output(self.fh, 123)
+        with self.assertRaises(FileNotFoundError):
+            job_output(self.fh, 123, nowait=True)
+        with self.assertRaises(FileNotFoundError):
+            for event in output_event_watch(self.fh, 123):
+                print(event)
+
+    def test_job_output(self):
+        jobid = self.submit()
+        output = job_output(self.fh, jobid)
+        self.assertEqual(output.stdout, self.test_stdout)
+        self.assertEqual(output.stderr, self.test_stderr)
+        self.assertEqual(output.log, "")
+
+    def test_job_output_labelio(self):
+        jobid = self.submit(ntasks=2)
+        output = job_output(self.fh, jobid, labelio=True)
+        self.assertIn("1: line 1", output.stdout)
+        self.assertIn("1: error 1", output.stderr)
+
+    def test_job_output_labelio_task_filter(self):
+        jobid = self.submit(ntasks=2)
+        output = job_output(self.fh, jobid, tasks="1", labelio=True)
+        self.assertIn("1: line 1", output.stdout)
+        self.assertIn("1: error 1", output.stderr)
+        self.assertNotIn("0: line 1", output.stdout)
+        self.assertNotIn("0: error 1", output.stderr)
+
+    def test_job_output_with_logs(self):
+        jobid = self.submit(verbose=True)
+        output = job_output(self.fh, jobid)
+        self.assertEqual(output.stdout, self.test_stdout)
+        # By default log messages are folded in with stderr:
+        self.assertIn("error 1", output.stderr)
+        self.assertIn("flux-shell[0]", output.stderr)
+        self.assertEqual(output.log, "")
+
+        # Now, set log_stderr_level=-1.
+        # output.log should contain all log messages
+        output = job_output(self.fh, jobid, log_stderr_level=-1)
+        self.assertEqual(output.stdout, self.test_stdout)
+        self.assertEqual(output.stderr, self.test_stderr)
+        self.assertTrue(len(output.log) > 0)
+
+    def test_job_output_nowait(self):
+        jobid = self.submit(cmd="echo first; sleep 100; echo second")
+        # wait for output to be ready:
+        event_wait(self.fh, jobid, "shell.init", "guest.exec.eventlog")
+        output = job_output(self.fh, jobid, nowait=True)
+        self.assertNotIn("second", output.stdout)
+        flux.job.cancel(self.fh, jobid)
+
+    def test_pending_job_output(self):
+        jobid = self.submit(hold=True)
+
+        def get_output():
+            # This function will be run in another thread, so use a new
+            # Flux handle:
+            return job_output(flux.Flux(), jobid)
+
+        # get_output() should block until the job is released, so run it
+        # in a separate thread using the ThreadPoolExecutor
+        future = self.executor.submit(get_output)
+
+        self.release_job(jobid)
+        try:
+            output = future.result(timeout=15)
+        except TimeoutError:
+            os.system(f"flux job eventlog {jobid}")
+            try:
+                flux.job.cancel(self.fh, jobid)
+            except FileNotFoundError:
+                pass
+            raise TimeoutError from None
+        self.assertEqual(output.stdout, self.test_stdout)
+        self.assertEqual(output.stderr, self.test_stderr)
+        self.assertEqual(output.log, "")
+
+    def test_pending_job_cancel(self):
+        jobid = self.submit(hold=True)
+
+        def get_output():
+            # This function will be run in another thread, so use a new
+            # Flux handle:
+            return job_output(flux.Flux(), jobid)
+
+        # get_output() should block until the job is released, so run it
+        # in a separate thread using the ThreadPoolExecutor
+        future = self.executor.submit(get_output)
+
+        # Cancel pending job
+        flux.job.cancel(self.fh, jobid)
+
+        # Job output should raise JobException
+        with self.assertRaises(flux.job.JobException):
+            future.result(timeout=15)
+
+    def test_pending_job_nowait(self):
+        jobid = self.submit(hold=True)
+        with self.assertRaises(FileNotFoundError):
+            job_output(self.fh, jobid, nowait=True)
+        flux.job.cancel(self.fh, jobid)
+
+    def test_output_event_watch(self):
+        jobid = self.submit()
+        stdout = ""
+        stderr = ""
+        for event in output_event_watch(self.fh, jobid):
+            if event.name == "data" and event.data:
+                if event.stream == "stdout":
+                    stdout += event.data
+                else:
+                    stderr += event.data
+        self.assertEqual(stdout, self.test_stdout)
+        self.assertEqual(stderr, self.test_stderr)
+
+    def test_output_event_watch_labelio(self):
+        jobid = self.submit(ntasks=2)
+        got_task = [False, False]
+        for event in output_event_watch(self.fh, jobid, labelio=True):
+            if event.name == "data" and event.data and "line 3" in event.data:
+                for rank in event.rank.ids:
+                    got_task[rank] = True
+        self.assertTrue(got_task[0])
+        self.assertTrue(got_task[1])
+
+    def test_output_event_watch_nowait(self):
+        def event_watch(jobid):
+            stdout = ""
+            stderr = ""
+            #  This function may be run in a thread, create new flux handle
+            fh = flux.Flux()
+            for event in output_event_watch(fh, jobid, nowait=True):
+                if event.name == "data" and event.data:
+                    if event.stream == "stdout":
+                        stdout += event.data
+                    else:
+                        stderr += event.data
+            return stdout, stderr
+
+        jobid = self.submit(hold=True)
+
+        #  Note: We don't test for FileNotFoundError here because there
+        #  ultimately no way to make a race free test. There would have to
+        #  be some way to ensure the watch request has been registered first.
+        self.release_job(jobid)
+        event_wait(self.fh, jobid, "shell.init", "guest.exec.eventlog")
+
+        # Now nowait should work:
+        stdout, stderr = event_watch(jobid)
+        self.assertEqual(stdout, self.test_stdout)
+        self.assertEqual(stderr, self.test_stderr)
+
+    def test_output_event_watch_async(self):
+        jobid = self.submit(verbose=True)
+
+        result = {"stdout": "", "stderr": "", "log": ""}
+
+        def output_event_watch_cb(future, result):
+            event = future.get_event()
+            if event is None:
+                return
+            if event.name == "data" and event.data is not None:
+                result[event.stream] += event.data
+            elif event.name == "log":
+                result["log"] += event.message + "\n"
+
+        future = output_event_watch_async(self.fh, jobid)
+        self.assertIsInstance(future, flux.job.output.JobOutputEventWatch)
+        future.then(output_event_watch_cb, result, timeout=60)
+        self.fh.reactor_run()
+
+        self.assertEqual(result["stdout"], self.test_stdout)
+        self.assertEqual(result["stderr"], self.test_stderr)
+        self.assertTrue(len(result["log"]) > 0)
+
+    def test_output_event_watch_async_cancel(self):
+        jobid = self.submit(cmd="echo before; sleep 30; echo after")
+
+        result = {"stdout": "", "stderr": "", "log": ""}
+
+        def output_event_watch_cb(future, result):
+            event = future.get_event()
+            if event is None:
+                return
+            if event.name == "data" and event.data is not None:
+                result[event.stream] += event.data
+                if "before" in event.data:
+                    flux.job.cancel(self.fh, jobid)
+            elif event.name == "log":
+                result["log"] += event.message + "\n"
+
+        future = output_event_watch_async(self.fh, jobid)
+        future.then(output_event_watch_cb, result, timeout=15)
+        self.fh.reactor_run()
+        self.assertIn("before", result["stdout"])
+
+    def test_output_watch(self):
+        def do_watch(
+            jobid,
+            result=None,
+            label=False,
+            log_stderr_level=LOG_TRACE,
+            nowait=False,
+        ):
+            if result is None:
+                result = {}
+            for stream, data in output_watch(
+                self.fh,
+                jobid,
+                labelio=label,
+                log_stderr_level=log_stderr_level,
+                nowait=nowait,
+            ):
+                result.setdefault(stream, []).append(data)
+            return {
+                key: "".join(result.get(key, [])) for key in ("stdout", "stderr", "log")
+            }
+
+        result = do_watch(self.submit())
+        self.assertEqual(result["stdout"], self.test_stdout)
+        self.assertEqual(result["stderr"], self.test_stderr)
+        self.assertEqual(result["log"], "")
+
+        # Separate logs
+        result = do_watch(self.submit(verbose=2), log_stderr_level=-1)
+        self.assertEqual(result["stdout"], self.test_stdout)
+        self.assertEqual(result["stderr"], self.test_stderr)
+        self.assertTrue(len(result["log"]) > 0)
+
+        # Redirect
+        result = do_watch(self.submit(redirect="{{tmpdir}}/test.out"))
+        self.assertRegex(result["stderr"], "stdout redirected to .*/test.out")
+        self.assertRegex(result["stderr"], "stderr redirected to .*/test.out")
+
+        # Labelio
+        result = do_watch(self.submit(ntasks=2), label=True)
+        self.assertIn("0: line 1", result["stdout"])
+        self.assertIn("1: line 1", result["stdout"])
+        self.assertIn("0: error 1", result["stderr"])
+        self.assertIn("1: error 1", result["stderr"])
+
+        # nowait=True works
+        jobid = self.submit()
+        event_wait(self.fh, jobid, "shell.init", "guest.exec.eventlog")
+
+        result = do_watch(jobid, nowait=True)
+        self.assertIn("line 1", result["stdout"])
+        self.assertIn("error 1", result["stderr"])
+
+        # canceled job
+        jobid = self.submit(hold=True)
+        flux.job.cancel(self.fh, jobid)
+
+        # catch OSError raised from non-started job:
+        with self.assertRaises(OSError) as context:
+            result = do_watch(jobid)
+            # Also, should have received a job.exception message on stderr:
+            self.assertIn("job.exception", "\n".join(result[0]["stderr"]))
+        exception = context.exception
+        self.assertEqual(exception.strerror, f"job {jobid} never started")
+        self.assertEqual(exception.errno, errno.EIO)
+
+    def test_output_watch_async(self):
+        def watch_cb(future, result):
+            try:
+                stream, data = future.get_output()
+            except Exception as exc:
+                stream = "stderr"
+                data = str(exc)
+                future.reset()
+            result.setdefault(stream, []).append(data)
+
+        def get_results(result):
+            return {
+                key: "".join(result.get(key, [])) for key in ("stdout", "stderr", "log")
+            }
+
+        results = {}
+
+        # Basic
+        results["basic"] = {}
+        output_watch_async(self.fh, self.submit()).then(watch_cb, results["basic"])
+
+        # Separate logs
+        results["log"] = {}
+        output_watch_async(
+            self.fh, self.submit(verbose=2), log_stderr_level=LOG_QUIET
+        ).then(watch_cb, results["log"])
+
+        # Labelio
+        results["labelio"] = {}
+        output_watch_async(self.fh, self.submit(ntasks=2), labelio=True).then(
+            watch_cb, results["labelio"]
+        )
+
+        # Cancel
+        jobid = self.submit(hold=True)
+        flux.job.cancel(self.fh, jobid)
+        results["cancel"] = {}
+        output_watch_async(self.fh, jobid).then(watch_cb, results["cancel"])
+
+        self.fh.reactor_run()
+
+        for test in ("basic", "log", "labelio", "cancel"):
+            results[test] = get_results(results[test])
+
+        self.assertEqual(results["basic"]["stdout"], self.test_stdout)
+        self.assertEqual(results["basic"]["stderr"], self.test_stderr)
+        self.assertEqual(results["basic"]["log"], "")
+
+        self.assertEqual(results["log"]["stdout"], self.test_stdout)
+        self.assertEqual(results["log"]["stderr"], self.test_stderr)
+        self.assertTrue(len(results["log"]["log"]) > 0)
+
+        self.assertIn("0: line 1", results["labelio"]["stdout"])
+        self.assertIn("1: line 1", results["labelio"]["stdout"])
+        self.assertIn("0: error 1", results["labelio"]["stderr"])
+        self.assertIn("1: error 1", results["labelio"]["stderr"])
+
+        # Also, should have received a job.exception message on stderr:
+        self.assertIn("job.exception", results["cancel"]["stderr"])
+        self.assertIn("never started", results["cancel"]["stderr"])
+
+    def test_output_watch_async_exception(self):
+        def watch_cb_with_error(future):
+            # Test raising error from typo:
+            stream, line = notfuture.getline()  # noqa: F821
+
+        jobid = self.submit()
+        output_watch_async(self.fh, jobid).then(watch_cb_with_error)
+        with self.assertRaises(NameError):
+            self.fh.reactor_run()
+
+    def test_output_watch_lines(self):
+        def do_watch_lines(
+            jobid,
+            result=None,
+            label=False,
+            log_stderr_level=LOG_TRACE,
+            keepends=False,
+            nojoin=False,
+            nowait=False,
+        ):
+            if result is None:
+                result = {}
+            count = 0
+            for stream, line in output_watch_lines(
+                self.fh,
+                jobid,
+                labelio=label,
+                log_stderr_level=log_stderr_level,
+                keepends=keepends,
+                nowait=nowait,
+            ):
+                count += 1
+                result.setdefault(stream, []).append(line)
+            if nojoin:
+                return result, count
+            return {
+                key: "\n".join(result.get(key, [])) + "\n"
+                for key in ("stdout", "stderr", "log")
+            }, count
+
+        exp_count = self.test_stdout.count("\n") + self.test_stderr.count("\n")
+
+        result, count = do_watch_lines(self.submit())
+        self.assertEqual(count, exp_count)
+        self.assertEqual(result["stdout"], self.test_stdout)
+        self.assertEqual(result["stderr"], self.test_stderr)
+        self.assertEqual(result["log"], "\n")
+
+        # Separate logs
+        result, count = do_watch_lines(self.submit(verbose=2), log_stderr_level=-1)
+        # Don't check `count`, unknown number of "log" lines
+        self.assertEqual(result["stdout"], self.test_stdout)
+        self.assertEqual(result["stderr"], self.test_stderr)
+        self.assertTrue(len(result["log"]) > 0)
+
+        # Redirect
+        result, count = do_watch_lines(self.submit(redirect="{{tmpdir}}/test.out"))
+        self.assertEqual(count, 2)
+        self.assertRegex(result["stderr"], "stdout redirected to .*/test.out")
+        self.assertRegex(result["stderr"], "stderr redirected to .*/test.out")
+
+        # Labelio
+        result, count = do_watch_lines(self.submit(ntasks=2), label=True)
+        self.assertEqual(count, 2 * exp_count)
+        self.assertIn("0: line 1", result["stdout"])
+        self.assertIn("1: line 1", result["stdout"])
+        self.assertIn("0: error 1", result["stderr"])
+        self.assertIn("1: error 1", result["stderr"])
+
+        # keepends=True
+        result, count = do_watch_lines(self.submit(), keepends=True, nojoin=True)
+        self.assertEqual(count, exp_count)
+        self.assertIn("line 1\n", result["stdout"])
+        self.assertIn("error 1\n", result["stderr"])
+
+        # nowait=True works
+        jobid = self.submit()
+        event_wait(self.fh, jobid, "shell.init", "guest.exec.eventlog")
+
+        result, count = do_watch_lines(jobid, nowait=True)
+        self.assertEqual(count, exp_count)
+        self.assertIn("line 1", result["stdout"])
+        self.assertIn("error 1", result["stderr"])
+
+        # canceled job
+        jobid = self.submit(hold=True)
+        flux.job.cancel(self.fh, jobid)
+
+        # catch OSError raised from non-started job:
+        with self.assertRaises(OSError) as context:
+            result, count = do_watch_lines(jobid)
+            # Also, should have received a job.exception message on stderr:
+            self.assertIn("job.exception", "\n".join(result[0]["stderr"]))
+        exception = context.exception
+        self.assertEqual(exception.strerror, f"job {jobid} never started")
+        self.assertEqual(exception.errno, errno.EIO)
+
+    def test_output_watch_lines_async(self):
+        def watch_cb(future, result):
+            try:
+                stream, line = future.getline()
+            except Exception as exc:
+                stream = "stderr"
+                line = str(exc)
+                future.reset()
+            result.setdefault(stream, []).append(line)
+            if "count" in result:
+                if stream is not None:
+                    result["count"] += 1
+            else:
+                result["count"] = 1
+
+        def get_results(result):
+            results = {
+                key: "\n".join(result.get(key, [])) + "\n"
+                for key in ("stdout", "stderr", "log")
+            }
+            results["count"] = result["count"]
+            return results
+
+        results = {}
+        exp_count = self.test_stdout.count("\n") + self.test_stderr.count("\n")
+
+        # Basic
+        results["basic"] = {}
+        output_watch_lines_async(self.fh, self.submit()).then(
+            watch_cb, results["basic"]
+        )
+        # Separate logs
+        results["log"] = {}
+        output_watch_lines_async(
+            self.fh, self.submit(verbose=2), log_stderr_level=LOG_QUIET
+        ).then(watch_cb, results["log"])
+        # Labelio
+        results["labelio"] = {}
+        output_watch_lines_async(self.fh, self.submit(ntasks=2), labelio=True).then(
+            watch_cb, results["labelio"]
+        )
+
+        # Cancel
+        jobid = self.submit(hold=True)
+        flux.job.cancel(self.fh, jobid)
+        results["cancel"] = {}
+        output_watch_lines_async(self.fh, jobid).then(watch_cb, results["cancel"])
+
+        self.fh.reactor_run()
+
+        for test in ("basic", "log", "labelio", "cancel"):
+            results[test] = get_results(results[test])
+
+        # check results:
+        self.assertEqual(results["basic"]["count"], exp_count)
+        self.assertEqual(results["basic"]["stdout"], self.test_stdout)
+        self.assertEqual(results["basic"]["stderr"], self.test_stderr)
+        self.assertEqual(results["basic"]["log"], "\n")
+
+        #  Unknown number of "log" entries so don't test count
+        self.assertEqual(results["log"]["stdout"], self.test_stdout)
+        self.assertEqual(results["log"]["stderr"], self.test_stderr)
+        self.assertTrue(len(results["log"]["log"]) > 0)
+
+        self.assertEqual(results["labelio"]["count"], 2 * exp_count)
+        self.assertIn("0: line 1", results["labelio"]["stdout"])
+        self.assertIn("1: line 1", results["labelio"]["stdout"])
+        self.assertIn("0: error 1", results["labelio"]["stderr"])
+        self.assertIn("1: error 1", results["labelio"]["stderr"])
+
+        # check canceled job for a job.exception message on stderr:
+        self.assertIn("job.exception", results["cancel"]["stderr"])
+        self.assertIn("never started", results["cancel"]["stderr"])
+
+    def test_output_watch_lines_async_exception(self):
+        def watch_cb_with_error(future):
+            # Test raising error from typo:
+            stream, line = notfuture.getline()  # noqa: F821
+
+        jobid = self.submit()
+        output_watch_lines_async(self.fh, jobid).then(watch_cb_with_error)
+        with self.assertRaises(NameError):
+            self.fh.reactor_run()
+
+
+if __name__ == "__main__":
+    from subflux import rerun_under_flux
+
+    if rerun_under_flux(size=__flux_size(), personality="job"):
+        from pycotap import LogMode, TAPTestRunner
+
+        unittest.main(testRunner=TAPTestRunner(LogMode.LogToYAML, LogMode.LogToError))
diff --git a/t/python/t0020-hostlist.py b/t/python/t0020-hostlist.py
new file mode 100755
index 000000000000..90335f057aaf
--- /dev/null
+++ b/t/python/t0020-hostlist.py
@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import unittest
+
+import flux.hostlist as hostlist
+import subflux  # noqa: F401
+from flux.idset import IDset
+from pycotap import TAPTestRunner
+
+
+class TestHostlistMethods(unittest.TestCase):
+    def test_basic_decode(self):
+        # simple string works
+        hl = hostlist.decode("host")
+        self.assertEqual(hl.count(), 1)
+
+        # iterables works
+        hl = hostlist.decode(["foo1", "foo2"])
+        self.assertEqual(str(hl), "foo[1-2]")
+
+        # set works, but must sort to get guaranteed order
+        hl = hostlist.decode({"foo1", "foo2"}).sort()
+        self.assertEqual(str(hl), "foo[1-2]")
+
+        with self.assertRaises(TypeError):
+            hl = hostlist.decode(["foo1", 42])
+
+    def test_invalid_decode(self):
+        test_invalid = [
+            "[]",
+            "foo[]",
+            "foo[",
+            "foo[1,3",
+            "foo[[1,3]",
+            "foo]",
+            "foo[x-y]",
+            "foo[0-1,2--5]",
+        ]
+        for string in test_invalid:
+            with self.assertRaises(ValueError):
+                hostlist.decode(string)
+
+        # TypeError tests
+        with self.assertRaises(TypeError):
+            hostlist.decode(42)
+        with self.assertRaises(TypeError):
+            hostlist.decode(1.0)
+        with self.assertRaises(TypeError):
+            hostlist.decode(["foo", 42])
+        with self.assertRaises(TypeError):
+            hostlist.decode()
+
+    def test_str(self):
+        tests = [
+            {"input": "", "output": ""},
+            {"input": "foo0", "output": "foo0"},
+            {"input": "foo0,foo1", "output": "foo[0-1]"},
+            {"input": "foo0,bar1", "output": "foo0,bar1"},
+            {"input": "foo[0-10]", "output": "foo[0-10]"},
+        ]
+        for test in tests:
+            hl = hostlist.decode(test["input"])
+            expected = test["output"]
+            self.assertEqual(str(hl), expected)
+            self.assertEqual(repr(hl), f"Hostlist('{expected}')")
+
+    def test_count(self):
+        tests = [
+            {"input": "", "result": 0},
+            {"input": "foo0", "result": 1},
+            {"input": "foo0,foo1", "result": 2},
+            {"input": "foo0,bar1", "result": 2},
+            {"input": "foo[0-10]", "result": 11},
+        ]
+        for test in tests:
+            hl = hostlist.decode(test["input"])
+            self.assertEqual(len(hl), test["result"])
+            self.assertEqual(hl.count(), test["result"])
+
+    def test_index(self):
+        hl = hostlist.decode("foo[0-9]")
+        self.assertEqual(hl[0], "foo0")
+        self.assertEqual(hl[1], "foo1")
+        self.assertEqual(hl[9], "foo9")
+        self.assertEqual(hl[-1], "foo9")
+        self.assertEqual(hl[-2], "foo8")
+        self.assertListEqual(list(hl[1:3]), ["foo1", "foo2"])
+        hl2 = hl[1, 2, 3]
+        self.assertIsInstance(hl2, hostlist.Hostlist)
+        self.assertListEqual(list(hl2), ["foo1", "foo2", "foo3"])
+        ids = IDset("0-1")
+        hl2 = hl[ids]
+        self.assertIsInstance(hl2, hostlist.Hostlist)
+        self.assertListEqual(list(hl2), ["foo0", "foo1"])
+
+    def test_index_exceptions(self):
+        hl = hostlist.decode("foo[0-9]")
+        self.assertRaises(TypeError, lambda x: x["a"], hl)
+        self.assertRaises(IndexError, lambda x: x[10], hl)
+        self.assertRaises(TypeError, lambda x: x["abc"], hl)
+
+    def test_contains(self):
+        hl = hostlist.decode("foo[0-9]")
+        self.assertIn("foo0", hl)
+        self.assertIn("foo5", hl)
+        self.assertNotIn("foo10", hl)
+        self.assertNotIn("foo", hl)
+
+    def test_iterator(self):
+        hl = hostlist.decode("foo[0-3]")
+        self.assertListEqual([host for host in hl], hl.expand())
+        hl = hostlist.decode("")
+        self.assertListEqual([host for host in hl], [])
+
+    def test_append(self):
+        hl = hostlist.decode("")
+        self.assertEqual(hl.append("foo[0-3]"), 4)
+        self.assertEqual(str(hl), "foo[0-3]")
+        self.assertEqual(hl.append("foo[7-9]"), 3)
+        self.assertEqual(str(hl), "foo[0-3,7-9]")
+        nl = hostlist.decode("foo1,bar")
+        self.assertEqual(hl.append(nl), 2)
+        self.assertEqual(str(hl), "foo[0-3,7-9,1],bar")
+        hl.append(["bar0", "bar1"])
+        self.assertEqual(str(hl), "foo[0-3,7-9,1],bar,bar[0-1]")
+
+        with self.assertRaises(TypeError):
+            hl.append(42)
+        with self.assertRaises(TypeError):
+            hl.append(["bar2", 42])
+
+    def test_delete(self):
+        hl = hostlist.decode("foo[0-10]")
+        self.assertEqual(hl.delete("foo5"), 1)
+        self.assertEqual(hl.delete("foo1,foo3"), 2)
+        self.assertEqual(str(hl), "foo[0,2,4,6-10]")
+
+        self.assertEqual(hl.delete(hostlist.decode("foo[6-7]")), 2)
+        self.assertEqual(str(hl), "foo[0,2,4,8-10]")
+
+    def test_sort(self):
+        hl = hostlist.decode("foo[7,6,5,4,3,2,1]").sort()
+        self.assertEqual(str(hl), "foo[1-7]")
+
+    def test_uniq(self):
+        hl = hostlist.decode("foo[1-7,1-7,1-7]").uniq()
+        self.assertEqual(str(hl), "foo[1-7]")
+
+    def test_copy(self):
+        hl = hostlist.decode("foo[0-3]")
+        cp = hl.copy()
+        hl.delete("foo[0-3]")
+        self.assertEqual(str(cp), "foo[0-3]")
+
+    def test_find(self):
+        hl = hostlist.decode("foo[0-3]")
+        self.assertEqual(hl.find("foo1"), 1)
+        self.assertEqual(hl.find("foo3"), 3)
+        with self.assertRaises(FileNotFoundError):
+            hl.find("foo4")
+
+    def test_index_method(self):
+        hl = hostlist.decode("foo[0-10]")
+        self.assertListEqual(hl.index("foo[1,5,7]"), [1, 5, 7])
+        self.assertListEqual(hl.index(hostlist.decode("foo1")), [1])
+        self.assertListEqual(hl.index("foo[9-11]", ignore_nomatch=True), [9, 10])
+        self.assertListEqual(hl.index("foo11", ignore_nomatch=True), [])
+        with self.assertRaises(FileNotFoundError):
+            hl.index("foo[9-11]")
+        with self.assertRaises(FileNotFoundError):
+            hl.index("foo11")
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=TAPTestRunner())
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/python/t0021-idset.py b/t/python/t0021-idset.py
new file mode 100755
index 000000000000..83b5cb221931
--- /dev/null
+++ b/t/python/t0021-idset.py
@@ -0,0 +1,283 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import unittest
+
+import flux.idset as idset
+import subflux  # noqa: F401
+from pycotap import TAPTestRunner
+
+
+class TestIDsetMethods(unittest.TestCase):
+    def test_constructor(self):
+        self.assertEqual(str(idset.IDset()), "")
+        self.assertEqual(str(idset.IDset(idset.IDset())), "")
+        self.assertEqual(str(idset.IDset(0)), "0")
+        self.assertEqual(str(idset.IDset(42)), "42")
+        self.assertEqual(str(idset.IDset([42])), "42")
+        self.assertEqual(str(idset.IDset("40,41,42")), "40-42")
+        self.assertEqual(str(idset.IDset([40, 41, 42])), "40-42")
+        self.assertEqual(str(idset.IDset([42, 41, 40])), "40-42")
+
+    def test_str(self):
+        tests = [
+            {"input": "", "flags": None, "output": ""},
+            {"input": "0", "flags": None, "output": "0"},
+            {"input": "0,1", "flags": None, "output": "0-1"},
+            {"input": "0-10", "flags": None, "output": "0-10"},
+            {
+                "input": "0-10",
+                "flags": idset.IDSET_FLAG_RANGE | idset.IDSET_FLAG_BRACKETS,
+                "output": "[0-10]",
+            },
+            {
+                "input": "0-3",
+                "flags": idset.IDSET_FLAG_BRACKETS,
+                "output": "[0,1,2,3]",
+            },
+            {"input": "0-3", "flags": 0, "output": "0,1,2,3"},
+        ]
+        for test in tests:
+            ids = idset.decode(test["input"])
+            if test["flags"] is not None:
+                ids.set_flags(test["flags"])
+            self.assertEqual(str(ids), test["output"])
+            expected = str(idset.decode(test["output"]))
+            self.assertEqual(repr(ids), f"IDset('{expected}')")
+
+    def test_count(self):
+        tests = [
+            {"input": "", "result": 0},
+            {"input": "0", "result": 1},
+            {"input": "0,1", "result": 2},
+            {"input": "0-10", "result": 11},
+            {"input": "0,5,9", "result": 3},
+        ]
+        i = 0
+        for test in tests:
+            with self.subTest(i=i):
+                ids = idset.decode(test["input"])
+                self.assertEqual(len(ids), test["result"])
+                self.assertEqual(ids.count(), test["result"])
+
+    def test_index(self):
+        ids = idset.decode("0-9")
+        self.assertTrue(ids[0])
+        self.assertTrue(ids[5])
+        self.assertFalse(ids[10])
+        ids[10] = True
+        ids[21] = 1
+        ids[5] = False
+        self.assertTrue(ids[10])
+        self.assertTrue(ids[21])
+        self.assertFalse(ids[5])
+        self.assertEqual(str(ids), "0-4,6-10,21")
+
+    def test_index_exceptions(self):
+        ids = idset.decode("0-9")
+        self.assertRaises(TypeError, lambda x: x["a"], ids)
+        self.assertRaises(ValueError, lambda x: x[-1], ids)
+        with self.assertRaises(TypeError):
+            ids[0] = 7
+        with self.assertRaises(TypeError):
+            ids[0] = "hello"
+
+    def test_contains(self):
+        ids = idset.decode("0-9")
+        self.assertIn(0, ids)
+        self.assertIn(5, ids)
+        self.assertNotIn(10, ids)
+        self.assertNotIn(1024, ids)
+
+    def test_contains_exceptions(self):
+        ids = idset.decode("0-9")
+        with self.assertRaises(ValueError):
+            -1 in ids
+        with self.assertRaises(TypeError):
+            "foo" in ids
+
+    def test_iterator(self):
+        ids = idset.decode("0-3,7")
+        self.assertListEqual([i for i in ids], [0, 1, 2, 3, 7])
+        ids = idset.decode("3")
+        self.assertListEqual([i for i in ids], [3])
+        ids = idset.decode("")
+        self.assertListEqual([i for i in ids], [])
+
+    def test_expand(self):
+        self.assertListEqual(idset.decode("0-3").expand(), [0, 1, 2, 3])
+        self.assertListEqual(idset.decode("").expand(), [])
+
+    def test_set_and_clear(self):
+        ids = idset.IDset()
+        ids.set(3)
+        self.assertTrue(ids[3])
+        self.assertEqual(str(ids), "3")
+        ids.clear(3)
+        self.assertFalse(ids[3])
+        self.assertEqual(str(ids), "")
+
+        ids.set(3, 7)
+        self.assertEqual(str(ids), "3-7")
+
+        ids.clear(4, 5)
+        self.assertEqual(str(ids), "3,6-7")
+
+    def test_set_and_clear_exceptions(self):
+        ids = idset.IDset()
+        self.assertRaises(ValueError, lambda x: x.set(-1), ids)
+        self.assertRaises(ValueError, lambda x: x.set(1, -1), ids)
+        self.assertRaises(TypeError, lambda x: x.set("a"), ids)
+        self.assertRaises(ValueError, lambda x: x.set(5, 1), ids)
+
+        self.assertRaises(ValueError, lambda x: x.clear(-1), ids)
+        self.assertRaises(ValueError, lambda x: x.clear(1, -1), ids)
+        self.assertRaises(TypeError, lambda x: x.clear("a"), ids)
+        self.assertRaises(ValueError, lambda x: x.clear(5, 1), ids)
+
+    def test_copy(self):
+        ids = idset.decode("0-9")
+        cp = ids.copy()
+        cp.clear(1, 9)
+        self.assertEqual(str(cp), "0")
+
+    def test_equal(self):
+        ids1 = idset.IDset()
+        ids2 = idset.IDset()
+        self.assertEqual(ids1, ids2)
+        self.assertTrue(ids1.equal(ids2))
+        ids1.set(0, 9)
+        ids2.set(0, 9)
+        self.assertEqual(ids1, ids2)
+        self.assertTrue(ids2.equal(ids1))
+        ids1.clear(0)
+        self.assertNotEqual(ids1, ids2)
+        self.assertFalse(ids1.equal(ids2))
+        with self.assertRaises(TypeError):
+            ids1 == "foo"
+        with self.assertRaises(TypeError):
+            ids1.equal("foo")
+
+    def test_first_last_next(self):
+        ids = idset.decode("0-9")
+        self.assertEqual(ids.first(), 0)
+        self.assertEqual(ids.last(), 9)
+        self.assertEqual(ids.next(5), 6)
+        self.assertEqual(ids.next(9), idset.IDSET_INVALID_ID)
+        self.assertRaises(ValueError, lambda x: x.next(-1), ids)
+        self.assertRaises(TypeError, lambda x: x.next("a"), ids)
+
+    def test_add_subtract(self):
+        ids = idset.decode("0-9")
+        ids2 = ids.copy()
+
+        self.assertEqual(str(ids.add("10-11")), "0-11")
+        self.assertEqual(str(ids.add([20, 21])), "0-11,20-21")
+        self.assertEqual(str(ids.add(idset.decode(""))), "0-11,20-21")
+
+        ids2 += "10-11"
+        self.assertEqual(str(ids2), "0-11")
+        ids2 += [20, 21]
+        self.assertEqual(str(ids2), "0-11,20-21")
+        ids2 += idset.decode("")
+        self.assertEqual(str(ids2), "0-11,20-21")
+
+        self.assertEqual(str(ids.subtract([])), "0-11,20-21")
+        self.assertEqual(str(ids.subtract("11-20")), "0-10,21")
+        self.assertEqual(str(ids.subtract(idset.decode("0-10"))), "21")
+        self.assertEqual(str(ids.subtract([21])), "")
+
+        ids2 -= ""
+        self.assertEqual(str(ids2), "0-11,20-21")
+        ids2 -= idset.IDset()
+        self.assertEqual(str(ids2), "0-11,20-21")
+        ids2 -= "11-20"
+        self.assertEqual(str(ids2), "0-10,21")
+        ids2 -= idset.decode("0-10")
+        self.assertEqual(str(ids2), "21")
+        ids2 -= 21
+        self.assertEqual(str(ids2), "")
+
+        with self.assertRaises(ValueError):
+            ids.subtract("foo")
+        with self.assertRaises(TypeError):
+            ids.subtract(42.0)
+        with self.assertRaises(ValueError):
+            ids.add("foo")
+        with self.assertRaises(TypeError):
+            ids.add(42.0)
+
+    def test_intersect(self):
+        tests = [
+            {"idset": "0-10", "args": ["0-5", "0-3"], "result": "0-3"},
+            {"idset": "0-1", "args": ["3-4"], "result": ""},
+            {"idset": "0-1024", "args": ["500-600"], "result": "500-600"},
+        ]
+        for test in tests:
+            ids = idset.decode(test["idset"])
+            # first, works with encoded idsets
+            result = ids.intersect(*test["args"])
+            self.assertEqual(str(result), test["result"])
+
+            # also try with decoded IDset objects
+            result = ids.intersect(*map(idset.decode, test["args"]))
+            self.assertEqual(str(result), test["result"])
+
+            # and finally with & operator
+            result = ids.copy()
+            for arg in test["args"]:
+                result = result & arg
+            self.assertEqual(str(result), test["result"])
+
+    def test_union(self):
+        tests = [
+            {"idset": "0-10", "args": ["5-15", "0-3"], "result": "0-15"},
+            {"idset": "0-1", "args": ["3-4"], "result": "0-1,3-4"},
+        ]
+        for test in tests:
+            ids = idset.decode(test["idset"])
+            result = ids.union(*test["args"])
+            self.assertEqual(str(result), test["result"])
+
+            result = ids.copy()
+            for arg in test["args"]:
+                result = result + arg
+            self.assertEqual(str(result), test["result"])
+
+            result = ids.copy()
+            for arg in test["args"]:
+                result = result | arg
+            self.assertEqual(str(result), test["result"])
+
+    def test_difference(self):
+        tests = [
+            {"idset": "0-10", "args": ["0-3"], "result": "4-10"},
+            {"idset": "0-10", "args": ["0-10"], "result": ""},
+            {"idset": "0-10", "args": ["5-7", "1-3"], "result": "0,4,8-10"},
+            {"idset": "0-1", "args": ["0-10"], "result": ""},
+            {"idset": "0-1024", "args": ["500-600"], "result": "0-499,601-1024"},
+        ]
+        for test in tests:
+            ids = idset.decode(test["idset"])
+            result = ids.difference(*test["args"])
+            self.assertEqual(str(result), test["result"])
+
+            result = ids.copy()
+            for arg in test["args"]:
+                result = result - arg
+            self.assertEqual(str(result), test["result"])
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=TAPTestRunner())
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/python/t0022-resource-set.py b/t/python/t0022-resource-set.py
new file mode 100755
index 000000000000..410c79bc0f1a
--- /dev/null
+++ b/t/python/t0022-resource-set.py
@@ -0,0 +1,296 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import json
+import unittest
+
+import subflux  # noqa: F401 - for PYTHONPATH
+from flux.hostlist import Hostlist
+from flux.idset import IDset
+from flux.resource import ResourceSet, Rlist
+from pycotap import TAPTestRunner
+
+
+class TestRSet(unittest.TestCase):
+    R_input = """
+    {
+      "version": 1,
+      "execution": {
+        "R_lite": [
+          {
+            "rank": "0-3",
+            "children": {
+                "core": "0-3",
+                "gpu": "0"
+             }
+          }
+        ],
+        "starttime": 0,
+        "expiration": 0,
+        "nodelist": [
+          "fluke[0-3]"
+        ]
+      }
+    }
+    """
+    R2 = """
+    {
+      "version": 1,
+      "execution": {
+        "R_lite": [
+          {
+            "rank": "10-13",
+            "children": {
+                "core": "0-3",
+                "gpu": "0"
+             }
+          }
+        ],
+        "starttime": 0,
+        "expiration": 0,
+        "nodelist": [
+          "fluke[10-13]"
+        ]
+      }
+    }
+    """
+
+    def test_init_string(self):
+        #  init by string
+        rset = ResourceSet(self.R_input)
+        self.assertEqual(str(rset), "rank[0-3]/core[0-3],gpu0")
+        self.assertEqual(rset.ncores, 16)
+        self.assertEqual(rset.ngpus, 4)
+
+        rset = ResourceSet(self.R2)
+        self.assertEqual(str(rset), "rank[10-13]/core[0-3],gpu0")
+        self.assertEqual(rset.ncores, 16)
+        self.assertEqual(rset.ngpus, 4)
+
+    def test_init_dict(self):
+        #  init by dict
+        rdict = json.loads(self.R_input)
+        rset = ResourceSet(rdict)
+        self.assertEqual(str(rset), "rank[0-3]/core[0-3],gpu0")
+        self.assertEqual(rset.ncores, 16)
+        self.assertEqual(rset.ngpus, 4)
+        self.assertEqual(rset.nnodes, 4)
+
+        rdict = json.loads(self.R2)
+        rset = ResourceSet(rdict)
+        self.assertEqual(str(rset), "rank[10-13]/core[0-3],gpu0")
+        self.assertEqual(rset.ncores, 16)
+        self.assertEqual(rset.ngpus, 4)
+        self.assertEqual(rset.nnodes, 4)
+
+    def test_init_implementation(self):
+        #  init by resource set implementation
+        rlist = Rlist().add_rank(0, cores="0-1").add_child(0, "gpu", "0")
+        rset = ResourceSet(rlist)
+        self.assertEqual(str(rset), "rank0/core[0-1],gpu0")
+        self.assertEqual(rset.ncores, 2)
+        self.assertEqual(rset.ngpus, 1)
+        self.assertEqual(rset.nnodes, 1)
+
+    def test_init_empty(self):
+        rset = ResourceSet()
+        self.assertEqual(str(rset), "")
+        self.assertEqual(rset.nnodes, 0)
+        self.assertEqual(rset.ncores, 0)
+        self.assertEqual(rset.ngpus, 0)
+
+    def test_init_assertions(self):
+        #  test invalid implementation
+        with self.assertRaises(TypeError):
+            ResourceSet((1, 2))
+
+        #  test invalid resource set string:
+        with self.assertRaises(ValueError):
+            ResourceSet("")
+
+        #  test invalid version
+        with self.assertRaises(ValueError):
+            ResourceSet({"version": 199})
+
+        #  test string with invalid version
+        with self.assertRaises(ValueError):
+            arg = json.dumps({"version": 199})
+            ResourceSet(arg)
+
+        #  test dict without 'version' string
+        with self.assertRaises(KeyError):
+            ResourceSet({})
+
+        #  test invalid JSON string
+        with self.assertRaises(json.decoder.JSONDecodeError):
+            ResourceSet("{")
+
+    def test_op(self):
+        set1 = ResourceSet(self.R_input)
+        set2 = set1.copy().remove_ranks("2-3")
+
+        # difference
+        result = set1 - set2
+        self.assertIsInstance(result, ResourceSet)
+        self.assertEqual(str(result), "rank[2-3]/core[0-3],gpu0")
+
+        set3 = result
+        # union
+        result = set3 | set1
+        self.assertIsInstance(result, ResourceSet)
+        self.assertEqual(str(result), "rank[0-3]/core[0-3],gpu0")
+
+        # intersection
+        result = set1 & set2
+        self.assertIsInstance(result, ResourceSet)
+        self.assertEqual(str(result), "rank[0-1]/core[0-3],gpu0")
+
+    def test_encode(self):
+        rset = ResourceSet(self.R_input)
+        rstring = rset.encode()
+        rset2 = ResourceSet(rstring)
+        self.assertEqual(str(rset), str(rset2))
+
+        rset = ResourceSet(self.R2)
+        rstring = rset.encode()
+        rset2 = ResourceSet(rstring)
+        self.assertEqual(str(rset), str(rset2))
+
+    def test_append(self):
+        rset = ResourceSet(self.R_input)
+        rset2 = ResourceSet(Rlist().add_rank(4, cores="0-3").add_child(4, "gpu", "0"))
+        rset.append(rset2)
+        self.assertEqual(str(rset), "rank[0-4]/core[0-3],gpu0")
+
+    def test_add(self):
+        rset = ResourceSet(self.R_input)
+        rset2 = ResourceSet(Rlist().add_rank(4, cores="0-3").add_child(4, "gpu", "0"))
+        rset.add(rset2)
+        self.assertEqual(str(rset), "rank[0-4]/core[0-3],gpu0")
+        # adding same resources allowed
+        rset2 = ResourceSet(Rlist().add_rank(4, cores="0-3").add_child(4, "gpu", "0"))
+        rset.add(rset2)
+        self.assertEqual(str(rset), "rank[0-4]/core[0-3],gpu0")
+
+    def test_nodelist(self):
+        rset = ResourceSet(self.R_input)
+        self.assertIsInstance(rset.nodelist, Hostlist)
+        self.assertEqual(rset.nodelist.count(), 4)
+        self.assertEqual(str(rset.nodelist), "fluke[0-3]")
+
+        rset = ResourceSet(self.R2)
+        self.assertIsInstance(rset.nodelist, Hostlist)
+        self.assertEqual(rset.nodelist.count(), 4)
+        self.assertEqual(str(rset.nodelist), "fluke[10-13]")
+
+    def test_ranks(self):
+        rset = ResourceSet(self.R_input)
+        self.assertIsInstance(rset.ranks, IDset)
+        self.assertEqual(rset.ranks.count(), 4)
+        self.assertEqual(str(rset.ranks), "0-3")
+
+        rset = ResourceSet(self.R2)
+        self.assertIsInstance(rset.ranks, IDset)
+        self.assertEqual(rset.ranks.count(), 4)
+        self.assertEqual(str(rset.ranks), "10-13")
+
+    def test_state(self):
+        rset = ResourceSet(self.R_input)
+        self.assertIsNone(rset.state)
+        rset.state = "up"
+        self.assertEqual(rset.state, "up")
+
+    def test_host_ranks(self):
+        rset = ResourceSet(self.R_input)
+        self.assertIsInstance(rset, ResourceSet)
+        self.assertEqual(rset.host_ranks("fluke0"), [0])
+        self.assertEqual(rset.host_ranks("fluke1"), [1])
+        self.assertEqual(rset.host_ranks("fluke2"), [2])
+        self.assertEqual(rset.host_ranks("fluke3"), [3])
+        self.assertEqual(rset.host_ranks("fluke[0,3]"), [0, 3])
+        self.assertEqual(rset.host_ranks("fluke[0-2]"), [0, 1, 2])
+        self.assertEqual(
+            rset.host_ranks("fluke[0-2,7]", ignore_nomatch=True), [0, 1, 2]
+        )
+        with self.assertRaises(FileNotFoundError):
+            rset.host_ranks("fluke7")
+
+        rset = ResourceSet(self.R2)
+        self.assertIsInstance(rset, ResourceSet)
+        self.assertEqual(rset.host_ranks("fluke10"), [10])
+        self.assertEqual(rset.host_ranks("fluke11"), [11])
+        self.assertEqual(rset.host_ranks("fluke12"), [12])
+        self.assertEqual(rset.host_ranks("fluke13"), [13])
+        self.assertEqual(rset.host_ranks("fluke[10,13]"), [10, 13])
+        self.assertEqual(rset.host_ranks("fluke[10-12]"), [10, 11, 12])
+        self.assertEqual(
+            rset.host_ranks("fluke[0-2,10-12]", ignore_nomatch=True), [10, 11, 12]
+        )
+        with self.assertRaises(FileNotFoundError):
+            rset.host_ranks("fluke[0-3,10-12]")
+
+    def test_properties(self):
+        rset = ResourceSet(self.R_input)
+        self.assertIsInstance(rset, ResourceSet)
+
+        rset.set_property("xx", "0-1")
+        rset.set_property("zz", "2-3")
+        rset.set_property("aa", "1")
+
+        with self.assertRaises(ValueError):
+            rset.set_property("x^y")
+        with self.assertRaises(ValueError):
+            rset.set_property("yy", "0-6")
+        with self.assertRaises(ValueError):
+            rset.set_property("yy", "foo")
+
+        # copy_constraint() with invalid property returns empty set
+        empty = rset.copy_constraint({"properties": ["foo"]})
+        self.assertIsInstance(empty, ResourceSet)
+        self.assertEqual(str(empty), "")
+
+        xx = rset.copy_constraint({"properties": ["xx"]})
+        self.assertIsInstance(xx, ResourceSet)
+        self.assertEqual(str(xx), "rank[0-1]/core[0-3],gpu0")
+
+        zz = rset.copy_constraint({"properties": ["zz"]})
+        self.assertIsInstance(zz, ResourceSet)
+        self.assertEqual(str(zz), "rank[2-3]/core[0-3],gpu0")
+
+        aa = rset.copy_constraint({"properties": ["aa"]})
+        self.assertIsInstance(aa, ResourceSet)
+        self.assertEqual(str(aa), "rank1/core[0-3],gpu0")
+
+        aa = xx.copy_constraint({"properties": ["aa"]})
+        self.assertIsInstance(aa, ResourceSet)
+        self.assertEqual(str(aa), "rank1/core[0-3],gpu0")
+
+        #  not (xx and aa)
+        notax = rset.copy_constraint({"not": [{"properties": ["aa", "xx"]}]})
+        self.assertIsInstance(notax, ResourceSet)
+        self.assertEqual(str(notax), "rank[0,2-3]/core[0-3],gpu0")
+
+        #  not xx and not aa
+        notax2 = rset.copy_constraint({"properties": ["^aa", "^xx"]})
+        self.assertIsInstance(notax2, ResourceSet)
+        self.assertEqual(str(notax2), "rank[2-3]/core[0-3],gpu0")
+
+        with self.assertRaises(ValueError):
+            rset.copy_constraint({"foo": []})
+        with self.assertRaises(ValueError):
+            rset.copy_constraint({"properties": [42]})
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=TAPTestRunner())
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/python/t0023-executor.py b/t/python/t0023-executor.py
new file mode 100755
index 000000000000..895246a4d913
--- /dev/null
+++ b/t/python/t0023-executor.py
@@ -0,0 +1,564 @@
+#!/usr/bin/env python3
+
+###############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import collections
+import concurrent.futures as cf
+import itertools
+import os
+import threading
+import types
+import unittest
+
+from flux.job import EventLogEvent, JobException, JobspecV1
+from flux.job.executor import (
+    FluxExecutor,
+    FluxExecutorFuture,
+    _AttachPackage,
+    _FluxExecutorThread,
+    _SubmitPackage,
+)
+
+
+def __flux_size():
+    return 1
+
+
+class ShamJobEventWatchFuture:
+    """A wrapper around a LogEvent to act like a JobEventWatchFuture."""
+
+    def __init__(self, log_event):
+        self.log_event = log_event
+
+    def get_event(self):
+        return self.log_event
+
+
+class TestFluxExecutor(unittest.TestCase):
+    """Tests for FluxExecutor."""
+
+    def test_as_completed(self):
+        with FluxExecutor() as executor:
+            jobspec = JobspecV1.from_command(["true"])
+            futures = [executor.submit(jobspec) for _ in range(3)]
+            attach_futures = []
+            for fut in cf.as_completed(futures):
+                self.assertEqual(fut.result(timeout=0), 0)
+                self.assertIsNone(fut.exception())
+                attach_fut = executor.attach(fut.jobid())
+                self.assertEqual(fut.jobid(), attach_fut.jobid())
+                attach_futures.append(attach_fut)
+            for attach_fut in cf.as_completed(attach_futures):
+                self.assertEqual(attach_fut.result(timeout=0), 0)
+                self.assertIsNone(attach_fut.exception())
+        self.assertFalse(executor._broken_event.is_set())
+
+    def test_failed_job(self):
+        with FluxExecutor(thread_name_prefix="foobar") as executor:
+            jobspec = JobspecV1.from_command(["false"])
+            future = executor.submit(jobspec).add_jobid_callback(
+                lambda future: event.set()
+            )
+            event = threading.Event()
+            jobid = future.jobid()
+            self.assertGreater(jobid, 0)
+            self.assertTrue(event.is_set())
+            self.assertEqual(future.result(), 1)
+            self.assertIsNone(future.exception())
+            # now attach to the same future
+            future = executor.attach(jobid)
+            self.assertEqual(jobid, future.jobid())
+            self.assertEqual(future.result(), 1)
+            self.assertIsNone(future.exception())
+        self.assertFalse(executor._broken_event.is_set())
+
+    def test_cancel_submit(self):
+        with FluxExecutor() as executor:
+            jobspec = JobspecV1.from_command(["false"])
+            for _ in range(3):
+                future = executor.submit(jobspec)
+                if future.cancel():
+                    self.assertFalse(future.running())
+                    self.assertTrue(future.cancelled())
+                    with self.assertRaises(cf.CancelledError):
+                        future.jobid()
+                    with self.assertRaises(cf.CancelledError):
+                        future.exception()
+                else:
+                    self.assertEqual(future.result(), 1)
+                    self.assertIsNone(future.exception())
+        self.assertFalse(executor._broken_event.is_set())
+
+    def test_cancel_attach(self):
+        with FluxExecutor() as executor:
+            jobspec = JobspecV1.from_command(["true"])
+            jobid = executor.submit(jobspec).jobid()
+            for _ in range(3):
+                future = executor.attach(jobid)
+                if future.cancel():
+                    self.assertFalse(future.running())
+                    self.assertTrue(future.cancelled())
+                    self.assertEqual(future.jobid(), jobid)
+                    with self.assertRaises(cf.CancelledError):
+                        future.exception()
+                else:
+                    self.assertEqual(future.result(), 0)
+                    self.assertIsNone(future.exception())
+        self.assertFalse(executor._broken_event.is_set())
+
+    def test_bad_arguments(self):
+        with FluxExecutor() as executor:
+            submit_future = executor.submit(None)  # not a valid jobspec
+            attach_future = executor.attach(None)  # not a valid job ID
+            attach_future2 = executor.attach(-1)  # invalid, immediately rejected
+            # invalid job IDs but rejected only in callback
+            zero_jobids = [executor.attach(0) for _ in range(5)]
+        # all futures should be fulfilled after exiting context manager
+        with self.assertRaisesRegex(RuntimeError, r"job could not be submitted.*"):
+            # trying to fetch jobid should raise an error
+            submit_future.jobid()
+        with self.assertRaises(OSError):
+            submit_future.result(timeout=0)
+        self.assertIsInstance(submit_future.exception(), OSError)
+        self.assertEqual(attach_future.jobid(), None)
+        with self.assertRaises(TypeError):
+            attach_future.result(timeout=0)
+        self.assertEqual(attach_future2.jobid(), -1)
+        with self.assertRaises(OverflowError):
+            attach_future2.result(timeout=0)
+        for future in zero_jobids:
+            self.assertEqual(future.jobid(), 0)
+            with self.assertRaisesRegex(ValueError, r".*does not match any job.*"):
+                future.result(timeout=0)
+        self.assertFalse(executor._broken_event.is_set())
+
+    def test_submit_after_shutdown(self):
+        executor = FluxExecutor()
+        executor.shutdown(wait=True)
+        with self.assertRaises(RuntimeError):
+            executor.submit(JobspecV1.from_command(["true"]))
+        with self.assertRaises(RuntimeError):
+            executor.submit(None)
+        with self.assertRaises(RuntimeError):
+            executor.attach(5)
+        with self.assertRaises(RuntimeError):
+            executor.attach(None)
+        self.assertFalse(executor._broken_event.is_set())
+
+    def test_wait(self):
+        with FluxExecutor(threads=3) as executor:
+            jobspec = JobspecV1.from_command(["false"])
+            futures = [executor.submit(jobspec) for _ in range(3)]
+            done, not_done = cf.wait(futures, return_when=cf.FIRST_COMPLETED)
+            self._check_done(done)
+            done, not_done = cf.wait(futures, return_when=cf.FIRST_EXCEPTION)
+            self._check_done(done)
+            done, not_done = cf.wait(futures)
+            self._check_done(done)
+            self.assertEqual(len(not_done), 0)
+        self.assertFalse(executor._broken_event.is_set())
+
+    def _check_done(self, done_futures):
+        self.assertGreater(len(done_futures), 0)
+        for fut in done_futures:
+            self.assertEqual(fut.result(timeout=0), 1)
+
+    def test_executor_event_callbacks(self):
+        with FluxExecutor() as executor:
+            expected_events = set(["start", "finish", "depend", "priority", "free"])
+            future = executor.submit(JobspecV1.from_command(["false"]))
+            for event in executor.EVENTS:
+                future.add_event_callback(
+                    event, lambda fut, event: expected_events.discard(event.name)
+                )
+        self.assertFalse(expected_events)  # no more expected events
+        self.assertFalse(executor._broken_event.is_set())
+
+    def test_exception_event(self):
+        with FluxExecutor() as executor:
+            flag = threading.Event()
+            future = executor.submit(JobspecV1.from_command(["/not/a/real/app"]))
+            future.add_event_callback("exception", lambda fut, event: flag.set())
+            self.assertIsInstance(future.exception(), JobException)
+            self.assertTrue(flag.is_set())
+            # repeat the test, attaching to the same job
+            jobid = future.jobid()
+            flag = threading.Event()
+            future = executor.attach(jobid)
+            self.assertEqual(jobid, future.jobid())
+            future.add_event_callback("exception", lambda fut, event: flag.set())
+            self.assertIsInstance(future.exception(), JobException)
+            self.assertTrue(flag.is_set())
+        self.assertFalse(executor._broken_event.is_set())
+
+    def test_broken_executor(self):
+        with FluxExecutor() as executor:
+            executor._broken_event.set()
+            with self.assertRaisesRegex(RuntimeError, "Executor is broken.*"):
+                executor.submit(JobspecV1.from_command(["/not/a/real/app"]))
+            with self.assertRaisesRegex(RuntimeError, "Executor is broken.*"):
+                executor.attach(25979)
+
+
+class TestFluxExecutorThread(unittest.TestCase):
+    """Simple synchronous tests for _FluxExecutorThread."""
+
+    def test_exit_condition(self):
+        deq = collections.deque()
+        event = threading.Event()
+        thread = _FluxExecutorThread(threading.Event(), event, deq, 0.01, (), {})
+        self.assertTrue(thread._FluxExecutorThread__work_remains())
+        event.set()
+        self.assertFalse(thread._FluxExecutorThread__work_remains())
+        deq.append(None)
+        self.assertTrue(thread._FluxExecutorThread__work_remains())
+
+    def test_bad_jobspecs(self):
+        deq = collections.deque()
+        event = threading.Event()
+        thread = _FluxExecutorThread(threading.Event(), event, deq, 0.01, (), {})
+        futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
+        deq.extend(
+            _SubmitPackage((None,), {}, f) for f in futures
+        )  # send jobspec of None
+        event.set()
+        thread.run()
+        self.assertFalse(deq)
+        self.assertFalse(thread._FluxExecutorThread__running_user_futures)
+        for fut in futures:
+            self.assertIsInstance(fut.exception(), OSError)
+
+    def test_bad_submit_arguments(self):
+        """send bad arguments to ``flux.job.submit``"""
+        deq = collections.deque()
+        event = threading.Event()
+        thread = _FluxExecutorThread(threading.Event(), event, deq, 0.01, (), {})
+        futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
+        jobspec = JobspecV1.from_command(["false"])
+        deq.extend(_SubmitPackage((jobspec,), {"not_an_arg": 42}, f) for f in futures)
+        event.set()
+        thread.run()
+        self.assertFalse(deq)
+        self.assertFalse(thread._FluxExecutorThread__running_user_futures)
+        for fut in futures:
+            self.assertIsInstance(fut.exception(), TypeError)
+
+    def test_bad_attach_arguments(self):
+        deq = collections.deque()
+        event = threading.Event()
+        thread = _FluxExecutorThread(threading.Event(), event, deq, 0.01, (), {})
+        futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
+        deq.extend(_AttachPackage(None, f) for f in futures)  # send jobid of None
+        event.set()
+        thread.run()
+        self.assertFalse(deq)
+        self.assertFalse(thread._FluxExecutorThread__running_user_futures)
+        for fut in futures:
+            self.assertIsInstance(fut.exception(), TypeError)
+        futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
+        deq.extend(_AttachPackage(0, f) for f in futures)  # send jobid of None
+        event.set()
+        thread.run()
+        self.assertFalse(deq)
+        self.assertFalse(thread._FluxExecutorThread__running_user_futures)
+        for fut in futures:
+            with self.assertRaisesRegex(ValueError, r".*does not match any job.*"):
+                fut.result(timeout=0)
+
+    def test_cancel(self):
+        deq = collections.deque()
+        event = threading.Event()
+        jobspec = JobspecV1.from_command(["false"])
+        thread = _FluxExecutorThread(threading.Event(), event, deq, 0.01, (), {})
+        futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
+        for fut in futures:
+            deq.append(_SubmitPackage((jobspec,), {}, fut))
+            fut.cancel()
+        event.set()
+        thread.run()
+        for fut in futures:
+            with self.assertRaises(cf.CancelledError):
+                fut.result()
+            with self.assertRaises(cf.CancelledError):
+                fut.jobid()
+
+    def test_exception_completion(self):
+        thread = _FluxExecutorThread(
+            threading.Event(), threading.Event(), collections.deque(), 0.01, (), {}
+        )
+        fut = FluxExecutorFuture(threading.get_ident())
+        self.assertFalse(fut.done())
+        fut._set_event(EventLogEvent({"name": "start", "timestamp": 0}))
+        self.assertFalse(fut.done())
+        thread._FluxExecutorThread__event_update(
+            ShamJobEventWatchFuture(
+                EventLogEvent(
+                    {
+                        "name": "exception",
+                        "timestamp": 0,
+                        "context": {"severity": 1, "type": "foobar", "note": ""},
+                    }
+                )
+            ),
+            fut,
+        )
+        self.assertFalse(fut.done())
+        thread._FluxExecutorThread__event_update(
+            ShamJobEventWatchFuture(
+                EventLogEvent(
+                    {
+                        "name": "exception",
+                        "timestamp": 0,
+                        "context": {"severity": 0, "type": "foobar", "note": ""},
+                    }
+                )
+            ),
+            fut,
+        )
+        self.assertTrue(fut.done())
+        self.assertIsInstance(fut.exception(), JobException)
+
+    def test_finish_completion(self):
+        thread = _FluxExecutorThread(
+            threading.Event(), threading.Event(), collections.deque(), 0.01, (), {}
+        )
+        for exit_status in (0, 1, 15, 120, 255):
+            flag = threading.Event()
+            fut = FluxExecutorFuture(threading.get_ident()).add_done_callback(
+                lambda fut: flag.set()
+            )
+            thread._FluxExecutorThread__event_update(
+                ShamJobEventWatchFuture(
+                    EventLogEvent(
+                        {
+                            "name": "finish",
+                            "timestamp": 0,
+                            "context": {"status": exit_status},
+                        }
+                    )
+                ),
+                fut,
+            )
+            self.assertTrue(fut.done())
+            self.assertTrue(flag.is_set())
+            if os.WIFEXITED(exit_status):
+                self.assertEqual(fut.result(), os.WEXITSTATUS(exit_status))
+            elif os.WIFSIGNALED(exit_status):
+                self.assertEqual(fut.result(), -os.WTERMSIG(exit_status))
+
+    def test_exception_catching(self):
+        """Test that the thread exits cleanly when __run() raises."""
+
+        def new_run(*args):
+            raise TypeError("foobar")
+
+        error_event = threading.Event()
+        stop_event = threading.Event()
+        stop_event.set()
+        futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
+        attach_futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
+        running_futures = set(
+            [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
+        )
+        deque = collections.deque([_SubmitPackage((None,), {}, fut) for fut in futures])
+        deque.extend([_AttachPackage(None, fut) for fut in attach_futures])
+        thread = _FluxExecutorThread(error_event, stop_event, deque, 0.01, (), {})
+        thread._FluxExecutorThread__running_user_futures.update(running_futures)
+        # replace the __run method
+        thread._FluxExecutorThread__run = types.MethodType(new_run, thread)
+        with self.assertRaisesRegex(TypeError, "foobar"):
+            thread.run()
+        self.assertTrue(error_event.is_set())
+        for fut in list(futures) + list(running_futures) + list(attach_futures):
+            with self.assertRaisesRegex(TypeError, "foobar"):
+                fut.result()
+
+    def test_exception_catching_reactor_run(self):
+        """Test that the thread exits cleanly when reactor_run returns < 0"""
+
+        def new_reactor_run(*args, **kwargs):
+            # returning < 0 causes the thread to raise
+            return -1
+
+        jobspec = JobspecV1.from_command(["true"])
+        error_event = threading.Event()
+        stop_event = threading.Event()
+        stop_event.set()
+        futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
+        deque = collections.deque(
+            [_SubmitPackage((jobspec,), {}, fut) for fut in futures]
+        )
+        thread = _FluxExecutorThread(error_event, stop_event, deque, 0.01, (), {})
+        # replace the reactor_run method of the thread's flux handle
+        thread._FluxExecutorThread__flux_handle.reactor_run = types.MethodType(
+            new_reactor_run, thread._FluxExecutorThread__flux_handle
+        )
+        with self.assertRaisesRegex(RuntimeError, "reactor start failed"):
+            thread.run()
+        self.assertTrue(error_event.is_set())
+        for fut in list(futures):
+            with self.assertRaisesRegex(RuntimeError, "reactor start failed"):
+                fut.result()
+
+
+class TestFluxExecutorFuture(unittest.TestCase):
+    """Tests for FluxExecutorFuture."""
+
+    def test_set_jobid(self):
+        for jobid in (21, 594240, None, -1, "foobar"):
+            fut = FluxExecutorFuture(threading.get_ident())
+            with self.assertRaises(cf.TimeoutError):
+                fut.jobid(timeout=0)
+            fut._set_jobid(jobid)
+            self.assertEqual(jobid, fut.jobid(timeout=0))
+            with self.assertRaises(RuntimeError):
+                fut._set_jobid(jobid)
+
+    def test_jobid_unavailable(self):
+        """Tests for cases where fetching the jobid should raise an exception."""
+        fut = FluxExecutorFuture(threading.get_ident())
+        fut._set_jobid(None, exc=OverflowError("foobar"))
+        with self.assertRaisesRegex(OverflowError, "foobar"):
+            fut.jobid(0)
+        with self.assertRaises(RuntimeError):
+            fut._set_jobid(1)
+        # test that jobid raises if future is cancelled
+        fut = FluxExecutorFuture(threading.get_ident())
+        fut.cancel()
+        with self.assertRaises(cf.CancelledError):
+            fut.jobid(0)
+        # test that jobid doesn't raise if future is cancelled but jobid already set
+        fut = FluxExecutorFuture(threading.get_ident())
+        jobid = 10
+        fut._set_jobid(jobid)
+        fut.cancel()
+        self.assertEqual(fut.jobid(0), jobid)
+        # test that jobid raises if an exception is set before jobid set
+        fut = FluxExecutorFuture(threading.get_ident())
+        fut.set_exception(ValueError("foobar"))
+        with self.assertRaisesRegex(RuntimeError, r"job could not be submitted.*"):
+            fut.jobid()
+
+    def test_jobid_callback(self):
+        fut = FluxExecutorFuture(threading.get_ident())
+        flag = threading.Event()
+        fut.add_jobid_callback(lambda f: flag.set())
+        self.assertFalse(flag.is_set())
+        fut._set_jobid(5)
+        self.assertTrue(flag.is_set())
+        # now check that adding callbacks fires them immediately
+        flag.clear()
+        fut.add_jobid_callback(lambda f: flag.set())
+        self.assertTrue(flag.is_set())
+
+    def test_event_callback(self):
+        fut = FluxExecutorFuture(threading.get_ident())
+        flag = threading.Event()
+        for state in ("clean", "start", "alloc"):
+            flag.clear()
+            fut.add_event_callback(
+                state, lambda f, log: (flag.set(), self.assertEqual(log.name, state))
+            )
+            log_event = EventLogEvent({"name": state, "timestamp": 0})
+            fut._set_event(log_event)
+            self.assertTrue(flag.is_set())
+            # now check that adding callbacks fires them immediately
+            flag.clear()
+            fut.add_event_callback(
+                state, lambda f, log: (flag.set(), self.assertEqual(log.name, state))
+            )
+            self.assertTrue(flag.is_set())
+
+    def test_bad_event(self):
+        fut = FluxExecutorFuture(threading.get_ident())
+        event = "not an event"
+        log_event = EventLogEvent({"name": event, "timestamp": 0})
+        with self.assertRaises(ValueError):
+            fut._set_event(log_event)
+        with self.assertRaises(ValueError):
+            fut.add_event_callback(event, None)
+
+    def test_callback_deadlock(self):
+        flag = threading.Event()
+        # try waiting for the future from the callback
+        fut = FluxExecutorFuture(threading.get_ident()).add_jobid_callback(
+            lambda fut: (fut.result(), flag.set())
+        )
+        fut._set_jobid(5)
+        # check that flag wasn't set---because the callback raised
+        self.assertFalse(flag.is_set())
+        # try the same with an event callback
+        log_event = EventLogEvent({"name": "debug", "timestamp": 0})
+        fut.add_event_callback(
+            log_event.name, lambda fut, event: (fut.result(), flag.set())
+        )
+        fut._set_event(log_event)
+        self.assertFalse(flag.is_set())
+        # now complete the future and try again
+        fut.set_result(21)
+        fut.add_jobid_callback(lambda fut: (fut.result(), flag.set()))
+        self.assertTrue(flag.is_set())
+        flag.clear()
+        fut.add_event_callback(
+            log_event.name, lambda fut, event: (fut.result(), flag.set())
+        )
+        self.assertTrue(flag.is_set())
+
+    def test_multiple_events(self):
+        fut = FluxExecutorFuture(threading.get_ident())
+        counter = itertools.count()
+        fut.add_event_callback("urgency", lambda fut, event: next(counter))
+        total_iterations = 5
+        for _ in range(5):
+            fut._set_event(EventLogEvent({"name": "urgency", "timestamp": 0}))
+        self.assertEqual(next(counter), total_iterations)
+        new_counter = itertools.count()
+        fut.add_event_callback("urgency", lambda fut, event: next(new_counter))
+        self.assertEqual(next(new_counter), total_iterations)
+        fut._set_event(EventLogEvent({"name": "urgency", "timestamp": 0}))
+        # invoking the callback increments the counter, as does checking the counter
+        self.assertEqual(next(counter), total_iterations + 2)
+        self.assertEqual(next(new_counter), total_iterations + 2)
+
+    def test_adding_callbacks_from_callbacks(self):
+        fut = FluxExecutorFuture(threading.get_ident())
+        flag = threading.Event()
+
+        def nested_callback(fut, event):
+            flag.set()
+
+        fut.add_event_callback(
+            "start", lambda fut, event: fut.add_event_callback("start", nested_callback)
+        )
+        fut._set_event(EventLogEvent({"name": "start", "timestamp": 0}))
+        self.assertTrue(flag.is_set())
+
+    def test_multiple_cancel(self):
+        fut = FluxExecutorFuture(threading.get_ident())
+        invocations = itertools.count()
+        fut.add_done_callback(lambda _: next(invocations))
+        self.assertFalse(fut.cancelled())
+        for _ in range(3):  # test cancelling more than once
+            self.assertTrue(fut.cancel())
+        self.assertEqual(next(invocations), 1)
+        with self.assertRaises(cf.CancelledError):
+            fut.jobid()
+
+
+if __name__ == "__main__":
+    from subflux import rerun_under_flux
+
+    if rerun_under_flux(size=__flux_size(), personality="job"):
+        from pycotap import TAPTestRunner
+
+        unittest.main(testRunner=TAPTestRunner())
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/python/t0024-util.py b/t/python/t0024-util.py
new file mode 100755
index 000000000000..d8b6c6792f11
--- /dev/null
+++ b/t/python/t0024-util.py
@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import os
+import time
+import unittest
+from collections import namedtuple
+from datetime import datetime
+
+import subflux  # noqa: F401 - To set up PYTHONPATH
+from flux.util import OutputFormat, UtilDatetime, parse_datetime
+from pycotap import TAPTestRunner
+
+
+def ts(year, month, day, hour=0, minute=0, sec=0, us=0):
+    return datetime(year, month, day, hour, minute, sec, us).astimezone().timestamp()
+
+
+class TestParseDatetime(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.now = datetime(2021, 6, 10, 8, 0, 0)
+        self.ts = self.now.timestamp()
+
+    def parse(self, string):
+        return parse_datetime(string, self.now).timestamp()
+
+    def parsedt(self, string):
+        return parse_datetime(string).astimezone()
+
+    def test_now(self):
+        # "now" returns parse_datetime(now=) when set
+        self.assertEqual(self.parse("now"), self.ts)
+        # "now" has better than second precision
+        now = parse_datetime("now").timestamp()
+        self.assertNotEqual(now % 1, 0)
+
+    def test_fsd(self):
+        self.assertEqual(self.parse("+1"), self.ts + 1)
+        self.assertEqual(self.parse("-1"), self.ts - 1)
+        self.assertEqual(self.parse("+1m"), self.ts + 60)
+        self.assertEqual(self.parse("+1.5m"), self.ts + 90)
+        self.assertEqual(self.parse("+1h"), self.ts + 3600)
+        self.assertEqual(self.parse("+100ms"), self.ts + 0.1)
+        self.assertEqual(self.parse("+1ms"), self.ts + 0.001)
+
+    def test_fsd_invalid(self):
+        with self.assertRaises(ValueError):
+            self.parse("+1ns")
+        with self.assertRaises(ValueError):
+            self.parse("+1x")
+        with self.assertRaises(ValueError):
+            self.parse("+x")
+        with self.assertRaises(ValueError):
+            self.parse("-")
+
+    def test_basic_datetime(self):
+        self.assertEqual(self.parse("6/10/2021 08:00"), self.ts)
+        self.assertEqual(self.parse("2021-06-10 08:00:00"), self.ts)
+        self.assertEqual(self.parse("Jun 10, 2021 8am"), self.ts)
+
+    def test_basic_invalid(self):
+        with self.assertRaises(ValueError):
+            self.parse("blursday")
+        with self.assertRaises(ValueError):
+            self.parse("6/45/2021")
+
+    def test_nlp(self):
+        self.assertEqual(self.parse("in 10 min"), self.ts + 600)
+        self.assertEqual(self.parse("tomorrow"), ts(2021, 6, 11))
+        self.assertEqual(self.parse("noon tomorrow"), ts(2021, 6, 11, 12))
+        self.assertEqual(self.parse("last wed"), ts(2021, 6, 9))
+
+
+class TestUtilDatetime(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.zero = UtilDatetime.fromtimestamp(0.0)
+        self.ts = UtilDatetime(2021, 6, 10, 8, 0, 0)
+
+    def test_zero(self):
+        self.assertEqual(f"{self.zero}", "")
+        self.assertEqual(f"{self.zero:::h}", "-")
+        self.assertEqual(f"{self.zero:%FT%T::<6}", "      ")
+        self.assertEqual(f"{self.zero:%FT%T::<6h}", "-     ")
+
+    def test_fmt(self):
+        self.assertEqual(f"{self.ts}", "2021-06-10T08:00:00")
+        self.assertEqual(f"{self.ts:::h}", "2021-06-10T08:00:00")
+        self.assertEqual(f"{self.ts:%b%d %R}", "Jun10 08:00")
+        self.assertEqual(f"{self.ts:%b%d %R::h}", "Jun10 08:00")
+        self.assertEqual(f"{self.ts:%b%d %R::>12}", " Jun10 08:00")
+        self.assertEqual(f"{self.ts:%b%d %R::>12h}", " Jun10 08:00")
+
+
+class Item:
+    def __init__(self, s="foo", i=42, f=1234.567, d=978336000.0):
+        self.s = s
+        self.i = i
+        self.f = f
+        self.d = d
+
+
+TestData = namedtuple(
+    "TestData",
+    (
+        "input",
+        "fmt",
+        "hdrfmt",
+        "fields",
+        "header",
+        "result",
+    ),
+)
+
+
+class TestOutputFormat(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+
+        # Set timezone to UTC so rendered datetimes match expected value
+        os.environ["TZ"] = "UTC"
+        time.tzset()
+
+        self.headings = {
+            "s": "STRING",
+            "i": "INTEGER",
+            "f": "FLOAT",
+            "d": "DATETIME",
+        }
+        self.item = Item()
+        self.cases = [
+            TestData("", "", "", [], "", ""),
+            TestData("{s}", "{0.s}", "{s}", ["s"], "STRING", "foo"),
+            TestData("{i}", "{0.i}", "{i}", ["i"], "INTEGER", "42"),
+            TestData("{f}", "{0.f}", "{f}", ["f"], "FLOAT", "1234.567"),
+            TestData("{d}", "{0.d}", "{d}", ["d"], "DATETIME", "978336000.0"),
+            TestData(
+                "{s:<6} {i:>7d}",
+                "{0.s:<6} {0.i:>7d}",
+                "{s:<6} {i:>7}",
+                ["s", "i"],
+                "STRING INTEGER",
+                "foo         42",
+            ),
+            TestData(
+                "{s:<6} {f:>8.2f}",
+                "{0.s:<6} {0.f:>8.2f}",
+                "{s:<6} {f:>8}",
+                ["s", "f"],
+                "STRING    FLOAT",
+                "foo     1234.57",
+            ),
+            TestData(
+                "a {s:<6} b{f:>8.2f} ",
+                "a {0.s:<6} b{0.f:>8.2f} ",
+                "a {s:<6} b{f:>8} ",
+                ["s", "f"],
+                "a STRING b   FLOAT ",
+                "a foo    b 1234.57 ",
+            ),
+            TestData(
+                "{i:>7d} {d!d:%b%d %R::^20}",
+                "{0.i:>7d} {0.d!d:%b%d %R::^20}",
+                "{i:>7} {d:^20}",
+                ["i", "d"],
+                "INTEGER       DATETIME      ",
+                "     42     Jan01 08:00     ",
+            ),
+            TestData(
+                "sort:i,-d {i:>7d} {d!d:%b%d %R::^20}",
+                "{0.i:>7d} {0.d!d:%b%d %R::^20}",
+                "{i:>7} {d:^20}",
+                ["i", "d"],
+                "INTEGER       DATETIME      ",
+                "     42     Jan01 08:00     ",
+            ),
+        ]
+
+    def test_basic(self):
+        for t in self.cases:
+            fmt = OutputFormat(t.input, headings=self.headings)
+            self.assertEqual(fmt.get_format(include_sort_prefix=False), t.fmt)
+            self.assertEqual(fmt.header_format(), t.hdrfmt)
+            self.assertCountEqual(fmt.fields, t.fields)
+            self.assertEqual(fmt.header(), t.header)
+            self.assertEqual(fmt.format(self.item), t.result)
+
+    def test_filter(self):
+        a = Item("", 0, 2.2)
+        d = Item("", 1, 1.1)
+        z = Item("", 23456789, 1.0)
+
+        items = [a, d, z]
+        fmt = OutputFormat(
+            "?+:{i:>7} ?:{s:>6} ?:{f:8.2}", headings=self.headings
+        ).filter(items)
+        self.assertEqual(fmt, "{i:>8} {f:8.2}")
+
+        fmt = OutputFormat(
+            "?+:{i:>7} ?:{s:>6} ?+:{f:.2}", headings=self.headings
+        ).filter(items)
+        self.assertEqual(fmt, "{i:>8} {f:3.2}")
+
+    def test_sort(self):
+        a = Item("a", 0, 2.2)
+        d = Item("d", 33, 1.1)
+        z = Item("z", 11, 1.0)
+
+        items = [z, a, d]
+        formatter = OutputFormat("{s}:{i}:{f}", headings=self.headings)
+        formatter.set_sort_keys("s")
+        formatter.sort_items(items)
+        self.assertListEqual(items, [a, d, z])
+
+        items = [z, a, d]
+        formatter.set_sort_keys("-s")
+        formatter.sort_items(items)
+        self.assertListEqual(items, [z, d, a])
+
+        items = [z, a, d]
+        formatter.set_sort_keys("i")
+        formatter.sort_items(items)
+        self.assertListEqual(items, [a, z, d])
+
+        items = [z, a, d]
+        formatter.set_sort_keys("f")
+        formatter.sort_items(items)
+        self.assertListEqual(items, [z, d, a])
+
+        # Embedded sort prefix works
+        items = [z, a, d]
+        formatter = OutputFormat("sort:i {s}:{i}:{f}", headings=self.headings)
+        formatter.sort_items(items)
+        self.assertListEqual(items, [a, z, d])
+
+        # Embedded sort prefix can be overridden
+        items = [z, a, d]
+        formatter = OutputFormat("sort:i {s}:{i}:{f}", headings=self.headings)
+        formatter.set_sort_keys("f")
+        formatter.sort_items(items)
+        self.assertListEqual(items, [z, d, a])
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t0025-uri.py b/t/python/t0025-uri.py
new file mode 100755
index 000000000000..538f74e516ed
--- /dev/null
+++ b/t/python/t0025-uri.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import platform
+import unittest
+
+import subflux  # noqa: F401 - To set up PYTHONPATH
+from flux.uri import JobURI
+from pycotap import TAPTestRunner
+
+
+class TestJobURI(unittest.TestCase):
+    def test_parse_remote(self):
+        uri = JobURI("ssh://foo.com/tmp/foo?tag=bar&x=y")
+        self.assertEqual(uri.uri, "ssh://foo.com/tmp/foo?tag=bar&x=y")
+        self.assertEqual(uri.remote, "ssh://foo.com/tmp/foo?tag=bar&x=y")
+        self.assertEqual(uri.local, "local:///tmp/foo")
+        self.assertEqual(uri.scheme, "ssh")
+        self.assertEqual(uri.netloc, "foo.com")
+        self.assertEqual(uri.path, "/tmp/foo")
+        self.assertEqual(uri.query, "tag=bar&x=y")
+        self.assertEqual(uri.fragment, "")
+        self.assertEqual(uri.params, "")
+
+    def test_parse_local(self):
+        hostname = platform.uname()[1]
+        uri = JobURI("local:///tmp/foo")
+        self.assertEqual(uri.uri, "local:///tmp/foo")
+        self.assertEqual(str(uri), "local:///tmp/foo")
+        self.assertEqual(uri.remote, f"ssh://{hostname}/tmp/foo")
+        self.assertEqual(uri.local, "local:///tmp/foo")
+        self.assertEqual(uri.scheme, "local")
+        self.assertEqual(uri.netloc, "")
+        self.assertEqual(uri.path, "/tmp/foo")
+        self.assertEqual(uri.query, "")
+        self.assertEqual(uri.fragment, "")
+        self.assertEqual(uri.params, "")
+
+    def test_parse_errors(self):
+        with self.assertRaises(ValueError):
+            JobURI("foo:///tmp/bar").remote
+        with self.assertRaises(ValueError):
+            JobURI("foo:///tmp/bar").local
+        with self.assertRaises(ValueError):
+            JobURI("")
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t0026-tree.py b/t/python/t0026-tree.py
new file mode 100755
index 000000000000..7213df79b7a4
--- /dev/null
+++ b/t/python/t0026-tree.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2022 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import io
+import os
+import unittest
+import unittest.mock
+
+import subflux  # noqa: F401 - To set up PYTHONPATH
+from flux.util import Tree
+from pycotap import TAPTestRunner
+
+
+class TestTree(unittest.TestCase):
+    @unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
+    def assertTreeRender(
+        self,
+        tree,
+        mock_stdout,
+        style="box",
+        level=None,
+        skip_root=False,
+        truncate=True,
+        expected_output=None,
+    ):
+        tree.render(style=style, level=level, skip_root=skip_root, truncate=truncate)
+        self.assertEqual(mock_stdout.getvalue(), expected_output)
+
+    @classmethod
+    def setUpClass(self):
+        self.trees = []
+        for combine in [True, False]:
+            root = Tree("Root", combine_children=combine)
+            child = Tree("flux", combine_children=combine)
+            child.append("sleep")
+
+            child2 = child.append("flux")
+            child2.append("foo")
+            child2.append("bar")
+
+            child.append("sleep")
+            child.append("sleep")
+
+            root.append_tree(child)
+            root.append("sleep")
+            root.append("sleep")
+            root.append("sleep")
+            self.trees.append(root)
+
+    def test_create_tree(self):
+        for tree in self.trees:
+            self.assertIsNotNone(tree)
+
+    def test_render_not_combined(self):
+        expected_output = """\
+Root
+├── flux
+│   ├── sleep
+│   ├── flux
+│   │   ├── foo
+│   │   └── bar
+│   ├── sleep
+│   └── sleep
+├── sleep
+├── sleep
+└── sleep
+"""
+        self.assertTreeRender(self.trees[1], expected_output=expected_output)
+
+    def test_render_combined(self):
+        expected_output = """\
+Root
+├── flux
+│   ├── 3*[sleep]
+│   └── flux
+│       ├── foo
+│       └── bar
+└── 3*[sleep]
+"""
+        self.assertTreeRender(self.trees[0], expected_output=expected_output)
+
+    def test_render_skip_root(self):
+        expected_output = """\
+flux
+├── 3*[sleep]
+└── flux
+    ├── foo
+    └── bar
+3*[sleep]
+"""
+        self.assertTreeRender(
+            self.trees[0], skip_root=True, expected_output=expected_output
+        )
+
+    def test_render_level0(self):
+        expected_output = """\
+Root
+"""
+        self.assertTreeRender(self.trees[1], level=0, expected_output=expected_output)
+
+    def test_render_level1(self):
+        expected_output = """\
+Root
+├── flux
+├── sleep
+├── sleep
+└── sleep
+"""
+        self.assertTreeRender(self.trees[1], level=1, expected_output=expected_output)
+
+    def test_render_level2(self):
+        expected_output = """\
+Root
+├── flux
+│   ├── sleep
+│   ├── flux
+│   ├── sleep
+│   └── sleep
+├── sleep
+├── sleep
+└── sleep
+"""
+        self.assertTreeRender(self.trees[1], level=2, expected_output=expected_output)
+
+    def test_render_style_compact(self):
+        expected_output = """\
+Root
+├─flux
+│ ├─3*[sleep]
+│ └─flux
+│   ├─foo
+│   └─bar
+└─3*[sleep]
+"""
+        self.assertTreeRender(
+            self.trees[0], style="compact", expected_output=expected_output
+        )
+
+    def test_render_style_ascii(self):
+        expected_output = """\
+Root
+|-- flux
+|   |-- sleep
+|   |-- flux
+|   |   |-- foo
+|   |   `-- bar
+|   |-- sleep
+|   `-- sleep
+|-- sleep
+|-- sleep
+`-- sleep
+"""
+        self.assertTreeRender(
+            self.trees[1], style="ascii", expected_output=expected_output
+        )
+
+    def test_render_truncate(self):
+        expected_output = """\
+Root
+|-- flux
+|   |-- sl+
+|   |-- fl+
+|   |   |-+
+|   |   `-+
+|   |-- sl+
+|   `-- sl+
+|-- sleep
+|-- sleep
+`-- sleep
+"""
+        columns = os.environ.get("COLUMNS")
+        os.environ["COLUMNS"] = "11"
+        self.assertTreeRender(
+            self.trees[1], style="ascii", truncate=True, expected_output=expected_output
+        )
+        if columns:
+            os.environ["COLUMNS"] = columns
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t0027-constraint-parser.py b/t/python/t0027-constraint-parser.py
new file mode 100755
index 000000000000..892abdb01c30
--- /dev/null
+++ b/t/python/t0027-constraint-parser.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import unittest
+
+import subflux  # noqa: F401 - To set up PYTHONPATH
+from flux.constraint.parser import ConstraintParser, ConstraintSyntaxError
+from pycotap import TAPTestRunner
+
+VALID = [
+    {"in": "a", "out": {"t": ["a"]}},
+    {"in": "(a)", "out": {"t": ["a"]}},
+    {"in": "( a )", "out": {"t": ["a"]}},
+    {"in": "'a'", "out": {"t": ["a"]}},
+    {"in": "('a')", "out": {"t": ["a"]}},
+    {"in": "a|b", "out": {"or": [{"t": ["a"]}, {"t": ["b"]}]}},
+    {"in": "a or b", "out": {"or": [{"t": ["a"]}, {"t": ["b"]}]}},
+    {"in": "a || b", "out": {"or": [{"t": ["a"]}, {"t": ["b"]}]}},
+    {"in": "a||b", "out": {"or": [{"t": ["a"]}, {"t": ["b"]}]}},
+    {"in": "a b c", "out": {"and": [{"t": ["a"]}, {"t": ["b"]}, {"t": ["c"]}]}},
+    {"in": "a&b&c", "out": {"and": [{"t": ["a"]}, {"t": ["b"]}, {"t": ["c"]}]}},
+    {"in": "a and b and c", "out": {"and": [{"t": ["a"]}, {"t": ["b"]}, {"t": ["c"]}]}},
+    {"in": "a && b && c", "out": {"and": [{"t": ["a"]}, {"t": ["b"]}, {"t": ["c"]}]}},
+    {
+        "in": "a&b|c",
+        "out": {"or": [{"and": [{"t": ["a"]}, {"t": ["b"]}]}, {"t": ["c"]}]},
+    },
+    {
+        "in": "a&(b|c)",
+        "out": {"and": [{"t": ["a"]}, {"or": [{"t": ["b"]}, {"t": ["c"]}]}]},
+    },
+    {
+        "in": "(a&(b|c))",
+        "out": {"and": [{"t": ["a"]}, {"or": [{"t": ["b"]}, {"t": ["c"]}]}]},
+    },
+    {
+        "in": "(a&(b|-c))",
+        "out": {"and": [{"t": ["a"]}, {"or": [{"t": ["b"]}, {"not": [{"t": ["c"]}]}]}]},
+    },
+    {
+        "in": "a or host:foo",
+        "out": {"or": [{"t": ["a"]}, {"host": ["foo"]}]},
+    },
+    {
+        "in": "a or -host:foo",
+        "out": {"or": [{"t": ["a"]}, {"not": [{"host": ["foo"]}]}]},
+    },
+    {
+        "in": "a or time:11:00am",
+        "out": {"or": [{"t": ["a"]}, {"time": ["11:00am"]}]},
+    },
+    {
+        "in": "a or test:'a quoted string'",
+        "out": {"or": [{"t": ["a"]}, {"test": ["a quoted string"]}]},
+    },
+    {
+        "in": "a or name:'/foo|(bar)|baz.*/'",
+        "out": {"or": [{"t": ["a"]}, {"name": ["/foo|(bar)|baz.*/"]}]},
+    },
+]
+
+INVALID = ["a|", "a:' ", "(a", "(a))", "-(a|b)", "4g,host:foo", "foo and a:"]
+
+
+class TestConstraintParser(ConstraintParser):
+    operator_map = {None: "t", "a": "op-a"}
+
+
+class TestSplitParser(ConstraintParser):
+    operator_map = {None: "t"}
+    split_values = {"t": ","}
+
+
+class TestCombineParser(ConstraintParser):
+    operator_map = {None: "t"}
+    combined_terms = set("t")
+
+
+class TestParser(unittest.TestCase):
+    def test_parse_valid(self):
+        parser = TestConstraintParser()
+        for test in VALID:
+            print(f"checking `{test['in']}'")
+            result = parser.parse(test["in"])
+            self.assertDictEqual(test["out"], result)
+
+    def test_parse_invalid(self):
+        parser = TestConstraintParser()
+        for test in INVALID:
+            print(f"checking invalid input `{test}'")
+            with self.assertRaises((SyntaxError, ConstraintSyntaxError)):
+                parser.parse(test)
+
+    def test_default_parser(self):
+        parser = ConstraintParser()
+        print("ConstraintParser requires operator by default")
+        with self.assertRaises(ConstraintSyntaxError):
+            parser.parse("foo")
+
+    def test_split_values(self):
+        parser = TestSplitParser()
+        print("ConstraintParser can split values if configured")
+        result = parser.parse("xx,yy")
+        print(f"got {result}")
+        self.assertDictEqual({"t": ["xx", "yy"]}, result)
+
+        result = parser.parse("xx")
+        print(f"got {result}")
+        self.assertDictEqual({"t": ["xx"]}, result)
+
+        result = parser.parse("t:xx,yy")
+        print(f"got {result}")
+        self.assertDictEqual({"t": ["xx", "yy"]}, result)
+
+    def test_combine_terms(self):
+        parser = TestCombineParser()
+        print("ConstraintParser doesn't combine unconfigured like terms")
+        result = parser.parse("a:xx b:yy")
+        print(f"got {result}")
+        self.assertDictEqual({"and": [{"a": ["xx"]}, {"b": ["yy"]}]}, result)
+
+        print("ConstraintParser combines configured like terms")
+        result = parser.parse("xx and yy")
+        print(f"got {result}")
+        self.assertDictEqual({"t": ["xx", "yy"]}, result)
+
+        print("ConstraintParser combines configured like terms (nested)")
+        result = parser.parse("not (xx and yy)")
+        print(f"got {result}")
+        self.assertDictEqual({"not": [{"t": ["xx", "yy"]}]}, result)
+
+        print("ConstraintParser doesn't combine like terms (or)")
+        result = parser.parse("xx|yy")
+        print(f"got {result}")
+        self.assertDictEqual({"or": [{"t": ["xx"]}, {"t": ["yy"]}]}, result)
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t0028-compat36.py b/t/python/t0028-compat36.py
new file mode 100755
index 000000000000..6a9ef60135be
--- /dev/null
+++ b/t/python/t0028-compat36.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import signal
+import unittest
+
+import flux.compat36
+import subflux  # noqa: F401 - To set up PYTHONPATH
+from pycotap import TAPTestRunner
+
+
+class TestCompat36(unittest.TestCase):
+    def test_strsignal(self):
+        # Cover getting all signals strings, sanity check
+        # values of most common ones.
+        # N.B. SIGSTKFLT not defined until Python 3.11, will get ValueError
+        for i in range(signal.SIGHUP, signal.SIGIO):
+            try:
+                flux.compat36.strsignal(i)
+            except ValueError:
+                pass
+
+        self.assertEqual(flux.compat36.strsignal(signal.SIGHUP), "Hangup")
+        self.assertEqual(flux.compat36.strsignal(signal.SIGINT), "Interrupt")
+        self.assertEqual(flux.compat36.strsignal(signal.SIGKILL), "Killed")
+        self.assertEqual(flux.compat36.strsignal(signal.SIGSEGV), "Segmentation Fault")
+        self.assertEqual(flux.compat36.strsignal(signal.SIGTERM), "Terminated")
+
+    def test_strsignal_invalid(self):
+        gotvalueerror = False
+
+        try:
+            flux.compat36.strsignal(0)
+        except ValueError:
+            gotvalueerror = True
+
+        self.assertIs(gotvalueerror, True)
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t0029-fileref.py b/t/python/t0029-fileref.py
new file mode 100755
index 000000000000..f6a4ca7a0a4b
--- /dev/null
+++ b/t/python/t0029-fileref.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import base64
+import random
+import stat
+import tempfile
+import unittest
+
+import subflux  # noqa: F401 -  To set up PYTHONPATH
+from flux.util import Fileref
+from pycotap import TAPTestRunner
+
+
+class TestFileref(unittest.TestCase):
+    def test_dict(self):
+        data = {"foo": True}
+        mode = stat.S_IFREG | 0o600
+        result = Fileref(data)
+        self.assertDictEqual(result, {"mode": mode, "data": data})
+
+        mode = stat.S_IFREG | 0o700
+        result = Fileref(data, perms=0o700)
+        self.assertDictEqual(result, {"mode": mode, "data": data})
+
+    def test_raw(self):
+        data = "this is a testfile\n"
+        mode = stat.S_IFREG | 0o600
+        encoding = "utf-8"
+        result = Fileref(data, encoding=encoding)
+        expected = {"mode": mode, "encoding": encoding, "data": data}
+        self.assertDictEqual(result, expected)
+
+        with self.assertRaises(ValueError):
+            Fileref(data, encoding="bad")
+
+    def test_file(self):
+
+        # utf-8
+        data = "This is a text file\n"
+        mode = stat.S_IFREG | 0o600
+        with tempfile.NamedTemporaryFile(mode="w") as fp:
+            fp.write(data)
+            fp.flush()
+            path = fp.name
+            result = Fileref(path)
+            expected = {
+                "mode": mode,
+                "encoding": "utf-8",
+                "data": data,
+                "size": len(data),
+            }
+            self.assertDictEqual(result, expected)
+
+        # binary
+        data = bytearray(map(random.getrandbits, (8,) * 32))
+        mode = stat.S_IFREG | 0o600
+        with tempfile.NamedTemporaryFile() as fp:
+            fp.write(data)
+            fp.flush()
+            path = fp.name
+            result = Fileref(path)
+            expected = {
+                "mode": mode,
+                "encoding": "base64",
+                "data": base64.b64encode(data).decode("utf-8"),
+                "size": 32,
+            }
+            self.assertDictEqual(result, expected)
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t0030-journal.py b/t/python/t0030-journal.py
new file mode 100755
index 000000000000..acbf8b663660
--- /dev/null
+++ b/t/python/t0030-journal.py
@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2024 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import errno
+import unittest
+
+import flux
+from flux.job import JobspecV1, JournalConsumer
+from subflux import rerun_under_flux
+
+
+def __flux_size():
+    return 2
+
+
+def waitall(flux_handle):
+    while True:
+        try:
+            flux.job.wait(flux_handle)
+        except OSError as exc:
+            if exc.errno == errno.ECHILD:
+                # No more jobs to wait for
+                break
+
+
+class TestJournalConsumer(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(self):
+        self.fh = flux.Flux()
+
+        # Run a couple jobs for historical data
+        jobspec = JobspecV1.from_command(["sleep", "0"])
+        for i in range(4):
+            flux.job.submit(self.fh, jobspec, waitable=True)
+        waitall(self.fh)
+
+    def test_history(self):
+        consumer = JournalConsumer(self.fh).start()
+
+        count = 0
+        events = []
+        while True:
+            # Read events until we get 4 clean events
+            event = consumer.poll(timeout=5.0)
+            events.append(event)
+            if event.name == "clean":
+                count += 1
+                if count == 4:
+                    break
+        consumer.stop()
+
+        # events are ordered
+        self.assertTrue(
+            all(
+                events[index].timestamp < events[index + 1].timestamp
+                for index in range(len(events) - 1)
+            )
+        )
+
+        # get timestamp of arbitrary event in history:
+        seq = 10
+        timestamp = events[seq].timestamp
+
+        # new consumer with since timestamp
+        consumer = JournalConsumer(self.fh, since=timestamp).start()
+        events2 = []
+        while True:
+            event = consumer.poll(timeout=5.0)
+            events2.append(event)
+            if event.timestamp == events[-1].timestamp:
+                break
+
+        consumer.stop()
+
+        # events2 should be equal to the events[seq+1:]:
+        self.assertListEqual(events2, events[seq + 1 :])
+
+        # ensure JournalEvent can be converted to a string:
+        print(f"{events[-1]}")
+
+    def test_async(self):
+        events = []
+        test_arg2 = 42
+        test_kw_arg = "foo"
+
+        count = [0]
+
+        def test_cb(event, arg2, kw_arg=None):
+            events.append(event)
+            self.assertEqual(arg2, test_arg2)
+            self.assertEqual(kw_arg, test_kw_arg)
+            if event.name == "clean":
+                count[0] += 1
+                if count[0] == 4:
+                    self.fh.reactor_stop()
+
+        consumer = JournalConsumer(self.fh).start()
+        consumer.set_callback(test_cb, test_arg2, kw_arg=test_kw_arg)
+
+        self.fh.reactor_run()
+
+        self.assertEqual(count[0], 4)
+
+        # historical events are ordered
+        self.assertTrue(
+            all(
+                events[index].timestamp < events[index + 1].timestamp
+                for index in range(len(events) - 1)
+            )
+        )
+
+        new_events = []
+
+        def new_cb(event):
+            if event is None:
+                # End of stream
+                self.fh.reactor_stop()
+            new_events.append(event)
+            if event.name == "clean":
+                consumer.stop()
+
+        # Reset callback to append to new_events
+        consumer.set_callback(new_cb)
+
+        jobid = flux.job.submit(self.fh, JobspecV1.from_command(["sleep", "0"]))
+
+        self.fh.reactor_run()
+
+        for event in new_events:
+            self.assertTrue(event.jobid == jobid)
+
+    def test_nonfull(self):
+        consumer = flux.job.JournalConsumer(self.fh, full=False).start()
+
+        events = []
+
+        def cb(event):
+            if event is None:
+                self.fh.reactor_stop()
+                return
+            events.append(event)
+            if event.name == "clean":
+                consumer.stop()
+
+        consumer.set_callback(cb)
+
+        jobid = flux.job.submit(self.fh, JobspecV1.from_command(["sleep", "0"]))
+
+        self.fh.reactor_run()
+
+        for event in events:
+            self.assertTrue(event.jobid == jobid)
+
+    def test_sentinel(self):
+        self.assertTrue(flux.job.journal.SENTINEL_EVENT.is_empty())
+        # Ensure empty event can be converted to a string:
+        print(flux.job.journal.SENTINEL_EVENT)
+
+        consumer = flux.job.JournalConsumer(self.fh, include_sentinel=True)
+        consumer.start()
+        while True:
+            event = consumer.poll(0.1)
+            if event == flux.job.journal.SENTINEL_EVENT and event.is_empty():
+                break
+
+    def test_sentinel_async(self):
+
+        def cb(event):
+            if event is None:
+                self.fh.reactor_stop()
+            elif event.is_empty():
+                consumer.stop()
+
+        consumer = flux.job.JournalConsumer(self.fh, include_sentinel=True)
+        consumer.start()
+        consumer.set_callback(cb)
+        self.fh.reactor_run()
+
+    def test_poll_timeout(self):
+
+        consumer = flux.job.JournalConsumer(self.fh, full=False).start()
+        with self.assertRaises(TimeoutError):
+            consumer.poll(timeout=0.1)
+        consumer.stop()
+
+    def test_poll_ENODATA(self):
+
+        consumer = flux.job.JournalConsumer(self.fh, full=False).start()
+        consumer.stop()
+        self.assertIsNone(consumer.poll(timeout=5.0))
+
+    def test_poll_RuntimeError(self):
+
+        with self.assertRaises(RuntimeError):
+            flux.job.JournalConsumer(self.fh).poll(5.0)
+
+    def test_poll_cb_set_before_start(self):
+
+        def cb(event):
+            if event is None:
+                self.fh.reactor_stop()
+            elif event.is_empty():
+                consumer.stop()
+
+        consumer = flux.job.JournalConsumer(self.fh, include_sentinel=True)
+        consumer.set_callback(cb)
+        consumer.start()
+        self.fh.reactor_run()
+
+    def test_poll_cb_reset(self):
+        """test that the consumer callback can be reset"""
+
+        def cb(event):
+            self.fail("incorrect callback called")
+
+        def cb2(event):
+            if event is None:
+                self.fh.reactor_stop()
+            elif event.is_empty():
+                consumer.stop()
+
+        consumer = flux.job.JournalConsumer(self.fh, include_sentinel=True)
+        consumer.set_callback(cb)
+        consumer.start()
+        consumer.set_callback(cb2)
+        self.fh.reactor_run()
+
+
+if __name__ == "__main__":
+    if rerun_under_flux(__flux_size()):
+        from pycotap import TAPTestRunner
+
+        unittest.main(testRunner=TAPTestRunner(), buffer=False)
diff --git a/t/python/t0031-conf-builtin.py b/t/python/t0031-conf-builtin.py
new file mode 100755
index 000000000000..14795efaae71
--- /dev/null
+++ b/t/python/t0031-conf-builtin.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2024 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import unittest
+
+import subflux  # noqa: F401
+from flux.conf_builtin import conf_builtin_get
+from pycotap import TAPTestRunner
+
+
+class TestConfBuiltin(unittest.TestCase):
+
+    def test_conf_builtin_get(self):
+        with self.assertRaises(ValueError):
+            conf_builtin_get("foo")
+
+        with self.assertRaises(ValueError):
+            conf_builtin_get("confdir", which="badarg")
+
+        self.assertIsNotNone(conf_builtin_get("confdir"))
+        self.assertIsNotNone(conf_builtin_get("confdir", which="intree"))
+        self.assertIsNotNone(conf_builtin_get("confdir", which="installed"))
+
+
+if __name__ == "__main__":
+    unittest.main(testRunner=TAPTestRunner())
diff --git a/t/python/t1000-service-add-remove.py b/t/python/t1000-service-add-remove.py
index 4cff9e09ce9c..205a475d00a2 100755
--- a/t/python/t1000-service-add-remove.py
+++ b/t/python/t1000-service-add-remove.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################
 # Copyright 2014 Lawrence Livermore National Security, LLC
@@ -10,14 +10,14 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 
-import unittest
 import errno
+import os
+import sys
+import unittest
 
 import flux
-from flux.message import Message
-from flux.core.inner import ffi
 from flux.constants import FLUX_MSGTYPE_REQUEST, FLUX_MSGTYPE_RESPONSE
-
+from flux.message import Message
 from subflux import rerun_under_flux
 
 
@@ -25,28 +25,22 @@ def __flux_size():
     return 2
 
 
-def service_add(f, name):
-    future = f.service_register(name)
-    return f.future_get(future, ffi.NULL)
-
-
-def service_remove(f, name):
-    future = f.service_unregister(name)
-    return f.future_get(future, ffi.NULL)
-
-
 class TestServiceAddRemove(unittest.TestCase):
     @classmethod
     def setUpClass(self):
         self.f = flux.Flux()
 
     def test_001_register_service(self):
-        rc = service_add(self.f, "foo")
-        self.assertEqual(rc, 0)
+        rc = self.f.service_register("foo").get()
+        self.assertEqual(rc, None)
+
+    def test_001_register_owner_service(self):
+        rc = self.f.service_register("owner").get()
+        self.assertEqual(rc, None)
 
     def test_002_service_add_eexist(self):
         with self.assertRaises(EnvironmentError) as e:
-            service_add(self.f, "foo")
+            self.f.service_register("foo").get()
         self.assertEqual(e.exception.errno, errno.EEXIST)
 
     def test_003_add_service_message_watcher(self):
@@ -55,11 +49,29 @@ def service_cb(f, t, msg, arg):
             self.assertEqual(rc, 0)
 
         self.f.watcher = self.f.msg_watcher_create(
-            service_cb, FLUX_MSGTYPE_REQUEST, "foo.*"
+            service_cb,
+            FLUX_MSGTYPE_REQUEST,
+            "foo.echo",
+            rolemask=flux.constants.FLUX_ROLE_ALL,
         )
         self.assertIsNotNone(self.f.watcher)
         self.f.watcher.start()
 
+    def test_003_add_owner_message_watcher(self):
+        def service_cb(f, t, msg, arg):
+            rc = f.respond(msg, msg.payload_str)
+            self.assertEqual(rc, 0)
+
+        # Note: default rolemask for handle.msg_watcher_create()
+        # should allow instance-owner only:
+        self.f.watcher2 = self.f.msg_watcher_create(
+            service_cb,
+            FLUX_MSGTYPE_REQUEST,
+            "owner.echo",
+        )
+        self.assertIsNotNone(self.f.watcher2)
+        self.f.watcher2.start()
+
     def test_004_service_rpc(self):
         cb_called = [False]  # So that cb_called[0] is mutable in inner function
         p = {"test": "foo"}
@@ -67,7 +79,7 @@ def test_004_service_rpc(self):
         def cb(f, t, msg, arg):
             cb_called[0] = True
             self.assertEqual(msg.payload, p)
-            f.reactor_stop(f.get_reactor())
+            f.reactor_stop()
 
         w2 = self.f.msg_watcher_create(cb, FLUX_MSGTYPE_RESPONSE, "foo.echo")
         w2.start()
@@ -80,22 +92,107 @@ def cb(f, t, msg, arg):
         ret = self.f.send(m)
         self.assertEqual(ret, 0)
 
-        ret = self.f.reactor_run(self.f.get_reactor(), 0)
+        ret = self.f.reactor_run()
         self.assertTrue(ret >= 0)
         self.assertTrue(cb_called[0])
         w2.stop()
         w2.destroy()
 
+    def send_user_request(self, topic, payload):
+        m = Message()
+        m.topic = topic
+        m.payload = payload
+        m.pimpl.set_rolemask(flux.constants.FLUX_ROLE_USER)
+        m.pimpl.set_userid(os.getuid() + 1)
+        self.assertTrue(m is not None)
+        ret = self.f.send(m)
+        self.assertEqual(ret, 0)
+
+    def test_004_service_rpc_user(self):
+        cb_called = [False]  # So that cb_called[0] is mutable in inner function
+        p = {"test": "bar"}
+
+        def cb(f, t, msg, arg):
+            cb_called[0] = True
+            self.assertEqual(msg.payload, p)
+            f.reactor_stop()
+
+        w2 = self.f.msg_watcher_create(cb, FLUX_MSGTYPE_RESPONSE, "foo.echo")
+        w2.start()
+        self.assertIsNotNone(w2, msg="msg_watcher_create response handler")
+
+        self.send_user_request("foo.echo", p)
+
+        ret = self.f.reactor_run()
+        self.assertTrue(ret >= 0)
+        self.assertTrue(cb_called[0])
+        w2.stop()
+        w2.destroy()
+
+    def test_004_service_rpc_user_eperm(self):
+        cb_called = [False]  # So that cb_called[0] is mutable in inner function
+        p = {"test": "baz"}
+
+        def cb(f, t, msg, arg):
+            cb_called[0] = True
+            with self.assertRaises(OSError) as exc:
+                msgtype, topic, data = msg.decode()
+            self.assertEqual(exc.exception.errno, errno.EPERM)
+            f.reactor_stop()
+
+        w2 = self.f.msg_watcher_create(cb, FLUX_MSGTYPE_RESPONSE, "owner.echo")
+        w2.start()
+        self.assertIsNotNone(w2, msg="msg_watcher_create response handler")
+
+        self.send_user_request("owner.echo", p)
+
+        ret = self.f.reactor_run()
+        self.assertTrue(ret >= 0)
+        self.assertTrue(cb_called[0])
+        w2.stop()
+        w2.destroy()
+
+    def test_003_add_service_message_watcher_respond_error(self):
+        def error_service_cb(f, t, msg, arg):
+            rc = f.respond_error(msg, errno.EINVAL, "Test error response")
+            self.assertEqual(rc, 0)
+
+        self.f.error_watcher = self.f.msg_watcher_create(
+            error_service_cb, FLUX_MSGTYPE_REQUEST, "foo.error"
+        )
+        self.assertIsNotNone(self.f.error_watcher)
+        self.f.error_watcher.start()
+
+    def test_004_service_rpc_error(self):
+        cb_called = [False]  # So that cb_called[0] is mutable in inner function
+
+        def then_cb(future):
+            cb_called[0] = True
+            with self.assertRaises(OSError):
+                future.get()
+            future.get_flux().reactor_stop()
+
+        self.f.rpc("foo.error").then(then_cb)
+
+        ret = self.f.reactor_run()
+        self.assertTrue(ret >= 0)
+        self.assertTrue(cb_called[0])
+
     def test_005_unregister_service(self):
-        rc = service_remove(self.f, "foo")
-        self.assertEqual(rc, 0)
+        rc = self.f.service_unregister("foo").get()
+        self.assertEqual(rc, None)
+
+        rc = self.f.service_unregister("owner").get()
+        self.assertEqual(rc, None)
 
         # done with message handler
         self.f.watcher.destroy()
+        self.f.watcher2.destroy()
+        self.f.error_watcher.destroy()
 
     def test_006_unregister_service_enoent(self):
         with self.assertRaises(EnvironmentError) as e:
-            service_remove(self.f, "foo")
+            self.f.service_unregister("foo").get()
         self.assertEqual(e.exception.errno, errno.ENOENT)
 
     def test_007_service_rpc_enosys(self):
@@ -112,10 +209,12 @@ def test_008_service_remove_on_disconnect(self):
         #   then, ensure "baz" service was removed.
         #
         def add_service_and_disconnect():
-            import sys
-
             h = flux.Flux()
-            sys.exit(service_add(h, "baz"))
+            try:
+                h.service_register("baz").get()
+            except Exception():
+                sys.exit(-1)
+            sys.exit(0)
 
         p = Process(target=add_service_and_disconnect)
         p.start()
diff --git a/t/python/tap/example/test_example.py b/t/python/tap/example/test_example.py
index 2848f650cc41..9120baa921c5 100755
--- a/t/python/tap/example/test_example.py
+++ b/t/python/tap/example/test_example.py
@@ -1,12 +1,14 @@
 #!/usr/bin/env python
 # pylint: disable=C0325
 
-import sys, os
+import os
+import sys
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 
 import unittest
-from pycotap import TAPTestRunner, LogMode
+
+from pycotap import LogMode, TAPTestRunner
 
 
 class MyTests(unittest.TestCase):
diff --git a/t/python/tap/pycotap/__init__.py b/t/python/tap/pycotap/__init__.py
old mode 100644
new mode 100755
index 3b4062d93546..0828c440c509
--- a/t/python/tap/pycotap/__init__.py
+++ b/t/python/tap/pycotap/__init__.py
@@ -6,15 +6,16 @@
 # See COPYING for details
 
 
-import unittest
-import sys
 import base64
+import sys
+import unittest
 
 if sys.hexversion >= 0x03000000:
     from io import StringIO
 else:
     from StringIO import StringIO
 
+
 # Log modes
 class LogMode(object):
     LogToError, LogToDiagnostics, LogToYAML, LogToAttachment = range(4)
@@ -135,7 +136,7 @@ def printErrors(self):
         self.print_raw("1..%d\n" % self.testsRun)
 
 
-class TAPTestRunner(object):
+class TAPTestRunner(unittest.TextTestRunner):
     def __init__(
         self,
         message_log=LogMode.LogToYAML,
diff --git a/t/python/tap/setup.py b/t/python/tap/setup.py
index 46131b0c390b..6f34494cce45 100755
--- a/t/python/tap/setup.py
+++ b/t/python/tap/setup.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # coding=utf-8
 
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 setup(
     name="pycotap",
diff --git a/t/python/tap/test/test.py b/t/python/tap/test/test.py
old mode 100644
new mode 100755
index 43b233509c31..82902a476c12
--- a/t/python/tap/test/test.py
+++ b/t/python/tap/test/test.py
@@ -2,19 +2,20 @@
 
 # pylint: disable=C0325
 
-import sys, os
+import os
+import sys
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 
-import unittest
 import re
+import unittest
 
 if sys.hexversion >= 0x03000000:
     from io import StringIO
 else:
     from StringIO import StringIO
 
-from pycotap import TAPTestRunner, LogMode
+from pycotap import LogMode, TAPTestRunner
 
 
 class TAPTestRunnerTest(unittest.TestCase):
diff --git a/t/rc/rc1-job b/t/rc/rc1-job
index bc264684928c..1a02897c874c 100755
--- a/t/rc/rc1-job
+++ b/t/rc/rc1-job
@@ -1,32 +1,74 @@
 #!/bin/bash -e
 
-set_fake_hwloc_by_rank() {
+idset_from_count() {
+    if test $1 -eq 1; then
+        echo "0"
+    else
+        echo "0-$(($1 - 1))"
+    fi
+}
+
+set_fake_resources() {
     cores=${1}
-    ranklist="0-$(($(flux getattr size) - 1))"
-    corelist="0-$((${cores} - 1))"
-    by_rank="{\"${ranklist}\":{\"Core\":${cores},\"cpuset\":\"${corelist}\"}}"
-    echo Setting fake by_rank="${by_rank}" >&2
-    flux kvs put resource.hwloc.by_rank="${by_rank}"
+    ranklist=$(idset_from_count $(flux getattr size))
+    corelist=$(idset_from_count ${cores})
+    R=$(flux R encode -r${ranklist} -c${corelist})
+    echo Setting fake resource.R="$R" >&2
+    flux kvs put resource.R="$R"
 }
 
-flux module load content-sqlite
-flux module load kvs
-flux exec -r all -x 0 flux module load kvs
-flux exec -r all flux module load kvs-watch
+RANK=$(flux getattr rank)
+
+# Usage: modload {all|<rank>} modname [args ...]
+modload() {
+    local where=$1; shift
+    if test "$where" = "all" || test $where -eq $RANK; then
+        flux module load $*
+    fi
+}
+
+
+modload all content
+modload 0 content-sqlite
+modload all kvs
+modload all kvs-watch
+
+modload 0 job-manager
 
+modload all job-ingest
+modload all job-info
+modload 0 job-list
+modload all barrier
+modload 0 heartbeat
 
-flux exec -r all flux module load job-ingest & pids="$pids $!"
-flux exec -r all flux module load job-info & pids="$pids $!"
-flux exec -r all flux module load barrier & pids="$pids $!"
+if test $RANK -eq 0; then
+    # Set fake resources for testing
+    set_fake_resources ${TEST_UNDER_FLUX_CORES_PER_RANK:-2}
+fi
+modload all resource noverify
 
-# Load a fake resource.hwloc.by_rank key for sched-simple
-# (avoids requirement to run "flux hwloc reload")
-#
-(set_fake_hwloc_by_rank ${TEST_UNDER_FLUX_CORES_PER_RANK:-2}) & pids="$pids $1"
+if [ "${TEST_UNDER_FLUX_NO_JOB_EXEC}" != "y" ]
+then
+    modload 0 job-exec
+fi
 
-wait $pids
+# mirror sched-simple default of limited=8
+if [ "${TEST_UNDER_FLUX_SCHED_SIMPLE_MODE}x" != "x" ]
+then
+    mode=${TEST_UNDER_FLUX_SCHED_SIMPLE_MODE}
+else
+    mode="limited=8"
+fi
 
-flux module load job-manager
-flux module load job-exec
+modload 0 sched-simple mode=${mode}
+#--setbit 0x2 enables creation of reason_pending field
+if [ $RANK -eq 0 ]
+then
+    flux module debug --setbit 0x2 sched-simple
+fi
 
-flux module load sched-simple
+test $RANK -ne 0 || flux admin cleanup-push <<-EOT
+	flux queue stop --all --nocheckpoint
+	flux cancel --all --states RUN
+	flux queue idle
+EOT
diff --git a/t/rc/rc1-kvs b/t/rc/rc1-kvs
index 9978dd9fe744..33cf0d90b8f1 100755
--- a/t/rc/rc1-kvs
+++ b/t/rc/rc1-kvs
@@ -1,6 +1,17 @@
 #!/bin/bash -e
 
-flux module load content-sqlite
-flux module load kvs
-flux exec -r all -x 0 flux module load kvs
-flux exec -r all flux module load kvs-watch
+RANK=$(flux getattr rank)
+
+# Usage: modload {all|<rank>} modname [args ...]
+modload() {
+    local where=$1; shift
+    if test "$where" = "all" || test $where -eq $RANK; then
+        flux module load $*
+    fi
+}
+
+modload all content blob-size-limit=1048576
+modload 0 content-sqlite
+modload all kvs
+modload all kvs-watch
+modload 0 heartbeat
diff --git a/t/rc/rc1-testenv b/t/rc/rc1-testenv
deleted file mode 100755
index 70431ba202dd..000000000000
--- a/t/rc/rc1-testenv
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash -e
-
-# t0014-runlevel.t loads this to verify the rc1 environment
-
-# Check that stdio is logged appropriately
-echo stderr-1 >&2
-echo stderr-2 >&2
-echo stdout-1
-echo stdout-2
-
-# These are explicitly unset in all rc subprocess environments
-# since Flux doesn't yet support simple v1 PMI in jobs that it runs.
-if test -n "$PMI_FD" || test -n "$PMI_RANK" || test -n "$PMI_SIZE"; then
-    echo "simple v1 PMI variables are set but they should not be" >&2
-    exit 1
-else
-    echo "simple v1 PMI variables are not set"
-fi
-
-# We need this to talk to broker
-if ! test -n "$FLUX_URI"; then
-    echo "FLUX_URI is not set" >&2
-    exit 1
-else
-    echo "FLUX_URI is set"
-fi
-
-# Run level should be 1 (and we can talk to broker)
-runlevel=$(flux getattr init.run-level)
-if test $runlevel -ne 1; then
-    echo "run level is $runlevel not 1" >&2
-    exit 1
-else
-    echo "run level is 1"
-fi
diff --git a/t/rc/rc3-job b/t/rc/rc3-job
index 61f7ecb2c391..533489858cb1 100755
--- a/t/rc/rc3-job
+++ b/t/rc/rc3-job
@@ -1,17 +1,32 @@
 #!/bin/bash -e
 
-flux queue stop
-flux job cancelall -f --states RUN
-flux queue idle
+RANK=$(flux getattr rank)
 
-flux module remove -f job-exec
-flux module remove -f sched-simple
-flux module remove job-manager
-flux exec -r all flux module remove barrier
-flux exec -r all flux module remove job-info
-flux exec -r all flux module remove kvs-watch
-flux exec -r all flux module remove job-ingest
+# Usage: modrm {all|<rank>} modname
+modrm() {
+    local where=$1; shift
+    if test "$where" = "all" || test $where -eq $RANK; then
+        flux module remove -f $*
+    fi
+}
 
-flux exec -r all -x 0 flux module remove kvs
-flux module remove kvs
-flux module remove content-sqlite
+if [ "${TEST_UNDER_FLUX_NO_EXEC}" != "y" ]
+then
+    modrm 0 job-exec
+fi
+modrm 0 heartbeat
+modrm 0 sched-simple
+modrm all resource
+modrm 0 job-list
+modrm all job-info
+modrm 0 job-manager
+modrm all barrier
+modrm all kvs-watch
+modrm all job-ingest
+
+modrm all kvs
+
+flux content flush
+
+modrm 0 content-sqlite
+modrm all content
diff --git a/t/rc/rc3-kvs b/t/rc/rc3-kvs
index 4ec89ebcb304..afb19ac48687 100755
--- a/t/rc/rc3-kvs
+++ b/t/rc/rc3-kvs
@@ -1,6 +1,20 @@
 #!/bin/bash -e
 
-flux exec -r all flux module remove kvs-watch
-flux exec -r all -x 0 flux module remove kvs
-flux module remove kvs
-flux module remove content-sqlite
+RANK=$(flux getattr rank)
+
+# Usage: modrm {all|<rank>} modname
+modrm() {
+    local where=$1; shift
+    if test "$where" = "all" || test $where -eq $RANK; then
+        flux module remove -f $*
+    fi
+}
+
+
+modrm 0 heartbeat
+modrm all kvs-watch
+modrm all kvs
+
+flux content flush
+modrm 0 content-sqlite
+modrm all content
diff --git a/t/rc/rc3-testenv b/t/rc/rc3-testenv
deleted file mode 100755
index 7f615156ea78..000000000000
--- a/t/rc/rc3-testenv
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash -e
-
-# t0014-runlevel.t loads rc1-testenv so we need corresponding rc3-testenv
diff --git a/t/reactor/reactorcat.c b/t/reactor/reactorcat.c
index eae1fb1c412f..b2658bb1c39b 100644
--- a/t/reactor/reactorcat.c
+++ b/t/reactor/reactorcat.c
@@ -16,6 +16,8 @@
 #include <fcntl.h>
 
 #include <flux/core.h>
+#include "src/common/libsubprocess/fbuf.h"
+#include "src/common/libsubprocess/fbuf_watcher.h"
 
 static int total_bytes = 0;
 
@@ -35,11 +37,12 @@ static void write_cb (flux_reactor_t *r, flux_watcher_t *w,
     if (revents & FLUX_POLLERR)
         die ("got POLLERR on stdout. Aborting\n");
 
-    if (flux_buffer_write_watcher_is_closed (w, &errnum)) {
+    if (fbuf_write_watcher_is_closed (w, &errnum)) {
         if (errnum)
             fprintf (stderr, "error: close: %s\n", strerror (errnum));
         flux_watcher_stop (w);
     }
+    /* else ignore reports of buffer space changes */
 }
 
 static void read_cb (flux_reactor_t *r, flux_watcher_t *w,
@@ -48,26 +51,24 @@ static void read_cb (flux_reactor_t *r, flux_watcher_t *w,
     const void *data;
     int len, n = 0;
     flux_watcher_t *writer = arg;
-    flux_buffer_t *wfb = NULL;
-    flux_buffer_t *rfb = NULL;;
+    struct fbuf *wfb = NULL;
+    struct fbuf *rfb = NULL;;
 
-    if (!(wfb = flux_buffer_write_watcher_get_buffer (writer))
-        || !(rfb = flux_buffer_read_watcher_get_buffer (w)))
+    if (!(wfb = fbuf_write_watcher_get_buffer (writer))
+        || !(rfb = fbuf_read_watcher_get_buffer (w)))
         die ("failed to get read/write buffers from watchers!\n");
 
-    if (!(data = flux_buffer_peek (rfb, -1, &len)))
-        die ("flux_buffer_peek: %s\n", strerror (errno));
+    if (!(data = fbuf_read (rfb, -1, &len)))
+        die ("fbuf_read: %s\n", strerror (errno));
 
-    if ((len > 0) && ((n = flux_buffer_write (wfb, data, len)) < 0))
-        die ("flux_buffer_write: %s\n", strerror (errno));
+    if ((len > 0) && ((n = fbuf_write (wfb, data, len)) < 0))
+        die ("fbuf_write: %s\n", strerror (errno));
     else if (len == 0) {
         /* Propagate EOF to writer, stop reader */
-        flux_buffer_write_watcher_close (writer);
+        fbuf_write_watcher_close (writer);
         flux_watcher_stop (w);
     }
 
-    /* Drop data in read buffer that was successfully written to writer */
-    flux_buffer_drop (rfb, n);
     total_bytes += n;
 }
 
@@ -97,9 +98,9 @@ int main (int argc, char *argv[])
     if (!(r = flux_reactor_create (0)))
         die ("flux_reactor_create failed\n");
 
-    ww = flux_buffer_write_watcher_create (r, STDOUT_FILENO, 4096,
+    ww = fbuf_write_watcher_create (r, STDOUT_FILENO, 4096,
                                            write_cb, 0, NULL);
-    rw = flux_buffer_read_watcher_create (r, STDIN_FILENO, 4096,
+    rw = fbuf_read_watcher_create (r, STDIN_FILENO, 4096,
                                           read_cb, 0, (void *) ww);
     if (!rw || !ww)
         die ("flux buffer watcher create failed\n");
diff --git a/t/request/req.c b/t/request/req.c
index 11c12a8db40f..ec86c342f019 100644
--- a/t/request/req.c
+++ b/t/request/req.c
@@ -12,12 +12,14 @@
 #include "config.h"
 #endif
 #include <flux/core.h>
-#include <czmq.h>
 #include <jansson.h>
+#include <inttypes.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/oom.h"
 #include "src/common/libutil/xzmalloc.h"
 #include "src/common/libutil/log.h"
+#include "ccan/str/str.h"
 
 typedef struct {
     flux_t *h;
@@ -72,20 +74,26 @@ static t_req_ctx_t *getctx (flux_t *h)
 
 /* Return number of queued clog requests
  */
-void count_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
+void count_request_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
 {
     t_req_ctx_t *ctx = getctx (h);
 
-    if (flux_respond_pack (h, msg, "{s:i}",
+    if (flux_respond_pack (h,
+                           msg,
+                           "{s:i}",
                            "count", zlist_size (ctx->clog_requests)) < 0)
         flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
 }
 
 /* Don't reply to request - just queue it for later.
  */
-void clog_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+void clog_request_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     t_req_ctx_t *ctx = getctx (h);
     flux_msg_t *cpy = flux_msg_copy (msg, true);
@@ -96,8 +104,10 @@ void clog_request_cb (flux_t *h, flux_msg_handler_t *mh,
 
 /* Reply to all queued requests.
  */
-void flush_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
+void flush_request_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
 {
     t_req_ctx_t *ctx = getctx (h);
     flux_msg_t *req;
@@ -115,8 +125,10 @@ void flush_request_cb (flux_t *h, flux_msg_handler_t *mh,
 /* Accept a json payload, verify it and return error if it doesn't
  * match expected.
  */
-void sink_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+void sink_request_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     double d;
 
@@ -136,8 +148,10 @@ void sink_request_cb (flux_t *h, flux_msg_handler_t *mh,
 
 /* Return a fixed json payload
  */
-void src_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                     const flux_msg_t *msg, void *arg)
+void src_request_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
 {
     if (flux_respond_pack (h, msg, "{s:i}", "wormz", 42) < 0)
         flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__);
@@ -145,8 +159,10 @@ void src_request_cb (flux_t *h, flux_msg_handler_t *mh,
 
 /* Return 'n' sequenced responses.
  */
-void nsrc_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+void nsrc_request_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     int i, count;
 
@@ -164,8 +180,10 @@ void nsrc_request_cb (flux_t *h, flux_msg_handler_t *mh,
 
 /* Always return an error 42
  */
-void err_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                     const flux_msg_t *msg, void *arg)
+void err_request_cb (flux_t *h,
+                     flux_msg_handler_t *mh,
+                     const flux_msg_t *msg,
+                     void *arg)
 {
     if (flux_respond_error (h, msg, 42, NULL) < 0)
         flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
@@ -173,8 +191,10 @@ void err_request_cb (flux_t *h, flux_msg_handler_t *mh,
 
 /* Echo a json payload back to requestor.
  */
-void echo_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+void echo_request_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     const char *json_str;
 
@@ -194,8 +214,10 @@ void echo_request_cb (flux_t *h, flux_msg_handler_t *mh,
 
 /* Proxy ping.
  */
-void xping_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
+void xping_request_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
 {
     t_req_ctx_t *ctx = arg;
     int rank, seq = ctx->ping_seq++;
@@ -203,16 +225,23 @@ void xping_request_cb (flux_t *h, flux_msg_handler_t *mh,
     char *hashkey = NULL;
     flux_msg_t *cpy;
 
-    if (flux_request_unpack (msg, NULL, "{s:i s:s}", "rank", &rank,
-                                                     "service", &service) < 0)
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s:i s:s}",
+                             "rank", &rank,
+                             "service", &service) < 0)
         goto error;
     flux_log (h, LOG_DEBUG, "Rxping rank=%d service=%s", rank, service);
 
     flux_log (h, LOG_DEBUG, "Tping seq=%d %d!%s", seq, rank, service);
 
     flux_future_t *f;
-    if (!(f = flux_rpc_pack (h, service, rank, FLUX_RPC_NORESPONSE,
-                             "{s:i}", "seq", seq)))
+    if (!(f = flux_rpc_pack (h,
+                             service,
+                             rank,
+                             0,
+                             "{s:i}",
+                             "seq", seq)))
         goto error;
     flux_future_destroy (f);
     if (!(cpy = flux_msg_copy (msg, true)))
@@ -230,8 +259,10 @@ void xping_request_cb (flux_t *h, flux_msg_handler_t *mh,
 /* Handle ping response for proxy ping.
  * Match it with a request and respond to that request.
  */
-void ping_response_cb (flux_t *h, flux_msg_handler_t *mh,
-                       const flux_msg_t *msg, void *arg)
+void ping_response_cb (flux_t *h,
+                       flux_msg_handler_t *mh,
+                       const flux_msg_t *msg,
+                       void *arg)
 {
     t_req_ctx_t *ctx = arg;
     int seq;
@@ -246,8 +277,7 @@ void ping_response_cb (flux_t *h, flux_msg_handler_t *mh,
         return;
     }
     if (!json_str || !(o = json_loads (json_str, 0, NULL))
-            || json_unpack (o, "{s:i s:s}", "seq", &seq,
-                                            "route", &route) < 0) {
+        || json_unpack (o, "{s:i s:s}", "seq", &seq, "route", &route) < 0) {
         flux_log (h, LOG_ERR, "%s: error decoding payload", __FUNCTION__);
         goto done;
     }
@@ -268,12 +298,15 @@ void ping_response_cb (flux_t *h, flux_msg_handler_t *mh,
 /* Handle the simplest possible request.
  * Verify that everything is as expected; log it and stop the reactor if not.
  */
-void null_request_cb (flux_t *h, flux_msg_handler_t *mh,
-                      const flux_msg_t *msg, void *arg)
+void null_request_cb (flux_t *h,
+                      flux_msg_handler_t *mh,
+                      const flux_msg_t *msg,
+                      void *arg)
 {
     t_req_ctx_t *ctx = arg;
     const char *topic;
-    int type, size;
+    int type;
+    size_t size;
     const void *buf;
     uint32_t nodeid;
 
@@ -286,7 +319,10 @@ void null_request_cb (flux_t *h, flux_msg_handler_t *mh,
         goto error;
     }
     if (type != FLUX_MSGTYPE_REQUEST) {
-        flux_log (h, LOG_ERR, "%s: unexpected type %s", __FUNCTION__,
+        flux_log (h,
+                  LOG_ERR,
+                  "%s: unexpected type %s",
+                  __FUNCTION__,
                   flux_msg_typestr (type));
         goto error;
     }
@@ -295,7 +331,10 @@ void null_request_cb (flux_t *h, flux_msg_handler_t *mh,
         goto error;
     }
     if (nodeid != ctx->rank && nodeid != FLUX_NODEID_ANY) {
-        flux_log (h, LOG_ERR, "%s: unexpected nodeid: %"PRIu32"", __FUNCTION__,
+        flux_log (h,
+                  LOG_ERR,
+                  "%s: unexpected nodeid: %"PRIu32"",
+                  __FUNCTION__,
                   nodeid);
         goto error;
     }
@@ -303,18 +342,24 @@ void null_request_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_log_error (h, "%s: flux_msg_get_topic", __FUNCTION__);
         goto error;
     }
-    if (strcmp (topic, "req.null") != 0) {
+    if (!streq (topic, "req.null")) {
         flux_log (h, LOG_ERR, "%s: unexpected topic: %s", __FUNCTION__,
                   topic);
         goto error;
     }
     if (flux_msg_get_payload (msg, &buf, &size) == 0) {
-        flux_log (h, LOG_ERR, "%s: unexpected payload size %d", __FUNCTION__,
+        flux_log (h,
+                  LOG_ERR,
+                  "%s: unexpected payload size %zu",
+                  __FUNCTION__,
                   size);
         goto error;
     }
     if (errno != EPROTO) {
-        flux_log (h, LOG_ERR, "%s: get nonexistent payload: %s", __FUNCTION__,
+        flux_log (h,
+                  LOG_ERR,
+                  "%s: get nonexistent payload: %s",
+                  __FUNCTION__,
                   strerror (errno));
         goto error;
     }
@@ -371,8 +416,6 @@ int mod_main (flux_t *h, int argc, char **argv)
     return -1;
 }
 
-MOD_NAME ("req");
-
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */
diff --git a/t/request/rpc.c b/t/request/rpc.c
index 3d9812f4b4bf..c834e6da385b 100644
--- a/t/request/rpc.c
+++ b/t/request/rpc.c
@@ -12,12 +12,27 @@
 #include "config.h"
 #endif
 #include <unistd.h>
+#include <getopt.h>
 #include <stdio.h>
 #include <flux/core.h>
 
 #include "src/common/libutil/read_all.h"
 #include "src/common/libutil/log.h"
 
+
+#define OPTIONS "rR"
+static const struct option longopts[] = {
+    {"raw-request", no_argument,  0, 'r'},
+    {"raw-response", no_argument,  0, 'R'},
+    { 0, 0, 0, 0 },
+};
+
+void usage (void)
+{
+    fprintf (stderr, "Usage: rpc [-r] [-R] topic [errnum] <payload >payload\n");
+    exit (1);
+}
+
 int main (int argc, char *argv[])
 {
     flux_t *h;
@@ -25,36 +40,61 @@ int main (int argc, char *argv[])
     const char *topic;
     ssize_t inlen;
     void *inbuf;
-    const void *outbuf;
-    int outlen;
+    const char *outbuf;
+    size_t outlen;
     int expected_errno = -1;
+    int ch;
+    bool raw_request = false;
+    bool raw_response = false;
+    int rc;
+
+    while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
+        switch (ch) {
+            case 'r':
+                raw_request = true;
+                break;
+            case 'R':
+                raw_response = true;
+                break;
+            default:
+                usage ();
+        }
+    }
+    if (argc - optind != 1 && argc - optind != 2)
+        usage ();
+    topic = argv[optind++];
+    if (argc - optind > 0)
+        expected_errno = strtoul (argv[optind++], NULL, 10);
 
     if (!(h = flux_open (NULL, 0)))
         log_err_exit ("flux_open");
 
-    if (argc != 2 && argc != 3) {
-        fprintf (stderr, "Usage: rpc topic [errnum] <payload >payload\n");
-        exit (1);
-    }
-    topic = argv[1];
-    if (argc == 3)
-        expected_errno = strtoul (argv[2], NULL, 10);
-
+    /* N.B. As a safety measure, read_all() adds a NUL char to the buffer
+     * that is not accounted for in the returned length.
+     */
     if ((inlen = read_all (STDIN_FILENO, &inbuf)) < 0)
         log_err_exit ("read from stdin");
-    if (inlen > 0)  // flux stringified JSON payloads are sent with \0-term
-        inlen++;    //  and read_all() ensures inbuf has one, not acct in inlen
+    if (raw_request)
+        f = flux_rpc_raw (h, topic, inbuf, inlen, FLUX_NODEID_ANY, 0);
+    else
+        f = flux_rpc (h, topic, inlen > 0 ? inbuf : NULL, FLUX_NODEID_ANY, 0);
+    if (!f)
+        log_err_exit ("error sending RPC");
 
-    if (!(f = flux_rpc_raw (h, topic, inbuf, inlen, FLUX_NODEID_ANY, 0)))
-        log_err_exit ("flux_rpc_raw %s", topic);
-    if (flux_rpc_get_raw (f, &outbuf, &outlen) < 0) {
+    if (raw_response)
+        rc = flux_rpc_get_raw (f, (const void **)&outbuf, &outlen);
+    else {
+        if ((rc = flux_rpc_get (f, &outbuf)) == 0)
+            outlen = outbuf ? strlen (outbuf) : 0;
+    }
+    if (rc < 0) {
         if (expected_errno > 0) {
             if (errno != expected_errno)
                 log_msg_exit ("%s: failed with errno=%d != expected %d",
                               topic, errno, expected_errno);
         }
         else
-            log_err_exit ("%s", topic);
+            log_msg_exit ("%s: %s", topic, future_strerror (f, errno));
     }
     else {
         if (expected_errno > 0)
diff --git a/t/request/rpc_stream.c b/t/request/rpc_stream.c
new file mode 100644
index 000000000000..e09502e46929
--- /dev/null
+++ b/t/request/rpc_stream.c
@@ -0,0 +1,80 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <jansson.h>
+#include <stdio.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/log.h"
+
+int main (int argc, char *argv[])
+{
+    flux_t *h;
+    flux_future_t *f;
+    const char *topic;
+    ssize_t inlen;
+    void *inbuf;
+    const char *out;
+    const char *end_key = NULL;
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (argc != 2 && argc != 3) {
+        fprintf (stderr, "Usage: rpc_stream topic [end-key] <payload\n");
+        exit (1);
+    }
+    topic = argv[1];
+    if (argc == 3)
+        end_key = argv[2];
+
+    /* N.B. As a safety measure, read_all() adds a NUL char to the buffer
+     * that is not accounted for in the returned length.
+     */
+    if ((inlen = read_all (STDIN_FILENO, &inbuf)) < 0)
+        log_err_exit ("read from stdin");
+    if (!(f = flux_rpc (h,
+                        topic,
+                        inlen > 0 ? inbuf : NULL,
+                        FLUX_NODEID_ANY,
+                        FLUX_RPC_STREAMING)))
+        log_err_exit ("flux_rpc %s", topic);
+
+    bool done = false;
+    do {
+        if (flux_rpc_get (f, &out) < 0)
+            log_msg_exit ("%s: %s", topic, future_strerror (f, errno));
+        printf ("%s\n", out);
+        fflush (stdout);
+        if (end_key) {
+            json_t *o = json_loads (out, 0, NULL);
+            if (!o)
+                log_msg_exit ("failed to parse response payload");
+            if (json_object_get (o, end_key))
+                done = true;
+            json_decref (o);
+        }
+        if (!done)
+            flux_future_reset (f);
+    } while (!done);
+    flux_future_destroy (f);
+    free (inbuf);
+    flux_close (h);
+    return (0);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/request/treq.c b/t/request/treq.c
index 91556bdf5e39..7de29f9ddfcb 100644
--- a/t/request/treq.c
+++ b/t/request/treq.c
@@ -14,16 +14,17 @@
 #include <getopt.h>
 #include <assert.h>
 #include <libgen.h>
-#include <czmq.h>
 #include <pthread.h>
 #include <flux/core.h>
-#include <czmq.h>
 #include <jansson.h>
 
+#include "src/common/libczmqcontainers/czmq_containers.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/oom.h"
 #include "src/common/libutil/monotime.h"
 #include "src/common/libutil/xzmalloc.h"
+#include "ccan/array_size/array_size.h"
+#include "ccan/str/str.h"
 
 void test_null (flux_t *h, uint32_t nodeid);
 void test_echo (flux_t *h, uint32_t nodeid);
@@ -61,8 +62,8 @@ static test_t tests[] = {
 test_t *test_lookup (const char *name)
 {
     int i;
-    for (i = 0; i < sizeof (tests) / sizeof (tests[0]); i++)
-        if (!strcmp (tests[i].name, name))
+    for (i = 0; i < ARRAY_SIZE (tests); i++)
+        if (streq (tests[i].name, name))
             return &tests[i];
     return NULL;
 }
@@ -139,7 +140,7 @@ void test_echo (flux_t *h, uint32_t nodeid)
                              "{s:s}", "mumble", "burble"))
              || flux_rpc_get_unpack (f, "{s:s}", "mumble", &s) < 0)
         log_err_exit ("%s", __FUNCTION__);
-    if (strcmp (s, "burble") != 0)
+    if (!streq (s, "burble"))
         log_msg_exit ("%s: returned payload wasn't an echo", __FUNCTION__);
     flux_future_destroy (f);
 }
@@ -189,7 +190,7 @@ void test_nsrc (flux_t *h, uint32_t nodeid)
     json_t *o;
 
     if (!(f = flux_rpc_pack (h, "req.nsrc",
-                             FLUX_NODEID_ANY, FLUX_RPC_NORESPONSE,
+                             FLUX_NODEID_ANY, 0,
                              "{s:i}", "count", count)))
         log_err_exit ("%s", __FUNCTION__);
     flux_future_destroy (f);
@@ -231,7 +232,7 @@ void test_putmsg (flux_t *h, uint32_t nodeid)
         oom ();
 
     if (!(f = flux_rpc_pack (h, "req.nsrc",
-                             FLUX_NODEID_ANY, FLUX_RPC_NORESPONSE,
+                             FLUX_NODEID_ANY, 0,
                              "{s:i}", "count", count)))
         log_err_exit ("%s", __FUNCTION__);
     flux_future_destroy (f);
diff --git a/t/resource/resource.eventlog.thing1 b/t/resource/resource.eventlog.thing1
new file mode 100644
index 000000000000..2786bc934332
--- /dev/null
+++ b/t/resource/resource.eventlog.thing1
@@ -0,0 +1,91329 @@
+{"timestamp":1691516967.2848825,"name":"resource-init","context":{"restart":false,"drain":{},"online":"","exclude":"0"}}
+{"timestamp":1691516967.2855024,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1691516968.7433197,"name":"online","context":{"idset":"0"}}
+{"timestamp":1691528076.2200217,"name":"online","context":{"idset":"6,13,27,32,52"}}
+{"timestamp":1691528076.3286021,"name":"online","context":{"idset":"2-4,8-11,14-16,18-19,21,23-26,28-29,33-35,37,39-43,47,49,51,53-54,58,60"}}
+{"timestamp":1691528076.429496,"name":"online","context":{"idset":"1,7,12,17,20,22,30-31,36,38,44-46,48,55-57,59"}}
+{"timestamp":1691528076.5462575,"name":"online","context":{"idset":"5,50"}}
+{"timestamp":1691680839.5358648,"name":"drain","context":{"idset":"48","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1691680839.6239004,"name":"drain","context":{"idset":"60","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692294896.7844155,"name":"undrain","context":{"idset":"48"}}
+{"timestamp":1692294916.441484,"name":"undrain","context":{"idset":"60"}}
+{"timestamp":1692295078.9258068,"name":"drain","context":{"idset":"48","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692392398.3978171,"name":"drain","context":{"idset":"13,16,18-29","reason":"epilog failed for jobid f2ygnmWC1hh","overwrite":0}}
+{"timestamp":1692639728.7625329,"name":"drain","context":{"idset":"60","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692823489.8290925,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1692823551.8280346,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1692849528.545357,"name":"undrain","context":{"idset":"36"}}
+{"timestamp":1692849546.2044473,"name":"undrain","context":{"idset":"13,16,18-29"}}
+{"timestamp":1692907718.423723,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1692908042.2125642,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1693078407.3944068,"name":"drain","context":{"idset":"1","reason":"prolog failed for jobid f4XZVTys1sV","overwrite":0}}
+{"timestamp":1693078465.0767128,"name":"drain","context":{"idset":"2","reason":"prolog failed for jobid f4XZvu4aSQo","overwrite":0}}
+{"timestamp":1693080904.6628013,"name":"drain","context":{"idset":"7-11","reason":"prolog failed for jobid f4XtU1gEtK9","overwrite":0}}
+{"timestamp":1693242251.6210065,"name":"undrain","context":{"idset":"1-2,7-11"}}
+{"timestamp":1693243249.7282867,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1693243249.8279536,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1693243251.7283232,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1693243251.7284067,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1693243251.7284474,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1693243251.8290865,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7277386,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7278173,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7278631,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7279069,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7279477,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.727988,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7280467,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7280869,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7281277,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7281656,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.728205,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7282445,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7282822,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7283232,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.7283611,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1693243253.8283486,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1693243255.7279978,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1693243255.7280753,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1693243255.7281148,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1693243255.728159,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1693243255.7281902,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1693243255.8282042,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7284501,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7285411,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7285872,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7286313,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7286689,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7287052,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7287436,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7287776,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7288249,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7288597,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7289031,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.7289391,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1693243257.8285685,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1693243259.7277036,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1693243259.7277668,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1693243259.7278178,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1693243259.727864,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1693243259.8277667,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1693243261.7262337,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1693243261.7263157,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1693243261.7263706,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1693243261.726414,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1693243261.7264531,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1693243261.7264996,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1693243261.7265401,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1693243261.7265866,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1693243261.8270559,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1693243263.728621,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1693243263.8290665,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1693243315.7277851,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1693243315.7279525,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693243315.728056,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693243315.7281473,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693243315.7282877,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693243315.8277068,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1693243317.7305899,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693243317.7307243,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693243317.7308128,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693243317.7308943,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693243317.7309804,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693243317.7311141,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693243317.7312346,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693243317.827987,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693243319.4795196,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693243319.4796436,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693243319.4797282,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693243319.4798105,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693243319.4799116,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693243319.4800127,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693243319.4801185,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693243319.4802096,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693243319.4803419,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693243319.4804618,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693243319.4805655,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693243319.4806433,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693243319.4807172,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693243319.4808114,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693243319.4809101,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693243319.4810171,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693243319.5796912,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693243325.7276006,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693243325.7277591,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693243325.8277395,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693243327.415417,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693243327.5168467,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693243332.3597686,"name":"online","context":{"idset":"2"}}
+{"timestamp":1693243337.5091608,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693243341.1189642,"name":"online","context":{"idset":"22"}}
+{"timestamp":1693243457.8281326,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1693243459.7282343,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693243459.8284688,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693243461.7276247,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693243461.7277231,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693243461.7277956,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693243461.7278736,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693243461.7279317,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693243461.7279909,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693243461.7280669,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693243461.8277719,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693243463.7274675,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693243463.7275913,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693243463.727726,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693243463.7277966,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693243463.8279846,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693243465.7282524,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693243465.7283823,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693243465.7284412,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693243465.7285032,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693243465.7285535,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693243465.8289726,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693243465.8298883,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693243465.8299336,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693243465.8299904,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693243465.9309011,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693243559.7274342,"name":"online","context":{"idset":"14"}}
+{"timestamp":1693243561.416652,"name":"online","context":{"idset":"20"}}
+{"timestamp":1693243562.7315712,"name":"online","context":{"idset":"6,8"}}
+{"timestamp":1693243564.9704435,"name":"online","context":{"idset":"21"}}
+{"timestamp":1693243565.8290105,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693243566.7234991,"name":"online","context":{"idset":"1"}}
+{"timestamp":1693243567.3744609,"name":"online","context":{"idset":"7"}}
+{"timestamp":1693243568.0165319,"name":"online","context":{"idset":"22"}}
+{"timestamp":1693243568.2384231,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693243568.4990485,"name":"online","context":{"idset":"10,38"}}
+{"timestamp":1693243568.7609582,"name":"online","context":{"idset":"46"}}
+{"timestamp":1693243569.3058956,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693243570.9268701,"name":"online","context":{"idset":"2"}}
+{"timestamp":1693243571.4490223,"name":"online","context":{"idset":"50"}}
+{"timestamp":1693243573.2159762,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693243577.2302222,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693243577.4072168,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693243578.14663,"name":"online","context":{"idset":"16"}}
+{"timestamp":1693243578.2773774,"name":"online","context":{"idset":"17,26"}}
+{"timestamp":1693243578.4136145,"name":"online","context":{"idset":"18"}}
+{"timestamp":1693243578.6725256,"name":"online","context":{"idset":"27"}}
+{"timestamp":1693243579.0637891,"name":"online","context":{"idset":"35,51"}}
+{"timestamp":1693243579.4636002,"name":"online","context":{"idset":"23"}}
+{"timestamp":1693243580.220175,"name":"online","context":{"idset":"30-31,34"}}
+{"timestamp":1693243581.2043476,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693243582.310385,"name":"online","context":{"idset":"4"}}
+{"timestamp":1693243582.4606915,"name":"online","context":{"idset":"5,32"}}
+{"timestamp":1693243582.584759,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693243583.4918492,"name":"online","context":{"idset":"25,43"}}
+{"timestamp":1693243583.6346445,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693243584.4534225,"name":"online","context":{"idset":"40"}}
+{"timestamp":1693243585.6124332,"name":"online","context":{"idset":"42"}}
+{"timestamp":1693243588.5406752,"name":"online","context":{"idset":"41"}}
+{"timestamp":1693243645.4716268,"name":"online","context":{"idset":"45"}}
+{"timestamp":1693243645.7092111,"name":"online","context":{"idset":"44"}}
+{"timestamp":1693243649.4400098,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693243709.7277493,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693243709.7278554,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693243709.7279494,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693243709.8282509,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693243713.7275889,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693243713.7276971,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693243713.8279226,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693243715.7282004,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693243715.828644,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693243731.357506,"name":"online","context":{"idset":"57"}}
+{"timestamp":1693243731.6971202,"name":"online","context":{"idset":"52"}}
+{"timestamp":1693243736.0580544,"name":"online","context":{"idset":"56"}}
+{"timestamp":1693243736.2073064,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693243737.7276614,"name":"online","context":{"idset":"26"}}
+{"timestamp":1693243737.9338493,"name":"online","context":{"idset":"25,35"}}
+{"timestamp":1693243739.1451521,"name":"online","context":{"idset":"54"}}
+{"timestamp":1693243741.7157466,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693243756.850131,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693243761.3604119,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693243787.7257643,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693243787.7258661,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693243787.8257334,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693243789.8286664,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693243791.7275319,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693243791.7276843,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693243791.7277975,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693243791.7279046,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693243791.7280128,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693243791.7281179,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693243791.7282157,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693243791.7283154,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693243791.8275108,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693243793.8287728,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693243795.7277184,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693243795.7278559,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693243795.7279582,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693243795.72807,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693243795.8282585,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693243853.7282507,"name":"online","context":{"idset":"44"}}
+{"timestamp":1693243856.0496151,"name":"online","context":{"idset":"45"}}
+{"timestamp":1693243857.8282471,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693243857.9553437,"name":"online","context":{"idset":"38"}}
+{"timestamp":1693243859.7463658,"name":"online","context":{"idset":"27"}}
+{"timestamp":1693243861.0679801,"name":"online","context":{"idset":"59"}}
+{"timestamp":1693243873.8285723,"name":"online","context":{"idset":"1-2"}}
+{"timestamp":1693243873.9954374,"name":"online","context":{"idset":"7"}}
+{"timestamp":1693243874.1156178,"name":"online","context":{"idset":"6"}}
+{"timestamp":1693243877.8683147,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693243878.8869436,"name":"online","context":{"idset":"4-5,8-9"}}
+{"timestamp":1693243879.8292222,"name":"online","context":{"idset":"18"}}
+{"timestamp":1693243880.063277,"name":"online","context":{"idset":"41"}}
+{"timestamp":1693243883.8279359,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693243885.8281546,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693243887.8280594,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693243896.0705812,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693243898.8954575,"name":"online","context":{"idset":"21"}}
+{"timestamp":1693243899.4174647,"name":"online","context":{"idset":"42"}}
+{"timestamp":1693243900.9419894,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693243901.4794736,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693243903.0672388,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693243904.8570709,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693243932.0244594,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693243932.1258061,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693243936.0506232,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693243936.5747621,"name":"online","context":{"idset":"37"}}
+{"timestamp":1693243962.6162436,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693243963.9942544,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693244945.7283187,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693244945.7284634,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693244945.8287315,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693244947.7278743,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693244947.7279844,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693244947.7280519,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693244947.8279972,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693244949.7275174,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693244949.8275785,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693244991.8282561,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693244992.2968392,"name":"online","context":{"idset":"50,56"}}
+{"timestamp":1693244992.9698365,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693245021.7285433,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693245021.7287772,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693245021.7288671,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693245021.7289376,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693245021.8288291,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693245023.7282338,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693245025.7285209,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693245025.7287288,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693245025.7288563,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693245025.729001,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693245025.7291377,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693245025.729254,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693245025.729367,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693245025.7295008,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693245025.7296019,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693245025.7296901,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693245025.7297595,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693245025.7298636,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693245025.7299285,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693245025.730015,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693245025.7300923,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693245025.7301719,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693245025.7302477,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693245025.7303247,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693245025.8284807,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693245039.8282776,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693245051.028193,"name":"online","context":{"idset":"2,37"}}
+{"timestamp":1693245053.828073,"name":"online","context":{"idset":"21"}}
+{"timestamp":1693245054.4759896,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693245055.8281653,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693245057.7280977,"name":"online","context":{"idset":"44"}}
+{"timestamp":1693245057.8281596,"name":"online","context":{"idset":"6,40,42"}}
+{"timestamp":1693245057.9545863,"name":"online","context":{"idset":"38,46"}}
+{"timestamp":1693245058.1325943,"name":"online","context":{"idset":"8"}}
+{"timestamp":1693245058.9953468,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693245059.8288748,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693245063.8280945,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693245071.8285434,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693245072.7220731,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693245117.8270416,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693245121.8287172,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693245154.1307325,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693245154.2311184,"name":"online","context":{"idset":"57"}}
+{"timestamp":1693245161.7278893,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693245161.7281659,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693245162.9344995,"name":"online","context":{"idset":"10"}}
+{"timestamp":1693245163.0617633,"name":"online","context":{"idset":"39,41"}}
+{"timestamp":1693245164.9514716,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693245165.7275646,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693245166.5049577,"name":"online","context":{"idset":"7"}}
+{"timestamp":1693245172.9747462,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693245175.0190058,"name":"online","context":{"idset":"59"}}
+{"timestamp":1693245189.8290539,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693245190.3173759,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693245196.0145895,"name":"online","context":{"idset":"25"}}
+{"timestamp":1693245197.7274332,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693245197.7275579,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693245197.7276392,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693245197.8280611,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693245199.8274672,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693245237.8281269,"name":"online","context":{"idset":"5"}}
+{"timestamp":1693245242.2404499,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693245249.9044878,"name":"online","context":{"idset":"38,42"}}
+{"timestamp":1693245252.1780553,"name":"online","context":{"idset":"2"}}
+{"timestamp":1693245259.8658366,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693245309.7278171,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693245309.7279401,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693245309.7283955,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693245313.5117064,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693245317.8292558,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693245321.0441949,"name":"online","context":{"idset":"45"}}
+{"timestamp":1693245323.091084,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693245330.1688707,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693245331.7276464,"name":"online","context":{"idset":"57"}}
+{"timestamp":1693245331.9775953,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693245334.9627099,"name":"online","context":{"idset":"54"}}
+{"timestamp":1693245338.0620799,"name":"online","context":{"idset":"1"}}
+{"timestamp":1693245343.8283567,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693245387.7276227,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693245387.7277491,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693245387.7278326,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693245387.7279165,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693245387.827749,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693245394.92749,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693245401.2460172,"name":"online","context":{"idset":"35"}}
+{"timestamp":1693245410.0195904,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693245414.7887893,"name":"online","context":{"idset":"29-30"}}
+{"timestamp":1693245419.8717437,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693245421.913944,"name":"online","context":{"idset":"17"}}
+{"timestamp":1693245430.4317951,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693245435.1060102,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693245477.0523071,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693245478.1151259,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693245483.2020292,"name":"online","context":{"idset":"4"}}
+{"timestamp":1693245485.1860523,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693245539.3845508,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693245617.7279711,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693245617.728117,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693245617.7282026,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693245617.8283792,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693245619.7275493,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693245619.7276826,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693245619.7277539,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693245619.8282127,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693245621.7281373,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693245621.7282588,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693245621.7283301,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693245621.828763,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693245638.9324327,"name":"online","context":{"idset":"10"}}
+{"timestamp":1693245639.7284625,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693245641.6155879,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693245641.8279939,"name":"online","context":{"idset":"18"}}
+{"timestamp":1693245642.0245957,"name":"online","context":{"idset":"22"}}
+{"timestamp":1693245695.7136779,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693245695.8161478,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693245699.7279906,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693245699.7281239,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693245699.7282114,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693245699.7282844,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693245699.7283783,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693245699.828639,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693245707.799572,"name":"online","context":{"idset":"44"}}
+{"timestamp":1693245711.7283752,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693245716.9118414,"name":"online","context":{"idset":"21"}}
+{"timestamp":1693245719.828315,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693245723.7271149,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693245723.827347,"name":"online","context":{"idset":"6"}}
+{"timestamp":1693245724.0070035,"name":"online","context":{"idset":"26"}}
+{"timestamp":1693245726.9732089,"name":"online","context":{"idset":"35"}}
+{"timestamp":1693245735.8282142,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693245801.8286357,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693245825.0685067,"name":"online","context":{"idset":"27"}}
+{"timestamp":1693245829.1061637,"name":"online","context":{"idset":"23"}}
+{"timestamp":1693245854.9400706,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693245881.9997525,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693246009.8285861,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693246011.826993,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693246013.7281888,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693246013.8286488,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693246035.6504755,"name":"online","context":{"idset":"8"}}
+{"timestamp":1693246089.727484,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693246089.7276194,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693246089.7277033,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693246089.7277927,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693246089.8273578,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693246091.7281485,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693246091.8284247,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693246093.7276521,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693246093.7277868,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693246093.8282547,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693246095.8284061,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693246099.8273437,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693246125.7277887,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693246125.8277342,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693246149.8283901,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693246169.7274909,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693246169.7276292,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693246169.7277229,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693246169.8283031,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693246171.7280397,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693246171.7281554,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693246171.7282329,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693246171.7283027,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693246171.7283819,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693246171.7284493,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693246171.7285144,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693246171.7285848,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693246171.728687,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693246171.7287619,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693246171.7288461,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693246171.728924,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693246171.7290032,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693246171.7290823,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693246171.7291608,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693246171.7292297,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693246171.7293005,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693246171.7293546,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693246171.7294259,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693246171.7294812,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693246171.729543,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693246171.8282402,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693246175.8284395,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693246179.8276627,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693246181.8283632,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693246181.8284228,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693246185.0004382,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693246186.4699233,"name":"online","context":{"idset":"35"}}
+{"timestamp":1693246186.8566606,"name":"online","context":{"idset":"23"}}
+{"timestamp":1693246187.7275937,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693246190.9962223,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693246191.9344552,"name":"online","context":{"idset":"10"}}
+{"timestamp":1693246193.7269096,"name":"online","context":{"idset":"1"}}
+{"timestamp":1693246193.8271039,"name":"online","context":{"idset":"5-6,21"}}
+{"timestamp":1693246193.9295094,"name":"online","context":{"idset":"2,7,26"}}
+{"timestamp":1693246194.719919,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693246195.7280848,"name":"online","context":{"idset":"20"}}
+{"timestamp":1693246196.0938826,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693246196.8300855,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693246198.8936236,"name":"online","context":{"idset":"8"}}
+{"timestamp":1693246203.7278874,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693246204.514179,"name":"online","context":{"idset":"22"}}
+{"timestamp":1693246206.0063236,"name":"online","context":{"idset":"59"}}
+{"timestamp":1693246207.3095963,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693246208.8157332,"name":"online","context":{"idset":"45"}}
+{"timestamp":1693246212.02356,"name":"online","context":{"idset":"50"}}
+{"timestamp":1693246213.9579253,"name":"online","context":{"idset":"37"}}
+{"timestamp":1693246216.1700146,"name":"online","context":{"idset":"14"}}
+{"timestamp":1693246216.8407245,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693246217.0237191,"name":"online","context":{"idset":"46"}}
+{"timestamp":1693246218.8399,"name":"online","context":{"idset":"17-18,29,40,42"}}
+{"timestamp":1693246218.951452,"name":"online","context":{"idset":"4,38,44,54"}}
+{"timestamp":1693246221.8279619,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693246223.0656474,"name":"online","context":{"idset":"30,56"}}
+{"timestamp":1693246223.8285654,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693246227.8557756,"name":"online","context":{"idset":"16"}}
+{"timestamp":1693246241.9552279,"name":"online","context":{"idset":"52"}}
+{"timestamp":1693246251.977952,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693246269.0609796,"name":"online","context":{"idset":"41"}}
+{"timestamp":1693246347.5121765,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693246403.8278356,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693246405.7279687,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693246405.7281282,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693246405.8279791,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693246407.7275803,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693246407.7277434,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693246407.7278528,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693246407.7279761,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693246407.8281734,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693246409.728585,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693246409.7287865,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693246409.7289016,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693246409.7290099,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693246409.729121,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693246409.8291919,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693246411.7269721,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693246411.7274008,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693246414.7063668,"name":"online","context":{"idset":"38"}}
+{"timestamp":1693246418.9659293,"name":"online","context":{"idset":"52"}}
+{"timestamp":1693246422.0623431,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693246423.8301656,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693246425.2303531,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693246426.4060974,"name":"online","context":{"idset":"42"}}
+{"timestamp":1693246428.8213654,"name":"online","context":{"idset":"27"}}
+{"timestamp":1693246429.7275553,"name":"online","context":{"idset":"44"}}
+{"timestamp":1693246430.4738114,"name":"online","context":{"idset":"37"}}
+{"timestamp":1693246431.5493667,"name":"online","context":{"idset":"45"}}
+{"timestamp":1693246431.957324,"name":"online","context":{"idset":"46-47"}}
+{"timestamp":1693246432.8499196,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693246434.5567625,"name":"online","context":{"idset":"41"}}
+{"timestamp":1693246436.8399706,"name":"online","context":{"idset":"40"}}
+{"timestamp":1693246511.7284448,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693246529.7285445,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693246541.5652854,"name":"online","context":{"idset":"57"}}
+{"timestamp":1693246541.9438522,"name":"online","context":{"idset":"25"}}
+{"timestamp":1693246567.7284677,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693246567.728611,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693246567.7286997,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693246567.7287903,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693246567.7289534,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693246567.8290372,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693246569.7274964,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693246569.8273969,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693246571.7279224,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693246571.7280362,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693246571.7281158,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693246571.728188,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693246571.7282465,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693246571.7283151,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693246571.7283723,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693246571.7284272,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693246571.7284946,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693246571.7285545,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693246571.7286301,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693246571.7286904,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693246571.7287471,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693246571.7288074,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693246571.7288673,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693246571.7289295,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693246571.7289834,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693246571.7290337,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693246571.7290823,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693246571.72913,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693246571.7291954,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693246571.7292476,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693246571.7292938,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693246571.7293396,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693246571.8283172,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693246573.7279005,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693246573.7280147,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693246573.7280841,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693246573.8286283,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693246575.7283657,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693246575.8289266,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693246609.8276556,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693246612.3085384,"name":"online","context":{"idset":"44"}}
+{"timestamp":1693246614.8734934,"name":"online","context":{"idset":"2,6,8,17,31"}}
+{"timestamp":1693246615.0699437,"name":"online","context":{"idset":"27"}}
+{"timestamp":1693246615.2978337,"name":"online","context":{"idset":"40,42"}}
+{"timestamp":1693246615.7280602,"name":"online","context":{"idset":"56"}}
+{"timestamp":1693246615.9845328,"name":"online","context":{"idset":"5"}}
+{"timestamp":1693246616.1043854,"name":"online","context":{"idset":"7"}}
+{"timestamp":1693246616.8306539,"name":"online","context":{"idset":"38,41,46"}}
+{"timestamp":1693246616.9998405,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693246617.3015158,"name":"online","context":{"idset":"37,45"}}
+{"timestamp":1693246617.9405127,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693246618.8259475,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693246619.0103018,"name":"online","context":{"idset":"50"}}
+{"timestamp":1693246620.3006837,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693246623.828541,"name":"online","context":{"idset":"14"}}
+{"timestamp":1693246681.7282598,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693246685.3385677,"name":"online","context":{"idset":"52"}}
+{"timestamp":1693246687.1659594,"name":"online","context":{"idset":"20"}}
+{"timestamp":1693246689.7284229,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693246696.2478094,"name":"online","context":{"idset":"59"}}
+{"timestamp":1693246711.8280735,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693246715.8288367,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693246747.8285139,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693246749.7274325,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693246749.7275262,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693246749.727592,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693246749.8278911,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693246751.7270606,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693246751.7271605,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693246751.8278766,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693246752.6943457,"name":"online","context":{"idset":"18"}}
+{"timestamp":1693246752.7970271,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693246758.8765078,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693246760.9101913,"name":"online","context":{"idset":"21"}}
+{"timestamp":1693246771.866812,"name":"online","context":{"idset":"2,5-6"}}
+{"timestamp":1693246774.0130818,"name":"online","context":{"idset":"50"}}
+{"timestamp":1693246774.1194491,"name":"online","context":{"idset":"23"}}
+{"timestamp":1693246825.8280327,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693246841.7280676,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693246853.8279057,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693246869.8280826,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693246891.7278624,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693246894.9233077,"name":"online","context":{"idset":"4"}}
+{"timestamp":1693246897.7275445,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693246899.7260237,"name":"online","context":{"idset":"56"}}
+{"timestamp":1693246901.8285396,"name":"online","context":{"idset":"8"}}
+{"timestamp":1693246903.8273146,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693246907.7284334,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693246907.7285495,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693246907.7286246,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693246907.828872,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693246911.7270865,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693246911.8278718,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693246915.8281791,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693246919.8277948,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693246989.8281097,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693246990.1996622,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693246996.4254527,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693246998.6789823,"name":"online","context":{"idset":"2"}}
+{"timestamp":1693246999.6497729,"name":"online","context":{"idset":"5"}}
+{"timestamp":1693246999.8284488,"name":"online","context":{"idset":"17"}}
+{"timestamp":1693247006.3961992,"name":"online","context":{"idset":"21"}}
+{"timestamp":1693247007.0697167,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693247031.7283447,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693247031.8281662,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693247033.7278512,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693247033.7279513,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693247033.7280281,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693247033.8286324,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693247037.8282018,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693247066.3751566,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693247067.180301,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693247082.1615579,"name":"online","context":{"idset":"42"}}
+{"timestamp":1693247083.7275469,"name":"online","context":{"idset":"45"}}
+{"timestamp":1693247084.036773,"name":"online","context":{"idset":"46"}}
+{"timestamp":1693247087.7281713,"name":"online","context":{"idset":"56"}}
+{"timestamp":1693247088.2518494,"name":"online","context":{"idset":"26"}}
+{"timestamp":1693247090.8157396,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693247094.2229569,"name":"online","context":{"idset":"35"}}
+{"timestamp":1693247099.2305932,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693247141.7276337,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693247141.8275089,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693247147.8285022,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693247157.7276671,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693247161.728169,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693247218.3531928,"name":"online","context":{"idset":"17"}}
+{"timestamp":1693247218.456598,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693247221.7280188,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693247221.8287649,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693247223.7281735,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693247223.7282741,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693247223.728358,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693247223.7284234,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693247223.7284939,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693247223.7285471,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693247223.8280191,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693247225.1107428,"name":"online","context":{"idset":"50"}}
+{"timestamp":1693247227.1695411,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693247227.1723678,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693247227.2725356,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693247227.5570841,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693247231.7273517,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693247231.7277148,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693247232.2266719,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693247233.908936,"name":"online","context":{"idset":"38,42"}}
+{"timestamp":1693247235.7281055,"name":"online","context":{"idset":"41,43,45-46"}}
+{"timestamp":1693247235.8288655,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693247237.2533014,"name":"online","context":{"idset":"7"}}
+{"timestamp":1693247279.9203253,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693247317.6929073,"name":"online","context":{"idset":"25"}}
+{"timestamp":1693247321.0475767,"name":"online","context":{"idset":"44"}}
+{"timestamp":1693247321.9306896,"name":"online","context":{"idset":"4"}}
+{"timestamp":1693247335.9546912,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693247403.6560361,"name":"online","context":{"idset":"14"}}
+{"timestamp":1693247407.8282571,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693247414.3521643,"name":"online","context":{"idset":"16"}}
+{"timestamp":1693247417.6046586,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693247418.8273289,"name":"online","context":{"idset":"27"}}
+{"timestamp":1693247421.2278509,"name":"online","context":{"idset":"54"}}
+{"timestamp":1693247426.303942,"name":"online","context":{"idset":"57"}}
+{"timestamp":1693247477.7284214,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693247477.8290424,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693247487.8287523,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693247490.9777305,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693247496.1243544,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693247506.2124846,"name":"online","context":{"idset":"37"}}
+{"timestamp":1693247561.5240464,"name":"online","context":{"idset":"59"}}
+{"timestamp":1693247575.2401025,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693247600.5041318,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693247627.7281117,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693247627.7282307,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693247627.7288096,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693247629.8272994,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693247631.7283914,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693247631.7284975,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693247631.7285626,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693247631.7286441,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693247631.728703,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693247631.72876,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693247631.7288151,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693247631.7288775,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693247631.7289374,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693247631.7289889,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693247631.7290418,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693247631.7290959,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693247631.7291479,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693247631.7291994,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693247631.7292471,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693247631.7292955,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693247631.7293522,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693247631.7294025,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693247631.7294638,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693247631.8282695,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693247665.3006978,"name":"online","context":{"idset":"1"}}
+{"timestamp":1693247665.4360294,"name":"online","context":{"idset":"10"}}
+{"timestamp":1693247669.3122108,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693247675.9578905,"name":"online","context":{"idset":"21"}}
+{"timestamp":1693247676.7991834,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693247678.7951756,"name":"online","context":{"idset":"27,30-31,55"}}
+{"timestamp":1693247678.919481,"name":"online","context":{"idset":"34,50,58"}}
+{"timestamp":1693247679.2107301,"name":"online","context":{"idset":"26"}}
+{"timestamp":1693247679.850805,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693247683.8279848,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693247683.9787223,"name":"online","context":{"idset":"20,49"}}
+{"timestamp":1693247688.8124483,"name":"online","context":{"idset":"56"}}
+{"timestamp":1693247696.9690955,"name":"online","context":{"idset":"35"}}
+{"timestamp":1693247703.914669,"name":"online","context":{"idset":"17"}}
+{"timestamp":1693247739.8285422,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693247835.8279738,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693247839.7282696,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693247839.7283754,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693247839.7284431,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693247839.7285163,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693247839.7285841,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693247839.7286549,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693247839.7287123,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693247839.7287681,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693247839.7288239,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693247839.7289672,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693247839.7290699,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693247839.7291484,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693247839.7292051,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693247839.7292778,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693247839.7293351,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693247839.8282318,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693247867.0841136,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693247876.2319517,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693247881.8285556,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693247886.8193557,"name":"online","context":{"idset":"1,5-6,8,55,58"}}
+{"timestamp":1693247886.9331937,"name":"online","context":{"idset":"2,7,10,50"}}
+{"timestamp":1693247891.9786141,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693247897.2075086,"name":"online","context":{"idset":"52"}}
+{"timestamp":1693247941.9539957,"name":"online","context":{"idset":"4,51"}}
+{"timestamp":1693247947.3234401,"name":"online","context":{"idset":"56"}}
+{"timestamp":1693247959.1155753,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693247964.1578486,"name":"online","context":{"idset":"14"}}
+{"timestamp":1693247966.7460475,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693247979.1039138,"name":"online","context":{"idset":"16"}}
+{"timestamp":1693247984.1065822,"name":"online","context":{"idset":"23"}}
+{"timestamp":1693247984.2260528,"name":"online","context":{"idset":"54,57"}}
+{"timestamp":1693248122.505698,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693248147.7277219,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693248147.827687,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693248149.7281175,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693248149.7282372,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693248149.8284523,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693248151.7275338,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693248151.7276597,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693248151.7277317,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693248151.7278106,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693248151.7278736,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693248151.7279336,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693248151.7279935,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693248151.7280536,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693248151.7281115,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693248151.7281682,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693248151.728225,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693248151.7282801,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693248151.7283437,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693248151.7284083,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693248151.7284698,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693248151.7285211,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693248151.7285714,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693248151.7286267,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693248151.7286775,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693248151.7287331,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693248151.7287824,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693248151.828002,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693248218.9777732,"name":"online","context":{"idset":"9,49"}}
+{"timestamp":1693248220.122956,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693248223.8154457,"name":"online","context":{"idset":"1,5,8"}}
+{"timestamp":1693248223.9270294,"name":"online","context":{"idset":"2,7,10,58"}}
+{"timestamp":1693248224.1156971,"name":"online","context":{"idset":"6"}}
+{"timestamp":1693248224.8191993,"name":"online","context":{"idset":"27"}}
+{"timestamp":1693248229.0514832,"name":"online","context":{"idset":"57"}}
+{"timestamp":1693248243.8286254,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693248248.9575667,"name":"online","context":{"idset":"52"}}
+{"timestamp":1693248249.0670018,"name":"online","context":{"idset":"56"}}
+{"timestamp":1693248249.4755399,"name":"online","context":{"idset":"54"}}
+{"timestamp":1693248249.8279853,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693248253.8294196,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693248253.9506502,"name":"online","context":{"idset":"4,26"}}
+{"timestamp":1693248254.070699,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693248256.8553047,"name":"online","context":{"idset":"23"}}
+{"timestamp":1693248259.0645566,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693248264.0147831,"name":"online","context":{"idset":"59"}}
+{"timestamp":1693248287.8265457,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693248315.4223993,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693248331.0062666,"name":"online","context":{"idset":"50"}}
+{"timestamp":1693248389.7270641,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693248389.7272062,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693248389.7272794,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693248389.7273555,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693248389.7274184,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693248389.727479,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693248389.7275386,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693248389.7276006,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693248389.8269923,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693248438.711961,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693248456.8545606,"name":"online","context":{"idset":"2,5-6,8,23"}}
+{"timestamp":1693248457.0055575,"name":"online","context":{"idset":"7"}}
+{"timestamp":1693248486.951257,"name":"online","context":{"idset":"4"}}
+{"timestamp":1693248492.0700982,"name":"online","context":{"idset":"1"}}
+{"timestamp":1693248505.8278461,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693248635.8283539,"name":"online","context":{"idset":"18"}}
+{"timestamp":1693248672.5057969,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693249209.7278256,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693249209.7279246,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693249209.7279851,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693249209.8284388,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693249211.8279312,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693249213.8280454,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693249313.7283864,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693249313.7285211,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693249313.7285991,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693249313.7286761,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693249313.72875,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693249313.728868,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693249313.8287084,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693249315.728529,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693249315.7286837,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693249315.728755,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693249315.7288203,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693249315.7288959,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693249315.729003,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693249315.8282902,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693249317.727442,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693249317.7275367,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693249317.7276242,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693249317.7277045,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693249317.7277973,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693249317.7279005,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693249317.7279956,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693249317.7280853,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693249317.7281764,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693249317.7282679,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693249317.7283695,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693249317.7284522,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693249317.7285345,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693249317.7286229,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693249317.7287066,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693249317.7287855,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693249317.7288623,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693249317.7289343,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693249317.7290077,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693249317.729084,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693249317.7291486,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693249317.7292237,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693249317.7292824,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693249317.7293463,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693249317.7294154,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693249317.7294703,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693249317.7295301,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693249317.7295988,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693249317.828172,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693249319.7279484,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693249319.7280269,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693249319.8283713,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693249323.7276661,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693249323.8285038,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693249350.8079863,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693249354.0226929,"name":"online","context":{"idset":"50"}}
+{"timestamp":1693249355.0132251,"name":"online","context":{"idset":"59"}}
+{"timestamp":1693249355.9447956,"name":"online","context":{"idset":"23"}}
+{"timestamp":1693249358.920095,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693249361.4557779,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693249361.9324372,"name":"online","context":{"idset":"10"}}
+{"timestamp":1693249362.813643,"name":"online","context":{"idset":"18,41"}}
+{"timestamp":1693249362.9555368,"name":"online","context":{"idset":"35,47"}}
+{"timestamp":1693249364.0501544,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693249364.8036423,"name":"online","context":{"idset":"1,5-6,8,16,27,30,32,39-40,43,45,57"}}
+{"timestamp":1693249364.9054291,"name":"online","context":{"idset":"2,4,7,26,37-38,42,49"}}
+{"timestamp":1693249365.0205197,"name":"online","context":{"idset":"28,44,46"}}
+{"timestamp":1693249365.827616,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693249365.9224463,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693249366.5617809,"name":"online","context":{"idset":"15,56"}}
+{"timestamp":1693249366.9084935,"name":"online","context":{"idset":"14,17,20"}}
+{"timestamp":1693249370.7576461,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693249371.3993964,"name":"online","context":{"idset":"21"}}
+{"timestamp":1693249386.6857207,"name":"online","context":{"idset":"25"}}
+{"timestamp":1693249395.0298822,"name":"online","context":{"idset":"22"}}
+{"timestamp":1693249490.4765286,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693249523.9785867,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693249526.9627035,"name":"online","context":{"idset":"52"}}
+{"timestamp":1693249592.1699815,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693249602.8501151,"name":"online","context":{"idset":"54"}}
+{"timestamp":1693249646.6258879,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693249701.9962292,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693249769.7276776,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693249769.7278004,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693249769.7283349,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693249771.7282107,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693249771.7283175,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693249771.7283823,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693249771.8282726,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693249773.7277534,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693249773.7278488,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693249773.7279139,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693249773.7279768,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693249773.7280741,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693249773.7281494,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693249773.7282591,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693249773.7283859,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693249773.7284715,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693249773.7285581,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693249773.7286553,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693249773.8285248,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693249813.9586864,"name":"online","context":{"idset":"38"}}
+{"timestamp":1693249815.8839612,"name":"online","context":{"idset":"8"}}
+{"timestamp":1693249820.9058623,"name":"online","context":{"idset":"10,42"}}
+{"timestamp":1693249821.0265381,"name":"online","context":{"idset":"46"}}
+{"timestamp":1693249821.7275698,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693249825.7987168,"name":"online","context":{"idset":"45"}}
+{"timestamp":1693249830.823122,"name":"online","context":{"idset":"1"}}
+{"timestamp":1693249835.7281854,"name":"online","context":{"idset":"41"}}
+{"timestamp":1693249837.0483673,"name":"online","context":{"idset":"44"}}
+{"timestamp":1693249840.8755522,"name":"online","context":{"idset":"2,6"}}
+{"timestamp":1693249869.8281777,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693249873.7283161,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693249873.7285039,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693249873.7285862,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693249873.7286699,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693249873.8282955,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693249877.82849,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693249915.9544225,"name":"online","context":{"idset":"49,52"}}
+{"timestamp":1693249918.972245,"name":"online","context":{"idset":"37"}}
+{"timestamp":1693249932.760421,"name":"online","context":{"idset":"59"}}
+{"timestamp":1693250026.991787,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693250125.9914112,"name":"online","context":{"idset":"7"}}
+{"timestamp":1693250126.1369481,"name":"online","context":{"idset":"5"}}
+{"timestamp":1693250189.7283154,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693250189.8286779,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693250191.7277105,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693250191.7278502,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693250191.7279644,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693250191.8277013,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693250193.7282069,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693250193.8282816,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693250197.8280065,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693250201.8285422,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693250219.5859504,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693250234.8660038,"name":"online","context":{"idset":"15-16"}}
+{"timestamp":1693250239.0159345,"name":"online","context":{"idset":"26"}}
+{"timestamp":1693250241.8483305,"name":"online","context":{"idset":"29,32"}}
+{"timestamp":1693250244.2271683,"name":"online","context":{"idset":"54"}}
+{"timestamp":1693250245.8285556,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693250246.9854813,"name":"online","context":{"idset":"35"}}
+{"timestamp":1693250250.8556998,"name":"online","context":{"idset":"23"}}
+{"timestamp":1693250342.0500836,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693250346.0647924,"name":"online","context":{"idset":"12,56"}}
+{"timestamp":1693250346.2020891,"name":"online","context":{"idset":"4"}}
+{"timestamp":1693250397.8279624,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693250399.8284357,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693250401.7284689,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693250401.7286391,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693250401.72878,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693250401.8287477,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693250499.7277167,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693250499.7278032,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693250499.7278564,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693250499.8284161,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693250501.7282386,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693250501.7283409,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693250501.7284191,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693250501.728493,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693250501.7285695,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693250501.7286687,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693250501.8284683,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693250503.7280321,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693250503.7281303,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693250503.7282076,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693250503.7282939,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693250503.7283819,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693250503.7284741,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693250503.7285626,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693250503.7286599,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693250503.7287438,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693250503.728827,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693250503.7289078,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693250503.7289853,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693250503.7290599,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693250503.7291319,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693250503.7292013,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693250503.729269,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693250503.7293377,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693250503.7293985,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693250503.7294633,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693250503.729521,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693250503.729583,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693250503.7296486,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693250503.7297094,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693250503.7297611,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693250503.7298198,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693250503.7298679,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693250503.7299297,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693250503.7299759,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693250503.8283412,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693250507.7274666,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693250507.7275398,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693250507.7275827,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693250507.7276411,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693250507.8284366,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693250630.9560134,"name":"online","context":{"idset":"38"}}
+{"timestamp":1693250631.3571861,"name":"online","context":{"idset":"5-6"}}
+{"timestamp":1693250639.3214192,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693250640.851176,"name":"online","context":{"idset":"23"}}
+{"timestamp":1693250640.9575999,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693250643.1669884,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693250645.9322531,"name":"online","context":{"idset":"10"}}
+{"timestamp":1693250648.2035992,"name":"online","context":{"idset":"52"}}
+{"timestamp":1693250651.9965572,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693250653.2262347,"name":"online","context":{"idset":"50"}}
+{"timestamp":1693250653.8281572,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693250653.9743721,"name":"online","context":{"idset":"9,20,22,43,46"}}
+{"timestamp":1693250654.0787053,"name":"online","context":{"idset":"24,40"}}
+{"timestamp":1693250655.8284891,"name":"online","context":{"idset":"2,8,14,42"}}
+{"timestamp":1693250656.0026195,"name":"online","context":{"idset":"1,7,41,44-45,57"}}
+{"timestamp":1693250658.1021299,"name":"online","context":{"idset":"25,29"}}
+{"timestamp":1693250658.2548478,"name":"online","context":{"idset":"26"}}
+{"timestamp":1693250658.8945799,"name":"online","context":{"idset":"21,54"}}
+{"timestamp":1693250659.1045763,"name":"online","context":{"idset":"16"}}
+{"timestamp":1693250660.9759171,"name":"online","context":{"idset":"37"}}
+{"timestamp":1693250661.0947049,"name":"online","context":{"idset":"18"}}
+{"timestamp":1693250661.4791937,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693250662.8466451,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693250663.2150648,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693250663.8289287,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693250664.126379,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693250665.9629242,"name":"online","context":{"idset":"4"}}
+{"timestamp":1693250666.1921289,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693250672.6732056,"name":"online","context":{"idset":"59"}}
+{"timestamp":1693250673.3092747,"name":"online","context":{"idset":"56"}}
+{"timestamp":1693250767.7282393,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693250852.671515,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693250861.5034227,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693250954.0578132,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693250962.0685518,"name":"online","context":{"idset":"27"}}
+{"timestamp":1693251117.8279755,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693251119.728014,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693251119.7281141,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693251119.7281764,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693251119.7282276,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693251119.7282784,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693251119.7283309,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693251119.7283826,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693251119.8285589,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693251121.7279451,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693251121.8282111,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693251225.7276278,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693251225.7278109,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693251225.8276732,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693251227.7280262,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693251227.7281415,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693251227.7282217,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693251227.7283001,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693251227.7283731,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693251227.7284436,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693251227.7285132,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693251227.7285845,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693251227.7286658,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693251227.7287502,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693251227.8280931,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693251229.7280929,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693251229.7282059,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693251229.728277,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693251229.7283437,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693251229.7284069,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693251229.7284687,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693251229.7285278,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693251229.7285881,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693251229.7286582,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693251229.7287164,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693251229.7287714,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693251229.7288284,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693251229.728884,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693251229.7289591,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693251229.7290173,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693251229.7290676,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693251229.7291405,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693251229.7292173,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693251229.7293017,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693251229.7330067,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693251229.8331585,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693251233.8277719,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693251237.7280312,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693251237.7281263,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693251237.8283551,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693251243.8270841,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693251321.8210068,"name":"online","context":{"idset":"21"}}
+{"timestamp":1693251322.0816302,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693251322.4593964,"name":"online","context":{"idset":"37"}}
+{"timestamp":1693251322.9764016,"name":"online","context":{"idset":"19-20"}}
+{"timestamp":1693251324.0605283,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693251324.8296452,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693251324.9552641,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693251325.828054,"name":"online","context":{"idset":"8"}}
+{"timestamp":1693251326.9349396,"name":"online","context":{"idset":"10,26,38,50"}}
+{"timestamp":1693251327.2262843,"name":"online","context":{"idset":"41"}}
+{"timestamp":1693251327.7279689,"name":"online","context":{"idset":"22"}}
+{"timestamp":1693251327.8287475,"name":"online","context":{"idset":"15,18"}}
+{"timestamp":1693251328.0542421,"name":"online","context":{"idset":"57"}}
+{"timestamp":1693251328.8877747,"name":"online","context":{"idset":"54,58"}}
+{"timestamp":1693251329.0500107,"name":"online","context":{"idset":"27,30,56"}}
+{"timestamp":1693251329.1908898,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693251329.5264368,"name":"online","context":{"idset":"46"}}
+{"timestamp":1693251330.4772656,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693251330.8764143,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693251331.6582744,"name":"online","context":{"idset":"14"}}
+{"timestamp":1693251331.8281381,"name":"online","context":{"idset":"5,12,29"}}
+{"timestamp":1693251331.9294434,"name":"online","context":{"idset":"2,7,25"}}
+{"timestamp":1693251332.0619757,"name":"online","context":{"idset":"1,23,45"}}
+{"timestamp":1693251332.7077026,"name":"online","context":{"idset":"52"}}
+{"timestamp":1693251333.4084635,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693251333.7279158,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693251334.0084414,"name":"online","context":{"idset":"32,44,59"}}
+{"timestamp":1693251334.1543717,"name":"online","context":{"idset":"42"}}
+{"timestamp":1693251337.1108379,"name":"online","context":{"idset":"6"}}
+{"timestamp":1693251339.8290968,"name":"online","context":{"idset":"40"}}
+{"timestamp":1693251341.9518526,"name":"online","context":{"idset":"4,51"}}
+{"timestamp":1693251415.415638,"name":"online","context":{"idset":"17"}}
+{"timestamp":1693251416.4716733,"name":"online","context":{"idset":"35"}}
+{"timestamp":1693251434.5689206,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693251485.8286095,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693251487.8277962,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693251491.8288057,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693251515.8286171,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693251587.8288262,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693251589.7278519,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693251589.7280548,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693251589.7281928,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693251589.7282832,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693251589.7290378,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693251590.1950302,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693251590.1992748,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693251590.1994133,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693251590.1995609,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693251590.1996784,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693251590.1997859,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693251590.1998916,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693251590.2999098,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693251593.7278228,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693251593.7279546,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693251593.7280409,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693251593.7281208,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693251593.7281947,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693251593.7283347,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693251593.7284312,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693251593.7285299,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693251593.7286372,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693251593.7287428,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693251593.7288218,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693251593.7289112,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693251593.728992,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693251593.7290812,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693251593.7291734,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693251593.7292583,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693251593.7293322,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693251593.729418,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693251593.7294934,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693251593.7295675,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693251593.7296457,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693251593.7297323,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693251593.7298031,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693251593.7298667,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693251593.7299418,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693251593.827919,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693251593.9325905,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693251597.7275393,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693251597.7276435,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693251597.7276981,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693251597.7277458,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693251597.8281169,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693251600.8253686,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693251600.9302022,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693251600.9302354,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693251605.7280309,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693251606.9544141,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693251608.3949335,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693251608.8013794,"name":"online","context":{"idset":"57"}}
+{"timestamp":1693251610.8297935,"name":"online","context":{"idset":"21,39-40"}}
+{"timestamp":1693251610.9072332,"name":"online","context":{"idset":"20,54"}}
+{"timestamp":1693251611.0121319,"name":"online","context":{"idset":"59"}}
+{"timestamp":1693251611.545907,"name":"online","context":{"idset":"44"}}
+{"timestamp":1693251612.0138402,"name":"online","context":{"idset":"22"}}
+{"timestamp":1693251613.7079823,"name":"online","context":{"idset":"38"}}
+{"timestamp":1693251613.8273585,"name":"online","context":{"idset":"42"}}
+{"timestamp":1693251614.6642272,"name":"online","context":{"idset":"17"}}
+{"timestamp":1693251615.7279313,"name":"online","context":{"idset":"1,45,55"}}
+{"timestamp":1693251615.8287203,"name":"online","context":{"idset":"5-6,8,14,18"}}
+{"timestamp":1693251615.9292252,"name":"online","context":{"idset":"2,4,7,52,58"}}
+{"timestamp":1693251617.9337277,"name":"online","context":{"idset":"10,27"}}
+{"timestamp":1693251618.0820732,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693251618.2247114,"name":"online","context":{"idset":"35"}}
+{"timestamp":1693251619.8289905,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693251626.0251019,"name":"online","context":{"idset":"46"}}
+{"timestamp":1693251630.1173167,"name":"online","context":{"idset":"16"}}
+{"timestamp":1693251637.1154118,"name":"online","context":{"idset":"23"}}
+{"timestamp":1693251637.9807327,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693251638.8522604,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693251639.0053697,"name":"online","context":{"idset":"26"}}
+{"timestamp":1693251639.8025968,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693251640.3590527,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693251640.9610474,"name":"online","context":{"idset":"19,37"}}
+{"timestamp":1693251641.4287882,"name":"online","context":{"idset":"25"}}
+{"timestamp":1693251645.7274544,"name":"online","context":{"idset":"41,56"}}
+{"timestamp":1693251645.827693,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693251648.8390005,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693251747.8289204,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693251849.0513117,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693253631.8282828,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693253633.7282653,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693253633.7283764,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693253633.7284627,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693253633.7285419,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693253633.7286623,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693253633.7287362,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693253633.728802,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693253633.8282263,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693253673.9798386,"name":"online","context":{"idset":"20"}}
+{"timestamp":1693253680.8875849,"name":"online","context":{"idset":"25,31"}}
+{"timestamp":1693253682.5982401,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693253840.8647473,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693253860.9315009,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693253861.0733101,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693253861.5720398,"name":"online","context":{"idset":"27"}}
+{"timestamp":1693254036.419229,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693254253.8285711,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693254350.9943945,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693254361.8287251,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693254373.8110509,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693254507.7280087,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693254507.8286881,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693254509.8287618,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693254517.8280869,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693254560.980438,"name":"online","context":{"idset":"20"}}
+{"timestamp":1693254561.8294449,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693254566.527462,"name":"online","context":{"idset":"23,28"}}
+{"timestamp":1693254621.7286854,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693254625.7283931,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693254625.7285209,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693254625.7285936,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693254625.7286706,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693254625.72875,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693254625.7288134,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693254625.8290501,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693254669.7281466,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693254723.0638697,"name":"online","context":{"idset":"56"}}
+{"timestamp":1693254725.7278531,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693254725.7279818,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693254725.7284942,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693254727.7283275,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693254727.7284451,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693254727.7285247,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693254727.7286329,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693254727.7287185,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693254727.7288296,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693254727.7299631,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693254727.9566422,"name":"online","context":{"idset":"50,52,57"}}
+{"timestamp":1693254728.0588846,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693254729.7285614,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693254731.7277696,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693254731.8279901,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693254731.9330044,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693254732.9762888,"name":"online","context":{"idset":"54"}}
+{"timestamp":1693254734.8920453,"name":"online","context":{"idset":"21"}}
+{"timestamp":1693254734.8958378,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693254734.8959253,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693254734.9965837,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693254744.8507445,"name":"online","context":{"idset":"12,17"}}
+{"timestamp":1693254747.8288474,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693254749.0787232,"name":"online","context":{"idset":"2,24"}}
+{"timestamp":1693254749.1918745,"name":"online","context":{"idset":"7"}}
+{"timestamp":1693254749.8282735,"name":"online","context":{"idset":"15,39"}}
+{"timestamp":1693254749.958926,"name":"online","context":{"idset":"35,46"}}
+{"timestamp":1693254750.7438488,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693254751.5498507,"name":"online","context":{"idset":"45"}}
+{"timestamp":1693254751.9627802,"name":"online","context":{"idset":"37"}}
+{"timestamp":1693254756.8049965,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693254759.3173931,"name":"online","context":{"idset":"1"}}
+{"timestamp":1693254774.9445207,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693254777.8287218,"name":"online","context":{"idset":"16"}}
+{"timestamp":1693254778.0929148,"name":"online","context":{"idset":"18"}}
+{"timestamp":1693254778.8739839,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693255099.7276561,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693255099.8283579,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693255103.7284472,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693255103.7285554,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693255103.8293011,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693255150.8667035,"name":"online","context":{"idset":"2,4,6"}}
+{"timestamp":1693255151.727566,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693255152.0074213,"name":"online","context":{"idset":"50"}}
+{"timestamp":1693255413.7282999,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693255413.7284224,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693255413.8284681,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693255415.7279458,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693255415.8279619,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693255417.7280772,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693255417.7281885,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693255417.7282629,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693255417.7283356,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693255417.7283957,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693255417.7284553,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693255417.7285123,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693255417.7285666,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693255417.7286344,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693255417.728693,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693255417.7287471,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693255417.7287991,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693255417.7288501,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693255417.7289135,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693255417.8283391,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693255459.8284254,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693255464.8203862,"name":"online","context":{"idset":"40,42-43"}}
+{"timestamp":1693255464.9571726,"name":"online","context":{"idset":"38"}}
+{"timestamp":1693255465.8122294,"name":"online","context":{"idset":"41"}}
+{"timestamp":1693255467.8010271,"name":"online","context":{"idset":"45"}}
+{"timestamp":1693255468.0290062,"name":"online","context":{"idset":"46"}}
+{"timestamp":1693255469.8786981,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693255470.0473862,"name":"online","context":{"idset":"44"}}
+{"timestamp":1693255479.9635086,"name":"online","context":{"idset":"37"}}
+{"timestamp":1693255484.9567225,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693255523.7284245,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693255523.7285247,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693255523.728579,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693255523.8287282,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693255525.8282893,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693255562.8180475,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693255572.7641864,"name":"online","context":{"idset":"26"}}
+{"timestamp":1693255572.9537148,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693255590.9780042,"name":"online","context":{"idset":"49"}}
+{"timestamp":1693255596.0637815,"name":"online","context":{"idset":"57"}}
+{"timestamp":1693255625.8285022,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693255627.8278694,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693255629.7280278,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693255629.8286018,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693255727.9970677,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693255733.8280127,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693255737.728394,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693255737.7285128,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693255737.8285792,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693255783.7281635,"name":"online","context":{"idset":"54"}}
+{"timestamp":1693255791.5089231,"name":"online","context":{"idset":"52"}}
+{"timestamp":1693255792.7386715,"name":"online","context":{"idset":"9"}}
+{"timestamp":1693255793.9556961,"name":"online","context":{"idset":"38"}}
+{"timestamp":1693255794.4488392,"name":"online","context":{"idset":"51"}}
+{"timestamp":1693255795.9754062,"name":"online","context":{"idset":"35"}}
+{"timestamp":1693255802.915926,"name":"online","context":{"idset":"58"}}
+{"timestamp":1693255821.0073483,"name":"online","context":{"idset":"50"}}
+{"timestamp":1693255853.8284473,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693255855.7257397,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693255855.7258396,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693255855.7259061,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693255855.8261759,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693255957.7277002,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693255957.8284173,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693255959.8284674,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693255995.065619,"name":"online","context":{"idset":"27,29"}}
+{"timestamp":1693256002.933326,"name":"online","context":{"idset":"25"}}
+{"timestamp":1693256013.0952597,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693256015.2731969,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693256057.8286335,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693256061.8283105,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693256088.1864371,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693256109.9785988,"name":"online","context":{"idset":"54"}}
+{"timestamp":1693256111.0048032,"name":"online","context":{"idset":"26"}}
+{"timestamp":1693256111.9499555,"name":"online","context":{"idset":"4"}}
+{"timestamp":1693256113.5918822,"name":"online","context":{"idset":"18"}}
+{"timestamp":1693256116.3024168,"name":"online","context":{"idset":"56"}}
+{"timestamp":1693256118.3097334,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693256161.8283064,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693256163.7274756,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693256163.7276511,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693256163.7277813,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693256163.7279162,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693256163.7280295,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693256163.7281342,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693256163.7282131,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693256163.828269,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693256191.2434163,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693256206.3293948,"name":"online","context":{"idset":"39"}}
+{"timestamp":1693256216.1921885,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693256220.9559236,"name":"online","context":{"idset":"37,47"}}
+{"timestamp":1693256221.206394,"name":"online","context":{"idset":"38"}}
+{"timestamp":1693256221.3106899,"name":"online","context":{"idset":"41,44"}}
+{"timestamp":1693256221.4196596,"name":"online","context":{"idset":"42"}}
+{"timestamp":1693256223.838403,"name":"online","context":{"idset":"40"}}
+{"timestamp":1693256303.1249208,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693256307.0705628,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693256421.3078907,"name":"online","context":{"idset":"43"}}
+{"timestamp":1693256438.3060017,"name":"online","context":{"idset":"57"}}
+{"timestamp":1693259867.728287,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693259867.7284322,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693259867.7285266,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693259867.728631,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693259867.7287486,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693259867.728837,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693259867.7289176,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693259867.7290003,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693259867.7290809,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693259867.7291622,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693259867.7292428,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693259867.7293262,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693259867.729414,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693259867.7294974,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693259867.7295914,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693259867.7296898,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693259867.7297647,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693259867.729845,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693259867.7299323,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693259867.7300117,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693259867.7300951,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693259867.7301743,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693259867.7302563,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693259867.7303336,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693259867.7304122,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693259867.730485,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693259867.7305629,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693259867.7306414,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693259867.7307181,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693259867.7307825,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693259867.7308562,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693259867.7309246,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693259867.7309906,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693259867.7310624,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693259867.7311285,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693259867.7311852,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693259867.7312508,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693259867.7313108,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693259867.7313709,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693259867.7314341,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693259867.7314925,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693259867.7315433,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693259867.7316148,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693259867.7316673,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693259867.7317233,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693259867.7317691,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693259867.7318196,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693259867.7318747,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693259867.7319217,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693259867.7319639,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693259867.7320042,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693259867.7320471,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693259867.7320857,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693259867.828738,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693261490.4632242,"name":"online","context":{"idset":"6-7,9,13,15,21-23,26,34-35,39,47,49,57-58"}}
+{"timestamp":1693261490.5774987,"name":"online","context":{"idset":"2,5,8,10,14,16,18-19,27-31,33,36-38,41-42,45,53-56,59"}}
+{"timestamp":1693261490.6802516,"name":"online","context":{"idset":"1,3-4,11,17,24-25,32,43-44,46,50-52"}}
+{"timestamp":1693261490.7914672,"name":"online","context":{"idset":"12,20,40"}}
+{"timestamp":1693262658.6002738,"name":"undrain","context":{"idset":"1"}}
+{"timestamp":1693262675.4027605,"name":"undrain","context":{"idset":"2-35,37-60"}}
+{"timestamp":1693352765.728044,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1693352765.7281997,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1693352765.7282627,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1693352765.8287747,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7282388,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7283065,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7283421,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7283812,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7284131,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7284534,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.728487,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7285171,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7285521,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.728586,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7286272,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7286654,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7286935,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7287273,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7287569,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7287872,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7288182,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7288485,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.7288818,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1693352767.8292484,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7272124,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7273011,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7273481,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7273939,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7274349,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7274747,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7275112,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7275467,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7275836,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7276332,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7276754,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7277246,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7277629,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7278016,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7278388,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7278783,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7279162,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7279587,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7279987,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7280393,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7280805,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7281239,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7281673,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7282119,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7282524,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7282937,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7283342,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7283785,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7284172,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7284565,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7285175,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7285719,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.7286236,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1693352769.8275144,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1693352831.7276905,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693352831.727967,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693352831.7281656,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693352831.728343,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693352831.7285054,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693352831.7286842,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693352831.7288578,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693352831.7290227,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693352831.7291906,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693352831.7293556,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693352831.7295191,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1693352831.729686,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693352831.7298453,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1693352831.7300029,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693352831.7301571,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693352831.7303097,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693352831.7304609,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693352831.7306178,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693352831.7307684,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693352831.7309144,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693352831.7310581,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693352831.7311978,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693352831.7313378,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693352831.7314723,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693352831.7315733,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693352831.7316532,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693352831.7317204,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693352831.7318027,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693352831.7319002,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693352831.7319655,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693352831.7320766,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693352831.7321556,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693352831.7322547,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693352831.7323289,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693352831.732424,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1693352831.7324948,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693352831.7325895,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693352831.7326703,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693352831.7327862,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693352831.7328801,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693352831.7329521,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693352831.7330422,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693352831.7331183,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693352831.7332149,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693352831.7332811,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693352831.733382,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693352831.7334454,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693352831.7335262,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693352831.7335954,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693352831.7336879,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1693352831.7337608,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693352831.7338402,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693352831.7339146,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693352831.8277373,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693352833.7278011,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693352833.7279332,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693352833.7280202,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693352833.8284607,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693492682.5262864,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1693352767.7282388,"reason":"broker was unresponsive"},"2":{"timestamp":1693352767.7283065,"reason":"broker was unresponsive"},"3":{"timestamp":1693352767.7283421,"reason":"broker was unresponsive"},"4":{"timestamp":1693352765.728044,"reason":"broker was unresponsive"},"5":{"timestamp":1693352767.7283812,"reason":"broker was unresponsive"},"6":{"timestamp":1693352767.7284131,"reason":"broker was unresponsive"},"7":{"timestamp":1693352767.7284534,"reason":"broker was unresponsive"},"8":{"timestamp":1693352769.7272124,"reason":"broker was unresponsive"},"9":{"timestamp":1693352767.728487,"reason":"broker was unresponsive"},"10":{"timestamp":1693352769.7273011,"reason":"broker was unresponsive"},"11":{"timestamp":1693352767.7285171,"reason":"broker was unresponsive"},"12":{"timestamp":1693352769.7273481,"reason":"broker was unresponsive"},"13":{"timestamp":1693352769.7273939,"reason":"broker was unresponsive"},"14":{"timestamp":1693352767.7285521,"reason":"broker was unresponsive"},"15":{"timestamp":1693352769.7274349,"reason":"broker was unresponsive"},"16":{"timestamp":1693352765.7281997,"reason":"broker was unresponsive"},"17":{"timestamp":1693352767.728586,"reason":"broker was unresponsive"},"18":{"timestamp":1693352767.7286272,"reason":"broker was unresponsive"},"19":{"timestamp":1693352767.7286654,"reason":"broker was unresponsive"},"20":{"timestamp":1693352767.7286935,"reason":"broker was unresponsive"},"21":{"timestamp":1693352765.7282627,"reason":"broker was unresponsive"},"22":{"timestamp":1693352769.7274747,"reason":"broker was unresponsive"},"23":{"timestamp":1693352767.7287273,"reason":"broker was unresponsive"},"24":{"timestamp":1693352765.8287747,"reason":"broker was unresponsive"},"25":{"timestamp":1693352767.7287569,"reason":"broker was unresponsive"},"26":{"timestamp":1693352767.7287872,"reason":"broker was unresponsive"},"27":{"timestamp":1693352767.7288182,"reason":"broker was unresponsive"},"28":{"timestamp":1693352769.7275112,"reason":"broker was unresponsive"},"29":{"timestamp":1693352769.7275467,"reason":"broker was unresponsive"},"30":{"timestamp":1693352767.7288485,"reason":"broker was unresponsive"},"31":{"timestamp":1693352767.7288818,"reason":"broker was unresponsive"},"32":{"timestamp":1693352767.8292484,"reason":"broker was unresponsive"},"33":{"timestamp":1693352769.7275836,"reason":"broker was unresponsive"},"34":{"timestamp":1693352769.7276332,"reason":"broker was unresponsive"},"35":{"timestamp":1693352769.7276754,"reason":"broker was unresponsive"},"36":{"timestamp":1693352769.7277246,"reason":"broker was unresponsive"},"37":{"timestamp":1693352769.7277629,"reason":"broker was unresponsive"},"38":{"timestamp":1693352769.7278016,"reason":"broker was unresponsive"},"39":{"timestamp":1693352769.7278388,"reason":"broker was unresponsive"},"40":{"timestamp":1693352769.7278783,"reason":"broker was unresponsive"},"41":{"timestamp":1693352769.7279162,"reason":"broker was unresponsive"},"42":{"timestamp":1693352769.7279587,"reason":"broker was unresponsive"},"43":{"timestamp":1693352769.7279987,"reason":"broker was unresponsive"},"44":{"timestamp":1693352769.7280393,"reason":"broker was unresponsive"},"45":{"timestamp":1693352769.7280805,"reason":"broker was unresponsive"},"46":{"timestamp":1693352769.7281239,"reason":"broker was unresponsive"},"47":{"timestamp":1693352769.7281673,"reason":"broker was unresponsive"},"49":{"timestamp":1693352769.7282119,"reason":"broker was unresponsive"},"50":{"timestamp":1693352769.7282524,"reason":"broker was unresponsive"},"51":{"timestamp":1693352769.7282937,"reason":"broker was unresponsive"},"52":{"timestamp":1693352769.7283342,"reason":"broker was unresponsive"},"53":{"timestamp":1693352769.7283785,"reason":"broker was unresponsive"},"54":{"timestamp":1693352769.7284172,"reason":"broker was unresponsive"},"55":{"timestamp":1693352769.7284565,"reason":"broker was unresponsive"},"56":{"timestamp":1693352769.7285175,"reason":"broker was unresponsive"},"57":{"timestamp":1693352769.7285719,"reason":"broker was unresponsive"},"58":{"timestamp":1693352769.7286236,"reason":"broker was unresponsive"},"59":{"timestamp":1693352769.8275144,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1693492682.5306196,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1693492694.1551588,"name":"online","context":{"idset":"0"}}
+{"timestamp":1693537503.4105253,"name":"online","context":{"idset":"1"}}
+{"timestamp":1693537528.1918509,"name":"online","context":{"idset":"6,11-12,14-15,21-22,24,30"}}
+{"timestamp":1693537528.3038995,"name":"online","context":{"idset":"2-3,8-9,13,18-20,23,26,28,35-37,39-40,44,50"}}
+{"timestamp":1693537528.41085,"name":"online","context":{"idset":"4-5,7,10,16-17,27,29,31-32,34,38,41-42,45,47,51-52,54,57-58"}}
+{"timestamp":1693537528.5580263,"name":"online","context":{"idset":"25,33,43,49,55-56,59-60"}}
+{"timestamp":1693537528.6603544,"name":"online","context":{"idset":"46,53"}}
+{"timestamp":1693537567.8300478,"name":"undrain","context":{"idset":"1-47,49-59"}}
+{"timestamp":1693538078.9379101,"name":"online","context":{"idset":"48"}}
+{"timestamp":1693694943.3867974,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1693695011.3857727,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693695069.2865586,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.286694,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2867548,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2867925,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2868257,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2868564,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2868881,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2869265,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2869587,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2869978,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2870383,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.287077,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2871127,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2871463,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2871799,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2872162,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2872515,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.2872941,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1693695069.3874495,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1693695073.2863324,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1693695073.2864509,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1693695073.286536,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1693695073.3870902,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1693695097.2866645,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1693695097.3875396,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1693695723.3862612,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693695979.7910051,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693696213.3864031,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693696315.1277812,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693696975.3864825,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693697069.0099201,"name":"online","context":{"idset":"46"}}
+{"timestamp":1693697891.286464,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1693697891.2867064,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1693697891.2867622,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1693697891.2868063,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1693697891.387296,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2859883,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2861421,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2862203,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2862933,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2863638,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2864363,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2865071,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2866189,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2866967,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2868054,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2869196,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.286998,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2870793,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2871609,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2873082,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2873948,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2874799,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2875633,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2877204,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2878063,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.2878833,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1693697893.3868427,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1693697895.2865872,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1693697895.2867243,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1693697895.2868671,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1693697895.2869587,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1693697895.2872043,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1693697895.287298,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1693697895.3867836,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1693697957.2863674,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1693697957.286516,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1693697957.2866275,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693697957.2867262,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693697957.2868586,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693697957.286948,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693697957.2870548,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693697957.2871749,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693697957.2872765,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693697957.287375,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1693697957.2874722,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693697957.2875555,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1693697957.2876599,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693697957.2877507,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693697957.2878394,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693697957.2879286,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693697957.2880161,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693697957.2881031,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693697957.2881885,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693697957.2882729,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693697957.2883565,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693697957.2884417,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693697957.2885251,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693697957.2886147,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693697957.2886875,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693697957.2887714,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693697957.2888458,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693697957.2889254,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693697957.2889934,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693697957.2890704,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693697957.2891414,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693697957.2892094,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693697957.2892818,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693697957.2893531,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693697957.2894168,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1693697957.2894914,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1693697957.2895565,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1693697957.289619,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1693697957.2896833,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1693697957.2897453,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1693697957.2898049,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1693697957.2898705,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1693697957.2899258,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1693697957.2899892,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1693697957.2900465,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1693697957.2900989,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1693697957.2901602,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1693697957.2902105,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1693697957.2902668,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1693697957.290318,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1693697957.2903662,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1693697957.2904158,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1693697957.2904692,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1693697957.2905161,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1693697957.290566,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1693697957.290616,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1693697957.2906599,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1693697957.2907062,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1693697957.3867562,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1693697957.7489026,"name":"drain","context":{"idset":"1-55","reason":"epilog failed for jobid f5bsmkoeXhq","overwrite":0}}
+{"timestamp":1693698594.2417126,"name":"online","context":{"idset":"54"}}
+{"timestamp":1693698594.3725836,"name":"online","context":{"idset":"1,8,22-23,59"}}
+{"timestamp":1693698594.4905193,"name":"online","context":{"idset":"4,43,51"}}
+{"timestamp":1693698594.8196485,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693698594.9209061,"name":"online","context":{"idset":"6,11,15,19,34,37,56"}}
+{"timestamp":1693698595.0353584,"name":"online","context":{"idset":"2-3,12-13,16-17,20,24,26-27,29,31-32,48,58,60"}}
+{"timestamp":1693698595.188755,"name":"online","context":{"idset":"5,7,14,18,21,30,49,53"}}
+{"timestamp":1693698595.6875076,"name":"online","context":{"idset":"47"}}
+{"timestamp":1693698595.8034315,"name":"online","context":{"idset":"10"}}
+{"timestamp":1693699615.1986578,"name":"online","context":{"idset":"38,41,57"}}
+{"timestamp":1693699615.2863805,"name":"online","context":{"idset":"39,45-46,50"}}
+{"timestamp":1693699615.3866997,"name":"online","context":{"idset":"9,25,28,42,44,52"}}
+{"timestamp":1693699615.5259337,"name":"online","context":{"idset":"55"}}
+{"timestamp":1693699790.3652742,"name":"undrain","context":{"idset":"1-34,37-60"}}
+{"timestamp":1693944248.8095753,"name":"drain","context":{"idset":"29","reason":"reason=Memory failure: buffer overflow when queuing memory failure","overwrite":0}}
+{"timestamp":1693953895.1715546,"name":"online","context":{"idset":"40"}}
+{"timestamp":1693958069.7878468,"name":"drain","context":{"idset":"29","reason":"Memory failure: buffer overflow when queuing memory failure","overwrite":1}}
+{"timestamp":1693958112.0448079,"name":"drain","context":{"idset":"35-36","reason":"PY: hsi0 stuck in starting","overwrite":1}}
+{"timestamp":1693958119.1671712,"name":"drain","context":{"idset":"29","reason":"PY: Memory failure: buffer overflow when queuing memory failure","overwrite":1}}
+{"timestamp":1693958124.5412202,"name":"drain","context":{"idset":"29","reason":"PY: memory failure: buffer overflow when queuing memory failure","overwrite":1}}
+{"timestamp":1694018413.5962143,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1694031160.7603185,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1694032534.1937561,"name":"online","context":{"idset":"17"}}
+{"timestamp":1694034069.5428836,"name":"online","context":{"idset":"29"}}
+{"timestamp":1694035763.5869501,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1694035821.781873,"name":"drain","context":{"idset":"17","reason":"epilog failed for jobid f6Na8TwZjAo","overwrite":0}}
+{"timestamp":1694045081.3865988,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1694045083.285749,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1694045083.2858407,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1694045083.2858973,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1694045083.285933,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1694045083.2859795,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1694045083.2860203,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1694045083.3858693,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2861023,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2861745,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2862296,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2862649,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2863011,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2863326,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.286366,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2864058,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2864432,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2864785,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.28652,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2865622,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2866223,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2866721,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2867312,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2867692,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2868152,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2868686,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2869189,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2869754,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2870274,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2870786,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2871203,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.287174,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2872231,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.287266,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2873178,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2873669,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2874191,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2874725,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2875249,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2875757,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2876472,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2877064,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2877612,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2878191,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2878747,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.287931,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2879899,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2880406,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.288101,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2881501,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2881939,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2882488,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.288286,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2883401,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.2883787,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1694045085.3870142,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1694045147.2859612,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1694045147.2861226,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1694045147.2862184,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1694045147.2863064,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1694045147.2863917,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1694045147.2864733,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1694045147.2865536,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1694045147.2866549,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1694045147.2867665,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1694045147.2868922,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1694045147.2870009,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1694045147.2871079,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1694045147.2872157,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1694045147.2873237,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1694045147.2874303,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1694045147.2875509,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1694045147.2876651,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1694045147.2877684,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1694045147.2878699,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1694045147.2879634,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1694045147.2880628,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1694045147.288161,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1694045147.2882628,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1694045147.2883689,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1694045147.2884619,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1694045147.2885578,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1694045147.2886622,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1694045147.288749,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1694045147.2888379,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1694045147.2889295,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1694045147.2890115,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1694045147.2891066,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1694045147.289201,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1694045147.2892847,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1694045147.2893658,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1694045147.289443,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1694045147.2895203,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1694045147.2896008,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1694045147.2896667,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1694045147.2897553,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1694045147.2898405,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1694045147.2899215,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1694045147.2900031,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1694045147.2900701,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1694045147.2901514,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1694045147.2902217,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1694045147.2902868,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1694045147.2903395,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1694045147.2904127,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1694045147.2904768,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1694045147.2905345,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1694045147.290585,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1694045147.290663,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1694045147.2907128,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1694045147.2907782,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1694045147.2908373,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1694045147.3856883,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1694048131.4953961,"name":"online","context":{"idset":"16"}}
+{"timestamp":1694048131.6316783,"name":"online","context":{"idset":"28"}}
+{"timestamp":1694048131.7973208,"name":"online","context":{"idset":"26,29"}}
+{"timestamp":1694048131.9278238,"name":"online","context":{"idset":"4,7,21,25,27"}}
+{"timestamp":1694048132.0547011,"name":"online","context":{"idset":"3,33,46,54"}}
+{"timestamp":1694048132.1135743,"name":"online","context":{"idset":"1"}}
+{"timestamp":1694048132.1622367,"name":"online","context":{"idset":"50"}}
+{"timestamp":1694048132.2649224,"name":"online","context":{"idset":"32"}}
+{"timestamp":1694048132.3134923,"name":"online","context":{"idset":"34"}}
+{"timestamp":1694048132.6105139,"name":"online","context":{"idset":"39,55"}}
+{"timestamp":1694048132.6696324,"name":"online","context":{"idset":"31"}}
+{"timestamp":1694048132.7236373,"name":"online","context":{"idset":"17"}}
+{"timestamp":1694048132.8553081,"name":"online","context":{"idset":"6,10,24,57"}}
+{"timestamp":1694048132.9835989,"name":"online","context":{"idset":"60"}}
+{"timestamp":1694048133.085777,"name":"online","context":{"idset":"11"}}
+{"timestamp":1694048133.1601117,"name":"online","context":{"idset":"2,58"}}
+{"timestamp":1694048133.2096198,"name":"online","context":{"idset":"14,19"}}
+{"timestamp":1694048133.386055,"name":"online","context":{"idset":"8,12,23"}}
+{"timestamp":1694048133.5177369,"name":"online","context":{"idset":"43"}}
+{"timestamp":1694048133.6705806,"name":"online","context":{"idset":"18,37"}}
+{"timestamp":1694048133.9778192,"name":"online","context":{"idset":"45,47"}}
+{"timestamp":1694048133.9949996,"name":"online","context":{"idset":"41"}}
+{"timestamp":1694048134.0645719,"name":"online","context":{"idset":"49"}}
+{"timestamp":1694048134.127471,"name":"online","context":{"idset":"51"}}
+{"timestamp":1694048134.20052,"name":"online","context":{"idset":"5,44,52"}}
+{"timestamp":1694048134.5137467,"name":"online","context":{"idset":"9,42"}}
+{"timestamp":1694048134.7068307,"name":"online","context":{"idset":"30,38"}}
+{"timestamp":1694048134.9857271,"name":"online","context":{"idset":"15,48"}}
+{"timestamp":1694048135.1020162,"name":"online","context":{"idset":"20,22"}}
+{"timestamp":1694048135.2244043,"name":"online","context":{"idset":"13,53,56"}}
+{"timestamp":1694048135.2862284,"name":"online","context":{"idset":"40"}}
+{"timestamp":1694048135.5937591,"name":"online","context":{"idset":"59"}}
+{"timestamp":1694048214.6161592,"name":"undrain","context":{"idset":"1-16,18-28,30-34,37-60"}}
+{"timestamp":1694048224.2075765,"name":"undrain","context":{"idset":"17"}}
+{"timestamp":1694117325.2842343,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2843432,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2843821,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2844198,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2844524,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2844846,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2845161,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2845454,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.28459,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.284632,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2846723,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2847083,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2847383,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2847686,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2848098,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2848406,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.2848721,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1694117325.3842671,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.285378,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.2854533,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.2854917,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.285527,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.2855604,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.285593,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.2856348,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.2856658,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.2856965,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.2857263,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.2857563,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.2857921,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1694117327.385433,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2887416,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2888792,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2889516,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2890208,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2890999,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2891672,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2892327,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2892988,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2893651,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2894316,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2894971,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2895663,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2896383,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2897048,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2897711,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2898397,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2899063,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2899733,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2900405,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2901092,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2901862,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2902517,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2903171,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2903783,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.2904413,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1694117329.3907495,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1694117391.2863674,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1694117391.2864997,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1694117391.2866008,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1694117391.2867041,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1694117391.286787,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1694117391.2868664,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1694117391.2869446,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1694117391.2870312,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1694117391.2871346,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1694117391.2872682,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1694117391.2873702,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1694117391.2874682,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1694117391.2875581,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1694117391.2876661,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1694117391.2877493,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1694117391.287838,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1694117391.2879348,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1694117391.2880127,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1694117391.2880909,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1694117391.2881753,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1694117391.2882595,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1694117391.288347,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1694117391.2884207,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1694117391.2885044,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1694117391.2885861,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1694117391.288667,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1694117391.2887547,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1694117391.2888281,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1694117391.2889059,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1694117391.2889743,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1694117391.2890522,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1694117391.2891166,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1694117391.2891924,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1694117391.2892542,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1694117391.2893202,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1694117391.289382,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1694117391.2894437,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1694117391.2895038,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1694117391.2895632,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1694117391.2896309,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1694117391.2896888,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1694117391.2897437,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1694117391.2897992,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1694117391.2898536,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1694117391.2899072,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1694117391.2899592,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1694117391.2900312,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1694117391.2900815,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1694117391.2901316,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1694117391.2901831,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1694117391.2902315,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1694117391.2902784,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1694117391.2903244,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1694117391.2903647,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1694117391.2904131,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1694117391.2904484,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1694117391.2904925,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1694117391.3868921,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1694118117.3570485,"name":"online","context":{"idset":"55"}}
+{"timestamp":1694118118.0715115,"name":"online","context":{"idset":"7-8,11,13,15,17-18,21-22,33,40,44,49"}}
+{"timestamp":1694118118.189326,"name":"online","context":{"idset":"1-4,6,10,12,16,19-20,25-26,31-32,41,46-48,51"}}
+{"timestamp":1694118118.2968609,"name":"online","context":{"idset":"5,14,23-24,27,29-30,34,36-39,43,50,52-53,56,58"}}
+{"timestamp":1694118118.4074306,"name":"online","context":{"idset":"9,28,45,54,59-60"}}
+{"timestamp":1694118191.2673721,"name":"undrain","context":{"idset":"1-28,30-34,37-41,43-56,58-60"}}
+{"timestamp":1694118197.6390676,"name":"undrain","context":{"idset":"42,57"}}
+{"timestamp":1694120616.3183222,"name":"online","context":{"idset":"57"}}
+{"timestamp":1694120616.4809484,"name":"online","context":{"idset":"42"}}
+{"timestamp":1694120666.7382557,"name":"undrain","context":{"idset":"29"}}
+{"timestamp":1694556436.5394912,"name":"undrain","context":{"idset":"35-36"}}
+{"timestamp":1694556477.6744776,"name":"online","context":{"idset":"35"}}
+{"timestamp":1694718541.2663739,"name":"drain","context":{"idset":"1","overwrite":0}}
+{"timestamp":1694721526.4193428,"name":"undrain","context":{"idset":"1"}}
+{"timestamp":1694726131.3866611,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1694726199.3849168,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1694916209.5536907,"name":"online","context":{"idset":"16"}}
+{"timestamp":1694916261.1391032,"name":"undrain","context":{"idset":"16"}}
+{"timestamp":1694917411.3871126,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1694917525.3854392,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1694918213.0599301,"name":"online","context":{"idset":"7"}}
+{"timestamp":1694919167.3859253,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1694922999.3870785,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1694923059.3856814,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1694923063.3854282,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1694923121.3860753,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1694936625.3874714,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1695143480.5960422,"name":"drain","context":{"idset":"36","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695143483.9077022,"name":"drain","context":{"idset":"48","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695143488.0956278,"name":"drain","context":{"idset":"57","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695143494.4467995,"name":"drain","context":{"idset":"39","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695143524.0210464,"name":"drain","context":{"idset":"55","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695143603.2890434,"name":"drain","context":{"idset":"36","reason":"epilog failed for jobid f8gm4AkxYcF","overwrite":0}}
+{"timestamp":1695143603.3867235,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1695143607.2884161,"name":"drain","context":{"idset":"48","reason":"epilog failed for jobid f8gm4Jackqm","overwrite":0}}
+{"timestamp":1695143607.3854547,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1695143611.2895341,"name":"drain","context":{"idset":"57","reason":"epilog failed for jobid f8gm4R6fbhh","overwrite":0}}
+{"timestamp":1695143611.3857496,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1695143617.2891502,"name":"drain","context":{"idset":"39","reason":"epilog failed for jobid f8gm4Bhp631","overwrite":0}}
+{"timestamp":1695143617.3855488,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1695143647.2892299,"name":"drain","context":{"idset":"55","reason":"epilog failed for jobid f8gm4PPLRpb","overwrite":0}}
+{"timestamp":1695143647.3861513,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1695144827.7909341,"name":"online","context":{"idset":"22,36"}}
+{"timestamp":1695144827.918772,"name":"online","context":{"idset":"11,15,39,48,55,57"}}
+{"timestamp":1695146427.3350294,"name":"undrain","context":{"idset":"11,15,22,36,39,48,54-55,57"}}
+{"timestamp":1695167973.382906,"name":"drain","context":{"idset":"7","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4008362,"name":"drain","context":{"idset":"3","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4068353,"name":"drain","context":{"idset":"1","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4127369,"name":"drain","context":{"idset":"19","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.428154,"name":"drain","context":{"idset":"30","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4400141,"name":"drain","context":{"idset":"16","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4407525,"name":"drain","context":{"idset":"38","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4444413,"name":"drain","context":{"idset":"2","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4505448,"name":"drain","context":{"idset":"12","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4553015,"name":"drain","context":{"idset":"8","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4562488,"name":"drain","context":{"idset":"24","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.463201,"name":"drain","context":{"idset":"40","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4644499,"name":"drain","context":{"idset":"20","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4651349,"name":"drain","context":{"idset":"17","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.465821,"name":"drain","context":{"idset":"14","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4664979,"name":"drain","context":{"idset":"31","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4673097,"name":"drain","context":{"idset":"50","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4682784,"name":"drain","context":{"idset":"10","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4691496,"name":"drain","context":{"idset":"18","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4698496,"name":"drain","context":{"idset":"34","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4705482,"name":"drain","context":{"idset":"23","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4773858,"name":"drain","context":{"idset":"37","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4806194,"name":"drain","context":{"idset":"29","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4813592,"name":"drain","context":{"idset":"56","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4820962,"name":"drain","context":{"idset":"53","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4839141,"name":"drain","context":{"idset":"52","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4848716,"name":"drain","context":{"idset":"49","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4864545,"name":"drain","context":{"idset":"4","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4921458,"name":"drain","context":{"idset":"41","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.493541,"name":"drain","context":{"idset":"59","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4942694,"name":"drain","context":{"idset":"25","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.4976008,"name":"drain","context":{"idset":"46","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5004926,"name":"drain","context":{"idset":"43","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5021207,"name":"drain","context":{"idset":"21","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5043561,"name":"drain","context":{"idset":"47","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5132973,"name":"drain","context":{"idset":"27","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5197859,"name":"drain","context":{"idset":"42","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5290151,"name":"drain","context":{"idset":"51","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.533005,"name":"drain","context":{"idset":"33","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5346301,"name":"drain","context":{"idset":"26","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5353112,"name":"drain","context":{"idset":"45","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5407667,"name":"drain","context":{"idset":"28","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5416007,"name":"drain","context":{"idset":"35","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5467086,"name":"drain","context":{"idset":"6","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5535858,"name":"drain","context":{"idset":"13","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5543361,"name":"drain","context":{"idset":"9","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.555588,"name":"drain","context":{"idset":"32","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5718591,"name":"drain","context":{"idset":"5","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5895262,"name":"drain","context":{"idset":"44","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695167973.5979128,"name":"drain","context":{"idset":"58","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1695168035.2866926,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1695168095.28511,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1695168095.2852685,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1695168095.2853692,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1695168095.2854567,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1695168095.2855418,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1695168095.2856746,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1695168095.2858062,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1695168095.2859364,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1695168095.2860663,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1695168095.2862203,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1695168095.2863612,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1695168095.2864902,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1695168095.2866299,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1695168095.2867625,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1695168095.2869108,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1695168095.2870276,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1695168095.2871389,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1695168095.2872522,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1695168095.2873554,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1695168095.2874551,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1695168095.287555,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1695168095.2876656,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1695168095.2877667,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1695168095.2878673,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1695168095.2879651,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1695168095.2880628,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1695168095.2881598,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1695168095.2882576,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1695168095.2883492,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1695168095.2884359,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1695168095.2885258,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1695168095.2886329,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1695168095.2887354,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1695168095.2888038,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1695168095.2889099,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1695168095.2890153,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1695168095.2891283,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1695168095.2892416,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1695168095.2893479,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1695168095.289449,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1695168095.2895546,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1695168095.2897148,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1695168095.3070042,"name":"drain","context":{"idset":"1","reason":"epilog failed for jobid f8sqVSJr8fH","overwrite":0}}
+{"timestamp":1695168095.3071241,"name":"drain","context":{"idset":"3","reason":"epilog failed for jobid f8sqVT6okPy","overwrite":0}}
+{"timestamp":1695168095.3071816,"name":"drain","context":{"idset":"2","reason":"epilog failed for jobid f8sqVSk3vT9","overwrite":0}}
+{"timestamp":1695168095.307291,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid f8sqVTL9dw5","overwrite":0}}
+{"timestamp":1695168095.3082364,"name":"drain","context":{"idset":"9","reason":"epilog failed for jobid f8sqVXHwicX","overwrite":0}}
+{"timestamp":1695168095.308301,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid f8sqVUnevzj","overwrite":0}}
+{"timestamp":1695168095.3097064,"name":"drain","context":{"idset":"7","reason":"epilog failed for jobid f8sqVVf4WaT","overwrite":0}}
+{"timestamp":1695168095.3103783,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid f8sqVb9orAb","overwrite":0}}
+{"timestamp":1695168095.310972,"name":"drain","context":{"idset":"12","reason":"epilog failed for jobid f8sqVYoQzEs","overwrite":0}}
+{"timestamp":1695168095.3110344,"name":"drain","context":{"idset":"10","reason":"epilog failed for jobid f8sqVYVd8rj","overwrite":0}}
+{"timestamp":1695168095.31249,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid f8sqVWwBtfh","overwrite":0}}
+{"timestamp":1695168095.3130143,"name":"drain","context":{"idset":"13","reason":"epilog failed for jobid f8sqVZSUgHV","overwrite":0}}
+{"timestamp":1695168095.3131895,"name":"drain","context":{"idset":"18","reason":"epilog failed for jobid f8sqVcmD4vK","overwrite":0}}
+{"timestamp":1695168095.3143253,"name":"drain","context":{"idset":"17","reason":"epilog failed for jobid f8sqVbyFTBd","overwrite":0}}
+{"timestamp":1695168095.3143861,"name":"drain","context":{"idset":"16","reason":"epilog failed for jobid f8sqVbLBm91","overwrite":0}}
+{"timestamp":1695168095.3144772,"name":"drain","context":{"idset":"21","reason":"epilog failed for jobid f8sqVfdFgUw","overwrite":0}}
+{"timestamp":1695168095.3145463,"name":"drain","context":{"idset":"23","reason":"epilog failed for jobid f8sqVfw3Xs5","overwrite":0}}
+{"timestamp":1695168095.3146489,"name":"drain","context":{"idset":"20","reason":"epilog failed for jobid f8sqVe1rTjD","overwrite":0}}
+{"timestamp":1695168095.3147514,"name":"drain","context":{"idset":"19","reason":"epilog failed for jobid f8sqVd7xts9","overwrite":0}}
+{"timestamp":1695168095.3152583,"name":"drain","context":{"idset":"27","reason":"epilog failed for jobid f8sqViLQNMV","overwrite":0}}
+{"timestamp":1695168095.3165019,"name":"drain","context":{"idset":"24","reason":"epilog failed for jobid f8sqVhACwPd","overwrite":0}}
+{"timestamp":1695168095.3165629,"name":"drain","context":{"idset":"30","reason":"epilog failed for jobid f8sqVkwd76f","overwrite":0}}
+{"timestamp":1695168095.3172226,"name":"drain","context":{"idset":"25","reason":"epilog failed for jobid f8sqVhTznmm","overwrite":0}}
+{"timestamp":1695168095.3172851,"name":"drain","context":{"idset":"38","reason":"epilog failed for jobid f8sqVrJxWH5","overwrite":0}}
+{"timestamp":1695168095.3173525,"name":"drain","context":{"idset":"31","reason":"epilog failed for jobid f8sqVmCSyv7","overwrite":0}}
+{"timestamp":1695168095.317421,"name":"drain","context":{"idset":"32","reason":"epilog failed for jobid f8sqVoAc2cf","overwrite":0}}
+{"timestamp":1695168095.3174913,"name":"drain","context":{"idset":"37","reason":"epilog failed for jobid f8sqVqcvqfm","overwrite":0}}
+{"timestamp":1695168095.3177488,"name":"drain","context":{"idset":"29","reason":"epilog failed for jobid f8sqVkWRKJo","overwrite":0}}
+{"timestamp":1695168095.3187273,"name":"drain","context":{"idset":"40","reason":"epilog failed for jobid f8sqVsh1qVh","overwrite":0}}
+{"timestamp":1695168095.3187921,"name":"drain","context":{"idset":"43","reason":"epilog failed for jobid f8sqVvT8Vvw","overwrite":0}}
+{"timestamp":1695168095.3190255,"name":"drain","context":{"idset":"41","reason":"epilog failed for jobid f8sqVtauQMm","overwrite":0}}
+{"timestamp":1695168095.3215837,"name":"drain","context":{"idset":"45","reason":"epilog failed for jobid f8sqVyfvwSP","overwrite":0}}
+{"timestamp":1695168095.3218572,"name":"drain","context":{"idset":"42","reason":"epilog failed for jobid f8sqVucCudZ","overwrite":0}}
+{"timestamp":1695168095.3219175,"name":"drain","context":{"idset":"34","reason":"epilog failed for jobid f8sqVpXBNYw","overwrite":0}}
+{"timestamp":1695168095.3219812,"name":"drain","context":{"idset":"47","reason":"epilog failed for jobid f8sqW1PG7KV","overwrite":0}}
+{"timestamp":1695168095.3222225,"name":"drain","context":{"idset":"49","reason":"epilog failed for jobid f8sqW1fZyRH","overwrite":0}}
+{"timestamp":1695168095.3240871,"name":"drain","context":{"idset":"46","reason":"epilog failed for jobid f8sqVzQvacP","overwrite":0}}
+{"timestamp":1695168095.3247609,"name":"drain","context":{"idset":"52","reason":"epilog failed for jobid f8sqW3am3Z9","overwrite":0}}
+{"timestamp":1695168095.3248286,"name":"drain","context":{"idset":"50","reason":"epilog failed for jobid f8sqW22KoN7","overwrite":0}}
+{"timestamp":1695168095.3248825,"name":"drain","context":{"idset":"59","reason":"epilog failed for jobid f8sqW5Q2AZd","overwrite":0}}
+{"timestamp":1695168095.3249328,"name":"drain","context":{"idset":"53","reason":"epilog failed for jobid f8sqW3s4uew","overwrite":0}}
+{"timestamp":1695168095.3249819,"name":"drain","context":{"idset":"56","reason":"epilog failed for jobid f8sqW4QCeaB","overwrite":0}}
+{"timestamp":1695168095.3251998,"name":"drain","context":{"idset":"51","reason":"epilog failed for jobid f8sqW2mKSY7","overwrite":0}}
+{"timestamp":1695168095.3848226,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1695168097.2862015,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1695168097.2863019,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1695168097.2863641,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1695168097.2864287,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1695168097.2865164,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1695168097.286597,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1695168097.2904251,"name":"drain","context":{"idset":"26","reason":"epilog failed for jobid f8sqVhipfbD","overwrite":0}}
+{"timestamp":1695168097.2906294,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid f8sqVTjsSSb","overwrite":0}}
+{"timestamp":1695168097.2906849,"name":"drain","context":{"idset":"28","reason":"epilog failed for jobid f8sqVibEFAw","overwrite":0}}
+{"timestamp":1695168097.2909234,"name":"drain","context":{"idset":"33","reason":"epilog failed for jobid f8sqVokhk6b","overwrite":0}}
+{"timestamp":1695168097.2911417,"name":"drain","context":{"idset":"44","reason":"epilog failed for jobid f8sqVwz5kqd","overwrite":0}}
+{"timestamp":1695168097.2911866,"name":"drain","context":{"idset":"35","reason":"epilog failed for jobid f8sqVq8m5KD","overwrite":0}}
+{"timestamp":1695168097.2916768,"name":"drain","context":{"idset":"58","reason":"epilog failed for jobid f8sqW4jUVEf","overwrite":0}}
+{"timestamp":1695168097.3869884,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1695168241.3861058,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1695169805.5437605,"name":"online","context":{"idset":"1,8,14,17-18,20,29,31,35,43,49"}}
+{"timestamp":1695169805.6524453,"name":"online","context":{"idset":"2,5-7,9-10,12-13,16,19,23-28,34,38,41-42,45-47,51,56,59"}}
+{"timestamp":1695169805.7559636,"name":"online","context":{"idset":"4,21,30,32-33,53-54,58"}}
+{"timestamp":1695169805.8611193,"name":"online","context":{"idset":"3,37,44,50,52"}}
+{"timestamp":1695169818.4425914,"name":"undrain","context":{"idset":"1-10,12-14,16-21,23-35,37-38,40-47,49-54,56,58-59"}}
+{"timestamp":1695223577.2864504,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2866154,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2866683,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2867117,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2867513,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2867863,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.286818,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2868507,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2868814,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2869105,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2869415,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2869699,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.2870047,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1695223577.3873975,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2857616,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2858551,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2858915,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2859249,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2859566,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2859952,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2860255,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2860589,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2860887,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.28613,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2861598,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2861907,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2862213,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2862539,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2862918,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2863278,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2863591,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2863886,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.286418,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2864509,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2864814,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2865188,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2865534,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2865868,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2866294,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2866619,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2866967,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2867296,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.286763,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.286797,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2868299,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2868648,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2868965,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2869315,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.2869647,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1695223579.3859115,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1695223643.2845676,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1695223643.2847083,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1695223643.2847798,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1695223643.2848432,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1695223643.2849388,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1695223643.2850442,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1695223643.285157,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1695223643.2852597,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1695223643.2853792,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1695223643.28549,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1695223643.285593,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1695223643.2857046,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1695223643.2858167,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1695223643.2859156,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1695223643.2860062,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1695223643.2860994,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1695223643.2861881,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1695223643.2862771,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1695223643.286366,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1695223643.2864692,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1695223643.2865453,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1695223643.2866449,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1695223643.2867279,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1695223643.2868173,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1695223643.2868969,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1695223643.2869802,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1695223643.2870791,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1695223643.2871614,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1695223643.2872393,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1695223643.287317,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1695223643.2873921,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1695223643.2874715,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1695223643.2875552,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1695223643.2876422,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1695223643.2877214,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1695223643.2877944,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1695223643.2878897,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1695223643.2880177,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1695223643.2881377,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1695223643.2882552,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1695223643.2883692,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1695223643.2884877,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1695223643.2885792,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1695223643.2886996,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1695223643.288774,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1695223643.2888803,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1695223643.2889502,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1695223643.2917857,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1695223643.2918928,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1695223643.3731954,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid f8wDWyPbVzw","overwrite":0}}
+{"timestamp":1695223643.3732848,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid f8wDWz2fC3Z","overwrite":0}}
+{"timestamp":1695223643.373323,"name":"drain","context":{"idset":"1","reason":"epilog failed for jobid f8wDWxXBvRD","overwrite":0}}
+{"timestamp":1695223643.3735974,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid f8wDX2Fe7ZZ","overwrite":0}}
+{"timestamp":1695223643.3736691,"name":"drain","context":{"idset":"2","reason":"epilog failed for jobid f8wDWxswkN3","overwrite":0}}
+{"timestamp":1695223643.37379,"name":"drain","context":{"idset":"3","reason":"epilog failed for jobid f8wDWy7Hdu9","overwrite":0}}
+{"timestamp":1695223643.3748939,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid f8wDWyfuN6j","overwrite":0}}
+{"timestamp":1695223643.3757875,"name":"drain","context":{"idset":"7","reason":"epilog failed for jobid f8wDX15Sgbh","overwrite":0}}
+{"timestamp":1695223643.3759129,"name":"drain","context":{"idset":"11","reason":"epilog failed for jobid f8wDX5Aehgs","overwrite":0}}
+{"timestamp":1695223643.3759811,"name":"drain","context":{"idset":"9","reason":"epilog failed for jobid f8wDX2XwyfM","overwrite":0}}
+{"timestamp":1695223643.3761165,"name":"drain","context":{"idset":"10","reason":"epilog failed for jobid f8wDX3wVJAK","overwrite":0}}
+{"timestamp":1695223643.3762808,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid f8wDX82hKFV","overwrite":0}}
+{"timestamp":1695223643.3774824,"name":"drain","context":{"idset":"13","reason":"epilog failed for jobid f8wDX662FqH","overwrite":0}}
+{"timestamp":1695223643.3778503,"name":"drain","context":{"idset":"18","reason":"epilog failed for jobid f8wDXABEGvT","overwrite":0}}
+{"timestamp":1695223643.3778861,"name":"drain","context":{"idset":"17","reason":"epilog failed for jobid f8wDX9BQkw1","overwrite":0}}
+{"timestamp":1695223643.3779316,"name":"drain","context":{"idset":"12","reason":"epilog failed for jobid f8wDX5SxZnf","overwrite":0}}
+{"timestamp":1695223643.3780088,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid f8wDX78okPR","overwrite":0}}
+{"timestamp":1695223643.3784547,"name":"drain","context":{"idset":"16","reason":"epilog failed for jobid f8wDX8kCy99","overwrite":0}}
+{"timestamp":1695223643.3785641,"name":"drain","context":{"idset":"19","reason":"epilog failed for jobid f8wDXAnoygj","overwrite":0}}
+{"timestamp":1695223643.3787405,"name":"drain","context":{"idset":"25","reason":"epilog failed for jobid f8wDXGK3JZD","overwrite":0}}
+{"timestamp":1695223643.3788943,"name":"drain","context":{"idset":"21","reason":"epilog failed for jobid f8wDXBwXRNF","overwrite":0}}
+{"timestamp":1695223643.379195,"name":"drain","context":{"idset":"27","reason":"epilog failed for jobid f8wDXJCkPQj","overwrite":0}}
+{"timestamp":1695223643.3793538,"name":"drain","context":{"idset":"22","reason":"epilog failed for jobid f8wDXCf35Fu","overwrite":0}}
+{"timestamp":1695223643.3794384,"name":"drain","context":{"idset":"30","reason":"epilog failed for jobid f8wDXLJKNX1","overwrite":0}}
+{"timestamp":1695223643.3794684,"name":"drain","context":{"idset":"31","reason":"epilog failed for jobid f8wDXMQ4qdq","overwrite":0}}
+{"timestamp":1695223643.379719,"name":"drain","context":{"idset":"34","reason":"epilog failed for jobid f8wDXQvf8XR","overwrite":0}}
+{"timestamp":1695223643.3797808,"name":"drain","context":{"idset":"28","reason":"epilog failed for jobid f8wDXKE3tgX","overwrite":0}}
+{"timestamp":1695223643.379868,"name":"drain","context":{"idset":"20","reason":"epilog failed for jobid f8wDXB6bq4s","overwrite":0}}
+{"timestamp":1695223643.3798964,"name":"drain","context":{"idset":"48","reason":"epilog failed for jobid f8wDXdAoCWb","overwrite":0}}
+{"timestamp":1695223643.3799226,"name":"drain","context":{"idset":"41","reason":"epilog failed for jobid f8wDXVtGjCK","overwrite":0}}
+{"timestamp":1695223643.3799479,"name":"drain","context":{"idset":"44","reason":"epilog failed for jobid f8wDXZHT5gB","overwrite":0}}
+{"timestamp":1695223643.3799763,"name":"drain","context":{"idset":"43","reason":"epilog failed for jobid f8wDXXj1qV9","overwrite":0}}
+{"timestamp":1695223643.3800046,"name":"drain","context":{"idset":"39","reason":"epilog failed for jobid f8wDXVW2uy9","overwrite":0}}
+{"timestamp":1695223643.3800302,"name":"drain","context":{"idset":"45","reason":"epilog failed for jobid f8wDXaCpdpb","overwrite":0}}
+{"timestamp":1695223643.3800547,"name":"drain","context":{"idset":"51","reason":"epilog failed for jobid f8wDXfNJ8kF","overwrite":0}}
+{"timestamp":1695223643.3800995,"name":"drain","context":{"idset":"47","reason":"epilog failed for jobid f8wDXbzbmYj","overwrite":0}}
+{"timestamp":1695223643.3801539,"name":"drain","context":{"idset":"37","reason":"epilog failed for jobid f8wDXTUuthu","overwrite":0}}
+{"timestamp":1695223643.3802114,"name":"drain","context":{"idset":"50","reason":"epilog failed for jobid f8wDXf4WHN7","overwrite":0}}
+{"timestamp":1695223643.3802392,"name":"drain","context":{"idset":"26","reason":"epilog failed for jobid f8wDXHvSXJw","overwrite":0}}
+{"timestamp":1695223643.3802645,"name":"drain","context":{"idset":"46","reason":"epilog failed for jobid f8wDXbgovAb","overwrite":0}}
+{"timestamp":1695223643.3802896,"name":"drain","context":{"idset":"49","reason":"epilog failed for jobid f8wDXeKWeC7","overwrite":0}}
+{"timestamp":1695223643.3803165,"name":"drain","context":{"idset":"35","reason":"epilog failed for jobid f8wDXSJiTk3","overwrite":0}}
+{"timestamp":1695223643.3803432,"name":"drain","context":{"idset":"33","reason":"epilog failed for jobid f8wDXQHbSUo","overwrite":0}}
+{"timestamp":1695223643.3803701,"name":"drain","context":{"idset":"42","reason":"epilog failed for jobid f8wDXWhiLDM","overwrite":0}}
+{"timestamp":1695223643.3803995,"name":"drain","context":{"idset":"24","reason":"epilog failed for jobid f8wDXFBorA3","overwrite":0}}
+{"timestamp":1695223643.3804281,"name":"drain","context":{"idset":"38","reason":"epilog failed for jobid f8wDXTpBjNP","overwrite":0}}
+{"timestamp":1695223643.3804543,"name":"drain","context":{"idset":"29","reason":"epilog failed for jobid f8wDXKaoidM","overwrite":0}}
+{"timestamp":1695223643.3804805,"name":"drain","context":{"idset":"23","reason":"epilog failed for jobid f8wDXENNF91","overwrite":0}}
+{"timestamp":1695223643.3805096,"name":"drain","context":{"idset":"32","reason":"epilog failed for jobid f8wDXNkeBa7","overwrite":0}}
+{"timestamp":1695223643.3824813,"name":"drain","context":{"idset":"36","reason":"epilog failed for jobid f8wDXSjvFXu","overwrite":0}}
+{"timestamp":1695223643.3842368,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1695224941.5912683,"name":"online","context":{"idset":"10,19,27,34,38,41,47,49,51"}}
+{"timestamp":1695224941.6924627,"name":"online","context":{"idset":"1-2,4,7-8,11-13,16,18,24,36,39,46,50"}}
+{"timestamp":1695224941.8006899,"name":"online","context":{"idset":"3,5-6,17,20,22-23,25-26,28-30,32,35,42-45,48"}}
+{"timestamp":1695224941.9123342,"name":"online","context":{"idset":"9,14-15,21,31,33,37"}}
+{"timestamp":1695224982.1399109,"name":"online","context":{"idset":"40"}}
+{"timestamp":1695225082.7038238,"name":"undrain","context":{"idset":"1-39"}}
+{"timestamp":1695225098.368444,"name":"undrain","context":{"idset":"41-51"}}
+{"timestamp":1695227770.7225554,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1695228533.7310395,"name":"online","context":{"idset":"40"}}
+{"timestamp":1695656772.8165178,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1695656772.8169332,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1695656772.8186934,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1695656772.826237,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1695656772.8276165,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1695656772.8342037,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1695656772.8372331,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1695656772.8394434,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1695656772.8399973,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1695656772.8400784,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1695656772.8409586,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1695656772.8425314,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1695656772.8499539,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1695656772.8500535,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1695656772.8503528,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1695656772.8537571,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1695656772.8538651,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1695656772.8567312,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1695656772.8570037,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1695656772.8591042,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1695656772.8645165,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1695656772.8656485,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1695656772.8659217,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1695656772.866231,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1695656772.8665521,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1695656772.8671441,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1695656772.8682442,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1695656772.8684373,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1695656772.8685255,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1695656772.8688445,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1695656772.8709288,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1695656772.8713844,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1695656772.8729103,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1695656772.8738894,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1695656772.873986,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1695656772.8743279,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1695656772.8768802,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1695656772.8774662,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1695656772.8813365,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1695656772.8840847,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1695656772.8852506,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1695656772.8864632,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1695656772.8865521,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1695656772.8873127,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1695656772.8897936,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1695656772.8902912,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1695656772.8907666,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1695656772.8911638,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1695656772.8919857,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1695656772.8923352,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1695656772.8938899,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1695656772.8949118,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1695656772.8953269,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1695656772.8955164,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1695656772.8959322,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1695656772.8964393,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1695656772.8971488,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1695656772.9070091,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1695656772.9085174,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1695656773.0086229,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1695656798.0004523,"name":"resource-init","context":{"restart":true,"drain":{},"online":"","exclude":"0"}}
+{"timestamp":1695656798.0084624,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1695656879.915585,"name":"online","context":{"idset":"0"}}
+{"timestamp":1695656880.9509895,"name":"online","context":{"idset":"40,51-60"}}
+{"timestamp":1695656881.0802124,"name":"online","context":{"idset":"1-39,41-50"}}
+{"timestamp":1695656930.9993465,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1695656931.0076478,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1695656931.0210612,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1695656931.0281007,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1695656931.0375421,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1695656931.0398431,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1695656931.0403204,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1695656931.0467262,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1695656931.048188,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1695656931.0487998,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1695656931.050926,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1695656931.051491,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1695656931.0548012,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1695656931.0551913,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1695656931.0587554,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1695656931.0634758,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1695656931.0650804,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1695656931.0668287,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1695656931.071275,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1695656931.0723448,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1695656931.0794835,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1695656931.0825779,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1695656931.0915656,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1695656931.0925305,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1695656931.0932896,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1695656931.0966184,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1695656931.0967996,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1695656931.0995731,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1695656931.1010933,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1695656931.1021535,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1695656931.103961,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1695656931.1068249,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1695656931.10905,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1695656931.1157625,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1695656931.1189666,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1695656931.1205924,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1695656931.1243496,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1695656931.1260927,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1695656931.1269522,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1695656931.1272786,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1695656931.1278691,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1695656931.130419,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1695656931.1335349,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1695656931.1357989,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1695656931.1374812,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1695656931.1376688,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1695656931.1503587,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1695656931.1505947,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1695656931.1531377,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1695656931.1533124,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1695656931.1611955,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1695656931.1650465,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1695656931.1665981,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1695656931.1688321,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1695656931.1817045,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1695656931.1871524,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1695656931.195219,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1695656931.2024848,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1695656931.2090087,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1695656931.2097886,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1695656931.9313586,"name":"online","context":{"idset":"1-2,5-7,9-11,14,17,20,22,25,38,41,45-46,49-50,58,60"}}
+{"timestamp":1695656932.0345283,"name":"online","context":{"idset":"3-4,8,12,15-16,19,21,23-24,26-35,39-40,42-44,47,51,53-55,57,59"}}
+{"timestamp":1695656932.1365454,"name":"online","context":{"idset":"13,18,36-37,48,52,56"}}
+{"timestamp":1695931499.7372658,"name":"drain","context":{"idset":"60","overwrite":0}}
+{"timestamp":1695932695.5268135,"name":"undrain","context":{"idset":"60"}}
+{"timestamp":1695952001.0116124,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.0117424,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.0117869,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.011833,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.0118756,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.01192,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.0119548,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.0119891,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.0120237,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.012059,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.012094,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.0121312,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1695952001.1121068,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0117874,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0118608,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0118973,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0119309,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0119636,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0119946,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0120251,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0120547,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0120931,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0121243,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0121543,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0121868,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0122194,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0122523,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0122881,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0123284,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0123618,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0123925,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0124233,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0124545,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0124867,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0125172,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0125489,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0125799,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0126243,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0126593,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0126929,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0127385,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0127773,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0128105,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0128422,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0128746,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.0129085,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1695952003.1123564,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0121439,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0122259,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0122714,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0123239,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0123641,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0124035,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0124447,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.012486,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0125263,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0125651,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0126011,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.0126464,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1695952005.1126869,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1695952067.0120559,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1695952067.012249,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1695952067.0123639,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1695952067.0124533,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1695952067.0125382,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1695952067.0126386,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1695952067.0127275,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1695952067.0128095,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1695952067.0128887,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1695952067.0129669,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1695952067.0130529,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1695952067.0131578,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1695952067.0132599,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1695952067.0133452,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1695952067.0134494,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1695952067.0135512,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1695952067.0136571,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1695952067.0137534,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1695952067.0138474,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1695952067.0139477,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1695952067.0140402,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1695952067.0141392,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1695952067.0142174,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1695952067.0143127,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1695952067.01441,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1695952067.0145118,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1695952067.0145872,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1695952067.0146952,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1695952067.0147877,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1695952067.0148759,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1695952067.0149465,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1695952067.015029,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1695952067.015115,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1695952067.0151949,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1695952067.0152595,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1695952067.0153389,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1695952067.0154047,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1695952067.0154843,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1695952067.0155647,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1695952067.0156512,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1695952067.0157125,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1695952067.0157895,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1695952067.0158505,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1695952067.0159206,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1695952067.0159771,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1695952067.0160549,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1695952067.0161088,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1695952067.0161791,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1695952067.0162354,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1695952067.0162864,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1695952067.0163558,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1695952067.0164235,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1695952067.0164917,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1695952067.016541,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1695952067.0166111,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1695952067.0166602,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1695952067.0167232,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1695952067.0167644,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1695952067.111973,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1695952069.1127021,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1695961531.6037056,"name":"online","context":{"idset":"1"}}
+{"timestamp":1695961608.8549304,"name":"online","context":{"idset":"7,38"}}
+{"timestamp":1695961608.9487877,"name":"online","context":{"idset":"31"}}
+{"timestamp":1695961609.0636213,"name":"online","context":{"idset":"33"}}
+{"timestamp":1695961609.1568029,"name":"online","context":{"idset":"10"}}
+{"timestamp":1695961609.3516896,"name":"online","context":{"idset":"58"}}
+{"timestamp":1695961609.3927119,"name":"online","context":{"idset":"47"}}
+{"timestamp":1695961609.4964292,"name":"online","context":{"idset":"22,29,39,60"}}
+{"timestamp":1695961609.6461415,"name":"online","context":{"idset":"23"}}
+{"timestamp":1695961609.7178314,"name":"online","context":{"idset":"6,9,19"}}
+{"timestamp":1695961609.772171,"name":"online","context":{"idset":"16,24,45"}}
+{"timestamp":1695961609.8037896,"name":"online","context":{"idset":"27"}}
+{"timestamp":1695961609.8889322,"name":"online","context":{"idset":"5,20,28,34"}}
+{"timestamp":1695961609.9913096,"name":"online","context":{"idset":"4,59"}}
+{"timestamp":1695961610.0949554,"name":"online","context":{"idset":"15,21,44,53"}}
+{"timestamp":1695961610.1055682,"name":"online","context":{"idset":"48"}}
+{"timestamp":1695961610.1409249,"name":"online","context":{"idset":"14,41"}}
+{"timestamp":1695961610.1773634,"name":"online","context":{"idset":"46"}}
+{"timestamp":1695961610.2790754,"name":"online","context":{"idset":"11,18,25-26,30,35,37"}}
+{"timestamp":1695961610.4813156,"name":"online","context":{"idset":"43,51-52"}}
+{"timestamp":1695961610.5937214,"name":"online","context":{"idset":"3"}}
+{"timestamp":1695961610.7062001,"name":"online","context":{"idset":"12-13,57"}}
+{"timestamp":1695961610.8297088,"name":"online","context":{"idset":"2,32,42,49,54"}}
+{"timestamp":1695961610.9830742,"name":"online","context":{"idset":"50"}}
+{"timestamp":1695961611.0097864,"name":"online","context":{"idset":"8,17"}}
+{"timestamp":1695961611.1107509,"name":"online","context":{"idset":"55-56"}}
+{"timestamp":1696001711.8926432,"name":"undrain","context":{"idset":"1-35,37-39,41-60"}}
+{"timestamp":1696027553.5676372,"name":"online","context":{"idset":"36"}}
+{"timestamp":1696027627.9737971,"name":"undrain","context":{"idset":"36,40"}}
+{"timestamp":1696027674.9433367,"name":"online","context":{"idset":"40"}}
+{"timestamp":1696372312.9254923,"name":"drain","context":{"idset":"1","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372312.9801416,"name":"drain","context":{"idset":"15","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.089087,"name":"drain","context":{"idset":"11","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.12679,"name":"drain","context":{"idset":"19","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.1283131,"name":"drain","context":{"idset":"7","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.1302197,"name":"drain","context":{"idset":"47","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.1589861,"name":"drain","context":{"idset":"58","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.1722143,"name":"drain","context":{"idset":"54","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.1781251,"name":"drain","context":{"idset":"25","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.1891251,"name":"drain","context":{"idset":"42","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.194597,"name":"drain","context":{"idset":"37","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.1960514,"name":"drain","context":{"idset":"13","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.2102478,"name":"drain","context":{"idset":"59","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.2338655,"name":"drain","context":{"idset":"35","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.246964,"name":"drain","context":{"idset":"49","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.2476947,"name":"drain","context":{"idset":"30","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.2483881,"name":"drain","context":{"idset":"3","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.252265,"name":"drain","context":{"idset":"34","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.2766964,"name":"drain","context":{"idset":"23","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696372313.3770962,"name":"drain","context":{"idset":"46","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1696379231.0113978,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.011497,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0115409,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0115812,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0116253,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0116634,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0117016,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0117385,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0117753,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0118101,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0118446,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0118787,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0119143,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0119498,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0119848,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0120168,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0120513,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0120828,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0121148,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0121477,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0121806,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0122132,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0122471,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0122817,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0123143,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0123477,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0123818,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0124245,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0124619,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.012495,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0125284,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.012563,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0125959,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0126395,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0126753,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0127127,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.012748,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0127833,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.0128179,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1696379231.1113248,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1696379293.0124581,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1696379293.0125864,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1696379293.0126829,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1696379293.0127506,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1696379293.0128248,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1696379293.0128946,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1696379293.0129871,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1696379293.0131032,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1696379293.0132043,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1696379293.0133004,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1696379293.0133963,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1696379293.0134869,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1696379293.0135779,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1696379293.0136759,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1696379293.0137641,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1696379293.0138688,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1696379293.013958,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1696379293.014044,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1696379293.0141354,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1696379293.01422,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1696379293.0143034,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1696379293.0143859,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1696379293.0144715,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1696379293.0145507,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1696379293.0146394,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1696379293.0147197,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1696379293.0147989,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1696379293.0148766,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1696379293.014951,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1696379293.0150268,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1696379293.0151029,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1696379293.0151751,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1696379293.0152471,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1696379293.0153186,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1696379293.0153909,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1696379293.0154605,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1696379293.0155315,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1696379293.0155985,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1696379293.0156755,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1696379293.0157497,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1696379293.0158162,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1696379293.0158803,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1696379293.0159435,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1696379293.0159998,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1696379293.0160639,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1696379293.016125,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1696379293.0161829,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1696379293.0162389,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1696379293.0163021,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1696379293.0163658,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1696379293.0164201,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1696379293.0164738,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1696379293.0165241,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1696379293.0165772,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1696379293.0166285,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1696379293.0166841,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1696379293.0167227,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1696379293.0167766,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1696379293.0168221,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1696379293.112551,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1696380226.260294,"name":"online","context":{"idset":"4,8,31,37,53"}}
+{"timestamp":1696380226.3644047,"name":"online","context":{"idset":"2-3,7,10,14-16,18,25,27,29-30,32,42,46,48,56-57,59-60"}}
+{"timestamp":1696380226.4655426,"name":"online","context":{"idset":"6,9,11-13,17,20-24,26,34-36,38-41,43-45,49-51,55,58"}}
+{"timestamp":1696380226.5683141,"name":"online","context":{"idset":"5,19,28,33,47,52,54"}}
+{"timestamp":1696380265.836725,"name":"undrain","context":{"idset":"3,7,11,13,15,19,23,25,30,34-35,37,42,46-47,49,54,58-59"}}
+{"timestamp":1696380273.0837543,"name":"undrain","context":{"idset":"2,4-6,8-10,12,14,16-18,20-22,24,26-29,31-33,36,38-41,43-45,48,50-53,55-57,60"}}
+{"timestamp":1696380304.8996906,"name":"online","context":{"idset":"1"}}
+{"timestamp":1696380373.9642808,"name":"undrain","context":{"idset":"1"}}
+{"timestamp":1696521257.8886666,"name":"drain","context":{"idset":"9","overwrite":0}}
+{"timestamp":1696545549.5990632,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1696545549.5994678,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1696545549.6011281,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1696545549.6055353,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1696545549.6090155,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1696545549.6145551,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1696545549.6162918,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1696545549.6207671,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1696545549.6327159,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1696545549.6329372,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1696545549.6331675,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1696545549.6334462,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1696545549.635756,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1696545549.6391304,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1696545549.6421783,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1696545549.6425803,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1696545549.651314,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1696545549.6536324,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1696545549.6583622,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1696545549.6585646,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1696545549.6623504,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1696545549.6629064,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1696545549.6651769,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1696545549.670063,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1696545549.6709514,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1696545549.6721833,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1696545549.6730342,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1696545549.6757889,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1696545549.6780331,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1696545549.6819549,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1696545549.6822042,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1696545549.6841478,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1696545549.6844168,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1696545549.6862662,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1696545549.6888103,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1696545549.690233,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1696545549.6915755,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1696545549.6946876,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1696545549.7022772,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1696545549.7072749,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1696545549.712328,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1696545549.7130401,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1696545549.7196383,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1696545549.7232552,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1696545549.725327,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1696545549.72841,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1696545549.733006,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1696545549.734781,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1696545549.7385776,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1696545549.7415564,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1696545549.7427006,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1696545549.7558506,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1696545549.7606301,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1696545549.7607915,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1696545549.7609184,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1696545549.7722673,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1696545549.8113461,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1696545549.8189847,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1696545549.916496,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1696545550.01758,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1696549531.8720355,"name":"online","context":{"idset":"2-4,6,9,15,17-18,22-23,26,36,42,47,53,56"}}
+{"timestamp":1696549531.9858787,"name":"online","context":{"idset":"1,5,7-8,10,12,14,16,19-20,24-25,27-28,30-33,35,38-40,43,45-46,48-49,51-52,54,57-58"}}
+{"timestamp":1696549532.0938511,"name":"online","context":{"idset":"11,21,34,41,44"}}
+{"timestamp":1696549532.2010622,"name":"online","context":{"idset":"13,29,37,50,55,60"}}
+{"timestamp":1696549532.3337371,"name":"online","context":{"idset":"59"}}
+{"timestamp":1697045693.0115476,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.011668,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.0117099,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.0117409,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.0117702,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.0118012,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.0118439,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.0118897,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.0119274,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.0119586,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.0119927,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.0120263,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1697045693.1119843,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1697045755.0112529,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1697045755.0113788,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1697045755.0115719,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1697045755.011806,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1697045755.0119152,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1697045755.0120492,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1697045755.0121827,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1697045755.0123954,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1697045755.0125427,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1697045755.0126703,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1697045755.0134766,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1697045755.0135956,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1697045755.1114225,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1697046119.0121522,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0122485,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.012291,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0123291,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0123644,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0124018,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0124362,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0124722,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0125074,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0125418,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0125773,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0126443,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0126872,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0127306,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0127797,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0128191,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0128543,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0128887,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0129223,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0129583,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0129933,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0130284,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0130658,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0131044,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0131412,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0131781,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0132282,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0132685,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0133064,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0133438,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0133886,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.0134342,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1697046119.1124752,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1697046121.0122468,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1697046121.012331,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1697046121.0123737,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1697046121.0124133,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1697046121.0124533,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1697046121.1128528,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1697046123.0112259,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1697046123.0113125,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1697046123.0113645,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1697046123.0114112,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1697046123.0114563,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1697046123.0115016,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1697046123.1116154,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1697046185.0098922,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1697046185.0102527,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1697046185.0103564,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1697046185.0104353,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1697046185.0105107,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1697046185.0105822,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1697046185.010669,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1697046185.0107424,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1697046185.0108113,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1697046185.0108809,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1697046185.0109482,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1697046185.0110145,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1697046185.0110879,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1697046185.0111613,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1697046185.0112495,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1697046185.0113449,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1697046185.0114112,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1697046185.0114958,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1697046185.0115995,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1697046185.0117161,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1697046185.0118172,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1697046185.0119221,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1697046185.0120227,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1697046185.0121264,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1697046185.0122356,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1697046185.0123394,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1697046185.0124371,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1697046185.0125313,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1697046185.0126338,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1697046185.0127382,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1697046185.0128379,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1697046185.0129471,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1697046185.0130517,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1697046185.013155,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1697046185.0132539,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1697046185.0133522,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1697046185.0134444,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1697046185.0135374,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1697046185.0136342,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1697046185.0137255,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1697046185.0138075,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1697046185.0138891,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1697046185.0139685,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1697046185.0140486,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1697046185.0141225,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1697046185.0260789,"name":"drain","context":{"idset":"4-5,10,13,17,22-23,27,31,34-35,41,44,46,49,54,60","reason":"epilog failed for jobid fDAWwBpQyQb","overwrite":0}}
+{"timestamp":1697046185.1099424,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1697047147.6190727,"name":"online","context":{"idset":"51"}}
+{"timestamp":1697047147.7043083,"name":"online","context":{"idset":"26"}}
+{"timestamp":1697047147.7644672,"name":"online","context":{"idset":"29"}}
+{"timestamp":1697047147.8753908,"name":"online","context":{"idset":"35,37"}}
+{"timestamp":1697047148.022316,"name":"online","context":{"idset":"7,34"}}
+{"timestamp":1697047148.1850209,"name":"online","context":{"idset":"18-19,22,43"}}
+{"timestamp":1697047148.3052526,"name":"online","context":{"idset":"4,16,54"}}
+{"timestamp":1697047148.4341588,"name":"online","context":{"idset":"1-3,11,21,56,60"}}
+{"timestamp":1697047148.5396655,"name":"online","context":{"idset":"13,23-24,31,33,36,59"}}
+{"timestamp":1697047148.642128,"name":"online","context":{"idset":"20,49,58"}}
+{"timestamp":1697047148.894702,"name":"online","context":{"idset":"14,48"}}
+{"timestamp":1697047149.275995,"name":"online","context":{"idset":"17"}}
+{"timestamp":1697047181.4018629,"name":"undrain","context":{"idset":"1-8,10-60"}}
+{"timestamp":1697047210.8872375,"name":"online","context":{"idset":"10,44"}}
+{"timestamp":1697047211.1125824,"name":"online","context":{"idset":"32"}}
+{"timestamp":1697047211.217757,"name":"online","context":{"idset":"5"}}
+{"timestamp":1697047211.3763325,"name":"online","context":{"idset":"8,12,27-28,41"}}
+{"timestamp":1697047211.5264852,"name":"online","context":{"idset":"38-39,50,53"}}
+{"timestamp":1697047211.7039084,"name":"online","context":{"idset":"15,25,30,47"}}
+{"timestamp":1697047211.8138885,"name":"online","context":{"idset":"6,40,45"}}
+{"timestamp":1697047212.8102994,"name":"online","context":{"idset":"46"}}
+{"timestamp":1697047391.1105404,"name":"online","context":{"idset":"42,52,57"}}
+{"timestamp":1697047391.247041,"name":"online","context":{"idset":"55"}}
+{"timestamp":1697049123.0113137,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1697049123.1121414,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1697049125.1115921,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1697049185.011795,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1697049185.1125205,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1697049187.1124723,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1697050270.7211137,"name":"online","context":{"idset":"17"}}
+{"timestamp":1697050271.1123269,"name":"online","context":{"idset":"7"}}
+{"timestamp":1697050271.4903154,"name":"online","context":{"idset":"55"}}
+{"timestamp":1697053836.4186749,"name":"undrain","context":{"idset":"9,17,55"}}
+{"timestamp":1697054273.1611769,"name":"drain","context":{"idset":"41","overwrite":0}}
+{"timestamp":1697054587.0168276,"name":"drain","context":{"idset":"7,17,41,55","reason":"epilog failed for jobid fDAsT469Csq","overwrite":0}}
+{"timestamp":1697054587.1120777,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1697054771.1885264,"name":"drain","context":{"idset":"9","overwrite":0}}
+{"timestamp":1697054792.5733986,"name":"undrain","context":{"idset":"7,17,55"}}
+{"timestamp":1697132561.9032125,"name":"undrain","context":{"idset":"9"}}
+{"timestamp":1697232201.7334201,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1697232201.7415504,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1697232201.7459903,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1697232201.7610633,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1697232201.7945688,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1697232201.808619,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1697232201.8153605,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1697232201.8246472,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1697232201.8300531,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1697232201.8306489,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1697232201.8329518,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1697232201.8404884,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1697232201.8411043,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1697232201.8436701,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1697232201.8446238,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1697232201.8517499,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1697232201.8526127,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1697232201.857187,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1697232201.8574317,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1697232201.8748686,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1697232201.8753152,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1697232201.877125,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1697232201.8783369,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1697232201.8823695,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1697232201.889137,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1697232201.8896925,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1697232201.8908033,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1697232201.8934221,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1697232201.8980429,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1697232201.8995216,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1697232201.9001408,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1697232201.9151835,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1697232201.9163301,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1697232201.9176526,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1697232201.9197743,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1697232201.9211931,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1697232201.9277697,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1697232201.9363523,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1697232201.9377992,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1697232201.9436998,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1697232201.9457955,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1697232201.9463329,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1697232201.9601231,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1697232201.9621854,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1697232201.9736443,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1697232201.9787166,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1697232201.9800193,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1697232201.98511,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1697232201.9860957,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1697232201.9923685,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1697232201.9946225,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1697232202.0088024,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1697232202.0229461,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1697232202.0278494,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1697232202.0333734,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1697232202.0481615,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1697232202.0548396,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1697232202.0609918,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1697232202.1612909,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1697235085.0111401,"name":"online","context":{"idset":"2-3"}}
+{"timestamp":1697235085.1111324,"name":"online","context":{"idset":"21,43,52,54"}}
+{"timestamp":1697235085.2164421,"name":"online","context":{"idset":"4,7,24,36,38,58"}}
+{"timestamp":1697235085.3213806,"name":"online","context":{"idset":"6,9-10,17-19,22,29,46,49-50,60"}}
+{"timestamp":1697235085.4229219,"name":"online","context":{"idset":"1,8,11-12,16,20,30-33,40,42,48,53,56"}}
+{"timestamp":1697235085.5361474,"name":"online","context":{"idset":"5,13,15,23,25,35,37,41,44-45,47"}}
+{"timestamp":1697235085.6573513,"name":"online","context":{"idset":"26,39,55,57"}}
+{"timestamp":1697235085.7649593,"name":"online","context":{"idset":"14,27-28,34,59"}}
+{"timestamp":1697235085.893712,"name":"online","context":{"idset":"51"}}
+{"timestamp":1697235211.0471578,"name":"undrain","context":{"idset":"41"}}
+{"timestamp":1697235845.2757034,"name":"drain","context":{"idset":"9","reason":"reason=Attached to switch being failed","overwrite":0}}
+{"timestamp":1697733219.0144615,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1697733219.0305984,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1697733219.0403991,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1697733219.0562606,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1697733219.066431,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1697733219.0678799,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1697733219.0727823,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1697733219.0940003,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1697733219.1062758,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1697733219.1088376,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1697733219.1108859,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1697733219.1177709,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1697733219.1198723,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1697733219.1211174,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1697733219.127646,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1697733219.1288815,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1697733219.1381657,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1697733219.1385064,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1697733219.1428616,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1697733219.1431484,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1697733219.1469352,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1697733219.1478782,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1697733219.1507752,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1697733219.1511998,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1697733219.1534295,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1697733219.1700287,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1697733219.170486,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1697733219.1734526,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1697733219.1740944,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1697733219.1822338,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1697733219.1840336,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1697733219.1905088,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1697733219.1986253,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1697733219.2005017,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1697733219.2034771,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1697733219.2038803,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1697733219.2103269,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1697733219.2111533,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1697733219.212852,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1697733219.2195799,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1697733219.2275317,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1697733219.2278161,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1697733219.2345047,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1697733219.2368746,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1697733219.2386391,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1697733219.2424006,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1697733219.2512522,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1697733219.2542052,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1697733219.2567716,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1697733219.267942,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1697733219.272295,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1697733219.2741106,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1697733219.2826486,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1697733219.2830555,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1697733219.2887042,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1697733219.2890916,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1697733219.3020806,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1697733219.3030233,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1697733219.3240559,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1697733219.4250817,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1697734962.6037159,"name":"online","context":{"idset":"1"}}
+{"timestamp":1697734963.9721732,"name":"online","context":{"idset":"22"}}
+{"timestamp":1697734964.1266093,"name":"online","context":{"idset":"9"}}
+{"timestamp":1697734964.8313565,"name":"online","context":{"idset":"4"}}
+{"timestamp":1697734965.3045256,"name":"online","context":{"idset":"49"}}
+{"timestamp":1697734965.7322166,"name":"online","context":{"idset":"23"}}
+{"timestamp":1697734965.7516193,"name":"online","context":{"idset":"11"}}
+{"timestamp":1697734965.9582479,"name":"online","context":{"idset":"42"}}
+{"timestamp":1697734966.0881875,"name":"online","context":{"idset":"6"}}
+{"timestamp":1697734966.1407361,"name":"online","context":{"idset":"24"}}
+{"timestamp":1697734966.1585753,"name":"online","context":{"idset":"3"}}
+{"timestamp":1697734966.1882658,"name":"online","context":{"idset":"2"}}
+{"timestamp":1697734966.2647355,"name":"online","context":{"idset":"37,58"}}
+{"timestamp":1697734966.3978686,"name":"online","context":{"idset":"10,12-13"}}
+{"timestamp":1697734966.4832113,"name":"online","context":{"idset":"20"}}
+{"timestamp":1697734966.6476772,"name":"online","context":{"idset":"8,35,51,54"}}
+{"timestamp":1697734966.6691966,"name":"online","context":{"idset":"50"}}
+{"timestamp":1697734966.7387555,"name":"online","context":{"idset":"14,26,34,39,41,47"}}
+{"timestamp":1697734966.8853898,"name":"online","context":{"idset":"19,38,46"}}
+{"timestamp":1697734966.8994887,"name":"online","context":{"idset":"16"}}
+{"timestamp":1697734967.0313365,"name":"online","context":{"idset":"29,44,55,57"}}
+{"timestamp":1697734967.0790515,"name":"online","context":{"idset":"28,48"}}
+{"timestamp":1697734967.1347225,"name":"online","context":{"idset":"31,45"}}
+{"timestamp":1697734967.2446971,"name":"online","context":{"idset":"18,30,32,40,53"}}
+{"timestamp":1697734967.3531458,"name":"online","context":{"idset":"36,59"}}
+{"timestamp":1697734967.4670494,"name":"online","context":{"idset":"21,27,52"}}
+{"timestamp":1697734967.5981765,"name":"online","context":{"idset":"7,15,56"}}
+{"timestamp":1697734967.746985,"name":"online","context":{"idset":"5,17,33"}}
+{"timestamp":1697734967.977391,"name":"online","context":{"idset":"25,60"}}
+{"timestamp":1697734968.098459,"name":"online","context":{"idset":"43"}}
+{"timestamp":1697763059.2546873,"name":"drain","context":{"idset":"36","reason":"olaf-testing-cxi_rh.target","overwrite":0}}
+{"timestamp":1698080843.707685,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1698080843.8068209,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1698080843.8083918,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1698080843.8092387,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1698080843.8155782,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1698080843.8196461,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1698080843.821523,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1698080843.8313639,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1698080843.8368576,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1698080843.8434346,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1698080843.8491781,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1698080843.8519831,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1698080843.8534985,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1698080843.8572977,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1698080843.8643231,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1698080843.8695962,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1698080843.8700681,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1698080843.88287,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1698080843.8852787,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1698080843.8854942,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1698080843.8862677,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1698080843.8865609,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1698080843.8868978,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1698080843.8907275,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1698080843.8974555,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1698080843.9088147,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1698080843.9195714,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1698080843.921679,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1698080843.9302516,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1698080843.9304955,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1698080843.9442108,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1698080843.9452534,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1698080843.9579785,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1698080843.9591055,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1698080843.9627707,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1698080843.9630334,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1698080843.9698713,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1698080843.9754686,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1698080843.9825423,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1698080843.9827564,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1698080843.9846497,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1698080843.9854422,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1698080843.9880855,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1698080843.9892538,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1698080843.9913852,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1698080843.9956493,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1698080843.9982414,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1698080844.0025606,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1698080844.0109627,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1698080844.0133796,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1698080844.0194969,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1698080844.0206797,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1698080844.0258625,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1698080844.0330863,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1698080844.0416489,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1698080844.0451407,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1698080844.0505295,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1698080844.091018,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1698080844.0988531,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1698080844.1996942,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1698085031.2457416,"name":"online","context":{"idset":"14"}}
+{"timestamp":1698085032.5097294,"name":"online","context":{"idset":"7,38"}}
+{"timestamp":1698085032.6193373,"name":"online","context":{"idset":"18"}}
+{"timestamp":1698085032.9374077,"name":"online","context":{"idset":"52"}}
+{"timestamp":1698085033.636941,"name":"online","context":{"idset":"37"}}
+{"timestamp":1698085033.8621092,"name":"online","context":{"idset":"51"}}
+{"timestamp":1698085034.2449389,"name":"online","context":{"idset":"39,53-54"}}
+{"timestamp":1698085034.3319004,"name":"online","context":{"idset":"45"}}
+{"timestamp":1698085034.4060969,"name":"online","context":{"idset":"2"}}
+{"timestamp":1698085034.4771969,"name":"online","context":{"idset":"11,15"}}
+{"timestamp":1698085034.8992605,"name":"online","context":{"idset":"48"}}
+{"timestamp":1698085035.0557194,"name":"online","context":{"idset":"6,44,55"}}
+{"timestamp":1698085035.1231866,"name":"online","context":{"idset":"34"}}
+{"timestamp":1698085035.2184808,"name":"online","context":{"idset":"5,12"}}
+{"timestamp":1698085035.3341017,"name":"online","context":{"idset":"16,19,23"}}
+{"timestamp":1698085035.3644598,"name":"online","context":{"idset":"49"}}
+{"timestamp":1698085035.3840353,"name":"online","context":{"idset":"57"}}
+{"timestamp":1698085035.4994731,"name":"online","context":{"idset":"4,29-30,33,60"}}
+{"timestamp":1698085035.5776923,"name":"online","context":{"idset":"8,27-28,46"}}
+{"timestamp":1698085035.6793489,"name":"online","context":{"idset":"17,22,26,42,56"}}
+{"timestamp":1698085035.8239617,"name":"online","context":{"idset":"36,41,58-59"}}
+{"timestamp":1698085035.9343662,"name":"online","context":{"idset":"21,35"}}
+{"timestamp":1698085036.0386465,"name":"online","context":{"idset":"1,3,9,47"}}
+{"timestamp":1698085036.1695893,"name":"online","context":{"idset":"13,25"}}
+{"timestamp":1698085036.3225751,"name":"online","context":{"idset":"24,31-32,50"}}
+{"timestamp":1698085036.431916,"name":"online","context":{"idset":"20"}}
+{"timestamp":1698085036.5434268,"name":"online","context":{"idset":"10"}}
+{"timestamp":1698085036.6546524,"name":"online","context":{"idset":"43"}}
+{"timestamp":1698085036.7657881,"name":"online","context":{"idset":"40"}}
+{"timestamp":1698092744.6867001,"name":"drain","context":{"idset":"29","reason":"reason=Attached to switch being tested","overwrite":0}}
+{"timestamp":1698182159.279958,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1698439023.1118371,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1698439085.1127532,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1698682583.0115232,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.0116565,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.0117035,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.0117447,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.0117819,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.011821,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.0118575,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.0118978,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.0119333,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.0119705,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.012008,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.0120506,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.012115,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1698682583.1119626,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0117903,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0118785,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0119274,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0120275,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0120964,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0121667,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0122392,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0123124,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0123858,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0124607,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0125351,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0126202,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0127118,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0127928,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.012876,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.0129576,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1698682585.1123235,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0120988,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0121748,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0122209,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0123093,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.012394,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0124755,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0125604,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0126581,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0127425,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.012826,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0129097,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0130017,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0130847,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0131679,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0133278,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0134072,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0134873,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0135686,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0136595,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0137398,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0138166,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0138958,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.0139744,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1698682587.1123459,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1698682589.112762,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1698682649.0124722,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1698682649.0126357,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1698682649.0128262,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1698682649.0129938,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1698682649.0131531,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1698682649.0133047,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1698682649.0134587,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1698682649.0136166,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1698682649.0137746,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1698682649.0139167,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1698682649.0140612,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1698682649.0142126,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1698682649.0143538,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1698682649.0144963,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1698682649.0146446,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1698682649.0147848,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1698682649.0149262,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1698682649.0150604,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1698682649.0151961,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1698682649.0153379,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1698682649.0154717,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1698682649.0156097,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1698682649.0157363,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1698682649.0158682,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1698682649.0160038,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1698682649.0161273,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1698682649.0162582,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1698682649.0163848,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1698682649.0165217,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1698682649.0166442,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1698682649.0167673,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1698682649.0168893,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1698682649.0170081,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1698682649.0171254,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1698682649.0172467,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1698682649.017359,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1698682649.0174718,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1698682649.0175834,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1698682649.0177147,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1698682649.0178332,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1698682649.017947,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1698682649.0180616,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1698682649.0181696,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1698682649.0182774,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1698682649.0183828,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1698682649.0184865,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1698682649.0185885,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1698682649.0187016,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1698682649.0188029,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1698682649.0189137,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1698682649.0190222,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1698682649.0191207,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1698682649.1124589,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1698682651.0115833,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1698682651.0116847,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1698682651.0117354,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1698682651.0118032,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1698682651.111886,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1698685161.3824401,"name":"online","context":{"idset":"5,11,35,43"}}
+{"timestamp":1698685161.5413246,"name":"online","context":{"idset":"15,17,22-23,36,46,55"}}
+{"timestamp":1698685161.6421847,"name":"online","context":{"idset":"2-3,6,8,12-13,16,19,27,32-33,37,44,54,59"}}
+{"timestamp":1698685161.7441404,"name":"online","context":{"idset":"9-10,14,25-26,29,31,39-40,47-48,57-58"}}
+{"timestamp":1698685161.8511479,"name":"online","context":{"idset":"1,4,7,20-21,28,30,34,38,41,45,50,52-53"}}
+{"timestamp":1698685162.0037234,"name":"online","context":{"idset":"18,24,42,49,56"}}
+{"timestamp":1698685162.1661849,"name":"online","context":{"idset":"51"}}
+{"timestamp":1698685181.3335643,"name":"undrain","context":{"idset":"1-59"}}
+{"timestamp":1698685204.3591313,"name":"online","context":{"idset":"60"}}
+{"timestamp":1698685265.5490093,"name":"undrain","context":{"idset":"60"}}
+{"timestamp":1698685288.6167469,"name":"drain","context":{"idset":"9,29","reason":"reason=Disable for switch pull","overwrite":0}}
+{"timestamp":1699370071.0121398,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1699370071.0122681,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1699370071.0123236,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1699370071.0123665,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1699370071.0124061,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1699370071.0124495,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1699370071.1124201,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1699370073.011651,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1699370073.0117218,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1699370073.0117631,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1699370073.0117993,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1699370073.011847,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1699370073.1118796,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1699370127.1126804,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1699370133.0117378,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1699370133.0118532,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1699370133.1117935,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1699370149.0123622,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1699370149.1126213,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1699370164.421212,"name":"drain","context":{"idset":"13,19,21","reason":"epilog failed for jobid fJQx39eziEK","overwrite":0}}
+{"timestamp":1699370237.0118895,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1699370237.0119958,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1699370237.0120351,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1699370237.0120697,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1699370237.0121131,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1699370237.0121479,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1699370237.0121865,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1699370237.1118197,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1699370241.0123918,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1699370299.0121169,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1699370299.0122228,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1699370299.1119266,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1699370315.0118184,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1699370315.0119135,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1699370315.0119481,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1699370315.0119803,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1699370315.012013,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1699370315.0120454,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1699370315.1121831,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1699370321.011555,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1699370321.0116818,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1699370321.0117788,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1699370321.0118325,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1699370321.0118794,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1699370321.0119205,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1699370327.0393019,"name":"online","context":{"idset":"59"}}
+{"timestamp":1699370340.9126713,"name":"online","context":{"idset":"19"}}
+{"timestamp":1699370341.0934982,"name":"online","context":{"idset":"21"}}
+{"timestamp":1699370381.0108469,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1699370381.0109506,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1699370381.011008,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1699370381.0110643,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1699370381.1107388,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1699370401.0110424,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1699370401.0111308,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1699370401.0111754,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1699370401.1115975,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1699370403.01243,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1699370403.0125179,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1699370403.0125625,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1699370421.0118177,"name":"online","context":{"idset":"13"}}
+{"timestamp":1699370508.2570565,"name":"online","context":{"idset":"3"}}
+{"timestamp":1699370510.3901331,"name":"online","context":{"idset":"49"}}
+{"timestamp":1699370567.112299,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1699370571.0100193,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1699370573.1131005,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1699370584.5067291,"name":"online","context":{"idset":"15,17,23-24"}}
+{"timestamp":1699370584.7511361,"name":"online","context":{"idset":"14"}}
+{"timestamp":1699370651.0110095,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1699370651.0111082,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1699370655.0117974,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1699370713.1117277,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1699370715.1117828,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1699370717.0116942,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1699370717.0118446,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1699370717.1124198,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1699370719.0124898,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1699370719.012629,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1699370719.0127044,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1699370719.0127721,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1699370719.0128324,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1699370719.012892,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1699370719.0129499,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1699370719.0130241,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1699370719.0130951,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1699370719.0131524,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1699370719.0132201,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1699370719.0132759,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1699370719.01333,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1699370719.0133934,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1699370719.0134573,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1699370719.0135093,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1699370719.1122317,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1699370740.2516739,"name":"online","context":{"idset":"9"}}
+{"timestamp":1699370742.0930569,"name":"online","context":{"idset":"60"}}
+{"timestamp":1699370821.3288946,"name":"online","context":{"idset":"59"}}
+{"timestamp":1699370877.012156,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1699370877.1127517,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1699370879.0116277,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1699370879.1116452,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1699370903.0111151,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1699370922.547174,"name":"online","context":{"idset":"58"}}
+{"timestamp":1699370923.0953207,"name":"online","context":{"idset":"6"}}
+{"timestamp":1699370923.2769165,"name":"online","context":{"idset":"12"}}
+{"timestamp":1699370923.6234844,"name":"online","context":{"idset":"52"}}
+{"timestamp":1699370923.9245296,"name":"online","context":{"idset":"54"}}
+{"timestamp":1699370924.0773518,"name":"online","context":{"idset":"8,10"}}
+{"timestamp":1699370924.371233,"name":"online","context":{"idset":"50"}}
+{"timestamp":1699370925.8018639,"name":"online","context":{"idset":"1"}}
+{"timestamp":1699370961.1122715,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1699371000.4475009,"name":"online","context":{"idset":"57"}}
+{"timestamp":1699371002.0799127,"name":"online","context":{"idset":"51"}}
+{"timestamp":1699371002.5538948,"name":"online","context":{"idset":"56"}}
+{"timestamp":1699371043.1101689,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1699371045.0121274,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1699371045.2433906,"name":"online","context":{"idset":"3"}}
+{"timestamp":1699371055.5028501,"name":"online","context":{"idset":"55"}}
+{"timestamp":1699371129.1104898,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1699371143.9068182,"name":"online","context":{"idset":"42"}}
+{"timestamp":1699371147.012465,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1699371160.5817528,"name":"online","context":{"idset":"47"}}
+{"timestamp":1699371244.7426238,"name":"online","context":{"idset":"27"}}
+{"timestamp":1699371251.6211054,"name":"online","context":{"idset":"5"}}
+{"timestamp":1699371302.4420047,"name":"online","context":{"idset":"58"}}
+{"timestamp":1699371310.5137825,"name":"online","context":{"idset":"2"}}
+{"timestamp":1699371321.0373373,"name":"online","context":{"idset":"4"}}
+{"timestamp":1699371373.1128902,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1699371402.3467555,"name":"online","context":{"idset":"59"}}
+{"timestamp":1699371457.1123583,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1699371475.0125661,"name":"online","context":{"idset":"39"}}
+{"timestamp":1699371478.4104965,"name":"online","context":{"idset":"38"}}
+{"timestamp":1699371481.012641,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1699371542.4346228,"name":"online","context":{"idset":"43"}}
+{"timestamp":1699371650.5133421,"name":"online","context":{"idset":"7"}}
+{"timestamp":1699371699.0101418,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1699371699.0102484,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1699371699.1103759,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1699371701.1122992,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1699371703.0119684,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1699371703.1125798,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1699371781.112231,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1699371785.0111854,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1699371785.0112994,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1699371785.0113523,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1699371785.011399,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1699371785.011447,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1699371785.1113737,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1699371787.1103981,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1699371857.0122335,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1699371857.0123589,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1699371857.1121082,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1699371859.0117316,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1699371859.0118315,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1699371859.1116774,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1699371861.0122309,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1699371861.0123284,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1699371861.1129467,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1699371895.5999281,"name":"online","context":{"idset":"29"}}
+{"timestamp":1699371899.0343273,"name":"online","context":{"idset":"43"}}
+{"timestamp":1699371902.6285064,"name":"online","context":{"idset":"47"}}
+{"timestamp":1699371949.1123819,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1699371989.2887063,"name":"online","context":{"idset":"35"}}
+{"timestamp":1699371990.111732,"name":"online","context":{"idset":"34"}}
+{"timestamp":1699372031.0121362,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1699372031.0125768,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1699372035.0118735,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1699372035.112509,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1699372053.9111786,"name":"online","context":{"idset":"48"}}
+{"timestamp":1699372064.5563095,"name":"online","context":{"idset":"46"}}
+{"timestamp":1699372065.1118534,"name":"online","context":{"idset":"59"}}
+{"timestamp":1699372073.1649675,"name":"online","context":{"idset":"44"}}
+{"timestamp":1699372075.3491743,"name":"online","context":{"idset":"38"}}
+{"timestamp":1699372075.7118075,"name":"online","context":{"idset":"40"}}
+{"timestamp":1699372111.0121455,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1699372111.0122516,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1699372111.0124068,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1699372111.1124713,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1699372138.480931,"name":"online","context":{"idset":"29"}}
+{"timestamp":1699372140.5321617,"name":"online","context":{"idset":"9"}}
+{"timestamp":1699372195.0100226,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1699372195.010129,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1699372195.0101914,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1699372195.0102484,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1699372195.0103028,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1699372195.0103595,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1699372195.0104258,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1699372195.0104895,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1699372195.0105438,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1699372195.0106392,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1699372195.0107102,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1699372195.0107875,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1699372195.1101167,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1699372197.0122104,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1699372197.1122189,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1699372199.011714,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1699372199.0118082,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1699372199.112529,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1699372201.0122657,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1699372201.1127851,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1699372214.1523786,"name":"online","context":{"idset":"30"}}
+{"timestamp":1699372223.1126838,"name":"online","context":{"idset":"60"}}
+{"timestamp":1699372233.9454446,"name":"online","context":{"idset":"26"}}
+{"timestamp":1699372283.0114074,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1699372283.0114877,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1699372283.0115292,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1699372283.0115788,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1699372283.0116274,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1699372283.0116818,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1699372283.0117314,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1699372283.1115594,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1699372296.5675128,"name":"online","context":{"idset":"43"}}
+{"timestamp":1699372299.3593905,"name":"online","context":{"idset":"25"}}
+{"timestamp":1699372320.8048754,"name":"online","context":{"idset":"1"}}
+{"timestamp":1699372321.1117013,"name":"online","context":{"idset":"35"}}
+{"timestamp":1699372374.0645735,"name":"online","context":{"idset":"42"}}
+{"timestamp":1699372382.8587453,"name":"online","context":{"idset":"36"}}
+{"timestamp":1699372383.7147424,"name":"online","context":{"idset":"32"}}
+{"timestamp":1699372405.1114738,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1699372447.7371631,"name":"online","context":{"idset":"44,48"}}
+{"timestamp":1699372448.84692,"name":"online","context":{"idset":"55"}}
+{"timestamp":1699372448.9563553,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1699372454.9960721,"name":"online","context":{"idset":"24"}}
+{"timestamp":1699372455.3926511,"name":"online","context":{"idset":"3"}}
+{"timestamp":1699372456.0191586,"name":"online","context":{"idset":"15"}}
+{"timestamp":1699372456.4452794,"name":"online","context":{"idset":"13"}}
+{"timestamp":1699372458.3275898,"name":"online","context":{"idset":"14"}}
+{"timestamp":1699372458.5721226,"name":"online","context":{"idset":"17"}}
+{"timestamp":1699372461.3946378,"name":"online","context":{"idset":"27"}}
+{"timestamp":1699372463.7506018,"name":"online","context":{"idset":"45"}}
+{"timestamp":1699372464.3058958,"name":"online","context":{"idset":"37"}}
+{"timestamp":1699372465.2906792,"name":"online","context":{"idset":"9"}}
+{"timestamp":1699372472.3511846,"name":"online","context":{"idset":"39"}}
+{"timestamp":1699372517.1122863,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1699372519.0114238,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1699372519.0115139,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1699372519.1116979,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1699372521.0120578,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1699372521.0121341,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1699372521.1122363,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1699372523.0117538,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1699372523.0118484,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1699372523.0119045,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1699372523.0119519,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1699372523.0119982,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1699372523.1116798,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1699372525.3039973,"name":"online","context":{"idset":"50"}}
+{"timestamp":1699372525.5098195,"name":"online","context":{"idset":"52"}}
+{"timestamp":1699372526.7643559,"name":"online","context":{"idset":"8"}}
+{"timestamp":1699372526.8090441,"name":"online","context":{"idset":"12"}}
+{"timestamp":1699372527.3810806,"name":"online","context":{"idset":"2"}}
+{"timestamp":1699372527.5986848,"name":"online","context":{"idset":"4"}}
+{"timestamp":1699372529.4204466,"name":"online","context":{"idset":"54"}}
+{"timestamp":1699372531.6369171,"name":"online","context":{"idset":"10"}}
+{"timestamp":1699372531.8413205,"name":"online","context":{"idset":"5"}}
+{"timestamp":1699372536.1427283,"name":"online","context":{"idset":"21"}}
+{"timestamp":1699372548.401464,"name":"online","context":{"idset":"31"}}
+{"timestamp":1699372558.1755195,"name":"online","context":{"idset":"51"}}
+{"timestamp":1699372563.1107347,"name":"online","context":{"idset":"56"}}
+{"timestamp":1699372567.9198549,"name":"online","context":{"idset":"47"}}
+{"timestamp":1699372569.4125333,"name":"online","context":{"idset":"6"}}
+{"timestamp":1699372599.0105367,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1699372599.1109426,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1699372601.0119829,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1699372601.0120625,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1699372601.0121109,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1699372601.1127517,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1699372643.0661368,"name":"online","context":{"idset":"1"}}
+{"timestamp":1699372655.2842569,"name":"online","context":{"idset":"28"}}
+{"timestamp":1699372695.1120136,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1699372697.0099852,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1699372697.0100622,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1699372697.0101123,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1699372697.0101855,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1699372697.0102582,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1699372697.1108584,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1699372720.2776916,"name":"online","context":{"idset":"9"}}
+{"timestamp":1699372724.9519835,"name":"online","context":{"idset":"25"}}
+{"timestamp":1699372725.5819447,"name":"online","context":{"idset":"30"}}
+{"timestamp":1699372725.9488704,"name":"online","context":{"idset":"36"}}
+{"timestamp":1699372730.0797856,"name":"online","context":{"idset":"35"}}
+{"timestamp":1699372731.0115728,"name":"online","context":{"idset":"32"}}
+{"timestamp":1699372732.2967279,"name":"online","context":{"idset":"20"}}
+{"timestamp":1699372741.0571427,"name":"online","context":{"idset":"57"}}
+{"timestamp":1699372745.8929493,"name":"online","context":{"idset":"49"}}
+{"timestamp":1699372748.7302775,"name":"online","context":{"idset":"29"}}
+{"timestamp":1699372782.6730464,"name":"online","context":{"idset":"18"}}
+{"timestamp":1699372794.9060733,"name":"online","context":{"idset":"22"}}
+{"timestamp":1699372804.8474517,"name":"online","context":{"idset":"58"}}
+{"timestamp":1699372814.1115479,"name":"online","context":{"idset":"14"}}
+{"timestamp":1699372814.9612718,"name":"online","context":{"idset":"15"}}
+{"timestamp":1699372816.9835479,"name":"online","context":{"idset":"55"}}
+{"timestamp":1699372861.4987693,"name":"online","context":{"idset":"24"}}
+{"timestamp":1699372862.8337286,"name":"online","context":{"idset":"13"}}
+{"timestamp":1699372880.2425444,"name":"online","context":{"idset":"41"}}
+{"timestamp":1699372882.4821684,"name":"online","context":{"idset":"16"}}
+{"timestamp":1699372884.2026401,"name":"online","context":{"idset":"17"}}
+{"timestamp":1699372890.1775632,"name":"online","context":{"idset":"12"}}
+{"timestamp":1699372900.1463149,"name":"online","context":{"idset":"6"}}
+{"timestamp":1699372900.9440918,"name":"online","context":{"idset":"4"}}
+{"timestamp":1699372904.0029085,"name":"online","context":{"idset":"26"}}
+{"timestamp":1699372939.0099592,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1699372939.010268,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1699372939.1365597,"name":"online","context":{"idset":"7"}}
+{"timestamp":1699372939.9148343,"name":"online","context":{"idset":"8"}}
+{"timestamp":1699372943.011179,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1699372943.0112917,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1699372943.0113533,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1699372943.1113007,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1699372945.1114333,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1699372945.3443904,"name":"online","context":{"idset":"2"}}
+{"timestamp":1699372961.3126304,"name":"online","context":{"idset":"34"}}
+{"timestamp":1699372978.3374836,"name":"online","context":{"idset":"10"}}
+{"timestamp":1699373021.0103536,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1699373021.0104585,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1699373021.1103814,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1699373023.1122355,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1699373025.1125669,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1699373050.5369406,"name":"online","context":{"idset":"9,59"}}
+{"timestamp":1699373055.8319831,"name":"online","context":{"idset":"53"}}
+{"timestamp":1699373105.0098855,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1699373105.1104453,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1699373147.2248874,"name":"online","context":{"idset":"25"}}
+{"timestamp":1699373148.2371972,"name":"online","context":{"idset":"55"}}
+{"timestamp":1699373218.0981617,"name":"online","context":{"idset":"6"}}
+{"timestamp":1699373225.6505833,"name":"online","context":{"idset":"37"}}
+{"timestamp":1699373226.0441253,"name":"online","context":{"idset":"44"}}
+{"timestamp":1699373226.1843934,"name":"online","context":{"idset":"21"}}
+{"timestamp":1699373226.7235708,"name":"online","context":{"idset":"42"}}
+{"timestamp":1699373227.8798573,"name":"online","context":{"idset":"46"}}
+{"timestamp":1699373228.193737,"name":"online","context":{"idset":"47"}}
+{"timestamp":1699373231.0097136,"name":"online","context":{"idset":"1"}}
+{"timestamp":1699373303.1768878,"name":"online","context":{"idset":"15,17"}}
+{"timestamp":1699396485.9352615,"name":"online","context":{"idset":"23"}}
+{"timestamp":1699396486.325258,"name":"online","context":{"idset":"11,19"}}
+{"timestamp":1699396486.4733412,"name":"online","context":{"idset":"33"}}
+{"timestamp":1699396525.8244505,"name":"undrain","context":{"idset":"1-60"}}
+{"timestamp":1699471517.10517,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1699471517.6829226,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1699471517.6852622,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1699471517.6870761,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1699471517.6898749,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1699471517.6983275,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1699471517.708334,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1699471517.7164528,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1699471517.7228768,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1699471517.7661355,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1699471517.7664909,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1699471517.7683432,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1699471517.7684309,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1699471517.7685416,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1699471517.7687268,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1699471517.768868,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1699471517.7689223,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1699471517.7789464,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1699471517.780108,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1699471517.7888083,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1699471517.8008568,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1699471517.8017695,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1699471517.8035326,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1699471517.8048675,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1699471517.8058181,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1699471517.8075166,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1699471517.8088057,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1699471517.8156691,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1699471517.8300307,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1699471517.8359668,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1699471517.8382168,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1699471517.8423254,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1699471517.8436782,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1699471517.8464069,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1699471517.8485115,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1699471517.8521266,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1699471517.8594992,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1699471517.8695047,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1699471517.8712118,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1699471517.8787019,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1699471517.8819628,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1699471517.8843129,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1699471517.889524,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1699471517.8944042,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1699471517.8957267,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1699471517.8974094,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1699471517.9143293,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1699471517.9203751,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1699471517.943337,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1699471517.9556124,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1699471517.9647987,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1699471517.9761388,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1699471518.0315659,"name":"drain","context":{"idset":"9-20","reason":"prolog failed for jobid fJeRS5pVKG3","overwrite":0}}
+{"timestamp":1699471518.0770957,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1699471521.4197593,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1699471521.52053,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1699471521.6029565,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1699471521.6179771,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1699471521.6181312,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1699471521.6851611,"name":"drain","context":{"idset":"1","reason":"epilog failed for jobid fJeQSeHVDcb","overwrite":0}}
+{"timestamp":1699471521.6862845,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1699471521.786736,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1699475501.5075717,"name":"online","context":{"idset":"29"}}
+{"timestamp":1699475501.6266255,"name":"online","context":{"idset":"5,15,18"}}
+{"timestamp":1699475501.7276533,"name":"online","context":{"idset":"1,6,8,14,24,30,34,37,45,47"}}
+{"timestamp":1699475501.8342052,"name":"online","context":{"idset":"2,11,13,16-17,25-26,28,32-33,46,48,50,53"}}
+{"timestamp":1699475501.9368899,"name":"online","context":{"idset":"3,7,20-23,31,38-39,51-52,57"}}
+{"timestamp":1699475502.0445859,"name":"online","context":{"idset":"4,10,12,19,40,43-44,54-55,58-60"}}
+{"timestamp":1699475502.1630766,"name":"online","context":{"idset":"35-36,41,49,56"}}
+{"timestamp":1699475502.2735174,"name":"online","context":{"idset":"9,42"}}
+{"timestamp":1699475502.4019966,"name":"online","context":{"idset":"27"}}
+{"timestamp":1699475557.7652125,"name":"undrain","context":{"idset":"1,9-20"}}
+{"timestamp":1699551137.4294446,"name":"drain","context":{"idset":"9","reason":"reason=Disabled for testing","overwrite":0}}
+{"timestamp":1699563893.0113256,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.0114346,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.0114772,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.0115125,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.0115454,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.0115802,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.01162,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.0116525,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.0116825,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.0117116,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.0117443,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1699563893.1113973,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1699563895.0114131,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1699563895.0115168,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1699563895.0115888,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1699563895.0116389,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1699563895.0116761,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1699563895.0117075,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1699563895.0117543,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1699563895.0117877,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1699563895.0118222,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1699563895.1119919,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.011867,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0119801,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0120337,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.012084,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0121322,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0121791,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0122252,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.01227,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0123179,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0123627,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0124085,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0124555,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0125022,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0125556,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0125997,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.012696,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0127406,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0127845,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0128274,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0128703,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0129149,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0129592,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0130043,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0130496,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0130937,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0131381,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0131838,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0132322,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0132868,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0133305,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0133748,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0134175,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0134828,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0135272,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0135708,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.0136318,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1699563897.1123362,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1699563959.0119336,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1699563959.0120754,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1699563959.0121737,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1699563959.0122533,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1699563959.0123291,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1699563959.012404,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1699563959.01248,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1699563959.0125544,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1699563959.012639,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1699563959.0127153,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1699563959.0128477,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1699563959.0131097,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1699563959.0132613,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1699563959.0133581,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1699563959.0134511,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1699563959.0135398,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1699563959.0136356,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1699563959.0137219,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1699563959.013828,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1699563959.0139441,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1699563959.0140874,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1699563959.0141718,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1699563959.0142896,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1699563959.0144267,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1699563959.0145245,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1699563959.0146329,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1699563959.0148525,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1699563959.0149615,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1699563959.0150659,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1699563959.015136,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1699563959.0151846,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1699563959.0152376,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1699563959.0153036,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1699563959.0153625,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1699563959.0154061,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1699563959.0154636,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1699563959.0155094,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1699563959.015569,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1699563959.0156279,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1699563959.0156872,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1699563959.0157413,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1699563959.0158029,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1699563959.015866,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1699563959.0159099,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1699563959.0159626,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1699563959.0160213,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1699563959.0160708,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1699563959.0161116,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1699563959.0161762,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1699563959.0162215,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1699563959.0162661,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1699563959.0163152,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1699563959.0163529,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1699563959.0163989,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1699563959.0164411,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1699563959.0164752,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1699563959.016516,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1699563959.016562,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1699563959.0222816,"name":"drain","context":{"idset":"5-8,10-13,22-29","reason":"epilog failed for jobid fJrUngbDAf1","overwrite":0}}
+{"timestamp":1699563959.1121597,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1699563961.1121347,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1699565471.6402295,"name":"online","context":{"idset":"39"}}
+{"timestamp":1699565472.0600736,"name":"online","context":{"idset":"2,5,12"}}
+{"timestamp":1699565472.1899393,"name":"online","context":{"idset":"3,27-28"}}
+{"timestamp":1699565472.2924056,"name":"online","context":{"idset":"10,14,16-17,33,37,46,53-54"}}
+{"timestamp":1699565472.4025941,"name":"online","context":{"idset":"1,8-9,13,15,21,26,29,31-32,35,38,41,45,50,52,57"}}
+{"timestamp":1699565472.5049183,"name":"online","context":{"idset":"4,6,18,22-23,34,49,56,58"}}
+{"timestamp":1699565472.613286,"name":"online","context":{"idset":"7,11,25,30,42-44,47,60"}}
+{"timestamp":1699565472.7141793,"name":"online","context":{"idset":"19-20,24,36,40,48,51,59"}}
+{"timestamp":1699565472.8945057,"name":"online","context":{"idset":"55"}}
+{"timestamp":1699565530.1469662,"name":"undrain","context":{"idset":"1-8,10-60"}}
+{"timestamp":1699572907.8246007,"name":"drain","context":{"idset":"29","reason":"reason=Disabled for testing","overwrite":0}}
+{"timestamp":1699984173.0117822,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0119116,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0119612,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0120099,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.012053,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0120943,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0121312,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.012166,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.012203,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0122423,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0122836,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0123196,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0123563,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0123901,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0124247,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0124609,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0124996,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0125391,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0125792,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0126376,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.0126882,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1699984173.1124246,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1699984175.0120602,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1699984175.0121589,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1699984175.0122118,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1699984175.1126649,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.012476,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0125799,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0126443,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0126948,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0127437,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0128105,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0128562,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0129001,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0129426,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0129852,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0130296,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0130744,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0131168,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0131614,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0132139,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0132582,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0133026,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0133469,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0133917,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0134368,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.013484,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0135319,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0135772,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.013633,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.0137155,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1699984177.1126513,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1699984179.0119619,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1699984179.0120676,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1699984179.0121274,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1699984179.0121808,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1699984179.012233,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1699984179.1120622,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1699984235.0118814,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1699984235.0119963,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1699984235.012068,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1699984235.0121419,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1699984235.0122378,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1699984235.0123336,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1699984235.0124292,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1699984235.0125227,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1699984235.0126252,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1699984235.0127213,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1699984235.0128181,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1699984235.0129125,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1699984235.1125534,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1699984237.0114129,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1699984237.1116149,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1699984239.0118976,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1699984239.0119908,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1699984239.0120735,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1699984239.0121653,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1699984239.0122616,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1699984239.0123529,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1699984239.0124462,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1699984239.0125399,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1699984239.1126466,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1699984241.0120394,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1699984241.0121689,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1699984241.0122769,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1699984241.0123577,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1699984241.0124297,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1699984241.0124974,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1699984241.0125656,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1699984241.0126429,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1699984241.0127065,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1699984241.0127766,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1699984241.01285,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1699984241.0129397,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1699984241.0130167,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1699984241.0130968,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1699984241.0131707,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1699984241.0132504,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1699984241.0133276,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1699984241.0133922,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1699984241.0134659,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1699984241.0135374,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1699984241.0136161,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1699984241.0136843,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1699984241.013752,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1699984241.0138195,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1699984241.0138831,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1699984241.0139465,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1699984241.0140061,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1699984241.0140576,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1699984241.014116,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1699984241.0141747,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1699984241.0142307,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1699984241.0142796,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1699984241.0143352,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1699984241.0144174,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1699984241.014473,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1699984241.1121402,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1699986127.5367072,"name":"online","context":{"idset":"9"}}
+{"timestamp":1699986127.7432961,"name":"online","context":{"idset":"6"}}
+{"timestamp":1699986141.7974577,"name":"online","context":{"idset":"38"}}
+{"timestamp":1699986141.920656,"name":"online","context":{"idset":"41"}}
+{"timestamp":1699986143.1100125,"name":"online","context":{"idset":"31"}}
+{"timestamp":1699986143.6425252,"name":"online","context":{"idset":"33"}}
+{"timestamp":1699986143.7673032,"name":"online","context":{"idset":"26"}}
+{"timestamp":1702081943.1126602,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1702081951.1122491,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1702082957.0119872,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1702082957.0120699,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1702082957.012121,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1702082957.012177,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1702082957.112186,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1702084115.6180418,"name":"online","context":{"idset":"12"}}
+{"timestamp":1702084115.8287883,"name":"online","context":{"idset":"4,25,27,53"}}
+{"timestamp":1702084115.9932916,"name":"online","context":{"idset":"3,9,21,23,29,34,38,45,49"}}
+{"timestamp":1702084116.0982895,"name":"online","context":{"idset":"1-2,10,14,17,28,30,32,42-44,48"}}
+{"timestamp":1702084116.2312157,"name":"online","context":{"idset":"7-8,18,24,35-36,56"}}
+{"timestamp":1702084116.3334,"name":"online","context":{"idset":"6,15-16,19,26,31,39,47,52,54"}}
+{"timestamp":1702084116.4378216,"name":"online","context":{"idset":"20,37,46,50,55"}}
+{"timestamp":1702084116.5665119,"name":"online","context":{"idset":"5,51"}}
+{"timestamp":1702084116.6857729,"name":"online","context":{"idset":"57"}}
+{"timestamp":1702084117.1117158,"name":"online","context":{"idset":"58"}}
+{"timestamp":1702084117.9776442,"name":"online","context":{"idset":"41"}}
+{"timestamp":1702084118.1801748,"name":"online","context":{"idset":"33"}}
+{"timestamp":1702084185.8059788,"name":"undrain","context":{"idset":"1-8,10,12,14-21,23-28,30-39,41-58"}}
+{"timestamp":1702256913.1741412,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1702256913.1789446,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1702256913.1830633,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1702256913.1852405,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1702256913.1863272,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1702256913.187058,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1702256913.194128,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1702256913.1944873,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702256913.199662,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1702256913.1997602,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1702256913.2072692,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1702256913.213593,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1702256913.2153101,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1702256913.2156134,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1702256913.2157166,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1702256913.2209554,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1702256913.2210519,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1702256913.2243063,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1702256913.2249258,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1702256913.2267179,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1702256913.2278144,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1702256913.2284241,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1702256913.2288158,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1702256913.2289281,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1702256913.2314031,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1702256913.2345481,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1702256913.2384565,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1702256913.2388248,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1702256913.2392428,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1702256913.2397649,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1702256913.2399054,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1702256913.2433977,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1702256913.2435091,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1702256913.2450507,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1702256913.2476912,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1702256913.2479429,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1702256913.2500448,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1702256913.2522357,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1702256913.2530105,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1702256913.2532499,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1702256913.2535706,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1702256913.2551129,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1702256913.2566743,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1702256913.2572451,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1702256913.2589295,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1702256913.2601552,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1702256913.2623434,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1702256913.2735274,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1702256913.3358877,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1702256913.3440735,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1702256913.3470092,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1702256913.4478099,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1702256914.4425352,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1702256914.6555452,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1702258447.5965109,"name":"resource-init","context":{"restart":true,"drain":{"9":{"timestamp":1699551137.4294446,"reason":"reason=Disabled for testing"},"11":{"timestamp":1699984177.0128562,"reason":"broker was unresponsive"},"13":{"timestamp":1699984173.012053,"reason":"broker was unresponsive"},"22":{"timestamp":1699984173.0123901,"reason":"broker was unresponsive"},"29":{"timestamp":1699572907.8246007,"reason":"reason=Disabled for testing"},"40":{"timestamp":1699984177.0133469,"reason":"broker was unresponsive"},"59":{"timestamp":1699984177.1126513,"reason":"broker was unresponsive"},"60":{"timestamp":1699984175.1126649,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1702258447.6074293,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1702258457.9180639,"name":"online","context":{"idset":"0"}}
+{"timestamp":1702292249.6323502,"name":"online","context":{"idset":"56"}}
+{"timestamp":1702292249.7870235,"name":"online","context":{"idset":"20"}}
+{"timestamp":1702292249.8139119,"name":"online","context":{"idset":"6"}}
+{"timestamp":1702292249.831316,"name":"online","context":{"idset":"36"}}
+{"timestamp":1702292250.0198505,"name":"online","context":{"idset":"7-8,14,47"}}
+{"timestamp":1702292250.1205933,"name":"online","context":{"idset":"17,35,37,54"}}
+{"timestamp":1702292250.2230587,"name":"online","context":{"idset":"12,15,28,30,50"}}
+{"timestamp":1702292250.3253438,"name":"online","context":{"idset":"9-10,18-19,25,27,29,33,42,45,53,57"}}
+{"timestamp":1702292250.4269836,"name":"online","context":{"idset":"2,5,16,23,32,38,44,49,52"}}
+{"timestamp":1702292250.5288475,"name":"online","context":{"idset":"21,26,31,34,39-41,46,55"}}
+{"timestamp":1702292250.6308696,"name":"online","context":{"idset":"3,24,43,48"}}
+{"timestamp":1702292534.5398672,"name":"drain","context":{"idset":"13,22,59","reason":"PY: hsi0 stuck in starting","overwrite":1}}
+{"timestamp":1702292593.4380839,"name":"drain","context":{"idset":"1,4,11,51,58,60","reason":"PY: rebooting","overwrite":1}}
+{"timestamp":1702292624.9044437,"name":"undrain","context":{"idset":"9,29"}}
+{"timestamp":1702292632.8093367,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1702292795.7206175,"name":"drain","context":{"idset":"2,44,46,48","reason":"PY: connection timed out on 'df -ht lustre'","overwrite":0}}
+{"timestamp":1702293665.5422151,"name":"drain","context":{"idset":"60","reason":"PY: hsi0 stuck in starting","overwrite":1}}
+{"timestamp":1702293735.074461,"name":"online","context":{"idset":"1,51"}}
+{"timestamp":1702293735.2078235,"name":"online","context":{"idset":"4"}}
+{"timestamp":1702293735.3516707,"name":"online","context":{"idset":"11,58"}}
+{"timestamp":1702293785.5267444,"name":"undrain","context":{"idset":"1,4,11,51,58"}}
+{"timestamp":1702491203.7145364,"name":"online","context":{"idset":"22"}}
+{"timestamp":1702491224.9776416,"name":"undrain","context":{"idset":"22"}}
+{"timestamp":1702491512.1824293,"name":"online","context":{"idset":"60"}}
+{"timestamp":1702491512.4355521,"name":"online","context":{"idset":"59"}}
+{"timestamp":1702491544.4851344,"name":"undrain","context":{"idset":"59-60"}}
+{"timestamp":1702491940.2006738,"name":"undrain","context":{"idset":"2,44,46,48"}}
+{"timestamp":1702515527.1213748,"name":"drain","context":{"idset":"39","reason":"reason=Lustre hung","overwrite":0}}
+{"timestamp":1702516835.0597153,"name":"drain","context":{"idset":"39","reason":"epilog failed for jobid fR8RV8gEpsy","overwrite":0}}
+{"timestamp":1702516835.1581616,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1702519010.6400938,"name":"drain","context":{"idset":"2","reason":"reason=Lustre hang","overwrite":0}}
+{"timestamp":1702605501.1257808,"name":"drain","context":{"idset":"21","reason":"reason=Lustre hung","overwrite":0}}
+{"timestamp":1702658505.2510126,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1702658505.251651,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1702658505.2533643,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1702658505.2560444,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1702658505.2563565,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1702658505.2564261,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1702658505.2647092,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1702658505.265049,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1702658505.2659917,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1702658505.267566,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1702658505.2679174,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1702658505.2686357,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1702658505.2706788,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1702658505.2724869,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1702658505.2726793,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1702658505.2736151,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1702658505.2744274,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1702658505.2746558,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1702658505.2774913,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1702658505.2783918,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1702658505.2800913,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1702658505.2815425,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1702658505.2823236,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1702658505.2827148,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1702658505.2844481,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1702658505.2845292,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1702658505.285058,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1702658505.2851849,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1702658505.2857263,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1702658505.2857902,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1702658505.2858996,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1702658505.2859583,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1702658505.2861223,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1702658505.2868857,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1702658505.2886274,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1702658505.2886698,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1702658505.2888105,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1702658505.2892535,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1702658505.2895958,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1702658505.2905221,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1702658505.2922962,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1702658505.292592,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1702658505.2929249,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1702658505.2933524,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1702658505.2939348,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1702658505.2943325,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1702658505.2953568,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1702658505.2983966,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1702658505.3005846,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1702658505.3011878,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1702658505.3019085,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1702658505.3025043,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1702658505.302779,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1702658505.3032179,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702658505.3044751,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1702658505.3050086,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1702658505.3420873,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1702658505.4426029,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1702658568.2506688,"name":"resource-init","context":{"restart":true,"drain":{"2":{"timestamp":1702519010.6400938,"reason":"reason=Lustre hang"},"13":{"timestamp":1699984173.012053,"reason":"PY: hsi0 stuck in starting"},"21":{"timestamp":1702605501.1257808,"reason":"reason=Lustre hung"},"39":{"timestamp":1702515527.1213748,"reason":"reason=Lustre hung"}},"online":"","exclude":"0"}}
+{"timestamp":1702658568.2755749,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1702658579.3024321,"name":"online","context":{"idset":"0"}}
+{"timestamp":1702658580.3378205,"name":"online","context":{"idset":"1-2,5-10,12,15-16,18-19,22-23,27-28,32-38,40-41,43-52,54-58,60"}}
+{"timestamp":1702658580.4398093,"name":"online","context":{"idset":"3-4,11,14,17,20-21,24-26,29-31,42,53,59"}}
+{"timestamp":1702678646.575665,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1702678646.6762557,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1702678710.5753672,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1702678710.5754857,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1702678710.6755872,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1702678712.6762073,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1702680225.3120198,"name":"online","context":{"idset":"6"}}
+{"timestamp":1702680227.0330284,"name":"online","context":{"idset":"21"}}
+{"timestamp":1702680281.2989914,"name":"online","context":{"idset":"39"}}
+{"timestamp":1702680295.0445921,"name":"online","context":{"idset":"44"}}
+{"timestamp":1702680510.4011998,"name":"undrain","context":{"idset":"6,21,39,44"}}
+{"timestamp":1702681828.5752239,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.5753076,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.5753379,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.5753636,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.5753882,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.5754149,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.5754364,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.5754573,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.5754795,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.5755086,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.5755339,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1702681828.6760621,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5757616,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5758567,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5759094,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5759354,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5759611,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5759871,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5760117,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5760367,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5760624,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5760865,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5761144,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.5761521,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1702681832.6765606,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5745773,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5746775,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5747461,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5748119,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5748539,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5748892,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5749249,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5749598,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5749967,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5750322,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5750678,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5751023,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5751483,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5751851,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5752218,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5752568,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5752916,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5753267,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.575362,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5753968,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5754316,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5754671,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5755031,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.575541,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5755773,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5756147,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5756519,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5756874,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.575727,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5757651,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5758035,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.5758455,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1702681834.6750965,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1702681892.575099,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1702681892.5752623,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1702681892.575388,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1702681892.5754955,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1702681892.5755956,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1702681892.5756938,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1702681892.5757887,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1702681892.5758882,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1702681892.5759833,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1702681892.5760791,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1702681892.6749847,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1702681894.5754838,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1702681894.6753578,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1702681896.5759633,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1702681896.5761054,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1702681896.576272,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1702681896.5764048,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1702681896.5765376,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1702681896.576679,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1702681896.5768151,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1702681896.5769382,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1702681896.5770581,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1702681896.5771885,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1702681896.5773056,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1702681896.5774198,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1702681896.5775406,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1702681896.5776517,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1702681896.5777631,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1702681896.5778711,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1702681896.5779908,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1702681896.578109,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1702681896.5782301,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1702681896.5783436,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1702681896.5784545,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1702681896.5785596,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1702681896.5786631,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1702681896.5787752,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1702681896.5788774,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1702681896.57898,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1702681896.5790889,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1702681896.5791972,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1702681896.6763639,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1702681897.5768621,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1702681897.5769324,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1702681897.5769808,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1702681897.5770266,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1702681897.5770695,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1702681897.5771108,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702681897.5771577,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1702681897.5771964,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1702681897.5772331,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1702681897.5772693,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1702681897.5773053,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1702681897.5773382,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1702681897.5773704,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1702681897.5774012,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1702681897.5774274,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1702681897.6771021,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1702682835.833782,"name":"online","context":{"idset":"59"}}
+{"timestamp":1702682960.6744556,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1702683133.5361986,"name":"online","context":{"idset":"6"}}
+{"timestamp":1702683134.5748324,"name":"online","context":{"idset":"21"}}
+{"timestamp":1702683136.5750356,"name":"online","context":{"idset":"44"}}
+{"timestamp":1702683137.1009781,"name":"online","context":{"idset":"39"}}
+{"timestamp":1702683140.3306992,"name":"online","context":{"idset":"17"}}
+{"timestamp":1702683142.2836499,"name":"online","context":{"idset":"45"}}
+{"timestamp":1702683143.0614569,"name":"online","context":{"idset":"27"}}
+{"timestamp":1702683143.1963935,"name":"online","context":{"idset":"28"}}
+{"timestamp":1702683143.7795696,"name":"online","context":{"idset":"40"}}
+{"timestamp":1702683144.6762938,"name":"online","context":{"idset":"60"}}
+{"timestamp":1702683148.5745373,"name":"online","context":{"idset":"38"}}
+{"timestamp":1702683149.8948052,"name":"online","context":{"idset":"59"}}
+{"timestamp":1702683151.1494055,"name":"online","context":{"idset":"49"}}
+{"timestamp":1702683203.1904249,"name":"online","context":{"idset":"10"}}
+{"timestamp":1702683203.3512518,"name":"online","context":{"idset":"4"}}
+{"timestamp":1702683203.9914007,"name":"online","context":{"idset":"7"}}
+{"timestamp":1702683204.1177304,"name":"online","context":{"idset":"9,19"}}
+{"timestamp":1702683204.2683625,"name":"online","context":{"idset":"12"}}
+{"timestamp":1702683204.4817467,"name":"online","context":{"idset":"22"}}
+{"timestamp":1702683204.5757418,"name":"online","context":{"idset":"11"}}
+{"timestamp":1702683204.7866733,"name":"online","context":{"idset":"16"}}
+{"timestamp":1702683204.9225852,"name":"online","context":{"idset":"18"}}
+{"timestamp":1702683205.3154588,"name":"online","context":{"idset":"14"}}
+{"timestamp":1702683205.4592113,"name":"online","context":{"idset":"50"}}
+{"timestamp":1702683205.9238408,"name":"online","context":{"idset":"15,51,56"}}
+{"timestamp":1702683206.0443547,"name":"online","context":{"idset":"5,31"}}
+{"timestamp":1702683206.1392217,"name":"online","context":{"idset":"57"}}
+{"timestamp":1702683206.5736334,"name":"online","context":{"idset":"8,30"}}
+{"timestamp":1702683207.0464079,"name":"online","context":{"idset":"20,33-34"}}
+{"timestamp":1702683207.1496947,"name":"online","context":{"idset":"1,3,23"}}
+{"timestamp":1702683207.2870722,"name":"online","context":{"idset":"52,55"}}
+{"timestamp":1702683207.4308612,"name":"online","context":{"idset":"41"}}
+{"timestamp":1702683207.572217,"name":"online","context":{"idset":"29"}}
+{"timestamp":1702683207.6882501,"name":"online","context":{"idset":"24,58"}}
+{"timestamp":1702683209.0447884,"name":"online","context":{"idset":"37,54"}}
+{"timestamp":1702683209.1475282,"name":"online","context":{"idset":"53"}}
+{"timestamp":1702683243.9385216,"name":"online","context":{"idset":"42"}}
+{"timestamp":1702683244.8482685,"name":"online","context":{"idset":"48"}}
+{"timestamp":1702683329.1260564,"name":"online","context":{"idset":"47"}}
+{"timestamp":1702683330.0766358,"name":"online","context":{"idset":"35"}}
+{"timestamp":1702683330.2614896,"name":"online","context":{"idset":"25"}}
+{"timestamp":1702683331.336601,"name":"online","context":{"idset":"43"}}
+{"timestamp":1702683331.4714537,"name":"online","context":{"idset":"26,46"}}
+{"timestamp":1702683332.4038391,"name":"online","context":{"idset":"36"}}
+{"timestamp":1702683332.6764905,"name":"online","context":{"idset":"32"}}
+{"timestamp":1702688247.4793563,"name":"undrain","context":{"idset":"1,3-12,14-60"}}
+{"timestamp":1702925048.9389622,"name":"drain","context":{"idset":"40","reason":"PY: check_ama fails","overwrite":0}}
+{"timestamp":1702925111.9160309,"name":"online","context":{"idset":"2"}}
+{"timestamp":1702925855.9089143,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1702934542.1105003,"name":"drain","context":{"idset":"40","reason":"PY: check_ama fails","overwrite":0}}
+{"timestamp":1702948314.676919,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1702948326.6744945,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1702948332.6765754,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1702948342.6765735,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1702948364.6756625,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1702948376.6758783,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1702948392.6759474,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1702948396.6758986,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1702948398.6746054,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1702948408.6759567,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1702948430.6761448,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1702948450.6765311,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1702948456.6769848,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1702948462.6755264,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1702948516.6758816,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1702948518.6755276,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1702948554.6768339,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1702948620.6756501,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1702949186.6765082,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1702949254.6746185,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1702951446.6764624,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1702951510.6760116,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1703010912.6762655,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1703010978.6762018,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1703018620.6759999,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1703018686.6753159,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1703117692.6767783,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1703117754.6768308,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1703118596.6755142,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1703118660.6764677,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1703268040.6760943,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1703268104.6755469,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1703275458.6745811,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1703275522.6757452,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1703275544.6769209,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1703275606.6759582,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1703275736.676528,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1703275802.6748891,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1704224359.9204025,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1704224359.9662204,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1704224359.9922366,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1704224360.00984,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1704224360.0168755,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1704224360.0307326,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1704224360.0327878,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1704224360.0448413,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1704224360.048589,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1704224360.0488966,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1704224360.0490725,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1704224360.050622,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1704224360.0574505,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1704224360.0683296,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1704224360.0705872,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1704224360.0706413,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1704224360.0720627,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1704224360.0750899,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1704224360.0842845,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1704224360.0900099,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1704224360.1001356,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1704224360.1129332,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1704224360.1724334,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1704224360.1747684,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1704224360.177712,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1704224360.1828465,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1704224360.1920125,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1704224360.1930778,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1704224360.2141612,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1704224360.2482619,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1704224360.2547541,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1704224360.2628746,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1704224360.2669983,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1704224360.2780223,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1704224360.3016086,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1704224360.323236,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1704224360.4240284,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1704224360.7242932,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1704228702.9121153,"name":"online","context":{"idset":"4,32"}}
+{"timestamp":1704228703.0320511,"name":"online","context":{"idset":"9,21-22,25,42"}}
+{"timestamp":1704228703.1723394,"name":"online","context":{"idset":"7,28,31,33,36"}}
+{"timestamp":1704228703.3021333,"name":"online","context":{"idset":"6,14,24,43,56-57,60"}}
+{"timestamp":1704228703.4050238,"name":"online","context":{"idset":"5,11-12,15,17,37,39,44,48,51,53"}}
+{"timestamp":1704228703.5236244,"name":"online","context":{"idset":"16,27,45,47,52,59"}}
+{"timestamp":1704228703.6259642,"name":"online","context":{"idset":"34"}}
+{"timestamp":1704228703.8672876,"name":"online","context":{"idset":"41"}}
+{"timestamp":1704231584.6763091,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1704231648.6745708,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1704234746.2065663,"name":"online","context":{"idset":"54"}}
+{"timestamp":1704234941.1850383,"name":"online","context":{"idset":"8,30,50"}}
+{"timestamp":1704234941.3461411,"name":"online","context":{"idset":"3,10,46,49"}}
+{"timestamp":1704234941.4524136,"name":"online","context":{"idset":"23,35,55"}}
+{"timestamp":1704234941.5788522,"name":"online","context":{"idset":"19,29,58"}}
+{"timestamp":1704234941.6865964,"name":"online","context":{"idset":"26"}}
+{"timestamp":1704234941.9408884,"name":"online","context":{"idset":"18"}}
+{"timestamp":1704235511.5314405,"name":"undrain","context":{"idset":"3,8,10,18-19,23,26,29-30,35,46,49-50,54-55,58"}}
+{"timestamp":1704236860.6761949,"name":"online","context":{"idset":"38"}}
+{"timestamp":1704236882.4228125,"name":"undrain","context":{"idset":"38"}}
+{"timestamp":1704236934.5502968,"name":"online","context":{"idset":"37"}}
+{"timestamp":1704236941.067349,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1704237000.3174419,"name":"online","context":{"idset":"1"}}
+{"timestamp":1704237008.9382279,"name":"undrain","context":{"idset":"1"}}
+{"timestamp":1704237072.6750932,"name":"online","context":{"idset":"20"}}
+{"timestamp":1704237088.8487213,"name":"undrain","context":{"idset":"20"}}
+{"timestamp":1704237226.3410127,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1704237641.0437946,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1704238681.3736224,"name":"online","context":{"idset":"2"}}
+{"timestamp":1704238691.6497083,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1704241399.402348,"name":"online","context":{"idset":"13"}}
+{"timestamp":1704241412.5562179,"name":"undrain","context":{"idset":"13"}}
+{"timestamp":1704242177.0969388,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1704242299.0678082,"name":"drain","context":{"idset":"40","reason":"JRG: Failed cxi_healthcheck","overwrite":0}}
+{"timestamp":1704243698.5791121,"name":"online","context":{"idset":"40"}}
+{"timestamp":1704243708.7657821,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1704246345.5469472,"name":"drain","context":{"idset":"16","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.5823963,"name":"drain","context":{"idset":"7","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.6105297,"name":"drain","context":{"idset":"12","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.638021,"name":"drain","context":{"idset":"5","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.6650667,"name":"drain","context":{"idset":"9","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.6921933,"name":"drain","context":{"idset":"4","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.7197502,"name":"drain","context":{"idset":"6","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.7468817,"name":"drain","context":{"idset":"14","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.7747321,"name":"drain","context":{"idset":"17","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.8025064,"name":"drain","context":{"idset":"15","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.829962,"name":"drain","context":{"idset":"21","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246345.8574214,"name":"drain","context":{"idset":"11","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246719.9898899,"name":"drain","context":{"idset":"32","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246720.0174832,"name":"drain","context":{"idset":"24","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246720.0450404,"name":"drain","context":{"idset":"31","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246720.0879643,"name":"drain","context":{"idset":"27","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246720.1622436,"name":"drain","context":{"idset":"28","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246720.1949925,"name":"drain","context":{"idset":"25","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246721.3557668,"name":"drain","context":{"idset":"39","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246721.4409375,"name":"drain","context":{"idset":"41","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246721.4863527,"name":"drain","context":{"idset":"42","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246721.5351,"name":"drain","context":{"idset":"33","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246721.6313341,"name":"drain","context":{"idset":"34","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246721.6693325,"name":"drain","context":{"idset":"36","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246732.3151388,"name":"drain","context":{"idset":"22","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246748.2698257,"name":"drain","context":{"idset":"48","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246748.2980838,"name":"drain","context":{"idset":"43","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246748.3265007,"name":"drain","context":{"idset":"44","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246748.3546851,"name":"drain","context":{"idset":"51","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246748.3827939,"name":"drain","context":{"idset":"45","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246748.4755301,"name":"drain","context":{"idset":"47","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246748.5242271,"name":"drain","context":{"idset":"52","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246887.2501743,"name":"drain","context":{"idset":"56","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246887.4298999,"name":"drain","context":{"idset":"53","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704246887.6021895,"name":"drain","context":{"idset":"57","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704305155.4914291,"name":"undrain","context":{"idset":"27-28,31-34"}}
+{"timestamp":1704310861.7871666,"name":"undrain","context":{"idset":"4-7,9,11-12,14-17,21-22,24-25,36,39,41-45,47-48,51-53,56-57"}}
+{"timestamp":1704400378.9826291,"name":"drain","context":{"idset":"4","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704400379.0323837,"name":"drain","context":{"idset":"11","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704400379.0835128,"name":"drain","context":{"idset":"9","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704400379.1302075,"name":"drain","context":{"idset":"15","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704400379.1632304,"name":"drain","context":{"idset":"7","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704400379.1925983,"name":"drain","context":{"idset":"12","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704400379.2304654,"name":"drain","context":{"idset":"5","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704400379.2781975,"name":"drain","context":{"idset":"14","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704400379.308882,"name":"drain","context":{"idset":"6","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704400379.3392653,"name":"drain","context":{"idset":"16","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704401880.3127789,"name":"drain","context":{"idset":"17","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704401886.8830318,"name":"drain","context":{"idset":"22","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704401894.8365057,"name":"drain","context":{"idset":"25","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704401899.5115166,"name":"drain","context":{"idset":"21","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704401914.3171771,"name":"drain","context":{"idset":"24","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704403226.2588379,"name":"drain","context":{"idset":"31","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704403226.3259783,"name":"drain","context":{"idset":"27","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704403226.3599064,"name":"drain","context":{"idset":"28","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704403897.0943618,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1704404666.800724,"name":"drain","context":{"idset":"32","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404669.9288597,"name":"drain","context":{"idset":"34","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404685.0200262,"name":"drain","context":{"idset":"33","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404897.2390711,"name":"drain","context":{"idset":"41","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404904.034977,"name":"drain","context":{"idset":"51","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404905.8228064,"name":"drain","context":{"idset":"44","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404907.7336984,"name":"drain","context":{"idset":"36","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404908.3314121,"name":"drain","context":{"idset":"43","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404908.9555621,"name":"drain","context":{"idset":"47","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404910.9524491,"name":"drain","context":{"idset":"42","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404911.7537403,"name":"drain","context":{"idset":"39","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404912.3472645,"name":"drain","context":{"idset":"48","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704404929.2659218,"name":"drain","context":{"idset":"45","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704405712.2999094,"name":"drain","context":{"idset":"52","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704405725.3420799,"name":"drain","context":{"idset":"53","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704407100.9589999,"name":"online","context":{"idset":"2"}}
+{"timestamp":1704407790.7358072,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1704408262.1079504,"name":"undrain","context":{"idset":"4-7,9,11-12,14-17,21-22,24-25,27-28,31-34,36,39,41-45,47-48,51-53"}}
+{"timestamp":1704408887.6957195,"name":"drain","context":{"idset":"16","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408888.756072,"name":"drain","context":{"idset":"34","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408891.6092818,"name":"drain","context":{"idset":"48","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408891.8075292,"name":"drain","context":{"idset":"31","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408893.7868586,"name":"drain","context":{"idset":"12","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408893.9493976,"name":"drain","context":{"idset":"27","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408894.1853027,"name":"drain","context":{"idset":"28","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408895.1392095,"name":"drain","context":{"idset":"36","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408896.8922303,"name":"drain","context":{"idset":"45","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408898.6012671,"name":"drain","context":{"idset":"17","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408898.9763329,"name":"drain","context":{"idset":"6","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408900.6780276,"name":"drain","context":{"idset":"21","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408901.5823519,"name":"drain","context":{"idset":"25","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408904.3295279,"name":"drain","context":{"idset":"43","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408904.71504,"name":"drain","context":{"idset":"47","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408904.7507422,"name":"drain","context":{"idset":"33","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408905.9539349,"name":"drain","context":{"idset":"15","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408906.0708032,"name":"drain","context":{"idset":"5","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408906.4219396,"name":"drain","context":{"idset":"41","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408906.4602416,"name":"drain","context":{"idset":"11","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408907.5218883,"name":"drain","context":{"idset":"44","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408907.6484294,"name":"drain","context":{"idset":"4","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408908.0385785,"name":"drain","context":{"idset":"9","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408908.1797121,"name":"drain","context":{"idset":"14","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408909.140697,"name":"drain","context":{"idset":"22","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408909.939045,"name":"drain","context":{"idset":"32","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408911.2337706,"name":"drain","context":{"idset":"7","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408912.2378721,"name":"drain","context":{"idset":"42","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408916.1912811,"name":"drain","context":{"idset":"39","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704408925.2620032,"name":"drain","context":{"idset":"24","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409378.7848465,"name":"undrain","context":{"idset":"4-7,9,11-12,14-17,21-22,24-25,27-28,31-34,36,39,41-45,47-48"}}
+{"timestamp":1704409544.6159039,"name":"drain","context":{"idset":"49","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.679256,"name":"drain","context":{"idset":"3","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.7119851,"name":"drain","context":{"idset":"44","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.7443008,"name":"drain","context":{"idset":"32","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.7773731,"name":"drain","context":{"idset":"31","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.8098369,"name":"drain","context":{"idset":"28","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.8384268,"name":"drain","context":{"idset":"48","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.8660886,"name":"drain","context":{"idset":"7","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.8952584,"name":"drain","context":{"idset":"6","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.9225228,"name":"drain","context":{"idset":"4","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.949774,"name":"drain","context":{"idset":"30","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409544.9770172,"name":"drain","context":{"idset":"25","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.0125129,"name":"drain","context":{"idset":"15","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.0484097,"name":"drain","context":{"idset":"38","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.0757422,"name":"drain","context":{"idset":"1","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.1051793,"name":"drain","context":{"idset":"11","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.1354136,"name":"drain","context":{"idset":"29","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.1641061,"name":"drain","context":{"idset":"17","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.1925631,"name":"drain","context":{"idset":"39","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.2202611,"name":"drain","context":{"idset":"40","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.2474539,"name":"drain","context":{"idset":"46","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.2745612,"name":"drain","context":{"idset":"13","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.3018429,"name":"drain","context":{"idset":"26","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.3291688,"name":"drain","context":{"idset":"10","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.3564696,"name":"drain","context":{"idset":"20","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.3837798,"name":"drain","context":{"idset":"8","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.4115231,"name":"drain","context":{"idset":"35","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.4389632,"name":"drain","context":{"idset":"27","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.466239,"name":"drain","context":{"idset":"5","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.4933107,"name":"drain","context":{"idset":"9","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.5203862,"name":"drain","context":{"idset":"14","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.5478251,"name":"drain","context":{"idset":"16","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.5749404,"name":"drain","context":{"idset":"45","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.6021245,"name":"drain","context":{"idset":"37","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.6293929,"name":"drain","context":{"idset":"42","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.6567466,"name":"drain","context":{"idset":"18","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.6840346,"name":"drain","context":{"idset":"34","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.7120538,"name":"drain","context":{"idset":"12","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.7391887,"name":"drain","context":{"idset":"36","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.7666838,"name":"drain","context":{"idset":"47","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.794064,"name":"drain","context":{"idset":"43","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.8220885,"name":"drain","context":{"idset":"19","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.8493845,"name":"drain","context":{"idset":"23","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.8765664,"name":"drain","context":{"idset":"33","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.9043205,"name":"drain","context":{"idset":"41","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.9317617,"name":"drain","context":{"idset":"24","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.959213,"name":"drain","context":{"idset":"22","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409545.9881792,"name":"drain","context":{"idset":"21","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704409826.6177506,"name":"undrain","context":{"idset":"1,3-49"}}
+{"timestamp":1704411547.4184349,"name":"drain","context":{"idset":"30","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411547.4480827,"name":"drain","context":{"idset":"27","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411548.4959595,"name":"drain","context":{"idset":"5","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411549.2912233,"name":"drain","context":{"idset":"7","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411550.9096742,"name":"drain","context":{"idset":"34","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411551.039134,"name":"drain","context":{"idset":"48","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411551.6513591,"name":"drain","context":{"idset":"42","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411551.8159783,"name":"drain","context":{"idset":"37","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411551.8854198,"name":"drain","context":{"idset":"17","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411552.0491493,"name":"drain","context":{"idset":"44","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411552.1409793,"name":"drain","context":{"idset":"9","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411552.4995241,"name":"drain","context":{"idset":"38","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411552.6280594,"name":"drain","context":{"idset":"47","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411552.6606622,"name":"drain","context":{"idset":"15","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411552.689513,"name":"drain","context":{"idset":"29","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411552.718425,"name":"drain","context":{"idset":"11","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411552.746516,"name":"drain","context":{"idset":"33","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411552.809613,"name":"drain","context":{"idset":"46","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411552.8845828,"name":"drain","context":{"idset":"28","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411553.4876206,"name":"drain","context":{"idset":"31","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411553.8607466,"name":"drain","context":{"idset":"35","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411553.9887285,"name":"drain","context":{"idset":"49","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411554.1149404,"name":"drain","context":{"idset":"45","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411554.4870574,"name":"drain","context":{"idset":"43","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411555.2706001,"name":"drain","context":{"idset":"21","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411555.9490325,"name":"drain","context":{"idset":"16","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411556.5627444,"name":"drain","context":{"idset":"8","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411558.6088595,"name":"drain","context":{"idset":"12","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411559.029635,"name":"drain","context":{"idset":"3","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411559.2951987,"name":"drain","context":{"idset":"18","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411559.3729651,"name":"drain","context":{"idset":"26","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411560.1439567,"name":"drain","context":{"idset":"36","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411560.8544853,"name":"drain","context":{"idset":"6","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411560.9113591,"name":"drain","context":{"idset":"24","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411561.2386167,"name":"drain","context":{"idset":"1","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411561.5629344,"name":"drain","context":{"idset":"32","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411561.7021544,"name":"drain","context":{"idset":"41","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411562.1601171,"name":"drain","context":{"idset":"23","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411562.1931851,"name":"drain","context":{"idset":"40","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411562.5170257,"name":"drain","context":{"idset":"14","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411563.5273576,"name":"drain","context":{"idset":"4","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411563.7593267,"name":"drain","context":{"idset":"39","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411564.5283425,"name":"drain","context":{"idset":"10","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411564.8755119,"name":"drain","context":{"idset":"20","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411566.5026276,"name":"drain","context":{"idset":"19","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411567.2839773,"name":"drain","context":{"idset":"25","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411567.9109938,"name":"drain","context":{"idset":"22","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704411580.0126176,"name":"drain","context":{"idset":"13","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704412218.450896,"name":"undrain","context":{"idset":"1,3-49"}}
+{"timestamp":1704413524.9298251,"name":"drain","context":{"idset":"6","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413525.3335433,"name":"drain","context":{"idset":"26","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413525.5781906,"name":"drain","context":{"idset":"23","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413525.6179943,"name":"drain","context":{"idset":"3","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413525.6783917,"name":"drain","context":{"idset":"49","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413525.8565004,"name":"drain","context":{"idset":"1","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413525.8896463,"name":"drain","context":{"idset":"31","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413525.9561572,"name":"drain","context":{"idset":"28","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413525.9842327,"name":"drain","context":{"idset":"27","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.0123713,"name":"drain","context":{"idset":"25","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.0407917,"name":"drain","context":{"idset":"47","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.0701187,"name":"drain","context":{"idset":"10","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.0990918,"name":"drain","context":{"idset":"43","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.1282308,"name":"drain","context":{"idset":"7","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.1565838,"name":"drain","context":{"idset":"32","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.1840639,"name":"drain","context":{"idset":"30","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.2114885,"name":"drain","context":{"idset":"38","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.2391977,"name":"drain","context":{"idset":"48","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.2666738,"name":"drain","context":{"idset":"4","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.293987,"name":"drain","context":{"idset":"19","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.321842,"name":"drain","context":{"idset":"18","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.3492646,"name":"drain","context":{"idset":"42","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.3765318,"name":"drain","context":{"idset":"36","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.4042418,"name":"drain","context":{"idset":"44","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.4320843,"name":"drain","context":{"idset":"8","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.4594755,"name":"drain","context":{"idset":"11","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.4868798,"name":"drain","context":{"idset":"5","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.514535,"name":"drain","context":{"idset":"13","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.5419645,"name":"drain","context":{"idset":"33","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.5696349,"name":"drain","context":{"idset":"20","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.5970781,"name":"drain","context":{"idset":"46","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.6247327,"name":"drain","context":{"idset":"40","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.6520863,"name":"drain","context":{"idset":"39","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.6797738,"name":"drain","context":{"idset":"29","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.7070575,"name":"drain","context":{"idset":"16","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.7346981,"name":"drain","context":{"idset":"15","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.7620041,"name":"drain","context":{"idset":"34","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.789674,"name":"drain","context":{"idset":"12","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.8170197,"name":"drain","context":{"idset":"35","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.8448746,"name":"drain","context":{"idset":"37","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.8720577,"name":"drain","context":{"idset":"21","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.8999,"name":"drain","context":{"idset":"24","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.9281535,"name":"drain","context":{"idset":"17","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.9553359,"name":"drain","context":{"idset":"9","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413526.9831243,"name":"drain","context":{"idset":"14","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413527.0105274,"name":"drain","context":{"idset":"22","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413527.0387287,"name":"drain","context":{"idset":"41","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704413527.0659652,"name":"drain","context":{"idset":"45","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704481847.510705,"name":"undrain","context":{"idset":"1,3-49"}}
+{"timestamp":1704482084.7746108,"name":"drain","context":{"idset":"48","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482084.8808222,"name":"drain","context":{"idset":"7","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482084.9216866,"name":"drain","context":{"idset":"3","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482084.9731593,"name":"drain","context":{"idset":"27","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.0195289,"name":"drain","context":{"idset":"51","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.047821,"name":"drain","context":{"idset":"49","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.0755596,"name":"drain","context":{"idset":"8","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.1035247,"name":"drain","context":{"idset":"4","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.1314745,"name":"drain","context":{"idset":"26","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.1593225,"name":"drain","context":{"idset":"1","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.1876752,"name":"drain","context":{"idset":"28","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.216043,"name":"drain","context":{"idset":"15","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.2444291,"name":"drain","context":{"idset":"5","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.2723169,"name":"drain","context":{"idset":"31","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.300189,"name":"drain","context":{"idset":"12","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.3284645,"name":"drain","context":{"idset":"50","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.3564408,"name":"drain","context":{"idset":"34","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.3836761,"name":"drain","context":{"idset":"19","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.411212,"name":"drain","context":{"idset":"6","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.4385002,"name":"drain","context":{"idset":"29","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.4655905,"name":"drain","context":{"idset":"10","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.4926846,"name":"drain","context":{"idset":"32","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.5195751,"name":"drain","context":{"idset":"53","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.5455956,"name":"drain","context":{"idset":"42","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.5710161,"name":"drain","context":{"idset":"55","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.5955713,"name":"drain","context":{"idset":"30","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.6198034,"name":"drain","context":{"idset":"46","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.6433282,"name":"drain","context":{"idset":"9","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.66664,"name":"drain","context":{"idset":"39","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.689636,"name":"drain","context":{"idset":"36","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.7119493,"name":"drain","context":{"idset":"56","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.7339625,"name":"drain","context":{"idset":"16","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.755594,"name":"drain","context":{"idset":"40","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.7779024,"name":"drain","context":{"idset":"11","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.7986398,"name":"drain","context":{"idset":"25","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.8192844,"name":"drain","context":{"idset":"54","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.8393681,"name":"drain","context":{"idset":"52","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.8722534,"name":"drain","context":{"idset":"17","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.9051597,"name":"drain","context":{"idset":"18","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.9351773,"name":"drain","context":{"idset":"43","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.9679301,"name":"drain","context":{"idset":"20","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482085.9947495,"name":"drain","context":{"idset":"13","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.0231128,"name":"drain","context":{"idset":"38","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.0470107,"name":"drain","context":{"idset":"45","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.0636675,"name":"drain","context":{"idset":"47","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.080302,"name":"drain","context":{"idset":"23","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.0972147,"name":"drain","context":{"idset":"37","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.1138506,"name":"drain","context":{"idset":"22","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.1304362,"name":"drain","context":{"idset":"44","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.1470764,"name":"drain","context":{"idset":"35","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.163903,"name":"drain","context":{"idset":"58","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.180536,"name":"drain","context":{"idset":"57","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.1972005,"name":"drain","context":{"idset":"59","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.2140071,"name":"drain","context":{"idset":"60","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.2306161,"name":"drain","context":{"idset":"14","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.247216,"name":"drain","context":{"idset":"33","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.2642868,"name":"drain","context":{"idset":"41","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.2809656,"name":"drain","context":{"idset":"24","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704482086.2979519,"name":"drain","context":{"idset":"21","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1704755320.57498,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1704755320.575089,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1704755320.575165,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1704755320.5752442,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1704755320.57533,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1704755320.5754123,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1704755320.5754962,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1704755320.5755742,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1704755320.5756595,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1704755320.5757427,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1704755320.5758214,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1704755320.5758958,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1704755320.5759702,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1704755320.576062,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1704755320.5761473,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1704755320.5762384,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1704755320.5763261,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1704755320.5764041,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1704755320.5764797,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1704755320.57656,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1704755320.5766404,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1704755320.5767269,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1704755320.5768099,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1704755320.5769334,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1704755320.6757951,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1704755322.5745127,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1704755322.5746176,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1704755322.5746839,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1704755322.5747488,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1704755322.5748286,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1704755322.5749142,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1704755322.5749953,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1704755322.5750716,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1704755322.5751474,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1704755322.5752146,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1704755322.5752842,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1704755322.5753541,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1704755322.5754166,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1704755322.5754783,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1704755322.5755379,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1704755322.575597,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1704755322.5756531,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1704755322.5757074,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1704755322.5757663,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1704755322.5758209,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1704755322.575877,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1704755322.5759313,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1704755322.5759764,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1704755322.5760252,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1704755322.5760667,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1704755322.5761161,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1704755322.5761659,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1704755322.5762138,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1704755322.5762606,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1704755322.5763047,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1704755322.5763457,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1704755322.576386,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1704755322.576427,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1704755322.6751835,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1704756853.636205,"name":"online","context":{"idset":"18"}}
+{"timestamp":1704756854.1328704,"name":"online","context":{"idset":"3,11,14,29-30,40,51"}}
+{"timestamp":1704756854.2546701,"name":"online","context":{"idset":"5-6,12,17,19-20,22,35,39,43-44,46,55"}}
+{"timestamp":1704756854.3672984,"name":"online","context":{"idset":"1-2,4,7,9-10,15-16,25-28,32-33,36,41,56"}}
+{"timestamp":1704756854.4737909,"name":"online","context":{"idset":"13,24,31,34,37-38,45,48-50,52-54,57"}}
+{"timestamp":1704756854.5750644,"name":"online","context":{"idset":"8,21,23,47,58,60"}}
+{"timestamp":1704756854.6753678,"name":"online","context":{"idset":"42,59"}}
+{"timestamp":1704826273.9489272,"name":"online","context":{"idset":"710"}}
+{"timestamp":1704826297.8485823,"name":"online","context":{"idset":"709"}}
+{"timestamp":1704826300.3398557,"name":"online","context":{"idset":"711,715-716,718-719,721,723,725,727,733-735,739"}}
+{"timestamp":1704826300.4418426,"name":"online","context":{"idset":"712-714,717,720,722,726,728-729,731,736,740-741,743-746,748-749,751,754-755"}}
+{"timestamp":1704826300.5437682,"name":"online","context":{"idset":"724,730,732,737,742,747,750,752-753,756"}}
+{"timestamp":1704827015.2090194,"name":"undrain","context":{"idset":"1,3-60"}}
+{"timestamp":1704834931.0490909,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1704835907.9274664,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1704835907.9291532,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1704835907.9325352,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1704835907.9537339,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1704835907.9556909,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1704835907.9561911,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1704835907.9607894,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1704835907.9779103,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1704835907.9814024,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1704835907.9864681,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1704835907.9894342,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1704835907.9920545,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1704835907.9921353,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1704835907.9923806,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1704835907.9957952,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1704835907.9973054,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1704835907.9978278,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1704835907.9996369,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1704835908.0032024,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1704835908.0040796,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1704835908.0083153,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1704835908.0100045,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1704835908.0114131,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1704835908.0123899,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1704835908.0128503,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1704835908.015306,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1704835908.0178115,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1704835908.0185971,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1704835908.0243266,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1704835908.0472445,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1704835908.0473208,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1704835908.0507834,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1704835908.051317,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1704835908.0534179,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1704835908.0545251,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1704835908.0562801,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1704835908.0571477,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1704835908.0574472,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1704835908.059335,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1704835908.0611696,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1704835908.0741749,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1704835908.0760674,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1704835908.0893631,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1704835908.1020093,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1704835908.1075284,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1704835908.2077057,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1704906045.3939471,"name":"drain","context":{"idset":"37","reason":"reason=SS C1_HNI error: rx_pause_err","overwrite":0}}
+{"timestamp":1704917882.1253517,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1704924287.8003252,"name":"online","context":{"idset":"756"}}
+{"timestamp":1704925741.562922,"name":"online","context":{"idset":"755"}}
+{"timestamp":1704925882.9095814,"name":"online","context":{"idset":"710-711,713-719,721-726,731,733,735,740"}}
+{"timestamp":1704925883.0140843,"name":"online","context":{"idset":"709,712,720,727-730,732,734,737,739,741,743-745,747-749,751-754"}}
+{"timestamp":1704925883.1192913,"name":"online","context":{"idset":"736,742,746,750"}}
+{"timestamp":1704926003.8013618,"name":"drain","context":{"idset":"61-708","reason":"reason=HPE bringup","overwrite":0}}
+{"timestamp":1704926012.6552303,"name":"drain","context":{"idset":"738","reason":"reason=HPE bringup","overwrite":0}}
+{"timestamp":1704928829.6370149,"name":"online","context":{"idset":"37"}}
+{"timestamp":1704929025.1443765,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1705017446.6759231,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1705017458.6750081,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1705017462.6752374,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1705017474.6751335,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1705017480.6746757,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1705017480.7851641,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1705017488.5748365,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1705017488.6756229,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1705017492.5751686,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1705017492.6752255,"name":"drain","context":{"idset":"729","reason":"broker was unresponsive"}}
+{"timestamp":1705017494.6749277,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1705017498.5758386,"name":"drain","context":{"idset":"718","reason":"broker was unresponsive"}}
+{"timestamp":1705017498.5931041,"name":"drain","context":{"idset":"727","reason":"broker was unresponsive"}}
+{"timestamp":1705017498.6931512,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1705017500.6749334,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1705017504.675535,"name":"drain","context":{"idset":"713","reason":"broker was unresponsive"}}
+{"timestamp":1705017506.6739464,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1705017512.5749919,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1705017512.6756539,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1705017514.6760049,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1705017524.5756524,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1705017524.675637,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1705017536.6759646,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1705017548.6759672,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1705017554.6764162,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1705017556.6759226,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1705017558.5749025,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1705017558.6747847,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1705017562.5762601,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1705017562.6764796,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1705017566.675837,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1705017570.675107,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1705017574.6761389,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1705017576.574836,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1705017576.6753488,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1705017598.6756439,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1705017600.6759186,"name":"drain","context":{"idset":"739","reason":"broker was unresponsive"}}
+{"timestamp":1705017604.675987,"name":"drain","context":{"idset":"716","reason":"broker was unresponsive"}}
+{"timestamp":1705017612.6759534,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1705017628.5760996,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1705017628.5762436,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1705017628.6763253,"name":"drain","context":{"idset":"710","reason":"broker was unresponsive"}}
+{"timestamp":1705017638.6762776,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1705017640.6762211,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1705017648.6763198,"name":"drain","context":{"idset":"709","reason":"broker was unresponsive"}}
+{"timestamp":1705017650.6761403,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1705017664.67609,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1705017672.6756964,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1705017676.6764586,"name":"drain","context":{"idset":"711","reason":"broker was unresponsive"}}
+{"timestamp":1705017680.5754552,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1705017680.6759176,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1705017690.6756763,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1705017690.7887695,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1705017692.5759456,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1705017692.6769145,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1705017696.6760368,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1705017700.6748853,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1705017702.6755199,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1705017704.575922,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1705017704.6761985,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1705017712.6747065,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1705017714.6749053,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1705017740.5748882,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1705017740.5749662,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1705017740.6754415,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1705017746.5748158,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1705017746.6748559,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1705017752.6758385,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1705017756.6750307,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1705017768.6760221,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1705017802.6751018,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1705017806.6744292,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1705017818.676028,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1705017854.6762333,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1705017864.6754079,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1705017870.5761161,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1705017870.6766601,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1705017890.6766086,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1705017896.6765947,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1705017902.5753253,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1705017902.6763079,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1705017908.6758463,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1705017920.6740592,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1705017938.5755124,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1705017938.6761513,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1705017944.6757631,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1705017956.6757228,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1705017960.6758349,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1705017964.5763743,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1705017964.6769261,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1705017968.6757383,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1705017974.6745658,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1705018008.675072,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1705018030.6762989,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1705018044.6765358,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1705018096.676461,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1705018104.675703,"name":"drain","context":{"idset":"719","reason":"broker was unresponsive"}}
+{"timestamp":1705018108.6760099,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1705018162.6761608,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1705018170.6756227,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1705018190.6757319,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1705018196.6765161,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1705018214.6753194,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1705018226.6761672,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1705018232.6757104,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1705018262.675663,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1705018276.6754887,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1705018292.6753595,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1705018294.6748302,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1705018394.67541,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1705018422.6756959,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1705018460.6756148,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1705018994.6761067,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1705021328.6753986,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1705021422.6762245,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1705021472.6763046,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1705021594.6761377,"name":"drain","context":{"idset":"747","reason":"broker was unresponsive"}}
+{"timestamp":1705021620.6766696,"name":"drain","context":{"idset":"744","reason":"broker was unresponsive"}}
+{"timestamp":1705021658.6753373,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1705021682.6759512,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1705024490.5748892,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1705024490.5749855,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1705024490.6750531,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.5760162,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.5760908,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.5761406,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.5761683,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.5761936,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.5762181,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.5762439,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.576268,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.5762985,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.5763333,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.5763633,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1705024492.6766274,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1705024494.574821,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1705024494.5749078,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1705024494.5749457,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1705024494.5749996,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1705024494.5750341,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1705024494.575069,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1705024494.575105,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1705024494.6753578,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1705024496.6759758,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1705024556.575254,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1705024556.5753713,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1705024556.5754313,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1705024556.5754874,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1705024556.5755455,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1705024556.5756414,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1705024556.5757143,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1705024556.5758212,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1705024556.5758986,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1705024556.5759759,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1705024556.5760639,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1705024556.576148,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1705024556.5762258,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1705024556.5762906,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1705024556.5763619,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1705024556.5764544,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1705024556.5765274,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1705024556.576592,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1705024556.5766513,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1705024556.5767093,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1705024556.5767713,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1705024556.5768282,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1705024556.6751828,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1705024558.6759672,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1705025728.5897624,"name":"online","context":{"idset":"44"}}
+{"timestamp":1705025728.8355169,"name":"online","context":{"idset":"20,25,33,51,53"}}
+{"timestamp":1705025728.9480767,"name":"online","context":{"idset":"2,6,58,60"}}
+{"timestamp":1705025729.0612803,"name":"online","context":{"idset":"4-5,10,13-14,29,34,41,43"}}
+{"timestamp":1705025729.1631141,"name":"online","context":{"idset":"3,7,9,11-12,15,17-19,21,23-24,27-28,30,36,38,46"}}
+{"timestamp":1705025729.2654054,"name":"online","context":{"idset":"1,8,26,35,37,39,50,52,59"}}
+{"timestamp":1705025729.3748593,"name":"online","context":{"idset":"22,31,40,42,48-49,55-57"}}
+{"timestamp":1705025729.4840903,"name":"online","context":{"idset":"32,45"}}
+{"timestamp":1705025729.6629412,"name":"online","context":{"idset":"47,54"}}
+{"timestamp":1705025772.3982029,"name":"undrain","context":{"idset":"1-60"}}
+{"timestamp":1705026098.9045706,"name":"online","context":{"idset":"16"}}
+{"timestamp":1705077771.1198332,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1705078000.8998497,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1705080867.5213161,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1705080876.3817363,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1705080885.9222879,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1705080896.7852771,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1705080907.4389951,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1705080915.1543405,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1705080925.4540291,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1705080933.052628,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1705080941.1974096,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1705095216.6761749,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1705095228.6750743,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1705095232.677191,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1705095242.6749921,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1705095246.6768277,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1705095248.6756043,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1705095252.67555,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1705095254.5747843,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1705095254.674943,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1705095258.5751638,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1705095258.675802,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1705095264.5748646,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1705095264.6754739,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1705095266.57516,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1705095266.6758931,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1705095272.6751902,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1705095276.5750632,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1705095276.5751865,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1705095276.5752339,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1705095276.6753919,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1705095278.5750849,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1705095278.6760445,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1705095280.6748962,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1705095288.5758035,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1705095288.5759156,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1705095288.5759633,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1705095288.6762767,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1705095290.5752945,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1705095290.6758835,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1705095296.5753639,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1705095296.6753967,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1705095304.675776,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1705095306.6757205,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1705095308.5754702,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1705095308.6758232,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1705095310.675539,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1705095312.6741419,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1705095318.5755205,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1705095318.6757519,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1705095320.5746217,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1705095320.6754422,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1705095324.6755939,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1705095330.5748944,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1705095330.5751011,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1705095330.6754014,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1705095332.6749942,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1705095336.6758385,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1705095338.6749289,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1705095340.6754367,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1705095342.5752225,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1705095342.5753117,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1705095342.5753651,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1705095342.6757643,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1705095350.5754945,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1705095350.6761851,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1705095352.5751486,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1705095352.6755896,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1705095356.6758933,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1705095358.6747274,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1705095368.6760352,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1705095372.5748742,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1705095372.6758523,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1705095374.6750059,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1705095378.6757524,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1705095396.6763568,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1705095402.6752214,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1705095434.6758745,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1705095438.676538,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1705095460.6764808,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1705095464.6758595,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1705095564.676224,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1705095582.6765313,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1705095630.6760581,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1705095634.6765501,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1705095650.6761003,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1705095700.675915,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1705095732.6751482,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1705095794.6760678,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1705099514.0838149,"name":"resource-init","context":{"restart":true,"drain":{"13":{"timestamp":1705095368.6760352,"reason":"broker was unresponsive"},"14":{"timestamp":1705095288.5758035,"reason":"broker was unresponsive"},"15":{"timestamp":1705095306.6757205,"reason":"broker was unresponsive"},"16":{"timestamp":1705095732.6751482,"reason":"broker was unresponsive"},"17":{"timestamp":1705095242.6749921,"reason":"broker was unresponsive"},"18":{"timestamp":1705095276.5750632,"reason":"broker was unresponsive"},"19":{"timestamp":1705095258.5751638,"reason":"broker was unresponsive"},"20":{"timestamp":1705095288.5759156,"reason":"broker was unresponsive"},"21":{"timestamp":1705095266.57516,"reason":"broker was unresponsive"},"22":{"timestamp":1705095232.677191,"reason":"broker was unresponsive"},"23":{"timestamp":1705095254.5747843,"reason":"broker was unresponsive"},"24":{"timestamp":1705095252.67555,"reason":"broker was unresponsive"},"25":{"timestamp":1705095228.6750743,"reason":"broker was unresponsive"},"26":{"timestamp":1705095372.6758523,"reason":"broker was unresponsive"},"27":{"timestamp":1705095288.5759633,"reason":"broker was unresponsive"},"29":{"timestamp":1705095276.5751865,"reason":"broker was unresponsive"},"30":{"timestamp":1705095308.6758232,"reason":"broker was unresponsive"},"31":{"timestamp":1705095288.6762767,"reason":"broker was unresponsive"},"32":{"timestamp":1705095246.6768277,"reason":"broker was unresponsive"},"33":{"timestamp":1705095312.6741419,"reason":"broker was unresponsive"},"34":{"timestamp":1705095216.6761749,"reason":"broker was unresponsive"},"36":{"timestamp":1705095278.5750849,"reason":"broker was unresponsive"},"37":{"timestamp":1705095396.6763568,"reason":"broker was unresponsive"},"38":{"timestamp":1705095582.6765313,"reason":"broker was unresponsive"},"41":{"timestamp":1705095272.6751902,"reason":"broker was unresponsive"},"44":{"timestamp":1705095278.6760445,"reason":"broker was unresponsive"},"45":{"timestamp":1705095276.5752339,"reason":"broker was unresponsive"},"46":{"timestamp":1705095290.6758835,"reason":"broker was unresponsive"},"47":{"timestamp":1705095634.6765501,"reason":"broker was unresponsive"},"50":{"timestamp":1705095258.675802,"reason":"broker was unresponsive"},"51":{"timestamp":1705095264.5748646,"reason":"broker was unresponsive"},"52":{"timestamp":1705095402.6752214,"reason":"broker was unresponsive"},"53":{"timestamp":1705095264.6754739,"reason":"broker was unresponsive"},"55":{"timestamp":1705095276.6753919,"reason":"broker was unresponsive"},"56":{"timestamp":1705095254.674943,"reason":"broker was unresponsive"},"57":{"timestamp":1705095296.6753967,"reason":"broker was unresponsive"},"58":{"timestamp":1705095266.6758931,"reason":"broker was unresponsive"},"59":{"timestamp":1705095248.6756043,"reason":"broker was unresponsive"},"60":{"timestamp":1705095564.676224,"reason":"broker was unresponsive"},"61-708":{"timestamp":1704926003.8013618,"reason":"reason=HPE bringup"},"709":{"timestamp":1705017648.6763198,"reason":"broker was unresponsive"},"710":{"timestamp":1705017628.6763253,"reason":"broker was unresponsive"},"711":{"timestamp":1705017676.6764586,"reason":"broker was unresponsive"},"713":{"timestamp":1705017504.675535,"reason":"broker was unresponsive"},"716":{"timestamp":1705017604.675987,"reason":"broker was unresponsive"},"718":{"timestamp":1705017498.5758386,"reason":"broker was unresponsive"},"719":{"timestamp":1705018104.675703,"reason":"broker was unresponsive"},"727":{"timestamp":1705017498.5931041,"reason":"broker was unresponsive"},"729":{"timestamp":1705017492.6752255,"reason":"broker was unresponsive"},"738":{"timestamp":1704926012.6552303,"reason":"reason=HPE bringup"},"739":{"timestamp":1705017600.6759186,"reason":"broker was unresponsive"},"744":{"timestamp":1705021620.6766696,"reason":"broker was unresponsive"},"747":{"timestamp":1705021594.6761377,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1705099514.101321,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1705099528.5185101,"name":"online","context":{"idset":"0"}}
+{"timestamp":1705099529.5864952,"name":"online","context":{"idset":"1-12,28,35,39-40,42-43,48-49,54"}}
+{"timestamp":1705708981.7377601,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7378395,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7378738,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7379045,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7379327,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7379601,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.737987,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7380157,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7380438,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7380719,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7381008,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7381301,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7381589,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7381899,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7382216,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.73826,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7382932,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7383373,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7383757,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.7384131,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1705708981.838336,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1705709045.7350945,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1705709045.7351863,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1705709045.7352407,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1705709045.7352893,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1705709045.7353368,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1705709045.7353828,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1705709045.7354271,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1705709045.7354689,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1705709045.7355113,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1705709045.7355607,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1705709045.7356246,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1705709045.7356725,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1705709045.7357244,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1705709045.7357681,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1705709045.7358162,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1705709045.7358694,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1705709045.7359076,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1705709045.7359543,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1705709045.735991,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1705709045.8355639,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1705709047.8372617,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1705718415.7039151,"name":"online","context":{"idset":"10,28,43"}}
+{"timestamp":1705718415.837203,"name":"online","context":{"idset":"5,11-13,23,27,41,44"}}
+{"timestamp":1705718415.9736035,"name":"online","context":{"idset":"1,4,9,14,18-19,21,33,39,49,57,60"}}
+{"timestamp":1705718416.0984428,"name":"online","context":{"idset":"7,15,17,20,30-32,53"}}
+{"timestamp":1705718416.1993339,"name":"online","context":{"idset":"3,6,8,16,22,24-25,29,34,36,46,48,50-51,56,59"}}
+{"timestamp":1705718416.3039372,"name":"online","context":{"idset":"2,26,42,45,47,52,55,58"}}
+{"timestamp":1705718416.4264386,"name":"online","context":{"idset":"37-38"}}
+{"timestamp":1705718416.6568162,"name":"online","context":{"idset":"54"}}
+{"timestamp":1705718567.2676594,"name":"undrain","context":{"idset":"1-60"}}
+{"timestamp":1705811366.8521595,"name":"online","context":{"idset":"40"}}
+{"timestamp":1705949761.6847224,"name":"drain","context":{"idset":"35","reason":"reason=check_ama: LLDPAD service is inactive","overwrite":0}}
+{"timestamp":1706024721.1267467,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1706024721.2223833,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1706024721.2681358,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1706024721.2809312,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1706024721.2890251,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1706024721.2894084,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1706024721.3410416,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1706024721.3411493,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1706024721.3421907,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1706024721.3432367,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1706024721.3511908,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1706024721.3600569,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1706024721.3809516,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1706024721.3818216,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1706024721.3841672,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1706024721.3916237,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1706024721.3930819,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1706024721.398113,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1706024721.4009998,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1706024721.4030788,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1706024721.4091637,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1706024721.4185195,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1706024721.4224288,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1706024721.4370201,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1706024721.4437191,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1706024721.4477308,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1706024721.4479189,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1706024721.4480865,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1706024721.4632986,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1706024721.4667969,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1706024721.4678719,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1706024721.4732254,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1706024721.473768,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1706024721.4803169,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1706024721.4867558,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1706024721.530694,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1706024721.5375223,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1706024721.5467243,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1706024721.5537977,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1706024721.5572298,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1706024721.5629666,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1706024721.5644326,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1706024721.5646019,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1706024721.5658066,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1706024721.5666809,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1706024721.5687716,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1706024721.5718224,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1706024721.5739574,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1706024721.5837247,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1706024721.5840828,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1706024721.6043937,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1706024721.6058564,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1706024721.6172333,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1706024721.6233242,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1706024721.6584685,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1706024721.6641457,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1706024721.6812167,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1706024721.7028055,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1706024721.802942,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1706040798.3147728,"name":"resource-init","context":{"restart":true,"drain":{"35":{"timestamp":1705949761.6847224,"reason":"reason=check_ama: LLDPAD service is inactive"},"61-708":{"timestamp":1704926003.8013618,"reason":"reason=HPE bringup"},"709":{"timestamp":1705017648.6763198,"reason":"broker was unresponsive"},"710":{"timestamp":1705017628.6763253,"reason":"broker was unresponsive"},"711":{"timestamp":1705017676.6764586,"reason":"broker was unresponsive"},"713":{"timestamp":1705017504.675535,"reason":"broker was unresponsive"},"716":{"timestamp":1705017604.675987,"reason":"broker was unresponsive"},"718":{"timestamp":1705017498.5758386,"reason":"broker was unresponsive"},"719":{"timestamp":1705018104.675703,"reason":"broker was unresponsive"},"727":{"timestamp":1705017498.5931041,"reason":"broker was unresponsive"},"729":{"timestamp":1705017492.6752255,"reason":"broker was unresponsive"},"738":{"timestamp":1704926012.6552303,"reason":"reason=HPE bringup"},"739":{"timestamp":1705017600.6759186,"reason":"broker was unresponsive"},"744":{"timestamp":1705021620.6766696,"reason":"broker was unresponsive"},"747":{"timestamp":1705021594.6761377,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1706040798.329607,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1706040817.4601207,"name":"online","context":{"idset":"0"}}
+{"timestamp":1706132222.6106372,"name":"online","context":{"idset":"712"}}
+{"timestamp":1706132259.0130649,"name":"online","context":{"idset":"710-711,714-715,725,729"}}
+{"timestamp":1706132259.1223428,"name":"online","context":{"idset":"709,716,718,720-721,723-724,727-728,734,749"}}
+{"timestamp":1706132259.2280111,"name":"online","context":{"idset":"717,726,730-733,737,740,742,748,750,753,755"}}
+{"timestamp":1706132259.3393414,"name":"online","context":{"idset":"752"}}
+{"timestamp":1706132259.6839769,"name":"online","context":{"idset":"739"}}
+{"timestamp":1706132261.5921195,"name":"online","context":{"idset":"713,722"}}
+{"timestamp":1706132261.7054143,"name":"online","context":{"idset":"719,735,738,745-747"}}
+{"timestamp":1706132261.8159926,"name":"online","context":{"idset":"741,743-744,751,756"}}
+{"timestamp":1706132261.9174969,"name":"online","context":{"idset":"736"}}
+{"timestamp":1706132262.0481665,"name":"online","context":{"idset":"754"}}
+{"timestamp":1706138935.4820075,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1706142570.5544922,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1706142571.0727134,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1706142571.4991601,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1706142571.6617584,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1706142571.7277,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1706142571.7284405,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1706142571.7517195,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1706142571.8013351,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1706142571.8458302,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1706142571.9465101,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1706142572.046196,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1706142572.0774002,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1706142572.1036854,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1706142572.1118062,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1706142572.1174104,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1706142572.1940811,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1706142572.2387488,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1706142572.2544022,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1706142572.306848,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1706142572.4076548,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1706142572.5305843,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1706142572.5759246,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1706142572.6765325,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1706142572.7412307,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1706142572.7430913,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1706142572.7729647,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1706142572.8396194,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1706142572.9341168,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1706142573.003211,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1706142573.0226102,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1706142573.0242877,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1706142573.0740745,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1706142573.0907538,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1706142573.1322756,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1706142573.2323885,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1706142573.2471185,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1706142573.3479812,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1706142573.3979189,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1706142573.4201815,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1706142573.5203173,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1706142573.7232921,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1706142573.7659931,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1706142573.866432,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1706142574.3485856,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1706142574.5956767,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1706142574.8904541,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1706142576.1753049,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1706146172.6573272,"name":"online","context":{"idset":"395"}}
+{"timestamp":1706146173.4322565,"name":"online","context":{"idset":"75"}}
+{"timestamp":1706146173.5596235,"name":"online","context":{"idset":"69-73,101-103,106,359,368,377"}}
+{"timestamp":1706146173.6044736,"name":"online","context":{"idset":"105,107-108,357-358,360-361,367,371-373,378,392"}}
+{"timestamp":1706146173.7055726,"name":"online","context":{"idset":"363-364,366,369-370,374-376,380,389,394,408,410-411,413,418-420,477"}}
+{"timestamp":1706146173.7702787,"name":"online","context":{"idset":"104,379,393,396,405,409,414,416,478-479,481,484-486,489,713,719-720"}}
+{"timestamp":1706146173.858458,"name":"online","context":{"idset":"76,400,406,417,480,482,487-488,492,717,721,723-724"}}
+{"timestamp":1706146173.9278524,"name":"online","context":{"idset":"403,491"}}
+{"timestamp":1706146174.0449703,"name":"online","context":{"idset":"391,407,412,718,722"}}
+{"timestamp":1706146174.2037151,"name":"online","context":{"idset":"74,390,483"}}
+{"timestamp":1706146174.2358623,"name":"online","context":{"idset":"490"}}
+{"timestamp":1706146174.4196961,"name":"online","context":{"idset":"471"}}
+{"timestamp":1706146174.5246298,"name":"online","context":{"idset":"715"}}
+{"timestamp":1706146174.80724,"name":"online","context":{"idset":"711,732"}}
+{"timestamp":1706146175.1025009,"name":"online","context":{"idset":"399,401"}}
+{"timestamp":1706146175.3326974,"name":"online","context":{"idset":"469,730"}}
+{"timestamp":1706146175.419322,"name":"online","context":{"idset":"470"}}
+{"timestamp":1706146175.6117227,"name":"online","context":{"idset":"716"}}
+{"timestamp":1706146175.7893548,"name":"online","context":{"idset":"402,731"}}
+{"timestamp":1706146176.0843384,"name":"online","context":{"idset":"397"}}
+{"timestamp":1706146176.256737,"name":"online","context":{"idset":"476,710"}}
+{"timestamp":1706146176.3778253,"name":"online","context":{"idset":"712,725"}}
+{"timestamp":1706146176.6683273,"name":"online","context":{"idset":"404,472"}}
+{"timestamp":1706146176.9805307,"name":"online","context":{"idset":"729"}}
+{"timestamp":1706146177.2267635,"name":"online","context":{"idset":"474"}}
+{"timestamp":1706146177.3495469,"name":"online","context":{"idset":"398"}}
+{"timestamp":1706146177.8177044,"name":"online","context":{"idset":"726"}}
+{"timestamp":1706146178.0012803,"name":"online","context":{"idset":"709"}}
+{"timestamp":1706146178.1335738,"name":"online","context":{"idset":"714"}}
+{"timestamp":1706146178.9839725,"name":"online","context":{"idset":"473,728"}}
+{"timestamp":1706146179.0911121,"name":"online","context":{"idset":"727"}}
+{"timestamp":1706146180.7218108,"name":"online","context":{"idset":"475,733"}}
+{"timestamp":1706146182.2951579,"name":"online","context":{"idset":"734,737"}}
+{"timestamp":1706146182.9249663,"name":"online","context":{"idset":"735"}}
+{"timestamp":1706146183.0964458,"name":"online","context":{"idset":"739-740,744,747,752,754"}}
+{"timestamp":1706146183.2050965,"name":"online","context":{"idset":"743,745,756"}}
+{"timestamp":1706146183.4189284,"name":"online","context":{"idset":"742,751,753"}}
+{"timestamp":1706146183.6118441,"name":"online","context":{"idset":"749"}}
+{"timestamp":1706146184.0418956,"name":"online","context":{"idset":"746"}}
+{"timestamp":1706146184.440275,"name":"online","context":{"idset":"736,738,741,748,750,755"}}
+{"timestamp":1706146208.7195952,"name":"online","context":{"idset":"415"}}
+{"timestamp":1706146239.7533765,"name":"online","context":{"idset":"100"}}
+{"timestamp":1706146263.6598008,"name":"online","context":{"idset":"96"}}
+{"timestamp":1706146266.2796507,"name":"online","context":{"idset":"95,362"}}
+{"timestamp":1706146290.5791457,"name":"online","context":{"idset":"94"}}
+{"timestamp":1706146333.3916779,"name":"online","context":{"idset":"97"}}
+{"timestamp":1706146354.9041865,"name":"online","context":{"idset":"93"}}
+{"timestamp":1706146462.240207,"name":"online","context":{"idset":"98"}}
+{"timestamp":1706204616.9514954,"name":"online","context":{"idset":"8,11"}}
+{"timestamp":1706204617.1116455,"name":"online","context":{"idset":"24,31,41"}}
+{"timestamp":1706204617.2640483,"name":"online","context":{"idset":"5,7,26"}}
+{"timestamp":1706204617.3838718,"name":"online","context":{"idset":"1,3-4,13,16,25,28,32,36"}}
+{"timestamp":1706204617.4894404,"name":"online","context":{"idset":"2,9,12,14-15,19,22,29-30,33-34,45,47,51-52,54-55"}}
+{"timestamp":1706204617.5949595,"name":"online","context":{"idset":"17,20,27,35,37-38,40,43-44"}}
+{"timestamp":1706204617.702302,"name":"online","context":{"idset":"6,18,21,23,39,42,46,48-49,53,56,60"}}
+{"timestamp":1706204617.8033741,"name":"online","context":{"idset":"10,50,57-59"}}
+{"timestamp":1706204763.6528177,"name":"undrain","context":{"idset":"35"}}
+{"timestamp":1706205058.9322414,"name":"online","context":{"idset":"99"}}
+{"timestamp":1706205137.4422255,"name":"undrain","context":{"idset":"711"}}
+{"timestamp":1706205147.8731992,"name":"undrain","context":{"idset":"713,718,727,729"}}
+{"timestamp":1706205151.855511,"name":"undrain","context":{"idset":"719"}}
+{"timestamp":1706205155.8803651,"name":"undrain","context":{"idset":"744"}}
+{"timestamp":1706205157.9443424,"name":"undrain","context":{"idset":"747"}}
+{"timestamp":1706205172.3329864,"name":"undrain","context":{"idset":"709-710,716,739"}}
+{"timestamp":1706205196.6505814,"name":"undrain","context":{"idset":"61-68,77-92,109-356,365,381-388,421-468,493-708"}}
+{"timestamp":1706205230.0179095,"name":"undrain","context":{"idset":"69-76,93-108,357-364,366-380,389-420,469-492,738"}}
+{"timestamp":1706205251.7074873,"name":"drain","context":{"idset":"61-756","reason":"reason=HPE bringup","overwrite":0}}
+{"timestamp":1706205277.9937892,"name":"undrain","context":{"idset":"69-76,93-108,357-380,389-420,469-492,709-756"}}
+{"timestamp":1706205362.428031,"name":"drain","context":{"idset":"741","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.4553845,"name":"drain","context":{"idset":"743","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.4838495,"name":"drain","context":{"idset":"744","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.5248091,"name":"drain","context":{"idset":"72","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.5543137,"name":"drain","context":{"idset":"746","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.5865822,"name":"drain","context":{"idset":"745","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.6239374,"name":"drain","context":{"idset":"380","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.6478071,"name":"drain","context":{"idset":"747","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.6734903,"name":"drain","context":{"idset":"740","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.6988769,"name":"drain","context":{"idset":"379","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.7216938,"name":"drain","context":{"idset":"754","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.7471752,"name":"drain","context":{"idset":"392","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.768873,"name":"drain","context":{"idset":"376","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.7903748,"name":"drain","context":{"idset":"378","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.8178535,"name":"drain","context":{"idset":"389","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.8607099,"name":"drain","context":{"idset":"377","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.904928,"name":"drain","context":{"idset":"69","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.9542472,"name":"drain","context":{"idset":"70","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205362.9987097,"name":"drain","context":{"idset":"406","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.0383599,"name":"drain","context":{"idset":"71","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.0774801,"name":"drain","context":{"idset":"390","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.113034,"name":"drain","context":{"idset":"103","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.1551507,"name":"drain","context":{"idset":"395","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.193785,"name":"drain","context":{"idset":"391","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.2293773,"name":"drain","context":{"idset":"742","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.2680786,"name":"drain","context":{"idset":"393","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.3067448,"name":"drain","context":{"idset":"74","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.3466563,"name":"drain","context":{"idset":"73","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.3850162,"name":"drain","context":{"idset":"748","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.4183147,"name":"drain","context":{"idset":"753","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.4444306,"name":"drain","context":{"idset":"357","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.4705276,"name":"drain","context":{"idset":"75","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.4969451,"name":"drain","context":{"idset":"397","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.5217409,"name":"drain","context":{"idset":"394","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.5466008,"name":"drain","context":{"idset":"396","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.5711994,"name":"drain","context":{"idset":"398","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.5954568,"name":"drain","context":{"idset":"358","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.6205366,"name":"drain","context":{"idset":"76","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.6457469,"name":"drain","context":{"idset":"749","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.6712556,"name":"drain","context":{"idset":"752","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.6967711,"name":"drain","context":{"idset":"93","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.7225091,"name":"drain","context":{"idset":"415","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.7468061,"name":"drain","context":{"idset":"750","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.7710493,"name":"drain","context":{"idset":"416","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.795382,"name":"drain","context":{"idset":"101","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.8199377,"name":"drain","context":{"idset":"755","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.8441739,"name":"drain","context":{"idset":"399","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.8686147,"name":"drain","context":{"idset":"401","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.8987532,"name":"drain","context":{"idset":"751","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.9230468,"name":"drain","context":{"idset":"403","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.9469554,"name":"drain","context":{"idset":"400","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.9713709,"name":"drain","context":{"idset":"404","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205363.9961383,"name":"drain","context":{"idset":"405","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.0201373,"name":"drain","context":{"idset":"402","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.0441742,"name":"drain","context":{"idset":"102","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.0681727,"name":"drain","context":{"idset":"418","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.0924718,"name":"drain","context":{"idset":"94","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.1171212,"name":"drain","context":{"idset":"368","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.1419959,"name":"drain","context":{"idset":"362","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.1794736,"name":"drain","context":{"idset":"104","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.2227733,"name":"drain","context":{"idset":"105","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.2570281,"name":"drain","context":{"idset":"99","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.2859566,"name":"drain","context":{"idset":"408","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.3217423,"name":"drain","context":{"idset":"420","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.345433,"name":"drain","context":{"idset":"97","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.3693838,"name":"drain","context":{"idset":"106","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.3936322,"name":"drain","context":{"idset":"95","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.4176202,"name":"drain","context":{"idset":"407","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.4489963,"name":"drain","context":{"idset":"471","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.4899087,"name":"drain","context":{"idset":"98","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.5359669,"name":"drain","context":{"idset":"477","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.5609074,"name":"drain","context":{"idset":"414","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.5851159,"name":"drain","context":{"idset":"411","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.6097398,"name":"drain","context":{"idset":"108","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.6341162,"name":"drain","context":{"idset":"417","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.6586025,"name":"drain","context":{"idset":"360","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.684814,"name":"drain","context":{"idset":"359","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.7092671,"name":"drain","context":{"idset":"107","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.7335019,"name":"drain","context":{"idset":"96","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.7583413,"name":"drain","context":{"idset":"364","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.7828598,"name":"drain","context":{"idset":"412","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.8074801,"name":"drain","context":{"idset":"369","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.8315585,"name":"drain","context":{"idset":"410","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.8559346,"name":"drain","context":{"idset":"100","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.87989,"name":"drain","context":{"idset":"409","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.9051805,"name":"drain","context":{"idset":"363","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.9296505,"name":"drain","context":{"idset":"361","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.9535868,"name":"drain","context":{"idset":"478","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205364.9775391,"name":"drain","context":{"idset":"366","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.0014882,"name":"drain","context":{"idset":"367","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.0253353,"name":"drain","context":{"idset":"370","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.0496755,"name":"drain","context":{"idset":"488","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.0734091,"name":"drain","context":{"idset":"473","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.0973029,"name":"drain","context":{"idset":"485","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.1209943,"name":"drain","context":{"idset":"482","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.1448839,"name":"drain","context":{"idset":"469","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.1696718,"name":"drain","context":{"idset":"719","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.1937826,"name":"drain","context":{"idset":"372","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.2181759,"name":"drain","context":{"idset":"713","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.2421145,"name":"drain","context":{"idset":"481","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.2660289,"name":"drain","context":{"idset":"374","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.2899621,"name":"drain","context":{"idset":"475","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.313905,"name":"drain","context":{"idset":"375","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.3376992,"name":"drain","context":{"idset":"373","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.3615277,"name":"drain","context":{"idset":"472","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.3858888,"name":"drain","context":{"idset":"484","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.4096894,"name":"drain","context":{"idset":"470","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.4335845,"name":"drain","context":{"idset":"486","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.4573812,"name":"drain","context":{"idset":"480","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.4812083,"name":"drain","context":{"idset":"489","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.5050375,"name":"drain","context":{"idset":"487","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.5294969,"name":"drain","context":{"idset":"736","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.5546629,"name":"drain","context":{"idset":"738","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.5785758,"name":"drain","context":{"idset":"492","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.6023579,"name":"drain","context":{"idset":"735","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.626209,"name":"drain","context":{"idset":"490","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.6499965,"name":"drain","context":{"idset":"479","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.6743731,"name":"drain","context":{"idset":"709","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.702338,"name":"drain","context":{"idset":"491","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.7322333,"name":"drain","context":{"idset":"474","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.7561345,"name":"drain","context":{"idset":"710","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.7804842,"name":"drain","context":{"idset":"711","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.8043554,"name":"drain","context":{"idset":"718","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.8281562,"name":"drain","context":{"idset":"483","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.8519673,"name":"drain","context":{"idset":"371","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.8767791,"name":"drain","context":{"idset":"715","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.9009514,"name":"drain","context":{"idset":"714","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.9250877,"name":"drain","context":{"idset":"723","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.9491503,"name":"drain","context":{"idset":"419","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.973213,"name":"drain","context":{"idset":"720","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205365.9936657,"name":"drain","context":{"idset":"717","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.0132236,"name":"drain","context":{"idset":"721","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.0323799,"name":"drain","context":{"idset":"476","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.0514853,"name":"drain","context":{"idset":"727","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.0706017,"name":"drain","context":{"idset":"413","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.0899227,"name":"drain","context":{"idset":"729","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.1091261,"name":"drain","context":{"idset":"722","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.1287563,"name":"drain","context":{"idset":"730","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.1478031,"name":"drain","context":{"idset":"726","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.1676087,"name":"drain","context":{"idset":"728","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.1873763,"name":"drain","context":{"idset":"737","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.2068596,"name":"drain","context":{"idset":"732","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.2261236,"name":"drain","context":{"idset":"734","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.2461646,"name":"drain","context":{"idset":"724","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.2661757,"name":"drain","context":{"idset":"731","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.2863336,"name":"drain","context":{"idset":"716","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.3103428,"name":"drain","context":{"idset":"725","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.3310742,"name":"drain","context":{"idset":"712","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.3575094,"name":"drain","context":{"idset":"733","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205366.3852854,"name":"drain","context":{"idset":"739","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205474.5860033,"name":"undrain","context":{"idset":"69-76,93-108,357-364,366-380,389-420,469-492,709-755"}}
+{"timestamp":1706205486.0503614,"name":"drain","context":{"idset":"378","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.0920258,"name":"drain","context":{"idset":"741","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.1333766,"name":"drain","context":{"idset":"743","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.1718361,"name":"drain","context":{"idset":"376","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.209444,"name":"drain","context":{"idset":"70","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.2457838,"name":"drain","context":{"idset":"746","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.2742629,"name":"drain","context":{"idset":"740","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.3091779,"name":"drain","context":{"idset":"391","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.3599124,"name":"drain","context":{"idset":"744","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.3980906,"name":"drain","context":{"idset":"389","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.4454949,"name":"drain","context":{"idset":"377","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.4830692,"name":"drain","context":{"idset":"69","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.5207458,"name":"drain","context":{"idset":"742","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.5674126,"name":"drain","context":{"idset":"747","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.6094191,"name":"drain","context":{"idset":"745","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.6364024,"name":"drain","context":{"idset":"72","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.667079,"name":"drain","context":{"idset":"380","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.6911635,"name":"drain","context":{"idset":"74","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.7126181,"name":"drain","context":{"idset":"379","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.7342768,"name":"drain","context":{"idset":"390","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.7563419,"name":"drain","context":{"idset":"71","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.7766111,"name":"drain","context":{"idset":"394","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.7966316,"name":"drain","context":{"idset":"392","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.816057,"name":"drain","context":{"idset":"75","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.8359005,"name":"drain","context":{"idset":"754","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.8552847,"name":"drain","context":{"idset":"748","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.8745017,"name":"drain","context":{"idset":"73","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.8936181,"name":"drain","context":{"idset":"395","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.9127986,"name":"drain","context":{"idset":"393","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.9323487,"name":"drain","context":{"idset":"397","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.9515839,"name":"drain","context":{"idset":"750","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.9708617,"name":"drain","context":{"idset":"752","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205486.9900291,"name":"drain","context":{"idset":"101","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.0096872,"name":"drain","context":{"idset":"753","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.0288799,"name":"drain","context":{"idset":"749","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.0479009,"name":"drain","context":{"idset":"399","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.0669212,"name":"drain","context":{"idset":"405","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.085933,"name":"drain","context":{"idset":"396","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.105449,"name":"drain","context":{"idset":"751","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.1246016,"name":"drain","context":{"idset":"398","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.1442394,"name":"drain","context":{"idset":"400","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.1641366,"name":"drain","context":{"idset":"403","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.1842945,"name":"drain","context":{"idset":"406","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.2046771,"name":"drain","context":{"idset":"99","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.2239852,"name":"drain","context":{"idset":"401","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.243088,"name":"drain","context":{"idset":"106","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.2620769,"name":"drain","context":{"idset":"94","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.2810464,"name":"drain","context":{"idset":"404","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.3001306,"name":"drain","context":{"idset":"76","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.3249922,"name":"drain","context":{"idset":"96","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.3444068,"name":"drain","context":{"idset":"93","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.3651443,"name":"drain","context":{"idset":"402","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.3841839,"name":"drain","context":{"idset":"755","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.4039626,"name":"drain","context":{"idset":"102","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.4237626,"name":"drain","context":{"idset":"409","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.4434075,"name":"drain","context":{"idset":"98","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.462218,"name":"drain","context":{"idset":"107","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.4817884,"name":"drain","context":{"idset":"103","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.5007365,"name":"drain","context":{"idset":"100","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.5196526,"name":"drain","context":{"idset":"97","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.5387907,"name":"drain","context":{"idset":"105","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.5585549,"name":"drain","context":{"idset":"108","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.5773931,"name":"drain","context":{"idset":"357","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.5962281,"name":"drain","context":{"idset":"412","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.6151919,"name":"drain","context":{"idset":"408","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.634356,"name":"drain","context":{"idset":"415","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.6532497,"name":"drain","context":{"idset":"359","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.6721945,"name":"drain","context":{"idset":"104","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.6910472,"name":"drain","context":{"idset":"358","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.7098386,"name":"drain","context":{"idset":"407","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.7287703,"name":"drain","context":{"idset":"362","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.753387,"name":"drain","context":{"idset":"360","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.7820733,"name":"drain","context":{"idset":"410","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.8127286,"name":"drain","context":{"idset":"416","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.8341925,"name":"drain","context":{"idset":"414","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.8532104,"name":"drain","context":{"idset":"418","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.872155,"name":"drain","context":{"idset":"364","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.8915551,"name":"drain","context":{"idset":"411","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.9106765,"name":"drain","context":{"idset":"368","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.9297571,"name":"drain","context":{"idset":"367","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.9487007,"name":"drain","context":{"idset":"95","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.9682524,"name":"drain","context":{"idset":"366","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205487.9872773,"name":"drain","context":{"idset":"370","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.0063415,"name":"drain","context":{"idset":"471","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.0254252,"name":"drain","context":{"idset":"369","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.0445211,"name":"drain","context":{"idset":"361","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.0636263,"name":"drain","context":{"idset":"477","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.0826735,"name":"drain","context":{"idset":"374","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.1015899,"name":"drain","context":{"idset":"417","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.1204851,"name":"drain","context":{"idset":"420","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.1400707,"name":"drain","context":{"idset":"473","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.1587536,"name":"drain","context":{"idset":"372","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.1777723,"name":"drain","context":{"idset":"475","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.1966805,"name":"drain","context":{"idset":"481","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.2156298,"name":"drain","context":{"idset":"472","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.2346406,"name":"drain","context":{"idset":"373","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.253988,"name":"drain","context":{"idset":"375","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.2731655,"name":"drain","context":{"idset":"480","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.2923503,"name":"drain","context":{"idset":"363","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.3116157,"name":"drain","context":{"idset":"474","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.3309147,"name":"drain","context":{"idset":"470","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.3499334,"name":"drain","context":{"idset":"487","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.3689141,"name":"drain","context":{"idset":"478","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.3880982,"name":"drain","context":{"idset":"483","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.4071996,"name":"drain","context":{"idset":"484","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.4265344,"name":"drain","context":{"idset":"486","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.4456453,"name":"drain","context":{"idset":"469","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.465451,"name":"drain","context":{"idset":"482","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.4847219,"name":"drain","context":{"idset":"485","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.5039597,"name":"drain","context":{"idset":"413","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.5227489,"name":"drain","context":{"idset":"488","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.5415449,"name":"drain","context":{"idset":"492","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.5617356,"name":"drain","context":{"idset":"709","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.5806346,"name":"drain","context":{"idset":"479","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.6001701,"name":"drain","context":{"idset":"371","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.6195359,"name":"drain","context":{"idset":"489","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.6401792,"name":"drain","context":{"idset":"490","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.6589391,"name":"drain","context":{"idset":"710","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.6778643,"name":"drain","context":{"idset":"711","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.6974094,"name":"drain","context":{"idset":"491","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.716898,"name":"drain","context":{"idset":"715","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.7354903,"name":"drain","context":{"idset":"714","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.7544868,"name":"drain","context":{"idset":"716","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.7741179,"name":"drain","context":{"idset":"713","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.7932043,"name":"drain","context":{"idset":"719","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.8130631,"name":"drain","context":{"idset":"718","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.8320129,"name":"drain","context":{"idset":"717","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.851028,"name":"drain","context":{"idset":"723","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.8820536,"name":"drain","context":{"idset":"727","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.9160705,"name":"drain","context":{"idset":"722","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.9489927,"name":"drain","context":{"idset":"721","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205488.975306,"name":"drain","context":{"idset":"724","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.0055165,"name":"drain","context":{"idset":"734","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.0268464,"name":"drain","context":{"idset":"737","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.0563123,"name":"drain","context":{"idset":"729","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.0919466,"name":"drain","context":{"idset":"736","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.1260393,"name":"drain","context":{"idset":"730","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.1529307,"name":"drain","context":{"idset":"726","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.1799452,"name":"drain","context":{"idset":"476","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.1990836,"name":"drain","context":{"idset":"738","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.2185469,"name":"drain","context":{"idset":"732","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.2378666,"name":"drain","context":{"idset":"419","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.2579162,"name":"drain","context":{"idset":"728","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.2775228,"name":"drain","context":{"idset":"735","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.2975245,"name":"drain","context":{"idset":"712","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.3174694,"name":"drain","context":{"idset":"739","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.3366759,"name":"drain","context":{"idset":"725","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.3564029,"name":"drain","context":{"idset":"731","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.3761268,"name":"drain","context":{"idset":"720","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706205489.398622,"name":"drain","context":{"idset":"733","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706212516.3256865,"name":"undrain","context":{"idset":"69-76,93-108,357-364,366-380,389-420,469-492,709-755"}}
+{"timestamp":1706224866.465081,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1706550328.6230378,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1706550390.62201,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1706638994.9035203,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1706639366.8043408,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1706639535.663702,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1706642508.3693256,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1706643700.5224135,"name":"drain","context":{"idset":"70","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5225041,"name":"drain","context":{"idset":"71","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5225573,"name":"drain","context":{"idset":"73","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.522593,"name":"drain","context":{"idset":"74","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5226243,"name":"drain","context":{"idset":"75","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5226569,"name":"drain","context":{"idset":"76","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5226886,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5227263,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5227592,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5227964,"name":"drain","context":{"idset":"96","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5228307,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5228627,"name":"drain","context":{"idset":"98","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5228968,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5229299,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5229673,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5230043,"name":"drain","context":{"idset":"102","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5230498,"name":"drain","context":{"idset":"104","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5230834,"name":"drain","context":{"idset":"105","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5231187,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.5231569,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1706643700.6223643,"name":"drain","context":{"idset":"108","reason":"broker was unresponsive"}}
+{"timestamp":1706643704.6238132,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5304887,"name":"drain","context":{"idset":"358","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5305703,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5306137,"name":"drain","context":{"idset":"360","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.530658,"name":"drain","context":{"idset":"361","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5306916,"name":"drain","context":{"idset":"362","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5307288,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5307624,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5307941,"name":"drain","context":{"idset":"366","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5308301,"name":"drain","context":{"idset":"367","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5308657,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5309012,"name":"drain","context":{"idset":"369","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5309348,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5309677,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5310004,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5310459,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5310941,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5311317,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5311785,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5312126,"name":"drain","context":{"idset":"377","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5312462,"name":"drain","context":{"idset":"378","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.53128,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5313137,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5313504,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5313857,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5314267,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5314705,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5315051,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5315382,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5315747,"name":"drain","context":{"idset":"395","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5316174,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5316565,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5316947,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5317326,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5317686,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5318053,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5318406,"name":"drain","context":{"idset":"402","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5318785,"name":"drain","context":{"idset":"403","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5319157,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5319552,"name":"drain","context":{"idset":"405","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5319963,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5320454,"name":"drain","context":{"idset":"407","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5320842,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5321217,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5321701,"name":"drain","context":{"idset":"410","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5322094,"name":"drain","context":{"idset":"411","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5322471,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5322897,"name":"drain","context":{"idset":"413","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5323288,"name":"drain","context":{"idset":"416","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5323701,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5324178,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5324602,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5325067,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5325489,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5325894,"name":"drain","context":{"idset":"470","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.53263,"name":"drain","context":{"idset":"471","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5326967,"name":"drain","context":{"idset":"472","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5327427,"name":"drain","context":{"idset":"473","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5327854,"name":"drain","context":{"idset":"474","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5328288,"name":"drain","context":{"idset":"475","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5328717,"name":"drain","context":{"idset":"476","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.532917,"name":"drain","context":{"idset":"477","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5329599,"name":"drain","context":{"idset":"478","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5330031,"name":"drain","context":{"idset":"479","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5330565,"name":"drain","context":{"idset":"480","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5331044,"name":"drain","context":{"idset":"481","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5331478,"name":"drain","context":{"idset":"482","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5331917,"name":"drain","context":{"idset":"483","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5332472,"name":"drain","context":{"idset":"484","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5332928,"name":"drain","context":{"idset":"485","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5333378,"name":"drain","context":{"idset":"486","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5333822,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5334308,"name":"drain","context":{"idset":"488","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5334766,"name":"drain","context":{"idset":"489","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5335226,"name":"drain","context":{"idset":"490","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5335751,"name":"drain","context":{"idset":"491","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5336223,"name":"drain","context":{"idset":"492","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5336697,"name":"drain","context":{"idset":"709","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5337195,"name":"drain","context":{"idset":"710","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5337718,"name":"drain","context":{"idset":"712","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.533819,"name":"drain","context":{"idset":"714","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5338674,"name":"drain","context":{"idset":"715","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5339179,"name":"drain","context":{"idset":"716","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5339656,"name":"drain","context":{"idset":"717","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5340159,"name":"drain","context":{"idset":"718","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5340717,"name":"drain","context":{"idset":"719","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.534121,"name":"drain","context":{"idset":"720","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5341702,"name":"drain","context":{"idset":"721","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5342214,"name":"drain","context":{"idset":"722","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5342715,"name":"drain","context":{"idset":"723","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5343282,"name":"drain","context":{"idset":"724","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5343802,"name":"drain","context":{"idset":"725","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5344326,"name":"drain","context":{"idset":"726","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5344937,"name":"drain","context":{"idset":"727","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5345466,"name":"drain","context":{"idset":"728","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5345974,"name":"drain","context":{"idset":"729","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.534652,"name":"drain","context":{"idset":"730","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5347042,"name":"drain","context":{"idset":"731","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5347555,"name":"drain","context":{"idset":"732","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5348153,"name":"drain","context":{"idset":"733","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5348766,"name":"drain","context":{"idset":"734","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5349293,"name":"drain","context":{"idset":"735","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5349789,"name":"drain","context":{"idset":"736","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5350425,"name":"drain","context":{"idset":"737","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5350947,"name":"drain","context":{"idset":"738","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5351496,"name":"drain","context":{"idset":"739","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5352025,"name":"drain","context":{"idset":"740","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5352533,"name":"drain","context":{"idset":"741","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.535306,"name":"drain","context":{"idset":"742","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.535368,"name":"drain","context":{"idset":"743","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5354221,"name":"drain","context":{"idset":"744","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5354757,"name":"drain","context":{"idset":"745","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5355294,"name":"drain","context":{"idset":"746","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.535584,"name":"drain","context":{"idset":"747","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5356374,"name":"drain","context":{"idset":"748","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.535707,"name":"drain","context":{"idset":"749","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5357668,"name":"drain","context":{"idset":"750","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5358222,"name":"drain","context":{"idset":"752","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5358865,"name":"drain","context":{"idset":"753","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.5359447,"name":"drain","context":{"idset":"754","reason":"broker was unresponsive"}}
+{"timestamp":1706643706.6222198,"name":"drain","context":{"idset":"755","reason":"broker was unresponsive"}}
+{"timestamp":1706643708.622967,"name":"drain","context":{"idset":"713","reason":"broker was unresponsive"}}
+{"timestamp":1706643710.52336,"name":"drain","context":{"idset":"711","reason":"broker was unresponsive"}}
+{"timestamp":1706643710.6241217,"name":"drain","context":{"idset":"756","reason":"broker was unresponsive"}}
+{"timestamp":1706643766.526454,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1706643766.5276196,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1706643766.5287139,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1706643766.529748,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1706643766.5307667,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1706643766.5317667,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1706643766.5327542,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1706643766.5337474,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1706643766.5347691,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1706643766.5357463,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1706643766.5367107,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1706643766.5376701,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1706643766.5386283,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1706643766.539583,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1706643766.5405333,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1706643766.5414715,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1706643766.5424075,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1706643766.5433309,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1706643766.5442538,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1706643766.5451674,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1706643766.622766,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1706643768.5312185,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1706643768.532387,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1706643768.5334775,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1706643768.5345347,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1706643768.5355504,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1706643768.5365546,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1706643768.5375514,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1706643768.5385437,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1706643768.5395279,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1706643768.5405076,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1706643768.5414784,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1706643768.5424507,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1706643768.5434172,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1706643768.5443726,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1706643768.5453312,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1706643768.5463545,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1706643768.5472755,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1706643768.548218,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1706643768.5491862,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1706643768.5501232,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1706643768.5512588,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1706643768.5522478,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1706643768.5531604,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1706643768.5540521,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1706643768.5549352,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1706643768.55581,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1706643768.5567071,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1706643768.5575805,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1706643768.5584571,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1706643768.5593307,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1706643768.5601935,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1706643768.5610421,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1706643768.561873,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1706643768.5627151,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1706643768.5635576,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1706643768.5643933,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1706643768.565218,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1706643768.5660336,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1706643768.5668416,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1706643768.5676591,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1706643768.5684564,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1706643768.5692761,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1706643768.570106,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1706643768.5708995,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1706643768.5716689,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1706643768.5724452,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1706643768.5732219,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1706643768.5740361,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1706643768.5747747,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1706643768.5755475,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1706643768.5763412,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1706643768.5771532,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1706643768.5779009,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1706643768.5786796,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1706643768.5794806,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1706643768.5808868,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1706643768.5810349,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1706643768.5817215,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1706643768.5824952,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1706643768.5832105,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1706643768.5839276,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1706643768.6221874,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1706643770.5248544,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1706643770.5261784,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1706643770.5273755,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1706643770.5285113,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1706643770.5296392,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1706643770.5307467,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1706643770.5318427,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1706643770.5329561,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1706643770.5340166,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1706643770.5350966,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1706643770.5361526,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1706643770.5372019,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1706643770.5382764,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1706643770.5393405,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1706643770.6218057,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1706643772.5272899,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1706643772.5281081,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1706643772.528841,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1706643772.5295415,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1706643772.5302324,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1706643772.530905,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1706643772.5316021,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1706643772.53227,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1706643772.5329244,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1706643772.533586,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1706643772.5342467,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1706643772.5348897,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1706643772.5355411,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1706643772.53618,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1706643772.5368073,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1706643772.5374422,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1706643772.5380676,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1706643772.5386791,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1706643772.5392933,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1706643772.5398905,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1706643772.5404861,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1706643772.541086,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1706643772.5416682,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1706643772.5422652,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1706643772.5428386,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1706643772.5434165,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1706643772.5439818,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1706643772.544548,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1706643772.5451043,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1706643772.5456529,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1706643772.5462036,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1706643772.5467422,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1706643772.5472772,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1706643772.5478029,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1706643772.548337,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1706643772.5488505,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1706643772.5493739,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1706643772.5498779,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1706643772.5503876,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1706643772.5508778,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1706643772.551393,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1706643772.5518749,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1706643772.5523615,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1706643772.5528312,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1706643772.5532987,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1706643772.5537562,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1706643772.6230121,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1706664320.6229515,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1706664386.5224779,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1706664386.6229351,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1706664452.5235631,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1706664452.6240973,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1706664464.6233039,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1706664512.6237733,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1706664516.6214771,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1706664530.5216782,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1706664530.6223965,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1706664576.6232057,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1706664596.6227796,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1706665676.6228352,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1706665682.6238208,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1706665706.62307,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1706665712.6229784,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1706665724.6219532,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1706665736.6238973,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1706665742.6225536,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1706665748.6229119,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1706665752.6235087,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1706665772.6233628,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1706665778.6228347,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1706665786.6225719,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1706665798.6224215,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1706665814.6217563,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1706665816.6225717,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1706665820.6235552,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1706665848.6220164,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1706665878.6231284,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1706665882.623771,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1706665912.6225226,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1706666144.6234725,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1706666208.6230674,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1706666274.6225016,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1706666312.6235311,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1706666338.6223543,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1706666378.6233277,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1706666666.6234331,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1706666726.6232285,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1706666732.6217601,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1706666792.6218503,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1706667140.6235723,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1706667190.6231809,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1706667202.6222703,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1706667242.6234763,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1706667252.6236632,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1706667254.6234071,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1706667304.6237018,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1706667316.6222849,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1706667584.6234779,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1706667650.6225646,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1706668046.6232135,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1706668112.6231892,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1706668208.6229718,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1706668274.623162,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1706669306.6234145,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1706669370.6229584,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1706669916.6238022,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1706669984.623198,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1706673092.622416,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1706673158.6227446,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1706753268.623363,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1706753270.6231773,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1706753278.6234145,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1706753334.522568,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1706753334.6227627,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1706753342.6232221,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1706817842.6225615,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1706817904.6234837,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1706830982.5222063,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5223689,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.522424,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5224741,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5225239,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5225706,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5226185,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5226657,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5227127,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5227621,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5228136,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5228651,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5229175,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5229704,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5230403,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5230947,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5231473,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5232034,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.5232642,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1706830982.6224263,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1706830984.5230994,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1706830984.5232491,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1706830984.5233288,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1706830984.5234056,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1706830984.6240444,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1706831046.5237181,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1706831046.5239632,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1706831046.5242071,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1706831046.5244265,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1706831046.5245984,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1706831046.5247781,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1706831046.5249541,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1706831046.525146,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1706831046.5253253,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1706831046.5254879,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1706831046.5256376,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1706831046.5257845,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1706831046.5259206,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1706831046.5260656,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1706831046.5262098,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1706831046.5263748,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1706831046.526499,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1706831046.5266256,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1706831046.5267344,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1706831046.5268216,"name":"drain","context":{"idset":"1","reason":"epilog failed for jobid fafZPpxiZpb","overwrite":0}}
+{"timestamp":1706831046.5268674,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1706831046.5269673,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1706831046.5270886,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1706831046.5271792,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1706831046.527266,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1706831046.6226544,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1707358689.473824,"name":"online","context":{"idset":"639-642,645-646"}}
+{"timestamp":1707358689.5843108,"name":"online","context":{"idset":"643"}}
+{"timestamp":1707358689.7437415,"name":"online","context":{"idset":"644,648,650,652"}}
+{"timestamp":1707358689.9250088,"name":"online","context":{"idset":"637-638,649"}}
+{"timestamp":1707358690.1383023,"name":"online","context":{"idset":"647,653"}}
+{"timestamp":1707358690.4499235,"name":"online","context":{"idset":"651"}}
+{"timestamp":1707358718.9965537,"name":"undrain","context":{"idset":"637-653"}}
+{"timestamp":1707409247.4984436,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1707409247.5531557,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1707409247.5582299,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1707409247.5664337,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1707409247.5776229,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1707409247.5804968,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1707409247.5832062,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1707409247.5942154,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1707409247.6209562,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1707409247.6238539,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1707409247.6357193,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1707409247.6364393,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1707409247.6436405,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1707409247.6587644,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1707409247.6786733,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1707409247.7166703,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1707409247.8172143,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1707409623.0872655,"name":"online","context":{"idset":"621"}}
+{"timestamp":1707409748.3302925,"name":"undrain","context":{"idset":"621"}}
+{"timestamp":1707412055.2168877,"name":"online","context":{"idset":"640-641,645,647"}}
+{"timestamp":1707412055.3370578,"name":"online","context":{"idset":"643,649,652"}}
+{"timestamp":1707412055.5288715,"name":"online","context":{"idset":"638-639,642,651"}}
+{"timestamp":1707412055.6452501,"name":"online","context":{"idset":"644,646,650"}}
+{"timestamp":1707412055.8794975,"name":"online","context":{"idset":"653"}}
+{"timestamp":1707412056.0000803,"name":"online","context":{"idset":"637"}}
+{"timestamp":1707412056.2723005,"name":"online","context":{"idset":"648"}}
+{"timestamp":1707416389.51916,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1707416389.5195937,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1707416389.5200162,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1707416389.5223801,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1707416389.5232728,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1707416389.5267501,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1707416389.5312462,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1707416389.5315361,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1707416389.5365825,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1707416389.546068,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1707416389.5847394,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1707416389.5849903,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1707416389.5878994,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1707416389.5884063,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1707416389.6886823,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1707416390.0723579,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1707416390.1729405,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1707418366.5763891,"name":"online","context":{"idset":"637,642"}}
+{"timestamp":1707418366.7229328,"name":"online","context":{"idset":"638"}}
+{"timestamp":1707418366.9432955,"name":"online","context":{"idset":"639-640,643-644,651"}}
+{"timestamp":1707418367.1498802,"name":"online","context":{"idset":"653"}}
+{"timestamp":1707418367.3134162,"name":"online","context":{"idset":"646"}}
+{"timestamp":1707418367.438798,"name":"online","context":{"idset":"650"}}
+{"timestamp":1707418367.8433194,"name":"online","context":{"idset":"647"}}
+{"timestamp":1707418368.1441309,"name":"online","context":{"idset":"641,652"}}
+{"timestamp":1707418369.7123914,"name":"online","context":{"idset":"645"}}
+{"timestamp":1707418370.7570274,"name":"online","context":{"idset":"648"}}
+{"timestamp":1707418378.9903755,"name":"online","context":{"idset":"649"}}
+{"timestamp":1707433772.2247481,"name":"online","context":{"idset":"61"}}
+{"timestamp":1707434005.3122516,"name":"online","context":{"idset":"63"}}
+{"timestamp":1707434005.3859298,"name":"online","context":{"idset":"80"}}
+{"timestamp":1707434005.4503593,"name":"online","context":{"idset":"67,71,74"}}
+{"timestamp":1707434005.4974456,"name":"online","context":{"idset":"73"}}
+{"timestamp":1707434005.4997747,"name":"online","context":{"idset":"136"}}
+{"timestamp":1707434005.5094175,"name":"online","context":{"idset":"65,77"}}
+{"timestamp":1707434005.5223069,"name":"online","context":{"idset":"94"}}
+{"timestamp":1707434005.5713198,"name":"online","context":{"idset":"82"}}
+{"timestamp":1707434005.5794694,"name":"online","context":{"idset":"87"}}
+{"timestamp":1707434005.5938871,"name":"online","context":{"idset":"83,109"}}
+{"timestamp":1707434005.6047299,"name":"online","context":{"idset":"116"}}
+{"timestamp":1707434005.6689897,"name":"online","context":{"idset":"64,81"}}
+{"timestamp":1707434005.6878159,"name":"online","context":{"idset":"69"}}
+{"timestamp":1707434005.8025417,"name":"online","context":{"idset":"62,70,85,95,112,152"}}
+{"timestamp":1707434005.81846,"name":"online","context":{"idset":"93"}}
+{"timestamp":1707434005.8692355,"name":"online","context":{"idset":"79,88,131"}}
+{"timestamp":1707434005.9150543,"name":"online","context":{"idset":"76,84,89,97,143"}}
+{"timestamp":1707434005.9391255,"name":"online","context":{"idset":"113,137"}}
+{"timestamp":1707434005.9457178,"name":"online","context":{"idset":"135"}}
+{"timestamp":1707434005.9609158,"name":"online","context":{"idset":"72"}}
+{"timestamp":1707434005.9701858,"name":"online","context":{"idset":"96,129,145"}}
+{"timestamp":1707434006.0008225,"name":"online","context":{"idset":"68,101,103"}}
+{"timestamp":1707434006.0315189,"name":"online","context":{"idset":"75,92,139,147"}}
+{"timestamp":1707434006.0438144,"name":"online","context":{"idset":"66,134"}}
+{"timestamp":1707434006.109833,"name":"online","context":{"idset":"78,106,108,119,146,149"}}
+{"timestamp":1707434006.2148356,"name":"online","context":{"idset":"105,107,111,122,151,156"}}
+{"timestamp":1707434006.3756568,"name":"online","context":{"idset":"90,99,120,132,182"}}
+{"timestamp":1707434006.4825277,"name":"online","context":{"idset":"86,91,100,102,104,114,121,125,130,138,140,142,153,164,172,180"}}
+{"timestamp":1707434006.5223749,"name":"online","context":{"idset":"118,127-128,148,150,161,194"}}
+{"timestamp":1707434006.6227975,"name":"online","context":{"idset":"144,154,159,165,188"}}
+{"timestamp":1707434006.756181,"name":"online","context":{"idset":"98,110,179,190"}}
+{"timestamp":1707434006.8686914,"name":"online","context":{"idset":"141,155,178,193"}}
+{"timestamp":1707434006.9887443,"name":"online","context":{"idset":"124,126,133,157,168,185,191"}}
+{"timestamp":1707434007.0965371,"name":"online","context":{"idset":"163,171,181,192,195"}}
+{"timestamp":1707434007.1983302,"name":"online","context":{"idset":"162,166,196-197"}}
+{"timestamp":1707434007.3251092,"name":"online","context":{"idset":"186,189"}}
+{"timestamp":1707434007.4459989,"name":"online","context":{"idset":"167,187"}}
+{"timestamp":1707434007.550698,"name":"online","context":{"idset":"158,160,170,183-184"}}
+{"timestamp":1707434007.6872282,"name":"online","context":{"idset":"169"}}
+{"timestamp":1707434015.1790452,"name":"online","context":{"idset":"201"}}
+{"timestamp":1707434015.4410911,"name":"online","context":{"idset":"208"}}
+{"timestamp":1707434015.5545342,"name":"online","context":{"idset":"206"}}
+{"timestamp":1707434015.6166286,"name":"online","context":{"idset":"202,212"}}
+{"timestamp":1707434015.6516237,"name":"online","context":{"idset":"210"}}
+{"timestamp":1707434015.733355,"name":"online","context":{"idset":"198"}}
+{"timestamp":1707434015.7837384,"name":"online","context":{"idset":"214"}}
+{"timestamp":1707434015.8743839,"name":"online","context":{"idset":"199"}}
+{"timestamp":1707434016.0236781,"name":"online","context":{"idset":"224,230"}}
+{"timestamp":1707434016.0621789,"name":"online","context":{"idset":"203"}}
+{"timestamp":1707434016.103682,"name":"online","context":{"idset":"200,204"}}
+{"timestamp":1707434016.1378958,"name":"online","context":{"idset":"226"}}
+{"timestamp":1707434016.1501079,"name":"online","context":{"idset":"213"}}
+{"timestamp":1707434016.1605091,"name":"online","context":{"idset":"244"}}
+{"timestamp":1707434016.1748047,"name":"online","context":{"idset":"205"}}
+{"timestamp":1707434016.3096704,"name":"online","context":{"idset":"209,216-217,220"}}
+{"timestamp":1707434016.3482566,"name":"online","context":{"idset":"222,242,258"}}
+{"timestamp":1707434016.4423344,"name":"online","context":{"idset":"211,215,218,228,240,243,284"}}
+{"timestamp":1707434016.4949431,"name":"online","context":{"idset":"277"}}
+{"timestamp":1707434016.5563121,"name":"online","context":{"idset":"207,237,254,262"}}
+{"timestamp":1707434016.6000922,"name":"online","context":{"idset":"223,239,256"}}
+{"timestamp":1707434016.7016737,"name":"online","context":{"idset":"221,227,234,241,251,257,260,278,285,294,300"}}
+{"timestamp":1707434016.7842207,"name":"online","context":{"idset":"232,236,246,288-289,298"}}
+{"timestamp":1707434016.9047701,"name":"online","context":{"idset":"263,279,286-287,292-293,295,301,304,313"}}
+{"timestamp":1707434016.9932363,"name":"online","context":{"idset":"219,250,255,264,281,297,307,318"}}
+{"timestamp":1707434017.1012645,"name":"online","context":{"idset":"229,231,235,247,253,259,283,290-291,296,309,312,316,320-321,323,328"}}
+{"timestamp":1707434017.2145441,"name":"online","context":{"idset":"225,252,267,270,273,276,306,315,322,327"}}
+{"timestamp":1707434017.3443432,"name":"online","context":{"idset":"233,245,266,268,272,275,282"}}
+{"timestamp":1707434017.4798486,"name":"online","context":{"idset":"261,299,310-311,314,324"}}
+{"timestamp":1707434017.5821726,"name":"online","context":{"idset":"280,317,325,329"}}
+{"timestamp":1707434017.6925523,"name":"online","context":{"idset":"249,265,269,271,305"}}
+{"timestamp":1707434017.8951128,"name":"online","context":{"idset":"248,274"}}
+{"timestamp":1707434018.0395648,"name":"online","context":{"idset":"326"}}
+{"timestamp":1707434024.9460845,"name":"online","context":{"idset":"330"}}
+{"timestamp":1707434025.1191425,"name":"online","context":{"idset":"331"}}
+{"timestamp":1707434025.4326053,"name":"online","context":{"idset":"332-333,335-338,342,344"}}
+{"timestamp":1707434025.5086286,"name":"online","context":{"idset":"339-340,345"}}
+{"timestamp":1707434025.6145403,"name":"online","context":{"idset":"343,348"}}
+{"timestamp":1707434025.6539609,"name":"online","context":{"idset":"346"}}
+{"timestamp":1707434025.7790792,"name":"online","context":{"idset":"347"}}
+{"timestamp":1707434025.8663063,"name":"online","context":{"idset":"334"}}
+{"timestamp":1707434026.271349,"name":"online","context":{"idset":"362,368"}}
+{"timestamp":1707434026.291008,"name":"online","context":{"idset":"349"}}
+{"timestamp":1707434026.3931127,"name":"online","context":{"idset":"350"}}
+{"timestamp":1707434026.4299061,"name":"online","context":{"idset":"351"}}
+{"timestamp":1707434026.5650547,"name":"online","context":{"idset":"354-356,359"}}
+{"timestamp":1707434026.5993035,"name":"online","context":{"idset":"357-358"}}
+{"timestamp":1707434026.612931,"name":"online","context":{"idset":"363"}}
+{"timestamp":1707434026.7220304,"name":"online","context":{"idset":"353"}}
+{"timestamp":1707434026.8466456,"name":"online","context":{"idset":"352,361,367,370"}}
+{"timestamp":1707434027.0239525,"name":"online","context":{"idset":"360,369,373,380"}}
+{"timestamp":1707434027.1095126,"name":"online","context":{"idset":"372,378"}}
+{"timestamp":1707434027.1535544,"name":"online","context":{"idset":"376-377,402"}}
+{"timestamp":1707434027.2574289,"name":"online","context":{"idset":"364-365,371,403,445"}}
+{"timestamp":1707434027.3640747,"name":"online","context":{"idset":"366,374-375,379,382,385,390,397-398,400,408,452,456"}}
+{"timestamp":1707434027.4833906,"name":"online","context":{"idset":"381,387,391,405-406,411"}}
+{"timestamp":1707434027.5519848,"name":"online","context":{"idset":"394,399,401,418,427"}}
+{"timestamp":1707434027.6132407,"name":"online","context":{"idset":"412-413,421,435"}}
+{"timestamp":1707434027.7359395,"name":"online","context":{"idset":"383,415-416,419,423,428,430-431,444"}}
+{"timestamp":1707434027.8554189,"name":"online","context":{"idset":"410,432,438,441-442,447-448,457"}}
+{"timestamp":1707434027.9567428,"name":"online","context":{"idset":"407,429,436,450-451,453"}}
+{"timestamp":1707434028.0630481,"name":"online","context":{"idset":"409,422,426,439,458"}}
+{"timestamp":1707434028.073781,"name":"online","context":{"idset":"420,424"}}
+{"timestamp":1707434028.1818717,"name":"online","context":{"idset":"417,425,433-434,440,446,459"}}
+{"timestamp":1707434028.2830925,"name":"online","context":{"idset":"449"}}
+{"timestamp":1707434028.4178691,"name":"online","context":{"idset":"460"}}
+{"timestamp":1707434028.5243974,"name":"online","context":{"idset":"389,414"}}
+{"timestamp":1707434028.6238961,"name":"online","context":{"idset":"395,437"}}
+{"timestamp":1707434028.7778034,"name":"online","context":{"idset":"404"}}
+{"timestamp":1707434029.1306214,"name":"online","context":{"idset":"443,454"}}
+{"timestamp":1707434029.6500421,"name":"online","context":{"idset":"396"}}
+{"timestamp":1707434030.017457,"name":"online","context":{"idset":"455"}}
+{"timestamp":1707434030.9722512,"name":"online","context":{"idset":"392"}}
+{"timestamp":1707434033.8067901,"name":"online","context":{"idset":"393"}}
+{"timestamp":1707434035.3570893,"name":"online","context":{"idset":"461,468,472"}}
+{"timestamp":1707434035.607578,"name":"online","context":{"idset":"462"}}
+{"timestamp":1707434035.6910689,"name":"online","context":{"idset":"464"}}
+{"timestamp":1707434035.7301567,"name":"online","context":{"idset":"463,471"}}
+{"timestamp":1707434035.7777872,"name":"online","context":{"idset":"465"}}
+{"timestamp":1707434035.9048064,"name":"online","context":{"idset":"466,470"}}
+{"timestamp":1707434035.9160004,"name":"online","context":{"idset":"469,475"}}
+{"timestamp":1707434035.9955657,"name":"online","context":{"idset":"477"}}
+{"timestamp":1707434036.0040131,"name":"online","context":{"idset":"474"}}
+{"timestamp":1707434036.0591648,"name":"online","context":{"idset":"476"}}
+{"timestamp":1707434036.1082895,"name":"online","context":{"idset":"473,478"}}
+{"timestamp":1707434036.2237611,"name":"online","context":{"idset":"467"}}
+{"timestamp":1707434036.2485564,"name":"online","context":{"idset":"479"}}
+{"timestamp":1707434036.3544466,"name":"online","context":{"idset":"482,484"}}
+{"timestamp":1707434036.3819122,"name":"online","context":{"idset":"483"}}
+{"timestamp":1707434036.4301538,"name":"online","context":{"idset":"480,487"}}
+{"timestamp":1707434036.47982,"name":"online","context":{"idset":"486"}}
+{"timestamp":1707434036.4946904,"name":"online","context":{"idset":"488"}}
+{"timestamp":1707434036.5464587,"name":"online","context":{"idset":"489"}}
+{"timestamp":1707434036.5904565,"name":"online","context":{"idset":"481,491"}}
+{"timestamp":1707434036.6597269,"name":"online","context":{"idset":"492"}}
+{"timestamp":1707434036.670929,"name":"online","context":{"idset":"494"}}
+{"timestamp":1707434036.7176864,"name":"online","context":{"idset":"490"}}
+{"timestamp":1707434036.7905693,"name":"online","context":{"idset":"496"}}
+{"timestamp":1707434036.9054496,"name":"online","context":{"idset":"497,499-500,502"}}
+{"timestamp":1707434036.9400971,"name":"online","context":{"idset":"493"}}
+{"timestamp":1707434036.9845774,"name":"online","context":{"idset":"485"}}
+{"timestamp":1707434037.0731452,"name":"online","context":{"idset":"498,503-504,507,548"}}
+{"timestamp":1707434037.1772604,"name":"online","context":{"idset":"506,508-510"}}
+{"timestamp":1707434037.294739,"name":"online","context":{"idset":"511-514,520,522,532"}}
+{"timestamp":1707434037.4030464,"name":"online","context":{"idset":"515,529,567,569"}}
+{"timestamp":1707434037.5713003,"name":"online","context":{"idset":"525,528,530-531,535,538"}}
+{"timestamp":1707434037.695478,"name":"online","context":{"idset":"518,533-534,536-537,541-544,546"}}
+{"timestamp":1707434037.8008163,"name":"online","context":{"idset":"516-517,521,523,526,539-540,547,549,551-556"}}
+{"timestamp":1707434037.9051001,"name":"online","context":{"idset":"524,558-559,565,575"}}
+{"timestamp":1707434037.9285104,"name":"online","context":{"idset":"557,572"}}
+{"timestamp":1707434038.0683484,"name":"online","context":{"idset":"545,563-564,566,568,570-571"}}
+{"timestamp":1707434038.1692734,"name":"online","context":{"idset":"560,562,573,576,578"}}
+{"timestamp":1707434038.2852554,"name":"online","context":{"idset":"519,527,561"}}
+{"timestamp":1707434038.4241109,"name":"online","context":{"idset":"574,579-580"}}
+{"timestamp":1707434038.5351241,"name":"online","context":{"idset":"577,582-583"}}
+{"timestamp":1707434038.667577,"name":"online","context":{"idset":"581"}}
+{"timestamp":1707434038.9044764,"name":"online","context":{"idset":"584"}}
+{"timestamp":1707434039.0138421,"name":"online","context":{"idset":"585"}}
+{"timestamp":1707434039.5236073,"name":"online","context":{"idset":"586"}}
+{"timestamp":1707434040.0148625,"name":"online","context":{"idset":"587"}}
+{"timestamp":1707434040.9205487,"name":"online","context":{"idset":"588"}}
+{"timestamp":1707434043.1040378,"name":"online","context":{"idset":"589"}}
+{"timestamp":1707434044.9017775,"name":"online","context":{"idset":"591"}}
+{"timestamp":1707434044.9504523,"name":"online","context":{"idset":"592"}}
+{"timestamp":1707434045.0908365,"name":"online","context":{"idset":"590"}}
+{"timestamp":1707434045.1488888,"name":"online","context":{"idset":"594"}}
+{"timestamp":1707434045.3671014,"name":"online","context":{"idset":"595"}}
+{"timestamp":1707434045.6794918,"name":"online","context":{"idset":"599"}}
+{"timestamp":1707434045.7359207,"name":"online","context":{"idset":"600"}}
+{"timestamp":1707434045.8754756,"name":"online","context":{"idset":"597,601-602,620"}}
+{"timestamp":1707434045.9111545,"name":"online","context":{"idset":"604-605"}}
+{"timestamp":1707434046.1248534,"name":"online","context":{"idset":"598,607"}}
+{"timestamp":1707434046.2588334,"name":"online","context":{"idset":"606,608"}}
+{"timestamp":1707434046.2759678,"name":"online","context":{"idset":"596"}}
+{"timestamp":1707434046.3085577,"name":"online","context":{"idset":"610"}}
+{"timestamp":1707434046.3264582,"name":"online","context":{"idset":"654"}}
+{"timestamp":1707434046.3550549,"name":"online","context":{"idset":"611-612"}}
+{"timestamp":1707434046.3918669,"name":"online","context":{"idset":"636"}}
+{"timestamp":1707434046.425765,"name":"online","context":{"idset":"613"}}
+{"timestamp":1707434046.4412334,"name":"online","context":{"idset":"616"}}
+{"timestamp":1707434046.5052495,"name":"online","context":{"idset":"618"}}
+{"timestamp":1707434046.5125716,"name":"online","context":{"idset":"617"}}
+{"timestamp":1707434046.5693979,"name":"online","context":{"idset":"603,619"}}
+{"timestamp":1707434046.6122022,"name":"online","context":{"idset":"609"}}
+{"timestamp":1707434046.6812794,"name":"online","context":{"idset":"614,622"}}
+{"timestamp":1707434046.715647,"name":"online","context":{"idset":"624"}}
+{"timestamp":1707434046.7511072,"name":"online","context":{"idset":"625-627"}}
+{"timestamp":1707434046.8921063,"name":"online","context":{"idset":"623,628,635"}}
+{"timestamp":1707434046.9048729,"name":"online","context":{"idset":"615,629"}}
+{"timestamp":1707434046.9777691,"name":"online","context":{"idset":"631,633,658"}}
+{"timestamp":1707434047.1183043,"name":"online","context":{"idset":"634,655-656"}}
+{"timestamp":1707434047.1697893,"name":"online","context":{"idset":"661"}}
+{"timestamp":1707434047.3107162,"name":"online","context":{"idset":"630,659,675,687,691,694"}}
+{"timestamp":1707434047.397491,"name":"online","context":{"idset":"663,689"}}
+{"timestamp":1707434047.502682,"name":"online","context":{"idset":"657,666,671,686"}}
+{"timestamp":1707434047.6093378,"name":"online","context":{"idset":"660,664-665,667-670,672"}}
+{"timestamp":1707434047.7070847,"name":"online","context":{"idset":"674,682,685"}}
+{"timestamp":1707434047.8301075,"name":"online","context":{"idset":"662,676-677,684,696"}}
+{"timestamp":1707434047.9368904,"name":"online","context":{"idset":"679,688,690,692,695,697,703,706-707"}}
+{"timestamp":1707434048.0437565,"name":"online","context":{"idset":"680,693,698,700,704,708-710,712,720"}}
+{"timestamp":1707434048.1570072,"name":"online","context":{"idset":"683,702,705,711,713-715,717,719,722"}}
+{"timestamp":1707434048.2611933,"name":"online","context":{"idset":"673,681,699,718,721"}}
+{"timestamp":1707434048.326472,"name":"online","context":{"idset":"724-725"}}
+{"timestamp":1707434048.4743266,"name":"online","context":{"idset":"701,728"}}
+{"timestamp":1707434048.6254108,"name":"online","context":{"idset":"723"}}
+{"timestamp":1707434048.7354369,"name":"online","context":{"idset":"716,726,729"}}
+{"timestamp":1707434048.9791627,"name":"online","context":{"idset":"727"}}
+{"timestamp":1707434049.3619745,"name":"online","context":{"idset":"730"}}
+{"timestamp":1707434049.9150052,"name":"online","context":{"idset":"731"}}
+{"timestamp":1707434050.7503722,"name":"online","context":{"idset":"732"}}
+{"timestamp":1707434053.5131485,"name":"online","context":{"idset":"733"}}
+{"timestamp":1707434054.929229,"name":"online","context":{"idset":"734-735"}}
+{"timestamp":1707434055.0969434,"name":"online","context":{"idset":"737"}}
+{"timestamp":1707434055.4678755,"name":"online","context":{"idset":"736"}}
+{"timestamp":1707434055.5798473,"name":"online","context":{"idset":"741-742,746"}}
+{"timestamp":1707434055.7919774,"name":"online","context":{"idset":"738,743,752"}}
+{"timestamp":1707434055.8971531,"name":"online","context":{"idset":"740,744-745"}}
+{"timestamp":1707434056.0193923,"name":"online","context":{"idset":"739,748"}}
+{"timestamp":1707434056.2444251,"name":"online","context":{"idset":"747,749-750,754"}}
+{"timestamp":1707434056.3745563,"name":"online","context":{"idset":"756"}}
+{"timestamp":1707434056.5239801,"name":"online","context":{"idset":"755"}}
+{"timestamp":1707434056.7864311,"name":"online","context":{"idset":"753"}}
+{"timestamp":1707434109.9103398,"name":"online","context":{"idset":"319"}}
+{"timestamp":1707434129.8377991,"name":"online","context":{"idset":"495"}}
+{"timestamp":1707434129.9550867,"name":"online","context":{"idset":"505"}}
+{"timestamp":1707434138.4164155,"name":"online","context":{"idset":"593"}}
+{"timestamp":1707434140.0956285,"name":"online","context":{"idset":"632"}}
+{"timestamp":1707434928.0677447,"name":"online","context":{"idset":"123"}}
+{"timestamp":1707434928.7002375,"name":"online","context":{"idset":"117"}}
+{"timestamp":1707435712.4176075,"name":"online","context":{"idset":"341"}}
+{"timestamp":1707436671.5106862,"name":"undrain","context":{"idset":"61-68,77-92,109-114,116-172,178-237,239-301,304-307,309-356,381-383,385,387,421-468,493-500,502-549,551-620,622-636,654-677,679-708"}}
+{"timestamp":1707436682.653646,"name":"undrain","context":{"idset":"70-71,73-76,93-102,104-108,358-364,366-380,389-413,415-420,469-492,709-750,752-756"}}
+{"timestamp":1707436691.9775903,"name":"undrain","context":{"idset":"115,173-177,238,302-303,308,384,386,388,501,550,678"}}
+{"timestamp":1707436698.4863107,"name":"undrain","context":{"idset":"414"}}
+{"timestamp":1707436834.2522268,"name":"undrain","context":{"idset":"1-60"}}
+{"timestamp":1707436969.4478452,"name":"drain","context":{"idset":"115,173-177,238,302-303,308,384,386,388,501,550,678,751","reason":"reason=Slingslot Issues","overwrite":0}}
+{"timestamp":1707438312.3876901,"name":"drain","context":{"idset":"668","reason":"reason=Slingshot RDMA timeout/errors","overwrite":0}}
+{"timestamp":1707438516.6932464,"name":"drain","context":{"idset":"348","reason":"reason=BAD WRITE CHECKSUMs from this node","overwrite":0}}
+{"timestamp":1707440725.7445505,"name":"drain","context":{"idset":"40","reason":"reason=Failed to retrieve AMA","overwrite":0}}
+{"timestamp":1707440794.0440924,"name":"online","context":{"idset":"7"}}
+{"timestamp":1707440794.0875058,"name":"online","context":{"idset":"1"}}
+{"timestamp":1707440794.0966794,"name":"online","context":{"idset":"2"}}
+{"timestamp":1707440794.2763588,"name":"online","context":{"idset":"3,17-18,52"}}
+{"timestamp":1707440794.4161732,"name":"online","context":{"idset":"15,30,38"}}
+{"timestamp":1707440794.5196962,"name":"online","context":{"idset":"8,13,20-21,25,35,44,46-47,59-60"}}
+{"timestamp":1707440794.6254427,"name":"online","context":{"idset":"9-11,19,26-27,37,39,58"}}
+{"timestamp":1707440794.7323282,"name":"online","context":{"idset":"5,12,16,22-23,34,36,45"}}
+{"timestamp":1707440794.846946,"name":"online","context":{"idset":"4,6,28-29,41-43,48-49,53,55-56"}}
+{"timestamp":1707440794.9534748,"name":"online","context":{"idset":"24,33,51,57"}}
+{"timestamp":1707440795.1140285,"name":"online","context":{"idset":"14,31-32,50,54"}}
+{"timestamp":1707498338.6236048,"name":"drain","context":{"idset":"257","reason":"broker was unresponsive"}}
+{"timestamp":1707498400.623745,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1707505250.4484279,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1707505297.6321774,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1707505330.4610994,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1707505359.9232275,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1707505463.0930507,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1707510855.247272,"name":"online","context":{"idset":"40"}}
+{"timestamp":1707510864.0906918,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1707758062.8263419,"name":"drain","context":{"idset":"151","reason":"broker was unresponsive"}}
+{"timestamp":1707758063.2695007,"name":"drain","context":{"idset":"166","reason":"broker was unresponsive"}}
+{"timestamp":1707758067.6176951,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1707758067.9257121,"name":"drain","context":{"idset":"141","reason":"broker was unresponsive"}}
+{"timestamp":1707758068.261302,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1707758068.6189797,"name":"drain","context":{"idset":"150","reason":"broker was unresponsive"}}
+{"timestamp":1707758068.9555268,"name":"drain","context":{"idset":"153","reason":"broker was unresponsive"}}
+{"timestamp":1707758068.9556725,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1707758068.9557807,"name":"drain","context":{"idset":"190","reason":"broker was unresponsive"}}
+{"timestamp":1707758068.9558313,"name":"drain","context":{"idset":"264","reason":"broker was unresponsive"}}
+{"timestamp":1707758068.955884,"name":"drain","context":{"idset":"265","reason":"broker was unresponsive"}}
+{"timestamp":1707758068.9559317,"name":"drain","context":{"idset":"267","reason":"broker was unresponsive"}}
+{"timestamp":1707758068.9559731,"name":"drain","context":{"idset":"268","reason":"broker was unresponsive"}}
+{"timestamp":1707758068.9560165,"name":"drain","context":{"idset":"269","reason":"broker was unresponsive"}}
+{"timestamp":1707758085.8219562,"name":"drain","context":{"idset":"187","reason":"broker was unresponsive"}}
+{"timestamp":1707758092.5241265,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1707758092.5242217,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1707758092.5242791,"name":"drain","context":{"idset":"132","reason":"broker was unresponsive"}}
+{"timestamp":1707758092.5243194,"name":"drain","context":{"idset":"171","reason":"broker was unresponsive"}}
+{"timestamp":1707758092.52437,"name":"drain","context":{"idset":"178","reason":"broker was unresponsive"}}
+{"timestamp":1707758092.5244141,"name":"drain","context":{"idset":"192","reason":"broker was unresponsive"}}
+{"timestamp":1707758092.6239402,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1707758094.5229478,"name":"drain","context":{"idset":"138","reason":"broker was unresponsive"}}
+{"timestamp":1707758094.5230751,"name":"drain","context":{"idset":"182","reason":"broker was unresponsive"}}
+{"timestamp":1707758094.5231295,"name":"drain","context":{"idset":"186","reason":"broker was unresponsive"}}
+{"timestamp":1707758094.6229112,"name":"drain","context":{"idset":"188","reason":"broker was unresponsive"}}
+{"timestamp":1707758096.5230641,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1707758096.5232358,"name":"drain","context":{"idset":"169","reason":"broker was unresponsive"}}
+{"timestamp":1707758096.5233295,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1707758096.6235068,"name":"drain","context":{"idset":"262","reason":"broker was unresponsive"}}
+{"timestamp":1707758098.5241079,"name":"drain","context":{"idset":"152","reason":"broker was unresponsive"}}
+{"timestamp":1707758098.6248641,"name":"drain","context":{"idset":"163","reason":"broker was unresponsive"}}
+{"timestamp":1707758116.6233981,"name":"drain","context":{"idset":"133","reason":"broker was unresponsive"}}
+{"timestamp":1707758120.1516304,"name":"drain","context":{"idset":"146","reason":"broker was unresponsive"}}
+{"timestamp":1707758122.6230888,"name":"drain","context":{"idset":"142","reason":"broker was unresponsive"}}
+{"timestamp":1707758195.7718611,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1707758197.4965999,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1707758197.7561049,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1707758198.069608,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1707758198.305783,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1707758198.5483494,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1707758198.835242,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1707758199.090502,"name":"drain","context":{"idset":"221","reason":"broker was unresponsive"}}
+{"timestamp":1707758199.3096511,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1707758199.5535083,"name":"drain","context":{"idset":"228","reason":"broker was unresponsive"}}
+{"timestamp":1707758199.8250983,"name":"drain","context":{"idset":"230","reason":"broker was unresponsive"}}
+{"timestamp":1707758200.066638,"name":"drain","context":{"idset":"260","reason":"broker was unresponsive"}}
+{"timestamp":1707758200.340585,"name":"drain","context":{"idset":"270","reason":"broker was unresponsive"}}
+{"timestamp":1707758200.6147997,"name":"drain","context":{"idset":"275","reason":"broker was unresponsive"}}
+{"timestamp":1707758200.878428,"name":"drain","context":{"idset":"276","reason":"broker was unresponsive"}}
+{"timestamp":1707758201.1616771,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1707758201.4706059,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1707758201.7958157,"name":"drain","context":{"idset":"310","reason":"broker was unresponsive"}}
+{"timestamp":1707758202.0803478,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1707758202.3838408,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.0304141,"name":"drain","context":{"idset":"199","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3319764,"name":"drain","context":{"idset":"204","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3323269,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3323786,"name":"drain","context":{"idset":"216","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3324232,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.332473,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3325136,"name":"drain","context":{"idset":"231","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3325522,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3325937,"name":"drain","context":{"idset":"236","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3326342,"name":"drain","context":{"idset":"247","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3332,"name":"drain","context":{"idset":"279","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3336716,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3341904,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3342257,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3342702,"name":"drain","context":{"idset":"293","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3343132,"name":"drain","context":{"idset":"297","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3343546,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3343971,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3344603,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3345468,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3345954,"name":"drain","context":{"idset":"220","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3346431,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3346877,"name":"drain","context":{"idset":"227","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3347538,"name":"drain","context":{"idset":"234","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3347962,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3348403,"name":"drain","context":{"idset":"259","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3348882,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3349328,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3349771,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3350286,"name":"drain","context":{"idset":"288","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3350739,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3351409,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3351898,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3352559,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3353202,"name":"drain","context":{"idset":"314","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3353629,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1707758203.3354034,"name":"drain","context":{"idset":"325","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5233669,"name":"drain","context":{"idset":"205","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5234675,"name":"drain","context":{"idset":"209","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5235579,"name":"drain","context":{"idset":"217","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5236251,"name":"drain","context":{"idset":"224","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5236919,"name":"drain","context":{"idset":"232","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5237567,"name":"drain","context":{"idset":"237","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5238304,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5238972,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.523963,"name":"drain","context":{"idset":"248","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5240421,"name":"drain","context":{"idset":"271","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5241115,"name":"drain","context":{"idset":"273","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5241778,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5242445,"name":"drain","context":{"idset":"291","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5243144,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5243812,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5244479,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.5245156,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1707758206.6232378,"name":"drain","context":{"idset":"324","reason":"broker was unresponsive"}}
+{"timestamp":1707758215.9260287,"name":"drain","context":{"idset":"203","reason":"broker was unresponsive"}}
+{"timestamp":1707758219.0339756,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1707758219.3117564,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1707758230.5245521,"name":"drain","context":{"idset":"196","reason":"broker was unresponsive"}}
+{"timestamp":1707758230.5246971,"name":"drain","context":{"idset":"200","reason":"broker was unresponsive"}}
+{"timestamp":1707758230.524785,"name":"drain","context":{"idset":"202","reason":"broker was unresponsive"}}
+{"timestamp":1707758230.5248742,"name":"drain","context":{"idset":"244","reason":"broker was unresponsive"}}
+{"timestamp":1707758230.6251211,"name":"drain","context":{"idset":"272","reason":"broker was unresponsive"}}
+{"timestamp":1707758232.522861,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1707758232.522975,"name":"drain","context":{"idset":"252","reason":"broker was unresponsive"}}
+{"timestamp":1707758232.523108,"name":"drain","context":{"idset":"253","reason":"broker was unresponsive"}}
+{"timestamp":1707758232.6236539,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1707758238.6235902,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1707758251.9418776,"name":"drain","context":{"idset":"326","reason":"broker was unresponsive"}}
+{"timestamp":1707758254.5223365,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1707758254.5224435,"name":"drain","context":{"idset":"261","reason":"broker was unresponsive"}}
+{"timestamp":1707758254.5225172,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1707758254.6223567,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1707758256.5244858,"name":"drain","context":{"idset":"233","reason":"broker was unresponsive"}}
+{"timestamp":1707758256.524569,"name":"drain","context":{"idset":"245","reason":"broker was unresponsive"}}
+{"timestamp":1707758256.5246215,"name":"drain","context":{"idset":"256","reason":"broker was unresponsive"}}
+{"timestamp":1707758256.5246742,"name":"drain","context":{"idset":"274","reason":"broker was unresponsive"}}
+{"timestamp":1707758256.5247197,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1707758258.5237515,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1707758258.5238543,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1707758258.5239191,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1707758258.6244318,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1707758416.5829248,"name":"drain","context":{"idset":"330","reason":"broker was unresponsive"}}
+{"timestamp":1707758416.9213564,"name":"drain","context":{"idset":"360","reason":"broker was unresponsive"}}
+{"timestamp":1707758419.2284575,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1707758419.2285454,"name":"drain","context":{"idset":"350","reason":"broker was unresponsive"}}
+{"timestamp":1707758419.6963875,"name":"drain","context":{"idset":"383","reason":"broker was unresponsive"}}
+{"timestamp":1707758419.6964514,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1707758419.6964889,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1707758420.9380829,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1707758420.9381747,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1707758420.9382191,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1707758420.9382515,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1707758420.9382873,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1707758421.310801,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1707758421.3108902,"name":"drain","context":{"idset":"402","reason":"broker was unresponsive"}}
+{"timestamp":1707758421.3114095,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1707758432.5232587,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1707758432.5233445,"name":"drain","context":{"idset":"351","reason":"broker was unresponsive"}}
+{"timestamp":1707758432.5233943,"name":"drain","context":{"idset":"355","reason":"broker was unresponsive"}}
+{"timestamp":1707758432.6238964,"name":"drain","context":{"idset":"357","reason":"broker was unresponsive"}}
+{"timestamp":1707758450.9256656,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1707758451.2867508,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1707758456.5243418,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1707758456.6245883,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1707758466.7299225,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1707758467.1130071,"name":"drain","context":{"idset":"369","reason":"broker was unresponsive"}}
+{"timestamp":1707758468.5339668,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1707758469.0580955,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1707758472.1378779,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1707758472.1380265,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1707758472.1380703,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1707758476.5234313,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1707758476.6239202,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1707758478.6236322,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1707758485.8036954,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1707761262.8398883,"name":"undrain","context":{"idset":"127,129-130,132,138,140-141,147,150-153,163,165-166,169,171,178,182,186-188,190-193,262,264-265,267-269"}}
+{"timestamp":1707761272.0351818,"name":"undrain","context":{"idset":"133,142,146"}}
+{"timestamp":1707761278.1271689,"name":"undrain","context":{"idset":"194,196,200,202,218,233,244-246,250,252-253,256,261,272,274,278,281,286,304,307,315,319,326"}}
+{"timestamp":1707761308.4506173,"name":"undrain","context":{"idset":"199,203-206,208-217,219-228,230-232,234-237,239,243,247-248,251,259-260,270-271,273,275-276,279-280,282-285,287-293,296-297,299-300,305,309-314,316-318,320-325,327"}}
+{"timestamp":1707761340.0000601,"name":"undrain","context":{"idset":"330-331,336,343-344,350-351,355-357,360,370-371,373,379,383,385,392-393,397,402,404,408"}}
+{"timestamp":1707761349.1583865,"name":"undrain","context":{"idset":"338,349,359,363-364,369,391,394,398,400-401"}}
+{"timestamp":1707761394.4320669,"name":"online","context":{"idset":"257"}}
+{"timestamp":1707761419.6104774,"name":"undrain","context":{"idset":"257"}}
+{"timestamp":1707762644.8812275,"name":"undrain","context":{"idset":"115"}}
+{"timestamp":1707762647.8810673,"name":"drain","context":{"idset":"115","reason":"reason=Drops to UEFI shell on boot","overwrite":0}}
+{"timestamp":1707762703.5934386,"name":"undrain","context":{"idset":"173-177,238,302-303,308,384,386,388,501,550,678,751"}}
+{"timestamp":1707762722.999548,"name":"drain","context":{"idset":"173","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762727.720232,"name":"drain","context":{"idset":"174","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762732.4814703,"name":"drain","context":{"idset":"175","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762737.6935868,"name":"drain","context":{"idset":"176","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762741.8386209,"name":"drain","context":{"idset":"177","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762774.7029541,"name":"drain","context":{"idset":"238","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762783.8695598,"name":"drain","context":{"idset":"302","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762787.4225008,"name":"drain","context":{"idset":"304","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762791.3636012,"name":"drain","context":{"idset":"303","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762797.8650391,"name":"undrain","context":{"idset":"304"}}
+{"timestamp":1707762818.9545493,"name":"drain","context":{"idset":"308","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762907.9360695,"name":"drain","context":{"idset":"348","reason":"reason=cxi1[hsi1]: C_EC_UNCOR_NS: C1_HNI error: tx_flit_ucor (25) + BAD WRITE CHECKSUM","overwrite":1}}
+{"timestamp":1707762927.3457448,"name":"drain","context":{"idset":"384","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762932.5336778,"name":"drain","context":{"idset":"386","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762936.4478533,"name":"drain","context":{"idset":"388","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762955.2379341,"name":"drain","context":{"idset":"474","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762962.5267787,"name":"drain","context":{"idset":"501","reason":"reason=cxi0[hsi0] link != up. Actual value = starting","overwrite":0}}
+{"timestamp":1707762968.3723316,"name":"undrain","context":{"idset":"474"}}
+{"timestamp":1707763006.6207442,"name":"drain","context":{"idset":"515","reason":"reason=*** FAIL: cxi0 Good Codeword rate=30187347.1903/s (expecting >= 35000000/s)","overwrite":0}}
+{"timestamp":1707763119.3080339,"name":"drain","context":{"idset":"550","reason":"reason=cxi1[hsi1]: p0: resetting serdes PLLs","overwrite":0}}
+{"timestamp":1707763160.3862872,"name":"drain","context":{"idset":"678","reason":"reason=cxi0[hsi0] link != up. Actual value = starting.","overwrite":0}}
+{"timestamp":1707763165.571362,"name":"drain","context":{"idset":"751","reason":"reason=cxi0[hsi0] link != up. Actual value = starting.","overwrite":0}}
+{"timestamp":1707764574.4539936,"name":"resource-init","context":{"restart":true,"drain":{"115":{"timestamp":1707762647.8810673,"reason":"reason=Drops to UEFI shell on boot"},"173":{"timestamp":1707762722.999548,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"174":{"timestamp":1707762727.720232,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"175":{"timestamp":1707762732.4814703,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"176":{"timestamp":1707762737.6935868,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"177":{"timestamp":1707762741.8386209,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"238":{"timestamp":1707762774.7029541,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"302":{"timestamp":1707762783.8695598,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"303":{"timestamp":1707762791.3636012,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"308":{"timestamp":1707762818.9545493,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"348":{"timestamp":1707438516.6932464,"reason":"reason=cxi1[hsi1]: C_EC_UNCOR_NS: C1_HNI error: tx_flit_ucor (25) + BAD WRITE CHECKSUM"},"384":{"timestamp":1707762927.3457448,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"386":{"timestamp":1707762932.5336778,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"388":{"timestamp":1707762936.4478533,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"501":{"timestamp":1707762962.5267787,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"515":{"timestamp":1707763006.6207442,"reason":"reason=*** FAIL: cxi0 Good Codeword rate=30187347.1903/s (expecting >= 35000000/s)"},"550":{"timestamp":1707763119.3080339,"reason":"reason=cxi1[hsi1]: p0: resetting serdes PLLs"},"668":{"timestamp":1707438312.3876901,"reason":"reason=Slingshot RDMA timeout/errors"},"678":{"timestamp":1707763160.3862872,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."},"751":{"timestamp":1707763165.571362,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."}},"online":"","exclude":"0"}}
+{"timestamp":1707764574.4796629,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1707764587.663693,"name":"online","context":{"idset":"0"}}
+{"timestamp":1707764588.5030415,"name":"online","context":{"idset":"57-60"}}
+{"timestamp":1707764589.504518,"name":"online","context":{"idset":"84,240,324,393"}}
+{"timestamp":1707764589.6448264,"name":"online","context":{"idset":"20,34,72,122,168,218,318,330,387,417,419"}}
+{"timestamp":1707764589.7483232,"name":"online","context":{"idset":"41,92,161,180,199,227,273,297,307,312,325,339,358,374,391,430,437,442"}}
+{"timestamp":1707764589.8573294,"name":"online","context":{"idset":"2,9,29-30,56,63,91,99,126,132,135,139,145,148,151,157,166,170,182-183,185-186,195,205,207,221,247,251,256,266,309,320-321,331,349,360,366,369-370,382,392,396,398-399,401,403,409,423,426-427,433-434,444"}}
+{"timestamp":1707764589.959893,"name":"online","context":{"idset":"16,25,31,38,40,46,48,52,61,66-68,71,79,81,86,88,96,106-108,113,121,123,130,133,140,142,146,150,153,158,167,169,171,184,188,192,198,211,217,219,230,232,241,249,253,260,274,293,296,298,300,322,326-329,338,341-345,350,355-356,363,365,380,390,394,402,408,413,415-416,420,422,428-429,432,435,446-447,563,653"}}
+{"timestamp":1707764590.0610104,"name":"online","context":{"idset":"5,18-19,24,70,73,76,78,85,87,94,97,101,104,112,114,118,137-138,143-144,154,159,162-163,165,179,181,189,196,200,203,208-209,212,215,223,231,233-234,237,243,245,257,259,267-268,270-272,278,281,289,299,301,311,313-314,319,352-353,357,361-362,364,372-373,375-378,395,404-405,410,414,418,421,424,438,455,476,581"}}
+{"timestamp":1707764590.1626279,"name":"online","context":{"idset":"12,21,51,65,69,77,80,89,93,95,98,100,102-103,110,119-120,125,127-129,136,155,172,178,187,190-191,193,201-202,213,226,269,276-277,279,283,285,290-291,294-295,304,315-316,336,346-347,351,354,368,381,383,385,389,400,406,411-412,425,431,568,732"}}
+{"timestamp":1707764590.2631235,"name":"online","context":{"idset":"6,13,141,147,149,152,194,197,206,228-229,239,244,258,262,332,367,443,735"}}
+{"timestamp":1707764590.3709712,"name":"online","context":{"idset":"11,14,53,55,82-83,109,111,131,156,160,164,210,224-225,235,246,250,254,261,263,265,282,284,292,305,317,334-335,340,359,379,439,441,445,483,524,528,533,598,633,748"}}
+{"timestamp":1707764590.4722753,"name":"online","context":{"idset":"45,64,74-75,117,124,134,236,248,286,310,397,407,436,572,668,698"}}
+{"timestamp":1707764590.5897512,"name":"online","context":{"idset":"27,49,90,204,214,287,333,337,371,471,486-487,579,624,666,707"}}
+{"timestamp":1707764590.7272441,"name":"online","context":{"idset":"23,37,39,50,62,220,280,474,496,512,520,531,537,557,706"}}
+{"timestamp":1707764590.8444226,"name":"online","context":{"idset":"3,8,42,47,116,216,252,275,288,323,451,463,495,570,576,593,595,631,641,709,722,725,736"}}
+{"timestamp":1707764590.9511693,"name":"online","context":{"idset":"4,10,28,32,54,348,475,478,480,493,503,509,517,521-522,529,534,543-544,558,582,599,612,625,627,645,650-651,660,663,665,681,720,723,745-746"}}
+{"timestamp":1707764591.0576074,"name":"online","context":{"idset":"306,458,490,492,507,513-514,516,547,561-562,569,616,643,658,664,682,689,692,696,728,740,744,749"}}
+{"timestamp":1707764591.1616347,"name":"online","context":{"idset":"1,7,26,449,491,494,500,504,536,541,573,577,600,606-607,611,614,700,710,712,717-718,726,747,750,752"}}
+{"timestamp":1707764591.2649286,"name":"online","context":{"idset":"35,44,264,454,465,468,470,473,481,511,519,527,530,540,548,556,571,574,587,596,602,608-609,613,615,621,632,634-635,637,646-647,656-657,674,683,685,693,721,743"}}
+{"timestamp":1707764591.3651485,"name":"online","context":{"idset":"15,17,36,43,105,222,450,453,457,459,489,499,506,523,525,532,538,542,549,554-555,566,585,588,597,601,604,617,619,622,630,638,642,644,652,673,675-676,679,684,690-691,695,703,714,727,737,739,741,756"}}
+{"timestamp":1707764591.4656982,"name":"online","context":{"idset":"33,448,456,464,466-467,472,484-485,498,502,526,545-546,580,590,618,620,623,640,648,655,659,672,688,699,702,704,708,713,729-730,753"}}
+{"timestamp":1707764591.5699956,"name":"online","context":{"idset":"22,469,518,535,553,559-560,594,639,661-662,667,701,711,716,719,724,738,754"}}
+{"timestamp":1707764591.6949732,"name":"online","context":{"idset":"460,462,477,482,539,552,578,584,603,680"}}
+{"timestamp":1707764591.8105111,"name":"online","context":{"idset":"452,461,479,488,497,551,565,589,629,636,649,654,697,715,731,733-734,755"}}
+{"timestamp":1707764591.9116259,"name":"online","context":{"idset":"505,508,510,564,567,575,586,605,610,626,628,670-671,687,694"}}
+{"timestamp":1707764592.0194254,"name":"online","context":{"idset":"515,583,669,677,705,742"}}
+{"timestamp":1707765292.6025467,"name":"drain","context":{"idset":"355","reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err","overwrite":0}}
+{"timestamp":1707765297.7466619,"name":"drain","context":{"idset":"387","reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err","overwrite":0}}
+{"timestamp":1707765302.0042863,"name":"drain","context":{"idset":"398","reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err","overwrite":0}}
+{"timestamp":1707765306.791353,"name":"drain","context":{"idset":"496","reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err","overwrite":0}}
+{"timestamp":1707765313.5718749,"name":"drain","context":{"idset":"589","reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err","overwrite":0}}
+{"timestamp":1707765427.3514349,"name":"drain","context":{"idset":"525","reason":"reason=*** FAIL: cxi0 Good Codeword rate=30453213.5106/s (expecting >= 35000000/s)","overwrite":0}}
+{"timestamp":1707770954.6759088,"name":"drain","context":{"idset":"2","reason":"prolog failed for jobid fcomRrAfLrT","overwrite":0}}
+{"timestamp":1707774592.9740059,"name":"drain","context":{"idset":"127-128,145-148,150-151,168-170,178","reason":"prolog failed for jobid fcpF4zyBUfh","overwrite":0}}
+{"timestamp":1707848679.6268177,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1707848898.6244495,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1707855877.2615316,"name":"resource-init","context":{"restart":true,"drain":{"2":{"timestamp":1707770954.6759088,"reason":"prolog failed for jobid fcomRrAfLrT"},"115":{"timestamp":1707762647.8810673,"reason":"reason=Drops to UEFI shell on boot"},"173":{"timestamp":1707762722.999548,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"174":{"timestamp":1707762727.720232,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"175":{"timestamp":1707762732.4814703,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"176":{"timestamp":1707762737.6935868,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"177":{"timestamp":1707762741.8386209,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"127-128,145-148,150-151,168-170,178":{"timestamp":1707774592.9740059,"reason":"prolog failed for jobid fcpF4zyBUfh"},"238":{"timestamp":1707762774.7029541,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"302":{"timestamp":1707762783.8695598,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"303":{"timestamp":1707762791.3636012,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"308":{"timestamp":1707762818.9545493,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"348":{"timestamp":1707438516.6932464,"reason":"reason=cxi1[hsi1]: C_EC_UNCOR_NS: C1_HNI error: tx_flit_ucor (25) + BAD WRITE CHECKSUM"},"355":{"timestamp":1707765292.6025467,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"384":{"timestamp":1707762927.3457448,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"386":{"timestamp":1707762932.5336778,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"387":{"timestamp":1707765297.7466619,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"388":{"timestamp":1707762936.4478533,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"398":{"timestamp":1707765302.0042863,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"496":{"timestamp":1707765306.791353,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"501":{"timestamp":1707762962.5267787,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"515":{"timestamp":1707763006.6207442,"reason":"reason=*** FAIL: cxi0 Good Codeword rate=30187347.1903/s (expecting >= 35000000/s)"},"525":{"timestamp":1707765427.3514349,"reason":"reason=*** FAIL: cxi0 Good Codeword rate=30453213.5106/s (expecting >= 35000000/s)"},"550":{"timestamp":1707763119.3080339,"reason":"reason=cxi1[hsi1]: p0: resetting serdes PLLs"},"589":{"timestamp":1707765313.5718749,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"668":{"timestamp":1707438312.3876901,"reason":"reason=Slingshot RDMA timeout/errors"},"678":{"timestamp":1707763160.3862872,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."},"751":{"timestamp":1707763165.571362,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."}},"online":"","exclude":"0"}}
+{"timestamp":1707855877.2829058,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1707855898.076144,"name":"online","context":{"idset":"0"}}
+{"timestamp":1707855898.9752076,"name":"online","context":{"idset":"54-60"}}
+{"timestamp":1707855899.4221237,"name":"online","context":{"idset":"348,515,668"}}
+{"timestamp":1707855899.7050493,"name":"online","context":{"idset":"20-21,39,51-52"}}
+{"timestamp":1707855899.8093185,"name":"online","context":{"idset":"5,7,9,11,13,16,19,24,27,29,34,40-41,46,48,207,370"}}
+{"timestamp":1707855899.9120762,"name":"online","context":{"idset":"1-4,12,14,18,23,25,28,30-31,35,37,49-50,96,153,180,227,307,350,358,374,408,426,476,481,483,487,582,624,665"}}
+{"timestamp":1707855900.0138755,"name":"online","context":{"idset":"6,8,10,26,33,38,43-45,47,53,81,102,135,198,240,251,356,393,409,422-423,427,429,437,455-456,502,524,563,581,653,656,666"}}
+{"timestamp":1707855900.1187363,"name":"online","context":{"idset":"17,32,36,42,66-67,97,126,132,157-158,182-183,185-186,195,223,233,247,266,281,300-301,318,325,328,339,360,366,369,380,382,394,399,401,420,428,430,434,444,474,504,512,520-521,523,536,548,590,612,625,627,630,641,650,660"}}
+{"timestamp":1707855900.2209275,"name":"online","context":{"idset":"15,63,73,86,92-93,103,105-107,118,121,123,130,133,139,161,171,179,188,205,208,211,218,221,230,234,253,260,267,270,273-274,283,290,293,296,298,313,324,327,338,343-344,349,361,365,375,377,392,405,415-417,419,424,432,442,446,448,451,463-465,471,475,478,486,489,493,495,509,513,517,519,529,531,533,542-543,547,557,561,568,572,576,579,593,595,599,604,611,616,621,623,631,633,639,642-643,646,651,663,673,679,698,736"}}
+{"timestamp":1707855900.323549,"name":"online","context":{"idset":"65,71-72,75-76,78-79,82,84,88,90-91,98-99,101,104,108,110-114,117,120,122,125,129,136,138,140-146,148-149,151,154,159,162-163,165-166,168,170,172,184,191-193,196,200-203,215,217,219,226,228-229,231-232,236,241,243,245,248-249,252,254,256,258-259,262,265,268-269,271,277-279,284,287,289,294,304,309,311,314-316,320-321,326,329-330,332,334,340-342,345-346,351,354,362-364,368,372-373,376,378-379,383,385,390-391,396,400,402-404,407,410-411,413,421,425,431,433,435,438,447,450,453-454,457-459,462,466-468,470,472-473,480,484,491-492,499-500,503,506-507,511,514,516,522,526-527,530,532,534-535,539-541,544-546,553-556,558-559,562,566,569,573-574,577,585,587-588,596-598,600,602-603,606-609,614-615,617,622,632,635,637-638,644-645,647-648,652,657-658,661-662,664,667,674,676,706-707,709,723,732,746,748"}}
+{"timestamp":1707855900.4249752,"name":"online","context":{"idset":"22,61-62,64,68-70,74,77,80,83,85,87,89,94-95,109,116,119,124,131,134,137,147,150,152,155-156,160,164,167,169,181,187,189-190,194,197,199,204,206,209-210,212-214,216,220,222,225,235,237,239,244,246,250,257,261,263-264,272,275-276,280,282,285-286,288,291-292,295,297,299,305-306,310,312,317,319,322-323,331,333,335-337,347,352-353,355,357,359,367,371,381,389,397,406,412,414,418,436,439,441,443,445,449,452,460-461,469,477,479,482,485,488,490,494,497-498,505,508,510,518,528,537-538,549,551-552,560,564-565,567,570-571,578,580,583-584,586,594,601,610,613,618-620,626,628-629,634,636,640,649,654-655,659,669-672,675,677,681,689,692-693,696,710,717,720,725,744-745,750,752,754,756"}}
+{"timestamp":1707855900.5407724,"name":"online","context":{"idset":"100,178,224,387,525,575,589,605,683-685,688,691,699,705,712,714,719,722,727,729,739-741,743,747"}}
+{"timestamp":1707855900.6415551,"name":"online","context":{"idset":"687,701,704,711,713,716,721,726,730,738,749"}}
+{"timestamp":1707855900.9032321,"name":"online","context":{"idset":"128,682,703,718,724,733,755"}}
+{"timestamp":1707855901.0343096,"name":"online","context":{"idset":"127,680,690,695,700,702,708,715,728,731,734-735,737,742,753"}}
+{"timestamp":1707855901.1700826,"name":"online","context":{"idset":"694,697"}}
+{"timestamp":1707931944.922622,"name":"drain","context":{"idset":"39","reason":"reason=Bad Link cxi0","overwrite":0}}
+{"timestamp":1707932554.7650969,"name":"drain","context":{"idset":"39","reason":"epilog failed for jobid fd1e3nxTyGT","overwrite":0}}
+{"timestamp":1707932554.8632236,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1708008739.3440754,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1708008801.3425488,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1708023465.9582236,"name":"resource-init","context":{"restart":true,"drain":{"2":{"timestamp":1707770954.6759088,"reason":"prolog failed for jobid fcomRrAfLrT"},"39":{"timestamp":1707931944.922622,"reason":"reason=Bad Link cxi0"},"115":{"timestamp":1707762647.8810673,"reason":"reason=Drops to UEFI shell on boot"},"173":{"timestamp":1707762722.999548,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"174":{"timestamp":1707762727.720232,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"175":{"timestamp":1707762732.4814703,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"176":{"timestamp":1707762737.6935868,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"177":{"timestamp":1707762741.8386209,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"127-128,145-148,150-151,168-170,178":{"timestamp":1707774592.9740059,"reason":"prolog failed for jobid fcpF4zyBUfh"},"238":{"timestamp":1707762774.7029541,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"302":{"timestamp":1707762783.8695598,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"303":{"timestamp":1707762791.3636012,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"308":{"timestamp":1707762818.9545493,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"348":{"timestamp":1707438516.6932464,"reason":"reason=cxi1[hsi1]: C_EC_UNCOR_NS: C1_HNI error: tx_flit_ucor (25) + BAD WRITE CHECKSUM"},"353":{"timestamp":1708008739.3440754,"reason":"broker was unresponsive"},"355":{"timestamp":1707765292.6025467,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"384":{"timestamp":1707762927.3457448,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"386":{"timestamp":1707762932.5336778,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"387":{"timestamp":1707765297.7466619,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"388":{"timestamp":1707762936.4478533,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"398":{"timestamp":1707765302.0042863,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"496":{"timestamp":1707765306.791353,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"501":{"timestamp":1707762962.5267787,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"515":{"timestamp":1707763006.6207442,"reason":"reason=*** FAIL: cxi0 Good Codeword rate=30187347.1903/s (expecting >= 35000000/s)"},"525":{"timestamp":1707765427.3514349,"reason":"reason=*** FAIL: cxi0 Good Codeword rate=30453213.5106/s (expecting >= 35000000/s)"},"550":{"timestamp":1707763119.3080339,"reason":"reason=cxi1[hsi1]: p0: resetting serdes PLLs"},"589":{"timestamp":1707765313.5718749,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"668":{"timestamp":1707438312.3876901,"reason":"reason=Slingshot RDMA timeout/errors"},"678":{"timestamp":1707763160.3862872,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."},"751":{"timestamp":1707763165.571362,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."}},"online":"","exclude":"0"}}
+{"timestamp":1708023465.9842961,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1708023490.0881379,"name":"online","context":{"idset":"0"}}
+{"timestamp":1708023491.0057845,"name":"online","context":{"idset":"2,58-60"}}
+{"timestamp":1708023491.3413794,"name":"online","context":{"idset":"12,21,51,349,381,415,430,439,477,639,641-642,644-645,647-648,650,722"}}
+{"timestamp":1708023491.4438007,"name":"online","context":{"idset":"1,3,5-11,13-17,19-20,22,25,27,30,32,34-35,37-38,40-43,45-46,48-49,52,54-55,57,127-128,145-148,150-151,168-170,178,342-348,350-352,354-375,377-380,382-383,385,387,389-390,392-394,397,399-400,402-406,408-414,416-419,421,423-429,431-433,435-438,441-476,478-495,497-499,502-508,510-549,551-590,593-638,640,643,646,649,651-677,679-685,687-717,719-721,723-750,752-756"}}
+{"timestamp":1708023491.544174,"name":"online","context":{"idset":"4,18,23-24,26,28-29,31,33,36,44,47,50,53,56,171,185,290,718"}}
+{"timestamp":1708023491.6709833,"name":"online","context":{"idset":"63-64,66-67,81,84,87,90,94,97-98,104,107,110,122-123,130,135-136,138-139,143,157,162,166-167,183,191-192,194-197,199,201,205,211-213,215,218,226,229-230,237,240-241,243,245,250,262-263,266,269,277-278,280,293,296,300,309,315-316,318,320-321,325,328,332,334,337,341,376,391,396,401,420,422,509"}}
+{"timestamp":1708023491.7717688,"name":"online","context":{"idset":"61-62,65,68-72,74-80,82-83,85-86,88,91-92,95-96,99-103,105-106,108-109,111-113,118-121,124-125,129,131-133,137,140-142,144,149,152-156,158,160-161,163-165,172,179-182,184,186-190,193,198,200,202-204,207,209-210,214,216-217,219,221-225,227-228,231-234,236,239,244,247,249,251-254,256-261,264-265,267-268,270-276,279,281-289,291-292,294-295,297-299,301,304-307,310-312,314,317,319,322-324,326-327,329-331,333,335-336,339-340,407,434,500"}}
+{"timestamp":1708023491.8797057,"name":"online","context":{"idset":"73,89,93,114,116-117,126,159,206,208,235,246,313,338"}}
+{"timestamp":1708023491.992831,"name":"online","context":{"idset":"134,220,248"}}
+{"timestamp":1708023926.0827367,"name":"drain","context":{"idset":"248","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1708028647.4673784,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1708125925.9665933,"name":"drain","context":{"idset":"123","reason":"reason=IOR timeouts","overwrite":0}}
+{"timestamp":1708125929.9910574,"name":"drain","context":{"idset":"117","reason":"reason=IOR timeouts","overwrite":0}}
+{"timestamp":1708132127.265264,"name":"drain","context":{"idset":"314,320-347,352,354,356-357","reason":"prolog failed for jobid fdconJq7sd1","overwrite":0}}
+{"timestamp":1708148996.1763897,"name":"resource-init","context":{"restart":true,"drain":{"2":{"timestamp":1707770954.6759088,"reason":"prolog failed for jobid fcomRrAfLrT"},"39":{"timestamp":1707931944.922622,"reason":"reason=Bad Link cxi0"},"115":{"timestamp":1707762647.8810673,"reason":"reason=Drops to UEFI shell on boot"},"117":{"timestamp":1708125929.9910574,"reason":"reason=IOR timeouts"},"123":{"timestamp":1708125925.9665933,"reason":"reason=IOR timeouts"},"173":{"timestamp":1707762722.999548,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"174":{"timestamp":1707762727.720232,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"175":{"timestamp":1707762732.4814703,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"176":{"timestamp":1707762737.6935868,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"177":{"timestamp":1707762741.8386209,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"127-128,145-148,150-151,168-170,178":{"timestamp":1707774592.9740059,"reason":"prolog failed for jobid fcpF4zyBUfh"},"238":{"timestamp":1707762774.7029541,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"248":{"timestamp":1708023926.0827367,"reason":"nodediag failed clocksource"},"302":{"timestamp":1707762783.8695598,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"303":{"timestamp":1707762791.3636012,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"308":{"timestamp":1707762818.9545493,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"348":{"timestamp":1707438516.6932464,"reason":"reason=cxi1[hsi1]: C_EC_UNCOR_NS: C1_HNI error: tx_flit_ucor (25) + BAD WRITE CHECKSUM"},"353":{"timestamp":1708008739.3440754,"reason":"broker was unresponsive"},"355":{"timestamp":1707765292.6025467,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"314,320-347,352,354,356-357":{"timestamp":1708132127.265264,"reason":"prolog failed for jobid fdconJq7sd1"},"384":{"timestamp":1707762927.3457448,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"386":{"timestamp":1707762932.5336778,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"387":{"timestamp":1707765297.7466619,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"388":{"timestamp":1707762936.4478533,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"398":{"timestamp":1707765302.0042863,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"496":{"timestamp":1707765306.791353,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"501":{"timestamp":1707762962.5267787,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"515":{"timestamp":1707763006.6207442,"reason":"reason=*** FAIL: cxi0 Good Codeword rate=30187347.1903/s (expecting >= 35000000/s)"},"525":{"timestamp":1707765427.3514349,"reason":"reason=*** FAIL: cxi0 Good Codeword rate=30453213.5106/s (expecting >= 35000000/s)"},"550":{"timestamp":1707763119.3080339,"reason":"reason=cxi1[hsi1]: p0: resetting serdes PLLs"},"589":{"timestamp":1707765313.5718749,"reason":"reason=cxi0[hsi0]: C_EC_BADCON_NS: C1_HNI error: rx_pause_err"},"668":{"timestamp":1707438312.3876901,"reason":"reason=Slingshot RDMA timeout/errors"},"678":{"timestamp":1707763160.3862872,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."},"751":{"timestamp":1707763165.571362,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."}},"online":"","exclude":"0"}}
+{"timestamp":1708148996.203058,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1708149023.2603023,"name":"online","context":{"idset":"0"}}
+{"timestamp":1708149024.4443574,"name":"online","context":{"idset":"2"}}
+{"timestamp":1708149024.619566,"name":"online","context":{"idset":"127-128,145-148,150-151,168-170,178,355,387,515,525,589,668"}}
+{"timestamp":1708149025.1075623,"name":"online","context":{"idset":"251"}}
+{"timestamp":1708149025.2998545,"name":"online","context":{"idset":"76,166,432"}}
+{"timestamp":1708149025.4493926,"name":"online","context":{"idset":"96,158,195,394,557,568,576"}}
+{"timestamp":1708149025.5639031,"name":"online","context":{"idset":"64,68,186,309,366,369,393,399,403,413,423,427,455,478,509,513"}}
+{"timestamp":1708149025.7072573,"name":"online","context":{"idset":"66-67,136,185,212,217,237,268,307,313,358,364,396,416,422,481,520,524,593"}}
+{"timestamp":1708149025.8132291,"name":"online","context":{"idset":"60,81,112,129-130,139-140,180,184,207,213,221,231,234,240,259,294,318,363,365,374,380,419,421,430,446-447,468,490,493,531-532,534,543,563,607"}}
+{"timestamp":1708149025.9149518,"name":"online","context":{"idset":"63,72,122,133,153-154,157,159,182-183,188,192,223,241,281,319,368,392,408-410,442,448,470,521-523,529,537,544,582,585"}}
+{"timestamp":1708149026.0153017,"name":"online","context":{"idset":"83,86,106,121,142,161,196,205,211,227,278,293,316,350,370,417,429,437,463,465,467,473,483,495,511,542,555,558,561,571,579,581,597,614,616"}}
+{"timestamp":1708149026.1157463,"name":"online","context":{"idset":"73,91,126,171,209,263,269,296,298,385,391,434,449,471,476,486,492,517,530,600,602,604,608"}}
+{"timestamp":1708149026.2186649,"name":"online","context":{"idset":"5,9,19-20,58,101-102,113,118,132,143,179,219,260,270,273,300,343,360,373,401,404,420,424,433,435,450-453,474,484,512,516,533,547,562,580,599"}}
+{"timestamp":1708149026.3212874,"name":"online","context":{"idset":"21,59,69,74,78,84,135,201,228,230,232,247,249,266,272,276,285,312,329,341,375,383,405,415,426,444,494,499,519,535-536,595,609,660,725,745"}}
+{"timestamp":1708149026.4238236,"name":"online","context":{"idset":"25,40,51,70,79-80,99,149,189,200,202,208,229,233,236,245,253,262,274,279,289,297,305,311,339,361,377-378,390,428,454,456,475,480,487,503-504,514,541,553,566,572-573,588,615,617,653,723,736"}}
+{"timestamp":1708149026.5280941,"name":"online","context":{"idset":"11,16,34,48-49,61,65,82,87-88,94,123,141,172,203,215,226,243,256,258,287,301,315,325,351,379,402,457,491,500,526-527,570,574,577,587,590,598,611,624,627,641,643,663,666,681,698,709,748"}}
+{"timestamp":1708149026.6333964,"name":"online","context":{"idset":"12,46,52,90,92,100,110,138,155,162,165,190,267,290,299,328,338,359,372,418,438,466,506,528,556,569,594,596,613,631,689,706-707,720,740,746"}}
+{"timestamp":1708149026.7396083,"name":"online","context":{"idset":"23,27,30,38,71,89,98,114,152,163,167,191,193,199,244,261,277,304,321,327,345,357,362,376,414,425,460,539,545,548,606,612,638,650,664,679,696,722,727,749"}}
+{"timestamp":1708149026.8507183,"name":"online","context":{"idset":"6,18,24,29,31,41-42,93,109,181,197-198,218,248,254,257,265,283-284,332,344,349,389,464,489,502,507,540,554,583,621,646,652,658,674,676,682,684,693,718,726,732"}}
+{"timestamp":1708149026.9579356,"name":"online","context":{"idset":"1,7-8,10,14,55,77,97,107,119-120,125,137,156,194,246,271,280,282,286,291,320,324,334,340,347,354,381-382,400,406,411-412,431,441,459,472,479,482,485,488,498,508,518,549,559,601,618,625,633,637,647,657,665,672,691,705,714,741,752"}}
+{"timestamp":1708149027.0628741,"name":"online","context":{"idset":"3-4,13,28,32,35,37,45,50,53-54,56-57,103,111,164,206,225,252,295,317,326,330-331,342,346,367,371,443,445,458,462,477,546,552,567,642,651,667,673,692,699,712,729,738,743-744"}}
+{"timestamp":1708149027.1668935,"name":"online","context":{"idset":"15,43,47,62,75,85,108,131,187,222,224,235,275,292,310,314,356,407,436,510,560,578,586,610,620,622-623,632,634,639,644,648,656,675,690,700-702,711,735,737,739,747,753-754,756"}}
+{"timestamp":1708149027.2679379,"name":"online","context":{"idset":"36,95,124,144,204,210,220,239,250,322,336,439,461,469,497,551,564-565,575,584,630,645,659,680,685,708,713,717,721,730,750"}}
+{"timestamp":1708149027.3766577,"name":"online","context":{"idset":"22,33,44,104-105,116,134,160,214,216,335,337,352,397,538,619,635-636,640,662,683,688,695,703-704,710,716,724"}}
+{"timestamp":1708149027.4946856,"name":"online","context":{"idset":"17,26,264,288,323,505,626,649,654,661,719,731,733,755"}}
+{"timestamp":1708149027.6029329,"name":"online","context":{"idset":"333,605,628-629,655,669-671,677,687,697,742"}}
+{"timestamp":1708149027.7082953,"name":"online","context":{"idset":"117,306,715,728,734"}}
+{"timestamp":1708149028.0015435,"name":"online","context":{"idset":"603,694"}}
+{"timestamp":1708289624.5998359,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1708292108.6001906,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1708442580.5997272,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1708460624.6021802,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1708460658.6006479,"name":"drain","context":{"idset":"613","reason":"broker was unresponsive"}}
+{"timestamp":1708460690.5997632,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1708460724.5985808,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1708464186.6007586,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1708553162.3043294,"name":"online","context":{"idset":"613"}}
+{"timestamp":1708553192.0657754,"name":"undrain","context":{"idset":"613"}}
+{"timestamp":1708553240.6073287,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1708553294.0036228,"name":"online","context":{"idset":"109"}}
+{"timestamp":1708553439.4818976,"name":"undrain","context":{"idset":"109"}}
+{"timestamp":1708553593.1097629,"name":"undrain","context":{"idset":"117"}}
+{"timestamp":1708553594.8015115,"name":"drain","context":{"idset":"117","reason":"reason=cxi_healthcheck fails","overwrite":0}}
+{"timestamp":1708553605.5930724,"name":"undrain","context":{"idset":"123"}}
+{"timestamp":1708553609.848176,"name":"drain","context":{"idset":"123","reason":"reason=cxi_healthcheck fails","overwrite":0}}
+{"timestamp":1708553659.1808445,"name":"online","context":{"idset":"115"}}
+{"timestamp":1708553685.5564771,"name":"undrain","context":{"idset":"115"}}
+{"timestamp":1708553787.7402794,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1708553787.881156,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1708553788.1645372,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1708553788.8588185,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1708553790.4458339,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1708554402.7036648,"name":"undrain","context":{"idset":"127-128,145-148,150-151,168-170,178"}}
+{"timestamp":1708554450.4419475,"name":"online","context":{"idset":"353"}}
+{"timestamp":1708554471.8951521,"name":"undrain","context":{"idset":"353"}}
+{"timestamp":1708554528.3013031,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1708554623.1867409,"name":"undrain","context":{"idset":"314,320-347,352,354,356-357"}}
+{"timestamp":1708554697.8147955,"name":"online","context":{"idset":"678"}}
+{"timestamp":1708554719.034142,"name":"undrain","context":{"idset":"678"}}
+{"timestamp":1708555045.4059496,"name":"online","context":{"idset":"173,308"}}
+{"timestamp":1708555045.7919452,"name":"online","context":{"idset":"174-175,242,255,302,384,388,395,398,440,496"}}
+{"timestamp":1708555045.8923602,"name":"online","context":{"idset":"177,348,386,501,591"}}
+{"timestamp":1708555045.9554744,"name":"online","context":{"idset":"303,592"}}
+{"timestamp":1708555045.9964614,"name":"online","context":{"idset":"176,686"}}
+{"timestamp":1708555047.5217311,"name":"online","context":{"idset":"77,531"}}
+{"timestamp":1708555049.505084,"name":"online","context":{"idset":"702"}}
+{"timestamp":1708555049.6255109,"name":"online","context":{"idset":"709,723,728"}}
+{"timestamp":1708555049.7632298,"name":"online","context":{"idset":"717"}}
+{"timestamp":1708555106.7802472,"name":"undrain","context":{"idset":"173-177"}}
+{"timestamp":1708555131.8687196,"name":"undrain","context":{"idset":"302-303,308"}}
+{"timestamp":1708555190.0793078,"name":"undrain","context":{"idset":"348"}}
+{"timestamp":1708555193.3111835,"name":"undrain","context":{"idset":"355"}}
+{"timestamp":1708555201.5358098,"name":"undrain","context":{"idset":"387"}}
+{"timestamp":1708555211.8198702,"name":"undrain","context":{"idset":"384,386,388"}}
+{"timestamp":1708555229.1740286,"name":"undrain","context":{"idset":"398,496,589"}}
+{"timestamp":1708555330.5166793,"name":"undrain","context":{"idset":"668"}}
+{"timestamp":1708555352.2878246,"name":"undrain","context":{"idset":"515"}}
+{"timestamp":1708555354.3008425,"name":"undrain","context":{"idset":"525"}}
+{"timestamp":1708555359.9996033,"name":"undrain","context":{"idset":"501"}}
+{"timestamp":1708555402.4921618,"name":"undrain","context":{"idset":"123"}}
+{"timestamp":1708555411.809015,"name":"drain","context":{"idset":"123","reason":"reason=not ok 3 - check_ama: Failed","overwrite":0}}
+{"timestamp":1708555502.4454331,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1708555884.2955313,"name":"online","context":{"idset":"248"}}
+{"timestamp":1708555919.6013906,"name":"drain","context":{"idset":"540","reason":"epilog failed for jobid feZmGM1MKmZ","overwrite":0}}
+{"timestamp":1708555922.6426342,"name":"undrain","context":{"idset":"248"}}
+{"timestamp":1708556285.1936443,"name":"drain","context":{"idset":"348","reason":"reason=C_EC_COR + Lustre Bad Write Checksums","overwrite":0}}
+{"timestamp":1708557636.8754659,"name":"drain","context":{"idset":"175","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708557701.009788,"name":"drain","context":{"idset":"385","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708557704.6921465,"name":"drain","context":{"idset":"395","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708557710.1518886,"name":"undrain","context":{"idset":"395"}}
+{"timestamp":1708557750.2687364,"name":"drain","context":{"idset":"308","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708557755.7074418,"name":"drain","context":{"idset":"525","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708558887.4445813,"name":"drain","context":{"idset":"663","reason":"reason=cxi_healthcheck fails","overwrite":0}}
+{"timestamp":1708558996.3552485,"name":"drain","context":{"idset":"440","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559002.8795025,"name":"drain","context":{"idset":"242","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559058.1636477,"name":"drain","context":{"idset":"176","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559087.3058999,"name":"drain","context":{"idset":"386","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559116.4876347,"name":"drain","context":{"idset":"398","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559195.3493366,"name":"drain","context":{"idset":"115","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559216.8833275,"name":"drain","context":{"idset":"255","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559261.2453611,"name":"drain","context":{"idset":"353","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559300.4250147,"name":"drain","context":{"idset":"174","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559393.7865355,"name":"drain","context":{"idset":"177","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559544.9615338,"name":"drain","context":{"idset":"496","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559628.0742533,"name":"drain","context":{"idset":"388","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708559716.4510095,"name":"drain","context":{"idset":"303","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708560007.3056688,"name":"drain","context":{"idset":"384","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708560415.8146031,"name":"drain","context":{"idset":"302","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708561926.3667958,"name":"undrain","context":{"idset":"255"}}
+{"timestamp":1708562166.6530974,"name":"drain","context":{"idset":"173","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708562338.0717275,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1708562338.1240473,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1708562338.1294494,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1708562338.1339982,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1708562338.1377892,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1708562338.1415472,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1708562338.146816,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1708562338.1556039,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1708562338.1599176,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1708562338.1641037,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1708562338.1679478,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1708562338.1716857,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1708562338.1760676,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1708562338.17996,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1708562338.1841097,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1708562338.1915224,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1708562338.1956301,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1708562338.1997647,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1708562338.2037706,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1708562338.2892404,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1708562338.7896843,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1708563031.762502,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1708563031.800283,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1708563031.9004834,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1708563035.7762673,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1708563036.0750036,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1708563036.1712258,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1708563036.1778316,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1708563036.2777672,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1708563036.387713,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1708563036.7214377,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1708563036.7394977,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1708563036.7562985,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1708563036.7610545,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1708563036.7770996,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1708563036.7938054,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1708563036.8251023,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1708563036.8335383,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1708563036.8425941,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1708563036.8591413,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1708563036.8630557,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1708563036.867713,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1708563036.8837931,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1708563036.8889103,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1708563036.8922591,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1708563036.8983188,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1708563036.9087355,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1708563036.9151912,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1708563036.9197843,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1708563036.9389818,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1708563036.9436646,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1708563036.9473698,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1708563036.9486709,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1708563036.9549594,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1708563036.9595094,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1708563036.9673538,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1708563036.9711633,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1708563036.9794955,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1708563036.9799304,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1708563036.9805226,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1708563036.9811826,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1708563036.993412,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1708563036.9978571,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1708563037.0067842,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1708563037.0132163,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1708563037.013768,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1708563037.0144415,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1708563037.0266798,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1708563037.044982,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1708563037.0808997,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1708563037.1079562,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1708563037.131721,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1708563037.2326796,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1708563049.4777262,"name":"drain","context":{"idset":"127-128,145-148,150-151,168-170,178,320-325,327-330,333-338,340-344,346-347,354-357,387,501,515","reason":"epilog failed for jobid feazHnG5GHM","overwrite":0}}
+{"timestamp":1708563236.9446952,"name":"undrain","context":{"idset":"176"}}
+{"timestamp":1708563246.2713401,"name":"drain","context":{"idset":"176","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708563255.137079,"name":"undrain","context":{"idset":"242"}}
+{"timestamp":1708563260.2633569,"name":"drain","context":{"idset":"242","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708563264.9852955,"name":"undrain","context":{"idset":"308"}}
+{"timestamp":1708563269.8069086,"name":"drain","context":{"idset":"308","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708563289.2226794,"name":"undrain","context":{"idset":"242"}}
+{"timestamp":1708563292.4330308,"name":"undrain","context":{"idset":"308"}}
+{"timestamp":1708563325.1050918,"name":"drain","context":{"idset":"242","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708563363.4001403,"name":"drain","context":{"idset":"308","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708563974.116257,"name":"online","context":{"idset":"321,328"}}
+{"timestamp":1708563974.2337391,"name":"online","context":{"idset":"169,324"}}
+{"timestamp":1708564046.2750382,"name":"online","context":{"idset":"145-146,148,333"}}
+{"timestamp":1708564046.3977709,"name":"online","context":{"idset":"127,168,170,178,322,327,329-330,335,338,344,347,354"}}
+{"timestamp":1708564046.4983232,"name":"online","context":{"idset":"128,147,151,334,336-337,341-342,355,357,515"}}
+{"timestamp":1708564046.6069269,"name":"online","context":{"idset":"323,343,346,356,387,501"}}
+{"timestamp":1708564046.7172508,"name":"online","context":{"idset":"320,340"}}
+{"timestamp":1708564046.91889,"name":"online","context":{"idset":"150,325"}}
+{"timestamp":1708564126.3283899,"name":"undrain","context":{"idset":"127-128,145-148,150-151,168-170,178,320-325,327-330,333-338,340-344,346-347,354-357,387,501,515"}}
+{"timestamp":1708565880.3856373,"name":"drain","context":{"idset":"395","reason":"reason=Lustre reconnects","overwrite":0}}
+{"timestamp":1708566435.2312269,"name":"undrain","context":{"idset":"348"}}
+{"timestamp":1708566533.6179767,"name":"online","context":{"idset":"348"}}
+{"timestamp":1708567417.1524184,"name":"drain","context":{"idset":"348","reason":"reason=Bad Write Checksum","overwrite":0}}
+{"timestamp":1708628067.8886561,"name":"drain","context":{"idset":"668","reason":"reason=Slingshot errors","overwrite":0}}
+{"timestamp":1708634946.5998828,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1708653101.6627135,"name":"drain","context":{"idset":"557","reason":"reason=Bad Write Checksum","overwrite":0}}
+{"timestamp":1708733251.7193215,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1707931944.922622,"reason":"reason=Bad Link cxi0"},"115":{"timestamp":1708559195.3493366,"reason":"reason=Lustre reconnects"},"117":{"timestamp":1708553594.8015115,"reason":"reason=cxi_healthcheck fails"},"123":{"timestamp":1708555411.809015,"reason":"reason=not ok 3 - check_ama: Failed"},"173":{"timestamp":1708562166.6530974,"reason":"reason=Lustre reconnects"},"174":{"timestamp":1708559300.4250147,"reason":"reason=Lustre reconnects"},"175":{"timestamp":1708557636.8754659,"reason":"reason=Lustre reconnects"},"176":{"timestamp":1708563246.2713401,"reason":"reason=Lustre reconnects"},"177":{"timestamp":1708559393.7865355,"reason":"reason=Lustre reconnects"},"238":{"timestamp":1707762774.7029541,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"242":{"timestamp":1708563325.1050918,"reason":"reason=Lustre reconnects"},"302":{"timestamp":1708560415.8146031,"reason":"reason=Lustre reconnects"},"303":{"timestamp":1708559716.4510095,"reason":"reason=Lustre reconnects"},"308":{"timestamp":1708563363.4001403,"reason":"reason=Lustre reconnects"},"348":{"timestamp":1708567417.1524184,"reason":"reason=Bad Write Checksum"},"353":{"timestamp":1708559261.2453611,"reason":"reason=Lustre reconnects"},"384":{"timestamp":1708560007.3056688,"reason":"reason=Lustre reconnects"},"385":{"timestamp":1708557701.009788,"reason":"reason=Lustre reconnects"},"386":{"timestamp":1708559087.3058999,"reason":"reason=Lustre reconnects"},"388":{"timestamp":1708559628.0742533,"reason":"reason=Lustre reconnects"},"395":{"timestamp":1708565880.3856373,"reason":"reason=Lustre reconnects"},"398":{"timestamp":1708559116.4876347,"reason":"reason=Lustre reconnects"},"440":{"timestamp":1708558996.3552485,"reason":"reason=Lustre reconnects"},"496":{"timestamp":1708559544.9615338,"reason":"reason=Lustre reconnects"},"525":{"timestamp":1708557755.7074418,"reason":"reason=Lustre reconnects"},"540":{"timestamp":1708555919.6013906,"reason":"epilog failed for jobid feZmGM1MKmZ"},"550":{"timestamp":1707763119.3080339,"reason":"reason=cxi1[hsi1]: p0: resetting serdes PLLs"},"557":{"timestamp":1708653101.6627135,"reason":"reason=Bad Write Checksum"},"663":{"timestamp":1708558887.4445813,"reason":"reason=cxi_healthcheck fails"},"668":{"timestamp":1708628067.8886561,"reason":"reason=Slingshot errors"},"751":{"timestamp":1707763165.571362,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."}},"online":"","exclude":"0"}}
+{"timestamp":1708733251.7483261,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1708733291.3395934,"name":"online","context":{"idset":"0"}}
+{"timestamp":1708733292.0522451,"name":"online","context":{"idset":"8,10-12,16-23,25-26,29-33,35-36,40-41,43-44,46-47,51-53,56,58-59"}}
+{"timestamp":1708733292.1532712,"name":"online","context":{"idset":"9,13-15,24,27-28,34,37-38,42,45,48-50,54-55,57,60"}}
+{"timestamp":1708733292.5612242,"name":"online","context":{"idset":"1,754-755"}}
+{"timestamp":1708733292.6633022,"name":"online","context":{"idset":"2,753,756"}}
+{"timestamp":1708733292.7891376,"name":"online","context":{"idset":"3-5,7,66,136,157,190,193,271-272,375-376,387,427,442-443,471,482-483,501,521,543,580,587"}}
+{"timestamp":1708733292.8908279,"name":"online","context":{"idset":"6,63,72,74,84,89,94,104,126,135,142-143,155,161,165-166,171,180,183,189,202,210,220,227,252,255,263-264,267,280-281,284,288-289,294,300,305,307,315,327,334,343,347,351,359,367,372-373,380,382-383,391-392,406-410,414,417,421,428,432-433,435,437,444,446,450-451,454,473,478,481,488,499,508,515-517,528,535,568,572,577,588,596,603-604,621-622,643"}}
+{"timestamp":1708733292.9977596,"name":"online","context":{"idset":"65,69,71,75,78-79,87,91,105,107,110-112,119,124-125,128-130,132-133,138,145-147,149,153,158,160,163-164,168,170,172,178,184-185,191,195,200-201,206,209,215,217,219,222,224-225,230-231,234,237,244,248-249,251,253,258,265,269,277,282-283,287,296,298-299,306,309,311,316,320-323,325,333,336-338,340-341,346,349,356-357,361-362,364-366,370-371,379,381,389,393-394,396,401-403,411,413,415-416,420,424,426,430-431,434,439,441,445,453,457,464-468,472,476-477,480,484-487,492,495,498,504,507,519,524,526-527,530,536,539,541,544-545,547-548,553-554,556,558,564,574,581,585-586,590-591,593,595,597,599,607,610-612,614-615,617,620,625,632-633,635,638,644,647,650,661,665,699,701,706"}}
+{"timestamp":1708733293.1000094,"name":"online","context":{"idset":"61-62,64,67-68,70,73,76-77,80-83,85-86,88,90,92-93,95,97-103,106,108-109,113-114,116,118,120-122,127,131,134,137,139-141,144,148,150-152,154,156,159,162,167,169,179,181-182,186-188,192,194,196-199,203-205,207-208,211-214,216,218,221,223,226,228-229,232-233,235-236,239-241,243,245-247,250,254,256-257,259-262,266,268,270,273-276,278-279,285-286,290-293,295,297,301,304,310,312-313,317-319,324,328-330,335,342,344,350,354-355,358,360,363,368-369,374,377-378,390,397,399-400,404-405,412,418-419,422-423,425,429,436,438,447-448,452,455-456,458-463,469-470,474-475,479,489-491,493-494,497,500,502-503,505-506,509-514,518,520,522-523,529,531-534,537-538,542,546,549,551-552,555,559-563,565-567,569-571,573,575-576,578-579,582-584,592,594,598,600-602,605-606,608-609,613,616,618-619,623-624,626-628,630-631,634,636-637,639-642,645-646,648-649,651-660,662,664,666-667,669-677,679,681-698,700,702-705,707-726,728-739"}}
+{"timestamp":1708733293.2217581,"name":"online","context":{"idset":"395,629,680,727,745,752"}}
+{"timestamp":1708733293.3229597,"name":"online","context":{"idset":"96,348,557,740-743,746-750"}}
+{"timestamp":1708733293.4441857,"name":"online","context":{"idset":"744"}}
+{"timestamp":1708752798.5318031,"name":"drain","context":{"idset":"259","reason":"broker was unresponsive"}}
+{"timestamp":1708752836.4325898,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1708752836.5334687,"name":"drain","context":{"idset":"270","reason":"broker was unresponsive"}}
+{"timestamp":1708752840.6255207,"name":"drain","context":{"idset":"198","reason":"broker was unresponsive"}}
+{"timestamp":1708753876.531302,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1708753878.4316299,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1708753878.4325604,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1708753878.4326386,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1708753878.4326899,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1708753878.4327359,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1708753878.5319221,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1708753880.4317369,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1708753880.5326788,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1708753882.4323184,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1708753882.4323938,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1708753882.4324355,"name":"drain","context":{"idset":"240","reason":"broker was unresponsive"}}
+{"timestamp":1708753882.4324746,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1708753882.4325056,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1708753882.4325364,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1708753882.4325709,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1708753882.5323,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1708753886.5314078,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1708963344.531615,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1708963406.5313487,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1708963958.5319269,"name":"drain","context":{"idset":"637","reason":"broker was unresponsive"}}
+{"timestamp":1708964022.5313773,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1708970332.044276,"name":"undrain","context":{"idset":"206,214-215,218,226,235,239-240,281-287,289-290,316"}}
+{"timestamp":1708970381.2033691,"name":"undrain","context":{"idset":"249,259,270"}}
+{"timestamp":1708970588.7713649,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1708970588.9193602,"name":"drain","context":{"idset":"262","reason":"broker was unresponsive"}}
+{"timestamp":1708970589.0693886,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1708970589.229274,"name":"drain","context":{"idset":"333","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2221014,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2221825,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2222183,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2222559,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2227495,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2227769,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2228036,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2228303,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2228751,"name":"drain","context":{"idset":"254","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2229035,"name":"drain","context":{"idset":"255","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2229309,"name":"drain","context":{"idset":"267","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2229602,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2229915,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2239141,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2239511,"name":"drain","context":{"idset":"288","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2239811,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2240105,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1708970591.2240522,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1708970592.431057,"name":"drain","context":{"idset":"197","reason":"broker was unresponsive"}}
+{"timestamp":1708970592.4311645,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1708970592.4312155,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1708970592.4312663,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1708970592.4313147,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1708970592.5310943,"name":"drain","context":{"idset":"334","reason":"broker was unresponsive"}}
+{"timestamp":1708970594.4323967,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1708970594.432482,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1708970594.4325407,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1708970594.5327139,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1708970600.5307312,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1708970633.9121051,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1708970634.9911051,"name":"drain","context":{"idset":"253","reason":"broker was unresponsive"}}
+{"timestamp":1708971257.6930614,"name":"undrain","context":{"idset":"197,206,208,212-215,218-219,222-223,225-226,239,243,253-255,262,267,280-283,287-288,290,304,311,313,315,333-336"}}
+{"timestamp":1708971518.4322112,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1708971518.5329058,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1708971520.7538762,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1708971520.9059246,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1708971521.0960071,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1708971521.242522,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1708971521.4195166,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1708971521.5908866,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1708971521.7942173,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1708971521.994621,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1708971522.2146256,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1708971522.4179895,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1708971522.6321652,"name":"drain","context":{"idset":"268","reason":"broker was unresponsive"}}
+{"timestamp":1708971522.8523819,"name":"drain","context":{"idset":"291","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.0700088,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.2789912,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.2791276,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4900446,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4901769,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4902387,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4902978,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4912729,"name":"drain","context":{"idset":"207","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4913328,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4913881,"name":"drain","context":{"idset":"217","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4914441,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4915001,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4915748,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4916315,"name":"drain","context":{"idset":"256","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4916887,"name":"drain","context":{"idset":"258","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4917452,"name":"drain","context":{"idset":"273","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4918034,"name":"drain","context":{"idset":"275","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.4918659,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1708971523.491924,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1708971528.640563,"name":"drain","context":{"idset":"234","reason":"broker was unresponsive"}}
+{"timestamp":1708971535.4567928,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1708971582.5312283,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1708973001.4024563,"name":"drain","context":{"idset":"61,63-79,82,157-172,178-179,181-182,594-612,615-616","reason":"prolog failed for jobid ffWYeVFkiVD","overwrite":0}}
+{"timestamp":1708973172.9547913,"name":"undrain","context":{"idset":"206-208,210,212-215,217-219,223,225,229,234-235,239,246,251,256,258,268,273,275,283,291-292,299,305,307,311-312,315,317-318"}}
+{"timestamp":1708973248.3388977,"name":"online","context":{"idset":"308"}}
+{"timestamp":1708973248.7273943,"name":"online","context":{"idset":"115,117,174-177,242,302-303,326,331,339,353,384-385,388"}}
+{"timestamp":1708973248.8282259,"name":"online","context":{"idset":"123,173,314,332,345,352,386,398,440,449,496,525,540,550,589,668,678"}}
+{"timestamp":1708973248.9294481,"name":"online","context":{"idset":"663"}}
+{"timestamp":1708973254.7542465,"name":"online","context":{"idset":"238"}}
+{"timestamp":1708973255.35131,"name":"online","context":{"idset":"751"}}
+{"timestamp":1708973293.1545887,"name":"undrain","context":{"idset":"61,63-79,82,157-172,178-179,181-182,594-612,615-616"}}
+{"timestamp":1708974786.5334918,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1708974806.4319308,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1708974806.5327249,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1708974810.5337398,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1708974812.5315721,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1708974848.5321143,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1708974866.4314961,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1708974866.4316077,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1708974866.5320284,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1708974868.4341183,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1708974868.4429383,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1708974868.4490597,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1708974870.5312083,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1708974872.4347503,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1708974872.4414062,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1708974872.4468887,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1708974872.4524899,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1708974872.457947,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1708974872.46398,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1708974872.5304303,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1708974874.5308027,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1708974882.4310422,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1708974882.5317855,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1708974889.3445711,"name":"online","context":{"idset":"7"}}
+{"timestamp":1708974890.0399678,"name":"online","context":{"idset":"31"}}
+{"timestamp":1708974891.0560772,"name":"online","context":{"idset":"33"}}
+{"timestamp":1708974894.8201315,"name":"online","context":{"idset":"55"}}
+{"timestamp":1708974942.5315204,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1708974974.082417,"name":"online","context":{"idset":"1"}}
+{"timestamp":1708974974.3279963,"name":"online","context":{"idset":"57"}}
+{"timestamp":1708974974.4317973,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1708974974.4318678,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1708974974.4319272,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1708974974.5321226,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1708974975.0822363,"name":"online","context":{"idset":"59"}}
+{"timestamp":1708974976.4331133,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1708974977.2196193,"name":"online","context":{"idset":"9"}}
+{"timestamp":1708974980.3144367,"name":"online","context":{"idset":"27"}}
+{"timestamp":1708975036.4334817,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1708975036.5314879,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1708975050.0935264,"name":"online","context":{"idset":"29"}}
+{"timestamp":1708975056.4334106,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1708975056.4335175,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1708975056.4335701,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1708975056.433624,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1708975056.433682,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1708975056.5335128,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1708975058.4328375,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1708975058.4329059,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1708975058.4329405,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1708975058.4329875,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1708975058.4330344,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1708975058.4330699,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1708975058.5326684,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1708975064.0968182,"name":"online","context":{"idset":"24"}}
+{"timestamp":1708975064.5869317,"name":"online","context":{"idset":"16"}}
+{"timestamp":1708975116.4329538,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1708975120.4362183,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1708975120.4430478,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1708975120.4496882,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1708975120.4562998,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1708975120.4628942,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1708975120.5356345,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1708975122.5305767,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1708975124.5318322,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1708975135.9241054,"name":"online","context":{"idset":"51"}}
+{"timestamp":1708975138.4329267,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1708975139.8668458,"name":"online","context":{"idset":"53"}}
+{"timestamp":1708975140.4325256,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1708975140.4326401,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1708975140.4327674,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1708975140.4328129,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1708975140.432858,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1708975142.4328158,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1708975144.1419497,"name":"online","context":{"idset":"56"}}
+{"timestamp":1708975202.5324321,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1708975220.4314635,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1708975220.5319574,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1708975222.4335999,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1708975222.4337113,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1708975222.4338443,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1708975222.4339216,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1708975222.5336795,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1708975224.0755236,"name":"online","context":{"idset":"4,59"}}
+{"timestamp":1708975227.4806683,"name":"online","context":{"idset":"3"}}
+{"timestamp":1708975229.0980995,"name":"online","context":{"idset":"49"}}
+{"timestamp":1708975241.8152273,"name":"online","context":{"idset":"29"}}
+{"timestamp":1708975302.4321592,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1708975304.4331653,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1708975304.4333398,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1708975367.8298438,"name":"online","context":{"idset":"25"}}
+{"timestamp":1708975390.4322991,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1708975392.4325457,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1708975392.432622,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1708975392.4326804,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1708975452.4343073,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1708975452.4401896,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1708975452.5321259,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1708975454.4357033,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1708975454.5316973,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1708975465.7230852,"name":"online","context":{"idset":"15"}}
+{"timestamp":1708975472.4340017,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1708975472.4341464,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1708975472.4343162,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1708975476.9494698,"name":"online","context":{"idset":"46"}}
+{"timestamp":1708975500.2132783,"name":"online","context":{"idset":"45"}}
+{"timestamp":1708975544.8242412,"name":"online","context":{"idset":"11"}}
+{"timestamp":1708975555.3636558,"name":"online","context":{"idset":"24"}}
+{"timestamp":1708975557.0320523,"name":"online","context":{"idset":"48"}}
+{"timestamp":1708975720.4331932,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1708975724.5325935,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1708975776.5294333,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1708975780.5320995,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1708975782.5310159,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1708975808.0933881,"name":"online","context":{"idset":"24"}}
+{"timestamp":1708975811.1384425,"name":"online","context":{"idset":"23"}}
+{"timestamp":1708975866.4333444,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1708975866.5309989,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1708975868.4355199,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1708975868.4410589,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1708975868.4492414,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1708975868.4574254,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1708975868.4659128,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1708975868.5319455,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1708975890.7671387,"name":"online","context":{"idset":"48"}}
+{"timestamp":1708975891.2313325,"name":"online","context":{"idset":"40"}}
+{"timestamp":1708975944.4328036,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1708975944.4364259,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1708975944.4397542,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1708975946.4345307,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1708975946.4414322,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1708975946.447907,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1708975946.4538641,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1708975946.4605896,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1708975948.4367807,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1708975948.441098,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1708975948.4453487,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1708975948.4494281,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1708975948.4535553,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1708975948.4576943,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1708975948.4618084,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1708975948.4659338,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1708975948.4700832,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1708975948.5317533,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1708975950.438787,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1708975950.4440558,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1708975950.4492092,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1708975950.4543128,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1708975950.4593387,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1708975950.4643755,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1708975950.4694984,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1708975950.4746192,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1708975950.4797063,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1708975950.4869885,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1708975950.4931881,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1708975950.4994476,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1708975950.5046561,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1708975950.5098717,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1708975950.5149891,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1708975950.5199621,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1708975950.531446,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1708975966.2117696,"name":"online","context":{"idset":"18"}}
+{"timestamp":1708975968.9022446,"name":"online","context":{"idset":"44,46"}}
+{"timestamp":1708975969.1455765,"name":"online","context":{"idset":"23"}}
+{"timestamp":1708975971.0232608,"name":"online","context":{"idset":"19"}}
+{"timestamp":1708975971.3237107,"name":"online","context":{"idset":"38,47"}}
+{"timestamp":1708975972.325582,"name":"online","context":{"idset":"15"}}
+{"timestamp":1708975973.2846799,"name":"online","context":{"idset":"53"}}
+{"timestamp":1708975973.3067062,"name":"online","context":{"idset":"56"}}
+{"timestamp":1708975973.5801742,"name":"online","context":{"idset":"14,16"}}
+{"timestamp":1708975973.8463857,"name":"online","context":{"idset":"54"}}
+{"timestamp":1708975974.2675776,"name":"online","context":{"idset":"37"}}
+{"timestamp":1708975977.1905022,"name":"online","context":{"idset":"20,22"}}
+{"timestamp":1708975978.307466,"name":"online","context":{"idset":"10"}}
+{"timestamp":1708976029.409714,"name":"online","context":{"idset":"36"}}
+{"timestamp":1708976029.5553901,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1708976032.5307291,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1708976033.7893054,"name":"online","context":{"idset":"42,45"}}
+{"timestamp":1708976033.9186916,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1708976049.2283003,"name":"online","context":{"idset":"41"}}
+{"timestamp":1708976051.2081263,"name":"online","context":{"idset":"1"}}
+{"timestamp":1708976051.5541172,"name":"online","context":{"idset":"58"}}
+{"timestamp":1708976052.1385741,"name":"online","context":{"idset":"60"}}
+{"timestamp":1708976052.2514555,"name":"online","context":{"idset":"17"}}
+{"timestamp":1708976052.432795,"name":"online","context":{"idset":"6"}}
+{"timestamp":1708976052.8808761,"name":"online","context":{"idset":"7,11,28"}}
+{"timestamp":1708976053.1295235,"name":"online","context":{"idset":"51"}}
+{"timestamp":1708976053.265193,"name":"online","context":{"idset":"3,26"}}
+{"timestamp":1708976054.1387451,"name":"online","context":{"idset":"29"}}
+{"timestamp":1708976058.0565212,"name":"online","context":{"idset":"33"}}
+{"timestamp":1708976132.4321697,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1708976134.4325867,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1708976140.9654219,"name":"online","context":{"idset":"21"}}
+{"timestamp":1708976152.7510767,"name":"online","context":{"idset":"13"}}
+{"timestamp":1708976156.0818467,"name":"online","context":{"idset":"43"}}
+{"timestamp":1708976211.4362667,"name":"online","context":{"idset":"35"}}
+{"timestamp":1708976213.081111,"name":"online","context":{"idset":"32,59"}}
+{"timestamp":1708976213.5250733,"name":"online","context":{"idset":"27"}}
+{"timestamp":1708976214.3995945,"name":"online","context":{"idset":"31"}}
+{"timestamp":1708976216.5333493,"name":"online","context":{"idset":"25"}}
+{"timestamp":1708976228.3861599,"name":"online","context":{"idset":"30"}}
+{"timestamp":1708976300.4326873,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1708976606.4351609,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1708976606.442029,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1708976606.5324283,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1708976608.4332459,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1708976608.5321949,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1708976629.1271935,"name":"online","context":{"idset":"47"}}
+{"timestamp":1708976634.08097,"name":"online","context":{"idset":"41"}}
+{"timestamp":1708976637.9053919,"name":"online","context":{"idset":"43"}}
+{"timestamp":1708976708.2278578,"name":"online","context":{"idset":"29"}}
+{"timestamp":1708976723.012444,"name":"online","context":{"idset":"27"}}
+{"timestamp":1708977020.4351003,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1708977020.444062,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1708977020.4526248,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1708977024.4345419,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1708977024.4403679,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1708977024.5316224,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1708977026.4335554,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1708977026.5309925,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1708977033.1429844,"name":"online","context":{"idset":"11"}}
+{"timestamp":1708977048.7882223,"name":"online","context":{"idset":"46"}}
+{"timestamp":1708977048.9345119,"name":"online","context":{"idset":"44"}}
+{"timestamp":1708977053.0408499,"name":"online","context":{"idset":"55"}}
+{"timestamp":1708977053.301012,"name":"online","context":{"idset":"57"}}
+{"timestamp":1708977188.5313153,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1708977190.43625,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1708977190.4449668,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1708977190.4536963,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1708977190.5321875,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1708977192.4328804,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1708977192.4376178,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1708977192.4423068,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1708977192.5306475,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1708977204.8232529,"name":"online","context":{"idset":"17"}}
+{"timestamp":1708977205.0121107,"name":"online","context":{"idset":"20"}}
+{"timestamp":1708977205.677186,"name":"online","context":{"idset":"14"}}
+{"timestamp":1708977206.9771571,"name":"online","context":{"idset":"48"}}
+{"timestamp":1708977208.0255027,"name":"online","context":{"idset":"24"}}
+{"timestamp":1708977233.3822336,"name":"online","context":{"idset":"47"}}
+{"timestamp":1708977260.4353254,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1708977260.4407592,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1708977260.4460433,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1708977260.4512908,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1708977260.531893,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1708977262.5326283,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1708977264.4361045,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1708977264.4413214,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1708977264.4463553,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1708977264.4517729,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1708977264.4570565,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1708977264.4622562,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1708977264.5315964,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1708977293.2548478,"name":"online","context":{"idset":"19"}}
+{"timestamp":1708977295.4136393,"name":"online","context":{"idset":"42"}}
+{"timestamp":1708977296.8775301,"name":"online","context":{"idset":"29"}}
+{"timestamp":1708977297.2524524,"name":"online","context":{"idset":"36"}}
+{"timestamp":1708977298.073307,"name":"online","context":{"idset":"34,38,44"}}
+{"timestamp":1708977298.2084103,"name":"online","context":{"idset":"27-28,35"}}
+{"timestamp":1708977298.5339918,"name":"online","context":{"idset":"31"}}
+{"timestamp":1708977302.4329464,"name":"online","context":{"idset":"45"}}
+{"timestamp":1708977303.6292918,"name":"online","context":{"idset":"25"}}
+{"timestamp":1708977308.2832298,"name":"online","context":{"idset":"37"}}
+{"timestamp":1708977317.1527615,"name":"online","context":{"idset":"30"}}
+{"timestamp":1708977356.5318332,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1708977359.9831402,"name":"online","context":{"idset":"16"}}
+{"timestamp":1708977367.241061,"name":"online","context":{"idset":"33"}}
+{"timestamp":1708977370.0719774,"name":"online","context":{"idset":"53"}}
+{"timestamp":1708977389.8269668,"name":"online","context":{"idset":"41"}}
+{"timestamp":1708977453.4096911,"name":"online","context":{"idset":"43"}}
+{"timestamp":1708981225.0348291,"name":"undrain","context":{"idset":"1-60"}}
+{"timestamp":1708981365.6981018,"name":"drain","context":{"idset":"39","reason":"reason=slingshot","overwrite":0}}
+{"timestamp":1708984018.5310788,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1708986881.2772908,"name":"drain","context":{"idset":"185,188,309-310,439,695,698,700","reason":"reason=Luustre hung","overwrite":0}}
+{"timestamp":1708989464.935338,"name":"drain","context":{"idset":"443","reason":"reason=lustre timeout","overwrite":0}}
+{"timestamp":1708989638.2017982,"name":"drain","context":{"idset":"568","reason":"reason=lustre timeout","overwrite":0}}
+{"timestamp":1708990999.8284125,"name":"drain","context":{"idset":"181,184-188,309,311,437,441,443-444,568,571,695-696,698","reason":"lctl ping fails","overwrite":1}}
+{"timestamp":1709058456.7185023,"name":"drain","context":{"idset":"61-76","reason":"reason=HPE Rabbit CMM work","overwrite":0}}
+{"timestamp":1709059440.5300641,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1709059536.4387715,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1709059536.4450011,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1709059536.4503949,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1709059536.4544797,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1709059536.4577723,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1709059536.4611125,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1709059536.4644222,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1709059536.4677169,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1709059536.4710147,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1709059536.4743578,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1709059536.5320637,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1709060484.5320497,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1709071170.5978725,"name":"drain","context":{"idset":"119-120,122,124","reason":"reason=lctl ping fails","overwrite":1}}
+{"timestamp":1709072276.6204038,"name":"drain","context":{"idset":"118-120,122,124,438,442","reason":"reason=RMA timeout to RX context NIC","overwrite":1}}
+{"timestamp":1709072886.4759777,"name":"drain","context":{"idset":"118-120,122,124,182,312-313,315,438,442,566-567,572,694,697","reason":"reason=RMA timeout to RX context NIC","overwrite":1}}
+{"timestamp":1709073322.652173,"name":"drain","context":{"idset":"118-122,124,182-183,312-316,438,442,566-567,570,572,694,697,699","reason":"reason=RMA timeout to RX context NIC","overwrite":1}}
+{"timestamp":1709074302.3847253,"name":"drain","context":{"idset":"118-122,124,182-183,312-316,438,442,566-567,570,572,693-694,697,699","reason":"reason=RMA timeout to RX context NIC","overwrite":1}}
+{"timestamp":1709075943.0017097,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1709075952.5102174,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1709076517.6726611,"name":"drain","context":{"idset":"118-122,124,182-183,312-316,438,442,566,569-570,693-694,697,699","reason":"reason=RMA timeout to RX context NIC","overwrite":1}}
+{"timestamp":1709076754.44486,"name":"drain","context":{"idset":"118-122,124,182-183,312-316,438,442,566,569-570,693-694,697,699","reason":"reason=RMA timeout to RX context NIC","overwrite":1}}
+{"timestamp":1709076817.926255,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1709100851.9288552,"name":"drain","context":{"idset":"252","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1709134754.5198977,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1709136202.5319188,"name":"offline","context":{"idset":"550"}}
+{"timestamp":1709169100.8475556,"name":"online","context":{"idset":"213"}}
+{"timestamp":1709169101.5582757,"name":"online","context":{"idset":"567"}}
+{"timestamp":1709169102.4028032,"name":"online","context":{"idset":"565,572"}}
+{"timestamp":1709169102.5325685,"name":"online","context":{"idset":"637"}}
+{"timestamp":1709169103.1112101,"name":"online","context":{"idset":"569"}}
+{"timestamp":1709234948.906889,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1709234948.9168751,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1709234949.0057962,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1709234949.0188017,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1709234949.0258813,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1709234949.031709,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1709234949.0378695,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1709234949.0468874,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1709234949.084811,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1709234949.085669,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1709234949.1028259,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1709234949.1118491,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1709234949.119483,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1709234949.1275065,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1709234949.1335144,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1709234949.1391625,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1709234949.1472154,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1709234949.1559179,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1709234949.1668086,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1709234949.1754549,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1709234949.1844118,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1709234949.1933303,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1709234949.2022583,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1709234949.2111626,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1709234949.2201252,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1709234949.2275097,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1709234949.2336116,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1709234949.2417839,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1709234949.2505729,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1709234949.260077,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1709234949.2703493,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1709234949.2806058,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1709234949.2893038,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1709234949.2981243,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1709234949.3067997,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1709234949.3162818,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1709234949.3193419,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1709234949.3261182,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1709234949.3403726,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1709234949.3489659,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1709234949.3575792,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1709234949.3662345,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1709234949.3748548,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1709234949.3834245,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1709234949.3919821,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1709234949.4006586,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1709234949.4092541,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1709234949.4179068,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1709234949.4247363,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1709234949.4303062,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1709234949.4374182,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1709234949.4473538,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1709234949.4573019,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1709234949.4674616,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1709234949.4768803,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1709234950.9392552,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1709234950.9517643,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1709234950.9607081,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1709234950.9713945,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1709234950.9800093,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1709234950.9880366,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1709234950.9941769,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1709234951.0000942,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1709234951.0156853,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1709234951.0233319,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1709234951.0286462,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1709234951.0337951,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1709234951.0389943,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1709234951.0444977,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1709234951.0491695,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1709234951.0540528,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1709234951.0589185,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1709234951.0592828,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1709234951.0647759,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1709234951.0704405,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1709234951.0707819,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1709234951.0764563,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1709234951.0767863,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1709234951.0818498,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1709234951.0868869,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1709234951.0918469,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1709234951.1074214,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1709234951.120033,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1709234951.1203833,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1709234951.1207201,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1709234951.1439452,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1709234951.1498189,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1709234951.1552703,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1709234951.1657813,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1709234951.1716468,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1709234951.171973,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1709234951.1723149,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1709234951.1928082,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1709234951.1990356,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1709234951.2041252,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1709234951.2089577,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1709234951.213881,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1709234951.2187629,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1709234951.228884,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1709234951.2348242,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1709234951.2352018,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1709234951.2396829,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1709234951.2399979,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1709234951.2403221,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1709234951.2447019,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1709234951.2499526,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1709234951.276639,"name":"offline","context":{"idset":"549"}}
+{"timestamp":1709234951.2770085,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1709234951.2822886,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1709234951.2919662,"name":"offline","context":{"idset":"121"}}
+{"timestamp":1709234951.2969282,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1709234951.3016424,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1709234951.306407,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1709234951.3110821,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1709234951.3158476,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1709234951.3213301,"name":"offline","context":{"idset":"456"}}
+{"timestamp":1709234951.3257234,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1709234951.3266857,"name":"offline","context":{"idset":"553"}}
+{"timestamp":1709234951.3303382,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1709234951.3306308,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1709234951.3439178,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1709234951.3493226,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1709234951.3541837,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1709234951.354491,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1709234951.3548555,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1709234951.3638675,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1709234951.3733928,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1709234951.382853,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1709234951.3922637,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1709234951.3925669,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1709234951.4017005,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1709234951.4070051,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1709234951.4119632,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1709234951.4167597,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1709234951.4214463,"name":"offline","context":{"idset":"672"}}
+{"timestamp":1709234951.4256706,"name":"offline","context":{"idset":"675"}}
+{"timestamp":1709234951.4398038,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1709234951.445415,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1709234951.4499891,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1709234951.454524,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1709234951.4592168,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1709234951.4640069,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1709234951.4685392,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1709234951.4731047,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1709234951.4775155,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1709234951.4821813,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1709234951.4865491,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1709234951.4911551,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1709234951.495898,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1709234951.5007885,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1709234951.5061543,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1709234951.5113606,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1709234951.5160313,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1709234951.5207257,"name":"offline","context":{"idset":"445"}}
+{"timestamp":1709234951.5253026,"name":"offline","context":{"idset":"448"}}
+{"timestamp":1709234951.5295396,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1709234951.5343428,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1709234951.5434382,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1709234951.5438051,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1709234951.5538182,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1709234951.5584352,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1709234951.5628624,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1709234951.5672286,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1709234951.6398072,"name":"offline","context":{"idset":"685"}}
+{"timestamp":1709234952.1042719,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1709234952.111975,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1709234952.1174934,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1709234952.1800685,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1709234952.2113941,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1709234952.219188,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1709234952.2239087,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1709234952.2321868,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1709234952.2461059,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1709234952.2658198,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1709234952.2838931,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1709234952.2891896,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1709234952.2936373,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1709234952.3175247,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1709234952.3362682,"name":"offline","context":{"idset":"661"}}
+{"timestamp":1709234952.3500042,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1709234952.354955,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1709234952.3624368,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1709234952.3676097,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1709234952.3720691,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1709234952.3766534,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1709234952.3904417,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1709234952.3948948,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1709234952.399554,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1709234952.4043407,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1709234952.4165952,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1709234952.4220369,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1709234952.4266806,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1709234952.4321289,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1709234952.4364872,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1709234952.4441078,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1709234952.445739,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1709234952.481318,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1709234952.4871209,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1709234952.5011661,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1709234952.5091662,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1709234952.5417287,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1709234952.5469663,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1709234952.550878,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1709234952.555366,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1709234952.562984,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1709234952.5760093,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1709234952.6159382,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1709234952.6227834,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1709234952.6287863,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1709234952.6473815,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1709234952.6522543,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1709234952.7010489,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1709234952.7070603,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1709234952.7406974,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1709234952.7720106,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1709234952.8165429,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1709234952.8238285,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1709234952.8303583,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1709234952.8542144,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1709234952.8711524,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1709234952.8871152,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1709234952.9057369,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1709234952.9225087,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1709234952.9409804,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1709234952.9458253,"name":"offline","context":{"idset":"667"}}
+{"timestamp":1709234952.9662101,"name":"offline","context":{"idset":"662"}}
+{"timestamp":1709234952.9797342,"name":"offline","context":{"idset":"450"}}
+{"timestamp":1709234952.9869525,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1709234953.002799,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1709234953.0090525,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1709234953.0142095,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1709234953.0179656,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1709234953.023947,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1709234953.0243311,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1709234953.0283408,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1709234953.0384238,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1709234953.0429707,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1709234953.04667,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1709234953.0527923,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1709234953.053206,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1709234953.062429,"name":"offline","context":{"idset":"555"}}
+{"timestamp":1709234953.0673614,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1709234953.0709875,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1709234953.0784185,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1709234953.0787659,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1709234953.0827186,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1709234953.0865564,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1709234953.0948765,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1709234953.107173,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1709234953.111644,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1709234953.1159096,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1709234953.1198368,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1709234953.133039,"name":"offline","context":{"idset":"666"}}
+{"timestamp":1709234953.1372621,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1709234953.1421242,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1709234953.1478736,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1709234953.1507518,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1709234953.1511979,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1709234953.1550796,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1709234953.1589396,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1709234953.1625354,"name":"offline","context":{"idset":"453"}}
+{"timestamp":1709234953.167748,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1709234953.1768723,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1709234953.1804042,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1709234953.1857769,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1709234953.1911719,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1709234953.1945484,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1709234953.1948988,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1709234953.1987195,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1709234953.2025242,"name":"offline","context":{"idset":"451"}}
+{"timestamp":1709234953.2059753,"name":"offline","context":{"idset":"558"}}
+{"timestamp":1709234953.2148697,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1709234953.2188168,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1709234953.2229726,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1709234953.2835834,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1709234953.3169708,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1709234953.3215964,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1709234953.3340394,"name":"offline","context":{"idset":"554"}}
+{"timestamp":1709234953.346411,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1709234953.3518188,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1709234953.3551912,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1709234953.3598959,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1709234953.3635461,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1709234953.3668177,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1709234953.3715289,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1709234953.3778644,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1709234953.3823998,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1709234953.3861272,"name":"offline","context":{"idset":"561"}}
+{"timestamp":1709234953.3903654,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1709234953.3966372,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1709234953.3969908,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1709234953.4038804,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1709234953.4072027,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1709234953.4104207,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1709234953.4149668,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1709234953.4204512,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1709234953.423281,"name":"offline","context":{"idset":"692"}}
+{"timestamp":1709234953.4318204,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1709234953.4353967,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1709234953.4385474,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1709234953.4417281,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1709234953.4420569,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1709234953.4449604,"name":"offline","context":{"idset":"462"}}
+{"timestamp":1709234953.448113,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1709234953.4512722,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1709234953.451575,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1709234953.4545314,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1709234953.466856,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1709234953.4707119,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1709234953.4710126,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1709234953.4739146,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1709234953.4774351,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1709234953.4876435,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1709234953.4907451,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1709234953.497489,"name":"offline","context":{"idset":"655"}}
+{"timestamp":1709234953.5061874,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1709234953.5211153,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1709234953.5224414,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1709234953.5310044,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1709234953.535038,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1709234953.5403082,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1709234953.5436676,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1709234953.549885,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1709234953.5535204,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1709234953.5574064,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1709234953.5607905,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1709234953.5645061,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1709234953.5686004,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1709234953.5724831,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1709234953.5759141,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1709234953.5762229,"name":"offline","context":{"idset":"682"}}
+{"timestamp":1709234953.5848489,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1709234953.588424,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1709234953.5913396,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1709234953.5948436,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1709234953.5977347,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1709234953.6053703,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1709234953.6089203,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1709234953.6124885,"name":"offline","context":{"idset":"452"}}
+{"timestamp":1709234953.6160057,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1709234953.6195428,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1709234953.6226978,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1709234953.6264441,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1709234953.6295857,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1709234953.6332524,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1709234953.6356618,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1709234953.6359377,"name":"offline","context":{"idset":"559"}}
+{"timestamp":1709234953.6381521,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1709234953.6384263,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1709234953.6475396,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1709234953.6504922,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1709234953.6507759,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1709234953.6532657,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1709234953.6535444,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1709234953.6561325,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1709234953.6587741,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1709234953.6641314,"name":"offline","context":{"idset":"454"}}
+{"timestamp":1709234953.6695032,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1709234953.674979,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1709234953.6803145,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1709234953.6806002,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1709234953.6857107,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1709234953.6884077,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1709234953.6910717,"name":"offline","context":{"idset":"551"}}
+{"timestamp":1709234953.6937847,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1709234953.6972818,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1709234953.7488627,"name":"offline","context":{"idset":"660"}}
+{"timestamp":1709234953.7822316,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1709234953.790153,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1709234953.8219318,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1709234953.8532329,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1709234953.8988998,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1709234953.9121504,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1709234953.9263029,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1709234953.9295142,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1709234953.9643896,"name":"offline","context":{"idset":"464"}}
+{"timestamp":1709234953.9673233,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1709234953.9737241,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1709234953.9977963,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1709234954.0009146,"name":"offline","context":{"idset":"676"}}
+{"timestamp":1709234954.0323658,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1709234954.0416338,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1709234954.0498261,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1709234954.0526118,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1709234954.0636458,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1709234954.1546121,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1709234954.1665425,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1709234954.1696899,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1709234954.2317631,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1709234954.3553419,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1709234954.3861279,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1709234954.4464095,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1709234954.4691968,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1709234954.5033982,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1709234954.5066836,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1709234954.5162249,"name":"offline","context":{"idset":"708"}}
+{"timestamp":1709234954.5219359,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1709234954.5412567,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1709234954.5604153,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1709234954.5643117,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1709234954.5684783,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1709234954.5769444,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1709234954.5865343,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1709234954.5897417,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1709234954.5933137,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1709234954.5971918,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1709234954.6029546,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1709234954.6245668,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1709234954.6284611,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1709234954.6311297,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1709234954.6343379,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1709234954.6379335,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1709234954.642318,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1709234954.6458702,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1709234954.6461833,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1709234954.6487577,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1709234954.6553719,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1709234954.6582887,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1709234954.661145,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1709234954.6637321,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1709234954.6667778,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1709234954.6785054,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1709234954.6813228,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1709234954.6854086,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1709234954.6877024,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1709234954.7062793,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1709234954.7089581,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1709234954.7119734,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1709234954.7362525,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1709234954.7389841,"name":"offline","context":{"idset":"657"}}
+{"timestamp":1709234954.7499564,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1709234954.7585266,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1709234954.7683253,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1709234954.7760968,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1709234954.7790504,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1709234954.7932777,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1709234954.7960131,"name":"offline","context":{"idset":"669"}}
+{"timestamp":1709234954.7992053,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1709234954.802001,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1709234954.8051779,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1709234954.8124316,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1709234954.8151782,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1709234954.8182027,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1709234954.8212721,"name":"offline","context":{"idset":"665"}}
+{"timestamp":1709234954.8234119,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1709234954.826741,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1709234954.8291295,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1709234954.8312519,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1709234954.8351188,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1709234954.8376908,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1709234954.8410895,"name":"offline","context":{"idset":"654"}}
+{"timestamp":1709234954.8439996,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1709234954.8460703,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1709234954.8488712,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1709234954.8512685,"name":"offline","context":{"idset":"671"}}
+{"timestamp":1709234954.8533692,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1709234954.8565629,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1709234954.8588789,"name":"offline","context":{"idset":"680"}}
+{"timestamp":1709234954.8612239,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1709234954.8655372,"name":"offline","context":{"idset":"552"}}
+{"timestamp":1709234954.8680775,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1709234954.8683317,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1709234954.868566,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1709234954.8704937,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1709234954.8725109,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1709234954.8787813,"name":"offline","context":{"idset":"670"}}
+{"timestamp":1709234954.8816698,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1709234954.884059,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1709234954.9413929,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1709234954.9485612,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1709234954.9488113,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1709234954.9490857,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1709234954.9796026,"name":"offline","context":{"idset":"656"}}
+{"timestamp":1709234954.9836576,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1709234955.0028212,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1709234955.0138001,"name":"offline","context":{"idset":"659"}}
+{"timestamp":1709234955.0167887,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1709234955.0210738,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1709234955.0319006,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1709234955.0407774,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1709234955.0433495,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1709234955.0457048,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1709234955.0485506,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1709234955.0504057,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1709234955.0543251,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1709234955.0545502,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1709234955.0584199,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1709234955.0587273,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1709234955.0623009,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1709234955.0641093,"name":"offline","context":{"idset":"658"}}
+{"timestamp":1709234955.0663912,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1709234955.0694342,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1709234955.0714734,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1709234955.0734975,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1709234955.0759232,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1709234955.0782239,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1709234955.0807056,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1709234955.0843232,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1709234955.0866659,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1709234955.0961068,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1709234955.09937,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1709234955.1013565,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1709234955.1034851,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1709234955.1056237,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1709234955.1076813,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1709234955.1218061,"name":"offline","context":{"idset":"556"}}
+{"timestamp":1709234955.1300898,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1709234955.1352537,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1709234955.1445849,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1709234955.158026,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1709234955.1669514,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1709234955.1755872,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1709234955.1775241,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1709234955.1849544,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1709234955.1868627,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1709234955.1951716,"name":"offline","context":{"idset":"694"}}
+{"timestamp":1709234955.2055085,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1709234955.207684,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1709234955.2096584,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1709234955.2116911,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1709234955.2132819,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1709234955.2275131,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1709234955.2458332,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1709234955.2476993,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1709234955.2515292,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1709234955.2925441,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1709234955.2972531,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1709234955.3240802,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1709234955.3261254,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1709234955.327991,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1709234955.3300033,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1709234955.332402,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1709234955.3339541,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1709234955.3362038,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1709234955.3384557,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1709234955.3407202,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1709234955.3430088,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1709234955.3448718,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1709234955.3465729,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1709234955.348772,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1709234955.3506808,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1709234955.3524892,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1709234955.3541062,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1709234955.3562632,"name":"offline","context":{"idset":"564"}}
+{"timestamp":1709234955.3582137,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1709234955.360203,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1709234955.3625131,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1709234955.3626812,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1709234955.3665466,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1709234955.3679025,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1709234955.3696804,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1709234955.3716671,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1709234955.3736317,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1709234955.3749721,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1709234955.3822908,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1709234955.3839734,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1709234955.3865089,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1709234955.388659,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1709234955.3900292,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1709234955.3943198,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1709234955.3944838,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1709234955.3950067,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1709234955.39851,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1709234955.399823,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1709234955.402024,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1709234955.4035094,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1709234955.4051373,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1709234955.4065819,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1709234955.4080706,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1709234955.4095268,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1709234955.4111269,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1709234955.4123707,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1709234955.4140081,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1709234955.4163499,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1709234955.4177375,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1709234955.4191103,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1709234955.4202983,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1709234955.421855,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1709234955.4282215,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1709234955.4287834,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1709234955.429224,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1709234955.4306827,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1709234955.432368,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1709234955.4338264,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1709234955.4374981,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1709234955.4389572,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1709234955.4402354,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1709234955.4427958,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1709234955.4564171,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1709234955.4589977,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1709234955.465749,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1709234955.485811,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1709234955.4893167,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1709234955.4985461,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1709234955.4986808,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1709234955.5029602,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1709234955.517432,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1709234955.5223818,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1709234955.5242035,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1709234955.5287769,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1709234955.5433438,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1709234955.5452099,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1709234955.5490005,"name":"offline","context":{"idset":"683"}}
+{"timestamp":1709234955.5511434,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1709234955.5550697,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1709234955.5605462,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1709234955.5615573,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1709234955.5643823,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1709234955.5660958,"name":"offline","context":{"idset":"677"}}
+{"timestamp":1709234955.5684686,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1709234955.5708861,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1709234955.5726845,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1709234955.5736523,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1709234955.5748186,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1709234955.5762262,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1709234955.5773537,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1709234955.5785563,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1709234955.579447,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1709234955.594393,"name":"offline","context":{"idset":"457"}}
+{"timestamp":1709234955.5977571,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1709234955.6010618,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1709234955.6038642,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1709234955.6049542,"name":"offline","context":{"idset":"466"}}
+{"timestamp":1709234955.6061859,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1709234955.6073589,"name":"offline","context":{"idset":"701"}}
+{"timestamp":1709234955.6086633,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1709234955.6135056,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1709234955.6146059,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1709234955.624146,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1709234955.6255016,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1709234955.6266274,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1709234955.6323674,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1709234955.652365,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1709234955.6722608,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1709234955.67642,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1709234955.682267,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1709234955.7771437,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1709234955.8168073,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1709234955.838515,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1709234955.916383,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1709234955.9409788,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1709234955.9583316,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1709234955.9804285,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1709234955.9915938,"name":"offline","context":{"idset":"706"}}
+{"timestamp":1709234955.9932086,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1709234955.9952729,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1709234956.0201108,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1709234956.0242977,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1709234956.0660625,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1709234956.0668733,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1709234956.0720999,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1709234956.0824883,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1709234956.0910435,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1709234956.1123674,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1709234956.1151364,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1709234956.1222794,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1709234956.1230645,"name":"offline","context":{"idset":"681"}}
+{"timestamp":1709234956.1268976,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1709234956.1364233,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1709234956.1495831,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1709234956.1566217,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1709234956.1709146,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1709234956.1784737,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1709234956.1807916,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1709234956.1816847,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1709234956.1824965,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1709234956.1833689,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1709234956.1877475,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1709234956.1939371,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1709234956.1945879,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1709234956.1951387,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1709234956.203326,"name":"offline","context":{"idset":"707"}}
+{"timestamp":1709234956.2048819,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1709234956.2083111,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1709234956.2093811,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1709234956.2105217,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1709234956.2129991,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1709234956.2185199,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1709234956.2298303,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1709234956.2435763,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1709234956.2463298,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1709234956.2559786,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1709234956.2677312,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1709234956.2695224,"name":"offline","context":{"idset":"679"}}
+{"timestamp":1709234956.2770402,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1709234956.2839503,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1709234956.2869756,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1709234956.2876139,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1709234956.2925365,"name":"offline","context":{"idset":"563"}}
+{"timestamp":1709234956.2932513,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1709234956.2938654,"name":"offline","context":{"idset":"690"}}
+{"timestamp":1709234956.2944143,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1709234956.3000746,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1709234956.3107095,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1709234956.3116286,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1709234956.3153679,"name":"offline","context":{"idset":"447"}}
+{"timestamp":1709234956.316226,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1709234956.3166108,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1709234956.3173161,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1709234956.3178906,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1709234956.3204327,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1709234956.3210387,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1709234956.3221178,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1709234956.3283672,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1709234956.3410575,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1709234956.3422465,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1709234956.3446479,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1709234956.3522987,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1709234956.3565681,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1709234956.3777165,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1709234956.3841572,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1709234956.3862441,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1709234956.3881352,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1709234956.3884997,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1709234956.3904655,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1709234956.3908906,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1709234956.3963625,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1709234956.3972201,"name":"offline","context":{"idset":"468"}}
+{"timestamp":1709234956.4018633,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1709234956.4022088,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1709234956.4087889,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1709234956.410311,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1709234956.4105914,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1709234956.4220104,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1709234956.4246807,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1709234956.4300442,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1709234956.4402306,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1709234956.4439473,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1709234956.4516275,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1709234956.4552894,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1709234956.4561198,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1709234956.4561512,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1709234956.4652858,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1709234956.4829972,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1709234956.4848125,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1709234956.4991326,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1709234956.5020344,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1709234956.5147426,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1709236319.0723307,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1708981365.6981018,"reason":"reason=slingshot"},"61-76":{"timestamp":1709058456.7185023,"reason":"reason=HPE Rabbit CMM work"},"115":{"timestamp":1708559195.3493366,"reason":"reason=Lustre reconnects"},"117":{"timestamp":1708553594.8015115,"reason":"reason=cxi_healthcheck fails"},"123":{"timestamp":1708555411.809015,"reason":"reason=not ok 3 - check_ama: Failed"},"119-120,122,124":{"timestamp":1709071170.5978725,"reason":"reason=RMA timeout to RX context NIC"},"173":{"timestamp":1708562166.6530974,"reason":"reason=Lustre reconnects"},"174":{"timestamp":1708559300.4250147,"reason":"reason=Lustre reconnects"},"175":{"timestamp":1708557636.8754659,"reason":"reason=Lustre reconnects"},"176":{"timestamp":1708563246.2713401,"reason":"reason=Lustre reconnects"},"177":{"timestamp":1708559393.7865355,"reason":"reason=Lustre reconnects"},"198":{"timestamp":1708752840.6255207,"reason":"broker was unresponsive"},"238":{"timestamp":1707762774.7029541,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"242":{"timestamp":1708563325.1050918,"reason":"reason=Lustre reconnects"},"252":{"timestamp":1709100851.9288552,"reason":"nodediag failed cxi"},"280":{"timestamp":1708971582.5312283,"reason":"broker was unresponsive"},"302":{"timestamp":1708560415.8146031,"reason":"reason=Lustre reconnects"},"303":{"timestamp":1708559716.4510095,"reason":"reason=Lustre reconnects"},"308":{"timestamp":1708563363.4001403,"reason":"reason=Lustre reconnects"},"348":{"timestamp":1708567417.1524184,"reason":"reason=Bad Write Checksum"},"353":{"timestamp":1708559261.2453611,"reason":"reason=Lustre reconnects"},"384":{"timestamp":1708560007.3056688,"reason":"reason=Lustre reconnects"},"385":{"timestamp":1708557701.009788,"reason":"reason=Lustre reconnects"},"386":{"timestamp":1708559087.3058999,"reason":"reason=Lustre reconnects"},"388":{"timestamp":1708559628.0742533,"reason":"reason=Lustre reconnects"},"395":{"timestamp":1708565880.3856373,"reason":"reason=Lustre reconnects"},"398":{"timestamp":1708559116.4876347,"reason":"reason=Lustre reconnects"},"440":{"timestamp":1708558996.3552485,"reason":"reason=Lustre reconnects"},"118,438,442":{"timestamp":1709072276.6204038,"reason":"reason=RMA timeout to RX context NIC"},"443":{"timestamp":1708989464.935338,"reason":"lctl ping fails"},"487":{"timestamp":1708974786.5334918,"reason":"broker was unresponsive"},"496":{"timestamp":1708559544.9615338,"reason":"reason=Lustre reconnects"},"525":{"timestamp":1708557755.7074418,"reason":"reason=Lustre reconnects"},"540":{"timestamp":1708555919.6013906,"reason":"epilog failed for jobid feZmGM1MKmZ"},"550":{"timestamp":1707763119.3080339,"reason":"reason=cxi1[hsi1]: p0: resetting serdes PLLs"},"557":{"timestamp":1708653101.6627135,"reason":"reason=Bad Write Checksum"},"565":{"timestamp":1708963344.531615,"reason":"broker was unresponsive"},"568":{"timestamp":1708989638.2017982,"reason":"lctl ping fails"},"569":{"timestamp":1709076517.6726611,"reason":"reason=RMA timeout to RX context NIC"},"637":{"timestamp":1708963958.5319269,"reason":"broker was unresponsive"},"663":{"timestamp":1708558887.4445813,"reason":"reason=cxi_healthcheck fails"},"668":{"timestamp":1708628067.8886561,"reason":"reason=Slingshot errors"},"693":{"timestamp":1709074302.3847253,"reason":"reason=RMA timeout to RX context NIC"},"181,184,186-187,311,437,441,444,571,696":{"timestamp":1708990999.8284125,"reason":"lctl ping fails"},"182,312-313,315,566-567,572,694,697":{"timestamp":1709072886.4759777,"reason":"reason=RMA timeout to RX context NIC"},"185,188,309,695,698":{"timestamp":1708986881.2772908,"reason":"lctl ping fails"},"121,183,314,316,570,699":{"timestamp":1709073322.652173,"reason":"reason=RMA timeout to RX context NIC"},"310,439,700":{"timestamp":1708986881.2772908,"reason":"reason=Luustre hung"},"751":{"timestamp":1707763165.571362,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."}},"online":"","exclude":"0"}}
+{"timestamp":1709236319.0962739,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709236355.2792511,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709236356.1727407,"name":"online","context":{"idset":"28,51-52,59"}}
+{"timestamp":1709236356.372843,"name":"online","context":{"idset":"6-7,9-11,13-27,29-38,40-50,53-58,60"}}
+{"timestamp":1709236356.5280776,"name":"online","context":{"idset":"115,123,173-177,213,242,302-303,308,348,353,384-386,388,395,398,440,496,525,540,565,567,569,572,637,663,668,751"}}
+{"timestamp":1709236356.6291032,"name":"online","context":{"idset":"557"}}
+{"timestamp":1709236357.6033392,"name":"online","context":{"idset":"90"}}
+{"timestamp":1709236358.4124539,"name":"online","context":{"idset":"439,605,710"}}
+{"timestamp":1709236358.6087253,"name":"online","context":{"idset":"298,328,340,375,574"}}
+{"timestamp":1709236358.744215,"name":"online","context":{"idset":"408,527"}}
+{"timestamp":1709236358.8646438,"name":"online","context":{"idset":"89,111,116,274,310,330,382,468,531,642,671,722,736"}}
+{"timestamp":1709236358.9673753,"name":"online","context":{"idset":"99,102,104,114,161,169,191,253,270,288,307,334,352,446,486,498,593-594,623,659,662,707"}}
+{"timestamp":1709236359.0726569,"name":"online","context":{"idset":"79,105,124,138,142,155,209,247,290,297,304,316,354-355,378,390,434,458,471,586,589,609,614,640,658,731"}}
+{"timestamp":1709236359.1780214,"name":"online","context":{"idset":"5,140,158,167,218,227-228,232,241,248,258,292-293,306,416,424,443,451,464,556,602,610,648,655"}}
+{"timestamp":1709236359.2798054,"name":"online","context":{"idset":"181,233,245,320,346,366,381,411,462,469-470,541,568,672,677,682,718"}}
+{"timestamp":1709236359.3835452,"name":"online","context":{"idset":"3,145,203,212,226,236-237,311,324,326,337,365,368,372,400,483,495,515,544,601,611,613,616,639,686,703,742,746"}}
+{"timestamp":1709236359.4936714,"name":"online","context":{"idset":"69,82,110,127,184,188,204,215,219,221,225,229,259,266,314,322,357,364,380,393,402,420,432,450,456,463,499,506-507,514,517,529,532,538,545,570,578,587,591,606-607,619,641,646,652,678,681,689,704-705,711,720,723,734,745"}}
+{"timestamp":1709236359.5940821,"name":"online","context":{"idset":"86,150,153,168,199,240,261,273,332,341,345,349,359,362,367,391,407,418,423,425,431,433,453,505,563,585,618,629,669,684,691,749,752"}}
+{"timestamp":1709236359.6991391,"name":"online","context":{"idset":"96,108,119,143,148,154,185,189,193,202,207-208,223,244,279,283,289,356,361,392,429,447,485,494,523,548,555,599,612,622,635,638,645,698,715,738-739,747"}}
+{"timestamp":1709236359.815732,"name":"online","context":{"idset":"8,94-95,100,103,113,135-136,164,178-180,182,255,263,276,379,501,522,577,598,643,649,717"}}
+{"timestamp":1709236359.9243338,"name":"online","context":{"idset":"1,62,77,93,97,118,121,125,131,147,149,151-152,160,162,166,183,192,195-196,201,224,254,260,275,285-286,291,300-301,325,339,370-371,413,444-445,472,475,482,488,492,497,504,513,524,535,539,542,564,575-576,579,588,621,625,630-631,661,664,683,695-696,706,709,714,728,750"}}
+{"timestamp":1709236360.0278602,"name":"online","context":{"idset":"83,91,101,107,112,141,200,211,214,216,220,222,250,282,318,336,397,399,427,436,479,489,509,520-521,533,537,543,547,549,551,573,582,595-597,624,647,651,667,679,685,687,693,701,727,732,756"}}
+{"timestamp":1709236360.1379671,"name":"online","context":{"idset":"4,12,122,126,129,144,187,262,268,271,294,309,312,329,333,405-406,419,452,474,477,554,617,636,644,675,716,733,744"}}
+{"timestamp":1709236360.2502639,"name":"online","context":{"idset":"106,120,128,137,139,159,172,198,205-206,210,246,251,278,280,287,299,305,344,373,377,387,396,401,409,421-422,428,441-442,448,454,457,460,466-467,473,481,484,493,500,511,518,581,600,604,615,628,633,650,653-654,660,674,676,680,713,721,729-730,754"}}
+{"timestamp":1709236360.3544967,"name":"online","context":{"idset":"2,70,78,84,88,130,132,156,170,197,231,239,265,267,281,296,315,327,347,363,369,374,389,426,437,459,476,508,512,516,526,528,536,558,561,566,580,592,620,688,694,708,735,741,743,755"}}
+{"timestamp":1709236360.4593587,"name":"online","context":{"idset":"87,249,313,323,350-351,394,465,480,510,534,656,700"}}
+{"timestamp":1709236360.5652401,"name":"online","context":{"idset":"80,98,109,165,194,272,319,342-343,360,376,403,414,449,455,478,491,608,665,697,725-726"}}
+{"timestamp":1709236360.6786449,"name":"online","context":{"idset":"81,133,157,234,284,338,410,412,415,417,430,461,562,583,603,627,699,702,748"}}
+{"timestamp":1709236360.8002577,"name":"online","context":{"idset":"163,171,230,252,257,264,331,438,502,519,530,590,657,666,670,673,692,719,724,740,753"}}
+{"timestamp":1709236360.90536,"name":"online","context":{"idset":"85,92,146,190,217,235,243,256,269,277,490,546,552,559-560,571,584,626,690,712"}}
+{"timestamp":1709236361.0113361,"name":"online","context":{"idset":"186,295,321,335,358,404,435,503,632,737"}}
+{"timestamp":1709236361.1125176,"name":"online","context":{"idset":"383,553,634"}}
+{"timestamp":1709236361.2254219,"name":"online","context":{"idset":"134"}}
+{"timestamp":1709236361.5076694,"name":"online","context":{"idset":"317"}}
+{"timestamp":1709238398.629365,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1708981365.6981018,"reason":"reason=slingshot"},"61-76":{"timestamp":1709058456.7185023,"reason":"reason=HPE Rabbit CMM work"},"115":{"timestamp":1708559195.3493366,"reason":"reason=Lustre reconnects"},"117":{"timestamp":1708553594.8015115,"reason":"reason=cxi_healthcheck fails"},"123":{"timestamp":1708555411.809015,"reason":"reason=not ok 3 - check_ama: Failed"},"119-120,122,124":{"timestamp":1709071170.5978725,"reason":"reason=RMA timeout to RX context NIC"},"173":{"timestamp":1708562166.6530974,"reason":"reason=Lustre reconnects"},"174":{"timestamp":1708559300.4250147,"reason":"reason=Lustre reconnects"},"175":{"timestamp":1708557636.8754659,"reason":"reason=Lustre reconnects"},"176":{"timestamp":1708563246.2713401,"reason":"reason=Lustre reconnects"},"177":{"timestamp":1708559393.7865355,"reason":"reason=Lustre reconnects"},"198":{"timestamp":1708752840.6255207,"reason":"broker was unresponsive"},"238":{"timestamp":1707762774.7029541,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting"},"242":{"timestamp":1708563325.1050918,"reason":"reason=Lustre reconnects"},"252":{"timestamp":1709100851.9288552,"reason":"nodediag failed cxi"},"280":{"timestamp":1708971582.5312283,"reason":"broker was unresponsive"},"302":{"timestamp":1708560415.8146031,"reason":"reason=Lustre reconnects"},"303":{"timestamp":1708559716.4510095,"reason":"reason=Lustre reconnects"},"308":{"timestamp":1708563363.4001403,"reason":"reason=Lustre reconnects"},"348":{"timestamp":1708567417.1524184,"reason":"reason=Bad Write Checksum"},"353":{"timestamp":1708559261.2453611,"reason":"reason=Lustre reconnects"},"384":{"timestamp":1708560007.3056688,"reason":"reason=Lustre reconnects"},"385":{"timestamp":1708557701.009788,"reason":"reason=Lustre reconnects"},"386":{"timestamp":1708559087.3058999,"reason":"reason=Lustre reconnects"},"388":{"timestamp":1708559628.0742533,"reason":"reason=Lustre reconnects"},"395":{"timestamp":1708565880.3856373,"reason":"reason=Lustre reconnects"},"398":{"timestamp":1708559116.4876347,"reason":"reason=Lustre reconnects"},"440":{"timestamp":1708558996.3552485,"reason":"reason=Lustre reconnects"},"118,438,442":{"timestamp":1709072276.6204038,"reason":"reason=RMA timeout to RX context NIC"},"443":{"timestamp":1708989464.935338,"reason":"lctl ping fails"},"487":{"timestamp":1708974786.5334918,"reason":"broker was unresponsive"},"496":{"timestamp":1708559544.9615338,"reason":"reason=Lustre reconnects"},"525":{"timestamp":1708557755.7074418,"reason":"reason=Lustre reconnects"},"540":{"timestamp":1708555919.6013906,"reason":"epilog failed for jobid feZmGM1MKmZ"},"550":{"timestamp":1707763119.3080339,"reason":"reason=cxi1[hsi1]: p0: resetting serdes PLLs"},"557":{"timestamp":1708653101.6627135,"reason":"reason=Bad Write Checksum"},"565":{"timestamp":1708963344.531615,"reason":"broker was unresponsive"},"568":{"timestamp":1708989638.2017982,"reason":"lctl ping fails"},"569":{"timestamp":1709076517.6726611,"reason":"reason=RMA timeout to RX context NIC"},"637":{"timestamp":1708963958.5319269,"reason":"broker was unresponsive"},"663":{"timestamp":1708558887.4445813,"reason":"reason=cxi_healthcheck fails"},"668":{"timestamp":1708628067.8886561,"reason":"reason=Slingshot errors"},"693":{"timestamp":1709074302.3847253,"reason":"reason=RMA timeout to RX context NIC"},"181,184,186-187,311,437,441,444,571,696":{"timestamp":1708990999.8284125,"reason":"lctl ping fails"},"182,312-313,315,566-567,572,694,697":{"timestamp":1709072886.4759777,"reason":"reason=RMA timeout to RX context NIC"},"185,188,309,695,698":{"timestamp":1708986881.2772908,"reason":"lctl ping fails"},"121,183,314,316,570,699":{"timestamp":1709073322.652173,"reason":"reason=RMA timeout to RX context NIC"},"310,439,700":{"timestamp":1708986881.2772908,"reason":"reason=Luustre hung"},"751":{"timestamp":1707763165.571362,"reason":"reason=cxi0[hsi0] link != up. Actual value = starting."}},"online":"","exclude":"0"}}
+{"timestamp":1709238398.6519594,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709238435.049705,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709238435.8737693,"name":"online","context":{"idset":"3-4,7-10,12-19,21-22,25-28,30-35,37-38,40,42,44,46-53,55-60"}}
+{"timestamp":1709238435.9742601,"name":"online","context":{"idset":"1-2,5-6,11,20,23-24,29,36,41,43,45,54"}}
+{"timestamp":1709238436.3771935,"name":"online","context":{"idset":"62,70,88-90,92,94-108,115,118-122,124-129,131-133,135-139,141,143-148,150-151,153,156-159,161-171,173-188,197-202,204,213-215,217,219,229,231,233,235,242,245-247,250,252,255,259-263,265-272,274,277-278,280,282-285,287,289-291,294,296-297,300-312,314-316,325-332,341-344,348-364,373-389,391-396,398-400,402-403,405-409,411-412,414-423,425-428,437-444,461-469,474-479,481-484,493,495-500,509-511,514-522,524-529,531-533,535-537,539-541,543-548,552,554-556,558,560,562,564-572,589-598,600-613,615-616,619,637-644,654-658,661,663-666,668,677-679,681-683,685,687-688,690,692-694,696-701,704-712,715,717,720-723,726-727,731,733-740,749-752,754-756"}}
+{"timestamp":1709238436.477772,"name":"online","context":{"idset":"69,85-87,91,93,123,130,134,140,142,149,152,154,160,172,203,218,220,230,232,236,253,256-258,264,273,275-276,281,286,288,292-293,298-299,313,390,397,404,410,413,470,472,480,512-513,523,530,549,557,561,614,617-618,620,659-660,662,667,680,684,686,689,691,695,703,713,716,718,724-725,730,753"}}
+{"timestamp":1709238436.6701183,"name":"online","context":{"idset":"155,216,234,248-249,251,345-347,401,424,471,473,534,538,542,563,599,653,702,714,728-729,732"}}
+{"timestamp":1709238436.7707121,"name":"online","context":{"idset":"254,279,295,494,553,559,719"}}
+{"timestamp":1709238455.568481,"name":"online","context":{"idset":"746"}}
+{"timestamp":1709238456.0575681,"name":"online","context":{"idset":"445"}}
+{"timestamp":1709238457.8655539,"name":"online","context":{"idset":"489"}}
+{"timestamp":1709238458.0358288,"name":"online","context":{"idset":"745"}}
+{"timestamp":1709238458.6512821,"name":"online","context":{"idset":"505"}}
+{"timestamp":1709238464.4067719,"name":"online","context":{"idset":"551"}}
+{"timestamp":1709238464.8986573,"name":"online","context":{"idset":"338,434"}}
+{"timestamp":1709238474.3646998,"name":"online","context":{"idset":"575"}}
+{"timestamp":1709238476.074662,"name":"online","context":{"idset":"676"}}
+{"timestamp":1709238478.7106841,"name":"online","context":{"idset":"573"}}
+{"timestamp":1709238483.6878219,"name":"online","context":{"idset":"626"}}
+{"timestamp":1709238491.4103258,"name":"online","context":{"idset":"458"}}
+{"timestamp":1709238495.1277745,"name":"online","context":{"idset":"243"}}
+{"timestamp":1709238500.5142159,"name":"online","context":{"idset":"324"}}
+{"timestamp":1709238501.9853644,"name":"online","context":{"idset":"365"}}
+{"timestamp":1709238503.5628955,"name":"online","context":{"idset":"744"}}
+{"timestamp":1709238506.2655246,"name":"online","context":{"idset":"492"}}
+{"timestamp":1709238509.4876964,"name":"online","context":{"idset":"508"}}
+{"timestamp":1709238515.9289658,"name":"online","context":{"idset":"77"}}
+{"timestamp":1709238518.6764417,"name":"online","context":{"idset":"460"}}
+{"timestamp":1709238520.0248256,"name":"online","context":{"idset":"502"}}
+{"timestamp":1709238521.6419423,"name":"online","context":{"idset":"631"}}
+{"timestamp":1709238522.950521,"name":"online","context":{"idset":"624"}}
+{"timestamp":1709238523.4550607,"name":"online","context":{"idset":"79"}}
+{"timestamp":1709238531.3243625,"name":"online","context":{"idset":"506"}}
+{"timestamp":1709238538.5577614,"name":"online","context":{"idset":"84"}}
+{"timestamp":1709238538.7370517,"name":"online","context":{"idset":"212"}}
+{"timestamp":1709238541.6734571,"name":"online","context":{"idset":"80"}}
+{"timestamp":1709238541.9519091,"name":"online","context":{"idset":"669"}}
+{"timestamp":1709238544.8990052,"name":"online","context":{"idset":"581"}}
+{"timestamp":1709238546.6070528,"name":"online","context":{"idset":"451"}}
+{"timestamp":1709238548.06952,"name":"online","context":{"idset":"430,649"}}
+{"timestamp":1709238550.3764429,"name":"online","context":{"idset":"109"}}
+{"timestamp":1709238557.2362196,"name":"online","context":{"idset":"673"}}
+{"timestamp":1709238558.4661241,"name":"online","context":{"idset":"241"}}
+{"timestamp":1709238562.4427915,"name":"online","context":{"idset":"635"}}
+{"timestamp":1709238564.0241971,"name":"online","context":{"idset":"196"}}
+{"timestamp":1709238565.323149,"name":"online","context":{"idset":"224"}}
+{"timestamp":1709238566.8630159,"name":"online","context":{"idset":"366"}}
+{"timestamp":1709238571.02809,"name":"online","context":{"idset":"239"}}
+{"timestamp":1709238572.1708305,"name":"online","context":{"idset":"633"}}
+{"timestamp":1709238572.8552113,"name":"online","context":{"idset":"244"}}
+{"timestamp":1709238574.7585516,"name":"online","context":{"idset":"211"}}
+{"timestamp":1709238575.9383357,"name":"online","context":{"idset":"748"}}
+{"timestamp":1709238576.6458642,"name":"online","context":{"idset":"585"}}
+{"timestamp":1709238577.2229214,"name":"online","context":{"idset":"634"}}
+{"timestamp":1709238577.760062,"name":"online","context":{"idset":"625"}}
+{"timestamp":1709238579.5510087,"name":"online","context":{"idset":"192"}}
+{"timestamp":1709238581.231972,"name":"online","context":{"idset":"671"}}
+{"timestamp":1709238581.3953929,"name":"online","context":{"idset":"743"}}
+{"timestamp":1709238581.5285251,"name":"online","context":{"idset":"111,113,190,194,228,240,321,333,339-340,370,431,435,448,450,485,576,623,627,646,650"}}
+{"timestamp":1709238581.643641,"name":"online","context":{"idset":"78,189,191,195,205,221,223,323,369,372,455,457,486,501,503,582-583,588,621,645,647-648,672,675"}}
+{"timestamp":1709238581.7442179,"name":"online","context":{"idset":"82-83,110,206,210,237,318,334,336-337,367,371,429,433,436,446-447,449,456,490,504,574,577,580,586,628-629"}}
+{"timestamp":1709238581.8568547,"name":"online","context":{"idset":"116,207-208,222,226-227,317,320,335,452,454,488,491,507,584,651,741"}}
+{"timestamp":1709238582.0167327,"name":"online","context":{"idset":"81,319"}}
+{"timestamp":1709238582.6016459,"name":"online","context":{"idset":"112,114,193,209,225,322,622,636,652,674,742"}}
+{"timestamp":1709238582.7028279,"name":"online","context":{"idset":"368,432,453,578-579,630,632,670"}}
+{"timestamp":1709238582.8180759,"name":"online","context":{"idset":"459,587,747"}}
+{"timestamp":1709242352.1051919,"name":"drain","context":{"idset":"1-36,38,40-60","reason":"reason=Disabled for IOR test","overwrite":0}}
+{"timestamp":1709246654.6660411,"name":"undrain","context":{"idset":"36"}}
+{"timestamp":1709248350.6873209,"name":"undrain","context":{"idset":"1-35,38-60"}}
+{"timestamp":1709248366.7295237,"name":"drain","context":{"idset":"39","reason":"reason=cxi","overwrite":0}}
+{"timestamp":1709250136.1293676,"name":"drain","context":{"idset":"25-48","reason":"cxi","overwrite":1}}
+{"timestamp":1709336299.7795324,"name":"undrain","context":{"idset":"61-76,115,117-124,173-177,181-188,198,238,242,252,280,302-303,308-316,348,353,384-386,388,395,398,437-444,487,496,525,540,550,557,565-572,637,663,668,693-700,751"}}
+{"timestamp":1709336332.4828737,"name":"drain","context":{"idset":"238","reason":"reason=cxi_healthcheck fails","overwrite":0}}
+{"timestamp":1709336341.4207382,"name":"drain","context":{"idset":"751","reason":"reason=cxi_healthcheck fails","overwrite":0}}
+{"timestamp":1709336358.0870526,"name":"drain","context":{"idset":"348","reason":"reason=Lustre Checksum fails","overwrite":0}}
+{"timestamp":1709336806.3089643,"name":"drain","context":{"idset":"550","reason":"reason=No power control","overwrite":0}}
+{"timestamp":1709338058.1759651,"name":"online","context":{"idset":"61,65-66,68,71,73,76"}}
+{"timestamp":1709338058.276221,"name":"online","context":{"idset":"63-64,67,72,74"}}
+{"timestamp":1709338058.4301603,"name":"online","context":{"idset":"75"}}
+{"timestamp":1709338191.8377986,"name":"online","context":{"idset":"117"}}
+{"timestamp":1709338296.8350663,"name":"online","context":{"idset":"487"}}
+{"timestamp":1709338314.7408817,"name":"undrain","context":{"idset":"25-38,40-48"}}
+{"timestamp":1709338350.2706728,"name":"drain","context":{"idset":"39","reason":"reason=Flipped SS Y cable","overwrite":1}}
+{"timestamp":1709338714.9651678,"name":"drain","context":{"idset":"39","reason":"reason=Flipped SS Y cable with merced255","overwrite":1}}
+{"timestamp":1709339223.7730668,"name":"undrain","context":{"idset":"348"}}
+{"timestamp":1709342944.7175205,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709342944.7417302,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709342976.7542698,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709342977.5764418,"name":"online","context":{"idset":"5,7,9-11,14,16-17,19,22-34,36-37,41,43,46,48,50,52-54,56-58,60"}}
+{"timestamp":1709342977.6790307,"name":"online","context":{"idset":"6,8,12-13,15,18,20-21,35,38,40,42,44-45,47,49,51,55,59"}}
+{"timestamp":1709342978.0065241,"name":"online","context":{"idset":"1-4,77,127-128,146-148,169,197,242,248,255,302-303,314,321-322,324,326,328,330,332-334,337-340,342,345,353,355-357,375,381,385,388,398,425,432,440-441,443,449,473,478,485,496,515,525,531,540,567,569,572,640-641,645,647-649,663,701,717,722,751"}}
+{"timestamp":1709342978.109386,"name":"online","context":{"idset":"61-62,65-76,79-81,83-84,86-92,94-99,101-115,118-126,129-133,135-141,143-145,150-151,153-154,156-168,170-171,173-185,187,189-196,198-201,204-205,207-211,213-219,221-223,225,228-229,231-237,239-241,243-247,249-254,256-257,259,261-275,277-280,282,284-285,287,289-291,293-298,300-301,304-312,315-318,320,323,325,327,329,331,335-336,341,343-344,346-352,354,358-374,376-380,382-384,386-387,389,391-397,399-403,405-412,414-424,426-431,433-439,442,444-448,450-463,465-469,471-472,474-477,479,481-484,486-488,490-495,497-503,505-511,513-514,516-524,526-530,532-539,541-548,551-560,562-566,568,570-571,573-578,580-605,607-613,615-617,619-639,642-644,646,650-658,660-662,664-674,676-679,681-683,685,687-688,690-694,696-700,702,704-712,714-716,718-721,723,725-729,731-743,745-750,752,754-756"}}
+{"timestamp":1709342978.2106795,"name":"online","context":{"idset":"63-64,78,82,85,93,100,116-117,134,142,149,152,155,172,186,188,202-203,206,212,220,224,226-227,230,258,260,276,281,283,286,288,292,299,313,319,390,404,413,464,470,480,489,504,512,549,561,579,606,614,618,659,675,680,684,686,689,695,703,713,724,730,744,753"}}
+{"timestamp":1709343084.1450529,"name":"drain","context":{"idset":"348","reason":"reason=Lustre CHECKSUM errors","overwrite":0}}
+{"timestamp":1709345985.6236742,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709345985.6628237,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709346019.6598508,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709346020.3377686,"name":"online","context":{"idset":"6-7,9-11,15-19,21-31,34,36-38,40-44,47-49,52-57,59-60"}}
+{"timestamp":1709346020.4386716,"name":"online","context":{"idset":"5,8,12-14,20,32-33,35,45-46,50-51,58"}}
+{"timestamp":1709346020.8371317,"name":"online","context":{"idset":"748-756"}}
+{"timestamp":1709346022.155313,"name":"online","context":{"idset":"722"}}
+{"timestamp":1709346022.8075171,"name":"online","context":{"idset":"114,116,642,736"}}
+{"timestamp":1709346023.0291383,"name":"online","context":{"idset":"468,655,731"}}
+{"timestamp":1709346023.1741478,"name":"online","context":{"idset":"253,275,381,399,515,524,710"}}
+{"timestamp":1709346023.2909586,"name":"online","context":{"idset":"73,90,138,240,247,378,431,620,662"}}
+{"timestamp":1709346023.437408,"name":"online","context":{"idset":"104,297,352,439,609,671,684"}}
+{"timestamp":1709346023.5415521,"name":"online","context":{"idset":"102,124,142,175,233,242,293,307,355,392,498,574,586,594,610,614,640,658,723"}}
+{"timestamp":1709346023.6465158,"name":"online","context":{"idset":"67,517,544"}}
+{"timestamp":1709346023.7484901,"name":"online","context":{"idset":"62,79,161,168,218,220,258,298,330,340,382,453,463,506,659,707"}}
+{"timestamp":1709346023.8564701,"name":"online","context":{"idset":"85,93,99,221,224,226,237,248,290,310,334,365,424,451,497,509,522,527,553,602,605,622-623,685,739"}}
+{"timestamp":1709346023.9616516,"name":"online","context":{"idset":"108,110,120,160,181,245,349,354,388,441,446,541,638,647,672,677"}}
+{"timestamp":1709346024.0663996,"name":"online","context":{"idset":"69-70,96,101,140-141,188,195,215,229,241,255,261,287,311,357,367,423,469,486,540,542,591,618,646,686,688,746"}}
+{"timestamp":1709346024.1720383,"name":"online","context":{"idset":"1,3,109,219,263,274,283,289,314,316,320,332,337,356,366,368,370-371,375,402,407-408,416,450,483,494-495,604,619,669,678,682"}}
+{"timestamp":1709346024.2738824,"name":"online","context":{"idset":"63,71,74,125,154-155,204,208,228,232,279,300-301,308,362,372,433-434,443,458,464,467,470,482,492,529,538,555,565,576,578,611,613,624,629,641,643,648,683,698,705-706,711,741"}}
+{"timestamp":1709346024.3772063,"name":"online","context":{"idset":"66,77,86,105,111-112,123,136,145,150,153,158,166,170,174,180,184,191-192,196,203,209,212,225,227,273,282,292,294,304,306,323-324,326,328,345,359,361,364,377,379,390,400,445,459,504,514,523,532,563,570,616,621,633,636,639,645,664,674,681,696,728"}}
+{"timestamp":1709346024.4819345,"name":"online","context":{"idset":"115,117,119,127,135,167,176,193,222-223,236,266,268,288,303,325,420,440,456,462,471,477,485,499,505,507,525,535,556,561,569,607,649,652,703-704,742"}}
+{"timestamp":1709346024.5848031,"name":"online","context":{"idset":"82,100,131,143,178,182,200,254,280,347,380,397-398,401,403,411,488,508,531,580,587,598,600,612,617,628,644,689,701,717,744-745"}}
+{"timestamp":1709346024.6876962,"name":"online","context":{"idset":"2,4,106,118,126,128,162,169,183,198,210,244,265,291,315,318,336,339,346,387,391,393,413,418,421,472,479,510,512,516,518,520,564,568,573,575,577,588-589,599,601,606,630,651,667-668,680,695,714,716,730,732"}}
+{"timestamp":1709346024.793092,"name":"online","context":{"idset":"68,76,78,95,113,129,132,144,147-148,156,164,173,179,185,199,202,211,249,262,271,369,432,457,487,500,513,526,537,566,572,582,593,631,650,660,687,693-694,708,718,720,735,738"}}
+{"timestamp":1709346024.9021895,"name":"online","context":{"idset":"64,91-92,103,107,152,172,187,207,250,270,296,322,329,333,350,353,384,389,396,406,425,427,436-437,447,476,484,501,521,536,539,548-549,551-552,583,656,661,725,734"}}
+{"timestamp":1709346025.003958,"name":"online","context":{"idset":"65,72,84,88,177,189,205-206,214,231,239,259-260,267,281,285-286,299,305,313,331,338,363,373,394,474,489,533,543,545,547,558,579,595,608,625,635,663,665,676,691,709,715,721,747"}}
+{"timestamp":1709346025.1079001,"name":"online","context":{"idset":"83,97,121,130,139,157,159,165,197,243,251,256,272,276,284,309,312,319,327,342-343,386,405,409,417,426,428-429,448,460,465,473,475,478,481,511,534,557,592,653,666,727,729"}}
+{"timestamp":1709346025.2085681,"name":"online","context":{"idset":"80,134,137,146,151,201,246,278,438,444,466,493,546,554,560,581,585,596-597,615,626,634,637,654,673,675,697,733"}}
+{"timestamp":1709346025.3266823,"name":"online","context":{"idset":"87,98,133,171,341,344,422,452,454,490,530,562,567,670,700,712"}}
+{"timestamp":1709346025.4276493,"name":"online","context":{"idset":"61,89,186,235,257,264,395,410,412,419,442,449,455,491,496,528,590,679,743"}}
+{"timestamp":1709346025.5348942,"name":"online","context":{"idset":"194,217,234,302,317,351,360,385,415,502,657,690,702,724,726"}}
+{"timestamp":1709346025.6512969,"name":"online","context":{"idset":"81,122,216,230,252,295,335,348,376,414,430,435,480,503,603,692,699"}}
+{"timestamp":1709346025.7613323,"name":"online","context":{"idset":"75,94,163,190,213,269,277,321,374,383,584,632,713,719,737,740"}}
+{"timestamp":1709346025.8767915,"name":"online","context":{"idset":"149,358,404,559,571"}}
+{"timestamp":1709346026.1992543,"name":"online","context":{"idset":"461,519,627"}}
+{"timestamp":1709346516.196058,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709346516.2309995,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709346516.2312772,"name":"online","context":{"idset":"0-38,40-237,239-549,551-756"}}
+{"timestamp":1709348219.6938589,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709348219.7218783,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709348219.7221003,"name":"online","context":{"idset":"0-38,40-237,239-549,551-756"}}
+{"timestamp":1709348350.2855484,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709348350.3129451,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709348350.3131549,"name":"online","context":{"idset":"0-38,40-237,239-549,551-756"}}
+{"timestamp":1709348381.3822258,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709348381.4108262,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709348381.4115224,"name":"online","context":{"idset":"0-38,40-237,239-549,551-756"}}
+{"timestamp":1709417592.8369837,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709417592.8634663,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709417624.7008688,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709417625.8097584,"name":"online","context":{"idset":"7-38,40-60"}}
+{"timestamp":1709417625.9620194,"name":"online","context":{"idset":"348,748-756"}}
+{"timestamp":1709417627.651912,"name":"online","context":{"idset":"330,669"}}
+{"timestamp":1709417627.7580512,"name":"online","context":{"idset":"113"}}
+{"timestamp":1709417627.8785894,"name":"online","context":{"idset":"586"}}
+{"timestamp":1709417628.0718653,"name":"online","context":{"idset":"162,416"}}
+{"timestamp":1709417628.3266971,"name":"online","context":{"idset":"181,248,291,375,468,723"}}
+{"timestamp":1709417628.4330201,"name":"online","context":{"idset":"6,310,340,424,495,517,584,605"}}
+{"timestamp":1709417628.5545845,"name":"online","context":{"idset":"122,161,298,357,362,439,450,529,642,645"}}
+{"timestamp":1709417628.6574576,"name":"online","context":{"idset":"79,158,199,253,352,365,382,434,540,565,610,641,691,746"}}
+{"timestamp":1709417628.7577317,"name":"online","context":{"idset":"140,155,273,307,316,458,473,578,596,618,672,684-686,731"}}
+{"timestamp":1709417628.8777378,"name":"online","context":{"idset":"67,73,86,102,114,116,136,143-144,174,185,191,223,240,247,250,262-263,346,355-356,381,406,433,470,485,492,523,556,594,614,652,679,696,698,722,739"}}
+{"timestamp":1709417628.9796796,"name":"online","context":{"idset":"71,74,91,97,101,104,109,120,124,131,142,154,180,188-189,195,202,210-211,227,233,274,290,293,297,300,311,402,446,457,476,482,504,518,527,532,555,574,582,587,602,623,629,640,658,677,702,710"}}
+{"timestamp":1709417629.0820475,"name":"online","context":{"idset":"85,88,96,118,153,167,169,182,193,212,226,228-229,237,242,245,258,264,303,332,334,338,364,367,379,388,408,443,524,564,570,589,611,616,621,649,659,662,682,711,717,745"}}
+{"timestamp":1709417629.1884086,"name":"online","context":{"idset":"66,117,147,203-204,282,325,339,345,349,366,370-371,393,399,407,423,453,481,486,494,500,538,544,599,609,619,646,648,706"}}
+{"timestamp":1709417629.2912841,"name":"online","context":{"idset":"62-63,110,150,159,183,196,219,222,225,241,259,261,275,278,301,309,347,359,431,448,451,454,463,483,501,512,514,522,531,534,539,542,575-577,593,600,606,625,636,664,693,713,734"}}
+{"timestamp":1709417629.3984797,"name":"online","context":{"idset":"1,5,72,157,168,172,175-176,178,184,218,220-221,224,254,268,279,314,320,326,353,361,390,418,459,462,475,489,510,516,526,558,561,569,579,620,624,674,678,681,695,714-715,721"}}
+{"timestamp":1709417629.4995754,"name":"online","context":{"idset":"3,75,77,89,125-126,135,141,207,209,270,280,287-288,308,318,328,336-337,378,380,387,389,392,400,456,477,487,497-498,508-509,520-521,535,541,545,553,567,573,612-613,631,643,655,667,683,687,703,709,716"}}
+{"timestamp":1709417629.6060174,"name":"online","context":{"idset":"2,76,100,106,112,115,123,128,137,145,156,197,200,215,232,234,236,249,271,292,299,304-306,313,324,333,354,368,377,401,411,413,427,432,440,442,447,464,471-472,484,488,506-507,515,537,548,580,585,598,604,630,633,639,738,742"}}
+{"timestamp":1709417629.7091396,"name":"online","context":{"idset":"4,61,65,69,92,111,119,121,127,129,138-139,151,166,208,255,260,266,276,283,319,322,343,350,373,384,394,398,403,429,436,499,505,568,572,588,591,638,650,653,668,689,720,728,741,747"}}
+{"timestamp":1709417629.8221898,"name":"online","context":{"idset":"68,70,78,82-83,93,160,164,187,205-206,244,256,265,286,312,315,358,385,391,396,420-421,428,437,441,449,452,460,467,469,490,493,530,543,547,549,551,557,566,581,595,628,635,647,660,663,666,688,692,701,704,708,725,730,733"}}
+{"timestamp":1709417629.9308584,"name":"online","context":{"idset":"84,94-95,108,170,177,198,231,239,251,267,289,296,327,363,372,376,397,409,414,425,445,491,622,632,644,654,676,694,727,729,735-736"}}
+{"timestamp":1709417630.0351682,"name":"online","context":{"idset":"148,230,246,285,295,323,369,412,419,422,426,465,474,478,503,528,533,583,592,608,656,661,718"}}
+{"timestamp":1709417630.1406825,"name":"online","context":{"idset":"80,149,165,173,179,331,386,410,444,479,525,554,601,626,634,665,675"}}
+{"timestamp":1709417630.2574849,"name":"online","context":{"idset":"64,214,216,284,341,344,351,360,374,395,415,430,455,466,511,513,552,560,563,607,617,637,651,670,700,712,724,726,732"}}
+{"timestamp":1709417630.362304,"name":"online","context":{"idset":"130,132,152,186,194,201,217,294,321,329,335,502,562,590,597,680,705,707,740"}}
+{"timestamp":1709417630.4678793,"name":"online","context":{"idset":"81,107,133,146,163,171,190,235,252,257,277,302,317,383,405,435,438,480,496,536,546,571,690,699,744"}}
+{"timestamp":1709417630.5893884,"name":"online","context":{"idset":"87,105,134,192,213,243,272,281,342,404,461,519,559,603,615,627,657,673,719,743"}}
+{"timestamp":1709417630.6903541,"name":"online","context":{"idset":"90,98-99,269,417,671,737"}}
+{"timestamp":1709417630.8430088,"name":"online","context":{"idset":"103"}}
+{"timestamp":1709417630.9832442,"name":"online","context":{"idset":"697"}}
+{"timestamp":1709425589.9902151,"name":"drain","context":{"idset":"1","reason":"prolog failed for jobid fgNs1vQEBm9","overwrite":0}}
+{"timestamp":1709429070.6501646,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709429070.66818,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709429103.4013865,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709429104.2178597,"name":"online","context":{"idset":"6-7,9-10,12,14,16-17,19,21-26,28-30,34-35,38,41-42,44-48,51-59"}}
+{"timestamp":1709429104.3058012,"name":"online","context":{"idset":"8,11,13,15,18,20,27,31-33,36-37,40,43,49-50,60"}}
+{"timestamp":1709429104.7085834,"name":"online","context":{"idset":"348,748-752,754-756"}}
+{"timestamp":1709429104.8432722,"name":"online","context":{"idset":"753"}}
+{"timestamp":1709429106.8384836,"name":"online","context":{"idset":"113,677"}}
+{"timestamp":1709429106.9606838,"name":"online","context":{"idset":"122,458,485"}}
+{"timestamp":1709429107.2069123,"name":"online","context":{"idset":"67,253,723"}}
+{"timestamp":1709429107.3167815,"name":"online","context":{"idset":"731"}}
+{"timestamp":1709429107.4401696,"name":"online","context":{"idset":"424,468,642"}}
+{"timestamp":1709429107.5438626,"name":"online","context":{"idset":"470,517,565,602,606,636,648,669"}}
+{"timestamp":1709429107.6483295,"name":"online","context":{"idset":"457,609,623"}}
+{"timestamp":1709429107.7488797,"name":"online","context":{"idset":"79,86,94,124,136,248,262-263,307,310,352,370,416,434,439,446,450,555,596,605,659,679,691,710,715,722"}}
+{"timestamp":1709429107.8534606,"name":"online","context":{"idset":"104,142,193,224,240,258,266,316,346,356,494,523,527,535,586,611,693,696,746"}}
+{"timestamp":1709429107.9751101,"name":"online","context":{"idset":"181,298,311,339,379,381,473,524"}}
+{"timestamp":1709429108.0861044,"name":"online","context":{"idset":"117,143,157,237,354,362,372,382,433,495,529,604,641,645,662,686"}}
+{"timestamp":1709429108.1965899,"name":"online","context":{"idset":"1,109-110,114-116,140,160-161,188,261,274,290,330,337,340,355,357,367,375,377,402,561,564,574,587,610,619,658,706"}}
+{"timestamp":1709429108.2987347,"name":"online","context":{"idset":"88,155,174,203,245,247,254,273,283,297,332-333,365,540,567,584,594,614,618,629,652,698"}}
+{"timestamp":1709429108.3995252,"name":"online","context":{"idset":"66,77,92,131,148,154,199,202,219,225,291,293,309,325,338,359,406,418,431,463,504,522,532,551,578,593,612,631,664,739"}}
+{"timestamp":1709429108.5014122,"name":"online","context":{"idset":"3,5,89,95,97,125,150,210,212,241,270,334,349,361,401,469,484,488,500,520,538,569-570,613,635,640,650,716,721"}}
+{"timestamp":1709429108.6046307,"name":"online","context":{"idset":"73,96,137,200,206,268,308,326,368,392,400,451,482,497,514-515,572,582,589,630,646,667"}}
+{"timestamp":1709429108.7070203,"name":"online","context":{"idset":"71,76,123,145,167,178,189,228,233,242,251,264,282,303,306,314,345,371,378,388,407-408,420,427,456,464,498,507,534,537,541,568,575,591,616,639,649,655,681,695,702,711,742"}}
+{"timestamp":1709429108.8117254,"name":"online","context":{"idset":"62,65,70,74,112,127,158,162,168-169,172,176,182,195,209,211,223,259,278-279,292,301,304,324,347,364,413,423,432,440,453,462,487,509,544,556,598,621,672,684,687,689,735"}}
+{"timestamp":1709429108.915761,"name":"online","context":{"idset":"63,69,72,118,135,153,180,183-184,196-197,218,222,227,229,234,249,275,280,296,300,380,384,390,399,411,459,477,492,505,510,553,576-577,580,583,599,620,638,643,665,668,682-683,714,720,726,738"}}
+{"timestamp":1709429109.0211811,"name":"online","context":{"idset":"100,103,106,120,126,144,175,208,221,236,244,260,271,328,331,387,393-394,415,429,437,442,499,518,542,585,590,653,678,705,709,736"}}
+{"timestamp":1709429109.125607,"name":"online","context":{"idset":"2,4,83,101,128,159,177,215,226,232,288,305,385,398,426,443,449,486,506,547,558,579,595,628,644,651,663,688,692,703,730"}}
+{"timestamp":1709429109.2331424,"name":"online","context":{"idset":"61,75,82,84,87,99,121,141,147,156,179,185,191,220,246,250,265,267,299,318,323,336,341,344,369,376,421,428,436,445,471-472,474,476,508,511-512,521,526,548-549,557,573,588,632,685,708,713,717,725,729,741,747"}}
+{"timestamp":1709429109.3399806,"name":"online","context":{"idset":"68,129,170,256,286,289,295,327,353,373,483,501,545,563,607,625-626,654,676,680,694"}}
+{"timestamp":1709429109.4428015,"name":"online","context":{"idset":"119,198,205,207,285,312-313,412,447,452,466,489,491,502,531,543,624,627,647,674,707,728,734"}}
+{"timestamp":1709429109.5487578,"name":"online","context":{"idset":"130,138,173,187,194,255,257,276,343,363,389,391,409,414,441,467,516,539,554,617,633,656,700-701,733,745"}}
+{"timestamp":1709429109.6540463,"name":"online","context":{"idset":"80,102,111,139,151,164,166,201,204,231,239,284,287,315,342,358,386,395-397,403,419,425,444,448,454,475,478,481,493,503,513,530,566,600-601,603,622,660-661"}}
+{"timestamp":1709429109.7705677,"name":"online","context":{"idset":"78,134,152,216,320,351,360,460,490,597,608,615,637,666,704,724,744"}}
+{"timestamp":1709429109.8758798,"name":"online","context":{"idset":"81,146,165,321-322,329,417,435,455,479,496,525,533,560,562,634,675,718,732,743"}}
+{"timestamp":1709429110.0126805,"name":"online","context":{"idset":"64,133,294,317,335,383,410,422,465,528,536,592,657,670,673"}}
+{"timestamp":1709429110.1183798,"name":"online","context":{"idset":"98,107-108,132,163,230,235,302,430,438,552,571,671,697,699,712,719,740"}}
+{"timestamp":1709429110.2346599,"name":"online","context":{"idset":"85,90,93,186,190,213,217,277,281,366,404,546"}}
+{"timestamp":1709429110.3352416,"name":"online","context":{"idset":"91,192,243,350"}}
+{"timestamp":1709429110.4821079,"name":"online","context":{"idset":"171,214,272,319,374,461,480,690,727"}}
+{"timestamp":1709429110.5983746,"name":"online","context":{"idset":"149,252,405,519,581"}}
+{"timestamp":1709429110.755233,"name":"online","context":{"idset":"105,269,559"}}
+{"timestamp":1709429110.9122453,"name":"online","context":{"idset":"737"}}
+{"timestamp":1709572066.0958493,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709572066.121917,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709572066.1221917,"name":"online","context":{"idset":"0-38,40-237,239-549,551-756"}}
+{"timestamp":1709572405.3469982,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709572405.3786802,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709572405.378916,"name":"online","context":{"idset":"0-38,40-237,239-549,551-756"}}
+{"timestamp":1709583475.6141074,"name":"drain","context":{"idset":"569","reason":"prolog failed for jobid fgjQELtBquR","overwrite":0}}
+{"timestamp":1709603932.3996222,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"569":{"timestamp":1709583475.6141074,"reason":"prolog failed for jobid fgjQELtBquR"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709603932.4307849,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709603971.0369244,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709603972.1954389,"name":"online","context":{"idset":"348,749-755"}}
+{"timestamp":1709603972.9629014,"name":"online","context":{"idset":"375,431,492,502,542,578,693"}}
+{"timestamp":1709603973.0899165,"name":"online","context":{"idset":"48,59,307,397"}}
+{"timestamp":1709603973.2588792,"name":"online","context":{"idset":"24,45,449"}}
+{"timestamp":1709603973.3908484,"name":"online","context":{"idset":"8,18,47,393,455,481,641,663,732,747"}}
+{"timestamp":1709603973.4968977,"name":"online","context":{"idset":"1,28,30,49,54,370,615"}}
+{"timestamp":1709603973.6012611,"name":"online","context":{"idset":"478,514"}}
+{"timestamp":1709603973.7462034,"name":"online","context":{"idset":"715"}}
+{"timestamp":1709603973.8735962,"name":"online","context":{"idset":"330,376,587,687"}}
+{"timestamp":1709603973.9898551,"name":"online","context":{"idset":"450,561,702"}}
+{"timestamp":1709603974.093801,"name":"online","context":{"idset":"329,408,467,621,721-722"}}
+{"timestamp":1709603974.1984036,"name":"online","context":{"idset":"310,380,391,525,562,584,598,652,672"}}
+{"timestamp":1709603974.3157604,"name":"online","context":{"idset":"356,378,456,464,474,495,535,538,607,659,728,734"}}
+{"timestamp":1709603974.4547603,"name":"online","context":{"idset":"283,300,337,345,365,371,423,437,461,472,516,540,574,586,596,614,629,649,673,710,714,718,729,731"}}
+{"timestamp":1709603974.5608766,"name":"online","context":{"idset":"385,404-405,416,427,585,642,698"}}
+{"timestamp":1709603974.6778984,"name":"online","context":{"idset":"314,347,368,377,390,394,398,413,417-418,475,487,494,539,590,606,643,662,668,683,700,727"}}
+{"timestamp":1709603974.7796125,"name":"online","context":{"idset":"339,401,432,549,566,594,645,665,720"}}
+{"timestamp":1709603974.8849454,"name":"online","context":{"idset":"195,302,304,312,316,338,340,349,358,360,362,379,381,396,451,458,468-469,483,486,515,537,547,593,647,669,674,679,742-743,746"}}
+{"timestamp":1709603974.9850533,"name":"online","context":{"idset":"83,115,167,246,306,321,346,354,374,388,440-441,457,462,476,479,485,493,526-527,544,589,591,618,632,684,719"}}
+{"timestamp":1709603975.090131,"name":"online","context":{"idset":"80,156,236,273,285,290,305,318,334,410,428,447,482,488,505,509,519,541,556-559,575,580-581,604,609,617,623-625,666,675,677,682,701,709,713,716,733"}}
+{"timestamp":1709603975.1928141,"name":"online","context":{"idset":"90,174,180,240,254,277,303,336,400,406,425,511,522,532,563,565,567,588,613,620,637,660-661,707,725"}}
+{"timestamp":1709603975.307579,"name":"online","context":{"idset":"3,70,78,94,108,124,140,142,152,188,221,223,228,235,245,261,287,297,301,359,399,414,420,435-436,459,499,521,534,573,576,583,612,638,646,664,667,686,730"}}
+{"timestamp":1709603975.4125636,"name":"online","context":{"idset":"168,173,225,299,322,430,443,446,454,622,653,723"}}
+{"timestamp":1709603975.5198178,"name":"online","context":{"idset":"67,85,106,118,136,144,155,216,231-232,260,262-263,278,319,361,409,470,520,536,553,602,619,636,648,654,658,724"}}
+{"timestamp":1709603975.6220443,"name":"online","context":{"idset":"95,99,104-105,123,126,131,135,139,150,160,185,191,208,230,255-256,282,293,327,363,373,384,412,442,445,448,453,496,517,533,572,592,600,605,627,639-640,678,696,735"}}
+{"timestamp":1709603975.7264087,"name":"online","context":{"idset":"2,89,103,113,116,125,161,166,177,209,222,229,234,247,249-250,275,280,286,308,320,326,343,350,402,415,424,438,473,529-530,543,551,568,570,595,608,610-611,631,635,689,695,697,705"}}
+{"timestamp":1709603975.8320231,"name":"online","context":{"idset":"79,87,93,97,110,143,193,294,323,344,351,353,357,364,367,369,387,389,392,421,434,463,477,480,489,497-498,501,506,512,528,531,548,552,564,582,597,616,688,706,708,717,741,745"}}
+{"timestamp":1709603975.9367239,"name":"online","context":{"idset":"198,215,243,269,295,313,325,332,366,383,419,426,429,491,524,560,571,603,633,657,691,712,726,740"}}
+{"timestamp":1709603976.0459177,"name":"online","context":{"idset":"100,107,151,194,201,219,241,252,267-268,284,291,342,471,554-555,650,655,699,736,738"}}
+{"timestamp":1709603976.1493099,"name":"online","context":{"idset":"65,84,86,102,112,119,122,127,133,146,169,199,205,218,272,276,324,335,341,352,403,460,508,510,601,634,681,739,748"}}
+{"timestamp":1709603976.2576983,"name":"online","context":{"idset":"96,129,138,145,153-154,157-159,162-164,176,181-182,189,204,211,214,217,239,248,264,281,288,386,439,452,465,513,546,626,644"}}
+{"timestamp":1709603976.3620927,"name":"online","context":{"idset":"4,82,88,92,98,120-121,132,172,187,200,202,210,233,251,253,257,259,265,271,292,309,315,328,331,355,466,504,628,671,692,744"}}
+{"timestamp":1709603976.465822,"name":"online","context":{"idset":"111,147,170,183,311,407,500,518,545,694"}}
+{"timestamp":1709603976.5799758,"name":"online","context":{"idset":"91,101,109,137,141,186,190,196,206,220,242,244,289,296,411,433,484,490,523,630,651,690,737"}}
+{"timestamp":1709603976.6843235,"name":"online","context":{"idset":"149,165,184,192,197,203,207,213,226-227,266,279,333,382,395,444,579,670,676,685,703,711"}}
+{"timestamp":1709603976.7884178,"name":"online","context":{"idset":"117,148,179,237,258,274,317,372,422,503,507,577,599,704"}}
+{"timestamp":1709603976.8892887,"name":"online","context":{"idset":"114,128,134,171,175,224,270,298"}}
+{"timestamp":1709603977.0770588,"name":"online","context":{"idset":"212,656"}}
+{"timestamp":1709603977.3482652,"name":"online","context":{"idset":"680"}}
+{"timestamp":1709603977.518434,"name":"online","context":{"idset":"130,178"}}
+{"timestamp":1709678554.9041376,"name":"online","context":{"idset":"61"}}
+{"timestamp":1709678612.3001077,"name":"online","context":{"idset":"63-64,66,68,73-75,77"}}
+{"timestamp":1709678612.401186,"name":"online","context":{"idset":"62,69,71-72,76,81"}}
+{"timestamp":1709679616.9269445,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1709679616.9343245,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1709679616.9411206,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1709679616.9488688,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1709679617.0488274,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1709679617.1050417,"name":"offline","context":{"idset":"563"}}
+{"timestamp":1709679617.1202452,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1709679617.1320612,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1709679617.1912277,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1709679617.2035811,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1709679617.2145922,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1709679617.2234697,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1709679617.2538731,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1709679617.2584045,"name":"offline","context":{"idset":"659"}}
+{"timestamp":1709679617.2662239,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1709679617.2669091,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1709679617.2914188,"name":"offline","context":{"idset":"666"}}
+{"timestamp":1709679617.2920659,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1709679617.2976449,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1709679617.3043132,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1709679617.3278964,"name":"offline","context":{"idset":"677"}}
+{"timestamp":1709679617.3336763,"name":"offline","context":{"idset":"462"}}
+{"timestamp":1709679617.3421304,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1709679617.3427622,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1709679617.34779,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1709679617.3484726,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1709679617.352885,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1709679617.3587446,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1709679617.3885348,"name":"offline","context":{"idset":"658"}}
+{"timestamp":1709679617.38922,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1709679617.3899276,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1709679617.3905835,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1709679617.3912382,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1709679617.3918924,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1709679617.3925962,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1709679617.3932462,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1709679617.3938994,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1709679617.4065795,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1709679617.4073093,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1709679617.412266,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1709679617.4154539,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1709679617.4198217,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1709679617.4315031,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1709679617.4406521,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1709679617.4413075,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1709679617.4454744,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1709679617.4461155,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1709679617.4500232,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1709679617.4508188,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1709679617.4549019,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1709679617.4630055,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1709679617.4633613,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1709679617.4636753,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1709679617.4701395,"name":"offline","context":{"idset":"549"}}
+{"timestamp":1709679617.4721777,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1709679617.4763689,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1709679617.4773939,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1709679617.4804554,"name":"offline","context":{"idset":"675"}}
+{"timestamp":1709679617.4807782,"name":"offline","context":{"idset":"679"}}
+{"timestamp":1709679617.4847116,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1709679617.4850488,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1709679617.5451453,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1709679617.5474384,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1709679617.5504234,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1709679617.5555212,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1709679617.5559213,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1709679617.5861998,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1709679617.5868099,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1709679617.5951564,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1709679617.6007683,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1709679617.604156,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1709679617.6095362,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1709679617.6120949,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1709679617.6149909,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1709679617.6324394,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1709679617.632941,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1709679617.6370869,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1709679617.6378729,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1709679617.6417189,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1709679617.6424038,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1709679617.6464794,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1709679617.6471479,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1709679617.6519961,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1709679617.6560876,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1709679617.6568151,"name":"offline","context":{"idset":"665"}}
+{"timestamp":1709679617.6575508,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1709679617.6603463,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1709679617.6610632,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1709679617.6655552,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1709679617.6662676,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1709679617.6669495,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1709679617.6720922,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1709679617.6726441,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1709679617.6796527,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1709679617.6801238,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1709679617.6806607,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1709679617.686991,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1709679617.6967778,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1709679617.6984055,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1709679617.7018645,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1709679617.7025888,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1709679617.7065511,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1709679617.7111216,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1709679617.7424428,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1709679617.746876,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1709679617.747529,"name":"offline","context":{"idset":"445"}}
+{"timestamp":1709679617.7512364,"name":"offline","context":{"idset":"456"}}
+{"timestamp":1709679617.7521062,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1709679617.7559903,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1709679617.7567382,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1709679617.7596424,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1709679617.7603714,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1709679617.7610781,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1709679617.7677884,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1709679617.7722456,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1709679617.7769928,"name":"offline","context":{"idset":"561"}}
+{"timestamp":1709679617.7787192,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1709679617.7795203,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1709679617.7803178,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1709679617.7810988,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1709679617.7818868,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1709679617.7826881,"name":"offline","context":{"idset":"656"}}
+{"timestamp":1709679617.7834883,"name":"offline","context":{"idset":"672"}}
+{"timestamp":1709679617.801482,"name":"offline","context":{"idset":"706"}}
+{"timestamp":1709679617.8019435,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1709679617.807601,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1709679617.8320584,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1709679617.8493881,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1709679617.85448,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1709679617.8548541,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1709679617.8608239,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1709679617.8670967,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1709679617.8675971,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1709679617.8680277,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1709679617.8683851,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1709679617.8687007,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1709679617.8690472,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1709679617.8693962,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1709679617.8697257,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1709679617.8700802,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1709679617.8704314,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1709679617.8707602,"name":"offline","context":{"idset":"468"}}
+{"timestamp":1709679617.8710647,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1709679617.8713734,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1709679617.8717287,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1709679617.872134,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1709679617.8724597,"name":"offline","context":{"idset":"559"}}
+{"timestamp":1709679617.8732483,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1709679617.8736329,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1709679617.8739834,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1709679617.8747728,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1709679617.8753777,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1709679617.8808208,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1709679617.8866472,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1709679617.8871696,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1709679617.8936176,"name":"offline","context":{"idset":"654"}}
+{"timestamp":1709679617.8958864,"name":"offline","context":{"idset":"660"}}
+{"timestamp":1709679617.9001367,"name":"offline","context":{"idset":"661"}}
+{"timestamp":1709679617.90712,"name":"offline","context":{"idset":"662"}}
+{"timestamp":1709679617.9074726,"name":"offline","context":{"idset":"667"}}
+{"timestamp":1709679617.9338078,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1709679617.9341652,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1709679617.9391766,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1709679617.9447596,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1709679617.9451056,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1709679617.9607072,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1709679617.9610946,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1709679617.9658678,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1709679617.9662197,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1709679617.9665956,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1709679617.9716234,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1709679617.9719875,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1709679617.976881,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1709679617.981976,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1709679617.9823685,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1709679617.9873645,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1709679617.9877312,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1709679617.9913819,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1709679617.9949868,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1709679617.9968841,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1709679618.0017142,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1709679618.0020497,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1709679618.0056777,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1709679618.0107944,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1709679618.0112789,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1709679618.0165603,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1709679618.0168586,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1709679618.0221527,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1709679618.0224891,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1709679618.0534446,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1709679618.0570927,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1709679618.0574193,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1709679618.061197,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1709679618.0615304,"name":"offline","context":{"idset":"682"}}
+{"timestamp":1709679618.0650101,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1709679618.0689988,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1709679618.0793257,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1709679618.0842154,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1709679618.0865793,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1709679618.0891821,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1709679618.0923526,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1709679618.0964203,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1709679618.1003859,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1709679618.1010301,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1709679618.1016819,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1709679618.105423,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1709679618.1233327,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1709679618.1239886,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1709679618.1246276,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1709679618.1252396,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1709679618.1258852,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1709679618.1265061,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1709679618.1271155,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1709679618.1277292,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1709679618.1283448,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1709679618.1290035,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1709679618.1296072,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1709679618.130204,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1709679618.130806,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1709679618.1318517,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1709679618.1322508,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1709679618.1363885,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1709679618.1367433,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1709679618.1410315,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1709679618.1414235,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1709679618.1453941,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1709679618.1498075,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1709679618.1518664,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1709679618.1544318,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1709679618.1562803,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1709679618.1602089,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1709679618.1656547,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1709679618.1676903,"name":"offline","context":{"idset":"447"}}
+{"timestamp":1709679618.1721895,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1709679618.1918693,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1709679618.1980517,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1709679618.1983907,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1709679618.2041879,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1709679618.2101934,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1709679618.2105429,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1709679618.2162802,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1709679618.2166216,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1709679618.2225668,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1709679618.2228668,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1709679618.2297525,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1709679618.2360713,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1709679618.241596,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1709679618.2457287,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1709679618.2493312,"name":"offline","context":{"idset":"655"}}
+{"timestamp":1709679618.256485,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1709679618.2605083,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1709679618.2657039,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1709679618.2693803,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1709679618.270808,"name":"offline","context":{"idset":"681"}}
+{"timestamp":1709679618.2762451,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1709679618.2811205,"name":"offline","context":{"idset":"692"}}
+{"timestamp":1709679618.286041,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1709679618.2908566,"name":"offline","context":{"idset":"707"}}
+{"timestamp":1709679618.295846,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1709679618.3015058,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1709679618.3260982,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1709679618.3306265,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1709679618.3692842,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1709679618.3747892,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1709679618.3786893,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1709679618.382534,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1709679618.3866825,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1709679618.3916612,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1709679618.3955014,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1709679618.3996265,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1709679618.4029739,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1709679618.4050837,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1709679618.4165351,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1709679618.4201267,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1709679618.4235995,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1709679618.4240558,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1709679618.4280362,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1709679618.4308879,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1709679618.4337504,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1709679618.4365749,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1709679618.4393992,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1709679618.44221,"name":"offline","context":{"idset":"464"}}
+{"timestamp":1709679618.445013,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1709679618.4478061,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1709679618.4506035,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1709679618.4510062,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1709679618.4514294,"name":"offline","context":{"idset":"564"}}
+{"timestamp":1709679618.4518366,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1709679618.4522352,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1709679618.4530039,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1709679618.4551153,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1709679618.4633567,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1709679618.4681172,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1709679618.4726119,"name":"offline","context":{"idset":"708"}}
+{"timestamp":1709679618.4776015,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1709679618.480068,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1709679618.482482,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1709679618.4832721,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1709679618.4904606,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1709679618.4910874,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1709679618.4948423,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1709679618.4953563,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1709679618.4987109,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1709679618.4992435,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1709679618.5024116,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1709679618.5028419,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1709679618.506043,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1709679618.5062854,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1709679618.5109656,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1709679618.5124936,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1709679618.514951,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1709679618.5172274,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1709679618.517473,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1709679618.5177078,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1709679618.5241077,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1709679618.5243559,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1709679618.5245724,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1709679618.5247858,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1709679618.5249979,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1709679618.5252097,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1709679618.5254397,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1709679618.5256534,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1709679618.5259461,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1709679618.5560868,"name":"offline","context":{"idset":"657"}}
+{"timestamp":1709679618.5603178,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1709679618.5629647,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1709679618.5632339,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1709679618.566829,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1709679618.5708234,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1709679618.5736444,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1709679618.5761998,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1709679618.5784242,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1709679618.5810697,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1709679618.5823038,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1709679618.5848238,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1709679618.5873759,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1709679618.5876131,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1709679618.5899689,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1709679618.5925179,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1709679618.5951407,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1709679618.5975573,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1709679618.6000526,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1709679618.6002843,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1709679618.6026535,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1709679618.6052592,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1709679618.605499,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1709679618.6057432,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1709679618.6080303,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1709679618.6105666,"name":"offline","context":{"idset":"555"}}
+{"timestamp":1709679618.6107447,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1709679618.6137977,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1709679618.6140881,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1709679618.6165266,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1709679618.6168401,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1709679618.6193068,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1709679618.6195886,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1709679618.6221871,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1709679618.6224821,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1709679618.6250196,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1709679618.6253452,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1709679618.6603863,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1709679618.662977,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1709679618.6653764,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1709679618.6656232,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1709679618.6680348,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1709679618.6682603,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1709679618.670229,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1709679618.6704736,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1709679618.6726346,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1709679618.6728437,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1709679618.6750882,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1709679618.6773643,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1709679618.6796911,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1709679618.682085,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1709679618.6823242,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1709679618.6845281,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1709679618.6867936,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1709679618.6871083,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1709679618.6874344,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1709679618.6877446,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1709679618.6880703,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1709679618.6883812,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1709679618.6886845,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1709679618.6889901,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1709679618.6892724,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1709679618.6895618,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1709679618.6899502,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1709679618.690232,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1709679618.6905248,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1709679618.6908035,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1709679618.6911063,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1709679618.6923592,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1709679618.6926448,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1709679618.6947024,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1709679618.6950026,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1709679618.6970851,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1709679618.697454,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1709679618.6994598,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1709679618.6998458,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1709679618.7018824,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1709679618.7022307,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1709679618.704205,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1709679618.7045569,"name":"offline","context":{"idset":"551"}}
+{"timestamp":1709679618.704917,"name":"offline","context":{"idset":"552"}}
+{"timestamp":1709679618.7064648,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1709679618.7066662,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1709679618.7088852,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1709679618.7092206,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1709679618.7114828,"name":"offline","context":{"idset":"669"}}
+{"timestamp":1709679618.7118235,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1709679618.7121484,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1709679618.7140896,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1709679618.7144558,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1709679618.7164757,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1709679618.717741,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1709679618.7199168,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1709679618.7214828,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1709679618.72332,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1709679618.7428341,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1709679618.7461441,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1709679618.7463739,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1709679618.7489274,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1709679618.7516475,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1709679618.7518561,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1709679618.7540655,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1709679618.7542083,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1709679618.7564812,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1709679618.7567008,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1709679618.7587292,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1709679618.7609513,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1709679618.7611299,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1709679618.7634315,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1709679618.7656443,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1709679618.7678459,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1709679618.7721949,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1709679618.7744143,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1709679618.7765689,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1709679618.7788332,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1709679618.7810183,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1709679618.7811849,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1709679618.7813685,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1709679618.7815261,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1709679618.7816815,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1709679618.7818327,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1709679618.7819843,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1709679618.7821684,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1709679618.7837665,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1709679618.7879908,"name":"offline","context":{"idset":"680"}}
+{"timestamp":1709679618.7901759,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1709679618.7922719,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1709679618.7964911,"name":"offline","context":{"idset":"701"}}
+{"timestamp":1709679618.7988307,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1709679618.8362787,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1709679618.8364744,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1709679618.8366408,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1709679618.8368554,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1709679618.8387079,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1709679618.8407853,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1709679618.8428798,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1709679618.8470697,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1709679618.8472335,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1709679618.8494542,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1709679618.8496637,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1709679618.8517904,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1709679618.8539081,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1709679618.8540661,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1709679618.8561084,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1709679618.8581607,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1709679618.8583779,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1709679618.860307,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1709679618.8624377,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1709679618.8664491,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1709679618.8684292,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1709679618.8714082,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1709679618.8747051,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1709679618.8769164,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1709679618.8791587,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1709679618.8826435,"name":"offline","context":{"idset":"450"}}
+{"timestamp":1709679618.8849161,"name":"offline","context":{"idset":"454"}}
+{"timestamp":1709679618.8872011,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1709679618.8902023,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1709679618.8921785,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1709679618.8940938,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1709679618.8979087,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1709679618.8998322,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1709679618.9017284,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1709679618.9053924,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1709679618.9072969,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1709679618.9092305,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1709679618.9113758,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1709679618.9115117,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1709679618.9116457,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1709679618.9117911,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1709679618.9145563,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1709679618.9290926,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1709679618.9326153,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1709679618.9344096,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1709679618.9362028,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1709679618.9397445,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1709679618.941534,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1709679618.9433205,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1709679618.9450588,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1709679618.948487,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1709679618.9502213,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1709679618.9519804,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1709679618.9554248,"name":"offline","context":{"idset":"121"}}
+{"timestamp":1709679618.9571192,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1709679618.9588459,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1709679618.9622145,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1709679618.963928,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1709679618.9657044,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1709679618.9690037,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1709679618.970701,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1709679618.9724061,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1709679618.9756985,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1709679618.9773769,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1709679618.9790401,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1709679618.9823203,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1709679618.9839835,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1709679618.9856384,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1709679618.9888666,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1709679618.9905515,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1709679618.992178,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1709679618.9938149,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1709679618.9954951,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1709679618.998682,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1709679619.00037,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1709679619.0020823,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1709679619.0051789,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1709679619.0068917,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1709679619.008486,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1709679619.0086076,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1709679619.0101671,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1709679619.011785,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1709679619.0119035,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1709679619.0120194,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1709679619.0121334,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1709679619.012249,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1709679619.0123837,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1709679619.0124984,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1709679619.0126259,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1709679619.0127411,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1709679619.0128965,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1709679619.0130086,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1709679619.013118,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1709679619.0132456,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1709679619.0133648,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1709679619.0138366,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1709679619.0152514,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1709679619.0168941,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1709679619.0181971,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1709679619.0201895,"name":"offline","context":{"idset":"448"}}
+{"timestamp":1709679619.022037,"name":"offline","context":{"idset":"453"}}
+{"timestamp":1709679619.0239065,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1709679619.0264292,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1709679619.0284269,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1709679619.0306983,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1709679619.033716,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1709679619.0359409,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1709679619.0381944,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1709679619.0414736,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1709679619.0436487,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1709679619.0446284,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1709679619.0482485,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1709679619.0501549,"name":"offline","context":{"idset":"683"}}
+{"timestamp":1709679619.0518041,"name":"offline","context":{"idset":"685"}}
+{"timestamp":1709679619.0553684,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1709679619.0572402,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1709679619.0739956,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1709679619.075767,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1709679619.0791705,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1709679619.0809019,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1709679619.0823469,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1709679619.0853796,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1709679619.087065,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1709679619.088764,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1709679619.0902107,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1709679619.0934901,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1709679619.0951476,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1709679619.0968044,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1709679619.0997331,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1709679619.101321,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1709679619.1027956,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1709679619.1057072,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1709679619.1073191,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1709679619.1089253,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1709679619.1105378,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1709679619.1106281,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1709679619.1122139,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1709679619.1123097,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1709679619.1137328,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1709679619.115237,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1709679619.1153343,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1709679619.1169097,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1709679619.1184766,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1709679619.118562,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1709679619.1201224,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1709679619.1202126,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1709679619.1217825,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1709679619.1233592,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1709679619.1234465,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1709679619.125011,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1709679619.1265635,"name":"offline","context":{"idset":"553"}}
+{"timestamp":1709679619.1281035,"name":"offline","context":{"idset":"556"}}
+{"timestamp":1709679619.1296887,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1709679619.1313269,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1709679619.1328568,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1709679619.135637,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1709679619.1371577,"name":"offline","context":{"idset":"690"}}
+{"timestamp":1709679619.138402,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1709679619.1410818,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1709679619.1424816,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1709679619.14381,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1709679619.1467288,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1709679619.1482413,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1709679619.1496475,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1709679619.1523798,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1709679619.1538732,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1709679619.1553464,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1709679619.1581104,"name":"offline","context":{"idset":"452"}}
+{"timestamp":1709679619.1596861,"name":"offline","context":{"idset":"457"}}
+{"timestamp":1709679619.1611068,"name":"offline","context":{"idset":"466"}}
+{"timestamp":1709679619.1638288,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1709679619.1652534,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1709679619.1666546,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1709679619.169312,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1709679619.1707048,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1709679619.1847069,"name":"offline","context":{"idset":"671"}}
+{"timestamp":1709679619.1867123,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1709679619.1882863,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1709679619.1898847,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1709679619.1919458,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1709679619.193507,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1709679619.1950598,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1709679619.1969948,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1709679619.1985514,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1709679619.1999435,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1709679619.2019272,"name":"offline","context":{"idset":"451"}}
+{"timestamp":1709679619.2040744,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1709679619.2054439,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1709679619.2067001,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1709679619.2079449,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1709679619.2081816,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1709679619.2228539,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1709679619.2229316,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1709679619.2238946,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1709679619.2248781,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1709679619.2249634,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1709679619.2259881,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1709679619.2261012,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1709679619.2270446,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1709679619.2279925,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1709679619.2280664,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1709679619.2291358,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1709679619.230103,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1709679619.230185,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1709679619.2311184,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1709679619.2320845,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1709679619.2321594,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1709679619.233089,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1709679619.2331707,"name":"offline","context":{"idset":"558"}}
+{"timestamp":1709679619.2332444,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1709679619.23335,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1709679619.2334242,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1709679619.2335346,"name":"offline","context":{"idset":"670"}}
+{"timestamp":1709679619.2419162,"name":"offline","context":{"idset":"676"}}
+{"timestamp":1709679619.2428098,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1709679619.2437131,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1709679619.2453902,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1709679619.2462518,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1709679619.2471027,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1709679619.24877,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1709679619.2496226,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1709679619.2505021,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1709679619.2521074,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1709679619.2529588,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1709679619.2538927,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1709679619.2554717,"name":"offline","context":{"idset":"554"}}
+{"timestamp":1709679619.2563059,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1709679619.2785058,"name":"offline","context":{"idset":"694"}}
+{"timestamp":1709679619.2791073,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1709679619.2797289,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1709679619.2808692,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1709679619.2809865,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1709679619.2811027,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1709679619.2812135,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1709679619.2813551,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1709679619.2814324,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1709679619.281549,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1709679619.2816281,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1709679619.2822189,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1709679619.2823215,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1709679619.2829409,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1709679619.2929871,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1709679619.2934861,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1709679619.2943633,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1709679619.2948177,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1709679619.2952609,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1709679619.296093,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1709679619.3091967,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1709679619.3221006,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1709679619.3239319,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1709679619.3261447,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1709679619.3273406,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1709679619.328517,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1709679619.3376598,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1709679619.3517766,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1709680020.212115,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"569":{"timestamp":1709583475.6141074,"reason":"prolog failed for jobid fgjQELtBquR"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709680021.1070499,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709681304.6951547,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709681316.6497581,"name":"online","context":{"idset":"1"}}
+{"timestamp":1709681319.172157,"name":"online","context":{"idset":"66-67,89,93,95,100,102-103,109-110,119,125,131,136-137,139,144-145,167,177,191,194,211,217,221,229,235-236,245,248-249,255,263-264,284-285,296,301,304-305,309,318,320,325,334,344"}}
+{"timestamp":1709681320.0191898,"name":"online","context":{"idset":"354"}}
+{"timestamp":1709681320.9889803,"name":"online","context":{"idset":"372,377"}}
+{"timestamp":1709681321.4627545,"name":"online","context":{"idset":"387-388,390"}}
+{"timestamp":1709681321.6674995,"name":"online","context":{"idset":"396"}}
+{"timestamp":1709681321.8818252,"name":"online","context":{"idset":"400-401"}}
+{"timestamp":1709681323.3160005,"name":"online","context":{"idset":"418"}}
+{"timestamp":1709681323.7751479,"name":"online","context":{"idset":"428"}}
+{"timestamp":1709681324.6646998,"name":"online","context":{"idset":"437"}}
+{"timestamp":1709681325.6112149,"name":"online","context":{"idset":"442,445"}}
+{"timestamp":1709681326.5214229,"name":"online","context":{"idset":"447"}}
+{"timestamp":1709681327.0933342,"name":"online","context":{"idset":"456,458-459"}}
+{"timestamp":1709681327.5751209,"name":"online","context":{"idset":"462,465"}}
+{"timestamp":1709681328.9069183,"name":"online","context":{"idset":"497-498,503"}}
+{"timestamp":1709681329.6116135,"name":"online","context":{"idset":"505-506"}}
+{"timestamp":1709681330.3289826,"name":"online","context":{"idset":"510,513-514"}}
+{"timestamp":1709681331.3458376,"name":"online","context":{"idset":"517,526"}}
+{"timestamp":1709681332.3183477,"name":"online","context":{"idset":"536,542-544"}}
+{"timestamp":1709681332.7406147,"name":"online","context":{"idset":"546"}}
+{"timestamp":1709681334.0867653,"name":"online","context":{"idset":"553"}}
+{"timestamp":1709681335.1028109,"name":"online","context":{"idset":"564,566"}}
+{"timestamp":1709681336.1232619,"name":"online","context":{"idset":"572"}}
+{"timestamp":1709681337.2227199,"name":"online","context":{"idset":"584,593"}}
+{"timestamp":1709681337.8565626,"name":"online","context":{"idset":"599"}}
+{"timestamp":1709681338.5338497,"name":"online","context":{"idset":"601"}}
+{"timestamp":1709681339.4429386,"name":"online","context":{"idset":"609"}}
+{"timestamp":1709681340.317426,"name":"online","context":{"idset":"615"}}
+{"timestamp":1709681340.4701486,"name":"online","context":{"idset":"619"}}
+{"timestamp":1709681342.5731244,"name":"online","context":{"idset":"624,626"}}
+{"timestamp":1709681344.4493508,"name":"online","context":{"idset":"673,678,680-681,685"}}
+{"timestamp":1709681344.6634598,"name":"online","context":{"idset":"687-688,691"}}
+{"timestamp":1709681346.0494664,"name":"online","context":{"idset":"699"}}
+{"timestamp":1709681347.0110672,"name":"online","context":{"idset":"704"}}
+{"timestamp":1709681347.8967841,"name":"online","context":{"idset":"708,710,712"}}
+{"timestamp":1709681348.6978254,"name":"online","context":{"idset":"719"}}
+{"timestamp":1709681350.2921627,"name":"online","context":{"idset":"3-4,8,18,24,28,30,45,47-49,54,59,61-65"}}
+{"timestamp":1709681351.003305,"name":"online","context":{"idset":"68-71"}}
+{"timestamp":1709681351.039274,"name":"online","context":{"idset":"72-73"}}
+{"timestamp":1709681351.8361664,"name":"online","context":{"idset":"74-77"}}
+{"timestamp":1709681352.9003944,"name":"online","context":{"idset":"80-83,85,88"}}
+{"timestamp":1709681353.817466,"name":"online","context":{"idset":"90-91,94"}}
+{"timestamp":1709681353.8574064,"name":"online","context":{"idset":"97"}}
+{"timestamp":1709681354.6855376,"name":"online","context":{"idset":"99,101"}}
+{"timestamp":1709681355.7034616,"name":"online","context":{"idset":"106-107,112"}}
+{"timestamp":1709681356.0734138,"name":"online","context":{"idset":"113-116"}}
+{"timestamp":1709681356.7338967,"name":"online","context":{"idset":"117-118"}}
+{"timestamp":1709681357.4346821,"name":"online","context":{"idset":"123-124"}}
+{"timestamp":1709681358.2324431,"name":"online","context":{"idset":"126-130"}}
+{"timestamp":1709681358.2637198,"name":"online","context":{"idset":"132"}}
+{"timestamp":1709681359.225996,"name":"online","context":{"idset":"134,140"}}
+{"timestamp":1709681360.0503988,"name":"online","context":{"idset":"143,147"}}
+{"timestamp":1709681360.7455735,"name":"online","context":{"idset":"156"}}
+{"timestamp":1709681361.6501305,"name":"online","context":{"idset":"160"}}
+{"timestamp":1709681362.8626916,"name":"online","context":{"idset":"173-174"}}
+{"timestamp":1709681363.8427753,"name":"online","context":{"idset":"185"}}
+{"timestamp":1709681364.539387,"name":"online","context":{"idset":"193"}}
+{"timestamp":1709681364.6526716,"name":"online","context":{"idset":"195"}}
+{"timestamp":1709681366.1356387,"name":"online","context":{"idset":"216"}}
+{"timestamp":1709681366.7575688,"name":"online","context":{"idset":"223"}}
+{"timestamp":1709681367.4135206,"name":"online","context":{"idset":"225,228"}}
+{"timestamp":1709681368.4424286,"name":"online","context":{"idset":"727"}}
+{"timestamp":1709681369.2838304,"name":"online","context":{"idset":"250,724-726"}}
+{"timestamp":1709681369.995141,"name":"online","context":{"idset":"254,722"}}
+{"timestamp":1709681371.1789021,"name":"online","context":{"idset":"278,721,723"}}
+{"timestamp":1709681371.9944592,"name":"online","context":{"idset":"283,287,289,718,720"}}
+{"timestamp":1709681372.0797801,"name":"online","context":{"idset":"290"}}
+{"timestamp":1709681372.8155081,"name":"online","context":{"idset":"294-295,717"}}
+{"timestamp":1709681373.7867091,"name":"online","context":{"idset":"300,302,715-716"}}
+{"timestamp":1709681373.9353061,"name":"online","context":{"idset":"306,713"}}
+{"timestamp":1709681374.8975203,"name":"online","context":{"idset":"308,310,711"}}
+{"timestamp":1709681375.9886758,"name":"online","context":{"idset":"312-314,316"}}
+{"timestamp":1709681376.0682819,"name":"online","context":{"idset":"317,714"}}
+{"timestamp":1709681376.2587614,"name":"online","context":{"idset":"319,321,709"}}
+{"timestamp":1709681377.182143,"name":"online","context":{"idset":"322-323,706"}}
+{"timestamp":1709681377.4847786,"name":"online","context":{"idset":"327"}}
+{"timestamp":1709681378.3071342,"name":"online","context":{"idset":"330-332"}}
+{"timestamp":1709681378.6239386,"name":"online","context":{"idset":"335,337-338,340,342,707"}}
+{"timestamp":1709681379.0970802,"name":"online","context":{"idset":"343,705"}}
+{"timestamp":1709681379.9134421,"name":"online","context":{"idset":"347,349"}}
+{"timestamp":1709681381.0310335,"name":"online","context":{"idset":"352-353,356,703"}}
+{"timestamp":1709681381.2293751,"name":"drain","context":{"idset":"66","reason":"broker was unresponsive"}}
+{"timestamp":1709681381.2403064,"name":"drain","context":{"idset":"67","reason":"broker was unresponsive"}}
+{"timestamp":1709681381.2628613,"name":"drain","context":{"idset":"89","reason":"broker was unresponsive"}}
+{"timestamp":1709681381.2866113,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1709681381.3943751,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1709681381.3979478,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.3681943,"name":"drain","context":{"idset":"102","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.419282,"name":"drain","context":{"idset":"103","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.4660995,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.4833553,"name":"drain","context":{"idset":"110","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.5015171,"name":"drain","context":{"idset":"119","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.5165713,"name":"drain","context":{"idset":"125","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.5881133,"name":"drain","context":{"idset":"131","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.591558,"name":"drain","context":{"idset":"136","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.5965333,"name":"drain","context":{"idset":"137","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.601145,"name":"drain","context":{"idset":"139","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.6199274,"name":"drain","context":{"idset":"144","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.6291566,"name":"drain","context":{"idset":"145","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.6716454,"name":"drain","context":{"idset":"167","reason":"broker was unresponsive"}}
+{"timestamp":1709681382.9935443,"name":"drain","context":{"idset":"177","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.1277764,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.1364768,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.1609488,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.1667616,"name":"drain","context":{"idset":"217","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.1720257,"name":"drain","context":{"idset":"221","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.1771307,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.1813967,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.1856766,"name":"drain","context":{"idset":"236","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.1935496,"name":"drain","context":{"idset":"245","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.3082535,"name":"drain","context":{"idset":"248","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.3107648,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.3291364,"name":"drain","context":{"idset":"255","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.3358667,"name":"drain","context":{"idset":"263","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.3633454,"name":"drain","context":{"idset":"264","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.3796248,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.3889556,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.3927896,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.4029431,"name":"drain","context":{"idset":"301","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.4564159,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.5732772,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.5834227,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.5877018,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.5910974,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.5974274,"name":"drain","context":{"idset":"325","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.6015635,"name":"drain","context":{"idset":"334","reason":"broker was unresponsive"}}
+{"timestamp":1709681384.6050229,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.2968731,"name":"drain","context":{"idset":"354","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.3563325,"name":"online","context":{"idset":"359"}}
+{"timestamp":1709681386.3797965,"name":"online","context":{"idset":"365"}}
+{"timestamp":1709681386.3891141,"name":"online","context":{"idset":"367-368,370"}}
+{"timestamp":1709681386.4042463,"name":"online","context":{"idset":"371,373-375,702"}}
+{"timestamp":1709681386.4294631,"name":"online","context":{"idset":"376,378,701"}}
+{"timestamp":1709681386.4606612,"name":"online","context":{"idset":"379-381,384-385"}}
+{"timestamp":1709681386.6351016,"name":"online","context":{"idset":"389,697-698,700"}}
+{"timestamp":1709681386.6589758,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.6731384,"name":"drain","context":{"idset":"377","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.6789196,"name":"drain","context":{"idset":"387","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.6861463,"name":"drain","context":{"idset":"388","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.6887178,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.692827,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.7261381,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.7330506,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.7380211,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.7861958,"name":"drain","context":{"idset":"428","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.8500264,"name":"drain","context":{"idset":"437","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.8726375,"name":"drain","context":{"idset":"442","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.8894696,"name":"drain","context":{"idset":"445","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.911958,"name":"drain","context":{"idset":"447","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.9361956,"name":"drain","context":{"idset":"456","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.9580297,"name":"drain","context":{"idset":"458","reason":"broker was unresponsive"}}
+{"timestamp":1709681386.9728115,"name":"drain","context":{"idset":"459","reason":"broker was unresponsive"}}
+{"timestamp":1709681387.0292575,"name":"online","context":{"idset":"391"}}
+{"timestamp":1709681387.0464168,"name":"online","context":{"idset":"393-394,397,696"}}
+{"timestamp":1709681387.0674472,"name":"online","context":{"idset":"398-399,404,406,694-695"}}
+{"timestamp":1709681387.0832424,"name":"online","context":{"idset":"408-410"}}
+{"timestamp":1709681387.099658,"name":"online","context":{"idset":"412,414-417,419,693"}}
+{"timestamp":1709681387.1055453,"name":"drain","context":{"idset":"462","reason":"broker was unresponsive"}}
+{"timestamp":1709681387.1210575,"name":"drain","context":{"idset":"465","reason":"broker was unresponsive"}}
+{"timestamp":1709681387.1334167,"name":"online","context":{"idset":"420-421,423,425"}}
+{"timestamp":1709681387.1835766,"name":"online","context":{"idset":"426,432,692"}}
+{"timestamp":1709681389.873574,"name":"online","context":{"idset":"435-436,438-440,690"}}
+{"timestamp":1709681390.1100252,"name":"online","context":{"idset":"443-444,449-450,684,686,689"}}
+{"timestamp":1709681390.1232228,"name":"drain","context":{"idset":"497","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.130908,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.1417434,"name":"drain","context":{"idset":"503","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.1518021,"name":"drain","context":{"idset":"505","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.1594672,"name":"drain","context":{"idset":"506","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.166888,"name":"drain","context":{"idset":"510","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.1765316,"name":"drain","context":{"idset":"513","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.1830907,"name":"drain","context":{"idset":"514","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.1913648,"name":"drain","context":{"idset":"517","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.2014601,"name":"drain","context":{"idset":"526","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.2076747,"name":"drain","context":{"idset":"536","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.2179804,"name":"drain","context":{"idset":"542","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.2264061,"name":"drain","context":{"idset":"543","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.2353237,"name":"drain","context":{"idset":"544","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.256367,"name":"drain","context":{"idset":"546","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.2609541,"name":"drain","context":{"idset":"553","reason":"broker was unresponsive"}}
+{"timestamp":1709681390.3034678,"name":"online","context":{"idset":"455,461,463-464,467,675,677,679,682-683"}}
+{"timestamp":1709681390.582773,"name":"online","context":{"idset":"468-470"}}
+{"timestamp":1709681393.2550955,"name":"online","context":{"idset":"496,499,522,674,676"}}
+{"timestamp":1709681393.2815576,"name":"online","context":{"idset":"500-501"}}
+{"timestamp":1709681393.4408503,"name":"online","context":{"idset":"502,504,512,672"}}
+{"timestamp":1709681393.5922072,"name":"online","context":{"idset":"527,670-671"}}
+{"timestamp":1709681393.71051,"name":"online","context":{"idset":"507-509"}}
+{"timestamp":1709681393.8813236,"name":"online","context":{"idset":"511,668-669"}}
+{"timestamp":1709681393.892544,"name":"drain","context":{"idset":"564","reason":"broker was unresponsive"}}
+{"timestamp":1709681393.8968093,"name":"drain","context":{"idset":"566","reason":"broker was unresponsive"}}
+{"timestamp":1709681393.9307537,"name":"drain","context":{"idset":"572","reason":"broker was unresponsive"}}
+{"timestamp":1709681393.9365761,"name":"drain","context":{"idset":"584","reason":"broker was unresponsive"}}
+{"timestamp":1709681393.9513502,"name":"drain","context":{"idset":"593","reason":"broker was unresponsive"}}
+{"timestamp":1709681393.9726408,"name":"drain","context":{"idset":"599","reason":"broker was unresponsive"}}
+{"timestamp":1709681393.9864061,"name":"drain","context":{"idset":"601","reason":"broker was unresponsive"}}
+{"timestamp":1709681394.0069788,"name":"drain","context":{"idset":"609","reason":"broker was unresponsive"}}
+{"timestamp":1709681394.0380058,"name":"online","context":{"idset":"515-516,518,520"}}
+{"timestamp":1709681394.1865003,"name":"online","context":{"idset":"519,521,523,525"}}
+{"timestamp":1709681394.3210816,"name":"online","context":{"idset":"524"}}
+{"timestamp":1709681394.5171268,"name":"online","context":{"idset":"528-534,537"}}
+{"timestamp":1709681394.6839268,"name":"online","context":{"idset":"535,538"}}
+{"timestamp":1709681396.2014048,"name":"online","context":{"idset":"539-541"}}
+{"timestamp":1709681396.2597802,"name":"online","context":{"idset":"545"}}
+{"timestamp":1709681396.4696147,"name":"online","context":{"idset":"547-549,551-552"}}
+{"timestamp":1709681396.6712723,"name":"online","context":{"idset":"554-558"}}
+{"timestamp":1709681396.83937,"name":"online","context":{"idset":"559-560"}}
+{"timestamp":1709681396.8579962,"name":"drain","context":{"idset":"615","reason":"broker was unresponsive"}}
+{"timestamp":1709681396.9853489,"name":"drain","context":{"idset":"619","reason":"broker was unresponsive"}}
+{"timestamp":1709681397.0042694,"name":"drain","context":{"idset":"624","reason":"broker was unresponsive"}}
+{"timestamp":1709681397.0206926,"name":"drain","context":{"idset":"626","reason":"broker was unresponsive"}}
+{"timestamp":1709681397.0639446,"name":"online","context":{"idset":"561-563,565"}}
+{"timestamp":1709681397.2813587,"name":"online","context":{"idset":"567-568"}}
+{"timestamp":1709681398.2433593,"name":"online","context":{"idset":"570-571"}}
+{"timestamp":1709681398.6379139,"name":"online","context":{"idset":"573-574"}}
+{"timestamp":1709681398.6832447,"name":"online","context":{"idset":"575-576"}}
+{"timestamp":1709681398.8509932,"name":"online","context":{"idset":"577-581"}}
+{"timestamp":1709681398.973279,"name":"online","context":{"idset":"582-583,585"}}
+{"timestamp":1709681399.091485,"name":"online","context":{"idset":"586-591"}}
+{"timestamp":1709681399.1990819,"name":"drain","context":{"idset":"673","reason":"broker was unresponsive"}}
+{"timestamp":1709681399.2509706,"name":"online","context":{"idset":"592,594-598"}}
+{"timestamp":1709681400.0894535,"name":"online","context":{"idset":"600,602-603"}}
+{"timestamp":1709681400.4210095,"name":"online","context":{"idset":"604,607"}}
+{"timestamp":1709681400.5736582,"name":"online","context":{"idset":"605-606,608,610"}}
+{"timestamp":1709681400.8027725,"name":"online","context":{"idset":"611-613"}}
+{"timestamp":1709681401.8290458,"name":"online","context":{"idset":"614,616-617"}}
+{"timestamp":1709681401.839561,"name":"online","context":{"idset":"618"}}
+{"timestamp":1709681401.9779341,"name":"online","context":{"idset":"620-623,631"}}
+{"timestamp":1709681402.1100249,"name":"online","context":{"idset":"625,627-630,633"}}
+{"timestamp":1709681402.919697,"name":"online","context":{"idset":"632,634"}}
+{"timestamp":1709681402.934957,"name":"drain","context":{"idset":"678","reason":"broker was unresponsive"}}
+{"timestamp":1709681402.9408548,"name":"drain","context":{"idset":"680","reason":"broker was unresponsive"}}
+{"timestamp":1709681402.9469535,"name":"drain","context":{"idset":"681","reason":"broker was unresponsive"}}
+{"timestamp":1709681402.9542124,"name":"drain","context":{"idset":"685","reason":"broker was unresponsive"}}
+{"timestamp":1709681402.962131,"name":"drain","context":{"idset":"687","reason":"broker was unresponsive"}}
+{"timestamp":1709681402.9762506,"name":"drain","context":{"idset":"688","reason":"broker was unresponsive"}}
+{"timestamp":1709681402.9895954,"name":"drain","context":{"idset":"691","reason":"broker was unresponsive"}}
+{"timestamp":1709681403.006067,"name":"drain","context":{"idset":"699","reason":"broker was unresponsive"}}
+{"timestamp":1709681403.0211267,"name":"drain","context":{"idset":"704","reason":"broker was unresponsive"}}
+{"timestamp":1709681403.035203,"name":"drain","context":{"idset":"708","reason":"broker was unresponsive"}}
+{"timestamp":1709681403.0472131,"name":"drain","context":{"idset":"710","reason":"broker was unresponsive"}}
+{"timestamp":1709681404.9189875,"name":"online","context":{"idset":"2,645-646"}}
+{"timestamp":1709681405.0222077,"name":"online","context":{"idset":"643-644"}}
+{"timestamp":1709681406.2039976,"name":"online","context":{"idset":"639-642"}}
+{"timestamp":1709681406.3381953,"name":"online","context":{"idset":"636-638"}}
+{"timestamp":1709681406.6114905,"name":"online","context":{"idset":"635,647-648"}}
+{"timestamp":1709681407.4882238,"name":"online","context":{"idset":"649-652"}}
+{"timestamp":1709681408.7758594,"name":"online","context":{"idset":"653-659"}}
+{"timestamp":1709681408.8212817,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1709681408.8318951,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1709681408.8635392,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1709681408.8827724,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1709681408.9095361,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1709681408.9368012,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1709681408.9620736,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1709681408.9713809,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1709681408.978828,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1709681409.8044841,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1709681409.8323469,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1709681409.8677628,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1709681409.8796892,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1709681409.8881745,"name":"drain","context":{"idset":"61","reason":"broker was unresponsive"}}
+{"timestamp":1709681409.8981011,"name":"drain","context":{"idset":"62","reason":"broker was unresponsive"}}
+{"timestamp":1709681409.9095984,"name":"drain","context":{"idset":"63","reason":"broker was unresponsive"}}
+{"timestamp":1709681410.1160378,"name":"drain","context":{"idset":"64","reason":"broker was unresponsive"}}
+{"timestamp":1709681410.1490726,"name":"drain","context":{"idset":"65","reason":"broker was unresponsive"}}
+{"timestamp":1709681410.1563275,"name":"drain","context":{"idset":"68","reason":"broker was unresponsive"}}
+{"timestamp":1709681410.162261,"name":"drain","context":{"idset":"69","reason":"broker was unresponsive"}}
+{"timestamp":1709681410.1708121,"name":"drain","context":{"idset":"70","reason":"broker was unresponsive"}}
+{"timestamp":1709681410.1939435,"name":"drain","context":{"idset":"71","reason":"broker was unresponsive"}}
+{"timestamp":1709681410.2934506,"name":"drain","context":{"idset":"72","reason":"broker was unresponsive"}}
+{"timestamp":1709681410.2965677,"name":"drain","context":{"idset":"73","reason":"broker was unresponsive"}}
+{"timestamp":1709681410.3037362,"name":"drain","context":{"idset":"74","reason":"broker was unresponsive"}}
+{"timestamp":1709681410.9298322,"name":"drain","context":{"idset":"75","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.4928811,"name":"drain","context":{"idset":"76","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.5084949,"name":"drain","context":{"idset":"77","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.5167923,"name":"drain","context":{"idset":"80","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.5255532,"name":"drain","context":{"idset":"81","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.5611663,"name":"drain","context":{"idset":"82","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.5831027,"name":"drain","context":{"idset":"83","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.5988719,"name":"drain","context":{"idset":"85","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.6055169,"name":"drain","context":{"idset":"88","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.6190681,"name":"drain","context":{"idset":"90","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.6429272,"name":"drain","context":{"idset":"91","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.6575837,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.6683059,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.6712108,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.6777511,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.6898465,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.6997364,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.7236922,"name":"drain","context":{"idset":"112","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.73578,"name":"drain","context":{"idset":"712","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.7581737,"name":"drain","context":{"idset":"719","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.8009629,"name":"online","context":{"idset":"660,663"}}
+{"timestamp":1709681411.810921,"name":"online","context":{"idset":"78-79,661-662,664-665"}}
+{"timestamp":1709681411.8237572,"name":"online","context":{"idset":"84,86-87,666-667"}}
+{"timestamp":1709681411.842701,"name":"online","context":{"idset":"493-495,752"}}
+{"timestamp":1709681411.8679347,"name":"online","context":{"idset":"492,750"}}
+{"timestamp":1709681411.8774562,"name":"drain","context":{"idset":"113","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.8837709,"name":"drain","context":{"idset":"114","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.8925068,"name":"drain","context":{"idset":"115","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.8994727,"name":"drain","context":{"idset":"116","reason":"broker was unresponsive"}}
+{"timestamp":1709681411.9239893,"name":"drain","context":{"idset":"117","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.8736432,"name":"drain","context":{"idset":"118","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.889564,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9137659,"name":"drain","context":{"idset":"124","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9186139,"name":"drain","context":{"idset":"126","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9231811,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9277308,"name":"drain","context":{"idset":"128","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9315896,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.935252,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9394503,"name":"drain","context":{"idset":"132","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9431677,"name":"drain","context":{"idset":"134","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9498262,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9536166,"name":"drain","context":{"idset":"143","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9614174,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9705269,"name":"drain","context":{"idset":"156","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9775803,"name":"drain","context":{"idset":"160","reason":"broker was unresponsive"}}
+{"timestamp":1709681413.9935381,"name":"online","context":{"idset":"98,491"}}
+{"timestamp":1709681414.0138175,"name":"online","context":{"idset":"490,749"}}
+{"timestamp":1709681414.0287967,"name":"online","context":{"idset":"104,489,747-748"}}
+{"timestamp":1709681414.0449657,"name":"online","context":{"idset":"105,108,487-488,745"}}
+{"timestamp":1709681414.0811834,"name":"online","context":{"idset":"111,483-486,746"}}
+{"timestamp":1709681414.0981498,"name":"online","context":{"idset":"482"}}
+{"timestamp":1709681414.1123083,"name":"online","context":{"idset":"120-122,476-481,743-744"}}
+{"timestamp":1709681414.1369171,"name":"online","context":{"idset":"473-475,742"}}
+{"timestamp":1709681414.3250537,"name":"drain","context":{"idset":"173","reason":"broker was unresponsive"}}
+{"timestamp":1709681414.3712187,"name":"drain","context":{"idset":"174","reason":"broker was unresponsive"}}
+{"timestamp":1709681414.5448501,"name":"drain","context":{"idset":"185","reason":"broker was unresponsive"}}
+{"timestamp":1709681414.5669005,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1709681414.6065304,"name":"drain","context":{"idset":"195","reason":"broker was unresponsive"}}
+{"timestamp":1709681414.7003641,"name":"online","context":{"idset":"133,135,138,471-472,740-741"}}
+{"timestamp":1709681414.7417979,"name":"online","context":{"idset":"466,739"}}
+{"timestamp":1709681414.7899187,"name":"online","context":{"idset":"141-142,460,738"}}
+{"timestamp":1709681414.8763158,"name":"online","context":{"idset":"146,148-150,457,737"}}
+{"timestamp":1709681416.8056576,"name":"online","context":{"idset":"151-155,157-159,453-454"}}
+{"timestamp":1709681417.6220672,"name":"online","context":{"idset":"161-162"}}
+{"timestamp":1709681418.0466228,"name":"online","context":{"idset":"164-165"}}
+{"timestamp":1709681418.0989313,"name":"online","context":{"idset":"166,736"}}
+{"timestamp":1709681418.2667911,"name":"online","context":{"idset":"168-172,175-176,451-452"}}
+{"timestamp":1709681418.4111035,"name":"online","context":{"idset":"735"}}
+{"timestamp":1709681418.5265214,"name":"online","context":{"idset":"178,180"}}
+{"timestamp":1709681418.623168,"name":"online","context":{"idset":"182"}}
+{"timestamp":1709681418.6313856,"name":"drain","context":{"idset":"216","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.7555635,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.7607675,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.771693,"name":"drain","context":{"idset":"228","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.7830951,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.7910876,"name":"drain","context":{"idset":"254","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.7963738,"name":"drain","context":{"idset":"721","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.8056767,"name":"drain","context":{"idset":"722","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.8102629,"name":"drain","context":{"idset":"723","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.8166225,"name":"drain","context":{"idset":"724","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.8237262,"name":"drain","context":{"idset":"725","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.8508961,"name":"drain","context":{"idset":"726","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.883795,"name":"drain","context":{"idset":"727","reason":"broker was unresponsive"}}
+{"timestamp":1709681418.9983933,"name":"online","context":{"idset":"183"}}
+{"timestamp":1709681419.038728,"name":"online","context":{"idset":"186,188,190,448"}}
+{"timestamp":1709681419.2065833,"name":"online","context":{"idset":"192,446,734"}}
+{"timestamp":1709681419.3115284,"name":"online","context":{"idset":"441,733"}}
+{"timestamp":1709681419.482738,"name":"online","context":{"idset":"196-199"}}
+{"timestamp":1709681419.8491592,"name":"online","context":{"idset":"202,204-208"}}
+{"timestamp":1709681422.6781099,"name":"online","context":{"idset":"209"}}
+{"timestamp":1709681422.8298028,"name":"online","context":{"idset":"218-220,434,731-732"}}
+{"timestamp":1709681423.0013232,"name":"online","context":{"idset":"222,433,729"}}
+{"timestamp":1709681423.3343508,"name":"online","context":{"idset":"224,431"}}
+{"timestamp":1709681425.4642551,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.4762263,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.4811175,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.5408835,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.5603411,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.5680461,"name":"drain","context":{"idset":"294","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.5816805,"name":"drain","context":{"idset":"295","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.5854528,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.6227891,"name":"drain","context":{"idset":"302","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.680407,"name":"drain","context":{"idset":"306","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.6869066,"name":"drain","context":{"idset":"308","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.7020419,"name":"drain","context":{"idset":"713","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.7187603,"name":"drain","context":{"idset":"715","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.7829328,"name":"drain","context":{"idset":"716","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.8226144,"name":"drain","context":{"idset":"717","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.9167569,"name":"drain","context":{"idset":"718","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.9422958,"name":"drain","context":{"idset":"720","reason":"broker was unresponsive"}}
+{"timestamp":1709681425.986192,"name":"online","context":{"idset":"430,730"}}
+{"timestamp":1709681426.1908889,"name":"online","context":{"idset":"230-234,728"}}
+{"timestamp":1709681426.3674686,"name":"online","context":{"idset":"239-241,429"}}
+{"timestamp":1709681426.8083749,"name":"online","context":{"idset":"242-244,427"}}
+{"timestamp":1709681428.6819887,"name":"drain","context":{"idset":"310","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.7005777,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.7034364,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.710573,"name":"drain","context":{"idset":"314","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.7154613,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.7214339,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.738225,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.7530956,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.786428,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.8370903,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.8597717,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.8692617,"name":"drain","context":{"idset":"706","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.8727474,"name":"drain","context":{"idset":"709","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.8797991,"name":"drain","context":{"idset":"711","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.8989787,"name":"drain","context":{"idset":"714","reason":"broker was unresponsive"}}
+{"timestamp":1709681428.9188991,"name":"online","context":{"idset":"246"}}
+{"timestamp":1709681429.0096669,"name":"online","context":{"idset":"251,413,422,424"}}
+{"timestamp":1709681429.9696949,"name":"online","context":{"idset":"407,411"}}
+{"timestamp":1709681430.0106034,"name":"online","context":{"idset":"256-257"}}
+{"timestamp":1709681430.0250697,"name":"online","context":{"idset":"259"}}
+{"timestamp":1709681430.0365591,"name":"online","context":{"idset":"260"}}
+{"timestamp":1709681430.1978335,"name":"online","context":{"idset":"261-262,405"}}
+{"timestamp":1709681430.2201972,"name":"online","context":{"idset":"266,403"}}
+{"timestamp":1709681432.5538156,"name":"drain","context":{"idset":"330","reason":"broker was unresponsive"}}
+{"timestamp":1709681432.5761559,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1709681432.5861928,"name":"drain","context":{"idset":"332","reason":"broker was unresponsive"}}
+{"timestamp":1709681432.5986011,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1709681432.611671,"name":"drain","context":{"idset":"337","reason":"broker was unresponsive"}}
+{"timestamp":1709681432.6890168,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1709681432.7105832,"name":"drain","context":{"idset":"340","reason":"broker was unresponsive"}}
+{"timestamp":1709681432.7151535,"name":"drain","context":{"idset":"342","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.1126852,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.3106022,"name":"drain","context":{"idset":"347","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.3506317,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.3695176,"name":"drain","context":{"idset":"352","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.3850417,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.3995831,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.4171181,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.4350603,"name":"drain","context":{"idset":"365","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.4505336,"name":"drain","context":{"idset":"367","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.4663775,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.4783361,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.4914904,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.4972129,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.5550256,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.565757,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.5699155,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.6082144,"name":"drain","context":{"idset":"378","reason":"broker was unresponsive"}}
+{"timestamp":1709681433.9189322,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1709681434.6768849,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1709681434.8973839,"name":"drain","context":{"idset":"381","reason":"broker was unresponsive"}}
+{"timestamp":1709681435.0834649,"name":"drain","context":{"idset":"384","reason":"broker was unresponsive"}}
+{"timestamp":1709681435.3205395,"name":"drain","context":{"idset":"701","reason":"broker was unresponsive"}}
+{"timestamp":1709681435.4848008,"name":"drain","context":{"idset":"702","reason":"broker was unresponsive"}}
+{"timestamp":1709681435.5398629,"name":"drain","context":{"idset":"703","reason":"broker was unresponsive"}}
+{"timestamp":1709681435.5721347,"name":"drain","context":{"idset":"705","reason":"broker was unresponsive"}}
+{"timestamp":1709681435.5822656,"name":"drain","context":{"idset":"707","reason":"broker was unresponsive"}}
+{"timestamp":1709681435.7556934,"name":"online","context":{"idset":"267"}}
+{"timestamp":1709681435.7797794,"name":"online","context":{"idset":"268-269"}}
+{"timestamp":1709681435.8501236,"name":"online","context":{"idset":"271-277,402"}}
+{"timestamp":1709681435.8712265,"name":"online","context":{"idset":"279-280"}}
+{"timestamp":1709681435.9045582,"name":"online","context":{"idset":"281"}}
+{"timestamp":1709681435.9313223,"name":"online","context":{"idset":"282,386,392,395"}}
+{"timestamp":1709681436.0175006,"name":"online","context":{"idset":"288,382-383"}}
+{"timestamp":1709681436.0495615,"name":"online","context":{"idset":"369"}}
+{"timestamp":1709681436.1259444,"name":"online","context":{"idset":"291"}}
+{"timestamp":1709681436.1645441,"name":"online","context":{"idset":"293"}}
+{"timestamp":1709681436.2169738,"name":"online","context":{"idset":"297-298,363-364,366"}}
+{"timestamp":1709681436.2573786,"name":"online","context":{"idset":"299,361-362"}}
+{"timestamp":1709681438.9891098,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.1813583,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.1841698,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.189379,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.372478,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.3800902,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.3844814,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.3888657,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.3964338,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4022353,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4120338,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4245939,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4314182,"name":"drain","context":{"idset":"410","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4390786,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4490526,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4601121,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4665427,"name":"drain","context":{"idset":"416","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4774132,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4800189,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4906983,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.4978843,"name":"drain","context":{"idset":"421","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.5023873,"name":"drain","context":{"idset":"423","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.5080755,"name":"drain","context":{"idset":"425","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.512033,"name":"drain","context":{"idset":"426","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.5167994,"name":"drain","context":{"idset":"432","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.5256426,"name":"drain","context":{"idset":"435","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.5335879,"name":"drain","context":{"idset":"436","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.5403166,"name":"drain","context":{"idset":"438","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.547081,"name":"drain","context":{"idset":"439","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.6330161,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.639883,"name":"drain","context":{"idset":"443","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.6500537,"name":"drain","context":{"idset":"444","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.6567371,"name":"drain","context":{"idset":"449","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.6654961,"name":"drain","context":{"idset":"450","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.6844475,"name":"drain","context":{"idset":"455","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.7049119,"name":"drain","context":{"idset":"461","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.7256489,"name":"drain","context":{"idset":"463","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.7991068,"name":"drain","context":{"idset":"464","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.8154979,"name":"drain","context":{"idset":"467","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.8309884,"name":"drain","context":{"idset":"468","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.8468037,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.915415,"name":"drain","context":{"idset":"470","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.9394345,"name":"drain","context":{"idset":"675","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.9599004,"name":"drain","context":{"idset":"677","reason":"broker was unresponsive"}}
+{"timestamp":1709681439.9872706,"name":"drain","context":{"idset":"679","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.0139453,"name":"drain","context":{"idset":"682","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.0388384,"name":"drain","context":{"idset":"683","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.0586998,"name":"drain","context":{"idset":"684","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.0766542,"name":"drain","context":{"idset":"686","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.0956101,"name":"drain","context":{"idset":"689","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.1842473,"name":"drain","context":{"idset":"690","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.2477794,"name":"drain","context":{"idset":"692","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.3140657,"name":"drain","context":{"idset":"693","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.3206289,"name":"drain","context":{"idset":"694","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.4924848,"name":"drain","context":{"idset":"695","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.5031359,"name":"drain","context":{"idset":"696","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.55181,"name":"drain","context":{"idset":"697","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.5581985,"name":"drain","context":{"idset":"698","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.5640523,"name":"drain","context":{"idset":"700","reason":"broker was unresponsive"}}
+{"timestamp":1709681440.5872309,"name":"online","context":{"idset":"360"}}
+{"timestamp":1709681440.6124048,"name":"online","context":{"idset":"303,307,355,357-358"}}
+{"timestamp":1709681440.6343167,"name":"online","context":{"idset":"350-351"}}
+{"timestamp":1709681440.7101848,"name":"online","context":{"idset":"311,345-346"}}
+{"timestamp":1709681441.7159755,"name":"online","context":{"idset":"315,341"}}
+{"timestamp":1709681443.2938185,"name":"online","context":{"idset":"333,336,339"}}
+{"timestamp":1709681443.3736787,"name":"online","context":{"idset":"326,328-329"}}
+{"timestamp":1709681443.6755061,"name":"online","context":{"idset":"324,348,755"}}
+{"timestamp":1709681443.7155585,"name":"online","context":{"idset":"258,265,270,286,292"}}
+{"timestamp":1709681443.7438657,"name":"online","context":{"idset":"237,247,252-253"}}
+{"timestamp":1709681443.8234639,"name":"online","context":{"idset":"227"}}
+{"timestamp":1709681443.8379345,"name":"online","context":{"idset":"226"}}
+{"timestamp":1709681444.656419,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1709681444.7323587,"name":"drain","context":{"idset":"499","reason":"broker was unresponsive"}}
+{"timestamp":1709681444.7795947,"name":"drain","context":{"idset":"500","reason":"broker was unresponsive"}}
+{"timestamp":1709681444.8375535,"name":"drain","context":{"idset":"501","reason":"broker was unresponsive"}}
+{"timestamp":1709681444.893415,"name":"drain","context":{"idset":"502","reason":"broker was unresponsive"}}
+{"timestamp":1709681444.9844787,"name":"drain","context":{"idset":"504","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.0348909,"name":"drain","context":{"idset":"507","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.2083032,"name":"drain","context":{"idset":"508","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.3055527,"name":"drain","context":{"idset":"509","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.3199012,"name":"drain","context":{"idset":"511","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.3323352,"name":"drain","context":{"idset":"512","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.3458421,"name":"drain","context":{"idset":"515","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.366606,"name":"drain","context":{"idset":"516","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.4460552,"name":"drain","context":{"idset":"518","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.5083697,"name":"drain","context":{"idset":"519","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.5508111,"name":"drain","context":{"idset":"520","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.5768547,"name":"drain","context":{"idset":"521","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.5895514,"name":"drain","context":{"idset":"522","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.6053863,"name":"drain","context":{"idset":"523","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.6180704,"name":"drain","context":{"idset":"524","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.6335545,"name":"drain","context":{"idset":"525","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.7080548,"name":"drain","context":{"idset":"527","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.7290118,"name":"drain","context":{"idset":"528","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.7520633,"name":"drain","context":{"idset":"529","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.7737257,"name":"drain","context":{"idset":"530","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.7902575,"name":"drain","context":{"idset":"531","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.8077328,"name":"drain","context":{"idset":"532","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.8283398,"name":"drain","context":{"idset":"533","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.8443666,"name":"drain","context":{"idset":"534","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.8696365,"name":"drain","context":{"idset":"535","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.8917572,"name":"drain","context":{"idset":"537","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.9068663,"name":"drain","context":{"idset":"538","reason":"broker was unresponsive"}}
+{"timestamp":1709681445.9770317,"name":"drain","context":{"idset":"539","reason":"broker was unresponsive"}}
+{"timestamp":1709681446.0297892,"name":"drain","context":{"idset":"668","reason":"broker was unresponsive"}}
+{"timestamp":1709681446.0477581,"name":"drain","context":{"idset":"669","reason":"broker was unresponsive"}}
+{"timestamp":1709681446.1408155,"name":"drain","context":{"idset":"670","reason":"broker was unresponsive"}}
+{"timestamp":1709681446.1588423,"name":"drain","context":{"idset":"671","reason":"broker was unresponsive"}}
+{"timestamp":1709681446.1744151,"name":"drain","context":{"idset":"672","reason":"broker was unresponsive"}}
+{"timestamp":1709681446.1792593,"name":"drain","context":{"idset":"674","reason":"broker was unresponsive"}}
+{"timestamp":1709681446.2576051,"name":"drain","context":{"idset":"676","reason":"broker was unresponsive"}}
+{"timestamp":1709681446.3335888,"name":"online","context":{"idset":"215"}}
+{"timestamp":1709681446.4778149,"name":"online","context":{"idset":"184,187,189,200-201,203,210,212,214,753-754"}}
+{"timestamp":1709681446.6904705,"name":"online","context":{"idset":"96,163,179,181,751"}}
+{"timestamp":1709681446.8293266,"name":"online","context":{"idset":"92"}}
+{"timestamp":1709681451.1550701,"name":"drain","context":{"idset":"540","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.4069738,"name":"drain","context":{"idset":"541","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.4169655,"name":"drain","context":{"idset":"545","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.4255848,"name":"drain","context":{"idset":"547","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.4344933,"name":"drain","context":{"idset":"548","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.5093682,"name":"drain","context":{"idset":"549","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.537534,"name":"drain","context":{"idset":"551","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.5410399,"name":"drain","context":{"idset":"552","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.6104763,"name":"drain","context":{"idset":"554","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.6717699,"name":"drain","context":{"idset":"555","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.7294922,"name":"drain","context":{"idset":"556","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.7342887,"name":"drain","context":{"idset":"557","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.7904696,"name":"drain","context":{"idset":"558","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.79654,"name":"drain","context":{"idset":"559","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.9456582,"name":"drain","context":{"idset":"560","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.9605997,"name":"drain","context":{"idset":"561","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.9817872,"name":"drain","context":{"idset":"562","reason":"broker was unresponsive"}}
+{"timestamp":1709681451.9978135,"name":"drain","context":{"idset":"563","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.020858,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.0408056,"name":"drain","context":{"idset":"567","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.0699167,"name":"drain","context":{"idset":"568","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.092833,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.1108153,"name":"drain","context":{"idset":"571","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.1288238,"name":"drain","context":{"idset":"573","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.1517413,"name":"drain","context":{"idset":"574","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.1698351,"name":"drain","context":{"idset":"575","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.1891766,"name":"drain","context":{"idset":"576","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.2081373,"name":"drain","context":{"idset":"577","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.2572849,"name":"drain","context":{"idset":"578","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.2732096,"name":"drain","context":{"idset":"579","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.2905619,"name":"drain","context":{"idset":"580","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.3076477,"name":"drain","context":{"idset":"581","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.3279881,"name":"drain","context":{"idset":"582","reason":"broker was unresponsive"}}
+{"timestamp":1709681452.3447528,"name":"drain","context":{"idset":"583","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.3321126,"name":"drain","context":{"idset":"585","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.3353755,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.3492317,"name":"drain","context":{"idset":"587","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.3592567,"name":"drain","context":{"idset":"588","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.3682387,"name":"drain","context":{"idset":"589","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.3771901,"name":"drain","context":{"idset":"590","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.3901806,"name":"drain","context":{"idset":"591","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.3956764,"name":"drain","context":{"idset":"592","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.4047773,"name":"drain","context":{"idset":"594","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.4122798,"name":"drain","context":{"idset":"595","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.4224672,"name":"drain","context":{"idset":"596","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.4281836,"name":"drain","context":{"idset":"597","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.4379952,"name":"drain","context":{"idset":"598","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.4440696,"name":"drain","context":{"idset":"600","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.4533715,"name":"drain","context":{"idset":"602","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.463896,"name":"drain","context":{"idset":"603","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.4704916,"name":"drain","context":{"idset":"604","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.4770453,"name":"drain","context":{"idset":"605","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.4861119,"name":"drain","context":{"idset":"606","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.6161799,"name":"drain","context":{"idset":"607","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.6727376,"name":"drain","context":{"idset":"608","reason":"broker was unresponsive"}}
+{"timestamp":1709681457.6784515,"name":"drain","context":{"idset":"610","reason":"broker was unresponsive"}}
+{"timestamp":1709681458.6046863,"name":"drain","context":{"idset":"611","reason":"broker was unresponsive"}}
+{"timestamp":1709681458.6408889,"name":"drain","context":{"idset":"612","reason":"broker was unresponsive"}}
+{"timestamp":1709681458.6858304,"name":"drain","context":{"idset":"613","reason":"broker was unresponsive"}}
+{"timestamp":1709681458.7003932,"name":"drain","context":{"idset":"614","reason":"broker was unresponsive"}}
+{"timestamp":1709681458.7200763,"name":"drain","context":{"idset":"616","reason":"broker was unresponsive"}}
+{"timestamp":1709681458.7348564,"name":"drain","context":{"idset":"617","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.029072,"name":"drain","context":{"idset":"618","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.0506647,"name":"drain","context":{"idset":"620","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.0722833,"name":"drain","context":{"idset":"621","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.0948062,"name":"drain","context":{"idset":"622","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.1201332,"name":"drain","context":{"idset":"623","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.1516814,"name":"drain","context":{"idset":"625","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.2327936,"name":"drain","context":{"idset":"627","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.30176,"name":"drain","context":{"idset":"628","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.3216348,"name":"drain","context":{"idset":"629","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.4090993,"name":"drain","context":{"idset":"630","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.5167036,"name":"drain","context":{"idset":"631","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.5370102,"name":"drain","context":{"idset":"632","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.5534525,"name":"drain","context":{"idset":"633","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.5877225,"name":"drain","context":{"idset":"634","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.6571665,"name":"drain","context":{"idset":"635","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.7134571,"name":"drain","context":{"idset":"636","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.753026,"name":"drain","context":{"idset":"637","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.779016,"name":"drain","context":{"idset":"638","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.798821,"name":"drain","context":{"idset":"639","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.8620541,"name":"drain","context":{"idset":"640","reason":"broker was unresponsive"}}
+{"timestamp":1709681459.927917,"name":"drain","context":{"idset":"641","reason":"broker was unresponsive"}}
+{"timestamp":1709681460.2144468,"name":"drain","context":{"idset":"642","reason":"broker was unresponsive"}}
+{"timestamp":1709681463.1880548,"name":"drain","context":{"idset":"643","reason":"broker was unresponsive"}}
+{"timestamp":1709681463.1968279,"name":"drain","context":{"idset":"644","reason":"broker was unresponsive"}}
+{"timestamp":1709681463.5096676,"name":"drain","context":{"idset":"645","reason":"broker was unresponsive"}}
+{"timestamp":1709681463.6065159,"name":"drain","context":{"idset":"646","reason":"broker was unresponsive"}}
+{"timestamp":1709681463.6185288,"name":"drain","context":{"idset":"647","reason":"broker was unresponsive"}}
+{"timestamp":1709681463.960393,"name":"drain","context":{"idset":"648","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.0335288,"name":"drain","context":{"idset":"649","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.0864179,"name":"drain","context":{"idset":"650","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.0899096,"name":"drain","context":{"idset":"651","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.0974226,"name":"drain","context":{"idset":"652","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.1021588,"name":"drain","context":{"idset":"653","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.1101449,"name":"drain","context":{"idset":"654","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.1750033,"name":"drain","context":{"idset":"655","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.1845353,"name":"drain","context":{"idset":"656","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.194587,"name":"drain","context":{"idset":"657","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.2021778,"name":"drain","context":{"idset":"658","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.2108328,"name":"drain","context":{"idset":"659","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.2247074,"name":"drain","context":{"idset":"660","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.2394729,"name":"drain","context":{"idset":"661","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.2477934,"name":"drain","context":{"idset":"662","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.2582645,"name":"drain","context":{"idset":"663","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.2764733,"name":"drain","context":{"idset":"664","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.2877569,"name":"drain","context":{"idset":"665","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.3000238,"name":"drain","context":{"idset":"666","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.3133538,"name":"drain","context":{"idset":"667","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.3254611,"name":"drain","context":{"idset":"742","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.3341618,"name":"drain","context":{"idset":"743","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.3459511,"name":"drain","context":{"idset":"744","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.3592403,"name":"drain","context":{"idset":"745","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.3835535,"name":"drain","context":{"idset":"746","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.3917551,"name":"drain","context":{"idset":"747","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.4060469,"name":"drain","context":{"idset":"748","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.4340608,"name":"drain","context":{"idset":"749","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.4442739,"name":"drain","context":{"idset":"750","reason":"broker was unresponsive"}}
+{"timestamp":1709681464.4573925,"name":"drain","context":{"idset":"752","reason":"broker was unresponsive"}}
+{"timestamp":1709681468.3484478,"name":"drain","context":{"idset":"740","reason":"broker was unresponsive"}}
+{"timestamp":1709681468.4152417,"name":"drain","context":{"idset":"741","reason":"broker was unresponsive"}}
+{"timestamp":1709681469.894057,"name":"drain","context":{"idset":"728","reason":"broker was unresponsive"}}
+{"timestamp":1709681470.0985878,"name":"drain","context":{"idset":"729","reason":"broker was unresponsive"}}
+{"timestamp":1709681470.9172387,"name":"drain","context":{"idset":"730","reason":"broker was unresponsive"}}
+{"timestamp":1709681470.950161,"name":"drain","context":{"idset":"731","reason":"broker was unresponsive"}}
+{"timestamp":1709681471.0538638,"name":"drain","context":{"idset":"732","reason":"broker was unresponsive"}}
+{"timestamp":1709681471.4289656,"name":"drain","context":{"idset":"733","reason":"broker was unresponsive"}}
+{"timestamp":1709681472.0564315,"name":"drain","context":{"idset":"734","reason":"broker was unresponsive"}}
+{"timestamp":1709681472.1440833,"name":"drain","context":{"idset":"735","reason":"broker was unresponsive"}}
+{"timestamp":1709681472.2474334,"name":"drain","context":{"idset":"736","reason":"broker was unresponsive"}}
+{"timestamp":1709681472.2672853,"name":"drain","context":{"idset":"737","reason":"broker was unresponsive"}}
+{"timestamp":1709681472.4184427,"name":"drain","context":{"idset":"738","reason":"broker was unresponsive"}}
+{"timestamp":1709681472.6889544,"name":"drain","context":{"idset":"739","reason":"broker was unresponsive"}}
+{"timestamp":1709681474.0712359,"name":"drain","context":{"idset":"753","reason":"broker was unresponsive"}}
+{"timestamp":1709681474.0774148,"name":"drain","context":{"idset":"754","reason":"broker was unresponsive"}}
+{"timestamp":1709681474.0830388,"name":"drain","context":{"idset":"755","reason":"broker was unresponsive"}}
+{"timestamp":1709684300.0615292,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1709684300.219054,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1709684300.7670026,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1709684301.3645978,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1709684301.4696734,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1709684301.5437365,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1709684302.1811032,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1709684302.4487095,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1709684302.764009,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1709684303.9956412,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1709684304.1866858,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1709684304.3120773,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1709684304.5358474,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1709684304.5446088,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1709684305.3977389,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1709684305.6032243,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1709684305.763231,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1709684307.2764826,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1709684307.415966,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1709684307.5782659,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1709684308.1993721,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1709684312.6284614,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1709684313.0433319,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1709684313.081991,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1709684313.2973766,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1709684313.4024057,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1709684314.4365509,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1709684314.5849791,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1709684314.6453335,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1709684315.4436355,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1709684315.78282,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1709684315.9840307,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1709684316.0942147,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1709684316.2013881,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1709684317.0204468,"name":"offline","context":{"idset":"454"}}
+{"timestamp":1709684317.1898649,"name":"offline","context":{"idset":"468"}}
+{"timestamp":1709684317.4143877,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1709684318.4788022,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1709684318.5974803,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1709684318.6870887,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1709684318.8785431,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1709684320.028358,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1709684320.1273682,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1709684320.4749029,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1709684321.5858047,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1709684321.7565153,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1709684321.8724298,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1709684322.7767875,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1709684322.8103998,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1709684322.9553161,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1709684323.1084628,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1709684323.7551193,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1709684323.9360828,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1709684324.0419919,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1709684324.2047453,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1709684325.3816822,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1709684325.4263566,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1709684325.594198,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1709684325.7498565,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1709684326.7604182,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1709684326.8688562,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1709684326.9151728,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1709684327.0396049,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1709684327.220171,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1709684328.1381969,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1709684328.2582865,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1709684328.3157809,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1709684328.7922683,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1709684329.5985947,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1709684329.6613302,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1709684329.8289492,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1709684329.9135432,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1709684330.8742383,"name":"offline","context":{"idset":"659"}}
+{"timestamp":1709684331.0144742,"name":"offline","context":{"idset":"670"}}
+{"timestamp":1709684331.1508214,"name":"offline","context":{"idset":"671"}}
+{"timestamp":1709684331.2636697,"name":"offline","context":{"idset":"675"}}
+{"timestamp":1709684332.3189528,"name":"offline","context":{"idset":"677"}}
+{"timestamp":1709684332.482142,"name":"offline","context":{"idset":"682"}}
+{"timestamp":1709684332.6184387,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1709684333.5134339,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1709684333.7162328,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1709684333.8023233,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1709684333.9169929,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1709684334.8355558,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1709684334.8884833,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1709684335.1125579,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1709684335.8354416,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1709684336.3523035,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1709684336.4513302,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1709684336.4970682,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1709684337.5407054,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1709684337.760673,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1709684337.9251485,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1709684338.1115677,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1709684339.1312883,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1709684339.240396,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1709684339.3520105,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1709684340.4917455,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1709684340.5860798,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1709684340.7623158,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1709684340.8900342,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1709684341.8219991,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1709684341.8630419,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1709684341.9972076,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1709684342.1055615,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1709684342.7811165,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1709684342.9257526,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1709684343.0674465,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1709684344.1141591,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1709684344.2555895,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1709684344.3700914,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1709684345.3675578,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1709684345.5314767,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1709684345.7169814,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1709684345.7906048,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1709684346.2930257,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1709684347.0720215,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1709684347.169647,"name":"offline","context":{"idset":"121"}}
+{"timestamp":1709684347.5070529,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1709684347.9763775,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1709684348.0797467,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1709684348.2306411,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1709684349.2882233,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1709684349.5493257,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1709684349.5720494,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1709684349.8173487,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1709684350.9326303,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1709684351.0666893,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1709684351.1047766,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1709684351.4198756,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1709684352.1682754,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1709684352.443763,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1709684352.5960357,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1709684352.7089744,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1709684353.683439,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1709684353.8375845,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1709684353.8891315,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1709684353.9750452,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1709684355.0023839,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1709684355.1619384,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1709684355.3785961,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1709684355.4159446,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1709684356.5154102,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1709684356.64749,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1709684356.7367105,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1709684356.7703955,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1709684357.6687636,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1709684357.9818847,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1709684358.1189628,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1709684358.1491039,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1709684359.1900339,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1709684359.3323491,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1709684359.4773538,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1709684360.162549,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1709684360.3640237,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1709684360.5541363,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1709684360.6834657,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1709684360.8863149,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1709684361.4111171,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1709684362.078047,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1709684362.095098,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1709684362.2764971,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1709684362.4618747,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1709684362.7007577,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1709684363.3936884,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1709684363.5326622,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1709684363.7906559,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1709684363.8632188,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1709684364.6618352,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1709684364.9043548,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1709684364.9891317,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1709684365.1705821,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1709684365.4070849,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1709684366.2798862,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1709684366.7305279,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1709684366.8227355,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1709684366.977452,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1709684367.7735705,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1709684368.0609038,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1709684368.1776583,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1709684368.2146771,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1709684368.4063766,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1709684368.6951714,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1709684369.2559314,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1709684369.3972573,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1709684369.4523702,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1709684369.6446388,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1709684370.7180152,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1709684370.7666738,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1709684370.8277619,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1709684370.8779533,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1709684371.0509114,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1709684371.3416686,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1709684372.2849257,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1709684372.303684,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1709684372.5244796,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1709684372.5630465,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1709684372.6252041,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1709684373.5267367,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1709684373.6225307,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1709684373.6978381,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1709684373.9120159,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1709684374.828429,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1709684374.9934599,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1709684375.1236024,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1709684375.2604327,"name":"offline","context":{"idset":"448"}}
+{"timestamp":1709684376.2623966,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1709684376.2795308,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1709684376.4704378,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1709684376.8636773,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1709684377.3438561,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1709684377.6918712,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1709684377.7606318,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1709684377.8859508,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1709684378.6027789,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1709684379.2441535,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1709684379.3264987,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1709684379.4312062,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1709684379.5274549,"name":"offline","context":{"idset":"551"}}
+{"timestamp":1709684379.5613518,"name":"offline","context":{"idset":"552"}}
+{"timestamp":1709684380.6642029,"name":"offline","context":{"idset":"554"}}
+{"timestamp":1709684380.9526803,"name":"offline","context":{"idset":"556"}}
+{"timestamp":1709684380.9695113,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1709684381.1582465,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1709684381.5413747,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1709684382.3107491,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1709684382.6283493,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1709684382.748075,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1709684383.0378509,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1709684383.8347266,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1709684384.3087828,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1709684384.4385247,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1709684384.5539358,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1709684385.5926845,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1709684385.7359271,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1709684385.7999616,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1709684385.9362297,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1709684386.3278043,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1709684386.6723032,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1709684387.1588888,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1709684387.3339224,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1709684387.3694696,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1709684387.7227693,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1709684388.4414783,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1709684388.5539911,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1709684388.6081469,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1709684388.6877789,"name":"offline","context":{"idset":"669"}}
+{"timestamp":1709684389.625946,"name":"offline","context":{"idset":"690"}}
+{"timestamp":1709684389.6575866,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1709684389.6933062,"name":"offline","context":{"idset":"694"}}
+{"timestamp":1709684389.863651,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1709684389.9498351,"name":"offline","context":{"idset":"701"}}
+{"timestamp":1709684390.0433261,"name":"offline","context":{"idset":"706"}}
+{"timestamp":1709684390.3008471,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1709684391.1315238,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1709684391.142848,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1709684391.2367496,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1709684391.2837389,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1709684391.3574622,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1709684392.6076307,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1709684392.6904802,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1709684392.7143004,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1709684392.8668556,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1709684393.0911305,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1709684395.6834145,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1709684396.3041914,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1709684396.4905939,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1709684397.2096488,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1709684397.6188335,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1709684397.6865237,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1709684398.608608,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1709684398.7269504,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1709684398.8547873,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1709684399.631846,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1709684399.7402954,"name":"offline","context":{"idset":"451"}}
+{"timestamp":1709684400.3061974,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1709684400.4770968,"name":"offline","context":{"idset":"555"}}
+{"timestamp":1709684400.5938087,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1709684400.6768177,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1709684401.9094951,"name":"offline","context":{"idset":"692"}}
+{"timestamp":1709684402.0503738,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1709684402.2656918,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1709684403.8223379,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1709684404.1596239,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1709684404.7764888,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1709684404.8611648,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1709684405.7060602,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1709684405.8820972,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1709684405.9645307,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1709684406.328218,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1709684407.0109174,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1709684407.0664256,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1709684407.1229131,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1709684408.0527713,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1709684408.1004519,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1709684408.2137768,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1709684408.2565536,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1709684408.8807514,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1709684408.8912001,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1709684409.3894937,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1709684409.4029052,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1709684409.4930613,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1709684410.3079274,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1709684410.4281955,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1709684410.4464383,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1709684410.5244474,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1709684410.9119291,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1709684411.6449378,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1709684412.1676159,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1709684412.2586901,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1709684412.3950016,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1709684413.1943195,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1709684413.6572802,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1709684413.7502875,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1709684413.8469169,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1709684415.0214424,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1709684415.1191707,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1709684415.1797495,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1709684416.2485886,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1709684416.3630285,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1709684416.4278901,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1709684416.5378528,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1709684416.89534,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1709684417.6441071,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1709684417.8011961,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1709684417.8595366,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1709684418.6958656,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1709684418.7065358,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1709684418.7451477,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1709684418.8209801,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1709684418.8969784,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1709684420.0808482,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1709684420.26301,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1709684420.2865217,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1709684420.4999766,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1709684421.3457761,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1709684421.3973808,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1709684421.4512804,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1709684421.5160534,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1709684422.1926067,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1709684422.314466,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1709684422.4177725,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1709684423.2510889,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1709684423.5906899,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1709684423.7561612,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1709684423.8866649,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1709684423.9001012,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1709684424.9587789,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1709684425.0307972,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1709684425.1800263,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1709684425.2727609,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1709684426.1707225,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1709684426.4427741,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1709684426.554745,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1709684426.6874976,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1709684426.7478213,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1709684428.1231964,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1709684428.1774821,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1709684428.2662027,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1709684428.4190259,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1709684429.227015,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1709684429.3155808,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1709684429.3603406,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1709684430.3695772,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1709684430.6343775,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1709684430.6927843,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1709684431.1705651,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1709684432.0493343,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1709684432.223078,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1709684432.3036621,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1709684432.5892448,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1709684433.3028209,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1709684433.4781432,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1709684433.7395821,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1709684433.7589905,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1709684433.8523555,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1709684434.7932138,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1709684434.9213262,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1709684435.0196149,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1709684435.0920792,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1709684435.1482465,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1709684435.7781782,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1709684435.93067,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1709684436.0636532,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1709684436.1466978,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1709684437.2029481,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1709684437.263144,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1709684437.3883023,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1709684437.7038157,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1709684437.8297906,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1709684438.4589009,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1709684438.5721009,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1709684438.7318406,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1709684439.5893409,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1709684439.8592355,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1709684440.051152,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1709684440.1089494,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1709684441.1343348,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1709684441.3219752,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1709684441.4149399,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1709684441.4576242,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1709684442.4119396,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1709684442.4481587,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1709684442.5292859,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1709684442.6264477,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1709684443.4671011,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1709684443.5563586,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1709684443.5895724,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1709684443.6416121,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1709684444.4904165,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1709684444.6978805,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1709684444.7904699,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1709684445.1024485,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1709684445.4811044,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1709684446.1231122,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1709684446.2267182,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1709684446.4149549,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1709684447.5643914,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1709684447.6334009,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1709684447.6924391,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1709684447.7626786,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1709684447.8887308,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1709684448.5059717,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1709684448.9677863,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1709684449.0587649,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1709684449.1910162,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1709684449.2456372,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1709684450.322118,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1709684450.3353333,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1709684450.5071888,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1709684451.3502328,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1709684451.4294937,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1709684451.5621421,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1709684451.8799105,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1709684451.9842165,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1709684452.0638442,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1709684452.6386011,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1709684452.7514229,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1709684453.3587673,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1709684453.6313393,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1709684453.7081404,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1709684453.860872,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1709684454.0117052,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1709684455.0807929,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1709684455.1198156,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1709684455.2659972,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1709684455.3787792,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1709684455.4498024,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1709684456.3591111,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1709684456.4958346,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1709684456.7185936,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1709684456.7799664,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1709684458.0266407,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1709684458.2551439,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1709684458.3376496,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1709684458.3966715,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1709684458.5171499,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1709684459.8054109,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1709684459.8666527,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1709684459.9555807,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1709684460.1423342,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1709684460.2248046,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1709684460.7331414,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1709684461.4221723,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1709684461.5000067,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1709684461.6855516,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1709684461.7449396,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1709684462.6434505,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1709684462.7023687,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1709684462.7478838,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1709684462.8436625,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1709684462.8570359,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1709684463.6636896,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1709684464.007791,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1709684464.133996,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1709684464.2084842,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1709684465.1887298,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1709684465.3694994,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1709684465.4557512,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1709684465.5809395,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1709684466.6050794,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1709684466.642117,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1709684466.8731043,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1709684466.9088118,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1709684466.9543993,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1709684466.9630105,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1709684467.304898,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1709684468.2708578,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1709684468.5050023,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1709684468.6039813,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1709684468.9400032,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1709684469.0420384,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1709684470.0890427,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1709684470.2010412,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1709684470.295665,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1709684470.3521342,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1709684470.3955791,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1709684471.7277641,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1709684471.805768,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1709684471.8458619,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1709684471.9223409,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1709684471.9711788,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1709684472.8864875,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1709684472.9289968,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1709684473.0472498,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1709684473.5595508,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1709684474.1931427,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1709684474.247546,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1709684474.5761647,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1709684474.6816266,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1709684475.3717628,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1709684475.4422586,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1709684475.7064338,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1709684476.9391234,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1709684477.0841024,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1709684477.120574,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1709684477.1764779,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1709684478.054544,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1709684478.1667869,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1709684478.2062318,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1709684478.2878182,"name":"offline","context":{"idset":"445"}}
+{"timestamp":1709684478.6524482,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1709684479.4487967,"name":"offline","context":{"idset":"447"}}
+{"timestamp":1709684479.5335665,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1709684480.0493071,"name":"offline","context":{"idset":"450"}}
+{"timestamp":1709684480.4216559,"name":"offline","context":{"idset":"452"}}
+{"timestamp":1709684480.5817306,"name":"offline","context":{"idset":"453"}}
+{"timestamp":1709684480.8923249,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1709684480.9427335,"name":"offline","context":{"idset":"456"}}
+{"timestamp":1709684481.0069079,"name":"offline","context":{"idset":"457"}}
+{"timestamp":1709684481.9071879,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1709684482.0642219,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1709684482.0986822,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1709684482.1525738,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1709684482.1877675,"name":"offline","context":{"idset":"462"}}
+{"timestamp":1709684482.9754667,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1709684483.2264476,"name":"offline","context":{"idset":"464"}}
+{"timestamp":1709684483.2646012,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1709684483.4142549,"name":"offline","context":{"idset":"466"}}
+{"timestamp":1709684483.5154753,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1709684483.6873617,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1709684484.1165183,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1709684484.185092,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1709684484.8532531,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1709684485.1455302,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1709684485.2093506,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1709684485.3697805,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1709684485.3811586,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1709684486.2281246,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1709684486.3006122,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1709684486.3816457,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1709684486.9945035,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1709684487.0891593,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1709684487.1499929,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1709684487.1895425,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1709684487.9772375,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1709684488.0327523,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1709684488.1027327,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1709684488.2928159,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1709684488.7827511,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1709684489.0695274,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1709684489.1188655,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1709684489.1521528,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1709684490.0927336,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1709684490.1565034,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1709684490.1956618,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1709684490.2301838,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1709684490.799258,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1709684491.1772728,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1709684491.2821243,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1709684491.3363898,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1709684492.0393946,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1709684492.1641464,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1709684492.2111657,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1709684492.2845633,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1709684492.5461051,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1709684493.3024659,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1709684493.4621575,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1709684493.5104029,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1709684493.5618317,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1709684494.0978923,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1709684494.9075677,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1709684494.985539,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1709684495.1770363,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1709684495.2117999,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1709684496.1310079,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1709684496.4103372,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1709684496.5777013,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1709684496.6819658,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1709684496.7199008,"name":"offline","context":{"idset":"549"}}
+{"timestamp":1709684497.2602656,"name":"offline","context":{"idset":"553"}}
+{"timestamp":1709684497.6359274,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1709684497.841486,"name":"offline","context":{"idset":"558"}}
+{"timestamp":1709684497.870224,"name":"offline","context":{"idset":"559"}}
+{"timestamp":1709684498.1026824,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1709684498.2023616,"name":"offline","context":{"idset":"561"}}
+{"timestamp":1709684498.2961028,"name":"offline","context":{"idset":"563"}}
+{"timestamp":1709684499.2082839,"name":"offline","context":{"idset":"564"}}
+{"timestamp":1709684499.486692,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1709684499.5107698,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1709684499.5590937,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1709684499.6032057,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1709684499.643666,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1709684500.3051984,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1709684500.4626079,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1709684500.8897359,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1709684500.9544485,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1709684501.0391517,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1709684501.1394324,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1709684501.5744753,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1709684501.7869279,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1709684502.045177,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1709684502.6318862,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1709684502.7153161,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1709684502.7626276,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1709684502.8069828,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1709684502.8528354,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1709684503.985554,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1709684503.9899609,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1709684504.0122859,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1709684504.1040435,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1709684504.1728287,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1709684504.2205923,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1709684505.2140353,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1709684505.2752075,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1709684505.3370323,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1709684506.1147852,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1709684506.1717615,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1709684506.2851858,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1709684506.3234229,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1709684506.3926349,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1709684506.7057743,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1709684507.3700082,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1709684507.4695435,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1709684507.5303376,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1709684508.6004012,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1709684508.7215462,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1709684508.7696695,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1709684508.8879235,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1709684509.3092456,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1709684509.5101497,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1709684509.829694,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1709684509.9214008,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1709684509.9965768,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1709684510.8935835,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1709684510.9434304,"name":"offline","context":{"idset":"654"}}
+{"timestamp":1709684510.9718182,"name":"offline","context":{"idset":"655"}}
+{"timestamp":1709684511.0182388,"name":"offline","context":{"idset":"656"}}
+{"timestamp":1709684511.9336076,"name":"offline","context":{"idset":"657"}}
+{"timestamp":1709684512.01296,"name":"offline","context":{"idset":"658"}}
+{"timestamp":1709684512.0638127,"name":"offline","context":{"idset":"660"}}
+{"timestamp":1709684512.0938351,"name":"offline","context":{"idset":"661"}}
+{"timestamp":1709684512.8179328,"name":"offline","context":{"idset":"662"}}
+{"timestamp":1709684512.9641349,"name":"offline","context":{"idset":"665"}}
+{"timestamp":1709684513.0460863,"name":"offline","context":{"idset":"666"}}
+{"timestamp":1709684513.4261801,"name":"offline","context":{"idset":"667"}}
+{"timestamp":1709684513.8014257,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1709684513.9372966,"name":"offline","context":{"idset":"672"}}
+{"timestamp":1709684513.9998407,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1709684514.7340291,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1709684514.8161695,"name":"offline","context":{"idset":"676"}}
+{"timestamp":1709684514.8775311,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1709684515.9083936,"name":"offline","context":{"idset":"679"}}
+{"timestamp":1709684516.0674174,"name":"offline","context":{"idset":"680"}}
+{"timestamp":1709684516.1060576,"name":"offline","context":{"idset":"681"}}
+{"timestamp":1709684516.1454275,"name":"offline","context":{"idset":"683"}}
+{"timestamp":1709684517.1495049,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1709684517.3148711,"name":"offline","context":{"idset":"685"}}
+{"timestamp":1709684517.3785021,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1709684518.2056682,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1709684518.2740271,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1709684518.3034389,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1709684518.3421187,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1709684518.9713151,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1709684519.0443048,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1709684519.4077845,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1709684519.6066489,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1709684520.2209537,"name":"offline","context":{"idset":"707"}}
+{"timestamp":1709684520.4530082,"name":"offline","context":{"idset":"708"}}
+{"timestamp":1709684520.5393436,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1709684520.6054728,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1709684521.7268965,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1709684521.8188317,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1709684521.8898082,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1709684521.9239721,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1709684522.716444,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1709684523.2551458,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1709684523.2756045,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1709684523.3606925,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1709684524.1603918,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1709684524.2953079,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1709684524.3919535,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1709684524.4651225,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1709684524.6025653,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1709684525.5459979,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1709684525.7487993,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1709684525.803885,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1709684525.8688588,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1709684525.8892853,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1709684526.9129176,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1709684527.0166817,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1709684527.0937586,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1709684527.1606457,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1709684527.1621151,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1709684527.3898053,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1709684527.7271695,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1709684612.962815,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"3":{"timestamp":1709681408.8212817,"reason":"broker was unresponsive"},"4":{"timestamp":1709681408.8318951,"reason":"broker was unresponsive"},"8":{"timestamp":1709681408.8635392,"reason":"broker was unresponsive"},"18":{"timestamp":1709681408.8827724,"reason":"broker was unresponsive"},"24":{"timestamp":1709681408.9095361,"reason":"broker was unresponsive"},"28":{"timestamp":1709681408.9368012,"reason":"broker was unresponsive"},"30":{"timestamp":1709681408.9620736,"reason":"broker was unresponsive"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"45":{"timestamp":1709681408.9713809,"reason":"broker was unresponsive"},"47":{"timestamp":1709681408.978828,"reason":"broker was unresponsive"},"48":{"timestamp":1709681409.8044841,"reason":"broker was unresponsive"},"49":{"timestamp":1709681409.8323469,"reason":"broker was unresponsive"},"54":{"timestamp":1709681409.8677628,"reason":"broker was unresponsive"},"59":{"timestamp":1709681409.8796892,"reason":"broker was unresponsive"},"61":{"timestamp":1709681409.8881745,"reason":"broker was unresponsive"},"62":{"timestamp":1709681409.8981011,"reason":"broker was unresponsive"},"63":{"timestamp":1709681409.9095984,"reason":"broker was unresponsive"},"64":{"timestamp":1709681410.1160378,"reason":"broker was unresponsive"},"65":{"timestamp":1709681410.1490726,"reason":"broker was unresponsive"},"66":{"timestamp":1709681381.2293751,"reason":"broker was unresponsive"},"67":{"timestamp":1709681381.2403064,"reason":"broker was unresponsive"},"68":{"timestamp":1709681410.1563275,"reason":"broker was unresponsive"},"69":{"timestamp":1709681410.162261,"reason":"broker was unresponsive"},"70":{"timestamp":1709681410.1708121,"reason":"broker was unresponsive"},"71":{"timestamp":1709681410.1939435,"reason":"broker was unresponsive"},"72":{"timestamp":1709681410.2934506,"reason":"broker was unresponsive"},"73":{"timestamp":1709681410.2965677,"reason":"broker was unresponsive"},"74":{"timestamp":1709681410.3037362,"reason":"broker was unresponsive"},"75":{"timestamp":1709681410.9298322,"reason":"broker was unresponsive"},"76":{"timestamp":1709681411.4928811,"reason":"broker was unresponsive"},"77":{"timestamp":1709681411.5084949,"reason":"broker was unresponsive"},"80":{"timestamp":1709681411.5167923,"reason":"broker was unresponsive"},"81":{"timestamp":1709681411.5255532,"reason":"broker was unresponsive"},"82":{"timestamp":1709681411.5611663,"reason":"broker was unresponsive"},"83":{"timestamp":1709681411.5831027,"reason":"broker was unresponsive"},"85":{"timestamp":1709681411.5988719,"reason":"broker was unresponsive"},"88":{"timestamp":1709681411.6055169,"reason":"broker was unresponsive"},"89":{"timestamp":1709681381.2628613,"reason":"broker was unresponsive"},"90":{"timestamp":1709681411.6190681,"reason":"broker was unresponsive"},"91":{"timestamp":1709681411.6429272,"reason":"broker was unresponsive"},"93":{"timestamp":1709681381.2866113,"reason":"broker was unresponsive"},"94":{"timestamp":1709681411.6575837,"reason":"broker was unresponsive"},"95":{"timestamp":1709681381.3943751,"reason":"broker was unresponsive"},"97":{"timestamp":1709681411.6683059,"reason":"broker was unresponsive"},"99":{"timestamp":1709681411.6712108,"reason":"broker was unresponsive"},"100":{"timestamp":1709681381.3979478,"reason":"broker was unresponsive"},"101":{"timestamp":1709681411.6777511,"reason":"broker was unresponsive"},"102":{"timestamp":1709681382.3681943,"reason":"broker was unresponsive"},"103":{"timestamp":1709681382.419282,"reason":"broker was unresponsive"},"106":{"timestamp":1709681411.6898465,"reason":"broker was unresponsive"},"107":{"timestamp":1709681411.6997364,"reason":"broker was unresponsive"},"109":{"timestamp":1709681382.4660995,"reason":"broker was unresponsive"},"110":{"timestamp":1709681382.4833553,"reason":"broker was unresponsive"},"112":{"timestamp":1709681411.7236922,"reason":"broker was unresponsive"},"113":{"timestamp":1709681411.8774562,"reason":"broker was unresponsive"},"114":{"timestamp":1709681411.8837709,"reason":"broker was unresponsive"},"115":{"timestamp":1709681411.8925068,"reason":"broker was unresponsive"},"116":{"timestamp":1709681411.8994727,"reason":"broker was unresponsive"},"117":{"timestamp":1709681411.9239893,"reason":"broker was unresponsive"},"118":{"timestamp":1709681413.8736432,"reason":"broker was unresponsive"},"119":{"timestamp":1709681382.5015171,"reason":"broker was unresponsive"},"123":{"timestamp":1709681413.889564,"reason":"broker was unresponsive"},"124":{"timestamp":1709681413.9137659,"reason":"broker was unresponsive"},"125":{"timestamp":1709681382.5165713,"reason":"broker was unresponsive"},"126":{"timestamp":1709681413.9186139,"reason":"broker was unresponsive"},"127":{"timestamp":1709681413.9231811,"reason":"broker was unresponsive"},"128":{"timestamp":1709681413.9277308,"reason":"broker was unresponsive"},"129":{"timestamp":1709681413.9315896,"reason":"broker was unresponsive"},"130":{"timestamp":1709681413.935252,"reason":"broker was unresponsive"},"131":{"timestamp":1709681382.5881133,"reason":"broker was unresponsive"},"132":{"timestamp":1709681413.9394503,"reason":"broker was unresponsive"},"134":{"timestamp":1709681413.9431677,"reason":"broker was unresponsive"},"136":{"timestamp":1709681382.591558,"reason":"broker was unresponsive"},"137":{"timestamp":1709681382.5965333,"reason":"broker was unresponsive"},"139":{"timestamp":1709681382.601145,"reason":"broker was unresponsive"},"140":{"timestamp":1709681413.9498262,"reason":"broker was unresponsive"},"143":{"timestamp":1709681413.9536166,"reason":"broker was unresponsive"},"144":{"timestamp":1709681382.6199274,"reason":"broker was unresponsive"},"145":{"timestamp":1709681382.6291566,"reason":"broker was unresponsive"},"147":{"timestamp":1709681413.9614174,"reason":"broker was unresponsive"},"156":{"timestamp":1709681413.9705269,"reason":"broker was unresponsive"},"160":{"timestamp":1709681413.9775803,"reason":"broker was unresponsive"},"167":{"timestamp":1709681382.6716454,"reason":"broker was unresponsive"},"173":{"timestamp":1709681414.3250537,"reason":"broker was unresponsive"},"174":{"timestamp":1709681414.3712187,"reason":"broker was unresponsive"},"177":{"timestamp":1709681382.9935443,"reason":"broker was unresponsive"},"185":{"timestamp":1709681414.5448501,"reason":"broker was unresponsive"},"191":{"timestamp":1709681384.1277764,"reason":"broker was unresponsive"},"193":{"timestamp":1709681414.5669005,"reason":"broker was unresponsive"},"194":{"timestamp":1709681384.1364768,"reason":"broker was unresponsive"},"195":{"timestamp":1709681414.6065304,"reason":"broker was unresponsive"},"211":{"timestamp":1709681384.1609488,"reason":"broker was unresponsive"},"216":{"timestamp":1709681418.6313856,"reason":"broker was unresponsive"},"217":{"timestamp":1709681384.1667616,"reason":"broker was unresponsive"},"221":{"timestamp":1709681384.1720257,"reason":"broker was unresponsive"},"223":{"timestamp":1709681418.7555635,"reason":"broker was unresponsive"},"225":{"timestamp":1709681418.7607675,"reason":"broker was unresponsive"},"228":{"timestamp":1709681418.771693,"reason":"broker was unresponsive"},"229":{"timestamp":1709681384.1771307,"reason":"broker was unresponsive"},"235":{"timestamp":1709681384.1813967,"reason":"broker was unresponsive"},"236":{"timestamp":1709681384.1856766,"reason":"broker was unresponsive"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"245":{"timestamp":1709681384.1935496,"reason":"broker was unresponsive"},"248":{"timestamp":1709681384.3082535,"reason":"broker was unresponsive"},"249":{"timestamp":1709681384.3107648,"reason":"broker was unresponsive"},"250":{"timestamp":1709681418.7830951,"reason":"broker was unresponsive"},"254":{"timestamp":1709681418.7910876,"reason":"broker was unresponsive"},"255":{"timestamp":1709681384.3291364,"reason":"broker was unresponsive"},"263":{"timestamp":1709681384.3358667,"reason":"broker was unresponsive"},"264":{"timestamp":1709681384.3633454,"reason":"broker was unresponsive"},"278":{"timestamp":1709681425.4642551,"reason":"broker was unresponsive"},"283":{"timestamp":1709681425.4762263,"reason":"broker was unresponsive"},"284":{"timestamp":1709681384.3796248,"reason":"broker was unresponsive"},"285":{"timestamp":1709681384.3889556,"reason":"broker was unresponsive"},"287":{"timestamp":1709681425.4811175,"reason":"broker was unresponsive"},"289":{"timestamp":1709681425.5408835,"reason":"broker was unresponsive"},"290":{"timestamp":1709681425.5603411,"reason":"broker was unresponsive"},"294":{"timestamp":1709681425.5680461,"reason":"broker was unresponsive"},"295":{"timestamp":1709681425.5816805,"reason":"broker was unresponsive"},"296":{"timestamp":1709681384.3927896,"reason":"broker was unresponsive"},"300":{"timestamp":1709681425.5854528,"reason":"broker was unresponsive"},"301":{"timestamp":1709681384.4029431,"reason":"broker was unresponsive"},"302":{"timestamp":1709681425.6227891,"reason":"broker was unresponsive"},"304":{"timestamp":1709681384.4564159,"reason":"broker was unresponsive"},"305":{"timestamp":1709681384.5732772,"reason":"broker was unresponsive"},"306":{"timestamp":1709681425.680407,"reason":"broker was unresponsive"},"308":{"timestamp":1709681425.6869066,"reason":"broker was unresponsive"},"309":{"timestamp":1709681384.5834227,"reason":"broker was unresponsive"},"310":{"timestamp":1709681428.6819887,"reason":"broker was unresponsive"},"312":{"timestamp":1709681428.7005777,"reason":"broker was unresponsive"},"313":{"timestamp":1709681428.7034364,"reason":"broker was unresponsive"},"314":{"timestamp":1709681428.710573,"reason":"broker was unresponsive"},"316":{"timestamp":1709681428.7154613,"reason":"broker was unresponsive"},"317":{"timestamp":1709681428.7214339,"reason":"broker was unresponsive"},"318":{"timestamp":1709681384.5877018,"reason":"broker was unresponsive"},"319":{"timestamp":1709681428.738225,"reason":"broker was unresponsive"},"320":{"timestamp":1709681384.5910974,"reason":"broker was unresponsive"},"321":{"timestamp":1709681428.7530956,"reason":"broker was unresponsive"},"322":{"timestamp":1709681428.786428,"reason":"broker was unresponsive"},"323":{"timestamp":1709681428.8370903,"reason":"broker was unresponsive"},"325":{"timestamp":1709681384.5974274,"reason":"broker was unresponsive"},"327":{"timestamp":1709681428.8597717,"reason":"broker was unresponsive"},"330":{"timestamp":1709681432.5538156,"reason":"broker was unresponsive"},"331":{"timestamp":1709681432.5761559,"reason":"broker was unresponsive"},"332":{"timestamp":1709681432.5861928,"reason":"broker was unresponsive"},"334":{"timestamp":1709681384.6015635,"reason":"broker was unresponsive"},"335":{"timestamp":1709681432.5986011,"reason":"broker was unresponsive"},"337":{"timestamp":1709681432.611671,"reason":"broker was unresponsive"},"338":{"timestamp":1709681432.6890168,"reason":"broker was unresponsive"},"340":{"timestamp":1709681432.7105832,"reason":"broker was unresponsive"},"342":{"timestamp":1709681432.7151535,"reason":"broker was unresponsive"},"343":{"timestamp":1709681433.1126852,"reason":"broker was unresponsive"},"344":{"timestamp":1709681384.6050229,"reason":"broker was unresponsive"},"347":{"timestamp":1709681433.3106022,"reason":"broker was unresponsive"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"349":{"timestamp":1709681433.3506317,"reason":"broker was unresponsive"},"352":{"timestamp":1709681433.3695176,"reason":"broker was unresponsive"},"353":{"timestamp":1709681433.3850417,"reason":"broker was unresponsive"},"354":{"timestamp":1709681386.2968731,"reason":"broker was unresponsive"},"356":{"timestamp":1709681433.3995831,"reason":"broker was unresponsive"},"359":{"timestamp":1709681433.4171181,"reason":"broker was unresponsive"},"365":{"timestamp":1709681433.4350603,"reason":"broker was unresponsive"},"367":{"timestamp":1709681433.4505336,"reason":"broker was unresponsive"},"368":{"timestamp":1709681433.4663775,"reason":"broker was unresponsive"},"370":{"timestamp":1709681433.4783361,"reason":"broker was unresponsive"},"371":{"timestamp":1709681433.4914904,"reason":"broker was unresponsive"},"372":{"timestamp":1709681386.6589758,"reason":"broker was unresponsive"},"373":{"timestamp":1709681433.4972129,"reason":"broker was unresponsive"},"374":{"timestamp":1709681433.5550256,"reason":"broker was unresponsive"},"375":{"timestamp":1709681433.565757,"reason":"broker was unresponsive"},"376":{"timestamp":1709681433.5699155,"reason":"broker was unresponsive"},"377":{"timestamp":1709681386.6731384,"reason":"broker was unresponsive"},"378":{"timestamp":1709681433.6082144,"reason":"broker was unresponsive"},"379":{"timestamp":1709681433.9189322,"reason":"broker was unresponsive"},"380":{"timestamp":1709681434.6768849,"reason":"broker was unresponsive"},"381":{"timestamp":1709681434.8973839,"reason":"broker was unresponsive"},"384":{"timestamp":1709681435.0834649,"reason":"broker was unresponsive"},"385":{"timestamp":1709681438.9891098,"reason":"broker was unresponsive"},"387":{"timestamp":1709681386.6789196,"reason":"broker was unresponsive"},"388":{"timestamp":1709681386.6861463,"reason":"broker was unresponsive"},"389":{"timestamp":1709681439.1813583,"reason":"broker was unresponsive"},"390":{"timestamp":1709681386.6887178,"reason":"broker was unresponsive"},"391":{"timestamp":1709681439.1841698,"reason":"broker was unresponsive"},"393":{"timestamp":1709681439.189379,"reason":"broker was unresponsive"},"394":{"timestamp":1709681439.372478,"reason":"broker was unresponsive"},"396":{"timestamp":1709681386.692827,"reason":"broker was unresponsive"},"397":{"timestamp":1709681439.3800902,"reason":"broker was unresponsive"},"398":{"timestamp":1709681439.3844814,"reason":"broker was unresponsive"},"399":{"timestamp":1709681439.3888657,"reason":"broker was unresponsive"},"400":{"timestamp":1709681386.7261381,"reason":"broker was unresponsive"},"401":{"timestamp":1709681386.7330506,"reason":"broker was unresponsive"},"404":{"timestamp":1709681439.3964338,"reason":"broker was unresponsive"},"406":{"timestamp":1709681439.4022353,"reason":"broker was unresponsive"},"408":{"timestamp":1709681439.4120338,"reason":"broker was unresponsive"},"409":{"timestamp":1709681439.4245939,"reason":"broker was unresponsive"},"410":{"timestamp":1709681439.4314182,"reason":"broker was unresponsive"},"412":{"timestamp":1709681439.4390786,"reason":"broker was unresponsive"},"414":{"timestamp":1709681439.4490526,"reason":"broker was unresponsive"},"415":{"timestamp":1709681439.4601121,"reason":"broker was unresponsive"},"416":{"timestamp":1709681439.4665427,"reason":"broker was unresponsive"},"417":{"timestamp":1709681439.4774132,"reason":"broker was unresponsive"},"418":{"timestamp":1709681386.7380211,"reason":"broker was unresponsive"},"419":{"timestamp":1709681439.4800189,"reason":"broker was unresponsive"},"420":{"timestamp":1709681439.4906983,"reason":"broker was unresponsive"},"421":{"timestamp":1709681439.4978843,"reason":"broker was unresponsive"},"423":{"timestamp":1709681439.5023873,"reason":"broker was unresponsive"},"425":{"timestamp":1709681439.5080755,"reason":"broker was unresponsive"},"426":{"timestamp":1709681439.512033,"reason":"broker was unresponsive"},"428":{"timestamp":1709681386.7861958,"reason":"broker was unresponsive"},"432":{"timestamp":1709681439.5167994,"reason":"broker was unresponsive"},"435":{"timestamp":1709681439.5256426,"reason":"broker was unresponsive"},"436":{"timestamp":1709681439.5335879,"reason":"broker was unresponsive"},"437":{"timestamp":1709681386.8500264,"reason":"broker was unresponsive"},"438":{"timestamp":1709681439.5403166,"reason":"broker was unresponsive"},"439":{"timestamp":1709681439.547081,"reason":"broker was unresponsive"},"440":{"timestamp":1709681439.6330161,"reason":"broker was unresponsive"},"442":{"timestamp":1709681386.8726375,"reason":"broker was unresponsive"},"443":{"timestamp":1709681439.639883,"reason":"broker was unresponsive"},"444":{"timestamp":1709681439.6500537,"reason":"broker was unresponsive"},"445":{"timestamp":1709681386.8894696,"reason":"broker was unresponsive"},"447":{"timestamp":1709681386.911958,"reason":"broker was unresponsive"},"449":{"timestamp":1709681439.6567371,"reason":"broker was unresponsive"},"450":{"timestamp":1709681439.6654961,"reason":"broker was unresponsive"},"455":{"timestamp":1709681439.6844475,"reason":"broker was unresponsive"},"456":{"timestamp":1709681386.9361956,"reason":"broker was unresponsive"},"458":{"timestamp":1709681386.9580297,"reason":"broker was unresponsive"},"459":{"timestamp":1709681386.9728115,"reason":"broker was unresponsive"},"461":{"timestamp":1709681439.7049119,"reason":"broker was unresponsive"},"462":{"timestamp":1709681387.1055453,"reason":"broker was unresponsive"},"463":{"timestamp":1709681439.7256489,"reason":"broker was unresponsive"},"464":{"timestamp":1709681439.7991068,"reason":"broker was unresponsive"},"465":{"timestamp":1709681387.1210575,"reason":"broker was unresponsive"},"467":{"timestamp":1709681439.8154979,"reason":"broker was unresponsive"},"468":{"timestamp":1709681439.8309884,"reason":"broker was unresponsive"},"469":{"timestamp":1709681439.8468037,"reason":"broker was unresponsive"},"470":{"timestamp":1709681439.915415,"reason":"broker was unresponsive"},"496":{"timestamp":1709681444.656419,"reason":"broker was unresponsive"},"497":{"timestamp":1709681390.1232228,"reason":"broker was unresponsive"},"498":{"timestamp":1709681390.130908,"reason":"broker was unresponsive"},"499":{"timestamp":1709681444.7323587,"reason":"broker was unresponsive"},"500":{"timestamp":1709681444.7795947,"reason":"broker was unresponsive"},"501":{"timestamp":1709681444.8375535,"reason":"broker was unresponsive"},"502":{"timestamp":1709681444.893415,"reason":"broker was unresponsive"},"503":{"timestamp":1709681390.1417434,"reason":"broker was unresponsive"},"504":{"timestamp":1709681444.9844787,"reason":"broker was unresponsive"},"505":{"timestamp":1709681390.1518021,"reason":"broker was unresponsive"},"506":{"timestamp":1709681390.1594672,"reason":"broker was unresponsive"},"507":{"timestamp":1709681445.0348909,"reason":"broker was unresponsive"},"508":{"timestamp":1709681445.2083032,"reason":"broker was unresponsive"},"509":{"timestamp":1709681445.3055527,"reason":"broker was unresponsive"},"510":{"timestamp":1709681390.166888,"reason":"broker was unresponsive"},"511":{"timestamp":1709681445.3199012,"reason":"broker was unresponsive"},"512":{"timestamp":1709681445.3323352,"reason":"broker was unresponsive"},"513":{"timestamp":1709681390.1765316,"reason":"broker was unresponsive"},"514":{"timestamp":1709681390.1830907,"reason":"broker was unresponsive"},"515":{"timestamp":1709681445.3458421,"reason":"broker was unresponsive"},"516":{"timestamp":1709681445.366606,"reason":"broker was unresponsive"},"517":{"timestamp":1709681390.1913648,"reason":"broker was unresponsive"},"518":{"timestamp":1709681445.4460552,"reason":"broker was unresponsive"},"519":{"timestamp":1709681445.5083697,"reason":"broker was unresponsive"},"520":{"timestamp":1709681445.5508111,"reason":"broker was unresponsive"},"521":{"timestamp":1709681445.5768547,"reason":"broker was unresponsive"},"522":{"timestamp":1709681445.5895514,"reason":"broker was unresponsive"},"523":{"timestamp":1709681445.6053863,"reason":"broker was unresponsive"},"524":{"timestamp":1709681445.6180704,"reason":"broker was unresponsive"},"525":{"timestamp":1709681445.6335545,"reason":"broker was unresponsive"},"526":{"timestamp":1709681390.2014601,"reason":"broker was unresponsive"},"527":{"timestamp":1709681445.7080548,"reason":"broker was unresponsive"},"528":{"timestamp":1709681445.7290118,"reason":"broker was unresponsive"},"529":{"timestamp":1709681445.7520633,"reason":"broker was unresponsive"},"530":{"timestamp":1709681445.7737257,"reason":"broker was unresponsive"},"531":{"timestamp":1709681445.7902575,"reason":"broker was unresponsive"},"532":{"timestamp":1709681445.8077328,"reason":"broker was unresponsive"},"533":{"timestamp":1709681445.8283398,"reason":"broker was unresponsive"},"534":{"timestamp":1709681445.8443666,"reason":"broker was unresponsive"},"535":{"timestamp":1709681445.8696365,"reason":"broker was unresponsive"},"536":{"timestamp":1709681390.2076747,"reason":"broker was unresponsive"},"537":{"timestamp":1709681445.8917572,"reason":"broker was unresponsive"},"538":{"timestamp":1709681445.9068663,"reason":"broker was unresponsive"},"539":{"timestamp":1709681445.9770317,"reason":"broker was unresponsive"},"540":{"timestamp":1709681451.1550701,"reason":"broker was unresponsive"},"541":{"timestamp":1709681451.4069738,"reason":"broker was unresponsive"},"542":{"timestamp":1709681390.2179804,"reason":"broker was unresponsive"},"543":{"timestamp":1709681390.2264061,"reason":"broker was unresponsive"},"544":{"timestamp":1709681390.2353237,"reason":"broker was unresponsive"},"545":{"timestamp":1709681451.4169655,"reason":"broker was unresponsive"},"546":{"timestamp":1709681390.256367,"reason":"broker was unresponsive"},"547":{"timestamp":1709681451.4255848,"reason":"broker was unresponsive"},"548":{"timestamp":1709681451.4344933,"reason":"broker was unresponsive"},"549":{"timestamp":1709681451.5093682,"reason":"broker was unresponsive"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"551":{"timestamp":1709681451.537534,"reason":"broker was unresponsive"},"552":{"timestamp":1709681451.5410399,"reason":"broker was unresponsive"},"553":{"timestamp":1709681390.2609541,"reason":"broker was unresponsive"},"554":{"timestamp":1709681451.6104763,"reason":"broker was unresponsive"},"555":{"timestamp":1709681451.6717699,"reason":"broker was unresponsive"},"556":{"timestamp":1709681451.7294922,"reason":"broker was unresponsive"},"557":{"timestamp":1709681451.7342887,"reason":"broker was unresponsive"},"558":{"timestamp":1709681451.7904696,"reason":"broker was unresponsive"},"559":{"timestamp":1709681451.79654,"reason":"broker was unresponsive"},"560":{"timestamp":1709681451.9456582,"reason":"broker was unresponsive"},"561":{"timestamp":1709681451.9605997,"reason":"broker was unresponsive"},"562":{"timestamp":1709681451.9817872,"reason":"broker was unresponsive"},"563":{"timestamp":1709681451.9978135,"reason":"broker was unresponsive"},"564":{"timestamp":1709681393.892544,"reason":"broker was unresponsive"},"565":{"timestamp":1709681452.020858,"reason":"broker was unresponsive"},"566":{"timestamp":1709681393.8968093,"reason":"broker was unresponsive"},"567":{"timestamp":1709681452.0408056,"reason":"broker was unresponsive"},"568":{"timestamp":1709681452.0699167,"reason":"broker was unresponsive"},"569":{"timestamp":1709583475.6141074,"reason":"prolog failed for jobid fgjQELtBquR"},"570":{"timestamp":1709681452.092833,"reason":"broker was unresponsive"},"571":{"timestamp":1709681452.1108153,"reason":"broker was unresponsive"},"572":{"timestamp":1709681393.9307537,"reason":"broker was unresponsive"},"573":{"timestamp":1709681452.1288238,"reason":"broker was unresponsive"},"574":{"timestamp":1709681452.1517413,"reason":"broker was unresponsive"},"575":{"timestamp":1709681452.1698351,"reason":"broker was unresponsive"},"576":{"timestamp":1709681452.1891766,"reason":"broker was unresponsive"},"577":{"timestamp":1709681452.2081373,"reason":"broker was unresponsive"},"578":{"timestamp":1709681452.2572849,"reason":"broker was unresponsive"},"579":{"timestamp":1709681452.2732096,"reason":"broker was unresponsive"},"580":{"timestamp":1709681452.2905619,"reason":"broker was unresponsive"},"581":{"timestamp":1709681452.3076477,"reason":"broker was unresponsive"},"582":{"timestamp":1709681452.3279881,"reason":"broker was unresponsive"},"583":{"timestamp":1709681452.3447528,"reason":"broker was unresponsive"},"584":{"timestamp":1709681393.9365761,"reason":"broker was unresponsive"},"585":{"timestamp":1709681457.3321126,"reason":"broker was unresponsive"},"586":{"timestamp":1709681457.3353755,"reason":"broker was unresponsive"},"587":{"timestamp":1709681457.3492317,"reason":"broker was unresponsive"},"588":{"timestamp":1709681457.3592567,"reason":"broker was unresponsive"},"589":{"timestamp":1709681457.3682387,"reason":"broker was unresponsive"},"590":{"timestamp":1709681457.3771901,"reason":"broker was unresponsive"},"591":{"timestamp":1709681457.3901806,"reason":"broker was unresponsive"},"592":{"timestamp":1709681457.3956764,"reason":"broker was unresponsive"},"593":{"timestamp":1709681393.9513502,"reason":"broker was unresponsive"},"594":{"timestamp":1709681457.4047773,"reason":"broker was unresponsive"},"595":{"timestamp":1709681457.4122798,"reason":"broker was unresponsive"},"596":{"timestamp":1709681457.4224672,"reason":"broker was unresponsive"},"597":{"timestamp":1709681457.4281836,"reason":"broker was unresponsive"},"598":{"timestamp":1709681457.4379952,"reason":"broker was unresponsive"},"599":{"timestamp":1709681393.9726408,"reason":"broker was unresponsive"},"600":{"timestamp":1709681457.4440696,"reason":"broker was unresponsive"},"601":{"timestamp":1709681393.9864061,"reason":"broker was unresponsive"},"602":{"timestamp":1709681457.4533715,"reason":"broker was unresponsive"},"603":{"timestamp":1709681457.463896,"reason":"broker was unresponsive"},"604":{"timestamp":1709681457.4704916,"reason":"broker was unresponsive"},"605":{"timestamp":1709681457.4770453,"reason":"broker was unresponsive"},"606":{"timestamp":1709681457.4861119,"reason":"broker was unresponsive"},"607":{"timestamp":1709681457.6161799,"reason":"broker was unresponsive"},"608":{"timestamp":1709681457.6727376,"reason":"broker was unresponsive"},"609":{"timestamp":1709681394.0069788,"reason":"broker was unresponsive"},"610":{"timestamp":1709681457.6784515,"reason":"broker was unresponsive"},"611":{"timestamp":1709681458.6046863,"reason":"broker was unresponsive"},"612":{"timestamp":1709681458.6408889,"reason":"broker was unresponsive"},"613":{"timestamp":1709681458.6858304,"reason":"broker was unresponsive"},"614":{"timestamp":1709681458.7003932,"reason":"broker was unresponsive"},"615":{"timestamp":1709681396.8579962,"reason":"broker was unresponsive"},"616":{"timestamp":1709681458.7200763,"reason":"broker was unresponsive"},"617":{"timestamp":1709681458.7348564,"reason":"broker was unresponsive"},"618":{"timestamp":1709681459.029072,"reason":"broker was unresponsive"},"619":{"timestamp":1709681396.9853489,"reason":"broker was unresponsive"},"620":{"timestamp":1709681459.0506647,"reason":"broker was unresponsive"},"621":{"timestamp":1709681459.0722833,"reason":"broker was unresponsive"},"622":{"timestamp":1709681459.0948062,"reason":"broker was unresponsive"},"623":{"timestamp":1709681459.1201332,"reason":"broker was unresponsive"},"624":{"timestamp":1709681397.0042694,"reason":"broker was unresponsive"},"625":{"timestamp":1709681459.1516814,"reason":"broker was unresponsive"},"626":{"timestamp":1709681397.0206926,"reason":"broker was unresponsive"},"627":{"timestamp":1709681459.2327936,"reason":"broker was unresponsive"},"628":{"timestamp":1709681459.30176,"reason":"broker was unresponsive"},"629":{"timestamp":1709681459.3216348,"reason":"broker was unresponsive"},"630":{"timestamp":1709681459.4090993,"reason":"broker was unresponsive"},"631":{"timestamp":1709681459.5167036,"reason":"broker was unresponsive"},"632":{"timestamp":1709681459.5370102,"reason":"broker was unresponsive"},"633":{"timestamp":1709681459.5534525,"reason":"broker was unresponsive"},"634":{"timestamp":1709681459.5877225,"reason":"broker was unresponsive"},"635":{"timestamp":1709681459.6571665,"reason":"broker was unresponsive"},"636":{"timestamp":1709681459.7134571,"reason":"broker was unresponsive"},"637":{"timestamp":1709681459.753026,"reason":"broker was unresponsive"},"638":{"timestamp":1709681459.779016,"reason":"broker was unresponsive"},"639":{"timestamp":1709681459.798821,"reason":"broker was unresponsive"},"640":{"timestamp":1709681459.8620541,"reason":"broker was unresponsive"},"641":{"timestamp":1709681459.927917,"reason":"broker was unresponsive"},"642":{"timestamp":1709681460.2144468,"reason":"broker was unresponsive"},"643":{"timestamp":1709681463.1880548,"reason":"broker was unresponsive"},"644":{"timestamp":1709681463.1968279,"reason":"broker was unresponsive"},"645":{"timestamp":1709681463.5096676,"reason":"broker was unresponsive"},"646":{"timestamp":1709681463.6065159,"reason":"broker was unresponsive"},"647":{"timestamp":1709681463.6185288,"reason":"broker was unresponsive"},"648":{"timestamp":1709681463.960393,"reason":"broker was unresponsive"},"649":{"timestamp":1709681464.0335288,"reason":"broker was unresponsive"},"650":{"timestamp":1709681464.0864179,"reason":"broker was unresponsive"},"651":{"timestamp":1709681464.0899096,"reason":"broker was unresponsive"},"652":{"timestamp":1709681464.0974226,"reason":"broker was unresponsive"},"653":{"timestamp":1709681464.1021588,"reason":"broker was unresponsive"},"654":{"timestamp":1709681464.1101449,"reason":"broker was unresponsive"},"655":{"timestamp":1709681464.1750033,"reason":"broker was unresponsive"},"656":{"timestamp":1709681464.1845353,"reason":"broker was unresponsive"},"657":{"timestamp":1709681464.194587,"reason":"broker was unresponsive"},"658":{"timestamp":1709681464.2021778,"reason":"broker was unresponsive"},"659":{"timestamp":1709681464.2108328,"reason":"broker was unresponsive"},"660":{"timestamp":1709681464.2247074,"reason":"broker was unresponsive"},"661":{"timestamp":1709681464.2394729,"reason":"broker was unresponsive"},"662":{"timestamp":1709681464.2477934,"reason":"broker was unresponsive"},"663":{"timestamp":1709681464.2582645,"reason":"broker was unresponsive"},"664":{"timestamp":1709681464.2764733,"reason":"broker was unresponsive"},"665":{"timestamp":1709681464.2877569,"reason":"broker was unresponsive"},"666":{"timestamp":1709681464.3000238,"reason":"broker was unresponsive"},"667":{"timestamp":1709681464.3133538,"reason":"broker was unresponsive"},"668":{"timestamp":1709681446.0297892,"reason":"broker was unresponsive"},"669":{"timestamp":1709681446.0477581,"reason":"broker was unresponsive"},"670":{"timestamp":1709681446.1408155,"reason":"broker was unresponsive"},"671":{"timestamp":1709681446.1588423,"reason":"broker was unresponsive"},"672":{"timestamp":1709681446.1744151,"reason":"broker was unresponsive"},"673":{"timestamp":1709681399.1990819,"reason":"broker was unresponsive"},"674":{"timestamp":1709681446.1792593,"reason":"broker was unresponsive"},"675":{"timestamp":1709681439.9394345,"reason":"broker was unresponsive"},"676":{"timestamp":1709681446.2576051,"reason":"broker was unresponsive"},"677":{"timestamp":1709681439.9599004,"reason":"broker was unresponsive"},"678":{"timestamp":1709681402.934957,"reason":"broker was unresponsive"},"679":{"timestamp":1709681439.9872706,"reason":"broker was unresponsive"},"680":{"timestamp":1709681402.9408548,"reason":"broker was unresponsive"},"681":{"timestamp":1709681402.9469535,"reason":"broker was unresponsive"},"682":{"timestamp":1709681440.0139453,"reason":"broker was unresponsive"},"683":{"timestamp":1709681440.0388384,"reason":"broker was unresponsive"},"684":{"timestamp":1709681440.0586998,"reason":"broker was unresponsive"},"685":{"timestamp":1709681402.9542124,"reason":"broker was unresponsive"},"686":{"timestamp":1709681440.0766542,"reason":"broker was unresponsive"},"687":{"timestamp":1709681402.962131,"reason":"broker was unresponsive"},"688":{"timestamp":1709681402.9762506,"reason":"broker was unresponsive"},"689":{"timestamp":1709681440.0956101,"reason":"broker was unresponsive"},"690":{"timestamp":1709681440.1842473,"reason":"broker was unresponsive"},"691":{"timestamp":1709681402.9895954,"reason":"broker was unresponsive"},"692":{"timestamp":1709681440.2477794,"reason":"broker was unresponsive"},"693":{"timestamp":1709681440.3140657,"reason":"broker was unresponsive"},"694":{"timestamp":1709681440.3206289,"reason":"broker was unresponsive"},"695":{"timestamp":1709681440.4924848,"reason":"broker was unresponsive"},"696":{"timestamp":1709681440.5031359,"reason":"broker was unresponsive"},"697":{"timestamp":1709681440.55181,"reason":"broker was unresponsive"},"698":{"timestamp":1709681440.5581985,"reason":"broker was unresponsive"},"699":{"timestamp":1709681403.006067,"reason":"broker was unresponsive"},"700":{"timestamp":1709681440.5640523,"reason":"broker was unresponsive"},"701":{"timestamp":1709681435.3205395,"reason":"broker was unresponsive"},"702":{"timestamp":1709681435.4848008,"reason":"broker was unresponsive"},"703":{"timestamp":1709681435.5398629,"reason":"broker was unresponsive"},"704":{"timestamp":1709681403.0211267,"reason":"broker was unresponsive"},"705":{"timestamp":1709681435.5721347,"reason":"broker was unresponsive"},"706":{"timestamp":1709681428.8692617,"reason":"broker was unresponsive"},"707":{"timestamp":1709681435.5822656,"reason":"broker was unresponsive"},"708":{"timestamp":1709681403.035203,"reason":"broker was unresponsive"},"709":{"timestamp":1709681428.8727474,"reason":"broker was unresponsive"},"710":{"timestamp":1709681403.0472131,"reason":"broker was unresponsive"},"711":{"timestamp":1709681428.8797991,"reason":"broker was unresponsive"},"712":{"timestamp":1709681411.73578,"reason":"broker was unresponsive"},"713":{"timestamp":1709681425.7020419,"reason":"broker was unresponsive"},"714":{"timestamp":1709681428.8989787,"reason":"broker was unresponsive"},"715":{"timestamp":1709681425.7187603,"reason":"broker was unresponsive"},"716":{"timestamp":1709681425.7829328,"reason":"broker was unresponsive"},"717":{"timestamp":1709681425.8226144,"reason":"broker was unresponsive"},"718":{"timestamp":1709681425.9167569,"reason":"broker was unresponsive"},"719":{"timestamp":1709681411.7581737,"reason":"broker was unresponsive"},"720":{"timestamp":1709681425.9422958,"reason":"broker was unresponsive"},"721":{"timestamp":1709681418.7963738,"reason":"broker was unresponsive"},"722":{"timestamp":1709681418.8056767,"reason":"broker was unresponsive"},"723":{"timestamp":1709681418.8102629,"reason":"broker was unresponsive"},"724":{"timestamp":1709681418.8166225,"reason":"broker was unresponsive"},"725":{"timestamp":1709681418.8237262,"reason":"broker was unresponsive"},"726":{"timestamp":1709681418.8508961,"reason":"broker was unresponsive"},"727":{"timestamp":1709681418.883795,"reason":"broker was unresponsive"},"728":{"timestamp":1709681469.894057,"reason":"broker was unresponsive"},"729":{"timestamp":1709681470.0985878,"reason":"broker was unresponsive"},"730":{"timestamp":1709681470.9172387,"reason":"broker was unresponsive"},"731":{"timestamp":1709681470.950161,"reason":"broker was unresponsive"},"732":{"timestamp":1709681471.0538638,"reason":"broker was unresponsive"},"733":{"timestamp":1709681471.4289656,"reason":"broker was unresponsive"},"734":{"timestamp":1709681472.0564315,"reason":"broker was unresponsive"},"735":{"timestamp":1709681472.1440833,"reason":"broker was unresponsive"},"736":{"timestamp":1709681472.2474334,"reason":"broker was unresponsive"},"737":{"timestamp":1709681472.2672853,"reason":"broker was unresponsive"},"738":{"timestamp":1709681472.4184427,"reason":"broker was unresponsive"},"739":{"timestamp":1709681472.6889544,"reason":"broker was unresponsive"},"740":{"timestamp":1709681468.3484478,"reason":"broker was unresponsive"},"741":{"timestamp":1709681468.4152417,"reason":"broker was unresponsive"},"742":{"timestamp":1709681464.3254611,"reason":"broker was unresponsive"},"743":{"timestamp":1709681464.3341618,"reason":"broker was unresponsive"},"744":{"timestamp":1709681464.3459511,"reason":"broker was unresponsive"},"745":{"timestamp":1709681464.3592403,"reason":"broker was unresponsive"},"746":{"timestamp":1709681464.3835535,"reason":"broker was unresponsive"},"747":{"timestamp":1709681464.3917551,"reason":"broker was unresponsive"},"748":{"timestamp":1709681464.4060469,"reason":"broker was unresponsive"},"749":{"timestamp":1709681464.4340608,"reason":"broker was unresponsive"},"750":{"timestamp":1709681464.4442739,"reason":"broker was unresponsive"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"},"752":{"timestamp":1709681464.4573925,"reason":"broker was unresponsive"},"753":{"timestamp":1709681474.0712359,"reason":"broker was unresponsive"},"754":{"timestamp":1709681474.0774148,"reason":"broker was unresponsive"},"755":{"timestamp":1709681474.0830388,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1709684612.9962223,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709684649.918644,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709684650.7717795,"name":"online","context":{"idset":"1-4,8,24,30,45,47-49,54,59"}}
+{"timestamp":1709684650.8724031,"name":"online","context":{"idset":"18,28"}}
+{"timestamp":1709684651.2529552,"name":"online","context":{"idset":"254,282-285,287,289-290,293-294,300-302,304-305,307-312,314-317,320-389,391-403,405-409,411-412,414-444,446-468,471-479,481-488,490-493,495-503,505-511,513-526,528-529,531-546,548,551-558,560,562-568,570-573,575-578,580-594,596-617,619-622,624-630,632-658,660-679,681-685,687-688,690-694,696-702,704,706-708,710-712,714-717,719-723,726-729,731-743,745-755"}}
+{"timestamp":1709684651.3542054,"name":"online","context":{"idset":"273,288,295,299,313,318-319,390,404,410,413,445,469-470,480,489,494,504,512,527,530,547,549,559,561,574,579,595,618,623,631,659,680,686,689,695,703,705,709,713,718,724-725,730,744"}}
+{"timestamp":1709684651.7388618,"name":"online","context":{"idset":"78"}}
+{"timestamp":1709684652.2837186,"name":"online","context":{"idset":"61-62,65-66,68,70-74,76-77,80,83-84,87,89-90,92-98,101-107,109-112,115-116,118-120,122,124,126,128-133,135-139,141-148,150-151,153,156-157,159-171,173-178,180-182,184-185,187,189-200,207-211,217-219,221-225,228-229,231,233-235,237,239-242,244-245,248-252,255,257,261-262,264,266-272,274,277,279-280,291,296-298,303"}}
+{"timestamp":1709684652.384469,"name":"online","context":{"idset":"63-64,67,69,75,79,81-82,85-86,88,91,99-100,108,113-114,117,121,123,125,127,134,140,149,152,154-155,158,172,179,183,186,188,201-206,212,214-216,220,226-227,230,232,236,243,246-247,253,256,258-260,263,265,275-276,278,281,286,292,306"}}
+{"timestamp":1709684676.0138731,"name":"undrain","context":{"idset":"3-4,8,18,24,28,30,45,47-49,54,59,61-65,68-77,80-83,85,88,90-91,94,97,99,101,106-107,112-118,123-124,126-130,132,134,140,143,147,156,160,173-174,185,193,195,216,223,225,228,250,254,278,283,287,289-290,294-295,300,302,306,308,310,312-314,316-317,319,321-323,327,330-332,335,337-338,340,342-343,347,349,352-353,356,359,365,367-368,370-371,373-376,378-381,384-385,389,391,393-394,397-399,404,406,408-410,412,414-417,419-421,423,425-426,432,435-436,438-440,443-444,449-450,455,461,463-464,467-470,496,499-502,504,507-509,511-512,515-516,518-525,527-535,537-541,545,547-549,551-552,554-563,565,567-568,570-571,573-583,585-592,594-598,600,602-608,610-614,616-618,620-623,625,627-641,668-672,674-727"}}
+{"timestamp":1709684688.2943962,"name":"undrain","context":{"idset":"66-67,89,93,95,100,102-103,109-110,119,125,131,136-137,139,144-145,167,177,191,194,211,217,221,229,235-236,245,248-249,255,263-264,284-285,296,301,304-305,309,318,320,325,334,344,354,372,377,387-388,390,396,400-401,418,428,437,442,445,447,456,458-459,462,465,497-498,503,505-506,510,513-514,517,526,536,542-544,546,553,564,566,572,584,593,599,601,609,615,619,624,626,673"}}
+{"timestamp":1709684701.2056167,"name":"undrain","context":{"idset":"642-667,728-750,752-755"}}
+{"timestamp":1709696347.5038266,"name":"drain","context":{"idset":"232","reason":"reason=unresponsive","overwrite":0}}
+{"timestamp":1709698050.5847478,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"232":{"timestamp":1709696347.5038266,"reason":"reason=unresponsive"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"348":{"timestamp":1709343084.1450529,"reason":"reason=Lustre CHECKSUM errors"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"569":{"timestamp":1709583475.6141074,"reason":"prolog failed for jobid fgjQELtBquR"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"}},"online":"","exclude":"0"}}
+{"timestamp":1709698050.6113563,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709698085.1842389,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709698086.2215548,"name":"online","context":{"idset":"1,8,18,24,28,30,45,47-49,54,59"}}
+{"timestamp":1709698086.4903572,"name":"online","context":{"idset":"348,751-755"}}
+{"timestamp":1709698087.4984915,"name":"online","context":{"idset":"278"}}
+{"timestamp":1709698087.6571407,"name":"online","context":{"idset":"525"}}
+{"timestamp":1709698088.6205146,"name":"online","context":{"idset":"380,393,418,455,464,472,593"}}
+{"timestamp":1709698088.9963763,"name":"online","context":{"idset":"69,156,188,193,195,254,356,374,378,410,416,423,432,449,478-479,575,580,609,615,632,649,674,701,718,722"}}
+{"timestamp":1709698089.1300576,"name":"online","context":{"idset":"78,152,225,260-261,277,307,376,394,397,408,450,468,495,509,542,561,567,607,621,647,659,662,669,702,710,715,729,734,742"}}
+{"timestamp":1709698089.2104988,"name":"online","context":{"idset":"83,180,306,340,482,562,652,700"}}
+{"timestamp":1709698089.3108611,"name":"online","context":{"idset":"85,90,246,303-304,330,338,346,351,360,365,377,406,413,417,437,467,470,474,493,511,514,516,537,539,547,559,614,617,645,679,716,727,731-732"}}
+{"timestamp":1709698089.4145186,"name":"online","context":{"idset":"105,113,283,297,339,390,486,538,566,598"}}
+{"timestamp":1709698089.5169799,"name":"online","context":{"idset":"2,116,167,185,216,223,300,345,391,425,502,578,584,642,672,693,725"}}
+{"timestamp":1709698089.6175935,"name":"online","context":{"idset":"79,123-124,136,155,208,230,255,314,368,375,457,461,469,481,499,573,591,597,637,667-668,687,698,721,749"}}
+{"timestamp":1709698089.7239764,"name":"online","context":{"idset":"3,70,115,140,143-144,160,231,236,240,245,249-250,286,312,316,336,342,379,404,458,475,487-488,527,557,574,581,585-586,588-589,596,613,625,629,641,663,665,707,746"}}
+{"timestamp":1709698089.8244107,"name":"online","context":{"idset":"61,67,76,80,89,94,103,106,108,142,173-174,234,262-263,290,301-302,310,329,334,361,371,381,388,396,401,420,447,456,483,505,515,535,540-541,549,587,590,594,653,675,677,713,719,728"}}
+{"timestamp":1709698089.9268303,"name":"online","context":{"idset":"63,87,99,118,166,168,177,198,247,256,273,275,280,319,337,362,370,409,431,445,451,491,494,521-522,563,620,623,660,709,714,720,743"}}
+{"timestamp":1709698090.0288167,"name":"online","context":{"idset":"66,86,95,110,126,131,139,215,268,428,434-436,485,492,519,531-532,612,640,661"}}
+{"timestamp":1709698090.1348214,"name":"online","context":{"idset":"150,199,201,206,221,228,269,282,299,305,321,347,349-350,358,400,430,544,556,558,572,602,605,618,624,682,688,726,733"}}
+{"timestamp":1709698090.2533159,"name":"online","context":{"idset":"84,97,112,170,194,219,222,235,285,287,320,326,384,399,441,462,480,496,528,534,536,583,604,606,622,638-639,643,646,686"}}
+{"timestamp":1709698090.3704634,"name":"online","context":{"idset":"68,71,75,77,119,125,151,218,293-294,308,318,323,343-344,427,443,448,489,568,570,654,658,664,683"}}
+{"timestamp":1709698090.4709766,"name":"online","context":{"idset":"65,72-73,101,107,122,127,129,135,145-146,153,158,161,163,172,182,187,191,205,209,211,229,243,267,272,276,281,322,327,352,354,359,363,385,389,405,426,454,473,497,517,529,543,551-552,555,565,595,608,616,627,635,648,684,689,697,717,724,735,740-741"}}
+{"timestamp":1709698090.5752256,"name":"online","context":{"idset":"4,102,159,169,181,189,202,214,220,233,248,252,257,264,266,271,296,364,367,373,402,415,439,442,446,459,471,476,501,506,526,533,545,548,592,610-611,619,631,633,636,666,678,695,699,706,745"}}
+{"timestamp":1709698090.6789839,"name":"online","context":{"idset":"82,133,138,154,157,162,164,184,197,217,224,239,241,251,253,265,279,284,291-292,328,440,463,465,508,530,564,571,708,747-748"}}
+{"timestamp":1709698090.7835851,"name":"online","context":{"idset":"62,92,96,98,104,120,132,147,171,200,210,313,332,353,357,369,398,453,498,512,520,524,560,634,650,657,681,690,712,723,750"}}
+{"timestamp":1709698090.8870151,"name":"online","context":{"idset":"64,88,100,130,148,165,183,186,192,227,244,259,288,295,325,341,366,372,383,386-387,414,429,477,510,518,553,576,582,644,685,696,736,739"}}
+{"timestamp":1709698091.0046718,"name":"online","context":{"idset":"204,331,382,411-412,438,452,500,600,673,692,705,730,738"}}
+{"timestamp":1709698091.1095848,"name":"online","context":{"idset":"93,109,121,175-176,190,196,207,242,289,395,403,419,424,490,579,603,711,737,744"}}
+{"timestamp":1709698091.2156334,"name":"online","context":{"idset":"74,81,111,117,141,149,270,309,333,433,444,546,577,601,628,651,656,671,691"}}
+{"timestamp":1709698091.319186,"name":"online","context":{"idset":"91,137,179,203,274,315,324,335,355,460,484,507,513,523,630,655,694,703"}}
+{"timestamp":1709698091.4388881,"name":"online","context":{"idset":"114,178,226,237,258,298,311,392,407,421-422,503,626,670,676,680,704"}}
+{"timestamp":1709698091.5435405,"name":"online","context":{"idset":"212,317,554"}}
+{"timestamp":1709698091.7345405,"name":"online","context":{"idset":"134,466,504"}}
+{"timestamp":1709698092.0391197,"name":"online","context":{"idset":"128,599"}}
+{"timestamp":1709739157.4391561,"name":"online","context":{"idset":"232"}}
+{"timestamp":1709751296.5399144,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1709751310.8328795,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1709751390.5275269,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1709751409.4151535,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1709751436.0459356,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1709751442.5904524,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1709751458.5037024,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1709751465.6899984,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1709751472.288692,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1709751773.3627174,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1709751773.7541974,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1709751773.8108532,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1709751773.8569567,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1709751773.9571419,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1709751773.9967637,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1709751774.0048873,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1709751774.0212448,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1709751774.036994,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1709751774.0848653,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1709751774.0955868,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1709751774.1008003,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1709751774.1486597,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1709751774.1573079,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1709751774.1615272,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1709751774.1928329,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1709751774.2182853,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1709751774.2230253,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1709751774.2286415,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1709751774.318166,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1709751774.3941748,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1709751774.4443939,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1709751774.4498119,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1709751774.550765,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1709753012.3897939,"name":"online","context":{"idset":"201,213,226,373,460,581-582,587,713,715"}}
+{"timestamp":1709753012.5020258,"name":"online","context":{"idset":"221,247,305,343,355,455,459,565,567,572,583-584,637"}}
+{"timestamp":1709753012.656373,"name":"online","context":{"idset":"199"}}
+{"timestamp":1709753220.0420954,"name":"online","context":{"idset":"69,73,87,95,165,171-172"}}
+{"timestamp":1709753220.1512847,"name":"online","context":{"idset":"166,170"}}
+{"timestamp":1709753268.6533275,"name":"undrain","context":{"idset":"348"}}
+{"timestamp":1709761076.2339444,"name":"drain","context":{"idset":"348","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1709762566.3530021,"name":"drain","context":{"idset":"348","reason":"reason=BAD CHECKSUM","overwrite":1}}
+{"timestamp":1709766282.6244557,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1709784425.5568194,"name":"drain","context":{"idset":"208,302","reason":"reason=Unable to log in","overwrite":0}}
+{"timestamp":1709784507.6836026,"name":"drain","context":{"idset":"262","reason":"reason=Unable to log in","overwrite":0}}
+{"timestamp":1709784667.5992875,"name":"drain","context":{"idset":"103","reason":"reason=Unable to log in","overwrite":0}}
+{"timestamp":1709784694.554915,"name":"drain","context":{"idset":"301","reason":"reason=Unable to log in","overwrite":0}}
+{"timestamp":1709784700.3730869,"name":"undrain","context":{"idset":"302"}}
+{"timestamp":1709784722.3057358,"name":"drain","context":{"idset":"756","reason":"reason=Unable to log in","overwrite":0}}
+{"timestamp":1709820070.6714931,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1709820070.987385,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1709820071.5233355,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1709820076.9918146,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1709821866.4556227,"name":"drain","context":{"idset":"663","reason":"broker was unresponsive"}}
+{"timestamp":1709821932.4547484,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1709822428.8666167,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1709822429.459641,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1709822429.7567563,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1709822431.1618006,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1709822467.9651554,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1709822468.3658478,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1709822468.447799,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1709822468.5345132,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1709822468.6338489,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1709822468.8784926,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1709822468.9430225,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1709822469.0437121,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1709822469.3037643,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1709822469.3214631,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1709822469.3567169,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1709822469.3803942,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1709822469.4668434,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1709822469.5028551,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1709822469.5178401,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1709822469.5837152,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1709822469.6091311,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1709822469.6255922,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1709822469.6329062,"name":"offline","context":{"idset":"121"}}
+{"timestamp":1709822469.7379851,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1709822469.9063766,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1709822469.9587905,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1709822470.0594296,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1709822470.3218896,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1709822470.3887086,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1709822470.4897377,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1709822470.5986655,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1709822470.7086079,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1709822470.9345529,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1709822471.0097194,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1709822471.0222812,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1709822471.1222386,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1709823367.5547183,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1709823367.990545,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1709823368.5067234,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1709823368.6024687,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1709823368.6788373,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1709823368.6887665,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1709823368.7889323,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1709823368.8790762,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1709823368.9329748,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1709823368.9642146,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1709823368.9863408,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1709823369.0867741,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1709823369.1442835,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1709823369.1741989,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1709823369.2189527,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1709823369.2256074,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1709823369.3215778,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1709823369.3709478,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1709823369.3911572,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1709823369.4070957,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1709823369.5084732,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1709823369.6936996,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1709823369.7079587,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1709823369.8084805,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1709823370.1968927,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1709823370.28936,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1709823370.3685074,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1709823370.4258432,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1709823370.5264871,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1709823370.7693918,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1709823371.1281276,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1709823371.4299116,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1709823801.7666874,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1709823801.7778182,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1709823801.8780363,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1709823801.9369664,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1709823801.9582028,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1709823802.0592275,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1709823802.5151067,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1709823802.7925854,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1709823802.8425066,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1709823802.849371,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1709823802.8828428,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1709823802.9760804,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1709823803.0257676,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1709823803.0651822,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1709823803.0734289,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1709823803.1736424,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1709823803.2934523,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1709823803.3219523,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1709823803.4233794,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1709823803.5116906,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1709823803.528254,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1709823803.5768156,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1709823803.5861702,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1709823803.6771345,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1709823804.0542817,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1709823804.0818748,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1709823804.1823039,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1709823804.2313793,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1709823804.3301911,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1709823804.3688822,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1709823804.4729002,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1709823804.5736508,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1709823804.6725621,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1709823804.8795562,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1709823805.6184916,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1709823806.0209782,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1709823891.8092244,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1709823892.4689991,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1709823892.5618157,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1709823892.5848415,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1709823892.5951619,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1709823892.6037755,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1709823892.6340563,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1709823892.6469116,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1709823892.7475917,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1709823892.7998972,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1709823892.8414941,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1709823892.9422991,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1709823893.0333591,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1709823893.0452969,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1709823893.1338284,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1709823893.1420743,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1709823893.1683059,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1709823893.2163851,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1709823893.2668412,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1709823893.272048,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1709823893.3219292,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1709823893.42296,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1709823893.6206567,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1709823893.7324049,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1709823894.0074308,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1709823894.030021,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1709823894.1283989,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1709823894.1853333,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1709823894.2300174,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1709823894.249217,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1709823894.3495109,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1709823894.4116974,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1709823894.5113966,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1709823895.4088495,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1709823895.5662735,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1709823895.6667867,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1709823981.3585618,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1709823981.545151,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1709823981.8686569,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1709823982.2846351,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1709823982.3705294,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1709823982.3937516,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1709823982.4937208,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1709823982.601644,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1709823982.6899934,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1709823982.7272103,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1709823982.8270411,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1709823983.0631666,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1709823983.2050483,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1709823983.3059046,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1709823983.3768682,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1709823983.3812969,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1709823983.4814456,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1709823983.5071497,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1709823983.5154772,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1709823983.5539193,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1709823983.6099327,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1709823983.6732469,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1709823983.6767945,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1709823983.6976166,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1709823983.7007728,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1709823983.7547965,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1709823983.8558278,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1709823984.1095014,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1709823984.3061395,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1709823984.564575,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1709823984.7620108,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1709823985.0969405,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1709823985.1970844,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1709823985.3580482,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1709823985.4581258,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1709823986.0262268,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1709824072.0758779,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1709824072.1741097,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1709824072.266984,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1709824072.4167371,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1709824072.5167918,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1709824072.639761,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1709824072.6494286,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1709824072.6954956,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1709824072.7016037,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1709824072.7364924,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1709824072.7425501,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1709824072.7485769,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1709824072.8221791,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1709824072.8714936,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1709824072.8763385,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1709824072.9672396,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1709824072.9837267,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1709824073.0840642,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1709824073.1573708,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1709824073.1873908,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1709824073.2883286,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1709824073.3635364,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1709824073.4512739,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1709824073.5063412,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1709824073.5400474,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1709824073.6392388,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1709824073.8098588,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1709824073.9358628,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1709824074.2746136,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1709824074.4890742,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1709824074.6667242,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1709824074.8872595,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1709824074.9216142,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1709824075.0032971,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1709824075.1046784,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1709824075.5604603,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1709824161.8841166,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1709824161.9983771,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1709824162.3258083,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1709824162.4498286,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1709824162.6361623,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1709824162.6499348,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1709824162.750212,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1709824162.8599453,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1709824162.9667368,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1709824163.067157,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1709824163.2108638,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1709824163.267848,"name":"offline","context":{"idset":"451"}}
+{"timestamp":1709824163.3026669,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1709824163.3929546,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1709824163.4590452,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1709824163.4708071,"name":"offline","context":{"idset":"447"}}
+{"timestamp":1709824163.4961028,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1709824163.6115429,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1709824163.7013848,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1709824163.7800515,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1709824163.8011513,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1709824163.8258514,"name":"offline","context":{"idset":"453"}}
+{"timestamp":1709824163.8332798,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1709824163.9299767,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1709824163.9539576,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1709824164.0542023,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1709824164.23699,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1709824164.3931749,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1709824164.6377237,"name":"offline","context":{"idset":"457"}}
+{"timestamp":1709824164.7784023,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1709824164.8097658,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1709824164.9104993,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1709824165.1555929,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1709824165.2479124,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1709824165.3487108,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1709824165.6541576,"name":"offline","context":{"idset":"445"}}
+{"timestamp":1709824224.4543412,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1709824252.4025259,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1709824252.4654341,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1709824252.4852414,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1709824252.5856714,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1709824252.7263968,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1709824252.8965566,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1709824252.9436455,"name":"offline","context":{"idset":"555"}}
+{"timestamp":1709824252.9512482,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1709824252.9739428,"name":"offline","context":{"idset":"549"}}
+{"timestamp":1709824253.0648727,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1709824253.1500549,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1709824253.1563206,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1709824253.2568607,"name":"offline","context":{"idset":"553"}}
+{"timestamp":1709824253.4657292,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1709824253.5458515,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1709824253.601836,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1709824253.6264145,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1709824253.6488953,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1709824253.6812849,"name":"offline","context":{"idset":"561"}}
+{"timestamp":1709824253.6888328,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1709824253.7278223,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1709824253.8288808,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1709824253.9258716,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1709824254.0238097,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1709824254.1527812,"name":"offline","context":{"idset":"559"}}
+{"timestamp":1709824254.15904,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1709824254.2617188,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1709824254.4464698,"name":"offline","context":{"idset":"551"}}
+{"timestamp":1709824254.531919,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1709824254.6319315,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1709824254.7375491,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1709824255.1933384,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1709824255.33761,"name":"offline","context":{"idset":"563"}}
+{"timestamp":1709824255.5860255,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1709824256.0197394,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1709824256.4353552,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1709824290.4539196,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1709824318.4545081,"name":"drain","context":{"idset":"532","reason":"broker was unresponsive"}}
+{"timestamp":1709824342.1986244,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1709824342.321981,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1709824342.3266056,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1709824342.4270105,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1709824342.4754457,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1709824342.5762873,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1709824342.8429456,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1709824342.9007392,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1709824342.9543018,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1709824343.0547297,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1709824343.1486044,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1709824343.2484314,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1709824343.3811481,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1709824343.4815605,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1709824343.614938,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1709824343.7194843,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1709824343.7254009,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1709824343.7472692,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1709824343.7640176,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1709824343.8154862,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1709824343.8440919,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1709824343.877605,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1709824343.9206305,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1709824344.0196795,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1709824344.1868758,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1709824344.1905124,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1709824344.1952577,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1709824344.2537034,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1709824344.2590358,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1709824344.2917116,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1709824344.366966,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1709824344.4667611,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1709824344.918736,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1709824345.3639684,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1709824345.7637284,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1709824382.4535284,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1709824432.0129695,"name":"offline","context":{"idset":"655"}}
+{"timestamp":1709824432.4390311,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1709824432.5475175,"name":"offline","context":{"idset":"681"}}
+{"timestamp":1709824432.6473093,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1709824432.7394092,"name":"offline","context":{"idset":"665"}}
+{"timestamp":1709824432.745755,"name":"offline","context":{"idset":"657"}}
+{"timestamp":1709824432.8455105,"name":"offline","context":{"idset":"679"}}
+{"timestamp":1709824432.9031446,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1709824432.9490356,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1709824432.9699504,"name":"offline","context":{"idset":"661"}}
+{"timestamp":1709824433.000412,"name":"offline","context":{"idset":"667"}}
+{"timestamp":1709824433.1006231,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1709824433.2926154,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1709824433.313803,"name":"offline","context":{"idset":"683"}}
+{"timestamp":1709824433.362972,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1709824433.368901,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1709824433.4690003,"name":"offline","context":{"idset":"707"}}
+{"timestamp":1709824433.5911794,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1709824433.6672633,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1709824433.7528436,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1709824433.8288422,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1709824433.8808005,"name":"offline","context":{"idset":"675"}}
+{"timestamp":1709824433.8847754,"name":"offline","context":{"idset":"701"}}
+{"timestamp":1709824433.9188142,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1709824434.0169082,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1709824434.1753788,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1709824434.2352607,"name":"offline","context":{"idset":"685"}}
+{"timestamp":1709824434.2433896,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1709824434.3446629,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1709824434.3999181,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1709824434.434721,"name":"offline","context":{"idset":"677"}}
+{"timestamp":1709824434.5349205,"name":"offline","context":{"idset":"671"}}
+{"timestamp":1709824434.6806898,"name":"offline","context":{"idset":"669"}}
+{"timestamp":1709824435.6526525,"name":"offline","context":{"idset":"659"}}
+{"timestamp":1709824436.0670311,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1709824522.2894378,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1709824522.3900619,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1709824522.5189824,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1709824522.6196814,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1709824522.7816782,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1709824522.8817041,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1709824523.0207984,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1709824523.0847406,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1709824523.1850679,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1709824523.3695445,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1709824523.4157972,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1709824523.4176583,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1709824523.5179513,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1709824523.5901828,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1709824523.6909182,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1709824524.1695974,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1709824524.269733,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1709824524.4373057,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1709824524.6763241,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1709824524.722028,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1709824524.8123746,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1709824524.8426478,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1709824524.9438972,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1709824588.4548297,"name":"drain","context":{"idset":"710","reason":"broker was unresponsive"}}
+{"timestamp":1709824650.4550178,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1709824701.9294143,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1709824702.2819774,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1709824702.4493794,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1709824702.700685,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1709824702.8007009,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1709824702.8488894,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1709824702.8927357,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1709824702.917836,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1709824702.9449806,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1709824702.9498415,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1709824702.9881454,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1709824703.0885277,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1709824703.1664677,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1709824703.2322109,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1709824703.2678668,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1709824703.285351,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1709824703.2978709,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1709824703.3118594,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1709824703.4122722,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1709824703.4980624,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1709824703.6024883,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1709824703.644453,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1709824703.6525593,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1709824703.7536879,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1709824703.8306382,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1709824703.9102843,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1709824703.9279366,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1709824704.0265839,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1709824704.1815088,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1709824704.2212026,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1709824704.3216114,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1709824704.466053,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1709824705.008656,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1709824705.0201218,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1709824705.119962,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1709824705.7699606,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1709824791.8436069,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1709824792.2489836,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1709824792.46714,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1709824792.5665057,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1709824792.6346524,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1709824792.6394162,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1709824792.6652215,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1709824792.765836,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1709824792.9436214,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1709824792.9720709,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1709824792.982398,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1709824793.0830123,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1709824793.1061404,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1709824793.1413805,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1709824793.1694143,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1709824793.223825,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1709824793.2630761,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1709824793.2967074,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1709824793.3976719,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1709824793.4404576,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1709824793.4632335,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1709824793.5044916,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1709824793.6066263,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1709824793.6826701,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1709824793.7834332,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1709824793.9428487,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1709824794.0427434,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1709824794.1401448,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1709824794.2400234,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1709824794.3847075,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1709824794.4898148,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1709824794.69327,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1709824794.9403863,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1709824795.0407009,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1709824795.1620777,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1709824881.7736576,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1709824882.320668,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1709824882.42097,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1709824882.5658729,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1709824882.5925057,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1709824882.6356223,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1709824882.6390445,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1709824882.7394571,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1709824882.9026561,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1709824882.913192,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1709824882.9496841,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1709824882.9656696,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1709824883.0017464,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1709824883.0531771,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1709824883.1540425,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1709824883.2511981,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1709824883.3425758,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1709824883.4428837,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1709824883.5507331,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1709824883.5834539,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1709824883.6848104,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1709824883.739543,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1709824883.8385072,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1709824883.9097173,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1709824883.9405906,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1709824884.0409834,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1709824884.0853777,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1709824884.1843486,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1709824884.4209907,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1709824884.483413,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1709824884.5605419,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1709824884.6613593,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1709824884.8893549,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1709824884.9900768,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1709824885.3977649,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1709824972.1066368,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1709824972.4659877,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1709824972.5667462,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1709824972.7834463,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1709824972.9149621,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1709824973.0289509,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1709824973.0964439,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1709824973.14133,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1709824973.2420297,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1709824973.3287213,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1709824973.428669,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1709824973.4759548,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1709824973.5313146,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1709824973.5869637,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1709824973.605581,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1709824973.6226075,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1709824973.639956,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1709824973.7135637,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1709824973.8146598,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1709824973.9016075,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1709824973.9125819,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1709824973.9165828,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1709824974.0125539,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1709824974.086782,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1709824974.1193783,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1709824974.2089062,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1709824974.2458665,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1709824974.3001266,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1709824974.4057574,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1709824974.6047883,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1709824974.7048163,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1709824974.8211141,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1709824974.9360104,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1709824975.6938198,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1709824976.1319046,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1709825038.454864,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1709825062.1297214,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1709825062.2042811,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1709825062.2147391,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1709825062.2446835,"name":"offline","context":{"idset":"456"}}
+{"timestamp":1709825062.2475274,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1709825062.3486197,"name":"offline","context":{"idset":"452"}}
+{"timestamp":1709825062.6924767,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1709825062.7873845,"name":"offline","context":{"idset":"454"}}
+{"timestamp":1709825062.7946329,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1709825062.8953633,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1709825063.009552,"name":"offline","context":{"idset":"464"}}
+{"timestamp":1709825063.1177676,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1709825063.183342,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1709825063.2837336,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1709825063.3179312,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1709825063.4180839,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1709825063.4736991,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1709825063.4974697,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1709825063.5482175,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1709825063.5842736,"name":"offline","context":{"idset":"466"}}
+{"timestamp":1709825063.659128,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1709825063.6664674,"name":"offline","context":{"idset":"450"}}
+{"timestamp":1709825063.7103696,"name":"offline","context":{"idset":"462"}}
+{"timestamp":1709825063.7288902,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1709825063.7462871,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1709825063.7943785,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1709825063.8644893,"name":"offline","context":{"idset":"468"}}
+{"timestamp":1709825063.9618905,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1709825063.9694016,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1709825063.9809611,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1709825064.0062501,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1709825064.0644705,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1709825064.1646614,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1709825064.9892139,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1709825066.1216569,"name":"offline","context":{"idset":"448"}}
+{"timestamp":1709825100.4535391,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1709825151.776232,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1709825151.9411364,"name":"offline","context":{"idset":"556"}}
+{"timestamp":1709825152.0410988,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1709825152.5399451,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1709825152.6403236,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1709825152.8430858,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1709825152.9437885,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1709825153.05639,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1709825153.0950978,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1709825153.1051772,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1709825153.1119392,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1709825153.1862626,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1709825153.2288144,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1709825153.2642019,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1709825153.2684054,"name":"offline","context":{"idset":"552"}}
+{"timestamp":1709825153.3176084,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1709825153.3292744,"name":"offline","context":{"idset":"554"}}
+{"timestamp":1709825153.4305303,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1709825153.6201842,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1709825153.6497371,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1709825153.6512475,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1709825153.727613,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1709825153.772754,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1709825153.8052998,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1709825153.9047356,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1709825154.1140919,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1709825154.3383296,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1709825154.3659549,"name":"offline","context":{"idset":"558"}}
+{"timestamp":1709825154.4667568,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1709825154.5392778,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1709825154.6395693,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1709825155.0998199,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1709825155.2068136,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1709825156.5341582,"name":"offline","context":{"idset":"564"}}
+{"timestamp":1709825242.0058954,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1709825242.2127144,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1709825242.3208752,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1709825242.3993342,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1709825242.4071834,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1709825242.4491925,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1709825242.5284257,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1709825242.5302529,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1709825242.5664644,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1709825242.5991967,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1709825242.6993976,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1709825242.9429593,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1709825242.977936,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1709825242.989428,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1709825243.0897386,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1709825243.1756771,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1709825243.2492075,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1709825243.2716596,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1709825243.2876127,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1709825243.3469894,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1709825243.4475076,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1709825243.5556273,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1709825243.6107421,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1709825243.7080965,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1709825243.7534239,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1709825243.8536239,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1709825243.985225,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1709825244.0839155,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1709825244.2514584,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1709825244.7261164,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1709825245.005538,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1709825245.103272,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1709825245.8484199,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1709825245.8497629,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1709825245.9499664,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1709825246.1529825,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1709825331.5717065,"name":"offline","context":{"idset":"692"}}
+{"timestamp":1709825331.867656,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1709825331.8757572,"name":"offline","context":{"idset":"690"}}
+{"timestamp":1709825331.9759953,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1709825332.4496555,"name":"offline","context":{"idset":"658"}}
+{"timestamp":1709825332.5691497,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1709825332.59183,"name":"offline","context":{"idset":"656"}}
+{"timestamp":1709825332.5953765,"name":"offline","context":{"idset":"672"}}
+{"timestamp":1709825332.6503229,"name":"offline","context":{"idset":"708"}}
+{"timestamp":1709825332.6678326,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1709825332.695487,"name":"offline","context":{"idset":"694"}}
+{"timestamp":1709825332.7779047,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1709825332.8296916,"name":"offline","context":{"idset":"666"}}
+{"timestamp":1709825332.9296162,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1709825333.1538484,"name":"offline","context":{"idset":"680"}}
+{"timestamp":1709825333.1864514,"name":"offline","context":{"idset":"670"}}
+{"timestamp":1709825333.202728,"name":"offline","context":{"idset":"662"}}
+{"timestamp":1709825333.2188947,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1709825333.2772224,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1709825333.2830997,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1709825333.3832059,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1709825333.4635153,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1709825333.4725459,"name":"offline","context":{"idset":"654"}}
+{"timestamp":1709825333.5134523,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1709825333.6135316,"name":"offline","context":{"idset":"660"}}
+{"timestamp":1709825333.7297788,"name":"offline","context":{"idset":"676"}}
+{"timestamp":1709825333.8212667,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1709825333.8645411,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1709825333.9157174,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1709825333.9676635,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1709825334.0675621,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1709825334.249819,"name":"offline","context":{"idset":"706"}}
+{"timestamp":1709825334.3081684,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1709825334.4091797,"name":"offline","context":{"idset":"682"}}
+{"timestamp":1709825334.96068,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1709825335.3425596,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1709825421.9042683,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1709825422.5674331,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1709825422.5883067,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1709825422.6890326,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1709825423.0198414,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1709825423.1232662,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1709825423.2233655,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1709825423.3633907,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1709825423.4763446,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1709825423.6357818,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1709825423.7365015,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1709825423.7718861,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1709825423.792129,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1709825423.8022625,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1709825423.9025128,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1709825424.0943973,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1709825424.1043677,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1709825424.1126266,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1709825424.2131963,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1709825424.3305674,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1709825424.6443479,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1709825424.7216794,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1709849900.2906215,"name":"online","context":{"idset":"69,73-74"}}
+{"timestamp":1709849900.4961162,"name":"online","context":{"idset":"71"}}
+{"timestamp":1709849900.6212685,"name":"online","context":{"idset":"75"}}
+{"timestamp":1709849900.8070099,"name":"online","context":{"idset":"72"}}
+{"timestamp":1709849901.4149001,"name":"online","context":{"idset":"79"}}
+{"timestamp":1709849901.4319882,"name":"online","context":{"idset":"80"}}
+{"timestamp":1709849901.5805116,"name":"online","context":{"idset":"102"}}
+{"timestamp":1709849901.7335408,"name":"online","context":{"idset":"76,82,84,87,92,94-95,100,108"}}
+{"timestamp":1709849901.8403621,"name":"online","context":{"idset":"70,77,83,89,91,98,104-105,114,119,123,148"}}
+{"timestamp":1709849901.9643643,"name":"online","context":{"idset":"93,120,128,142,146,155,161,170"}}
+{"timestamp":1709849902.0694513,"name":"online","context":{"idset":"81,96,103,107,113,124,136,143,145,149,153,167-168"}}
+{"timestamp":1709849902.1863854,"name":"online","context":{"idset":"68,115,117,121-122,129,138,140-141,147,151,157,160,165,171"}}
+{"timestamp":1709849902.2890422,"name":"online","context":{"idset":"61,86,106,109,116,132,139,169,172,176,178"}}
+{"timestamp":1709849902.2960486,"name":"online","context":{"idset":"125-127"}}
+{"timestamp":1709849902.3986037,"name":"online","context":{"idset":"85,88,111,130,133-134,137,150,152,154,158-159,163-164,174-175,177,179-180"}}
+{"timestamp":1709849902.5007572,"name":"online","context":{"idset":"112,118"}}
+{"timestamp":1709849902.612114,"name":"online","context":{"idset":"78,90,97,101,110,131"}}
+{"timestamp":1709849903.2308221,"name":"online","context":{"idset":"64"}}
+{"timestamp":1709849903.9408946,"name":"online","context":{"idset":"66-67,188"}}
+{"timestamp":1709849904.138952,"name":"online","context":{"idset":"156,166"}}
+{"timestamp":1709849904.2370875,"name":"online","context":{"idset":"173,181"}}
+{"timestamp":1709849904.353411,"name":"online","context":{"idset":"62,184"}}
+{"timestamp":1709849905.1802301,"name":"online","context":{"idset":"187"}}
+{"timestamp":1709849906.130564,"name":"online","context":{"idset":"65,183"}}
+{"timestamp":1709849906.9045074,"name":"online","context":{"idset":"144,162"}}
+{"timestamp":1709849907.2168179,"name":"online","context":{"idset":"182"}}
+{"timestamp":1709849907.7718842,"name":"online","context":{"idset":"63"}}
+{"timestamp":1709849908.2912014,"name":"online","context":{"idset":"185"}}
+{"timestamp":1709849908.4070997,"name":"online","context":{"idset":"135"}}
+{"timestamp":1709849908.8211806,"name":"online","context":{"idset":"186"}}
+{"timestamp":1709849909.6344759,"name":"online","context":{"idset":"189"}}
+{"timestamp":1709849909.8362007,"name":"online","context":{"idset":"190-191"}}
+{"timestamp":1709849909.9762652,"name":"online","context":{"idset":"192"}}
+{"timestamp":1709849910.0953522,"name":"online","context":{"idset":"193"}}
+{"timestamp":1709849910.3448012,"name":"online","context":{"idset":"194"}}
+{"timestamp":1709849911.2710259,"name":"online","context":{"idset":"196"}}
+{"timestamp":1709849911.3943751,"name":"online","context":{"idset":"195"}}
+{"timestamp":1709849911.475919,"name":"online","context":{"idset":"197"}}
+{"timestamp":1709849911.4794967,"name":"online","context":{"idset":"198"}}
+{"timestamp":1709849911.5414662,"name":"online","context":{"idset":"199"}}
+{"timestamp":1709849911.554208,"name":"online","context":{"idset":"201"}}
+{"timestamp":1709849911.5905247,"name":"online","context":{"idset":"206"}}
+{"timestamp":1709849911.6077838,"name":"online","context":{"idset":"200"}}
+{"timestamp":1709849911.6529298,"name":"online","context":{"idset":"209"}}
+{"timestamp":1709849911.6910357,"name":"online","context":{"idset":"211"}}
+{"timestamp":1709849911.7399113,"name":"online","context":{"idset":"202-203,207"}}
+{"timestamp":1709849911.7854857,"name":"online","context":{"idset":"204-205,213,215"}}
+{"timestamp":1709849911.8015482,"name":"online","context":{"idset":"208"}}
+{"timestamp":1709849911.8558774,"name":"online","context":{"idset":"223"}}
+{"timestamp":1709849911.8715014,"name":"online","context":{"idset":"214"}}
+{"timestamp":1709849911.9857862,"name":"online","context":{"idset":"216"}}
+{"timestamp":1709849912.1073861,"name":"online","context":{"idset":"210,212,217-218,220,230-231"}}
+{"timestamp":1709849912.2124546,"name":"online","context":{"idset":"219,221,232-233,237,240,259"}}
+{"timestamp":1709849912.3164685,"name":"online","context":{"idset":"222,225,227,229,235,238,241,258,260,275"}}
+{"timestamp":1709849912.4305217,"name":"online","context":{"idset":"226,228,236,239,243,245,247-248,250,262,264,267"}}
+{"timestamp":1709849912.5294993,"name":"online","context":{"idset":"224,244,249,252-253,255,257,261,266,268,279-280,283-285,287-288,290"}}
+{"timestamp":1709849912.6410773,"name":"online","context":{"idset":"234,242,246,251,254,256,263,265,269,272,277-278,281-282,286,292,296"}}
+{"timestamp":1709849912.6734676,"name":"online","context":{"idset":"273,276"}}
+{"timestamp":1709849912.7823291,"name":"online","context":{"idset":"270-271,274,289,293-295"}}
+{"timestamp":1709849912.8339212,"name":"online","context":{"idset":"291"}}
+{"timestamp":1709849912.9030719,"name":"online","context":{"idset":"297"}}
+{"timestamp":1709849913.6616521,"name":"online","context":{"idset":"299-300"}}
+{"timestamp":1709849913.8060558,"name":"online","context":{"idset":"298,301"}}
+{"timestamp":1709849913.9269164,"name":"online","context":{"idset":"302-303"}}
+{"timestamp":1709849914.0036783,"name":"online","context":{"idset":"305-306"}}
+{"timestamp":1709849914.1203144,"name":"online","context":{"idset":"304"}}
+{"timestamp":1709849915.0296504,"name":"online","context":{"idset":"307"}}
+{"timestamp":1709849915.8358068,"name":"online","context":{"idset":"308-309"}}
+{"timestamp":1709849916.4817502,"name":"online","context":{"idset":"310"}}
+{"timestamp":1709849916.6220117,"name":"online","context":{"idset":"311"}}
+{"timestamp":1709849916.8583496,"name":"online","context":{"idset":"312"}}
+{"timestamp":1709849917.4827495,"name":"online","context":{"idset":"313"}}
+{"timestamp":1709849917.9807453,"name":"online","context":{"idset":"314"}}
+{"timestamp":1709849918.1345425,"name":"online","context":{"idset":"315"}}
+{"timestamp":1709849918.5699944,"name":"online","context":{"idset":"316"}}
+{"timestamp":1709849919.6029282,"name":"online","context":{"idset":"318"}}
+{"timestamp":1709849919.8393281,"name":"online","context":{"idset":"321"}}
+{"timestamp":1709849921.2128227,"name":"online","context":{"idset":"319,326-327"}}
+{"timestamp":1709849921.3121822,"name":"online","context":{"idset":"317"}}
+{"timestamp":1709849921.3981774,"name":"online","context":{"idset":"325"}}
+{"timestamp":1709849921.4294124,"name":"online","context":{"idset":"328"}}
+{"timestamp":1709849921.444562,"name":"online","context":{"idset":"329-330"}}
+{"timestamp":1709849921.4961455,"name":"online","context":{"idset":"331"}}
+{"timestamp":1709849921.502599,"name":"online","context":{"idset":"333,337"}}
+{"timestamp":1709849921.5418108,"name":"online","context":{"idset":"335-336"}}
+{"timestamp":1709849921.5695107,"name":"online","context":{"idset":"332,334"}}
+{"timestamp":1709849921.5901575,"name":"online","context":{"idset":"338-339"}}
+{"timestamp":1709849921.6210172,"name":"online","context":{"idset":"341"}}
+{"timestamp":1709849921.658417,"name":"online","context":{"idset":"342"}}
+{"timestamp":1709849921.7485027,"name":"online","context":{"idset":"340"}}
+{"timestamp":1709849921.8926182,"name":"online","context":{"idset":"345"}}
+{"timestamp":1709849921.9090812,"name":"online","context":{"idset":"344"}}
+{"timestamp":1709849921.9740903,"name":"online","context":{"idset":"349"}}
+{"timestamp":1709849922.1013546,"name":"online","context":{"idset":"346-347,350"}}
+{"timestamp":1709849922.2455044,"name":"online","context":{"idset":"324,348,352,355-356,362"}}
+{"timestamp":1709849922.3235533,"name":"online","context":{"idset":"354,372"}}
+{"timestamp":1709849922.3568177,"name":"online","context":{"idset":"357,366"}}
+{"timestamp":1709849922.3909919,"name":"online","context":{"idset":"351,358,360,373,375"}}
+{"timestamp":1709849922.3993847,"name":"online","context":{"idset":"379"}}
+{"timestamp":1709849922.4421389,"name":"online","context":{"idset":"361,363,365,367-368,374,376"}}
+{"timestamp":1709849922.4931707,"name":"online","context":{"idset":"322,370,392"}}
+{"timestamp":1709849922.5909657,"name":"online","context":{"idset":"359,364,369,380,384,386,388-389"}}
+{"timestamp":1709849922.6228359,"name":"online","context":{"idset":"377,391"}}
+{"timestamp":1709849922.7239747,"name":"online","context":{"idset":"320,381,383,385,393,395-397,399,401,407,410,412,414,424"}}
+{"timestamp":1709849922.7411182,"name":"online","context":{"idset":"394,404,417"}}
+{"timestamp":1709849922.7805536,"name":"online","context":{"idset":"387,406,408,415,418,420-423"}}
+{"timestamp":1709849922.9002714,"name":"online","context":{"idset":"390,398,402-403,411,413,416"}}
+{"timestamp":1709849923.0047011,"name":"online","context":{"idset":"409,419,425"}}
+{"timestamp":1709849923.1166079,"name":"online","context":{"idset":"353"}}
+{"timestamp":1709849923.3529379,"name":"online","context":{"idset":"343,378,382,400,405"}}
+{"timestamp":1709849923.4685481,"name":"online","context":{"idset":"371,426-427"}}
+{"timestamp":1709849923.6145475,"name":"online","context":{"idset":"428-431"}}
+{"timestamp":1709849923.7493234,"name":"online","context":{"idset":"432-434"}}
+{"timestamp":1709849924.585377,"name":"online","context":{"idset":"435"}}
+{"timestamp":1709849924.7848973,"name":"online","context":{"idset":"323"}}
+{"timestamp":1709849925.6063204,"name":"online","context":{"idset":"436"}}
+{"timestamp":1709849926.9332075,"name":"online","context":{"idset":"438"}}
+{"timestamp":1709849927.7700198,"name":"online","context":{"idset":"442"}}
+{"timestamp":1709849928.366611,"name":"online","context":{"idset":"444"}}
+{"timestamp":1709849929.7777584,"name":"online","context":{"idset":"440"}}
+{"timestamp":1709849930.5179565,"name":"online","context":{"idset":"441"}}
+{"timestamp":1709849931.0844386,"name":"online","context":{"idset":"439,446"}}
+{"timestamp":1709849931.2005568,"name":"online","context":{"idset":"449"}}
+{"timestamp":1709849931.2491302,"name":"online","context":{"idset":"453"}}
+{"timestamp":1709849931.3230598,"name":"online","context":{"idset":"462-463"}}
+{"timestamp":1709849931.379307,"name":"online","context":{"idset":"443,455,457-459"}}
+{"timestamp":1709849931.4305065,"name":"online","context":{"idset":"454"}}
+{"timestamp":1709849931.447664,"name":"online","context":{"idset":"464"}}
+{"timestamp":1709849931.4597027,"name":"online","context":{"idset":"465"}}
+{"timestamp":1709849931.5400994,"name":"online","context":{"idset":"437,461"}}
+{"timestamp":1709849931.5820541,"name":"online","context":{"idset":"466"}}
+{"timestamp":1709849932.020735,"name":"online","context":{"idset":"470"}}
+{"timestamp":1709849932.0797527,"name":"online","context":{"idset":"477"}}
+{"timestamp":1709849932.1096189,"name":"online","context":{"idset":"471"}}
+{"timestamp":1709849932.1798944,"name":"online","context":{"idset":"445"}}
+{"timestamp":1709849932.2144022,"name":"online","context":{"idset":"482"}}
+{"timestamp":1709849932.298424,"name":"online","context":{"idset":"478"}}
+{"timestamp":1709849932.3216214,"name":"online","context":{"idset":"479"}}
+{"timestamp":1709849932.3719563,"name":"online","context":{"idset":"483,486,494"}}
+{"timestamp":1709849932.4029567,"name":"online","context":{"idset":"476,481"}}
+{"timestamp":1709849932.4631484,"name":"online","context":{"idset":"487,489-490,492,498"}}
+{"timestamp":1709849932.5094967,"name":"online","context":{"idset":"501,506"}}
+{"timestamp":1709849932.6766045,"name":"online","context":{"idset":"493,497,516-517"}}
+{"timestamp":1709849932.7316468,"name":"online","context":{"idset":"460,499,508,532"}}
+{"timestamp":1709849932.7546358,"name":"online","context":{"idset":"531"}}
+{"timestamp":1709849932.7707491,"name":"online","context":{"idset":"512,524"}}
+{"timestamp":1709849932.7747111,"name":"online","context":{"idset":"496,526"}}
+{"timestamp":1709849932.8905759,"name":"online","context":{"idset":"491,503,505,510-511,513,520,523,527,534-535"}}
+{"timestamp":1709849932.9554415,"name":"online","context":{"idset":"515,519,521,533,540,542"}}
+{"timestamp":1709849933.0656674,"name":"online","context":{"idset":"502,528,530,536,538-539,541,544-545,547,549,551"}}
+{"timestamp":1709849933.170517,"name":"online","context":{"idset":"500,546,548,553-554"}}
+{"timestamp":1709849933.3225956,"name":"online","context":{"idset":"467,480,485,509,522,529,556-558"}}
+{"timestamp":1709849933.4322689,"name":"online","context":{"idset":"473,559-561"}}
+{"timestamp":1709849933.5397446,"name":"online","context":{"idset":"468,504"}}
+{"timestamp":1709849933.7074916,"name":"online","context":{"idset":"456,495,507,514,518,525,537,543"}}
+{"timestamp":1709849933.7959857,"name":"online","context":{"idset":"484,488"}}
+{"timestamp":1709849933.9499159,"name":"online","context":{"idset":"450"}}
+{"timestamp":1709849934.2010412,"name":"online","context":{"idset":"469"}}
+{"timestamp":1709849934.6465712,"name":"online","context":{"idset":"451,552"}}
+{"timestamp":1709849934.7471688,"name":"online","context":{"idset":"562-563"}}
+{"timestamp":1709849934.9223819,"name":"online","context":{"idset":"555"}}
+{"timestamp":1709849935.9424157,"name":"online","context":{"idset":"474-475,564"}}
+{"timestamp":1709849936.4238567,"name":"online","context":{"idset":"565"}}
+{"timestamp":1709849936.5889366,"name":"online","context":{"idset":"566"}}
+{"timestamp":1709849937.4256837,"name":"online","context":{"idset":"567"}}
+{"timestamp":1709849937.9706738,"name":"online","context":{"idset":"568"}}
+{"timestamp":1709849938.462729,"name":"online","context":{"idset":"448,472"}}
+{"timestamp":1709849939.7960522,"name":"online","context":{"idset":"569"}}
+{"timestamp":1709849940.0235677,"name":"online","context":{"idset":"452"}}
+{"timestamp":1709849940.2862103,"name":"online","context":{"idset":"570"}}
+{"timestamp":1709849940.879344,"name":"online","context":{"idset":"571"}}
+{"timestamp":1709849941.0422983,"name":"online","context":{"idset":"572,574"}}
+{"timestamp":1709849941.1869051,"name":"online","context":{"idset":"582"}}
+{"timestamp":1709849941.2244771,"name":"online","context":{"idset":"581,583"}}
+{"timestamp":1709849941.2964356,"name":"online","context":{"idset":"584"}}
+{"timestamp":1709849941.3595905,"name":"online","context":{"idset":"586"}}
+{"timestamp":1709849941.4378266,"name":"online","context":{"idset":"585"}}
+{"timestamp":1709849941.4620202,"name":"online","context":{"idset":"587"}}
+{"timestamp":1709849941.5978808,"name":"online","context":{"idset":"579"}}
+{"timestamp":1709849941.7258472,"name":"online","context":{"idset":"589"}}
+{"timestamp":1709849941.8369696,"name":"online","context":{"idset":"588"}}
+{"timestamp":1709849941.9816985,"name":"online","context":{"idset":"591"}}
+{"timestamp":1709849942.0497139,"name":"online","context":{"idset":"590"}}
+{"timestamp":1709849942.1309297,"name":"online","context":{"idset":"592"}}
+{"timestamp":1709849942.1588521,"name":"online","context":{"idset":"595,602"}}
+{"timestamp":1709849942.17858,"name":"online","context":{"idset":"597"}}
+{"timestamp":1709849942.2115884,"name":"online","context":{"idset":"593"}}
+{"timestamp":1709849942.2491138,"name":"online","context":{"idset":"599"}}
+{"timestamp":1709849942.280942,"name":"online","context":{"idset":"594"}}
+{"timestamp":1709849942.2897606,"name":"online","context":{"idset":"598,603,605"}}
+{"timestamp":1709849942.3511741,"name":"online","context":{"idset":"596,600,604"}}
+{"timestamp":1709849942.4297132,"name":"online","context":{"idset":"573,601,609"}}
+{"timestamp":1709849942.456136,"name":"online","context":{"idset":"606"}}
+{"timestamp":1709849942.4831622,"name":"online","context":{"idset":"608"}}
+{"timestamp":1709849942.5226536,"name":"online","context":{"idset":"607,610"}}
+{"timestamp":1709849942.5569744,"name":"online","context":{"idset":"613"}}
+{"timestamp":1709849942.5630231,"name":"online","context":{"idset":"611"}}
+{"timestamp":1709849942.7305,"name":"online","context":{"idset":"612,615"}}
+{"timestamp":1709849942.8301179,"name":"online","context":{"idset":"614,616-617,619,621-625,627,633"}}
+{"timestamp":1709849942.9350884,"name":"online","context":{"idset":"578,618,620,626,628-629,631-632,634-635,637-638,640,644"}}
+{"timestamp":1709849942.9398551,"name":"online","context":{"idset":"642"}}
+{"timestamp":1709849943.0522642,"name":"online","context":{"idset":"580,630,636,639,641,643,645,647,649,651,653-654"}}
+{"timestamp":1709849943.1589339,"name":"online","context":{"idset":"648,650,652,656-657,661"}}
+{"timestamp":1709849943.2794199,"name":"online","context":{"idset":"646,658-659,662,664,666"}}
+{"timestamp":1709849943.3390961,"name":"online","context":{"idset":"655,668"}}
+{"timestamp":1709849943.3943281,"name":"online","context":{"idset":"665,677"}}
+{"timestamp":1709849943.5124235,"name":"online","context":{"idset":"660,663,667,669,671-672,674-675,678"}}
+{"timestamp":1709849943.564599,"name":"online","context":{"idset":"670,676"}}
+{"timestamp":1709849943.6877682,"name":"online","context":{"idset":"673,679"}}
+{"timestamp":1709849943.9681883,"name":"online","context":{"idset":"680"}}
+{"timestamp":1709849944.3548913,"name":"online","context":{"idset":"681,683"}}
+{"timestamp":1709849944.5857918,"name":"online","context":{"idset":"684"}}
+{"timestamp":1709849944.7870889,"name":"online","context":{"idset":"685"}}
+{"timestamp":1709849945.111238,"name":"online","context":{"idset":"575,682"}}
+{"timestamp":1709849945.6070404,"name":"online","context":{"idset":"577,687"}}
+{"timestamp":1709849945.8029099,"name":"online","context":{"idset":"447,686,688"}}
+{"timestamp":1709849946.210201,"name":"online","context":{"idset":"576,689"}}
+{"timestamp":1709849946.3161283,"name":"online","context":{"idset":"690"}}
+{"timestamp":1709849947.3518567,"name":"online","context":{"idset":"691"}}
+{"timestamp":1709849947.6795905,"name":"online","context":{"idset":"692"}}
+{"timestamp":1709849948.1816344,"name":"online","context":{"idset":"693-694"}}
+{"timestamp":1709849949.6214201,"name":"online","context":{"idset":"695"}}
+{"timestamp":1709849950.6964982,"name":"online","context":{"idset":"697"}}
+{"timestamp":1709849950.8540313,"name":"online","context":{"idset":"696"}}
+{"timestamp":1709849950.966722,"name":"online","context":{"idset":"698"}}
+{"timestamp":1709849951.0719793,"name":"online","context":{"idset":"699"}}
+{"timestamp":1709849951.2555087,"name":"online","context":{"idset":"700"}}
+{"timestamp":1709849951.3688524,"name":"online","context":{"idset":"707"}}
+{"timestamp":1709849951.9781101,"name":"online","context":{"idset":"705,709-710,713"}}
+{"timestamp":1709849952.0035489,"name":"online","context":{"idset":"712"}}
+{"timestamp":1709849952.0881002,"name":"online","context":{"idset":"708"}}
+{"timestamp":1709849952.1852295,"name":"online","context":{"idset":"715"}}
+{"timestamp":1709849952.2926865,"name":"online","context":{"idset":"714,716,727"}}
+{"timestamp":1709849952.3544316,"name":"online","context":{"idset":"728,731"}}
+{"timestamp":1709849952.4943318,"name":"online","context":{"idset":"729,732-735,737"}}
+{"timestamp":1709849952.576519,"name":"online","context":{"idset":"739-740"}}
+{"timestamp":1709849952.6818066,"name":"online","context":{"idset":"736,738,741-743,747,751"}}
+{"timestamp":1709849952.6996546,"name":"online","context":{"idset":"744"}}
+{"timestamp":1709849952.760787,"name":"online","context":{"idset":"749,752"}}
+{"timestamp":1709849952.8706291,"name":"online","context":{"idset":"704,748"}}
+{"timestamp":1709849952.9641843,"name":"online","context":{"idset":"750,755-756"}}
+{"timestamp":1709849953.1261537,"name":"online","context":{"idset":"703"}}
+{"timestamp":1709849953.2327175,"name":"online","context":{"idset":"719"}}
+{"timestamp":1709849953.3688037,"name":"online","context":{"idset":"724"}}
+{"timestamp":1709849953.6037958,"name":"online","context":{"idset":"745"}}
+{"timestamp":1709849953.6688032,"name":"online","context":{"idset":"754"}}
+{"timestamp":1709849953.7805519,"name":"online","context":{"idset":"726"}}
+{"timestamp":1709849954.1329432,"name":"online","context":{"idset":"701,711,753"}}
+{"timestamp":1709849954.2207818,"name":"online","context":{"idset":"725,746"}}
+{"timestamp":1709849954.3763721,"name":"online","context":{"idset":"706,730"}}
+{"timestamp":1709849954.7154214,"name":"online","context":{"idset":"717"}}
+{"timestamp":1709849954.9701297,"name":"online","context":{"idset":"722"}}
+{"timestamp":1709849955.2459083,"name":"online","context":{"idset":"702"}}
+{"timestamp":1709849955.3110189,"name":"online","context":{"idset":"723"}}
+{"timestamp":1709849955.4214168,"name":"online","context":{"idset":"721"}}
+{"timestamp":1709849956.3303471,"name":"online","context":{"idset":"720"}}
+{"timestamp":1709849963.0316107,"name":"online","context":{"idset":"718"}}
+{"timestamp":1709851677.6815581,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"103":{"timestamp":1709784667.5992875,"reason":"reason=Unable to log in"},"208":{"timestamp":1709784425.5568194,"reason":"reason=Unable to log in"},"232":{"timestamp":1709696347.5038266,"reason":"reason=unresponsive"},"238":{"timestamp":1709336332.4828737,"reason":"reason=cxi_healthcheck fails"},"262":{"timestamp":1709784507.6836026,"reason":"reason=Unable to log in"},"301":{"timestamp":1709784694.554915,"reason":"reason=Unable to log in"},"348":{"timestamp":1709761076.2339444,"reason":"reason=BAD CHECKSUM"},"374":{"timestamp":1709825038.454864,"reason":"broker was unresponsive"},"440":{"timestamp":1709824224.4543412,"reason":"broker was unresponsive"},"532":{"timestamp":1709824318.4545081,"reason":"broker was unresponsive"},"550":{"timestamp":1709336806.3089643,"reason":"reason=No power control"},"569":{"timestamp":1709583475.6141074,"reason":"prolog failed for jobid fgjQELtBquR"},"663":{"timestamp":1709821866.4556227,"reason":"broker was unresponsive"},"710":{"timestamp":1709824588.4548297,"reason":"broker was unresponsive"},"751":{"timestamp":1709336341.4207382,"reason":"reason=cxi_healthcheck fails"},"756":{"timestamp":1709784722.3057358,"reason":"reason=Unable to log in"}},"online":"","exclude":"0"}}
+{"timestamp":1709851677.7166612,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709851715.9389014,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709851716.729059,"name":"online","context":{"idset":"61-68,70,76-98,100-150,154-352,354-484,486-549,551-681,683-756"}}
+{"timestamp":1709851716.8466451,"name":"online","context":{"idset":"1,24,28,30,45,47-49,54,59"}}
+{"timestamp":1709851717.1168525,"name":"online","context":{"idset":"3,18,151-153"}}
+{"timestamp":1709851717.2176187,"name":"online","context":{"idset":"2,4,8,69,74,485,682"}}
+{"timestamp":1709851717.318682,"name":"online","context":{"idset":"71-73,75,353"}}
+{"timestamp":1709858832.2912085,"name":"drain","context":{"idset":"99","reason":"reason=HPE Replacing Rabbit","overwrite":0}}
+{"timestamp":1709858868.28846,"name":"drain","context":{"idset":"550","reason":"reason=Awaiting MAC from Py","overwrite":1}}
+{"timestamp":1709860749.6626327,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1709861755.002317,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1709861755.0117257,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1709861755.0188584,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1709861755.027843,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1709861755.0778279,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1709861755.0853744,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1709861755.090441,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1709861755.1792693,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1709861755.253551,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1709861755.2592475,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1709861755.3522635,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1709861755.3839524,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1709861755.4579298,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1709861755.4644518,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1709861755.4703674,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1709861755.476048,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1709861755.4821787,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1709861755.4872048,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1709861755.4921513,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1709861755.4973195,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1709861755.5021303,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1709861755.5072331,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1709861755.5120966,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1709861755.5174966,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1709861755.5223527,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1709861755.5274606,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1709861755.5502672,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1709861755.5564072,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1709861755.574173,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1709861755.5806754,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1709861755.5860486,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1709861755.5928841,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1709861755.6165466,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1709861755.6256533,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1709861755.6390662,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1709861755.646219,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1709861755.6535616,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1709861755.7078278,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1709861755.7914002,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1709861755.7990854,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1709861755.8074689,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1709861755.8127484,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1709861755.8190928,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1709861755.8233254,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1709861755.8301039,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1709861755.8367033,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1709861755.8442848,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1709861755.8842323,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1709861756.0695882,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1709861756.0820158,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1709861756.0867758,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1709861756.0918891,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1709861756.0968773,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1709861756.103879,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1709861756.1108618,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1709861756.1114488,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1709861756.1159885,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1709861756.1218541,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1709861756.1268067,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1709861756.1273568,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1709861756.1305809,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1709861756.1336038,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1709861756.1416731,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1709861756.2127407,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1709861756.2176433,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1709861756.2452836,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1709861756.2888644,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1709861756.3631995,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1709861756.3638284,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1709861756.3644593,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1709861756.3649819,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1709861756.4468558,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1709861756.5272124,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1709861756.6334867,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1709861756.7156482,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1709861756.7880805,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1709861756.7933552,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1709861756.8728874,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1709861756.9087679,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1709861756.9127076,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1709861756.920105,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1709861756.936579,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1709861756.9468257,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1709861756.9781976,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1709861757.0123906,"name":"offline","context":{"idset":"121"}}
+{"timestamp":1709861757.0153894,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1709861757.0383418,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1709861757.0717971,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1709861757.1348181,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1709861757.139852,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1709861757.1798131,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1709861757.1857793,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1709861757.1947796,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1709861757.1988115,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1709861757.2027979,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1709861757.2065053,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1709861757.2107635,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1709861757.2147901,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1709861757.2177997,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1709861757.2217603,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1709861757.2257628,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1709861757.2262437,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1709861757.2297606,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1709861757.2337887,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1709861757.2377515,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1709861757.2417588,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1709861757.2457705,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1709861757.2488451,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1709861757.251601,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1709861757.255775,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1709861757.2597387,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1709861757.2637396,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1709861757.2717483,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1709861757.2757452,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1709861757.2957454,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1709861757.2961552,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1709861757.3027608,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1709861757.3290033,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1709861757.3337858,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1709861757.3737931,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1709861757.4720144,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1709861757.6149499,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1709861757.6262209,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1709861757.6314547,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1709861757.7214468,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1709861757.8004599,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1709861757.8432839,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1709861757.8677964,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1709861757.9690633,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1709861758.0650311,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1709861758.1358373,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1709861758.1949642,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1709861758.2091501,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1709861758.2609286,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1709861758.3616982,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1709861758.4729562,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1709861758.6207538,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1709861758.6743388,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1709861758.691766,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1709861758.7279902,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1709861758.7698526,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1709861758.7870207,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1709861758.8023546,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1709861758.8692992,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1709861758.8947473,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1709861758.9008822,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1709861758.9298995,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1709861758.9673293,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1709861758.9881768,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1709861759.0322497,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1709861759.0386322,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1709861759.042932,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1709861759.0516076,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1709861759.0740602,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1709861759.0831358,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1709861759.186281,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1709861759.2364764,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1709861759.24178,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1709861759.2655764,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1709861759.2995343,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1709861759.3169899,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1709861759.3483083,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1709861759.3686161,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1709861759.3770444,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1709861759.416816,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1709861759.4219873,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1709861759.4411356,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1709861759.4582582,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1709861759.4648781,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1709861759.4728646,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1709861759.522583,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1709861759.5718338,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1709861759.6727614,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1709861759.893791,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1709861759.9939044,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1709861760.3098693,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1709861760.4098153,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1709861760.8275859,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1709861760.9277918,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1709861761.1018147,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1709861761.1591187,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1709861761.2411087,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1709861761.3267853,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1709861761.3578608,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1709861761.4497647,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1709861761.5016983,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1709861761.5147331,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1709861761.5196993,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1709861761.5441523,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1709861761.5616879,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1709861761.5620863,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1709861761.5667319,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1709861761.5704916,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1709861761.574394,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1709861761.6041102,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1709861761.6187897,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1709861761.7086957,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1709861761.7166083,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1709861761.7217133,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1709861761.7437062,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1709861761.7587101,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1709861761.76369,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1709861761.8127425,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1709861761.8166568,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1709861761.8286119,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1709861761.8326862,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1709861761.8366709,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1709861761.8405032,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1709861761.8455257,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1709861761.8496904,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1709861761.8686187,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1709861761.8796172,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1709861761.8926396,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1709861761.8976347,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1709861761.9003863,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1709861761.9051514,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1709861761.9162738,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1709861761.9296639,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1709861761.9336019,"name":"offline","context":{"idset":"448"}}
+{"timestamp":1709861761.9456255,"name":"offline","context":{"idset":"450"}}
+{"timestamp":1709861761.9537702,"name":"offline","context":{"idset":"452"}}
+{"timestamp":1709861761.9646969,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1709861761.982312,"name":"offline","context":{"idset":"447"}}
+{"timestamp":1709861761.9872479,"name":"offline","context":{"idset":"451"}}
+{"timestamp":1709861761.9981544,"name":"offline","context":{"idset":"453"}}
+{"timestamp":1709861762.0085824,"name":"offline","context":{"idset":"445"}}
+{"timestamp":1709861762.0123608,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1709861762.0299044,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1709861762.0340805,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1709861762.0476665,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1709861762.0591817,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1709861762.0974629,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1709861762.1080062,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1709861762.1111343,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1709861762.1151481,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1709861762.1186488,"name":"offline","context":{"idset":"457"}}
+{"timestamp":1709861762.121007,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1709861762.1243408,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1709861762.1281457,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1709861762.1440215,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1709861762.183588,"name":"offline","context":{"idset":"462"}}
+{"timestamp":1709861762.1909673,"name":"offline","context":{"idset":"456"}}
+{"timestamp":1709861762.1973681,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1709861762.2011793,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1709861762.2056313,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1709861762.2125895,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1709861762.2340164,"name":"offline","context":{"idset":"464"}}
+{"timestamp":1709861762.2460546,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1709861762.2656596,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1709861762.2706347,"name":"offline","context":{"idset":"466"}}
+{"timestamp":1709861762.2846236,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1709861762.2957087,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1709861762.3106852,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1709861762.3296483,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1709861762.333668,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1709861762.3376362,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1709861762.3466258,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1709861762.354609,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1709861762.3916411,"name":"offline","context":{"idset":"468"}}
+{"timestamp":1709861762.4051862,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1709861762.4205413,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1709861762.4453554,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1709861762.4570124,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1709861762.4599314,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1709861762.4792049,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1709861762.4818654,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1709861762.5446553,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1709861762.5485995,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1709861762.5659399,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1709861762.5744152,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1709861762.5832226,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1709861762.5886295,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1709861762.6013188,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1709861762.6202836,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1709861762.6233363,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1709861762.6264782,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1709861762.6293736,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1709861762.637748,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1709861762.6431341,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1709861762.6621978,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1709861762.6706359,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1709861762.6745794,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1709861762.6989245,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1709861762.7035496,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1709861762.7171128,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1709861762.7226338,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1709861762.7625799,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1709861762.8116245,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1709861762.8135931,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1709861762.8306432,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1709861762.8401849,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1709861762.8435142,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1709861762.8476353,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1709861762.8492558,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1709861762.8536355,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1709861762.8726268,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1709861762.8866374,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1709861762.8995678,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1709861762.902566,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1709861762.9116554,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1709861762.9119999,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1709861762.9137838,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1709861762.9232225,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1709861762.9313822,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1709861762.9497776,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1709861762.9625721,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1709861762.9696093,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1709861762.9733603,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1709861762.979218,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1709861762.9867244,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1709861762.9871137,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1709861763.0044386,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1709861763.0105457,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1709861763.0167537,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1709861763.0281308,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1709861763.0379369,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1709861763.0402365,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1709861763.043021,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1709861763.070328,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1709861763.0914261,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1709861763.0987866,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1709861763.1025581,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1709861763.1067774,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1709861763.1163981,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1709861763.1203253,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1709861763.1247568,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1709861763.1281297,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1709861763.1317546,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1709861763.1348073,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1709861763.1535687,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1709861763.1558664,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1709861763.188592,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1709861763.199594,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1709861763.2035971,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1709861763.2295907,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1709861763.2345533,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1709861763.2381256,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1709861763.2410657,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1709861763.2425644,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1709861763.2575345,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1709861763.2606063,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1709861763.2646239,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1709861763.2866368,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1709861763.2905462,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1709861763.3075712,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1709861763.3116908,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1709861763.312006,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1709861763.3285449,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1709861763.3325598,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1709861763.3405855,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1709861763.3705935,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1709861763.3996692,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1709861763.4095435,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1709861763.4158061,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1709861763.4276729,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1709861763.4315286,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1709861763.4355505,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1709861763.4703624,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1709861763.4731491,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1709861763.4825375,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1709861763.4995308,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1709861763.5025716,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1709861763.5295897,"name":"offline","context":{"idset":"551"}}
+{"timestamp":1709861763.5392268,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1709861763.5435429,"name":"offline","context":{"idset":"549"}}
+{"timestamp":1709861763.5666237,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1709861763.5761266,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1709861763.5786848,"name":"offline","context":{"idset":"552"}}
+{"timestamp":1709861763.5883484,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1709861763.6285319,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1709861763.6362355,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1709861763.6525202,"name":"offline","context":{"idset":"556"}}
+{"timestamp":1709861763.6615472,"name":"offline","context":{"idset":"555"}}
+{"timestamp":1709861763.6825767,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1709861763.7211607,"name":"offline","context":{"idset":"564"}}
+{"timestamp":1709861763.7458215,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1709861763.7489219,"name":"offline","context":{"idset":"563"}}
+{"timestamp":1709861763.7557747,"name":"offline","context":{"idset":"554"}}
+{"timestamp":1709861763.7775908,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1709861763.7815447,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1709861763.7818031,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1709861763.7855589,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1709861763.8045702,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1709861763.8085749,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1709861763.8193264,"name":"offline","context":{"idset":"558"}}
+{"timestamp":1709861763.826982,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1709861763.8369803,"name":"offline","context":{"idset":"561"}}
+{"timestamp":1709861763.8615277,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1709861763.8617759,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1709861763.8725207,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1709861763.876519,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1709861763.8805764,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1709861763.8808675,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1709861763.8845177,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1709861763.8885045,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1709861763.9156463,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1709861763.9195559,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1709861763.9198143,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1709861763.9315271,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1709861763.931767,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1709861763.9849367,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1709861763.9915745,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1709861763.9935763,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1709861764.0001147,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1709861764.0185754,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1709861764.0329943,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1709861764.0431261,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1709861764.0479162,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1709861764.0690725,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1709861764.0767739,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1709861764.0886302,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1709861764.1089952,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1709861764.1330044,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1709861764.1347673,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1709861764.1363561,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1709861764.1501119,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1709861764.1521876,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1709861764.173034,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1709861764.1931243,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1709861764.2050769,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1709861764.2150512,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1709861764.2186952,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1709861764.2266109,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1709861764.2339995,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1709861764.24858,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1709861764.2562888,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1709861764.2619171,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1709861764.2665722,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1709861764.268326,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1709861764.2815738,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1709861764.3155522,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1709861764.3298659,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1709861764.38715,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1709861764.4029257,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1709861764.411339,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1709861764.4219165,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1709861764.4400978,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1709861764.4544287,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1709861764.4655247,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1709861764.5165453,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1709861764.5167956,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1709861764.5195324,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1709861764.5383501,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1709861764.5425253,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1709861764.5895348,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1709861764.6045482,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1709861764.6117399,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1709861764.6214595,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1709861764.6500936,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1709861764.6553676,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1709861764.6659272,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1709861764.6697514,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1709861764.6723316,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1709861764.6775815,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1709861764.6796932,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1709861764.7051895,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1709861764.7072725,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1709861764.7349012,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1709861764.7443831,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1709861764.7445683,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1709861764.7613397,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1709861764.7835777,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1709861764.7856865,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1709861764.8005929,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1709861764.8026373,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1709861764.8047519,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1709861764.8058019,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1709861764.8068621,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1709861764.8174818,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1709861764.8206158,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1709861764.8216746,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1709861764.8439641,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1709861764.8460822,"name":"offline","context":{"idset":"454"}}
+{"timestamp":1709861764.8471797,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1709861764.8481879,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1709861764.8513799,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1709861764.8545356,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1709861764.856632,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1709861764.8587704,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1709861764.8650939,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1709861764.8767815,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1709861764.8874991,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1709861764.8934987,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1709861764.8970442,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1709861764.9020252,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1709861764.9088414,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1709861764.9194593,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1709861764.9234488,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1709861764.9556313,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1709861764.9724593,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1709861765.0065038,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1709861765.0095251,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1709861765.0113697,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1709861765.0194945,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1709861765.0235353,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1709861765.0304985,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1709861765.031884,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1709861765.0434797,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1709861765.0604992,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1709861765.0635743,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1709861765.0648549,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1709861765.0726571,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1709861765.0748508,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1709861765.0810726,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1709861765.0874076,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1709861765.0906823,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1709861765.0925553,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1709861765.0993066,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1709861765.1142864,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1709861765.1161494,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1709861765.1197317,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1709861765.1434131,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1709861765.1436005,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1709861765.143775,"name":"offline","context":{"idset":"656"}}
+{"timestamp":1709861765.1494846,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1709861765.1505992,"name":"offline","context":{"idset":"657"}}
+{"timestamp":1709861765.1526697,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1709861765.1602998,"name":"offline","context":{"idset":"654"}}
+{"timestamp":1709861765.1911836,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1709861765.1934533,"name":"offline","context":{"idset":"662"}}
+{"timestamp":1709861765.2328808,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1709861765.2535238,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1709861765.25756,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1709861765.2924514,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1709861765.2955935,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1709861765.2965117,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1709861765.3116896,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1709861765.3227751,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1709861765.32903,"name":"offline","context":{"idset":"655"}}
+{"timestamp":1709861765.3320189,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1709861765.3321919,"name":"offline","context":{"idset":"672"}}
+{"timestamp":1709861765.3384287,"name":"offline","context":{"idset":"658"}}
+{"timestamp":1709861765.3424387,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1709861765.348429,"name":"offline","context":{"idset":"659"}}
+{"timestamp":1709861765.3628042,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1709861765.3670971,"name":"offline","context":{"idset":"660"}}
+{"timestamp":1709861765.3730874,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1709861765.3764384,"name":"offline","context":{"idset":"666"}}
+{"timestamp":1709861765.3804359,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1709861765.3854322,"name":"offline","context":{"idset":"665"}}
+{"timestamp":1709861765.3894198,"name":"offline","context":{"idset":"675"}}
+{"timestamp":1709861765.3895571,"name":"offline","context":{"idset":"661"}}
+{"timestamp":1709861765.3934481,"name":"offline","context":{"idset":"669"}}
+{"timestamp":1709861765.3994298,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1709861765.3995681,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1709861765.3996973,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1709861765.399822,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1709861765.4034133,"name":"offline","context":{"idset":"671"}}
+{"timestamp":1709861765.4074166,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1709861765.4125152,"name":"offline","context":{"idset":"667"}}
+{"timestamp":1709861765.4544568,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1709861765.4559999,"name":"offline","context":{"idset":"670"}}
+{"timestamp":1709861765.4574435,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1709861765.4584758,"name":"offline","context":{"idset":"681"}}
+{"timestamp":1709861765.4681466,"name":"offline","context":{"idset":"559"}}
+{"timestamp":1709861765.4729435,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1709861765.4842808,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1709861765.4896197,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1709861765.5179467,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1709861765.5225751,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1709861765.5341697,"name":"offline","context":{"idset":"677"}}
+{"timestamp":1709861765.5351148,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1709861765.5362749,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1709861765.536485,"name":"offline","context":{"idset":"682"}}
+{"timestamp":1709861765.541784,"name":"offline","context":{"idset":"683"}}
+{"timestamp":1709861765.5455256,"name":"offline","context":{"idset":"679"}}
+{"timestamp":1709861765.5565143,"name":"offline","context":{"idset":"690"}}
+{"timestamp":1709861765.5574188,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1709861765.5711589,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1709861765.5723295,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1709861765.5813782,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1709861765.5987349,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1709861765.6181536,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1709861765.6221905,"name":"offline","context":{"idset":"676"}}
+{"timestamp":1709861765.6347241,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1709861765.6459515,"name":"offline","context":{"idset":"694"}}
+{"timestamp":1709861765.6763029,"name":"offline","context":{"idset":"692"}}
+{"timestamp":1709861765.6788177,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1709861765.7070789,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1709861765.7083552,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1709861765.7112195,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1709861765.7295754,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1709861765.7512593,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1709861765.754653,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1709861765.7605503,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1709861765.7688665,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1709861765.7756836,"name":"offline","context":{"idset":"685"}}
+{"timestamp":1709861765.7777407,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1709861765.7834854,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1709861765.7844651,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1709861765.7968302,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1709861765.8044074,"name":"offline","context":{"idset":"708"}}
+{"timestamp":1709861765.8061898,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1709861765.8100004,"name":"offline","context":{"idset":"680"}}
+{"timestamp":1709861765.8225095,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1709861765.8270938,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1709861765.8282461,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1709861765.8348203,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1709861765.8362131,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1709861765.8433049,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1709861765.8627431,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1709861765.8778965,"name":"offline","context":{"idset":"707"}}
+{"timestamp":1709861765.886579,"name":"offline","context":{"idset":"701"}}
+{"timestamp":1709861765.8880527,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1709861765.8889551,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1709861765.8928323,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1709861765.8937185,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1709861765.8956699,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1709861765.9051754,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1709861765.9084835,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1709861765.9231536,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1709861765.9249327,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1709861765.9351656,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1709861765.9408035,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1709861765.9474311,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1709861765.9486735,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1709861765.9516709,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1709861765.9527252,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1709861765.9540296,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1709861765.9633009,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1709861765.9644394,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1709861765.9706843,"name":"offline","context":{"idset":"706"}}
+{"timestamp":1709861765.9978092,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1709861765.9992616,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1709861766.019547,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1709861766.0207429,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1709861766.0232129,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1709861766.0255368,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1709861766.0325863,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1709861766.0403917,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1709861766.0637536,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1709861766.0804033,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1709861766.0942192,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1709861766.1006949,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1709861766.1052225,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1709861766.1114955,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1709861766.1193752,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1709861766.1640322,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1709861766.1681397,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1709861766.1755934,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1709861766.1797819,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1709861766.1846428,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1709861766.1854525,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1709861766.1859982,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1709861766.2120645,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1709861766.2178497,"name":"offline","context":{"idset":"553"}}
+{"timestamp":1709861766.2194016,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1709861766.2250738,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1709861766.2360044,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1709861766.2392621,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1709861766.2476671,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1709861766.2532327,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1709861766.256114,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1709861766.2632246,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1709861766.2639558,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1709861766.2676921,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1709861766.2735999,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1709861766.2805524,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1709861766.3071766,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1709861766.3169622,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1709861766.326283,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1709861766.3467135,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1709861766.3505239,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1709861766.363822,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1709861766.368073,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1709861766.3788338,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1709861766.3795395,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1709861766.3833313,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1709861766.4003148,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1709861766.4096253,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1709861766.4101119,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1709861766.4119866,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1709861766.4440727,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1709861766.4477403,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1709861766.4573476,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1709861766.4655051,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1709861766.4938109,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1709861766.4974799,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1709861766.5001168,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1709861766.5182889,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1709861766.5316234,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1709861766.537154,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1709861766.5544653,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1709861766.5975659,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1709861766.6846743,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1709861766.6881881,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1709861766.7887375,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1709861767.0616848,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1709861767.162205,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1709861767.4203284,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1709861767.5209002,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1709861768.2240541,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1709861768.5609298,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1709863241.8896568,"name":"online","context":{"idset":"66"}}
+{"timestamp":1709863241.9730387,"name":"online","context":{"idset":"93"}}
+{"timestamp":1709863242.1704488,"name":"online","context":{"idset":"83"}}
+{"timestamp":1709863242.184212,"name":"online","context":{"idset":"97"}}
+{"timestamp":1709863242.2022061,"name":"online","context":{"idset":"65"}}
+{"timestamp":1709863242.2791972,"name":"online","context":{"idset":"67,101,117,121"}}
+{"timestamp":1709863242.3800936,"name":"online","context":{"idset":"62-64,77,95-96,103-104,116,123,129"}}
+{"timestamp":1709863242.4827788,"name":"online","context":{"idset":"73,82,98,105-106,112,119,139"}}
+{"timestamp":1709863242.591753,"name":"online","context":{"idset":"70,89,107,109,111,133"}}
+{"timestamp":1709863242.6078513,"name":"online","context":{"idset":"168"}}
+{"timestamp":1709863242.7087789,"name":"online","context":{"idset":"61,81,85,100,102,110,114-115,124,128,131,134,140,145,153,164,167,169,176"}}
+{"timestamp":1709863242.8103309,"name":"online","context":{"idset":"68-69,79,86-87,92,94,113,120,122,126-127,130,137-138,144,148,150,160,166,171-172,178,180"}}
+{"timestamp":1709863242.9124775,"name":"online","context":{"idset":"75-76,84,88,90-91,108,118,125,135,146,155-156,158-159,161-162,170,174,179,188"}}
+{"timestamp":1709863243.0159941,"name":"online","context":{"idset":"74,78,132,141,147,152,154,157,163,173,175,181"}}
+{"timestamp":1709863243.1205313,"name":"online","context":{"idset":"136,142-143,151,177,182,184-185,187"}}
+{"timestamp":1709863243.2295856,"name":"online","context":{"idset":"149,165,183,186"}}
+{"timestamp":1709863243.5394697,"name":"online","context":{"idset":"72,80"}}
+{"timestamp":1709863243.6584649,"name":"online","context":{"idset":"71"}}
+{"timestamp":1709863250.6114161,"name":"online","context":{"idset":"189"}}
+{"timestamp":1709863250.7962015,"name":"online","context":{"idset":"190"}}
+{"timestamp":1709863250.842833,"name":"online","context":{"idset":"191"}}
+{"timestamp":1709863251.0451097,"name":"online","context":{"idset":"198-199"}}
+{"timestamp":1709863251.068414,"name":"online","context":{"idset":"192"}}
+{"timestamp":1709863251.1543853,"name":"online","context":{"idset":"193"}}
+{"timestamp":1709863251.2253337,"name":"online","context":{"idset":"200"}}
+{"timestamp":1709863251.2784913,"name":"online","context":{"idset":"195"}}
+{"timestamp":1709863251.3398495,"name":"online","context":{"idset":"202"}}
+{"timestamp":1709863251.3868144,"name":"online","context":{"idset":"196"}}
+{"timestamp":1709863251.4322031,"name":"online","context":{"idset":"194"}}
+{"timestamp":1709863251.4925621,"name":"online","context":{"idset":"197,203"}}
+{"timestamp":1709863251.5062509,"name":"online","context":{"idset":"201"}}
+{"timestamp":1709863251.5181062,"name":"online","context":{"idset":"212"}}
+{"timestamp":1709863251.5486238,"name":"online","context":{"idset":"209,211"}}
+{"timestamp":1709863251.5910559,"name":"online","context":{"idset":"204,218"}}
+{"timestamp":1709863251.6064408,"name":"online","context":{"idset":"222"}}
+{"timestamp":1709863251.6339097,"name":"online","context":{"idset":"214"}}
+{"timestamp":1709863251.6651046,"name":"online","context":{"idset":"220"}}
+{"timestamp":1709863251.7094803,"name":"online","context":{"idset":"223"}}
+{"timestamp":1709863251.7284555,"name":"online","context":{"idset":"206"}}
+{"timestamp":1709863251.750581,"name":"online","context":{"idset":"210"}}
+{"timestamp":1709863251.7755227,"name":"online","context":{"idset":"205"}}
+{"timestamp":1709863251.8026373,"name":"online","context":{"idset":"217"}}
+{"timestamp":1709863251.832495,"name":"online","context":{"idset":"215"}}
+{"timestamp":1709863251.9536259,"name":"online","context":{"idset":"207,213"}}
+{"timestamp":1709863252.0256419,"name":"online","context":{"idset":"208,225,228-229,237,249,253"}}
+{"timestamp":1709863252.1272464,"name":"online","context":{"idset":"216,219,227,234,238,241,243,248"}}
+{"timestamp":1709863252.2454243,"name":"online","context":{"idset":"226,230-231,233,239-240,242,256,261"}}
+{"timestamp":1709863252.3616729,"name":"online","context":{"idset":"224,232,235-236,250,257,260,271,285,294,299"}}
+{"timestamp":1709863252.4699662,"name":"online","context":{"idset":"251-252,264-267,273-275,281,284,291,295,301,308"}}
+{"timestamp":1709863252.6101153,"name":"online","context":{"idset":"258-259,287,289,306,310-311,313,315"}}
+{"timestamp":1709863252.7127509,"name":"online","context":{"idset":"221,247,263,269,277,280,286,288,292,302,304-305"}}
+{"timestamp":1709863252.8152204,"name":"online","context":{"idset":"244-246,262,270,272,276,278-279,282,290,293,296-298,300,303,307,309,314,316"}}
+{"timestamp":1709863252.9288268,"name":"online","context":{"idset":"254-255,283"}}
+{"timestamp":1709863253.0515935,"name":"online","context":{"idset":"312"}}
+{"timestamp":1709863254.6864216,"name":"online","context":{"idset":"317"}}
+{"timestamp":1709863259.6398823,"name":"online","context":{"idset":"319"}}
+{"timestamp":1709863259.6677501,"name":"online","context":{"idset":"318"}}
+{"timestamp":1709863259.9861329,"name":"online","context":{"idset":"320-322"}}
+{"timestamp":1709863260.0976171,"name":"online","context":{"idset":"323,325"}}
+{"timestamp":1709863260.1506429,"name":"online","context":{"idset":"326"}}
+{"timestamp":1709863260.1846223,"name":"online","context":{"idset":"324"}}
+{"timestamp":1709863260.2320149,"name":"online","context":{"idset":"328"}}
+{"timestamp":1709863260.3604803,"name":"online","context":{"idset":"329"}}
+{"timestamp":1709863260.416805,"name":"online","context":{"idset":"327"}}
+{"timestamp":1709863260.5119188,"name":"online","context":{"idset":"334,339"}}
+{"timestamp":1709863260.5177989,"name":"online","context":{"idset":"330"}}
+{"timestamp":1709863260.5718071,"name":"online","context":{"idset":"332"}}
+{"timestamp":1709863260.5872996,"name":"online","context":{"idset":"336"}}
+{"timestamp":1709863260.606128,"name":"online","context":{"idset":"335"}}
+{"timestamp":1709863260.6585376,"name":"online","context":{"idset":"333,338,340"}}
+{"timestamp":1709863260.6821156,"name":"online","context":{"idset":"331"}}
+{"timestamp":1709863260.7025075,"name":"online","context":{"idset":"337"}}
+{"timestamp":1709863260.722384,"name":"online","context":{"idset":"342"}}
+{"timestamp":1709863260.7747002,"name":"online","context":{"idset":"341"}}
+{"timestamp":1709863260.8243573,"name":"online","context":{"idset":"344"}}
+{"timestamp":1709863260.9276791,"name":"online","context":{"idset":"345"}}
+{"timestamp":1709863260.9819646,"name":"online","context":{"idset":"346"}}
+{"timestamp":1709863261.0418479,"name":"online","context":{"idset":"343"}}
+{"timestamp":1709863261.0782788,"name":"online","context":{"idset":"347"}}
+{"timestamp":1709863261.0973716,"name":"online","context":{"idset":"352"}}
+{"timestamp":1709863261.168313,"name":"online","context":{"idset":"350,356,360"}}
+{"timestamp":1709863261.187614,"name":"online","context":{"idset":"357"}}
+{"timestamp":1709863261.2413788,"name":"online","context":{"idset":"349"}}
+{"timestamp":1709863261.2599914,"name":"online","context":{"idset":"358"}}
+{"timestamp":1709863261.2776208,"name":"online","context":{"idset":"354"}}
+{"timestamp":1709863261.2989619,"name":"online","context":{"idset":"362"}}
+{"timestamp":1709863261.3251791,"name":"online","context":{"idset":"375"}}
+{"timestamp":1709863261.3496301,"name":"online","context":{"idset":"348,351"}}
+{"timestamp":1709863261.5431201,"name":"online","context":{"idset":"355,363,365,367,369,379"}}
+{"timestamp":1709863261.6505308,"name":"online","context":{"idset":"359,366,370,373-374,382-383"}}
+{"timestamp":1709863261.752882,"name":"online","context":{"idset":"364,368,377-378,381,384,388-389"}}
+{"timestamp":1709863261.8627045,"name":"online","context":{"idset":"361,372,380,385-386,390-391,393,398,400,404-405"}}
+{"timestamp":1709863261.9782362,"name":"online","context":{"idset":"376,394,396,401,406,408-409"}}
+{"timestamp":1709863262.0817025,"name":"online","context":{"idset":"387,392,395,399,402,407,412,414,418,423"}}
+{"timestamp":1709863262.1932013,"name":"online","context":{"idset":"353,397,410-411,413,416,421-422,425,428-432,435,437,439,441"}}
+{"timestamp":1709863262.3021712,"name":"online","context":{"idset":"403,415,417,419-420,424,426-427,433-434,438,440,442-444"}}
+{"timestamp":1709863262.4476192,"name":"online","context":{"idset":"436"}}
+{"timestamp":1709863263.4698186,"name":"online","context":{"idset":"445"}}
+{"timestamp":1709863263.6593037,"name":"online","context":{"idset":"446"}}
+{"timestamp":1709863268.4686775,"name":"online","context":{"idset":"448"}}
+{"timestamp":1709863268.5167029,"name":"online","context":{"idset":"447"}}
+{"timestamp":1709863268.8355567,"name":"online","context":{"idset":"449"}}
+{"timestamp":1709863268.9154191,"name":"online","context":{"idset":"450,452"}}
+{"timestamp":1709863269.027746,"name":"online","context":{"idset":"451,453,455"}}
+{"timestamp":1709863269.0841811,"name":"online","context":{"idset":"456"}}
+{"timestamp":1709863269.0959408,"name":"online","context":{"idset":"454"}}
+{"timestamp":1709863269.3017237,"name":"online","context":{"idset":"458"}}
+{"timestamp":1709863269.447736,"name":"online","context":{"idset":"457,459,461,464"}}
+{"timestamp":1709863269.4655311,"name":"online","context":{"idset":"463"}}
+{"timestamp":1709863269.5236874,"name":"online","context":{"idset":"460"}}
+{"timestamp":1709863269.6508904,"name":"online","context":{"idset":"462,465,469-470"}}
+{"timestamp":1709863269.6705091,"name":"online","context":{"idset":"467-468"}}
+{"timestamp":1709863269.7925339,"name":"online","context":{"idset":"471-472"}}
+{"timestamp":1709863269.8399239,"name":"online","context":{"idset":"473"}}
+{"timestamp":1709863269.9478383,"name":"online","context":{"idset":"476"}}
+{"timestamp":1709863270.0097985,"name":"online","context":{"idset":"474"}}
+{"timestamp":1709863270.0585935,"name":"online","context":{"idset":"475,477"}}
+{"timestamp":1709863270.0660906,"name":"online","context":{"idset":"479"}}
+{"timestamp":1709863270.1676335,"name":"online","context":{"idset":"481"}}
+{"timestamp":1709863270.2140074,"name":"online","context":{"idset":"478,483"}}
+{"timestamp":1709863270.2634847,"name":"online","context":{"idset":"480,484"}}
+{"timestamp":1709863270.3933549,"name":"online","context":{"idset":"482"}}
+{"timestamp":1709863270.4520235,"name":"online","context":{"idset":"487"}}
+{"timestamp":1709863270.4583516,"name":"online","context":{"idset":"488"}}
+{"timestamp":1709863270.6069167,"name":"online","context":{"idset":"491"}}
+{"timestamp":1709863270.6346498,"name":"online","context":{"idset":"486"}}
+{"timestamp":1709863270.6990955,"name":"online","context":{"idset":"489"}}
+{"timestamp":1709863270.7518413,"name":"online","context":{"idset":"490,493,496,498"}}
+{"timestamp":1709863270.7846332,"name":"online","context":{"idset":"492,502,505"}}
+{"timestamp":1709863270.8438671,"name":"online","context":{"idset":"499"}}
+{"timestamp":1709863270.8747356,"name":"online","context":{"idset":"495"}}
+{"timestamp":1709863270.9780862,"name":"online","context":{"idset":"485,494,500,503,506"}}
+{"timestamp":1709863271.0840952,"name":"online","context":{"idset":"497,509,512-513,517,528-529"}}
+{"timestamp":1709863271.1950915,"name":"online","context":{"idset":"501,504,510,518-520,522,526"}}
+{"timestamp":1709863271.2690289,"name":"online","context":{"idset":"525,530,534-535,537,539"}}
+{"timestamp":1709863271.3776498,"name":"online","context":{"idset":"508,511,514,516,521,524,527,536,538,541-542,546,555"}}
+{"timestamp":1709863271.4861846,"name":"online","context":{"idset":"507,515,533,540,549,551-552,556-557,559-560,562-565,572"}}
+{"timestamp":1709863271.5866718,"name":"online","context":{"idset":"531-532,544-545,548,558,566,568,570"}}
+{"timestamp":1709863271.6962693,"name":"online","context":{"idset":"523,543,547,553-554,567,569"}}
+{"timestamp":1709863271.7458305,"name":"online","context":{"idset":"561"}}
+{"timestamp":1709863271.8514013,"name":"online","context":{"idset":"571,573"}}
+{"timestamp":1709863272.281018,"name":"online","context":{"idset":"574"}}
+{"timestamp":1709863272.7612479,"name":"online","context":{"idset":"575"}}
+{"timestamp":1709863274.5482709,"name":"online","context":{"idset":"576"}}
+{"timestamp":1709863277.4868357,"name":"online","context":{"idset":"577-578"}}
+{"timestamp":1709863277.6806822,"name":"online","context":{"idset":"579"}}
+{"timestamp":1709863277.8104801,"name":"online","context":{"idset":"580"}}
+{"timestamp":1709863277.8655462,"name":"online","context":{"idset":"582"}}
+{"timestamp":1709863277.9192326,"name":"online","context":{"idset":"581,583-584"}}
+{"timestamp":1709863278.0580699,"name":"online","context":{"idset":"585"}}
+{"timestamp":1709863278.1368287,"name":"online","context":{"idset":"586"}}
+{"timestamp":1709863278.2329521,"name":"online","context":{"idset":"587"}}
+{"timestamp":1709863278.4329588,"name":"online","context":{"idset":"588,593"}}
+{"timestamp":1709863278.4463854,"name":"online","context":{"idset":"590"}}
+{"timestamp":1709863278.5566156,"name":"online","context":{"idset":"589,591-592,594,596"}}
+{"timestamp":1709863278.6485262,"name":"online","context":{"idset":"595,597,599"}}
+{"timestamp":1709863278.6727741,"name":"online","context":{"idset":"598"}}
+{"timestamp":1709863278.7644405,"name":"online","context":{"idset":"600"}}
+{"timestamp":1709863278.8127718,"name":"online","context":{"idset":"602"}}
+{"timestamp":1709863278.8929183,"name":"online","context":{"idset":"601,603"}}
+{"timestamp":1709863278.946321,"name":"online","context":{"idset":"606"}}
+{"timestamp":1709863279.0031159,"name":"online","context":{"idset":"605,607"}}
+{"timestamp":1709863279.0752873,"name":"online","context":{"idset":"608"}}
+{"timestamp":1709863279.1597161,"name":"online","context":{"idset":"609"}}
+{"timestamp":1709863279.2451484,"name":"online","context":{"idset":"604,611-612"}}
+{"timestamp":1709863279.2770786,"name":"online","context":{"idset":"610"}}
+{"timestamp":1709863279.3867106,"name":"online","context":{"idset":"613-614"}}
+{"timestamp":1709863279.5730662,"name":"online","context":{"idset":"615-616"}}
+{"timestamp":1709863279.6102309,"name":"online","context":{"idset":"617"}}
+{"timestamp":1709863279.712724,"name":"online","context":{"idset":"618"}}
+{"timestamp":1709863279.7383497,"name":"online","context":{"idset":"622"}}
+{"timestamp":1709863279.815284,"name":"online","context":{"idset":"619"}}
+{"timestamp":1709863279.8640871,"name":"online","context":{"idset":"621,624,627"}}
+{"timestamp":1709863279.8838446,"name":"online","context":{"idset":"620"}}
+{"timestamp":1709863279.9163306,"name":"online","context":{"idset":"626"}}
+{"timestamp":1709863279.9363792,"name":"online","context":{"idset":"630"}}
+{"timestamp":1709863279.989327,"name":"online","context":{"idset":"623,625"}}
+{"timestamp":1709863279.9994061,"name":"online","context":{"idset":"629"}}
+{"timestamp":1709863280.0803726,"name":"online","context":{"idset":"632-633"}}
+{"timestamp":1709863280.1021895,"name":"online","context":{"idset":"628"}}
+{"timestamp":1709863280.1731753,"name":"online","context":{"idset":"631"}}
+{"timestamp":1709863280.2287033,"name":"online","context":{"idset":"645"}}
+{"timestamp":1709863280.3566682,"name":"online","context":{"idset":"634-636,642"}}
+{"timestamp":1709863280.4677844,"name":"online","context":{"idset":"647-648,652"}}
+{"timestamp":1709863280.5870476,"name":"online","context":{"idset":"637-638,640-641,646,650,654,656"}}
+{"timestamp":1709863280.681514,"name":"online","context":{"idset":"643-644,653,657-658,667,669-670,672"}}
+{"timestamp":1709863280.8068359,"name":"online","context":{"idset":"649,655,660,664-666,668,675,680,686"}}
+{"timestamp":1709863280.9151895,"name":"online","context":{"idset":"659,661-662,674,678-679,681,689,691,695-696"}}
+{"timestamp":1709863281.0205214,"name":"online","context":{"idset":"651,663,671,673,677,683-685,690,692-694,697,699-701"}}
+{"timestamp":1709863281.136905,"name":"online","context":{"idset":"676,687-688"}}
+{"timestamp":1709863281.2436662,"name":"online","context":{"idset":"698,702"}}
+{"timestamp":1709863281.6882713,"name":"online","context":{"idset":"682,703"}}
+{"timestamp":1709863282.4796205,"name":"online","context":{"idset":"704"}}
+{"timestamp":1709863283.5161436,"name":"online","context":{"idset":"705"}}
+{"timestamp":1709863286.4031174,"name":"online","context":{"idset":"706-707"}}
+{"timestamp":1709863286.5578649,"name":"online","context":{"idset":"708"}}
+{"timestamp":1709863286.7485807,"name":"online","context":{"idset":"709,711"}}
+{"timestamp":1709863286.8020949,"name":"online","context":{"idset":"710"}}
+{"timestamp":1709863286.9535346,"name":"online","context":{"idset":"713"}}
+{"timestamp":1709863286.9998412,"name":"online","context":{"idset":"712"}}
+{"timestamp":1709863287.0489142,"name":"online","context":{"idset":"714-715"}}
+{"timestamp":1709863287.15769,"name":"online","context":{"idset":"716"}}
+{"timestamp":1709863287.4150651,"name":"online","context":{"idset":"717-718"}}
+{"timestamp":1709863287.4811771,"name":"online","context":{"idset":"720,726"}}
+{"timestamp":1709863287.575583,"name":"online","context":{"idset":"719,721,724"}}
+{"timestamp":1709863287.6305952,"name":"online","context":{"idset":"723"}}
+{"timestamp":1709863287.6839852,"name":"online","context":{"idset":"722,725,727,729"}}
+{"timestamp":1709863287.7324646,"name":"online","context":{"idset":"728,730"}}
+{"timestamp":1709863287.8457391,"name":"online","context":{"idset":"731"}}
+{"timestamp":1709863287.9023621,"name":"online","context":{"idset":"733"}}
+{"timestamp":1709863287.9283729,"name":"online","context":{"idset":"732"}}
+{"timestamp":1709863287.9836411,"name":"online","context":{"idset":"735"}}
+{"timestamp":1709863288.0341518,"name":"online","context":{"idset":"736"}}
+{"timestamp":1709863288.1920066,"name":"online","context":{"idset":"734,737-738,740"}}
+{"timestamp":1709863288.3375823,"name":"online","context":{"idset":"741"}}
+{"timestamp":1709863288.4514165,"name":"online","context":{"idset":"739,742"}}
+{"timestamp":1709863288.6180022,"name":"online","context":{"idset":"743-746"}}
+{"timestamp":1709863288.7632835,"name":"online","context":{"idset":"747-748"}}
+{"timestamp":1709863288.8728461,"name":"online","context":{"idset":"749-752"}}
+{"timestamp":1709863288.9830558,"name":"online","context":{"idset":"753-756"}}
+{"timestamp":1709863460.9979403,"name":"undrain","context":{"idset":"103,208,232,238,262,301,348,374,440,532,569,663,710,751,756"}}
+{"timestamp":1709866839.8394222,"name":"drain","context":{"idset":"348","reason":"reason=BAD WRITE CHECKSUM","overwrite":0}}
+{"timestamp":1709867072.3431404,"name":"drain","context":{"idset":"663","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1709916152.4323595,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"99":{"timestamp":1709858832.2912085,"reason":"reason=HPE Replacing Rabbit"},"348":{"timestamp":1709866839.8394222,"reason":"reason=BAD WRITE CHECKSUM"},"550":{"timestamp":1709336806.3089643,"reason":"reason=Awaiting MAC from Py"},"663":{"timestamp":1709867072.3431404,"reason":"nodediag failed cxi"}},"online":"","exclude":"0"}}
+{"timestamp":1709916152.4659843,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709916190.5649524,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709916191.3961182,"name":"online","context":{"idset":"1,18,24,28,30,45,47-49,54,59"}}
+{"timestamp":1709916191.8793089,"name":"online","context":{"idset":"753-756"}}
+{"timestamp":1709916192.0070329,"name":"online","context":{"idset":"238"}}
+{"timestamp":1709916192.247134,"name":"online","context":{"idset":"2-4,8,75,111,113,123,131,133,154-155,159,166,202,223,229,247,257,259,317,329-330,362,409,414,417,444,446,449,457,489,517,519,530,532,549,575,577,585,589,624,637,650,658-659,661,676-677,691,707-708,729-731,734,745"}}
+{"timestamp":1709916192.3501592,"name":"online","context":{"idset":"64,70,74,82,85,87,115,125,129,135,139,141,147,152,157,160,162,178-179,184,188,190,192-193,197-198,209-211,213,219,221-222,241,249,253,260-261,265,285-286,297-299,304,312,319-320,323,328,342,344,351,353,358-361,364,373,386,395,397,401,406,416,422,431,433,436,448,458,461,464,468,478,480-482,495,507,516,518,524,536,542,552-553,555-556,559,562,573,578,581-584,588,602,607,609,611-612,617,623,630,641,651-653,656,660,669,671,680,688,693,700,706,710,713,720,722,724-726,728,733,737-739,746"}}
+{"timestamp":1709916192.4535708,"name":"online","context":{"idset":"71,80-81,84,104-106,112,114,117,119,121,124,132,134,156,164,176,181,187,191,196,199,203,206,212,214,217-218,226-228,236,254,270,274,283,288,291-293,305,310,313,316,324,333-334,355,367,372,374,378,384,389,391,400,404,411-412,415,430,439,445,450,453-455,462,483,485,488,494,499,502-505,512,515,528,531,535,538,543,545-546,560,565,567,569,580,587,596,598,600,605,621,631,633,643-644,647-648,673,675,678,681-682,684,711,714-715,721,740,744"}}
+{"timestamp":1709916192.5580602,"name":"online","context":{"idset":"61,66-67,69,73,77,79,88,93-94,96-98,102-103,107,109,116,122,127,130,140,142,144-145,148,151,165,167,169,171-173,177,180,182,185-186,189,195,200-201,205,207-208,225,232-233,235,244,256,258,262-264,266,273,276-278,287,290,294,296,302,307,309,311,314,318,322,325,335,337-340,345,349-350,352,354,356-357,365-366,368-370,379-383,385,388,390,392-394,396,398,402,407,410,421,423-424,426-427,432,435,441,451,456,459,463,465,470,475,477,479,486-487,490-492,508-509,521,529,533,537,547,551,557,561,564,570-571,576,579,591-593,595,597,599,608,614,616,619,625,628-629,632,634-636,638,640,642,645,649,664-666,668,672,674,679,685,687,695,697-698,703,709,716-718,732,747,750"}}
+{"timestamp":1709916192.6596851,"name":"online","context":{"idset":"62-63,65,68,72,76,78,83,86,89-92,95,100-101,108,110,118,120,126,128,136-138,143,146,149-150,153,158,161,163,168,170,174-175,183,194,204,215-216,220,224,230-231,234,237,239-240,242-243,245-246,248,250-252,255,267,269,271-272,275,279-282,284,289,295,300-301,303,306,308,315,321,326-327,331-332,336,341,343,346-347,363,375-377,387,399,403,405,408,413,418-420,425,428-429,434,437-438,440,442-443,447,452,460,467,469,471-474,476,484,493,496-498,500-501,506,510-511,513-514,520,522-523,525-527,534,539-541,544,548,554,558,563,566,568,572,574,586,590,594,601,603-604,606,610,613,615,618,620,622,626-627,646,654-655,657,662-663,667,670,683,686,689-690,692,694,696,699,701-702,704-705,712,719,723,727,735-736,741-743,748-749,751-752"}}
+{"timestamp":1709916192.8231356,"name":"online","context":{"idset":"348"}}
+{"timestamp":1709917499.1823676,"name":"drain","context":{"idset":"624,626","reason":"drained by /admin/scripts/drain_node","overwrite":0}}
+{"timestamp":1709921686.5896759,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1709921686.6357012,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1709921686.6401815,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1709921686.6648407,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1709921686.6738021,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1709921686.6792421,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1709921686.7198243,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1709921686.7276418,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1709921686.7355275,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1709921686.7469687,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1709921686.74774,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1709921686.7846055,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1709921686.8176632,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1709921686.841748,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1709921686.8524554,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1709921686.9526842,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1709921687.0697646,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1709921687.1013088,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1709921687.1140049,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1709921687.1195588,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1709921687.1242337,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1709921687.1328037,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1709921687.1331856,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1709921687.1502864,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1709921687.1750247,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1709921687.175611,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1709921687.1808758,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1709921687.1813848,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1709921687.18804,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1709921687.1886497,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1709921687.192781,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1709921687.1997404,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1709921687.2007039,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1709921687.208586,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1709921687.2141607,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1709921687.2332499,"name":"offline","context":{"idset":"661"}}
+{"timestamp":1709921687.2644992,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1709921687.2647982,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1709921687.269129,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1709921687.2700112,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1709921687.2703221,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1709921687.2746096,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1709921687.2748849,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1709921687.2751458,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1709921687.2765474,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1709921687.2768111,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1709921687.2774525,"name":"offline","context":{"idset":"448"}}
+{"timestamp":1709921687.2778058,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1709921687.2781122,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1709921687.2783711,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1709921687.2786272,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1709921687.2816775,"name":"offline","context":{"idset":"659"}}
+{"timestamp":1709921687.2874758,"name":"offline","context":{"idset":"676"}}
+{"timestamp":1709921687.2880113,"name":"offline","context":{"idset":"677"}}
+{"timestamp":1709921687.294205,"name":"offline","context":{"idset":"708"}}
+{"timestamp":1709921687.4606633,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1709921687.4611502,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1709921687.4649365,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1709921687.4653718,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1709921687.4724703,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1709921687.6130941,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1709921687.6184101,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1709921687.6233823,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1709921687.6283212,"name":"offline","context":{"idset":"121"}}
+{"timestamp":1709921687.6332593,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1709921687.6381054,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1709921687.6429164,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1709921687.6438584,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1709921687.6447978,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1709921687.6457453,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1709921687.6484759,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1709921687.6494164,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1709921687.6528652,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1709921687.6537998,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1709921687.6547296,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1709921687.6556578,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1709921687.6565745,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1709921687.657496,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1709921687.6584218,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1709921687.6593492,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1709921687.6602547,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1709921687.6611726,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1709921687.6620905,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1709921687.6630106,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1709921687.6639345,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1709921687.6648469,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1709921687.6657517,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1709921687.666661,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1709921687.6675751,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1709921687.6684771,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1709921687.6693778,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1709921687.67027,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1709921687.6721785,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1709921687.6759422,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1709921687.6768453,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1709921687.6777363,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1709921687.6786268,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1709921687.679518,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1709921687.680409,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1709921687.6812875,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1709921687.6821766,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1709921687.6840649,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1709921687.687763,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1709921687.691848,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1709921687.6922433,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1709921687.6963935,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1709921687.7005279,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1709921687.7046967,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1709921687.7052832,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1709921687.709491,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1709921687.7149265,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1709921687.715512,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1709921687.7197435,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1709921687.7203696,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1709921687.7250051,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1709921687.7302387,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1709921687.7353597,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1709921687.7407351,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1709921687.7459755,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1709921687.755759,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1709921687.7604704,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1709921687.7649033,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1709921687.7735906,"name":"offline","context":{"idset":"450"}}
+{"timestamp":1709921687.7784324,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1709921687.7851419,"name":"offline","context":{"idset":"468"}}
+{"timestamp":1709921687.7914896,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1709921687.7959764,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1709921687.8005667,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1709921687.8090527,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1709921687.8132217,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1709921687.8175297,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1709921687.8256636,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1709921687.8299527,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1709921687.8343084,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1709921687.8384624,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1709921687.8437784,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1709921687.8558867,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1709921687.8620269,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1709921687.8628125,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1709921687.8678613,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1709921687.8721237,"name":"offline","context":{"idset":"552"}}
+{"timestamp":1709921687.8777924,"name":"offline","context":{"idset":"553"}}
+{"timestamp":1709921687.8895504,"name":"offline","context":{"idset":"555"}}
+{"timestamp":1709921687.8954601,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1709921687.9024138,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1709921687.9130862,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1709921687.9191053,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1709921687.9259951,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1709921687.9378119,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1709921687.9437103,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1709921687.9497843,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1709921687.9613857,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1709921687.9683909,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1709921687.9744039,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1709921687.9808521,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1709921687.9933083,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1709921688.0009451,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1709921688.0064776,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1709921688.0140224,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1709921688.0194957,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1709921688.024097,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1709921688.0277679,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1709921688.033694,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1709921688.0383353,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1709921688.0423052,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1709921688.0462489,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1709921688.0538049,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1709921688.0576775,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1709921688.0615292,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1709921688.0694873,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1709921688.0732627,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1709921688.0771646,"name":"offline","context":{"idset":"654"}}
+{"timestamp":1709921688.0846536,"name":"offline","context":{"idset":"666"}}
+{"timestamp":1709921688.0884795,"name":"offline","context":{"idset":"672"}}
+{"timestamp":1709921688.0922482,"name":"offline","context":{"idset":"681"}}
+{"timestamp":1709921688.0996156,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1709921688.1033762,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1709921688.107096,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1709921688.1144617,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1709921688.1183233,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1709921688.1220624,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1709921688.1258178,"name":"offline","context":{"idset":"706"}}
+{"timestamp":1709921688.1338232,"name":"offline","context":{"idset":"707"}}
+{"timestamp":1709921688.1375837,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1709921688.137928,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1709921688.1414952,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1709921688.1452987,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1709921688.1456387,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1709921688.149189,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1709921688.1512053,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1709921688.1545477,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1709921688.1582377,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1709921688.1608756,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1709921688.1632857,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1709921688.1657057,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1709921688.1681211,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1709921688.1705239,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1709921688.1733503,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1709921688.1798346,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1709921688.1839349,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1709921688.1866822,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1709921688.2271142,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1709921688.2341578,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1709921688.2377751,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1709921688.2413588,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1709921688.2483876,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1709921688.2518802,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1709921688.2558932,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1709921688.2626121,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1709921688.2665412,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1709921688.2699997,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1709921688.2722311,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1709921688.2744908,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1709921688.2766807,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1709921688.2789099,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1709921688.2811913,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1709921688.2834094,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1709921688.2842343,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1709921688.2865777,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1709921688.2869492,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1709921688.2899737,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1709921688.2933221,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1709921688.2936885,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1709921688.296771,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1709921688.3000596,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1709921688.3004436,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1709921688.3008006,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1709921688.3035798,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1709921688.3039548,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1709921688.3043249,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1709921688.3046801,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1709921688.3050318,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1709921688.3053911,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1709921688.3057423,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1709921688.3060904,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1709921688.3064516,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1709921688.3068025,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1709921688.3077776,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1709921688.3143222,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1709921688.3175128,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1709921688.3207676,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1709921688.3270538,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1709921688.3302419,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1709921688.3305976,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1709921688.3335798,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1709921688.3357205,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1709921688.3384621,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1709921688.3405588,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1709921688.3427134,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1709921688.3447659,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1709921688.3469653,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1709921688.348578,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1709921688.3496933,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1709921688.3500357,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1709921688.3528478,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1709921688.3559844,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1709921688.3563366,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1709921688.3589976,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1709921688.3593273,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1709921688.3620718,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1709921688.3652968,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1709921688.365624,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1709921688.3692055,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1709921688.3695309,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1709921688.3722258,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1709921688.3772852,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1709921688.3822501,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1709921688.3921475,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1709921688.3955736,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1709921688.3984132,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1709921688.4043934,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1709921688.4073677,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1709921688.4103155,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1709921688.4160256,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1709921688.4191172,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1709921688.4219849,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1709921688.4273305,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1709921688.430666,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1709921688.4333913,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1709921688.4398711,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1709921688.4429078,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1709921688.4459224,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1709921688.4509737,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1709921688.4532244,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1709921688.4535201,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1709921688.455987,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1709921688.4562819,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1709921688.4588346,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1709921688.4614859,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1709921688.461787,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1709921688.4640512,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1709921688.4643424,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1709921688.4668689,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1709921688.4694405,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1709921688.4697225,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1709921688.4720345,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1709921688.4723272,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1709921688.4746504,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1709921688.4771709,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1709921688.4797041,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1709921688.4821823,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1709921688.4846306,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1709921688.4893959,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1709921688.4918296,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1709921688.4942498,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1709921688.4991417,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1709921688.5015187,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1709921688.5038874,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1709921688.5086327,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1709921688.5109854,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1709921688.5133088,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1709921688.5178726,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1709921688.5207744,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1709921688.5233431,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1709921688.5278428,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1709921688.5301199,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1709921688.5324097,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1709921688.5349448,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1709921688.5372052,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1709921688.537466,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1709921688.5395689,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1709921688.541929,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1709921688.5429409,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1709921688.5451381,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1709921688.5468087,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1709921688.5503833,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1709921688.5537,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1709921688.5572734,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1709921688.5610578,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1709921688.5649023,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1709921688.5685039,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1709921688.5738416,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1709921688.5760796,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1709921688.578207,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1709921688.5822785,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1709921688.5843372,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1709921688.5863779,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1709921688.5903518,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1709921688.5923996,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1709921688.5943646,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1709921688.5982702,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1709921688.6002567,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1709921688.6022272,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1709921688.6060543,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1709921688.608047,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1709921688.6099896,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1709921688.6137435,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1709921688.6157138,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1709921688.6180277,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1709921688.6220529,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1709921688.6239536,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1709921688.6258519,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1709921688.6295109,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1709921688.6313708,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1709921688.6332192,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1709921688.636837,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1709921688.6370475,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1709921688.6387446,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1709921688.6405699,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1709921688.6407773,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1709921688.6424825,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1709921688.6426902,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1709921688.6443813,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1709921688.6462533,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1709921688.646606,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1709921688.6482286,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1709921688.6500838,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1709921688.6503007,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1709921688.6520059,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1709921688.6538279,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1709921688.6541069,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1709921688.6557324,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1709921688.65593,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1709921688.6576567,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1709921688.65978,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1709921688.6612766,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1709921688.6627517,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1709921688.6645162,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1709921688.6668179,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1709921688.6688738,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1709921688.6717036,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1709921688.6737487,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1709921688.6757991,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1709921688.6785645,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1709921688.6800013,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1709921688.6817484,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1709921688.6850295,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1709921688.6867414,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1709921688.6884017,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1709921688.6917191,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1709921688.6933525,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1709921688.69484,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1709921688.6979828,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1709921688.6999943,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1709921688.7019227,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1709921688.7051761,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1709921688.7067463,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1709921688.7083149,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1709921688.7113557,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1709921688.7128851,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1709921688.7144253,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1709921688.7174065,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1709921688.7189193,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1709921688.720433,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1709921688.7234473,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1709921688.7249458,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1709921688.7264419,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1709921688.7266202,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1709921688.7280376,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1709921688.7298927,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1709921688.7300732,"name":"offline","context":{"idset":"445"}}
+{"timestamp":1709921688.7314725,"name":"offline","context":{"idset":"447"}}
+{"timestamp":1709921688.7329507,"name":"offline","context":{"idset":"451"}}
+{"timestamp":1709921688.7331257,"name":"offline","context":{"idset":"452"}}
+{"timestamp":1709921688.7345655,"name":"offline","context":{"idset":"453"}}
+{"timestamp":1709921688.7360723,"name":"offline","context":{"idset":"454"}}
+{"timestamp":1709921688.7362435,"name":"offline","context":{"idset":"456"}}
+{"timestamp":1709921688.7376523,"name":"offline","context":{"idset":"457"}}
+{"timestamp":1709921688.7378252,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1709921688.7395089,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1709921688.7411206,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1709921688.741298,"name":"offline","context":{"idset":"462"}}
+{"timestamp":1709921688.7435143,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1709921688.7436848,"name":"offline","context":{"idset":"464"}}
+{"timestamp":1709921688.7451031,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1709921688.7465382,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1709921688.7467067,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1709921688.7481008,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1709921688.7482677,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1709921688.7496307,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1709921688.7510982,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1709921688.7525659,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1709921688.7540078,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1709921688.7569597,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1709921688.7583678,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1709921688.7598758,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1709921688.7627361,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1709921688.7640605,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1709921688.7654684,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1709921688.768424,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1709921688.7697921,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1709921688.7711558,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1709921688.7740564,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1709921688.7757645,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1709921688.777132,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1709921688.7797806,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1709921688.7811363,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1709921688.7824988,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1709921688.785111,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1709921688.7865446,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1709921688.7881937,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1709921688.7909963,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1709921688.7923806,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1709921688.7937148,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1709921688.7969134,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1709921688.7981915,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1709921688.7998867,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1709921688.8027492,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1709921688.8044682,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1709921688.8062198,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1709921688.8085556,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1709921688.8103566,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1709921688.8119373,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1709921688.8137665,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1709921688.8154392,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1709921688.8170393,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1709921688.8187563,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1709921688.8200858,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1709921688.821666,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1709921688.8240826,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1709921688.825721,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1709921688.8270867,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1709921688.8295057,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1709921688.8307154,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1709921688.8319116,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1709921688.8342233,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1709921688.8353984,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1709921688.8365862,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1709921688.8389533,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1709921688.8401232,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1709921688.8412857,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1709921688.8424475,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1709921688.8425972,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1709921688.8436766,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1709921688.844831,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1709921688.8449707,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1709921688.8460436,"name":"offline","context":{"idset":"549"}}
+{"timestamp":1709921688.8461812,"name":"offline","context":{"idset":"551"}}
+{"timestamp":1709921688.8472533,"name":"offline","context":{"idset":"554"}}
+{"timestamp":1709921688.8484051,"name":"offline","context":{"idset":"556"}}
+{"timestamp":1709921688.848541,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1709921688.8496103,"name":"offline","context":{"idset":"558"}}
+{"timestamp":1709921688.8507583,"name":"offline","context":{"idset":"559"}}
+{"timestamp":1709921688.8508921,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1709921688.851985,"name":"offline","context":{"idset":"561"}}
+{"timestamp":1709921688.8531005,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1709921688.8532369,"name":"offline","context":{"idset":"563"}}
+{"timestamp":1709921688.854558,"name":"offline","context":{"idset":"564"}}
+{"timestamp":1709921688.855679,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1709921688.8558137,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1709921688.856849,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1709921688.8569818,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1709921688.8580117,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1709921688.8590987,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1709921688.8592269,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1709921688.8603392,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1709921688.8624101,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1709921688.8634884,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1709921688.8645887,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1709921688.8666229,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1709921688.8676732,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1709921688.8687158,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1709921688.8707163,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1709921688.8717802,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1709921688.8727756,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1709921688.8750308,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1709921688.876121,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1709921688.877197,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1709921688.8791332,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1709921688.8801594,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1709921688.8811927,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1709921688.8831172,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1709921688.8841012,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1709921688.885093,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1709921688.8869655,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1709921688.8879845,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1709921688.8889389,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1709921688.8907712,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1709921688.8917065,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1709921688.8926697,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1709921688.8944955,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1709921688.8954234,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1709921688.8963969,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1709921688.8983569,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1709921688.8993158,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1709921688.9001877,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1709921688.9019094,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1709921688.9028363,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1709921688.9040148,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1709921688.905802,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1709921688.906678,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1709921688.9075487,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1709921688.9092019,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1709921688.9100597,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1709921688.9109178,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1709921688.9125392,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1709921688.9133844,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1709921688.9141998,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1709921688.9157875,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1709921688.9166691,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1709921688.9176133,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1709921688.9191608,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1709921688.9199588,"name":"offline","context":{"idset":"655"}}
+{"timestamp":1709921688.9207995,"name":"offline","context":{"idset":"656"}}
+{"timestamp":1709921688.9222624,"name":"offline","context":{"idset":"657"}}
+{"timestamp":1709921688.9230635,"name":"offline","context":{"idset":"658"}}
+{"timestamp":1709921688.9238415,"name":"offline","context":{"idset":"660"}}
+{"timestamp":1709921688.925318,"name":"offline","context":{"idset":"662"}}
+{"timestamp":1709921688.9260836,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1709921688.9268248,"name":"offline","context":{"idset":"665"}}
+{"timestamp":1709921688.9282837,"name":"offline","context":{"idset":"667"}}
+{"timestamp":1709921688.9291527,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1709921688.929882,"name":"offline","context":{"idset":"669"}}
+{"timestamp":1709921688.9312785,"name":"offline","context":{"idset":"670"}}
+{"timestamp":1709921688.9320378,"name":"offline","context":{"idset":"671"}}
+{"timestamp":1709921688.9327586,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1709921688.9341202,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1709921688.9348369,"name":"offline","context":{"idset":"675"}}
+{"timestamp":1709921688.9357886,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1709921688.9373825,"name":"offline","context":{"idset":"679"}}
+{"timestamp":1709921688.9382327,"name":"offline","context":{"idset":"680"}}
+{"timestamp":1709921688.9389327,"name":"offline","context":{"idset":"682"}}
+{"timestamp":1709921688.9402077,"name":"offline","context":{"idset":"683"}}
+{"timestamp":1709921688.9408867,"name":"offline","context":{"idset":"685"}}
+{"timestamp":1709921688.9416382,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1709921688.9430268,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1709921688.9437025,"name":"offline","context":{"idset":"690"}}
+{"timestamp":1709921688.9443445,"name":"offline","context":{"idset":"692"}}
+{"timestamp":1709921688.9455888,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1709921688.9462206,"name":"offline","context":{"idset":"694"}}
+{"timestamp":1709921688.9468925,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1709921688.9481952,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1709921688.9488277,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1709921688.9494798,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1709921688.9506969,"name":"offline","context":{"idset":"701"}}
+{"timestamp":1709921688.95139,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1709921688.9520171,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1709921688.9532113,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1709921688.9542601,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1709921688.9550536,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1709921688.9562364,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1709921688.9568708,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1709921688.9574883,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1709921688.9586165,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1709921688.9592192,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1709921688.9598441,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1709921688.9609404,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1709921688.9615366,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1709921688.962131,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1709921688.9631901,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1709921688.9637671,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1709921688.9643569,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1709921688.9656146,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1709921688.9669218,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1709921688.9682379,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1709921688.9695413,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1709921688.9708369,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1709921688.9721346,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1709921688.9734366,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1709921688.9747288,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1709921688.9760196,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1709921688.9773164,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1709921688.9782999,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1709921688.982213,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1709921688.9822855,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1709921688.9823589,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1709921688.9824235,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1709921688.9824877,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1709921688.9825521,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1709921688.9826167,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1709921688.9826832,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1709921688.9827447,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1709921688.9828053,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1709921688.9828656,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1709921688.9829257,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1709921688.9829943,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1709921688.9830613,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1709921688.9831214,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1709921688.9831791,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1709921688.9832428,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1709921688.9833105,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1709921688.9833677,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1709921688.983423,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1709921688.9834785,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1709921688.9835339,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1709921688.983588,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1709921688.9836407,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1709921688.9836924,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1709921688.9837449,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1709921688.9837968,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1709921688.9838483,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1709921688.9838996,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1709921688.9839509,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1709921688.9840038,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1709921688.9843247,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1709921688.9846978,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1709921688.9850738,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1709921688.9854567,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1709921688.9858279,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1709921688.9864795,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1709921688.9868317,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1709921688.9871809,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1709921688.9878001,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1709921688.9881403,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1709921688.9884887,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1709921688.9890692,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1709921688.989435,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1709921688.9897952,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1709921688.9903541,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1709921688.9906542,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1709921688.9909368,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1709921688.9914699,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1709921688.9917512,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1709921688.9920201,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1709921688.9925096,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1709921688.9927735,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1709921688.9930403,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1709921688.9933238,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1709921688.9935849,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1709921688.9938359,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1709921688.9940827,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1709921688.9943423,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1709921688.9945886,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1709921688.9948316,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1709921688.99506,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1709921688.9952841,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1709921688.995527,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1709921688.9957464,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1709921688.9959626,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1709921688.9961715,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1709921688.9963884,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1709921688.9965897,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1709921688.9967873,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1709921688.996979,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1709921688.9971676,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1709921688.9973586,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1709921688.9975519,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1709921688.9977288,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1709921688.9979188,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1709921688.9980752,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1709921688.9982393,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1709921688.9983873,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1709921688.9985545,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1709921688.9987035,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1709921688.998848,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1709921688.9989851,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1709921688.9991438,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1709921688.9993017,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1709921688.9994364,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1709921688.9995666,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1709921688.9996874,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1709921688.9998021,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1709921688.9999132,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1709921689.0000186,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1709921689.0022609,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1709921788.8161356,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"99":{"timestamp":1709858832.2912085,"reason":"reason=HPE Replacing Rabbit"},"348":{"timestamp":1709866839.8394222,"reason":"reason=BAD WRITE CHECKSUM"},"550":{"timestamp":1709336806.3089643,"reason":"reason=Awaiting MAC from Py"},"624,626":{"timestamp":1709917499.1823676,"reason":"drained by /admin/scripts/drain_node"},"663":{"timestamp":1709867072.3431404,"reason":"nodediag failed cxi"}},"online":"","exclude":"0"}}
+{"timestamp":1709921788.8212729,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1709921811.790669,"name":"online","context":{"idset":"0"}}
+{"timestamp":1709921812.7071695,"name":"online","context":{"idset":"1,8,18,24,28,30,45,47-49,54,59"}}
+{"timestamp":1709921813.1399589,"name":"online","context":{"idset":"348,663,755-756"}}
+{"timestamp":1709921814.4294171,"name":"online","context":{"idset":"583,745"}}
+{"timestamp":1709921814.6730199,"name":"online","context":{"idset":"184"}}
+{"timestamp":1709921814.8234951,"name":"online","context":{"idset":"222,395,729"}}
+{"timestamp":1709921815.1411057,"name":"online","context":{"idset":"190,344,675"}}
+{"timestamp":1709921815.3598888,"name":"online","context":{"idset":"331,503,677"}}
+{"timestamp":1709921815.4766703,"name":"online","context":{"idset":"125,131,154,198,351,630,643"}}
+{"timestamp":1709921815.5989549,"name":"online","context":{"idset":"188,235,259,301,378,391,483,494,708"}}
+{"timestamp":1709921815.7257645,"name":"online","context":{"idset":"141,158,263,330,383,447,449"}}
+{"timestamp":1709921815.8462126,"name":"online","context":{"idset":"121,457,464,553,722"}}
+{"timestamp":1709921815.963131,"name":"online","context":{"idset":"64,70,202,340,446,524,536,621"}}
+{"timestamp":1709921816.0642796,"name":"online","context":{"idset":"82,128,178,203,229,249,257,319,329,364,397,412,433,436,461-462,518,520,530,589,650,661,676,724,731,734"}}
+{"timestamp":1709921816.1809652,"name":"online","context":{"idset":"2,75,85,113,123,139,159,164,193,196,213,223,241,247,266,285,304,323,342,360,375,416,444,448,453,455,495,504-505,581,607,609,673,691,714"}}
+{"timestamp":1709921816.2816827,"name":"online","context":{"idset":"111,129,147,152,155,179,209,212,227,260,265,305,312,316,362,458,468,512,555,585,587,611,624,647,653,659,711,730,737,744,746"}}
+{"timestamp":1709921816.3996327,"name":"online","context":{"idset":"122,264,288,355,384,409,414,430,480,532,569,602,641,706"}}
+{"timestamp":1709921816.5019033,"name":"online","context":{"idset":"74,106,156,236,333,338,358,361,372,386,411,478,489,537,619,644"}}
+{"timestamp":1709921816.6234617,"name":"online","context":{"idset":"80,84,142,191,261,286,298,302,402,435,450-451,481,508,515,529,535,546,567,570,600,612,681,693,707,715,733"}}
+{"timestamp":1709921816.7263649,"name":"online","context":{"idset":"107,117,130,132-133,144,166,173,176,181,187,211,217-218,254,290,310,313,317,324,359,388,404,424,454,475,492,519,521,531,543,571,575,577,594,597,637,651,666,671,680,720,728,732"}}
+{"timestamp":1709921816.8286397,"name":"online","context":{"idset":"61,101,105,115,160,199,206,253,267,314,318,334,373,547,557,578,580,603,633,688,753"}}
+{"timestamp":1709921816.9292467,"name":"online","context":{"idset":"3,87,124,232,287,307,325,369,374,389,392,401,410,434,517,528,533,545,572,584,588,592,595,605,614,642,654,712,718,749"}}
+{"timestamp":1709921817.0337224,"name":"online","context":{"idset":"4,79,92,100,109,114,140,205,269,281,327,335,341,343,377,405,407,417,421,465,476,502,506-507,564-565,596,632,634,638,656,664,672,684,697,699,704,725,736,747,750"}}
+{"timestamp":1709921817.1577742,"name":"online","context":{"idset":"71,104,118-119,167-168,172,185,195,243,256,278,280,294,308,320,337,356,393,479,482,485,490,499,540,586,591,601,617,622,631,635,645,687,709,738"}}
+{"timestamp":1709921817.2621017,"name":"online","context":{"idset":"67,77,93,103,135-136,143,145,153,157,171,177,197,214,216,252,270,273-274,276,297,309,322,350,354,366-368,370,379,400,406,418,420,422,426,432,441,509,523,549,559,561,573,579,593,599,613,625,627,662,679,703,717,721,726,741,754"}}
+{"timestamp":1709921817.3637989,"name":"online","context":{"idset":"66,73,95,137,146,165,169,174-175,183,192,207,219,225,228,231,233,291-293,299,332,365,380,439,445,469,488,522,538,542,552,556,558,560,590,604,626,646,665,682,685,689,700,710,713"}}
+{"timestamp":1709921817.4826703,"name":"online","context":{"idset":"69,81,86,88,97-98,108,134,170,200,210,220-221,226,234,239-240,245-246,255,277,283,289,296,328,336,347,353,357,376,385,387,390,425,428,440,456,460,467,470,477,513,516,525,534,539,541,562,576,598,608,616,623,652,657-658,660,668-669,674,683,690,695-696,701,716,735,740,742,748,752"}}
+{"timestamp":1709921817.5860169,"name":"online","context":{"idset":"72,96,112,120,150,162,186,201,208,237,271,311,315,321,339,345-346,349,352,381,394,398,415,419,423,427,459,463,471,487,493,496-497,510,526,548,554,574,610,628,636,655,692,739,743"}}
+{"timestamp":1709921817.6869226,"name":"online","context":{"idset":"63,68,89-90,94,102,110,127,148,151,242,244,250,272,275,382,399,429,431,474,486,566,568,582,618,648,667,670,694,723"}}
+{"timestamp":1709921817.7942581,"name":"online","context":{"idset":"65,78,189,224,238,248,262,279,282,326,498,606,620,686,705,719,751"}}
+{"timestamp":1709921817.9161818,"name":"online","context":{"idset":"76,163,251,258,284,303,396,442,452,473,491,500,511,563,615,629,640,698,727"}}
+{"timestamp":1709921818.0250857,"name":"online","context":{"idset":"295,484,501,527,544,551,649,678"}}
+{"timestamp":1709921818.1323962,"name":"online","context":{"idset":"83,91,116,138,149,161,180,182,194,204,215,230,300,306,403,408,413,437-438,443,472,702"}}
+{"timestamp":1709921818.2337136,"name":"online","context":{"idset":"62,126,363,514"}}
+{"timestamp":1709925207.659452,"name":"undrain","context":{"idset":"624,626"}}
+{"timestamp":1709937391.2303963,"name":"drain","context":{"idset":"557","reason":"broker was unresponsive"}}
+{"timestamp":1709937457.2286365,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1710134373.7191205,"name":"drain","context":{"idset":"61-98,100-125","reason":"drained by /admin/scripts/drain_node","overwrite":0}}
+{"timestamp":1710134434.2267177,"name":"undrain","context":{"idset":"61-98,100-125"}}
+{"timestamp":1710136167.4201007,"name":"drain","context":{"idset":"121","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1710195214.2853191,"name":"online","context":{"idset":"557"}}
+{"timestamp":1710195266.0155272,"name":"online","context":{"idset":"550"}}
+{"timestamp":1710195309.4781427,"name":"online","context":{"idset":"99"}}
+{"timestamp":1710195997.9093578,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1709425589.9902151,"reason":"prolog failed for jobid fgNs1vQEBm9"},"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"99":{"timestamp":1709858832.2912085,"reason":"reason=HPE Replacing Rabbit"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"348":{"timestamp":1709866839.8394222,"reason":"reason=BAD WRITE CHECKSUM"},"550":{"timestamp":1709336806.3089643,"reason":"reason=Awaiting MAC from Py"},"557":{"timestamp":1709937391.2303963,"reason":"broker was unresponsive"},"663":{"timestamp":1709867072.3431404,"reason":"nodediag failed cxi"}},"online":"","exclude":"0"}}
+{"timestamp":1710195997.9100885,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1710196020.2737815,"name":"online","context":{"idset":"0"}}
+{"timestamp":1710196021.1143703,"name":"online","context":{"idset":"1,8,18,24,28,30,45,47-49,54,59"}}
+{"timestamp":1710196021.6178174,"name":"online","context":{"idset":"99,348,550,557,663"}}
+{"timestamp":1710196021.7927275,"name":"online","context":{"idset":"238"}}
+{"timestamp":1710196021.916487,"name":"online","context":{"idset":"2,123,353,640,708,728,741"}}
+{"timestamp":1710196022.0386817,"name":"online","context":{"idset":"3-4,77,111,121,125,133,154-155,174,184,209,212,239,241,253,257,259,269,271,297,310,317,330,335,343,360,372,375,402,406,417,431,433,435-436,442,446,449,462,479,526,559,570,582,585,588-589,600,611-612,636,643,660,673,676-677,688,700,709,729-730,742"}}
+{"timestamp":1710196022.1391337,"name":"online","context":{"idset":"70,78,85,87,93,104,106,113-115,129,131,139-141,152,163,186,193,202-203,207,211,222,227,229,260-261,265,278,298,304,312,323,329,342,351,355-356,364,385,412,416,422,444,448,457,468,481,494-495,512,515,518,552-553,575,583,594,596,609,624,631,633,641,646,653,658,671-672,686,691,706-707,721-722,731,734,737,739,744-746"}}
+{"timestamp":1710196022.240921,"name":"online","context":{"idset":"64,71,73,75,80,82,92,105,108,117-118,124,132,147-148,157,164,179,185,187-188,190,196,198-199,206,213,217-218,223,231,247,264,276,285,288,291,299,302,305,314,316,320,324-325,338-339,341,344,361-362,365,367,374,384,388,395,397,409-411,414,430,450-451,453,455,458,461,464,485,488-489,502-505,522,524,529-532,536-537,549,551,555-556,565,567,569,574,576,581,584,587,590-591,605,607-608,619,621-622,630,635,649-650,659,669,681-682,711,715,720,733,743"}}
+{"timestamp":1710196022.3468668,"name":"online","context":{"idset":"61,65,67,69,84,88,94,100,107,109,119,128,130,137,143-144,151,156,160,166,168,173,176-178,180-181,195,197,205,216,219,221,224-225,235-236,240,243,246,249-250,252,256,262,266,270,273,280,282,292,294,296,301,308,313,318-319,322,328,334,337,340,352,357-359,368-370,373,377-378,381-382,386,389,392,394,396,398,404-405,418,420-421,424,426-427,447,456,459,469,478,480,482-483,490-492,497,499,501,506,509,517,519,523,528,533,541-542,545-547,558,560,564,571,573,577-578,580,593,601-602,625,629,637,644-645,647,651,664,666,668,674-675,683-684,687,693,695,697,703,714,718-719,724,726,736,748,750"}}
+{"timestamp":1710196022.4484434,"name":"online","context":{"idset":"63,66,68,72,74,76,79,81,83,86,89-91,95-98,101-103,112,116,120,122,126-127,134-136,142,145,149-150,153,158-159,161-162,165,167,169-172,175,182-183,189,191-192,200-201,204,208,210,214-215,220,226,228,230,232-234,237,242,244-245,248,251,254-255,258,263,267,272,274-275,277,279,281,283-284,286-287,289-290,293,295,300,303,306-307,309,311,315,321,326-327,331-333,336,345-347,349-350,354,363,366,376,379-380,383,387,390-391,393,399-401,403,407-408,415,419,423,425,428-429,432,434,437-441,443,445,452,454,460,463,465,467,470-477,484,486-487,493,496,498,500,507-508,510-511,513-514,516,520-521,525,527,534-535,538-540,543-544,548,554,561-563,566,568,572,579,586,592,595,597-599,603-604,606,610,613-617,620,623,626-628,632,634,638,642,648,652,654-657,661-662,665,667,670,678-680,685,689-690,692,694,696,698-699,701-702,704-705,710,712-713,716-717,723,725,727,732,735,738,740,747,749,751,753"}}
+{"timestamp":1710196022.5498118,"name":"online","context":{"idset":"62,110,138,146,194,413,618,752,754-755"}}
+{"timestamp":1710196022.6655247,"name":"online","context":{"idset":"756"}}
+{"timestamp":1710196240.863158,"name":"undrain","context":{"idset":"99"}}
+{"timestamp":1710196255.9192953,"name":"undrain","context":{"idset":"550"}}
+{"timestamp":1710196264.9345191,"name":"undrain","context":{"idset":"557"}}
+{"timestamp":1710196386.9142146,"name":"undrain","context":{"idset":"1"}}
+{"timestamp":1710199393.6852264,"name":"drain","context":{"idset":"709-756","reason":"reason=HPE PFG testing","overwrite":0}}
+{"timestamp":1710199586.6126552,"name":"online","context":{"idset":"268"}}
+{"timestamp":1710199645.8546257,"name":"online","context":{"idset":"371,466,639"}}
+{"timestamp":1710302700.1621139,"name":"undrain","context":{"idset":"709-756"}}
+{"timestamp":1710307679.7246292,"name":"resource-init","context":{"restart":true,"drain":{"39":{"timestamp":1709248366.7295237,"reason":"reason=Flipped SS Y cable with merced255"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"348":{"timestamp":1709866839.8394222,"reason":"reason=BAD WRITE CHECKSUM"},"663":{"timestamp":1709867072.3431404,"reason":"nodediag failed cxi"}},"online":"","exclude":"0"}}
+{"timestamp":1710307679.7255821,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1710307703.1810915,"name":"online","context":{"idset":"0"}}
+{"timestamp":1710307703.9566877,"name":"online","context":{"idset":"41,45,48,60"}}
+{"timestamp":1710307704.0721338,"name":"online","context":{"idset":"5,15,20-21,23,25,32-33,38,42-44,46-47,49-59"}}
+{"timestamp":1710307704.2450058,"name":"online","context":{"idset":"121"}}
+{"timestamp":1710307704.4770396,"name":"online","context":{"idset":"348,663"}}
+{"timestamp":1710307704.8160534,"name":"online","context":{"idset":"753"}}
+{"timestamp":1710307704.9946575,"name":"online","context":{"idset":"91,285,292,552,623"}}
+{"timestamp":1710307705.1122773,"name":"online","context":{"idset":"95,180,269,378"}}
+{"timestamp":1710307705.2358296,"name":"online","context":{"idset":"187,195,215,257,268,352,457,531,554,728"}}
+{"timestamp":1710307705.350106,"name":"online","context":{"idset":"114,118,235,242,328,461,494,567,569,579,581,635,730,746"}}
+{"timestamp":1710307705.4521666,"name":"online","context":{"idset":"70,123,151,155,211,264,323,332,361,374,377,414,425,433,503,523,536,541,585,589,605,607,611,631,659,672"}}
+{"timestamp":1710307705.5691893,"name":"online","context":{"idset":"125,131,223,262,298,312,405,444,495,573,619,653,671,676,716"}}
+{"timestamp":1710307705.6903055,"name":"online","context":{"idset":"13,73,82,104,111,113,128,132,139,154,177,202,209-210,216,219,252,256,259,261,272,302,308,316,364,372,379,382,384,403,409,432,439,448,482,489,505-506,532,553,555,587,601,614,638,650,654,656,677-678,682,694,700,724,745,750"}}
+{"timestamp":1710307705.8514047,"name":"online","context":{"idset":"19,35,75,78,84,89-90,106,109,119,129,137,147-148,156,160,164,168,178-179,192,214,231,236,238,241,247,253,273,287,291,299,318,335,338,356,362,368,386,392,399,416-417,420,430,447,458,462,467,483,501,504,508,526,529,537,547,582,584,588,600,643,651-652,681,684,690-691,697,704,725"}}
+{"timestamp":1710307705.9524868,"name":"online","context":{"idset":"6,10,18,30,37,68,77,83,99,103,112,130,152,157,173,175,183-185,191,197,199,207,212,232,234,249,278,280,288-289,300,325,329,336,343-344,350,385,388-389,391,394,402,410,418,424,434,436,445-446,449-452,464,481,486,515,518,525,540,549,558,563,570,592,596,616,621,625,632,634,636,649,661,675,683,689,696,706,708,711,713,726,729,733,736,744"}}
+{"timestamp":1710307706.0565305,"name":"online","context":{"idset":"8,12,26,28,85,120,127,133,162,213,265,294,314,320,327,360,470,484,516,557,609,630,644,666,741"}}
+{"timestamp":1710307706.1798496,"name":"online","context":{"idset":"4,7,11,14,16,31,61,86-88,92-93,98,100,102,108,126,134,142-143,165,174,188,193,196,203,205-206,217,221-222,227,233,239,254,260,263,279,304,309-310,313,315,334,341,353-354,358,375,395,398,400-401,411,422,428,431,442,454-455,468-469,476-477,485,512,521-522,527,538,543-544,546,560,562,572,574,576,590-591,593,608,612,617,624,629,637,642,646,655,657-658,660,668-669,673,680,687,695,699,707,710,715,720,722,727,735,738,747,755"}}
+{"timestamp":1710307706.2837448,"name":"online","context":{"idset":"2-3,17,24,36,40,64,71,80,94,97,115,117,124,140-141,145,158,166,170,176,181,189-190,218,224,240,245,250,258,267,275,277,281,283,296,301,305,324,330,342,351,357,359,365,367,371,376,387,396,406,426,440,456,472,474,478,490,499,502,511,517,530,533,539,561,565,571,577,583,594,613,633,641,670,688,731,734,737"}}
+{"timestamp":1710307706.3867507,"name":"online","context":{"idset":"9,22,65,72,79,101,116,146,150,163,220,237,243,246,274,282,295,317,321,337,340,355,370,373,397,438,475,479,507,580,603,622,647,662,674,686,702,709,714,717,732,754"}}
+{"timestamp":1710307706.490521,"name":"online","context":{"idset":"27,29,34,63,67,69,81,107,135-136,138,153,159,161,171-172,201,208,226,248,251,270,286,290,307,319,346,369,380,393,419,435,453,460,463,466,491-493,509-510,514,519-520,535,542,556,578,602,615,620,627,640,667,679,693,703,718,721,739-740,742,748"}}
+{"timestamp":1710307706.6078379,"name":"online","context":{"idset":"96,122,144,149,169,182,186,198,200,225,228-230,255,266,271,293,297,303,311,322,326,331,333,339,345,349,363,366,381,383,390,404,412-413,415,421,427,429,441,443,459,465,471,480,487-488,496-498,513,524,528,534,545,548,550-551,559,564,566,575,586,595,597-598,604,606,610,626,639,645,664-665,685,701,705,712,719,723,743,749,751,756"}}
+{"timestamp":1710307706.7091682,"name":"online","context":{"idset":"1,62,66,74,76,105,110,167,194,204,276,347,407,568,599,618,628,648,692,698,752"}}
+{"timestamp":1710307706.8404372,"name":"online","context":{"idset":"423"}}
+{"timestamp":1710307706.9574838,"name":"online","context":{"idset":"244,408,437,473"}}
+{"timestamp":1710307707.0586562,"name":"online","context":{"idset":"284,306,500"}}
+{"timestamp":1710350462.5830762,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1710351292.5834293,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1710351358.5816128,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1710367935.7172248,"name":"online","context":{"idset":"39"}}
+{"timestamp":1710367941.968823,"name":"undrain","context":{"idset":"39"}}
+{"timestamp":1710368400.7770364,"name":"undrain","context":{"idset":"97"}}
+{"timestamp":1710369309.1457515,"name":"online","context":{"idset":"353"}}
+{"timestamp":1710369322.0631607,"name":"undrain","context":{"idset":"353"}}
+{"timestamp":1710371144.0952022,"name":"drain","context":{"idset":"755","reason":"reason=MPI Failure","overwrite":0}}
+{"timestamp":1710371148.4794686,"name":"drain","context":{"idset":"756","reason":"reason=MPI Failure","overwrite":0}}
+{"timestamp":1710371658.3311131,"name":"drain","context":{"idset":"753","reason":"reason=MPI Failure","overwrite":0}}
+{"timestamp":1710371696.1206582,"name":"drain","context":{"idset":"531","reason":"reason=MPI Failure","overwrite":0}}
+{"timestamp":1710372275.699584,"name":"drain","context":{"idset":"696-756","reason":"reason=MPI Failure","overwrite":1}}
+{"timestamp":1710372785.4881094,"name":"drain","context":{"idset":"664","reason":"reason=MPI Failure","overwrite":1}}
+{"timestamp":1710372978.5898032,"name":"drain","context":{"idset":"665","reason":"reason=MPI Failure","overwrite":1}}
+{"timestamp":1710373051.4201543,"name":"drain","context":{"idset":"660-756","reason":"reason=MPI Failure","overwrite":1}}
+{"timestamp":1710373675.0398071,"name":"drain","context":{"idset":"238","reason":"reason=Cronjob lc-ansible-pull failed to complete","overwrite":0}}
+{"timestamp":1710373770.9666505,"name":"undrain","context":{"idset":"531,660-756"}}
+{"timestamp":1710373827.5937934,"name":"drain","context":{"idset":"663","reason":"reason=Cassini firmware is at 1.4.238, expected 1.5.41","overwrite":0}}
+{"timestamp":1710374039.7401021,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1710375011.6336055,"name":"online","context":{"idset":"238"}}
+{"timestamp":1710375026.8056121,"name":"undrain","context":{"idset":"238"}}
+{"timestamp":1710440162.7613657,"name":"drain","context":{"idset":"709-756","reason":"HPE Testing","overwrite":0}}
+{"timestamp":1710442769.7260041,"name":"drain","context":{"idset":"2","reason":"prolog failed for jobid fib2qTpngu5","overwrite":0}}
+{"timestamp":1710459891.7540319,"name":"undrain","context":{"idset":"709-756"}}
+{"timestamp":1710516112.5825348,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1710516150.4830616,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1710516150.5810568,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1710517720.5826221,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1710517784.5808043,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1710525744.5812607,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1710533382.1204836,"name":"online","context":{"idset":"348"}}
+{"timestamp":1710533900.5912287,"name":"undrain","context":{"idset":"348"}}
+{"timestamp":1710538272.5824068,"name":"drain","context":{"idset":"199","reason":"broker was unresponsive"}}
+{"timestamp":1710538336.5821183,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1710543041.6912529,"name":"online","context":{"idset":"570"}}
+{"timestamp":1710543055.8126793,"name":"undrain","context":{"idset":"570"}}
+{"timestamp":1710545965.9089558,"name":"online","context":{"idset":"134,167,717"}}
+{"timestamp":1710546344.6974525,"name":"drain","context":{"idset":"753","reason":"Observed errors last two nights","overwrite":0}}
+{"timestamp":1710549161.9161172,"name":"drain","context":{"idset":"734","reason":"connect() to 10.85.101.192:1039 failed","overwrite":0}}
+{"timestamp":1710695705.8661742,"name":"drain","context":{"idset":"750-752,754-756","reason":"IOR Errors","overwrite":0}}
+{"timestamp":1710697839.5410402,"name":"resource-init","context":{"restart":true,"drain":{"2":{"timestamp":1710442769.7260041,"reason":"prolog failed for jobid fib2qTpngu5"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"199":{"timestamp":1710538272.5824068,"reason":"broker was unresponsive"},"663":{"timestamp":1710373827.5937934,"reason":"reason=Cassini firmware is at 1.4.238, expected 1.5.41"},"734":{"timestamp":1710549161.9161172,"reason":"connect() to 10.85.101.192:1039 failed"},"753":{"timestamp":1710546344.6974525,"reason":"Observed errors last two nights"},"750-752,754-756":{"timestamp":1710695705.8661742,"reason":"IOR Errors"}},"online":"","exclude":"0"}}
+{"timestamp":1710697839.5466592,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1710697871.6012406,"name":"online","context":{"idset":"0"}}
+{"timestamp":1710697872.5648179,"name":"online","context":{"idset":"121"}}
+{"timestamp":1710697872.9125626,"name":"online","context":{"idset":"663"}}
+{"timestamp":1710697873.0583541,"name":"online","context":{"idset":"2,4,8-12,14,16,19,21,23-24,27,30-33,35,40,42,44,46-47,53-56,59,136,257,261,517,525"}}
+{"timestamp":1710697873.1619024,"name":"online","context":{"idset":"1,3,5-7,13,15,17,20,22,25-26,28-29,34,36,38-39,41,43,45,48-52,57-58,65,87,122,144,149,152,169,179,181,232,247,262,268,278,280,292,304,312,324,330,341,344,356,358,362,394,397-398,409,411,418,424,426,428,432,471,474,477,498,500,519,530-531,541,545,562-563,570,575,591,603,605,624,634,640,653,669,673-674,677,707,715-716,719,722,744"}}
+{"timestamp":1710697873.2640696,"name":"online","context":{"idset":"18,37,60,69,71-72,81,83,85,88-89,108-109,113,124,134,139,141,143,147-148,150,157,160,178,200,214,221,228,250,258,270,285,287,295,306,316,318,337-338,351,363-364,370-371,379,381,388,392,400,406,423,433,446,454,458,466,469,482,488,497,506,518,524,529,533,535-536,540,555,571-572,576,579,583,589-590,598-599,612,615,617-618,632,638,644-645,650,664,672,682,690,692,702-704,713,721,731-732"}}
+{"timestamp":1710697873.3705518,"name":"online","context":{"idset":"61,63-64,67,73-77,79,86,93,96-100,104-106,115,119,125-133,135,137,140,145,155-156,158-159,162-165,167-168,174-177,180,182-183,186,190,192-193,196-197,201-202,207,211,213,215,217-220,223-225,227,229-231,235,238-242,248-249,254-256,263-264,273-274,276,279,281,283-284,286,294,296,299,301-303,305,308,310-311,313-315,317,321-323,325-328,332-336,339-340,343,345-346,348-349,352-353,355,357,360-361,365-369,372,374,376,378,382-384,387,389-391,393,395-396,399,401,403-405,412-414,416-417,419-422,425,430-431,434-438,441-445,448-453,455-457,459,461-464,467-468,470,473,475-476,478-481,483,485-487,489-494,499,501,503,505,507-511,513-516,520,522,526-528,532,534,537,539,542,544,546,548-554,556-558,561,566-567,569,573-574,577-578,581-582,585-588,592-593,595-597,601-602,606-607,610,613-614,620-623,627,631,633,635-637,641-643,646,649,651,654-656,658-659,661-662,665-667,675-676,680-681,683-689,691,693-695,699,701,705-706,712,714,717-718,720,725,729-730,733,737-738,741-742"}}
+{"timestamp":1710697873.4715998,"name":"online","context":{"idset":"62,66,68,70,78,80,82,84,90-92,94-95,101-103,107,110-112,114,116-118,120,123,138,142,146,151,153-154,161,166,170-173,184-185,187-189,191,194-195,198,203-206,208-210,212,216,222,226,233-234,236-237,243-246,251-253,259-260,265-267,269,271-272,275,277,282,288-291,293,297-298,300,307,309,319-320,329,331,342,347,350,354,359,373,375,377,380,385-386,402,407-408,410,415,427,429,439-440,447,460,465,472,484,495-496,502,504,512,521,523,538,543,547,559-560,564-565,568,580,584,594,600,604,608-609,611,616,619,625-626,628-630,639,647-648,652,657,660,668,670-671,678-679,696-698,700,708-711,723-724,726-728,734-736,739-740,743,746,751,753"}}
+{"timestamp":1710697873.59566,"name":"online","context":{"idset":"745,747-748,752,754-756"}}
+{"timestamp":1710697873.7194059,"name":"online","context":{"idset":"749-750"}}
+{"timestamp":1710800159.5611808,"name":"drain","context":{"idset":"745","reason":"vgs hang","overwrite":0}}
+{"timestamp":1710800370.1495643,"name":"drain","context":{"idset":"745","reason":"epilog failed for jobid fjMU2cp8nP9","overwrite":0}}
+{"timestamp":1710800370.2453022,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1710804301.4680445,"name":"undrain","context":{"idset":"745"}}
+{"timestamp":1710804480.6500688,"name":"undrain","context":{"idset":"750-752,754-756"}}
+{"timestamp":1710804483.9735229,"name":"undrain","context":{"idset":"753"}}
+{"timestamp":1710804524.4170768,"name":"drain","context":{"idset":"744-756","reason":"Implicated in IOR failures","overwrite":0}}
+{"timestamp":1710804556.0472746,"name":"online","context":{"idset":"745"}}
+{"timestamp":1710804563.0881329,"name":"undrain","context":{"idset":"744-756"}}
+{"timestamp":1710804566.3281977,"name":"drain","context":{"idset":"744-756","reason":"Implicated in IOR failures","overwrite":0}}
+{"timestamp":1710808179.4946334,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1710808179.8331401,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1710808180.5613432,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1710808180.6003954,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1710808180.6079969,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1710808180.6131473,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1710808180.6181791,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1710808180.6217718,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1710808180.7046916,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1710808180.7284629,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1710808180.7437532,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1710808180.7494941,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1710808180.7533233,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1710808180.7599082,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1710808180.766083,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1710808180.772491,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1710808180.7791963,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1710808180.786273,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1710808180.7934248,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1710808180.8003328,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1710808180.8078332,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1710808180.8147175,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1710808180.8229055,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1710808180.828445,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1710808180.8339751,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1710808180.8419373,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1710808180.8464549,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1710808180.8511393,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1710808180.8554089,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1710808180.8726552,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1710808180.8811507,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1710808180.8886492,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1710808180.8959658,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1710808180.9034603,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1710808180.9114358,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1710808180.9186652,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1710808180.9251552,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1710808180.9303977,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1710808180.9353964,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1710808180.9408672,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1710808180.9517319,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1710808180.956986,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1710808180.9697666,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1710808180.9774809,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1710808180.982167,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1710808180.9875522,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1710808180.9927223,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1710808180.9987786,"name":"offline","context":{"idset":"658"}}
+{"timestamp":1710808181.0187783,"name":"offline","context":{"idset":"660"}}
+{"timestamp":1710808181.0311551,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1710808181.0378046,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1710808181.0466101,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1710808181.0556405,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1710808181.0638452,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1710808181.0706854,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1710808181.1253278,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1710808181.16956,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1710808181.1772635,"name":"offline","context":{"idset":"676"}}
+{"timestamp":1710808181.221719,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1710808181.3030732,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1710808181.308171,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1710808181.3301246,"name":"offline","context":{"idset":"706"}}
+{"timestamp":1710808181.3432722,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1710808181.3510706,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1710808181.358743,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1710808181.363652,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1710808181.3690467,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1710808181.3985331,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1710808181.4055862,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1710808181.4125271,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1710808181.416739,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1710808181.421021,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1710808181.4252019,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1710808181.4308658,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1710808181.4358778,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1710808181.4394853,"name":"offline","context":{"idset":"669"}}
+{"timestamp":1710808181.4445782,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1710808181.4488468,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1710808181.4542108,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1710808181.4676301,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1710808181.474746,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1710808181.4816673,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1710808181.4887059,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1710808181.4942255,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1710808181.4996436,"name":"offline","context":{"idset":"454"}}
+{"timestamp":1710808181.5038633,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1710808181.5077493,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1710808181.5192413,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1710808181.532995,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1710808181.5404415,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1710808181.5474734,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1710808181.5545671,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1710808181.5616682,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1710808181.5687776,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1710808181.5806515,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1710808181.5892375,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1710808181.596689,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1710808181.601084,"name":"offline","context":{"idset":"457"}}
+{"timestamp":1710808181.6326554,"name":"offline","context":{"idset":"553"}}
+{"timestamp":1710808181.6427233,"name":"offline","context":{"idset":"453"}}
+{"timestamp":1710808181.6831095,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1710808181.6905437,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1710808181.6951039,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1710808181.7296944,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1710808181.7406554,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1710808181.7477019,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1710808181.752104,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1710808181.7799692,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1710808181.7884355,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1710808181.7955096,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1710808181.8372536,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1710808181.8451502,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1710808181.8516741,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1710808181.8583918,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1710808181.9001625,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1710808181.9099863,"name":"offline","context":{"idset":"445"}}
+{"timestamp":1710808181.9144495,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1710808181.9422781,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1710808181.9475586,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1710808181.9994578,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1710808182.0069861,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1710808182.0140667,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1710808182.0649393,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1710808182.0741568,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1710808182.1218359,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1710808182.1811187,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1710808182.1880987,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1710808182.276607,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1710808182.3811891,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1710808182.3879905,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1710808182.4379129,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1710808182.4535191,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1710808182.4613221,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1710808182.4680805,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1710808182.475126,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1710808182.482137,"name":"offline","context":{"idset":"555"}}
+{"timestamp":1710808182.4897249,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1710808182.5021188,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1710808182.5114286,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1710808182.5185151,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1710808182.5746245,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1710808182.583087,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1710808182.591759,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1710808182.5991545,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1710808182.6049345,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1710808182.6134953,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1710808182.6426945,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1710808182.6499641,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1710808182.729706,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1710808182.756073,"name":"offline","context":{"idset":"692"}}
+{"timestamp":1710808182.7947345,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1710808182.8106174,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1710808182.8705213,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1710808182.8762517,"name":"offline","context":{"idset":"447"}}
+{"timestamp":1710808182.9279754,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1710808182.9371555,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1710808182.9838903,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1710808182.9896305,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1710808183.0438764,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1710808183.1372039,"name":"offline","context":{"idset":"666"}}
+{"timestamp":1710808183.4848368,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1710808183.5803998,"name":"offline","context":{"idset":"550"}}
+{"timestamp":1710808184.7154713,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1710808185.4990308,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1710808419.0724847,"name":"drain","context":{"idset":"61","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.072612,"name":"drain","context":{"idset":"62","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.072649,"name":"drain","context":{"idset":"63","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0726817,"name":"drain","context":{"idset":"64","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0727127,"name":"drain","context":{"idset":"65","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0727422,"name":"drain","context":{"idset":"66","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0727699,"name":"drain","context":{"idset":"68","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0727999,"name":"drain","context":{"idset":"69","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0728276,"name":"drain","context":{"idset":"73","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0728576,"name":"drain","context":{"idset":"75","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0728915,"name":"drain","context":{"idset":"76","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0729187,"name":"drain","context":{"idset":"77","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0729473,"name":"drain","context":{"idset":"78","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0729744,"name":"drain","context":{"idset":"79","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0730021,"name":"drain","context":{"idset":"81","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0730317,"name":"drain","context":{"idset":"82","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0730603,"name":"drain","context":{"idset":"85","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0730917,"name":"drain","context":{"idset":"86","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.073128,"name":"drain","context":{"idset":"88","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0731606,"name":"drain","context":{"idset":"89","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0731926,"name":"drain","context":{"idset":"90","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0732219,"name":"drain","context":{"idset":"91","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0732567,"name":"drain","context":{"idset":"92","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0732994,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0733328,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0733681,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0734017,"name":"drain","context":{"idset":"96","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0734365,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0734746,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0735087,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0735431,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0735803,"name":"drain","context":{"idset":"103","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0736146,"name":"drain","context":{"idset":"104","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0736535,"name":"drain","context":{"idset":"105","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0736873,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0737228,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0737584,"name":"drain","context":{"idset":"108","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0737948,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0738328,"name":"drain","context":{"idset":"110","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0738685,"name":"drain","context":{"idset":"112","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0739074,"name":"drain","context":{"idset":"113","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0739467,"name":"drain","context":{"idset":"115","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0739846,"name":"drain","context":{"idset":"116","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0740249,"name":"drain","context":{"idset":"117","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0740676,"name":"drain","context":{"idset":"120","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0741067,"name":"drain","context":{"idset":"122","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0741501,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0741911,"name":"drain","context":{"idset":"125","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0742307,"name":"drain","context":{"idset":"126","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0742738,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0743189,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0743632,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0744057,"name":"drain","context":{"idset":"131","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0744462,"name":"drain","context":{"idset":"133","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0744901,"name":"drain","context":{"idset":"134","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0745332,"name":"drain","context":{"idset":"135","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0745764,"name":"drain","context":{"idset":"136","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.074621,"name":"drain","context":{"idset":"137","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0746648,"name":"drain","context":{"idset":"139","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0747111,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0747633,"name":"drain","context":{"idset":"141","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.074815,"name":"drain","context":{"idset":"143","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0748818,"name":"drain","context":{"idset":"145","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0749402,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0749891,"name":"drain","context":{"idset":"148","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0750344,"name":"drain","context":{"idset":"149","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0750818,"name":"drain","context":{"idset":"150","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.075124,"name":"drain","context":{"idset":"151","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0751705,"name":"drain","context":{"idset":"152","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0752158,"name":"drain","context":{"idset":"153","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0752594,"name":"drain","context":{"idset":"155","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0753112,"name":"drain","context":{"idset":"156","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.075356,"name":"drain","context":{"idset":"157","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0754044,"name":"drain","context":{"idset":"158","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0754526,"name":"drain","context":{"idset":"160","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0754988,"name":"drain","context":{"idset":"161","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0755491,"name":"drain","context":{"idset":"164","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0755968,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0756457,"name":"drain","context":{"idset":"166","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0756946,"name":"drain","context":{"idset":"169","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0757558,"name":"drain","context":{"idset":"170","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0758057,"name":"drain","context":{"idset":"171","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0758524,"name":"drain","context":{"idset":"172","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0759022,"name":"drain","context":{"idset":"173","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0759523,"name":"drain","context":{"idset":"175","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0760007,"name":"drain","context":{"idset":"490","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0760527,"name":"drain","context":{"idset":"491","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0761013,"name":"drain","context":{"idset":"492","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.076159,"name":"drain","context":{"idset":"493","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0762115,"name":"drain","context":{"idset":"494","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0762613,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0763204,"name":"drain","context":{"idset":"497","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0763717,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0764244,"name":"drain","context":{"idset":"499","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0764763,"name":"drain","context":{"idset":"500","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0765798,"name":"drain","context":{"idset":"501","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0766356,"name":"drain","context":{"idset":"502","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0766916,"name":"drain","context":{"idset":"503","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.076755,"name":"drain","context":{"idset":"505","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0768161,"name":"drain","context":{"idset":"507","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0768788,"name":"drain","context":{"idset":"508","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0769334,"name":"drain","context":{"idset":"509","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0769894,"name":"drain","context":{"idset":"510","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0770464,"name":"drain","context":{"idset":"511","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.077101,"name":"drain","context":{"idset":"512","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0771635,"name":"drain","context":{"idset":"513","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0772247,"name":"drain","context":{"idset":"514","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.077297,"name":"drain","context":{"idset":"516","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0773644,"name":"drain","context":{"idset":"519","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0774267,"name":"drain","context":{"idset":"521","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0774903,"name":"drain","context":{"idset":"523","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0775528,"name":"drain","context":{"idset":"525","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0776179,"name":"drain","context":{"idset":"528","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0776846,"name":"drain","context":{"idset":"529","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0777478,"name":"drain","context":{"idset":"530","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0778151,"name":"drain","context":{"idset":"531","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0778797,"name":"drain","context":{"idset":"533","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0779457,"name":"drain","context":{"idset":"535","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0780113,"name":"drain","context":{"idset":"537","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0780797,"name":"drain","context":{"idset":"538","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0781488,"name":"drain","context":{"idset":"539","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0782158,"name":"drain","context":{"idset":"540","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0782845,"name":"drain","context":{"idset":"541","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0783548,"name":"drain","context":{"idset":"543","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0784283,"name":"drain","context":{"idset":"544","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0784991,"name":"drain","context":{"idset":"545","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0785649,"name":"drain","context":{"idset":"546","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0786331,"name":"drain","context":{"idset":"549","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0787046,"name":"drain","context":{"idset":"551","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0787742,"name":"drain","context":{"idset":"552","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0788462,"name":"drain","context":{"idset":"554","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0789165,"name":"drain","context":{"idset":"556","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0789912,"name":"drain","context":{"idset":"557","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0790627,"name":"drain","context":{"idset":"558","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0791371,"name":"drain","context":{"idset":"559","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0792103,"name":"drain","context":{"idset":"560","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0792761,"name":"drain","context":{"idset":"561","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0793495,"name":"drain","context":{"idset":"562","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0794172,"name":"drain","context":{"idset":"563","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.079488,"name":"drain","context":{"idset":"564","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0795584,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0796254,"name":"drain","context":{"idset":"567","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0798182,"name":"drain","context":{"idset":"568","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.079886,"name":"drain","context":{"idset":"569","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0799534,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0800219,"name":"drain","context":{"idset":"572","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0800881,"name":"drain","context":{"idset":"573","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0801725,"name":"drain","context":{"idset":"575","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0802391,"name":"drain","context":{"idset":"577","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0803154,"name":"drain","context":{"idset":"578","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0804036,"name":"drain","context":{"idset":"579","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0804772,"name":"drain","context":{"idset":"580","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0805478,"name":"drain","context":{"idset":"583","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.080617,"name":"drain","context":{"idset":"584","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0806894,"name":"drain","context":{"idset":"585","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0807626,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0808332,"name":"drain","context":{"idset":"587","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.080905,"name":"drain","context":{"idset":"589","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.080981,"name":"drain","context":{"idset":"590","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0810547,"name":"drain","context":{"idset":"591","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0811303,"name":"drain","context":{"idset":"592","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.081203,"name":"drain","context":{"idset":"593","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0812786,"name":"drain","context":{"idset":"594","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0813584,"name":"drain","context":{"idset":"595","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0814338,"name":"drain","context":{"idset":"597","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0815089,"name":"drain","context":{"idset":"598","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0815849,"name":"drain","context":{"idset":"599","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.081665,"name":"drain","context":{"idset":"600","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0817423,"name":"drain","context":{"idset":"601","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0818214,"name":"drain","context":{"idset":"602","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0819077,"name":"drain","context":{"idset":"603","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.081985,"name":"drain","context":{"idset":"604","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0820627,"name":"drain","context":{"idset":"605","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0821433,"name":"drain","context":{"idset":"606","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0822217,"name":"drain","context":{"idset":"608","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0823081,"name":"drain","context":{"idset":"609","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0823898,"name":"drain","context":{"idset":"610","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0824685,"name":"drain","context":{"idset":"611","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0825498,"name":"drain","context":{"idset":"613","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.082629,"name":"drain","context":{"idset":"614","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0827103,"name":"drain","context":{"idset":"616","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0827935,"name":"drain","context":{"idset":"617","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0828731,"name":"drain","context":{"idset":"618","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0829561,"name":"drain","context":{"idset":"619","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0830402,"name":"drain","context":{"idset":"620","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0831208,"name":"drain","context":{"idset":"621","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0832055,"name":"drain","context":{"idset":"622","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.083288,"name":"drain","context":{"idset":"623","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0833797,"name":"drain","context":{"idset":"626","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0834653,"name":"drain","context":{"idset":"628","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0835478,"name":"drain","context":{"idset":"630","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0836349,"name":"drain","context":{"idset":"632","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0837266,"name":"drain","context":{"idset":"633","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0838325,"name":"drain","context":{"idset":"634","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0839198,"name":"drain","context":{"idset":"635","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0840054,"name":"drain","context":{"idset":"636","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0840936,"name":"drain","context":{"idset":"638","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0841782,"name":"drain","context":{"idset":"639","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0842671,"name":"drain","context":{"idset":"640","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0843601,"name":"drain","context":{"idset":"641","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0844467,"name":"drain","context":{"idset":"642","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.084537,"name":"drain","context":{"idset":"643","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0846233,"name":"drain","context":{"idset":"644","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0847111,"name":"drain","context":{"idset":"645","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0847998,"name":"drain","context":{"idset":"646","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0848894,"name":"drain","context":{"idset":"649","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0849795,"name":"drain","context":{"idset":"650","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0850682,"name":"drain","context":{"idset":"651","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0851595,"name":"drain","context":{"idset":"654","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0852504,"name":"drain","context":{"idset":"655","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0853453,"name":"drain","context":{"idset":"656","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0854387,"name":"drain","context":{"idset":"657","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0855286,"name":"drain","context":{"idset":"659","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0856206,"name":"drain","context":{"idset":"661","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0857108,"name":"drain","context":{"idset":"662","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0858049,"name":"drain","context":{"idset":"664","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0858955,"name":"drain","context":{"idset":"665","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0859907,"name":"drain","context":{"idset":"667","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0860863,"name":"drain","context":{"idset":"668","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0861807,"name":"drain","context":{"idset":"671","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0862768,"name":"drain","context":{"idset":"672","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0863769,"name":"drain","context":{"idset":"673","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.086473,"name":"drain","context":{"idset":"674","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0865693,"name":"drain","context":{"idset":"675","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0866702,"name":"drain","context":{"idset":"677","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.0867701,"name":"drain","context":{"idset":"678","reason":"broker was unresponsive"}}
+{"timestamp":1710808419.1873004,"name":"drain","context":{"idset":"680","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1045525,"name":"drain","context":{"idset":"146","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1048052,"name":"drain","context":{"idset":"167","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1050005,"name":"drain","context":{"idset":"174","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1051931,"name":"drain","context":{"idset":"176","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1054688,"name":"drain","context":{"idset":"177","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.105654,"name":"drain","context":{"idset":"178","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1058364,"name":"drain","context":{"idset":"179","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1060185,"name":"drain","context":{"idset":"180","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1062062,"name":"drain","context":{"idset":"181","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1063986,"name":"drain","context":{"idset":"183","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1065888,"name":"drain","context":{"idset":"184","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1067739,"name":"drain","context":{"idset":"185","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1069643,"name":"drain","context":{"idset":"187","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1071532,"name":"drain","context":{"idset":"188","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1073561,"name":"drain","context":{"idset":"189","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1075637,"name":"drain","context":{"idset":"190","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1077662,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1079738,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1081696,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1083651,"name":"drain","context":{"idset":"196","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1085577,"name":"drain","context":{"idset":"197","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1087539,"name":"drain","context":{"idset":"200","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.108943,"name":"drain","context":{"idset":"201","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.109139,"name":"drain","context":{"idset":"202","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1093342,"name":"drain","context":{"idset":"203","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1095378,"name":"drain","context":{"idset":"204","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1097355,"name":"drain","context":{"idset":"205","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1099288,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1101255,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1103423,"name":"drain","context":{"idset":"209","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1105402,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1107311,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1109366,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1111431,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1113484,"name":"drain","context":{"idset":"217","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1115551,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1117673,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1119747,"name":"drain","context":{"idset":"221","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1121709,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1123867,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1125898,"name":"drain","context":{"idset":"224","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1127946,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1130033,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1132123,"name":"drain","context":{"idset":"227","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1134353,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1136458,"name":"drain","context":{"idset":"230","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.113852,"name":"drain","context":{"idset":"231","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1140766,"name":"drain","context":{"idset":"233","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1142967,"name":"drain","context":{"idset":"237","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1145043,"name":"drain","context":{"idset":"238","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.114737,"name":"drain","context":{"idset":"240","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1149507,"name":"drain","context":{"idset":"241","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1151631,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1153767,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1155872,"name":"drain","context":{"idset":"247","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1158018,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.115999,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1161313,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1162565,"name":"drain","context":{"idset":"254","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1163938,"name":"drain","context":{"idset":"257","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1165202,"name":"drain","context":{"idset":"258","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1166489,"name":"drain","context":{"idset":"259","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1167889,"name":"drain","context":{"idset":"260","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1169183,"name":"drain","context":{"idset":"261","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1170499,"name":"drain","context":{"idset":"262","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1171815,"name":"drain","context":{"idset":"264","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1173203,"name":"drain","context":{"idset":"266","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1174524,"name":"drain","context":{"idset":"268","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1175852,"name":"drain","context":{"idset":"270","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1177306,"name":"drain","context":{"idset":"273","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.117873,"name":"drain","context":{"idset":"275","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1180077,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1181622,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.118314,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1184497,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1185837,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1187274,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1188753,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1190147,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1191571,"name":"drain","context":{"idset":"295","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1193008,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1194389,"name":"drain","context":{"idset":"297","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1195748,"name":"drain","context":{"idset":"298","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1197107,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1198475,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1199944,"name":"drain","context":{"idset":"301","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1201324,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1202705,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1204185,"name":"drain","context":{"idset":"308","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1205585,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1206982,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1208367,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1209919,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1211436,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1212831,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1214316,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1215732,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1217129,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1218538,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1219947,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1221509,"name":"drain","context":{"idset":"324","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1223056,"name":"drain","context":{"idset":"326","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1224582,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1225991,"name":"drain","context":{"idset":"328","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1227462,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.122885,"name":"drain","context":{"idset":"332","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1230252,"name":"drain","context":{"idset":"333","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1231744,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1233287,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1234751,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1236355,"name":"drain","context":{"idset":"341","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1237822,"name":"drain","context":{"idset":"342","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1239276,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1240711,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1242235,"name":"drain","context":{"idset":"345","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1243792,"name":"drain","context":{"idset":"346","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1245277,"name":"drain","context":{"idset":"347","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.124675,"name":"drain","context":{"idset":"348","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1248217,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1249702,"name":"drain","context":{"idset":"351","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1251187,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1252761,"name":"drain","context":{"idset":"354","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1254339,"name":"drain","context":{"idset":"355","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1255841,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1257331,"name":"drain","context":{"idset":"357","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1258821,"name":"drain","context":{"idset":"358","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1260309,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1261814,"name":"drain","context":{"idset":"361","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.126348,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1265032,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1266544,"name":"drain","context":{"idset":"365","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1268055,"name":"drain","context":{"idset":"366","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1269584,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1271105,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1272626,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1274281,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1275835,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1277368,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1278901,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1280451,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1281989,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1283817,"name":"drain","context":{"idset":"383","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1285529,"name":"drain","context":{"idset":"384","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1287091,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.128865,"name":"drain","context":{"idset":"386","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1290216,"name":"drain","context":{"idset":"388","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1291783,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1293428,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1295125,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.129673,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1298299,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1299865,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1301432,"name":"drain","context":{"idset":"395","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1303077,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1304667,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.130641,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1308193,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1309803,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1311393,"name":"drain","context":{"idset":"405","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1313088,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.131469,"name":"drain","context":{"idset":"407","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1316459,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1318073,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.131968,"name":"drain","context":{"idset":"411","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1322269,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1325357,"name":"drain","context":{"idset":"413","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1328259,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1331122,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1334057,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1335909,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1337588,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.133925,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1340919,"name":"drain","context":{"idset":"421","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1343625,"name":"drain","context":{"idset":"423","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1346614,"name":"drain","context":{"idset":"424","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1348522,"name":"drain","context":{"idset":"426","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.135021,"name":"drain","context":{"idset":"427","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1351862,"name":"drain","context":{"idset":"429","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1353643,"name":"drain","context":{"idset":"430","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1355314,"name":"drain","context":{"idset":"432","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1356995,"name":"drain","context":{"idset":"433","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1358659,"name":"drain","context":{"idset":"434","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1360383,"name":"drain","context":{"idset":"435","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1362073,"name":"drain","context":{"idset":"436","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1363883,"name":"drain","context":{"idset":"438","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1365612,"name":"drain","context":{"idset":"439","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1367323,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1369016,"name":"drain","context":{"idset":"441","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.137073,"name":"drain","context":{"idset":"442","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1372468,"name":"drain","context":{"idset":"444","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1374276,"name":"drain","context":{"idset":"448","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1376009,"name":"drain","context":{"idset":"449","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.137773,"name":"drain","context":{"idset":"450","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1379445,"name":"drain","context":{"idset":"451","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1381159,"name":"drain","context":{"idset":"452","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1382987,"name":"drain","context":{"idset":"455","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1384723,"name":"drain","context":{"idset":"456","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1386471,"name":"drain","context":{"idset":"459","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1388192,"name":"drain","context":{"idset":"460","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.138993,"name":"drain","context":{"idset":"461","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1391661,"name":"drain","context":{"idset":"462","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1393464,"name":"drain","context":{"idset":"463","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1395218,"name":"drain","context":{"idset":"464","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1396976,"name":"drain","context":{"idset":"466","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1398745,"name":"drain","context":{"idset":"467","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1400509,"name":"drain","context":{"idset":"468","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1402271,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1404097,"name":"drain","context":{"idset":"471","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1405852,"name":"drain","context":{"idset":"472","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1407614,"name":"drain","context":{"idset":"475","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1409385,"name":"drain","context":{"idset":"476","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1411161,"name":"drain","context":{"idset":"478","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1413012,"name":"drain","context":{"idset":"479","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1415517,"name":"drain","context":{"idset":"480","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.141731,"name":"drain","context":{"idset":"481","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1419106,"name":"drain","context":{"idset":"483","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1420901,"name":"drain","context":{"idset":"484","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.142271,"name":"drain","context":{"idset":"485","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1424572,"name":"drain","context":{"idset":"486","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1426396,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1428208,"name":"drain","context":{"idset":"488","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1429913,"name":"drain","context":{"idset":"670","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1431608,"name":"drain","context":{"idset":"679","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1433361,"name":"drain","context":{"idset":"681","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1435058,"name":"drain","context":{"idset":"682","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1436746,"name":"drain","context":{"idset":"683","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1438456,"name":"drain","context":{"idset":"685","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.144016,"name":"drain","context":{"idset":"686","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1441851,"name":"drain","context":{"idset":"688","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.144362,"name":"drain","context":{"idset":"690","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1445334,"name":"drain","context":{"idset":"691","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1447036,"name":"drain","context":{"idset":"694","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1448743,"name":"drain","context":{"idset":"695","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1450441,"name":"drain","context":{"idset":"698","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1452231,"name":"drain","context":{"idset":"699","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1453989,"name":"drain","context":{"idset":"700","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1455719,"name":"drain","context":{"idset":"701","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1457446,"name":"drain","context":{"idset":"702","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.145916,"name":"drain","context":{"idset":"703","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1460884,"name":"drain","context":{"idset":"704","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.14626,"name":"drain","context":{"idset":"705","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1464438,"name":"drain","context":{"idset":"707","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1466165,"name":"drain","context":{"idset":"708","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1467903,"name":"drain","context":{"idset":"709","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1469634,"name":"drain","context":{"idset":"711","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1471386,"name":"drain","context":{"idset":"712","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1473196,"name":"drain","context":{"idset":"713","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1474917,"name":"drain","context":{"idset":"714","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1476836,"name":"drain","context":{"idset":"715","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1478775,"name":"drain","context":{"idset":"716","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.148057,"name":"drain","context":{"idset":"717","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1482337,"name":"drain","context":{"idset":"718","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1484184,"name":"drain","context":{"idset":"720","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1485963,"name":"drain","context":{"idset":"721","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.148772,"name":"drain","context":{"idset":"722","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1489508,"name":"drain","context":{"idset":"723","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1491306,"name":"drain","context":{"idset":"724","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1493123,"name":"drain","context":{"idset":"725","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1494927,"name":"drain","context":{"idset":"726","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1496701,"name":"drain","context":{"idset":"727","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1498504,"name":"drain","context":{"idset":"729","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1500278,"name":"drain","context":{"idset":"730","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1502385,"name":"drain","context":{"idset":"732","reason":"broker was unresponsive"}}
+{"timestamp":1710808421.1909573,"name":"drain","context":{"idset":"733","reason":"broker was unresponsive"}}
+{"timestamp":1710808431.1586156,"name":"drain","context":{"idset":"446","reason":"broker was unresponsive"}}
+{"timestamp":1710808479.1610014,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1710808479.1648107,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1710808479.1680951,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1710808479.1714342,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1710808479.1746268,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1710808479.1779201,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1710808479.1812339,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1710808479.1896241,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1710808479.1934946,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1710808479.1966772,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1710808479.2040236,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1710808479.2066519,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1710808479.2099876,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1710808479.2141147,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1710808479.2192369,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1710808479.224124,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1710808479.2310076,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1710808479.2326853,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1710808479.2369158,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1710808479.2402263,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1710808479.24525,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1710808479.2466795,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1710808479.2542784,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1710808479.2585161,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1710808479.26372,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1710808479.26489,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1710808479.2688959,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1710808479.2720792,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1710808479.2764814,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1710808479.2791786,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1710808479.2827396,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1710808479.2859423,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1710808479.2895961,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1710808479.2944393,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1710808479.2951026,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1710808479.300915,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1710808479.3039961,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1710808479.3072121,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1710808479.3114185,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1710808479.3151596,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1710808479.3167906,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1710808479.3208046,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1710808479.3241537,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1710808479.3278625,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1710808479.3324039,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1710808479.3348532,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1710808479.3366995,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1710808479.3400729,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1710808479.3433027,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1710808479.346822,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1710808479.3518198,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1710808479.352798,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1710808479.3558366,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1710808479.3600802,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1710808479.3633571,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1710808479.3666244,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1710808479.3754191,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1710808479.3782089,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1710808479.380172,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1710808479.3835368,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1710808479.3867118,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1710808479.3898721,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1710808479.3930562,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1710808479.4002805,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1710808479.4036124,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1710808479.4056284,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1710808479.4087639,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1710808479.4119344,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1710808479.4151108,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1710808479.4205217,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1710808479.4226828,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1710808479.4246635,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1710808479.4278159,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1710808479.4310055,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1710808479.4352396,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1710808479.4391184,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1710808479.446744,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1710808479.4481056,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1710808479.4532418,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1710808479.456414,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1710808479.459758,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1710808479.4648621,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1710808479.469106,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1710808479.4712324,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1710808479.4745824,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1710808479.4775944,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1710808479.4808939,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1710808479.4829698,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1710808479.4872103,"name":"offline","context":{"idset":"549"}}
+{"timestamp":1710808479.490339,"name":"offline","context":{"idset":"551"}}
+{"timestamp":1710808479.4987493,"name":"offline","context":{"idset":"552"}}
+{"timestamp":1710808479.5019155,"name":"offline","context":{"idset":"554"}}
+{"timestamp":1710808479.5040462,"name":"offline","context":{"idset":"556"}}
+{"timestamp":1710808479.5082672,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1710808479.5148015,"name":"offline","context":{"idset":"558"}}
+{"timestamp":1710808479.5189836,"name":"offline","context":{"idset":"559"}}
+{"timestamp":1710808479.5221777,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1710808479.5295525,"name":"offline","context":{"idset":"561"}}
+{"timestamp":1710808479.5319748,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1710808479.5345964,"name":"offline","context":{"idset":"563"}}
+{"timestamp":1710808479.5442853,"name":"offline","context":{"idset":"564"}}
+{"timestamp":1710808479.546838,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1710808479.5536864,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1710808479.5630941,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1710808479.5661783,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1710808479.5707204,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1710808479.6259451,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1710808479.6289911,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1710808479.6317434,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1710808479.6360779,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1710808479.6389322,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1710808479.6399925,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1710808479.6427233,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1710808479.6488905,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1710808479.653239,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1710808479.654428,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1710808479.6571956,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1710808479.6620131,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1710808479.6644058,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1710808479.6657336,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1710808479.6680741,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1710808479.67401,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1710808479.6746831,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1710808479.6773612,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1710808479.680923,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1710808479.6844399,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1710808479.6863697,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1710808479.6882672,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1710808479.6913924,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1710808479.694237,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1710808479.697803,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1710808479.7012353,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1710808479.704953,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1710808479.707257,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1710808479.7096884,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1710808479.7116485,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1710808479.7143066,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1710808479.7183032,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1710808479.7211239,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1710808479.7291212,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1710808479.7295814,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1710808479.7300193,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1710808479.7358434,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1710808479.7362778,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1710808479.7379286,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1710808479.7420928,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1710808479.7434843,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1710808479.7471421,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1710808479.7496083,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1710808479.7515075,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1710808479.7563148,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1710808479.7566729,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1710808479.7591586,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1710808479.7617145,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1710808479.7665856,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1710808479.769938,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1710808479.7740464,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1710808479.7781041,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1710808479.7816219,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1710808479.7882233,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1710808479.7910326,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1710808479.7958269,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1710808479.8026001,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1710808479.8051705,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1710808479.810497,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1710808479.8136692,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1710808479.815918,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1710808479.8194683,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1710808479.8215415,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1710808479.8221211,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1710808479.8240395,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1710808479.8266642,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1710808479.8286605,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1710808479.8305173,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1710808479.8344712,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1710808479.835927,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1710808479.836364,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1710808479.838397,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1710808479.8408637,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1710808479.8424883,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1710808479.8466122,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1710808479.8481269,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1710808479.8498998,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1710808479.8521345,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1710808479.8555648,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1710808479.8582737,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1710808479.8591666,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1710808479.8620601,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1710808479.8646355,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1710808479.86746,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1710808479.8711765,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1710808479.8716195,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1710808479.8743434,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1710808479.8766987,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1710808479.8793166,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1710808479.8833172,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1710808479.8850698,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1710808479.8869469,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1710808479.8898239,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1710808479.8925734,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1710808479.8951354,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1710808479.8994348,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1710808479.9013753,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1710808479.9033017,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1710808479.9056096,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1710808479.9097414,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1710808479.9112124,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1710808479.9141147,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1710808479.9180863,"name":"offline","context":{"idset":"654"}}
+{"timestamp":1710808479.9197164,"name":"offline","context":{"idset":"655"}}
+{"timestamp":1710808479.9209986,"name":"offline","context":{"idset":"656"}}
+{"timestamp":1710808479.9240372,"name":"offline","context":{"idset":"657"}}
+{"timestamp":1710808479.9265964,"name":"offline","context":{"idset":"659"}}
+{"timestamp":1710808479.9292119,"name":"offline","context":{"idset":"661"}}
+{"timestamp":1710808479.9326713,"name":"offline","context":{"idset":"662"}}
+{"timestamp":1710808479.9341578,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1710808479.9367597,"name":"offline","context":{"idset":"665"}}
+{"timestamp":1710808479.9390612,"name":"offline","context":{"idset":"667"}}
+{"timestamp":1710808479.9413419,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1710808479.9444156,"name":"offline","context":{"idset":"671"}}
+{"timestamp":1710808479.9472671,"name":"offline","context":{"idset":"672"}}
+{"timestamp":1710808479.9488845,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1710808479.9509263,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1710808479.9526033,"name":"offline","context":{"idset":"675"}}
+{"timestamp":1710808479.9542487,"name":"offline","context":{"idset":"677"}}
+{"timestamp":1710808479.9588077,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1710808483.1334693,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1710808483.1360264,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1710808483.138438,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1710808483.1408207,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1710808483.1432729,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1710808483.1459794,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1710808483.148298,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1710808483.1513042,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1710808483.1531754,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1710808483.155755,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1710808483.1582556,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1710808483.161551,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1710808483.1633813,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1710808483.1652398,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1710808483.1677225,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1710808483.1700773,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1710808483.1729598,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1710808483.1763482,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1710808483.1806498,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1710808483.1810191,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1710808483.1822579,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1710808483.1858947,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1710808483.1877184,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1710808483.1887219,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1710808483.1925595,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1710808483.1965497,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1710808483.1974647,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1710808483.1999593,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1710808483.2003217,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1710808483.2045462,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1710808483.2049007,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1710808483.2086337,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1710808483.2111518,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1710808483.2114999,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1710808483.215549,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1710808483.2158995,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1710808483.218708,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1710808483.2206776,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1710808483.2220762,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1710808483.2246814,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1710808483.2256606,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1710808483.2277956,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1710808483.2294059,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1710808483.2352443,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1710808483.2366486,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1710808483.2369769,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1710808483.2405407,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1710808483.2408805,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1710808483.2446768,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1710808483.2473609,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1710808483.2518218,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1710808483.2521484,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1710808483.25247,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1710808483.2527857,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1710808483.2581606,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1710808483.2595809,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1710808483.2599001,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1710808483.2602127,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1710808483.2645335,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1710808483.2648702,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1710808483.2684393,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1710808483.2703691,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1710808483.2756062,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1710808483.2759316,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1710808483.2785878,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1710808483.2789032,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1710808483.2836781,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1710808483.2856331,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1710808483.2859437,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1710808483.2875562,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1710808483.2895317,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1710808483.2935143,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1710808483.2943676,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1710808483.2988698,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1710808483.3002996,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1710808483.3005962,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1710808483.3009052,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1710808483.3036091,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1710808483.3075867,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1710808483.3078876,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1710808483.3105009,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1710808483.3119168,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1710808483.3122127,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1710808483.3165119,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1710808483.3168068,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1710808483.3206327,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1710808483.3209319,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1710808483.3245139,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1710808483.3259087,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1710808483.3284898,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1710808483.3287737,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1710808483.3290505,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1710808483.3303607,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1710808483.3345056,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1710808483.3347874,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1710808483.3385854,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1710808483.3388638,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1710808483.3403394,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1710808483.3428121,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1710808483.3442194,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1710808483.3445082,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1710808483.3462715,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1710808483.3495502,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1710808483.3498213,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1710808483.3535635,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1710808483.3538418,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1710808483.3546844,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1710808483.3628647,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1710808483.365612,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1710808483.3658824,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1710808483.3661385,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1710808483.3664088,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1710808483.3666611,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1710808483.3669281,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1710808483.3671801,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1710808483.3689029,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1710808483.3719957,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1710808483.3758507,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1710808483.3761075,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1710808483.3763607,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1710808483.3766086,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1710808483.3774977,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1710808483.3789575,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1710808483.3803954,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1710808483.3817828,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1710808483.3855877,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1710808483.3881221,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1710808483.3889413,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1710808483.3891804,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1710808483.3897414,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1710808483.3911443,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1710808483.3925362,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1710808483.3938625,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1710808483.3957269,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1710808483.3987091,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1710808483.4013755,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1710808483.4022362,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1710808483.4024973,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1710808483.4027369,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1710808483.4039807,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1710808483.4052024,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1710808483.4070621,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1710808483.4100847,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1710808483.4125657,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1710808483.412792,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1710808483.4130104,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1710808483.413799,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1710808483.4148042,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1710808483.4158604,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1710808483.4170661,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1710808483.4194758,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1710808483.4219429,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1710808483.4238598,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1710808483.4240909,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1710808483.4243138,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1710808483.4252598,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1710808483.4263647,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1710808483.427501,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1710808483.4286923,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1710808483.4297972,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1710808483.4321039,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1710808483.434649,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1710808483.4359987,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1710808483.4362018,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1710808483.4366126,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1710808483.4376218,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1710808483.4386716,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1710808483.4396968,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1710808483.4408367,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1710808483.4419281,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1710808483.4452627,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1710808483.447866,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1710808483.4486594,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1710808483.4488542,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1710808483.4490449,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1710808483.4495547,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1710808483.4505708,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1710808483.45151,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1710808483.4526062,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1710808483.4541728,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1710808483.4570096,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1710808483.4594936,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1710808483.4602683,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1710808483.4604592,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1710808483.4606385,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1710808483.461005,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1710808483.4618382,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1710808483.462729,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1710808483.463625,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1710808483.4645526,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1710808483.4677181,"name":"offline","context":{"idset":"448"}}
+{"timestamp":1710808483.4702027,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1710808483.4709892,"name":"offline","context":{"idset":"450"}}
+{"timestamp":1710808483.4711626,"name":"offline","context":{"idset":"451"}}
+{"timestamp":1710808483.4713368,"name":"offline","context":{"idset":"452"}}
+{"timestamp":1710808483.4717276,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1710808483.4726982,"name":"offline","context":{"idset":"456"}}
+{"timestamp":1710808483.4735432,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1710808483.4744413,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1710808483.4754055,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1710808483.4785941,"name":"offline","context":{"idset":"462"}}
+{"timestamp":1710808483.4810927,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1710808483.4824145,"name":"offline","context":{"idset":"464"}}
+{"timestamp":1710808483.482583,"name":"offline","context":{"idset":"466"}}
+{"timestamp":1710808483.4827435,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1710808483.4830973,"name":"offline","context":{"idset":"468"}}
+{"timestamp":1710808483.4839573,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1710808483.4848638,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1710808483.4857624,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1710808483.4866543,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1710808483.4898057,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1710808483.4923263,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1710808483.4936528,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1710808483.4938054,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1710808483.4939504,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1710808483.4940946,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1710808483.4944103,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1710808483.495188,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1710808483.496058,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1710808483.496886,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1710808483.4988492,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1710808483.501339,"name":"offline","context":{"idset":"670"}}
+{"timestamp":1710808483.5038335,"name":"offline","context":{"idset":"679"}}
+{"timestamp":1710808483.5039754,"name":"offline","context":{"idset":"680"}}
+{"timestamp":1710808483.5041111,"name":"offline","context":{"idset":"681"}}
+{"timestamp":1710808483.5042439,"name":"offline","context":{"idset":"682"}}
+{"timestamp":1710808483.5047281,"name":"offline","context":{"idset":"683"}}
+{"timestamp":1710808483.5055647,"name":"offline","context":{"idset":"685"}}
+{"timestamp":1710808483.5063279,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1710808483.5070679,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1710808483.5078769,"name":"offline","context":{"idset":"690"}}
+{"timestamp":1710808483.5093935,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1710808483.5095284,"name":"offline","context":{"idset":"694"}}
+{"timestamp":1710808483.5102623,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1710808483.5127561,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1710808483.5152237,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1710808483.5171106,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1710808483.5172436,"name":"offline","context":{"idset":"701"}}
+{"timestamp":1710808483.5173764,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1710808483.5174944,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1710808483.5176105,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1710808483.5177286,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1710808483.5180612,"name":"offline","context":{"idset":"707"}}
+{"timestamp":1710808483.5187736,"name":"offline","context":{"idset":"708"}}
+{"timestamp":1710808483.5194328,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1710808483.520051,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1710808483.5207002,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1710808483.5230863,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1710808483.5255597,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1710808483.5280173,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1710808483.5280881,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1710808483.5281501,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1710808483.5282097,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1710808483.5282681,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1710808483.5283942,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1710808483.5285022,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1710808483.5291474,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1710808483.5297172,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1710808483.5302861,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1710808483.5308564,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1710808483.531415,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1710808483.5319636,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1710808483.5325086,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1710808483.5330594,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1710808483.5337372,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1710808493.1556668,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1710808493.2013681,"name":"drain","context":{"idset":"61-66,68-69,73,75-79,81-83,85-86,88-97,99-101,103-110,112-113,115-117,120,122-127,129-137,139-143,145-158,160-162,164-167,169-181,183-185,187-191,193-194,196-198,200-206,208-213,217-219,221-227,229-231,233,237-238,240-241,243,246-247,249-251,254-255,257-264,266,268,270,273,275,278,282-284,286-287,289,292,295-301,305,307-311,313,316-324,326-328,331-333,335-336,338,341-349,351,353-361,363-366,368,370-372,374-376,379-380,383-386,388-397,399-401,405-409,411-415,417-421,423-424,426-427,429-430,432-436,438-442,444,446,448-452,455-456,459-464,466-469,471-472,475-476,478-481,483-488,490-494,496-503,505-514,516,519,521,523,525,528-531,533,535,537-541,543-546,549,551-552,554,556-565,567-570,572-573,575,577-580,583-587,589-595,597-606,608-611,613-614,616-623,626,628-630,632-636,638-646,649-651,654-657,659,661-662,664-665,667-668,670-675,677-683,685-688,690-691,694-695,698-705,707-709,711-718,720-727,729-730,732-733","reason":"epilog failed for jobid fjQePTzX6kF","overwrite":0}}
+{"timestamp":1710817329.1571181,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1710817441.0567548,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1710817441.1569197,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1710817461.1561494,"name":"offline","context":{"idset":"121"}}
+{"timestamp":1710817689.0563293,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1710817689.1564579,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1710817749.1547937,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1710817783.1565175,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1710818391.056217,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1710818391.1563957,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1710818507.156311,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1710818591.1566999,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1710819089.1562781,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1710819251.1557908,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1710819333.1559651,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1710819367.1564162,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1710819393.0560434,"name":"drain","context":{"idset":"741","reason":"broker was unresponsive"}}
+{"timestamp":1710819393.1566978,"name":"drain","context":{"idset":"743","reason":"broker was unresponsive"}}
+{"timestamp":1710819395.0562375,"name":"drain","context":{"idset":"735","reason":"broker was unresponsive"}}
+{"timestamp":1710819395.056385,"name":"drain","context":{"idset":"739","reason":"broker was unresponsive"}}
+{"timestamp":1710819397.056211,"name":"drain","context":{"idset":"737","reason":"broker was unresponsive"}}
+{"timestamp":1710819405.0559831,"name":"drain","context":{"idset":"738","reason":"broker was unresponsive"}}
+{"timestamp":1710819405.0560517,"name":"drain","context":{"idset":"740","reason":"broker was unresponsive"}}
+{"timestamp":1710819405.0560977,"name":"drain","context":{"idset":"742","reason":"broker was unresponsive"}}
+{"timestamp":1710819409.0567255,"name":"drain","context":{"idset":"736","reason":"broker was unresponsive"}}
+{"timestamp":1710819459.0577605,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1710819459.0584817,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1710819459.0592136,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1710819459.0597832,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1710819459.0603263,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1710819459.0608225,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1710819459.0613253,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1710819459.0618067,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1710819459.0623116,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1710819459.0627789,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1710819459.1568565,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1710819469.0565956,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1710819469.0571549,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1710819469.0577276,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1710819469.0581832,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1710819469.0586557,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1710819469.0590968,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1710819469.1564577,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1710819471.0565236,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1710819471.0571544,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1710819471.0576024,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1710819471.0580559,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1710819471.1558311,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1710869263.0576186,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0577836,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0578148,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0578403,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0578637,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0578885,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0579154,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0579395,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.057965,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0579915,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0580194,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0580454,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0580699,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0580978,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0581241,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0581503,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0581782,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0582056,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.058234,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0582631,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0582998,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.058331,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0583601,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0583911,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0584216,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0584524,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0584841,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0585153,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0585501,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0585835,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0586157,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0586479,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0586817,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0587151,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0587494,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0587852,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.05882,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0588539,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0588887,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0589247,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0589612,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0589976,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0590351,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0590732,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0591111,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0591493,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0591884,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0592272,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0592654,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0593081,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0593476,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0593867,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0594268,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0594642,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0595012,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0595393,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0595779,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.0596201,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1710869263.1566422,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1710869325.0610645,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1710869325.0616241,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1710869325.0621486,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1710869325.0626669,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1710869325.063185,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1710869325.0636346,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1710869325.0640848,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1710869325.0645487,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1710869325.0649669,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1710869325.0654123,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1710869325.0658281,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1710869325.0662417,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1710869325.0666888,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1710869325.0671086,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1710869325.0675261,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1710869325.0679479,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1710869325.0683446,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1710869325.0687182,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1710869325.0690851,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1710869325.069454,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1710869325.0698054,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1710869325.0701485,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1710869325.070492,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1710869325.0708294,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1710869325.071151,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1710869325.0714862,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1710869325.0717998,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1710869325.0721107,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1710869325.0724187,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1710869325.0727103,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1710869325.0729926,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1710869325.0732684,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1710869325.0735784,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1710869325.0738885,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1710869325.0741804,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1710869325.0744736,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1710869325.0747809,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1710869325.0755484,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1710869325.075598,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1710869325.0756443,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1710869325.0757794,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1710869325.0760036,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1710869325.0762172,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1710869325.0764704,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1710869325.0766964,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1710869325.076884,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1710869325.0770671,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1710869325.0772407,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1710869325.077424,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1710869325.0775862,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1710869325.0777488,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1710869325.0778985,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1710869325.0780413,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1710869325.0781765,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1710869325.0783114,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1710869325.0784366,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1710869325.0785511,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1710869325.0786557,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1710869325.0787683,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1710869325.1568775,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1711043853.6097665,"name":"online","context":{"idset":"62"}}
+{"timestamp":1711043853.6169827,"name":"online","context":{"idset":"68"}}
+{"timestamp":1711043853.7263978,"name":"online","context":{"idset":"63"}}
+{"timestamp":1711043853.746753,"name":"online","context":{"idset":"66"}}
+{"timestamp":1711043853.7660441,"name":"online","context":{"idset":"86"}}
+{"timestamp":1711043853.7776821,"name":"online","context":{"idset":"85"}}
+{"timestamp":1711043853.8533976,"name":"online","context":{"idset":"78"}}
+{"timestamp":1711043853.8927028,"name":"online","context":{"idset":"75,133"}}
+{"timestamp":1711043853.9360697,"name":"online","context":{"idset":"69,82"}}
+{"timestamp":1711043853.9875276,"name":"online","context":{"idset":"65"}}
+{"timestamp":1711043854.0337083,"name":"online","context":{"idset":"100"}}
+{"timestamp":1711043854.0914009,"name":"online","context":{"idset":"64"}}
+{"timestamp":1711043854.1006334,"name":"online","context":{"idset":"61,141"}}
+{"timestamp":1711043854.6302047,"name":"online","context":{"idset":"148,167"}}
+{"timestamp":1711043854.6566427,"name":"online","context":{"idset":"156"}}
+{"timestamp":1711043854.8204851,"name":"online","context":{"idset":"77,135,149,157"}}
+{"timestamp":1711043854.9067965,"name":"online","context":{"idset":"140,153"}}
+{"timestamp":1711043855.0123675,"name":"online","context":{"idset":"218"}}
+{"timestamp":1711043855.057533,"name":"online","context":{"idset":"76,95,158,169"}}
+{"timestamp":1711043855.066983,"name":"online","context":{"idset":"212"}}
+{"timestamp":1711043855.1150224,"name":"online","context":{"idset":"99,197"}}
+{"timestamp":1711043855.1546736,"name":"online","context":{"idset":"127,210"}}
+{"timestamp":1711043855.2108786,"name":"online","context":{"idset":"107,109"}}
+{"timestamp":1711043855.230541,"name":"online","context":{"idset":"105,170"}}
+{"timestamp":1711043855.25337,"name":"online","context":{"idset":"108,208"}}
+{"timestamp":1711043855.2757843,"name":"online","context":{"idset":"136,179,226"}}
+{"timestamp":1711043855.3198934,"name":"online","context":{"idset":"90,143"}}
+{"timestamp":1711043855.3394427,"name":"online","context":{"idset":"194"}}
+{"timestamp":1711043855.361938,"name":"online","context":{"idset":"166,188"}}
+{"timestamp":1711043855.3730426,"name":"online","context":{"idset":"81,150"}}
+{"timestamp":1711043855.4416869,"name":"online","context":{"idset":"92,122,129,145,184,187"}}
+{"timestamp":1711043855.457623,"name":"online","context":{"idset":"88"}}
+{"timestamp":1711043855.4745526,"name":"online","context":{"idset":"101,201"}}
+{"timestamp":1711043855.4971251,"name":"online","context":{"idset":"73"}}
+{"timestamp":1711043855.5057347,"name":"online","context":{"idset":"125,160"}}
+{"timestamp":1711043855.5680835,"name":"online","context":{"idset":"91,137,161"}}
+{"timestamp":1711043855.6181493,"name":"online","context":{"idset":"97,104,106,120,152"}}
+{"timestamp":1711043855.6604161,"name":"online","context":{"idset":"172,219"}}
+{"timestamp":1711043855.8224227,"name":"online","context":{"idset":"96,205,213"}}
+{"timestamp":1711043855.9325089,"name":"online","context":{"idset":"89,134,177,193,196"}}
+{"timestamp":1711043856.0409937,"name":"online","context":{"idset":"94,113,116,171,191,200,211,217"}}
+{"timestamp":1711043856.1822288,"name":"online","context":{"idset":"139,178,202"}}
+{"timestamp":1711043856.4595878,"name":"online","context":{"idset":"180"}}
+{"timestamp":1711043856.6248088,"name":"online","context":{"idset":"93,112,155,181,204,224"}}
+{"timestamp":1711043856.7525764,"name":"online","context":{"idset":"110,164,174-175,185,206"}}
+{"timestamp":1711043856.8535709,"name":"online","context":{"idset":"103,130,147,151,173,189-190,221"}}
+{"timestamp":1711043857.0015433,"name":"online","context":{"idset":"131,146,176,209,222"}}
+{"timestamp":1711043857.0565093,"name":"online","context":{"idset":"79,165"}}
+{"timestamp":1711043857.155323,"name":"online","context":{"idset":"115,126,183,223"}}
+{"timestamp":1711043857.378252,"name":"online","context":{"idset":"203"}}
+{"timestamp":1711043858.0666997,"name":"online","context":{"idset":"225"}}
+{"timestamp":1711043862.945637,"name":"online","context":{"idset":"227,254"}}
+{"timestamp":1711043863.0421722,"name":"online","context":{"idset":"229-230"}}
+{"timestamp":1711043863.178509,"name":"online","context":{"idset":"231,233,237,241"}}
+{"timestamp":1711043863.2532473,"name":"online","context":{"idset":"240"}}
+{"timestamp":1711043863.2844708,"name":"online","context":{"idset":"238"}}
+{"timestamp":1711043863.3001859,"name":"online","context":{"idset":"243"}}
+{"timestamp":1711043863.3548727,"name":"online","context":{"idset":"251"}}
+{"timestamp":1711043863.4419155,"name":"online","context":{"idset":"246,250"}}
+{"timestamp":1711043863.5013776,"name":"online","context":{"idset":"247,249"}}
+{"timestamp":1711043863.9815829,"name":"online","context":{"idset":"259"}}
+{"timestamp":1711043864.0770781,"name":"online","context":{"idset":"258"}}
+{"timestamp":1711043864.0923066,"name":"online","context":{"idset":"257,260"}}
+{"timestamp":1711043864.1106663,"name":"online","context":{"idset":"262"}}
+{"timestamp":1711043864.229434,"name":"online","context":{"idset":"264,266"}}
+{"timestamp":1711043864.3925791,"name":"online","context":{"idset":"261,268,270,275"}}
+{"timestamp":1711043864.4778786,"name":"online","context":{"idset":"278,283-284"}}
+{"timestamp":1711043864.5019267,"name":"online","context":{"idset":"287,289,295"}}
+{"timestamp":1711043864.5141697,"name":"online","context":{"idset":"299"}}
+{"timestamp":1711043864.5162156,"name":"online","context":{"idset":"286"}}
+{"timestamp":1711043864.5786972,"name":"online","context":{"idset":"273,292,300"}}
+{"timestamp":1711043864.6447332,"name":"online","context":{"idset":"297"}}
+{"timestamp":1711043864.6649079,"name":"online","context":{"idset":"282"}}
+{"timestamp":1711043864.6830547,"name":"online","context":{"idset":"301"}}
+{"timestamp":1711043864.6970642,"name":"online","context":{"idset":"305"}}
+{"timestamp":1711043864.7250984,"name":"online","context":{"idset":"307"}}
+{"timestamp":1711043864.7616029,"name":"online","context":{"idset":"296,298"}}
+{"timestamp":1711043864.8412278,"name":"online","context":{"idset":"308,313,320"}}
+{"timestamp":1711043864.9634428,"name":"online","context":{"idset":"309,323"}}
+{"timestamp":1711043864.9894693,"name":"online","context":{"idset":"317,319,322"}}
+{"timestamp":1711043865.0053005,"name":"online","context":{"idset":"311,318,321"}}
+{"timestamp":1711043865.1568139,"name":"online","context":{"idset":"316,327-328,331,335,338,341,343,345"}}
+{"timestamp":1711043865.1883297,"name":"online","context":{"idset":"342"}}
+{"timestamp":1711043865.2916341,"name":"online","context":{"idset":"324,326,333,336,344"}}
+{"timestamp":1711043865.423614,"name":"online","context":{"idset":"332,346-348,355,357"}}
+{"timestamp":1711043865.5363388,"name":"online","context":{"idset":"349,351,353-354,356,359,361,363-364,366,371"}}
+{"timestamp":1711043865.5525229,"name":"online","context":{"idset":"368"}}
+{"timestamp":1711043865.6653364,"name":"online","context":{"idset":"358,370,372,374"}}
+{"timestamp":1711043865.7026339,"name":"online","context":{"idset":"375"}}
+{"timestamp":1711043866.0402312,"name":"online","context":{"idset":"376,379-380,383-386,389"}}
+{"timestamp":1711043866.1775675,"name":"online","context":{"idset":"388,390-391,396-397"}}
+{"timestamp":1711043866.2823098,"name":"online","context":{"idset":"392-393,395,399-401"}}
+{"timestamp":1711043866.3948863,"name":"online","context":{"idset":"405-408,411,413"}}
+{"timestamp":1711043866.5171127,"name":"online","context":{"idset":"409,412,414-415,417"}}
+{"timestamp":1711043866.7535062,"name":"online","context":{"idset":"418"}}
+{"timestamp":1711043867.0570314,"name":"online","context":{"idset":"394"}}
+{"timestamp":1711043867.2675965,"name":"online","context":{"idset":"419"}}
+{"timestamp":1711043872.3261156,"name":"online","context":{"idset":"420,423"}}
+{"timestamp":1711043872.4036798,"name":"online","context":{"idset":"421"}}
+{"timestamp":1711043872.425539,"name":"online","context":{"idset":"427"}}
+{"timestamp":1711043872.4409862,"name":"online","context":{"idset":"424"}}
+{"timestamp":1711043872.624186,"name":"online","context":{"idset":"429,432"}}
+{"timestamp":1711043872.6459727,"name":"online","context":{"idset":"426,430"}}
+{"timestamp":1711043872.7070951,"name":"online","context":{"idset":"435"}}
+{"timestamp":1711043872.7794058,"name":"online","context":{"idset":"433-434"}}
+{"timestamp":1711043872.7991343,"name":"online","context":{"idset":"436"}}
+{"timestamp":1711043872.8897681,"name":"online","context":{"idset":"439-440"}}
+{"timestamp":1711043872.9303873,"name":"online","context":{"idset":"438"}}
+{"timestamp":1711043873.2269223,"name":"online","context":{"idset":"441"}}
+{"timestamp":1711043873.4530399,"name":"online","context":{"idset":"444,448-450"}}
+{"timestamp":1711043873.4937477,"name":"online","context":{"idset":"442,446,451"}}
+{"timestamp":1711043873.7098074,"name":"online","context":{"idset":"455"}}
+{"timestamp":1711043873.7336686,"name":"online","context":{"idset":"452"}}
+{"timestamp":1711043873.8397915,"name":"online","context":{"idset":"460,463,467"}}
+{"timestamp":1711043873.887408,"name":"online","context":{"idset":"459,462,469"}}
+{"timestamp":1711043873.8952076,"name":"online","context":{"idset":"456"}}
+{"timestamp":1711043873.9342651,"name":"online","context":{"idset":"461"}}
+{"timestamp":1711043873.9739466,"name":"online","context":{"idset":"471,475-476"}}
+{"timestamp":1711043873.9948473,"name":"online","context":{"idset":"466,480"}}
+{"timestamp":1711043874.0423858,"name":"online","context":{"idset":"464,468"}}
+{"timestamp":1711043874.1415482,"name":"online","context":{"idset":"472,478,484-485"}}
+{"timestamp":1711043874.1825497,"name":"online","context":{"idset":"479,483"}}
+{"timestamp":1711043874.2083426,"name":"online","context":{"idset":"481"}}
+{"timestamp":1711043874.246043,"name":"online","context":{"idset":"491"}}
+{"timestamp":1711043874.2801111,"name":"online","context":{"idset":"493"}}
+{"timestamp":1711043874.2962403,"name":"online","context":{"idset":"488"}}
+{"timestamp":1711043874.3720953,"name":"online","context":{"idset":"486"}}
+{"timestamp":1711043874.4309316,"name":"online","context":{"idset":"490,492,496"}}
+{"timestamp":1711043874.5500824,"name":"online","context":{"idset":"487,494,497-498,502-503,505,508-509,512"}}
+{"timestamp":1711043874.62639,"name":"online","context":{"idset":"499,507"}}
+{"timestamp":1711043874.7407358,"name":"online","context":{"idset":"500-501,511,513,516,519"}}
+{"timestamp":1711043874.8583007,"name":"online","context":{"idset":"510,514,521"}}
+{"timestamp":1711043875.0247638,"name":"online","context":{"idset":"525,530-531,533,535,540,543"}}
+{"timestamp":1711043875.0587094,"name":"online","context":{"idset":"523,529,538,545"}}
+{"timestamp":1711043875.1226532,"name":"online","context":{"idset":"528,537,539,541,544,546"}}
+{"timestamp":1711043875.2743871,"name":"online","context":{"idset":"549,551"}}
+{"timestamp":1711043875.3931193,"name":"online","context":{"idset":"556-557,560"}}
+{"timestamp":1711043875.4940245,"name":"online","context":{"idset":"552,554,558,561-562"}}
+{"timestamp":1711043875.5988514,"name":"online","context":{"idset":"559,563-564,569"}}
+{"timestamp":1711043875.7414353,"name":"online","context":{"idset":"565,567-568,572-573,577,583,585"}}
+{"timestamp":1711043875.8467093,"name":"online","context":{"idset":"570,575,579-580,584,586-587,589"}}
+{"timestamp":1711043875.991456,"name":"online","context":{"idset":"578"}}
+{"timestamp":1711043876.1837392,"name":"online","context":{"idset":"590"}}
+{"timestamp":1711043876.4602706,"name":"online","context":{"idset":"591"}}
+{"timestamp":1711043876.6831961,"name":"online","context":{"idset":"592"}}
+{"timestamp":1711043881.722702,"name":"online","context":{"idset":"593"}}
+{"timestamp":1711043881.7957618,"name":"online","context":{"idset":"594-595,597"}}
+{"timestamp":1711043881.8189983,"name":"online","context":{"idset":"598"}}
+{"timestamp":1711043881.951098,"name":"online","context":{"idset":"602"}}
+{"timestamp":1711043882.052597,"name":"online","context":{"idset":"599-600,606"}}
+{"timestamp":1711043882.0800047,"name":"online","context":{"idset":"601"}}
+{"timestamp":1711043882.1618063,"name":"online","context":{"idset":"604,608"}}
+{"timestamp":1711043882.1775355,"name":"online","context":{"idset":"605"}}
+{"timestamp":1711043882.1985629,"name":"online","context":{"idset":"609"}}
+{"timestamp":1711043882.2585025,"name":"online","context":{"idset":"610"}}
+{"timestamp":1711043882.2781732,"name":"online","context":{"idset":"603"}}
+{"timestamp":1711043882.6137419,"name":"online","context":{"idset":"611"}}
+{"timestamp":1711043882.732568,"name":"online","context":{"idset":"614,616"}}
+{"timestamp":1711043882.7483504,"name":"online","context":{"idset":"613"}}
+{"timestamp":1711043882.9757915,"name":"online","context":{"idset":"617-620"}}
+{"timestamp":1711043883.0336962,"name":"online","context":{"idset":"621-622"}}
+{"timestamp":1711043883.1039362,"name":"online","context":{"idset":"623"}}
+{"timestamp":1711043883.1436698,"name":"online","context":{"idset":"626"}}
+{"timestamp":1711043883.2045076,"name":"online","context":{"idset":"630"}}
+{"timestamp":1711043883.2409606,"name":"online","context":{"idset":"632"}}
+{"timestamp":1711043883.2581818,"name":"online","context":{"idset":"633"}}
+{"timestamp":1711043883.2864559,"name":"online","context":{"idset":"628"}}
+{"timestamp":1711043883.3144734,"name":"online","context":{"idset":"640"}}
+{"timestamp":1711043883.3359888,"name":"online","context":{"idset":"635"}}
+{"timestamp":1711043883.3538744,"name":"online","context":{"idset":"638,642"}}
+{"timestamp":1711043883.3849497,"name":"online","context":{"idset":"634,643"}}
+{"timestamp":1711043883.4019237,"name":"online","context":{"idset":"641"}}
+{"timestamp":1711043883.4258163,"name":"online","context":{"idset":"636,639,644,649"}}
+{"timestamp":1711043883.5855112,"name":"online","context":{"idset":"645-646,656"}}
+{"timestamp":1711043883.7223222,"name":"online","context":{"idset":"650-651,654-655,657,662"}}
+{"timestamp":1711043883.8402314,"name":"online","context":{"idset":"661,664-665,667,670"}}
+{"timestamp":1711043883.957052,"name":"online","context":{"idset":"659,672-673,675"}}
+{"timestamp":1711043884.1060047,"name":"online","context":{"idset":"668,671,674,677-678,680"}}
+{"timestamp":1711043884.2222345,"name":"online","context":{"idset":"681-682,688,690,711"}}
+{"timestamp":1711043884.3234611,"name":"online","context":{"idset":"679,683,685-686,691,694"}}
+{"timestamp":1711043884.5396099,"name":"online","context":{"idset":"695,698-699,701-703,705,707"}}
+{"timestamp":1711043884.6454527,"name":"online","context":{"idset":"700,704,708-709,716"}}
+{"timestamp":1711043884.7933547,"name":"online","context":{"idset":"712-713,717,721,730"}}
+{"timestamp":1711043884.9096448,"name":"online","context":{"idset":"718,720,722,726,729"}}
+{"timestamp":1711043885.0159178,"name":"online","context":{"idset":"725,727"}}
+{"timestamp":1711043885.0575488,"name":"online","context":{"idset":"714,733"}}
+{"timestamp":1711043885.1573596,"name":"online","context":{"idset":"715,724"}}
+{"timestamp":1711043885.3971114,"name":"online","context":{"idset":"723"}}
+{"timestamp":1711043915.7395539,"name":"online","context":{"idset":"365"}}
+{"timestamp":1711044204.8532975,"name":"undrain","context":{"idset":"61-66,68-69,73,75-79,81-82,85-86,88-97,99-101,103-110,112-113,115-117,120,122-123,125-127,129-131,133-137,139-141,143,145-153,155-158,160-161,164-167,169-181,183-185,187-191,193-194,196-197,200-206,208-213,217-219,221-227,229-231,233,237-238,240-241,243,246-247,249-251,254,257-262,264,266,268,270,273,275,278,282-284,286-287,289,292,295-301,305,307-309,311,313,316-324,326-328,331-333,335-336,338,341-349,351,353-359,361,363-366,368,370-372,374-376,379-380,383-386,388-397,399-401,405-409,411-415,417-421,423-424,426-427,429-430,432-436,438-442,444,446,448-452,455-456,459-464,466-469,471-472,475-476,478-481,483-488,490-494,496-503,505,507-514,516,519,521,523,525,528-531,533,535,537-541,543-546,549,551-552,554,556-565,567-570,572-573,575,577-580,583-587,589-595,597-606,608-611,613-614,616-623,626,628,630,632-636,638-646,649-651,654-657,659,661-662,664-665,667-668,670-675,677-683,685-686,688,690-691,694-695,698-705,707-709,711-718,720-727,729-730,732-733"}}
+{"timestamp":1711045556.7758074,"name":"online","context":{"idset":"142,360"}}
+{"timestamp":1711045556.8960454,"name":"online","context":{"idset":"83,162,198,310,506"}}
+{"timestamp":1711045556.9973896,"name":"online","context":{"idset":"124,132,154,255,263,629,687"}}
+{"timestamp":1711045571.5172381,"name":"undrain","context":{"idset":"83,124,132,142,154,162,198,255,263,310,360,506,629,687"}}
+{"timestamp":1711045707.2440991,"name":"online","context":{"idset":"67"}}
+{"timestamp":1711045707.3939149,"name":"online","context":{"idset":"159"}}
+{"timestamp":1711045707.4370403,"name":"online","context":{"idset":"70,80,128,215"}}
+{"timestamp":1711045707.4587536,"name":"online","context":{"idset":"74"}}
+{"timestamp":1711045707.5696881,"name":"online","context":{"idset":"144"}}
+{"timestamp":1711045707.6542063,"name":"online","context":{"idset":"138"}}
+{"timestamp":1711045707.6616466,"name":"online","context":{"idset":"118"}}
+{"timestamp":1711045707.6889668,"name":"online","context":{"idset":"87,111,182"}}
+{"timestamp":1711045707.7335472,"name":"online","context":{"idset":"72"}}
+{"timestamp":1711045707.7559359,"name":"online","context":{"idset":"71"}}
+{"timestamp":1711045707.7894437,"name":"online","context":{"idset":"102"}}
+{"timestamp":1711045707.7938247,"name":"online","context":{"idset":"98,168"}}
+{"timestamp":1711045707.8135192,"name":"online","context":{"idset":"239"}}
+{"timestamp":1711045707.8333793,"name":"online","context":{"idset":"207,214"}}
+{"timestamp":1711045707.9038489,"name":"online","context":{"idset":"84"}}
+{"timestamp":1711045707.9221249,"name":"online","context":{"idset":"163"}}
+{"timestamp":1711045707.9371967,"name":"online","context":{"idset":"216,228"}}
+{"timestamp":1711045708.0049574,"name":"online","context":{"idset":"248"}}
+{"timestamp":1711045708.047612,"name":"online","context":{"idset":"114,119"}}
+{"timestamp":1711045708.1762588,"name":"online","context":{"idset":"195,220,242,252,293"}}
+{"timestamp":1711045708.3194551,"name":"online","context":{"idset":"234,245,253"}}
+{"timestamp":1711045708.5565684,"name":"online","context":{"idset":"236,267,294,306"}}
+{"timestamp":1711045708.7438583,"name":"online","context":{"idset":"256,277,279,329,337"}}
+{"timestamp":1711045708.8528554,"name":"online","context":{"idset":"232,271,403"}}
+{"timestamp":1711045708.9535406,"name":"online","context":{"idset":"186,192,285,304,314,350,425,437"}}
+{"timestamp":1711045709.0721719,"name":"online","context":{"idset":"235,244,274,288,291,302,312,352,382,398,410,422,457,520"}}
+{"timestamp":1711045709.1760237,"name":"online","context":{"idset":"265,269,340,381,416,431,447,547-548"}}
+{"timestamp":1711045709.2916746,"name":"online","context":{"idset":"272,276,339,367,387,402,404,445,453-454,465,470,474,482,515,517,526,532,536"}}
+{"timestamp":1711045709.4105935,"name":"online","context":{"idset":"280-281,303,315,330,334,362,369,373,377-378,428,443,458,473,522,524,527,534,550"}}
+{"timestamp":1711045709.5324194,"name":"online","context":{"idset":"290,325,477,489,495,504,518,542"}}
+{"timestamp":1711045716.7482092,"name":"online","context":{"idset":"553"}}
+{"timestamp":1711045717.0085201,"name":"online","context":{"idset":"566,571,576,582"}}
+{"timestamp":1711045717.0581036,"name":"online","context":{"idset":"555,574,581"}}
+{"timestamp":1711045717.1585116,"name":"online","context":{"idset":"588"}}
+{"timestamp":1711045717.2662265,"name":"online","context":{"idset":"596,615"}}
+{"timestamp":1711045717.3009255,"name":"online","context":{"idset":"612"}}
+{"timestamp":1711045717.427125,"name":"online","context":{"idset":"627,637,648,653,658"}}
+{"timestamp":1711045717.5278296,"name":"online","context":{"idset":"607,631,647,652,684,689"}}
+{"timestamp":1711045717.6330683,"name":"online","context":{"idset":"624-625,660,666,669,697"}}
+{"timestamp":1711045717.7337651,"name":"online","context":{"idset":"692,696,706"}}
+{"timestamp":1711045717.8563025,"name":"online","context":{"idset":"676,693,710,719,728"}}
+{"timestamp":1711045717.9761233,"name":"online","context":{"idset":"735,740"}}
+{"timestamp":1711045718.097568,"name":"online","context":{"idset":"736-739,741-742"}}
+{"timestamp":1711045718.8460517,"name":"online","context":{"idset":"743"}}
+{"timestamp":1711046097.961571,"name":"undrain","context":{"idset":"735-743"}}
+{"timestamp":1711046690.4270704,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1711046690.5264163,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1711046690.747952,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1711046692.3598983,"name":"online","context":{"idset":"61-62"}}
+{"timestamp":1711046692.6253867,"name":"online","context":{"idset":"63"}}
+{"timestamp":1711046695.2138464,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1711046695.2397008,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1711046695.2965109,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1711046695.3548515,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1711046695.3837342,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1711046695.3920996,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1711046695.4184899,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1711046695.423218,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1711046695.447567,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1711046695.4518905,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1711046695.4769163,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1711046695.4819975,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1711046695.486526,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1711046695.493279,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1711046695.520829,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1711046695.5250018,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1711046695.5535526,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1711046695.5578427,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1711046695.5651674,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1711046695.5882638,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1711046695.5934277,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1711046695.5979869,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1711046695.6061132,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1711046695.6110024,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1711046695.6159971,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1711046695.620178,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1711046695.6266372,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1711046695.6280782,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1711046695.640712,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1711046695.6422203,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1711046695.6724355,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1711046695.6774354,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1711046695.6826813,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1711046695.6875122,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1711046695.6949844,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1711046695.7004395,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1711046695.7073205,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1711046695.7134697,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1711046695.7139413,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1711046695.7314799,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1711046695.7324376,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1711046695.7337489,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1711046695.7554684,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1711046695.765311,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1711046695.8021703,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1711046695.8108351,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1711046695.8159196,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1711046695.822859,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1711046695.8599968,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1711046695.867183,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1711046695.8790309,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1711046695.8880229,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1711046695.9009786,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1711046695.9016933,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1711046695.903167,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1711046695.928951,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1711046695.9379516,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1711046695.9491551,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1711046695.9516196,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1711046695.9541731,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1711046695.9590871,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1711046695.9706819,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1711046695.9784365,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1711046695.9939969,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1711046696.0120077,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1711046696.0144222,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1711046696.025996,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1711046696.0265567,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1711046696.0330455,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1711046696.0389192,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1711046696.0445015,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1711046696.0460517,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1711046696.0617578,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1711046696.0625122,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1711046696.069989,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1711046696.0715475,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1711046696.0766537,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1711046696.0840681,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1711046696.0848236,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1711046696.1148131,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1711046696.1222382,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1711046696.1229808,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1711046696.130976,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1711046696.1720433,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1711046696.1785994,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1711046696.1802225,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1711046696.19892,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1711046696.205267,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1711046696.2107968,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1711046696.2115498,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1711046696.2198684,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1711046696.2259405,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1711046696.2380075,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1711046696.245281,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1711046696.251868,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1711046696.253432,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1711046696.2586257,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1711046696.2647417,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1711046696.2658029,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1711046696.2665594,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1711046696.2719798,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1711046696.2759414,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1711046696.2768717,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1711046696.2901001,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1711046696.2987185,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1711046696.30615,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1711046696.313596,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1711046696.3142574,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1711046696.3165047,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1711046696.3200815,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1711046696.3233542,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1711046696.3240857,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1711046696.3286409,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1711046696.3293624,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1711046696.4349201,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1711046696.4444664,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1711046696.4550405,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1711046696.4614496,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1711046696.4699471,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1711046696.4762881,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1711046696.482693,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1711046696.4833996,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1711046696.4879658,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1711046696.4900808,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1711046696.4964597,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1711046696.8359802,"name":"online","context":{"idset":"66,70,72"}}
+{"timestamp":1711046696.9724929,"name":"online","context":{"idset":"65,68-69,73,79,85,90"}}
+{"timestamp":1711046697.0397391,"name":"online","context":{"idset":"67,74,77,97"}}
+{"timestamp":1711046697.1484253,"name":"online","context":{"idset":"64,71,78,82-84,86,88-89,92,96,100-101,103"}}
+{"timestamp":1711046697.1908987,"name":"online","context":{"idset":"76,87,91,93-95,98,114,140"}}
+{"timestamp":1711046697.4039412,"name":"online","context":{"idset":"75,81,99,110,115-116,125,127,142,144,153"}}
+{"timestamp":1711046697.4299555,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1711046697.5300925,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1711046697.7472794,"name":"online","context":{"idset":"104,106,112,129,149,158,162,164-165,167,182"}}
+{"timestamp":1711046697.7473409,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1711046697.7875476,"name":"online","context":{"idset":"80,102,105,107,109,113,118-119,130-131,139,147"}}
+{"timestamp":1711046697.9752114,"name":"online","context":{"idset":"111,120,122,126,128,132-138,141,143,145-146,148,150-152,154-157,159-161,163,166,168-181,184-191"}}
+{"timestamp":1711046698.0984459,"name":"online","context":{"idset":"183"}}
+{"timestamp":1711046698.4877934,"name":"online","context":{"idset":"124"}}
+{"timestamp":1711046698.6322327,"name":"online","context":{"idset":"108"}}
+{"timestamp":1711046698.9958601,"name":"online","context":{"idset":"192-193"}}
+{"timestamp":1711046699.2428694,"name":"online","context":{"idset":"194"}}
+{"timestamp":1711046702.2261782,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1711046702.2629385,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1711046702.3024576,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1711046702.4976268,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1711046702.5235901,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1711046702.528821,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1711046702.5334477,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1711046702.5716581,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1711046702.6014771,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1711046702.6356802,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1711046702.6648281,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1711046702.6960611,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1711046702.7048602,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1711046702.731982,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1711046702.7364423,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1711046702.7411804,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1711046702.7460356,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1711046702.7675571,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1711046702.7897139,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1711046702.8111665,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1711046702.8229802,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1711046702.8309083,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1711046702.8351429,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1711046702.8586493,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1711046702.8647516,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1711046702.8894203,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1711046702.8952827,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1711046702.9616446,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1711046702.9701879,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1711046703.0380909,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1711046703.0862424,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1711046703.0951858,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1711046703.1026669,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1711046703.116065,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1711046703.1517668,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1711046703.1585989,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1711046703.1661165,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1711046703.1735954,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1711046703.2031019,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1711046703.2109923,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1711046703.2216821,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1711046703.2224965,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1711046703.263418,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1711046703.2715216,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1711046703.2782331,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1711046703.3102198,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1711046703.3188014,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1711046703.3308735,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1711046703.3757155,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1711046703.376123,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1711046703.3824563,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1711046703.3832366,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1711046703.3956852,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1711046703.430706,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1711046703.4394312,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1711046703.4477665,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1711046703.4622815,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1711046703.4638917,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1711046703.4699697,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1711046703.4822884,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1711046703.4918814,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1711046703.497843,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1711046703.4985874,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1711046703.5079131,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1711046703.5205746,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1711046703.5239606,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1711046703.5400085,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1711046703.5408435,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1711046703.5415947,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1711046703.5463865,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1711046703.558279,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1711046703.5886195,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1711046703.5896335,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1711046703.5916333,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1711046703.611156,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1711046703.6191564,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1711046703.6239014,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1711046703.6245763,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1711046703.63831,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1711046703.6451571,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1711046703.6515474,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1711046703.6586268,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1711046703.6688218,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1711046703.6695979,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1711046703.675534,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1711046703.6853671,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1711046703.6915808,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1711046703.6967793,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1711046703.7072353,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1711046703.7165196,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1711046703.7182415,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1711046703.7236128,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1711046703.732702,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1711046703.740536,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1711046703.7570496,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1711046703.7650876,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1711046703.7722409,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1711046703.7754846,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1711046703.781302,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1711046703.7868619,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1711046703.7946112,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1711046703.8096914,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1711046703.8104525,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1711046703.8111954,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1711046703.8174176,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1711046703.8256323,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1711046703.8307712,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1711046703.8345294,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1711046703.840574,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1711046703.8423483,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1711046703.8597441,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1711046703.8673706,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1711046703.8731003,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1711046703.8752677,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1711046703.8758891,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1711046703.8765483,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1711046703.893153,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1711046703.8939121,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1711046703.8988588,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1711046703.9077961,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1711046703.9116833,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1711046703.9178953,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1711046703.9238653,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1711046703.9244578,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1711046703.9592998,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1711046704.2490456,"name":"online","context":{"idset":"195-198,200-212,214"}}
+{"timestamp":1711046704.3845496,"name":"online","context":{"idset":"213,215,217-221,225,227"}}
+{"timestamp":1711046704.4222002,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1711046704.5220013,"name":"online","context":{"idset":"222,226"}}
+{"timestamp":1711046704.5220559,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1711046704.5714989,"name":"online","context":{"idset":"233"}}
+{"timestamp":1711046704.6155286,"name":"online","context":{"idset":"234"}}
+{"timestamp":1711046704.7157178,"name":"online","context":{"idset":"216,223-224,229,235,238,241,243,245"}}
+{"timestamp":1711046704.7157671,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1711046704.7539158,"name":"online","context":{"idset":"236,254,258"}}
+{"timestamp":1711046704.8597856,"name":"online","context":{"idset":"231,242,251"}}
+{"timestamp":1711046704.9709094,"name":"online","context":{"idset":"244,256,266"}}
+{"timestamp":1711046705.1420372,"name":"online","context":{"idset":"239"}}
+{"timestamp":1711046705.3633869,"name":"online","context":{"idset":"228,230,232,237,240,246-250,252-253,255,257,259-265,267-320"}}
+{"timestamp":1711046705.9688463,"name":"online","context":{"idset":"321-322"}}
+{"timestamp":1711046706.1971066,"name":"online","context":{"idset":"323"}}
+{"timestamp":1711046709.2021868,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1711046709.2970848,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1711046709.3437188,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1711046709.4995708,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1711046709.5456007,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1711046709.5732808,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1711046709.5803077,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1711046709.622766,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1711046709.6304929,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1711046709.6369867,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1711046709.6840646,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1711046709.7533367,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1711046709.7612629,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1711046709.767179,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1711046709.7934821,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1711046709.7989047,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1711046709.8041978,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1711046709.8412488,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1711046709.8618598,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1711046709.8689058,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1711046709.8912325,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1711046709.8979969,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1711046709.9209898,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1711046709.9263403,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1711046709.9518704,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1711046709.9795156,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1711046710.0411758,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1711046710.0641651,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1711046710.0693579,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1711046710.0985065,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1711046710.1674979,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1711046710.1756902,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1711046710.1821694,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1711046710.2251558,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1711046710.2544582,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1711046710.2829611,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1711046710.2959282,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1711046710.3197665,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1711046710.3263605,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1711046710.331737,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1711046710.3379023,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1711046710.3685675,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1711046710.3737791,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1711046710.4251959,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1711046710.4358115,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1711046710.5140345,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1711046710.5529804,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1711046710.5599496,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1711046710.5945807,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1711046710.6023805,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1711046710.6341372,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1711046710.6403761,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1711046710.6449761,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1711046710.6504724,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1711046710.6582074,"name":"online","context":{"idset":"324"}}
+{"timestamp":1711046710.6657045,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1711046710.6769028,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1711046710.6850789,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1711046710.6926973,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1711046710.7221766,"name":"online","context":{"idset":"325"}}
+{"timestamp":1711046710.7222083,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1711046710.728663,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1711046710.7336395,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1711046710.7386358,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1711046710.7689564,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1711046710.7746108,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1711046710.779731,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1711046710.7873275,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1711046710.8151381,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1711046710.8207464,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1711046710.852324,"name":"online","context":{"idset":"326"}}
+{"timestamp":1711046710.8523774,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1711046710.8580866,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1711046710.8657191,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1711046710.8665767,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1711046710.8734009,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1711046710.8923564,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1711046710.8932629,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1711046710.9103215,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1711046710.94628,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1711046710.9520147,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1711046710.9571149,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1711046710.957577,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1711046710.9626021,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1711046710.9776306,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1711046710.9784031,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1711046710.9846685,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1711046710.990931,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1711046710.9966707,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1711046710.9975679,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1711046711.0272865,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1711046711.0288272,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1711046711.031872,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1711046711.0325119,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1711046711.0333552,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1711046711.0453477,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1711046711.0599864,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1711046711.0744438,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1711046711.0822101,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1711046711.0839581,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1711046711.0926325,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1711046711.0930259,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1711046711.1045432,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1711046711.1129463,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1711046711.1199996,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1711046711.1207573,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1711046711.127126,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1711046711.1343489,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1711046711.1395223,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1711046711.147253,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1711046711.1479328,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1711046711.158313,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1711046711.1726887,"name":"online","context":{"idset":"327-328"}}
+{"timestamp":1711046711.1727221,"name":"offline","context":{"idset":"445"}}
+{"timestamp":1711046711.1869314,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1711046711.1929221,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1711046711.2005122,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1711046711.2159364,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1711046711.222615,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1711046711.2300811,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1711046711.2379408,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1711046711.2454977,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1711046711.2656033,"name":"offline","context":{"idset":"447"}}
+{"timestamp":1711046711.2744539,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1711046711.2750866,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1711046711.2828355,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1711046711.2902143,"name":"offline","context":{"idset":"448"}}
+{"timestamp":1711046711.3123322,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1711046711.3228052,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1711046711.376946,"name":"online","context":{"idset":"329-333"}}
+{"timestamp":1711046711.4259312,"name":"offline","context":{"idset":"450"}}
+{"timestamp":1711046711.4379301,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1711046711.5279605,"name":"online","context":{"idset":"334"}}
+{"timestamp":1711046711.5869496,"name":"online","context":{"idset":"335-351"}}
+{"timestamp":1711046711.6959851,"name":"online","context":{"idset":"352-355"}}
+{"timestamp":1711046711.8034487,"name":"online","context":{"idset":"356-360,363"}}
+{"timestamp":1711046711.9134216,"name":"online","context":{"idset":"361-362,364-366,370"}}
+{"timestamp":1711046711.9134753,"name":"offline","context":{"idset":"457"}}
+{"timestamp":1711046711.9631023,"name":"online","context":{"idset":"367"}}
+{"timestamp":1711046712.1191373,"name":"online","context":{"idset":"369,371-374,376"}}
+{"timestamp":1711046712.2274442,"name":"online","context":{"idset":"368,375,377,380,384-385,388"}}
+{"timestamp":1711046712.3533225,"name":"online","context":{"idset":"378-379,386-387,389,392,394-395,401"}}
+{"timestamp":1711046712.5366991,"name":"online","context":{"idset":"393,397"}}
+{"timestamp":1711046712.6860764,"name":"online","context":{"idset":"381,390,399,419-420,423,429,431,434-435,437,439,441,443,445"}}
+{"timestamp":1711046712.7862978,"name":"online","context":{"idset":"382-383,391,396,398,400,402-418,421-422,424-428,430,432,436,438,442,444,446-448"}}
+{"timestamp":1711046712.7863495,"name":"offline","context":{"idset":"451"}}
+{"timestamp":1711046712.8290598,"name":"online","context":{"idset":"440"}}
+{"timestamp":1711046712.9862163,"name":"online","context":{"idset":"449-450"}}
+{"timestamp":1711046713.1820123,"name":"online","context":{"idset":"433"}}
+{"timestamp":1711046713.7574637,"name":"online","context":{"idset":"457"}}
+{"timestamp":1711046714.25805,"name":"online","context":{"idset":"451"}}
+{"timestamp":1711046716.0342422,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1711046716.2026801,"name":"offline","context":{"idset":"452"}}
+{"timestamp":1711046716.2672818,"name":"offline","context":{"idset":"453"}}
+{"timestamp":1711046716.3414626,"name":"offline","context":{"idset":"454"}}
+{"timestamp":1711046716.495342,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1711046716.5606532,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1711046716.5705931,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1711046716.6170356,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1711046716.6283514,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1711046716.6554418,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1711046716.7029493,"name":"offline","context":{"idset":"462"}}
+{"timestamp":1711046716.7125747,"name":"offline","context":{"idset":"456"}}
+{"timestamp":1711046716.7899377,"name":"offline","context":{"idset":"464"}}
+{"timestamp":1711046716.7962141,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1711046716.8175504,"name":"offline","context":{"idset":"468"}}
+{"timestamp":1711046716.8226101,"name":"offline","context":{"idset":"466"}}
+{"timestamp":1711046716.8266866,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1711046716.8308551,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1711046716.8521092,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1711046716.856952,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1711046716.9118502,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1711046716.9326715,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1711046716.9376686,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1711046716.960274,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1711046716.9658093,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1711046717.0328913,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1711046717.0742834,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1711046717.1019545,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1711046717.141927,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1711046717.203377,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1711046717.2123575,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1711046717.2414258,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1711046717.2464106,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1711046717.2531879,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1711046717.2854354,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1711046717.291698,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1711046717.343926,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1711046717.3702517,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1711046717.3749778,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1711046717.40907,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1711046717.4193048,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1711046717.4283378,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1711046717.4901948,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1711046717.5492027,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1711046717.5950236,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1711046717.6130188,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1711046717.6289639,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1711046717.6750157,"name":"online","context":{"idset":"452"}}
+{"timestamp":1711046717.6957188,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1711046717.7364213,"name":"online","context":{"idset":"453"}}
+{"timestamp":1711046717.7364798,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1711046717.7520125,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1711046717.7980895,"name":"online","context":{"idset":"467"}}
+{"timestamp":1711046717.807759,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1711046717.8194022,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1711046717.8330462,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1711046717.8385656,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1711046717.8476915,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1711046717.8910038,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1711046717.900007,"name":"online","context":{"idset":"454"}}
+{"timestamp":1711046717.9000483,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1711046717.9109945,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1711046717.9560375,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1711046717.9640093,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1711046717.9720681,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1711046718.0177641,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1711046718.0300112,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1711046718.0369742,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1711046718.0462544,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1711046718.0567391,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1711046718.0575099,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1711046718.065443,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1711046718.1009569,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1711046718.1221993,"name":"online","context":{"idset":"455,458-459"}}
+{"timestamp":1711046718.1229868,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1711046718.1310387,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1711046718.1407647,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1711046718.149039,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1711046718.1590641,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1711046718.1699562,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1711046718.170723,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1711046718.1919608,"name":"online","context":{"idset":"460-461"}}
+{"timestamp":1711046718.1920037,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1711046718.202143,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1711046718.2152312,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1711046718.2259688,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1711046718.2646658,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1711046718.2728083,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1711046718.2829096,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1711046718.2986825,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1711046718.3081179,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1711046718.3157768,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1711046718.3330941,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1711046718.3338451,"name":"offline","context":{"idset":"552"}}
+{"timestamp":1711046718.3434343,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1711046718.3500602,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1711046718.3609061,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1711046718.3690116,"name":"offline","context":{"idset":"549"}}
+{"timestamp":1711046718.3769491,"name":"offline","context":{"idset":"551"}}
+{"timestamp":1711046718.3821709,"name":"offline","context":{"idset":"553"}}
+{"timestamp":1711046718.398469,"name":"offline","context":{"idset":"556"}}
+{"timestamp":1711046718.4039171,"name":"offline","context":{"idset":"558"}}
+{"timestamp":1711046718.4079704,"name":"offline","context":{"idset":"559"}}
+{"timestamp":1711046718.420855,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1711046718.4360232,"name":"online","context":{"idset":"456,462"}}
+{"timestamp":1711046718.4360681,"name":"offline","context":{"idset":"564"}}
+{"timestamp":1711046718.4520249,"name":"online","context":{"idset":"464"}}
+{"timestamp":1711046718.4592218,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1711046718.4765315,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1711046718.4854531,"name":"offline","context":{"idset":"554"}}
+{"timestamp":1711046718.4906955,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1711046718.4968355,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1711046718.4983308,"name":"offline","context":{"idset":"563"}}
+{"timestamp":1711046718.5063822,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1711046718.5150392,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1711046718.5239248,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1711046718.5246062,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1711046718.5308869,"name":"online","context":{"idset":"465"}}
+{"timestamp":1711046718.5482473,"name":"online","context":{"idset":"466,468"}}
+{"timestamp":1711046718.5598958,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1711046718.5605567,"name":"offline","context":{"idset":"550"}}
+{"timestamp":1711046718.569999,"name":"offline","context":{"idset":"555"}}
+{"timestamp":1711046718.5707238,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1711046718.5714138,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1711046718.5720966,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1711046718.5818715,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1711046718.5879474,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1711046718.5919194,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1711046718.6329482,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1711046718.6399605,"name":"offline","context":{"idset":"561"}}
+{"timestamp":1711046718.6449065,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1711046718.6455517,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1711046718.7411757,"name":"online","context":{"idset":"463,469-473,489"}}
+{"timestamp":1711046718.7412484,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1711046718.8069677,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1711046718.8490622,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1711046718.9253161,"name":"online","context":{"idset":"474-478,480-488,490,492-494"}}
+{"timestamp":1711046719.0937762,"name":"online","context":{"idset":"491,496,499"}}
+{"timestamp":1711046719.2284129,"name":"online","context":{"idset":"495,497-498,501"}}
+{"timestamp":1711046719.3062723,"name":"online","context":{"idset":"503"}}
+{"timestamp":1711046719.3119836,"name":"online","context":{"idset":"479"}}
+{"timestamp":1711046719.3120451,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1711046719.4128377,"name":"online","context":{"idset":"500,504,506-511"}}
+{"timestamp":1711046719.4129088,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1711046719.5593138,"name":"online","context":{"idset":"505,515-518,521-522,524"}}
+{"timestamp":1711046719.6730375,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1711046720.062377,"name":"online","context":{"idset":"513,520,523,525-527,529-543,545,548-549,551-552,554,557,560-562,564,568-569,572,575,578"}}
+{"timestamp":1711046720.1582861,"name":"online","context":{"idset":"528,544,546-547,550,553,555-556,558,563,565-567,571,574,576-577"}}
+{"timestamp":1711046720.3014646,"name":"online","context":{"idset":"514,579"}}
+{"timestamp":1711046720.4403532,"name":"online","context":{"idset":"502,512,519,559"}}
+{"timestamp":1711046720.8932219,"name":"online","context":{"idset":"570"}}
+{"timestamp":1711046721.127084,"name":"online","context":{"idset":"573"}}
+{"timestamp":1711046721.2578218,"name":"online","context":{"idset":"580"}}
+{"timestamp":1711046723.0253303,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1711046723.1737578,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1711046723.2543204,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1711046723.3161316,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1711046723.5273411,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1711046723.5603025,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1711046723.5898867,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1711046723.5939226,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1711046723.6356034,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1711046723.6701226,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1711046723.7057943,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1711046723.7750196,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1711046723.8266547,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1711046723.8614175,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1711046723.9013298,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1711046723.9299068,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1711046723.9468462,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1711046723.977777,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1711046723.9883912,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1711046724.0277555,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1711046724.0341246,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1711046724.1139069,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1711046724.1478169,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1711046724.2855124,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1711046724.2907541,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1711046724.3344693,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1711046724.4015882,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1711046724.4308205,"name":"online","context":{"idset":"581"}}
+{"timestamp":1711046724.4308577,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1711046724.5973396,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1711046724.6479793,"name":"online","context":{"idset":"582"}}
+{"timestamp":1711046724.6480408,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1711046724.7284901,"name":"online","context":{"idset":"583"}}
+{"timestamp":1711046724.7559412,"name":"online","context":{"idset":"584"}}
+{"timestamp":1711046724.7622516,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1711046724.7952266,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1711046724.892977,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1711046725.0379763,"name":"online","context":{"idset":"585"}}
+{"timestamp":1711046725.0380321,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1711046725.0904927,"name":"online","context":{"idset":"587-588,590"}}
+{"timestamp":1711046725.1179819,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1711046725.1471491,"name":"online","context":{"idset":"586,589"}}
+{"timestamp":1711046725.1471872,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1711046725.2247176,"name":"online","context":{"idset":"591"}}
+{"timestamp":1711046725.260103,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1711046725.3864555,"name":"online","context":{"idset":"592-593,595"}}
+{"timestamp":1711046725.4923186,"name":"online","context":{"idset":"594,598-601"}}
+{"timestamp":1711046725.4923832,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1711046725.535481,"name":"online","context":{"idset":"596-597"}}
+{"timestamp":1711046725.5449762,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1711046725.5507507,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1711046725.5750046,"name":"offline","context":{"idset":"659"}}
+{"timestamp":1711046725.5978327,"name":"offline","context":{"idset":"658"}}
+{"timestamp":1711046725.6357133,"name":"online","context":{"idset":"604"}}
+{"timestamp":1711046725.6357617,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1711046725.6424012,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1711046725.6476254,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1711046725.6526227,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1711046725.6530645,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1711046725.6664052,"name":"online","context":{"idset":"605"}}
+{"timestamp":1711046725.6664357,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1711046725.7078319,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1711046725.7204835,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1711046725.7336626,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1711046725.8093247,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1711046725.8341711,"name":"online","context":{"idset":"602"}}
+{"timestamp":1711046725.8342226,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1711046725.8409462,"name":"online","context":{"idset":"607"}}
+{"timestamp":1711046725.8409801,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1711046725.8470998,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1711046725.8532763,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1711046725.8615696,"name":"online","context":{"idset":"603"}}
+{"timestamp":1711046725.8684018,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1711046725.874474,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1711046725.8781996,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1711046725.8869283,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1711046725.8874311,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1711046725.8900676,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1711046725.8961356,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1711046725.9406719,"name":"online","context":{"idset":"610"}}
+{"timestamp":1711046725.9407134,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1711046726.0118394,"name":"online","context":{"idset":"608"}}
+{"timestamp":1711046726.0177534,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1711046726.0257573,"name":"offline","context":{"idset":"671"}}
+{"timestamp":1711046726.1505675,"name":"online","context":{"idset":"612-613"}}
+{"timestamp":1711046726.2855194,"name":"online","context":{"idset":"606,609,614"}}
+{"timestamp":1711046726.3450596,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1711046726.3507693,"name":"offline","context":{"idset":"661"}}
+{"timestamp":1711046726.3867595,"name":"offline","context":{"idset":"660"}}
+{"timestamp":1711046726.4277718,"name":"offline","context":{"idset":"675"}}
+{"timestamp":1711046726.4627337,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1711046726.475749,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1711046726.4977617,"name":"offline","context":{"idset":"669"}}
+{"timestamp":1711046726.5087237,"name":"online","context":{"idset":"620"}}
+{"timestamp":1711046726.5087519,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1711046726.5145671,"name":"offline","context":{"idset":"662"}}
+{"timestamp":1711046726.5196187,"name":"offline","context":{"idset":"655"}}
+{"timestamp":1711046726.5539443,"name":"offline","context":{"idset":"670"}}
+{"timestamp":1711046726.6053445,"name":"online","context":{"idset":"619"}}
+{"timestamp":1711046726.6102672,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1711046726.6186323,"name":"online","context":{"idset":"623"}}
+{"timestamp":1711046726.6186662,"name":"offline","context":{"idset":"677"}}
+{"timestamp":1711046726.6537411,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1711046726.6617422,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1711046726.6957574,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1711046726.7066059,"name":"offline","context":{"idset":"679"}}
+{"timestamp":1711046726.7458334,"name":"online","context":{"idset":"611"}}
+{"timestamp":1711046726.7458706,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1711046726.752701,"name":"offline","context":{"idset":"666"}}
+{"timestamp":1711046726.7568352,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1711046726.7700541,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1711046726.8114319,"name":"offline","context":{"idset":"654"}}
+{"timestamp":1711046726.8167274,"name":"offline","context":{"idset":"694"}}
+{"timestamp":1711046726.8211308,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1711046726.8291054,"name":"offline","context":{"idset":"672"}}
+{"timestamp":1711046726.8395493,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1711046726.8823407,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1711046726.8887768,"name":"offline","context":{"idset":"665"}}
+{"timestamp":1711046726.8979809,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1711046726.929039,"name":"online","context":{"idset":"618"}}
+{"timestamp":1711046726.9290762,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1711046726.9397857,"name":"offline","context":{"idset":"682"}}
+{"timestamp":1711046726.9628377,"name":"offline","context":{"idset":"683"}}
+{"timestamp":1711046726.9696867,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1711046726.9777138,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1711046726.978159,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1711046726.9786046,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1711046726.9837577,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1711046727.00072,"name":"offline","context":{"idset":"690"}}
+{"timestamp":1711046727.0307479,"name":"offline","context":{"idset":"681"}}
+{"timestamp":1711046727.0362849,"name":"online","context":{"idset":"658"}}
+{"timestamp":1711046727.0420537,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1711046727.0828204,"name":"online","context":{"idset":"621,631,659"}}
+{"timestamp":1711046727.0872035,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1711046727.1244206,"name":"online","context":{"idset":"624,627,636"}}
+{"timestamp":1711046727.1319232,"name":"online","context":{"idset":"635"}}
+{"timestamp":1711046727.1319785,"name":"offline","context":{"idset":"692"}}
+{"timestamp":1711046727.1711717,"name":"online","context":{"idset":"629,645"}}
+{"timestamp":1711046727.195483,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1711046727.2005265,"name":"offline","context":{"idset":"667"}}
+{"timestamp":1711046727.201004,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1711046727.2335246,"name":"offline","context":{"idset":"656"}}
+{"timestamp":1711046727.2397106,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1711046727.2432456,"name":"offline","context":{"idset":"676"}}
+{"timestamp":1711046727.2496195,"name":"offline","context":{"idset":"680"}}
+{"timestamp":1711046727.2739904,"name":"online","context":{"idset":"617,630"}}
+{"timestamp":1711046727.2740166,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1711046727.3245406,"name":"online","context":{"idset":"637,643"}}
+{"timestamp":1711046727.3576965,"name":"online","context":{"idset":"615,625,628,632,638,641,646,664"}}
+{"timestamp":1711046727.3888054,"name":"online","context":{"idset":"616"}}
+{"timestamp":1711046727.4164088,"name":"online","context":{"idset":"642"}}
+{"timestamp":1711046727.4336448,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1711046727.4444211,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1711046727.4597147,"name":"offline","context":{"idset":"657"}}
+{"timestamp":1711046727.4866378,"name":"offline","context":{"idset":"685"}}
+{"timestamp":1711046727.5186548,"name":"online","context":{"idset":"639-640,671"}}
+{"timestamp":1711046727.5187206,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1711046727.5569563,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1711046727.5866513,"name":"online","context":{"idset":"622"}}
+{"timestamp":1711046727.6037765,"name":"offline","context":{"idset":"701"}}
+{"timestamp":1711046727.8775582,"name":"online","context":{"idset":"697"}}
+{"timestamp":1711046727.9049656,"name":"online","context":{"idset":"660"}}
+{"timestamp":1711046728.0089152,"name":"online","context":{"idset":"647,649,661-662,669,675,703"}}
+{"timestamp":1711046728.0509176,"name":"online","context":{"idset":"655"}}
+{"timestamp":1711046728.0743284,"name":"drain","context":{"idset":"64-116,118-120,122,124-198,200-456,458-466,468-488,490-662,664-705","reason":"epilog failed for jobid fjwmzWL2SST","overwrite":0}}
+{"timestamp":1711046728.1304038,"name":"online","context":{"idset":"651,670,677"}}
+{"timestamp":1711046728.1304343,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1711046728.2805934,"name":"online","context":{"idset":"626,634,653,666,668,672,679,694,698"}}
+{"timestamp":1711046728.3992717,"name":"online","context":{"idset":"648,650,691,705"}}
+{"timestamp":1711046728.5062077,"name":"online","context":{"idset":"665,673,681-683,686,690,699,702"}}
+{"timestamp":1711046728.6290989,"name":"online","context":{"idset":"633,692,700"}}
+{"timestamp":1711046728.7713485,"name":"online","context":{"idset":"644,656,667,676,680,684,687-688,695"}}
+{"timestamp":1711046729.0111129,"name":"online","context":{"idset":"652,657,685,689,704"}}
+{"timestamp":1711046729.1118188,"name":"online","context":{"idset":"678,701"}}
+{"timestamp":1711046729.3440096,"name":"online","context":{"idset":"674"}}
+{"timestamp":1711046729.5715728,"name":"online","context":{"idset":"696"}}
+{"timestamp":1711046729.6980112,"name":"online","context":{"idset":"654"}}
+{"timestamp":1711046729.854259,"name":"online","context":{"idset":"693"}}
+{"timestamp":1711049829.838589,"name":"undrain","context":{"idset":"64"}}
+{"timestamp":1711049841.0505767,"name":"undrain","context":{"idset":"65-116,118-120,122,124-198,200-456,458-466,468-488,490-662,664-705"}}
+{"timestamp":1711050003.4969029,"name":"online","context":{"idset":"744,746,753,756"}}
+{"timestamp":1711050003.6322587,"name":"online","context":{"idset":"745,747,749-752,755"}}
+{"timestamp":1711050003.7697911,"name":"online","context":{"idset":"754"}}
+{"timestamp":1711050096.2918017,"name":"undrain","context":{"idset":"744-747,749-756"}}
+{"timestamp":1711050131.16482,"name":"online","context":{"idset":"2"}}
+{"timestamp":1711050134.5125079,"name":"online","context":{"idset":"8,52"}}
+{"timestamp":1711050134.6503382,"name":"online","context":{"idset":"10,13"}}
+{"timestamp":1711050134.7556248,"name":"online","context":{"idset":"7,49,54"}}
+{"timestamp":1711050134.861331,"name":"online","context":{"idset":"3-6,11,16,22,24"}}
+{"timestamp":1711050134.9884679,"name":"online","context":{"idset":"12,31,33,48"}}
+{"timestamp":1711050135.05878,"name":"online","context":{"idset":"15,20,40,46"}}
+{"timestamp":1711050135.1580994,"name":"online","context":{"idset":"9,19,30,32,38-39,43-44,47"}}
+{"timestamp":1711050135.2684431,"name":"online","context":{"idset":"18,29,34,37,41,45,51,53,59"}}
+{"timestamp":1711050135.3695765,"name":"online","context":{"idset":"17,21,25,35,42,55,57"}}
+{"timestamp":1711050135.496726,"name":"online","context":{"idset":"28,56"}}
+{"timestamp":1711050135.6025257,"name":"online","context":{"idset":"27,50,60"}}
+{"timestamp":1711050181.1338851,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1711050200.5378752,"name":"undrain","context":{"idset":"3-13,15-22,24-25,27-35,37-57,59-60"}}
+{"timestamp":1711053555.4964333,"name":"drain","context":{"idset":"748","reason":"nodeup fails","overwrite":1}}
+{"timestamp":1711053640.2865379,"name":"online","context":{"idset":"14,58"}}
+{"timestamp":1711053644.1649003,"name":"online","context":{"idset":"26,36"}}
+{"timestamp":1711053644.3478615,"name":"online","context":{"idset":"23"}}
+{"timestamp":1711053644.5258372,"name":"online","context":{"idset":"1"}}
+{"timestamp":1711053674.1570716,"name":"undrain","context":{"idset":"1,14,23,26,36,58"}}
+{"timestamp":1711056069.9131258,"name":"online","context":{"idset":"734"}}
+{"timestamp":1711056070.1185668,"name":"online","context":{"idset":"199"}}
+{"timestamp":1711056070.2622037,"name":"online","context":{"idset":"663"}}
+{"timestamp":1711056151.3030798,"name":"online","context":{"idset":"731"}}
+{"timestamp":1711056166.8884327,"name":"online","context":{"idset":"732"}}
+{"timestamp":1711056273.628207,"name":"undrain","context":{"idset":"199"}}
+{"timestamp":1711056284.2698681,"name":"undrain","context":{"idset":"734"}}
+{"timestamp":1711056749.1185374,"name":"drain","context":{"idset":"748","reason":"cxi: IP ping fails","overwrite":1}}
+{"timestamp":1711057563.9697526,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1711063432.9436467,"name":"online","context":{"idset":"663"}}
+{"timestamp":1711063488.3716516,"name":"undrain","context":{"idset":"663"}}
+{"timestamp":1711067404.8042161,"name":"resource-init","context":{"restart":true,"drain":{"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"}},"online":"","exclude":"0"}}
+{"timestamp":1711067404.8204334,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1711067435.5774016,"name":"online","context":{"idset":"0"}}
+{"timestamp":1711067437.69731,"name":"online","context":{"idset":"11470"}}
+{"timestamp":1711067437.7995906,"name":"online","context":{"idset":"254,663"}}
+{"timestamp":1711067437.9597876,"name":"online","context":{"idset":"711"}}
+{"timestamp":1711067438.0056853,"name":"online","context":{"idset":"11593"}}
+{"timestamp":1711067438.1003466,"name":"online","context":{"idset":"11473"}}
+{"timestamp":1711067438.1498466,"name":"online","context":{"idset":"11472"}}
+{"timestamp":1711067438.1977491,"name":"online","context":{"idset":"707,11171"}}
+{"timestamp":1711067438.2329135,"name":"online","context":{"idset":"73,85,88,91,93,103,106,108,110,113,116,120,128,131,133,136,144,146,148,156-158,160,162,165-166,173,181-182,184,190-191,197,199,202-203,206,210,214-215,217,221,223,226-227,229,232,236-239,241-243,253,255,260,262,264,266-267,269,272,274,279,286,288-289,300,304,308,322,325-326,329,334-337,339-341,350-351,355,357,360,362-364,367-368,371,378,386,389,391,394,411,414-415,430,432,434-435,438-439,441,443-444,446,449,451,458,461,465,467,470,494,498,503,511,513,529,560,562,571,583,585-586,602,610,612,624,627,631,639,641,644-646,657-658,669,673,684,696-697,735"}}
+{"timestamp":1711067438.2355604,"name":"online","context":{"idset":"65-66,153,167,205,290,347,453,459,491,635,648,681"}}
+{"timestamp":1711067438.236882,"name":"online","context":{"idset":"104,216,222,258,292,401,464,475,497,536,643,698,743"}}
+{"timestamp":1711067438.3251851,"name":"online","context":{"idset":"61-64,67-72,74-84,86-87,89-90,92,94-102,105,107,109,111-112,114-115,118-119,122,124-126,129-130,132,134-135,137-143,145,147,149-152,154-155,159,161,163-164,168,170-172,174-178,180,183,186-189,192-196,198,200-201,204,207-209,211-213,218-220,224-225,228,230-231,233-235,240,244-252,256-257,259,261,263,265,268,270-271,273,275-278,280-285,287,291,293-299,301-303,305-307,309-321,323-324,327-328,330-333,338,342-346,348-349,352-354,356,358-359,361,365-366,369-370,372-377,379-385,387-388,390,392-393,395-400,403-410,412-413,416-429,431,433,436-437,440,442,445,447-448,450,452,454-457,460,462-463,466,468-469,471-473,476-490,492-493,495-496,499-502,504-510,512,514-528,530-535,537-559,561,563-570,572-582,584,587-601,603-609,611,613-623,625-626,628-630,632,634,636-638,640,647,649-656,659-662,664-668,670-672,674-680,682-683,685-695,699-706,708,717-718,733-734,736,742,744-746,749,751-756"}}
+{"timestamp":1711067438.3805714,"name":"online","context":{"idset":"30,36,50,127,169,179,185,402,474,633,642,737,741,750,11471,11474-11475"}}
+{"timestamp":1711067438.3859916,"name":"online","context":{"idset":"21"}}
+{"timestamp":1711067438.3860338,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1711067438.4258733,"name":"online","context":{"idset":"19-20,40,49,11469,11476"}}
+{"timestamp":1711067438.4259205,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1711067438.4313705,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1711067438.4367464,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1711067438.4477265,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1711067438.506814,"name":"online","context":{"idset":"4"}}
+{"timestamp":1711067438.5068591,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1711067438.5077586,"name":"online","context":{"idset":"1"}}
+{"timestamp":1711067438.5077794,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1711067438.5086501,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1711067438.5094516,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1711067438.5132565,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1711067438.5185909,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1711067438.5236497,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1711067438.6081867,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1711067438.6087182,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1711067438.6091969,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1711067438.7430506,"name":"online","context":{"idset":"747"}}
+{"timestamp":1711067438.8724399,"name":"online","context":{"idset":"3,11-12,17-18,22,25,29,31-32,37,43,46-47,59"}}
+{"timestamp":1711067438.9148004,"name":"online","context":{"idset":"39"}}
+{"timestamp":1711067439.0814054,"name":"online","context":{"idset":"11747-11748"}}
+{"timestamp":1711067439.1115165,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1711067439.1416371,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1711067439.1473978,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1711067439.1526613,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1711067439.1576211,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1711067439.2077091,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1711067439.2086413,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1711067439.2095985,"name":"online","context":{"idset":"2"}}
+{"timestamp":1711067439.20962,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1711067439.2105818,"name":"online","context":{"idset":"23"}}
+{"timestamp":1711067439.2106011,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1711067439.211729,"name":"online","context":{"idset":"7,13,24"}}
+{"timestamp":1711067439.2117455,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1711067439.2161911,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1711067439.3279717,"name":"online","context":{"idset":"5-6,8,14-16,26-27,33-34,42,44,48,52-53,56"}}
+{"timestamp":1711067439.3280077,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1711067439.3289337,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1711067439.3298609,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1711067439.3308003,"name":"online","context":{"idset":"35,51,55,58"}}
+{"timestamp":1711067439.3317778,"name":"online","context":{"idset":"9,28,41,54,60"}}
+{"timestamp":1711067439.3317978,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1711067439.3327641,"name":"online","context":{"idset":"45"}}
+{"timestamp":1711067439.3327835,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1711067439.3337493,"name":"online","context":{"idset":"10"}}
+{"timestamp":1711067439.3337677,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1711067439.3346946,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1711067439.3357744,"name":"online","context":{"idset":"38"}}
+{"timestamp":1711067439.3366992,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1711067439.3376215,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1711067439.4726264,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1711067439.4732547,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1711067439.4737451,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1711067439.4742374,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1711067439.4748278,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1711067439.4753151,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1711067439.4758041,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1711067439.4763229,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1711067439.4768486,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1711067439.4773538,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1711067439.4778278,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1711067439.5624442,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1711067439.5631719,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1711067439.5637155,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1711067439.5644164,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1711067439.5650847,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1711067439.5656209,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1711067439.6355736,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1711067439.6361477,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1711067439.6367083,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1711067439.6371925,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1711067439.6376932,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1711067439.6381619,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1711067439.6387217,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1711067439.703476,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1711067439.7041268,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1711067439.7075856,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1711067439.7133055,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1711067439.7192309,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1711067439.7247119,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1711067439.7356575,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1711067439.7411306,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1711067439.7472763,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1711067439.7582538,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1711067439.7637563,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1711067439.8761086,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1711067439.8770373,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1711067439.8779302,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1711067439.8788207,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1711067439.8797121,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1711067439.8806009,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1711067439.8814971,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1711067439.8823919,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1711067439.8832676,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1711067439.8841491,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1711067439.985991,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1711067439.9868951,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1711067439.9877691,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1711067439.9886417,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1711067439.9895184,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1711067439.990386,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1711067439.9912493,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1711067439.9923329,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1711067439.993228,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1711067440.1058419,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1711067440.1067276,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1711067440.1076047,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1711067440.1084673,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1711067440.1093338,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1711067440.1101887,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1711067440.1110513,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1711067440.1119101,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1711067440.1127748,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1711067440.1136827,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1711067440.1153972,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1711067440.2290845,"name":"online","context":{"idset":"730"}}
+{"timestamp":1711067440.2291226,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1711067440.2298145,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1711067440.2304418,"name":"online","context":{"idset":"11468"}}
+{"timestamp":1711067440.2304604,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1711067440.2310138,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1711067440.2316287,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1711067440.2321682,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1711067440.2326665,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1711067440.2332847,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1711067440.2339151,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1711067440.2344201,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1711067440.2348864,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1711067440.2401717,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1711067440.3873255,"name":"online","context":{"idset":"11462"}}
+{"timestamp":1711067440.3873715,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1711067440.3882258,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1711067440.3890758,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1711067440.3899434,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1711067440.3908098,"name":"online","context":{"idset":"11461"}}
+{"timestamp":1711067440.3908339,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1711067440.3916643,"name":"online","context":{"idset":"11464,11467"}}
+{"timestamp":1711067440.3927021,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1711067440.3936141,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1711067440.3944635,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1711067440.3953485,"name":"online","context":{"idset":"11463"}}
+{"timestamp":1711067440.3953681,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1711067440.5565186,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1711067440.5574007,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1711067440.5583642,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1711067440.5592158,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1711067440.5600746,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1711067440.560931,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1711067440.5617611,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1711067440.5627141,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1711067440.5635469,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1711067440.5643554,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1711067440.5651853,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1711067440.5660276,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1711067440.7969735,"name":"online","context":{"idset":"11348"}}
+{"timestamp":1711067440.8970959,"name":"online","context":{"idset":"11347"}}
+{"timestamp":1711067440.8971431,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1711067440.982873,"name":"offline","context":{"idset":"11347"}}
+{"timestamp":1711067440.9894886,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1711067441.1401854,"name":"online","context":{"idset":"713,11189"}}
+{"timestamp":1711067441.1761825,"name":"online","context":{"idset":"11175"}}
+{"timestamp":1711067441.2552655,"name":"online","context":{"idset":"11172"}}
+{"timestamp":1711067441.2929883,"name":"online","context":{"idset":"11204"}}
+{"timestamp":1711067441.377759,"name":"online","context":{"idset":"11160"}}
+{"timestamp":1711067441.382875,"name":"online","context":{"idset":"11203"}}
+{"timestamp":1711067441.5581019,"name":"online","context":{"idset":"722"}}
+{"timestamp":1711067441.5962064,"name":"online","context":{"idset":"11170"}}
+{"timestamp":1711067441.6082835,"name":"online","context":{"idset":"11166"}}
+{"timestamp":1711067441.644876,"name":"online","context":{"idset":"712,714"}}
+{"timestamp":1711067441.6933384,"name":"online","context":{"idset":"11157,11167,11169"}}
+{"timestamp":1711067441.7341945,"name":"online","context":{"idset":"11162"}}
+{"timestamp":1711067441.7650745,"name":"online","context":{"idset":"11159"}}
+{"timestamp":1711067441.7957854,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1711067441.8302281,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1711067441.8559227,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1711067441.861306,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1711067441.8941162,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1711067441.8978677,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1711067441.9001851,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1711067441.9028015,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1711067441.9337258,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1711067441.9363511,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1711067441.9648392,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1711067441.9692032,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1711067441.9801028,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1711067441.9833062,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1711067442.0123382,"name":"online","context":{"idset":"11165"}}
+{"timestamp":1711067442.0123558,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1711067442.0149035,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1711067442.0211158,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1711067442.0238035,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1711067442.0294766,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1711067442.0804086,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1711067442.0825262,"name":"online","context":{"idset":"11590"}}
+{"timestamp":1711067442.0825474,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1711067442.0838361,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1711067442.1162746,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1711067442.1170056,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1711067442.1194282,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1711067442.1502075,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1711067442.1537454,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1711067442.157722,"name":"online","context":{"idset":"11612"}}
+{"timestamp":1711067442.1871848,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1711067442.1902018,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1711067442.2174568,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1711067442.2218077,"name":"online","context":{"idset":"11610"}}
+{"timestamp":1711067442.2273297,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1711067442.2532105,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1711067442.2578778,"name":"online","context":{"idset":"11589"}}
+{"timestamp":1711067442.2578948,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1711067442.2866869,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1711067442.291127,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1711067442.2952282,"name":"online","context":{"idset":"723"}}
+{"timestamp":1711067442.2985208,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1711067442.3052113,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1711067442.3095055,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1711067442.3505569,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1711067442.3516161,"name":"online","context":{"idset":"11594"}}
+{"timestamp":1711067442.3516333,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1711067442.3565829,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1711067442.3906155,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1711067442.3917558,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1711067442.3979204,"name":"online","context":{"idset":"91,140"}}
+{"timestamp":1711067442.4321947,"name":"online","context":{"idset":"88,11592"}}
+{"timestamp":1711067442.4322181,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1711067442.433989,"name":"online","context":{"idset":"11595"}}
+{"timestamp":1711067442.438674,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1711067442.4417303,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1711067442.4445434,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1711067442.4498389,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1711067442.5156734,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1711067442.5164437,"name":"online","context":{"idset":"127,162,11596"}}
+{"timestamp":1711067442.5169024,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1711067442.5173707,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1711067442.518477,"name":"online","context":{"idset":"11591"}}
+{"timestamp":1711067442.5189528,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1711067442.5194221,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1711067442.5198908,"name":"online","context":{"idset":"93,166,738"}}
+{"timestamp":1711067442.5203669,"name":"online","context":{"idset":"107,133,149-150"}}
+{"timestamp":1711067442.5203791,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1711067442.5238724,"name":"online","context":{"idset":"181"}}
+{"timestamp":1711067442.5271695,"name":"online","context":{"idset":"436"}}
+{"timestamp":1711067442.6371512,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1711067442.6381111,"name":"online","context":{"idset":"182,11501"}}
+{"timestamp":1711067442.639045,"name":"online","context":{"idset":"110"}}
+{"timestamp":1711067442.6404018,"name":"online","context":{"idset":"11489"}}
+{"timestamp":1711067442.6411519,"name":"online","context":{"idset":"11508"}}
+{"timestamp":1711067442.6417704,"name":"online","context":{"idset":"11456,11609"}}
+{"timestamp":1711067442.7291238,"name":"online","context":{"idset":"728"}}
+{"timestamp":1711067442.730526,"name":"online","context":{"idset":"11486"}}
+{"timestamp":1711067442.735826,"name":"online","context":{"idset":"11487"}}
+{"timestamp":1711067442.8059735,"name":"online","context":{"idset":"725,740,11603"}}
+{"timestamp":1711067442.8072095,"name":"online","context":{"idset":"11623,11630"}}
+{"timestamp":1711067442.8084378,"name":"online","context":{"idset":"11633"}}
+{"timestamp":1711067442.8097785,"name":"online","context":{"idset":"11459,11619"}}
+{"timestamp":1711067442.8109875,"name":"online","context":{"idset":"11451,11598"}}
+{"timestamp":1711067442.8122067,"name":"online","context":{"idset":"11611"}}
+{"timestamp":1711067442.8156068,"name":"online","context":{"idset":"11460"}}
+{"timestamp":1711067442.8322768,"name":"online","context":{"idset":"11602"}}
+{"timestamp":1711067442.937453,"name":"online","context":{"idset":"11478"}}
+{"timestamp":1711067442.9397459,"name":"online","context":{"idset":"11484,11597,11625"}}
+{"timestamp":1711067442.9414144,"name":"online","context":{"idset":"709,720,739,11455,11482,11616"}}
+{"timestamp":1711067442.9428005,"name":"online","context":{"idset":"11485,11506"}}
+{"timestamp":1711067442.9438758,"name":"online","context":{"idset":"729"}}
+{"timestamp":1711067442.9449441,"name":"online","context":{"idset":"719,727,11490"}}
+{"timestamp":1711067442.9460256,"name":"online","context":{"idset":"726,11607"}}
+{"timestamp":1711067442.9472241,"name":"online","context":{"idset":"11620"}}
+{"timestamp":1711067442.9484446,"name":"online","context":{"idset":"11480"}}
+{"timestamp":1711067442.9496872,"name":"online","context":{"idset":"11629"}}
+{"timestamp":1711067442.9509137,"name":"online","context":{"idset":"11615"}}
+{"timestamp":1711067442.9748352,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1711067442.9895849,"name":"online","context":{"idset":"716,721,731,11491,11493"}}
+{"timestamp":1711067443.1524253,"name":"online","context":{"idset":"11454,11458"}}
+{"timestamp":1711067443.1532788,"name":"online","context":{"idset":"11601,11626-11627"}}
+{"timestamp":1711067443.1533101,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1711067443.1544232,"name":"online","context":{"idset":"732,11477,11492,11496,11500,11503-11504,11507"}}
+{"timestamp":1711067443.154448,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1711067443.1552691,"name":"online","context":{"idset":"11599,11618"}}
+{"timestamp":1711067443.1560366,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1711067443.1569633,"name":"online","context":{"idset":"11600"}}
+{"timestamp":1711067443.1578486,"name":"online","context":{"idset":"11488"}}
+{"timestamp":1711067443.1585531,"name":"online","context":{"idset":"11628"}}
+{"timestamp":1711067443.1593208,"name":"online","context":{"idset":"11499,11622"}}
+{"timestamp":1711067443.1602061,"name":"online","context":{"idset":"11479"}}
+{"timestamp":1711067443.1612699,"name":"online","context":{"idset":"11635"}}
+{"timestamp":1711067443.1622016,"name":"online","context":{"idset":"11497,11505,11604,11614,11617"}}
+{"timestamp":1711067443.162884,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1711067443.1635776,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1711067443.1642056,"name":"online","context":{"idset":"11636"}}
+{"timestamp":1711067443.1649263,"name":"online","context":{"idset":"11502"}}
+{"timestamp":1711067443.1657174,"name":"online","context":{"idset":"11634"}}
+{"timestamp":1711067443.1664984,"name":"online","context":{"idset":"11632"}}
+{"timestamp":1711067443.1672735,"name":"online","context":{"idset":"11483"}}
+{"timestamp":1711067443.1763196,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1711067443.1792758,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1711067443.1872125,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1711067443.209312,"name":"online","context":{"idset":"108,131,139,146,148,11631"}}
+{"timestamp":1711067443.2093475,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1711067443.2099509,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1711067443.2105544,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1711067443.3210795,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1711067443.3990746,"name":"online","context":{"idset":"11624"}}
+{"timestamp":1711067443.4503982,"name":"online","context":{"idset":"11481,11495,11613"}}
+{"timestamp":1711067443.4510009,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1711067443.4515946,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1711067443.4521811,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1711067443.4527683,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1711067443.4534376,"name":"online","context":{"idset":"11494"}}
+{"timestamp":1711067443.4541476,"name":"online","context":{"idset":"153,156-157,160,163,184,205,724,11498"}}
+{"timestamp":1711067443.4547803,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1711067443.4555202,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1711067443.4561236,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1711067443.456784,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1711067443.457433,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1711067443.4580798,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1711067443.458662,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1711067443.4594169,"name":"online","context":{"idset":"197"}}
+{"timestamp":1711067443.4678531,"name":"online","context":{"idset":"169,173,191,209-210,216,222,710"}}
+{"timestamp":1711067443.4678848,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1711067443.4685636,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1711067443.4693453,"name":"online","context":{"idset":"715"}}
+{"timestamp":1711067443.4700012,"name":"online","context":{"idset":"212"}}
+{"timestamp":1711067443.4700189,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1711067443.4706514,"name":"online","context":{"idset":"226"}}
+{"timestamp":1711067443.4713881,"name":"online","context":{"idset":"254"}}
+{"timestamp":1711067443.472007,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1711067443.4804373,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1711067443.4810367,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1711067443.4817703,"name":"online","context":{"idset":"237"}}
+{"timestamp":1711067443.4825585,"name":"online","context":{"idset":"231"}}
+{"timestamp":1711067443.4833503,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1711067443.4839418,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1711067443.4847639,"name":"online","context":{"idset":"206"}}
+{"timestamp":1711067443.4847977,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1711067443.4854646,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1711067443.4861414,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1711067443.4868562,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1711067443.4875729,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1711067443.7041247,"name":"online","context":{"idset":"217,230"}}
+{"timestamp":1711067443.7050879,"name":"online","context":{"idset":"165,211"}}
+{"timestamp":1711067443.7055819,"name":"online","context":{"idset":"202,214"}}
+{"timestamp":1711067443.7055969,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1711067443.7060623,"name":"online","context":{"idset":"168,215"}}
+{"timestamp":1711067443.7065437,"name":"online","context":{"idset":"246"}}
+{"timestamp":1711067443.7069948,"name":"online","context":{"idset":"11608"}}
+{"timestamp":1711067443.7070072,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1711067443.7074656,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1711067443.7079318,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1711067443.7083941,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1711067443.7088408,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1711067443.7093518,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1711067443.7098136,"name":"online","context":{"idset":"167"}}
+{"timestamp":1711067443.7098222,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1711067443.7102656,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1711067443.710881,"name":"online","context":{"idset":"243"}}
+{"timestamp":1711067443.7108915,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1711067443.7113559,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1711067443.711803,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1711067443.7123864,"name":"online","context":{"idset":"224"}}
+{"timestamp":1711067443.7128422,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1711067443.7133403,"name":"online","context":{"idset":"271-272"}}
+{"timestamp":1711067443.7138202,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1711067443.7143106,"name":"online","context":{"idset":"235"}}
+{"timestamp":1711067443.7147624,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1711067443.715734,"name":"online","context":{"idset":"218,223,236,274"}}
+{"timestamp":1711067443.7190642,"name":"online","context":{"idset":"203,238,253,261-262"}}
+{"timestamp":1711067443.7243242,"name":"online","context":{"idset":"240,286"}}
+{"timestamp":1711067443.7553062,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1711067443.9305532,"name":"online","context":{"idset":"228-229,251,267,289"}}
+{"timestamp":1711067443.9313452,"name":"online","context":{"idset":"264"}}
+{"timestamp":1711067443.9377728,"name":"online","context":{"idset":"227,252,263,280,287,304,312,318,337"}}
+{"timestamp":1711067443.9779022,"name":"online","context":{"idset":"234"}}
+{"timestamp":1711067444.0624332,"name":"online","context":{"idset":"57,221,232-233,239,242,282,292,321,338,11606"}}
+{"timestamp":1711067444.1634939,"name":"online","context":{"idset":"46,190,255,257,266,279,284,297,329,331,341,367,11605"}}
+{"timestamp":1711067444.2490966,"name":"online","context":{"idset":"241,258,275,298,340,378"}}
+{"timestamp":1711067444.3238041,"name":"online","context":{"idset":"260,327,339,347,350,362,368,11171"}}
+{"timestamp":1711067444.4568179,"name":"online","context":{"idset":"325-326,335-336,345,356,389,411,11199-11200"}}
+{"timestamp":1711067444.5296714,"name":"online","context":{"idset":"324,333,346,357,364,415,642"}}
+{"timestamp":1711067444.570893,"name":"online","context":{"idset":"402"}}
+{"timestamp":1711067444.6490307,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1711067444.6563301,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1711067444.7130263,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1711067444.714932,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1711067444.7168233,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1711067444.718714,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1711067444.7221999,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1711067444.7279778,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1711067444.8179426,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1711067444.8189573,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1711067444.8198075,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1711067444.820612,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1711067444.8214417,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1711067444.8885326,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1711067444.8893607,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1711067444.8975203,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1711067444.9644375,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1711067444.9652202,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1711067444.9660554,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1711067444.9688654,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1711067444.9748135,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1711067444.9800205,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1711067445.0504172,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1711067445.0511951,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1711067445.0520239,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1711067445.0529211,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1711067445.0537,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1711067445.1214767,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1711067445.1224809,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1711067445.1232717,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1711067445.1725097,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1711067445.1751273,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1711067445.1821084,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1711067445.187223,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1711067445.2376306,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1711067445.2387617,"name":"online","context":{"idset":"11348"}}
+{"timestamp":1711067445.2397208,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1711067445.2450094,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1711067445.2500889,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1711067445.3148816,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1711067445.3156772,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1711067445.3164859,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1711067445.3173487,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1711067445.4871533,"name":"online","context":{"idset":"11347"}}
+{"timestamp":1711067445.5326087,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1711067445.8122599,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1711067445.9197917,"name":"online","context":{"idset":"244,290,300,322,351,366,387,394,414"}}
+{"timestamp":1711067446.0371964,"name":"online","context":{"idset":"413,416,432,434,441,446,458,475"}}
+{"timestamp":1711067446.1041837,"name":"online","context":{"idset":"439,443-444,449,461,467,494"}}
+{"timestamp":1711067446.2517619,"name":"online","context":{"idset":"465,485,498,501,560,586,624"}}
+{"timestamp":1711067446.3228457,"name":"online","context":{"idset":"511,513,567,11470"}}
+{"timestamp":1711067446.4201789,"name":"online","context":{"idset":"412,562,571,583-584,597,611,627,639,644,650"}}
+{"timestamp":1711067446.420222,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1711067446.5225272,"name":"online","context":{"idset":"363,386,610,614,635,643,652"}}
+{"timestamp":1711067446.5949128,"name":"online","context":{"idset":"382,646,648"}}
+{"timestamp":1711067446.6088111,"name":"offline","context":{"idset":"11747"}}
+{"timestamp":1711067446.7926779,"name":"offline","context":{"idset":"11748"}}
+{"timestamp":1711067446.8925703,"name":"online","context":{"idset":"11473,11476"}}
+{"timestamp":1711067447.0085766,"name":"online","context":{"idset":"11469,11471-11472"}}
+{"timestamp":1711067447.1596291,"name":"online","context":{"idset":"11474-11475,11621"}}
+{"timestamp":1711067447.8849592,"name":"online","context":{"idset":"11205"}}
+{"timestamp":1711067447.9859874,"name":"online","context":{"idset":"11149,11153-11154,11351"}}
+{"timestamp":1711067448.0981524,"name":"online","context":{"idset":"11150,11181-11182,11185,11196,11206,11209-11210,11336,11352,11593"}}
+{"timestamp":1711067448.1367612,"name":"online","context":{"idset":"11156"}}
+{"timestamp":1711067448.1438236,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1711067448.1501002,"name":"online","context":{"idset":"11350"}}
+{"timestamp":1711067448.1501405,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1711067448.2113347,"name":"online","context":{"idset":"11152,11194,11354"}}
+{"timestamp":1711067448.2113659,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1711067448.2122216,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1711067448.2130547,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1711067448.2145305,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1711067448.2218509,"name":"online","context":{"idset":"11207"}}
+{"timestamp":1711067448.2218695,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1711067448.2278774,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1711067448.2368138,"name":"online","context":{"idset":"11174"}}
+{"timestamp":1711067448.2368448,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1711067448.3404713,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1711067448.3414869,"name":"online","context":{"idset":"11151"}}
+{"timestamp":1711067448.3415115,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1711067448.3424878,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1711067448.3433094,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1711067448.3441389,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1711067448.3449712,"name":"online","context":{"idset":"11186,11208"}}
+{"timestamp":1711067448.3449895,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1711067448.3481324,"name":"online","context":{"idset":"11462"}}
+{"timestamp":1711067448.454241,"name":"online","context":{"idset":"11467"}}
+{"timestamp":1711067448.454272,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1711067448.4551184,"name":"online","context":{"idset":"11461"}}
+{"timestamp":1711067448.4560773,"name":"online","context":{"idset":"11155,11190"}}
+{"timestamp":1711067448.4569108,"name":"online","context":{"idset":"11178"}}
+{"timestamp":1711067448.4578714,"name":"online","context":{"idset":"11183,11193"}}
+{"timestamp":1711067448.5278921,"name":"online","context":{"idset":"11173,11176-11177,11180,11353,11468"}}
+{"timestamp":1711067448.527925,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1711067448.6250174,"name":"online","context":{"idset":"11195,11216,11333,11337,11464"}}
+{"timestamp":1711067448.7368505,"name":"online","context":{"idset":"11218,11334"}}
+{"timestamp":1711067448.8758516,"name":"online","context":{"idset":"11217,11219-11220,11335,11463"}}
+{"timestamp":1711067448.9772742,"name":"online","context":{"idset":"11215,11410"}}
+{"timestamp":1711067449.1082032,"name":"online","context":{"idset":"11313,11390,11432"}}
+{"timestamp":1711067449.2453723,"name":"online","context":{"idset":"11378"}}
+{"timestamp":1711067449.4864583,"name":"online","context":{"idset":"11342"}}
+{"timestamp":1711067449.5873187,"name":"online","context":{"idset":"11242,11285,11395-11396"}}
+{"timestamp":1711067449.7188001,"name":"online","context":{"idset":"11305,11324,11327,11361,11392,11430,11445"}}
+{"timestamp":1711067449.8512685,"name":"online","context":{"idset":"11289,11391,11403,11427,11434"}}
+{"timestamp":1711067449.9599643,"name":"online","context":{"idset":"11139,11389"}}
+{"timestamp":1711067450.1292851,"name":"online","context":{"idset":"11129,11131,11240,11298,11322,11345,11426,11437"}}
+{"timestamp":1711067450.2346973,"name":"online","context":{"idset":"11296,11302,11386,11436"}}
+{"timestamp":1711067450.3403702,"name":"online","context":{"idset":"11138,11244,11246,11314,11331,11405,11408,11413,11423,11435"}}
+{"timestamp":1711067450.4784248,"name":"online","context":{"idset":"11126-11127,11250,11306,11339,11344,11415,11429,11448"}}
+{"timestamp":1711067450.5851381,"name":"online","context":{"idset":"11136-11137,11140,11243,11304,11316-11318,11343,11373,11443"}}
+{"timestamp":1711067450.6864042,"name":"online","context":{"idset":"11134,11249,11291-11292,11307,11312,11357,11360,11406-11407,11409,11412,11428,11446"}}
+{"timestamp":1711067450.7899289,"name":"online","context":{"idset":"11125,11128,11130,11132-11133,11135,11226,11237-11238,11241,11252,11286-11288,11290,11295,11300-11301,11303,11309-11311,11315,11319-11320,11323,11328-11329,11332,11340,11346,11377,11379,11381,11404,11411,11424-11425,11431,11433,11439,11447,11450-11452,11480"}}
+{"timestamp":1711067450.897362,"name":"online","context":{"idset":"11239,11247,11251,11293-11294,11297,11299,11321,11325-11326,11330,11358,11362-11363,11368,11372,11382-11383,11388,11397-11401,11416-11418,11420-11422,11444,11456,11460,11748"}}
+{"timestamp":1711067451.0105848,"name":"online","context":{"idset":"11228,11245,11248,11308,11359,11364,11375-11376,11385,11387,11402,11414,11419,11438,11440-11442,11449,11747"}}
+{"timestamp":1711067451.1118169,"name":"online","context":{"idset":"11236,11338,11365,11380,11384,11459"}}
+{"timestamp":1711067451.221838,"name":"online","context":{"idset":"11234,11366,11371,11394,11454-11455,11457-11458,11495"}}
+{"timestamp":1711067451.3229992,"name":"online","context":{"idset":"11224-11225,11227,11367,11374,11453,11477,11482,11485,11612"}}
+{"timestamp":1711067451.4249365,"name":"online","context":{"idset":"11223,11231,11369,11393,11478-11479,11483-11484,11489,11494,11496,11611"}}
+{"timestamp":1711067451.5556438,"name":"online","context":{"idset":"11221-11222,11481,11486-11488,11490-11493,11500-11501,11504,11508,11609"}}
+{"timestamp":1711067451.6566896,"name":"online","context":{"idset":"11141,11229,11232-11233,11235,11497,11499,11506"}}
+{"timestamp":1711067451.8047159,"name":"online","context":{"idset":"11498,11502,11505,11507,11607"}}
+{"timestamp":1711067451.9782326,"name":"online","context":{"idset":"11503,11592"}}
+{"timestamp":1711067452.0936849,"name":"online","context":{"idset":"11594-11595,11627"}}
+{"timestamp":1711067452.2340357,"name":"online","context":{"idset":"365,459,463,593,602,612,647,664,11589-11590,11596,11616,11626"}}
+{"timestamp":1711067452.335119,"name":"online","context":{"idset":"596,641,645,673,684,11591,11615,11625"}}
+{"timestamp":1711067452.4425573,"name":"online","context":{"idset":"696-697,11144,11598,11613-11614,11623"}}
+{"timestamp":1711067452.5778852,"name":"online","context":{"idset":"11597,11617,11620,11622,11628"}}
+{"timestamp":1711067452.731643,"name":"online","context":{"idset":"11143,11600,11602-11603,11619"}}
+{"timestamp":1711067452.9088097,"name":"online","context":{"idset":"11147-11148,11599,11601,11604,11624"}}
+{"timestamp":1711067453.0441074,"name":"online","context":{"idset":"11142,11145-11146,11630,11632-11633,11635-11636"}}
+{"timestamp":1711067453.2712302,"name":"online","context":{"idset":"11629,11631,11634"}}
+{"timestamp":1711067453.614944,"name":"online","context":{"idset":"11608,11610"}}
+{"timestamp":1711067453.751858,"name":"online","context":{"idset":"11606,11618"}}
+{"timestamp":1711067454.1923807,"name":"online","context":{"idset":"11605"}}
+{"timestamp":1711067599.7711329,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1711096934.358676,"name":"drain","context":{"idset":"431","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711117081.6739657,"name":"drain","context":{"idset":"11337","reason":"broker was unresponsive"}}
+{"timestamp":1711117081.7735705,"name":"drain","context":{"idset":"11338","reason":"broker was unresponsive"}}
+{"timestamp":1711117143.7718754,"name":"offline","context":{"idset":"11337"}}
+{"timestamp":1711117145.7713585,"name":"offline","context":{"idset":"11338"}}
+{"timestamp":1711117417.6734309,"name":"drain","context":{"idset":"11390","reason":"broker was unresponsive"}}
+{"timestamp":1711117417.6736095,"name":"drain","context":{"idset":"11393","reason":"broker was unresponsive"}}
+{"timestamp":1711117417.7733912,"name":"drain","context":{"idset":"11394","reason":"broker was unresponsive"}}
+{"timestamp":1711117429.7713454,"name":"drain","context":{"idset":"11339","reason":"broker was unresponsive"}}
+{"timestamp":1711117435.673979,"name":"drain","context":{"idset":"11340","reason":"broker was unresponsive"}}
+{"timestamp":1711117435.6741276,"name":"drain","context":{"idset":"11342","reason":"broker was unresponsive"}}
+{"timestamp":1711117435.6742618,"name":"drain","context":{"idset":"11343","reason":"broker was unresponsive"}}
+{"timestamp":1711117435.6744049,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1711117435.7733719,"name":"drain","context":{"idset":"11345","reason":"broker was unresponsive"}}
+{"timestamp":1711117481.6784155,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1711117481.6866014,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1711117481.7723415,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1711117495.6766078,"name":"offline","context":{"idset":"11339"}}
+{"timestamp":1711117495.6813545,"name":"drain","context":{"idset":"11606","reason":"broker was unresponsive"}}
+{"timestamp":1711117495.6814806,"name":"drain","context":{"idset":"11608","reason":"broker was unresponsive"}}
+{"timestamp":1711117495.7736769,"name":"drain","context":{"idset":"11614","reason":"broker was unresponsive"}}
+{"timestamp":1711117497.6801593,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1711117497.6868465,"name":"offline","context":{"idset":"11342"}}
+{"timestamp":1711117497.6978559,"name":"offline","context":{"idset":"11343"}}
+{"timestamp":1711117497.704747,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1711117497.7720282,"name":"offline","context":{"idset":"11345"}}
+{"timestamp":1711117557.6760695,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1711117557.7721107,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1711117559.772023,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1711121983.7724895,"name":"drain","context":{"idset":"11477","reason":"broker was unresponsive"}}
+{"timestamp":1711122049.7719808,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1711123043.7731605,"name":"drain","context":{"idset":"11478","reason":"broker was unresponsive"}}
+{"timestamp":1711123105.7733288,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1711123241.6735494,"name":"drain","context":{"idset":"11445","reason":"broker was unresponsive"}}
+{"timestamp":1711123241.6736929,"name":"drain","context":{"idset":"11446","reason":"broker was unresponsive"}}
+{"timestamp":1711123241.6738214,"name":"drain","context":{"idset":"11447","reason":"broker was unresponsive"}}
+{"timestamp":1711123241.6739299,"name":"drain","context":{"idset":"11448","reason":"broker was unresponsive"}}
+{"timestamp":1711123241.674036,"name":"drain","context":{"idset":"11449","reason":"broker was unresponsive"}}
+{"timestamp":1711123241.6741452,"name":"drain","context":{"idset":"11450","reason":"broker was unresponsive"}}
+{"timestamp":1711123241.7729518,"name":"drain","context":{"idset":"11451","reason":"broker was unresponsive"}}
+{"timestamp":1711123247.6733992,"name":"drain","context":{"idset":"11452","reason":"broker was unresponsive"}}
+{"timestamp":1711123247.6735187,"name":"drain","context":{"idset":"11453","reason":"broker was unresponsive"}}
+{"timestamp":1711123247.6736262,"name":"drain","context":{"idset":"11454","reason":"broker was unresponsive"}}
+{"timestamp":1711123247.6737466,"name":"drain","context":{"idset":"11455","reason":"broker was unresponsive"}}
+{"timestamp":1711123247.6738572,"name":"drain","context":{"idset":"11456","reason":"broker was unresponsive"}}
+{"timestamp":1711123247.6739681,"name":"drain","context":{"idset":"11457","reason":"broker was unresponsive"}}
+{"timestamp":1711123247.6740794,"name":"drain","context":{"idset":"11458","reason":"broker was unresponsive"}}
+{"timestamp":1711123247.6741953,"name":"drain","context":{"idset":"11459","reason":"broker was unresponsive"}}
+{"timestamp":1711123247.7819526,"name":"drain","context":{"idset":"11460","reason":"broker was unresponsive"}}
+{"timestamp":1711123305.6790771,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1711123305.6890707,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1711123305.7718179,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1711123307.6768332,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1711123307.684474,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1711123307.6905968,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1711123307.7713563,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1711123309.6760402,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1711123309.6833909,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1711123309.6966143,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1711123309.7709877,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1711123311.6788142,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1711123311.6847544,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1711123311.6905544,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1711123311.6964526,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1711123311.7725337,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1711123499.7731371,"name":"drain","context":{"idset":"11332","reason":"broker was unresponsive"}}
+{"timestamp":1711123565.7720993,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1711126273.9625854,"name":"drain","context":{"idset":"11125","reason":"nodediag failed bogomips cpucount dmi network pci","overwrite":0}}
+{"timestamp":1711126278.6191533,"name":"drain","context":{"idset":"11126","reason":"nodediag failed bogomips cpucount dmi network pci","overwrite":0}}
+{"timestamp":1711126278.7933528,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1711126283.1690614,"name":"online","context":{"idset":"11126"}}
+{"timestamp":1711126290.556175,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1711126295.1317222,"name":"online","context":{"idset":"11125"}}
+{"timestamp":1711126696.680208,"name":"drain","context":{"idset":"11127","reason":"nodediag failed bogomips cpucount dmi network pci","overwrite":0}}
+{"timestamp":1711129677.3918278,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1711129681.3539968,"name":"online","context":{"idset":"11125"}}
+{"timestamp":1711129913.5158849,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1711129913.6152568,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1711131158.7399614,"name":"online","context":{"idset":"11614"}}
+{"timestamp":1711131158.849622,"name":"online","context":{"idset":"11608"}}
+{"timestamp":1711131159.064873,"name":"online","context":{"idset":"11606"}}
+{"timestamp":1711131380.4368384,"name":"online","context":{"idset":"11339"}}
+{"timestamp":1711131380.7269692,"name":"online","context":{"idset":"11338,11340,11344"}}
+{"timestamp":1711131381.0113778,"name":"online","context":{"idset":"11337"}}
+{"timestamp":1711131381.1565099,"name":"online","context":{"idset":"11343"}}
+{"timestamp":1711131381.27613,"name":"online","context":{"idset":"11341-11342,11345"}}
+{"timestamp":1711131418.2051375,"name":"online","context":{"idset":"11393"}}
+{"timestamp":1711131418.749521,"name":"online","context":{"idset":"11390,11394"}}
+{"timestamp":1711132480.4518912,"name":"online","context":{"idset":"11125"}}
+{"timestamp":1711137477.2022595,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1711140105.6735849,"name":"drain","context":{"idset":"11501","reason":"broker was unresponsive"}}
+{"timestamp":1711140105.7732108,"name":"drain","context":{"idset":"11502","reason":"broker was unresponsive"}}
+{"timestamp":1711140171.6759124,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1711140171.7720423,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1711140321.4661839,"name":"online","context":{"idset":"11341"}}
+{"timestamp":1711140955.7734199,"name":"drain","context":{"idset":"11365","reason":"broker was unresponsive"}}
+{"timestamp":1711141019.7718654,"name":"offline","context":{"idset":"11365"}}
+{"timestamp":1711141273.7721725,"name":"drain","context":{"idset":"11165","reason":"broker was unresponsive"}}
+{"timestamp":1711141389.7715259,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1711142975.7731442,"name":"drain","context":{"idset":"11341","reason":"broker was unresponsive"}}
+{"timestamp":1711143039.7722123,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1711143325.7809885,"name":"drain","context":{"idset":"11331","reason":"broker was unresponsive"}}
+{"timestamp":1711143391.7722256,"name":"offline","context":{"idset":"11331"}}
+{"timestamp":1711143627.7719393,"name":"offline","context":{"idset":"11342"}}
+{"timestamp":1711146121.4358735,"name":"online","context":{"idset":"11341"}}
+{"timestamp":1711146900.3106725,"name":"online","context":{"idset":"11365"}}
+{"timestamp":1711147021.7717752,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1711148027.6723082,"name":"drain","context":{"idset":"11295","reason":"broker was unresponsive"}}
+{"timestamp":1711148027.7718606,"name":"drain","context":{"idset":"11296","reason":"broker was unresponsive"}}
+{"timestamp":1711148089.6754894,"name":"offline","context":{"idset":"11295"}}
+{"timestamp":1711148089.7706387,"name":"offline","context":{"idset":"11296"}}
+{"timestamp":1711149300.2261975,"name":"online","context":{"idset":"11341"}}
+{"timestamp":1711155708.5741615,"name":"online","context":{"idset":"123"}}
+{"timestamp":1711155709.3956616,"name":"online","context":{"idset":"117"}}
+{"timestamp":1711157017.064605,"name":"drain","context":{"idset":"92,151,170,176,179,202,213,220,241,311,343,399,436,472,550,596,617,674,699","reason":"Can't unmount Lustre","overwrite":0}}
+{"timestamp":1711157026.2696362,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1711157241.6862283,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1711157241.6947916,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1711157241.7029505,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1711157241.711108,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1711157241.7237465,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1711157241.7314069,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1711157241.7397223,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1711157241.7480044,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1711157241.768219,"name":"offline","context":{"idset":"550"}}
+{"timestamp":1711157241.7693892,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1711157241.7769868,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1711157242.3321545,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1711157242.3457415,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1711157242.3545823,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1711157242.3622344,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1711157242.3703074,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1711157242.3781095,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1711157242.4601262,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1711157261.7716401,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1711162481.7731988,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1711162547.7722442,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1711163915.5903628,"name":"undrain","context":{"idset":"92,151,170,176,179,202,213,220,241,311,343,399,436,472,550,596,617,674,699"}}
+{"timestamp":1711164142.0575831,"name":"online","context":{"idset":"123"}}
+{"timestamp":1711164142.3335404,"name":"online","context":{"idset":"117"}}
+{"timestamp":1711164142.6064122,"name":"online","context":{"idset":"151,179"}}
+{"timestamp":1711164142.855741,"name":"online","context":{"idset":"92,170,176"}}
+{"timestamp":1711164148.7119889,"name":"online","context":{"idset":"202"}}
+{"timestamp":1711164149.0899169,"name":"online","context":{"idset":"213"}}
+{"timestamp":1711164149.2066925,"name":"online","context":{"idset":"220,241"}}
+{"timestamp":1711164149.6518292,"name":"online","context":{"idset":"311"}}
+{"timestamp":1711164155.2789586,"name":"online","context":{"idset":"343"}}
+{"timestamp":1711164156.1410689,"name":"online","context":{"idset":"399"}}
+{"timestamp":1711164156.4284508,"name":"online","context":{"idset":"436"}}
+{"timestamp":1711164162.1215403,"name":"online","context":{"idset":"472"}}
+{"timestamp":1711164164.4700563,"name":"online","context":{"idset":"550"}}
+{"timestamp":1711164168.6210895,"name":"online","context":{"idset":"596"}}
+{"timestamp":1711164169.6842148,"name":"online","context":{"idset":"617"}}
+{"timestamp":1711164171.0499465,"name":"online","context":{"idset":"674"}}
+{"timestamp":1711164171.6652811,"name":"online","context":{"idset":"699"}}
+{"timestamp":1711164505.2318995,"name":"undrain","context":{"idset":"123"}}
+{"timestamp":1711165087.7736251,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1711165149.7714005,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1711168631.8140626,"name":"drain","context":{"idset":"372","reason":"Lustre timeouts","overwrite":0}}
+{"timestamp":1711215610.0095849,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1711216213.7730908,"name":"drain","context":{"idset":"755","reason":"broker was unresponsive"}}
+{"timestamp":1711216279.7713799,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1711216745.6805143,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6806085,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.680656,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6807451,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6808176,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6808617,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6809049,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6809447,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6809835,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6810231,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6811044,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6811471,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6811881,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6812286,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6812708,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6813264,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.681371,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6814148,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.681462,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6833789,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6835806,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6837616,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6838651,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6840565,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6841562,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6842697,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6843822,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6844778,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6845732,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6846764,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6847744,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6848724,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6850114,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6851158,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6852152,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6853395,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6854444,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6855464,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6856468,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.68575,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6858568,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6859612,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6861026,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.686213,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6863275,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6864519,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6865566,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6866629,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6867788,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6868875,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6869948,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6871991,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6874077,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6875894,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6877069,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6878214,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6879315,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6880448,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1711216745.6881592,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1711216746.2583418,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1711216806.1884465,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1711216806.3666975,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1711216806.5744188,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1711216806.5825315,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1711216806.6688526,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1711216806.7289634,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1711216806.7702527,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1711216806.774822,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1711216806.8363798,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1711216806.8477442,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1711216806.8562396,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1711216806.8634467,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1711216806.9144967,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1711216806.9155986,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1711216806.9195721,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1711216806.9802368,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1711216806.9895768,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1711216806.9995017,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1711216807.0514207,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1711216807.0843582,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1711216807.0983541,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1711216807.1419437,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1711216807.1866739,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1711216807.2012727,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1711216807.2090397,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1711216807.2535133,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1711216807.2571628,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1711216807.2956276,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1711216807.305357,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1711216807.313566,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1711216807.3239596,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1711216807.3774624,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1711216807.3785565,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1711216807.3836493,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1711216807.426796,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1711216807.4337564,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1711216807.4427488,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1711216807.4558921,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1711216807.4640939,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1711216807.4651945,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1711216807.4865177,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1711216807.4876251,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1711216807.495405,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1711216807.5267253,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1711216807.6013098,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1711216807.603143,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1711216807.6043737,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1711216807.6055648,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1711216807.6078808,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1711216807.6177576,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1711216807.7048976,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1711216807.706398,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1711216807.7078562,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1711216807.7090955,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1711216807.7105124,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1711216807.7184258,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1711216807.7271476,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1711216807.741271,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1711216807.7493558,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1711216807.7572842,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1711216807.7763438,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1711216807.7775464,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1711216807.7812526,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1711216807.7900157,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1711216807.8090224,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1711216807.8625758,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1711216807.8637397,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1711216807.8649056,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1711216807.8660417,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1711216807.8866377,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1711216807.8978672,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1711216807.8990164,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1711216807.9001746,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1711216807.9013889,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1711216807.902642,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1711216807.9037745,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1711216807.9567959,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1711216807.9681954,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1711216808.0889931,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1711216808.0900691,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1711216808.0911262,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1711216808.0921729,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1711216808.093226,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1711216808.0942831,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1711216808.0953574,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1711216808.0964973,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1711216808.1030695,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1711216808.1174147,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1711216808.127429,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1711216808.1354642,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1711216808.1544318,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1711216808.2771227,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1711216808.2782166,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1711216808.2793562,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1711216808.2803988,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1711216808.2814338,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1711216808.2825446,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1711216808.2836161,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1711216808.2846694,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1711216808.2857065,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1711216808.2867355,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1711216808.2877898,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1711216808.2937231,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1711216808.3045011,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1711216808.3934381,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1711216808.4014182,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1711216808.408628,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1711216808.4153719,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1711216808.4240084,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1711216808.440196,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1711216808.4419823,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1711216808.4438052,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1711216808.4456439,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1711216808.4530139,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1711216808.4585509,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1711216808.4685278,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1711216808.4756465,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1711216808.4839475,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1711216808.4850128,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1711216808.5136995,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1711216808.5421119,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1711216808.5726469,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1711216808.6009607,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1711216808.6020155,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1711216808.6030507,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1711216808.6040828,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1711216808.6141269,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1711216808.6419909,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1711216808.6698422,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1711216808.6888223,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1711216808.6898527,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1711216808.6908829,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1711216808.7098603,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1711216808.7378483,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1711216808.7478428,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1711216808.7488704,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1711216808.7724371,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1711216808.7734749,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1711216808.7744844,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1711216808.7756944,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1711216808.7855737,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1711216808.7865906,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1711216808.7876132,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1711216808.7888153,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1711216808.7898195,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1711216808.7908204,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1711216808.8154185,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1711216808.8470109,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1711216808.8481131,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1711216808.8493326,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1711216808.8653064,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1711216808.8800597,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1711216808.8814738,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1711216808.8825519,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1711216808.8837032,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1711216808.8938959,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1711216808.9128032,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1711216808.9137893,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1711216808.9148037,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1711216808.9157791,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1711216808.9167442,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1711216808.9270427,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1711216808.9370356,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1711216808.9380398,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1711216808.9390817,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1711216808.9400527,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1711216808.9410203,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1711216808.9420228,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1711216808.9430161,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1711216808.9439702,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1711216808.9449155,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1711216808.964077,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1711216808.9650693,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1711216808.9660945,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1711216808.9670799,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1711216808.9680884,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1711216808.9690685,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1711216808.9700787,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1711216808.9710429,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1711216808.9719977,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1711216808.9729517,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1711216808.9923832,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1711216809.0113201,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1711216809.0305388,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1711216809.0498707,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1711216809.059958,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1711216809.0609598,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1711216809.0619071,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1711216809.0628469,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1711216809.0638041,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1711216809.0648034,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1711216809.0748422,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1711216809.0848789,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1711216809.0858417,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1711216809.0868371,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1711216809.0883331,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1711216809.0898395,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1711216809.0914679,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1711216809.0929654,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1711216809.0944607,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1711216809.0959671,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1711216809.0974603,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1711216809.0989237,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1711216809.1006124,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1711216809.1018205,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1711216809.1027491,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1711216809.1036811,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1711216809.1046498,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1711216809.1245105,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1711216809.1437607,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1711216809.1627467,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1711216809.1817963,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1711216809.1827226,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1711216809.1838486,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1711216809.1847794,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1711216809.185698,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1711216809.1866169,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1711216809.187562,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1711216809.1884868,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1711216809.189404,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1711216809.1903198,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1711216809.1912208,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1711216809.1921508,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1711216809.196424,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1711216809.1975358,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1711216809.2026746,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1711216809.2035995,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1711216809.2125587,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1711216809.2134688,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1711216809.222811,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1711216809.2237308,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1711216809.2297883,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1711216809.2307415,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1711216809.2316554,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1711216809.2540345,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1711216809.273741,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1711216809.2927992,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1711216809.3138912,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1711216809.3341768,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1711216809.3564355,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1711216809.3812606,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1711216809.4057343,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1711216809.4274299,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1711216809.4284637,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1711216809.4297264,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1711216809.4307282,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1711216809.4318802,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1711216809.4329507,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1711216809.4339759,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1711216809.4350867,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1711216809.4361768,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1711216809.4372296,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1711216809.4382441,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1711216809.4395158,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1711216809.4406555,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1711216809.4418936,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1711216809.4430447,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1711216809.4441574,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1711216809.4452684,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1711216809.4462721,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1711216809.4474373,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1711216809.4484639,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1711216809.4495151,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1711216809.450815,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1711216809.4518394,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1711216809.4529147,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1711216809.4539208,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1711216809.4549115,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1711216809.4559581,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1711216809.4676447,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1711216809.4970801,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1711216809.5176675,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1711216809.5385087,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1711216809.5594192,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1711216809.5800433,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1711216809.6005101,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1711216809.6208625,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1711216809.6418939,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1711216809.6625357,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1711216809.6849592,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1711216809.7056828,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1711216809.7265596,"name":"offline","context":{"idset":"454"}}
+{"timestamp":1711216809.7275319,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1711216809.7285123,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1711216809.7294171,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1711216809.7304854,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1711216809.7314236,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1711216809.7323225,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1711216809.7332532,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1711216809.7341614,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1711216809.7351019,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1711216809.7360964,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1711216809.7370601,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1711216809.7379951,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1711216809.7389131,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1711216809.7398407,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1711216809.7407804,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1711216809.761415,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1711216809.7818718,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1711216809.8022981,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1711216809.8230853,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1711216809.8441796,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1711216809.8648043,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1711216809.8856127,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1711216809.906152,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1711216810.1336753,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1711216810.134547,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1711216810.1354089,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1711216810.1362543,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1711216810.1370904,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1711216810.1379173,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1711216810.1387413,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1711216810.1395605,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1711216810.1404016,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1711216810.1412134,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1711216810.1420386,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1711216810.1519494,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1711216810.1709824,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1711216810.1902583,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1711216810.2092805,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1711216810.2297299,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1711216810.2488186,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1711216810.2729473,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1711216810.2932429,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1711216810.3139575,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1711216810.334357,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1711216810.3566647,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1711216810.3826115,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1711216810.4030361,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1711216810.4041286,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1711216810.4050174,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1711216810.4059315,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1711216810.4068363,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1711216810.4076996,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1711216810.4085588,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1711216810.4094236,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1711216810.4102497,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1711216810.4110582,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1711216810.4118497,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1711216810.4126542,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1711216810.4135022,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1711216810.4143498,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1711216810.4152727,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1711216810.4162505,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1711216810.4540172,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1711216810.4762785,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1711216810.495548,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1711216810.5143273,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1711216810.533215,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1711216810.5522659,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1711216810.5711534,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1711216810.5898972,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1711216810.6090586,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1711216810.6098654,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1711216810.6106498,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1711216810.8354509,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1711216810.8364332,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1711216810.8372991,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1711216810.8380973,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1711216810.8388994,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1711216810.8398454,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1711216810.8407118,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1711216810.8415654,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1711216810.8424015,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1711216810.843209,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1711216810.8439825,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1711216810.8447535,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1711216810.8455334,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1711216810.8560953,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1711216810.8573918,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1711216810.8586354,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1711216810.8598654,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1711216810.861093,"name":"offline","context":{"idset":"555"}}
+{"timestamp":1711216810.8624654,"name":"offline","context":{"idset":"561"}}
+{"timestamp":1711216810.8637631,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1711216810.8649945,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1711216810.8662174,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1711216810.8671038,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1711216810.8864207,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1711216810.9055543,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1711216810.9247177,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1711216810.9443097,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1711216810.9451101,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1711216810.9459686,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1711216810.9467275,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1711216810.9474998,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1711216810.9573755,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1711216810.9762745,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1711216811.0135057,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1711216811.0142634,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1711216811.042351,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1711216811.0430951,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1711216811.0438485,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1711216811.0445964,"name":"offline","context":{"idset":"453"}}
+{"timestamp":1711216811.0545249,"name":"offline","context":{"idset":"466"}}
+{"timestamp":1711216811.0552723,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1711216811.0560195,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1711216811.0567949,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1711216811.0576191,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1711216811.0583882,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1711216811.0683665,"name":"offline","context":{"idset":"445"}}
+{"timestamp":1711216811.0691175,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1711216811.0790298,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1711216811.0797741,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1711216811.0805037,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1711216811.081239,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1711216811.0819693,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1711216811.0827062,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1711216811.0928624,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1711216811.0936403,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1711216811.094378,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1711216811.0951161,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1711216811.0958664,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1711216811.0966313,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1711216811.0973561,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1711216811.1550629,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1711216811.1557915,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1711216811.1565261,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1711216811.1662714,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1711216811.1852882,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1711216811.2045414,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1711216811.2288489,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1711216811.2559099,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1711216811.256665,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1711216811.2573931,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1711216811.2753978,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1711216811.2933815,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1711216811.29407,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1711216811.2947402,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1711216811.2954285,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1711216811.2961001,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1711216811.2967703,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1711216811.3309202,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1711216811.3475304,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1711216811.3562567,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1711216811.3568714,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1711216811.3574834,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1711216811.3580799,"name":"offline","context":{"idset":"552"}}
+{"timestamp":1711216811.3586752,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1711216811.3592696,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1711216811.3598773,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1711216811.3604851,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1711216811.3610995,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1711216811.3617165,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1711216811.3623424,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1711216811.3630128,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1711216811.3636403,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1711216811.3722868,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1711216811.3729014,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1711216811.3735175,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1711216811.3741565,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1711216811.374795,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1711216811.3754435,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1711216811.3760431,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1711216811.3766465,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1711216811.377243,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1711216811.3778818,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1711216811.3785014,"name":"offline","context":{"idset":"549"}}
+{"timestamp":1711216811.3790941,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1711216811.3796773,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1711216811.3802598,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1711216811.3808818,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1711216811.3815033,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1711216811.3898134,"name":"offline","context":{"idset":"447"}}
+{"timestamp":1711216811.4076009,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1711216811.4242108,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1711216811.4414241,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1711216811.4572759,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1711216811.4804976,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1711216811.4810314,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1711216811.4815791,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1711216811.4821007,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1711216811.4826438,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1711216811.4978802,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1711216811.5133247,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1711216811.5292587,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1711216811.5840971,"name":"offline","context":{"idset":"656"}}
+{"timestamp":1711216811.6292813,"name":"offline","context":{"idset":"451"}}
+{"timestamp":1711216811.6297779,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1711216811.6364257,"name":"offline","context":{"idset":"659"}}
+{"timestamp":1711216811.6368773,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1711216811.6435461,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1711216811.6439953,"name":"offline","context":{"idset":"462"}}
+{"timestamp":1711216811.7051437,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1711216811.7055764,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1711216811.7906265,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1711216811.7910273,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1711216811.7914255,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1711216811.7918191,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1711216811.7922075,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1711216811.792613,"name":"offline","context":{"idset":"452"}}
+{"timestamp":1711216811.793005,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1711216811.7988594,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1711216811.8046074,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1711216811.815841,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1711216811.8162391,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1711216811.8220272,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1711216811.8278787,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1711216811.8282657,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1711216811.8286519,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1711216811.8290472,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1711216811.8294303,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1711216811.835403,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1711216811.8358047,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1711216811.8361995,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1711216811.8365986,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1711216811.8369975,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1711216811.8374097,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1711216811.8434017,"name":"offline","context":{"idset":"665"}}
+{"timestamp":1711216811.8547735,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1711216811.8607152,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1711216811.8950648,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1711216811.895469,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1711216811.9014683,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1711216811.9018462,"name":"offline","context":{"idset":"701"}}
+{"timestamp":1711216811.9022188,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1711216811.9081261,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1711216811.9085157,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1711216811.9486084,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1711216811.9490247,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1711216811.9494107,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1711216811.9497862,"name":"offline","context":{"idset":"450"}}
+{"timestamp":1711216811.995049,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1711216811.9954751,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1711216811.9959226,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1711216812.0940051,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1711216812.0944233,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1711216812.0948293,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1711216812.1009033,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1711216812.1013114,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1711216812.1072338,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1711216812.1076128,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1711216812.1080303,"name":"offline","context":{"idset":"558"}}
+{"timestamp":1711216812.1084015,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1711216812.1143765,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1711216812.1371694,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1711216812.1375709,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1711216812.1379497,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1711216812.1439562,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1711216812.1443281,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1711216812.1447072,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1711216812.1564877,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1711216812.156898,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1711216812.1573167,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1711216812.1576822,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1711216812.1580896,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1711216812.1584737,"name":"offline","context":{"idset":"675"}}
+{"timestamp":1711216812.1644163,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1711216812.1647823,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1711216812.1651714,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1711216812.1655614,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1711216812.1659272,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1711216812.1662812,"name":"offline","context":{"idset":"690"}}
+{"timestamp":1711216812.1666434,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1711216812.1669962,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1711216812.1673691,"name":"offline","context":{"idset":"456"}}
+{"timestamp":1711216812.1677237,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1711216812.2015245,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1711216812.2019079,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1711216812.2022808,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1711216812.2026634,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1711216812.2030156,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1711216812.2034171,"name":"offline","context":{"idset":"448"}}
+{"timestamp":1711216812.2038095,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1711216812.2155817,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1711216812.2159345,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1711216812.2162752,"name":"offline","context":{"idset":"708"}}
+{"timestamp":1711216812.2166381,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1711216812.2169876,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1711216812.2173581,"name":"offline","context":{"idset":"669"}}
+{"timestamp":1711216812.2177095,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1711216812.2180524,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1711216812.218406,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1711216812.2242484,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1711216812.224607,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1711216812.286226,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1711216812.3464012,"name":"offline","context":{"idset":"564"}}
+{"timestamp":1711216812.35216,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1711216812.3525112,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1711216812.3583245,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1711216812.3641596,"name":"offline","context":{"idset":"553"}}
+{"timestamp":1711216812.3645167,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1711216812.364852,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1711216812.3651881,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1711216812.3655257,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1711216812.3713276,"name":"offline","context":{"idset":"464"}}
+{"timestamp":1711216812.4046071,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1711216812.4049945,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1711216812.4053593,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1711216812.411195,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1711216812.416954,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1711216812.4282939,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1711216812.428627,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1711216812.4289606,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1711216812.4292872,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1711216812.4296205,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1711216812.429945,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1711216812.4302676,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1711216812.4305921,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1711216812.441927,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1711216812.4477267,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1711216812.4480577,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1711216812.4483876,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1711216812.4487214,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1711216812.4490635,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1711216812.4494126,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1711216812.4497445,"name":"offline","context":{"idset":"660"}}
+{"timestamp":1711216812.450068,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1711216812.4504082,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1711216812.456301,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1711216812.4786365,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1711216812.478967,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1711216812.4903955,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1711216812.507236,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1711216812.5075808,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1711216812.5134671,"name":"offline","context":{"idset":"679"}}
+{"timestamp":1711216812.5413246,"name":"offline","context":{"idset":"550"}}
+{"timestamp":1711216812.541652,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1711216812.5419707,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1711216812.5422843,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1711216812.6296611,"name":"offline","context":{"idset":"672"}}
+{"timestamp":1711216812.6299813,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1711216812.6302984,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1711216812.6306093,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1711216812.6309204,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1711216812.6366267,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1711216812.7346973,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1711216812.7350402,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1711216812.735369,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1711216812.7356818,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1711216812.7359869,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1711216812.7362878,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1711216812.7365975,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1711216812.7369101,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1711216812.7427523,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1711216812.7430856,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1711216812.7434101,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1711216812.7437334,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1711216812.7440364,"name":"offline","context":{"idset":"676"}}
+{"timestamp":1711216812.7443538,"name":"offline","context":{"idset":"677"}}
+{"timestamp":1711216812.7446554,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1711216812.7506301,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1711216812.8010225,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1711216812.8013391,"name":"offline","context":{"idset":"658"}}
+{"timestamp":1711216812.8016362,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1711216812.8019338,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1711216812.8022304,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1711216812.802531,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1711216812.8309693,"name":"offline","context":{"idset":"682"}}
+{"timestamp":1711216812.8759077,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1711216812.8762279,"name":"offline","context":{"idset":"554"}}
+{"timestamp":1711216812.876549,"name":"offline","context":{"idset":"559"}}
+{"timestamp":1711216812.8824103,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1711216812.9323788,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1711216812.9326932,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1711216812.9551194,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1711216812.9554348,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1711216812.9557297,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1711216812.9732101,"name":"offline","context":{"idset":"556"}}
+{"timestamp":1711216812.9735193,"name":"offline","context":{"idset":"670"}}
+{"timestamp":1711216812.9738176,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1711216812.9797392,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1711216812.9856851,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1711216812.9970255,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1711216812.9973238,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1711216812.9976063,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1711216812.9978883,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1711216812.998179,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1711216812.9984663,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1711216812.9987602,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1711216812.9990504,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1711216812.9993412,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1711216812.9996207,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1711216813.0054557,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1711216813.0057595,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1711216813.0340581,"name":"offline","context":{"idset":"666"}}
+{"timestamp":1711216813.0343471,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1711216813.0346231,"name":"offline","context":{"idset":"680"}}
+{"timestamp":1711216813.0349011,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1711216813.0351756,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1711216813.0355465,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1711216813.0358365,"name":"offline","context":{"idset":"563"}}
+{"timestamp":1711216813.0361183,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1711216813.1231644,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1711216813.1234641,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1711216813.1237381,"name":"offline","context":{"idset":"661"}}
+{"timestamp":1711216813.1622097,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1711216813.1624911,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1711216813.1627595,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1711216813.1630242,"name":"offline","context":{"idset":"685"}}
+{"timestamp":1711216813.1632884,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1711216813.1635594,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1711216813.1858742,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1711216813.1861451,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1711216813.2139254,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1711216813.214195,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1711216813.2144749,"name":"offline","context":{"idset":"468"}}
+{"timestamp":1711216813.2202859,"name":"offline","context":{"idset":"667"}}
+{"timestamp":1711216813.2205548,"name":"offline","context":{"idset":"657"}}
+{"timestamp":1711216813.2973483,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1711216813.3518145,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1711216813.3520796,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1711216813.3687103,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1711216813.3689752,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1711216813.4129462,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1711216813.4132113,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1711216813.4134769,"name":"offline","context":{"idset":"662"}}
+{"timestamp":1711216813.4483464,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1711216813.4487548,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1711216813.4491179,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1711216813.449384,"name":"offline","context":{"idset":"551"}}
+{"timestamp":1711216813.4552426,"name":"offline","context":{"idset":"694"}}
+{"timestamp":1711216813.4555278,"name":"offline","context":{"idset":"692"}}
+{"timestamp":1711216813.4557834,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1711216813.4780405,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1711216813.4783359,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1711216813.4786546,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1711216813.4789107,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1711216813.4846797,"name":"offline","context":{"idset":"681"}}
+{"timestamp":1711216813.4849319,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1711216813.485182,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1711216813.4964364,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711216813.4967275,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1711216813.5192327,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1711216813.5900407,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1711216813.5902982,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1711216813.5905459,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1711216813.59079,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1711216813.6456439,"name":"offline","context":{"idset":"671"}}
+{"timestamp":1711216813.6823604,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1711216813.6853819,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1711216813.7517452,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1711216813.7987843,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1711216813.8626435,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1711216813.8829429,"name":"offline","context":{"idset":"707"}}
+{"timestamp":1711216813.8860445,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1711216813.9169805,"name":"offline","context":{"idset":"706"}}
+{"timestamp":1711216813.9972243,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1711216814.0279927,"name":"offline","context":{"idset":"654"}}
+{"timestamp":1711216814.1075547,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1711216814.2074969,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1711216814.313055,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1711216814.4008763,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1711216814.501111,"name":"offline","context":{"idset":"457"}}
+{"timestamp":1711216814.6075265,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1711216814.8362761,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1711216815.0795505,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1711216815.0911598,"name":"offline","context":{"idset":"655"}}
+{"timestamp":1711216815.1917672,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1711216815.501369,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1711216815.6017249,"name":"offline","context":{"idset":"683"}}
+{"timestamp":1711224267.0777807,"name":"online","context":{"idset":"25"}}
+{"timestamp":1711224267.3150415,"name":"online","context":{"idset":"5,9,32,52"}}
+{"timestamp":1711224267.4759977,"name":"online","context":{"idset":"14"}}
+{"timestamp":1711224267.6723018,"name":"online","context":{"idset":"3,6,40,44"}}
+{"timestamp":1711224267.7869537,"name":"online","context":{"idset":"4,8,24,27,34-35"}}
+{"timestamp":1711224267.8997908,"name":"online","context":{"idset":"1-2,11,13,16-17,23"}}
+{"timestamp":1711224268.0510058,"name":"online","context":{"idset":"7,15,22,26,31,38,59"}}
+{"timestamp":1711224268.1737287,"name":"online","context":{"idset":"19-20,30,39,45-46,53,57-58"}}
+{"timestamp":1711224268.2852328,"name":"online","context":{"idset":"18,21,41-42,51,54"}}
+{"timestamp":1711224268.408675,"name":"online","context":{"idset":"10,28-29,33,49,56,60"}}
+{"timestamp":1711224268.5292389,"name":"online","context":{"idset":"12,36,55"}}
+{"timestamp":1711224268.6975076,"name":"online","context":{"idset":"43,50"}}
+{"timestamp":1711224268.8340771,"name":"online","context":{"idset":"37,47-48"}}
+{"timestamp":1711224357.2388186,"name":"online","context":{"idset":"63"}}
+{"timestamp":1711224357.344645,"name":"online","context":{"idset":"68,75"}}
+{"timestamp":1711224357.5239673,"name":"online","context":{"idset":"61-62,66,72,78"}}
+{"timestamp":1711224357.6706388,"name":"online","context":{"idset":"65,69,83,105"}}
+{"timestamp":1711224357.7715516,"name":"online","context":{"idset":"74"}}
+{"timestamp":1711224357.9093292,"name":"online","context":{"idset":"64,84"}}
+{"timestamp":1711224358.0763519,"name":"online","context":{"idset":"67"}}
+{"timestamp":1711224358.4329362,"name":"online","context":{"idset":"79,82,101,108"}}
+{"timestamp":1711224358.5414834,"name":"online","context":{"idset":"73,110"}}
+{"timestamp":1711224358.6945946,"name":"online","context":{"idset":"70,76,99,107"}}
+{"timestamp":1711224358.8207955,"name":"online","context":{"idset":"71,96,115,142,175"}}
+{"timestamp":1711224358.9652967,"name":"online","context":{"idset":"85,141"}}
+{"timestamp":1711224359.1106234,"name":"online","context":{"idset":"80,111,116,133,136,159,166,174"}}
+{"timestamp":1711224359.2151303,"name":"online","context":{"idset":"95,114,123,158,179,185"}}
+{"timestamp":1711224359.3436711,"name":"online","context":{"idset":"77,87,113,151,184"}}
+{"timestamp":1711224359.6243718,"name":"online","context":{"idset":"91-92,119,129,137,181"}}
+{"timestamp":1711224359.7785709,"name":"online","context":{"idset":"81,103,140,160,163,178,182"}}
+{"timestamp":1711224359.9040334,"name":"online","context":{"idset":"126,135,152"}}
+{"timestamp":1711224360.0881081,"name":"online","context":{"idset":"112,127"}}
+{"timestamp":1711224360.2229831,"name":"online","context":{"idset":"104,117,120,134,138-139,147,164"}}
+{"timestamp":1711224360.3473821,"name":"online","context":{"idset":"106,109,161,186,188"}}
+{"timestamp":1711224360.458353,"name":"online","context":{"idset":"102,122,130,149,154,156,180"}}
+{"timestamp":1711224360.6734648,"name":"online","context":{"idset":"89-90,94,128,150,157,171,173,183"}}
+{"timestamp":1711224360.8366451,"name":"online","context":{"idset":"118,124,131,143,145,148,172,177"}}
+{"timestamp":1711224360.9608243,"name":"online","context":{"idset":"86,88,93,97-98,125,132,144,153,162,165,167-169,176"}}
+{"timestamp":1711224361.0624781,"name":"online","context":{"idset":"100,155,187"}}
+{"timestamp":1711224361.2678003,"name":"online","context":{"idset":"170"}}
+{"timestamp":1711224361.5973096,"name":"online","context":{"idset":"146"}}
+{"timestamp":1711224362.9220767,"name":"online","context":{"idset":"189"}}
+{"timestamp":1711224366.6733468,"name":"online","context":{"idset":"191"}}
+{"timestamp":1711224366.7100246,"name":"online","context":{"idset":"190"}}
+{"timestamp":1711224366.955493,"name":"online","context":{"idset":"192-193,195-197"}}
+{"timestamp":1711224367.0035148,"name":"online","context":{"idset":"194"}}
+{"timestamp":1711224367.2662444,"name":"online","context":{"idset":"198-201"}}
+{"timestamp":1711224367.3789213,"name":"online","context":{"idset":"202-205"}}
+{"timestamp":1711224367.838064,"name":"online","context":{"idset":"206,208-209"}}
+{"timestamp":1711224367.9963033,"name":"online","context":{"idset":"207"}}
+{"timestamp":1711224368.1512184,"name":"online","context":{"idset":"210-211,213-214"}}
+{"timestamp":1711224368.2630689,"name":"online","context":{"idset":"212,215-217,219"}}
+{"timestamp":1711224368.3961625,"name":"online","context":{"idset":"218"}}
+{"timestamp":1711224368.4670084,"name":"online","context":{"idset":"220-222"}}
+{"timestamp":1711224368.5749588,"name":"online","context":{"idset":"223-224,226-227"}}
+{"timestamp":1711224368.6874192,"name":"online","context":{"idset":"228-235"}}
+{"timestamp":1711224368.8134007,"name":"online","context":{"idset":"236-238,254"}}
+{"timestamp":1711224368.919569,"name":"online","context":{"idset":"239-240"}}
+{"timestamp":1711224369.0464418,"name":"online","context":{"idset":"241-243"}}
+{"timestamp":1711224369.2112467,"name":"online","context":{"idset":"244-245"}}
+{"timestamp":1711224369.3397636,"name":"online","context":{"idset":"225,248,250-251,253"}}
+{"timestamp":1711224369.4459364,"name":"online","context":{"idset":"246-247,249,252,255-256,258"}}
+{"timestamp":1711224369.6025643,"name":"online","context":{"idset":"257,262"}}
+{"timestamp":1711224369.7152281,"name":"online","context":{"idset":"261,266"}}
+{"timestamp":1711224369.8264544,"name":"online","context":{"idset":"259-260,264,267,269"}}
+{"timestamp":1711224369.9346981,"name":"online","context":{"idset":"263,265,268,270-272,274-276"}}
+{"timestamp":1711224370.0928493,"name":"online","context":{"idset":"273,277-279,281,286"}}
+{"timestamp":1711224370.1934967,"name":"online","context":{"idset":"280,282,285"}}
+{"timestamp":1711224370.3201499,"name":"online","context":{"idset":"283-284,287-291"}}
+{"timestamp":1711224370.5308976,"name":"online","context":{"idset":"292-295,301"}}
+{"timestamp":1711224370.6389661,"name":"online","context":{"idset":"296-297,299,303,307-308"}}
+{"timestamp":1711224370.7459936,"name":"online","context":{"idset":"298,300,302,306,310-311,313"}}
+{"timestamp":1711224370.8464808,"name":"online","context":{"idset":"304,314-315"}}
+{"timestamp":1711224370.9739726,"name":"online","context":{"idset":"305,309"}}
+{"timestamp":1711224371.1040685,"name":"online","context":{"idset":"316"}}
+{"timestamp":1711224372.439744,"name":"online","context":{"idset":"317"}}
+{"timestamp":1711224376.1385143,"name":"online","context":{"idset":"318"}}
+{"timestamp":1711224376.1877909,"name":"online","context":{"idset":"319"}}
+{"timestamp":1711224376.2325342,"name":"online","context":{"idset":"320"}}
+{"timestamp":1711224376.3348362,"name":"online","context":{"idset":"321"}}
+{"timestamp":1711224376.3950984,"name":"online","context":{"idset":"322"}}
+{"timestamp":1711224376.4460187,"name":"online","context":{"idset":"324"}}
+{"timestamp":1711224376.4868913,"name":"online","context":{"idset":"325"}}
+{"timestamp":1711224376.6173022,"name":"online","context":{"idset":"323,326-327"}}
+{"timestamp":1711224376.8199916,"name":"online","context":{"idset":"330,332"}}
+{"timestamp":1711224376.9288945,"name":"online","context":{"idset":"328-329,331,333"}}
+{"timestamp":1711224377.2996235,"name":"online","context":{"idset":"334-336"}}
+{"timestamp":1711224377.4928775,"name":"online","context":{"idset":"337-338,340"}}
+{"timestamp":1711224377.6772261,"name":"online","context":{"idset":"339,341,344,346"}}
+{"timestamp":1711224377.7878411,"name":"online","context":{"idset":"343,345"}}
+{"timestamp":1711224377.9187541,"name":"online","context":{"idset":"342,347-351"}}
+{"timestamp":1711224378.1304855,"name":"online","context":{"idset":"352-355,357,361"}}
+{"timestamp":1711224378.2638974,"name":"online","context":{"idset":"356,358-360,362-363,367"}}
+{"timestamp":1711224378.3652339,"name":"online","context":{"idset":"364-366"}}
+{"timestamp":1711224378.5719562,"name":"online","context":{"idset":"369"}}
+{"timestamp":1711224378.6722841,"name":"online","context":{"idset":"368,370,372"}}
+{"timestamp":1711224378.7791016,"name":"online","context":{"idset":"371"}}
+{"timestamp":1711224378.8852999,"name":"online","context":{"idset":"373-377,379-380,385"}}
+{"timestamp":1711224378.9868793,"name":"online","context":{"idset":"378,381-384"}}
+{"timestamp":1711224379.0966387,"name":"online","context":{"idset":"386-387"}}
+{"timestamp":1711224379.2334974,"name":"online","context":{"idset":"388-393"}}
+{"timestamp":1711224379.4140306,"name":"online","context":{"idset":"396-398,401"}}
+{"timestamp":1711224379.5021071,"name":"online","context":{"idset":"395,399-400,402-403"}}
+{"timestamp":1711224379.6318703,"name":"online","context":{"idset":"404-410,412"}}
+{"timestamp":1711224379.7380867,"name":"online","context":{"idset":"411,413,416"}}
+{"timestamp":1711224379.8530757,"name":"online","context":{"idset":"414,417-419"}}
+{"timestamp":1711224379.9833381,"name":"online","context":{"idset":"415,420-421,424"}}
+{"timestamp":1711224380.1126883,"name":"online","context":{"idset":"394,422,426-427"}}
+{"timestamp":1711224380.2183373,"name":"online","context":{"idset":"423,428-430,432-433"}}
+{"timestamp":1711224380.3202538,"name":"online","context":{"idset":"425,434,436-438"}}
+{"timestamp":1711224380.4274173,"name":"online","context":{"idset":"435,439,442"}}
+{"timestamp":1711224380.5591049,"name":"online","context":{"idset":"441,443"}}
+{"timestamp":1711224381.879456,"name":"online","context":{"idset":"444"}}
+{"timestamp":1711224383.4111898,"name":"online","context":{"idset":"445"}}
+{"timestamp":1711224385.5871353,"name":"online","context":{"idset":"446"}}
+{"timestamp":1711224385.652282,"name":"online","context":{"idset":"448"}}
+{"timestamp":1711224385.6997015,"name":"online","context":{"idset":"450"}}
+{"timestamp":1711224385.7752128,"name":"online","context":{"idset":"447"}}
+{"timestamp":1711224385.8229451,"name":"online","context":{"idset":"449"}}
+{"timestamp":1711224385.9438438,"name":"online","context":{"idset":"451"}}
+{"timestamp":1711224385.9862678,"name":"online","context":{"idset":"452,454"}}
+{"timestamp":1711224386.1282263,"name":"online","context":{"idset":"453"}}
+{"timestamp":1711224386.1716866,"name":"online","context":{"idset":"459"}}
+{"timestamp":1711224386.2789481,"name":"online","context":{"idset":"455-456"}}
+{"timestamp":1711224386.4464138,"name":"online","context":{"idset":"457-458,460-461"}}
+{"timestamp":1711224386.775321,"name":"online","context":{"idset":"463-464"}}
+{"timestamp":1711224386.92858,"name":"online","context":{"idset":"462,465-466"}}
+{"timestamp":1711224387.0678892,"name":"online","context":{"idset":"467-469"}}
+{"timestamp":1711224387.1799018,"name":"online","context":{"idset":"470-471,473"}}
+{"timestamp":1711224387.3819885,"name":"online","context":{"idset":"472,474,476-479"}}
+{"timestamp":1711224387.5018821,"name":"online","context":{"idset":"475"}}
+{"timestamp":1711224387.6127448,"name":"online","context":{"idset":"480,482-484,486"}}
+{"timestamp":1711224387.662199,"name":"online","context":{"idset":"485,487,494"}}
+{"timestamp":1711224387.7853608,"name":"online","context":{"idset":"481,488-489,495"}}
+{"timestamp":1711224387.8958955,"name":"online","context":{"idset":"491-493"}}
+{"timestamp":1711224388.0027092,"name":"online","context":{"idset":"490,496"}}
+{"timestamp":1711224388.2209706,"name":"online","context":{"idset":"497-500,502"}}
+{"timestamp":1711224388.3342643,"name":"online","context":{"idset":"501,503-505,507-509"}}
+{"timestamp":1711224388.5004976,"name":"online","context":{"idset":"506,511-514"}}
+{"timestamp":1711224388.6347342,"name":"online","context":{"idset":"510,515,517-518"}}
+{"timestamp":1711224388.7954321,"name":"online","context":{"idset":"516,519-521"}}
+{"timestamp":1711224388.9328983,"name":"online","context":{"idset":"522-524,529"}}
+{"timestamp":1711224388.9870799,"name":"online","context":{"idset":"525-528"}}
+{"timestamp":1711224389.1432762,"name":"online","context":{"idset":"531,533-534,536-538"}}
+{"timestamp":1711224389.2780938,"name":"online","context":{"idset":"530,532,535,539-541"}}
+{"timestamp":1711224389.3908246,"name":"online","context":{"idset":"542-547"}}
+{"timestamp":1711224389.5434685,"name":"online","context":{"idset":"548,550,554"}}
+{"timestamp":1711224389.681447,"name":"online","context":{"idset":"551,553"}}
+{"timestamp":1711224389.8152225,"name":"online","context":{"idset":"549,552,555-559,561,565"}}
+{"timestamp":1711224389.9241028,"name":"online","context":{"idset":"560,562-564,566-568"}}
+{"timestamp":1711224390.2131627,"name":"online","context":{"idset":"569"}}
+{"timestamp":1711224391.6226287,"name":"online","context":{"idset":"570"}}
+{"timestamp":1711224392.9414549,"name":"online","context":{"idset":"571"}}
+{"timestamp":1711224395.039299,"name":"online","context":{"idset":"572"}}
+{"timestamp":1711224395.1848152,"name":"online","context":{"idset":"573,575-576"}}
+{"timestamp":1711224395.2285151,"name":"online","context":{"idset":"574"}}
+{"timestamp":1711224395.2794983,"name":"online","context":{"idset":"577"}}
+{"timestamp":1711224395.3815413,"name":"online","context":{"idset":"580"}}
+{"timestamp":1711224395.4540279,"name":"online","context":{"idset":"578-579"}}
+{"timestamp":1711224395.564662,"name":"online","context":{"idset":"581"}}
+{"timestamp":1711224395.7292459,"name":"online","context":{"idset":"582-583"}}
+{"timestamp":1711224395.8817155,"name":"online","context":{"idset":"584-585,587"}}
+{"timestamp":1711224395.992569,"name":"online","context":{"idset":"586"}}
+{"timestamp":1711224396.2309375,"name":"online","context":{"idset":"588-589"}}
+{"timestamp":1711224396.368248,"name":"online","context":{"idset":"590"}}
+{"timestamp":1711224396.508198,"name":"online","context":{"idset":"591-593"}}
+{"timestamp":1711224396.620666,"name":"online","context":{"idset":"594-598"}}
+{"timestamp":1711224396.7582018,"name":"online","context":{"idset":"599,602"}}
+{"timestamp":1711224396.9391611,"name":"online","context":{"idset":"600-601,603,605"}}
+{"timestamp":1711224397.0794096,"name":"online","context":{"idset":"604,606-609,611"}}
+{"timestamp":1711224397.256557,"name":"online","context":{"idset":"610,612,614-618"}}
+{"timestamp":1711224397.3978794,"name":"online","context":{"idset":"619-620"}}
+{"timestamp":1711224397.5748806,"name":"online","context":{"idset":"613,621-623"}}
+{"timestamp":1711224397.7392051,"name":"online","context":{"idset":"624-628"}}
+{"timestamp":1711224397.9163046,"name":"online","context":{"idset":"629-630,633,637"}}
+{"timestamp":1711224398.029897,"name":"online","context":{"idset":"631-632,634-636,639-642"}}
+{"timestamp":1711224398.1494379,"name":"online","context":{"idset":"643"}}
+{"timestamp":1711224398.2629111,"name":"online","context":{"idset":"638,644,646-647"}}
+{"timestamp":1711224398.3645606,"name":"online","context":{"idset":"645,648-649,663"}}
+{"timestamp":1711224398.4962432,"name":"online","context":{"idset":"653"}}
+{"timestamp":1711224398.6122127,"name":"online","context":{"idset":"650-652,654-658"}}
+{"timestamp":1711224398.7234831,"name":"online","context":{"idset":"659-662,664-665"}}
+{"timestamp":1711224398.8366284,"name":"online","context":{"idset":"666-668,671,673"}}
+{"timestamp":1711224398.954823,"name":"online","context":{"idset":"669-670,672"}}
+{"timestamp":1711224399.0681117,"name":"online","context":{"idset":"676"}}
+{"timestamp":1711224399.236037,"name":"online","context":{"idset":"674-675,677-678,680"}}
+{"timestamp":1711224399.3576939,"name":"online","context":{"idset":"681-683,688"}}
+{"timestamp":1711224399.4591465,"name":"online","context":{"idset":"679,684-687,692-694"}}
+{"timestamp":1711224399.5806267,"name":"online","context":{"idset":"689-691,695"}}
+{"timestamp":1711224401.0540798,"name":"online","context":{"idset":"696"}}
+{"timestamp":1711224402.4423697,"name":"online","context":{"idset":"697"}}
+{"timestamp":1711224404.6187711,"name":"online","context":{"idset":"698-702"}}
+{"timestamp":1711224404.7619069,"name":"online","context":{"idset":"703-704,711"}}
+{"timestamp":1711224405.0460629,"name":"online","context":{"idset":"705-707"}}
+{"timestamp":1711224405.2189534,"name":"online","context":{"idset":"708-710"}}
+{"timestamp":1711224405.5038326,"name":"online","context":{"idset":"712"}}
+{"timestamp":1711224405.6050861,"name":"online","context":{"idset":"713"}}
+{"timestamp":1711224405.774493,"name":"online","context":{"idset":"714-715"}}
+{"timestamp":1711224405.9472075,"name":"online","context":{"idset":"716"}}
+{"timestamp":1711224406.0935538,"name":"online","context":{"idset":"717-720,726"}}
+{"timestamp":1711224406.2755487,"name":"online","context":{"idset":"721-722,724-725,730"}}
+{"timestamp":1711224406.4184952,"name":"online","context":{"idset":"723,727-729,733"}}
+{"timestamp":1711224406.5950794,"name":"online","context":{"idset":"731,734-736,748"}}
+{"timestamp":1711224406.7487853,"name":"online","context":{"idset":"738-741,744"}}
+{"timestamp":1711224406.8607664,"name":"online","context":{"idset":"732,737,742"}}
+{"timestamp":1711224407.0236728,"name":"online","context":{"idset":"745-747"}}
+{"timestamp":1711224407.1627252,"name":"online","context":{"idset":"749,751-754"}}
+{"timestamp":1711224407.2913032,"name":"online","context":{"idset":"750,756"}}
+{"timestamp":1711224407.6127548,"name":"online","context":{"idset":"743"}}
+{"timestamp":1711224407.7515447,"name":"online","context":{"idset":"755"}}
+{"timestamp":1711224463.7175395,"name":"online","context":{"idset":"312"}}
+{"timestamp":1711224566.3774354,"name":"drain","context":{"idset":"440","reason":"cxi: down","overwrite":0}}
+{"timestamp":1711225405.5338228,"name":"online","context":{"idset":"440"}}
+{"timestamp":1711225456.6227512,"name":"undrain","context":{"idset":"440"}}
+{"timestamp":1711225535.9264688,"name":"undrain","context":{"idset":"372"}}
+{"timestamp":1711225625.8485076,"name":"undrain","context":{"idset":"755"}}
+{"timestamp":1711321271.7724252,"name":"drain","context":{"idset":"11464","reason":"broker was unresponsive"}}
+{"timestamp":1711321335.7713799,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1711324333.0536292,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1711216745.6805143,"reason":"broker was unresponsive"},"2":{"timestamp":1711216745.6806085,"reason":"broker was unresponsive"},"3":{"timestamp":1711216745.680656,"reason":"broker was unresponsive"},"4":{"timestamp":1711216745.6807451,"reason":"broker was unresponsive"},"5":{"timestamp":1711216745.6808176,"reason":"broker was unresponsive"},"6":{"timestamp":1711216745.6808617,"reason":"broker was unresponsive"},"7":{"timestamp":1711216745.6809049,"reason":"broker was unresponsive"},"8":{"timestamp":1711216745.6809447,"reason":"broker was unresponsive"},"9":{"timestamp":1711216745.6809835,"reason":"broker was unresponsive"},"10":{"timestamp":1711216745.6810231,"reason":"broker was unresponsive"},"11":{"timestamp":1711216745.6811044,"reason":"broker was unresponsive"},"12":{"timestamp":1711216745.6811471,"reason":"broker was unresponsive"},"13":{"timestamp":1711216745.6811881,"reason":"broker was unresponsive"},"14":{"timestamp":1711216745.6812286,"reason":"broker was unresponsive"},"15":{"timestamp":1711216745.6812708,"reason":"broker was unresponsive"},"16":{"timestamp":1711216745.6813264,"reason":"broker was unresponsive"},"17":{"timestamp":1711216745.681371,"reason":"broker was unresponsive"},"18":{"timestamp":1711216745.6814148,"reason":"broker was unresponsive"},"19":{"timestamp":1711216745.681462,"reason":"broker was unresponsive"},"20":{"timestamp":1711216745.6833789,"reason":"broker was unresponsive"},"21":{"timestamp":1711216745.6835806,"reason":"broker was unresponsive"},"22":{"timestamp":1711216745.6837616,"reason":"broker was unresponsive"},"23":{"timestamp":1711216745.6838651,"reason":"broker was unresponsive"},"24":{"timestamp":1711216745.6840565,"reason":"broker was unresponsive"},"25":{"timestamp":1711216745.6841562,"reason":"broker was unresponsive"},"26":{"timestamp":1711216745.6842697,"reason":"broker was unresponsive"},"27":{"timestamp":1711216745.6843822,"reason":"broker was unresponsive"},"28":{"timestamp":1711216745.6844778,"reason":"broker was unresponsive"},"29":{"timestamp":1711216745.6845732,"reason":"broker was unresponsive"},"30":{"timestamp":1711216745.6846764,"reason":"broker was unresponsive"},"31":{"timestamp":1711216745.6847744,"reason":"broker was unresponsive"},"32":{"timestamp":1711216745.6848724,"reason":"broker was unresponsive"},"33":{"timestamp":1711216745.6850114,"reason":"broker was unresponsive"},"34":{"timestamp":1711216745.6851158,"reason":"broker was unresponsive"},"35":{"timestamp":1711216745.6852152,"reason":"broker was unresponsive"},"36":{"timestamp":1711216745.6853395,"reason":"broker was unresponsive"},"37":{"timestamp":1711216745.6854444,"reason":"broker was unresponsive"},"38":{"timestamp":1711216745.6855464,"reason":"broker was unresponsive"},"39":{"timestamp":1711216745.6856468,"reason":"broker was unresponsive"},"40":{"timestamp":1711216745.68575,"reason":"broker was unresponsive"},"41":{"timestamp":1711216745.6858568,"reason":"broker was unresponsive"},"42":{"timestamp":1711216745.6859612,"reason":"broker was unresponsive"},"43":{"timestamp":1711216745.6861026,"reason":"broker was unresponsive"},"44":{"timestamp":1711216745.686213,"reason":"broker was unresponsive"},"45":{"timestamp":1711216745.6863275,"reason":"broker was unresponsive"},"46":{"timestamp":1711216745.6864519,"reason":"broker was unresponsive"},"47":{"timestamp":1711216745.6865566,"reason":"broker was unresponsive"},"48":{"timestamp":1711216745.6866629,"reason":"broker was unresponsive"},"49":{"timestamp":1711216745.6867788,"reason":"broker was unresponsive"},"50":{"timestamp":1711216745.6868875,"reason":"broker was unresponsive"},"51":{"timestamp":1711216745.6869948,"reason":"broker was unresponsive"},"52":{"timestamp":1711216745.6871991,"reason":"broker was unresponsive"},"53":{"timestamp":1711216745.6874077,"reason":"broker was unresponsive"},"54":{"timestamp":1711216745.6875894,"reason":"broker was unresponsive"},"55":{"timestamp":1711216745.6877069,"reason":"broker was unresponsive"},"56":{"timestamp":1711216745.6878214,"reason":"broker was unresponsive"},"57":{"timestamp":1711216745.6879315,"reason":"broker was unresponsive"},"58":{"timestamp":1711216745.6880448,"reason":"broker was unresponsive"},"59":{"timestamp":1711216745.6881592,"reason":"broker was unresponsive"},"60":{"timestamp":1711216746.2583418,"reason":"broker was unresponsive"},"109":{"timestamp":1711165087.7736251,"reason":"broker was unresponsive"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"431":{"timestamp":1711096934.358676,"reason":"nodediag failed pci"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"},"11125":{"timestamp":1711126273.9625854,"reason":"nodediag failed bogomips cpucount dmi network pci"},"11126":{"timestamp":1711126278.6191533,"reason":"nodediag failed bogomips cpucount dmi network pci"},"11127":{"timestamp":1711126696.680208,"reason":"nodediag failed bogomips cpucount dmi network pci"},"11165":{"timestamp":1711141273.7721725,"reason":"broker was unresponsive"},"11295":{"timestamp":1711148027.6723082,"reason":"broker was unresponsive"},"11296":{"timestamp":1711148027.7718606,"reason":"broker was unresponsive"},"11331":{"timestamp":1711143325.7809885,"reason":"broker was unresponsive"},"11332":{"timestamp":1711123499.7731371,"reason":"broker was unresponsive"},"11337":{"timestamp":1711117081.6739657,"reason":"broker was unresponsive"},"11338":{"timestamp":1711117081.7735705,"reason":"broker was unresponsive"},"11339":{"timestamp":1711117429.7713454,"reason":"broker was unresponsive"},"11340":{"timestamp":1711117435.673979,"reason":"broker was unresponsive"},"11341":{"timestamp":1711142975.7731442,"reason":"broker was unresponsive"},"11342":{"timestamp":1711117435.6741276,"reason":"broker was unresponsive"},"11343":{"timestamp":1711117435.6742618,"reason":"broker was unresponsive"},"11344":{"timestamp":1711117435.6744049,"reason":"broker was unresponsive"},"11345":{"timestamp":1711117435.7733719,"reason":"broker was unresponsive"},"11365":{"timestamp":1711140955.7734199,"reason":"broker was unresponsive"},"11390":{"timestamp":1711117417.6734309,"reason":"broker was unresponsive"},"11393":{"timestamp":1711117417.6736095,"reason":"broker was unresponsive"},"11394":{"timestamp":1711117417.7733912,"reason":"broker was unresponsive"},"11445":{"timestamp":1711123241.6735494,"reason":"broker was unresponsive"},"11446":{"timestamp":1711123241.6736929,"reason":"broker was unresponsive"},"11447":{"timestamp":1711123241.6738214,"reason":"broker was unresponsive"},"11448":{"timestamp":1711123241.6739299,"reason":"broker was unresponsive"},"11449":{"timestamp":1711123241.674036,"reason":"broker was unresponsive"},"11450":{"timestamp":1711123241.6741452,"reason":"broker was unresponsive"},"11451":{"timestamp":1711123241.7729518,"reason":"broker was unresponsive"},"11452":{"timestamp":1711123247.6733992,"reason":"broker was unresponsive"},"11453":{"timestamp":1711123247.6735187,"reason":"broker was unresponsive"},"11454":{"timestamp":1711123247.6736262,"reason":"broker was unresponsive"},"11455":{"timestamp":1711123247.6737466,"reason":"broker was unresponsive"},"11456":{"timestamp":1711123247.6738572,"reason":"broker was unresponsive"},"11457":{"timestamp":1711123247.6739681,"reason":"broker was unresponsive"},"11458":{"timestamp":1711123247.6740794,"reason":"broker was unresponsive"},"11459":{"timestamp":1711123247.6741953,"reason":"broker was unresponsive"},"11460":{"timestamp":1711123247.7819526,"reason":"broker was unresponsive"},"11464":{"timestamp":1711321271.7724252,"reason":"broker was unresponsive"},"11477":{"timestamp":1711121983.7724895,"reason":"broker was unresponsive"},"11478":{"timestamp":1711123043.7731605,"reason":"broker was unresponsive"},"11501":{"timestamp":1711140105.6735849,"reason":"broker was unresponsive"},"11502":{"timestamp":1711140105.7732108,"reason":"broker was unresponsive"},"11606":{"timestamp":1711117495.6813545,"reason":"broker was unresponsive"},"11608":{"timestamp":1711117495.6814806,"reason":"broker was unresponsive"},"11614":{"timestamp":1711117495.7736769,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1711324333.0701568,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1711324370.3285036,"name":"online","context":{"idset":"0"}}
+{"timestamp":1711324372.3640244,"name":"online","context":{"idset":"11189,11203"}}
+{"timestamp":1711324372.5469553,"name":"online","context":{"idset":"11162,11175,11204,11593"}}
+{"timestamp":1711324372.7428334,"name":"online","context":{"idset":"109,748,11159-11160,11170-11171,11470-11471"}}
+{"timestamp":1711324372.908124,"name":"online","context":{"idset":"117,123,254,11157,11169,11172,11473-11475"}}
+{"timestamp":1711324373.0167179,"name":"online","context":{"idset":"711,11469,11472,11476"}}
+{"timestamp":1711324373.1390502,"name":"online","context":{"idset":"663,11166-11167"}}
+{"timestamp":1711324373.4227724,"name":"online","context":{"idset":"61,63,66,71,74-75,78,82-83,85-88,93,95-96,98,104-105,111,113,116,119,122,126,128,130,134,138-140,143,145,147-148,150-151,154,162-163,165,168-173,175,178,181,186-187,191,197,204-205,216-217,220,223-225,227-228,231,233-234,239,241-243,251,253,255-257,261,263,273,283,285,290,292-293,295,297-299,305,311,313-314,317,324,326-327,332,339,341,343,346,348-349,353,357-358,360-361,363-364,366,368-369,371,375,377-379,381,386-387,393-394,398-400,402,408-409,411,421-422,425-430,432,437-440,444,449,455,457,464,466,468-471,473-474,477,481,485,487,490-491,495-496,498-499,502,504,507-508,511-512,515,519,521-522,527-529,532-533,535,539-540,544,547-553,555,560,563-564,566,568,570,572,576-580,582,585-588,592,594,599,603,608,611-612,617-618,620-622,625,628,637-638,641,644-645,647,649,651,654-658,660,665-667,670,672,675-676,681-682,684,687,690,692-694,696-699,702,706-707,717,733,742,746,754"}}
+{"timestamp":1711324373.5268588,"name":"online","context":{"idset":"12,28,46,52,64-65,67,69,72,77,79-81,84,89-91,94,100-101,103,108,112,115,118,120,129,132-133,135,137,141,144,146,149,153,156-159,161,164,167,174,176,179-180,183-185,188,190,192-196,198,200,203,206-207,209,212-214,219,221,229-230,232,235-238,240,244-248,250,252,264-265,268-270,272,274-279,281,286-288,294,304,306,309,312,319-322,328,333,335-338,342,344-345,347,351-352,354-356,359,362,365,367,372-374,376,380,382,384-385,388-392,395-397,401,403,405-407,412,415,419-420,423,433-436,445-448,453,458-462,465,475-476,478-480,484,492,497,501,503,505,509,516-517,520,523-524,526,534,537-538,542-543,545-546,554,556-559,562,565,569,571,573,581,583-584,589,591,593,596,598,600,604-605,609-610,613,616,619,623-624,626,632,634-636,642-643,646,648,652-653,661,668-669,671,673,677-678,680,683,685,695,700-701,705,718,729,735-737,741,744-745,749-752,11747"}}
+{"timestamp":1711324373.6290665,"name":"online","context":{"idset":"62,70,73,76,92,97,99,102,106-107,110,114,124-125,127,131,136,142,152,155,160,166,177,182,189,199,201-202,208,210-211,215,218,222,226,249,258-260,262,266-267,271,280,282,284,289,291,300-303,307-308,310,315-316,318,323,325,330-331,340,350,370,383,404,410,413-414,416-418,424,441-443,450-452,454,456,463,467,472,482-483,486,488-489,493-494,500,506,510,513-514,518,525,530-531,536,561,567,574-575,590,595,601-602,606-607,615,627,629-631,633,639-640,650,659,662,664,674,679,686,688-689,691,703-704,708,734,743,753,756"}}
+{"timestamp":1711324373.7385681,"name":"online","context":{"idset":"296,329,334,541,614,755"}}
+{"timestamp":1711324373.8760755,"name":"online","context":{"idset":"6,15,24,26,31-34,38,53,57,68,597,747"}}
+{"timestamp":1711324373.9811718,"name":"online","context":{"idset":"3,22"}}
+{"timestamp":1711324374.1244609,"name":"online","context":{"idset":"11748"}}
+{"timestamp":1711324374.3046932,"name":"online","context":{"idset":"1-2,5,7-11,13-14,16-20,23,25,27,29-30,35-37,39,41-44,47-51,54-56,58-60,11199-11200"}}
+{"timestamp":1711324374.4056234,"name":"online","context":{"idset":"4,21,40,45"}}
+{"timestamp":1711324374.5443752,"name":"online","context":{"idset":"11467"}}
+{"timestamp":1711324374.6545405,"name":"online","context":{"idset":"11461,11463,11468"}}
+{"timestamp":1711324374.8124704,"name":"online","context":{"idset":"11462"}}
+{"timestamp":1711324374.9446139,"name":"online","context":{"idset":"730"}}
+{"timestamp":1711324375.5593798,"name":"online","context":{"idset":"713"}}
+{"timestamp":1711324375.9285522,"name":"online","context":{"idset":"11190,11193-11195"}}
+{"timestamp":1711324376.0800569,"name":"online","context":{"idset":"11151,11186,11196,11330,11479-11480"}}
+{"timestamp":1711324376.1867979,"name":"online","context":{"idset":"11174,11182,11328,11354,11482"}}
+{"timestamp":1711324376.2936034,"name":"online","context":{"idset":"11125,11150,11177,11180-11181,11183,11185,11207,11325-11327,11329,11350,11483"}}
+{"timestamp":1711324376.4377673,"name":"online","context":{"idset":"11149,11152-11156,11173,11176,11178,11210,11353,11397,11481,11484"}}
+{"timestamp":1711324376.5450735,"name":"online","context":{"idset":"712,11205-11206,11208-11209,11351-11352,11411"}}
+{"timestamp":1711324376.6892276,"name":"online","context":{"idset":"11503,11505"}}
+{"timestamp":1711324376.8027353,"name":"online","context":{"idset":"11504,11507-11508,11596,11598"}}
+{"timestamp":1711324376.9088364,"name":"online","context":{"idset":"11366,11409,11506,11589,11602"}}
+{"timestamp":1711324377.0303676,"name":"online","context":{"idset":"11130,11343,11365,11367-11369,11371,11398,11401,11590,11592,11594-11595,11597,11599,11603-11604"}}
+{"timestamp":1711324377.1279013,"name":"online","context":{"idset":"11344,11347,11372,11591,11601"}}
+{"timestamp":1711324377.2724113,"name":"online","context":{"idset":"721,731,738,11129,11131-11132,11341,11345,11348,11612"}}
+{"timestamp":1711324377.4189653,"name":"online","context":{"idset":"723,11127-11128,11220,11294,11297,11319,11435,11489,11622"}}
+{"timestamp":1711324377.5275176,"name":"online","context":{"idset":"714,11135-11136,11244,11250,11288,11298,11311,11320,11322,11389,11405,11414,11430,11434,11444,11486-11487,11492,11607,11609,11611,11617,11628,11635"}}
+{"timestamp":1711324377.6366079,"name":"online","context":{"idset":"11134,11137,11139,11216,11218-11219,11237-11239,11243,11245,11247-11248,11252,11286-11287,11289-11290,11293,11299-11301,11303-11304,11306-11308,11317-11318,11321,11323-11324,11357,11360,11390-11393,11399,11410,11415,11417,11419,11421,11424,11427,11431-11432,11436,11440,11442,11485,11488,11490-11491,11606,11608,11610,11614,11619-11621,11623-11626,11629-11630,11632-11634"}}
+{"timestamp":1711324377.7579379,"name":"online","context":{"idset":"709,11133,11138,11140,11215,11217,11240-11242,11251,11285,11291-11292,11302,11305,11309-11310,11312,11314,11358-11359,11361-11364,11381-11383,11385-11388,11395-11396,11413,11416,11418,11420,11422-11423,11426,11428-11429,11433,11437-11439,11443,11613,11615-11616,11618,11627,11631,11636"}}
+{"timestamp":1711324377.8664694,"name":"online","context":{"idset":"719,724,732,11221-11222,11225,11227,11231,11236,11246,11249,11313,11315-11316,11384,11394,11403,11425,11441"}}
+{"timestamp":1711324377.9755633,"name":"online","context":{"idset":"716,720,725,727-728,739-740,11224,11226,11228,11232-11234,11346,11375,11377,11400,11406-11407"}}
+{"timestamp":1711324378.099113,"name":"online","context":{"idset":"11223,11229,11235,11374,11380,11404,11412,11497"}}
+{"timestamp":1711324378.2110231,"name":"online","context":{"idset":"715,726,11373,11376,11378-11379,11402,11408,11493-11494,11496,11500"}}
+{"timestamp":1711324378.388355,"name":"online","context":{"idset":"11495,11498-11499,11605"}}
+{"timestamp":1711324378.685225,"name":"online","context":{"idset":"710"}}
+{"timestamp":1711324379.0804417,"name":"online","context":{"idset":"722"}}
+{"timestamp":1711324379.5691264,"name":"online","context":{"idset":"11141,11147"}}
+{"timestamp":1711324379.670511,"name":"online","context":{"idset":"11143-11146"}}
+{"timestamp":1711324379.7810535,"name":"online","context":{"idset":"11142,11148"}}
+{"timestamp":1711324380.1559916,"name":"online","context":{"idset":"11337-11340"}}
+{"timestamp":1711324380.2710674,"name":"online","context":{"idset":"11333-11336"}}
+{"timestamp":1711325310.3051066,"name":"drain","context":{"idset":"11149,11155,11159,11165,11175,11188-11189,11192,11195,11203-11204,11293,11296,11308,11341,11353,11369,11374,11384,11387,11394,11402,11431,11433,11448,11450,11453,11463,11481,11488,11490,11593,11607,11624,11634","reason":"badnode","overwrite":1}}
+{"timestamp":1711325839.4671319,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1711325839.5107687,"name":"offline","context":{"idset":"11149"}}
+{"timestamp":1711325839.5215116,"name":"offline","context":{"idset":"11353"}}
+{"timestamp":1711325839.5317075,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1711325839.5795214,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1711325839.5835383,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1711325839.5941536,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1711325839.604816,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1711325839.6207459,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1711325839.6302638,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1711325839.647697,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1711325839.6487873,"name":"offline","context":{"idset":"11293"}}
+{"timestamp":1711325839.6498451,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1711325839.661926,"name":"offline","context":{"idset":"11374"}}
+{"timestamp":1711325839.6715689,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1711325839.6726577,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1711325839.6891422,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1711325839.6990347,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1711325839.7241974,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1711325839.725337,"name":"offline","context":{"idset":"11308"}}
+{"timestamp":1711325839.7264466,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1711325839.7275605,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1711325839.7336748,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1711325839.7512839,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1711325839.7722588,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1711325839.7933197,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1711325839.8029284,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1711325839.9694829,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1711326134.6213346,"name":"drain","context":{"idset":"11346,11605","reason":"GPU memory in use failing apdgpu.t","overwrite":0}}
+{"timestamp":1711326347.0885549,"name":"online","context":{"idset":"11202"}}
+{"timestamp":1711326347.7982347,"name":"online","context":{"idset":"11459"}}
+{"timestamp":1711326348.4064236,"name":"online","context":{"idset":"11356"}}
+{"timestamp":1711326348.5729427,"name":"online","context":{"idset":"11212"}}
+{"timestamp":1711326348.8045812,"name":"online","context":{"idset":"11752"}}
+{"timestamp":1711326349.3092456,"name":"online","context":{"idset":"11457"}}
+{"timestamp":1711326349.6454592,"name":"online","context":{"idset":"11331,11758"}}
+{"timestamp":1711326349.8281436,"name":"online","context":{"idset":"11332,11755-11756,11759-11760"}}
+{"timestamp":1711326349.9964108,"name":"online","context":{"idset":"11761"}}
+{"timestamp":1711326350.1390166,"name":"online","context":{"idset":"11754,11764"}}
+{"timestamp":1711326350.320024,"name":"online","context":{"idset":"11750,11762-11763"}}
+{"timestamp":1711326350.4664655,"name":"online","context":{"idset":"11466,11477,11749"}}
+{"timestamp":1711326350.6050968,"name":"online","context":{"idset":"11456,11753,11757"}}
+{"timestamp":1711326350.7483783,"name":"online","context":{"idset":"11370,11751"}}
+{"timestamp":1711326350.9261773,"name":"online","context":{"idset":"11458"}}
+{"timestamp":1711326351.027154,"name":"online","context":{"idset":"11446"}}
+{"timestamp":1711326351.128973,"name":"online","context":{"idset":"11451"}}
+{"timestamp":1711326351.2502425,"name":"online","context":{"idset":"11455"}}
+{"timestamp":1711326351.4062545,"name":"online","context":{"idset":"11454"}}
+{"timestamp":1711326351.6455891,"name":"online","context":{"idset":"11445,11447,11452"}}
+{"timestamp":1711326792.8705738,"name":"drain","context":{"idset":"11317","reason":"epilog failed for jobid fka2yuNfWCo","overwrite":0}}
+{"timestamp":1711326792.9661782,"name":"offline","context":{"idset":"11317"}}
+{"timestamp":1711377214.5747344,"name":"offline","context":{"idset":"11356"}}
+{"timestamp":1711377606.9675629,"name":"drain","context":{"idset":"11434","reason":"broker was unresponsive"}}
+{"timestamp":1711377672.9673829,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1711378194.9686093,"name":"drain","context":{"idset":"11352","reason":"broker was unresponsive"}}
+{"timestamp":1711378258.967262,"name":"offline","context":{"idset":"11352"}}
+{"timestamp":1711379754.1735857,"name":"online","context":{"idset":"11607"}}
+{"timestamp":1711380036.2964509,"name":"online","context":{"idset":"11384"}}
+{"timestamp":1711380711.0640893,"name":"online","context":{"idset":"11159"}}
+{"timestamp":1711380711.8144889,"name":"online","context":{"idset":"11463"}}
+{"timestamp":1711380712.0241082,"name":"online","context":{"idset":"11195,11634"}}
+{"timestamp":1711380712.397625,"name":"online","context":{"idset":"11293"}}
+{"timestamp":1711380712.5762005,"name":"online","context":{"idset":"11308"}}
+{"timestamp":1711381520.9672534,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1711381652.9670918,"name":"offline","context":{"idset":"11308"}}
+{"timestamp":1711382845.6396153,"name":"online","context":{"idset":"11624"}}
+{"timestamp":1711382845.9703591,"name":"online","context":{"idset":"11481"}}
+{"timestamp":1711383304.9860458,"name":"online","context":{"idset":"11192"}}
+{"timestamp":1711383352.9718282,"name":"undrain","context":{"idset":"11175"}}
+{"timestamp":1711383462.9680221,"name":"drain","context":{"idset":"11426","reason":"broker was unresponsive"}}
+{"timestamp":1711383526.9669595,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1711383693.3026993,"name":"online","context":{"idset":"11356"}}
+{"timestamp":1711383699.4677567,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1711383733.4342835,"name":"online","context":{"idset":"11355"}}
+{"timestamp":1711384352.7499061,"name":"online","context":{"idset":"11634"}}
+{"timestamp":1711384352.8652163,"name":"online","context":{"idset":"11308"}}
+{"timestamp":1711384429.6746376,"name":"online","context":{"idset":"11352"}}
+{"timestamp":1711384650.8648033,"name":"online","context":{"idset":"11402"}}
+{"timestamp":1711386319.6955035,"name":"online","context":{"idset":"11433"}}
+{"timestamp":1711386319.9561217,"name":"online","context":{"idset":"11353,11431"}}
+{"timestamp":1711386320.0575633,"name":"online","context":{"idset":"11374"}}
+{"timestamp":1711386320.6523879,"name":"online","context":{"idset":"11369"}}
+{"timestamp":1711386330.8410144,"name":"undrain","context":{"idset":"11155,11159,11192,11195,11293,11308,11341,11353,11369,11374,11384,11402,11431,11433,11463,11481,11607,11624,11634"}}
+{"timestamp":1711386511.3684192,"name":"drain","context":{"idset":"11125-11127","reason":"ama fail","overwrite":1}}
+{"timestamp":1711386755.9220569,"name":"drain","context":{"idset":"11149,11189,11203-11204,11387,11394","reason":"Rabbit link Failure","overwrite":1}}
+{"timestamp":1711386882.9674518,"name":"drain","context":{"idset":"11229","reason":"broker was unresponsive"}}
+{"timestamp":1711386944.9666948,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1711388219.0690517,"name":"drain","context":{"idset":"11296,11448,11453","reason":"ama fail","overwrite":1}}
+{"timestamp":1711388382.3303399,"name":"drain","context":{"idset":"11188,11488,11490,11593","reason":"Rabbit link Failure","overwrite":1}}
+{"timestamp":1711388625.9909227,"name":"online","context":{"idset":"11165"}}
+{"timestamp":1711388656.0928767,"name":"undrain","context":{"idset":"11165"}}
+{"timestamp":1711388780.0590403,"name":"online","context":{"idset":"11464"}}
+{"timestamp":1711388780.6767459,"name":"online","context":{"idset":"11426"}}
+{"timestamp":1711388787.3195274,"name":"undrain","context":{"idset":"11426,11464"}}
+{"timestamp":1711389100.8684056,"name":"drain","context":{"idset":"11351","reason":"broker was unresponsive"}}
+{"timestamp":1711389162.8741028,"name":"offline","context":{"idset":"11351"}}
+{"timestamp":1711389162.9667544,"name":"offline","context":{"idset":"11352"}}
+{"timestamp":1711390647.7607419,"name":"online","context":{"idset":"11352"}}
+{"timestamp":1711391014.9677579,"name":"drain","context":{"idset":"11318","reason":"broker was unresponsive"}}
+{"timestamp":1711391043.7835953,"name":"online","context":{"idset":"11168"}}
+{"timestamp":1711391078.9668586,"name":"offline","context":{"idset":"11318"}}
+{"timestamp":1711391173.0856833,"name":"online","context":{"idset":"11163"}}
+{"timestamp":1711391226.7031353,"name":"online","context":{"idset":"11161"}}
+{"timestamp":1711391265.6120374,"name":"online","context":{"idset":"11158"}}
+{"timestamp":1711391417.8841317,"name":"online","context":{"idset":"11191"}}
+{"timestamp":1711392815.4005558,"name":"undrain","context":{"idset":"11331-11332,11337-11340,11343-11345,11352,11365,11390,11393,11445-11447,11451-11452,11454-11459,11477,11606,11608,11614"}}
+{"timestamp":1711392886.0817285,"name":"drain","context":{"idset":"11450","reason":"ama fail","overwrite":1}}
+{"timestamp":1711393750.8690577,"name":"drain","context":{"idset":"11343","reason":"broker was unresponsive"}}
+{"timestamp":1711393750.8691745,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1711393750.8692429,"name":"drain","context":{"idset":"11345","reason":"broker was unresponsive"}}
+{"timestamp":1711393750.8693571,"name":"drain","context":{"idset":"11347","reason":"broker was unresponsive"}}
+{"timestamp":1711393750.9687252,"name":"drain","context":{"idset":"11348","reason":"broker was unresponsive"}}
+{"timestamp":1711393810.4709265,"name":"online","context":{"idset":"11126"}}
+{"timestamp":1711393816.8758547,"name":"offline","context":{"idset":"11343"}}
+{"timestamp":1711393816.8854203,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1711393816.8934798,"name":"offline","context":{"idset":"11345"}}
+{"timestamp":1711393816.9017949,"name":"offline","context":{"idset":"11346"}}
+{"timestamp":1711393816.9092531,"name":"offline","context":{"idset":"11347"}}
+{"timestamp":1711393816.9824855,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1711393888.9688671,"name":"drain","context":{"idset":"11165","reason":"broker was unresponsive"}}
+{"timestamp":1711393954.9664285,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1711395127.3814554,"name":"drain","context":{"idset":"11295,11449,11460,11502","reason":"ama fail","overwrite":1}}
+{"timestamp":1711395678.9683194,"name":"drain","context":{"idset":"722","reason":"broker was unresponsive"}}
+{"timestamp":1711395744.9665804,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1711396048.0523934,"name":"online","context":{"idset":"11229"}}
+{"timestamp":1711396472.5749269,"name":"online","context":{"idset":"11165"}}
+{"timestamp":1711396595.5186663,"name":"online","context":{"idset":"11203-11204"}}
+{"timestamp":1711396595.6742485,"name":"online","context":{"idset":"11189"}}
+{"timestamp":1711397333.8162749,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1711398262.969044,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1711398264.9671254,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1711398293.5258851,"name":"online","context":{"idset":"11344"}}
+{"timestamp":1711398331.0066085,"name":"undrain","context":{"idset":"11344"}}
+{"timestamp":1711398620.7183728,"name":"online","context":{"idset":"11348"}}
+{"timestamp":1711398633.4381323,"name":"undrain","context":{"idset":"11348"}}
+{"timestamp":1711398726.4288077,"name":"online","context":{"idset":"11434"}}
+{"timestamp":1711398744.5483856,"name":"undrain","context":{"idset":"11434"}}
+{"timestamp":1711399127.4139991,"name":"online","context":{"idset":"11188"}}
+{"timestamp":1711399141.6832712,"name":"online","context":{"idset":"11211,11214"}}
+{"timestamp":1711399648.6946192,"name":"drain","context":{"idset":"11190","reason":"bad partner","overwrite":0}}
+{"timestamp":1711399689.1859024,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1711399743.1944523,"name":"online","context":{"idset":"11351"}}
+{"timestamp":1711399750.0673065,"name":"undrain","context":{"idset":"11351"}}
+{"timestamp":1711399858.8681102,"name":"drain","context":{"idset":"11157","reason":"broker was unresponsive"}}
+{"timestamp":1711399858.8682301,"name":"drain","context":{"idset":"11167","reason":"broker was unresponsive"}}
+{"timestamp":1711399858.9679446,"name":"drain","context":{"idset":"11168","reason":"broker was unresponsive"}}
+{"timestamp":1711399864.9688313,"name":"drain","context":{"idset":"11169","reason":"broker was unresponsive"}}
+{"timestamp":1711399910.9672837,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1711399924.8713844,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1711399924.8800735,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1711399924.9668744,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1711399926.9672084,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1711399996.9694977,"name":"drain","context":{"idset":"11202","reason":"broker was unresponsive"}}
+{"timestamp":1711400058.966701,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1711401067.3461077,"name":"drain","context":{"idset":"11187","overwrite":0}}
+{"timestamp":1711401109.754894,"name":"drain","context":{"idset":"11187","reason":"reason=drain for Rabbit connection","overwrite":0}}
+{"timestamp":1711401210.565743,"name":"online","context":{"idset":"11169"}}
+{"timestamp":1711401210.876626,"name":"online","context":{"idset":"11157,11167-11168"}}
+{"timestamp":1711401267.6071715,"name":"drain","context":{"idset":"11501","reason":"ama fail","overwrite":1}}
+{"timestamp":1711401359.6521537,"name":"undrain","context":{"idset":"11167"}}
+{"timestamp":1711401561.9171531,"name":"online","context":{"idset":"11318"}}
+{"timestamp":1711401570.2090859,"name":"undrain","context":{"idset":"11318"}}
+{"timestamp":1711401728.8686216,"name":"drain","context":{"idset":"11167","reason":"broker was unresponsive"}}
+{"timestamp":1711401778.8045268,"name":"online","context":{"idset":"11213"}}
+{"timestamp":1711401792.8723252,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1711401792.8819687,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1711401792.966511,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1711401794.9678535,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1711401846.9683437,"name":"drain","context":{"idset":"11393","reason":"broker was unresponsive"}}
+{"timestamp":1711401913.7476814,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1711402398.9669802,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1711402765.6523082,"name":"online","context":{"idset":"11179"}}
+{"timestamp":1711403791.5491529,"name":"online","context":{"idset":"11157"}}
+{"timestamp":1711403900.9178135,"name":"online","context":{"idset":"11167"}}
+{"timestamp":1711404185.9761777,"name":"online","context":{"idset":"11168"}}
+{"timestamp":1711404334.4127696,"name":"online","context":{"idset":"11169"}}
+{"timestamp":1711404572.9688854,"name":"drain","context":{"idset":"11179","reason":"broker was unresponsive"}}
+{"timestamp":1711404636.9670498,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1711404992.6359529,"name":"undrain","context":{"idset":"11157,11165,11167-11169,11229"}}
+{"timestamp":1711405087.0970409,"name":"online","context":{"idset":"11202"}}
+{"timestamp":1711405110.2980747,"name":"undrain","context":{"idset":"11202,11478"}}
+{"timestamp":1711405119.195823,"name":"drain","context":{"idset":"11150,11388","reason":"reason=Draining for partner Rabbit","overwrite":0}}
+{"timestamp":1711405824.9193692,"name":"online","context":{"idset":"11184"}}
+{"timestamp":1711406173.540354,"name":"online","context":{"idset":"11187"}}
+{"timestamp":1711406762.8675106,"name":"drain","context":{"idset":"11213","reason":"broker was unresponsive"}}
+{"timestamp":1711406762.9678555,"name":"drain","context":{"idset":"11216","reason":"broker was unresponsive"}}
+{"timestamp":1711406828.8715682,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1711406828.967382,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1711406918.2155173,"name":"online","context":{"idset":"11203"}}
+{"timestamp":1711406918.4292984,"name":"online","context":{"idset":"11204"}}
+{"timestamp":1711406918.9858203,"name":"online","context":{"idset":"11189"}}
+{"timestamp":1711406919.4089139,"name":"online","context":{"idset":"11197"}}
+{"timestamp":1711406919.9359078,"name":"online","context":{"idset":"11198,11201"}}
+{"timestamp":1711406991.364831,"name":"online","context":{"idset":"11460"}}
+{"timestamp":1711407067.4051404,"name":"online","context":{"idset":"11296"}}
+{"timestamp":1711407167.6842554,"name":"online","context":{"idset":"11450"}}
+{"timestamp":1711407175.2087779,"name":"online","context":{"idset":"11449"}}
+{"timestamp":1711407252.6312881,"name":"online","context":{"idset":"11453"}}
+{"timestamp":1711407860.45856,"name":"drain","context":{"idset":"11128-11148,11151-11154,11156-11163,11165-11174,11177-11178,11180-11186,11191-11202,11205-11212,11214-11215,11217-11229,11231-11252,11285-11294,11297-11316,11318-11340,11344,11348,11350-11386,11389-11392,11395-11447,11451-11452,11454-11459,11461-11464,11466-11477,11479-11487,11489,11491-11500,11503-11508,11589-11592,11594-11599,11601-11604,11606-11636,11747-11764","reason":"FW/BIOS updates","overwrite":0}}
+{"timestamp":1711407906.2318344,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1711407906.3408237,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1711407906.3898435,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1711407906.4314086,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1711407906.4421663,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1711407906.4502881,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1711407906.462676,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1711407906.5123668,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1711407906.5134897,"name":"offline","context":{"idset":"11146"}}
+{"timestamp":1711407906.522907,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1711407906.532439,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1711407906.5418291,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1711407906.550704,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1711407906.5616663,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1711407906.6416471,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1711407906.6427975,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1711407906.6439171,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1711407906.6945131,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1711407906.6964302,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1711407906.705718,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1711407906.7153587,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1711407906.729399,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1711407906.7376089,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1711407906.748693,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1711407906.7599425,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1711407906.776299,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1711407906.7848928,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1711407906.7862473,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1711407906.7930822,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1711407906.809442,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1711407906.8281233,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1711407906.8292241,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1711407906.8322754,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1711407906.8403916,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1711407906.8414996,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1711407906.842593,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1711407906.8613455,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1711407906.8800902,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1711407906.8817608,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1711407906.8894656,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1711407906.8905587,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1711407906.9004891,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1711407906.9200454,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1711407906.93943,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1711407906.9405208,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1711407906.9416063,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1711407906.9426837,"name":"offline","context":{"idset":"11153"}}
+{"timestamp":1711407906.9527905,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1711407906.9717145,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1711407906.9908297,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1711407907.0009153,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1711407907.0024602,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1711407907.0035851,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1711407907.0047297,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1711407907.0241587,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1711407907.0345194,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1711407907.0357342,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1711407907.0368059,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1711407907.0378742,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1711407907.0389409,"name":"offline","context":{"idset":"11154"}}
+{"timestamp":1711407907.0400505,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1711407907.0411103,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1711407907.0604353,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1711407907.0800245,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1711407907.0810952,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1711407907.0821617,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1711407907.0832114,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1711407907.0842731,"name":"offline","context":{"idset":"11300"}}
+{"timestamp":1711407907.0853271,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1711407907.0863762,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1711407907.0874166,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1711407907.097856,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1711407907.1171072,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1711407907.136313,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1711407907.1373625,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1711407907.1384039,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1711407907.13956,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1711407907.147733,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1711407907.1554418,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1711407907.163075,"name":"offline","context":{"idset":"11292"}}
+{"timestamp":1711407907.170953,"name":"offline","context":{"idset":"11299"}}
+{"timestamp":1711407907.1862271,"name":"offline","context":{"idset":"11301"}}
+{"timestamp":1711407907.1874497,"name":"offline","context":{"idset":"11304"}}
+{"timestamp":1711407907.1941085,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1711407907.2197533,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1711407907.2390282,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1711407907.258574,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1711407907.2596154,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1711407907.260643,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1711407907.261667,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1711407907.262697,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1711407907.2689059,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1711407907.2926552,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1711407907.3083882,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1711407907.315815,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1711407907.3230433,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1711407907.3303566,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1711407907.3376946,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1711407907.345001,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1711407907.3594701,"name":"offline","context":{"idset":"11287"}}
+{"timestamp":1711407907.373878,"name":"offline","context":{"idset":"11293"}}
+{"timestamp":1711407907.3815348,"name":"offline","context":{"idset":"11297"}}
+{"timestamp":1711407907.382565,"name":"offline","context":{"idset":"11302"}}
+{"timestamp":1711407907.398134,"name":"offline","context":{"idset":"11303"}}
+{"timestamp":1711407907.4173064,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1711407907.4366732,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1711407907.4562693,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1711407907.4755502,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1711407907.4954917,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1711407907.5055907,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1711407907.5066202,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1711407907.5076466,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1711407907.508743,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1711407907.5100107,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1711407907.5225205,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1711407907.529633,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1711407907.541434,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1711407907.5493696,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1711407907.5606949,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1711407907.5669858,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1711407907.5754342,"name":"offline","context":{"idset":"11285"}}
+{"timestamp":1711407907.5766478,"name":"offline","context":{"idset":"11288"}}
+{"timestamp":1711407907.5827472,"name":"offline","context":{"idset":"11289"}}
+{"timestamp":1711407907.590152,"name":"offline","context":{"idset":"11290"}}
+{"timestamp":1711407907.5973105,"name":"offline","context":{"idset":"11291"}}
+{"timestamp":1711407907.6046264,"name":"offline","context":{"idset":"11298"}}
+{"timestamp":1711407907.6189399,"name":"offline","context":{"idset":"11305"}}
+{"timestamp":1711407907.6560962,"name":"offline","context":{"idset":"11294"}}
+{"timestamp":1711407910.7138937,"name":"offline","context":{"idset":"11306"}}
+{"timestamp":1711407910.7750001,"name":"offline","context":{"idset":"11308"}}
+{"timestamp":1711407910.8749142,"name":"offline","context":{"idset":"11307"}}
+{"timestamp":1711407910.9717386,"name":"offline","context":{"idset":"11310"}}
+{"timestamp":1711407911.0709257,"name":"offline","context":{"idset":"11309"}}
+{"timestamp":1711407911.1845891,"name":"offline","context":{"idset":"11311"}}
+{"timestamp":1711407911.2139866,"name":"offline","context":{"idset":"11313"}}
+{"timestamp":1711407911.3138411,"name":"offline","context":{"idset":"11312"}}
+{"timestamp":1711407911.6853857,"name":"offline","context":{"idset":"11314"}}
+{"timestamp":1711407911.7835131,"name":"offline","context":{"idset":"11316"}}
+{"timestamp":1711407911.8016171,"name":"offline","context":{"idset":"11318"}}
+{"timestamp":1711407911.8771932,"name":"offline","context":{"idset":"11315"}}
+{"timestamp":1711407911.8924234,"name":"offline","context":{"idset":"11319"}}
+{"timestamp":1711407911.9485002,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1711407911.950094,"name":"offline","context":{"idset":"11325"}}
+{"timestamp":1711407911.9580469,"name":"offline","context":{"idset":"11331"}}
+{"timestamp":1711407911.9684966,"name":"offline","context":{"idset":"11320"}}
+{"timestamp":1711407911.9778101,"name":"offline","context":{"idset":"11324"}}
+{"timestamp":1711407912.0564833,"name":"offline","context":{"idset":"11323"}}
+{"timestamp":1711407912.0574777,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1711407912.0584652,"name":"offline","context":{"idset":"11322"}}
+{"timestamp":1711407912.059464,"name":"offline","context":{"idset":"11326"}}
+{"timestamp":1711407912.0974505,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1711407912.126936,"name":"offline","context":{"idset":"11338"}}
+{"timestamp":1711407912.1389494,"name":"offline","context":{"idset":"11328"}}
+{"timestamp":1711407912.1399512,"name":"offline","context":{"idset":"11329"}}
+{"timestamp":1711407912.1463718,"name":"offline","context":{"idset":"11351"}}
+{"timestamp":1711407912.1616011,"name":"offline","context":{"idset":"11352"}}
+{"timestamp":1711407912.1699221,"name":"offline","context":{"idset":"11327"}}
+{"timestamp":1711407912.1777556,"name":"offline","context":{"idset":"11333"}}
+{"timestamp":1711407912.1845355,"name":"offline","context":{"idset":"11339"}}
+{"timestamp":1711407912.2003894,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1711407912.2019706,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1711407912.2494423,"name":"offline","context":{"idset":"11337"}}
+{"timestamp":1711407912.2508225,"name":"offline","context":{"idset":"11330"}}
+{"timestamp":1711407912.3199503,"name":"offline","context":{"idset":"11335"}}
+{"timestamp":1711407912.3477015,"name":"offline","context":{"idset":"11336"}}
+{"timestamp":1711407912.3487737,"name":"offline","context":{"idset":"11353"}}
+{"timestamp":1711407912.3498311,"name":"offline","context":{"idset":"11334"}}
+{"timestamp":1711407912.4031222,"name":"offline","context":{"idset":"11350"}}
+{"timestamp":1711407912.5591977,"name":"offline","context":{"idset":"11356"}}
+{"timestamp":1711407912.5959098,"name":"offline","context":{"idset":"11355"}}
+{"timestamp":1711407912.6960206,"name":"offline","context":{"idset":"11354"}}
+{"timestamp":1711407912.7165043,"name":"offline","context":{"idset":"11357"}}
+{"timestamp":1711407912.7553837,"name":"offline","context":{"idset":"11358"}}
+{"timestamp":1711407912.76459,"name":"offline","context":{"idset":"11360"}}
+{"timestamp":1711407912.7715459,"name":"offline","context":{"idset":"11361"}}
+{"timestamp":1711407912.8244574,"name":"offline","context":{"idset":"11359"}}
+{"timestamp":1711407912.832669,"name":"offline","context":{"idset":"11365"}}
+{"timestamp":1711407912.8412461,"name":"offline","context":{"idset":"11363"}}
+{"timestamp":1711407912.9352047,"name":"offline","context":{"idset":"11364"}}
+{"timestamp":1711407912.9546111,"name":"offline","context":{"idset":"11362"}}
+{"timestamp":1711407913.0462427,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1711407913.0573728,"name":"offline","context":{"idset":"11367"}}
+{"timestamp":1711407913.1027677,"name":"offline","context":{"idset":"11370"}}
+{"timestamp":1711407913.110929,"name":"offline","context":{"idset":"11366"}}
+{"timestamp":1711407913.1177552,"name":"offline","context":{"idset":"11372"}}
+{"timestamp":1711407913.1856742,"name":"offline","context":{"idset":"11368"}}
+{"timestamp":1711407913.195152,"name":"offline","context":{"idset":"11375"}}
+{"timestamp":1711407913.2022023,"name":"offline","context":{"idset":"11374"}}
+{"timestamp":1711407913.2514312,"name":"offline","context":{"idset":"11376"}}
+{"timestamp":1711407913.2820785,"name":"offline","context":{"idset":"11373"}}
+{"timestamp":1711407913.2942832,"name":"offline","context":{"idset":"11371"}}
+{"timestamp":1711407913.3024633,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1711407913.3499155,"name":"offline","context":{"idset":"11377"}}
+{"timestamp":1711407913.3854482,"name":"offline","context":{"idset":"11379"}}
+{"timestamp":1711407913.3954444,"name":"offline","context":{"idset":"11380"}}
+{"timestamp":1711407913.4365525,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1711407913.4682052,"name":"offline","context":{"idset":"11378"}}
+{"timestamp":1711407913.4761467,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1711407913.5173357,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1711407913.551019,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1711407913.5869789,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1711407913.5953479,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1711407913.6386292,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1711407913.6450446,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1711407913.652442,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1711407913.6589417,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1711407913.6703188,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1711407913.6714857,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1711407913.6776602,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1711407913.7008877,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1711407913.7019365,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1711407913.70292,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1711407913.7038774,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1711407913.7528546,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1711407913.7538614,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1711407913.754812,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1711407913.755764,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1711407913.7902408,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1711407913.7919555,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1711407913.7936752,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1711407913.7954133,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1711407913.8602724,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1711407913.8620346,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1711407913.8637946,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1711407913.8655391,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1711407913.8672745,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1711407913.9207623,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1711407913.9397857,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1711407913.9498129,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1711407913.9507778,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1711407913.9517865,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1711407913.9527445,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1711407913.9536877,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1711407913.954627,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1711407913.9555902,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1711407913.9565585,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1711407913.9575243,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1711407913.9677026,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1711407913.9686775,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1711407913.9696441,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1711407913.9705732,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1711407913.9714992,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1711407913.9724562,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1711407913.9822738,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1711407914.0016885,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1711407914.0118585,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1711407914.0127764,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1711407914.0136888,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1711407914.0146484,"name":"offline","context":{"idset":"11439"}}
+{"timestamp":1711407914.0155852,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1711407914.0342116,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1711407914.0383258,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1711407914.0451663,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1711407914.0514522,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1711407914.1362507,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1711407914.3172209,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1711407915.9176786,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1711407916.0540369,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1711407916.154722,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1711407916.2132037,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1711407916.22508,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1711407916.3178086,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1711407916.4165208,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1711407916.6447103,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1711407916.7359431,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1711407916.7500956,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1711407916.8401382,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1711407916.900841,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1711407916.9124963,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1711407916.9763312,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1711407916.9780483,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1711407916.9871671,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1711407916.9940226,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1711407917.0777094,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1711407917.1649075,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1711407917.1720908,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1711407917.251183,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1711407917.2953651,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1711407917.3355157,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1711407917.3464105,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1711407917.3561742,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1711407917.3644433,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1711407917.3850064,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1711407917.4417379,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1711407917.4822676,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1711407917.5180027,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1711407917.525048,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1711407917.5800784,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1711407917.5815947,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1711407917.5878847,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1711407917.6399207,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1711407917.6408112,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1711407917.6940248,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1711407917.7862215,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1711407918.0869815,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1711407918.1524401,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1711407918.1602769,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1711407918.2209373,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1711407918.2549219,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1711407918.2678049,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1711407918.2771142,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1711407918.2897828,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1711407918.2972848,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1711407918.3871181,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1711407918.43413,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1711407918.444731,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1711407918.453428,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1711407918.4594781,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1711407918.4708886,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1711407918.4717774,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1711407918.5676026,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1711407918.6098166,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1711407918.6205649,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1711407918.6265259,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1711407918.656117,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1711407918.6779044,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1711407918.6931925,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1711407918.6943312,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1711407918.702389,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1711407918.7111533,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1711407918.7286212,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1711407918.7814043,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1711407918.7916281,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1711407918.79248,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1711407918.8441231,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1711407918.8567073,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1711407918.9066334,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1711407918.9074707,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1711407918.9082866,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1711407918.914387,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1711407918.9200485,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1711407918.926887,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1711407918.9475548,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1711407919.0220377,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1711407919.0229654,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1711407919.0238729,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1711407919.0247781,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1711407919.0260534,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1711407919.0329399,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1711407919.055752,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1711407919.0641892,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1711407919.1638494,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1711407919.1647701,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1711407919.1656408,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1711407919.166503,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1711407919.1904666,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1711407919.2570221,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1711407919.2578442,"name":"offline","context":{"idset":"11747"}}
+{"timestamp":1711407919.2586837,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1711407919.2595375,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1711407919.2603664,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1711407919.26122,"name":"offline","context":{"idset":"11759"}}
+{"timestamp":1711407919.2677515,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1711407919.2747278,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1711407919.2930951,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1711407919.3106408,"name":"offline","context":{"idset":"11750"}}
+{"timestamp":1711407919.4073775,"name":"offline","context":{"idset":"11748"}}
+{"timestamp":1711407919.4081919,"name":"offline","context":{"idset":"11752"}}
+{"timestamp":1711407919.4090052,"name":"offline","context":{"idset":"11755"}}
+{"timestamp":1711407919.4098113,"name":"offline","context":{"idset":"11749"}}
+{"timestamp":1711407919.4106154,"name":"offline","context":{"idset":"11754"}}
+{"timestamp":1711407919.4197936,"name":"offline","context":{"idset":"11751"}}
+{"timestamp":1711407919.4949782,"name":"offline","context":{"idset":"11757"}}
+{"timestamp":1711407919.49579,"name":"offline","context":{"idset":"11758"}}
+{"timestamp":1711407919.4965944,"name":"offline","context":{"idset":"11756"}}
+{"timestamp":1711407919.4974756,"name":"offline","context":{"idset":"11760"}}
+{"timestamp":1711407919.4982834,"name":"offline","context":{"idset":"11753"}}
+{"timestamp":1711407919.5013154,"name":"offline","context":{"idset":"11761"}}
+{"timestamp":1711407919.5805831,"name":"offline","context":{"idset":"11763"}}
+{"timestamp":1711407919.5831871,"name":"offline","context":{"idset":"11762"}}
+{"timestamp":1711407919.6831076,"name":"offline","context":{"idset":"11764"}}
+{"timestamp":1711408178.5247927,"name":"offline","context":{"idset":"11150"}}
+{"timestamp":1711408178.624217,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1711408356.9669778,"name":"drain","context":{"idset":"717","reason":"broker was unresponsive"}}
+{"timestamp":1711408382.96789,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1711408386.9673128,"name":"drain","context":{"idset":"727","reason":"broker was unresponsive"}}
+{"timestamp":1711408392.9664531,"name":"drain","context":{"idset":"731","reason":"broker was unresponsive"}}
+{"timestamp":1711408398.9665821,"name":"drain","context":{"idset":"729","reason":"broker was unresponsive"}}
+{"timestamp":1711408414.9684844,"name":"drain","context":{"idset":"721","reason":"broker was unresponsive"}}
+{"timestamp":1711408490.8689344,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1711408490.9666882,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1711408618.8669355,"name":"drain","context":{"idset":"713","reason":"broker was unresponsive"}}
+{"timestamp":1711408620.9668899,"name":"drain","context":{"idset":"709","reason":"broker was unresponsive"}}
+{"timestamp":1711408652.9666452,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1711408820.967556,"name":"drain","context":{"idset":"723","reason":"broker was unresponsive"}}
+{"timestamp":1711408924.9672146,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1711408968.9664094,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1711409098.9674611,"name":"drain","context":{"idset":"711","reason":"broker was unresponsive"}}
+{"timestamp":1711409100.9664223,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1711409132.9680476,"name":"drain","context":{"idset":"715","reason":"broker was unresponsive"}}
+{"timestamp":1711409186.966481,"name":"drain","context":{"idset":"725","reason":"broker was unresponsive"}}
+{"timestamp":1711409234.9663491,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1711409358.9656224,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1711409724.9680018,"name":"drain","context":{"idset":"732","reason":"broker was unresponsive"}}
+{"timestamp":1711409778.9655137,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1711409910.967391,"name":"drain","context":{"idset":"710","reason":"broker was unresponsive"}}
+{"timestamp":1711409922.96631,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1711410106.9664729,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1711410610.9660466,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1711410624.9664106,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1711410932.5020099,"name":"online","context":{"idset":"732"}}
+{"timestamp":1711411098.9658978,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1711411120.9668686,"name":"offline","context":{"idset":"11296"}}
+{"timestamp":1711411176.9675019,"name":"drain","context":{"idset":"719","reason":"broker was unresponsive"}}
+{"timestamp":1711411204.967303,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1711411242.96644,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1711411352.9673057,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1711411606.9678445,"name":"drain","context":{"idset":"712","reason":"broker was unresponsive"}}
+{"timestamp":1711411668.9673212,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1711411808.967082,"name":"drain","context":{"idset":"728","reason":"broker was unresponsive"}}
+{"timestamp":1711411874.9666126,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1711412254.9660211,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1711412524.9665248,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1711412638.9668939,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1711412720.9673684,"name":"drain","context":{"idset":"718","reason":"broker was unresponsive"}}
+{"timestamp":1711412782.8687725,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1711412782.9676855,"name":"drain","context":{"idset":"730","reason":"broker was unresponsive"}}
+{"timestamp":1711412816.9666514,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1711412846.9658864,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1711413056.9654508,"name":"drain","context":{"idset":"724","reason":"broker was unresponsive"}}
+{"timestamp":1711413078.9673193,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1711413084.9665074,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1711413119.2147803,"name":"online","context":{"idset":"732"}}
+{"timestamp":1711413158.9666805,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1711413259.4957604,"name":"online","context":{"idset":"728"}}
+{"timestamp":1711413264.6469223,"name":"online","context":{"idset":"712"}}
+{"timestamp":1711413266.9522231,"name":"online","context":{"idset":"11187"}}
+{"timestamp":1711413267.1660202,"name":"online","context":{"idset":"718"}}
+{"timestamp":1711413285.229969,"name":"online","context":{"idset":"730"}}
+{"timestamp":1711413596.9657536,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1711416172.8679979,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1711416172.9662242,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1711459580.3667469,"name":"online","context":{"idset":"809"}}
+{"timestamp":1711460364.8795934,"name":"drain","context":{"idset":"809","overwrite":0}}
+{"timestamp":1711460371.1431019,"name":"drain","context":{"idset":"810","overwrite":0}}
+{"timestamp":1711460384.6069784,"name":"offline","context":{"idset":"809"}}
+{"timestamp":1711461777.1299369,"name":"drain","context":{"idset":"789-796","overwrite":0}}
+{"timestamp":1711462355.7072415,"name":"online","context":{"idset":"869"}}
+{"timestamp":1711463310.2784338,"name":"online","context":{"idset":"870"}}
+{"timestamp":1711467020.428874,"name":"online","context":{"idset":"11168"}}
+{"timestamp":1711468480.9672196,"name":"drain","context":{"idset":"745","reason":"broker was unresponsive"}}
+{"timestamp":1711468542.9659345,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1711471512.6612787,"name":"online","context":{"idset":"809"}}
+{"timestamp":1711473645.2017839,"name":"online","context":{"idset":"11169"}}
+{"timestamp":1711474166.3998945,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1711474557.9183731,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1711475091.3763654,"name":"online","context":{"idset":"11167"}}
+{"timestamp":1711475465.7430615,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1711475830.7560318,"name":"online","context":{"idset":"11157"}}
+{"timestamp":1711476187.8017988,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1711479781.7078962,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1711481313.0768132,"name":"undrain","context":{"idset":"809"}}
+{"timestamp":1711481976.2077589,"name":"online","context":{"idset":"747"}}
+{"timestamp":1711485254.866616,"name":"drain","context":{"idset":"739","reason":"broker was unresponsive"}}
+{"timestamp":1711485256.8670223,"name":"drain","context":{"idset":"751","reason":"broker was unresponsive"}}
+{"timestamp":1711485260.9668798,"name":"drain","context":{"idset":"737","reason":"broker was unresponsive"}}
+{"timestamp":1711485262.9674547,"name":"drain","context":{"idset":"753","reason":"broker was unresponsive"}}
+{"timestamp":1711485264.9682837,"name":"drain","context":{"idset":"741","reason":"broker was unresponsive"}}
+{"timestamp":1711485268.9671292,"name":"drain","context":{"idset":"747","reason":"broker was unresponsive"}}
+{"timestamp":1711485272.9671066,"name":"drain","context":{"idset":"743","reason":"broker was unresponsive"}}
+{"timestamp":1711485314.9663484,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1711485348.9668539,"name":"drain","context":{"idset":"755","reason":"broker was unresponsive"}}
+{"timestamp":1711485352.9676051,"name":"drain","context":{"idset":"735","reason":"broker was unresponsive"}}
+{"timestamp":1711485354.9740343,"name":"drain","context":{"idset":"749","reason":"broker was unresponsive"}}
+{"timestamp":1711485404.9666295,"name":"drain","context":{"idset":"740","reason":"broker was unresponsive"}}
+{"timestamp":1711486000.9676955,"name":"drain","context":{"idset":"734","reason":"broker was unresponsive"}}
+{"timestamp":1711486062.9658523,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1711486170.9677472,"name":"drain","context":{"idset":"744","reason":"broker was unresponsive"}}
+{"timestamp":1711486210.9669952,"name":"drain","context":{"idset":"750","reason":"broker was unresponsive"}}
+{"timestamp":1711486226.9662178,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1711486252.9676309,"name":"drain","context":{"idset":"754","reason":"broker was unresponsive"}}
+{"timestamp":1711486276.9662235,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1711486308.9655349,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1711486342.8685734,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1711486472.428997,"name":"online","context":{"idset":"734"}}
+{"timestamp":1711486600.9662437,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1711486908.9673815,"name":"drain","context":{"idset":"742","reason":"broker was unresponsive"}}
+{"timestamp":1711486958.9672997,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711486964.9660981,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1711487096.966197,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1711487238.9656997,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1711487390.9656293,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1711487432.9659314,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1711487434.9660881,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1711487806.6537106,"name":"online","context":{"idset":"748"}}
+{"timestamp":1711487954.966521,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1711488184.9667988,"name":"drain","context":{"idset":"752","reason":"broker was unresponsive"}}
+{"timestamp":1711488242.9666128,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1711488510.9678004,"name":"drain","context":{"idset":"756","reason":"broker was unresponsive"}}
+{"timestamp":1711488578.9661462,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1711488668.9666712,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1711488768.9655547,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1711488872.1284454,"name":"online","context":{"idset":"744"}}
+{"timestamp":1711489178.4198384,"name":"online","context":{"idset":"734"}}
+{"timestamp":1711489248.9662826,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1711489454.4899852,"name":"online","context":{"idset":"742"}}
+{"timestamp":1711489712.9658885,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1711489956.9655786,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1711490552.8214061,"name":"online","context":{"idset":"744"}}
+{"timestamp":1711490658.9658115,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711491146.965421,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1711491198.0330884,"name":"online","context":{"idset":"742"}}
+{"timestamp":1711491636.4871819,"name":"online","context":{"idset":"748"}}
+{"timestamp":1711492178.083214,"name":"online","context":{"idset":"740"}}
+{"timestamp":1711492434.9655707,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1711492978.440742,"name":"online","context":{"idset":"734"}}
+{"timestamp":1711493038.9656069,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1711493579.1628041,"name":"online","context":{"idset":"744"}}
+{"timestamp":1711493714.9646575,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1711494216.9655938,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711494251.6124043,"name":"online","context":{"idset":"740"}}
+{"timestamp":1711494506.9655647,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1711494656.7326295,"name":"online","context":{"idset":"742"}}
+{"timestamp":1711494698.9669604,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1711495197.9819038,"name":"online","context":{"idset":"748"}}
+{"timestamp":1711495394.7515178,"name":"online","context":{"idset":"11184"}}
+{"timestamp":1711495399.8939109,"name":"online","context":{"idset":"734"}}
+{"timestamp":1711495964.6402583,"name":"online","context":{"idset":"11188"}}
+{"timestamp":1711496147.5891454,"name":"online","context":{"idset":"11186"}}
+{"timestamp":1711496195.4102504,"name":"online","context":{"idset":"11187"}}
+{"timestamp":1711496902.9672594,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711497246.9661179,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1711497374.9666057,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1711497445.7445054,"name":"online","context":{"idset":"742"}}
+{"timestamp":1711497631.8353779,"name":"online","context":{"idset":"11179"}}
+{"timestamp":1711497813.3760293,"name":"online","context":{"idset":"744"}}
+{"timestamp":1711497874.9664283,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1711497956.7183192,"name":"online","context":{"idset":"740"}}
+{"timestamp":1711498092.966965,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1711498422.4835629,"name":"online","context":{"idset":"748"}}
+{"timestamp":1711498651.1921585,"name":"online","context":{"idset":"734"}}
+{"timestamp":1711499638.9664085,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1711500048.9668565,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711500183.4188232,"name":"online","context":{"idset":"744"}}
+{"timestamp":1711500607.4831009,"name":"online","context":{"idset":"742"}}
+{"timestamp":1711500622.9675112,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1711500810.9662619,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1711501155.1565468,"name":"online","context":{"idset":"734"}}
+{"timestamp":1711501518.5438426,"name":"online","context":{"idset":"740"}}
+{"timestamp":1711501704.9665241,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1711501928.9665906,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1711502248.4583273,"name":"online","context":{"idset":"748"}}
+{"timestamp":1711502416.2559359,"name":"online","context":{"idset":"744"}}
+{"timestamp":1711503336.9658265,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711503406.9661996,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1711503802.6428437,"name":"online","context":{"idset":"742"}}
+{"timestamp":1711503901.7220819,"name":"online","context":{"idset":"734"}}
+{"timestamp":1711504192.9672291,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1711506096.9450383,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1711216745.6805143,"reason":"broker was unresponsive"},"2":{"timestamp":1711216745.6806085,"reason":"broker was unresponsive"},"3":{"timestamp":1711216745.680656,"reason":"broker was unresponsive"},"4":{"timestamp":1711216745.6807451,"reason":"broker was unresponsive"},"5":{"timestamp":1711216745.6808176,"reason":"broker was unresponsive"},"6":{"timestamp":1711216745.6808617,"reason":"broker was unresponsive"},"7":{"timestamp":1711216745.6809049,"reason":"broker was unresponsive"},"8":{"timestamp":1711216745.6809447,"reason":"broker was unresponsive"},"9":{"timestamp":1711216745.6809835,"reason":"broker was unresponsive"},"10":{"timestamp":1711216745.6810231,"reason":"broker was unresponsive"},"11":{"timestamp":1711216745.6811044,"reason":"broker was unresponsive"},"12":{"timestamp":1711216745.6811471,"reason":"broker was unresponsive"},"13":{"timestamp":1711216745.6811881,"reason":"broker was unresponsive"},"14":{"timestamp":1711216745.6812286,"reason":"broker was unresponsive"},"15":{"timestamp":1711216745.6812708,"reason":"broker was unresponsive"},"16":{"timestamp":1711216745.6813264,"reason":"broker was unresponsive"},"17":{"timestamp":1711216745.681371,"reason":"broker was unresponsive"},"18":{"timestamp":1711216745.6814148,"reason":"broker was unresponsive"},"19":{"timestamp":1711216745.681462,"reason":"broker was unresponsive"},"20":{"timestamp":1711216745.6833789,"reason":"broker was unresponsive"},"21":{"timestamp":1711216745.6835806,"reason":"broker was unresponsive"},"22":{"timestamp":1711216745.6837616,"reason":"broker was unresponsive"},"23":{"timestamp":1711216745.6838651,"reason":"broker was unresponsive"},"24":{"timestamp":1711216745.6840565,"reason":"broker was unresponsive"},"25":{"timestamp":1711216745.6841562,"reason":"broker was unresponsive"},"26":{"timestamp":1711216745.6842697,"reason":"broker was unresponsive"},"27":{"timestamp":1711216745.6843822,"reason":"broker was unresponsive"},"28":{"timestamp":1711216745.6844778,"reason":"broker was unresponsive"},"29":{"timestamp":1711216745.6845732,"reason":"broker was unresponsive"},"30":{"timestamp":1711216745.6846764,"reason":"broker was unresponsive"},"31":{"timestamp":1711216745.6847744,"reason":"broker was unresponsive"},"32":{"timestamp":1711216745.6848724,"reason":"broker was unresponsive"},"33":{"timestamp":1711216745.6850114,"reason":"broker was unresponsive"},"34":{"timestamp":1711216745.6851158,"reason":"broker was unresponsive"},"35":{"timestamp":1711216745.6852152,"reason":"broker was unresponsive"},"36":{"timestamp":1711216745.6853395,"reason":"broker was unresponsive"},"37":{"timestamp":1711216745.6854444,"reason":"broker was unresponsive"},"38":{"timestamp":1711216745.6855464,"reason":"broker was unresponsive"},"39":{"timestamp":1711216745.6856468,"reason":"broker was unresponsive"},"40":{"timestamp":1711216745.68575,"reason":"broker was unresponsive"},"41":{"timestamp":1711216745.6858568,"reason":"broker was unresponsive"},"42":{"timestamp":1711216745.6859612,"reason":"broker was unresponsive"},"43":{"timestamp":1711216745.6861026,"reason":"broker was unresponsive"},"44":{"timestamp":1711216745.686213,"reason":"broker was unresponsive"},"45":{"timestamp":1711216745.6863275,"reason":"broker was unresponsive"},"46":{"timestamp":1711216745.6864519,"reason":"broker was unresponsive"},"47":{"timestamp":1711216745.6865566,"reason":"broker was unresponsive"},"48":{"timestamp":1711216745.6866629,"reason":"broker was unresponsive"},"49":{"timestamp":1711216745.6867788,"reason":"broker was unresponsive"},"50":{"timestamp":1711216745.6868875,"reason":"broker was unresponsive"},"51":{"timestamp":1711216745.6869948,"reason":"broker was unresponsive"},"52":{"timestamp":1711216745.6871991,"reason":"broker was unresponsive"},"53":{"timestamp":1711216745.6874077,"reason":"broker was unresponsive"},"54":{"timestamp":1711216745.6875894,"reason":"broker was unresponsive"},"55":{"timestamp":1711216745.6877069,"reason":"broker was unresponsive"},"56":{"timestamp":1711216745.6878214,"reason":"broker was unresponsive"},"57":{"timestamp":1711216745.6879315,"reason":"broker was unresponsive"},"58":{"timestamp":1711216745.6880448,"reason":"broker was unresponsive"},"59":{"timestamp":1711216745.6881592,"reason":"broker was unresponsive"},"60":{"timestamp":1711216746.2583418,"reason":"broker was unresponsive"},"109":{"timestamp":1711165087.7736251,"reason":"broker was unresponsive"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"431":{"timestamp":1711096934.358676,"reason":"nodediag failed pci"},"709":{"timestamp":1711408620.9668899,"reason":"broker was unresponsive"},"710":{"timestamp":1711409910.967391,"reason":"broker was unresponsive"},"711":{"timestamp":1711409098.9674611,"reason":"broker was unresponsive"},"712":{"timestamp":1711411606.9678445,"reason":"broker was unresponsive"},"713":{"timestamp":1711408618.8669355,"reason":"broker was unresponsive"},"715":{"timestamp":1711409132.9680476,"reason":"broker was unresponsive"},"717":{"timestamp":1711408356.9669778,"reason":"broker was unresponsive"},"718":{"timestamp":1711412720.9673684,"reason":"broker was unresponsive"},"719":{"timestamp":1711411176.9675019,"reason":"broker was unresponsive"},"721":{"timestamp":1711408414.9684844,"reason":"broker was unresponsive"},"722":{"timestamp":1711395678.9683194,"reason":"broker was unresponsive"},"723":{"timestamp":1711408820.967556,"reason":"broker was unresponsive"},"724":{"timestamp":1711413056.9654508,"reason":"broker was unresponsive"},"725":{"timestamp":1711409186.966481,"reason":"broker was unresponsive"},"727":{"timestamp":1711408386.9673128,"reason":"broker was unresponsive"},"728":{"timestamp":1711411808.967082,"reason":"broker was unresponsive"},"729":{"timestamp":1711408398.9665821,"reason":"broker was unresponsive"},"730":{"timestamp":1711412782.9676855,"reason":"broker was unresponsive"},"731":{"timestamp":1711408392.9664531,"reason":"broker was unresponsive"},"732":{"timestamp":1711409724.9680018,"reason":"broker was unresponsive"},"734":{"timestamp":1711486000.9676955,"reason":"broker was unresponsive"},"735":{"timestamp":1711485352.9676051,"reason":"broker was unresponsive"},"737":{"timestamp":1711485260.9668798,"reason":"broker was unresponsive"},"739":{"timestamp":1711485254.866616,"reason":"broker was unresponsive"},"740":{"timestamp":1711485404.9666295,"reason":"broker was unresponsive"},"741":{"timestamp":1711485264.9682837,"reason":"broker was unresponsive"},"742":{"timestamp":1711486908.9673815,"reason":"broker was unresponsive"},"743":{"timestamp":1711485272.9671066,"reason":"broker was unresponsive"},"744":{"timestamp":1711486170.9677472,"reason":"broker was unresponsive"},"745":{"timestamp":1711468480.9672196,"reason":"broker was unresponsive"},"747":{"timestamp":1711485268.9671292,"reason":"broker was unresponsive"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"},"749":{"timestamp":1711485354.9740343,"reason":"broker was unresponsive"},"750":{"timestamp":1711486210.9669952,"reason":"broker was unresponsive"},"751":{"timestamp":1711485256.8670223,"reason":"broker was unresponsive"},"752":{"timestamp":1711488184.9667988,"reason":"broker was unresponsive"},"753":{"timestamp":1711485262.9674547,"reason":"broker was unresponsive"},"754":{"timestamp":1711486252.9676309,"reason":"broker was unresponsive"},"755":{"timestamp":1711485348.9668539,"reason":"broker was unresponsive"},"756":{"timestamp":1711488510.9678004,"reason":"broker was unresponsive"},"789-796":{"timestamp":1711461777.1299369,"reason":""},"810":{"timestamp":1711460371.1431019,"reason":""},"11125":{"timestamp":1711126273.9625854,"reason":"ama fail"},"11126":{"timestamp":1711126278.6191533,"reason":"ama fail"},"11127":{"timestamp":1711126696.680208,"reason":"ama fail"},"11179":{"timestamp":1711404572.9688854,"reason":"broker was unresponsive"},"11187":{"timestamp":1711401067.3461077,"reason":"reason=drain for Rabbit connection"},"11190":{"timestamp":1711399648.6946192,"reason":"bad partner"},"11213":{"timestamp":1711406762.8675106,"reason":"broker was unresponsive"},"11216":{"timestamp":1711406762.9678555,"reason":"broker was unresponsive"},"11295":{"timestamp":1711148027.6723082,"reason":"ama fail"},"11296":{"timestamp":1711148027.7718606,"reason":"ama fail"},"11317":{"timestamp":1711326792.8705738,"reason":"epilog failed for jobid fka2yuNfWCo"},"11342":{"timestamp":1711117435.6741276,"reason":"broker was unresponsive"},"11343":{"timestamp":1711393750.8690577,"reason":"broker was unresponsive"},"11345":{"timestamp":1711393750.8692429,"reason":"broker was unresponsive"},"11347":{"timestamp":1711393750.8693571,"reason":"broker was unresponsive"},"11150,11388":{"timestamp":1711405119.195823,"reason":"reason=Draining for partner Rabbit"},"11393":{"timestamp":1711401846.9683437,"reason":"broker was unresponsive"},"11394":{"timestamp":1711117417.7733912,"reason":"Rabbit link Failure"},"11448":{"timestamp":1711123241.6739299,"reason":"ama fail"},"11449":{"timestamp":1711123241.674036,"reason":"ama fail"},"11450":{"timestamp":1711123241.6741452,"reason":"ama fail"},"11453":{"timestamp":1711123247.6735187,"reason":"ama fail"},"11460":{"timestamp":1711123247.7819526,"reason":"ama fail"},"11501":{"timestamp":1711140105.6735849,"reason":"ama fail"},"11502":{"timestamp":1711140105.7732108,"reason":"ama fail"},"11149,11188-11189,11203-11204,11387,11488,11490,11593":{"timestamp":1711325310.3051066,"reason":"Rabbit link Failure"},"11346,11605":{"timestamp":1711326134.6213346,"reason":"GPU memory in use failing apdgpu.t"},"11128-11148,11151-11154,11156-11163,11165-11174,11177-11178,11180-11186,11191-11202,11205-11212,11214-11215,11217-11229,11231-11252,11285-11294,11297-11316,11318-11340,11344,11348,11350-11386,11389-11392,11395-11447,11451-11452,11454-11459,11461-11464,11466-11477,11479-11487,11489,11491-11500,11503-11508,11589-11592,11594-11599,11601-11604,11606-11636,11747-11764":{"timestamp":1711407860.45856,"reason":"FW/BIOS updates"}},"online":"","exclude":"0"}}
+{"timestamp":1711506096.9650342,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1711506133.8172729,"name":"online","context":{"idset":"0"}}
+{"timestamp":1711506136.0187039,"name":"online","context":{"idset":"11186,11188"}}
+{"timestamp":1711506136.1824903,"name":"online","context":{"idset":"109,11179,11184,11187"}}
+{"timestamp":1711506136.2931459,"name":"online","context":{"idset":"11230"}}
+{"timestamp":1711506136.606585,"name":"online","context":{"idset":"123,254,730,748"}}
+{"timestamp":1711506136.7082582,"name":"online","context":{"idset":"52,663"}}
+{"timestamp":1711506136.859648,"name":"online","context":{"idset":"8,45,718,728,732,869"}}
+{"timestamp":1711506136.9615815,"name":"online","context":{"idset":"39"}}
+{"timestamp":1711506137.0649996,"name":"online","context":{"idset":"128,136,141,143,152,154,164,172,181,189,196-197,199,217,220,227,230,234,240,243,290,320-321,323,329-330,337-338,340,342,352,356,360,367,371,400-401,403,423-424,432,435,439-440,445,477-478,489,499,502,507,511,518,520-521,523,529,540-542,549,582,585,590,604,609,612,614,616,618,626,650,652,658,683-684,687,694,698,707,809,870"}}
+{"timestamp":1711506137.1691153,"name":"online","context":{"idset":"124-127,129-135,137-140,142,144-151,153,155-163,165-167,169-171,173-175,177-180,182-188,190-195,198,200-216,218-219,221-226,229,231-233,235-239,241-242,244-247,259,274,280,287,293,299,310-316,318-319,322,324-328,331-336,339,341,343-345,347-351,353-355,357-359,361-366,368-370,372-381,383-399,402,404,406-422,425-430,433-434,436-438,441-443,446-454,456-458,461-476,479-480,482-488,490-498,500-501,503-506,508-510,512-517,519,524-528,530-539,543-548,550-556,558-581,583-584,586-589,591-597,599-603,605-607,610-611,613,615,617,620-625,627-633,635-644,646-649,651,653-657,659-661,664,666-667,669-677,679-682,685-686,688-693,695-697,699-706,708,733"}}
+{"timestamp":1711506137.2785926,"name":"online","context":{"idset":"18,22,37,49,59,168,176,228,248-250,252-253,255-256,268,270,282-283,294-295,298,305,309,317,346,382,405,444,455,459-460,481,522,557,598,608,619,634,645,662,665,678,714,716,720"}}
+{"timestamp":1711506137.3852196,"name":"online","context":{"idset":"11,15,257-258,261-263,265-266,271,275,277-278,284-286,288,292,296,300,303,306-307,734"}}
+{"timestamp":1711506137.4860222,"name":"online","context":{"idset":"251,260,264,267,269,272-273,276,279,281,289,291,297,301-302,304,308,668,726"}}
+{"timestamp":1711506137.7198665,"name":"online","context":{"idset":"1-7,9-10,12-14,16-17,19,21,23-36,38,41-44,46-48,50-51,53-58,60"}}
+{"timestamp":1711506137.8211083,"name":"online","context":{"idset":"20,40"}}
+{"timestamp":1711506138.0562258,"name":"online","context":{"idset":"740"}}
+{"timestamp":1711506138.5831938,"name":"online","context":{"idset":"744,11157,11164-11165"}}
+{"timestamp":1711506138.6954601,"name":"online","context":{"idset":"11158,11161,11169,11172"}}
+{"timestamp":1711506138.8544796,"name":"online","context":{"idset":"11160,11162-11163,11166-11167"}}
+{"timestamp":1711506138.9872489,"name":"online","context":{"idset":"11159,11168,11170-11171"}}
+{"timestamp":1711506141.1617801,"name":"online","context":{"idset":"742"}}
+{"timestamp":1711506254.2684541,"name":"drain","context":{"idset":"736","reason":"broker was unresponsive"}}
+{"timestamp":1711506258.2527435,"name":"drain","context":{"idset":"746","reason":"broker was unresponsive"}}
+{"timestamp":1711506302.2536957,"name":"drain","context":{"idset":"738","reason":"broker was unresponsive"}}
+{"timestamp":1711506388.2544255,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711506928.7801869,"name":"online","context":{"idset":"742"}}
+{"timestamp":1711507118.2529836,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1711507152.4430182,"name":"online","context":{"idset":"736"}}
+{"timestamp":1711507155.1200352,"name":"online","context":{"idset":"738"}}
+{"timestamp":1711507157.6246181,"name":"online","context":{"idset":"756"}}
+{"timestamp":1711507158.6616218,"name":"online","context":{"idset":"752"}}
+{"timestamp":1711507159.7356229,"name":"online","context":{"idset":"747"}}
+{"timestamp":1711507159.9565287,"name":"online","context":{"idset":"737,750"}}
+{"timestamp":1711507160.097111,"name":"online","context":{"idset":"741,11605"}}
+{"timestamp":1711507160.2327921,"name":"online","context":{"idset":"749,751,753,755"}}
+{"timestamp":1711507160.3677404,"name":"online","context":{"idset":"739"}}
+{"timestamp":1711507160.7171764,"name":"online","context":{"idset":"754"}}
+{"timestamp":1711507161.3737409,"name":"online","context":{"idset":"735"}}
+{"timestamp":1711507164.080219,"name":"online","context":{"idset":"748"}}
+{"timestamp":1711507165.8248181,"name":"online","context":{"idset":"743"}}
+{"timestamp":1711514850.2193763,"name":"online","context":{"idset":"11502"}}
+{"timestamp":1711515216.8958383,"name":"online","context":{"idset":"11189,11203"}}
+{"timestamp":1711515217.0129309,"name":"online","context":{"idset":"11204"}}
+{"timestamp":1711515217.3746839,"name":"online","context":{"idset":"11190"}}
+{"timestamp":1711515232.3871598,"name":"online","context":{"idset":"11191"}}
+{"timestamp":1711515232.5234027,"name":"online","context":{"idset":"11192,11195"}}
+{"timestamp":1711515232.8018811,"name":"online","context":{"idset":"11201"}}
+{"timestamp":1711515233.1086226,"name":"online","context":{"idset":"11193,11196"}}
+{"timestamp":1711515233.2484336,"name":"online","context":{"idset":"11194,11197-11198"}}
+{"timestamp":1711515233.4442577,"name":"online","context":{"idset":"11199-11200,11202"}}
+{"timestamp":1711515886.4852986,"name":"online","context":{"idset":"11317"}}
+{"timestamp":1711515982.3572645,"name":"online","context":{"idset":"11128,11138"}}
+{"timestamp":1711515982.6609943,"name":"online","context":{"idset":"11133"}}
+{"timestamp":1711515982.7934926,"name":"online","context":{"idset":"11134-11135,11389"}}
+{"timestamp":1711515982.9572868,"name":"online","context":{"idset":"11129,11136-11137,11139-11140,11383,11385,11390"}}
+{"timestamp":1711515983.3128107,"name":"online","context":{"idset":"11131,11333,11338"}}
+{"timestamp":1711515983.4197011,"name":"online","context":{"idset":"11335,11382"}}
+{"timestamp":1711515984.0467708,"name":"online","context":{"idset":"11334"}}
+{"timestamp":1711515984.2306113,"name":"online","context":{"idset":"11336"}}
+{"timestamp":1711515984.5298822,"name":"online","context":{"idset":"11214"}}
+{"timestamp":1711515984.9394803,"name":"online","context":{"idset":"11210-11211"}}
+{"timestamp":1711515985.6068814,"name":"online","context":{"idset":"11206,11209"}}
+{"timestamp":1711516620.157371,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1711516620.1629186,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1711516620.2540109,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1711516622.159555,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1711516622.1647887,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1711516622.1698933,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1711516622.1748478,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1711516622.2543945,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1711516624.1585052,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1711516624.1638985,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1711516624.168993,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1711516624.2538803,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1711516722.2534847,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1711516724.1578357,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1711516724.2550278,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1711516726.254431,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1711516728.2544413,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1711523843.021579,"name":"online","context":{"idset":"11337"}}
+{"timestamp":1711523843.1984251,"name":"online","context":{"idset":"11339-11340,11342"}}
+{"timestamp":1711523843.347769,"name":"online","context":{"idset":"11341"}}
+{"timestamp":1711523843.4960997,"name":"online","context":{"idset":"11344-11345,11348"}}
+{"timestamp":1711523843.6622517,"name":"online","context":{"idset":"11346-11347"}}
+{"timestamp":1711523843.7630754,"name":"online","context":{"idset":"11343"}}
+{"timestamp":1711525549.753552,"name":"online","context":{"idset":"11213"}}
+{"timestamp":1711525761.0426831,"name":"online","context":{"idset":"11216"}}
+{"timestamp":1711529541.8019686,"name":"online","context":{"idset":"11208"}}
+{"timestamp":1711529542.0389955,"name":"online","context":{"idset":"11205,11212"}}
+{"timestamp":1711529542.1462033,"name":"online","context":{"idset":"11207"}}
+{"timestamp":1711529542.5526965,"name":"online","context":{"idset":"11215,11217-11218,11220"}}
+{"timestamp":1711529542.6898851,"name":"online","context":{"idset":"11219"}}
+{"timestamp":1711546880.2932708,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1711547318.2542987,"name":"drain","context":{"idset":"11164","reason":"broker was unresponsive"}}
+{"timestamp":1711547346.7946427,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1711547384.1570749,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1711547384.1618512,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1711547384.2535505,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1711547386.157901,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1711547386.1622419,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1711547386.2545965,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1711547388.1587076,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1711547388.1627231,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1711547388.166702,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1711547388.1705058,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1711547388.2536342,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1711547392.2526884,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1711547394.1624355,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1711547394.1697977,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1711547394.1776192,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1711547394.2863233,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1711547396.1585023,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1711547396.1626787,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1711547396.1661384,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1711547396.2537773,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1711547398.1573617,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1711547398.2537806,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1711547436.2547872,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1711547616.1653366,"name":"drain","context":{"idset":"11230","reason":"broker was unresponsive"}}
+{"timestamp":1711547616.1661725,"name":"drain","context":{"idset":"11341","reason":"broker was unresponsive"}}
+{"timestamp":1711547680.1884511,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1711547680.193301,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1711547680.1981807,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1711547680.2031341,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1711547680.2079349,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1711547680.2124734,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1711547680.2171826,"name":"offline","context":{"idset":"11317"}}
+{"timestamp":1711547680.2217,"name":"offline","context":{"idset":"11333"}}
+{"timestamp":1711547680.2276945,"name":"offline","context":{"idset":"11334"}}
+{"timestamp":1711547680.2556784,"name":"offline","context":{"idset":"11336"}}
+{"timestamp":1711547682.1835537,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1711547682.1877451,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1711547682.1934526,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1711547682.1975834,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1711547682.2015367,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1711547682.2056437,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1711547682.2098198,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1711547682.2325933,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1711547682.2378647,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1711547682.252368,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1711547682.2533398,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1711547682.254283,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1711547682.2592757,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1711547682.2720141,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1711547682.2762723,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1711547682.2867141,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1711547682.3276091,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1711547682.3542004,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1711547682.3551769,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1711547682.3561466,"name":"offline","context":{"idset":"11335"}}
+{"timestamp":1711547682.3571095,"name":"offline","context":{"idset":"11337"}}
+{"timestamp":1711547682.3580885,"name":"offline","context":{"idset":"11338"}}
+{"timestamp":1711547682.3697324,"name":"offline","context":{"idset":"11339"}}
+{"timestamp":1711547682.4075274,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1711547682.4085033,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1711547682.4094877,"name":"offline","context":{"idset":"11342"}}
+{"timestamp":1711547682.410471,"name":"offline","context":{"idset":"11343"}}
+{"timestamp":1711547682.4114373,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1711547682.4228945,"name":"offline","context":{"idset":"11345"}}
+{"timestamp":1711547682.4554942,"name":"offline","context":{"idset":"11346"}}
+{"timestamp":1711547682.4779966,"name":"offline","context":{"idset":"11347"}}
+{"timestamp":1711547682.478893,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1711547682.4798329,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1711547682.4917357,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1711547698.2546904,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1711547700.1559672,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1711547700.254189,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1711552107.76998,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1711552107.7856216,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1711552107.802561,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1711552107.8184135,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1711552107.8241014,"name":"offline","context":{"idset":"734"}}
+{"timestamp":1711552107.8287635,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1711552107.8401124,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1711552107.8405886,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1711552107.8442435,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1711552107.8446617,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1711552107.8498998,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1711552107.86744,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1711552107.8678918,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1711552107.8729632,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1711552107.8768868,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1711552107.8802881,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1711552107.8837819,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1711552107.9069908,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1711552107.9074097,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1711552107.9077957,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711552107.9081807,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1711552107.9146276,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1711552107.9180417,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1711552107.9608886,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1711552107.9612813,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1711552107.9616702,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1711552108.0173087,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1711552108.0337148,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1711552108.0579815,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1711552108.0969899,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1711554000.1541724,"name":"online","context":{"idset":"726"}}
+{"timestamp":1711554000.2548325,"name":"online","context":{"idset":"713"}}
+{"timestamp":1711554000.4269345,"name":"online","context":{"idset":"711"}}
+{"timestamp":1711554000.6125908,"name":"online","context":{"idset":"715"}}
+{"timestamp":1711554000.8584371,"name":"online","context":{"idset":"722,730,742"}}
+{"timestamp":1711554001.0549567,"name":"online","context":{"idset":"723,749"}}
+{"timestamp":1711554001.2167306,"name":"online","context":{"idset":"716,719,729,731,738,752,756"}}
+{"timestamp":1711554001.34658,"name":"online","context":{"idset":"709-710,717"}}
+{"timestamp":1711554001.462502,"name":"online","context":{"idset":"740"}}
+{"timestamp":1711554001.5671463,"name":"online","context":{"idset":"733,736,741,754"}}
+{"timestamp":1711554001.707829,"name":"online","context":{"idset":"712,748"}}
+{"timestamp":1711554001.8090534,"name":"online","context":{"idset":"718,721,724-725,743"}}
+{"timestamp":1711554001.9780364,"name":"online","context":{"idset":"746,753"}}
+{"timestamp":1711554002.4919088,"name":"online","context":{"idset":"745,755"}}
+{"timestamp":1711554002.6409004,"name":"online","context":{"idset":"739,744"}}
+{"timestamp":1711554002.8144836,"name":"online","context":{"idset":"714,720"}}
+{"timestamp":1711556879.3685722,"name":"online","context":{"idset":"11340"}}
+{"timestamp":1711556879.5156772,"name":"online","context":{"idset":"11339"}}
+{"timestamp":1711558153.7393165,"name":"online","context":{"idset":"11213"}}
+{"timestamp":1711558162.6973076,"name":"online","context":{"idset":"11208"}}
+{"timestamp":1711558671.8405721,"name":"online","context":{"idset":"11715-11716"}}
+{"timestamp":1711559927.802604,"name":"offline","context":{"idset":"448"}}
+{"timestamp":1711559927.8451362,"name":"offline","context":{"idset":"445"}}
+{"timestamp":1711559927.8578608,"name":"offline","context":{"idset":"450"}}
+{"timestamp":1711559927.8643847,"name":"offline","context":{"idset":"452"}}
+{"timestamp":1711559927.8868492,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1711559927.9024856,"name":"offline","context":{"idset":"462"}}
+{"timestamp":1711559927.9033659,"name":"offline","context":{"idset":"447"}}
+{"timestamp":1711559927.9041402,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1711559927.9338615,"name":"offline","context":{"idset":"463"}}
+{"timestamp":1711559927.9346304,"name":"offline","context":{"idset":"449"}}
+{"timestamp":1711559927.9353914,"name":"offline","context":{"idset":"464"}}
+{"timestamp":1711559927.9361367,"name":"offline","context":{"idset":"467"}}
+{"timestamp":1711559927.9369996,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1711559927.9633269,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1711559927.9642127,"name":"offline","context":{"idset":"451"}}
+{"timestamp":1711559927.9650924,"name":"offline","context":{"idset":"455"}}
+{"timestamp":1711559927.965955,"name":"offline","context":{"idset":"457"}}
+{"timestamp":1711559927.9669142,"name":"offline","context":{"idset":"465"}}
+{"timestamp":1711559927.9709921,"name":"offline","context":{"idset":"466"}}
+{"timestamp":1711559927.9801223,"name":"offline","context":{"idset":"468"}}
+{"timestamp":1711559927.980978,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1711559927.9841814,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1711559928.0071459,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1711559928.0195887,"name":"offline","context":{"idset":"549"}}
+{"timestamp":1711559928.0203934,"name":"offline","context":{"idset":"446"}}
+{"timestamp":1711559928.0213375,"name":"offline","context":{"idset":"456"}}
+{"timestamp":1711559928.022372,"name":"offline","context":{"idset":"458"}}
+{"timestamp":1711559928.0231771,"name":"offline","context":{"idset":"459"}}
+{"timestamp":1711559928.0239692,"name":"offline","context":{"idset":"460"}}
+{"timestamp":1711559928.0248168,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1711559928.025815,"name":"offline","context":{"idset":"550"}}
+{"timestamp":1711559928.0333138,"name":"offline","context":{"idset":"552"}}
+{"timestamp":1711559928.0386949,"name":"offline","context":{"idset":"554"}}
+{"timestamp":1711559928.1046529,"name":"offline","context":{"idset":"559"}}
+{"timestamp":1711559928.1054499,"name":"offline","context":{"idset":"453"}}
+{"timestamp":1711559928.1063104,"name":"offline","context":{"idset":"454"}}
+{"timestamp":1711559928.1070876,"name":"offline","context":{"idset":"553"}}
+{"timestamp":1711559928.107861,"name":"offline","context":{"idset":"555"}}
+{"timestamp":1711559928.10867,"name":"offline","context":{"idset":"558"}}
+{"timestamp":1711559928.1097705,"name":"offline","context":{"idset":"560"}}
+{"timestamp":1711559928.211277,"name":"offline","context":{"idset":"564"}}
+{"timestamp":1711559928.2341588,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1711559928.2577825,"name":"offline","context":{"idset":"551"}}
+{"timestamp":1711559928.2819197,"name":"offline","context":{"idset":"557"}}
+{"timestamp":1711559928.3358028,"name":"offline","context":{"idset":"562"}}
+{"timestamp":1711559928.3366988,"name":"offline","context":{"idset":"561"}}
+{"timestamp":1711559928.4071736,"name":"offline","context":{"idset":"563"}}
+{"timestamp":1711559928.4416049,"name":"offline","context":{"idset":"556"}}
+{"timestamp":1711560189.6111379,"name":"drain","context":{"idset":"445-468,541-564","reason":"CDU work TB","overwrite":0}}
+{"timestamp":1711561031.4380157,"name":"online","context":{"idset":"11216"}}
+{"timestamp":1711561738.6809998,"name":"online","context":{"idset":"11679"}}
+{"timestamp":1711562258.0263138,"name":"drain","context":{"idset":"809","reason":"faaland-lustre-test","overwrite":0}}
+{"timestamp":1711562897.4001534,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1711563210.156266,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1711563366.2710364,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1711564709.7804377,"name":"drain","context":{"idset":"709-716","reason":"Cabinet power work needed -jrg","overwrite":1}}
+{"timestamp":1711564754.0607774,"name":"offline","context":{"idset":"709"}}
+{"timestamp":1711564754.065975,"name":"offline","context":{"idset":"712"}}
+{"timestamp":1711564754.0696533,"name":"offline","context":{"idset":"711"}}
+{"timestamp":1711564754.0741539,"name":"offline","context":{"idset":"714"}}
+{"timestamp":1711564754.0789814,"name":"offline","context":{"idset":"713"}}
+{"timestamp":1711564754.0825586,"name":"offline","context":{"idset":"716"}}
+{"timestamp":1711564754.1795156,"name":"offline","context":{"idset":"710"}}
+{"timestamp":1711564754.2337692,"name":"offline","context":{"idset":"715"}}
+{"timestamp":1711564941.680872,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1711564941.7809265,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1711564941.8921366,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1711572322.7760353,"name":"undrain","context":{"idset":"809"}}
+{"timestamp":1711572695.5852365,"name":"offline","context":{"idset":"11679"}}
+{"timestamp":1711574282.1579652,"name":"offline","context":{"idset":"11339"}}
+{"timestamp":1711574282.2552657,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1711575776.2527885,"name":"offline","context":{"idset":"746"}}
+{"timestamp":1711576490.2548447,"name":"drain","context":{"idset":"348","reason":"broker was unresponsive"}}
+{"timestamp":1711576592.2540176,"name":"offline","context":{"idset":"348"}}
+{"timestamp":1711576602.207341,"name":"offline","context":{"idset":"869"}}
+{"timestamp":1711576713.5907419,"name":"drain","context":{"idset":"870","reason":"epilog failed for jobid fm4cUUWjqfd","overwrite":0}}
+{"timestamp":1711576713.6899419,"name":"offline","context":{"idset":"870"}}
+{"timestamp":1711579917.7431555,"name":"drain","context":{"idset":"720","reason":"Rebooting to fix blades - JRG","overwrite":0}}
+{"timestamp":1711579934.6107867,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1711580820.6064806,"name":"online","context":{"idset":"10428"}}
+{"timestamp":1711580910.6101434,"name":"online","context":{"idset":"10425"}}
+{"timestamp":1711580910.7548978,"name":"online","context":{"idset":"10421"}}
+{"timestamp":1711580910.9134994,"name":"online","context":{"idset":"10436"}}
+{"timestamp":1711580911.1137772,"name":"online","context":{"idset":"10431-10432"}}
+{"timestamp":1711580911.3695276,"name":"online","context":{"idset":"10433"}}
+{"timestamp":1711581375.5704784,"name":"online","context":{"idset":"720"}}
+{"timestamp":1711581390.9532583,"name":"undrain","context":{"idset":"720"}}
+{"timestamp":1711583800.3683445,"name":"online","context":{"idset":"10100"}}
+{"timestamp":1711583983.6469295,"name":"online","context":{"idset":"10099"}}
+{"timestamp":1711584134.7613592,"name":"online","context":{"idset":"10098"}}
+{"timestamp":1711584169.8392792,"name":"online","context":{"idset":"10097"}}
+{"timestamp":1711584219.0157764,"name":"online","context":{"idset":"10096"}}
+{"timestamp":1711584242.9422877,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1711584416.5853815,"name":"online","context":{"idset":"10093"}}
+{"timestamp":1711584422.2367082,"name":"online","context":{"idset":"10094"}}
+{"timestamp":1711585269.8131826,"name":"online","context":{"idset":"10092"}}
+{"timestamp":1711585271.5280063,"name":"online","context":{"idset":"10091"}}
+{"timestamp":1711585746.2665091,"name":"online","context":{"idset":"10090"}}
+{"timestamp":1711585830.3764477,"name":"online","context":{"idset":"10088"}}
+{"timestamp":1711585832.3375561,"name":"online","context":{"idset":"10087"}}
+{"timestamp":1711585978.3326135,"name":"online","context":{"idset":"740"}}
+{"timestamp":1711585984.7540858,"name":"online","context":{"idset":"10089"}}
+{"timestamp":1711586003.3401599,"name":"undrain","context":{"idset":"740"}}
+{"timestamp":1711586075.6410885,"name":"online","context":{"idset":"10085"}}
+{"timestamp":1711586077.8477638,"name":"online","context":{"idset":"10086"}}
+{"timestamp":1711586779.6185637,"name":"offline","context":{"idset":"645"}}
+{"timestamp":1711586857.201689,"name":"drain","context":{"idset":"646","reason":"shutting down to test rebooting blades without rabbits - wendy","overwrite":0}}
+{"timestamp":1711586974.5105598,"name":"offline","context":{"idset":"646"}}
+{"timestamp":1711587136.0002518,"name":"drain","context":{"idset":"647","reason":"shutting down to test rebooting blades without rabbits - wendy","overwrite":0}}
+{"timestamp":1711587148.3273411,"name":"offline","context":{"idset":"647"}}
+{"timestamp":1711587197.7047591,"name":"drain","context":{"idset":"648","reason":"shutting down to test rebooting blades without rabbits - wendy","overwrite":0}}
+{"timestamp":1711587204.8627186,"name":"offline","context":{"idset":"648"}}
+{"timestamp":1711587211.9606316,"name":"drain","context":{"idset":"649","reason":"shutting down to test rebooting blades without rabbits - wendy","overwrite":0}}
+{"timestamp":1711587216.0661559,"name":"offline","context":{"idset":"649"}}
+{"timestamp":1711587232.1780922,"name":"drain","context":{"idset":"650","reason":"shutting down to test rebooting blades without rabbits - wendy","overwrite":0}}
+{"timestamp":1711587236.9471939,"name":"offline","context":{"idset":"650"}}
+{"timestamp":1711587244.2040467,"name":"drain","context":{"idset":"651","reason":"shutting down to test rebooting blades without rabbits - wendy","overwrite":0}}
+{"timestamp":1711587248.4449039,"name":"offline","context":{"idset":"651"}}
+{"timestamp":1711587255.7559283,"name":"drain","context":{"idset":"652","reason":"shutting down to test rebooting blades without rabbits - wendy","overwrite":0}}
+{"timestamp":1711587260.5475185,"name":"offline","context":{"idset":"652"}}
+{"timestamp":1711589508.2545855,"name":"drain","context":{"idset":"11715","reason":"broker was unresponsive"}}
+{"timestamp":1711589520.2540197,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1711589642.4670117,"name":"online","context":{"idset":"752"}}
+{"timestamp":1711589660.2541702,"name":"drain","context":{"idset":"11716","reason":"broker was unresponsive"}}
+{"timestamp":1711589674.2546487,"name":"offline","context":{"idset":"11715"}}
+{"timestamp":1711589850.254847,"name":"offline","context":{"idset":"11716"}}
+{"timestamp":1711590030.1238039,"name":"drain","context":{"idset":"11715-11716","reason":"epilog failed for jobid fm6fejYRcuV","overwrite":0}}
+{"timestamp":1711591134.2537625,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1711591264.2536607,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1711591460.254168,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1711591630.2554383,"name":"drain","context":{"idset":"10096","reason":"broker was unresponsive"}}
+{"timestamp":1711591694.2549069,"name":"offline","context":{"idset":"10096"}}
+{"timestamp":1711591750.6331117,"name":"online","context":{"idset":"748"}}
+{"timestamp":1711591796.4583309,"name":"online","context":{"idset":"10083"}}
+{"timestamp":1711591798.7639241,"name":"online","context":{"idset":"10084"}}
+{"timestamp":1711591810.2558079,"name":"online","context":{"idset":"11493"}}
+{"timestamp":1711591842.2540739,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711591867.1989987,"name":"online","context":{"idset":"11494"}}
+{"timestamp":1711591882.2531021,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1711591903.107955,"name":"online","context":{"idset":"11495"}}
+{"timestamp":1711591925.8134308,"name":"online","context":{"idset":"11496"}}
+{"timestamp":1711592209.4111493,"name":"online","context":{"idset":"742"}}
+{"timestamp":1711592302.2542377,"name":"drain","context":{"idset":"10083","reason":"broker was unresponsive"}}
+{"timestamp":1711592366.2544851,"name":"offline","context":{"idset":"10083"}}
+{"timestamp":1711592453.8429365,"name":"online","context":{"idset":"752"}}
+{"timestamp":1711592480.2538371,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1711592574.6011972,"name":"online","context":{"idset":"10081"}}
+{"timestamp":1711592576.2549477,"name":"drain","context":{"idset":"10421","reason":"broker was unresponsive"}}
+{"timestamp":1711592576.9828663,"name":"online","context":{"idset":"10082"}}
+{"timestamp":1711592578.2555511,"name":"drain","context":{"idset":"10425","reason":"broker was unresponsive"}}
+{"timestamp":1711592580.1550264,"name":"drain","context":{"idset":"10428","reason":"broker was unresponsive"}}
+{"timestamp":1711592580.2556479,"name":"drain","context":{"idset":"10433","reason":"broker was unresponsive"}}
+{"timestamp":1711592582.1546719,"name":"drain","context":{"idset":"10431","reason":"broker was unresponsive"}}
+{"timestamp":1711592582.1547906,"name":"drain","context":{"idset":"10432","reason":"broker was unresponsive"}}
+{"timestamp":1711592582.2541113,"name":"drain","context":{"idset":"10436","reason":"broker was unresponsive"}}
+{"timestamp":1711592622.2542515,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1711592640.2536836,"name":"offline","context":{"idset":"10421"}}
+{"timestamp":1711592642.1577966,"name":"offline","context":{"idset":"10425"}}
+{"timestamp":1711592642.2548974,"name":"offline","context":{"idset":"10428"}}
+{"timestamp":1711592644.1563308,"name":"offline","context":{"idset":"10431"}}
+{"timestamp":1711592644.2538464,"name":"offline","context":{"idset":"10432"}}
+{"timestamp":1711592646.1579771,"name":"offline","context":{"idset":"10433"}}
+{"timestamp":1711592646.2550931,"name":"offline","context":{"idset":"10436"}}
+{"timestamp":1711592883.0892525,"name":"online","context":{"idset":"117"}}
+{"timestamp":1711592883.4005618,"name":"online","context":{"idset":"62,66-67,72,78,94"}}
+{"timestamp":1711592883.5018482,"name":"online","context":{"idset":"61,63,65,69-71,73-76,79,81-90,92-93"}}
+{"timestamp":1711592883.6117992,"name":"online","context":{"idset":"64,68,77,80,91,95-108,110-116,119-120,122"}}
+{"timestamp":1711592883.7442615,"name":"online","context":{"idset":"118"}}
+{"timestamp":1711592887.0303607,"name":"online","context":{"idset":"258,270"}}
+{"timestamp":1711592915.0918593,"name":"online","context":{"idset":"10080"}}
+{"timestamp":1711592918.1285887,"name":"online","context":{"idset":"10078"}}
+{"timestamp":1711592920.5116122,"name":"online","context":{"idset":"10077"}}
+{"timestamp":1711592936.1811764,"name":"online","context":{"idset":"743"}}
+{"timestamp":1711592936.5975587,"name":"online","context":{"idset":"749"}}
+{"timestamp":1711592936.6987894,"name":"online","context":{"idset":"739"}}
+{"timestamp":1711592941.2115734,"name":"online","context":{"idset":"11715"}}
+{"timestamp":1711592943.566869,"name":"online","context":{"idset":"744"}}
+{"timestamp":1711593030.7261975,"name":"online","context":{"idset":"722,727-728,732,735,737,747,750-751"}}
+{"timestamp":1711593077.7462919,"name":"online","context":{"idset":"10076"}}
+{"timestamp":1711593211.6404903,"name":"undrain","context":{"idset":"109,717-719,721-725,727-732,735-739,741-745,747,749-756"}}
+{"timestamp":1711593222.2542431,"name":"drain","context":{"idset":"725","reason":"broker was unresponsive"}}
+{"timestamp":1711593286.2534502,"name":"offline","context":{"idset":"725"}}
+{"timestamp":1711593309.9014494,"name":"online","context":{"idset":"10074"}}
+{"timestamp":1711593312.2549782,"name":"drain","context":{"idset":"726","reason":"broker was unresponsive"}}
+{"timestamp":1711593312.5357049,"name":"online","context":{"idset":"10073"}}
+{"timestamp":1711593314.6767268,"name":"online","context":{"idset":"10072"}}
+{"timestamp":1711593317.77953,"name":"online","context":{"idset":"10071"}}
+{"timestamp":1711593320.7058425,"name":"online","context":{"idset":"10070"}}
+{"timestamp":1711593322.7210696,"name":"online","context":{"idset":"10069"}}
+{"timestamp":1711593332.2552569,"name":"drain","context":{"idset":"727","reason":"broker was unresponsive"}}
+{"timestamp":1711593356.2547033,"name":"drain","context":{"idset":"728","reason":"broker was unresponsive"}}
+{"timestamp":1711593374.2541909,"name":"offline","context":{"idset":"726"}}
+{"timestamp":1711593396.2544746,"name":"offline","context":{"idset":"727"}}
+{"timestamp":1711593414.254441,"name":"drain","context":{"idset":"729","reason":"broker was unresponsive"}}
+{"timestamp":1711593418.25544,"name":"offline","context":{"idset":"728"}}
+{"timestamp":1711593462.2552798,"name":"drain","context":{"idset":"730","reason":"broker was unresponsive"}}
+{"timestamp":1711593476.2547545,"name":"offline","context":{"idset":"729"}}
+{"timestamp":1711593510.2550235,"name":"drain","context":{"idset":"731","reason":"broker was unresponsive"}}
+{"timestamp":1711593526.2546184,"name":"offline","context":{"idset":"730"}}
+{"timestamp":1711593530.2558599,"name":"drain","context":{"idset":"732","reason":"broker was unresponsive"}}
+{"timestamp":1711593574.2545404,"name":"offline","context":{"idset":"731"}}
+{"timestamp":1711593596.2546251,"name":"offline","context":{"idset":"732"}}
+{"timestamp":1711594128.1822991,"name":"online","context":{"idset":"10079"}}
+{"timestamp":1711595711.1683691,"name":"offline","context":{"idset":"10084"}}
+{"timestamp":1711596358.1753509,"name":"drain","context":{"idset":"717","reason":"rebooting to fix blades - kpn ","overwrite":0}}
+{"timestamp":1711596380.3398762,"name":"offline","context":{"idset":"717"}}
+{"timestamp":1711596521.3173139,"name":"online","context":{"idset":"10397"}}
+{"timestamp":1711596521.4691985,"name":"online","context":{"idset":"10389,10398"}}
+{"timestamp":1711596522.0051486,"name":"online","context":{"idset":"10439"}}
+{"timestamp":1711596522.1726267,"name":"online","context":{"idset":"10401,10450"}}
+{"timestamp":1711596522.3929055,"name":"online","context":{"idset":"10360,10402"}}
+{"timestamp":1711596522.7105787,"name":"online","context":{"idset":"10447,10458"}}
+{"timestamp":1711596522.8526647,"name":"online","context":{"idset":"10385,10444"}}
+{"timestamp":1711596523.0095782,"name":"online","context":{"idset":"10441-10442"}}
+{"timestamp":1711596523.1789656,"name":"online","context":{"idset":"10406,10417,10451"}}
+{"timestamp":1711596523.3147335,"name":"online","context":{"idset":"10410,10438,10440,10448,10462"}}
+{"timestamp":1711596523.5745196,"name":"online","context":{"idset":"10359,10416,10452"}}
+{"timestamp":1711596523.6926823,"name":"online","context":{"idset":"10368,10407,10419,10445,10464,10467"}}
+{"timestamp":1711596523.8238468,"name":"online","context":{"idset":"10361,10365,10372"}}
+{"timestamp":1711596524.0066729,"name":"online","context":{"idset":"10463"}}
+{"timestamp":1711596524.4173868,"name":"online","context":{"idset":"10380-10381,10384"}}
+{"timestamp":1711596524.8746808,"name":"online","context":{"idset":"10375,10377,10379"}}
+{"timestamp":1711596525.015204,"name":"online","context":{"idset":"10374"}}
+{"timestamp":1711596584.4448881,"name":"online","context":{"idset":"11386"}}
+{"timestamp":1711596584.6297131,"name":"online","context":{"idset":"11383,11392"}}
+{"timestamp":1711596584.827395,"name":"online","context":{"idset":"11382,11393"}}
+{"timestamp":1711596584.9419954,"name":"online","context":{"idset":"11389,11391"}}
+{"timestamp":1711596585.2259903,"name":"online","context":{"idset":"11384"}}
+{"timestamp":1711596594.304997,"name":"online","context":{"idset":"11401"}}
+{"timestamp":1711596594.589201,"name":"online","context":{"idset":"11400,11402"}}
+{"timestamp":1711596594.9261796,"name":"online","context":{"idset":"11403"}}
+{"timestamp":1711596595.6829448,"name":"online","context":{"idset":"11408-11409"}}
+{"timestamp":1711596595.7941904,"name":"online","context":{"idset":"11411"}}
+{"timestamp":1711596596.0805502,"name":"online","context":{"idset":"11413"}}
+{"timestamp":1711596603.399277,"name":"online","context":{"idset":"11414"}}
+{"timestamp":1711596604.2411125,"name":"online","context":{"idset":"11415"}}
+{"timestamp":1711596605.4221933,"name":"online","context":{"idset":"11422"}}
+{"timestamp":1711596605.9212329,"name":"online","context":{"idset":"11417,11420"}}
+{"timestamp":1711596606.1026747,"name":"online","context":{"idset":"11419,11423"}}
+{"timestamp":1711596606.2729864,"name":"online","context":{"idset":"11421"}}
+{"timestamp":1711596607.081013,"name":"online","context":{"idset":"11428"}}
+{"timestamp":1711596613.1295607,"name":"online","context":{"idset":"11430"}}
+{"timestamp":1711596616.1410048,"name":"online","context":{"idset":"11431"}}
+{"timestamp":1711596616.9191568,"name":"online","context":{"idset":"11436"}}
+{"timestamp":1711596619.0766995,"name":"online","context":{"idset":"11437-11438"}}
+{"timestamp":1711596619.4710586,"name":"online","context":{"idset":"11439,11441-11443"}}
+{"timestamp":1711596619.8608513,"name":"online","context":{"idset":"11444"}}
+{"timestamp":1711596620.6667902,"name":"online","context":{"idset":"11446"}}
+{"timestamp":1711596625.7794161,"name":"online","context":{"idset":"11449-11450"}}
+{"timestamp":1711596625.9452991,"name":"online","context":{"idset":"11447"}}
+{"timestamp":1711596626.8577771,"name":"online","context":{"idset":"11452"}}
+{"timestamp":1711596628.5635378,"name":"online","context":{"idset":"11459"}}
+{"timestamp":1711596628.8793733,"name":"online","context":{"idset":"11460"}}
+{"timestamp":1711596631.0899382,"name":"online","context":{"idset":"11462"}}
+{"timestamp":1711596635.1978824,"name":"online","context":{"idset":"11465"}}
+{"timestamp":1711596636.8141732,"name":"online","context":{"idset":"11467"}}
+{"timestamp":1711596637.6129215,"name":"online","context":{"idset":"11469"}}
+{"timestamp":1711596639.7267838,"name":"online","context":{"idset":"11472"}}
+{"timestamp":1711596640.0430319,"name":"online","context":{"idset":"11470,11473"}}
+{"timestamp":1711596640.2698762,"name":"online","context":{"idset":"11474"}}
+{"timestamp":1711596640.9262874,"name":"online","context":{"idset":"11475"}}
+{"timestamp":1711596641.0614395,"name":"online","context":{"idset":"11476"}}
+{"timestamp":1711596642.6439598,"name":"online","context":{"idset":"11479"}}
+{"timestamp":1711596645.1159062,"name":"online","context":{"idset":"11480"}}
+{"timestamp":1711596646.6560361,"name":"online","context":{"idset":"11481"}}
+{"timestamp":1711596647.4463737,"name":"online","context":{"idset":"11483"}}
+{"timestamp":1711596648.4995625,"name":"online","context":{"idset":"11484"}}
+{"timestamp":1711596648.865473,"name":"online","context":{"idset":"11485"}}
+{"timestamp":1711596650.06148,"name":"online","context":{"idset":"11489"}}
+{"timestamp":1711596650.4331636,"name":"online","context":{"idset":"11487"}}
+{"timestamp":1711596650.8457592,"name":"online","context":{"idset":"11491"}}
+{"timestamp":1711596651.4617443,"name":"online","context":{"idset":"11492"}}
+{"timestamp":1711596656.564677,"name":"online","context":{"idset":"11497"}}
+{"timestamp":1711596660.8403211,"name":"online","context":{"idset":"11508"}}
+{"timestamp":1711596851.0527842,"name":"drain","context":{"idset":"717","reason":"epilog failed for jobid fm8DJpZWRx3","overwrite":0}}
+{"timestamp":1711597103.0687871,"name":"online","context":{"idset":"10084"}}
+{"timestamp":1711597541.4691079,"name":"drain","context":{"idset":"10076","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711597598.2555497,"name":"drain","context":{"idset":"10082","reason":"broker was unresponsive"}}
+{"timestamp":1711597664.1579545,"name":"drain","context":{"idset":"10082","reason":"prolog failed for jobid fm8igdC4XuR","overwrite":0}}
+{"timestamp":1711597664.2542503,"name":"offline","context":{"idset":"10082"}}
+{"timestamp":1711597874.1479568,"name":"online","context":{"idset":"10101-10102,10105"}}
+{"timestamp":1711597874.2686222,"name":"online","context":{"idset":"10103-10104,10106,10116"}}
+{"timestamp":1711597874.4480026,"name":"online","context":{"idset":"10109-10110,10113"}}
+{"timestamp":1711597874.5696421,"name":"online","context":{"idset":"10107-10108,10112,10115"}}
+{"timestamp":1711597874.6710663,"name":"online","context":{"idset":"10111,10114"}}
+{"timestamp":1711597879.7632606,"name":"online","context":{"idset":"10117-10119"}}
+{"timestamp":1711597879.9052639,"name":"online","context":{"idset":"10120"}}
+{"timestamp":1711597880.0643315,"name":"online","context":{"idset":"10122"}}
+{"timestamp":1711597880.1658378,"name":"online","context":{"idset":"10123,10125,10128"}}
+{"timestamp":1711597880.2735713,"name":"online","context":{"idset":"10127,10130"}}
+{"timestamp":1711597880.3743095,"name":"online","context":{"idset":"10121,10129"}}
+{"timestamp":1711597880.5978267,"name":"online","context":{"idset":"10124,10131-10132"}}
+{"timestamp":1711597880.8058176,"name":"online","context":{"idset":"10126"}}
+{"timestamp":1711597885.440419,"name":"online","context":{"idset":"10133"}}
+{"timestamp":1711597885.7357476,"name":"online","context":{"idset":"10135-10138"}}
+{"timestamp":1711597885.8995667,"name":"online","context":{"idset":"10134"}}
+{"timestamp":1711597886.0897057,"name":"online","context":{"idset":"10139,10142,10144"}}
+{"timestamp":1711597886.286411,"name":"online","context":{"idset":"10140,10143"}}
+{"timestamp":1711597886.3929899,"name":"online","context":{"idset":"10145-10146"}}
+{"timestamp":1711597886.4991808,"name":"online","context":{"idset":"10141"}}
+{"timestamp":1711597886.6544912,"name":"online","context":{"idset":"10147"}}
+{"timestamp":1711597887.080406,"name":"online","context":{"idset":"10148"}}
+{"timestamp":1711597891.3245909,"name":"online","context":{"idset":"10149-10150"}}
+{"timestamp":1711597891.468812,"name":"online","context":{"idset":"10151-10152"}}
+{"timestamp":1711597891.6793356,"name":"online","context":{"idset":"10154,10158"}}
+{"timestamp":1711597891.940419,"name":"online","context":{"idset":"10159"}}
+{"timestamp":1711597892.0420802,"name":"online","context":{"idset":"10156-10157,10161"}}
+{"timestamp":1711597892.2053025,"name":"online","context":{"idset":"10162"}}
+{"timestamp":1711597892.3503149,"name":"online","context":{"idset":"10160"}}
+{"timestamp":1711597892.5011446,"name":"online","context":{"idset":"10163"}}
+{"timestamp":1711597892.6429019,"name":"online","context":{"idset":"10164"}}
+{"timestamp":1711597892.9105952,"name":"online","context":{"idset":"10165-10166"}}
+{"timestamp":1711597896.9694369,"name":"online","context":{"idset":"10170"}}
+{"timestamp":1711597897.2569399,"name":"online","context":{"idset":"10167-10169,10171"}}
+{"timestamp":1711597897.752192,"name":"online","context":{"idset":"10172"}}
+{"timestamp":1711597897.8940079,"name":"online","context":{"idset":"10173-10174,10176-10177"}}
+{"timestamp":1711597898.1937191,"name":"online","context":{"idset":"10175,10178"}}
+{"timestamp":1711597898.4310532,"name":"online","context":{"idset":"10179"}}
+{"timestamp":1711597898.7145131,"name":"online","context":{"idset":"10180-10181"}}
+{"timestamp":1711597898.9562244,"name":"online","context":{"idset":"10182"}}
+{"timestamp":1711597902.9373691,"name":"online","context":{"idset":"10183"}}
+{"timestamp":1711597903.084904,"name":"online","context":{"idset":"10184-10185,10187"}}
+{"timestamp":1711597903.2416496,"name":"online","context":{"idset":"10186"}}
+{"timestamp":1711597903.4197383,"name":"online","context":{"idset":"10188-10189"}}
+{"timestamp":1711597903.8971879,"name":"online","context":{"idset":"10190,10192"}}
+{"timestamp":1711597904.0084152,"name":"online","context":{"idset":"10191,10193"}}
+{"timestamp":1711597904.110368,"name":"online","context":{"idset":"10194"}}
+{"timestamp":1711597904.45681,"name":"online","context":{"idset":"10195,10197-10198"}}
+{"timestamp":1711597904.6325271,"name":"online","context":{"idset":"10196"}}
+{"timestamp":1711597908.7490151,"name":"online","context":{"idset":"10199"}}
+{"timestamp":1711597908.8915725,"name":"online","context":{"idset":"10200-10202"}}
+{"timestamp":1711597909.0036097,"name":"online","context":{"idset":"10203"}}
+{"timestamp":1711597909.463691,"name":"online","context":{"idset":"10204"}}
+{"timestamp":1711597909.6650589,"name":"online","context":{"idset":"10207-10208"}}
+{"timestamp":1711597909.8317256,"name":"online","context":{"idset":"10209-10211"}}
+{"timestamp":1711597910.1223075,"name":"online","context":{"idset":"10212-10213,10215"}}
+{"timestamp":1711597910.3933594,"name":"online","context":{"idset":"10214"}}
+{"timestamp":1711597910.7064836,"name":"online","context":{"idset":"10216"}}
+{"timestamp":1711597914.5367885,"name":"online","context":{"idset":"10217,10219"}}
+{"timestamp":1711597914.7220116,"name":"online","context":{"idset":"10220"}}
+{"timestamp":1711597914.9556744,"name":"online","context":{"idset":"10218,10221"}}
+{"timestamp":1711597915.2081966,"name":"online","context":{"idset":"10222-10223"}}
+{"timestamp":1711597915.3885548,"name":"online","context":{"idset":"10225"}}
+{"timestamp":1711597915.6868584,"name":"online","context":{"idset":"10224,10226-10227"}}
+{"timestamp":1711597915.893265,"name":"online","context":{"idset":"10228"}}
+{"timestamp":1711597944.2539268,"name":"drain","context":{"idset":"10081","reason":"broker was unresponsive"}}
+{"timestamp":1711598008.1592054,"name":"drain","context":{"idset":"10081","reason":"prolog failed for jobid fm8kw2NQgm5","overwrite":0}}
+{"timestamp":1711598008.2556539,"name":"offline","context":{"idset":"10081"}}
+{"timestamp":1711598162.2552085,"name":"drain","context":{"idset":"10080","reason":"broker was unresponsive"}}
+{"timestamp":1711598226.2534738,"name":"offline","context":{"idset":"10080"}}
+{"timestamp":1711598228.2556102,"name":"drain","context":{"idset":"10079","reason":"broker was unresponsive"}}
+{"timestamp":1711598294.2544048,"name":"offline","context":{"idset":"10079"}}
+{"timestamp":1711598420.2553101,"name":"drain","context":{"idset":"10078","reason":"broker was unresponsive"}}
+{"timestamp":1711598424.2559545,"name":"drain","context":{"idset":"10077","reason":"broker was unresponsive"}}
+{"timestamp":1711598482.2547524,"name":"offline","context":{"idset":"10078"}}
+{"timestamp":1711598488.2544618,"name":"offline","context":{"idset":"10077"}}
+{"timestamp":1711598948.7929192,"name":"undrain","context":{"idset":"717"}}
+{"timestamp":1711599018.4842365,"name":"online","context":{"idset":"10082"}}
+{"timestamp":1711599321.9595537,"name":"online","context":{"idset":"10081"}}
+{"timestamp":1711599552.2543674,"name":"offline","context":{"idset":"10076"}}
+{"timestamp":1711599734.2550299,"name":"drain","context":{"idset":"10074","reason":"broker was unresponsive"}}
+{"timestamp":1711599774.2550957,"name":"drain","context":{"idset":"10073","reason":"broker was unresponsive"}}
+{"timestamp":1711599800.2554057,"name":"offline","context":{"idset":"10074"}}
+{"timestamp":1711599830.1547806,"name":"drain","context":{"idset":"10085","reason":"broker was unresponsive"}}
+{"timestamp":1711599830.2550619,"name":"drain","context":{"idset":"10086","reason":"broker was unresponsive"}}
+{"timestamp":1711599836.2540793,"name":"drain","context":{"idset":"10071","reason":"broker was unresponsive"}}
+{"timestamp":1711599838.2547846,"name":"offline","context":{"idset":"10073"}}
+{"timestamp":1711599840.2556009,"name":"drain","context":{"idset":"10072","reason":"broker was unresponsive"}}
+{"timestamp":1711599894.2539496,"name":"offline","context":{"idset":"10085"}}
+{"timestamp":1711599896.2548406,"name":"offline","context":{"idset":"10086"}}
+{"timestamp":1711599900.2534122,"name":"offline","context":{"idset":"10071"}}
+{"timestamp":1711599902.2548754,"name":"offline","context":{"idset":"10072"}}
+{"timestamp":1711600252.2546053,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1711600478.1543531,"name":"drain","context":{"idset":"10154","reason":"broker was unresponsive"}}
+{"timestamp":1711600478.2542264,"name":"drain","context":{"idset":"10156","reason":"broker was unresponsive"}}
+{"timestamp":1711600542.1577208,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1711600542.2545779,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1711600556.2563355,"name":"drain","context":{"idset":"10183","reason":"broker was unresponsive"}}
+{"timestamp":1711600622.2549057,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1711600791.2288516,"name":"online","context":{"idset":"10076"}}
+{"timestamp":1711600792.7640219,"name":"online","context":{"idset":"10075"}}
+{"timestamp":1711600811.1517384,"name":"drain","context":{"idset":"10154,10156","reason":"epilog failed for jobid fm948nkaQej","overwrite":0}}
+{"timestamp":1711601600.4910355,"name":"online","context":{"idset":"10071"}}
+{"timestamp":1711601602.4652805,"name":"online","context":{"idset":"10072"}}
+{"timestamp":1711603028.2555907,"name":"drain","context":{"idset":"10157","reason":"broker was unresponsive"}}
+{"timestamp":1711603090.255111,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1711603290.1493776,"name":"online","context":{"idset":"10561"}}
+{"timestamp":1711603293.7077463,"name":"online","context":{"idset":"10549"}}
+{"timestamp":1711603293.8528173,"name":"online","context":{"idset":"10552"}}
+{"timestamp":1711603294.4780948,"name":"online","context":{"idset":"10559"}}
+{"timestamp":1711603294.6215425,"name":"online","context":{"idset":"10551,10553,10555,10557"}}
+{"timestamp":1711603294.72334,"name":"online","context":{"idset":"10554,10558,10560,10563-10564"}}
+{"timestamp":1711603294.8681073,"name":"online","context":{"idset":"10550,10562"}}
+{"timestamp":1711603295.0088515,"name":"online","context":{"idset":"10556"}}
+{"timestamp":1711604254.5171294,"name":"online","context":{"idset":"10157"}}
+{"timestamp":1711604326.2549248,"name":"drain","context":{"idset":"695","reason":"broker was unresponsive"}}
+{"timestamp":1711604340.2561386,"name":"drain","context":{"idset":"701","reason":"broker was unresponsive"}}
+{"timestamp":1711604392.2562771,"name":"drain","context":{"idset":"697","reason":"broker was unresponsive"}}
+{"timestamp":1711604580.2552168,"name":"drain","context":{"idset":"689","reason":"broker was unresponsive"}}
+{"timestamp":1711604590.2548223,"name":"drain","context":{"idset":"704","reason":"broker was unresponsive"}}
+{"timestamp":1711604648.2542455,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1711604740.2553332,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1711604908.2558758,"name":"drain","context":{"idset":"707","reason":"broker was unresponsive"}}
+{"timestamp":1711604918.2557685,"name":"drain","context":{"idset":"703","reason":"broker was unresponsive"}}
+{"timestamp":1711604948.2558951,"name":"drain","context":{"idset":"699","reason":"broker was unresponsive"}}
+{"timestamp":1711604954.2552111,"name":"drain","context":{"idset":"687","reason":"broker was unresponsive"}}
+{"timestamp":1711604958.2564192,"name":"drain","context":{"idset":"691","reason":"broker was unresponsive"}}
+{"timestamp":1711604960.2562068,"name":"drain","context":{"idset":"685","reason":"broker was unresponsive"}}
+{"timestamp":1711604968.2570915,"name":"drain","context":{"idset":"705","reason":"broker was unresponsive"}}
+{"timestamp":1711605080.9739125,"name":"online","context":{"idset":"10183"}}
+{"timestamp":1711605174.1568608,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1711605716.2557297,"name":"drain","context":{"idset":"698","reason":"broker was unresponsive"}}
+{"timestamp":1711605778.2555597,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1711605976.2549586,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1711605994.2548351,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1711606062.2558303,"name":"drain","context":{"idset":"688","reason":"broker was unresponsive"}}
+{"timestamp":1711606122.2549007,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1711606406.1556187,"name":"drain","context":{"idset":"692","reason":"broker was unresponsive"}}
+{"timestamp":1711606406.256216,"name":"drain","context":{"idset":"696","reason":"broker was unresponsive"}}
+{"timestamp":1711606408.2602317,"name":"drain","context":{"idset":"702","reason":"broker was unresponsive"}}
+{"timestamp":1711606467.8751817,"name":"offline","context":{"idset":"692"}}
+{"timestamp":1711606467.887804,"name":"offline","context":{"idset":"696"}}
+{"timestamp":1711606467.9816139,"name":"offline","context":{"idset":"702"}}
+{"timestamp":1711606868.2557678,"name":"drain","context":{"idset":"700","reason":"broker was unresponsive"}}
+{"timestamp":1711606944.2557213,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1711606988.255033,"name":"offline","context":{"idset":"701"}}
+{"timestamp":1711607030.2542584,"name":"offline","context":{"idset":"697"}}
+{"timestamp":1711607271.0253379,"name":"offline","context":{"idset":"685"}}
+{"timestamp":1711607620.2544515,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1711607808.2546859,"name":"offline","context":{"idset":"707"}}
+{"timestamp":1711608214.2551055,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1711611392.2550452,"name":"offline","context":{"idset":"10549"}}
+{"timestamp":1711615908.1602454,"name":"drain","context":{"idset":"685-687,690-692,694,696-697,699-702,706-708","reason":"epilog failed for jobid fm9HsYwFjjd","overwrite":0}}
+{"timestamp":1711615908.2551167,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1711633824.8577917,"name":"undrain","context":{"idset":"11382-11384,11386,11389,11391-11392,11400-11403,11408-11409,11411,11413-11415,11417,11419-11423,11428,11430-11431,11436-11439,11441-11443,11446-11447,11452,11459,11462,11467,11469-11470,11472-11476,11479-11481,11483-11485,11487,11489,11491-11497,11508"}}
+{"timestamp":1711634633.9882171,"name":"drain","context":{"idset":"10095","overwrite":0}}
+{"timestamp":1711634770.5287678,"name":"drain","context":{"idset":"10096","reason":"Partner node needs hardware action","overwrite":1}}
+{"timestamp":1711636554.7017479,"name":"online","context":{"idset":"698"}}
+{"timestamp":1711636556.1562276,"name":"online","context":{"idset":"704"}}
+{"timestamp":1711636556.6786811,"name":"online","context":{"idset":"700,703"}}
+{"timestamp":1711636556.8098023,"name":"online","context":{"idset":"695,705"}}
+{"timestamp":1711636558.6863632,"name":"online","context":{"idset":"688"}}
+{"timestamp":1711636561.6337671,"name":"online","context":{"idset":"699"}}
+{"timestamp":1711636562.5612662,"name":"online","context":{"idset":"707"}}
+{"timestamp":1711636562.7243369,"name":"online","context":{"idset":"691"}}
+{"timestamp":1711636566.4037151,"name":"online","context":{"idset":"689"}}
+{"timestamp":1711636566.7557507,"name":"online","context":{"idset":"687"}}
+{"timestamp":1711636838.2555192,"name":"drain","context":{"idset":"693","reason":"broker was unresponsive"}}
+{"timestamp":1711637020.2548931,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1711637152.3827591,"name":"online","context":{"idset":"874"}}
+{"timestamp":1711637242.0846999,"name":"online","context":{"idset":"873"}}
+{"timestamp":1711637294.6260407,"name":"online","context":{"idset":"11685"}}
+{"timestamp":1711637340.2547159,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1711638502.2543736,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1711640818.2548335,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1711640864.2543795,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1711640994.078238,"name":"online","context":{"idset":"704"}}
+{"timestamp":1711641026.2550433,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1711641043.3408628,"name":"online","context":{"idset":"700"}}
+{"timestamp":1711641178.2542269,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1711641702.254425,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1711641777.2951829,"name":"online","context":{"idset":"693"}}
+{"timestamp":1711641833.1627128,"name":"online","context":{"idset":"11669"}}
+{"timestamp":1711641994.2554824,"name":"offline","context":{"idset":"688"}}
+{"timestamp":1711642210.1588371,"name":"drain","context":{"idset":"718","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1589563,"name":"drain","context":{"idset":"719","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1590128,"name":"drain","context":{"idset":"720","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1590643,"name":"drain","context":{"idset":"721","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1590991,"name":"drain","context":{"idset":"722","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1591821,"name":"drain","context":{"idset":"723","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1592195,"name":"drain","context":{"idset":"724","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1592536,"name":"drain","context":{"idset":"733","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1592872,"name":"drain","context":{"idset":"735","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1593204,"name":"drain","context":{"idset":"736","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1593742,"name":"drain","context":{"idset":"737","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1594355,"name":"drain","context":{"idset":"738","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1594796,"name":"drain","context":{"idset":"739","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.159517,"name":"drain","context":{"idset":"740","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1595554,"name":"drain","context":{"idset":"741","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1595943,"name":"drain","context":{"idset":"742","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1596365,"name":"drain","context":{"idset":"743","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1596746,"name":"drain","context":{"idset":"744","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1597111,"name":"drain","context":{"idset":"745","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1597559,"name":"drain","context":{"idset":"747","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1598115,"name":"drain","context":{"idset":"749","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1598501,"name":"drain","context":{"idset":"750","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.159888,"name":"drain","context":{"idset":"751","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1599293,"name":"drain","context":{"idset":"752","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1599746,"name":"drain","context":{"idset":"753","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1600144,"name":"drain","context":{"idset":"754","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.1600802,"name":"drain","context":{"idset":"755","reason":"broker was unresponsive"}}
+{"timestamp":1711642210.5039413,"name":"drain","context":{"idset":"756","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1616328,"name":"drain","context":{"idset":"637","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1617086,"name":"drain","context":{"idset":"638","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.161751,"name":"drain","context":{"idset":"639","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1617904,"name":"drain","context":{"idset":"640","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1618268,"name":"drain","context":{"idset":"641","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1618621,"name":"drain","context":{"idset":"642","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1618974,"name":"drain","context":{"idset":"643","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1619322,"name":"drain","context":{"idset":"644","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1619685,"name":"drain","context":{"idset":"653","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.16201,"name":"drain","context":{"idset":"654","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.162055,"name":"drain","context":{"idset":"655","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1620932,"name":"drain","context":{"idset":"656","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1621311,"name":"drain","context":{"idset":"657","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1621685,"name":"drain","context":{"idset":"658","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1622062,"name":"drain","context":{"idset":"659","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1622446,"name":"drain","context":{"idset":"660","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1622829,"name":"drain","context":{"idset":"661","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1623201,"name":"drain","context":{"idset":"662","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1623569,"name":"drain","context":{"idset":"663","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.162394,"name":"drain","context":{"idset":"664","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1624658,"name":"drain","context":{"idset":"665","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1625173,"name":"drain","context":{"idset":"666","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1625559,"name":"drain","context":{"idset":"668","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1625948,"name":"drain","context":{"idset":"669","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1626337,"name":"drain","context":{"idset":"670","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.162673,"name":"drain","context":{"idset":"671","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1627119,"name":"drain","context":{"idset":"672","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1627512,"name":"drain","context":{"idset":"673","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1627915,"name":"drain","context":{"idset":"674","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1628311,"name":"drain","context":{"idset":"675","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1628704,"name":"drain","context":{"idset":"676","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1629126,"name":"drain","context":{"idset":"679","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1629534,"name":"drain","context":{"idset":"680","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1629946,"name":"drain","context":{"idset":"682","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.1630385,"name":"drain","context":{"idset":"683","reason":"broker was unresponsive"}}
+{"timestamp":1711642252.5597043,"name":"drain","context":{"idset":"684","reason":"broker was unresponsive"}}
+{"timestamp":1711642258.156193,"name":"drain","context":{"idset":"667","reason":"broker was unresponsive"}}
+{"timestamp":1711642258.1562793,"name":"drain","context":{"idset":"677","reason":"broker was unresponsive"}}
+{"timestamp":1711642258.156333,"name":"drain","context":{"idset":"678","reason":"broker was unresponsive"}}
+{"timestamp":1711642258.1563835,"name":"drain","context":{"idset":"681","reason":"broker was unresponsive"}}
+{"timestamp":1711642270.7459354,"name":"online","context":{"idset":"699"}}
+{"timestamp":1711642271.3365471,"name":"online","context":{"idset":"698"}}
+{"timestamp":1711642271.3977184,"name":"offline","context":{"idset":"718"}}
+{"timestamp":1711642271.3988247,"name":"offline","context":{"idset":"719"}}
+{"timestamp":1711642271.3998904,"name":"offline","context":{"idset":"720"}}
+{"timestamp":1711642271.401772,"name":"offline","context":{"idset":"721"}}
+{"timestamp":1711642271.4070156,"name":"offline","context":{"idset":"722"}}
+{"timestamp":1711642271.4131529,"name":"offline","context":{"idset":"723"}}
+{"timestamp":1711642271.4198377,"name":"offline","context":{"idset":"724"}}
+{"timestamp":1711642271.4255407,"name":"offline","context":{"idset":"733"}}
+{"timestamp":1711642271.4306777,"name":"offline","context":{"idset":"735"}}
+{"timestamp":1711642271.4377973,"name":"offline","context":{"idset":"736"}}
+{"timestamp":1711642271.4444513,"name":"offline","context":{"idset":"737"}}
+{"timestamp":1711642271.4501531,"name":"offline","context":{"idset":"738"}}
+{"timestamp":1711642271.4561925,"name":"offline","context":{"idset":"739"}}
+{"timestamp":1711642271.4626303,"name":"offline","context":{"idset":"740"}}
+{"timestamp":1711642271.4675915,"name":"offline","context":{"idset":"741"}}
+{"timestamp":1711642271.4836984,"name":"offline","context":{"idset":"742"}}
+{"timestamp":1711642271.48487,"name":"offline","context":{"idset":"743"}}
+{"timestamp":1711642271.4860499,"name":"offline","context":{"idset":"744"}}
+{"timestamp":1711642271.4887245,"name":"offline","context":{"idset":"745"}}
+{"timestamp":1711642271.496582,"name":"offline","context":{"idset":"747"}}
+{"timestamp":1711642271.5025551,"name":"offline","context":{"idset":"748"}}
+{"timestamp":1711642271.5084474,"name":"offline","context":{"idset":"749"}}
+{"timestamp":1711642271.5141201,"name":"offline","context":{"idset":"750"}}
+{"timestamp":1711642271.5192616,"name":"offline","context":{"idset":"751"}}
+{"timestamp":1711642271.5233102,"name":"offline","context":{"idset":"753"}}
+{"timestamp":1711642271.5276234,"name":"offline","context":{"idset":"754"}}
+{"timestamp":1711642271.566371,"name":"offline","context":{"idset":"755"}}
+{"timestamp":1711642271.5995269,"name":"offline","context":{"idset":"756"}}
+{"timestamp":1711642288.2546556,"name":"offline","context":{"idset":"752"}}
+{"timestamp":1711642318.1894867,"name":"offline","context":{"idset":"637"}}
+{"timestamp":1711642318.1948001,"name":"offline","context":{"idset":"639"}}
+{"timestamp":1711642318.2000339,"name":"offline","context":{"idset":"640"}}
+{"timestamp":1711642318.2052128,"name":"offline","context":{"idset":"641"}}
+{"timestamp":1711642318.2102711,"name":"offline","context":{"idset":"642"}}
+{"timestamp":1711642318.2153268,"name":"offline","context":{"idset":"643"}}
+{"timestamp":1711642318.2203863,"name":"offline","context":{"idset":"644"}}
+{"timestamp":1711642318.2253728,"name":"offline","context":{"idset":"653"}}
+{"timestamp":1711642318.2303727,"name":"offline","context":{"idset":"654"}}
+{"timestamp":1711642318.2376361,"name":"offline","context":{"idset":"655"}}
+{"timestamp":1711642318.2431774,"name":"offline","context":{"idset":"656"}}
+{"timestamp":1711642318.2483091,"name":"offline","context":{"idset":"657"}}
+{"timestamp":1711642318.2543108,"name":"offline","context":{"idset":"658"}}
+{"timestamp":1711642318.2592404,"name":"offline","context":{"idset":"659"}}
+{"timestamp":1711642318.2733517,"name":"offline","context":{"idset":"660"}}
+{"timestamp":1711642318.274291,"name":"offline","context":{"idset":"661"}}
+{"timestamp":1711642318.2752156,"name":"offline","context":{"idset":"662"}}
+{"timestamp":1711642318.279259,"name":"offline","context":{"idset":"663"}}
+{"timestamp":1711642318.2864203,"name":"offline","context":{"idset":"664"}}
+{"timestamp":1711642318.2929595,"name":"offline","context":{"idset":"665"}}
+{"timestamp":1711642318.2993448,"name":"offline","context":{"idset":"666"}}
+{"timestamp":1711642318.304518,"name":"offline","context":{"idset":"668"}}
+{"timestamp":1711642318.3094058,"name":"offline","context":{"idset":"669"}}
+{"timestamp":1711642318.3684959,"name":"offline","context":{"idset":"670"}}
+{"timestamp":1711642318.4222679,"name":"offline","context":{"idset":"671"}}
+{"timestamp":1711642318.4365716,"name":"offline","context":{"idset":"672"}}
+{"timestamp":1711642318.437525,"name":"offline","context":{"idset":"673"}}
+{"timestamp":1711642318.4384584,"name":"offline","context":{"idset":"674"}}
+{"timestamp":1711642318.4393821,"name":"offline","context":{"idset":"675"}}
+{"timestamp":1711642318.4402885,"name":"offline","context":{"idset":"676"}}
+{"timestamp":1711642318.4412072,"name":"offline","context":{"idset":"679"}}
+{"timestamp":1711642318.4421272,"name":"offline","context":{"idset":"680"}}
+{"timestamp":1711642318.4430459,"name":"offline","context":{"idset":"683"}}
+{"timestamp":1711642318.4819684,"name":"offline","context":{"idset":"684"}}
+{"timestamp":1711642320.1623349,"name":"offline","context":{"idset":"667"}}
+{"timestamp":1711642320.167789,"name":"offline","context":{"idset":"677"}}
+{"timestamp":1711642320.1730199,"name":"offline","context":{"idset":"678"}}
+{"timestamp":1711642320.1782365,"name":"offline","context":{"idset":"681"}}
+{"timestamp":1711642320.1835165,"name":"offline","context":{"idset":"686"}}
+{"timestamp":1711642320.1887536,"name":"offline","context":{"idset":"690"}}
+{"timestamp":1711642320.1939833,"name":"offline","context":{"idset":"706"}}
+{"timestamp":1711642320.2841983,"name":"offline","context":{"idset":"708"}}
+{"timestamp":1711642322.2549641,"name":"offline","context":{"idset":"704"}}
+{"timestamp":1711642324.1589651,"name":"offline","context":{"idset":"694"}}
+{"timestamp":1711642324.2545993,"name":"offline","context":{"idset":"700"}}
+{"timestamp":1711642366.1580346,"name":"offline","context":{"idset":"638"}}
+{"timestamp":1711642366.2548316,"name":"offline","context":{"idset":"682"}}
+{"timestamp":1711642382.2544386,"name":"offline","context":{"idset":"693"}}
+{"timestamp":1711642484.1572287,"name":"offline","context":{"idset":"698"}}
+{"timestamp":1711642484.2552466,"name":"offline","context":{"idset":"699"}}
+{"timestamp":1711643242.7329867,"name":"online","context":{"idset":"691,703"}}
+{"timestamp":1711643242.9061217,"name":"online","context":{"idset":"687"}}
+{"timestamp":1711643424.1555297,"name":"offline","context":{"idset":"687"}}
+{"timestamp":1711643488.2532897,"name":"offline","context":{"idset":"689"}}
+{"timestamp":1711643506.253685,"name":"offline","context":{"idset":"691"}}
+{"timestamp":1711643556.2542388,"name":"offline","context":{"idset":"695"}}
+{"timestamp":1711643616.15783,"name":"offline","context":{"idset":"703"}}
+{"timestamp":1711643616.1630802,"name":"offline","context":{"idset":"705"}}
+{"timestamp":1711643616.2583759,"name":"offline","context":{"idset":"707"}}
+{"timestamp":1711643659.2512302,"name":"online","context":{"idset":"875"}}
+{"timestamp":1711644351.1504831,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1711645642.8167577,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1711645642.8277376,"name":"offline","context":{"idset":"71"}}
+{"timestamp":1711645642.8347065,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1711645642.8498607,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1711645642.8578875,"name":"offline","context":{"idset":"72"}}
+{"timestamp":1711645642.8654649,"name":"offline","context":{"idset":"73"}}
+{"timestamp":1711645642.8784328,"name":"offline","context":{"idset":"76"}}
+{"timestamp":1711645642.8794136,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1711645642.9102252,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1711645642.9111273,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1711645642.9120507,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1711645642.9130058,"name":"offline","context":{"idset":"69"}}
+{"timestamp":1711645642.9232888,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1711645642.9429803,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1711645642.9438934,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1711645642.944948,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1711645642.9458759,"name":"offline","context":{"idset":"70"}}
+{"timestamp":1711645642.9481339,"name":"offline","context":{"idset":"74"}}
+{"timestamp":1711645642.9528506,"name":"offline","context":{"idset":"75"}}
+{"timestamp":1711645642.9618752,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1711645642.9757473,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1711645642.9861586,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1711645643.1201799,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1711646931.6480982,"name":"drain","context":{"idset":"69-76","overwrite":0}}
+{"timestamp":1711647021.3459654,"name":"online","context":{"idset":"11302,11308,11310"}}
+{"timestamp":1711647021.4871552,"name":"online","context":{"idset":"11301,11313"}}
+{"timestamp":1711647021.7886832,"name":"online","context":{"idset":"11304"}}
+{"timestamp":1711647021.9056427,"name":"online","context":{"idset":"11307,11309,11311-11312,11314"}}
+{"timestamp":1711647022.0430727,"name":"online","context":{"idset":"11303,11305-11306,11315"}}
+{"timestamp":1711647022.155683,"name":"online","context":{"idset":"11316"}}
+{"timestamp":1711647232.5255136,"name":"online","context":{"idset":"61-62,64,66-67,78-79,81-83"}}
+{"timestamp":1711647232.6341705,"name":"online","context":{"idset":"63,65,68,77,80,84"}}
+{"timestamp":1711647594.3615842,"name":"online","context":{"idset":"11857"}}
+{"timestamp":1711647594.5015261,"name":"online","context":{"idset":"11848,11850"}}
+{"timestamp":1711647594.62152,"name":"online","context":{"idset":"11846"}}
+{"timestamp":1711647594.777308,"name":"online","context":{"idset":"11855"}}
+{"timestamp":1711647594.8854239,"name":"online","context":{"idset":"11851,11854,11860"}}
+{"timestamp":1711647594.996557,"name":"online","context":{"idset":"11847,11849,11853,11856,11858"}}
+{"timestamp":1711647595.1357989,"name":"online","context":{"idset":"11845,11859"}}
+{"timestamp":1711647615.5256357,"name":"online","context":{"idset":"11885"}}
+{"timestamp":1711647615.8665714,"name":"online","context":{"idset":"11877-11878,11890"}}
+{"timestamp":1711647616.0425949,"name":"online","context":{"idset":"11886"}}
+{"timestamp":1711647616.1875236,"name":"online","context":{"idset":"11879,11887-11889"}}
+{"timestamp":1711647616.3333273,"name":"online","context":{"idset":"11882,11891"}}
+{"timestamp":1711647616.4700174,"name":"online","context":{"idset":"11880-11881,11883-11884"}}
+{"timestamp":1711648606.1955028,"name":"online","context":{"idset":"11286,11297,11321"}}
+{"timestamp":1711648606.4926741,"name":"online","context":{"idset":"11290-11291"}}
+{"timestamp":1711648606.7698321,"name":"online","context":{"idset":"11300"}}
+{"timestamp":1711648608.8558404,"name":"online","context":{"idset":"11324"}}
+{"timestamp":1711648608.9748373,"name":"online","context":{"idset":"11323"}}
+{"timestamp":1711648609.1351645,"name":"online","context":{"idset":"11326"}}
+{"timestamp":1711648609.2564611,"name":"online","context":{"idset":"11325"}}
+{"timestamp":1711648612.3787053,"name":"online","context":{"idset":"11330"}}
+{"timestamp":1711648615.0247762,"name":"online","context":{"idset":"11332"}}
+{"timestamp":1711648615.655369,"name":"online","context":{"idset":"11335,11344"}}
+{"timestamp":1711648615.8955524,"name":"online","context":{"idset":"11338"}}
+{"timestamp":1711648615.9985602,"name":"online","context":{"idset":"11357,11366"}}
+{"timestamp":1711648616.2306554,"name":"online","context":{"idset":"11348,11364"}}
+{"timestamp":1711648616.3367431,"name":"online","context":{"idset":"11351,11353,11361"}}
+{"timestamp":1711648616.4827685,"name":"online","context":{"idset":"11355,11359,11372,11375,11377"}}
+{"timestamp":1711648616.6622939,"name":"online","context":{"idset":"11354,11374,11604,11607"}}
+{"timestamp":1711648616.7729936,"name":"online","context":{"idset":"11370,11378"}}
+{"timestamp":1711648616.8346384,"name":"online","context":{"idset":"11380"}}
+{"timestamp":1711648616.9488957,"name":"online","context":{"idset":"11340,11395,11592,11612"}}
+{"timestamp":1711648617.048687,"name":"online","context":{"idset":"11608"}}
+{"timestamp":1711648617.155762,"name":"online","context":{"idset":"11595,11597,11599,11611"}}
+{"timestamp":1711648617.3216004,"name":"online","context":{"idset":"11603"}}
+{"timestamp":1711648617.4418912,"name":"online","context":{"idset":"11360"}}
+{"timestamp":1711648617.4863193,"name":"online","context":{"idset":"11601"}}
+{"timestamp":1711648617.7766957,"name":"online","context":{"idset":"11416,11427,11618"}}
+{"timestamp":1711648617.9394605,"name":"online","context":{"idset":"11615,11620-11621"}}
+{"timestamp":1711648618.2221718,"name":"online","context":{"idset":"11614,11619"}}
+{"timestamp":1711648618.4849021,"name":"online","context":{"idset":"11410,11444"}}
+{"timestamp":1711648618.6913009,"name":"online","context":{"idset":"11397,11622"}}
+{"timestamp":1711648618.7437398,"name":"online","context":{"idset":"11412"}}
+{"timestamp":1711648618.8545589,"name":"online","context":{"idset":"11405"}}
+{"timestamp":1711648618.9635823,"name":"online","context":{"idset":"11433"}}
+{"timestamp":1711648619.0705035,"name":"online","context":{"idset":"11435,11500"}}
+{"timestamp":1711648619.2154806,"name":"online","context":{"idset":"11398,11429"}}
+{"timestamp":1711648619.3330514,"name":"online","context":{"idset":"11623"}}
+{"timestamp":1711648619.4761202,"name":"online","context":{"idset":"11461,11624"}}
+{"timestamp":1711648620.273454,"name":"online","context":{"idset":"11625-11627"}}
+{"timestamp":1711648620.4457617,"name":"online","context":{"idset":"11628"}}
+{"timestamp":1711648620.7689905,"name":"online","context":{"idset":"11630"}}
+{"timestamp":1711648620.9358747,"name":"online","context":{"idset":"11629"}}
+{"timestamp":1711648621.2109995,"name":"online","context":{"idset":"11631"}}
+{"timestamp":1711648621.3179579,"name":"online","context":{"idset":"11445,11632"}}
+{"timestamp":1711648621.8240428,"name":"online","context":{"idset":"11457"}}
+{"timestamp":1711648622.2902608,"name":"online","context":{"idset":"11634"}}
+{"timestamp":1711648622.4655609,"name":"online","context":{"idset":"11633"}}
+{"timestamp":1711648623.8754861,"name":"online","context":{"idset":"11635-11636"}}
+{"timestamp":1711648624.3637838,"name":"online","context":{"idset":"11747"}}
+{"timestamp":1711648625.1203809,"name":"online","context":{"idset":"11748"}}
+{"timestamp":1711648625.8826056,"name":"online","context":{"idset":"11754"}}
+{"timestamp":1711648626.2538016,"name":"online","context":{"idset":"11750,11755"}}
+{"timestamp":1711648626.4796205,"name":"online","context":{"idset":"11756"}}
+{"timestamp":1711648626.5952208,"name":"online","context":{"idset":"11751,11761"}}
+{"timestamp":1711648626.7609365,"name":"online","context":{"idset":"11757,11759-11760,11764"}}
+{"timestamp":1711648626.9874179,"name":"online","context":{"idset":"11752,11762"}}
+{"timestamp":1711648650.1510217,"name":"undrain","context":{"idset":"11128-11148,11151-11154,11156-11163,11165-11174,11177-11178,11180-11186,11191-11202,11205-11212,11214-11215,11217-11229,11231-11252,11285-11294,11297-11300,11318-11340,11344,11348,11350-11381,11385,11390,11395-11399,11404-11407,11410,11412,11416,11418,11424-11427,11429,11432-11435,11440,11444-11445,11451,11454-11458,11461,11463-11464,11466,11468,11471,11477,11482,11486,11498-11500,11503-11507,11589-11592,11594-11599,11601-11604,11606-11636,11747-11764"}}
+{"timestamp":1711649550.0213203,"name":"undrain","context":{"idset":"10076"}}
+{"timestamp":1711650472.2563207,"name":"drain","context":{"idset":"11290","reason":"broker was unresponsive"}}
+{"timestamp":1711650474.2557621,"name":"drain","context":{"idset":"11286","reason":"broker was unresponsive"}}
+{"timestamp":1711650476.2556355,"name":"drain","context":{"idset":"11291","reason":"broker was unresponsive"}}
+{"timestamp":1711650480.2563696,"name":"drain","context":{"idset":"11297","reason":"broker was unresponsive"}}
+{"timestamp":1711650482.25527,"name":"drain","context":{"idset":"11300","reason":"broker was unresponsive"}}
+{"timestamp":1711650536.2544315,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1711650538.2556913,"name":"offline","context":{"idset":"11290"}}
+{"timestamp":1711650540.2551498,"name":"offline","context":{"idset":"11291"}}
+{"timestamp":1711650542.2546799,"name":"offline","context":{"idset":"11297"}}
+{"timestamp":1711650544.1599052,"name":"offline","context":{"idset":"11300"}}
+{"timestamp":1711650544.1654844,"name":"offline","context":{"idset":"11301"}}
+{"timestamp":1711650544.1708443,"name":"offline","context":{"idset":"11302"}}
+{"timestamp":1711650544.2541547,"name":"offline","context":{"idset":"11303"}}
+{"timestamp":1711650546.1641712,"name":"offline","context":{"idset":"11304"}}
+{"timestamp":1711650546.1741357,"name":"offline","context":{"idset":"11305"}}
+{"timestamp":1711650546.1827526,"name":"offline","context":{"idset":"11306"}}
+{"timestamp":1711650546.2766237,"name":"offline","context":{"idset":"11307"}}
+{"timestamp":1711650548.1649418,"name":"offline","context":{"idset":"11308"}}
+{"timestamp":1711650548.1854224,"name":"offline","context":{"idset":"11309"}}
+{"timestamp":1711650548.192595,"name":"offline","context":{"idset":"11310"}}
+{"timestamp":1711650548.1997538,"name":"offline","context":{"idset":"11311"}}
+{"timestamp":1711650548.2841282,"name":"offline","context":{"idset":"11312"}}
+{"timestamp":1711650550.1714754,"name":"offline","context":{"idset":"11313"}}
+{"timestamp":1711650550.1824524,"name":"offline","context":{"idset":"11314"}}
+{"timestamp":1711650550.1939783,"name":"offline","context":{"idset":"11315"}}
+{"timestamp":1711650550.2691786,"name":"offline","context":{"idset":"11316"}}
+{"timestamp":1711650620.1558492,"name":"drain","context":{"idset":"11321","reason":"broker was unresponsive"}}
+{"timestamp":1711650620.2557573,"name":"drain","context":{"idset":"11323","reason":"broker was unresponsive"}}
+{"timestamp":1711650622.1556945,"name":"drain","context":{"idset":"11325","reason":"broker was unresponsive"}}
+{"timestamp":1711650622.1558068,"name":"drain","context":{"idset":"11326","reason":"broker was unresponsive"}}
+{"timestamp":1711650622.2556465,"name":"drain","context":{"idset":"11330","reason":"broker was unresponsive"}}
+{"timestamp":1711650624.2569983,"name":"drain","context":{"idset":"11324","reason":"broker was unresponsive"}}
+{"timestamp":1711650628.1561337,"name":"drain","context":{"idset":"11332","reason":"broker was unresponsive"}}
+{"timestamp":1711650628.1562393,"name":"drain","context":{"idset":"11335","reason":"broker was unresponsive"}}
+{"timestamp":1711650628.2561018,"name":"drain","context":{"idset":"11338","reason":"broker was unresponsive"}}
+{"timestamp":1711650632.155652,"name":"drain","context":{"idset":"11340","reason":"broker was unresponsive"}}
+{"timestamp":1711650632.255296,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1711650634.2559328,"name":"drain","context":{"idset":"11351","reason":"broker was unresponsive"}}
+{"timestamp":1711650636.1550829,"name":"drain","context":{"idset":"11348","reason":"broker was unresponsive"}}
+{"timestamp":1711650636.1552191,"name":"drain","context":{"idset":"11353","reason":"broker was unresponsive"}}
+{"timestamp":1711650636.2551146,"name":"drain","context":{"idset":"11357","reason":"broker was unresponsive"}}
+{"timestamp":1711650638.1569469,"name":"drain","context":{"idset":"11354","reason":"broker was unresponsive"}}
+{"timestamp":1711650638.2565691,"name":"drain","context":{"idset":"11355","reason":"broker was unresponsive"}}
+{"timestamp":1711650640.1562665,"name":"drain","context":{"idset":"11359","reason":"broker was unresponsive"}}
+{"timestamp":1711650640.1564136,"name":"drain","context":{"idset":"11360","reason":"broker was unresponsive"}}
+{"timestamp":1711650640.2558939,"name":"drain","context":{"idset":"11366","reason":"broker was unresponsive"}}
+{"timestamp":1711650642.1560137,"name":"drain","context":{"idset":"11361","reason":"broker was unresponsive"}}
+{"timestamp":1711650642.255878,"name":"drain","context":{"idset":"11364","reason":"broker was unresponsive"}}
+{"timestamp":1711650644.1551466,"name":"drain","context":{"idset":"11372","reason":"broker was unresponsive"}}
+{"timestamp":1711650644.1552951,"name":"drain","context":{"idset":"11374","reason":"broker was unresponsive"}}
+{"timestamp":1711650644.1554286,"name":"drain","context":{"idset":"11375","reason":"broker was unresponsive"}}
+{"timestamp":1711650644.2543528,"name":"drain","context":{"idset":"11378","reason":"broker was unresponsive"}}
+{"timestamp":1711650646.15609,"name":"drain","context":{"idset":"11370","reason":"broker was unresponsive"}}
+{"timestamp":1711650646.2556603,"name":"drain","context":{"idset":"11377","reason":"broker was unresponsive"}}
+{"timestamp":1711650648.2546284,"name":"drain","context":{"idset":"11380","reason":"broker was unresponsive"}}
+{"timestamp":1711650650.1564307,"name":"drain","context":{"idset":"11383","reason":"broker was unresponsive"}}
+{"timestamp":1711650650.1565728,"name":"drain","context":{"idset":"11384","reason":"broker was unresponsive"}}
+{"timestamp":1711650650.1567149,"name":"drain","context":{"idset":"11389","reason":"broker was unresponsive"}}
+{"timestamp":1711650650.2555413,"name":"drain","context":{"idset":"11392","reason":"broker was unresponsive"}}
+{"timestamp":1711650652.1564956,"name":"drain","context":{"idset":"11382","reason":"broker was unresponsive"}}
+{"timestamp":1711650652.1566389,"name":"drain","context":{"idset":"11386","reason":"broker was unresponsive"}}
+{"timestamp":1711650652.2568946,"name":"drain","context":{"idset":"11395","reason":"broker was unresponsive"}}
+{"timestamp":1711650654.156652,"name":"drain","context":{"idset":"11391","reason":"broker was unresponsive"}}
+{"timestamp":1711650656.1558905,"name":"drain","context":{"idset":"11397","reason":"broker was unresponsive"}}
+{"timestamp":1711650656.2558155,"name":"drain","context":{"idset":"11398","reason":"broker was unresponsive"}}
+{"timestamp":1711650660.1558089,"name":"drain","context":{"idset":"11400","reason":"broker was unresponsive"}}
+{"timestamp":1711650660.1559827,"name":"drain","context":{"idset":"11401","reason":"broker was unresponsive"}}
+{"timestamp":1711650660.156127,"name":"drain","context":{"idset":"11402","reason":"broker was unresponsive"}}
+{"timestamp":1711650660.156266,"name":"drain","context":{"idset":"11403","reason":"broker was unresponsive"}}
+{"timestamp":1711650660.1564043,"name":"drain","context":{"idset":"11405","reason":"broker was unresponsive"}}
+{"timestamp":1711650660.2544689,"name":"drain","context":{"idset":"11412","reason":"broker was unresponsive"}}
+{"timestamp":1711650662.2556987,"name":"drain","context":{"idset":"11415","reason":"broker was unresponsive"}}
+{"timestamp":1711650664.1576862,"name":"drain","context":{"idset":"11408","reason":"broker was unresponsive"}}
+{"timestamp":1711650664.1578712,"name":"drain","context":{"idset":"11409","reason":"broker was unresponsive"}}
+{"timestamp":1711650664.1580465,"name":"drain","context":{"idset":"11410","reason":"broker was unresponsive"}}
+{"timestamp":1711650664.1581936,"name":"drain","context":{"idset":"11411","reason":"broker was unresponsive"}}
+{"timestamp":1711650664.1583388,"name":"drain","context":{"idset":"11414","reason":"broker was unresponsive"}}
+{"timestamp":1711650664.1584907,"name":"drain","context":{"idset":"11416","reason":"broker was unresponsive"}}
+{"timestamp":1711650664.1586566,"name":"drain","context":{"idset":"11417","reason":"broker was unresponsive"}}
+{"timestamp":1711650664.2576492,"name":"drain","context":{"idset":"11419","reason":"broker was unresponsive"}}
+{"timestamp":1711650666.2565143,"name":"drain","context":{"idset":"11413","reason":"broker was unresponsive"}}
+{"timestamp":1711650668.1566327,"name":"drain","context":{"idset":"11420","reason":"broker was unresponsive"}}
+{"timestamp":1711650668.1567967,"name":"drain","context":{"idset":"11421","reason":"broker was unresponsive"}}
+{"timestamp":1711650668.1569288,"name":"drain","context":{"idset":"11423","reason":"broker was unresponsive"}}
+{"timestamp":1711650668.2563472,"name":"drain","context":{"idset":"11428","reason":"broker was unresponsive"}}
+{"timestamp":1711650670.1555846,"name":"drain","context":{"idset":"11422","reason":"broker was unresponsive"}}
+{"timestamp":1711650670.2561576,"name":"drain","context":{"idset":"11427","reason":"broker was unresponsive"}}
+{"timestamp":1711650672.1563449,"name":"drain","context":{"idset":"11429","reason":"broker was unresponsive"}}
+{"timestamp":1711650672.2569659,"name":"drain","context":{"idset":"11433","reason":"broker was unresponsive"}}
+{"timestamp":1711650674.1572955,"name":"drain","context":{"idset":"11430","reason":"broker was unresponsive"}}
+{"timestamp":1711650674.1574328,"name":"drain","context":{"idset":"11431","reason":"broker was unresponsive"}}
+{"timestamp":1711650674.1575251,"name":"drain","context":{"idset":"11435","reason":"broker was unresponsive"}}
+{"timestamp":1711650674.1576116,"name":"drain","context":{"idset":"11437","reason":"broker was unresponsive"}}
+{"timestamp":1711650674.1576991,"name":"drain","context":{"idset":"11438","reason":"broker was unresponsive"}}
+{"timestamp":1711650674.2569783,"name":"drain","context":{"idset":"11439","reason":"broker was unresponsive"}}
+{"timestamp":1711650676.1566427,"name":"drain","context":{"idset":"11436","reason":"broker was unresponsive"}}
+{"timestamp":1711650676.2562418,"name":"drain","context":{"idset":"11442","reason":"broker was unresponsive"}}
+{"timestamp":1711650680.1577346,"name":"drain","context":{"idset":"11441","reason":"broker was unresponsive"}}
+{"timestamp":1711650680.1579924,"name":"drain","context":{"idset":"11443","reason":"broker was unresponsive"}}
+{"timestamp":1711650680.1581464,"name":"drain","context":{"idset":"11444","reason":"broker was unresponsive"}}
+{"timestamp":1711650680.158303,"name":"drain","context":{"idset":"11445","reason":"broker was unresponsive"}}
+{"timestamp":1711650680.2565355,"name":"drain","context":{"idset":"11447","reason":"broker was unresponsive"}}
+{"timestamp":1711650682.1555905,"name":"drain","context":{"idset":"11446","reason":"broker was unresponsive"}}
+{"timestamp":1711650682.2549489,"name":"drain","context":{"idset":"11452","reason":"broker was unresponsive"}}
+{"timestamp":1711650684.1603127,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1711650684.1687078,"name":"drain","context":{"idset":"11459","reason":"broker was unresponsive"}}
+{"timestamp":1711650686.1633856,"name":"offline","context":{"idset":"11323"}}
+{"timestamp":1711650686.1741121,"name":"offline","context":{"idset":"11324"}}
+{"timestamp":1711650686.1813509,"name":"offline","context":{"idset":"11325"}}
+{"timestamp":1711650686.1868598,"name":"offline","context":{"idset":"11326"}}
+{"timestamp":1711650686.1949894,"name":"drain","context":{"idset":"11457","reason":"broker was unresponsive"}}
+{"timestamp":1711650686.2636492,"name":"drain","context":{"idset":"11461","reason":"broker was unresponsive"}}
+{"timestamp":1711650688.1588573,"name":"offline","context":{"idset":"11330"}}
+{"timestamp":1711650688.163574,"name":"drain","context":{"idset":"11462","reason":"broker was unresponsive"}}
+{"timestamp":1711650688.2572289,"name":"drain","context":{"idset":"11467","reason":"broker was unresponsive"}}
+{"timestamp":1711650690.1594172,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1711650690.1642787,"name":"drain","context":{"idset":"11465","reason":"broker was unresponsive"}}
+{"timestamp":1711650690.1644809,"name":"drain","context":{"idset":"11469","reason":"broker was unresponsive"}}
+{"timestamp":1711650690.2565198,"name":"drain","context":{"idset":"11472","reason":"broker was unresponsive"}}
+{"timestamp":1711650692.159215,"name":"offline","context":{"idset":"11335"}}
+{"timestamp":1711650692.2571719,"name":"drain","context":{"idset":"11470","reason":"broker was unresponsive"}}
+{"timestamp":1711650694.1576324,"name":"offline","context":{"idset":"11338"}}
+{"timestamp":1711650694.1654859,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1711650694.1702652,"name":"drain","context":{"idset":"11474","reason":"broker was unresponsive"}}
+{"timestamp":1711650694.1704659,"name":"drain","context":{"idset":"11475","reason":"broker was unresponsive"}}
+{"timestamp":1711650694.2565086,"name":"drain","context":{"idset":"11480","reason":"broker was unresponsive"}}
+{"timestamp":1711650696.1591239,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1711650696.1632161,"name":"drain","context":{"idset":"11473","reason":"broker was unresponsive"}}
+{"timestamp":1711650696.1634502,"name":"drain","context":{"idset":"11481","reason":"broker was unresponsive"}}
+{"timestamp":1711650696.2568576,"name":"drain","context":{"idset":"11483","reason":"broker was unresponsive"}}
+{"timestamp":1711650698.1591575,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1711650698.1669459,"name":"offline","context":{"idset":"11351"}}
+{"timestamp":1711650698.1710629,"name":"drain","context":{"idset":"11476","reason":"broker was unresponsive"}}
+{"timestamp":1711650698.1712503,"name":"drain","context":{"idset":"11479","reason":"broker was unresponsive"}}
+{"timestamp":1711650698.1714315,"name":"drain","context":{"idset":"11484","reason":"broker was unresponsive"}}
+{"timestamp":1711650698.2583613,"name":"drain","context":{"idset":"11489","reason":"broker was unresponsive"}}
+{"timestamp":1711650700.1615145,"name":"offline","context":{"idset":"11353"}}
+{"timestamp":1711650700.1724532,"name":"offline","context":{"idset":"11354"}}
+{"timestamp":1711650700.1801558,"name":"offline","context":{"idset":"11355"}}
+{"timestamp":1711650700.1837032,"name":"drain","context":{"idset":"11491","reason":"broker was unresponsive"}}
+{"timestamp":1711650700.2607677,"name":"drain","context":{"idset":"11493","reason":"broker was unresponsive"}}
+{"timestamp":1711650702.1584597,"name":"offline","context":{"idset":"11357"}}
+{"timestamp":1711650702.1702683,"name":"offline","context":{"idset":"11359"}}
+{"timestamp":1711650702.1794801,"name":"offline","context":{"idset":"11360"}}
+{"timestamp":1711650702.1866462,"name":"drain","context":{"idset":"11485","reason":"broker was unresponsive"}}
+{"timestamp":1711650702.1868434,"name":"drain","context":{"idset":"11487","reason":"broker was unresponsive"}}
+{"timestamp":1711650702.1870284,"name":"drain","context":{"idset":"11495","reason":"broker was unresponsive"}}
+{"timestamp":1711650702.2742891,"name":"drain","context":{"idset":"11497","reason":"broker was unresponsive"}}
+{"timestamp":1711650704.1576824,"name":"offline","context":{"idset":"11361"}}
+{"timestamp":1711650704.1648741,"name":"offline","context":{"idset":"11364"}}
+{"timestamp":1711650704.1696942,"name":"drain","context":{"idset":"11492","reason":"broker was unresponsive"}}
+{"timestamp":1711650704.2572072,"name":"drain","context":{"idset":"11496","reason":"broker was unresponsive"}}
+{"timestamp":1711650706.1572392,"name":"offline","context":{"idset":"11366"}}
+{"timestamp":1711650706.2562897,"name":"drain","context":{"idset":"11494","reason":"broker was unresponsive"}}
+{"timestamp":1711650708.1590323,"name":"offline","context":{"idset":"11370"}}
+{"timestamp":1711650708.1655955,"name":"offline","context":{"idset":"11372"}}
+{"timestamp":1711650708.1696668,"name":"drain","context":{"idset":"11500","reason":"broker was unresponsive"}}
+{"timestamp":1711650708.2582283,"name":"drain","context":{"idset":"11508","reason":"broker was unresponsive"}}
+{"timestamp":1711650710.1607666,"name":"offline","context":{"idset":"11374"}}
+{"timestamp":1711650710.1664143,"name":"offline","context":{"idset":"11375"}}
+{"timestamp":1711650710.1718895,"name":"offline","context":{"idset":"11377"}}
+{"timestamp":1711650710.2547941,"name":"offline","context":{"idset":"11378"}}
+{"timestamp":1711650712.2542529,"name":"offline","context":{"idset":"11380"}}
+{"timestamp":1711650714.1602218,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1711650714.1671042,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1711650714.1763749,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1711650714.2546058,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1711650716.1627362,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1711650716.1745932,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1711650716.2840991,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1711650718.1740801,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1711650718.2547657,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1711650720.1665483,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1711650720.2557645,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1711650722.163718,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1711650722.1730402,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1711650722.1822236,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1711650722.2546291,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1711650726.1762998,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1711650726.1849709,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1711650726.1938353,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1711650726.2029076,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1711650726.2122092,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1711650726.336519,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1711650728.1618724,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1711650728.1677909,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1711650728.1740093,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1711650728.2542024,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1711650730.1595674,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1711650730.1649294,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1711650730.2553461,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1711650732.1578951,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1711650732.254241,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1711650734.1583395,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1711650734.2551341,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1711650736.1583011,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1711650736.1649456,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1711650736.2545269,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1711650738.157846,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1711650738.2548621,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1711650740.1595635,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1711650740.1665912,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1711650740.2553828,"name":"offline","context":{"idset":"11439"}}
+{"timestamp":1711650742.1583023,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1711650742.1632717,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1711650742.2546661,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1711650744.1599185,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1711650744.1659622,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1711650744.170996,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1711650744.1758692,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1711650744.2600148,"name":"drain","context":{"idset":"11592","reason":"broker was unresponsive"}}
+{"timestamp":1711650746.1587319,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1711650746.1656764,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1711650746.1706169,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1711650746.1751504,"name":"drain","context":{"idset":"11595","reason":"broker was unresponsive"}}
+{"timestamp":1711650746.1752539,"name":"drain","context":{"idset":"11597","reason":"broker was unresponsive"}}
+{"timestamp":1711650746.1753533,"name":"drain","context":{"idset":"11599","reason":"broker was unresponsive"}}
+{"timestamp":1711650746.1754766,"name":"drain","context":{"idset":"11601","reason":"broker was unresponsive"}}
+{"timestamp":1711650746.2862706,"name":"drain","context":{"idset":"11604","reason":"broker was unresponsive"}}
+{"timestamp":1711650748.1579359,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1711650748.2566125,"name":"drain","context":{"idset":"11608","reason":"broker was unresponsive"}}
+{"timestamp":1711650750.1594191,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1711650750.1690311,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1711650750.1775641,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1711650750.1843369,"name":"drain","context":{"idset":"11603","reason":"broker was unresponsive"}}
+{"timestamp":1711650750.1844945,"name":"drain","context":{"idset":"11607","reason":"broker was unresponsive"}}
+{"timestamp":1711650750.2807338,"name":"drain","context":{"idset":"11612","reason":"broker was unresponsive"}}
+{"timestamp":1711650752.158036,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1711650752.1636662,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1711650752.1684825,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1711650752.2583437,"name":"drain","context":{"idset":"11615","reason":"broker was unresponsive"}}
+{"timestamp":1711650754.1587009,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1711650754.164547,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1711650754.1694586,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1711650754.1736095,"name":"drain","context":{"idset":"11611","reason":"broker was unresponsive"}}
+{"timestamp":1711650754.2588882,"name":"drain","context":{"idset":"11614","reason":"broker was unresponsive"}}
+{"timestamp":1711650756.1590137,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1711650756.1669726,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1711650756.1752253,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1711650756.2583568,"name":"drain","context":{"idset":"11618","reason":"broker was unresponsive"}}
+{"timestamp":1711650758.1584115,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1711650758.164093,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1711650758.2544663,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1711650760.1579711,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1711650760.254817,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1711650762.1584408,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1711650762.1636841,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1711650762.2556813,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1711650764.1589155,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1711650764.1645758,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1711650764.255178,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1711650766.1601546,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1711650766.1671538,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1711650766.2547362,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1711650768.1592472,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1711650768.1634393,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1711650768.1681724,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1711650768.2532909,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1711650770.2531426,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1711650774.2540088,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1711650786.1581087,"name":"drain","context":{"idset":"11623","reason":"broker was unresponsive"}}
+{"timestamp":1711650786.1582921,"name":"drain","context":{"idset":"11626","reason":"broker was unresponsive"}}
+{"timestamp":1711650786.158462,"name":"drain","context":{"idset":"11628","reason":"broker was unresponsive"}}
+{"timestamp":1711650786.1586142,"name":"drain","context":{"idset":"11631","reason":"broker was unresponsive"}}
+{"timestamp":1711650786.1587439,"name":"drain","context":{"idset":"11633","reason":"broker was unresponsive"}}
+{"timestamp":1711650786.3133466,"name":"drain","context":{"idset":"11635","reason":"broker was unresponsive"}}
+{"timestamp":1711650788.1567943,"name":"drain","context":{"idset":"11620","reason":"broker was unresponsive"}}
+{"timestamp":1711650788.1569326,"name":"drain","context":{"idset":"11625","reason":"broker was unresponsive"}}
+{"timestamp":1711650788.1570282,"name":"drain","context":{"idset":"11629","reason":"broker was unresponsive"}}
+{"timestamp":1711650788.1571083,"name":"drain","context":{"idset":"11632","reason":"broker was unresponsive"}}
+{"timestamp":1711650788.1571879,"name":"drain","context":{"idset":"11634","reason":"broker was unresponsive"}}
+{"timestamp":1711650788.1572692,"name":"drain","context":{"idset":"11636","reason":"broker was unresponsive"}}
+{"timestamp":1711650788.2682078,"name":"drain","context":{"idset":"11685","reason":"broker was unresponsive"}}
+{"timestamp":1711650790.1558547,"name":"drain","context":{"idset":"11619","reason":"broker was unresponsive"}}
+{"timestamp":1711650790.1559811,"name":"drain","context":{"idset":"11621","reason":"broker was unresponsive"}}
+{"timestamp":1711650790.1560779,"name":"drain","context":{"idset":"11622","reason":"broker was unresponsive"}}
+{"timestamp":1711650790.1561599,"name":"drain","context":{"idset":"11624","reason":"broker was unresponsive"}}
+{"timestamp":1711650790.1562457,"name":"drain","context":{"idset":"11627","reason":"broker was unresponsive"}}
+{"timestamp":1711650790.2545576,"name":"drain","context":{"idset":"11630","reason":"broker was unresponsive"}}
+{"timestamp":1711650792.1551094,"name":"drain","context":{"idset":"11669","reason":"broker was unresponsive"}}
+{"timestamp":1711650800.1572526,"name":"drain","context":{"idset":"11747","reason":"broker was unresponsive"}}
+{"timestamp":1711650800.157377,"name":"drain","context":{"idset":"11748","reason":"broker was unresponsive"}}
+{"timestamp":1711650800.1574733,"name":"drain","context":{"idset":"11751","reason":"broker was unresponsive"}}
+{"timestamp":1711650800.2567852,"name":"drain","context":{"idset":"11754","reason":"broker was unresponsive"}}
+{"timestamp":1711650802.1570642,"name":"drain","context":{"idset":"11752","reason":"broker was unresponsive"}}
+{"timestamp":1711650802.1571805,"name":"drain","context":{"idset":"11755","reason":"broker was unresponsive"}}
+{"timestamp":1711650802.1572621,"name":"drain","context":{"idset":"11756","reason":"broker was unresponsive"}}
+{"timestamp":1711650802.1573403,"name":"drain","context":{"idset":"11761","reason":"broker was unresponsive"}}
+{"timestamp":1711650802.1574311,"name":"drain","context":{"idset":"11762","reason":"broker was unresponsive"}}
+{"timestamp":1711650802.1575131,"name":"drain","context":{"idset":"11764","reason":"broker was unresponsive"}}
+{"timestamp":1711650802.1575909,"name":"drain","context":{"idset":"11847","reason":"broker was unresponsive"}}
+{"timestamp":1711650802.1576722,"name":"drain","context":{"idset":"11851","reason":"broker was unresponsive"}}
+{"timestamp":1711650802.2696717,"name":"drain","context":{"idset":"11858","reason":"broker was unresponsive"}}
+{"timestamp":1711650804.1562026,"name":"drain","context":{"idset":"11750","reason":"broker was unresponsive"}}
+{"timestamp":1711650804.1563218,"name":"drain","context":{"idset":"11757","reason":"broker was unresponsive"}}
+{"timestamp":1711650804.1564288,"name":"drain","context":{"idset":"11759","reason":"broker was unresponsive"}}
+{"timestamp":1711650804.1565218,"name":"drain","context":{"idset":"11760","reason":"broker was unresponsive"}}
+{"timestamp":1711650804.1566057,"name":"drain","context":{"idset":"11846","reason":"broker was unresponsive"}}
+{"timestamp":1711650804.1567078,"name":"drain","context":{"idset":"11849","reason":"broker was unresponsive"}}
+{"timestamp":1711650804.1568165,"name":"drain","context":{"idset":"11853","reason":"broker was unresponsive"}}
+{"timestamp":1711650804.1569026,"name":"drain","context":{"idset":"11860","reason":"broker was unresponsive"}}
+{"timestamp":1711650804.1569905,"name":"drain","context":{"idset":"11882","reason":"broker was unresponsive"}}
+{"timestamp":1711650804.2783105,"name":"drain","context":{"idset":"11885","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1620014,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1711650806.166317,"name":"drain","context":{"idset":"11845","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1664228,"name":"drain","context":{"idset":"11848","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1665132,"name":"drain","context":{"idset":"11850","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1665988,"name":"drain","context":{"idset":"11854","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.166683,"name":"drain","context":{"idset":"11855","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1667683,"name":"drain","context":{"idset":"11856","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1668537,"name":"drain","context":{"idset":"11857","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1669383,"name":"drain","context":{"idset":"11859","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1670232,"name":"drain","context":{"idset":"11877","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1671085,"name":"drain","context":{"idset":"11879","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1671929,"name":"drain","context":{"idset":"11880","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1672783,"name":"drain","context":{"idset":"11881","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1673639,"name":"drain","context":{"idset":"11883","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1674566,"name":"drain","context":{"idset":"11886","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.1675458,"name":"drain","context":{"idset":"11887","reason":"broker was unresponsive"}}
+{"timestamp":1711650806.354965,"name":"drain","context":{"idset":"11889","reason":"broker was unresponsive"}}
+{"timestamp":1711650808.1589382,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1711650808.1631656,"name":"drain","context":{"idset":"11878","reason":"broker was unresponsive"}}
+{"timestamp":1711650808.1632943,"name":"drain","context":{"idset":"11884","reason":"broker was unresponsive"}}
+{"timestamp":1711650808.1633935,"name":"drain","context":{"idset":"11888","reason":"broker was unresponsive"}}
+{"timestamp":1711650808.1635201,"name":"drain","context":{"idset":"11890","reason":"broker was unresponsive"}}
+{"timestamp":1711650808.2568216,"name":"drain","context":{"idset":"11891","reason":"broker was unresponsive"}}
+{"timestamp":1711650810.158088,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1711650810.1630783,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1711650810.254612,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1711650812.1584079,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1711650812.2555931,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1711650814.1575525,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1711650814.2547734,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1711650816.1580482,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1711650816.2544498,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1711650818.1591184,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1711650818.1684871,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1711650818.2545946,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1711650852.1828425,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1711650852.1885309,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1711650852.1931307,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1711650852.198679,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1711650852.2037587,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1711650852.2091391,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1711650852.2151663,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1711650852.2208824,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1711650852.2288601,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1711650852.2349234,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1711650852.2409079,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1711650852.2468169,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1711650852.251014,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1711650852.2660837,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1711650852.2670476,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1711650852.2680154,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1711650852.269002,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1711650852.2753248,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1711650852.2807143,"name":"offline","context":{"idset":"11669"}}
+{"timestamp":1711650852.2862625,"name":"offline","context":{"idset":"11685"}}
+{"timestamp":1711650854.2535098,"name":"offline","context":{"idset":"11715"}}
+{"timestamp":1711650866.3495402,"name":"offline","context":{"idset":"11747"}}
+{"timestamp":1711650866.3508751,"name":"offline","context":{"idset":"11748"}}
+{"timestamp":1711650866.3525736,"name":"offline","context":{"idset":"11751"}}
+{"timestamp":1711650866.3542411,"name":"offline","context":{"idset":"11752"}}
+{"timestamp":1711650866.3559053,"name":"offline","context":{"idset":"11754"}}
+{"timestamp":1711650866.3575766,"name":"offline","context":{"idset":"11755"}}
+{"timestamp":1711650866.359225,"name":"offline","context":{"idset":"11756"}}
+{"timestamp":1711650866.3608854,"name":"offline","context":{"idset":"11757"}}
+{"timestamp":1711650866.3625605,"name":"offline","context":{"idset":"11759"}}
+{"timestamp":1711650866.3642361,"name":"offline","context":{"idset":"11760"}}
+{"timestamp":1711650866.3658447,"name":"offline","context":{"idset":"11761"}}
+{"timestamp":1711650866.3675113,"name":"offline","context":{"idset":"11764"}}
+{"timestamp":1711650868.1687222,"name":"offline","context":{"idset":"11845"}}
+{"timestamp":1711650868.1742861,"name":"offline","context":{"idset":"11846"}}
+{"timestamp":1711650868.1787722,"name":"offline","context":{"idset":"11847"}}
+{"timestamp":1711650868.1831968,"name":"offline","context":{"idset":"11848"}}
+{"timestamp":1711650868.1875689,"name":"offline","context":{"idset":"11849"}}
+{"timestamp":1711650868.191417,"name":"offline","context":{"idset":"11850"}}
+{"timestamp":1711650868.1956494,"name":"offline","context":{"idset":"11851"}}
+{"timestamp":1711650868.2002418,"name":"offline","context":{"idset":"11853"}}
+{"timestamp":1711650868.205591,"name":"offline","context":{"idset":"11854"}}
+{"timestamp":1711650868.2116406,"name":"offline","context":{"idset":"11855"}}
+{"timestamp":1711650868.2163427,"name":"offline","context":{"idset":"11856"}}
+{"timestamp":1711650868.2210369,"name":"offline","context":{"idset":"11857"}}
+{"timestamp":1711650868.2258461,"name":"offline","context":{"idset":"11858"}}
+{"timestamp":1711650868.229804,"name":"offline","context":{"idset":"11859"}}
+{"timestamp":1711650868.2552269,"name":"offline","context":{"idset":"11860"}}
+{"timestamp":1711650870.1670222,"name":"offline","context":{"idset":"11877"}}
+{"timestamp":1711650870.1715856,"name":"offline","context":{"idset":"11878"}}
+{"timestamp":1711650870.1759703,"name":"offline","context":{"idset":"11879"}}
+{"timestamp":1711650870.1803725,"name":"offline","context":{"idset":"11880"}}
+{"timestamp":1711650870.184689,"name":"offline","context":{"idset":"11882"}}
+{"timestamp":1711650870.1891415,"name":"offline","context":{"idset":"11883"}}
+{"timestamp":1711650870.1941278,"name":"offline","context":{"idset":"11884"}}
+{"timestamp":1711650870.1977191,"name":"offline","context":{"idset":"11885"}}
+{"timestamp":1711650870.2035711,"name":"offline","context":{"idset":"11886"}}
+{"timestamp":1711650870.208194,"name":"offline","context":{"idset":"11887"}}
+{"timestamp":1711650870.2127616,"name":"offline","context":{"idset":"11888"}}
+{"timestamp":1711650870.2172053,"name":"offline","context":{"idset":"11889"}}
+{"timestamp":1711650870.2222164,"name":"offline","context":{"idset":"11890"}}
+{"timestamp":1711650870.3547785,"name":"offline","context":{"idset":"11891"}}
+{"timestamp":1711650878.2547717,"name":"offline","context":{"idset":"11762"}}
+{"timestamp":1711650882.2537034,"name":"offline","context":{"idset":"11881"}}
+{"timestamp":1711650914.2548065,"name":"offline","context":{"idset":"11750"}}
+{"timestamp":1711651396.240273,"name":"drain","context":{"idset":"10098","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651401.4053178,"name":"drain","context":{"idset":"10089","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651402.3096604,"name":"drain","context":{"idset":"10069","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651407.056253,"name":"drain","context":{"idset":"10088","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651408.5980887,"name":"drain","context":{"idset":"10099","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651415.2621243,"name":"drain","context":{"idset":"10094","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651424.0299106,"name":"drain","context":{"idset":"10091","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651430.1663055,"name":"drain","context":{"idset":"10097","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651431.6536202,"name":"drain","context":{"idset":"10087","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651432.9528325,"name":"drain","context":{"idset":"10090","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651436.1179669,"name":"drain","context":{"idset":"10084","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651441.8821375,"name":"drain","context":{"idset":"10092","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651447.6579275,"name":"drain","context":{"idset":"10093","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651448.6048837,"name":"drain","context":{"idset":"10100","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651452.9712451,"name":"drain","context":{"idset":"10070","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711651454.7350454,"name":"drain","context":{"idset":"10075","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1711653725.8084719,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1711216745.6805143,"reason":"broker was unresponsive"},"2":{"timestamp":1711216745.6806085,"reason":"broker was unresponsive"},"3":{"timestamp":1711216745.680656,"reason":"broker was unresponsive"},"4":{"timestamp":1711216745.6807451,"reason":"broker was unresponsive"},"5":{"timestamp":1711216745.6808176,"reason":"broker was unresponsive"},"6":{"timestamp":1711216745.6808617,"reason":"broker was unresponsive"},"7":{"timestamp":1711216745.6809049,"reason":"broker was unresponsive"},"8":{"timestamp":1711216745.6809447,"reason":"broker was unresponsive"},"9":{"timestamp":1711216745.6809835,"reason":"broker was unresponsive"},"10":{"timestamp":1711216745.6810231,"reason":"broker was unresponsive"},"11":{"timestamp":1711216745.6811044,"reason":"broker was unresponsive"},"12":{"timestamp":1711216745.6811471,"reason":"broker was unresponsive"},"13":{"timestamp":1711216745.6811881,"reason":"broker was unresponsive"},"14":{"timestamp":1711216745.6812286,"reason":"broker was unresponsive"},"15":{"timestamp":1711216745.6812708,"reason":"broker was unresponsive"},"16":{"timestamp":1711216745.6813264,"reason":"broker was unresponsive"},"17":{"timestamp":1711216745.681371,"reason":"broker was unresponsive"},"18":{"timestamp":1711216745.6814148,"reason":"broker was unresponsive"},"19":{"timestamp":1711216745.681462,"reason":"broker was unresponsive"},"20":{"timestamp":1711216745.6833789,"reason":"broker was unresponsive"},"21":{"timestamp":1711216745.6835806,"reason":"broker was unresponsive"},"22":{"timestamp":1711216745.6837616,"reason":"broker was unresponsive"},"23":{"timestamp":1711216745.6838651,"reason":"broker was unresponsive"},"24":{"timestamp":1711216745.6840565,"reason":"broker was unresponsive"},"25":{"timestamp":1711216745.6841562,"reason":"broker was unresponsive"},"26":{"timestamp":1711216745.6842697,"reason":"broker was unresponsive"},"27":{"timestamp":1711216745.6843822,"reason":"broker was unresponsive"},"28":{"timestamp":1711216745.6844778,"reason":"broker was unresponsive"},"29":{"timestamp":1711216745.6845732,"reason":"broker was unresponsive"},"30":{"timestamp":1711216745.6846764,"reason":"broker was unresponsive"},"31":{"timestamp":1711216745.6847744,"reason":"broker was unresponsive"},"32":{"timestamp":1711216745.6848724,"reason":"broker was unresponsive"},"33":{"timestamp":1711216745.6850114,"reason":"broker was unresponsive"},"34":{"timestamp":1711216745.6851158,"reason":"broker was unresponsive"},"35":{"timestamp":1711216745.6852152,"reason":"broker was unresponsive"},"36":{"timestamp":1711216745.6853395,"reason":"broker was unresponsive"},"37":{"timestamp":1711216745.6854444,"reason":"broker was unresponsive"},"38":{"timestamp":1711216745.6855464,"reason":"broker was unresponsive"},"39":{"timestamp":1711216745.6856468,"reason":"broker was unresponsive"},"40":{"timestamp":1711216745.68575,"reason":"broker was unresponsive"},"41":{"timestamp":1711216745.6858568,"reason":"broker was unresponsive"},"42":{"timestamp":1711216745.6859612,"reason":"broker was unresponsive"},"43":{"timestamp":1711216745.6861026,"reason":"broker was unresponsive"},"44":{"timestamp":1711216745.686213,"reason":"broker was unresponsive"},"45":{"timestamp":1711216745.6863275,"reason":"broker was unresponsive"},"46":{"timestamp":1711216745.6864519,"reason":"broker was unresponsive"},"47":{"timestamp":1711216745.6865566,"reason":"broker was unresponsive"},"48":{"timestamp":1711216745.6866629,"reason":"broker was unresponsive"},"49":{"timestamp":1711216745.6867788,"reason":"broker was unresponsive"},"50":{"timestamp":1711216745.6868875,"reason":"broker was unresponsive"},"51":{"timestamp":1711216745.6869948,"reason":"broker was unresponsive"},"52":{"timestamp":1711216745.6871991,"reason":"broker was unresponsive"},"53":{"timestamp":1711216745.6874077,"reason":"broker was unresponsive"},"54":{"timestamp":1711216745.6875894,"reason":"broker was unresponsive"},"55":{"timestamp":1711216745.6877069,"reason":"broker was unresponsive"},"56":{"timestamp":1711216745.6878214,"reason":"broker was unresponsive"},"57":{"timestamp":1711216745.6879315,"reason":"broker was unresponsive"},"58":{"timestamp":1711216745.6880448,"reason":"broker was unresponsive"},"59":{"timestamp":1711216745.6881592,"reason":"broker was unresponsive"},"60":{"timestamp":1711216746.2583418,"reason":"broker was unresponsive"},"69-76":{"timestamp":1711646931.6480982,"reason":""},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"348":{"timestamp":1711576490.2548447,"reason":"broker was unresponsive"},"431":{"timestamp":1711096934.358676,"reason":"nodediag failed pci"},"445-468,541-564":{"timestamp":1711560189.6111379,"reason":"CDU work TB"},"637":{"timestamp":1711642252.1616328,"reason":"broker was unresponsive"},"638":{"timestamp":1711642252.1617086,"reason":"broker was unresponsive"},"639":{"timestamp":1711642252.161751,"reason":"broker was unresponsive"},"640":{"timestamp":1711642252.1617904,"reason":"broker was unresponsive"},"641":{"timestamp":1711642252.1618268,"reason":"broker was unresponsive"},"642":{"timestamp":1711642252.1618621,"reason":"broker was unresponsive"},"643":{"timestamp":1711642252.1618974,"reason":"broker was unresponsive"},"644":{"timestamp":1711642252.1619322,"reason":"broker was unresponsive"},"646":{"timestamp":1711586857.201689,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"647":{"timestamp":1711587136.0002518,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"648":{"timestamp":1711587197.7047591,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"649":{"timestamp":1711587211.9606316,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"650":{"timestamp":1711587232.1780922,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"651":{"timestamp":1711587244.2040467,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"652":{"timestamp":1711587255.7559283,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"653":{"timestamp":1711642252.1619685,"reason":"broker was unresponsive"},"654":{"timestamp":1711642252.16201,"reason":"broker was unresponsive"},"655":{"timestamp":1711642252.162055,"reason":"broker was unresponsive"},"656":{"timestamp":1711642252.1620932,"reason":"broker was unresponsive"},"657":{"timestamp":1711642252.1621311,"reason":"broker was unresponsive"},"658":{"timestamp":1711642252.1621685,"reason":"broker was unresponsive"},"659":{"timestamp":1711642252.1622062,"reason":"broker was unresponsive"},"660":{"timestamp":1711642252.1622446,"reason":"broker was unresponsive"},"661":{"timestamp":1711642252.1622829,"reason":"broker was unresponsive"},"662":{"timestamp":1711642252.1623201,"reason":"broker was unresponsive"},"663":{"timestamp":1711642252.1623569,"reason":"broker was unresponsive"},"664":{"timestamp":1711642252.162394,"reason":"broker was unresponsive"},"665":{"timestamp":1711642252.1624658,"reason":"broker was unresponsive"},"666":{"timestamp":1711642252.1625173,"reason":"broker was unresponsive"},"667":{"timestamp":1711642258.156193,"reason":"broker was unresponsive"},"668":{"timestamp":1711642252.1625559,"reason":"broker was unresponsive"},"669":{"timestamp":1711642252.1625948,"reason":"broker was unresponsive"},"670":{"timestamp":1711642252.1626337,"reason":"broker was unresponsive"},"671":{"timestamp":1711642252.162673,"reason":"broker was unresponsive"},"672":{"timestamp":1711642252.1627119,"reason":"broker was unresponsive"},"673":{"timestamp":1711642252.1627512,"reason":"broker was unresponsive"},"674":{"timestamp":1711642252.1627915,"reason":"broker was unresponsive"},"675":{"timestamp":1711642252.1628311,"reason":"broker was unresponsive"},"676":{"timestamp":1711642252.1628704,"reason":"broker was unresponsive"},"677":{"timestamp":1711642258.1562793,"reason":"broker was unresponsive"},"678":{"timestamp":1711642258.156333,"reason":"broker was unresponsive"},"679":{"timestamp":1711642252.1629126,"reason":"broker was unresponsive"},"680":{"timestamp":1711642252.1629534,"reason":"broker was unresponsive"},"681":{"timestamp":1711642258.1563835,"reason":"broker was unresponsive"},"682":{"timestamp":1711642252.1629946,"reason":"broker was unresponsive"},"683":{"timestamp":1711642252.1630385,"reason":"broker was unresponsive"},"684":{"timestamp":1711642252.5597043,"reason":"broker was unresponsive"},"685":{"timestamp":1711604960.2562068,"reason":"broker was unresponsive"},"687":{"timestamp":1711604954.2552111,"reason":"broker was unresponsive"},"688":{"timestamp":1711606062.2558303,"reason":"broker was unresponsive"},"689":{"timestamp":1711604580.2552168,"reason":"broker was unresponsive"},"691":{"timestamp":1711604958.2564192,"reason":"broker was unresponsive"},"692":{"timestamp":1711606406.1556187,"reason":"broker was unresponsive"},"693":{"timestamp":1711636838.2555192,"reason":"broker was unresponsive"},"695":{"timestamp":1711604326.2549248,"reason":"broker was unresponsive"},"696":{"timestamp":1711606406.256216,"reason":"broker was unresponsive"},"697":{"timestamp":1711604392.2562771,"reason":"broker was unresponsive"},"698":{"timestamp":1711605716.2557297,"reason":"broker was unresponsive"},"699":{"timestamp":1711604948.2558951,"reason":"broker was unresponsive"},"700":{"timestamp":1711606868.2557678,"reason":"broker was unresponsive"},"701":{"timestamp":1711604340.2561386,"reason":"broker was unresponsive"},"702":{"timestamp":1711606408.2602317,"reason":"broker was unresponsive"},"703":{"timestamp":1711604918.2557685,"reason":"broker was unresponsive"},"704":{"timestamp":1711604590.2548223,"reason":"broker was unresponsive"},"705":{"timestamp":1711604968.2570915,"reason":"broker was unresponsive"},"707":{"timestamp":1711604908.2558758,"reason":"broker was unresponsive"},"686,690,694,706,708":{"timestamp":1711615908.1602454,"reason":"epilog failed for jobid fm9HsYwFjjd"},"709":{"timestamp":1711408620.9668899,"reason":"Cabinet power work needed -jrg"},"710":{"timestamp":1711409910.967391,"reason":"Cabinet power work needed -jrg"},"711":{"timestamp":1711409098.9674611,"reason":"Cabinet power work needed -jrg"},"712":{"timestamp":1711411606.9678445,"reason":"Cabinet power work needed -jrg"},"713":{"timestamp":1711408618.8669355,"reason":"Cabinet power work needed -jrg"},"715":{"timestamp":1711409132.9680476,"reason":"Cabinet power work needed -jrg"},"714,716":{"timestamp":1711564709.7804377,"reason":"Cabinet power work needed -jrg"},"718":{"timestamp":1711642210.1588371,"reason":"broker was unresponsive"},"719":{"timestamp":1711642210.1589563,"reason":"broker was unresponsive"},"720":{"timestamp":1711642210.1590128,"reason":"broker was unresponsive"},"721":{"timestamp":1711642210.1590643,"reason":"broker was unresponsive"},"722":{"timestamp":1711642210.1590991,"reason":"broker was unresponsive"},"723":{"timestamp":1711642210.1591821,"reason":"broker was unresponsive"},"724":{"timestamp":1711642210.1592195,"reason":"broker was unresponsive"},"725":{"timestamp":1711593222.2542431,"reason":"broker was unresponsive"},"726":{"timestamp":1711593312.2549782,"reason":"broker was unresponsive"},"727":{"timestamp":1711593332.2552569,"reason":"broker was unresponsive"},"728":{"timestamp":1711593356.2547033,"reason":"broker was unresponsive"},"729":{"timestamp":1711593414.254441,"reason":"broker was unresponsive"},"730":{"timestamp":1711593462.2552798,"reason":"broker was unresponsive"},"731":{"timestamp":1711593510.2550235,"reason":"broker was unresponsive"},"732":{"timestamp":1711593530.2558599,"reason":"broker was unresponsive"},"733":{"timestamp":1711642210.1592536,"reason":"broker was unresponsive"},"734":{"timestamp":1711486000.9676955,"reason":"broker was unresponsive"},"735":{"timestamp":1711642210.1592872,"reason":"broker was unresponsive"},"736":{"timestamp":1711642210.1593204,"reason":"broker was unresponsive"},"737":{"timestamp":1711642210.1593742,"reason":"broker was unresponsive"},"738":{"timestamp":1711642210.1594355,"reason":"broker was unresponsive"},"739":{"timestamp":1711642210.1594796,"reason":"broker was unresponsive"},"740":{"timestamp":1711642210.159517,"reason":"broker was unresponsive"},"741":{"timestamp":1711642210.1595554,"reason":"broker was unresponsive"},"742":{"timestamp":1711642210.1595943,"reason":"broker was unresponsive"},"743":{"timestamp":1711642210.1596365,"reason":"broker was unresponsive"},"744":{"timestamp":1711642210.1596746,"reason":"broker was unresponsive"},"745":{"timestamp":1711642210.1597111,"reason":"broker was unresponsive"},"746":{"timestamp":1711506258.2527435,"reason":"broker was unresponsive"},"747":{"timestamp":1711642210.1597559,"reason":"broker was unresponsive"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"},"749":{"timestamp":1711642210.1598115,"reason":"broker was unresponsive"},"750":{"timestamp":1711642210.1598501,"reason":"broker was unresponsive"},"751":{"timestamp":1711642210.159888,"reason":"broker was unresponsive"},"752":{"timestamp":1711642210.1599293,"reason":"broker was unresponsive"},"753":{"timestamp":1711642210.1599746,"reason":"broker was unresponsive"},"754":{"timestamp":1711642210.1600144,"reason":"broker was unresponsive"},"755":{"timestamp":1711642210.1600802,"reason":"broker was unresponsive"},"756":{"timestamp":1711642210.5039413,"reason":"broker was unresponsive"},"789-796":{"timestamp":1711461777.1299369,"reason":""},"810":{"timestamp":1711460371.1431019,"reason":""},"870":{"timestamp":1711576713.5907419,"reason":"epilog failed for jobid fm4cUUWjqfd"},"10069":{"timestamp":1711651402.3096604,"reason":"nodediag failed amdapu"},"10070":{"timestamp":1711651452.9712451,"reason":"nodediag failed amdapu"},"10071":{"timestamp":1711599836.2540793,"reason":"broker was unresponsive"},"10072":{"timestamp":1711599840.2556009,"reason":"broker was unresponsive"},"10073":{"timestamp":1711599774.2550957,"reason":"broker was unresponsive"},"10074":{"timestamp":1711599734.2550299,"reason":"broker was unresponsive"},"10075":{"timestamp":1711651454.7350454,"reason":"nodediag failed amdapu"},"10077":{"timestamp":1711598424.2559545,"reason":"broker was unresponsive"},"10078":{"timestamp":1711598420.2553101,"reason":"broker was unresponsive"},"10079":{"timestamp":1711598228.2556102,"reason":"broker was unresponsive"},"10080":{"timestamp":1711598162.2552085,"reason":"broker was unresponsive"},"10081":{"timestamp":1711597944.2539268,"reason":"broker was unresponsive"},"10082":{"timestamp":1711597598.2555497,"reason":"broker was unresponsive"},"10083":{"timestamp":1711592302.2542377,"reason":"broker was unresponsive"},"10084":{"timestamp":1711651436.1179669,"reason":"nodediag failed amdapu"},"10085":{"timestamp":1711599830.1547806,"reason":"broker was unresponsive"},"10086":{"timestamp":1711599830.2550619,"reason":"broker was unresponsive"},"10087":{"timestamp":1711651431.6536202,"reason":"nodediag failed amdapu"},"10088":{"timestamp":1711651407.056253,"reason":"nodediag failed amdapu"},"10089":{"timestamp":1711651401.4053178,"reason":"nodediag failed amdapu"},"10090":{"timestamp":1711651432.9528325,"reason":"nodediag failed amdapu"},"10091":{"timestamp":1711651424.0299106,"reason":"nodediag failed amdapu"},"10092":{"timestamp":1711651441.8821375,"reason":"nodediag failed amdapu"},"10093":{"timestamp":1711651447.6579275,"reason":"nodediag failed amdapu"},"10094":{"timestamp":1711651415.2621243,"reason":"nodediag failed amdapu"},"10095":{"timestamp":1711634633.9882171,"reason":""},"10096":{"timestamp":1711591630.2554383,"reason":"Partner node needs hardware action"},"10097":{"timestamp":1711651430.1663055,"reason":"nodediag failed amdapu"},"10098":{"timestamp":1711651396.240273,"reason":"nodediag failed amdapu"},"10099":{"timestamp":1711651408.5980887,"reason":"nodediag failed amdapu"},"10100":{"timestamp":1711651448.6048837,"reason":"nodediag failed amdapu"},"10154":{"timestamp":1711600478.1543531,"reason":"broker was unresponsive"},"10156":{"timestamp":1711600478.2542264,"reason":"broker was unresponsive"},"10157":{"timestamp":1711603028.2555907,"reason":"broker was unresponsive"},"10183":{"timestamp":1711600556.2563355,"reason":"broker was unresponsive"},"10421":{"timestamp":1711592576.2549477,"reason":"broker was unresponsive"},"10425":{"timestamp":1711592578.2555511,"reason":"broker was unresponsive"},"10428":{"timestamp":1711592580.1550264,"reason":"broker was unresponsive"},"10431":{"timestamp":1711592582.1546719,"reason":"broker was unresponsive"},"10432":{"timestamp":1711592582.1547906,"reason":"broker was unresponsive"},"10433":{"timestamp":1711592580.2556479,"reason":"broker was unresponsive"},"10436":{"timestamp":1711592582.2541113,"reason":"broker was unresponsive"},"11125":{"timestamp":1711126273.9625854,"reason":"ama fail"},"11126":{"timestamp":1711126278.6191533,"reason":"ama fail"},"11127":{"timestamp":1711126696.680208,"reason":"ama fail"},"11164":{"timestamp":1711547318.2542987,"reason":"broker was unresponsive"},"11179":{"timestamp":1711404572.9688854,"reason":"broker was unresponsive"},"11187":{"timestamp":1711401067.3461077,"reason":"reason=drain for Rabbit connection"},"11190":{"timestamp":1711399648.6946192,"reason":"bad partner"},"11213":{"timestamp":1711406762.8675106,"reason":"broker was unresponsive"},"11216":{"timestamp":1711406762.9678555,"reason":"broker was unresponsive"},"11230":{"timestamp":1711547616.1653366,"reason":"broker was unresponsive"},"11286":{"timestamp":1711650474.2557621,"reason":"broker was unresponsive"},"11290":{"timestamp":1711650472.2563207,"reason":"broker was unresponsive"},"11291":{"timestamp":1711650476.2556355,"reason":"broker was unresponsive"},"11295":{"timestamp":1711148027.6723082,"reason":"ama fail"},"11296":{"timestamp":1711148027.7718606,"reason":"ama fail"},"11297":{"timestamp":1711650480.2563696,"reason":"broker was unresponsive"},"11300":{"timestamp":1711650482.25527,"reason":"broker was unresponsive"},"11301-11316":{"timestamp":1711407860.45856,"reason":"FW/BIOS updates"},"11317":{"timestamp":1711326792.8705738,"reason":"epilog failed for jobid fka2yuNfWCo"},"11321":{"timestamp":1711650620.1558492,"reason":"broker was unresponsive"},"11323":{"timestamp":1711650620.2557573,"reason":"broker was unresponsive"},"11324":{"timestamp":1711650624.2569983,"reason":"broker was unresponsive"},"11325":{"timestamp":1711650622.1556945,"reason":"broker was unresponsive"},"11326":{"timestamp":1711650622.1558068,"reason":"broker was unresponsive"},"11330":{"timestamp":1711650622.2556465,"reason":"broker was unresponsive"},"11332":{"timestamp":1711650628.1561337,"reason":"broker was unresponsive"},"11335":{"timestamp":1711650628.1562393,"reason":"broker was unresponsive"},"11338":{"timestamp":1711650628.2561018,"reason":"broker was unresponsive"},"11340":{"timestamp":1711650632.155652,"reason":"broker was unresponsive"},"11341":{"timestamp":1711547616.1661725,"reason":"broker was unresponsive"},"11342":{"timestamp":1711117435.6741276,"reason":"broker was unresponsive"},"11343":{"timestamp":1711393750.8690577,"reason":"broker was unresponsive"},"11344":{"timestamp":1711650632.255296,"reason":"broker was unresponsive"},"11345":{"timestamp":1711393750.8692429,"reason":"broker was unresponsive"},"11347":{"timestamp":1711393750.8693571,"reason":"broker was unresponsive"},"11348":{"timestamp":1711650636.1550829,"reason":"broker was unresponsive"},"11351":{"timestamp":1711650634.2559328,"reason":"broker was unresponsive"},"11353":{"timestamp":1711650636.1552191,"reason":"broker was unresponsive"},"11354":{"timestamp":1711650638.1569469,"reason":"broker was unresponsive"},"11355":{"timestamp":1711650638.2565691,"reason":"broker was unresponsive"},"11357":{"timestamp":1711650636.2551146,"reason":"broker was unresponsive"},"11359":{"timestamp":1711650640.1562665,"reason":"broker was unresponsive"},"11360":{"timestamp":1711650640.1564136,"reason":"broker was unresponsive"},"11361":{"timestamp":1711650642.1560137,"reason":"broker was unresponsive"},"11364":{"timestamp":1711650642.255878,"reason":"broker was unresponsive"},"11366":{"timestamp":1711650640.2558939,"reason":"broker was unresponsive"},"11370":{"timestamp":1711650646.15609,"reason":"broker was unresponsive"},"11372":{"timestamp":1711650644.1551466,"reason":"broker was unresponsive"},"11374":{"timestamp":1711650644.1552951,"reason":"broker was unresponsive"},"11375":{"timestamp":1711650644.1554286,"reason":"broker was unresponsive"},"11377":{"timestamp":1711650646.2556603,"reason":"broker was unresponsive"},"11378":{"timestamp":1711650644.2543528,"reason":"broker was unresponsive"},"11380":{"timestamp":1711650648.2546284,"reason":"broker was unresponsive"},"11382":{"timestamp":1711650652.1564956,"reason":"broker was unresponsive"},"11383":{"timestamp":1711650650.1564307,"reason":"broker was unresponsive"},"11384":{"timestamp":1711650650.1565728,"reason":"broker was unresponsive"},"11386":{"timestamp":1711650652.1566389,"reason":"broker was unresponsive"},"11150,11388":{"timestamp":1711405119.195823,"reason":"reason=Draining for partner Rabbit"},"11389":{"timestamp":1711650650.1567149,"reason":"broker was unresponsive"},"11391":{"timestamp":1711650654.156652,"reason":"broker was unresponsive"},"11392":{"timestamp":1711650650.2555413,"reason":"broker was unresponsive"},"11393":{"timestamp":1711401846.9683437,"reason":"broker was unresponsive"},"11394":{"timestamp":1711117417.7733912,"reason":"Rabbit link Failure"},"11395":{"timestamp":1711650652.2568946,"reason":"broker was unresponsive"},"11397":{"timestamp":1711650656.1558905,"reason":"broker was unresponsive"},"11398":{"timestamp":1711650656.2558155,"reason":"broker was unresponsive"},"11400":{"timestamp":1711650660.1558089,"reason":"broker was unresponsive"},"11401":{"timestamp":1711650660.1559827,"reason":"broker was unresponsive"},"11402":{"timestamp":1711650660.156127,"reason":"broker was unresponsive"},"11403":{"timestamp":1711650660.156266,"reason":"broker was unresponsive"},"11405":{"timestamp":1711650660.1564043,"reason":"broker was unresponsive"},"11408":{"timestamp":1711650664.1576862,"reason":"broker was unresponsive"},"11409":{"timestamp":1711650664.1578712,"reason":"broker was unresponsive"},"11410":{"timestamp":1711650664.1580465,"reason":"broker was unresponsive"},"11411":{"timestamp":1711650664.1581936,"reason":"broker was unresponsive"},"11412":{"timestamp":1711650660.2544689,"reason":"broker was unresponsive"},"11413":{"timestamp":1711650666.2565143,"reason":"broker was unresponsive"},"11414":{"timestamp":1711650664.1583388,"reason":"broker was unresponsive"},"11415":{"timestamp":1711650662.2556987,"reason":"broker was unresponsive"},"11416":{"timestamp":1711650664.1584907,"reason":"broker was unresponsive"},"11417":{"timestamp":1711650664.1586566,"reason":"broker was unresponsive"},"11419":{"timestamp":1711650664.2576492,"reason":"broker was unresponsive"},"11420":{"timestamp":1711650668.1566327,"reason":"broker was unresponsive"},"11421":{"timestamp":1711650668.1567967,"reason":"broker was unresponsive"},"11422":{"timestamp":1711650670.1555846,"reason":"broker was unresponsive"},"11423":{"timestamp":1711650668.1569288,"reason":"broker was unresponsive"},"11427":{"timestamp":1711650670.2561576,"reason":"broker was unresponsive"},"11428":{"timestamp":1711650668.2563472,"reason":"broker was unresponsive"},"11429":{"timestamp":1711650672.1563449,"reason":"broker was unresponsive"},"11430":{"timestamp":1711650674.1572955,"reason":"broker was unresponsive"},"11431":{"timestamp":1711650674.1574328,"reason":"broker was unresponsive"},"11433":{"timestamp":1711650672.2569659,"reason":"broker was unresponsive"},"11435":{"timestamp":1711650674.1575251,"reason":"broker was unresponsive"},"11436":{"timestamp":1711650676.1566427,"reason":"broker was unresponsive"},"11437":{"timestamp":1711650674.1576116,"reason":"broker was unresponsive"},"11438":{"timestamp":1711650674.1576991,"reason":"broker was unresponsive"},"11439":{"timestamp":1711650674.2569783,"reason":"broker was unresponsive"},"11441":{"timestamp":1711650680.1577346,"reason":"broker was unresponsive"},"11442":{"timestamp":1711650676.2562418,"reason":"broker was unresponsive"},"11443":{"timestamp":1711650680.1579924,"reason":"broker was unresponsive"},"11444":{"timestamp":1711650680.1581464,"reason":"broker was unresponsive"},"11445":{"timestamp":1711650680.158303,"reason":"broker was unresponsive"},"11446":{"timestamp":1711650682.1555905,"reason":"broker was unresponsive"},"11447":{"timestamp":1711650680.2565355,"reason":"broker was unresponsive"},"11448":{"timestamp":1711123241.6739299,"reason":"ama fail"},"11449":{"timestamp":1711123241.674036,"reason":"ama fail"},"11450":{"timestamp":1711123241.6741452,"reason":"ama fail"},"11452":{"timestamp":1711650682.2549489,"reason":"broker was unresponsive"},"11453":{"timestamp":1711123247.6735187,"reason":"ama fail"},"11457":{"timestamp":1711650686.1949894,"reason":"broker was unresponsive"},"11459":{"timestamp":1711650684.1687078,"reason":"broker was unresponsive"},"11460":{"timestamp":1711123247.7819526,"reason":"ama fail"},"11461":{"timestamp":1711650686.2636492,"reason":"broker was unresponsive"},"11462":{"timestamp":1711650688.163574,"reason":"broker was unresponsive"},"11465":{"timestamp":1711650690.1642787,"reason":"broker was unresponsive"},"11467":{"timestamp":1711650688.2572289,"reason":"broker was unresponsive"},"11469":{"timestamp":1711650690.1644809,"reason":"broker was unresponsive"},"11470":{"timestamp":1711650692.2571719,"reason":"broker was unresponsive"},"11472":{"timestamp":1711650690.2565198,"reason":"broker was unresponsive"},"11473":{"timestamp":1711650696.1632161,"reason":"broker was unresponsive"},"11474":{"timestamp":1711650694.1702652,"reason":"broker was unresponsive"},"11475":{"timestamp":1711650694.1704659,"reason":"broker was unresponsive"},"11476":{"timestamp":1711650698.1710629,"reason":"broker was unresponsive"},"11479":{"timestamp":1711650698.1712503,"reason":"broker was unresponsive"},"11480":{"timestamp":1711650694.2565086,"reason":"broker was unresponsive"},"11481":{"timestamp":1711650696.1634502,"reason":"broker was unresponsive"},"11483":{"timestamp":1711650696.2568576,"reason":"broker was unresponsive"},"11484":{"timestamp":1711650698.1714315,"reason":"broker was unresponsive"},"11485":{"timestamp":1711650702.1866462,"reason":"broker was unresponsive"},"11487":{"timestamp":1711650702.1868434,"reason":"broker was unresponsive"},"11489":{"timestamp":1711650698.2583613,"reason":"broker was unresponsive"},"11491":{"timestamp":1711650700.1837032,"reason":"broker was unresponsive"},"11492":{"timestamp":1711650704.1696942,"reason":"broker was unresponsive"},"11493":{"timestamp":1711650700.2607677,"reason":"broker was unresponsive"},"11494":{"timestamp":1711650706.2562897,"reason":"broker was unresponsive"},"11495":{"timestamp":1711650702.1870284,"reason":"broker was unresponsive"},"11496":{"timestamp":1711650704.2572072,"reason":"broker was unresponsive"},"11497":{"timestamp":1711650702.2742891,"reason":"broker was unresponsive"},"11500":{"timestamp":1711650708.1696668,"reason":"broker was unresponsive"},"11501":{"timestamp":1711140105.6735849,"reason":"ama fail"},"11502":{"timestamp":1711140105.7732108,"reason":"ama fail"},"11508":{"timestamp":1711650708.2582283,"reason":"broker was unresponsive"},"11592":{"timestamp":1711650744.2600148,"reason":"broker was unresponsive"},"11149,11188-11189,11203-11204,11387,11488,11490,11593":{"timestamp":1711325310.3051066,"reason":"Rabbit link Failure"},"11595":{"timestamp":1711650746.1751504,"reason":"broker was unresponsive"},"11597":{"timestamp":1711650746.1752539,"reason":"broker was unresponsive"},"11599":{"timestamp":1711650746.1753533,"reason":"broker was unresponsive"},"11601":{"timestamp":1711650746.1754766,"reason":"broker was unresponsive"},"11603":{"timestamp":1711650750.1843369,"reason":"broker was unresponsive"},"11604":{"timestamp":1711650746.2862706,"reason":"broker was unresponsive"},"11346,11605":{"timestamp":1711326134.6213346,"reason":"GPU memory in use failing apdgpu.t"},"11607":{"timestamp":1711650750.1844945,"reason":"broker was unresponsive"},"11608":{"timestamp":1711650748.2566125,"reason":"broker was unresponsive"},"11611":{"timestamp":1711650754.1736095,"reason":"broker was unresponsive"},"11612":{"timestamp":1711650750.2807338,"reason":"broker was unresponsive"},"11614":{"timestamp":1711650754.2588882,"reason":"broker was unresponsive"},"11615":{"timestamp":1711650752.2583437,"reason":"broker was unresponsive"},"11618":{"timestamp":1711650756.2583568,"reason":"broker was unresponsive"},"11619":{"timestamp":1711650790.1558547,"reason":"broker was unresponsive"},"11620":{"timestamp":1711650788.1567943,"reason":"broker was unresponsive"},"11621":{"timestamp":1711650790.1559811,"reason":"broker was unresponsive"},"11622":{"timestamp":1711650790.1560779,"reason":"broker was unresponsive"},"11623":{"timestamp":1711650786.1581087,"reason":"broker was unresponsive"},"11624":{"timestamp":1711650790.1561599,"reason":"broker was unresponsive"},"11625":{"timestamp":1711650788.1569326,"reason":"broker was unresponsive"},"11626":{"timestamp":1711650786.1582921,"reason":"broker was unresponsive"},"11627":{"timestamp":1711650790.1562457,"reason":"broker was unresponsive"},"11628":{"timestamp":1711650786.158462,"reason":"broker was unresponsive"},"11629":{"timestamp":1711650788.1570282,"reason":"broker was unresponsive"},"11630":{"timestamp":1711650790.2545576,"reason":"broker was unresponsive"},"11631":{"timestamp":1711650786.1586142,"reason":"broker was unresponsive"},"11632":{"timestamp":1711650788.1571083,"reason":"broker was unresponsive"},"11633":{"timestamp":1711650786.1587439,"reason":"broker was unresponsive"},"11634":{"timestamp":1711650788.1571879,"reason":"broker was unresponsive"},"11635":{"timestamp":1711650786.3133466,"reason":"broker was unresponsive"},"11636":{"timestamp":1711650788.1572692,"reason":"broker was unresponsive"},"11669":{"timestamp":1711650792.1551094,"reason":"broker was unresponsive"},"11685":{"timestamp":1711650788.2682078,"reason":"broker was unresponsive"},"11715":{"timestamp":1711589508.2545855,"reason":"broker was unresponsive"},"11716":{"timestamp":1711589660.2541702,"reason":"broker was unresponsive"},"11747":{"timestamp":1711650800.1572526,"reason":"broker was unresponsive"},"11748":{"timestamp":1711650800.157377,"reason":"broker was unresponsive"},"11750":{"timestamp":1711650804.1562026,"reason":"broker was unresponsive"},"11751":{"timestamp":1711650800.1574733,"reason":"broker was unresponsive"},"11752":{"timestamp":1711650802.1570642,"reason":"broker was unresponsive"},"11754":{"timestamp":1711650800.2567852,"reason":"broker was unresponsive"},"11755":{"timestamp":1711650802.1571805,"reason":"broker was unresponsive"},"11756":{"timestamp":1711650802.1572621,"reason":"broker was unresponsive"},"11757":{"timestamp":1711650804.1563218,"reason":"broker was unresponsive"},"11759":{"timestamp":1711650804.1564288,"reason":"broker was unresponsive"},"11760":{"timestamp":1711650804.1565218,"reason":"broker was unresponsive"},"11761":{"timestamp":1711650802.1573403,"reason":"broker was unresponsive"},"11762":{"timestamp":1711650802.1574311,"reason":"broker was unresponsive"},"11764":{"timestamp":1711650802.1575131,"reason":"broker was unresponsive"},"11845":{"timestamp":1711650806.166317,"reason":"broker was unresponsive"},"11846":{"timestamp":1711650804.1566057,"reason":"broker was unresponsive"},"11847":{"timestamp":1711650802.1575909,"reason":"broker was unresponsive"},"11848":{"timestamp":1711650806.1664228,"reason":"broker was unresponsive"},"11849":{"timestamp":1711650804.1567078,"reason":"broker was unresponsive"},"11850":{"timestamp":1711650806.1665132,"reason":"broker was unresponsive"},"11851":{"timestamp":1711650802.1576722,"reason":"broker was unresponsive"},"11853":{"timestamp":1711650804.1568165,"reason":"broker was unresponsive"},"11854":{"timestamp":1711650806.1665988,"reason":"broker was unresponsive"},"11855":{"timestamp":1711650806.166683,"reason":"broker was unresponsive"},"11856":{"timestamp":1711650806.1667683,"reason":"broker was unresponsive"},"11857":{"timestamp":1711650806.1668537,"reason":"broker was unresponsive"},"11858":{"timestamp":1711650802.2696717,"reason":"broker was unresponsive"},"11859":{"timestamp":1711650806.1669383,"reason":"broker was unresponsive"},"11860":{"timestamp":1711650804.1569026,"reason":"broker was unresponsive"},"11877":{"timestamp":1711650806.1670232,"reason":"broker was unresponsive"},"11878":{"timestamp":1711650808.1631656,"reason":"broker was unresponsive"},"11879":{"timestamp":1711650806.1671085,"reason":"broker was unresponsive"},"11880":{"timestamp":1711650806.1671929,"reason":"broker was unresponsive"},"11881":{"timestamp":1711650806.1672783,"reason":"broker was unresponsive"},"11882":{"timestamp":1711650804.1569905,"reason":"broker was unresponsive"},"11883":{"timestamp":1711650806.1673639,"reason":"broker was unresponsive"},"11884":{"timestamp":1711650808.1632943,"reason":"broker was unresponsive"},"11885":{"timestamp":1711650804.2783105,"reason":"broker was unresponsive"},"11886":{"timestamp":1711650806.1674566,"reason":"broker was unresponsive"},"11887":{"timestamp":1711650806.1675458,"reason":"broker was unresponsive"},"11888":{"timestamp":1711650808.1633935,"reason":"broker was unresponsive"},"11889":{"timestamp":1711650806.354965,"reason":"broker was unresponsive"},"11890":{"timestamp":1711650808.1635201,"reason":"broker was unresponsive"},"11891":{"timestamp":1711650808.2568216,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1711653725.8478873,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1711653754.6419284,"name":"online","context":{"idset":"0"}}
+{"timestamp":1711653756.4849737,"name":"online","context":{"idset":"965,972,974,11264,11286,11289,11324,11327,11332,11352,11357,11377,11385,11402,11406,11415,11421,11423,11436,11446,11461,11481,11525,11536,11575,11584,11634,11681,11688,11694,11723,11733,11746,11750,11759,11800,11806,11813,11819,11838-11839,11844,11869,11879,11881,11884"}}
+{"timestamp":1711653756.590699,"name":"online","context":{"idset":"966,979,11253-11254,11259,11265,11269,11280,11282,11284,11292,11295,11300,11309,11319,11329,11346,11349,11354-11355,11363-11365,11372,11375,11382,11386,11388,11395,11399,11403-11405,11408,11447,11449-11451,11459,11463,11466-11467,11477,11480,11482-11483,11487,11503,11514,11524,11531,11534,11538,11548,11555,11557-11558,11580-11581,11585,11587,11589,11600-11601,11603,11616,11621,11638,11640,11651,11657,11662,11666-11667,11673,11677,11684,11689,11691-11692,11698,11703,11706,11708,11710,11713-11714,11716-11718,11724,11730,11734,11741,11744,11747,11756,11763,11783,11807-11811,11821-11822,11825,11829-11830,11835,11843,11845,11849,11858,11865-11866,11891"}}
+{"timestamp":1711653756.707032,"name":"online","context":{"idset":"967-971,973,975-978,980,10157,11255,11257-11258,11260-11263,11266-11268,11270,11272,11274-11279,11281,11285,11287-11288,11291,11293-11294,11296-11299,11301,11303,11305-11307,11310-11314,11316,11320-11323,11325-11326,11328,11330-11331,11334-11336,11338-11345,11347-11348,11350-11351,11353,11356,11358-11362,11366-11368,11371,11376,11378-11379,11381,11383-11384,11387,11389,11391-11394,11397-11398,11400-11401,11409-11414,11416-11417,11420,11422,11424-11426,11428-11432,11434-11435,11437-11438,11440-11445,11448,11452-11458,11460,11462,11464-11465,11468-11469,11472-11476,11478-11479,11484-11486,11488-11491,11494,11496-11502,11504-11510,11512-11513,11515-11518,11520-11521,11523,11527-11530,11532-11533,11537,11539-11540,11542-11544,11546-11547,11549,11551-11554,11556,11559-11569,11571-11574,11576-11579,11582-11583,11588,11590,11592-11599,11602,11604-11606,11608-11614,11618,11622-11626,11628,11632-11633,11637,11639,11641-11650,11653-11656,11658-11661,11663,11665,11668-11672,11674-11676,11678-11679,11683,11685,11687,11690,11695-11697,11699-11702,11704-11705,11707,11709,11712,11715,11719-11722,11725-11729,11731-11732,11735-11740,11742-11743,11745,11748-11749,11751-11755,11757-11758,11760-11762,11766-11770,11772-11782,11784-11785,11787,11790,11792-11799,11801,11803-11805,11812,11814-11815,11817-11818,11820,11823-11824,11826-11828,11832-11834,11836-11837,11840-11842,11847-11848,11850-11857,11859-11864,11868,11870-11872,11874,11876-11878,11880,11883,11885-11886,11888-11890,11892"}}
+{"timestamp":1711653756.8101058,"name":"online","context":{"idset":"10183,11271,11273,11283,11290,11302,11304,11308,11315,11333,11337,11369-11370,11373,11380,11390,11396,11407,11419,11427,11433,11470,11492,11495,11519,11522,11526,11535,11541,11550,11570,11586,11591,11607,11615,11617,11620,11627,11629-11631,11635-11636,11664,11682,11686,11693,11764-11765,11771,11786,11788,11831,11846,11867,11873,11875,11882,11887"}}
+{"timestamp":1711653756.956208,"name":"online","context":{"idset":"11374,11418,11493,11545,11619,11652"}}
+{"timestamp":1711653757.1183476,"name":"online","context":{"idset":"109,117,123,254,11511"}}
+{"timestamp":1711653757.3249066,"name":"online","context":{"idset":"10207,10561"}}
+{"timestamp":1711653757.4744265,"name":"online","context":{"idset":"29,59,10110,10226,10398,10402"}}
+{"timestamp":1711653757.5804875,"name":"online","context":{"idset":"217,585,10103,10133,10181,10225,10389,10401"}}
+{"timestamp":1711653757.6868746,"name":"online","context":{"idset":"34,55,61-62,65-67,77-82,84-85,87-104,106-108,110-113,115-116,118,120,122,125,127-136,138-150,152-153,156-157,159-176,178-190,192-206,208-216,218-253,255-264,267-276,278-280,282-292,294-301,303-305,307-315,317-334,336-346,349-358,360-373,376-377,379-384,386-390,392,394-404,407-412,414-418,420-421,423-430,432-434,436-444,469-480,482-499,501-502,504-513,515-522,524-540,565-578,580-582,584,586-609,611-617,619-620,10118,10131,10149,10161-10162,10188,10210"}}
+{"timestamp":1711653757.8361263,"name":"online","context":{"idset":"63,83,86,105,114,119,124,126,137,151,154-155,158,177,191,207,265-266,277,281,293,302,306,316,335,347,359,374-375,378,385,391,393,405-406,413,419,422,435,481,500,503,514,523,579,583,610,618,627-628,630,10140,10150,10160,10177,10190,10397,10444"}}
+{"timestamp":1711653757.93751,"name":"online","context":{"idset":"621-626,629,631-636,10148,10159,10168,10173,10221"}}
+{"timestamp":1711653758.0449407,"name":"online","context":{"idset":"9,12,19,24,27,31,64,809,10130,10151,10171,10192,10211,10224"}}
+{"timestamp":1711653758.1470537,"name":"online","context":{"idset":"4,21,35,44-45,10115,10122,10128,10152,10163,10176,10194,10200-10203,10227"}}
+{"timestamp":1711653758.2555518,"name":"online","context":{"idset":"10101,10111,10120,10132,10141,10143,10146,10158,10164,10178,10184-10185,10196-10197,10199,10204,10208-10209,10218-10220,10441"}}
+{"timestamp":1711653758.3647249,"name":"online","context":{"idset":"10102,10107-10108,10114,10125-10126,10134,10144-10145,10174-10175,10179,10182,10186-10187,10191,10195,10198,10212,10217,10222-10223,10228,10451"}}
+{"timestamp":1711653758.4743614,"name":"online","context":{"idset":"1-3,5-8,10-11,14,16-18,20,23,26,32-33,37-38,41-43,47-48,51,54,56-57,60,68,10104-10106,10109,10112-10113,10116-10117,10121,10136,10138-10139,10142,10147,10165-10167,10169-10170,10172,10189,10193,10213-10216,10439,10442"}}
+{"timestamp":1711653758.5756998,"name":"online","context":{"idset":"13,15,22,25,28,30,36,39-40,46,49-50,52-53,58,873,10123-10124,10129,10135,10137,10180,10452"}}
+{"timestamp":1711653758.6896696,"name":"online","context":{"idset":"875,10119,10127,10438,10440"}}
+{"timestamp":1711653758.8969288,"name":"online","context":{"idset":"874,10445"}}
+{"timestamp":1711653759.0471833,"name":"online","context":{"idset":"10447-10448"}}
+{"timestamp":1711653759.1700256,"name":"online","context":{"idset":"10450"}}
+{"timestamp":1711653759.646874,"name":"online","context":{"idset":"10361"}}
+{"timestamp":1711653759.8139243,"name":"online","context":{"idset":"10372"}}
+{"timestamp":1711653760.0262375,"name":"online","context":{"idset":"10365,10368"}}
+{"timestamp":1711653760.2578747,"name":"online","context":{"idset":"10359-10360"}}
+{"timestamp":1711653760.802103,"name":"online","context":{"idset":"10416"}}
+{"timestamp":1711653760.9639974,"name":"online","context":{"idset":"10410"}}
+{"timestamp":1711653761.1340206,"name":"online","context":{"idset":"10092,10406-10407,10417,10419"}}
+{"timestamp":1711653761.6696649,"name":"online","context":{"idset":"10089-10090"}}
+{"timestamp":1711653761.7800419,"name":"online","context":{"idset":"10091"}}
+{"timestamp":1711653762.0290334,"name":"online","context":{"idset":"10087-10088"}}
+{"timestamp":1711653762.2805536,"name":"online","context":{"idset":"10098"}}
+{"timestamp":1711653762.8320694,"name":"online","context":{"idset":"10099-10100"}}
+{"timestamp":1711653762.981004,"name":"online","context":{"idset":"10094"}}
+{"timestamp":1711653763.1078422,"name":"online","context":{"idset":"10093,10097"}}
+{"timestamp":1711653827.4929831,"name":"online","context":{"idset":"10458"}}
+{"timestamp":1711653832.9647021,"name":"drain","context":{"idset":"10076","reason":"broker was unresponsive"}}
+{"timestamp":1711653832.9648218,"name":"drain","context":{"idset":"10374","reason":"broker was unresponsive"}}
+{"timestamp":1711653832.9648952,"name":"drain","context":{"idset":"10375","reason":"broker was unresponsive"}}
+{"timestamp":1711653832.9649596,"name":"drain","context":{"idset":"10377","reason":"broker was unresponsive"}}
+{"timestamp":1711653832.9650226,"name":"drain","context":{"idset":"10379","reason":"broker was unresponsive"}}
+{"timestamp":1711653832.9650848,"name":"drain","context":{"idset":"10380","reason":"broker was unresponsive"}}
+{"timestamp":1711653832.9651468,"name":"drain","context":{"idset":"10462","reason":"broker was unresponsive"}}
+{"timestamp":1711653832.9652078,"name":"drain","context":{"idset":"10463","reason":"broker was unresponsive"}}
+{"timestamp":1711653832.9652689,"name":"drain","context":{"idset":"10464","reason":"broker was unresponsive"}}
+{"timestamp":1711653832.9653337,"name":"drain","context":{"idset":"10467","reason":"broker was unresponsive"}}
+{"timestamp":1711653833.1115868,"name":"drain","context":{"idset":"10553","reason":"broker was unresponsive"}}
+{"timestamp":1711653834.9610977,"name":"drain","context":{"idset":"10550","reason":"broker was unresponsive"}}
+{"timestamp":1711653834.9612203,"name":"drain","context":{"idset":"10551","reason":"broker was unresponsive"}}
+{"timestamp":1711653834.9612973,"name":"drain","context":{"idset":"10552","reason":"broker was unresponsive"}}
+{"timestamp":1711653834.9613669,"name":"drain","context":{"idset":"10554","reason":"broker was unresponsive"}}
+{"timestamp":1711653834.9614487,"name":"drain","context":{"idset":"10555","reason":"broker was unresponsive"}}
+{"timestamp":1711653835.0605469,"name":"drain","context":{"idset":"10556","reason":"broker was unresponsive"}}
+{"timestamp":1711654516.9775007,"name":"online","context":{"idset":"10082"}}
+{"timestamp":1711654528.3864894,"name":"online","context":{"idset":"10381"}}
+{"timestamp":1711654530.1733875,"name":"online","context":{"idset":"10081,10385"}}
+{"timestamp":1711654532.5658596,"name":"online","context":{"idset":"10384,10560"}}
+{"timestamp":1711654534.7652142,"name":"online","context":{"idset":"10084"}}
+{"timestamp":1711654542.4507637,"name":"online","context":{"idset":"10557"}}
+{"timestamp":1711654546.5649676,"name":"online","context":{"idset":"10559"}}
+{"timestamp":1711654547.1376653,"name":"online","context":{"idset":"10564"}}
+{"timestamp":1711654547.5232611,"name":"online","context":{"idset":"10563"}}
+{"timestamp":1711654549.8955102,"name":"online","context":{"idset":"10562"}}
+{"timestamp":1711654550.201447,"name":"online","context":{"idset":"10558"}}
+{"timestamp":1711656464.7456715,"name":"drain","context":{"idset":"10102","reason":"rvs dumped core","overwrite":0}}
+{"timestamp":1711659795.061769,"name":"drain","context":{"idset":"11702","reason":"broker was unresponsive"}}
+{"timestamp":1711659857.0590909,"name":"offline","context":{"idset":"11702"}}
+{"timestamp":1711659885.061022,"name":"drain","context":{"idset":"11637","reason":"broker was unresponsive"}}
+{"timestamp":1711659892.0492222,"name":"drain","context":{"idset":"11157","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659899.9074621,"name":"drain","context":{"idset":"11158","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659909.3810983,"name":"drain","context":{"idset":"11159","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659917.3405585,"name":"drain","context":{"idset":"11160","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659924.5419488,"name":"drain","context":{"idset":"11161","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659931.3902264,"name":"drain","context":{"idset":"11162","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659939.3624558,"name":"drain","context":{"idset":"11163","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659950.8494904,"name":"offline","context":{"idset":"11637"}}
+{"timestamp":1711659950.9614942,"name":"drain","context":{"idset":"11517","reason":"broker was unresponsive"}}
+{"timestamp":1711659951.0613308,"name":"drain","context":{"idset":"11536","reason":"broker was unresponsive"}}
+{"timestamp":1711659959.8101106,"name":"drain","context":{"idset":"11165","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659968.5539007,"name":"drain","context":{"idset":"11166","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659975.7040341,"name":"drain","context":{"idset":"11167","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659983.0627556,"name":"drain","context":{"idset":"11168","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659991.2328937,"name":"drain","context":{"idset":"11169","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711659997.8838148,"name":"drain","context":{"idset":"11170","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711660005.6104357,"name":"drain","context":{"idset":"11171","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711660013.3725522,"name":"drain","context":{"idset":"11172","reason":"shutting down to move blades - wendy","overwrite":0}}
+{"timestamp":1711660013.3739777,"name":"offline","context":{"idset":"11517"}}
+{"timestamp":1711660013.3753695,"name":"offline","context":{"idset":"11536"}}
+{"timestamp":1711660124.9618423,"name":"drain","context":{"idset":"10557","reason":"broker was unresponsive"}}
+{"timestamp":1711660124.9619589,"name":"drain","context":{"idset":"10558","reason":"broker was unresponsive"}}
+{"timestamp":1711660124.9620354,"name":"drain","context":{"idset":"10559","reason":"broker was unresponsive"}}
+{"timestamp":1711660124.962131,"name":"drain","context":{"idset":"10560","reason":"broker was unresponsive"}}
+{"timestamp":1711660124.9622042,"name":"drain","context":{"idset":"10561","reason":"broker was unresponsive"}}
+{"timestamp":1711660124.9622781,"name":"drain","context":{"idset":"10562","reason":"broker was unresponsive"}}
+{"timestamp":1711660124.9623649,"name":"drain","context":{"idset":"10563","reason":"broker was unresponsive"}}
+{"timestamp":1711660125.0727112,"name":"drain","context":{"idset":"10564","reason":"broker was unresponsive"}}
+{"timestamp":1711660186.9660041,"name":"offline","context":{"idset":"10557"}}
+{"timestamp":1711660186.974066,"name":"offline","context":{"idset":"10558"}}
+{"timestamp":1711660187.0592868,"name":"offline","context":{"idset":"10559"}}
+{"timestamp":1711660188.9681492,"name":"offline","context":{"idset":"10560"}}
+{"timestamp":1711660188.9767673,"name":"offline","context":{"idset":"10561"}}
+{"timestamp":1711660188.9909101,"name":"offline","context":{"idset":"10562"}}
+{"timestamp":1711660189.0605257,"name":"offline","context":{"idset":"10563"}}
+{"timestamp":1711660191.0601063,"name":"offline","context":{"idset":"10564"}}
+{"timestamp":1711660461.1080077,"name":"online","context":{"idset":"11680"}}
+{"timestamp":1711660486.6421392,"name":"drain","context":{"idset":"10158","reason":"draining to repair blade - wendy","overwrite":0}}
+{"timestamp":1711660636.9673107,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1711660637.0601275,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1711661236.4636974,"name":"drain","context":{"idset":"11765-11780","reason":"Per Brick - Rabbits Off in x19xx","overwrite":0}}
+{"timestamp":1711661477.244117,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1711661538.4584198,"name":"offline","context":{"idset":"10084"}}
+{"timestamp":1711662583.0842521,"name":"online","context":{"idset":"10072"}}
+{"timestamp":1711662626.7690742,"name":"online","context":{"idset":"10071"}}
+{"timestamp":1711662783.98808,"name":"online","context":{"idset":"10080"}}
+{"timestamp":1711662801.4923022,"name":"offline","context":{"idset":"873"}}
+{"timestamp":1711662806.4539137,"name":"online","context":{"idset":"10079"}}
+{"timestamp":1711662951.2999039,"name":"online","context":{"idset":"10073"}}
+{"timestamp":1711662977.8272679,"name":"online","context":{"idset":"10074"}}
+{"timestamp":1711663272.7887158,"name":"drain","context":{"idset":"10587,10590","reason":"core dumps -KK","overwrite":0}}
+{"timestamp":1711663344.7754133,"name":"online","context":{"idset":"10069"}}
+{"timestamp":1711663377.1481729,"name":"online","context":{"idset":"10070"}}
+{"timestamp":1711663460.6123624,"name":"undrain","context":{"idset":"10587,10590"}}
+{"timestamp":1711663526.290566,"name":"drain","context":{"idset":"10137,10140,11566,11587","reason":"core dumps -KK","overwrite":0}}
+{"timestamp":1711663617.0607617,"name":"drain","context":{"idset":"11704","reason":"broker was unresponsive"}}
+{"timestamp":1711663679.1858916,"name":"offline","context":{"idset":"11704"}}
+{"timestamp":1711663706.9630723,"name":"drain","context":{"idset":"965","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9631405,"name":"drain","context":{"idset":"966","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.963171,"name":"drain","context":{"idset":"967","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9631999,"name":"drain","context":{"idset":"968","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9632339,"name":"drain","context":{"idset":"969","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9632568,"name":"drain","context":{"idset":"970","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9632795,"name":"drain","context":{"idset":"971","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9633036,"name":"drain","context":{"idset":"972","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9633393,"name":"drain","context":{"idset":"973","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9633627,"name":"drain","context":{"idset":"974","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9633861,"name":"drain","context":{"idset":"975","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9634261,"name":"drain","context":{"idset":"976","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9634516,"name":"drain","context":{"idset":"977","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9634771,"name":"drain","context":{"idset":"978","reason":"broker was unresponsive"}}
+{"timestamp":1711663706.9635053,"name":"drain","context":{"idset":"979","reason":"broker was unresponsive"}}
+{"timestamp":1711663707.147469,"name":"drain","context":{"idset":"980","reason":"broker was unresponsive"}}
+{"timestamp":1711664552.0380454,"name":"drain","context":{"idset":"11717-11732","reason":"Per Brick - Rabbits Off in x19xx","overwrite":0}}
+{"timestamp":1711666915.3795209,"name":"online","context":{"idset":"10068"}}
+{"timestamp":1711666959.4738243,"name":"online","context":{"idset":"10067"}}
+{"timestamp":1711666992.1131492,"name":"online","context":{"idset":"10066"}}
+{"timestamp":1711667019.6706176,"name":"online","context":{"idset":"10065"}}
+{"timestamp":1711667251.2030878,"name":"drain","context":{"idset":"809","reason":"broker was unresponsive"}}
+{"timestamp":1711667258.6265669,"name":"online","context":{"idset":"10063"}}
+{"timestamp":1711667313.0598228,"name":"offline","context":{"idset":"809"}}
+{"timestamp":1711667773.0594749,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1711667909.9296641,"name":"online","context":{"idset":"10062"}}
+{"timestamp":1711667910.529552,"name":"online","context":{"idset":"10061"}}
+{"timestamp":1711668088.9625077,"name":"drain","context":{"idset":"10361","reason":"broker was unresponsive"}}
+{"timestamp":1711668088.9626274,"name":"drain","context":{"idset":"10365","reason":"broker was unresponsive"}}
+{"timestamp":1711668088.9627261,"name":"drain","context":{"idset":"10368","reason":"broker was unresponsive"}}
+{"timestamp":1711668089.0613325,"name":"drain","context":{"idset":"10372","reason":"broker was unresponsive"}}
+{"timestamp":1711668113.3647482,"name":"online","context":{"idset":"10059"}}
+{"timestamp":1711668150.9679074,"name":"offline","context":{"idset":"10361"}}
+{"timestamp":1711668150.9777699,"name":"offline","context":{"idset":"10365"}}
+{"timestamp":1711668150.9870763,"name":"offline","context":{"idset":"10368"}}
+{"timestamp":1711668151.2100959,"name":"offline","context":{"idset":"10372"}}
+{"timestamp":1711668211.0619664,"name":"drain","context":{"idset":"10059","reason":"broker was unresponsive"}}
+{"timestamp":1711668218.2995338,"name":"online","context":{"idset":"10057"}}
+{"timestamp":1711668273.0615561,"name":"offline","context":{"idset":"10059"}}
+{"timestamp":1711668315.177243,"name":"online","context":{"idset":"10053"}}
+{"timestamp":1711668438.9624472,"name":"drain","context":{"idset":"65","reason":"broker was unresponsive"}}
+{"timestamp":1711668439.0630105,"name":"drain","context":{"idset":"67","reason":"broker was unresponsive"}}
+{"timestamp":1711668444.9627461,"name":"drain","context":{"idset":"61","reason":"broker was unresponsive"}}
+{"timestamp":1711668445.0625052,"name":"drain","context":{"idset":"63","reason":"broker was unresponsive"}}
+{"timestamp":1711668457.0617182,"name":"drain","context":{"idset":"80","reason":"broker was unresponsive"}}
+{"timestamp":1711668462.9610252,"name":"drain","context":{"idset":"78","reason":"broker was unresponsive"}}
+{"timestamp":1711668462.9611192,"name":"drain","context":{"idset":"82","reason":"broker was unresponsive"}}
+{"timestamp":1711668463.0615458,"name":"drain","context":{"idset":"84","reason":"broker was unresponsive"}}
+{"timestamp":1711668502.963347,"name":"offline","context":{"idset":"67"}}
+{"timestamp":1711668503.0621645,"name":"drain","context":{"idset":"875","reason":"broker was unresponsive"}}
+{"timestamp":1711668505.0593822,"name":"offline","context":{"idset":"65"}}
+{"timestamp":1711668506.966094,"name":"offline","context":{"idset":"63"}}
+{"timestamp":1711668506.9728317,"name":"drain","context":{"idset":"62","reason":"broker was unresponsive"}}
+{"timestamp":1711668506.9729121,"name":"drain","context":{"idset":"64","reason":"broker was unresponsive"}}
+{"timestamp":1711668506.9729731,"name":"drain","context":{"idset":"66","reason":"broker was unresponsive"}}
+{"timestamp":1711668506.9730346,"name":"drain","context":{"idset":"68","reason":"broker was unresponsive"}}
+{"timestamp":1711668507.0706048,"name":"drain","context":{"idset":"874","reason":"broker was unresponsive"}}
+{"timestamp":1711668509.0601213,"name":"offline","context":{"idset":"61"}}
+{"timestamp":1711668522.1719112,"name":"offline","context":{"idset":"80"}}
+{"timestamp":1711668525.0606718,"name":"offline","context":{"idset":"82"}}
+{"timestamp":1711668526.9621451,"name":"drain","context":{"idset":"77","reason":"broker was unresponsive"}}
+{"timestamp":1711668526.96223,"name":"drain","context":{"idset":"79","reason":"broker was unresponsive"}}
+{"timestamp":1711668526.9622741,"name":"drain","context":{"idset":"81","reason":"broker was unresponsive"}}
+{"timestamp":1711668527.0620673,"name":"drain","context":{"idset":"83","reason":"broker was unresponsive"}}
+{"timestamp":1711668528.9651561,"name":"offline","context":{"idset":"78"}}
+{"timestamp":1711668529.2192309,"name":"offline","context":{"idset":"84"}}
+{"timestamp":1711668548.9821446,"name":"offline","context":{"idset":"965"}}
+{"timestamp":1711668548.9919443,"name":"offline","context":{"idset":"966"}}
+{"timestamp":1711668549.0159955,"name":"offline","context":{"idset":"967"}}
+{"timestamp":1711668549.0256727,"name":"offline","context":{"idset":"968"}}
+{"timestamp":1711668549.0348721,"name":"offline","context":{"idset":"969"}}
+{"timestamp":1711668549.0452664,"name":"offline","context":{"idset":"970"}}
+{"timestamp":1711668549.0605979,"name":"offline","context":{"idset":"971"}}
+{"timestamp":1711668549.0707712,"name":"offline","context":{"idset":"972"}}
+{"timestamp":1711668549.0810466,"name":"offline","context":{"idset":"974"}}
+{"timestamp":1711668549.101656,"name":"offline","context":{"idset":"975"}}
+{"timestamp":1711668549.1050115,"name":"offline","context":{"idset":"976"}}
+{"timestamp":1711668549.1199934,"name":"offline","context":{"idset":"977"}}
+{"timestamp":1711668549.1296315,"name":"offline","context":{"idset":"978"}}
+{"timestamp":1711668549.1401618,"name":"offline","context":{"idset":"979"}}
+{"timestamp":1711668549.1489778,"name":"offline","context":{"idset":"980"}}
+{"timestamp":1711668568.9785724,"name":"offline","context":{"idset":"62"}}
+{"timestamp":1711668568.9942276,"name":"offline","context":{"idset":"64"}}
+{"timestamp":1711668569.0073094,"name":"offline","context":{"idset":"66"}}
+{"timestamp":1711668569.0196214,"name":"offline","context":{"idset":"68"}}
+{"timestamp":1711668569.0271316,"name":"offline","context":{"idset":"874"}}
+{"timestamp":1711668569.0593936,"name":"offline","context":{"idset":"875"}}
+{"timestamp":1711668588.9667981,"name":"offline","context":{"idset":"79"}}
+{"timestamp":1711668588.974663,"name":"offline","context":{"idset":"81"}}
+{"timestamp":1711668588.9821775,"name":"offline","context":{"idset":"83"}}
+{"timestamp":1711668589.064811,"name":"offline","context":{"idset":"973"}}
+{"timestamp":1711668600.9660704,"name":"drain","context":{"idset":"62,64,66,68,77,79,81,83","reason":"epilog failed for jobid fmHfa9LYeLw","overwrite":0}}
+{"timestamp":1711668601.0603831,"name":"offline","context":{"idset":"77"}}
+{"timestamp":1711668729.1260347,"name":"online","context":{"idset":"10064"}}
+{"timestamp":1711668755.4346271,"name":"drain","context":{"idset":"10381","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711668789.0508697,"name":"drain","context":{"idset":"10360","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711668795.1309788,"name":"drain","context":{"idset":"10359","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711668961.0618374,"name":"drain","context":{"idset":"11803","reason":"broker was unresponsive"}}
+{"timestamp":1711669023.190449,"name":"offline","context":{"idset":"11803"}}
+{"timestamp":1711669099.0599494,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1711669868.5219145,"name":"online","context":{"idset":"10058"}}
+{"timestamp":1711669897.2096248,"name":"drain","context":{"idset":"10061","reason":"nodediag failed pci amdapu","overwrite":0}}
+{"timestamp":1711669954.9420981,"name":"drain","context":{"idset":"10062","reason":"nodediag failed pci amdapu","overwrite":0}}
+{"timestamp":1711670140.9621673,"name":"drain","context":{"idset":"10384","reason":"broker was unresponsive"}}
+{"timestamp":1711670141.0619121,"name":"drain","context":{"idset":"10385","reason":"broker was unresponsive"}}
+{"timestamp":1711670202.9663434,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1711670203.0595696,"name":"offline","context":{"idset":"10384"}}
+{"timestamp":1711670205.0601485,"name":"offline","context":{"idset":"10385"}}
+{"timestamp":1711670413.5395412,"name":"online","context":{"idset":"10056"}}
+{"timestamp":1711670431.1045146,"name":"online","context":{"idset":"10055"}}
+{"timestamp":1711670455.5958602,"name":"online","context":{"idset":"10054"}}
+{"timestamp":1711670794.0494463,"name":"drain","context":{"idset":"10410","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711670794.9424369,"name":"drain","context":{"idset":"10389","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711670810.5146904,"name":"drain","context":{"idset":"10398","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711670812.9259496,"name":"drain","context":{"idset":"10402","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711670827.7794442,"name":"drain","context":{"idset":"10397","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711670828.6399806,"name":"drain","context":{"idset":"10406","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711670837.7224295,"name":"drain","context":{"idset":"10401","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711670850.8804054,"name":"drain","context":{"idset":"10407","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711670943.8737538,"name":"online","context":{"idset":"10059"}}
+{"timestamp":1711671173.0613143,"name":"drain","context":{"idset":"10053","reason":"broker was unresponsive"}}
+{"timestamp":1711671235.0605152,"name":"offline","context":{"idset":"10053"}}
+{"timestamp":1711671275.7183051,"name":"drain","context":{"idset":"10065","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711671280.8218336,"name":"drain","context":{"idset":"10063","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711671451.0608497,"name":"offline","context":{"idset":"11793"}}
+{"timestamp":1711671717.1271069,"name":"undrain","context":{"idset":"1-60,10059,10071-10074,10079-10082,11286,11290-11291,11297,11300,11321,11323-11326,11330,11332,11335,11338,11340-11345,11347-11348,11351,11353-11355,11357,11359-11361,11364,11366,11370,11372,11374-11375,11377-11378,11380,11382-11384,11386,11389,11391-11393,11395,11397-11398,11400-11403,11405,11408-11417,11419-11423,11427-11431,11433,11435-11438,11441-11447,11452,11457,11459,11461-11462,11465,11467,11469-11470,11472-11476,11479-11481,11483-11485,11487,11489,11491-11497,11500,11508,11592,11595,11597,11599,11601,11603-11604,11607-11608,11611-11612,11614-11615,11618-11636,11669,11685,11715-11716,11747-11748,11750-11752,11754-11757,11759-11762,11764,11845-11851,11853-11860,11877-11891"}}
+{"timestamp":1711672367.8617034,"name":"drain","context":{"idset":"10057","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711672953.0546682,"name":"online","context":{"idset":"10060"}}
+{"timestamp":1711673058.7506936,"name":"online","context":{"idset":"10053"}}
+{"timestamp":1711673806.1731946,"name":"drain","context":{"idset":"11759","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711673808.8159473,"name":"drain","context":{"idset":"11750","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711673814.9281149,"name":"drain","context":{"idset":"11761","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711673823.8231194,"name":"drain","context":{"idset":"11762","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711673825.5554583,"name":"drain","context":{"idset":"11760","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711673826.2768984,"name":"drain","context":{"idset":"11754","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711673827.1037047,"name":"drain","context":{"idset":"11755","reason":"nodediag failed clocksource pci","overwrite":0}}
+{"timestamp":1711673828.1390667,"name":"drain","context":{"idset":"11757","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711673836.7761922,"name":"drain","context":{"idset":"11752","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711673842.0654182,"name":"drain","context":{"idset":"11751","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711673843.0750339,"name":"drain","context":{"idset":"11756","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711673855.6725125,"name":"drain","context":{"idset":"11764","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711674574.9615963,"name":"drain","context":{"idset":"11657","reason":"broker was unresponsive"}}
+{"timestamp":1711674575.0620043,"name":"drain","context":{"idset":"11666","reason":"broker was unresponsive"}}
+{"timestamp":1711674639.1941695,"name":"offline","context":{"idset":"11666"}}
+{"timestamp":1711674687.2015226,"name":"offline","context":{"idset":"11657"}}
+{"timestamp":1711675205.0027633,"name":"undrain","context":{"idset":"11750,11754-11755,11757,11759-11762"}}
+{"timestamp":1711675629.0607488,"name":"offline","context":{"idset":"11750"}}
+{"timestamp":1711675641.04655,"name":"offline","context":{"idset":"11749"}}
+{"timestamp":1711675641.0768738,"name":"offline","context":{"idset":"11751"}}
+{"timestamp":1711675641.1026213,"name":"offline","context":{"idset":"11752"}}
+{"timestamp":1711675641.1089585,"name":"offline","context":{"idset":"11753"}}
+{"timestamp":1711675641.1947713,"name":"offline","context":{"idset":"11754"}}
+{"timestamp":1711675641.2345388,"name":"offline","context":{"idset":"11755"}}
+{"timestamp":1711675641.2361002,"name":"offline","context":{"idset":"11756"}}
+{"timestamp":1711675641.26106,"name":"offline","context":{"idset":"11757"}}
+{"timestamp":1711675641.2778199,"name":"offline","context":{"idset":"11758"}}
+{"timestamp":1711675641.3025079,"name":"offline","context":{"idset":"11759"}}
+{"timestamp":1711675641.3146873,"name":"offline","context":{"idset":"11760"}}
+{"timestamp":1711675641.324831,"name":"offline","context":{"idset":"11761"}}
+{"timestamp":1711675641.3383656,"name":"offline","context":{"idset":"11762"}}
+{"timestamp":1711675641.35514,"name":"offline","context":{"idset":"11763"}}
+{"timestamp":1711675641.3610561,"name":"offline","context":{"idset":"11764"}}
+{"timestamp":1711675985.9264374,"name":"online","context":{"idset":"10052"}}
+{"timestamp":1711676055.6795862,"name":"online","context":{"idset":"10051"}}
+{"timestamp":1711676110.2417102,"name":"online","context":{"idset":"10050"}}
+{"timestamp":1711676131.0820878,"name":"online","context":{"idset":"10049"}}
+{"timestamp":1711676235.3206046,"name":"online","context":{"idset":"10048"}}
+{"timestamp":1711676237.568356,"name":"online","context":{"idset":"10047"}}
+{"timestamp":1711676309.1323595,"name":"online","context":{"idset":"10046"}}
+{"timestamp":1711676310.9459987,"name":"online","context":{"idset":"10045"}}
+{"timestamp":1711676950.5066323,"name":"online","context":{"idset":"10042"}}
+{"timestamp":1711676951.8946292,"name":"online","context":{"idset":"10041"}}
+{"timestamp":1711676999.819129,"name":"online","context":{"idset":"10040"}}
+{"timestamp":1711677002.1622698,"name":"online","context":{"idset":"10039"}}
+{"timestamp":1711677312.1135077,"name":"online","context":{"idset":"10380"}}
+{"timestamp":1711677312.2865224,"name":"online","context":{"idset":"10374,10379"}}
+{"timestamp":1711677983.0617237,"name":"drain","context":{"idset":"11668","reason":"broker was unresponsive"}}
+{"timestamp":1711678045.0606792,"name":"offline","context":{"idset":"11668"}}
+{"timestamp":1711679469.8185863,"name":"online","context":{"idset":"10044"}}
+{"timestamp":1711679473.1391535,"name":"online","context":{"idset":"10043"}}
+{"timestamp":1711679627.1751299,"name":"online","context":{"idset":"10038"}}
+{"timestamp":1711679630.7999594,"name":"online","context":{"idset":"10037"}}
+{"timestamp":1711681144.9635539,"name":"drain","context":{"idset":"11653","reason":"broker was unresponsive"}}
+{"timestamp":1711681144.9637053,"name":"drain","context":{"idset":"11654","reason":"broker was unresponsive"}}
+{"timestamp":1711681144.9637702,"name":"drain","context":{"idset":"11655","reason":"broker was unresponsive"}}
+{"timestamp":1711681144.9638529,"name":"drain","context":{"idset":"11656","reason":"broker was unresponsive"}}
+{"timestamp":1711681144.9639428,"name":"drain","context":{"idset":"11658","reason":"broker was unresponsive"}}
+{"timestamp":1711681144.964011,"name":"drain","context":{"idset":"11659","reason":"broker was unresponsive"}}
+{"timestamp":1711681144.9640872,"name":"drain","context":{"idset":"11660","reason":"broker was unresponsive"}}
+{"timestamp":1711681144.9641614,"name":"drain","context":{"idset":"11661","reason":"broker was unresponsive"}}
+{"timestamp":1711681144.9642324,"name":"drain","context":{"idset":"11662","reason":"broker was unresponsive"}}
+{"timestamp":1711681144.9643033,"name":"drain","context":{"idset":"11663","reason":"broker was unresponsive"}}
+{"timestamp":1711681145.1130784,"name":"drain","context":{"idset":"11664","reason":"broker was unresponsive"}}
+{"timestamp":1711681146.9624338,"name":"drain","context":{"idset":"11665","reason":"broker was unresponsive"}}
+{"timestamp":1711681147.0618732,"name":"drain","context":{"idset":"11667","reason":"broker was unresponsive"}}
+{"timestamp":1711681208.9937327,"name":"offline","context":{"idset":"11653"}}
+{"timestamp":1711681209.0112464,"name":"offline","context":{"idset":"11654"}}
+{"timestamp":1711681209.0272925,"name":"offline","context":{"idset":"11655"}}
+{"timestamp":1711681209.3435454,"name":"offline","context":{"idset":"11656"}}
+{"timestamp":1711681209.3609653,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1711681209.378711,"name":"offline","context":{"idset":"11659"}}
+{"timestamp":1711681209.4079702,"name":"offline","context":{"idset":"11660"}}
+{"timestamp":1711681209.4499094,"name":"offline","context":{"idset":"11661"}}
+{"timestamp":1711681209.4523952,"name":"offline","context":{"idset":"11662"}}
+{"timestamp":1711681209.4687557,"name":"offline","context":{"idset":"11663"}}
+{"timestamp":1711681209.4851463,"name":"offline","context":{"idset":"11664"}}
+{"timestamp":1711681209.5027292,"name":"offline","context":{"idset":"11665"}}
+{"timestamp":1711681209.8578393,"name":"offline","context":{"idset":"11667"}}
+{"timestamp":1711682951.4881909,"name":"undrain","context":{"idset":"10053,10057,10061-10063,10065"}}
+{"timestamp":1711683116.9607275,"name":"drain","context":{"idset":"10043","reason":"broker was unresponsive"}}
+{"timestamp":1711683117.060648,"name":"drain","context":{"idset":"10044","reason":"broker was unresponsive"}}
+{"timestamp":1711683168.9612861,"name":"drain","context":{"idset":"10037","reason":"broker was unresponsive"}}
+{"timestamp":1711683169.0618935,"name":"drain","context":{"idset":"10038","reason":"broker was unresponsive"}}
+{"timestamp":1711683180.9637754,"name":"offline","context":{"idset":"10043"}}
+{"timestamp":1711683181.0592999,"name":"offline","context":{"idset":"10044"}}
+{"timestamp":1711683212.9615688,"name":"drain","context":{"idset":"10051","reason":"broker was unresponsive"}}
+{"timestamp":1711683213.0621202,"name":"drain","context":{"idset":"10052","reason":"broker was unresponsive"}}
+{"timestamp":1711683231.1823461,"name":"offline","context":{"idset":"10037"}}
+{"timestamp":1711683232.2595778,"name":"offline","context":{"idset":"10038"}}
+{"timestamp":1711683277.0599546,"name":"offline","context":{"idset":"10052"}}
+{"timestamp":1711683279.1945198,"name":"offline","context":{"idset":"10051"}}
+{"timestamp":1711683815.4715815,"name":"drain","context":{"idset":"10046","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711683819.2361462,"name":"drain","context":{"idset":"10045","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711683826.9287212,"name":"drain","context":{"idset":"10042","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711683857.581295,"name":"drain","context":{"idset":"10047","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711683873.0078707,"name":"drain","context":{"idset":"10056","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711683874.0295269,"name":"drain","context":{"idset":"10048","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711683881.8280046,"name":"drain","context":{"idset":"10058","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711683884.1152647,"name":"drain","context":{"idset":"10054","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711683889.1029561,"name":"drain","context":{"idset":"10055","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711685015.725915,"name":"drain","context":{"idset":"10053-10068","reason":"reason HP L0 test","overwrite":1}}
+{"timestamp":1711685344.6436086,"name":"online","context":{"idset":"10044"}}
+{"timestamp":1711685346.8078432,"name":"online","context":{"idset":"10043"}}
+{"timestamp":1711685398.1046758,"name":"online","context":{"idset":"10038"}}
+{"timestamp":1711685399.9302452,"name":"online","context":{"idset":"10037"}}
+{"timestamp":1711685587.0810034,"name":"drain","context":{"idset":"10059-10060,10064,10066-10068","reason":"HP L0 test","overwrite":1}}
+{"timestamp":1711685840.4403684,"name":"online","context":{"idset":"10741"}}
+{"timestamp":1711686071.0616689,"name":"drain","context":{"idset":"10741","reason":"broker was unresponsive"}}
+{"timestamp":1711686092.6599398,"name":"online","context":{"idset":"11655"}}
+{"timestamp":1711686092.9397464,"name":"online","context":{"idset":"11665"}}
+{"timestamp":1711686093.0964801,"name":"online","context":{"idset":"11656-11657,11660"}}
+{"timestamp":1711686093.2924852,"name":"online","context":{"idset":"11658,11661,11666"}}
+{"timestamp":1711686093.4139802,"name":"online","context":{"idset":"11653,11659,11662-11663,11667"}}
+{"timestamp":1711686093.5292554,"name":"online","context":{"idset":"11664,11668"}}
+{"timestamp":1711686093.6787515,"name":"online","context":{"idset":"11654"}}
+{"timestamp":1711686108.889544,"name":"drain","context":{"idset":"10071","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711686112.8132043,"name":"drain","context":{"idset":"10072","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711686133.0593002,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1711686343.8520756,"name":"undrain","context":{"idset":"10053-10068"}}
+{"timestamp":1711686418.965126,"name":"offline","context":{"idset":"10037"}}
+{"timestamp":1711686419.0598762,"name":"offline","context":{"idset":"10038"}}
+{"timestamp":1711686424.9643276,"name":"offline","context":{"idset":"10043"}}
+{"timestamp":1711686425.059186,"name":"offline","context":{"idset":"10044"}}
+{"timestamp":1711686627.0607119,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1711687556.3339245,"name":"drain","context":{"idset":"10053","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711687563.3166091,"name":"drain","context":{"idset":"10062","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711687564.0639358,"name":"drain","context":{"idset":"10061","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711687586.4930465,"name":"drain","context":{"idset":"10057","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711687588.3164465,"name":"online","context":{"idset":"10747"}}
+{"timestamp":1711687588.4810724,"name":"online","context":{"idset":"10743"}}
+{"timestamp":1711687588.6647534,"name":"online","context":{"idset":"10751"}}
+{"timestamp":1711687588.833087,"name":"online","context":{"idset":"10744,10752"}}
+{"timestamp":1711687589.0450118,"name":"online","context":{"idset":"10741,10746,10749"}}
+{"timestamp":1711687589.1672642,"name":"online","context":{"idset":"10745,10754"}}
+{"timestamp":1711687589.2921491,"name":"online","context":{"idset":"10742,10748,10750,10756"}}
+{"timestamp":1711687589.669138,"name":"online","context":{"idset":"10753"}}
+{"timestamp":1711687589.7831151,"name":"online","context":{"idset":"10755"}}
+{"timestamp":1711687594.652056,"name":"drain","context":{"idset":"10065","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711687600.900475,"name":"online","context":{"idset":"10759"}}
+{"timestamp":1711687601.0903788,"name":"online","context":{"idset":"10757"}}
+{"timestamp":1711687601.5426066,"name":"online","context":{"idset":"10764"}}
+{"timestamp":1711687601.667084,"name":"online","context":{"idset":"10760-10762,10769"}}
+{"timestamp":1711687601.8882728,"name":"online","context":{"idset":"10766,10768"}}
+{"timestamp":1711687602.1909914,"name":"online","context":{"idset":"10767,10772"}}
+{"timestamp":1711687602.3675539,"name":"online","context":{"idset":"10758,10765"}}
+{"timestamp":1711687602.4895685,"name":"online","context":{"idset":"10763,10770"}}
+{"timestamp":1711687602.6634181,"name":"online","context":{"idset":"10771"}}
+{"timestamp":1711687613.8336368,"name":"drain","context":{"idset":"10063","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711687613.835237,"name":"online","context":{"idset":"10773"}}
+{"timestamp":1711687613.8369603,"name":"online","context":{"idset":"10774"}}
+{"timestamp":1711687613.8386054,"name":"online","context":{"idset":"10775,10777"}}
+{"timestamp":1711687614.2181134,"name":"online","context":{"idset":"10778,10783"}}
+{"timestamp":1711687614.388618,"name":"online","context":{"idset":"10779"}}
+{"timestamp":1711687614.5807807,"name":"online","context":{"idset":"10781,10784"}}
+{"timestamp":1711687614.696943,"name":"online","context":{"idset":"10782"}}
+{"timestamp":1711687614.8488891,"name":"online","context":{"idset":"10776,10780,10786"}}
+{"timestamp":1711687615.2415879,"name":"online","context":{"idset":"10785,10787"}}
+{"timestamp":1711687615.6606243,"name":"online","context":{"idset":"10788"}}
+{"timestamp":1711687626.1157503,"name":"online","context":{"idset":"10789"}}
+{"timestamp":1711687626.4183886,"name":"online","context":{"idset":"10790-10791"}}
+{"timestamp":1711687626.5353863,"name":"online","context":{"idset":"10792-10793"}}
+{"timestamp":1711687626.9898791,"name":"online","context":{"idset":"10798-10799"}}
+{"timestamp":1711687627.4494536,"name":"online","context":{"idset":"10794-10796,10801"}}
+{"timestamp":1711687627.6056561,"name":"online","context":{"idset":"10802"}}
+{"timestamp":1711687627.9123726,"name":"online","context":{"idset":"10797"}}
+{"timestamp":1711687628.2026196,"name":"online","context":{"idset":"10804"}}
+{"timestamp":1711687628.3567989,"name":"online","context":{"idset":"10800"}}
+{"timestamp":1711687628.463572,"name":"online","context":{"idset":"10803"}}
+{"timestamp":1711687638.7836928,"name":"online","context":{"idset":"10805"}}
+{"timestamp":1711687638.8842273,"name":"online","context":{"idset":"10806"}}
+{"timestamp":1711687639.1353045,"name":"online","context":{"idset":"10808"}}
+{"timestamp":1711687639.309231,"name":"online","context":{"idset":"10809"}}
+{"timestamp":1711687639.4741261,"name":"online","context":{"idset":"10810"}}
+{"timestamp":1711687639.6329925,"name":"online","context":{"idset":"10807,10811"}}
+{"timestamp":1711687639.9802999,"name":"online","context":{"idset":"10812,10814-10815"}}
+{"timestamp":1711687640.519335,"name":"online","context":{"idset":"10817"}}
+{"timestamp":1711687640.6394255,"name":"online","context":{"idset":"10813,10816"}}
+{"timestamp":1711687641.2002599,"name":"online","context":{"idset":"10818-10820"}}
+{"timestamp":1711687651.8755627,"name":"online","context":{"idset":"10825"}}
+{"timestamp":1711687652.0984032,"name":"online","context":{"idset":"10822"}}
+{"timestamp":1711687652.3535659,"name":"online","context":{"idset":"10828"}}
+{"timestamp":1711687652.5067964,"name":"online","context":{"idset":"10826-10827"}}
+{"timestamp":1711687652.6084464,"name":"online","context":{"idset":"10829-10830,10832"}}
+{"timestamp":1711687652.9063497,"name":"online","context":{"idset":"10823-10824"}}
+{"timestamp":1711687653.0912824,"name":"online","context":{"idset":"10821,10831"}}
+{"timestamp":1711687653.2156417,"name":"online","context":{"idset":"10833-10834"}}
+{"timestamp":1711687653.4961762,"name":"online","context":{"idset":"10835-10836"}}
+{"timestamp":1711687664.1584914,"name":"online","context":{"idset":"10837"}}
+{"timestamp":1711687664.5786779,"name":"online","context":{"idset":"10838-10839"}}
+{"timestamp":1711687664.9863245,"name":"online","context":{"idset":"10842-10843"}}
+{"timestamp":1711687665.1370382,"name":"online","context":{"idset":"10840,10844"}}
+{"timestamp":1711687665.3309309,"name":"online","context":{"idset":"10841,10847"}}
+{"timestamp":1711687665.5202739,"name":"online","context":{"idset":"10845,10848-10849"}}
+{"timestamp":1711687665.6215878,"name":"online","context":{"idset":"10846"}}
+{"timestamp":1711687665.9018927,"name":"online","context":{"idset":"10851"}}
+{"timestamp":1711687666.0590434,"name":"online","context":{"idset":"10850"}}
+{"timestamp":1711687666.2485113,"name":"online","context":{"idset":"10852"}}
+{"timestamp":1711687676.4937732,"name":"online","context":{"idset":"10853"}}
+{"timestamp":1711687677.1330144,"name":"online","context":{"idset":"10855"}}
+{"timestamp":1711687677.3203154,"name":"online","context":{"idset":"10854"}}
+{"timestamp":1711687677.5843811,"name":"online","context":{"idset":"10857,10861"}}
+{"timestamp":1711687677.7798157,"name":"online","context":{"idset":"10856,10859-10860"}}
+{"timestamp":1711687678.0121546,"name":"online","context":{"idset":"10863"}}
+{"timestamp":1711687678.1637187,"name":"online","context":{"idset":"10865"}}
+{"timestamp":1711687678.3182163,"name":"online","context":{"idset":"10862,10864,10866"}}
+{"timestamp":1711687678.4196846,"name":"online","context":{"idset":"10858"}}
+{"timestamp":1711687678.5221393,"name":"online","context":{"idset":"10867"}}
+{"timestamp":1711687679.4477613,"name":"online","context":{"idset":"10868"}}
+{"timestamp":1711687800.5550568,"name":"online","context":{"idset":"10044"}}
+{"timestamp":1711687801.783128,"name":"online","context":{"idset":"10043"}}
+{"timestamp":1711688152.8651168,"name":"online","context":{"idset":"11125"}}
+{"timestamp":1711688153.0243895,"name":"online","context":{"idset":"11129"}}
+{"timestamp":1711688153.2207904,"name":"online","context":{"idset":"11130,11151,11154"}}
+{"timestamp":1711688153.3765359,"name":"online","context":{"idset":"11128,11146,11166"}}
+{"timestamp":1711688153.4893119,"name":"online","context":{"idset":"11134,11139,11159"}}
+{"timestamp":1711688153.6415,"name":"online","context":{"idset":"11196"}}
+{"timestamp":1711688153.7427428,"name":"online","context":{"idset":"11148"}}
+{"timestamp":1711688154.1644962,"name":"online","context":{"idset":"11178,11204"}}
+{"timestamp":1711688154.2656047,"name":"online","context":{"idset":"11131,11156,11181"}}
+{"timestamp":1711688154.3821881,"name":"online","context":{"idset":"11127,11168"}}
+{"timestamp":1711688154.5322015,"name":"online","context":{"idset":"11132-11133,11177,11186"}}
+{"timestamp":1711688154.6809642,"name":"online","context":{"idset":"11140,11142,11161,11170,11242"}}
+{"timestamp":1711688154.7917516,"name":"online","context":{"idset":"11143,11169"}}
+{"timestamp":1711688154.9018905,"name":"online","context":{"idset":"11141,11165,11180"}}
+{"timestamp":1711688155.0160048,"name":"online","context":{"idset":"11135,11144,11152-11153,11189,11249"}}
+{"timestamp":1711688155.132911,"name":"online","context":{"idset":"11155,11162,11171-11172,11176,11184-11185,11203,11206,11239,11246"}}
+{"timestamp":1711688155.2429311,"name":"online","context":{"idset":"11136-11137,11145,11157,11167,11213,11245"}}
+{"timestamp":1711688155.3606057,"name":"online","context":{"idset":"11164,11192,11195,11200-11201,11211,11217,11223"}}
+{"timestamp":1711688155.4712157,"name":"online","context":{"idset":"11138,11147,11158,11163,11173,11183,11187,11190,11197,11199,11208,11210,11218-11219,11226,11228,11230,11235,11238,11250-11251"}}
+{"timestamp":1711688155.5815718,"name":"online","context":{"idset":"11160,11174,11179,11182,11188,11191,11193-11194,11198,11205,11207,11209,11212,11224,11232-11233,11236-11237,11240-11241,11244"}}
+{"timestamp":1711688155.7334776,"name":"online","context":{"idset":"11175,11220,11222,11225,11227,11231,11234,11243"}}
+{"timestamp":1711688155.852143,"name":"online","context":{"idset":"11214,11221,11229,11247-11248,11252"}}
+{"timestamp":1711688156.1868076,"name":"online","context":{"idset":"11202"}}
+{"timestamp":1711688255.0613749,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1711688559.6569977,"name":"online","context":{"idset":"10038"}}
+{"timestamp":1711688560.9071727,"name":"online","context":{"idset":"10037"}}
+{"timestamp":1711689952.154294,"name":"drain","context":{"idset":"10794","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711689952.9087276,"name":"drain","context":{"idset":"10793","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711689953.484504,"name":"drain","context":{"idset":"10789","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711689953.9435339,"name":"drain","context":{"idset":"10795","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711689954.5076962,"name":"drain","context":{"idset":"10797","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711689954.9819255,"name":"drain","context":{"idset":"10790","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711689955.4526725,"name":"drain","context":{"idset":"10798","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711689955.9200354,"name":"drain","context":{"idset":"10792","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711689956.3654644,"name":"drain","context":{"idset":"10791","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711689956.8180678,"name":"drain","context":{"idset":"10796","reason":"nodediag failed clocksource dmi pci","overwrite":0}}
+{"timestamp":1711690255.3257957,"name":"online","context":{"idset":"10157"}}
+{"timestamp":1711690272.9735875,"name":"online","context":{"idset":"10156,10158"}}
+{"timestamp":1711690273.286653,"name":"online","context":{"idset":"10154"}}
+{"timestamp":1711692175.0600023,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1711705905.1869528,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1711706364.8885615,"name":"drain","context":{"idset":"10451","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706376.0832734,"name":"drain","context":{"idset":"10417","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706376.8219044,"name":"drain","context":{"idset":"10416","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706381.4783027,"name":"drain","context":{"idset":"10441","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706386.5250204,"name":"drain","context":{"idset":"10458","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706392.7732136,"name":"drain","context":{"idset":"10450","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706397.1331835,"name":"drain","context":{"idset":"10438","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706401.0427601,"name":"drain","context":{"idset":"10442","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706412.8626382,"name":"drain","context":{"idset":"10419","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706418.4330242,"name":"drain","context":{"idset":"10447","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706427.7553701,"name":"drain","context":{"idset":"10448","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706428.5088429,"name":"drain","context":{"idset":"10439","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706431.9480107,"name":"drain","context":{"idset":"10440","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706432.7034392,"name":"drain","context":{"idset":"10452","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706433.3454251,"name":"drain","context":{"idset":"10444","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706463.9188411,"name":"drain","context":{"idset":"10445","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706543.524848,"name":"drain","context":{"idset":"11681","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706564.1413035,"name":"drain","context":{"idset":"11683","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706566.4348085,"name":"drain","context":{"idset":"11680","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706570.4257567,"name":"drain","context":{"idset":"11684","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706575.0096402,"name":"drain","context":{"idset":"11679","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711706579.4838822,"name":"drain","context":{"idset":"11682","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711712431.0591199,"name":"offline","context":{"idset":"11891"}}
+{"timestamp":1711724002.3414977,"name":"offline","context":{"idset":"11767"}}
+{"timestamp":1711724002.4155176,"name":"offline","context":{"idset":"11768"}}
+{"timestamp":1711724002.4260945,"name":"offline","context":{"idset":"11765"}}
+{"timestamp":1711724002.5262866,"name":"offline","context":{"idset":"11766"}}
+{"timestamp":1711724085.9849486,"name":"drain","context":{"idset":"10082","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711724106.1604726,"name":"drain","context":{"idset":"10081","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711724223.0590506,"name":"drain","context":{"idset":"11678","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711724232.0323668,"name":"drain","context":{"idset":"11677","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711724555.0610635,"name":"offline","context":{"idset":"11717"}}
+{"timestamp":1711724707.039973,"name":"drain","context":{"idset":"11675","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711724720.1368284,"name":"online","context":{"idset":"10871"}}
+{"timestamp":1711725464.3559108,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1711725474.2077906,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1711725520.9707153,"name":"offline","context":{"idset":"10389"}}
+{"timestamp":1711725520.9821439,"name":"offline","context":{"idset":"10401"}}
+{"timestamp":1711725521.0707319,"name":"offline","context":{"idset":"10402"}}
+{"timestamp":1711725690.0250506,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1711725690.1899219,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1711725690.3340855,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1711725690.3474994,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1711725690.3593259,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1711725690.3689899,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1711725690.3877623,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1711725690.4068985,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1711725690.4083948,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1711725690.4156537,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1711725690.511903,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1711725755.0632687,"name":"drain","context":{"idset":"11784","reason":"broker was unresponsive"}}
+{"timestamp":1711725767.0656931,"name":"drain","context":{"idset":"11794","reason":"broker was unresponsive"}}
+{"timestamp":1711725778.6114855,"name":"offline","context":{"idset":"11669"}}
+{"timestamp":1711725810.3775454,"name":"offline","context":{"idset":"11670"}}
+{"timestamp":1711725817.0591738,"name":"offline","context":{"idset":"11784"}}
+{"timestamp":1711725829.059655,"name":"offline","context":{"idset":"11794"}}
+{"timestamp":1711725933.0596557,"name":"offline","context":{"idset":"11651"}}
+{"timestamp":1711725965.8510883,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1711725982.7424746,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1711726238.1123741,"name":"undrain","context":{"idset":"11501-11502"}}
+{"timestamp":1711726281.020335,"name":"drain","context":{"idset":"10748","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711726285.468183,"name":"drain","context":{"idset":"10747","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711726288.3614619,"name":"drain","context":{"idset":"10743","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711726310.4522023,"name":"drain","context":{"idset":"10742","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711726311.2278292,"name":"drain","context":{"idset":"10745","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711726319.1374547,"name":"drain","context":{"idset":"10746","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711726327.3123736,"name":"drain","context":{"idset":"10744","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711726388.1491804,"name":"drain","context":{"idset":"11673","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711726392.3737485,"name":"drain","context":{"idset":"11674","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711726396.3991082,"name":"drain","context":{"idset":"11672","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711726427.1922398,"name":"drain","context":{"idset":"11671","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711726427.2945235,"name":"drain","context":{"idset":"11651,11670","reason":"epilog failed for jobid fmLXtp1qajy","overwrite":0}}
+{"timestamp":1711727083.0628095,"name":"drain","context":{"idset":"10871","reason":"broker was unresponsive"}}
+{"timestamp":1711727147.0256267,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1711727723.2028818,"name":"drain","context":{"idset":"11148","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711727725.8331397,"name":"drain","context":{"idset":"10222","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711727735.0622723,"name":"drain","context":{"idset":"11860","reason":"broker was unresponsive"}}
+{"timestamp":1711727742.2625678,"name":"drain","context":{"idset":"11151","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711727743.1796935,"name":"drain","context":{"idset":"10224","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711727750.2855296,"name":"drain","context":{"idset":"10228","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711727757.2006228,"name":"drain","context":{"idset":"10223","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711727761.1825933,"name":"drain","context":{"idset":"10225","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711727768.6712646,"name":"drain","context":{"idset":"10226","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711727772.7067671,"name":"drain","context":{"idset":"10227","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711727797.0602093,"name":"offline","context":{"idset":"11860"}}
+{"timestamp":1711727957.0627635,"name":"drain","context":{"idset":"11892","reason":"broker was unresponsive"}}
+{"timestamp":1711727982.2853529,"name":"online","context":{"idset":"10146-10147,10160"}}
+{"timestamp":1711727982.3868382,"name":"online","context":{"idset":"10149,10152,10159"}}
+{"timestamp":1711727982.5534127,"name":"online","context":{"idset":"10150-10151,10161"}}
+{"timestamp":1711728021.0605328,"name":"offline","context":{"idset":"11892"}}
+{"timestamp":1711728125.5287814,"name":"online","context":{"idset":"11637"}}
+{"timestamp":1711728323.9877291,"name":"drain","context":{"idset":"10114","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711728334.9796245,"name":"drain","context":{"idset":"10113","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711728343.3254752,"name":"drain","context":{"idset":"10118","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711728352.8618662,"name":"drain","context":{"idset":"10119","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711728355.0613661,"name":"offline","context":{"idset":"11776"}}
+{"timestamp":1711728361.0575018,"name":"drain","context":{"idset":"10111","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711728361.8150437,"name":"drain","context":{"idset":"10117","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711728363.1796095,"name":"drain","context":{"idset":"10112","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711728364.1678636,"name":"drain","context":{"idset":"10116","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711728368.2850928,"name":"drain","context":{"idset":"10110","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711728380.2118144,"name":"drain","context":{"idset":"10115","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711728627.5913291,"name":"online","context":{"idset":"11717"}}
+{"timestamp":1711729094.9820085,"name":"drain","context":{"idset":"11493-11508","reason":"reason HP L0 testing","overwrite":0}}
+{"timestamp":1711729137.2745626,"name":"undrain","context":{"idset":"11493-11508"}}
+{"timestamp":1711729178.3428459,"name":"online","context":{"idset":"884"}}
+{"timestamp":1711730444.282562,"name":"online","context":{"idset":"11651"}}
+{"timestamp":1711730561.3231311,"name":"undrain","context":{"idset":"11637,11651"}}
+{"timestamp":1711730671.154022,"name":"drain","context":{"idset":"11493-11508","overwrite":0}}
+{"timestamp":1711730824.5530663,"name":"undrain","context":{"idset":"11493-11508"}}
+{"timestamp":1711732016.350565,"name":"online","context":{"idset":"883"}}
+{"timestamp":1711733069.8399937,"name":"offline","context":{"idset":"11154"}}
+{"timestamp":1711733155.1427765,"name":"undrain","context":{"idset":"11148-11151,11230,11301-11316"}}
+{"timestamp":1711733176.5633008,"name":"online","context":{"idset":"11702"}}
+{"timestamp":1711733631.3445716,"name":"drain","context":{"idset":"10763","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733644.2002394,"name":"drain","context":{"idset":"10783","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733646.8067858,"name":"drain","context":{"idset":"10754","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733648.0229697,"name":"drain","context":{"idset":"10755","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733652.9365022,"name":"drain","context":{"idset":"10762","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733653.8923004,"name":"drain","context":{"idset":"10770","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733654.7146134,"name":"drain","context":{"idset":"10760","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733655.3274591,"name":"drain","context":{"idset":"10758","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733655.7736757,"name":"drain","context":{"idset":"10764","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733657.139148,"name":"drain","context":{"idset":"10805","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733657.8636148,"name":"drain","context":{"idset":"10761","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733658.3502553,"name":"drain","context":{"idset":"10753","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733659.6447358,"name":"drain","context":{"idset":"10752","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733663.8206584,"name":"drain","context":{"idset":"10750","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733666.2542951,"name":"drain","context":{"idset":"10807","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733667.0522788,"name":"drain","context":{"idset":"10782","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733667.8328431,"name":"drain","context":{"idset":"10775","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733668.6052599,"name":"drain","context":{"idset":"10774","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733669.349016,"name":"drain","context":{"idset":"10768","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733670.1288216,"name":"drain","context":{"idset":"10771","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733670.8903363,"name":"drain","context":{"idset":"10772","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733671.594311,"name":"drain","context":{"idset":"10799","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733672.0661569,"name":"drain","context":{"idset":"10800","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733672.6788509,"name":"drain","context":{"idset":"10788","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733673.4463534,"name":"drain","context":{"idset":"10781","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733676.932476,"name":"drain","context":{"idset":"10826","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733677.9403784,"name":"drain","context":{"idset":"11248","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733680.9899652,"name":"drain","context":{"idset":"10757","reason":"nodediag failed clocksource dmi pci","overwrite":0}}
+{"timestamp":1711733682.1142714,"name":"drain","context":{"idset":"10808","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733682.6906681,"name":"drain","context":{"idset":"10804","reason":"nodediag failed clocksource dmi pci","overwrite":0}}
+{"timestamp":1711733683.499532,"name":"drain","context":{"idset":"10769","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733684.3732729,"name":"drain","context":{"idset":"11249","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733685.2440166,"name":"drain","context":{"idset":"10749","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733686.0992904,"name":"drain","context":{"idset":"10841","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733686.9549427,"name":"drain","context":{"idset":"10765","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733687.8642294,"name":"drain","context":{"idset":"10756","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733688.7280917,"name":"drain","context":{"idset":"10814","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733689.3770008,"name":"drain","context":{"idset":"10829","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733689.844337,"name":"drain","context":{"idset":"10812","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733690.3280208,"name":"drain","context":{"idset":"10780","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733690.8045485,"name":"drain","context":{"idset":"10784","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733691.6027968,"name":"drain","context":{"idset":"10779","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733692.358212,"name":"drain","context":{"idset":"10821","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733693.1085598,"name":"drain","context":{"idset":"10759","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733693.6073937,"name":"drain","context":{"idset":"10766","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733694.074327,"name":"drain","context":{"idset":"10767","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733694.5160217,"name":"drain","context":{"idset":"10803","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733694.9659727,"name":"drain","context":{"idset":"10832","reason":"nodediag failed clocksource dmi pci","overwrite":0}}
+{"timestamp":1711733695.434325,"name":"drain","context":{"idset":"10786","reason":"nodediag failed clocksource dmi pci","overwrite":0}}
+{"timestamp":1711733695.9094985,"name":"drain","context":{"idset":"10751","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733696.3637471,"name":"drain","context":{"idset":"10823","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733696.8085656,"name":"drain","context":{"idset":"10809","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733697.2708282,"name":"drain","context":{"idset":"11152","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733697.7252762,"name":"drain","context":{"idset":"10828","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733698.1820004,"name":"drain","context":{"idset":"10819","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733698.6518373,"name":"drain","context":{"idset":"10785","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733699.1434879,"name":"drain","context":{"idset":"10776","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733699.6309912,"name":"drain","context":{"idset":"11155","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733700.0951796,"name":"drain","context":{"idset":"10838","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733700.5743349,"name":"drain","context":{"idset":"10778","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733701.0410399,"name":"drain","context":{"idset":"10836","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733701.5006347,"name":"drain","context":{"idset":"10855","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733701.9776583,"name":"drain","context":{"idset":"10815","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733702.4539537,"name":"drain","context":{"idset":"10787","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733702.917141,"name":"drain","context":{"idset":"10773","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733703.4170635,"name":"drain","context":{"idset":"11135","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733704.2163916,"name":"drain","context":{"idset":"10811","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733704.9789884,"name":"drain","context":{"idset":"10844","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733705.7467763,"name":"undrain","context":{"idset":"11702"}}
+{"timestamp":1711733706.5187273,"name":"drain","context":{"idset":"10842","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733707.2587597,"name":"drain","context":{"idset":"10848","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733707.7425423,"name":"drain","context":{"idset":"11134","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733708.496907,"name":"drain","context":{"idset":"10810","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733709.3148644,"name":"drain","context":{"idset":"11142","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733710.0873327,"name":"drain","context":{"idset":"10825","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733710.8221874,"name":"drain","context":{"idset":"11131","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733711.3039272,"name":"drain","context":{"idset":"10777","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733711.7876866,"name":"drain","context":{"idset":"10817","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733712.2439032,"name":"drain","context":{"idset":"10847","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733712.7122905,"name":"drain","context":{"idset":"11133","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733713.2322202,"name":"drain","context":{"idset":"10851","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733713.6947486,"name":"drain","context":{"idset":"10801","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733714.170361,"name":"drain","context":{"idset":"10834","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733714.623822,"name":"drain","context":{"idset":"11139","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733715.0908253,"name":"drain","context":{"idset":"10858","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733715.8392277,"name":"drain","context":{"idset":"11137","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733716.6519105,"name":"drain","context":{"idset":"11153","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733717.4611535,"name":"drain","context":{"idset":"10816","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733718.2463894,"name":"drain","context":{"idset":"10822","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733718.8872747,"name":"drain","context":{"idset":"10854","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733719.3541987,"name":"drain","context":{"idset":"10837","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733719.8553531,"name":"drain","context":{"idset":"10857","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733720.3389573,"name":"drain","context":{"idset":"11145","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733720.8079438,"name":"drain","context":{"idset":"11156","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733721.2832353,"name":"drain","context":{"idset":"10806","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733721.741982,"name":"drain","context":{"idset":"10802","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733722.3066049,"name":"drain","context":{"idset":"11197","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733723.0788264,"name":"drain","context":{"idset":"10813","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733723.8547161,"name":"drain","context":{"idset":"10820","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733724.6029553,"name":"drain","context":{"idset":"10850","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733725.1298466,"name":"drain","context":{"idset":"11128","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733725.7487013,"name":"drain","context":{"idset":"11146","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733726.5102046,"name":"drain","context":{"idset":"10843","reason":"nodediag failed clocksource dmi pci","overwrite":0}}
+{"timestamp":1711733727.2783854,"name":"drain","context":{"idset":"10827","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733727.9338088,"name":"drain","context":{"idset":"11208","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733728.662195,"name":"drain","context":{"idset":"10868","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733729.4170434,"name":"drain","context":{"idset":"10860","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733730.1968377,"name":"drain","context":{"idset":"10835","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733730.9831386,"name":"drain","context":{"idset":"10830","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733731.7706234,"name":"drain","context":{"idset":"10833","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733732.5315197,"name":"drain","context":{"idset":"10864","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733733.3034852,"name":"drain","context":{"idset":"10867","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733734.0151284,"name":"drain","context":{"idset":"10863","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733734.9523857,"name":"drain","context":{"idset":"11176","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733735.7078552,"name":"drain","context":{"idset":"11192","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733736.1889868,"name":"drain","context":{"idset":"11209","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733736.6401687,"name":"drain","context":{"idset":"10846","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733737.1100254,"name":"drain","context":{"idset":"10849","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733737.5644846,"name":"drain","context":{"idset":"11250","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733738.064044,"name":"drain","context":{"idset":"10861","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733738.7628512,"name":"drain","context":{"idset":"10856","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733739.7074654,"name":"drain","context":{"idset":"10831","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733740.5875123,"name":"drain","context":{"idset":"10818","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733741.3673873,"name":"drain","context":{"idset":"10824","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733741.9539423,"name":"drain","context":{"idset":"10852","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733742.7857056,"name":"drain","context":{"idset":"10866","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733743.6580968,"name":"drain","context":{"idset":"11144","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733744.5511096,"name":"drain","context":{"idset":"10840","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733745.450212,"name":"drain","context":{"idset":"11186","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733746.3380299,"name":"drain","context":{"idset":"11191","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733747.2219884,"name":"drain","context":{"idset":"10865","reason":"nodediag failed clocksource dmi pci","overwrite":0}}
+{"timestamp":1711733748.0834906,"name":"drain","context":{"idset":"10853","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733748.6721213,"name":"drain","context":{"idset":"11132","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733749.1720347,"name":"drain","context":{"idset":"10862","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733749.6325366,"name":"drain","context":{"idset":"10839","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733750.0912073,"name":"drain","context":{"idset":"11141","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733750.5770993,"name":"drain","context":{"idset":"11226","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733751.0504472,"name":"drain","context":{"idset":"11138","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733751.5286865,"name":"drain","context":{"idset":"11129","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733751.9738102,"name":"drain","context":{"idset":"11222","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733752.4245155,"name":"drain","context":{"idset":"11196","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733752.8965876,"name":"drain","context":{"idset":"11225","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733753.3470843,"name":"drain","context":{"idset":"11184","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733753.8216982,"name":"drain","context":{"idset":"11180","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733754.2711582,"name":"drain","context":{"idset":"11194","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733757.6044221,"name":"drain","context":{"idset":"11220","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733758.0766449,"name":"drain","context":{"idset":"11178","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733758.5277925,"name":"drain","context":{"idset":"11136","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733758.9772053,"name":"drain","context":{"idset":"11214","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733759.4154129,"name":"drain","context":{"idset":"10845","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733759.8675582,"name":"drain","context":{"idset":"11212","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733760.3350453,"name":"drain","context":{"idset":"11245","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733760.7928731,"name":"drain","context":{"idset":"10859","reason":"nodediag failed dmi pci","overwrite":0}}
+{"timestamp":1711733761.2802315,"name":"drain","context":{"idset":"11201","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733761.8135316,"name":"drain","context":{"idset":"11182","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733762.5457294,"name":"drain","context":{"idset":"11241","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733763.3638685,"name":"drain","context":{"idset":"11143","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733764.1444685,"name":"drain","context":{"idset":"11140","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733766.19999,"name":"drain","context":{"idset":"11224","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733766.982336,"name":"drain","context":{"idset":"11193","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733767.7279243,"name":"drain","context":{"idset":"11130","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733768.3725736,"name":"drain","context":{"idset":"11147","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733769.1186938,"name":"drain","context":{"idset":"11238","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733769.9507768,"name":"drain","context":{"idset":"11246","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733771.590981,"name":"drain","context":{"idset":"11221","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733772.3564627,"name":"drain","context":{"idset":"11207","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733772.9328082,"name":"drain","context":{"idset":"11243","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733773.3974895,"name":"drain","context":{"idset":"11198","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733776.1908786,"name":"drain","context":{"idset":"11202","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733777.3370647,"name":"drain","context":{"idset":"11195","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733778.9921753,"name":"drain","context":{"idset":"11199","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733779.8067582,"name":"drain","context":{"idset":"11206","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733780.5838475,"name":"drain","context":{"idset":"11223","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733781.3524117,"name":"drain","context":{"idset":"11205","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733781.9836645,"name":"drain","context":{"idset":"11211","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733782.4445305,"name":"drain","context":{"idset":"11219","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733782.9099724,"name":"drain","context":{"idset":"11200","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733787.8340917,"name":"drain","context":{"idset":"11218","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733789.0230632,"name":"drain","context":{"idset":"11217","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733793.8033855,"name":"drain","context":{"idset":"11239","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733794.2501934,"name":"drain","context":{"idset":"11210","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733796.6682739,"name":"drain","context":{"idset":"11244","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733801.554213,"name":"drain","context":{"idset":"11242","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733805.2468588,"name":"drain","context":{"idset":"11247","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733848.4729207,"name":"drain","context":{"idset":"11228","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733851.3643103,"name":"drain","context":{"idset":"11232","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733871.2564409,"name":"drain","context":{"idset":"11231","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733873.7400775,"name":"drain","context":{"idset":"11227","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733874.6906664,"name":"drain","context":{"idset":"11234","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733885.8408241,"name":"drain","context":{"idset":"11229","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733893.4895463,"name":"drain","context":{"idset":"11237","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733896.0478668,"name":"drain","context":{"idset":"11236","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733901.9148343,"name":"drain","context":{"idset":"11235","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711733902.6849751,"name":"drain","context":{"idset":"11233","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711734055.3149543,"name":"drain","context":{"idset":"11230","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711734121.1560712,"name":"drain","context":{"idset":"883-884","overwrite":0}}
+{"timestamp":1711734157.7365916,"name":"offline","context":{"idset":"883"}}
+{"timestamp":1711734162.0172753,"name":"offline","context":{"idset":"884"}}
+{"timestamp":1711734731.8132401,"name":"drain","context":{"idset":"10150","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711734735.8542626,"name":"drain","context":{"idset":"10149","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711734763.3099208,"name":"drain","context":{"idset":"10147","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711734764.9003315,"name":"drain","context":{"idset":"10151","reason":"nodediag failed clocksource pci","overwrite":0}}
+{"timestamp":1711734767.9628785,"name":"drain","context":{"idset":"10146","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711736024.8756938,"name":"undrain","context":{"idset":"10361"}}
+{"timestamp":1711736043.7617981,"name":"undrain","context":{"idset":"10117"}}
+{"timestamp":1711736045.6395082,"name":"undrain","context":{"idset":"10118"}}
+{"timestamp":1711736452.8531506,"name":"undrain","context":{"idset":"11125,11127"}}
+{"timestamp":1711737076.1203187,"name":"drain","context":{"idset":"11127","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711737088.4452558,"name":"drain","context":{"idset":"11125","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711738272.1335626,"name":"undrain","context":{"idset":"11157-11172"}}
+{"timestamp":1711738630.6939547,"name":"offline","context":{"idset":"11769"}}
+{"timestamp":1711738630.6950343,"name":"offline","context":{"idset":"11770"}}
+{"timestamp":1711738630.6960714,"name":"offline","context":{"idset":"11771"}}
+{"timestamp":1711738630.7272332,"name":"offline","context":{"idset":"11772"}}
+{"timestamp":1711738630.7356069,"name":"offline","context":{"idset":"11773"}}
+{"timestamp":1711738630.7547636,"name":"offline","context":{"idset":"11774"}}
+{"timestamp":1711738630.8271899,"name":"offline","context":{"idset":"11775"}}
+{"timestamp":1711738631.5492342,"name":"offline","context":{"idset":"11778"}}
+{"timestamp":1711738631.5760486,"name":"offline","context":{"idset":"11779"}}
+{"timestamp":1711738631.6346912,"name":"offline","context":{"idset":"11780"}}
+{"timestamp":1711738683.0610936,"name":"offline","context":{"idset":"11777"}}
+{"timestamp":1711739252.8187349,"name":"drain","context":{"idset":"11157","reason":"nodediag failed pci cxi","overwrite":0}}
+{"timestamp":1711739601.9629896,"name":"offline","context":{"idset":"11836"}}
+{"timestamp":1711739749.5053444,"name":"drain","context":{"idset":"11158","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711740170.0871279,"name":"drain","context":{"idset":"11161","reason":"nodediag failed pci cxi","overwrite":0}}
+{"timestamp":1711740196.4025867,"name":"drain","context":{"idset":"11160","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711740203.2337866,"name":"drain","context":{"idset":"11159","reason":"nodediag failed pci cxi","overwrite":0}}
+{"timestamp":1711740209.9484718,"name":"drain","context":{"idset":"11162","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711740213.3964698,"name":"drain","context":{"idset":"11163","reason":"nodediag failed pci cxi","overwrite":0}}
+{"timestamp":1711740238.3827271,"name":"drain","context":{"idset":"11167","reason":"nodediag failed pci cxi","overwrite":0}}
+{"timestamp":1711740247.1264496,"name":"drain","context":{"idset":"11171","reason":"nodediag failed pci cxi","overwrite":0}}
+{"timestamp":1711740247.8759899,"name":"drain","context":{"idset":"11166","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711740248.7028427,"name":"drain","context":{"idset":"11168","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711740249.4637747,"name":"drain","context":{"idset":"11165","reason":"nodediag failed pci cxi","overwrite":0}}
+{"timestamp":1711740252.6594939,"name":"drain","context":{"idset":"11164","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711740272.7305949,"name":"drain","context":{"idset":"11169","reason":"nodediag failed pci cxi","overwrite":0}}
+{"timestamp":1711740285.7003391,"name":"drain","context":{"idset":"11170","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711740292.4341311,"name":"drain","context":{"idset":"11172","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1711740772.0469997,"name":"undrain","context":{"idset":"11157-11172"}}
+{"timestamp":1711741470.5727289,"name":"undrain","context":{"idset":"10047-10048,10053,10057,10061-10063,10065,10071-10072,10081-10082,10110-10116,10119,10146-10147,10149-10150,10222-10228,10359-10360,10397-10398,10406-10407,10410,10416-10417,10419,10438-10442,10444-10445,10447-10448,10450-10452,10458,11125,11127-11147,11152-11153,11155-11156,11176,11178,11180,11182,11184,11186,11191-11202,11205-11212,11214,11217-11237,11248-11250,11671-11675,11677-11684"}}
+{"timestamp":1711741567.0797455,"name":"undrain","context":{"idset":"10381,10389,10401-10402,11751-11752,11756,11764"}}
+{"timestamp":1711742686.2489491,"name":"drain","context":{"idset":"10161","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1711742687.1360803,"name":"drain","context":{"idset":"10162","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1711742688.0076518,"name":"drain","context":{"idset":"10169","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1711742688.869864,"name":"drain","context":{"idset":"10170","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1711742688.8722303,"name":"online","context":{"idset":"10095"}}
+{"timestamp":1711743415.7833021,"name":"drain","context":{"idset":"11417","reason":"epilog failed for jobid fmKGqZRsvpP","overwrite":0}}
+{"timestamp":1711743415.7834704,"name":"drain","context":{"idset":"11240","reason":"epilog failed for jobid fmLXp3j1Rgj","overwrite":0}}
+{"timestamp":1711743415.783556,"name":"drain","context":{"idset":"10038","reason":"epilog failed for jobid fmKGNquzZew","overwrite":0}}
+{"timestamp":1711743745.0155947,"name":"offline","context":{"idset":"217"}}
+{"timestamp":1711743745.4175208,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1711743745.843679,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1711743746.5866387,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1711743747.0244615,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1711743876.1497376,"name":"offline","context":{"idset":"10098"}}
+{"timestamp":1711743876.1694734,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1711743876.1853263,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1711743877.5359328,"name":"offline","context":{"idset":"10087"}}
+{"timestamp":1711743877.5465777,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1711743877.5541348,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1711743877.5633247,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1711743877.5648899,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1711743877.590929,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1711743877.6061978,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1711743877.608423,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1711743877.6122465,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1711743877.621835,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1711743877.6305392,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1711743877.6413589,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1711743877.6479418,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1711743877.6569624,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1711743877.6661975,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1711743877.6754894,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1711743877.68507,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1711743877.7211995,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1711743877.7319131,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1711743877.7334344,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1711743877.7349586,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1711743877.7364798,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1711743877.7380192,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1711743877.7424464,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1711743877.7721827,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1711743877.8153243,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1711743877.8169208,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1711743877.818464,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1711743877.8199663,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1711743877.8214779,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1711743877.8230634,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1711743877.8612609,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1711743877.8719449,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1711743877.8734539,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1711743877.8750241,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1711743877.8765566,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1711743877.9053926,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1711743877.9441946,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1711743877.9457023,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1711743877.9472063,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1711743877.9486892,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1711743877.9501555,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1711743877.9612842,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1711743877.962898,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1711743877.9643881,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1711743877.965868,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1711743877.9673345,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1711743877.9688225,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1711743877.9702921,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1711743878.0002258,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1711743878.0290771,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1711743878.0306032,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1711743878.032078,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1711743878.0335569,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1711743878.0624008,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1711743878.0821917,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1711743878.0836768,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1711743878.0851426,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1711743878.0866511,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1711743878.088155,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1711743878.0987558,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1711743878.1002986,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1711743878.1017721,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1711743878.103235,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1711743878.1047182,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1711743878.1061664,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1711743878.1076279,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1711743878.1090736,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1711743878.1472342,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1711743878.1580186,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1711743878.1595013,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1711743878.160985,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1711743878.1624849,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1711743878.1639395,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1711743878.1653841,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1711743878.1668546,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1711743878.2047975,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1711743878.2429638,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1711743878.2445629,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1711743878.2460294,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1711743878.2474802,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1711743878.2489228,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1711743878.2503927,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1711743878.2611649,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1711743878.2901583,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1711743878.2916222,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1711743878.293057,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1711743878.2945442,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1711743878.2959967,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1711743878.297435,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1711743878.2988708,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1711743878.3003612,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1711743878.3017962,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1711743878.3305542,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1711743878.3683722,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1711743878.3788767,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1711743878.3803084,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1711743878.3817987,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1711743878.3832803,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1711743878.3847234,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1711743878.3861594,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1711743878.3875868,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1711743878.4256294,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1711743878.4544389,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1711743878.4558518,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1711743878.4572532,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1711743878.4587145,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1711743878.4601552,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1711743878.4615636,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1711743878.4629622,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1711743878.473505,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1711743878.5112116,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1711743878.5407312,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1711743878.5421712,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1711743878.5435925,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1711743878.545001,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1711743878.5464864,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1711743878.5480607,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1711743878.5494695,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1711743878.5508935,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1711743878.5522907,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1711743878.5910904,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1711743878.6202774,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1711743878.6217561,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1711743878.6232452,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1711743878.6246541,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1711743878.6261513,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1711743878.6275485,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1711743878.6289389,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1711743878.6670187,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1711743878.705009,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1711743878.7155578,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1711743878.7169526,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1711743878.7183385,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1711743878.7197335,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1711743878.7211175,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1711743878.7408361,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1711743878.7422283,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1711743878.7436244,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1711743878.7450337,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1711743878.746419,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1711743878.7478311,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1711743878.7583661,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1711743878.7962966,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1711743878.8160763,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1711743878.8174736,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1711743878.8190093,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1711743878.8203928,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1711743878.8217733,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1711743878.823173,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1711743878.8245533,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1711743878.8352695,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1711743878.8366721,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1711743878.8380692,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1711743878.8394926,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1711743878.8408697,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1711743878.8422616,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1711743878.8436525,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1711743878.8728774,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1711743878.9107707,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1711743878.9121361,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1711743878.9134994,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1711743878.9148526,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1711743878.9162133,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1711743878.9175775,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1711743878.9189265,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1711743878.9387994,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1711743878.9401526,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1711743878.9415081,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1711743878.9428582,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1711743878.9442346,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1711743878.9455903,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1711743878.9469807,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1711743878.9483392,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1711743878.9682031,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1711743879.0066829,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1711743879.0080609,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1711743879.009408,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1711743879.0107782,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1711743879.0121315,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1711743879.0135305,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1711743879.0149031,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1711743879.0162697,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1711743879.0476856,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1711743879.0608108,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1711743879.0628197,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1711743879.064883,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1711743879.0670168,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1711743879.0692179,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1711743879.0713215,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1711743879.1054661,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1711743879.1576362,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1711743879.1604695,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1711743879.1629009,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1711743879.183923,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1711743879.1866832,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1711743879.1893661,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1711743879.1922164,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1711743879.194947,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1711743879.1976821,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1711743879.2005367,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1711743879.2032683,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1711743879.2245107,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1711743879.2272391,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1711743879.2300761,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1711743879.2325752,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1711743879.2348154,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1711743879.2372265,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1711743879.2396338,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1711743879.242002,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1711743879.2791057,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1711743879.333571,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1711743879.3362043,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1711743879.3390083,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1711743879.3417737,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1711743879.3445594,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1711743879.3471968,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1711743879.4243147,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1711743879.4824927,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1711743879.4851587,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1711743879.4877768,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1711743879.4903822,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1711743879.4929512,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1711743879.4954939,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1711743879.4980359,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1711743879.500567,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1711743879.5028169,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1711743879.5053437,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1711743879.5416293,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1711743879.544085,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1711743879.5467393,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1711743879.5491982,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1711743879.5515778,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1711743879.5540268,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1711743879.556159,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1711743879.5583086,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1711743879.5605028,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1711743879.5627263,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1711743879.6244633,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1711743879.6834216,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1711743879.7412324,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1711743879.756542,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1711743879.7582555,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1711743879.759937,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1711743879.7617383,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1711743879.7637613,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1711743879.7657731,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1711743879.7675984,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1711743879.7693319,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1711743879.7833421,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1711743879.8380435,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1711743879.842864,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1711743879.8447113,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1711743879.8465519,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1711743879.8483124,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1711743879.8499796,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1711743879.8517768,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1711743879.8536079,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1711743879.855485,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1711743879.8572047,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1711743879.9085858,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1711743879.9549129,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1711743879.9905591,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1711743879.9920981,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1711743879.9937189,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1711743879.9953527,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1711743879.9969742,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1711743879.9985862,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1711743880.0000947,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1711743880.0017955,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1711743880.0509429,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1711743880.0953493,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1711743880.1081171,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1711743880.1096091,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1711743880.1111155,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1711743880.112571,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1711743880.1139693,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1711743880.1153405,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1711743880.1169879,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1711743880.1527805,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1711743880.1972797,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1711743880.1989567,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1711743880.2005718,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1711743880.2021339,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1711743880.2037089,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1711743880.2052581,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1711743880.2067864,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1711743880.2082548,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1711743880.2096858,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1711743880.2112727,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1711743880.2436779,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1711743880.2848425,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1711743880.3302479,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1711743880.3316479,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1711743880.333101,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1711743880.3343952,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1711743880.3357162,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1711743880.3370137,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1711743880.342581,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1711743880.3441925,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1711743880.3568408,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1711743880.402673,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1711743880.4483812,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1711743880.4956653,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1711743880.4970634,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1711743880.4984636,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1711743880.500037,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1711743880.5016484,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1711743880.503365,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1711743880.5049975,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1711743880.5063572,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1711743880.5076554,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1711743880.5089691,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1711743880.5436375,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1711743880.58761,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1711743880.6001585,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1711743880.6016474,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1711743880.6031191,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1711743880.6047263,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1711743880.6061952,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1711743880.6075466,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1711743880.6088953,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1711743880.6103909,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1711743880.6119022,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1711743880.6133711,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1711743880.6148553,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1711743880.6165125,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1711743880.6395216,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1711743880.6829402,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1711743880.727241,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1711743880.7756314,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1711743880.7984829,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1711743880.7999218,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1711743880.8013599,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1711743880.802808,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1711743880.8042364,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1711743880.8057263,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1711743880.8071799,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1711743880.819932,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1711743880.8638558,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1711743880.9065716,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1711743880.9394741,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1711743880.9407318,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1711743880.941937,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1711743880.9431579,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1711743880.9443634,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1711743880.9456148,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1711743880.9468255,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1711743880.9480517,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1711743880.9493449,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1711743880.9721682,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1711743881.0152378,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1711743881.0571909,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1711743881.0682342,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1711743881.0696039,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1711743881.0709047,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1711743881.0721188,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1711743881.0733588,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1711743881.0745838,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1711743881.0758612,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1711743881.0985222,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1711743881.1420829,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1711743881.185678,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1711743881.1871316,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1711743881.1885171,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1711743881.1899445,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1711743881.1913471,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1711743881.1927714,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1711743881.194154,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1711743881.1955483,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1711743881.1969092,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1711743881.2385943,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1711743881.2762702,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1711743881.2971513,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1711743881.2985446,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1711743881.2999198,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1711743881.3012846,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1711743881.3026204,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1711743881.3039403,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1711743881.3052006,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1711743881.3064117,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1711743881.3076777,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1711743881.3488362,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1711743881.3928742,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1711743881.4240718,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1711743881.4253771,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1711743881.4269347,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1711743881.4282389,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1711743881.429523,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1711743881.4307983,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1711743881.4320376,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1711743881.433286,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1711743881.4343932,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1711743881.4354649,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1711743881.448123,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1711743881.4976435,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1711743881.5417473,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1711743881.5646055,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1711743881.5659246,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1711743881.5672772,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1711743881.5684485,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1711743881.5695727,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1711743881.5708618,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1711743881.5720692,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1711743881.5733111,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1711743881.574475,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1711743881.5756614,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1711743881.5768499,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1711743881.577981,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1711743881.5886905,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1711743881.6278744,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1711743881.6706138,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1711743881.693259,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1711743881.6946054,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1711743881.6959229,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1711743881.6972263,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1711743881.6985357,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1711743881.6997895,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1711743881.7010379,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1711743881.7022731,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1711743881.7035689,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1711743881.7048259,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1711743881.7380652,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1711743881.7814791,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1711743881.8269482,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1711743881.8699675,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1711743881.8818438,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1711743881.8831468,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1711743881.8845148,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1711743881.8858895,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1711743881.8872659,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1711743881.8886292,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1711743881.8899853,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1711743881.8913741,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1711743881.9339366,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1711743881.972321,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1711743882.0049548,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1711743882.0060599,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1711743882.007319,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1711743882.0085273,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1711743882.0097847,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1711743882.0110102,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1711743882.0121508,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1711743882.013334,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1711743882.0145988,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1711743882.0254169,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1711743882.0635684,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1711743882.1031532,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1711743882.1256902,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1711743882.1267476,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1711743882.1277399,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1711743882.1286526,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1711743882.1298709,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1711743882.131098,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1711743882.1323946,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1711743882.1722746,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1711743882.2051978,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1711743882.2191036,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1711743882.2198362,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1711743882.2205856,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1711743882.2213054,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1711743882.222002,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1711743882.2227027,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1711743882.2234867,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1711743882.2242162,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1711743882.225033,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1711743882.225848,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1711743882.246882,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1711743882.2737451,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1711743882.2929726,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1711743882.2937822,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1711743882.2945294,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1711743882.295249,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1711743882.2960372,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1711743882.2967541,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1711743882.2974744,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1711743882.2983229,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1711743882.2990179,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1711743882.2998178,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1711743882.321264,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1711743882.3469422,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1711743882.3744602,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1711743882.3751822,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1711743882.3759308,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1711743882.3767178,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1711743882.3773777,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1711743882.3780558,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1711743882.3851113,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1711743882.4111402,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1711743882.4370968,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1711743882.4564323,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1711743882.4571922,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1711743882.4578865,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1711743882.4585936,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1711743882.4593363,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1711743882.4601085,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1711743882.4608343,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1711743882.4748256,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1711743882.4831114,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1711743882.4841585,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1711743882.4850261,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1711743882.4859054,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1711743882.486733,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1711743882.4875224,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1711743882.4882281,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1711743882.4890268,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1711743882.5147979,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1711743882.5407472,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1711743882.5661664,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1711743882.5669501,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1711743882.5676863,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1711743882.5683682,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1711743882.5691302,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1711743882.5698705,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1711743882.5767944,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1711743882.5838387,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1711743882.5845714,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1711743882.5853579,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1711743882.5860569,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1711743882.5867815,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1711743882.5875444,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1711743882.5882449,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1711743882.59548,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1711743882.6216087,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1711743882.6590731,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1711743882.6661081,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1711743882.6668158,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1711743882.6675019,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1711743882.6681826,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1711743882.6817441,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1711743882.6824772,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1711743882.6833861,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1711743882.6842136,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1711743882.6850512,"name":"offline","context":{"idset":"10037"}}
+{"timestamp":1711743882.6857641,"name":"offline","context":{"idset":"10038"}}
+{"timestamp":1711743882.6864452,"name":"offline","context":{"idset":"10039"}}
+{"timestamp":1711743882.6871381,"name":"offline","context":{"idset":"10040"}}
+{"timestamp":1711743882.6878326,"name":"offline","context":{"idset":"10041"}}
+{"timestamp":1711743882.6950734,"name":"offline","context":{"idset":"10042"}}
+{"timestamp":1711743882.6960633,"name":"offline","context":{"idset":"10043"}}
+{"timestamp":1711743882.6967289,"name":"offline","context":{"idset":"10044"}}
+{"timestamp":1711743882.6973567,"name":"offline","context":{"idset":"10045"}}
+{"timestamp":1711743882.6981571,"name":"offline","context":{"idset":"10046"}}
+{"timestamp":1711743882.6990674,"name":"offline","context":{"idset":"10047"}}
+{"timestamp":1711743882.6999421,"name":"offline","context":{"idset":"10048"}}
+{"timestamp":1711743882.7006164,"name":"offline","context":{"idset":"10049"}}
+{"timestamp":1711743882.7289908,"name":"offline","context":{"idset":"10050"}}
+{"timestamp":1711743882.7570112,"name":"offline","context":{"idset":"10053"}}
+{"timestamp":1711743882.7648544,"name":"offline","context":{"idset":"10054"}}
+{"timestamp":1711743882.7658486,"name":"offline","context":{"idset":"10055"}}
+{"timestamp":1711743882.7666965,"name":"offline","context":{"idset":"10056"}}
+{"timestamp":1711743882.7673514,"name":"offline","context":{"idset":"10057"}}
+{"timestamp":1711743882.7831779,"name":"offline","context":{"idset":"10058"}}
+{"timestamp":1711743882.7843657,"name":"offline","context":{"idset":"10059"}}
+{"timestamp":1711743882.785763,"name":"offline","context":{"idset":"10060"}}
+{"timestamp":1711743882.787143,"name":"offline","context":{"idset":"10061"}}
+{"timestamp":1711743882.7879651,"name":"offline","context":{"idset":"10062"}}
+{"timestamp":1711743882.7892122,"name":"offline","context":{"idset":"10063"}}
+{"timestamp":1711743882.7903016,"name":"offline","context":{"idset":"10064"}}
+{"timestamp":1711743882.7918103,"name":"offline","context":{"idset":"10065"}}
+{"timestamp":1711743882.7998359,"name":"offline","context":{"idset":"10066"}}
+{"timestamp":1711743882.8320396,"name":"offline","context":{"idset":"10067"}}
+{"timestamp":1711743882.8327951,"name":"offline","context":{"idset":"10068"}}
+{"timestamp":1711743882.833797,"name":"offline","context":{"idset":"10069"}}
+{"timestamp":1711743882.8344293,"name":"offline","context":{"idset":"10070"}}
+{"timestamp":1711743882.8351455,"name":"offline","context":{"idset":"10071"}}
+{"timestamp":1711743882.8359234,"name":"offline","context":{"idset":"10072"}}
+{"timestamp":1711743882.8367872,"name":"offline","context":{"idset":"10073"}}
+{"timestamp":1711743882.8374794,"name":"offline","context":{"idset":"10074"}}
+{"timestamp":1711743882.8460236,"name":"offline","context":{"idset":"10079"}}
+{"timestamp":1711743882.8905814,"name":"offline","context":{"idset":"10080"}}
+{"timestamp":1711743882.9001155,"name":"offline","context":{"idset":"10081"}}
+{"timestamp":1711743882.9007881,"name":"offline","context":{"idset":"10082"}}
+{"timestamp":1711743882.901742,"name":"offline","context":{"idset":"10088"}}
+{"timestamp":1711743882.9026093,"name":"offline","context":{"idset":"10089"}}
+{"timestamp":1711743882.9036238,"name":"offline","context":{"idset":"10090"}}
+{"timestamp":1711743882.904578,"name":"offline","context":{"idset":"10091"}}
+{"timestamp":1711743882.9056599,"name":"offline","context":{"idset":"10092"}}
+{"timestamp":1711743882.9486535,"name":"offline","context":{"idset":"10093"}}
+{"timestamp":1711743882.9779606,"name":"offline","context":{"idset":"10094"}}
+{"timestamp":1711743882.9792893,"name":"offline","context":{"idset":"10095"}}
+{"timestamp":1711743882.9805577,"name":"offline","context":{"idset":"10097"}}
+{"timestamp":1711743882.9818122,"name":"offline","context":{"idset":"10099"}}
+{"timestamp":1711743883.005882,"name":"offline","context":{"idset":"10100"}}
+{"timestamp":1711743883.0066614,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1711743883.0078757,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1711743883.0091646,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1711743883.0105848,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1711743883.0118928,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1711743883.0132058,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1711743883.0144279,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1711743883.0156257,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1711743883.0253677,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1711743883.0270145,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1711743883.0280671,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1711743883.0290084,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1711743883.0299537,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1711743883.0310187,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1711743883.0320449,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1711743883.0433216,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1711743883.0849204,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1711743883.1279063,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1711743883.1385167,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1711743883.1395321,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1711743883.1405032,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1711743883.1415122,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1711743883.1425676,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1711743883.1436327,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1711743883.1657145,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1711743883.1668367,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1711743883.1679833,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1711743883.169106,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1711743883.1702316,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1711743883.1712813,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1711743883.1724088,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1711743883.1947379,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1711743883.2377279,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1711743883.2808254,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1711743883.2819319,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1711743883.2829978,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1711743883.2840695,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1711743883.3076046,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1711743883.3087361,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1711743883.3098347,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1711743883.3109484,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1711743883.3120489,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1711743883.3130748,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1711743883.3141026,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1711743883.315145,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1711743883.3263502,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1711743883.3274457,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1711743883.3285489,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1711743883.3296258,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1711743883.3306549,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1711743883.3317447,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1711743883.3432074,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1711743883.3859799,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1711743883.4081368,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1711743883.4091992,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1711743883.4102325,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1711743883.4215715,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1711743883.4225962,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1711743883.4236724,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1711743883.4247537,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1711743883.4257874,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1711743883.4268806,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1711743883.4279847,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1711743883.4394825,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1711743883.4405792,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1711743883.4416723,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1711743883.4427557,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1711743883.4438229,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1711743883.4448807,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1711743883.4458678,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1711743883.4468801,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1711743883.4479139,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1711743883.4489887,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1711743883.4500766,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1711743883.4822335,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1711743883.5261004,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1711743883.5272233,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1711743883.5282521,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1711743883.5292606,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1711743883.5302584,"name":"offline","context":{"idset":"10188"}}
+{"timestamp":1711743883.5312307,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1711743883.5321219,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1711743883.5331566,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1711743883.5768573,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1711743883.617949,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1711743883.6289327,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1711743883.6299796,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1711743883.630996,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1711743883.631938,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1711743883.6541386,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1711743883.6551614,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1711743883.6562016,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1711743883.6571331,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1711743883.6580238,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1711743883.6590078,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1711743883.6600997,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1711743883.6611693,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1711743883.6621614,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1711743883.6843791,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1711743883.6853421,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1711743883.6862237,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1711743883.6871307,"name":"offline","context":{"idset":"10213"}}
+{"timestamp":1711743883.6880283,"name":"offline","context":{"idset":"10214"}}
+{"timestamp":1711743883.6889017,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1711743883.6897736,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1711743883.6906354,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1711743883.6915388,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1711743883.7036192,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1711743883.7430959,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1711743883.7849576,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1711743883.7958977,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1711743883.7968035,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1711743883.7977417,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1711743883.7986124,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1711743883.7995265,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1711743883.8003876,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1711743883.8200421,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1711743883.8412831,"name":"offline","context":{"idset":"10359"}}
+{"timestamp":1711743883.8423173,"name":"offline","context":{"idset":"10360"}}
+{"timestamp":1711743883.8433142,"name":"offline","context":{"idset":"10374"}}
+{"timestamp":1711743883.8443527,"name":"offline","context":{"idset":"10379"}}
+{"timestamp":1711743883.8454251,"name":"offline","context":{"idset":"10380"}}
+{"timestamp":1711743883.8464727,"name":"offline","context":{"idset":"10397"}}
+{"timestamp":1711743883.847508,"name":"offline","context":{"idset":"10398"}}
+{"timestamp":1711743883.8483634,"name":"offline","context":{"idset":"10406"}}
+{"timestamp":1711743883.849287,"name":"offline","context":{"idset":"10407"}}
+{"timestamp":1711743883.8503411,"name":"offline","context":{"idset":"10410"}}
+{"timestamp":1711743883.8513949,"name":"offline","context":{"idset":"10416"}}
+{"timestamp":1711743883.8523061,"name":"offline","context":{"idset":"10417"}}
+{"timestamp":1711743883.8533568,"name":"offline","context":{"idset":"10419"}}
+{"timestamp":1711743883.8862994,"name":"offline","context":{"idset":"10438"}}
+{"timestamp":1711743883.9145939,"name":"offline","context":{"idset":"10439"}}
+{"timestamp":1711743883.915566,"name":"offline","context":{"idset":"10440"}}
+{"timestamp":1711743883.9164853,"name":"offline","context":{"idset":"10441"}}
+{"timestamp":1711743883.9174132,"name":"offline","context":{"idset":"10442"}}
+{"timestamp":1711743883.9182441,"name":"offline","context":{"idset":"10444"}}
+{"timestamp":1711743883.9191561,"name":"offline","context":{"idset":"10445"}}
+{"timestamp":1711743883.9200916,"name":"offline","context":{"idset":"10447"}}
+{"timestamp":1711743883.9209886,"name":"offline","context":{"idset":"10448"}}
+{"timestamp":1711743883.9218349,"name":"offline","context":{"idset":"10450"}}
+{"timestamp":1711743883.922729,"name":"offline","context":{"idset":"10451"}}
+{"timestamp":1711743883.9236798,"name":"offline","context":{"idset":"10452"}}
+{"timestamp":1711743883.9246428,"name":"offline","context":{"idset":"10458"}}
+{"timestamp":1711743883.9255979,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1711743883.9580863,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1711743884.0001562,"name":"offline","context":{"idset":"10743"}}
+{"timestamp":1711743884.0422914,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1711743884.0852063,"name":"offline","context":{"idset":"10745"}}
+{"timestamp":1711743884.1073358,"name":"offline","context":{"idset":"10746"}}
+{"timestamp":1711743884.1083653,"name":"offline","context":{"idset":"10747"}}
+{"timestamp":1711743884.1093791,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1711743884.1103671,"name":"offline","context":{"idset":"10749"}}
+{"timestamp":1711743884.111311,"name":"offline","context":{"idset":"10750"}}
+{"timestamp":1711743884.1122367,"name":"offline","context":{"idset":"10751"}}
+{"timestamp":1711743884.1132307,"name":"offline","context":{"idset":"10752"}}
+{"timestamp":1711743884.1142414,"name":"offline","context":{"idset":"10753"}}
+{"timestamp":1711743884.1152124,"name":"offline","context":{"idset":"10754"}}
+{"timestamp":1711743884.1367548,"name":"offline","context":{"idset":"10755"}}
+{"timestamp":1711743884.178798,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1711743884.2010996,"name":"offline","context":{"idset":"10757"}}
+{"timestamp":1711743884.2021093,"name":"offline","context":{"idset":"10758"}}
+{"timestamp":1711743884.2031047,"name":"offline","context":{"idset":"10759"}}
+{"timestamp":1711743884.2040868,"name":"offline","context":{"idset":"10760"}}
+{"timestamp":1711743884.2050297,"name":"offline","context":{"idset":"10761"}}
+{"timestamp":1711743884.2059622,"name":"offline","context":{"idset":"10762"}}
+{"timestamp":1711743884.2069118,"name":"offline","context":{"idset":"10763"}}
+{"timestamp":1711743884.2078784,"name":"offline","context":{"idset":"10764"}}
+{"timestamp":1711743884.208693,"name":"offline","context":{"idset":"10765"}}
+{"timestamp":1711743884.2095988,"name":"offline","context":{"idset":"10766"}}
+{"timestamp":1711743884.2208796,"name":"offline","context":{"idset":"10767"}}
+{"timestamp":1711743884.2635012,"name":"offline","context":{"idset":"10768"}}
+{"timestamp":1711743884.3060625,"name":"offline","context":{"idset":"10769"}}
+{"timestamp":1711743884.3487191,"name":"offline","context":{"idset":"10770"}}
+{"timestamp":1711743884.3709977,"name":"offline","context":{"idset":"10771"}}
+{"timestamp":1711743884.3718922,"name":"offline","context":{"idset":"10772"}}
+{"timestamp":1711743884.3728628,"name":"offline","context":{"idset":"10773"}}
+{"timestamp":1711743884.3738227,"name":"offline","context":{"idset":"10774"}}
+{"timestamp":1711743884.3747988,"name":"offline","context":{"idset":"10775"}}
+{"timestamp":1711743884.3757601,"name":"offline","context":{"idset":"10776"}}
+{"timestamp":1711743884.3982959,"name":"offline","context":{"idset":"10777"}}
+{"timestamp":1711743884.4420381,"name":"offline","context":{"idset":"10778"}}
+{"timestamp":1711743884.4737558,"name":"offline","context":{"idset":"10779"}}
+{"timestamp":1711743884.4746656,"name":"offline","context":{"idset":"10780"}}
+{"timestamp":1711743884.4755239,"name":"offline","context":{"idset":"10781"}}
+{"timestamp":1711743884.4764545,"name":"offline","context":{"idset":"10782"}}
+{"timestamp":1711743884.4773402,"name":"offline","context":{"idset":"10783"}}
+{"timestamp":1711743884.4782596,"name":"offline","context":{"idset":"10784"}}
+{"timestamp":1711743884.4791865,"name":"offline","context":{"idset":"10785"}}
+{"timestamp":1711743884.4800787,"name":"offline","context":{"idset":"10786"}}
+{"timestamp":1711743884.481051,"name":"offline","context":{"idset":"10787"}}
+{"timestamp":1711743884.4819551,"name":"offline","context":{"idset":"10788"}}
+{"timestamp":1711743884.4829292,"name":"offline","context":{"idset":"10789"}}
+{"timestamp":1711743884.4947562,"name":"offline","context":{"idset":"10790"}}
+{"timestamp":1711743884.5375752,"name":"offline","context":{"idset":"10791"}}
+{"timestamp":1711743884.5384989,"name":"offline","context":{"idset":"10792"}}
+{"timestamp":1711743884.539387,"name":"offline","context":{"idset":"10793"}}
+{"timestamp":1711743884.5402687,"name":"offline","context":{"idset":"10794"}}
+{"timestamp":1711743884.5411515,"name":"offline","context":{"idset":"10795"}}
+{"timestamp":1711743884.542032,"name":"offline","context":{"idset":"10796"}}
+{"timestamp":1711743884.5429044,"name":"offline","context":{"idset":"10797"}}
+{"timestamp":1711743884.5438011,"name":"offline","context":{"idset":"10798"}}
+{"timestamp":1711743884.5446384,"name":"offline","context":{"idset":"10799"}}
+{"timestamp":1711743884.5455208,"name":"offline","context":{"idset":"10800"}}
+{"timestamp":1711743884.5463662,"name":"offline","context":{"idset":"10801"}}
+{"timestamp":1711743884.5908251,"name":"offline","context":{"idset":"10802"}}
+{"timestamp":1711743884.631475,"name":"offline","context":{"idset":"10803"}}
+{"timestamp":1711743884.6739166,"name":"offline","context":{"idset":"10804"}}
+{"timestamp":1711743884.7159171,"name":"offline","context":{"idset":"10805"}}
+{"timestamp":1711743884.7167716,"name":"offline","context":{"idset":"10806"}}
+{"timestamp":1711743884.7176611,"name":"offline","context":{"idset":"10807"}}
+{"timestamp":1711743884.7185414,"name":"offline","context":{"idset":"10808"}}
+{"timestamp":1711743884.719357,"name":"offline","context":{"idset":"10809"}}
+{"timestamp":1711743884.7201915,"name":"offline","context":{"idset":"10810"}}
+{"timestamp":1711743884.7209799,"name":"offline","context":{"idset":"10811"}}
+{"timestamp":1711743884.7421033,"name":"offline","context":{"idset":"10812"}}
+{"timestamp":1711743884.7426038,"name":"offline","context":{"idset":"10813"}}
+{"timestamp":1711743884.7430785,"name":"offline","context":{"idset":"10814"}}
+{"timestamp":1711743884.7436934,"name":"offline","context":{"idset":"10815"}}
+{"timestamp":1711743884.7443621,"name":"offline","context":{"idset":"10816"}}
+{"timestamp":1711743884.7450147,"name":"offline","context":{"idset":"10817"}}
+{"timestamp":1711743884.7457817,"name":"offline","context":{"idset":"10818"}}
+{"timestamp":1711743884.7465367,"name":"offline","context":{"idset":"10819"}}
+{"timestamp":1711743884.7473049,"name":"offline","context":{"idset":"10820"}}
+{"timestamp":1711743884.7480609,"name":"offline","context":{"idset":"10821"}}
+{"timestamp":1711743884.7488027,"name":"offline","context":{"idset":"10822"}}
+{"timestamp":1711743884.7702668,"name":"offline","context":{"idset":"10823"}}
+{"timestamp":1711743884.8125873,"name":"offline","context":{"idset":"10824"}}
+{"timestamp":1711743884.8238821,"name":"offline","context":{"idset":"10825"}}
+{"timestamp":1711743884.8248098,"name":"offline","context":{"idset":"10826"}}
+{"timestamp":1711743884.8257394,"name":"offline","context":{"idset":"10827"}}
+{"timestamp":1711743884.8266418,"name":"offline","context":{"idset":"10828"}}
+{"timestamp":1711743884.8275576,"name":"offline","context":{"idset":"10829"}}
+{"timestamp":1711743884.8284779,"name":"offline","context":{"idset":"10830"}}
+{"timestamp":1711743884.8293805,"name":"offline","context":{"idset":"10831"}}
+{"timestamp":1711743884.8413932,"name":"offline","context":{"idset":"10832"}}
+{"timestamp":1711743884.8857799,"name":"offline","context":{"idset":"10833"}}
+{"timestamp":1711743884.9183896,"name":"offline","context":{"idset":"10834"}}
+{"timestamp":1711743884.9192331,"name":"offline","context":{"idset":"10835"}}
+{"timestamp":1711743884.9200389,"name":"offline","context":{"idset":"10836"}}
+{"timestamp":1711743884.9208665,"name":"offline","context":{"idset":"10837"}}
+{"timestamp":1711743884.9216948,"name":"offline","context":{"idset":"10838"}}
+{"timestamp":1711743884.9225531,"name":"offline","context":{"idset":"10839"}}
+{"timestamp":1711743884.9341993,"name":"offline","context":{"idset":"10840"}}
+{"timestamp":1711743884.9349897,"name":"offline","context":{"idset":"10841"}}
+{"timestamp":1711743884.935787,"name":"offline","context":{"idset":"10842"}}
+{"timestamp":1711743884.9365916,"name":"offline","context":{"idset":"10843"}}
+{"timestamp":1711743884.9373865,"name":"offline","context":{"idset":"10844"}}
+{"timestamp":1711743884.9382315,"name":"offline","context":{"idset":"10845"}}
+{"timestamp":1711743884.9390697,"name":"offline","context":{"idset":"10846"}}
+{"timestamp":1711743884.9399378,"name":"offline","context":{"idset":"10847"}}
+{"timestamp":1711743884.9407964,"name":"offline","context":{"idset":"10848"}}
+{"timestamp":1711743884.9416344,"name":"offline","context":{"idset":"10849"}}
+{"timestamp":1711743884.9424751,"name":"offline","context":{"idset":"10850"}}
+{"timestamp":1711743884.9433124,"name":"offline","context":{"idset":"10851"}}
+{"timestamp":1711743884.944123,"name":"offline","context":{"idset":"10852"}}
+{"timestamp":1711743884.944943,"name":"offline","context":{"idset":"10853"}}
+{"timestamp":1711743884.9762826,"name":"offline","context":{"idset":"10854"}}
+{"timestamp":1711743884.9862714,"name":"offline","context":{"idset":"10855"}}
+{"timestamp":1711743884.986975,"name":"offline","context":{"idset":"10856"}}
+{"timestamp":1711743884.9876456,"name":"offline","context":{"idset":"10857"}}
+{"timestamp":1711743884.9883504,"name":"offline","context":{"idset":"10858"}}
+{"timestamp":1711743884.9891284,"name":"offline","context":{"idset":"10859"}}
+{"timestamp":1711743884.9899037,"name":"offline","context":{"idset":"10860"}}
+{"timestamp":1711743884.9907401,"name":"offline","context":{"idset":"10861"}}
+{"timestamp":1711743884.9915342,"name":"offline","context":{"idset":"10862"}}
+{"timestamp":1711743884.9924207,"name":"offline","context":{"idset":"10863"}}
+{"timestamp":1711743884.9932957,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1711743884.9941249,"name":"offline","context":{"idset":"10865"}}
+{"timestamp":1711743884.9950192,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1711743884.9959018,"name":"offline","context":{"idset":"10867"}}
+{"timestamp":1711743884.9967711,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1711743885.0177782,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1711743885.0599775,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1711743885.103617,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1711743885.1520083,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1711743885.1642547,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1711743885.1651402,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1711743885.166024,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1711743885.1669095,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1711743885.1677654,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1711743885.1685653,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1711743885.1692533,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1711743885.1699774,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1711743885.170753,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1711743885.1931479,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1711743885.23785,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1711743885.2821169,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1711743885.2926512,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1711743885.2934041,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1711743885.2941973,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1711743885.2949994,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1711743885.2957685,"name":"offline","context":{"idset":"11146"}}
+{"timestamp":1711743885.296536,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1711743885.2973287,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1711743885.2981236,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1711743885.2989309,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1711743885.2997279,"name":"offline","context":{"idset":"11153"}}
+{"timestamp":1711743885.3450074,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1711743885.3889909,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1711743885.4301937,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1711743885.4310133,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1711743885.431643,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1711743885.4320805,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1711743885.432518,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1711743885.4330797,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1711743885.4338541,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1711743885.4346447,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1711743885.4354877,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1711743885.436317,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1711743885.4696488,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1711743885.5122538,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1711743885.5234933,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1711743885.5242736,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1711743885.5250673,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1711743885.5258489,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1711743885.5265672,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1711743885.5273225,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1711743885.5280619,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1711743885.5287294,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1711743885.529417,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1711743885.5301564,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1711743885.5309105,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1711743885.5316548,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1711743885.5323715,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1711743885.5331111,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1711743885.57656,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1711743885.5972974,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1711743885.5980084,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1711743885.5987301,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1711743885.5994883,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1711743885.6002154,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1711743885.6009684,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1711743885.6017172,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1711743885.6024723,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1711743885.603215,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1711743885.6039693,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1711743885.6047211,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1711743885.6263037,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1711743885.6691337,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1711743885.7126892,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1711743885.757221,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1711743885.8022978,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1711743885.8031116,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1711743885.8078694,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1711743885.8086965,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1711743885.809449,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1711743885.8102627,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1711743885.8445659,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1711743885.8454082,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1711743885.8461924,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1711743885.8469715,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1711743885.8477275,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1711743885.848424,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1711743885.849081,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1711743885.8498008,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1711743885.8506131,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1711743885.8513777,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1711743885.8520737,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1711743885.8527687,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1711743885.853545,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1711743885.8660917,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1711743885.9057498,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1711743885.9290259,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1711743885.929795,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1711743885.9305789,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1711743885.9313567,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1711743885.9321678,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1711743885.9330239,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1711743885.9338636,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1711743885.9346766,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1711743885.9356167,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1711743885.9364278,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1711743885.9372404,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1711743885.9380894,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1711743885.9388883,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1711743885.9396787,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1711743885.9406414,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1711743885.9414072,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1711743885.9881058,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1711743886.0305829,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1711743886.0735438,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1711743886.0851769,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1711743886.0858755,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1711743886.0865204,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1711743886.087157,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1711743886.0878057,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1711743886.0884397,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1711743886.0891171,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1711743886.089879,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1711743886.1468925,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1711743886.1897225,"name":"offline","context":{"idset":"11253"}}
+{"timestamp":1711743886.2332788,"name":"offline","context":{"idset":"11254"}}
+{"timestamp":1711743886.2549758,"name":"offline","context":{"idset":"11255"}}
+{"timestamp":1711743886.2557406,"name":"offline","context":{"idset":"11257"}}
+{"timestamp":1711743886.2565067,"name":"offline","context":{"idset":"11258"}}
+{"timestamp":1711743886.257237,"name":"offline","context":{"idset":"11259"}}
+{"timestamp":1711743886.2695546,"name":"offline","context":{"idset":"11260"}}
+{"timestamp":1711743886.280148,"name":"offline","context":{"idset":"11261"}}
+{"timestamp":1711743886.280833,"name":"offline","context":{"idset":"11262"}}
+{"timestamp":1711743886.2909994,"name":"offline","context":{"idset":"11263"}}
+{"timestamp":1711743886.2916284,"name":"offline","context":{"idset":"11264"}}
+{"timestamp":1711743886.2922688,"name":"offline","context":{"idset":"11265"}}
+{"timestamp":1711743886.2929721,"name":"offline","context":{"idset":"11266"}}
+{"timestamp":1711743886.2936389,"name":"offline","context":{"idset":"11267"}}
+{"timestamp":1711743886.2943461,"name":"offline","context":{"idset":"11268"}}
+{"timestamp":1711743886.2950575,"name":"offline","context":{"idset":"11269"}}
+{"timestamp":1711743886.2958031,"name":"offline","context":{"idset":"11270"}}
+{"timestamp":1711743886.2965033,"name":"offline","context":{"idset":"11271"}}
+{"timestamp":1711743886.2971234,"name":"offline","context":{"idset":"11272"}}
+{"timestamp":1711743886.2978163,"name":"offline","context":{"idset":"11273"}}
+{"timestamp":1711743886.2985229,"name":"offline","context":{"idset":"11274"}}
+{"timestamp":1711743886.2992225,"name":"offline","context":{"idset":"11275"}}
+{"timestamp":1711743886.2999506,"name":"offline","context":{"idset":"11276"}}
+{"timestamp":1711743886.3006313,"name":"offline","context":{"idset":"11277"}}
+{"timestamp":1711743886.3012874,"name":"offline","context":{"idset":"11278"}}
+{"timestamp":1711743886.3019614,"name":"offline","context":{"idset":"11279"}}
+{"timestamp":1711743886.3026452,"name":"offline","context":{"idset":"11280"}}
+{"timestamp":1711743886.3033104,"name":"offline","context":{"idset":"11281"}}
+{"timestamp":1711743886.3040397,"name":"offline","context":{"idset":"11282"}}
+{"timestamp":1711743886.3047147,"name":"offline","context":{"idset":"11283"}}
+{"timestamp":1711743886.3053734,"name":"offline","context":{"idset":"11284"}}
+{"timestamp":1711743886.3173075,"name":"offline","context":{"idset":"11285"}}
+{"timestamp":1711743886.3179202,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1711743886.318619,"name":"offline","context":{"idset":"11287"}}
+{"timestamp":1711743886.3193355,"name":"offline","context":{"idset":"11288"}}
+{"timestamp":1711743886.3200321,"name":"offline","context":{"idset":"11289"}}
+{"timestamp":1711743886.3207462,"name":"offline","context":{"idset":"11290"}}
+{"timestamp":1711743886.3214619,"name":"offline","context":{"idset":"11291"}}
+{"timestamp":1711743886.3221633,"name":"offline","context":{"idset":"11292"}}
+{"timestamp":1711743886.322871,"name":"offline","context":{"idset":"11293"}}
+{"timestamp":1711743886.3235774,"name":"offline","context":{"idset":"11294"}}
+{"timestamp":1711743886.3328023,"name":"offline","context":{"idset":"11295"}}
+{"timestamp":1711743886.3697383,"name":"offline","context":{"idset":"11296"}}
+{"timestamp":1711743886.4113929,"name":"offline","context":{"idset":"11297"}}
+{"timestamp":1711743886.4122972,"name":"offline","context":{"idset":"11298"}}
+{"timestamp":1711743886.4129632,"name":"offline","context":{"idset":"11299"}}
+{"timestamp":1711743886.4136121,"name":"offline","context":{"idset":"11300"}}
+{"timestamp":1711743886.4142604,"name":"offline","context":{"idset":"11301"}}
+{"timestamp":1711743886.4149113,"name":"offline","context":{"idset":"11302"}}
+{"timestamp":1711743886.415554,"name":"offline","context":{"idset":"11303"}}
+{"timestamp":1711743886.4161844,"name":"offline","context":{"idset":"11304"}}
+{"timestamp":1711743886.4167724,"name":"offline","context":{"idset":"11305"}}
+{"timestamp":1711743886.4173617,"name":"offline","context":{"idset":"11306"}}
+{"timestamp":1711743886.4284718,"name":"offline","context":{"idset":"11307"}}
+{"timestamp":1711743886.4292057,"name":"offline","context":{"idset":"11308"}}
+{"timestamp":1711743886.4298196,"name":"offline","context":{"idset":"11309"}}
+{"timestamp":1711743886.4305067,"name":"offline","context":{"idset":"11310"}}
+{"timestamp":1711743886.431169,"name":"offline","context":{"idset":"11311"}}
+{"timestamp":1711743886.4317749,"name":"offline","context":{"idset":"11312"}}
+{"timestamp":1711743886.4324555,"name":"offline","context":{"idset":"11313"}}
+{"timestamp":1711743886.4332197,"name":"offline","context":{"idset":"11314"}}
+{"timestamp":1711743886.4339025,"name":"offline","context":{"idset":"11315"}}
+{"timestamp":1711743886.4345882,"name":"offline","context":{"idset":"11316"}}
+{"timestamp":1711743886.4464471,"name":"offline","context":{"idset":"11319"}}
+{"timestamp":1711743886.489825,"name":"offline","context":{"idset":"11320"}}
+{"timestamp":1711743886.5133173,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1711743886.5140061,"name":"offline","context":{"idset":"11322"}}
+{"timestamp":1711743886.5146852,"name":"offline","context":{"idset":"11323"}}
+{"timestamp":1711743886.5153522,"name":"offline","context":{"idset":"11324"}}
+{"timestamp":1711743886.5160217,"name":"offline","context":{"idset":"11325"}}
+{"timestamp":1711743886.5166941,"name":"offline","context":{"idset":"11326"}}
+{"timestamp":1711743886.5271111,"name":"offline","context":{"idset":"11327"}}
+{"timestamp":1711743886.5668337,"name":"offline","context":{"idset":"11328"}}
+{"timestamp":1711743886.5674319,"name":"offline","context":{"idset":"11329"}}
+{"timestamp":1711743886.5679994,"name":"offline","context":{"idset":"11330"}}
+{"timestamp":1711743886.5685737,"name":"offline","context":{"idset":"11331"}}
+{"timestamp":1711743886.5691423,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1711743886.5697603,"name":"offline","context":{"idset":"11333"}}
+{"timestamp":1711743886.5702238,"name":"offline","context":{"idset":"11334"}}
+{"timestamp":1711743886.5708261,"name":"offline","context":{"idset":"11335"}}
+{"timestamp":1711743886.5714853,"name":"offline","context":{"idset":"11336"}}
+{"timestamp":1711743886.5721307,"name":"offline","context":{"idset":"11337"}}
+{"timestamp":1711743886.5727873,"name":"offline","context":{"idset":"11338"}}
+{"timestamp":1711743886.5844638,"name":"offline","context":{"idset":"11339"}}
+{"timestamp":1711743886.5850847,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1711743886.5857182,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1711743886.5863872,"name":"offline","context":{"idset":"11342"}}
+{"timestamp":1711743886.5870056,"name":"offline","context":{"idset":"11343"}}
+{"timestamp":1711743886.5876107,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1711743886.5882046,"name":"offline","context":{"idset":"11345"}}
+{"timestamp":1711743886.588799,"name":"offline","context":{"idset":"11346"}}
+{"timestamp":1711743886.6002686,"name":"offline","context":{"idset":"11347"}}
+{"timestamp":1711743886.6009252,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1711743886.6015751,"name":"offline","context":{"idset":"11349"}}
+{"timestamp":1711743886.6022141,"name":"offline","context":{"idset":"11350"}}
+{"timestamp":1711743886.6028495,"name":"offline","context":{"idset":"11351"}}
+{"timestamp":1711743886.6034567,"name":"offline","context":{"idset":"11352"}}
+{"timestamp":1711743886.6038256,"name":"offline","context":{"idset":"11353"}}
+{"timestamp":1711743886.6041532,"name":"offline","context":{"idset":"11354"}}
+{"timestamp":1711743886.6045346,"name":"offline","context":{"idset":"11355"}}
+{"timestamp":1711743886.605092,"name":"offline","context":{"idset":"11356"}}
+{"timestamp":1711743886.6056824,"name":"offline","context":{"idset":"11357"}}
+{"timestamp":1711743886.6062803,"name":"offline","context":{"idset":"11358"}}
+{"timestamp":1711743886.6177871,"name":"offline","context":{"idset":"11359"}}
+{"timestamp":1711743886.6183929,"name":"offline","context":{"idset":"11360"}}
+{"timestamp":1711743886.6189919,"name":"offline","context":{"idset":"11361"}}
+{"timestamp":1711743886.6195793,"name":"offline","context":{"idset":"11362"}}
+{"timestamp":1711743886.6201677,"name":"offline","context":{"idset":"11363"}}
+{"timestamp":1711743886.6207573,"name":"offline","context":{"idset":"11364"}}
+{"timestamp":1711743886.6213312,"name":"offline","context":{"idset":"11365"}}
+{"timestamp":1711743886.6325173,"name":"offline","context":{"idset":"11366"}}
+{"timestamp":1711743886.6770344,"name":"offline","context":{"idset":"11367"}}
+{"timestamp":1711743886.6880887,"name":"offline","context":{"idset":"11368"}}
+{"timestamp":1711743886.6886377,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1711743886.6891787,"name":"offline","context":{"idset":"11370"}}
+{"timestamp":1711743886.6897521,"name":"offline","context":{"idset":"11371"}}
+{"timestamp":1711743886.6903737,"name":"offline","context":{"idset":"11372"}}
+{"timestamp":1711743886.6909988,"name":"offline","context":{"idset":"11373"}}
+{"timestamp":1711743886.6916254,"name":"offline","context":{"idset":"11374"}}
+{"timestamp":1711743886.6922448,"name":"offline","context":{"idset":"11375"}}
+{"timestamp":1711743886.7043099,"name":"offline","context":{"idset":"11376"}}
+{"timestamp":1711743886.7048986,"name":"offline","context":{"idset":"11377"}}
+{"timestamp":1711743886.7054844,"name":"offline","context":{"idset":"11378"}}
+{"timestamp":1711743886.7060463,"name":"offline","context":{"idset":"11379"}}
+{"timestamp":1711743886.706636,"name":"offline","context":{"idset":"11380"}}
+{"timestamp":1711743886.7071989,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1711743886.7077653,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1711743886.7083161,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1711743886.7088709,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1711743886.709424,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1711743886.7099867,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1711743886.7105396,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1711743886.7110848,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1711743886.7116387,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1711743886.7230659,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1711743886.7236168,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1711743886.7241848,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1711743886.7247531,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1711743886.7253392,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1711743886.7259297,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1711743886.7264769,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1711743886.7270238,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1711743886.727592,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1711743886.7281704,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1711743886.7287488,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1711743886.7292871,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1711743886.7298534,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1711743886.7303901,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1711743886.7309566,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1711743886.7315419,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1711743886.7320979,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1711743886.7326484,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1711743886.7557991,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1711743886.7973275,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1711743886.8404729,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1711743886.8518639,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1711743886.8524573,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1711743886.8530312,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1711743886.8536115,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1711743886.8541737,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1711743886.8547139,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1711743886.8551047,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1711743886.8554118,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1711743886.8559706,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1711743886.8565433,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1711743886.8571048,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1711743886.8577008,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1711743886.9011946,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1711743886.9402761,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1711743886.9806652,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1711743886.9811835,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1711743886.9817362,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1711743886.9823036,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1711743886.9828634,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1711743886.9834185,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1711743886.9839544,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1711743886.9844933,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1711743886.9958143,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1711743886.9963076,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1711743886.9968059,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1711743886.9972892,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1711743886.9977839,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1711743886.9982662,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1711743886.9987681,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1711743886.9992473,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1711743886.9997337,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1711743887.0002594,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1711743887.0111935,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1711743887.0117056,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1711743887.0121784,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1711743887.0126395,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1711743887.013057,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1711743887.0241876,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1711743887.024761,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1711743887.0253181,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1711743887.0258806,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1711743887.0264404,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1711743887.026999,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1711743887.0275574,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1711743887.0280814,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1711743887.0286098,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1711743887.0291312,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1711743887.0296488,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1711743887.0301666,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1711743887.0307012,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1711743887.0312154,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1711743887.0317273,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1711743887.0322309,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1711743887.0326865,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1711743887.0331264,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1711743887.0335772,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1711743887.0340397,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1711743887.0344844,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1711743887.0349181,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1711743887.0353427,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1711743887.0357597,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1711743887.0362101,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1711743887.0366638,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1711743887.0370936,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1711743887.0375438,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1711743887.037998,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1711743887.0385306,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1711743887.0390723,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1711743887.0396023,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1711743887.051892,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1711743887.0524378,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1711743887.0530221,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1711743887.0534546,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1711743887.0541742,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1711743887.0545783,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1711743887.0549712,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1711743887.0553551,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1711743887.0557423,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1711743887.0561674,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1711743887.056941,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1711743887.0578222,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1711743887.0583179,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1711743887.0587807,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1711743887.0592577,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1711743887.0596738,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1711743887.0809319,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1711743887.132623,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1711743887.174931,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1711743887.2178769,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1711743887.2183685,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1711743887.218853,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1711743887.219327,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1711743887.2197986,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1711743887.2202635,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1711743887.2207332,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1711743887.2211936,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1711743887.221662,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1711743887.2221227,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1711743887.2225919,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1711743887.2230642,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1711743887.2235367,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1711743887.2239959,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1711743887.2352777,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1711743887.2670665,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1711743887.2674587,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1711743887.2678599,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1711743887.2682209,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1711743887.2686317,"name":"offline","context":{"idset":"11525"}}
+{"timestamp":1711743887.2690036,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1711743887.2693644,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1711743887.2697308,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1711743887.2701037,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1711743887.2704895,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1711743887.2708638,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1711743887.2712052,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1711743887.2715743,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1711743887.2720027,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1711743887.2723942,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1711743887.2827795,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1711743887.3266792,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1711743887.3720138,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1711743887.3724604,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1711743887.3728693,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1711743887.3733118,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1711743887.373719,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1711743887.3741283,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1711743887.3745348,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1711743887.3749282,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1711743887.3753185,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1711743887.3757222,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1711743887.376116,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1711743887.3765047,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1711743887.387784,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1711743887.3881555,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1711743887.3885396,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1711743887.3889289,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1711743887.3893251,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1711743887.389704,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1711743887.3900886,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1711743887.3904915,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1711743887.3908827,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1711743887.3912528,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1711743887.3916225,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1711743887.3919942,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1711743887.3923626,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1711743887.392756,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1711743887.3931372,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1711743887.3935406,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1711743887.3939164,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1711743887.3942893,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1711743887.3946719,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1711743887.3950427,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1711743887.3954165,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1711743887.3957865,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1711743887.3961527,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1711743887.3965237,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1711743887.3968978,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1711743887.397248,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1711743887.3975952,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1711743887.4081242,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1711743887.4084733,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1711743887.4088206,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1711743887.4091637,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1711743887.4094887,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1711743887.409811,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1711743887.4101579,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1711743887.4104903,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1711743887.4107981,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1711743887.4111013,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1711743887.4114201,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1711743887.4117424,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1711743887.4120646,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1711743887.412384,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1711743887.4126961,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1711743887.413008,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1711743887.4133272,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1711743887.4137025,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1711743887.4140148,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1711743887.4143059,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1711743887.414607,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1711743887.414916,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1711743887.4152379,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1711743887.4155846,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1711743887.4159243,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1711743887.4162602,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1711743887.4165962,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1711743887.4577944,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1711743887.5010905,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1711743887.5407977,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1711743887.582732,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1711743887.6148062,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1711743887.6151698,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1711743887.6155205,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1711743887.6158588,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1711743887.6161947,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1711743887.6165419,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1711743887.6168687,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1711743887.617218,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1711743887.6175506,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1711743887.6178761,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1711743887.6182027,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1711743887.6185317,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1711743887.6188607,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1711743887.6191907,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1711743887.6195221,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1711743887.6620023,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1711743887.7066495,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1711743887.7506099,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1711743887.7812517,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1711743887.8026693,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1711743887.8030422,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1711743887.8033574,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1711743887.8036466,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1711743887.8039861,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1711743887.8043339,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1711743887.8046808,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1711743887.8050241,"name":"offline","context":{"idset":"11637"}}
+{"timestamp":1711743887.8053679,"name":"offline","context":{"idset":"11638"}}
+{"timestamp":1711743887.8057144,"name":"offline","context":{"idset":"11639"}}
+{"timestamp":1711743887.8060999,"name":"offline","context":{"idset":"11640"}}
+{"timestamp":1711743887.806448,"name":"offline","context":{"idset":"11641"}}
+{"timestamp":1711743887.806776,"name":"offline","context":{"idset":"11642"}}
+{"timestamp":1711743887.8071144,"name":"offline","context":{"idset":"11643"}}
+{"timestamp":1711743887.8074579,"name":"offline","context":{"idset":"11644"}}
+{"timestamp":1711743887.8187165,"name":"offline","context":{"idset":"11645"}}
+{"timestamp":1711743887.819052,"name":"offline","context":{"idset":"11646"}}
+{"timestamp":1711743887.8193805,"name":"offline","context":{"idset":"11647"}}
+{"timestamp":1711743887.820133,"name":"offline","context":{"idset":"11648"}}
+{"timestamp":1711743887.8204732,"name":"offline","context":{"idset":"11649"}}
+{"timestamp":1711743887.8207963,"name":"offline","context":{"idset":"11650"}}
+{"timestamp":1711743887.8211434,"name":"offline","context":{"idset":"11651"}}
+{"timestamp":1711743887.8214757,"name":"offline","context":{"idset":"11652"}}
+{"timestamp":1711743887.8218017,"name":"offline","context":{"idset":"11653"}}
+{"timestamp":1711743887.8221629,"name":"offline","context":{"idset":"11654"}}
+{"timestamp":1711743887.8224885,"name":"offline","context":{"idset":"11655"}}
+{"timestamp":1711743887.8228586,"name":"offline","context":{"idset":"11656"}}
+{"timestamp":1711743887.8231807,"name":"offline","context":{"idset":"11657"}}
+{"timestamp":1711743887.8235035,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1711743887.8238258,"name":"offline","context":{"idset":"11659"}}
+{"timestamp":1711743887.8241456,"name":"offline","context":{"idset":"11660"}}
+{"timestamp":1711743887.8244698,"name":"offline","context":{"idset":"11661"}}
+{"timestamp":1711743887.8247862,"name":"offline","context":{"idset":"11662"}}
+{"timestamp":1711743887.8251014,"name":"offline","context":{"idset":"11663"}}
+{"timestamp":1711743887.8254185,"name":"offline","context":{"idset":"11664"}}
+{"timestamp":1711743887.825732,"name":"offline","context":{"idset":"11665"}}
+{"timestamp":1711743887.8260574,"name":"offline","context":{"idset":"11666"}}
+{"timestamp":1711743887.8263845,"name":"offline","context":{"idset":"11667"}}
+{"timestamp":1711743887.826709,"name":"offline","context":{"idset":"11668"}}
+{"timestamp":1711743887.8270161,"name":"offline","context":{"idset":"11671"}}
+{"timestamp":1711743887.8273237,"name":"offline","context":{"idset":"11672"}}
+{"timestamp":1711743887.8276322,"name":"offline","context":{"idset":"11673"}}
+{"timestamp":1711743887.8279393,"name":"offline","context":{"idset":"11674"}}
+{"timestamp":1711743887.8282437,"name":"offline","context":{"idset":"11675"}}
+{"timestamp":1711743887.8285503,"name":"offline","context":{"idset":"11676"}}
+{"timestamp":1711743887.8288581,"name":"offline","context":{"idset":"11677"}}
+{"timestamp":1711743887.8291585,"name":"offline","context":{"idset":"11678"}}
+{"timestamp":1711743887.8294697,"name":"offline","context":{"idset":"11679"}}
+{"timestamp":1711743887.8297942,"name":"offline","context":{"idset":"11680"}}
+{"timestamp":1711743887.8300917,"name":"offline","context":{"idset":"11681"}}
+{"timestamp":1711743887.8303876,"name":"offline","context":{"idset":"11682"}}
+{"timestamp":1711743887.8307228,"name":"offline","context":{"idset":"11683"}}
+{"timestamp":1711743887.8310113,"name":"offline","context":{"idset":"11684"}}
+{"timestamp":1711743887.8313048,"name":"offline","context":{"idset":"11685"}}
+{"timestamp":1711743887.8316011,"name":"offline","context":{"idset":"11686"}}
+{"timestamp":1711743887.8318903,"name":"offline","context":{"idset":"11687"}}
+{"timestamp":1711743887.8321795,"name":"offline","context":{"idset":"11688"}}
+{"timestamp":1711743887.8324807,"name":"offline","context":{"idset":"11689"}}
+{"timestamp":1711743887.8327725,"name":"offline","context":{"idset":"11690"}}
+{"timestamp":1711743887.8330607,"name":"offline","context":{"idset":"11691"}}
+{"timestamp":1711743887.833349,"name":"offline","context":{"idset":"11692"}}
+{"timestamp":1711743887.8336377,"name":"offline","context":{"idset":"11693"}}
+{"timestamp":1711743887.8339257,"name":"offline","context":{"idset":"11694"}}
+{"timestamp":1711743887.8342094,"name":"offline","context":{"idset":"11695"}}
+{"timestamp":1711743887.8345013,"name":"offline","context":{"idset":"11696"}}
+{"timestamp":1711743887.8347838,"name":"offline","context":{"idset":"11697"}}
+{"timestamp":1711743887.8350661,"name":"offline","context":{"idset":"11698"}}
+{"timestamp":1711743887.8353474,"name":"offline","context":{"idset":"11699"}}
+{"timestamp":1711743887.8465011,"name":"offline","context":{"idset":"11700"}}
+{"timestamp":1711743887.8467846,"name":"offline","context":{"idset":"11701"}}
+{"timestamp":1711743887.84706,"name":"offline","context":{"idset":"11702"}}
+{"timestamp":1711743887.8473265,"name":"offline","context":{"idset":"11703"}}
+{"timestamp":1711743887.8476069,"name":"offline","context":{"idset":"11705"}}
+{"timestamp":1711743887.8478715,"name":"offline","context":{"idset":"11706"}}
+{"timestamp":1711743887.8481367,"name":"offline","context":{"idset":"11707"}}
+{"timestamp":1711743887.8484151,"name":"offline","context":{"idset":"11708"}}
+{"timestamp":1711743887.8486905,"name":"offline","context":{"idset":"11709"}}
+{"timestamp":1711743887.8489568,"name":"offline","context":{"idset":"11710"}}
+{"timestamp":1711743887.8492208,"name":"offline","context":{"idset":"11712"}}
+{"timestamp":1711743887.8494911,"name":"offline","context":{"idset":"11713"}}
+{"timestamp":1711743887.8497508,"name":"offline","context":{"idset":"11714"}}
+{"timestamp":1711743887.8500111,"name":"offline","context":{"idset":"11715"}}
+{"timestamp":1711743887.850271,"name":"offline","context":{"idset":"11716"}}
+{"timestamp":1711743887.8505349,"name":"offline","context":{"idset":"11717"}}
+{"timestamp":1711743887.8507936,"name":"offline","context":{"idset":"11718"}}
+{"timestamp":1711743887.8510482,"name":"offline","context":{"idset":"11719"}}
+{"timestamp":1711743887.8513021,"name":"offline","context":{"idset":"11720"}}
+{"timestamp":1711743887.8515651,"name":"offline","context":{"idset":"11721"}}
+{"timestamp":1711743887.8626797,"name":"offline","context":{"idset":"11722"}}
+{"timestamp":1711743887.9063401,"name":"offline","context":{"idset":"11723"}}
+{"timestamp":1711743887.9474912,"name":"offline","context":{"idset":"11724"}}
+{"timestamp":1711743887.9477496,"name":"offline","context":{"idset":"11725"}}
+{"timestamp":1711743887.947998,"name":"offline","context":{"idset":"11726"}}
+{"timestamp":1711743887.9482417,"name":"offline","context":{"idset":"11727"}}
+{"timestamp":1711743887.948489,"name":"offline","context":{"idset":"11728"}}
+{"timestamp":1711743887.9487133,"name":"offline","context":{"idset":"11729"}}
+{"timestamp":1711743887.9489479,"name":"offline","context":{"idset":"11730"}}
+{"timestamp":1711743887.9491844,"name":"offline","context":{"idset":"11731"}}
+{"timestamp":1711743887.9494233,"name":"offline","context":{"idset":"11732"}}
+{"timestamp":1711743887.9496539,"name":"offline","context":{"idset":"11733"}}
+{"timestamp":1711743887.9498775,"name":"offline","context":{"idset":"11734"}}
+{"timestamp":1711743887.950105,"name":"offline","context":{"idset":"11735"}}
+{"timestamp":1711743887.9503326,"name":"offline","context":{"idset":"11736"}}
+{"timestamp":1711743887.9505646,"name":"offline","context":{"idset":"11737"}}
+{"timestamp":1711743887.9507957,"name":"offline","context":{"idset":"11738"}}
+{"timestamp":1711743887.9510233,"name":"offline","context":{"idset":"11739"}}
+{"timestamp":1711743887.9512417,"name":"offline","context":{"idset":"11740"}}
+{"timestamp":1711743887.9514771,"name":"offline","context":{"idset":"11741"}}
+{"timestamp":1711743887.9517112,"name":"offline","context":{"idset":"11742"}}
+{"timestamp":1711743887.9519377,"name":"offline","context":{"idset":"11743"}}
+{"timestamp":1711743887.9521601,"name":"offline","context":{"idset":"11744"}}
+{"timestamp":1711743887.9523885,"name":"offline","context":{"idset":"11745"}}
+{"timestamp":1711743887.9633784,"name":"offline","context":{"idset":"11746"}}
+{"timestamp":1711743887.9635799,"name":"offline","context":{"idset":"11747"}}
+{"timestamp":1711743887.963768,"name":"offline","context":{"idset":"11748"}}
+{"timestamp":1711743887.9639549,"name":"offline","context":{"idset":"11781"}}
+{"timestamp":1711743887.9747422,"name":"offline","context":{"idset":"11782"}}
+{"timestamp":1711743887.9749663,"name":"offline","context":{"idset":"11783"}}
+{"timestamp":1711743887.9751766,"name":"offline","context":{"idset":"11785"}}
+{"timestamp":1711743887.9753814,"name":"offline","context":{"idset":"11786"}}
+{"timestamp":1711743887.975596,"name":"offline","context":{"idset":"11787"}}
+{"timestamp":1711743887.9757965,"name":"offline","context":{"idset":"11788"}}
+{"timestamp":1711743887.9759965,"name":"offline","context":{"idset":"11790"}}
+{"timestamp":1711743887.9761992,"name":"offline","context":{"idset":"11792"}}
+{"timestamp":1711743887.9764092,"name":"offline","context":{"idset":"11795"}}
+{"timestamp":1711743887.9766068,"name":"offline","context":{"idset":"11796"}}
+{"timestamp":1711743887.976805,"name":"offline","context":{"idset":"11797"}}
+{"timestamp":1711743887.9770017,"name":"offline","context":{"idset":"11798"}}
+{"timestamp":1711743887.9771969,"name":"offline","context":{"idset":"11799"}}
+{"timestamp":1711743887.9773917,"name":"offline","context":{"idset":"11800"}}
+{"timestamp":1711743887.9775908,"name":"offline","context":{"idset":"11801"}}
+{"timestamp":1711743887.9777682,"name":"offline","context":{"idset":"11804"}}
+{"timestamp":1711743887.9779458,"name":"offline","context":{"idset":"11805"}}
+{"timestamp":1711743887.9781239,"name":"offline","context":{"idset":"11806"}}
+{"timestamp":1711743887.9782922,"name":"offline","context":{"idset":"11807"}}
+{"timestamp":1711743887.9784658,"name":"offline","context":{"idset":"11808"}}
+{"timestamp":1711743887.9786301,"name":"offline","context":{"idset":"11809"}}
+{"timestamp":1711743887.9787822,"name":"offline","context":{"idset":"11810"}}
+{"timestamp":1711743887.9789319,"name":"offline","context":{"idset":"11811"}}
+{"timestamp":1711743887.9790955,"name":"offline","context":{"idset":"11812"}}
+{"timestamp":1711743887.9792626,"name":"offline","context":{"idset":"11813"}}
+{"timestamp":1711743887.9794407,"name":"offline","context":{"idset":"11814"}}
+{"timestamp":1711743887.9796104,"name":"offline","context":{"idset":"11815"}}
+{"timestamp":1711743887.9797754,"name":"offline","context":{"idset":"11817"}}
+{"timestamp":1711743887.9799387,"name":"offline","context":{"idset":"11818"}}
+{"timestamp":1711743887.9801059,"name":"offline","context":{"idset":"11819"}}
+{"timestamp":1711743887.9802659,"name":"offline","context":{"idset":"11820"}}
+{"timestamp":1711743887.9806876,"name":"offline","context":{"idset":"11821"}}
+{"timestamp":1711743887.9808471,"name":"offline","context":{"idset":"11822"}}
+{"timestamp":1711743887.9810009,"name":"offline","context":{"idset":"11823"}}
+{"timestamp":1711743887.9811442,"name":"offline","context":{"idset":"11824"}}
+{"timestamp":1711743887.9812856,"name":"offline","context":{"idset":"11825"}}
+{"timestamp":1711743887.9814472,"name":"offline","context":{"idset":"11826"}}
+{"timestamp":1711743887.9815979,"name":"offline","context":{"idset":"11827"}}
+{"timestamp":1711743887.9817505,"name":"offline","context":{"idset":"11828"}}
+{"timestamp":1711743887.9818969,"name":"offline","context":{"idset":"11829"}}
+{"timestamp":1711743887.9820426,"name":"offline","context":{"idset":"11830"}}
+{"timestamp":1711743887.9821856,"name":"offline","context":{"idset":"11831"}}
+{"timestamp":1711743887.9823287,"name":"offline","context":{"idset":"11832"}}
+{"timestamp":1711743887.9824836,"name":"offline","context":{"idset":"11833"}}
+{"timestamp":1711743887.982626,"name":"offline","context":{"idset":"11834"}}
+{"timestamp":1711743887.9827638,"name":"offline","context":{"idset":"11835"}}
+{"timestamp":1711743887.982903,"name":"offline","context":{"idset":"11837"}}
+{"timestamp":1711743887.9830396,"name":"offline","context":{"idset":"11838"}}
+{"timestamp":1711743887.9831765,"name":"offline","context":{"idset":"11839"}}
+{"timestamp":1711743887.9833145,"name":"offline","context":{"idset":"11840"}}
+{"timestamp":1711743887.9834654,"name":"offline","context":{"idset":"11841"}}
+{"timestamp":1711743887.9836013,"name":"offline","context":{"idset":"11842"}}
+{"timestamp":1711743887.9837303,"name":"offline","context":{"idset":"11843"}}
+{"timestamp":1711743887.9838569,"name":"offline","context":{"idset":"11844"}}
+{"timestamp":1711743887.98399,"name":"offline","context":{"idset":"11845"}}
+{"timestamp":1711743887.9841242,"name":"offline","context":{"idset":"11846"}}
+{"timestamp":1711743887.9842501,"name":"offline","context":{"idset":"11847"}}
+{"timestamp":1711743887.9843876,"name":"offline","context":{"idset":"11848"}}
+{"timestamp":1711743887.9845569,"name":"offline","context":{"idset":"11849"}}
+{"timestamp":1711743887.9847054,"name":"offline","context":{"idset":"11850"}}
+{"timestamp":1711743887.9848588,"name":"offline","context":{"idset":"11851"}}
+{"timestamp":1711743887.9850068,"name":"offline","context":{"idset":"11852"}}
+{"timestamp":1711743887.9851525,"name":"offline","context":{"idset":"11853"}}
+{"timestamp":1711743887.9852982,"name":"offline","context":{"idset":"11854"}}
+{"timestamp":1711743887.9854474,"name":"offline","context":{"idset":"11855"}}
+{"timestamp":1711743887.9855926,"name":"offline","context":{"idset":"11856"}}
+{"timestamp":1711743887.9857335,"name":"offline","context":{"idset":"11857"}}
+{"timestamp":1711743887.9858809,"name":"offline","context":{"idset":"11858"}}
+{"timestamp":1711743887.9860301,"name":"offline","context":{"idset":"11859"}}
+{"timestamp":1711743887.9861772,"name":"offline","context":{"idset":"11861"}}
+{"timestamp":1711743887.986321,"name":"offline","context":{"idset":"11862"}}
+{"timestamp":1711743887.986465,"name":"offline","context":{"idset":"11863"}}
+{"timestamp":1711743887.986608,"name":"offline","context":{"idset":"11864"}}
+{"timestamp":1711743887.986757,"name":"offline","context":{"idset":"11865"}}
+{"timestamp":1711743887.9869137,"name":"offline","context":{"idset":"11866"}}
+{"timestamp":1711743887.987061,"name":"offline","context":{"idset":"11867"}}
+{"timestamp":1711743887.9871981,"name":"offline","context":{"idset":"11868"}}
+{"timestamp":1711743887.9873407,"name":"offline","context":{"idset":"11869"}}
+{"timestamp":1711743887.9874852,"name":"offline","context":{"idset":"11870"}}
+{"timestamp":1711743887.9876232,"name":"offline","context":{"idset":"11871"}}
+{"timestamp":1711743887.9877605,"name":"offline","context":{"idset":"11872"}}
+{"timestamp":1711743887.9878964,"name":"offline","context":{"idset":"11873"}}
+{"timestamp":1711743887.9880288,"name":"offline","context":{"idset":"11874"}}
+{"timestamp":1711743887.9881601,"name":"offline","context":{"idset":"11875"}}
+{"timestamp":1711743887.9882965,"name":"offline","context":{"idset":"11876"}}
+{"timestamp":1711743887.9884326,"name":"offline","context":{"idset":"11877"}}
+{"timestamp":1711743887.9990907,"name":"offline","context":{"idset":"11878"}}
+{"timestamp":1711743887.9992449,"name":"offline","context":{"idset":"11879"}}
+{"timestamp":1711743887.9993689,"name":"offline","context":{"idset":"11880"}}
+{"timestamp":1711743887.9995027,"name":"offline","context":{"idset":"11881"}}
+{"timestamp":1711743887.9996266,"name":"offline","context":{"idset":"11882"}}
+{"timestamp":1711743887.9997528,"name":"offline","context":{"idset":"11883"}}
+{"timestamp":1711743887.9998715,"name":"offline","context":{"idset":"11884"}}
+{"timestamp":1711743887.9999964,"name":"offline","context":{"idset":"11885"}}
+{"timestamp":1711743888.000118,"name":"offline","context":{"idset":"11886"}}
+{"timestamp":1711743888.0418427,"name":"offline","context":{"idset":"11887"}}
+{"timestamp":1711743888.0854747,"name":"offline","context":{"idset":"11888"}}
+{"timestamp":1711743888.1106832,"name":"offline","context":{"idset":"11889"}}
+{"timestamp":1711743888.1108065,"name":"offline","context":{"idset":"11890"}}
+{"timestamp":1711743888.1109078,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1711743888.1110048,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1711743888.1110971,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1711743888.1111882,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1711743888.1112814,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1711743888.111393,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1711743888.1114852,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1711743888.1115706,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1711743888.1116543,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1711743888.1117353,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1711743888.1118162,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1711743888.1118977,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1711743888.11198,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1711743888.1120572,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1711743888.1121345,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1711743888.1122091,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1711743888.112282,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1711743888.1123555,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1711743888.1124556,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1711743888.1125305,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1711743888.1125977,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1711743888.1126628,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1711743888.1127272,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1711743888.1324294,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1711743888.1324677,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1711744235.0356531,"name":"resource-init","context":{"restart":true,"drain":{"61":{"timestamp":1711668444.9627461,"reason":"broker was unresponsive"},"62":{"timestamp":1711668506.9728317,"reason":"broker was unresponsive"},"63":{"timestamp":1711668445.0625052,"reason":"broker was unresponsive"},"64":{"timestamp":1711668506.9729121,"reason":"broker was unresponsive"},"65":{"timestamp":1711668438.9624472,"reason":"broker was unresponsive"},"66":{"timestamp":1711668506.9729731,"reason":"broker was unresponsive"},"67":{"timestamp":1711668439.0630105,"reason":"broker was unresponsive"},"68":{"timestamp":1711668506.9730346,"reason":"broker was unresponsive"},"69-76":{"timestamp":1711646931.6480982,"reason":""},"77":{"timestamp":1711668526.9621451,"reason":"broker was unresponsive"},"78":{"timestamp":1711668462.9610252,"reason":"broker was unresponsive"},"79":{"timestamp":1711668526.96223,"reason":"broker was unresponsive"},"80":{"timestamp":1711668457.0617182,"reason":"broker was unresponsive"},"81":{"timestamp":1711668526.9622741,"reason":"broker was unresponsive"},"82":{"timestamp":1711668462.9611192,"reason":"broker was unresponsive"},"83":{"timestamp":1711668527.0620673,"reason":"broker was unresponsive"},"84":{"timestamp":1711668463.0615458,"reason":"broker was unresponsive"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"348":{"timestamp":1711576490.2548447,"reason":"broker was unresponsive"},"431":{"timestamp":1711096934.358676,"reason":"nodediag failed pci"},"445-468,541-564":{"timestamp":1711560189.6111379,"reason":"CDU work TB"},"637":{"timestamp":1711642252.1616328,"reason":"broker was unresponsive"},"638":{"timestamp":1711642252.1617086,"reason":"broker was unresponsive"},"639":{"timestamp":1711642252.161751,"reason":"broker was unresponsive"},"640":{"timestamp":1711642252.1617904,"reason":"broker was unresponsive"},"641":{"timestamp":1711642252.1618268,"reason":"broker was unresponsive"},"642":{"timestamp":1711642252.1618621,"reason":"broker was unresponsive"},"643":{"timestamp":1711642252.1618974,"reason":"broker was unresponsive"},"644":{"timestamp":1711642252.1619322,"reason":"broker was unresponsive"},"646":{"timestamp":1711586857.201689,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"647":{"timestamp":1711587136.0002518,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"648":{"timestamp":1711587197.7047591,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"649":{"timestamp":1711587211.9606316,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"650":{"timestamp":1711587232.1780922,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"651":{"timestamp":1711587244.2040467,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"652":{"timestamp":1711587255.7559283,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"653":{"timestamp":1711642252.1619685,"reason":"broker was unresponsive"},"654":{"timestamp":1711642252.16201,"reason":"broker was unresponsive"},"655":{"timestamp":1711642252.162055,"reason":"broker was unresponsive"},"656":{"timestamp":1711642252.1620932,"reason":"broker was unresponsive"},"657":{"timestamp":1711642252.1621311,"reason":"broker was unresponsive"},"658":{"timestamp":1711642252.1621685,"reason":"broker was unresponsive"},"659":{"timestamp":1711642252.1622062,"reason":"broker was unresponsive"},"660":{"timestamp":1711642252.1622446,"reason":"broker was unresponsive"},"661":{"timestamp":1711642252.1622829,"reason":"broker was unresponsive"},"662":{"timestamp":1711642252.1623201,"reason":"broker was unresponsive"},"663":{"timestamp":1711642252.1623569,"reason":"broker was unresponsive"},"664":{"timestamp":1711642252.162394,"reason":"broker was unresponsive"},"665":{"timestamp":1711642252.1624658,"reason":"broker was unresponsive"},"666":{"timestamp":1711642252.1625173,"reason":"broker was unresponsive"},"667":{"timestamp":1711642258.156193,"reason":"broker was unresponsive"},"668":{"timestamp":1711642252.1625559,"reason":"broker was unresponsive"},"669":{"timestamp":1711642252.1625948,"reason":"broker was unresponsive"},"670":{"timestamp":1711642252.1626337,"reason":"broker was unresponsive"},"671":{"timestamp":1711642252.162673,"reason":"broker was unresponsive"},"672":{"timestamp":1711642252.1627119,"reason":"broker was unresponsive"},"673":{"timestamp":1711642252.1627512,"reason":"broker was unresponsive"},"674":{"timestamp":1711642252.1627915,"reason":"broker was unresponsive"},"675":{"timestamp":1711642252.1628311,"reason":"broker was unresponsive"},"676":{"timestamp":1711642252.1628704,"reason":"broker was unresponsive"},"677":{"timestamp":1711642258.1562793,"reason":"broker was unresponsive"},"678":{"timestamp":1711642258.156333,"reason":"broker was unresponsive"},"679":{"timestamp":1711642252.1629126,"reason":"broker was unresponsive"},"680":{"timestamp":1711642252.1629534,"reason":"broker was unresponsive"},"681":{"timestamp":1711642258.1563835,"reason":"broker was unresponsive"},"682":{"timestamp":1711642252.1629946,"reason":"broker was unresponsive"},"683":{"timestamp":1711642252.1630385,"reason":"broker was unresponsive"},"684":{"timestamp":1711642252.5597043,"reason":"broker was unresponsive"},"685":{"timestamp":1711604960.2562068,"reason":"broker was unresponsive"},"687":{"timestamp":1711604954.2552111,"reason":"broker was unresponsive"},"688":{"timestamp":1711606062.2558303,"reason":"broker was unresponsive"},"689":{"timestamp":1711604580.2552168,"reason":"broker was unresponsive"},"691":{"timestamp":1711604958.2564192,"reason":"broker was unresponsive"},"692":{"timestamp":1711606406.1556187,"reason":"broker was unresponsive"},"693":{"timestamp":1711636838.2555192,"reason":"broker was unresponsive"},"695":{"timestamp":1711604326.2549248,"reason":"broker was unresponsive"},"696":{"timestamp":1711606406.256216,"reason":"broker was unresponsive"},"697":{"timestamp":1711604392.2562771,"reason":"broker was unresponsive"},"698":{"timestamp":1711605716.2557297,"reason":"broker was unresponsive"},"699":{"timestamp":1711604948.2558951,"reason":"broker was unresponsive"},"700":{"timestamp":1711606868.2557678,"reason":"broker was unresponsive"},"701":{"timestamp":1711604340.2561386,"reason":"broker was unresponsive"},"702":{"timestamp":1711606408.2602317,"reason":"broker was unresponsive"},"703":{"timestamp":1711604918.2557685,"reason":"broker was unresponsive"},"704":{"timestamp":1711604590.2548223,"reason":"broker was unresponsive"},"705":{"timestamp":1711604968.2570915,"reason":"broker was unresponsive"},"707":{"timestamp":1711604908.2558758,"reason":"broker was unresponsive"},"686,690,694,706,708":{"timestamp":1711615908.1602454,"reason":"epilog failed for jobid fm9HsYwFjjd"},"709":{"timestamp":1711408620.9668899,"reason":"Cabinet power work needed -jrg"},"710":{"timestamp":1711409910.967391,"reason":"Cabinet power work needed -jrg"},"711":{"timestamp":1711409098.9674611,"reason":"Cabinet power work needed -jrg"},"712":{"timestamp":1711411606.9678445,"reason":"Cabinet power work needed -jrg"},"713":{"timestamp":1711408618.8669355,"reason":"Cabinet power work needed -jrg"},"715":{"timestamp":1711409132.9680476,"reason":"Cabinet power work needed -jrg"},"714,716":{"timestamp":1711564709.7804377,"reason":"Cabinet power work needed -jrg"},"718":{"timestamp":1711642210.1588371,"reason":"broker was unresponsive"},"719":{"timestamp":1711642210.1589563,"reason":"broker was unresponsive"},"720":{"timestamp":1711642210.1590128,"reason":"broker was unresponsive"},"721":{"timestamp":1711642210.1590643,"reason":"broker was unresponsive"},"722":{"timestamp":1711642210.1590991,"reason":"broker was unresponsive"},"723":{"timestamp":1711642210.1591821,"reason":"broker was unresponsive"},"724":{"timestamp":1711642210.1592195,"reason":"broker was unresponsive"},"725":{"timestamp":1711593222.2542431,"reason":"broker was unresponsive"},"726":{"timestamp":1711593312.2549782,"reason":"broker was unresponsive"},"727":{"timestamp":1711593332.2552569,"reason":"broker was unresponsive"},"728":{"timestamp":1711593356.2547033,"reason":"broker was unresponsive"},"729":{"timestamp":1711593414.254441,"reason":"broker was unresponsive"},"730":{"timestamp":1711593462.2552798,"reason":"broker was unresponsive"},"731":{"timestamp":1711593510.2550235,"reason":"broker was unresponsive"},"732":{"timestamp":1711593530.2558599,"reason":"broker was unresponsive"},"733":{"timestamp":1711642210.1592536,"reason":"broker was unresponsive"},"734":{"timestamp":1711486000.9676955,"reason":"broker was unresponsive"},"735":{"timestamp":1711642210.1592872,"reason":"broker was unresponsive"},"736":{"timestamp":1711642210.1593204,"reason":"broker was unresponsive"},"737":{"timestamp":1711642210.1593742,"reason":"broker was unresponsive"},"738":{"timestamp":1711642210.1594355,"reason":"broker was unresponsive"},"739":{"timestamp":1711642210.1594796,"reason":"broker was unresponsive"},"740":{"timestamp":1711642210.159517,"reason":"broker was unresponsive"},"741":{"timestamp":1711642210.1595554,"reason":"broker was unresponsive"},"742":{"timestamp":1711642210.1595943,"reason":"broker was unresponsive"},"743":{"timestamp":1711642210.1596365,"reason":"broker was unresponsive"},"744":{"timestamp":1711642210.1596746,"reason":"broker was unresponsive"},"745":{"timestamp":1711642210.1597111,"reason":"broker was unresponsive"},"746":{"timestamp":1711506258.2527435,"reason":"broker was unresponsive"},"747":{"timestamp":1711642210.1597559,"reason":"broker was unresponsive"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"},"749":{"timestamp":1711642210.1598115,"reason":"broker was unresponsive"},"750":{"timestamp":1711642210.1598501,"reason":"broker was unresponsive"},"751":{"timestamp":1711642210.159888,"reason":"broker was unresponsive"},"752":{"timestamp":1711642210.1599293,"reason":"broker was unresponsive"},"753":{"timestamp":1711642210.1599746,"reason":"broker was unresponsive"},"754":{"timestamp":1711642210.1600144,"reason":"broker was unresponsive"},"755":{"timestamp":1711642210.1600802,"reason":"broker was unresponsive"},"756":{"timestamp":1711642210.5039413,"reason":"broker was unresponsive"},"789-796":{"timestamp":1711461777.1299369,"reason":""},"809":{"timestamp":1711667251.2030878,"reason":"broker was unresponsive"},"810":{"timestamp":1711460371.1431019,"reason":""},"870":{"timestamp":1711576713.5907419,"reason":"epilog failed for jobid fm4cUUWjqfd"},"874":{"timestamp":1711668507.0706048,"reason":"broker was unresponsive"},"875":{"timestamp":1711668503.0621645,"reason":"broker was unresponsive"},"883-884":{"timestamp":1711734121.1560712,"reason":""},"965":{"timestamp":1711663706.9630723,"reason":"broker was unresponsive"},"966":{"timestamp":1711663706.9631405,"reason":"broker was unresponsive"},"967":{"timestamp":1711663706.963171,"reason":"broker was unresponsive"},"968":{"timestamp":1711663706.9631999,"reason":"broker was unresponsive"},"969":{"timestamp":1711663706.9632339,"reason":"broker was unresponsive"},"970":{"timestamp":1711663706.9632568,"reason":"broker was unresponsive"},"971":{"timestamp":1711663706.9632795,"reason":"broker was unresponsive"},"972":{"timestamp":1711663706.9633036,"reason":"broker was unresponsive"},"973":{"timestamp":1711663706.9633393,"reason":"broker was unresponsive"},"974":{"timestamp":1711663706.9633627,"reason":"broker was unresponsive"},"975":{"timestamp":1711663706.9633861,"reason":"broker was unresponsive"},"976":{"timestamp":1711663706.9634261,"reason":"broker was unresponsive"},"977":{"timestamp":1711663706.9634516,"reason":"broker was unresponsive"},"978":{"timestamp":1711663706.9634771,"reason":"broker was unresponsive"},"979":{"timestamp":1711663706.9635053,"reason":"broker was unresponsive"},"980":{"timestamp":1711663707.147469,"reason":"broker was unresponsive"},"10037":{"timestamp":1711683168.9612861,"reason":"broker was unresponsive"},"10038":{"timestamp":1711683169.0618935,"reason":"broker was unresponsive"},"10042":{"timestamp":1711683826.9287212,"reason":"nodediag failed pci"},"10043":{"timestamp":1711683116.9607275,"reason":"broker was unresponsive"},"10044":{"timestamp":1711683117.060648,"reason":"broker was unresponsive"},"10045":{"timestamp":1711683819.2361462,"reason":"nodediag failed pci"},"10046":{"timestamp":1711683815.4715815,"reason":"nodediag failed pci"},"10051":{"timestamp":1711683212.9615688,"reason":"broker was unresponsive"},"10052":{"timestamp":1711683213.0621202,"reason":"broker was unresponsive"},"10069":{"timestamp":1711651402.3096604,"reason":"nodediag failed amdapu"},"10070":{"timestamp":1711651452.9712451,"reason":"nodediag failed amdapu"},"10075":{"timestamp":1711651454.7350454,"reason":"nodediag failed amdapu"},"10076":{"timestamp":1711653832.9647021,"reason":"broker was unresponsive"},"10077":{"timestamp":1711598424.2559545,"reason":"broker was unresponsive"},"10078":{"timestamp":1711598420.2553101,"reason":"broker was unresponsive"},"10083":{"timestamp":1711592302.2542377,"reason":"broker was unresponsive"},"10084":{"timestamp":1711651436.1179669,"reason":"nodediag failed amdapu"},"10085":{"timestamp":1711599830.1547806,"reason":"broker was unresponsive"},"10086":{"timestamp":1711599830.2550619,"reason":"broker was unresponsive"},"10087":{"timestamp":1711651431.6536202,"reason":"nodediag failed amdapu"},"10088":{"timestamp":1711651407.056253,"reason":"nodediag failed amdapu"},"10089":{"timestamp":1711651401.4053178,"reason":"nodediag failed amdapu"},"10090":{"timestamp":1711651432.9528325,"reason":"nodediag failed amdapu"},"10091":{"timestamp":1711651424.0299106,"reason":"nodediag failed amdapu"},"10092":{"timestamp":1711651441.8821375,"reason":"nodediag failed amdapu"},"10093":{"timestamp":1711651447.6579275,"reason":"nodediag failed amdapu"},"10094":{"timestamp":1711651415.2621243,"reason":"nodediag failed amdapu"},"10095":{"timestamp":1711634633.9882171,"reason":""},"10096":{"timestamp":1711591630.2554383,"reason":"Partner node needs hardware action"},"10097":{"timestamp":1711651430.1663055,"reason":"nodediag failed amdapu"},"10098":{"timestamp":1711651396.240273,"reason":"nodediag failed amdapu"},"10099":{"timestamp":1711651408.5980887,"reason":"nodediag failed amdapu"},"10100":{"timestamp":1711651448.6048837,"reason":"nodediag failed amdapu"},"10102":{"timestamp":1711656464.7456715,"reason":"rvs dumped core"},"10151":{"timestamp":1711734764.9003315,"reason":"nodediag failed clocksource pci"},"10154":{"timestamp":1711600478.1543531,"reason":"broker was unresponsive"},"10156":{"timestamp":1711600478.2542264,"reason":"broker was unresponsive"},"10157":{"timestamp":1711603028.2555907,"reason":"broker was unresponsive"},"10158":{"timestamp":1711660486.6421392,"reason":"draining to repair blade - wendy"},"10161":{"timestamp":1711742686.2489491,"reason":"nodediag failed cxi"},"10162":{"timestamp":1711742687.1360803,"reason":"nodediag failed cxi"},"10169":{"timestamp":1711742688.0076518,"reason":"nodediag failed cxi"},"10170":{"timestamp":1711742688.869864,"reason":"nodediag failed cxi"},"10183":{"timestamp":1711600556.2563355,"reason":"broker was unresponsive"},"10365":{"timestamp":1711668088.9626274,"reason":"broker was unresponsive"},"10368":{"timestamp":1711668088.9627261,"reason":"broker was unresponsive"},"10372":{"timestamp":1711668089.0613325,"reason":"broker was unresponsive"},"10374":{"timestamp":1711653832.9648218,"reason":"broker was unresponsive"},"10375":{"timestamp":1711653832.9648952,"reason":"broker was unresponsive"},"10377":{"timestamp":1711653832.9649596,"reason":"broker was unresponsive"},"10379":{"timestamp":1711653832.9650226,"reason":"broker was unresponsive"},"10380":{"timestamp":1711653832.9650848,"reason":"broker was unresponsive"},"10384":{"timestamp":1711670140.9621673,"reason":"broker was unresponsive"},"10385":{"timestamp":1711670141.0619121,"reason":"broker was unresponsive"},"10421":{"timestamp":1711592576.2549477,"reason":"broker was unresponsive"},"10425":{"timestamp":1711592578.2555511,"reason":"broker was unresponsive"},"10428":{"timestamp":1711592580.1550264,"reason":"broker was unresponsive"},"10431":{"timestamp":1711592582.1546719,"reason":"broker was unresponsive"},"10432":{"timestamp":1711592582.1547906,"reason":"broker was unresponsive"},"10433":{"timestamp":1711592580.2556479,"reason":"broker was unresponsive"},"10436":{"timestamp":1711592582.2541113,"reason":"broker was unresponsive"},"10462":{"timestamp":1711653832.9651468,"reason":"broker was unresponsive"},"10463":{"timestamp":1711653832.9652078,"reason":"broker was unresponsive"},"10464":{"timestamp":1711653832.9652689,"reason":"broker was unresponsive"},"10467":{"timestamp":1711653832.9653337,"reason":"broker was unresponsive"},"10550":{"timestamp":1711653834.9610977,"reason":"broker was unresponsive"},"10551":{"timestamp":1711653834.9612203,"reason":"broker was unresponsive"},"10552":{"timestamp":1711653834.9612973,"reason":"broker was unresponsive"},"10553":{"timestamp":1711653833.1115868,"reason":"broker was unresponsive"},"10554":{"timestamp":1711653834.9613669,"reason":"broker was unresponsive"},"10555":{"timestamp":1711653834.9614487,"reason":"broker was unresponsive"},"10556":{"timestamp":1711653835.0605469,"reason":"broker was unresponsive"},"10557":{"timestamp":1711660124.9618423,"reason":"broker was unresponsive"},"10558":{"timestamp":1711660124.9619589,"reason":"broker was unresponsive"},"10559":{"timestamp":1711660124.9620354,"reason":"broker was unresponsive"},"10560":{"timestamp":1711660124.962131,"reason":"broker was unresponsive"},"10561":{"timestamp":1711660124.9622042,"reason":"broker was unresponsive"},"10562":{"timestamp":1711660124.9622781,"reason":"broker was unresponsive"},"10563":{"timestamp":1711660124.9623649,"reason":"broker was unresponsive"},"10564":{"timestamp":1711660125.0727112,"reason":"broker was unresponsive"},"10741":{"timestamp":1711686071.0616689,"reason":"broker was unresponsive"},"10742":{"timestamp":1711726310.4522023,"reason":"nodediag failed dmi pci"},"10743":{"timestamp":1711726288.3614619,"reason":"nodediag failed dmi pci"},"10744":{"timestamp":1711726327.3123736,"reason":"nodediag failed dmi pci"},"10745":{"timestamp":1711726311.2278292,"reason":"nodediag failed dmi pci"},"10746":{"timestamp":1711726319.1374547,"reason":"nodediag failed dmi pci"},"10747":{"timestamp":1711726285.468183,"reason":"nodediag failed dmi pci"},"10748":{"timestamp":1711726281.020335,"reason":"nodediag failed dmi pci"},"10749":{"timestamp":1711733685.2440166,"reason":"nodediag failed dmi pci"},"10750":{"timestamp":1711733663.8206584,"reason":"nodediag failed dmi pci"},"10751":{"timestamp":1711733695.9094985,"reason":"nodediag failed dmi pci"},"10752":{"timestamp":1711733659.6447358,"reason":"nodediag failed dmi pci"},"10753":{"timestamp":1711733658.3502553,"reason":"nodediag failed dmi pci"},"10754":{"timestamp":1711733646.8067858,"reason":"nodediag failed dmi pci"},"10755":{"timestamp":1711733648.0229697,"reason":"nodediag failed dmi pci"},"10756":{"timestamp":1711733687.8642294,"reason":"nodediag failed dmi pci"},"10757":{"timestamp":1711733680.9899652,"reason":"nodediag failed clocksource dmi pci"},"10758":{"timestamp":1711733655.3274591,"reason":"nodediag failed dmi pci"},"10759":{"timestamp":1711733693.1085598,"reason":"nodediag failed dmi pci"},"10760":{"timestamp":1711733654.7146134,"reason":"nodediag failed dmi pci"},"10761":{"timestamp":1711733657.8636148,"reason":"nodediag failed dmi pci"},"10762":{"timestamp":1711733652.9365022,"reason":"nodediag failed dmi pci"},"10763":{"timestamp":1711733631.3445716,"reason":"nodediag failed dmi pci"},"10764":{"timestamp":1711733655.7736757,"reason":"nodediag failed dmi pci"},"10765":{"timestamp":1711733686.9549427,"reason":"nodediag failed dmi pci"},"10766":{"timestamp":1711733693.6073937,"reason":"nodediag failed dmi pci"},"10767":{"timestamp":1711733694.074327,"reason":"nodediag failed dmi pci"},"10768":{"timestamp":1711733669.349016,"reason":"nodediag failed dmi pci"},"10769":{"timestamp":1711733683.499532,"reason":"nodediag failed dmi pci"},"10770":{"timestamp":1711733653.8923004,"reason":"nodediag failed dmi pci"},"10771":{"timestamp":1711733670.1288216,"reason":"nodediag failed dmi pci"},"10772":{"timestamp":1711733670.8903363,"reason":"nodediag failed dmi pci"},"10773":{"timestamp":1711733702.917141,"reason":"nodediag failed dmi pci"},"10774":{"timestamp":1711733668.6052599,"reason":"nodediag failed dmi pci"},"10775":{"timestamp":1711733667.8328431,"reason":"nodediag failed dmi pci"},"10776":{"timestamp":1711733699.1434879,"reason":"nodediag failed dmi pci"},"10777":{"timestamp":1711733711.3039272,"reason":"nodediag failed dmi pci"},"10778":{"timestamp":1711733700.5743349,"reason":"nodediag failed dmi pci"},"10779":{"timestamp":1711733691.6027968,"reason":"nodediag failed dmi pci"},"10780":{"timestamp":1711733690.3280208,"reason":"nodediag failed dmi pci"},"10781":{"timestamp":1711733673.4463534,"reason":"nodediag failed dmi pci"},"10782":{"timestamp":1711733667.0522788,"reason":"nodediag failed dmi pci"},"10783":{"timestamp":1711733644.2002394,"reason":"nodediag failed dmi pci"},"10784":{"timestamp":1711733690.8045485,"reason":"nodediag failed dmi pci"},"10785":{"timestamp":1711733698.6518373,"reason":"nodediag failed dmi pci"},"10786":{"timestamp":1711733695.434325,"reason":"nodediag failed clocksource dmi pci"},"10787":{"timestamp":1711733702.4539537,"reason":"nodediag failed dmi pci"},"10788":{"timestamp":1711733672.6788509,"reason":"nodediag failed dmi pci"},"10789":{"timestamp":1711689953.484504,"reason":"nodediag failed dmi pci"},"10790":{"timestamp":1711689954.9819255,"reason":"nodediag failed dmi pci"},"10791":{"timestamp":1711689956.3654644,"reason":"nodediag failed dmi pci"},"10792":{"timestamp":1711689955.9200354,"reason":"nodediag failed dmi pci"},"10793":{"timestamp":1711689952.9087276,"reason":"nodediag failed dmi pci"},"10794":{"timestamp":1711689952.154294,"reason":"nodediag failed dmi pci"},"10795":{"timestamp":1711689953.9435339,"reason":"nodediag failed dmi pci"},"10796":{"timestamp":1711689956.8180678,"reason":"nodediag failed clocksource dmi pci"},"10797":{"timestamp":1711689954.5076962,"reason":"nodediag failed dmi pci"},"10798":{"timestamp":1711689955.4526725,"reason":"nodediag failed dmi pci"},"10799":{"timestamp":1711733671.594311,"reason":"nodediag failed dmi pci"},"10800":{"timestamp":1711733672.0661569,"reason":"nodediag failed dmi pci"},"10801":{"timestamp":1711733713.6947486,"reason":"nodediag failed dmi pci"},"10802":{"timestamp":1711733721.741982,"reason":"nodediag failed dmi pci"},"10803":{"timestamp":1711733694.5160217,"reason":"nodediag failed dmi pci"},"10804":{"timestamp":1711733682.6906681,"reason":"nodediag failed clocksource dmi pci"},"10805":{"timestamp":1711733657.139148,"reason":"nodediag failed dmi pci"},"10806":{"timestamp":1711733721.2832353,"reason":"nodediag failed dmi pci"},"10807":{"timestamp":1711733666.2542951,"reason":"nodediag failed dmi pci"},"10808":{"timestamp":1711733682.1142714,"reason":"nodediag failed dmi pci"},"10809":{"timestamp":1711733696.8085656,"reason":"nodediag failed dmi pci"},"10810":{"timestamp":1711733708.496907,"reason":"nodediag failed dmi pci"},"10811":{"timestamp":1711733704.2163916,"reason":"nodediag failed dmi pci"},"10812":{"timestamp":1711733689.844337,"reason":"nodediag failed dmi pci"},"10813":{"timestamp":1711733723.0788264,"reason":"nodediag failed dmi pci"},"10814":{"timestamp":1711733688.7280917,"reason":"nodediag failed dmi pci"},"10815":{"timestamp":1711733701.9776583,"reason":"nodediag failed dmi pci"},"10816":{"timestamp":1711733717.4611535,"reason":"nodediag failed dmi pci"},"10817":{"timestamp":1711733711.7876866,"reason":"nodediag failed dmi pci"},"10818":{"timestamp":1711733740.5875123,"reason":"nodediag failed dmi pci"},"10819":{"timestamp":1711733698.1820004,"reason":"nodediag failed dmi pci"},"10820":{"timestamp":1711733723.8547161,"reason":"nodediag failed dmi pci"},"10821":{"timestamp":1711733692.358212,"reason":"nodediag failed dmi pci"},"10822":{"timestamp":1711733718.2463894,"reason":"nodediag failed dmi pci"},"10823":{"timestamp":1711733696.3637471,"reason":"nodediag failed dmi pci"},"10824":{"timestamp":1711733741.3673873,"reason":"nodediag failed dmi pci"},"10825":{"timestamp":1711733710.0873327,"reason":"nodediag failed dmi pci"},"10826":{"timestamp":1711733676.932476,"reason":"nodediag failed dmi pci"},"10827":{"timestamp":1711733727.2783854,"reason":"nodediag failed dmi pci"},"10828":{"timestamp":1711733697.7252762,"reason":"nodediag failed dmi pci"},"10829":{"timestamp":1711733689.3770008,"reason":"nodediag failed dmi pci"},"10830":{"timestamp":1711733730.9831386,"reason":"nodediag failed dmi pci"},"10831":{"timestamp":1711733739.7074654,"reason":"nodediag failed dmi pci"},"10832":{"timestamp":1711733694.9659727,"reason":"nodediag failed clocksource dmi pci"},"10833":{"timestamp":1711733731.7706234,"reason":"nodediag failed dmi pci"},"10834":{"timestamp":1711733714.170361,"reason":"nodediag failed dmi pci"},"10835":{"timestamp":1711733730.1968377,"reason":"nodediag failed dmi pci"},"10836":{"timestamp":1711733701.0410399,"reason":"nodediag failed dmi pci"},"10837":{"timestamp":1711733719.3541987,"reason":"nodediag failed dmi pci"},"10838":{"timestamp":1711733700.0951796,"reason":"nodediag failed dmi pci"},"10839":{"timestamp":1711733749.6325366,"reason":"nodediag failed dmi pci"},"10840":{"timestamp":1711733744.5511096,"reason":"nodediag failed dmi pci"},"10841":{"timestamp":1711733686.0992904,"reason":"nodediag failed dmi pci"},"10842":{"timestamp":1711733706.5187273,"reason":"nodediag failed dmi pci"},"10843":{"timestamp":1711733726.5102046,"reason":"nodediag failed clocksource dmi pci"},"10844":{"timestamp":1711733704.9789884,"reason":"nodediag failed dmi pci"},"10845":{"timestamp":1711733759.4154129,"reason":"nodediag failed dmi pci"},"10846":{"timestamp":1711733736.6401687,"reason":"nodediag failed dmi pci"},"10847":{"timestamp":1711733712.2439032,"reason":"nodediag failed dmi pci"},"10848":{"timestamp":1711733707.2587597,"reason":"nodediag failed dmi pci"},"10849":{"timestamp":1711733737.1100254,"reason":"nodediag failed dmi pci"},"10850":{"timestamp":1711733724.6029553,"reason":"nodediag failed dmi pci"},"10851":{"timestamp":1711733713.2322202,"reason":"nodediag failed dmi pci"},"10852":{"timestamp":1711733741.9539423,"reason":"nodediag failed dmi pci"},"10853":{"timestamp":1711733748.0834906,"reason":"nodediag failed dmi pci"},"10854":{"timestamp":1711733718.8872747,"reason":"nodediag failed dmi pci"},"10855":{"timestamp":1711733701.5006347,"reason":"nodediag failed dmi pci"},"10856":{"timestamp":1711733738.7628512,"reason":"nodediag failed dmi pci"},"10857":{"timestamp":1711733719.8553531,"reason":"nodediag failed dmi pci"},"10858":{"timestamp":1711733715.0908253,"reason":"nodediag failed dmi pci"},"10859":{"timestamp":1711733760.7928731,"reason":"nodediag failed dmi pci"},"10860":{"timestamp":1711733729.4170434,"reason":"nodediag failed dmi pci"},"10861":{"timestamp":1711733738.064044,"reason":"nodediag failed dmi pci"},"10862":{"timestamp":1711733749.1720347,"reason":"nodediag failed dmi pci"},"10863":{"timestamp":1711733734.0151284,"reason":"nodediag failed dmi pci"},"10864":{"timestamp":1711733732.5315197,"reason":"nodediag failed dmi pci"},"10865":{"timestamp":1711733747.2219884,"reason":"nodediag failed clocksource dmi pci"},"10866":{"timestamp":1711733742.7857056,"reason":"nodediag failed dmi pci"},"10867":{"timestamp":1711733733.3034852,"reason":"nodediag failed dmi pci"},"10868":{"timestamp":1711733728.662195,"reason":"nodediag failed dmi pci"},"10871":{"timestamp":1711727083.0628095,"reason":"broker was unresponsive"},"11126":{"timestamp":1711126278.6191533,"reason":"ama fail"},"11179":{"timestamp":1711404572.9688854,"reason":"broker was unresponsive"},"11187":{"timestamp":1711401067.3461077,"reason":"reason=drain for Rabbit connection"},"11190":{"timestamp":1711399648.6946192,"reason":"bad partner"},"11213":{"timestamp":1711406762.8675106,"reason":"broker was unresponsive"},"11216":{"timestamp":1711406762.9678555,"reason":"broker was unresponsive"},"11238":{"timestamp":1711733769.1186938,"reason":"nodediag failed pci"},"11239":{"timestamp":1711733793.8033855,"reason":"nodediag failed pci"},"11240":{"timestamp":1711743415.7834704,"reason":"epilog failed for jobid fmLXp3j1Rgj"},"11241":{"timestamp":1711733762.5457294,"reason":"nodediag failed pci"},"11242":{"timestamp":1711733801.554213,"reason":"nodediag failed pci"},"11243":{"timestamp":1711733772.9328082,"reason":"nodediag failed pci"},"11244":{"timestamp":1711733796.6682739,"reason":"nodediag failed pci"},"11245":{"timestamp":1711733760.3350453,"reason":"nodediag failed pci"},"11246":{"timestamp":1711733769.9507768,"reason":"nodediag failed pci"},"11247":{"timestamp":1711733805.2468588,"reason":"nodediag failed pci"},"11295":{"timestamp":1711148027.6723082,"reason":"ama fail"},"11296":{"timestamp":1711148027.7718606,"reason":"ama fail"},"11317":{"timestamp":1711326792.8705738,"reason":"epilog failed for jobid fka2yuNfWCo"},"11388":{"timestamp":1711405119.195823,"reason":"reason=Draining for partner Rabbit"},"11394":{"timestamp":1711117417.7733912,"reason":"Rabbit link Failure"},"11417":{"timestamp":1711743415.7833021,"reason":"epilog failed for jobid fmKGqZRsvpP"},"11439":{"timestamp":1711650674.2569783,"reason":"broker was unresponsive"},"11448":{"timestamp":1711123241.6739299,"reason":"ama fail"},"11449":{"timestamp":1711123241.674036,"reason":"ama fail"},"11450":{"timestamp":1711123241.6741452,"reason":"ama fail"},"11453":{"timestamp":1711123247.6735187,"reason":"ama fail"},"11460":{"timestamp":1711123247.7819526,"reason":"ama fail"},"11517":{"timestamp":1711659950.9614942,"reason":"broker was unresponsive"},"11536":{"timestamp":1711659951.0613308,"reason":"broker was unresponsive"},"10137,10140,11566,11587":{"timestamp":1711663526.290566,"reason":"core dumps -KK"},"11188-11189,11203-11204,11387,11488,11490,11593":{"timestamp":1711325310.3051066,"reason":"Rabbit link Failure"},"11346,11605":{"timestamp":1711326134.6213346,"reason":"GPU memory in use failing apdgpu.t"},"11653":{"timestamp":1711681144.9635539,"reason":"broker was unresponsive"},"11654":{"timestamp":1711681144.9637053,"reason":"broker was unresponsive"},"11655":{"timestamp":1711681144.9637702,"reason":"broker was unresponsive"},"11656":{"timestamp":1711681144.9638529,"reason":"broker was unresponsive"},"11657":{"timestamp":1711674574.9615963,"reason":"broker was unresponsive"},"11658":{"timestamp":1711681144.9639428,"reason":"broker was unresponsive"},"11659":{"timestamp":1711681144.964011,"reason":"broker was unresponsive"},"11660":{"timestamp":1711681144.9640872,"reason":"broker was unresponsive"},"11661":{"timestamp":1711681144.9641614,"reason":"broker was unresponsive"},"11662":{"timestamp":1711681144.9642324,"reason":"broker was unresponsive"},"11663":{"timestamp":1711681144.9643033,"reason":"broker was unresponsive"},"11664":{"timestamp":1711681145.1130784,"reason":"broker was unresponsive"},"11665":{"timestamp":1711681146.9624338,"reason":"broker was unresponsive"},"11666":{"timestamp":1711674575.0620043,"reason":"broker was unresponsive"},"11667":{"timestamp":1711681147.0618732,"reason":"broker was unresponsive"},"11668":{"timestamp":1711677983.0617237,"reason":"broker was unresponsive"},"11670":{"timestamp":1711726427.2945235,"reason":"epilog failed for jobid fmLXtp1qajy"},"11704":{"timestamp":1711663617.0607617,"reason":"broker was unresponsive"},"11717-11732":{"timestamp":1711664552.0380454,"reason":"Per Brick - Rabbits Off in x19xx"},"11765-11780":{"timestamp":1711661236.4636974,"reason":"Per Brick - Rabbits Off in x19xx"},"11784":{"timestamp":1711725755.0632687,"reason":"broker was unresponsive"},"11794":{"timestamp":1711725767.0656931,"reason":"broker was unresponsive"},"11803":{"timestamp":1711668961.0618374,"reason":"broker was unresponsive"},"11860":{"timestamp":1711727735.0622723,"reason":"broker was unresponsive"},"11892":{"timestamp":1711727957.0627635,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1711744235.0495574,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1711744268.4762194,"name":"online","context":{"idset":"0"}}
+{"timestamp":1711744270.2791479,"name":"online","context":{"idset":"10097,10374,11187,11448-11449,11655,11661-11663"}}
+{"timestamp":1711744270.4145353,"name":"online","context":{"idset":"10038,10044,10088-10089,10094,10098-10099,10741,11190,11394,11490,11605,11653-11654,11656,11659,11717"}}
+{"timestamp":1711744270.5449774,"name":"online","context":{"idset":"10037,10043,10069-10070,10087,10090-10093,10100,10379-10380,11179,11188-11189,11203-11204,11213,11346,11450,11453,11488,11657-11658,11660,11664-11668"}}
+{"timestamp":1711744270.6755924,"name":"online","context":{"idset":"10095,10398"}}
+{"timestamp":1711744270.8344305,"name":"online","context":{"idset":"10397,10458,11141,11386,11878"}}
+{"timestamp":1711744270.9654086,"name":"online","context":{"idset":"10082,10359-10360,10406,10419,10850,11125,11148,11198,11224,11237-11238,11274,11309,11316,11480,11541,11676,11700,11823,11861"}}
+{"timestamp":1711744271.0972779,"name":"online","context":{"idset":"10152,10161-10162,10416,10747,10756,10818,11167,11177,11199,11206,11209,11219,11221,11334,11345,11438,11446,11451,11467,11489,11501,11508-11509,11535,11564,11616,11680,11712,11716,11790,11839,11883"}}
+{"timestamp":1711744271.1997349,"name":"online","context":{"idset":"109,117,123,254,10047,10209,10410,10757,11182,11192,11202,11214,11368,11373,11402,11599,11607,11647,11679,11689,11732,11787,11800,11805,11838,11847,11856,11868"}}
+{"timestamp":1711744271.3037763,"name":"online","context":{"idset":"44,10048,10074,10159-10160,10407,10440,10444,10451,10764,10783,10815,10834,10843,10856,11180,11185,11210,11246,11260,11289,11343,11429,11433,11514,11621,11675,11719,11726,11729,11738,11748,11814"}}
+{"timestamp":1711744271.4387739,"name":"online","context":{"idset":"59,630-635,10072,10107,10128,10137,10184,10188,10417,10441,10760,10778,10793,10795,10797,10802,10859,11142,11158,11175,11178,11184,11191,11197,11207,11218,11220,11222,11226,11235,11245,11259,11329,11331-11332,11354,11384,11399,11414-11415,11418-11419,11421,11458-11459,11464,11550,11579,11613,11630,11643,11671,11673,11677,11734,11783,11785,11792,11850,11855,11889"}}
+{"timestamp":1711744271.5441055,"name":"online","context":{"idset":"39,10040,10045,10079,10150,10167,10170,10189,10200,10445,10748,10752,10785,10788,10840,10851-10852,11138,11146,11152,11181,11183,11193,11195-11196,11201,11205,11211,11217,11230,11232,11234,11247-11249,11281,11290,11304,11348,11357,11365,11395,11412,11423,11425,11441,11452,11455,11474,11481,11507,11510,11518,11538,11546,11557,11585,11591,11611,11637,11674,11678,11681,11683,11718,11720,11723,11725,11736,11781,11795,11806,11812,11817,11848,11886,11888"}}
+{"timestamp":1711744271.6564288,"name":"online","context":{"idset":"107,194,10127,10164,10439,10447,10749,10765,10796,10808,10832,10857,11130,11137,11144-11145,11153,11156,11194,11212,11223,11225,11241,11244,11250,11252,11254,11291,11301,11303,11312,11326,11351,11372,11382,11389-11391,11408,11416,11427,11435,11445,11447,11454,11456,11468,11478,11482,11484,11486,11504,11527-11528,11532,11563,11573,11578,11580,11595,11598,11618,11627,11634,11639,11672,11682,11702,11710,11721,11724,11727,11730,11745,11747,11782,11786,11796,11799,11801,11820,11834,11840,11844,11852,11862,11869,11881"}}
+{"timestamp":1711744271.7733407,"name":"online","context":{"idset":"85,87,89,91,93,95,97,99,101,103,105,111,113,115,119,132-134,148,151,166,178,199,227-228,339,360,379,384,401,404,424,470,537,540,569,577,581,585,608,614,616,628,10050,10071,10080,10146,10151,10187,10193,10216,10452,10742,10767,10772,10787,10810,10822,10845-10846,10848,10854,10860,10865,11132,11140,11147,11151,11155,11166,11170,11176,11186,11208,11231,11236,11239,11243,11251,11258,11262,11264,11266,11283-11284,11286,11293-11294,11300,11302,11306-11307,11321,11324-11325,11327-11328,11335,11344,11347,11349,11374,11376,11381,11383,11392-11393,11396,11406,11420,11434,11443,11457,11469,11487,11492,11500,11513,11519,11521,11523,11531,11534,11539-11540,11558,11569,11572,11589-11590,11609,11615,11619,11631,11645,11649,11684,11688,11698,11706,11708,11713-11715,11722,11731,11740-11741,11744,11788,11808,11810-11811,11821-11822,11826,11828,11831,11837,11841-11842,11845-11846,11854,11857,11864,11870,11879,11882,11887,11890"}}
+{"timestamp":1711744271.8869505,"name":"online","context":{"idset":"26,31-32,38,42,45-46,58,98,116,120,125,129,131,135-137,139-140,142-146,149,152-153,155,158,160,164-165,167-168,170-171,173,177,179-180,182,189-191,193,196-198,201-205,208,210,212,215-216,218-220,222,226,230-232,234-236,239-240,242,244,247,251,253,255,258-259,270,278,285,289,291-293,299,301,309-313,315-316,318,320,322-325,329-331,333-334,336,338,340,342,344,346,349,351,353-354,356,358-359,361-363,365,373,381-383,385,387-390,406,408,411-412,414,416,418-422,426,428-430,433-434,437,439,441-442,473-474,476,478-479,481-486,488-490,494,496-497,502-503,507-509,511-516,520-521,523-525,527-529,532-533,535-536,538-539,570-573,575,590,592-593,595,598,602-604,606,610,613,615,619,622-627,629,10073,10163,10191,10198-10199,10201,10227,10448,10745,10751,10753-10754,10758-10759,10763,10799,10803,10809,10811,10816,10820,10830,10842,10844,10858,11128,11134,11143,11157,11159,11162,11200,11227-11229,11240,11255,11265,11268,11280,11285,11287-11288,11292,11297-11299,11311,11315,11319-11320,11322-11323,11330,11336,11338-11342,11359,11363,11367,11375,11378-11380,11385,11401,11403,11413,11422,11431,11436-11437,11444,11462-11463,11465-11466,11470,11472,11475,11485,11495,11503,11505-11506,11512,11515,11525-11526,11529,11543,11545,11547,11551,11555-11556,11561,11566,11574-11575,11577,11581,11592,11597,11600-11601,11603,11612,11614,11622,11624,11633,11638,11641,11652,11690,11692,11705,11709,11737,11743,11797-11798,11804,11807,11809,11818,11824,11829,11832-11833,11849,11858,11863,11865,11871,11877,11880,11884-11885"}}
+{"timestamp":1711744272.0034556,"name":"online","context":{"idset":"29,51,86,88,90,92,94,96,100,102,104,106,108,110,112,114,118,122,124,126-128,130,141,147,150,154,156-157,159,161-163,169,172,174-176,181,183-187,192,200,206-207,209,211,213,224-225,229,237-238,241,243,245,248-250,252,256-257,260-266,268-269,271,273-275,279-281,284,286-288,290,294-298,300,302,304-308,314,317,319,321,327-328,332,335,337,341,343,345,347,350,352,357,364,366-372,374-378,380,386,391-400,402-403,405,407,409-410,415,417,425,427,432,435-436,440,443-444,471-472,475,477,480,487,491-493,495,500-501,504-506,510,517-519,522,530-531,534,565-568,574,576,579-580,582-584,587-589,591,594,596-597,599,601,605,607,609,611-612,617-618,620-621,10039,10041,10046,10049,10081,10106,10129,10133,10138,10179,10186,10192,10196,10220-10221,10438,10442,10450,10750,10782,10812-10813,10817,10819,10826,10836-10838,10847,10862-10863,11129,11131,11136,11161,11168,11171,11233,11242,11253,11257,11263,11267,11269,11271-11273,11275-11279,11305,11308,11310,11314,11333,11350,11353,11361,11366,11369-11371,11377,11397,11405,11410-11411,11428,11430,11432,11440,11442,11461,11473,11476-11477,11479,11491,11493,11497-11499,11502,11511,11524,11530,11537,11542,11544,11554,11559-11560,11562,11565,11567,11570-11571,11576,11582-11583,11586,11588,11594,11606,11608,11610,11617,11620,11625,11628-11629,11636,11685,11687,11693,11695-11697,11699,11701,11703,11707,11739,11813,11815,11830,11866,11872-11874,11876"}}
+{"timestamp":1711744272.1193516,"name":"online","context":{"idset":"138,195,214,233,246,267,272,276-277,282-283,303,355,413,423,438,469,498-499,526,578,586,600,10042,10130,10134-10135,10142,10149,10171,10176,10181,10185,10194-10195,10204,10207,10211,10214-10215,10217,10743,10755,10762,10770,10775,10781,10786,10789-10790,10805,10814,10821,10828,10833,10849,10868,11127,11133,11135,11139,11160,11163-11165,11169,11172,11261,11270,11282,11313,11337,11355,11358,11360,11364,11398,11400,11404,11407,11409,11424,11483,11494,11496,11516,11520,11522,11548-11549,11552-11553,11568,11584,11596,11602,11604,11623,11626,11632,11635,11640,11644,11646,11648,11686,11691,11694,11728,11733,11819,11825,11827,11835,11843,11851,11853,11859,11867,11875"}}
+{"timestamp":1711744272.2398214,"name":"online","context":{"idset":"15,19,23-24,50,10068,10102,10126,10147,10180,10182,10197,10210,10212,10218,10746,10761,10766,10768,10771,10791,10794,10801,10835,10861,11352,11356,11362,11426,11642,11651,11735,11742,11746"}}
+{"timestamp":1711744272.3507328,"name":"online","context":{"idset":"6,16,25,27-28,30,33-37,40-41,43,47,49,52-57,10101,10105,10109-10110,10120,10122,10132,10136,10140,10173-10175,10177-10178,10190,10203,10213,10769,10777,10784,10800,10804,10806-10807,10823-10825,10827,10831,10839,10841,10853,10866,11650"}}
+{"timestamp":1711744272.4523945,"name":"online","context":{"idset":"48,60,10053,10062,10104,10116,10222,10226,10774,10779-10780,10798,10855,10864,10867"}}
+{"timestamp":1711744272.5628879,"name":"online","context":{"idset":"13,10060,10112,10123,10125,10131,10139,10141,10144-10145,10165-10166,10172,10208,10224,10228,10744,10773,10776"}}
+{"timestamp":1711744272.6710863,"name":"online","context":{"idset":"1-5,7-12,17-18,20,10054,10059,10064,10108,10114-10115,10118,10792,10829"}}
+{"timestamp":1711744272.7722106,"name":"online","context":{"idset":"14,21-22,10055-10058,10061,10063,10113,10124,10143,10169,10219,10223,10225"}}
+{"timestamp":1711744272.8796489,"name":"online","context":{"idset":"10065,10067,10103"}}
+{"timestamp":1711744273.0222795,"name":"online","context":{"idset":"10066,10111,10119,10121,10168"}}
+{"timestamp":1711744273.1628144,"name":"online","context":{"idset":"10117"}}
+{"timestamp":1711744474.7068515,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1711744485.1922078,"name":"drain","context":{"idset":"10152","reason":"broker was unresponsive"}}
+{"timestamp":1711744485.19349,"name":"drain","context":{"idset":"10159","reason":"broker was unresponsive"}}
+{"timestamp":1711744485.1935451,"name":"drain","context":{"idset":"10160","reason":"broker was unresponsive"}}
+{"timestamp":1711744542.7249076,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1711744542.7357981,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1711744542.7461743,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1711744542.8151145,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1711745028.8140945,"name":"offline","context":{"idset":"10095"}}
+{"timestamp":1711746186.9952898,"name":"online","context":{"idset":"10095"}}
+{"timestamp":1711747044.7180438,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1711747044.7239637,"name":"offline","context":{"idset":"10743"}}
+{"timestamp":1711747044.7297392,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1711747044.8121042,"name":"offline","context":{"idset":"10745"}}
+{"timestamp":1711747046.7210033,"name":"offline","context":{"idset":"10746"}}
+{"timestamp":1711747046.7299392,"name":"offline","context":{"idset":"10747"}}
+{"timestamp":1711747046.7384727,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1711747046.8129272,"name":"offline","context":{"idset":"10749"}}
+{"timestamp":1711747048.7220497,"name":"offline","context":{"idset":"10750"}}
+{"timestamp":1711747048.7319927,"name":"offline","context":{"idset":"10751"}}
+{"timestamp":1711747048.7441046,"name":"offline","context":{"idset":"10752"}}
+{"timestamp":1711747048.7531943,"name":"offline","context":{"idset":"10753"}}
+{"timestamp":1711747048.8120885,"name":"offline","context":{"idset":"10754"}}
+{"timestamp":1711747050.7228243,"name":"offline","context":{"idset":"10755"}}
+{"timestamp":1711747050.7325497,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1711747050.7423394,"name":"offline","context":{"idset":"10758"}}
+{"timestamp":1711747050.7514527,"name":"offline","context":{"idset":"10759"}}
+{"timestamp":1711747050.8120522,"name":"offline","context":{"idset":"10760"}}
+{"timestamp":1711747052.7179284,"name":"offline","context":{"idset":"10761"}}
+{"timestamp":1711747052.7246566,"name":"offline","context":{"idset":"10762"}}
+{"timestamp":1711747052.7305832,"name":"offline","context":{"idset":"10763"}}
+{"timestamp":1711747052.8114042,"name":"offline","context":{"idset":"10764"}}
+{"timestamp":1711747054.7209401,"name":"offline","context":{"idset":"10765"}}
+{"timestamp":1711747054.7309029,"name":"offline","context":{"idset":"10766"}}
+{"timestamp":1711747054.7398343,"name":"offline","context":{"idset":"10767"}}
+{"timestamp":1711747054.8122649,"name":"offline","context":{"idset":"10768"}}
+{"timestamp":1711747056.7172296,"name":"offline","context":{"idset":"10769"}}
+{"timestamp":1711747056.7231448,"name":"offline","context":{"idset":"10770"}}
+{"timestamp":1711747056.8118229,"name":"offline","context":{"idset":"10771"}}
+{"timestamp":1711747058.7177019,"name":"offline","context":{"idset":"10772"}}
+{"timestamp":1711747058.8116615,"name":"offline","context":{"idset":"10773"}}
+{"timestamp":1711747060.7300489,"name":"offline","context":{"idset":"10775"}}
+{"timestamp":1711747060.7452531,"name":"offline","context":{"idset":"10776"}}
+{"timestamp":1711747060.7561376,"name":"offline","context":{"idset":"10777"}}
+{"timestamp":1711747060.7729642,"name":"offline","context":{"idset":"10778"}}
+{"timestamp":1711747060.8634093,"name":"offline","context":{"idset":"10779"}}
+{"timestamp":1711747062.7176592,"name":"offline","context":{"idset":"10780"}}
+{"timestamp":1711747062.7247155,"name":"offline","context":{"idset":"10782"}}
+{"timestamp":1711747062.8114812,"name":"offline","context":{"idset":"10784"}}
+{"timestamp":1711747064.7203147,"name":"offline","context":{"idset":"10785"}}
+{"timestamp":1711747064.7295058,"name":"offline","context":{"idset":"10787"}}
+{"timestamp":1711747064.7382772,"name":"offline","context":{"idset":"10788"}}
+{"timestamp":1711747064.8114784,"name":"offline","context":{"idset":"10789"}}
+{"timestamp":1711747066.7217209,"name":"offline","context":{"idset":"10790"}}
+{"timestamp":1711747066.732347,"name":"offline","context":{"idset":"10791"}}
+{"timestamp":1711747066.7411399,"name":"offline","context":{"idset":"10792"}}
+{"timestamp":1711747066.812314,"name":"offline","context":{"idset":"10793"}}
+{"timestamp":1711747068.7235203,"name":"offline","context":{"idset":"10794"}}
+{"timestamp":1711747068.7335362,"name":"offline","context":{"idset":"10795"}}
+{"timestamp":1711747068.7440674,"name":"offline","context":{"idset":"10797"}}
+{"timestamp":1711747068.753238,"name":"offline","context":{"idset":"10798"}}
+{"timestamp":1711747068.8123631,"name":"offline","context":{"idset":"10799"}}
+{"timestamp":1711747070.7172227,"name":"offline","context":{"idset":"10800"}}
+{"timestamp":1711747070.723541,"name":"offline","context":{"idset":"10801"}}
+{"timestamp":1711747070.8116188,"name":"offline","context":{"idset":"10802"}}
+{"timestamp":1711747072.7149522,"name":"offline","context":{"idset":"10803"}}
+{"timestamp":1711747072.8107438,"name":"offline","context":{"idset":"10805"}}
+{"timestamp":1711747074.7166622,"name":"offline","context":{"idset":"10806"}}
+{"timestamp":1711747074.7222569,"name":"offline","context":{"idset":"10807"}}
+{"timestamp":1711747074.8115931,"name":"offline","context":{"idset":"10808"}}
+{"timestamp":1711747076.7224565,"name":"offline","context":{"idset":"10809"}}
+{"timestamp":1711747076.7315392,"name":"offline","context":{"idset":"10810"}}
+{"timestamp":1711747076.7401812,"name":"offline","context":{"idset":"10811"}}
+{"timestamp":1711747076.7487619,"name":"offline","context":{"idset":"10812"}}
+{"timestamp":1711747076.8125405,"name":"offline","context":{"idset":"10813"}}
+{"timestamp":1711747078.7168319,"name":"offline","context":{"idset":"10814"}}
+{"timestamp":1711747078.7226875,"name":"offline","context":{"idset":"10815"}}
+{"timestamp":1711747078.8122666,"name":"offline","context":{"idset":"10816"}}
+{"timestamp":1711747080.7166445,"name":"offline","context":{"idset":"10817"}}
+{"timestamp":1711747080.7219167,"name":"offline","context":{"idset":"10818"}}
+{"timestamp":1711747080.8121946,"name":"offline","context":{"idset":"10819"}}
+{"timestamp":1711747082.7205868,"name":"offline","context":{"idset":"10820"}}
+{"timestamp":1711747082.7298899,"name":"offline","context":{"idset":"10821"}}
+{"timestamp":1711747082.7387383,"name":"offline","context":{"idset":"10822"}}
+{"timestamp":1711747082.8126614,"name":"offline","context":{"idset":"10823"}}
+{"timestamp":1711747084.7210789,"name":"offline","context":{"idset":"10824"}}
+{"timestamp":1711747084.7313108,"name":"offline","context":{"idset":"10825"}}
+{"timestamp":1711747084.7435544,"name":"offline","context":{"idset":"10826"}}
+{"timestamp":1711747084.8127756,"name":"offline","context":{"idset":"10827"}}
+{"timestamp":1711747086.7181163,"name":"offline","context":{"idset":"10828"}}
+{"timestamp":1711747086.7242768,"name":"offline","context":{"idset":"10829"}}
+{"timestamp":1711747086.7325015,"name":"offline","context":{"idset":"10830"}}
+{"timestamp":1711747086.8118258,"name":"offline","context":{"idset":"10831"}}
+{"timestamp":1711747088.7176349,"name":"offline","context":{"idset":"10833"}}
+{"timestamp":1711747088.7240434,"name":"offline","context":{"idset":"10834"}}
+{"timestamp":1711747088.7299724,"name":"offline","context":{"idset":"10835"}}
+{"timestamp":1711747088.8119192,"name":"offline","context":{"idset":"10836"}}
+{"timestamp":1711747090.7206826,"name":"offline","context":{"idset":"10837"}}
+{"timestamp":1711747090.730222,"name":"offline","context":{"idset":"10838"}}
+{"timestamp":1711747090.7421634,"name":"offline","context":{"idset":"10839"}}
+{"timestamp":1711747090.8127682,"name":"offline","context":{"idset":"10840"}}
+{"timestamp":1711747092.7191794,"name":"offline","context":{"idset":"10841"}}
+{"timestamp":1711747092.7276912,"name":"offline","context":{"idset":"10842"}}
+{"timestamp":1711747092.7383506,"name":"offline","context":{"idset":"10844"}}
+{"timestamp":1711747092.8119724,"name":"offline","context":{"idset":"10845"}}
+{"timestamp":1711747094.7175176,"name":"offline","context":{"idset":"10846"}}
+{"timestamp":1711747094.7231736,"name":"offline","context":{"idset":"10847"}}
+{"timestamp":1711747094.7308943,"name":"offline","context":{"idset":"10848"}}
+{"timestamp":1711747094.8118806,"name":"offline","context":{"idset":"10849"}}
+{"timestamp":1711747096.7206719,"name":"offline","context":{"idset":"10850"}}
+{"timestamp":1711747096.7287724,"name":"offline","context":{"idset":"10851"}}
+{"timestamp":1711747096.7356997,"name":"offline","context":{"idset":"10852"}}
+{"timestamp":1711747096.7431388,"name":"offline","context":{"idset":"10853"}}
+{"timestamp":1711747096.8113794,"name":"offline","context":{"idset":"10854"}}
+{"timestamp":1711747098.7212954,"name":"offline","context":{"idset":"10855"}}
+{"timestamp":1711747098.730927,"name":"offline","context":{"idset":"10856"}}
+{"timestamp":1711747098.7398927,"name":"offline","context":{"idset":"10857"}}
+{"timestamp":1711747098.8133726,"name":"offline","context":{"idset":"10858"}}
+{"timestamp":1711747100.7202241,"name":"offline","context":{"idset":"10859"}}
+{"timestamp":1711747100.7295465,"name":"offline","context":{"idset":"10860"}}
+{"timestamp":1711747100.7385225,"name":"offline","context":{"idset":"10861"}}
+{"timestamp":1711747100.8120275,"name":"offline","context":{"idset":"10862"}}
+{"timestamp":1711747102.7171993,"name":"offline","context":{"idset":"10863"}}
+{"timestamp":1711747102.722904,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1711747102.7283225,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1711747102.7334855,"name":"offline","context":{"idset":"10867"}}
+{"timestamp":1711747102.8110616,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1711747108.812413,"name":"offline","context":{"idset":"10774"}}
+{"timestamp":1711747112.7179685,"name":"offline","context":{"idset":"10781"}}
+{"timestamp":1711747112.8129613,"name":"offline","context":{"idset":"10783"}}
+{"timestamp":1711748181.7182167,"name":"online","context":{"idset":"11892"}}
+{"timestamp":1711748181.9834838,"name":"online","context":{"idset":"11891"}}
+{"timestamp":1711749450.6987739,"name":"online","context":{"idset":"841"}}
+{"timestamp":1711749532.0779164,"name":"online","context":{"idset":"11704"}}
+{"timestamp":1711749793.9952123,"name":"undrain","context":{"idset":"11238-11247"}}
+{"timestamp":1711750837.0583324,"name":"undrain","context":{"idset":"11189-11190,11203-11204"}}
+{"timestamp":1711752270.6693094,"name":"online","context":{"idset":"10931"}}
+{"timestamp":1711752270.6708956,"name":"online","context":{"idset":"10917,10926-10927,10932"}}
+{"timestamp":1711752270.6720691,"name":"online","context":{"idset":"10928,10930"}}
+{"timestamp":1711752270.6733024,"name":"online","context":{"idset":"10918,10921-10923"}}
+{"timestamp":1711752270.6745274,"name":"online","context":{"idset":"10924,10929"}}
+{"timestamp":1711752502.8126442,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1711752921.7045662,"name":"online","context":{"idset":"10742,10751"}}
+{"timestamp":1711752921.7619171,"name":"online","context":{"idset":"10750"}}
+{"timestamp":1711752921.9444401,"name":"online","context":{"idset":"10769"}}
+{"timestamp":1711752921.9797313,"name":"online","context":{"idset":"10766"}}
+{"timestamp":1711752922.242368,"name":"online","context":{"idset":"10746,10749"}}
+{"timestamp":1711752922.2525098,"name":"online","context":{"idset":"10753"}}
+{"timestamp":1711752922.3430297,"name":"online","context":{"idset":"10763"}}
+{"timestamp":1711752922.5799491,"name":"online","context":{"idset":"10755,10758,10762"}}
+{"timestamp":1711752922.6937079,"name":"online","context":{"idset":"10743,10754,10791"}}
+{"timestamp":1711752922.7706103,"name":"online","context":{"idset":"10760"}}
+{"timestamp":1711752922.9166505,"name":"online","context":{"idset":"10748,10782"}}
+{"timestamp":1711752923.0739458,"name":"online","context":{"idset":"10756,10784"}}
+{"timestamp":1711752923.2169104,"name":"online","context":{"idset":"10759,10779"}}
+{"timestamp":1711752923.357935,"name":"online","context":{"idset":"10744,10767,10770"}}
+{"timestamp":1711752923.5886481,"name":"online","context":{"idset":"10778,10793,10812,10822"}}
+{"timestamp":1711752923.698801,"name":"online","context":{"idset":"10745,10764"}}
+{"timestamp":1711752923.8600674,"name":"online","context":{"idset":"10747,10752,10761,10800"}}
+{"timestamp":1711752923.9707601,"name":"online","context":{"idset":"10777,10785,10799"}}
+{"timestamp":1711752924.1214783,"name":"online","context":{"idset":"10765,10768,10771,10775,10781,10801,10811"}}
+{"timestamp":1711752924.2298691,"name":"online","context":{"idset":"10773,10790,10802,10835,10840"}}
+{"timestamp":1711752924.3759322,"name":"online","context":{"idset":"10772,10774,10808,10823,10828,10841,10867"}}
+{"timestamp":1711752924.5140634,"name":"online","context":{"idset":"10794,10845"}}
+{"timestamp":1711752924.6299806,"name":"online","context":{"idset":"10783,10798,10807,10809,10826,10862"}}
+{"timestamp":1711752924.7158887,"name":"online","context":{"idset":"10787-10788,10803,10816,10834"}}
+{"timestamp":1711752924.8159029,"name":"online","context":{"idset":"10780,10814-10815,10818,10821,10831,10833,10838,10844,10847,10855,10858,10860"}}
+{"timestamp":1711752924.9558206,"name":"online","context":{"idset":"10795,10820,10839,10846,10853,10861,10863-10864,10866"}}
+{"timestamp":1711752925.0659878,"name":"online","context":{"idset":"10797,10805,10817,10829-10830,10837,10850"}}
+{"timestamp":1711752925.1764629,"name":"online","context":{"idset":"10792,10810,10836,10842,10848-10849,10859"}}
+{"timestamp":1711752925.2871318,"name":"online","context":{"idset":"10806,10852,10854,10856-10857,10868"}}
+{"timestamp":1711752925.4991035,"name":"online","context":{"idset":"10819,10824-10825"}}
+{"timestamp":1711752925.666393,"name":"online","context":{"idset":"10776,10813,10827"}}
+{"timestamp":1711752925.9724364,"name":"online","context":{"idset":"10851"}}
+{"timestamp":1711752926.0837753,"name":"online","context":{"idset":"10789"}}
+{"timestamp":1711754625.594959,"name":"drain","context":{"idset":"11237-11252","overwrite":0}}
+{"timestamp":1711754668.9253089,"name":"undrain","context":{"idset":"11237-11252"}}
+{"timestamp":1711755918.2565866,"name":"undrain","context":{"idset":"11346"}}
+{"timestamp":1711756264.9512467,"name":"drain","context":{"idset":"348","reason":"pci: 0000:03:00.1 width x8, expected x16","overwrite":1}}
+{"timestamp":1711756699.007709,"name":"offline","context":{"idset":"11891"}}
+{"timestamp":1711758318.1100662,"name":"undrain","context":{"idset":"10742-10756,10758-10785,10787-10795,10797-10803,10805-10831,10833-10842,10844-10864,10866-10868"}}
+{"timestamp":1711758579.6413181,"name":"online","context":{"idset":"10083"}}
+{"timestamp":1711758966.0068121,"name":"undrain","context":{"idset":"11387-11388,11394,11439"}}
+{"timestamp":1711760064.5222507,"name":"offline","context":{"idset":"10083"}}
+{"timestamp":1711761631.084892,"name":"undrain","context":{"idset":"11488,11490"}}
+{"timestamp":1711761956.8501387,"name":"undrain","context":{"idset":"11593,11605"}}
+{"timestamp":1711762170.0976398,"name":"drain","context":{"idset":"11593","overwrite":0}}
+{"timestamp":1711762210.5458412,"name":"online","context":{"idset":"10083"}}
+{"timestamp":1711763403.3143895,"name":"drain","context":{"idset":"10741","reason":"dmi nodediag failed dmi","overwrite":1}}
+{"timestamp":1711763445.765372,"name":"undrain","context":{"idset":"10037-10038,10043-10044,10083,10374,10379-10380,11179,11213,11653-11668,11704,11892"}}
+{"timestamp":1711765178.6653776,"name":"undrain","context":{"idset":"10042,10045-10046"}}
+{"timestamp":1711765697.594394,"name":"undrain","context":{"idset":"11717-11732"}}
+{"timestamp":1711765762.6662257,"name":"undrain","context":{"idset":"10069-10070,10087-10094,10097-10100"}}
+{"timestamp":1711774951.8277757,"name":"drain","context":{"idset":"11240","reason":"epilog failed for jobid fmUJGNzW5E7","overwrite":0}}
+{"timestamp":1711775098.7875574,"name":"undrain","context":{"idset":"11448-11450,11453"}}
+{"timestamp":1711852278.0364819,"name":"resource-init","context":{"restart":true,"drain":{"61":{"timestamp":1711668444.9627461,"reason":"broker was unresponsive"},"62":{"timestamp":1711668506.9728317,"reason":"broker was unresponsive"},"63":{"timestamp":1711668445.0625052,"reason":"broker was unresponsive"},"64":{"timestamp":1711668506.9729121,"reason":"broker was unresponsive"},"65":{"timestamp":1711668438.9624472,"reason":"broker was unresponsive"},"66":{"timestamp":1711668506.9729731,"reason":"broker was unresponsive"},"67":{"timestamp":1711668439.0630105,"reason":"broker was unresponsive"},"68":{"timestamp":1711668506.9730346,"reason":"broker was unresponsive"},"69-76":{"timestamp":1711646931.6480982,"reason":""},"77":{"timestamp":1711668526.9621451,"reason":"broker was unresponsive"},"78":{"timestamp":1711668462.9610252,"reason":"broker was unresponsive"},"79":{"timestamp":1711668526.96223,"reason":"broker was unresponsive"},"80":{"timestamp":1711668457.0617182,"reason":"broker was unresponsive"},"81":{"timestamp":1711668526.9622741,"reason":"broker was unresponsive"},"82":{"timestamp":1711668462.9611192,"reason":"broker was unresponsive"},"83":{"timestamp":1711668527.0620673,"reason":"broker was unresponsive"},"84":{"timestamp":1711668463.0615458,"reason":"broker was unresponsive"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"348":{"timestamp":1711576490.2548447,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"431":{"timestamp":1711096934.358676,"reason":"nodediag failed pci"},"445-468,541-564":{"timestamp":1711560189.6111379,"reason":"CDU work TB"},"637":{"timestamp":1711642252.1616328,"reason":"broker was unresponsive"},"638":{"timestamp":1711642252.1617086,"reason":"broker was unresponsive"},"639":{"timestamp":1711642252.161751,"reason":"broker was unresponsive"},"640":{"timestamp":1711642252.1617904,"reason":"broker was unresponsive"},"641":{"timestamp":1711642252.1618268,"reason":"broker was unresponsive"},"642":{"timestamp":1711642252.1618621,"reason":"broker was unresponsive"},"643":{"timestamp":1711642252.1618974,"reason":"broker was unresponsive"},"644":{"timestamp":1711642252.1619322,"reason":"broker was unresponsive"},"646":{"timestamp":1711586857.201689,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"647":{"timestamp":1711587136.0002518,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"648":{"timestamp":1711587197.7047591,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"649":{"timestamp":1711587211.9606316,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"650":{"timestamp":1711587232.1780922,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"651":{"timestamp":1711587244.2040467,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"652":{"timestamp":1711587255.7559283,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"653":{"timestamp":1711642252.1619685,"reason":"broker was unresponsive"},"654":{"timestamp":1711642252.16201,"reason":"broker was unresponsive"},"655":{"timestamp":1711642252.162055,"reason":"broker was unresponsive"},"656":{"timestamp":1711642252.1620932,"reason":"broker was unresponsive"},"657":{"timestamp":1711642252.1621311,"reason":"broker was unresponsive"},"658":{"timestamp":1711642252.1621685,"reason":"broker was unresponsive"},"659":{"timestamp":1711642252.1622062,"reason":"broker was unresponsive"},"660":{"timestamp":1711642252.1622446,"reason":"broker was unresponsive"},"661":{"timestamp":1711642252.1622829,"reason":"broker was unresponsive"},"662":{"timestamp":1711642252.1623201,"reason":"broker was unresponsive"},"663":{"timestamp":1711642252.1623569,"reason":"broker was unresponsive"},"664":{"timestamp":1711642252.162394,"reason":"broker was unresponsive"},"665":{"timestamp":1711642252.1624658,"reason":"broker was unresponsive"},"666":{"timestamp":1711642252.1625173,"reason":"broker was unresponsive"},"667":{"timestamp":1711642258.156193,"reason":"broker was unresponsive"},"668":{"timestamp":1711642252.1625559,"reason":"broker was unresponsive"},"669":{"timestamp":1711642252.1625948,"reason":"broker was unresponsive"},"670":{"timestamp":1711642252.1626337,"reason":"broker was unresponsive"},"671":{"timestamp":1711642252.162673,"reason":"broker was unresponsive"},"672":{"timestamp":1711642252.1627119,"reason":"broker was unresponsive"},"673":{"timestamp":1711642252.1627512,"reason":"broker was unresponsive"},"674":{"timestamp":1711642252.1627915,"reason":"broker was unresponsive"},"675":{"timestamp":1711642252.1628311,"reason":"broker was unresponsive"},"676":{"timestamp":1711642252.1628704,"reason":"broker was unresponsive"},"677":{"timestamp":1711642258.1562793,"reason":"broker was unresponsive"},"678":{"timestamp":1711642258.156333,"reason":"broker was unresponsive"},"679":{"timestamp":1711642252.1629126,"reason":"broker was unresponsive"},"680":{"timestamp":1711642252.1629534,"reason":"broker was unresponsive"},"681":{"timestamp":1711642258.1563835,"reason":"broker was unresponsive"},"682":{"timestamp":1711642252.1629946,"reason":"broker was unresponsive"},"683":{"timestamp":1711642252.1630385,"reason":"broker was unresponsive"},"684":{"timestamp":1711642252.5597043,"reason":"broker was unresponsive"},"685":{"timestamp":1711604960.2562068,"reason":"broker was unresponsive"},"687":{"timestamp":1711604954.2552111,"reason":"broker was unresponsive"},"688":{"timestamp":1711606062.2558303,"reason":"broker was unresponsive"},"689":{"timestamp":1711604580.2552168,"reason":"broker was unresponsive"},"691":{"timestamp":1711604958.2564192,"reason":"broker was unresponsive"},"692":{"timestamp":1711606406.1556187,"reason":"broker was unresponsive"},"693":{"timestamp":1711636838.2555192,"reason":"broker was unresponsive"},"695":{"timestamp":1711604326.2549248,"reason":"broker was unresponsive"},"696":{"timestamp":1711606406.256216,"reason":"broker was unresponsive"},"697":{"timestamp":1711604392.2562771,"reason":"broker was unresponsive"},"698":{"timestamp":1711605716.2557297,"reason":"broker was unresponsive"},"699":{"timestamp":1711604948.2558951,"reason":"broker was unresponsive"},"700":{"timestamp":1711606868.2557678,"reason":"broker was unresponsive"},"701":{"timestamp":1711604340.2561386,"reason":"broker was unresponsive"},"702":{"timestamp":1711606408.2602317,"reason":"broker was unresponsive"},"703":{"timestamp":1711604918.2557685,"reason":"broker was unresponsive"},"704":{"timestamp":1711604590.2548223,"reason":"broker was unresponsive"},"705":{"timestamp":1711604968.2570915,"reason":"broker was unresponsive"},"707":{"timestamp":1711604908.2558758,"reason":"broker was unresponsive"},"686,690,694,706,708":{"timestamp":1711615908.1602454,"reason":"epilog failed for jobid fm9HsYwFjjd"},"709":{"timestamp":1711408620.9668899,"reason":"Cabinet power work needed -jrg"},"710":{"timestamp":1711409910.967391,"reason":"Cabinet power work needed -jrg"},"711":{"timestamp":1711409098.9674611,"reason":"Cabinet power work needed -jrg"},"712":{"timestamp":1711411606.9678445,"reason":"Cabinet power work needed -jrg"},"713":{"timestamp":1711408618.8669355,"reason":"Cabinet power work needed -jrg"},"715":{"timestamp":1711409132.9680476,"reason":"Cabinet power work needed -jrg"},"714,716":{"timestamp":1711564709.7804377,"reason":"Cabinet power work needed -jrg"},"718":{"timestamp":1711642210.1588371,"reason":"broker was unresponsive"},"719":{"timestamp":1711642210.1589563,"reason":"broker was unresponsive"},"720":{"timestamp":1711642210.1590128,"reason":"broker was unresponsive"},"721":{"timestamp":1711642210.1590643,"reason":"broker was unresponsive"},"722":{"timestamp":1711642210.1590991,"reason":"broker was unresponsive"},"723":{"timestamp":1711642210.1591821,"reason":"broker was unresponsive"},"724":{"timestamp":1711642210.1592195,"reason":"broker was unresponsive"},"725":{"timestamp":1711593222.2542431,"reason":"broker was unresponsive"},"726":{"timestamp":1711593312.2549782,"reason":"broker was unresponsive"},"727":{"timestamp":1711593332.2552569,"reason":"broker was unresponsive"},"728":{"timestamp":1711593356.2547033,"reason":"broker was unresponsive"},"729":{"timestamp":1711593414.254441,"reason":"broker was unresponsive"},"730":{"timestamp":1711593462.2552798,"reason":"broker was unresponsive"},"731":{"timestamp":1711593510.2550235,"reason":"broker was unresponsive"},"732":{"timestamp":1711593530.2558599,"reason":"broker was unresponsive"},"733":{"timestamp":1711642210.1592536,"reason":"broker was unresponsive"},"734":{"timestamp":1711486000.9676955,"reason":"broker was unresponsive"},"735":{"timestamp":1711642210.1592872,"reason":"broker was unresponsive"},"736":{"timestamp":1711642210.1593204,"reason":"broker was unresponsive"},"737":{"timestamp":1711642210.1593742,"reason":"broker was unresponsive"},"738":{"timestamp":1711642210.1594355,"reason":"broker was unresponsive"},"739":{"timestamp":1711642210.1594796,"reason":"broker was unresponsive"},"740":{"timestamp":1711642210.159517,"reason":"broker was unresponsive"},"741":{"timestamp":1711642210.1595554,"reason":"broker was unresponsive"},"742":{"timestamp":1711642210.1595943,"reason":"broker was unresponsive"},"743":{"timestamp":1711642210.1596365,"reason":"broker was unresponsive"},"744":{"timestamp":1711642210.1596746,"reason":"broker was unresponsive"},"745":{"timestamp":1711642210.1597111,"reason":"broker was unresponsive"},"746":{"timestamp":1711506258.2527435,"reason":"broker was unresponsive"},"747":{"timestamp":1711642210.1597559,"reason":"broker was unresponsive"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"},"749":{"timestamp":1711642210.1598115,"reason":"broker was unresponsive"},"750":{"timestamp":1711642210.1598501,"reason":"broker was unresponsive"},"751":{"timestamp":1711642210.159888,"reason":"broker was unresponsive"},"752":{"timestamp":1711642210.1599293,"reason":"broker was unresponsive"},"753":{"timestamp":1711642210.1599746,"reason":"broker was unresponsive"},"754":{"timestamp":1711642210.1600144,"reason":"broker was unresponsive"},"755":{"timestamp":1711642210.1600802,"reason":"broker was unresponsive"},"756":{"timestamp":1711642210.5039413,"reason":"broker was unresponsive"},"789-796":{"timestamp":1711461777.1299369,"reason":""},"809":{"timestamp":1711667251.2030878,"reason":"broker was unresponsive"},"810":{"timestamp":1711460371.1431019,"reason":""},"870":{"timestamp":1711576713.5907419,"reason":"epilog failed for jobid fm4cUUWjqfd"},"874":{"timestamp":1711668507.0706048,"reason":"broker was unresponsive"},"875":{"timestamp":1711668503.0621645,"reason":"broker was unresponsive"},"883-884":{"timestamp":1711734121.1560712,"reason":""},"965":{"timestamp":1711663706.9630723,"reason":"broker was unresponsive"},"966":{"timestamp":1711663706.9631405,"reason":"broker was unresponsive"},"967":{"timestamp":1711663706.963171,"reason":"broker was unresponsive"},"968":{"timestamp":1711663706.9631999,"reason":"broker was unresponsive"},"969":{"timestamp":1711663706.9632339,"reason":"broker was unresponsive"},"970":{"timestamp":1711663706.9632568,"reason":"broker was unresponsive"},"971":{"timestamp":1711663706.9632795,"reason":"broker was unresponsive"},"972":{"timestamp":1711663706.9633036,"reason":"broker was unresponsive"},"973":{"timestamp":1711663706.9633393,"reason":"broker was unresponsive"},"974":{"timestamp":1711663706.9633627,"reason":"broker was unresponsive"},"975":{"timestamp":1711663706.9633861,"reason":"broker was unresponsive"},"976":{"timestamp":1711663706.9634261,"reason":"broker was unresponsive"},"977":{"timestamp":1711663706.9634516,"reason":"broker was unresponsive"},"978":{"timestamp":1711663706.9634771,"reason":"broker was unresponsive"},"979":{"timestamp":1711663706.9635053,"reason":"broker was unresponsive"},"980":{"timestamp":1711663707.147469,"reason":"broker was unresponsive"},"10051":{"timestamp":1711683212.9615688,"reason":"broker was unresponsive"},"10052":{"timestamp":1711683213.0621202,"reason":"broker was unresponsive"},"10075":{"timestamp":1711651454.7350454,"reason":"nodediag failed amdapu"},"10076":{"timestamp":1711653832.9647021,"reason":"broker was unresponsive"},"10077":{"timestamp":1711598424.2559545,"reason":"broker was unresponsive"},"10078":{"timestamp":1711598420.2553101,"reason":"broker was unresponsive"},"10084":{"timestamp":1711651436.1179669,"reason":"nodediag failed amdapu"},"10085":{"timestamp":1711599830.1547806,"reason":"broker was unresponsive"},"10086":{"timestamp":1711599830.2550619,"reason":"broker was unresponsive"},"10095":{"timestamp":1711634633.9882171,"reason":""},"10096":{"timestamp":1711591630.2554383,"reason":"Partner node needs hardware action"},"10102":{"timestamp":1711656464.7456715,"reason":"rvs dumped core"},"10151":{"timestamp":1711734764.9003315,"reason":"nodediag failed clocksource pci"},"10152":{"timestamp":1711744485.1922078,"reason":"broker was unresponsive"},"10154":{"timestamp":1711600478.1543531,"reason":"broker was unresponsive"},"10156":{"timestamp":1711600478.2542264,"reason":"broker was unresponsive"},"10157":{"timestamp":1711603028.2555907,"reason":"broker was unresponsive"},"10158":{"timestamp":1711660486.6421392,"reason":"draining to repair blade - wendy"},"10159":{"timestamp":1711744485.19349,"reason":"broker was unresponsive"},"10160":{"timestamp":1711744485.1935451,"reason":"broker was unresponsive"},"10161":{"timestamp":1711742686.2489491,"reason":"nodediag failed cxi"},"10162":{"timestamp":1711742687.1360803,"reason":"nodediag failed cxi"},"10169":{"timestamp":1711742688.0076518,"reason":"nodediag failed cxi"},"10170":{"timestamp":1711742688.869864,"reason":"nodediag failed cxi"},"10183":{"timestamp":1711600556.2563355,"reason":"broker was unresponsive"},"10365":{"timestamp":1711668088.9626274,"reason":"broker was unresponsive"},"10368":{"timestamp":1711668088.9627261,"reason":"broker was unresponsive"},"10372":{"timestamp":1711668089.0613325,"reason":"broker was unresponsive"},"10375":{"timestamp":1711653832.9648952,"reason":"broker was unresponsive"},"10377":{"timestamp":1711653832.9649596,"reason":"broker was unresponsive"},"10384":{"timestamp":1711670140.9621673,"reason":"broker was unresponsive"},"10385":{"timestamp":1711670141.0619121,"reason":"broker was unresponsive"},"10421":{"timestamp":1711592576.2549477,"reason":"broker was unresponsive"},"10425":{"timestamp":1711592578.2555511,"reason":"broker was unresponsive"},"10428":{"timestamp":1711592580.1550264,"reason":"broker was unresponsive"},"10431":{"timestamp":1711592582.1546719,"reason":"broker was unresponsive"},"10432":{"timestamp":1711592582.1547906,"reason":"broker was unresponsive"},"10433":{"timestamp":1711592580.2556479,"reason":"broker was unresponsive"},"10436":{"timestamp":1711592582.2541113,"reason":"broker was unresponsive"},"10462":{"timestamp":1711653832.9651468,"reason":"broker was unresponsive"},"10463":{"timestamp":1711653832.9652078,"reason":"broker was unresponsive"},"10464":{"timestamp":1711653832.9652689,"reason":"broker was unresponsive"},"10467":{"timestamp":1711653832.9653337,"reason":"broker was unresponsive"},"10550":{"timestamp":1711653834.9610977,"reason":"broker was unresponsive"},"10551":{"timestamp":1711653834.9612203,"reason":"broker was unresponsive"},"10552":{"timestamp":1711653834.9612973,"reason":"broker was unresponsive"},"10553":{"timestamp":1711653833.1115868,"reason":"broker was unresponsive"},"10554":{"timestamp":1711653834.9613669,"reason":"broker was unresponsive"},"10555":{"timestamp":1711653834.9614487,"reason":"broker was unresponsive"},"10556":{"timestamp":1711653835.0605469,"reason":"broker was unresponsive"},"10557":{"timestamp":1711660124.9618423,"reason":"broker was unresponsive"},"10558":{"timestamp":1711660124.9619589,"reason":"broker was unresponsive"},"10559":{"timestamp":1711660124.9620354,"reason":"broker was unresponsive"},"10560":{"timestamp":1711660124.962131,"reason":"broker was unresponsive"},"10561":{"timestamp":1711660124.9622042,"reason":"broker was unresponsive"},"10562":{"timestamp":1711660124.9622781,"reason":"broker was unresponsive"},"10563":{"timestamp":1711660124.9623649,"reason":"broker was unresponsive"},"10564":{"timestamp":1711660125.0727112,"reason":"broker was unresponsive"},"10741":{"timestamp":1711686071.0616689,"reason":"dmi nodediag failed dmi"},"10757":{"timestamp":1711733680.9899652,"reason":"nodediag failed clocksource dmi pci"},"10786":{"timestamp":1711733695.434325,"reason":"nodediag failed clocksource dmi pci"},"10796":{"timestamp":1711689956.8180678,"reason":"nodediag failed clocksource dmi pci"},"10804":{"timestamp":1711733682.6906681,"reason":"nodediag failed clocksource dmi pci"},"10832":{"timestamp":1711733694.9659727,"reason":"nodediag failed clocksource dmi pci"},"10843":{"timestamp":1711733726.5102046,"reason":"nodediag failed clocksource dmi pci"},"10865":{"timestamp":1711733747.2219884,"reason":"nodediag failed clocksource dmi pci"},"10871":{"timestamp":1711727083.0628095,"reason":"broker was unresponsive"},"11126":{"timestamp":1711126278.6191533,"reason":"ama fail"},"11187":{"timestamp":1711401067.3461077,"reason":"reason=drain for Rabbit connection"},"11188":{"timestamp":1711325310.3051066,"reason":"Rabbit link Failure"},"11216":{"timestamp":1711406762.9678555,"reason":"broker was unresponsive"},"11240":{"timestamp":1711774951.8277757,"reason":"epilog failed for jobid fmUJGNzW5E7"},"11295":{"timestamp":1711148027.6723082,"reason":"ama fail"},"11296":{"timestamp":1711148027.7718606,"reason":"ama fail"},"11317":{"timestamp":1711326792.8705738,"reason":"epilog failed for jobid fka2yuNfWCo"},"11417":{"timestamp":1711743415.7833021,"reason":"epilog failed for jobid fmKGqZRsvpP"},"11460":{"timestamp":1711123247.7819526,"reason":"ama fail"},"11517":{"timestamp":1711659950.9614942,"reason":"broker was unresponsive"},"11536":{"timestamp":1711659951.0613308,"reason":"broker was unresponsive"},"10137,10140,11566,11587":{"timestamp":1711663526.290566,"reason":"core dumps -KK"},"11593":{"timestamp":1711762170.0976398,"reason":""},"11670":{"timestamp":1711726427.2945235,"reason":"epilog failed for jobid fmLXtp1qajy"},"11765-11780":{"timestamp":1711661236.4636974,"reason":"Per Brick - Rabbits Off in x19xx"},"11784":{"timestamp":1711725755.0632687,"reason":"broker was unresponsive"},"11794":{"timestamp":1711725767.0656931,"reason":"broker was unresponsive"},"11803":{"timestamp":1711668961.0618374,"reason":"broker was unresponsive"},"11860":{"timestamp":1711727735.0622723,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1711852278.0447996,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1711852315.791842,"name":"online","context":{"idset":"0"}}
+{"timestamp":1711852317.5740266,"name":"online","context":{"idset":"10044,10049,10093,10097,10102,10750,10759,10771,10780,10792,10813,10823,10831,10932,11448,11726"}}
+{"timestamp":1711852317.7145772,"name":"online","context":{"idset":"10046,10050,10058,10065-10066,10072-10073,10083,10087,10089-10090,10094-10095,10140,10170,10741,10745,10748-10749,10752,10760-10761,10766,10772-10774,10776,10778,10783-10785,10790,10793,10801,10805-10806,10812,10814-10816,10819-10822,10825,10828-10829,10835,10841,10846-10847,10851,10853,10859,10863,10923,10927,10929,11179,11450,11566,11653-11654,11659,11662,11665-11666,11717,11721,11723-11724,11727,11729,11892"}}
+{"timestamp":1711852317.8259697,"name":"online","context":{"idset":"10037-10038,10043,10045,10053-10057,10059-10061,10063,10067-10071,10074,10079-10082,10088,10091,10099-10100,10137,10162,10169,10360,10379-10380,10743-10744,10746,10751,10753-10756,10758,10762-10765,10768-10770,10775,10777,10781-10782,10787,10789,10791,10794,10796-10798,10800,10802-10804,10807,10809-10811,10817-10818,10824,10826-10827,10830,10832-10834,10836-10840,10842,10844,10848,10850,10852,10854-10858,10860-10862,10864,10866-10868,10917-10918,10921-10922,10924,10926,10928,10930-10931,11187-11188,11213,11449,11453,11605,11657-11658,11660-11661,11663-11664,11667,11704,11719-11720,11722,11725,11728,11730-11732"}}
+{"timestamp":1711852317.9251609,"name":"online","context":{"idset":"10042,10048,10062,10064,10092,10098,10150,10359,10374,10742,10747,10757,10767,10779,10786,10788,10795,10799,10808,10843,10845,10849,10865,11655-11656,11668,11718"}}
+{"timestamp":1711852318.0676763,"name":"online","context":{"idset":"10041,10163,10397-10398,10458,11156"}}
+{"timestamp":1711852318.2000935,"name":"online","context":{"idset":"841,10047,10149,10164,11181,11254,11343,11353,11547,11747,11804,11820,11865"}}
+{"timestamp":1711852318.3035533,"name":"online","context":{"idset":"10039-10040,10204,10406,10416,10419,11160,11191,11252,11291,11386,11423,11468,11476,11530,11571,11683,11716,11785,11797,11830,11872"}}
+{"timestamp":1711852318.4085379,"name":"online","context":{"idset":"10129,10168,10186,10192,10407,10410,10417,11178,11251,11314,11380,11429,11443,11447,11452,11498,11538,11542,11554,11601,11620,11673,11684-11685,11699,11787,11858,11880"}}
+{"timestamp":1711852318.54476,"name":"online","context":{"idset":"10108,10138,10145,10165,10197,10213,11135,11148,11163,11204,11214,11217,11232,11239,11271,11290,11304,11326,11383,11407,11458,11482-11483,11509,11523,11525,11529,11564,11596,11603,11609,11628,11639,11681-11682,11690,11701,11714-11715,11790,11805,11815,11834,11856-11857,11869,11879,11885"}}
+{"timestamp":1711852318.6481974,"name":"online","context":{"idset":"10116,10128,10130,10166,10174,10195,10203,10211,10221,10438,11175-11177,11180,11183,11185-11186,11206,11228,11234,11261,11280,11297,11300,11305,11336,11344,11384,11393,11395,11415,11420,11425,11446,11454,11456-11457,11459,11510,11514,11521,11534,11537,11572-11573,11579,11606,11608,11627,11638,11641,11646,11650,11689,11708,11742,11746,11783,11786,11795,11808,11825,11832,11842,11844,11878,11881"}}
+{"timestamp":1711852318.7554405,"name":"online","context":{"idset":"15,37,479,10123,10135,10141-10142,10144,10167,10177-10178,10184,10193,10198,10209,10219,10439,10445,11128,11130,11136,11142,11144,11152,11182,11190,11195,11205,11208-11209,11219,11235,11259,11265,11267,11282,11319,11328,11361,11375,11389,11410,11419,11421,11437,11462,11472,11475,11500,11503,11519,11539,11545,11551,11577-11578,11583,11588,11607,11635,11702,11710,11740,11743,11782,11788,11799,11824,11835,11838,11862,11864,11868"}}
+{"timestamp":1711852318.8661685,"name":"online","context":{"idset":"49,10134,10136,10182,10190,10199-10201,10214,10447,10451-10452,11134,11137,11140-11141,11143,11146-11147,11157,11184,11189,11192,11201,11207,11210-11212,11222,11237,11241,11244,11249-11250,11255,11262,11264,11268,11281,11283,11285,11287,11289,11293-11294,11299,11303,11311-11312,11325,11327,11330-11332,11334,11337,11349-11350,11359,11363,11370,11382,11390,11396,11403-11404,11427,11431,11433,11440,11442,11445,11451,11455,11464,11469-11470,11481,11484,11489,11495,11497,11502,11505,11511,11520,11522,11527,11544,11557-11558,11568-11569,11575,11584-11585,11589,11592,11594,11602,11622,11637,11640,11645,11649,11671,11676-11678,11694,11696,11698,11706,11709,11734,11741,11781,11792,11800,11806,11809,11811,11814,11818,11823,11826-11829,11845,11850,11852,11855,11863,11870,11883,11888-11890"}}
+{"timestamp":1711852318.9791367,"name":"online","context":{"idset":"105,10104,10109,10113-10114,10147,10175-10176,10181,10188,10196,10212,10218,10224,10441,11125,11132,11145,11153,11155,11161,11164,11168-11170,11218,11220,11223,11226,11230,11238,11242,11247-11248,11270,11272,11274,11277-11278,11284,11286,11292,11298,11301-11302,11308,11310,11320-11322,11324,11329,11341,11345,11351,11357,11369,11377,11381,11385,11391-11392,11394,11399,11402,11414,11418,11435,11441,11461,11463,11477,11480,11488,11493-11494,11499,11501,11506-11508,11513,11518,11526,11532,11535,11540-11541,11543,11549-11550,11556,11559-11560,11562,11565,11567,11570,11576,11581,11590,11595,11597,11599,11611,11615,11623,11625-11626,11629,11643,11647,11672,11674,11679,11688,11700,11713,11737-11738,11745,11796,11798,11801,11813,11817,11821-11822,11833,11843,11846-11848,11851,11854,11871,11882,11886-11887"}}
+{"timestamp":1711852319.0951209,"name":"online","context":{"idset":"101-104,10105-10106,10118,10121,10124,10132,10139,10143,10146,10173,10179-10180,10189,10207,10215,10440,10448,11129,11131,11133,11138-11139,11151,11158-11159,11162,11165-11166,11171-11172,11193-11194,11197,11199-11200,11203,11224,11227,11231,11246,11257,11260,11263,11266,11269,11273,11276,11288,11306-11307,11309,11313,11315-11316,11323,11333,11335,11338,11340,11342,11346-11348,11352,11354,11362,11364-11366,11368,11371-11374,11376,11378-11379,11397-11398,11401,11406,11408-11409,11411-11413,11430,11432,11434,11436,11438,11444,11467,11473-11474,11478,11485,11491,11496,11512,11515-11516,11528,11548,11552,11555,11561,11563,11574,11582,11591,11598,11604,11610,11612-11613,11616-11619,11621,11624,11631-11632,11634,11642,11651,11675,11680,11686-11687,11691,11693,11695,11697,11703,11707,11712,11739,11810,11812,11819,11831,11837,11841,11853,11861,11866-11867,11874-11875"}}
+{"timestamp":1711852319.2015846,"name":"online","context":{"idset":"8,13,20,27,43,50,10107,10110,10112,10115,10120,10122,10126,10133,10171-10172,10185,10191,10194,10208,10210,10216-10217,10220,10442,10444,11127,11167,11196,11198,11202,11221,11229,11233,11253,11258,11275,11339,11355-11356,11360,11367,11400,11405,11416,11424,11465,11479,11486-11487,11490,11492,11504,11524,11531,11546,11553,11580,11586,11600,11614,11630,11633,11636,11644,11652,11692,11705,11744,11807,11839-11840,11859,11873,11876-11877,11884"}}
+{"timestamp":1711852319.3096576,"name":"online","context":{"idset":"3-4,16,25,10101,10187,10225,10450,11225,11236,11243,11279,11358,11422,11426,11428,11735-11736,11748,11849"}}
+{"timestamp":1711852319.4176309,"name":"online","context":{"idset":"10103,10119,10125,10131,10223,10227,11245,11648"}}
+{"timestamp":1711852319.5235732,"name":"online","context":{"idset":"42,10111,10117,10222,10226,10228,11733"}}
+{"timestamp":1711852319.6315913,"name":"online","context":{"idset":"1-2,5-7,9-11,14,17-19,21-24,26,28-36,38,40-41,44-48,51-60"}}
+{"timestamp":1711852319.7399836,"name":"online","context":{"idset":"12,39,10127"}}
+{"timestamp":1711852457.310375,"name":"online","context":{"idset":"86,119,164,170,187,193,281,285,291,294,296,330,472,633"}}
+{"timestamp":1711852457.418736,"name":"online","context":{"idset":"141,151-153,190,202,209,211,225-226,228,238,247-248,251,260-261,271,283,302,319,344,360,363,371,377,381,406,423-424,436,439,470,474,485,487,507-508,522,526,528,530,567,576,585,588,590,611,613,616,619,625"}}
+{"timestamp":1711852457.5262544,"name":"online","context":{"idset":"131,140,160,171-172,175,201,203,210,240,263,289,298,300-301,331,338,340,343,347,369-370,374,387,396,399,427,433,444,471,496-497,533,565,580,599,604,617,620,623"}}
+{"timestamp":1711852457.6619644,"name":"online","context":{"idset":"88,98,106,108,114-115,125,130,133-135,143,147,167,183,199,204,213,220,224,227,242,244,246,262,266,269,277,284,297,305,314,322,324-325,332,368,373,380,390-391,398,400,404,412-413,420,422,428,435,437,441,443,475,480,484,498,503,509-510,535-536,539,570,582-584,591,598,614,624"}}
+{"timestamp":1711852457.7701278,"name":"online","context":{"idset":"118,136,174,252,274,292,425,430,532,595,605,612,632"}}
+{"timestamp":1711852457.8783937,"name":"online","context":{"idset":"85,100,146,161,249,388,415,600"}}
+{"timestamp":1711852457.9867661,"name":"online","context":{"idset":"375,622"}}
+{"timestamp":1711852458.1328228,"name":"online","context":{"idset":"109"}}
+{"timestamp":1711852458.3064251,"name":"online","context":{"idset":"117,241,299,317,492,504,596"}}
+{"timestamp":1711852458.4598153,"name":"online","context":{"idset":"90,97,107,111,120,155,169,173,180,186,197,206-207,232-233,235,243,250,255-256,259,268,270,279,307,316,320,327,334,337,339,341,349,352,366,392,395,402,410,421,426,432,481,486,521,537,569,578,593,603,606,610,618"}}
+{"timestamp":1711852458.5684927,"name":"online","context":{"idset":"95,126-127,129,132,137,139,145,148,154,163,168,179,184,191,194,200,208,219,222,236,253,258,265,267,287-288,295,308,329,333,342,345,357-359,361-362,376,382,393,403,405,416-417,469,506,513-514,525,538,572,575,587,597,607-608,621,626,628-629,631,634"}}
+{"timestamp":1711852458.6778104,"name":"online","context":{"idset":"87,110,116,122,142,162,189,205,215-216,230,237,264,286,310,355,372,378-379,386,414,442,477,511-512,516-517,579,627"}}
+{"timestamp":1711852458.8152051,"name":"online","context":{"idset":"156,181,196,276,383,394,397,429,491,493,568"}}
+{"timestamp":1711852460.256021,"name":"online","context":{"idset":"123,254"}}
+{"timestamp":1711852460.4603877,"name":"online","context":{"idset":"113,144,176,306,351,385,434,494,500-501,519,540,573,609"}}
+{"timestamp":1711852460.5100939,"name":"online","context":{"idset":"89,124,157,195,273,309,318,346,350,401,476,482-483,490,527,531,594,601,615"}}
+{"timestamp":1711852460.5599101,"name":"online","context":{"idset":"150,165,229,328,335,364-365,384,389,438,489,499,524,529"}}
+{"timestamp":1711852460.5670359,"name":"online","context":{"idset":"96,138"}}
+{"timestamp":1711852460.5732365,"name":"online","context":{"idset":"473"}}
+{"timestamp":1711852460.5912788,"name":"online","context":{"idset":"94,185"}}
+{"timestamp":1711852460.6599705,"name":"online","context":{"idset":"99,112,159,198,212,245,278,304,311,315,321,354,495,518,592,630"}}
+{"timestamp":1711852460.7133727,"name":"online","context":{"idset":"93,178,214,239,272,523,581,586"}}
+{"timestamp":1711852460.7643507,"name":"online","context":{"idset":"158,182,231,312,419,566"}}
+{"timestamp":1711852460.9225669,"name":"online","context":{"idset":"534"}}
+{"timestamp":1711852463.0199015,"name":"online","context":{"idset":"11240"}}
+{"timestamp":1711852464.5032213,"name":"online","context":{"idset":"290,602"}}
+{"timestamp":1711852464.6408124,"name":"online","context":{"idset":"91-92,128,149,166,218,234,257,280,282,293,313,356,367,408,411,418,440,478,502,505,515,571,574,577,635"}}
+{"timestamp":1711852464.7503412,"name":"online","context":{"idset":"177,192,275,303,323,336,353,407,409,488,520,589"}}
+{"timestamp":1711884332.2251968,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1711892806.2263796,"name":"offline","context":{"idset":"10448"}}
+{"timestamp":1711903037.5381749,"name":"online","context":{"idset":"11777"}}
+{"timestamp":1711903037.7492483,"name":"online","context":{"idset":"11778"}}
+{"timestamp":1711946370.2288194,"name":"drain","context":{"idset":"11569","reason":"broker was unresponsive"}}
+{"timestamp":1711946434.2588818,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1711979866.1342788,"name":"offline","context":{"idset":"10095"}}
+{"timestamp":1711979866.2126184,"name":"offline","context":{"idset":"11777"}}
+{"timestamp":1711979866.2403402,"name":"offline","context":{"idset":"11778"}}
+{"timestamp":1711979866.2692094,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1711979866.2720394,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1711979866.2800543,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1711979866.2878587,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1711979866.312582,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1711979866.3153841,"name":"offline","context":{"idset":"10786"}}
+{"timestamp":1711979866.3843606,"name":"offline","context":{"idset":"10843"}}
+{"timestamp":1711979866.3863096,"name":"offline","context":{"idset":"10804"}}
+{"timestamp":1711979866.3878181,"name":"offline","context":{"idset":"10832"}}
+{"timestamp":1711979866.3893042,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1711979866.3907876,"name":"offline","context":{"idset":"10757"}}
+{"timestamp":1711979866.4464481,"name":"offline","context":{"idset":"10865"}}
+{"timestamp":1711979866.5208638,"name":"offline","context":{"idset":"10796"}}
+{"timestamp":1711980914.97558,"name":"online","context":{"idset":"9974,9977,9980"}}
+{"timestamp":1711980915.1127546,"name":"online","context":{"idset":"9973,9979,9984,9988"}}
+{"timestamp":1711980915.2739971,"name":"online","context":{"idset":"9978,9982-9983,9985-9986"}}
+{"timestamp":1711980915.5029502,"name":"online","context":{"idset":"9975,9981,9987"}}
+{"timestamp":1711982453.3271847,"name":"online","context":{"idset":"10036"}}
+{"timestamp":1711982548.2285702,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1711982590.2767313,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1711982863.7287045,"name":"online","context":{"idset":"10169-10170"}}
+{"timestamp":1711982863.8792543,"name":"online","context":{"idset":"10161"}}
+{"timestamp":1711982876.8056242,"name":"undrain","context":{"idset":"10161,10169-10170"}}
+{"timestamp":1711984506.2282465,"name":"drain","context":{"idset":"10036","reason":"broker was unresponsive"}}
+{"timestamp":1711984570.2262237,"name":"offline","context":{"idset":"10036"}}
+{"timestamp":1711985590.1298366,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.1299634,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.129997,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.1300287,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.130055,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.1301,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.1301718,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.1302528,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.1303506,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.1304929,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.1305854,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1711985590.2635758,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1711985656.1373813,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1711985656.1433253,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1711985656.1488583,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1711985656.1543856,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1711985656.1598189,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1711985656.1651049,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1711985656.1736152,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1711985656.1797945,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1711985656.1859214,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1711985656.2013555,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1711985656.2029259,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1711985656.2991719,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1711986027.384675,"name":"online","context":{"idset":"5"}}
+{"timestamp":1711986028.076611,"name":"online","context":{"idset":"10-11"}}
+{"timestamp":1711986028.1866274,"name":"online","context":{"idset":"1-2,6-7,12"}}
+{"timestamp":1711986028.2631547,"name":"online","context":{"idset":"4"}}
+{"timestamp":1711986028.3478348,"name":"online","context":{"idset":"8"}}
+{"timestamp":1711986032.7706509,"name":"online","context":{"idset":"3"}}
+{"timestamp":1711986033.2548268,"name":"online","context":{"idset":"9"}}
+{"timestamp":1711986172.126236,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1711986172.226109,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1711986174.2265673,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1711986176.1277797,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1711986176.127861,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1711986176.1279109,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1711986176.1279769,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1711986176.1280706,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1711986176.1281965,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1711986176.128314,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1711986176.1284461,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1711986176.2268984,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1711986238.1443899,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1711986238.1531076,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1711986238.1616335,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1711986238.1699917,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1711986238.1784751,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1711986238.1869667,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1711986238.1953318,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1711986238.2067847,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1711986238.2158806,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1711986238.234988,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1711986238.2365215,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1711986238.2429607,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1711986604.6325216,"name":"online","context":{"idset":"14-15"}}
+{"timestamp":1711986604.7899656,"name":"online","context":{"idset":"18,22"}}
+{"timestamp":1711986605.1161032,"name":"online","context":{"idset":"16-17,19-21,24"}}
+{"timestamp":1711986605.2744031,"name":"online","context":{"idset":"13,23"}}
+{"timestamp":1711986712.1284616,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.1285477,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.1286163,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.1287522,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.1288626,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.1289904,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.1291039,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.1292276,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.129328,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.1294577,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.1295605,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1711986712.2507603,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1711986776.1398599,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1711986776.1462762,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1711986776.1527071,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1711986776.1590581,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1711986776.1650379,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1711986776.1711962,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1711986776.1767731,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1711986776.1857634,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1711986776.1923683,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1711986776.2085216,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1711986776.2100985,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1711986776.3100297,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1711987152.3790879,"name":"online","context":{"idset":"33"}}
+{"timestamp":1711987152.7745769,"name":"online","context":{"idset":"26-28,36"}}
+{"timestamp":1711987153.2172978,"name":"online","context":{"idset":"29-30,32,34"}}
+{"timestamp":1711987153.3282056,"name":"online","context":{"idset":"25,31,35"}}
+{"timestamp":1711987342.1267064,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.1267867,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.1268172,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.1268439,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.1269064,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.1269703,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.127022,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.1270769,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.1271369,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.1271925,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.1272507,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1711987342.2340004,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1711987406.1416831,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1711987406.149559,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1711987406.158206,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1711987406.1656199,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1711987406.1730599,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1711987406.1799791,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1711987406.1877124,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1711987406.1983473,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1711987406.2065549,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1711987406.2243862,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1711987406.2259285,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1711987406.228965,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1711987778.2604277,"name":"online","context":{"idset":"46"}}
+{"timestamp":1711987782.8075516,"name":"online","context":{"idset":"38-39"}}
+{"timestamp":1711987783.087503,"name":"online","context":{"idset":"37,41,45"}}
+{"timestamp":1711987783.1934886,"name":"online","context":{"idset":"40,42-43,47-48"}}
+{"timestamp":1711987783.3579834,"name":"online","context":{"idset":"44"}}
+{"timestamp":1711988713.3839881,"name":"undrain","context":{"idset":"10157-10158"}}
+{"timestamp":1711988819.8451352,"name":"online","context":{"idset":"10157"}}
+{"timestamp":1711988820.1296086,"name":"online","context":{"idset":"10158"}}
+{"timestamp":1711989703.1926322,"name":"online","context":{"idset":"11711"}}
+{"timestamp":1711991215.013855,"name":"online","context":{"idset":"10981"}}
+{"timestamp":1711991215.3639174,"name":"online","context":{"idset":"10982"}}
+{"timestamp":1711992762.598196,"name":"undrain","context":{"idset":"11765-11780"}}
+{"timestamp":1711993308.7730825,"name":"drain","context":{"idset":"11765-11780","reason":"Diags Running","overwrite":0}}
+{"timestamp":1711993972.7841954,"name":"drain","context":{"idset":"11189","reason":"broker was unresponsive"}}
+{"timestamp":1711994034.6875093,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1711994034.7821665,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1711994548.6854541,"name":"drain","context":{"idset":"10406","reason":"broker was unresponsive"}}
+{"timestamp":1711994548.6855533,"name":"drain","context":{"idset":"10416","reason":"broker was unresponsive"}}
+{"timestamp":1711994548.6856179,"name":"drain","context":{"idset":"10417","reason":"broker was unresponsive"}}
+{"timestamp":1711994548.6856782,"name":"drain","context":{"idset":"10438","reason":"broker was unresponsive"}}
+{"timestamp":1711994548.685781,"name":"drain","context":{"idset":"10444","reason":"broker was unresponsive"}}
+{"timestamp":1711994548.6858656,"name":"drain","context":{"idset":"10445","reason":"broker was unresponsive"}}
+{"timestamp":1711994548.6859574,"name":"drain","context":{"idset":"10447","reason":"broker was unresponsive"}}
+{"timestamp":1711994548.7840729,"name":"drain","context":{"idset":"10450","reason":"broker was unresponsive"}}
+{"timestamp":1711994566.6834636,"name":"drain","context":{"idset":"10419","reason":"broker was unresponsive"}}
+{"timestamp":1711994566.6835759,"name":"drain","context":{"idset":"10451","reason":"broker was unresponsive"}}
+{"timestamp":1711994566.7830374,"name":"drain","context":{"idset":"10452","reason":"broker was unresponsive"}}
+{"timestamp":1711994612.6965337,"name":"offline","context":{"idset":"10406"}}
+{"timestamp":1711994612.7065203,"name":"offline","context":{"idset":"10416"}}
+{"timestamp":1711994612.7167473,"name":"offline","context":{"idset":"10417"}}
+{"timestamp":1711994612.7259502,"name":"offline","context":{"idset":"10438"}}
+{"timestamp":1711994612.735487,"name":"offline","context":{"idset":"10444"}}
+{"timestamp":1711994612.7448318,"name":"offline","context":{"idset":"10445"}}
+{"timestamp":1711994612.7566886,"name":"offline","context":{"idset":"10447"}}
+{"timestamp":1711994613.0181673,"name":"offline","context":{"idset":"10450"}}
+{"timestamp":1711994632.6901104,"name":"offline","context":{"idset":"10419"}}
+{"timestamp":1711994632.7008755,"name":"offline","context":{"idset":"10451"}}
+{"timestamp":1711994632.9913383,"name":"offline","context":{"idset":"10452"}}
+{"timestamp":1711994926.6849668,"name":"drain","context":{"idset":"10407","reason":"broker was unresponsive"}}
+{"timestamp":1711994926.6850717,"name":"drain","context":{"idset":"10439","reason":"broker was unresponsive"}}
+{"timestamp":1711994926.7842457,"name":"drain","context":{"idset":"10440","reason":"broker was unresponsive"}}
+{"timestamp":1711994990.6875479,"name":"offline","context":{"idset":"10407"}}
+{"timestamp":1711994990.6952863,"name":"offline","context":{"idset":"10439"}}
+{"timestamp":1711994990.7834189,"name":"offline","context":{"idset":"10440"}}
+{"timestamp":1711995233.6952045,"name":"drain","context":{"idset":"10407","reason":"epilog failed for jobid fmiWJV6Dp9d","overwrite":0}}
+{"timestamp":1711997290.6712449,"name":"online","context":{"idset":"851"}}
+{"timestamp":1711997290.8234544,"name":"online","context":{"idset":"852,881"}}
+{"timestamp":1711997290.9722114,"name":"online","context":{"idset":"840"}}
+{"timestamp":1711997291.1220839,"name":"online","context":{"idset":"882"}}
+{"timestamp":1711997291.2759402,"name":"online","context":{"idset":"11188-11189"}}
+{"timestamp":1711997318.3053694,"name":"offline","context":{"idset":"851"}}
+{"timestamp":1711997318.3741148,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1711997318.4466469,"name":"offline","context":{"idset":"882"}}
+{"timestamp":1711997318.4733241,"name":"offline","context":{"idset":"840"}}
+{"timestamp":1711997318.511502,"name":"offline","context":{"idset":"881"}}
+{"timestamp":1711997318.5229895,"name":"offline","context":{"idset":"852"}}
+{"timestamp":1711997318.6226616,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1711997464.393141,"name":"online","context":{"idset":"851,11189"}}
+{"timestamp":1711997464.861089,"name":"online","context":{"idset":"852,882,11188"}}
+{"timestamp":1711997465.0460682,"name":"online","context":{"idset":"881"}}
+{"timestamp":1711997465.3073556,"name":"online","context":{"idset":"840"}}
+{"timestamp":1711997467.3466318,"name":"online","context":{"idset":"839"}}
+{"timestamp":1711997908.7831469,"name":"offline","context":{"idset":"11711"}}
+{"timestamp":1711998176.6840956,"name":"drain","context":{"idset":"10410","reason":"broker was unresponsive"}}
+{"timestamp":1711998176.7837813,"name":"drain","context":{"idset":"10458","reason":"broker was unresponsive"}}
+{"timestamp":1711998240.6873949,"name":"offline","context":{"idset":"10410"}}
+{"timestamp":1711998240.7823551,"name":"offline","context":{"idset":"10458"}}
+{"timestamp":1711999054.6863794,"name":"drain","context":{"idset":"11493","reason":"broker was unresponsive"}}
+{"timestamp":1711999054.6865115,"name":"drain","context":{"idset":"11494","reason":"broker was unresponsive"}}
+{"timestamp":1711999054.6865766,"name":"drain","context":{"idset":"11495","reason":"broker was unresponsive"}}
+{"timestamp":1711999054.6866362,"name":"drain","context":{"idset":"11496","reason":"broker was unresponsive"}}
+{"timestamp":1711999054.686723,"name":"drain","context":{"idset":"11497","reason":"broker was unresponsive"}}
+{"timestamp":1711999054.6868224,"name":"drain","context":{"idset":"11498","reason":"broker was unresponsive"}}
+{"timestamp":1711999054.6869185,"name":"drain","context":{"idset":"11499","reason":"broker was unresponsive"}}
+{"timestamp":1711999054.6870217,"name":"drain","context":{"idset":"11500","reason":"broker was unresponsive"}}
+{"timestamp":1711999054.687109,"name":"drain","context":{"idset":"11501","reason":"broker was unresponsive"}}
+{"timestamp":1711999054.7876759,"name":"drain","context":{"idset":"11502","reason":"broker was unresponsive"}}
+{"timestamp":1711999060.6864119,"name":"drain","context":{"idset":"11503","reason":"broker was unresponsive"}}
+{"timestamp":1711999060.6865165,"name":"drain","context":{"idset":"11504","reason":"broker was unresponsive"}}
+{"timestamp":1711999060.6865866,"name":"drain","context":{"idset":"11505","reason":"broker was unresponsive"}}
+{"timestamp":1711999060.6866503,"name":"drain","context":{"idset":"11506","reason":"broker was unresponsive"}}
+{"timestamp":1711999060.6867461,"name":"drain","context":{"idset":"11507","reason":"broker was unresponsive"}}
+{"timestamp":1711999060.7851932,"name":"drain","context":{"idset":"11508","reason":"broker was unresponsive"}}
+{"timestamp":1711999116.7831385,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1711999118.6926816,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1711999118.7035172,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1711999118.7138577,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1711999118.7284064,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1711999118.7836373,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1711999120.6898899,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1711999120.6980245,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1711999120.7062621,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1711999120.7829237,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1711999121.7090328,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1711999121.7204485,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1711999121.7311747,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1711999121.74576,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1711999121.7979882,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1711999124.7822692,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1711999678.6867654,"name":"drain","context":{"idset":"11285","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6868987,"name":"drain","context":{"idset":"11286","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6869974,"name":"drain","context":{"idset":"11287","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6870911,"name":"drain","context":{"idset":"11288","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6871724,"name":"drain","context":{"idset":"11289","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.687247,"name":"drain","context":{"idset":"11290","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6873426,"name":"drain","context":{"idset":"11291","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6874483,"name":"drain","context":{"idset":"11292","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6875474,"name":"drain","context":{"idset":"11293","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6876364,"name":"drain","context":{"idset":"11294","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6877365,"name":"drain","context":{"idset":"11297","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6878426,"name":"drain","context":{"idset":"11298","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.6879513,"name":"drain","context":{"idset":"11299","reason":"broker was unresponsive"}}
+{"timestamp":1711999678.8150098,"name":"drain","context":{"idset":"11300","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6861956,"name":"drain","context":{"idset":"11269","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6863399,"name":"drain","context":{"idset":"11270","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6864691,"name":"drain","context":{"idset":"11271","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6865888,"name":"drain","context":{"idset":"11272","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.686703,"name":"drain","context":{"idset":"11273","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6868193,"name":"drain","context":{"idset":"11274","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6869347,"name":"drain","context":{"idset":"11275","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6870532,"name":"drain","context":{"idset":"11276","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6871679,"name":"drain","context":{"idset":"11277","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6872816,"name":"drain","context":{"idset":"11278","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6874027,"name":"drain","context":{"idset":"11279","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6875162,"name":"drain","context":{"idset":"11280","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6876323,"name":"drain","context":{"idset":"11281","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6877465,"name":"drain","context":{"idset":"11282","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.6878541,"name":"drain","context":{"idset":"11283","reason":"broker was unresponsive"}}
+{"timestamp":1711999684.834142,"name":"drain","context":{"idset":"11284","reason":"broker was unresponsive"}}
+{"timestamp":1711999744.7007332,"name":"offline","context":{"idset":"11285"}}
+{"timestamp":1711999744.7091072,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1711999744.71716,"name":"offline","context":{"idset":"11287"}}
+{"timestamp":1711999744.7252514,"name":"offline","context":{"idset":"11288"}}
+{"timestamp":1711999744.7333238,"name":"offline","context":{"idset":"11289"}}
+{"timestamp":1711999744.7412407,"name":"offline","context":{"idset":"11290"}}
+{"timestamp":1711999744.7513139,"name":"offline","context":{"idset":"11291"}}
+{"timestamp":1711999744.7693474,"name":"offline","context":{"idset":"11292"}}
+{"timestamp":1711999744.7709386,"name":"offline","context":{"idset":"11293"}}
+{"timestamp":1711999744.778275,"name":"offline","context":{"idset":"11294"}}
+{"timestamp":1711999744.8142211,"name":"offline","context":{"idset":"11297"}}
+{"timestamp":1711999744.8250668,"name":"offline","context":{"idset":"11298"}}
+{"timestamp":1711999744.8266175,"name":"offline","context":{"idset":"11299"}}
+{"timestamp":1711999744.8281925,"name":"offline","context":{"idset":"11300"}}
+{"timestamp":1711999748.7025228,"name":"offline","context":{"idset":"11269"}}
+{"timestamp":1711999748.7113302,"name":"offline","context":{"idset":"11270"}}
+{"timestamp":1711999748.7173834,"name":"offline","context":{"idset":"11271"}}
+{"timestamp":1711999748.7238629,"name":"offline","context":{"idset":"11272"}}
+{"timestamp":1711999748.7334716,"name":"offline","context":{"idset":"11273"}}
+{"timestamp":1711999748.7473845,"name":"offline","context":{"idset":"11274"}}
+{"timestamp":1711999748.7489645,"name":"offline","context":{"idset":"11275"}}
+{"timestamp":1711999748.7718606,"name":"offline","context":{"idset":"11276"}}
+{"timestamp":1711999748.7729156,"name":"offline","context":{"idset":"11277"}}
+{"timestamp":1711999748.7782698,"name":"offline","context":{"idset":"11278"}}
+{"timestamp":1711999748.7863526,"name":"offline","context":{"idset":"11279"}}
+{"timestamp":1711999748.7944126,"name":"offline","context":{"idset":"11280"}}
+{"timestamp":1711999749.0881915,"name":"offline","context":{"idset":"11281"}}
+{"timestamp":1711999749.1021974,"name":"offline","context":{"idset":"11282"}}
+{"timestamp":1711999749.1132314,"name":"offline","context":{"idset":"11283"}}
+{"timestamp":1711999749.1388609,"name":"offline","context":{"idset":"11284"}}
+{"timestamp":1711999768.783828,"name":"drain","context":{"idset":"11260","reason":"broker was unresponsive"}}
+{"timestamp":1711999834.7834251,"name":"offline","context":{"idset":"11260"}}
+{"timestamp":1712001086.7968159,"name":"undrain","context":{"idset":"11493-11508"}}
+{"timestamp":1712001639.4554269,"name":"online","context":{"idset":"11497"}}
+{"timestamp":1712001640.1551712,"name":"online","context":{"idset":"11494"}}
+{"timestamp":1712001640.784385,"name":"online","context":{"idset":"11502"}}
+{"timestamp":1712001641.11519,"name":"online","context":{"idset":"11506"}}
+{"timestamp":1712001641.3627412,"name":"online","context":{"idset":"11505,11508"}}
+{"timestamp":1712001641.8249955,"name":"online","context":{"idset":"11501"}}
+{"timestamp":1712001642.0324128,"name":"online","context":{"idset":"11498,11503"}}
+{"timestamp":1712001642.676259,"name":"online","context":{"idset":"11499"}}
+{"timestamp":1712001643.1486042,"name":"online","context":{"idset":"11495"}}
+{"timestamp":1712001643.4054434,"name":"online","context":{"idset":"11496"}}
+{"timestamp":1712001643.6115007,"name":"online","context":{"idset":"11493"}}
+{"timestamp":1712001645.4639077,"name":"online","context":{"idset":"11500"}}
+{"timestamp":1712001648.252341,"name":"online","context":{"idset":"11504"}}
+{"timestamp":1712001686.680938,"name":"online","context":{"idset":"843"}}
+{"timestamp":1712001687.8928235,"name":"online","context":{"idset":"844"}}
+{"timestamp":1712001884.8805854,"name":"online","context":{"idset":"11260"}}
+{"timestamp":1712001885.0379868,"name":"online","context":{"idset":"11256"}}
+{"timestamp":1712003029.9715586,"name":"online","context":{"idset":"10993"}}
+{"timestamp":1712003048.0476468,"name":"online","context":{"idset":"11289"}}
+{"timestamp":1712003048.2986243,"name":"online","context":{"idset":"11285,11293"}}
+{"timestamp":1712003048.4521415,"name":"online","context":{"idset":"11288,11291"}}
+{"timestamp":1712003048.6613069,"name":"online","context":{"idset":"11286,11292,11295,11297"}}
+{"timestamp":1712003048.8164532,"name":"online","context":{"idset":"11296"}}
+{"timestamp":1712003048.9311905,"name":"online","context":{"idset":"11290,11294,11298-11300"}}
+{"timestamp":1712003049.070219,"name":"online","context":{"idset":"11287"}}
+{"timestamp":1712003988.7822118,"name":"offline","context":{"idset":"11812"}}
+{"timestamp":1712004514.6037207,"name":"drain","context":{"idset":"11812","reason":"epilog failed for jobid fmoe85E2cFq","overwrite":0}}
+{"timestamp":1712004602.484952,"name":"drain","context":{"idset":"11781-11796","reason":"Running Diags - Rabbits Off","overwrite":1}}
+{"timestamp":1712004735.8780181,"name":"undrain","context":{"idset":"10365,10368,10372"}}
+{"timestamp":1712004754.4849381,"name":"online","context":{"idset":"10363"}}
+{"timestamp":1712004754.8663058,"name":"online","context":{"idset":"10368"}}
+{"timestamp":1712004755.0811477,"name":"online","context":{"idset":"10357,10364-10366,10369-10372"}}
+{"timestamp":1712004755.2249863,"name":"online","context":{"idset":"10361-10362"}}
+{"timestamp":1712004755.3666732,"name":"online","context":{"idset":"10358,10367"}}
+{"timestamp":1712004812.8396358,"name":"drain","context":{"idset":"849-850","overwrite":0}}
+{"timestamp":1712004859.9543238,"name":"drain","context":{"idset":"11797-11812","reason":"Running Diags - Rabbits Off","overwrite":1}}
+{"timestamp":1712004866.1869416,"name":"online","context":{"idset":"10026"}}
+{"timestamp":1712004867.3367534,"name":"online","context":{"idset":"10025"}}
+{"timestamp":1712004868.6903009,"name":"online","context":{"idset":"10024"}}
+{"timestamp":1712004870.3881259,"name":"online","context":{"idset":"10023"}}
+{"timestamp":1712004873.069634,"name":"online","context":{"idset":"10022"}}
+{"timestamp":1712004876.0560839,"name":"online","context":{"idset":"10021"}}
+{"timestamp":1712005063.5043457,"name":"online","context":{"idset":"10027"}}
+{"timestamp":1712005065.312259,"name":"online","context":{"idset":"10028"}}
+{"timestamp":1712005066.3941195,"name":"online","context":{"idset":"10030"}}
+{"timestamp":1712005088.7732615,"name":"drain","context":{"idset":"11813-11828","reason":"Running Diags - Rabbits Off","overwrite":0}}
+{"timestamp":1712006085.2090132,"name":"online","context":{"idset":"10029"}}
+{"timestamp":1712006127.4265921,"name":"online","context":{"idset":"10033"}}
+{"timestamp":1712006131.3763545,"name":"online","context":{"idset":"10035"}}
+{"timestamp":1712006133.6289308,"name":"online","context":{"idset":"10034"}}
+{"timestamp":1712006137.5862479,"name":"online","context":{"idset":"10032"}}
+{"timestamp":1712006140.2507365,"name":"online","context":{"idset":"10031"}}
+{"timestamp":1712006423.2592289,"name":"undrain","context":{"idset":"11188-11189"}}
+{"timestamp":1712006434.9212751,"name":"drain","context":{"idset":"11188-11189","overwrite":0}}
+{"timestamp":1712006582.4820375,"name":"undrain","context":{"idset":"11188-11189"}}
+{"timestamp":1712007116.7823906,"name":"offline","context":{"idset":"11800"}}
+{"timestamp":1712007433.6860094,"name":"online","context":{"idset":"10036"}}
+{"timestamp":1712007892.3421161,"name":"online","context":{"idset":"10373,10383,10449"}}
+{"timestamp":1712007892.4900484,"name":"online","context":{"idset":"10381,10386"}}
+{"timestamp":1712007892.6872938,"name":"online","context":{"idset":"10378"}}
+{"timestamp":1712007892.8642418,"name":"online","context":{"idset":"10387,10395"}}
+{"timestamp":1712007893.1541729,"name":"online","context":{"idset":"10401,10470"}}
+{"timestamp":1712007893.3137331,"name":"online","context":{"idset":"10414,10422,10446"}}
+{"timestamp":1712007893.4546711,"name":"online","context":{"idset":"10388-10389,10426,10437"}}
+{"timestamp":1712007893.5596333,"name":"online","context":{"idset":"10394,10396,10427"}}
+{"timestamp":1712007893.6603534,"name":"online","context":{"idset":"10382,10390,10405,10443,10448,10472"}}
+{"timestamp":1712007893.7975278,"name":"online","context":{"idset":"10391,10412,10415,10468"}}
+{"timestamp":1712007893.9058764,"name":"online","context":{"idset":"10376,10393,10424,10429,10434,10456,10461"}}
+{"timestamp":1712007894.0137861,"name":"online","context":{"idset":"10408,10413,10420,10430,10435,10469,10478"}}
+{"timestamp":1712007894.1341,"name":"online","context":{"idset":"10392,10399,10403,10409,10411,10423,10454"}}
+{"timestamp":1712007894.2804716,"name":"online","context":{"idset":"10400,10402,10404,10418,10455,10460,10465,10481-10482"}}
+{"timestamp":1712007894.4325171,"name":"online","context":{"idset":"10457,10459,10466,10471,10474-10476,10479-10480,10483-10484"}}
+{"timestamp":1712007894.5685382,"name":"online","context":{"idset":"10453,10477"}}
+{"timestamp":1712007894.7051384,"name":"online","context":{"idset":"10473"}}
+{"timestamp":1712007928.6388364,"name":"undrain","context":{"idset":"10375,10377,10384-10385,10406-10407,10410,10416-10417,10419,10421,10425,10428,10431-10433,10436,10438-10440,10444-10445,10447,10450-10452,10458,10462-10464,10467"}}
+{"timestamp":1712008068.8969369,"name":"online","context":{"idset":"10428,10432,10440"}}
+{"timestamp":1712008069.0439496,"name":"online","context":{"idset":"10425,10433,10458,10463,10467"}}
+{"timestamp":1712008069.1516206,"name":"online","context":{"idset":"10406,10417,10421,10439,10447"}}
+{"timestamp":1712008069.2594264,"name":"online","context":{"idset":"10407,10416,10431,10436,10438,10445,10451"}}
+{"timestamp":1712008069.394124,"name":"online","context":{"idset":"10410,10419,10444,10450,10452,10462"}}
+{"timestamp":1712008311.7992265,"name":"online","context":{"idset":"10377"}}
+{"timestamp":1712008311.9124794,"name":"online","context":{"idset":"10375,10384-10385"}}
+{"timestamp":1712008312.1105008,"name":"online","context":{"idset":"10464"}}
+{"timestamp":1712008730.7825291,"name":"offline","context":{"idset":"11828"}}
+{"timestamp":1712009097.7897105,"name":"drain","context":{"idset":"10399","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1712009104.252106,"name":"drain","context":{"idset":"10454","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1712009118.0677142,"name":"drain","context":{"idset":"10475","reason":"nodediag failed network cxi","overwrite":0}}
+{"timestamp":1712009129.1789448,"name":"drain","context":{"idset":"10464","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1712009130.9063466,"name":"drain","context":{"idset":"10478","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1712009893.0227642,"name":"online","context":{"idset":"849"}}
+{"timestamp":1712009923.577239,"name":"online","context":{"idset":"850"}}
+{"timestamp":1712010026.2590911,"name":"online","context":{"idset":"10012"}}
+{"timestamp":1712010027.8774679,"name":"online","context":{"idset":"10011"}}
+{"timestamp":1712010029.2138245,"name":"online","context":{"idset":"10010"}}
+{"timestamp":1712010031.1295588,"name":"online","context":{"idset":"10009"}}
+{"timestamp":1712010032.7167122,"name":"online","context":{"idset":"10008"}}
+{"timestamp":1712010034.1336293,"name":"online","context":{"idset":"10007"}}
+{"timestamp":1712010035.4755661,"name":"online","context":{"idset":"10006"}}
+{"timestamp":1712010037.0164366,"name":"online","context":{"idset":"10005"}}
+{"timestamp":1712010199.6079128,"name":"undrain","context":{"idset":"10399,10454,10464,10475,10478"}}
+{"timestamp":1712010802.1967065,"name":"undrain","context":{"idset":"11285-11300"}}
+{"timestamp":1712011059.9506621,"name":"undrain","context":{"idset":"11269-11284"}}
+{"timestamp":1712011568.0889397,"name":"online","context":{"idset":"11187"}}
+{"timestamp":1712011896.6217515,"name":"online","context":{"idset":"10020"}}
+{"timestamp":1712011898.1398132,"name":"online","context":{"idset":"10019"}}
+{"timestamp":1712011899.4510486,"name":"online","context":{"idset":"10018"}}
+{"timestamp":1712011900.542558,"name":"online","context":{"idset":"10017"}}
+{"timestamp":1712011901.6609716,"name":"online","context":{"idset":"10016"}}
+{"timestamp":1712011902.699856,"name":"online","context":{"idset":"10015"}}
+{"timestamp":1712011903.8480141,"name":"online","context":{"idset":"10014"}}
+{"timestamp":1712011907.8633473,"name":"online","context":{"idset":"10013"}}
+{"timestamp":1712012077.482532,"name":"online","context":{"idset":"11828"}}
+{"timestamp":1712013031.9132471,"name":"online","context":{"idset":"11277"}}
+{"timestamp":1712013032.1191094,"name":"online","context":{"idset":"11271,11279,11282"}}
+{"timestamp":1712013032.2803688,"name":"online","context":{"idset":"11270,11273"}}
+{"timestamp":1712013032.4265947,"name":"online","context":{"idset":"11269,11272,11275-11276,11283"}}
+{"timestamp":1712013032.5392005,"name":"online","context":{"idset":"11280-11281,11284"}}
+{"timestamp":1712013032.6928787,"name":"online","context":{"idset":"11274"}}
+{"timestamp":1712013032.9033434,"name":"online","context":{"idset":"11278"}}
+{"timestamp":1712013513.0749462,"name":"undrain","context":{"idset":"11260"}}
+{"timestamp":1712014009.6115859,"name":"online","context":{"idset":"9996"}}
+{"timestamp":1712014011.1563425,"name":"online","context":{"idset":"9995"}}
+{"timestamp":1712014012.8118844,"name":"online","context":{"idset":"9994"}}
+{"timestamp":1712014013.9204147,"name":"online","context":{"idset":"9993"}}
+{"timestamp":1712014015.191999,"name":"online","context":{"idset":"9992"}}
+{"timestamp":1712014016.5179267,"name":"online","context":{"idset":"9991"}}
+{"timestamp":1712014018.3929789,"name":"online","context":{"idset":"9990"}}
+{"timestamp":1712014020.4358125,"name":"online","context":{"idset":"9989"}}
+{"timestamp":1712015195.6642647,"name":"online","context":{"idset":"869"}}
+{"timestamp":1712015232.0077794,"name":"online","context":{"idset":"972,980"}}
+{"timestamp":1712015232.1212797,"name":"online","context":{"idset":"971,973,977-978"}}
+{"timestamp":1712015232.2760508,"name":"online","context":{"idset":"974,976,979"}}
+{"timestamp":1712015232.5232062,"name":"online","context":{"idset":"975"}}
+{"timestamp":1712015395.704354,"name":"online","context":{"idset":"970"}}
+{"timestamp":1712015396.1724503,"name":"online","context":{"idset":"969"}}
+{"timestamp":1712015793.2960994,"name":"drain","context":{"idset":"843-844","overwrite":0}}
+{"timestamp":1712016015.5758197,"name":"online","context":{"idset":"870"}}
+{"timestamp":1712016126.0143788,"name":"drain","context":{"idset":"869","overwrite":0}}
+{"timestamp":1712016412.3276682,"name":"online","context":{"idset":"11802"}}
+{"timestamp":1712016489.3744466,"name":"undrain","context":{"idset":"11802"}}
+{"timestamp":1712017026.0187469,"name":"online","context":{"idset":"9997"}}
+{"timestamp":1712017026.6559396,"name":"online","context":{"idset":"9998"}}
+{"timestamp":1712017027.7026606,"name":"online","context":{"idset":"9999"}}
+{"timestamp":1712017029.2957456,"name":"online","context":{"idset":"10000"}}
+{"timestamp":1712017029.7088056,"name":"online","context":{"idset":"10001"}}
+{"timestamp":1712017031.0996721,"name":"online","context":{"idset":"10002"}}
+{"timestamp":1712017032.6705844,"name":"online","context":{"idset":"10003"}}
+{"timestamp":1712017033.7279842,"name":"online","context":{"idset":"10004"}}
+{"timestamp":1712017343.4676061,"name":"online","context":{"idset":"9976"}}
+{"timestamp":1712017426.8392973,"name":"online","context":{"idset":"11800"}}
+{"timestamp":1712017427.1276965,"name":"online","context":{"idset":"11812"}}
+{"timestamp":1712017460.4460509,"name":"online","context":{"idset":"11803"}}
+{"timestamp":1712018212.7836771,"name":"drain","context":{"idset":"9973","reason":"broker was unresponsive"}}
+{"timestamp":1712018276.7832551,"name":"offline","context":{"idset":"9973"}}
+{"timestamp":1712019578.9623601,"name":"online","context":{"idset":"9973"}}
+{"timestamp":1712024995.6795928,"name":"online","context":{"idset":"10884"}}
+{"timestamp":1712024996.0200057,"name":"online","context":{"idset":"10905"}}
+{"timestamp":1712026940.6850164,"name":"drain","context":{"idset":"11592","reason":"broker was unresponsive"}}
+{"timestamp":1712026940.6851528,"name":"drain","context":{"idset":"11603","reason":"broker was unresponsive"}}
+{"timestamp":1712026940.6852219,"name":"drain","context":{"idset":"11609","reason":"broker was unresponsive"}}
+{"timestamp":1712026940.6852841,"name":"drain","context":{"idset":"11619","reason":"broker was unresponsive"}}
+{"timestamp":1712026940.6853518,"name":"drain","context":{"idset":"11631","reason":"broker was unresponsive"}}
+{"timestamp":1712026940.7842064,"name":"drain","context":{"idset":"11633","reason":"broker was unresponsive"}}
+{"timestamp":1712027003.0212746,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1712027004.6935833,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1712027004.707114,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1712027005.0940239,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1712027006.6924298,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1712027007.0940049,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1712027625.9002004,"name":"online","context":{"idset":"10874"}}
+{"timestamp":1712027626.1351318,"name":"online","context":{"idset":"10871,10906,10911"}}
+{"timestamp":1712027626.2984548,"name":"online","context":{"idset":"10873,10883"}}
+{"timestamp":1712028840.7845249,"name":"drain","context":{"idset":"11612","reason":"broker was unresponsive"}}
+{"timestamp":1712028902.7822435,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1712030020.2899971,"name":"online","context":{"idset":"11592,11619"}}
+{"timestamp":1712030020.4914134,"name":"online","context":{"idset":"11609"}}
+{"timestamp":1712030020.6439605,"name":"online","context":{"idset":"11603,11631"}}
+{"timestamp":1712030580.7826135,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1712031205.6575637,"name":"drain","context":{"idset":"10884","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1712043086.7846026,"name":"drain","context":{"idset":"11527","reason":"broker was unresponsive"}}
+{"timestamp":1712043151.0011849,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1712064760.0042455,"name":"offline","context":{"idset":"843"}}
+{"timestamp":1712064781.189095,"name":"offline","context":{"idset":"844"}}
+{"timestamp":1712064926.9195154,"name":"online","context":{"idset":"844"}}
+{"timestamp":1712064927.2511854,"name":"online","context":{"idset":"843"}}
+{"timestamp":1712065402.4546242,"name":"drain","context":{"idset":"844","reason":"epilog failed for jobid fn4AXyriRmq","overwrite":0}}
+{"timestamp":1712067818.7824368,"name":"offline","context":{"idset":"10765"}}
+{"timestamp":1712068080.9443812,"name":"drain","context":{"idset":"11829-11844","reason":"Running Diags - Rabbits Off","overwrite":0}}
+{"timestamp":1712068141.3627269,"name":"drain","context":{"idset":"11845-11860","reason":"Running Diags - Rabbits Off","overwrite":1}}
+{"timestamp":1712068186.5559592,"name":"drain","context":{"idset":"11861-11876","reason":"Running Diags - Rabbits Off","overwrite":0}}
+{"timestamp":1712069192.4458654,"name":"online","context":{"idset":"11860"}}
+{"timestamp":1712070261.6910651,"name":"offline","context":{"idset":"11842"}}
+{"timestamp":1712070271.6910779,"name":"offline","context":{"idset":"11841"}}
+{"timestamp":1712071031.6908116,"name":"offline","context":{"idset":"11874"}}
+{"timestamp":1712071187.6048658,"name":"online","context":{"idset":"10154"}}
+{"timestamp":1712071600.1941323,"name":"undrain","context":{"idset":"843-844,849-850,869-870,969-980,11187"}}
+{"timestamp":1712073532.3928831,"name":"online","context":{"idset":"11841"}}
+{"timestamp":1712073532.5057764,"name":"online","context":{"idset":"11842"}}
+{"timestamp":1712073541.647979,"name":"online","context":{"idset":"11836"}}
+{"timestamp":1712073755.5918038,"name":"drain","context":{"idset":"11319","reason":"broker was unresponsive"}}
+{"timestamp":1712073755.5919735,"name":"drain","context":{"idset":"11320","reason":"broker was unresponsive"}}
+{"timestamp":1712073755.6913724,"name":"drain","context":{"idset":"11321","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.5950892,"name":"drain","context":{"idset":"11322","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.5952139,"name":"drain","context":{"idset":"11323","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.595295,"name":"drain","context":{"idset":"11324","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.5953906,"name":"drain","context":{"idset":"11325","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.5954952,"name":"drain","context":{"idset":"11326","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.5955708,"name":"drain","context":{"idset":"11327","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.5956366,"name":"drain","context":{"idset":"11328","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.5957003,"name":"drain","context":{"idset":"11329","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.5957878,"name":"drain","context":{"idset":"11330","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.5958869,"name":"drain","context":{"idset":"11331","reason":"broker was unresponsive"}}
+{"timestamp":1712073761.7011204,"name":"drain","context":{"idset":"11332","reason":"broker was unresponsive"}}
+{"timestamp":1712073789.1198373,"name":"online","context":{"idset":"10869"}}
+{"timestamp":1712073789.1824834,"name":"online","context":{"idset":"10895"}}
+{"timestamp":1712073789.2475567,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1712073789.3048887,"name":"online","context":{"idset":"10878"}}
+{"timestamp":1712073789.3745039,"name":"online","context":{"idset":"10887,10889"}}
+{"timestamp":1712073789.432339,"name":"online","context":{"idset":"10875"}}
+{"timestamp":1712073789.4976227,"name":"online","context":{"idset":"10903"}}
+{"timestamp":1712073789.7234869,"name":"online","context":{"idset":"10870,10885,10901"}}
+{"timestamp":1712073789.784107,"name":"online","context":{"idset":"10880"}}
+{"timestamp":1712073790.0902436,"name":"online","context":{"idset":"10913"}}
+{"timestamp":1712073790.2705636,"name":"online","context":{"idset":"10881"}}
+{"timestamp":1712073790.4795582,"name":"online","context":{"idset":"10876,10882,10893,10898"}}
+{"timestamp":1712073790.628623,"name":"online","context":{"idset":"10879,10891,10914,10938,10955"}}
+{"timestamp":1712073790.9073088,"name":"online","context":{"idset":"10886,10910,10915"}}
+{"timestamp":1712073791.0195553,"name":"online","context":{"idset":"10902,10936"}}
+{"timestamp":1712073791.1726809,"name":"online","context":{"idset":"10892,10894,10896,10904,10907-10908,10950"}}
+{"timestamp":1712073791.3610823,"name":"online","context":{"idset":"10877,10916,10920"}}
+{"timestamp":1712073791.5086544,"name":"online","context":{"idset":"10888,10897,10900,10909,10919,10925,10933,10940,10949,10991"}}
+{"timestamp":1712073791.6224983,"name":"online","context":{"idset":"10937,10944,10965,10967"}}
+{"timestamp":1712073791.7702363,"name":"online","context":{"idset":"10954,10980,10984"}}
+{"timestamp":1712073791.918792,"name":"online","context":{"idset":"10945,10948,10951,10959,10974,10983,10996"}}
+{"timestamp":1712073792.0654182,"name":"online","context":{"idset":"10899,10946,10952-10953,10957,10961,10969,10977,10990"}}
+{"timestamp":1712073792.1789896,"name":"online","context":{"idset":"10943,10956,10972-10973,10985-10986,10992,10995"}}
+{"timestamp":1712073792.2921069,"name":"online","context":{"idset":"10941,10958,10970,10975,10987"}}
+{"timestamp":1712073792.4430132,"name":"online","context":{"idset":"10890,10942,10963-10964,10979"}}
+{"timestamp":1712073792.5585046,"name":"online","context":{"idset":"10935,10939,10960,10968,10978,10989"}}
+{"timestamp":1712073792.7402236,"name":"online","context":{"idset":"10962,10966,10971,10994"}}
+{"timestamp":1712073792.9621081,"name":"online","context":{"idset":"10947,10976,10988"}}
+{"timestamp":1712073793.3481345,"name":"online","context":{"idset":"10934"}}
+{"timestamp":1712073821.5969577,"name":"offline","context":{"idset":"11319"}}
+{"timestamp":1712073821.6047647,"name":"offline","context":{"idset":"11320"}}
+{"timestamp":1712073821.6905663,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1712073823.5958357,"name":"offline","context":{"idset":"11322"}}
+{"timestamp":1712073823.603636,"name":"offline","context":{"idset":"11323"}}
+{"timestamp":1712073823.9606686,"name":"offline","context":{"idset":"11324"}}
+{"timestamp":1712073825.6031599,"name":"offline","context":{"idset":"11325"}}
+{"timestamp":1712073825.6168633,"name":"offline","context":{"idset":"11326"}}
+{"timestamp":1712073825.6298335,"name":"offline","context":{"idset":"11327"}}
+{"timestamp":1712073825.6420181,"name":"offline","context":{"idset":"11328"}}
+{"timestamp":1712073825.6960793,"name":"offline","context":{"idset":"11329"}}
+{"timestamp":1712073827.597734,"name":"offline","context":{"idset":"11330"}}
+{"timestamp":1712073827.6056886,"name":"offline","context":{"idset":"11331"}}
+{"timestamp":1712073827.6900003,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1712074162.6833539,"name":"online","context":{"idset":"11439"}}
+{"timestamp":1712074619.1270335,"name":"undrain","context":{"idset":"10036"}}
+{"timestamp":1712074635.0891457,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1712074785.0442247,"name":"online","context":{"idset":"11874"}}
+{"timestamp":1712075292.8792295,"name":"drain","context":{"idset":"10869","overwrite":0}}
+{"timestamp":1712075295.592694,"name":"drain","context":{"idset":"10870","overwrite":0}}
+{"timestamp":1712075429.6147048,"name":"drain","context":{"idset":"10872","overwrite":0}}
+{"timestamp":1712075432.3684921,"name":"drain","context":{"idset":"10875","overwrite":0}}
+{"timestamp":1712075435.4121058,"name":"drain","context":{"idset":"10876","overwrite":0}}
+{"timestamp":1712075438.2462318,"name":"drain","context":{"idset":"10877","overwrite":0}}
+{"timestamp":1712075441.0759542,"name":"drain","context":{"idset":"10878","overwrite":0}}
+{"timestamp":1712075443.8583369,"name":"drain","context":{"idset":"10879","overwrite":0}}
+{"timestamp":1712075446.6319361,"name":"drain","context":{"idset":"10880","overwrite":0}}
+{"timestamp":1712075449.4528251,"name":"drain","context":{"idset":"10881","overwrite":0}}
+{"timestamp":1712075452.2567382,"name":"drain","context":{"idset":"10882","overwrite":0}}
+{"timestamp":1712075457.797327,"name":"drain","context":{"idset":"10885","overwrite":0}}
+{"timestamp":1712075460.5568137,"name":"drain","context":{"idset":"10886","overwrite":0}}
+{"timestamp":1712075463.370959,"name":"drain","context":{"idset":"10887","overwrite":0}}
+{"timestamp":1712075466.042644,"name":"drain","context":{"idset":"10888","overwrite":0}}
+{"timestamp":1712075468.8039017,"name":"drain","context":{"idset":"10889","overwrite":0}}
+{"timestamp":1712075471.6518552,"name":"drain","context":{"idset":"10890","overwrite":0}}
+{"timestamp":1712075474.3596859,"name":"drain","context":{"idset":"10891","overwrite":0}}
+{"timestamp":1712075477.0721288,"name":"drain","context":{"idset":"10892","overwrite":0}}
+{"timestamp":1712075479.8134344,"name":"drain","context":{"idset":"10893","overwrite":0}}
+{"timestamp":1712075482.6898034,"name":"drain","context":{"idset":"10894","overwrite":0}}
+{"timestamp":1712075485.3636742,"name":"drain","context":{"idset":"10895","overwrite":0}}
+{"timestamp":1712075488.848227,"name":"drain","context":{"idset":"10896","overwrite":0}}
+{"timestamp":1712075491.9506924,"name":"drain","context":{"idset":"10897","overwrite":0}}
+{"timestamp":1712075495.4196017,"name":"drain","context":{"idset":"10898","overwrite":0}}
+{"timestamp":1712075498.1424453,"name":"drain","context":{"idset":"10899","overwrite":0}}
+{"timestamp":1712075500.9534094,"name":"drain","context":{"idset":"10900","overwrite":0}}
+{"timestamp":1712075503.6977541,"name":"drain","context":{"idset":"10901","overwrite":0}}
+{"timestamp":1712075506.5308251,"name":"drain","context":{"idset":"10902","overwrite":0}}
+{"timestamp":1712075509.4403024,"name":"drain","context":{"idset":"10903","overwrite":0}}
+{"timestamp":1712075512.099128,"name":"drain","context":{"idset":"10904","overwrite":0}}
+{"timestamp":1712075514.8124657,"name":"drain","context":{"idset":"10907","overwrite":0}}
+{"timestamp":1712075517.5694385,"name":"drain","context":{"idset":"10908","overwrite":0}}
+{"timestamp":1712075520.3469443,"name":"drain","context":{"idset":"10909","overwrite":0}}
+{"timestamp":1712075523.2204323,"name":"drain","context":{"idset":"10910","overwrite":0}}
+{"timestamp":1712075654.6259868,"name":"drain","context":{"idset":"10913","overwrite":0}}
+{"timestamp":1712075657.3611178,"name":"drain","context":{"idset":"10914","overwrite":0}}
+{"timestamp":1712075660.1857314,"name":"drain","context":{"idset":"10915","overwrite":0}}
+{"timestamp":1712075663.0238438,"name":"drain","context":{"idset":"10916","overwrite":0}}
+{"timestamp":1712075665.8771789,"name":"drain","context":{"idset":"10919","overwrite":0}}
+{"timestamp":1712075668.8911669,"name":"drain","context":{"idset":"10920","overwrite":0}}
+{"timestamp":1712075671.8149934,"name":"drain","context":{"idset":"10933","overwrite":0}}
+{"timestamp":1712075674.9389839,"name":"drain","context":{"idset":"10934","overwrite":0}}
+{"timestamp":1712075677.6372676,"name":"drain","context":{"idset":"10935","overwrite":0}}
+{"timestamp":1712075680.3478642,"name":"drain","context":{"idset":"10936","overwrite":0}}
+{"timestamp":1712075683.2029836,"name":"drain","context":{"idset":"10937","overwrite":0}}
+{"timestamp":1712075685.9562981,"name":"drain","context":{"idset":"10938","overwrite":0}}
+{"timestamp":1712075688.6737597,"name":"drain","context":{"idset":"10939","overwrite":0}}
+{"timestamp":1712075691.5527005,"name":"drain","context":{"idset":"10940","overwrite":0}}
+{"timestamp":1712075694.3174336,"name":"drain","context":{"idset":"10941","overwrite":0}}
+{"timestamp":1712075697.0958941,"name":"drain","context":{"idset":"10942","overwrite":0}}
+{"timestamp":1712075699.8340847,"name":"drain","context":{"idset":"10943","overwrite":0}}
+{"timestamp":1712075702.5863547,"name":"drain","context":{"idset":"10944","overwrite":0}}
+{"timestamp":1712075705.3843641,"name":"drain","context":{"idset":"10945","overwrite":0}}
+{"timestamp":1712075708.1143255,"name":"drain","context":{"idset":"10946","overwrite":0}}
+{"timestamp":1712075710.9195693,"name":"drain","context":{"idset":"10947","overwrite":0}}
+{"timestamp":1712075713.7172279,"name":"drain","context":{"idset":"10948","overwrite":0}}
+{"timestamp":1712075716.5015185,"name":"drain","context":{"idset":"10949","overwrite":0}}
+{"timestamp":1712075719.2634649,"name":"drain","context":{"idset":"10950","overwrite":0}}
+{"timestamp":1712075722.0004787,"name":"drain","context":{"idset":"10951","overwrite":0}}
+{"timestamp":1712075724.7967033,"name":"drain","context":{"idset":"10952","overwrite":0}}
+{"timestamp":1712075728.200984,"name":"drain","context":{"idset":"10953","overwrite":0}}
+{"timestamp":1712075731.4775343,"name":"drain","context":{"idset":"10954","overwrite":0}}
+{"timestamp":1712075734.7953773,"name":"drain","context":{"idset":"10955","overwrite":0}}
+{"timestamp":1712075737.5193365,"name":"drain","context":{"idset":"10956","overwrite":0}}
+{"timestamp":1712075740.2685692,"name":"drain","context":{"idset":"10957","overwrite":0}}
+{"timestamp":1712075743.0691776,"name":"drain","context":{"idset":"10958","overwrite":0}}
+{"timestamp":1712075745.8575826,"name":"drain","context":{"idset":"10959","overwrite":0}}
+{"timestamp":1712075748.5080221,"name":"drain","context":{"idset":"10960","overwrite":0}}
+{"timestamp":1712075751.2755439,"name":"drain","context":{"idset":"10961","overwrite":0}}
+{"timestamp":1712075754.0125215,"name":"drain","context":{"idset":"10962","overwrite":0}}
+{"timestamp":1712075756.8840353,"name":"drain","context":{"idset":"10963","overwrite":0}}
+{"timestamp":1712075759.6015449,"name":"drain","context":{"idset":"10964","overwrite":0}}
+{"timestamp":1712075762.288167,"name":"drain","context":{"idset":"10965","overwrite":0}}
+{"timestamp":1712075765.0413864,"name":"drain","context":{"idset":"10966","overwrite":0}}
+{"timestamp":1712075767.8569207,"name":"drain","context":{"idset":"10967","overwrite":0}}
+{"timestamp":1712075770.6701047,"name":"drain","context":{"idset":"10968","overwrite":0}}
+{"timestamp":1712075773.3886943,"name":"drain","context":{"idset":"10969","overwrite":0}}
+{"timestamp":1712075776.2587945,"name":"drain","context":{"idset":"10970","overwrite":0}}
+{"timestamp":1712075779.1861453,"name":"drain","context":{"idset":"10971","overwrite":0}}
+{"timestamp":1712075781.9166141,"name":"drain","context":{"idset":"10972","overwrite":0}}
+{"timestamp":1712075784.6050305,"name":"drain","context":{"idset":"10973","overwrite":0}}
+{"timestamp":1712075788.0068188,"name":"drain","context":{"idset":"10974","overwrite":0}}
+{"timestamp":1712075791.4100859,"name":"drain","context":{"idset":"10975","overwrite":0}}
+{"timestamp":1712075794.2928064,"name":"drain","context":{"idset":"10976","overwrite":0}}
+{"timestamp":1712075797.1236913,"name":"drain","context":{"idset":"10977","overwrite":0}}
+{"timestamp":1712075799.7588124,"name":"drain","context":{"idset":"10978","overwrite":0}}
+{"timestamp":1712075802.6327755,"name":"drain","context":{"idset":"10979","overwrite":0}}
+{"timestamp":1712075805.495199,"name":"drain","context":{"idset":"10980","overwrite":0}}
+{"timestamp":1712075808.3565192,"name":"drain","context":{"idset":"10983","overwrite":0}}
+{"timestamp":1712075811.1280558,"name":"drain","context":{"idset":"10984","overwrite":0}}
+{"timestamp":1712075813.935683,"name":"drain","context":{"idset":"10985","overwrite":0}}
+{"timestamp":1712075816.6926124,"name":"drain","context":{"idset":"10986","overwrite":0}}
+{"timestamp":1712075819.5536861,"name":"drain","context":{"idset":"10987","overwrite":0}}
+{"timestamp":1712075822.3387408,"name":"drain","context":{"idset":"10988","overwrite":0}}
+{"timestamp":1712075825.2598395,"name":"drain","context":{"idset":"10989","overwrite":0}}
+{"timestamp":1712075828.1209693,"name":"drain","context":{"idset":"10990","overwrite":0}}
+{"timestamp":1712075830.8867323,"name":"drain","context":{"idset":"10991","overwrite":0}}
+{"timestamp":1712075833.5121868,"name":"drain","context":{"idset":"10992","overwrite":0}}
+{"timestamp":1712075836.3265541,"name":"drain","context":{"idset":"10994","overwrite":0}}
+{"timestamp":1712075839.1943069,"name":"drain","context":{"idset":"10995","overwrite":0}}
+{"timestamp":1712075841.9888263,"name":"drain","context":{"idset":"10996","overwrite":0}}
+{"timestamp":1712076244.4205413,"name":"undrain","context":{"idset":"10869"}}
+{"timestamp":1712076247.2144651,"name":"undrain","context":{"idset":"10870"}}
+{"timestamp":1712076379.5499036,"name":"undrain","context":{"idset":"10872"}}
+{"timestamp":1712076382.2665579,"name":"undrain","context":{"idset":"10875"}}
+{"timestamp":1712076385.0656292,"name":"undrain","context":{"idset":"10876"}}
+{"timestamp":1712076388.2172365,"name":"undrain","context":{"idset":"10877"}}
+{"timestamp":1712076392.2090325,"name":"undrain","context":{"idset":"10878"}}
+{"timestamp":1712076395.1525078,"name":"undrain","context":{"idset":"10879"}}
+{"timestamp":1712076397.9845252,"name":"undrain","context":{"idset":"10880"}}
+{"timestamp":1712076400.7316546,"name":"undrain","context":{"idset":"10881"}}
+{"timestamp":1712076404.4184027,"name":"undrain","context":{"idset":"10882"}}
+{"timestamp":1712076408.2109797,"name":"undrain","context":{"idset":"10884"}}
+{"timestamp":1712076413.3862524,"name":"undrain","context":{"idset":"10885"}}
+{"timestamp":1712076416.222759,"name":"undrain","context":{"idset":"10886"}}
+{"timestamp":1712076419.1587887,"name":"undrain","context":{"idset":"10887"}}
+{"timestamp":1712076421.8949997,"name":"undrain","context":{"idset":"10888"}}
+{"timestamp":1712076424.7212896,"name":"undrain","context":{"idset":"10889"}}
+{"timestamp":1712076427.6348083,"name":"undrain","context":{"idset":"10890"}}
+{"timestamp":1712076430.3109276,"name":"undrain","context":{"idset":"10891"}}
+{"timestamp":1712076433.1564848,"name":"undrain","context":{"idset":"10892"}}
+{"timestamp":1712076435.7004876,"name":"undrain","context":{"idset":"10893"}}
+{"timestamp":1712076438.3805535,"name":"undrain","context":{"idset":"10894"}}
+{"timestamp":1712076441.1224384,"name":"undrain","context":{"idset":"10895"}}
+{"timestamp":1712076443.9007037,"name":"undrain","context":{"idset":"10896"}}
+{"timestamp":1712076446.9864974,"name":"undrain","context":{"idset":"10897"}}
+{"timestamp":1712076449.9463453,"name":"undrain","context":{"idset":"10898"}}
+{"timestamp":1712076452.8807306,"name":"undrain","context":{"idset":"10899"}}
+{"timestamp":1712076456.1879599,"name":"undrain","context":{"idset":"10900"}}
+{"timestamp":1712076458.9819133,"name":"undrain","context":{"idset":"10901"}}
+{"timestamp":1712076461.8224978,"name":"undrain","context":{"idset":"10902"}}
+{"timestamp":1712076464.5794237,"name":"undrain","context":{"idset":"10903"}}
+{"timestamp":1712076467.2769136,"name":"undrain","context":{"idset":"10904"}}
+{"timestamp":1712076469.8869658,"name":"undrain","context":{"idset":"10907"}}
+{"timestamp":1712076472.6804974,"name":"undrain","context":{"idset":"10908"}}
+{"timestamp":1712076475.3685801,"name":"undrain","context":{"idset":"10909"}}
+{"timestamp":1712076478.1575925,"name":"undrain","context":{"idset":"10910"}}
+{"timestamp":1712076613.1504118,"name":"undrain","context":{"idset":"10913"}}
+{"timestamp":1712076615.8838284,"name":"undrain","context":{"idset":"10914"}}
+{"timestamp":1712076618.6688695,"name":"undrain","context":{"idset":"10915"}}
+{"timestamp":1712076621.4812531,"name":"undrain","context":{"idset":"10916"}}
+{"timestamp":1712076624.2290711,"name":"undrain","context":{"idset":"10919"}}
+{"timestamp":1712076627.4174709,"name":"undrain","context":{"idset":"10920"}}
+{"timestamp":1712076630.3752985,"name":"undrain","context":{"idset":"10933"}}
+{"timestamp":1712076633.2809477,"name":"undrain","context":{"idset":"10934"}}
+{"timestamp":1712076636.1765876,"name":"undrain","context":{"idset":"10935"}}
+{"timestamp":1712076638.9633534,"name":"undrain","context":{"idset":"10936"}}
+{"timestamp":1712076641.6473856,"name":"undrain","context":{"idset":"10937"}}
+{"timestamp":1712076644.3826854,"name":"undrain","context":{"idset":"10938"}}
+{"timestamp":1712076647.3114069,"name":"undrain","context":{"idset":"10939"}}
+{"timestamp":1712076650.1150262,"name":"undrain","context":{"idset":"10940"}}
+{"timestamp":1712076652.8231287,"name":"undrain","context":{"idset":"10941"}}
+{"timestamp":1712076655.5796843,"name":"undrain","context":{"idset":"10942"}}
+{"timestamp":1712076658.4129519,"name":"undrain","context":{"idset":"10943"}}
+{"timestamp":1712076661.1387603,"name":"undrain","context":{"idset":"10944"}}
+{"timestamp":1712076663.9957685,"name":"undrain","context":{"idset":"10945"}}
+{"timestamp":1712076666.6752613,"name":"undrain","context":{"idset":"10946"}}
+{"timestamp":1712076669.4965441,"name":"undrain","context":{"idset":"10947"}}
+{"timestamp":1712076672.3423204,"name":"undrain","context":{"idset":"10948"}}
+{"timestamp":1712076675.2323258,"name":"undrain","context":{"idset":"10949"}}
+{"timestamp":1712076678.0345511,"name":"undrain","context":{"idset":"10950"}}
+{"timestamp":1712076680.8605561,"name":"undrain","context":{"idset":"10951"}}
+{"timestamp":1712076683.6834807,"name":"undrain","context":{"idset":"10952"}}
+{"timestamp":1712076686.5523288,"name":"undrain","context":{"idset":"10953"}}
+{"timestamp":1712076689.8088026,"name":"undrain","context":{"idset":"10954"}}
+{"timestamp":1712076692.5361893,"name":"undrain","context":{"idset":"10955"}}
+{"timestamp":1712076695.4346242,"name":"undrain","context":{"idset":"10956"}}
+{"timestamp":1712076697.9765167,"name":"undrain","context":{"idset":"10957"}}
+{"timestamp":1712076700.7296174,"name":"undrain","context":{"idset":"10958"}}
+{"timestamp":1712076703.4548845,"name":"undrain","context":{"idset":"10959"}}
+{"timestamp":1712076706.1840894,"name":"undrain","context":{"idset":"10960"}}
+{"timestamp":1712076708.7216828,"name":"undrain","context":{"idset":"10961"}}
+{"timestamp":1712076711.4425118,"name":"undrain","context":{"idset":"10962"}}
+{"timestamp":1712076714.2875788,"name":"undrain","context":{"idset":"10963"}}
+{"timestamp":1712076717.0526371,"name":"undrain","context":{"idset":"10964"}}
+{"timestamp":1712076720.0896626,"name":"undrain","context":{"idset":"10965"}}
+{"timestamp":1712076723.2179456,"name":"undrain","context":{"idset":"10966"}}
+{"timestamp":1712076726.0195475,"name":"undrain","context":{"idset":"10967"}}
+{"timestamp":1712076728.7270808,"name":"undrain","context":{"idset":"10968"}}
+{"timestamp":1712076731.6104257,"name":"undrain","context":{"idset":"10969"}}
+{"timestamp":1712076734.3815386,"name":"undrain","context":{"idset":"10970"}}
+{"timestamp":1712076736.9337127,"name":"undrain","context":{"idset":"10971"}}
+{"timestamp":1712076739.5968995,"name":"undrain","context":{"idset":"10972"}}
+{"timestamp":1712076742.3058186,"name":"undrain","context":{"idset":"10973"}}
+{"timestamp":1712076745.1589425,"name":"undrain","context":{"idset":"10974"}}
+{"timestamp":1712076748.42664,"name":"undrain","context":{"idset":"10975"}}
+{"timestamp":1712076751.2788029,"name":"undrain","context":{"idset":"10976"}}
+{"timestamp":1712076754.5779049,"name":"undrain","context":{"idset":"10977"}}
+{"timestamp":1712076757.2486124,"name":"undrain","context":{"idset":"10978"}}
+{"timestamp":1712076760.0004389,"name":"undrain","context":{"idset":"10979"}}
+{"timestamp":1712076763.0402324,"name":"undrain","context":{"idset":"10980"}}
+{"timestamp":1712076765.6431158,"name":"undrain","context":{"idset":"10983"}}
+{"timestamp":1712076768.128938,"name":"undrain","context":{"idset":"10984"}}
+{"timestamp":1712076770.6134071,"name":"undrain","context":{"idset":"10985"}}
+{"timestamp":1712076773.0539017,"name":"undrain","context":{"idset":"10986"}}
+{"timestamp":1712076775.639195,"name":"undrain","context":{"idset":"10987"}}
+{"timestamp":1712076778.1416931,"name":"undrain","context":{"idset":"10988"}}
+{"timestamp":1712076780.8092299,"name":"undrain","context":{"idset":"10989"}}
+{"timestamp":1712076783.472959,"name":"undrain","context":{"idset":"10990"}}
+{"timestamp":1712076786.5951734,"name":"undrain","context":{"idset":"10991"}}
+{"timestamp":1712076789.4541113,"name":"undrain","context":{"idset":"10992"}}
+{"timestamp":1712076792.3903658,"name":"undrain","context":{"idset":"10994"}}
+{"timestamp":1712076795.3869371,"name":"undrain","context":{"idset":"10995"}}
+{"timestamp":1712076798.3930774,"name":"undrain","context":{"idset":"10996"}}
+{"timestamp":1712077371.0823727,"name":"online","context":{"idset":"11125"}}
+{"timestamp":1712077371.551857,"name":"online","context":{"idset":"11126"}}
+{"timestamp":1712079043.0907531,"name":"drain","context":{"idset":"11125","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9967983,"name":"drain","context":{"idset":"11127","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9969132,"name":"drain","context":{"idset":"11128","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9969907,"name":"drain","context":{"idset":"11129","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9970613,"name":"drain","context":{"idset":"11130","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9971497,"name":"drain","context":{"idset":"11131","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9972458,"name":"drain","context":{"idset":"11132","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9973373,"name":"drain","context":{"idset":"11133","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9974139,"name":"drain","context":{"idset":"11134","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9975028,"name":"drain","context":{"idset":"11135","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9975903,"name":"drain","context":{"idset":"11136","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9976768,"name":"drain","context":{"idset":"11137","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9977582,"name":"drain","context":{"idset":"11138","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9978447,"name":"drain","context":{"idset":"11139","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.997932,"name":"drain","context":{"idset":"11140","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9980164,"name":"drain","context":{"idset":"11141","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9981048,"name":"drain","context":{"idset":"11142","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9981937,"name":"drain","context":{"idset":"11143","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9982798,"name":"drain","context":{"idset":"11144","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9983644,"name":"drain","context":{"idset":"11145","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.998455,"name":"drain","context":{"idset":"11146","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9985433,"name":"drain","context":{"idset":"11147","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9986298,"name":"drain","context":{"idset":"11148","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9987171,"name":"drain","context":{"idset":"11151","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9988039,"name":"drain","context":{"idset":"11152","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9988978,"name":"drain","context":{"idset":"11153","reason":"broker was unresponsive"}}
+{"timestamp":1712079046.9989817,"name":"drain","context":{"idset":"11155","reason":"broker was unresponsive"}}
+{"timestamp":1712079047.2586949,"name":"drain","context":{"idset":"11156","reason":"broker was unresponsive"}}
+{"timestamp":1712079109.0226793,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1712079109.0305324,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1712079109.0378277,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1712079109.0452096,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1712079109.0527632,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1712079109.0598118,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1712079109.0705338,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1712079109.078671,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1712079109.0956604,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1712079109.0974395,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1712079109.1016357,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1712079109.1213577,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1712079109.1613283,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1712079109.163275,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1712079109.1650178,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1712079109.1668959,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1712079109.1687543,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1712079109.1706851,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1712079109.1727426,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1712079109.1745567,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1712079109.1785147,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1712079109.1856546,"name":"offline","context":{"idset":"11146"}}
+{"timestamp":1712079109.1925986,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1712079109.1998048,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1712079109.2068994,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1712079109.2142828,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1712079109.2212739,"name":"offline","context":{"idset":"11153"}}
+{"timestamp":1712079109.2283671,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1712079109.2380843,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1712079171.5353351,"name":"drain","context":{"idset":"10120-10129","overwrite":0}}
+{"timestamp":1712079203.2770023,"name":"drain","context":{"idset":"11375-11382","reason":"draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1712079317.090462,"name":"offline","context":{"idset":"11439"}}
+{"timestamp":1712079667.9903784,"name":"online","context":{"idset":"10871"}}
+{"timestamp":1712080247.9727101,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1712080298.8582935,"name":"online","context":{"idset":"10871"}}
+{"timestamp":1712080321.4871211,"name":"undrain","context":{"idset":"10871"}}
+{"timestamp":1712084137.9055309,"name":"drain","context":{"idset":"841","overwrite":0}}
+{"timestamp":1712084431.9012132,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1712084505.1794877,"name":"online","context":{"idset":"11139"}}
+{"timestamp":1712084505.2287405,"name":"online","context":{"idset":"11127,11129"}}
+{"timestamp":1712084505.3756237,"name":"online","context":{"idset":"11132-11133"}}
+{"timestamp":1712084505.5673411,"name":"online","context":{"idset":"11128,11137"}}
+{"timestamp":1712084505.7153852,"name":"online","context":{"idset":"11134,11138"}}
+{"timestamp":1712084505.8882749,"name":"online","context":{"idset":"11125,11130,11136,11140"}}
+{"timestamp":1712084506.2440164,"name":"online","context":{"idset":"11131"}}
+{"timestamp":1712084506.6988308,"name":"online","context":{"idset":"11135"}}
+{"timestamp":1712084858.0621107,"name":"offline","context":{"idset":"839"}}
+{"timestamp":1712084858.1468177,"name":"offline","context":{"idset":"840"}}
+{"timestamp":1712085370.4801667,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1712085370.4996798,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1712085370.5117886,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1712085370.5321295,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1712085370.6022561,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1712085370.6549685,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1712085370.7034957,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1712085370.7275994,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1712085370.7363698,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1712085370.7453649,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1712085370.7700589,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1712085370.7825706,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1712085370.8465362,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1712085370.8754485,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1712085370.8771954,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1712085370.8868124,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1712085370.8966079,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1712085370.9056251,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1712085370.9151227,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1712085370.9168518,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1712085370.9398937,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1712085370.9507668,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1712085370.9833465,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1712085370.9850719,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1712085370.995676,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1712085371.0289369,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1712085371.0308142,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1712085371.0384521,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1712085371.0422571,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1712085371.0525711,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1712085371.0728474,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1712085371.111043,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1712085371.1225858,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1712085371.1244247,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1712085371.126205,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1712085371.1279292,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1712085371.1296644,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1712085371.1314266,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1712085371.1331596,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1712085371.1466577,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1712085371.1673117,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1712085371.1887865,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1712085371.2094359,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1712085371.2111609,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1712085371.223943,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1712085371.2406602,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1712085371.2423251,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1712085371.2440181,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1712085371.2495267,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1712085371.2696166,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1712085371.2908146,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1712085371.3117597,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1712085371.3135412,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1712085371.3152471,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1712085371.3173785,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1712085371.3190663,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1712085371.3233809,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1712085371.3314641,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1712085371.3392751,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1712085371.3553042,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1712085371.3637981,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1712085371.3655882,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1712085371.3799608,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1712085371.3951256,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1712085371.4101706,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1712085371.4179041,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1712085371.4451728,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1712085371.4570067,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1712085371.4834533,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1712085371.514312,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1712085371.9637945,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1712085371.9672351,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1712085371.9701102,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1712085372.1788702,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1712085372.1944921,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1712085372.2445126,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1712085372.2666287,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1712085372.281332,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1712085372.3257201,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1712085372.3374512,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1712085372.3392301,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1712085372.3695681,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1712085372.3713565,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1712085372.3807642,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1712085372.4082162,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1712085372.5802627,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1712085372.8410947,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1712085372.8620915,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1712085372.873219,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1712085372.9100864,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1712085372.911839,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1712086500.4114876,"name":"online","context":{"idset":"10496"}}
+{"timestamp":1712086500.5236831,"name":"online","context":{"idset":"10494"}}
+{"timestamp":1712086500.5758035,"name":"online","context":{"idset":"10511"}}
+{"timestamp":1712086500.6732755,"name":"online","context":{"idset":"10485,10491"}}
+{"timestamp":1712086501.027251,"name":"online","context":{"idset":"10495"}}
+{"timestamp":1712086501.3691413,"name":"online","context":{"idset":"10490,10492,10501,10505"}}
+{"timestamp":1712086501.5244756,"name":"online","context":{"idset":"10486,10488,10500,10513-10514"}}
+{"timestamp":1712086501.6756198,"name":"online","context":{"idset":"10493,10515"}}
+{"timestamp":1712086502.0912397,"name":"online","context":{"idset":"10489,10520"}}
+{"timestamp":1712086502.2260578,"name":"online","context":{"idset":"10503-10504,10526"}}
+{"timestamp":1712086503.1044476,"name":"online","context":{"idset":"10499,10507,10523,10531"}}
+{"timestamp":1712086503.1070948,"name":"online","context":{"idset":"10487,10497,10508,10512,10517,10532,10537,10553,10578"}}
+{"timestamp":1712086503.1099639,"name":"online","context":{"idset":"10525,10528,10572"}}
+{"timestamp":1712086503.1138582,"name":"online","context":{"idset":"10502,10506,10527,10529-10530,10533,10546"}}
+{"timestamp":1712086503.1163898,"name":"online","context":{"idset":"10539,10543,10550,10555,10589"}}
+{"timestamp":1712086503.1456709,"name":"online","context":{"idset":"10498,10524,10536,10564,10566,10577,10580,10602"}}
+{"timestamp":1712086503.3100555,"name":"online","context":{"idset":"10516,10534,10547,10552,10556,10558,10561,10568,10590-10591,10594,10598"}}
+{"timestamp":1712086503.8691139,"name":"online","context":{"idset":"10535,10544,10548-10549,10551,10560,10574,10576,10579,10587,10593,10595,10600-10601,10603,10605,10611"}}
+{"timestamp":1712086504.7611115,"name":"online","context":{"idset":"10538,10540-10542,10554,10557,10559,10562-10563,10567,10569-10570,10573,10575,10581-10586,10588,10592,10596,10606-10608,10610,10612"}}
+{"timestamp":1712086504.7630987,"name":"online","context":{"idset":"10565,10571,10599,10609"}}
+{"timestamp":1712086504.7649651,"name":"online","context":{"idset":"10597,10604"}}
+{"timestamp":1712086504.7670012,"name":"online","context":{"idset":"10545"}}
+{"timestamp":1712086629.3178353,"name":"undrain","context":{"idset":"10550-10564"}}
+{"timestamp":1712086791.4119098,"name":"drain","context":{"idset":"10509,10518","reason":"CXI Issues --JRG","overwrite":1}}
+{"timestamp":1712086793.5077596,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1712086793.6059012,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1712086793.6176586,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1712086793.6791298,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1712086793.693141,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1712086793.7070887,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1712086793.716099,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1712086793.7288697,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1712086793.7497137,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1712086793.7577994,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1712086793.7673264,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1712086793.7692418,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1712086793.7762077,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1712086793.7843485,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1712086793.8025134,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1712086793.8104274,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1712086793.8205535,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1712086793.8283637,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1712086793.838073,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1712086793.8597507,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1712086793.869468,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1712086793.8776412,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1712086793.8856943,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1712086793.8936777,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1712086793.9016681,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1712086793.9097354,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1712086793.9178901,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1712086793.95471,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1712086793.9565682,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1712086793.9645343,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1712086793.966501,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1712086793.9777977,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1712086793.9797239,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1712086793.9816172,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1712086793.9834285,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1712086793.9852602,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1712086793.9965663,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1712086794.0078924,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1712086794.0288298,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1712086794.0506117,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1712086794.0770903,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1712086794.1070039,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1712086794.1087515,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1712086794.1105227,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1712086794.1123059,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1712086794.114043,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1712086794.1158628,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1712086794.1176665,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1712086794.1378191,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1712086794.1583529,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1712086794.1694429,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1712086794.1906023,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1712086794.2117329,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1712086794.213557,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1712086794.215291,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1712086794.2170587,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1712086794.2188501,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1712086794.2205803,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1712086794.2224569,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1712086794.2241704,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1712086794.2259581,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1712086794.2371895,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1712086794.2481194,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1712086794.3440588,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1712086794.3459117,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1712086794.3476655,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1712086794.3494186,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1712086794.3993301,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1712086794.4103868,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1712086794.4317825,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1712086794.5948393,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1712086794.6105607,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1712086794.6265075,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1712086794.628778,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1712086794.6310921,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1712086794.6452446,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1712086794.6759124,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1712086794.734875,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1712086794.7366619,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1712086794.8334911,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1712086796.1215405,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1712086796.1235394,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1712086796.125329,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1712086796.1271145,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1712086796.2015011,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1712086796.2980728,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1712086796.3139176,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1712086796.3280714,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1712086796.345818,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1712086797.4388263,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1712086797.4681633,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1712086797.4699569,"name":"offline","context":{"idset":"10188"}}
+{"timestamp":1712086797.4898305,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1712086797.491519,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1712086797.4931931,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1712086797.4948645,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1712086797.4965413,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1712086797.4982073,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1712086797.499893,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1712086797.5015776,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1712086797.5032344,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1712086797.5049076,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1712086797.5065708,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1712086797.5082359,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1712086797.5098984,"name":"offline","context":{"idset":"10213"}}
+{"timestamp":1712086797.5115705,"name":"offline","context":{"idset":"10214"}}
+{"timestamp":1712086797.5132542,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1712086797.5624237,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1712086797.6027236,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1712086797.6182475,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1712086797.6236434,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1712086797.6437957,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1712086800.9439588,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1712086800.9457142,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1712086831.8068576,"name":"drain","context":{"idset":"10510,10519","reason":"Bad partner node --JRG","overwrite":1}}
+{"timestamp":1712087312.0516427,"name":"drain","context":{"idset":"10157-10158","reason":"epilog failed for jobid fn4AXyJ6haF","overwrite":0}}
+{"timestamp":1712087815.0916734,"name":"drain","context":{"idset":"10871","reason":"broker was unresponsive"}}
+{"timestamp":1712087879.0871756,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1712088013.9111192,"name":"online","context":{"idset":"11158"}}
+{"timestamp":1712088014.0673859,"name":"online","context":{"idset":"11157,11162,11165"}}
+{"timestamp":1712088014.2086627,"name":"online","context":{"idset":"11159-11160,11168"}}
+{"timestamp":1712088014.5327411,"name":"online","context":{"idset":"11163,11194"}}
+{"timestamp":1712088014.6436906,"name":"online","context":{"idset":"11161,11180"}}
+{"timestamp":1712088014.8574488,"name":"online","context":{"idset":"11175-11176"}}
+{"timestamp":1712088015.132921,"name":"online","context":{"idset":"11171,11173,11195"}}
+{"timestamp":1712088015.2967844,"name":"online","context":{"idset":"11170,11183"}}
+{"timestamp":1712088015.5247455,"name":"online","context":{"idset":"11192,11199"}}
+{"timestamp":1712088015.6807973,"name":"online","context":{"idset":"11164,11172,11177,11190-11191"}}
+{"timestamp":1712088015.8242853,"name":"online","context":{"idset":"11169,11179,11182,11187,11196,11209,11211,11231"}}
+{"timestamp":1712088015.967391,"name":"online","context":{"idset":"11181,11186,11203,11218"}}
+{"timestamp":1712088016.1085644,"name":"online","context":{"idset":"11166,11178,11185,11193,11200-11201,11225"}}
+{"timestamp":1712088016.2491872,"name":"online","context":{"idset":"11167,11184,11188,11205,11219,11233,11251"}}
+{"timestamp":1712088016.3648117,"name":"online","context":{"idset":"11202,11216-11217,11220,11227"}}
+{"timestamp":1712088016.5116839,"name":"online","context":{"idset":"11189,11207-11208,11212,11214,11223,11243-11244,11247,11250,11252"}}
+{"timestamp":1712088016.6256797,"name":"online","context":{"idset":"11198,11206,11210,11215,11224,11238,11245"}}
+{"timestamp":1712088016.7377689,"name":"online","context":{"idset":"11221,11228-11229,11232,11234,11236-11237,11239,11248-11249"}}
+{"timestamp":1712088016.8509948,"name":"online","context":{"idset":"11222,11235,11242,11246"}}
+{"timestamp":1712088016.9648609,"name":"online","context":{"idset":"11226,11230"}}
+{"timestamp":1712088721.1281126,"name":"drain","context":{"idset":"843","reason":"broker was unresponsive"}}
+{"timestamp":1712088721.1284544,"name":"drain","context":{"idset":"844","reason":"broker was unresponsive"}}
+{"timestamp":1712088789.6261647,"name":"offline","context":{"idset":"843"}}
+{"timestamp":1712088790.4787905,"name":"offline","context":{"idset":"844"}}
+{"timestamp":1712089522.2106011,"name":"online","context":{"idset":"11204,11213"}}
+{"timestamp":1712089523.1388314,"name":"online","context":{"idset":"11197,11241"}}
+{"timestamp":1712089719.7270143,"name":"offline","context":{"idset":"849"}}
+{"timestamp":1712089719.7858484,"name":"offline","context":{"idset":"851"}}
+{"timestamp":1712089719.8020015,"name":"offline","context":{"idset":"852"}}
+{"timestamp":1712089719.9021389,"name":"offline","context":{"idset":"850"}}
+{"timestamp":1712090473.9734473,"name":"undrain","context":{"idset":"9973"}}
+{"timestamp":1712090829.5657814,"name":"online","context":{"idset":"11240"}}
+{"timestamp":1712090960.6537855,"name":"offline","context":{"idset":"11802"}}
+{"timestamp":1712092101.9103787,"name":"online","context":{"idset":"10085"}}
+{"timestamp":1712092149.3642948,"name":"online","context":{"idset":"10086"}}
+{"timestamp":1712092190.3451016,"name":"online","context":{"idset":"10095"}}
+{"timestamp":1712092542.2450297,"name":"online","context":{"idset":"11417"}}
+{"timestamp":1712092990.9561598,"name":"drain","context":{"idset":"11793","reason":"kernel panics","overwrite":1}}
+{"timestamp":1712092996.5080049,"name":"drain","context":{"idset":"11793","reason":"kernel panics - JRG","overwrite":1}}
+{"timestamp":1712093017.1467195,"name":"drain","context":{"idset":"11794","reason":"bad partner node - JRG","overwrite":1}}
+{"timestamp":1712093097.8897839,"name":"drain","context":{"idset":"11789","reason":"kernel panics - JRG","overwrite":1}}
+{"timestamp":1712093110.6747224,"name":"drain","context":{"idset":"11790","reason":"bad partner node - JRG","overwrite":1}}
+{"timestamp":1712093177.2344193,"name":"drain","context":{"idset":"11587","reason":"kernel panics - JRG","overwrite":1}}
+{"timestamp":1712093194.2606094,"name":"drain","context":{"idset":"11588","reason":"bad partner node - JRG","overwrite":1}}
+{"timestamp":1712093550.5135028,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1712093550.6066678,"name":"offline","context":{"idset":"11790"}}
+{"timestamp":1712093863.2931864,"name":"undrain","context":{"idset":"11216"}}
+{"timestamp":1712093958.2443516,"name":"undrain","context":{"idset":"11240"}}
+{"timestamp":1712094140.4139462,"name":"drain","context":{"idset":"10357-10468","reason":"--reason HP Diags","overwrite":0}}
+{"timestamp":1712094315.2472501,"name":"undrain","context":{"idset":"10096"}}
+{"timestamp":1712094618.5177488,"name":"online","context":{"idset":"10096"}}
+{"timestamp":1712094718.9294119,"name":"undrain","context":{"idset":"10085"}}
+{"timestamp":1712094734.2452819,"name":"undrain","context":{"idset":"10086"}}
+{"timestamp":1712094746.094012,"name":"undrain","context":{"idset":"10095"}}
+{"timestamp":1712094905.0890455,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1712094978.8228164,"name":"drain","context":{"idset":"10906","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712094989.8756378,"name":"drain","context":{"idset":"10873","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712094992.5093184,"name":"drain","context":{"idset":"10883","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712094993.9039888,"name":"drain","context":{"idset":"10884","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712095007.7999616,"name":"drain","context":{"idset":"10874","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712095015.6494093,"name":"drain","context":{"idset":"10905","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712095025.0662155,"name":"drain","context":{"idset":"11173","reason":"epilog failed for jobid fnFN1DHfSM5","overwrite":0}}
+{"timestamp":1712096188.4627929,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1711985590.1298366,"reason":"broker was unresponsive"},"2":{"timestamp":1711985590.1299634,"reason":"broker was unresponsive"},"3":{"timestamp":1711985590.129997,"reason":"broker was unresponsive"},"4":{"timestamp":1711985590.1300287,"reason":"broker was unresponsive"},"5":{"timestamp":1711985590.130055,"reason":"broker was unresponsive"},"6":{"timestamp":1711985590.1301,"reason":"broker was unresponsive"},"7":{"timestamp":1711985590.1301718,"reason":"broker was unresponsive"},"8":{"timestamp":1711985590.1302528,"reason":"broker was unresponsive"},"9":{"timestamp":1711985590.1303506,"reason":"broker was unresponsive"},"10":{"timestamp":1711985590.1304929,"reason":"broker was unresponsive"},"11":{"timestamp":1711985590.1305854,"reason":"broker was unresponsive"},"12":{"timestamp":1711985590.2635758,"reason":"broker was unresponsive"},"13":{"timestamp":1711986176.1277797,"reason":"broker was unresponsive"},"14":{"timestamp":1711986176.127861,"reason":"broker was unresponsive"},"15":{"timestamp":1711986172.126236,"reason":"broker was unresponsive"},"16":{"timestamp":1711986176.1279109,"reason":"broker was unresponsive"},"17":{"timestamp":1711986172.226109,"reason":"broker was unresponsive"},"18":{"timestamp":1711986176.1279769,"reason":"broker was unresponsive"},"19":{"timestamp":1711986174.2265673,"reason":"broker was unresponsive"},"20":{"timestamp":1711986176.1280706,"reason":"broker was unresponsive"},"21":{"timestamp":1711986176.1281965,"reason":"broker was unresponsive"},"22":{"timestamp":1711986176.128314,"reason":"broker was unresponsive"},"23":{"timestamp":1711986176.1284461,"reason":"broker was unresponsive"},"24":{"timestamp":1711986176.2268984,"reason":"broker was unresponsive"},"25":{"timestamp":1711986712.1284616,"reason":"broker was unresponsive"},"26":{"timestamp":1711986712.1285477,"reason":"broker was unresponsive"},"27":{"timestamp":1711986712.1286163,"reason":"broker was unresponsive"},"28":{"timestamp":1711986712.1287522,"reason":"broker was unresponsive"},"29":{"timestamp":1711986712.1288626,"reason":"broker was unresponsive"},"30":{"timestamp":1711986712.1289904,"reason":"broker was unresponsive"},"31":{"timestamp":1711986712.1291039,"reason":"broker was unresponsive"},"32":{"timestamp":1711986712.1292276,"reason":"broker was unresponsive"},"33":{"timestamp":1711986712.129328,"reason":"broker was unresponsive"},"34":{"timestamp":1711986712.1294577,"reason":"broker was unresponsive"},"35":{"timestamp":1711986712.1295605,"reason":"broker was unresponsive"},"36":{"timestamp":1711986712.2507603,"reason":"broker was unresponsive"},"37":{"timestamp":1711987342.1267064,"reason":"broker was unresponsive"},"38":{"timestamp":1711987342.1267867,"reason":"broker was unresponsive"},"39":{"timestamp":1711987342.1268172,"reason":"broker was unresponsive"},"40":{"timestamp":1711987342.1268439,"reason":"broker was unresponsive"},"41":{"timestamp":1711987342.1269064,"reason":"broker was unresponsive"},"42":{"timestamp":1711987342.1269703,"reason":"broker was unresponsive"},"43":{"timestamp":1711987342.127022,"reason":"broker was unresponsive"},"44":{"timestamp":1711987342.1270769,"reason":"broker was unresponsive"},"45":{"timestamp":1711987342.1271369,"reason":"broker was unresponsive"},"46":{"timestamp":1711987342.1271925,"reason":"broker was unresponsive"},"47":{"timestamp":1711987342.1272507,"reason":"broker was unresponsive"},"48":{"timestamp":1711987342.2340004,"reason":"broker was unresponsive"},"61":{"timestamp":1711668444.9627461,"reason":"broker was unresponsive"},"62":{"timestamp":1711668506.9728317,"reason":"broker was unresponsive"},"63":{"timestamp":1711668445.0625052,"reason":"broker was unresponsive"},"64":{"timestamp":1711668506.9729121,"reason":"broker was unresponsive"},"65":{"timestamp":1711668438.9624472,"reason":"broker was unresponsive"},"66":{"timestamp":1711668506.9729731,"reason":"broker was unresponsive"},"67":{"timestamp":1711668439.0630105,"reason":"broker was unresponsive"},"68":{"timestamp":1711668506.9730346,"reason":"broker was unresponsive"},"69-76":{"timestamp":1711646931.6480982,"reason":""},"77":{"timestamp":1711668526.9621451,"reason":"broker was unresponsive"},"78":{"timestamp":1711668462.9610252,"reason":"broker was unresponsive"},"79":{"timestamp":1711668526.96223,"reason":"broker was unresponsive"},"80":{"timestamp":1711668457.0617182,"reason":"broker was unresponsive"},"81":{"timestamp":1711668526.9622741,"reason":"broker was unresponsive"},"82":{"timestamp":1711668462.9611192,"reason":"broker was unresponsive"},"83":{"timestamp":1711668527.0620673,"reason":"broker was unresponsive"},"84":{"timestamp":1711668463.0615458,"reason":"broker was unresponsive"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"246":{"timestamp":1711982548.2285702,"reason":"broker was unresponsive"},"348":{"timestamp":1711576490.2548447,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"431":{"timestamp":1711096934.358676,"reason":"nodediag failed pci"},"445-468,541-564":{"timestamp":1711560189.6111379,"reason":"CDU work TB"},"637":{"timestamp":1711642252.1616328,"reason":"broker was unresponsive"},"638":{"timestamp":1711642252.1617086,"reason":"broker was unresponsive"},"639":{"timestamp":1711642252.161751,"reason":"broker was unresponsive"},"640":{"timestamp":1711642252.1617904,"reason":"broker was unresponsive"},"641":{"timestamp":1711642252.1618268,"reason":"broker was unresponsive"},"642":{"timestamp":1711642252.1618621,"reason":"broker was unresponsive"},"643":{"timestamp":1711642252.1618974,"reason":"broker was unresponsive"},"644":{"timestamp":1711642252.1619322,"reason":"broker was unresponsive"},"646":{"timestamp":1711586857.201689,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"647":{"timestamp":1711587136.0002518,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"648":{"timestamp":1711587197.7047591,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"649":{"timestamp":1711587211.9606316,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"650":{"timestamp":1711587232.1780922,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"651":{"timestamp":1711587244.2040467,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"652":{"timestamp":1711587255.7559283,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"653":{"timestamp":1711642252.1619685,"reason":"broker was unresponsive"},"654":{"timestamp":1711642252.16201,"reason":"broker was unresponsive"},"655":{"timestamp":1711642252.162055,"reason":"broker was unresponsive"},"656":{"timestamp":1711642252.1620932,"reason":"broker was unresponsive"},"657":{"timestamp":1711642252.1621311,"reason":"broker was unresponsive"},"658":{"timestamp":1711642252.1621685,"reason":"broker was unresponsive"},"659":{"timestamp":1711642252.1622062,"reason":"broker was unresponsive"},"660":{"timestamp":1711642252.1622446,"reason":"broker was unresponsive"},"661":{"timestamp":1711642252.1622829,"reason":"broker was unresponsive"},"662":{"timestamp":1711642252.1623201,"reason":"broker was unresponsive"},"663":{"timestamp":1711642252.1623569,"reason":"broker was unresponsive"},"664":{"timestamp":1711642252.162394,"reason":"broker was unresponsive"},"665":{"timestamp":1711642252.1624658,"reason":"broker was unresponsive"},"666":{"timestamp":1711642252.1625173,"reason":"broker was unresponsive"},"667":{"timestamp":1711642258.156193,"reason":"broker was unresponsive"},"668":{"timestamp":1711642252.1625559,"reason":"broker was unresponsive"},"669":{"timestamp":1711642252.1625948,"reason":"broker was unresponsive"},"670":{"timestamp":1711642252.1626337,"reason":"broker was unresponsive"},"671":{"timestamp":1711642252.162673,"reason":"broker was unresponsive"},"672":{"timestamp":1711642252.1627119,"reason":"broker was unresponsive"},"673":{"timestamp":1711642252.1627512,"reason":"broker was unresponsive"},"674":{"timestamp":1711642252.1627915,"reason":"broker was unresponsive"},"675":{"timestamp":1711642252.1628311,"reason":"broker was unresponsive"},"676":{"timestamp":1711642252.1628704,"reason":"broker was unresponsive"},"677":{"timestamp":1711642258.1562793,"reason":"broker was unresponsive"},"678":{"timestamp":1711642258.156333,"reason":"broker was unresponsive"},"679":{"timestamp":1711642252.1629126,"reason":"broker was unresponsive"},"680":{"timestamp":1711642252.1629534,"reason":"broker was unresponsive"},"681":{"timestamp":1711642258.1563835,"reason":"broker was unresponsive"},"682":{"timestamp":1711642252.1629946,"reason":"broker was unresponsive"},"683":{"timestamp":1711642252.1630385,"reason":"broker was unresponsive"},"684":{"timestamp":1711642252.5597043,"reason":"broker was unresponsive"},"685":{"timestamp":1711604960.2562068,"reason":"broker was unresponsive"},"687":{"timestamp":1711604954.2552111,"reason":"broker was unresponsive"},"688":{"timestamp":1711606062.2558303,"reason":"broker was unresponsive"},"689":{"timestamp":1711604580.2552168,"reason":"broker was unresponsive"},"691":{"timestamp":1711604958.2564192,"reason":"broker was unresponsive"},"692":{"timestamp":1711606406.1556187,"reason":"broker was unresponsive"},"693":{"timestamp":1711636838.2555192,"reason":"broker was unresponsive"},"695":{"timestamp":1711604326.2549248,"reason":"broker was unresponsive"},"696":{"timestamp":1711606406.256216,"reason":"broker was unresponsive"},"697":{"timestamp":1711604392.2562771,"reason":"broker was unresponsive"},"698":{"timestamp":1711605716.2557297,"reason":"broker was unresponsive"},"699":{"timestamp":1711604948.2558951,"reason":"broker was unresponsive"},"700":{"timestamp":1711606868.2557678,"reason":"broker was unresponsive"},"701":{"timestamp":1711604340.2561386,"reason":"broker was unresponsive"},"702":{"timestamp":1711606408.2602317,"reason":"broker was unresponsive"},"703":{"timestamp":1711604918.2557685,"reason":"broker was unresponsive"},"704":{"timestamp":1711604590.2548223,"reason":"broker was unresponsive"},"705":{"timestamp":1711604968.2570915,"reason":"broker was unresponsive"},"707":{"timestamp":1711604908.2558758,"reason":"broker was unresponsive"},"686,690,694,706,708":{"timestamp":1711615908.1602454,"reason":"epilog failed for jobid fm9HsYwFjjd"},"709":{"timestamp":1711408620.9668899,"reason":"Cabinet power work needed -jrg"},"710":{"timestamp":1711409910.967391,"reason":"Cabinet power work needed -jrg"},"711":{"timestamp":1711409098.9674611,"reason":"Cabinet power work needed -jrg"},"712":{"timestamp":1711411606.9678445,"reason":"Cabinet power work needed -jrg"},"713":{"timestamp":1711408618.8669355,"reason":"Cabinet power work needed -jrg"},"715":{"timestamp":1711409132.9680476,"reason":"Cabinet power work needed -jrg"},"714,716":{"timestamp":1711564709.7804377,"reason":"Cabinet power work needed -jrg"},"718":{"timestamp":1711642210.1588371,"reason":"broker was unresponsive"},"719":{"timestamp":1711642210.1589563,"reason":"broker was unresponsive"},"720":{"timestamp":1711642210.1590128,"reason":"broker was unresponsive"},"721":{"timestamp":1711642210.1590643,"reason":"broker was unresponsive"},"722":{"timestamp":1711642210.1590991,"reason":"broker was unresponsive"},"723":{"timestamp":1711642210.1591821,"reason":"broker was unresponsive"},"724":{"timestamp":1711642210.1592195,"reason":"broker was unresponsive"},"725":{"timestamp":1711593222.2542431,"reason":"broker was unresponsive"},"726":{"timestamp":1711593312.2549782,"reason":"broker was unresponsive"},"727":{"timestamp":1711593332.2552569,"reason":"broker was unresponsive"},"728":{"timestamp":1711593356.2547033,"reason":"broker was unresponsive"},"729":{"timestamp":1711593414.254441,"reason":"broker was unresponsive"},"730":{"timestamp":1711593462.2552798,"reason":"broker was unresponsive"},"731":{"timestamp":1711593510.2550235,"reason":"broker was unresponsive"},"732":{"timestamp":1711593530.2558599,"reason":"broker was unresponsive"},"733":{"timestamp":1711642210.1592536,"reason":"broker was unresponsive"},"734":{"timestamp":1711486000.9676955,"reason":"broker was unresponsive"},"735":{"timestamp":1711642210.1592872,"reason":"broker was unresponsive"},"736":{"timestamp":1711642210.1593204,"reason":"broker was unresponsive"},"737":{"timestamp":1711642210.1593742,"reason":"broker was unresponsive"},"738":{"timestamp":1711642210.1594355,"reason":"broker was unresponsive"},"739":{"timestamp":1711642210.1594796,"reason":"broker was unresponsive"},"740":{"timestamp":1711642210.159517,"reason":"broker was unresponsive"},"741":{"timestamp":1711642210.1595554,"reason":"broker was unresponsive"},"742":{"timestamp":1711642210.1595943,"reason":"broker was unresponsive"},"743":{"timestamp":1711642210.1596365,"reason":"broker was unresponsive"},"744":{"timestamp":1711642210.1596746,"reason":"broker was unresponsive"},"745":{"timestamp":1711642210.1597111,"reason":"broker was unresponsive"},"746":{"timestamp":1711506258.2527435,"reason":"broker was unresponsive"},"747":{"timestamp":1711642210.1597559,"reason":"broker was unresponsive"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"},"749":{"timestamp":1711642210.1598115,"reason":"broker was unresponsive"},"750":{"timestamp":1711642210.1598501,"reason":"broker was unresponsive"},"751":{"timestamp":1711642210.159888,"reason":"broker was unresponsive"},"752":{"timestamp":1711642210.1599293,"reason":"broker was unresponsive"},"753":{"timestamp":1711642210.1599746,"reason":"broker was unresponsive"},"754":{"timestamp":1711642210.1600144,"reason":"broker was unresponsive"},"755":{"timestamp":1711642210.1600802,"reason":"broker was unresponsive"},"756":{"timestamp":1711642210.5039413,"reason":"broker was unresponsive"},"789-796":{"timestamp":1711461777.1299369,"reason":""},"809":{"timestamp":1711667251.2030878,"reason":"broker was unresponsive"},"810":{"timestamp":1711460371.1431019,"reason":""},"841":{"timestamp":1712084137.9055309,"reason":""},"843":{"timestamp":1712088721.1281126,"reason":"broker was unresponsive"},"844":{"timestamp":1712088721.1284544,"reason":"broker was unresponsive"},"874":{"timestamp":1711668507.0706048,"reason":"broker was unresponsive"},"875":{"timestamp":1711668503.0621645,"reason":"broker was unresponsive"},"883-884":{"timestamp":1711734121.1560712,"reason":""},"965":{"timestamp":1711663706.9630723,"reason":"broker was unresponsive"},"966":{"timestamp":1711663706.9631405,"reason":"broker was unresponsive"},"967":{"timestamp":1711663706.963171,"reason":"broker was unresponsive"},"968":{"timestamp":1711663706.9631999,"reason":"broker was unresponsive"},"10051":{"timestamp":1711683212.9615688,"reason":"broker was unresponsive"},"10052":{"timestamp":1711683213.0621202,"reason":"broker was unresponsive"},"10075":{"timestamp":1711651454.7350454,"reason":"nodediag failed amdapu"},"10076":{"timestamp":1711653832.9647021,"reason":"broker was unresponsive"},"10077":{"timestamp":1711598424.2559545,"reason":"broker was unresponsive"},"10078":{"timestamp":1711598420.2553101,"reason":"broker was unresponsive"},"10084":{"timestamp":1711651436.1179669,"reason":"nodediag failed amdapu"},"10102":{"timestamp":1711656464.7456715,"reason":"rvs dumped core"},"10120-10129":{"timestamp":1712079171.5353351,"reason":""},"10151":{"timestamp":1711734764.9003315,"reason":"nodediag failed clocksource pci"},"10152":{"timestamp":1711744485.1922078,"reason":"broker was unresponsive"},"10154":{"timestamp":1711600478.1543531,"reason":"broker was unresponsive"},"10156":{"timestamp":1711600478.2542264,"reason":"broker was unresponsive"},"10157-10158":{"timestamp":1712087312.0516427,"reason":"epilog failed for jobid fn4AXyJ6haF"},"10159":{"timestamp":1711744485.19349,"reason":"broker was unresponsive"},"10160":{"timestamp":1711744485.1935451,"reason":"broker was unresponsive"},"10162":{"timestamp":1711742687.1360803,"reason":"nodediag failed cxi"},"10183":{"timestamp":1711600556.2563355,"reason":"broker was unresponsive"},"10357-10468":{"timestamp":1712094140.4139462,"reason":"--reason HP Diags"},"10509,10518":{"timestamp":1712086791.4119098,"reason":"CXI Issues --JRG"},"10510,10519":{"timestamp":1712086831.8068576,"reason":"Bad partner node --JRG"},"10741":{"timestamp":1711686071.0616689,"reason":"dmi nodediag failed dmi"},"10757":{"timestamp":1711733680.9899652,"reason":"nodediag failed clocksource dmi pci"},"10786":{"timestamp":1711733695.434325,"reason":"nodediag failed clocksource dmi pci"},"10796":{"timestamp":1711689956.8180678,"reason":"nodediag failed clocksource dmi pci"},"10804":{"timestamp":1711733682.6906681,"reason":"nodediag failed clocksource dmi pci"},"10832":{"timestamp":1711733694.9659727,"reason":"nodediag failed clocksource dmi pci"},"10843":{"timestamp":1711733726.5102046,"reason":"nodediag failed clocksource dmi pci"},"10865":{"timestamp":1711733747.2219884,"reason":"nodediag failed clocksource dmi pci"},"10871":{"timestamp":1712087815.0916734,"reason":"broker was unresponsive"},"10873":{"timestamp":1712094989.8756378,"reason":"nodediag failed cxi"},"10874":{"timestamp":1712095007.7999616,"reason":"nodediag failed cxi"},"10883":{"timestamp":1712094992.5093184,"reason":"nodediag failed cxi"},"10884":{"timestamp":1712094993.9039888,"reason":"nodediag failed cxi"},"10905":{"timestamp":1712095015.6494093,"reason":"nodediag failed cxi"},"10906":{"timestamp":1712094978.8228164,"reason":"nodediag failed cxi"},"11125":{"timestamp":1712079043.0907531,"reason":"broker was unresponsive"},"11126":{"timestamp":1711126278.6191533,"reason":"ama fail"},"11127":{"timestamp":1712079046.9967983,"reason":"broker was unresponsive"},"11128":{"timestamp":1712079046.9969132,"reason":"broker was unresponsive"},"11129":{"timestamp":1712079046.9969907,"reason":"broker was unresponsive"},"11130":{"timestamp":1712079046.9970613,"reason":"broker was unresponsive"},"11131":{"timestamp":1712079046.9971497,"reason":"broker was unresponsive"},"11132":{"timestamp":1712079046.9972458,"reason":"broker was unresponsive"},"11133":{"timestamp":1712079046.9973373,"reason":"broker was unresponsive"},"11134":{"timestamp":1712079046.9974139,"reason":"broker was unresponsive"},"11135":{"timestamp":1712079046.9975028,"reason":"broker was unresponsive"},"11136":{"timestamp":1712079046.9975903,"reason":"broker was unresponsive"},"11137":{"timestamp":1712079046.9976768,"reason":"broker was unresponsive"},"11138":{"timestamp":1712079046.9977582,"reason":"broker was unresponsive"},"11139":{"timestamp":1712079046.9978447,"reason":"broker was unresponsive"},"11140":{"timestamp":1712079046.997932,"reason":"broker was unresponsive"},"11141":{"timestamp":1712079046.9980164,"reason":"broker was unresponsive"},"11142":{"timestamp":1712079046.9981048,"reason":"broker was unresponsive"},"11143":{"timestamp":1712079046.9981937,"reason":"broker was unresponsive"},"11144":{"timestamp":1712079046.9982798,"reason":"broker was unresponsive"},"11145":{"timestamp":1712079046.9983644,"reason":"broker was unresponsive"},"11146":{"timestamp":1712079046.998455,"reason":"broker was unresponsive"},"11147":{"timestamp":1712079046.9985433,"reason":"broker was unresponsive"},"11148":{"timestamp":1712079046.9986298,"reason":"broker was unresponsive"},"11151":{"timestamp":1712079046.9987171,"reason":"broker was unresponsive"},"11152":{"timestamp":1712079046.9988039,"reason":"broker was unresponsive"},"11153":{"timestamp":1712079046.9988978,"reason":"broker was unresponsive"},"11155":{"timestamp":1712079046.9989817,"reason":"broker was unresponsive"},"11156":{"timestamp":1712079047.2586949,"reason":"broker was unresponsive"},"11173":{"timestamp":1712095025.0662155,"reason":"epilog failed for jobid fnFN1DHfSM5"},"11317":{"timestamp":1711326792.8705738,"reason":"epilog failed for jobid fka2yuNfWCo"},"11319":{"timestamp":1712073755.5918038,"reason":"broker was unresponsive"},"11320":{"timestamp":1712073755.5919735,"reason":"broker was unresponsive"},"11321":{"timestamp":1712073755.6913724,"reason":"broker was unresponsive"},"11322":{"timestamp":1712073761.5950892,"reason":"broker was unresponsive"},"11323":{"timestamp":1712073761.5952139,"reason":"broker was unresponsive"},"11324":{"timestamp":1712073761.595295,"reason":"broker was unresponsive"},"11325":{"timestamp":1712073761.5953906,"reason":"broker was unresponsive"},"11326":{"timestamp":1712073761.5954952,"reason":"broker was unresponsive"},"11327":{"timestamp":1712073761.5955708,"reason":"broker was unresponsive"},"11328":{"timestamp":1712073761.5956366,"reason":"broker was unresponsive"},"11329":{"timestamp":1712073761.5957003,"reason":"broker was unresponsive"},"11330":{"timestamp":1712073761.5957878,"reason":"broker was unresponsive"},"11331":{"timestamp":1712073761.5958869,"reason":"broker was unresponsive"},"11332":{"timestamp":1712073761.7011204,"reason":"broker was unresponsive"},"11375-11382":{"timestamp":1712079203.2770023,"reason":"draining to run on-node diags - wendy"},"11417":{"timestamp":1711743415.7833021,"reason":"epilog failed for jobid fmKGqZRsvpP"},"11460":{"timestamp":1711123247.7819526,"reason":"ama fail"},"11517":{"timestamp":1711659950.9614942,"reason":"broker was unresponsive"},"11527":{"timestamp":1712043086.7846026,"reason":"broker was unresponsive"},"11536":{"timestamp":1711659951.0613308,"reason":"broker was unresponsive"},"10137,10140,11566":{"timestamp":1711663526.290566,"reason":"core dumps -KK"},"11569":{"timestamp":1711946370.2288194,"reason":"broker was unresponsive"},"11587":{"timestamp":1711663526.290566,"reason":"kernel panics - JRG"},"11588":{"timestamp":1712093194.2606094,"reason":"bad partner node - JRG"},"11592":{"timestamp":1712026940.6850164,"reason":"broker was unresponsive"},"11593":{"timestamp":1711762170.0976398,"reason":""},"11603":{"timestamp":1712026940.6851528,"reason":"broker was unresponsive"},"11609":{"timestamp":1712026940.6852219,"reason":"broker was unresponsive"},"11612":{"timestamp":1712028840.7845249,"reason":"broker was unresponsive"},"11619":{"timestamp":1712026940.6852841,"reason":"broker was unresponsive"},"11631":{"timestamp":1712026940.6853518,"reason":"broker was unresponsive"},"11633":{"timestamp":1712026940.7842064,"reason":"broker was unresponsive"},"11670":{"timestamp":1711726427.2945235,"reason":"epilog failed for jobid fmLXtp1qajy"},"11765-11780":{"timestamp":1711993308.7730825,"reason":"Diags Running"},"11784":{"timestamp":1711725755.0632687,"reason":"Running Diags - Rabbits Off"},"11790":{"timestamp":1712004602.484952,"reason":"bad partner node - JRG"},"11789,11793":{"timestamp":1712004602.484952,"reason":"kernel panics - JRG"},"11794":{"timestamp":1711725767.0656931,"reason":"bad partner node - JRG"},"11781-11783,11785-11788,11791-11792,11795-11796":{"timestamp":1712004602.484952,"reason":"Running Diags - Rabbits Off"},"11803":{"timestamp":1711668961.0618374,"reason":"Running Diags - Rabbits Off"},"11797-11801,11804-11811":{"timestamp":1712004859.9543238,"reason":"Running Diags - Rabbits Off"},"11812":{"timestamp":1712004514.6037207,"reason":"Running Diags - Rabbits Off"},"11813-11828":{"timestamp":1712005088.7732615,"reason":"Running Diags - Rabbits Off"},"11829-11844":{"timestamp":1712068080.9443812,"reason":"Running Diags - Rabbits Off"},"11845-11859":{"timestamp":1712068141.3627269,"reason":"Running Diags - Rabbits Off"},"11860":{"timestamp":1711727735.0622723,"reason":"Running Diags - Rabbits Off"},"11861-11876":{"timestamp":1712068186.5559592,"reason":"Running Diags - Rabbits Off"}},"online":"","exclude":"0"}}
+{"timestamp":1712096188.4816427,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1712096237.6941073,"name":"online","context":{"idset":"0"}}
+{"timestamp":1712096239.5381815,"name":"online","context":{"idset":"10045,10053,10063,10077,10082,10775,10789,10836,10852,10858,11128,11592,11619,11803,11828,11874"}}
+{"timestamp":1712096239.6405292,"name":"online","context":{"idset":"10039-10040,10054,10059,10071-10072,10596,10773,10777,10780,10782,10785,10787,10793,10807-10808,10818,10820,10825,10829,10834,10857,10859-10861,11130,11132-11133,11137-11138,11566,11800"}}
+{"timestamp":1712096239.7445307,"name":"online","context":{"idset":"841,10037-10038,10041,10043-10044,10047-10050,10055-10058,10060-10062,10064-10070,10073-10076,10078-10081,10083,10748-10753,10758,10771-10772,10781,10792,10797,10800,10802,10809,10814,10819,10821,10823,10830,10838,10841,10856,11125,11127,11129,11131,11134-11136,11139-11140,11603,11609,11631,11812,11836,11841-11842"}}
+{"timestamp":1712096239.8861721,"name":"online","context":{"idset":"10042,10046,10837,10840,10868,11417,11522,11860"}}
+{"timestamp":1712096240.0427325,"name":"online","context":{"idset":"882,10003,10505,10544,10744,11299,11308,11344,11351-11352,11389,11414,11458,11654,11679,11714,11808"}}
+{"timestamp":1712096240.1476722,"name":"online","context":{"idset":"117,881,973,10524,10531,10590,10743,10767,10917,11179,11182,11227,11252,11305,11347,11349,11381,11421,11442,11449,11487,11495,11499,11567,11580,11599,11601,11622,11639-11640,11651,11677,11696,11712,11731,11734,11738-11740,11787,11809,11827"}}
+{"timestamp":1712096240.2525427,"name":"online","context":{"idset":"109,123,254,869-870,9975,10018,10474,10476,10487,10506,10543,10556,10578,10582,10747,10768,10882,10892,10908,10941,10951-10952,10972-10973,10983,11195,11206,11220,11234,11241,11265,11272,11281,11304,11346,11377,11425,11441,11444-11445,11464,11472,11475-11476,11494,11537-11540,11554,11563-11564,11572,11579,11607,11610,11620,11629,11667,11682,11718,11788,11824,11846,11851,11863"}}
+{"timestamp":1712096240.3989115,"name":"online","context":{"idset":"9976,10004,10488,10525,10529,10557,10570,10591,10599,10761,10764,10913,10920,10981,10996,11166,11190,11247,11296,11303,11311,11340,11350,11376,11386,11392,11394,11409,11411,11415,11431,11452,11461,11489-11490,11500,11502,11505,11509,11523-11525,11531,11535,11541,11546,11551,11553,11556,11562,11591,11594-11595,11598,11611,11613,11617-11618,11621,11623,11627,11655,11658,11665,11672,11676,11686,11692,11706,11710,11723,11732,11782,11830,11844,11876"}}
+{"timestamp":1712096240.5044332,"name":"online","context":{"idset":"29,10019,10034,10520,10545,10606,10766,10877,10914,10943,10946,10960,10984,11159,11162,11175,11214,11223,11268,11289,11341-11342,11382,11402,11404,11407,11416,11423,11435,11447,11450,11456,11467,11470,11488,11497-11498,11521,11526,11532,11557,11561,11581,11590,11597,11606,11615,11624,11673,11678,11683,11702,11715-11717,11722,11727-11729,11785,11795-11796,11806,11832-11833,11837"}}
+{"timestamp":1712096240.6109982,"name":"online","context":{"idset":"246,631-633,972,9989,10001,10011,10021,10026,10483,10507,10517,10533,10572,10587,10600,10608,10889,10897,10930,11160,11185-11186,11191,11198,11207,11228,11287,11343,11348,11353,11399,11405,11408,11410,11412,11419,11429,11436,11438,11463,11469,11478,11480-11481,11485,11492,11506,11513-11514,11516,11558,11560,11570-11571,11576,11602,11614,11625-11626,11630,11641,11647,11653,11657,11660,11663,11681,11694,11705,11707,11709,11720-11721,11781,11783,11792,11797-11798,11804,11810,11822,11868,11879,11881"}}
+{"timestamp":1712096240.7250965,"name":"online","context":{"idset":"46,89,99,101,105,107,113,115,119,136-137,145,148,181,195,197,263,270,272,293,310,320,324,330,334,345,349,352,360-361,401,408,410,423,433,435,482,499,504,510,515,575,581,587,605,625,9986,9996,10009,10014,10022,10025,10033,10477,10495,10497,10512,10526,10550,10555,10558,10567,10573,10576,10602,10604,10754-10756,10769-10770,10774,10776,10778-10779,10783-10784,10788,10791,10794-10795,10798-10799,10801,10803,10805-10806,10810-10813,10815-10817,10822,10824,10826-10828,10831,10835,10842,10844,10846-10848,10850-10851,10853-10854,10862-10863,10879,10881,10884-10885,10902,10905,10915,10970-10971,11167,11181,11183-11184,11196,11199,11239,11244,11250-11251,11256,11269,11277,11280,11286,11302,11310,11406,11413,11420,11424,11426-11428,11443,11453-11454,11459,11462,11474,11491,11493,11501,11508,11511,11519,11529,11534,11542-11544,11548,11565,11577-11578,11582-11584,11586,11600,11608,11634,11637,11642-11644,11648,11656,11659,11662,11671,11674,11701,11704,11708,11735-11736,11741,11786,11805,11818,11820-11821,11831,11838,11854,11856,11870"}}
+{"timestamp":1712096240.83867,"name":"online","context":{"idset":"103,108,111-112,114,116,120,122,124-126,128-129,134,138-144,149,153,156,158-160,162-166,168,170,172-173,175-178,180,182,185-186,189-192,196,198-201,204-215,219-220,222,224-225,227-228,231-232,234-244,255-256,258-259,262,265-266,276,278,283-285,295,297-300,304,307,309,311-314,316-317,319,323,325,327,331,333,335-337,339,342-344,350,354,356-357,359,362,364,368,370,373-381,383-385,388-389,391,393-396,398-399,402,407,409,412-416,419,421-422,424-430,432,434,437,439-440,443-444,470-471,473-475,477-481,483-484,488-492,494-498,500-502,505,509,511-512,514,516-517,519-521,523-535,537,539-540,566-569,571-574,577-578,582-586,589-592,596-600,602-604,606,608-610,613-614,616-618,620-621,623,626-627,629-630,634-635,975-976,979,9983,9990,10013,10024,10027,10475,10490-10491,10500-10501,10504,10513,10515,10523,10527,10538,10542,10580,10584,10588,10601,10790,10833,10839,10845,10849,10855,10869-10870,10873,10880,10888,10898,10906,10921,10933,10947,10954,10959,10975,10988,11170,11187-11188,11192,11194,11200-11201,11208,11210,11217,11245,11253,11258,11261-11262,11267,11273,11282,11284-11285,11293,11297,11312,11335,11337,11390,11418,11430,11433,11437,11440,11446,11448,11451,11455,11468,11482-11483,11496,11510,11512,11520,11528,11530,11547,11549,11552,11559,11575,11585,11605,11628,11632,11635-11636,11645,11680,11703,11807,11814-11815,11819,11825,11829,11834,11845,11867,11871-11872,11892"}}
+{"timestamp":1712096240.9527776,"name":"online","context":{"idset":"87,90-92,95,98,100,102,104,106,110,127,130-133,135,146-147,150-152,154-155,157,161,167,169,171,174,179,183-184,187,193-194,202-203,216,218,226,229-230,233,245,247-253,257,260-261,264,267-269,271,273-275,277,279-282,286-292,294,296,301-303,305-306,308,315,318,321-322,328-329,332,338,340-341,346-347,351,353,355,358,363,365-367,369,371-372,382,386-387,390,392,397,400,403-406,411,417-418,420,436,438,441-442,469,472,476,485-487,493,503,506-508,513,518,522,536,538,570,576,579-580,588,593-595,601,607,611-612,615,619,622,624,628,977-978,9974,9979,9992,10000,10012,10020,10096,10471-10472,10481,10484-10485,10489,10493,10496,10503,10511,10516,10528,10532,10534,10536-10537,10539-10540,10549,10551,10560,10565-10566,10568,10574,10581,10589,10592-10593,10598,10610,10876,10883,10890,10900,10904,10907,10919,10923-10924,10927,10929,10932,10937,10949,10957-10958,10961-10962,10965,10979-10980,10982,10987,10994-10995,11161,11163,11168-11169,11172,11176,11178,11180,11202-11203,11205,11218-11219,11221,11224,11226,11231-11233,11235,11237,11240,11242,11248-11249,11254,11260,11264,11266,11271,11274,11276,11278-11279,11288,11295,11307,11309,11333-11334,11336,11338-11339,11345,11359,11374,11432,11434,11477,11486,11503-11504,11515,11518,11545,11555,11568,11573-11574,11596,11616,11638,11649,11661,11664,11666,11675,11684,11719,11730,11737,11745,11799,11801,11811,11823,11826,11840,11848-11850,11853,11859,11861,11866,11869,11887"}}
+{"timestamp":1712096241.0667248,"name":"online","context":{"idset":"85-86,88,93-94,96-97,118,565,969,971,974,980,9978,9985,9987,9993-9994,9997,9999,10002,10005,10010,10028-10029,10032,10086,10089,10092,10469-10470,10478,10482,10492,10498-10499,10508,10514,10530,10541,10546,10548,10552-10554,10561-10562,10569,10571,10577,10579,10583,10586,10595,10609,10872,10886,10893,10901,10903,10909-10910,10916,10922,10925,10928,10934,10936,10938,10944,10948,10955,10963,10966,10969,10978,10990,10992-10993,11157,11164-11165,11171,11177,11193,11197,11209,11211-11213,11215,11222,11229,11238,11246,11255,11270,11275,11283,11290-11291,11294,11300-11301,11306,11313-11314,11316,11354,11357,11361-11363,11397-11398,11400-11401,11422,11457,11465,11473,11479,11484,11589,11604,11646,11650,11652,11668,11691,11713,11724-11726,11733,11743,11747-11748,11813,11817,11835,11839,11843,11852,11857-11858,11862,11864,11873,11877,11884-11886,11889"}}
+{"timestamp":1712096241.178875,"name":"online","context":{"idset":"4,9,11,17,21,33,39,45,48,970,9973,9981-9982,9984,9988,10007,10016,10035,10090,10094,10473,10502,10535,10585,10597,10603,10605,10607,10611-10612,10745-10746,10864,10867,10874-10875,10894-10895,10918,10935,10939,10945,10950,10953,10956,10967,10974,10976-10977,10986,10989,11158,11189,11204,11216,11225,11230,11243,11257,11259,11263,11292,11298,11315,11360,11365,11373,11378-11379,11550,11847,11865,11875,11878,11883"}}
+{"timestamp":1712096241.2910702,"name":"online","context":{"idset":"9977,9998,10006,10015,10017,10023,10031,10036,10088,10095,10097-10098,10100,10479-10480,10486,10563-10564,10887,10891,10926,10940,10964,10968,10985,10991,11236,11364,11366-11371,11380,11395,11742,11744,11746,11855,11880,11882,11888,11890"}}
+{"timestamp":1712096241.4036925,"name":"online","context":{"idset":"9980,9991,9995,10008,10030,10085,10494,10547,10559,10575,10594,10742,10759-10760,10866,10878,10896,10899,10931,10942,11355-11356,11358,11372,11375,11393,11688,11698"}}
+{"timestamp":1712096241.5195286,"name":"online","context":{"idset":"1-2,5-8,10,12-15,18-19,22-23,26-27,30-31,37-38,41-44,47,49-50,52,54,56-58,10087,10093,10099,10762-10763,11384,11391,11396,11697"}}
+{"timestamp":1712096241.6312089,"name":"online","context":{"idset":"16,20,24-25,28,32,34-36,40,51,53,55,59-60,10091,11385"}}
+{"timestamp":1712096241.7862999,"name":"online","context":{"idset":"3,11685,11687,11689-11690,11693,11700"}}
+{"timestamp":1712096241.9395182,"name":"online","context":{"idset":"10383,10450,11383,11695,11699"}}
+{"timestamp":1712096242.0597086,"name":"online","context":{"idset":"10451,10463"}}
+{"timestamp":1712096242.1757987,"name":"online","context":{"idset":"10396,10415,10466"}}
+{"timestamp":1712096242.3224919,"name":"online","context":{"idset":"10377,10402,10420,10430,10435"}}
+{"timestamp":1712096242.4838018,"name":"online","context":{"idset":"10369,10375,10382,10400-10401,10414,10434,10437"}}
+{"timestamp":1712096242.5983443,"name":"online","context":{"idset":"10358,10364,10367-10368,10384-10385,10390-10391,10398,10403,10421,10461-10462,10468"}}
+{"timestamp":1712096242.7120676,"name":"online","context":{"idset":"10381,10393,10413,10419,10441,10455,10459"}}
+{"timestamp":1712096242.8273604,"name":"online","context":{"idset":"10359,10365-10366,10399,10423-10424,10438,10454,10465,10467"}}
+{"timestamp":1712096242.9398274,"name":"online","context":{"idset":"10362,10379,10386,10406,10411,10431,10439-10440,10458,10460"}}
+{"timestamp":1712096243.0938601,"name":"online","context":{"idset":"10357,10370-10371,10374,10408,10410,10417,10426,10428-10429,10436,10442,10453,10456"}}
+{"timestamp":1712096243.2058794,"name":"online","context":{"idset":"10361,10376,10387,10397,10405,10412,10422,10425,10446,10448-10449,10464"}}
+{"timestamp":1712096243.3210974,"name":"online","context":{"idset":"10388-10389,10392,10404,10407,10409,10416,10427,10433,10443,10445,10452,10457"}}
+{"timestamp":1712096243.4757388,"name":"online","context":{"idset":"10363,10432,10447"}}
+{"timestamp":1712096243.5870965,"name":"online","context":{"idset":"10360,10372,10378,10380,10394-10395,10418,10444"}}
+{"timestamp":1712096243.8380704,"name":"online","context":{"idset":"10373"}}
+{"timestamp":1712096585.8629279,"name":"drain","context":{"idset":"10301","reason":"Bad CXI Link --JRG","overwrite":0}}
+{"timestamp":1712096599.7608726,"name":"drain","context":{"idset":"10302","reason":"bad partner --JRG","overwrite":0}}
+{"timestamp":1712096633.8221953,"name":"drain","context":{"idset":"10469-10474","reason":"--reason HP Diags","overwrite":0}}
+{"timestamp":1712096636.0097418,"name":"offline","context":{"idset":"10075"}}
+{"timestamp":1712096658.8781381,"name":"drain","context":{"idset":"10476-10484","reason":"--reason HP Diags","overwrite":0}}
+{"timestamp":1712096681.148495,"name":"drain","context":{"idset":"10475","reason":"--reason repair cxi1 issue HP","overwrite":0}}
+{"timestamp":1712096714.0101504,"name":"offline","context":{"idset":"10076"}}
+{"timestamp":1712096795.6175246,"name":"offline","context":{"idset":"10077"}}
+{"timestamp":1712096886.009376,"name":"offline","context":{"idset":"10078"}}
+{"timestamp":1712097564.0118735,"name":"drain","context":{"idset":"11892","reason":"broker was unresponsive"}}
+{"timestamp":1712097626.2539406,"name":"offline","context":{"idset":"11892"}}
+{"timestamp":1712099279.1659129,"name":"online","context":{"idset":"11320,11327,11332"}}
+{"timestamp":1712099279.3743422,"name":"online","context":{"idset":"11319,11321-11322,11326,11328"}}
+{"timestamp":1712099279.5286777,"name":"online","context":{"idset":"11323,11325,11331"}}
+{"timestamp":1712099318.9978116,"name":"online","context":{"idset":"10107"}}
+{"timestamp":1712099319.0476577,"name":"online","context":{"idset":"10113"}}
+{"timestamp":1712099319.0948527,"name":"online","context":{"idset":"10106"}}
+{"timestamp":1712099319.136574,"name":"online","context":{"idset":"10101,10109,10118"}}
+{"timestamp":1712099319.3598247,"name":"online","context":{"idset":"10110"}}
+{"timestamp":1712099319.3689642,"name":"online","context":{"idset":"10104"}}
+{"timestamp":1712099319.5152133,"name":"online","context":{"idset":"10138"}}
+{"timestamp":1712099319.5671234,"name":"online","context":{"idset":"10121"}}
+{"timestamp":1712099319.6102901,"name":"online","context":{"idset":"10105"}}
+{"timestamp":1712099319.6189775,"name":"online","context":{"idset":"10152"}}
+{"timestamp":1712099319.6667044,"name":"online","context":{"idset":"10134,10139"}}
+{"timestamp":1712099319.7167819,"name":"online","context":{"idset":"10114"}}
+{"timestamp":1712099319.7956736,"name":"online","context":{"idset":"10126"}}
+{"timestamp":1712099319.8566899,"name":"online","context":{"idset":"10108,10145,10155"}}
+{"timestamp":1712099319.9966934,"name":"online","context":{"idset":"10103,10111,10150,10156"}}
+{"timestamp":1712099320.1355662,"name":"online","context":{"idset":"10112,10115,10117,10119,10146,10159,10217"}}
+{"timestamp":1712099320.2727358,"name":"online","context":{"idset":"10123,10168,10173,10180,10218"}}
+{"timestamp":1712099320.3822429,"name":"online","context":{"idset":"10102,10116,10149,10189,10198"}}
+{"timestamp":1712099320.534421,"name":"online","context":{"idset":"10143,10185"}}
+{"timestamp":1712099320.6887083,"name":"online","context":{"idset":"10141"}}
+{"timestamp":1712099320.9092534,"name":"online","context":{"idset":"10164,10187,10191"}}
+{"timestamp":1712099321.0655353,"name":"online","context":{"idset":"10122,10125,10161,10175,10194,10211"}}
+{"timestamp":1712099321.261642,"name":"online","context":{"idset":"10120,10142"}}
+{"timestamp":1712099321.413347,"name":"online","context":{"idset":"10124,10165,10179"}}
+{"timestamp":1712099321.6276083,"name":"online","context":{"idset":"10157,10174,10178,10190"}}
+{"timestamp":1712099321.783632,"name":"online","context":{"idset":"10136,10222"}}
+{"timestamp":1712099321.8970304,"name":"online","context":{"idset":"10147,10182,10199"}}
+{"timestamp":1712099322.0100555,"name":"online","context":{"idset":"10129,10131,10195"}}
+{"timestamp":1712099322.1230364,"name":"online","context":{"idset":"10171-10172"}}
+{"timestamp":1712099322.2836163,"name":"online","context":{"idset":"10151,10163,10166,10181,10188,10207,10213,10220,10223"}}
+{"timestamp":1712099322.397145,"name":"online","context":{"idset":"10137,10154,10167,10177,10200,10212,10224,10228"}}
+{"timestamp":1712099322.5845664,"name":"online","context":{"idset":"10127,10130,10132,10140,10144,10169,10193,10196-10197,10209,10214"}}
+{"timestamp":1712099322.7357864,"name":"online","context":{"idset":"10128,10135,10158,10170,10176,10184,10186,10204,10215-10216,10219"}}
+{"timestamp":1712099322.8862472,"name":"online","context":{"idset":"10133,10148,10160,10162,10192,10203,10221,10226"}}
+{"timestamp":1712099323.0016456,"name":"online","context":{"idset":"10208,10210,10225,10227"}}
+{"timestamp":1712099323.1708722,"name":"online","context":{"idset":"10201"}}
+{"timestamp":1712099335.9212527,"name":"undrain","context":{"idset":"11319-11329,11331-11332"}}
+{"timestamp":1712099988.0099347,"name":"offline","context":{"idset":"10469"}}
+{"timestamp":1712100438.009603,"name":"offline","context":{"idset":"10470"}}
+{"timestamp":1712100614.163697,"name":"online","context":{"idset":"10741"}}
+{"timestamp":1712100699.5581965,"name":"online","context":{"idset":"10757"}}
+{"timestamp":1712100700.401135,"name":"online","context":{"idset":"10765"}}
+{"timestamp":1712100830.1743488,"name":"undrain","context":{"idset":"10873-10874,10883-10884"}}
+{"timestamp":1712101031.7547829,"name":"online","context":{"idset":"10236"}}
+{"timestamp":1712101032.0500593,"name":"online","context":{"idset":"10276"}}
+{"timestamp":1712101032.1102517,"name":"online","context":{"idset":"10244"}}
+{"timestamp":1712101032.1270485,"name":"online","context":{"idset":"10254"}}
+{"timestamp":1712101032.3155308,"name":"online","context":{"idset":"10230"}}
+{"timestamp":1712101032.398428,"name":"online","context":{"idset":"10234,10256"}}
+{"timestamp":1712101032.4583097,"name":"online","context":{"idset":"10233"}}
+{"timestamp":1712101032.5865896,"name":"online","context":{"idset":"10248"}}
+{"timestamp":1712101032.8489809,"name":"online","context":{"idset":"10251,10264,10267,10283"}}
+{"timestamp":1712101033.0443623,"name":"online","context":{"idset":"10242-10243"}}
+{"timestamp":1712101033.1560638,"name":"online","context":{"idset":"10235,10255"}}
+{"timestamp":1712101033.3195429,"name":"online","context":{"idset":"10262"}}
+{"timestamp":1712101033.5341167,"name":"online","context":{"idset":"10239,10269,10281"}}
+{"timestamp":1712101033.6831334,"name":"online","context":{"idset":"10272"}}
+{"timestamp":1712101033.8683455,"name":"online","context":{"idset":"10231"}}
+{"timestamp":1712101034.12869,"name":"online","context":{"idset":"10229,10312"}}
+{"timestamp":1712101034.2786214,"name":"online","context":{"idset":"10237,10258,10260"}}
+{"timestamp":1712101034.4682248,"name":"online","context":{"idset":"10246,10282,10286,10299"}}
+{"timestamp":1712101034.6073241,"name":"online","context":{"idset":"10249,10338,10344,10348"}}
+{"timestamp":1712101034.7586212,"name":"online","context":{"idset":"10238,10241,10257,10273,10278,10290,10310,10315,10322,10339"}}
+{"timestamp":1712101034.8662183,"name":"online","context":{"idset":"10240,10253,10268,10284,10305,10324"}}
+{"timestamp":1712101034.9740932,"name":"online","context":{"idset":"10298,10308,10327,10343"}}
+{"timestamp":1712101035.1156263,"name":"online","context":{"idset":"10232,10245,10250,10266,10270-10271,10274,10279,10285,10287,10293,10295-10296,10319,10341,10346,10349"}}
+{"timestamp":1712101035.2245808,"name":"online","context":{"idset":"10259,10261,10280,10300,10307,10309,10333,10342,10350"}}
+{"timestamp":1712101035.3337054,"name":"online","context":{"idset":"10289,10291,10297,10306,10311,10320-10321,10328,10354"}}
+{"timestamp":1712101035.4468377,"name":"online","context":{"idset":"10247,10263,10277,10292,10294,10316-10318,10332"}}
+{"timestamp":1712101035.5548136,"name":"online","context":{"idset":"10252,10265,10288,10303-10304,10323,10325,10329,10331,10334,10356"}}
+{"timestamp":1712101035.704735,"name":"online","context":{"idset":"10275,10313,10330,10335-10337,10340,10351"}}
+{"timestamp":1712101035.8560503,"name":"online","context":{"idset":"10345,10353,10355"}}
+{"timestamp":1712101036.0411108,"name":"online","context":{"idset":"10314,10326,10347,10352"}}
+{"timestamp":1712101301.9300735,"name":"offline","context":{"idset":"10357"}}
+{"timestamp":1712101301.9379816,"name":"offline","context":{"idset":"10358"}}
+{"timestamp":1712101301.9455125,"name":"offline","context":{"idset":"10360"}}
+{"timestamp":1712101301.9533455,"name":"offline","context":{"idset":"10361"}}
+{"timestamp":1712101301.961601,"name":"offline","context":{"idset":"10362"}}
+{"timestamp":1712101301.9696076,"name":"offline","context":{"idset":"10363"}}
+{"timestamp":1712101301.9776824,"name":"offline","context":{"idset":"10364"}}
+{"timestamp":1712101301.985713,"name":"offline","context":{"idset":"10365"}}
+{"timestamp":1712101301.9973812,"name":"offline","context":{"idset":"10366"}}
+{"timestamp":1712101302.0044897,"name":"offline","context":{"idset":"10368"}}
+{"timestamp":1712101302.0130455,"name":"offline","context":{"idset":"10369"}}
+{"timestamp":1712101302.021693,"name":"offline","context":{"idset":"10370"}}
+{"timestamp":1712101302.0412385,"name":"offline","context":{"idset":"10371"}}
+{"timestamp":1712101302.0435832,"name":"offline","context":{"idset":"10372"}}
+{"timestamp":1712101303.9150939,"name":"offline","context":{"idset":"10359"}}
+{"timestamp":1712101304.0099037,"name":"offline","context":{"idset":"10367"}}
+{"timestamp":1712101313.9238601,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1712101313.9424047,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1712101314.0110807,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1712101315.9186039,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1712101315.9268596,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1712101315.9355869,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1712101316.0112491,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1712101317.9261057,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1712101317.9436228,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1712101317.9608436,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1712101318.0123022,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1712101319.9240201,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1712101319.9382715,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1712101319.9511304,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1712101320.0106165,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1712101321.9225824,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1712101321.934974,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1712101321.9479635,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1712101322.0096526,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1712101323.9170346,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1712101323.9268024,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1712101324.0100894,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1712101685.2148933,"name":"online","context":{"idset":"10786"}}
+{"timestamp":1712101775.6791904,"name":"online","context":{"idset":"10796"}}
+{"timestamp":1712101776.0703475,"name":"online","context":{"idset":"10804"}}
+{"timestamp":1712101843.9122288,"name":"drain","context":{"idset":"11877","reason":"broker was unresponsive"}}
+{"timestamp":1712101843.9123399,"name":"drain","context":{"idset":"11878","reason":"broker was unresponsive"}}
+{"timestamp":1712101844.0117874,"name":"drain","context":{"idset":"11879","reason":"broker was unresponsive"}}
+{"timestamp":1712101849.9135923,"name":"drain","context":{"idset":"11880","reason":"broker was unresponsive"}}
+{"timestamp":1712101849.9136949,"name":"drain","context":{"idset":"11881","reason":"broker was unresponsive"}}
+{"timestamp":1712101849.9137807,"name":"drain","context":{"idset":"11882","reason":"broker was unresponsive"}}
+{"timestamp":1712101849.913851,"name":"drain","context":{"idset":"11883","reason":"broker was unresponsive"}}
+{"timestamp":1712101849.9139225,"name":"drain","context":{"idset":"11884","reason":"broker was unresponsive"}}
+{"timestamp":1712101849.9139903,"name":"drain","context":{"idset":"11885","reason":"broker was unresponsive"}}
+{"timestamp":1712101849.9140527,"name":"drain","context":{"idset":"11886","reason":"broker was unresponsive"}}
+{"timestamp":1712101849.9141397,"name":"drain","context":{"idset":"11887","reason":"broker was unresponsive"}}
+{"timestamp":1712101849.9142251,"name":"drain","context":{"idset":"11888","reason":"broker was unresponsive"}}
+{"timestamp":1712101849.9143026,"name":"drain","context":{"idset":"11889","reason":"broker was unresponsive"}}
+{"timestamp":1712101850.048907,"name":"drain","context":{"idset":"11890","reason":"broker was unresponsive"}}
+{"timestamp":1712101899.169733,"name":"online","context":{"idset":"10832"}}
+{"timestamp":1712101909.9188712,"name":"offline","context":{"idset":"11877"}}
+{"timestamp":1712101909.9304583,"name":"offline","context":{"idset":"11878"}}
+{"timestamp":1712101910.0151405,"name":"offline","context":{"idset":"11879"}}
+{"timestamp":1712101911.9236786,"name":"offline","context":{"idset":"11880"}}
+{"timestamp":1712101911.9365396,"name":"offline","context":{"idset":"11881"}}
+{"timestamp":1712101911.9530356,"name":"offline","context":{"idset":"11882"}}
+{"timestamp":1712101911.9619656,"name":"offline","context":{"idset":"11883"}}
+{"timestamp":1712101912.3208542,"name":"offline","context":{"idset":"11884"}}
+{"timestamp":1712101913.9197972,"name":"offline","context":{"idset":"11885"}}
+{"timestamp":1712101913.9301949,"name":"offline","context":{"idset":"11886"}}
+{"timestamp":1712101914.0091164,"name":"offline","context":{"idset":"11887"}}
+{"timestamp":1712101915.9191844,"name":"offline","context":{"idset":"11888"}}
+{"timestamp":1712101915.9330175,"name":"offline","context":{"idset":"11889"}}
+{"timestamp":1712101916.0101018,"name":"offline","context":{"idset":"11890"}}
+{"timestamp":1712102953.3257546,"name":"undrain","context":{"idset":"10357-10372"}}
+{"timestamp":1712103046.2298036,"name":"online","context":{"idset":"10370"}}
+{"timestamp":1712103046.3810339,"name":"online","context":{"idset":"10365"}}
+{"timestamp":1712103046.5358894,"name":"online","context":{"idset":"10361,10368-10369"}}
+{"timestamp":1712103046.7975516,"name":"online","context":{"idset":"10358-10359,10366-10367"}}
+{"timestamp":1712103046.9645336,"name":"online","context":{"idset":"10357,10360,10362-10363"}}
+{"timestamp":1712103047.1253309,"name":"online","context":{"idset":"10364,10371-10372"}}
+{"timestamp":1712103234.0097408,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1712103808.051157,"name":"drain","context":{"idset":"10886","reason":"epilog failed for jobid fnFWHjJ8JpF","overwrite":0}}
+{"timestamp":1712104091.9124708,"name":"drain","context":{"idset":"10837","reason":"broker was unresponsive"}}
+{"timestamp":1712104091.9125915,"name":"drain","context":{"idset":"10838","reason":"broker was unresponsive"}}
+{"timestamp":1712104092.012002,"name":"drain","context":{"idset":"10839","reason":"broker was unresponsive"}}
+{"timestamp":1712104097.9162123,"name":"drain","context":{"idset":"10840","reason":"broker was unresponsive"}}
+{"timestamp":1712104097.9163101,"name":"drain","context":{"idset":"10841","reason":"broker was unresponsive"}}
+{"timestamp":1712104097.9163918,"name":"drain","context":{"idset":"10842","reason":"broker was unresponsive"}}
+{"timestamp":1712104097.916522,"name":"drain","context":{"idset":"10844","reason":"broker was unresponsive"}}
+{"timestamp":1712104097.9166572,"name":"drain","context":{"idset":"10845","reason":"broker was unresponsive"}}
+{"timestamp":1712104097.916791,"name":"drain","context":{"idset":"10846","reason":"broker was unresponsive"}}
+{"timestamp":1712104097.9169168,"name":"drain","context":{"idset":"10847","reason":"broker was unresponsive"}}
+{"timestamp":1712104097.9170182,"name":"drain","context":{"idset":"10848","reason":"broker was unresponsive"}}
+{"timestamp":1712104097.9171381,"name":"drain","context":{"idset":"10849","reason":"broker was unresponsive"}}
+{"timestamp":1712104097.9172451,"name":"drain","context":{"idset":"10850","reason":"broker was unresponsive"}}
+{"timestamp":1712104098.022404,"name":"drain","context":{"idset":"10851","reason":"broker was unresponsive"}}
+{"timestamp":1712104104.0097284,"name":"drain","context":{"idset":"10852","reason":"broker was unresponsive"}}
+{"timestamp":1712104151.9118407,"name":"drain","context":{"idset":"10853","reason":"broker was unresponsive"}}
+{"timestamp":1712104151.9119496,"name":"drain","context":{"idset":"10854","reason":"broker was unresponsive"}}
+{"timestamp":1712104151.912024,"name":"drain","context":{"idset":"10855","reason":"broker was unresponsive"}}
+{"timestamp":1712104151.9121008,"name":"drain","context":{"idset":"10856","reason":"broker was unresponsive"}}
+{"timestamp":1712104151.9121745,"name":"drain","context":{"idset":"10857","reason":"broker was unresponsive"}}
+{"timestamp":1712104152.0106273,"name":"drain","context":{"idset":"10858","reason":"broker was unresponsive"}}
+{"timestamp":1712104157.9229591,"name":"offline","context":{"idset":"10837"}}
+{"timestamp":1712104157.9352257,"name":"offline","context":{"idset":"10838"}}
+{"timestamp":1712104157.9458966,"name":"offline","context":{"idset":"10839"}}
+{"timestamp":1712104157.9590466,"name":"drain","context":{"idset":"10859","reason":"broker was unresponsive"}}
+{"timestamp":1712104157.9591551,"name":"drain","context":{"idset":"10860","reason":"broker was unresponsive"}}
+{"timestamp":1712104157.9592886,"name":"drain","context":{"idset":"10861","reason":"broker was unresponsive"}}
+{"timestamp":1712104157.9650466,"name":"drain","context":{"idset":"10862","reason":"broker was unresponsive"}}
+{"timestamp":1712104157.9651451,"name":"drain","context":{"idset":"10863","reason":"broker was unresponsive"}}
+{"timestamp":1712104157.9756937,"name":"drain","context":{"idset":"10864","reason":"broker was unresponsive"}}
+{"timestamp":1712104157.9757817,"name":"drain","context":{"idset":"10866","reason":"broker was unresponsive"}}
+{"timestamp":1712104157.9758604,"name":"drain","context":{"idset":"10867","reason":"broker was unresponsive"}}
+{"timestamp":1712104158.0859213,"name":"drain","context":{"idset":"10868","reason":"broker was unresponsive"}}
+{"timestamp":1712104159.9230025,"name":"offline","context":{"idset":"10840"}}
+{"timestamp":1712104159.9347403,"name":"offline","context":{"idset":"10841"}}
+{"timestamp":1712104159.945641,"name":"offline","context":{"idset":"10842"}}
+{"timestamp":1712104160.3234756,"name":"offline","context":{"idset":"10844"}}
+{"timestamp":1712104161.9201107,"name":"offline","context":{"idset":"10845"}}
+{"timestamp":1712104161.9329689,"name":"offline","context":{"idset":"10846"}}
+{"timestamp":1712104162.0107894,"name":"offline","context":{"idset":"10847"}}
+{"timestamp":1712104163.9173863,"name":"offline","context":{"idset":"10848"}}
+{"timestamp":1712104163.9250321,"name":"offline","context":{"idset":"10849"}}
+{"timestamp":1712104163.9329393,"name":"offline","context":{"idset":"10850"}}
+{"timestamp":1712104164.0108657,"name":"offline","context":{"idset":"10851"}}
+{"timestamp":1712104165.0528533,"name":"offline","context":{"idset":"10852"}}
+{"timestamp":1712104217.9247088,"name":"offline","context":{"idset":"10853"}}
+{"timestamp":1712104217.9364369,"name":"offline","context":{"idset":"10854"}}
+{"timestamp":1712104217.9476714,"name":"offline","context":{"idset":"10855"}}
+{"timestamp":1712104218.291219,"name":"offline","context":{"idset":"10856"}}
+{"timestamp":1712104218.3046503,"name":"offline","context":{"idset":"10857"}}
+{"timestamp":1712104218.3410792,"name":"offline","context":{"idset":"10858"}}
+{"timestamp":1712104219.6514571,"name":"offline","context":{"idset":"10859"}}
+{"timestamp":1712104219.664572,"name":"offline","context":{"idset":"10860"}}
+{"timestamp":1712104219.7412906,"name":"offline","context":{"idset":"10861"}}
+{"timestamp":1712104221.915782,"name":"offline","context":{"idset":"10862"}}
+{"timestamp":1712104221.9243419,"name":"offline","context":{"idset":"10863"}}
+{"timestamp":1712104222.2768605,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1712104223.6225903,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1712104223.6360488,"name":"offline","context":{"idset":"10867"}}
+{"timestamp":1712104223.7133009,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1712105835.2316346,"name":"drain","context":{"idset":"11329","overwrite":0}}
+{"timestamp":1712108474.7654357,"name":"online","context":{"idset":"11329"}}
+{"timestamp":1712108522.3242645,"name":"undrain","context":{"idset":"11329"}}
+{"timestamp":1712109306.3389504,"name":"drain","context":{"idset":"10357-10372","reason":"--reason HP Diags","overwrite":0}}
+{"timestamp":1712112992.6878266,"name":"online","context":{"idset":"11324"}}
+{"timestamp":1712114834.9992273,"name":"undrain","context":{"idset":"11317"}}
+{"timestamp":1712114853.3112199,"name":"undrain","context":{"idset":"11330"}}
+{"timestamp":1712114865.2275217,"name":"undrain","context":{"idset":"11375"}}
+{"timestamp":1712114901.245966,"name":"undrain","context":{"idset":"11376"}}
+{"timestamp":1712114904.5899942,"name":"undrain","context":{"idset":"11377"}}
+{"timestamp":1712114908.0361202,"name":"undrain","context":{"idset":"11378"}}
+{"timestamp":1712114912.1998992,"name":"undrain","context":{"idset":"11379"}}
+{"timestamp":1712114917.4271567,"name":"undrain","context":{"idset":"11380"}}
+{"timestamp":1712115660.0096495,"name":"offline","context":{"idset":"10310"}}
+{"timestamp":1712151247.4846132,"name":"online","context":{"idset":"10075"}}
+{"timestamp":1712151262.7285554,"name":"online","context":{"idset":"10076"}}
+{"timestamp":1712151290.3362055,"name":"online","context":{"idset":"10077"}}
+{"timestamp":1712151309.5566361,"name":"online","context":{"idset":"10078"}}
+{"timestamp":1712151324.5038192,"name":"online","context":{"idset":"10084"}}
+{"timestamp":1712151779.7830231,"name":"undrain","context":{"idset":"10075-10078,10084"}}
+{"timestamp":1712153415.704289,"name":"online","context":{"idset":"10051"}}
+{"timestamp":1712153431.1302588,"name":"online","context":{"idset":"10052"}}
+{"timestamp":1712153610.0102735,"name":"offline","context":{"idset":"10357"}}
+{"timestamp":1712153677.8937376,"name":"undrain","context":{"idset":"10051-10052"}}
+{"timestamp":1712154691.6398871,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6401901,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6406775,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6407955,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6478553,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6479921,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6481254,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6482697,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6483908,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6485405,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6486492,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6555567,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6597042,"name":"drain","context":{"idset":"85","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6604736,"name":"drain","context":{"idset":"86","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.660594,"name":"drain","context":{"idset":"87","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.660717,"name":"drain","context":{"idset":"88","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6608331,"name":"drain","context":{"idset":"89","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6609375,"name":"drain","context":{"idset":"90","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6610425,"name":"drain","context":{"idset":"91","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6611438,"name":"drain","context":{"idset":"92","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6612563,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6613679,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6635458,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6652935,"name":"drain","context":{"idset":"96","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6653471,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6798809,"name":"drain","context":{"idset":"98","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6844938,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6845479,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.684592,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6846371,"name":"drain","context":{"idset":"102","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6846807,"name":"drain","context":{"idset":"103","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6847265,"name":"drain","context":{"idset":"104","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6847749,"name":"drain","context":{"idset":"105","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6848218,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6848698,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6849141,"name":"drain","context":{"idset":"108","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.684962,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.685008,"name":"drain","context":{"idset":"110","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6850562,"name":"drain","context":{"idset":"111","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6851053,"name":"drain","context":{"idset":"112","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6851523,"name":"drain","context":{"idset":"113","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6851997,"name":"drain","context":{"idset":"114","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.685246,"name":"drain","context":{"idset":"115","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6852949,"name":"drain","context":{"idset":"116","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6853426,"name":"drain","context":{"idset":"117","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6853926,"name":"drain","context":{"idset":"118","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6854501,"name":"drain","context":{"idset":"119","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6854975,"name":"drain","context":{"idset":"120","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6855476,"name":"drain","context":{"idset":"122","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6855998,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6856515,"name":"drain","context":{"idset":"124","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6857026,"name":"drain","context":{"idset":"125","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6857526,"name":"drain","context":{"idset":"126","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6858027,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6858518,"name":"drain","context":{"idset":"128","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6859028,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6859548,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6860077,"name":"drain","context":{"idset":"131","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6860585,"name":"drain","context":{"idset":"132","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6861086,"name":"drain","context":{"idset":"133","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6861615,"name":"drain","context":{"idset":"134","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6875262,"name":"drain","context":{"idset":"135","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6875885,"name":"drain","context":{"idset":"136","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6899619,"name":"drain","context":{"idset":"137","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6900213,"name":"drain","context":{"idset":"138","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6900744,"name":"drain","context":{"idset":"139","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6901298,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6901815,"name":"drain","context":{"idset":"141","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6902332,"name":"drain","context":{"idset":"142","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6902819,"name":"drain","context":{"idset":"143","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6903317,"name":"drain","context":{"idset":"144","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6903832,"name":"drain","context":{"idset":"145","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6904464,"name":"drain","context":{"idset":"146","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6904979,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6905484,"name":"drain","context":{"idset":"148","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6905997,"name":"drain","context":{"idset":"149","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6906526,"name":"drain","context":{"idset":"150","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6907089,"name":"drain","context":{"idset":"151","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6907649,"name":"drain","context":{"idset":"152","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.69082,"name":"drain","context":{"idset":"153","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6908724,"name":"drain","context":{"idset":"154","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6909251,"name":"drain","context":{"idset":"155","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6909785,"name":"drain","context":{"idset":"156","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6910326,"name":"drain","context":{"idset":"157","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6910849,"name":"drain","context":{"idset":"158","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6911387,"name":"drain","context":{"idset":"159","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6911941,"name":"drain","context":{"idset":"160","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6912525,"name":"drain","context":{"idset":"161","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6913092,"name":"drain","context":{"idset":"162","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6913657,"name":"drain","context":{"idset":"163","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6914692,"name":"drain","context":{"idset":"164","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6915326,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.691592,"name":"drain","context":{"idset":"166","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6916521,"name":"drain","context":{"idset":"167","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6917136,"name":"drain","context":{"idset":"168","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6917765,"name":"drain","context":{"idset":"169","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6918373,"name":"drain","context":{"idset":"170","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6918993,"name":"drain","context":{"idset":"171","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6919615,"name":"drain","context":{"idset":"172","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6920226,"name":"drain","context":{"idset":"173","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6920846,"name":"drain","context":{"idset":"174","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6921449,"name":"drain","context":{"idset":"175","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6922057,"name":"drain","context":{"idset":"176","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6922667,"name":"drain","context":{"idset":"177","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6923275,"name":"drain","context":{"idset":"178","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6923888,"name":"drain","context":{"idset":"179","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6924729,"name":"drain","context":{"idset":"180","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6925418,"name":"drain","context":{"idset":"181","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.692605,"name":"drain","context":{"idset":"182","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6926687,"name":"drain","context":{"idset":"183","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6927319,"name":"drain","context":{"idset":"184","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6927969,"name":"drain","context":{"idset":"185","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6928613,"name":"drain","context":{"idset":"186","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6929262,"name":"drain","context":{"idset":"187","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6929927,"name":"drain","context":{"idset":"189","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6930578,"name":"drain","context":{"idset":"190","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6931233,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6931884,"name":"drain","context":{"idset":"192","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6932549,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6933224,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6933856,"name":"drain","context":{"idset":"195","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6934624,"name":"drain","context":{"idset":"196","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6935284,"name":"drain","context":{"idset":"197","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6935964,"name":"drain","context":{"idset":"198","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6936646,"name":"drain","context":{"idset":"199","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6937323,"name":"drain","context":{"idset":"200","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6938033,"name":"drain","context":{"idset":"201","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6938739,"name":"drain","context":{"idset":"202","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6939418,"name":"drain","context":{"idset":"203","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.694011,"name":"drain","context":{"idset":"204","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6940792,"name":"drain","context":{"idset":"205","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6941504,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6942205,"name":"drain","context":{"idset":"207","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6942902,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6943579,"name":"drain","context":{"idset":"209","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6944435,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6945148,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6945879,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.694659,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6947279,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6948023,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6948771,"name":"drain","context":{"idset":"216","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6949482,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6950197,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.695096,"name":"drain","context":{"idset":"220","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6951735,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6952496,"name":"drain","context":{"idset":"224","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.695329,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.6954112,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7015183,"name":"drain","context":{"idset":"227","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7017403,"name":"drain","context":{"idset":"228","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.702008,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7021568,"name":"drain","context":{"idset":"230","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.702256,"name":"drain","context":{"idset":"231","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7023766,"name":"drain","context":{"idset":"232","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7027056,"name":"drain","context":{"idset":"233","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7028174,"name":"drain","context":{"idset":"234","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7030404,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.703239,"name":"drain","context":{"idset":"236","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7033858,"name":"drain","context":{"idset":"237","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.703666,"name":"drain","context":{"idset":"238","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7038407,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7040567,"name":"drain","context":{"idset":"240","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7042947,"name":"drain","context":{"idset":"241","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7044706,"name":"drain","context":{"idset":"242","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7046885,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.704932,"name":"drain","context":{"idset":"244","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.705106,"name":"drain","context":{"idset":"245","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7084284,"name":"drain","context":{"idset":"247","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7085428,"name":"drain","context":{"idset":"248","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7086403,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7087467,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7088439,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7089396,"name":"drain","context":{"idset":"252","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7090359,"name":"drain","context":{"idset":"253","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.709131,"name":"drain","context":{"idset":"254","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7092288,"name":"drain","context":{"idset":"255","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7093244,"name":"drain","context":{"idset":"256","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7094264,"name":"drain","context":{"idset":"257","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7095149,"name":"drain","context":{"idset":"258","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7096033,"name":"drain","context":{"idset":"259","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7096915,"name":"drain","context":{"idset":"260","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7097797,"name":"drain","context":{"idset":"261","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7098682,"name":"drain","context":{"idset":"262","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7099583,"name":"drain","context":{"idset":"263","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7100489,"name":"drain","context":{"idset":"264","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7101383,"name":"drain","context":{"idset":"265","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.710228,"name":"drain","context":{"idset":"266","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7103202,"name":"drain","context":{"idset":"267","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7104197,"name":"drain","context":{"idset":"268","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7105114,"name":"drain","context":{"idset":"269","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.710604,"name":"drain","context":{"idset":"270","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7106969,"name":"drain","context":{"idset":"271","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7107887,"name":"drain","context":{"idset":"272","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7108822,"name":"drain","context":{"idset":"273","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.710974,"name":"drain","context":{"idset":"274","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7110667,"name":"drain","context":{"idset":"275","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7111592,"name":"drain","context":{"idset":"276","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7112534,"name":"drain","context":{"idset":"277","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7113464,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7114518,"name":"drain","context":{"idset":"279","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7115457,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7116408,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.711735,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7118301,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7119248,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7120202,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7121158,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.712213,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7123086,"name":"drain","context":{"idset":"288","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7124102,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.712508,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7126062,"name":"drain","context":{"idset":"291","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7127049,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7128024,"name":"drain","context":{"idset":"293","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7129002,"name":"drain","context":{"idset":"294","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7129996,"name":"drain","context":{"idset":"295","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7130992,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7131979,"name":"drain","context":{"idset":"297","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7132986,"name":"drain","context":{"idset":"298","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7134199,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7135201,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7136211,"name":"drain","context":{"idset":"301","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.713722,"name":"drain","context":{"idset":"302","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7138236,"name":"drain","context":{"idset":"303","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7139258,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7140279,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7141302,"name":"drain","context":{"idset":"306","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.714232,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7143342,"name":"drain","context":{"idset":"308","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7144449,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7145491,"name":"drain","context":{"idset":"310","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7146521,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7147582,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7148652,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.714973,"name":"drain","context":{"idset":"314","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7150784,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7151866,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.715292,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7154064,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7155137,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7156217,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7157314,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7158403,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7159488,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7160597,"name":"drain","context":{"idset":"324","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7161684,"name":"drain","context":{"idset":"325","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7162766,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.716387,"name":"drain","context":{"idset":"328","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7165065,"name":"drain","context":{"idset":"329","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7166176,"name":"drain","context":{"idset":"330","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7167275,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7168386,"name":"drain","context":{"idset":"332","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7169497,"name":"drain","context":{"idset":"333","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.717062,"name":"drain","context":{"idset":"334","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7171738,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7172842,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7214179,"name":"drain","context":{"idset":"337","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7215626,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7216904,"name":"drain","context":{"idset":"339","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7218168,"name":"drain","context":{"idset":"340","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7219422,"name":"drain","context":{"idset":"341","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7220695,"name":"drain","context":{"idset":"342","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7221949,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.722321,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7224567,"name":"drain","context":{"idset":"345","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.722584,"name":"drain","context":{"idset":"346","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7227106,"name":"drain","context":{"idset":"347","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7228374,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.722964,"name":"drain","context":{"idset":"350","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7230921,"name":"drain","context":{"idset":"351","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7232213,"name":"drain","context":{"idset":"352","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7233503,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7234862,"name":"drain","context":{"idset":"354","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7236207,"name":"drain","context":{"idset":"355","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7237515,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7238832,"name":"drain","context":{"idset":"357","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7240148,"name":"drain","context":{"idset":"358","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7241466,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7242789,"name":"drain","context":{"idset":"360","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7244198,"name":"drain","context":{"idset":"361","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7245531,"name":"drain","context":{"idset":"362","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7246847,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7248199,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7249532,"name":"drain","context":{"idset":"365","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7250876,"name":"drain","context":{"idset":"366","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7252233,"name":"drain","context":{"idset":"367","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7253551,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7254889,"name":"drain","context":{"idset":"369","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7256119,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7257328,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7258542,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7259772,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7261055,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7262285,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7263508,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7264836,"name":"drain","context":{"idset":"377","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7266088,"name":"drain","context":{"idset":"378","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7267334,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7268579,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7269838,"name":"drain","context":{"idset":"381","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7271094,"name":"drain","context":{"idset":"382","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7272341,"name":"drain","context":{"idset":"383","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7273588,"name":"drain","context":{"idset":"384","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7324924,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7326367,"name":"drain","context":{"idset":"386","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7327676,"name":"drain","context":{"idset":"387","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7328908,"name":"drain","context":{"idset":"388","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7330179,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7331469,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7332675,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7333806,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7335205,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7336516,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7337768,"name":"drain","context":{"idset":"395","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7339132,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7340493,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7341723,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7342989,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7344391,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7345707,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7347057,"name":"drain","context":{"idset":"402","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7348571,"name":"drain","context":{"idset":"403","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7350051,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7351537,"name":"drain","context":{"idset":"405","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7352865,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7354391,"name":"drain","context":{"idset":"407","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7355785,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7357144,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7358544,"name":"drain","context":{"idset":"410","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.73599,"name":"drain","context":{"idset":"411","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7361226,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7362478,"name":"drain","context":{"idset":"413","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7363818,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7365298,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7366652,"name":"drain","context":{"idset":"416","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7367952,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.736927,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7370627,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7371974,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7373383,"name":"drain","context":{"idset":"421","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7374895,"name":"drain","context":{"idset":"422","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7376387,"name":"drain","context":{"idset":"423","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.737777,"name":"drain","context":{"idset":"424","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7379177,"name":"drain","context":{"idset":"425","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.743413,"name":"drain","context":{"idset":"426","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7435708,"name":"drain","context":{"idset":"427","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7437136,"name":"drain","context":{"idset":"428","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7438543,"name":"drain","context":{"idset":"429","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7439935,"name":"drain","context":{"idset":"430","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7441335,"name":"drain","context":{"idset":"432","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7442758,"name":"drain","context":{"idset":"433","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7444334,"name":"drain","context":{"idset":"434","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7445731,"name":"drain","context":{"idset":"435","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7447121,"name":"drain","context":{"idset":"436","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7448521,"name":"drain","context":{"idset":"437","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7449913,"name":"drain","context":{"idset":"438","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7451386,"name":"drain","context":{"idset":"439","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7452812,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7454312,"name":"drain","context":{"idset":"441","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7455742,"name":"drain","context":{"idset":"442","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.745717,"name":"drain","context":{"idset":"443","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7458611,"name":"drain","context":{"idset":"444","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7460058,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7461517,"name":"drain","context":{"idset":"470","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7463119,"name":"drain","context":{"idset":"471","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7464659,"name":"drain","context":{"idset":"472","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7466099,"name":"drain","context":{"idset":"473","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7467546,"name":"drain","context":{"idset":"474","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7468998,"name":"drain","context":{"idset":"475","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7470443,"name":"drain","context":{"idset":"476","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7471905,"name":"drain","context":{"idset":"477","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7473381,"name":"drain","context":{"idset":"478","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7474983,"name":"drain","context":{"idset":"479","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7476485,"name":"drain","context":{"idset":"480","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7477965,"name":"drain","context":{"idset":"481","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7479465,"name":"drain","context":{"idset":"482","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7480977,"name":"drain","context":{"idset":"483","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7482476,"name":"drain","context":{"idset":"484","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7484069,"name":"drain","context":{"idset":"485","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7485554,"name":"drain","context":{"idset":"486","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7487063,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7488544,"name":"drain","context":{"idset":"488","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7490056,"name":"drain","context":{"idset":"489","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7491577,"name":"drain","context":{"idset":"490","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7493105,"name":"drain","context":{"idset":"491","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.749469,"name":"drain","context":{"idset":"492","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7496233,"name":"drain","context":{"idset":"493","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7604153,"name":"drain","context":{"idset":"494","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7605824,"name":"drain","context":{"idset":"495","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7607298,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7608891,"name":"drain","context":{"idset":"497","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7610483,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7612071,"name":"drain","context":{"idset":"499","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7613707,"name":"drain","context":{"idset":"500","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7615492,"name":"drain","context":{"idset":"501","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7617049,"name":"drain","context":{"idset":"502","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7618606,"name":"drain","context":{"idset":"503","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7620151,"name":"drain","context":{"idset":"504","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7621713,"name":"drain","context":{"idset":"505","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7623255,"name":"drain","context":{"idset":"506","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.762495,"name":"drain","context":{"idset":"507","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7626514,"name":"drain","context":{"idset":"508","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7628117,"name":"drain","context":{"idset":"509","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7629704,"name":"drain","context":{"idset":"510","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7631311,"name":"drain","context":{"idset":"511","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7632918,"name":"drain","context":{"idset":"512","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7634594,"name":"drain","context":{"idset":"513","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7636204,"name":"drain","context":{"idset":"514","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7637794,"name":"drain","context":{"idset":"515","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7639389,"name":"drain","context":{"idset":"516","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7640977,"name":"drain","context":{"idset":"517","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7642608,"name":"drain","context":{"idset":"518","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7704351,"name":"drain","context":{"idset":"519","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7706141,"name":"drain","context":{"idset":"520","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7707744,"name":"drain","context":{"idset":"521","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7709324,"name":"drain","context":{"idset":"522","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7710917,"name":"drain","context":{"idset":"523","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7712505,"name":"drain","context":{"idset":"524","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7714393,"name":"drain","context":{"idset":"525","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.771601,"name":"drain","context":{"idset":"526","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7717626,"name":"drain","context":{"idset":"527","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7719235,"name":"drain","context":{"idset":"528","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.772085,"name":"drain","context":{"idset":"529","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7722466,"name":"drain","context":{"idset":"530","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.772414,"name":"drain","context":{"idset":"531","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7725685,"name":"drain","context":{"idset":"532","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7727175,"name":"drain","context":{"idset":"533","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7728746,"name":"drain","context":{"idset":"534","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7730417,"name":"drain","context":{"idset":"535","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7732072,"name":"drain","context":{"idset":"536","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7733757,"name":"drain","context":{"idset":"537","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7735519,"name":"drain","context":{"idset":"538","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7737143,"name":"drain","context":{"idset":"539","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7738755,"name":"drain","context":{"idset":"540","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7740443,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7742167,"name":"drain","context":{"idset":"566","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.774384,"name":"drain","context":{"idset":"567","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7745574,"name":"drain","context":{"idset":"568","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7747273,"name":"drain","context":{"idset":"569","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7749047,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7750826,"name":"drain","context":{"idset":"571","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7752657,"name":"drain","context":{"idset":"572","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.775445,"name":"drain","context":{"idset":"573","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7756157,"name":"drain","context":{"idset":"574","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7757812,"name":"drain","context":{"idset":"575","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7759368,"name":"drain","context":{"idset":"576","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7760947,"name":"drain","context":{"idset":"577","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7762668,"name":"drain","context":{"idset":"578","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7764611,"name":"drain","context":{"idset":"579","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7766304,"name":"drain","context":{"idset":"580","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7767966,"name":"drain","context":{"idset":"581","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7769599,"name":"drain","context":{"idset":"582","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7771285,"name":"drain","context":{"idset":"583","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7773046,"name":"drain","context":{"idset":"584","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7774849,"name":"drain","context":{"idset":"585","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7776549,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.777828,"name":"drain","context":{"idset":"587","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7780101,"name":"drain","context":{"idset":"588","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7782385,"name":"drain","context":{"idset":"589","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7894335,"name":"drain","context":{"idset":"590","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7896128,"name":"drain","context":{"idset":"591","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7897892,"name":"drain","context":{"idset":"592","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7899675,"name":"drain","context":{"idset":"593","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7901475,"name":"drain","context":{"idset":"594","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7903228,"name":"drain","context":{"idset":"595","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7904975,"name":"drain","context":{"idset":"596","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7906611,"name":"drain","context":{"idset":"597","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7908409,"name":"drain","context":{"idset":"598","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7910223,"name":"drain","context":{"idset":"599","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7911994,"name":"drain","context":{"idset":"600","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7913766,"name":"drain","context":{"idset":"601","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7915711,"name":"drain","context":{"idset":"602","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7917581,"name":"drain","context":{"idset":"603","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7919462,"name":"drain","context":{"idset":"604","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7921228,"name":"drain","context":{"idset":"605","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7922993,"name":"drain","context":{"idset":"606","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7924814,"name":"drain","context":{"idset":"607","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7926571,"name":"drain","context":{"idset":"608","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7928505,"name":"drain","context":{"idset":"609","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7930374,"name":"drain","context":{"idset":"610","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7932198,"name":"drain","context":{"idset":"611","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7934685,"name":"drain","context":{"idset":"612","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7936568,"name":"drain","context":{"idset":"613","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7938452,"name":"drain","context":{"idset":"614","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7940338,"name":"drain","context":{"idset":"615","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7942197,"name":"drain","context":{"idset":"616","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.794416,"name":"drain","context":{"idset":"617","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7946017,"name":"drain","context":{"idset":"618","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7947829,"name":"drain","context":{"idset":"619","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7949672,"name":"drain","context":{"idset":"620","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7951493,"name":"drain","context":{"idset":"621","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7953339,"name":"drain","context":{"idset":"622","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7955184,"name":"drain","context":{"idset":"623","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7956882,"name":"drain","context":{"idset":"624","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7958715,"name":"drain","context":{"idset":"625","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7960589,"name":"drain","context":{"idset":"626","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7962403,"name":"drain","context":{"idset":"627","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.796427,"name":"drain","context":{"idset":"628","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7966206,"name":"drain","context":{"idset":"629","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7968118,"name":"drain","context":{"idset":"630","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.7969999,"name":"drain","context":{"idset":"631","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.8208048,"name":"drain","context":{"idset":"632","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.8343539,"name":"drain","context":{"idset":"633","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.8508646,"name":"drain","context":{"idset":"634","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.861587,"name":"drain","context":{"idset":"635","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.8909535,"name":"drain","context":{"idset":"869","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9068179,"name":"drain","context":{"idset":"870","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9234848,"name":"drain","context":{"idset":"881","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9423342,"name":"drain","context":{"idset":"882","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9622352,"name":"drain","context":{"idset":"969","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9624407,"name":"drain","context":{"idset":"970","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.962635,"name":"drain","context":{"idset":"971","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9628313,"name":"drain","context":{"idset":"972","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9630353,"name":"drain","context":{"idset":"973","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9632485,"name":"drain","context":{"idset":"974","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9844565,"name":"drain","context":{"idset":"975","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9846833,"name":"drain","context":{"idset":"976","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9848833,"name":"drain","context":{"idset":"977","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9850831,"name":"drain","context":{"idset":"978","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.98528,"name":"drain","context":{"idset":"979","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9854927,"name":"drain","context":{"idset":"980","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9859102,"name":"drain","context":{"idset":"9973","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9863,"name":"drain","context":{"idset":"9974","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9867265,"name":"drain","context":{"idset":"9975","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9871423,"name":"drain","context":{"idset":"9976","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9875474,"name":"drain","context":{"idset":"9977","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9879613,"name":"drain","context":{"idset":"9978","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9883626,"name":"drain","context":{"idset":"9979","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9887779,"name":"drain","context":{"idset":"9980","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9891841,"name":"drain","context":{"idset":"9981","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9895952,"name":"drain","context":{"idset":"9982","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.99,"name":"drain","context":{"idset":"9983","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9905057,"name":"drain","context":{"idset":"9984","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9910107,"name":"drain","context":{"idset":"9985","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9914284,"name":"drain","context":{"idset":"9986","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9929512,"name":"drain","context":{"idset":"9987","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9933772,"name":"drain","context":{"idset":"9988","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9938779,"name":"drain","context":{"idset":"9989","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9942834,"name":"drain","context":{"idset":"9990","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.994761,"name":"drain","context":{"idset":"9991","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9951608,"name":"drain","context":{"idset":"9992","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9966958,"name":"drain","context":{"idset":"9993","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9973536,"name":"drain","context":{"idset":"9994","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9977736,"name":"drain","context":{"idset":"9995","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9987879,"name":"drain","context":{"idset":"9996","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9992223,"name":"drain","context":{"idset":"9997","reason":"broker was unresponsive"}}
+{"timestamp":1712154691.9998438,"name":"drain","context":{"idset":"9998","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0004478,"name":"drain","context":{"idset":"9999","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0009925,"name":"drain","context":{"idset":"10000","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.00158,"name":"drain","context":{"idset":"10001","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0020142,"name":"drain","context":{"idset":"10002","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0027688,"name":"drain","context":{"idset":"10003","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.003221,"name":"drain","context":{"idset":"10004","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0038302,"name":"drain","context":{"idset":"10005","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0042663,"name":"drain","context":{"idset":"10006","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0048888,"name":"drain","context":{"idset":"10007","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0053651,"name":"drain","context":{"idset":"10008","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0059037,"name":"drain","context":{"idset":"10009","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0063317,"name":"drain","context":{"idset":"10010","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0069458,"name":"drain","context":{"idset":"10011","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0076416,"name":"drain","context":{"idset":"10012","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.008126,"name":"drain","context":{"idset":"10013","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0088105,"name":"drain","context":{"idset":"10014","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0138414,"name":"drain","context":{"idset":"10015","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0142868,"name":"drain","context":{"idset":"10016","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0147307,"name":"drain","context":{"idset":"10017","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0152342,"name":"drain","context":{"idset":"10018","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0156934,"name":"drain","context":{"idset":"10019","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0160887,"name":"drain","context":{"idset":"10020","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0171306,"name":"drain","context":{"idset":"10021","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0175445,"name":"drain","context":{"idset":"10022","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0179458,"name":"drain","context":{"idset":"10023","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0183384,"name":"drain","context":{"idset":"10024","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0187528,"name":"drain","context":{"idset":"10025","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0195444,"name":"drain","context":{"idset":"10026","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.020762,"name":"drain","context":{"idset":"10027","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0211778,"name":"drain","context":{"idset":"10028","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0217192,"name":"drain","context":{"idset":"10029","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0221164,"name":"drain","context":{"idset":"10030","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0256703,"name":"drain","context":{"idset":"10031","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0261612,"name":"drain","context":{"idset":"10032","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0266223,"name":"drain","context":{"idset":"10033","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0280805,"name":"drain","context":{"idset":"10034","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0284531,"name":"drain","context":{"idset":"10035","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0288522,"name":"drain","context":{"idset":"10036","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0301805,"name":"drain","context":{"idset":"10037","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.030616,"name":"drain","context":{"idset":"10038","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0310261,"name":"drain","context":{"idset":"10039","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0314457,"name":"drain","context":{"idset":"10040","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0318673,"name":"drain","context":{"idset":"10041","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0367901,"name":"drain","context":{"idset":"10042","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0372682,"name":"drain","context":{"idset":"10043","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.03771,"name":"drain","context":{"idset":"10044","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0381563,"name":"drain","context":{"idset":"10045","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.038584,"name":"drain","context":{"idset":"10046","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0390396,"name":"drain","context":{"idset":"10047","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0394773,"name":"drain","context":{"idset":"10048","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0398867,"name":"drain","context":{"idset":"10049","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.040309,"name":"drain","context":{"idset":"10050","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0407417,"name":"drain","context":{"idset":"10051","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0437491,"name":"drain","context":{"idset":"10052","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0466192,"name":"drain","context":{"idset":"10053","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0470922,"name":"drain","context":{"idset":"10054","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0477114,"name":"drain","context":{"idset":"10055","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0483823,"name":"drain","context":{"idset":"10056","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0498736,"name":"drain","context":{"idset":"10057","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0503008,"name":"drain","context":{"idset":"10058","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0507491,"name":"drain","context":{"idset":"10059","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0511656,"name":"drain","context":{"idset":"10060","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0515821,"name":"drain","context":{"idset":"10061","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.053834,"name":"drain","context":{"idset":"10062","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0550618,"name":"drain","context":{"idset":"10063","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0554938,"name":"drain","context":{"idset":"10064","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0561919,"name":"drain","context":{"idset":"10065","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0566978,"name":"drain","context":{"idset":"10066","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0572069,"name":"drain","context":{"idset":"10067","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0576293,"name":"drain","context":{"idset":"10068","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0579636,"name":"drain","context":{"idset":"10069","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.058814,"name":"drain","context":{"idset":"10070","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0591509,"name":"drain","context":{"idset":"10071","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0596557,"name":"drain","context":{"idset":"10072","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0601408,"name":"drain","context":{"idset":"10073","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0605791,"name":"drain","context":{"idset":"10074","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0615735,"name":"drain","context":{"idset":"10075","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.062603,"name":"drain","context":{"idset":"10076","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0630455,"name":"drain","context":{"idset":"10077","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0634887,"name":"drain","context":{"idset":"10078","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0653596,"name":"drain","context":{"idset":"10079","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0658143,"name":"drain","context":{"idset":"10080","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0662339,"name":"drain","context":{"idset":"10081","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0666721,"name":"drain","context":{"idset":"10082","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0671015,"name":"drain","context":{"idset":"10083","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0697711,"name":"drain","context":{"idset":"10084","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0754991,"name":"drain","context":{"idset":"10085","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0759709,"name":"drain","context":{"idset":"10086","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0763938,"name":"drain","context":{"idset":"10087","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.076843,"name":"drain","context":{"idset":"10088","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0786448,"name":"drain","context":{"idset":"10089","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0790851,"name":"drain","context":{"idset":"10090","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0800772,"name":"drain","context":{"idset":"10091","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.080476,"name":"drain","context":{"idset":"10092","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.081007,"name":"drain","context":{"idset":"10093","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0816131,"name":"drain","context":{"idset":"10094","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0847113,"name":"drain","context":{"idset":"10095","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0851505,"name":"drain","context":{"idset":"10096","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0855951,"name":"drain","context":{"idset":"10097","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0860367,"name":"drain","context":{"idset":"10098","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0875695,"name":"drain","context":{"idset":"10099","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0891881,"name":"drain","context":{"idset":"10100","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0895898,"name":"drain","context":{"idset":"10101","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0900357,"name":"drain","context":{"idset":"10103","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0904779,"name":"drain","context":{"idset":"10104","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0909159,"name":"drain","context":{"idset":"10105","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0914791,"name":"drain","context":{"idset":"10106","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0919535,"name":"drain","context":{"idset":"10107","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.09252,"name":"drain","context":{"idset":"10108","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0930083,"name":"drain","context":{"idset":"10109","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0935347,"name":"drain","context":{"idset":"10110","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0941203,"name":"drain","context":{"idset":"10111","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0945811,"name":"drain","context":{"idset":"10112","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0950613,"name":"drain","context":{"idset":"10113","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0955501,"name":"drain","context":{"idset":"10114","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0960312,"name":"drain","context":{"idset":"10115","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0965364,"name":"drain","context":{"idset":"10116","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0970156,"name":"drain","context":{"idset":"10117","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0974627,"name":"drain","context":{"idset":"10118","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.097945,"name":"drain","context":{"idset":"10119","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0984499,"name":"drain","context":{"idset":"10130","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0988858,"name":"drain","context":{"idset":"10131","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0993621,"name":"drain","context":{"idset":"10132","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.0998168,"name":"drain","context":{"idset":"10133","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1003056,"name":"drain","context":{"idset":"10134","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1007507,"name":"drain","context":{"idset":"10135","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1011977,"name":"drain","context":{"idset":"10136","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1016693,"name":"drain","context":{"idset":"10138","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1021886,"name":"drain","context":{"idset":"10139","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1027501,"name":"drain","context":{"idset":"10141","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1032383,"name":"drain","context":{"idset":"10142","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1036949,"name":"drain","context":{"idset":"10143","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1041818,"name":"drain","context":{"idset":"10144","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1046801,"name":"drain","context":{"idset":"10145","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1051702,"name":"drain","context":{"idset":"10146","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1056852,"name":"drain","context":{"idset":"10147","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1061764,"name":"drain","context":{"idset":"10148","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1066544,"name":"drain","context":{"idset":"10149","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1071053,"name":"drain","context":{"idset":"10150","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1075859,"name":"drain","context":{"idset":"10155","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1080379,"name":"drain","context":{"idset":"10161","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1085031,"name":"drain","context":{"idset":"10163","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.109813,"name":"drain","context":{"idset":"10164","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1102936,"name":"drain","context":{"idset":"10165","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1107855,"name":"drain","context":{"idset":"10166","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1112411,"name":"drain","context":{"idset":"10167","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1116893,"name":"drain","context":{"idset":"10168","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1121376,"name":"drain","context":{"idset":"10169","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1126459,"name":"drain","context":{"idset":"10170","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1132081,"name":"drain","context":{"idset":"10171","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1137207,"name":"drain","context":{"idset":"10172","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1142459,"name":"drain","context":{"idset":"10173","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1147118,"name":"drain","context":{"idset":"10174","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1152544,"name":"drain","context":{"idset":"10175","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1157835,"name":"drain","context":{"idset":"10176","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1163208,"name":"drain","context":{"idset":"10177","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1168191,"name":"drain","context":{"idset":"10178","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1172884,"name":"drain","context":{"idset":"10179","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1178503,"name":"drain","context":{"idset":"10180","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1183202,"name":"drain","context":{"idset":"10181","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1187963,"name":"drain","context":{"idset":"10182","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.119262,"name":"drain","context":{"idset":"10184","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1201177,"name":"drain","context":{"idset":"10185","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1205254,"name":"drain","context":{"idset":"10186","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1209128,"name":"drain","context":{"idset":"10187","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1213179,"name":"drain","context":{"idset":"10188","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1237216,"name":"drain","context":{"idset":"10189","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1242039,"name":"drain","context":{"idset":"10190","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1246049,"name":"drain","context":{"idset":"10191","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1263187,"name":"drain","context":{"idset":"10192","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.126775,"name":"drain","context":{"idset":"10193","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1283445,"name":"drain","context":{"idset":"10194","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1288474,"name":"drain","context":{"idset":"10195","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1293375,"name":"drain","context":{"idset":"10196","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1311934,"name":"drain","context":{"idset":"10197","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1316781,"name":"drain","context":{"idset":"10198","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1321611,"name":"drain","context":{"idset":"10199","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1326504,"name":"drain","context":{"idset":"10200","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1383219,"name":"drain","context":{"idset":"10201","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1402805,"name":"drain","context":{"idset":"10203","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1407681,"name":"drain","context":{"idset":"10204","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1412749,"name":"drain","context":{"idset":"10207","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1417551,"name":"drain","context":{"idset":"10208","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1422639,"name":"drain","context":{"idset":"10209","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1427567,"name":"drain","context":{"idset":"10210","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1432779,"name":"drain","context":{"idset":"10211","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.143784,"name":"drain","context":{"idset":"10212","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1442785,"name":"drain","context":{"idset":"10213","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1447704,"name":"drain","context":{"idset":"10214","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1452756,"name":"drain","context":{"idset":"10215","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1465502,"name":"drain","context":{"idset":"10216","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1472189,"name":"drain","context":{"idset":"10217","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1477337,"name":"drain","context":{"idset":"10218","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1484256,"name":"drain","context":{"idset":"10219","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1492729,"name":"drain","context":{"idset":"10220","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.150162,"name":"drain","context":{"idset":"10221","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1508093,"name":"drain","context":{"idset":"10222","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.151557,"name":"drain","context":{"idset":"10223","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1521697,"name":"drain","context":{"idset":"10224","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1529777,"name":"drain","context":{"idset":"10225","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1536686,"name":"drain","context":{"idset":"10226","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1539881,"name":"drain","context":{"idset":"10227","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1544161,"name":"drain","context":{"idset":"10228","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1549852,"name":"drain","context":{"idset":"10229","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.155328,"name":"drain","context":{"idset":"10230","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1560159,"name":"drain","context":{"idset":"10231","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1565709,"name":"drain","context":{"idset":"10232","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1570976,"name":"drain","context":{"idset":"10233","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1575892,"name":"drain","context":{"idset":"10234","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1581612,"name":"drain","context":{"idset":"10235","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1586955,"name":"drain","context":{"idset":"10236","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1592197,"name":"drain","context":{"idset":"10237","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1597676,"name":"drain","context":{"idset":"10238","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1603186,"name":"drain","context":{"idset":"10239","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1608613,"name":"drain","context":{"idset":"10240","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1613708,"name":"drain","context":{"idset":"10241","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1619284,"name":"drain","context":{"idset":"10242","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.162426,"name":"drain","context":{"idset":"10243","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1629293,"name":"drain","context":{"idset":"10244","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1634564,"name":"drain","context":{"idset":"10245","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1639669,"name":"drain","context":{"idset":"10246","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1645501,"name":"drain","context":{"idset":"10247","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1650479,"name":"drain","context":{"idset":"10248","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1655612,"name":"drain","context":{"idset":"10249","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1660788,"name":"drain","context":{"idset":"10250","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1665885,"name":"drain","context":{"idset":"10251","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1671071,"name":"drain","context":{"idset":"10252","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1676214,"name":"drain","context":{"idset":"10253","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1680996,"name":"drain","context":{"idset":"10254","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1684422,"name":"drain","context":{"idset":"10255","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.168859,"name":"drain","context":{"idset":"10256","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1692717,"name":"drain","context":{"idset":"10257","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1697695,"name":"drain","context":{"idset":"10258","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1702056,"name":"drain","context":{"idset":"10259","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1705289,"name":"drain","context":{"idset":"10260","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1708262,"name":"drain","context":{"idset":"10261","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1711252,"name":"drain","context":{"idset":"10262","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1715286,"name":"drain","context":{"idset":"10263","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1718686,"name":"drain","context":{"idset":"10264","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1721611,"name":"drain","context":{"idset":"10265","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1725268,"name":"drain","context":{"idset":"10266","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1729248,"name":"drain","context":{"idset":"10267","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1732593,"name":"drain","context":{"idset":"10268","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1735897,"name":"drain","context":{"idset":"10269","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1739256,"name":"drain","context":{"idset":"10270","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1743321,"name":"drain","context":{"idset":"10271","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1747069,"name":"drain","context":{"idset":"10272","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1750937,"name":"drain","context":{"idset":"10273","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1754203,"name":"drain","context":{"idset":"10274","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1757326,"name":"drain","context":{"idset":"10275","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1760244,"name":"drain","context":{"idset":"10276","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1764064,"name":"drain","context":{"idset":"10277","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1767967,"name":"drain","context":{"idset":"10278","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1771843,"name":"drain","context":{"idset":"10279","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1775587,"name":"drain","context":{"idset":"10280","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.17788,"name":"drain","context":{"idset":"10281","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1782088,"name":"drain","context":{"idset":"10282","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1785915,"name":"drain","context":{"idset":"10283","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.178915,"name":"drain","context":{"idset":"10284","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1792443,"name":"drain","context":{"idset":"10285","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1795595,"name":"drain","context":{"idset":"10286","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1798904,"name":"drain","context":{"idset":"10287","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1801946,"name":"drain","context":{"idset":"10288","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1805344,"name":"drain","context":{"idset":"10289","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1808658,"name":"drain","context":{"idset":"10290","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1812718,"name":"drain","context":{"idset":"10291","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1816318,"name":"drain","context":{"idset":"10292","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1819527,"name":"drain","context":{"idset":"10293","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1824872,"name":"drain","context":{"idset":"10294","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1828856,"name":"drain","context":{"idset":"10295","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1831956,"name":"drain","context":{"idset":"10296","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.183738,"name":"drain","context":{"idset":"10297","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1842871,"name":"drain","context":{"idset":"10298","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1850612,"name":"drain","context":{"idset":"10299","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1856375,"name":"drain","context":{"idset":"10300","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1863458,"name":"drain","context":{"idset":"10303","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.18731,"name":"drain","context":{"idset":"10304","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1884067,"name":"drain","context":{"idset":"10305","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1891046,"name":"drain","context":{"idset":"10306","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.189537,"name":"drain","context":{"idset":"10307","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1899049,"name":"drain","context":{"idset":"10308","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1902196,"name":"drain","context":{"idset":"10309","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1906848,"name":"drain","context":{"idset":"10311","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1912055,"name":"drain","context":{"idset":"10312","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1917057,"name":"drain","context":{"idset":"10313","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1922157,"name":"drain","context":{"idset":"10314","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.192723,"name":"drain","context":{"idset":"10315","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1932271,"name":"drain","context":{"idset":"10316","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1937413,"name":"drain","context":{"idset":"10317","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1942444,"name":"drain","context":{"idset":"10318","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1947479,"name":"drain","context":{"idset":"10319","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.195245,"name":"drain","context":{"idset":"10320","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1957593,"name":"drain","context":{"idset":"10321","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1962512,"name":"drain","context":{"idset":"10322","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.196739,"name":"drain","context":{"idset":"10323","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1972318,"name":"drain","context":{"idset":"10324","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1977785,"name":"drain","context":{"idset":"10325","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1982744,"name":"drain","context":{"idset":"10326","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.198787,"name":"drain","context":{"idset":"10327","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1992407,"name":"drain","context":{"idset":"10328","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.1996758,"name":"drain","context":{"idset":"10329","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2002022,"name":"drain","context":{"idset":"10330","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2005742,"name":"drain","context":{"idset":"10331","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2010007,"name":"drain","context":{"idset":"10332","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2015157,"name":"drain","context":{"idset":"10333","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.201833,"name":"drain","context":{"idset":"10334","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2021644,"name":"drain","context":{"idset":"10335","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2026806,"name":"drain","context":{"idset":"10336","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2032039,"name":"drain","context":{"idset":"10337","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2037508,"name":"drain","context":{"idset":"10338","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2043078,"name":"drain","context":{"idset":"10339","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2046561,"name":"drain","context":{"idset":"10340","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2051589,"name":"drain","context":{"idset":"10341","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2057502,"name":"drain","context":{"idset":"10342","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2061348,"name":"drain","context":{"idset":"10343","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2064543,"name":"drain","context":{"idset":"10344","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2067556,"name":"drain","context":{"idset":"10345","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2071021,"name":"drain","context":{"idset":"10346","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2074149,"name":"drain","context":{"idset":"10347","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2077267,"name":"drain","context":{"idset":"10348","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2082362,"name":"drain","context":{"idset":"10349","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2087827,"name":"drain","context":{"idset":"10350","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2093384,"name":"drain","context":{"idset":"10351","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.209904,"name":"drain","context":{"idset":"10352","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2104762,"name":"drain","context":{"idset":"10353","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2109931,"name":"drain","context":{"idset":"10354","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2115319,"name":"drain","context":{"idset":"10355","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2119184,"name":"drain","context":{"idset":"10356","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2848744,"name":"drain","context":{"idset":"10485","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2855797,"name":"drain","context":{"idset":"10486","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2862186,"name":"drain","context":{"idset":"10487","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2869651,"name":"drain","context":{"idset":"10488","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2876391,"name":"drain","context":{"idset":"10489","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2882817,"name":"drain","context":{"idset":"10490","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2889614,"name":"drain","context":{"idset":"10491","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2896385,"name":"drain","context":{"idset":"10492","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.290364,"name":"drain","context":{"idset":"10493","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2910125,"name":"drain","context":{"idset":"10494","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2919216,"name":"drain","context":{"idset":"10495","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2922721,"name":"drain","context":{"idset":"10496","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2928631,"name":"drain","context":{"idset":"10497","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2933228,"name":"drain","context":{"idset":"10498","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2936754,"name":"drain","context":{"idset":"10499","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2940159,"name":"drain","context":{"idset":"10500","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2944803,"name":"drain","context":{"idset":"10501","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2952039,"name":"drain","context":{"idset":"10502","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2958465,"name":"drain","context":{"idset":"10503","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2964497,"name":"drain","context":{"idset":"10504","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2970898,"name":"drain","context":{"idset":"10505","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2979319,"name":"drain","context":{"idset":"10506","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2985799,"name":"drain","context":{"idset":"10507","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.2993052,"name":"drain","context":{"idset":"10508","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3001246,"name":"drain","context":{"idset":"10511","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3008513,"name":"drain","context":{"idset":"10512","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.30162,"name":"drain","context":{"idset":"10513","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3022187,"name":"drain","context":{"idset":"10514","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3026826,"name":"drain","context":{"idset":"10515","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.303062,"name":"drain","context":{"idset":"10516","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3034911,"name":"drain","context":{"idset":"10517","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3039613,"name":"drain","context":{"idset":"10520","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3043089,"name":"drain","context":{"idset":"10523","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3047626,"name":"drain","context":{"idset":"10524","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3053615,"name":"drain","context":{"idset":"10525","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3059833,"name":"drain","context":{"idset":"10526","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3067749,"name":"drain","context":{"idset":"10527","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3075652,"name":"drain","context":{"idset":"10528","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3081651,"name":"drain","context":{"idset":"10529","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3089309,"name":"drain","context":{"idset":"10530","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3095362,"name":"drain","context":{"idset":"10531","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3101959,"name":"drain","context":{"idset":"10532","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3112245,"name":"drain","context":{"idset":"10533","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3118629,"name":"drain","context":{"idset":"10534","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3125131,"name":"drain","context":{"idset":"10535","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3131442,"name":"drain","context":{"idset":"10536","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3146098,"name":"drain","context":{"idset":"10537","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3152812,"name":"drain","context":{"idset":"10538","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3159089,"name":"drain","context":{"idset":"10539","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3169928,"name":"drain","context":{"idset":"10540","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3177123,"name":"drain","context":{"idset":"10541","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3183446,"name":"drain","context":{"idset":"10542","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3189867,"name":"drain","context":{"idset":"10543","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3198016,"name":"drain","context":{"idset":"10544","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3205104,"name":"drain","context":{"idset":"10545","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3212693,"name":"drain","context":{"idset":"10546","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3223677,"name":"drain","context":{"idset":"10547","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3239694,"name":"drain","context":{"idset":"10548","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3252914,"name":"drain","context":{"idset":"10549","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.326052,"name":"drain","context":{"idset":"10550","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3268077,"name":"drain","context":{"idset":"10551","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.327435,"name":"drain","context":{"idset":"10552","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3287976,"name":"drain","context":{"idset":"10553","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3296967,"name":"drain","context":{"idset":"10554","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3303504,"name":"drain","context":{"idset":"10555","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3311803,"name":"drain","context":{"idset":"10556","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3325636,"name":"drain","context":{"idset":"10557","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3333406,"name":"drain","context":{"idset":"10558","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3341837,"name":"drain","context":{"idset":"10559","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3350315,"name":"drain","context":{"idset":"10560","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3357589,"name":"drain","context":{"idset":"10561","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3364799,"name":"drain","context":{"idset":"10562","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3372164,"name":"drain","context":{"idset":"10563","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3378868,"name":"drain","context":{"idset":"10564","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3385117,"name":"drain","context":{"idset":"10565","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3392389,"name":"drain","context":{"idset":"10566","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3402104,"name":"drain","context":{"idset":"10567","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3409209,"name":"drain","context":{"idset":"10568","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3415594,"name":"drain","context":{"idset":"10569","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3426354,"name":"drain","context":{"idset":"10570","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3437395,"name":"drain","context":{"idset":"10571","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3448856,"name":"drain","context":{"idset":"10572","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3457575,"name":"drain","context":{"idset":"10573","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3464994,"name":"drain","context":{"idset":"10574","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3473284,"name":"drain","context":{"idset":"10575","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3482213,"name":"drain","context":{"idset":"10576","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3491318,"name":"drain","context":{"idset":"10577","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3498058,"name":"drain","context":{"idset":"10578","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3505373,"name":"drain","context":{"idset":"10579","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3512614,"name":"drain","context":{"idset":"10580","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3521948,"name":"drain","context":{"idset":"10581","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3531559,"name":"drain","context":{"idset":"10582","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3539686,"name":"drain","context":{"idset":"10583","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3547308,"name":"drain","context":{"idset":"10584","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3553751,"name":"drain","context":{"idset":"10585","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3562109,"name":"drain","context":{"idset":"10586","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3569131,"name":"drain","context":{"idset":"10587","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3576562,"name":"drain","context":{"idset":"10588","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3583889,"name":"drain","context":{"idset":"10589","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3597434,"name":"drain","context":{"idset":"10590","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3604431,"name":"drain","context":{"idset":"10591","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3611217,"name":"drain","context":{"idset":"10592","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3618419,"name":"drain","context":{"idset":"10593","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3625395,"name":"drain","context":{"idset":"10594","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3633015,"name":"drain","context":{"idset":"10595","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3641078,"name":"drain","context":{"idset":"10596","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3648114,"name":"drain","context":{"idset":"10597","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3654938,"name":"drain","context":{"idset":"10598","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3661163,"name":"drain","context":{"idset":"10599","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3667905,"name":"drain","context":{"idset":"10600","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3674822,"name":"drain","context":{"idset":"10601","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3681419,"name":"drain","context":{"idset":"10602","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3688738,"name":"drain","context":{"idset":"10603","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3695886,"name":"drain","context":{"idset":"10604","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3704796,"name":"drain","context":{"idset":"10605","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3711882,"name":"drain","context":{"idset":"10606","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3718636,"name":"drain","context":{"idset":"10607","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3725467,"name":"drain","context":{"idset":"10608","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3732421,"name":"drain","context":{"idset":"10609","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3739498,"name":"drain","context":{"idset":"10610","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3748868,"name":"drain","context":{"idset":"10611","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3756571,"name":"drain","context":{"idset":"10612","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3769879,"name":"drain","context":{"idset":"10742","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3776464,"name":"drain","context":{"idset":"10743","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3783202,"name":"drain","context":{"idset":"10744","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3790207,"name":"drain","context":{"idset":"10745","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3797064,"name":"drain","context":{"idset":"10746","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3805423,"name":"drain","context":{"idset":"10747","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3812046,"name":"drain","context":{"idset":"10748","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3819404,"name":"drain","context":{"idset":"10749","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3826568,"name":"drain","context":{"idset":"10750","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3833318,"name":"drain","context":{"idset":"10751","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.384012,"name":"drain","context":{"idset":"10752","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3847342,"name":"drain","context":{"idset":"10753","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3854253,"name":"drain","context":{"idset":"10754","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3861046,"name":"drain","context":{"idset":"10755","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3870304,"name":"drain","context":{"idset":"10756","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3885562,"name":"drain","context":{"idset":"10758","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3892276,"name":"drain","context":{"idset":"10759","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3898606,"name":"drain","context":{"idset":"10760","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3905506,"name":"drain","context":{"idset":"10761","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3912003,"name":"drain","context":{"idset":"10762","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3918478,"name":"drain","context":{"idset":"10763","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3924794,"name":"drain","context":{"idset":"10764","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3931131,"name":"drain","context":{"idset":"10765","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3947837,"name":"drain","context":{"idset":"10766","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.395452,"name":"drain","context":{"idset":"10767","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3962033,"name":"drain","context":{"idset":"10768","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3968582,"name":"drain","context":{"idset":"10769","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3974874,"name":"drain","context":{"idset":"10770","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3981576,"name":"drain","context":{"idset":"10771","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3989351,"name":"drain","context":{"idset":"10772","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.3996425,"name":"drain","context":{"idset":"10773","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4003255,"name":"drain","context":{"idset":"10774","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4010527,"name":"drain","context":{"idset":"10775","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4017422,"name":"drain","context":{"idset":"10776","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4024112,"name":"drain","context":{"idset":"10777","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4032459,"name":"drain","context":{"idset":"10778","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4039946,"name":"drain","context":{"idset":"10779","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4047318,"name":"drain","context":{"idset":"10780","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4054728,"name":"drain","context":{"idset":"10781","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4061332,"name":"drain","context":{"idset":"10782","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4068439,"name":"drain","context":{"idset":"10783","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4075408,"name":"drain","context":{"idset":"10784","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4082389,"name":"drain","context":{"idset":"10785","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4097347,"name":"drain","context":{"idset":"10787","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.410491,"name":"drain","context":{"idset":"10788","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4111774,"name":"drain","context":{"idset":"10789","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4119377,"name":"drain","context":{"idset":"10790","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.412993,"name":"drain","context":{"idset":"10791","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4137211,"name":"drain","context":{"idset":"10792","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4144263,"name":"drain","context":{"idset":"10793","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4151042,"name":"drain","context":{"idset":"10794","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4159729,"name":"drain","context":{"idset":"10795","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4173253,"name":"drain","context":{"idset":"10797","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4180102,"name":"drain","context":{"idset":"10798","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4188068,"name":"drain","context":{"idset":"10799","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4194548,"name":"drain","context":{"idset":"10800","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4200866,"name":"drain","context":{"idset":"10801","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4207356,"name":"drain","context":{"idset":"10802","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4214437,"name":"drain","context":{"idset":"10803","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4228704,"name":"drain","context":{"idset":"10805","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4236355,"name":"drain","context":{"idset":"10806","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4243872,"name":"drain","context":{"idset":"10807","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4251552,"name":"drain","context":{"idset":"10808","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4259276,"name":"drain","context":{"idset":"10809","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4266846,"name":"drain","context":{"idset":"10810","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4274397,"name":"drain","context":{"idset":"10811","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4281826,"name":"drain","context":{"idset":"10812","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4289408,"name":"drain","context":{"idset":"10813","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4296727,"name":"drain","context":{"idset":"10814","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4303629,"name":"drain","context":{"idset":"10815","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4310246,"name":"drain","context":{"idset":"10816","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4317398,"name":"drain","context":{"idset":"10817","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4323542,"name":"drain","context":{"idset":"10818","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4330842,"name":"drain","context":{"idset":"10819","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4338861,"name":"drain","context":{"idset":"10820","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4346492,"name":"drain","context":{"idset":"10821","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.435416,"name":"drain","context":{"idset":"10822","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4361839,"name":"drain","context":{"idset":"10823","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4369462,"name":"drain","context":{"idset":"10824","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4377201,"name":"drain","context":{"idset":"10825","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4385562,"name":"drain","context":{"idset":"10826","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4393222,"name":"drain","context":{"idset":"10827","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4400873,"name":"drain","context":{"idset":"10828","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4408145,"name":"drain","context":{"idset":"10829","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4415514,"name":"drain","context":{"idset":"10830","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4432015,"name":"drain","context":{"idset":"10831","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4468665,"name":"drain","context":{"idset":"10833","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4483485,"name":"drain","context":{"idset":"10834","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4498746,"name":"drain","context":{"idset":"10835","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4514556,"name":"drain","context":{"idset":"10836","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4530044,"name":"drain","context":{"idset":"10869","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4547071,"name":"drain","context":{"idset":"10870","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4562738,"name":"drain","context":{"idset":"10872","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4582243,"name":"drain","context":{"idset":"10873","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4596572,"name":"drain","context":{"idset":"10874","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4613819,"name":"drain","context":{"idset":"10875","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4627881,"name":"drain","context":{"idset":"10876","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4643116,"name":"drain","context":{"idset":"10877","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4654522,"name":"drain","context":{"idset":"10878","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4662533,"name":"drain","context":{"idset":"10879","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4670525,"name":"drain","context":{"idset":"10880","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4678516,"name":"drain","context":{"idset":"10881","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4686468,"name":"drain","context":{"idset":"10882","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4696405,"name":"drain","context":{"idset":"10883","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4704447,"name":"drain","context":{"idset":"10884","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4712358,"name":"drain","context":{"idset":"10885","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4719651,"name":"drain","context":{"idset":"10887","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4727702,"name":"drain","context":{"idset":"10888","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4735653,"name":"drain","context":{"idset":"10889","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4742692,"name":"drain","context":{"idset":"10890","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4750276,"name":"drain","context":{"idset":"10891","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4758477,"name":"drain","context":{"idset":"10892","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4765317,"name":"drain","context":{"idset":"10893","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4772632,"name":"drain","context":{"idset":"10894","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4779906,"name":"drain","context":{"idset":"10895","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4786465,"name":"drain","context":{"idset":"10896","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4793742,"name":"drain","context":{"idset":"10897","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.480077,"name":"drain","context":{"idset":"10898","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4808145,"name":"drain","context":{"idset":"10899","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4815516,"name":"drain","context":{"idset":"10900","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4822826,"name":"drain","context":{"idset":"10901","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4830072,"name":"drain","context":{"idset":"10902","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4837763,"name":"drain","context":{"idset":"10903","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4844949,"name":"drain","context":{"idset":"10904","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4864881,"name":"drain","context":{"idset":"10907","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4871569,"name":"drain","context":{"idset":"10908","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4878094,"name":"drain","context":{"idset":"10909","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4886243,"name":"drain","context":{"idset":"10910","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.489284,"name":"drain","context":{"idset":"10913","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4899786,"name":"drain","context":{"idset":"10914","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4907162,"name":"drain","context":{"idset":"10915","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4914048,"name":"drain","context":{"idset":"10916","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4920766,"name":"drain","context":{"idset":"10917","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4927423,"name":"drain","context":{"idset":"10918","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4934251,"name":"drain","context":{"idset":"10919","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4941397,"name":"drain","context":{"idset":"10920","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4949517,"name":"drain","context":{"idset":"10921","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4956431,"name":"drain","context":{"idset":"10922","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4963577,"name":"drain","context":{"idset":"10923","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4971218,"name":"drain","context":{"idset":"10924","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4978392,"name":"drain","context":{"idset":"10925","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4985492,"name":"drain","context":{"idset":"10926","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4992132,"name":"drain","context":{"idset":"10927","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.4999094,"name":"drain","context":{"idset":"10928","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5005817,"name":"drain","context":{"idset":"10929","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5012736,"name":"drain","context":{"idset":"10930","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5019968,"name":"drain","context":{"idset":"10931","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5027313,"name":"drain","context":{"idset":"10932","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5035186,"name":"drain","context":{"idset":"10933","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5042305,"name":"drain","context":{"idset":"10934","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5049505,"name":"drain","context":{"idset":"10935","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5056741,"name":"drain","context":{"idset":"10936","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.506387,"name":"drain","context":{"idset":"10937","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5071805,"name":"drain","context":{"idset":"10938","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5079291,"name":"drain","context":{"idset":"10939","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5086536,"name":"drain","context":{"idset":"10940","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5094101,"name":"drain","context":{"idset":"10941","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.510149,"name":"drain","context":{"idset":"10942","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5109775,"name":"drain","context":{"idset":"10943","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5117195,"name":"drain","context":{"idset":"10944","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.512455,"name":"drain","context":{"idset":"10945","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5132055,"name":"drain","context":{"idset":"10946","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5139494,"name":"drain","context":{"idset":"10947","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5147402,"name":"drain","context":{"idset":"10948","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5154896,"name":"drain","context":{"idset":"10949","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5162227,"name":"drain","context":{"idset":"10950","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.516917,"name":"drain","context":{"idset":"10951","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5194244,"name":"drain","context":{"idset":"10952","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5204279,"name":"drain","context":{"idset":"10953","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5214152,"name":"drain","context":{"idset":"10954","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5231087,"name":"drain","context":{"idset":"10955","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5246379,"name":"drain","context":{"idset":"10956","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5257483,"name":"drain","context":{"idset":"10957","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5268803,"name":"drain","context":{"idset":"10958","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5280797,"name":"drain","context":{"idset":"10959","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5292435,"name":"drain","context":{"idset":"10960","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5305285,"name":"drain","context":{"idset":"10961","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5318334,"name":"drain","context":{"idset":"10962","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5330911,"name":"drain","context":{"idset":"10963","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5342257,"name":"drain","context":{"idset":"10964","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5356715,"name":"drain","context":{"idset":"10965","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5373554,"name":"drain","context":{"idset":"10966","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5382643,"name":"drain","context":{"idset":"10967","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5395999,"name":"drain","context":{"idset":"10968","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5407925,"name":"drain","context":{"idset":"10969","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5421367,"name":"drain","context":{"idset":"10970","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5436044,"name":"drain","context":{"idset":"10971","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5451784,"name":"drain","context":{"idset":"10972","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5472317,"name":"drain","context":{"idset":"10973","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5483532,"name":"drain","context":{"idset":"10974","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5499618,"name":"drain","context":{"idset":"10975","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5515492,"name":"drain","context":{"idset":"10976","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5528789,"name":"drain","context":{"idset":"10977","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5542946,"name":"drain","context":{"idset":"10978","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5557637,"name":"drain","context":{"idset":"10979","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5570686,"name":"drain","context":{"idset":"10980","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5581162,"name":"drain","context":{"idset":"10981","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5591586,"name":"drain","context":{"idset":"10982","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5606191,"name":"drain","context":{"idset":"10983","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5617664,"name":"drain","context":{"idset":"10984","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.562763,"name":"drain","context":{"idset":"10985","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5635364,"name":"drain","context":{"idset":"10986","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5646999,"name":"drain","context":{"idset":"10987","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5657291,"name":"drain","context":{"idset":"10988","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5667839,"name":"drain","context":{"idset":"10989","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5678964,"name":"drain","context":{"idset":"10990","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5693498,"name":"drain","context":{"idset":"10991","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5706637,"name":"drain","context":{"idset":"10992","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5716524,"name":"drain","context":{"idset":"10993","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5726616,"name":"drain","context":{"idset":"10994","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5738223,"name":"drain","context":{"idset":"10995","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5749226,"name":"drain","context":{"idset":"10996","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5891516,"name":"drain","context":{"idset":"11157","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5897508,"name":"drain","context":{"idset":"11158","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5906279,"name":"drain","context":{"idset":"11159","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5914931,"name":"drain","context":{"idset":"11160","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.592314,"name":"drain","context":{"idset":"11161","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5931911,"name":"drain","context":{"idset":"11162","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.594203,"name":"drain","context":{"idset":"11163","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5950673,"name":"drain","context":{"idset":"11164","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5959294,"name":"drain","context":{"idset":"11165","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5967844,"name":"drain","context":{"idset":"11166","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5976548,"name":"drain","context":{"idset":"11167","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.59852,"name":"drain","context":{"idset":"11168","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.5993781,"name":"drain","context":{"idset":"11169","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6002505,"name":"drain","context":{"idset":"11170","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6013787,"name":"drain","context":{"idset":"11171","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6022067,"name":"drain","context":{"idset":"11172","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6030674,"name":"drain","context":{"idset":"11175","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6039093,"name":"drain","context":{"idset":"11176","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6047168,"name":"drain","context":{"idset":"11177","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6051571,"name":"drain","context":{"idset":"11178","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6055932,"name":"drain","context":{"idset":"11179","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6060183,"name":"drain","context":{"idset":"11180","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6066427,"name":"drain","context":{"idset":"11181","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6073895,"name":"drain","context":{"idset":"11182","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6082635,"name":"drain","context":{"idset":"11183","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6090174,"name":"drain","context":{"idset":"11184","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.609787,"name":"drain","context":{"idset":"11185","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6105771,"name":"drain","context":{"idset":"11186","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6113768,"name":"drain","context":{"idset":"11187","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6122303,"name":"drain","context":{"idset":"11188","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.612987,"name":"drain","context":{"idset":"11189","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6138051,"name":"drain","context":{"idset":"11190","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6146841,"name":"drain","context":{"idset":"11191","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6155779,"name":"drain","context":{"idset":"11192","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6164539,"name":"drain","context":{"idset":"11193","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6173141,"name":"drain","context":{"idset":"11194","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6180403,"name":"drain","context":{"idset":"11195","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6188531,"name":"drain","context":{"idset":"11196","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6197391,"name":"drain","context":{"idset":"11197","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6206214,"name":"drain","context":{"idset":"11198","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6215026,"name":"drain","context":{"idset":"11199","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6223729,"name":"drain","context":{"idset":"11200","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6232476,"name":"drain","context":{"idset":"11201","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6242516,"name":"drain","context":{"idset":"11202","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6251273,"name":"drain","context":{"idset":"11203","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6260109,"name":"drain","context":{"idset":"11204","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6268666,"name":"drain","context":{"idset":"11205","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6276445,"name":"drain","context":{"idset":"11206","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6284437,"name":"drain","context":{"idset":"11207","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6292162,"name":"drain","context":{"idset":"11208","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6299369,"name":"drain","context":{"idset":"11209","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6306694,"name":"drain","context":{"idset":"11210","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6313751,"name":"drain","context":{"idset":"11211","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6321084,"name":"drain","context":{"idset":"11212","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.632833,"name":"drain","context":{"idset":"11213","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6335218,"name":"drain","context":{"idset":"11214","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6341536,"name":"drain","context":{"idset":"11215","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6348433,"name":"drain","context":{"idset":"11216","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6355832,"name":"drain","context":{"idset":"11217","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6360271,"name":"drain","context":{"idset":"11218","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6365154,"name":"drain","context":{"idset":"11219","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6373053,"name":"drain","context":{"idset":"11220","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6381056,"name":"drain","context":{"idset":"11221","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.638907,"name":"drain","context":{"idset":"11222","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6396916,"name":"drain","context":{"idset":"11223","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6404746,"name":"drain","context":{"idset":"11224","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6412561,"name":"drain","context":{"idset":"11225","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6420529,"name":"drain","context":{"idset":"11226","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6427951,"name":"drain","context":{"idset":"11227","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6435363,"name":"drain","context":{"idset":"11228","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6442616,"name":"drain","context":{"idset":"11229","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6450257,"name":"drain","context":{"idset":"11230","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6458087,"name":"drain","context":{"idset":"11231","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6466007,"name":"drain","context":{"idset":"11232","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6473877,"name":"drain","context":{"idset":"11233","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6481757,"name":"drain","context":{"idset":"11234","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.649344,"name":"drain","context":{"idset":"11235","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6501386,"name":"drain","context":{"idset":"11236","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6509383,"name":"drain","context":{"idset":"11237","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6517637,"name":"drain","context":{"idset":"11238","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6525581,"name":"drain","context":{"idset":"11239","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6533463,"name":"drain","context":{"idset":"11240","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6542008,"name":"drain","context":{"idset":"11241","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6550143,"name":"drain","context":{"idset":"11242","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.655849,"name":"drain","context":{"idset":"11243","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6567733,"name":"drain","context":{"idset":"11244","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6575952,"name":"drain","context":{"idset":"11245","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6583927,"name":"drain","context":{"idset":"11246","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6591909,"name":"drain","context":{"idset":"11247","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6599851,"name":"drain","context":{"idset":"11248","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6607814,"name":"drain","context":{"idset":"11249","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6615689,"name":"drain","context":{"idset":"11250","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6623478,"name":"drain","context":{"idset":"11251","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6631436,"name":"drain","context":{"idset":"11252","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6640122,"name":"drain","context":{"idset":"11253","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6649065,"name":"drain","context":{"idset":"11254","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6657479,"name":"drain","context":{"idset":"11255","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6669991,"name":"drain","context":{"idset":"11256","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.667803,"name":"drain","context":{"idset":"11257","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6686072,"name":"drain","context":{"idset":"11258","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6694062,"name":"drain","context":{"idset":"11259","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6702125,"name":"drain","context":{"idset":"11260","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.671042,"name":"drain","context":{"idset":"11261","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6718421,"name":"drain","context":{"idset":"11262","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6726789,"name":"drain","context":{"idset":"11263","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6734924,"name":"drain","context":{"idset":"11264","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6742995,"name":"drain","context":{"idset":"11265","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6750948,"name":"drain","context":{"idset":"11266","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6758456,"name":"drain","context":{"idset":"11267","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6766286,"name":"drain","context":{"idset":"11268","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6774092,"name":"drain","context":{"idset":"11269","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6781652,"name":"drain","context":{"idset":"11270","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6789205,"name":"drain","context":{"idset":"11271","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6796877,"name":"drain","context":{"idset":"11272","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6805077,"name":"drain","context":{"idset":"11273","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.681289,"name":"drain","context":{"idset":"11274","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6820729,"name":"drain","context":{"idset":"11275","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6828883,"name":"drain","context":{"idset":"11276","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6837211,"name":"drain","context":{"idset":"11277","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6846209,"name":"drain","context":{"idset":"11278","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6855164,"name":"drain","context":{"idset":"11279","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6863096,"name":"drain","context":{"idset":"11280","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.687103,"name":"drain","context":{"idset":"11281","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6879213,"name":"drain","context":{"idset":"11282","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6887715,"name":"drain","context":{"idset":"11283","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6896114,"name":"drain","context":{"idset":"11284","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6905005,"name":"drain","context":{"idset":"11285","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6913316,"name":"drain","context":{"idset":"11286","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6921604,"name":"drain","context":{"idset":"11287","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6929648,"name":"drain","context":{"idset":"11288","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6937521,"name":"drain","context":{"idset":"11289","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6948855,"name":"drain","context":{"idset":"11290","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6957214,"name":"drain","context":{"idset":"11291","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6965785,"name":"drain","context":{"idset":"11292","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6973817,"name":"drain","context":{"idset":"11293","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6982257,"name":"drain","context":{"idset":"11294","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6990471,"name":"drain","context":{"idset":"11295","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.6998746,"name":"drain","context":{"idset":"11296","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.700716,"name":"drain","context":{"idset":"11297","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7015886,"name":"drain","context":{"idset":"11298","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7024364,"name":"drain","context":{"idset":"11299","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.703263,"name":"drain","context":{"idset":"11300","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7040951,"name":"drain","context":{"idset":"11301","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7049065,"name":"drain","context":{"idset":"11302","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7058074,"name":"drain","context":{"idset":"11303","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7067287,"name":"drain","context":{"idset":"11304","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7076378,"name":"drain","context":{"idset":"11305","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7084641,"name":"drain","context":{"idset":"11306","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7092736,"name":"drain","context":{"idset":"11307","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7100933,"name":"drain","context":{"idset":"11308","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7109213,"name":"drain","context":{"idset":"11309","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7117696,"name":"drain","context":{"idset":"11310","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7126141,"name":"drain","context":{"idset":"11311","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7134395,"name":"drain","context":{"idset":"11312","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7142987,"name":"drain","context":{"idset":"11313","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7151473,"name":"drain","context":{"idset":"11314","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7159693,"name":"drain","context":{"idset":"11315","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7168384,"name":"drain","context":{"idset":"11316","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7176626,"name":"drain","context":{"idset":"11319","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7185509,"name":"drain","context":{"idset":"11320","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7193894,"name":"drain","context":{"idset":"11321","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7202344,"name":"drain","context":{"idset":"11322","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7210872,"name":"drain","context":{"idset":"11323","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7219379,"name":"drain","context":{"idset":"11324","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7227912,"name":"drain","context":{"idset":"11325","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7236772,"name":"drain","context":{"idset":"11326","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.72453,"name":"drain","context":{"idset":"11327","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7253735,"name":"drain","context":{"idset":"11328","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7262185,"name":"drain","context":{"idset":"11329","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7270534,"name":"drain","context":{"idset":"11331","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7278821,"name":"drain","context":{"idset":"11332","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7287097,"name":"drain","context":{"idset":"11333","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7296131,"name":"drain","context":{"idset":"11334","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7305033,"name":"drain","context":{"idset":"11335","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7313645,"name":"drain","context":{"idset":"11336","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7322342,"name":"drain","context":{"idset":"11337","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7331452,"name":"drain","context":{"idset":"11338","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7340038,"name":"drain","context":{"idset":"11339","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7348406,"name":"drain","context":{"idset":"11340","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7356975,"name":"drain","context":{"idset":"11341","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7365856,"name":"drain","context":{"idset":"11342","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7374482,"name":"drain","context":{"idset":"11343","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7382832,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7391167,"name":"drain","context":{"idset":"11345","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.740042,"name":"drain","context":{"idset":"11346","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7409012,"name":"drain","context":{"idset":"11347","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7417147,"name":"drain","context":{"idset":"11348","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.742522,"name":"drain","context":{"idset":"11349","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7433016,"name":"drain","context":{"idset":"11350","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7441294,"name":"drain","context":{"idset":"11351","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7449949,"name":"drain","context":{"idset":"11352","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7459545,"name":"drain","context":{"idset":"11353","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7469089,"name":"drain","context":{"idset":"11354","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7478616,"name":"drain","context":{"idset":"11355","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7488129,"name":"drain","context":{"idset":"11356","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7497623,"name":"drain","context":{"idset":"11357","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7509179,"name":"drain","context":{"idset":"11358","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7518847,"name":"drain","context":{"idset":"11359","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7528374,"name":"drain","context":{"idset":"11360","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7537308,"name":"drain","context":{"idset":"11361","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7545819,"name":"drain","context":{"idset":"11362","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7554591,"name":"drain","context":{"idset":"11363","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7564318,"name":"drain","context":{"idset":"11364","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7573259,"name":"drain","context":{"idset":"11365","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7581871,"name":"drain","context":{"idset":"11366","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7591572,"name":"drain","context":{"idset":"11367","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.760112,"name":"drain","context":{"idset":"11368","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7609518,"name":"drain","context":{"idset":"11369","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7617881,"name":"drain","context":{"idset":"11370","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.762639,"name":"drain","context":{"idset":"11371","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.763489,"name":"drain","context":{"idset":"11372","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7643514,"name":"drain","context":{"idset":"11373","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7651522,"name":"drain","context":{"idset":"11374","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7659426,"name":"drain","context":{"idset":"11375","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7667835,"name":"drain","context":{"idset":"11376","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7676272,"name":"drain","context":{"idset":"11377","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7684815,"name":"drain","context":{"idset":"11378","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7693639,"name":"drain","context":{"idset":"11379","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7713802,"name":"drain","context":{"idset":"11380","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7738781,"name":"drain","context":{"idset":"11383","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7747688,"name":"drain","context":{"idset":"11384","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7756054,"name":"drain","context":{"idset":"11385","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7764227,"name":"drain","context":{"idset":"11386","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7772832,"name":"drain","context":{"idset":"11389","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7781432,"name":"drain","context":{"idset":"11390","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7789931,"name":"drain","context":{"idset":"11391","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7798281,"name":"drain","context":{"idset":"11392","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7806766,"name":"drain","context":{"idset":"11393","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7815275,"name":"drain","context":{"idset":"11394","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7823181,"name":"drain","context":{"idset":"11395","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.783165,"name":"drain","context":{"idset":"11396","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7840011,"name":"drain","context":{"idset":"11397","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7848673,"name":"drain","context":{"idset":"11398","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7857177,"name":"drain","context":{"idset":"11399","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7864349,"name":"drain","context":{"idset":"11400","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7871354,"name":"drain","context":{"idset":"11401","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7878435,"name":"drain","context":{"idset":"11402","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7885523,"name":"drain","context":{"idset":"11404","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7892518,"name":"drain","context":{"idset":"11405","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7900887,"name":"drain","context":{"idset":"11406","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7909517,"name":"drain","context":{"idset":"11407","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7918444,"name":"drain","context":{"idset":"11408","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7927401,"name":"drain","context":{"idset":"11409","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7936668,"name":"drain","context":{"idset":"11410","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7945628,"name":"drain","context":{"idset":"11411","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7954464,"name":"drain","context":{"idset":"11412","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7963233,"name":"drain","context":{"idset":"11413","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7972093,"name":"drain","context":{"idset":"11414","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7980816,"name":"drain","context":{"idset":"11415","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.7989514,"name":"drain","context":{"idset":"11416","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8006811,"name":"drain","context":{"idset":"11418","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.801528,"name":"drain","context":{"idset":"11419","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.802325,"name":"drain","context":{"idset":"11420","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8031952,"name":"drain","context":{"idset":"11421","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8041372,"name":"drain","context":{"idset":"11422","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8050284,"name":"drain","context":{"idset":"11423","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8059421,"name":"drain","context":{"idset":"11424","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8068275,"name":"drain","context":{"idset":"11425","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8077354,"name":"drain","context":{"idset":"11426","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8085849,"name":"drain","context":{"idset":"11427","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8094218,"name":"drain","context":{"idset":"11428","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8102601,"name":"drain","context":{"idset":"11429","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8111579,"name":"drain","context":{"idset":"11430","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8120186,"name":"drain","context":{"idset":"11431","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8128703,"name":"drain","context":{"idset":"11432","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8137786,"name":"drain","context":{"idset":"11433","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8146782,"name":"drain","context":{"idset":"11434","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8156159,"name":"drain","context":{"idset":"11435","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8165164,"name":"drain","context":{"idset":"11436","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.817409,"name":"drain","context":{"idset":"11437","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8182616,"name":"drain","context":{"idset":"11438","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8191555,"name":"drain","context":{"idset":"11440","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8200519,"name":"drain","context":{"idset":"11441","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8209813,"name":"drain","context":{"idset":"11442","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8218842,"name":"drain","context":{"idset":"11443","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8227746,"name":"drain","context":{"idset":"11444","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8236468,"name":"drain","context":{"idset":"11445","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8245139,"name":"drain","context":{"idset":"11446","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8253784,"name":"drain","context":{"idset":"11447","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8261838,"name":"drain","context":{"idset":"11448","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8270147,"name":"drain","context":{"idset":"11449","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8278799,"name":"drain","context":{"idset":"11450","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8287663,"name":"drain","context":{"idset":"11451","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8297007,"name":"drain","context":{"idset":"11452","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8306162,"name":"drain","context":{"idset":"11453","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8315661,"name":"drain","context":{"idset":"11454","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8324704,"name":"drain","context":{"idset":"11455","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8333681,"name":"drain","context":{"idset":"11456","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8342683,"name":"drain","context":{"idset":"11457","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8351686,"name":"drain","context":{"idset":"11458","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8360796,"name":"drain","context":{"idset":"11459","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8369684,"name":"drain","context":{"idset":"11461","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8378525,"name":"drain","context":{"idset":"11462","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8387308,"name":"drain","context":{"idset":"11463","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8396258,"name":"drain","context":{"idset":"11464","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8404713,"name":"drain","context":{"idset":"11465","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8413599,"name":"drain","context":{"idset":"11467","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8422604,"name":"drain","context":{"idset":"11468","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8431501,"name":"drain","context":{"idset":"11469","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8440495,"name":"drain","context":{"idset":"11470","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8449426,"name":"drain","context":{"idset":"11472","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.845834,"name":"drain","context":{"idset":"11473","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8467212,"name":"drain","context":{"idset":"11474","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8476095,"name":"drain","context":{"idset":"11475","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8485229,"name":"drain","context":{"idset":"11476","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8494375,"name":"drain","context":{"idset":"11477","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8504064,"name":"drain","context":{"idset":"11478","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.851279,"name":"drain","context":{"idset":"11479","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8522575,"name":"drain","context":{"idset":"11480","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8531055,"name":"drain","context":{"idset":"11481","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8539455,"name":"drain","context":{"idset":"11482","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8548,"name":"drain","context":{"idset":"11483","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8556988,"name":"drain","context":{"idset":"11484","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8565514,"name":"drain","context":{"idset":"11485","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8573911,"name":"drain","context":{"idset":"11486","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8583002,"name":"drain","context":{"idset":"11487","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8591919,"name":"drain","context":{"idset":"11488","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8600814,"name":"drain","context":{"idset":"11489","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8610039,"name":"drain","context":{"idset":"11490","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8618779,"name":"drain","context":{"idset":"11491","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8627224,"name":"drain","context":{"idset":"11492","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8635724,"name":"drain","context":{"idset":"11493","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8644898,"name":"drain","context":{"idset":"11494","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8654335,"name":"drain","context":{"idset":"11495","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.866354,"name":"drain","context":{"idset":"11496","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8672559,"name":"drain","context":{"idset":"11497","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8681912,"name":"drain","context":{"idset":"11498","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.869103,"name":"drain","context":{"idset":"11499","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8700061,"name":"drain","context":{"idset":"11500","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8709049,"name":"drain","context":{"idset":"11501","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8718345,"name":"drain","context":{"idset":"11502","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8727145,"name":"drain","context":{"idset":"11503","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8737223,"name":"drain","context":{"idset":"11504","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8746645,"name":"drain","context":{"idset":"11505","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.875694,"name":"drain","context":{"idset":"11506","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8767233,"name":"drain","context":{"idset":"11508","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8777416,"name":"drain","context":{"idset":"11509","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8787465,"name":"drain","context":{"idset":"11510","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8796964,"name":"drain","context":{"idset":"11511","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.880748,"name":"drain","context":{"idset":"11512","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8817775,"name":"drain","context":{"idset":"11513","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8828058,"name":"drain","context":{"idset":"11514","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8838191,"name":"drain","context":{"idset":"11515","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8847611,"name":"drain","context":{"idset":"11516","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.885607,"name":"drain","context":{"idset":"11518","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8864586,"name":"drain","context":{"idset":"11519","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8874149,"name":"drain","context":{"idset":"11520","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8884397,"name":"drain","context":{"idset":"11521","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8897202,"name":"drain","context":{"idset":"11522","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.89062,"name":"drain","context":{"idset":"11523","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.89149,"name":"drain","context":{"idset":"11524","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8923569,"name":"drain","context":{"idset":"11525","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8932726,"name":"drain","context":{"idset":"11526","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8949468,"name":"drain","context":{"idset":"11528","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8958304,"name":"drain","context":{"idset":"11529","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8967347,"name":"drain","context":{"idset":"11530","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8976376,"name":"drain","context":{"idset":"11531","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8985436,"name":"drain","context":{"idset":"11532","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.8994722,"name":"drain","context":{"idset":"11534","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.900347,"name":"drain","context":{"idset":"11535","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9012654,"name":"drain","context":{"idset":"11537","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9021657,"name":"drain","context":{"idset":"11538","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9030745,"name":"drain","context":{"idset":"11539","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9039884,"name":"drain","context":{"idset":"11540","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9048347,"name":"drain","context":{"idset":"11541","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9056971,"name":"drain","context":{"idset":"11542","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9065795,"name":"drain","context":{"idset":"11543","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.907474,"name":"drain","context":{"idset":"11544","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9083641,"name":"drain","context":{"idset":"11545","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9092846,"name":"drain","context":{"idset":"11546","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9101894,"name":"drain","context":{"idset":"11547","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9111967,"name":"drain","context":{"idset":"11548","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9120445,"name":"drain","context":{"idset":"11549","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.912874,"name":"drain","context":{"idset":"11550","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9136961,"name":"drain","context":{"idset":"11551","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9145219,"name":"drain","context":{"idset":"11552","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9153473,"name":"drain","context":{"idset":"11553","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9162686,"name":"drain","context":{"idset":"11554","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9224265,"name":"drain","context":{"idset":"11555","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9234829,"name":"drain","context":{"idset":"11556","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9245389,"name":"drain","context":{"idset":"11557","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9256041,"name":"drain","context":{"idset":"11558","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9266512,"name":"drain","context":{"idset":"11559","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9276841,"name":"drain","context":{"idset":"11560","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9286895,"name":"drain","context":{"idset":"11561","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9297462,"name":"drain","context":{"idset":"11562","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9307995,"name":"drain","context":{"idset":"11563","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9318008,"name":"drain","context":{"idset":"11564","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9327316,"name":"drain","context":{"idset":"11565","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9345765,"name":"drain","context":{"idset":"11567","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9363933,"name":"drain","context":{"idset":"11568","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9373949,"name":"drain","context":{"idset":"11570","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9383559,"name":"drain","context":{"idset":"11571","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9393115,"name":"drain","context":{"idset":"11572","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9562876,"name":"drain","context":{"idset":"11573","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.957289,"name":"drain","context":{"idset":"11574","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9584115,"name":"drain","context":{"idset":"11575","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9744682,"name":"drain","context":{"idset":"11576","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9754434,"name":"drain","context":{"idset":"11577","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9764154,"name":"drain","context":{"idset":"11578","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9773655,"name":"drain","context":{"idset":"11579","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9783056,"name":"drain","context":{"idset":"11580","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9792244,"name":"drain","context":{"idset":"11581","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9902475,"name":"drain","context":{"idset":"11582","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9912097,"name":"drain","context":{"idset":"11583","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9921758,"name":"drain","context":{"idset":"11584","reason":"broker was unresponsive"}}
+{"timestamp":1712154692.9931383,"name":"drain","context":{"idset":"11585","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0131507,"name":"drain","context":{"idset":"11586","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.014164,"name":"drain","context":{"idset":"11589","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0151422,"name":"drain","context":{"idset":"11590","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.016113,"name":"drain","context":{"idset":"11591","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0198882,"name":"drain","context":{"idset":"11594","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.020884,"name":"drain","context":{"idset":"11595","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0218966,"name":"drain","context":{"idset":"11596","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0228806,"name":"drain","context":{"idset":"11597","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0250845,"name":"drain","context":{"idset":"11598","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0272567,"name":"drain","context":{"idset":"11599","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0281804,"name":"drain","context":{"idset":"11600","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0290883,"name":"drain","context":{"idset":"11601","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0299966,"name":"drain","context":{"idset":"11602","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0328157,"name":"drain","context":{"idset":"11604","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0337353,"name":"drain","context":{"idset":"11605","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0346334,"name":"drain","context":{"idset":"11606","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0386083,"name":"drain","context":{"idset":"11607","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0394509,"name":"drain","context":{"idset":"11608","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0410652,"name":"drain","context":{"idset":"11610","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0418851,"name":"drain","context":{"idset":"11611","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0439832,"name":"drain","context":{"idset":"11613","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0448203,"name":"drain","context":{"idset":"11614","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0456433,"name":"drain","context":{"idset":"11615","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0464704,"name":"drain","context":{"idset":"11616","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0472839,"name":"drain","context":{"idset":"11617","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0482194,"name":"drain","context":{"idset":"11618","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0500429,"name":"drain","context":{"idset":"11620","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0508971,"name":"drain","context":{"idset":"11621","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0517426,"name":"drain","context":{"idset":"11622","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0525949,"name":"drain","context":{"idset":"11623","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0546894,"name":"drain","context":{"idset":"11624","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0555315,"name":"drain","context":{"idset":"11625","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0564125,"name":"drain","context":{"idset":"11626","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0572789,"name":"drain","context":{"idset":"11627","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.058116,"name":"drain","context":{"idset":"11628","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0590928,"name":"drain","context":{"idset":"11629","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0598869,"name":"drain","context":{"idset":"11630","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0617664,"name":"drain","context":{"idset":"11632","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0629814,"name":"drain","context":{"idset":"11634","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0641026,"name":"drain","context":{"idset":"11635","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0652032,"name":"drain","context":{"idset":"11636","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0662534,"name":"drain","context":{"idset":"11637","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0686688,"name":"drain","context":{"idset":"11638","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0697999,"name":"drain","context":{"idset":"11639","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0708745,"name":"drain","context":{"idset":"11640","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0720477,"name":"drain","context":{"idset":"11641","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0730958,"name":"drain","context":{"idset":"11642","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.074157,"name":"drain","context":{"idset":"11643","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0752761,"name":"drain","context":{"idset":"11644","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0763602,"name":"drain","context":{"idset":"11645","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0774431,"name":"drain","context":{"idset":"11646","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.079812,"name":"drain","context":{"idset":"11647","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.080842,"name":"drain","context":{"idset":"11648","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0818453,"name":"drain","context":{"idset":"11649","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0829546,"name":"drain","context":{"idset":"11650","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0841453,"name":"drain","context":{"idset":"11651","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0852122,"name":"drain","context":{"idset":"11652","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0862155,"name":"drain","context":{"idset":"11653","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.087213,"name":"drain","context":{"idset":"11654","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.089514,"name":"drain","context":{"idset":"11655","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0906324,"name":"drain","context":{"idset":"11656","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0917158,"name":"drain","context":{"idset":"11657","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0928659,"name":"drain","context":{"idset":"11658","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0939064,"name":"drain","context":{"idset":"11659","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0949571,"name":"drain","context":{"idset":"11660","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.09605,"name":"drain","context":{"idset":"11661","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0983422,"name":"drain","context":{"idset":"11662","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.0992951,"name":"drain","context":{"idset":"11663","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1002965,"name":"drain","context":{"idset":"11664","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1011734,"name":"drain","context":{"idset":"11665","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1033723,"name":"drain","context":{"idset":"11666","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1043112,"name":"drain","context":{"idset":"11667","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1053383,"name":"drain","context":{"idset":"11668","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1066477,"name":"drain","context":{"idset":"11671","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1077561,"name":"drain","context":{"idset":"11672","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1088331,"name":"drain","context":{"idset":"11673","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1112273,"name":"drain","context":{"idset":"11674","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1123614,"name":"drain","context":{"idset":"11675","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1140089,"name":"drain","context":{"idset":"11676","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1151218,"name":"drain","context":{"idset":"11677","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1162241,"name":"drain","context":{"idset":"11678","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.117327,"name":"drain","context":{"idset":"11679","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.130173,"name":"drain","context":{"idset":"11680","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1311958,"name":"drain","context":{"idset":"11681","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1321731,"name":"drain","context":{"idset":"11682","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1331494,"name":"drain","context":{"idset":"11683","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1341619,"name":"drain","context":{"idset":"11684","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1351497,"name":"drain","context":{"idset":"11685","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1361265,"name":"drain","context":{"idset":"11686","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1370687,"name":"drain","context":{"idset":"11687","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1380403,"name":"drain","context":{"idset":"11688","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1510615,"name":"drain","context":{"idset":"11689","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1520512,"name":"drain","context":{"idset":"11690","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1530178,"name":"drain","context":{"idset":"11691","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1540155,"name":"drain","context":{"idset":"11692","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1551945,"name":"drain","context":{"idset":"11693","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1561847,"name":"drain","context":{"idset":"11694","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1571891,"name":"drain","context":{"idset":"11695","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1581762,"name":"drain","context":{"idset":"11696","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1637199,"name":"drain","context":{"idset":"11697","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1648033,"name":"drain","context":{"idset":"11698","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1659079,"name":"drain","context":{"idset":"11699","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1669827,"name":"drain","context":{"idset":"11700","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1680729,"name":"drain","context":{"idset":"11701","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1720645,"name":"drain","context":{"idset":"11702","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1731017,"name":"drain","context":{"idset":"11703","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1741474,"name":"drain","context":{"idset":"11704","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1751597,"name":"drain","context":{"idset":"11705","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1798201,"name":"drain","context":{"idset":"11706","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1808448,"name":"drain","context":{"idset":"11707","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1817787,"name":"drain","context":{"idset":"11708","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1827078,"name":"drain","context":{"idset":"11709","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1836538,"name":"drain","context":{"idset":"11710","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1846871,"name":"drain","context":{"idset":"11712","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1857395,"name":"drain","context":{"idset":"11713","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1868286,"name":"drain","context":{"idset":"11714","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1879213,"name":"drain","context":{"idset":"11715","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1890233,"name":"drain","context":{"idset":"11716","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1901295,"name":"drain","context":{"idset":"11717","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1912267,"name":"drain","context":{"idset":"11718","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1923048,"name":"drain","context":{"idset":"11719","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1934376,"name":"drain","context":{"idset":"11720","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.19455,"name":"drain","context":{"idset":"11721","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1956434,"name":"drain","context":{"idset":"11722","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1967309,"name":"drain","context":{"idset":"11723","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1978269,"name":"drain","context":{"idset":"11724","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.1989326,"name":"drain","context":{"idset":"11725","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2006493,"name":"drain","context":{"idset":"11726","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2017837,"name":"drain","context":{"idset":"11727","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2028847,"name":"drain","context":{"idset":"11728","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.204004,"name":"drain","context":{"idset":"11729","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2050366,"name":"drain","context":{"idset":"11730","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2060685,"name":"drain","context":{"idset":"11731","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2071798,"name":"drain","context":{"idset":"11732","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2083478,"name":"drain","context":{"idset":"11733","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2094789,"name":"drain","context":{"idset":"11734","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.210587,"name":"drain","context":{"idset":"11735","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2117074,"name":"drain","context":{"idset":"11736","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2127905,"name":"drain","context":{"idset":"11737","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2139282,"name":"drain","context":{"idset":"11738","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2150459,"name":"drain","context":{"idset":"11739","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.216145,"name":"drain","context":{"idset":"11740","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2200994,"name":"drain","context":{"idset":"11741","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2211487,"name":"drain","context":{"idset":"11742","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2222831,"name":"drain","context":{"idset":"11743","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2233424,"name":"drain","context":{"idset":"11744","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2243903,"name":"drain","context":{"idset":"11745","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2299895,"name":"drain","context":{"idset":"11746","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2310464,"name":"drain","context":{"idset":"11747","reason":"broker was unresponsive"}}
+{"timestamp":1712154693.2321258,"name":"drain","context":{"idset":"11748","reason":"broker was unresponsive"}}
+{"timestamp":1712154776.915616,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1712154837.6094444,"name":"offline","context":{"idset":"10517"}}
+{"timestamp":1712154873.9160409,"name":"offline","context":{"idset":"10557"}}
+{"timestamp":1712154874.0097699,"name":"offline","context":{"idset":"10558"}}
+{"timestamp":1712156751.1098735,"name":"undrain","context":{"idset":"10741-10868"}}
+{"timestamp":1712160521.3064172,"name":"offline","context":{"idset":"594"}}
+{"timestamp":1712161524.5548441,"name":"drain","context":{"idset":"10742","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712161525.023258,"name":"drain","context":{"idset":"10745","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712161552.3722243,"name":"drain","context":{"idset":"10754","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712161827.7108033,"name":"drain","context":{"idset":"10746","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712161829.3900695,"name":"undrain","context":{"idset":"10357-10372"}}
+{"timestamp":1712161829.9059019,"name":"drain","context":{"idset":"10760","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712161830.8916755,"name":"drain","context":{"idset":"10756","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712161830.893002,"name":"online","context":{"idset":"10357"}}
+{"timestamp":1712162025.3048179,"name":"drain","context":{"idset":"10742","reason":"epilog failed for jobid fnPT1xJ292F","overwrite":0}}
+{"timestamp":1712162025.3049252,"name":"drain","context":{"idset":"10745","reason":"epilog failed for jobid fnPT2gUDi1D","overwrite":0}}
+{"timestamp":1712162025.305017,"name":"drain","context":{"idset":"10746","reason":"epilog failed for jobid fnPT2vTAw9q","overwrite":0}}
+{"timestamp":1712162178.7159617,"name":"online","context":{"idset":"11711"}}
+{"timestamp":1712162334.132664,"name":"undrain","context":{"idset":"11712"}}
+{"timestamp":1712162360.5720921,"name":"undrain","context":{"idset":"11713-11746"}}
+{"timestamp":1712162683.1316421,"name":"undrain","context":{"idset":"11637"}}
+{"timestamp":1712162685.9411156,"name":"undrain","context":{"idset":"11638"}}
+{"timestamp":1712162711.6362917,"name":"undrain","context":{"idset":"11639"}}
+{"timestamp":1712162736.2156773,"name":"undrain","context":{"idset":"11640"}}
+{"timestamp":1712162737.6866121,"name":"undrain","context":{"idset":"11641"}}
+{"timestamp":1712162738.9760704,"name":"undrain","context":{"idset":"11642"}}
+{"timestamp":1712162739.717273,"name":"undrain","context":{"idset":"11643"}}
+{"timestamp":1712162740.4448924,"name":"undrain","context":{"idset":"11644"}}
+{"timestamp":1712162741.1542842,"name":"undrain","context":{"idset":"11645"}}
+{"timestamp":1712162741.8970594,"name":"undrain","context":{"idset":"11646"}}
+{"timestamp":1712162742.8925748,"name":"undrain","context":{"idset":"11647"}}
+{"timestamp":1712162743.933378,"name":"undrain","context":{"idset":"11648"}}
+{"timestamp":1712162745.0904312,"name":"undrain","context":{"idset":"11649"}}
+{"timestamp":1712162746.5289166,"name":"undrain","context":{"idset":"11650"}}
+{"timestamp":1712162747.7423022,"name":"undrain","context":{"idset":"11651"}}
+{"timestamp":1712162748.5845044,"name":"undrain","context":{"idset":"11652"}}
+{"timestamp":1712162749.6727769,"name":"undrain","context":{"idset":"11653"}}
+{"timestamp":1712162750.7926965,"name":"undrain","context":{"idset":"11654"}}
+{"timestamp":1712162751.9299181,"name":"undrain","context":{"idset":"11655"}}
+{"timestamp":1712162754.2495587,"name":"undrain","context":{"idset":"11656"}}
+{"timestamp":1712162755.4810698,"name":"undrain","context":{"idset":"11657"}}
+{"timestamp":1712162756.6260636,"name":"undrain","context":{"idset":"11658"}}
+{"timestamp":1712162757.2441666,"name":"undrain","context":{"idset":"11659"}}
+{"timestamp":1712162758.1166451,"name":"undrain","context":{"idset":"11660"}}
+{"timestamp":1712162759.0952184,"name":"undrain","context":{"idset":"11661"}}
+{"timestamp":1712162760.0694506,"name":"undrain","context":{"idset":"11662"}}
+{"timestamp":1712162761.1227829,"name":"undrain","context":{"idset":"11663"}}
+{"timestamp":1712162762.1024659,"name":"undrain","context":{"idset":"11664"}}
+{"timestamp":1712162763.8690095,"name":"undrain","context":{"idset":"11665"}}
+{"timestamp":1712162764.8729465,"name":"undrain","context":{"idset":"11666"}}
+{"timestamp":1712162765.873136,"name":"undrain","context":{"idset":"11667"}}
+{"timestamp":1712162766.8067451,"name":"undrain","context":{"idset":"11668"}}
+{"timestamp":1712162768.3715818,"name":"undrain","context":{"idset":"11670"}}
+{"timestamp":1712162769.984812,"name":"undrain","context":{"idset":"11671"}}
+{"timestamp":1712162770.8474088,"name":"undrain","context":{"idset":"11672"}}
+{"timestamp":1712162771.7838988,"name":"undrain","context":{"idset":"11673"}}
+{"timestamp":1712162772.8240509,"name":"undrain","context":{"idset":"11674"}}
+{"timestamp":1712162773.8123553,"name":"undrain","context":{"idset":"11675"}}
+{"timestamp":1712162774.752578,"name":"undrain","context":{"idset":"11676"}}
+{"timestamp":1712162775.7649007,"name":"undrain","context":{"idset":"11677"}}
+{"timestamp":1712162776.7762794,"name":"undrain","context":{"idset":"11678"}}
+{"timestamp":1712162777.7977958,"name":"undrain","context":{"idset":"11679"}}
+{"timestamp":1712162778.7816606,"name":"undrain","context":{"idset":"11680"}}
+{"timestamp":1712162779.7282665,"name":"undrain","context":{"idset":"11681"}}
+{"timestamp":1712162780.8183284,"name":"undrain","context":{"idset":"11682"}}
+{"timestamp":1712162781.8311524,"name":"undrain","context":{"idset":"11683"}}
+{"timestamp":1712162782.8254244,"name":"undrain","context":{"idset":"11684"}}
+{"timestamp":1712162784.0558829,"name":"undrain","context":{"idset":"11685"}}
+{"timestamp":1712162785.4248264,"name":"undrain","context":{"idset":"11686"}}
+{"timestamp":1712162786.1226959,"name":"undrain","context":{"idset":"11687"}}
+{"timestamp":1712162786.8473275,"name":"undrain","context":{"idset":"11688"}}
+{"timestamp":1712162787.7808318,"name":"undrain","context":{"idset":"11689"}}
+{"timestamp":1712162788.7442396,"name":"undrain","context":{"idset":"11690"}}
+{"timestamp":1712162789.7333093,"name":"undrain","context":{"idset":"11691"}}
+{"timestamp":1712162790.7900062,"name":"undrain","context":{"idset":"11692"}}
+{"timestamp":1712162791.8299732,"name":"undrain","context":{"idset":"11693"}}
+{"timestamp":1712162792.882653,"name":"undrain","context":{"idset":"11694"}}
+{"timestamp":1712162793.9136081,"name":"undrain","context":{"idset":"11695"}}
+{"timestamp":1712162794.9759068,"name":"undrain","context":{"idset":"11696"}}
+{"timestamp":1712162795.9590869,"name":"undrain","context":{"idset":"11697"}}
+{"timestamp":1712162796.9659607,"name":"undrain","context":{"idset":"11698"}}
+{"timestamp":1712162797.9773636,"name":"undrain","context":{"idset":"11699"}}
+{"timestamp":1712162799.1345112,"name":"undrain","context":{"idset":"11700"}}
+{"timestamp":1712162800.5276077,"name":"undrain","context":{"idset":"11701"}}
+{"timestamp":1712162801.2879083,"name":"undrain","context":{"idset":"11702"}}
+{"timestamp":1712162802.2991984,"name":"undrain","context":{"idset":"11703"}}
+{"timestamp":1712162803.3091686,"name":"undrain","context":{"idset":"11704"}}
+{"timestamp":1712162804.309413,"name":"undrain","context":{"idset":"11705"}}
+{"timestamp":1712162805.817034,"name":"undrain","context":{"idset":"11706"}}
+{"timestamp":1712162806.9998388,"name":"undrain","context":{"idset":"11707"}}
+{"timestamp":1712162808.1464269,"name":"undrain","context":{"idset":"11708"}}
+{"timestamp":1712162809.1428924,"name":"undrain","context":{"idset":"11709"}}
+{"timestamp":1712162810.2589307,"name":"undrain","context":{"idset":"11710"}}
+{"timestamp":1712162880.8958092,"name":"undrain","context":{"idset":"11747"}}
+{"timestamp":1712162881.9824996,"name":"undrain","context":{"idset":"11748"}}
+{"timestamp":1712163281.6492143,"name":"undrain","context":{"idset":"10101"}}
+{"timestamp":1712163520.0446463,"name":"undrain","context":{"idset":"10102"}}
+{"timestamp":1712163949.8727562,"name":"online","context":{"idset":"10102,10137"}}
+{"timestamp":1712163950.0734816,"name":"online","context":{"idset":"10123,10125"}}
+{"timestamp":1712163950.1738732,"name":"online","context":{"idset":"10151"}}
+{"timestamp":1712163950.2946405,"name":"online","context":{"idset":"10120,10127,10159"}}
+{"timestamp":1712163950.551857,"name":"online","context":{"idset":"10122,10128-10129,10152"}}
+{"timestamp":1712163950.6667671,"name":"online","context":{"idset":"10124,10140,10157,10160,10202"}}
+{"timestamp":1712163950.7674427,"name":"online","context":{"idset":"10126,10156,10158"}}
+{"timestamp":1712165100.1289933,"name":"offline","context":{"idset":"10936"}}
+{"timestamp":1712166188.6337581,"name":"undrain","context":{"idset":"10917-10920"}}
+{"timestamp":1712166978.8280637,"name":"offline","context":{"idset":"589"}}
+{"timestamp":1712166978.8293383,"name":"offline","context":{"idset":"595"}}
+{"timestamp":1712166978.8305993,"name":"offline","context":{"idset":"593"}}
+{"timestamp":1712166978.8318667,"name":"offline","context":{"idset":"590"}}
+{"timestamp":1712166978.8331201,"name":"offline","context":{"idset":"596"}}
+{"timestamp":1712166978.8343704,"name":"offline","context":{"idset":"591"}}
+{"timestamp":1712166978.8356245,"name":"offline","context":{"idset":"592"}}
+{"timestamp":1712167414.6142466,"name":"drain","context":{"idset":"10917-10920","overwrite":0}}
+{"timestamp":1712167584.2672238,"name":"drain","context":{"idset":"10917-10920","overwrite":0}}
+{"timestamp":1712168452.8310819,"name":"offline","context":{"idset":"607"}}
+{"timestamp":1712168452.919234,"name":"offline","context":{"idset":"611"}}
+{"timestamp":1712168452.9296527,"name":"offline","context":{"idset":"608"}}
+{"timestamp":1712168452.9505141,"name":"offline","context":{"idset":"609"}}
+{"timestamp":1712168452.96243,"name":"offline","context":{"idset":"610"}}
+{"timestamp":1712168453.03859,"name":"offline","context":{"idset":"612"}}
+{"timestamp":1712168453.1373255,"name":"offline","context":{"idset":"606"}}
+{"timestamp":1712168453.5165465,"name":"offline","context":{"idset":"605"}}
+{"timestamp":1712168754.0582001,"name":"drain","context":{"idset":"10917-10918,10921-10924","overwrite":1}}
+{"timestamp":1712169545.8930812,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1712169545.9872701,"name":"offline","context":{"idset":"10918"}}
+{"timestamp":1712169546.0242531,"name":"offline","context":{"idset":"10920"}}
+{"timestamp":1712169546.1086068,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1712169567.3675952,"name":"online","context":{"idset":"10920"}}
+{"timestamp":1712169567.963526,"name":"online","context":{"idset":"10919"}}
+{"timestamp":1712169570.1029983,"name":"online","context":{"idset":"10918"}}
+{"timestamp":1712169570.2696102,"name":"online","context":{"idset":"10917"}}
+{"timestamp":1712169640.4419613,"name":"undrain","context":{"idset":"10917-10920"}}
+{"timestamp":1712169686.9176335,"name":"undrain","context":{"idset":"11253"}}
+{"timestamp":1712169751.557744,"name":"undrain","context":{"idset":"11254"}}
+{"timestamp":1712169752.5632493,"name":"undrain","context":{"idset":"11255"}}
+{"timestamp":1712169753.6233516,"name":"undrain","context":{"idset":"11256"}}
+{"timestamp":1712169754.7820113,"name":"undrain","context":{"idset":"11257"}}
+{"timestamp":1712169755.8087027,"name":"undrain","context":{"idset":"11258"}}
+{"timestamp":1712169756.8502054,"name":"undrain","context":{"idset":"11259"}}
+{"timestamp":1712169757.9158757,"name":"undrain","context":{"idset":"11260"}}
+{"timestamp":1712169758.9353669,"name":"undrain","context":{"idset":"11261"}}
+{"timestamp":1712169759.981952,"name":"undrain","context":{"idset":"11262"}}
+{"timestamp":1712169760.9750767,"name":"undrain","context":{"idset":"11263"}}
+{"timestamp":1712169761.9722259,"name":"undrain","context":{"idset":"11264"}}
+{"timestamp":1712169763.1274784,"name":"undrain","context":{"idset":"11265"}}
+{"timestamp":1712169764.5392132,"name":"undrain","context":{"idset":"11266"}}
+{"timestamp":1712169765.5268216,"name":"undrain","context":{"idset":"11267"}}
+{"timestamp":1712169766.5238843,"name":"undrain","context":{"idset":"11268"}}
+{"timestamp":1712169768.0210104,"name":"undrain","context":{"idset":"11269"}}
+{"timestamp":1712169769.3527982,"name":"undrain","context":{"idset":"11270"}}
+{"timestamp":1712169770.656292,"name":"undrain","context":{"idset":"11271"}}
+{"timestamp":1712169772.1964438,"name":"undrain","context":{"idset":"11272"}}
+{"timestamp":1712169773.3509285,"name":"undrain","context":{"idset":"11273"}}
+{"timestamp":1712169774.514642,"name":"undrain","context":{"idset":"11274"}}
+{"timestamp":1712169841.0012393,"name":"undrain","context":{"idset":"11275"}}
+{"timestamp":1712169841.9803848,"name":"undrain","context":{"idset":"11276"}}
+{"timestamp":1712169843.0344498,"name":"undrain","context":{"idset":"11277"}}
+{"timestamp":1712169844.0614429,"name":"undrain","context":{"idset":"11278"}}
+{"timestamp":1712169845.107852,"name":"undrain","context":{"idset":"11279"}}
+{"timestamp":1712169846.1587226,"name":"undrain","context":{"idset":"11280"}}
+{"timestamp":1712169847.1815801,"name":"undrain","context":{"idset":"11281"}}
+{"timestamp":1712169848.2361922,"name":"undrain","context":{"idset":"11282"}}
+{"timestamp":1712169849.5470016,"name":"undrain","context":{"idset":"11283"}}
+{"timestamp":1712169850.7925203,"name":"undrain","context":{"idset":"11284"}}
+{"timestamp":1712169851.4446747,"name":"undrain","context":{"idset":"11285"}}
+{"timestamp":1712169852.1060321,"name":"undrain","context":{"idset":"11286"}}
+{"timestamp":1712169852.853121,"name":"undrain","context":{"idset":"11287"}}
+{"timestamp":1712169853.6835277,"name":"undrain","context":{"idset":"11288"}}
+{"timestamp":1712169854.6742475,"name":"undrain","context":{"idset":"11289"}}
+{"timestamp":1712169855.682389,"name":"undrain","context":{"idset":"11290"}}
+{"timestamp":1712169856.706604,"name":"undrain","context":{"idset":"11291"}}
+{"timestamp":1712169857.7217567,"name":"undrain","context":{"idset":"11292"}}
+{"timestamp":1712169858.7302928,"name":"undrain","context":{"idset":"11293"}}
+{"timestamp":1712169859.8186817,"name":"undrain","context":{"idset":"11294"}}
+{"timestamp":1712169860.8440108,"name":"undrain","context":{"idset":"11295"}}
+{"timestamp":1712169861.8242533,"name":"undrain","context":{"idset":"11296"}}
+{"timestamp":1712169862.8416715,"name":"undrain","context":{"idset":"11297"}}
+{"timestamp":1712169863.8308599,"name":"undrain","context":{"idset":"11298"}}
+{"timestamp":1712169864.8084202,"name":"undrain","context":{"idset":"11299"}}
+{"timestamp":1712169865.8060415,"name":"undrain","context":{"idset":"11300"}}
+{"timestamp":1712169866.8656824,"name":"undrain","context":{"idset":"11301"}}
+{"timestamp":1712169867.7424085,"name":"undrain","context":{"idset":"11302"}}
+{"timestamp":1712169868.7612531,"name":"undrain","context":{"idset":"11303"}}
+{"timestamp":1712169869.8022668,"name":"undrain","context":{"idset":"11304"}}
+{"timestamp":1712169870.8381307,"name":"undrain","context":{"idset":"11305"}}
+{"timestamp":1712169871.8714435,"name":"undrain","context":{"idset":"11306"}}
+{"timestamp":1712169872.8807721,"name":"undrain","context":{"idset":"11307"}}
+{"timestamp":1712169873.9119029,"name":"undrain","context":{"idset":"11308"}}
+{"timestamp":1712169874.9124472,"name":"undrain","context":{"idset":"11309"}}
+{"timestamp":1712169875.9281747,"name":"undrain","context":{"idset":"11310"}}
+{"timestamp":1712169876.9614253,"name":"undrain","context":{"idset":"11311"}}
+{"timestamp":1712169877.9613554,"name":"undrain","context":{"idset":"11312"}}
+{"timestamp":1712169878.9648752,"name":"undrain","context":{"idset":"11313"}}
+{"timestamp":1712169879.9635353,"name":"undrain","context":{"idset":"11314"}}
+{"timestamp":1712169880.7651892,"name":"undrain","context":{"idset":"11315"}}
+{"timestamp":1712169881.5741522,"name":"undrain","context":{"idset":"11316"}}
+{"timestamp":1712169884.5249534,"name":"undrain","context":{"idset":"11319"}}
+{"timestamp":1712169885.8213749,"name":"undrain","context":{"idset":"11320"}}
+{"timestamp":1712169887.0766582,"name":"undrain","context":{"idset":"11321"}}
+{"timestamp":1712169888.2865334,"name":"undrain","context":{"idset":"11322"}}
+{"timestamp":1712169889.1155713,"name":"undrain","context":{"idset":"11323"}}
+{"timestamp":1712169890.3067489,"name":"undrain","context":{"idset":"11324"}}
+{"timestamp":1712169891.4941676,"name":"undrain","context":{"idset":"11325"}}
+{"timestamp":1712169892.7235641,"name":"undrain","context":{"idset":"11326"}}
+{"timestamp":1712170128.9576046,"name":"online","context":{"idset":"10864"}}
+{"timestamp":1712170129.0669725,"name":"online","context":{"idset":"10867"}}
+{"timestamp":1712170129.5618243,"name":"online","context":{"idset":"10866,10868"}}
+{"timestamp":1712170196.628001,"name":"drain","context":{"idset":"10921","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712170199.1652176,"name":"drain","context":{"idset":"10920","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712170204.2542255,"name":"drain","context":{"idset":"10922","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712170212.6221066,"name":"drain","context":{"idset":"10923","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712170223.6940391,"name":"drain","context":{"idset":"10917","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712170228.1558609,"name":"drain","context":{"idset":"10918","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712170230.1759303,"name":"drain","context":{"idset":"10919","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712170273.445899,"name":"undrain","context":{"idset":"10917-10918,10921-10924"}}
+{"timestamp":1712170314.2268863,"name":"drain","context":{"idset":"10357-10359","overwrite":0}}
+{"timestamp":1712170418.5547442,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1712170445.8718538,"name":"online","context":{"idset":"10924"}}
+{"timestamp":1712170561.6227057,"name":"drain","context":{"idset":"10924","overwrite":1}}
+{"timestamp":1712170973.5266407,"name":"drain","context":{"idset":"10924","overwrite":1}}
+{"timestamp":1712171580.0102785,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1712174312.4487283,"name":"drain","context":{"idset":"10825","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174313.2261338,"name":"drain","context":{"idset":"10829","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174316.1914029,"name":"drain","context":{"idset":"10830","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174316.6733592,"name":"drain","context":{"idset":"10823","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174366.1198924,"name":"drain","context":{"idset":"10831","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174369.619107,"name":"drain","context":{"idset":"10822","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174370.8748741,"name":"drain","context":{"idset":"10826","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174378.7271705,"name":"drain","context":{"idset":"10827","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174384.0462465,"name":"drain","context":{"idset":"10824","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174386.6559324,"name":"drain","context":{"idset":"10828","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174456.1854422,"name":"drain","context":{"idset":"11660","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174464.6082194,"name":"drain","context":{"idset":"11656","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174465.4122262,"name":"drain","context":{"idset":"11657","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174466.0412419,"name":"drain","context":{"idset":"11653","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174466.6267278,"name":"drain","context":{"idset":"11658","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174468.6355774,"name":"drain","context":{"idset":"11659","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174469.7052834,"name":"drain","context":{"idset":"11299","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174470.4574249,"name":"drain","context":{"idset":"11295","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174483.5145023,"name":"drain","context":{"idset":"11654","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174485.5970631,"name":"drain","context":{"idset":"11655","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174514.4612403,"name":"drain","context":{"idset":"11297","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174520.7652485,"name":"drain","context":{"idset":"11298","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174525.1720884,"name":"drain","context":{"idset":"11300","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174531.2918873,"name":"drain","context":{"idset":"11294","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174532.0124931,"name":"drain","context":{"idset":"11296","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174534.9495943,"name":"drain","context":{"idset":"11292","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174539.6623499,"name":"drain","context":{"idset":"11293","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174838.6317751,"name":"drain","context":{"idset":"11676","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712174973.081986,"name":"drain","context":{"idset":"10924","overwrite":1}}
+{"timestamp":1712175202.5836709,"name":"undrain","context":{"idset":"10924"}}
+{"timestamp":1712175381.0046866,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1712175381.0232029,"name":"offline","context":{"idset":"9999"}}
+{"timestamp":1712175383.3246059,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1712175383.3362498,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1712175383.3484128,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1712175383.3508258,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1712175383.3627582,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1712175383.3828294,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1712175383.3967781,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1712175383.4069974,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1712175383.4164495,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1712175383.426724,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1712175383.4456978,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1712175383.4666049,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1712175383.4687965,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1712175383.4703689,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1712175383.4859743,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1712175383.4959838,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1712175383.5471475,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1712175383.5496955,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1712175383.5521793,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1712175383.5546267,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1712175383.5570362,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1712175383.5838223,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1712175383.5912461,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1712175383.5925543,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1712175383.5938718,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1712175383.6107495,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1712175383.6354887,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1712175383.6367085,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1712175383.6379244,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1712175383.6391284,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1712175383.656775,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1712175383.6688869,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1712175383.6700871,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1712175383.6712823,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1712175383.672478,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1712175383.6835737,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1712175383.7065098,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1712175383.7077014,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1712175383.7088828,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1712175383.7100656,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1712175383.7132161,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1712175383.7360647,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1712175383.7426128,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1712175383.7437873,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1712175383.7449565,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1712175383.7481418,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1712175383.7656715,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1712175383.7775724,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1712175383.7787521,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1712175383.7799194,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1712175383.7834554,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1712175383.7903788,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1712175383.8025815,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1712175383.8252246,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1712175383.8263946,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1712175383.8275592,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1712175383.828721,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1712175383.8314054,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1712175383.8436837,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1712175383.8662543,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1712175383.8675268,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1712175383.8686769,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1712175383.8698328,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1712175383.8709793,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1712175383.8848882,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1712175383.907434,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1712175383.9139249,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1712175383.915071,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1712175383.9162099,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1712175383.9173501,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1712175383.9263365,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1712175383.9535036,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1712175383.9620039,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1712175383.963593,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1712175383.9651022,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1712175383.9665184,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1712175383.9679983,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1712175383.9874589,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1712175384.0046957,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1712175384.0058427,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1712175384.0069914,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1712175384.008137,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1712175384.0092843,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1712175384.031714,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1712175384.0435193,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1712175384.0446572,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1712175384.0457895,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1712175384.0469244,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1712175384.0587595,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1712175384.0812275,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1712175384.0823624,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1712175384.0834963,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1712175384.0846434,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1712175384.09639,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1712175384.1188161,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1712175384.1199496,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1712175384.1210749,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1712175384.1221941,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1712175384.1392903,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1712175384.1457908,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1712175384.1469181,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1712175384.1480575,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1712175384.1492064,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1712175384.1611116,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1712175384.1622486,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1712175384.1633649,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1712175384.1644826,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1712175384.1656048,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1712175384.1670184,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1712175384.1895308,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1712175384.1959515,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1712175384.1970663,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1712175384.1981833,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1712175384.2005084,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1712175384.2072158,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1712175384.2083442,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1712175384.2300007,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1712175384.2418149,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1712175384.2429633,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1712175384.2440817,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1712175384.2467842,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1712175384.2533329,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1712175384.2598455,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1712175384.26636,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1712175384.27844,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1712175384.3008749,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1712175384.3231332,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1712175384.3295338,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1712175384.330642,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1712175384.3317447,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1712175384.3328397,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1712175384.3339617,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1712175384.3350596,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1712175384.3414295,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1712175384.3637266,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1712175384.3807065,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1712175384.3818192,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1712175384.3829124,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1712175384.3840041,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1712175384.3850968,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1712175384.3861902,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1712175384.3925791,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1712175384.4151771,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1712175384.4322882,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1712175384.4333878,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1712175384.4345019,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1712175384.4356031,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1712175384.4366951,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1712175384.4377835,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1712175384.4495368,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1712175384.4719267,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1712175384.4889839,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1712175384.4900742,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1712175384.4911571,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1712175384.4922371,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1712175384.4933176,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1712175384.4996994,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1712175384.5220907,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1712175384.5392084,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1712175384.5403039,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1712175384.5413835,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1712175384.5424626,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1712175384.5542271,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1712175384.5767128,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1712175384.5831404,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1712175384.5842359,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1712175384.5906477,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1712175384.6024108,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1712175384.6035326,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1712175384.6046133,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1712175384.6056981,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1712175384.6067922,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1712175384.6184986,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1712175384.6355059,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1712175384.6365845,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1712175384.6376524,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1712175384.6387219,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1712175384.667083,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1712175384.6737018,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1712175384.6748831,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1712175384.6759648,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1712175384.6770375,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1712175384.698019,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1712175384.6993861,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1712175384.7007978,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1712175384.7021444,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1712175384.7035272,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1712175384.7049136,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1712175384.7198448,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1712175384.7483568,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1712175384.7577419,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1712175384.7591336,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1712175384.7607379,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1712175384.7622685,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1712175384.7957959,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1712175384.8064368,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1712175384.8079245,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1712175384.809468,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1712175384.8109181,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1712175384.8123548,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1712175384.8373899,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1712175384.8559699,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1712175384.8576865,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1712175384.8595159,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1712175384.8611937,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1712175384.8629944,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1712175384.873594,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1712175384.9078634,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1712175384.9256289,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1712175384.9271333,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1712175384.9366148,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1712175384.9714386,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1712175384.9732964,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1712175384.9749558,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1712175384.9767118,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1712175384.987087,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1712175384.9888728,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1712175384.9905016,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1712175384.9922514,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1712175385.0023739,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1712175385.0397792,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1712175385.0414853,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1712175385.0436563,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1712175385.0458422,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1712175385.0768845,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1712175385.0892253,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1712175385.0911953,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1712175385.0929019,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1712175385.1256886,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1712175385.1286952,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1712175385.1314614,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1712175385.1450102,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1712175385.146976,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1712175385.1488409,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1712175385.171108,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1712175385.1732931,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1712175385.1755207,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1712175385.1777809,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1712175385.1899343,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1712175385.191854,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1712175385.1938589,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1712175385.1961005,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1712175385.2088783,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1712175385.2380209,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1712175385.2399483,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1712175385.2420187,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1712175385.2437301,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1712175385.272366,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1712175385.2921591,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1712175385.293853,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1712175385.3049264,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1712175385.3155453,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1712175385.3172226,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1712175385.318893,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1712175385.3299096,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1712175385.3403492,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1712175385.3420198,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1712175385.344326,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1712175385.3641074,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1712175385.3658152,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1712175385.3674729,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1712175385.3692048,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1712175385.3708713,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1712175385.3725235,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1712175385.3926616,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1712175385.3943212,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1712175385.3959796,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1712175385.3977079,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1712175385.3999453,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1712175385.4016058,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1712175385.4032524,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1712175385.4049006,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1712175385.4341705,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1712175385.4545403,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1712175385.456202,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1712175385.4578459,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1712175385.4595757,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1712175385.4618456,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1712175385.4723372,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1712175385.5106821,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1712175385.5504401,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1712175385.5529644,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1712175385.5555186,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1712175385.5781698,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1712175385.607655,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1712175385.609314,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1712175385.6110857,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1712175385.6127412,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1712175385.6327291,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1712175385.634382,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1712175385.63605,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1712175385.6381581,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1712175385.6399221,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1712175385.678329,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1712175385.6799872,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1712175385.6816323,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1712175385.6838832,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1712175385.6855445,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1712175385.6968346,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1712175385.7359049,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1712175385.7375472,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1712175385.7391336,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1712175385.7407255,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1712175385.7423043,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1712175385.7439954,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1712175385.7456136,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1712175385.7472837,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1712175385.7581565,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1712175385.795223,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1712175385.8066268,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1712175385.8083107,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1712175385.8099868,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1712175385.8116109,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1712175385.8132842,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1712175385.8148487,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1712175385.8349149,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1712175385.8699138,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1712175385.8873348,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1712175385.8887417,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1712175385.8906353,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1712175385.9232311,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1712175385.9475439,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1712175385.9489424,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1712175385.9503005,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1712175385.959219,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1712175385.9605901,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1712175385.9619493,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1712175385.9709289,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1712175385.9722219,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1712175385.9735527,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1712175385.9748738,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1712175385.9910808,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1712175385.9925733,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1712175385.9938612,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1712175386.0023847,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1712175386.0041368,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1712175386.0053878,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1712175386.0066245,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1712175386.0078592,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1712175386.0091207,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1712175386.0322592,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1712175386.0403538,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1712175386.0417237,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1712175386.0434465,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1712175386.0513325,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1712175386.0801795,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1712175386.0814345,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1712175386.0826907,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1712175386.0910068,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1712175386.0922081,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1712175386.0933795,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1712175386.1079113,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1712175386.1288972,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1712175386.1301038,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1712175386.1312549,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1712175386.1324036,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1712175386.133528,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1712175386.1346204,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1712175386.1357598,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1712175386.1372247,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1712175386.1383569,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1712175386.1457486,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1712175386.146908,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1712175386.1480217,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1712175386.14922,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1712175386.1504412,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1712175386.1515708,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1712175386.1526718,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1712175386.1733162,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1712175386.1989999,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1712175386.2063365,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1712175386.2073803,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1712175386.2084138,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1712175386.2094595,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1712175386.2284844,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1712175386.2296236,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1712175386.2307518,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1712175386.2317722,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1712175386.2503519,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1712175386.2632318,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1712175386.2645555,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1712175386.2655962,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1712175386.272222,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1712175386.2733154,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1712175386.2743003,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1712175386.275485,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1712175386.2765031,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1712175386.277472,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1712175386.2958689,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1712175386.296927,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1712175386.2980568,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1712175386.2990706,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1712175386.3000932,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1712175386.3069069,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1712175386.3079271,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1712175386.3089249,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1712175386.3098876,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1712175386.3108938,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1712175386.3118682,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1712175386.3245382,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1712175386.3258398,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1712175386.3268557,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1712175386.3278089,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1712175386.3287723,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1712175386.3297343,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1712175386.3306978,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1712175386.3316629,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1712175386.3383934,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1712175386.3625522,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1712175386.3807969,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1712175386.3817542,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1712175386.382756,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1712175386.383709,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1712175386.3846445,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1712175386.3855803,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1712175386.3865113,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1712175386.4104316,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1712175386.4339297,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1712175386.4408503,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1712175386.4418235,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1712175386.4427605,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1712175386.4437211,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1712175386.4446988,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1712175386.4684293,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1712175386.4810259,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1712175386.4819913,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1712175386.4832609,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1712175386.4842134,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1712175386.4851375,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1712175386.4860582,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1712175386.4870019,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1712175386.487936,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1712175386.4945652,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1712175386.5184076,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1712175386.5249383,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1712175386.5259185,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1712175386.5268705,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1712175386.5278146,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1712175386.5287716,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1712175386.5298483,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1712175386.5307734,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1712175386.5317178,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1712175386.5327401,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1712175386.5451372,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1712175386.5686953,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1712175386.5920331,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1712175386.6207621,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1712175386.6336532,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1712175386.6346312,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1712175386.6356061,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1712175386.6365654,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1712175386.6374888,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1712175386.6551616,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1712175386.6781518,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1712175386.6958842,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1712175386.6968095,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1712175386.697727,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1712175386.6986499,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1712175386.6995654,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1712175386.7004981,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1712175386.7014334,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1712175386.7023494,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1712175386.7032776,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1712175386.7151132,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1712175386.7378922,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1712175386.7605696,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1712175386.7614942,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1712175386.7624104,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1712175386.7633102,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1712175386.7642419,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1712175386.7651567,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1712175386.766063,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1712175386.766984,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1712175386.7679126,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1712175386.7688301,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1712175386.7697484,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1712175386.7706585,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1712175386.7880654,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1712175386.8109436,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1712175386.8340871,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1712175386.8571112,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1712175386.8803797,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1712175386.8867466,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1712175386.8877172,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1712175386.8886921,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1712175386.8895946,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1712175386.8904948,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1712175386.9081171,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1712175386.9312999,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1712175386.9544091,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1712175386.977536,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1712175386.978442,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1712175386.9793618,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1712175386.9803095,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1712175386.9812036,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1712175386.9821062,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1712175386.9995935,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1712175387.0231402,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1712175387.0463603,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1712175387.0472643,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1712175387.0481648,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1712175387.0490649,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1712175387.0499775,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1712175387.0732067,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1712175387.0964119,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1712175387.1137979,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1712175387.1146967,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1712175387.1156089,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1712175387.1165218,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1712175387.1284933,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1712175387.1294091,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1712175387.1303389,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1712175387.1312542,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1712175387.1321516,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1712175387.1330609,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1712175387.1558146,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1712175387.1676443,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1712175387.1685801,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1712175387.1695209,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1712175387.170388,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1712175387.1712525,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1712175387.172137,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1712175387.1730053,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1712175387.1848469,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1712175387.1857162,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1712175387.1865768,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1712175387.1874382,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1712175387.1882949,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1712175387.1891706,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1712175387.1900699,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1712175387.1909604,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1712175387.1918309,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1712175387.1983311,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1712175387.2211428,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1712175387.2495055,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1712175387.2504029,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1712175387.251297,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1712175387.252167,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1712175387.253052,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1712175387.253917,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1712175387.2602942,"name":"offline","context":{"idset":"841"}}
+{"timestamp":1712175387.2830689,"name":"offline","context":{"idset":"869"}}
+{"timestamp":1712175387.3062148,"name":"offline","context":{"idset":"870"}}
+{"timestamp":1712175387.3125682,"name":"offline","context":{"idset":"881"}}
+{"timestamp":1712175387.3135288,"name":"offline","context":{"idset":"882"}}
+{"timestamp":1712175387.3144076,"name":"offline","context":{"idset":"969"}}
+{"timestamp":1712175387.3154655,"name":"offline","context":{"idset":"970"}}
+{"timestamp":1712175387.3392813,"name":"offline","context":{"idset":"971"}}
+{"timestamp":1712175387.3457336,"name":"offline","context":{"idset":"972"}}
+{"timestamp":1712175387.3466024,"name":"offline","context":{"idset":"973"}}
+{"timestamp":1712175387.3475182,"name":"offline","context":{"idset":"974"}}
+{"timestamp":1712175387.3486156,"name":"offline","context":{"idset":"975"}}
+{"timestamp":1712175387.3497245,"name":"offline","context":{"idset":"976"}}
+{"timestamp":1712175387.3619351,"name":"offline","context":{"idset":"977"}}
+{"timestamp":1712175387.3627999,"name":"offline","context":{"idset":"978"}}
+{"timestamp":1712175387.3636482,"name":"offline","context":{"idset":"979"}}
+{"timestamp":1712175387.3644962,"name":"offline","context":{"idset":"980"}}
+{"timestamp":1712175387.3653529,"name":"offline","context":{"idset":"9973"}}
+{"timestamp":1712175387.366473,"name":"offline","context":{"idset":"9974"}}
+{"timestamp":1712175387.3840368,"name":"offline","context":{"idset":"9975"}}
+{"timestamp":1712175387.4011719,"name":"offline","context":{"idset":"9976"}}
+{"timestamp":1712175387.4020519,"name":"offline","context":{"idset":"9977"}}
+{"timestamp":1712175387.4028983,"name":"offline","context":{"idset":"9978"}}
+{"timestamp":1712175387.4037688,"name":"offline","context":{"idset":"9979"}}
+{"timestamp":1712175387.4156208,"name":"offline","context":{"idset":"9980"}}
+{"timestamp":1712175387.4165478,"name":"offline","context":{"idset":"9981"}}
+{"timestamp":1712175387.4174066,"name":"offline","context":{"idset":"9982"}}
+{"timestamp":1712175387.4184015,"name":"offline","context":{"idset":"9983"}}
+{"timestamp":1712175387.424665,"name":"offline","context":{"idset":"9984"}}
+{"timestamp":1712175387.4255347,"name":"offline","context":{"idset":"9985"}}
+{"timestamp":1712175387.426384,"name":"offline","context":{"idset":"9986"}}
+{"timestamp":1712175387.4272296,"name":"offline","context":{"idset":"9987"}}
+{"timestamp":1712175387.4280665,"name":"offline","context":{"idset":"9988"}}
+{"timestamp":1712175387.4344981,"name":"offline","context":{"idset":"9989"}}
+{"timestamp":1712175387.4457204,"name":"offline","context":{"idset":"9990"}}
+{"timestamp":1712175387.4465652,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1712175387.4473901,"name":"offline","context":{"idset":"9992"}}
+{"timestamp":1712175387.4482305,"name":"offline","context":{"idset":"9993"}}
+{"timestamp":1712175387.449331,"name":"offline","context":{"idset":"9994"}}
+{"timestamp":1712175387.4715431,"name":"offline","context":{"idset":"9995"}}
+{"timestamp":1712175387.4724255,"name":"offline","context":{"idset":"9996"}}
+{"timestamp":1712175387.4732678,"name":"offline","context":{"idset":"9997"}}
+{"timestamp":1712175387.4741089,"name":"offline","context":{"idset":"9998"}}
+{"timestamp":1712175387.4750116,"name":"offline","context":{"idset":"10000"}}
+{"timestamp":1712175387.4758694,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1712175387.4767711,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1712175387.483048,"name":"offline","context":{"idset":"10003"}}
+{"timestamp":1712175387.4838796,"name":"offline","context":{"idset":"10004"}}
+{"timestamp":1712175387.4847038,"name":"offline","context":{"idset":"10005"}}
+{"timestamp":1712175387.4855385,"name":"offline","context":{"idset":"10006"}}
+{"timestamp":1712175387.4863544,"name":"offline","context":{"idset":"10007"}}
+{"timestamp":1712175387.4871867,"name":"offline","context":{"idset":"10008"}}
+{"timestamp":1712175387.488059,"name":"offline","context":{"idset":"10009"}}
+{"timestamp":1712175387.494458,"name":"offline","context":{"idset":"10010"}}
+{"timestamp":1712175387.5173647,"name":"offline","context":{"idset":"10011"}}
+{"timestamp":1712175387.5181935,"name":"offline","context":{"idset":"10012"}}
+{"timestamp":1712175387.5190182,"name":"offline","context":{"idset":"10013"}}
+{"timestamp":1712175387.5198479,"name":"offline","context":{"idset":"10014"}}
+{"timestamp":1712175387.52071,"name":"offline","context":{"idset":"10015"}}
+{"timestamp":1712175387.5215609,"name":"offline","context":{"idset":"10016"}}
+{"timestamp":1712175387.5223911,"name":"offline","context":{"idset":"10017"}}
+{"timestamp":1712175387.5232277,"name":"offline","context":{"idset":"10018"}}
+{"timestamp":1712175387.5241044,"name":"offline","context":{"idset":"10019"}}
+{"timestamp":1712175387.5309496,"name":"offline","context":{"idset":"10020"}}
+{"timestamp":1712175387.5370159,"name":"offline","context":{"idset":"10021"}}
+{"timestamp":1712175387.5378451,"name":"offline","context":{"idset":"10022"}}
+{"timestamp":1712175387.5386693,"name":"offline","context":{"idset":"10023"}}
+{"timestamp":1712175387.539578,"name":"offline","context":{"idset":"10024"}}
+{"timestamp":1712175387.5404215,"name":"offline","context":{"idset":"10025"}}
+{"timestamp":1712175387.5412364,"name":"offline","context":{"idset":"10026"}}
+{"timestamp":1712175387.5420711,"name":"offline","context":{"idset":"10027"}}
+{"timestamp":1712175387.5429995,"name":"offline","context":{"idset":"10028"}}
+{"timestamp":1712175387.5438142,"name":"offline","context":{"idset":"10029"}}
+{"timestamp":1712175387.5446451,"name":"offline","context":{"idset":"10030"}}
+{"timestamp":1712175387.5454564,"name":"offline","context":{"idset":"10031"}}
+{"timestamp":1712175387.5462899,"name":"offline","context":{"idset":"10032"}}
+{"timestamp":1712175387.5471084,"name":"offline","context":{"idset":"10033"}}
+{"timestamp":1712175387.5479388,"name":"offline","context":{"idset":"10034"}}
+{"timestamp":1712175387.5644281,"name":"offline","context":{"idset":"10035"}}
+{"timestamp":1712175387.5861974,"name":"offline","context":{"idset":"10036"}}
+{"timestamp":1712175387.6026113,"name":"offline","context":{"idset":"10037"}}
+{"timestamp":1712175387.6034286,"name":"offline","context":{"idset":"10038"}}
+{"timestamp":1712175387.6042423,"name":"offline","context":{"idset":"10039"}}
+{"timestamp":1712175387.6050873,"name":"offline","context":{"idset":"10040"}}
+{"timestamp":1712175387.6059811,"name":"offline","context":{"idset":"10041"}}
+{"timestamp":1712175387.6068029,"name":"offline","context":{"idset":"10042"}}
+{"timestamp":1712175387.6076345,"name":"offline","context":{"idset":"10043"}}
+{"timestamp":1712175387.6084876,"name":"offline","context":{"idset":"10044"}}
+{"timestamp":1712175387.6093235,"name":"offline","context":{"idset":"10045"}}
+{"timestamp":1712175387.6101491,"name":"offline","context":{"idset":"10046"}}
+{"timestamp":1712175387.631846,"name":"offline","context":{"idset":"10047"}}
+{"timestamp":1712175387.6532066,"name":"offline","context":{"idset":"10048"}}
+{"timestamp":1712175387.6747661,"name":"offline","context":{"idset":"10049"}}
+{"timestamp":1712175387.6964436,"name":"offline","context":{"idset":"10050"}}
+{"timestamp":1712175387.7077928,"name":"offline","context":{"idset":"10051"}}
+{"timestamp":1712175387.7086461,"name":"offline","context":{"idset":"10052"}}
+{"timestamp":1712175387.7094893,"name":"offline","context":{"idset":"10053"}}
+{"timestamp":1712175387.7154744,"name":"offline","context":{"idset":"10054"}}
+{"timestamp":1712175387.737484,"name":"offline","context":{"idset":"10055"}}
+{"timestamp":1712175387.7648211,"name":"offline","context":{"idset":"10056"}}
+{"timestamp":1712175387.766165,"name":"offline","context":{"idset":"10057"}}
+{"timestamp":1712175387.767467,"name":"offline","context":{"idset":"10058"}}
+{"timestamp":1712175387.7687676,"name":"offline","context":{"idset":"10059"}}
+{"timestamp":1712175387.7700536,"name":"offline","context":{"idset":"10060"}}
+{"timestamp":1712175387.7713516,"name":"offline","context":{"idset":"10061"}}
+{"timestamp":1712175387.7726474,"name":"offline","context":{"idset":"10062"}}
+{"timestamp":1712175387.7739344,"name":"offline","context":{"idset":"10063"}}
+{"timestamp":1712175387.7752159,"name":"offline","context":{"idset":"10064"}}
+{"timestamp":1712175387.7847941,"name":"offline","context":{"idset":"10065"}}
+{"timestamp":1712175387.8191311,"name":"offline","context":{"idset":"10066"}}
+{"timestamp":1712175387.8204429,"name":"offline","context":{"idset":"10067"}}
+{"timestamp":1712175387.8217211,"name":"offline","context":{"idset":"10068"}}
+{"timestamp":1712175387.8230019,"name":"offline","context":{"idset":"10069"}}
+{"timestamp":1712175387.8243048,"name":"offline","context":{"idset":"10070"}}
+{"timestamp":1712175387.8255882,"name":"offline","context":{"idset":"10071"}}
+{"timestamp":1712175387.8268712,"name":"offline","context":{"idset":"10072"}}
+{"timestamp":1712175387.8281455,"name":"offline","context":{"idset":"10073"}}
+{"timestamp":1712175387.8294246,"name":"offline","context":{"idset":"10074"}}
+{"timestamp":1712175387.8307185,"name":"offline","context":{"idset":"10075"}}
+{"timestamp":1712175387.8319914,"name":"offline","context":{"idset":"10076"}}
+{"timestamp":1712175387.8332629,"name":"offline","context":{"idset":"10077"}}
+{"timestamp":1712175387.8675547,"name":"offline","context":{"idset":"10078"}}
+{"timestamp":1712175387.9021976,"name":"offline","context":{"idset":"10079"}}
+{"timestamp":1712175387.9364986,"name":"offline","context":{"idset":"10080"}}
+{"timestamp":1712175387.9377735,"name":"offline","context":{"idset":"10081"}}
+{"timestamp":1712175387.9390633,"name":"offline","context":{"idset":"10082"}}
+{"timestamp":1712175387.9403284,"name":"offline","context":{"idset":"10083"}}
+{"timestamp":1712175387.9416051,"name":"offline","context":{"idset":"10084"}}
+{"timestamp":1712175387.9428735,"name":"offline","context":{"idset":"10085"}}
+{"timestamp":1712175387.9441583,"name":"offline","context":{"idset":"10086"}}
+{"timestamp":1712175387.9454546,"name":"offline","context":{"idset":"10087"}}
+{"timestamp":1712175387.9472849,"name":"offline","context":{"idset":"10088"}}
+{"timestamp":1712175387.9492559,"name":"offline","context":{"idset":"10089"}}
+{"timestamp":1712175387.9511824,"name":"offline","context":{"idset":"10090"}}
+{"timestamp":1712175387.9530635,"name":"offline","context":{"idset":"10091"}}
+{"timestamp":1712175387.9544942,"name":"offline","context":{"idset":"10092"}}
+{"timestamp":1712175387.9669347,"name":"offline","context":{"idset":"10093"}}
+{"timestamp":1712175388.0018189,"name":"offline","context":{"idset":"10094"}}
+{"timestamp":1712175388.0365081,"name":"offline","context":{"idset":"10095"}}
+{"timestamp":1712175388.0713255,"name":"offline","context":{"idset":"10096"}}
+{"timestamp":1712175388.0809705,"name":"offline","context":{"idset":"10097"}}
+{"timestamp":1712175388.0822394,"name":"offline","context":{"idset":"10098"}}
+{"timestamp":1712175388.0835235,"name":"offline","context":{"idset":"10099"}}
+{"timestamp":1712175388.0847936,"name":"offline","context":{"idset":"10100"}}
+{"timestamp":1712175388.0860946,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1712175388.0873621,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1712175388.0886471,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1712175388.0899193,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1712175388.0911753,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1712175388.0924504,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1712175388.0937381,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1712175388.0949836,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1712175388.0962238,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1712175388.0974963,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1712175388.0987718,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1712175388.1084306,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1712175388.1435606,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1712175388.1783648,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1712175388.213074,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1712175388.2478323,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1712175388.2825587,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1712175388.2838256,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1712175388.2850802,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1712175388.2863278,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1712175388.2875965,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1712175388.2888784,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1712175388.2901201,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1712175388.2913516,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1712175388.2926269,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1712175388.2938833,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1712175388.2951396,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1712175388.3301337,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1712175388.3649447,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1712175388.4063449,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1712175388.4423881,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1712175388.4793274,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1712175388.5153503,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1712175388.5254903,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1712175388.5269146,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1712175388.5283439,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1712175388.5470524,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1712175388.583653,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1712175388.619895,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1712175388.6474593,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1712175388.6487107,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1712175388.649941,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1712175388.6773489,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1712175388.7047417,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1712175388.7061551,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1712175388.7073896,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1712175388.7086275,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1712175388.7098525,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1712175388.7110863,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1712175388.7299039,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1712175388.7401264,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1712175388.7413599,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1712175388.7425861,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1712175388.7437999,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1712175388.7453532,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1712175388.7465982,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1712175388.7479954,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1712175388.7492349,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1712175388.7504528,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1712175388.7708626,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1712175388.8070655,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1712175388.8259592,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1712175388.8271897,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1712175388.8284104,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1712175388.8296206,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1712175388.8308268,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1712175388.8492999,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1712175388.8853683,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1712175388.9225988,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1712175388.944653,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1712175388.9460516,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1712175388.9649186,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1712175388.9661748,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1712175388.9674053,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1712175388.9686241,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1712175388.9698422,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1712175388.9710493,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1712175388.9722583,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1712175388.9734831,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1712175388.9748962,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1712175389.0025403,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1712175389.0037453,"name":"offline","context":{"idset":"10188"}}
+{"timestamp":1712175389.0049424,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1712175389.0061467,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1712175389.0073364,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1712175389.0085373,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1712175389.0097363,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1712175389.027648,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1712175389.0622942,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1712175389.1076474,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1712175389.1177261,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1712175389.1189921,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1712175389.1202285,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1712175389.1384504,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1712175389.1397145,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1712175389.1409724,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1712175389.142179,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1712175389.1434221,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1712175389.1446271,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1712175389.1543422,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1712175389.1556084,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1712175389.1567962,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1712175389.1579957,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1712175389.1592026,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1712175389.1604087,"name":"offline","context":{"idset":"10213"}}
+{"timestamp":1712175389.1615956,"name":"offline","context":{"idset":"10214"}}
+{"timestamp":1712175389.1627913,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1712175389.16397,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1712175389.19066,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1712175389.2257965,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1712175389.2668982,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1712175389.2681465,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1712175389.2693365,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1712175389.2705173,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1712175389.3057811,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1712175389.3239238,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1712175389.3251348,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1712175389.326437,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1712175389.3276103,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1712175389.328804,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1712175389.3385205,"name":"offline","context":{"idset":"10229"}}
+{"timestamp":1712175389.3397756,"name":"offline","context":{"idset":"10230"}}
+{"timestamp":1712175389.3410368,"name":"offline","context":{"idset":"10231"}}
+{"timestamp":1712175389.3423452,"name":"offline","context":{"idset":"10232"}}
+{"timestamp":1712175389.3523655,"name":"offline","context":{"idset":"10233"}}
+{"timestamp":1712175389.3537374,"name":"offline","context":{"idset":"10234"}}
+{"timestamp":1712175389.3552241,"name":"offline","context":{"idset":"10235"}}
+{"timestamp":1712175389.3567035,"name":"offline","context":{"idset":"10236"}}
+{"timestamp":1712175389.3582497,"name":"offline","context":{"idset":"10237"}}
+{"timestamp":1712175389.359606,"name":"offline","context":{"idset":"10238"}}
+{"timestamp":1712175389.3609312,"name":"offline","context":{"idset":"10239"}}
+{"timestamp":1712175389.371248,"name":"offline","context":{"idset":"10240"}}
+{"timestamp":1712175389.3726385,"name":"offline","context":{"idset":"10241"}}
+{"timestamp":1712175389.3739769,"name":"offline","context":{"idset":"10242"}}
+{"timestamp":1712175389.3752613,"name":"offline","context":{"idset":"10243"}}
+{"timestamp":1712175389.3766093,"name":"offline","context":{"idset":"10244"}}
+{"timestamp":1712175389.3779187,"name":"offline","context":{"idset":"10245"}}
+{"timestamp":1712175389.3797734,"name":"offline","context":{"idset":"10246"}}
+{"timestamp":1712175389.3810351,"name":"offline","context":{"idset":"10247"}}
+{"timestamp":1712175389.4175003,"name":"offline","context":{"idset":"10248"}}
+{"timestamp":1712175389.4449708,"name":"offline","context":{"idset":"10249"}}
+{"timestamp":1712175389.4461718,"name":"offline","context":{"idset":"10250"}}
+{"timestamp":1712175389.4474287,"name":"offline","context":{"idset":"10251"}}
+{"timestamp":1712175389.4576187,"name":"offline","context":{"idset":"10252"}}
+{"timestamp":1712175389.4958305,"name":"offline","context":{"idset":"10253"}}
+{"timestamp":1712175389.4970751,"name":"offline","context":{"idset":"10254"}}
+{"timestamp":1712175389.4982224,"name":"offline","context":{"idset":"10255"}}
+{"timestamp":1712175389.4993587,"name":"offline","context":{"idset":"10256"}}
+{"timestamp":1712175389.5005033,"name":"offline","context":{"idset":"10257"}}
+{"timestamp":1712175389.5101326,"name":"offline","context":{"idset":"10258"}}
+{"timestamp":1712175389.5113337,"name":"offline","context":{"idset":"10259"}}
+{"timestamp":1712175389.5125442,"name":"offline","context":{"idset":"10260"}}
+{"timestamp":1712175389.5137038,"name":"offline","context":{"idset":"10261"}}
+{"timestamp":1712175389.514869,"name":"offline","context":{"idset":"10262"}}
+{"timestamp":1712175389.5245328,"name":"offline","context":{"idset":"10263"}}
+{"timestamp":1712175389.5256751,"name":"offline","context":{"idset":"10264"}}
+{"timestamp":1712175389.5352819,"name":"offline","context":{"idset":"10265"}}
+{"timestamp":1712175389.5364423,"name":"offline","context":{"idset":"10266"}}
+{"timestamp":1712175389.5376282,"name":"offline","context":{"idset":"10267"}}
+{"timestamp":1712175389.5388024,"name":"offline","context":{"idset":"10268"}}
+{"timestamp":1712175389.5399244,"name":"offline","context":{"idset":"10269"}}
+{"timestamp":1712175389.5410757,"name":"offline","context":{"idset":"10270"}}
+{"timestamp":1712175389.5520682,"name":"offline","context":{"idset":"10271"}}
+{"timestamp":1712175389.5536215,"name":"offline","context":{"idset":"10272"}}
+{"timestamp":1712175389.5549104,"name":"offline","context":{"idset":"10273"}}
+{"timestamp":1712175389.5560741,"name":"offline","context":{"idset":"10274"}}
+{"timestamp":1712175389.557445,"name":"offline","context":{"idset":"10275"}}
+{"timestamp":1712175389.5587294,"name":"offline","context":{"idset":"10276"}}
+{"timestamp":1712175389.560113,"name":"offline","context":{"idset":"10277"}}
+{"timestamp":1712175389.5613608,"name":"offline","context":{"idset":"10278"}}
+{"timestamp":1712175389.562597,"name":"offline","context":{"idset":"10279"}}
+{"timestamp":1712175389.5732036,"name":"offline","context":{"idset":"10280"}}
+{"timestamp":1712175389.5745535,"name":"offline","context":{"idset":"10281"}}
+{"timestamp":1712175389.5756698,"name":"offline","context":{"idset":"10282"}}
+{"timestamp":1712175389.5770178,"name":"offline","context":{"idset":"10283"}}
+{"timestamp":1712175389.5784736,"name":"offline","context":{"idset":"10284"}}
+{"timestamp":1712175389.6105149,"name":"offline","context":{"idset":"10285"}}
+{"timestamp":1712175389.6121571,"name":"offline","context":{"idset":"10286"}}
+{"timestamp":1712175389.6136723,"name":"offline","context":{"idset":"10287"}}
+{"timestamp":1712175389.6246858,"name":"offline","context":{"idset":"10288"}}
+{"timestamp":1712175389.6263905,"name":"offline","context":{"idset":"10289"}}
+{"timestamp":1712175389.6276853,"name":"offline","context":{"idset":"10290"}}
+{"timestamp":1712175389.6387205,"name":"offline","context":{"idset":"10291"}}
+{"timestamp":1712175389.6399477,"name":"offline","context":{"idset":"10292"}}
+{"timestamp":1712175389.6412618,"name":"offline","context":{"idset":"10293"}}
+{"timestamp":1712175389.642498,"name":"offline","context":{"idset":"10294"}}
+{"timestamp":1712175389.6437747,"name":"offline","context":{"idset":"10295"}}
+{"timestamp":1712175389.6554871,"name":"offline","context":{"idset":"10296"}}
+{"timestamp":1712175389.6567323,"name":"offline","context":{"idset":"10297"}}
+{"timestamp":1712175389.6581938,"name":"offline","context":{"idset":"10298"}}
+{"timestamp":1712175389.6697433,"name":"offline","context":{"idset":"10299"}}
+{"timestamp":1712175389.6711321,"name":"offline","context":{"idset":"10300"}}
+{"timestamp":1712175389.672369,"name":"offline","context":{"idset":"10303"}}
+{"timestamp":1712175389.6820943,"name":"offline","context":{"idset":"10304"}}
+{"timestamp":1712175389.6916902,"name":"offline","context":{"idset":"10305"}}
+{"timestamp":1712175389.6928253,"name":"offline","context":{"idset":"10306"}}
+{"timestamp":1712175389.693949,"name":"offline","context":{"idset":"10307"}}
+{"timestamp":1712175389.6950507,"name":"offline","context":{"idset":"10308"}}
+{"timestamp":1712175389.7047606,"name":"offline","context":{"idset":"10309"}}
+{"timestamp":1712175389.7058804,"name":"offline","context":{"idset":"10311"}}
+{"timestamp":1712175389.7069788,"name":"offline","context":{"idset":"10312"}}
+{"timestamp":1712175389.7081299,"name":"offline","context":{"idset":"10313"}}
+{"timestamp":1712175389.7092836,"name":"offline","context":{"idset":"10314"}}
+{"timestamp":1712175389.7103846,"name":"offline","context":{"idset":"10315"}}
+{"timestamp":1712175389.711508,"name":"offline","context":{"idset":"10316"}}
+{"timestamp":1712175389.7126272,"name":"offline","context":{"idset":"10317"}}
+{"timestamp":1712175389.722913,"name":"offline","context":{"idset":"10318"}}
+{"timestamp":1712175389.7614994,"name":"offline","context":{"idset":"10319"}}
+{"timestamp":1712175389.7716715,"name":"offline","context":{"idset":"10320"}}
+{"timestamp":1712175389.7728865,"name":"offline","context":{"idset":"10321"}}
+{"timestamp":1712175389.7741106,"name":"offline","context":{"idset":"10322"}}
+{"timestamp":1712175389.7753837,"name":"offline","context":{"idset":"10323"}}
+{"timestamp":1712175389.7765551,"name":"offline","context":{"idset":"10324"}}
+{"timestamp":1712175389.7777185,"name":"offline","context":{"idset":"10325"}}
+{"timestamp":1712175389.7790635,"name":"offline","context":{"idset":"10326"}}
+{"timestamp":1712175389.8088043,"name":"offline","context":{"idset":"10327"}}
+{"timestamp":1712175389.8492687,"name":"offline","context":{"idset":"10328"}}
+{"timestamp":1712175389.8913484,"name":"offline","context":{"idset":"10329"}}
+{"timestamp":1712175389.8926742,"name":"offline","context":{"idset":"10330"}}
+{"timestamp":1712175389.8939989,"name":"offline","context":{"idset":"10331"}}
+{"timestamp":1712175389.9165001,"name":"offline","context":{"idset":"10332"}}
+{"timestamp":1712175389.957525,"name":"offline","context":{"idset":"10333"}}
+{"timestamp":1712175389.9588721,"name":"offline","context":{"idset":"10334"}}
+{"timestamp":1712175389.9600778,"name":"offline","context":{"idset":"10335"}}
+{"timestamp":1712175389.9612672,"name":"offline","context":{"idset":"10336"}}
+{"timestamp":1712175389.9829969,"name":"offline","context":{"idset":"10337"}}
+{"timestamp":1712175389.9842629,"name":"offline","context":{"idset":"10338"}}
+{"timestamp":1712175389.9855216,"name":"offline","context":{"idset":"10339"}}
+{"timestamp":1712175389.986943,"name":"offline","context":{"idset":"10340"}}
+{"timestamp":1712175389.9983909,"name":"offline","context":{"idset":"10341"}}
+{"timestamp":1712175389.9996881,"name":"offline","context":{"idset":"10342"}}
+{"timestamp":1712175390.0014806,"name":"offline","context":{"idset":"10343"}}
+{"timestamp":1712175390.0031438,"name":"offline","context":{"idset":"10344"}}
+{"timestamp":1712175390.0047359,"name":"offline","context":{"idset":"10345"}}
+{"timestamp":1712175390.0059118,"name":"offline","context":{"idset":"10346"}}
+{"timestamp":1712175390.0069642,"name":"offline","context":{"idset":"10347"}}
+{"timestamp":1712175390.0302489,"name":"offline","context":{"idset":"10348"}}
+{"timestamp":1712175390.0313087,"name":"offline","context":{"idset":"10349"}}
+{"timestamp":1712175390.0324085,"name":"offline","context":{"idset":"10350"}}
+{"timestamp":1712175390.0335906,"name":"offline","context":{"idset":"10351"}}
+{"timestamp":1712175390.0347068,"name":"offline","context":{"idset":"10352"}}
+{"timestamp":1712175390.0358193,"name":"offline","context":{"idset":"10353"}}
+{"timestamp":1712175390.0369341,"name":"offline","context":{"idset":"10354"}}
+{"timestamp":1712175390.0718687,"name":"offline","context":{"idset":"10355"}}
+{"timestamp":1712175390.1126914,"name":"offline","context":{"idset":"10356"}}
+{"timestamp":1712175390.1233852,"name":"offline","context":{"idset":"10357"}}
+{"timestamp":1712175390.1245766,"name":"offline","context":{"idset":"10358"}}
+{"timestamp":1712175390.1351063,"name":"offline","context":{"idset":"10359"}}
+{"timestamp":1712175390.1729174,"name":"offline","context":{"idset":"10360"}}
+{"timestamp":1712175390.1998177,"name":"offline","context":{"idset":"10361"}}
+{"timestamp":1712175390.2094791,"name":"offline","context":{"idset":"10362"}}
+{"timestamp":1712175390.2105854,"name":"offline","context":{"idset":"10363"}}
+{"timestamp":1712175390.2116878,"name":"offline","context":{"idset":"10364"}}
+{"timestamp":1712175390.2127595,"name":"offline","context":{"idset":"10365"}}
+{"timestamp":1712175390.2225258,"name":"offline","context":{"idset":"10366"}}
+{"timestamp":1712175390.2236328,"name":"offline","context":{"idset":"10367"}}
+{"timestamp":1712175390.2246883,"name":"offline","context":{"idset":"10368"}}
+{"timestamp":1712175390.2257724,"name":"offline","context":{"idset":"10369"}}
+{"timestamp":1712175390.2268491,"name":"offline","context":{"idset":"10370"}}
+{"timestamp":1712175390.227901,"name":"offline","context":{"idset":"10371"}}
+{"timestamp":1712175390.2461398,"name":"offline","context":{"idset":"10372"}}
+{"timestamp":1712175390.2557635,"name":"offline","context":{"idset":"10373"}}
+{"timestamp":1712175390.2568731,"name":"offline","context":{"idset":"10374"}}
+{"timestamp":1712175390.2579463,"name":"offline","context":{"idset":"10375"}}
+{"timestamp":1712175390.2590024,"name":"offline","context":{"idset":"10376"}}
+{"timestamp":1712175390.2600603,"name":"offline","context":{"idset":"10377"}}
+{"timestamp":1712175390.2611361,"name":"offline","context":{"idset":"10378"}}
+{"timestamp":1712175390.2622213,"name":"offline","context":{"idset":"10379"}}
+{"timestamp":1712175390.2633224,"name":"offline","context":{"idset":"10380"}}
+{"timestamp":1712175390.2729104,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1712175390.3080826,"name":"offline","context":{"idset":"10382"}}
+{"timestamp":1712175390.3348763,"name":"offline","context":{"idset":"10383"}}
+{"timestamp":1712175390.3359358,"name":"offline","context":{"idset":"10384"}}
+{"timestamp":1712175390.3369873,"name":"offline","context":{"idset":"10385"}}
+{"timestamp":1712175390.3380268,"name":"offline","context":{"idset":"10386"}}
+{"timestamp":1712175390.3390718,"name":"offline","context":{"idset":"10387"}}
+{"timestamp":1712175390.3487692,"name":"offline","context":{"idset":"10388"}}
+{"timestamp":1712175390.3841033,"name":"offline","context":{"idset":"10389"}}
+{"timestamp":1712175390.4099057,"name":"offline","context":{"idset":"10390"}}
+{"timestamp":1712175390.4109395,"name":"offline","context":{"idset":"10391"}}
+{"timestamp":1712175390.4119606,"name":"offline","context":{"idset":"10392"}}
+{"timestamp":1712175390.4290295,"name":"offline","context":{"idset":"10393"}}
+{"timestamp":1712175390.4457281,"name":"offline","context":{"idset":"10394"}}
+{"timestamp":1712175390.4466755,"name":"offline","context":{"idset":"10395"}}
+{"timestamp":1712175390.4476318,"name":"offline","context":{"idset":"10396"}}
+{"timestamp":1712175390.4485912,"name":"offline","context":{"idset":"10397"}}
+{"timestamp":1712175390.4572382,"name":"offline","context":{"idset":"10398"}}
+{"timestamp":1712175390.4659719,"name":"offline","context":{"idset":"10399"}}
+{"timestamp":1712175390.4669557,"name":"offline","context":{"idset":"10400"}}
+{"timestamp":1712175390.4678733,"name":"offline","context":{"idset":"10401"}}
+{"timestamp":1712175390.4688389,"name":"offline","context":{"idset":"10402"}}
+{"timestamp":1712175390.4698024,"name":"offline","context":{"idset":"10403"}}
+{"timestamp":1712175390.4707544,"name":"offline","context":{"idset":"10404"}}
+{"timestamp":1712175390.4717193,"name":"offline","context":{"idset":"10405"}}
+{"timestamp":1712175390.4726713,"name":"offline","context":{"idset":"10406"}}
+{"timestamp":1712175390.4813004,"name":"offline","context":{"idset":"10407"}}
+{"timestamp":1712175390.5123594,"name":"offline","context":{"idset":"10408"}}
+{"timestamp":1712175390.5208447,"name":"offline","context":{"idset":"10409"}}
+{"timestamp":1712175390.5217226,"name":"offline","context":{"idset":"10410"}}
+{"timestamp":1712175390.5226338,"name":"offline","context":{"idset":"10411"}}
+{"timestamp":1712175390.5235672,"name":"offline","context":{"idset":"10412"}}
+{"timestamp":1712175390.5245256,"name":"offline","context":{"idset":"10413"}}
+{"timestamp":1712175390.5253952,"name":"offline","context":{"idset":"10414"}}
+{"timestamp":1712175390.5262587,"name":"offline","context":{"idset":"10415"}}
+{"timestamp":1712175390.5271139,"name":"offline","context":{"idset":"10416"}}
+{"timestamp":1712175390.5352945,"name":"offline","context":{"idset":"10417"}}
+{"timestamp":1712175390.5649009,"name":"offline","context":{"idset":"10418"}}
+{"timestamp":1712175390.5945749,"name":"offline","context":{"idset":"10419"}}
+{"timestamp":1712175390.6231599,"name":"offline","context":{"idset":"10420"}}
+{"timestamp":1712175390.6440775,"name":"offline","context":{"idset":"10421"}}
+{"timestamp":1712175390.6448579,"name":"offline","context":{"idset":"10422"}}
+{"timestamp":1712175390.6456332,"name":"offline","context":{"idset":"10423"}}
+{"timestamp":1712175390.646826,"name":"offline","context":{"idset":"10424"}}
+{"timestamp":1712175390.6476145,"name":"offline","context":{"idset":"10425"}}
+{"timestamp":1712175390.6743631,"name":"offline","context":{"idset":"10426"}}
+{"timestamp":1712175390.69415,"name":"offline","context":{"idset":"10427"}}
+{"timestamp":1712175390.6949077,"name":"offline","context":{"idset":"10428"}}
+{"timestamp":1712175390.6956446,"name":"offline","context":{"idset":"10429"}}
+{"timestamp":1712175390.6963937,"name":"offline","context":{"idset":"10430"}}
+{"timestamp":1712175390.6971378,"name":"offline","context":{"idset":"10431"}}
+{"timestamp":1712175390.6978946,"name":"offline","context":{"idset":"10432"}}
+{"timestamp":1712175390.6986413,"name":"offline","context":{"idset":"10433"}}
+{"timestamp":1712175390.6993775,"name":"offline","context":{"idset":"10434"}}
+{"timestamp":1712175390.7001288,"name":"offline","context":{"idset":"10435"}}
+{"timestamp":1712175390.7008595,"name":"offline","context":{"idset":"10436"}}
+{"timestamp":1712175390.7140956,"name":"offline","context":{"idset":"10437"}}
+{"timestamp":1712175390.7398779,"name":"offline","context":{"idset":"10438"}}
+{"timestamp":1712175390.7586052,"name":"offline","context":{"idset":"10439"}}
+{"timestamp":1712175390.7593129,"name":"offline","context":{"idset":"10440"}}
+{"timestamp":1712175390.7600107,"name":"offline","context":{"idset":"10441"}}
+{"timestamp":1712175390.7607219,"name":"offline","context":{"idset":"10442"}}
+{"timestamp":1712175390.7614391,"name":"offline","context":{"idset":"10443"}}
+{"timestamp":1712175390.7621274,"name":"offline","context":{"idset":"10444"}}
+{"timestamp":1712175390.7628503,"name":"offline","context":{"idset":"10445"}}
+{"timestamp":1712175390.7635882,"name":"offline","context":{"idset":"10446"}}
+{"timestamp":1712175390.7642851,"name":"offline","context":{"idset":"10447"}}
+{"timestamp":1712175390.7650208,"name":"offline","context":{"idset":"10448"}}
+{"timestamp":1712175390.7657218,"name":"offline","context":{"idset":"10449"}}
+{"timestamp":1712175390.7664082,"name":"offline","context":{"idset":"10450"}}
+{"timestamp":1712175390.7670856,"name":"offline","context":{"idset":"10451"}}
+{"timestamp":1712175390.7906904,"name":"offline","context":{"idset":"10452"}}
+{"timestamp":1712175390.8147314,"name":"offline","context":{"idset":"10453"}}
+{"timestamp":1712175390.8382237,"name":"offline","context":{"idset":"10454"}}
+{"timestamp":1712175390.8619366,"name":"offline","context":{"idset":"10455"}}
+{"timestamp":1712175390.8873155,"name":"offline","context":{"idset":"10456"}}
+{"timestamp":1712175390.8933392,"name":"offline","context":{"idset":"10457"}}
+{"timestamp":1712175390.8939779,"name":"offline","context":{"idset":"10458"}}
+{"timestamp":1712175390.8946037,"name":"offline","context":{"idset":"10459"}}
+{"timestamp":1712175390.8952248,"name":"offline","context":{"idset":"10460"}}
+{"timestamp":1712175390.8958485,"name":"offline","context":{"idset":"10461"}}
+{"timestamp":1712175390.8964691,"name":"offline","context":{"idset":"10462"}}
+{"timestamp":1712175390.8970742,"name":"offline","context":{"idset":"10463"}}
+{"timestamp":1712175390.9084902,"name":"offline","context":{"idset":"10464"}}
+{"timestamp":1712175390.9302192,"name":"offline","context":{"idset":"10465"}}
+{"timestamp":1712175390.9514258,"name":"offline","context":{"idset":"10466"}}
+{"timestamp":1712175390.9722102,"name":"offline","context":{"idset":"10467"}}
+{"timestamp":1712175390.9930084,"name":"offline","context":{"idset":"10468"}}
+{"timestamp":1712175390.9987082,"name":"offline","context":{"idset":"10471"}}
+{"timestamp":1712175390.9992812,"name":"offline","context":{"idset":"10472"}}
+{"timestamp":1712175390.9998708,"name":"offline","context":{"idset":"10473"}}
+{"timestamp":1712175391.0004473,"name":"offline","context":{"idset":"10474"}}
+{"timestamp":1712175391.0010104,"name":"offline","context":{"idset":"10475"}}
+{"timestamp":1712175391.0066557,"name":"offline","context":{"idset":"10476"}}
+{"timestamp":1712175391.0276072,"name":"offline","context":{"idset":"10477"}}
+{"timestamp":1712175391.0488949,"name":"offline","context":{"idset":"10478"}}
+{"timestamp":1712175391.0700991,"name":"offline","context":{"idset":"10479"}}
+{"timestamp":1712175391.080806,"name":"offline","context":{"idset":"10480"}}
+{"timestamp":1712175391.0814123,"name":"offline","context":{"idset":"10481"}}
+{"timestamp":1712175391.0819855,"name":"offline","context":{"idset":"10482"}}
+{"timestamp":1712175391.0877318,"name":"offline","context":{"idset":"10483"}}
+{"timestamp":1712175391.1090024,"name":"offline","context":{"idset":"10484"}}
+{"timestamp":1712175391.134423,"name":"offline","context":{"idset":"10485"}}
+{"timestamp":1712175391.1351035,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1712175391.1357484,"name":"offline","context":{"idset":"10487"}}
+{"timestamp":1712175391.1363194,"name":"offline","context":{"idset":"10488"}}
+{"timestamp":1712175391.1420171,"name":"offline","context":{"idset":"10489"}}
+{"timestamp":1712175391.1425934,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1712175391.143167,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1712175391.1437473,"name":"offline","context":{"idset":"10492"}}
+{"timestamp":1712175391.1443386,"name":"offline","context":{"idset":"10493"}}
+{"timestamp":1712175391.144918,"name":"offline","context":{"idset":"10494"}}
+{"timestamp":1712175391.1663105,"name":"offline","context":{"idset":"10495"}}
+{"timestamp":1712175391.1668863,"name":"offline","context":{"idset":"10496"}}
+{"timestamp":1712175391.1674769,"name":"offline","context":{"idset":"10497"}}
+{"timestamp":1712175391.1680372,"name":"offline","context":{"idset":"10498"}}
+{"timestamp":1712175391.1686642,"name":"offline","context":{"idset":"10499"}}
+{"timestamp":1712175391.1744907,"name":"offline","context":{"idset":"10500"}}
+{"timestamp":1712175391.1750619,"name":"offline","context":{"idset":"10501"}}
+{"timestamp":1712175391.1756635,"name":"offline","context":{"idset":"10502"}}
+{"timestamp":1712175391.176235,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1712175391.176806,"name":"offline","context":{"idset":"10504"}}
+{"timestamp":1712175391.1773605,"name":"offline","context":{"idset":"10505"}}
+{"timestamp":1712175391.1779234,"name":"offline","context":{"idset":"10506"}}
+{"timestamp":1712175391.1784811,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1712175391.1790357,"name":"offline","context":{"idset":"10508"}}
+{"timestamp":1712175391.1951869,"name":"offline","context":{"idset":"10511"}}
+{"timestamp":1712175391.200891,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1712175391.2014565,"name":"offline","context":{"idset":"10513"}}
+{"timestamp":1712175391.2020192,"name":"offline","context":{"idset":"10514"}}
+{"timestamp":1712175391.2025795,"name":"offline","context":{"idset":"10515"}}
+{"timestamp":1712175391.2031403,"name":"offline","context":{"idset":"10516"}}
+{"timestamp":1712175391.203696,"name":"offline","context":{"idset":"10520"}}
+{"timestamp":1712175391.2042558,"name":"offline","context":{"idset":"10523"}}
+{"timestamp":1712175391.2099178,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1712175391.2309694,"name":"offline","context":{"idset":"10525"}}
+{"timestamp":1712175391.2521911,"name":"offline","context":{"idset":"10526"}}
+{"timestamp":1712175391.2578766,"name":"offline","context":{"idset":"10527"}}
+{"timestamp":1712175391.2584398,"name":"offline","context":{"idset":"10528"}}
+{"timestamp":1712175391.2589931,"name":"offline","context":{"idset":"10529"}}
+{"timestamp":1712175391.2595475,"name":"offline","context":{"idset":"10530"}}
+{"timestamp":1712175391.2652922,"name":"offline","context":{"idset":"10531"}}
+{"timestamp":1712175391.2863243,"name":"offline","context":{"idset":"10532"}}
+{"timestamp":1712175391.3023136,"name":"offline","context":{"idset":"10533"}}
+{"timestamp":1712175391.3028634,"name":"offline","context":{"idset":"10534"}}
+{"timestamp":1712175391.3034139,"name":"offline","context":{"idset":"10535"}}
+{"timestamp":1712175391.3142705,"name":"offline","context":{"idset":"10536"}}
+{"timestamp":1712175391.3250976,"name":"offline","context":{"idset":"10537"}}
+{"timestamp":1712175391.3256638,"name":"offline","context":{"idset":"10538"}}
+{"timestamp":1712175391.3262451,"name":"offline","context":{"idset":"10539"}}
+{"timestamp":1712175391.3319376,"name":"offline","context":{"idset":"10540"}}
+{"timestamp":1712175391.332489,"name":"offline","context":{"idset":"10541"}}
+{"timestamp":1712175391.3330486,"name":"offline","context":{"idset":"10542"}}
+{"timestamp":1712175391.3336022,"name":"offline","context":{"idset":"10543"}}
+{"timestamp":1712175391.3341651,"name":"offline","context":{"idset":"10544"}}
+{"timestamp":1712175391.3399405,"name":"offline","context":{"idset":"10545"}}
+{"timestamp":1712175391.340497,"name":"offline","context":{"idset":"10546"}}
+{"timestamp":1712175391.3410528,"name":"offline","context":{"idset":"10547"}}
+{"timestamp":1712175391.341615,"name":"offline","context":{"idset":"10548"}}
+{"timestamp":1712175391.3421571,"name":"offline","context":{"idset":"10549"}}
+{"timestamp":1712175391.342695,"name":"offline","context":{"idset":"10550"}}
+{"timestamp":1712175391.343231,"name":"offline","context":{"idset":"10551"}}
+{"timestamp":1712175391.3437834,"name":"offline","context":{"idset":"10552"}}
+{"timestamp":1712175391.3443568,"name":"offline","context":{"idset":"10553"}}
+{"timestamp":1712175391.3449004,"name":"offline","context":{"idset":"10554"}}
+{"timestamp":1712175391.3505497,"name":"offline","context":{"idset":"10555"}}
+{"timestamp":1712175391.3818946,"name":"offline","context":{"idset":"10556"}}
+{"timestamp":1712175391.3928893,"name":"offline","context":{"idset":"10559"}}
+{"timestamp":1712175391.3934677,"name":"offline","context":{"idset":"10560"}}
+{"timestamp":1712175391.3940027,"name":"offline","context":{"idset":"10561"}}
+{"timestamp":1712175391.3945408,"name":"offline","context":{"idset":"10562"}}
+{"timestamp":1712175391.3950963,"name":"offline","context":{"idset":"10563"}}
+{"timestamp":1712175391.3956928,"name":"offline","context":{"idset":"10564"}}
+{"timestamp":1712175391.4117892,"name":"offline","context":{"idset":"10565"}}
+{"timestamp":1712175391.4333758,"name":"offline","context":{"idset":"10566"}}
+{"timestamp":1712175391.4548929,"name":"offline","context":{"idset":"10567"}}
+{"timestamp":1712175391.460758,"name":"offline","context":{"idset":"10568"}}
+{"timestamp":1712175391.4613495,"name":"offline","context":{"idset":"10569"}}
+{"timestamp":1712175391.46193,"name":"offline","context":{"idset":"10570"}}
+{"timestamp":1712175391.4730487,"name":"offline","context":{"idset":"10571"}}
+{"timestamp":1712175391.4736023,"name":"offline","context":{"idset":"10572"}}
+{"timestamp":1712175391.4741762,"name":"offline","context":{"idset":"10573"}}
+{"timestamp":1712175391.4747205,"name":"offline","context":{"idset":"10574"}}
+{"timestamp":1712175391.4752471,"name":"offline","context":{"idset":"10575"}}
+{"timestamp":1712175391.4809248,"name":"offline","context":{"idset":"10576"}}
+{"timestamp":1712175391.502697,"name":"offline","context":{"idset":"10577"}}
+{"timestamp":1712175391.5083923,"name":"offline","context":{"idset":"10578"}}
+{"timestamp":1712175391.5089571,"name":"offline","context":{"idset":"10579"}}
+{"timestamp":1712175391.5095057,"name":"offline","context":{"idset":"10580"}}
+{"timestamp":1712175391.5151484,"name":"offline","context":{"idset":"10581"}}
+{"timestamp":1712175391.5156925,"name":"offline","context":{"idset":"10582"}}
+{"timestamp":1712175391.516243,"name":"offline","context":{"idset":"10583"}}
+{"timestamp":1712175391.5167737,"name":"offline","context":{"idset":"10584"}}
+{"timestamp":1712175391.5172942,"name":"offline","context":{"idset":"10585"}}
+{"timestamp":1712175391.5178308,"name":"offline","context":{"idset":"10586"}}
+{"timestamp":1712175391.52369,"name":"offline","context":{"idset":"10587"}}
+{"timestamp":1712175391.5347166,"name":"offline","context":{"idset":"10588"}}
+{"timestamp":1712175391.5352778,"name":"offline","context":{"idset":"10589"}}
+{"timestamp":1712175391.5360944,"name":"offline","context":{"idset":"10590"}}
+{"timestamp":1712175391.5370114,"name":"offline","context":{"idset":"10591"}}
+{"timestamp":1712175391.5379374,"name":"offline","context":{"idset":"10592"}}
+{"timestamp":1712175391.5388653,"name":"offline","context":{"idset":"10593"}}
+{"timestamp":1712175391.5398266,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1712175391.5556562,"name":"offline","context":{"idset":"10595"}}
+{"timestamp":1712175391.5739541,"name":"offline","context":{"idset":"10596"}}
+{"timestamp":1712175391.5748076,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1712175391.5753632,"name":"offline","context":{"idset":"10598"}}
+{"timestamp":1712175391.5759475,"name":"offline","context":{"idset":"10599"}}
+{"timestamp":1712175391.5765185,"name":"offline","context":{"idset":"10600"}}
+{"timestamp":1712175391.5771205,"name":"offline","context":{"idset":"10601"}}
+{"timestamp":1712175391.5779181,"name":"offline","context":{"idset":"10602"}}
+{"timestamp":1712175391.5787416,"name":"offline","context":{"idset":"10603"}}
+{"timestamp":1712175391.5934501,"name":"offline","context":{"idset":"10604"}}
+{"timestamp":1712175391.6159308,"name":"offline","context":{"idset":"10605"}}
+{"timestamp":1712175391.6426377,"name":"offline","context":{"idset":"10606"}}
+{"timestamp":1712175391.6431658,"name":"offline","context":{"idset":"10607"}}
+{"timestamp":1712175391.6437154,"name":"offline","context":{"idset":"10608"}}
+{"timestamp":1712175391.6442397,"name":"offline","context":{"idset":"10609"}}
+{"timestamp":1712175391.6551418,"name":"offline","context":{"idset":"10610"}}
+{"timestamp":1712175391.6767814,"name":"offline","context":{"idset":"10611"}}
+{"timestamp":1712175391.6982055,"name":"offline","context":{"idset":"10612"}}
+{"timestamp":1712175391.6987433,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1712175391.7097173,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1712175391.7102478,"name":"offline","context":{"idset":"10743"}}
+{"timestamp":1712175391.7107813,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1712175391.7113073,"name":"offline","context":{"idset":"10745"}}
+{"timestamp":1712175391.711823,"name":"offline","context":{"idset":"10746"}}
+{"timestamp":1712175391.7123327,"name":"offline","context":{"idset":"10747"}}
+{"timestamp":1712175391.7180979,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1712175391.7186203,"name":"offline","context":{"idset":"10749"}}
+{"timestamp":1712175391.7191238,"name":"offline","context":{"idset":"10750"}}
+{"timestamp":1712175391.719646,"name":"offline","context":{"idset":"10751"}}
+{"timestamp":1712175391.7254868,"name":"offline","context":{"idset":"10752"}}
+{"timestamp":1712175391.7471242,"name":"offline","context":{"idset":"10753"}}
+{"timestamp":1712175391.7476389,"name":"offline","context":{"idset":"10754"}}
+{"timestamp":1712175391.7481411,"name":"offline","context":{"idset":"10755"}}
+{"timestamp":1712175391.7486632,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1712175391.7491667,"name":"offline","context":{"idset":"10757"}}
+{"timestamp":1712175391.7497199,"name":"offline","context":{"idset":"10758"}}
+{"timestamp":1712175391.7659497,"name":"offline","context":{"idset":"10759"}}
+{"timestamp":1712175391.7717316,"name":"offline","context":{"idset":"10760"}}
+{"timestamp":1712175391.7722378,"name":"offline","context":{"idset":"10761"}}
+{"timestamp":1712175391.7727423,"name":"offline","context":{"idset":"10762"}}
+{"timestamp":1712175391.7732511,"name":"offline","context":{"idset":"10763"}}
+{"timestamp":1712175391.7737579,"name":"offline","context":{"idset":"10764"}}
+{"timestamp":1712175391.7742541,"name":"offline","context":{"idset":"10765"}}
+{"timestamp":1712175391.7747569,"name":"offline","context":{"idset":"10766"}}
+{"timestamp":1712175391.7752538,"name":"offline","context":{"idset":"10767"}}
+{"timestamp":1712175391.7757709,"name":"offline","context":{"idset":"10768"}}
+{"timestamp":1712175391.7762828,"name":"offline","context":{"idset":"10769"}}
+{"timestamp":1712175391.7980413,"name":"offline","context":{"idset":"10770"}}
+{"timestamp":1712175391.8197865,"name":"offline","context":{"idset":"10771"}}
+{"timestamp":1712175391.8203065,"name":"offline","context":{"idset":"10772"}}
+{"timestamp":1712175391.8208184,"name":"offline","context":{"idset":"10773"}}
+{"timestamp":1712175391.8213162,"name":"offline","context":{"idset":"10774"}}
+{"timestamp":1712175391.8218236,"name":"offline","context":{"idset":"10775"}}
+{"timestamp":1712175391.8223209,"name":"offline","context":{"idset":"10776"}}
+{"timestamp":1712175391.822839,"name":"offline","context":{"idset":"10777"}}
+{"timestamp":1712175391.8233402,"name":"offline","context":{"idset":"10778"}}
+{"timestamp":1712175391.8449175,"name":"offline","context":{"idset":"10779"}}
+{"timestamp":1712175391.8664372,"name":"offline","context":{"idset":"10780"}}
+{"timestamp":1712175391.8925147,"name":"offline","context":{"idset":"10781"}}
+{"timestamp":1712175391.9037056,"name":"offline","context":{"idset":"10782"}}
+{"timestamp":1712175391.9042156,"name":"offline","context":{"idset":"10783"}}
+{"timestamp":1712175391.904727,"name":"offline","context":{"idset":"10784"}}
+{"timestamp":1712175391.9104216,"name":"offline","context":{"idset":"10785"}}
+{"timestamp":1712175391.9265351,"name":"offline","context":{"idset":"10786"}}
+{"timestamp":1712175391.9270267,"name":"offline","context":{"idset":"10787"}}
+{"timestamp":1712175391.9275208,"name":"offline","context":{"idset":"10788"}}
+{"timestamp":1712175391.9280038,"name":"offline","context":{"idset":"10789"}}
+{"timestamp":1712175391.9284947,"name":"offline","context":{"idset":"10790"}}
+{"timestamp":1712175391.9289901,"name":"offline","context":{"idset":"10791"}}
+{"timestamp":1712175391.9294827,"name":"offline","context":{"idset":"10792"}}
+{"timestamp":1712175391.9403367,"name":"offline","context":{"idset":"10793"}}
+{"timestamp":1712175391.9615791,"name":"offline","context":{"idset":"10794"}}
+{"timestamp":1712175391.9724591,"name":"offline","context":{"idset":"10795"}}
+{"timestamp":1712175391.9729607,"name":"offline","context":{"idset":"10796"}}
+{"timestamp":1712175391.9837322,"name":"offline","context":{"idset":"10797"}}
+{"timestamp":1712175391.9893787,"name":"offline","context":{"idset":"10798"}}
+{"timestamp":1712175391.9898751,"name":"offline","context":{"idset":"10799"}}
+{"timestamp":1712175391.9903595,"name":"offline","context":{"idset":"10800"}}
+{"timestamp":1712175391.9908488,"name":"offline","context":{"idset":"10801"}}
+{"timestamp":1712175391.9913461,"name":"offline","context":{"idset":"10802"}}
+{"timestamp":1712175391.9970126,"name":"offline","context":{"idset":"10803"}}
+{"timestamp":1712175392.0026581,"name":"offline","context":{"idset":"10804"}}
+{"timestamp":1712175392.0031602,"name":"offline","context":{"idset":"10805"}}
+{"timestamp":1712175392.0036559,"name":"offline","context":{"idset":"10806"}}
+{"timestamp":1712175392.0041406,"name":"offline","context":{"idset":"10807"}}
+{"timestamp":1712175392.0046234,"name":"offline","context":{"idset":"10808"}}
+{"timestamp":1712175392.0051012,"name":"offline","context":{"idset":"10809"}}
+{"timestamp":1712175392.0209877,"name":"offline","context":{"idset":"10810"}}
+{"timestamp":1712175392.0370042,"name":"offline","context":{"idset":"10811"}}
+{"timestamp":1712175392.0374939,"name":"offline","context":{"idset":"10812"}}
+{"timestamp":1712175392.0379906,"name":"offline","context":{"idset":"10813"}}
+{"timestamp":1712175392.0384662,"name":"offline","context":{"idset":"10814"}}
+{"timestamp":1712175392.0389686,"name":"offline","context":{"idset":"10815"}}
+{"timestamp":1712175392.0497372,"name":"offline","context":{"idset":"10816"}}
+{"timestamp":1712175392.0502291,"name":"offline","context":{"idset":"10817"}}
+{"timestamp":1712175392.050709,"name":"offline","context":{"idset":"10818"}}
+{"timestamp":1712175392.051192,"name":"offline","context":{"idset":"10819"}}
+{"timestamp":1712175392.0516639,"name":"offline","context":{"idset":"10820"}}
+{"timestamp":1712175392.0625889,"name":"offline","context":{"idset":"10821"}}
+{"timestamp":1712175392.0785201,"name":"offline","context":{"idset":"10822"}}
+{"timestamp":1712175392.0789907,"name":"offline","context":{"idset":"10823"}}
+{"timestamp":1712175392.0794692,"name":"offline","context":{"idset":"10824"}}
+{"timestamp":1712175392.0799448,"name":"offline","context":{"idset":"10825"}}
+{"timestamp":1712175392.080442,"name":"offline","context":{"idset":"10826"}}
+{"timestamp":1712175392.0860496,"name":"offline","context":{"idset":"10827"}}
+{"timestamp":1712175392.0865295,"name":"offline","context":{"idset":"10828"}}
+{"timestamp":1712175392.086997,"name":"offline","context":{"idset":"10829"}}
+{"timestamp":1712175392.0874631,"name":"offline","context":{"idset":"10830"}}
+{"timestamp":1712175392.0879226,"name":"offline","context":{"idset":"10831"}}
+{"timestamp":1712175392.0883803,"name":"offline","context":{"idset":"10832"}}
+{"timestamp":1712175392.0888453,"name":"offline","context":{"idset":"10833"}}
+{"timestamp":1712175392.089299,"name":"offline","context":{"idset":"10834"}}
+{"timestamp":1712175392.089777,"name":"offline","context":{"idset":"10835"}}
+{"timestamp":1712175392.0902348,"name":"offline","context":{"idset":"10836"}}
+{"timestamp":1712175392.0906959,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1712175392.0911453,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1712175392.0916023,"name":"offline","context":{"idset":"10867"}}
+{"timestamp":1712175392.1075723,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1712175392.129802,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1712175392.150584,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1712175392.151046,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1712175392.1515059,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1712175392.1519582,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1712175392.1524298,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1712175392.1528809,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1712175392.1533337,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1712175392.1537902,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1712175392.1542354,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1712175392.1546977,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1712175392.1551423,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1712175392.1762414,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1712175392.1973834,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1712175392.2185857,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1712175392.2396767,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1712175392.2608593,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1712175392.2664769,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1712175392.2669253,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1712175392.2673905,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1712175392.2678587,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1712175392.2683251,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1712175392.2687807,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1712175392.2692459,"name":"offline","context":{"idset":"10894"}}
+{"timestamp":1712175392.2696953,"name":"offline","context":{"idset":"10895"}}
+{"timestamp":1712175392.270139,"name":"offline","context":{"idset":"10896"}}
+{"timestamp":1712175392.2706032,"name":"offline","context":{"idset":"10897"}}
+{"timestamp":1712175392.2866969,"name":"offline","context":{"idset":"10898"}}
+{"timestamp":1712175392.3082297,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1712175392.329653,"name":"offline","context":{"idset":"10900"}}
+{"timestamp":1712175392.3353357,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1712175392.3358114,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1712175392.3362653,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1712175392.336719,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1712175392.3371861,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1712175392.3376372,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1712175392.3381073,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1712175392.3385739,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1712175392.3390489,"name":"offline","context":{"idset":"10909"}}
+{"timestamp":1712175392.3395269,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1712175392.3557556,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1712175392.3772938,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1712175392.4049208,"name":"offline","context":{"idset":"10915"}}
+{"timestamp":1712175392.4260268,"name":"offline","context":{"idset":"10916"}}
+{"timestamp":1712175392.4473269,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1712175392.458143,"name":"offline","context":{"idset":"10918"}}
+{"timestamp":1712175392.4586027,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1712175392.4590306,"name":"offline","context":{"idset":"10920"}}
+{"timestamp":1712175392.4594605,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1712175392.4598899,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1712175392.4603329,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1712175392.4607801,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1712175392.4612055,"name":"offline","context":{"idset":"10926"}}
+{"timestamp":1712175392.4616327,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1712175392.4826384,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1712175392.5036726,"name":"offline","context":{"idset":"10929"}}
+{"timestamp":1712175392.5247335,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1712175392.5302787,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1712175392.5307271,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1712175392.5311477,"name":"offline","context":{"idset":"10933"}}
+{"timestamp":1712175392.5315804,"name":"offline","context":{"idset":"10934"}}
+{"timestamp":1712175392.5320084,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1712175392.5324318,"name":"offline","context":{"idset":"10937"}}
+{"timestamp":1712175392.5328507,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1712175392.5332787,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1712175392.5337036,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1712175392.5341237,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1712175392.5345492,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1712175392.5349672,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1712175392.5508733,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1712175392.571892,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1712175392.592983,"name":"offline","context":{"idset":"10946"}}
+{"timestamp":1712175392.6140535,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1712175392.6378391,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1712175392.6386387,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1712175392.6394334,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1712175392.6402285,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1712175392.6410246,"name":"offline","context":{"idset":"10952"}}
+{"timestamp":1712175392.6418219,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1712175392.642606,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1712175392.6656618,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1712175392.6874847,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1712175392.7132199,"name":"offline","context":{"idset":"10957"}}
+{"timestamp":1712175392.7352579,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1712175392.7409284,"name":"offline","context":{"idset":"10959"}}
+{"timestamp":1712175392.7413518,"name":"offline","context":{"idset":"10960"}}
+{"timestamp":1712175392.7417812,"name":"offline","context":{"idset":"10961"}}
+{"timestamp":1712175392.7422135,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1712175392.7426443,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1712175392.7430689,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1712175392.7434938,"name":"offline","context":{"idset":"10965"}}
+{"timestamp":1712175392.7491007,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1712175392.7704728,"name":"offline","context":{"idset":"10967"}}
+{"timestamp":1712175392.7916729,"name":"offline","context":{"idset":"10968"}}
+{"timestamp":1712175392.7920942,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1712175392.7925107,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1712175392.7929122,"name":"offline","context":{"idset":"10971"}}
+{"timestamp":1712175392.7933197,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1712175392.7937505,"name":"offline","context":{"idset":"10973"}}
+{"timestamp":1712175392.7941611,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1712175392.7945764,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1712175392.7949812,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1712175392.7953858,"name":"offline","context":{"idset":"10977"}}
+{"timestamp":1712175392.7958124,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1712175392.7962174,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1712175392.7966237,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1712175392.8022137,"name":"offline","context":{"idset":"10981"}}
+{"timestamp":1712175392.8233309,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1712175392.8445539,"name":"offline","context":{"idset":"10983"}}
+{"timestamp":1712175392.8659279,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1712175392.8819604,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1712175392.8823678,"name":"offline","context":{"idset":"10986"}}
+{"timestamp":1712175392.8830757,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1712175392.8838382,"name":"offline","context":{"idset":"10988"}}
+{"timestamp":1712175392.884594,"name":"offline","context":{"idset":"10989"}}
+{"timestamp":1712175392.8853509,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1712175392.8861079,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1712175392.8966825,"name":"offline","context":{"idset":"10992"}}
+{"timestamp":1712175392.9211466,"name":"offline","context":{"idset":"10993"}}
+{"timestamp":1712175392.9472265,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1712175392.9719868,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1712175393.0141644,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1712175393.0441694,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1712175393.0449388,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1712175393.0456095,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1712175393.0464461,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1712175393.0665357,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1712175393.1058166,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1712175393.1065366,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1712175393.1073852,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1712175393.1081789,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1712175393.1088974,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1712175393.1097736,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1712175393.11044,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1712175393.1299195,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1712175393.1776638,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1712175393.207381,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1712175393.2081265,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1712175393.2088845,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1712175393.2096219,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1712175393.2103431,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1712175393.211189,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1712175393.2118819,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1712175393.2125816,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1712175393.2132545,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1712175393.2510452,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1712175393.251693,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1712175393.2523596,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1712175393.2530444,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1712175393.2537284,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1712175393.2543747,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1712175393.2550492,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1712175393.2557106,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1712175393.2563331,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1712175393.2569475,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1712175393.2576096,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1712175393.2582307,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1712175393.2943833,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1712175393.3303187,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1712175393.3665438,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1712175393.3671639,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1712175393.3678048,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1712175393.3684165,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1712175393.3690615,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1712175393.3696876,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1712175393.3703277,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1712175393.3709552,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1712175393.3716238,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1712175393.372246,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1712175393.3728685,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1712175393.382134,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1712175393.4176922,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1712175393.4533503,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1712175393.48053,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1712175393.4811926,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1712175393.481833,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1712175393.4825699,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1712175393.4833238,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1712175393.4840105,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1712175393.4846799,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1712175393.4853513,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1712175393.4860556,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1712175393.4867637,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1712175393.4874282,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1712175393.4881864,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1712175393.5196159,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1712175393.5580845,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1712175393.5945823,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1712175393.6311736,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1712175393.6672773,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1712175393.6679087,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1712175393.6685462,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1712175393.6691935,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1712175393.7054844,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1712175393.7418442,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1712175393.7783685,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1712175393.7879834,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1712175393.7886953,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1712175393.8071363,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1712175393.8077419,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1712175393.8084717,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1712175393.8091295,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1712175393.8097591,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1712175393.8103569,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1712175393.8109465,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1712175393.8116083,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1712175393.8211901,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1712175393.8218009,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1712175393.822377,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1712175393.822948,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1712175393.8235216,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1712175393.8241336,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1712175393.8247538,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1712175393.8253763,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1712175393.8259752,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1712175393.8535049,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1712175393.8893371,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1712175393.9169888,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1712175393.9175878,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1712175393.9181583,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1712175393.9187677,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1712175393.9193289,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1712175393.9199207,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1712175393.9472876,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1712175393.9478886,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1712175393.9485209,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1712175393.9490716,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1712175393.9496579,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1712175393.9502032,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1712175393.9509573,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1712175393.9515455,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1712175393.961179,"name":"offline","context":{"idset":"11253"}}
+{"timestamp":1712175393.9996655,"name":"offline","context":{"idset":"11254"}}
+{"timestamp":1712175394.0383935,"name":"offline","context":{"idset":"11255"}}
+{"timestamp":1712175394.0390518,"name":"offline","context":{"idset":"11256"}}
+{"timestamp":1712175394.0398116,"name":"offline","context":{"idset":"11257"}}
+{"timestamp":1712175394.0404782,"name":"offline","context":{"idset":"11258"}}
+{"timestamp":1712175394.0599711,"name":"offline","context":{"idset":"11259"}}
+{"timestamp":1712175394.0793395,"name":"offline","context":{"idset":"11260"}}
+{"timestamp":1712175394.0799398,"name":"offline","context":{"idset":"11261"}}
+{"timestamp":1712175394.0805974,"name":"offline","context":{"idset":"11262"}}
+{"timestamp":1712175394.0812681,"name":"offline","context":{"idset":"11263"}}
+{"timestamp":1712175394.091583,"name":"offline","context":{"idset":"11264"}}
+{"timestamp":1712175394.0923362,"name":"offline","context":{"idset":"11265"}}
+{"timestamp":1712175394.0929868,"name":"offline","context":{"idset":"11266"}}
+{"timestamp":1712175394.0935626,"name":"offline","context":{"idset":"11267"}}
+{"timestamp":1712175394.0942154,"name":"offline","context":{"idset":"11268"}}
+{"timestamp":1712175394.1041403,"name":"offline","context":{"idset":"11269"}}
+{"timestamp":1712175394.1047823,"name":"offline","context":{"idset":"11270"}}
+{"timestamp":1712175394.1054652,"name":"offline","context":{"idset":"11271"}}
+{"timestamp":1712175394.106041,"name":"offline","context":{"idset":"11272"}}
+{"timestamp":1712175394.1067216,"name":"offline","context":{"idset":"11273"}}
+{"timestamp":1712175394.1074882,"name":"offline","context":{"idset":"11274"}}
+{"timestamp":1712175394.1081157,"name":"offline","context":{"idset":"11275"}}
+{"timestamp":1712175394.1086874,"name":"offline","context":{"idset":"11276"}}
+{"timestamp":1712175394.1192961,"name":"offline","context":{"idset":"11277"}}
+{"timestamp":1712175394.1199515,"name":"offline","context":{"idset":"11278"}}
+{"timestamp":1712175394.1207094,"name":"offline","context":{"idset":"11279"}}
+{"timestamp":1712175394.1213114,"name":"offline","context":{"idset":"11280"}}
+{"timestamp":1712175394.1218855,"name":"offline","context":{"idset":"11281"}}
+{"timestamp":1712175394.1225419,"name":"offline","context":{"idset":"11282"}}
+{"timestamp":1712175394.1231213,"name":"offline","context":{"idset":"11283"}}
+{"timestamp":1712175394.1237786,"name":"offline","context":{"idset":"11284"}}
+{"timestamp":1712175394.163229,"name":"offline","context":{"idset":"11285"}}
+{"timestamp":1712175394.2052078,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1712175394.2156341,"name":"offline","context":{"idset":"11287"}}
+{"timestamp":1712175394.2161553,"name":"offline","context":{"idset":"11288"}}
+{"timestamp":1712175394.2166867,"name":"offline","context":{"idset":"11289"}}
+{"timestamp":1712175394.2172012,"name":"offline","context":{"idset":"11290"}}
+{"timestamp":1712175394.2177162,"name":"offline","context":{"idset":"11291"}}
+{"timestamp":1712175394.2182226,"name":"offline","context":{"idset":"11292"}}
+{"timestamp":1712175394.2354853,"name":"offline","context":{"idset":"11293"}}
+{"timestamp":1712175394.2359984,"name":"offline","context":{"idset":"11294"}}
+{"timestamp":1712175394.2365086,"name":"offline","context":{"idset":"11295"}}
+{"timestamp":1712175394.2370112,"name":"offline","context":{"idset":"11296"}}
+{"timestamp":1712175394.2375174,"name":"offline","context":{"idset":"11297"}}
+{"timestamp":1712175394.2380171,"name":"offline","context":{"idset":"11298"}}
+{"timestamp":1712175394.2385309,"name":"offline","context":{"idset":"11299"}}
+{"timestamp":1712175394.2390301,"name":"offline","context":{"idset":"11300"}}
+{"timestamp":1712175394.2478802,"name":"offline","context":{"idset":"11301"}}
+{"timestamp":1712175394.2817848,"name":"offline","context":{"idset":"11302"}}
+{"timestamp":1712175394.3158362,"name":"offline","context":{"idset":"11303"}}
+{"timestamp":1712175394.3247328,"name":"offline","context":{"idset":"11304"}}
+{"timestamp":1712175394.3252347,"name":"offline","context":{"idset":"11305"}}
+{"timestamp":1712175394.3257353,"name":"offline","context":{"idset":"11306"}}
+{"timestamp":1712175394.3262289,"name":"offline","context":{"idset":"11307"}}
+{"timestamp":1712175394.3434737,"name":"offline","context":{"idset":"11308"}}
+{"timestamp":1712175394.3689928,"name":"offline","context":{"idset":"11309"}}
+{"timestamp":1712175394.3694944,"name":"offline","context":{"idset":"11310"}}
+{"timestamp":1712175394.3699846,"name":"offline","context":{"idset":"11311"}}
+{"timestamp":1712175394.3704786,"name":"offline","context":{"idset":"11312"}}
+{"timestamp":1712175394.3709667,"name":"offline","context":{"idset":"11313"}}
+{"timestamp":1712175394.3798177,"name":"offline","context":{"idset":"11314"}}
+{"timestamp":1712175394.3803062,"name":"offline","context":{"idset":"11315"}}
+{"timestamp":1712175394.3807993,"name":"offline","context":{"idset":"11316"}}
+{"timestamp":1712175394.3812835,"name":"offline","context":{"idset":"11319"}}
+{"timestamp":1712175394.3817718,"name":"offline","context":{"idset":"11320"}}
+{"timestamp":1712175394.382257,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1712175394.3827474,"name":"offline","context":{"idset":"11322"}}
+{"timestamp":1712175394.3832288,"name":"offline","context":{"idset":"11323"}}
+{"timestamp":1712175394.3921223,"name":"offline","context":{"idset":"11324"}}
+{"timestamp":1712175394.3926206,"name":"offline","context":{"idset":"11325"}}
+{"timestamp":1712175394.3931005,"name":"offline","context":{"idset":"11326"}}
+{"timestamp":1712175394.3935931,"name":"offline","context":{"idset":"11327"}}
+{"timestamp":1712175394.3940716,"name":"offline","context":{"idset":"11328"}}
+{"timestamp":1712175394.3945577,"name":"offline","context":{"idset":"11329"}}
+{"timestamp":1712175394.4117303,"name":"offline","context":{"idset":"11331"}}
+{"timestamp":1712175394.4455924,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1712175394.4543908,"name":"offline","context":{"idset":"11333"}}
+{"timestamp":1712175394.4548705,"name":"offline","context":{"idset":"11334"}}
+{"timestamp":1712175394.4553425,"name":"offline","context":{"idset":"11335"}}
+{"timestamp":1712175394.4558203,"name":"offline","context":{"idset":"11336"}}
+{"timestamp":1712175394.4646127,"name":"offline","context":{"idset":"11337"}}
+{"timestamp":1712175394.4733949,"name":"offline","context":{"idset":"11338"}}
+{"timestamp":1712175394.4738712,"name":"offline","context":{"idset":"11339"}}
+{"timestamp":1712175394.4743369,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1712175394.4748108,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1712175394.4836175,"name":"offline","context":{"idset":"11342"}}
+{"timestamp":1712175394.4840832,"name":"offline","context":{"idset":"11343"}}
+{"timestamp":1712175394.484551,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1712175394.4850113,"name":"offline","context":{"idset":"11345"}}
+{"timestamp":1712175394.485476,"name":"offline","context":{"idset":"11346"}}
+{"timestamp":1712175394.4859354,"name":"offline","context":{"idset":"11347"}}
+{"timestamp":1712175394.4863946,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1712175394.495218,"name":"offline","context":{"idset":"11349"}}
+{"timestamp":1712175394.4956889,"name":"offline","context":{"idset":"11350"}}
+{"timestamp":1712175394.4961467,"name":"offline","context":{"idset":"11351"}}
+{"timestamp":1712175394.4966075,"name":"offline","context":{"idset":"11352"}}
+{"timestamp":1712175394.4970615,"name":"offline","context":{"idset":"11353"}}
+{"timestamp":1712175394.4975188,"name":"offline","context":{"idset":"11354"}}
+{"timestamp":1712175394.4979718,"name":"offline","context":{"idset":"11355"}}
+{"timestamp":1712175394.5067978,"name":"offline","context":{"idset":"11356"}}
+{"timestamp":1712175394.5416682,"name":"offline","context":{"idset":"11357"}}
+{"timestamp":1712175394.568145,"name":"offline","context":{"idset":"11358"}}
+{"timestamp":1712175394.5686138,"name":"offline","context":{"idset":"11359"}}
+{"timestamp":1712175394.5690682,"name":"offline","context":{"idset":"11360"}}
+{"timestamp":1712175394.5695324,"name":"offline","context":{"idset":"11361"}}
+{"timestamp":1712175394.5699952,"name":"offline","context":{"idset":"11362"}}
+{"timestamp":1712175394.5706472,"name":"offline","context":{"idset":"11363"}}
+{"timestamp":1712175394.5711,"name":"offline","context":{"idset":"11364"}}
+{"timestamp":1712175394.5715656,"name":"offline","context":{"idset":"11365"}}
+{"timestamp":1712175394.5720131,"name":"offline","context":{"idset":"11366"}}
+{"timestamp":1712175394.5724773,"name":"offline","context":{"idset":"11367"}}
+{"timestamp":1712175394.5816686,"name":"offline","context":{"idset":"11368"}}
+{"timestamp":1712175394.5821233,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1712175394.5825741,"name":"offline","context":{"idset":"11370"}}
+{"timestamp":1712175394.5830145,"name":"offline","context":{"idset":"11371"}}
+{"timestamp":1712175394.5834594,"name":"offline","context":{"idset":"11372"}}
+{"timestamp":1712175394.5838974,"name":"offline","context":{"idset":"11373"}}
+{"timestamp":1712175394.5843339,"name":"offline","context":{"idset":"11374"}}
+{"timestamp":1712175394.5847762,"name":"offline","context":{"idset":"11375"}}
+{"timestamp":1712175394.5852134,"name":"offline","context":{"idset":"11376"}}
+{"timestamp":1712175394.5856636,"name":"offline","context":{"idset":"11377"}}
+{"timestamp":1712175394.6123121,"name":"offline","context":{"idset":"11378"}}
+{"timestamp":1712175394.6465008,"name":"offline","context":{"idset":"11379"}}
+{"timestamp":1712175394.6803267,"name":"offline","context":{"idset":"11380"}}
+{"timestamp":1712175394.7058544,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1712175394.7062981,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1712175394.7067392,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1712175394.7071702,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1712175394.7242866,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1712175394.7247345,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1712175394.7251627,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1712175394.7255938,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1712175394.726018,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1712175394.7264488,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1712175394.7268722,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1712175394.7356293,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1712175394.7360554,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1712175394.7364852,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1712175394.7369056,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1712175394.7373233,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1712175394.7377477,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1712175394.7381659,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1712175394.7385898,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1712175394.7723532,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1712175394.7979965,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1712175394.7984288,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1712175394.7988427,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1712175394.7992547,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1712175394.7996917,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1712175394.8001032,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1712175394.8088431,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1712175394.8175964,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1712175394.8263824,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1712175394.8268173,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1712175394.8272269,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1712175394.827636,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1712175394.8280394,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1712175394.8284488,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1712175394.8288515,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1712175394.8376257,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1712175394.838033,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1712175394.8384421,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1712175394.83884,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1712175394.8392365,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1712175394.8396413,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1712175394.8400369,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1712175394.8404613,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1712175394.8408566,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1712175394.841249,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1712175394.8416481,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1712175394.8420391,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1712175394.8424356,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1712175394.8511624,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1712175394.851563,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1712175394.851954,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1712175394.85234,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1712175394.8527374,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1712175394.8531234,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1712175394.8535175,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1712175394.8539011,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1712175394.8542831,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1712175394.8546689,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1712175394.8550479,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1712175394.8554332,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1712175394.855813,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1712175394.8561897,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1712175394.8649104,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1712175394.8652904,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1712175394.8656723,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1712175394.8660462,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1712175394.8664398,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1712175394.8668139,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1712175394.875551,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1712175394.8926058,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1712175394.9014058,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1712175394.9017873,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1712175394.902159,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1712175394.9025366,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1712175394.9029081,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1712175394.9032764,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1712175394.9036503,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1712175394.9040141,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1712175394.9043765,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1712175394.9047456,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1712175394.9051077,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1712175394.9054737,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1712175394.9058337,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1712175394.9061899,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1712175394.906563,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1712175394.9069204,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1712175394.915674,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1712175394.9160328,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1712175394.9163868,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1712175394.9167507,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1712175394.9171047,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1712175394.9174697,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1712175394.9178212,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1712175394.9181693,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1712175394.9268742,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1712175394.9607339,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1712175394.9777493,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1712175394.978101,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1712175394.9784536,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1712175394.9787977,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1712175394.9791403,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1712175394.9794886,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1712175394.9798315,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1712175394.9801745,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1712175394.9805548,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1712175394.9809031,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1712175394.9896219,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1712175394.9899695,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1712175394.9903071,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1712175394.9906509,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1712175394.9909859,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1712175394.9913192,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1712175394.9916606,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1712175394.9919939,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1712175394.9923248,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1712175394.9926641,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1712175394.9930191,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1712175394.9933622,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1712175394.9937,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1712175394.9940295,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1712175394.9943564,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1712175394.9946909,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1712175394.9950163,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1712175394.9953399,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1712175394.9956768,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1712175394.996002,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1712175395.0046721,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1712175395.0049963,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1712175395.0053177,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1712175395.0056458,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1712175395.0059662,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1712175395.0062845,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1712175395.0066068,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1712175395.0069234,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1712175395.0072391,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1712175395.0075703,"name":"offline","context":{"idset":"11525"}}
+{"timestamp":1712175395.0078859,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1712175395.008198,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1712175395.008517,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1712175395.0088274,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1712175395.0091362,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1712175395.0094514,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1712175395.0320339,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1712175395.0710242,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1712175395.0897999,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1712175395.0901182,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1712175395.0904334,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1712175395.0907416,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1712175395.0910437,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1712175395.0913444,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1712175395.0916543,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1712175395.0919552,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1712175395.092253,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1712175395.1009448,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1712175395.1347427,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1712175395.1692536,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1712175395.1947093,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1712175395.2033632,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1712175395.2036803,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1712175395.2039771,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1712175395.2042704,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1712175395.2045784,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1712175395.2048721,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1712175395.2051616,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1712175395.205457,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1712175395.205745,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1712175395.206032,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1712175395.2063167,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1712175395.2149756,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1712175395.215287,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1712175395.2156231,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1712175395.2159088,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1712175395.2161899,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1712175395.2165055,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1712175395.2168155,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1712175395.2170961,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1712175395.2173731,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1712175395.2176638,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1712175395.2179396,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1712175395.2182133,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1712175395.2184973,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1712175395.2187715,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1712175395.2190437,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1712175395.2193151,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1712175395.2195935,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1712175395.2198651,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1712175395.220135,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1712175395.2204087,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1712175395.2206771,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1712175395.2209435,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1712175395.2212095,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1712175395.2214949,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1712175395.2217612,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1712175395.2220244,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1712175395.2222862,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1712175395.2309461,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1712175395.2312114,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1712175395.2314973,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1712175395.2317674,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1712175395.2320261,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1712175395.2322814,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1712175395.2325628,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1712175395.2328284,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1712175395.233084,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1712175395.2333372,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1712175395.2336109,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1712175395.2338738,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1712175395.2341263,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1712175395.2428412,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1712175395.281595,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1712175395.3227987,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1712175395.3484509,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1712175395.3487129,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1712175395.3489666,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1712175395.3492179,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1712175395.3494701,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1712175395.3497152,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1712175395.3499599,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1712175395.3502021,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1712175395.35045,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1712175395.350692,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1712175395.3509324,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1712175395.3511732,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1712175395.3514199,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1712175395.3516686,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1712175395.351907,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1712175395.3521433,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1712175395.3523784,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1712175395.3526282,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1712175395.3528645,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1712175395.3530989,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1712175395.3533306,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1712175395.3620293,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1712175395.362267,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1712175395.3625047,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1712175395.3627326,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1712175395.3629587,"name":"offline","context":{"idset":"11637"}}
+{"timestamp":1712175395.3631842,"name":"offline","context":{"idset":"11638"}}
+{"timestamp":1712175395.3634415,"name":"offline","context":{"idset":"11639"}}
+{"timestamp":1712175395.3636694,"name":"offline","context":{"idset":"11640"}}
+{"timestamp":1712175395.3638916,"name":"offline","context":{"idset":"11641"}}
+{"timestamp":1712175395.3641126,"name":"offline","context":{"idset":"11642"}}
+{"timestamp":1712175395.3643327,"name":"offline","context":{"idset":"11643"}}
+{"timestamp":1712175395.3645632,"name":"offline","context":{"idset":"11644"}}
+{"timestamp":1712175395.3647816,"name":"offline","context":{"idset":"11645"}}
+{"timestamp":1712175395.365,"name":"offline","context":{"idset":"11646"}}
+{"timestamp":1712175395.365217,"name":"offline","context":{"idset":"11647"}}
+{"timestamp":1712175395.3654413,"name":"offline","context":{"idset":"11648"}}
+{"timestamp":1712175395.3656588,"name":"offline","context":{"idset":"11649"}}
+{"timestamp":1712175395.3658738,"name":"offline","context":{"idset":"11650"}}
+{"timestamp":1712175395.3660867,"name":"offline","context":{"idset":"11651"}}
+{"timestamp":1712175395.3662994,"name":"offline","context":{"idset":"11652"}}
+{"timestamp":1712175395.3665195,"name":"offline","context":{"idset":"11653"}}
+{"timestamp":1712175395.3667314,"name":"offline","context":{"idset":"11654"}}
+{"timestamp":1712175395.3669419,"name":"offline","context":{"idset":"11655"}}
+{"timestamp":1712175395.367151,"name":"offline","context":{"idset":"11656"}}
+{"timestamp":1712175395.3673596,"name":"offline","context":{"idset":"11657"}}
+{"timestamp":1712175395.3675754,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1712175395.3677828,"name":"offline","context":{"idset":"11659"}}
+{"timestamp":1712175395.3679891,"name":"offline","context":{"idset":"11660"}}
+{"timestamp":1712175395.3681946,"name":"offline","context":{"idset":"11661"}}
+{"timestamp":1712175395.3684225,"name":"offline","context":{"idset":"11662"}}
+{"timestamp":1712175395.3686299,"name":"offline","context":{"idset":"11663"}}
+{"timestamp":1712175395.3688347,"name":"offline","context":{"idset":"11664"}}
+{"timestamp":1712175395.3690388,"name":"offline","context":{"idset":"11665"}}
+{"timestamp":1712175395.3778031,"name":"offline","context":{"idset":"11666"}}
+{"timestamp":1712175395.3780096,"name":"offline","context":{"idset":"11667"}}
+{"timestamp":1712175395.3782084,"name":"offline","context":{"idset":"11668"}}
+{"timestamp":1712175395.3784242,"name":"offline","context":{"idset":"11671"}}
+{"timestamp":1712175395.3786204,"name":"offline","context":{"idset":"11672"}}
+{"timestamp":1712175395.3788147,"name":"offline","context":{"idset":"11673"}}
+{"timestamp":1712175395.3790085,"name":"offline","context":{"idset":"11674"}}
+{"timestamp":1712175395.3792043,"name":"offline","context":{"idset":"11675"}}
+{"timestamp":1712175395.3794038,"name":"offline","context":{"idset":"11676"}}
+{"timestamp":1712175395.379595,"name":"offline","context":{"idset":"11677"}}
+{"timestamp":1712175395.3797858,"name":"offline","context":{"idset":"11678"}}
+{"timestamp":1712175395.3799753,"name":"offline","context":{"idset":"11679"}}
+{"timestamp":1712175395.3801637,"name":"offline","context":{"idset":"11680"}}
+{"timestamp":1712175395.3803508,"name":"offline","context":{"idset":"11681"}}
+{"timestamp":1712175395.3890898,"name":"offline","context":{"idset":"11682"}}
+{"timestamp":1712175395.4232941,"name":"offline","context":{"idset":"11683"}}
+{"timestamp":1712175395.4575374,"name":"offline","context":{"idset":"11684"}}
+{"timestamp":1712175395.4746912,"name":"offline","context":{"idset":"11685"}}
+{"timestamp":1712175395.474884,"name":"offline","context":{"idset":"11686"}}
+{"timestamp":1712175395.4750702,"name":"offline","context":{"idset":"11687"}}
+{"timestamp":1712175395.4752541,"name":"offline","context":{"idset":"11688"}}
+{"timestamp":1712175395.4754424,"name":"offline","context":{"idset":"11689"}}
+{"timestamp":1712175395.4756227,"name":"offline","context":{"idset":"11690"}}
+{"timestamp":1712175395.475805,"name":"offline","context":{"idset":"11691"}}
+{"timestamp":1712175395.4759867,"name":"offline","context":{"idset":"11692"}}
+{"timestamp":1712175395.4761665,"name":"offline","context":{"idset":"11693"}}
+{"timestamp":1712175395.4763441,"name":"offline","context":{"idset":"11694"}}
+{"timestamp":1712175395.4765379,"name":"offline","context":{"idset":"11695"}}
+{"timestamp":1712175395.4852006,"name":"offline","context":{"idset":"11696"}}
+{"timestamp":1712175395.4853783,"name":"offline","context":{"idset":"11697"}}
+{"timestamp":1712175395.4940782,"name":"offline","context":{"idset":"11698"}}
+{"timestamp":1712175395.4942555,"name":"offline","context":{"idset":"11699"}}
+{"timestamp":1712175395.4944353,"name":"offline","context":{"idset":"11700"}}
+{"timestamp":1712175395.4946098,"name":"offline","context":{"idset":"11701"}}
+{"timestamp":1712175395.4947827,"name":"offline","context":{"idset":"11702"}}
+{"timestamp":1712175395.4949532,"name":"offline","context":{"idset":"11703"}}
+{"timestamp":1712175395.4951229,"name":"offline","context":{"idset":"11704"}}
+{"timestamp":1712175395.4952912,"name":"offline","context":{"idset":"11705"}}
+{"timestamp":1712175395.4954643,"name":"offline","context":{"idset":"11706"}}
+{"timestamp":1712175395.4956312,"name":"offline","context":{"idset":"11707"}}
+{"timestamp":1712175395.4957967,"name":"offline","context":{"idset":"11708"}}
+{"timestamp":1712175395.4959614,"name":"offline","context":{"idset":"11709"}}
+{"timestamp":1712175395.4961267,"name":"offline","context":{"idset":"11710"}}
+{"timestamp":1712175395.4962904,"name":"offline","context":{"idset":"11711"}}
+{"timestamp":1712175395.4964588,"name":"offline","context":{"idset":"11712"}}
+{"timestamp":1712175395.4966211,"name":"offline","context":{"idset":"11713"}}
+{"timestamp":1712175395.4967835,"name":"offline","context":{"idset":"11714"}}
+{"timestamp":1712175395.496944,"name":"offline","context":{"idset":"11715"}}
+{"timestamp":1712175395.4971042,"name":"offline","context":{"idset":"11716"}}
+{"timestamp":1712175395.4972632,"name":"offline","context":{"idset":"11717"}}
+{"timestamp":1712175395.4974296,"name":"offline","context":{"idset":"11718"}}
+{"timestamp":1712175395.4975855,"name":"offline","context":{"idset":"11719"}}
+{"timestamp":1712175395.4977417,"name":"offline","context":{"idset":"11720"}}
+{"timestamp":1712175395.4978969,"name":"offline","context":{"idset":"11721"}}
+{"timestamp":1712175395.4980507,"name":"offline","context":{"idset":"11722"}}
+{"timestamp":1712175395.4982045,"name":"offline","context":{"idset":"11723"}}
+{"timestamp":1712175395.4983582,"name":"offline","context":{"idset":"11724"}}
+{"timestamp":1712175395.4985178,"name":"offline","context":{"idset":"11725"}}
+{"timestamp":1712175395.4986687,"name":"offline","context":{"idset":"11726"}}
+{"timestamp":1712175395.4988189,"name":"offline","context":{"idset":"11727"}}
+{"timestamp":1712175395.4989703,"name":"offline","context":{"idset":"11728"}}
+{"timestamp":1712175395.4991188,"name":"offline","context":{"idset":"11729"}}
+{"timestamp":1712175395.507776,"name":"offline","context":{"idset":"11730"}}
+{"timestamp":1712175395.507925,"name":"offline","context":{"idset":"11731"}}
+{"timestamp":1712175395.5080709,"name":"offline","context":{"idset":"11732"}}
+{"timestamp":1712175395.508215,"name":"offline","context":{"idset":"11733"}}
+{"timestamp":1712175395.5083597,"name":"offline","context":{"idset":"11734"}}
+{"timestamp":1712175395.5085232,"name":"offline","context":{"idset":"11735"}}
+{"timestamp":1712175395.5086691,"name":"offline","context":{"idset":"11736"}}
+{"timestamp":1712175395.5088103,"name":"offline","context":{"idset":"11737"}}
+{"timestamp":1712175395.5089512,"name":"offline","context":{"idset":"11738"}}
+{"timestamp":1712175395.5090909,"name":"offline","context":{"idset":"11739"}}
+{"timestamp":1712175395.5092294,"name":"offline","context":{"idset":"11740"}}
+{"timestamp":1712175395.5093668,"name":"offline","context":{"idset":"11741"}}
+{"timestamp":1712175395.5095205,"name":"offline","context":{"idset":"11742"}}
+{"timestamp":1712175395.5096583,"name":"offline","context":{"idset":"11743"}}
+{"timestamp":1712175395.5097933,"name":"offline","context":{"idset":"11744"}}
+{"timestamp":1712175395.5099285,"name":"offline","context":{"idset":"11745"}}
+{"timestamp":1712175395.510062,"name":"offline","context":{"idset":"11746"}}
+{"timestamp":1712175395.510195,"name":"offline","context":{"idset":"11747"}}
+{"timestamp":1712175395.5103271,"name":"offline","context":{"idset":"11748"}}
+{"timestamp":1712175395.5104685,"name":"offline","context":{"idset":"11781"}}
+{"timestamp":1712175395.5106003,"name":"offline","context":{"idset":"11782"}}
+{"timestamp":1712175395.5107303,"name":"offline","context":{"idset":"11783"}}
+{"timestamp":1712175395.5108578,"name":"offline","context":{"idset":"11785"}}
+{"timestamp":1712175395.5109842,"name":"offline","context":{"idset":"11786"}}
+{"timestamp":1712175395.5111129,"name":"offline","context":{"idset":"11787"}}
+{"timestamp":1712175395.5112369,"name":"offline","context":{"idset":"11788"}}
+{"timestamp":1712175395.5113604,"name":"offline","context":{"idset":"11792"}}
+{"timestamp":1712175395.5114923,"name":"offline","context":{"idset":"11795"}}
+{"timestamp":1712175395.5116148,"name":"offline","context":{"idset":"11796"}}
+{"timestamp":1712175395.5117371,"name":"offline","context":{"idset":"11797"}}
+{"timestamp":1712175395.511858,"name":"offline","context":{"idset":"11798"}}
+{"timestamp":1712175395.5204866,"name":"offline","context":{"idset":"11799"}}
+{"timestamp":1712175395.5206099,"name":"offline","context":{"idset":"11800"}}
+{"timestamp":1712175395.5207276,"name":"offline","context":{"idset":"11801"}}
+{"timestamp":1712175395.5208447,"name":"offline","context":{"idset":"11803"}}
+{"timestamp":1712175395.5209603,"name":"offline","context":{"idset":"11804"}}
+{"timestamp":1712175395.5210757,"name":"offline","context":{"idset":"11805"}}
+{"timestamp":1712175395.521189,"name":"offline","context":{"idset":"11806"}}
+{"timestamp":1712175395.5213027,"name":"offline","context":{"idset":"11807"}}
+{"timestamp":1712175395.5214264,"name":"offline","context":{"idset":"11808"}}
+{"timestamp":1712175395.521538,"name":"offline","context":{"idset":"11809"}}
+{"timestamp":1712175395.5216477,"name":"offline","context":{"idset":"11810"}}
+{"timestamp":1712175395.5217583,"name":"offline","context":{"idset":"11811"}}
+{"timestamp":1712175395.5218678,"name":"offline","context":{"idset":"11812"}}
+{"timestamp":1712175395.5219758,"name":"offline","context":{"idset":"11813"}}
+{"timestamp":1712175395.522083,"name":"offline","context":{"idset":"11814"}}
+{"timestamp":1712175395.5221889,"name":"offline","context":{"idset":"11815"}}
+{"timestamp":1712175395.5222924,"name":"offline","context":{"idset":"11817"}}
+{"timestamp":1712175395.5223954,"name":"offline","context":{"idset":"11818"}}
+{"timestamp":1712175395.5225039,"name":"offline","context":{"idset":"11819"}}
+{"timestamp":1712175395.5226054,"name":"offline","context":{"idset":"11820"}}
+{"timestamp":1712175395.522706,"name":"offline","context":{"idset":"11821"}}
+{"timestamp":1712175395.522805,"name":"offline","context":{"idset":"11822"}}
+{"timestamp":1712175395.5229032,"name":"offline","context":{"idset":"11823"}}
+{"timestamp":1712175395.5229995,"name":"offline","context":{"idset":"11824"}}
+{"timestamp":1712175395.5230961,"name":"offline","context":{"idset":"11825"}}
+{"timestamp":1712175395.5231919,"name":"offline","context":{"idset":"11826"}}
+{"timestamp":1712175395.5232866,"name":"offline","context":{"idset":"11827"}}
+{"timestamp":1712175395.5233812,"name":"offline","context":{"idset":"11828"}}
+{"timestamp":1712175395.5234799,"name":"offline","context":{"idset":"11829"}}
+{"timestamp":1712175395.5235722,"name":"offline","context":{"idset":"11830"}}
+{"timestamp":1712175395.523665,"name":"offline","context":{"idset":"11831"}}
+{"timestamp":1712175395.5237565,"name":"offline","context":{"idset":"11832"}}
+{"timestamp":1712175395.5238469,"name":"offline","context":{"idset":"11833"}}
+{"timestamp":1712175395.5239365,"name":"offline","context":{"idset":"11834"}}
+{"timestamp":1712175395.5240242,"name":"offline","context":{"idset":"11835"}}
+{"timestamp":1712175395.5241117,"name":"offline","context":{"idset":"11836"}}
+{"timestamp":1712175395.5241985,"name":"offline","context":{"idset":"11837"}}
+{"timestamp":1712175395.5242846,"name":"offline","context":{"idset":"11838"}}
+{"timestamp":1712175395.52437,"name":"offline","context":{"idset":"11839"}}
+{"timestamp":1712175395.5244584,"name":"offline","context":{"idset":"11840"}}
+{"timestamp":1712175395.5245414,"name":"offline","context":{"idset":"11841"}}
+{"timestamp":1712175395.5246246,"name":"offline","context":{"idset":"11842"}}
+{"timestamp":1712175395.5247068,"name":"offline","context":{"idset":"11843"}}
+{"timestamp":1712175395.5247879,"name":"offline","context":{"idset":"11844"}}
+{"timestamp":1712175395.5248685,"name":"offline","context":{"idset":"11845"}}
+{"timestamp":1712175395.5249469,"name":"offline","context":{"idset":"11846"}}
+{"timestamp":1712175395.5250256,"name":"offline","context":{"idset":"11847"}}
+{"timestamp":1712175395.5251024,"name":"offline","context":{"idset":"11848"}}
+{"timestamp":1712175395.5251791,"name":"offline","context":{"idset":"11849"}}
+{"timestamp":1712175395.5252554,"name":"offline","context":{"idset":"11850"}}
+{"timestamp":1712175395.525331,"name":"offline","context":{"idset":"11851"}}
+{"timestamp":1712175395.5254452,"name":"offline","context":{"idset":"11852"}}
+{"timestamp":1712175395.5255275,"name":"offline","context":{"idset":"11853"}}
+{"timestamp":1712175395.5340507,"name":"offline","context":{"idset":"11854"}}
+{"timestamp":1712175395.5341582,"name":"offline","context":{"idset":"11855"}}
+{"timestamp":1712175395.5342565,"name":"offline","context":{"idset":"11856"}}
+{"timestamp":1712175395.5521576,"name":"offline","context":{"idset":"11857"}}
+{"timestamp":1712175395.5607154,"name":"offline","context":{"idset":"11858"}}
+{"timestamp":1712175395.5607922,"name":"offline","context":{"idset":"11859"}}
+{"timestamp":1712175395.5608625,"name":"offline","context":{"idset":"11860"}}
+{"timestamp":1712175395.56093,"name":"offline","context":{"idset":"11861"}}
+{"timestamp":1712175395.5609977,"name":"offline","context":{"idset":"11862"}}
+{"timestamp":1712175395.5610626,"name":"offline","context":{"idset":"11863"}}
+{"timestamp":1712175395.5611279,"name":"offline","context":{"idset":"11864"}}
+{"timestamp":1712175395.561192,"name":"offline","context":{"idset":"11865"}}
+{"timestamp":1712175395.5612571,"name":"offline","context":{"idset":"11866"}}
+{"timestamp":1712175395.5613194,"name":"offline","context":{"idset":"11867"}}
+{"timestamp":1712175395.5613797,"name":"offline","context":{"idset":"11868"}}
+{"timestamp":1712175395.5614488,"name":"offline","context":{"idset":"11869"}}
+{"timestamp":1712175395.5615098,"name":"offline","context":{"idset":"11870"}}
+{"timestamp":1712175395.5615675,"name":"offline","context":{"idset":"11871"}}
+{"timestamp":1712175395.5616245,"name":"offline","context":{"idset":"11872"}}
+{"timestamp":1712175395.561681,"name":"offline","context":{"idset":"11873"}}
+{"timestamp":1712175395.5617368,"name":"offline","context":{"idset":"11874"}}
+{"timestamp":1712175395.5617909,"name":"offline","context":{"idset":"11875"}}
+{"timestamp":1712175395.5618126,"name":"offline","context":{"idset":"11876"}}
+{"timestamp":1712178469.6317399,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1711985590.1298366,"reason":"broker was unresponsive"},"2":{"timestamp":1711985590.1299634,"reason":"broker was unresponsive"},"3":{"timestamp":1711985590.129997,"reason":"broker was unresponsive"},"4":{"timestamp":1711985590.1300287,"reason":"broker was unresponsive"},"5":{"timestamp":1711985590.130055,"reason":"broker was unresponsive"},"6":{"timestamp":1711985590.1301,"reason":"broker was unresponsive"},"7":{"timestamp":1711985590.1301718,"reason":"broker was unresponsive"},"8":{"timestamp":1711985590.1302528,"reason":"broker was unresponsive"},"9":{"timestamp":1711985590.1303506,"reason":"broker was unresponsive"},"10":{"timestamp":1711985590.1304929,"reason":"broker was unresponsive"},"11":{"timestamp":1711985590.1305854,"reason":"broker was unresponsive"},"12":{"timestamp":1711985590.2635758,"reason":"broker was unresponsive"},"13":{"timestamp":1711986176.1277797,"reason":"broker was unresponsive"},"14":{"timestamp":1711986176.127861,"reason":"broker was unresponsive"},"15":{"timestamp":1711986172.126236,"reason":"broker was unresponsive"},"16":{"timestamp":1711986176.1279109,"reason":"broker was unresponsive"},"17":{"timestamp":1711986172.226109,"reason":"broker was unresponsive"},"18":{"timestamp":1711986176.1279769,"reason":"broker was unresponsive"},"19":{"timestamp":1711986174.2265673,"reason":"broker was unresponsive"},"20":{"timestamp":1711986176.1280706,"reason":"broker was unresponsive"},"21":{"timestamp":1711986176.1281965,"reason":"broker was unresponsive"},"22":{"timestamp":1711986176.128314,"reason":"broker was unresponsive"},"23":{"timestamp":1711986176.1284461,"reason":"broker was unresponsive"},"24":{"timestamp":1711986176.2268984,"reason":"broker was unresponsive"},"25":{"timestamp":1711986712.1284616,"reason":"broker was unresponsive"},"26":{"timestamp":1711986712.1285477,"reason":"broker was unresponsive"},"27":{"timestamp":1711986712.1286163,"reason":"broker was unresponsive"},"28":{"timestamp":1711986712.1287522,"reason":"broker was unresponsive"},"29":{"timestamp":1711986712.1288626,"reason":"broker was unresponsive"},"30":{"timestamp":1711986712.1289904,"reason":"broker was unresponsive"},"31":{"timestamp":1711986712.1291039,"reason":"broker was unresponsive"},"32":{"timestamp":1711986712.1292276,"reason":"broker was unresponsive"},"33":{"timestamp":1711986712.129328,"reason":"broker was unresponsive"},"34":{"timestamp":1711986712.1294577,"reason":"broker was unresponsive"},"35":{"timestamp":1711986712.1295605,"reason":"broker was unresponsive"},"36":{"timestamp":1711986712.2507603,"reason":"broker was unresponsive"},"37":{"timestamp":1711987342.1267064,"reason":"broker was unresponsive"},"38":{"timestamp":1711987342.1267867,"reason":"broker was unresponsive"},"39":{"timestamp":1711987342.1268172,"reason":"broker was unresponsive"},"40":{"timestamp":1711987342.1268439,"reason":"broker was unresponsive"},"41":{"timestamp":1711987342.1269064,"reason":"broker was unresponsive"},"42":{"timestamp":1711987342.1269703,"reason":"broker was unresponsive"},"43":{"timestamp":1711987342.127022,"reason":"broker was unresponsive"},"44":{"timestamp":1711987342.1270769,"reason":"broker was unresponsive"},"45":{"timestamp":1711987342.1271369,"reason":"broker was unresponsive"},"46":{"timestamp":1711987342.1271925,"reason":"broker was unresponsive"},"47":{"timestamp":1711987342.1272507,"reason":"broker was unresponsive"},"48":{"timestamp":1711987342.2340004,"reason":"broker was unresponsive"},"49":{"timestamp":1712154691.6398871,"reason":"broker was unresponsive"},"50":{"timestamp":1712154691.6401901,"reason":"broker was unresponsive"},"51":{"timestamp":1712154691.6406775,"reason":"broker was unresponsive"},"52":{"timestamp":1712154691.6407955,"reason":"broker was unresponsive"},"53":{"timestamp":1712154691.6478553,"reason":"broker was unresponsive"},"54":{"timestamp":1712154691.6479921,"reason":"broker was unresponsive"},"55":{"timestamp":1712154691.6481254,"reason":"broker was unresponsive"},"56":{"timestamp":1712154691.6482697,"reason":"broker was unresponsive"},"57":{"timestamp":1712154691.6483908,"reason":"broker was unresponsive"},"58":{"timestamp":1712154691.6485405,"reason":"broker was unresponsive"},"59":{"timestamp":1712154691.6486492,"reason":"broker was unresponsive"},"60":{"timestamp":1712154691.6555567,"reason":"broker was unresponsive"},"61":{"timestamp":1711668444.9627461,"reason":"broker was unresponsive"},"62":{"timestamp":1711668506.9728317,"reason":"broker was unresponsive"},"63":{"timestamp":1711668445.0625052,"reason":"broker was unresponsive"},"64":{"timestamp":1711668506.9729121,"reason":"broker was unresponsive"},"65":{"timestamp":1711668438.9624472,"reason":"broker was unresponsive"},"66":{"timestamp":1711668506.9729731,"reason":"broker was unresponsive"},"67":{"timestamp":1711668439.0630105,"reason":"broker was unresponsive"},"68":{"timestamp":1711668506.9730346,"reason":"broker was unresponsive"},"69-76":{"timestamp":1711646931.6480982,"reason":""},"77":{"timestamp":1711668526.9621451,"reason":"broker was unresponsive"},"78":{"timestamp":1711668462.9610252,"reason":"broker was unresponsive"},"79":{"timestamp":1711668526.96223,"reason":"broker was unresponsive"},"80":{"timestamp":1711668457.0617182,"reason":"broker was unresponsive"},"81":{"timestamp":1711668526.9622741,"reason":"broker was unresponsive"},"82":{"timestamp":1711668462.9611192,"reason":"broker was unresponsive"},"83":{"timestamp":1711668527.0620673,"reason":"broker was unresponsive"},"84":{"timestamp":1711668463.0615458,"reason":"broker was unresponsive"},"85":{"timestamp":1712154691.6597042,"reason":"broker was unresponsive"},"86":{"timestamp":1712154691.6604736,"reason":"broker was unresponsive"},"87":{"timestamp":1712154691.660594,"reason":"broker was unresponsive"},"88":{"timestamp":1712154691.660717,"reason":"broker was unresponsive"},"89":{"timestamp":1712154691.6608331,"reason":"broker was unresponsive"},"90":{"timestamp":1712154691.6609375,"reason":"broker was unresponsive"},"91":{"timestamp":1712154691.6610425,"reason":"broker was unresponsive"},"92":{"timestamp":1712154691.6611438,"reason":"broker was unresponsive"},"93":{"timestamp":1712154691.6612563,"reason":"broker was unresponsive"},"94":{"timestamp":1712154691.6613679,"reason":"broker was unresponsive"},"95":{"timestamp":1712154691.6635458,"reason":"broker was unresponsive"},"96":{"timestamp":1712154691.6652935,"reason":"broker was unresponsive"},"97":{"timestamp":1712154691.6653471,"reason":"broker was unresponsive"},"98":{"timestamp":1712154691.6798809,"reason":"broker was unresponsive"},"99":{"timestamp":1712154691.6844938,"reason":"broker was unresponsive"},"100":{"timestamp":1712154691.6845479,"reason":"broker was unresponsive"},"101":{"timestamp":1712154691.684592,"reason":"broker was unresponsive"},"102":{"timestamp":1712154691.6846371,"reason":"broker was unresponsive"},"103":{"timestamp":1712154691.6846807,"reason":"broker was unresponsive"},"104":{"timestamp":1712154691.6847265,"reason":"broker was unresponsive"},"105":{"timestamp":1712154691.6847749,"reason":"broker was unresponsive"},"106":{"timestamp":1712154691.6848218,"reason":"broker was unresponsive"},"107":{"timestamp":1712154691.6848698,"reason":"broker was unresponsive"},"108":{"timestamp":1712154691.6849141,"reason":"broker was unresponsive"},"109":{"timestamp":1712154691.684962,"reason":"broker was unresponsive"},"110":{"timestamp":1712154691.685008,"reason":"broker was unresponsive"},"111":{"timestamp":1712154691.6850562,"reason":"broker was unresponsive"},"112":{"timestamp":1712154691.6851053,"reason":"broker was unresponsive"},"113":{"timestamp":1712154691.6851523,"reason":"broker was unresponsive"},"114":{"timestamp":1712154691.6851997,"reason":"broker was unresponsive"},"115":{"timestamp":1712154691.685246,"reason":"broker was unresponsive"},"116":{"timestamp":1712154691.6852949,"reason":"broker was unresponsive"},"117":{"timestamp":1712154691.6853426,"reason":"broker was unresponsive"},"118":{"timestamp":1712154691.6853926,"reason":"broker was unresponsive"},"119":{"timestamp":1712154691.6854501,"reason":"broker was unresponsive"},"120":{"timestamp":1712154691.6854975,"reason":"broker was unresponsive"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"122":{"timestamp":1712154691.6855476,"reason":"broker was unresponsive"},"123":{"timestamp":1712154691.6855998,"reason":"broker was unresponsive"},"124":{"timestamp":1712154691.6856515,"reason":"broker was unresponsive"},"125":{"timestamp":1712154691.6857026,"reason":"broker was unresponsive"},"126":{"timestamp":1712154691.6857526,"reason":"broker was unresponsive"},"127":{"timestamp":1712154691.6858027,"reason":"broker was unresponsive"},"128":{"timestamp":1712154691.6858518,"reason":"broker was unresponsive"},"129":{"timestamp":1712154691.6859028,"reason":"broker was unresponsive"},"130":{"timestamp":1712154691.6859548,"reason":"broker was unresponsive"},"131":{"timestamp":1712154691.6860077,"reason":"broker was unresponsive"},"132":{"timestamp":1712154691.6860585,"reason":"broker was unresponsive"},"133":{"timestamp":1712154691.6861086,"reason":"broker was unresponsive"},"134":{"timestamp":1712154691.6861615,"reason":"broker was unresponsive"},"135":{"timestamp":1712154691.6875262,"reason":"broker was unresponsive"},"136":{"timestamp":1712154691.6875885,"reason":"broker was unresponsive"},"137":{"timestamp":1712154691.6899619,"reason":"broker was unresponsive"},"138":{"timestamp":1712154691.6900213,"reason":"broker was unresponsive"},"139":{"timestamp":1712154691.6900744,"reason":"broker was unresponsive"},"140":{"timestamp":1712154691.6901298,"reason":"broker was unresponsive"},"141":{"timestamp":1712154691.6901815,"reason":"broker was unresponsive"},"142":{"timestamp":1712154691.6902332,"reason":"broker was unresponsive"},"143":{"timestamp":1712154691.6902819,"reason":"broker was unresponsive"},"144":{"timestamp":1712154691.6903317,"reason":"broker was unresponsive"},"145":{"timestamp":1712154691.6903832,"reason":"broker was unresponsive"},"146":{"timestamp":1712154691.6904464,"reason":"broker was unresponsive"},"147":{"timestamp":1712154691.6904979,"reason":"broker was unresponsive"},"148":{"timestamp":1712154691.6905484,"reason":"broker was unresponsive"},"149":{"timestamp":1712154691.6905997,"reason":"broker was unresponsive"},"150":{"timestamp":1712154691.6906526,"reason":"broker was unresponsive"},"151":{"timestamp":1712154691.6907089,"reason":"broker was unresponsive"},"152":{"timestamp":1712154691.6907649,"reason":"broker was unresponsive"},"153":{"timestamp":1712154691.69082,"reason":"broker was unresponsive"},"154":{"timestamp":1712154691.6908724,"reason":"broker was unresponsive"},"155":{"timestamp":1712154691.6909251,"reason":"broker was unresponsive"},"156":{"timestamp":1712154691.6909785,"reason":"broker was unresponsive"},"157":{"timestamp":1712154691.6910326,"reason":"broker was unresponsive"},"158":{"timestamp":1712154691.6910849,"reason":"broker was unresponsive"},"159":{"timestamp":1712154691.6911387,"reason":"broker was unresponsive"},"160":{"timestamp":1712154691.6911941,"reason":"broker was unresponsive"},"161":{"timestamp":1712154691.6912525,"reason":"broker was unresponsive"},"162":{"timestamp":1712154691.6913092,"reason":"broker was unresponsive"},"163":{"timestamp":1712154691.6913657,"reason":"broker was unresponsive"},"164":{"timestamp":1712154691.6914692,"reason":"broker was unresponsive"},"165":{"timestamp":1712154691.6915326,"reason":"broker was unresponsive"},"166":{"timestamp":1712154691.691592,"reason":"broker was unresponsive"},"167":{"timestamp":1712154691.6916521,"reason":"broker was unresponsive"},"168":{"timestamp":1712154691.6917136,"reason":"broker was unresponsive"},"169":{"timestamp":1712154691.6917765,"reason":"broker was unresponsive"},"170":{"timestamp":1712154691.6918373,"reason":"broker was unresponsive"},"171":{"timestamp":1712154691.6918993,"reason":"broker was unresponsive"},"172":{"timestamp":1712154691.6919615,"reason":"broker was unresponsive"},"173":{"timestamp":1712154691.6920226,"reason":"broker was unresponsive"},"174":{"timestamp":1712154691.6920846,"reason":"broker was unresponsive"},"175":{"timestamp":1712154691.6921449,"reason":"broker was unresponsive"},"176":{"timestamp":1712154691.6922057,"reason":"broker was unresponsive"},"177":{"timestamp":1712154691.6922667,"reason":"broker was unresponsive"},"178":{"timestamp":1712154691.6923275,"reason":"broker was unresponsive"},"179":{"timestamp":1712154691.6923888,"reason":"broker was unresponsive"},"180":{"timestamp":1712154691.6924729,"reason":"broker was unresponsive"},"181":{"timestamp":1712154691.6925418,"reason":"broker was unresponsive"},"182":{"timestamp":1712154691.692605,"reason":"broker was unresponsive"},"183":{"timestamp":1712154691.6926687,"reason":"broker was unresponsive"},"184":{"timestamp":1712154691.6927319,"reason":"broker was unresponsive"},"185":{"timestamp":1712154691.6927969,"reason":"broker was unresponsive"},"186":{"timestamp":1712154691.6928613,"reason":"broker was unresponsive"},"187":{"timestamp":1712154691.6929262,"reason":"broker was unresponsive"},"189":{"timestamp":1712154691.6929927,"reason":"broker was unresponsive"},"190":{"timestamp":1712154691.6930578,"reason":"broker was unresponsive"},"191":{"timestamp":1712154691.6931233,"reason":"broker was unresponsive"},"192":{"timestamp":1712154691.6931884,"reason":"broker was unresponsive"},"193":{"timestamp":1712154691.6932549,"reason":"broker was unresponsive"},"194":{"timestamp":1712154691.6933224,"reason":"broker was unresponsive"},"195":{"timestamp":1712154691.6933856,"reason":"broker was unresponsive"},"196":{"timestamp":1712154691.6934624,"reason":"broker was unresponsive"},"197":{"timestamp":1712154691.6935284,"reason":"broker was unresponsive"},"198":{"timestamp":1712154691.6935964,"reason":"broker was unresponsive"},"199":{"timestamp":1712154691.6936646,"reason":"broker was unresponsive"},"200":{"timestamp":1712154691.6937323,"reason":"broker was unresponsive"},"201":{"timestamp":1712154691.6938033,"reason":"broker was unresponsive"},"202":{"timestamp":1712154691.6938739,"reason":"broker was unresponsive"},"203":{"timestamp":1712154691.6939418,"reason":"broker was unresponsive"},"204":{"timestamp":1712154691.694011,"reason":"broker was unresponsive"},"205":{"timestamp":1712154691.6940792,"reason":"broker was unresponsive"},"206":{"timestamp":1712154691.6941504,"reason":"broker was unresponsive"},"207":{"timestamp":1712154691.6942205,"reason":"broker was unresponsive"},"208":{"timestamp":1712154691.6942902,"reason":"broker was unresponsive"},"209":{"timestamp":1712154691.6943579,"reason":"broker was unresponsive"},"210":{"timestamp":1712154691.6944435,"reason":"broker was unresponsive"},"211":{"timestamp":1712154691.6945148,"reason":"broker was unresponsive"},"212":{"timestamp":1712154691.6945879,"reason":"broker was unresponsive"},"213":{"timestamp":1712154691.694659,"reason":"broker was unresponsive"},"214":{"timestamp":1712154691.6947279,"reason":"broker was unresponsive"},"215":{"timestamp":1712154691.6948023,"reason":"broker was unresponsive"},"216":{"timestamp":1712154691.6948771,"reason":"broker was unresponsive"},"218":{"timestamp":1712154691.6949482,"reason":"broker was unresponsive"},"219":{"timestamp":1712154691.6950197,"reason":"broker was unresponsive"},"220":{"timestamp":1712154691.695096,"reason":"broker was unresponsive"},"222":{"timestamp":1712154691.6951735,"reason":"broker was unresponsive"},"224":{"timestamp":1712154691.6952496,"reason":"broker was unresponsive"},"225":{"timestamp":1712154691.695329,"reason":"broker was unresponsive"},"226":{"timestamp":1712154691.6954112,"reason":"broker was unresponsive"},"227":{"timestamp":1712154691.7015183,"reason":"broker was unresponsive"},"228":{"timestamp":1712154691.7017403,"reason":"broker was unresponsive"},"229":{"timestamp":1712154691.702008,"reason":"broker was unresponsive"},"230":{"timestamp":1712154691.7021568,"reason":"broker was unresponsive"},"231":{"timestamp":1712154691.702256,"reason":"broker was unresponsive"},"232":{"timestamp":1712154691.7023766,"reason":"broker was unresponsive"},"233":{"timestamp":1712154691.7027056,"reason":"broker was unresponsive"},"234":{"timestamp":1712154691.7028174,"reason":"broker was unresponsive"},"235":{"timestamp":1712154691.7030404,"reason":"broker was unresponsive"},"236":{"timestamp":1712154691.703239,"reason":"broker was unresponsive"},"237":{"timestamp":1712154691.7033858,"reason":"broker was unresponsive"},"238":{"timestamp":1712154691.703666,"reason":"broker was unresponsive"},"239":{"timestamp":1712154691.7038407,"reason":"broker was unresponsive"},"240":{"timestamp":1712154691.7040567,"reason":"broker was unresponsive"},"241":{"timestamp":1712154691.7042947,"reason":"broker was unresponsive"},"242":{"timestamp":1712154691.7044706,"reason":"broker was unresponsive"},"243":{"timestamp":1712154691.7046885,"reason":"broker was unresponsive"},"244":{"timestamp":1712154691.704932,"reason":"broker was unresponsive"},"245":{"timestamp":1712154691.705106,"reason":"broker was unresponsive"},"246":{"timestamp":1711982548.2285702,"reason":"broker was unresponsive"},"247":{"timestamp":1712154691.7084284,"reason":"broker was unresponsive"},"248":{"timestamp":1712154691.7085428,"reason":"broker was unresponsive"},"249":{"timestamp":1712154691.7086403,"reason":"broker was unresponsive"},"250":{"timestamp":1712154691.7087467,"reason":"broker was unresponsive"},"251":{"timestamp":1712154691.7088439,"reason":"broker was unresponsive"},"252":{"timestamp":1712154691.7089396,"reason":"broker was unresponsive"},"253":{"timestamp":1712154691.7090359,"reason":"broker was unresponsive"},"254":{"timestamp":1712154691.709131,"reason":"broker was unresponsive"},"255":{"timestamp":1712154691.7092288,"reason":"broker was unresponsive"},"256":{"timestamp":1712154691.7093244,"reason":"broker was unresponsive"},"257":{"timestamp":1712154691.7094264,"reason":"broker was unresponsive"},"258":{"timestamp":1712154691.7095149,"reason":"broker was unresponsive"},"259":{"timestamp":1712154691.7096033,"reason":"broker was unresponsive"},"260":{"timestamp":1712154691.7096915,"reason":"broker was unresponsive"},"261":{"timestamp":1712154691.7097797,"reason":"broker was unresponsive"},"262":{"timestamp":1712154691.7098682,"reason":"broker was unresponsive"},"263":{"timestamp":1712154691.7099583,"reason":"broker was unresponsive"},"264":{"timestamp":1712154691.7100489,"reason":"broker was unresponsive"},"265":{"timestamp":1712154691.7101383,"reason":"broker was unresponsive"},"266":{"timestamp":1712154691.710228,"reason":"broker was unresponsive"},"267":{"timestamp":1712154691.7103202,"reason":"broker was unresponsive"},"268":{"timestamp":1712154691.7104197,"reason":"broker was unresponsive"},"269":{"timestamp":1712154691.7105114,"reason":"broker was unresponsive"},"270":{"timestamp":1712154691.710604,"reason":"broker was unresponsive"},"271":{"timestamp":1712154691.7106969,"reason":"broker was unresponsive"},"272":{"timestamp":1712154691.7107887,"reason":"broker was unresponsive"},"273":{"timestamp":1712154691.7108822,"reason":"broker was unresponsive"},"274":{"timestamp":1712154691.710974,"reason":"broker was unresponsive"},"275":{"timestamp":1712154691.7110667,"reason":"broker was unresponsive"},"276":{"timestamp":1712154691.7111592,"reason":"broker was unresponsive"},"277":{"timestamp":1712154691.7112534,"reason":"broker was unresponsive"},"278":{"timestamp":1712154691.7113464,"reason":"broker was unresponsive"},"279":{"timestamp":1712154691.7114518,"reason":"broker was unresponsive"},"280":{"timestamp":1712154691.7115457,"reason":"broker was unresponsive"},"281":{"timestamp":1712154691.7116408,"reason":"broker was unresponsive"},"282":{"timestamp":1712154691.711735,"reason":"broker was unresponsive"},"283":{"timestamp":1712154691.7118301,"reason":"broker was unresponsive"},"284":{"timestamp":1712154691.7119248,"reason":"broker was unresponsive"},"285":{"timestamp":1712154691.7120202,"reason":"broker was unresponsive"},"286":{"timestamp":1712154691.7121158,"reason":"broker was unresponsive"},"287":{"timestamp":1712154691.712213,"reason":"broker was unresponsive"},"288":{"timestamp":1712154691.7123086,"reason":"broker was unresponsive"},"289":{"timestamp":1712154691.7124102,"reason":"broker was unresponsive"},"290":{"timestamp":1712154691.712508,"reason":"broker was unresponsive"},"291":{"timestamp":1712154691.7126062,"reason":"broker was unresponsive"},"292":{"timestamp":1712154691.7127049,"reason":"broker was unresponsive"},"293":{"timestamp":1712154691.7128024,"reason":"broker was unresponsive"},"294":{"timestamp":1712154691.7129002,"reason":"broker was unresponsive"},"295":{"timestamp":1712154691.7129996,"reason":"broker was unresponsive"},"296":{"timestamp":1712154691.7130992,"reason":"broker was unresponsive"},"297":{"timestamp":1712154691.7131979,"reason":"broker was unresponsive"},"298":{"timestamp":1712154691.7132986,"reason":"broker was unresponsive"},"299":{"timestamp":1712154691.7134199,"reason":"broker was unresponsive"},"300":{"timestamp":1712154691.7135201,"reason":"broker was unresponsive"},"301":{"timestamp":1712154691.7136211,"reason":"broker was unresponsive"},"302":{"timestamp":1712154691.713722,"reason":"broker was unresponsive"},"303":{"timestamp":1712154691.7138236,"reason":"broker was unresponsive"},"304":{"timestamp":1712154691.7139258,"reason":"broker was unresponsive"},"305":{"timestamp":1712154691.7140279,"reason":"broker was unresponsive"},"306":{"timestamp":1712154691.7141302,"reason":"broker was unresponsive"},"307":{"timestamp":1712154691.714232,"reason":"broker was unresponsive"},"308":{"timestamp":1712154691.7143342,"reason":"broker was unresponsive"},"309":{"timestamp":1712154691.7144449,"reason":"broker was unresponsive"},"310":{"timestamp":1712154691.7145491,"reason":"broker was unresponsive"},"311":{"timestamp":1712154691.7146521,"reason":"broker was unresponsive"},"312":{"timestamp":1712154691.7147582,"reason":"broker was unresponsive"},"313":{"timestamp":1712154691.7148652,"reason":"broker was unresponsive"},"314":{"timestamp":1712154691.714973,"reason":"broker was unresponsive"},"315":{"timestamp":1712154691.7150784,"reason":"broker was unresponsive"},"316":{"timestamp":1712154691.7151866,"reason":"broker was unresponsive"},"317":{"timestamp":1712154691.715292,"reason":"broker was unresponsive"},"318":{"timestamp":1712154691.7154064,"reason":"broker was unresponsive"},"319":{"timestamp":1712154691.7155137,"reason":"broker was unresponsive"},"320":{"timestamp":1712154691.7156217,"reason":"broker was unresponsive"},"321":{"timestamp":1712154691.7157314,"reason":"broker was unresponsive"},"322":{"timestamp":1712154691.7158403,"reason":"broker was unresponsive"},"323":{"timestamp":1712154691.7159488,"reason":"broker was unresponsive"},"324":{"timestamp":1712154691.7160597,"reason":"broker was unresponsive"},"325":{"timestamp":1712154691.7161684,"reason":"broker was unresponsive"},"327":{"timestamp":1712154691.7162766,"reason":"broker was unresponsive"},"328":{"timestamp":1712154691.716387,"reason":"broker was unresponsive"},"329":{"timestamp":1712154691.7165065,"reason":"broker was unresponsive"},"330":{"timestamp":1712154691.7166176,"reason":"broker was unresponsive"},"331":{"timestamp":1712154691.7167275,"reason":"broker was unresponsive"},"332":{"timestamp":1712154691.7168386,"reason":"broker was unresponsive"},"333":{"timestamp":1712154691.7169497,"reason":"broker was unresponsive"},"334":{"timestamp":1712154691.717062,"reason":"broker was unresponsive"},"335":{"timestamp":1712154691.7171738,"reason":"broker was unresponsive"},"336":{"timestamp":1712154691.7172842,"reason":"broker was unresponsive"},"337":{"timestamp":1712154691.7214179,"reason":"broker was unresponsive"},"338":{"timestamp":1712154691.7215626,"reason":"broker was unresponsive"},"339":{"timestamp":1712154691.7216904,"reason":"broker was unresponsive"},"340":{"timestamp":1712154691.7218168,"reason":"broker was unresponsive"},"341":{"timestamp":1712154691.7219422,"reason":"broker was unresponsive"},"342":{"timestamp":1712154691.7220695,"reason":"broker was unresponsive"},"343":{"timestamp":1712154691.7221949,"reason":"broker was unresponsive"},"344":{"timestamp":1712154691.722321,"reason":"broker was unresponsive"},"345":{"timestamp":1712154691.7224567,"reason":"broker was unresponsive"},"346":{"timestamp":1712154691.722584,"reason":"broker was unresponsive"},"347":{"timestamp":1712154691.7227106,"reason":"broker was unresponsive"},"348":{"timestamp":1711576490.2548447,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"349":{"timestamp":1712154691.7228374,"reason":"broker was unresponsive"},"350":{"timestamp":1712154691.722964,"reason":"broker was unresponsive"},"351":{"timestamp":1712154691.7230921,"reason":"broker was unresponsive"},"352":{"timestamp":1712154691.7232213,"reason":"broker was unresponsive"},"353":{"timestamp":1712154691.7233503,"reason":"broker was unresponsive"},"354":{"timestamp":1712154691.7234862,"reason":"broker was unresponsive"},"355":{"timestamp":1712154691.7236207,"reason":"broker was unresponsive"},"356":{"timestamp":1712154691.7237515,"reason":"broker was unresponsive"},"357":{"timestamp":1712154691.7238832,"reason":"broker was unresponsive"},"358":{"timestamp":1712154691.7240148,"reason":"broker was unresponsive"},"359":{"timestamp":1712154691.7241466,"reason":"broker was unresponsive"},"360":{"timestamp":1712154691.7242789,"reason":"broker was unresponsive"},"361":{"timestamp":1712154691.7244198,"reason":"broker was unresponsive"},"362":{"timestamp":1712154691.7245531,"reason":"broker was unresponsive"},"363":{"timestamp":1712154691.7246847,"reason":"broker was unresponsive"},"364":{"timestamp":1712154691.7248199,"reason":"broker was unresponsive"},"365":{"timestamp":1712154691.7249532,"reason":"broker was unresponsive"},"366":{"timestamp":1712154691.7250876,"reason":"broker was unresponsive"},"367":{"timestamp":1712154691.7252233,"reason":"broker was unresponsive"},"368":{"timestamp":1712154691.7253551,"reason":"broker was unresponsive"},"369":{"timestamp":1712154691.7254889,"reason":"broker was unresponsive"},"370":{"timestamp":1712154691.7256119,"reason":"broker was unresponsive"},"371":{"timestamp":1712154691.7257328,"reason":"broker was unresponsive"},"372":{"timestamp":1712154691.7258542,"reason":"broker was unresponsive"},"373":{"timestamp":1712154691.7259772,"reason":"broker was unresponsive"},"374":{"timestamp":1712154691.7261055,"reason":"broker was unresponsive"},"375":{"timestamp":1712154691.7262285,"reason":"broker was unresponsive"},"376":{"timestamp":1712154691.7263508,"reason":"broker was unresponsive"},"377":{"timestamp":1712154691.7264836,"reason":"broker was unresponsive"},"378":{"timestamp":1712154691.7266088,"reason":"broker was unresponsive"},"379":{"timestamp":1712154691.7267334,"reason":"broker was unresponsive"},"380":{"timestamp":1712154691.7268579,"reason":"broker was unresponsive"},"381":{"timestamp":1712154691.7269838,"reason":"broker was unresponsive"},"382":{"timestamp":1712154691.7271094,"reason":"broker was unresponsive"},"383":{"timestamp":1712154691.7272341,"reason":"broker was unresponsive"},"384":{"timestamp":1712154691.7273588,"reason":"broker was unresponsive"},"385":{"timestamp":1712154691.7324924,"reason":"broker was unresponsive"},"386":{"timestamp":1712154691.7326367,"reason":"broker was unresponsive"},"387":{"timestamp":1712154691.7327676,"reason":"broker was unresponsive"},"388":{"timestamp":1712154691.7328908,"reason":"broker was unresponsive"},"389":{"timestamp":1712154691.7330179,"reason":"broker was unresponsive"},"390":{"timestamp":1712154691.7331469,"reason":"broker was unresponsive"},"391":{"timestamp":1712154691.7332675,"reason":"broker was unresponsive"},"392":{"timestamp":1712154691.7333806,"reason":"broker was unresponsive"},"393":{"timestamp":1712154691.7335205,"reason":"broker was unresponsive"},"394":{"timestamp":1712154691.7336516,"reason":"broker was unresponsive"},"395":{"timestamp":1712154691.7337768,"reason":"broker was unresponsive"},"396":{"timestamp":1712154691.7339132,"reason":"broker was unresponsive"},"397":{"timestamp":1712154691.7340493,"reason":"broker was unresponsive"},"398":{"timestamp":1712154691.7341723,"reason":"broker was unresponsive"},"399":{"timestamp":1712154691.7342989,"reason":"broker was unresponsive"},"400":{"timestamp":1712154691.7344391,"reason":"broker was unresponsive"},"401":{"timestamp":1712154691.7345707,"reason":"broker was unresponsive"},"402":{"timestamp":1712154691.7347057,"reason":"broker was unresponsive"},"403":{"timestamp":1712154691.7348571,"reason":"broker was unresponsive"},"404":{"timestamp":1712154691.7350051,"reason":"broker was unresponsive"},"405":{"timestamp":1712154691.7351537,"reason":"broker was unresponsive"},"406":{"timestamp":1712154691.7352865,"reason":"broker was unresponsive"},"407":{"timestamp":1712154691.7354391,"reason":"broker was unresponsive"},"408":{"timestamp":1712154691.7355785,"reason":"broker was unresponsive"},"409":{"timestamp":1712154691.7357144,"reason":"broker was unresponsive"},"410":{"timestamp":1712154691.7358544,"reason":"broker was unresponsive"},"411":{"timestamp":1712154691.73599,"reason":"broker was unresponsive"},"412":{"timestamp":1712154691.7361226,"reason":"broker was unresponsive"},"413":{"timestamp":1712154691.7362478,"reason":"broker was unresponsive"},"414":{"timestamp":1712154691.7363818,"reason":"broker was unresponsive"},"415":{"timestamp":1712154691.7365298,"reason":"broker was unresponsive"},"416":{"timestamp":1712154691.7366652,"reason":"broker was unresponsive"},"417":{"timestamp":1712154691.7367952,"reason":"broker was unresponsive"},"418":{"timestamp":1712154691.736927,"reason":"broker was unresponsive"},"419":{"timestamp":1712154691.7370627,"reason":"broker was unresponsive"},"420":{"timestamp":1712154691.7371974,"reason":"broker was unresponsive"},"421":{"timestamp":1712154691.7373383,"reason":"broker was unresponsive"},"422":{"timestamp":1712154691.7374895,"reason":"broker was unresponsive"},"423":{"timestamp":1712154691.7376387,"reason":"broker was unresponsive"},"424":{"timestamp":1712154691.737777,"reason":"broker was unresponsive"},"425":{"timestamp":1712154691.7379177,"reason":"broker was unresponsive"},"426":{"timestamp":1712154691.743413,"reason":"broker was unresponsive"},"427":{"timestamp":1712154691.7435708,"reason":"broker was unresponsive"},"428":{"timestamp":1712154691.7437136,"reason":"broker was unresponsive"},"429":{"timestamp":1712154691.7438543,"reason":"broker was unresponsive"},"430":{"timestamp":1712154691.7439935,"reason":"broker was unresponsive"},"431":{"timestamp":1711096934.358676,"reason":"nodediag failed pci"},"432":{"timestamp":1712154691.7441335,"reason":"broker was unresponsive"},"433":{"timestamp":1712154691.7442758,"reason":"broker was unresponsive"},"434":{"timestamp":1712154691.7444334,"reason":"broker was unresponsive"},"435":{"timestamp":1712154691.7445731,"reason":"broker was unresponsive"},"436":{"timestamp":1712154691.7447121,"reason":"broker was unresponsive"},"437":{"timestamp":1712154691.7448521,"reason":"broker was unresponsive"},"438":{"timestamp":1712154691.7449913,"reason":"broker was unresponsive"},"439":{"timestamp":1712154691.7451386,"reason":"broker was unresponsive"},"440":{"timestamp":1712154691.7452812,"reason":"broker was unresponsive"},"441":{"timestamp":1712154691.7454312,"reason":"broker was unresponsive"},"442":{"timestamp":1712154691.7455742,"reason":"broker was unresponsive"},"443":{"timestamp":1712154691.745717,"reason":"broker was unresponsive"},"444":{"timestamp":1712154691.7458611,"reason":"broker was unresponsive"},"469":{"timestamp":1712154691.7460058,"reason":"broker was unresponsive"},"470":{"timestamp":1712154691.7461517,"reason":"broker was unresponsive"},"471":{"timestamp":1712154691.7463119,"reason":"broker was unresponsive"},"472":{"timestamp":1712154691.7464659,"reason":"broker was unresponsive"},"473":{"timestamp":1712154691.7466099,"reason":"broker was unresponsive"},"474":{"timestamp":1712154691.7467546,"reason":"broker was unresponsive"},"475":{"timestamp":1712154691.7468998,"reason":"broker was unresponsive"},"476":{"timestamp":1712154691.7470443,"reason":"broker was unresponsive"},"477":{"timestamp":1712154691.7471905,"reason":"broker was unresponsive"},"478":{"timestamp":1712154691.7473381,"reason":"broker was unresponsive"},"479":{"timestamp":1712154691.7474983,"reason":"broker was unresponsive"},"480":{"timestamp":1712154691.7476485,"reason":"broker was unresponsive"},"481":{"timestamp":1712154691.7477965,"reason":"broker was unresponsive"},"482":{"timestamp":1712154691.7479465,"reason":"broker was unresponsive"},"483":{"timestamp":1712154691.7480977,"reason":"broker was unresponsive"},"484":{"timestamp":1712154691.7482476,"reason":"broker was unresponsive"},"485":{"timestamp":1712154691.7484069,"reason":"broker was unresponsive"},"486":{"timestamp":1712154691.7485554,"reason":"broker was unresponsive"},"487":{"timestamp":1712154691.7487063,"reason":"broker was unresponsive"},"488":{"timestamp":1712154691.7488544,"reason":"broker was unresponsive"},"489":{"timestamp":1712154691.7490056,"reason":"broker was unresponsive"},"490":{"timestamp":1712154691.7491577,"reason":"broker was unresponsive"},"491":{"timestamp":1712154691.7493105,"reason":"broker was unresponsive"},"492":{"timestamp":1712154691.749469,"reason":"broker was unresponsive"},"493":{"timestamp":1712154691.7496233,"reason":"broker was unresponsive"},"494":{"timestamp":1712154691.7604153,"reason":"broker was unresponsive"},"495":{"timestamp":1712154691.7605824,"reason":"broker was unresponsive"},"496":{"timestamp":1712154691.7607298,"reason":"broker was unresponsive"},"497":{"timestamp":1712154691.7608891,"reason":"broker was unresponsive"},"498":{"timestamp":1712154691.7610483,"reason":"broker was unresponsive"},"499":{"timestamp":1712154691.7612071,"reason":"broker was unresponsive"},"500":{"timestamp":1712154691.7613707,"reason":"broker was unresponsive"},"501":{"timestamp":1712154691.7615492,"reason":"broker was unresponsive"},"502":{"timestamp":1712154691.7617049,"reason":"broker was unresponsive"},"503":{"timestamp":1712154691.7618606,"reason":"broker was unresponsive"},"504":{"timestamp":1712154691.7620151,"reason":"broker was unresponsive"},"505":{"timestamp":1712154691.7621713,"reason":"broker was unresponsive"},"506":{"timestamp":1712154691.7623255,"reason":"broker was unresponsive"},"507":{"timestamp":1712154691.762495,"reason":"broker was unresponsive"},"508":{"timestamp":1712154691.7626514,"reason":"broker was unresponsive"},"509":{"timestamp":1712154691.7628117,"reason":"broker was unresponsive"},"510":{"timestamp":1712154691.7629704,"reason":"broker was unresponsive"},"511":{"timestamp":1712154691.7631311,"reason":"broker was unresponsive"},"512":{"timestamp":1712154691.7632918,"reason":"broker was unresponsive"},"513":{"timestamp":1712154691.7634594,"reason":"broker was unresponsive"},"514":{"timestamp":1712154691.7636204,"reason":"broker was unresponsive"},"515":{"timestamp":1712154691.7637794,"reason":"broker was unresponsive"},"516":{"timestamp":1712154691.7639389,"reason":"broker was unresponsive"},"517":{"timestamp":1712154691.7640977,"reason":"broker was unresponsive"},"518":{"timestamp":1712154691.7642608,"reason":"broker was unresponsive"},"519":{"timestamp":1712154691.7704351,"reason":"broker was unresponsive"},"520":{"timestamp":1712154691.7706141,"reason":"broker was unresponsive"},"521":{"timestamp":1712154691.7707744,"reason":"broker was unresponsive"},"522":{"timestamp":1712154691.7709324,"reason":"broker was unresponsive"},"523":{"timestamp":1712154691.7710917,"reason":"broker was unresponsive"},"524":{"timestamp":1712154691.7712505,"reason":"broker was unresponsive"},"525":{"timestamp":1712154691.7714393,"reason":"broker was unresponsive"},"526":{"timestamp":1712154691.771601,"reason":"broker was unresponsive"},"527":{"timestamp":1712154691.7717626,"reason":"broker was unresponsive"},"528":{"timestamp":1712154691.7719235,"reason":"broker was unresponsive"},"529":{"timestamp":1712154691.772085,"reason":"broker was unresponsive"},"530":{"timestamp":1712154691.7722466,"reason":"broker was unresponsive"},"531":{"timestamp":1712154691.772414,"reason":"broker was unresponsive"},"532":{"timestamp":1712154691.7725685,"reason":"broker was unresponsive"},"533":{"timestamp":1712154691.7727175,"reason":"broker was unresponsive"},"534":{"timestamp":1712154691.7728746,"reason":"broker was unresponsive"},"535":{"timestamp":1712154691.7730417,"reason":"broker was unresponsive"},"536":{"timestamp":1712154691.7732072,"reason":"broker was unresponsive"},"537":{"timestamp":1712154691.7733757,"reason":"broker was unresponsive"},"538":{"timestamp":1712154691.7735519,"reason":"broker was unresponsive"},"539":{"timestamp":1712154691.7737143,"reason":"broker was unresponsive"},"540":{"timestamp":1712154691.7738755,"reason":"broker was unresponsive"},"445-468,541-564":{"timestamp":1711560189.6111379,"reason":"CDU work TB"},"565":{"timestamp":1712154691.7740443,"reason":"broker was unresponsive"},"566":{"timestamp":1712154691.7742167,"reason":"broker was unresponsive"},"567":{"timestamp":1712154691.774384,"reason":"broker was unresponsive"},"568":{"timestamp":1712154691.7745574,"reason":"broker was unresponsive"},"569":{"timestamp":1712154691.7747273,"reason":"broker was unresponsive"},"570":{"timestamp":1712154691.7749047,"reason":"broker was unresponsive"},"571":{"timestamp":1712154691.7750826,"reason":"broker was unresponsive"},"572":{"timestamp":1712154691.7752657,"reason":"broker was unresponsive"},"573":{"timestamp":1712154691.775445,"reason":"broker was unresponsive"},"574":{"timestamp":1712154691.7756157,"reason":"broker was unresponsive"},"575":{"timestamp":1712154691.7757812,"reason":"broker was unresponsive"},"576":{"timestamp":1712154691.7759368,"reason":"broker was unresponsive"},"577":{"timestamp":1712154691.7760947,"reason":"broker was unresponsive"},"578":{"timestamp":1712154691.7762668,"reason":"broker was unresponsive"},"579":{"timestamp":1712154691.7764611,"reason":"broker was unresponsive"},"580":{"timestamp":1712154691.7766304,"reason":"broker was unresponsive"},"581":{"timestamp":1712154691.7767966,"reason":"broker was unresponsive"},"582":{"timestamp":1712154691.7769599,"reason":"broker was unresponsive"},"583":{"timestamp":1712154691.7771285,"reason":"broker was unresponsive"},"584":{"timestamp":1712154691.7773046,"reason":"broker was unresponsive"},"585":{"timestamp":1712154691.7774849,"reason":"broker was unresponsive"},"586":{"timestamp":1712154691.7776549,"reason":"broker was unresponsive"},"587":{"timestamp":1712154691.777828,"reason":"broker was unresponsive"},"588":{"timestamp":1712154691.7780101,"reason":"broker was unresponsive"},"589":{"timestamp":1712154691.7782385,"reason":"broker was unresponsive"},"590":{"timestamp":1712154691.7894335,"reason":"broker was unresponsive"},"591":{"timestamp":1712154691.7896128,"reason":"broker was unresponsive"},"592":{"timestamp":1712154691.7897892,"reason":"broker was unresponsive"},"593":{"timestamp":1712154691.7899675,"reason":"broker was unresponsive"},"594":{"timestamp":1712154691.7901475,"reason":"broker was unresponsive"},"595":{"timestamp":1712154691.7903228,"reason":"broker was unresponsive"},"596":{"timestamp":1712154691.7904975,"reason":"broker was unresponsive"},"597":{"timestamp":1712154691.7906611,"reason":"broker was unresponsive"},"598":{"timestamp":1712154691.7908409,"reason":"broker was unresponsive"},"599":{"timestamp":1712154691.7910223,"reason":"broker was unresponsive"},"600":{"timestamp":1712154691.7911994,"reason":"broker was unresponsive"},"601":{"timestamp":1712154691.7913766,"reason":"broker was unresponsive"},"602":{"timestamp":1712154691.7915711,"reason":"broker was unresponsive"},"603":{"timestamp":1712154691.7917581,"reason":"broker was unresponsive"},"604":{"timestamp":1712154691.7919462,"reason":"broker was unresponsive"},"605":{"timestamp":1712154691.7921228,"reason":"broker was unresponsive"},"606":{"timestamp":1712154691.7922993,"reason":"broker was unresponsive"},"607":{"timestamp":1712154691.7924814,"reason":"broker was unresponsive"},"608":{"timestamp":1712154691.7926571,"reason":"broker was unresponsive"},"609":{"timestamp":1712154691.7928505,"reason":"broker was unresponsive"},"610":{"timestamp":1712154691.7930374,"reason":"broker was unresponsive"},"611":{"timestamp":1712154691.7932198,"reason":"broker was unresponsive"},"612":{"timestamp":1712154691.7934685,"reason":"broker was unresponsive"},"613":{"timestamp":1712154691.7936568,"reason":"broker was unresponsive"},"614":{"timestamp":1712154691.7938452,"reason":"broker was unresponsive"},"615":{"timestamp":1712154691.7940338,"reason":"broker was unresponsive"},"616":{"timestamp":1712154691.7942197,"reason":"broker was unresponsive"},"617":{"timestamp":1712154691.794416,"reason":"broker was unresponsive"},"618":{"timestamp":1712154691.7946017,"reason":"broker was unresponsive"},"619":{"timestamp":1712154691.7947829,"reason":"broker was unresponsive"},"620":{"timestamp":1712154691.7949672,"reason":"broker was unresponsive"},"621":{"timestamp":1712154691.7951493,"reason":"broker was unresponsive"},"622":{"timestamp":1712154691.7953339,"reason":"broker was unresponsive"},"623":{"timestamp":1712154691.7955184,"reason":"broker was unresponsive"},"624":{"timestamp":1712154691.7956882,"reason":"broker was unresponsive"},"625":{"timestamp":1712154691.7958715,"reason":"broker was unresponsive"},"626":{"timestamp":1712154691.7960589,"reason":"broker was unresponsive"},"627":{"timestamp":1712154691.7962403,"reason":"broker was unresponsive"},"628":{"timestamp":1712154691.796427,"reason":"broker was unresponsive"},"629":{"timestamp":1712154691.7966206,"reason":"broker was unresponsive"},"630":{"timestamp":1712154691.7968118,"reason":"broker was unresponsive"},"631":{"timestamp":1712154691.7969999,"reason":"broker was unresponsive"},"632":{"timestamp":1712154691.8208048,"reason":"broker was unresponsive"},"633":{"timestamp":1712154691.8343539,"reason":"broker was unresponsive"},"634":{"timestamp":1712154691.8508646,"reason":"broker was unresponsive"},"635":{"timestamp":1712154691.861587,"reason":"broker was unresponsive"},"637":{"timestamp":1711642252.1616328,"reason":"broker was unresponsive"},"638":{"timestamp":1711642252.1617086,"reason":"broker was unresponsive"},"639":{"timestamp":1711642252.161751,"reason":"broker was unresponsive"},"640":{"timestamp":1711642252.1617904,"reason":"broker was unresponsive"},"641":{"timestamp":1711642252.1618268,"reason":"broker was unresponsive"},"642":{"timestamp":1711642252.1618621,"reason":"broker was unresponsive"},"643":{"timestamp":1711642252.1618974,"reason":"broker was unresponsive"},"644":{"timestamp":1711642252.1619322,"reason":"broker was unresponsive"},"646":{"timestamp":1711586857.201689,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"647":{"timestamp":1711587136.0002518,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"648":{"timestamp":1711587197.7047591,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"649":{"timestamp":1711587211.9606316,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"650":{"timestamp":1711587232.1780922,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"651":{"timestamp":1711587244.2040467,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"652":{"timestamp":1711587255.7559283,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"653":{"timestamp":1711642252.1619685,"reason":"broker was unresponsive"},"654":{"timestamp":1711642252.16201,"reason":"broker was unresponsive"},"655":{"timestamp":1711642252.162055,"reason":"broker was unresponsive"},"656":{"timestamp":1711642252.1620932,"reason":"broker was unresponsive"},"657":{"timestamp":1711642252.1621311,"reason":"broker was unresponsive"},"658":{"timestamp":1711642252.1621685,"reason":"broker was unresponsive"},"659":{"timestamp":1711642252.1622062,"reason":"broker was unresponsive"},"660":{"timestamp":1711642252.1622446,"reason":"broker was unresponsive"},"661":{"timestamp":1711642252.1622829,"reason":"broker was unresponsive"},"662":{"timestamp":1711642252.1623201,"reason":"broker was unresponsive"},"663":{"timestamp":1711642252.1623569,"reason":"broker was unresponsive"},"664":{"timestamp":1711642252.162394,"reason":"broker was unresponsive"},"665":{"timestamp":1711642252.1624658,"reason":"broker was unresponsive"},"666":{"timestamp":1711642252.1625173,"reason":"broker was unresponsive"},"667":{"timestamp":1711642258.156193,"reason":"broker was unresponsive"},"668":{"timestamp":1711642252.1625559,"reason":"broker was unresponsive"},"669":{"timestamp":1711642252.1625948,"reason":"broker was unresponsive"},"670":{"timestamp":1711642252.1626337,"reason":"broker was unresponsive"},"671":{"timestamp":1711642252.162673,"reason":"broker was unresponsive"},"672":{"timestamp":1711642252.1627119,"reason":"broker was unresponsive"},"673":{"timestamp":1711642252.1627512,"reason":"broker was unresponsive"},"674":{"timestamp":1711642252.1627915,"reason":"broker was unresponsive"},"675":{"timestamp":1711642252.1628311,"reason":"broker was unresponsive"},"676":{"timestamp":1711642252.1628704,"reason":"broker was unresponsive"},"677":{"timestamp":1711642258.1562793,"reason":"broker was unresponsive"},"678":{"timestamp":1711642258.156333,"reason":"broker was unresponsive"},"679":{"timestamp":1711642252.1629126,"reason":"broker was unresponsive"},"680":{"timestamp":1711642252.1629534,"reason":"broker was unresponsive"},"681":{"timestamp":1711642258.1563835,"reason":"broker was unresponsive"},"682":{"timestamp":1711642252.1629946,"reason":"broker was unresponsive"},"683":{"timestamp":1711642252.1630385,"reason":"broker was unresponsive"},"684":{"timestamp":1711642252.5597043,"reason":"broker was unresponsive"},"685":{"timestamp":1711604960.2562068,"reason":"broker was unresponsive"},"687":{"timestamp":1711604954.2552111,"reason":"broker was unresponsive"},"688":{"timestamp":1711606062.2558303,"reason":"broker was unresponsive"},"689":{"timestamp":1711604580.2552168,"reason":"broker was unresponsive"},"691":{"timestamp":1711604958.2564192,"reason":"broker was unresponsive"},"692":{"timestamp":1711606406.1556187,"reason":"broker was unresponsive"},"693":{"timestamp":1711636838.2555192,"reason":"broker was unresponsive"},"695":{"timestamp":1711604326.2549248,"reason":"broker was unresponsive"},"696":{"timestamp":1711606406.256216,"reason":"broker was unresponsive"},"697":{"timestamp":1711604392.2562771,"reason":"broker was unresponsive"},"698":{"timestamp":1711605716.2557297,"reason":"broker was unresponsive"},"699":{"timestamp":1711604948.2558951,"reason":"broker was unresponsive"},"700":{"timestamp":1711606868.2557678,"reason":"broker was unresponsive"},"701":{"timestamp":1711604340.2561386,"reason":"broker was unresponsive"},"702":{"timestamp":1711606408.2602317,"reason":"broker was unresponsive"},"703":{"timestamp":1711604918.2557685,"reason":"broker was unresponsive"},"704":{"timestamp":1711604590.2548223,"reason":"broker was unresponsive"},"705":{"timestamp":1711604968.2570915,"reason":"broker was unresponsive"},"707":{"timestamp":1711604908.2558758,"reason":"broker was unresponsive"},"686,690,694,706,708":{"timestamp":1711615908.1602454,"reason":"epilog failed for jobid fm9HsYwFjjd"},"709":{"timestamp":1711408620.9668899,"reason":"Cabinet power work needed -jrg"},"710":{"timestamp":1711409910.967391,"reason":"Cabinet power work needed -jrg"},"711":{"timestamp":1711409098.9674611,"reason":"Cabinet power work needed -jrg"},"712":{"timestamp":1711411606.9678445,"reason":"Cabinet power work needed -jrg"},"713":{"timestamp":1711408618.8669355,"reason":"Cabinet power work needed -jrg"},"715":{"timestamp":1711409132.9680476,"reason":"Cabinet power work needed -jrg"},"714,716":{"timestamp":1711564709.7804377,"reason":"Cabinet power work needed -jrg"},"718":{"timestamp":1711642210.1588371,"reason":"broker was unresponsive"},"719":{"timestamp":1711642210.1589563,"reason":"broker was unresponsive"},"720":{"timestamp":1711642210.1590128,"reason":"broker was unresponsive"},"721":{"timestamp":1711642210.1590643,"reason":"broker was unresponsive"},"722":{"timestamp":1711642210.1590991,"reason":"broker was unresponsive"},"723":{"timestamp":1711642210.1591821,"reason":"broker was unresponsive"},"724":{"timestamp":1711642210.1592195,"reason":"broker was unresponsive"},"725":{"timestamp":1711593222.2542431,"reason":"broker was unresponsive"},"726":{"timestamp":1711593312.2549782,"reason":"broker was unresponsive"},"727":{"timestamp":1711593332.2552569,"reason":"broker was unresponsive"},"728":{"timestamp":1711593356.2547033,"reason":"broker was unresponsive"},"729":{"timestamp":1711593414.254441,"reason":"broker was unresponsive"},"730":{"timestamp":1711593462.2552798,"reason":"broker was unresponsive"},"731":{"timestamp":1711593510.2550235,"reason":"broker was unresponsive"},"732":{"timestamp":1711593530.2558599,"reason":"broker was unresponsive"},"733":{"timestamp":1711642210.1592536,"reason":"broker was unresponsive"},"734":{"timestamp":1711486000.9676955,"reason":"broker was unresponsive"},"735":{"timestamp":1711642210.1592872,"reason":"broker was unresponsive"},"736":{"timestamp":1711642210.1593204,"reason":"broker was unresponsive"},"737":{"timestamp":1711642210.1593742,"reason":"broker was unresponsive"},"738":{"timestamp":1711642210.1594355,"reason":"broker was unresponsive"},"739":{"timestamp":1711642210.1594796,"reason":"broker was unresponsive"},"740":{"timestamp":1711642210.159517,"reason":"broker was unresponsive"},"741":{"timestamp":1711642210.1595554,"reason":"broker was unresponsive"},"742":{"timestamp":1711642210.1595943,"reason":"broker was unresponsive"},"743":{"timestamp":1711642210.1596365,"reason":"broker was unresponsive"},"744":{"timestamp":1711642210.1596746,"reason":"broker was unresponsive"},"745":{"timestamp":1711642210.1597111,"reason":"broker was unresponsive"},"746":{"timestamp":1711506258.2527435,"reason":"broker was unresponsive"},"747":{"timestamp":1711642210.1597559,"reason":"broker was unresponsive"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"},"749":{"timestamp":1711642210.1598115,"reason":"broker was unresponsive"},"750":{"timestamp":1711642210.1598501,"reason":"broker was unresponsive"},"751":{"timestamp":1711642210.159888,"reason":"broker was unresponsive"},"752":{"timestamp":1711642210.1599293,"reason":"broker was unresponsive"},"753":{"timestamp":1711642210.1599746,"reason":"broker was unresponsive"},"754":{"timestamp":1711642210.1600144,"reason":"broker was unresponsive"},"755":{"timestamp":1711642210.1600802,"reason":"broker was unresponsive"},"756":{"timestamp":1711642210.5039413,"reason":"broker was unresponsive"},"789-796":{"timestamp":1711461777.1299369,"reason":""},"809":{"timestamp":1711667251.2030878,"reason":"broker was unresponsive"},"810":{"timestamp":1711460371.1431019,"reason":""},"841":{"timestamp":1712084137.9055309,"reason":""},"843":{"timestamp":1712088721.1281126,"reason":"broker was unresponsive"},"844":{"timestamp":1712088721.1284544,"reason":"broker was unresponsive"},"869":{"timestamp":1712154691.8909535,"reason":"broker was unresponsive"},"870":{"timestamp":1712154691.9068179,"reason":"broker was unresponsive"},"874":{"timestamp":1711668507.0706048,"reason":"broker was unresponsive"},"875":{"timestamp":1711668503.0621645,"reason":"broker was unresponsive"},"881":{"timestamp":1712154691.9234848,"reason":"broker was unresponsive"},"882":{"timestamp":1712154691.9423342,"reason":"broker was unresponsive"},"883-884":{"timestamp":1711734121.1560712,"reason":""},"965":{"timestamp":1711663706.9630723,"reason":"broker was unresponsive"},"966":{"timestamp":1711663706.9631405,"reason":"broker was unresponsive"},"967":{"timestamp":1711663706.963171,"reason":"broker was unresponsive"},"968":{"timestamp":1711663706.9631999,"reason":"broker was unresponsive"},"969":{"timestamp":1712154691.9622352,"reason":"broker was unresponsive"},"970":{"timestamp":1712154691.9624407,"reason":"broker was unresponsive"},"971":{"timestamp":1712154691.962635,"reason":"broker was unresponsive"},"972":{"timestamp":1712154691.9628313,"reason":"broker was unresponsive"},"973":{"timestamp":1712154691.9630353,"reason":"broker was unresponsive"},"974":{"timestamp":1712154691.9632485,"reason":"broker was unresponsive"},"975":{"timestamp":1712154691.9844565,"reason":"broker was unresponsive"},"976":{"timestamp":1712154691.9846833,"reason":"broker was unresponsive"},"977":{"timestamp":1712154691.9848833,"reason":"broker was unresponsive"},"978":{"timestamp":1712154691.9850831,"reason":"broker was unresponsive"},"979":{"timestamp":1712154691.98528,"reason":"broker was unresponsive"},"980":{"timestamp":1712154691.9854927,"reason":"broker was unresponsive"},"9973":{"timestamp":1712154691.9859102,"reason":"broker was unresponsive"},"9974":{"timestamp":1712154691.9863,"reason":"broker was unresponsive"},"9975":{"timestamp":1712154691.9867265,"reason":"broker was unresponsive"},"9976":{"timestamp":1712154691.9871423,"reason":"broker was unresponsive"},"9977":{"timestamp":1712154691.9875474,"reason":"broker was unresponsive"},"9978":{"timestamp":1712154691.9879613,"reason":"broker was unresponsive"},"9979":{"timestamp":1712154691.9883626,"reason":"broker was unresponsive"},"9980":{"timestamp":1712154691.9887779,"reason":"broker was unresponsive"},"9981":{"timestamp":1712154691.9891841,"reason":"broker was unresponsive"},"9982":{"timestamp":1712154691.9895952,"reason":"broker was unresponsive"},"9983":{"timestamp":1712154691.99,"reason":"broker was unresponsive"},"9984":{"timestamp":1712154691.9905057,"reason":"broker was unresponsive"},"9985":{"timestamp":1712154691.9910107,"reason":"broker was unresponsive"},"9986":{"timestamp":1712154691.9914284,"reason":"broker was unresponsive"},"9987":{"timestamp":1712154691.9929512,"reason":"broker was unresponsive"},"9988":{"timestamp":1712154691.9933772,"reason":"broker was unresponsive"},"9989":{"timestamp":1712154691.9938779,"reason":"broker was unresponsive"},"9990":{"timestamp":1712154691.9942834,"reason":"broker was unresponsive"},"9991":{"timestamp":1712154691.994761,"reason":"broker was unresponsive"},"9992":{"timestamp":1712154691.9951608,"reason":"broker was unresponsive"},"9993":{"timestamp":1712154691.9966958,"reason":"broker was unresponsive"},"9994":{"timestamp":1712154691.9973536,"reason":"broker was unresponsive"},"9995":{"timestamp":1712154691.9977736,"reason":"broker was unresponsive"},"9996":{"timestamp":1712154691.9987879,"reason":"broker was unresponsive"},"9997":{"timestamp":1712154691.9992223,"reason":"broker was unresponsive"},"9998":{"timestamp":1712154691.9998438,"reason":"broker was unresponsive"},"9999":{"timestamp":1712154692.0004478,"reason":"broker was unresponsive"},"10000":{"timestamp":1712154692.0009925,"reason":"broker was unresponsive"},"10001":{"timestamp":1712154692.00158,"reason":"broker was unresponsive"},"10002":{"timestamp":1712154692.0020142,"reason":"broker was unresponsive"},"10003":{"timestamp":1712154692.0027688,"reason":"broker was unresponsive"},"10004":{"timestamp":1712154692.003221,"reason":"broker was unresponsive"},"10005":{"timestamp":1712154692.0038302,"reason":"broker was unresponsive"},"10006":{"timestamp":1712154692.0042663,"reason":"broker was unresponsive"},"10007":{"timestamp":1712154692.0048888,"reason":"broker was unresponsive"},"10008":{"timestamp":1712154692.0053651,"reason":"broker was unresponsive"},"10009":{"timestamp":1712154692.0059037,"reason":"broker was unresponsive"},"10010":{"timestamp":1712154692.0063317,"reason":"broker was unresponsive"},"10011":{"timestamp":1712154692.0069458,"reason":"broker was unresponsive"},"10012":{"timestamp":1712154692.0076416,"reason":"broker was unresponsive"},"10013":{"timestamp":1712154692.008126,"reason":"broker was unresponsive"},"10014":{"timestamp":1712154692.0088105,"reason":"broker was unresponsive"},"10015":{"timestamp":1712154692.0138414,"reason":"broker was unresponsive"},"10016":{"timestamp":1712154692.0142868,"reason":"broker was unresponsive"},"10017":{"timestamp":1712154692.0147307,"reason":"broker was unresponsive"},"10018":{"timestamp":1712154692.0152342,"reason":"broker was unresponsive"},"10019":{"timestamp":1712154692.0156934,"reason":"broker was unresponsive"},"10020":{"timestamp":1712154692.0160887,"reason":"broker was unresponsive"},"10021":{"timestamp":1712154692.0171306,"reason":"broker was unresponsive"},"10022":{"timestamp":1712154692.0175445,"reason":"broker was unresponsive"},"10023":{"timestamp":1712154692.0179458,"reason":"broker was unresponsive"},"10024":{"timestamp":1712154692.0183384,"reason":"broker was unresponsive"},"10025":{"timestamp":1712154692.0187528,"reason":"broker was unresponsive"},"10026":{"timestamp":1712154692.0195444,"reason":"broker was unresponsive"},"10027":{"timestamp":1712154692.020762,"reason":"broker was unresponsive"},"10028":{"timestamp":1712154692.0211778,"reason":"broker was unresponsive"},"10029":{"timestamp":1712154692.0217192,"reason":"broker was unresponsive"},"10030":{"timestamp":1712154692.0221164,"reason":"broker was unresponsive"},"10031":{"timestamp":1712154692.0256703,"reason":"broker was unresponsive"},"10032":{"timestamp":1712154692.0261612,"reason":"broker was unresponsive"},"10033":{"timestamp":1712154692.0266223,"reason":"broker was unresponsive"},"10034":{"timestamp":1712154692.0280805,"reason":"broker was unresponsive"},"10035":{"timestamp":1712154692.0284531,"reason":"broker was unresponsive"},"10036":{"timestamp":1712154692.0288522,"reason":"broker was unresponsive"},"10037":{"timestamp":1712154692.0301805,"reason":"broker was unresponsive"},"10038":{"timestamp":1712154692.030616,"reason":"broker was unresponsive"},"10039":{"timestamp":1712154692.0310261,"reason":"broker was unresponsive"},"10040":{"timestamp":1712154692.0314457,"reason":"broker was unresponsive"},"10041":{"timestamp":1712154692.0318673,"reason":"broker was unresponsive"},"10042":{"timestamp":1712154692.0367901,"reason":"broker was unresponsive"},"10043":{"timestamp":1712154692.0372682,"reason":"broker was unresponsive"},"10044":{"timestamp":1712154692.03771,"reason":"broker was unresponsive"},"10045":{"timestamp":1712154692.0381563,"reason":"broker was unresponsive"},"10046":{"timestamp":1712154692.038584,"reason":"broker was unresponsive"},"10047":{"timestamp":1712154692.0390396,"reason":"broker was unresponsive"},"10048":{"timestamp":1712154692.0394773,"reason":"broker was unresponsive"},"10049":{"timestamp":1712154692.0398867,"reason":"broker was unresponsive"},"10050":{"timestamp":1712154692.040309,"reason":"broker was unresponsive"},"10051":{"timestamp":1712154692.0407417,"reason":"broker was unresponsive"},"10052":{"timestamp":1712154692.0437491,"reason":"broker was unresponsive"},"10053":{"timestamp":1712154692.0466192,"reason":"broker was unresponsive"},"10054":{"timestamp":1712154692.0470922,"reason":"broker was unresponsive"},"10055":{"timestamp":1712154692.0477114,"reason":"broker was unresponsive"},"10056":{"timestamp":1712154692.0483823,"reason":"broker was unresponsive"},"10057":{"timestamp":1712154692.0498736,"reason":"broker was unresponsive"},"10058":{"timestamp":1712154692.0503008,"reason":"broker was unresponsive"},"10059":{"timestamp":1712154692.0507491,"reason":"broker was unresponsive"},"10060":{"timestamp":1712154692.0511656,"reason":"broker was unresponsive"},"10061":{"timestamp":1712154692.0515821,"reason":"broker was unresponsive"},"10062":{"timestamp":1712154692.053834,"reason":"broker was unresponsive"},"10063":{"timestamp":1712154692.0550618,"reason":"broker was unresponsive"},"10064":{"timestamp":1712154692.0554938,"reason":"broker was unresponsive"},"10065":{"timestamp":1712154692.0561919,"reason":"broker was unresponsive"},"10066":{"timestamp":1712154692.0566978,"reason":"broker was unresponsive"},"10067":{"timestamp":1712154692.0572069,"reason":"broker was unresponsive"},"10068":{"timestamp":1712154692.0576293,"reason":"broker was unresponsive"},"10069":{"timestamp":1712154692.0579636,"reason":"broker was unresponsive"},"10070":{"timestamp":1712154692.058814,"reason":"broker was unresponsive"},"10071":{"timestamp":1712154692.0591509,"reason":"broker was unresponsive"},"10072":{"timestamp":1712154692.0596557,"reason":"broker was unresponsive"},"10073":{"timestamp":1712154692.0601408,"reason":"broker was unresponsive"},"10074":{"timestamp":1712154692.0605791,"reason":"broker was unresponsive"},"10075":{"timestamp":1712154692.0615735,"reason":"broker was unresponsive"},"10076":{"timestamp":1712154692.062603,"reason":"broker was unresponsive"},"10077":{"timestamp":1712154692.0630455,"reason":"broker was unresponsive"},"10078":{"timestamp":1712154692.0634887,"reason":"broker was unresponsive"},"10079":{"timestamp":1712154692.0653596,"reason":"broker was unresponsive"},"10080":{"timestamp":1712154692.0658143,"reason":"broker was unresponsive"},"10081":{"timestamp":1712154692.0662339,"reason":"broker was unresponsive"},"10082":{"timestamp":1712154692.0666721,"reason":"broker was unresponsive"},"10083":{"timestamp":1712154692.0671015,"reason":"broker was unresponsive"},"10084":{"timestamp":1712154692.0697711,"reason":"broker was unresponsive"},"10085":{"timestamp":1712154692.0754991,"reason":"broker was unresponsive"},"10086":{"timestamp":1712154692.0759709,"reason":"broker was unresponsive"},"10087":{"timestamp":1712154692.0763938,"reason":"broker was unresponsive"},"10088":{"timestamp":1712154692.076843,"reason":"broker was unresponsive"},"10089":{"timestamp":1712154692.0786448,"reason":"broker was unresponsive"},"10090":{"timestamp":1712154692.0790851,"reason":"broker was unresponsive"},"10091":{"timestamp":1712154692.0800772,"reason":"broker was unresponsive"},"10092":{"timestamp":1712154692.080476,"reason":"broker was unresponsive"},"10093":{"timestamp":1712154692.081007,"reason":"broker was unresponsive"},"10094":{"timestamp":1712154692.0816131,"reason":"broker was unresponsive"},"10095":{"timestamp":1712154692.0847113,"reason":"broker was unresponsive"},"10096":{"timestamp":1712154692.0851505,"reason":"broker was unresponsive"},"10097":{"timestamp":1712154692.0855951,"reason":"broker was unresponsive"},"10098":{"timestamp":1712154692.0860367,"reason":"broker was unresponsive"},"10099":{"timestamp":1712154692.0875695,"reason":"broker was unresponsive"},"10100":{"timestamp":1712154692.0891881,"reason":"broker was unresponsive"},"10103":{"timestamp":1712154692.0900357,"reason":"broker was unresponsive"},"10104":{"timestamp":1712154692.0904779,"reason":"broker was unresponsive"},"10105":{"timestamp":1712154692.0909159,"reason":"broker was unresponsive"},"10106":{"timestamp":1712154692.0914791,"reason":"broker was unresponsive"},"10107":{"timestamp":1712154692.0919535,"reason":"broker was unresponsive"},"10108":{"timestamp":1712154692.09252,"reason":"broker was unresponsive"},"10109":{"timestamp":1712154692.0930083,"reason":"broker was unresponsive"},"10110":{"timestamp":1712154692.0935347,"reason":"broker was unresponsive"},"10111":{"timestamp":1712154692.0941203,"reason":"broker was unresponsive"},"10112":{"timestamp":1712154692.0945811,"reason":"broker was unresponsive"},"10113":{"timestamp":1712154692.0950613,"reason":"broker was unresponsive"},"10114":{"timestamp":1712154692.0955501,"reason":"broker was unresponsive"},"10115":{"timestamp":1712154692.0960312,"reason":"broker was unresponsive"},"10116":{"timestamp":1712154692.0965364,"reason":"broker was unresponsive"},"10117":{"timestamp":1712154692.0970156,"reason":"broker was unresponsive"},"10118":{"timestamp":1712154692.0974627,"reason":"broker was unresponsive"},"10119":{"timestamp":1712154692.097945,"reason":"broker was unresponsive"},"10120-10129":{"timestamp":1712079171.5353351,"reason":""},"10130":{"timestamp":1712154692.0984499,"reason":"broker was unresponsive"},"10131":{"timestamp":1712154692.0988858,"reason":"broker was unresponsive"},"10132":{"timestamp":1712154692.0993621,"reason":"broker was unresponsive"},"10133":{"timestamp":1712154692.0998168,"reason":"broker was unresponsive"},"10134":{"timestamp":1712154692.1003056,"reason":"broker was unresponsive"},"10135":{"timestamp":1712154692.1007507,"reason":"broker was unresponsive"},"10136":{"timestamp":1712154692.1011977,"reason":"broker was unresponsive"},"10138":{"timestamp":1712154692.1016693,"reason":"broker was unresponsive"},"10139":{"timestamp":1712154692.1021886,"reason":"broker was unresponsive"},"10141":{"timestamp":1712154692.1027501,"reason":"broker was unresponsive"},"10142":{"timestamp":1712154692.1032383,"reason":"broker was unresponsive"},"10143":{"timestamp":1712154692.1036949,"reason":"broker was unresponsive"},"10144":{"timestamp":1712154692.1041818,"reason":"broker was unresponsive"},"10145":{"timestamp":1712154692.1046801,"reason":"broker was unresponsive"},"10146":{"timestamp":1712154692.1051702,"reason":"broker was unresponsive"},"10147":{"timestamp":1712154692.1056852,"reason":"broker was unresponsive"},"10148":{"timestamp":1712154692.1061764,"reason":"broker was unresponsive"},"10149":{"timestamp":1712154692.1066544,"reason":"broker was unresponsive"},"10150":{"timestamp":1712154692.1071053,"reason":"broker was unresponsive"},"10151":{"timestamp":1711734764.9003315,"reason":"nodediag failed clocksource pci"},"10152":{"timestamp":1711744485.1922078,"reason":"broker was unresponsive"},"10154":{"timestamp":1711600478.1543531,"reason":"broker was unresponsive"},"10155":{"timestamp":1712154692.1075859,"reason":"broker was unresponsive"},"10156":{"timestamp":1711600478.2542264,"reason":"broker was unresponsive"},"10157-10158":{"timestamp":1712087312.0516427,"reason":"epilog failed for jobid fn4AXyJ6haF"},"10159":{"timestamp":1711744485.19349,"reason":"broker was unresponsive"},"10160":{"timestamp":1711744485.1935451,"reason":"broker was unresponsive"},"10161":{"timestamp":1712154692.1080379,"reason":"broker was unresponsive"},"10162":{"timestamp":1711742687.1360803,"reason":"nodediag failed cxi"},"10163":{"timestamp":1712154692.1085031,"reason":"broker was unresponsive"},"10164":{"timestamp":1712154692.109813,"reason":"broker was unresponsive"},"10165":{"timestamp":1712154692.1102936,"reason":"broker was unresponsive"},"10166":{"timestamp":1712154692.1107855,"reason":"broker was unresponsive"},"10167":{"timestamp":1712154692.1112411,"reason":"broker was unresponsive"},"10168":{"timestamp":1712154692.1116893,"reason":"broker was unresponsive"},"10169":{"timestamp":1712154692.1121376,"reason":"broker was unresponsive"},"10170":{"timestamp":1712154692.1126459,"reason":"broker was unresponsive"},"10171":{"timestamp":1712154692.1132081,"reason":"broker was unresponsive"},"10172":{"timestamp":1712154692.1137207,"reason":"broker was unresponsive"},"10173":{"timestamp":1712154692.1142459,"reason":"broker was unresponsive"},"10174":{"timestamp":1712154692.1147118,"reason":"broker was unresponsive"},"10175":{"timestamp":1712154692.1152544,"reason":"broker was unresponsive"},"10176":{"timestamp":1712154692.1157835,"reason":"broker was unresponsive"},"10177":{"timestamp":1712154692.1163208,"reason":"broker was unresponsive"},"10178":{"timestamp":1712154692.1168191,"reason":"broker was unresponsive"},"10179":{"timestamp":1712154692.1172884,"reason":"broker was unresponsive"},"10180":{"timestamp":1712154692.1178503,"reason":"broker was unresponsive"},"10181":{"timestamp":1712154692.1183202,"reason":"broker was unresponsive"},"10182":{"timestamp":1712154692.1187963,"reason":"broker was unresponsive"},"10183":{"timestamp":1711600556.2563355,"reason":"broker was unresponsive"},"10184":{"timestamp":1712154692.119262,"reason":"broker was unresponsive"},"10185":{"timestamp":1712154692.1201177,"reason":"broker was unresponsive"},"10186":{"timestamp":1712154692.1205254,"reason":"broker was unresponsive"},"10187":{"timestamp":1712154692.1209128,"reason":"broker was unresponsive"},"10188":{"timestamp":1712154692.1213179,"reason":"broker was unresponsive"},"10189":{"timestamp":1712154692.1237216,"reason":"broker was unresponsive"},"10190":{"timestamp":1712154692.1242039,"reason":"broker was unresponsive"},"10191":{"timestamp":1712154692.1246049,"reason":"broker was unresponsive"},"10192":{"timestamp":1712154692.1263187,"reason":"broker was unresponsive"},"10193":{"timestamp":1712154692.126775,"reason":"broker was unresponsive"},"10194":{"timestamp":1712154692.1283445,"reason":"broker was unresponsive"},"10195":{"timestamp":1712154692.1288474,"reason":"broker was unresponsive"},"10196":{"timestamp":1712154692.1293375,"reason":"broker was unresponsive"},"10197":{"timestamp":1712154692.1311934,"reason":"broker was unresponsive"},"10198":{"timestamp":1712154692.1316781,"reason":"broker was unresponsive"},"10199":{"timestamp":1712154692.1321611,"reason":"broker was unresponsive"},"10200":{"timestamp":1712154692.1326504,"reason":"broker was unresponsive"},"10201":{"timestamp":1712154692.1383219,"reason":"broker was unresponsive"},"10203":{"timestamp":1712154692.1402805,"reason":"broker was unresponsive"},"10204":{"timestamp":1712154692.1407681,"reason":"broker was unresponsive"},"10207":{"timestamp":1712154692.1412749,"reason":"broker was unresponsive"},"10208":{"timestamp":1712154692.1417551,"reason":"broker was unresponsive"},"10209":{"timestamp":1712154692.1422639,"reason":"broker was unresponsive"},"10210":{"timestamp":1712154692.1427567,"reason":"broker was unresponsive"},"10211":{"timestamp":1712154692.1432779,"reason":"broker was unresponsive"},"10212":{"timestamp":1712154692.143784,"reason":"broker was unresponsive"},"10213":{"timestamp":1712154692.1442785,"reason":"broker was unresponsive"},"10214":{"timestamp":1712154692.1447704,"reason":"broker was unresponsive"},"10215":{"timestamp":1712154692.1452756,"reason":"broker was unresponsive"},"10216":{"timestamp":1712154692.1465502,"reason":"broker was unresponsive"},"10217":{"timestamp":1712154692.1472189,"reason":"broker was unresponsive"},"10218":{"timestamp":1712154692.1477337,"reason":"broker was unresponsive"},"10219":{"timestamp":1712154692.1484256,"reason":"broker was unresponsive"},"10220":{"timestamp":1712154692.1492729,"reason":"broker was unresponsive"},"10221":{"timestamp":1712154692.150162,"reason":"broker was unresponsive"},"10222":{"timestamp":1712154692.1508093,"reason":"broker was unresponsive"},"10223":{"timestamp":1712154692.151557,"reason":"broker was unresponsive"},"10224":{"timestamp":1712154692.1521697,"reason":"broker was unresponsive"},"10225":{"timestamp":1712154692.1529777,"reason":"broker was unresponsive"},"10226":{"timestamp":1712154692.1536686,"reason":"broker was unresponsive"},"10227":{"timestamp":1712154692.1539881,"reason":"broker was unresponsive"},"10228":{"timestamp":1712154692.1544161,"reason":"broker was unresponsive"},"10229":{"timestamp":1712154692.1549852,"reason":"broker was unresponsive"},"10230":{"timestamp":1712154692.155328,"reason":"broker was unresponsive"},"10231":{"timestamp":1712154692.1560159,"reason":"broker was unresponsive"},"10232":{"timestamp":1712154692.1565709,"reason":"broker was unresponsive"},"10233":{"timestamp":1712154692.1570976,"reason":"broker was unresponsive"},"10234":{"timestamp":1712154692.1575892,"reason":"broker was unresponsive"},"10235":{"timestamp":1712154692.1581612,"reason":"broker was unresponsive"},"10236":{"timestamp":1712154692.1586955,"reason":"broker was unresponsive"},"10237":{"timestamp":1712154692.1592197,"reason":"broker was unresponsive"},"10238":{"timestamp":1712154692.1597676,"reason":"broker was unresponsive"},"10239":{"timestamp":1712154692.1603186,"reason":"broker was unresponsive"},"10240":{"timestamp":1712154692.1608613,"reason":"broker was unresponsive"},"10241":{"timestamp":1712154692.1613708,"reason":"broker was unresponsive"},"10242":{"timestamp":1712154692.1619284,"reason":"broker was unresponsive"},"10243":{"timestamp":1712154692.162426,"reason":"broker was unresponsive"},"10244":{"timestamp":1712154692.1629293,"reason":"broker was unresponsive"},"10245":{"timestamp":1712154692.1634564,"reason":"broker was unresponsive"},"10246":{"timestamp":1712154692.1639669,"reason":"broker was unresponsive"},"10247":{"timestamp":1712154692.1645501,"reason":"broker was unresponsive"},"10248":{"timestamp":1712154692.1650479,"reason":"broker was unresponsive"},"10249":{"timestamp":1712154692.1655612,"reason":"broker was unresponsive"},"10250":{"timestamp":1712154692.1660788,"reason":"broker was unresponsive"},"10251":{"timestamp":1712154692.1665885,"reason":"broker was unresponsive"},"10252":{"timestamp":1712154692.1671071,"reason":"broker was unresponsive"},"10253":{"timestamp":1712154692.1676214,"reason":"broker was unresponsive"},"10254":{"timestamp":1712154692.1680996,"reason":"broker was unresponsive"},"10255":{"timestamp":1712154692.1684422,"reason":"broker was unresponsive"},"10256":{"timestamp":1712154692.168859,"reason":"broker was unresponsive"},"10257":{"timestamp":1712154692.1692717,"reason":"broker was unresponsive"},"10258":{"timestamp":1712154692.1697695,"reason":"broker was unresponsive"},"10259":{"timestamp":1712154692.1702056,"reason":"broker was unresponsive"},"10260":{"timestamp":1712154692.1705289,"reason":"broker was unresponsive"},"10261":{"timestamp":1712154692.1708262,"reason":"broker was unresponsive"},"10262":{"timestamp":1712154692.1711252,"reason":"broker was unresponsive"},"10263":{"timestamp":1712154692.1715286,"reason":"broker was unresponsive"},"10264":{"timestamp":1712154692.1718686,"reason":"broker was unresponsive"},"10265":{"timestamp":1712154692.1721611,"reason":"broker was unresponsive"},"10266":{"timestamp":1712154692.1725268,"reason":"broker was unresponsive"},"10267":{"timestamp":1712154692.1729248,"reason":"broker was unresponsive"},"10268":{"timestamp":1712154692.1732593,"reason":"broker was unresponsive"},"10269":{"timestamp":1712154692.1735897,"reason":"broker was unresponsive"},"10270":{"timestamp":1712154692.1739256,"reason":"broker was unresponsive"},"10271":{"timestamp":1712154692.1743321,"reason":"broker was unresponsive"},"10272":{"timestamp":1712154692.1747069,"reason":"broker was unresponsive"},"10273":{"timestamp":1712154692.1750937,"reason":"broker was unresponsive"},"10274":{"timestamp":1712154692.1754203,"reason":"broker was unresponsive"},"10275":{"timestamp":1712154692.1757326,"reason":"broker was unresponsive"},"10276":{"timestamp":1712154692.1760244,"reason":"broker was unresponsive"},"10277":{"timestamp":1712154692.1764064,"reason":"broker was unresponsive"},"10278":{"timestamp":1712154692.1767967,"reason":"broker was unresponsive"},"10279":{"timestamp":1712154692.1771843,"reason":"broker was unresponsive"},"10280":{"timestamp":1712154692.1775587,"reason":"broker was unresponsive"},"10281":{"timestamp":1712154692.17788,"reason":"broker was unresponsive"},"10282":{"timestamp":1712154692.1782088,"reason":"broker was unresponsive"},"10283":{"timestamp":1712154692.1785915,"reason":"broker was unresponsive"},"10284":{"timestamp":1712154692.178915,"reason":"broker was unresponsive"},"10285":{"timestamp":1712154692.1792443,"reason":"broker was unresponsive"},"10286":{"timestamp":1712154692.1795595,"reason":"broker was unresponsive"},"10287":{"timestamp":1712154692.1798904,"reason":"broker was unresponsive"},"10288":{"timestamp":1712154692.1801946,"reason":"broker was unresponsive"},"10289":{"timestamp":1712154692.1805344,"reason":"broker was unresponsive"},"10290":{"timestamp":1712154692.1808658,"reason":"broker was unresponsive"},"10291":{"timestamp":1712154692.1812718,"reason":"broker was unresponsive"},"10292":{"timestamp":1712154692.1816318,"reason":"broker was unresponsive"},"10293":{"timestamp":1712154692.1819527,"reason":"broker was unresponsive"},"10294":{"timestamp":1712154692.1824872,"reason":"broker was unresponsive"},"10295":{"timestamp":1712154692.1828856,"reason":"broker was unresponsive"},"10296":{"timestamp":1712154692.1831956,"reason":"broker was unresponsive"},"10297":{"timestamp":1712154692.183738,"reason":"broker was unresponsive"},"10298":{"timestamp":1712154692.1842871,"reason":"broker was unresponsive"},"10299":{"timestamp":1712154692.1850612,"reason":"broker was unresponsive"},"10300":{"timestamp":1712154692.1856375,"reason":"broker was unresponsive"},"10301":{"timestamp":1712096585.8629279,"reason":"Bad CXI Link --JRG"},"10302":{"timestamp":1712096599.7608726,"reason":"bad partner --JRG"},"10303":{"timestamp":1712154692.1863458,"reason":"broker was unresponsive"},"10304":{"timestamp":1712154692.18731,"reason":"broker was unresponsive"},"10305":{"timestamp":1712154692.1884067,"reason":"broker was unresponsive"},"10306":{"timestamp":1712154692.1891046,"reason":"broker was unresponsive"},"10307":{"timestamp":1712154692.189537,"reason":"broker was unresponsive"},"10308":{"timestamp":1712154692.1899049,"reason":"broker was unresponsive"},"10309":{"timestamp":1712154692.1902196,"reason":"broker was unresponsive"},"10311":{"timestamp":1712154692.1906848,"reason":"broker was unresponsive"},"10312":{"timestamp":1712154692.1912055,"reason":"broker was unresponsive"},"10313":{"timestamp":1712154692.1917057,"reason":"broker was unresponsive"},"10314":{"timestamp":1712154692.1922157,"reason":"broker was unresponsive"},"10315":{"timestamp":1712154692.192723,"reason":"broker was unresponsive"},"10316":{"timestamp":1712154692.1932271,"reason":"broker was unresponsive"},"10317":{"timestamp":1712154692.1937413,"reason":"broker was unresponsive"},"10318":{"timestamp":1712154692.1942444,"reason":"broker was unresponsive"},"10319":{"timestamp":1712154692.1947479,"reason":"broker was unresponsive"},"10320":{"timestamp":1712154692.195245,"reason":"broker was unresponsive"},"10321":{"timestamp":1712154692.1957593,"reason":"broker was unresponsive"},"10322":{"timestamp":1712154692.1962512,"reason":"broker was unresponsive"},"10323":{"timestamp":1712154692.196739,"reason":"broker was unresponsive"},"10324":{"timestamp":1712154692.1972318,"reason":"broker was unresponsive"},"10325":{"timestamp":1712154692.1977785,"reason":"broker was unresponsive"},"10326":{"timestamp":1712154692.1982744,"reason":"broker was unresponsive"},"10327":{"timestamp":1712154692.198787,"reason":"broker was unresponsive"},"10328":{"timestamp":1712154692.1992407,"reason":"broker was unresponsive"},"10329":{"timestamp":1712154692.1996758,"reason":"broker was unresponsive"},"10330":{"timestamp":1712154692.2002022,"reason":"broker was unresponsive"},"10331":{"timestamp":1712154692.2005742,"reason":"broker was unresponsive"},"10332":{"timestamp":1712154692.2010007,"reason":"broker was unresponsive"},"10333":{"timestamp":1712154692.2015157,"reason":"broker was unresponsive"},"10334":{"timestamp":1712154692.201833,"reason":"broker was unresponsive"},"10335":{"timestamp":1712154692.2021644,"reason":"broker was unresponsive"},"10336":{"timestamp":1712154692.2026806,"reason":"broker was unresponsive"},"10337":{"timestamp":1712154692.2032039,"reason":"broker was unresponsive"},"10338":{"timestamp":1712154692.2037508,"reason":"broker was unresponsive"},"10339":{"timestamp":1712154692.2043078,"reason":"broker was unresponsive"},"10340":{"timestamp":1712154692.2046561,"reason":"broker was unresponsive"},"10341":{"timestamp":1712154692.2051589,"reason":"broker was unresponsive"},"10342":{"timestamp":1712154692.2057502,"reason":"broker was unresponsive"},"10343":{"timestamp":1712154692.2061348,"reason":"broker was unresponsive"},"10344":{"timestamp":1712154692.2064543,"reason":"broker was unresponsive"},"10345":{"timestamp":1712154692.2067556,"reason":"broker was unresponsive"},"10346":{"timestamp":1712154692.2071021,"reason":"broker was unresponsive"},"10347":{"timestamp":1712154692.2074149,"reason":"broker was unresponsive"},"10348":{"timestamp":1712154692.2077267,"reason":"broker was unresponsive"},"10349":{"timestamp":1712154692.2082362,"reason":"broker was unresponsive"},"10350":{"timestamp":1712154692.2087827,"reason":"broker was unresponsive"},"10351":{"timestamp":1712154692.2093384,"reason":"broker was unresponsive"},"10352":{"timestamp":1712154692.209904,"reason":"broker was unresponsive"},"10353":{"timestamp":1712154692.2104762,"reason":"broker was unresponsive"},"10354":{"timestamp":1712154692.2109931,"reason":"broker was unresponsive"},"10355":{"timestamp":1712154692.2115319,"reason":"broker was unresponsive"},"10356":{"timestamp":1712154692.2119184,"reason":"broker was unresponsive"},"10357-10359":{"timestamp":1712170314.2268863,"reason":""},"10373-10468":{"timestamp":1712094140.4139462,"reason":"--reason HP Diags"},"10469-10474":{"timestamp":1712096633.8221953,"reason":"--reason HP Diags"},"10475":{"timestamp":1712096681.148495,"reason":"--reason repair cxi1 issue HP"},"10476-10484":{"timestamp":1712096658.8781381,"reason":"--reason HP Diags"},"10485":{"timestamp":1712154692.2848744,"reason":"broker was unresponsive"},"10486":{"timestamp":1712154692.2855797,"reason":"broker was unresponsive"},"10487":{"timestamp":1712154692.2862186,"reason":"broker was unresponsive"},"10488":{"timestamp":1712154692.2869651,"reason":"broker was unresponsive"},"10489":{"timestamp":1712154692.2876391,"reason":"broker was unresponsive"},"10490":{"timestamp":1712154692.2882817,"reason":"broker was unresponsive"},"10491":{"timestamp":1712154692.2889614,"reason":"broker was unresponsive"},"10492":{"timestamp":1712154692.2896385,"reason":"broker was unresponsive"},"10493":{"timestamp":1712154692.290364,"reason":"broker was unresponsive"},"10494":{"timestamp":1712154692.2910125,"reason":"broker was unresponsive"},"10495":{"timestamp":1712154692.2919216,"reason":"broker was unresponsive"},"10496":{"timestamp":1712154692.2922721,"reason":"broker was unresponsive"},"10497":{"timestamp":1712154692.2928631,"reason":"broker was unresponsive"},"10498":{"timestamp":1712154692.2933228,"reason":"broker was unresponsive"},"10499":{"timestamp":1712154692.2936754,"reason":"broker was unresponsive"},"10500":{"timestamp":1712154692.2940159,"reason":"broker was unresponsive"},"10501":{"timestamp":1712154692.2944803,"reason":"broker was unresponsive"},"10502":{"timestamp":1712154692.2952039,"reason":"broker was unresponsive"},"10503":{"timestamp":1712154692.2958465,"reason":"broker was unresponsive"},"10504":{"timestamp":1712154692.2964497,"reason":"broker was unresponsive"},"10505":{"timestamp":1712154692.2970898,"reason":"broker was unresponsive"},"10506":{"timestamp":1712154692.2979319,"reason":"broker was unresponsive"},"10507":{"timestamp":1712154692.2985799,"reason":"broker was unresponsive"},"10508":{"timestamp":1712154692.2993052,"reason":"broker was unresponsive"},"10511":{"timestamp":1712154692.3001246,"reason":"broker was unresponsive"},"10512":{"timestamp":1712154692.3008513,"reason":"broker was unresponsive"},"10513":{"timestamp":1712154692.30162,"reason":"broker was unresponsive"},"10514":{"timestamp":1712154692.3022187,"reason":"broker was unresponsive"},"10515":{"timestamp":1712154692.3026826,"reason":"broker was unresponsive"},"10516":{"timestamp":1712154692.303062,"reason":"broker was unresponsive"},"10517":{"timestamp":1712154692.3034911,"reason":"broker was unresponsive"},"10509,10518":{"timestamp":1712086791.4119098,"reason":"CXI Issues --JRG"},"10510,10519":{"timestamp":1712086831.8068576,"reason":"Bad partner node --JRG"},"10520":{"timestamp":1712154692.3039613,"reason":"broker was unresponsive"},"10523":{"timestamp":1712154692.3043089,"reason":"broker was unresponsive"},"10524":{"timestamp":1712154692.3047626,"reason":"broker was unresponsive"},"10525":{"timestamp":1712154692.3053615,"reason":"broker was unresponsive"},"10526":{"timestamp":1712154692.3059833,"reason":"broker was unresponsive"},"10527":{"timestamp":1712154692.3067749,"reason":"broker was unresponsive"},"10528":{"timestamp":1712154692.3075652,"reason":"broker was unresponsive"},"10529":{"timestamp":1712154692.3081651,"reason":"broker was unresponsive"},"10530":{"timestamp":1712154692.3089309,"reason":"broker was unresponsive"},"10531":{"timestamp":1712154692.3095362,"reason":"broker was unresponsive"},"10532":{"timestamp":1712154692.3101959,"reason":"broker was unresponsive"},"10533":{"timestamp":1712154692.3112245,"reason":"broker was unresponsive"},"10534":{"timestamp":1712154692.3118629,"reason":"broker was unresponsive"},"10535":{"timestamp":1712154692.3125131,"reason":"broker was unresponsive"},"10536":{"timestamp":1712154692.3131442,"reason":"broker was unresponsive"},"10537":{"timestamp":1712154692.3146098,"reason":"broker was unresponsive"},"10538":{"timestamp":1712154692.3152812,"reason":"broker was unresponsive"},"10539":{"timestamp":1712154692.3159089,"reason":"broker was unresponsive"},"10540":{"timestamp":1712154692.3169928,"reason":"broker was unresponsive"},"10541":{"timestamp":1712154692.3177123,"reason":"broker was unresponsive"},"10542":{"timestamp":1712154692.3183446,"reason":"broker was unresponsive"},"10543":{"timestamp":1712154692.3189867,"reason":"broker was unresponsive"},"10544":{"timestamp":1712154692.3198016,"reason":"broker was unresponsive"},"10545":{"timestamp":1712154692.3205104,"reason":"broker was unresponsive"},"10546":{"timestamp":1712154692.3212693,"reason":"broker was unresponsive"},"10547":{"timestamp":1712154692.3223677,"reason":"broker was unresponsive"},"10548":{"timestamp":1712154692.3239694,"reason":"broker was unresponsive"},"10549":{"timestamp":1712154692.3252914,"reason":"broker was unresponsive"},"10550":{"timestamp":1712154692.326052,"reason":"broker was unresponsive"},"10551":{"timestamp":1712154692.3268077,"reason":"broker was unresponsive"},"10552":{"timestamp":1712154692.327435,"reason":"broker was unresponsive"},"10553":{"timestamp":1712154692.3287976,"reason":"broker was unresponsive"},"10554":{"timestamp":1712154692.3296967,"reason":"broker was unresponsive"},"10555":{"timestamp":1712154692.3303504,"reason":"broker was unresponsive"},"10556":{"timestamp":1712154692.3311803,"reason":"broker was unresponsive"},"10557":{"timestamp":1712154692.3325636,"reason":"broker was unresponsive"},"10558":{"timestamp":1712154692.3333406,"reason":"broker was unresponsive"},"10559":{"timestamp":1712154692.3341837,"reason":"broker was unresponsive"},"10560":{"timestamp":1712154692.3350315,"reason":"broker was unresponsive"},"10561":{"timestamp":1712154692.3357589,"reason":"broker was unresponsive"},"10562":{"timestamp":1712154692.3364799,"reason":"broker was unresponsive"},"10563":{"timestamp":1712154692.3372164,"reason":"broker was unresponsive"},"10564":{"timestamp":1712154692.3378868,"reason":"broker was unresponsive"},"10565":{"timestamp":1712154692.3385117,"reason":"broker was unresponsive"},"10566":{"timestamp":1712154692.3392389,"reason":"broker was unresponsive"},"10567":{"timestamp":1712154692.3402104,"reason":"broker was unresponsive"},"10568":{"timestamp":1712154692.3409209,"reason":"broker was unresponsive"},"10569":{"timestamp":1712154692.3415594,"reason":"broker was unresponsive"},"10570":{"timestamp":1712154692.3426354,"reason":"broker was unresponsive"},"10571":{"timestamp":1712154692.3437395,"reason":"broker was unresponsive"},"10572":{"timestamp":1712154692.3448856,"reason":"broker was unresponsive"},"10573":{"timestamp":1712154692.3457575,"reason":"broker was unresponsive"},"10574":{"timestamp":1712154692.3464994,"reason":"broker was unresponsive"},"10575":{"timestamp":1712154692.3473284,"reason":"broker was unresponsive"},"10576":{"timestamp":1712154692.3482213,"reason":"broker was unresponsive"},"10577":{"timestamp":1712154692.3491318,"reason":"broker was unresponsive"},"10578":{"timestamp":1712154692.3498058,"reason":"broker was unresponsive"},"10579":{"timestamp":1712154692.3505373,"reason":"broker was unresponsive"},"10580":{"timestamp":1712154692.3512614,"reason":"broker was unresponsive"},"10581":{"timestamp":1712154692.3521948,"reason":"broker was unresponsive"},"10582":{"timestamp":1712154692.3531559,"reason":"broker was unresponsive"},"10583":{"timestamp":1712154692.3539686,"reason":"broker was unresponsive"},"10584":{"timestamp":1712154692.3547308,"reason":"broker was unresponsive"},"10585":{"timestamp":1712154692.3553751,"reason":"broker was unresponsive"},"10586":{"timestamp":1712154692.3562109,"reason":"broker was unresponsive"},"10587":{"timestamp":1712154692.3569131,"reason":"broker was unresponsive"},"10588":{"timestamp":1712154692.3576562,"reason":"broker was unresponsive"},"10589":{"timestamp":1712154692.3583889,"reason":"broker was unresponsive"},"10590":{"timestamp":1712154692.3597434,"reason":"broker was unresponsive"},"10591":{"timestamp":1712154692.3604431,"reason":"broker was unresponsive"},"10592":{"timestamp":1712154692.3611217,"reason":"broker was unresponsive"},"10593":{"timestamp":1712154692.3618419,"reason":"broker was unresponsive"},"10594":{"timestamp":1712154692.3625395,"reason":"broker was unresponsive"},"10595":{"timestamp":1712154692.3633015,"reason":"broker was unresponsive"},"10596":{"timestamp":1712154692.3641078,"reason":"broker was unresponsive"},"10597":{"timestamp":1712154692.3648114,"reason":"broker was unresponsive"},"10598":{"timestamp":1712154692.3654938,"reason":"broker was unresponsive"},"10599":{"timestamp":1712154692.3661163,"reason":"broker was unresponsive"},"10600":{"timestamp":1712154692.3667905,"reason":"broker was unresponsive"},"10601":{"timestamp":1712154692.3674822,"reason":"broker was unresponsive"},"10602":{"timestamp":1712154692.3681419,"reason":"broker was unresponsive"},"10603":{"timestamp":1712154692.3688738,"reason":"broker was unresponsive"},"10604":{"timestamp":1712154692.3695886,"reason":"broker was unresponsive"},"10605":{"timestamp":1712154692.3704796,"reason":"broker was unresponsive"},"10606":{"timestamp":1712154692.3711882,"reason":"broker was unresponsive"},"10607":{"timestamp":1712154692.3718636,"reason":"broker was unresponsive"},"10608":{"timestamp":1712154692.3725467,"reason":"broker was unresponsive"},"10609":{"timestamp":1712154692.3732421,"reason":"broker was unresponsive"},"10610":{"timestamp":1712154692.3739498,"reason":"broker was unresponsive"},"10611":{"timestamp":1712154692.3748868,"reason":"broker was unresponsive"},"10612":{"timestamp":1712154692.3756571,"reason":"broker was unresponsive"},"10742":{"timestamp":1712161524.5548441,"reason":"nodediag failed cxi"},"10745":{"timestamp":1712161525.023258,"reason":"nodediag failed cxi"},"10746":{"timestamp":1712161827.7108033,"reason":"nodediag failed cxi"},"10754":{"timestamp":1712161552.3722243,"reason":"nodediag failed cxi"},"10756":{"timestamp":1712161830.8916755,"reason":"nodediag failed cxi"},"10760":{"timestamp":1712161829.9059019,"reason":"nodediag failed cxi"},"10822":{"timestamp":1712174369.619107,"reason":"nodediag failed cxi"},"10823":{"timestamp":1712174316.6733592,"reason":"nodediag failed cxi"},"10824":{"timestamp":1712174384.0462465,"reason":"nodediag failed cxi"},"10825":{"timestamp":1712174312.4487283,"reason":"nodediag failed cxi"},"10826":{"timestamp":1712174370.8748741,"reason":"nodediag failed cxi"},"10827":{"timestamp":1712174378.7271705,"reason":"nodediag failed cxi"},"10828":{"timestamp":1712174386.6559324,"reason":"nodediag failed cxi"},"10829":{"timestamp":1712174313.2261338,"reason":"nodediag failed cxi"},"10830":{"timestamp":1712174316.1914029,"reason":"nodediag failed cxi"},"10831":{"timestamp":1712174366.1198924,"reason":"nodediag failed cxi"},"10869":{"timestamp":1712154692.4530044,"reason":"broker was unresponsive"},"10870":{"timestamp":1712154692.4547071,"reason":"broker was unresponsive"},"10871":{"timestamp":1712087815.0916734,"reason":"broker was unresponsive"},"10872":{"timestamp":1712154692.4562738,"reason":"broker was unresponsive"},"10873":{"timestamp":1712154692.4582243,"reason":"broker was unresponsive"},"10874":{"timestamp":1712154692.4596572,"reason":"broker was unresponsive"},"10875":{"timestamp":1712154692.4613819,"reason":"broker was unresponsive"},"10876":{"timestamp":1712154692.4627881,"reason":"broker was unresponsive"},"10877":{"timestamp":1712154692.4643116,"reason":"broker was unresponsive"},"10878":{"timestamp":1712154692.4654522,"reason":"broker was unresponsive"},"10879":{"timestamp":1712154692.4662533,"reason":"broker was unresponsive"},"10880":{"timestamp":1712154692.4670525,"reason":"broker was unresponsive"},"10881":{"timestamp":1712154692.4678516,"reason":"broker was unresponsive"},"10882":{"timestamp":1712154692.4686468,"reason":"broker was unresponsive"},"10883":{"timestamp":1712154692.4696405,"reason":"broker was unresponsive"},"10884":{"timestamp":1712154692.4704447,"reason":"broker was unresponsive"},"10885":{"timestamp":1712154692.4712358,"reason":"broker was unresponsive"},"10886":{"timestamp":1712103808.051157,"reason":"epilog failed for jobid fnFWHjJ8JpF"},"10887":{"timestamp":1712154692.4719651,"reason":"broker was unresponsive"},"10888":{"timestamp":1712154692.4727702,"reason":"broker was unresponsive"},"10889":{"timestamp":1712154692.4735653,"reason":"broker was unresponsive"},"10890":{"timestamp":1712154692.4742692,"reason":"broker was unresponsive"},"10891":{"timestamp":1712154692.4750276,"reason":"broker was unresponsive"},"10892":{"timestamp":1712154692.4758477,"reason":"broker was unresponsive"},"10893":{"timestamp":1712154692.4765317,"reason":"broker was unresponsive"},"10894":{"timestamp":1712154692.4772632,"reason":"broker was unresponsive"},"10895":{"timestamp":1712154692.4779906,"reason":"broker was unresponsive"},"10896":{"timestamp":1712154692.4786465,"reason":"broker was unresponsive"},"10897":{"timestamp":1712154692.4793742,"reason":"broker was unresponsive"},"10898":{"timestamp":1712154692.480077,"reason":"broker was unresponsive"},"10899":{"timestamp":1712154692.4808145,"reason":"broker was unresponsive"},"10900":{"timestamp":1712154692.4815516,"reason":"broker was unresponsive"},"10901":{"timestamp":1712154692.4822826,"reason":"broker was unresponsive"},"10902":{"timestamp":1712154692.4830072,"reason":"broker was unresponsive"},"10903":{"timestamp":1712154692.4837763,"reason":"broker was unresponsive"},"10904":{"timestamp":1712154692.4844949,"reason":"broker was unresponsive"},"10905":{"timestamp":1712095015.6494093,"reason":"nodediag failed cxi"},"10906":{"timestamp":1712094978.8228164,"reason":"nodediag failed cxi"},"10907":{"timestamp":1712154692.4864881,"reason":"broker was unresponsive"},"10908":{"timestamp":1712154692.4871569,"reason":"broker was unresponsive"},"10909":{"timestamp":1712154692.4878094,"reason":"broker was unresponsive"},"10910":{"timestamp":1712154692.4886243,"reason":"broker was unresponsive"},"10913":{"timestamp":1712154692.489284,"reason":"broker was unresponsive"},"10914":{"timestamp":1712154692.4899786,"reason":"broker was unresponsive"},"10915":{"timestamp":1712154692.4907162,"reason":"broker was unresponsive"},"10916":{"timestamp":1712154692.4914048,"reason":"broker was unresponsive"},"10919":{"timestamp":1712170230.1759303,"reason":"nodediag failed cxi"},"10920":{"timestamp":1712170199.1652176,"reason":"nodediag failed cxi"},"10925":{"timestamp":1712154692.4978392,"reason":"broker was unresponsive"},"10926":{"timestamp":1712154692.4985492,"reason":"broker was unresponsive"},"10927":{"timestamp":1712154692.4992132,"reason":"broker was unresponsive"},"10928":{"timestamp":1712154692.4999094,"reason":"broker was unresponsive"},"10929":{"timestamp":1712154692.5005817,"reason":"broker was unresponsive"},"10930":{"timestamp":1712154692.5012736,"reason":"broker was unresponsive"},"10931":{"timestamp":1712154692.5019968,"reason":"broker was unresponsive"},"10932":{"timestamp":1712154692.5027313,"reason":"broker was unresponsive"},"10933":{"timestamp":1712154692.5035186,"reason":"broker was unresponsive"},"10934":{"timestamp":1712154692.5042305,"reason":"broker was unresponsive"},"10935":{"timestamp":1712154692.5049505,"reason":"broker was unresponsive"},"10936":{"timestamp":1712154692.5056741,"reason":"broker was unresponsive"},"10937":{"timestamp":1712154692.506387,"reason":"broker was unresponsive"},"10938":{"timestamp":1712154692.5071805,"reason":"broker was unresponsive"},"10939":{"timestamp":1712154692.5079291,"reason":"broker was unresponsive"},"10940":{"timestamp":1712154692.5086536,"reason":"broker was unresponsive"},"10941":{"timestamp":1712154692.5094101,"reason":"broker was unresponsive"},"10942":{"timestamp":1712154692.510149,"reason":"broker was unresponsive"},"10943":{"timestamp":1712154692.5109775,"reason":"broker was unresponsive"},"10944":{"timestamp":1712154692.5117195,"reason":"broker was unresponsive"},"10945":{"timestamp":1712154692.512455,"reason":"broker was unresponsive"},"10946":{"timestamp":1712154692.5132055,"reason":"broker was unresponsive"},"10947":{"timestamp":1712154692.5139494,"reason":"broker was unresponsive"},"10948":{"timestamp":1712154692.5147402,"reason":"broker was unresponsive"},"10949":{"timestamp":1712154692.5154896,"reason":"broker was unresponsive"},"10950":{"timestamp":1712154692.5162227,"reason":"broker was unresponsive"},"10951":{"timestamp":1712154692.516917,"reason":"broker was unresponsive"},"10952":{"timestamp":1712154692.5194244,"reason":"broker was unresponsive"},"10953":{"timestamp":1712154692.5204279,"reason":"broker was unresponsive"},"10954":{"timestamp":1712154692.5214152,"reason":"broker was unresponsive"},"10955":{"timestamp":1712154692.5231087,"reason":"broker was unresponsive"},"10956":{"timestamp":1712154692.5246379,"reason":"broker was unresponsive"},"10957":{"timestamp":1712154692.5257483,"reason":"broker was unresponsive"},"10958":{"timestamp":1712154692.5268803,"reason":"broker was unresponsive"},"10959":{"timestamp":1712154692.5280797,"reason":"broker was unresponsive"},"10960":{"timestamp":1712154692.5292435,"reason":"broker was unresponsive"},"10961":{"timestamp":1712154692.5305285,"reason":"broker was unresponsive"},"10962":{"timestamp":1712154692.5318334,"reason":"broker was unresponsive"},"10963":{"timestamp":1712154692.5330911,"reason":"broker was unresponsive"},"10964":{"timestamp":1712154692.5342257,"reason":"broker was unresponsive"},"10965":{"timestamp":1712154692.5356715,"reason":"broker was unresponsive"},"10966":{"timestamp":1712154692.5373554,"reason":"broker was unresponsive"},"10967":{"timestamp":1712154692.5382643,"reason":"broker was unresponsive"},"10968":{"timestamp":1712154692.5395999,"reason":"broker was unresponsive"},"10969":{"timestamp":1712154692.5407925,"reason":"broker was unresponsive"},"10970":{"timestamp":1712154692.5421367,"reason":"broker was unresponsive"},"10971":{"timestamp":1712154692.5436044,"reason":"broker was unresponsive"},"10972":{"timestamp":1712154692.5451784,"reason":"broker was unresponsive"},"10973":{"timestamp":1712154692.5472317,"reason":"broker was unresponsive"},"10974":{"timestamp":1712154692.5483532,"reason":"broker was unresponsive"},"10975":{"timestamp":1712154692.5499618,"reason":"broker was unresponsive"},"10976":{"timestamp":1712154692.5515492,"reason":"broker was unresponsive"},"10977":{"timestamp":1712154692.5528789,"reason":"broker was unresponsive"},"10978":{"timestamp":1712154692.5542946,"reason":"broker was unresponsive"},"10979":{"timestamp":1712154692.5557637,"reason":"broker was unresponsive"},"10980":{"timestamp":1712154692.5570686,"reason":"broker was unresponsive"},"10981":{"timestamp":1712154692.5581162,"reason":"broker was unresponsive"},"10982":{"timestamp":1712154692.5591586,"reason":"broker was unresponsive"},"10983":{"timestamp":1712154692.5606191,"reason":"broker was unresponsive"},"10984":{"timestamp":1712154692.5617664,"reason":"broker was unresponsive"},"10985":{"timestamp":1712154692.562763,"reason":"broker was unresponsive"},"10986":{"timestamp":1712154692.5635364,"reason":"broker was unresponsive"},"10987":{"timestamp":1712154692.5646999,"reason":"broker was unresponsive"},"10988":{"timestamp":1712154692.5657291,"reason":"broker was unresponsive"},"10989":{"timestamp":1712154692.5667839,"reason":"broker was unresponsive"},"10990":{"timestamp":1712154692.5678964,"reason":"broker was unresponsive"},"10991":{"timestamp":1712154692.5693498,"reason":"broker was unresponsive"},"10992":{"timestamp":1712154692.5706637,"reason":"broker was unresponsive"},"10993":{"timestamp":1712154692.5716524,"reason":"broker was unresponsive"},"10994":{"timestamp":1712154692.5726616,"reason":"broker was unresponsive"},"10995":{"timestamp":1712154692.5738223,"reason":"broker was unresponsive"},"10996":{"timestamp":1712154692.5749226,"reason":"broker was unresponsive"},"11125":{"timestamp":1712079043.0907531,"reason":"broker was unresponsive"},"11126":{"timestamp":1711126278.6191533,"reason":"ama fail"},"11127":{"timestamp":1712079046.9967983,"reason":"broker was unresponsive"},"11128":{"timestamp":1712079046.9969132,"reason":"broker was unresponsive"},"11129":{"timestamp":1712079046.9969907,"reason":"broker was unresponsive"},"11130":{"timestamp":1712079046.9970613,"reason":"broker was unresponsive"},"11131":{"timestamp":1712079046.9971497,"reason":"broker was unresponsive"},"11132":{"timestamp":1712079046.9972458,"reason":"broker was unresponsive"},"11133":{"timestamp":1712079046.9973373,"reason":"broker was unresponsive"},"11134":{"timestamp":1712079046.9974139,"reason":"broker was unresponsive"},"11135":{"timestamp":1712079046.9975028,"reason":"broker was unresponsive"},"11136":{"timestamp":1712079046.9975903,"reason":"broker was unresponsive"},"11137":{"timestamp":1712079046.9976768,"reason":"broker was unresponsive"},"11138":{"timestamp":1712079046.9977582,"reason":"broker was unresponsive"},"11139":{"timestamp":1712079046.9978447,"reason":"broker was unresponsive"},"11140":{"timestamp":1712079046.997932,"reason":"broker was unresponsive"},"11141":{"timestamp":1712079046.9980164,"reason":"broker was unresponsive"},"11142":{"timestamp":1712079046.9981048,"reason":"broker was unresponsive"},"11143":{"timestamp":1712079046.9981937,"reason":"broker was unresponsive"},"11144":{"timestamp":1712079046.9982798,"reason":"broker was unresponsive"},"11145":{"timestamp":1712079046.9983644,"reason":"broker was unresponsive"},"11146":{"timestamp":1712079046.998455,"reason":"broker was unresponsive"},"11147":{"timestamp":1712079046.9985433,"reason":"broker was unresponsive"},"11148":{"timestamp":1712079046.9986298,"reason":"broker was unresponsive"},"11151":{"timestamp":1712079046.9987171,"reason":"broker was unresponsive"},"11152":{"timestamp":1712079046.9988039,"reason":"broker was unresponsive"},"11153":{"timestamp":1712079046.9988978,"reason":"broker was unresponsive"},"11155":{"timestamp":1712079046.9989817,"reason":"broker was unresponsive"},"11156":{"timestamp":1712079047.2586949,"reason":"broker was unresponsive"},"11157":{"timestamp":1712154692.5891516,"reason":"broker was unresponsive"},"11158":{"timestamp":1712154692.5897508,"reason":"broker was unresponsive"},"11159":{"timestamp":1712154692.5906279,"reason":"broker was unresponsive"},"11160":{"timestamp":1712154692.5914931,"reason":"broker was unresponsive"},"11161":{"timestamp":1712154692.592314,"reason":"broker was unresponsive"},"11162":{"timestamp":1712154692.5931911,"reason":"broker was unresponsive"},"11163":{"timestamp":1712154692.594203,"reason":"broker was unresponsive"},"11164":{"timestamp":1712154692.5950673,"reason":"broker was unresponsive"},"11165":{"timestamp":1712154692.5959294,"reason":"broker was unresponsive"},"11166":{"timestamp":1712154692.5967844,"reason":"broker was unresponsive"},"11167":{"timestamp":1712154692.5976548,"reason":"broker was unresponsive"},"11168":{"timestamp":1712154692.59852,"reason":"broker was unresponsive"},"11169":{"timestamp":1712154692.5993781,"reason":"broker was unresponsive"},"11170":{"timestamp":1712154692.6002505,"reason":"broker was unresponsive"},"11171":{"timestamp":1712154692.6013787,"reason":"broker was unresponsive"},"11172":{"timestamp":1712154692.6022067,"reason":"broker was unresponsive"},"11173":{"timestamp":1712095025.0662155,"reason":"epilog failed for jobid fnFN1DHfSM5"},"11175":{"timestamp":1712154692.6030674,"reason":"broker was unresponsive"},"11176":{"timestamp":1712154692.6039093,"reason":"broker was unresponsive"},"11177":{"timestamp":1712154692.6047168,"reason":"broker was unresponsive"},"11178":{"timestamp":1712154692.6051571,"reason":"broker was unresponsive"},"11179":{"timestamp":1712154692.6055932,"reason":"broker was unresponsive"},"11180":{"timestamp":1712154692.6060183,"reason":"broker was unresponsive"},"11181":{"timestamp":1712154692.6066427,"reason":"broker was unresponsive"},"11182":{"timestamp":1712154692.6073895,"reason":"broker was unresponsive"},"11183":{"timestamp":1712154692.6082635,"reason":"broker was unresponsive"},"11184":{"timestamp":1712154692.6090174,"reason":"broker was unresponsive"},"11185":{"timestamp":1712154692.609787,"reason":"broker was unresponsive"},"11186":{"timestamp":1712154692.6105771,"reason":"broker was unresponsive"},"11187":{"timestamp":1712154692.6113768,"reason":"broker was unresponsive"},"11188":{"timestamp":1712154692.6122303,"reason":"broker was unresponsive"},"11189":{"timestamp":1712154692.612987,"reason":"broker was unresponsive"},"11190":{"timestamp":1712154692.6138051,"reason":"broker was unresponsive"},"11191":{"timestamp":1712154692.6146841,"reason":"broker was unresponsive"},"11192":{"timestamp":1712154692.6155779,"reason":"broker was unresponsive"},"11193":{"timestamp":1712154692.6164539,"reason":"broker was unresponsive"},"11194":{"timestamp":1712154692.6173141,"reason":"broker was unresponsive"},"11195":{"timestamp":1712154692.6180403,"reason":"broker was unresponsive"},"11196":{"timestamp":1712154692.6188531,"reason":"broker was unresponsive"},"11197":{"timestamp":1712154692.6197391,"reason":"broker was unresponsive"},"11198":{"timestamp":1712154692.6206214,"reason":"broker was unresponsive"},"11199":{"timestamp":1712154692.6215026,"reason":"broker was unresponsive"},"11200":{"timestamp":1712154692.6223729,"reason":"broker was unresponsive"},"11201":{"timestamp":1712154692.6232476,"reason":"broker was unresponsive"},"11202":{"timestamp":1712154692.6242516,"reason":"broker was unresponsive"},"11203":{"timestamp":1712154692.6251273,"reason":"broker was unresponsive"},"11204":{"timestamp":1712154692.6260109,"reason":"broker was unresponsive"},"11205":{"timestamp":1712154692.6268666,"reason":"broker was unresponsive"},"11206":{"timestamp":1712154692.6276445,"reason":"broker was unresponsive"},"11207":{"timestamp":1712154692.6284437,"reason":"broker was unresponsive"},"11208":{"timestamp":1712154692.6292162,"reason":"broker was unresponsive"},"11209":{"timestamp":1712154692.6299369,"reason":"broker was unresponsive"},"11210":{"timestamp":1712154692.6306694,"reason":"broker was unresponsive"},"11211":{"timestamp":1712154692.6313751,"reason":"broker was unresponsive"},"11212":{"timestamp":1712154692.6321084,"reason":"broker was unresponsive"},"11213":{"timestamp":1712154692.632833,"reason":"broker was unresponsive"},"11214":{"timestamp":1712154692.6335218,"reason":"broker was unresponsive"},"11215":{"timestamp":1712154692.6341536,"reason":"broker was unresponsive"},"11216":{"timestamp":1712154692.6348433,"reason":"broker was unresponsive"},"11217":{"timestamp":1712154692.6355832,"reason":"broker was unresponsive"},"11218":{"timestamp":1712154692.6360271,"reason":"broker was unresponsive"},"11219":{"timestamp":1712154692.6365154,"reason":"broker was unresponsive"},"11220":{"timestamp":1712154692.6373053,"reason":"broker was unresponsive"},"11221":{"timestamp":1712154692.6381056,"reason":"broker was unresponsive"},"11222":{"timestamp":1712154692.638907,"reason":"broker was unresponsive"},"11223":{"timestamp":1712154692.6396916,"reason":"broker was unresponsive"},"11224":{"timestamp":1712154692.6404746,"reason":"broker was unresponsive"},"11225":{"timestamp":1712154692.6412561,"reason":"broker was unresponsive"},"11226":{"timestamp":1712154692.6420529,"reason":"broker was unresponsive"},"11227":{"timestamp":1712154692.6427951,"reason":"broker was unresponsive"},"11228":{"timestamp":1712154692.6435363,"reason":"broker was unresponsive"},"11229":{"timestamp":1712154692.6442616,"reason":"broker was unresponsive"},"11230":{"timestamp":1712154692.6450257,"reason":"broker was unresponsive"},"11231":{"timestamp":1712154692.6458087,"reason":"broker was unresponsive"},"11232":{"timestamp":1712154692.6466007,"reason":"broker was unresponsive"},"11233":{"timestamp":1712154692.6473877,"reason":"broker was unresponsive"},"11234":{"timestamp":1712154692.6481757,"reason":"broker was unresponsive"},"11235":{"timestamp":1712154692.649344,"reason":"broker was unresponsive"},"11236":{"timestamp":1712154692.6501386,"reason":"broker was unresponsive"},"11237":{"timestamp":1712154692.6509383,"reason":"broker was unresponsive"},"11238":{"timestamp":1712154692.6517637,"reason":"broker was unresponsive"},"11239":{"timestamp":1712154692.6525581,"reason":"broker was unresponsive"},"11240":{"timestamp":1712154692.6533463,"reason":"broker was unresponsive"},"11241":{"timestamp":1712154692.6542008,"reason":"broker was unresponsive"},"11242":{"timestamp":1712154692.6550143,"reason":"broker was unresponsive"},"11243":{"timestamp":1712154692.655849,"reason":"broker was unresponsive"},"11244":{"timestamp":1712154692.6567733,"reason":"broker was unresponsive"},"11245":{"timestamp":1712154692.6575952,"reason":"broker was unresponsive"},"11246":{"timestamp":1712154692.6583927,"reason":"broker was unresponsive"},"11247":{"timestamp":1712154692.6591909,"reason":"broker was unresponsive"},"11248":{"timestamp":1712154692.6599851,"reason":"broker was unresponsive"},"11249":{"timestamp":1712154692.6607814,"reason":"broker was unresponsive"},"11250":{"timestamp":1712154692.6615689,"reason":"broker was unresponsive"},"11251":{"timestamp":1712154692.6623478,"reason":"broker was unresponsive"},"11252":{"timestamp":1712154692.6631436,"reason":"broker was unresponsive"},"11292":{"timestamp":1712174534.9495943,"reason":"nodediag failed cxi"},"11293":{"timestamp":1712174539.6623499,"reason":"nodediag failed cxi"},"11294":{"timestamp":1712174531.2918873,"reason":"nodediag failed cxi"},"11295":{"timestamp":1712174470.4574249,"reason":"nodediag failed cxi"},"11296":{"timestamp":1712174532.0124931,"reason":"nodediag failed cxi"},"11297":{"timestamp":1712174514.4612403,"reason":"nodediag failed cxi"},"11298":{"timestamp":1712174520.7652485,"reason":"nodediag failed cxi"},"11299":{"timestamp":1712174469.7052834,"reason":"nodediag failed cxi"},"11300":{"timestamp":1712174525.1720884,"reason":"nodediag failed cxi"},"11327":{"timestamp":1712154692.72453,"reason":"broker was unresponsive"},"11328":{"timestamp":1712154692.7253735,"reason":"broker was unresponsive"},"11329":{"timestamp":1712154692.7262185,"reason":"broker was unresponsive"},"11331":{"timestamp":1712154692.7270534,"reason":"broker was unresponsive"},"11332":{"timestamp":1712154692.7278821,"reason":"broker was unresponsive"},"11333":{"timestamp":1712154692.7287097,"reason":"broker was unresponsive"},"11334":{"timestamp":1712154692.7296131,"reason":"broker was unresponsive"},"11335":{"timestamp":1712154692.7305033,"reason":"broker was unresponsive"},"11336":{"timestamp":1712154692.7313645,"reason":"broker was unresponsive"},"11337":{"timestamp":1712154692.7322342,"reason":"broker was unresponsive"},"11338":{"timestamp":1712154692.7331452,"reason":"broker was unresponsive"},"11339":{"timestamp":1712154692.7340038,"reason":"broker was unresponsive"},"11340":{"timestamp":1712154692.7348406,"reason":"broker was unresponsive"},"11341":{"timestamp":1712154692.7356975,"reason":"broker was unresponsive"},"11342":{"timestamp":1712154692.7365856,"reason":"broker was unresponsive"},"11343":{"timestamp":1712154692.7374482,"reason":"broker was unresponsive"},"11344":{"timestamp":1712154692.7382832,"reason":"broker was unresponsive"},"11345":{"timestamp":1712154692.7391167,"reason":"broker was unresponsive"},"11346":{"timestamp":1712154692.740042,"reason":"broker was unresponsive"},"11347":{"timestamp":1712154692.7409012,"reason":"broker was unresponsive"},"11348":{"timestamp":1712154692.7417147,"reason":"broker was unresponsive"},"11349":{"timestamp":1712154692.742522,"reason":"broker was unresponsive"},"11350":{"timestamp":1712154692.7433016,"reason":"broker was unresponsive"},"11351":{"timestamp":1712154692.7441294,"reason":"broker was unresponsive"},"11352":{"timestamp":1712154692.7449949,"reason":"broker was unresponsive"},"11353":{"timestamp":1712154692.7459545,"reason":"broker was unresponsive"},"11354":{"timestamp":1712154692.7469089,"reason":"broker was unresponsive"},"11355":{"timestamp":1712154692.7478616,"reason":"broker was unresponsive"},"11356":{"timestamp":1712154692.7488129,"reason":"broker was unresponsive"},"11357":{"timestamp":1712154692.7497623,"reason":"broker was unresponsive"},"11358":{"timestamp":1712154692.7509179,"reason":"broker was unresponsive"},"11359":{"timestamp":1712154692.7518847,"reason":"broker was unresponsive"},"11360":{"timestamp":1712154692.7528374,"reason":"broker was unresponsive"},"11361":{"timestamp":1712154692.7537308,"reason":"broker was unresponsive"},"11362":{"timestamp":1712154692.7545819,"reason":"broker was unresponsive"},"11363":{"timestamp":1712154692.7554591,"reason":"broker was unresponsive"},"11364":{"timestamp":1712154692.7564318,"reason":"broker was unresponsive"},"11365":{"timestamp":1712154692.7573259,"reason":"broker was unresponsive"},"11366":{"timestamp":1712154692.7581871,"reason":"broker was unresponsive"},"11367":{"timestamp":1712154692.7591572,"reason":"broker was unresponsive"},"11368":{"timestamp":1712154692.760112,"reason":"broker was unresponsive"},"11369":{"timestamp":1712154692.7609518,"reason":"broker was unresponsive"},"11370":{"timestamp":1712154692.7617881,"reason":"broker was unresponsive"},"11371":{"timestamp":1712154692.762639,"reason":"broker was unresponsive"},"11372":{"timestamp":1712154692.763489,"reason":"broker was unresponsive"},"11373":{"timestamp":1712154692.7643514,"reason":"broker was unresponsive"},"11374":{"timestamp":1712154692.7651522,"reason":"broker was unresponsive"},"11375":{"timestamp":1712154692.7659426,"reason":"broker was unresponsive"},"11376":{"timestamp":1712154692.7667835,"reason":"broker was unresponsive"},"11377":{"timestamp":1712154692.7676272,"reason":"broker was unresponsive"},"11378":{"timestamp":1712154692.7684815,"reason":"broker was unresponsive"},"11379":{"timestamp":1712154692.7693639,"reason":"broker was unresponsive"},"11380":{"timestamp":1712154692.7713802,"reason":"broker was unresponsive"},"11381-11382":{"timestamp":1712079203.2770023,"reason":"draining to run on-node diags - wendy"},"11383":{"timestamp":1712154692.7738781,"reason":"broker was unresponsive"},"11384":{"timestamp":1712154692.7747688,"reason":"broker was unresponsive"},"11385":{"timestamp":1712154692.7756054,"reason":"broker was unresponsive"},"11386":{"timestamp":1712154692.7764227,"reason":"broker was unresponsive"},"11389":{"timestamp":1712154692.7772832,"reason":"broker was unresponsive"},"11390":{"timestamp":1712154692.7781432,"reason":"broker was unresponsive"},"11391":{"timestamp":1712154692.7789931,"reason":"broker was unresponsive"},"11392":{"timestamp":1712154692.7798281,"reason":"broker was unresponsive"},"11393":{"timestamp":1712154692.7806766,"reason":"broker was unresponsive"},"11394":{"timestamp":1712154692.7815275,"reason":"broker was unresponsive"},"11395":{"timestamp":1712154692.7823181,"reason":"broker was unresponsive"},"11396":{"timestamp":1712154692.783165,"reason":"broker was unresponsive"},"11397":{"timestamp":1712154692.7840011,"reason":"broker was unresponsive"},"11398":{"timestamp":1712154692.7848673,"reason":"broker was unresponsive"},"11399":{"timestamp":1712154692.7857177,"reason":"broker was unresponsive"},"11400":{"timestamp":1712154692.7864349,"reason":"broker was unresponsive"},"11401":{"timestamp":1712154692.7871354,"reason":"broker was unresponsive"},"11402":{"timestamp":1712154692.7878435,"reason":"broker was unresponsive"},"11404":{"timestamp":1712154692.7885523,"reason":"broker was unresponsive"},"11405":{"timestamp":1712154692.7892518,"reason":"broker was unresponsive"},"11406":{"timestamp":1712154692.7900887,"reason":"broker was unresponsive"},"11407":{"timestamp":1712154692.7909517,"reason":"broker was unresponsive"},"11408":{"timestamp":1712154692.7918444,"reason":"broker was unresponsive"},"11409":{"timestamp":1712154692.7927401,"reason":"broker was unresponsive"},"11410":{"timestamp":1712154692.7936668,"reason":"broker was unresponsive"},"11411":{"timestamp":1712154692.7945628,"reason":"broker was unresponsive"},"11412":{"timestamp":1712154692.7954464,"reason":"broker was unresponsive"},"11413":{"timestamp":1712154692.7963233,"reason":"broker was unresponsive"},"11414":{"timestamp":1712154692.7972093,"reason":"broker was unresponsive"},"11415":{"timestamp":1712154692.7980816,"reason":"broker was unresponsive"},"11416":{"timestamp":1712154692.7989514,"reason":"broker was unresponsive"},"11417":{"timestamp":1711743415.7833021,"reason":"epilog failed for jobid fmKGqZRsvpP"},"11418":{"timestamp":1712154692.8006811,"reason":"broker was unresponsive"},"11419":{"timestamp":1712154692.801528,"reason":"broker was unresponsive"},"11420":{"timestamp":1712154692.802325,"reason":"broker was unresponsive"},"11421":{"timestamp":1712154692.8031952,"reason":"broker was unresponsive"},"11422":{"timestamp":1712154692.8041372,"reason":"broker was unresponsive"},"11423":{"timestamp":1712154692.8050284,"reason":"broker was unresponsive"},"11424":{"timestamp":1712154692.8059421,"reason":"broker was unresponsive"},"11425":{"timestamp":1712154692.8068275,"reason":"broker was unresponsive"},"11426":{"timestamp":1712154692.8077354,"reason":"broker was unresponsive"},"11427":{"timestamp":1712154692.8085849,"reason":"broker was unresponsive"},"11428":{"timestamp":1712154692.8094218,"reason":"broker was unresponsive"},"11429":{"timestamp":1712154692.8102601,"reason":"broker was unresponsive"},"11430":{"timestamp":1712154692.8111579,"reason":"broker was unresponsive"},"11431":{"timestamp":1712154692.8120186,"reason":"broker was unresponsive"},"11432":{"timestamp":1712154692.8128703,"reason":"broker was unresponsive"},"11433":{"timestamp":1712154692.8137786,"reason":"broker was unresponsive"},"11434":{"timestamp":1712154692.8146782,"reason":"broker was unresponsive"},"11435":{"timestamp":1712154692.8156159,"reason":"broker was unresponsive"},"11436":{"timestamp":1712154692.8165164,"reason":"broker was unresponsive"},"11437":{"timestamp":1712154692.817409,"reason":"broker was unresponsive"},"11438":{"timestamp":1712154692.8182616,"reason":"broker was unresponsive"},"11440":{"timestamp":1712154692.8191555,"reason":"broker was unresponsive"},"11441":{"timestamp":1712154692.8200519,"reason":"broker was unresponsive"},"11442":{"timestamp":1712154692.8209813,"reason":"broker was unresponsive"},"11443":{"timestamp":1712154692.8218842,"reason":"broker was unresponsive"},"11444":{"timestamp":1712154692.8227746,"reason":"broker was unresponsive"},"11445":{"timestamp":1712154692.8236468,"reason":"broker was unresponsive"},"11446":{"timestamp":1712154692.8245139,"reason":"broker was unresponsive"},"11447":{"timestamp":1712154692.8253784,"reason":"broker was unresponsive"},"11448":{"timestamp":1712154692.8261838,"reason":"broker was unresponsive"},"11449":{"timestamp":1712154692.8270147,"reason":"broker was unresponsive"},"11450":{"timestamp":1712154692.8278799,"reason":"broker was unresponsive"},"11451":{"timestamp":1712154692.8287663,"reason":"broker was unresponsive"},"11452":{"timestamp":1712154692.8297007,"reason":"broker was unresponsive"},"11453":{"timestamp":1712154692.8306162,"reason":"broker was unresponsive"},"11454":{"timestamp":1712154692.8315661,"reason":"broker was unresponsive"},"11455":{"timestamp":1712154692.8324704,"reason":"broker was unresponsive"},"11456":{"timestamp":1712154692.8333681,"reason":"broker was unresponsive"},"11457":{"timestamp":1712154692.8342683,"reason":"broker was unresponsive"},"11458":{"timestamp":1712154692.8351686,"reason":"broker was unresponsive"},"11459":{"timestamp":1712154692.8360796,"reason":"broker was unresponsive"},"11460":{"timestamp":1711123247.7819526,"reason":"ama fail"},"11461":{"timestamp":1712154692.8369684,"reason":"broker was unresponsive"},"11462":{"timestamp":1712154692.8378525,"reason":"broker was unresponsive"},"11463":{"timestamp":1712154692.8387308,"reason":"broker was unresponsive"},"11464":{"timestamp":1712154692.8396258,"reason":"broker was unresponsive"},"11465":{"timestamp":1712154692.8404713,"reason":"broker was unresponsive"},"11467":{"timestamp":1712154692.8413599,"reason":"broker was unresponsive"},"11468":{"timestamp":1712154692.8422604,"reason":"broker was unresponsive"},"11469":{"timestamp":1712154692.8431501,"reason":"broker was unresponsive"},"11470":{"timestamp":1712154692.8440495,"reason":"broker was unresponsive"},"11472":{"timestamp":1712154692.8449426,"reason":"broker was unresponsive"},"11473":{"timestamp":1712154692.845834,"reason":"broker was unresponsive"},"11474":{"timestamp":1712154692.8467212,"reason":"broker was unresponsive"},"11475":{"timestamp":1712154692.8476095,"reason":"broker was unresponsive"},"11476":{"timestamp":1712154692.8485229,"reason":"broker was unresponsive"},"11477":{"timestamp":1712154692.8494375,"reason":"broker was unresponsive"},"11478":{"timestamp":1712154692.8504064,"reason":"broker was unresponsive"},"11479":{"timestamp":1712154692.851279,"reason":"broker was unresponsive"},"11480":{"timestamp":1712154692.8522575,"reason":"broker was unresponsive"},"11481":{"timestamp":1712154692.8531055,"reason":"broker was unresponsive"},"11482":{"timestamp":1712154692.8539455,"reason":"broker was unresponsive"},"11483":{"timestamp":1712154692.8548,"reason":"broker was unresponsive"},"11484":{"timestamp":1712154692.8556988,"reason":"broker was unresponsive"},"11485":{"timestamp":1712154692.8565514,"reason":"broker was unresponsive"},"11486":{"timestamp":1712154692.8573911,"reason":"broker was unresponsive"},"11487":{"timestamp":1712154692.8583002,"reason":"broker was unresponsive"},"11488":{"timestamp":1712154692.8591919,"reason":"broker was unresponsive"},"11489":{"timestamp":1712154692.8600814,"reason":"broker was unresponsive"},"11490":{"timestamp":1712154692.8610039,"reason":"broker was unresponsive"},"11491":{"timestamp":1712154692.8618779,"reason":"broker was unresponsive"},"11492":{"timestamp":1712154692.8627224,"reason":"broker was unresponsive"},"11493":{"timestamp":1712154692.8635724,"reason":"broker was unresponsive"},"11494":{"timestamp":1712154692.8644898,"reason":"broker was unresponsive"},"11495":{"timestamp":1712154692.8654335,"reason":"broker was unresponsive"},"11496":{"timestamp":1712154692.866354,"reason":"broker was unresponsive"},"11497":{"timestamp":1712154692.8672559,"reason":"broker was unresponsive"},"11498":{"timestamp":1712154692.8681912,"reason":"broker was unresponsive"},"11499":{"timestamp":1712154692.869103,"reason":"broker was unresponsive"},"11500":{"timestamp":1712154692.8700061,"reason":"broker was unresponsive"},"11501":{"timestamp":1712154692.8709049,"reason":"broker was unresponsive"},"11502":{"timestamp":1712154692.8718345,"reason":"broker was unresponsive"},"11503":{"timestamp":1712154692.8727145,"reason":"broker was unresponsive"},"11504":{"timestamp":1712154692.8737223,"reason":"broker was unresponsive"},"11505":{"timestamp":1712154692.8746645,"reason":"broker was unresponsive"},"11506":{"timestamp":1712154692.875694,"reason":"broker was unresponsive"},"11508":{"timestamp":1712154692.8767233,"reason":"broker was unresponsive"},"11509":{"timestamp":1712154692.8777416,"reason":"broker was unresponsive"},"11510":{"timestamp":1712154692.8787465,"reason":"broker was unresponsive"},"11511":{"timestamp":1712154692.8796964,"reason":"broker was unresponsive"},"11512":{"timestamp":1712154692.880748,"reason":"broker was unresponsive"},"11513":{"timestamp":1712154692.8817775,"reason":"broker was unresponsive"},"11514":{"timestamp":1712154692.8828058,"reason":"broker was unresponsive"},"11515":{"timestamp":1712154692.8838191,"reason":"broker was unresponsive"},"11516":{"timestamp":1712154692.8847611,"reason":"broker was unresponsive"},"11517":{"timestamp":1711659950.9614942,"reason":"broker was unresponsive"},"11518":{"timestamp":1712154692.885607,"reason":"broker was unresponsive"},"11519":{"timestamp":1712154692.8864586,"reason":"broker was unresponsive"},"11520":{"timestamp":1712154692.8874149,"reason":"broker was unresponsive"},"11521":{"timestamp":1712154692.8884397,"reason":"broker was unresponsive"},"11522":{"timestamp":1712154692.8897202,"reason":"broker was unresponsive"},"11523":{"timestamp":1712154692.89062,"reason":"broker was unresponsive"},"11524":{"timestamp":1712154692.89149,"reason":"broker was unresponsive"},"11525":{"timestamp":1712154692.8923569,"reason":"broker was unresponsive"},"11526":{"timestamp":1712154692.8932726,"reason":"broker was unresponsive"},"11527":{"timestamp":1712043086.7846026,"reason":"broker was unresponsive"},"11528":{"timestamp":1712154692.8949468,"reason":"broker was unresponsive"},"11529":{"timestamp":1712154692.8958304,"reason":"broker was unresponsive"},"11530":{"timestamp":1712154692.8967347,"reason":"broker was unresponsive"},"11531":{"timestamp":1712154692.8976376,"reason":"broker was unresponsive"},"11532":{"timestamp":1712154692.8985436,"reason":"broker was unresponsive"},"11534":{"timestamp":1712154692.8994722,"reason":"broker was unresponsive"},"11535":{"timestamp":1712154692.900347,"reason":"broker was unresponsive"},"11536":{"timestamp":1711659951.0613308,"reason":"broker was unresponsive"},"11537":{"timestamp":1712154692.9012654,"reason":"broker was unresponsive"},"11538":{"timestamp":1712154692.9021657,"reason":"broker was unresponsive"},"11539":{"timestamp":1712154692.9030745,"reason":"broker was unresponsive"},"11540":{"timestamp":1712154692.9039884,"reason":"broker was unresponsive"},"11541":{"timestamp":1712154692.9048347,"reason":"broker was unresponsive"},"11542":{"timestamp":1712154692.9056971,"reason":"broker was unresponsive"},"11543":{"timestamp":1712154692.9065795,"reason":"broker was unresponsive"},"11544":{"timestamp":1712154692.907474,"reason":"broker was unresponsive"},"11545":{"timestamp":1712154692.9083641,"reason":"broker was unresponsive"},"11546":{"timestamp":1712154692.9092846,"reason":"broker was unresponsive"},"11547":{"timestamp":1712154692.9101894,"reason":"broker was unresponsive"},"11548":{"timestamp":1712154692.9111967,"reason":"broker was unresponsive"},"11549":{"timestamp":1712154692.9120445,"reason":"broker was unresponsive"},"11550":{"timestamp":1712154692.912874,"reason":"broker was unresponsive"},"11551":{"timestamp":1712154692.9136961,"reason":"broker was unresponsive"},"11552":{"timestamp":1712154692.9145219,"reason":"broker was unresponsive"},"11553":{"timestamp":1712154692.9153473,"reason":"broker was unresponsive"},"11554":{"timestamp":1712154692.9162686,"reason":"broker was unresponsive"},"11555":{"timestamp":1712154692.9224265,"reason":"broker was unresponsive"},"11556":{"timestamp":1712154692.9234829,"reason":"broker was unresponsive"},"11557":{"timestamp":1712154692.9245389,"reason":"broker was unresponsive"},"11558":{"timestamp":1712154692.9256041,"reason":"broker was unresponsive"},"11559":{"timestamp":1712154692.9266512,"reason":"broker was unresponsive"},"11560":{"timestamp":1712154692.9276841,"reason":"broker was unresponsive"},"11561":{"timestamp":1712154692.9286895,"reason":"broker was unresponsive"},"11562":{"timestamp":1712154692.9297462,"reason":"broker was unresponsive"},"11563":{"timestamp":1712154692.9307995,"reason":"broker was unresponsive"},"11564":{"timestamp":1712154692.9318008,"reason":"broker was unresponsive"},"11565":{"timestamp":1712154692.9327316,"reason":"broker was unresponsive"},"10137,10140,11566":{"timestamp":1711663526.290566,"reason":"core dumps -KK"},"11567":{"timestamp":1712154692.9345765,"reason":"broker was unresponsive"},"11568":{"timestamp":1712154692.9363933,"reason":"broker was unresponsive"},"11569":{"timestamp":1711946370.2288194,"reason":"broker was unresponsive"},"11570":{"timestamp":1712154692.9373949,"reason":"broker was unresponsive"},"11571":{"timestamp":1712154692.9383559,"reason":"broker was unresponsive"},"11572":{"timestamp":1712154692.9393115,"reason":"broker was unresponsive"},"11573":{"timestamp":1712154692.9562876,"reason":"broker was unresponsive"},"11574":{"timestamp":1712154692.957289,"reason":"broker was unresponsive"},"11575":{"timestamp":1712154692.9584115,"reason":"broker was unresponsive"},"11576":{"timestamp":1712154692.9744682,"reason":"broker was unresponsive"},"11577":{"timestamp":1712154692.9754434,"reason":"broker was unresponsive"},"11578":{"timestamp":1712154692.9764154,"reason":"broker was unresponsive"},"11579":{"timestamp":1712154692.9773655,"reason":"broker was unresponsive"},"11580":{"timestamp":1712154692.9783056,"reason":"broker was unresponsive"},"11581":{"timestamp":1712154692.9792244,"reason":"broker was unresponsive"},"11582":{"timestamp":1712154692.9902475,"reason":"broker was unresponsive"},"11583":{"timestamp":1712154692.9912097,"reason":"broker was unresponsive"},"11584":{"timestamp":1712154692.9921758,"reason":"broker was unresponsive"},"11585":{"timestamp":1712154692.9931383,"reason":"broker was unresponsive"},"11586":{"timestamp":1712154693.0131507,"reason":"broker was unresponsive"},"11587":{"timestamp":1711663526.290566,"reason":"kernel panics - JRG"},"11588":{"timestamp":1712093194.2606094,"reason":"bad partner node - JRG"},"11589":{"timestamp":1712154693.014164,"reason":"broker was unresponsive"},"11590":{"timestamp":1712154693.0151422,"reason":"broker was unresponsive"},"11591":{"timestamp":1712154693.016113,"reason":"broker was unresponsive"},"11592":{"timestamp":1712026940.6850164,"reason":"broker was unresponsive"},"11593":{"timestamp":1711762170.0976398,"reason":""},"11594":{"timestamp":1712154693.0198882,"reason":"broker was unresponsive"},"11595":{"timestamp":1712154693.020884,"reason":"broker was unresponsive"},"11596":{"timestamp":1712154693.0218966,"reason":"broker was unresponsive"},"11597":{"timestamp":1712154693.0228806,"reason":"broker was unresponsive"},"11598":{"timestamp":1712154693.0250845,"reason":"broker was unresponsive"},"11599":{"timestamp":1712154693.0272567,"reason":"broker was unresponsive"},"11600":{"timestamp":1712154693.0281804,"reason":"broker was unresponsive"},"11601":{"timestamp":1712154693.0290883,"reason":"broker was unresponsive"},"11602":{"timestamp":1712154693.0299966,"reason":"broker was unresponsive"},"11603":{"timestamp":1712026940.6851528,"reason":"broker was unresponsive"},"11604":{"timestamp":1712154693.0328157,"reason":"broker was unresponsive"},"11605":{"timestamp":1712154693.0337353,"reason":"broker was unresponsive"},"11606":{"timestamp":1712154693.0346334,"reason":"broker was unresponsive"},"11607":{"timestamp":1712154693.0386083,"reason":"broker was unresponsive"},"11608":{"timestamp":1712154693.0394509,"reason":"broker was unresponsive"},"11609":{"timestamp":1712026940.6852219,"reason":"broker was unresponsive"},"11610":{"timestamp":1712154693.0410652,"reason":"broker was unresponsive"},"11611":{"timestamp":1712154693.0418851,"reason":"broker was unresponsive"},"11612":{"timestamp":1712028840.7845249,"reason":"broker was unresponsive"},"11613":{"timestamp":1712154693.0439832,"reason":"broker was unresponsive"},"11614":{"timestamp":1712154693.0448203,"reason":"broker was unresponsive"},"11615":{"timestamp":1712154693.0456433,"reason":"broker was unresponsive"},"11616":{"timestamp":1712154693.0464704,"reason":"broker was unresponsive"},"11617":{"timestamp":1712154693.0472839,"reason":"broker was unresponsive"},"11618":{"timestamp":1712154693.0482194,"reason":"broker was unresponsive"},"11619":{"timestamp":1712026940.6852841,"reason":"broker was unresponsive"},"11620":{"timestamp":1712154693.0500429,"reason":"broker was unresponsive"},"11621":{"timestamp":1712154693.0508971,"reason":"broker was unresponsive"},"11622":{"timestamp":1712154693.0517426,"reason":"broker was unresponsive"},"11623":{"timestamp":1712154693.0525949,"reason":"broker was unresponsive"},"11624":{"timestamp":1712154693.0546894,"reason":"broker was unresponsive"},"11625":{"timestamp":1712154693.0555315,"reason":"broker was unresponsive"},"11626":{"timestamp":1712154693.0564125,"reason":"broker was unresponsive"},"11627":{"timestamp":1712154693.0572789,"reason":"broker was unresponsive"},"11628":{"timestamp":1712154693.058116,"reason":"broker was unresponsive"},"11629":{"timestamp":1712154693.0590928,"reason":"broker was unresponsive"},"11630":{"timestamp":1712154693.0598869,"reason":"broker was unresponsive"},"11631":{"timestamp":1712026940.6853518,"reason":"broker was unresponsive"},"11632":{"timestamp":1712154693.0617664,"reason":"broker was unresponsive"},"11633":{"timestamp":1712026940.7842064,"reason":"broker was unresponsive"},"11634":{"timestamp":1712154693.0629814,"reason":"broker was unresponsive"},"11635":{"timestamp":1712154693.0641026,"reason":"broker was unresponsive"},"11636":{"timestamp":1712154693.0652032,"reason":"broker was unresponsive"},"11653":{"timestamp":1712174466.0412419,"reason":"nodediag failed cxi"},"11654":{"timestamp":1712174483.5145023,"reason":"nodediag failed cxi"},"11655":{"timestamp":1712174485.5970631,"reason":"nodediag failed cxi"},"11656":{"timestamp":1712174464.6082194,"reason":"nodediag failed cxi"},"11657":{"timestamp":1712174465.4122262,"reason":"nodediag failed cxi"},"11658":{"timestamp":1712174466.6267278,"reason":"nodediag failed cxi"},"11659":{"timestamp":1712174468.6355774,"reason":"nodediag failed cxi"},"11660":{"timestamp":1712174456.1854422,"reason":"nodediag failed cxi"},"11676":{"timestamp":1712174838.6317751,"reason":"nodediag failed cxi"},"11765-11780":{"timestamp":1711993308.7730825,"reason":"Diags Running"},"11784":{"timestamp":1711725755.0632687,"reason":"Running Diags - Rabbits Off"},"11790":{"timestamp":1712004602.484952,"reason":"bad partner node - JRG"},"11789,11793":{"timestamp":1712004602.484952,"reason":"kernel panics - JRG"},"11794":{"timestamp":1711725767.0656931,"reason":"bad partner node - JRG"},"11781-11783,11785-11788,11791-11792,11795-11796":{"timestamp":1712004602.484952,"reason":"Running Diags - Rabbits Off"},"11803":{"timestamp":1711668961.0618374,"reason":"Running Diags - Rabbits Off"},"11797-11801,11804-11811":{"timestamp":1712004859.9543238,"reason":"Running Diags - Rabbits Off"},"11812":{"timestamp":1712004514.6037207,"reason":"Running Diags - Rabbits Off"},"11813-11828":{"timestamp":1712005088.7732615,"reason":"Running Diags - Rabbits Off"},"11829-11844":{"timestamp":1712068080.9443812,"reason":"Running Diags - Rabbits Off"},"11845-11859":{"timestamp":1712068141.3627269,"reason":"Running Diags - Rabbits Off"},"11860":{"timestamp":1711727735.0622723,"reason":"Running Diags - Rabbits Off"},"11861-11876":{"timestamp":1712068186.5559592,"reason":"Running Diags - Rabbits Off"},"11877":{"timestamp":1712101843.9122288,"reason":"broker was unresponsive"},"11878":{"timestamp":1712101843.9123399,"reason":"broker was unresponsive"},"11879":{"timestamp":1712101844.0117874,"reason":"broker was unresponsive"},"11880":{"timestamp":1712101849.9135923,"reason":"broker was unresponsive"},"11881":{"timestamp":1712101849.9136949,"reason":"broker was unresponsive"},"11882":{"timestamp":1712101849.9137807,"reason":"broker was unresponsive"},"11883":{"timestamp":1712101849.913851,"reason":"broker was unresponsive"},"11884":{"timestamp":1712101849.9139225,"reason":"broker was unresponsive"},"11885":{"timestamp":1712101849.9139903,"reason":"broker was unresponsive"},"11886":{"timestamp":1712101849.9140527,"reason":"broker was unresponsive"},"11887":{"timestamp":1712101849.9141397,"reason":"broker was unresponsive"},"11888":{"timestamp":1712101849.9142251,"reason":"broker was unresponsive"},"11889":{"timestamp":1712101849.9143026,"reason":"broker was unresponsive"},"11890":{"timestamp":1712101850.048907,"reason":"broker was unresponsive"},"11892":{"timestamp":1712097564.0118735,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1712178469.6455793,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1712178485.0422347,"name":"online","context":{"idset":"0"}}
+{"timestamp":1712188002.7828493,"name":"online","context":{"idset":"841"}}
+{"timestamp":1712188116.0589819,"name":"undrain","context":{"idset":"841"}}
+{"timestamp":1712188279.3106575,"name":"online","context":{"idset":"951"}}
+{"timestamp":1712188279.4238133,"name":"online","context":{"idset":"969"}}
+{"timestamp":1712188279.4743979,"name":"online","context":{"idset":"970"}}
+{"timestamp":1712188279.5168154,"name":"online","context":{"idset":"972"}}
+{"timestamp":1712188279.5390491,"name":"online","context":{"idset":"975"}}
+{"timestamp":1712188279.7048562,"name":"online","context":{"idset":"9207"}}
+{"timestamp":1712188279.7160225,"name":"online","context":{"idset":"9213"}}
+{"timestamp":1712188279.8047452,"name":"online","context":{"idset":"971,974"}}
+{"timestamp":1712188279.9446213,"name":"online","context":{"idset":"952,973,976,978-979,9208"}}
+{"timestamp":1712188279.9648964,"name":"online","context":{"idset":"9228"}}
+{"timestamp":1712188280.0211737,"name":"online","context":{"idset":"980"}}
+{"timestamp":1712188280.09848,"name":"online","context":{"idset":"9218"}}
+{"timestamp":1712188280.1135337,"name":"online","context":{"idset":"9216"}}
+{"timestamp":1712188280.1722147,"name":"online","context":{"idset":"9215"}}
+{"timestamp":1712188280.275651,"name":"online","context":{"idset":"9209"}}
+{"timestamp":1712188280.3288364,"name":"online","context":{"idset":"9219"}}
+{"timestamp":1712188280.3539641,"name":"online","context":{"idset":"9212"}}
+{"timestamp":1712188280.5711441,"name":"online","context":{"idset":"9231"}}
+{"timestamp":1712188280.6355097,"name":"online","context":{"idset":"9248"}}
+{"timestamp":1712188280.7283168,"name":"online","context":{"idset":"9206"}}
+{"timestamp":1712188280.7570953,"name":"online","context":{"idset":"9211,9234"}}
+{"timestamp":1712188280.8294814,"name":"online","context":{"idset":"9233"}}
+{"timestamp":1712188280.852844,"name":"online","context":{"idset":"9222"}}
+{"timestamp":1712188280.9237926,"name":"online","context":{"idset":"9220,9225"}}
+{"timestamp":1712188280.952575,"name":"online","context":{"idset":"9210"}}
+{"timestamp":1712188281.1009624,"name":"online","context":{"idset":"9227"}}
+{"timestamp":1712188281.235033,"name":"online","context":{"idset":"9232"}}
+{"timestamp":1712188281.277076,"name":"online","context":{"idset":"9235"}}
+{"timestamp":1712188281.3015079,"name":"online","context":{"idset":"9226"}}
+{"timestamp":1712188281.4822066,"name":"online","context":{"idset":"9230,9240"}}
+{"timestamp":1712188281.6307549,"name":"online","context":{"idset":"9244,9246"}}
+{"timestamp":1712188281.6665986,"name":"online","context":{"idset":"9221"}}
+{"timestamp":1712188281.8048275,"name":"online","context":{"idset":"9224,9238,9254,9257,9264"}}
+{"timestamp":1712188281.927583,"name":"online","context":{"idset":"9205,9217,9223,9229,9241,9249,9302"}}
+{"timestamp":1712188282.2137625,"name":"online","context":{"idset":"9250-9251"}}
+{"timestamp":1712188282.32425,"name":"online","context":{"idset":"9236,9260,9262-9263,9282,9293"}}
+{"timestamp":1712188282.4307864,"name":"online","context":{"idset":"9237,9242-9243,9245,9247,9252,9256,9278,9283,9307"}}
+{"timestamp":1712188282.5483158,"name":"online","context":{"idset":"9267,9274,9300"}}
+{"timestamp":1712188282.6539679,"name":"online","context":{"idset":"9258,9265,9271,9287,9309,9314,9317"}}
+{"timestamp":1712188282.7547836,"name":"online","context":{"idset":"9259,9266,9268,9272,9277,9295-9296,9299,9318"}}
+{"timestamp":1712188282.8786144,"name":"online","context":{"idset":"9255,9261,9269,9276,9313,9319"}}
+{"timestamp":1712188282.9893126,"name":"online","context":{"idset":"9279,9292,9301,9306,9308,9310,9312"}}
+{"timestamp":1712188283.1069841,"name":"online","context":{"idset":"9253,9273,9275,9281,9285,9291"}}
+{"timestamp":1712188283.2355828,"name":"online","context":{"idset":"9270,9284,9288-9289,9298,9305,9316"}}
+{"timestamp":1712188283.3411882,"name":"online","context":{"idset":"9297,9304"}}
+{"timestamp":1712188283.5109684,"name":"online","context":{"idset":"9280,9290,9294,9303"}}
+{"timestamp":1712188283.6549072,"name":"online","context":{"idset":"9286,9315"}}
+{"timestamp":1712188283.854593,"name":"online","context":{"idset":"9311,9320"}}
+{"timestamp":1712188286.1251051,"name":"online","context":{"idset":"9321"}}
+{"timestamp":1712188290.9552331,"name":"online","context":{"idset":"9325"}}
+{"timestamp":1712188291.0916641,"name":"online","context":{"idset":"9323"}}
+{"timestamp":1712188291.1863623,"name":"online","context":{"idset":"9322"}}
+{"timestamp":1712188291.4295161,"name":"online","context":{"idset":"9324,9328"}}
+{"timestamp":1712188291.5991406,"name":"online","context":{"idset":"9973"}}
+{"timestamp":1712188291.6524332,"name":"online","context":{"idset":"9329"}}
+{"timestamp":1712188291.767215,"name":"online","context":{"idset":"9975"}}
+{"timestamp":1712188291.7880857,"name":"online","context":{"idset":"9326,9982"}}
+{"timestamp":1712188291.8647182,"name":"online","context":{"idset":"9327,9331"}}
+{"timestamp":1712188291.9687111,"name":"online","context":{"idset":"9330,9974"}}
+{"timestamp":1712188292.0448229,"name":"online","context":{"idset":"9977"}}
+{"timestamp":1712188292.1571753,"name":"online","context":{"idset":"9978"}}
+{"timestamp":1712188292.2347517,"name":"online","context":{"idset":"9976"}}
+{"timestamp":1712188292.2636445,"name":"online","context":{"idset":"9981"}}
+{"timestamp":1712188292.2844429,"name":"online","context":{"idset":"9983"}}
+{"timestamp":1712188292.3040955,"name":"online","context":{"idset":"9979"}}
+{"timestamp":1712188292.4900885,"name":"online","context":{"idset":"9980,9985,9990-9991,9995"}}
+{"timestamp":1712188292.5580604,"name":"online","context":{"idset":"9332"}}
+{"timestamp":1712188292.6037247,"name":"online","context":{"idset":"9993"}}
+{"timestamp":1712188292.7329636,"name":"online","context":{"idset":"9984"}}
+{"timestamp":1712188292.7751386,"name":"online","context":{"idset":"9986-9987"}}
+{"timestamp":1712188293.0357637,"name":"online","context":{"idset":"9988"}}
+{"timestamp":1712188293.2348273,"name":"online","context":{"idset":"9994,10001"}}
+{"timestamp":1712188293.259079,"name":"online","context":{"idset":"9996"}}
+{"timestamp":1712188293.3011012,"name":"online","context":{"idset":"9999,10003"}}
+{"timestamp":1712188293.3533685,"name":"online","context":{"idset":"9997,10000"}}
+{"timestamp":1712188293.4714539,"name":"online","context":{"idset":"9992"}}
+{"timestamp":1712188293.6349854,"name":"online","context":{"idset":"10002"}}
+{"timestamp":1712188293.7419884,"name":"online","context":{"idset":"10005"}}
+{"timestamp":1712188293.8029866,"name":"online","context":{"idset":"10023"}}
+{"timestamp":1712188293.8308337,"name":"online","context":{"idset":"10009"}}
+{"timestamp":1712188293.8746281,"name":"online","context":{"idset":"10006"}}
+{"timestamp":1712188293.9662175,"name":"online","context":{"idset":"10014"}}
+{"timestamp":1712188293.9888794,"name":"online","context":{"idset":"10008,10022"}}
+{"timestamp":1712188294.0368044,"name":"online","context":{"idset":"10029"}}
+{"timestamp":1712188294.0737195,"name":"online","context":{"idset":"10010"}}
+{"timestamp":1712188294.1041751,"name":"online","context":{"idset":"9998"}}
+{"timestamp":1712188294.2061172,"name":"online","context":{"idset":"10011-10012,10016,10019"}}
+{"timestamp":1712188294.2760522,"name":"online","context":{"idset":"10007,10018,10021"}}
+{"timestamp":1712188294.3406458,"name":"online","context":{"idset":"10004,10013,10025"}}
+{"timestamp":1712188294.3784032,"name":"online","context":{"idset":"10015,10024"}}
+{"timestamp":1712188294.464155,"name":"online","context":{"idset":"10033"}}
+{"timestamp":1712188294.484221,"name":"online","context":{"idset":"10028"}}
+{"timestamp":1712188294.5336444,"name":"online","context":{"idset":"10017,10026"}}
+{"timestamp":1712188294.6728888,"name":"online","context":{"idset":"10027,10039"}}
+{"timestamp":1712188294.7849777,"name":"online","context":{"idset":"10030-10032,10036,10038,10044"}}
+{"timestamp":1712188294.9464779,"name":"online","context":{"idset":"10035,10040"}}
+{"timestamp":1712188295.0574379,"name":"online","context":{"idset":"10020"}}
+{"timestamp":1712188295.4750037,"name":"online","context":{"idset":"10034,10041,10043,10054,10056"}}
+{"timestamp":1712188295.5757194,"name":"online","context":{"idset":"10042,10045"}}
+{"timestamp":1712188295.7380714,"name":"online","context":{"idset":"10046,10049,10088"}}
+{"timestamp":1712188295.8884077,"name":"online","context":{"idset":"10048,10052-10053,10087"}}
+{"timestamp":1712188295.9077289,"name":"online","context":{"idset":"10086"}}
+{"timestamp":1712188296.0172143,"name":"online","context":{"idset":"10051,10057,10060,10066"}}
+{"timestamp":1712188296.1174021,"name":"online","context":{"idset":"10050,10065,10067"}}
+{"timestamp":1712188296.2461829,"name":"online","context":{"idset":"10055,10061-10062,10068,10075,10077,10084"}}
+{"timestamp":1712188296.3086128,"name":"online","context":{"idset":"10058,10071,10074,10089"}}
+{"timestamp":1712188296.4438548,"name":"online","context":{"idset":"10072-10073,10081"}}
+{"timestamp":1712188296.5559137,"name":"online","context":{"idset":"10047,10079-10080,10085"}}
+{"timestamp":1712188296.67485,"name":"online","context":{"idset":"10063,10069-10070,10078,10083"}}
+{"timestamp":1712188296.7880492,"name":"online","context":{"idset":"10076,10082"}}
+{"timestamp":1712188297.5376389,"name":"online","context":{"idset":"10090"}}
+{"timestamp":1712188298.1362429,"name":"online","context":{"idset":"10091-10092"}}
+{"timestamp":1712188298.6269376,"name":"online","context":{"idset":"10093"}}
+{"timestamp":1712188302.8205669,"name":"online","context":{"idset":"10094"}}
+{"timestamp":1712188303.1471448,"name":"online","context":{"idset":"10096"}}
+{"timestamp":1712188303.3968074,"name":"online","context":{"idset":"10099"}}
+{"timestamp":1712188303.4724069,"name":"online","context":{"idset":"10095"}}
+{"timestamp":1712188303.5284584,"name":"online","context":{"idset":"10098,10103"}}
+{"timestamp":1712188303.6444552,"name":"online","context":{"idset":"10101-10102"}}
+{"timestamp":1712188303.8084955,"name":"online","context":{"idset":"10106"}}
+{"timestamp":1712188303.831913,"name":"online","context":{"idset":"10097"}}
+{"timestamp":1712188303.8557765,"name":"online","context":{"idset":"10105"}}
+{"timestamp":1712188304.0667136,"name":"online","context":{"idset":"10104"}}
+{"timestamp":1712188304.1353226,"name":"online","context":{"idset":"10100"}}
+{"timestamp":1712188304.5886819,"name":"online","context":{"idset":"10112"}}
+{"timestamp":1712188304.655076,"name":"online","context":{"idset":"10110"}}
+{"timestamp":1712188304.8074031,"name":"online","context":{"idset":"10118"}}
+{"timestamp":1712188304.9335966,"name":"online","context":{"idset":"10107-10109,10117,10119"}}
+{"timestamp":1712188304.9570301,"name":"online","context":{"idset":"10114"}}
+{"timestamp":1712188305.0640082,"name":"online","context":{"idset":"10116,10120"}}
+{"timestamp":1712188305.1959407,"name":"online","context":{"idset":"10113,10115,10129"}}
+{"timestamp":1712188305.3386343,"name":"online","context":{"idset":"10124"}}
+{"timestamp":1712188305.4119811,"name":"online","context":{"idset":"10127-10128"}}
+{"timestamp":1712188305.5179489,"name":"online","context":{"idset":"10122"}}
+{"timestamp":1712188305.607995,"name":"online","context":{"idset":"10125-10126,10141"}}
+{"timestamp":1712188305.7119274,"name":"online","context":{"idset":"10134"}}
+{"timestamp":1712188305.7836838,"name":"online","context":{"idset":"10123"}}
+{"timestamp":1712188305.8147638,"name":"online","context":{"idset":"10130"}}
+{"timestamp":1712188305.8340418,"name":"online","context":{"idset":"10121,10144"}}
+{"timestamp":1712188305.8868377,"name":"online","context":{"idset":"10139,10142"}}
+{"timestamp":1712188306.000035,"name":"online","context":{"idset":"10111,10131-10132,10137"}}
+{"timestamp":1712188306.0827298,"name":"online","context":{"idset":"10133"}}
+{"timestamp":1712188306.123733,"name":"online","context":{"idset":"10140"}}
+{"timestamp":1712188306.1483929,"name":"online","context":{"idset":"10151"}}
+{"timestamp":1712188306.207175,"name":"online","context":{"idset":"10146"}}
+{"timestamp":1712188306.3191593,"name":"online","context":{"idset":"10155,10160"}}
+{"timestamp":1712188306.383321,"name":"online","context":{"idset":"10156"}}
+{"timestamp":1712188306.4720917,"name":"online","context":{"idset":"10157"}}
+{"timestamp":1712188306.4916234,"name":"online","context":{"idset":"10152"}}
+{"timestamp":1712188306.5285199,"name":"online","context":{"idset":"10150,10163"}}
+{"timestamp":1712188306.5678163,"name":"online","context":{"idset":"10158,10161"}}
+{"timestamp":1712188306.629019,"name":"online","context":{"idset":"10169"}}
+{"timestamp":1712188306.6891606,"name":"online","context":{"idset":"10159,10165-10166"}}
+{"timestamp":1712188306.7999773,"name":"online","context":{"idset":"10168,10172"}}
+{"timestamp":1712188307.0345981,"name":"online","context":{"idset":"10164,10167,10171,10175-10176,10179"}}
+{"timestamp":1712188307.0618978,"name":"online","context":{"idset":"10170"}}
+{"timestamp":1712188307.1028473,"name":"online","context":{"idset":"10149,10174,10178"}}
+{"timestamp":1712188307.2636721,"name":"online","context":{"idset":"10173,10177,10181"}}
+{"timestamp":1712188307.3925564,"name":"online","context":{"idset":"10184"}}
+{"timestamp":1712188307.5083556,"name":"online","context":{"idset":"10180,10188"}}
+{"timestamp":1712188307.6213641,"name":"online","context":{"idset":"10182,10192"}}
+{"timestamp":1712188307.8378022,"name":"online","context":{"idset":"10185,10193,10211,10213"}}
+{"timestamp":1712188307.9279389,"name":"online","context":{"idset":"10190,10196-10197"}}
+{"timestamp":1712188308.0311072,"name":"online","context":{"idset":"10187,10200,10209"}}
+{"timestamp":1712188308.1600654,"name":"online","context":{"idset":"10191,10195,10202,10215"}}
+{"timestamp":1712188308.2750783,"name":"online","context":{"idset":"10186,10189,10194,10201,10212,10216"}}
+{"timestamp":1712188308.4415882,"name":"online","context":{"idset":"10198,10217,10225"}}
+{"timestamp":1712188308.5593278,"name":"online","context":{"idset":"10203,10214,10230"}}
+{"timestamp":1712188308.6823852,"name":"online","context":{"idset":"10204,10210,10220"}}
+{"timestamp":1712188308.7003436,"name":"online","context":{"idset":"10221,10224"}}
+{"timestamp":1712188308.8122082,"name":"online","context":{"idset":"10199,10208,10226-10227"}}
+{"timestamp":1712188309.0170748,"name":"online","context":{"idset":"10218,10222"}}
+{"timestamp":1712188309.1861632,"name":"online","context":{"idset":"10223,10229"}}
+{"timestamp":1712188309.3312604,"name":"online","context":{"idset":"10228,10231"}}
+{"timestamp":1712188309.5703242,"name":"online","context":{"idset":"10219"}}
+{"timestamp":1712188310.1237946,"name":"online","context":{"idset":"10233-10234"}}
+{"timestamp":1712188310.649713,"name":"online","context":{"idset":"10232"}}
+{"timestamp":1712188310.7621779,"name":"online","context":{"idset":"10235"}}
+{"timestamp":1712188314.7296829,"name":"online","context":{"idset":"10236"}}
+{"timestamp":1712188315.0344794,"name":"online","context":{"idset":"10238"}}
+{"timestamp":1712188315.1218972,"name":"online","context":{"idset":"10237"}}
+{"timestamp":1712188315.5191321,"name":"online","context":{"idset":"10239,10241,10245-10246"}}
+{"timestamp":1712188315.5567904,"name":"online","context":{"idset":"10240,10242-10243"}}
+{"timestamp":1712188315.5901785,"name":"online","context":{"idset":"10244"}}
+{"timestamp":1712188315.9561386,"name":"online","context":{"idset":"10247"}}
+{"timestamp":1712188316.2168036,"name":"online","context":{"idset":"10248"}}
+{"timestamp":1712188316.2845273,"name":"online","context":{"idset":"10249"}}
+{"timestamp":1712188316.504813,"name":"online","context":{"idset":"10253"}}
+{"timestamp":1712188316.5387421,"name":"online","context":{"idset":"10252"}}
+{"timestamp":1712188316.5963004,"name":"online","context":{"idset":"10254"}}
+{"timestamp":1712188316.7488055,"name":"online","context":{"idset":"10263"}}
+{"timestamp":1712188316.8270903,"name":"online","context":{"idset":"10257,10262"}}
+{"timestamp":1712188316.8468244,"name":"online","context":{"idset":"10251"}}
+{"timestamp":1712188316.9195602,"name":"online","context":{"idset":"10256,10259,10265"}}
+{"timestamp":1712188317.0114107,"name":"online","context":{"idset":"10250,10258"}}
+{"timestamp":1712188317.1449125,"name":"online","context":{"idset":"10267"}}
+{"timestamp":1712188317.2116294,"name":"online","context":{"idset":"10260"}}
+{"timestamp":1712188317.3325899,"name":"online","context":{"idset":"10264"}}
+{"timestamp":1712188317.3363838,"name":"online","context":{"idset":"10270"}}
+{"timestamp":1712188317.3709319,"name":"online","context":{"idset":"10266,10268"}}
+{"timestamp":1712188317.5186274,"name":"online","context":{"idset":"10269,10275"}}
+{"timestamp":1712188317.5950634,"name":"online","context":{"idset":"10271"}}
+{"timestamp":1712188317.7324374,"name":"online","context":{"idset":"10255,10276,10279"}}
+{"timestamp":1712188317.8362889,"name":"online","context":{"idset":"10272-10274,10277-10278,10280"}}
+{"timestamp":1712188317.8871455,"name":"online","context":{"idset":"10281"}}
+{"timestamp":1712188317.9094486,"name":"online","context":{"idset":"10283"}}
+{"timestamp":1712188318.0784454,"name":"online","context":{"idset":"10282"}}
+{"timestamp":1712188318.1454668,"name":"online","context":{"idset":"10284"}}
+{"timestamp":1712188318.1934571,"name":"online","context":{"idset":"10306"}}
+{"timestamp":1712188318.3201146,"name":"online","context":{"idset":"10285-10286,10289"}}
+{"timestamp":1712188318.4385936,"name":"online","context":{"idset":"10291-10292,10296"}}
+{"timestamp":1712188318.5073838,"name":"online","context":{"idset":"10287-10288"}}
+{"timestamp":1712188318.5348749,"name":"online","context":{"idset":"10295,10312"}}
+{"timestamp":1712188318.6100664,"name":"online","context":{"idset":"10309"}}
+{"timestamp":1712188318.6646404,"name":"online","context":{"idset":"10293,10297"}}
+{"timestamp":1712188318.7164102,"name":"online","context":{"idset":"10305"}}
+{"timestamp":1712188318.7982152,"name":"online","context":{"idset":"10298,10303,10308,10314"}}
+{"timestamp":1712188318.8277457,"name":"online","context":{"idset":"10311"}}
+{"timestamp":1712188318.8959143,"name":"online","context":{"idset":"10299,10310"}}
+{"timestamp":1712188319.0143666,"name":"online","context":{"idset":"10294,10313"}}
+{"timestamp":1712188319.1653247,"name":"online","context":{"idset":"10300,10315-10316,10320"}}
+{"timestamp":1712188319.2716823,"name":"online","context":{"idset":"10290"}}
+{"timestamp":1712188319.4563019,"name":"online","context":{"idset":"10318,10328"}}
+{"timestamp":1712188319.5750496,"name":"online","context":{"idset":"10317,10319,10321-10322,10327"}}
+{"timestamp":1712188319.7064211,"name":"online","context":{"idset":"10307,10323,10326"}}
+{"timestamp":1712188319.8160937,"name":"online","context":{"idset":"10329,10331"}}
+{"timestamp":1712188320.1648355,"name":"online","context":{"idset":"10332-10334,10337"}}
+{"timestamp":1712188320.2817736,"name":"online","context":{"idset":"10330,10338,10342-10344,10346"}}
+{"timestamp":1712188320.4337814,"name":"online","context":{"idset":"10324-10325,10335-10336,10340-10341,10351"}}
+{"timestamp":1712188320.6651595,"name":"online","context":{"idset":"10339,10348"}}
+{"timestamp":1712188320.806555,"name":"online","context":{"idset":"10345,10349,10357-10358,10360"}}
+{"timestamp":1712188320.9299142,"name":"online","context":{"idset":"10347,10350,10353-10354,10356,10359,10362"}}
+{"timestamp":1712188321.0657413,"name":"online","context":{"idset":"10352,10361"}}
+{"timestamp":1712188321.4070847,"name":"online","context":{"idset":"10355,10363"}}
+{"timestamp":1712188321.6585524,"name":"online","context":{"idset":"10366"}}
+{"timestamp":1712188321.8796611,"name":"online","context":{"idset":"10364"}}
+{"timestamp":1712188322.2119536,"name":"online","context":{"idset":"10365"}}
+{"timestamp":1712188322.8318877,"name":"online","context":{"idset":"10367"}}
+{"timestamp":1712188326.6721873,"name":"online","context":{"idset":"10368-10369"}}
+{"timestamp":1712188326.8486331,"name":"online","context":{"idset":"10370"}}
+{"timestamp":1712188327.179029,"name":"online","context":{"idset":"10371-10372"}}
+{"timestamp":1712188327.3607848,"name":"online","context":{"idset":"10373-10374"}}
+{"timestamp":1712188327.573051,"name":"online","context":{"idset":"10375"}}
+{"timestamp":1712188327.8467777,"name":"online","context":{"idset":"10378"}}
+{"timestamp":1712188327.9604955,"name":"online","context":{"idset":"10377"}}
+{"timestamp":1712188328.1325195,"name":"online","context":{"idset":"10376"}}
+{"timestamp":1712188328.4536114,"name":"online","context":{"idset":"10379"}}
+{"timestamp":1712188328.9793463,"name":"online","context":{"idset":"10381"}}
+{"timestamp":1712188329.0125456,"name":"online","context":{"idset":"10386"}}
+{"timestamp":1712188329.0790067,"name":"online","context":{"idset":"10385,10389"}}
+{"timestamp":1712188329.2726796,"name":"online","context":{"idset":"10384,10387"}}
+{"timestamp":1712188329.4103415,"name":"online","context":{"idset":"10394"}}
+{"timestamp":1712188329.4698102,"name":"online","context":{"idset":"10380"}}
+{"timestamp":1712188329.550647,"name":"online","context":{"idset":"10391,10395,10397"}}
+{"timestamp":1712188329.7524283,"name":"online","context":{"idset":"10383,10388,10390"}}
+{"timestamp":1712188329.8619311,"name":"online","context":{"idset":"10392,10406"}}
+{"timestamp":1712188330.0883408,"name":"online","context":{"idset":"10402,10405"}}
+{"timestamp":1712188330.1268876,"name":"online","context":{"idset":"10382,10396"}}
+{"timestamp":1712188330.1565526,"name":"online","context":{"idset":"10393"}}
+{"timestamp":1712188330.2991209,"name":"online","context":{"idset":"10401,10410"}}
+{"timestamp":1712188330.3568201,"name":"online","context":{"idset":"10399"}}
+{"timestamp":1712188330.4120312,"name":"online","context":{"idset":"10398,10414"}}
+{"timestamp":1712188330.4470189,"name":"online","context":{"idset":"10407-10408"}}
+{"timestamp":1712188330.6329467,"name":"online","context":{"idset":"10404,10411-10412"}}
+{"timestamp":1712188330.9069068,"name":"online","context":{"idset":"10400,10419"}}
+{"timestamp":1712188330.9363062,"name":"online","context":{"idset":"10416"}}
+{"timestamp":1712188331.0013156,"name":"online","context":{"idset":"10413"}}
+{"timestamp":1712188331.0524547,"name":"online","context":{"idset":"10417,10421"}}
+{"timestamp":1712188331.0882854,"name":"online","context":{"idset":"10403"}}
+{"timestamp":1712188331.1404722,"name":"online","context":{"idset":"10420,10422"}}
+{"timestamp":1712188331.3343377,"name":"online","context":{"idset":"10409,10426"}}
+{"timestamp":1712188331.3934197,"name":"online","context":{"idset":"10415"}}
+{"timestamp":1712188331.5390031,"name":"online","context":{"idset":"10423,10444"}}
+{"timestamp":1712188331.7607739,"name":"online","context":{"idset":"10425,10429"}}
+{"timestamp":1712188331.8852746,"name":"online","context":{"idset":"10424,10431-10434"}}
+{"timestamp":1712188331.9163496,"name":"online","context":{"idset":"10428,10452"}}
+{"timestamp":1712188331.9848809,"name":"online","context":{"idset":"10443"}}
+{"timestamp":1712188332.0719998,"name":"online","context":{"idset":"10418,10430,10438"}}
+{"timestamp":1712188332.2318857,"name":"online","context":{"idset":"10427,10449,10453"}}
+{"timestamp":1712188332.3663483,"name":"online","context":{"idset":"10439,10473"}}
+{"timestamp":1712188332.4854147,"name":"online","context":{"idset":"10441,10451,10454"}}
+{"timestamp":1712188332.5894561,"name":"online","context":{"idset":"10436-10437,10459,10489"}}
+{"timestamp":1712188332.7128575,"name":"online","context":{"idset":"10448,10450,10455,10462"}}
+{"timestamp":1712188332.8323612,"name":"online","context":{"idset":"10446,10461,10474"}}
+{"timestamp":1712188332.9990077,"name":"online","context":{"idset":"10435,10440,10442,10447,10457,10464,10468,10477,10483,10486-10487,10493"}}
+{"timestamp":1712188333.1418078,"name":"online","context":{"idset":"10445,10465-10466,10471,10476,10481"}}
+{"timestamp":1712188333.2665684,"name":"online","context":{"idset":"10456,10460,10485"}}
+{"timestamp":1712188333.418252,"name":"online","context":{"idset":"10458,10463,10472,10491,10494,10496"}}
+{"timestamp":1712188333.5511329,"name":"online","context":{"idset":"10490"}}
+{"timestamp":1712188333.6671543,"name":"online","context":{"idset":"10470,10488,10500-10501"}}
+{"timestamp":1712188333.7888982,"name":"online","context":{"idset":"10492,10495,10497"}}
+{"timestamp":1712188334.1988435,"name":"online","context":{"idset":"10467,10498-10499"}}
+{"timestamp":1712188334.7004652,"name":"online","context":{"idset":"10502"}}
+{"timestamp":1712188338.0268202,"name":"online","context":{"idset":"10505"}}
+{"timestamp":1712188338.1456852,"name":"online","context":{"idset":"10503"}}
+{"timestamp":1712188338.7207351,"name":"online","context":{"idset":"10506"}}
+{"timestamp":1712188338.8992352,"name":"online","context":{"idset":"10507-10508,10511"}}
+{"timestamp":1712188339.5918868,"name":"online","context":{"idset":"10516"}}
+{"timestamp":1712188339.6322858,"name":"online","context":{"idset":"10512"}}
+{"timestamp":1712188339.7544537,"name":"online","context":{"idset":"10514"}}
+{"timestamp":1712188339.9108477,"name":"online","context":{"idset":"10515"}}
+{"timestamp":1712188340.0072277,"name":"online","context":{"idset":"10513"}}
+{"timestamp":1712188340.2383683,"name":"online","context":{"idset":"10520"}}
+{"timestamp":1712188340.7910798,"name":"online","context":{"idset":"10525"}}
+{"timestamp":1712188340.8710725,"name":"online","context":{"idset":"10528"}}
+{"timestamp":1712188341.0328062,"name":"online","context":{"idset":"10523"}}
+{"timestamp":1712188341.1300356,"name":"online","context":{"idset":"10530,10533"}}
+{"timestamp":1712188341.1772971,"name":"online","context":{"idset":"10536"}}
+{"timestamp":1712188341.2269473,"name":"online","context":{"idset":"10529"}}
+{"timestamp":1712188341.3876405,"name":"online","context":{"idset":"10531"}}
+{"timestamp":1712188341.5126004,"name":"online","context":{"idset":"10532,10535,10537"}}
+{"timestamp":1712188341.6598635,"name":"online","context":{"idset":"10526,10539"}}
+{"timestamp":1712188341.7812572,"name":"online","context":{"idset":"10524,10527,10542"}}
+{"timestamp":1712188341.8231025,"name":"online","context":{"idset":"10543"}}
+{"timestamp":1712188341.8588068,"name":"online","context":{"idset":"10540"}}
+{"timestamp":1712188342.0067601,"name":"online","context":{"idset":"10550"}}
+{"timestamp":1712188342.1164041,"name":"online","context":{"idset":"10534,10538,10547"}}
+{"timestamp":1712188342.1558053,"name":"online","context":{"idset":"10552"}}
+{"timestamp":1712188342.2165725,"name":"online","context":{"idset":"10544-10546"}}
+{"timestamp":1712188342.3390095,"name":"online","context":{"idset":"10551"}}
+{"timestamp":1712188342.4878771,"name":"online","context":{"idset":"10549"}}
+{"timestamp":1712188342.5783045,"name":"online","context":{"idset":"10553-10554"}}
+{"timestamp":1712188342.6413016,"name":"online","context":{"idset":"10555-10556,10559"}}
+{"timestamp":1712188342.7232356,"name":"online","context":{"idset":"10560"}}
+{"timestamp":1712188342.8525131,"name":"online","context":{"idset":"10548,10562"}}
+{"timestamp":1712188342.9806533,"name":"online","context":{"idset":"10563"}}
+{"timestamp":1712188343.0359383,"name":"online","context":{"idset":"10561"}}
+{"timestamp":1712188343.0559239,"name":"online","context":{"idset":"10565"}}
+{"timestamp":1712188343.1627541,"name":"online","context":{"idset":"10568"}}
+{"timestamp":1712188343.1948767,"name":"online","context":{"idset":"10564"}}
+{"timestamp":1712188343.2641144,"name":"online","context":{"idset":"10566-10567"}}
+{"timestamp":1712188343.3971207,"name":"online","context":{"idset":"10569"}}
+{"timestamp":1712188343.5154459,"name":"online","context":{"idset":"10577"}}
+{"timestamp":1712188343.5807712,"name":"online","context":{"idset":"10572"}}
+{"timestamp":1712188343.6214609,"name":"online","context":{"idset":"10571"}}
+{"timestamp":1712188343.7390423,"name":"online","context":{"idset":"10570,10573"}}
+{"timestamp":1712188343.8944507,"name":"online","context":{"idset":"10575-10576,10587"}}
+{"timestamp":1712188343.9462032,"name":"online","context":{"idset":"10579"}}
+{"timestamp":1712188344.0846014,"name":"online","context":{"idset":"10574,10578,10580-10581"}}
+{"timestamp":1712188344.3151453,"name":"online","context":{"idset":"10586"}}
+{"timestamp":1712188344.358866,"name":"online","context":{"idset":"10585"}}
+{"timestamp":1712188344.5339031,"name":"online","context":{"idset":"10584,10588-10589,10591"}}
+{"timestamp":1712188344.6752877,"name":"online","context":{"idset":"10583,10597"}}
+{"timestamp":1712188344.7911713,"name":"online","context":{"idset":"10594-10595,10598"}}
+{"timestamp":1712188344.8970776,"name":"online","context":{"idset":"10590,10592,10596,10600,10606"}}
+{"timestamp":1712188345.0325716,"name":"online","context":{"idset":"10601,10604,10616"}}
+{"timestamp":1712188345.1827681,"name":"online","context":{"idset":"10593,10605,10612,10631,10641"}}
+{"timestamp":1712188345.3187156,"name":"online","context":{"idset":"10599,10607,10609,10613,10621"}}
+{"timestamp":1712188345.4821162,"name":"online","context":{"idset":"10610,10620,10622,10630,10640,10645"}}
+{"timestamp":1712188345.6155353,"name":"online","context":{"idset":"10611,10614,10617-10619,10635-10636"}}
+{"timestamp":1712188345.7511435,"name":"online","context":{"idset":"10608,10623,10626,10648"}}
+{"timestamp":1712188345.8537612,"name":"online","context":{"idset":"10615,10624-10625,10627-10628,10632"}}
+{"timestamp":1712188346.1432121,"name":"online","context":{"idset":"10603,10649"}}
+{"timestamp":1712188346.2878854,"name":"online","context":{"idset":"10633,10646"}}
+{"timestamp":1712188346.4672008,"name":"online","context":{"idset":"10647"}}
+{"timestamp":1712188347.0567286,"name":"online","context":{"idset":"10650"}}
+{"timestamp":1712188348.0785439,"name":"online","context":{"idset":"10651"}}
+{"timestamp":1712188349.9761755,"name":"online","context":{"idset":"10652"}}
+{"timestamp":1712188350.1276605,"name":"online","context":{"idset":"10653"}}
+{"timestamp":1712188350.8296049,"name":"online","context":{"idset":"10656,10658"}}
+{"timestamp":1712188350.8634045,"name":"online","context":{"idset":"10654"}}
+{"timestamp":1712188350.9708552,"name":"online","context":{"idset":"10655,10659"}}
+{"timestamp":1712188351.4071176,"name":"online","context":{"idset":"10660"}}
+{"timestamp":1712188351.5017571,"name":"online","context":{"idset":"10661"}}
+{"timestamp":1712188351.7226036,"name":"online","context":{"idset":"10663"}}
+{"timestamp":1712188351.8917441,"name":"online","context":{"idset":"10662,10664"}}
+{"timestamp":1712188352.4966102,"name":"online","context":{"idset":"10665"}}
+{"timestamp":1712188352.9073567,"name":"online","context":{"idset":"10671"}}
+{"timestamp":1712188353.0145137,"name":"online","context":{"idset":"10667"}}
+{"timestamp":1712188353.2333705,"name":"online","context":{"idset":"10676,10678"}}
+{"timestamp":1712188353.3868954,"name":"online","context":{"idset":"10666,10672,10674,10677,10680"}}
+{"timestamp":1712188353.4639702,"name":"online","context":{"idset":"10670,10673"}}
+{"timestamp":1712188353.5018208,"name":"online","context":{"idset":"10675,10679"}}
+{"timestamp":1712188353.6111314,"name":"online","context":{"idset":"10668"}}
+{"timestamp":1712188353.6858656,"name":"online","context":{"idset":"10669,10681"}}
+{"timestamp":1712188353.8427911,"name":"online","context":{"idset":"10686,10689"}}
+{"timestamp":1712188353.9654088,"name":"online","context":{"idset":"10682"}}
+{"timestamp":1712188354.0923634,"name":"online","context":{"idset":"10695"}}
+{"timestamp":1712188354.1325841,"name":"online","context":{"idset":"10687,10690-10691,10696"}}
+{"timestamp":1712188354.2124906,"name":"online","context":{"idset":"10692"}}
+{"timestamp":1712188354.4183137,"name":"online","context":{"idset":"10685,10688,10697"}}
+{"timestamp":1712188354.455956,"name":"online","context":{"idset":"10698"}}
+{"timestamp":1712188354.5111341,"name":"online","context":{"idset":"10704"}}
+{"timestamp":1712188354.8332534,"name":"online","context":{"idset":"10700,10709"}}
+{"timestamp":1712188354.9015622,"name":"online","context":{"idset":"10699,10705,10710"}}
+{"timestamp":1712188355.1382558,"name":"online","context":{"idset":"10701,10706-10708,10715"}}
+{"timestamp":1712188355.3223712,"name":"online","context":{"idset":"10703,10711,10716"}}
+{"timestamp":1712188355.3966672,"name":"online","context":{"idset":"10714"}}
+{"timestamp":1712188355.4642227,"name":"online","context":{"idset":"10722"}}
+{"timestamp":1712188355.5039763,"name":"online","context":{"idset":"10727"}}
+{"timestamp":1712188355.7985461,"name":"online","context":{"idset":"10713,10717,10719,10723"}}
+{"timestamp":1712188355.8389294,"name":"online","context":{"idset":"10724"}}
+{"timestamp":1712188355.9080615,"name":"online","context":{"idset":"10725"}}
+{"timestamp":1712188356.0876784,"name":"online","context":{"idset":"10720,10729"}}
+{"timestamp":1712188356.2826612,"name":"online","context":{"idset":"10721,10730,10737"}}
+{"timestamp":1712188356.3595426,"name":"online","context":{"idset":"10733"}}
+{"timestamp":1712188356.44859,"name":"online","context":{"idset":"10732"}}
+{"timestamp":1712188356.4732366,"name":"online","context":{"idset":"10736"}}
+{"timestamp":1712188356.5202329,"name":"online","context":{"idset":"10741"}}
+{"timestamp":1712188356.6688361,"name":"online","context":{"idset":"10731,10734,10738,10742,10744,10746-10747"}}
+{"timestamp":1712188356.8013301,"name":"online","context":{"idset":"10726,10735,10748"}}
+{"timestamp":1712188356.8423116,"name":"online","context":{"idset":"10749"}}
+{"timestamp":1712188356.9210835,"name":"online","context":{"idset":"10751"}}
+{"timestamp":1712188357.0547442,"name":"online","context":{"idset":"10728,10758"}}
+{"timestamp":1712188357.2337503,"name":"online","context":{"idset":"10750,10760"}}
+{"timestamp":1712188357.3684704,"name":"online","context":{"idset":"10743,10745,10752,10754"}}
+{"timestamp":1712188357.4586349,"name":"online","context":{"idset":"10762"}}
+{"timestamp":1712188357.6497252,"name":"online","context":{"idset":"10756,10759"}}
+{"timestamp":1712188357.7511551,"name":"online","context":{"idset":"10753,10755,10769"}}
+{"timestamp":1712188357.8780944,"name":"online","context":{"idset":"10757,10766,10770"}}
+{"timestamp":1712188358.0335882,"name":"online","context":{"idset":"10768,10775"}}
+{"timestamp":1712188358.1667793,"name":"online","context":{"idset":"10771-10772,10785"}}
+{"timestamp":1712188358.3087144,"name":"online","context":{"idset":"10763,10780"}}
+{"timestamp":1712188358.4515765,"name":"online","context":{"idset":"10767,10773-10774,10779,10782"}}
+{"timestamp":1712188358.5924382,"name":"online","context":{"idset":"10761,10765,10777,10783-10784"}}
+{"timestamp":1712188358.6991606,"name":"online","context":{"idset":"10764,10776,10778,10781"}}
+{"timestamp":1712188358.8302844,"name":"online","context":{"idset":"10788"}}
+{"timestamp":1712188359.0514498,"name":"online","context":{"idset":"10786"}}
+{"timestamp":1712188359.2140565,"name":"online","context":{"idset":"10787"}}
+{"timestamp":1712188359.7276092,"name":"online","context":{"idset":"10789"}}
+{"timestamp":1712188361.7233524,"name":"online","context":{"idset":"10790"}}
+{"timestamp":1712188361.9646332,"name":"online","context":{"idset":"10791"}}
+{"timestamp":1712188362.6996298,"name":"online","context":{"idset":"10794"}}
+{"timestamp":1712188362.8934295,"name":"online","context":{"idset":"10793,10795-10796"}}
+{"timestamp":1712188363.2192035,"name":"online","context":{"idset":"10792"}}
+{"timestamp":1712188363.5074191,"name":"online","context":{"idset":"10799-10800"}}
+{"timestamp":1712188363.5535147,"name":"online","context":{"idset":"10798"}}
+{"timestamp":1712188363.9858913,"name":"online","context":{"idset":"10797"}}
+{"timestamp":1712188364.054569,"name":"online","context":{"idset":"10801"}}
+{"timestamp":1712188364.349134,"name":"online","context":{"idset":"10802"}}
+{"timestamp":1712188364.6828039,"name":"online","context":{"idset":"10804"}}
+{"timestamp":1712188364.8017755,"name":"online","context":{"idset":"10803,10808"}}
+{"timestamp":1712188365.0218465,"name":"online","context":{"idset":"10811"}}
+{"timestamp":1712188365.0517106,"name":"online","context":{"idset":"10806"}}
+{"timestamp":1712188365.2224576,"name":"online","context":{"idset":"10805,10807,10810"}}
+{"timestamp":1712188365.4231181,"name":"online","context":{"idset":"10813"}}
+{"timestamp":1712188365.5083861,"name":"online","context":{"idset":"10814"}}
+{"timestamp":1712188365.6673851,"name":"online","context":{"idset":"10809,10817-10818,10826,10828"}}
+{"timestamp":1712188365.7773633,"name":"online","context":{"idset":"10815-10816"}}
+{"timestamp":1712188365.8194685,"name":"online","context":{"idset":"10819,10823-10824"}}
+{"timestamp":1712188365.9384358,"name":"online","context":{"idset":"10812,10827"}}
+{"timestamp":1712188366.0586035,"name":"online","context":{"idset":"10822"}}
+{"timestamp":1712188366.2132275,"name":"online","context":{"idset":"10820"}}
+{"timestamp":1712188366.3465471,"name":"online","context":{"idset":"10821,10831"}}
+{"timestamp":1712188366.3942616,"name":"online","context":{"idset":"10833"}}
+{"timestamp":1712188366.4301665,"name":"online","context":{"idset":"10825,10829"}}
+{"timestamp":1712188366.5505874,"name":"online","context":{"idset":"10836,10839-10840"}}
+{"timestamp":1712188366.593056,"name":"online","context":{"idset":"10832"}}
+{"timestamp":1712188366.6411538,"name":"online","context":{"idset":"10837"}}
+{"timestamp":1712188366.7045093,"name":"online","context":{"idset":"10830,10834-10835"}}
+{"timestamp":1712188366.80619,"name":"online","context":{"idset":"10838"}}
+{"timestamp":1712188366.9997087,"name":"online","context":{"idset":"10843"}}
+{"timestamp":1712188367.1354427,"name":"online","context":{"idset":"10841,10844,10847"}}
+{"timestamp":1712188367.3092351,"name":"online","context":{"idset":"10846,10849,10853"}}
+{"timestamp":1712188367.4162984,"name":"online","context":{"idset":"10842,10848,10854"}}
+{"timestamp":1712188367.469137,"name":"online","context":{"idset":"10845,10855"}}
+{"timestamp":1712188367.5871904,"name":"online","context":{"idset":"10852"}}
+{"timestamp":1712188367.5945623,"name":"online","context":{"idset":"10850"}}
+{"timestamp":1712188367.6481607,"name":"online","context":{"idset":"10851"}}
+{"timestamp":1712188367.9857509,"name":"online","context":{"idset":"10859"}}
+{"timestamp":1712188368.0880246,"name":"online","context":{"idset":"10861,10864"}}
+{"timestamp":1712188368.1192844,"name":"online","context":{"idset":"10857"}}
+{"timestamp":1712188368.2350235,"name":"online","context":{"idset":"10862,10869"}}
+{"timestamp":1712188368.2720635,"name":"online","context":{"idset":"10865"}}
+{"timestamp":1712188368.3861697,"name":"online","context":{"idset":"10856,10866,10877"}}
+{"timestamp":1712188368.4216826,"name":"online","context":{"idset":"10870,10875"}}
+{"timestamp":1712188368.5480635,"name":"online","context":{"idset":"10858,10860"}}
+{"timestamp":1712188368.6906326,"name":"online","context":{"idset":"10863,10867"}}
+{"timestamp":1712188368.8228137,"name":"online","context":{"idset":"10868,10872,10876,10879"}}
+{"timestamp":1712188368.9580421,"name":"online","context":{"idset":"10878,10881"}}
+{"timestamp":1712188369.1764305,"name":"online","context":{"idset":"10882,10885,10888"}}
+{"timestamp":1712188369.3360717,"name":"online","context":{"idset":"10887,10890"}}
+{"timestamp":1712188369.4546354,"name":"online","context":{"idset":"10880"}}
+{"timestamp":1712188369.6112857,"name":"online","context":{"idset":"10891-10892"}}
+{"timestamp":1712188369.7618806,"name":"online","context":{"idset":"10889,10900"}}
+{"timestamp":1712188369.8879256,"name":"online","context":{"idset":"10893,10901,10903,10909"}}
+{"timestamp":1712188370.0130248,"name":"online","context":{"idset":"10895,10897"}}
+{"timestamp":1712188370.1389837,"name":"online","context":{"idset":"10896,10902,10904"}}
+{"timestamp":1712188370.3323293,"name":"online","context":{"idset":"10898-10899,10910"}}
+{"timestamp":1712188370.4609482,"name":"online","context":{"idset":"10908,10913,10918"}}
+{"timestamp":1712188370.5629263,"name":"online","context":{"idset":"10894,10914-10916,10919"}}
+{"timestamp":1712188370.6637576,"name":"online","context":{"idset":"10917"}}
+{"timestamp":1712188370.8218703,"name":"online","context":{"idset":"10907"}}
+{"timestamp":1712188371.3117054,"name":"online","context":{"idset":"10920-10921"}}
+{"timestamp":1712188371.4922566,"name":"online","context":{"idset":"10923"}}
+{"timestamp":1712188371.6868355,"name":"online","context":{"idset":"10924-10925"}}
+{"timestamp":1712188371.9990745,"name":"online","context":{"idset":"10922"}}
+{"timestamp":1712188373.001904,"name":"online","context":{"idset":"10926"}}
+{"timestamp":1712188373.1434665,"name":"online","context":{"idset":"10927"}}
+{"timestamp":1712188373.5968208,"name":"online","context":{"idset":"10929"}}
+{"timestamp":1712188373.6274035,"name":"online","context":{"idset":"10928"}}
+{"timestamp":1712188374.5299907,"name":"online","context":{"idset":"10931,10933"}}
+{"timestamp":1712188374.6318784,"name":"online","context":{"idset":"10930"}}
+{"timestamp":1712188374.7711701,"name":"online","context":{"idset":"10932"}}
+{"timestamp":1712188375.0082545,"name":"online","context":{"idset":"10939"}}
+{"timestamp":1712188375.1197317,"name":"online","context":{"idset":"10938"}}
+{"timestamp":1712188375.1958504,"name":"online","context":{"idset":"10934,10940"}}
+{"timestamp":1712188375.7897556,"name":"online","context":{"idset":"10942"}}
+{"timestamp":1712188376.0481992,"name":"online","context":{"idset":"10941"}}
+{"timestamp":1712188376.2238884,"name":"online","context":{"idset":"10945"}}
+{"timestamp":1712188376.3923323,"name":"online","context":{"idset":"10946"}}
+{"timestamp":1712188376.4365165,"name":"online","context":{"idset":"10947"}}
+{"timestamp":1712188376.5314219,"name":"online","context":{"idset":"10944"}}
+{"timestamp":1712188376.790194,"name":"online","context":{"idset":"10948,10952"}}
+{"timestamp":1712188377.037858,"name":"online","context":{"idset":"10949,10953"}}
+{"timestamp":1712188377.0811038,"name":"online","context":{"idset":"10954"}}
+{"timestamp":1712188377.1335421,"name":"online","context":{"idset":"10950"}}
+{"timestamp":1712188377.3285029,"name":"online","context":{"idset":"10951,10956,10962"}}
+{"timestamp":1712188377.4062555,"name":"online","context":{"idset":"10959"}}
+{"timestamp":1712188377.6598053,"name":"online","context":{"idset":"10955"}}
+{"timestamp":1712188377.7035377,"name":"online","context":{"idset":"10965"}}
+{"timestamp":1712188377.7425218,"name":"online","context":{"idset":"10957"}}
+{"timestamp":1712188377.8364742,"name":"online","context":{"idset":"10958,10961,10964,10970,10973"}}
+{"timestamp":1712188377.9013786,"name":"online","context":{"idset":"10960,10963,10968,10972"}}
+{"timestamp":1712188378.0090051,"name":"online","context":{"idset":"10971"}}
+{"timestamp":1712188378.1573651,"name":"online","context":{"idset":"10967,10974"}}
+{"timestamp":1712188378.2896898,"name":"online","context":{"idset":"10975"}}
+{"timestamp":1712188378.4198563,"name":"online","context":{"idset":"10977"}}
+{"timestamp":1712188378.5510228,"name":"online","context":{"idset":"10985"}}
+{"timestamp":1712188378.5593166,"name":"online","context":{"idset":"10981"}}
+{"timestamp":1712188378.7273703,"name":"online","context":{"idset":"10966,10969,10978-10980,10982"}}
+{"timestamp":1712188379.0445273,"name":"online","context":{"idset":"10983,10986,10989"}}
+{"timestamp":1712188379.292908,"name":"online","context":{"idset":"10987,10991,10997"}}
+{"timestamp":1712188379.4000871,"name":"online","context":{"idset":"10984,10990,10992-10993"}}
+{"timestamp":1712188379.4987533,"name":"online","context":{"idset":"10988,10996"}}
+{"timestamp":1712188379.5695381,"name":"online","context":{"idset":"10995"}}
+{"timestamp":1712188379.6251137,"name":"online","context":{"idset":"10994"}}
+{"timestamp":1712188379.814342,"name":"online","context":{"idset":"11005"}}
+{"timestamp":1712188379.8819768,"name":"online","context":{"idset":"11007"}}
+{"timestamp":1712188380.0478067,"name":"online","context":{"idset":"10998,11001,11006"}}
+{"timestamp":1712188380.1935613,"name":"online","context":{"idset":"11011"}}
+{"timestamp":1712188380.2363076,"name":"online","context":{"idset":"11010"}}
+{"timestamp":1712188380.2767608,"name":"online","context":{"idset":"11000"}}
+{"timestamp":1712188380.328824,"name":"online","context":{"idset":"11014"}}
+{"timestamp":1712188380.407743,"name":"online","context":{"idset":"11002,11004"}}
+{"timestamp":1712188380.536097,"name":"online","context":{"idset":"11008"}}
+{"timestamp":1712188380.6840456,"name":"online","context":{"idset":"11016,11018,11020-11021"}}
+{"timestamp":1712188380.8271255,"name":"online","context":{"idset":"11009,11013,11015,11017,11019,11022"}}
+{"timestamp":1712188381.112886,"name":"online","context":{"idset":"11026"}}
+{"timestamp":1712188381.2146983,"name":"online","context":{"idset":"11024-11025"}}
+{"timestamp":1712188381.4542053,"name":"online","context":{"idset":"11023"}}
+{"timestamp":1712188381.745708,"name":"online","context":{"idset":"11028,11034-11035"}}
+{"timestamp":1712188381.8921144,"name":"online","context":{"idset":"11032,11036"}}
+{"timestamp":1712188382.0304866,"name":"online","context":{"idset":"11029,11038,11042,11046,11049"}}
+{"timestamp":1712188382.1723721,"name":"online","context":{"idset":"11027,11040,11043,11045,11048"}}
+{"timestamp":1712188382.3105438,"name":"online","context":{"idset":"11033,11037,11039"}}
+{"timestamp":1712188382.4390256,"name":"online","context":{"idset":"11030-11031,11041"}}
+{"timestamp":1712188382.5806372,"name":"online","context":{"idset":"11044,11047,11050-11051,11053,11055"}}
+{"timestamp":1712188382.7107885,"name":"online","context":{"idset":"11052,11054"}}
+{"timestamp":1712188383.0149162,"name":"online","context":{"idset":"11057"}}
+{"timestamp":1712188383.145498,"name":"online","context":{"idset":"11056,11058"}}
+{"timestamp":1712188383.5974231,"name":"online","context":{"idset":"11059-11060"}}
+{"timestamp":1712188383.7398179,"name":"online","context":{"idset":"11061"}}
+{"timestamp":1712188384.6836424,"name":"online","context":{"idset":"11062"}}
+{"timestamp":1712188384.9077554,"name":"online","context":{"idset":"11063"}}
+{"timestamp":1712188385.2294085,"name":"online","context":{"idset":"11064"}}
+{"timestamp":1712188385.546344,"name":"online","context":{"idset":"11065"}}
+{"timestamp":1712188386.1744604,"name":"online","context":{"idset":"11067"}}
+{"timestamp":1712188386.357213,"name":"online","context":{"idset":"11066,11068"}}
+{"timestamp":1712188386.6668129,"name":"online","context":{"idset":"11069-11070"}}
+{"timestamp":1712188387.2359352,"name":"online","context":{"idset":"11071"}}
+{"timestamp":1712188387.4092195,"name":"online","context":{"idset":"11073"}}
+{"timestamp":1712188387.8820884,"name":"online","context":{"idset":"11074"}}
+{"timestamp":1712188388.0968881,"name":"online","context":{"idset":"11072,11075"}}
+{"timestamp":1712188388.6500971,"name":"online","context":{"idset":"11076"}}
+{"timestamp":1712188388.6995788,"name":"online","context":{"idset":"11079"}}
+{"timestamp":1712188388.7658279,"name":"online","context":{"idset":"11077"}}
+{"timestamp":1712188388.8172414,"name":"online","context":{"idset":"11082"}}
+{"timestamp":1712188388.8642533,"name":"online","context":{"idset":"11078"}}
+{"timestamp":1712188388.9551303,"name":"online","context":{"idset":"11080-11081"}}
+{"timestamp":1712188389.1390352,"name":"online","context":{"idset":"11084"}}
+{"timestamp":1712188389.4585137,"name":"online","context":{"idset":"11085-11086"}}
+{"timestamp":1712188389.4980404,"name":"online","context":{"idset":"11083"}}
+{"timestamp":1712188389.7075946,"name":"online","context":{"idset":"11090,11093-11094,11097"}}
+{"timestamp":1712188389.7513342,"name":"online","context":{"idset":"11087"}}
+{"timestamp":1712188389.8363099,"name":"online","context":{"idset":"11088,11092"}}
+{"timestamp":1712188389.9495242,"name":"online","context":{"idset":"11089,11095-11096,11098"}}
+{"timestamp":1712188390.0719018,"name":"online","context":{"idset":"11091,11100,11109"}}
+{"timestamp":1712188390.1718462,"name":"online","context":{"idset":"11099,11101"}}
+{"timestamp":1712188390.4254754,"name":"online","context":{"idset":"11103,11106-11107,11110"}}
+{"timestamp":1712188390.5563707,"name":"online","context":{"idset":"11105,11113,11118"}}
+{"timestamp":1712188390.6359565,"name":"online","context":{"idset":"11104,11111"}}
+{"timestamp":1712188390.6996698,"name":"online","context":{"idset":"11102,11114-11115,11121"}}
+{"timestamp":1712188390.7738996,"name":"online","context":{"idset":"11117"}}
+{"timestamp":1712188390.9041457,"name":"online","context":{"idset":"11108,11119"}}
+{"timestamp":1712188391.0406127,"name":"online","context":{"idset":"11112,11123,11127-11128"}}
+{"timestamp":1712188391.1373873,"name":"online","context":{"idset":"11120,11125"}}
+{"timestamp":1712188391.2753651,"name":"online","context":{"idset":"11131"}}
+{"timestamp":1712188391.4473243,"name":"online","context":{"idset":"11129,11132"}}
+{"timestamp":1712188391.6857927,"name":"online","context":{"idset":"11133"}}
+{"timestamp":1712188391.7454093,"name":"online","context":{"idset":"11122"}}
+{"timestamp":1712188391.9047265,"name":"online","context":{"idset":"11135"}}
+{"timestamp":1712188391.9438,"name":"online","context":{"idset":"11137"}}
+{"timestamp":1712188392.1091273,"name":"online","context":{"idset":"11139,11144"}}
+{"timestamp":1712188392.3077176,"name":"online","context":{"idset":"11147"}}
+{"timestamp":1712188392.3560524,"name":"online","context":{"idset":"11148"}}
+{"timestamp":1712188392.3970132,"name":"online","context":{"idset":"11141,11160"}}
+{"timestamp":1712188392.4046597,"name":"online","context":{"idset":"11146"}}
+{"timestamp":1712188392.4629257,"name":"online","context":{"idset":"11151"}}
+{"timestamp":1712188392.5354862,"name":"online","context":{"idset":"11142-11143"}}
+{"timestamp":1712188392.6947899,"name":"online","context":{"idset":"11145,11152"}}
+{"timestamp":1712188392.78332,"name":"online","context":{"idset":"11154"}}
+{"timestamp":1712188392.8656859,"name":"online","context":{"idset":"11156"}}
+{"timestamp":1712188392.9593582,"name":"online","context":{"idset":"11155,11161"}}
+{"timestamp":1712188393.2080412,"name":"online","context":{"idset":"11159"}}
+{"timestamp":1712188393.276998,"name":"online","context":{"idset":"11153,11164"}}
+{"timestamp":1712188393.4285052,"name":"online","context":{"idset":"11158,11163"}}
+{"timestamp":1712188393.4754522,"name":"online","context":{"idset":"11162"}}
+{"timestamp":1712188393.5835602,"name":"online","context":{"idset":"11157,11166"}}
+{"timestamp":1712188393.7186961,"name":"online","context":{"idset":"11167-11168"}}
+{"timestamp":1712188393.8646941,"name":"online","context":{"idset":"11169"}}
+{"timestamp":1712188394.0058465,"name":"online","context":{"idset":"11170,11172,11175-11176"}}
+{"timestamp":1712188394.1883731,"name":"online","context":{"idset":"11184"}}
+{"timestamp":1712188394.2883694,"name":"online","context":{"idset":"11171,11188,11190"}}
+{"timestamp":1712188394.4405408,"name":"online","context":{"idset":"11177,11181,11183,11191,11193"}}
+{"timestamp":1712188394.5770612,"name":"online","context":{"idset":"11165,11182,11185,11189,11192"}}
+{"timestamp":1712188394.7211461,"name":"online","context":{"idset":"11179-11180,11186,11196"}}
+{"timestamp":1712188394.8151188,"name":"online","context":{"idset":"11194"}}
+{"timestamp":1712188394.9235873,"name":"online","context":{"idset":"11198,11200"}}
+{"timestamp":1712188394.9623559,"name":"online","context":{"idset":"11195"}}
+{"timestamp":1712188395.1793611,"name":"online","context":{"idset":"11197,11199"}}
+{"timestamp":1712188395.475786,"name":"online","context":{"idset":"11201,11203"}}
+{"timestamp":1712188395.6926467,"name":"online","context":{"idset":"11202"}}
+{"timestamp":1712188396.4406903,"name":"online","context":{"idset":"11204-11205"}}
+{"timestamp":1712188397.062356,"name":"online","context":{"idset":"11206"}}
+{"timestamp":1712188397.2334006,"name":"online","context":{"idset":"11207"}}
+{"timestamp":1712188398.0618198,"name":"online","context":{"idset":"11208"}}
+{"timestamp":1712188398.323792,"name":"online","context":{"idset":"11209"}}
+{"timestamp":1712188398.3826375,"name":"online","context":{"idset":"11210"}}
+{"timestamp":1712188398.4529684,"name":"online","context":{"idset":"11211"}}
+{"timestamp":1712188398.784843,"name":"online","context":{"idset":"11214"}}
+{"timestamp":1712188399.2611928,"name":"online","context":{"idset":"11212-11213"}}
+{"timestamp":1712188399.4664173,"name":"online","context":{"idset":"11215"}}
+{"timestamp":1712188399.5657878,"name":"online","context":{"idset":"11217"}}
+{"timestamp":1712188400.7621331,"name":"online","context":{"idset":"11216"}}
+{"timestamp":1712188400.7637839,"name":"online","context":{"idset":"11219"}}
+{"timestamp":1712188400.7650461,"name":"online","context":{"idset":"11220"}}
+{"timestamp":1712188400.7664342,"name":"online","context":{"idset":"11222"}}
+{"timestamp":1712188400.7676542,"name":"online","context":{"idset":"11221"}}
+{"timestamp":1712188400.8226752,"name":"online","context":{"idset":"11224"}}
+{"timestamp":1712188400.8807077,"name":"online","context":{"idset":"11218"}}
+{"timestamp":1712188401.0271089,"name":"online","context":{"idset":"11225-11226"}}
+{"timestamp":1712188401.206805,"name":"online","context":{"idset":"11223,11227"}}
+{"timestamp":1712188401.3012266,"name":"online","context":{"idset":"11230"}}
+{"timestamp":1712188401.3492765,"name":"online","context":{"idset":"11228"}}
+{"timestamp":1712188401.3971372,"name":"online","context":{"idset":"11229"}}
+{"timestamp":1712188401.5129497,"name":"online","context":{"idset":"11232,11240"}}
+{"timestamp":1712188401.6795928,"name":"online","context":{"idset":"11235,11239,11241"}}
+{"timestamp":1712188401.7814608,"name":"online","context":{"idset":"11231,11233-11234,11236-11237"}}
+{"timestamp":1712188401.9936364,"name":"online","context":{"idset":"11238,11243"}}
+{"timestamp":1712188402.2241752,"name":"online","context":{"idset":"11248"}}
+{"timestamp":1712188402.4388192,"name":"online","context":{"idset":"11242,11247,11252-11253,11256-11257"}}
+{"timestamp":1712188402.4792864,"name":"online","context":{"idset":"11254"}}
+{"timestamp":1712188402.5363543,"name":"online","context":{"idset":"11249,11251"}}
+{"timestamp":1712188402.5759814,"name":"online","context":{"idset":"11245"}}
+{"timestamp":1712188402.6286287,"name":"online","context":{"idset":"11246,11260"}}
+{"timestamp":1712188402.7690973,"name":"online","context":{"idset":"11244,11255"}}
+{"timestamp":1712188402.8196826,"name":"online","context":{"idset":"11250,11261,11264"}}
+{"timestamp":1712188403.0458932,"name":"online","context":{"idset":"11258,11262-11263,11265"}}
+{"timestamp":1712188403.0973811,"name":"online","context":{"idset":"11259,11266"}}
+{"timestamp":1712188403.207031,"name":"online","context":{"idset":"11269"}}
+{"timestamp":1712188403.4285114,"name":"online","context":{"idset":"11267-11268"}}
+{"timestamp":1712188403.4776933,"name":"online","context":{"idset":"11271"}}
+{"timestamp":1712188403.5721817,"name":"online","context":{"idset":"11272"}}
+{"timestamp":1712188403.697772,"name":"online","context":{"idset":"11273"}}
+{"timestamp":1712188404.0929482,"name":"online","context":{"idset":"11281"}}
+{"timestamp":1712188404.2400496,"name":"online","context":{"idset":"11270,11274-11275,11277,11280"}}
+{"timestamp":1712188404.4692168,"name":"online","context":{"idset":"11283"}}
+{"timestamp":1712188404.5440307,"name":"online","context":{"idset":"11278,11284,11289"}}
+{"timestamp":1712188404.5901122,"name":"online","context":{"idset":"11285"}}
+{"timestamp":1712188404.6709261,"name":"online","context":{"idset":"11276"}}
+{"timestamp":1712188404.7437434,"name":"online","context":{"idset":"11282"}}
+{"timestamp":1712188404.8142531,"name":"online","context":{"idset":"11291"}}
+{"timestamp":1712188404.8830428,"name":"online","context":{"idset":"11279,11290"}}
+{"timestamp":1712188405.1264694,"name":"online","context":{"idset":"11294-11295"}}
+{"timestamp":1712188405.198312,"name":"online","context":{"idset":"11286,11292"}}
+{"timestamp":1712188405.2669823,"name":"online","context":{"idset":"11293"}}
+{"timestamp":1712188405.2810686,"name":"online","context":{"idset":"11297"}}
+{"timestamp":1712188405.3653562,"name":"online","context":{"idset":"11296,11298"}}
+{"timestamp":1712188405.4773626,"name":"online","context":{"idset":"11301"}}
+{"timestamp":1712188405.5893843,"name":"online","context":{"idset":"11288,11300"}}
+{"timestamp":1712188405.7026634,"name":"online","context":{"idset":"11299"}}
+{"timestamp":1712188405.8600342,"name":"online","context":{"idset":"11302,11304-11305"}}
+{"timestamp":1712188406.0294719,"name":"online","context":{"idset":"11306-11307,11309"}}
+{"timestamp":1712188406.3223338,"name":"online","context":{"idset":"11315,11318-11319"}}
+{"timestamp":1712188406.4800851,"name":"online","context":{"idset":"11303,11311,11313,11316"}}
+{"timestamp":1712188406.5891726,"name":"online","context":{"idset":"11310,11314,11320,11324"}}
+{"timestamp":1712188406.7153325,"name":"online","context":{"idset":"11317"}}
+{"timestamp":1712188406.8247392,"name":"online","context":{"idset":"11308,11321,11327"}}
+{"timestamp":1712188406.9871547,"name":"online","context":{"idset":"11312,11328-11329"}}
+{"timestamp":1712188407.1779041,"name":"online","context":{"idset":"11325"}}
+{"timestamp":1712188407.3777478,"name":"online","context":{"idset":"11323,11331,11333"}}
+{"timestamp":1712188407.5263579,"name":"online","context":{"idset":"11322,11332"}}
+{"timestamp":1712188407.5882554,"name":"online","context":{"idset":"11326"}}
+{"timestamp":1712188407.9283421,"name":"online","context":{"idset":"11334"}}
+{"timestamp":1712188408.3709323,"name":"online","context":{"idset":"11335"}}
+{"timestamp":1712188408.9051883,"name":"online","context":{"idset":"11337"}}
+{"timestamp":1712188409.0705028,"name":"online","context":{"idset":"11336"}}
+{"timestamp":1712188409.8335016,"name":"online","context":{"idset":"11338"}}
+{"timestamp":1712188409.9188006,"name":"online","context":{"idset":"11339,11341"}}
+{"timestamp":1712188410.3633873,"name":"online","context":{"idset":"11340,11342"}}
+{"timestamp":1712188411.0753758,"name":"online","context":{"idset":"11343"}}
+{"timestamp":1712188411.2510641,"name":"online","context":{"idset":"11346"}}
+{"timestamp":1712188411.4761,"name":"online","context":{"idset":"11344"}}
+{"timestamp":1712188411.537097,"name":"online","context":{"idset":"11347"}}
+{"timestamp":1712188411.9092586,"name":"online","context":{"idset":"11345"}}
+{"timestamp":1712188412.0785856,"name":"online","context":{"idset":"11349"}}
+{"timestamp":1712188412.268769,"name":"online","context":{"idset":"11348,11350-11351"}}
+{"timestamp":1712188412.4386172,"name":"online","context":{"idset":"11352"}}
+{"timestamp":1712188412.5657413,"name":"online","context":{"idset":"11355"}}
+{"timestamp":1712188412.6323104,"name":"online","context":{"idset":"11354"}}
+{"timestamp":1712188412.822063,"name":"online","context":{"idset":"11353,11359"}}
+{"timestamp":1712188413.0777588,"name":"online","context":{"idset":"11357,11360"}}
+{"timestamp":1712188413.2104514,"name":"online","context":{"idset":"11361-11362,11364"}}
+{"timestamp":1712188413.3172352,"name":"online","context":{"idset":"11366"}}
+{"timestamp":1712188413.4000657,"name":"online","context":{"idset":"11363,11369"}}
+{"timestamp":1712188413.5106356,"name":"online","context":{"idset":"11356,11371"}}
+{"timestamp":1712188413.6210465,"name":"online","context":{"idset":"11375"}}
+{"timestamp":1712188413.7090182,"name":"online","context":{"idset":"11367,11370"}}
+{"timestamp":1712188413.8830326,"name":"online","context":{"idset":"11368,11372"}}
+{"timestamp":1712188414.0248322,"name":"online","context":{"idset":"11382"}}
+{"timestamp":1712188414.1869929,"name":"online","context":{"idset":"11378,11381"}}
+{"timestamp":1712188414.2377458,"name":"online","context":{"idset":"11374"}}
+{"timestamp":1712188414.3127353,"name":"online","context":{"idset":"11365,11376,11379"}}
+{"timestamp":1712188414.4712546,"name":"online","context":{"idset":"11377,11384-11385,11389,11391,11398"}}
+{"timestamp":1712188414.5117648,"name":"online","context":{"idset":"11399"}}
+{"timestamp":1712188414.5528944,"name":"online","context":{"idset":"11380"}}
+{"timestamp":1712188414.5944026,"name":"online","context":{"idset":"11383,11392"}}
+{"timestamp":1712188414.7041199,"name":"online","context":{"idset":"11402"}}
+{"timestamp":1712188414.7751188,"name":"online","context":{"idset":"11388"}}
+{"timestamp":1712188414.8906186,"name":"online","context":{"idset":"11396"}}
+{"timestamp":1712188414.9293892,"name":"online","context":{"idset":"11386,11394"}}
+{"timestamp":1712188414.9836085,"name":"online","context":{"idset":"11393,11401"}}
+{"timestamp":1712188415.2159836,"name":"online","context":{"idset":"11397"}}
+{"timestamp":1712188415.3414228,"name":"online","context":{"idset":"11400,11403-11404"}}
+{"timestamp":1712188415.6471515,"name":"online","context":{"idset":"11405-11406"}}
+{"timestamp":1712188415.9110048,"name":"online","context":{"idset":"11408,11411"}}
+{"timestamp":1712188416.1165648,"name":"online","context":{"idset":"11407,11409-11410,11412,11415"}}
+{"timestamp":1712188416.2180371,"name":"online","context":{"idset":"11414"}}
+{"timestamp":1712188416.3321257,"name":"online","context":{"idset":"11417"}}
+{"timestamp":1712188416.4498546,"name":"online","context":{"idset":"11416"}}
+{"timestamp":1712188416.4915831,"name":"online","context":{"idset":"11421"}}
+{"timestamp":1712188416.5009718,"name":"online","context":{"idset":"11420"}}
+{"timestamp":1712188416.6567607,"name":"online","context":{"idset":"11419"}}
+{"timestamp":1712188416.7074862,"name":"online","context":{"idset":"11413,11418,11433"}}
+{"timestamp":1712188416.8431158,"name":"online","context":{"idset":"11430"}}
+{"timestamp":1712188416.9197407,"name":"online","context":{"idset":"11422-11423,11426"}}
+{"timestamp":1712188416.9728353,"name":"online","context":{"idset":"11427"}}
+{"timestamp":1712188417.1190836,"name":"online","context":{"idset":"11424,11428"}}
+{"timestamp":1712188417.175622,"name":"online","context":{"idset":"11432,11435,11437"}}
+{"timestamp":1712188417.1911261,"name":"online","context":{"idset":"11425"}}
+{"timestamp":1712188417.3941875,"name":"online","context":{"idset":"11434"}}
+{"timestamp":1712188417.6444638,"name":"online","context":{"idset":"11436"}}
+{"timestamp":1712188417.712486,"name":"online","context":{"idset":"11443"}}
+{"timestamp":1712188417.9297631,"name":"online","context":{"idset":"11444"}}
+{"timestamp":1712188418.1733959,"name":"online","context":{"idset":"11438"}}
+{"timestamp":1712188418.3281465,"name":"online","context":{"idset":"11446"}}
+{"timestamp":1712188418.4876711,"name":"online","context":{"idset":"11447-11448,11451"}}
+{"timestamp":1712188418.5809777,"name":"online","context":{"idset":"11453"}}
+{"timestamp":1712188418.7815583,"name":"online","context":{"idset":"11445"}}
+{"timestamp":1712188418.9365764,"name":"online","context":{"idset":"11449-11450,11459"}}
+{"timestamp":1712188419.1657765,"name":"online","context":{"idset":"11452,11454,11456,11463"}}
+{"timestamp":1712188419.3637331,"name":"online","context":{"idset":"11457,11460,11468"}}
+{"timestamp":1712188419.5266747,"name":"online","context":{"idset":"11461-11462,11467"}}
+{"timestamp":1712188419.5897326,"name":"online","context":{"idset":"11474"}}
+{"timestamp":1712188419.6995509,"name":"online","context":{"idset":"11458,11469"}}
+{"timestamp":1712188419.8004947,"name":"online","context":{"idset":"11472"}}
+{"timestamp":1712188419.9904296,"name":"online","context":{"idset":"11465,11471,11475,11477"}}
+{"timestamp":1712188420.0809271,"name":"online","context":{"idset":"11476"}}
+{"timestamp":1712188420.2294157,"name":"online","context":{"idset":"11470,11473"}}
+{"timestamp":1712188420.7834585,"name":"online","context":{"idset":"11478-11479"}}
+{"timestamp":1712188421.5483229,"name":"online","context":{"idset":"11480"}}
+{"timestamp":1712188421.7477996,"name":"online","context":{"idset":"11482"}}
+{"timestamp":1712188422.1477864,"name":"online","context":{"idset":"11481,11484"}}
+{"timestamp":1712188422.2209747,"name":"online","context":{"idset":"11483"}}
+{"timestamp":1712188422.8103619,"name":"online","context":{"idset":"11485"}}
+{"timestamp":1712188422.9930933,"name":"online","context":{"idset":"11486"}}
+{"timestamp":1712188423.5076559,"name":"online","context":{"idset":"11488"}}
+{"timestamp":1712188423.8083062,"name":"online","context":{"idset":"11487"}}
+{"timestamp":1712188423.8983271,"name":"online","context":{"idset":"11489-11490"}}
+{"timestamp":1712188424.447964,"name":"online","context":{"idset":"11491"}}
+{"timestamp":1712188424.5704322,"name":"online","context":{"idset":"11492-11493"}}
+{"timestamp":1712188424.6177468,"name":"online","context":{"idset":"11497"}}
+{"timestamp":1712188424.7476447,"name":"online","context":{"idset":"11496"}}
+{"timestamp":1712188425.0970709,"name":"online","context":{"idset":"11501"}}
+{"timestamp":1712188425.1743565,"name":"online","context":{"idset":"11498"}}
+{"timestamp":1712188425.2290444,"name":"online","context":{"idset":"11503-11504"}}
+{"timestamp":1712188425.3111858,"name":"online","context":{"idset":"11494-11495,11502"}}
+{"timestamp":1712188425.4181011,"name":"online","context":{"idset":"11499-11500,11512"}}
+{"timestamp":1712188425.6339529,"name":"online","context":{"idset":"11511"}}
+{"timestamp":1712188425.7045028,"name":"online","context":{"idset":"11506"}}
+{"timestamp":1712188425.7554715,"name":"online","context":{"idset":"11510"}}
+{"timestamp":1712188425.8737752,"name":"online","context":{"idset":"11507,11509,11514"}}
+{"timestamp":1712188425.920579,"name":"online","context":{"idset":"11505"}}
+{"timestamp":1712188426.1244047,"name":"online","context":{"idset":"11508,11513"}}
+{"timestamp":1712188426.2144856,"name":"online","context":{"idset":"11519"}}
+{"timestamp":1712188426.994344,"name":"online","context":{"idset":"11517,11520"}}
+{"timestamp":1712188426.9961059,"name":"online","context":{"idset":"11515"}}
+{"timestamp":1712188426.9979196,"name":"online","context":{"idset":"11522-11524"}}
+{"timestamp":1712188426.9997144,"name":"online","context":{"idset":"11516"}}
+{"timestamp":1712188427.0011802,"name":"online","context":{"idset":"11521"}}
+{"timestamp":1712188427.0026619,"name":"online","context":{"idset":"11518"}}
+{"timestamp":1712188427.0043015,"name":"online","context":{"idset":"11542"}}
+{"timestamp":1712188427.0060577,"name":"online","context":{"idset":"11525,11541"}}
+{"timestamp":1712188427.0076339,"name":"online","context":{"idset":"11533"}}
+{"timestamp":1712188427.009316,"name":"online","context":{"idset":"11530,11532,11539"}}
+{"timestamp":1712188427.1272161,"name":"online","context":{"idset":"11529,11534,11545"}}
+{"timestamp":1712188427.1904087,"name":"online","context":{"idset":"11531,11535,11537,11544"}}
+{"timestamp":1712188427.3523815,"name":"online","context":{"idset":"11540,11543"}}
+{"timestamp":1712188427.4992039,"name":"online","context":{"idset":"11526,11528,11536,11538,11547"}}
+{"timestamp":1712188427.615521,"name":"online","context":{"idset":"11548-11549"}}
+{"timestamp":1712188427.7070966,"name":"online","context":{"idset":"11546,11550"}}
+{"timestamp":1712188427.8888905,"name":"online","context":{"idset":"11553"}}
+{"timestamp":1712188427.9545591,"name":"online","context":{"idset":"11551"}}
+{"timestamp":1712188428.1571214,"name":"online","context":{"idset":"11556"}}
+{"timestamp":1712188428.2131064,"name":"online","context":{"idset":"11562"}}
+{"timestamp":1712188428.300631,"name":"online","context":{"idset":"11552,11555,11558"}}
+{"timestamp":1712188428.3539233,"name":"online","context":{"idset":"11557"}}
+{"timestamp":1712188428.438303,"name":"online","context":{"idset":"11554"}}
+{"timestamp":1712188428.5267487,"name":"online","context":{"idset":"11560-11561"}}
+{"timestamp":1712188428.724576,"name":"online","context":{"idset":"11559"}}
+{"timestamp":1712188428.7710199,"name":"online","context":{"idset":"11565"}}
+{"timestamp":1712188428.8672607,"name":"online","context":{"idset":"11564"}}
+{"timestamp":1712188428.9779737,"name":"online","context":{"idset":"11568-11570"}}
+{"timestamp":1712188429.0338848,"name":"online","context":{"idset":"11573"}}
+{"timestamp":1712188429.1054933,"name":"online","context":{"idset":"11566-11567,11572"}}
+{"timestamp":1712188429.2298555,"name":"online","context":{"idset":"11571"}}
+{"timestamp":1712188429.4936049,"name":"online","context":{"idset":"11574"}}
+{"timestamp":1712188429.6380675,"name":"online","context":{"idset":"11577"}}
+{"timestamp":1712188429.7740569,"name":"online","context":{"idset":"11576"}}
+{"timestamp":1712188429.8945894,"name":"online","context":{"idset":"11575"}}
+{"timestamp":1712188430.0117643,"name":"online","context":{"idset":"11579,11581"}}
+{"timestamp":1712188430.3528492,"name":"online","context":{"idset":"11578,11585"}}
+{"timestamp":1712188430.5201716,"name":"online","context":{"idset":"11580,11582,11584"}}
+{"timestamp":1712188430.7227988,"name":"online","context":{"idset":"11589,11591"}}
+{"timestamp":1712188430.8725493,"name":"online","context":{"idset":"11586"}}
+{"timestamp":1712188430.9741545,"name":"online","context":{"idset":"11583,11590"}}
+{"timestamp":1712188431.1675277,"name":"online","context":{"idset":"11592"}}
+{"timestamp":1712188431.2620962,"name":"online","context":{"idset":"11596"}}
+{"timestamp":1712188431.4534461,"name":"online","context":{"idset":"11594,11599,11607"}}
+{"timestamp":1712188431.5644879,"name":"online","context":{"idset":"11595,11597"}}
+{"timestamp":1712188431.6943462,"name":"online","context":{"idset":"11598,11601,11604,11606"}}
+{"timestamp":1712188431.7908969,"name":"online","context":{"idset":"11605,11608"}}
+{"timestamp":1712188431.9008892,"name":"online","context":{"idset":"11593,11600,11602"}}
+{"timestamp":1712188432.0458081,"name":"online","context":{"idset":"11603"}}
+{"timestamp":1712188432.2474027,"name":"online","context":{"idset":"11609"}}
+{"timestamp":1712188432.629384,"name":"online","context":{"idset":"11610"}}
+{"timestamp":1712188433.4140146,"name":"online","context":{"idset":"11611-11612"}}
+{"timestamp":1712188433.8246224,"name":"online","context":{"idset":"11613-11614"}}
+{"timestamp":1712188433.895236,"name":"online","context":{"idset":"11615"}}
+{"timestamp":1712188458.1863453,"name":"online","context":{"idset":"11617"}}
+{"timestamp":1712188458.1873944,"name":"online","context":{"idset":"11616"}}
+{"timestamp":1712188458.188381,"name":"online","context":{"idset":"11618"}}
+{"timestamp":1712188458.1894674,"name":"online","context":{"idset":"11622"}}
+{"timestamp":1712188458.1904821,"name":"online","context":{"idset":"11619"}}
+{"timestamp":1712188458.1914165,"name":"online","context":{"idset":"11620-11621"}}
+{"timestamp":1712188458.1926084,"name":"online","context":{"idset":"11623"}}
+{"timestamp":1712188458.1937397,"name":"online","context":{"idset":"11624"}}
+{"timestamp":1712188458.1947403,"name":"online","context":{"idset":"11625"}}
+{"timestamp":1712188458.1957295,"name":"online","context":{"idset":"11627"}}
+{"timestamp":1712188458.1967206,"name":"online","context":{"idset":"11626,11628"}}
+{"timestamp":1712188458.1977663,"name":"online","context":{"idset":"11634"}}
+{"timestamp":1712188458.1988018,"name":"online","context":{"idset":"11629,11631,11633"}}
+{"timestamp":1712188458.1997287,"name":"online","context":{"idset":"11630"}}
+{"timestamp":1712188458.2006705,"name":"online","context":{"idset":"11636"}}
+{"timestamp":1712188458.2017801,"name":"online","context":{"idset":"11635,11638"}}
+{"timestamp":1712188458.2028193,"name":"online","context":{"idset":"11637"}}
+{"timestamp":1712188458.203768,"name":"online","context":{"idset":"11639"}}
+{"timestamp":1712188458.2047319,"name":"online","context":{"idset":"11640"}}
+{"timestamp":1712188458.2058921,"name":"online","context":{"idset":"11646"}}
+{"timestamp":1712188458.207005,"name":"online","context":{"idset":"11641-11644"}}
+{"timestamp":1712188458.2081077,"name":"online","context":{"idset":"11647"}}
+{"timestamp":1712188458.2091579,"name":"online","context":{"idset":"11645,11650,11655"}}
+{"timestamp":1712188458.210114,"name":"online","context":{"idset":"11649,11652,11656"}}
+{"timestamp":1712188458.2112172,"name":"online","context":{"idset":"11653-11654"}}
+{"timestamp":1712188458.2123837,"name":"online","context":{"idset":"11657"}}
+{"timestamp":1712188482.9782913,"name":"online","context":{"idset":"11648,11659"}}
+{"timestamp":1712188482.9806199,"name":"online","context":{"idset":"11651,11658,11662,11665,11673"}}
+{"timestamp":1712188482.9825342,"name":"online","context":{"idset":"11664,11671,11674"}}
+{"timestamp":1712188482.9848998,"name":"online","context":{"idset":"11663"}}
+{"timestamp":1712188482.9884794,"name":"online","context":{"idset":"11660,11668,11672,11675,11678"}}
+{"timestamp":1712188482.9919095,"name":"online","context":{"idset":"11661,11667,11681"}}
+{"timestamp":1712188482.9953251,"name":"online","context":{"idset":"11676"}}
+{"timestamp":1712188482.9986784,"name":"online","context":{"idset":"11666,11679"}}
+{"timestamp":1712188483.002022,"name":"online","context":{"idset":"11677,11680"}}
+{"timestamp":1712188483.005399,"name":"online","context":{"idset":"11682-11683,11685"}}
+{"timestamp":1712188483.008673,"name":"online","context":{"idset":"11684"}}
+{"timestamp":1712188483.0124457,"name":"online","context":{"idset":"11690"}}
+{"timestamp":1712188483.0158842,"name":"online","context":{"idset":"11687"}}
+{"timestamp":1712188483.0195155,"name":"online","context":{"idset":"11689"}}
+{"timestamp":1712188483.0230088,"name":"online","context":{"idset":"11691,11695"}}
+{"timestamp":1712188483.0268316,"name":"online","context":{"idset":"11686,11688,11692"}}
+{"timestamp":1712188483.0302505,"name":"online","context":{"idset":"11693-11694,11701-11702"}}
+{"timestamp":1712188483.0338771,"name":"online","context":{"idset":"11696,11700"}}
+{"timestamp":1712188483.0370879,"name":"online","context":{"idset":"11698"}}
+{"timestamp":1712188483.0403044,"name":"online","context":{"idset":"11705"}}
+{"timestamp":1712188483.0436935,"name":"online","context":{"idset":"11703"}}
+{"timestamp":1712188483.0470457,"name":"online","context":{"idset":"11697,11704"}}
+{"timestamp":1712188483.0502365,"name":"online","context":{"idset":"11699"}}
+{"timestamp":1712188483.053791,"name":"online","context":{"idset":"11708"}}
+{"timestamp":1712188483.0578473,"name":"online","context":{"idset":"11707,11709"}}
+{"timestamp":1712188483.0618489,"name":"online","context":{"idset":"11706"}}
+{"timestamp":1712188483.0651584,"name":"online","context":{"idset":"11710-11712"}}
+{"timestamp":1712188483.0687845,"name":"online","context":{"idset":"11716,11720"}}
+{"timestamp":1712188483.0720236,"name":"online","context":{"idset":"11713-11715,11717-11719"}}
+{"timestamp":1712188483.0759418,"name":"online","context":{"idset":"11722-11723"}}
+{"timestamp":1712188483.0806048,"name":"online","context":{"idset":"11724"}}
+{"timestamp":1712188483.0840337,"name":"online","context":{"idset":"11721,11728-11729"}}
+{"timestamp":1712188483.0871639,"name":"online","context":{"idset":"11730"}}
+{"timestamp":1712188483.0904505,"name":"online","context":{"idset":"11726-11727,11731,11733-11734,11736,11739"}}
+{"timestamp":1712188483.093936,"name":"online","context":{"idset":"11725,11737"}}
+{"timestamp":1712188483.0970776,"name":"online","context":{"idset":"11738"}}
+{"timestamp":1712188483.1005445,"name":"online","context":{"idset":"11735"}}
+{"timestamp":1712188483.104022,"name":"online","context":{"idset":"11732"}}
+{"timestamp":1712188483.1073542,"name":"online","context":{"idset":"11741"}}
+{"timestamp":1712188483.1101904,"name":"online","context":{"idset":"11740"}}
+{"timestamp":1712188483.1135137,"name":"online","context":{"idset":"11742"}}
+{"timestamp":1712188483.1173005,"name":"online","context":{"idset":"11743"}}
+{"timestamp":1712188483.1205375,"name":"online","context":{"idset":"11744"}}
+{"timestamp":1712188483.1240282,"name":"online","context":{"idset":"11745"}}
+{"timestamp":1712188483.1273229,"name":"online","context":{"idset":"11746-11747"}}
+{"timestamp":1712188483.1307361,"name":"online","context":{"idset":"11749"}}
+{"timestamp":1712188483.1343818,"name":"online","context":{"idset":"11752"}}
+{"timestamp":1712188483.1378579,"name":"online","context":{"idset":"11750"}}
+{"timestamp":1712188483.1413569,"name":"online","context":{"idset":"11751"}}
+{"timestamp":1712188483.1447432,"name":"online","context":{"idset":"11753"}}
+{"timestamp":1712188483.1484203,"name":"online","context":{"idset":"11754"}}
+{"timestamp":1712188483.1521349,"name":"online","context":{"idset":"11757"}}
+{"timestamp":1712188483.1555085,"name":"online","context":{"idset":"11755"}}
+{"timestamp":1712188483.1590908,"name":"online","context":{"idset":"11759"}}
+{"timestamp":1712188483.1624687,"name":"online","context":{"idset":"11758"}}
+{"timestamp":1712188483.1660943,"name":"online","context":{"idset":"11764"}}
+{"timestamp":1712188483.169704,"name":"online","context":{"idset":"11756,11760"}}
+{"timestamp":1712188483.1742299,"name":"online","context":{"idset":"11762,11771"}}
+{"timestamp":1712188483.1783338,"name":"online","context":{"idset":"11763,11768"}}
+{"timestamp":1712188483.1822348,"name":"online","context":{"idset":"11765-11766"}}
+{"timestamp":1712188483.1859009,"name":"online","context":{"idset":"11761,11770"}}
+{"timestamp":1712188483.1895084,"name":"online","context":{"idset":"11775"}}
+{"timestamp":1712188483.1929922,"name":"online","context":{"idset":"11769"}}
+{"timestamp":1712188483.1964121,"name":"online","context":{"idset":"11773,11778"}}
+{"timestamp":1712188483.2002194,"name":"online","context":{"idset":"11774,11782,11787"}}
+{"timestamp":1712188483.2036142,"name":"online","context":{"idset":"11772,11777,11779-11780"}}
+{"timestamp":1712188483.2073207,"name":"online","context":{"idset":"11784"}}
+{"timestamp":1712188483.2107291,"name":"online","context":{"idset":"11781"}}
+{"timestamp":1712188483.2144885,"name":"online","context":{"idset":"11767,11776,11785"}}
+{"timestamp":1712188483.2180469,"name":"online","context":{"idset":"11783,11800"}}
+{"timestamp":1712188483.2217953,"name":"online","context":{"idset":"11786,11791"}}
+{"timestamp":1712188483.2255664,"name":"online","context":{"idset":"11792"}}
+{"timestamp":1712188483.229872,"name":"online","context":{"idset":"11788,11795-11796,11804"}}
+{"timestamp":1712188483.2336578,"name":"online","context":{"idset":"11805"}}
+{"timestamp":1712188483.2371235,"name":"online","context":{"idset":"11797,11818"}}
+{"timestamp":1712188483.241056,"name":"online","context":{"idset":"11808-11809"}}
+{"timestamp":1712188483.2439716,"name":"online","context":{"idset":"11801-11802,11806,11813,11815,11822,11824"}}
+{"timestamp":1712188483.2462242,"name":"online","context":{"idset":"11799,11803,11811-11812"}}
+{"timestamp":1712188483.2482834,"name":"online","context":{"idset":"11798"}}
+{"timestamp":1712188483.2505383,"name":"online","context":{"idset":"11814"}}
+{"timestamp":1712188483.2528415,"name":"online","context":{"idset":"11810,11819-11820,11823"}}
+{"timestamp":1712188483.2549558,"name":"online","context":{"idset":"11817"}}
+{"timestamp":1712188483.2570984,"name":"online","context":{"idset":"11825-11826,11828-11829"}}
+{"timestamp":1712188483.2590528,"name":"online","context":{"idset":"11827,11830"}}
+{"timestamp":1712188483.2612042,"name":"online","context":{"idset":"11821,11836"}}
+{"timestamp":1712188483.2633662,"name":"online","context":{"idset":"11831,11835"}}
+{"timestamp":1712188483.265394,"name":"online","context":{"idset":"11838"}}
+{"timestamp":1712188483.267256,"name":"online","context":{"idset":"11833"}}
+{"timestamp":1712188483.2689712,"name":"online","context":{"idset":"11834"}}
+{"timestamp":1712188483.2706904,"name":"online","context":{"idset":"11840"}}
+{"timestamp":1712188483.2724137,"name":"online","context":{"idset":"11832"}}
+{"timestamp":1712188483.274776,"name":"online","context":{"idset":"11843"}}
+{"timestamp":1712188483.2770281,"name":"online","context":{"idset":"11842"}}
+{"timestamp":1712188483.279213,"name":"online","context":{"idset":"11845"}}
+{"timestamp":1712188483.2814281,"name":"online","context":{"idset":"11844"}}
+{"timestamp":1712188483.2836473,"name":"online","context":{"idset":"11837"}}
+{"timestamp":1712188483.2856791,"name":"online","context":{"idset":"11846,11848"}}
+{"timestamp":1712188483.2885857,"name":"online","context":{"idset":"11856"}}
+{"timestamp":1712188483.2905462,"name":"online","context":{"idset":"11862"}}
+{"timestamp":1712188483.2936182,"name":"online","context":{"idset":"11850,11855"}}
+{"timestamp":1712188483.2965295,"name":"online","context":{"idset":"11849"}}
+{"timestamp":1712188483.2995534,"name":"online","context":{"idset":"11857"}}
+{"timestamp":1712188483.3024087,"name":"online","context":{"idset":"11858,11861"}}
+{"timestamp":1712188483.3049662,"name":"online","context":{"idset":"11847"}}
+{"timestamp":1712188483.3077147,"name":"online","context":{"idset":"11852-11854,11872"}}
+{"timestamp":1712188483.3095222,"name":"online","context":{"idset":"11851,11859,11867"}}
+{"timestamp":1712188483.3114347,"name":"online","context":{"idset":"11864-11865,11869-11870"}}
+{"timestamp":1712188483.3135092,"name":"online","context":{"idset":"11866,11873-11874"}}
+{"timestamp":1712188483.3153713,"name":"online","context":{"idset":"11860,11871"}}
+{"timestamp":1712188483.3173449,"name":"online","context":{"idset":"11863"}}
+{"timestamp":1712188483.3192475,"name":"online","context":{"idset":"11868,11878"}}
+{"timestamp":1712188508.9155045,"name":"online","context":{"idset":"11876,11879"}}
+{"timestamp":1712188508.9169304,"name":"online","context":{"idset":"11875"}}
+{"timestamp":1712188508.9180789,"name":"online","context":{"idset":"11877"}}
+{"timestamp":1712188508.919179,"name":"online","context":{"idset":"11881"}}
+{"timestamp":1712188508.9202073,"name":"online","context":{"idset":"11882"}}
+{"timestamp":1712188508.9212363,"name":"online","context":{"idset":"11885"}}
+{"timestamp":1712188508.9223351,"name":"online","context":{"idset":"11880"}}
+{"timestamp":1712188508.9233689,"name":"online","context":{"idset":"11883"}}
+{"timestamp":1712188509.0809233,"name":"online","context":{"idset":"11886"}}
+{"timestamp":1712188509.0820756,"name":"online","context":{"idset":"11887"}}
+{"timestamp":1712188509.0831676,"name":"online","context":{"idset":"11889-11890"}}
+{"timestamp":1712188509.0842538,"name":"online","context":{"idset":"11888"}}
+{"timestamp":1712188509.0854545,"name":"online","context":{"idset":"11892"}}
+{"timestamp":1712188509.7440648,"name":"online","context":{"idset":"7"}}
+{"timestamp":1712188509.7454104,"name":"online","context":{"idset":"29,41"}}
+{"timestamp":1712188509.7467558,"name":"online","context":{"idset":"45-46"}}
+{"timestamp":1712188509.7480025,"name":"online","context":{"idset":"17,51,59"}}
+{"timestamp":1712188509.750349,"name":"online","context":{"idset":"11,21,23,34,50"}}
+{"timestamp":1712188509.7515678,"name":"online","context":{"idset":"2"}}
+{"timestamp":1712188509.7527711,"name":"online","context":{"idset":"4,15,28,37-38"}}
+{"timestamp":1712188509.7539964,"name":"online","context":{"idset":"1,9-10,20,35-36,40,43,58"}}
+{"timestamp":1712188509.7552915,"name":"online","context":{"idset":"5,13-14,25-26,44,60"}}
+{"timestamp":1712188509.7565556,"name":"online","context":{"idset":"6,8,18,32,39,42,48,54"}}
+{"timestamp":1712188509.7578306,"name":"online","context":{"idset":"22,24,30,47,56-57"}}
+{"timestamp":1712188509.759012,"name":"online","context":{"idset":"19,27,31,53"}}
+{"timestamp":1712188509.7602837,"name":"online","context":{"idset":"3,16,52"}}
+{"timestamp":1712188509.761476,"name":"online","context":{"idset":"49"}}
+{"timestamp":1712188509.7626975,"name":"online","context":{"idset":"12,33,55"}}
+{"timestamp":1712188641.1543598,"name":"undrain","context":{"idset":"969-976,978-980,9973-9988,9990-10036,10038-10058,10060-10063,10065-10100,10103-10134,10137,10139-10142,10144,10146,10149-10152,10155-10161,10163-10182,10184-10201,10203-10204,10208-10260,10262-10300,10303,10305-10309,10311-10359,10373-10468,10470-10474,10476-10477,10481,10483,10485-10503,10505-10508,10511-10516,10520,10523-10540,10542-10556,10559-10581,10583-10601,10603-10612,10742,10745-10746,10754,10756,10760,10822-10831,10869-10870,10872,10875-10882,10885,10887-10904,10907-10910,10913-10916,10919-10920,10925-10934,10938-10942,10944-10975,10977-10996,11125,11127-11129,11131-11133,11135,11137,11139,11141-11148,11151-11153,11155-11172,11175-11177,11179-11186,11188-11252,11292-11300,11327-11329,11331-11357,11359-11372,11374-11386,11389,11391-11394,11396-11402,11404-11428,11430,11432-11438,11443-11454,11456-11463,11465,11467-11470,11472-11506,11508-11526,11528-11532,11534-11562,11564-11586,11589-11615"}}
+{"timestamp":1712189140.919699,"name":"online","context":{"idset":"91"}}
+{"timestamp":1712189141.0565567,"name":"online","context":{"idset":"88"}}
+{"timestamp":1712189141.3382499,"name":"online","context":{"idset":"90,93"}}
+{"timestamp":1712189141.5844603,"name":"online","context":{"idset":"99"}}
+{"timestamp":1712189141.8163462,"name":"online","context":{"idset":"100"}}
+{"timestamp":1712189142.0606537,"name":"online","context":{"idset":"122,176"}}
+{"timestamp":1712189142.1267996,"name":"online","context":{"idset":"97"}}
+{"timestamp":1712189142.3577006,"name":"online","context":{"idset":"125"}}
+{"timestamp":1712189142.4424024,"name":"online","context":{"idset":"107"}}
+{"timestamp":1712189142.5406315,"name":"online","context":{"idset":"98"}}
+{"timestamp":1712189142.8700461,"name":"online","context":{"idset":"94-95,101"}}
+{"timestamp":1712189142.9981899,"name":"online","context":{"idset":"151"}}
+{"timestamp":1712189143.22628,"name":"online","context":{"idset":"126,160"}}
+{"timestamp":1712189143.3343706,"name":"online","context":{"idset":"127"}}
+{"timestamp":1712189143.4599888,"name":"online","context":{"idset":"103,109,124,149"}}
+{"timestamp":1712189143.6495607,"name":"online","context":{"idset":"87,135"}}
+{"timestamp":1712189143.8453076,"name":"online","context":{"idset":"185"}}
+{"timestamp":1712189143.95,"name":"online","context":{"idset":"118,120,157"}}
+{"timestamp":1712189144.1373878,"name":"online","context":{"idset":"96,106,112,114,138,172,174"}}
+{"timestamp":1712189144.4608817,"name":"online","context":{"idset":"86,110,184"}}
+{"timestamp":1712189144.626018,"name":"online","context":{"idset":"117,130,143,169,186"}}
+{"timestamp":1712189144.7257059,"name":"online","context":{"idset":"85,89,129,132,141,144,164"}}
+{"timestamp":1712189144.8861058,"name":"online","context":{"idset":"92,102,123,128,131,139,142,154-156,162,181"}}
+{"timestamp":1712189145.049552,"name":"online","context":{"idset":"104,108,115-116,119,133,137,148,150,153,158,161,166,179"}}
+{"timestamp":1712189145.2031887,"name":"online","context":{"idset":"105,113,134,136,140,159,163,165,168,170-171,175,178"}}
+{"timestamp":1712189145.3046439,"name":"online","context":{"idset":"145,147,167,173,182-183,187,190-192"}}
+{"timestamp":1712189145.406641,"name":"online","context":{"idset":"111,177,188-189"}}
+{"timestamp":1712189145.5754168,"name":"online","context":{"idset":"152,193"}}
+{"timestamp":1712189145.7911346,"name":"online","context":{"idset":"180"}}
+{"timestamp":1712189145.9827411,"name":"online","context":{"idset":"146"}}
+{"timestamp":1712189146.5073669,"name":"online","context":{"idset":"194"}}
+{"timestamp":1712189150.6295633,"name":"online","context":{"idset":"199"}}
+{"timestamp":1712189150.7413914,"name":"online","context":{"idset":"195"}}
+{"timestamp":1712189150.8925099,"name":"online","context":{"idset":"200"}}
+{"timestamp":1712189151.0170815,"name":"online","context":{"idset":"196"}}
+{"timestamp":1712189151.1298027,"name":"online","context":{"idset":"197,202,204,208"}}
+{"timestamp":1712189151.2480526,"name":"online","context":{"idset":"201,203"}}
+{"timestamp":1712189151.3113055,"name":"online","context":{"idset":"198,206,209"}}
+{"timestamp":1712189151.3781011,"name":"online","context":{"idset":"205,210-211"}}
+{"timestamp":1712189151.5001938,"name":"online","context":{"idset":"207,213"}}
+{"timestamp":1712189151.6166477,"name":"online","context":{"idset":"212,214-215"}}
+{"timestamp":1712189151.6305163,"name":"online","context":{"idset":"216"}}
+{"timestamp":1712189151.7099543,"name":"online","context":{"idset":"218"}}
+{"timestamp":1712189151.7817059,"name":"online","context":{"idset":"219"}}
+{"timestamp":1712189151.9195693,"name":"online","context":{"idset":"220-221"}}
+{"timestamp":1712189152.0966003,"name":"online","context":{"idset":"222"}}
+{"timestamp":1712189152.2608559,"name":"online","context":{"idset":"223"}}
+{"timestamp":1712189152.6170704,"name":"online","context":{"idset":"224"}}
+{"timestamp":1712189152.8319283,"name":"online","context":{"idset":"226-229"}}
+{"timestamp":1712189153.0011337,"name":"online","context":{"idset":"225"}}
+{"timestamp":1712189153.1676033,"name":"online","context":{"idset":"230-232"}}
+{"timestamp":1712189153.3307183,"name":"online","context":{"idset":"233-234,236-237"}}
+{"timestamp":1712189153.500526,"name":"online","context":{"idset":"235,239"}}
+{"timestamp":1712189153.7279058,"name":"online","context":{"idset":"240-241"}}
+{"timestamp":1712189153.7982974,"name":"online","context":{"idset":"238"}}
+{"timestamp":1712189153.9850392,"name":"online","context":{"idset":"242-244"}}
+{"timestamp":1712189154.1288815,"name":"online","context":{"idset":"247"}}
+{"timestamp":1712189154.2224028,"name":"online","context":{"idset":"245,248-249"}}
+{"timestamp":1712189154.3822229,"name":"online","context":{"idset":"246,251"}}
+{"timestamp":1712189154.4854305,"name":"online","context":{"idset":"250,253"}}
+{"timestamp":1712189154.5897412,"name":"online","context":{"idset":"254"}}
+{"timestamp":1712189154.6943023,"name":"online","context":{"idset":"252,256"}}
+{"timestamp":1712189154.8772898,"name":"online","context":{"idset":"255"}}
+{"timestamp":1712189155.0743811,"name":"online","context":{"idset":"257"}}
+{"timestamp":1712189155.1829622,"name":"online","context":{"idset":"258"}}
+{"timestamp":1712189156.064528,"name":"online","context":{"idset":"263"}}
+{"timestamp":1712189156.2616031,"name":"online","context":{"idset":"259-260"}}
+{"timestamp":1712189156.6874909,"name":"online","context":{"idset":"262,264"}}
+{"timestamp":1712189157.0554116,"name":"online","context":{"idset":"265"}}
+{"timestamp":1712189157.1574199,"name":"online","context":{"idset":"261,266"}}
+{"timestamp":1712189157.4549918,"name":"online","context":{"idset":"267"}}
+{"timestamp":1712189157.7667034,"name":"online","context":{"idset":"268-269,273-274"}}
+{"timestamp":1712189157.9451222,"name":"online","context":{"idset":"270-272,276,285"}}
+{"timestamp":1712189158.0996838,"name":"online","context":{"idset":"278,281"}}
+{"timestamp":1712189158.2721117,"name":"online","context":{"idset":"275,279-280,282-283,289,295"}}
+{"timestamp":1712189158.3518951,"name":"online","context":{"idset":"287,293"}}
+{"timestamp":1712189158.4645817,"name":"online","context":{"idset":"286,290,299"}}
+{"timestamp":1712189158.5721807,"name":"online","context":{"idset":"277,284,288,291-292,294,296,298,300,302-303,305"}}
+{"timestamp":1712189158.6606026,"name":"online","context":{"idset":"297,301,304,306,308,310-312,314-315"}}
+{"timestamp":1712189158.7779591,"name":"online","context":{"idset":"309,313,316-317"}}
+{"timestamp":1712189158.879823,"name":"online","context":{"idset":"307,318-323"}}
+{"timestamp":1712189160.3346982,"name":"online","context":{"idset":"324"}}
+{"timestamp":1712189160.6474683,"name":"online","context":{"idset":"325"}}
+{"timestamp":1712189160.7499831,"name":"online","context":{"idset":"327"}}
+{"timestamp":1712189160.8646643,"name":"online","context":{"idset":"326"}}
+{"timestamp":1712189161.1351991,"name":"online","context":{"idset":"328"}}
+{"timestamp":1712189161.15324,"name":"online","context":{"idset":"329"}}
+{"timestamp":1712189161.4696434,"name":"online","context":{"idset":"332"}}
+{"timestamp":1712189161.5627358,"name":"online","context":{"idset":"331,333"}}
+{"timestamp":1712189161.6795869,"name":"online","context":{"idset":"330,334"}}
+{"timestamp":1712189161.8923979,"name":"online","context":{"idset":"337,339-340"}}
+{"timestamp":1712189161.9777908,"name":"online","context":{"idset":"335-336,338,341-342,344"}}
+{"timestamp":1712189162.054935,"name":"online","context":{"idset":"345-346"}}
+{"timestamp":1712189162.0696962,"name":"online","context":{"idset":"343,347,350"}}
+{"timestamp":1712189162.1650882,"name":"online","context":{"idset":"349"}}
+{"timestamp":1712189162.3349688,"name":"online","context":{"idset":"351"}}
+{"timestamp":1712189162.4091694,"name":"online","context":{"idset":"352"}}
+{"timestamp":1712189162.5277674,"name":"online","context":{"idset":"353"}}
+{"timestamp":1712189162.613564,"name":"online","context":{"idset":"354-355"}}
+{"timestamp":1712189162.8102083,"name":"online","context":{"idset":"356-357"}}
+{"timestamp":1712189163.1995659,"name":"online","context":{"idset":"358,360-362"}}
+{"timestamp":1712189163.3778806,"name":"online","context":{"idset":"359,365-366"}}
+{"timestamp":1712189163.4904377,"name":"online","context":{"idset":"363-364"}}
+{"timestamp":1712189163.7089758,"name":"online","context":{"idset":"367-368"}}
+{"timestamp":1712189163.9136674,"name":"online","context":{"idset":"369"}}
+{"timestamp":1712189164.0603631,"name":"online","context":{"idset":"370"}}
+{"timestamp":1712189164.2111297,"name":"online","context":{"idset":"371"}}
+{"timestamp":1712189164.26875,"name":"online","context":{"idset":"372"}}
+{"timestamp":1712189164.5335212,"name":"online","context":{"idset":"373-375,377"}}
+{"timestamp":1712189164.62551,"name":"online","context":{"idset":"376"}}
+{"timestamp":1712189164.9150145,"name":"online","context":{"idset":"378,380"}}
+{"timestamp":1712189165.0105472,"name":"online","context":{"idset":"379,384"}}
+{"timestamp":1712189165.1759195,"name":"online","context":{"idset":"381-383,385-386"}}
+{"timestamp":1712189166.1121724,"name":"online","context":{"idset":"387-388"}}
+{"timestamp":1712189166.2623816,"name":"online","context":{"idset":"389"}}
+{"timestamp":1712189166.7393577,"name":"online","context":{"idset":"390-391"}}
+{"timestamp":1712189167.2226686,"name":"online","context":{"idset":"392"}}
+{"timestamp":1712189167.4674559,"name":"online","context":{"idset":"393,395"}}
+{"timestamp":1712189167.6867168,"name":"online","context":{"idset":"396"}}
+{"timestamp":1712189167.9039972,"name":"online","context":{"idset":"398"}}
+{"timestamp":1712189167.9758968,"name":"online","context":{"idset":"397,400"}}
+{"timestamp":1712189167.9906969,"name":"online","context":{"idset":"402"}}
+{"timestamp":1712189168.3103991,"name":"online","context":{"idset":"399"}}
+{"timestamp":1712189168.6680672,"name":"online","context":{"idset":"401,411"}}
+{"timestamp":1712189168.8491662,"name":"online","context":{"idset":"394,403-406,409-410,412,414-415"}}
+{"timestamp":1712189168.9509153,"name":"online","context":{"idset":"407-408"}}
+{"timestamp":1712189169.06498,"name":"online","context":{"idset":"413,416"}}
+{"timestamp":1712189169.3347673,"name":"online","context":{"idset":"417"}}
+{"timestamp":1712189169.591995,"name":"online","context":{"idset":"420,426"}}
+{"timestamp":1712189169.7433889,"name":"online","context":{"idset":"422"}}
+{"timestamp":1712189169.9065621,"name":"online","context":{"idset":"425"}}
+{"timestamp":1712189169.9847846,"name":"online","context":{"idset":"418,424"}}
+{"timestamp":1712189170.0556343,"name":"online","context":{"idset":"419,421"}}
+{"timestamp":1712189170.2165112,"name":"online","context":{"idset":"423"}}
+{"timestamp":1712189170.3214574,"name":"online","context":{"idset":"429,431"}}
+{"timestamp":1712189170.4866846,"name":"online","context":{"idset":"427-428,433-438"}}
+{"timestamp":1712189170.5667183,"name":"online","context":{"idset":"432,440,442,444"}}
+{"timestamp":1712189170.6370571,"name":"online","context":{"idset":"430,439,441,443"}}
+{"timestamp":1712189172.0133581,"name":"online","context":{"idset":"472-473"}}
+{"timestamp":1712189172.0942924,"name":"online","context":{"idset":"475"}}
+{"timestamp":1712189172.1150899,"name":"online","context":{"idset":"471"}}
+{"timestamp":1712189172.31726,"name":"online","context":{"idset":"470,474"}}
+{"timestamp":1712189172.4121187,"name":"online","context":{"idset":"476"}}
+{"timestamp":1712189172.4857967,"name":"online","context":{"idset":"479"}}
+{"timestamp":1712189172.6618063,"name":"online","context":{"idset":"469,477-478,481,483"}}
+{"timestamp":1712189172.7427638,"name":"online","context":{"idset":"482"}}
+{"timestamp":1712189172.8175509,"name":"online","context":{"idset":"480,485"}}
+{"timestamp":1712189173.0037999,"name":"online","context":{"idset":"484"}}
+{"timestamp":1712189173.176075,"name":"online","context":{"idset":"486-490"}}
+{"timestamp":1712189173.2931528,"name":"online","context":{"idset":"491,493"}}
+{"timestamp":1712189173.4056261,"name":"online","context":{"idset":"492,494,496-497"}}
+{"timestamp":1712189173.4888823,"name":"online","context":{"idset":"495,498-499"}}
+{"timestamp":1712189173.7740595,"name":"online","context":{"idset":"500,502"}}
+{"timestamp":1712189174.1611238,"name":"online","context":{"idset":"501,504"}}
+{"timestamp":1712189174.3344769,"name":"online","context":{"idset":"503,505"}}
+{"timestamp":1712189174.5849349,"name":"online","context":{"idset":"508,510"}}
+{"timestamp":1712189174.7136955,"name":"online","context":{"idset":"506-507,509,511"}}
+{"timestamp":1712189175.0757968,"name":"online","context":{"idset":"512-513"}}
+{"timestamp":1712189175.2984965,"name":"online","context":{"idset":"514,516,519"}}
+{"timestamp":1712189175.3789105,"name":"online","context":{"idset":"515,517,520,522"}}
+{"timestamp":1712189175.5654368,"name":"online","context":{"idset":"518,521"}}
+{"timestamp":1712189175.8663292,"name":"online","context":{"idset":"524"}}
+{"timestamp":1712189175.9443462,"name":"online","context":{"idset":"523"}}
+{"timestamp":1712189176.0600746,"name":"online","context":{"idset":"525"}}
+{"timestamp":1712189176.1614182,"name":"online","context":{"idset":"526"}}
+{"timestamp":1712189176.4856212,"name":"online","context":{"idset":"527-528"}}
+{"timestamp":1712189177.1541841,"name":"online","context":{"idset":"529"}}
+{"timestamp":1712189177.324595,"name":"online","context":{"idset":"530-531"}}
+{"timestamp":1712189177.495223,"name":"online","context":{"idset":"532"}}
+{"timestamp":1712189177.8700387,"name":"online","context":{"idset":"534,536"}}
+{"timestamp":1712189177.9954078,"name":"online","context":{"idset":"533,535"}}
+{"timestamp":1712189178.1956246,"name":"online","context":{"idset":"537"}}
+{"timestamp":1712189178.4613378,"name":"online","context":{"idset":"538,540"}}
+{"timestamp":1712189178.7744603,"name":"online","context":{"idset":"539"}}
+{"timestamp":1712189180.0385864,"name":"online","context":{"idset":"565"}}
+{"timestamp":1712189180.1406288,"name":"online","context":{"idset":"566"}}
+{"timestamp":1712189180.3483372,"name":"online","context":{"idset":"567"}}
+{"timestamp":1712189180.8470461,"name":"online","context":{"idset":"568,570"}}
+{"timestamp":1712189181.0102034,"name":"online","context":{"idset":"569"}}
+{"timestamp":1712189181.0738525,"name":"online","context":{"idset":"573"}}
+{"timestamp":1712189181.136584,"name":"online","context":{"idset":"574-575"}}
+{"timestamp":1712189181.267653,"name":"online","context":{"idset":"572,577"}}
+{"timestamp":1712189181.3498867,"name":"online","context":{"idset":"571,578,581"}}
+{"timestamp":1712189181.4426048,"name":"online","context":{"idset":"576,580,582,585"}}
+{"timestamp":1712189181.5101585,"name":"online","context":{"idset":"579,583,587-588"}}
+{"timestamp":1712189181.6026418,"name":"online","context":{"idset":"586"}}
+{"timestamp":1712189181.7400939,"name":"online","context":{"idset":"584"}}
+{"timestamp":1712189181.9944818,"name":"online","context":{"idset":"597-598,600"}}
+{"timestamp":1712189182.1633003,"name":"online","context":{"idset":"599,602-603"}}
+{"timestamp":1712189182.3291554,"name":"online","context":{"idset":"601,604"}}
+{"timestamp":1712189182.7018883,"name":"online","context":{"idset":"613-614"}}
+{"timestamp":1712189182.9709342,"name":"online","context":{"idset":"615"}}
+{"timestamp":1712189183.2395072,"name":"online","context":{"idset":"616-617,619"}}
+{"timestamp":1712189183.461345,"name":"online","context":{"idset":"620"}}
+{"timestamp":1712189183.6278102,"name":"online","context":{"idset":"618,621-623"}}
+{"timestamp":1712189183.7474303,"name":"online","context":{"idset":"624"}}
+{"timestamp":1712189183.8521707,"name":"online","context":{"idset":"625,627-628"}}
+{"timestamp":1712189184.0378408,"name":"online","context":{"idset":"626,629-630"}}
+{"timestamp":1712189184.1532588,"name":"online","context":{"idset":"631-633,635"}}
+{"timestamp":1712189184.36022,"name":"online","context":{"idset":"634,636"}}
+{"timestamp":1712189812.7981689,"name":"undrain","context":{"idset":"85-120,122-187,189-216,218-220,222,224-245,247-325,327-347,349-430,432-444,469-540,565-588,597-604,613-635"}}
+{"timestamp":1712190084.1469963,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1712190086.666708,"name":"online","context":{"idset":"160"}}
+{"timestamp":1712190768.7994049,"name":"undrain","context":{"idset":"1-60"}}
+{"timestamp":1712190808.5399854,"name":"undrain","context":{"idset":"246"}}
+{"timestamp":1712198623.6408765,"name":"resource-init","context":{"restart":true,"drain":{"61":{"timestamp":1711668444.9627461,"reason":"broker was unresponsive"},"62":{"timestamp":1711668506.9728317,"reason":"broker was unresponsive"},"63":{"timestamp":1711668445.0625052,"reason":"broker was unresponsive"},"64":{"timestamp":1711668506.9729121,"reason":"broker was unresponsive"},"65":{"timestamp":1711668438.9624472,"reason":"broker was unresponsive"},"66":{"timestamp":1711668506.9729731,"reason":"broker was unresponsive"},"67":{"timestamp":1711668439.0630105,"reason":"broker was unresponsive"},"68":{"timestamp":1711668506.9730346,"reason":"broker was unresponsive"},"69-76":{"timestamp":1711646931.6480982,"reason":""},"77":{"timestamp":1711668526.9621451,"reason":"broker was unresponsive"},"78":{"timestamp":1711668462.9610252,"reason":"broker was unresponsive"},"79":{"timestamp":1711668526.96223,"reason":"broker was unresponsive"},"80":{"timestamp":1711668457.0617182,"reason":"broker was unresponsive"},"81":{"timestamp":1711668526.9622741,"reason":"broker was unresponsive"},"82":{"timestamp":1711668462.9611192,"reason":"broker was unresponsive"},"83":{"timestamp":1711668527.0620673,"reason":"broker was unresponsive"},"84":{"timestamp":1711668463.0615458,"reason":"broker was unresponsive"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"348":{"timestamp":1711576490.2548447,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"431":{"timestamp":1711096934.358676,"reason":"nodediag failed pci"},"445-468,541-564":{"timestamp":1711560189.6111379,"reason":"CDU work TB"},"589":{"timestamp":1712154691.7782385,"reason":"broker was unresponsive"},"590":{"timestamp":1712154691.7894335,"reason":"broker was unresponsive"},"591":{"timestamp":1712154691.7896128,"reason":"broker was unresponsive"},"592":{"timestamp":1712154691.7897892,"reason":"broker was unresponsive"},"593":{"timestamp":1712154691.7899675,"reason":"broker was unresponsive"},"594":{"timestamp":1712154691.7901475,"reason":"broker was unresponsive"},"595":{"timestamp":1712154691.7903228,"reason":"broker was unresponsive"},"596":{"timestamp":1712154691.7904975,"reason":"broker was unresponsive"},"605":{"timestamp":1712154691.7921228,"reason":"broker was unresponsive"},"606":{"timestamp":1712154691.7922993,"reason":"broker was unresponsive"},"607":{"timestamp":1712154691.7924814,"reason":"broker was unresponsive"},"608":{"timestamp":1712154691.7926571,"reason":"broker was unresponsive"},"609":{"timestamp":1712154691.7928505,"reason":"broker was unresponsive"},"610":{"timestamp":1712154691.7930374,"reason":"broker was unresponsive"},"611":{"timestamp":1712154691.7932198,"reason":"broker was unresponsive"},"612":{"timestamp":1712154691.7934685,"reason":"broker was unresponsive"},"637":{"timestamp":1711642252.1616328,"reason":"broker was unresponsive"},"638":{"timestamp":1711642252.1617086,"reason":"broker was unresponsive"},"639":{"timestamp":1711642252.161751,"reason":"broker was unresponsive"},"640":{"timestamp":1711642252.1617904,"reason":"broker was unresponsive"},"641":{"timestamp":1711642252.1618268,"reason":"broker was unresponsive"},"642":{"timestamp":1711642252.1618621,"reason":"broker was unresponsive"},"643":{"timestamp":1711642252.1618974,"reason":"broker was unresponsive"},"644":{"timestamp":1711642252.1619322,"reason":"broker was unresponsive"},"646":{"timestamp":1711586857.201689,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"647":{"timestamp":1711587136.0002518,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"648":{"timestamp":1711587197.7047591,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"649":{"timestamp":1711587211.9606316,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"650":{"timestamp":1711587232.1780922,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"651":{"timestamp":1711587244.2040467,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"652":{"timestamp":1711587255.7559283,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"653":{"timestamp":1711642252.1619685,"reason":"broker was unresponsive"},"654":{"timestamp":1711642252.16201,"reason":"broker was unresponsive"},"655":{"timestamp":1711642252.162055,"reason":"broker was unresponsive"},"656":{"timestamp":1711642252.1620932,"reason":"broker was unresponsive"},"657":{"timestamp":1711642252.1621311,"reason":"broker was unresponsive"},"658":{"timestamp":1711642252.1621685,"reason":"broker was unresponsive"},"659":{"timestamp":1711642252.1622062,"reason":"broker was unresponsive"},"660":{"timestamp":1711642252.1622446,"reason":"broker was unresponsive"},"661":{"timestamp":1711642252.1622829,"reason":"broker was unresponsive"},"662":{"timestamp":1711642252.1623201,"reason":"broker was unresponsive"},"663":{"timestamp":1711642252.1623569,"reason":"broker was unresponsive"},"664":{"timestamp":1711642252.162394,"reason":"broker was unresponsive"},"665":{"timestamp":1711642252.1624658,"reason":"broker was unresponsive"},"666":{"timestamp":1711642252.1625173,"reason":"broker was unresponsive"},"667":{"timestamp":1711642258.156193,"reason":"broker was unresponsive"},"668":{"timestamp":1711642252.1625559,"reason":"broker was unresponsive"},"669":{"timestamp":1711642252.1625948,"reason":"broker was unresponsive"},"670":{"timestamp":1711642252.1626337,"reason":"broker was unresponsive"},"671":{"timestamp":1711642252.162673,"reason":"broker was unresponsive"},"672":{"timestamp":1711642252.1627119,"reason":"broker was unresponsive"},"673":{"timestamp":1711642252.1627512,"reason":"broker was unresponsive"},"674":{"timestamp":1711642252.1627915,"reason":"broker was unresponsive"},"675":{"timestamp":1711642252.1628311,"reason":"broker was unresponsive"},"676":{"timestamp":1711642252.1628704,"reason":"broker was unresponsive"},"677":{"timestamp":1711642258.1562793,"reason":"broker was unresponsive"},"678":{"timestamp":1711642258.156333,"reason":"broker was unresponsive"},"679":{"timestamp":1711642252.1629126,"reason":"broker was unresponsive"},"680":{"timestamp":1711642252.1629534,"reason":"broker was unresponsive"},"681":{"timestamp":1711642258.1563835,"reason":"broker was unresponsive"},"682":{"timestamp":1711642252.1629946,"reason":"broker was unresponsive"},"683":{"timestamp":1711642252.1630385,"reason":"broker was unresponsive"},"684":{"timestamp":1711642252.5597043,"reason":"broker was unresponsive"},"685":{"timestamp":1711604960.2562068,"reason":"broker was unresponsive"},"687":{"timestamp":1711604954.2552111,"reason":"broker was unresponsive"},"688":{"timestamp":1711606062.2558303,"reason":"broker was unresponsive"},"689":{"timestamp":1711604580.2552168,"reason":"broker was unresponsive"},"691":{"timestamp":1711604958.2564192,"reason":"broker was unresponsive"},"692":{"timestamp":1711606406.1556187,"reason":"broker was unresponsive"},"693":{"timestamp":1711636838.2555192,"reason":"broker was unresponsive"},"695":{"timestamp":1711604326.2549248,"reason":"broker was unresponsive"},"696":{"timestamp":1711606406.256216,"reason":"broker was unresponsive"},"697":{"timestamp":1711604392.2562771,"reason":"broker was unresponsive"},"698":{"timestamp":1711605716.2557297,"reason":"broker was unresponsive"},"699":{"timestamp":1711604948.2558951,"reason":"broker was unresponsive"},"700":{"timestamp":1711606868.2557678,"reason":"broker was unresponsive"},"701":{"timestamp":1711604340.2561386,"reason":"broker was unresponsive"},"702":{"timestamp":1711606408.2602317,"reason":"broker was unresponsive"},"703":{"timestamp":1711604918.2557685,"reason":"broker was unresponsive"},"704":{"timestamp":1711604590.2548223,"reason":"broker was unresponsive"},"705":{"timestamp":1711604968.2570915,"reason":"broker was unresponsive"},"707":{"timestamp":1711604908.2558758,"reason":"broker was unresponsive"},"686,690,694,706,708":{"timestamp":1711615908.1602454,"reason":"epilog failed for jobid fm9HsYwFjjd"},"709":{"timestamp":1711408620.9668899,"reason":"Cabinet power work needed -jrg"},"710":{"timestamp":1711409910.967391,"reason":"Cabinet power work needed -jrg"},"711":{"timestamp":1711409098.9674611,"reason":"Cabinet power work needed -jrg"},"712":{"timestamp":1711411606.9678445,"reason":"Cabinet power work needed -jrg"},"713":{"timestamp":1711408618.8669355,"reason":"Cabinet power work needed -jrg"},"715":{"timestamp":1711409132.9680476,"reason":"Cabinet power work needed -jrg"},"714,716":{"timestamp":1711564709.7804377,"reason":"Cabinet power work needed -jrg"},"718":{"timestamp":1711642210.1588371,"reason":"broker was unresponsive"},"719":{"timestamp":1711642210.1589563,"reason":"broker was unresponsive"},"720":{"timestamp":1711642210.1590128,"reason":"broker was unresponsive"},"721":{"timestamp":1711642210.1590643,"reason":"broker was unresponsive"},"722":{"timestamp":1711642210.1590991,"reason":"broker was unresponsive"},"723":{"timestamp":1711642210.1591821,"reason":"broker was unresponsive"},"724":{"timestamp":1711642210.1592195,"reason":"broker was unresponsive"},"725":{"timestamp":1711593222.2542431,"reason":"broker was unresponsive"},"726":{"timestamp":1711593312.2549782,"reason":"broker was unresponsive"},"727":{"timestamp":1711593332.2552569,"reason":"broker was unresponsive"},"728":{"timestamp":1711593356.2547033,"reason":"broker was unresponsive"},"729":{"timestamp":1711593414.254441,"reason":"broker was unresponsive"},"730":{"timestamp":1711593462.2552798,"reason":"broker was unresponsive"},"731":{"timestamp":1711593510.2550235,"reason":"broker was unresponsive"},"732":{"timestamp":1711593530.2558599,"reason":"broker was unresponsive"},"733":{"timestamp":1711642210.1592536,"reason":"broker was unresponsive"},"734":{"timestamp":1711486000.9676955,"reason":"broker was unresponsive"},"735":{"timestamp":1711642210.1592872,"reason":"broker was unresponsive"},"736":{"timestamp":1711642210.1593204,"reason":"broker was unresponsive"},"737":{"timestamp":1711642210.1593742,"reason":"broker was unresponsive"},"738":{"timestamp":1711642210.1594355,"reason":"broker was unresponsive"},"739":{"timestamp":1711642210.1594796,"reason":"broker was unresponsive"},"740":{"timestamp":1711642210.159517,"reason":"broker was unresponsive"},"741":{"timestamp":1711642210.1595554,"reason":"broker was unresponsive"},"742":{"timestamp":1711642210.1595943,"reason":"broker was unresponsive"},"743":{"timestamp":1711642210.1596365,"reason":"broker was unresponsive"},"744":{"timestamp":1711642210.1596746,"reason":"broker was unresponsive"},"745":{"timestamp":1711642210.1597111,"reason":"broker was unresponsive"},"746":{"timestamp":1711506258.2527435,"reason":"broker was unresponsive"},"747":{"timestamp":1711642210.1597559,"reason":"broker was unresponsive"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"},"749":{"timestamp":1711642210.1598115,"reason":"broker was unresponsive"},"750":{"timestamp":1711642210.1598501,"reason":"broker was unresponsive"},"751":{"timestamp":1711642210.159888,"reason":"broker was unresponsive"},"752":{"timestamp":1711642210.1599293,"reason":"broker was unresponsive"},"753":{"timestamp":1711642210.1599746,"reason":"broker was unresponsive"},"754":{"timestamp":1711642210.1600144,"reason":"broker was unresponsive"},"755":{"timestamp":1711642210.1600802,"reason":"broker was unresponsive"},"756":{"timestamp":1711642210.5039413,"reason":"broker was unresponsive"},"789-796":{"timestamp":1711461777.1299369,"reason":""},"809":{"timestamp":1711667251.2030878,"reason":"broker was unresponsive"},"810":{"timestamp":1711460371.1431019,"reason":""},"843":{"timestamp":1712088721.1281126,"reason":"broker was unresponsive"},"844":{"timestamp":1712088721.1284544,"reason":"broker was unresponsive"},"869":{"timestamp":1712154691.8909535,"reason":"broker was unresponsive"},"870":{"timestamp":1712154691.9068179,"reason":"broker was unresponsive"},"874":{"timestamp":1711668507.0706048,"reason":"broker was unresponsive"},"875":{"timestamp":1711668503.0621645,"reason":"broker was unresponsive"},"881":{"timestamp":1712154691.9234848,"reason":"broker was unresponsive"},"882":{"timestamp":1712154691.9423342,"reason":"broker was unresponsive"},"883-884":{"timestamp":1711734121.1560712,"reason":""},"965":{"timestamp":1711663706.9630723,"reason":"broker was unresponsive"},"966":{"timestamp":1711663706.9631405,"reason":"broker was unresponsive"},"967":{"timestamp":1711663706.963171,"reason":"broker was unresponsive"},"968":{"timestamp":1711663706.9631999,"reason":"broker was unresponsive"},"977":{"timestamp":1712154691.9848833,"reason":"broker was unresponsive"},"9989":{"timestamp":1712154691.9938779,"reason":"broker was unresponsive"},"10037":{"timestamp":1712154692.0301805,"reason":"broker was unresponsive"},"10059":{"timestamp":1712154692.0507491,"reason":"broker was unresponsive"},"10064":{"timestamp":1712154692.0554938,"reason":"broker was unresponsive"},"10135":{"timestamp":1712154692.1007507,"reason":"broker was unresponsive"},"10136":{"timestamp":1712154692.1011977,"reason":"broker was unresponsive"},"10138":{"timestamp":1712154692.1016693,"reason":"broker was unresponsive"},"10143":{"timestamp":1712154692.1036949,"reason":"broker was unresponsive"},"10145":{"timestamp":1712154692.1046801,"reason":"broker was unresponsive"},"10147":{"timestamp":1712154692.1056852,"reason":"broker was unresponsive"},"10148":{"timestamp":1712154692.1061764,"reason":"broker was unresponsive"},"10154":{"timestamp":1711600478.1543531,"reason":"broker was unresponsive"},"10162":{"timestamp":1711742687.1360803,"reason":"nodediag failed cxi"},"10183":{"timestamp":1711600556.2563355,"reason":"broker was unresponsive"},"10207":{"timestamp":1712154692.1412749,"reason":"broker was unresponsive"},"10261":{"timestamp":1712154692.1708262,"reason":"broker was unresponsive"},"10301":{"timestamp":1712096585.8629279,"reason":"Bad CXI Link --JRG"},"10302":{"timestamp":1712096599.7608726,"reason":"bad partner --JRG"},"10304":{"timestamp":1712154692.18731,"reason":"broker was unresponsive"},"10469":{"timestamp":1712096633.8221953,"reason":"--reason HP Diags"},"10475":{"timestamp":1712096681.148495,"reason":"--reason repair cxi1 issue HP"},"10478-10480,10482,10484":{"timestamp":1712096658.8781381,"reason":"--reason HP Diags"},"10504":{"timestamp":1712154692.2964497,"reason":"broker was unresponsive"},"10517":{"timestamp":1712154692.3034911,"reason":"broker was unresponsive"},"10509,10518":{"timestamp":1712086791.4119098,"reason":"CXI Issues --JRG"},"10510,10519":{"timestamp":1712086831.8068576,"reason":"Bad partner node --JRG"},"10541":{"timestamp":1712154692.3177123,"reason":"broker was unresponsive"},"10557":{"timestamp":1712154692.3325636,"reason":"broker was unresponsive"},"10558":{"timestamp":1712154692.3333406,"reason":"broker was unresponsive"},"10582":{"timestamp":1712154692.3531559,"reason":"broker was unresponsive"},"10602":{"timestamp":1712154692.3681419,"reason":"broker was unresponsive"},"10871":{"timestamp":1712087815.0916734,"reason":"broker was unresponsive"},"10873":{"timestamp":1712154692.4582243,"reason":"broker was unresponsive"},"10874":{"timestamp":1712154692.4596572,"reason":"broker was unresponsive"},"10883":{"timestamp":1712154692.4696405,"reason":"broker was unresponsive"},"10884":{"timestamp":1712154692.4704447,"reason":"broker was unresponsive"},"10886":{"timestamp":1712103808.051157,"reason":"epilog failed for jobid fnFWHjJ8JpF"},"10905":{"timestamp":1712095015.6494093,"reason":"nodediag failed cxi"},"10906":{"timestamp":1712094978.8228164,"reason":"nodediag failed cxi"},"10935":{"timestamp":1712154692.5049505,"reason":"broker was unresponsive"},"10936":{"timestamp":1712154692.5056741,"reason":"broker was unresponsive"},"10937":{"timestamp":1712154692.506387,"reason":"broker was unresponsive"},"10943":{"timestamp":1712154692.5109775,"reason":"broker was unresponsive"},"10976":{"timestamp":1712154692.5515492,"reason":"broker was unresponsive"},"11126":{"timestamp":1711126278.6191533,"reason":"ama fail"},"11130":{"timestamp":1712079046.9970613,"reason":"broker was unresponsive"},"11134":{"timestamp":1712079046.9974139,"reason":"broker was unresponsive"},"11136":{"timestamp":1712079046.9975903,"reason":"broker was unresponsive"},"11138":{"timestamp":1712079046.9977582,"reason":"broker was unresponsive"},"11140":{"timestamp":1712079046.997932,"reason":"broker was unresponsive"},"11173":{"timestamp":1712095025.0662155,"reason":"epilog failed for jobid fnFN1DHfSM5"},"11178":{"timestamp":1712154692.6051571,"reason":"broker was unresponsive"},"11187":{"timestamp":1712154692.6113768,"reason":"broker was unresponsive"},"11358":{"timestamp":1712154692.7509179,"reason":"broker was unresponsive"},"11373":{"timestamp":1712154692.7643514,"reason":"broker was unresponsive"},"11390":{"timestamp":1712154692.7781432,"reason":"broker was unresponsive"},"11395":{"timestamp":1712154692.7823181,"reason":"broker was unresponsive"},"11429":{"timestamp":1712154692.8102601,"reason":"broker was unresponsive"},"11431":{"timestamp":1712154692.8120186,"reason":"broker was unresponsive"},"11440":{"timestamp":1712154692.8191555,"reason":"broker was unresponsive"},"11441":{"timestamp":1712154692.8200519,"reason":"broker was unresponsive"},"11442":{"timestamp":1712154692.8209813,"reason":"broker was unresponsive"},"11455":{"timestamp":1712154692.8324704,"reason":"broker was unresponsive"},"11464":{"timestamp":1712154692.8396258,"reason":"broker was unresponsive"},"11527":{"timestamp":1712043086.7846026,"reason":"broker was unresponsive"},"11563":{"timestamp":1712154692.9307995,"reason":"broker was unresponsive"},"11587":{"timestamp":1711663526.290566,"reason":"kernel panics - JRG"},"11588":{"timestamp":1712093194.2606094,"reason":"bad partner node - JRG"},"11616":{"timestamp":1712154693.0464704,"reason":"broker was unresponsive"},"11617":{"timestamp":1712154693.0472839,"reason":"broker was unresponsive"},"11618":{"timestamp":1712154693.0482194,"reason":"broker was unresponsive"},"11619":{"timestamp":1712026940.6852841,"reason":"broker was unresponsive"},"11620":{"timestamp":1712154693.0500429,"reason":"broker was unresponsive"},"11621":{"timestamp":1712154693.0508971,"reason":"broker was unresponsive"},"11622":{"timestamp":1712154693.0517426,"reason":"broker was unresponsive"},"11623":{"timestamp":1712154693.0525949,"reason":"broker was unresponsive"},"11624":{"timestamp":1712154693.0546894,"reason":"broker was unresponsive"},"11625":{"timestamp":1712154693.0555315,"reason":"broker was unresponsive"},"11626":{"timestamp":1712154693.0564125,"reason":"broker was unresponsive"},"11627":{"timestamp":1712154693.0572789,"reason":"broker was unresponsive"},"11628":{"timestamp":1712154693.058116,"reason":"broker was unresponsive"},"11629":{"timestamp":1712154693.0590928,"reason":"broker was unresponsive"},"11630":{"timestamp":1712154693.0598869,"reason":"broker was unresponsive"},"11631":{"timestamp":1712026940.6853518,"reason":"broker was unresponsive"},"11632":{"timestamp":1712154693.0617664,"reason":"broker was unresponsive"},"11633":{"timestamp":1712026940.7842064,"reason":"broker was unresponsive"},"11634":{"timestamp":1712154693.0629814,"reason":"broker was unresponsive"},"11635":{"timestamp":1712154693.0641026,"reason":"broker was unresponsive"},"11636":{"timestamp":1712154693.0652032,"reason":"broker was unresponsive"},"11653":{"timestamp":1712174466.0412419,"reason":"nodediag failed cxi"},"11654":{"timestamp":1712174483.5145023,"reason":"nodediag failed cxi"},"11655":{"timestamp":1712174485.5970631,"reason":"nodediag failed cxi"},"11656":{"timestamp":1712174464.6082194,"reason":"nodediag failed cxi"},"11657":{"timestamp":1712174465.4122262,"reason":"nodediag failed cxi"},"11658":{"timestamp":1712174466.6267278,"reason":"nodediag failed cxi"},"11659":{"timestamp":1712174468.6355774,"reason":"nodediag failed cxi"},"11660":{"timestamp":1712174456.1854422,"reason":"nodediag failed cxi"},"11676":{"timestamp":1712174838.6317751,"reason":"nodediag failed cxi"},"11765-11780":{"timestamp":1711993308.7730825,"reason":"Diags Running"},"11784":{"timestamp":1711725755.0632687,"reason":"Running Diags - Rabbits Off"},"11790":{"timestamp":1712004602.484952,"reason":"bad partner node - JRG"},"11789,11793":{"timestamp":1712004602.484952,"reason":"kernel panics - JRG"},"11794":{"timestamp":1711725767.0656931,"reason":"bad partner node - JRG"},"11781-11783,11785-11788,11791-11792,11795-11796":{"timestamp":1712004602.484952,"reason":"Running Diags - Rabbits Off"},"11803":{"timestamp":1711668961.0618374,"reason":"Running Diags - Rabbits Off"},"11797-11801,11804-11811":{"timestamp":1712004859.9543238,"reason":"Running Diags - Rabbits Off"},"11812":{"timestamp":1712004514.6037207,"reason":"Running Diags - Rabbits Off"},"11813-11828":{"timestamp":1712005088.7732615,"reason":"Running Diags - Rabbits Off"},"11829-11844":{"timestamp":1712068080.9443812,"reason":"Running Diags - Rabbits Off"},"11845-11859":{"timestamp":1712068141.3627269,"reason":"Running Diags - Rabbits Off"},"11860":{"timestamp":1711727735.0622723,"reason":"Running Diags - Rabbits Off"},"11861-11876":{"timestamp":1712068186.5559592,"reason":"Running Diags - Rabbits Off"},"11877":{"timestamp":1712101843.9122288,"reason":"broker was unresponsive"},"11878":{"timestamp":1712101843.9123399,"reason":"broker was unresponsive"},"11879":{"timestamp":1712101844.0117874,"reason":"broker was unresponsive"},"11880":{"timestamp":1712101849.9135923,"reason":"broker was unresponsive"},"11881":{"timestamp":1712101849.9136949,"reason":"broker was unresponsive"},"11882":{"timestamp":1712101849.9137807,"reason":"broker was unresponsive"},"11883":{"timestamp":1712101849.913851,"reason":"broker was unresponsive"},"11884":{"timestamp":1712101849.9139225,"reason":"broker was unresponsive"},"11885":{"timestamp":1712101849.9139903,"reason":"broker was unresponsive"},"11886":{"timestamp":1712101849.9140527,"reason":"broker was unresponsive"},"11887":{"timestamp":1712101849.9141397,"reason":"broker was unresponsive"},"11888":{"timestamp":1712101849.9142251,"reason":"broker was unresponsive"},"11889":{"timestamp":1712101849.9143026,"reason":"broker was unresponsive"},"11890":{"timestamp":1712101850.048907,"reason":"broker was unresponsive"},"11892":{"timestamp":1712097564.0118735,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1712198623.6465318,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1712198639.4913704,"name":"online","context":{"idset":"0"}}
+{"timestamp":1712198641.1991713,"name":"online","context":{"idset":"11541"}}
+{"timestamp":1712198641.2195597,"name":"online","context":{"idset":"9331,10003,10062,10115,10187,10189,10294,11211,11271,11681,11885"}}
+{"timestamp":1712198641.3273065,"name":"online","context":{"idset":"974,9973,10036,10116,10280,10335,10399,11032,11049,11198,11359,11378,11803,11814"}}
+{"timestamp":1712198641.4365275,"name":"online","context":{"idset":"951,9984,10020,10121,10150,10199,10265,10292-10293,10347,11015,11197,11201,11223,11248,11309,11340,11417,11522-11523,11682,11685,11694,11768,11826,11850,11873"}}
+{"timestamp":1712198641.525492,"name":"online","context":{"idset":"9219,9252,9289,9313,10018,10076,10220,10245,10263,10270,10309,10337,10387,11021,11052,11065,11257,11260,11410,11483,11486,11489,11557,11618,11642,11645,11650,11747,11759,11805,11847"}}
+{"timestamp":1712198641.6428561,"name":"online","context":{"idset":"970,975,9205,9213,9230,9237,9240,9242,9247-9248,9260,9269,9274,9276,9283,9294,9298,9307-9308,9315-9316,9321,9324,9977,9982-9983,9985,9990,9994,10019,10021,10027,10033-10035,10047,10056-10057,10065,10067,10074,10086,10091,10094-10095,10103-10104,10125,10128-10129,10142,10157,10159,10165,10167,10172,10178,10203,10208,10213,10219,10223,10234-10235,10237,10281,10291,10298,10307,10316,10318,10320-10321,10324-10325,10328,10354,10363-10364,10367,10373,10378,10390,10401,10407,10419-10420,10430-10431,10433,10438,10446,10454,11019,11035,11050,11061,11064,11070,11075,11084,11088,11105,11117,11160,11170,11189,11214,11224,11227,11233,11236,11239-11240,11249,11261-11262,11275,11280-11282,11286,11293,11299-11300,11327,11332,11334,11347-11348,11361-11363,11365,11369,11375,11377,11379-11380,11382,11402,11413,11426,11446,11454,11456,11462,11475,11496,11516,11520-11521,11531,11538,11544,11548-11549,11554,11560,11573,11580,11585,11591,11612,11619,11623,11631,11637-11638,11649,11654,11656,11660,11665,11671,11689,11700,11703-11705,11716-11717,11722,11724,11734,11737-11740,11742,11744,11749,11756,11761,11778,11780,11783,11787,11795,11804,11809-11811,11825,11840,11851,11857-11858,11860,11870,11875,11877,11886,11888"}}
+{"timestamp":1712198641.7351353,"name":"online","context":{"idset":"971,978,980,9244,9246,9257,9261,9270,9291,9300,9303,9980,9987,9991-9992,10001,10004,10010-10011,10022,10026,10029,10038,10042,10051,10068,10075,10077-10078,10084,10099,10118,10123,10127,10133-10134,10139,10141,10156,10177,10180,10185,10188,10191,10194,10196,10209,10215,10221-10222,10226,10230,10236,10243-10244,10246,10248,10250,10254,10257,10266,10269,10274,10289,10300,10315,10332,10362,10371,10376,10379-10380,10382-10383,10394,10403,10406,10414-10415,10429,10432,10436,10449,10456-10457,10998,11010,11013,11020,11046,11060,11079,11095,11103,11157,11182,11184,11186,11202,11229,11234,11254,11256,11289,11305,11316,11322,11344,11357,11403-11404,11421,11435,11449,11467-11468,11473,11485,11511,11513,11552,11562,11568-11569,11586,11600-11601,11603,11608,11624-11625,11633,11647,11662,11674-11675,11684,11697-11698,11712,11733,11741,11758,11760,11769,11771,11797,11802,11813,11815,11831,11843,11855,11868,11872,11876,11879,11882"}}
+{"timestamp":1712198641.8318689,"name":"online","context":{"idset":"979,9206,9210-9211,9215-9218,9223-9224,9227-9228,9232-9233,9245,9251,9253-9255,9258,9263-9264,9266,9272-9273,9275,9284,9288,9290,9292,9296-9297,9302,9306,9309-9311,9314,9319-9320,9325-9328,9332,9974,9981,9986,9988,9993,9995,9997,10009,10013-10017,10024-10025,10039,10043-10044,10052-10054,10060,10063,10069,10071-10072,10081,10087,10096,10098,10106,10112,10119-10120,10131,10144,10155,10164,10168,10171,10173,10175,10181,10186,10197,10212,10217,10225,10227-10228,10232,10239-10241,10258,10260,10264,10272,10282-10283,10285,10287-10288,10290,10303,10305,10308,10322-10323,10329-10331,10334,10338,10341-10344,10349-10350,10353,10355,10359,10361,10366,10369,10372,10377,10388-10389,10391,10397-10398,10405,10410,10412,10417-10418,10422-10423,10425-10428,10435,10439,10443,10445,10447,10450-10451,10453,10455,11000,11002,11004,11006,11008,11011,11016-11017,11022,11024,11028,11030,11034,11039-11042,11045,11047,11053-11054,11057,11059,11063,11067,11076-11078,11080-11081,11090,11092-11093,11096-11098,11104,11106,11111-11112,11133,11139,11141,11143,11152,11156,11159,11161,11163,11168,11172,11179,11190-11191,11194,11199-11200,11204-11205,11208,11216,11219,11225,11228,11238,11242-11243,11247,11250,11252,11255,11259,11263,11267-11268,11270,11274,11276-11279,11284-11285,11288,11290,11295-11296,11301,11306-11307,11314,11319,11321,11323,11328,11331,11342-11343,11350,11352-11353,11356,11360,11367-11368,11370-11372,11374,11385,11400,11412,11416,11418-11419,11422,11424-11425,11432-11434,11436,11444-11445,11448,11451-11453,11469,11474,11477,11479,11482,11484,11492,11494-11495,11497,11499,11502,11508,11510,11514,11524,11526,11529,11547,11550,11553,11555,11561,11564,11566,11570-11571,11574,11576-11577,11583-11584,11592,11594,11596,11598,11605-11606,11610-11611,11614,11621-11622,11629-11630,11636,11639,11644,11657,11659,11661,11664,11666,11678-11680,11686-11688,11690-11691,11695,11699,11708-11711,11719,11723,11731-11732,11735,11743,11750-11751,11755,11775-11777,11779,11785,11788,11791,11798-11799,11801,11808,11821,11823,11827,11833,11837-11838,11844,11848-11849,11852-11854,11869,11871,11878,11883"}}
+{"timestamp":1712198641.8480136,"name":"online","context":{"idset":"841,972,9225,9229,9236,9241,9250,9262,9279,9282,9286,9295,9329,9975,9978,9998-9999,10061,10080,10101,10107,10110,10130,10137,10146,10152,10182,10200-10201,10218,10256,10259,10262,10275,10277,10310,10336,10348,10356,10358,10404,10413,10421,10996,11001,11014,11038,11044,11051,11062,11074,11083,11107-11108,11119,11164,11185,11203,11210,11213,11251,11273,11283,11298,11311,11313,11338-11339,11376,11384,11415,11438,11480,11493,11506,11533-11534,11567,11616-11617,11620,11626,11668,11754,11784,11796,11830,11864-11865"}}
+{"timestamp":1712198641.9157729,"name":"online","context":{"idset":"952,969,9256,9299,9322,9976,9979,10005,10028,10030,10040,10045-10046,10049,10055,10070,10083,10092-10093,10100,10105,10124,10126,10160,10174,10176,10179,10198,10249,10268,10284,10286,10296,10299,10314,10319,10333,10345,10352,10360,10375,10386,10402,10409,10416,10440,11043,11058,11068,11073,11085,11089,11091,11094,11109-11110,11144,11147-11148,11153,11155,11158,11167,11180,11183,11188,11195-11196,11206-11207,11212,11230,11232,11244,11258,11294,11317,11320,11326,11333,11351,11354,11366,11381,11386,11406,11411,11420,11423,11437,11443,11463,11465,11471,11476,11488,11490,11505,11507,11515,11525,11528,11530,11542,11546,11609,11628,11635,11640,11652-11653,11673,11701,11706,11714,11718,11720-11721,11730,11765,11773,11800,11820,11856,11863,11874"}}
+{"timestamp":1712198641.9257865,"name":"online","context":{"idset":"11215"}}
+{"timestamp":1712198642.0377305,"name":"online","context":{"idset":"117,123,254,431,973,9208-9209,9212,9220,9226,9235,9249,9259,9267-9268,9281,9287,9293,9301,9312,9317-9318,9323,9330,10002,10007,10012,10023,10031,10048,10058,10088,10090,10097,10109,10111,10113-10114,10117,10140,10149,10151,10166,10170,10190,10192-10193,10195,10210,10214,10216,10224,10229,10233,10251,10253,10267,10271,10273,10276,10278,10306,10313,10317,10326,10357,10365,10368,10370,10374,10381,10384-10385,10392-10393,10396,10400,10437,10441-10442,10444,10448,10452,10458,10997,11005,11007,11025-11027,11031,11033,11037,11048,11056,11071,11101,11113-11114,11137,11145-11146,11162,11166,11175-11177,11181,11193,11217,11237,11264,11266,11269,11272,11292,11302,11318,11335,11349,11405,11408,11414,11427,11430,11450,11457-11458,11470,11478,11498,11501,11503,11517-11518,11532,11535,11537,11539,11556,11572,11578,11581,11590,11595,11597,11615,11634,11641,11646,11651,11655,11663,11672,11702,11713,11727,11746,11753,11767,11770,11772,11792,11806,11817,11822,11824,11828,11832,11834-11835,11842,11862,11880,11889"}}
+{"timestamp":1712198642.0725029,"name":"online","context":{"idset":"85,87,92,95-96,100,105,114,120,130-132,134,136,142,146-147,152,156,161,165-166,175-177,182,187,190,192,197,203,206,208,212-213,216,218,221,224-228,235-239,244-245,248,251-253,255,259,262,266-267,270-271,275-276,283,288,292-293,295,303-304,306,308,310,312,318-319,322,330,335-336,342,344,346,349,352-353,355-356,358,360,363,365,368,378-380,386,393,406,409,411,417,419,423-424,427-430,433-434,436-437,440,442-444,469-470,474-477,483,488,490-492,494,496,498,500,502,504,514,526,528-529,533,535,539,567-568,570,572,575-577,579-580,582-584,588,597-598,602,619,629,631,11312"}}
+{"timestamp":1712198642.2074933,"name":"online","context":{"idset":"91,93-94,99,101-104,107-108,110-113,115,119,122,124-128,135,138-140,143-145,148,150-151,153,157-160,164,167-171,174,178,180-181,183-186,191,193-194,196,198-202,204-205,207,210,214-215,219-220,223,229,231,233-234,240-242,246,249,256-258,260-261,263-265,269,274,277,280-282,284-286,290,294,297-302,305,307,309,311,313-316,321,323,325,327-328,332-334,338-341,343,345,347,350,357,359,361,367,369-373,376,381-384,388-392,394-395,397-398,400-405,407-408,410,412,415-416,420,422,426,432,435,438-439,441,471-472,478,480,482,486-487,489,493,495,497,499,501,503,506,508-511,513,515-516,519,521-524,527,530-532,534,536-538,540,565-566,569,571,574,578,586-587,600-601,604,613-617,620-623,625-626,628,630,634,636,10894,10901,11394"}}
+{"timestamp":1712198642.4145815,"name":"online","context":{"idset":"635,10866-10867,10922,10948,10961,10988,11086,11125,11131,11397,11602,11752,11763,11774,11781,11812,11892"}}
+{"timestamp":1712198642.5199411,"name":"online","context":{"idset":"10,9222,9231,9238,9271,9280,9305,10000,10006,10008,10041,10102,10108,10122,10132,10158,10163,10169,10184,10242,10247,10252,10279,10295,10312,10327,10346,10351,10395,10411,10857,10864,10875,10879,10881-10882,10897,10903-10904,10920,10931,10960,10962,11087,11127,11129,11388,11551,11589,11627,11676,11692,11715,11725-11726,11728-11729,11736,11818,11829,11845,11859,11861,11887,11890"}}
+{"timestamp":1712198642.6420996,"name":"online","context":{"idset":"1-3,5-9,11-24,26-28,49,10939,10966"}}
+{"timestamp":1712198642.7492185,"name":"online","context":{"idset":"25,30-32,35-47,50-54,56,58,60,97,154,189,232,278,485,9207,9221,9234,9243,9265,9277-9278,9304,9996,10032,10050,10066,10079,10082,10089,10161,10202,10204,10211,10231,10238,10297,10311,10339,10424,10434,10835-10836,10838,10847-10848,10850,10858,10860-10861,10865,10869-10870,10887,10890,10895-10896,10913,10916-10917,10919,10925,10930,10940,10942,10947,10965,10970-10972,10977,10981-10982,10985-10987,11120,11122,11128,11132,11392,11559,11582,11593,11607,11677,11683,11693,11696,11762,11764,11766,11782,11786,11819,11846,11867,11881"}}
+{"timestamp":1712198642.869122,"name":"online","context":{"idset":"109,243,291,399,512,517,9285,10085,10255,10893,10899,10908,10918,10946,10959,10980,10983-10984,10991-10992,11398,11599,11604,11745,11757,11836"}}
+{"timestamp":1712198642.9726131,"name":"online","context":{"idset":"10837,10929,10993,11393,11707,11866"}}
+{"timestamp":1712198643.075717,"name":"online","context":{"idset":"90,98,118,149,172,179,209,222,230,250,268,272-273,296,317,326,331,337,354,362,364,366,374-375,377,385,387,413,421,479,481,484,518,520,525,573,581,585,603,618,632-633,976,10073,10340,10408,10833,10840-10842,10851,10854-10855,10862-10863,10878,10885,10888-10889,10892,10900,10907,10909-10910,10915,10923,10926-10928,10932,10934,10941,10945,10949,10951,10953,10956-10957,10963,10968,10973-10975,10990,11082,11123,11389,11519,11558,11658"}}
+{"timestamp":1712198643.2646096,"name":"online","context":{"idset":"88-89,116,137,141,155,162,173,188,195,211,247,289,320,324,329,414,418,507,624,627,10839,10843,10845,10853,10868,10877,10933,10944,10950,10967,10969"}}
+{"timestamp":1712198643.3785491,"name":"online","context":{"idset":"106,133,163,279,287,351,425,599,10849,10859,10876,10891,10898,10902,10938,10979,11396"}}
+{"timestamp":1712198643.5538237,"name":"online","context":{"idset":"129,396,473,505,10832,10914,11399"}}
+{"timestamp":1712198643.6714232,"name":"online","context":{"idset":"10921,10924,10954,10989"}}
+{"timestamp":1712198643.8517821,"name":"online","context":{"idset":"55,57,10550,10784,10834,10844,10856,10872,10955"}}
+{"timestamp":1712198644.0198519,"name":"online","context":{"idset":"86,10474,10515,10596,10626,10640,10714"}}
+{"timestamp":1712198644.1354527,"name":"online","context":{"idset":"10498,10520,10528,10648,10688,10764,10806"}}
+{"timestamp":1712198644.2384806,"name":"online","context":{"idset":"10463,10568,10583,10651,10665,10719,10749,10757"}}
+{"timestamp":1712198644.349905,"name":"online","context":{"idset":"10465,10512,10529,10536,10561-10562,10588,10601,10623,10650,10756,10779,10808,10821,10825,10852"}}
+{"timestamp":1712198644.4641008,"name":"online","context":{"idset":"10459,10496,10513,10531-10532,10552-10553,10564,10579,10581,10590,10606,10636,10649,10653,10670,10676,10682,10689-10690,10706,10717,10726,10728,10734,10743,10754,10767,10773,10778,10781,10790-10791,10799,10807,10813,10831,10978,10995,11121,11391"}}
+{"timestamp":1712198644.574481,"name":"online","context":{"idset":"10483,10495,10497,10535,10546-10547,10565,10567,10570,10585,10594-10595,10604,10624,10632,10652,10661,10686,10697,10704,10725,10737,10745,10753,10766,10769,10788,10794-10795,10802-10803,10815,10880,10958"}}
+{"timestamp":1712198644.7297447,"name":"online","context":{"idset":"10524,10543,10576,10666,10677,10691,10708,10826,10846,10994"}}
+{"timestamp":1712198644.8408625,"name":"online","context":{"idset":"10477,10486,10544,10616,10746,10772,10792,10952"}}
+{"timestamp":1712198644.9433897,"name":"online","context":{"idset":"33,10476,10500,10503,10573,10586-10587,10591,10603,10613,10619,10633,10641,10646,10667,10675,10699-10700,10713,10720,10727,10735,10747,10750,10759,10789,10817,10824,10964"}}
+{"timestamp":1712198645.0461268,"name":"online","context":{"idset":"4,10467,10471,10533,10549,10556,10560,10607,10611,10622,10630,10658,10678,10696,10710,10723,10730,10776,10800,10804,10811,10820"}}
+{"timestamp":1712198645.2280829,"name":"online","context":{"idset":"29,34,48,59,10460,10485,10499,10507,10540,10548,10569,10577,10609,10612,10645,10659,10672,10698,10709,10711,10716,10721,10724,10732-10733,10748,10751,10755,10758,10760,10770,10782-10783,10798,10810,10822-10823,10829"}}
+{"timestamp":1712198645.3009853,"name":"online","context":{"idset":"10462,10470,10487,10494,10506,10537,10592,10656,10668-10669,10673,10680,10692,10722,10736,10738,10741-10742,10761,10774,10797,10812"}}
+{"timestamp":1712198645.3166659,"name":"online","context":{"idset":"10472,10481,10617,10662,10715,10762,10801,10814"}}
+{"timestamp":1712198645.4029307,"name":"online","context":{"idset":"10501,10530,10534,10545,10578,10589,10598,10605,10627,10674,10681,10695,10731,10768,10785-10786,10793,10819,10827"}}
+{"timestamp":1712198645.4783375,"name":"online","context":{"idset":"10461,10466,10566,10593,10599,10620,10775,10805,10809"}}
+{"timestamp":1712198645.5736029,"name":"online","context":{"idset":"10492,10511,10554,10614,10671,10744"}}
+{"timestamp":1712198645.6107626,"name":"online","context":{"idset":"10493,10523,10575,10580,10608,10679,10707"}}
+{"timestamp":1712198645.6609273,"name":"online","context":{"idset":"10505"}}
+{"timestamp":1712198645.7670696,"name":"online","context":{"idset":"10473,10489,10508,10542,10563,10572,10584,10615,10618,10635,10655,10663,10685,10729,10787,10828,10830"}}
+{"timestamp":1712198645.781461,"name":"online","context":{"idset":"10526"}}
+{"timestamp":1712198645.8204591,"name":"online","context":{"idset":"10539"}}
+{"timestamp":1712198645.8335967,"name":"online","context":{"idset":"10464,10555"}}
+{"timestamp":1712198645.8578446,"name":"online","context":{"idset":"10502"}}
+{"timestamp":1712198645.9511924,"name":"online","context":{"idset":"10551,10574,10654,10771,10780,10818"}}
+{"timestamp":1712198646.0485892,"name":"online","context":{"idset":"10514,10571,10597,10600,10625,10664,10687,10701,10703,10752,10765,10796"}}
+{"timestamp":1712198646.1646507,"name":"online","context":{"idset":"10468,10491,10621,10816"}}
+{"timestamp":1712198646.3376441,"name":"online","context":{"idset":"10488,10490,10516,10525,10527,10538,10559,10610,10631,10647,10660,10705,10763,10777"}}
+{"timestamp":1712198646.5124671,"name":"online","context":{"idset":"10628"}}
+{"timestamp":1712198647.3560157,"name":"online","context":{"idset":"11055,11100,11169,11265,11543,11545,11643"}}
+{"timestamp":1712198647.4580395,"name":"online","context":{"idset":"11102,11220,11222,11235,11310,11337,11401,11409,11447,11491,11500,11509"}}
+{"timestamp":1712198647.628191,"name":"online","context":{"idset":"11069,11209,11231,11241,11245-11246,11291,11324,11341,11346,11472,11575"}}
+{"timestamp":1712198647.7300131,"name":"online","context":{"idset":"11036,11066,11099,11192,11218,11226,11407,11504,11536,11579,11648"}}
+{"timestamp":1712198647.8320024,"name":"online","context":{"idset":"11115,11118,11135,11142,11154,11165,11171,11297,11315,11325,11329"}}
+{"timestamp":1712198647.9484811,"name":"online","context":{"idset":"11009,11018,11023,11029,11151,11253,11303-11304,11308,11345,11364,11459-11461,11481,11512,11540,11613,11667"}}
+{"timestamp":1712198648.1124518,"name":"online","context":{"idset":"11072,11221,11336,11355,11383,11428,11487,11565"}}
+{"timestamp":1712198803.768636,"name":"undrain","context":{"idset":"10886"}}
+{"timestamp":1712202504.6240864,"name":"drain","context":{"idset":"11226","reason":"broker was unresponsive"}}
+{"timestamp":1712202508.5270298,"name":"drain","context":{"idset":"11233","reason":"broker was unresponsive"}}
+{"timestamp":1712202508.5271747,"name":"drain","context":{"idset":"11235","reason":"broker was unresponsive"}}
+{"timestamp":1712202508.5273066,"name":"drain","context":{"idset":"11236","reason":"broker was unresponsive"}}
+{"timestamp":1712202508.5273933,"name":"drain","context":{"idset":"11243","reason":"broker was unresponsive"}}
+{"timestamp":1712202508.5274906,"name":"drain","context":{"idset":"11247","reason":"broker was unresponsive"}}
+{"timestamp":1712202508.6267445,"name":"drain","context":{"idset":"11249","reason":"broker was unresponsive"}}
+{"timestamp":1712202570.5437157,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1712202570.559953,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1712202570.9447944,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1712202570.962734,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1712202570.9929368,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1712202571.0358684,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1712202571.0478528,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1712202843.9507849,"name":"online","context":{"idset":"10873-10874"}}
+{"timestamp":1712202985.3164032,"name":"drain","context":{"idset":"10886","overwrite":0}}
+{"timestamp":1712203043.8665249,"name":"undrain","context":{"idset":"10886"}}
+{"timestamp":1712203134.8095155,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1712203134.9093862,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1712203179.4876909,"name":"online","context":{"idset":"10873"}}
+{"timestamp":1712203180.0907226,"name":"online","context":{"idset":"10874"}}
+{"timestamp":1712203320.8157983,"name":"drain","context":{"idset":"10885-10886","overwrite":0}}
+{"timestamp":1712203374.470021,"name":"undrain","context":{"idset":"10885-10886"}}
+{"timestamp":1712203445.1255968,"name":"drain","context":{"idset":"10886","overwrite":0}}
+{"timestamp":1712203448.0168498,"name":"drain","context":{"idset":"10886","overwrite":0}}
+{"timestamp":1712204412.3644712,"name":"undrain","context":{"idset":"10871,10883-10884,10886,10905-10906,10935-10937,10943,10976"}}
+{"timestamp":1712204443.0008402,"name":"undrain","context":{"idset":"10873-10874"}}
+{"timestamp":1712206762.1156664,"name":"online","context":{"idset":"10937"}}
+{"timestamp":1712206881.5149281,"name":"online","context":{"idset":"10976"}}
+{"timestamp":1712207043.2204895,"name":"online","context":{"idset":"10906"}}
+{"timestamp":1712207211.7895062,"name":"drain","context":{"idset":"10871,10883-10884,10886,10905,10911-10912,10935-10936,10943","overwrite":0}}
+{"timestamp":1712207295.0937617,"name":"drain","context":{"idset":"10871,10883-10884,10886,10905,10911-10912,10935-10936,10943","overwrite":0}}
+{"timestamp":1712207332.4251831,"name":"undrain","context":{"idset":"10871,10883-10884,10886,10905,10911-10912,10935-10936,10943"}}
+{"timestamp":1712207740.5101039,"name":"drain","context":{"idset":"10883-10884,10886,10905","overwrite":0}}
+{"timestamp":1712207933.1157787,"name":"online","context":{"idset":"10886"}}
+{"timestamp":1712207979.450958,"name":"undrain","context":{"idset":"10883-10884,10886,10905"}}
+{"timestamp":1712235954.4362509,"name":"online","context":{"idset":"796"}}
+{"timestamp":1712235954.7123239,"name":"online","context":{"idset":"795,807"}}
+{"timestamp":1712235954.911006,"name":"online","context":{"idset":"804"}}
+{"timestamp":1712235955.0939374,"name":"online","context":{"idset":"826"}}
+{"timestamp":1712235955.2785854,"name":"online","context":{"idset":"799"}}
+{"timestamp":1712235955.4749274,"name":"online","context":{"idset":"819"}}
+{"timestamp":1712235955.5931251,"name":"online","context":{"idset":"802,827"}}
+{"timestamp":1712235955.7629888,"name":"online","context":{"idset":"797,809-810,814,816-817,825,834"}}
+{"timestamp":1712235955.9389918,"name":"online","context":{"idset":"798,800-801,812,815,818"}}
+{"timestamp":1712235956.0605361,"name":"online","context":{"idset":"805,813,820,822,828,830"}}
+{"timestamp":1712235956.2395213,"name":"online","context":{"idset":"803,811,821,832-833"}}
+{"timestamp":1712235956.4147727,"name":"online","context":{"idset":"806,835"}}
+{"timestamp":1712235956.5841262,"name":"online","context":{"idset":"823-824,829,831"}}
+{"timestamp":1712235956.7504661,"name":"online","context":{"idset":"808,836"}}
+{"timestamp":1712236010.7951918,"name":"undrain","context":{"idset":"795-796,809-810"}}
+{"timestamp":1712236762.3760471,"name":"offline","context":{"idset":"10303"}}
+{"timestamp":1712236781.0769484,"name":"drain","context":{"idset":"10303","reason":"bad partner node --JRG","overwrite":0}}
+{"timestamp":1712236811.3206971,"name":"drain","context":{"idset":"10304","reason":"fails loopback test --JRG","overwrite":1}}
+{"timestamp":1712237659.4009576,"name":"drain","context":{"idset":"621-628","reason":"New blade install --JRG","overwrite":0}}
+{"timestamp":1712237683.0216222,"name":"offline","context":{"idset":"622"}}
+{"timestamp":1712237683.0496402,"name":"offline","context":{"idset":"623"}}
+{"timestamp":1712237683.0790248,"name":"offline","context":{"idset":"624"}}
+{"timestamp":1712237683.1043208,"name":"offline","context":{"idset":"627"}}
+{"timestamp":1712237683.1227357,"name":"offline","context":{"idset":"628"}}
+{"timestamp":1712237683.158798,"name":"offline","context":{"idset":"626"}}
+{"timestamp":1712237683.1823199,"name":"offline","context":{"idset":"621"}}
+{"timestamp":1712237683.2656078,"name":"offline","context":{"idset":"625"}}
+{"timestamp":1712238675.3197408,"name":"online","context":{"idset":"10302"}}
+{"timestamp":1712238675.4971955,"name":"online","context":{"idset":"10301"}}
+{"timestamp":1712240146.0447352,"name":"drain","context":{"idset":"795-836","reason":"Doing Local Benchmarks --JRG","overwrite":0}}
+{"timestamp":1712241030.3444023,"name":"drain","context":{"idset":"969","reason":"broker was unresponsive"}}
+{"timestamp":1712241031.3465993,"name":"drain","context":{"idset":"970","reason":"broker was unresponsive"}}
+{"timestamp":1712241112.5309589,"name":"offline","context":{"idset":"969"}}
+{"timestamp":1712241112.6237938,"name":"offline","context":{"idset":"970"}}
+{"timestamp":1712241434.5243275,"name":"drain","context":{"idset":"971","reason":"broker was unresponsive"}}
+{"timestamp":1712241434.6244709,"name":"drain","context":{"idset":"972","reason":"broker was unresponsive"}}
+{"timestamp":1712241436.6227129,"name":"offline","context":{"idset":"11815"}}
+{"timestamp":1712241452.6253233,"name":"drain","context":{"idset":"11125","reason":"broker was unresponsive"}}
+{"timestamp":1712241476.5255573,"name":"drain","context":{"idset":"973","reason":"broker was unresponsive"}}
+{"timestamp":1712241476.6259952,"name":"drain","context":{"idset":"974","reason":"broker was unresponsive"}}
+{"timestamp":1712241482.6270466,"name":"drain","context":{"idset":"978","reason":"broker was unresponsive"}}
+{"timestamp":1712241488.5256779,"name":"drain","context":{"idset":"979","reason":"broker was unresponsive"}}
+{"timestamp":1712241488.6261418,"name":"drain","context":{"idset":"980","reason":"broker was unresponsive"}}
+{"timestamp":1712241496.5307703,"name":"offline","context":{"idset":"971"}}
+{"timestamp":1712241496.6227157,"name":"offline","context":{"idset":"972"}}
+{"timestamp":1712241518.6226952,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1712241542.5427792,"name":"offline","context":{"idset":"973"}}
+{"timestamp":1712241542.6244879,"name":"offline","context":{"idset":"974"}}
+{"timestamp":1712241548.6229863,"name":"offline","context":{"idset":"978"}}
+{"timestamp":1712241552.5316226,"name":"offline","context":{"idset":"979"}}
+{"timestamp":1712241552.6236181,"name":"offline","context":{"idset":"980"}}
+{"timestamp":1712242176.6260459,"name":"drain","context":{"idset":"11388","reason":"broker was unresponsive"}}
+{"timestamp":1712242240.622834,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1712243893.6651571,"name":"undrain","context":{"idset":"795-836"}}
+{"timestamp":1712245166.6261528,"name":"drain","context":{"idset":"11115","reason":"broker was unresponsive"}}
+{"timestamp":1712245232.8818815,"name":"offline","context":{"idset":"11115"}}
+{"timestamp":1712246105.8709884,"name":"drain","context":{"idset":"10518","reason":"Node wont power on --JRG","overwrite":1}}
+{"timestamp":1712246171.0250499,"name":"online","context":{"idset":"10504,10519"}}
+{"timestamp":1712246171.1272244,"name":"online","context":{"idset":"10510,10517,10541"}}
+{"timestamp":1712246171.3318853,"name":"online","context":{"idset":"10509"}}
+{"timestamp":1712246171.5132902,"name":"online","context":{"idset":"10521,10557-10558,10582"}}
+{"timestamp":1712246171.6870356,"name":"online","context":{"idset":"10522"}}
+{"timestamp":1712246171.8952961,"name":"online","context":{"idset":"10602"}}
+{"timestamp":1712246217.3801992,"name":"undrain","context":{"idset":"10504,10509-10510,10517,10519,10541,10557-10558,10582,10602"}}
+{"timestamp":1712247263.0627384,"name":"online","context":{"idset":"11012"}}
+{"timestamp":1712247703.4706407,"name":"online","context":{"idset":"10136"}}
+{"timestamp":1712247703.7578716,"name":"online","context":{"idset":"10135"}}
+{"timestamp":1712247850.6299593,"name":"online","context":{"idset":"11226,11249"}}
+{"timestamp":1712247850.79405,"name":"online","context":{"idset":"11235-11236,11243,11247"}}
+{"timestamp":1712247851.0921681,"name":"online","context":{"idset":"11233"}}
+{"timestamp":1712250096.3743248,"name":"online","context":{"idset":"10684"}}
+{"timestamp":1712250233.9716916,"name":"online","context":{"idset":"10683"}}
+{"timestamp":1712251076.6770136,"name":"online","context":{"idset":"11115"}}
+{"timestamp":1712251257.9561243,"name":"online","context":{"idset":"9989"}}
+{"timestamp":1712251323.8498173,"name":"online","context":{"idset":"10143"}}
+{"timestamp":1712251324.1459725,"name":"online","context":{"idset":"10154"}}
+{"timestamp":1712251324.3457251,"name":"online","context":{"idset":"10138,10147"}}
+{"timestamp":1712251324.9959002,"name":"online","context":{"idset":"10162,10207"}}
+{"timestamp":1712251368.6907787,"name":"undrain","context":{"idset":"11115"}}
+{"timestamp":1712251371.3539946,"name":"online","context":{"idset":"9722"}}
+{"timestamp":1712251371.7735214,"name":"online","context":{"idset":"9733,9746"}}
+{"timestamp":1712251371.8571105,"name":"online","context":{"idset":"9718"}}
+{"timestamp":1712251371.9350808,"name":"online","context":{"idset":"9726"}}
+{"timestamp":1712251372.0652146,"name":"online","context":{"idset":"9720,9723"}}
+{"timestamp":1712251372.1576502,"name":"online","context":{"idset":"9724,9727"}}
+{"timestamp":1712251372.352139,"name":"online","context":{"idset":"9721,9730-9731"}}
+{"timestamp":1712251372.4849455,"name":"online","context":{"idset":"9741"}}
+{"timestamp":1712251373.2975893,"name":"online","context":{"idset":"9748"}}
+{"timestamp":1712251373.3000743,"name":"online","context":{"idset":"9753"}}
+{"timestamp":1712251373.302479,"name":"online","context":{"idset":"9719"}}
+{"timestamp":1712251373.3050625,"name":"online","context":{"idset":"9764"}}
+{"timestamp":1712251374.0598295,"name":"online","context":{"idset":"9742"}}
+{"timestamp":1712251374.0628121,"name":"online","context":{"idset":"9737"}}
+{"timestamp":1712251374.0654027,"name":"online","context":{"idset":"9717"}}
+{"timestamp":1712251374.0678759,"name":"online","context":{"idset":"9752,9782"}}
+{"timestamp":1712251374.1743176,"name":"online","context":{"idset":"9736,9738"}}
+{"timestamp":1712251374.2668786,"name":"online","context":{"idset":"9735"}}
+{"timestamp":1712251374.54495,"name":"online","context":{"idset":"9732,9771"}}
+{"timestamp":1712251374.6543589,"name":"online","context":{"idset":"9744,9780"}}
+{"timestamp":1712251374.7504082,"name":"online","context":{"idset":"9734,9788"}}
+{"timestamp":1712251374.9416687,"name":"online","context":{"idset":"9725,9772,9783"}}
+{"timestamp":1712251375.0616581,"name":"online","context":{"idset":"9728,9739-9740,9750-9751,9756,9768,9776,9810"}}
+{"timestamp":1712251375.2447219,"name":"online","context":{"idset":"9747,9757,9763,9770,9774-9775,9777,9781,9791,9836"}}
+{"timestamp":1712251375.4402366,"name":"online","context":{"idset":"9743,9755,9773,9784,9814"}}
+{"timestamp":1712251375.5619938,"name":"online","context":{"idset":"9729,9745,9749,9769,9786,9837-9838"}}
+{"timestamp":1712251375.7480388,"name":"online","context":{"idset":"9754,9762,9765,9767,9789,9792,9795,9804-9805,9807,9811,9813,9819,9821,9825,9830"}}
+{"timestamp":1712251376.7174332,"name":"online","context":{"idset":"9761,9766,9785,9787,9798-9799,9803,9835,9841"}}
+{"timestamp":1712251376.7210138,"name":"online","context":{"idset":"9758-9759,9779,9793,9796,9822,9826,9834"}}
+{"timestamp":1712251376.7243063,"name":"online","context":{"idset":"9800,9806,9808,9817-9818,9832"}}
+{"timestamp":1712251376.7274134,"name":"online","context":{"idset":"9778,9790,9794,9809,9816,9823-9824"}}
+{"timestamp":1712251376.7307508,"name":"online","context":{"idset":"9760,9802,9829,9840"}}
+{"timestamp":1712251376.734061,"name":"online","context":{"idset":"9812,9815,9828,9842-9843"}}
+{"timestamp":1712251376.7371647,"name":"online","context":{"idset":"9801,9820,9831,9839,9844"}}
+{"timestamp":1712251376.9437916,"name":"online","context":{"idset":"9797,9827,9833"}}
+{"timestamp":1712251423.3449986,"name":"undrain","context":{"idset":"9989"}}
+{"timestamp":1712251896.0884097,"name":"online","context":{"idset":"11003"}}
+{"timestamp":1712252115.930419,"name":"drain","context":{"idset":"10517","reason":"bad partner node","overwrite":0}}
+{"timestamp":1712252148.3576784,"name":"offline","context":{"idset":"10517"}}
+{"timestamp":1712252261.9308169,"name":"online","context":{"idset":"11136"}}
+{"timestamp":1712252262.4500918,"name":"online","context":{"idset":"11178"}}
+{"timestamp":1712252263.0088377,"name":"online","context":{"idset":"11138"}}
+{"timestamp":1712252263.1105163,"name":"online","context":{"idset":"11130"}}
+{"timestamp":1712252263.9549375,"name":"online","context":{"idset":"11134,11140,11187"}}
+{"timestamp":1712253210.808218,"name":"online","context":{"idset":"10037"}}
+{"timestamp":1712253601.0106037,"name":"online","context":{"idset":"11124"}}
+{"timestamp":1712253997.2632515,"name":"online","context":{"idset":"9466"}}
+{"timestamp":1712253997.3370574,"name":"online","context":{"idset":"9463"}}
+{"timestamp":1712253997.5393717,"name":"online","context":{"idset":"9462,9478"}}
+{"timestamp":1712253997.6311142,"name":"online","context":{"idset":"9464"}}
+{"timestamp":1712253997.8467937,"name":"online","context":{"idset":"9475"}}
+{"timestamp":1712253997.8675992,"name":"online","context":{"idset":"9487"}}
+{"timestamp":1712253998.025636,"name":"online","context":{"idset":"9461,9467,9470,9474,9483"}}
+{"timestamp":1712253998.1849563,"name":"online","context":{"idset":"9472"}}
+{"timestamp":1712253998.4215717,"name":"online","context":{"idset":"9465"}}
+{"timestamp":1712253998.6749592,"name":"online","context":{"idset":"9477"}}
+{"timestamp":1712253998.9377735,"name":"online","context":{"idset":"9473,9509"}}
+{"timestamp":1712253999.0162773,"name":"online","context":{"idset":"9481"}}
+{"timestamp":1712253999.1354272,"name":"online","context":{"idset":"9480"}}
+{"timestamp":1712253999.2310853,"name":"online","context":{"idset":"9496"}}
+{"timestamp":1712253999.249007,"name":"online","context":{"idset":"9486"}}
+{"timestamp":1712253999.4787219,"name":"online","context":{"idset":"9492,9507"}}
+{"timestamp":1712253999.6550343,"name":"online","context":{"idset":"9494"}}
+{"timestamp":1712253999.6714344,"name":"online","context":{"idset":"9479"}}
+{"timestamp":1712253999.8330648,"name":"online","context":{"idset":"9499"}}
+{"timestamp":1712253999.9228604,"name":"online","context":{"idset":"9513,9524"}}
+{"timestamp":1712254000.0010405,"name":"online","context":{"idset":"9519"}}
+{"timestamp":1712254000.0777457,"name":"online","context":{"idset":"9485,9500"}}
+{"timestamp":1712254000.2999225,"name":"online","context":{"idset":"9489,9493,9506"}}
+{"timestamp":1712254000.3915112,"name":"online","context":{"idset":"9476,9497,9514"}}
+{"timestamp":1712254000.4912429,"name":"online","context":{"idset":"9471,9542,9547,9553"}}
+{"timestamp":1712254000.7130234,"name":"online","context":{"idset":"9469,9491,9511,9522,9552"}}
+{"timestamp":1712254000.8926764,"name":"online","context":{"idset":"9503-9504,9523,9556,9558,9582"}}
+{"timestamp":1712254001.0098047,"name":"online","context":{"idset":"9488,9510,9534,9538-9539,9567,9573"}}
+{"timestamp":1712254001.1869159,"name":"online","context":{"idset":"9516,9521,9555,9585"}}
+{"timestamp":1712254001.304311,"name":"online","context":{"idset":"9490,9501,9505,9518,9526,9536,9545,9548,9557,9575,9586"}}
+{"timestamp":1712254001.4882045,"name":"online","context":{"idset":"9484,9508,9517,9531,9544,9550,9560,9569,9577,9580,9588"}}
+{"timestamp":1712254001.5913589,"name":"online","context":{"idset":"9468,9495,9498,9502,9515,9520,9525,9528,9564,9572"}}
+{"timestamp":1712254001.7149792,"name":"online","context":{"idset":"9482,9529,9535,9537,9543,9546,9554,9566,9571,9576,9579"}}
+{"timestamp":1712254001.8329375,"name":"online","context":{"idset":"9530,9532,9551,9563,9578,9581"}}
+{"timestamp":1712254002.0074639,"name":"online","context":{"idset":"9527,9541,9574,9584"}}
+{"timestamp":1712254002.1253941,"name":"online","context":{"idset":"9549,9559,9561,9568,9583"}}
+{"timestamp":1712254002.3162215,"name":"online","context":{"idset":"9512,9565"}}
+{"timestamp":1712254002.4444449,"name":"online","context":{"idset":"9587"}}
+{"timestamp":1712254002.6184585,"name":"online","context":{"idset":"9540,9570"}}
+{"timestamp":1712254192.1381111,"name":"offline","context":{"idset":"795"}}
+{"timestamp":1712254194.6270406,"name":"drain","context":{"idset":"10184","reason":"broker was unresponsive"}}
+{"timestamp":1712254237.9585514,"name":"online","context":{"idset":"9214"}}
+{"timestamp":1712254256.6264846,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1712254304.6269586,"name":"drain","context":{"idset":"796","reason":"broker was unresponsive"}}
+{"timestamp":1712254368.6228797,"name":"offline","context":{"idset":"796"}}
+{"timestamp":1712254526.6264279,"name":"drain","context":{"idset":"11115","reason":"broker was unresponsive"}}
+{"timestamp":1712254589.7422478,"name":"offline","context":{"idset":"11115"}}
+{"timestamp":1712254984.7410762,"name":"online","context":{"idset":"10059"}}
+{"timestamp":1712255268.9196324,"name":"drain","context":{"idset":"597-604,613-620","reason":"New blade installations","overwrite":0}}
+{"timestamp":1712255301.3562946,"name":"online","context":{"idset":"10064"}}
+{"timestamp":1712255332.6318717,"name":"offline","context":{"idset":"599"}}
+{"timestamp":1712255332.6568608,"name":"offline","context":{"idset":"600"}}
+{"timestamp":1712255332.6751232,"name":"offline","context":{"idset":"614"}}
+{"timestamp":1712255332.7133107,"name":"offline","context":{"idset":"604"}}
+{"timestamp":1712255332.7331109,"name":"offline","context":{"idset":"597"}}
+{"timestamp":1712255332.7649722,"name":"offline","context":{"idset":"603"}}
+{"timestamp":1712255332.7714648,"name":"offline","context":{"idset":"617"}}
+{"timestamp":1712255332.8177843,"name":"offline","context":{"idset":"619"}}
+{"timestamp":1712255332.821116,"name":"offline","context":{"idset":"598"}}
+{"timestamp":1712255332.8590255,"name":"offline","context":{"idset":"601"}}
+{"timestamp":1712255332.8890958,"name":"offline","context":{"idset":"602"}}
+{"timestamp":1712255332.9074397,"name":"offline","context":{"idset":"613"}}
+{"timestamp":1712255332.9151144,"name":"offline","context":{"idset":"615"}}
+{"timestamp":1712255332.9331892,"name":"offline","context":{"idset":"616"}}
+{"timestamp":1712255332.9595788,"name":"offline","context":{"idset":"618"}}
+{"timestamp":1712255333.1224437,"name":"offline","context":{"idset":"620"}}
+{"timestamp":1712255642.6251159,"name":"drain","context":{"idset":"9561","reason":"broker was unresponsive"}}
+{"timestamp":1712255708.6239488,"name":"offline","context":{"idset":"9561"}}
+{"timestamp":1712255961.7708464,"name":"undrain","context":{"idset":"10037,10059,10064"}}
+{"timestamp":1712256888.3709071,"name":"online","context":{"idset":"11670"}}
+{"timestamp":1712256889.3070269,"name":"online","context":{"idset":"11669"}}
+{"timestamp":1712256974.0910864,"name":"undrain","context":{"idset":"11653-11660"}}
+{"timestamp":1712256986.1714008,"name":"undrain","context":{"idset":"11676"}}
+{"timestamp":1712257278.5871131,"name":"online","context":{"idset":"11287"}}
+{"timestamp":1712257405.3082883,"name":"undrain","context":{"idset":"11358,11373"}}
+{"timestamp":1712257638.6263337,"name":"drain","context":{"idset":"11003","reason":"broker was unresponsive"}}
+{"timestamp":1712257702.6227002,"name":"offline","context":{"idset":"11003"}}
+{"timestamp":1712258983.6482201,"name":"online","context":{"idset":"11149"}}
+{"timestamp":1712258984.0422602,"name":"online","context":{"idset":"11150"}}
+{"timestamp":1712259390.627492,"name":"drain","context":{"idset":"10873","reason":"broker was unresponsive"}}
+{"timestamp":1712259506.6230671,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1712260786.5586481,"name":"online","context":{"idset":"11115"}}
+{"timestamp":1712260819.3331053,"name":"undrain","context":{"idset":"11115"}}
+{"timestamp":1712260878.7878225,"name":"online","context":{"idset":"11003"}}
+{"timestamp":1712260905.4979541,"name":"undrain","context":{"idset":"11003"}}
+{"timestamp":1712261467.3244655,"name":"online","context":{"idset":"11748"}}
+{"timestamp":1712261862.602201,"name":"drain","context":{"idset":"797-798","reason":"--reason swapping blades into x1900 - wendy","overwrite":0}}
+{"timestamp":1712262098.5272985,"name":"offline","context":{"idset":"797"}}
+{"timestamp":1712262098.6208756,"name":"offline","context":{"idset":"798"}}
+{"timestamp":1712262308.5184369,"name":"online","context":{"idset":"9239"}}
+{"timestamp":1712262440.8716927,"name":"offline","context":{"idset":"9205"}}
+{"timestamp":1712262440.9452493,"name":"offline","context":{"idset":"9207"}}
+{"timestamp":1712262440.962465,"name":"offline","context":{"idset":"9206"}}
+{"timestamp":1712262440.9895356,"name":"offline","context":{"idset":"9226"}}
+{"timestamp":1712262441.0046933,"name":"offline","context":{"idset":"9214"}}
+{"timestamp":1712262441.0291932,"name":"offline","context":{"idset":"9222"}}
+{"timestamp":1712262441.0433283,"name":"offline","context":{"idset":"9237"}}
+{"timestamp":1712262441.0584433,"name":"offline","context":{"idset":"9208"}}
+{"timestamp":1712262441.1356668,"name":"offline","context":{"idset":"9234"}}
+{"timestamp":1712262441.1617117,"name":"offline","context":{"idset":"9210"}}
+{"timestamp":1712262441.2403877,"name":"offline","context":{"idset":"9235"}}
+{"timestamp":1712262441.2604032,"name":"offline","context":{"idset":"9231"}}
+{"timestamp":1712262441.2749836,"name":"offline","context":{"idset":"9251"}}
+{"timestamp":1712262441.2898083,"name":"offline","context":{"idset":"9263"}}
+{"timestamp":1712262441.3169374,"name":"offline","context":{"idset":"9253"}}
+{"timestamp":1712262441.3199115,"name":"offline","context":{"idset":"9221"}}
+{"timestamp":1712262441.3334291,"name":"offline","context":{"idset":"9248"}}
+{"timestamp":1712262441.4378712,"name":"offline","context":{"idset":"9257"}}
+{"timestamp":1712262441.4405363,"name":"offline","context":{"idset":"9243"}}
+{"timestamp":1712262441.4431715,"name":"offline","context":{"idset":"9232"}}
+{"timestamp":1712262441.4567068,"name":"offline","context":{"idset":"9272"}}
+{"timestamp":1712262441.5245347,"name":"offline","context":{"idset":"9223"}}
+{"timestamp":1712262441.5381584,"name":"offline","context":{"idset":"9211"}}
+{"timestamp":1712262441.5533521,"name":"offline","context":{"idset":"9213"}}
+{"timestamp":1712262441.567941,"name":"offline","context":{"idset":"9215"}}
+{"timestamp":1712262441.5706735,"name":"offline","context":{"idset":"9217"}}
+{"timestamp":1712262441.582865,"name":"offline","context":{"idset":"9230"}}
+{"timestamp":1712262441.6989083,"name":"offline","context":{"idset":"9236"}}
+{"timestamp":1712262441.7298923,"name":"offline","context":{"idset":"9240"}}
+{"timestamp":1712262441.7326386,"name":"offline","context":{"idset":"9241"}}
+{"timestamp":1712262441.7448657,"name":"offline","context":{"idset":"9245"}}
+{"timestamp":1712262441.757163,"name":"offline","context":{"idset":"9246"}}
+{"timestamp":1712262441.7790689,"name":"offline","context":{"idset":"9270"}}
+{"timestamp":1712262441.7816939,"name":"offline","context":{"idset":"9275"}}
+{"timestamp":1712262441.7843406,"name":"offline","context":{"idset":"9277"}}
+{"timestamp":1712262441.789048,"name":"offline","context":{"idset":"9281"}}
+{"timestamp":1712262441.864599,"name":"offline","context":{"idset":"9282"}}
+{"timestamp":1712262441.8798161,"name":"offline","context":{"idset":"9285"}}
+{"timestamp":1712262441.8946781,"name":"offline","context":{"idset":"9292"}}
+{"timestamp":1712262441.9110181,"name":"offline","context":{"idset":"9332"}}
+{"timestamp":1712262441.9264817,"name":"offline","context":{"idset":"9209"}}
+{"timestamp":1712262441.952769,"name":"offline","context":{"idset":"9212"}}
+{"timestamp":1712262441.9741423,"name":"offline","context":{"idset":"9216"}}
+{"timestamp":1712262441.9861469,"name":"offline","context":{"idset":"9218"}}
+{"timestamp":1712262441.9887404,"name":"offline","context":{"idset":"9219"}}
+{"timestamp":1712262442.0003564,"name":"offline","context":{"idset":"9224"}}
+{"timestamp":1712262442.0801516,"name":"offline","context":{"idset":"9225"}}
+{"timestamp":1712262442.0971496,"name":"offline","context":{"idset":"9227"}}
+{"timestamp":1712262442.1112764,"name":"offline","context":{"idset":"9228"}}
+{"timestamp":1712262442.1296215,"name":"offline","context":{"idset":"9233"}}
+{"timestamp":1712262442.1451988,"name":"offline","context":{"idset":"9238"}}
+{"timestamp":1712262442.1499352,"name":"offline","context":{"idset":"9239"}}
+{"timestamp":1712262442.1776211,"name":"offline","context":{"idset":"9242"}}
+{"timestamp":1712262442.2031221,"name":"offline","context":{"idset":"9244"}}
+{"timestamp":1712262442.2253928,"name":"offline","context":{"idset":"9250"}}
+{"timestamp":1712262442.247431,"name":"offline","context":{"idset":"9254"}}
+{"timestamp":1712262442.2619414,"name":"offline","context":{"idset":"9256"}}
+{"timestamp":1712262442.2765341,"name":"offline","context":{"idset":"9260"}}
+{"timestamp":1712262442.2913206,"name":"offline","context":{"idset":"9261"}}
+{"timestamp":1712262442.3081987,"name":"offline","context":{"idset":"9262"}}
+{"timestamp":1712262442.3284283,"name":"offline","context":{"idset":"9264"}}
+{"timestamp":1712262442.3443713,"name":"offline","context":{"idset":"9265"}}
+{"timestamp":1712262442.3645682,"name":"offline","context":{"idset":"9266"}}
+{"timestamp":1712262442.3968449,"name":"offline","context":{"idset":"9268"}}
+{"timestamp":1712262442.4261861,"name":"offline","context":{"idset":"9269"}}
+{"timestamp":1712262442.4401946,"name":"offline","context":{"idset":"9274"}}
+{"timestamp":1712262442.4540896,"name":"offline","context":{"idset":"9278"}}
+{"timestamp":1712262442.4677503,"name":"offline","context":{"idset":"9279"}}
+{"timestamp":1712262442.5004749,"name":"offline","context":{"idset":"9283"}}
+{"timestamp":1712262442.5232189,"name":"offline","context":{"idset":"9286"}}
+{"timestamp":1712262442.5521655,"name":"offline","context":{"idset":"9289"}}
+{"timestamp":1712262442.5761604,"name":"offline","context":{"idset":"9290"}}
+{"timestamp":1712262442.5975945,"name":"offline","context":{"idset":"9291"}}
+{"timestamp":1712262442.6001124,"name":"offline","context":{"idset":"9293"}}
+{"timestamp":1712262442.6026418,"name":"offline","context":{"idset":"9299"}}
+{"timestamp":1712262442.6051733,"name":"offline","context":{"idset":"9300"}}
+{"timestamp":1712262442.6078596,"name":"offline","context":{"idset":"9301"}}
+{"timestamp":1712262442.6104307,"name":"offline","context":{"idset":"9302"}}
+{"timestamp":1712262442.6130245,"name":"offline","context":{"idset":"9306"}}
+{"timestamp":1712262442.621773,"name":"offline","context":{"idset":"9311"}}
+{"timestamp":1712262442.6351838,"name":"offline","context":{"idset":"9314"}}
+{"timestamp":1712262442.6380589,"name":"offline","context":{"idset":"9324"}}
+{"timestamp":1712262442.6482558,"name":"offline","context":{"idset":"9328"}}
+{"timestamp":1712262442.6609569,"name":"offline","context":{"idset":"9220"}}
+{"timestamp":1712262442.6637948,"name":"offline","context":{"idset":"9229"}}
+{"timestamp":1712262442.6728356,"name":"offline","context":{"idset":"9247"}}
+{"timestamp":1712262442.7030711,"name":"offline","context":{"idset":"9249"}}
+{"timestamp":1712262442.7241285,"name":"offline","context":{"idset":"9252"}}
+{"timestamp":1712262442.7448566,"name":"offline","context":{"idset":"9255"}}
+{"timestamp":1712262442.7649245,"name":"offline","context":{"idset":"9259"}}
+{"timestamp":1712262442.7846773,"name":"offline","context":{"idset":"9267"}}
+{"timestamp":1712262442.8040726,"name":"offline","context":{"idset":"9271"}}
+{"timestamp":1712262442.8147583,"name":"offline","context":{"idset":"9273"}}
+{"timestamp":1712262442.8170154,"name":"offline","context":{"idset":"9276"}}
+{"timestamp":1712262442.819237,"name":"offline","context":{"idset":"9284"}}
+{"timestamp":1712262442.8214624,"name":"offline","context":{"idset":"9287"}}
+{"timestamp":1712262442.8236775,"name":"offline","context":{"idset":"9288"}}
+{"timestamp":1712262442.8275974,"name":"offline","context":{"idset":"9294"}}
+{"timestamp":1712262442.8382442,"name":"offline","context":{"idset":"9295"}}
+{"timestamp":1712262442.8489165,"name":"offline","context":{"idset":"9296"}}
+{"timestamp":1712262442.8595066,"name":"offline","context":{"idset":"9304"}}
+{"timestamp":1712262442.8694851,"name":"offline","context":{"idset":"9308"}}
+{"timestamp":1712262442.8789289,"name":"offline","context":{"idset":"9310"}}
+{"timestamp":1712262442.8883746,"name":"offline","context":{"idset":"9312"}}
+{"timestamp":1712262442.9068353,"name":"offline","context":{"idset":"9313"}}
+{"timestamp":1712262442.9251449,"name":"offline","context":{"idset":"9315"}}
+{"timestamp":1712262442.9430273,"name":"offline","context":{"idset":"9316"}}
+{"timestamp":1712262442.9594269,"name":"offline","context":{"idset":"9320"}}
+{"timestamp":1712262442.976203,"name":"offline","context":{"idset":"9322"}}
+{"timestamp":1712262442.9927952,"name":"offline","context":{"idset":"9325"}}
+{"timestamp":1712262443.0090902,"name":"offline","context":{"idset":"9326"}}
+{"timestamp":1712262443.0253232,"name":"offline","context":{"idset":"9331"}}
+{"timestamp":1712262443.0413032,"name":"offline","context":{"idset":"9258"}}
+{"timestamp":1712262443.0574081,"name":"offline","context":{"idset":"9280"}}
+{"timestamp":1712262443.0737979,"name":"offline","context":{"idset":"9297"}}
+{"timestamp":1712262443.0828393,"name":"offline","context":{"idset":"9298"}}
+{"timestamp":1712262443.084758,"name":"offline","context":{"idset":"9303"}}
+{"timestamp":1712262443.0866725,"name":"offline","context":{"idset":"9305"}}
+{"timestamp":1712262443.0885849,"name":"offline","context":{"idset":"9307"}}
+{"timestamp":1712262443.090487,"name":"offline","context":{"idset":"9309"}}
+{"timestamp":1712262443.0923712,"name":"offline","context":{"idset":"9317"}}
+{"timestamp":1712262443.0942938,"name":"offline","context":{"idset":"9318"}}
+{"timestamp":1712262443.0961688,"name":"offline","context":{"idset":"9319"}}
+{"timestamp":1712262443.098073,"name":"offline","context":{"idset":"9321"}}
+{"timestamp":1712262443.1043861,"name":"offline","context":{"idset":"9323"}}
+{"timestamp":1712262443.112783,"name":"offline","context":{"idset":"9327"}}
+{"timestamp":1712262443.1211498,"name":"offline","context":{"idset":"9329"}}
+{"timestamp":1712262443.1295433,"name":"offline","context":{"idset":"9330"}}
+{"timestamp":1712265206.1693995,"name":"drain","context":{"idset":"10741-10868","overwrite":0}}
+{"timestamp":1712265398.553122,"name":"undrain","context":{"idset":"10741-10868"}}
+{"timestamp":1712265498.6265624,"name":"drain","context":{"idset":"9534","reason":"broker was unresponsive"}}
+{"timestamp":1712265560.6227524,"name":"offline","context":{"idset":"9534"}}
+{"timestamp":1712265740.0424361,"name":"online","context":{"idset":"11174"}}
+{"timestamp":1712265741.51299,"name":"online","context":{"idset":"11173"}}
+{"timestamp":1712267422.3869722,"name":"drain","context":{"idset":"11220","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712267849.1048238,"name":"drain","context":{"idset":"10470","reason":"broker was unresponsive"}}
+{"timestamp":1712267849.1065321,"name":"drain","context":{"idset":"10471","reason":"broker was unresponsive"}}
+{"timestamp":1712267849.1068006,"name":"drain","context":{"idset":"10472","reason":"broker was unresponsive"}}
+{"timestamp":1712267849.1069198,"name":"drain","context":{"idset":"10473","reason":"broker was unresponsive"}}
+{"timestamp":1712267849.107039,"name":"drain","context":{"idset":"10474","reason":"broker was unresponsive"}}
+{"timestamp":1712267849.1071479,"name":"drain","context":{"idset":"10476","reason":"broker was unresponsive"}}
+{"timestamp":1712267849.1072385,"name":"drain","context":{"idset":"10477","reason":"broker was unresponsive"}}
+{"timestamp":1712267849.1073437,"name":"drain","context":{"idset":"10481","reason":"broker was unresponsive"}}
+{"timestamp":1712267849.1271958,"name":"drain","context":{"idset":"10483","reason":"broker was unresponsive"}}
+{"timestamp":1712267914.5564258,"name":"offline","context":{"idset":"10470"}}
+{"timestamp":1712267914.599426,"name":"offline","context":{"idset":"10471"}}
+{"timestamp":1712267914.6290171,"name":"offline","context":{"idset":"10472"}}
+{"timestamp":1712267914.6520965,"name":"offline","context":{"idset":"10473"}}
+{"timestamp":1712267915.5955,"name":"offline","context":{"idset":"10474"}}
+{"timestamp":1712267915.6097567,"name":"offline","context":{"idset":"10476"}}
+{"timestamp":1712267915.6129,"name":"offline","context":{"idset":"10477"}}
+{"timestamp":1712267915.6156809,"name":"offline","context":{"idset":"10481"}}
+{"timestamp":1712267915.6430709,"name":"offline","context":{"idset":"10483"}}
+{"timestamp":1712268024.5265024,"name":"drain","context":{"idset":"10874","reason":"broker was unresponsive"}}
+{"timestamp":1712268025.2693172,"name":"drain","context":{"idset":"10906","reason":"broker was unresponsive"}}
+{"timestamp":1712268087.3168221,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1712268089.2948828,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1712268121.4638283,"name":"drain","context":{"idset":"10412","reason":"broker was unresponsive"}}
+{"timestamp":1712268187.4189229,"name":"offline","context":{"idset":"10412"}}
+{"timestamp":1712268702.5263739,"name":"drain","context":{"idset":"951","reason":"broker was unresponsive"}}
+{"timestamp":1712268703.3344858,"name":"drain","context":{"idset":"952","reason":"broker was unresponsive"}}
+{"timestamp":1712268743.9535029,"name":"undrain","context":{"idset":"11173,11178,11187"}}
+{"timestamp":1712268764.5355663,"name":"offline","context":{"idset":"951"}}
+{"timestamp":1712268765.3795581,"name":"offline","context":{"idset":"952"}}
+{"timestamp":1712269099.4085643,"name":"drain","context":{"idset":"11173","overwrite":0}}
+{"timestamp":1712269682.1395669,"name":"undrain","context":{"idset":"11173"}}
+{"timestamp":1712270043.3290925,"name":"online","context":{"idset":"10873"}}
+{"timestamp":1712270721.4578836,"name":"online","context":{"idset":"10474"}}
+{"timestamp":1712270721.9461238,"name":"online","context":{"idset":"10472"}}
+{"timestamp":1712270722.1332943,"name":"online","context":{"idset":"10475-10476"}}
+{"timestamp":1712270722.2554369,"name":"online","context":{"idset":"10478,10480,10484"}}
+{"timestamp":1712270722.4334419,"name":"online","context":{"idset":"10470,10477,10479,10482-10483"}}
+{"timestamp":1712270722.5534339,"name":"online","context":{"idset":"10471,10473"}}
+{"timestamp":1712270723.3085105,"name":"online","context":{"idset":"10481"}}
+{"timestamp":1712270723.3111658,"name":"online","context":{"idset":"10469"}}
+{"timestamp":1712270817.4812977,"name":"drain","context":{"idset":"11397-11412","overwrite":0}}
+{"timestamp":1712270841.743036,"name":"undrain","context":{"idset":"11397-11412"}}
+{"timestamp":1712270907.4221985,"name":"drain","context":{"idset":"11400-11412","overwrite":0}}
+{"timestamp":1712271436.9838583,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1712271808.3093069,"name":"drain","context":{"idset":"11413-11428","overwrite":0}}
+{"timestamp":1712272276.4058383,"name":"drain","context":{"idset":"9447","reason":"CXI Stuck at Starting --JRG","overwrite":1}}
+{"timestamp":1712272297.6105912,"name":"drain","context":{"idset":"9448","reason":"bad partner node --JRG","overwrite":0}}
+{"timestamp":1712273007.6605749,"name":"drain","context":{"idset":"11187","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712273154.5289214,"name":"online","context":{"idset":"964"}}
+{"timestamp":1712273155.3101871,"name":"online","context":{"idset":"949,951,953"}}
+{"timestamp":1712273155.3127587,"name":"online","context":{"idset":"952,954"}}
+{"timestamp":1712273155.3152339,"name":"online","context":{"idset":"950"}}
+{"timestamp":1712273155.6017079,"name":"online","context":{"idset":"963"}}
+{"timestamp":1712273813.4920654,"name":"online","context":{"idset":"10183-10184"}}
+{"timestamp":1712274216.6631169,"name":"drain","context":{"idset":"11253-11380","reason":"Adam: testing","overwrite":0}}
+{"timestamp":1712274256.249831,"name":"undrain","context":{"idset":"11253-11380"}}
+{"timestamp":1712274334.3545144,"name":"drain","context":{"idset":"11253-11380","reason":"Adam: testing","overwrite":0}}
+{"timestamp":1712274357.4410529,"name":"undrain","context":{"idset":"11253-11380"}}
+{"timestamp":1712274414.439393,"name":"drain","context":{"idset":"9634","reason":"Boot loops --JRG","overwrite":0}}
+{"timestamp":1712274417.508244,"name":"drain","context":{"idset":"11253-11380","reason":"Adam: testing","overwrite":0}}
+{"timestamp":1712274429.6281929,"name":"undrain","context":{"idset":"11253-11380"}}
+{"timestamp":1712274432.5613508,"name":"drain","context":{"idset":"9633","reason":"bad partner node --JRG","overwrite":0}}
+{"timestamp":1712274472.057575,"name":"online","context":{"idset":"10205"}}
+{"timestamp":1712274472.5323987,"name":"online","context":{"idset":"10206"}}
+{"timestamp":1712274513.762475,"name":"online","context":{"idset":"869"}}
+{"timestamp":1712274514.0566504,"name":"online","context":{"idset":"870"}}
+{"timestamp":1712274769.4600027,"name":"drain","context":{"idset":"10206","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1712275231.4141004,"name":"undrain","context":{"idset":"10183-10184"}}
+{"timestamp":1712275294.0289221,"name":"drain","context":{"idset":"11253-11380","reason":"adam: test","overwrite":0}}
+{"timestamp":1712275304.4046972,"name":"undrain","context":{"idset":"11253-11380"}}
+{"timestamp":1712275337.2562318,"name":"drain","context":{"idset":"11642","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.528425,"name":"drain","context":{"idset":"11637","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5285213,"name":"drain","context":{"idset":"11638","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5286081,"name":"drain","context":{"idset":"11639","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5286763,"name":"drain","context":{"idset":"11640","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5287418,"name":"drain","context":{"idset":"11641","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5288122,"name":"drain","context":{"idset":"11643","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5288818,"name":"drain","context":{"idset":"11644","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5289531,"name":"drain","context":{"idset":"11645","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.529022,"name":"drain","context":{"idset":"11646","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5290902,"name":"drain","context":{"idset":"11647","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.52916,"name":"drain","context":{"idset":"11648","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5294018,"name":"drain","context":{"idset":"11649","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5294805,"name":"drain","context":{"idset":"11650","reason":"broker was unresponsive"}}
+{"timestamp":1712275384.5295577,"name":"drain","context":{"idset":"11651","reason":"broker was unresponsive"}}
+{"timestamp":1712275385.3217044,"name":"drain","context":{"idset":"11652","reason":"broker was unresponsive"}}
+{"timestamp":1712275403.416642,"name":"offline","context":{"idset":"11642"}}
+{"timestamp":1712275448.5711417,"name":"offline","context":{"idset":"11637"}}
+{"timestamp":1712275448.5879512,"name":"offline","context":{"idset":"11638"}}
+{"timestamp":1712275448.6030333,"name":"offline","context":{"idset":"11639"}}
+{"timestamp":1712275448.6183467,"name":"offline","context":{"idset":"11640"}}
+{"timestamp":1712275448.646378,"name":"offline","context":{"idset":"11641"}}
+{"timestamp":1712275449.5384986,"name":"offline","context":{"idset":"11643"}}
+{"timestamp":1712275449.5520771,"name":"offline","context":{"idset":"11644"}}
+{"timestamp":1712275449.5547063,"name":"offline","context":{"idset":"11645"}}
+{"timestamp":1712275449.5647779,"name":"offline","context":{"idset":"11646"}}
+{"timestamp":1712275449.5912421,"name":"offline","context":{"idset":"11647"}}
+{"timestamp":1712275449.5942848,"name":"offline","context":{"idset":"11648"}}
+{"timestamp":1712275449.5973122,"name":"offline","context":{"idset":"11649"}}
+{"timestamp":1712275449.6003475,"name":"offline","context":{"idset":"11650"}}
+{"timestamp":1712275449.6033666,"name":"offline","context":{"idset":"11651"}}
+{"timestamp":1712275449.6063907,"name":"offline","context":{"idset":"11652"}}
+{"timestamp":1712275607.3432059,"name":"online","context":{"idset":"842"}}
+{"timestamp":1712275619.6370995,"name":"drain","context":{"idset":"10205","overwrite":0}}
+{"timestamp":1712275736.2819407,"name":"drain","context":{"idset":"11269-11284","reason":"adam: test","overwrite":0}}
+{"timestamp":1712275845.8010297,"name":"undrain","context":{"idset":"11269-11284"}}
+{"timestamp":1712275875.555192,"name":"online","context":{"idset":"871-872"}}
+{"timestamp":1712275982.6992898,"name":"drain","context":{"idset":"11253-11380","reason":"adam: test","overwrite":0}}
+{"timestamp":1712276009.5492091,"name":"drain","context":{"idset":"11637-11641,11643","reason":"epilog failed for jobid fnTejN4cZHR","overwrite":0}}
+{"timestamp":1712276041.4657481,"name":"undrain","context":{"idset":"11253-11380"}}
+{"timestamp":1712276076.2928488,"name":"drain","context":{"idset":"11269-11284","reason":"adam: test","overwrite":0}}
+{"timestamp":1712276087.408658,"name":"undrain","context":{"idset":"11269-11284"}}
+{"timestamp":1712276705.2031317,"name":"online","context":{"idset":"10873,10905"}}
+{"timestamp":1712276705.2059059,"name":"online","context":{"idset":"10874,10906,10935-10936"}}
+{"timestamp":1712276705.2696457,"name":"online","context":{"idset":"10883"}}
+{"timestamp":1712276705.7620511,"name":"online","context":{"idset":"10884"}}
+{"timestamp":1712276897.4650202,"name":"undrain","context":{"idset":"10205-10206"}}
+{"timestamp":1712277505.2805984,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1712277522.2330797,"name":"online","context":{"idset":"10874"}}
+{"timestamp":1712277745.2407494,"name":"undrain","context":{"idset":"10873-10874,10906"}}
+{"timestamp":1712278201.5165467,"name":"online","context":{"idset":"10412"}}
+{"timestamp":1712278349.6227779,"name":"undrain","context":{"idset":"10412"}}
+{"timestamp":1712278885.2695019,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1712279439.4657333,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1712279664.5066693,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1712279665.2668674,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1712279665.2694035,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1712279665.2720218,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1712279665.2746291,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1712279665.2772808,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1712280455.4487255,"name":"online","context":{"idset":"10162"}}
+{"timestamp":1712280479.6446254,"name":"undrain","context":{"idset":"10162"}}
+{"timestamp":1712280717.455169,"name":"undrain","context":{"idset":"11187"}}
+{"timestamp":1712281427.4661973,"name":"drain","context":{"idset":"10374","reason":"nodediag failed dgemm_perf tapwrap","overwrite":0}}
+{"timestamp":1712281662.0422585,"name":"undrain","context":{"idset":"10207"}}
+{"timestamp":1712282651.4123049,"name":"online","context":{"idset":"11187"}}
+{"timestamp":1712283342.0240512,"name":"drain","context":{"idset":"11187","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712283366.0228555,"name":"drain","context":{"idset":"11175","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712283371.618238,"name":"drain","context":{"idset":"11180","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712283372.3378575,"name":"drain","context":{"idset":"11188","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712283373.4206862,"name":"drain","context":{"idset":"11181","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712283375.6480954,"name":"drain","context":{"idset":"11183","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712283376.4794605,"name":"drain","context":{"idset":"11177","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712283377.3986776,"name":"drain","context":{"idset":"11186","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712283385.4774759,"name":"drain","context":{"idset":"11179","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712283392.2250698,"name":"drain","context":{"idset":"11184","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712283395.9794703,"name":"drain","context":{"idset":"11176","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284193.296082,"name":"drain","context":{"idset":"11691","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284211.6072376,"name":"drain","context":{"idset":"11698","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284591.9534943,"name":"drain","context":{"idset":"11704","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284602.5684547,"name":"drain","context":{"idset":"11703","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284608.0223868,"name":"drain","context":{"idset":"11702","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284618.4463491,"name":"drain","context":{"idset":"11714","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284621.2324533,"name":"drain","context":{"idset":"11713","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712284623.5991323,"name":"drain","context":{"idset":"11705","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284624.6863372,"name":"drain","context":{"idset":"11715","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284626.2511165,"name":"drain","context":{"idset":"11709","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284630.7035587,"name":"drain","context":{"idset":"11712","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284630.9228861,"name":"drain","context":{"idset":"11706","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284631.8040702,"name":"drain","context":{"idset":"11707","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284633.3568602,"name":"drain","context":{"idset":"11711","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284635.6152203,"name":"drain","context":{"idset":"11710","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284759.9565878,"name":"drain","context":{"idset":"11685","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284762.0035577,"name":"drain","context":{"idset":"11687","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284790.0051301,"name":"drain","context":{"idset":"11689","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284793.6694391,"name":"drain","context":{"idset":"11692","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284797.8105862,"name":"drain","context":{"idset":"11696","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284799.217145,"name":"drain","context":{"idset":"11686","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284799.3191228,"name":"drain","context":{"idset":"11700","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712284799.4201963,"name":"drain","context":{"idset":"11695","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712284802.4379079,"name":"drain","context":{"idset":"11697","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284810.0878494,"name":"drain","context":{"idset":"11701","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284832.1324215,"name":"drain","context":{"idset":"11708","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284869.6989889,"name":"drain","context":{"idset":"11716","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284938.6257942,"name":"drain","context":{"idset":"11690","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284949.2307355,"name":"drain","context":{"idset":"11693","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712284964.5834908,"name":"drain","context":{"idset":"11694","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287699.3791277,"name":"drain","context":{"idset":"10579","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287699.5493069,"name":"drain","context":{"idset":"10576","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287701.5757816,"name":"drain","context":{"idset":"10890","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287703.8468909,"name":"drain","context":{"idset":"10630","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287704.043112,"name":"drain","context":{"idset":"10577","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287704.2476439,"name":"drain","context":{"idset":"10835","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287704.3974419,"name":"drain","context":{"idset":"10632","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287704.5413883,"name":"drain","context":{"idset":"10575","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287704.7054522,"name":"drain","context":{"idset":"10704","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287704.8681021,"name":"drain","context":{"idset":"10598","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287705.0545318,"name":"drain","context":{"idset":"10635","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287705.2732902,"name":"drain","context":{"idset":"10580","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287705.464963,"name":"drain","context":{"idset":"10636","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287705.6685755,"name":"drain","context":{"idset":"10742","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287705.8661294,"name":"drain","context":{"idset":"10633","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287706.0617137,"name":"drain","context":{"idset":"10708","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287706.2639267,"name":"drain","context":{"idset":"10706","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287706.4813592,"name":"drain","context":{"idset":"10602","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287706.6767247,"name":"drain","context":{"idset":"10631","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287706.8779249,"name":"drain","context":{"idset":"10646","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287707.0717456,"name":"drain","context":{"idset":"10578","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287708.1824534,"name":"drain","context":{"idset":"10608","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287708.388345,"name":"drain","context":{"idset":"10757","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287708.6029525,"name":"drain","context":{"idset":"10615","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287708.8102152,"name":"drain","context":{"idset":"10689","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287709.0070341,"name":"drain","context":{"idset":"10748","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287709.2045541,"name":"drain","context":{"idset":"10824","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287709.3894453,"name":"drain","context":{"idset":"10679","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287709.5814235,"name":"drain","context":{"idset":"10623","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287709.7851751,"name":"drain","context":{"idset":"10585","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287709.9256654,"name":"drain","context":{"idset":"10716","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287710.0311058,"name":"drain","context":{"idset":"10622","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287710.1308887,"name":"drain","context":{"idset":"10590","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287710.2423677,"name":"drain","context":{"idset":"10745","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287710.3375134,"name":"drain","context":{"idset":"10822","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287710.430999,"name":"drain","context":{"idset":"10624","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287710.531796,"name":"drain","context":{"idset":"10611","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287710.6249774,"name":"drain","context":{"idset":"10707","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287710.7177784,"name":"drain","context":{"idset":"10732","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287710.8108892,"name":"drain","context":{"idset":"10695","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287710.9052792,"name":"drain","context":{"idset":"10816","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287711.0138216,"name":"drain","context":{"idset":"10605","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287711.1076367,"name":"drain","context":{"idset":"10727","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287711.2001698,"name":"drain","context":{"idset":"10723","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287711.2931118,"name":"drain","context":{"idset":"10812","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287711.3870361,"name":"drain","context":{"idset":"10790","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287711.4820123,"name":"drain","context":{"idset":"10793","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287711.5972722,"name":"drain","context":{"idset":"10582","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287711.6904573,"name":"drain","context":{"idset":"10869","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287711.7841115,"name":"drain","context":{"idset":"10714","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287711.8790607,"name":"drain","context":{"idset":"10771","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287712.0336328,"name":"drain","context":{"idset":"10791","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287712.2134554,"name":"drain","context":{"idset":"10818","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287712.3774705,"name":"drain","context":{"idset":"10906","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287712.552248,"name":"drain","context":{"idset":"10648","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287712.726505,"name":"drain","context":{"idset":"10933","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287712.883291,"name":"drain","context":{"idset":"10950","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287713.0511169,"name":"drain","context":{"idset":"10738","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287713.2441761,"name":"drain","context":{"idset":"10658","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287713.3984108,"name":"drain","context":{"idset":"10826","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287713.5644364,"name":"drain","context":{"idset":"10765","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287713.7256114,"name":"drain","context":{"idset":"10752","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287713.8947227,"name":"drain","context":{"idset":"10743","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287714.0743251,"name":"drain","context":{"idset":"10779","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287714.2412488,"name":"drain","context":{"idset":"10856","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287714.4043067,"name":"drain","context":{"idset":"10830","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287714.5884416,"name":"drain","context":{"idset":"11221","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287714.7698772,"name":"drain","context":{"idset":"10831","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287714.9398844,"name":"drain","context":{"idset":"10690","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287715.1243868,"name":"drain","context":{"idset":"10862","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287715.3136594,"name":"drain","context":{"idset":"11396","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287715.4988544,"name":"drain","context":{"idset":"10888","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287715.7146099,"name":"drain","context":{"idset":"10957","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287715.889534,"name":"drain","context":{"idset":"10850","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287716.0405416,"name":"drain","context":{"idset":"10941","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287716.1917722,"name":"drain","context":{"idset":"10650","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287716.3535378,"name":"drain","context":{"idset":"10870","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287716.5126004,"name":"drain","context":{"idset":"11604","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287716.6737995,"name":"drain","context":{"idset":"10902","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287716.8254092,"name":"drain","context":{"idset":"10914","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287716.9830236,"name":"drain","context":{"idset":"10942","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287717.1152017,"name":"drain","context":{"idset":"10977","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287717.2337639,"name":"drain","context":{"idset":"10827","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287717.3416638,"name":"drain","context":{"idset":"10848","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287717.4557946,"name":"drain","context":{"idset":"11329","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287717.5577898,"name":"drain","context":{"idset":"10931","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287717.6641262,"name":"drain","context":{"idset":"10836","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287717.7564704,"name":"drain","context":{"idset":"10787","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287717.8598919,"name":"drain","context":{"idset":"11112","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287717.9545407,"name":"drain","context":{"idset":"10979","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287718.0542786,"name":"drain","context":{"idset":"841","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287718.1471729,"name":"drain","context":{"idset":"10952","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287718.2586834,"name":"drain","context":{"idset":"11399","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287718.3548954,"name":"drain","context":{"idset":"10946","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287718.4489665,"name":"drain","context":{"idset":"11114","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287718.5510008,"name":"drain","context":{"idset":"10958","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287718.6436677,"name":"drain","context":{"idset":"11393","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287718.7369328,"name":"drain","context":{"idset":"11366","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287718.8299868,"name":"drain","context":{"idset":"11149","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287718.9234316,"name":"drain","context":{"idset":"10992","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287719.0231054,"name":"drain","context":{"idset":"802","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287719.1337745,"name":"drain","context":{"idset":"11118","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287719.2325447,"name":"drain","context":{"idset":"804","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287719.3642321,"name":"drain","context":{"idset":"10944","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287719.4807901,"name":"drain","context":{"idset":"11154","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287719.6009474,"name":"drain","context":{"idset":"10975","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287719.6983087,"name":"drain","context":{"idset":"11165","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287719.7952366,"name":"drain","context":{"idset":"11121","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287719.8921857,"name":"drain","context":{"idset":"11199","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287719.9884224,"name":"drain","context":{"idset":"11332","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287720.0995593,"name":"drain","context":{"idset":"822","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287720.196629,"name":"drain","context":{"idset":"11610","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287720.2949595,"name":"drain","context":{"idset":"11189","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287720.3930612,"name":"drain","context":{"idset":"807","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287720.4963152,"name":"drain","context":{"idset":"11151","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287720.6040399,"name":"drain","context":{"idset":"10974","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287720.7027638,"name":"drain","context":{"idset":"11389","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287720.8008254,"name":"drain","context":{"idset":"11672","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287720.9020956,"name":"drain","context":{"idset":"11368","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287720.9971881,"name":"drain","context":{"idset":"11375","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287721.0977228,"name":"drain","context":{"idset":"954","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287721.2044544,"name":"drain","context":{"idset":"11671","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287721.2984507,"name":"drain","context":{"idset":"801","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287721.3956032,"name":"drain","context":{"idset":"11376","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287721.4904878,"name":"drain","context":{"idset":"9488","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287721.5869358,"name":"drain","context":{"idset":"10597","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287721.7073989,"name":"drain","context":{"idset":"10700","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287721.8147967,"name":"drain","context":{"idset":"11201","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287721.909903,"name":"drain","context":{"idset":"11143","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287722.0145152,"name":"drain","context":{"idset":"819","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287722.1234715,"name":"drain","context":{"idset":"11195","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287722.2201936,"name":"drain","context":{"idset":"10721","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287722.3369739,"name":"drain","context":{"idset":"10617","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287722.4498458,"name":"drain","context":{"idset":"10664","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287722.5536652,"name":"drain","context":{"idset":"826","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287722.6503549,"name":"drain","context":{"idset":"10591","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287722.7463446,"name":"drain","context":{"idset":"10696","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287722.8417733,"name":"drain","context":{"idset":"10720","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287722.9374146,"name":"drain","context":{"idset":"11211","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287723.0627584,"name":"drain","context":{"idset":"806","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287723.1619766,"name":"drain","context":{"idset":"11679","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287723.2749295,"name":"drain","context":{"idset":"10809","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287723.3759935,"name":"drain","context":{"idset":"821","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287723.4733086,"name":"drain","context":{"idset":"10640","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287723.5776687,"name":"drain","context":{"idset":"10651","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287723.6755226,"name":"drain","context":{"idset":"10596","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287723.7739847,"name":"drain","context":{"idset":"10600","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287723.8763452,"name":"drain","context":{"idset":"10760","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287723.9728272,"name":"drain","context":{"idset":"10728","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287724.0795486,"name":"drain","context":{"idset":"11196","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287724.1844614,"name":"drain","context":{"idset":"10613","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287724.2815742,"name":"drain","context":{"idset":"812","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287724.3827655,"name":"drain","context":{"idset":"11012","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287724.4899275,"name":"drain","context":{"idset":"10592","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287724.5927753,"name":"drain","context":{"idset":"11719","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287724.6861162,"name":"drain","context":{"idset":"830","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287724.7794216,"name":"drain","context":{"idset":"11115","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287724.877558,"name":"drain","context":{"idset":"810","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287724.9734519,"name":"drain","context":{"idset":"10587","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287725.0781927,"name":"drain","context":{"idset":"10619","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287725.1756699,"name":"drain","context":{"idset":"10785","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287725.2792153,"name":"drain","context":{"idset":"10620","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287725.3879111,"name":"drain","context":{"idset":"823","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287725.4858587,"name":"drain","context":{"idset":"11003","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287725.6095364,"name":"drain","context":{"idset":"11653","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287725.706893,"name":"drain","context":{"idset":"10609","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287725.8095596,"name":"drain","context":{"idset":"10722","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287725.9161544,"name":"drain","context":{"idset":"10676","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287726.0134957,"name":"drain","context":{"idset":"11675","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287726.1195664,"name":"drain","context":{"idset":"10581","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287726.2227576,"name":"drain","context":{"idset":"10599","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287726.3219922,"name":"drain","context":{"idset":"10641","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287726.4302101,"name":"drain","context":{"idset":"10724","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287726.5353925,"name":"drain","context":{"idset":"10680","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287726.6334224,"name":"drain","context":{"idset":"10647","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287726.7450976,"name":"drain","context":{"idset":"10701","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287726.8383338,"name":"drain","context":{"idset":"10672","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287726.9314618,"name":"drain","context":{"idset":"10715","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287727.0245764,"name":"drain","context":{"idset":"10588","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287727.1181054,"name":"drain","context":{"idset":"10659","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287727.2138774,"name":"drain","context":{"idset":"825","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287727.3091774,"name":"drain","context":{"idset":"10616","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287727.4155517,"name":"drain","context":{"idset":"10697","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287727.5123832,"name":"drain","context":{"idset":"834","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287727.6081111,"name":"drain","context":{"idset":"10759","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287727.7095556,"name":"drain","context":{"idset":"10698","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287727.8068743,"name":"drain","context":{"idset":"10583","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287727.9042814,"name":"drain","context":{"idset":"10710","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.0013559,"name":"drain","context":{"idset":"10854","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.1059794,"name":"drain","context":{"idset":"10682","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.2011881,"name":"drain","context":{"idset":"10774","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.2958114,"name":"drain","context":{"idset":"10625","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.3906708,"name":"drain","context":{"idset":"11749","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.4978573,"name":"drain","context":{"idset":"10778","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.5971897,"name":"drain","context":{"idset":"10703","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.6897736,"name":"drain","context":{"idset":"815","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.7815657,"name":"drain","context":{"idset":"10691","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.8734033,"name":"drain","context":{"idset":"10730","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287728.9655478,"name":"drain","context":{"idset":"10674","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287729.0593047,"name":"drain","context":{"idset":"10686","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287729.1597316,"name":"drain","context":{"idset":"10649","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287729.2538831,"name":"drain","context":{"idset":"10595","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287729.3534372,"name":"drain","context":{"idset":"10725","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287729.4472134,"name":"drain","context":{"idset":"11680","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287729.5428617,"name":"drain","context":{"idset":"10653","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287729.6368923,"name":"drain","context":{"idset":"10746","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287729.7399895,"name":"drain","context":{"idset":"10683","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287729.8367009,"name":"drain","context":{"idset":"11736","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287729.9327536,"name":"drain","context":{"idset":"10734","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287730.0277469,"name":"drain","context":{"idset":"10744","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287730.1296248,"name":"drain","context":{"idset":"11664","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287730.2334955,"name":"drain","context":{"idset":"10789","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287730.3632951,"name":"drain","context":{"idset":"10675","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287730.4596548,"name":"drain","context":{"idset":"10593","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287730.5591161,"name":"drain","context":{"idset":"10621","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287730.6586504,"name":"drain","context":{"idset":"10726","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287730.7728391,"name":"drain","context":{"idset":"10805","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287730.8693113,"name":"drain","context":{"idset":"11722","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287730.9660542,"name":"drain","context":{"idset":"11665","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287731.0626354,"name":"drain","context":{"idset":"11684","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287731.2380967,"name":"drain","context":{"idset":"10717","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287731.394068,"name":"drain","context":{"idset":"831","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287731.5566967,"name":"drain","context":{"idset":"10855","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287731.719326,"name":"drain","context":{"idset":"10687","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287731.8803518,"name":"drain","context":{"idset":"10904","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287732.1475255,"name":"drain","context":{"idset":"10737","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287732.4863164,"name":"drain","context":{"idset":"10655","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287732.8097041,"name":"drain","context":{"idset":"10667","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287733.0791256,"name":"drain","context":{"idset":"10584","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287733.2975934,"name":"drain","context":{"idset":"11734","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287733.5002091,"name":"drain","context":{"idset":"10753","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287733.7003558,"name":"drain","context":{"idset":"10662","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287733.8947728,"name":"drain","context":{"idset":"10709","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287734.0948653,"name":"drain","context":{"idset":"10797","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287734.2925067,"name":"drain","context":{"idset":"10802","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287734.5016785,"name":"drain","context":{"idset":"11656","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287734.7015038,"name":"drain","context":{"idset":"10770","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287734.9014351,"name":"drain","context":{"idset":"9469","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287735.0680747,"name":"drain","context":{"idset":"10731","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287735.2073853,"name":"drain","context":{"idset":"11663","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287735.3707082,"name":"drain","context":{"idset":"824","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287735.5004475,"name":"drain","context":{"idset":"10685","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287735.6456468,"name":"drain","context":{"idset":"10678","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287735.8204639,"name":"drain","context":{"idset":"10749","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287736.0180719,"name":"drain","context":{"idset":"10626","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287736.2134514,"name":"drain","context":{"idset":"10614","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287736.4044771,"name":"drain","context":{"idset":"11666","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287736.5988729,"name":"drain","context":{"idset":"10735","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287736.7979498,"name":"drain","context":{"idset":"814","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287736.9970434,"name":"drain","context":{"idset":"10847","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287737.2015517,"name":"drain","context":{"idset":"10665","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287737.4131536,"name":"drain","context":{"idset":"10666","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287737.6076825,"name":"drain","context":{"idset":"10671","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287737.8071718,"name":"drain","context":{"idset":"11717","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287738.0069444,"name":"drain","context":{"idset":"10681","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287738.2065997,"name":"drain","context":{"idset":"10794","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287738.4125371,"name":"drain","context":{"idset":"10733","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287738.6068218,"name":"drain","context":{"idset":"10705","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287738.8029716,"name":"drain","context":{"idset":"11735","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287739.0051982,"name":"drain","context":{"idset":"11219","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287739.2136252,"name":"drain","context":{"idset":"10601","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287739.4339921,"name":"drain","context":{"idset":"10603","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287739.6245327,"name":"drain","context":{"idset":"10663","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287739.8181801,"name":"drain","context":{"idset":"10873","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287740.0211921,"name":"drain","context":{"idset":"11728","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287740.2196381,"name":"drain","context":{"idset":"11762","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287740.413182,"name":"drain","context":{"idset":"10652","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287740.6081851,"name":"drain","context":{"idset":"11748","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287740.8119049,"name":"drain","context":{"idset":"10769","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287740.9393239,"name":"drain","context":{"idset":"10612","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287741.0505936,"name":"drain","context":{"idset":"10767","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287741.161902,"name":"drain","context":{"idset":"10833","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287741.2564752,"name":"drain","context":{"idset":"10884","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287741.3516946,"name":"drain","context":{"idset":"10736","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287741.4491372,"name":"drain","context":{"idset":"10803","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287741.5518558,"name":"drain","context":{"idset":"10684","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287741.6471386,"name":"drain","context":{"idset":"10607","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287741.7467122,"name":"drain","context":{"idset":"836","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287741.8402393,"name":"drain","context":{"idset":"10692","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287741.933527,"name":"drain","context":{"idset":"11750","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287742.0299904,"name":"drain","context":{"idset":"10777","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287742.1233878,"name":"drain","context":{"idset":"10903","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287742.2166562,"name":"drain","context":{"idset":"11733","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287742.3101285,"name":"drain","context":{"idset":"10879","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287742.403898,"name":"drain","context":{"idset":"11747","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287742.5079806,"name":"drain","context":{"idset":"10610","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287742.6042964,"name":"drain","context":{"idset":"11746","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287742.7077608,"name":"drain","context":{"idset":"10755","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287742.802592,"name":"drain","context":{"idset":"10711","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287742.9047346,"name":"drain","context":{"idset":"10719","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287743.0035939,"name":"drain","context":{"idset":"10781","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287743.1084929,"name":"drain","context":{"idset":"10782","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287743.2190149,"name":"drain","context":{"idset":"10853","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287743.314579,"name":"drain","context":{"idset":"10905","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287743.409704,"name":"drain","context":{"idset":"10934","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287743.5147266,"name":"drain","context":{"idset":"10800","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287743.6106293,"name":"drain","context":{"idset":"10872","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287743.7064497,"name":"drain","context":{"idset":"10877","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287743.807492,"name":"drain","context":{"idset":"10828","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287743.9132466,"name":"drain","context":{"idset":"10604","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287744.013056,"name":"drain","context":{"idset":"10713","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287744.1096625,"name":"drain","context":{"idset":"10754","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287744.2097218,"name":"drain","context":{"idset":"10763","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287744.3412523,"name":"drain","context":{"idset":"10775","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287744.5136611,"name":"drain","context":{"idset":"10807","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287744.6830065,"name":"drain","context":{"idset":"11745","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287744.8522594,"name":"drain","context":{"idset":"10792","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287745.0284162,"name":"drain","context":{"idset":"10806","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287745.2086246,"name":"drain","context":{"idset":"10945","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287745.3936777,"name":"drain","context":{"idset":"10844","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287745.5651987,"name":"drain","context":{"idset":"10586","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287745.7190311,"name":"drain","context":{"idset":"10881","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287745.8846984,"name":"drain","context":{"idset":"10783","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287746.0454659,"name":"drain","context":{"idset":"10859","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287746.1977293,"name":"drain","context":{"idset":"10810","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287746.3514411,"name":"drain","context":{"idset":"10821","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287746.5244267,"name":"drain","context":{"idset":"11756","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287746.6982925,"name":"drain","context":{"idset":"10915","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287746.8564467,"name":"drain","context":{"idset":"833","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287747.0143776,"name":"drain","context":{"idset":"10841","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287747.1497512,"name":"drain","context":{"idset":"10780","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287747.2797635,"name":"drain","context":{"idset":"10936","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287747.3939519,"name":"drain","context":{"idset":"10668","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287747.5025408,"name":"drain","context":{"idset":"11333","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287747.5966361,"name":"drain","context":{"idset":"10814","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287747.6955466,"name":"drain","context":{"idset":"10982","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287747.7885461,"name":"drain","context":{"idset":"10966","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287747.8913405,"name":"drain","context":{"idset":"11287","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287747.9845436,"name":"drain","context":{"idset":"10677","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287748.0771732,"name":"drain","context":{"idset":"10699","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287748.1747947,"name":"drain","context":{"idset":"10750","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287748.2705705,"name":"drain","context":{"idset":"10849","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287748.3644178,"name":"drain","context":{"idset":"10897","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287748.464242,"name":"drain","context":{"idset":"11178","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287748.5574751,"name":"drain","context":{"idset":"871","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287748.6508689,"name":"drain","context":{"idset":"10729","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287748.7671535,"name":"drain","context":{"idset":"10808","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287748.8858333,"name":"drain","context":{"idset":"10922","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287749.006062,"name":"drain","context":{"idset":"10660","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287749.1061382,"name":"drain","context":{"idset":"10845","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287749.2109027,"name":"drain","context":{"idset":"10894","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287749.3531337,"name":"drain","context":{"idset":"10820","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287749.5211895,"name":"drain","context":{"idset":"11109","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287749.6948698,"name":"drain","context":{"idset":"10670","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287749.8593445,"name":"drain","context":{"idset":"10773","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287750.0365355,"name":"drain","context":{"idset":"10857","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287750.2193897,"name":"drain","context":{"idset":"10876","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287750.4131451,"name":"drain","context":{"idset":"10993","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287750.6019523,"name":"drain","context":{"idset":"10926","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287750.7881618,"name":"drain","context":{"idset":"10747","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287750.9467082,"name":"drain","context":{"idset":"10851","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287751.140908,"name":"drain","context":{"idset":"11737","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287751.3254187,"name":"drain","context":{"idset":"10840","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287751.516525,"name":"drain","context":{"idset":"10628","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287751.6953096,"name":"drain","context":{"idset":"11764","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287751.8688173,"name":"drain","context":{"idset":"11699","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287752.0296011,"name":"drain","context":{"idset":"10795","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287752.1903639,"name":"drain","context":{"idset":"11157","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287752.3647213,"name":"drain","context":{"idset":"10919","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287752.5308537,"name":"drain","context":{"idset":"10764","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287752.703351,"name":"drain","context":{"idset":"10892","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287752.8658755,"name":"drain","context":{"idset":"11763","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287753.0264766,"name":"drain","context":{"idset":"10786","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287753.1617198,"name":"drain","context":{"idset":"10813","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287753.3381009,"name":"drain","context":{"idset":"10907","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287753.4959655,"name":"drain","context":{"idset":"10798","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287753.6519673,"name":"drain","context":{"idset":"10837","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287753.8315914,"name":"drain","context":{"idset":"11755","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287754.0009236,"name":"drain","context":{"idset":"10825","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287754.1669352,"name":"drain","context":{"idset":"11124","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287754.336323,"name":"drain","context":{"idset":"10801","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287754.5118902,"name":"drain","context":{"idset":"10861","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287754.6736286,"name":"drain","context":{"idset":"10937","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287754.8319187,"name":"drain","context":{"idset":"11759","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287754.997905,"name":"drain","context":{"idset":"10910","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287755.1754711,"name":"drain","context":{"idset":"10860","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287755.3365378,"name":"drain","context":{"idset":"10796","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287755.4967248,"name":"drain","context":{"idset":"10916","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287755.6774111,"name":"drain","context":{"idset":"11137","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287755.8012259,"name":"drain","context":{"idset":"10967","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287755.9130602,"name":"drain","context":{"idset":"11222","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287756.0172975,"name":"drain","context":{"idset":"11327","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287756.1413519,"name":"drain","context":{"idset":"10887","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287756.249373,"name":"drain","context":{"idset":"10829","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287756.3553238,"name":"drain","context":{"idset":"10838","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287756.4800155,"name":"drain","context":{"idset":"10880","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287756.6198666,"name":"drain","context":{"idset":"10882","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287756.7216885,"name":"drain","context":{"idset":"10953","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287756.8220108,"name":"drain","context":{"idset":"10963","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287756.9200268,"name":"drain","context":{"idset":"11758","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287757.0181251,"name":"drain","context":{"idset":"10784","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287757.1160486,"name":"drain","context":{"idset":"10940","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287757.2142274,"name":"drain","context":{"idset":"10863","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287757.3231757,"name":"drain","context":{"idset":"10927","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287757.4248395,"name":"drain","context":{"idset":"10923","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287757.5323894,"name":"drain","context":{"idset":"10768","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287757.6351144,"name":"drain","context":{"idset":"10839","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287757.8117676,"name":"drain","context":{"idset":"10823","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287757.971719,"name":"drain","context":{"idset":"10909","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287758.1690755,"name":"drain","context":{"idset":"10954","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287758.3343287,"name":"drain","context":{"idset":"10865","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287758.5125532,"name":"drain","context":{"idset":"10654","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287758.6754589,"name":"drain","context":{"idset":"10935","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287758.8349361,"name":"drain","context":{"idset":"11398","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287758.9951198,"name":"drain","context":{"idset":"10766","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287759.1833577,"name":"drain","context":{"idset":"10924","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287759.3993604,"name":"drain","context":{"idset":"10893","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287759.5428605,"name":"drain","context":{"idset":"11139","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287759.6737416,"name":"drain","context":{"idset":"10970","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287759.7971451,"name":"drain","context":{"idset":"10868","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287759.9035623,"name":"drain","context":{"idset":"11760","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287760.0037661,"name":"drain","context":{"idset":"10669","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287760.1046515,"name":"drain","context":{"idset":"10898","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287760.2067704,"name":"drain","context":{"idset":"10948","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287760.3250003,"name":"drain","context":{"idset":"11324","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287760.4897041,"name":"drain","context":{"idset":"10762","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287760.6885598,"name":"drain","context":{"idset":"10846","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287760.8555627,"name":"drain","context":{"idset":"10799","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287761.0249093,"name":"drain","context":{"idset":"10875","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287761.1941493,"name":"drain","context":{"idset":"11394","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287761.373971,"name":"drain","context":{"idset":"800","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287761.5436575,"name":"drain","context":{"idset":"10900","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287761.7218482,"name":"drain","context":{"idset":"10889","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287761.8885593,"name":"drain","context":{"idset":"10864","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287762.1674781,"name":"drain","context":{"idset":"10983","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287762.4689491,"name":"drain","context":{"idset":"10891","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287762.7135031,"name":"drain","context":{"idset":"10956","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287762.9196916,"name":"drain","context":{"idset":"10817","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287763.1189821,"name":"drain","context":{"idset":"803","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287763.3238995,"name":"drain","context":{"idset":"10984","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287763.5231857,"name":"drain","context":{"idset":"10986","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287763.7181346,"name":"drain","context":{"idset":"10896","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287763.9349661,"name":"drain","context":{"idset":"10772","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287764.1924303,"name":"drain","context":{"idset":"10866","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287764.3948781,"name":"drain","context":{"idset":"10834","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287764.593473,"name":"drain","context":{"idset":"10885","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287764.7952545,"name":"drain","context":{"idset":"10973","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287764.9890053,"name":"drain","context":{"idset":"11158","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287765.1567528,"name":"drain","context":{"idset":"11369","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287765.2929411,"name":"drain","context":{"idset":"10991","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287765.4271567,"name":"drain","context":{"idset":"10918","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287765.5599437,"name":"drain","context":{"idset":"11325","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287765.7203295,"name":"drain","context":{"idset":"10968","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287765.9099004,"name":"drain","context":{"idset":"11159","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287766.1019092,"name":"drain","context":{"idset":"11122","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287766.2985754,"name":"drain","context":{"idset":"10964","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287766.4995661,"name":"drain","context":{"idset":"10987","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287766.696316,"name":"drain","context":{"idset":"10989","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287766.898721,"name":"drain","context":{"idset":"10994","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287767.1000476,"name":"drain","context":{"idset":"10895","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287767.2958548,"name":"drain","context":{"idset":"10965","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287767.4929531,"name":"drain","context":{"idset":"10981","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287767.6949313,"name":"drain","context":{"idset":"10858","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287767.9002836,"name":"drain","context":{"idset":"10938","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287768.097949,"name":"drain","context":{"idset":"10932","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287768.306551,"name":"drain","context":{"idset":"10959","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287768.5075853,"name":"drain","context":{"idset":"11113","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287768.7258809,"name":"drain","context":{"idset":"11135","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287768.9319942,"name":"drain","context":{"idset":"10930","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287769.128473,"name":"drain","context":{"idset":"10917","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287769.3177159,"name":"drain","context":{"idset":"11110","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287769.5289493,"name":"drain","context":{"idset":"11367","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287769.7378199,"name":"drain","context":{"idset":"10811","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287769.9431181,"name":"drain","context":{"idset":"11334","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287770.1428342,"name":"drain","context":{"idset":"10920","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287770.3443127,"name":"drain","context":{"idset":"10929","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287770.5429602,"name":"drain","context":{"idset":"10899","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287770.7402244,"name":"drain","context":{"idset":"11391","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287770.8498361,"name":"drain","context":{"idset":"949","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287770.9500444,"name":"drain","context":{"idset":"11218","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287771.0495481,"name":"drain","context":{"idset":"10815","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287771.148515,"name":"drain","context":{"idset":"10804","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287771.2463298,"name":"drain","context":{"idset":"11603","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287771.3441832,"name":"drain","context":{"idset":"799","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287771.4421282,"name":"drain","context":{"idset":"11123","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287771.5492928,"name":"drain","context":{"idset":"11210","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287771.6460042,"name":"drain","context":{"idset":"953","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287771.7415504,"name":"drain","context":{"idset":"11162","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287771.8362002,"name":"drain","context":{"idset":"10955","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287771.931237,"name":"drain","context":{"idset":"10788","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287772.0275779,"name":"drain","context":{"idset":"11111","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287772.1507373,"name":"drain","context":{"idset":"11392","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287772.2685318,"name":"drain","context":{"idset":"11173","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287772.3827295,"name":"drain","context":{"idset":"10947","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287772.4910493,"name":"drain","context":{"idset":"11216","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287772.5859985,"name":"drain","context":{"idset":"11397","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287772.7475309,"name":"drain","context":{"idset":"976","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287772.9138818,"name":"drain","context":{"idset":"11326","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287773.0924029,"name":"drain","context":{"idset":"10939","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287773.2630641,"name":"drain","context":{"idset":"11166","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287773.4329989,"name":"drain","context":{"idset":"11174","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287773.6127477,"name":"drain","context":{"idset":"11170","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287773.795857,"name":"drain","context":{"idset":"11117","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287773.971293,"name":"drain","context":{"idset":"10776","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287774.1453352,"name":"drain","context":{"idset":"10960","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287774.3165293,"name":"drain","context":{"idset":"10972","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287774.5102985,"name":"drain","context":{"idset":"11150","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287774.7046168,"name":"drain","context":{"idset":"11119","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287774.8857465,"name":"drain","context":{"idset":"11213","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287775.052223,"name":"drain","context":{"idset":"11194","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287775.2254405,"name":"drain","context":{"idset":"10878","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287775.3955724,"name":"drain","context":{"idset":"11193","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287775.5778661,"name":"drain","context":{"idset":"11160","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287775.7523551,"name":"drain","context":{"idset":"11120","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287775.9187174,"name":"drain","context":{"idset":"11209","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287776.0879745,"name":"drain","context":{"idset":"10928","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287776.2367194,"name":"drain","context":{"idset":"10969","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287776.3749905,"name":"drain","context":{"idset":"10995","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287776.4853354,"name":"drain","context":{"idset":"10921","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287776.6081727,"name":"drain","context":{"idset":"11605","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287776.7112405,"name":"drain","context":{"idset":"11145","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287776.8334708,"name":"drain","context":{"idset":"11168","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287776.9607484,"name":"drain","context":{"idset":"11214","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287777.0612011,"name":"drain","context":{"idset":"11171","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287777.156708,"name":"drain","context":{"idset":"11141","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287777.2518561,"name":"drain","context":{"idset":"9461","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287777.3475637,"name":"drain","context":{"idset":"10961","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287777.4437177,"name":"drain","context":{"idset":"11133","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287777.5499182,"name":"drain","context":{"idset":"11142","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287777.6497183,"name":"drain","context":{"idset":"11182","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287777.7486846,"name":"drain","context":{"idset":"964","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287777.8537872,"name":"drain","context":{"idset":"10988","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287777.9524739,"name":"drain","context":{"idset":"11161","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287778.0649624,"name":"drain","context":{"idset":"10589","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287778.1690969,"name":"drain","context":{"idset":"9512","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287778.270004,"name":"drain","context":{"idset":"11372","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287778.3768198,"name":"drain","context":{"idset":"11163","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287778.4972503,"name":"drain","context":{"idset":"11144","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287778.5964062,"name":"drain","context":{"idset":"11185","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287778.6959751,"name":"drain","context":{"idset":"11164","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287778.8000312,"name":"drain","context":{"idset":"9518","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287778.9126866,"name":"drain","context":{"idset":"11328","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287779.041678,"name":"drain","context":{"idset":"11206","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287779.1470447,"name":"drain","context":{"idset":"11155","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287779.2478118,"name":"drain","context":{"idset":"10990","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287779.3472593,"name":"drain","context":{"idset":"11374","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287779.452085,"name":"drain","context":{"idset":"10976","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287779.5515101,"name":"drain","context":{"idset":"9510","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287779.6758299,"name":"drain","context":{"idset":"11152","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287779.7762966,"name":"drain","context":{"idset":"11607","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287779.8757355,"name":"drain","context":{"idset":"11191","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287779.9781034,"name":"drain","context":{"idset":"9495","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287780.0829067,"name":"drain","context":{"idset":"11208","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287780.1867657,"name":"drain","context":{"idset":"9468","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287780.2828991,"name":"drain","context":{"idset":"11217","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287780.3929996,"name":"drain","context":{"idset":"11147","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287780.5408502,"name":"drain","context":{"idset":"808","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287780.639787,"name":"drain","context":{"idset":"11669","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287780.735249,"name":"drain","context":{"idset":"11198","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287780.832109,"name":"drain","context":{"idset":"11148","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287780.92746,"name":"drain","context":{"idset":"11611","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287781.0227673,"name":"drain","context":{"idset":"11172","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287781.117897,"name":"drain","context":{"idset":"11169","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287781.2186017,"name":"drain","context":{"idset":"9473","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287781.331001,"name":"drain","context":{"idset":"11215","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287781.4289038,"name":"drain","context":{"idset":"11190","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287781.5246544,"name":"drain","context":{"idset":"11371","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287781.620188,"name":"drain","context":{"idset":"832","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287781.728822,"name":"drain","context":{"idset":"11204","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287781.8434386,"name":"drain","context":{"idset":"9494","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287781.9563708,"name":"drain","context":{"idset":"11613","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287782.0572424,"name":"drain","context":{"idset":"10978","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287782.153862,"name":"drain","context":{"idset":"11197","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287782.2600467,"name":"drain","context":{"idset":"11200","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287782.3795295,"name":"drain","context":{"idset":"11156","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287782.5017829,"name":"drain","context":{"idset":"9504","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287782.5987475,"name":"drain","context":{"idset":"805","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287782.6990407,"name":"drain","context":{"idset":"11370","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287782.7990179,"name":"drain","context":{"idset":"9481","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287782.8963928,"name":"drain","context":{"idset":"11677","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287782.9922714,"name":"drain","context":{"idset":"11606","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287783.0888991,"name":"drain","context":{"idset":"11608","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287783.1868505,"name":"drain","context":{"idset":"820","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287783.2908812,"name":"drain","context":{"idset":"11609","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287783.3897557,"name":"drain","context":{"idset":"11688","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287783.4870198,"name":"drain","context":{"idset":"11654","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287783.5860567,"name":"drain","context":{"idset":"11670","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287783.6907158,"name":"drain","context":{"idset":"11205","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287783.7915001,"name":"drain","context":{"idset":"11614","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287783.8923311,"name":"drain","context":{"idset":"11720","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287783.991426,"name":"drain","context":{"idset":"11234","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287784.0921612,"name":"drain","context":{"idset":"11676","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287784.1938751,"name":"drain","context":{"idset":"11615","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287784.2932303,"name":"drain","context":{"idset":"11212","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287784.404706,"name":"drain","context":{"idset":"828","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287784.5040643,"name":"drain","context":{"idset":"11612","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287784.6064494,"name":"drain","context":{"idset":"9482","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287784.8043282,"name":"drain","context":{"idset":"9466","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287784.9725435,"name":"drain","context":{"idset":"9464","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287785.133472,"name":"drain","context":{"idset":"11153","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287785.294039,"name":"drain","context":{"idset":"11731","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287785.4469671,"name":"drain","context":{"idset":"11146","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287785.6038334,"name":"drain","context":{"idset":"811","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287785.7820492,"name":"drain","context":{"idset":"10594","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287785.9570026,"name":"drain","context":{"idset":"11251","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287786.1454279,"name":"drain","context":{"idset":"9471","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287786.2990842,"name":"drain","context":{"idset":"9516","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287786.4547551,"name":"drain","context":{"idset":"9484","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287786.6277978,"name":"drain","context":{"idset":"9499","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287786.7588851,"name":"drain","context":{"idset":"9485","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287786.8706026,"name":"drain","context":{"idset":"11225","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287786.9725482,"name":"drain","context":{"idset":"10980","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287787.0678613,"name":"drain","context":{"idset":"11668","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287787.1685324,"name":"drain","context":{"idset":"9501","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287787.2642791,"name":"drain","context":{"idset":"11741","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287787.3595881,"name":"drain","context":{"idset":"11207","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287787.5060904,"name":"drain","context":{"idset":"829","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287787.6795857,"name":"drain","context":{"idset":"9462","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287787.8496604,"name":"drain","context":{"idset":"9472","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287788.0123823,"name":"drain","context":{"idset":"9507","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287788.180882,"name":"drain","context":{"idset":"10606","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287788.3522358,"name":"drain","context":{"idset":"818","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287788.5180073,"name":"drain","context":{"idset":"9496","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287788.6830184,"name":"drain","context":{"idset":"11673","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287788.8587766,"name":"drain","context":{"idset":"9503","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287789.0714202,"name":"drain","context":{"idset":"11678","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287789.2388337,"name":"drain","context":{"idset":"9475","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287789.3946333,"name":"drain","context":{"idset":"9519","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287789.5254138,"name":"drain","context":{"idset":"10645","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287789.6419942,"name":"drain","context":{"idset":"9487","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287789.7482834,"name":"drain","context":{"idset":"9492","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287789.8560834,"name":"drain","context":{"idset":"11655","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287789.9570866,"name":"drain","context":{"idset":"10627","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287790.0564544,"name":"drain","context":{"idset":"11659","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287790.1544106,"name":"drain","context":{"idset":"9498","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287790.2539766,"name":"drain","context":{"idset":"11658","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287790.3514886,"name":"drain","context":{"idset":"9506","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287790.4568083,"name":"drain","context":{"idset":"11724","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287790.5539856,"name":"drain","context":{"idset":"817","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287790.657546,"name":"drain","context":{"idset":"963","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287790.7541416,"name":"drain","context":{"idset":"9463","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287790.8516881,"name":"drain","context":{"idset":"11661","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287790.9491887,"name":"drain","context":{"idset":"809","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287791.05496,"name":"drain","context":{"idset":"9513","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287791.153085,"name":"drain","context":{"idset":"11727","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287791.2544844,"name":"drain","context":{"idset":"9509","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287791.3502793,"name":"drain","context":{"idset":"10741","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287791.4769933,"name":"drain","context":{"idset":"9478","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287791.6360567,"name":"drain","context":{"idset":"11732","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287791.7512586,"name":"drain","context":{"idset":"11660","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287791.9278543,"name":"drain","context":{"idset":"827","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287792.1416659,"name":"drain","context":{"idset":"11740","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287792.3552792,"name":"drain","context":{"idset":"9515","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287792.5588706,"name":"drain","context":{"idset":"11662","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287792.7648845,"name":"drain","context":{"idset":"10874","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287792.9673443,"name":"drain","context":{"idset":"835","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287793.1866412,"name":"drain","context":{"idset":"813","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287793.3907087,"name":"drain","context":{"idset":"11723","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287793.5820305,"name":"drain","context":{"idset":"11738","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287793.7945745,"name":"drain","context":{"idset":"9505","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287793.9883201,"name":"drain","context":{"idset":"10661","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287794.1866083,"name":"drain","context":{"idset":"11742","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287794.3898139,"name":"drain","context":{"idset":"9474","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287794.5906327,"name":"drain","context":{"idset":"11721","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287794.788908,"name":"drain","context":{"idset":"11238","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287794.9362085,"name":"drain","context":{"idset":"10618","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287795.0749812,"name":"drain","context":{"idset":"10756","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287795.2188046,"name":"drain","context":{"idset":"10688","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287795.3719783,"name":"drain","context":{"idset":"11729","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287795.563123,"name":"drain","context":{"idset":"10673","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287795.756228,"name":"drain","context":{"idset":"816","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287795.9503772,"name":"drain","context":{"idset":"11683","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287796.1597402,"name":"drain","context":{"idset":"10886","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287796.3614869,"name":"drain","context":{"idset":"11739","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287796.573256,"name":"drain","context":{"idset":"11743","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287796.770854,"name":"drain","context":{"idset":"10843","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287796.9877274,"name":"drain","context":{"idset":"11681","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287797.1875412,"name":"drain","context":{"idset":"11744","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287797.3882248,"name":"drain","context":{"idset":"10908","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287797.5823996,"name":"drain","context":{"idset":"11730","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287797.7985165,"name":"drain","context":{"idset":"10656","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287797.9966404,"name":"drain","context":{"idset":"10751","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287798.1908708,"name":"drain","context":{"idset":"11132","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287798.3984916,"name":"drain","context":{"idset":"10761","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287798.6010208,"name":"drain","context":{"idset":"10913","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287798.8098509,"name":"drain","context":{"idset":"10758","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287799.0220451,"name":"drain","context":{"idset":"11753","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287799.2305744,"name":"drain","context":{"idset":"10949","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287799.4346461,"name":"drain","context":{"idset":"11752","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287799.6404712,"name":"drain","context":{"idset":"11754","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287799.8547978,"name":"drain","context":{"idset":"11667","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287800.0513878,"name":"drain","context":{"idset":"11757","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287800.239933,"name":"drain","context":{"idset":"872","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287800.4274237,"name":"drain","context":{"idset":"11227","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287800.5573771,"name":"drain","context":{"idset":"10951","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287800.655477,"name":"drain","context":{"idset":"10842","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287800.7527435,"name":"drain","context":{"idset":"11725","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287800.8638422,"name":"drain","context":{"idset":"10852","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287800.9680226,"name":"drain","context":{"idset":"842","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287801.0679495,"name":"drain","context":{"idset":"10883","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287801.1724622,"name":"drain","context":{"idset":"10985","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287801.2912266,"name":"drain","context":{"idset":"10925","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287801.3927875,"name":"drain","context":{"idset":"11761","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287801.4928966,"name":"drain","context":{"idset":"950","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287801.6004865,"name":"drain","context":{"idset":"10832","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287801.704119,"name":"drain","context":{"idset":"11331","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287801.8138957,"name":"drain","context":{"idset":"10962","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287801.9256649,"name":"drain","context":{"idset":"11751","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287802.0336337,"name":"drain","context":{"idset":"10819","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287802.1406641,"name":"drain","context":{"idset":"10971","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287802.2566957,"name":"drain","context":{"idset":"10901","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287802.3604734,"name":"drain","context":{"idset":"11167","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287802.4781544,"name":"drain","context":{"idset":"11202","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287802.5917573,"name":"drain","context":{"idset":"9517","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287802.6965077,"name":"drain","context":{"idset":"10867","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287802.7987463,"name":"drain","context":{"idset":"975","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287802.906229,"name":"drain","context":{"idset":"9514","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287803.0130603,"name":"drain","context":{"idset":"9467","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287803.1291716,"name":"drain","context":{"idset":"9465","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287803.7500322,"name":"drain","context":{"idset":"11192","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287803.857219,"name":"drain","context":{"idset":"9479","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287803.9627032,"name":"drain","context":{"idset":"9476","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287804.0568528,"name":"drain","context":{"idset":"9500","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287804.1708484,"name":"drain","context":{"idset":"11237","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287804.3873379,"name":"drain","context":{"idset":"11203","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287804.5682311,"name":"drain","context":{"idset":"9491","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287804.7315121,"name":"drain","context":{"idset":"11718","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287804.8989153,"name":"drain","context":{"idset":"11242","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287805.0694988,"name":"drain","context":{"idset":"11802","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287805.2365179,"name":"drain","context":{"idset":"9486","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287805.423743,"name":"drain","context":{"idset":"9483","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287805.6056478,"name":"drain","context":{"idset":"11682","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287805.7905166,"name":"drain","context":{"idset":"9489","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287805.9391329,"name":"drain","context":{"idset":"9490","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287806.1001534,"name":"drain","context":{"idset":"9521","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287806.2197592,"name":"drain","context":{"idset":"9477","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287806.3251121,"name":"drain","context":{"idset":"9508","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287806.4507258,"name":"drain","context":{"idset":"9497","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287806.5514257,"name":"drain","context":{"idset":"11726","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287806.6597972,"name":"drain","context":{"idset":"11228","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287806.7641778,"name":"drain","context":{"idset":"9493","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287806.8739061,"name":"drain","context":{"idset":"9502","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287807.0079861,"name":"drain","context":{"idset":"11224","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287807.1253622,"name":"drain","context":{"idset":"11674","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287807.2410009,"name":"drain","context":{"idset":"11240","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287807.3524427,"name":"drain","context":{"idset":"11657","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287807.473506,"name":"drain","context":{"idset":"9522","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287807.5927818,"name":"drain","context":{"idset":"9480","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287807.7359817,"name":"drain","context":{"idset":"9559","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1712287807.8549757,"name":"drain","context":{"idset":"11245","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287808.0217147,"name":"drain","context":{"idset":"9542","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287808.1345868,"name":"drain","context":{"idset":"9470","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287808.2456951,"name":"drain","context":{"idset":"9527","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287808.3506114,"name":"drain","context":{"idset":"11232","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287808.4705589,"name":"drain","context":{"idset":"11248","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287808.5780883,"name":"drain","context":{"idset":"9558","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287808.6787817,"name":"drain","context":{"idset":"11230","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287808.7852409,"name":"drain","context":{"idset":"9520","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287808.9142606,"name":"drain","context":{"idset":"9524","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287809.0377865,"name":"drain","context":{"idset":"11229","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287809.1440537,"name":"drain","context":{"idset":"11252","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287809.2494283,"name":"drain","context":{"idset":"9511","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287809.3549588,"name":"drain","context":{"idset":"9551","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287809.4723039,"name":"drain","context":{"idset":"9523","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287809.5786884,"name":"drain","context":{"idset":"9540","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287809.6937335,"name":"drain","context":{"idset":"9536","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287809.7946138,"name":"drain","context":{"idset":"9535","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287809.919168,"name":"drain","context":{"idset":"9539","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287810.0419717,"name":"drain","context":{"idset":"9724","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287810.1514931,"name":"drain","context":{"idset":"9717","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287810.2671237,"name":"drain","context":{"idset":"11241","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287810.3842585,"name":"drain","context":{"idset":"9742","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287810.4922237,"name":"drain","context":{"idset":"9982","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287810.5937264,"name":"drain","context":{"idset":"9537","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287810.7060533,"name":"drain","context":{"idset":"9525","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287810.8267956,"name":"drain","context":{"idset":"9997","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287810.9275908,"name":"drain","context":{"idset":"9974","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287811.0283124,"name":"drain","context":{"idset":"9722","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287811.1358671,"name":"drain","context":{"idset":"9789","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287811.2371111,"name":"drain","context":{"idset":"9572","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287811.3485119,"name":"drain","context":{"idset":"11250","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287811.4676199,"name":"drain","context":{"idset":"9973","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287811.6065342,"name":"drain","context":{"idset":"10046","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287811.7109444,"name":"drain","context":{"idset":"9977","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287811.8177638,"name":"drain","context":{"idset":"9990","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287811.9316273,"name":"drain","context":{"idset":"9577","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287812.0340729,"name":"drain","context":{"idset":"10011","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287812.142055,"name":"drain","context":{"idset":"9560","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287812.2453299,"name":"drain","context":{"idset":"9786","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287812.368921,"name":"drain","context":{"idset":"10019","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287812.4808753,"name":"drain","context":{"idset":"9575","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287812.5840321,"name":"drain","context":{"idset":"9545","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287812.6933362,"name":"drain","context":{"idset":"10038","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287812.7962182,"name":"drain","context":{"idset":"9569","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287812.9002981,"name":"drain","context":{"idset":"9985","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287813.0120955,"name":"drain","context":{"idset":"10003","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287813.1194389,"name":"drain","context":{"idset":"9994","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287813.2215681,"name":"drain","context":{"idset":"9532","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287813.3599203,"name":"drain","context":{"idset":"9526","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287813.5369239,"name":"drain","context":{"idset":"9729","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287813.7568402,"name":"drain","context":{"idset":"10016","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287813.9560459,"name":"drain","context":{"idset":"9727","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287814.1681168,"name":"drain","context":{"idset":"9530","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287814.3324039,"name":"drain","context":{"idset":"10041","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287814.517695,"name":"drain","context":{"idset":"9538","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287814.7095108,"name":"drain","context":{"idset":"9739","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287814.8718722,"name":"drain","context":{"idset":"9750","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287815.0443408,"name":"drain","context":{"idset":"9765","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287815.2164881,"name":"drain","context":{"idset":"9988","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287815.3777018,"name":"drain","context":{"idset":"10048","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287815.5485129,"name":"drain","context":{"idset":"10053","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287815.721251,"name":"drain","context":{"idset":"9574","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287815.8789582,"name":"drain","context":{"idset":"10056","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287816.0333734,"name":"drain","context":{"idset":"10034","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287816.1899619,"name":"drain","context":{"idset":"9800","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287816.3626664,"name":"drain","context":{"idset":"9772","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287816.5155115,"name":"drain","context":{"idset":"9737","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287816.6813104,"name":"drain","context":{"idset":"10015","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287816.8395658,"name":"drain","context":{"idset":"9778","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287817.0070713,"name":"drain","context":{"idset":"9783","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287817.1828666,"name":"drain","context":{"idset":"9563","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287817.3387225,"name":"drain","context":{"idset":"9552","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287817.460448,"name":"drain","context":{"idset":"9981","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287817.5712597,"name":"drain","context":{"idset":"10064","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287817.7013922,"name":"drain","context":{"idset":"9557","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287817.8114262,"name":"drain","context":{"idset":"10018","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287817.9206126,"name":"drain","context":{"idset":"9837","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287818.0237942,"name":"drain","context":{"idset":"10084","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287818.119024,"name":"drain","context":{"idset":"10078","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287818.2212155,"name":"drain","context":{"idset":"9976","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287818.3209398,"name":"drain","context":{"idset":"9578","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287818.4188228,"name":"drain","context":{"idset":"10005","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287818.5297158,"name":"drain","context":{"idset":"9726","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287818.6245589,"name":"drain","context":{"idset":"10023","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287818.7250373,"name":"drain","context":{"idset":"9773","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287818.8196087,"name":"drain","context":{"idset":"9549","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287818.9223406,"name":"drain","context":{"idset":"9836","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287819.0302546,"name":"drain","context":{"idset":"9529","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287819.1254265,"name":"drain","context":{"idset":"9806","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287819.2209978,"name":"drain","context":{"idset":"9751","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287819.3285766,"name":"drain","context":{"idset":"9758","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287819.4283342,"name":"drain","context":{"idset":"10080","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287819.528091,"name":"drain","context":{"idset":"9814","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287819.63539,"name":"drain","context":{"idset":"9777","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287819.7395973,"name":"drain","context":{"idset":"9565","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287819.836519,"name":"drain","context":{"idset":"9544","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287819.9381742,"name":"drain","context":{"idset":"9556","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287820.0343304,"name":"drain","context":{"idset":"9775","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287820.1420751,"name":"drain","context":{"idset":"9546","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287820.2496958,"name":"drain","context":{"idset":"9991","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287820.3632455,"name":"drain","context":{"idset":"9528","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287820.4761484,"name":"drain","context":{"idset":"9995","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287820.583858,"name":"drain","context":{"idset":"9547","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287820.6811433,"name":"drain","context":{"idset":"10098","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287820.7873874,"name":"drain","context":{"idset":"9766","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287820.8857799,"name":"drain","context":{"idset":"9566","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287820.986886,"name":"drain","context":{"idset":"9588","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287821.0835562,"name":"drain","context":{"idset":"9817","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287821.1872363,"name":"drain","context":{"idset":"10013","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287821.2858365,"name":"drain","context":{"idset":"9555","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287821.3930926,"name":"drain","context":{"idset":"9989","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287821.4884083,"name":"drain","context":{"idset":"10137","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287821.6095738,"name":"drain","context":{"idset":"9554","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287821.8141944,"name":"drain","context":{"idset":"10021","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287822.0268581,"name":"drain","context":{"idset":"10110","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287822.2308376,"name":"drain","context":{"idset":"9582","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287822.4500623,"name":"drain","context":{"idset":"10179","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287822.6484835,"name":"drain","context":{"idset":"10007","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287822.8503602,"name":"drain","context":{"idset":"10002","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287823.0528171,"name":"drain","context":{"idset":"9584","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287823.2742956,"name":"drain","context":{"idset":"9732","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287823.4780738,"name":"drain","context":{"idset":"9734","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287823.682373,"name":"drain","context":{"idset":"9999","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287823.8785505,"name":"drain","context":{"idset":"10054","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287824.0829177,"name":"drain","context":{"idset":"10108","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287824.2918701,"name":"drain","context":{"idset":"10009","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287824.4915423,"name":"drain","context":{"idset":"9763","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287824.6586778,"name":"drain","context":{"idset":"9797","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287824.8112047,"name":"drain","context":{"idset":"10010","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287824.9388773,"name":"drain","context":{"idset":"9580","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287825.0899062,"name":"drain","context":{"idset":"9978","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287825.2532854,"name":"drain","context":{"idset":"10022","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287825.4299905,"name":"drain","context":{"idset":"9568","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287825.6190672,"name":"drain","context":{"idset":"9550","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287825.8123298,"name":"drain","context":{"idset":"9564","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287826.0032976,"name":"drain","context":{"idset":"10047","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287826.2060618,"name":"drain","context":{"idset":"10057","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287826.4068928,"name":"drain","context":{"idset":"10120","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287826.6093452,"name":"drain","context":{"idset":"10069","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287826.8116221,"name":"drain","context":{"idset":"9570","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287827.0100279,"name":"drain","context":{"idset":"9755","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287827.2077065,"name":"drain","context":{"idset":"9718","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287827.4082601,"name":"drain","context":{"idset":"10006","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287827.6138835,"name":"drain","context":{"idset":"9586","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287827.8143046,"name":"drain","context":{"idset":"10125","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287827.9971514,"name":"drain","context":{"idset":"10124","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287828.2071934,"name":"drain","context":{"idset":"9983","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287828.4257009,"name":"drain","context":{"idset":"10096","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287828.6525428,"name":"drain","context":{"idset":"10203","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287828.8721776,"name":"drain","context":{"idset":"10025","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287829.077395,"name":"drain","context":{"idset":"10001","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287829.2737007,"name":"drain","context":{"idset":"9813","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287829.4631217,"name":"drain","context":{"idset":"9725","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287829.6684561,"name":"drain","context":{"idset":"10185","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287829.9018469,"name":"drain","context":{"idset":"10172","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287830.1106713,"name":"drain","context":{"idset":"9571","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287830.327563,"name":"drain","context":{"idset":"9744","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287830.4534423,"name":"drain","context":{"idset":"10039","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287830.5591614,"name":"drain","context":{"idset":"9741","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287830.657948,"name":"drain","context":{"idset":"9774","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287830.7540369,"name":"drain","context":{"idset":"10000","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287830.8497722,"name":"drain","context":{"idset":"10014","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287830.94648,"name":"drain","context":{"idset":"9788","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287831.0427549,"name":"drain","context":{"idset":"9730","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287831.1432228,"name":"drain","context":{"idset":"10024","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287831.2392299,"name":"drain","context":{"idset":"9811","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287831.3348215,"name":"drain","context":{"idset":"9720","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287831.4361076,"name":"drain","context":{"idset":"10221","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287831.5317466,"name":"drain","context":{"idset":"9583","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287831.6278613,"name":"drain","context":{"idset":"9736","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287831.7282317,"name":"drain","context":{"idset":"9760","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287831.824261,"name":"drain","context":{"idset":"9782","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287831.918555,"name":"drain","context":{"idset":"9821","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287832.012851,"name":"drain","context":{"idset":"10070","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287832.1068327,"name":"drain","context":{"idset":"9804","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287832.2169805,"name":"drain","context":{"idset":"10072","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287832.3159318,"name":"drain","context":{"idset":"9805","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287832.4172094,"name":"drain","context":{"idset":"10116","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287832.5136468,"name":"drain","context":{"idset":"9984","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287832.6077781,"name":"drain","context":{"idset":"9790","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287832.7076712,"name":"drain","context":{"idset":"9740","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287832.8018787,"name":"drain","context":{"idset":"9835","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287832.9069126,"name":"drain","context":{"idset":"9980","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.0046048,"name":"drain","context":{"idset":"10086","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.1056166,"name":"drain","context":{"idset":"10196","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.202178,"name":"drain","context":{"idset":"10126","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.2972486,"name":"drain","context":{"idset":"9986","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.391017,"name":"drain","context":{"idset":"10202","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.4980252,"name":"drain","context":{"idset":"9719","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.5955286,"name":"drain","context":{"idset":"9768","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.6911001,"name":"drain","context":{"idset":"9733","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.7861786,"name":"drain","context":{"idset":"10027","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.8808801,"name":"drain","context":{"idset":"10031","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287833.9954312,"name":"drain","context":{"idset":"9748","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287834.0951943,"name":"drain","context":{"idset":"10216","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287834.1969757,"name":"drain","context":{"idset":"9840","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287834.2938073,"name":"drain","context":{"idset":"10051","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287834.3931196,"name":"drain","context":{"idset":"9770","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287834.4986069,"name":"drain","context":{"idset":"10130","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287834.5965247,"name":"drain","context":{"idset":"9987","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287834.6946743,"name":"drain","context":{"idset":"9723","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287834.7919362,"name":"drain","context":{"idset":"10246","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287834.8890116,"name":"drain","context":{"idset":"9780","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287834.9865575,"name":"drain","context":{"idset":"10198","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287835.0923488,"name":"drain","context":{"idset":"9998","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287835.2063091,"name":"drain","context":{"idset":"9531","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287835.3045235,"name":"drain","context":{"idset":"10092","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287835.4003029,"name":"drain","context":{"idset":"10026","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287835.4959579,"name":"drain","context":{"idset":"10059","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287835.5917363,"name":"drain","context":{"idset":"10061","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287835.698199,"name":"drain","context":{"idset":"10207","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287835.7968252,"name":"drain","context":{"idset":"10017","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287835.8966186,"name":"drain","context":{"idset":"9996","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287835.9906693,"name":"drain","context":{"idset":"9831","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287836.084836,"name":"drain","context":{"idset":"9787","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287836.1949573,"name":"drain","context":{"idset":"10222","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287836.2945442,"name":"drain","context":{"idset":"10058","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287836.3923275,"name":"drain","context":{"idset":"10029","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287836.4932523,"name":"drain","context":{"idset":"10008","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287836.594311,"name":"drain","context":{"idset":"10004","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287836.6956906,"name":"drain","context":{"idset":"9785","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287836.7921791,"name":"drain","context":{"idset":"10012","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287836.8891094,"name":"drain","context":{"idset":"10045","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287836.984714,"name":"drain","context":{"idset":"9979","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287837.0797091,"name":"drain","context":{"idset":"9735","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287837.1734986,"name":"drain","context":{"idset":"10212","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287837.27179,"name":"drain","context":{"idset":"9567","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287837.3737621,"name":"drain","context":{"idset":"10055","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287837.4702005,"name":"drain","context":{"idset":"9776","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287837.566061,"name":"drain","context":{"idset":"10035","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287837.6614306,"name":"drain","context":{"idset":"10036","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287837.7568476,"name":"drain","context":{"idset":"10305","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287837.8587797,"name":"drain","context":{"idset":"9756","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287837.9689107,"name":"drain","context":{"idset":"9798","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287838.0729008,"name":"drain","context":{"idset":"10267","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287838.1732178,"name":"drain","context":{"idset":"9816","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287838.2707522,"name":"drain","context":{"idset":"10258","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287838.3651271,"name":"drain","context":{"idset":"10102","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287838.4660316,"name":"drain","context":{"idset":"10060","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287838.5611978,"name":"drain","context":{"idset":"10065","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287838.6553946,"name":"drain","context":{"idset":"9749","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287838.7499104,"name":"drain","context":{"idset":"10049","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287838.8502908,"name":"drain","context":{"idset":"10079","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287838.9547887,"name":"drain","context":{"idset":"10265","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287839.0500038,"name":"drain","context":{"idset":"9573","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287839.1447918,"name":"drain","context":{"idset":"10033","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287839.2390592,"name":"drain","context":{"idset":"10171","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287839.3351467,"name":"drain","context":{"idset":"10052","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287839.4379809,"name":"drain","context":{"idset":"10075","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287839.5327783,"name":"drain","context":{"idset":"10320","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287839.6273267,"name":"drain","context":{"idset":"9747","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287839.7218208,"name":"drain","context":{"idset":"10193","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287839.8164132,"name":"drain","context":{"idset":"10192","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287839.9104054,"name":"drain","context":{"idset":"9826","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287840.0044589,"name":"drain","context":{"idset":"10229","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287840.1076329,"name":"drain","context":{"idset":"9738","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287840.2125378,"name":"drain","context":{"idset":"10050","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287840.3084235,"name":"drain","context":{"idset":"9585","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287840.4123597,"name":"drain","context":{"idset":"10250","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287840.5344877,"name":"drain","context":{"idset":"10173","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287840.6589315,"name":"drain","context":{"idset":"10307","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287840.7887723,"name":"drain","context":{"idset":"9543","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287840.9225876,"name":"drain","context":{"idset":"10160","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287841.0700679,"name":"drain","context":{"idset":"10074","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287841.2395864,"name":"drain","context":{"idset":"10044","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287841.4125948,"name":"drain","context":{"idset":"9745","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287841.5756826,"name":"drain","context":{"idset":"9771","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287841.7354293,"name":"drain","context":{"idset":"9746","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287841.9011319,"name":"drain","context":{"idset":"9802","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287842.0711975,"name":"drain","context":{"idset":"10256","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287842.280659,"name":"drain","context":{"idset":"9754","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287842.4550912,"name":"drain","context":{"idset":"9838","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287842.5966847,"name":"drain","context":{"idset":"9769","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287842.7164643,"name":"drain","context":{"idset":"9803","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287842.8261654,"name":"drain","context":{"idset":"10089","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287842.9487402,"name":"drain","context":{"idset":"10215","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287843.0505164,"name":"drain","context":{"idset":"10175","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287843.1489391,"name":"drain","context":{"idset":"9815","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287843.2468812,"name":"drain","context":{"idset":"9833","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287843.354825,"name":"drain","context":{"idset":"9779","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287843.453685,"name":"drain","context":{"idset":"9795","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287843.5507817,"name":"drain","context":{"idset":"9842","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287843.6472182,"name":"drain","context":{"idset":"10339","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287843.744729,"name":"drain","context":{"idset":"10088","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287843.8434315,"name":"drain","context":{"idset":"10104","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287843.9526169,"name":"drain","context":{"idset":"9731","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287844.0497229,"name":"drain","context":{"idset":"10225","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287844.1463172,"name":"drain","context":{"idset":"10077","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287844.2433622,"name":"drain","context":{"idset":"9975","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287844.3504701,"name":"drain","context":{"idset":"9791","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287844.469991,"name":"drain","context":{"idset":"9796","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287844.5680742,"name":"drain","context":{"idset":"10073","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287844.6719718,"name":"drain","context":{"idset":"9799","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287844.7705512,"name":"drain","context":{"idset":"10295","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287844.8707657,"name":"drain","context":{"idset":"10028","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287844.9693148,"name":"drain","context":{"idset":"9820","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287845.0752091,"name":"drain","context":{"idset":"10264","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287845.2014413,"name":"drain","context":{"idset":"10271","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287845.3253345,"name":"drain","context":{"idset":"10299","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287845.4545317,"name":"drain","context":{"idset":"10257","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287845.6016364,"name":"drain","context":{"idset":"10285","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287845.768472,"name":"drain","context":{"idset":"9822","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287845.9368508,"name":"drain","context":{"idset":"9823","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287846.1069283,"name":"drain","context":{"idset":"9794","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287846.268883,"name":"drain","context":{"idset":"10063","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287846.4451559,"name":"drain","context":{"idset":"10189","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287846.6350117,"name":"drain","context":{"idset":"10085","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287846.8050444,"name":"drain","context":{"idset":"9757","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287846.9735205,"name":"drain","context":{"idset":"9807","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287847.1192133,"name":"drain","context":{"idset":"10032","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287847.2484586,"name":"drain","context":{"idset":"9781","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287847.3657675,"name":"drain","context":{"idset":"10315","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287847.477783,"name":"drain","context":{"idset":"10118","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287847.5992279,"name":"drain","context":{"idset":"10067","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287847.6951745,"name":"drain","context":{"idset":"9541","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287847.7914093,"name":"drain","context":{"idset":"9829","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287847.8868222,"name":"drain","context":{"idset":"10330","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287847.9805167,"name":"drain","context":{"idset":"10091","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287848.0818441,"name":"drain","context":{"idset":"9819","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287848.1771755,"name":"drain","context":{"idset":"10273","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287848.2845151,"name":"drain","context":{"idset":"10190","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287848.3961947,"name":"drain","context":{"idset":"9553","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287848.5782604,"name":"drain","context":{"idset":"10375","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287848.7482531,"name":"drain","context":{"idset":"10066","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287848.9212224,"name":"drain","context":{"idset":"10093","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287849.0825095,"name":"drain","context":{"idset":"9743","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287849.2432263,"name":"drain","context":{"idset":"9581","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287849.4141037,"name":"drain","context":{"idset":"10068","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287849.5824976,"name":"drain","context":{"idset":"10294","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287849.745409,"name":"drain","context":{"idset":"9828","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287849.9060876,"name":"drain","context":{"idset":"10338","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287850.0662158,"name":"drain","context":{"idset":"10275","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287850.2581515,"name":"drain","context":{"idset":"9844","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287850.4443252,"name":"drain","context":{"idset":"10094","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287850.6081607,"name":"drain","context":{"idset":"9839","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287850.7923231,"name":"drain","context":{"idset":"10134","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287850.9900913,"name":"drain","context":{"idset":"10297","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287851.1627107,"name":"drain","context":{"idset":"10103","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287851.3264842,"name":"drain","context":{"idset":"10361","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287851.5665307,"name":"drain","context":{"idset":"10123","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287851.8811071,"name":"drain","context":{"idset":"10087","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287852.1400652,"name":"drain","context":{"idset":"10287","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287852.3605914,"name":"drain","context":{"idset":"10272","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287852.5702305,"name":"drain","context":{"idset":"10115","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287852.8034294,"name":"drain","context":{"idset":"9762","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287853.0568988,"name":"drain","context":{"idset":"10095","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287853.3220899,"name":"drain","context":{"idset":"10106","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287853.5783155,"name":"drain","context":{"idset":"10348","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287853.8238885,"name":"drain","context":{"idset":"10343","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287854.0365665,"name":"drain","context":{"idset":"10340","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287854.2472272,"name":"drain","context":{"idset":"9992","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287854.4506598,"name":"drain","context":{"idset":"10141","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287854.5937781,"name":"drain","context":{"idset":"9832","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287854.730958,"name":"drain","context":{"idset":"10183","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287854.889991,"name":"drain","context":{"idset":"10239","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287855.0733688,"name":"drain","context":{"idset":"9548","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287855.2705731,"name":"drain","context":{"idset":"10237","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287855.4726269,"name":"drain","context":{"idset":"10289","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287855.6723206,"name":"drain","context":{"idset":"9827","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287855.8759906,"name":"drain","context":{"idset":"10168","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287856.0641754,"name":"drain","context":{"idset":"9841","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287856.2668335,"name":"drain","context":{"idset":"10152","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287856.4876151,"name":"drain","context":{"idset":"10113","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287856.6937313,"name":"drain","context":{"idset":"10142","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287856.9052527,"name":"drain","context":{"idset":"10182","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287857.1203759,"name":"drain","context":{"idset":"9761","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287857.3383641,"name":"drain","context":{"idset":"9993","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287857.5561838,"name":"drain","context":{"idset":"10270","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287857.7709439,"name":"drain","context":{"idset":"9812","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287857.9886556,"name":"drain","context":{"idset":"10350","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287858.2066884,"name":"drain","context":{"idset":"10201","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287858.4244928,"name":"drain","context":{"idset":"10158","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287858.6453009,"name":"drain","context":{"idset":"9753","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287858.8693478,"name":"drain","context":{"idset":"10164","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287859.075006,"name":"drain","context":{"idset":"9576","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287859.3004746,"name":"drain","context":{"idset":"10122","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287859.4981067,"name":"drain","context":{"idset":"10133","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287859.6942189,"name":"drain","context":{"idset":"10157","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287859.8986666,"name":"drain","context":{"idset":"10322","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287860.1047971,"name":"drain","context":{"idset":"10083","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287860.2663295,"name":"drain","context":{"idset":"10205","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287860.3883955,"name":"drain","context":{"idset":"9728","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287860.5014277,"name":"drain","context":{"idset":"9824","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287860.6095171,"name":"drain","context":{"idset":"10099","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287860.710417,"name":"drain","context":{"idset":"10100","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287860.8189859,"name":"drain","context":{"idset":"10378","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287860.9218955,"name":"drain","context":{"idset":"10082","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287861.0254869,"name":"drain","context":{"idset":"10181","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287861.1269648,"name":"drain","context":{"idset":"10380","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287861.2254896,"name":"drain","context":{"idset":"10081","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287861.3406126,"name":"drain","context":{"idset":"10170","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287861.4441991,"name":"drain","context":{"idset":"9721","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287861.5538516,"name":"drain","context":{"idset":"10184","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287861.6535103,"name":"drain","context":{"idset":"10176","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287861.7560585,"name":"drain","context":{"idset":"10335","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287861.8643615,"name":"drain","context":{"idset":"10165","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287861.9656613,"name":"drain","context":{"idset":"10191","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287862.0692124,"name":"drain","context":{"idset":"10146","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287862.194808,"name":"drain","context":{"idset":"10383","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287862.3309019,"name":"drain","context":{"idset":"10114","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287862.4790118,"name":"drain","context":{"idset":"10037","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287862.6517074,"name":"drain","context":{"idset":"10040","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287862.822084,"name":"drain","context":{"idset":"10187","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287862.9915888,"name":"drain","context":{"idset":"10387","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287863.1478415,"name":"drain","context":{"idset":"10043","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287863.3212504,"name":"drain","context":{"idset":"10139","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287863.4885998,"name":"drain","context":{"idset":"10188","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287863.6513293,"name":"drain","context":{"idset":"10247","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287863.8052056,"name":"drain","context":{"idset":"10214","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287863.9381063,"name":"drain","context":{"idset":"10174","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287864.057621,"name":"drain","context":{"idset":"9834","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287864.1849551,"name":"drain","context":{"idset":"10371","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287864.2826829,"name":"drain","context":{"idset":"10278","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287864.380322,"name":"drain","context":{"idset":"10180","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287864.4848549,"name":"drain","context":{"idset":"10129","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287864.5992925,"name":"drain","context":{"idset":"10200","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287864.711834,"name":"drain","context":{"idset":"10169","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287864.8147502,"name":"drain","context":{"idset":"9808","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287864.9136553,"name":"drain","context":{"idset":"10213","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287865.0122004,"name":"drain","context":{"idset":"10370","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287865.1120236,"name":"drain","context":{"idset":"10368","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287865.2311277,"name":"drain","context":{"idset":"10090","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287865.3309851,"name":"drain","context":{"idset":"9767","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287865.4329479,"name":"drain","context":{"idset":"10131","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287865.5452909,"name":"drain","context":{"idset":"10167","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287865.6430335,"name":"drain","context":{"idset":"10209","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287865.7411444,"name":"drain","context":{"idset":"10132","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287865.8428383,"name":"drain","context":{"idset":"10097","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287865.9453843,"name":"drain","context":{"idset":"10109","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287866.0564263,"name":"drain","context":{"idset":"10248","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287866.1697347,"name":"drain","context":{"idset":"10159","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287866.297117,"name":"drain","context":{"idset":"10149","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287866.402544,"name":"drain","context":{"idset":"10280","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287866.5047703,"name":"drain","context":{"idset":"10186","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287866.635407,"name":"drain","context":{"idset":"10155","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287866.8023267,"name":"drain","context":{"idset":"10112","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287866.9674571,"name":"drain","context":{"idset":"10042","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287867.1326947,"name":"drain","context":{"idset":"10163","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287867.3146796,"name":"drain","context":{"idset":"9759","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287867.4806733,"name":"drain","context":{"idset":"10178","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287867.6410289,"name":"drain","context":{"idset":"10226","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287867.8156574,"name":"drain","context":{"idset":"9801","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287867.9796848,"name":"drain","context":{"idset":"10224","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287868.1444337,"name":"drain","context":{"idset":"10284","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287868.3166084,"name":"drain","context":{"idset":"10071","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287868.449893,"name":"drain","context":{"idset":"10204","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287868.5671844,"name":"drain","context":{"idset":"9843","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287868.6753099,"name":"drain","context":{"idset":"10030","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287868.7758877,"name":"drain","context":{"idset":"10254","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287868.8782883,"name":"drain","context":{"idset":"10263","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287868.9740422,"name":"drain","context":{"idset":"10177","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287869.0734847,"name":"drain","context":{"idset":"10227","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287869.171711,"name":"drain","context":{"idset":"9792","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287869.2687263,"name":"drain","context":{"idset":"10228","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287869.3841457,"name":"drain","context":{"idset":"10252","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287869.5019403,"name":"drain","context":{"idset":"10262","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287869.6395009,"name":"drain","context":{"idset":"9810","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287869.7621739,"name":"drain","context":{"idset":"10230","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287869.9010663,"name":"drain","context":{"idset":"10151","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287870.0561862,"name":"drain","context":{"idset":"10314","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287870.1918869,"name":"drain","context":{"idset":"10242","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287870.314095,"name":"drain","context":{"idset":"10293","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287870.4373057,"name":"drain","context":{"idset":"9818","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287870.5525181,"name":"drain","context":{"idset":"10253","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287870.6694119,"name":"drain","context":{"idset":"10020","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287870.7879186,"name":"drain","context":{"idset":"10390","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287870.8894286,"name":"drain","context":{"idset":"10195","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287870.9898245,"name":"drain","context":{"idset":"10231","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287871.089529,"name":"drain","context":{"idset":"10236","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287871.1891785,"name":"drain","context":{"idset":"10161","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287871.288944,"name":"drain","context":{"idset":"10197","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287871.4150753,"name":"drain","context":{"idset":"10255","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287871.5402064,"name":"drain","context":{"idset":"10328","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287871.69262,"name":"drain","context":{"idset":"9830","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287871.8719616,"name":"drain","context":{"idset":"9793","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287872.03336,"name":"drain","context":{"idset":"10310","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287872.1973646,"name":"drain","context":{"idset":"10311","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287872.3690453,"name":"drain","context":{"idset":"10162","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287872.5438337,"name":"drain","context":{"idset":"10210","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287872.7063425,"name":"drain","context":{"idset":"10101","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287872.8703642,"name":"drain","context":{"idset":"10194","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287873.0596261,"name":"drain","context":{"idset":"10233","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287873.2335541,"name":"drain","context":{"idset":"10235","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287873.3828728,"name":"drain","context":{"idset":"10240","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287873.5381005,"name":"drain","context":{"idset":"10286","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287873.6599123,"name":"drain","context":{"idset":"10298","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287873.7906446,"name":"drain","context":{"idset":"10251","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287873.9175498,"name":"drain","context":{"idset":"10296","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287874.0372074,"name":"drain","context":{"idset":"10062","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287874.1438422,"name":"drain","context":{"idset":"10292","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287874.2454638,"name":"drain","context":{"idset":"10260","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287874.3516123,"name":"drain","context":{"idset":"10223","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287874.4569128,"name":"drain","context":{"idset":"10290","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287874.5655382,"name":"drain","context":{"idset":"10217","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287874.6726141,"name":"drain","context":{"idset":"10268","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287874.7934189,"name":"drain","context":{"idset":"10117","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287874.8954012,"name":"drain","context":{"idset":"10313","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287874.9972794,"name":"drain","context":{"idset":"10208","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287875.0977297,"name":"drain","context":{"idset":"10282","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287875.2307053,"name":"drain","context":{"idset":"10211","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287875.3435326,"name":"drain","context":{"idset":"9764","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287875.4477942,"name":"drain","context":{"idset":"10391","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287875.5501611,"name":"drain","context":{"idset":"10306","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287875.6516192,"name":"drain","context":{"idset":"10288","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287875.8163095,"name":"drain","context":{"idset":"10276","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287876.0047548,"name":"drain","context":{"idset":"10232","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287876.1949947,"name":"drain","context":{"idset":"10283","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287876.3935342,"name":"drain","context":{"idset":"10274","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287876.5647233,"name":"drain","context":{"idset":"10316","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287876.728523,"name":"drain","context":{"idset":"10317","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287876.8891168,"name":"drain","context":{"idset":"10323","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287877.0571814,"name":"drain","context":{"idset":"10243","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287877.2216933,"name":"drain","context":{"idset":"10319","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287877.3826511,"name":"drain","context":{"idset":"10300","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287877.5621653,"name":"drain","context":{"idset":"10076","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287877.7203488,"name":"drain","context":{"idset":"10329","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287877.8585198,"name":"drain","context":{"idset":"9784","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287877.9931617,"name":"drain","context":{"idset":"10241","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287878.1009977,"name":"drain","context":{"idset":"10269","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287878.2007115,"name":"drain","context":{"idset":"10345","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287878.3037317,"name":"drain","context":{"idset":"10259","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287878.4072456,"name":"drain","context":{"idset":"10156","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287878.5138829,"name":"drain","context":{"idset":"9809","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287878.6112399,"name":"drain","context":{"idset":"9752","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287878.7084305,"name":"drain","context":{"idset":"10308","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287878.8052237,"name":"drain","context":{"idset":"10312","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287878.9095378,"name":"drain","context":{"idset":"10128","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287879.0082202,"name":"drain","context":{"idset":"10355","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287879.1201968,"name":"drain","context":{"idset":"10358","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287879.2209573,"name":"drain","context":{"idset":"10105","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287879.3193972,"name":"drain","context":{"idset":"10326","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287879.416435,"name":"drain","context":{"idset":"10362","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287879.5160561,"name":"drain","context":{"idset":"10144","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287879.6156654,"name":"drain","context":{"idset":"9825","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287879.7216034,"name":"drain","context":{"idset":"9579","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287879.8279548,"name":"drain","context":{"idset":"10353","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287879.9260795,"name":"drain","context":{"idset":"10166","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287880.0246215,"name":"drain","context":{"idset":"10373","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287880.1261799,"name":"drain","context":{"idset":"10332","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287880.253448,"name":"drain","context":{"idset":"10324","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287880.3658953,"name":"drain","context":{"idset":"10266","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287880.4838245,"name":"drain","context":{"idset":"10325","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287880.5895245,"name":"drain","context":{"idset":"10357","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287880.6939585,"name":"drain","context":{"idset":"10321","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287880.7998316,"name":"drain","context":{"idset":"10318","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287880.9037805,"name":"drain","context":{"idset":"10333","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287881.0135264,"name":"drain","context":{"idset":"10349","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287881.1152308,"name":"drain","context":{"idset":"10150","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287881.2166767,"name":"drain","context":{"idset":"10347","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287881.407867,"name":"drain","context":{"idset":"10376","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287881.6252141,"name":"drain","context":{"idset":"10277","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287881.8297548,"name":"drain","context":{"idset":"10344","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287882.041496,"name":"drain","context":{"idset":"10352","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287882.241385,"name":"drain","context":{"idset":"10107","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287882.4564967,"name":"drain","context":{"idset":"10379","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287882.7260296,"name":"drain","context":{"idset":"10336","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287882.9709311,"name":"drain","context":{"idset":"10342","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287883.2316594,"name":"drain","context":{"idset":"10359","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287883.4601531,"name":"drain","context":{"idset":"10366","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287883.6795585,"name":"drain","context":{"idset":"10346","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287883.8894851,"name":"drain","context":{"idset":"10367","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287884.1153154,"name":"drain","context":{"idset":"10395","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287884.3084295,"name":"drain","context":{"idset":"10238","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287884.4900153,"name":"drain","context":{"idset":"10360","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287884.6541176,"name":"drain","context":{"idset":"10111","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287884.8200591,"name":"drain","context":{"idset":"10363","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287885.0099878,"name":"drain","context":{"idset":"10384","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287885.199806,"name":"drain","context":{"idset":"10356","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287885.4177594,"name":"drain","context":{"idset":"10121","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287885.6003861,"name":"drain","context":{"idset":"10382","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287885.790004,"name":"drain","context":{"idset":"10385","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287885.9901855,"name":"drain","context":{"idset":"10365","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287886.2158225,"name":"drain","context":{"idset":"10388","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287886.4254317,"name":"drain","context":{"idset":"10354","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287886.6182222,"name":"drain","context":{"idset":"10377","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287886.8191276,"name":"drain","context":{"idset":"10351","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287887.0148897,"name":"drain","context":{"idset":"9587","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287887.2356286,"name":"drain","context":{"idset":"10127","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287887.4434597,"name":"drain","context":{"idset":"10291","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287887.6587441,"name":"drain","context":{"idset":"10369","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287887.8765373,"name":"drain","context":{"idset":"10119","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287888.0973754,"name":"drain","context":{"idset":"10381","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287888.3174734,"name":"drain","context":{"idset":"10372","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287888.558722,"name":"drain","context":{"idset":"10393","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287888.7683423,"name":"drain","context":{"idset":"10140","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287888.9754059,"name":"drain","context":{"idset":"10199","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287889.1806073,"name":"drain","context":{"idset":"10364","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287889.4211397,"name":"drain","context":{"idset":"10234","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287889.6334026,"name":"drain","context":{"idset":"10220","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287889.8446796,"name":"drain","context":{"idset":"10279","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287889.9778028,"name":"drain","context":{"idset":"10245","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287890.0889697,"name":"drain","context":{"idset":"10218","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287890.1948986,"name":"drain","context":{"idset":"10400","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287890.3057411,"name":"drain","context":{"idset":"10309","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287890.4144833,"name":"drain","context":{"idset":"10386","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287890.5187285,"name":"drain","context":{"idset":"10281","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287890.6269979,"name":"drain","context":{"idset":"10249","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287890.7277701,"name":"drain","context":{"idset":"10408","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287890.8350961,"name":"drain","context":{"idset":"10327","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287890.9348321,"name":"drain","context":{"idset":"10341","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287891.0357909,"name":"drain","context":{"idset":"10331","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287891.1362381,"name":"drain","context":{"idset":"10244","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287891.2534537,"name":"drain","context":{"idset":"10334","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287891.3590405,"name":"drain","context":{"idset":"10219","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287891.4793081,"name":"drain","context":{"idset":"10399","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287891.5902987,"name":"drain","context":{"idset":"10401","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287891.717047,"name":"drain","context":{"idset":"10404","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287891.8656492,"name":"drain","context":{"idset":"10337","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287892.0367112,"name":"drain","context":{"idset":"10397","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287892.2094858,"name":"drain","context":{"idset":"10437","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287892.368983,"name":"drain","context":{"idset":"10392","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287892.55898,"name":"drain","context":{"idset":"10409","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287892.7211769,"name":"drain","context":{"idset":"10414","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287892.9139316,"name":"drain","context":{"idset":"10394","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287893.0846493,"name":"drain","context":{"idset":"10440","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287893.2466805,"name":"drain","context":{"idset":"10424","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287893.4054072,"name":"drain","context":{"idset":"10415","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287893.5756166,"name":"drain","context":{"idset":"10406","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287893.7184069,"name":"drain","context":{"idset":"10405","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287893.8382378,"name":"drain","context":{"idset":"10426","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287893.9980257,"name":"drain","context":{"idset":"10403","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287894.1346791,"name":"drain","context":{"idset":"10428","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287894.3035517,"name":"drain","context":{"idset":"10456","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287894.4563856,"name":"drain","context":{"idset":"10396","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287894.608201,"name":"drain","context":{"idset":"10421","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287894.8097749,"name":"drain","context":{"idset":"10430","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287894.9909739,"name":"drain","context":{"idset":"10522","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287895.1750813,"name":"drain","context":{"idset":"10490","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287895.3567753,"name":"drain","context":{"idset":"10446","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287895.5327303,"name":"drain","context":{"idset":"10457","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287895.7323656,"name":"drain","context":{"idset":"10398","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287895.8744686,"name":"drain","context":{"idset":"10420","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287896.0102496,"name":"drain","context":{"idset":"10417","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287896.1170835,"name":"drain","context":{"idset":"10519","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287896.2208354,"name":"drain","context":{"idset":"10533","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287896.3290854,"name":"drain","context":{"idset":"10450","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287896.440078,"name":"drain","context":{"idset":"10552","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287896.5533075,"name":"drain","context":{"idset":"10463","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287896.6504095,"name":"drain","context":{"idset":"10458","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287896.7497373,"name":"drain","context":{"idset":"10423","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287896.8557692,"name":"drain","context":{"idset":"10498","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287896.953939,"name":"drain","context":{"idset":"10419","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287897.0637033,"name":"drain","context":{"idset":"10452","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287897.1707654,"name":"drain","context":{"idset":"10465","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287897.2747178,"name":"drain","context":{"idset":"10438","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287897.3763354,"name":"drain","context":{"idset":"10467","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287897.4753063,"name":"drain","context":{"idset":"10486","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287897.5743046,"name":"drain","context":{"idset":"10422","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287897.6865799,"name":"drain","context":{"idset":"10499","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287897.7940197,"name":"drain","context":{"idset":"10427","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287897.8941541,"name":"drain","context":{"idset":"10520","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287897.9920831,"name":"drain","context":{"idset":"10550","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287898.0895824,"name":"drain","context":{"idset":"10537","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287898.1968398,"name":"drain","context":{"idset":"10441","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287898.325531,"name":"drain","context":{"idset":"10435","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287898.4654517,"name":"drain","context":{"idset":"10432","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287898.6046991,"name":"drain","context":{"idset":"10466","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287898.7532468,"name":"drain","context":{"idset":"10402","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287898.9279554,"name":"drain","context":{"idset":"10447","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287899.1051922,"name":"drain","context":{"idset":"10425","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287899.2780566,"name":"drain","context":{"idset":"10523","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287899.4516869,"name":"drain","context":{"idset":"10410","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287899.6201732,"name":"drain","context":{"idset":"10535","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287899.7887573,"name":"drain","context":{"idset":"10407","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287899.9768755,"name":"drain","context":{"idset":"10551","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287900.1519654,"name":"drain","context":{"idset":"10524","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287900.3198109,"name":"drain","context":{"idset":"10434","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287900.4924817,"name":"drain","context":{"idset":"10503","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287900.6571112,"name":"drain","context":{"idset":"10512","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287900.7964172,"name":"drain","context":{"idset":"10451","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287900.9263148,"name":"drain","context":{"idset":"10510","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287901.0384185,"name":"drain","context":{"idset":"10468","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287901.1407504,"name":"drain","context":{"idset":"10412","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287901.2424743,"name":"drain","context":{"idset":"10532","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287901.3385201,"name":"drain","context":{"idset":"10464","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287901.4415321,"name":"drain","context":{"idset":"10418","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287901.5464561,"name":"drain","context":{"idset":"10567","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287901.6581097,"name":"drain","context":{"idset":"10431","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287901.7698314,"name":"drain","context":{"idset":"10549","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287901.8934472,"name":"drain","context":{"idset":"10436","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287902.0701344,"name":"drain","context":{"idset":"10444","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287902.229069,"name":"drain","context":{"idset":"10459","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287902.4154739,"name":"drain","context":{"idset":"10527","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287902.5955815,"name":"drain","context":{"idset":"10525","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287902.7712617,"name":"drain","context":{"idset":"10500","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287902.941031,"name":"drain","context":{"idset":"10411","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287903.1195061,"name":"drain","context":{"idset":"10561","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287903.3033125,"name":"drain","context":{"idset":"10439","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287903.4690456,"name":"drain","context":{"idset":"10557","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712287903.6432595,"name":"drain","context":{"idset":"10416","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287903.7977684,"name":"drain","context":{"idset":"10548","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287903.9371231,"name":"drain","context":{"idset":"10454","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287904.0564995,"name":"drain","context":{"idset":"10516","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287904.1653442,"name":"drain","context":{"idset":"10574","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287904.2762244,"name":"drain","context":{"idset":"10530","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287904.3902471,"name":"drain","context":{"idset":"10485","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287904.5115814,"name":"drain","context":{"idset":"10502","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287904.6211758,"name":"drain","context":{"idset":"10429","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287904.7251551,"name":"drain","context":{"idset":"10487","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287904.8286381,"name":"drain","context":{"idset":"10413","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287904.9376202,"name":"drain","context":{"idset":"10546","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287905.0537541,"name":"drain","context":{"idset":"10453","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287905.1833491,"name":"drain","context":{"idset":"10448","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287905.2894766,"name":"drain","context":{"idset":"10571","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287905.4019809,"name":"drain","context":{"idset":"10555","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287905.5074069,"name":"drain","context":{"idset":"10501","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287905.6179709,"name":"drain","context":{"idset":"10531","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287905.719897,"name":"drain","context":{"idset":"10566","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287905.8336437,"name":"drain","context":{"idset":"10569","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287905.9466977,"name":"drain","context":{"idset":"10509","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287906.0763631,"name":"drain","context":{"idset":"10455","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287906.1792159,"name":"drain","context":{"idset":"10521","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287906.2852972,"name":"drain","context":{"idset":"10541","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287906.3911724,"name":"drain","context":{"idset":"10493","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287906.5077195,"name":"drain","context":{"idset":"10489","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287906.6108334,"name":"drain","context":{"idset":"10563","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287906.7109292,"name":"drain","context":{"idset":"10568","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287906.853302,"name":"drain","context":{"idset":"10443","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287906.9555957,"name":"drain","context":{"idset":"10547","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287907.06706,"name":"drain","context":{"idset":"10572","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287907.1657705,"name":"drain","context":{"idset":"10543","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287907.2631881,"name":"drain","context":{"idset":"10442","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287907.3878393,"name":"drain","context":{"idset":"10488","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287907.4943125,"name":"drain","context":{"idset":"10507","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287907.5915742,"name":"drain","context":{"idset":"10538","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287907.6938164,"name":"drain","context":{"idset":"10539","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287907.7909338,"name":"drain","context":{"idset":"10511","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287907.9136381,"name":"drain","context":{"idset":"10515","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287908.0364027,"name":"drain","context":{"idset":"10462","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287908.1327982,"name":"drain","context":{"idset":"10534","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287908.2282786,"name":"drain","context":{"idset":"10461","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287908.3236749,"name":"drain","context":{"idset":"10504","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287908.4312153,"name":"drain","context":{"idset":"10495","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287908.5421817,"name":"drain","context":{"idset":"10536","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287908.6459672,"name":"drain","context":{"idset":"10460","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287908.7440898,"name":"drain","context":{"idset":"10553","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287908.8471818,"name":"drain","context":{"idset":"10492","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287908.954937,"name":"drain","context":{"idset":"10494","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287909.0520701,"name":"drain","context":{"idset":"10497","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287909.1500227,"name":"drain","context":{"idset":"10505","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287909.2462754,"name":"drain","context":{"idset":"10496","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287909.3475964,"name":"drain","context":{"idset":"10508","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287909.4504015,"name":"drain","context":{"idset":"10529","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287909.5657287,"name":"drain","context":{"idset":"10445","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287909.6719556,"name":"drain","context":{"idset":"10491","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287909.7796321,"name":"drain","context":{"idset":"10559","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287909.8819044,"name":"drain","context":{"idset":"10514","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287909.977402,"name":"drain","context":{"idset":"10560","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287910.0896661,"name":"drain","context":{"idset":"10433","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287910.2287204,"name":"drain","context":{"idset":"10449","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287910.3715429,"name":"drain","context":{"idset":"10506","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287910.511426,"name":"drain","context":{"idset":"10558","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287910.7060983,"name":"drain","context":{"idset":"10526","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287910.8709428,"name":"drain","context":{"idset":"10540","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287911.0459037,"name":"drain","context":{"idset":"10573","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287911.2770622,"name":"drain","context":{"idset":"10570","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287911.4443905,"name":"drain","context":{"idset":"10556","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287911.6398208,"name":"drain","context":{"idset":"10562","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287911.8116648,"name":"drain","context":{"idset":"10545","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287912.0406618,"name":"drain","context":{"idset":"10542","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287912.3336027,"name":"drain","context":{"idset":"10564","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287912.6370256,"name":"drain","context":{"idset":"10554","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287912.9220188,"name":"drain","context":{"idset":"10528","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287913.1358523,"name":"drain","context":{"idset":"10513","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287913.4002271,"name":"drain","context":{"idset":"10565","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712287913.6972928,"name":"drain","context":{"idset":"10544","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712291529.2562842,"name":"offline","context":{"idset":"10356"}}
+{"timestamp":1712293577.6978974,"name":"undrain","context":{"idset":"10373"}}
+{"timestamp":1712295835.6838045,"name":"undrain","context":{"idset":"10375"}}
+{"timestamp":1712296019.4776642,"name":"undrain","context":{"idset":"9974-9981,9983-9994,9996-10004,10006,10008-10009,10011-10018,10020-10022,10024-10045,10048-10050,10052-10056,10058-10078,10080-10083,10085-10091,10093-10100"}}
+{"timestamp":1712297115.1155467,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1712297371.4359789,"name":"undrain","context":{"idset":"9973"}}
+{"timestamp":1712298145.934443,"name":"drain","context":{"idset":"9973","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712298781.4970379,"name":"undrain","context":{"idset":"10101-10134,10137,10139-10142,10144,10146,10149-10152,10155-10156,10158-10159,10161-10183,10185-10192,10194-10202,10204-10205,10208-10215,10217-10220,10222-10228"}}
+{"timestamp":1712299475.6428883,"name":"undrain","context":{"idset":"10229-10232,10234-10249,10251-10255,10257-10260,10262-10266,10268-10272,10274-10277,10279,10281-10300,10306-10355,10358-10372,10376-10388,10390-10399,10401-10420,10422-10431,10433-10435,10437-10443,10445-10449,10452-10455,10457-10465,10467-10468"}}
+{"timestamp":1712325698.8640437,"name":"offline","context":{"idset":"11115"}}
+{"timestamp":1712326095.4836364,"name":"undrain","context":{"idset":"9973,9982"}}
+{"timestamp":1712326495.0890651,"name":"drain","context":{"idset":"11000","reason":"broker was unresponsive"}}
+{"timestamp":1712326517.5432539,"name":"undrain","context":{"idset":"9995,10005,10007,10010,10019,10023,10046-10047,10051,10057,10079,10084,10092"}}
+{"timestamp":1712326559.1418533,"name":"offline","context":{"idset":"11000"}}
+{"timestamp":1712326683.3743525,"name":"drain","context":{"idset":"11005","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712326689.1067295,"name":"drain","context":{"idset":"11004","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712326689.2248931,"name":"drain","context":{"idset":"11002","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712326689.5956488,"name":"drain","context":{"idset":"11001","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712326691.3185263,"name":"drain","context":{"idset":"11006","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712327590.3344913,"name":"drain","context":{"idset":"11486","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3346167,"name":"drain","context":{"idset":"11487","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3346658,"name":"drain","context":{"idset":"11488","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3347228,"name":"drain","context":{"idset":"11489","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3347826,"name":"drain","context":{"idset":"11490","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3348348,"name":"drain","context":{"idset":"11491","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3348861,"name":"drain","context":{"idset":"11492","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3349328,"name":"drain","context":{"idset":"11493","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3349853,"name":"drain","context":{"idset":"11494","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3350406,"name":"drain","context":{"idset":"11495","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3350909,"name":"drain","context":{"idset":"11496","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3351414,"name":"drain","context":{"idset":"11497","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3351901,"name":"drain","context":{"idset":"11498","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3352365,"name":"drain","context":{"idset":"11499","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3353057,"name":"drain","context":{"idset":"11500","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3353543,"name":"drain","context":{"idset":"11501","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3354051,"name":"drain","context":{"idset":"11502","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3354595,"name":"drain","context":{"idset":"11503","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3355095,"name":"drain","context":{"idset":"11504","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3355818,"name":"drain","context":{"idset":"11505","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3356397,"name":"drain","context":{"idset":"11506","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.33569,"name":"drain","context":{"idset":"11507","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3357489,"name":"drain","context":{"idset":"11508","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3357995,"name":"drain","context":{"idset":"11509","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3358498,"name":"drain","context":{"idset":"11510","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3359003,"name":"drain","context":{"idset":"11511","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3359516,"name":"drain","context":{"idset":"11512","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3360035,"name":"drain","context":{"idset":"11513","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3360636,"name":"drain","context":{"idset":"11514","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.336123,"name":"drain","context":{"idset":"11515","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3361769,"name":"drain","context":{"idset":"11516","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3362298,"name":"drain","context":{"idset":"11517","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3363199,"name":"drain","context":{"idset":"11518","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3363781,"name":"drain","context":{"idset":"11519","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3364305,"name":"drain","context":{"idset":"11520","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.336483,"name":"drain","context":{"idset":"11521","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3365345,"name":"drain","context":{"idset":"11522","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3365889,"name":"drain","context":{"idset":"11523","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3366625,"name":"drain","context":{"idset":"11524","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3367252,"name":"drain","context":{"idset":"11525","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3367805,"name":"drain","context":{"idset":"11526","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3368354,"name":"drain","context":{"idset":"11528","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3368917,"name":"drain","context":{"idset":"11529","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3369472,"name":"drain","context":{"idset":"11530","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3370023,"name":"drain","context":{"idset":"11531","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3370771,"name":"drain","context":{"idset":"11532","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3371549,"name":"drain","context":{"idset":"11533","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3372176,"name":"drain","context":{"idset":"11534","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3372924,"name":"drain","context":{"idset":"11535","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3373487,"name":"drain","context":{"idset":"11536","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3374345,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3375325,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3375971,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3376594,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3377209,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3377807,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3378441,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3379085,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.337971,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.338033,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3380945,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3381579,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3382223,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.338305,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3383703,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3384328,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3384981,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3385623,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3386288,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3386955,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3387609,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3388252,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3388906,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3389592,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3390479,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3391175,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3391857,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.339257,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3393409,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3394077,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3394797,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3395483,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3396211,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3396893,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3397601,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.339829,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3398979,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3399746,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.340049,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3401203,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3401964,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3402896,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3403695,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3404412,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.340528,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.34061,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3406882,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3407614,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.340837,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3409314,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.341011,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3410935,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3411677,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3412478,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3413384,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3414345,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3415287,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3416078,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3416948,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3417723,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.341855,"name":"drain","context":{"idset":"85","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3419325,"name":"drain","context":{"idset":"86","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3420193,"name":"drain","context":{"idset":"87","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3421075,"name":"drain","context":{"idset":"88","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3421919,"name":"drain","context":{"idset":"89","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3422942,"name":"drain","context":{"idset":"90","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3423748,"name":"drain","context":{"idset":"91","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3424635,"name":"drain","context":{"idset":"92","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3425479,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3426356,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3427296,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3428237,"name":"drain","context":{"idset":"96","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3429053,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3429852,"name":"drain","context":{"idset":"98","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3430684,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3431506,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3432314,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.343334,"name":"drain","context":{"idset":"102","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3434184,"name":"drain","context":{"idset":"103","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3435011,"name":"drain","context":{"idset":"104","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3435822,"name":"drain","context":{"idset":"105","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3436635,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3437502,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.343833,"name":"drain","context":{"idset":"108","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3439169,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3440032,"name":"drain","context":{"idset":"110","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3440893,"name":"drain","context":{"idset":"111","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3441737,"name":"drain","context":{"idset":"112","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3442614,"name":"drain","context":{"idset":"113","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3443646,"name":"drain","context":{"idset":"114","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3444505,"name":"drain","context":{"idset":"115","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3445382,"name":"drain","context":{"idset":"116","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3446279,"name":"drain","context":{"idset":"117","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3447156,"name":"drain","context":{"idset":"118","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3448081,"name":"drain","context":{"idset":"119","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3448989,"name":"drain","context":{"idset":"120","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3449872,"name":"drain","context":{"idset":"122","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.345098,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3451905,"name":"drain","context":{"idset":"124","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3452988,"name":"drain","context":{"idset":"125","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3454003,"name":"drain","context":{"idset":"126","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3455002,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3455908,"name":"drain","context":{"idset":"128","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3456857,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3457773,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3458674,"name":"drain","context":{"idset":"131","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3459592,"name":"drain","context":{"idset":"132","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.346066,"name":"drain","context":{"idset":"133","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.34618,"name":"drain","context":{"idset":"134","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3462932,"name":"drain","context":{"idset":"135","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3463862,"name":"drain","context":{"idset":"136","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3464808,"name":"drain","context":{"idset":"137","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3465757,"name":"drain","context":{"idset":"138","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.346674,"name":"drain","context":{"idset":"139","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3467896,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3468871,"name":"drain","context":{"idset":"141","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3469813,"name":"drain","context":{"idset":"142","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3470967,"name":"drain","context":{"idset":"143","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3471966,"name":"drain","context":{"idset":"144","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3473186,"name":"drain","context":{"idset":"145","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3474336,"name":"drain","context":{"idset":"146","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3475325,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3476284,"name":"drain","context":{"idset":"148","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3477392,"name":"drain","context":{"idset":"149","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3478432,"name":"drain","context":{"idset":"150","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3479431,"name":"drain","context":{"idset":"151","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3480525,"name":"drain","context":{"idset":"152","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3481488,"name":"drain","context":{"idset":"153","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3482451,"name":"drain","context":{"idset":"154","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3483634,"name":"drain","context":{"idset":"155","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3484595,"name":"drain","context":{"idset":"156","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3485644,"name":"drain","context":{"idset":"157","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3486633,"name":"drain","context":{"idset":"158","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3487644,"name":"drain","context":{"idset":"159","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3488638,"name":"drain","context":{"idset":"160","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3489614,"name":"drain","context":{"idset":"161","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3490598,"name":"drain","context":{"idset":"162","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3491566,"name":"drain","context":{"idset":"163","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3492529,"name":"drain","context":{"idset":"164","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3493779,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3494813,"name":"drain","context":{"idset":"166","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3495877,"name":"drain","context":{"idset":"167","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3496919,"name":"drain","context":{"idset":"168","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3497937,"name":"drain","context":{"idset":"169","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3498926,"name":"drain","context":{"idset":"170","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3499913,"name":"drain","context":{"idset":"171","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3500924,"name":"drain","context":{"idset":"172","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3501945,"name":"drain","context":{"idset":"173","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.350317,"name":"drain","context":{"idset":"174","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3504357,"name":"drain","context":{"idset":"175","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3505402,"name":"drain","context":{"idset":"176","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3506408,"name":"drain","context":{"idset":"177","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3507485,"name":"drain","context":{"idset":"178","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.350852,"name":"drain","context":{"idset":"179","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3509548,"name":"drain","context":{"idset":"180","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.351064,"name":"drain","context":{"idset":"181","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3511703,"name":"drain","context":{"idset":"182","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3512926,"name":"drain","context":{"idset":"183","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3513968,"name":"drain","context":{"idset":"184","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3515136,"name":"drain","context":{"idset":"185","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3516214,"name":"drain","context":{"idset":"186","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3517284,"name":"drain","context":{"idset":"187","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3518333,"name":"drain","context":{"idset":"188","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.351939,"name":"drain","context":{"idset":"189","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3520436,"name":"drain","context":{"idset":"190","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3521519,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3522599,"name":"drain","context":{"idset":"192","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3523843,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3524976,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3526216,"name":"drain","context":{"idset":"195","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3527322,"name":"drain","context":{"idset":"196","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3528419,"name":"drain","context":{"idset":"197","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3529503,"name":"drain","context":{"idset":"198","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3530583,"name":"drain","context":{"idset":"199","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3531716,"name":"drain","context":{"idset":"200","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3532991,"name":"drain","context":{"idset":"201","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3534093,"name":"drain","context":{"idset":"202","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3535187,"name":"drain","context":{"idset":"203","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3536403,"name":"drain","context":{"idset":"204","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3537524,"name":"drain","context":{"idset":"205","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3538673,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3539786,"name":"drain","context":{"idset":"207","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.35409,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3542008,"name":"drain","context":{"idset":"209","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3543348,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3544493,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3545651,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3546786,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3548017,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3549168,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3550308,"name":"drain","context":{"idset":"216","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3551459,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3552749,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3554091,"name":"drain","context":{"idset":"220","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3555305,"name":"drain","context":{"idset":"221","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3556499,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3557727,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3558967,"name":"drain","context":{"idset":"224","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3560135,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3561318,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3562541,"name":"drain","context":{"idset":"227","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3563895,"name":"drain","context":{"idset":"228","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3565094,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3566277,"name":"drain","context":{"idset":"230","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3567462,"name":"drain","context":{"idset":"231","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3568773,"name":"drain","context":{"idset":"232","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.356998,"name":"drain","context":{"idset":"233","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3571179,"name":"drain","context":{"idset":"234","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3572376,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3573759,"name":"drain","context":{"idset":"236","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3574996,"name":"drain","context":{"idset":"237","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3576217,"name":"drain","context":{"idset":"238","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3577409,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3578627,"name":"drain","context":{"idset":"240","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3579977,"name":"drain","context":{"idset":"241","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3581204,"name":"drain","context":{"idset":"242","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3582406,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3583856,"name":"drain","context":{"idset":"244","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3585095,"name":"drain","context":{"idset":"245","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3586323,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3587635,"name":"drain","context":{"idset":"247","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3588901,"name":"drain","context":{"idset":"248","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3590324,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3591597,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3593061,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3594329,"name":"drain","context":{"idset":"252","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3595655,"name":"drain","context":{"idset":"253","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3597016,"name":"drain","context":{"idset":"254","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3598309,"name":"drain","context":{"idset":"255","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3599594,"name":"drain","context":{"idset":"256","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3601019,"name":"drain","context":{"idset":"257","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3602319,"name":"drain","context":{"idset":"258","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3603773,"name":"drain","context":{"idset":"259","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3605139,"name":"drain","context":{"idset":"260","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3606458,"name":"drain","context":{"idset":"261","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3607771,"name":"drain","context":{"idset":"262","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3609068,"name":"drain","context":{"idset":"263","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3610396,"name":"drain","context":{"idset":"264","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3611865,"name":"drain","context":{"idset":"265","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3613412,"name":"drain","context":{"idset":"266","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3614728,"name":"drain","context":{"idset":"267","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3616052,"name":"drain","context":{"idset":"268","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3617394,"name":"drain","context":{"idset":"269","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3618731,"name":"drain","context":{"idset":"270","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3620043,"name":"drain","context":{"idset":"271","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3621364,"name":"drain","context":{"idset":"272","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3623002,"name":"drain","context":{"idset":"273","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3624363,"name":"drain","context":{"idset":"274","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3625698,"name":"drain","context":{"idset":"275","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3627028,"name":"drain","context":{"idset":"276","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.362838,"name":"drain","context":{"idset":"277","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3629744,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3631089,"name":"drain","context":{"idset":"279","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3632433,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3634093,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3635464,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3636847,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3638201,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3639584,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3640962,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.364233,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3644016,"name":"drain","context":{"idset":"288","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.364543,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3646824,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3648207,"name":"drain","context":{"idset":"291","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3649604,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3650978,"name":"drain","context":{"idset":"293","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3652418,"name":"drain","context":{"idset":"294","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.365402,"name":"drain","context":{"idset":"295","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3655837,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3657322,"name":"drain","context":{"idset":"297","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3658733,"name":"drain","context":{"idset":"298","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3660138,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3661532,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3663125,"name":"drain","context":{"idset":"301","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3664551,"name":"drain","context":{"idset":"302","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3666129,"name":"drain","context":{"idset":"303","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3667579,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3669007,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3670447,"name":"drain","context":{"idset":"306","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3671882,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3673718,"name":"drain","context":{"idset":"308","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3675201,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3676791,"name":"drain","context":{"idset":"310","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3678229,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3679748,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3681173,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3682609,"name":"drain","context":{"idset":"314","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3684216,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3685665,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3687308,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3688788,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3690383,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3691869,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3693483,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3694952,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3696444,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3698099,"name":"drain","context":{"idset":"324","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3699579,"name":"drain","context":{"idset":"325","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3701069,"name":"drain","context":{"idset":"326","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.370255,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3704176,"name":"drain","context":{"idset":"328","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3705664,"name":"drain","context":{"idset":"329","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3707209,"name":"drain","context":{"idset":"330","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.370888,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.371043,"name":"drain","context":{"idset":"332","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3712242,"name":"drain","context":{"idset":"333","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3714068,"name":"drain","context":{"idset":"334","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3715589,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3717096,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3718653,"name":"drain","context":{"idset":"337","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3720303,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3721824,"name":"drain","context":{"idset":"339","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.372359,"name":"drain","context":{"idset":"340","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3725116,"name":"drain","context":{"idset":"341","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3726652,"name":"drain","context":{"idset":"342","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.372817,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3729923,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3731468,"name":"drain","context":{"idset":"345","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3733189,"name":"drain","context":{"idset":"346","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3734741,"name":"drain","context":{"idset":"347","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3736324,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3737864,"name":"drain","context":{"idset":"350","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3739419,"name":"drain","context":{"idset":"351","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3741112,"name":"drain","context":{"idset":"352","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3742824,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3744447,"name":"drain","context":{"idset":"354","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.374603,"name":"drain","context":{"idset":"355","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.374763,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3749189,"name":"drain","context":{"idset":"357","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3750784,"name":"drain","context":{"idset":"358","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3752501,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3754249,"name":"drain","context":{"idset":"360","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3755832,"name":"drain","context":{"idset":"361","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.375741,"name":"drain","context":{"idset":"362","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3758988,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3760586,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3762302,"name":"drain","context":{"idset":"365","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3764064,"name":"drain","context":{"idset":"366","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3765671,"name":"drain","context":{"idset":"367","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3767259,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3768849,"name":"drain","context":{"idset":"369","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3770483,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3772109,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3773949,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3775592,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3777227,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3778925,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3780563,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3782201,"name":"drain","context":{"idset":"377","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3784142,"name":"drain","context":{"idset":"378","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3785808,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3787456,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3789103,"name":"drain","context":{"idset":"381","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3790767,"name":"drain","context":{"idset":"382","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3792422,"name":"drain","context":{"idset":"383","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.379452,"name":"drain","context":{"idset":"384","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3796225,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3797884,"name":"drain","context":{"idset":"386","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3799539,"name":"drain","context":{"idset":"387","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3801208,"name":"drain","context":{"idset":"388","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3803058,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3804743,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3806555,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.380826,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.380996,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3811648,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3813632,"name":"drain","context":{"idset":"395","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3815458,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.381732,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3819041,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3820789,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3822503,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3824353,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3826306,"name":"drain","context":{"idset":"402","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3828218,"name":"drain","context":{"idset":"403","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3829949,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.383168,"name":"drain","context":{"idset":"405","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3833568,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3835299,"name":"drain","context":{"idset":"407","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3837037,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3838921,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3840697,"name":"drain","context":{"idset":"410","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3842435,"name":"drain","context":{"idset":"411","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3844383,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3846147,"name":"drain","context":{"idset":"413","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3847873,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3849754,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3851514,"name":"drain","context":{"idset":"416","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3853424,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3855176,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3856986,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.385879,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3860676,"name":"drain","context":{"idset":"421","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3862448,"name":"drain","context":{"idset":"422","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.386436,"name":"drain","context":{"idset":"423","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3866141,"name":"drain","context":{"idset":"424","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3867927,"name":"drain","context":{"idset":"425","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3869789,"name":"drain","context":{"idset":"426","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3871622,"name":"drain","context":{"idset":"427","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3873789,"name":"drain","context":{"idset":"428","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3875649,"name":"drain","context":{"idset":"429","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3877473,"name":"drain","context":{"idset":"430","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3880968,"name":"drain","context":{"idset":"432","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.388294,"name":"drain","context":{"idset":"433","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3884802,"name":"drain","context":{"idset":"434","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3886616,"name":"drain","context":{"idset":"435","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3888433,"name":"drain","context":{"idset":"436","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3890235,"name":"drain","context":{"idset":"437","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3892024,"name":"drain","context":{"idset":"438","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3893986,"name":"drain","context":{"idset":"439","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.38958,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3897638,"name":"drain","context":{"idset":"441","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3899455,"name":"drain","context":{"idset":"442","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3901267,"name":"drain","context":{"idset":"443","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3903286,"name":"drain","context":{"idset":"444","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3905137,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3906991,"name":"drain","context":{"idset":"470","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3908827,"name":"drain","context":{"idset":"471","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3910685,"name":"drain","context":{"idset":"472","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3912516,"name":"drain","context":{"idset":"473","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3914516,"name":"drain","context":{"idset":"474","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3916368,"name":"drain","context":{"idset":"475","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3918252,"name":"drain","context":{"idset":"476","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3920126,"name":"drain","context":{"idset":"477","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3921983,"name":"drain","context":{"idset":"478","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3923976,"name":"drain","context":{"idset":"479","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3925855,"name":"drain","context":{"idset":"480","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3927729,"name":"drain","context":{"idset":"481","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3929584,"name":"drain","context":{"idset":"482","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3931441,"name":"drain","context":{"idset":"483","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3933434,"name":"drain","context":{"idset":"484","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3935292,"name":"drain","context":{"idset":"485","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3937178,"name":"drain","context":{"idset":"486","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3939052,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3940949,"name":"drain","context":{"idset":"488","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3942988,"name":"drain","context":{"idset":"489","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3944888,"name":"drain","context":{"idset":"490","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3946786,"name":"drain","context":{"idset":"491","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3948696,"name":"drain","context":{"idset":"492","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.395057,"name":"drain","context":{"idset":"493","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3952467,"name":"drain","context":{"idset":"494","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3954535,"name":"drain","context":{"idset":"495","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3956416,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3958373,"name":"drain","context":{"idset":"497","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3960271,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3962204,"name":"drain","context":{"idset":"499","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3964274,"name":"drain","context":{"idset":"500","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3966191,"name":"drain","context":{"idset":"501","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3968108,"name":"drain","context":{"idset":"502","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3970029,"name":"drain","context":{"idset":"503","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3971932,"name":"drain","context":{"idset":"504","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3974011,"name":"drain","context":{"idset":"505","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.397594,"name":"drain","context":{"idset":"506","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3977878,"name":"drain","context":{"idset":"507","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3979821,"name":"drain","context":{"idset":"508","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3981752,"name":"drain","context":{"idset":"509","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3983886,"name":"drain","context":{"idset":"510","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3985879,"name":"drain","context":{"idset":"511","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3987856,"name":"drain","context":{"idset":"512","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3989813,"name":"drain","context":{"idset":"513","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3991778,"name":"drain","context":{"idset":"514","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3993926,"name":"drain","context":{"idset":"515","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3995891,"name":"drain","context":{"idset":"516","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3997872,"name":"drain","context":{"idset":"517","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.3999856,"name":"drain","context":{"idset":"518","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4001839,"name":"drain","context":{"idset":"519","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4003971,"name":"drain","context":{"idset":"520","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4005959,"name":"drain","context":{"idset":"521","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4007962,"name":"drain","context":{"idset":"522","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4009929,"name":"drain","context":{"idset":"523","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4011934,"name":"drain","context":{"idset":"524","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4014063,"name":"drain","context":{"idset":"525","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4016082,"name":"drain","context":{"idset":"526","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4018073,"name":"drain","context":{"idset":"527","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4020078,"name":"drain","context":{"idset":"528","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4022043,"name":"drain","context":{"idset":"529","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4024143,"name":"drain","context":{"idset":"530","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4026134,"name":"drain","context":{"idset":"531","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.402813,"name":"drain","context":{"idset":"532","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4030128,"name":"drain","context":{"idset":"533","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4032156,"name":"drain","context":{"idset":"534","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4034288,"name":"drain","context":{"idset":"535","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4036317,"name":"drain","context":{"idset":"536","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4038327,"name":"drain","context":{"idset":"537","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4040349,"name":"drain","context":{"idset":"538","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4042389,"name":"drain","context":{"idset":"539","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.404458,"name":"drain","context":{"idset":"540","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4046636,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4048696,"name":"drain","context":{"idset":"566","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4050732,"name":"drain","context":{"idset":"567","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4052896,"name":"drain","context":{"idset":"568","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4054952,"name":"drain","context":{"idset":"569","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4056973,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4059,"name":"drain","context":{"idset":"571","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4061055,"name":"drain","context":{"idset":"572","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4063246,"name":"drain","context":{"idset":"573","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4065278,"name":"drain","context":{"idset":"574","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4067342,"name":"drain","context":{"idset":"575","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4069395,"name":"drain","context":{"idset":"576","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4071438,"name":"drain","context":{"idset":"577","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4073677,"name":"drain","context":{"idset":"578","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4075761,"name":"drain","context":{"idset":"579","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4077849,"name":"drain","context":{"idset":"580","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4079907,"name":"drain","context":{"idset":"581","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4081974,"name":"drain","context":{"idset":"582","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4084194,"name":"drain","context":{"idset":"583","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4086301,"name":"drain","context":{"idset":"584","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4088399,"name":"drain","context":{"idset":"585","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4090495,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4092596,"name":"drain","context":{"idset":"587","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4094849,"name":"drain","context":{"idset":"588","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4096942,"name":"drain","context":{"idset":"629","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4099073,"name":"drain","context":{"idset":"630","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4101174,"name":"drain","context":{"idset":"631","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4103434,"name":"drain","context":{"idset":"632","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4105554,"name":"drain","context":{"idset":"633","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4107659,"name":"drain","context":{"idset":"634","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4109776,"name":"drain","context":{"idset":"635","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4111905,"name":"drain","context":{"idset":"636","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4860477,"name":"drain","context":{"idset":"9973","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4863653,"name":"drain","context":{"idset":"9974","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4866724,"name":"drain","context":{"idset":"9975","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4869659,"name":"drain","context":{"idset":"9976","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4872556,"name":"drain","context":{"idset":"9977","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4875646,"name":"drain","context":{"idset":"9978","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4878664,"name":"drain","context":{"idset":"9979","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4881577,"name":"drain","context":{"idset":"9980","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4884663,"name":"drain","context":{"idset":"9981","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4887726,"name":"drain","context":{"idset":"9982","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4890645,"name":"drain","context":{"idset":"9983","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4893718,"name":"drain","context":{"idset":"9984","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4896703,"name":"drain","context":{"idset":"9985","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4899735,"name":"drain","context":{"idset":"9986","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.490284,"name":"drain","context":{"idset":"9987","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4905787,"name":"drain","context":{"idset":"9988","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4908845,"name":"drain","context":{"idset":"9989","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4911773,"name":"drain","context":{"idset":"9990","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4914861,"name":"drain","context":{"idset":"9991","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4917793,"name":"drain","context":{"idset":"9992","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4920855,"name":"drain","context":{"idset":"9993","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4923978,"name":"drain","context":{"idset":"9994","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4926891,"name":"drain","context":{"idset":"9995","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4929969,"name":"drain","context":{"idset":"9996","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4933107,"name":"drain","context":{"idset":"9997","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.493607,"name":"drain","context":{"idset":"9998","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4939041,"name":"drain","context":{"idset":"9999","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4942126,"name":"drain","context":{"idset":"10000","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4945233,"name":"drain","context":{"idset":"10001","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4948199,"name":"drain","context":{"idset":"10002","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4951246,"name":"drain","context":{"idset":"10003","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4954529,"name":"drain","context":{"idset":"10004","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.49575,"name":"drain","context":{"idset":"10005","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4960461,"name":"drain","context":{"idset":"10006","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4963751,"name":"drain","context":{"idset":"10007","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4966736,"name":"drain","context":{"idset":"10008","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4969709,"name":"drain","context":{"idset":"10009","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4972851,"name":"drain","context":{"idset":"10010","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4975948,"name":"drain","context":{"idset":"10011","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4978948,"name":"drain","context":{"idset":"10012","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4981916,"name":"drain","context":{"idset":"10013","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4985213,"name":"drain","context":{"idset":"10014","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4988186,"name":"drain","context":{"idset":"10015","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4991171,"name":"drain","context":{"idset":"10016","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4994285,"name":"drain","context":{"idset":"10017","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.4997368,"name":"drain","context":{"idset":"10018","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5000362,"name":"drain","context":{"idset":"10019","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5003529,"name":"drain","context":{"idset":"10020","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5006669,"name":"drain","context":{"idset":"10021","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5009692,"name":"drain","context":{"idset":"10022","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5012851,"name":"drain","context":{"idset":"10023","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5015903,"name":"drain","context":{"idset":"10024","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5018952,"name":"drain","context":{"idset":"10025","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5021963,"name":"drain","context":{"idset":"10026","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5025163,"name":"drain","context":{"idset":"10027","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5028312,"name":"drain","context":{"idset":"10028","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5031335,"name":"drain","context":{"idset":"10029","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5034528,"name":"drain","context":{"idset":"10030","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5037684,"name":"drain","context":{"idset":"10031","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5040698,"name":"drain","context":{"idset":"10032","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5043871,"name":"drain","context":{"idset":"10033","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5046895,"name":"drain","context":{"idset":"10034","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5050066,"name":"drain","context":{"idset":"10035","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5053258,"name":"drain","context":{"idset":"10036","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5056264,"name":"drain","context":{"idset":"10037","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5059428,"name":"drain","context":{"idset":"10038","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5062473,"name":"drain","context":{"idset":"10039","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5065644,"name":"drain","context":{"idset":"10040","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5068729,"name":"drain","context":{"idset":"10041","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5071912,"name":"drain","context":{"idset":"10042","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5075107,"name":"drain","context":{"idset":"10043","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5078139,"name":"drain","context":{"idset":"10044","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5081296,"name":"drain","context":{"idset":"10045","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5084479,"name":"drain","context":{"idset":"10046","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5087521,"name":"drain","context":{"idset":"10047","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5090559,"name":"drain","context":{"idset":"10048","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.509387,"name":"drain","context":{"idset":"10049","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5096929,"name":"drain","context":{"idset":"10050","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5099969,"name":"drain","context":{"idset":"10051","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5103374,"name":"drain","context":{"idset":"10052","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5106454,"name":"drain","context":{"idset":"10053","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5109503,"name":"drain","context":{"idset":"10054","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5112581,"name":"drain","context":{"idset":"10055","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5115967,"name":"drain","context":{"idset":"10056","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5119035,"name":"drain","context":{"idset":"10057","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5122101,"name":"drain","context":{"idset":"10058","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5125501,"name":"drain","context":{"idset":"10059","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5128622,"name":"drain","context":{"idset":"10060","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5131671,"name":"drain","context":{"idset":"10061","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5135081,"name":"drain","context":{"idset":"10062","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5138159,"name":"drain","context":{"idset":"10063","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5141225,"name":"drain","context":{"idset":"10064","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5144398,"name":"drain","context":{"idset":"10065","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5147598,"name":"drain","context":{"idset":"10066","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5150743,"name":"drain","context":{"idset":"10067","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5154097,"name":"drain","context":{"idset":"10068","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5157461,"name":"drain","context":{"idset":"10069","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5160692,"name":"drain","context":{"idset":"10070","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5164151,"name":"drain","context":{"idset":"10071","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5167787,"name":"drain","context":{"idset":"10072","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5170956,"name":"drain","context":{"idset":"10073","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5174341,"name":"drain","context":{"idset":"10074","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5177574,"name":"drain","context":{"idset":"10075","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5180929,"name":"drain","context":{"idset":"10076","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5184281,"name":"drain","context":{"idset":"10077","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5187604,"name":"drain","context":{"idset":"10078","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5190842,"name":"drain","context":{"idset":"10079","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5194116,"name":"drain","context":{"idset":"10080","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.519731,"name":"drain","context":{"idset":"10081","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5200531,"name":"drain","context":{"idset":"10082","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5203903,"name":"drain","context":{"idset":"10083","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5207033,"name":"drain","context":{"idset":"10084","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.521033,"name":"drain","context":{"idset":"10085","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5213585,"name":"drain","context":{"idset":"10086","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.521677,"name":"drain","context":{"idset":"10087","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5219927,"name":"drain","context":{"idset":"10088","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5223346,"name":"drain","context":{"idset":"10089","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5226533,"name":"drain","context":{"idset":"10090","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5229785,"name":"drain","context":{"idset":"10091","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5233276,"name":"drain","context":{"idset":"10092","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5236983,"name":"drain","context":{"idset":"10093","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5240171,"name":"drain","context":{"idset":"10094","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5243597,"name":"drain","context":{"idset":"10095","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5246823,"name":"drain","context":{"idset":"10096","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5250037,"name":"drain","context":{"idset":"10097","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.525352,"name":"drain","context":{"idset":"10098","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5256696,"name":"drain","context":{"idset":"10099","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5259898,"name":"drain","context":{"idset":"10100","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5263517,"name":"drain","context":{"idset":"10101","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5266776,"name":"drain","context":{"idset":"10102","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5270004,"name":"drain","context":{"idset":"10103","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5273342,"name":"drain","context":{"idset":"10104","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5276649,"name":"drain","context":{"idset":"10105","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5279846,"name":"drain","context":{"idset":"10106","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5283279,"name":"drain","context":{"idset":"10107","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.528646,"name":"drain","context":{"idset":"10108","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.52897,"name":"drain","context":{"idset":"10109","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5293021,"name":"drain","context":{"idset":"10110","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5296288,"name":"drain","context":{"idset":"10111","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5299587,"name":"drain","context":{"idset":"10112","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5302989,"name":"drain","context":{"idset":"10113","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5306196,"name":"drain","context":{"idset":"10114","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5309434,"name":"drain","context":{"idset":"10115","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5312943,"name":"drain","context":{"idset":"10116","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5316148,"name":"drain","context":{"idset":"10117","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5319541,"name":"drain","context":{"idset":"10118","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5322938,"name":"drain","context":{"idset":"10119","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5326164,"name":"drain","context":{"idset":"10120","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5329394,"name":"drain","context":{"idset":"10121","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5332913,"name":"drain","context":{"idset":"10122","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5336204,"name":"drain","context":{"idset":"10123","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5339587,"name":"drain","context":{"idset":"10124","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5343177,"name":"drain","context":{"idset":"10125","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5346465,"name":"drain","context":{"idset":"10126","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5349874,"name":"drain","context":{"idset":"10127","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5353551,"name":"drain","context":{"idset":"10128","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.535687,"name":"drain","context":{"idset":"10129","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5360134,"name":"drain","context":{"idset":"10130","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5363553,"name":"drain","context":{"idset":"10131","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5366993,"name":"drain","context":{"idset":"10132","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5370471,"name":"drain","context":{"idset":"10133","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5374084,"name":"drain","context":{"idset":"10134","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5384128,"name":"drain","context":{"idset":"10137","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5387487,"name":"drain","context":{"idset":"10139","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5390711,"name":"drain","context":{"idset":"10140","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5394359,"name":"drain","context":{"idset":"10141","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5397656,"name":"drain","context":{"idset":"10142","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5401099,"name":"drain","context":{"idset":"10144","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5404706,"name":"drain","context":{"idset":"10146","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5407977,"name":"drain","context":{"idset":"10149","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5411267,"name":"drain","context":{"idset":"10150","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5414679,"name":"drain","context":{"idset":"10151","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5417986,"name":"drain","context":{"idset":"10152","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5421274,"name":"drain","context":{"idset":"10155","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5424914,"name":"drain","context":{"idset":"10156","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5431349,"name":"drain","context":{"idset":"10158","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5434849,"name":"drain","context":{"idset":"10159","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5441346,"name":"drain","context":{"idset":"10161","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5444822,"name":"drain","context":{"idset":"10162","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5448101,"name":"drain","context":{"idset":"10163","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5451374,"name":"drain","context":{"idset":"10164","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5454826,"name":"drain","context":{"idset":"10165","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5458288,"name":"drain","context":{"idset":"10166","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5461891,"name":"drain","context":{"idset":"10167","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5465319,"name":"drain","context":{"idset":"10168","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5468717,"name":"drain","context":{"idset":"10169","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5471997,"name":"drain","context":{"idset":"10170","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5475667,"name":"drain","context":{"idset":"10171","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5478981,"name":"drain","context":{"idset":"10172","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5482485,"name":"drain","context":{"idset":"10173","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5486085,"name":"drain","context":{"idset":"10174","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5489421,"name":"drain","context":{"idset":"10175","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5493059,"name":"drain","context":{"idset":"10176","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5496414,"name":"drain","context":{"idset":"10177","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5499976,"name":"drain","context":{"idset":"10178","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5503843,"name":"drain","context":{"idset":"10179","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5507379,"name":"drain","context":{"idset":"10180","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5511117,"name":"drain","context":{"idset":"10181","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5514665,"name":"drain","context":{"idset":"10182","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5518029,"name":"drain","context":{"idset":"10183","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5524726,"name":"drain","context":{"idset":"10185","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5528078,"name":"drain","context":{"idset":"10186","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5531428,"name":"drain","context":{"idset":"10187","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5534976,"name":"drain","context":{"idset":"10188","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5538304,"name":"drain","context":{"idset":"10189","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.554172,"name":"drain","context":{"idset":"10190","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5545344,"name":"drain","context":{"idset":"10191","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.554882,"name":"drain","context":{"idset":"10192","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5555627,"name":"drain","context":{"idset":"10194","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5558999,"name":"drain","context":{"idset":"10195","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5562367,"name":"drain","context":{"idset":"10196","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5565922,"name":"drain","context":{"idset":"10197","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5569315,"name":"drain","context":{"idset":"10198","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5572894,"name":"drain","context":{"idset":"10199","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5576289,"name":"drain","context":{"idset":"10200","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5580096,"name":"drain","context":{"idset":"10201","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5583668,"name":"drain","context":{"idset":"10202","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5590496,"name":"drain","context":{"idset":"10204","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.559427,"name":"drain","context":{"idset":"10205","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5597813,"name":"drain","context":{"idset":"10206","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5604839,"name":"drain","context":{"idset":"10208","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.560832,"name":"drain","context":{"idset":"10209","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5611775,"name":"drain","context":{"idset":"10210","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5615592,"name":"drain","context":{"idset":"10211","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5619023,"name":"drain","context":{"idset":"10212","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5622392,"name":"drain","context":{"idset":"10213","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5626059,"name":"drain","context":{"idset":"10214","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5629659,"name":"drain","context":{"idset":"10215","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5636599,"name":"drain","context":{"idset":"10217","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5639977,"name":"drain","context":{"idset":"10218","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5643754,"name":"drain","context":{"idset":"10219","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5647285,"name":"drain","context":{"idset":"10220","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5654542,"name":"drain","context":{"idset":"10222","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5658033,"name":"drain","context":{"idset":"10223","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5661533,"name":"drain","context":{"idset":"10224","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5665703,"name":"drain","context":{"idset":"10225","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5669308,"name":"drain","context":{"idset":"10226","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.567369,"name":"drain","context":{"idset":"10227","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5677304,"name":"drain","context":{"idset":"10228","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5680912,"name":"drain","context":{"idset":"10229","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5684569,"name":"drain","context":{"idset":"10230","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5688059,"name":"drain","context":{"idset":"10231","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5691638,"name":"drain","context":{"idset":"10232","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5698538,"name":"drain","context":{"idset":"10234","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5702131,"name":"drain","context":{"idset":"10235","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5705788,"name":"drain","context":{"idset":"10236","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5709255,"name":"drain","context":{"idset":"10237","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.57131,"name":"drain","context":{"idset":"10238","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5716588,"name":"drain","context":{"idset":"10239","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5720057,"name":"drain","context":{"idset":"10240","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.572381,"name":"drain","context":{"idset":"10241","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5727313,"name":"drain","context":{"idset":"10242","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5730774,"name":"drain","context":{"idset":"10243","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5734551,"name":"drain","context":{"idset":"10244","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5738041,"name":"drain","context":{"idset":"10245","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5741525,"name":"drain","context":{"idset":"10246","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5745289,"name":"drain","context":{"idset":"10247","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5748789,"name":"drain","context":{"idset":"10248","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5752265,"name":"drain","context":{"idset":"10249","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5759301,"name":"drain","context":{"idset":"10251","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5762978,"name":"drain","context":{"idset":"10252","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5766599,"name":"drain","context":{"idset":"10253","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5770099,"name":"drain","context":{"idset":"10254","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5773785,"name":"drain","context":{"idset":"10255","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5780764,"name":"drain","context":{"idset":"10257","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5784411,"name":"drain","context":{"idset":"10258","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5788031,"name":"drain","context":{"idset":"10259","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5791533,"name":"drain","context":{"idset":"10260","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5795214,"name":"drain","context":{"idset":"10262","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5798872,"name":"drain","context":{"idset":"10263","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5802431,"name":"drain","context":{"idset":"10264","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5806127,"name":"drain","context":{"idset":"10265","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5809791,"name":"drain","context":{"idset":"10266","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5816925,"name":"drain","context":{"idset":"10268","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5820611,"name":"drain","context":{"idset":"10269","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5824347,"name":"drain","context":{"idset":"10270","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5827866,"name":"drain","context":{"idset":"10271","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5831521,"name":"drain","context":{"idset":"10272","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5838609,"name":"drain","context":{"idset":"10274","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5842285,"name":"drain","context":{"idset":"10275","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5845973,"name":"drain","context":{"idset":"10276","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5849533,"name":"drain","context":{"idset":"10277","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5856805,"name":"drain","context":{"idset":"10279","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5864031,"name":"drain","context":{"idset":"10281","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5867612,"name":"drain","context":{"idset":"10282","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5871191,"name":"drain","context":{"idset":"10283","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5875115,"name":"drain","context":{"idset":"10284","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5878735,"name":"drain","context":{"idset":"10285","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5882318,"name":"drain","context":{"idset":"10286","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5886271,"name":"drain","context":{"idset":"10287","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5889881,"name":"drain","context":{"idset":"10288","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5893624,"name":"drain","context":{"idset":"10289","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5897315,"name":"drain","context":{"idset":"10290","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5900893,"name":"drain","context":{"idset":"10291","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5904727,"name":"drain","context":{"idset":"10292","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5908475,"name":"drain","context":{"idset":"10293","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5912087,"name":"drain","context":{"idset":"10294","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5915816,"name":"drain","context":{"idset":"10295","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5919535,"name":"drain","context":{"idset":"10296","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5923288,"name":"drain","context":{"idset":"10297","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5926888,"name":"drain","context":{"idset":"10298","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5930645,"name":"drain","context":{"idset":"10299","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5934403,"name":"drain","context":{"idset":"10300","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5948629,"name":"drain","context":{"idset":"10306","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5952365,"name":"drain","context":{"idset":"10307","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5956314,"name":"drain","context":{"idset":"10308","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5960073,"name":"drain","context":{"idset":"10309","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5963972,"name":"drain","context":{"idset":"10310","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5967638,"name":"drain","context":{"idset":"10311","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5971415,"name":"drain","context":{"idset":"10312","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5975368,"name":"drain","context":{"idset":"10313","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.597908,"name":"drain","context":{"idset":"10314","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.598305,"name":"drain","context":{"idset":"10315","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5987015,"name":"drain","context":{"idset":"10316","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5990744,"name":"drain","context":{"idset":"10317","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.5994682,"name":"drain","context":{"idset":"10318","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.599848,"name":"drain","context":{"idset":"10319","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6002128,"name":"drain","context":{"idset":"10320","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6006029,"name":"drain","context":{"idset":"10321","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6009834,"name":"drain","context":{"idset":"10322","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6013801,"name":"drain","context":{"idset":"10323","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6017594,"name":"drain","context":{"idset":"10324","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6021309,"name":"drain","context":{"idset":"10325","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6025214,"name":"drain","context":{"idset":"10326","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6028988,"name":"drain","context":{"idset":"10327","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6032894,"name":"drain","context":{"idset":"10328","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6036735,"name":"drain","context":{"idset":"10329","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6040592,"name":"drain","context":{"idset":"10330","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6044462,"name":"drain","context":{"idset":"10331","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6048295,"name":"drain","context":{"idset":"10332","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6052096,"name":"drain","context":{"idset":"10333","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6055942,"name":"drain","context":{"idset":"10334","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6059752,"name":"drain","context":{"idset":"10335","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.60637,"name":"drain","context":{"idset":"10336","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6067472,"name":"drain","context":{"idset":"10337","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6071198,"name":"drain","context":{"idset":"10338","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6075077,"name":"drain","context":{"idset":"10339","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6078923,"name":"drain","context":{"idset":"10340","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6082878,"name":"drain","context":{"idset":"10341","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.608659,"name":"drain","context":{"idset":"10342","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6090446,"name":"drain","context":{"idset":"10343","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.609437,"name":"drain","context":{"idset":"10344","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6098089,"name":"drain","context":{"idset":"10345","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6101918,"name":"drain","context":{"idset":"10346","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6105914,"name":"drain","context":{"idset":"10347","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6109664,"name":"drain","context":{"idset":"10348","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6113636,"name":"drain","context":{"idset":"10349","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.611748,"name":"drain","context":{"idset":"10350","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6121342,"name":"drain","context":{"idset":"10351","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6125402,"name":"drain","context":{"idset":"10352","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6129189,"name":"drain","context":{"idset":"10353","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6133208,"name":"drain","context":{"idset":"10354","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6137066,"name":"drain","context":{"idset":"10355","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6144719,"name":"drain","context":{"idset":"10358","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6148603,"name":"drain","context":{"idset":"10359","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6152339,"name":"drain","context":{"idset":"10360","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6156359,"name":"drain","context":{"idset":"10361","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6160247,"name":"drain","context":{"idset":"10362","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6164329,"name":"drain","context":{"idset":"10363","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.616823,"name":"drain","context":{"idset":"10364","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.617198,"name":"drain","context":{"idset":"10365","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6176009,"name":"drain","context":{"idset":"10366","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.617991,"name":"drain","context":{"idset":"10367","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6183822,"name":"drain","context":{"idset":"10368","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6187723,"name":"drain","context":{"idset":"10369","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.619163,"name":"drain","context":{"idset":"10370","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6195583,"name":"drain","context":{"idset":"10371","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6199458,"name":"drain","context":{"idset":"10372","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6203573,"name":"drain","context":{"idset":"10373","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6211312,"name":"drain","context":{"idset":"10375","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6215274,"name":"drain","context":{"idset":"10376","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6219215,"name":"drain","context":{"idset":"10377","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6223278,"name":"drain","context":{"idset":"10378","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6227064,"name":"drain","context":{"idset":"10379","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6230981,"name":"drain","context":{"idset":"10380","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6235042,"name":"drain","context":{"idset":"10381","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6238818,"name":"drain","context":{"idset":"10382","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6242845,"name":"drain","context":{"idset":"10383","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6246688,"name":"drain","context":{"idset":"10384","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6251051,"name":"drain","context":{"idset":"10385","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6255183,"name":"drain","context":{"idset":"10386","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6259038,"name":"drain","context":{"idset":"10387","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.626312,"name":"drain","context":{"idset":"10388","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6267054,"name":"drain","context":{"idset":"10389","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6270874,"name":"drain","context":{"idset":"10390","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6275034,"name":"drain","context":{"idset":"10391","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6278918,"name":"drain","context":{"idset":"10392","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6283002,"name":"drain","context":{"idset":"10393","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6286926,"name":"drain","context":{"idset":"10394","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6290746,"name":"drain","context":{"idset":"10395","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6294842,"name":"drain","context":{"idset":"10396","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6298823,"name":"drain","context":{"idset":"10397","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6302819,"name":"drain","context":{"idset":"10398","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.630851,"name":"drain","context":{"idset":"10399","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.631752,"name":"drain","context":{"idset":"10401","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6321428,"name":"drain","context":{"idset":"10402","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6325765,"name":"drain","context":{"idset":"10403","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6329734,"name":"drain","context":{"idset":"10404","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6333871,"name":"drain","context":{"idset":"10405","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6337972,"name":"drain","context":{"idset":"10406","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6341929,"name":"drain","context":{"idset":"10407","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6346033,"name":"drain","context":{"idset":"10408","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6350117,"name":"drain","context":{"idset":"10409","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6354256,"name":"drain","context":{"idset":"10410","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6358373,"name":"drain","context":{"idset":"10411","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6362326,"name":"drain","context":{"idset":"10412","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6366489,"name":"drain","context":{"idset":"10413","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6370614,"name":"drain","context":{"idset":"10414","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6374719,"name":"drain","context":{"idset":"10415","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6378675,"name":"drain","context":{"idset":"10416","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6382945,"name":"drain","context":{"idset":"10417","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6386979,"name":"drain","context":{"idset":"10418","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6391177,"name":"drain","context":{"idset":"10419","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6395347,"name":"drain","context":{"idset":"10420","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.640343,"name":"drain","context":{"idset":"10422","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6407447,"name":"drain","context":{"idset":"10423","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6411591,"name":"drain","context":{"idset":"10424","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6415725,"name":"drain","context":{"idset":"10425","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6419747,"name":"drain","context":{"idset":"10426","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6424038,"name":"drain","context":{"idset":"10427","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6428056,"name":"drain","context":{"idset":"10428","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6432076,"name":"drain","context":{"idset":"10429","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6436331,"name":"drain","context":{"idset":"10430","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6440313,"name":"drain","context":{"idset":"10431","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6448426,"name":"drain","context":{"idset":"10433","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6452448,"name":"drain","context":{"idset":"10434","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6456733,"name":"drain","context":{"idset":"10435","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6464684,"name":"drain","context":{"idset":"10437","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6468718,"name":"drain","context":{"idset":"10438","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6472921,"name":"drain","context":{"idset":"10439","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6476972,"name":"drain","context":{"idset":"10440","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.648102,"name":"drain","context":{"idset":"10441","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6485255,"name":"drain","context":{"idset":"10442","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.648932,"name":"drain","context":{"idset":"10443","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6497307,"name":"drain","context":{"idset":"10445","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6501355,"name":"drain","context":{"idset":"10446","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6505585,"name":"drain","context":{"idset":"10447","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6509666,"name":"drain","context":{"idset":"10448","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6513872,"name":"drain","context":{"idset":"10449","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6525676,"name":"drain","context":{"idset":"10452","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6529775,"name":"drain","context":{"idset":"10453","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6534047,"name":"drain","context":{"idset":"10454","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6538088,"name":"drain","context":{"idset":"10455","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6546156,"name":"drain","context":{"idset":"10457","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6550245,"name":"drain","context":{"idset":"10458","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6554506,"name":"drain","context":{"idset":"10459","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6558614,"name":"drain","context":{"idset":"10460","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6562798,"name":"drain","context":{"idset":"10461","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6566911,"name":"drain","context":{"idset":"10462","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6570983,"name":"drain","context":{"idset":"10463","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.657521,"name":"drain","context":{"idset":"10464","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6579289,"name":"drain","context":{"idset":"10465","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6587389,"name":"drain","context":{"idset":"10467","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.6591487,"name":"drain","context":{"idset":"10468","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9087579,"name":"drain","context":{"idset":"10996","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9093142,"name":"drain","context":{"idset":"10997","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9098587,"name":"drain","context":{"idset":"10998","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9135165,"name":"drain","context":{"idset":"11007","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9140501,"name":"drain","context":{"idset":"11008","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9145885,"name":"drain","context":{"idset":"11009","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9151227,"name":"drain","context":{"idset":"11010","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9156902,"name":"drain","context":{"idset":"11011","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9167793,"name":"drain","context":{"idset":"11013","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9173367,"name":"drain","context":{"idset":"11014","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.917856,"name":"drain","context":{"idset":"11015","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9184012,"name":"drain","context":{"idset":"11016","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9189241,"name":"drain","context":{"idset":"11017","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9194727,"name":"drain","context":{"idset":"11018","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9200003,"name":"drain","context":{"idset":"11019","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9205587,"name":"drain","context":{"idset":"11020","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.921088,"name":"drain","context":{"idset":"11021","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9216475,"name":"drain","context":{"idset":"11022","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9221759,"name":"drain","context":{"idset":"11023","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9227283,"name":"drain","context":{"idset":"11024","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9233172,"name":"drain","context":{"idset":"11025","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9239047,"name":"drain","context":{"idset":"11026","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9244971,"name":"drain","context":{"idset":"11027","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9250629,"name":"drain","context":{"idset":"11028","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9256351,"name":"drain","context":{"idset":"11029","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.926183,"name":"drain","context":{"idset":"11030","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.92676,"name":"drain","context":{"idset":"11031","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9273551,"name":"drain","context":{"idset":"11032","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9278975,"name":"drain","context":{"idset":"11033","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.928463,"name":"drain","context":{"idset":"11034","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.929076,"name":"drain","context":{"idset":"11035","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9296522,"name":"drain","context":{"idset":"11036","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.930203,"name":"drain","context":{"idset":"11037","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9307525,"name":"drain","context":{"idset":"11038","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9313085,"name":"drain","context":{"idset":"11039","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.931839,"name":"drain","context":{"idset":"11040","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9323926,"name":"drain","context":{"idset":"11041","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9329221,"name":"drain","context":{"idset":"11042","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9334776,"name":"drain","context":{"idset":"11043","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9340062,"name":"drain","context":{"idset":"11044","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9345689,"name":"drain","context":{"idset":"11045","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9350994,"name":"drain","context":{"idset":"11046","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9356637,"name":"drain","context":{"idset":"11047","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9361932,"name":"drain","context":{"idset":"11048","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9367526,"name":"drain","context":{"idset":"11049","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9372978,"name":"drain","context":{"idset":"11050","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9378414,"name":"drain","context":{"idset":"11051","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.93839,"name":"drain","context":{"idset":"11052","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9389751,"name":"drain","context":{"idset":"11053","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9398067,"name":"drain","context":{"idset":"11054","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9403739,"name":"drain","context":{"idset":"11055","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9409561,"name":"drain","context":{"idset":"11056","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9415841,"name":"drain","context":{"idset":"11057","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9421244,"name":"drain","context":{"idset":"11058","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9426796,"name":"drain","context":{"idset":"11059","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9432137,"name":"drain","context":{"idset":"11060","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9437613,"name":"drain","context":{"idset":"11061","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9443102,"name":"drain","context":{"idset":"11062","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9448414,"name":"drain","context":{"idset":"11063","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9453907,"name":"drain","context":{"idset":"11064","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9459233,"name":"drain","context":{"idset":"11065","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9464796,"name":"drain","context":{"idset":"11066","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9470108,"name":"drain","context":{"idset":"11067","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9475596,"name":"drain","context":{"idset":"11068","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9480915,"name":"drain","context":{"idset":"11069","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9486434,"name":"drain","context":{"idset":"11070","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9491765,"name":"drain","context":{"idset":"11071","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9497261,"name":"drain","context":{"idset":"11072","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9502592,"name":"drain","context":{"idset":"11073","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9508212,"name":"drain","context":{"idset":"11074","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9514351,"name":"drain","context":{"idset":"11075","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9520388,"name":"drain","context":{"idset":"11076","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9526832,"name":"drain","context":{"idset":"11077","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9533105,"name":"drain","context":{"idset":"11078","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9538796,"name":"drain","context":{"idset":"11079","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9544353,"name":"drain","context":{"idset":"11080","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9549735,"name":"drain","context":{"idset":"11081","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9555285,"name":"drain","context":{"idset":"11082","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9560697,"name":"drain","context":{"idset":"11083","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.956629,"name":"drain","context":{"idset":"11084","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9571671,"name":"drain","context":{"idset":"11085","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9577212,"name":"drain","context":{"idset":"11086","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9582589,"name":"drain","context":{"idset":"11087","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9588201,"name":"drain","context":{"idset":"11088","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9593759,"name":"drain","context":{"idset":"11089","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9599118,"name":"drain","context":{"idset":"11090","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9604702,"name":"drain","context":{"idset":"11091","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9610121,"name":"drain","context":{"idset":"11092","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9615688,"name":"drain","context":{"idset":"11093","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9621062,"name":"drain","context":{"idset":"11094","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9626632,"name":"drain","context":{"idset":"11095","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9632032,"name":"drain","context":{"idset":"11096","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9637587,"name":"drain","context":{"idset":"11097","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9643178,"name":"drain","context":{"idset":"11098","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9648595,"name":"drain","context":{"idset":"11099","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9654193,"name":"drain","context":{"idset":"11100","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9659607,"name":"drain","context":{"idset":"11101","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9665208,"name":"drain","context":{"idset":"11102","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9670901,"name":"drain","context":{"idset":"11103","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9676571,"name":"drain","context":{"idset":"11104","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9682019,"name":"drain","context":{"idset":"11105","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9687665,"name":"drain","context":{"idset":"11106","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9693274,"name":"drain","context":{"idset":"11107","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9698684,"name":"drain","context":{"idset":"11108","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.977936,"name":"drain","context":{"idset":"11127","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9785099,"name":"drain","context":{"idset":"11128","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9790606,"name":"drain","context":{"idset":"11129","reason":"broker was unresponsive"}}
+{"timestamp":1712327590.9801767,"name":"drain","context":{"idset":"11131","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.004632,"name":"drain","context":{"idset":"11537","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0051975,"name":"drain","context":{"idset":"11538","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0057843,"name":"drain","context":{"idset":"11539","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0063651,"name":"drain","context":{"idset":"11540","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0069366,"name":"drain","context":{"idset":"11541","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0075114,"name":"drain","context":{"idset":"11542","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0080833,"name":"drain","context":{"idset":"11543","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0086622,"name":"drain","context":{"idset":"11544","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0092337,"name":"drain","context":{"idset":"11545","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0098152,"name":"drain","context":{"idset":"11546","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0103991,"name":"drain","context":{"idset":"11547","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0109701,"name":"drain","context":{"idset":"11548","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0115471,"name":"drain","context":{"idset":"11549","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0121188,"name":"drain","context":{"idset":"11550","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0126956,"name":"drain","context":{"idset":"11551","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0132933,"name":"drain","context":{"idset":"11552","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0138562,"name":"drain","context":{"idset":"11553","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0144572,"name":"drain","context":{"idset":"11554","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0150182,"name":"drain","context":{"idset":"11555","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0156167,"name":"drain","context":{"idset":"11556","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0161793,"name":"drain","context":{"idset":"11557","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0167713,"name":"drain","context":{"idset":"11558","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0173571,"name":"drain","context":{"idset":"11559","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0179217,"name":"drain","context":{"idset":"11560","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0185175,"name":"drain","context":{"idset":"11561","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0190768,"name":"drain","context":{"idset":"11562","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0196681,"name":"drain","context":{"idset":"11564","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0202327,"name":"drain","context":{"idset":"11565","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0208254,"name":"drain","context":{"idset":"11566","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0214384,"name":"drain","context":{"idset":"11567","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0220182,"name":"drain","context":{"idset":"11568","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0226018,"name":"drain","context":{"idset":"11569","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0231812,"name":"drain","context":{"idset":"11570","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.023803,"name":"drain","context":{"idset":"11571","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0243976,"name":"drain","context":{"idset":"11572","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0249751,"name":"drain","context":{"idset":"11573","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0255635,"name":"drain","context":{"idset":"11574","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0261593,"name":"drain","context":{"idset":"11575","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0267904,"name":"drain","context":{"idset":"11576","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0273938,"name":"drain","context":{"idset":"11577","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0279622,"name":"drain","context":{"idset":"11578","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0285625,"name":"drain","context":{"idset":"11579","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0291319,"name":"drain","context":{"idset":"11580","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0297291,"name":"drain","context":{"idset":"11581","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0303302,"name":"drain","context":{"idset":"11582","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0309014,"name":"drain","context":{"idset":"11583","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0315046,"name":"drain","context":{"idset":"11584","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0320725,"name":"drain","context":{"idset":"11585","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0326738,"name":"drain","context":{"idset":"11586","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0332453,"name":"drain","context":{"idset":"11589","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.033849,"name":"drain","context":{"idset":"11590","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0344658,"name":"drain","context":{"idset":"11591","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0350592,"name":"drain","context":{"idset":"11592","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0356531,"name":"drain","context":{"idset":"11593","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.036258,"name":"drain","context":{"idset":"11594","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0368838,"name":"drain","context":{"idset":"11595","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0374701,"name":"drain","context":{"idset":"11596","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0380549,"name":"drain","context":{"idset":"11597","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0386474,"name":"drain","context":{"idset":"11598","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0392363,"name":"drain","context":{"idset":"11599","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0398617,"name":"drain","context":{"idset":"11600","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0404766,"name":"drain","context":{"idset":"11601","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.0410509,"name":"drain","context":{"idset":"11602","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2378466,"name":"drain","context":{"idset":"11223","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2431269,"name":"drain","context":{"idset":"11231","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.248414,"name":"drain","context":{"idset":"11239","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2517166,"name":"drain","context":{"idset":"11244","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2531064,"name":"drain","context":{"idset":"11246","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2577469,"name":"drain","context":{"idset":"11253","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2584586,"name":"drain","context":{"idset":"11254","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2591479,"name":"drain","context":{"idset":"11255","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2598441,"name":"drain","context":{"idset":"11256","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.260555,"name":"drain","context":{"idset":"11257","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2612493,"name":"drain","context":{"idset":"11258","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2619455,"name":"drain","context":{"idset":"11259","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2626569,"name":"drain","context":{"idset":"11260","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2633708,"name":"drain","context":{"idset":"11261","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2640526,"name":"drain","context":{"idset":"11262","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2647684,"name":"drain","context":{"idset":"11263","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2654829,"name":"drain","context":{"idset":"11264","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.266165,"name":"drain","context":{"idset":"11265","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2668793,"name":"drain","context":{"idset":"11266","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2675953,"name":"drain","context":{"idset":"11267","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2682948,"name":"drain","context":{"idset":"11268","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.268986,"name":"drain","context":{"idset":"11269","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2697039,"name":"drain","context":{"idset":"11270","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2704029,"name":"drain","context":{"idset":"11271","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2710946,"name":"drain","context":{"idset":"11272","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2717881,"name":"drain","context":{"idset":"11273","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2725,"name":"drain","context":{"idset":"11274","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2731948,"name":"drain","context":{"idset":"11275","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2739153,"name":"drain","context":{"idset":"11276","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2746301,"name":"drain","context":{"idset":"11277","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2753487,"name":"drain","context":{"idset":"11278","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2760286,"name":"drain","context":{"idset":"11279","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2767384,"name":"drain","context":{"idset":"11280","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2774544,"name":"drain","context":{"idset":"11281","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2781353,"name":"drain","context":{"idset":"11282","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.278841,"name":"drain","context":{"idset":"11283","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2795577,"name":"drain","context":{"idset":"11284","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2802393,"name":"drain","context":{"idset":"11285","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2809522,"name":"drain","context":{"idset":"11286","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2823305,"name":"drain","context":{"idset":"11288","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2830281,"name":"drain","context":{"idset":"11289","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2837672,"name":"drain","context":{"idset":"11290","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2845824,"name":"drain","context":{"idset":"11291","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2853863,"name":"drain","context":{"idset":"11292","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2861814,"name":"drain","context":{"idset":"11293","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2869735,"name":"drain","context":{"idset":"11294","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2877607,"name":"drain","context":{"idset":"11295","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2884858,"name":"drain","context":{"idset":"11296","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2891903,"name":"drain","context":{"idset":"11297","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2899146,"name":"drain","context":{"idset":"11298","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2907174,"name":"drain","context":{"idset":"11299","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.291523,"name":"drain","context":{"idset":"11300","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.29228,"name":"drain","context":{"idset":"11301","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2930491,"name":"drain","context":{"idset":"11302","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2937779,"name":"drain","context":{"idset":"11303","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2945073,"name":"drain","context":{"idset":"11304","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.295279,"name":"drain","context":{"idset":"11305","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2960703,"name":"drain","context":{"idset":"11306","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2968245,"name":"drain","context":{"idset":"11307","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2976091,"name":"drain","context":{"idset":"11308","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2983735,"name":"drain","context":{"idset":"11309","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2991149,"name":"drain","context":{"idset":"11310","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.2998338,"name":"drain","context":{"idset":"11311","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3005912,"name":"drain","context":{"idset":"11312","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3013582,"name":"drain","context":{"idset":"11313","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3020537,"name":"drain","context":{"idset":"11314","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3027582,"name":"drain","context":{"idset":"11315","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3034711,"name":"drain","context":{"idset":"11316","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3041975,"name":"drain","context":{"idset":"11317","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3049035,"name":"drain","context":{"idset":"11318","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3056154,"name":"drain","context":{"idset":"11319","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3063269,"name":"drain","context":{"idset":"11320","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3070312,"name":"drain","context":{"idset":"11321","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3077536,"name":"drain","context":{"idset":"11322","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.308502,"name":"drain","context":{"idset":"11323","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3162344,"name":"drain","context":{"idset":"11335","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3170049,"name":"drain","context":{"idset":"11336","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3177714,"name":"drain","context":{"idset":"11337","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3185468,"name":"drain","context":{"idset":"11338","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3193197,"name":"drain","context":{"idset":"11339","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3200583,"name":"drain","context":{"idset":"11340","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3208137,"name":"drain","context":{"idset":"11341","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3215833,"name":"drain","context":{"idset":"11342","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.322376,"name":"drain","context":{"idset":"11343","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.323113,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3238764,"name":"drain","context":{"idset":"11345","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3246517,"name":"drain","context":{"idset":"11346","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3254318,"name":"drain","context":{"idset":"11347","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3261893,"name":"drain","context":{"idset":"11348","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3269305,"name":"drain","context":{"idset":"11349","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3277256,"name":"drain","context":{"idset":"11350","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3286119,"name":"drain","context":{"idset":"11351","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3294153,"name":"drain","context":{"idset":"11352","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3301656,"name":"drain","context":{"idset":"11353","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3309226,"name":"drain","context":{"idset":"11354","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3317037,"name":"drain","context":{"idset":"11355","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3324792,"name":"drain","context":{"idset":"11356","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3331943,"name":"drain","context":{"idset":"11357","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3339174,"name":"drain","context":{"idset":"11359","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.334677,"name":"drain","context":{"idset":"11360","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3354013,"name":"drain","context":{"idset":"11361","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3361189,"name":"drain","context":{"idset":"11362","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3368359,"name":"drain","context":{"idset":"11363","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3375733,"name":"drain","context":{"idset":"11364","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3383009,"name":"drain","context":{"idset":"11365","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3462174,"name":"drain","context":{"idset":"11377","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3469515,"name":"drain","context":{"idset":"11378","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3476837,"name":"drain","context":{"idset":"11379","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3484085,"name":"drain","context":{"idset":"11380","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3491204,"name":"drain","context":{"idset":"11381","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3498476,"name":"drain","context":{"idset":"11382","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3505847,"name":"drain","context":{"idset":"11383","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3513114,"name":"drain","context":{"idset":"11384","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3520303,"name":"drain","context":{"idset":"11385","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3527756,"name":"drain","context":{"idset":"11386","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3832524,"name":"drain","context":{"idset":"11430","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.384357,"name":"drain","context":{"idset":"11432","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3854582,"name":"drain","context":{"idset":"11433","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3865533,"name":"drain","context":{"idset":"11434","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3876581,"name":"drain","context":{"idset":"11435","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3887517,"name":"drain","context":{"idset":"11436","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3897929,"name":"drain","context":{"idset":"11437","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3906813,"name":"drain","context":{"idset":"11438","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3914294,"name":"drain","context":{"idset":"11443","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3921533,"name":"drain","context":{"idset":"11444","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3929119,"name":"drain","context":{"idset":"11445","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3936567,"name":"drain","context":{"idset":"11446","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3943975,"name":"drain","context":{"idset":"11447","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3951194,"name":"drain","context":{"idset":"11448","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3958559,"name":"drain","context":{"idset":"11449","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3967297,"name":"drain","context":{"idset":"11450","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3974941,"name":"drain","context":{"idset":"11451","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3982477,"name":"drain","context":{"idset":"11452","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3990014,"name":"drain","context":{"idset":"11453","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.3997445,"name":"drain","context":{"idset":"11454","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4004893,"name":"drain","context":{"idset":"11456","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4012105,"name":"drain","context":{"idset":"11457","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4019568,"name":"drain","context":{"idset":"11458","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4026971,"name":"drain","context":{"idset":"11459","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4034297,"name":"drain","context":{"idset":"11460","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4041517,"name":"drain","context":{"idset":"11461","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4048967,"name":"drain","context":{"idset":"11462","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4056363,"name":"drain","context":{"idset":"11463","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.406383,"name":"drain","context":{"idset":"11465","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4071078,"name":"drain","context":{"idset":"11467","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4078469,"name":"drain","context":{"idset":"11468","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.408622,"name":"drain","context":{"idset":"11469","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4093895,"name":"drain","context":{"idset":"11470","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4101202,"name":"drain","context":{"idset":"11471","reason":"broker was unresponsive"}}
+{"timestamp":1712327591.4108677,"name":"drain","context":{"idset":"11472","reason":"broker was unresponsive"}}
+{"timestamp":1712328628.4681363,"name":"undrain","context":{"idset":"10474-10475"}}
+{"timestamp":1712328953.3829122,"name":"drain","context":{"idset":"10474","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712328977.256582,"name":"undrain","context":{"idset":"801,822-823,826,830,9537,9717,9758,9778,9786,10157,10160,10184,10193,10203,10207,10216,10221,10233,10250,10256,10267,10273,10278,10280,10305,10400,10421,10432,10436,10444,10450-10451,10456,10466,10486,10490,10522,10524,10552,10557,10561,10589,10598,10608,10701,10704,10746,10748,10812,10835,10869,10890,10927,10931,10957,10977,11149,11184,11196,11218,11228,11251,11396,11666,11695,11700,11713"}}
+{"timestamp":1712329120.7360339,"name":"drain","context":{"idset":"11466","overwrite":0}}
+{"timestamp":1712329439.8604815,"name":"online","context":{"idset":"845"}}
+{"timestamp":1712329439.8632472,"name":"online","context":{"idset":"846"}}
+{"timestamp":1712329442.7562137,"name":"drain","context":{"idset":"11695","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712329520.4584391,"name":"undrain","context":{"idset":"799-800,802-821,824-825,827-829,831-836,841-842,871-872,949-950,953-954,963-964,975-976,9461-9532,9536,9538-9558,9560,9563-9588,9718-9757,9759-9777,9779-9785,9787-9844,10357,10485,10487-10489,10491-10516,10519-10521,10523,10525-10551,10553-10556,10558-10560,10562-10588,10590-10597,10599-10607,10609-10628,10630-10633,10635-10636,10640-10641,10645-10656,10658-10692,10695-10700,10703,10705-10711,10713-10717,10719-10738,10741-10745,10747,10749-10811,10813-10834,10836-10868,10870,10872-10889,10891-10910,10913-10926,10928-10930,10932-10942,10944-10956,10958-10976,10978-10995,11001-11006,11012,11109-11114,11117-11124,11132-11133,11135,11137,11139,11141-11148,11150-11183,11185-11195,11197-11217,11219,11221-11222,11224-11225,11227,11229-11230,11232,11234,11237-11238,11240-11242,11245,11248,11250,11252,11287,11324-11329,11331-11334,11366-11372,11374-11376,11389,11391-11394,11397-11399,11603-11615,11653-11665,11667-11694,11696-11699,11701-11712,11714-11764,11802"}}
+{"timestamp":1712330176.622016,"name":"offline","context":{"idset":"11892"}}
+{"timestamp":1712330414.5834444,"name":"drain","context":{"idset":"11685","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330433.3751938,"name":"drain","context":{"idset":"11686","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1712330433.4678483,"name":"drain","context":{"idset":"11687","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330433.4694264,"name":"offline","context":{"idset":"817"}}
+{"timestamp":1712330433.561861,"name":"drain","context":{"idset":"11689","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330433.5634325,"name":"offline","context":{"idset":"818"}}
+{"timestamp":1712330433.6560903,"name":"drain","context":{"idset":"11691","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712330433.7542911,"name":"drain","context":{"idset":"11696","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330433.8468153,"name":"drain","context":{"idset":"11692","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712330434.0443096,"name":"drain","context":{"idset":"11698","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330434.1565216,"name":"drain","context":{"idset":"11703","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330434.3320162,"name":"drain","context":{"idset":"11702","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330434.4953003,"name":"drain","context":{"idset":"11704","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330434.653213,"name":"drain","context":{"idset":"11706","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330434.814641,"name":"drain","context":{"idset":"11707","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330434.9760258,"name":"drain","context":{"idset":"11709","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330450.3736897,"name":"drain","context":{"idset":"11710","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330450.4731157,"name":"drain","context":{"idset":"11711","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330469.6357894,"name":"drain","context":{"idset":"11714","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330470.6818855,"name":"drain","context":{"idset":"11715","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330471.2624297,"name":"drain","context":{"idset":"11712","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330471.3720269,"name":"drain","context":{"idset":"11664","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330471.4671679,"name":"drain","context":{"idset":"11663","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330471.5633116,"name":"drain","context":{"idset":"11665","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330471.6667771,"name":"drain","context":{"idset":"11668","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330471.7855048,"name":"drain","context":{"idset":"11657","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330471.9092953,"name":"drain","context":{"idset":"11655","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330472.0534272,"name":"drain","context":{"idset":"11662","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330472.219491,"name":"drain","context":{"idset":"11654","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330472.3879061,"name":"drain","context":{"idset":"11660","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330472.5537159,"name":"drain","context":{"idset":"11674","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330472.7166924,"name":"drain","context":{"idset":"11656","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330472.8800154,"name":"drain","context":{"idset":"11667","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330473.038471,"name":"drain","context":{"idset":"11658","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330473.210041,"name":"drain","context":{"idset":"11659","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712330473.3722389,"name":"drain","context":{"idset":"11653","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330474.1245604,"name":"drain","context":{"idset":"11661","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330474.6423876,"name":"drain","context":{"idset":"799","reason":"broker was unresponsive"}}
+{"timestamp":1712330474.6481371,"name":"drain","context":{"idset":"800","reason":"broker was unresponsive"}}
+{"timestamp":1712330475.6167927,"name":"drain","context":{"idset":"11683","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330477.8852117,"name":"drain","context":{"idset":"11670","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330478.9431214,"name":"drain","context":{"idset":"11679","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330479.399539,"name":"drain","context":{"idset":"11669","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330479.8758326,"name":"drain","context":{"idset":"11684","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330480.2057967,"name":"drain","context":{"idset":"11680","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330481.2855182,"name":"drain","context":{"idset":"11678","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330481.4549022,"name":"drain","context":{"idset":"11671","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330481.8438857,"name":"drain","context":{"idset":"11676","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330482.1072097,"name":"drain","context":{"idset":"11675","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330482.4356873,"name":"drain","context":{"idset":"11672","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330484.216573,"name":"drain","context":{"idset":"11682","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330486.251406,"name":"drain","context":{"idset":"11681","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330488.75214,"name":"drain","context":{"idset":"11673","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330510.0196822,"name":"drain","context":{"idset":"11720","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330513.2824974,"name":"drain","context":{"idset":"11731","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330514.6673982,"name":"drain","context":{"idset":"11728","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330514.8791695,"name":"drain","context":{"idset":"11730","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330517.2515867,"name":"drain","context":{"idset":"11762","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1712330517.5905797,"name":"drain","context":{"idset":"11726","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330517.8098376,"name":"drain","context":{"idset":"11724","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330519.4148438,"name":"drain","context":{"idset":"11727","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330520.7425258,"name":"drain","context":{"idset":"11729","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330521.2568207,"name":"drain","context":{"idset":"11725","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330522.770314,"name":"drain","context":{"idset":"11718","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330523.247088,"name":"drain","context":{"idset":"11719","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330523.4613914,"name":"drain","context":{"idset":"11722","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330523.9044576,"name":"drain","context":{"idset":"11763","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712330524.3487394,"name":"drain","context":{"idset":"11721","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330525.2744215,"name":"drain","context":{"idset":"11753","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330525.6103771,"name":"drain","context":{"idset":"11723","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330525.7885973,"name":"drain","context":{"idset":"11717","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330527.6357992,"name":"drain","context":{"idset":"11749","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330529.0053041,"name":"drain","context":{"idset":"11761","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330529.6364772,"name":"drain","context":{"idset":"11758","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330530.4874673,"name":"drain","context":{"idset":"11751","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330531.2442417,"name":"drain","context":{"idset":"11752","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330531.9181197,"name":"drain","context":{"idset":"11764","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712330532.1245584,"name":"drain","context":{"idset":"11760","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330533.0370331,"name":"drain","context":{"idset":"11757","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330534.0661736,"name":"drain","context":{"idset":"11759","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330534.7081432,"name":"drain","context":{"idset":"11755","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712330539.3480027,"name":"offline","context":{"idset":"799"}}
+{"timestamp":1712330539.4394834,"name":"offline","context":{"idset":"800"}}
+{"timestamp":1712330683.3417573,"name":"drain","context":{"idset":"809","reason":"broker was unresponsive"}}
+{"timestamp":1712330683.341866,"name":"drain","context":{"idset":"810","reason":"broker was unresponsive"}}
+{"timestamp":1712330683.3419075,"name":"drain","context":{"idset":"811","reason":"broker was unresponsive"}}
+{"timestamp":1712330683.4417996,"name":"drain","context":{"idset":"812","reason":"broker was unresponsive"}}
+{"timestamp":1712330749.3510489,"name":"offline","context":{"idset":"809"}}
+{"timestamp":1712330749.366044,"name":"offline","context":{"idset":"811"}}
+{"timestamp":1712330749.4418502,"name":"offline","context":{"idset":"812"}}
+{"timestamp":1712330797.4384682,"name":"offline","context":{"idset":"810"}}
+{"timestamp":1712330914.8115845,"name":"undrain","context":{"idset":"1-60,85-120,122-216,218-347,349-430,432-444,469-540,565-588,629-636,869-870,951-952,9974-9981,9983-9994,9996-10004,10006,10008-10009,10011-10018,10020-10022,10024-10045,10048-10050,10052-10056,10058-10070,10072-10075,10077-10078,10080-10083,10085-10086,10088-10091,10093-10097,10099-10137,10139-10142,10144,10146,10149-10152,10155-10156,10158-10159,10161-10183,10185-10192,10194-10202,10204-10205,10208-10215,10217-10220,10222-10232,10234-10249,10251-10255,10257-10260,10262-10266,10268-10272,10274-10277,10279,10281-10300,10306-10355,10358-10373,10375-10399,10401-10420,10422-10431,10433-10435,10437-10443,10445-10449,10452-10455,10457-10465,10467-10468,10470-10473,10476-10477,10481,10483,10996-10998,11007,11127-11131,11136,11138,11223,11226,11231,11233,11235-11236,11239,11243-11244,11246-11247,11249,11616-11631,11633-11636,11877-11883,11885-11890"}}
+{"timestamp":1712331487.341608,"name":"drain","context":{"idset":"807","reason":"broker was unresponsive"}}
+{"timestamp":1712331487.4412344,"name":"drain","context":{"idset":"808","reason":"broker was unresponsive"}}
+{"timestamp":1712331551.3508463,"name":"offline","context":{"idset":"807"}}
+{"timestamp":1712331551.4415381,"name":"offline","context":{"idset":"808"}}
+{"timestamp":1712331685.0469942,"name":"undrain","context":{"idset":"10474,11653-11658,11660-11665,11667-11676,11678-11685,11687,11689,11695-11696,11698,11702-11704,11706-11707,11709-11712,11714-11715,11717-11731,11749,11751-11753,11755,11757-11761"}}
+{"timestamp":1712331720.2338269,"name":"undrain","context":{"idset":"9973,9982,9995,10005,10007,10010,10019,10023,10046-10047,10051,10057,10071,10076,10079,10084,10087,10092,10098,10206"}}
+{"timestamp":1712331833.0961521,"name":"drain","context":{"idset":"11713","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331841.4010427,"name":"drain","context":{"idset":"11704","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331842.5592225,"name":"drain","context":{"idset":"11703","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331845.1807196,"name":"drain","context":{"idset":"11709","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712331846.8321049,"name":"drain","context":{"idset":"11710","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331847.0185637,"name":"drain","context":{"idset":"11711","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331848.5531778,"name":"drain","context":{"idset":"11712","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331848.7271087,"name":"drain","context":{"idset":"11707","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331849.5751133,"name":"drain","context":{"idset":"11706","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331850.0461085,"name":"drain","context":{"idset":"11716","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331850.3809431,"name":"drain","context":{"idset":"11701","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331850.6001527,"name":"drain","context":{"idset":"11702","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331850.8486133,"name":"drain","context":{"idset":"11715","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331851.6838648,"name":"drain","context":{"idset":"11708","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331855.3141694,"name":"drain","context":{"idset":"11714","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331928.1370986,"name":"drain","context":{"idset":"9728","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712331928.8997169,"name":"drain","context":{"idset":"9722","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331930.8554306,"name":"drain","context":{"idset":"9720","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712331931.0427904,"name":"drain","context":{"idset":"9723","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331931.2539451,"name":"drain","context":{"idset":"9732","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331931.6041188,"name":"drain","context":{"idset":"9731","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331932.3308575,"name":"drain","context":{"idset":"9730","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331934.2667007,"name":"drain","context":{"idset":"9724","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331934.4973326,"name":"drain","context":{"idset":"9725","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331934.9728818,"name":"drain","context":{"idset":"9726","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331935.6806746,"name":"drain","context":{"idset":"9727","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331935.835525,"name":"drain","context":{"idset":"9717","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331939.5668306,"name":"drain","context":{"idset":"9729","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712331978.4851689,"name":"online","context":{"idset":"10261"}}
+{"timestamp":1712332042.7343211,"name":"undrain","context":{"idset":"10261"}}
+{"timestamp":1712332151.9771981,"name":"undrain","context":{"idset":"10301-10302"}}
+{"timestamp":1712332168.7181618,"name":"drain","context":{"idset":"10475","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712332288.4253299,"name":"online","context":{"idset":"10304"}}
+{"timestamp":1712332304.9575491,"name":"undrain","context":{"idset":"10304"}}
+{"timestamp":1712332609.2860804,"name":"undrain","context":{"idset":"11777-11778"}}
+{"timestamp":1712333532.4485443,"name":"drain","context":{"idset":"9741","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333532.8583488,"name":"drain","context":{"idset":"9745","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333534.8247879,"name":"drain","context":{"idset":"9739","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333535.1554551,"name":"drain","context":{"idset":"9743","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333535.4313176,"name":"drain","context":{"idset":"9733","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333536.402988,"name":"drain","context":{"idset":"9744","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333537.0177345,"name":"drain","context":{"idset":"9747","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333537.4922156,"name":"drain","context":{"idset":"9740","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333538.3697252,"name":"drain","context":{"idset":"9737","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333538.5243113,"name":"drain","context":{"idset":"9735","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333538.9680157,"name":"drain","context":{"idset":"9742","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333539.2254002,"name":"drain","context":{"idset":"9738","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712333540.4401367,"name":"drain","context":{"idset":"9748","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333540.5943189,"name":"drain","context":{"idset":"9736","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333540.822227,"name":"drain","context":{"idset":"9746","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333544.8455789,"name":"drain","context":{"idset":"9734","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712333755.1207211,"name":"drain","context":{"idset":"821-824","overwrite":0}}
+{"timestamp":1712334275.3562083,"name":"offline","context":{"idset":"821"}}
+{"timestamp":1712334275.3639162,"name":"offline","context":{"idset":"822"}}
+{"timestamp":1712334275.3775108,"name":"offline","context":{"idset":"823"}}
+{"timestamp":1712334276.1949179,"name":"offline","context":{"idset":"824"}}
+{"timestamp":1712334582.9395163,"name":"undrain","context":{"idset":"11008-11011,11013-11108,11253-11286,11288-11323,11335-11357,11359-11365,11377-11386,11430,11432-11438,11443-11454,11456-11463,11465,11467-11472,11486-11526,11528-11562,11564-11586,11589-11602"}}
+{"timestamp":1712334717.243984,"name":"offline","context":{"idset":"869"}}
+{"timestamp":1712334859.4442852,"name":"drain","context":{"idset":"870","reason":"broker was unresponsive"}}
+{"timestamp":1712334885.9243846,"name":"drain","context":{"idset":"10045","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712334886.7581542,"name":"drain","context":{"idset":"10065","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334888.3983226,"name":"drain","context":{"idset":"10037","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334889.0101347,"name":"drain","context":{"idset":"10068","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334889.1826782,"name":"drain","context":{"idset":"10060","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334889.8289771,"name":"drain","context":{"idset":"10067","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334889.9913764,"name":"drain","context":{"idset":"10047","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334890.3313305,"name":"drain","context":{"idset":"10054","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334890.5034156,"name":"drain","context":{"idset":"10050","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334890.7620723,"name":"drain","context":{"idset":"10064","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334890.9249234,"name":"drain","context":{"idset":"10063","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334891.1112378,"name":"drain","context":{"idset":"10056","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334891.2628505,"name":"drain","context":{"idset":"10039","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334891.4321783,"name":"drain","context":{"idset":"10058","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334891.9778779,"name":"drain","context":{"idset":"10062","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334892.1386166,"name":"drain","context":{"idset":"10066","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334892.2994289,"name":"drain","context":{"idset":"10059","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334892.8327014,"name":"drain","context":{"idset":"10046","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334893.3933988,"name":"drain","context":{"idset":"10049","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334893.6943209,"name":"drain","context":{"idset":"10053","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334894.0056541,"name":"drain","context":{"idset":"10043","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334894.337343,"name":"drain","context":{"idset":"10048","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334894.6886494,"name":"drain","context":{"idset":"10057","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334894.9729452,"name":"drain","context":{"idset":"10051","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334895.2472034,"name":"drain","context":{"idset":"10042","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712334895.4550674,"name":"drain","context":{"idset":"10055","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334895.6693161,"name":"drain","context":{"idset":"10044","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334896.8568437,"name":"drain","context":{"idset":"10052","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334897.8948488,"name":"drain","context":{"idset":"10040","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334898.1348019,"name":"drain","context":{"idset":"10041","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334899.5964868,"name":"drain","context":{"idset":"10038","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334901.1249568,"name":"drain","context":{"idset":"10061","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712334925.44172,"name":"offline","context":{"idset":"870"}}
+{"timestamp":1712336387.2644234,"name":"drain","context":{"idset":"9764","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712336391.6983895,"name":"drain","context":{"idset":"9753","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712336400.5601459,"name":"drain","context":{"idset":"9751","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712336401.8097148,"name":"drain","context":{"idset":"9749","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712336403.1911836,"name":"drain","context":{"idset":"9758","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712336407.5557129,"name":"drain","context":{"idset":"9759","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712336412.2280688,"name":"drain","context":{"idset":"9763","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712336412.383817,"name":"drain","context":{"idset":"9762","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712336417.8116803,"name":"drain","context":{"idset":"9755","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712336418.2480905,"name":"drain","context":{"idset":"9750","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1712336821.5876768,"name":"drain","context":{"idset":"11777","reason":"nodediag failed dgemm_perf amdapu","overwrite":0}}
+{"timestamp":1712336857.936064,"name":"drain","context":{"idset":"11778","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712337593.3448501,"name":"drain","context":{"idset":"803","reason":"broker was unresponsive"}}
+{"timestamp":1712337593.4446397,"name":"drain","context":{"idset":"804","reason":"broker was unresponsive"}}
+{"timestamp":1712337659.3500962,"name":"offline","context":{"idset":"803"}}
+{"timestamp":1712337659.4409359,"name":"offline","context":{"idset":"804"}}
+{"timestamp":1712337661.3698051,"name":"drain","context":{"idset":"9719","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712337664.7496204,"name":"drain","context":{"idset":"10355","reason":"bad partner node --JRG","overwrite":0}}
+{"timestamp":1712337675.8685074,"name":"drain","context":{"idset":"9721","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712337681.6244354,"name":"offline","context":{"idset":"10355"}}
+{"timestamp":1712337869.3428621,"name":"drain","context":{"idset":"10343","reason":"broker was unresponsive"}}
+{"timestamp":1712337869.4427822,"name":"drain","context":{"idset":"10344","reason":"broker was unresponsive"}}
+{"timestamp":1712337883.442421,"name":"drain","context":{"idset":"11329","reason":"broker was unresponsive"}}
+{"timestamp":1712337931.3499961,"name":"offline","context":{"idset":"10343"}}
+{"timestamp":1712337931.4402955,"name":"offline","context":{"idset":"10344"}}
+{"timestamp":1712337945.4408681,"name":"offline","context":{"idset":"11329"}}
+{"timestamp":1712338123.4430783,"name":"drain","context":{"idset":"11374","reason":"broker was unresponsive"}}
+{"timestamp":1712338139.3428166,"name":"drain","context":{"idset":"801","reason":"broker was unresponsive"}}
+{"timestamp":1712338139.445405,"name":"drain","context":{"idset":"802","reason":"broker was unresponsive"}}
+{"timestamp":1712338189.4411459,"name":"offline","context":{"idset":"11374"}}
+{"timestamp":1712338201.3508165,"name":"offline","context":{"idset":"801"}}
+{"timestamp":1712338201.4419987,"name":"offline","context":{"idset":"802"}}
+{"timestamp":1712338210.6618073,"name":"undrain","context":{"idset":"9717,9719,9721-9727,9729-9737,9739-9749,9751,9753,9755,9758-9759,9764,10037-10041,10043-10044,10046-10068,10475,11701-11704,11706-11708,11710-11716,11778"}}
+{"timestamp":1712338223.4112215,"name":"undrain","context":{"idset":"9720,9728,9738,9762-9763,10042,10045,11659,11691-11692,11709,11763-11764,11777"}}
+{"timestamp":1712338298.4587221,"name":"undrain","context":{"idset":"9535,10356,11115"}}
+{"timestamp":1712338675.3469379,"name":"drain","context":{"idset":"10155","reason":"broker was unresponsive"}}
+{"timestamp":1712338675.4476702,"name":"drain","context":{"idset":"10156","reason":"broker was unresponsive"}}
+{"timestamp":1712338741.3508189,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1712338741.4412036,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1712338902.6244915,"name":"undrain","context":{"idset":"10374"}}
+{"timestamp":1712340241.1384423,"name":"undrain","context":{"idset":"10469,10478-10480,10482,10484"}}
+{"timestamp":1712340966.3513074,"name":"drain","context":{"idset":"11221-11236","reason":"--reason draining to conserve power in x1900c6","overwrite":0}}
+{"timestamp":1712342217.7655268,"name":"drain","context":{"idset":"820","overwrite":0}}
+{"timestamp":1712342238.6917338,"name":"offline","context":{"idset":"820"}}
+{"timestamp":1712342249.3357341,"name":"offline","context":{"idset":"819"}}
+{"timestamp":1712342273.5977426,"name":"drain","context":{"idset":"819-820","overwrite":0}}
+{"timestamp":1712342982.9400101,"name":"drain","context":{"idset":"11778","reason":"--reason Performing diags - JE","overwrite":0}}
+{"timestamp":1712343808.3417928,"name":"undrain","context":{"idset":"11455"}}
+{"timestamp":1712343866.9651802,"name":"online","context":{"idset":"11455"}}
+{"timestamp":1712344729.5503497,"name":"online","context":{"idset":"9334"}}
+{"timestamp":1712344729.7065573,"name":"online","context":{"idset":"9333"}}
+{"timestamp":1712344729.8167853,"name":"online","context":{"idset":"9337-9338,9350"}}
+{"timestamp":1712344730.004986,"name":"online","context":{"idset":"9339"}}
+{"timestamp":1712344730.2093685,"name":"online","context":{"idset":"9341"}}
+{"timestamp":1712344730.2954679,"name":"online","context":{"idset":"9336,9346"}}
+{"timestamp":1712344730.4038751,"name":"online","context":{"idset":"9349,9355"}}
+{"timestamp":1712344730.5925763,"name":"online","context":{"idset":"9343"}}
+{"timestamp":1712344730.6971762,"name":"online","context":{"idset":"9345"}}
+{"timestamp":1712344730.7852316,"name":"online","context":{"idset":"9348"}}
+{"timestamp":1712344730.8095348,"name":"online","context":{"idset":"9375"}}
+{"timestamp":1712344730.9355907,"name":"online","context":{"idset":"9358"}}
+{"timestamp":1712344731.2168574,"name":"online","context":{"idset":"9353,9360,9374"}}
+{"timestamp":1712344731.4099231,"name":"online","context":{"idset":"9363"}}
+{"timestamp":1712344731.5766203,"name":"online","context":{"idset":"9335"}}
+{"timestamp":1712344731.6635666,"name":"online","context":{"idset":"9366,9403"}}
+{"timestamp":1712344731.7627754,"name":"online","context":{"idset":"9361,9368"}}
+{"timestamp":1712344731.8546062,"name":"online","context":{"idset":"9340,9352"}}
+{"timestamp":1712344731.964385,"name":"online","context":{"idset":"9344,9392"}}
+{"timestamp":1712344732.0953383,"name":"online","context":{"idset":"9342,9369,9396"}}
+{"timestamp":1712344732.3453135,"name":"online","context":{"idset":"9371,9376,9433"}}
+{"timestamp":1712344732.4761956,"name":"online","context":{"idset":"9367,9398"}}
+{"timestamp":1712344732.5922596,"name":"online","context":{"idset":"9362,9380-9381"}}
+{"timestamp":1712344732.7779799,"name":"online","context":{"idset":"9357,9391,9397"}}
+{"timestamp":1712344732.9606819,"name":"online","context":{"idset":"9347,9351,9359,9365,9383,9414-9415,9451"}}
+{"timestamp":1712344733.0818901,"name":"online","context":{"idset":"9354,9364,9377,9382,9408,9411,9420"}}
+{"timestamp":1712344733.1988671,"name":"online","context":{"idset":"9356,9370,9379,9384-9385,9389,9412,9454"}}
+{"timestamp":1712344733.3154221,"name":"online","context":{"idset":"9378,9428,9450"}}
+{"timestamp":1712344733.5136497,"name":"online","context":{"idset":"9372,9386,9399,9407,9416-9417,9419,9429-9430,9432,9438,9446"}}
+{"timestamp":1712344733.6930511,"name":"online","context":{"idset":"9394-9395,9410,9418,9424-9425,9445,9456"}}
+{"timestamp":1712344733.8722212,"name":"online","context":{"idset":"9373,9387-9388,9390,9393,9400-9402,9404,9406,9409,9413,9426-9427,9431,9434-9435,9437,9440-9441,9443"}}
+{"timestamp":1712344733.9883211,"name":"online","context":{"idset":"9405,9421-9422,9452,9457-9459"}}
+{"timestamp":1712344734.0905845,"name":"online","context":{"idset":"9423,9436,9442,9444,9449,9453,9460"}}
+{"timestamp":1712344734.3524308,"name":"online","context":{"idset":"9439,9455"}}
+{"timestamp":1712344995.7979279,"name":"online","context":{"idset":"9591"}}
+{"timestamp":1712344996.0413911,"name":"online","context":{"idset":"9599"}}
+{"timestamp":1712344996.1480196,"name":"online","context":{"idset":"9607"}}
+{"timestamp":1712344996.6354072,"name":"online","context":{"idset":"9597"}}
+{"timestamp":1712344996.7356191,"name":"online","context":{"idset":"9610"}}
+{"timestamp":1712344996.8625066,"name":"online","context":{"idset":"9595"}}
+{"timestamp":1712344997.0343893,"name":"online","context":{"idset":"9593"}}
+{"timestamp":1712344997.2703159,"name":"online","context":{"idset":"9594"}}
+{"timestamp":1712344997.6141977,"name":"online","context":{"idset":"9596"}}
+{"timestamp":1712344997.7381222,"name":"online","context":{"idset":"9602"}}
+{"timestamp":1712344997.7675738,"name":"online","context":{"idset":"9592"}}
+{"timestamp":1712344997.8779325,"name":"online","context":{"idset":"9605"}}
+{"timestamp":1712344998.0507746,"name":"online","context":{"idset":"9621"}}
+{"timestamp":1712344998.2270844,"name":"online","context":{"idset":"9615"}}
+{"timestamp":1712344998.5295064,"name":"online","context":{"idset":"9598,9601,9606,9622-9623,9630,9639"}}
+{"timestamp":1712344998.5472522,"name":"online","context":{"idset":"9629"}}
+{"timestamp":1712344998.7505949,"name":"online","context":{"idset":"9604,9617,9625"}}
+{"timestamp":1712344998.9514651,"name":"online","context":{"idset":"9589,9616"}}
+{"timestamp":1712344999.0706229,"name":"online","context":{"idset":"9590,9600,9609,9611,9619-9620,9646"}}
+{"timestamp":1712344999.2720542,"name":"online","context":{"idset":"9603,9608,9612-9613,9618,9624,9631-9632,9635-9637,9641,9645,9649,9655,9664,9672,9696"}}
+{"timestamp":1712344999.5099797,"name":"online","context":{"idset":"9638,9640,9653,9657,9671,9676,9681"}}
+{"timestamp":1712344999.6342962,"name":"online","context":{"idset":"9614,9628,9660,9669,9694"}}
+{"timestamp":1712344999.8229871,"name":"online","context":{"idset":"9642,9644,9650-9652,9654,9663,9665-9666,9674,9684-9685,9691,9706,9711,9714"}}
+{"timestamp":1712344999.9409597,"name":"online","context":{"idset":"9647,9662,9680,9682-9683,9689,9693,9698"}}
+{"timestamp":1712345000.0595229,"name":"online","context":{"idset":"9626-9627,9648,9656,9659,9661,9667,9670,9675,9678,9695,9697,9700,9710"}}
+{"timestamp":1712345000.1780705,"name":"online","context":{"idset":"9643,9658,9673,9677,9686,9690,9702,9712-9713,9715"}}
+{"timestamp":1712345000.2969575,"name":"online","context":{"idset":"9679,9687,9692,9699,9701,9704,9707-9708"}}
+{"timestamp":1712345001.1376417,"name":"online","context":{"idset":"9688,9705"}}
+{"timestamp":1712345001.1403699,"name":"online","context":{"idset":"9668,9709"}}
+{"timestamp":1712345001.143085,"name":"online","context":{"idset":"9703"}}
+{"timestamp":1712345001.1458139,"name":"online","context":{"idset":"9716"}}
+{"timestamp":1712345226.7601151,"name":"drain","context":{"idset":"11660","reason":"nodediag failed babelstream","overwrite":0}}
+{"timestamp":1712346509.3631339,"name":"undrain","context":{"idset":"11220-11236"}}
+{"timestamp":1712347112.6973126,"name":"undrain","context":{"idset":"10138,10143,10145,10147-10148,10154-10156"}}
+{"timestamp":1712347359.3439841,"name":"drain","context":{"idset":"11226","reason":"broker was unresponsive"}}
+{"timestamp":1712347359.3441069,"name":"drain","context":{"idset":"11233","reason":"broker was unresponsive"}}
+{"timestamp":1712347359.3441837,"name":"drain","context":{"idset":"11235","reason":"broker was unresponsive"}}
+{"timestamp":1712347360.1209552,"name":"drain","context":{"idset":"11236","reason":"broker was unresponsive"}}
+{"timestamp":1712347363.3437819,"name":"drain","context":{"idset":"11247","reason":"broker was unresponsive"}}
+{"timestamp":1712347364.1011047,"name":"drain","context":{"idset":"11249","reason":"broker was unresponsive"}}
+{"timestamp":1712347425.3733168,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1712347425.3968487,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1712347425.4187725,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1712347425.4390528,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1712347426.1536522,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1712347426.1713021,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1712347508.0716112,"name":"drain","context":{"idset":"10944","reason":"broker was unresponsive"}}
+{"timestamp":1712347570.0595174,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1712347934.0538592,"name":"drain","context":{"idset":"10872","reason":"broker was unresponsive"}}
+{"timestamp":1712348000.0581076,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1712348147.3444083,"name":"drain","context":{"idset":"813","reason":"broker was unresponsive"}}
+{"timestamp":1712348148.0782108,"name":"drain","context":{"idset":"814","reason":"broker was unresponsive"}}
+{"timestamp":1712348167.1690047,"name":"undrain","context":{"idset":"11778"}}
+{"timestamp":1712348208.9644253,"name":"offline","context":{"idset":"813"}}
+{"timestamp":1712348209.0541351,"name":"offline","context":{"idset":"814"}}
+{"timestamp":1712348261.3433502,"name":"drain","context":{"idset":"805","reason":"broker was unresponsive"}}
+{"timestamp":1712348262.1058218,"name":"drain","context":{"idset":"806","reason":"broker was unresponsive"}}
+{"timestamp":1712348303.343436,"name":"drain","context":{"idset":"825","reason":"broker was unresponsive"}}
+{"timestamp":1712348304.0895774,"name":"drain","context":{"idset":"826","reason":"broker was unresponsive"}}
+{"timestamp":1712348325.3643086,"name":"offline","context":{"idset":"805"}}
+{"timestamp":1712348326.2745457,"name":"offline","context":{"idset":"806"}}
+{"timestamp":1712348367.3559361,"name":"offline","context":{"idset":"825"}}
+{"timestamp":1712348368.2287347,"name":"offline","context":{"idset":"826"}}
+{"timestamp":1712349359.343132,"name":"drain","context":{"idset":"815","reason":"broker was unresponsive"}}
+{"timestamp":1712349360.1984277,"name":"drain","context":{"idset":"816","reason":"broker was unresponsive"}}
+{"timestamp":1712349423.3529902,"name":"offline","context":{"idset":"815"}}
+{"timestamp":1712349424.2433324,"name":"offline","context":{"idset":"816"}}
+{"timestamp":1712349640.237411,"name":"drain","context":{"idset":"11778","reason":"broker was unresponsive"}}
+{"timestamp":1712349643.3440766,"name":"drain","context":{"idset":"11777","reason":"broker was unresponsive"}}
+{"timestamp":1712349699.3537469,"name":"offline","context":{"idset":"11765"}}
+{"timestamp":1712349699.3727901,"name":"offline","context":{"idset":"11766"}}
+{"timestamp":1712349700.1466262,"name":"offline","context":{"idset":"11767"}}
+{"timestamp":1712349701.3574419,"name":"offline","context":{"idset":"11768"}}
+{"timestamp":1712349701.3760204,"name":"offline","context":{"idset":"11769"}}
+{"timestamp":1712349701.3935306,"name":"offline","context":{"idset":"11770"}}
+{"timestamp":1712349702.1266272,"name":"offline","context":{"idset":"11771"}}
+{"timestamp":1712349702.7938135,"name":"offline","context":{"idset":"11772"}}
+{"timestamp":1712349703.0040543,"name":"offline","context":{"idset":"11773"}}
+{"timestamp":1712349703.0207167,"name":"offline","context":{"idset":"11774"}}
+{"timestamp":1712349703.0364203,"name":"offline","context":{"idset":"11775"}}
+{"timestamp":1712349705.3603942,"name":"offline","context":{"idset":"11776"}}
+{"timestamp":1712349705.3797336,"name":"offline","context":{"idset":"11777"}}
+{"timestamp":1712349705.398062,"name":"offline","context":{"idset":"11778"}}
+{"timestamp":1712349706.1505899,"name":"offline","context":{"idset":"11779"}}
+{"timestamp":1712349706.1534696,"name":"offline","context":{"idset":"11780"}}
+{"timestamp":1712349932.2236493,"name":"offline","context":{"idset":"11845"}}
+{"timestamp":1712349932.2664721,"name":"offline","context":{"idset":"11846"}}
+{"timestamp":1712349932.3021386,"name":"offline","context":{"idset":"11847"}}
+{"timestamp":1712349932.3056436,"name":"offline","context":{"idset":"11848"}}
+{"timestamp":1712349932.37639,"name":"offline","context":{"idset":"11849"}}
+{"timestamp":1712349935.3573232,"name":"offline","context":{"idset":"11850"}}
+{"timestamp":1712349935.3738444,"name":"offline","context":{"idset":"11851"}}
+{"timestamp":1712349936.2458801,"name":"offline","context":{"idset":"11852"}}
+{"timestamp":1712349936.2732344,"name":"offline","context":{"idset":"11853"}}
+{"timestamp":1712349937.3631778,"name":"offline","context":{"idset":"11854"}}
+{"timestamp":1712349937.3801575,"name":"offline","context":{"idset":"11855"}}
+{"timestamp":1712349937.3977184,"name":"offline","context":{"idset":"11856"}}
+{"timestamp":1712349937.4131792,"name":"offline","context":{"idset":"11857"}}
+{"timestamp":1712349938.2830276,"name":"offline","context":{"idset":"11858"}}
+{"timestamp":1712349938.2863877,"name":"offline","context":{"idset":"11859"}}
+{"timestamp":1712349938.4050756,"name":"offline","context":{"idset":"11860"}}
+{"timestamp":1712350320.9205794,"name":"online","context":{"idset":"9208"}}
+{"timestamp":1712350321.2942111,"name":"online","context":{"idset":"9215"}}
+{"timestamp":1712350322.2598066,"name":"online","context":{"idset":"9213"}}
+{"timestamp":1712350322.2631061,"name":"online","context":{"idset":"9206"}}
+{"timestamp":1712350322.2662821,"name":"online","context":{"idset":"9211,9242"}}
+{"timestamp":1712350322.2693951,"name":"online","context":{"idset":"9233"}}
+{"timestamp":1712350322.272613,"name":"online","context":{"idset":"9207,9216,9221,9223"}}
+{"timestamp":1712350322.2758315,"name":"online","context":{"idset":"9212"}}
+{"timestamp":1712350322.2788033,"name":"online","context":{"idset":"9225"}}
+{"timestamp":1712350322.2820935,"name":"online","context":{"idset":"9209"}}
+{"timestamp":1712350322.2853124,"name":"online","context":{"idset":"9217-9218,9230,9232"}}
+{"timestamp":1712350322.421977,"name":"online","context":{"idset":"9210"}}
+{"timestamp":1712350322.5351484,"name":"online","context":{"idset":"9205,9219,9224,9245,9248"}}
+{"timestamp":1712350322.7736397,"name":"online","context":{"idset":"9235"}}
+{"timestamp":1712350322.8915823,"name":"online","context":{"idset":"9234,9254"}}
+{"timestamp":1712350322.9341249,"name":"online","context":{"idset":"9251-9252"}}
+{"timestamp":1712350323.0723493,"name":"online","context":{"idset":"9250,9255,9257"}}
+{"timestamp":1712350323.1914721,"name":"online","context":{"idset":"9241"}}
+{"timestamp":1712350323.3214281,"name":"online","context":{"idset":"9229,9281"}}
+{"timestamp":1712350324.269495,"name":"online","context":{"idset":"9220"}}
+{"timestamp":1712350324.2736778,"name":"online","context":{"idset":"9249,9277"}}
+{"timestamp":1712350324.2772243,"name":"online","context":{"idset":"9222,9259,9269"}}
+{"timestamp":1712350324.2808492,"name":"online","context":{"idset":"9263,9279"}}
+{"timestamp":1712350324.284961,"name":"online","context":{"idset":"9226-9227,9273,9292,9295,9309"}}
+{"timestamp":1712350324.3895328,"name":"online","context":{"idset":"9297,9302"}}
+{"timestamp":1712350324.5978985,"name":"online","context":{"idset":"9247,9268,9323"}}
+{"timestamp":1712350324.7294803,"name":"online","context":{"idset":"9244"}}
+{"timestamp":1712350324.9594982,"name":"online","context":{"idset":"9236-9237,9243,9246,9253,9256,9260-9261,9264-9265,9267,9272,9275,9282,9287,9289-9290,9294,9296,9299,9304,9306,9313,9320,9322,9325,9329"}}
+{"timestamp":1712350325.0796959,"name":"online","context":{"idset":"9258,9284,9293,9301,9314-9315,9321"}}
+{"timestamp":1712350325.2046845,"name":"online","context":{"idset":"9228,9231,9238,9283,9288,9291,9307,9310,9324,9328,9332"}}
+{"timestamp":1712350325.3335605,"name":"online","context":{"idset":"9262,9271,9285,9312,9317,9319,9326"}}
+{"timestamp":1712350326.2560074,"name":"online","context":{"idset":"9270,9298,9300,9303,9305,9308,9311,9327,9331"}}
+{"timestamp":1712350326.2593713,"name":"online","context":{"idset":"9274"}}
+{"timestamp":1712350326.2627425,"name":"online","context":{"idset":"9266,9286,9330"}}
+{"timestamp":1712350326.2660842,"name":"online","context":{"idset":"9276,9278,9280"}}
+{"timestamp":1712350482.0513477,"name":"offline","context":{"idset":"11861"}}
+{"timestamp":1712350483.3737693,"name":"offline","context":{"idset":"11862"}}
+{"timestamp":1712350483.4083564,"name":"offline","context":{"idset":"11863"}}
+{"timestamp":1712350483.4444695,"name":"offline","context":{"idset":"11864"}}
+{"timestamp":1712350485.3531675,"name":"offline","context":{"idset":"11865"}}
+{"timestamp":1712350485.3689158,"name":"offline","context":{"idset":"11866"}}
+{"timestamp":1712350486.1104486,"name":"offline","context":{"idset":"11867"}}
+{"timestamp":1712350487.355608,"name":"offline","context":{"idset":"11868"}}
+{"timestamp":1712350487.3701231,"name":"offline","context":{"idset":"11869"}}
+{"timestamp":1712350487.3939798,"name":"offline","context":{"idset":"11870"}}
+{"timestamp":1712350488.1143601,"name":"offline","context":{"idset":"11871"}}
+{"timestamp":1712350489.3620279,"name":"offline","context":{"idset":"11872"}}
+{"timestamp":1712350489.3811157,"name":"offline","context":{"idset":"11873"}}
+{"timestamp":1712350489.4098973,"name":"offline","context":{"idset":"11874"}}
+{"timestamp":1712350490.1483457,"name":"offline","context":{"idset":"11875"}}
+{"timestamp":1712350490.1511648,"name":"offline","context":{"idset":"11876"}}
+{"timestamp":1712350524.2175126,"name":"drain","context":{"idset":"9214","reason":"CXI Starting --JRG","overwrite":0}}
+{"timestamp":1712350549.0126185,"name":"drain","context":{"idset":"9316","reason":"Fails loopback test --JRG","overwrite":0}}
+{"timestamp":1712350566.2323,"name":"drain","context":{"idset":"9318","reason":"Fails bogomips --JRG","overwrite":0}}
+{"timestamp":1712351272.2043664,"name":"undrain","context":{"idset":"9559"}}
+{"timestamp":1712352285.3513789,"name":"offline","context":{"idset":"11829"}}
+{"timestamp":1712352286.0757849,"name":"offline","context":{"idset":"11830"}}
+{"timestamp":1712352286.1124725,"name":"offline","context":{"idset":"11831"}}
+{"timestamp":1712352286.136554,"name":"offline","context":{"idset":"11832"}}
+{"timestamp":1712352286.1397705,"name":"offline","context":{"idset":"11833"}}
+{"timestamp":1712352286.1584046,"name":"offline","context":{"idset":"11834"}}
+{"timestamp":1712352286.2202332,"name":"offline","context":{"idset":"11835"}}
+{"timestamp":1712352289.351615,"name":"offline","context":{"idset":"11836"}}
+{"timestamp":1712352289.3643899,"name":"offline","context":{"idset":"11837"}}
+{"timestamp":1712352290.1066859,"name":"offline","context":{"idset":"11838"}}
+{"timestamp":1712352290.1456823,"name":"offline","context":{"idset":"11840"}}
+{"timestamp":1712352290.1771655,"name":"offline","context":{"idset":"11842"}}
+{"timestamp":1712352290.1840348,"name":"offline","context":{"idset":"11843"}}
+{"timestamp":1712352290.2478521,"name":"offline","context":{"idset":"11844"}}
+{"timestamp":1712352395.3461986,"name":"drain","context":{"idset":"835","reason":"broker was unresponsive"}}
+{"timestamp":1712352396.0807211,"name":"drain","context":{"idset":"836","reason":"broker was unresponsive"}}
+{"timestamp":1712352438.0526481,"name":"drain","context":{"idset":"9726","reason":"broker was unresponsive"}}
+{"timestamp":1712352442.0729523,"name":"drain","context":{"idset":"9725","reason":"broker was unresponsive"}}
+{"timestamp":1712352454.7358615,"name":"undrain","context":{"idset":"11637"}}
+{"timestamp":1712352461.3526495,"name":"offline","context":{"idset":"835"}}
+{"timestamp":1712352462.0893373,"name":"offline","context":{"idset":"836"}}
+{"timestamp":1712352500.2063875,"name":"undrain","context":{"idset":"11638-11652"}}
+{"timestamp":1712352503.3508506,"name":"offline","context":{"idset":"9725"}}
+{"timestamp":1712352504.107228,"name":"offline","context":{"idset":"9726"}}
+{"timestamp":1712352556.8019292,"name":"undrain","context":{"idset":"11660,11686,11762"}}
+{"timestamp":1712352882.0445571,"name":"drain","context":{"idset":"11694","reason":"broker was unresponsive"}}
+{"timestamp":1712352944.0679147,"name":"offline","context":{"idset":"11694"}}
+{"timestamp":1712353656.0801761,"name":"online","context":{"idset":"10147,10154"}}
+{"timestamp":1712353656.7871544,"name":"online","context":{"idset":"10148"}}
+{"timestamp":1712353656.97894,"name":"online","context":{"idset":"10153"}}
+{"timestamp":1712353686.2502072,"name":"drain","context":{"idset":"10147-10148,10153-10154","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1712354720.5063212,"name":"online","context":{"idset":"11125"}}
+{"timestamp":1712354721.2891684,"name":"online","context":{"idset":"11126"}}
+{"timestamp":1712355439.0755372,"name":"undrain","context":{"idset":"10147-10148,10153-10154"}}
+{"timestamp":1712355823.3308198,"name":"offline","context":{"idset":"846"}}
+{"timestamp":1712355824.0859051,"name":"offline","context":{"idset":"845"}}
+{"timestamp":1712355841.3457093,"name":"drain","context":{"idset":"10471","reason":"broker was unresponsive"}}
+{"timestamp":1712355842.2199931,"name":"drain","context":{"idset":"10473","reason":"broker was unresponsive"}}
+{"timestamp":1712355843.3485792,"name":"drain","context":{"idset":"10469","reason":"broker was unresponsive"}}
+{"timestamp":1712355843.3487108,"name":"drain","context":{"idset":"10470","reason":"broker was unresponsive"}}
+{"timestamp":1712355843.3487971,"name":"drain","context":{"idset":"10472","reason":"broker was unresponsive"}}
+{"timestamp":1712355843.348918,"name":"drain","context":{"idset":"10475","reason":"broker was unresponsive"}}
+{"timestamp":1712355844.270498,"name":"drain","context":{"idset":"10476","reason":"broker was unresponsive"}}
+{"timestamp":1712355845.345845,"name":"drain","context":{"idset":"10477","reason":"broker was unresponsive"}}
+{"timestamp":1712355846.0879886,"name":"drain","context":{"idset":"10478","reason":"broker was unresponsive"}}
+{"timestamp":1712355847.3450718,"name":"drain","context":{"idset":"10474","reason":"broker was unresponsive"}}
+{"timestamp":1712355847.3451836,"name":"drain","context":{"idset":"10479","reason":"broker was unresponsive"}}
+{"timestamp":1712355847.3452773,"name":"drain","context":{"idset":"10481","reason":"broker was unresponsive"}}
+{"timestamp":1712355848.0988836,"name":"drain","context":{"idset":"10484","reason":"broker was unresponsive"}}
+{"timestamp":1712355850.0757811,"name":"drain","context":{"idset":"10480","reason":"broker was unresponsive"}}
+{"timestamp":1712355851.3449905,"name":"drain","context":{"idset":"10482","reason":"broker was unresponsive"}}
+{"timestamp":1712355852.1232853,"name":"drain","context":{"idset":"10483","reason":"broker was unresponsive"}}
+{"timestamp":1712355906.0657945,"name":"offline","context":{"idset":"10469"}}
+{"timestamp":1712355907.3617601,"name":"offline","context":{"idset":"10470"}}
+{"timestamp":1712355907.3818667,"name":"offline","context":{"idset":"10471"}}
+{"timestamp":1712355907.4000776,"name":"offline","context":{"idset":"10472"}}
+{"timestamp":1712355908.1420999,"name":"offline","context":{"idset":"10473"}}
+{"timestamp":1712355909.3627994,"name":"offline","context":{"idset":"10474"}}
+{"timestamp":1712355909.3818831,"name":"offline","context":{"idset":"10475"}}
+{"timestamp":1712355909.3997612,"name":"offline","context":{"idset":"10476"}}
+{"timestamp":1712355910.0945828,"name":"offline","context":{"idset":"10477"}}
+{"timestamp":1712355910.0997403,"name":"offline","context":{"idset":"10478"}}
+{"timestamp":1712355910.1382136,"name":"offline","context":{"idset":"10479"}}
+{"timestamp":1712355910.1675329,"name":"offline","context":{"idset":"10480"}}
+{"timestamp":1712355910.221509,"name":"offline","context":{"idset":"10481"}}
+{"timestamp":1712355912.0473509,"name":"offline","context":{"idset":"10482"}}
+{"timestamp":1712355912.0900669,"name":"offline","context":{"idset":"10483"}}
+{"timestamp":1712355912.1687713,"name":"offline","context":{"idset":"10484"}}
+{"timestamp":1712356196.7819562,"name":"undrain","context":{"idset":"11125-11126"}}
+{"timestamp":1712358118.4097719,"name":"undrain","context":{"idset":"11247,11249"}}
+{"timestamp":1712358136.008389,"name":"undrain","context":{"idset":"11226,11233,11235-11236"}}
+{"timestamp":1712361648.1654882,"name":"drain","context":{"idset":"10261","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712365572.2220874,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2286646,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.228714,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2306376,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2307081,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2307422,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2307882,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2308214,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2308669,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2309122,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2309432,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2309718,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2310019,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2310357,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2310669,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2311132,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2311509,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2311814,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2312181,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2312882,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2313271,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2313788,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2314267,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2314601,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2314937,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2315323,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2315834,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.231638,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2316718,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2317057,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2317371,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2317684,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2318003,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.231853,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2318985,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2319705,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2320211,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2320583,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2320967,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2321327,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2321801,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2322273,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2322598,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.232311,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2323503,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2323849,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2324324,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2324872,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2325249,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2325625,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2325978,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2326546,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.232717,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2327638,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2328057,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.232842,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2328789,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2329352,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2329841,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2330277,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.233068,"name":"drain","context":{"idset":"85","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2331095,"name":"drain","context":{"idset":"86","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2332046,"name":"drain","context":{"idset":"87","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2332504,"name":"drain","context":{"idset":"88","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2333148,"name":"drain","context":{"idset":"89","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2333574,"name":"drain","context":{"idset":"90","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.233413,"name":"drain","context":{"idset":"91","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2334795,"name":"drain","context":{"idset":"92","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2335336,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.233588,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2336364,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2337134,"name":"drain","context":{"idset":"96","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.233762,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.233824,"name":"drain","context":{"idset":"98","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2338965,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2339897,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2340486,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2341108,"name":"drain","context":{"idset":"102","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2341814,"name":"drain","context":{"idset":"103","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2342274,"name":"drain","context":{"idset":"104","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2342942,"name":"drain","context":{"idset":"105","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2343431,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2344041,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2344565,"name":"drain","context":{"idset":"108","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2345219,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2345781,"name":"drain","context":{"idset":"110","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2346251,"name":"drain","context":{"idset":"111","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2346919,"name":"drain","context":{"idset":"112","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2347553,"name":"drain","context":{"idset":"113","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2348204,"name":"drain","context":{"idset":"114","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.234868,"name":"drain","context":{"idset":"115","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2349133,"name":"drain","context":{"idset":"116","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2350121,"name":"drain","context":{"idset":"117","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2350743,"name":"drain","context":{"idset":"118","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2351217,"name":"drain","context":{"idset":"119","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2351696,"name":"drain","context":{"idset":"120","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2352524,"name":"drain","context":{"idset":"122","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2353327,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2353818,"name":"drain","context":{"idset":"124","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.23543,"name":"drain","context":{"idset":"125","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2354929,"name":"drain","context":{"idset":"126","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2355638,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.235621,"name":"drain","context":{"idset":"128","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2356713,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2357309,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2358205,"name":"drain","context":{"idset":"131","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2358782,"name":"drain","context":{"idset":"132","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2359297,"name":"drain","context":{"idset":"133","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2360036,"name":"drain","context":{"idset":"134","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2360642,"name":"drain","context":{"idset":"135","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2361147,"name":"drain","context":{"idset":"136","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2361672,"name":"drain","context":{"idset":"137","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2362382,"name":"drain","context":{"idset":"138","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2363441,"name":"drain","context":{"idset":"139","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2364006,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2364745,"name":"drain","context":{"idset":"141","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2365518,"name":"drain","context":{"idset":"142","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2366059,"name":"drain","context":{"idset":"143","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.23666,"name":"drain","context":{"idset":"144","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.236732,"name":"drain","context":{"idset":"145","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2367985,"name":"drain","context":{"idset":"146","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2368526,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.236917,"name":"drain","context":{"idset":"148","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2369714,"name":"drain","context":{"idset":"149","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2370315,"name":"drain","context":{"idset":"150","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.237123,"name":"drain","context":{"idset":"151","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2371862,"name":"drain","context":{"idset":"152","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2372417,"name":"drain","context":{"idset":"153","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2373435,"name":"drain","context":{"idset":"154","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2374115,"name":"drain","context":{"idset":"155","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2374861,"name":"drain","context":{"idset":"156","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2375619,"name":"drain","context":{"idset":"157","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2376416,"name":"drain","context":{"idset":"158","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2377126,"name":"drain","context":{"idset":"159","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.237788,"name":"drain","context":{"idset":"160","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2379014,"name":"drain","context":{"idset":"161","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2379725,"name":"drain","context":{"idset":"162","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2380474,"name":"drain","context":{"idset":"163","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2381475,"name":"drain","context":{"idset":"164","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2382314,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2383361,"name":"drain","context":{"idset":"166","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2384217,"name":"drain","context":{"idset":"167","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2384849,"name":"drain","context":{"idset":"168","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.238564,"name":"drain","context":{"idset":"169","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2386551,"name":"drain","context":{"idset":"170","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2387521,"name":"drain","context":{"idset":"171","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2388451,"name":"drain","context":{"idset":"172","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2389157,"name":"drain","context":{"idset":"173","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2389863,"name":"drain","context":{"idset":"174","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2390757,"name":"drain","context":{"idset":"175","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2391434,"name":"drain","context":{"idset":"176","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2392077,"name":"drain","context":{"idset":"177","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2393053,"name":"drain","context":{"idset":"178","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2394068,"name":"drain","context":{"idset":"179","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2394729,"name":"drain","context":{"idset":"180","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.239537,"name":"drain","context":{"idset":"181","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2396276,"name":"drain","context":{"idset":"182","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2397325,"name":"drain","context":{"idset":"183","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2398086,"name":"drain","context":{"idset":"184","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2399015,"name":"drain","context":{"idset":"185","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2399721,"name":"drain","context":{"idset":"186","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2400663,"name":"drain","context":{"idset":"187","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2401721,"name":"drain","context":{"idset":"188","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.240247,"name":"drain","context":{"idset":"189","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2403345,"name":"drain","context":{"idset":"190","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.240447,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2405415,"name":"drain","context":{"idset":"192","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2406516,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2407241,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2407918,"name":"drain","context":{"idset":"195","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2408812,"name":"drain","context":{"idset":"196","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.240957,"name":"drain","context":{"idset":"197","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2410438,"name":"drain","context":{"idset":"198","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2411568,"name":"drain","context":{"idset":"199","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2412353,"name":"drain","context":{"idset":"200","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2413495,"name":"drain","context":{"idset":"201","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2414603,"name":"drain","context":{"idset":"202","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2415369,"name":"drain","context":{"idset":"203","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2416494,"name":"drain","context":{"idset":"204","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.241739,"name":"drain","context":{"idset":"205","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2418363,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2419224,"name":"drain","context":{"idset":"207","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.241993,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2420797,"name":"drain","context":{"idset":"209","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2421513,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2422457,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2423363,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2424362,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2425499,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2426443,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2427473,"name":"drain","context":{"idset":"216","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2428365,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2429252,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2430108,"name":"drain","context":{"idset":"220","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2430842,"name":"drain","context":{"idset":"221","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2431943,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2433102,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2433951,"name":"drain","context":{"idset":"224","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.243521,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2436035,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2437198,"name":"drain","context":{"idset":"227","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2438271,"name":"drain","context":{"idset":"228","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2439458,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2440476,"name":"drain","context":{"idset":"230","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2441368,"name":"drain","context":{"idset":"231","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2442374,"name":"drain","context":{"idset":"232","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2443538,"name":"drain","context":{"idset":"233","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2445238,"name":"drain","context":{"idset":"234","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2446132,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2447176,"name":"drain","context":{"idset":"236","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2448299,"name":"drain","context":{"idset":"237","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2449131,"name":"drain","context":{"idset":"238","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2450552,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2451694,"name":"drain","context":{"idset":"240","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2453053,"name":"drain","context":{"idset":"241","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2454159,"name":"drain","context":{"idset":"242","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2455254,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2456093,"name":"drain","context":{"idset":"244","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2456954,"name":"drain","context":{"idset":"245","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2458007,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2458837,"name":"drain","context":{"idset":"247","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2459664,"name":"drain","context":{"idset":"248","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2460852,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.246201,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2463498,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2464375,"name":"drain","context":{"idset":"252","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2465682,"name":"drain","context":{"idset":"253","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2466788,"name":"drain","context":{"idset":"254","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2468174,"name":"drain","context":{"idset":"255","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.246907,"name":"drain","context":{"idset":"256","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2470167,"name":"drain","context":{"idset":"257","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2471035,"name":"drain","context":{"idset":"258","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2471898,"name":"drain","context":{"idset":"259","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2473016,"name":"drain","context":{"idset":"260","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2474208,"name":"drain","context":{"idset":"261","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.247545,"name":"drain","context":{"idset":"262","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2476625,"name":"drain","context":{"idset":"263","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2477677,"name":"drain","context":{"idset":"264","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2479048,"name":"drain","context":{"idset":"265","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2480068,"name":"drain","context":{"idset":"266","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2481599,"name":"drain","context":{"idset":"267","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2482576,"name":"drain","context":{"idset":"268","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2483904,"name":"drain","context":{"idset":"269","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2484899,"name":"drain","context":{"idset":"270","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2486119,"name":"drain","context":{"idset":"271","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2487044,"name":"drain","context":{"idset":"272","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2488339,"name":"drain","context":{"idset":"273","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2489343,"name":"drain","context":{"idset":"274","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2491014,"name":"drain","context":{"idset":"275","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2492063,"name":"drain","context":{"idset":"276","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2493408,"name":"drain","context":{"idset":"277","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2494519,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2495797,"name":"drain","context":{"idset":"279","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2496936,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2497976,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2499115,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2500079,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2501473,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2503216,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2504642,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2505872,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2507181,"name":"drain","context":{"idset":"288","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2509167,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2510245,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2511661,"name":"drain","context":{"idset":"291","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2513063,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2514446,"name":"drain","context":{"idset":"293","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2515419,"name":"drain","context":{"idset":"294","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2516575,"name":"drain","context":{"idset":"295","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2517707,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2519009,"name":"drain","context":{"idset":"297","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2519977,"name":"drain","context":{"idset":"298","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2520924,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2522221,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2523494,"name":"drain","context":{"idset":"301","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2524884,"name":"drain","context":{"idset":"302","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2525928,"name":"drain","context":{"idset":"303","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2527246,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2528481,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2529981,"name":"drain","context":{"idset":"306","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2531021,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2532146,"name":"drain","context":{"idset":"308","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2533586,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2534888,"name":"drain","context":{"idset":"310","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2535899,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2537384,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.253876,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2540071,"name":"drain","context":{"idset":"314","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2541103,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2542439,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2543879,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2545335,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2546411,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2547669,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2548714,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2549934,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.255172,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2553525,"name":"drain","context":{"idset":"324","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2554677,"name":"drain","context":{"idset":"325","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.255615,"name":"drain","context":{"idset":"326","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2557478,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2559001,"name":"drain","context":{"idset":"328","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2560575,"name":"drain","context":{"idset":"329","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2561874,"name":"drain","context":{"idset":"330","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2563679,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2564769,"name":"drain","context":{"idset":"332","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2566125,"name":"drain","context":{"idset":"333","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2567232,"name":"drain","context":{"idset":"334","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2568758,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2569954,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2571297,"name":"drain","context":{"idset":"337","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2572548,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.257385,"name":"drain","context":{"idset":"339","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2574966,"name":"drain","context":{"idset":"340","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2576525,"name":"drain","context":{"idset":"341","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2577868,"name":"drain","context":{"idset":"342","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2579179,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2580254,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2581847,"name":"drain","context":{"idset":"345","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.258379,"name":"drain","context":{"idset":"346","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2584953,"name":"drain","context":{"idset":"347","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2586358,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2587464,"name":"drain","context":{"idset":"350","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2589319,"name":"drain","context":{"idset":"351","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2590652,"name":"drain","context":{"idset":"352","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2592273,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2594182,"name":"drain","context":{"idset":"354","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2595685,"name":"drain","context":{"idset":"355","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2596927,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2598264,"name":"drain","context":{"idset":"357","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2599926,"name":"drain","context":{"idset":"358","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.260107,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2602549,"name":"drain","context":{"idset":"360","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2604394,"name":"drain","context":{"idset":"361","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2605715,"name":"drain","context":{"idset":"362","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2607214,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2608461,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2609866,"name":"drain","context":{"idset":"365","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2611835,"name":"drain","context":{"idset":"366","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2613223,"name":"drain","context":{"idset":"367","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2614841,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2616174,"name":"drain","context":{"idset":"369","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2617965,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2619493,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2620864,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2622414,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2624378,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2625926,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2627516,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2628973,"name":"drain","context":{"idset":"377","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2630675,"name":"drain","context":{"idset":"378","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.263252,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2634366,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2635744,"name":"drain","context":{"idset":"381","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.263726,"name":"drain","context":{"idset":"382","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.263864,"name":"drain","context":{"idset":"383","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2640243,"name":"drain","context":{"idset":"384","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.264183,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2643523,"name":"drain","context":{"idset":"386","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2645359,"name":"drain","context":{"idset":"387","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.264689,"name":"drain","context":{"idset":"388","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2648473,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2650054,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2651813,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2653377,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2655056,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2656848,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2658558,"name":"drain","context":{"idset":"395","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2660236,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.266165,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2663405,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2665014,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2666881,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2668436,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2669966,"name":"drain","context":{"idset":"402","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2671745,"name":"drain","context":{"idset":"403","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.267381,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.267556,"name":"drain","context":{"idset":"405","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2677186,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2678699,"name":"drain","context":{"idset":"407","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2680836,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2682476,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2684119,"name":"drain","context":{"idset":"410","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2685771,"name":"drain","context":{"idset":"411","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2687278,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2688804,"name":"drain","context":{"idset":"413","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2690475,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2692211,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2694154,"name":"drain","context":{"idset":"416","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2695661,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2697184,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2698672,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.270031,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.270216,"name":"drain","context":{"idset":"421","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2703731,"name":"drain","context":{"idset":"422","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2705886,"name":"drain","context":{"idset":"423","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2707787,"name":"drain","context":{"idset":"424","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2709115,"name":"drain","context":{"idset":"425","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2710454,"name":"drain","context":{"idset":"426","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2711689,"name":"drain","context":{"idset":"427","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2713249,"name":"drain","context":{"idset":"428","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2714541,"name":"drain","context":{"idset":"429","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.271594,"name":"drain","context":{"idset":"430","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2718632,"name":"drain","context":{"idset":"432","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2720077,"name":"drain","context":{"idset":"433","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2721453,"name":"drain","context":{"idset":"434","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2722859,"name":"drain","context":{"idset":"435","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2724187,"name":"drain","context":{"idset":"436","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2725475,"name":"drain","context":{"idset":"437","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2726729,"name":"drain","context":{"idset":"438","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2728074,"name":"drain","context":{"idset":"439","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.272953,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2731025,"name":"drain","context":{"idset":"441","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2732365,"name":"drain","context":{"idset":"442","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2733817,"name":"drain","context":{"idset":"443","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2735665,"name":"drain","context":{"idset":"444","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2737708,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2739315,"name":"drain","context":{"idset":"470","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2740798,"name":"drain","context":{"idset":"471","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2742119,"name":"drain","context":{"idset":"472","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2743676,"name":"drain","context":{"idset":"473","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2745199,"name":"drain","context":{"idset":"474","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2746675,"name":"drain","context":{"idset":"475","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2747996,"name":"drain","context":{"idset":"476","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2749362,"name":"drain","context":{"idset":"477","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.275068,"name":"drain","context":{"idset":"478","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.275203,"name":"drain","context":{"idset":"479","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2753644,"name":"drain","context":{"idset":"480","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.275517,"name":"drain","context":{"idset":"481","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2756519,"name":"drain","context":{"idset":"482","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2758021,"name":"drain","context":{"idset":"483","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2759569,"name":"drain","context":{"idset":"484","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2760921,"name":"drain","context":{"idset":"485","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2762446,"name":"drain","context":{"idset":"486","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2764091,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2765806,"name":"drain","context":{"idset":"488","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2767344,"name":"drain","context":{"idset":"489","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2768693,"name":"drain","context":{"idset":"490","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2770088,"name":"drain","context":{"idset":"491","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2771466,"name":"drain","context":{"idset":"492","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2773159,"name":"drain","context":{"idset":"493","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2774546,"name":"drain","context":{"idset":"494","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2776456,"name":"drain","context":{"idset":"495","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.277786,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2779441,"name":"drain","context":{"idset":"497","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2780952,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2782397,"name":"drain","context":{"idset":"499","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2783957,"name":"drain","context":{"idset":"500","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2785323,"name":"drain","context":{"idset":"501","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2787144,"name":"drain","context":{"idset":"502","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2788572,"name":"drain","context":{"idset":"503","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2790089,"name":"drain","context":{"idset":"504","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2791617,"name":"drain","context":{"idset":"505","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.27932,"name":"drain","context":{"idset":"506","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.279459,"name":"drain","context":{"idset":"507","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2795956,"name":"drain","context":{"idset":"508","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2797847,"name":"drain","context":{"idset":"509","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2799296,"name":"drain","context":{"idset":"510","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2800832,"name":"drain","context":{"idset":"511","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2802305,"name":"drain","context":{"idset":"512","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2803993,"name":"drain","context":{"idset":"513","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2805479,"name":"drain","context":{"idset":"514","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2806888,"name":"drain","context":{"idset":"515","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2808588,"name":"drain","context":{"idset":"516","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2810044,"name":"drain","context":{"idset":"517","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2811444,"name":"drain","context":{"idset":"518","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2813232,"name":"drain","context":{"idset":"519","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2814908,"name":"drain","context":{"idset":"520","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2816381,"name":"drain","context":{"idset":"521","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2817841,"name":"drain","context":{"idset":"522","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2819545,"name":"drain","context":{"idset":"523","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2821147,"name":"drain","context":{"idset":"524","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2822933,"name":"drain","context":{"idset":"525","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2824483,"name":"drain","context":{"idset":"526","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.282604,"name":"drain","context":{"idset":"527","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2827566,"name":"drain","context":{"idset":"528","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2829111,"name":"drain","context":{"idset":"529","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2830932,"name":"drain","context":{"idset":"530","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2832422,"name":"drain","context":{"idset":"531","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.283426,"name":"drain","context":{"idset":"532","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2835739,"name":"drain","context":{"idset":"533","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2837248,"name":"drain","context":{"idset":"534","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2838998,"name":"drain","context":{"idset":"535","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2840562,"name":"drain","context":{"idset":"536","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2842324,"name":"drain","context":{"idset":"537","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2843966,"name":"drain","context":{"idset":"538","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2845466,"name":"drain","context":{"idset":"539","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.284693,"name":"drain","context":{"idset":"540","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2848408,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2850063,"name":"drain","context":{"idset":"566","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2852187,"name":"drain","context":{"idset":"567","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2854061,"name":"drain","context":{"idset":"568","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2855771,"name":"drain","context":{"idset":"569","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2857399,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2858939,"name":"drain","context":{"idset":"571","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2860723,"name":"drain","context":{"idset":"572","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2862372,"name":"drain","context":{"idset":"573","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2864635,"name":"drain","context":{"idset":"574","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2866285,"name":"drain","context":{"idset":"575","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2867982,"name":"drain","context":{"idset":"576","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2870302,"name":"drain","context":{"idset":"577","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2871921,"name":"drain","context":{"idset":"578","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2874193,"name":"drain","context":{"idset":"579","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2876408,"name":"drain","context":{"idset":"580","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.287806,"name":"drain","context":{"idset":"581","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2879577,"name":"drain","context":{"idset":"582","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2881575,"name":"drain","context":{"idset":"583","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2883339,"name":"drain","context":{"idset":"584","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2885106,"name":"drain","context":{"idset":"585","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2886982,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2888741,"name":"drain","context":{"idset":"587","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2890332,"name":"drain","context":{"idset":"588","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2892213,"name":"drain","context":{"idset":"629","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2894192,"name":"drain","context":{"idset":"630","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2896099,"name":"drain","context":{"idset":"631","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2898138,"name":"drain","context":{"idset":"632","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2899923,"name":"drain","context":{"idset":"633","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2901745,"name":"drain","context":{"idset":"634","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2903678,"name":"drain","context":{"idset":"635","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2905288,"name":"drain","context":{"idset":"636","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2907131,"name":"drain","context":{"idset":"827","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2909091,"name":"drain","context":{"idset":"828","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2911112,"name":"drain","context":{"idset":"829","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2913167,"name":"drain","context":{"idset":"830","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.291532,"name":"drain","context":{"idset":"831","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2917318,"name":"drain","context":{"idset":"832","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2919226,"name":"drain","context":{"idset":"833","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.292141,"name":"drain","context":{"idset":"834","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2923977,"name":"drain","context":{"idset":"841","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2926667,"name":"drain","context":{"idset":"842","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2928617,"name":"drain","context":{"idset":"871","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2930267,"name":"drain","context":{"idset":"872","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.293196,"name":"drain","context":{"idset":"949","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2933846,"name":"drain","context":{"idset":"950","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2935448,"name":"drain","context":{"idset":"951","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2937517,"name":"drain","context":{"idset":"952","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2939444,"name":"drain","context":{"idset":"953","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.294106,"name":"drain","context":{"idset":"954","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2943177,"name":"drain","context":{"idset":"963","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2944853,"name":"drain","context":{"idset":"964","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2946494,"name":"drain","context":{"idset":"975","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2948294,"name":"drain","context":{"idset":"976","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2952271,"name":"drain","context":{"idset":"9205","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2955875,"name":"drain","context":{"idset":"9206","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.296021,"name":"drain","context":{"idset":"9207","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2964427,"name":"drain","context":{"idset":"9208","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2968445,"name":"drain","context":{"idset":"9209","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2972867,"name":"drain","context":{"idset":"9210","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.297647,"name":"drain","context":{"idset":"9211","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2979949,"name":"drain","context":{"idset":"9212","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2983701,"name":"drain","context":{"idset":"9213","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2987416,"name":"drain","context":{"idset":"9215","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2990794,"name":"drain","context":{"idset":"9216","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2995129,"name":"drain","context":{"idset":"9217","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.2999234,"name":"drain","context":{"idset":"9218","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3003852,"name":"drain","context":{"idset":"9219","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3007545,"name":"drain","context":{"idset":"9220","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3011456,"name":"drain","context":{"idset":"9221","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3015721,"name":"drain","context":{"idset":"9222","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3019235,"name":"drain","context":{"idset":"9223","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3022864,"name":"drain","context":{"idset":"9224","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3026607,"name":"drain","context":{"idset":"9225","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3030438,"name":"drain","context":{"idset":"9226","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3034568,"name":"drain","context":{"idset":"9227","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3038936,"name":"drain","context":{"idset":"9228","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3042996,"name":"drain","context":{"idset":"9229","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3047299,"name":"drain","context":{"idset":"9230","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3051598,"name":"drain","context":{"idset":"9231","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3055997,"name":"drain","context":{"idset":"9232","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3059926,"name":"drain","context":{"idset":"9233","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3064158,"name":"drain","context":{"idset":"9234","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3068275,"name":"drain","context":{"idset":"9235","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3072319,"name":"drain","context":{"idset":"9236","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3076165,"name":"drain","context":{"idset":"9237","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3079703,"name":"drain","context":{"idset":"9238","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3083317,"name":"drain","context":{"idset":"9241","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3087118,"name":"drain","context":{"idset":"9242","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3090804,"name":"drain","context":{"idset":"9243","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3094623,"name":"drain","context":{"idset":"9244","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3098512,"name":"drain","context":{"idset":"9245","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3102007,"name":"drain","context":{"idset":"9246","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3105772,"name":"drain","context":{"idset":"9247","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3109539,"name":"drain","context":{"idset":"9248","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3113551,"name":"drain","context":{"idset":"9249","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3117914,"name":"drain","context":{"idset":"9250","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.312165,"name":"drain","context":{"idset":"9251","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3125522,"name":"drain","context":{"idset":"9252","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3129256,"name":"drain","context":{"idset":"9253","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.313297,"name":"drain","context":{"idset":"9254","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3137007,"name":"drain","context":{"idset":"9255","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.314054,"name":"drain","context":{"idset":"9256","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3144908,"name":"drain","context":{"idset":"9257","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.314899,"name":"drain","context":{"idset":"9258","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3152778,"name":"drain","context":{"idset":"9259","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3157024,"name":"drain","context":{"idset":"9260","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3161159,"name":"drain","context":{"idset":"9261","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3165081,"name":"drain","context":{"idset":"9262","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3169003,"name":"drain","context":{"idset":"9263","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.317277,"name":"drain","context":{"idset":"9264","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3176601,"name":"drain","context":{"idset":"9265","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3180387,"name":"drain","context":{"idset":"9266","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3184505,"name":"drain","context":{"idset":"9267","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3188241,"name":"drain","context":{"idset":"9268","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3192263,"name":"drain","context":{"idset":"9269","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3196049,"name":"drain","context":{"idset":"9270","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.320034,"name":"drain","context":{"idset":"9271","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3204567,"name":"drain","context":{"idset":"9272","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.320859,"name":"drain","context":{"idset":"9273","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3212461,"name":"drain","context":{"idset":"9274","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3216519,"name":"drain","context":{"idset":"9275","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3220909,"name":"drain","context":{"idset":"9276","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3225801,"name":"drain","context":{"idset":"9277","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3230762,"name":"drain","context":{"idset":"9278","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3235774,"name":"drain","context":{"idset":"9279","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.323946,"name":"drain","context":{"idset":"9280","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3243504,"name":"drain","context":{"idset":"9281","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3247416,"name":"drain","context":{"idset":"9282","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3251071,"name":"drain","context":{"idset":"9283","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3255732,"name":"drain","context":{"idset":"9284","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3259771,"name":"drain","context":{"idset":"9285","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3263881,"name":"drain","context":{"idset":"9286","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3268397,"name":"drain","context":{"idset":"9287","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.327239,"name":"drain","context":{"idset":"9288","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3276513,"name":"drain","context":{"idset":"9289","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3280807,"name":"drain","context":{"idset":"9290","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3286028,"name":"drain","context":{"idset":"9291","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3291101,"name":"drain","context":{"idset":"9292","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3296106,"name":"drain","context":{"idset":"9293","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3303347,"name":"drain","context":{"idset":"9294","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3307176,"name":"drain","context":{"idset":"9295","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3311355,"name":"drain","context":{"idset":"9296","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.331594,"name":"drain","context":{"idset":"9297","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3319633,"name":"drain","context":{"idset":"9298","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3323581,"name":"drain","context":{"idset":"9299","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.332725,"name":"drain","context":{"idset":"9300","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3330855,"name":"drain","context":{"idset":"9301","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3334589,"name":"drain","context":{"idset":"9302","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3338201,"name":"drain","context":{"idset":"9303","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3341796,"name":"drain","context":{"idset":"9304","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3345523,"name":"drain","context":{"idset":"9305","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3349161,"name":"drain","context":{"idset":"9306","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3352871,"name":"drain","context":{"idset":"9307","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3356507,"name":"drain","context":{"idset":"9308","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3360145,"name":"drain","context":{"idset":"9309","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3363879,"name":"drain","context":{"idset":"9310","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3367529,"name":"drain","context":{"idset":"9311","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3371174,"name":"drain","context":{"idset":"9312","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3375506,"name":"drain","context":{"idset":"9313","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3379519,"name":"drain","context":{"idset":"9314","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3383715,"name":"drain","context":{"idset":"9315","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3388176,"name":"drain","context":{"idset":"9317","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3392313,"name":"drain","context":{"idset":"9319","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3397138,"name":"drain","context":{"idset":"9320","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3401206,"name":"drain","context":{"idset":"9321","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3405697,"name":"drain","context":{"idset":"9322","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3409452,"name":"drain","context":{"idset":"9323","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3414855,"name":"drain","context":{"idset":"9324","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3418963,"name":"drain","context":{"idset":"9325","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3422897,"name":"drain","context":{"idset":"9326","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3426609,"name":"drain","context":{"idset":"9327","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3430295,"name":"drain","context":{"idset":"9328","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.343416,"name":"drain","context":{"idset":"9329","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3437881,"name":"drain","context":{"idset":"9330","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.344156,"name":"drain","context":{"idset":"9331","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3445544,"name":"drain","context":{"idset":"9332","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3449254,"name":"drain","context":{"idset":"9333","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3453109,"name":"drain","context":{"idset":"9334","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.345686,"name":"drain","context":{"idset":"9335","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3460591,"name":"drain","context":{"idset":"9336","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3464417,"name":"drain","context":{"idset":"9337","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3468146,"name":"drain","context":{"idset":"9338","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3472128,"name":"drain","context":{"idset":"9339","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3476036,"name":"drain","context":{"idset":"9340","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3479791,"name":"drain","context":{"idset":"9341","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3483694,"name":"drain","context":{"idset":"9342","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3487444,"name":"drain","context":{"idset":"9343","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3491178,"name":"drain","context":{"idset":"9344","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.349508,"name":"drain","context":{"idset":"9345","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3499067,"name":"drain","context":{"idset":"9346","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3503244,"name":"drain","context":{"idset":"9347","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3507071,"name":"drain","context":{"idset":"9348","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3511038,"name":"drain","context":{"idset":"9349","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3515794,"name":"drain","context":{"idset":"9350","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.352,"name":"drain","context":{"idset":"9351","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3524508,"name":"drain","context":{"idset":"9352","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3528392,"name":"drain","context":{"idset":"9353","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3532486,"name":"drain","context":{"idset":"9354","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3536842,"name":"drain","context":{"idset":"9355","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3541024,"name":"drain","context":{"idset":"9356","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3545299,"name":"drain","context":{"idset":"9357","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3549948,"name":"drain","context":{"idset":"9358","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3554275,"name":"drain","context":{"idset":"9359","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3558381,"name":"drain","context":{"idset":"9360","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3562491,"name":"drain","context":{"idset":"9361","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.356678,"name":"drain","context":{"idset":"9362","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.357111,"name":"drain","context":{"idset":"9363","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3575432,"name":"drain","context":{"idset":"9364","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3579662,"name":"drain","context":{"idset":"9365","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3583727,"name":"drain","context":{"idset":"9366","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3587983,"name":"drain","context":{"idset":"9367","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3591878,"name":"drain","context":{"idset":"9368","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3595841,"name":"drain","context":{"idset":"9369","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3599703,"name":"drain","context":{"idset":"9370","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3604333,"name":"drain","context":{"idset":"9371","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3608251,"name":"drain","context":{"idset":"9372","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3612773,"name":"drain","context":{"idset":"9373","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3616908,"name":"drain","context":{"idset":"9374","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3621163,"name":"drain","context":{"idset":"9375","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3625994,"name":"drain","context":{"idset":"9376","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3630276,"name":"drain","context":{"idset":"9377","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3634908,"name":"drain","context":{"idset":"9378","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3639109,"name":"drain","context":{"idset":"9379","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3643198,"name":"drain","context":{"idset":"9380","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3647523,"name":"drain","context":{"idset":"9381","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3651464,"name":"drain","context":{"idset":"9382","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3655953,"name":"drain","context":{"idset":"9383","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.365994,"name":"drain","context":{"idset":"9384","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3663993,"name":"drain","context":{"idset":"9385","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3667903,"name":"drain","context":{"idset":"9386","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3671815,"name":"drain","context":{"idset":"9387","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3675973,"name":"drain","context":{"idset":"9388","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3679898,"name":"drain","context":{"idset":"9389","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3683949,"name":"drain","context":{"idset":"9390","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3687873,"name":"drain","context":{"idset":"9391","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3691785,"name":"drain","context":{"idset":"9392","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3695843,"name":"drain","context":{"idset":"9393","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3699763,"name":"drain","context":{"idset":"9394","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3703845,"name":"drain","context":{"idset":"9395","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3707793,"name":"drain","context":{"idset":"9396","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.371172,"name":"drain","context":{"idset":"9397","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3715844,"name":"drain","context":{"idset":"9398","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3719792,"name":"drain","context":{"idset":"9399","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3723879,"name":"drain","context":{"idset":"9400","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3727884,"name":"drain","context":{"idset":"9401","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.373184,"name":"drain","context":{"idset":"9402","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3735936,"name":"drain","context":{"idset":"9403","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3739939,"name":"drain","context":{"idset":"9404","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3744025,"name":"drain","context":{"idset":"9405","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3748012,"name":"drain","context":{"idset":"9406","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3751996,"name":"drain","context":{"idset":"9407","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3756082,"name":"drain","context":{"idset":"9408","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.37604,"name":"drain","context":{"idset":"9409","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3765039,"name":"drain","context":{"idset":"9410","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3769233,"name":"drain","context":{"idset":"9411","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3773606,"name":"drain","context":{"idset":"9412","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3778069,"name":"drain","context":{"idset":"9413","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3782375,"name":"drain","context":{"idset":"9414","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3787589,"name":"drain","context":{"idset":"9415","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3793902,"name":"drain","context":{"idset":"9416","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3799071,"name":"drain","context":{"idset":"9417","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3804541,"name":"drain","context":{"idset":"9418","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3809621,"name":"drain","context":{"idset":"9419","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3814833,"name":"drain","context":{"idset":"9420","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3820152,"name":"drain","context":{"idset":"9421","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.382534,"name":"drain","context":{"idset":"9422","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3830402,"name":"drain","context":{"idset":"9423","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3835061,"name":"drain","context":{"idset":"9424","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3839669,"name":"drain","context":{"idset":"9425","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3845413,"name":"drain","context":{"idset":"9426","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3850296,"name":"drain","context":{"idset":"9427","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3855517,"name":"drain","context":{"idset":"9428","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3860972,"name":"drain","context":{"idset":"9429","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3866255,"name":"drain","context":{"idset":"9430","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3871996,"name":"drain","context":{"idset":"9431","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.387713,"name":"drain","context":{"idset":"9432","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3882332,"name":"drain","context":{"idset":"9433","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3887851,"name":"drain","context":{"idset":"9434","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3893387,"name":"drain","context":{"idset":"9435","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3899322,"name":"drain","context":{"idset":"9436","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3904872,"name":"drain","context":{"idset":"9437","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3910923,"name":"drain","context":{"idset":"9438","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3916361,"name":"drain","context":{"idset":"9439","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3921294,"name":"drain","context":{"idset":"9440","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3926804,"name":"drain","context":{"idset":"9441","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3932047,"name":"drain","context":{"idset":"9442","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3939195,"name":"drain","context":{"idset":"9443","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3945632,"name":"drain","context":{"idset":"9444","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.395148,"name":"drain","context":{"idset":"9445","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3956766,"name":"drain","context":{"idset":"9446","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3961937,"name":"drain","context":{"idset":"9449","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3967452,"name":"drain","context":{"idset":"9450","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3972845,"name":"drain","context":{"idset":"9451","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3978529,"name":"drain","context":{"idset":"9452","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.398407,"name":"drain","context":{"idset":"9453","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3989625,"name":"drain","context":{"idset":"9454","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.3995063,"name":"drain","context":{"idset":"9455","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4000347,"name":"drain","context":{"idset":"9456","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4005325,"name":"drain","context":{"idset":"9457","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4010484,"name":"drain","context":{"idset":"9458","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4016027,"name":"drain","context":{"idset":"9459","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4021955,"name":"drain","context":{"idset":"9460","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.402739,"name":"drain","context":{"idset":"9461","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4032285,"name":"drain","context":{"idset":"9462","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4037616,"name":"drain","context":{"idset":"9463","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4043248,"name":"drain","context":{"idset":"9464","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.404882,"name":"drain","context":{"idset":"9465","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4054239,"name":"drain","context":{"idset":"9466","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4059749,"name":"drain","context":{"idset":"9467","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4065804,"name":"drain","context":{"idset":"9468","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4071465,"name":"drain","context":{"idset":"9469","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4076934,"name":"drain","context":{"idset":"9470","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4081283,"name":"drain","context":{"idset":"9471","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4086525,"name":"drain","context":{"idset":"9472","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4092138,"name":"drain","context":{"idset":"9473","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4097822,"name":"drain","context":{"idset":"9474","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4103234,"name":"drain","context":{"idset":"9475","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4108651,"name":"drain","context":{"idset":"9476","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4114223,"name":"drain","context":{"idset":"9477","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4119496,"name":"drain","context":{"idset":"9478","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4125671,"name":"drain","context":{"idset":"9479","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4132147,"name":"drain","context":{"idset":"9480","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4138775,"name":"drain","context":{"idset":"9481","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4144835,"name":"drain","context":{"idset":"9482","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4149582,"name":"drain","context":{"idset":"9483","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4154117,"name":"drain","context":{"idset":"9484","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4158936,"name":"drain","context":{"idset":"9485","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4163501,"name":"drain","context":{"idset":"9486","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4167912,"name":"drain","context":{"idset":"9487","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4174592,"name":"drain","context":{"idset":"9488","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4190547,"name":"drain","context":{"idset":"9489","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4198856,"name":"drain","context":{"idset":"9490","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4205687,"name":"drain","context":{"idset":"9491","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4212465,"name":"drain","context":{"idset":"9492","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4219382,"name":"drain","context":{"idset":"9493","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4226267,"name":"drain","context":{"idset":"9494","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4233079,"name":"drain","context":{"idset":"9495","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4239669,"name":"drain","context":{"idset":"9496","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4246514,"name":"drain","context":{"idset":"9497","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4253368,"name":"drain","context":{"idset":"9498","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4259896,"name":"drain","context":{"idset":"9499","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4266763,"name":"drain","context":{"idset":"9500","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4273386,"name":"drain","context":{"idset":"9501","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4280002,"name":"drain","context":{"idset":"9502","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4288449,"name":"drain","context":{"idset":"9503","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4295268,"name":"drain","context":{"idset":"9504","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4301987,"name":"drain","context":{"idset":"9505","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4316468,"name":"drain","context":{"idset":"9506","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4323425,"name":"drain","context":{"idset":"9507","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.432992,"name":"drain","context":{"idset":"9508","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4336872,"name":"drain","context":{"idset":"9509","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4343328,"name":"drain","context":{"idset":"9510","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4347847,"name":"drain","context":{"idset":"9511","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4352288,"name":"drain","context":{"idset":"9512","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4356914,"name":"drain","context":{"idset":"9513","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4361334,"name":"drain","context":{"idset":"9514","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4365995,"name":"drain","context":{"idset":"9515","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4370401,"name":"drain","context":{"idset":"9516","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4375031,"name":"drain","context":{"idset":"9517","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4379911,"name":"drain","context":{"idset":"9518","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4384549,"name":"drain","context":{"idset":"9519","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4388983,"name":"drain","context":{"idset":"9520","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4393609,"name":"drain","context":{"idset":"9521","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4398031,"name":"drain","context":{"idset":"9522","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4402478,"name":"drain","context":{"idset":"9523","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4407229,"name":"drain","context":{"idset":"9524","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4411709,"name":"drain","context":{"idset":"9525","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4416361,"name":"drain","context":{"idset":"9526","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4420826,"name":"drain","context":{"idset":"9527","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4425514,"name":"drain","context":{"idset":"9528","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4429958,"name":"drain","context":{"idset":"9529","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4434624,"name":"drain","context":{"idset":"9530","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4439077,"name":"drain","context":{"idset":"9531","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4443798,"name":"drain","context":{"idset":"9532","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4448252,"name":"drain","context":{"idset":"9536","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4452925,"name":"drain","context":{"idset":"9537","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4457381,"name":"drain","context":{"idset":"9538","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4461846,"name":"drain","context":{"idset":"9539","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4466505,"name":"drain","context":{"idset":"9540","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4471033,"name":"drain","context":{"idset":"9541","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4475698,"name":"drain","context":{"idset":"9542","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4480226,"name":"drain","context":{"idset":"9543","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.448489,"name":"drain","context":{"idset":"9544","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4489384,"name":"drain","context":{"idset":"9545","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.449404,"name":"drain","context":{"idset":"9546","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4499049,"name":"drain","context":{"idset":"9547","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4503789,"name":"drain","context":{"idset":"9548","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4508326,"name":"drain","context":{"idset":"9549","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.451304,"name":"drain","context":{"idset":"9550","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4517753,"name":"drain","context":{"idset":"9551","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4522283,"name":"drain","context":{"idset":"9552","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4526987,"name":"drain","context":{"idset":"9553","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4531512,"name":"drain","context":{"idset":"9554","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4536245,"name":"drain","context":{"idset":"9555","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4540823,"name":"drain","context":{"idset":"9556","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4545624,"name":"drain","context":{"idset":"9557","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4550743,"name":"drain","context":{"idset":"9558","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4555583,"name":"drain","context":{"idset":"9559","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4560173,"name":"drain","context":{"idset":"9560","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4565005,"name":"drain","context":{"idset":"9563","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.456964,"name":"drain","context":{"idset":"9564","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4574506,"name":"drain","context":{"idset":"9565","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4579189,"name":"drain","context":{"idset":"9566","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4584017,"name":"drain","context":{"idset":"9567","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4588625,"name":"drain","context":{"idset":"9568","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4593434,"name":"drain","context":{"idset":"9569","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4598076,"name":"drain","context":{"idset":"9570","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4602907,"name":"drain","context":{"idset":"9571","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4607527,"name":"drain","context":{"idset":"9572","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4612164,"name":"drain","context":{"idset":"9573","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4617207,"name":"drain","context":{"idset":"9574","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4621904,"name":"drain","context":{"idset":"9575","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4626751,"name":"drain","context":{"idset":"9576","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.463136,"name":"drain","context":{"idset":"9577","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4636221,"name":"drain","context":{"idset":"9578","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4640868,"name":"drain","context":{"idset":"9579","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4645863,"name":"drain","context":{"idset":"9580","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.465054,"name":"drain","context":{"idset":"9581","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4655421,"name":"drain","context":{"idset":"9582","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.466007,"name":"drain","context":{"idset":"9583","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4664881,"name":"drain","context":{"idset":"9584","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4669533,"name":"drain","context":{"idset":"9585","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4674363,"name":"drain","context":{"idset":"9586","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4679072,"name":"drain","context":{"idset":"9587","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4683914,"name":"drain","context":{"idset":"9588","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4688601,"name":"drain","context":{"idset":"9589","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4693513,"name":"drain","context":{"idset":"9590","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4698212,"name":"drain","context":{"idset":"9591","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.47031,"name":"drain","context":{"idset":"9592","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4707782,"name":"drain","context":{"idset":"9593","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4712467,"name":"drain","context":{"idset":"9594","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4717357,"name":"drain","context":{"idset":"9595","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4722049,"name":"drain","context":{"idset":"9596","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4726954,"name":"drain","context":{"idset":"9597","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4731646,"name":"drain","context":{"idset":"9598","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4736826,"name":"drain","context":{"idset":"9599","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4741886,"name":"drain","context":{"idset":"9600","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4747031,"name":"drain","context":{"idset":"9601","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.475199,"name":"drain","context":{"idset":"9602","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4756994,"name":"drain","context":{"idset":"9603","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4761765,"name":"drain","context":{"idset":"9604","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4766712,"name":"drain","context":{"idset":"9605","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4771438,"name":"drain","context":{"idset":"9606","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4776549,"name":"drain","context":{"idset":"9607","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.478133,"name":"drain","context":{"idset":"9608","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4786286,"name":"drain","context":{"idset":"9609","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4791083,"name":"drain","context":{"idset":"9610","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4796093,"name":"drain","context":{"idset":"9611","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4800894,"name":"drain","context":{"idset":"9612","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4805832,"name":"drain","context":{"idset":"9613","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4810646,"name":"drain","context":{"idset":"9614","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4815581,"name":"drain","context":{"idset":"9615","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4820397,"name":"drain","context":{"idset":"9616","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4825373,"name":"drain","context":{"idset":"9617","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4830194,"name":"drain","context":{"idset":"9618","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4835131,"name":"drain","context":{"idset":"9619","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4839928,"name":"drain","context":{"idset":"9620","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4844897,"name":"drain","context":{"idset":"9621","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.484972,"name":"drain","context":{"idset":"9622","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4854732,"name":"drain","context":{"idset":"9623","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4859588,"name":"drain","context":{"idset":"9624","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4864573,"name":"drain","context":{"idset":"9625","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4869449,"name":"drain","context":{"idset":"9626","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.487442,"name":"drain","context":{"idset":"9627","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4879246,"name":"drain","context":{"idset":"9628","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4884274,"name":"drain","context":{"idset":"9629","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4889116,"name":"drain","context":{"idset":"9630","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4894142,"name":"drain","context":{"idset":"9631","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4898999,"name":"drain","context":{"idset":"9632","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4903982,"name":"drain","context":{"idset":"9635","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.490886,"name":"drain","context":{"idset":"9636","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.491389,"name":"drain","context":{"idset":"9637","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4918764,"name":"drain","context":{"idset":"9638","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4923739,"name":"drain","context":{"idset":"9639","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4928646,"name":"drain","context":{"idset":"9640","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4933784,"name":"drain","context":{"idset":"9641","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4938681,"name":"drain","context":{"idset":"9642","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.494369,"name":"drain","context":{"idset":"9643","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4948564,"name":"drain","context":{"idset":"9644","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4953611,"name":"drain","context":{"idset":"9645","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4958506,"name":"drain","context":{"idset":"9646","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4963596,"name":"drain","context":{"idset":"9647","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4968495,"name":"drain","context":{"idset":"9648","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4973547,"name":"drain","context":{"idset":"9649","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4978421,"name":"drain","context":{"idset":"9650","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4983506,"name":"drain","context":{"idset":"9651","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4988427,"name":"drain","context":{"idset":"9652","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.499347,"name":"drain","context":{"idset":"9653","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.4998434,"name":"drain","context":{"idset":"9654","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5003459,"name":"drain","context":{"idset":"9655","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5008368,"name":"drain","context":{"idset":"9656","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5013697,"name":"drain","context":{"idset":"9657","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5018671,"name":"drain","context":{"idset":"9658","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.502399,"name":"drain","context":{"idset":"9659","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5028989,"name":"drain","context":{"idset":"9660","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5034325,"name":"drain","context":{"idset":"9661","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5039339,"name":"drain","context":{"idset":"9662","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.504468,"name":"drain","context":{"idset":"9663","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5049689,"name":"drain","context":{"idset":"9664","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5054824,"name":"drain","context":{"idset":"9665","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5059915,"name":"drain","context":{"idset":"9666","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5065117,"name":"drain","context":{"idset":"9667","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5070307,"name":"drain","context":{"idset":"9668","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5075495,"name":"drain","context":{"idset":"9669","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5080671,"name":"drain","context":{"idset":"9670","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.508584,"name":"drain","context":{"idset":"9671","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5091031,"name":"drain","context":{"idset":"9672","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5096228,"name":"drain","context":{"idset":"9673","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5101483,"name":"drain","context":{"idset":"9674","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5106716,"name":"drain","context":{"idset":"9675","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5111911,"name":"drain","context":{"idset":"9676","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5117304,"name":"drain","context":{"idset":"9677","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5122499,"name":"drain","context":{"idset":"9678","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5127831,"name":"drain","context":{"idset":"9679","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5133045,"name":"drain","context":{"idset":"9680","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5138087,"name":"drain","context":{"idset":"9681","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5143297,"name":"drain","context":{"idset":"9682","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5148354,"name":"drain","context":{"idset":"9683","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5153601,"name":"drain","context":{"idset":"9684","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5158865,"name":"drain","context":{"idset":"9685","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5164032,"name":"drain","context":{"idset":"9686","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5169063,"name":"drain","context":{"idset":"9687","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5174186,"name":"drain","context":{"idset":"9688","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5179236,"name":"drain","context":{"idset":"9689","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5184402,"name":"drain","context":{"idset":"9690","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5189495,"name":"drain","context":{"idset":"9691","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5194695,"name":"drain","context":{"idset":"9692","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5199783,"name":"drain","context":{"idset":"9693","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5205081,"name":"drain","context":{"idset":"9694","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5210173,"name":"drain","context":{"idset":"9695","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5215447,"name":"drain","context":{"idset":"9696","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5220542,"name":"drain","context":{"idset":"9697","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.522583,"name":"drain","context":{"idset":"9698","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5230944,"name":"drain","context":{"idset":"9699","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5236182,"name":"drain","context":{"idset":"9700","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.524127,"name":"drain","context":{"idset":"9701","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5246537,"name":"drain","context":{"idset":"9702","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5251653,"name":"drain","context":{"idset":"9703","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5256927,"name":"drain","context":{"idset":"9704","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5262053,"name":"drain","context":{"idset":"9705","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5267353,"name":"drain","context":{"idset":"9706","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5272496,"name":"drain","context":{"idset":"9707","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5277846,"name":"drain","context":{"idset":"9708","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5283148,"name":"drain","context":{"idset":"9709","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.528827,"name":"drain","context":{"idset":"9710","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5293605,"name":"drain","context":{"idset":"9711","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5298758,"name":"drain","context":{"idset":"9712","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5304103,"name":"drain","context":{"idset":"9713","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5309248,"name":"drain","context":{"idset":"9714","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5314503,"name":"drain","context":{"idset":"9715","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5319638,"name":"drain","context":{"idset":"9716","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.532495,"name":"drain","context":{"idset":"9717","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5330083,"name":"drain","context":{"idset":"9718","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5335414,"name":"drain","context":{"idset":"9719","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5340581,"name":"drain","context":{"idset":"9720","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5345957,"name":"drain","context":{"idset":"9721","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5351126,"name":"drain","context":{"idset":"9722","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5356646,"name":"drain","context":{"idset":"9723","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5361876,"name":"drain","context":{"idset":"9724","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5367253,"name":"drain","context":{"idset":"9727","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5372455,"name":"drain","context":{"idset":"9728","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5377774,"name":"drain","context":{"idset":"9729","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5383177,"name":"drain","context":{"idset":"9730","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5388408,"name":"drain","context":{"idset":"9731","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5393727,"name":"drain","context":{"idset":"9732","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5398927,"name":"drain","context":{"idset":"9733","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5404253,"name":"drain","context":{"idset":"9734","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5409496,"name":"drain","context":{"idset":"9735","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5414903,"name":"drain","context":{"idset":"9736","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.542027,"name":"drain","context":{"idset":"9737","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5425735,"name":"drain","context":{"idset":"9738","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5430999,"name":"drain","context":{"idset":"9739","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5436428,"name":"drain","context":{"idset":"9740","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5441675,"name":"drain","context":{"idset":"9741","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5447049,"name":"drain","context":{"idset":"9742","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5452292,"name":"drain","context":{"idset":"9743","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5457706,"name":"drain","context":{"idset":"9744","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5463171,"name":"drain","context":{"idset":"9745","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.546845,"name":"drain","context":{"idset":"9746","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5473871,"name":"drain","context":{"idset":"9747","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5479121,"name":"drain","context":{"idset":"9748","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5484495,"name":"drain","context":{"idset":"9749","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5495033,"name":"drain","context":{"idset":"9751","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5500293,"name":"drain","context":{"idset":"9752","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5505738,"name":"drain","context":{"idset":"9753","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5511017,"name":"drain","context":{"idset":"9754","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5516443,"name":"drain","context":{"idset":"9755","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5521755,"name":"drain","context":{"idset":"9756","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5527234,"name":"drain","context":{"idset":"9757","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5532558,"name":"drain","context":{"idset":"9758","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5537963,"name":"drain","context":{"idset":"9759","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5543396,"name":"drain","context":{"idset":"9760","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5548689,"name":"drain","context":{"idset":"9761","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5554218,"name":"drain","context":{"idset":"9762","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5559607,"name":"drain","context":{"idset":"9763","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5565169,"name":"drain","context":{"idset":"9764","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5570476,"name":"drain","context":{"idset":"9765","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5575924,"name":"drain","context":{"idset":"9766","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.558125,"name":"drain","context":{"idset":"9767","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5586729,"name":"drain","context":{"idset":"9768","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5592058,"name":"drain","context":{"idset":"9769","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5597532,"name":"drain","context":{"idset":"9770","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5602984,"name":"drain","context":{"idset":"9771","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5608363,"name":"drain","context":{"idset":"9772","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5613887,"name":"drain","context":{"idset":"9773","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5619247,"name":"drain","context":{"idset":"9774","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5624716,"name":"drain","context":{"idset":"9775","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5630093,"name":"drain","context":{"idset":"9776","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5635655,"name":"drain","context":{"idset":"9777","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5641038,"name":"drain","context":{"idset":"9778","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5646513,"name":"drain","context":{"idset":"9779","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5651889,"name":"drain","context":{"idset":"9780","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5657415,"name":"drain","context":{"idset":"9781","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.566293,"name":"drain","context":{"idset":"9782","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5668309,"name":"drain","context":{"idset":"9783","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5673821,"name":"drain","context":{"idset":"9784","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.56792,"name":"drain","context":{"idset":"9785","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5684707,"name":"drain","context":{"idset":"9786","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5690169,"name":"drain","context":{"idset":"9787","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5695748,"name":"drain","context":{"idset":"9788","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5701172,"name":"drain","context":{"idset":"9789","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5706718,"name":"drain","context":{"idset":"9790","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5712104,"name":"drain","context":{"idset":"9791","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.571764,"name":"drain","context":{"idset":"9792","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5723226,"name":"drain","context":{"idset":"9793","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5728652,"name":"drain","context":{"idset":"9794","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5734196,"name":"drain","context":{"idset":"9795","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5739708,"name":"drain","context":{"idset":"9796","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5745399,"name":"drain","context":{"idset":"9797","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5750873,"name":"drain","context":{"idset":"9798","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5756762,"name":"drain","context":{"idset":"9799","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.576226,"name":"drain","context":{"idset":"9800","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5767908,"name":"drain","context":{"idset":"9801","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5773485,"name":"drain","context":{"idset":"9802","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5778935,"name":"drain","context":{"idset":"9803","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5784519,"name":"drain","context":{"idset":"9804","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5789957,"name":"drain","context":{"idset":"9805","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5795565,"name":"drain","context":{"idset":"9806","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5801044,"name":"drain","context":{"idset":"9807","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5806615,"name":"drain","context":{"idset":"9808","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5812097,"name":"drain","context":{"idset":"9809","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5817728,"name":"drain","context":{"idset":"9810","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5823357,"name":"drain","context":{"idset":"9811","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5828826,"name":"drain","context":{"idset":"9812","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5834484,"name":"drain","context":{"idset":"9813","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.583998,"name":"drain","context":{"idset":"9814","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5845556,"name":"drain","context":{"idset":"9815","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5851045,"name":"drain","context":{"idset":"9816","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.58567,"name":"drain","context":{"idset":"9817","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.586221,"name":"drain","context":{"idset":"9818","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5867889,"name":"drain","context":{"idset":"9819","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5873539,"name":"drain","context":{"idset":"9820","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5879078,"name":"drain","context":{"idset":"9821","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5884695,"name":"drain","context":{"idset":"9822","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5890257,"name":"drain","context":{"idset":"9823","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5895948,"name":"drain","context":{"idset":"9824","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5901487,"name":"drain","context":{"idset":"9825","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5907223,"name":"drain","context":{"idset":"9826","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5912886,"name":"drain","context":{"idset":"9827","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5918422,"name":"drain","context":{"idset":"9828","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5924091,"name":"drain","context":{"idset":"9829","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5929649,"name":"drain","context":{"idset":"9830","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5935335,"name":"drain","context":{"idset":"9831","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5940893,"name":"drain","context":{"idset":"9832","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5946641,"name":"drain","context":{"idset":"9833","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5952268,"name":"drain","context":{"idset":"9834","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.595803,"name":"drain","context":{"idset":"9835","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5963714,"name":"drain","context":{"idset":"9836","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5969286,"name":"drain","context":{"idset":"9837","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5974972,"name":"drain","context":{"idset":"9838","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5980561,"name":"drain","context":{"idset":"9839","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5986316,"name":"drain","context":{"idset":"9840","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5991912,"name":"drain","context":{"idset":"9841","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.5997603,"name":"drain","context":{"idset":"9842","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6003358,"name":"drain","context":{"idset":"9843","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6008949,"name":"drain","context":{"idset":"9844","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6014712,"name":"drain","context":{"idset":"9973","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6020353,"name":"drain","context":{"idset":"9974","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.602607,"name":"drain","context":{"idset":"9975","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.603168,"name":"drain","context":{"idset":"9976","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6037414,"name":"drain","context":{"idset":"9977","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6043212,"name":"drain","context":{"idset":"9978","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6048877,"name":"drain","context":{"idset":"9979","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6054645,"name":"drain","context":{"idset":"9980","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6060302,"name":"drain","context":{"idset":"9981","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6066124,"name":"drain","context":{"idset":"9982","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6071782,"name":"drain","context":{"idset":"9983","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6077561,"name":"drain","context":{"idset":"9984","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6083393,"name":"drain","context":{"idset":"9985","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6089053,"name":"drain","context":{"idset":"9986","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.609484,"name":"drain","context":{"idset":"9987","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6100547,"name":"drain","context":{"idset":"9988","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6106315,"name":"drain","context":{"idset":"9989","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6112003,"name":"drain","context":{"idset":"9990","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6117814,"name":"drain","context":{"idset":"9991","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6123636,"name":"drain","context":{"idset":"9992","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6129355,"name":"drain","context":{"idset":"9993","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6135185,"name":"drain","context":{"idset":"9994","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6140931,"name":"drain","context":{"idset":"9995","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6146774,"name":"drain","context":{"idset":"9996","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6152487,"name":"drain","context":{"idset":"9997","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6158333,"name":"drain","context":{"idset":"9998","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6164162,"name":"drain","context":{"idset":"9999","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6169884,"name":"drain","context":{"idset":"10000","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.617569,"name":"drain","context":{"idset":"10001","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6181395,"name":"drain","context":{"idset":"10002","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.618722,"name":"drain","context":{"idset":"10003","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6193061,"name":"drain","context":{"idset":"10004","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.61988,"name":"drain","context":{"idset":"10005","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6204653,"name":"drain","context":{"idset":"10006","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6210384,"name":"drain","context":{"idset":"10007","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.621628,"name":"drain","context":{"idset":"10008","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6222048,"name":"drain","context":{"idset":"10009","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6227961,"name":"drain","context":{"idset":"10010","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6233871,"name":"drain","context":{"idset":"10011","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6239636,"name":"drain","context":{"idset":"10012","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6245511,"name":"drain","context":{"idset":"10013","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6251276,"name":"drain","context":{"idset":"10014","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6257203,"name":"drain","context":{"idset":"10015","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.626308,"name":"drain","context":{"idset":"10016","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6268897,"name":"drain","context":{"idset":"10017","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6275012,"name":"drain","context":{"idset":"10018","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6280825,"name":"drain","context":{"idset":"10019","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6286852,"name":"drain","context":{"idset":"10020","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6292794,"name":"drain","context":{"idset":"10021","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6298611,"name":"drain","context":{"idset":"10022","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6304564,"name":"drain","context":{"idset":"10023","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.631036,"name":"drain","context":{"idset":"10024","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6316307,"name":"drain","context":{"idset":"10025","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6322126,"name":"drain","context":{"idset":"10026","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6328106,"name":"drain","context":{"idset":"10027","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6334112,"name":"drain","context":{"idset":"10028","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6339934,"name":"drain","context":{"idset":"10029","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6345897,"name":"drain","context":{"idset":"10030","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6351728,"name":"drain","context":{"idset":"10031","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6357706,"name":"drain","context":{"idset":"10032","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6363678,"name":"drain","context":{"idset":"10033","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6369488,"name":"drain","context":{"idset":"10034","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6375492,"name":"drain","context":{"idset":"10035","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6381335,"name":"drain","context":{"idset":"10036","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6387267,"name":"drain","context":{"idset":"10037","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6393211,"name":"drain","context":{"idset":"10038","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6399095,"name":"drain","context":{"idset":"10039","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6405127,"name":"drain","context":{"idset":"10040","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6410983,"name":"drain","context":{"idset":"10041","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6417019,"name":"drain","context":{"idset":"10042","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6423068,"name":"drain","context":{"idset":"10043","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6428921,"name":"drain","context":{"idset":"10044","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6434925,"name":"drain","context":{"idset":"10045","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6440773,"name":"drain","context":{"idset":"10046","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6446834,"name":"drain","context":{"idset":"10047","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6452863,"name":"drain","context":{"idset":"10048","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6458752,"name":"drain","context":{"idset":"10049","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6464801,"name":"drain","context":{"idset":"10050","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6470697,"name":"drain","context":{"idset":"10051","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.647671,"name":"drain","context":{"idset":"10052","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6482763,"name":"drain","context":{"idset":"10053","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6488667,"name":"drain","context":{"idset":"10054","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6494672,"name":"drain","context":{"idset":"10055","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6500626,"name":"drain","context":{"idset":"10056","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6506603,"name":"drain","context":{"idset":"10057","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6512506,"name":"drain","context":{"idset":"10058","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6518536,"name":"drain","context":{"idset":"10059","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6524644,"name":"drain","context":{"idset":"10060","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6530602,"name":"drain","context":{"idset":"10061","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6536629,"name":"drain","context":{"idset":"10062","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6542566,"name":"drain","context":{"idset":"10063","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6548688,"name":"drain","context":{"idset":"10064","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6554794,"name":"drain","context":{"idset":"10065","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.656075,"name":"drain","context":{"idset":"10066","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.656683,"name":"drain","context":{"idset":"10067","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6572964,"name":"drain","context":{"idset":"10068","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6578965,"name":"drain","context":{"idset":"10069","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.658505,"name":"drain","context":{"idset":"10070","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6591027,"name":"drain","context":{"idset":"10071","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6597164,"name":"drain","context":{"idset":"10072","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6603303,"name":"drain","context":{"idset":"10073","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6609309,"name":"drain","context":{"idset":"10074","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.661546,"name":"drain","context":{"idset":"10075","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6621461,"name":"drain","context":{"idset":"10076","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6627607,"name":"drain","context":{"idset":"10077","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6633747,"name":"drain","context":{"idset":"10078","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6639731,"name":"drain","context":{"idset":"10079","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6645932,"name":"drain","context":{"idset":"10080","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6651962,"name":"drain","context":{"idset":"10081","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6658111,"name":"drain","context":{"idset":"10082","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6664269,"name":"drain","context":{"idset":"10083","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6670268,"name":"drain","context":{"idset":"10084","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6676369,"name":"drain","context":{"idset":"10085","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6682403,"name":"drain","context":{"idset":"10086","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6688607,"name":"drain","context":{"idset":"10087","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6694894,"name":"drain","context":{"idset":"10088","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6700952,"name":"drain","context":{"idset":"10089","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6707127,"name":"drain","context":{"idset":"10090","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6713254,"name":"drain","context":{"idset":"10091","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6719296,"name":"drain","context":{"idset":"10092","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6725523,"name":"drain","context":{"idset":"10093","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6731603,"name":"drain","context":{"idset":"10094","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.673784,"name":"drain","context":{"idset":"10095","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6744008,"name":"drain","context":{"idset":"10096","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.675009,"name":"drain","context":{"idset":"10097","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6756308,"name":"drain","context":{"idset":"10098","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.67624,"name":"drain","context":{"idset":"10099","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6768615,"name":"drain","context":{"idset":"10100","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6774869,"name":"drain","context":{"idset":"10101","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.678097,"name":"drain","context":{"idset":"10102","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6787314,"name":"drain","context":{"idset":"10103","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6793542,"name":"drain","context":{"idset":"10104","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.679966,"name":"drain","context":{"idset":"10105","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6805925,"name":"drain","context":{"idset":"10106","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6812034,"name":"drain","context":{"idset":"10107","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6818275,"name":"drain","context":{"idset":"10108","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.682451,"name":"drain","context":{"idset":"10109","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6830635,"name":"drain","context":{"idset":"10110","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6836898,"name":"drain","context":{"idset":"10111","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.684314,"name":"drain","context":{"idset":"10112","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6849258,"name":"drain","context":{"idset":"10113","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6855493,"name":"drain","context":{"idset":"10114","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6861622,"name":"drain","context":{"idset":"10115","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6868007,"name":"drain","context":{"idset":"10116","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6874268,"name":"drain","context":{"idset":"10117","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6880391,"name":"drain","context":{"idset":"10118","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6886652,"name":"drain","context":{"idset":"10119","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6892951,"name":"drain","context":{"idset":"10120","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6899114,"name":"drain","context":{"idset":"10121","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6905406,"name":"drain","context":{"idset":"10122","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6911559,"name":"drain","context":{"idset":"10123","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6917815,"name":"drain","context":{"idset":"10124","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6924107,"name":"drain","context":{"idset":"10125","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6930273,"name":"drain","context":{"idset":"10126","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6936626,"name":"drain","context":{"idset":"10127","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.694303,"name":"drain","context":{"idset":"10128","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6949196,"name":"drain","context":{"idset":"10129","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6955514,"name":"drain","context":{"idset":"10130","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6961687,"name":"drain","context":{"idset":"10131","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6967976,"name":"drain","context":{"idset":"10132","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6974335,"name":"drain","context":{"idset":"10133","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6980529,"name":"drain","context":{"idset":"10134","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6986835,"name":"drain","context":{"idset":"10135","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.699317,"name":"drain","context":{"idset":"10136","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.6999376,"name":"drain","context":{"idset":"10137","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7005789,"name":"drain","context":{"idset":"10139","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7012033,"name":"drain","context":{"idset":"10140","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7018433,"name":"drain","context":{"idset":"10141","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.702491,"name":"drain","context":{"idset":"10142","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7031174,"name":"drain","context":{"idset":"10144","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.703752,"name":"drain","context":{"idset":"10146","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7043879,"name":"drain","context":{"idset":"10147","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7050161,"name":"drain","context":{"idset":"10148","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7056525,"name":"drain","context":{"idset":"10149","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7062938,"name":"drain","context":{"idset":"10150","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7069175,"name":"drain","context":{"idset":"10151","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7075539,"name":"drain","context":{"idset":"10152","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7081811,"name":"drain","context":{"idset":"10153","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7088189,"name":"drain","context":{"idset":"10154","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7094662,"name":"drain","context":{"idset":"10157","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7100945,"name":"drain","context":{"idset":"10158","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7107491,"name":"drain","context":{"idset":"10159","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7113934,"name":"drain","context":{"idset":"10160","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7120221,"name":"drain","context":{"idset":"10161","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7126665,"name":"drain","context":{"idset":"10162","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.713311,"name":"drain","context":{"idset":"10163","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7139418,"name":"drain","context":{"idset":"10164","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7145855,"name":"drain","context":{"idset":"10165","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7152166,"name":"drain","context":{"idset":"10166","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7158625,"name":"drain","context":{"idset":"10167","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7165082,"name":"drain","context":{"idset":"10168","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7171452,"name":"drain","context":{"idset":"10169","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7177925,"name":"drain","context":{"idset":"10170","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7184503,"name":"drain","context":{"idset":"10171","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7190886,"name":"drain","context":{"idset":"10172","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7197344,"name":"drain","context":{"idset":"10173","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7203803,"name":"drain","context":{"idset":"10174","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.721014,"name":"drain","context":{"idset":"10175","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7216597,"name":"drain","context":{"idset":"10176","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7223079,"name":"drain","context":{"idset":"10177","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7229462,"name":"drain","context":{"idset":"10178","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7235959,"name":"drain","context":{"idset":"10179","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7242301,"name":"drain","context":{"idset":"10180","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7248824,"name":"drain","context":{"idset":"10181","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7255385,"name":"drain","context":{"idset":"10182","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7262022,"name":"drain","context":{"idset":"10183","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.726871,"name":"drain","context":{"idset":"10184","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7275286,"name":"drain","context":{"idset":"10185","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7281671,"name":"drain","context":{"idset":"10186","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7288213,"name":"drain","context":{"idset":"10187","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.72948,"name":"drain","context":{"idset":"10188","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7301209,"name":"drain","context":{"idset":"10189","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7307687,"name":"drain","context":{"idset":"10190","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7314229,"name":"drain","context":{"idset":"10191","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.732064,"name":"drain","context":{"idset":"10192","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7327151,"name":"drain","context":{"idset":"10193","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7333694,"name":"drain","context":{"idset":"10194","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7340117,"name":"drain","context":{"idset":"10195","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7346685,"name":"drain","context":{"idset":"10196","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7353392,"name":"drain","context":{"idset":"10197","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7359846,"name":"drain","context":{"idset":"10198","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7366428,"name":"drain","context":{"idset":"10199","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7372978,"name":"drain","context":{"idset":"10200","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7379427,"name":"drain","context":{"idset":"10201","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7385962,"name":"drain","context":{"idset":"10202","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7392387,"name":"drain","context":{"idset":"10203","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7399004,"name":"drain","context":{"idset":"10204","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7405581,"name":"drain","context":{"idset":"10205","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7412045,"name":"drain","context":{"idset":"10206","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7418675,"name":"drain","context":{"idset":"10207","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7425313,"name":"drain","context":{"idset":"10208","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.743175,"name":"drain","context":{"idset":"10209","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7438438,"name":"drain","context":{"idset":"10210","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7445078,"name":"drain","context":{"idset":"10211","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7451587,"name":"drain","context":{"idset":"10212","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7458181,"name":"drain","context":{"idset":"10213","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7464828,"name":"drain","context":{"idset":"10214","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7471318,"name":"drain","context":{"idset":"10215","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7477911,"name":"drain","context":{"idset":"10216","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7484534,"name":"drain","context":{"idset":"10217","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7491021,"name":"drain","context":{"idset":"10218","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7497661,"name":"drain","context":{"idset":"10219","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7504568,"name":"drain","context":{"idset":"10220","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7511101,"name":"drain","context":{"idset":"10221","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7517745,"name":"drain","context":{"idset":"10222","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7524436,"name":"drain","context":{"idset":"10223","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7530951,"name":"drain","context":{"idset":"10224","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7537665,"name":"drain","context":{"idset":"10225","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7544343,"name":"drain","context":{"idset":"10226","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7550926,"name":"drain","context":{"idset":"10227","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7557595,"name":"drain","context":{"idset":"10228","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.756428,"name":"drain","context":{"idset":"10229","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7570839,"name":"drain","context":{"idset":"10230","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7577524,"name":"drain","context":{"idset":"10231","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7584257,"name":"drain","context":{"idset":"10232","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7590814,"name":"drain","context":{"idset":"10233","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7597554,"name":"drain","context":{"idset":"10234","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.760427,"name":"drain","context":{"idset":"10235","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7610805,"name":"drain","context":{"idset":"10236","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7617462,"name":"drain","context":{"idset":"10237","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7624183,"name":"drain","context":{"idset":"10238","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7630782,"name":"drain","context":{"idset":"10239","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7637465,"name":"drain","context":{"idset":"10240","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7644186,"name":"drain","context":{"idset":"10241","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7650795,"name":"drain","context":{"idset":"10242","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7657509,"name":"drain","context":{"idset":"10243","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7664204,"name":"drain","context":{"idset":"10244","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7670836,"name":"drain","context":{"idset":"10245","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7677605,"name":"drain","context":{"idset":"10246","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7684345,"name":"drain","context":{"idset":"10247","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7690954,"name":"drain","context":{"idset":"10248","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7697656,"name":"drain","context":{"idset":"10249","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7704449,"name":"drain","context":{"idset":"10250","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7711065,"name":"drain","context":{"idset":"10251","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7717836,"name":"drain","context":{"idset":"10252","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7724619,"name":"drain","context":{"idset":"10253","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7731254,"name":"drain","context":{"idset":"10254","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7737968,"name":"drain","context":{"idset":"10255","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7744751,"name":"drain","context":{"idset":"10256","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7751384,"name":"drain","context":{"idset":"10257","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7758152,"name":"drain","context":{"idset":"10258","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7764897,"name":"drain","context":{"idset":"10259","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7771556,"name":"drain","context":{"idset":"10260","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7784929,"name":"drain","context":{"idset":"10262","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7791626,"name":"drain","context":{"idset":"10263","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7798421,"name":"drain","context":{"idset":"10264","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7805238,"name":"drain","context":{"idset":"10265","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7811949,"name":"drain","context":{"idset":"10266","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7818754,"name":"drain","context":{"idset":"10267","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.782562,"name":"drain","context":{"idset":"10268","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7832305,"name":"drain","context":{"idset":"10269","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7839177,"name":"drain","context":{"idset":"10270","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7846031,"name":"drain","context":{"idset":"10271","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7852895,"name":"drain","context":{"idset":"10272","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7859626,"name":"drain","context":{"idset":"10273","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7866507,"name":"drain","context":{"idset":"10274","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7873333,"name":"drain","context":{"idset":"10275","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7880034,"name":"drain","context":{"idset":"10276","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7886899,"name":"drain","context":{"idset":"10277","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7893965,"name":"drain","context":{"idset":"10278","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7900739,"name":"drain","context":{"idset":"10279","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7907746,"name":"drain","context":{"idset":"10280","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7914841,"name":"drain","context":{"idset":"10281","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7921603,"name":"drain","context":{"idset":"10282","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7928519,"name":"drain","context":{"idset":"10283","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7935426,"name":"drain","context":{"idset":"10284","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7942181,"name":"drain","context":{"idset":"10285","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7949085,"name":"drain","context":{"idset":"10286","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7955964,"name":"drain","context":{"idset":"10287","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7962806,"name":"drain","context":{"idset":"10288","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7969596,"name":"drain","context":{"idset":"10289","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7976491,"name":"drain","context":{"idset":"10290","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7983379,"name":"drain","context":{"idset":"10291","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7990148,"name":"drain","context":{"idset":"10292","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.7997031,"name":"drain","context":{"idset":"10293","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8003981,"name":"drain","context":{"idset":"10294","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8010752,"name":"drain","context":{"idset":"10295","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.801784,"name":"drain","context":{"idset":"10296","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8024814,"name":"drain","context":{"idset":"10297","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8031602,"name":"drain","context":{"idset":"10298","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8038535,"name":"drain","context":{"idset":"10299","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8045502,"name":"drain","context":{"idset":"10300","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8052313,"name":"drain","context":{"idset":"10301","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8059192,"name":"drain","context":{"idset":"10302","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8066132,"name":"drain","context":{"idset":"10304","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8073051,"name":"drain","context":{"idset":"10305","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8079889,"name":"drain","context":{"idset":"10306","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.808682,"name":"drain","context":{"idset":"10307","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8093944,"name":"drain","context":{"idset":"10308","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8100817,"name":"drain","context":{"idset":"10309","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8107853,"name":"drain","context":{"idset":"10310","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8114851,"name":"drain","context":{"idset":"10311","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.812171,"name":"drain","context":{"idset":"10312","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.812886,"name":"drain","context":{"idset":"10313","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8135979,"name":"drain","context":{"idset":"10314","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.814296,"name":"drain","context":{"idset":"10315","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8149822,"name":"drain","context":{"idset":"10316","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8156869,"name":"drain","context":{"idset":"10317","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8163836,"name":"drain","context":{"idset":"10318","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8170693,"name":"drain","context":{"idset":"10319","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8177671,"name":"drain","context":{"idset":"10320","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8184693,"name":"drain","context":{"idset":"10321","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8192341,"name":"drain","context":{"idset":"10322","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8199363,"name":"drain","context":{"idset":"10323","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8206398,"name":"drain","context":{"idset":"10324","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8213398,"name":"drain","context":{"idset":"10325","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8220305,"name":"drain","context":{"idset":"10326","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8227303,"name":"drain","context":{"idset":"10327","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8234341,"name":"drain","context":{"idset":"10328","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8241277,"name":"drain","context":{"idset":"10329","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.824832,"name":"drain","context":{"idset":"10330","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8255439,"name":"drain","context":{"idset":"10331","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8262496,"name":"drain","context":{"idset":"10332","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8270316,"name":"drain","context":{"idset":"10333","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8279626,"name":"drain","context":{"idset":"10334","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8287179,"name":"drain","context":{"idset":"10335","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8297181,"name":"drain","context":{"idset":"10336","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8305345,"name":"drain","context":{"idset":"10337","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8315363,"name":"drain","context":{"idset":"10338","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8323243,"name":"drain","context":{"idset":"10339","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8330812,"name":"drain","context":{"idset":"10340","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.833797,"name":"drain","context":{"idset":"10341","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8345113,"name":"drain","context":{"idset":"10342","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8352108,"name":"drain","context":{"idset":"10345","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8359258,"name":"drain","context":{"idset":"10346","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8366439,"name":"drain","context":{"idset":"10347","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8373766,"name":"drain","context":{"idset":"10348","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8380766,"name":"drain","context":{"idset":"10349","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.838814,"name":"drain","context":{"idset":"10350","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8395448,"name":"drain","context":{"idset":"10351","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8402493,"name":"drain","context":{"idset":"10352","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8409641,"name":"drain","context":{"idset":"10353","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8416772,"name":"drain","context":{"idset":"10354","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8423979,"name":"drain","context":{"idset":"10357","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8430998,"name":"drain","context":{"idset":"10358","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8438184,"name":"drain","context":{"idset":"10359","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8445356,"name":"drain","context":{"idset":"10360","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8452363,"name":"drain","context":{"idset":"10361","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8459775,"name":"drain","context":{"idset":"10362","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8466973,"name":"drain","context":{"idset":"10363","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8474131,"name":"drain","context":{"idset":"10364","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8481154,"name":"drain","context":{"idset":"10365","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8488317,"name":"drain","context":{"idset":"10366","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8495479,"name":"drain","context":{"idset":"10367","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8502533,"name":"drain","context":{"idset":"10368","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8509703,"name":"drain","context":{"idset":"10369","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8516893,"name":"drain","context":{"idset":"10370","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8524089,"name":"drain","context":{"idset":"10371","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8531156,"name":"drain","context":{"idset":"10372","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8538339,"name":"drain","context":{"idset":"10373","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8545616,"name":"drain","context":{"idset":"10374","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8552866,"name":"drain","context":{"idset":"10375","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8559964,"name":"drain","context":{"idset":"10376","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8567138,"name":"drain","context":{"idset":"10377","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8574705,"name":"drain","context":{"idset":"10378","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8581994,"name":"drain","context":{"idset":"10379","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8589301,"name":"drain","context":{"idset":"10380","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8596814,"name":"drain","context":{"idset":"10381","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8604298,"name":"drain","context":{"idset":"10382","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8611393,"name":"drain","context":{"idset":"10383","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8618772,"name":"drain","context":{"idset":"10384","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8626282,"name":"drain","context":{"idset":"10385","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8633609,"name":"drain","context":{"idset":"10386","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8640823,"name":"drain","context":{"idset":"10387","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8648272,"name":"drain","context":{"idset":"10388","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8655748,"name":"drain","context":{"idset":"10389","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.866308,"name":"drain","context":{"idset":"10390","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8670402,"name":"drain","context":{"idset":"10391","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8677895,"name":"drain","context":{"idset":"10392","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8685358,"name":"drain","context":{"idset":"10393","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.869292,"name":"drain","context":{"idset":"10394","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8700292,"name":"drain","context":{"idset":"10395","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8707633,"name":"drain","context":{"idset":"10396","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8715243,"name":"drain","context":{"idset":"10397","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8722613,"name":"drain","context":{"idset":"10398","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8730001,"name":"drain","context":{"idset":"10399","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8737504,"name":"drain","context":{"idset":"10400","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8745008,"name":"drain","context":{"idset":"10401","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8752205,"name":"drain","context":{"idset":"10402","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.875967,"name":"drain","context":{"idset":"10403","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8767276,"name":"drain","context":{"idset":"10404","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8774903,"name":"drain","context":{"idset":"10405","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8782103,"name":"drain","context":{"idset":"10406","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8789659,"name":"drain","context":{"idset":"10407","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8797224,"name":"drain","context":{"idset":"10408","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.880466,"name":"drain","context":{"idset":"10409","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8812056,"name":"drain","context":{"idset":"10410","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8819704,"name":"drain","context":{"idset":"10411","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8827114,"name":"drain","context":{"idset":"10412","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8834713,"name":"drain","context":{"idset":"10413","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8842137,"name":"drain","context":{"idset":"10414","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8849554,"name":"drain","context":{"idset":"10415","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8857186,"name":"drain","context":{"idset":"10416","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8864768,"name":"drain","context":{"idset":"10417","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8872135,"name":"drain","context":{"idset":"10418","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8879604,"name":"drain","context":{"idset":"10419","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8887148,"name":"drain","context":{"idset":"10420","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8894806,"name":"drain","context":{"idset":"10421","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8902094,"name":"drain","context":{"idset":"10422","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8909731,"name":"drain","context":{"idset":"10423","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8917391,"name":"drain","context":{"idset":"10424","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8924842,"name":"drain","context":{"idset":"10425","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8932292,"name":"drain","context":{"idset":"10426","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8939965,"name":"drain","context":{"idset":"10427","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.894752,"name":"drain","context":{"idset":"10428","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8955173,"name":"drain","context":{"idset":"10429","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8962796,"name":"drain","context":{"idset":"10430","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8970304,"name":"drain","context":{"idset":"10431","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8977799,"name":"drain","context":{"idset":"10432","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8985479,"name":"drain","context":{"idset":"10433","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.8993146,"name":"drain","context":{"idset":"10434","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9000452,"name":"drain","context":{"idset":"10435","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9008093,"name":"drain","context":{"idset":"10436","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9015768,"name":"drain","context":{"idset":"10437","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9023242,"name":"drain","context":{"idset":"10438","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.903074,"name":"drain","context":{"idset":"10439","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9038441,"name":"drain","context":{"idset":"10440","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9046087,"name":"drain","context":{"idset":"10441","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9053605,"name":"drain","context":{"idset":"10442","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9061131,"name":"drain","context":{"idset":"10443","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9068825,"name":"drain","context":{"idset":"10444","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9076383,"name":"drain","context":{"idset":"10445","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9084084,"name":"drain","context":{"idset":"10446","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9091628,"name":"drain","context":{"idset":"10447","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9099174,"name":"drain","context":{"idset":"10448","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9106827,"name":"drain","context":{"idset":"10449","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9114556,"name":"drain","context":{"idset":"10450","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9122086,"name":"drain","context":{"idset":"10451","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9129622,"name":"drain","context":{"idset":"10452","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9137363,"name":"drain","context":{"idset":"10453","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9145091,"name":"drain","context":{"idset":"10454","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9152472,"name":"drain","context":{"idset":"10455","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9160945,"name":"drain","context":{"idset":"10456","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9169171,"name":"drain","context":{"idset":"10457","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.917737,"name":"drain","context":{"idset":"10458","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9185164,"name":"drain","context":{"idset":"10459","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9192972,"name":"drain","context":{"idset":"10460","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9200597,"name":"drain","context":{"idset":"10461","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.920836,"name":"drain","context":{"idset":"10462","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.921603,"name":"drain","context":{"idset":"10463","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9223852,"name":"drain","context":{"idset":"10464","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9231505,"name":"drain","context":{"idset":"10465","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9239168,"name":"drain","context":{"idset":"10466","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9246945,"name":"drain","context":{"idset":"10467","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9254715,"name":"drain","context":{"idset":"10468","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9262331,"name":"drain","context":{"idset":"10485","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9270127,"name":"drain","context":{"idset":"10486","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9277925,"name":"drain","context":{"idset":"10487","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9285741,"name":"drain","context":{"idset":"10488","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9293399,"name":"drain","context":{"idset":"10489","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9301057,"name":"drain","context":{"idset":"10490","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.930898,"name":"drain","context":{"idset":"10491","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9316812,"name":"drain","context":{"idset":"10492","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9324479,"name":"drain","context":{"idset":"10493","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9332445,"name":"drain","context":{"idset":"10494","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9340601,"name":"drain","context":{"idset":"10495","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9348292,"name":"drain","context":{"idset":"10496","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9356434,"name":"drain","context":{"idset":"10497","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9364321,"name":"drain","context":{"idset":"10498","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9372032,"name":"drain","context":{"idset":"10499","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9379728,"name":"drain","context":{"idset":"10500","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9387615,"name":"drain","context":{"idset":"10501","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9395902,"name":"drain","context":{"idset":"10502","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9403853,"name":"drain","context":{"idset":"10503","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9411416,"name":"drain","context":{"idset":"10504","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9419389,"name":"drain","context":{"idset":"10505","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9427316,"name":"drain","context":{"idset":"10506","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9435003,"name":"drain","context":{"idset":"10507","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9442813,"name":"drain","context":{"idset":"10508","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9450576,"name":"drain","context":{"idset":"10509","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9458508,"name":"drain","context":{"idset":"10510","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9466181,"name":"drain","context":{"idset":"10511","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9474154,"name":"drain","context":{"idset":"10512","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9481881,"name":"drain","context":{"idset":"10513","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9489825,"name":"drain","context":{"idset":"10514","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9497581,"name":"drain","context":{"idset":"10515","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9505618,"name":"drain","context":{"idset":"10516","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.951354,"name":"drain","context":{"idset":"10519","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9521089,"name":"drain","context":{"idset":"10520","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9528978,"name":"drain","context":{"idset":"10521","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9536836,"name":"drain","context":{"idset":"10522","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9544795,"name":"drain","context":{"idset":"10523","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9552381,"name":"drain","context":{"idset":"10524","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.956028,"name":"drain","context":{"idset":"10525","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9568243,"name":"drain","context":{"idset":"10526","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9576445,"name":"drain","context":{"idset":"10527","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9584339,"name":"drain","context":{"idset":"10528","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9592175,"name":"drain","context":{"idset":"10529","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9600136,"name":"drain","context":{"idset":"10530","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9607921,"name":"drain","context":{"idset":"10531","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9615884,"name":"drain","context":{"idset":"10532","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9623878,"name":"drain","context":{"idset":"10533","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9631686,"name":"drain","context":{"idset":"10534","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9639513,"name":"drain","context":{"idset":"10535","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9647496,"name":"drain","context":{"idset":"10536","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9655533,"name":"drain","context":{"idset":"10537","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9663455,"name":"drain","context":{"idset":"10538","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9671082,"name":"drain","context":{"idset":"10539","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9679122,"name":"drain","context":{"idset":"10540","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9687154,"name":"drain","context":{"idset":"10541","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9694993,"name":"drain","context":{"idset":"10542","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9702938,"name":"drain","context":{"idset":"10543","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9710782,"name":"drain","context":{"idset":"10544","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9718804,"name":"drain","context":{"idset":"10545","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9726734,"name":"drain","context":{"idset":"10546","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9734778,"name":"drain","context":{"idset":"10547","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9742801,"name":"drain","context":{"idset":"10548","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9750667,"name":"drain","context":{"idset":"10549","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9758546,"name":"drain","context":{"idset":"10550","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9766574,"name":"drain","context":{"idset":"10551","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9774628,"name":"drain","context":{"idset":"10552","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9782512,"name":"drain","context":{"idset":"10553","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.979044,"name":"drain","context":{"idset":"10554","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9798515,"name":"drain","context":{"idset":"10555","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9806664,"name":"drain","context":{"idset":"10556","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9814801,"name":"drain","context":{"idset":"10557","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9822564,"name":"drain","context":{"idset":"10558","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9830599,"name":"drain","context":{"idset":"10559","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9838684,"name":"drain","context":{"idset":"10560","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.984668,"name":"drain","context":{"idset":"10561","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9854679,"name":"drain","context":{"idset":"10562","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9862571,"name":"drain","context":{"idset":"10563","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9870772,"name":"drain","context":{"idset":"10564","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.987865,"name":"drain","context":{"idset":"10565","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.988678,"name":"drain","context":{"idset":"10566","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9894888,"name":"drain","context":{"idset":"10567","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9903092,"name":"drain","context":{"idset":"10568","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9910865,"name":"drain","context":{"idset":"10569","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9919076,"name":"drain","context":{"idset":"10570","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9927187,"name":"drain","context":{"idset":"10571","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9935393,"name":"drain","context":{"idset":"10572","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9943354,"name":"drain","context":{"idset":"10573","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9951277,"name":"drain","context":{"idset":"10574","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9959493,"name":"drain","context":{"idset":"10575","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9967685,"name":"drain","context":{"idset":"10576","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9975662,"name":"drain","context":{"idset":"10577","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9983752,"name":"drain","context":{"idset":"10578","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9991708,"name":"drain","context":{"idset":"10579","reason":"broker was unresponsive"}}
+{"timestamp":1712365572.9999917,"name":"drain","context":{"idset":"10580","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0007906,"name":"drain","context":{"idset":"10581","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0016079,"name":"drain","context":{"idset":"10582","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0024312,"name":"drain","context":{"idset":"10583","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0032296,"name":"drain","context":{"idset":"10584","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0040317,"name":"drain","context":{"idset":"10585","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0048473,"name":"drain","context":{"idset":"10586","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0056653,"name":"drain","context":{"idset":"10587","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0064816,"name":"drain","context":{"idset":"10588","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0072932,"name":"drain","context":{"idset":"10589","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0080955,"name":"drain","context":{"idset":"10590","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0089159,"name":"drain","context":{"idset":"10591","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0097389,"name":"drain","context":{"idset":"10592","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0105457,"name":"drain","context":{"idset":"10593","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0113752,"name":"drain","context":{"idset":"10594","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0121825,"name":"drain","context":{"idset":"10595","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.013006,"name":"drain","context":{"idset":"10596","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.013828,"name":"drain","context":{"idset":"10597","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.014647,"name":"drain","context":{"idset":"10598","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0154741,"name":"drain","context":{"idset":"10599","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0162997,"name":"drain","context":{"idset":"10600","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0170829,"name":"drain","context":{"idset":"10601","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0179217,"name":"drain","context":{"idset":"10602","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0187483,"name":"drain","context":{"idset":"10603","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0195701,"name":"drain","context":{"idset":"10604","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.020402,"name":"drain","context":{"idset":"10605","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.021193,"name":"drain","context":{"idset":"10606","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0220175,"name":"drain","context":{"idset":"10607","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0228496,"name":"drain","context":{"idset":"10608","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.02367,"name":"drain","context":{"idset":"10609","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0244899,"name":"drain","context":{"idset":"10610","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0253193,"name":"drain","context":{"idset":"10611","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0261273,"name":"drain","context":{"idset":"10612","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0269647,"name":"drain","context":{"idset":"10613","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0277812,"name":"drain","context":{"idset":"10614","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0286078,"name":"drain","context":{"idset":"10615","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0294335,"name":"drain","context":{"idset":"10616","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0302451,"name":"drain","context":{"idset":"10617","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0310516,"name":"drain","context":{"idset":"10618","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0318937,"name":"drain","context":{"idset":"10619","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0327175,"name":"drain","context":{"idset":"10620","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0335743,"name":"drain","context":{"idset":"10621","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.03439,"name":"drain","context":{"idset":"10622","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0352039,"name":"drain","context":{"idset":"10623","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0360417,"name":"drain","context":{"idset":"10624","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.036876,"name":"drain","context":{"idset":"10625","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0377133,"name":"drain","context":{"idset":"10626","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0385361,"name":"drain","context":{"idset":"10627","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0394185,"name":"drain","context":{"idset":"10628","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0402396,"name":"drain","context":{"idset":"10630","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0410807,"name":"drain","context":{"idset":"10631","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0418937,"name":"drain","context":{"idset":"10632","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0427399,"name":"drain","context":{"idset":"10633","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0436072,"name":"drain","context":{"idset":"10635","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0444498,"name":"drain","context":{"idset":"10636","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0452611,"name":"drain","context":{"idset":"10640","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0460789,"name":"drain","context":{"idset":"10641","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0469213,"name":"drain","context":{"idset":"10645","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.04776,"name":"drain","context":{"idset":"10646","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0486219,"name":"drain","context":{"idset":"10647","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0494418,"name":"drain","context":{"idset":"10648","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.050257,"name":"drain","context":{"idset":"10649","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0511172,"name":"drain","context":{"idset":"10650","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.05196,"name":"drain","context":{"idset":"10651","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0528162,"name":"drain","context":{"idset":"10652","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0536404,"name":"drain","context":{"idset":"10653","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0544887,"name":"drain","context":{"idset":"10654","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0553267,"name":"drain","context":{"idset":"10655","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0561504,"name":"drain","context":{"idset":"10656","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0569766,"name":"drain","context":{"idset":"10658","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0578148,"name":"drain","context":{"idset":"10659","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.058665,"name":"drain","context":{"idset":"10660","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0595052,"name":"drain","context":{"idset":"10661","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0603325,"name":"drain","context":{"idset":"10662","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0611575,"name":"drain","context":{"idset":"10663","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0620008,"name":"drain","context":{"idset":"10664","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0628598,"name":"drain","context":{"idset":"10665","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0637088,"name":"drain","context":{"idset":"10666","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0645459,"name":"drain","context":{"idset":"10667","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0653944,"name":"drain","context":{"idset":"10668","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0662284,"name":"drain","context":{"idset":"10669","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0670981,"name":"drain","context":{"idset":"10670","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.067935,"name":"drain","context":{"idset":"10671","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0687804,"name":"drain","context":{"idset":"10672","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0696244,"name":"drain","context":{"idset":"10673","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0704739,"name":"drain","context":{"idset":"10674","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0713198,"name":"drain","context":{"idset":"10675","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0721307,"name":"drain","context":{"idset":"10676","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0729833,"name":"drain","context":{"idset":"10677","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0738344,"name":"drain","context":{"idset":"10678","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0746858,"name":"drain","context":{"idset":"10679","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0755274,"name":"drain","context":{"idset":"10680","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.076369,"name":"drain","context":{"idset":"10681","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0772023,"name":"drain","context":{"idset":"10682","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0780585,"name":"drain","context":{"idset":"10683","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0789101,"name":"drain","context":{"idset":"10684","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0797427,"name":"drain","context":{"idset":"10685","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0806029,"name":"drain","context":{"idset":"10686","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0814638,"name":"drain","context":{"idset":"10687","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0823247,"name":"drain","context":{"idset":"10688","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0831583,"name":"drain","context":{"idset":"10689","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0839906,"name":"drain","context":{"idset":"10690","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0848472,"name":"drain","context":{"idset":"10691","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0857019,"name":"drain","context":{"idset":"10692","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0865667,"name":"drain","context":{"idset":"10695","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0874145,"name":"drain","context":{"idset":"10696","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0882497,"name":"drain","context":{"idset":"10697","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0891149,"name":"drain","context":{"idset":"10698","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0899694,"name":"drain","context":{"idset":"10699","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0908313,"name":"drain","context":{"idset":"10700","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.091676,"name":"drain","context":{"idset":"10701","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0925295,"name":"drain","context":{"idset":"10703","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0933933,"name":"drain","context":{"idset":"10704","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0942342,"name":"drain","context":{"idset":"10705","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0950918,"name":"drain","context":{"idset":"10706","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0959344,"name":"drain","context":{"idset":"10707","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0967982,"name":"drain","context":{"idset":"10708","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0976555,"name":"drain","context":{"idset":"10709","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.0985098,"name":"drain","context":{"idset":"10710","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.099354,"name":"drain","context":{"idset":"10711","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1001899,"name":"drain","context":{"idset":"10713","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1010525,"name":"drain","context":{"idset":"10714","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1019149,"name":"drain","context":{"idset":"10715","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1027777,"name":"drain","context":{"idset":"10716","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1036193,"name":"drain","context":{"idset":"10717","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1044838,"name":"drain","context":{"idset":"10719","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1053448,"name":"drain","context":{"idset":"10720","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1061928,"name":"drain","context":{"idset":"10721","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1070554,"name":"drain","context":{"idset":"10722","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1079028,"name":"drain","context":{"idset":"10723","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1087658,"name":"drain","context":{"idset":"10724","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1096323,"name":"drain","context":{"idset":"10725","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1105011,"name":"drain","context":{"idset":"10726","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1113672,"name":"drain","context":{"idset":"10727","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1121993,"name":"drain","context":{"idset":"10728","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1130645,"name":"drain","context":{"idset":"10729","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1139362,"name":"drain","context":{"idset":"10730","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1148014,"name":"drain","context":{"idset":"10731","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1156693,"name":"drain","context":{"idset":"10732","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1165202,"name":"drain","context":{"idset":"10733","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.117388,"name":"drain","context":{"idset":"10734","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1182439,"name":"drain","context":{"idset":"10735","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.119113,"name":"drain","context":{"idset":"10736","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1199801,"name":"drain","context":{"idset":"10737","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1208405,"name":"drain","context":{"idset":"10738","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1217148,"name":"drain","context":{"idset":"10741","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1225886,"name":"drain","context":{"idset":"10742","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.123461,"name":"drain","context":{"idset":"10743","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1243222,"name":"drain","context":{"idset":"10744","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1251574,"name":"drain","context":{"idset":"10745","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1260245,"name":"drain","context":{"idset":"10746","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.126895,"name":"drain","context":{"idset":"10747","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1277659,"name":"drain","context":{"idset":"10748","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1286366,"name":"drain","context":{"idset":"10749","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1294935,"name":"drain","context":{"idset":"10750","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1303656,"name":"drain","context":{"idset":"10751","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1312227,"name":"drain","context":{"idset":"10752","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1320941,"name":"drain","context":{"idset":"10753","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1329622,"name":"drain","context":{"idset":"10754","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1338165,"name":"drain","context":{"idset":"10755","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1346927,"name":"drain","context":{"idset":"10756","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1355712,"name":"drain","context":{"idset":"10757","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1364465,"name":"drain","context":{"idset":"10758","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1373212,"name":"drain","context":{"idset":"10759","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1381626,"name":"drain","context":{"idset":"10760","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1390309,"name":"drain","context":{"idset":"10761","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1399415,"name":"drain","context":{"idset":"10762","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1408238,"name":"drain","context":{"idset":"10763","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1417074,"name":"drain","context":{"idset":"10764","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1425729,"name":"drain","context":{"idset":"10765","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1434462,"name":"drain","context":{"idset":"10766","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1443226,"name":"drain","context":{"idset":"10767","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1451817,"name":"drain","context":{"idset":"10768","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1460595,"name":"drain","context":{"idset":"10769","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1469145,"name":"drain","context":{"idset":"10770","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1477904,"name":"drain","context":{"idset":"10771","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1486702,"name":"drain","context":{"idset":"10772","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1495631,"name":"drain","context":{"idset":"10773","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1504474,"name":"drain","context":{"idset":"10774","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1513188,"name":"drain","context":{"idset":"10775","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1521673,"name":"drain","context":{"idset":"10776","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1530473,"name":"drain","context":{"idset":"10777","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1539307,"name":"drain","context":{"idset":"10778","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1548109,"name":"drain","context":{"idset":"10779","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1556981,"name":"drain","context":{"idset":"10780","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1565633,"name":"drain","context":{"idset":"10781","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1574452,"name":"drain","context":{"idset":"10782","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1583347,"name":"drain","context":{"idset":"10783","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1592026,"name":"drain","context":{"idset":"10784","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1600885,"name":"drain","context":{"idset":"10785","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1609616,"name":"drain","context":{"idset":"10786","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1618497,"name":"drain","context":{"idset":"10787","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1628067,"name":"drain","context":{"idset":"10788","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1637442,"name":"drain","context":{"idset":"10789","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1646264,"name":"drain","context":{"idset":"10790","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.165499,"name":"drain","context":{"idset":"10791","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1663668,"name":"drain","context":{"idset":"10792","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1672223,"name":"drain","context":{"idset":"10793","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1680927,"name":"drain","context":{"idset":"10794","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.168962,"name":"drain","context":{"idset":"10795","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1698296,"name":"drain","context":{"idset":"10796","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.170702,"name":"drain","context":{"idset":"10797","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1715734,"name":"drain","context":{"idset":"10798","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1724429,"name":"drain","context":{"idset":"10799","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1733344,"name":"drain","context":{"idset":"10800","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.174211,"name":"drain","context":{"idset":"10801","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1751013,"name":"drain","context":{"idset":"10802","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1759739,"name":"drain","context":{"idset":"10803","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1768665,"name":"drain","context":{"idset":"10804","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1777604,"name":"drain","context":{"idset":"10805","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1786463,"name":"drain","context":{"idset":"10806","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1795409,"name":"drain","context":{"idset":"10807","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1804161,"name":"drain","context":{"idset":"10808","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1813056,"name":"drain","context":{"idset":"10809","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1821861,"name":"drain","context":{"idset":"10810","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1830876,"name":"drain","context":{"idset":"10811","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1839852,"name":"drain","context":{"idset":"10812","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1848783,"name":"drain","context":{"idset":"10813","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1857576,"name":"drain","context":{"idset":"10814","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1866503,"name":"drain","context":{"idset":"10815","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1875484,"name":"drain","context":{"idset":"10816","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1884432,"name":"drain","context":{"idset":"10817","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1893361,"name":"drain","context":{"idset":"10818","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1902153,"name":"drain","context":{"idset":"10819","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1910923,"name":"drain","context":{"idset":"10820","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1919847,"name":"drain","context":{"idset":"10821","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1928813,"name":"drain","context":{"idset":"10822","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.193784,"name":"drain","context":{"idset":"10823","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1946874,"name":"drain","context":{"idset":"10824","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1955686,"name":"drain","context":{"idset":"10825","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1964586,"name":"drain","context":{"idset":"10826","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1973672,"name":"drain","context":{"idset":"10827","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1982546,"name":"drain","context":{"idset":"10828","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.1991575,"name":"drain","context":{"idset":"10829","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2000601,"name":"drain","context":{"idset":"10830","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2009485,"name":"drain","context":{"idset":"10831","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2018456,"name":"drain","context":{"idset":"10832","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2027476,"name":"drain","context":{"idset":"10833","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2036524,"name":"drain","context":{"idset":"10834","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2045562,"name":"drain","context":{"idset":"10835","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2054584,"name":"drain","context":{"idset":"10836","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2063508,"name":"drain","context":{"idset":"10837","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2072315,"name":"drain","context":{"idset":"10838","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2081363,"name":"drain","context":{"idset":"10839","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2090437,"name":"drain","context":{"idset":"10840","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2099521,"name":"drain","context":{"idset":"10841","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2108572,"name":"drain","context":{"idset":"10842","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2117457,"name":"drain","context":{"idset":"10843","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2126482,"name":"drain","context":{"idset":"10844","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2135534,"name":"drain","context":{"idset":"10845","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2144616,"name":"drain","context":{"idset":"10846","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2153721,"name":"drain","context":{"idset":"10847","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2162795,"name":"drain","context":{"idset":"10848","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2171559,"name":"drain","context":{"idset":"10849","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2180581,"name":"drain","context":{"idset":"10850","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2189653,"name":"drain","context":{"idset":"10851","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2198789,"name":"drain","context":{"idset":"10852","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.220787,"name":"drain","context":{"idset":"10853","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2217011,"name":"drain","context":{"idset":"10854","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2226045,"name":"drain","context":{"idset":"10855","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2235055,"name":"drain","context":{"idset":"10856","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2244158,"name":"drain","context":{"idset":"10857","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2253308,"name":"drain","context":{"idset":"10858","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2262266,"name":"drain","context":{"idset":"10859","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2271376,"name":"drain","context":{"idset":"10860","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.228049,"name":"drain","context":{"idset":"10861","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2289436,"name":"drain","context":{"idset":"10862","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.229857,"name":"drain","context":{"idset":"10863","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.230773,"name":"drain","context":{"idset":"10864","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2316828,"name":"drain","context":{"idset":"10865","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2325985,"name":"drain","context":{"idset":"10866","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2335165,"name":"drain","context":{"idset":"10867","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2344093,"name":"drain","context":{"idset":"10868","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.235323,"name":"drain","context":{"idset":"10869","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2362199,"name":"drain","context":{"idset":"10870","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2371399,"name":"drain","context":{"idset":"10873","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2380564,"name":"drain","context":{"idset":"10874","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.238975,"name":"drain","context":{"idset":"10875","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2399347,"name":"drain","context":{"idset":"10876","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2408431,"name":"drain","context":{"idset":"10877","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2417583,"name":"drain","context":{"idset":"10878","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2426755,"name":"drain","context":{"idset":"10879","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2435966,"name":"drain","context":{"idset":"10880","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2445195,"name":"drain","context":{"idset":"10881","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2454352,"name":"drain","context":{"idset":"10882","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.246346,"name":"drain","context":{"idset":"10883","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2472394,"name":"drain","context":{"idset":"10884","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2481606,"name":"drain","context":{"idset":"10885","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2490864,"name":"drain","context":{"idset":"10886","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.250006,"name":"drain","context":{"idset":"10887","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2509241,"name":"drain","context":{"idset":"10888","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2518539,"name":"drain","context":{"idset":"10889","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2527649,"name":"drain","context":{"idset":"10890","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2536905,"name":"drain","context":{"idset":"10891","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2546239,"name":"drain","context":{"idset":"10892","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2555518,"name":"drain","context":{"idset":"10893","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2564771,"name":"drain","context":{"idset":"10894","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2574019,"name":"drain","context":{"idset":"10895","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2583287,"name":"drain","context":{"idset":"10896","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2592206,"name":"drain","context":{"idset":"10897","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2601435,"name":"drain","context":{"idset":"10898","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2610712,"name":"drain","context":{"idset":"10899","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2619967,"name":"drain","context":{"idset":"10900","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2631607,"name":"drain","context":{"idset":"10901","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2640898,"name":"drain","context":{"idset":"10902","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2650228,"name":"drain","context":{"idset":"10903","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2659504,"name":"drain","context":{"idset":"10904","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2668667,"name":"drain","context":{"idset":"10905","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2677903,"name":"drain","context":{"idset":"10906","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2687852,"name":"drain","context":{"idset":"10907","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2697341,"name":"drain","context":{"idset":"10908","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2706664,"name":"drain","context":{"idset":"10909","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2715962,"name":"drain","context":{"idset":"10910","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2725284,"name":"drain","context":{"idset":"10913","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2734654,"name":"drain","context":{"idset":"10914","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2743835,"name":"drain","context":{"idset":"10915","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2753191,"name":"drain","context":{"idset":"10916","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2762341,"name":"drain","context":{"idset":"10917","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2771664,"name":"drain","context":{"idset":"10918","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2780974,"name":"drain","context":{"idset":"10919","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2790387,"name":"drain","context":{"idset":"10920","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2799811,"name":"drain","context":{"idset":"10921","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2809002,"name":"drain","context":{"idset":"10922","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2818317,"name":"drain","context":{"idset":"10923","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2827816,"name":"drain","context":{"idset":"10924","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2837226,"name":"drain","context":{"idset":"10925","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2846601,"name":"drain","context":{"idset":"10926","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2855916,"name":"drain","context":{"idset":"10927","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2865283,"name":"drain","context":{"idset":"10928","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.287472,"name":"drain","context":{"idset":"10929","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2883973,"name":"drain","context":{"idset":"10930","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2893355,"name":"drain","context":{"idset":"10931","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2902584,"name":"drain","context":{"idset":"10932","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2911959,"name":"drain","context":{"idset":"10933","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2921329,"name":"drain","context":{"idset":"10934","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2930727,"name":"drain","context":{"idset":"10935","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2940137,"name":"drain","context":{"idset":"10936","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2949338,"name":"drain","context":{"idset":"10937","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2958746,"name":"drain","context":{"idset":"10938","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.296819,"name":"drain","context":{"idset":"10939","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2977691,"name":"drain","context":{"idset":"10940","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2987108,"name":"drain","context":{"idset":"10941","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.2996583,"name":"drain","context":{"idset":"10942","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3005991,"name":"drain","context":{"idset":"10945","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3015473,"name":"drain","context":{"idset":"10946","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3024745,"name":"drain","context":{"idset":"10947","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3034151,"name":"drain","context":{"idset":"10948","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3043621,"name":"drain","context":{"idset":"10949","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3053155,"name":"drain","context":{"idset":"10950","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3062425,"name":"drain","context":{"idset":"10951","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3071826,"name":"drain","context":{"idset":"10952","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3081312,"name":"drain","context":{"idset":"10953","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3090641,"name":"drain","context":{"idset":"10954","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.310003,"name":"drain","context":{"idset":"10955","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3109515,"name":"drain","context":{"idset":"10956","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3118947,"name":"drain","context":{"idset":"10957","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3128393,"name":"drain","context":{"idset":"10958","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3137898,"name":"drain","context":{"idset":"10959","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3147397,"name":"drain","context":{"idset":"10960","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3156886,"name":"drain","context":{"idset":"10961","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3166449,"name":"drain","context":{"idset":"10962","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.317574,"name":"drain","context":{"idset":"10963","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3185229,"name":"drain","context":{"idset":"10964","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.319474,"name":"drain","context":{"idset":"10965","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3204255,"name":"drain","context":{"idset":"10966","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3213785,"name":"drain","context":{"idset":"10967","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.32233,"name":"drain","context":{"idset":"10968","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3232865,"name":"drain","context":{"idset":"10969","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.324223,"name":"drain","context":{"idset":"10970","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3251567,"name":"drain","context":{"idset":"10971","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3261015,"name":"drain","context":{"idset":"10972","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3270519,"name":"drain","context":{"idset":"10973","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3280025,"name":"drain","context":{"idset":"10974","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3289611,"name":"drain","context":{"idset":"10975","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3299177,"name":"drain","context":{"idset":"10976","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3308704,"name":"drain","context":{"idset":"10977","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3318224,"name":"drain","context":{"idset":"10978","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3327558,"name":"drain","context":{"idset":"10979","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3337064,"name":"drain","context":{"idset":"10980","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3346622,"name":"drain","context":{"idset":"10981","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3356183,"name":"drain","context":{"idset":"10982","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3365712,"name":"drain","context":{"idset":"10983","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.337528,"name":"drain","context":{"idset":"10984","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3384824,"name":"drain","context":{"idset":"10985","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.339443,"name":"drain","context":{"idset":"10986","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3404334,"name":"drain","context":{"idset":"10987","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3413815,"name":"drain","context":{"idset":"10988","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3423364,"name":"drain","context":{"idset":"10989","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3432975,"name":"drain","context":{"idset":"10990","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3442407,"name":"drain","context":{"idset":"10991","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3451977,"name":"drain","context":{"idset":"10992","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3461599,"name":"drain","context":{"idset":"10993","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3471215,"name":"drain","context":{"idset":"10994","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3480818,"name":"drain","context":{"idset":"10995","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3490312,"name":"drain","context":{"idset":"10996","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3499928,"name":"drain","context":{"idset":"10997","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3509524,"name":"drain","context":{"idset":"10998","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3519187,"name":"drain","context":{"idset":"11001","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3528781,"name":"drain","context":{"idset":"11002","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3538444,"name":"drain","context":{"idset":"11003","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3548095,"name":"drain","context":{"idset":"11004","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3557754,"name":"drain","context":{"idset":"11005","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3567405,"name":"drain","context":{"idset":"11006","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3577011,"name":"drain","context":{"idset":"11007","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3586533,"name":"drain","context":{"idset":"11008","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3596182,"name":"drain","context":{"idset":"11009","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3605893,"name":"drain","context":{"idset":"11010","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3615584,"name":"drain","context":{"idset":"11011","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3625238,"name":"drain","context":{"idset":"11012","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3634937,"name":"drain","context":{"idset":"11013","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3644629,"name":"drain","context":{"idset":"11014","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3654284,"name":"drain","context":{"idset":"11015","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3663948,"name":"drain","context":{"idset":"11016","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3673639,"name":"drain","context":{"idset":"11017","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.36834,"name":"drain","context":{"idset":"11018","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3693151,"name":"drain","context":{"idset":"11019","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3702934,"name":"drain","context":{"idset":"11020","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.371249,"name":"drain","context":{"idset":"11021","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3722186,"name":"drain","context":{"idset":"11022","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3731875,"name":"drain","context":{"idset":"11023","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3741581,"name":"drain","context":{"idset":"11024","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3751321,"name":"drain","context":{"idset":"11025","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3760908,"name":"drain","context":{"idset":"11026","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.377053,"name":"drain","context":{"idset":"11027","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3780262,"name":"drain","context":{"idset":"11028","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3789978,"name":"drain","context":{"idset":"11029","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3799777,"name":"drain","context":{"idset":"11030","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3809519,"name":"drain","context":{"idset":"11031","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3819268,"name":"drain","context":{"idset":"11032","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3829026,"name":"drain","context":{"idset":"11033","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3838804,"name":"drain","context":{"idset":"11034","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3848588,"name":"drain","context":{"idset":"11035","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3858273,"name":"drain","context":{"idset":"11036","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.38679,"name":"drain","context":{"idset":"11037","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3877676,"name":"drain","context":{"idset":"11038","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3887463,"name":"drain","context":{"idset":"11039","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3897262,"name":"drain","context":{"idset":"11040","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3907056,"name":"drain","context":{"idset":"11041","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3916833,"name":"drain","context":{"idset":"11042","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3926568,"name":"drain","context":{"idset":"11043","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3936388,"name":"drain","context":{"idset":"11044","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3946178,"name":"drain","context":{"idset":"11045","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3955972,"name":"drain","context":{"idset":"11046","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.396558,"name":"drain","context":{"idset":"11047","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3975372,"name":"drain","context":{"idset":"11048","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3985217,"name":"drain","context":{"idset":"11049","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.3995016,"name":"drain","context":{"idset":"11050","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4004865,"name":"drain","context":{"idset":"11051","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4014704,"name":"drain","context":{"idset":"11052","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4024549,"name":"drain","context":{"idset":"11053","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4034369,"name":"drain","context":{"idset":"11054","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4044204,"name":"drain","context":{"idset":"11055","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4054,"name":"drain","context":{"idset":"11056","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4063787,"name":"drain","context":{"idset":"11057","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4073567,"name":"drain","context":{"idset":"11058","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4083426,"name":"drain","context":{"idset":"11059","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4093292,"name":"drain","context":{"idset":"11060","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4103243,"name":"drain","context":{"idset":"11061","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4113164,"name":"drain","context":{"idset":"11062","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4123027,"name":"drain","context":{"idset":"11063","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4132836,"name":"drain","context":{"idset":"11064","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4142513,"name":"drain","context":{"idset":"11065","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4152372,"name":"drain","context":{"idset":"11066","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4162254,"name":"drain","context":{"idset":"11067","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.417208,"name":"drain","context":{"idset":"11068","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4181757,"name":"drain","context":{"idset":"11069","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4191716,"name":"drain","context":{"idset":"11070","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4201617,"name":"drain","context":{"idset":"11071","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.421154,"name":"drain","context":{"idset":"11072","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4221473,"name":"drain","context":{"idset":"11073","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4231369,"name":"drain","context":{"idset":"11074","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4241257,"name":"drain","context":{"idset":"11075","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4251156,"name":"drain","context":{"idset":"11076","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4261086,"name":"drain","context":{"idset":"11077","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4271007,"name":"drain","context":{"idset":"11078","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4280951,"name":"drain","context":{"idset":"11079","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4290886,"name":"drain","context":{"idset":"11080","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4300656,"name":"drain","context":{"idset":"11081","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4310572,"name":"drain","context":{"idset":"11082","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4320664,"name":"drain","context":{"idset":"11083","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4330556,"name":"drain","context":{"idset":"11084","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4340529,"name":"drain","context":{"idset":"11085","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.435045,"name":"drain","context":{"idset":"11086","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4360409,"name":"drain","context":{"idset":"11087","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4370358,"name":"drain","context":{"idset":"11088","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4380314,"name":"drain","context":{"idset":"11089","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4390271,"name":"drain","context":{"idset":"11090","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.440032,"name":"drain","context":{"idset":"11091","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4410479,"name":"drain","context":{"idset":"11092","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4420466,"name":"drain","context":{"idset":"11093","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.443027,"name":"drain","context":{"idset":"11094","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4440267,"name":"drain","context":{"idset":"11095","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4450247,"name":"drain","context":{"idset":"11096","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4460225,"name":"drain","context":{"idset":"11097","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4470212,"name":"drain","context":{"idset":"11098","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4480162,"name":"drain","context":{"idset":"11099","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4490149,"name":"drain","context":{"idset":"11100","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4500136,"name":"drain","context":{"idset":"11101","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4510171,"name":"drain","context":{"idset":"11102","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4520192,"name":"drain","context":{"idset":"11103","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4530184,"name":"drain","context":{"idset":"11104","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4540186,"name":"drain","context":{"idset":"11105","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4550183,"name":"drain","context":{"idset":"11106","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4560068,"name":"drain","context":{"idset":"11107","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4570065,"name":"drain","context":{"idset":"11108","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4580102,"name":"drain","context":{"idset":"11109","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4590151,"name":"drain","context":{"idset":"11110","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4600148,"name":"drain","context":{"idset":"11111","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.46102,"name":"drain","context":{"idset":"11112","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4620256,"name":"drain","context":{"idset":"11113","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4630289,"name":"drain","context":{"idset":"11114","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4640336,"name":"drain","context":{"idset":"11117","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4650395,"name":"drain","context":{"idset":"11118","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4660468,"name":"drain","context":{"idset":"11119","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4670546,"name":"drain","context":{"idset":"11120","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4680631,"name":"drain","context":{"idset":"11121","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4690628,"name":"drain","context":{"idset":"11122","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4700487,"name":"drain","context":{"idset":"11123","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4710498,"name":"drain","context":{"idset":"11124","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4720569,"name":"drain","context":{"idset":"11125","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4730661,"name":"drain","context":{"idset":"11126","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4740667,"name":"drain","context":{"idset":"11127","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4750795,"name":"drain","context":{"idset":"11128","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4760923,"name":"drain","context":{"idset":"11129","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4771039,"name":"drain","context":{"idset":"11130","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4781134,"name":"drain","context":{"idset":"11131","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4791224,"name":"drain","context":{"idset":"11132","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4801333,"name":"drain","context":{"idset":"11133","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4811442,"name":"drain","context":{"idset":"11135","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4821565,"name":"drain","context":{"idset":"11136","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4831724,"name":"drain","context":{"idset":"11137","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.484189,"name":"drain","context":{"idset":"11138","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4852064,"name":"drain","context":{"idset":"11139","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4861984,"name":"drain","context":{"idset":"11141","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4872081,"name":"drain","context":{"idset":"11142","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4882188,"name":"drain","context":{"idset":"11143","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4892297,"name":"drain","context":{"idset":"11144","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.490242,"name":"drain","context":{"idset":"11145","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4912558,"name":"drain","context":{"idset":"11146","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4922874,"name":"drain","context":{"idset":"11147","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4933014,"name":"drain","context":{"idset":"11148","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4943211,"name":"drain","context":{"idset":"11149","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4953368,"name":"drain","context":{"idset":"11150","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4963517,"name":"drain","context":{"idset":"11151","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4973645,"name":"drain","context":{"idset":"11152","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4983842,"name":"drain","context":{"idset":"11153","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.4994006,"name":"drain","context":{"idset":"11154","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5004199,"name":"drain","context":{"idset":"11155","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.501442,"name":"drain","context":{"idset":"11156","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5024519,"name":"drain","context":{"idset":"11157","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5034575,"name":"drain","context":{"idset":"11158","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5044773,"name":"drain","context":{"idset":"11159","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5054955,"name":"drain","context":{"idset":"11160","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5065186,"name":"drain","context":{"idset":"11161","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5075397,"name":"drain","context":{"idset":"11162","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5085614,"name":"drain","context":{"idset":"11163","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5095837,"name":"drain","context":{"idset":"11164","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5106013,"name":"drain","context":{"idset":"11165","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5116234,"name":"drain","context":{"idset":"11166","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5126479,"name":"drain","context":{"idset":"11167","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5136647,"name":"drain","context":{"idset":"11168","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5146797,"name":"drain","context":{"idset":"11169","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5157032,"name":"drain","context":{"idset":"11170","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5167339,"name":"drain","context":{"idset":"11171","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5177591,"name":"drain","context":{"idset":"11172","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5187826,"name":"drain","context":{"idset":"11173","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5198059,"name":"drain","context":{"idset":"11174","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5208292,"name":"drain","context":{"idset":"11175","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5218394,"name":"drain","context":{"idset":"11176","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5228601,"name":"drain","context":{"idset":"11177","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5238829,"name":"drain","context":{"idset":"11178","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5249083,"name":"drain","context":{"idset":"11179","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5259378,"name":"drain","context":{"idset":"11180","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5269723,"name":"drain","context":{"idset":"11181","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5280004,"name":"drain","context":{"idset":"11182","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.529036,"name":"drain","context":{"idset":"11183","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5300672,"name":"drain","context":{"idset":"11184","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5311015,"name":"drain","context":{"idset":"11185","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5321302,"name":"drain","context":{"idset":"11186","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5331612,"name":"drain","context":{"idset":"11187","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5341883,"name":"drain","context":{"idset":"11188","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5352159,"name":"drain","context":{"idset":"11189","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5362518,"name":"drain","context":{"idset":"11190","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5372992,"name":"drain","context":{"idset":"11191","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.538331,"name":"drain","context":{"idset":"11192","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5393682,"name":"drain","context":{"idset":"11193","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5404055,"name":"drain","context":{"idset":"11194","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5414383,"name":"drain","context":{"idset":"11195","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5424705,"name":"drain","context":{"idset":"11196","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5434897,"name":"drain","context":{"idset":"11197","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5445127,"name":"drain","context":{"idset":"11198","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5455496,"name":"drain","context":{"idset":"11199","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5465832,"name":"drain","context":{"idset":"11200","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5476203,"name":"drain","context":{"idset":"11201","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5486538,"name":"drain","context":{"idset":"11202","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5496891,"name":"drain","context":{"idset":"11203","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5507255,"name":"drain","context":{"idset":"11204","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5517662,"name":"drain","context":{"idset":"11205","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5528021,"name":"drain","context":{"idset":"11206","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5538433,"name":"drain","context":{"idset":"11207","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5548754,"name":"drain","context":{"idset":"11208","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5559142,"name":"drain","context":{"idset":"11209","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5569506,"name":"drain","context":{"idset":"11210","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5579844,"name":"drain","context":{"idset":"11211","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5590277,"name":"drain","context":{"idset":"11212","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5600691,"name":"drain","context":{"idset":"11213","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5611057,"name":"drain","context":{"idset":"11214","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5621431,"name":"drain","context":{"idset":"11215","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5631819,"name":"drain","context":{"idset":"11216","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5642264,"name":"drain","context":{"idset":"11217","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5652785,"name":"drain","context":{"idset":"11218","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5663218,"name":"drain","context":{"idset":"11219","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5673594,"name":"drain","context":{"idset":"11220","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5684013,"name":"drain","context":{"idset":"11221","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5694482,"name":"drain","context":{"idset":"11222","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5704722,"name":"drain","context":{"idset":"11223","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5715098,"name":"drain","context":{"idset":"11224","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5725524,"name":"drain","context":{"idset":"11225","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.573597,"name":"drain","context":{"idset":"11227","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5746396,"name":"drain","context":{"idset":"11228","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5756869,"name":"drain","context":{"idset":"11229","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5767438,"name":"drain","context":{"idset":"11230","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5777903,"name":"drain","context":{"idset":"11231","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5788343,"name":"drain","context":{"idset":"11232","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5798798,"name":"drain","context":{"idset":"11234","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5809278,"name":"drain","context":{"idset":"11237","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5819693,"name":"drain","context":{"idset":"11238","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5830324,"name":"drain","context":{"idset":"11239","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5841503,"name":"drain","context":{"idset":"11240","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5852818,"name":"drain","context":{"idset":"11241","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5864182,"name":"drain","context":{"idset":"11242","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5875175,"name":"drain","context":{"idset":"11243","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5885744,"name":"drain","context":{"idset":"11244","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5896349,"name":"drain","context":{"idset":"11245","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5907195,"name":"drain","context":{"idset":"11246","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.591805,"name":"drain","context":{"idset":"11248","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5928757,"name":"drain","context":{"idset":"11250","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5939326,"name":"drain","context":{"idset":"11251","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5949852,"name":"drain","context":{"idset":"11252","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5960474,"name":"drain","context":{"idset":"11253","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5971084,"name":"drain","context":{"idset":"11254","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5981708,"name":"drain","context":{"idset":"11255","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.5992353,"name":"drain","context":{"idset":"11256","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6003189,"name":"drain","context":{"idset":"11257","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6013811,"name":"drain","context":{"idset":"11258","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6024418,"name":"drain","context":{"idset":"11259","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6035063,"name":"drain","context":{"idset":"11260","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6045749,"name":"drain","context":{"idset":"11261","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6056366,"name":"drain","context":{"idset":"11262","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6066999,"name":"drain","context":{"idset":"11263","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6077602,"name":"drain","context":{"idset":"11264","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6088283,"name":"drain","context":{"idset":"11265","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6098914,"name":"drain","context":{"idset":"11266","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6109536,"name":"drain","context":{"idset":"11267","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6120207,"name":"drain","context":{"idset":"11268","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6130877,"name":"drain","context":{"idset":"11269","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6141493,"name":"drain","context":{"idset":"11270","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.615212,"name":"drain","context":{"idset":"11271","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6162913,"name":"drain","context":{"idset":"11272","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6173549,"name":"drain","context":{"idset":"11273","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6184206,"name":"drain","context":{"idset":"11274","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6194868,"name":"drain","context":{"idset":"11275","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6205513,"name":"drain","context":{"idset":"11276","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6216207,"name":"drain","context":{"idset":"11277","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6226838,"name":"drain","context":{"idset":"11278","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6237607,"name":"drain","context":{"idset":"11279","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6248219,"name":"drain","context":{"idset":"11280","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6258876,"name":"drain","context":{"idset":"11281","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6269495,"name":"drain","context":{"idset":"11282","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6280189,"name":"drain","context":{"idset":"11283","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6291003,"name":"drain","context":{"idset":"11284","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6301715,"name":"drain","context":{"idset":"11285","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6312401,"name":"drain","context":{"idset":"11286","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6323245,"name":"drain","context":{"idset":"11287","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6333959,"name":"drain","context":{"idset":"11288","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6344633,"name":"drain","context":{"idset":"11289","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6355662,"name":"drain","context":{"idset":"11290","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6366439,"name":"drain","context":{"idset":"11291","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6377144,"name":"drain","context":{"idset":"11292","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6387858,"name":"drain","context":{"idset":"11293","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6400592,"name":"drain","context":{"idset":"11294","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6411662,"name":"drain","context":{"idset":"11295","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6422822,"name":"drain","context":{"idset":"11296","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6433797,"name":"drain","context":{"idset":"11297","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6444609,"name":"drain","context":{"idset":"11298","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6455371,"name":"drain","context":{"idset":"11299","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6466143,"name":"drain","context":{"idset":"11300","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6476901,"name":"drain","context":{"idset":"11301","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6487713,"name":"drain","context":{"idset":"11302","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6498525,"name":"drain","context":{"idset":"11303","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6509335,"name":"drain","context":{"idset":"11304","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6520147,"name":"drain","context":{"idset":"11305","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6530995,"name":"drain","context":{"idset":"11306","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6541793,"name":"drain","context":{"idset":"11307","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6552606,"name":"drain","context":{"idset":"11308","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6563673,"name":"drain","context":{"idset":"11309","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6574485,"name":"drain","context":{"idset":"11310","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6585312,"name":"drain","context":{"idset":"11311","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6596072,"name":"drain","context":{"idset":"11312","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6606891,"name":"drain","context":{"idset":"11313","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6617696,"name":"drain","context":{"idset":"11314","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6628633,"name":"drain","context":{"idset":"11315","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6639454,"name":"drain","context":{"idset":"11316","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6650267,"name":"drain","context":{"idset":"11317","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6661057,"name":"drain","context":{"idset":"11318","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6671877,"name":"drain","context":{"idset":"11319","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6682887,"name":"drain","context":{"idset":"11320","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6693714,"name":"drain","context":{"idset":"11321","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6704545,"name":"drain","context":{"idset":"11322","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6715362,"name":"drain","context":{"idset":"11323","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6726227,"name":"drain","context":{"idset":"11324","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6737058,"name":"drain","context":{"idset":"11325","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6747904,"name":"drain","context":{"idset":"11326","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6758742,"name":"drain","context":{"idset":"11327","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6769655,"name":"drain","context":{"idset":"11328","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6780519,"name":"drain","context":{"idset":"11331","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6791384,"name":"drain","context":{"idset":"11332","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6802421,"name":"drain","context":{"idset":"11333","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6813529,"name":"drain","context":{"idset":"11334","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6824431,"name":"drain","context":{"idset":"11335","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6835349,"name":"drain","context":{"idset":"11336","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6846213,"name":"drain","context":{"idset":"11337","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6857021,"name":"drain","context":{"idset":"11338","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6867895,"name":"drain","context":{"idset":"11339","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6878812,"name":"drain","context":{"idset":"11340","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.688972,"name":"drain","context":{"idset":"11341","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6900647,"name":"drain","context":{"idset":"11342","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.691153,"name":"drain","context":{"idset":"11343","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6922565,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6933806,"name":"drain","context":{"idset":"11345","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6944704,"name":"drain","context":{"idset":"11346","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6955621,"name":"drain","context":{"idset":"11347","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6966543,"name":"drain","context":{"idset":"11348","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6977494,"name":"drain","context":{"idset":"11349","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6988451,"name":"drain","context":{"idset":"11350","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.6999371,"name":"drain","context":{"idset":"11351","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7010262,"name":"drain","context":{"idset":"11352","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7021222,"name":"drain","context":{"idset":"11353","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7032163,"name":"drain","context":{"idset":"11354","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7043314,"name":"drain","context":{"idset":"11355","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7054284,"name":"drain","context":{"idset":"11356","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7065215,"name":"drain","context":{"idset":"11357","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7076168,"name":"drain","context":{"idset":"11359","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7087162,"name":"drain","context":{"idset":"11360","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7098145,"name":"drain","context":{"idset":"11361","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7109149,"name":"drain","context":{"idset":"11362","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7120113,"name":"drain","context":{"idset":"11363","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7131152,"name":"drain","context":{"idset":"11364","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7142148,"name":"drain","context":{"idset":"11365","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7153404,"name":"drain","context":{"idset":"11366","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7164388,"name":"drain","context":{"idset":"11367","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7175448,"name":"drain","context":{"idset":"11368","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7186477,"name":"drain","context":{"idset":"11369","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7197461,"name":"drain","context":{"idset":"11370","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7208521,"name":"drain","context":{"idset":"11371","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7220464,"name":"drain","context":{"idset":"11372","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7231631,"name":"drain","context":{"idset":"11375","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.724261,"name":"drain","context":{"idset":"11376","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7253742,"name":"drain","context":{"idset":"11377","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7264869,"name":"drain","context":{"idset":"11378","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7275951,"name":"drain","context":{"idset":"11379","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7287025,"name":"drain","context":{"idset":"11380","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7298064,"name":"drain","context":{"idset":"11381","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7309108,"name":"drain","context":{"idset":"11382","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7320132,"name":"drain","context":{"idset":"11383","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7331238,"name":"drain","context":{"idset":"11384","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.734225,"name":"drain","context":{"idset":"11385","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7353494,"name":"drain","context":{"idset":"11386","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7364607,"name":"drain","context":{"idset":"11389","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7375669,"name":"drain","context":{"idset":"11391","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7386825,"name":"drain","context":{"idset":"11392","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7397916,"name":"drain","context":{"idset":"11393","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7409284,"name":"drain","context":{"idset":"11394","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7420373,"name":"drain","context":{"idset":"11396","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7431452,"name":"drain","context":{"idset":"11397","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.744252,"name":"drain","context":{"idset":"11398","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7453833,"name":"drain","context":{"idset":"11399","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.778296,"name":"drain","context":{"idset":"11430","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7794206,"name":"drain","context":{"idset":"11432","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7805457,"name":"drain","context":{"idset":"11433","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7816722,"name":"drain","context":{"idset":"11434","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7828031,"name":"drain","context":{"idset":"11435","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7839198,"name":"drain","context":{"idset":"11436","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.785042,"name":"drain","context":{"idset":"11437","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7861645,"name":"drain","context":{"idset":"11438","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7873051,"name":"drain","context":{"idset":"11443","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7884271,"name":"drain","context":{"idset":"11444","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7895498,"name":"drain","context":{"idset":"11445","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7906895,"name":"drain","context":{"idset":"11446","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7918308,"name":"drain","context":{"idset":"11447","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7929759,"name":"drain","context":{"idset":"11448","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7941093,"name":"drain","context":{"idset":"11449","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7952402,"name":"drain","context":{"idset":"11450","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7963824,"name":"drain","context":{"idset":"11451","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7975113,"name":"drain","context":{"idset":"11452","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7986388,"name":"drain","context":{"idset":"11453","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.7997668,"name":"drain","context":{"idset":"11454","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8008986,"name":"drain","context":{"idset":"11455","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8020322,"name":"drain","context":{"idset":"11456","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8031595,"name":"drain","context":{"idset":"11457","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8043122,"name":"drain","context":{"idset":"11458","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8054457,"name":"drain","context":{"idset":"11459","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8065751,"name":"drain","context":{"idset":"11460","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8077266,"name":"drain","context":{"idset":"11461","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8088553,"name":"drain","context":{"idset":"11462","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8099849,"name":"drain","context":{"idset":"11463","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8111331,"name":"drain","context":{"idset":"11465","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8122854,"name":"drain","context":{"idset":"11467","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8134332,"name":"drain","context":{"idset":"11468","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8145852,"name":"drain","context":{"idset":"11469","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8157241,"name":"drain","context":{"idset":"11470","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8168702,"name":"drain","context":{"idset":"11471","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8180058,"name":"drain","context":{"idset":"11472","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8191481,"name":"drain","context":{"idset":"11473","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8203051,"name":"drain","context":{"idset":"11474","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8214438,"name":"drain","context":{"idset":"11475","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8225882,"name":"drain","context":{"idset":"11476","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8237481,"name":"drain","context":{"idset":"11477","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8248889,"name":"drain","context":{"idset":"11478","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.826026,"name":"drain","context":{"idset":"11479","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8271687,"name":"drain","context":{"idset":"11480","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8283229,"name":"drain","context":{"idset":"11481","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8294599,"name":"drain","context":{"idset":"11482","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.830601,"name":"drain","context":{"idset":"11483","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8317366,"name":"drain","context":{"idset":"11484","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8328736,"name":"drain","context":{"idset":"11485","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.834013,"name":"drain","context":{"idset":"11486","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8351524,"name":"drain","context":{"idset":"11487","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8363042,"name":"drain","context":{"idset":"11488","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8374436,"name":"drain","context":{"idset":"11489","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8385918,"name":"drain","context":{"idset":"11490","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8397274,"name":"drain","context":{"idset":"11491","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8408744,"name":"drain","context":{"idset":"11492","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8420322,"name":"drain","context":{"idset":"11493","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8431768,"name":"drain","context":{"idset":"11494","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8443353,"name":"drain","context":{"idset":"11495","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8454797,"name":"drain","context":{"idset":"11496","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8466315,"name":"drain","context":{"idset":"11497","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8477974,"name":"drain","context":{"idset":"11498","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8489382,"name":"drain","context":{"idset":"11499","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8500795,"name":"drain","context":{"idset":"11500","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8512239,"name":"drain","context":{"idset":"11501","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8524072,"name":"drain","context":{"idset":"11502","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8535519,"name":"drain","context":{"idset":"11503","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8546991,"name":"drain","context":{"idset":"11504","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8558791,"name":"drain","context":{"idset":"11505","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8570251,"name":"drain","context":{"idset":"11506","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8581719,"name":"drain","context":{"idset":"11507","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8593483,"name":"drain","context":{"idset":"11508","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.860502,"name":"drain","context":{"idset":"11509","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8616548,"name":"drain","context":{"idset":"11510","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.862803,"name":"drain","context":{"idset":"11511","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8639483,"name":"drain","context":{"idset":"11512","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8651066,"name":"drain","context":{"idset":"11513","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8662601,"name":"drain","context":{"idset":"11514","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8674266,"name":"drain","context":{"idset":"11515","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8685856,"name":"drain","context":{"idset":"11516","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8697486,"name":"drain","context":{"idset":"11517","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8709023,"name":"drain","context":{"idset":"11518","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.872051,"name":"drain","context":{"idset":"11519","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8732071,"name":"drain","context":{"idset":"11520","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8743899,"name":"drain","context":{"idset":"11521","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8755434,"name":"drain","context":{"idset":"11522","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8766971,"name":"drain","context":{"idset":"11523","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8778551,"name":"drain","context":{"idset":"11524","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8790076,"name":"drain","context":{"idset":"11525","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8801579,"name":"drain","context":{"idset":"11526","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8813295,"name":"drain","context":{"idset":"11528","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8824995,"name":"drain","context":{"idset":"11529","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8836577,"name":"drain","context":{"idset":"11530","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8848116,"name":"drain","context":{"idset":"11531","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8859637,"name":"drain","context":{"idset":"11532","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8871183,"name":"drain","context":{"idset":"11533","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8882921,"name":"drain","context":{"idset":"11534","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8894467,"name":"drain","context":{"idset":"11535","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8906085,"name":"drain","context":{"idset":"11536","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8917618,"name":"drain","context":{"idset":"11537","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8929169,"name":"drain","context":{"idset":"11538","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8940694,"name":"drain","context":{"idset":"11539","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8952346,"name":"drain","context":{"idset":"11540","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8964059,"name":"drain","context":{"idset":"11541","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8975644,"name":"drain","context":{"idset":"11542","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.8987365,"name":"drain","context":{"idset":"11543","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9001698,"name":"drain","context":{"idset":"11544","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9020455,"name":"drain","context":{"idset":"11545","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9040096,"name":"drain","context":{"idset":"11546","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9058547,"name":"drain","context":{"idset":"11547","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9075847,"name":"drain","context":{"idset":"11548","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9090986,"name":"drain","context":{"idset":"11549","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9108431,"name":"drain","context":{"idset":"11550","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9125543,"name":"drain","context":{"idset":"11551","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9142616,"name":"drain","context":{"idset":"11552","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.915961,"name":"drain","context":{"idset":"11553","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9174156,"name":"drain","context":{"idset":"11554","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9185796,"name":"drain","context":{"idset":"11555","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9197261,"name":"drain","context":{"idset":"11556","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9208729,"name":"drain","context":{"idset":"11557","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9220166,"name":"drain","context":{"idset":"11558","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9232931,"name":"drain","context":{"idset":"11559","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9244602,"name":"drain","context":{"idset":"11560","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9256153,"name":"drain","context":{"idset":"11561","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9267669,"name":"drain","context":{"idset":"11562","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9279335,"name":"drain","context":{"idset":"11564","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9290888,"name":"drain","context":{"idset":"11565","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9302316,"name":"drain","context":{"idset":"11566","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9313924,"name":"drain","context":{"idset":"11567","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.932548,"name":"drain","context":{"idset":"11568","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9336905,"name":"drain","context":{"idset":"11569","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9349179,"name":"drain","context":{"idset":"11570","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9361832,"name":"drain","context":{"idset":"11571","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9374418,"name":"drain","context":{"idset":"11572","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9386592,"name":"drain","context":{"idset":"11573","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.939939,"name":"drain","context":{"idset":"11574","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9411871,"name":"drain","context":{"idset":"11575","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9424169,"name":"drain","context":{"idset":"11576","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9435771,"name":"drain","context":{"idset":"11577","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9447336,"name":"drain","context":{"idset":"11578","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9459293,"name":"drain","context":{"idset":"11579","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9471509,"name":"drain","context":{"idset":"11580","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9484508,"name":"drain","context":{"idset":"11581","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9496152,"name":"drain","context":{"idset":"11582","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.950798,"name":"drain","context":{"idset":"11583","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9520173,"name":"drain","context":{"idset":"11584","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9531691,"name":"drain","context":{"idset":"11585","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9543386,"name":"drain","context":{"idset":"11586","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9554882,"name":"drain","context":{"idset":"11589","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.956645,"name":"drain","context":{"idset":"11590","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9577982,"name":"drain","context":{"idset":"11591","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9590485,"name":"drain","context":{"idset":"11592","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9602113,"name":"drain","context":{"idset":"11593","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9613798,"name":"drain","context":{"idset":"11594","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9625344,"name":"drain","context":{"idset":"11595","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.96369,"name":"drain","context":{"idset":"11596","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9648461,"name":"drain","context":{"idset":"11597","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9660347,"name":"drain","context":{"idset":"11598","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9672759,"name":"drain","context":{"idset":"11599","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9684441,"name":"drain","context":{"idset":"11600","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9695969,"name":"drain","context":{"idset":"11601","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9707575,"name":"drain","context":{"idset":"11602","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9719169,"name":"drain","context":{"idset":"11603","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9730785,"name":"drain","context":{"idset":"11604","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9742367,"name":"drain","context":{"idset":"11605","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9754145,"name":"drain","context":{"idset":"11606","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9765701,"name":"drain","context":{"idset":"11607","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9777808,"name":"drain","context":{"idset":"11608","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9789431,"name":"drain","context":{"idset":"11609","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9801011,"name":"drain","context":{"idset":"11610","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9812617,"name":"drain","context":{"idset":"11611","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9824805,"name":"drain","context":{"idset":"11612","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9836538,"name":"drain","context":{"idset":"11613","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9848158,"name":"drain","context":{"idset":"11614","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9859715,"name":"drain","context":{"idset":"11615","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9871686,"name":"drain","context":{"idset":"11616","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9884355,"name":"drain","context":{"idset":"11617","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9896021,"name":"drain","context":{"idset":"11618","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9908779,"name":"drain","context":{"idset":"11619","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9921825,"name":"drain","context":{"idset":"11620","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9934466,"name":"drain","context":{"idset":"11621","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9946382,"name":"drain","context":{"idset":"11622","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9959121,"name":"drain","context":{"idset":"11623","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.997138,"name":"drain","context":{"idset":"11624","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9983759,"name":"drain","context":{"idset":"11625","reason":"broker was unresponsive"}}
+{"timestamp":1712365573.9996562,"name":"drain","context":{"idset":"11626","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0008256,"name":"drain","context":{"idset":"11627","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0020335,"name":"drain","context":{"idset":"11628","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0032594,"name":"drain","context":{"idset":"11629","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0045254,"name":"drain","context":{"idset":"11630","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0057659,"name":"drain","context":{"idset":"11631","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.007026,"name":"drain","context":{"idset":"11633","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0082335,"name":"drain","context":{"idset":"11634","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0094814,"name":"drain","context":{"idset":"11635","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0107441,"name":"drain","context":{"idset":"11636","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0119636,"name":"drain","context":{"idset":"11653","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.013144,"name":"drain","context":{"idset":"11654","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0143993,"name":"drain","context":{"idset":"11655","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0155854,"name":"drain","context":{"idset":"11656","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0168338,"name":"drain","context":{"idset":"11657","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0180514,"name":"drain","context":{"idset":"11658","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0192924,"name":"drain","context":{"idset":"11659","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0205414,"name":"drain","context":{"idset":"11660","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0217526,"name":"drain","context":{"idset":"11661","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0230043,"name":"drain","context":{"idset":"11662","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0242088,"name":"drain","context":{"idset":"11663","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0253997,"name":"drain","context":{"idset":"11664","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0266085,"name":"drain","context":{"idset":"11665","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0277908,"name":"drain","context":{"idset":"11666","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0289652,"name":"drain","context":{"idset":"11667","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0302942,"name":"drain","context":{"idset":"11668","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0321054,"name":"drain","context":{"idset":"11669","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0339477,"name":"drain","context":{"idset":"11670","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0358055,"name":"drain","context":{"idset":"11671","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0376096,"name":"drain","context":{"idset":"11672","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.039391,"name":"drain","context":{"idset":"11673","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0411592,"name":"drain","context":{"idset":"11674","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0429301,"name":"drain","context":{"idset":"11675","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0444205,"name":"drain","context":{"idset":"11676","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0457098,"name":"drain","context":{"idset":"11677","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0469139,"name":"drain","context":{"idset":"11678","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0481505,"name":"drain","context":{"idset":"11679","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0495498,"name":"drain","context":{"idset":"11680","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0508928,"name":"drain","context":{"idset":"11681","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0521166,"name":"drain","context":{"idset":"11682","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0533624,"name":"drain","context":{"idset":"11683","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0547881,"name":"drain","context":{"idset":"11684","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.05601,"name":"drain","context":{"idset":"11685","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0573213,"name":"drain","context":{"idset":"11686","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0585124,"name":"drain","context":{"idset":"11687","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0596926,"name":"drain","context":{"idset":"11688","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0609007,"name":"drain","context":{"idset":"11689","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0622373,"name":"drain","context":{"idset":"11690","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0634422,"name":"drain","context":{"idset":"11691","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0646238,"name":"drain","context":{"idset":"11692","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0658562,"name":"drain","context":{"idset":"11693","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0670881,"name":"drain","context":{"idset":"11695","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0683165,"name":"drain","context":{"idset":"11696","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0695369,"name":"drain","context":{"idset":"11697","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0707932,"name":"drain","context":{"idset":"11698","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0721102,"name":"drain","context":{"idset":"11699","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0733862,"name":"drain","context":{"idset":"11700","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0746007,"name":"drain","context":{"idset":"11701","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0758116,"name":"drain","context":{"idset":"11702","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.077132,"name":"drain","context":{"idset":"11703","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0784664,"name":"drain","context":{"idset":"11704","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0797985,"name":"drain","context":{"idset":"11705","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0811372,"name":"drain","context":{"idset":"11706","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0824137,"name":"drain","context":{"idset":"11707","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0837054,"name":"drain","context":{"idset":"11708","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0849547,"name":"drain","context":{"idset":"11709","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0861518,"name":"drain","context":{"idset":"11710","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0873725,"name":"drain","context":{"idset":"11711","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0885665,"name":"drain","context":{"idset":"11712","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0897558,"name":"drain","context":{"idset":"11713","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0909441,"name":"drain","context":{"idset":"11714","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.092135,"name":"drain","context":{"idset":"11715","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0933781,"name":"drain","context":{"idset":"11716","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0949488,"name":"drain","context":{"idset":"11717","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0961504,"name":"drain","context":{"idset":"11718","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0973582,"name":"drain","context":{"idset":"11719","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0985529,"name":"drain","context":{"idset":"11720","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.0997965,"name":"drain","context":{"idset":"11721","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1010034,"name":"drain","context":{"idset":"11722","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1021972,"name":"drain","context":{"idset":"11723","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1034052,"name":"drain","context":{"idset":"11724","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1047161,"name":"drain","context":{"idset":"11725","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1060042,"name":"drain","context":{"idset":"11726","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.107301,"name":"drain","context":{"idset":"11727","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.108511,"name":"drain","context":{"idset":"11728","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1097074,"name":"drain","context":{"idset":"11729","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1109064,"name":"drain","context":{"idset":"11730","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1121089,"name":"drain","context":{"idset":"11731","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1133173,"name":"drain","context":{"idset":"11732","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1145139,"name":"drain","context":{"idset":"11733","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1157134,"name":"drain","context":{"idset":"11734","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.116914,"name":"drain","context":{"idset":"11735","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.118113,"name":"drain","context":{"idset":"11736","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1193531,"name":"drain","context":{"idset":"11737","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1206181,"name":"drain","context":{"idset":"11738","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1218214,"name":"drain","context":{"idset":"11739","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1230252,"name":"drain","context":{"idset":"11740","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1243427,"name":"drain","context":{"idset":"11741","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.125711,"name":"drain","context":{"idset":"11742","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1270363,"name":"drain","context":{"idset":"11743","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1282978,"name":"drain","context":{"idset":"11744","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1295116,"name":"drain","context":{"idset":"11745","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1307399,"name":"drain","context":{"idset":"11746","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1319807,"name":"drain","context":{"idset":"11747","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1332083,"name":"drain","context":{"idset":"11748","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.134516,"name":"drain","context":{"idset":"11749","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1357708,"name":"drain","context":{"idset":"11750","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1370347,"name":"drain","context":{"idset":"11751","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1382954,"name":"drain","context":{"idset":"11752","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1395168,"name":"drain","context":{"idset":"11753","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1407387,"name":"drain","context":{"idset":"11754","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1419568,"name":"drain","context":{"idset":"11755","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.143183,"name":"drain","context":{"idset":"11756","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1444252,"name":"drain","context":{"idset":"11757","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1456571,"name":"drain","context":{"idset":"11758","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1468844,"name":"drain","context":{"idset":"11759","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1481116,"name":"drain","context":{"idset":"11760","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1493554,"name":"drain","context":{"idset":"11761","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1505969,"name":"drain","context":{"idset":"11762","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1518533,"name":"drain","context":{"idset":"11763","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1532352,"name":"drain","context":{"idset":"11764","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.1757526,"name":"drain","context":{"idset":"11802","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2050045,"name":"drain","context":{"idset":"11877","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2062364,"name":"drain","context":{"idset":"11878","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2074754,"name":"drain","context":{"idset":"11879","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2086878,"name":"drain","context":{"idset":"11880","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.209893,"name":"drain","context":{"idset":"11881","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2110989,"name":"drain","context":{"idset":"11882","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2123373,"name":"drain","context":{"idset":"11883","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2135563,"name":"drain","context":{"idset":"11885","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2147608,"name":"drain","context":{"idset":"11886","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2159693,"name":"drain","context":{"idset":"11887","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2171705,"name":"drain","context":{"idset":"11888","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2183867,"name":"drain","context":{"idset":"11889","reason":"broker was unresponsive"}}
+{"timestamp":1712365574.2195978,"name":"drain","context":{"idset":"11890","reason":"broker was unresponsive"}}
+{"timestamp":1712372191.1068847,"name":"resource-init","context":{"restart":true,"drain":{"1":{"timestamp":1712365572.2220874,"reason":"broker was unresponsive"},"2":{"timestamp":1712365572.2286646,"reason":"broker was unresponsive"},"3":{"timestamp":1712365572.228714,"reason":"broker was unresponsive"},"4":{"timestamp":1712365572.2306376,"reason":"broker was unresponsive"},"5":{"timestamp":1712365572.2307081,"reason":"broker was unresponsive"},"6":{"timestamp":1712365572.2307422,"reason":"broker was unresponsive"},"7":{"timestamp":1712365572.2307882,"reason":"broker was unresponsive"},"8":{"timestamp":1712365572.2308214,"reason":"broker was unresponsive"},"9":{"timestamp":1712365572.2308669,"reason":"broker was unresponsive"},"10":{"timestamp":1712365572.2309122,"reason":"broker was unresponsive"},"11":{"timestamp":1712365572.2309432,"reason":"broker was unresponsive"},"12":{"timestamp":1712365572.2309718,"reason":"broker was unresponsive"},"13":{"timestamp":1712365572.2310019,"reason":"broker was unresponsive"},"14":{"timestamp":1712365572.2310357,"reason":"broker was unresponsive"},"15":{"timestamp":1712365572.2310669,"reason":"broker was unresponsive"},"16":{"timestamp":1712365572.2311132,"reason":"broker was unresponsive"},"17":{"timestamp":1712365572.2311509,"reason":"broker was unresponsive"},"18":{"timestamp":1712365572.2311814,"reason":"broker was unresponsive"},"19":{"timestamp":1712365572.2312181,"reason":"broker was unresponsive"},"20":{"timestamp":1712365572.2312882,"reason":"broker was unresponsive"},"21":{"timestamp":1712365572.2313271,"reason":"broker was unresponsive"},"22":{"timestamp":1712365572.2313788,"reason":"broker was unresponsive"},"23":{"timestamp":1712365572.2314267,"reason":"broker was unresponsive"},"24":{"timestamp":1712365572.2314601,"reason":"broker was unresponsive"},"25":{"timestamp":1712365572.2314937,"reason":"broker was unresponsive"},"26":{"timestamp":1712365572.2315323,"reason":"broker was unresponsive"},"27":{"timestamp":1712365572.2315834,"reason":"broker was unresponsive"},"28":{"timestamp":1712365572.231638,"reason":"broker was unresponsive"},"29":{"timestamp":1712365572.2316718,"reason":"broker was unresponsive"},"30":{"timestamp":1712365572.2317057,"reason":"broker was unresponsive"},"31":{"timestamp":1712365572.2317371,"reason":"broker was unresponsive"},"32":{"timestamp":1712365572.2317684,"reason":"broker was unresponsive"},"33":{"timestamp":1712365572.2318003,"reason":"broker was unresponsive"},"34":{"timestamp":1712365572.231853,"reason":"broker was unresponsive"},"35":{"timestamp":1712365572.2318985,"reason":"broker was unresponsive"},"36":{"timestamp":1712365572.2319705,"reason":"broker was unresponsive"},"37":{"timestamp":1712365572.2320211,"reason":"broker was unresponsive"},"38":{"timestamp":1712365572.2320583,"reason":"broker was unresponsive"},"39":{"timestamp":1712365572.2320967,"reason":"broker was unresponsive"},"40":{"timestamp":1712365572.2321327,"reason":"broker was unresponsive"},"41":{"timestamp":1712365572.2321801,"reason":"broker was unresponsive"},"42":{"timestamp":1712365572.2322273,"reason":"broker was unresponsive"},"43":{"timestamp":1712365572.2322598,"reason":"broker was unresponsive"},"44":{"timestamp":1712365572.232311,"reason":"broker was unresponsive"},"45":{"timestamp":1712365572.2323503,"reason":"broker was unresponsive"},"46":{"timestamp":1712365572.2323849,"reason":"broker was unresponsive"},"47":{"timestamp":1712365572.2324324,"reason":"broker was unresponsive"},"48":{"timestamp":1712365572.2324872,"reason":"broker was unresponsive"},"49":{"timestamp":1712365572.2325249,"reason":"broker was unresponsive"},"50":{"timestamp":1712365572.2325625,"reason":"broker was unresponsive"},"51":{"timestamp":1712365572.2325978,"reason":"broker was unresponsive"},"52":{"timestamp":1712365572.2326546,"reason":"broker was unresponsive"},"53":{"timestamp":1712365572.232717,"reason":"broker was unresponsive"},"54":{"timestamp":1712365572.2327638,"reason":"broker was unresponsive"},"55":{"timestamp":1712365572.2328057,"reason":"broker was unresponsive"},"56":{"timestamp":1712365572.232842,"reason":"broker was unresponsive"},"57":{"timestamp":1712365572.2328789,"reason":"broker was unresponsive"},"58":{"timestamp":1712365572.2329352,"reason":"broker was unresponsive"},"59":{"timestamp":1712365572.2329841,"reason":"broker was unresponsive"},"60":{"timestamp":1712365572.2330277,"reason":"broker was unresponsive"},"61":{"timestamp":1711668444.9627461,"reason":"broker was unresponsive"},"62":{"timestamp":1711668506.9728317,"reason":"broker was unresponsive"},"63":{"timestamp":1711668445.0625052,"reason":"broker was unresponsive"},"64":{"timestamp":1711668506.9729121,"reason":"broker was unresponsive"},"65":{"timestamp":1711668438.9624472,"reason":"broker was unresponsive"},"66":{"timestamp":1711668506.9729731,"reason":"broker was unresponsive"},"67":{"timestamp":1711668439.0630105,"reason":"broker was unresponsive"},"68":{"timestamp":1711668506.9730346,"reason":"broker was unresponsive"},"69-76":{"timestamp":1711646931.6480982,"reason":""},"77":{"timestamp":1711668526.9621451,"reason":"broker was unresponsive"},"78":{"timestamp":1711668462.9610252,"reason":"broker was unresponsive"},"79":{"timestamp":1711668526.96223,"reason":"broker was unresponsive"},"80":{"timestamp":1711668457.0617182,"reason":"broker was unresponsive"},"81":{"timestamp":1711668526.9622741,"reason":"broker was unresponsive"},"82":{"timestamp":1711668462.9611192,"reason":"broker was unresponsive"},"83":{"timestamp":1711668527.0620673,"reason":"broker was unresponsive"},"84":{"timestamp":1711668463.0615458,"reason":"broker was unresponsive"},"85":{"timestamp":1712365572.233068,"reason":"broker was unresponsive"},"86":{"timestamp":1712365572.2331095,"reason":"broker was unresponsive"},"87":{"timestamp":1712365572.2332046,"reason":"broker was unresponsive"},"88":{"timestamp":1712365572.2332504,"reason":"broker was unresponsive"},"89":{"timestamp":1712365572.2333148,"reason":"broker was unresponsive"},"90":{"timestamp":1712365572.2333574,"reason":"broker was unresponsive"},"91":{"timestamp":1712365572.233413,"reason":"broker was unresponsive"},"92":{"timestamp":1712365572.2334795,"reason":"broker was unresponsive"},"93":{"timestamp":1712365572.2335336,"reason":"broker was unresponsive"},"94":{"timestamp":1712365572.233588,"reason":"broker was unresponsive"},"95":{"timestamp":1712365572.2336364,"reason":"broker was unresponsive"},"96":{"timestamp":1712365572.2337134,"reason":"broker was unresponsive"},"97":{"timestamp":1712365572.233762,"reason":"broker was unresponsive"},"98":{"timestamp":1712365572.233824,"reason":"broker was unresponsive"},"99":{"timestamp":1712365572.2338965,"reason":"broker was unresponsive"},"100":{"timestamp":1712365572.2339897,"reason":"broker was unresponsive"},"101":{"timestamp":1712365572.2340486,"reason":"broker was unresponsive"},"102":{"timestamp":1712365572.2341108,"reason":"broker was unresponsive"},"103":{"timestamp":1712365572.2341814,"reason":"broker was unresponsive"},"104":{"timestamp":1712365572.2342274,"reason":"broker was unresponsive"},"105":{"timestamp":1712365572.2342942,"reason":"broker was unresponsive"},"106":{"timestamp":1712365572.2343431,"reason":"broker was unresponsive"},"107":{"timestamp":1712365572.2344041,"reason":"broker was unresponsive"},"108":{"timestamp":1712365572.2344565,"reason":"broker was unresponsive"},"109":{"timestamp":1712365572.2345219,"reason":"broker was unresponsive"},"110":{"timestamp":1712365572.2345781,"reason":"broker was unresponsive"},"111":{"timestamp":1712365572.2346251,"reason":"broker was unresponsive"},"112":{"timestamp":1712365572.2346919,"reason":"broker was unresponsive"},"113":{"timestamp":1712365572.2347553,"reason":"broker was unresponsive"},"114":{"timestamp":1712365572.2348204,"reason":"broker was unresponsive"},"115":{"timestamp":1712365572.234868,"reason":"broker was unresponsive"},"116":{"timestamp":1712365572.2349133,"reason":"broker was unresponsive"},"117":{"timestamp":1712365572.2350121,"reason":"broker was unresponsive"},"118":{"timestamp":1712365572.2350743,"reason":"broker was unresponsive"},"119":{"timestamp":1712365572.2351217,"reason":"broker was unresponsive"},"120":{"timestamp":1712365572.2351696,"reason":"broker was unresponsive"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"122":{"timestamp":1712365572.2352524,"reason":"broker was unresponsive"},"123":{"timestamp":1712365572.2353327,"reason":"broker was unresponsive"},"124":{"timestamp":1712365572.2353818,"reason":"broker was unresponsive"},"125":{"timestamp":1712365572.23543,"reason":"broker was unresponsive"},"126":{"timestamp":1712365572.2354929,"reason":"broker was unresponsive"},"127":{"timestamp":1712365572.2355638,"reason":"broker was unresponsive"},"128":{"timestamp":1712365572.235621,"reason":"broker was unresponsive"},"129":{"timestamp":1712365572.2356713,"reason":"broker was unresponsive"},"130":{"timestamp":1712365572.2357309,"reason":"broker was unresponsive"},"131":{"timestamp":1712365572.2358205,"reason":"broker was unresponsive"},"132":{"timestamp":1712365572.2358782,"reason":"broker was unresponsive"},"133":{"timestamp":1712365572.2359297,"reason":"broker was unresponsive"},"134":{"timestamp":1712365572.2360036,"reason":"broker was unresponsive"},"135":{"timestamp":1712365572.2360642,"reason":"broker was unresponsive"},"136":{"timestamp":1712365572.2361147,"reason":"broker was unresponsive"},"137":{"timestamp":1712365572.2361672,"reason":"broker was unresponsive"},"138":{"timestamp":1712365572.2362382,"reason":"broker was unresponsive"},"139":{"timestamp":1712365572.2363441,"reason":"broker was unresponsive"},"140":{"timestamp":1712365572.2364006,"reason":"broker was unresponsive"},"141":{"timestamp":1712365572.2364745,"reason":"broker was unresponsive"},"142":{"timestamp":1712365572.2365518,"reason":"broker was unresponsive"},"143":{"timestamp":1712365572.2366059,"reason":"broker was unresponsive"},"144":{"timestamp":1712365572.23666,"reason":"broker was unresponsive"},"145":{"timestamp":1712365572.236732,"reason":"broker was unresponsive"},"146":{"timestamp":1712365572.2367985,"reason":"broker was unresponsive"},"147":{"timestamp":1712365572.2368526,"reason":"broker was unresponsive"},"148":{"timestamp":1712365572.236917,"reason":"broker was unresponsive"},"149":{"timestamp":1712365572.2369714,"reason":"broker was unresponsive"},"150":{"timestamp":1712365572.2370315,"reason":"broker was unresponsive"},"151":{"timestamp":1712365572.237123,"reason":"broker was unresponsive"},"152":{"timestamp":1712365572.2371862,"reason":"broker was unresponsive"},"153":{"timestamp":1712365572.2372417,"reason":"broker was unresponsive"},"154":{"timestamp":1712365572.2373435,"reason":"broker was unresponsive"},"155":{"timestamp":1712365572.2374115,"reason":"broker was unresponsive"},"156":{"timestamp":1712365572.2374861,"reason":"broker was unresponsive"},"157":{"timestamp":1712365572.2375619,"reason":"broker was unresponsive"},"158":{"timestamp":1712365572.2376416,"reason":"broker was unresponsive"},"159":{"timestamp":1712365572.2377126,"reason":"broker was unresponsive"},"160":{"timestamp":1712365572.237788,"reason":"broker was unresponsive"},"161":{"timestamp":1712365572.2379014,"reason":"broker was unresponsive"},"162":{"timestamp":1712365572.2379725,"reason":"broker was unresponsive"},"163":{"timestamp":1712365572.2380474,"reason":"broker was unresponsive"},"164":{"timestamp":1712365572.2381475,"reason":"broker was unresponsive"},"165":{"timestamp":1712365572.2382314,"reason":"broker was unresponsive"},"166":{"timestamp":1712365572.2383361,"reason":"broker was unresponsive"},"167":{"timestamp":1712365572.2384217,"reason":"broker was unresponsive"},"168":{"timestamp":1712365572.2384849,"reason":"broker was unresponsive"},"169":{"timestamp":1712365572.238564,"reason":"broker was unresponsive"},"170":{"timestamp":1712365572.2386551,"reason":"broker was unresponsive"},"171":{"timestamp":1712365572.2387521,"reason":"broker was unresponsive"},"172":{"timestamp":1712365572.2388451,"reason":"broker was unresponsive"},"173":{"timestamp":1712365572.2389157,"reason":"broker was unresponsive"},"174":{"timestamp":1712365572.2389863,"reason":"broker was unresponsive"},"175":{"timestamp":1712365572.2390757,"reason":"broker was unresponsive"},"176":{"timestamp":1712365572.2391434,"reason":"broker was unresponsive"},"177":{"timestamp":1712365572.2392077,"reason":"broker was unresponsive"},"178":{"timestamp":1712365572.2393053,"reason":"broker was unresponsive"},"179":{"timestamp":1712365572.2394068,"reason":"broker was unresponsive"},"180":{"timestamp":1712365572.2394729,"reason":"broker was unresponsive"},"181":{"timestamp":1712365572.239537,"reason":"broker was unresponsive"},"182":{"timestamp":1712365572.2396276,"reason":"broker was unresponsive"},"183":{"timestamp":1712365572.2397325,"reason":"broker was unresponsive"},"184":{"timestamp":1712365572.2398086,"reason":"broker was unresponsive"},"185":{"timestamp":1712365572.2399015,"reason":"broker was unresponsive"},"186":{"timestamp":1712365572.2399721,"reason":"broker was unresponsive"},"187":{"timestamp":1712365572.2400663,"reason":"broker was unresponsive"},"188":{"timestamp":1712365572.2401721,"reason":"broker was unresponsive"},"189":{"timestamp":1712365572.240247,"reason":"broker was unresponsive"},"190":{"timestamp":1712365572.2403345,"reason":"broker was unresponsive"},"191":{"timestamp":1712365572.240447,"reason":"broker was unresponsive"},"192":{"timestamp":1712365572.2405415,"reason":"broker was unresponsive"},"193":{"timestamp":1712365572.2406516,"reason":"broker was unresponsive"},"194":{"timestamp":1712365572.2407241,"reason":"broker was unresponsive"},"195":{"timestamp":1712365572.2407918,"reason":"broker was unresponsive"},"196":{"timestamp":1712365572.2408812,"reason":"broker was unresponsive"},"197":{"timestamp":1712365572.240957,"reason":"broker was unresponsive"},"198":{"timestamp":1712365572.2410438,"reason":"broker was unresponsive"},"199":{"timestamp":1712365572.2411568,"reason":"broker was unresponsive"},"200":{"timestamp":1712365572.2412353,"reason":"broker was unresponsive"},"201":{"timestamp":1712365572.2413495,"reason":"broker was unresponsive"},"202":{"timestamp":1712365572.2414603,"reason":"broker was unresponsive"},"203":{"timestamp":1712365572.2415369,"reason":"broker was unresponsive"},"204":{"timestamp":1712365572.2416494,"reason":"broker was unresponsive"},"205":{"timestamp":1712365572.241739,"reason":"broker was unresponsive"},"206":{"timestamp":1712365572.2418363,"reason":"broker was unresponsive"},"207":{"timestamp":1712365572.2419224,"reason":"broker was unresponsive"},"208":{"timestamp":1712365572.241993,"reason":"broker was unresponsive"},"209":{"timestamp":1712365572.2420797,"reason":"broker was unresponsive"},"210":{"timestamp":1712365572.2421513,"reason":"broker was unresponsive"},"211":{"timestamp":1712365572.2422457,"reason":"broker was unresponsive"},"212":{"timestamp":1712365572.2423363,"reason":"broker was unresponsive"},"213":{"timestamp":1712365572.2424362,"reason":"broker was unresponsive"},"214":{"timestamp":1712365572.2425499,"reason":"broker was unresponsive"},"215":{"timestamp":1712365572.2426443,"reason":"broker was unresponsive"},"216":{"timestamp":1712365572.2427473,"reason":"broker was unresponsive"},"218":{"timestamp":1712365572.2428365,"reason":"broker was unresponsive"},"219":{"timestamp":1712365572.2429252,"reason":"broker was unresponsive"},"220":{"timestamp":1712365572.2430108,"reason":"broker was unresponsive"},"221":{"timestamp":1712365572.2430842,"reason":"broker was unresponsive"},"222":{"timestamp":1712365572.2431943,"reason":"broker was unresponsive"},"223":{"timestamp":1712365572.2433102,"reason":"broker was unresponsive"},"224":{"timestamp":1712365572.2433951,"reason":"broker was unresponsive"},"225":{"timestamp":1712365572.243521,"reason":"broker was unresponsive"},"226":{"timestamp":1712365572.2436035,"reason":"broker was unresponsive"},"227":{"timestamp":1712365572.2437198,"reason":"broker was unresponsive"},"228":{"timestamp":1712365572.2438271,"reason":"broker was unresponsive"},"229":{"timestamp":1712365572.2439458,"reason":"broker was unresponsive"},"230":{"timestamp":1712365572.2440476,"reason":"broker was unresponsive"},"231":{"timestamp":1712365572.2441368,"reason":"broker was unresponsive"},"232":{"timestamp":1712365572.2442374,"reason":"broker was unresponsive"},"233":{"timestamp":1712365572.2443538,"reason":"broker was unresponsive"},"234":{"timestamp":1712365572.2445238,"reason":"broker was unresponsive"},"235":{"timestamp":1712365572.2446132,"reason":"broker was unresponsive"},"236":{"timestamp":1712365572.2447176,"reason":"broker was unresponsive"},"237":{"timestamp":1712365572.2448299,"reason":"broker was unresponsive"},"238":{"timestamp":1712365572.2449131,"reason":"broker was unresponsive"},"239":{"timestamp":1712365572.2450552,"reason":"broker was unresponsive"},"240":{"timestamp":1712365572.2451694,"reason":"broker was unresponsive"},"241":{"timestamp":1712365572.2453053,"reason":"broker was unresponsive"},"242":{"timestamp":1712365572.2454159,"reason":"broker was unresponsive"},"243":{"timestamp":1712365572.2455254,"reason":"broker was unresponsive"},"244":{"timestamp":1712365572.2456093,"reason":"broker was unresponsive"},"245":{"timestamp":1712365572.2456954,"reason":"broker was unresponsive"},"246":{"timestamp":1712365572.2458007,"reason":"broker was unresponsive"},"247":{"timestamp":1712365572.2458837,"reason":"broker was unresponsive"},"248":{"timestamp":1712365572.2459664,"reason":"broker was unresponsive"},"249":{"timestamp":1712365572.2460852,"reason":"broker was unresponsive"},"250":{"timestamp":1712365572.246201,"reason":"broker was unresponsive"},"251":{"timestamp":1712365572.2463498,"reason":"broker was unresponsive"},"252":{"timestamp":1712365572.2464375,"reason":"broker was unresponsive"},"253":{"timestamp":1712365572.2465682,"reason":"broker was unresponsive"},"254":{"timestamp":1712365572.2466788,"reason":"broker was unresponsive"},"255":{"timestamp":1712365572.2468174,"reason":"broker was unresponsive"},"256":{"timestamp":1712365572.246907,"reason":"broker was unresponsive"},"257":{"timestamp":1712365572.2470167,"reason":"broker was unresponsive"},"258":{"timestamp":1712365572.2471035,"reason":"broker was unresponsive"},"259":{"timestamp":1712365572.2471898,"reason":"broker was unresponsive"},"260":{"timestamp":1712365572.2473016,"reason":"broker was unresponsive"},"261":{"timestamp":1712365572.2474208,"reason":"broker was unresponsive"},"262":{"timestamp":1712365572.247545,"reason":"broker was unresponsive"},"263":{"timestamp":1712365572.2476625,"reason":"broker was unresponsive"},"264":{"timestamp":1712365572.2477677,"reason":"broker was unresponsive"},"265":{"timestamp":1712365572.2479048,"reason":"broker was unresponsive"},"266":{"timestamp":1712365572.2480068,"reason":"broker was unresponsive"},"267":{"timestamp":1712365572.2481599,"reason":"broker was unresponsive"},"268":{"timestamp":1712365572.2482576,"reason":"broker was unresponsive"},"269":{"timestamp":1712365572.2483904,"reason":"broker was unresponsive"},"270":{"timestamp":1712365572.2484899,"reason":"broker was unresponsive"},"271":{"timestamp":1712365572.2486119,"reason":"broker was unresponsive"},"272":{"timestamp":1712365572.2487044,"reason":"broker was unresponsive"},"273":{"timestamp":1712365572.2488339,"reason":"broker was unresponsive"},"274":{"timestamp":1712365572.2489343,"reason":"broker was unresponsive"},"275":{"timestamp":1712365572.2491014,"reason":"broker was unresponsive"},"276":{"timestamp":1712365572.2492063,"reason":"broker was unresponsive"},"277":{"timestamp":1712365572.2493408,"reason":"broker was unresponsive"},"278":{"timestamp":1712365572.2494519,"reason":"broker was unresponsive"},"279":{"timestamp":1712365572.2495797,"reason":"broker was unresponsive"},"280":{"timestamp":1712365572.2496936,"reason":"broker was unresponsive"},"281":{"timestamp":1712365572.2497976,"reason":"broker was unresponsive"},"282":{"timestamp":1712365572.2499115,"reason":"broker was unresponsive"},"283":{"timestamp":1712365572.2500079,"reason":"broker was unresponsive"},"284":{"timestamp":1712365572.2501473,"reason":"broker was unresponsive"},"285":{"timestamp":1712365572.2503216,"reason":"broker was unresponsive"},"286":{"timestamp":1712365572.2504642,"reason":"broker was unresponsive"},"287":{"timestamp":1712365572.2505872,"reason":"broker was unresponsive"},"288":{"timestamp":1712365572.2507181,"reason":"broker was unresponsive"},"289":{"timestamp":1712365572.2509167,"reason":"broker was unresponsive"},"290":{"timestamp":1712365572.2510245,"reason":"broker was unresponsive"},"291":{"timestamp":1712365572.2511661,"reason":"broker was unresponsive"},"292":{"timestamp":1712365572.2513063,"reason":"broker was unresponsive"},"293":{"timestamp":1712365572.2514446,"reason":"broker was unresponsive"},"294":{"timestamp":1712365572.2515419,"reason":"broker was unresponsive"},"295":{"timestamp":1712365572.2516575,"reason":"broker was unresponsive"},"296":{"timestamp":1712365572.2517707,"reason":"broker was unresponsive"},"297":{"timestamp":1712365572.2519009,"reason":"broker was unresponsive"},"298":{"timestamp":1712365572.2519977,"reason":"broker was unresponsive"},"299":{"timestamp":1712365572.2520924,"reason":"broker was unresponsive"},"300":{"timestamp":1712365572.2522221,"reason":"broker was unresponsive"},"301":{"timestamp":1712365572.2523494,"reason":"broker was unresponsive"},"302":{"timestamp":1712365572.2524884,"reason":"broker was unresponsive"},"303":{"timestamp":1712365572.2525928,"reason":"broker was unresponsive"},"304":{"timestamp":1712365572.2527246,"reason":"broker was unresponsive"},"305":{"timestamp":1712365572.2528481,"reason":"broker was unresponsive"},"306":{"timestamp":1712365572.2529981,"reason":"broker was unresponsive"},"307":{"timestamp":1712365572.2531021,"reason":"broker was unresponsive"},"308":{"timestamp":1712365572.2532146,"reason":"broker was unresponsive"},"309":{"timestamp":1712365572.2533586,"reason":"broker was unresponsive"},"310":{"timestamp":1712365572.2534888,"reason":"broker was unresponsive"},"311":{"timestamp":1712365572.2535899,"reason":"broker was unresponsive"},"312":{"timestamp":1712365572.2537384,"reason":"broker was unresponsive"},"313":{"timestamp":1712365572.253876,"reason":"broker was unresponsive"},"314":{"timestamp":1712365572.2540071,"reason":"broker was unresponsive"},"315":{"timestamp":1712365572.2541103,"reason":"broker was unresponsive"},"316":{"timestamp":1712365572.2542439,"reason":"broker was unresponsive"},"317":{"timestamp":1712365572.2543879,"reason":"broker was unresponsive"},"318":{"timestamp":1712365572.2545335,"reason":"broker was unresponsive"},"319":{"timestamp":1712365572.2546411,"reason":"broker was unresponsive"},"320":{"timestamp":1712365572.2547669,"reason":"broker was unresponsive"},"321":{"timestamp":1712365572.2548714,"reason":"broker was unresponsive"},"322":{"timestamp":1712365572.2549934,"reason":"broker was unresponsive"},"323":{"timestamp":1712365572.255172,"reason":"broker was unresponsive"},"324":{"timestamp":1712365572.2553525,"reason":"broker was unresponsive"},"325":{"timestamp":1712365572.2554677,"reason":"broker was unresponsive"},"326":{"timestamp":1712365572.255615,"reason":"broker was unresponsive"},"327":{"timestamp":1712365572.2557478,"reason":"broker was unresponsive"},"328":{"timestamp":1712365572.2559001,"reason":"broker was unresponsive"},"329":{"timestamp":1712365572.2560575,"reason":"broker was unresponsive"},"330":{"timestamp":1712365572.2561874,"reason":"broker was unresponsive"},"331":{"timestamp":1712365572.2563679,"reason":"broker was unresponsive"},"332":{"timestamp":1712365572.2564769,"reason":"broker was unresponsive"},"333":{"timestamp":1712365572.2566125,"reason":"broker was unresponsive"},"334":{"timestamp":1712365572.2567232,"reason":"broker was unresponsive"},"335":{"timestamp":1712365572.2568758,"reason":"broker was unresponsive"},"336":{"timestamp":1712365572.2569954,"reason":"broker was unresponsive"},"337":{"timestamp":1712365572.2571297,"reason":"broker was unresponsive"},"338":{"timestamp":1712365572.2572548,"reason":"broker was unresponsive"},"339":{"timestamp":1712365572.257385,"reason":"broker was unresponsive"},"340":{"timestamp":1712365572.2574966,"reason":"broker was unresponsive"},"341":{"timestamp":1712365572.2576525,"reason":"broker was unresponsive"},"342":{"timestamp":1712365572.2577868,"reason":"broker was unresponsive"},"343":{"timestamp":1712365572.2579179,"reason":"broker was unresponsive"},"344":{"timestamp":1712365572.2580254,"reason":"broker was unresponsive"},"345":{"timestamp":1712365572.2581847,"reason":"broker was unresponsive"},"346":{"timestamp":1712365572.258379,"reason":"broker was unresponsive"},"347":{"timestamp":1712365572.2584953,"reason":"broker was unresponsive"},"348":{"timestamp":1711576490.2548447,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"349":{"timestamp":1712365572.2586358,"reason":"broker was unresponsive"},"350":{"timestamp":1712365572.2587464,"reason":"broker was unresponsive"},"351":{"timestamp":1712365572.2589319,"reason":"broker was unresponsive"},"352":{"timestamp":1712365572.2590652,"reason":"broker was unresponsive"},"353":{"timestamp":1712365572.2592273,"reason":"broker was unresponsive"},"354":{"timestamp":1712365572.2594182,"reason":"broker was unresponsive"},"355":{"timestamp":1712365572.2595685,"reason":"broker was unresponsive"},"356":{"timestamp":1712365572.2596927,"reason":"broker was unresponsive"},"357":{"timestamp":1712365572.2598264,"reason":"broker was unresponsive"},"358":{"timestamp":1712365572.2599926,"reason":"broker was unresponsive"},"359":{"timestamp":1712365572.260107,"reason":"broker was unresponsive"},"360":{"timestamp":1712365572.2602549,"reason":"broker was unresponsive"},"361":{"timestamp":1712365572.2604394,"reason":"broker was unresponsive"},"362":{"timestamp":1712365572.2605715,"reason":"broker was unresponsive"},"363":{"timestamp":1712365572.2607214,"reason":"broker was unresponsive"},"364":{"timestamp":1712365572.2608461,"reason":"broker was unresponsive"},"365":{"timestamp":1712365572.2609866,"reason":"broker was unresponsive"},"366":{"timestamp":1712365572.2611835,"reason":"broker was unresponsive"},"367":{"timestamp":1712365572.2613223,"reason":"broker was unresponsive"},"368":{"timestamp":1712365572.2614841,"reason":"broker was unresponsive"},"369":{"timestamp":1712365572.2616174,"reason":"broker was unresponsive"},"370":{"timestamp":1712365572.2617965,"reason":"broker was unresponsive"},"371":{"timestamp":1712365572.2619493,"reason":"broker was unresponsive"},"372":{"timestamp":1712365572.2620864,"reason":"broker was unresponsive"},"373":{"timestamp":1712365572.2622414,"reason":"broker was unresponsive"},"374":{"timestamp":1712365572.2624378,"reason":"broker was unresponsive"},"375":{"timestamp":1712365572.2625926,"reason":"broker was unresponsive"},"376":{"timestamp":1712365572.2627516,"reason":"broker was unresponsive"},"377":{"timestamp":1712365572.2628973,"reason":"broker was unresponsive"},"378":{"timestamp":1712365572.2630675,"reason":"broker was unresponsive"},"379":{"timestamp":1712365572.263252,"reason":"broker was unresponsive"},"380":{"timestamp":1712365572.2634366,"reason":"broker was unresponsive"},"381":{"timestamp":1712365572.2635744,"reason":"broker was unresponsive"},"382":{"timestamp":1712365572.263726,"reason":"broker was unresponsive"},"383":{"timestamp":1712365572.263864,"reason":"broker was unresponsive"},"384":{"timestamp":1712365572.2640243,"reason":"broker was unresponsive"},"385":{"timestamp":1712365572.264183,"reason":"broker was unresponsive"},"386":{"timestamp":1712365572.2643523,"reason":"broker was unresponsive"},"387":{"timestamp":1712365572.2645359,"reason":"broker was unresponsive"},"388":{"timestamp":1712365572.264689,"reason":"broker was unresponsive"},"389":{"timestamp":1712365572.2648473,"reason":"broker was unresponsive"},"390":{"timestamp":1712365572.2650054,"reason":"broker was unresponsive"},"391":{"timestamp":1712365572.2651813,"reason":"broker was unresponsive"},"392":{"timestamp":1712365572.2653377,"reason":"broker was unresponsive"},"393":{"timestamp":1712365572.2655056,"reason":"broker was unresponsive"},"394":{"timestamp":1712365572.2656848,"reason":"broker was unresponsive"},"395":{"timestamp":1712365572.2658558,"reason":"broker was unresponsive"},"396":{"timestamp":1712365572.2660236,"reason":"broker was unresponsive"},"397":{"timestamp":1712365572.266165,"reason":"broker was unresponsive"},"398":{"timestamp":1712365572.2663405,"reason":"broker was unresponsive"},"399":{"timestamp":1712365572.2665014,"reason":"broker was unresponsive"},"400":{"timestamp":1712365572.2666881,"reason":"broker was unresponsive"},"401":{"timestamp":1712365572.2668436,"reason":"broker was unresponsive"},"402":{"timestamp":1712365572.2669966,"reason":"broker was unresponsive"},"403":{"timestamp":1712365572.2671745,"reason":"broker was unresponsive"},"404":{"timestamp":1712365572.267381,"reason":"broker was unresponsive"},"405":{"timestamp":1712365572.267556,"reason":"broker was unresponsive"},"406":{"timestamp":1712365572.2677186,"reason":"broker was unresponsive"},"407":{"timestamp":1712365572.2678699,"reason":"broker was unresponsive"},"408":{"timestamp":1712365572.2680836,"reason":"broker was unresponsive"},"409":{"timestamp":1712365572.2682476,"reason":"broker was unresponsive"},"410":{"timestamp":1712365572.2684119,"reason":"broker was unresponsive"},"411":{"timestamp":1712365572.2685771,"reason":"broker was unresponsive"},"412":{"timestamp":1712365572.2687278,"reason":"broker was unresponsive"},"413":{"timestamp":1712365572.2688804,"reason":"broker was unresponsive"},"414":{"timestamp":1712365572.2690475,"reason":"broker was unresponsive"},"415":{"timestamp":1712365572.2692211,"reason":"broker was unresponsive"},"416":{"timestamp":1712365572.2694154,"reason":"broker was unresponsive"},"417":{"timestamp":1712365572.2695661,"reason":"broker was unresponsive"},"418":{"timestamp":1712365572.2697184,"reason":"broker was unresponsive"},"419":{"timestamp":1712365572.2698672,"reason":"broker was unresponsive"},"420":{"timestamp":1712365572.270031,"reason":"broker was unresponsive"},"421":{"timestamp":1712365572.270216,"reason":"broker was unresponsive"},"422":{"timestamp":1712365572.2703731,"reason":"broker was unresponsive"},"423":{"timestamp":1712365572.2705886,"reason":"broker was unresponsive"},"424":{"timestamp":1712365572.2707787,"reason":"broker was unresponsive"},"425":{"timestamp":1712365572.2709115,"reason":"broker was unresponsive"},"426":{"timestamp":1712365572.2710454,"reason":"broker was unresponsive"},"427":{"timestamp":1712365572.2711689,"reason":"broker was unresponsive"},"428":{"timestamp":1712365572.2713249,"reason":"broker was unresponsive"},"429":{"timestamp":1712365572.2714541,"reason":"broker was unresponsive"},"430":{"timestamp":1712365572.271594,"reason":"broker was unresponsive"},"431":{"timestamp":1711096934.358676,"reason":"nodediag failed pci"},"432":{"timestamp":1712365572.2718632,"reason":"broker was unresponsive"},"433":{"timestamp":1712365572.2720077,"reason":"broker was unresponsive"},"434":{"timestamp":1712365572.2721453,"reason":"broker was unresponsive"},"435":{"timestamp":1712365572.2722859,"reason":"broker was unresponsive"},"436":{"timestamp":1712365572.2724187,"reason":"broker was unresponsive"},"437":{"timestamp":1712365572.2725475,"reason":"broker was unresponsive"},"438":{"timestamp":1712365572.2726729,"reason":"broker was unresponsive"},"439":{"timestamp":1712365572.2728074,"reason":"broker was unresponsive"},"440":{"timestamp":1712365572.272953,"reason":"broker was unresponsive"},"441":{"timestamp":1712365572.2731025,"reason":"broker was unresponsive"},"442":{"timestamp":1712365572.2732365,"reason":"broker was unresponsive"},"443":{"timestamp":1712365572.2733817,"reason":"broker was unresponsive"},"444":{"timestamp":1712365572.2735665,"reason":"broker was unresponsive"},"469":{"timestamp":1712365572.2737708,"reason":"broker was unresponsive"},"470":{"timestamp":1712365572.2739315,"reason":"broker was unresponsive"},"471":{"timestamp":1712365572.2740798,"reason":"broker was unresponsive"},"472":{"timestamp":1712365572.2742119,"reason":"broker was unresponsive"},"473":{"timestamp":1712365572.2743676,"reason":"broker was unresponsive"},"474":{"timestamp":1712365572.2745199,"reason":"broker was unresponsive"},"475":{"timestamp":1712365572.2746675,"reason":"broker was unresponsive"},"476":{"timestamp":1712365572.2747996,"reason":"broker was unresponsive"},"477":{"timestamp":1712365572.2749362,"reason":"broker was unresponsive"},"478":{"timestamp":1712365572.275068,"reason":"broker was unresponsive"},"479":{"timestamp":1712365572.275203,"reason":"broker was unresponsive"},"480":{"timestamp":1712365572.2753644,"reason":"broker was unresponsive"},"481":{"timestamp":1712365572.275517,"reason":"broker was unresponsive"},"482":{"timestamp":1712365572.2756519,"reason":"broker was unresponsive"},"483":{"timestamp":1712365572.2758021,"reason":"broker was unresponsive"},"484":{"timestamp":1712365572.2759569,"reason":"broker was unresponsive"},"485":{"timestamp":1712365572.2760921,"reason":"broker was unresponsive"},"486":{"timestamp":1712365572.2762446,"reason":"broker was unresponsive"},"487":{"timestamp":1712365572.2764091,"reason":"broker was unresponsive"},"488":{"timestamp":1712365572.2765806,"reason":"broker was unresponsive"},"489":{"timestamp":1712365572.2767344,"reason":"broker was unresponsive"},"490":{"timestamp":1712365572.2768693,"reason":"broker was unresponsive"},"491":{"timestamp":1712365572.2770088,"reason":"broker was unresponsive"},"492":{"timestamp":1712365572.2771466,"reason":"broker was unresponsive"},"493":{"timestamp":1712365572.2773159,"reason":"broker was unresponsive"},"494":{"timestamp":1712365572.2774546,"reason":"broker was unresponsive"},"495":{"timestamp":1712365572.2776456,"reason":"broker was unresponsive"},"496":{"timestamp":1712365572.277786,"reason":"broker was unresponsive"},"497":{"timestamp":1712365572.2779441,"reason":"broker was unresponsive"},"498":{"timestamp":1712365572.2780952,"reason":"broker was unresponsive"},"499":{"timestamp":1712365572.2782397,"reason":"broker was unresponsive"},"500":{"timestamp":1712365572.2783957,"reason":"broker was unresponsive"},"501":{"timestamp":1712365572.2785323,"reason":"broker was unresponsive"},"502":{"timestamp":1712365572.2787144,"reason":"broker was unresponsive"},"503":{"timestamp":1712365572.2788572,"reason":"broker was unresponsive"},"504":{"timestamp":1712365572.2790089,"reason":"broker was unresponsive"},"505":{"timestamp":1712365572.2791617,"reason":"broker was unresponsive"},"506":{"timestamp":1712365572.27932,"reason":"broker was unresponsive"},"507":{"timestamp":1712365572.279459,"reason":"broker was unresponsive"},"508":{"timestamp":1712365572.2795956,"reason":"broker was unresponsive"},"509":{"timestamp":1712365572.2797847,"reason":"broker was unresponsive"},"510":{"timestamp":1712365572.2799296,"reason":"broker was unresponsive"},"511":{"timestamp":1712365572.2800832,"reason":"broker was unresponsive"},"512":{"timestamp":1712365572.2802305,"reason":"broker was unresponsive"},"513":{"timestamp":1712365572.2803993,"reason":"broker was unresponsive"},"514":{"timestamp":1712365572.2805479,"reason":"broker was unresponsive"},"515":{"timestamp":1712365572.2806888,"reason":"broker was unresponsive"},"516":{"timestamp":1712365572.2808588,"reason":"broker was unresponsive"},"517":{"timestamp":1712365572.2810044,"reason":"broker was unresponsive"},"518":{"timestamp":1712365572.2811444,"reason":"broker was unresponsive"},"519":{"timestamp":1712365572.2813232,"reason":"broker was unresponsive"},"520":{"timestamp":1712365572.2814908,"reason":"broker was unresponsive"},"521":{"timestamp":1712365572.2816381,"reason":"broker was unresponsive"},"522":{"timestamp":1712365572.2817841,"reason":"broker was unresponsive"},"523":{"timestamp":1712365572.2819545,"reason":"broker was unresponsive"},"524":{"timestamp":1712365572.2821147,"reason":"broker was unresponsive"},"525":{"timestamp":1712365572.2822933,"reason":"broker was unresponsive"},"526":{"timestamp":1712365572.2824483,"reason":"broker was unresponsive"},"527":{"timestamp":1712365572.282604,"reason":"broker was unresponsive"},"528":{"timestamp":1712365572.2827566,"reason":"broker was unresponsive"},"529":{"timestamp":1712365572.2829111,"reason":"broker was unresponsive"},"530":{"timestamp":1712365572.2830932,"reason":"broker was unresponsive"},"531":{"timestamp":1712365572.2832422,"reason":"broker was unresponsive"},"532":{"timestamp":1712365572.283426,"reason":"broker was unresponsive"},"533":{"timestamp":1712365572.2835739,"reason":"broker was unresponsive"},"534":{"timestamp":1712365572.2837248,"reason":"broker was unresponsive"},"535":{"timestamp":1712365572.2838998,"reason":"broker was unresponsive"},"536":{"timestamp":1712365572.2840562,"reason":"broker was unresponsive"},"537":{"timestamp":1712365572.2842324,"reason":"broker was unresponsive"},"538":{"timestamp":1712365572.2843966,"reason":"broker was unresponsive"},"539":{"timestamp":1712365572.2845466,"reason":"broker was unresponsive"},"540":{"timestamp":1712365572.284693,"reason":"broker was unresponsive"},"445-468,541-564":{"timestamp":1711560189.6111379,"reason":"CDU work TB"},"565":{"timestamp":1712365572.2848408,"reason":"broker was unresponsive"},"566":{"timestamp":1712365572.2850063,"reason":"broker was unresponsive"},"567":{"timestamp":1712365572.2852187,"reason":"broker was unresponsive"},"568":{"timestamp":1712365572.2854061,"reason":"broker was unresponsive"},"569":{"timestamp":1712365572.2855771,"reason":"broker was unresponsive"},"570":{"timestamp":1712365572.2857399,"reason":"broker was unresponsive"},"571":{"timestamp":1712365572.2858939,"reason":"broker was unresponsive"},"572":{"timestamp":1712365572.2860723,"reason":"broker was unresponsive"},"573":{"timestamp":1712365572.2862372,"reason":"broker was unresponsive"},"574":{"timestamp":1712365572.2864635,"reason":"broker was unresponsive"},"575":{"timestamp":1712365572.2866285,"reason":"broker was unresponsive"},"576":{"timestamp":1712365572.2867982,"reason":"broker was unresponsive"},"577":{"timestamp":1712365572.2870302,"reason":"broker was unresponsive"},"578":{"timestamp":1712365572.2871921,"reason":"broker was unresponsive"},"579":{"timestamp":1712365572.2874193,"reason":"broker was unresponsive"},"580":{"timestamp":1712365572.2876408,"reason":"broker was unresponsive"},"581":{"timestamp":1712365572.287806,"reason":"broker was unresponsive"},"582":{"timestamp":1712365572.2879577,"reason":"broker was unresponsive"},"583":{"timestamp":1712365572.2881575,"reason":"broker was unresponsive"},"584":{"timestamp":1712365572.2883339,"reason":"broker was unresponsive"},"585":{"timestamp":1712365572.2885106,"reason":"broker was unresponsive"},"586":{"timestamp":1712365572.2886982,"reason":"broker was unresponsive"},"587":{"timestamp":1712365572.2888741,"reason":"broker was unresponsive"},"588":{"timestamp":1712365572.2890332,"reason":"broker was unresponsive"},"589":{"timestamp":1712154691.7782385,"reason":"broker was unresponsive"},"590":{"timestamp":1712154691.7894335,"reason":"broker was unresponsive"},"591":{"timestamp":1712154691.7896128,"reason":"broker was unresponsive"},"592":{"timestamp":1712154691.7897892,"reason":"broker was unresponsive"},"593":{"timestamp":1712154691.7899675,"reason":"broker was unresponsive"},"594":{"timestamp":1712154691.7901475,"reason":"broker was unresponsive"},"595":{"timestamp":1712154691.7903228,"reason":"broker was unresponsive"},"596":{"timestamp":1712154691.7904975,"reason":"broker was unresponsive"},"605":{"timestamp":1712154691.7921228,"reason":"broker was unresponsive"},"606":{"timestamp":1712154691.7922993,"reason":"broker was unresponsive"},"607":{"timestamp":1712154691.7924814,"reason":"broker was unresponsive"},"608":{"timestamp":1712154691.7926571,"reason":"broker was unresponsive"},"609":{"timestamp":1712154691.7928505,"reason":"broker was unresponsive"},"610":{"timestamp":1712154691.7930374,"reason":"broker was unresponsive"},"611":{"timestamp":1712154691.7932198,"reason":"broker was unresponsive"},"612":{"timestamp":1712154691.7934685,"reason":"broker was unresponsive"},"597-604,613-620":{"timestamp":1712255268.9196324,"reason":"New blade installations"},"621-628":{"timestamp":1712237659.4009576,"reason":"New blade install --JRG"},"629":{"timestamp":1712365572.2892213,"reason":"broker was unresponsive"},"630":{"timestamp":1712365572.2894192,"reason":"broker was unresponsive"},"631":{"timestamp":1712365572.2896099,"reason":"broker was unresponsive"},"632":{"timestamp":1712365572.2898138,"reason":"broker was unresponsive"},"633":{"timestamp":1712365572.2899923,"reason":"broker was unresponsive"},"634":{"timestamp":1712365572.2901745,"reason":"broker was unresponsive"},"635":{"timestamp":1712365572.2903678,"reason":"broker was unresponsive"},"636":{"timestamp":1712365572.2905288,"reason":"broker was unresponsive"},"637":{"timestamp":1711642252.1616328,"reason":"broker was unresponsive"},"638":{"timestamp":1711642252.1617086,"reason":"broker was unresponsive"},"639":{"timestamp":1711642252.161751,"reason":"broker was unresponsive"},"640":{"timestamp":1711642252.1617904,"reason":"broker was unresponsive"},"641":{"timestamp":1711642252.1618268,"reason":"broker was unresponsive"},"642":{"timestamp":1711642252.1618621,"reason":"broker was unresponsive"},"643":{"timestamp":1711642252.1618974,"reason":"broker was unresponsive"},"644":{"timestamp":1711642252.1619322,"reason":"broker was unresponsive"},"646":{"timestamp":1711586857.201689,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"647":{"timestamp":1711587136.0002518,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"648":{"timestamp":1711587197.7047591,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"649":{"timestamp":1711587211.9606316,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"650":{"timestamp":1711587232.1780922,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"651":{"timestamp":1711587244.2040467,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"652":{"timestamp":1711587255.7559283,"reason":"shutting down to test rebooting blades without rabbits - wendy"},"653":{"timestamp":1711642252.1619685,"reason":"broker was unresponsive"},"654":{"timestamp":1711642252.16201,"reason":"broker was unresponsive"},"655":{"timestamp":1711642252.162055,"reason":"broker was unresponsive"},"656":{"timestamp":1711642252.1620932,"reason":"broker was unresponsive"},"657":{"timestamp":1711642252.1621311,"reason":"broker was unresponsive"},"658":{"timestamp":1711642252.1621685,"reason":"broker was unresponsive"},"659":{"timestamp":1711642252.1622062,"reason":"broker was unresponsive"},"660":{"timestamp":1711642252.1622446,"reason":"broker was unresponsive"},"661":{"timestamp":1711642252.1622829,"reason":"broker was unresponsive"},"662":{"timestamp":1711642252.1623201,"reason":"broker was unresponsive"},"663":{"timestamp":1711642252.1623569,"reason":"broker was unresponsive"},"664":{"timestamp":1711642252.162394,"reason":"broker was unresponsive"},"665":{"timestamp":1711642252.1624658,"reason":"broker was unresponsive"},"666":{"timestamp":1711642252.1625173,"reason":"broker was unresponsive"},"667":{"timestamp":1711642258.156193,"reason":"broker was unresponsive"},"668":{"timestamp":1711642252.1625559,"reason":"broker was unresponsive"},"669":{"timestamp":1711642252.1625948,"reason":"broker was unresponsive"},"670":{"timestamp":1711642252.1626337,"reason":"broker was unresponsive"},"671":{"timestamp":1711642252.162673,"reason":"broker was unresponsive"},"672":{"timestamp":1711642252.1627119,"reason":"broker was unresponsive"},"673":{"timestamp":1711642252.1627512,"reason":"broker was unresponsive"},"674":{"timestamp":1711642252.1627915,"reason":"broker was unresponsive"},"675":{"timestamp":1711642252.1628311,"reason":"broker was unresponsive"},"676":{"timestamp":1711642252.1628704,"reason":"broker was unresponsive"},"677":{"timestamp":1711642258.1562793,"reason":"broker was unresponsive"},"678":{"timestamp":1711642258.156333,"reason":"broker was unresponsive"},"679":{"timestamp":1711642252.1629126,"reason":"broker was unresponsive"},"680":{"timestamp":1711642252.1629534,"reason":"broker was unresponsive"},"681":{"timestamp":1711642258.1563835,"reason":"broker was unresponsive"},"682":{"timestamp":1711642252.1629946,"reason":"broker was unresponsive"},"683":{"timestamp":1711642252.1630385,"reason":"broker was unresponsive"},"684":{"timestamp":1711642252.5597043,"reason":"broker was unresponsive"},"685":{"timestamp":1711604960.2562068,"reason":"broker was unresponsive"},"687":{"timestamp":1711604954.2552111,"reason":"broker was unresponsive"},"688":{"timestamp":1711606062.2558303,"reason":"broker was unresponsive"},"689":{"timestamp":1711604580.2552168,"reason":"broker was unresponsive"},"691":{"timestamp":1711604958.2564192,"reason":"broker was unresponsive"},"692":{"timestamp":1711606406.1556187,"reason":"broker was unresponsive"},"693":{"timestamp":1711636838.2555192,"reason":"broker was unresponsive"},"695":{"timestamp":1711604326.2549248,"reason":"broker was unresponsive"},"696":{"timestamp":1711606406.256216,"reason":"broker was unresponsive"},"697":{"timestamp":1711604392.2562771,"reason":"broker was unresponsive"},"698":{"timestamp":1711605716.2557297,"reason":"broker was unresponsive"},"699":{"timestamp":1711604948.2558951,"reason":"broker was unresponsive"},"700":{"timestamp":1711606868.2557678,"reason":"broker was unresponsive"},"701":{"timestamp":1711604340.2561386,"reason":"broker was unresponsive"},"702":{"timestamp":1711606408.2602317,"reason":"broker was unresponsive"},"703":{"timestamp":1711604918.2557685,"reason":"broker was unresponsive"},"704":{"timestamp":1711604590.2548223,"reason":"broker was unresponsive"},"705":{"timestamp":1711604968.2570915,"reason":"broker was unresponsive"},"707":{"timestamp":1711604908.2558758,"reason":"broker was unresponsive"},"686,690,694,706,708":{"timestamp":1711615908.1602454,"reason":"epilog failed for jobid fm9HsYwFjjd"},"709":{"timestamp":1711408620.9668899,"reason":"Cabinet power work needed -jrg"},"710":{"timestamp":1711409910.967391,"reason":"Cabinet power work needed -jrg"},"711":{"timestamp":1711409098.9674611,"reason":"Cabinet power work needed -jrg"},"712":{"timestamp":1711411606.9678445,"reason":"Cabinet power work needed -jrg"},"713":{"timestamp":1711408618.8669355,"reason":"Cabinet power work needed -jrg"},"715":{"timestamp":1711409132.9680476,"reason":"Cabinet power work needed -jrg"},"714,716":{"timestamp":1711564709.7804377,"reason":"Cabinet power work needed -jrg"},"718":{"timestamp":1711642210.1588371,"reason":"broker was unresponsive"},"719":{"timestamp":1711642210.1589563,"reason":"broker was unresponsive"},"720":{"timestamp":1711642210.1590128,"reason":"broker was unresponsive"},"721":{"timestamp":1711642210.1590643,"reason":"broker was unresponsive"},"722":{"timestamp":1711642210.1590991,"reason":"broker was unresponsive"},"723":{"timestamp":1711642210.1591821,"reason":"broker was unresponsive"},"724":{"timestamp":1711642210.1592195,"reason":"broker was unresponsive"},"725":{"timestamp":1711593222.2542431,"reason":"broker was unresponsive"},"726":{"timestamp":1711593312.2549782,"reason":"broker was unresponsive"},"727":{"timestamp":1711593332.2552569,"reason":"broker was unresponsive"},"728":{"timestamp":1711593356.2547033,"reason":"broker was unresponsive"},"729":{"timestamp":1711593414.254441,"reason":"broker was unresponsive"},"730":{"timestamp":1711593462.2552798,"reason":"broker was unresponsive"},"731":{"timestamp":1711593510.2550235,"reason":"broker was unresponsive"},"732":{"timestamp":1711593530.2558599,"reason":"broker was unresponsive"},"733":{"timestamp":1711642210.1592536,"reason":"broker was unresponsive"},"734":{"timestamp":1711486000.9676955,"reason":"broker was unresponsive"},"735":{"timestamp":1711642210.1592872,"reason":"broker was unresponsive"},"736":{"timestamp":1711642210.1593204,"reason":"broker was unresponsive"},"737":{"timestamp":1711642210.1593742,"reason":"broker was unresponsive"},"738":{"timestamp":1711642210.1594355,"reason":"broker was unresponsive"},"739":{"timestamp":1711642210.1594796,"reason":"broker was unresponsive"},"740":{"timestamp":1711642210.159517,"reason":"broker was unresponsive"},"741":{"timestamp":1711642210.1595554,"reason":"broker was unresponsive"},"742":{"timestamp":1711642210.1595943,"reason":"broker was unresponsive"},"743":{"timestamp":1711642210.1596365,"reason":"broker was unresponsive"},"744":{"timestamp":1711642210.1596746,"reason":"broker was unresponsive"},"745":{"timestamp":1711642210.1597111,"reason":"broker was unresponsive"},"746":{"timestamp":1711506258.2527435,"reason":"broker was unresponsive"},"747":{"timestamp":1711642210.1597559,"reason":"broker was unresponsive"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"},"749":{"timestamp":1711642210.1598115,"reason":"broker was unresponsive"},"750":{"timestamp":1711642210.1598501,"reason":"broker was unresponsive"},"751":{"timestamp":1711642210.159888,"reason":"broker was unresponsive"},"752":{"timestamp":1711642210.1599293,"reason":"broker was unresponsive"},"753":{"timestamp":1711642210.1599746,"reason":"broker was unresponsive"},"754":{"timestamp":1711642210.1600144,"reason":"broker was unresponsive"},"755":{"timestamp":1711642210.1600802,"reason":"broker was unresponsive"},"756":{"timestamp":1711642210.5039413,"reason":"broker was unresponsive"},"789-794":{"timestamp":1711461777.1299369,"reason":""},"796":{"timestamp":1712254304.6269586,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261862.602201,"reason":"--reason swapping blades into x1900 - wendy"},"799":{"timestamp":1712330474.6423876,"reason":"broker was unresponsive"},"800":{"timestamp":1712330474.6481371,"reason":"broker was unresponsive"},"801":{"timestamp":1712338139.3428166,"reason":"broker was unresponsive"},"802":{"timestamp":1712338139.445405,"reason":"broker was unresponsive"},"803":{"timestamp":1712337593.3448501,"reason":"broker was unresponsive"},"804":{"timestamp":1712337593.4446397,"reason":"broker was unresponsive"},"805":{"timestamp":1712348261.3433502,"reason":"broker was unresponsive"},"806":{"timestamp":1712348262.1058218,"reason":"broker was unresponsive"},"807":{"timestamp":1712331487.341608,"reason":"broker was unresponsive"},"808":{"timestamp":1712331487.4412344,"reason":"broker was unresponsive"},"809":{"timestamp":1712330683.3417573,"reason":"broker was unresponsive"},"810":{"timestamp":1712330683.341866,"reason":"broker was unresponsive"},"811":{"timestamp":1712330683.3419075,"reason":"broker was unresponsive"},"812":{"timestamp":1712330683.4417996,"reason":"broker was unresponsive"},"813":{"timestamp":1712348147.3444083,"reason":"broker was unresponsive"},"814":{"timestamp":1712348148.0782108,"reason":"broker was unresponsive"},"815":{"timestamp":1712349359.343132,"reason":"broker was unresponsive"},"816":{"timestamp":1712349360.1984277,"reason":"broker was unresponsive"},"819":{"timestamp":1712342273.5977426,"reason":""},"820":{"timestamp":1712342217.7655268,"reason":""},"821-824":{"timestamp":1712333755.1207211,"reason":""},"825":{"timestamp":1712348303.343436,"reason":"broker was unresponsive"},"826":{"timestamp":1712348304.0895774,"reason":"broker was unresponsive"},"827":{"timestamp":1712365572.2907131,"reason":"broker was unresponsive"},"828":{"timestamp":1712365572.2909091,"reason":"broker was unresponsive"},"829":{"timestamp":1712365572.2911112,"reason":"broker was unresponsive"},"830":{"timestamp":1712365572.2913167,"reason":"broker was unresponsive"},"831":{"timestamp":1712365572.291532,"reason":"broker was unresponsive"},"832":{"timestamp":1712365572.2917318,"reason":"broker was unresponsive"},"833":{"timestamp":1712365572.2919226,"reason":"broker was unresponsive"},"834":{"timestamp":1712365572.292141,"reason":"broker was unresponsive"},"835":{"timestamp":1712352395.3461986,"reason":"broker was unresponsive"},"836":{"timestamp":1712352396.0807211,"reason":"broker was unresponsive"},"841":{"timestamp":1712365572.2923977,"reason":"broker was unresponsive"},"842":{"timestamp":1712365572.2926667,"reason":"broker was unresponsive"},"843":{"timestamp":1712088721.1281126,"reason":"broker was unresponsive"},"844":{"timestamp":1712088721.1284544,"reason":"broker was unresponsive"},"870":{"timestamp":1712334859.4442852,"reason":"broker was unresponsive"},"871":{"timestamp":1712365572.2928617,"reason":"broker was unresponsive"},"872":{"timestamp":1712365572.2930267,"reason":"broker was unresponsive"},"874":{"timestamp":1711668507.0706048,"reason":"broker was unresponsive"},"875":{"timestamp":1711668503.0621645,"reason":"broker was unresponsive"},"881":{"timestamp":1712154691.9234848,"reason":"broker was unresponsive"},"882":{"timestamp":1712154691.9423342,"reason":"broker was unresponsive"},"883-884":{"timestamp":1711734121.1560712,"reason":""},"949":{"timestamp":1712365572.293196,"reason":"broker was unresponsive"},"950":{"timestamp":1712365572.2933846,"reason":"broker was unresponsive"},"951":{"timestamp":1712365572.2935448,"reason":"broker was unresponsive"},"952":{"timestamp":1712365572.2937517,"reason":"broker was unresponsive"},"953":{"timestamp":1712365572.2939444,"reason":"broker was unresponsive"},"954":{"timestamp":1712365572.294106,"reason":"broker was unresponsive"},"963":{"timestamp":1712365572.2943177,"reason":"broker was unresponsive"},"964":{"timestamp":1712365572.2944853,"reason":"broker was unresponsive"},"965":{"timestamp":1711663706.9630723,"reason":"broker was unresponsive"},"966":{"timestamp":1711663706.9631405,"reason":"broker was unresponsive"},"967":{"timestamp":1711663706.963171,"reason":"broker was unresponsive"},"968":{"timestamp":1711663706.9631999,"reason":"broker was unresponsive"},"969":{"timestamp":1712241030.3444023,"reason":"broker was unresponsive"},"970":{"timestamp":1712241031.3465993,"reason":"broker was unresponsive"},"971":{"timestamp":1712241434.5243275,"reason":"broker was unresponsive"},"972":{"timestamp":1712241434.6244709,"reason":"broker was unresponsive"},"973":{"timestamp":1712241476.5255573,"reason":"broker was unresponsive"},"974":{"timestamp":1712241476.6259952,"reason":"broker was unresponsive"},"975":{"timestamp":1712365572.2946494,"reason":"broker was unresponsive"},"976":{"timestamp":1712365572.2948294,"reason":"broker was unresponsive"},"977":{"timestamp":1712154691.9848833,"reason":"broker was unresponsive"},"978":{"timestamp":1712241482.6270466,"reason":"broker was unresponsive"},"979":{"timestamp":1712241488.5256779,"reason":"broker was unresponsive"},"980":{"timestamp":1712241488.6261418,"reason":"broker was unresponsive"},"9205":{"timestamp":1712365572.2952271,"reason":"broker was unresponsive"},"9206":{"timestamp":1712365572.2955875,"reason":"broker was unresponsive"},"9207":{"timestamp":1712365572.296021,"reason":"broker was unresponsive"},"9208":{"timestamp":1712365572.2964427,"reason":"broker was unresponsive"},"9209":{"timestamp":1712365572.2968445,"reason":"broker was unresponsive"},"9210":{"timestamp":1712365572.2972867,"reason":"broker was unresponsive"},"9211":{"timestamp":1712365572.297647,"reason":"broker was unresponsive"},"9212":{"timestamp":1712365572.2979949,"reason":"broker was unresponsive"},"9213":{"timestamp":1712365572.2983701,"reason":"broker was unresponsive"},"9214":{"timestamp":1712350524.2175126,"reason":"CXI Starting --JRG"},"9215":{"timestamp":1712365572.2987416,"reason":"broker was unresponsive"},"9216":{"timestamp":1712365572.2990794,"reason":"broker was unresponsive"},"9217":{"timestamp":1712365572.2995129,"reason":"broker was unresponsive"},"9218":{"timestamp":1712365572.2999234,"reason":"broker was unresponsive"},"9219":{"timestamp":1712365572.3003852,"reason":"broker was unresponsive"},"9220":{"timestamp":1712365572.3007545,"reason":"broker was unresponsive"},"9221":{"timestamp":1712365572.3011456,"reason":"broker was unresponsive"},"9222":{"timestamp":1712365572.3015721,"reason":"broker was unresponsive"},"9223":{"timestamp":1712365572.3019235,"reason":"broker was unresponsive"},"9224":{"timestamp":1712365572.3022864,"reason":"broker was unresponsive"},"9225":{"timestamp":1712365572.3026607,"reason":"broker was unresponsive"},"9226":{"timestamp":1712365572.3030438,"reason":"broker was unresponsive"},"9227":{"timestamp":1712365572.3034568,"reason":"broker was unresponsive"},"9228":{"timestamp":1712365572.3038936,"reason":"broker was unresponsive"},"9229":{"timestamp":1712365572.3042996,"reason":"broker was unresponsive"},"9230":{"timestamp":1712365572.3047299,"reason":"broker was unresponsive"},"9231":{"timestamp":1712365572.3051598,"reason":"broker was unresponsive"},"9232":{"timestamp":1712365572.3055997,"reason":"broker was unresponsive"},"9233":{"timestamp":1712365572.3059926,"reason":"broker was unresponsive"},"9234":{"timestamp":1712365572.3064158,"reason":"broker was unresponsive"},"9235":{"timestamp":1712365572.3068275,"reason":"broker was unresponsive"},"9236":{"timestamp":1712365572.3072319,"reason":"broker was unresponsive"},"9237":{"timestamp":1712365572.3076165,"reason":"broker was unresponsive"},"9238":{"timestamp":1712365572.3079703,"reason":"broker was unresponsive"},"9241":{"timestamp":1712365572.3083317,"reason":"broker was unresponsive"},"9242":{"timestamp":1712365572.3087118,"reason":"broker was unresponsive"},"9243":{"timestamp":1712365572.3090804,"reason":"broker was unresponsive"},"9244":{"timestamp":1712365572.3094623,"reason":"broker was unresponsive"},"9245":{"timestamp":1712365572.3098512,"reason":"broker was unresponsive"},"9246":{"timestamp":1712365572.3102007,"reason":"broker was unresponsive"},"9247":{"timestamp":1712365572.3105772,"reason":"broker was unresponsive"},"9248":{"timestamp":1712365572.3109539,"reason":"broker was unresponsive"},"9249":{"timestamp":1712365572.3113551,"reason":"broker was unresponsive"},"9250":{"timestamp":1712365572.3117914,"reason":"broker was unresponsive"},"9251":{"timestamp":1712365572.312165,"reason":"broker was unresponsive"},"9252":{"timestamp":1712365572.3125522,"reason":"broker was unresponsive"},"9253":{"timestamp":1712365572.3129256,"reason":"broker was unresponsive"},"9254":{"timestamp":1712365572.313297,"reason":"broker was unresponsive"},"9255":{"timestamp":1712365572.3137007,"reason":"broker was unresponsive"},"9256":{"timestamp":1712365572.314054,"reason":"broker was unresponsive"},"9257":{"timestamp":1712365572.3144908,"reason":"broker was unresponsive"},"9258":{"timestamp":1712365572.314899,"reason":"broker was unresponsive"},"9259":{"timestamp":1712365572.3152778,"reason":"broker was unresponsive"},"9260":{"timestamp":1712365572.3157024,"reason":"broker was unresponsive"},"9261":{"timestamp":1712365572.3161159,"reason":"broker was unresponsive"},"9262":{"timestamp":1712365572.3165081,"reason":"broker was unresponsive"},"9263":{"timestamp":1712365572.3169003,"reason":"broker was unresponsive"},"9264":{"timestamp":1712365572.317277,"reason":"broker was unresponsive"},"9265":{"timestamp":1712365572.3176601,"reason":"broker was unresponsive"},"9266":{"timestamp":1712365572.3180387,"reason":"broker was unresponsive"},"9267":{"timestamp":1712365572.3184505,"reason":"broker was unresponsive"},"9268":{"timestamp":1712365572.3188241,"reason":"broker was unresponsive"},"9269":{"timestamp":1712365572.3192263,"reason":"broker was unresponsive"},"9270":{"timestamp":1712365572.3196049,"reason":"broker was unresponsive"},"9271":{"timestamp":1712365572.320034,"reason":"broker was unresponsive"},"9272":{"timestamp":1712365572.3204567,"reason":"broker was unresponsive"},"9273":{"timestamp":1712365572.320859,"reason":"broker was unresponsive"},"9274":{"timestamp":1712365572.3212461,"reason":"broker was unresponsive"},"9275":{"timestamp":1712365572.3216519,"reason":"broker was unresponsive"},"9276":{"timestamp":1712365572.3220909,"reason":"broker was unresponsive"},"9277":{"timestamp":1712365572.3225801,"reason":"broker was unresponsive"},"9278":{"timestamp":1712365572.3230762,"reason":"broker was unresponsive"},"9279":{"timestamp":1712365572.3235774,"reason":"broker was unresponsive"},"9280":{"timestamp":1712365572.323946,"reason":"broker was unresponsive"},"9281":{"timestamp":1712365572.3243504,"reason":"broker was unresponsive"},"9282":{"timestamp":1712365572.3247416,"reason":"broker was unresponsive"},"9283":{"timestamp":1712365572.3251071,"reason":"broker was unresponsive"},"9284":{"timestamp":1712365572.3255732,"reason":"broker was unresponsive"},"9285":{"timestamp":1712365572.3259771,"reason":"broker was unresponsive"},"9286":{"timestamp":1712365572.3263881,"reason":"broker was unresponsive"},"9287":{"timestamp":1712365572.3268397,"reason":"broker was unresponsive"},"9288":{"timestamp":1712365572.327239,"reason":"broker was unresponsive"},"9289":{"timestamp":1712365572.3276513,"reason":"broker was unresponsive"},"9290":{"timestamp":1712365572.3280807,"reason":"broker was unresponsive"},"9291":{"timestamp":1712365572.3286028,"reason":"broker was unresponsive"},"9292":{"timestamp":1712365572.3291101,"reason":"broker was unresponsive"},"9293":{"timestamp":1712365572.3296106,"reason":"broker was unresponsive"},"9294":{"timestamp":1712365572.3303347,"reason":"broker was unresponsive"},"9295":{"timestamp":1712365572.3307176,"reason":"broker was unresponsive"},"9296":{"timestamp":1712365572.3311355,"reason":"broker was unresponsive"},"9297":{"timestamp":1712365572.331594,"reason":"broker was unresponsive"},"9298":{"timestamp":1712365572.3319633,"reason":"broker was unresponsive"},"9299":{"timestamp":1712365572.3323581,"reason":"broker was unresponsive"},"9300":{"timestamp":1712365572.332725,"reason":"broker was unresponsive"},"9301":{"timestamp":1712365572.3330855,"reason":"broker was unresponsive"},"9302":{"timestamp":1712365572.3334589,"reason":"broker was unresponsive"},"9303":{"timestamp":1712365572.3338201,"reason":"broker was unresponsive"},"9304":{"timestamp":1712365572.3341796,"reason":"broker was unresponsive"},"9305":{"timestamp":1712365572.3345523,"reason":"broker was unresponsive"},"9306":{"timestamp":1712365572.3349161,"reason":"broker was unresponsive"},"9307":{"timestamp":1712365572.3352871,"reason":"broker was unresponsive"},"9308":{"timestamp":1712365572.3356507,"reason":"broker was unresponsive"},"9309":{"timestamp":1712365572.3360145,"reason":"broker was unresponsive"},"9310":{"timestamp":1712365572.3363879,"reason":"broker was unresponsive"},"9311":{"timestamp":1712365572.3367529,"reason":"broker was unresponsive"},"9312":{"timestamp":1712365572.3371174,"reason":"broker was unresponsive"},"9313":{"timestamp":1712365572.3375506,"reason":"broker was unresponsive"},"9314":{"timestamp":1712365572.3379519,"reason":"broker was unresponsive"},"9315":{"timestamp":1712365572.3383715,"reason":"broker was unresponsive"},"9316":{"timestamp":1712350549.0126185,"reason":"Fails loopback test --JRG"},"9317":{"timestamp":1712365572.3388176,"reason":"broker was unresponsive"},"9318":{"timestamp":1712350566.2323,"reason":"Fails bogomips --JRG"},"9319":{"timestamp":1712365572.3392313,"reason":"broker was unresponsive"},"9320":{"timestamp":1712365572.3397138,"reason":"broker was unresponsive"},"9321":{"timestamp":1712365572.3401206,"reason":"broker was unresponsive"},"9322":{"timestamp":1712365572.3405697,"reason":"broker was unresponsive"},"9323":{"timestamp":1712365572.3409452,"reason":"broker was unresponsive"},"9324":{"timestamp":1712365572.3414855,"reason":"broker was unresponsive"},"9325":{"timestamp":1712365572.3418963,"reason":"broker was unresponsive"},"9326":{"timestamp":1712365572.3422897,"reason":"broker was unresponsive"},"9327":{"timestamp":1712365572.3426609,"reason":"broker was unresponsive"},"9328":{"timestamp":1712365572.3430295,"reason":"broker was unresponsive"},"9329":{"timestamp":1712365572.343416,"reason":"broker was unresponsive"},"9330":{"timestamp":1712365572.3437881,"reason":"broker was unresponsive"},"9331":{"timestamp":1712365572.344156,"reason":"broker was unresponsive"},"9332":{"timestamp":1712365572.3445544,"reason":"broker was unresponsive"},"9333":{"timestamp":1712365572.3449254,"reason":"broker was unresponsive"},"9334":{"timestamp":1712365572.3453109,"reason":"broker was unresponsive"},"9335":{"timestamp":1712365572.345686,"reason":"broker was unresponsive"},"9336":{"timestamp":1712365572.3460591,"reason":"broker was unresponsive"},"9337":{"timestamp":1712365572.3464417,"reason":"broker was unresponsive"},"9338":{"timestamp":1712365572.3468146,"reason":"broker was unresponsive"},"9339":{"timestamp":1712365572.3472128,"reason":"broker was unresponsive"},"9340":{"timestamp":1712365572.3476036,"reason":"broker was unresponsive"},"9341":{"timestamp":1712365572.3479791,"reason":"broker was unresponsive"},"9342":{"timestamp":1712365572.3483694,"reason":"broker was unresponsive"},"9343":{"timestamp":1712365572.3487444,"reason":"broker was unresponsive"},"9344":{"timestamp":1712365572.3491178,"reason":"broker was unresponsive"},"9345":{"timestamp":1712365572.349508,"reason":"broker was unresponsive"},"9346":{"timestamp":1712365572.3499067,"reason":"broker was unresponsive"},"9347":{"timestamp":1712365572.3503244,"reason":"broker was unresponsive"},"9348":{"timestamp":1712365572.3507071,"reason":"broker was unresponsive"},"9349":{"timestamp":1712365572.3511038,"reason":"broker was unresponsive"},"9350":{"timestamp":1712365572.3515794,"reason":"broker was unresponsive"},"9351":{"timestamp":1712365572.352,"reason":"broker was unresponsive"},"9352":{"timestamp":1712365572.3524508,"reason":"broker was unresponsive"},"9353":{"timestamp":1712365572.3528392,"reason":"broker was unresponsive"},"9354":{"timestamp":1712365572.3532486,"reason":"broker was unresponsive"},"9355":{"timestamp":1712365572.3536842,"reason":"broker was unresponsive"},"9356":{"timestamp":1712365572.3541024,"reason":"broker was unresponsive"},"9357":{"timestamp":1712365572.3545299,"reason":"broker was unresponsive"},"9358":{"timestamp":1712365572.3549948,"reason":"broker was unresponsive"},"9359":{"timestamp":1712365572.3554275,"reason":"broker was unresponsive"},"9360":{"timestamp":1712365572.3558381,"reason":"broker was unresponsive"},"9361":{"timestamp":1712365572.3562491,"reason":"broker was unresponsive"},"9362":{"timestamp":1712365572.356678,"reason":"broker was unresponsive"},"9363":{"timestamp":1712365572.357111,"reason":"broker was unresponsive"},"9364":{"timestamp":1712365572.3575432,"reason":"broker was unresponsive"},"9365":{"timestamp":1712365572.3579662,"reason":"broker was unresponsive"},"9366":{"timestamp":1712365572.3583727,"reason":"broker was unresponsive"},"9367":{"timestamp":1712365572.3587983,"reason":"broker was unresponsive"},"9368":{"timestamp":1712365572.3591878,"reason":"broker was unresponsive"},"9369":{"timestamp":1712365572.3595841,"reason":"broker was unresponsive"},"9370":{"timestamp":1712365572.3599703,"reason":"broker was unresponsive"},"9371":{"timestamp":1712365572.3604333,"reason":"broker was unresponsive"},"9372":{"timestamp":1712365572.3608251,"reason":"broker was unresponsive"},"9373":{"timestamp":1712365572.3612773,"reason":"broker was unresponsive"},"9374":{"timestamp":1712365572.3616908,"reason":"broker was unresponsive"},"9375":{"timestamp":1712365572.3621163,"reason":"broker was unresponsive"},"9376":{"timestamp":1712365572.3625994,"reason":"broker was unresponsive"},"9377":{"timestamp":1712365572.3630276,"reason":"broker was unresponsive"},"9378":{"timestamp":1712365572.3634908,"reason":"broker was unresponsive"},"9379":{"timestamp":1712365572.3639109,"reason":"broker was unresponsive"},"9380":{"timestamp":1712365572.3643198,"reason":"broker was unresponsive"},"9381":{"timestamp":1712365572.3647523,"reason":"broker was unresponsive"},"9382":{"timestamp":1712365572.3651464,"reason":"broker was unresponsive"},"9383":{"timestamp":1712365572.3655953,"reason":"broker was unresponsive"},"9384":{"timestamp":1712365572.365994,"reason":"broker was unresponsive"},"9385":{"timestamp":1712365572.3663993,"reason":"broker was unresponsive"},"9386":{"timestamp":1712365572.3667903,"reason":"broker was unresponsive"},"9387":{"timestamp":1712365572.3671815,"reason":"broker was unresponsive"},"9388":{"timestamp":1712365572.3675973,"reason":"broker was unresponsive"},"9389":{"timestamp":1712365572.3679898,"reason":"broker was unresponsive"},"9390":{"timestamp":1712365572.3683949,"reason":"broker was unresponsive"},"9391":{"timestamp":1712365572.3687873,"reason":"broker was unresponsive"},"9392":{"timestamp":1712365572.3691785,"reason":"broker was unresponsive"},"9393":{"timestamp":1712365572.3695843,"reason":"broker was unresponsive"},"9394":{"timestamp":1712365572.3699763,"reason":"broker was unresponsive"},"9395":{"timestamp":1712365572.3703845,"reason":"broker was unresponsive"},"9396":{"timestamp":1712365572.3707793,"reason":"broker was unresponsive"},"9397":{"timestamp":1712365572.371172,"reason":"broker was unresponsive"},"9398":{"timestamp":1712365572.3715844,"reason":"broker was unresponsive"},"9399":{"timestamp":1712365572.3719792,"reason":"broker was unresponsive"},"9400":{"timestamp":1712365572.3723879,"reason":"broker was unresponsive"},"9401":{"timestamp":1712365572.3727884,"reason":"broker was unresponsive"},"9402":{"timestamp":1712365572.373184,"reason":"broker was unresponsive"},"9403":{"timestamp":1712365572.3735936,"reason":"broker was unresponsive"},"9404":{"timestamp":1712365572.3739939,"reason":"broker was unresponsive"},"9405":{"timestamp":1712365572.3744025,"reason":"broker was unresponsive"},"9406":{"timestamp":1712365572.3748012,"reason":"broker was unresponsive"},"9407":{"timestamp":1712365572.3751996,"reason":"broker was unresponsive"},"9408":{"timestamp":1712365572.3756082,"reason":"broker was unresponsive"},"9409":{"timestamp":1712365572.37604,"reason":"broker was unresponsive"},"9410":{"timestamp":1712365572.3765039,"reason":"broker was unresponsive"},"9411":{"timestamp":1712365572.3769233,"reason":"broker was unresponsive"},"9412":{"timestamp":1712365572.3773606,"reason":"broker was unresponsive"},"9413":{"timestamp":1712365572.3778069,"reason":"broker was unresponsive"},"9414":{"timestamp":1712365572.3782375,"reason":"broker was unresponsive"},"9415":{"timestamp":1712365572.3787589,"reason":"broker was unresponsive"},"9416":{"timestamp":1712365572.3793902,"reason":"broker was unresponsive"},"9417":{"timestamp":1712365572.3799071,"reason":"broker was unresponsive"},"9418":{"timestamp":1712365572.3804541,"reason":"broker was unresponsive"},"9419":{"timestamp":1712365572.3809621,"reason":"broker was unresponsive"},"9420":{"timestamp":1712365572.3814833,"reason":"broker was unresponsive"},"9421":{"timestamp":1712365572.3820152,"reason":"broker was unresponsive"},"9422":{"timestamp":1712365572.382534,"reason":"broker was unresponsive"},"9423":{"timestamp":1712365572.3830402,"reason":"broker was unresponsive"},"9424":{"timestamp":1712365572.3835061,"reason":"broker was unresponsive"},"9425":{"timestamp":1712365572.3839669,"reason":"broker was unresponsive"},"9426":{"timestamp":1712365572.3845413,"reason":"broker was unresponsive"},"9427":{"timestamp":1712365572.3850296,"reason":"broker was unresponsive"},"9428":{"timestamp":1712365572.3855517,"reason":"broker was unresponsive"},"9429":{"timestamp":1712365572.3860972,"reason":"broker was unresponsive"},"9430":{"timestamp":1712365572.3866255,"reason":"broker was unresponsive"},"9431":{"timestamp":1712365572.3871996,"reason":"broker was unresponsive"},"9432":{"timestamp":1712365572.387713,"reason":"broker was unresponsive"},"9433":{"timestamp":1712365572.3882332,"reason":"broker was unresponsive"},"9434":{"timestamp":1712365572.3887851,"reason":"broker was unresponsive"},"9435":{"timestamp":1712365572.3893387,"reason":"broker was unresponsive"},"9436":{"timestamp":1712365572.3899322,"reason":"broker was unresponsive"},"9437":{"timestamp":1712365572.3904872,"reason":"broker was unresponsive"},"9438":{"timestamp":1712365572.3910923,"reason":"broker was unresponsive"},"9439":{"timestamp":1712365572.3916361,"reason":"broker was unresponsive"},"9440":{"timestamp":1712365572.3921294,"reason":"broker was unresponsive"},"9441":{"timestamp":1712365572.3926804,"reason":"broker was unresponsive"},"9442":{"timestamp":1712365572.3932047,"reason":"broker was unresponsive"},"9443":{"timestamp":1712365572.3939195,"reason":"broker was unresponsive"},"9444":{"timestamp":1712365572.3945632,"reason":"broker was unresponsive"},"9445":{"timestamp":1712365572.395148,"reason":"broker was unresponsive"},"9446":{"timestamp":1712365572.3956766,"reason":"broker was unresponsive"},"9447":{"timestamp":1712272276.4058383,"reason":"CXI Stuck at Starting --JRG"},"9448":{"timestamp":1712272297.6105912,"reason":"bad partner node --JRG"},"9449":{"timestamp":1712365572.3961937,"reason":"broker was unresponsive"},"9450":{"timestamp":1712365572.3967452,"reason":"broker was unresponsive"},"9451":{"timestamp":1712365572.3972845,"reason":"broker was unresponsive"},"9452":{"timestamp":1712365572.3978529,"reason":"broker was unresponsive"},"9453":{"timestamp":1712365572.398407,"reason":"broker was unresponsive"},"9454":{"timestamp":1712365572.3989625,"reason":"broker was unresponsive"},"9455":{"timestamp":1712365572.3995063,"reason":"broker was unresponsive"},"9456":{"timestamp":1712365572.4000347,"reason":"broker was unresponsive"},"9457":{"timestamp":1712365572.4005325,"reason":"broker was unresponsive"},"9458":{"timestamp":1712365572.4010484,"reason":"broker was unresponsive"},"9459":{"timestamp":1712365572.4016027,"reason":"broker was unresponsive"},"9460":{"timestamp":1712365572.4021955,"reason":"broker was unresponsive"},"9461":{"timestamp":1712365572.402739,"reason":"broker was unresponsive"},"9462":{"timestamp":1712365572.4032285,"reason":"broker was unresponsive"},"9463":{"timestamp":1712365572.4037616,"reason":"broker was unresponsive"},"9464":{"timestamp":1712365572.4043248,"reason":"broker was unresponsive"},"9465":{"timestamp":1712365572.404882,"reason":"broker was unresponsive"},"9466":{"timestamp":1712365572.4054239,"reason":"broker was unresponsive"},"9467":{"timestamp":1712365572.4059749,"reason":"broker was unresponsive"},"9468":{"timestamp":1712365572.4065804,"reason":"broker was unresponsive"},"9469":{"timestamp":1712365572.4071465,"reason":"broker was unresponsive"},"9470":{"timestamp":1712365572.4076934,"reason":"broker was unresponsive"},"9471":{"timestamp":1712365572.4081283,"reason":"broker was unresponsive"},"9472":{"timestamp":1712365572.4086525,"reason":"broker was unresponsive"},"9473":{"timestamp":1712365572.4092138,"reason":"broker was unresponsive"},"9474":{"timestamp":1712365572.4097822,"reason":"broker was unresponsive"},"9475":{"timestamp":1712365572.4103234,"reason":"broker was unresponsive"},"9476":{"timestamp":1712365572.4108651,"reason":"broker was unresponsive"},"9477":{"timestamp":1712365572.4114223,"reason":"broker was unresponsive"},"9478":{"timestamp":1712365572.4119496,"reason":"broker was unresponsive"},"9479":{"timestamp":1712365572.4125671,"reason":"broker was unresponsive"},"9480":{"timestamp":1712365572.4132147,"reason":"broker was unresponsive"},"9481":{"timestamp":1712365572.4138775,"reason":"broker was unresponsive"},"9482":{"timestamp":1712365572.4144835,"reason":"broker was unresponsive"},"9483":{"timestamp":1712365572.4149582,"reason":"broker was unresponsive"},"9484":{"timestamp":1712365572.4154117,"reason":"broker was unresponsive"},"9485":{"timestamp":1712365572.4158936,"reason":"broker was unresponsive"},"9486":{"timestamp":1712365572.4163501,"reason":"broker was unresponsive"},"9487":{"timestamp":1712365572.4167912,"reason":"broker was unresponsive"},"9488":{"timestamp":1712365572.4174592,"reason":"broker was unresponsive"},"9489":{"timestamp":1712365572.4190547,"reason":"broker was unresponsive"},"9490":{"timestamp":1712365572.4198856,"reason":"broker was unresponsive"},"9491":{"timestamp":1712365572.4205687,"reason":"broker was unresponsive"},"9492":{"timestamp":1712365572.4212465,"reason":"broker was unresponsive"},"9493":{"timestamp":1712365572.4219382,"reason":"broker was unresponsive"},"9494":{"timestamp":1712365572.4226267,"reason":"broker was unresponsive"},"9495":{"timestamp":1712365572.4233079,"reason":"broker was unresponsive"},"9496":{"timestamp":1712365572.4239669,"reason":"broker was unresponsive"},"9497":{"timestamp":1712365572.4246514,"reason":"broker was unresponsive"},"9498":{"timestamp":1712365572.4253368,"reason":"broker was unresponsive"},"9499":{"timestamp":1712365572.4259896,"reason":"broker was unresponsive"},"9500":{"timestamp":1712365572.4266763,"reason":"broker was unresponsive"},"9501":{"timestamp":1712365572.4273386,"reason":"broker was unresponsive"},"9502":{"timestamp":1712365572.4280002,"reason":"broker was unresponsive"},"9503":{"timestamp":1712365572.4288449,"reason":"broker was unresponsive"},"9504":{"timestamp":1712365572.4295268,"reason":"broker was unresponsive"},"9505":{"timestamp":1712365572.4301987,"reason":"broker was unresponsive"},"9506":{"timestamp":1712365572.4316468,"reason":"broker was unresponsive"},"9507":{"timestamp":1712365572.4323425,"reason":"broker was unresponsive"},"9508":{"timestamp":1712365572.432992,"reason":"broker was unresponsive"},"9509":{"timestamp":1712365572.4336872,"reason":"broker was unresponsive"},"9510":{"timestamp":1712365572.4343328,"reason":"broker was unresponsive"},"9511":{"timestamp":1712365572.4347847,"reason":"broker was unresponsive"},"9512":{"timestamp":1712365572.4352288,"reason":"broker was unresponsive"},"9513":{"timestamp":1712365572.4356914,"reason":"broker was unresponsive"},"9514":{"timestamp":1712365572.4361334,"reason":"broker was unresponsive"},"9515":{"timestamp":1712365572.4365995,"reason":"broker was unresponsive"},"9516":{"timestamp":1712365572.4370401,"reason":"broker was unresponsive"},"9517":{"timestamp":1712365572.4375031,"reason":"broker was unresponsive"},"9518":{"timestamp":1712365572.4379911,"reason":"broker was unresponsive"},"9519":{"timestamp":1712365572.4384549,"reason":"broker was unresponsive"},"9520":{"timestamp":1712365572.4388983,"reason":"broker was unresponsive"},"9521":{"timestamp":1712365572.4393609,"reason":"broker was unresponsive"},"9522":{"timestamp":1712365572.4398031,"reason":"broker was unresponsive"},"9523":{"timestamp":1712365572.4402478,"reason":"broker was unresponsive"},"9524":{"timestamp":1712365572.4407229,"reason":"broker was unresponsive"},"9525":{"timestamp":1712365572.4411709,"reason":"broker was unresponsive"},"9526":{"timestamp":1712365572.4416361,"reason":"broker was unresponsive"},"9527":{"timestamp":1712365572.4420826,"reason":"broker was unresponsive"},"9528":{"timestamp":1712365572.4425514,"reason":"broker was unresponsive"},"9529":{"timestamp":1712365572.4429958,"reason":"broker was unresponsive"},"9530":{"timestamp":1712365572.4434624,"reason":"broker was unresponsive"},"9531":{"timestamp":1712365572.4439077,"reason":"broker was unresponsive"},"9532":{"timestamp":1712365572.4443798,"reason":"broker was unresponsive"},"9534":{"timestamp":1712265498.6265624,"reason":"broker was unresponsive"},"9536":{"timestamp":1712365572.4448252,"reason":"broker was unresponsive"},"9537":{"timestamp":1712365572.4452925,"reason":"broker was unresponsive"},"9538":{"timestamp":1712365572.4457381,"reason":"broker was unresponsive"},"9539":{"timestamp":1712365572.4461846,"reason":"broker was unresponsive"},"9540":{"timestamp":1712365572.4466505,"reason":"broker was unresponsive"},"9541":{"timestamp":1712365572.4471033,"reason":"broker was unresponsive"},"9542":{"timestamp":1712365572.4475698,"reason":"broker was unresponsive"},"9543":{"timestamp":1712365572.4480226,"reason":"broker was unresponsive"},"9544":{"timestamp":1712365572.448489,"reason":"broker was unresponsive"},"9545":{"timestamp":1712365572.4489384,"reason":"broker was unresponsive"},"9546":{"timestamp":1712365572.449404,"reason":"broker was unresponsive"},"9547":{"timestamp":1712365572.4499049,"reason":"broker was unresponsive"},"9548":{"timestamp":1712365572.4503789,"reason":"broker was unresponsive"},"9549":{"timestamp":1712365572.4508326,"reason":"broker was unresponsive"},"9550":{"timestamp":1712365572.451304,"reason":"broker was unresponsive"},"9551":{"timestamp":1712365572.4517753,"reason":"broker was unresponsive"},"9552":{"timestamp":1712365572.4522283,"reason":"broker was unresponsive"},"9553":{"timestamp":1712365572.4526987,"reason":"broker was unresponsive"},"9554":{"timestamp":1712365572.4531512,"reason":"broker was unresponsive"},"9555":{"timestamp":1712365572.4536245,"reason":"broker was unresponsive"},"9556":{"timestamp":1712365572.4540823,"reason":"broker was unresponsive"},"9557":{"timestamp":1712365572.4545624,"reason":"broker was unresponsive"},"9558":{"timestamp":1712365572.4550743,"reason":"broker was unresponsive"},"9559":{"timestamp":1712365572.4555583,"reason":"broker was unresponsive"},"9560":{"timestamp":1712365572.4560173,"reason":"broker was unresponsive"},"9561":{"timestamp":1712255642.6251159,"reason":"broker was unresponsive"},"9563":{"timestamp":1712365572.4565005,"reason":"broker was unresponsive"},"9564":{"timestamp":1712365572.456964,"reason":"broker was unresponsive"},"9565":{"timestamp":1712365572.4574506,"reason":"broker was unresponsive"},"9566":{"timestamp":1712365572.4579189,"reason":"broker was unresponsive"},"9567":{"timestamp":1712365572.4584017,"reason":"broker was unresponsive"},"9568":{"timestamp":1712365572.4588625,"reason":"broker was unresponsive"},"9569":{"timestamp":1712365572.4593434,"reason":"broker was unresponsive"},"9570":{"timestamp":1712365572.4598076,"reason":"broker was unresponsive"},"9571":{"timestamp":1712365572.4602907,"reason":"broker was unresponsive"},"9572":{"timestamp":1712365572.4607527,"reason":"broker was unresponsive"},"9573":{"timestamp":1712365572.4612164,"reason":"broker was unresponsive"},"9574":{"timestamp":1712365572.4617207,"reason":"broker was unresponsive"},"9575":{"timestamp":1712365572.4621904,"reason":"broker was unresponsive"},"9576":{"timestamp":1712365572.4626751,"reason":"broker was unresponsive"},"9577":{"timestamp":1712365572.463136,"reason":"broker was unresponsive"},"9578":{"timestamp":1712365572.4636221,"reason":"broker was unresponsive"},"9579":{"timestamp":1712365572.4640868,"reason":"broker was unresponsive"},"9580":{"timestamp":1712365572.4645863,"reason":"broker was unresponsive"},"9581":{"timestamp":1712365572.465054,"reason":"broker was unresponsive"},"9582":{"timestamp":1712365572.4655421,"reason":"broker was unresponsive"},"9583":{"timestamp":1712365572.466007,"reason":"broker was unresponsive"},"9584":{"timestamp":1712365572.4664881,"reason":"broker was unresponsive"},"9585":{"timestamp":1712365572.4669533,"reason":"broker was unresponsive"},"9586":{"timestamp":1712365572.4674363,"reason":"broker was unresponsive"},"9587":{"timestamp":1712365572.4679072,"reason":"broker was unresponsive"},"9588":{"timestamp":1712365572.4683914,"reason":"broker was unresponsive"},"9589":{"timestamp":1712365572.4688601,"reason":"broker was unresponsive"},"9590":{"timestamp":1712365572.4693513,"reason":"broker was unresponsive"},"9591":{"timestamp":1712365572.4698212,"reason":"broker was unresponsive"},"9592":{"timestamp":1712365572.47031,"reason":"broker was unresponsive"},"9593":{"timestamp":1712365572.4707782,"reason":"broker was unresponsive"},"9594":{"timestamp":1712365572.4712467,"reason":"broker was unresponsive"},"9595":{"timestamp":1712365572.4717357,"reason":"broker was unresponsive"},"9596":{"timestamp":1712365572.4722049,"reason":"broker was unresponsive"},"9597":{"timestamp":1712365572.4726954,"reason":"broker was unresponsive"},"9598":{"timestamp":1712365572.4731646,"reason":"broker was unresponsive"},"9599":{"timestamp":1712365572.4736826,"reason":"broker was unresponsive"},"9600":{"timestamp":1712365572.4741886,"reason":"broker was unresponsive"},"9601":{"timestamp":1712365572.4747031,"reason":"broker was unresponsive"},"9602":{"timestamp":1712365572.475199,"reason":"broker was unresponsive"},"9603":{"timestamp":1712365572.4756994,"reason":"broker was unresponsive"},"9604":{"timestamp":1712365572.4761765,"reason":"broker was unresponsive"},"9605":{"timestamp":1712365572.4766712,"reason":"broker was unresponsive"},"9606":{"timestamp":1712365572.4771438,"reason":"broker was unresponsive"},"9607":{"timestamp":1712365572.4776549,"reason":"broker was unresponsive"},"9608":{"timestamp":1712365572.478133,"reason":"broker was unresponsive"},"9609":{"timestamp":1712365572.4786286,"reason":"broker was unresponsive"},"9610":{"timestamp":1712365572.4791083,"reason":"broker was unresponsive"},"9611":{"timestamp":1712365572.4796093,"reason":"broker was unresponsive"},"9612":{"timestamp":1712365572.4800894,"reason":"broker was unresponsive"},"9613":{"timestamp":1712365572.4805832,"reason":"broker was unresponsive"},"9614":{"timestamp":1712365572.4810646,"reason":"broker was unresponsive"},"9615":{"timestamp":1712365572.4815581,"reason":"broker was unresponsive"},"9616":{"timestamp":1712365572.4820397,"reason":"broker was unresponsive"},"9617":{"timestamp":1712365572.4825373,"reason":"broker was unresponsive"},"9618":{"timestamp":1712365572.4830194,"reason":"broker was unresponsive"},"9619":{"timestamp":1712365572.4835131,"reason":"broker was unresponsive"},"9620":{"timestamp":1712365572.4839928,"reason":"broker was unresponsive"},"9621":{"timestamp":1712365572.4844897,"reason":"broker was unresponsive"},"9622":{"timestamp":1712365572.484972,"reason":"broker was unresponsive"},"9623":{"timestamp":1712365572.4854732,"reason":"broker was unresponsive"},"9624":{"timestamp":1712365572.4859588,"reason":"broker was unresponsive"},"9625":{"timestamp":1712365572.4864573,"reason":"broker was unresponsive"},"9626":{"timestamp":1712365572.4869449,"reason":"broker was unresponsive"},"9627":{"timestamp":1712365572.487442,"reason":"broker was unresponsive"},"9628":{"timestamp":1712365572.4879246,"reason":"broker was unresponsive"},"9629":{"timestamp":1712365572.4884274,"reason":"broker was unresponsive"},"9630":{"timestamp":1712365572.4889116,"reason":"broker was unresponsive"},"9631":{"timestamp":1712365572.4894142,"reason":"broker was unresponsive"},"9632":{"timestamp":1712365572.4898999,"reason":"broker was unresponsive"},"9633":{"timestamp":1712274432.5613508,"reason":"bad partner node --JRG"},"9634":{"timestamp":1712274414.439393,"reason":"Boot loops --JRG"},"9635":{"timestamp":1712365572.4903982,"reason":"broker was unresponsive"},"9636":{"timestamp":1712365572.490886,"reason":"broker was unresponsive"},"9637":{"timestamp":1712365572.491389,"reason":"broker was unresponsive"},"9638":{"timestamp":1712365572.4918764,"reason":"broker was unresponsive"},"9639":{"timestamp":1712365572.4923739,"reason":"broker was unresponsive"},"9640":{"timestamp":1712365572.4928646,"reason":"broker was unresponsive"},"9641":{"timestamp":1712365572.4933784,"reason":"broker was unresponsive"},"9642":{"timestamp":1712365572.4938681,"reason":"broker was unresponsive"},"9643":{"timestamp":1712365572.494369,"reason":"broker was unresponsive"},"9644":{"timestamp":1712365572.4948564,"reason":"broker was unresponsive"},"9645":{"timestamp":1712365572.4953611,"reason":"broker was unresponsive"},"9646":{"timestamp":1712365572.4958506,"reason":"broker was unresponsive"},"9647":{"timestamp":1712365572.4963596,"reason":"broker was unresponsive"},"9648":{"timestamp":1712365572.4968495,"reason":"broker was unresponsive"},"9649":{"timestamp":1712365572.4973547,"reason":"broker was unresponsive"},"9650":{"timestamp":1712365572.4978421,"reason":"broker was unresponsive"},"9651":{"timestamp":1712365572.4983506,"reason":"broker was unresponsive"},"9652":{"timestamp":1712365572.4988427,"reason":"broker was unresponsive"},"9653":{"timestamp":1712365572.499347,"reason":"broker was unresponsive"},"9654":{"timestamp":1712365572.4998434,"reason":"broker was unresponsive"},"9655":{"timestamp":1712365572.5003459,"reason":"broker was unresponsive"},"9656":{"timestamp":1712365572.5008368,"reason":"broker was unresponsive"},"9657":{"timestamp":1712365572.5013697,"reason":"broker was unresponsive"},"9658":{"timestamp":1712365572.5018671,"reason":"broker was unresponsive"},"9659":{"timestamp":1712365572.502399,"reason":"broker was unresponsive"},"9660":{"timestamp":1712365572.5028989,"reason":"broker was unresponsive"},"9661":{"timestamp":1712365572.5034325,"reason":"broker was unresponsive"},"9662":{"timestamp":1712365572.5039339,"reason":"broker was unresponsive"},"9663":{"timestamp":1712365572.504468,"reason":"broker was unresponsive"},"9664":{"timestamp":1712365572.5049689,"reason":"broker was unresponsive"},"9665":{"timestamp":1712365572.5054824,"reason":"broker was unresponsive"},"9666":{"timestamp":1712365572.5059915,"reason":"broker was unresponsive"},"9667":{"timestamp":1712365572.5065117,"reason":"broker was unresponsive"},"9668":{"timestamp":1712365572.5070307,"reason":"broker was unresponsive"},"9669":{"timestamp":1712365572.5075495,"reason":"broker was unresponsive"},"9670":{"timestamp":1712365572.5080671,"reason":"broker was unresponsive"},"9671":{"timestamp":1712365572.508584,"reason":"broker was unresponsive"},"9672":{"timestamp":1712365572.5091031,"reason":"broker was unresponsive"},"9673":{"timestamp":1712365572.5096228,"reason":"broker was unresponsive"},"9674":{"timestamp":1712365572.5101483,"reason":"broker was unresponsive"},"9675":{"timestamp":1712365572.5106716,"reason":"broker was unresponsive"},"9676":{"timestamp":1712365572.5111911,"reason":"broker was unresponsive"},"9677":{"timestamp":1712365572.5117304,"reason":"broker was unresponsive"},"9678":{"timestamp":1712365572.5122499,"reason":"broker was unresponsive"},"9679":{"timestamp":1712365572.5127831,"reason":"broker was unresponsive"},"9680":{"timestamp":1712365572.5133045,"reason":"broker was unresponsive"},"9681":{"timestamp":1712365572.5138087,"reason":"broker was unresponsive"},"9682":{"timestamp":1712365572.5143297,"reason":"broker was unresponsive"},"9683":{"timestamp":1712365572.5148354,"reason":"broker was unresponsive"},"9684":{"timestamp":1712365572.5153601,"reason":"broker was unresponsive"},"9685":{"timestamp":1712365572.5158865,"reason":"broker was unresponsive"},"9686":{"timestamp":1712365572.5164032,"reason":"broker was unresponsive"},"9687":{"timestamp":1712365572.5169063,"reason":"broker was unresponsive"},"9688":{"timestamp":1712365572.5174186,"reason":"broker was unresponsive"},"9689":{"timestamp":1712365572.5179236,"reason":"broker was unresponsive"},"9690":{"timestamp":1712365572.5184402,"reason":"broker was unresponsive"},"9691":{"timestamp":1712365572.5189495,"reason":"broker was unresponsive"},"9692":{"timestamp":1712365572.5194695,"reason":"broker was unresponsive"},"9693":{"timestamp":1712365572.5199783,"reason":"broker was unresponsive"},"9694":{"timestamp":1712365572.5205081,"reason":"broker was unresponsive"},"9695":{"timestamp":1712365572.5210173,"reason":"broker was unresponsive"},"9696":{"timestamp":1712365572.5215447,"reason":"broker was unresponsive"},"9697":{"timestamp":1712365572.5220542,"reason":"broker was unresponsive"},"9698":{"timestamp":1712365572.522583,"reason":"broker was unresponsive"},"9699":{"timestamp":1712365572.5230944,"reason":"broker was unresponsive"},"9700":{"timestamp":1712365572.5236182,"reason":"broker was unresponsive"},"9701":{"timestamp":1712365572.524127,"reason":"broker was unresponsive"},"9702":{"timestamp":1712365572.5246537,"reason":"broker was unresponsive"},"9703":{"timestamp":1712365572.5251653,"reason":"broker was unresponsive"},"9704":{"timestamp":1712365572.5256927,"reason":"broker was unresponsive"},"9705":{"timestamp":1712365572.5262053,"reason":"broker was unresponsive"},"9706":{"timestamp":1712365572.5267353,"reason":"broker was unresponsive"},"9707":{"timestamp":1712365572.5272496,"reason":"broker was unresponsive"},"9708":{"timestamp":1712365572.5277846,"reason":"broker was unresponsive"},"9709":{"timestamp":1712365572.5283148,"reason":"broker was unresponsive"},"9710":{"timestamp":1712365572.528827,"reason":"broker was unresponsive"},"9711":{"timestamp":1712365572.5293605,"reason":"broker was unresponsive"},"9712":{"timestamp":1712365572.5298758,"reason":"broker was unresponsive"},"9713":{"timestamp":1712365572.5304103,"reason":"broker was unresponsive"},"9714":{"timestamp":1712365572.5309248,"reason":"broker was unresponsive"},"9715":{"timestamp":1712365572.5314503,"reason":"broker was unresponsive"},"9716":{"timestamp":1712365572.5319638,"reason":"broker was unresponsive"},"9717":{"timestamp":1712365572.532495,"reason":"broker was unresponsive"},"9718":{"timestamp":1712365572.5330083,"reason":"broker was unresponsive"},"9719":{"timestamp":1712365572.5335414,"reason":"broker was unresponsive"},"9720":{"timestamp":1712365572.5340581,"reason":"broker was unresponsive"},"9721":{"timestamp":1712365572.5345957,"reason":"broker was unresponsive"},"9722":{"timestamp":1712365572.5351126,"reason":"broker was unresponsive"},"9723":{"timestamp":1712365572.5356646,"reason":"broker was unresponsive"},"9724":{"timestamp":1712365572.5361876,"reason":"broker was unresponsive"},"9725":{"timestamp":1712352442.0729523,"reason":"broker was unresponsive"},"9726":{"timestamp":1712352438.0526481,"reason":"broker was unresponsive"},"9727":{"timestamp":1712365572.5367253,"reason":"broker was unresponsive"},"9728":{"timestamp":1712365572.5372455,"reason":"broker was unresponsive"},"9729":{"timestamp":1712365572.5377774,"reason":"broker was unresponsive"},"9730":{"timestamp":1712365572.5383177,"reason":"broker was unresponsive"},"9731":{"timestamp":1712365572.5388408,"reason":"broker was unresponsive"},"9732":{"timestamp":1712365572.5393727,"reason":"broker was unresponsive"},"9733":{"timestamp":1712365572.5398927,"reason":"broker was unresponsive"},"9734":{"timestamp":1712365572.5404253,"reason":"broker was unresponsive"},"9735":{"timestamp":1712365572.5409496,"reason":"broker was unresponsive"},"9736":{"timestamp":1712365572.5414903,"reason":"broker was unresponsive"},"9737":{"timestamp":1712365572.542027,"reason":"broker was unresponsive"},"9738":{"timestamp":1712365572.5425735,"reason":"broker was unresponsive"},"9739":{"timestamp":1712365572.5430999,"reason":"broker was unresponsive"},"9740":{"timestamp":1712365572.5436428,"reason":"broker was unresponsive"},"9741":{"timestamp":1712365572.5441675,"reason":"broker was unresponsive"},"9742":{"timestamp":1712365572.5447049,"reason":"broker was unresponsive"},"9743":{"timestamp":1712365572.5452292,"reason":"broker was unresponsive"},"9744":{"timestamp":1712365572.5457706,"reason":"broker was unresponsive"},"9745":{"timestamp":1712365572.5463171,"reason":"broker was unresponsive"},"9746":{"timestamp":1712365572.546845,"reason":"broker was unresponsive"},"9747":{"timestamp":1712365572.5473871,"reason":"broker was unresponsive"},"9748":{"timestamp":1712365572.5479121,"reason":"broker was unresponsive"},"9749":{"timestamp":1712365572.5484495,"reason":"broker was unresponsive"},"9750":{"timestamp":1712336418.2480905,"reason":"nodediag failed amdapu"},"9751":{"timestamp":1712365572.5495033,"reason":"broker was unresponsive"},"9752":{"timestamp":1712365572.5500293,"reason":"broker was unresponsive"},"9753":{"timestamp":1712365572.5505738,"reason":"broker was unresponsive"},"9754":{"timestamp":1712365572.5511017,"reason":"broker was unresponsive"},"9755":{"timestamp":1712365572.5516443,"reason":"broker was unresponsive"},"9756":{"timestamp":1712365572.5521755,"reason":"broker was unresponsive"},"9757":{"timestamp":1712365572.5527234,"reason":"broker was unresponsive"},"9758":{"timestamp":1712365572.5532558,"reason":"broker was unresponsive"},"9759":{"timestamp":1712365572.5537963,"reason":"broker was unresponsive"},"9760":{"timestamp":1712365572.5543396,"reason":"broker was unresponsive"},"9761":{"timestamp":1712365572.5548689,"reason":"broker was unresponsive"},"9762":{"timestamp":1712365572.5554218,"reason":"broker was unresponsive"},"9763":{"timestamp":1712365572.5559607,"reason":"broker was unresponsive"},"9764":{"timestamp":1712365572.5565169,"reason":"broker was unresponsive"},"9765":{"timestamp":1712365572.5570476,"reason":"broker was unresponsive"},"9766":{"timestamp":1712365572.5575924,"reason":"broker was unresponsive"},"9767":{"timestamp":1712365572.558125,"reason":"broker was unresponsive"},"9768":{"timestamp":1712365572.5586729,"reason":"broker was unresponsive"},"9769":{"timestamp":1712365572.5592058,"reason":"broker was unresponsive"},"9770":{"timestamp":1712365572.5597532,"reason":"broker was unresponsive"},"9771":{"timestamp":1712365572.5602984,"reason":"broker was unresponsive"},"9772":{"timestamp":1712365572.5608363,"reason":"broker was unresponsive"},"9773":{"timestamp":1712365572.5613887,"reason":"broker was unresponsive"},"9774":{"timestamp":1712365572.5619247,"reason":"broker was unresponsive"},"9775":{"timestamp":1712365572.5624716,"reason":"broker was unresponsive"},"9776":{"timestamp":1712365572.5630093,"reason":"broker was unresponsive"},"9777":{"timestamp":1712365572.5635655,"reason":"broker was unresponsive"},"9778":{"timestamp":1712365572.5641038,"reason":"broker was unresponsive"},"9779":{"timestamp":1712365572.5646513,"reason":"broker was unresponsive"},"9780":{"timestamp":1712365572.5651889,"reason":"broker was unresponsive"},"9781":{"timestamp":1712365572.5657415,"reason":"broker was unresponsive"},"9782":{"timestamp":1712365572.566293,"reason":"broker was unresponsive"},"9783":{"timestamp":1712365572.5668309,"reason":"broker was unresponsive"},"9784":{"timestamp":1712365572.5673821,"reason":"broker was unresponsive"},"9785":{"timestamp":1712365572.56792,"reason":"broker was unresponsive"},"9786":{"timestamp":1712365572.5684707,"reason":"broker was unresponsive"},"9787":{"timestamp":1712365572.5690169,"reason":"broker was unresponsive"},"9788":{"timestamp":1712365572.5695748,"reason":"broker was unresponsive"},"9789":{"timestamp":1712365572.5701172,"reason":"broker was unresponsive"},"9790":{"timestamp":1712365572.5706718,"reason":"broker was unresponsive"},"9791":{"timestamp":1712365572.5712104,"reason":"broker was unresponsive"},"9792":{"timestamp":1712365572.571764,"reason":"broker was unresponsive"},"9793":{"timestamp":1712365572.5723226,"reason":"broker was unresponsive"},"9794":{"timestamp":1712365572.5728652,"reason":"broker was unresponsive"},"9795":{"timestamp":1712365572.5734196,"reason":"broker was unresponsive"},"9796":{"timestamp":1712365572.5739708,"reason":"broker was unresponsive"},"9797":{"timestamp":1712365572.5745399,"reason":"broker was unresponsive"},"9798":{"timestamp":1712365572.5750873,"reason":"broker was unresponsive"},"9799":{"timestamp":1712365572.5756762,"reason":"broker was unresponsive"},"9800":{"timestamp":1712365572.576226,"reason":"broker was unresponsive"},"9801":{"timestamp":1712365572.5767908,"reason":"broker was unresponsive"},"9802":{"timestamp":1712365572.5773485,"reason":"broker was unresponsive"},"9803":{"timestamp":1712365572.5778935,"reason":"broker was unresponsive"},"9804":{"timestamp":1712365572.5784519,"reason":"broker was unresponsive"},"9805":{"timestamp":1712365572.5789957,"reason":"broker was unresponsive"},"9806":{"timestamp":1712365572.5795565,"reason":"broker was unresponsive"},"9807":{"timestamp":1712365572.5801044,"reason":"broker was unresponsive"},"9808":{"timestamp":1712365572.5806615,"reason":"broker was unresponsive"},"9809":{"timestamp":1712365572.5812097,"reason":"broker was unresponsive"},"9810":{"timestamp":1712365572.5817728,"reason":"broker was unresponsive"},"9811":{"timestamp":1712365572.5823357,"reason":"broker was unresponsive"},"9812":{"timestamp":1712365572.5828826,"reason":"broker was unresponsive"},"9813":{"timestamp":1712365572.5834484,"reason":"broker was unresponsive"},"9814":{"timestamp":1712365572.583998,"reason":"broker was unresponsive"},"9815":{"timestamp":1712365572.5845556,"reason":"broker was unresponsive"},"9816":{"timestamp":1712365572.5851045,"reason":"broker was unresponsive"},"9817":{"timestamp":1712365572.58567,"reason":"broker was unresponsive"},"9818":{"timestamp":1712365572.586221,"reason":"broker was unresponsive"},"9819":{"timestamp":1712365572.5867889,"reason":"broker was unresponsive"},"9820":{"timestamp":1712365572.5873539,"reason":"broker was unresponsive"},"9821":{"timestamp":1712365572.5879078,"reason":"broker was unresponsive"},"9822":{"timestamp":1712365572.5884695,"reason":"broker was unresponsive"},"9823":{"timestamp":1712365572.5890257,"reason":"broker was unresponsive"},"9824":{"timestamp":1712365572.5895948,"reason":"broker was unresponsive"},"9825":{"timestamp":1712365572.5901487,"reason":"broker was unresponsive"},"9826":{"timestamp":1712365572.5907223,"reason":"broker was unresponsive"},"9827":{"timestamp":1712365572.5912886,"reason":"broker was unresponsive"},"9828":{"timestamp":1712365572.5918422,"reason":"broker was unresponsive"},"9829":{"timestamp":1712365572.5924091,"reason":"broker was unresponsive"},"9830":{"timestamp":1712365572.5929649,"reason":"broker was unresponsive"},"9831":{"timestamp":1712365572.5935335,"reason":"broker was unresponsive"},"9832":{"timestamp":1712365572.5940893,"reason":"broker was unresponsive"},"9833":{"timestamp":1712365572.5946641,"reason":"broker was unresponsive"},"9834":{"timestamp":1712365572.5952268,"reason":"broker was unresponsive"},"9835":{"timestamp":1712365572.595803,"reason":"broker was unresponsive"},"9836":{"timestamp":1712365572.5963714,"reason":"broker was unresponsive"},"9837":{"timestamp":1712365572.5969286,"reason":"broker was unresponsive"},"9838":{"timestamp":1712365572.5974972,"reason":"broker was unresponsive"},"9839":{"timestamp":1712365572.5980561,"reason":"broker was unresponsive"},"9840":{"timestamp":1712365572.5986316,"reason":"broker was unresponsive"},"9841":{"timestamp":1712365572.5991912,"reason":"broker was unresponsive"},"9842":{"timestamp":1712365572.5997603,"reason":"broker was unresponsive"},"9843":{"timestamp":1712365572.6003358,"reason":"broker was unresponsive"},"9844":{"timestamp":1712365572.6008949,"reason":"broker was unresponsive"},"9973":{"timestamp":1712365572.6014712,"reason":"broker was unresponsive"},"9974":{"timestamp":1712365572.6020353,"reason":"broker was unresponsive"},"9975":{"timestamp":1712365572.602607,"reason":"broker was unresponsive"},"9976":{"timestamp":1712365572.603168,"reason":"broker was unresponsive"},"9977":{"timestamp":1712365572.6037414,"reason":"broker was unresponsive"},"9978":{"timestamp":1712365572.6043212,"reason":"broker was unresponsive"},"9979":{"timestamp":1712365572.6048877,"reason":"broker was unresponsive"},"9980":{"timestamp":1712365572.6054645,"reason":"broker was unresponsive"},"9981":{"timestamp":1712365572.6060302,"reason":"broker was unresponsive"},"9982":{"timestamp":1712365572.6066124,"reason":"broker was unresponsive"},"9983":{"timestamp":1712365572.6071782,"reason":"broker was unresponsive"},"9984":{"timestamp":1712365572.6077561,"reason":"broker was unresponsive"},"9985":{"timestamp":1712365572.6083393,"reason":"broker was unresponsive"},"9986":{"timestamp":1712365572.6089053,"reason":"broker was unresponsive"},"9987":{"timestamp":1712365572.609484,"reason":"broker was unresponsive"},"9988":{"timestamp":1712365572.6100547,"reason":"broker was unresponsive"},"9989":{"timestamp":1712365572.6106315,"reason":"broker was unresponsive"},"9990":{"timestamp":1712365572.6112003,"reason":"broker was unresponsive"},"9991":{"timestamp":1712365572.6117814,"reason":"broker was unresponsive"},"9992":{"timestamp":1712365572.6123636,"reason":"broker was unresponsive"},"9993":{"timestamp":1712365572.6129355,"reason":"broker was unresponsive"},"9994":{"timestamp":1712365572.6135185,"reason":"broker was unresponsive"},"9995":{"timestamp":1712365572.6140931,"reason":"broker was unresponsive"},"9996":{"timestamp":1712365572.6146774,"reason":"broker was unresponsive"},"9997":{"timestamp":1712365572.6152487,"reason":"broker was unresponsive"},"9998":{"timestamp":1712365572.6158333,"reason":"broker was unresponsive"},"9999":{"timestamp":1712365572.6164162,"reason":"broker was unresponsive"},"10000":{"timestamp":1712365572.6169884,"reason":"broker was unresponsive"},"10001":{"timestamp":1712365572.617569,"reason":"broker was unresponsive"},"10002":{"timestamp":1712365572.6181395,"reason":"broker was unresponsive"},"10003":{"timestamp":1712365572.618722,"reason":"broker was unresponsive"},"10004":{"timestamp":1712365572.6193061,"reason":"broker was unresponsive"},"10005":{"timestamp":1712365572.61988,"reason":"broker was unresponsive"},"10006":{"timestamp":1712365572.6204653,"reason":"broker was unresponsive"},"10007":{"timestamp":1712365572.6210384,"reason":"broker was unresponsive"},"10008":{"timestamp":1712365572.621628,"reason":"broker was unresponsive"},"10009":{"timestamp":1712365572.6222048,"reason":"broker was unresponsive"},"10010":{"timestamp":1712365572.6227961,"reason":"broker was unresponsive"},"10011":{"timestamp":1712365572.6233871,"reason":"broker was unresponsive"},"10012":{"timestamp":1712365572.6239636,"reason":"broker was unresponsive"},"10013":{"timestamp":1712365572.6245511,"reason":"broker was unresponsive"},"10014":{"timestamp":1712365572.6251276,"reason":"broker was unresponsive"},"10015":{"timestamp":1712365572.6257203,"reason":"broker was unresponsive"},"10016":{"timestamp":1712365572.626308,"reason":"broker was unresponsive"},"10017":{"timestamp":1712365572.6268897,"reason":"broker was unresponsive"},"10018":{"timestamp":1712365572.6275012,"reason":"broker was unresponsive"},"10019":{"timestamp":1712365572.6280825,"reason":"broker was unresponsive"},"10020":{"timestamp":1712365572.6286852,"reason":"broker was unresponsive"},"10021":{"timestamp":1712365572.6292794,"reason":"broker was unresponsive"},"10022":{"timestamp":1712365572.6298611,"reason":"broker was unresponsive"},"10023":{"timestamp":1712365572.6304564,"reason":"broker was unresponsive"},"10024":{"timestamp":1712365572.631036,"reason":"broker was unresponsive"},"10025":{"timestamp":1712365572.6316307,"reason":"broker was unresponsive"},"10026":{"timestamp":1712365572.6322126,"reason":"broker was unresponsive"},"10027":{"timestamp":1712365572.6328106,"reason":"broker was unresponsive"},"10028":{"timestamp":1712365572.6334112,"reason":"broker was unresponsive"},"10029":{"timestamp":1712365572.6339934,"reason":"broker was unresponsive"},"10030":{"timestamp":1712365572.6345897,"reason":"broker was unresponsive"},"10031":{"timestamp":1712365572.6351728,"reason":"broker was unresponsive"},"10032":{"timestamp":1712365572.6357706,"reason":"broker was unresponsive"},"10033":{"timestamp":1712365572.6363678,"reason":"broker was unresponsive"},"10034":{"timestamp":1712365572.6369488,"reason":"broker was unresponsive"},"10035":{"timestamp":1712365572.6375492,"reason":"broker was unresponsive"},"10036":{"timestamp":1712365572.6381335,"reason":"broker was unresponsive"},"10037":{"timestamp":1712365572.6387267,"reason":"broker was unresponsive"},"10038":{"timestamp":1712365572.6393211,"reason":"broker was unresponsive"},"10039":{"timestamp":1712365572.6399095,"reason":"broker was unresponsive"},"10040":{"timestamp":1712365572.6405127,"reason":"broker was unresponsive"},"10041":{"timestamp":1712365572.6410983,"reason":"broker was unresponsive"},"10042":{"timestamp":1712365572.6417019,"reason":"broker was unresponsive"},"10043":{"timestamp":1712365572.6423068,"reason":"broker was unresponsive"},"10044":{"timestamp":1712365572.6428921,"reason":"broker was unresponsive"},"10045":{"timestamp":1712365572.6434925,"reason":"broker was unresponsive"},"10046":{"timestamp":1712365572.6440773,"reason":"broker was unresponsive"},"10047":{"timestamp":1712365572.6446834,"reason":"broker was unresponsive"},"10048":{"timestamp":1712365572.6452863,"reason":"broker was unresponsive"},"10049":{"timestamp":1712365572.6458752,"reason":"broker was unresponsive"},"10050":{"timestamp":1712365572.6464801,"reason":"broker was unresponsive"},"10051":{"timestamp":1712365572.6470697,"reason":"broker was unresponsive"},"10052":{"timestamp":1712365572.647671,"reason":"broker was unresponsive"},"10053":{"timestamp":1712365572.6482763,"reason":"broker was unresponsive"},"10054":{"timestamp":1712365572.6488667,"reason":"broker was unresponsive"},"10055":{"timestamp":1712365572.6494672,"reason":"broker was unresponsive"},"10056":{"timestamp":1712365572.6500626,"reason":"broker was unresponsive"},"10057":{"timestamp":1712365572.6506603,"reason":"broker was unresponsive"},"10058":{"timestamp":1712365572.6512506,"reason":"broker was unresponsive"},"10059":{"timestamp":1712365572.6518536,"reason":"broker was unresponsive"},"10060":{"timestamp":1712365572.6524644,"reason":"broker was unresponsive"},"10061":{"timestamp":1712365572.6530602,"reason":"broker was unresponsive"},"10062":{"timestamp":1712365572.6536629,"reason":"broker was unresponsive"},"10063":{"timestamp":1712365572.6542566,"reason":"broker was unresponsive"},"10064":{"timestamp":1712365572.6548688,"reason":"broker was unresponsive"},"10065":{"timestamp":1712365572.6554794,"reason":"broker was unresponsive"},"10066":{"timestamp":1712365572.656075,"reason":"broker was unresponsive"},"10067":{"timestamp":1712365572.656683,"reason":"broker was unresponsive"},"10068":{"timestamp":1712365572.6572964,"reason":"broker was unresponsive"},"10069":{"timestamp":1712365572.6578965,"reason":"broker was unresponsive"},"10070":{"timestamp":1712365572.658505,"reason":"broker was unresponsive"},"10071":{"timestamp":1712365572.6591027,"reason":"broker was unresponsive"},"10072":{"timestamp":1712365572.6597164,"reason":"broker was unresponsive"},"10073":{"timestamp":1712365572.6603303,"reason":"broker was unresponsive"},"10074":{"timestamp":1712365572.6609309,"reason":"broker was unresponsive"},"10075":{"timestamp":1712365572.661546,"reason":"broker was unresponsive"},"10076":{"timestamp":1712365572.6621461,"reason":"broker was unresponsive"},"10077":{"timestamp":1712365572.6627607,"reason":"broker was unresponsive"},"10078":{"timestamp":1712365572.6633747,"reason":"broker was unresponsive"},"10079":{"timestamp":1712365572.6639731,"reason":"broker was unresponsive"},"10080":{"timestamp":1712365572.6645932,"reason":"broker was unresponsive"},"10081":{"timestamp":1712365572.6651962,"reason":"broker was unresponsive"},"10082":{"timestamp":1712365572.6658111,"reason":"broker was unresponsive"},"10083":{"timestamp":1712365572.6664269,"reason":"broker was unresponsive"},"10084":{"timestamp":1712365572.6670268,"reason":"broker was unresponsive"},"10085":{"timestamp":1712365572.6676369,"reason":"broker was unresponsive"},"10086":{"timestamp":1712365572.6682403,"reason":"broker was unresponsive"},"10087":{"timestamp":1712365572.6688607,"reason":"broker was unresponsive"},"10088":{"timestamp":1712365572.6694894,"reason":"broker was unresponsive"},"10089":{"timestamp":1712365572.6700952,"reason":"broker was unresponsive"},"10090":{"timestamp":1712365572.6707127,"reason":"broker was unresponsive"},"10091":{"timestamp":1712365572.6713254,"reason":"broker was unresponsive"},"10092":{"timestamp":1712365572.6719296,"reason":"broker was unresponsive"},"10093":{"timestamp":1712365572.6725523,"reason":"broker was unresponsive"},"10094":{"timestamp":1712365572.6731603,"reason":"broker was unresponsive"},"10095":{"timestamp":1712365572.673784,"reason":"broker was unresponsive"},"10096":{"timestamp":1712365572.6744008,"reason":"broker was unresponsive"},"10097":{"timestamp":1712365572.675009,"reason":"broker was unresponsive"},"10098":{"timestamp":1712365572.6756308,"reason":"broker was unresponsive"},"10099":{"timestamp":1712365572.67624,"reason":"broker was unresponsive"},"10100":{"timestamp":1712365572.6768615,"reason":"broker was unresponsive"},"10101":{"timestamp":1712365572.6774869,"reason":"broker was unresponsive"},"10102":{"timestamp":1712365572.678097,"reason":"broker was unresponsive"},"10103":{"timestamp":1712365572.6787314,"reason":"broker was unresponsive"},"10104":{"timestamp":1712365572.6793542,"reason":"broker was unresponsive"},"10105":{"timestamp":1712365572.679966,"reason":"broker was unresponsive"},"10106":{"timestamp":1712365572.6805925,"reason":"broker was unresponsive"},"10107":{"timestamp":1712365572.6812034,"reason":"broker was unresponsive"},"10108":{"timestamp":1712365572.6818275,"reason":"broker was unresponsive"},"10109":{"timestamp":1712365572.682451,"reason":"broker was unresponsive"},"10110":{"timestamp":1712365572.6830635,"reason":"broker was unresponsive"},"10111":{"timestamp":1712365572.6836898,"reason":"broker was unresponsive"},"10112":{"timestamp":1712365572.684314,"reason":"broker was unresponsive"},"10113":{"timestamp":1712365572.6849258,"reason":"broker was unresponsive"},"10114":{"timestamp":1712365572.6855493,"reason":"broker was unresponsive"},"10115":{"timestamp":1712365572.6861622,"reason":"broker was unresponsive"},"10116":{"timestamp":1712365572.6868007,"reason":"broker was unresponsive"},"10117":{"timestamp":1712365572.6874268,"reason":"broker was unresponsive"},"10118":{"timestamp":1712365572.6880391,"reason":"broker was unresponsive"},"10119":{"timestamp":1712365572.6886652,"reason":"broker was unresponsive"},"10120":{"timestamp":1712365572.6892951,"reason":"broker was unresponsive"},"10121":{"timestamp":1712365572.6899114,"reason":"broker was unresponsive"},"10122":{"timestamp":1712365572.6905406,"reason":"broker was unresponsive"},"10123":{"timestamp":1712365572.6911559,"reason":"broker was unresponsive"},"10124":{"timestamp":1712365572.6917815,"reason":"broker was unresponsive"},"10125":{"timestamp":1712365572.6924107,"reason":"broker was unresponsive"},"10126":{"timestamp":1712365572.6930273,"reason":"broker was unresponsive"},"10127":{"timestamp":1712365572.6936626,"reason":"broker was unresponsive"},"10128":{"timestamp":1712365572.694303,"reason":"broker was unresponsive"},"10129":{"timestamp":1712365572.6949196,"reason":"broker was unresponsive"},"10130":{"timestamp":1712365572.6955514,"reason":"broker was unresponsive"},"10131":{"timestamp":1712365572.6961687,"reason":"broker was unresponsive"},"10132":{"timestamp":1712365572.6967976,"reason":"broker was unresponsive"},"10133":{"timestamp":1712365572.6974335,"reason":"broker was unresponsive"},"10134":{"timestamp":1712365572.6980529,"reason":"broker was unresponsive"},"10135":{"timestamp":1712365572.6986835,"reason":"broker was unresponsive"},"10136":{"timestamp":1712365572.699317,"reason":"broker was unresponsive"},"10137":{"timestamp":1712365572.6999376,"reason":"broker was unresponsive"},"10139":{"timestamp":1712365572.7005789,"reason":"broker was unresponsive"},"10140":{"timestamp":1712365572.7012033,"reason":"broker was unresponsive"},"10141":{"timestamp":1712365572.7018433,"reason":"broker was unresponsive"},"10142":{"timestamp":1712365572.702491,"reason":"broker was unresponsive"},"10144":{"timestamp":1712365572.7031174,"reason":"broker was unresponsive"},"10146":{"timestamp":1712365572.703752,"reason":"broker was unresponsive"},"10147":{"timestamp":1712365572.7043879,"reason":"broker was unresponsive"},"10148":{"timestamp":1712365572.7050161,"reason":"broker was unresponsive"},"10149":{"timestamp":1712365572.7056525,"reason":"broker was unresponsive"},"10150":{"timestamp":1712365572.7062938,"reason":"broker was unresponsive"},"10151":{"timestamp":1712365572.7069175,"reason":"broker was unresponsive"},"10152":{"timestamp":1712365572.7075539,"reason":"broker was unresponsive"},"10153":{"timestamp":1712365572.7081811,"reason":"broker was unresponsive"},"10154":{"timestamp":1712365572.7088189,"reason":"broker was unresponsive"},"10157":{"timestamp":1712365572.7094662,"reason":"broker was unresponsive"},"10158":{"timestamp":1712365572.7100945,"reason":"broker was unresponsive"},"10159":{"timestamp":1712365572.7107491,"reason":"broker was unresponsive"},"10160":{"timestamp":1712365572.7113934,"reason":"broker was unresponsive"},"10161":{"timestamp":1712365572.7120221,"reason":"broker was unresponsive"},"10162":{"timestamp":1712365572.7126665,"reason":"broker was unresponsive"},"10163":{"timestamp":1712365572.713311,"reason":"broker was unresponsive"},"10164":{"timestamp":1712365572.7139418,"reason":"broker was unresponsive"},"10165":{"timestamp":1712365572.7145855,"reason":"broker was unresponsive"},"10166":{"timestamp":1712365572.7152166,"reason":"broker was unresponsive"},"10167":{"timestamp":1712365572.7158625,"reason":"broker was unresponsive"},"10168":{"timestamp":1712365572.7165082,"reason":"broker was unresponsive"},"10169":{"timestamp":1712365572.7171452,"reason":"broker was unresponsive"},"10170":{"timestamp":1712365572.7177925,"reason":"broker was unresponsive"},"10171":{"timestamp":1712365572.7184503,"reason":"broker was unresponsive"},"10172":{"timestamp":1712365572.7190886,"reason":"broker was unresponsive"},"10173":{"timestamp":1712365572.7197344,"reason":"broker was unresponsive"},"10174":{"timestamp":1712365572.7203803,"reason":"broker was unresponsive"},"10175":{"timestamp":1712365572.721014,"reason":"broker was unresponsive"},"10176":{"timestamp":1712365572.7216597,"reason":"broker was unresponsive"},"10177":{"timestamp":1712365572.7223079,"reason":"broker was unresponsive"},"10178":{"timestamp":1712365572.7229462,"reason":"broker was unresponsive"},"10179":{"timestamp":1712365572.7235959,"reason":"broker was unresponsive"},"10180":{"timestamp":1712365572.7242301,"reason":"broker was unresponsive"},"10181":{"timestamp":1712365572.7248824,"reason":"broker was unresponsive"},"10182":{"timestamp":1712365572.7255385,"reason":"broker was unresponsive"},"10183":{"timestamp":1712365572.7262022,"reason":"broker was unresponsive"},"10184":{"timestamp":1712365572.726871,"reason":"broker was unresponsive"},"10185":{"timestamp":1712365572.7275286,"reason":"broker was unresponsive"},"10186":{"timestamp":1712365572.7281671,"reason":"broker was unresponsive"},"10187":{"timestamp":1712365572.7288213,"reason":"broker was unresponsive"},"10188":{"timestamp":1712365572.72948,"reason":"broker was unresponsive"},"10189":{"timestamp":1712365572.7301209,"reason":"broker was unresponsive"},"10190":{"timestamp":1712365572.7307687,"reason":"broker was unresponsive"},"10191":{"timestamp":1712365572.7314229,"reason":"broker was unresponsive"},"10192":{"timestamp":1712365572.732064,"reason":"broker was unresponsive"},"10193":{"timestamp":1712365572.7327151,"reason":"broker was unresponsive"},"10194":{"timestamp":1712365572.7333694,"reason":"broker was unresponsive"},"10195":{"timestamp":1712365572.7340117,"reason":"broker was unresponsive"},"10196":{"timestamp":1712365572.7346685,"reason":"broker was unresponsive"},"10197":{"timestamp":1712365572.7353392,"reason":"broker was unresponsive"},"10198":{"timestamp":1712365572.7359846,"reason":"broker was unresponsive"},"10199":{"timestamp":1712365572.7366428,"reason":"broker was unresponsive"},"10200":{"timestamp":1712365572.7372978,"reason":"broker was unresponsive"},"10201":{"timestamp":1712365572.7379427,"reason":"broker was unresponsive"},"10202":{"timestamp":1712365572.7385962,"reason":"broker was unresponsive"},"10203":{"timestamp":1712365572.7392387,"reason":"broker was unresponsive"},"10204":{"timestamp":1712365572.7399004,"reason":"broker was unresponsive"},"10205":{"timestamp":1712365572.7405581,"reason":"broker was unresponsive"},"10206":{"timestamp":1712365572.7412045,"reason":"broker was unresponsive"},"10207":{"timestamp":1712365572.7418675,"reason":"broker was unresponsive"},"10208":{"timestamp":1712365572.7425313,"reason":"broker was unresponsive"},"10209":{"timestamp":1712365572.743175,"reason":"broker was unresponsive"},"10210":{"timestamp":1712365572.7438438,"reason":"broker was unresponsive"},"10211":{"timestamp":1712365572.7445078,"reason":"broker was unresponsive"},"10212":{"timestamp":1712365572.7451587,"reason":"broker was unresponsive"},"10213":{"timestamp":1712365572.7458181,"reason":"broker was unresponsive"},"10214":{"timestamp":1712365572.7464828,"reason":"broker was unresponsive"},"10215":{"timestamp":1712365572.7471318,"reason":"broker was unresponsive"},"10216":{"timestamp":1712365572.7477911,"reason":"broker was unresponsive"},"10217":{"timestamp":1712365572.7484534,"reason":"broker was unresponsive"},"10218":{"timestamp":1712365572.7491021,"reason":"broker was unresponsive"},"10219":{"timestamp":1712365572.7497661,"reason":"broker was unresponsive"},"10220":{"timestamp":1712365572.7504568,"reason":"broker was unresponsive"},"10221":{"timestamp":1712365572.7511101,"reason":"broker was unresponsive"},"10222":{"timestamp":1712365572.7517745,"reason":"broker was unresponsive"},"10223":{"timestamp":1712365572.7524436,"reason":"broker was unresponsive"},"10224":{"timestamp":1712365572.7530951,"reason":"broker was unresponsive"},"10225":{"timestamp":1712365572.7537665,"reason":"broker was unresponsive"},"10226":{"timestamp":1712365572.7544343,"reason":"broker was unresponsive"},"10227":{"timestamp":1712365572.7550926,"reason":"broker was unresponsive"},"10228":{"timestamp":1712365572.7557595,"reason":"broker was unresponsive"},"10229":{"timestamp":1712365572.756428,"reason":"broker was unresponsive"},"10230":{"timestamp":1712365572.7570839,"reason":"broker was unresponsive"},"10231":{"timestamp":1712365572.7577524,"reason":"broker was unresponsive"},"10232":{"timestamp":1712365572.7584257,"reason":"broker was unresponsive"},"10233":{"timestamp":1712365572.7590814,"reason":"broker was unresponsive"},"10234":{"timestamp":1712365572.7597554,"reason":"broker was unresponsive"},"10235":{"timestamp":1712365572.760427,"reason":"broker was unresponsive"},"10236":{"timestamp":1712365572.7610805,"reason":"broker was unresponsive"},"10237":{"timestamp":1712365572.7617462,"reason":"broker was unresponsive"},"10238":{"timestamp":1712365572.7624183,"reason":"broker was unresponsive"},"10239":{"timestamp":1712365572.7630782,"reason":"broker was unresponsive"},"10240":{"timestamp":1712365572.7637465,"reason":"broker was unresponsive"},"10241":{"timestamp":1712365572.7644186,"reason":"broker was unresponsive"},"10242":{"timestamp":1712365572.7650795,"reason":"broker was unresponsive"},"10243":{"timestamp":1712365572.7657509,"reason":"broker was unresponsive"},"10244":{"timestamp":1712365572.7664204,"reason":"broker was unresponsive"},"10245":{"timestamp":1712365572.7670836,"reason":"broker was unresponsive"},"10246":{"timestamp":1712365572.7677605,"reason":"broker was unresponsive"},"10247":{"timestamp":1712365572.7684345,"reason":"broker was unresponsive"},"10248":{"timestamp":1712365572.7690954,"reason":"broker was unresponsive"},"10249":{"timestamp":1712365572.7697656,"reason":"broker was unresponsive"},"10250":{"timestamp":1712365572.7704449,"reason":"broker was unresponsive"},"10251":{"timestamp":1712365572.7711065,"reason":"broker was unresponsive"},"10252":{"timestamp":1712365572.7717836,"reason":"broker was unresponsive"},"10253":{"timestamp":1712365572.7724619,"reason":"broker was unresponsive"},"10254":{"timestamp":1712365572.7731254,"reason":"broker was unresponsive"},"10255":{"timestamp":1712365572.7737968,"reason":"broker was unresponsive"},"10256":{"timestamp":1712365572.7744751,"reason":"broker was unresponsive"},"10257":{"timestamp":1712365572.7751384,"reason":"broker was unresponsive"},"10258":{"timestamp":1712365572.7758152,"reason":"broker was unresponsive"},"10259":{"timestamp":1712365572.7764897,"reason":"broker was unresponsive"},"10260":{"timestamp":1712365572.7771556,"reason":"broker was unresponsive"},"10261":{"timestamp":1712361648.1654882,"reason":"nodediag failed dgemm_perf"},"10262":{"timestamp":1712365572.7784929,"reason":"broker was unresponsive"},"10263":{"timestamp":1712365572.7791626,"reason":"broker was unresponsive"},"10264":{"timestamp":1712365572.7798421,"reason":"broker was unresponsive"},"10265":{"timestamp":1712365572.7805238,"reason":"broker was unresponsive"},"10266":{"timestamp":1712365572.7811949,"reason":"broker was unresponsive"},"10267":{"timestamp":1712365572.7818754,"reason":"broker was unresponsive"},"10268":{"timestamp":1712365572.782562,"reason":"broker was unresponsive"},"10269":{"timestamp":1712365572.7832305,"reason":"broker was unresponsive"},"10270":{"timestamp":1712365572.7839177,"reason":"broker was unresponsive"},"10271":{"timestamp":1712365572.7846031,"reason":"broker was unresponsive"},"10272":{"timestamp":1712365572.7852895,"reason":"broker was unresponsive"},"10273":{"timestamp":1712365572.7859626,"reason":"broker was unresponsive"},"10274":{"timestamp":1712365572.7866507,"reason":"broker was unresponsive"},"10275":{"timestamp":1712365572.7873333,"reason":"broker was unresponsive"},"10276":{"timestamp":1712365572.7880034,"reason":"broker was unresponsive"},"10277":{"timestamp":1712365572.7886899,"reason":"broker was unresponsive"},"10278":{"timestamp":1712365572.7893965,"reason":"broker was unresponsive"},"10279":{"timestamp":1712365572.7900739,"reason":"broker was unresponsive"},"10280":{"timestamp":1712365572.7907746,"reason":"broker was unresponsive"},"10281":{"timestamp":1712365572.7914841,"reason":"broker was unresponsive"},"10282":{"timestamp":1712365572.7921603,"reason":"broker was unresponsive"},"10283":{"timestamp":1712365572.7928519,"reason":"broker was unresponsive"},"10284":{"timestamp":1712365572.7935426,"reason":"broker was unresponsive"},"10285":{"timestamp":1712365572.7942181,"reason":"broker was unresponsive"},"10286":{"timestamp":1712365572.7949085,"reason":"broker was unresponsive"},"10287":{"timestamp":1712365572.7955964,"reason":"broker was unresponsive"},"10288":{"timestamp":1712365572.7962806,"reason":"broker was unresponsive"},"10289":{"timestamp":1712365572.7969596,"reason":"broker was unresponsive"},"10290":{"timestamp":1712365572.7976491,"reason":"broker was unresponsive"},"10291":{"timestamp":1712365572.7983379,"reason":"broker was unresponsive"},"10292":{"timestamp":1712365572.7990148,"reason":"broker was unresponsive"},"10293":{"timestamp":1712365572.7997031,"reason":"broker was unresponsive"},"10294":{"timestamp":1712365572.8003981,"reason":"broker was unresponsive"},"10295":{"timestamp":1712365572.8010752,"reason":"broker was unresponsive"},"10296":{"timestamp":1712365572.801784,"reason":"broker was unresponsive"},"10297":{"timestamp":1712365572.8024814,"reason":"broker was unresponsive"},"10298":{"timestamp":1712365572.8031602,"reason":"broker was unresponsive"},"10299":{"timestamp":1712365572.8038535,"reason":"broker was unresponsive"},"10300":{"timestamp":1712365572.8045502,"reason":"broker was unresponsive"},"10301":{"timestamp":1712365572.8052313,"reason":"broker was unresponsive"},"10302":{"timestamp":1712365572.8059192,"reason":"broker was unresponsive"},"10303":{"timestamp":1712236781.0769484,"reason":"bad partner node --JRG"},"10304":{"timestamp":1712365572.8066132,"reason":"broker was unresponsive"},"10305":{"timestamp":1712365572.8073051,"reason":"broker was unresponsive"},"10306":{"timestamp":1712365572.8079889,"reason":"broker was unresponsive"},"10307":{"timestamp":1712365572.808682,"reason":"broker was unresponsive"},"10308":{"timestamp":1712365572.8093944,"reason":"broker was unresponsive"},"10309":{"timestamp":1712365572.8100817,"reason":"broker was unresponsive"},"10310":{"timestamp":1712365572.8107853,"reason":"broker was unresponsive"},"10311":{"timestamp":1712365572.8114851,"reason":"broker was unresponsive"},"10312":{"timestamp":1712365572.812171,"reason":"broker was unresponsive"},"10313":{"timestamp":1712365572.812886,"reason":"broker was unresponsive"},"10314":{"timestamp":1712365572.8135979,"reason":"broker was unresponsive"},"10315":{"timestamp":1712365572.814296,"reason":"broker was unresponsive"},"10316":{"timestamp":1712365572.8149822,"reason":"broker was unresponsive"},"10317":{"timestamp":1712365572.8156869,"reason":"broker was unresponsive"},"10318":{"timestamp":1712365572.8163836,"reason":"broker was unresponsive"},"10319":{"timestamp":1712365572.8170693,"reason":"broker was unresponsive"},"10320":{"timestamp":1712365572.8177671,"reason":"broker was unresponsive"},"10321":{"timestamp":1712365572.8184693,"reason":"broker was unresponsive"},"10322":{"timestamp":1712365572.8192341,"reason":"broker was unresponsive"},"10323":{"timestamp":1712365572.8199363,"reason":"broker was unresponsive"},"10324":{"timestamp":1712365572.8206398,"reason":"broker was unresponsive"},"10325":{"timestamp":1712365572.8213398,"reason":"broker was unresponsive"},"10326":{"timestamp":1712365572.8220305,"reason":"broker was unresponsive"},"10327":{"timestamp":1712365572.8227303,"reason":"broker was unresponsive"},"10328":{"timestamp":1712365572.8234341,"reason":"broker was unresponsive"},"10329":{"timestamp":1712365572.8241277,"reason":"broker was unresponsive"},"10330":{"timestamp":1712365572.824832,"reason":"broker was unresponsive"},"10331":{"timestamp":1712365572.8255439,"reason":"broker was unresponsive"},"10332":{"timestamp":1712365572.8262496,"reason":"broker was unresponsive"},"10333":{"timestamp":1712365572.8270316,"reason":"broker was unresponsive"},"10334":{"timestamp":1712365572.8279626,"reason":"broker was unresponsive"},"10335":{"timestamp":1712365572.8287179,"reason":"broker was unresponsive"},"10336":{"timestamp":1712365572.8297181,"reason":"broker was unresponsive"},"10337":{"timestamp":1712365572.8305345,"reason":"broker was unresponsive"},"10338":{"timestamp":1712365572.8315363,"reason":"broker was unresponsive"},"10339":{"timestamp":1712365572.8323243,"reason":"broker was unresponsive"},"10340":{"timestamp":1712365572.8330812,"reason":"broker was unresponsive"},"10341":{"timestamp":1712365572.833797,"reason":"broker was unresponsive"},"10342":{"timestamp":1712365572.8345113,"reason":"broker was unresponsive"},"10343":{"timestamp":1712337869.3428621,"reason":"broker was unresponsive"},"10344":{"timestamp":1712337869.4427822,"reason":"broker was unresponsive"},"10345":{"timestamp":1712365572.8352108,"reason":"broker was unresponsive"},"10346":{"timestamp":1712365572.8359258,"reason":"broker was unresponsive"},"10347":{"timestamp":1712365572.8366439,"reason":"broker was unresponsive"},"10348":{"timestamp":1712365572.8373766,"reason":"broker was unresponsive"},"10349":{"timestamp":1712365572.8380766,"reason":"broker was unresponsive"},"10350":{"timestamp":1712365572.838814,"reason":"broker was unresponsive"},"10351":{"timestamp":1712365572.8395448,"reason":"broker was unresponsive"},"10352":{"timestamp":1712365572.8402493,"reason":"broker was unresponsive"},"10353":{"timestamp":1712365572.8409641,"reason":"broker was unresponsive"},"10354":{"timestamp":1712365572.8416772,"reason":"broker was unresponsive"},"10355":{"timestamp":1712337664.7496204,"reason":"bad partner node --JRG"},"10357":{"timestamp":1712365572.8423979,"reason":"broker was unresponsive"},"10358":{"timestamp":1712365572.8430998,"reason":"broker was unresponsive"},"10359":{"timestamp":1712365572.8438184,"reason":"broker was unresponsive"},"10360":{"timestamp":1712365572.8445356,"reason":"broker was unresponsive"},"10361":{"timestamp":1712365572.8452363,"reason":"broker was unresponsive"},"10362":{"timestamp":1712365572.8459775,"reason":"broker was unresponsive"},"10363":{"timestamp":1712365572.8466973,"reason":"broker was unresponsive"},"10364":{"timestamp":1712365572.8474131,"reason":"broker was unresponsive"},"10365":{"timestamp":1712365572.8481154,"reason":"broker was unresponsive"},"10366":{"timestamp":1712365572.8488317,"reason":"broker was unresponsive"},"10367":{"timestamp":1712365572.8495479,"reason":"broker was unresponsive"},"10368":{"timestamp":1712365572.8502533,"reason":"broker was unresponsive"},"10369":{"timestamp":1712365572.8509703,"reason":"broker was unresponsive"},"10370":{"timestamp":1712365572.8516893,"reason":"broker was unresponsive"},"10371":{"timestamp":1712365572.8524089,"reason":"broker was unresponsive"},"10372":{"timestamp":1712365572.8531156,"reason":"broker was unresponsive"},"10373":{"timestamp":1712365572.8538339,"reason":"broker was unresponsive"},"10374":{"timestamp":1712365572.8545616,"reason":"broker was unresponsive"},"10375":{"timestamp":1712365572.8552866,"reason":"broker was unresponsive"},"10376":{"timestamp":1712365572.8559964,"reason":"broker was unresponsive"},"10377":{"timestamp":1712365572.8567138,"reason":"broker was unresponsive"},"10378":{"timestamp":1712365572.8574705,"reason":"broker was unresponsive"},"10379":{"timestamp":1712365572.8581994,"reason":"broker was unresponsive"},"10380":{"timestamp":1712365572.8589301,"reason":"broker was unresponsive"},"10381":{"timestamp":1712365572.8596814,"reason":"broker was unresponsive"},"10382":{"timestamp":1712365572.8604298,"reason":"broker was unresponsive"},"10383":{"timestamp":1712365572.8611393,"reason":"broker was unresponsive"},"10384":{"timestamp":1712365572.8618772,"reason":"broker was unresponsive"},"10385":{"timestamp":1712365572.8626282,"reason":"broker was unresponsive"},"10386":{"timestamp":1712365572.8633609,"reason":"broker was unresponsive"},"10387":{"timestamp":1712365572.8640823,"reason":"broker was unresponsive"},"10388":{"timestamp":1712365572.8648272,"reason":"broker was unresponsive"},"10389":{"timestamp":1712365572.8655748,"reason":"broker was unresponsive"},"10390":{"timestamp":1712365572.866308,"reason":"broker was unresponsive"},"10391":{"timestamp":1712365572.8670402,"reason":"broker was unresponsive"},"10392":{"timestamp":1712365572.8677895,"reason":"broker was unresponsive"},"10393":{"timestamp":1712365572.8685358,"reason":"broker was unresponsive"},"10394":{"timestamp":1712365572.869292,"reason":"broker was unresponsive"},"10395":{"timestamp":1712365572.8700292,"reason":"broker was unresponsive"},"10396":{"timestamp":1712365572.8707633,"reason":"broker was unresponsive"},"10397":{"timestamp":1712365572.8715243,"reason":"broker was unresponsive"},"10398":{"timestamp":1712365572.8722613,"reason":"broker was unresponsive"},"10399":{"timestamp":1712365572.8730001,"reason":"broker was unresponsive"},"10400":{"timestamp":1712365572.8737504,"reason":"broker was unresponsive"},"10401":{"timestamp":1712365572.8745008,"reason":"broker was unresponsive"},"10402":{"timestamp":1712365572.8752205,"reason":"broker was unresponsive"},"10403":{"timestamp":1712365572.875967,"reason":"broker was unresponsive"},"10404":{"timestamp":1712365572.8767276,"reason":"broker was unresponsive"},"10405":{"timestamp":1712365572.8774903,"reason":"broker was unresponsive"},"10406":{"timestamp":1712365572.8782103,"reason":"broker was unresponsive"},"10407":{"timestamp":1712365572.8789659,"reason":"broker was unresponsive"},"10408":{"timestamp":1712365572.8797224,"reason":"broker was unresponsive"},"10409":{"timestamp":1712365572.880466,"reason":"broker was unresponsive"},"10410":{"timestamp":1712365572.8812056,"reason":"broker was unresponsive"},"10411":{"timestamp":1712365572.8819704,"reason":"broker was unresponsive"},"10412":{"timestamp":1712365572.8827114,"reason":"broker was unresponsive"},"10413":{"timestamp":1712365572.8834713,"reason":"broker was unresponsive"},"10414":{"timestamp":1712365572.8842137,"reason":"broker was unresponsive"},"10415":{"timestamp":1712365572.8849554,"reason":"broker was unresponsive"},"10416":{"timestamp":1712365572.8857186,"reason":"broker was unresponsive"},"10417":{"timestamp":1712365572.8864768,"reason":"broker was unresponsive"},"10418":{"timestamp":1712365572.8872135,"reason":"broker was unresponsive"},"10419":{"timestamp":1712365572.8879604,"reason":"broker was unresponsive"},"10420":{"timestamp":1712365572.8887148,"reason":"broker was unresponsive"},"10421":{"timestamp":1712365572.8894806,"reason":"broker was unresponsive"},"10422":{"timestamp":1712365572.8902094,"reason":"broker was unresponsive"},"10423":{"timestamp":1712365572.8909731,"reason":"broker was unresponsive"},"10424":{"timestamp":1712365572.8917391,"reason":"broker was unresponsive"},"10425":{"timestamp":1712365572.8924842,"reason":"broker was unresponsive"},"10426":{"timestamp":1712365572.8932292,"reason":"broker was unresponsive"},"10427":{"timestamp":1712365572.8939965,"reason":"broker was unresponsive"},"10428":{"timestamp":1712365572.894752,"reason":"broker was unresponsive"},"10429":{"timestamp":1712365572.8955173,"reason":"broker was unresponsive"},"10430":{"timestamp":1712365572.8962796,"reason":"broker was unresponsive"},"10431":{"timestamp":1712365572.8970304,"reason":"broker was unresponsive"},"10432":{"timestamp":1712365572.8977799,"reason":"broker was unresponsive"},"10433":{"timestamp":1712365572.8985479,"reason":"broker was unresponsive"},"10434":{"timestamp":1712365572.8993146,"reason":"broker was unresponsive"},"10435":{"timestamp":1712365572.9000452,"reason":"broker was unresponsive"},"10436":{"timestamp":1712365572.9008093,"reason":"broker was unresponsive"},"10437":{"timestamp":1712365572.9015768,"reason":"broker was unresponsive"},"10438":{"timestamp":1712365572.9023242,"reason":"broker was unresponsive"},"10439":{"timestamp":1712365572.903074,"reason":"broker was unresponsive"},"10440":{"timestamp":1712365572.9038441,"reason":"broker was unresponsive"},"10441":{"timestamp":1712365572.9046087,"reason":"broker was unresponsive"},"10442":{"timestamp":1712365572.9053605,"reason":"broker was unresponsive"},"10443":{"timestamp":1712365572.9061131,"reason":"broker was unresponsive"},"10444":{"timestamp":1712365572.9068825,"reason":"broker was unresponsive"},"10445":{"timestamp":1712365572.9076383,"reason":"broker was unresponsive"},"10446":{"timestamp":1712365572.9084084,"reason":"broker was unresponsive"},"10447":{"timestamp":1712365572.9091628,"reason":"broker was unresponsive"},"10448":{"timestamp":1712365572.9099174,"reason":"broker was unresponsive"},"10449":{"timestamp":1712365572.9106827,"reason":"broker was unresponsive"},"10450":{"timestamp":1712365572.9114556,"reason":"broker was unresponsive"},"10451":{"timestamp":1712365572.9122086,"reason":"broker was unresponsive"},"10452":{"timestamp":1712365572.9129622,"reason":"broker was unresponsive"},"10453":{"timestamp":1712365572.9137363,"reason":"broker was unresponsive"},"10454":{"timestamp":1712365572.9145091,"reason":"broker was unresponsive"},"10455":{"timestamp":1712365572.9152472,"reason":"broker was unresponsive"},"10456":{"timestamp":1712365572.9160945,"reason":"broker was unresponsive"},"10457":{"timestamp":1712365572.9169171,"reason":"broker was unresponsive"},"10458":{"timestamp":1712365572.917737,"reason":"broker was unresponsive"},"10459":{"timestamp":1712365572.9185164,"reason":"broker was unresponsive"},"10460":{"timestamp":1712365572.9192972,"reason":"broker was unresponsive"},"10461":{"timestamp":1712365572.9200597,"reason":"broker was unresponsive"},"10462":{"timestamp":1712365572.920836,"reason":"broker was unresponsive"},"10463":{"timestamp":1712365572.921603,"reason":"broker was unresponsive"},"10464":{"timestamp":1712365572.9223852,"reason":"broker was unresponsive"},"10465":{"timestamp":1712365572.9231505,"reason":"broker was unresponsive"},"10466":{"timestamp":1712365572.9239168,"reason":"broker was unresponsive"},"10467":{"timestamp":1712365572.9246945,"reason":"broker was unresponsive"},"10468":{"timestamp":1712365572.9254715,"reason":"broker was unresponsive"},"10469":{"timestamp":1712355843.3485792,"reason":"broker was unresponsive"},"10470":{"timestamp":1712355843.3487108,"reason":"broker was unresponsive"},"10471":{"timestamp":1712355841.3457093,"reason":"broker was unresponsive"},"10472":{"timestamp":1712355843.3487971,"reason":"broker was unresponsive"},"10473":{"timestamp":1712355842.2199931,"reason":"broker was unresponsive"},"10474":{"timestamp":1712355847.3450718,"reason":"broker was unresponsive"},"10475":{"timestamp":1712355843.348918,"reason":"broker was unresponsive"},"10476":{"timestamp":1712355844.270498,"reason":"broker was unresponsive"},"10477":{"timestamp":1712355845.345845,"reason":"broker was unresponsive"},"10478":{"timestamp":1712355846.0879886,"reason":"broker was unresponsive"},"10479":{"timestamp":1712355847.3451836,"reason":"broker was unresponsive"},"10480":{"timestamp":1712355850.0757811,"reason":"broker was unresponsive"},"10481":{"timestamp":1712355847.3452773,"reason":"broker was unresponsive"},"10482":{"timestamp":1712355851.3449905,"reason":"broker was unresponsive"},"10483":{"timestamp":1712355852.1232853,"reason":"broker was unresponsive"},"10484":{"timestamp":1712355848.0988836,"reason":"broker was unresponsive"},"10485":{"timestamp":1712365572.9262331,"reason":"broker was unresponsive"},"10486":{"timestamp":1712365572.9270127,"reason":"broker was unresponsive"},"10487":{"timestamp":1712365572.9277925,"reason":"broker was unresponsive"},"10488":{"timestamp":1712365572.9285741,"reason":"broker was unresponsive"},"10489":{"timestamp":1712365572.9293399,"reason":"broker was unresponsive"},"10490":{"timestamp":1712365572.9301057,"reason":"broker was unresponsive"},"10491":{"timestamp":1712365572.930898,"reason":"broker was unresponsive"},"10492":{"timestamp":1712365572.9316812,"reason":"broker was unresponsive"},"10493":{"timestamp":1712365572.9324479,"reason":"broker was unresponsive"},"10494":{"timestamp":1712365572.9332445,"reason":"broker was unresponsive"},"10495":{"timestamp":1712365572.9340601,"reason":"broker was unresponsive"},"10496":{"timestamp":1712365572.9348292,"reason":"broker was unresponsive"},"10497":{"timestamp":1712365572.9356434,"reason":"broker was unresponsive"},"10498":{"timestamp":1712365572.9364321,"reason":"broker was unresponsive"},"10499":{"timestamp":1712365572.9372032,"reason":"broker was unresponsive"},"10500":{"timestamp":1712365572.9379728,"reason":"broker was unresponsive"},"10501":{"timestamp":1712365572.9387615,"reason":"broker was unresponsive"},"10502":{"timestamp":1712365572.9395902,"reason":"broker was unresponsive"},"10503":{"timestamp":1712365572.9403853,"reason":"broker was unresponsive"},"10504":{"timestamp":1712365572.9411416,"reason":"broker was unresponsive"},"10505":{"timestamp":1712365572.9419389,"reason":"broker was unresponsive"},"10506":{"timestamp":1712365572.9427316,"reason":"broker was unresponsive"},"10507":{"timestamp":1712365572.9435003,"reason":"broker was unresponsive"},"10508":{"timestamp":1712365572.9442813,"reason":"broker was unresponsive"},"10509":{"timestamp":1712365572.9450576,"reason":"broker was unresponsive"},"10510":{"timestamp":1712365572.9458508,"reason":"broker was unresponsive"},"10511":{"timestamp":1712365572.9466181,"reason":"broker was unresponsive"},"10512":{"timestamp":1712365572.9474154,"reason":"broker was unresponsive"},"10513":{"timestamp":1712365572.9481881,"reason":"broker was unresponsive"},"10514":{"timestamp":1712365572.9489825,"reason":"broker was unresponsive"},"10515":{"timestamp":1712365572.9497581,"reason":"broker was unresponsive"},"10516":{"timestamp":1712365572.9505618,"reason":"broker was unresponsive"},"10517":{"timestamp":1712252115.930419,"reason":"bad partner node"},"10518":{"timestamp":1712086791.4119098,"reason":"Node wont power on --JRG"},"10519":{"timestamp":1712365572.951354,"reason":"broker was unresponsive"},"10520":{"timestamp":1712365572.9521089,"reason":"broker was unresponsive"},"10521":{"timestamp":1712365572.9528978,"reason":"broker was unresponsive"},"10522":{"timestamp":1712365572.9536836,"reason":"broker was unresponsive"},"10523":{"timestamp":1712365572.9544795,"reason":"broker was unresponsive"},"10524":{"timestamp":1712365572.9552381,"reason":"broker was unresponsive"},"10525":{"timestamp":1712365572.956028,"reason":"broker was unresponsive"},"10526":{"timestamp":1712365572.9568243,"reason":"broker was unresponsive"},"10527":{"timestamp":1712365572.9576445,"reason":"broker was unresponsive"},"10528":{"timestamp":1712365572.9584339,"reason":"broker was unresponsive"},"10529":{"timestamp":1712365572.9592175,"reason":"broker was unresponsive"},"10530":{"timestamp":1712365572.9600136,"reason":"broker was unresponsive"},"10531":{"timestamp":1712365572.9607921,"reason":"broker was unresponsive"},"10532":{"timestamp":1712365572.9615884,"reason":"broker was unresponsive"},"10533":{"timestamp":1712365572.9623878,"reason":"broker was unresponsive"},"10534":{"timestamp":1712365572.9631686,"reason":"broker was unresponsive"},"10535":{"timestamp":1712365572.9639513,"reason":"broker was unresponsive"},"10536":{"timestamp":1712365572.9647496,"reason":"broker was unresponsive"},"10537":{"timestamp":1712365572.9655533,"reason":"broker was unresponsive"},"10538":{"timestamp":1712365572.9663455,"reason":"broker was unresponsive"},"10539":{"timestamp":1712365572.9671082,"reason":"broker was unresponsive"},"10540":{"timestamp":1712365572.9679122,"reason":"broker was unresponsive"},"10541":{"timestamp":1712365572.9687154,"reason":"broker was unresponsive"},"10542":{"timestamp":1712365572.9694993,"reason":"broker was unresponsive"},"10543":{"timestamp":1712365572.9702938,"reason":"broker was unresponsive"},"10544":{"timestamp":1712365572.9710782,"reason":"broker was unresponsive"},"10545":{"timestamp":1712365572.9718804,"reason":"broker was unresponsive"},"10546":{"timestamp":1712365572.9726734,"reason":"broker was unresponsive"},"10547":{"timestamp":1712365572.9734778,"reason":"broker was unresponsive"},"10548":{"timestamp":1712365572.9742801,"reason":"broker was unresponsive"},"10549":{"timestamp":1712365572.9750667,"reason":"broker was unresponsive"},"10550":{"timestamp":1712365572.9758546,"reason":"broker was unresponsive"},"10551":{"timestamp":1712365572.9766574,"reason":"broker was unresponsive"},"10552":{"timestamp":1712365572.9774628,"reason":"broker was unresponsive"},"10553":{"timestamp":1712365572.9782512,"reason":"broker was unresponsive"},"10554":{"timestamp":1712365572.979044,"reason":"broker was unresponsive"},"10555":{"timestamp":1712365572.9798515,"reason":"broker was unresponsive"},"10556":{"timestamp":1712365572.9806664,"reason":"broker was unresponsive"},"10557":{"timestamp":1712365572.9814801,"reason":"broker was unresponsive"},"10558":{"timestamp":1712365572.9822564,"reason":"broker was unresponsive"},"10559":{"timestamp":1712365572.9830599,"reason":"broker was unresponsive"},"10560":{"timestamp":1712365572.9838684,"reason":"broker was unresponsive"},"10561":{"timestamp":1712365572.984668,"reason":"broker was unresponsive"},"10562":{"timestamp":1712365572.9854679,"reason":"broker was unresponsive"},"10563":{"timestamp":1712365572.9862571,"reason":"broker was unresponsive"},"10564":{"timestamp":1712365572.9870772,"reason":"broker was unresponsive"},"10565":{"timestamp":1712365572.987865,"reason":"broker was unresponsive"},"10566":{"timestamp":1712365572.988678,"reason":"broker was unresponsive"},"10567":{"timestamp":1712365572.9894888,"reason":"broker was unresponsive"},"10568":{"timestamp":1712365572.9903092,"reason":"broker was unresponsive"},"10569":{"timestamp":1712365572.9910865,"reason":"broker was unresponsive"},"10570":{"timestamp":1712365572.9919076,"reason":"broker was unresponsive"},"10571":{"timestamp":1712365572.9927187,"reason":"broker was unresponsive"},"10572":{"timestamp":1712365572.9935393,"reason":"broker was unresponsive"},"10573":{"timestamp":1712365572.9943354,"reason":"broker was unresponsive"},"10574":{"timestamp":1712365572.9951277,"reason":"broker was unresponsive"},"10575":{"timestamp":1712365572.9959493,"reason":"broker was unresponsive"},"10576":{"timestamp":1712365572.9967685,"reason":"broker was unresponsive"},"10577":{"timestamp":1712365572.9975662,"reason":"broker was unresponsive"},"10578":{"timestamp":1712365572.9983752,"reason":"broker was unresponsive"},"10579":{"timestamp":1712365572.9991708,"reason":"broker was unresponsive"},"10580":{"timestamp":1712365572.9999917,"reason":"broker was unresponsive"},"10581":{"timestamp":1712365573.0007906,"reason":"broker was unresponsive"},"10582":{"timestamp":1712365573.0016079,"reason":"broker was unresponsive"},"10583":{"timestamp":1712365573.0024312,"reason":"broker was unresponsive"},"10584":{"timestamp":1712365573.0032296,"reason":"broker was unresponsive"},"10585":{"timestamp":1712365573.0040317,"reason":"broker was unresponsive"},"10586":{"timestamp":1712365573.0048473,"reason":"broker was unresponsive"},"10587":{"timestamp":1712365573.0056653,"reason":"broker was unresponsive"},"10588":{"timestamp":1712365573.0064816,"reason":"broker was unresponsive"},"10589":{"timestamp":1712365573.0072932,"reason":"broker was unresponsive"},"10590":{"timestamp":1712365573.0080955,"reason":"broker was unresponsive"},"10591":{"timestamp":1712365573.0089159,"reason":"broker was unresponsive"},"10592":{"timestamp":1712365573.0097389,"reason":"broker was unresponsive"},"10593":{"timestamp":1712365573.0105457,"reason":"broker was unresponsive"},"10594":{"timestamp":1712365573.0113752,"reason":"broker was unresponsive"},"10595":{"timestamp":1712365573.0121825,"reason":"broker was unresponsive"},"10596":{"timestamp":1712365573.013006,"reason":"broker was unresponsive"},"10597":{"timestamp":1712365573.013828,"reason":"broker was unresponsive"},"10598":{"timestamp":1712365573.014647,"reason":"broker was unresponsive"},"10599":{"timestamp":1712365573.0154741,"reason":"broker was unresponsive"},"10600":{"timestamp":1712365573.0162997,"reason":"broker was unresponsive"},"10601":{"timestamp":1712365573.0170829,"reason":"broker was unresponsive"},"10602":{"timestamp":1712365573.0179217,"reason":"broker was unresponsive"},"10603":{"timestamp":1712365573.0187483,"reason":"broker was unresponsive"},"10604":{"timestamp":1712365573.0195701,"reason":"broker was unresponsive"},"10605":{"timestamp":1712365573.020402,"reason":"broker was unresponsive"},"10606":{"timestamp":1712365573.021193,"reason":"broker was unresponsive"},"10607":{"timestamp":1712365573.0220175,"reason":"broker was unresponsive"},"10608":{"timestamp":1712365573.0228496,"reason":"broker was unresponsive"},"10609":{"timestamp":1712365573.02367,"reason":"broker was unresponsive"},"10610":{"timestamp":1712365573.0244899,"reason":"broker was unresponsive"},"10611":{"timestamp":1712365573.0253193,"reason":"broker was unresponsive"},"10612":{"timestamp":1712365573.0261273,"reason":"broker was unresponsive"},"10613":{"timestamp":1712365573.0269647,"reason":"broker was unresponsive"},"10614":{"timestamp":1712365573.0277812,"reason":"broker was unresponsive"},"10615":{"timestamp":1712365573.0286078,"reason":"broker was unresponsive"},"10616":{"timestamp":1712365573.0294335,"reason":"broker was unresponsive"},"10617":{"timestamp":1712365573.0302451,"reason":"broker was unresponsive"},"10618":{"timestamp":1712365573.0310516,"reason":"broker was unresponsive"},"10619":{"timestamp":1712365573.0318937,"reason":"broker was unresponsive"},"10620":{"timestamp":1712365573.0327175,"reason":"broker was unresponsive"},"10621":{"timestamp":1712365573.0335743,"reason":"broker was unresponsive"},"10622":{"timestamp":1712365573.03439,"reason":"broker was unresponsive"},"10623":{"timestamp":1712365573.0352039,"reason":"broker was unresponsive"},"10624":{"timestamp":1712365573.0360417,"reason":"broker was unresponsive"},"10625":{"timestamp":1712365573.036876,"reason":"broker was unresponsive"},"10626":{"timestamp":1712365573.0377133,"reason":"broker was unresponsive"},"10627":{"timestamp":1712365573.0385361,"reason":"broker was unresponsive"},"10628":{"timestamp":1712365573.0394185,"reason":"broker was unresponsive"},"10630":{"timestamp":1712365573.0402396,"reason":"broker was unresponsive"},"10631":{"timestamp":1712365573.0410807,"reason":"broker was unresponsive"},"10632":{"timestamp":1712365573.0418937,"reason":"broker was unresponsive"},"10633":{"timestamp":1712365573.0427399,"reason":"broker was unresponsive"},"10635":{"timestamp":1712365573.0436072,"reason":"broker was unresponsive"},"10636":{"timestamp":1712365573.0444498,"reason":"broker was unresponsive"},"10640":{"timestamp":1712365573.0452611,"reason":"broker was unresponsive"},"10641":{"timestamp":1712365573.0460789,"reason":"broker was unresponsive"},"10645":{"timestamp":1712365573.0469213,"reason":"broker was unresponsive"},"10646":{"timestamp":1712365573.04776,"reason":"broker was unresponsive"},"10647":{"timestamp":1712365573.0486219,"reason":"broker was unresponsive"},"10648":{"timestamp":1712365573.0494418,"reason":"broker was unresponsive"},"10649":{"timestamp":1712365573.050257,"reason":"broker was unresponsive"},"10650":{"timestamp":1712365573.0511172,"reason":"broker was unresponsive"},"10651":{"timestamp":1712365573.05196,"reason":"broker was unresponsive"},"10652":{"timestamp":1712365573.0528162,"reason":"broker was unresponsive"},"10653":{"timestamp":1712365573.0536404,"reason":"broker was unresponsive"},"10654":{"timestamp":1712365573.0544887,"reason":"broker was unresponsive"},"10655":{"timestamp":1712365573.0553267,"reason":"broker was unresponsive"},"10656":{"timestamp":1712365573.0561504,"reason":"broker was unresponsive"},"10658":{"timestamp":1712365573.0569766,"reason":"broker was unresponsive"},"10659":{"timestamp":1712365573.0578148,"reason":"broker was unresponsive"},"10660":{"timestamp":1712365573.058665,"reason":"broker was unresponsive"},"10661":{"timestamp":1712365573.0595052,"reason":"broker was unresponsive"},"10662":{"timestamp":1712365573.0603325,"reason":"broker was unresponsive"},"10663":{"timestamp":1712365573.0611575,"reason":"broker was unresponsive"},"10664":{"timestamp":1712365573.0620008,"reason":"broker was unresponsive"},"10665":{"timestamp":1712365573.0628598,"reason":"broker was unresponsive"},"10666":{"timestamp":1712365573.0637088,"reason":"broker was unresponsive"},"10667":{"timestamp":1712365573.0645459,"reason":"broker was unresponsive"},"10668":{"timestamp":1712365573.0653944,"reason":"broker was unresponsive"},"10669":{"timestamp":1712365573.0662284,"reason":"broker was unresponsive"},"10670":{"timestamp":1712365573.0670981,"reason":"broker was unresponsive"},"10671":{"timestamp":1712365573.067935,"reason":"broker was unresponsive"},"10672":{"timestamp":1712365573.0687804,"reason":"broker was unresponsive"},"10673":{"timestamp":1712365573.0696244,"reason":"broker was unresponsive"},"10674":{"timestamp":1712365573.0704739,"reason":"broker was unresponsive"},"10675":{"timestamp":1712365573.0713198,"reason":"broker was unresponsive"},"10676":{"timestamp":1712365573.0721307,"reason":"broker was unresponsive"},"10677":{"timestamp":1712365573.0729833,"reason":"broker was unresponsive"},"10678":{"timestamp":1712365573.0738344,"reason":"broker was unresponsive"},"10679":{"timestamp":1712365573.0746858,"reason":"broker was unresponsive"},"10680":{"timestamp":1712365573.0755274,"reason":"broker was unresponsive"},"10681":{"timestamp":1712365573.076369,"reason":"broker was unresponsive"},"10682":{"timestamp":1712365573.0772023,"reason":"broker was unresponsive"},"10683":{"timestamp":1712365573.0780585,"reason":"broker was unresponsive"},"10684":{"timestamp":1712365573.0789101,"reason":"broker was unresponsive"},"10685":{"timestamp":1712365573.0797427,"reason":"broker was unresponsive"},"10686":{"timestamp":1712365573.0806029,"reason":"broker was unresponsive"},"10687":{"timestamp":1712365573.0814638,"reason":"broker was unresponsive"},"10688":{"timestamp":1712365573.0823247,"reason":"broker was unresponsive"},"10689":{"timestamp":1712365573.0831583,"reason":"broker was unresponsive"},"10690":{"timestamp":1712365573.0839906,"reason":"broker was unresponsive"},"10691":{"timestamp":1712365573.0848472,"reason":"broker was unresponsive"},"10692":{"timestamp":1712365573.0857019,"reason":"broker was unresponsive"},"10695":{"timestamp":1712365573.0865667,"reason":"broker was unresponsive"},"10696":{"timestamp":1712365573.0874145,"reason":"broker was unresponsive"},"10697":{"timestamp":1712365573.0882497,"reason":"broker was unresponsive"},"10698":{"timestamp":1712365573.0891149,"reason":"broker was unresponsive"},"10699":{"timestamp":1712365573.0899694,"reason":"broker was unresponsive"},"10700":{"timestamp":1712365573.0908313,"reason":"broker was unresponsive"},"10701":{"timestamp":1712365573.091676,"reason":"broker was unresponsive"},"10703":{"timestamp":1712365573.0925295,"reason":"broker was unresponsive"},"10704":{"timestamp":1712365573.0933933,"reason":"broker was unresponsive"},"10705":{"timestamp":1712365573.0942342,"reason":"broker was unresponsive"},"10706":{"timestamp":1712365573.0950918,"reason":"broker was unresponsive"},"10707":{"timestamp":1712365573.0959344,"reason":"broker was unresponsive"},"10708":{"timestamp":1712365573.0967982,"reason":"broker was unresponsive"},"10709":{"timestamp":1712365573.0976555,"reason":"broker was unresponsive"},"10710":{"timestamp":1712365573.0985098,"reason":"broker was unresponsive"},"10711":{"timestamp":1712365573.099354,"reason":"broker was unresponsive"},"10713":{"timestamp":1712365573.1001899,"reason":"broker was unresponsive"},"10714":{"timestamp":1712365573.1010525,"reason":"broker was unresponsive"},"10715":{"timestamp":1712365573.1019149,"reason":"broker was unresponsive"},"10716":{"timestamp":1712365573.1027777,"reason":"broker was unresponsive"},"10717":{"timestamp":1712365573.1036193,"reason":"broker was unresponsive"},"10719":{"timestamp":1712365573.1044838,"reason":"broker was unresponsive"},"10720":{"timestamp":1712365573.1053448,"reason":"broker was unresponsive"},"10721":{"timestamp":1712365573.1061928,"reason":"broker was unresponsive"},"10722":{"timestamp":1712365573.1070554,"reason":"broker was unresponsive"},"10723":{"timestamp":1712365573.1079028,"reason":"broker was unresponsive"},"10724":{"timestamp":1712365573.1087658,"reason":"broker was unresponsive"},"10725":{"timestamp":1712365573.1096323,"reason":"broker was unresponsive"},"10726":{"timestamp":1712365573.1105011,"reason":"broker was unresponsive"},"10727":{"timestamp":1712365573.1113672,"reason":"broker was unresponsive"},"10728":{"timestamp":1712365573.1121993,"reason":"broker was unresponsive"},"10729":{"timestamp":1712365573.1130645,"reason":"broker was unresponsive"},"10730":{"timestamp":1712365573.1139362,"reason":"broker was unresponsive"},"10731":{"timestamp":1712365573.1148014,"reason":"broker was unresponsive"},"10732":{"timestamp":1712365573.1156693,"reason":"broker was unresponsive"},"10733":{"timestamp":1712365573.1165202,"reason":"broker was unresponsive"},"10734":{"timestamp":1712365573.117388,"reason":"broker was unresponsive"},"10735":{"timestamp":1712365573.1182439,"reason":"broker was unresponsive"},"10736":{"timestamp":1712365573.119113,"reason":"broker was unresponsive"},"10737":{"timestamp":1712365573.1199801,"reason":"broker was unresponsive"},"10738":{"timestamp":1712365573.1208405,"reason":"broker was unresponsive"},"10741":{"timestamp":1712365573.1217148,"reason":"broker was unresponsive"},"10742":{"timestamp":1712365573.1225886,"reason":"broker was unresponsive"},"10743":{"timestamp":1712365573.123461,"reason":"broker was unresponsive"},"10744":{"timestamp":1712365573.1243222,"reason":"broker was unresponsive"},"10745":{"timestamp":1712365573.1251574,"reason":"broker was unresponsive"},"10746":{"timestamp":1712365573.1260245,"reason":"broker was unresponsive"},"10747":{"timestamp":1712365573.126895,"reason":"broker was unresponsive"},"10748":{"timestamp":1712365573.1277659,"reason":"broker was unresponsive"},"10749":{"timestamp":1712365573.1286366,"reason":"broker was unresponsive"},"10750":{"timestamp":1712365573.1294935,"reason":"broker was unresponsive"},"10751":{"timestamp":1712365573.1303656,"reason":"broker was unresponsive"},"10752":{"timestamp":1712365573.1312227,"reason":"broker was unresponsive"},"10753":{"timestamp":1712365573.1320941,"reason":"broker was unresponsive"},"10754":{"timestamp":1712365573.1329622,"reason":"broker was unresponsive"},"10755":{"timestamp":1712365573.1338165,"reason":"broker was unresponsive"},"10756":{"timestamp":1712365573.1346927,"reason":"broker was unresponsive"},"10757":{"timestamp":1712365573.1355712,"reason":"broker was unresponsive"},"10758":{"timestamp":1712365573.1364465,"reason":"broker was unresponsive"},"10759":{"timestamp":1712365573.1373212,"reason":"broker was unresponsive"},"10760":{"timestamp":1712365573.1381626,"reason":"broker was unresponsive"},"10761":{"timestamp":1712365573.1390309,"reason":"broker was unresponsive"},"10762":{"timestamp":1712365573.1399415,"reason":"broker was unresponsive"},"10763":{"timestamp":1712365573.1408238,"reason":"broker was unresponsive"},"10764":{"timestamp":1712365573.1417074,"reason":"broker was unresponsive"},"10765":{"timestamp":1712365573.1425729,"reason":"broker was unresponsive"},"10766":{"timestamp":1712365573.1434462,"reason":"broker was unresponsive"},"10767":{"timestamp":1712365573.1443226,"reason":"broker was unresponsive"},"10768":{"timestamp":1712365573.1451817,"reason":"broker was unresponsive"},"10769":{"timestamp":1712365573.1460595,"reason":"broker was unresponsive"},"10770":{"timestamp":1712365573.1469145,"reason":"broker was unresponsive"},"10771":{"timestamp":1712365573.1477904,"reason":"broker was unresponsive"},"10772":{"timestamp":1712365573.1486702,"reason":"broker was unresponsive"},"10773":{"timestamp":1712365573.1495631,"reason":"broker was unresponsive"},"10774":{"timestamp":1712365573.1504474,"reason":"broker was unresponsive"},"10775":{"timestamp":1712365573.1513188,"reason":"broker was unresponsive"},"10776":{"timestamp":1712365573.1521673,"reason":"broker was unresponsive"},"10777":{"timestamp":1712365573.1530473,"reason":"broker was unresponsive"},"10778":{"timestamp":1712365573.1539307,"reason":"broker was unresponsive"},"10779":{"timestamp":1712365573.1548109,"reason":"broker was unresponsive"},"10780":{"timestamp":1712365573.1556981,"reason":"broker was unresponsive"},"10781":{"timestamp":1712365573.1565633,"reason":"broker was unresponsive"},"10782":{"timestamp":1712365573.1574452,"reason":"broker was unresponsive"},"10783":{"timestamp":1712365573.1583347,"reason":"broker was unresponsive"},"10784":{"timestamp":1712365573.1592026,"reason":"broker was unresponsive"},"10785":{"timestamp":1712365573.1600885,"reason":"broker was unresponsive"},"10786":{"timestamp":1712365573.1609616,"reason":"broker was unresponsive"},"10787":{"timestamp":1712365573.1618497,"reason":"broker was unresponsive"},"10788":{"timestamp":1712365573.1628067,"reason":"broker was unresponsive"},"10789":{"timestamp":1712365573.1637442,"reason":"broker was unresponsive"},"10790":{"timestamp":1712365573.1646264,"reason":"broker was unresponsive"},"10791":{"timestamp":1712365573.165499,"reason":"broker was unresponsive"},"10792":{"timestamp":1712365573.1663668,"reason":"broker was unresponsive"},"10793":{"timestamp":1712365573.1672223,"reason":"broker was unresponsive"},"10794":{"timestamp":1712365573.1680927,"reason":"broker was unresponsive"},"10795":{"timestamp":1712365573.168962,"reason":"broker was unresponsive"},"10796":{"timestamp":1712365573.1698296,"reason":"broker was unresponsive"},"10797":{"timestamp":1712365573.170702,"reason":"broker was unresponsive"},"10798":{"timestamp":1712365573.1715734,"reason":"broker was unresponsive"},"10799":{"timestamp":1712365573.1724429,"reason":"broker was unresponsive"},"10800":{"timestamp":1712365573.1733344,"reason":"broker was unresponsive"},"10801":{"timestamp":1712365573.174211,"reason":"broker was unresponsive"},"10802":{"timestamp":1712365573.1751013,"reason":"broker was unresponsive"},"10803":{"timestamp":1712365573.1759739,"reason":"broker was unresponsive"},"10804":{"timestamp":1712365573.1768665,"reason":"broker was unresponsive"},"10805":{"timestamp":1712365573.1777604,"reason":"broker was unresponsive"},"10806":{"timestamp":1712365573.1786463,"reason":"broker was unresponsive"},"10807":{"timestamp":1712365573.1795409,"reason":"broker was unresponsive"},"10808":{"timestamp":1712365573.1804161,"reason":"broker was unresponsive"},"10809":{"timestamp":1712365573.1813056,"reason":"broker was unresponsive"},"10810":{"timestamp":1712365573.1821861,"reason":"broker was unresponsive"},"10811":{"timestamp":1712365573.1830876,"reason":"broker was unresponsive"},"10812":{"timestamp":1712365573.1839852,"reason":"broker was unresponsive"},"10813":{"timestamp":1712365573.1848783,"reason":"broker was unresponsive"},"10814":{"timestamp":1712365573.1857576,"reason":"broker was unresponsive"},"10815":{"timestamp":1712365573.1866503,"reason":"broker was unresponsive"},"10816":{"timestamp":1712365573.1875484,"reason":"broker was unresponsive"},"10817":{"timestamp":1712365573.1884432,"reason":"broker was unresponsive"},"10818":{"timestamp":1712365573.1893361,"reason":"broker was unresponsive"},"10819":{"timestamp":1712365573.1902153,"reason":"broker was unresponsive"},"10820":{"timestamp":1712365573.1910923,"reason":"broker was unresponsive"},"10821":{"timestamp":1712365573.1919847,"reason":"broker was unresponsive"},"10822":{"timestamp":1712365573.1928813,"reason":"broker was unresponsive"},"10823":{"timestamp":1712365573.193784,"reason":"broker was unresponsive"},"10824":{"timestamp":1712365573.1946874,"reason":"broker was unresponsive"},"10825":{"timestamp":1712365573.1955686,"reason":"broker was unresponsive"},"10826":{"timestamp":1712365573.1964586,"reason":"broker was unresponsive"},"10827":{"timestamp":1712365573.1973672,"reason":"broker was unresponsive"},"10828":{"timestamp":1712365573.1982546,"reason":"broker was unresponsive"},"10829":{"timestamp":1712365573.1991575,"reason":"broker was unresponsive"},"10830":{"timestamp":1712365573.2000601,"reason":"broker was unresponsive"},"10831":{"timestamp":1712365573.2009485,"reason":"broker was unresponsive"},"10832":{"timestamp":1712365573.2018456,"reason":"broker was unresponsive"},"10833":{"timestamp":1712365573.2027476,"reason":"broker was unresponsive"},"10834":{"timestamp":1712365573.2036524,"reason":"broker was unresponsive"},"10835":{"timestamp":1712365573.2045562,"reason":"broker was unresponsive"},"10836":{"timestamp":1712365573.2054584,"reason":"broker was unresponsive"},"10837":{"timestamp":1712365573.2063508,"reason":"broker was unresponsive"},"10838":{"timestamp":1712365573.2072315,"reason":"broker was unresponsive"},"10839":{"timestamp":1712365573.2081363,"reason":"broker was unresponsive"},"10840":{"timestamp":1712365573.2090437,"reason":"broker was unresponsive"},"10841":{"timestamp":1712365573.2099521,"reason":"broker was unresponsive"},"10842":{"timestamp":1712365573.2108572,"reason":"broker was unresponsive"},"10843":{"timestamp":1712365573.2117457,"reason":"broker was unresponsive"},"10844":{"timestamp":1712365573.2126482,"reason":"broker was unresponsive"},"10845":{"timestamp":1712365573.2135534,"reason":"broker was unresponsive"},"10846":{"timestamp":1712365573.2144616,"reason":"broker was unresponsive"},"10847":{"timestamp":1712365573.2153721,"reason":"broker was unresponsive"},"10848":{"timestamp":1712365573.2162795,"reason":"broker was unresponsive"},"10849":{"timestamp":1712365573.2171559,"reason":"broker was unresponsive"},"10850":{"timestamp":1712365573.2180581,"reason":"broker was unresponsive"},"10851":{"timestamp":1712365573.2189653,"reason":"broker was unresponsive"},"10852":{"timestamp":1712365573.2198789,"reason":"broker was unresponsive"},"10853":{"timestamp":1712365573.220787,"reason":"broker was unresponsive"},"10854":{"timestamp":1712365573.2217011,"reason":"broker was unresponsive"},"10855":{"timestamp":1712365573.2226045,"reason":"broker was unresponsive"},"10856":{"timestamp":1712365573.2235055,"reason":"broker was unresponsive"},"10857":{"timestamp":1712365573.2244158,"reason":"broker was unresponsive"},"10858":{"timestamp":1712365573.2253308,"reason":"broker was unresponsive"},"10859":{"timestamp":1712365573.2262266,"reason":"broker was unresponsive"},"10860":{"timestamp":1712365573.2271376,"reason":"broker was unresponsive"},"10861":{"timestamp":1712365573.228049,"reason":"broker was unresponsive"},"10862":{"timestamp":1712365573.2289436,"reason":"broker was unresponsive"},"10863":{"timestamp":1712365573.229857,"reason":"broker was unresponsive"},"10864":{"timestamp":1712365573.230773,"reason":"broker was unresponsive"},"10865":{"timestamp":1712365573.2316828,"reason":"broker was unresponsive"},"10866":{"timestamp":1712365573.2325985,"reason":"broker was unresponsive"},"10867":{"timestamp":1712365573.2335165,"reason":"broker was unresponsive"},"10868":{"timestamp":1712365573.2344093,"reason":"broker was unresponsive"},"10869":{"timestamp":1712365573.235323,"reason":"broker was unresponsive"},"10870":{"timestamp":1712365573.2362199,"reason":"broker was unresponsive"},"10872":{"timestamp":1712347934.0538592,"reason":"broker was unresponsive"},"10873":{"timestamp":1712365573.2371399,"reason":"broker was unresponsive"},"10874":{"timestamp":1712365573.2380564,"reason":"broker was unresponsive"},"10875":{"timestamp":1712365573.238975,"reason":"broker was unresponsive"},"10876":{"timestamp":1712365573.2399347,"reason":"broker was unresponsive"},"10877":{"timestamp":1712365573.2408431,"reason":"broker was unresponsive"},"10878":{"timestamp":1712365573.2417583,"reason":"broker was unresponsive"},"10879":{"timestamp":1712365573.2426755,"reason":"broker was unresponsive"},"10880":{"timestamp":1712365573.2435966,"reason":"broker was unresponsive"},"10881":{"timestamp":1712365573.2445195,"reason":"broker was unresponsive"},"10882":{"timestamp":1712365573.2454352,"reason":"broker was unresponsive"},"10883":{"timestamp":1712365573.246346,"reason":"broker was unresponsive"},"10884":{"timestamp":1712365573.2472394,"reason":"broker was unresponsive"},"10885":{"timestamp":1712365573.2481606,"reason":"broker was unresponsive"},"10886":{"timestamp":1712365573.2490864,"reason":"broker was unresponsive"},"10887":{"timestamp":1712365573.250006,"reason":"broker was unresponsive"},"10888":{"timestamp":1712365573.2509241,"reason":"broker was unresponsive"},"10889":{"timestamp":1712365573.2518539,"reason":"broker was unresponsive"},"10890":{"timestamp":1712365573.2527649,"reason":"broker was unresponsive"},"10891":{"timestamp":1712365573.2536905,"reason":"broker was unresponsive"},"10892":{"timestamp":1712365573.2546239,"reason":"broker was unresponsive"},"10893":{"timestamp":1712365573.2555518,"reason":"broker was unresponsive"},"10894":{"timestamp":1712365573.2564771,"reason":"broker was unresponsive"},"10895":{"timestamp":1712365573.2574019,"reason":"broker was unresponsive"},"10896":{"timestamp":1712365573.2583287,"reason":"broker was unresponsive"},"10897":{"timestamp":1712365573.2592206,"reason":"broker was unresponsive"},"10898":{"timestamp":1712365573.2601435,"reason":"broker was unresponsive"},"10899":{"timestamp":1712365573.2610712,"reason":"broker was unresponsive"},"10900":{"timestamp":1712365573.2619967,"reason":"broker was unresponsive"},"10901":{"timestamp":1712365573.2631607,"reason":"broker was unresponsive"},"10902":{"timestamp":1712365573.2640898,"reason":"broker was unresponsive"},"10903":{"timestamp":1712365573.2650228,"reason":"broker was unresponsive"},"10904":{"timestamp":1712365573.2659504,"reason":"broker was unresponsive"},"10905":{"timestamp":1712365573.2668667,"reason":"broker was unresponsive"},"10906":{"timestamp":1712365573.2677903,"reason":"broker was unresponsive"},"10907":{"timestamp":1712365573.2687852,"reason":"broker was unresponsive"},"10908":{"timestamp":1712365573.2697341,"reason":"broker was unresponsive"},"10909":{"timestamp":1712365573.2706664,"reason":"broker was unresponsive"},"10910":{"timestamp":1712365573.2715962,"reason":"broker was unresponsive"},"10913":{"timestamp":1712365573.2725284,"reason":"broker was unresponsive"},"10914":{"timestamp":1712365573.2734654,"reason":"broker was unresponsive"},"10915":{"timestamp":1712365573.2743835,"reason":"broker was unresponsive"},"10916":{"timestamp":1712365573.2753191,"reason":"broker was unresponsive"},"10917":{"timestamp":1712365573.2762341,"reason":"broker was unresponsive"},"10918":{"timestamp":1712365573.2771664,"reason":"broker was unresponsive"},"10919":{"timestamp":1712365573.2780974,"reason":"broker was unresponsive"},"10920":{"timestamp":1712365573.2790387,"reason":"broker was unresponsive"},"10921":{"timestamp":1712365573.2799811,"reason":"broker was unresponsive"},"10922":{"timestamp":1712365573.2809002,"reason":"broker was unresponsive"},"10923":{"timestamp":1712365573.2818317,"reason":"broker was unresponsive"},"10924":{"timestamp":1712365573.2827816,"reason":"broker was unresponsive"},"10925":{"timestamp":1712365573.2837226,"reason":"broker was unresponsive"},"10926":{"timestamp":1712365573.2846601,"reason":"broker was unresponsive"},"10927":{"timestamp":1712365573.2855916,"reason":"broker was unresponsive"},"10928":{"timestamp":1712365573.2865283,"reason":"broker was unresponsive"},"10929":{"timestamp":1712365573.287472,"reason":"broker was unresponsive"},"10930":{"timestamp":1712365573.2883973,"reason":"broker was unresponsive"},"10931":{"timestamp":1712365573.2893355,"reason":"broker was unresponsive"},"10932":{"timestamp":1712365573.2902584,"reason":"broker was unresponsive"},"10933":{"timestamp":1712365573.2911959,"reason":"broker was unresponsive"},"10934":{"timestamp":1712365573.2921329,"reason":"broker was unresponsive"},"10935":{"timestamp":1712365573.2930727,"reason":"broker was unresponsive"},"10936":{"timestamp":1712365573.2940137,"reason":"broker was unresponsive"},"10937":{"timestamp":1712365573.2949338,"reason":"broker was unresponsive"},"10938":{"timestamp":1712365573.2958746,"reason":"broker was unresponsive"},"10939":{"timestamp":1712365573.296819,"reason":"broker was unresponsive"},"10940":{"timestamp":1712365573.2977691,"reason":"broker was unresponsive"},"10941":{"timestamp":1712365573.2987108,"reason":"broker was unresponsive"},"10942":{"timestamp":1712365573.2996583,"reason":"broker was unresponsive"},"10944":{"timestamp":1712347508.0716112,"reason":"broker was unresponsive"},"10945":{"timestamp":1712365573.3005991,"reason":"broker was unresponsive"},"10946":{"timestamp":1712365573.3015473,"reason":"broker was unresponsive"},"10947":{"timestamp":1712365573.3024745,"reason":"broker was unresponsive"},"10948":{"timestamp":1712365573.3034151,"reason":"broker was unresponsive"},"10949":{"timestamp":1712365573.3043621,"reason":"broker was unresponsive"},"10950":{"timestamp":1712365573.3053155,"reason":"broker was unresponsive"},"10951":{"timestamp":1712365573.3062425,"reason":"broker was unresponsive"},"10952":{"timestamp":1712365573.3071826,"reason":"broker was unresponsive"},"10953":{"timestamp":1712365573.3081312,"reason":"broker was unresponsive"},"10954":{"timestamp":1712365573.3090641,"reason":"broker was unresponsive"},"10955":{"timestamp":1712365573.310003,"reason":"broker was unresponsive"},"10956":{"timestamp":1712365573.3109515,"reason":"broker was unresponsive"},"10957":{"timestamp":1712365573.3118947,"reason":"broker was unresponsive"},"10958":{"timestamp":1712365573.3128393,"reason":"broker was unresponsive"},"10959":{"timestamp":1712365573.3137898,"reason":"broker was unresponsive"},"10960":{"timestamp":1712365573.3147397,"reason":"broker was unresponsive"},"10961":{"timestamp":1712365573.3156886,"reason":"broker was unresponsive"},"10962":{"timestamp":1712365573.3166449,"reason":"broker was unresponsive"},"10963":{"timestamp":1712365573.317574,"reason":"broker was unresponsive"},"10964":{"timestamp":1712365573.3185229,"reason":"broker was unresponsive"},"10965":{"timestamp":1712365573.319474,"reason":"broker was unresponsive"},"10966":{"timestamp":1712365573.3204255,"reason":"broker was unresponsive"},"10967":{"timestamp":1712365573.3213785,"reason":"broker was unresponsive"},"10968":{"timestamp":1712365573.32233,"reason":"broker was unresponsive"},"10969":{"timestamp":1712365573.3232865,"reason":"broker was unresponsive"},"10970":{"timestamp":1712365573.324223,"reason":"broker was unresponsive"},"10971":{"timestamp":1712365573.3251567,"reason":"broker was unresponsive"},"10972":{"timestamp":1712365573.3261015,"reason":"broker was unresponsive"},"10973":{"timestamp":1712365573.3270519,"reason":"broker was unresponsive"},"10974":{"timestamp":1712365573.3280025,"reason":"broker was unresponsive"},"10975":{"timestamp":1712365573.3289611,"reason":"broker was unresponsive"},"10976":{"timestamp":1712365573.3299177,"reason":"broker was unresponsive"},"10977":{"timestamp":1712365573.3308704,"reason":"broker was unresponsive"},"10978":{"timestamp":1712365573.3318224,"reason":"broker was unresponsive"},"10979":{"timestamp":1712365573.3327558,"reason":"broker was unresponsive"},"10980":{"timestamp":1712365573.3337064,"reason":"broker was unresponsive"},"10981":{"timestamp":1712365573.3346622,"reason":"broker was unresponsive"},"10982":{"timestamp":1712365573.3356183,"reason":"broker was unresponsive"},"10983":{"timestamp":1712365573.3365712,"reason":"broker was unresponsive"},"10984":{"timestamp":1712365573.337528,"reason":"broker was unresponsive"},"10985":{"timestamp":1712365573.3384824,"reason":"broker was unresponsive"},"10986":{"timestamp":1712365573.339443,"reason":"broker was unresponsive"},"10987":{"timestamp":1712365573.3404334,"reason":"broker was unresponsive"},"10988":{"timestamp":1712365573.3413815,"reason":"broker was unresponsive"},"10989":{"timestamp":1712365573.3423364,"reason":"broker was unresponsive"},"10990":{"timestamp":1712365573.3432975,"reason":"broker was unresponsive"},"10991":{"timestamp":1712365573.3442407,"reason":"broker was unresponsive"},"10992":{"timestamp":1712365573.3451977,"reason":"broker was unresponsive"},"10993":{"timestamp":1712365573.3461599,"reason":"broker was unresponsive"},"10994":{"timestamp":1712365573.3471215,"reason":"broker was unresponsive"},"10995":{"timestamp":1712365573.3480818,"reason":"broker was unresponsive"},"10996":{"timestamp":1712365573.3490312,"reason":"broker was unresponsive"},"10997":{"timestamp":1712365573.3499928,"reason":"broker was unresponsive"},"10998":{"timestamp":1712365573.3509524,"reason":"broker was unresponsive"},"11000":{"timestamp":1712326495.0890651,"reason":"broker was unresponsive"},"11001":{"timestamp":1712365573.3519187,"reason":"broker was unresponsive"},"11002":{"timestamp":1712365573.3528781,"reason":"broker was unresponsive"},"11003":{"timestamp":1712365573.3538444,"reason":"broker was unresponsive"},"11004":{"timestamp":1712365573.3548095,"reason":"broker was unresponsive"},"11005":{"timestamp":1712365573.3557754,"reason":"broker was unresponsive"},"11006":{"timestamp":1712365573.3567405,"reason":"broker was unresponsive"},"11007":{"timestamp":1712365573.3577011,"reason":"broker was unresponsive"},"11008":{"timestamp":1712365573.3586533,"reason":"broker was unresponsive"},"11009":{"timestamp":1712365573.3596182,"reason":"broker was unresponsive"},"11010":{"timestamp":1712365573.3605893,"reason":"broker was unresponsive"},"11011":{"timestamp":1712365573.3615584,"reason":"broker was unresponsive"},"11012":{"timestamp":1712365573.3625238,"reason":"broker was unresponsive"},"11013":{"timestamp":1712365573.3634937,"reason":"broker was unresponsive"},"11014":{"timestamp":1712365573.3644629,"reason":"broker was unresponsive"},"11015":{"timestamp":1712365573.3654284,"reason":"broker was unresponsive"},"11016":{"timestamp":1712365573.3663948,"reason":"broker was unresponsive"},"11017":{"timestamp":1712365573.3673639,"reason":"broker was unresponsive"},"11018":{"timestamp":1712365573.36834,"reason":"broker was unresponsive"},"11019":{"timestamp":1712365573.3693151,"reason":"broker was unresponsive"},"11020":{"timestamp":1712365573.3702934,"reason":"broker was unresponsive"},"11021":{"timestamp":1712365573.371249,"reason":"broker was unresponsive"},"11022":{"timestamp":1712365573.3722186,"reason":"broker was unresponsive"},"11023":{"timestamp":1712365573.3731875,"reason":"broker was unresponsive"},"11024":{"timestamp":1712365573.3741581,"reason":"broker was unresponsive"},"11025":{"timestamp":1712365573.3751321,"reason":"broker was unresponsive"},"11026":{"timestamp":1712365573.3760908,"reason":"broker was unresponsive"},"11027":{"timestamp":1712365573.377053,"reason":"broker was unresponsive"},"11028":{"timestamp":1712365573.3780262,"reason":"broker was unresponsive"},"11029":{"timestamp":1712365573.3789978,"reason":"broker was unresponsive"},"11030":{"timestamp":1712365573.3799777,"reason":"broker was unresponsive"},"11031":{"timestamp":1712365573.3809519,"reason":"broker was unresponsive"},"11032":{"timestamp":1712365573.3819268,"reason":"broker was unresponsive"},"11033":{"timestamp":1712365573.3829026,"reason":"broker was unresponsive"},"11034":{"timestamp":1712365573.3838804,"reason":"broker was unresponsive"},"11035":{"timestamp":1712365573.3848588,"reason":"broker was unresponsive"},"11036":{"timestamp":1712365573.3858273,"reason":"broker was unresponsive"},"11037":{"timestamp":1712365573.38679,"reason":"broker was unresponsive"},"11038":{"timestamp":1712365573.3877676,"reason":"broker was unresponsive"},"11039":{"timestamp":1712365573.3887463,"reason":"broker was unresponsive"},"11040":{"timestamp":1712365573.3897262,"reason":"broker was unresponsive"},"11041":{"timestamp":1712365573.3907056,"reason":"broker was unresponsive"},"11042":{"timestamp":1712365573.3916833,"reason":"broker was unresponsive"},"11043":{"timestamp":1712365573.3926568,"reason":"broker was unresponsive"},"11044":{"timestamp":1712365573.3936388,"reason":"broker was unresponsive"},"11045":{"timestamp":1712365573.3946178,"reason":"broker was unresponsive"},"11046":{"timestamp":1712365573.3955972,"reason":"broker was unresponsive"},"11047":{"timestamp":1712365573.396558,"reason":"broker was unresponsive"},"11048":{"timestamp":1712365573.3975372,"reason":"broker was unresponsive"},"11049":{"timestamp":1712365573.3985217,"reason":"broker was unresponsive"},"11050":{"timestamp":1712365573.3995016,"reason":"broker was unresponsive"},"11051":{"timestamp":1712365573.4004865,"reason":"broker was unresponsive"},"11052":{"timestamp":1712365573.4014704,"reason":"broker was unresponsive"},"11053":{"timestamp":1712365573.4024549,"reason":"broker was unresponsive"},"11054":{"timestamp":1712365573.4034369,"reason":"broker was unresponsive"},"11055":{"timestamp":1712365573.4044204,"reason":"broker was unresponsive"},"11056":{"timestamp":1712365573.4054,"reason":"broker was unresponsive"},"11057":{"timestamp":1712365573.4063787,"reason":"broker was unresponsive"},"11058":{"timestamp":1712365573.4073567,"reason":"broker was unresponsive"},"11059":{"timestamp":1712365573.4083426,"reason":"broker was unresponsive"},"11060":{"timestamp":1712365573.4093292,"reason":"broker was unresponsive"},"11061":{"timestamp":1712365573.4103243,"reason":"broker was unresponsive"},"11062":{"timestamp":1712365573.4113164,"reason":"broker was unresponsive"},"11063":{"timestamp":1712365573.4123027,"reason":"broker was unresponsive"},"11064":{"timestamp":1712365573.4132836,"reason":"broker was unresponsive"},"11065":{"timestamp":1712365573.4142513,"reason":"broker was unresponsive"},"11066":{"timestamp":1712365573.4152372,"reason":"broker was unresponsive"},"11067":{"timestamp":1712365573.4162254,"reason":"broker was unresponsive"},"11068":{"timestamp":1712365573.417208,"reason":"broker was unresponsive"},"11069":{"timestamp":1712365573.4181757,"reason":"broker was unresponsive"},"11070":{"timestamp":1712365573.4191716,"reason":"broker was unresponsive"},"11071":{"timestamp":1712365573.4201617,"reason":"broker was unresponsive"},"11072":{"timestamp":1712365573.421154,"reason":"broker was unresponsive"},"11073":{"timestamp":1712365573.4221473,"reason":"broker was unresponsive"},"11074":{"timestamp":1712365573.4231369,"reason":"broker was unresponsive"},"11075":{"timestamp":1712365573.4241257,"reason":"broker was unresponsive"},"11076":{"timestamp":1712365573.4251156,"reason":"broker was unresponsive"},"11077":{"timestamp":1712365573.4261086,"reason":"broker was unresponsive"},"11078":{"timestamp":1712365573.4271007,"reason":"broker was unresponsive"},"11079":{"timestamp":1712365573.4280951,"reason":"broker was unresponsive"},"11080":{"timestamp":1712365573.4290886,"reason":"broker was unresponsive"},"11081":{"timestamp":1712365573.4300656,"reason":"broker was unresponsive"},"11082":{"timestamp":1712365573.4310572,"reason":"broker was unresponsive"},"11083":{"timestamp":1712365573.4320664,"reason":"broker was unresponsive"},"11084":{"timestamp":1712365573.4330556,"reason":"broker was unresponsive"},"11085":{"timestamp":1712365573.4340529,"reason":"broker was unresponsive"},"11086":{"timestamp":1712365573.435045,"reason":"broker was unresponsive"},"11087":{"timestamp":1712365573.4360409,"reason":"broker was unresponsive"},"11088":{"timestamp":1712365573.4370358,"reason":"broker was unresponsive"},"11089":{"timestamp":1712365573.4380314,"reason":"broker was unresponsive"},"11090":{"timestamp":1712365573.4390271,"reason":"broker was unresponsive"},"11091":{"timestamp":1712365573.440032,"reason":"broker was unresponsive"},"11092":{"timestamp":1712365573.4410479,"reason":"broker was unresponsive"},"11093":{"timestamp":1712365573.4420466,"reason":"broker was unresponsive"},"11094":{"timestamp":1712365573.443027,"reason":"broker was unresponsive"},"11095":{"timestamp":1712365573.4440267,"reason":"broker was unresponsive"},"11096":{"timestamp":1712365573.4450247,"reason":"broker was unresponsive"},"11097":{"timestamp":1712365573.4460225,"reason":"broker was unresponsive"},"11098":{"timestamp":1712365573.4470212,"reason":"broker was unresponsive"},"11099":{"timestamp":1712365573.4480162,"reason":"broker was unresponsive"},"11100":{"timestamp":1712365573.4490149,"reason":"broker was unresponsive"},"11101":{"timestamp":1712365573.4500136,"reason":"broker was unresponsive"},"11102":{"timestamp":1712365573.4510171,"reason":"broker was unresponsive"},"11103":{"timestamp":1712365573.4520192,"reason":"broker was unresponsive"},"11104":{"timestamp":1712365573.4530184,"reason":"broker was unresponsive"},"11105":{"timestamp":1712365573.4540186,"reason":"broker was unresponsive"},"11106":{"timestamp":1712365573.4550183,"reason":"broker was unresponsive"},"11107":{"timestamp":1712365573.4560068,"reason":"broker was unresponsive"},"11108":{"timestamp":1712365573.4570065,"reason":"broker was unresponsive"},"11109":{"timestamp":1712365573.4580102,"reason":"broker was unresponsive"},"11110":{"timestamp":1712365573.4590151,"reason":"broker was unresponsive"},"11111":{"timestamp":1712365573.4600148,"reason":"broker was unresponsive"},"11112":{"timestamp":1712365573.46102,"reason":"broker was unresponsive"},"11113":{"timestamp":1712365573.4620256,"reason":"broker was unresponsive"},"11114":{"timestamp":1712365573.4630289,"reason":"broker was unresponsive"},"11117":{"timestamp":1712365573.4640336,"reason":"broker was unresponsive"},"11118":{"timestamp":1712365573.4650395,"reason":"broker was unresponsive"},"11119":{"timestamp":1712365573.4660468,"reason":"broker was unresponsive"},"11120":{"timestamp":1712365573.4670546,"reason":"broker was unresponsive"},"11121":{"timestamp":1712365573.4680631,"reason":"broker was unresponsive"},"11122":{"timestamp":1712365573.4690628,"reason":"broker was unresponsive"},"11123":{"timestamp":1712365573.4700487,"reason":"broker was unresponsive"},"11124":{"timestamp":1712365573.4710498,"reason":"broker was unresponsive"},"11125":{"timestamp":1712365573.4720569,"reason":"broker was unresponsive"},"11126":{"timestamp":1712365573.4730661,"reason":"broker was unresponsive"},"11127":{"timestamp":1712365573.4740667,"reason":"broker was unresponsive"},"11128":{"timestamp":1712365573.4750795,"reason":"broker was unresponsive"},"11129":{"timestamp":1712365573.4760923,"reason":"broker was unresponsive"},"11130":{"timestamp":1712365573.4771039,"reason":"broker was unresponsive"},"11131":{"timestamp":1712365573.4781134,"reason":"broker was unresponsive"},"11132":{"timestamp":1712365573.4791224,"reason":"broker was unresponsive"},"11133":{"timestamp":1712365573.4801333,"reason":"broker was unresponsive"},"11134":{"timestamp":1712079046.9974139,"reason":"broker was unresponsive"},"11135":{"timestamp":1712365573.4811442,"reason":"broker was unresponsive"},"11136":{"timestamp":1712365573.4821565,"reason":"broker was unresponsive"},"11137":{"timestamp":1712365573.4831724,"reason":"broker was unresponsive"},"11138":{"timestamp":1712365573.484189,"reason":"broker was unresponsive"},"11139":{"timestamp":1712365573.4852064,"reason":"broker was unresponsive"},"11140":{"timestamp":1712079046.997932,"reason":"broker was unresponsive"},"11141":{"timestamp":1712365573.4861984,"reason":"broker was unresponsive"},"11142":{"timestamp":1712365573.4872081,"reason":"broker was unresponsive"},"11143":{"timestamp":1712365573.4882188,"reason":"broker was unresponsive"},"11144":{"timestamp":1712365573.4892297,"reason":"broker was unresponsive"},"11145":{"timestamp":1712365573.490242,"reason":"broker was unresponsive"},"11146":{"timestamp":1712365573.4912558,"reason":"broker was unresponsive"},"11147":{"timestamp":1712365573.4922874,"reason":"broker was unresponsive"},"11148":{"timestamp":1712365573.4933014,"reason":"broker was unresponsive"},"11149":{"timestamp":1712365573.4943211,"reason":"broker was unresponsive"},"11150":{"timestamp":1712365573.4953368,"reason":"broker was unresponsive"},"11151":{"timestamp":1712365573.4963517,"reason":"broker was unresponsive"},"11152":{"timestamp":1712365573.4973645,"reason":"broker was unresponsive"},"11153":{"timestamp":1712365573.4983842,"reason":"broker was unresponsive"},"11154":{"timestamp":1712365573.4994006,"reason":"broker was unresponsive"},"11155":{"timestamp":1712365573.5004199,"reason":"broker was unresponsive"},"11156":{"timestamp":1712365573.501442,"reason":"broker was unresponsive"},"11157":{"timestamp":1712365573.5024519,"reason":"broker was unresponsive"},"11158":{"timestamp":1712365573.5034575,"reason":"broker was unresponsive"},"11159":{"timestamp":1712365573.5044773,"reason":"broker was unresponsive"},"11160":{"timestamp":1712365573.5054955,"reason":"broker was unresponsive"},"11161":{"timestamp":1712365573.5065186,"reason":"broker was unresponsive"},"11162":{"timestamp":1712365573.5075397,"reason":"broker was unresponsive"},"11163":{"timestamp":1712365573.5085614,"reason":"broker was unresponsive"},"11164":{"timestamp":1712365573.5095837,"reason":"broker was unresponsive"},"11165":{"timestamp":1712365573.5106013,"reason":"broker was unresponsive"},"11166":{"timestamp":1712365573.5116234,"reason":"broker was unresponsive"},"11167":{"timestamp":1712365573.5126479,"reason":"broker was unresponsive"},"11168":{"timestamp":1712365573.5136647,"reason":"broker was unresponsive"},"11169":{"timestamp":1712365573.5146797,"reason":"broker was unresponsive"},"11170":{"timestamp":1712365573.5157032,"reason":"broker was unresponsive"},"11171":{"timestamp":1712365573.5167339,"reason":"broker was unresponsive"},"11172":{"timestamp":1712365573.5177591,"reason":"broker was unresponsive"},"11173":{"timestamp":1712365573.5187826,"reason":"broker was unresponsive"},"11174":{"timestamp":1712365573.5198059,"reason":"broker was unresponsive"},"11175":{"timestamp":1712365573.5208292,"reason":"broker was unresponsive"},"11176":{"timestamp":1712365573.5218394,"reason":"broker was unresponsive"},"11177":{"timestamp":1712365573.5228601,"reason":"broker was unresponsive"},"11178":{"timestamp":1712365573.5238829,"reason":"broker was unresponsive"},"11179":{"timestamp":1712365573.5249083,"reason":"broker was unresponsive"},"11180":{"timestamp":1712365573.5259378,"reason":"broker was unresponsive"},"11181":{"timestamp":1712365573.5269723,"reason":"broker was unresponsive"},"11182":{"timestamp":1712365573.5280004,"reason":"broker was unresponsive"},"11183":{"timestamp":1712365573.529036,"reason":"broker was unresponsive"},"11184":{"timestamp":1712365573.5300672,"reason":"broker was unresponsive"},"11185":{"timestamp":1712365573.5311015,"reason":"broker was unresponsive"},"11186":{"timestamp":1712365573.5321302,"reason":"broker was unresponsive"},"11187":{"timestamp":1712365573.5331612,"reason":"broker was unresponsive"},"11188":{"timestamp":1712365573.5341883,"reason":"broker was unresponsive"},"11189":{"timestamp":1712365573.5352159,"reason":"broker was unresponsive"},"11190":{"timestamp":1712365573.5362518,"reason":"broker was unresponsive"},"11191":{"timestamp":1712365573.5372992,"reason":"broker was unresponsive"},"11192":{"timestamp":1712365573.538331,"reason":"broker was unresponsive"},"11193":{"timestamp":1712365573.5393682,"reason":"broker was unresponsive"},"11194":{"timestamp":1712365573.5404055,"reason":"broker was unresponsive"},"11195":{"timestamp":1712365573.5414383,"reason":"broker was unresponsive"},"11196":{"timestamp":1712365573.5424705,"reason":"broker was unresponsive"},"11197":{"timestamp":1712365573.5434897,"reason":"broker was unresponsive"},"11198":{"timestamp":1712365573.5445127,"reason":"broker was unresponsive"},"11199":{"timestamp":1712365573.5455496,"reason":"broker was unresponsive"},"11200":{"timestamp":1712365573.5465832,"reason":"broker was unresponsive"},"11201":{"timestamp":1712365573.5476203,"reason":"broker was unresponsive"},"11202":{"timestamp":1712365573.5486538,"reason":"broker was unresponsive"},"11203":{"timestamp":1712365573.5496891,"reason":"broker was unresponsive"},"11204":{"timestamp":1712365573.5507255,"reason":"broker was unresponsive"},"11205":{"timestamp":1712365573.5517662,"reason":"broker was unresponsive"},"11206":{"timestamp":1712365573.5528021,"reason":"broker was unresponsive"},"11207":{"timestamp":1712365573.5538433,"reason":"broker was unresponsive"},"11208":{"timestamp":1712365573.5548754,"reason":"broker was unresponsive"},"11209":{"timestamp":1712365573.5559142,"reason":"broker was unresponsive"},"11210":{"timestamp":1712365573.5569506,"reason":"broker was unresponsive"},"11211":{"timestamp":1712365573.5579844,"reason":"broker was unresponsive"},"11212":{"timestamp":1712365573.5590277,"reason":"broker was unresponsive"},"11213":{"timestamp":1712365573.5600691,"reason":"broker was unresponsive"},"11214":{"timestamp":1712365573.5611057,"reason":"broker was unresponsive"},"11215":{"timestamp":1712365573.5621431,"reason":"broker was unresponsive"},"11216":{"timestamp":1712365573.5631819,"reason":"broker was unresponsive"},"11217":{"timestamp":1712365573.5642264,"reason":"broker was unresponsive"},"11218":{"timestamp":1712365573.5652785,"reason":"broker was unresponsive"},"11219":{"timestamp":1712365573.5663218,"reason":"broker was unresponsive"},"11220":{"timestamp":1712365573.5673594,"reason":"broker was unresponsive"},"11221":{"timestamp":1712365573.5684013,"reason":"broker was unresponsive"},"11222":{"timestamp":1712365573.5694482,"reason":"broker was unresponsive"},"11223":{"timestamp":1712365573.5704722,"reason":"broker was unresponsive"},"11224":{"timestamp":1712365573.5715098,"reason":"broker was unresponsive"},"11225":{"timestamp":1712365573.5725524,"reason":"broker was unresponsive"},"11227":{"timestamp":1712365573.573597,"reason":"broker was unresponsive"},"11228":{"timestamp":1712365573.5746396,"reason":"broker was unresponsive"},"11229":{"timestamp":1712365573.5756869,"reason":"broker was unresponsive"},"11230":{"timestamp":1712365573.5767438,"reason":"broker was unresponsive"},"11231":{"timestamp":1712365573.5777903,"reason":"broker was unresponsive"},"11232":{"timestamp":1712365573.5788343,"reason":"broker was unresponsive"},"11234":{"timestamp":1712365573.5798798,"reason":"broker was unresponsive"},"11237":{"timestamp":1712365573.5809278,"reason":"broker was unresponsive"},"11238":{"timestamp":1712365573.5819693,"reason":"broker was unresponsive"},"11239":{"timestamp":1712365573.5830324,"reason":"broker was unresponsive"},"11240":{"timestamp":1712365573.5841503,"reason":"broker was unresponsive"},"11241":{"timestamp":1712365573.5852818,"reason":"broker was unresponsive"},"11242":{"timestamp":1712365573.5864182,"reason":"broker was unresponsive"},"11243":{"timestamp":1712365573.5875175,"reason":"broker was unresponsive"},"11244":{"timestamp":1712365573.5885744,"reason":"broker was unresponsive"},"11245":{"timestamp":1712365573.5896349,"reason":"broker was unresponsive"},"11246":{"timestamp":1712365573.5907195,"reason":"broker was unresponsive"},"11248":{"timestamp":1712365573.591805,"reason":"broker was unresponsive"},"11250":{"timestamp":1712365573.5928757,"reason":"broker was unresponsive"},"11251":{"timestamp":1712365573.5939326,"reason":"broker was unresponsive"},"11252":{"timestamp":1712365573.5949852,"reason":"broker was unresponsive"},"11253":{"timestamp":1712365573.5960474,"reason":"broker was unresponsive"},"11254":{"timestamp":1712365573.5971084,"reason":"broker was unresponsive"},"11255":{"timestamp":1712365573.5981708,"reason":"broker was unresponsive"},"11256":{"timestamp":1712365573.5992353,"reason":"broker was unresponsive"},"11257":{"timestamp":1712365573.6003189,"reason":"broker was unresponsive"},"11258":{"timestamp":1712365573.6013811,"reason":"broker was unresponsive"},"11259":{"timestamp":1712365573.6024418,"reason":"broker was unresponsive"},"11260":{"timestamp":1712365573.6035063,"reason":"broker was unresponsive"},"11261":{"timestamp":1712365573.6045749,"reason":"broker was unresponsive"},"11262":{"timestamp":1712365573.6056366,"reason":"broker was unresponsive"},"11263":{"timestamp":1712365573.6066999,"reason":"broker was unresponsive"},"11264":{"timestamp":1712365573.6077602,"reason":"broker was unresponsive"},"11265":{"timestamp":1712365573.6088283,"reason":"broker was unresponsive"},"11266":{"timestamp":1712365573.6098914,"reason":"broker was unresponsive"},"11267":{"timestamp":1712365573.6109536,"reason":"broker was unresponsive"},"11268":{"timestamp":1712365573.6120207,"reason":"broker was unresponsive"},"11269":{"timestamp":1712365573.6130877,"reason":"broker was unresponsive"},"11270":{"timestamp":1712365573.6141493,"reason":"broker was unresponsive"},"11271":{"timestamp":1712365573.615212,"reason":"broker was unresponsive"},"11272":{"timestamp":1712365573.6162913,"reason":"broker was unresponsive"},"11273":{"timestamp":1712365573.6173549,"reason":"broker was unresponsive"},"11274":{"timestamp":1712365573.6184206,"reason":"broker was unresponsive"},"11275":{"timestamp":1712365573.6194868,"reason":"broker was unresponsive"},"11276":{"timestamp":1712365573.6205513,"reason":"broker was unresponsive"},"11277":{"timestamp":1712365573.6216207,"reason":"broker was unresponsive"},"11278":{"timestamp":1712365573.6226838,"reason":"broker was unresponsive"},"11279":{"timestamp":1712365573.6237607,"reason":"broker was unresponsive"},"11280":{"timestamp":1712365573.6248219,"reason":"broker was unresponsive"},"11281":{"timestamp":1712365573.6258876,"reason":"broker was unresponsive"},"11282":{"timestamp":1712365573.6269495,"reason":"broker was unresponsive"},"11283":{"timestamp":1712365573.6280189,"reason":"broker was unresponsive"},"11284":{"timestamp":1712365573.6291003,"reason":"broker was unresponsive"},"11285":{"timestamp":1712365573.6301715,"reason":"broker was unresponsive"},"11286":{"timestamp":1712365573.6312401,"reason":"broker was unresponsive"},"11287":{"timestamp":1712365573.6323245,"reason":"broker was unresponsive"},"11288":{"timestamp":1712365573.6333959,"reason":"broker was unresponsive"},"11289":{"timestamp":1712365573.6344633,"reason":"broker was unresponsive"},"11290":{"timestamp":1712365573.6355662,"reason":"broker was unresponsive"},"11291":{"timestamp":1712365573.6366439,"reason":"broker was unresponsive"},"11292":{"timestamp":1712365573.6377144,"reason":"broker was unresponsive"},"11293":{"timestamp":1712365573.6387858,"reason":"broker was unresponsive"},"11294":{"timestamp":1712365573.6400592,"reason":"broker was unresponsive"},"11295":{"timestamp":1712365573.6411662,"reason":"broker was unresponsive"},"11296":{"timestamp":1712365573.6422822,"reason":"broker was unresponsive"},"11297":{"timestamp":1712365573.6433797,"reason":"broker was unresponsive"},"11298":{"timestamp":1712365573.6444609,"reason":"broker was unresponsive"},"11299":{"timestamp":1712365573.6455371,"reason":"broker was unresponsive"},"11300":{"timestamp":1712365573.6466143,"reason":"broker was unresponsive"},"11301":{"timestamp":1712365573.6476901,"reason":"broker was unresponsive"},"11302":{"timestamp":1712365573.6487713,"reason":"broker was unresponsive"},"11303":{"timestamp":1712365573.6498525,"reason":"broker was unresponsive"},"11304":{"timestamp":1712365573.6509335,"reason":"broker was unresponsive"},"11305":{"timestamp":1712365573.6520147,"reason":"broker was unresponsive"},"11306":{"timestamp":1712365573.6530995,"reason":"broker was unresponsive"},"11307":{"timestamp":1712365573.6541793,"reason":"broker was unresponsive"},"11308":{"timestamp":1712365573.6552606,"reason":"broker was unresponsive"},"11309":{"timestamp":1712365573.6563673,"reason":"broker was unresponsive"},"11310":{"timestamp":1712365573.6574485,"reason":"broker was unresponsive"},"11311":{"timestamp":1712365573.6585312,"reason":"broker was unresponsive"},"11312":{"timestamp":1712365573.6596072,"reason":"broker was unresponsive"},"11313":{"timestamp":1712365573.6606891,"reason":"broker was unresponsive"},"11314":{"timestamp":1712365573.6617696,"reason":"broker was unresponsive"},"11315":{"timestamp":1712365573.6628633,"reason":"broker was unresponsive"},"11316":{"timestamp":1712365573.6639454,"reason":"broker was unresponsive"},"11317":{"timestamp":1712365573.6650267,"reason":"broker was unresponsive"},"11318":{"timestamp":1712365573.6661057,"reason":"broker was unresponsive"},"11319":{"timestamp":1712365573.6671877,"reason":"broker was unresponsive"},"11320":{"timestamp":1712365573.6682887,"reason":"broker was unresponsive"},"11321":{"timestamp":1712365573.6693714,"reason":"broker was unresponsive"},"11322":{"timestamp":1712365573.6704545,"reason":"broker was unresponsive"},"11323":{"timestamp":1712365573.6715362,"reason":"broker was unresponsive"},"11324":{"timestamp":1712365573.6726227,"reason":"broker was unresponsive"},"11325":{"timestamp":1712365573.6737058,"reason":"broker was unresponsive"},"11326":{"timestamp":1712365573.6747904,"reason":"broker was unresponsive"},"11327":{"timestamp":1712365573.6758742,"reason":"broker was unresponsive"},"11328":{"timestamp":1712365573.6769655,"reason":"broker was unresponsive"},"11329":{"timestamp":1712337883.442421,"reason":"broker was unresponsive"},"11331":{"timestamp":1712365573.6780519,"reason":"broker was unresponsive"},"11332":{"timestamp":1712365573.6791384,"reason":"broker was unresponsive"},"11333":{"timestamp":1712365573.6802421,"reason":"broker was unresponsive"},"11334":{"timestamp":1712365573.6813529,"reason":"broker was unresponsive"},"11335":{"timestamp":1712365573.6824431,"reason":"broker was unresponsive"},"11336":{"timestamp":1712365573.6835349,"reason":"broker was unresponsive"},"11337":{"timestamp":1712365573.6846213,"reason":"broker was unresponsive"},"11338":{"timestamp":1712365573.6857021,"reason":"broker was unresponsive"},"11339":{"timestamp":1712365573.6867895,"reason":"broker was unresponsive"},"11340":{"timestamp":1712365573.6878812,"reason":"broker was unresponsive"},"11341":{"timestamp":1712365573.688972,"reason":"broker was unresponsive"},"11342":{"timestamp":1712365573.6900647,"reason":"broker was unresponsive"},"11343":{"timestamp":1712365573.691153,"reason":"broker was unresponsive"},"11344":{"timestamp":1712365573.6922565,"reason":"broker was unresponsive"},"11345":{"timestamp":1712365573.6933806,"reason":"broker was unresponsive"},"11346":{"timestamp":1712365573.6944704,"reason":"broker was unresponsive"},"11347":{"timestamp":1712365573.6955621,"reason":"broker was unresponsive"},"11348":{"timestamp":1712365573.6966543,"reason":"broker was unresponsive"},"11349":{"timestamp":1712365573.6977494,"reason":"broker was unresponsive"},"11350":{"timestamp":1712365573.6988451,"reason":"broker was unresponsive"},"11351":{"timestamp":1712365573.6999371,"reason":"broker was unresponsive"},"11352":{"timestamp":1712365573.7010262,"reason":"broker was unresponsive"},"11353":{"timestamp":1712365573.7021222,"reason":"broker was unresponsive"},"11354":{"timestamp":1712365573.7032163,"reason":"broker was unresponsive"},"11355":{"timestamp":1712365573.7043314,"reason":"broker was unresponsive"},"11356":{"timestamp":1712365573.7054284,"reason":"broker was unresponsive"},"11357":{"timestamp":1712365573.7065215,"reason":"broker was unresponsive"},"11359":{"timestamp":1712365573.7076168,"reason":"broker was unresponsive"},"11360":{"timestamp":1712365573.7087162,"reason":"broker was unresponsive"},"11361":{"timestamp":1712365573.7098145,"reason":"broker was unresponsive"},"11362":{"timestamp":1712365573.7109149,"reason":"broker was unresponsive"},"11363":{"timestamp":1712365573.7120113,"reason":"broker was unresponsive"},"11364":{"timestamp":1712365573.7131152,"reason":"broker was unresponsive"},"11365":{"timestamp":1712365573.7142148,"reason":"broker was unresponsive"},"11366":{"timestamp":1712365573.7153404,"reason":"broker was unresponsive"},"11367":{"timestamp":1712365573.7164388,"reason":"broker was unresponsive"},"11368":{"timestamp":1712365573.7175448,"reason":"broker was unresponsive"},"11369":{"timestamp":1712365573.7186477,"reason":"broker was unresponsive"},"11370":{"timestamp":1712365573.7197461,"reason":"broker was unresponsive"},"11371":{"timestamp":1712365573.7208521,"reason":"broker was unresponsive"},"11372":{"timestamp":1712365573.7220464,"reason":"broker was unresponsive"},"11374":{"timestamp":1712338123.4430783,"reason":"broker was unresponsive"},"11375":{"timestamp":1712365573.7231631,"reason":"broker was unresponsive"},"11376":{"timestamp":1712365573.724261,"reason":"broker was unresponsive"},"11377":{"timestamp":1712365573.7253742,"reason":"broker was unresponsive"},"11378":{"timestamp":1712365573.7264869,"reason":"broker was unresponsive"},"11379":{"timestamp":1712365573.7275951,"reason":"broker was unresponsive"},"11380":{"timestamp":1712365573.7287025,"reason":"broker was unresponsive"},"11381":{"timestamp":1712365573.7298064,"reason":"broker was unresponsive"},"11382":{"timestamp":1712365573.7309108,"reason":"broker was unresponsive"},"11383":{"timestamp":1712365573.7320132,"reason":"broker was unresponsive"},"11384":{"timestamp":1712365573.7331238,"reason":"broker was unresponsive"},"11385":{"timestamp":1712365573.734225,"reason":"broker was unresponsive"},"11386":{"timestamp":1712365573.7353494,"reason":"broker was unresponsive"},"11388":{"timestamp":1712242176.6260459,"reason":"broker was unresponsive"},"11389":{"timestamp":1712365573.7364607,"reason":"broker was unresponsive"},"11390":{"timestamp":1712154692.7781432,"reason":"broker was unresponsive"},"11391":{"timestamp":1712365573.7375669,"reason":"broker was unresponsive"},"11392":{"timestamp":1712365573.7386825,"reason":"broker was unresponsive"},"11393":{"timestamp":1712365573.7397916,"reason":"broker was unresponsive"},"11394":{"timestamp":1712365573.7409284,"reason":"broker was unresponsive"},"11395":{"timestamp":1712154692.7823181,"reason":"broker was unresponsive"},"11396":{"timestamp":1712365573.7420373,"reason":"broker was unresponsive"},"11397":{"timestamp":1712365573.7431452,"reason":"broker was unresponsive"},"11398":{"timestamp":1712365573.744252,"reason":"broker was unresponsive"},"11399":{"timestamp":1712365573.7453833,"reason":"broker was unresponsive"},"11400-11412":{"timestamp":1712270907.4221985,"reason":""},"11413-11428":{"timestamp":1712271808.3093069,"reason":""},"11429":{"timestamp":1712154692.8102601,"reason":"broker was unresponsive"},"11430":{"timestamp":1712365573.778296,"reason":"broker was unresponsive"},"11431":{"timestamp":1712154692.8120186,"reason":"broker was unresponsive"},"11432":{"timestamp":1712365573.7794206,"reason":"broker was unresponsive"},"11433":{"timestamp":1712365573.7805457,"reason":"broker was unresponsive"},"11434":{"timestamp":1712365573.7816722,"reason":"broker was unresponsive"},"11435":{"timestamp":1712365573.7828031,"reason":"broker was unresponsive"},"11436":{"timestamp":1712365573.7839198,"reason":"broker was unresponsive"},"11437":{"timestamp":1712365573.785042,"reason":"broker was unresponsive"},"11438":{"timestamp":1712365573.7861645,"reason":"broker was unresponsive"},"11440":{"timestamp":1712154692.8191555,"reason":"broker was unresponsive"},"11441":{"timestamp":1712154692.8200519,"reason":"broker was unresponsive"},"11442":{"timestamp":1712154692.8209813,"reason":"broker was unresponsive"},"11443":{"timestamp":1712365573.7873051,"reason":"broker was unresponsive"},"11444":{"timestamp":1712365573.7884271,"reason":"broker was unresponsive"},"11445":{"timestamp":1712365573.7895498,"reason":"broker was unresponsive"},"11446":{"timestamp":1712365573.7906895,"reason":"broker was unresponsive"},"11447":{"timestamp":1712365573.7918308,"reason":"broker was unresponsive"},"11448":{"timestamp":1712365573.7929759,"reason":"broker was unresponsive"},"11449":{"timestamp":1712365573.7941093,"reason":"broker was unresponsive"},"11450":{"timestamp":1712365573.7952402,"reason":"broker was unresponsive"},"11451":{"timestamp":1712365573.7963824,"reason":"broker was unresponsive"},"11452":{"timestamp":1712365573.7975113,"reason":"broker was unresponsive"},"11453":{"timestamp":1712365573.7986388,"reason":"broker was unresponsive"},"11454":{"timestamp":1712365573.7997668,"reason":"broker was unresponsive"},"11455":{"timestamp":1712365573.8008986,"reason":"broker was unresponsive"},"11456":{"timestamp":1712365573.8020322,"reason":"broker was unresponsive"},"11457":{"timestamp":1712365573.8031595,"reason":"broker was unresponsive"},"11458":{"timestamp":1712365573.8043122,"reason":"broker was unresponsive"},"11459":{"timestamp":1712365573.8054457,"reason":"broker was unresponsive"},"11460":{"timestamp":1712365573.8065751,"reason":"broker was unresponsive"},"11461":{"timestamp":1712365573.8077266,"reason":"broker was unresponsive"},"11462":{"timestamp":1712365573.8088553,"reason":"broker was unresponsive"},"11463":{"timestamp":1712365573.8099849,"reason":"broker was unresponsive"},"11464":{"timestamp":1712154692.8396258,"reason":"broker was unresponsive"},"11465":{"timestamp":1712365573.8111331,"reason":"broker was unresponsive"},"11466":{"timestamp":1712329120.7360339,"reason":""},"11467":{"timestamp":1712365573.8122854,"reason":"broker was unresponsive"},"11468":{"timestamp":1712365573.8134332,"reason":"broker was unresponsive"},"11469":{"timestamp":1712365573.8145852,"reason":"broker was unresponsive"},"11470":{"timestamp":1712365573.8157241,"reason":"broker was unresponsive"},"11471":{"timestamp":1712365573.8168702,"reason":"broker was unresponsive"},"11472":{"timestamp":1712365573.8180058,"reason":"broker was unresponsive"},"11473":{"timestamp":1712365573.8191481,"reason":"broker was unresponsive"},"11474":{"timestamp":1712365573.8203051,"reason":"broker was unresponsive"},"11475":{"timestamp":1712365573.8214438,"reason":"broker was unresponsive"},"11476":{"timestamp":1712365573.8225882,"reason":"broker was unresponsive"},"11477":{"timestamp":1712365573.8237481,"reason":"broker was unresponsive"},"11478":{"timestamp":1712365573.8248889,"reason":"broker was unresponsive"},"11479":{"timestamp":1712365573.826026,"reason":"broker was unresponsive"},"11480":{"timestamp":1712365573.8271687,"reason":"broker was unresponsive"},"11481":{"timestamp":1712365573.8283229,"reason":"broker was unresponsive"},"11482":{"timestamp":1712365573.8294599,"reason":"broker was unresponsive"},"11483":{"timestamp":1712365573.830601,"reason":"broker was unresponsive"},"11484":{"timestamp":1712365573.8317366,"reason":"broker was unresponsive"},"11485":{"timestamp":1712365573.8328736,"reason":"broker was unresponsive"},"11486":{"timestamp":1712365573.834013,"reason":"broker was unresponsive"},"11487":{"timestamp":1712365573.8351524,"reason":"broker was unresponsive"},"11488":{"timestamp":1712365573.8363042,"reason":"broker was unresponsive"},"11489":{"timestamp":1712365573.8374436,"reason":"broker was unresponsive"},"11490":{"timestamp":1712365573.8385918,"reason":"broker was unresponsive"},"11491":{"timestamp":1712365573.8397274,"reason":"broker was unresponsive"},"11492":{"timestamp":1712365573.8408744,"reason":"broker was unresponsive"},"11493":{"timestamp":1712365573.8420322,"reason":"broker was unresponsive"},"11494":{"timestamp":1712365573.8431768,"reason":"broker was unresponsive"},"11495":{"timestamp":1712365573.8443353,"reason":"broker was unresponsive"},"11496":{"timestamp":1712365573.8454797,"reason":"broker was unresponsive"},"11497":{"timestamp":1712365573.8466315,"reason":"broker was unresponsive"},"11498":{"timestamp":1712365573.8477974,"reason":"broker was unresponsive"},"11499":{"timestamp":1712365573.8489382,"reason":"broker was unresponsive"},"11500":{"timestamp":1712365573.8500795,"reason":"broker was unresponsive"},"11501":{"timestamp":1712365573.8512239,"reason":"broker was unresponsive"},"11502":{"timestamp":1712365573.8524072,"reason":"broker was unresponsive"},"11503":{"timestamp":1712365573.8535519,"reason":"broker was unresponsive"},"11504":{"timestamp":1712365573.8546991,"reason":"broker was unresponsive"},"11505":{"timestamp":1712365573.8558791,"reason":"broker was unresponsive"},"11506":{"timestamp":1712365573.8570251,"reason":"broker was unresponsive"},"11507":{"timestamp":1712365573.8581719,"reason":"broker was unresponsive"},"11508":{"timestamp":1712365573.8593483,"reason":"broker was unresponsive"},"11509":{"timestamp":1712365573.860502,"reason":"broker was unresponsive"},"11510":{"timestamp":1712365573.8616548,"reason":"broker was unresponsive"},"11511":{"timestamp":1712365573.862803,"reason":"broker was unresponsive"},"11512":{"timestamp":1712365573.8639483,"reason":"broker was unresponsive"},"11513":{"timestamp":1712365573.8651066,"reason":"broker was unresponsive"},"11514":{"timestamp":1712365573.8662601,"reason":"broker was unresponsive"},"11515":{"timestamp":1712365573.8674266,"reason":"broker was unresponsive"},"11516":{"timestamp":1712365573.8685856,"reason":"broker was unresponsive"},"11517":{"timestamp":1712365573.8697486,"reason":"broker was unresponsive"},"11518":{"timestamp":1712365573.8709023,"reason":"broker was unresponsive"},"11519":{"timestamp":1712365573.872051,"reason":"broker was unresponsive"},"11520":{"timestamp":1712365573.8732071,"reason":"broker was unresponsive"},"11521":{"timestamp":1712365573.8743899,"reason":"broker was unresponsive"},"11522":{"timestamp":1712365573.8755434,"reason":"broker was unresponsive"},"11523":{"timestamp":1712365573.8766971,"reason":"broker was unresponsive"},"11524":{"timestamp":1712365573.8778551,"reason":"broker was unresponsive"},"11525":{"timestamp":1712365573.8790076,"reason":"broker was unresponsive"},"11526":{"timestamp":1712365573.8801579,"reason":"broker was unresponsive"},"11527":{"timestamp":1712043086.7846026,"reason":"broker was unresponsive"},"11528":{"timestamp":1712365573.8813295,"reason":"broker was unresponsive"},"11529":{"timestamp":1712365573.8824995,"reason":"broker was unresponsive"},"11530":{"timestamp":1712365573.8836577,"reason":"broker was unresponsive"},"11531":{"timestamp":1712365573.8848116,"reason":"broker was unresponsive"},"11532":{"timestamp":1712365573.8859637,"reason":"broker was unresponsive"},"11533":{"timestamp":1712365573.8871183,"reason":"broker was unresponsive"},"11534":{"timestamp":1712365573.8882921,"reason":"broker was unresponsive"},"11535":{"timestamp":1712365573.8894467,"reason":"broker was unresponsive"},"11536":{"timestamp":1712365573.8906085,"reason":"broker was unresponsive"},"11537":{"timestamp":1712365573.8917618,"reason":"broker was unresponsive"},"11538":{"timestamp":1712365573.8929169,"reason":"broker was unresponsive"},"11539":{"timestamp":1712365573.8940694,"reason":"broker was unresponsive"},"11540":{"timestamp":1712365573.8952346,"reason":"broker was unresponsive"},"11541":{"timestamp":1712365573.8964059,"reason":"broker was unresponsive"},"11542":{"timestamp":1712365573.8975644,"reason":"broker was unresponsive"},"11543":{"timestamp":1712365573.8987365,"reason":"broker was unresponsive"},"11544":{"timestamp":1712365573.9001698,"reason":"broker was unresponsive"},"11545":{"timestamp":1712365573.9020455,"reason":"broker was unresponsive"},"11546":{"timestamp":1712365573.9040096,"reason":"broker was unresponsive"},"11547":{"timestamp":1712365573.9058547,"reason":"broker was unresponsive"},"11548":{"timestamp":1712365573.9075847,"reason":"broker was unresponsive"},"11549":{"timestamp":1712365573.9090986,"reason":"broker was unresponsive"},"11550":{"timestamp":1712365573.9108431,"reason":"broker was unresponsive"},"11551":{"timestamp":1712365573.9125543,"reason":"broker was unresponsive"},"11552":{"timestamp":1712365573.9142616,"reason":"broker was unresponsive"},"11553":{"timestamp":1712365573.915961,"reason":"broker was unresponsive"},"11554":{"timestamp":1712365573.9174156,"reason":"broker was unresponsive"},"11555":{"timestamp":1712365573.9185796,"reason":"broker was unresponsive"},"11556":{"timestamp":1712365573.9197261,"reason":"broker was unresponsive"},"11557":{"timestamp":1712365573.9208729,"reason":"broker was unresponsive"},"11558":{"timestamp":1712365573.9220166,"reason":"broker was unresponsive"},"11559":{"timestamp":1712365573.9232931,"reason":"broker was unresponsive"},"11560":{"timestamp":1712365573.9244602,"reason":"broker was unresponsive"},"11561":{"timestamp":1712365573.9256153,"reason":"broker was unresponsive"},"11562":{"timestamp":1712365573.9267669,"reason":"broker was unresponsive"},"11563":{"timestamp":1712154692.9307995,"reason":"broker was unresponsive"},"11564":{"timestamp":1712365573.9279335,"reason":"broker was unresponsive"},"11565":{"timestamp":1712365573.9290888,"reason":"broker was unresponsive"},"11566":{"timestamp":1712365573.9302316,"reason":"broker was unresponsive"},"11567":{"timestamp":1712365573.9313924,"reason":"broker was unresponsive"},"11568":{"timestamp":1712365573.932548,"reason":"broker was unresponsive"},"11569":{"timestamp":1712365573.9336905,"reason":"broker was unresponsive"},"11570":{"timestamp":1712365573.9349179,"reason":"broker was unresponsive"},"11571":{"timestamp":1712365573.9361832,"reason":"broker was unresponsive"},"11572":{"timestamp":1712365573.9374418,"reason":"broker was unresponsive"},"11573":{"timestamp":1712365573.9386592,"reason":"broker was unresponsive"},"11574":{"timestamp":1712365573.939939,"reason":"broker was unresponsive"},"11575":{"timestamp":1712365573.9411871,"reason":"broker was unresponsive"},"11576":{"timestamp":1712365573.9424169,"reason":"broker was unresponsive"},"11577":{"timestamp":1712365573.9435771,"reason":"broker was unresponsive"},"11578":{"timestamp":1712365573.9447336,"reason":"broker was unresponsive"},"11579":{"timestamp":1712365573.9459293,"reason":"broker was unresponsive"},"11580":{"timestamp":1712365573.9471509,"reason":"broker was unresponsive"},"11581":{"timestamp":1712365573.9484508,"reason":"broker was unresponsive"},"11582":{"timestamp":1712365573.9496152,"reason":"broker was unresponsive"},"11583":{"timestamp":1712365573.950798,"reason":"broker was unresponsive"},"11584":{"timestamp":1712365573.9520173,"reason":"broker was unresponsive"},"11585":{"timestamp":1712365573.9531691,"reason":"broker was unresponsive"},"11586":{"timestamp":1712365573.9543386,"reason":"broker was unresponsive"},"11587":{"timestamp":1711663526.290566,"reason":"kernel panics - JRG"},"11588":{"timestamp":1712093194.2606094,"reason":"bad partner node - JRG"},"11589":{"timestamp":1712365573.9554882,"reason":"broker was unresponsive"},"11590":{"timestamp":1712365573.956645,"reason":"broker was unresponsive"},"11591":{"timestamp":1712365573.9577982,"reason":"broker was unresponsive"},"11592":{"timestamp":1712365573.9590485,"reason":"broker was unresponsive"},"11593":{"timestamp":1712365573.9602113,"reason":"broker was unresponsive"},"11594":{"timestamp":1712365573.9613798,"reason":"broker was unresponsive"},"11595":{"timestamp":1712365573.9625344,"reason":"broker was unresponsive"},"11596":{"timestamp":1712365573.96369,"reason":"broker was unresponsive"},"11597":{"timestamp":1712365573.9648461,"reason":"broker was unresponsive"},"11598":{"timestamp":1712365573.9660347,"reason":"broker was unresponsive"},"11599":{"timestamp":1712365573.9672759,"reason":"broker was unresponsive"},"11600":{"timestamp":1712365573.9684441,"reason":"broker was unresponsive"},"11601":{"timestamp":1712365573.9695969,"reason":"broker was unresponsive"},"11602":{"timestamp":1712365573.9707575,"reason":"broker was unresponsive"},"11603":{"timestamp":1712365573.9719169,"reason":"broker was unresponsive"},"11604":{"timestamp":1712365573.9730785,"reason":"broker was unresponsive"},"11605":{"timestamp":1712365573.9742367,"reason":"broker was unresponsive"},"11606":{"timestamp":1712365573.9754145,"reason":"broker was unresponsive"},"11607":{"timestamp":1712365573.9765701,"reason":"broker was unresponsive"},"11608":{"timestamp":1712365573.9777808,"reason":"broker was unresponsive"},"11609":{"timestamp":1712365573.9789431,"reason":"broker was unresponsive"},"11610":{"timestamp":1712365573.9801011,"reason":"broker was unresponsive"},"11611":{"timestamp":1712365573.9812617,"reason":"broker was unresponsive"},"11612":{"timestamp":1712365573.9824805,"reason":"broker was unresponsive"},"11613":{"timestamp":1712365573.9836538,"reason":"broker was unresponsive"},"11614":{"timestamp":1712365573.9848158,"reason":"broker was unresponsive"},"11615":{"timestamp":1712365573.9859715,"reason":"broker was unresponsive"},"11616":{"timestamp":1712365573.9871686,"reason":"broker was unresponsive"},"11617":{"timestamp":1712365573.9884355,"reason":"broker was unresponsive"},"11618":{"timestamp":1712365573.9896021,"reason":"broker was unresponsive"},"11619":{"timestamp":1712365573.9908779,"reason":"broker was unresponsive"},"11620":{"timestamp":1712365573.9921825,"reason":"broker was unresponsive"},"11621":{"timestamp":1712365573.9934466,"reason":"broker was unresponsive"},"11622":{"timestamp":1712365573.9946382,"reason":"broker was unresponsive"},"11623":{"timestamp":1712365573.9959121,"reason":"broker was unresponsive"},"11624":{"timestamp":1712365573.997138,"reason":"broker was unresponsive"},"11625":{"timestamp":1712365573.9983759,"reason":"broker was unresponsive"},"11626":{"timestamp":1712365573.9996562,"reason":"broker was unresponsive"},"11627":{"timestamp":1712365574.0008256,"reason":"broker was unresponsive"},"11628":{"timestamp":1712365574.0020335,"reason":"broker was unresponsive"},"11629":{"timestamp":1712365574.0032594,"reason":"broker was unresponsive"},"11630":{"timestamp":1712365574.0045254,"reason":"broker was unresponsive"},"11631":{"timestamp":1712365574.0057659,"reason":"broker was unresponsive"},"11632":{"timestamp":1712154693.0617664,"reason":"broker was unresponsive"},"11633":{"timestamp":1712365574.007026,"reason":"broker was unresponsive"},"11634":{"timestamp":1712365574.0082335,"reason":"broker was unresponsive"},"11635":{"timestamp":1712365574.0094814,"reason":"broker was unresponsive"},"11636":{"timestamp":1712365574.0107441,"reason":"broker was unresponsive"},"11653":{"timestamp":1712365574.0119636,"reason":"broker was unresponsive"},"11654":{"timestamp":1712365574.013144,"reason":"broker was unresponsive"},"11655":{"timestamp":1712365574.0143993,"reason":"broker was unresponsive"},"11656":{"timestamp":1712365574.0155854,"reason":"broker was unresponsive"},"11657":{"timestamp":1712365574.0168338,"reason":"broker was unresponsive"},"11658":{"timestamp":1712365574.0180514,"reason":"broker was unresponsive"},"11659":{"timestamp":1712365574.0192924,"reason":"broker was unresponsive"},"11660":{"timestamp":1712365574.0205414,"reason":"broker was unresponsive"},"11661":{"timestamp":1712365574.0217526,"reason":"broker was unresponsive"},"11662":{"timestamp":1712365574.0230043,"reason":"broker was unresponsive"},"11663":{"timestamp":1712365574.0242088,"reason":"broker was unresponsive"},"11664":{"timestamp":1712365574.0253997,"reason":"broker was unresponsive"},"11665":{"timestamp":1712365574.0266085,"reason":"broker was unresponsive"},"11666":{"timestamp":1712365574.0277908,"reason":"broker was unresponsive"},"11667":{"timestamp":1712365574.0289652,"reason":"broker was unresponsive"},"11668":{"timestamp":1712365574.0302942,"reason":"broker was unresponsive"},"11669":{"timestamp":1712365574.0321054,"reason":"broker was unresponsive"},"11670":{"timestamp":1712365574.0339477,"reason":"broker was unresponsive"},"11671":{"timestamp":1712365574.0358055,"reason":"broker was unresponsive"},"11672":{"timestamp":1712365574.0376096,"reason":"broker was unresponsive"},"11673":{"timestamp":1712365574.039391,"reason":"broker was unresponsive"},"11674":{"timestamp":1712365574.0411592,"reason":"broker was unresponsive"},"11675":{"timestamp":1712365574.0429301,"reason":"broker was unresponsive"},"11676":{"timestamp":1712365574.0444205,"reason":"broker was unresponsive"},"11677":{"timestamp":1712365574.0457098,"reason":"broker was unresponsive"},"11678":{"timestamp":1712365574.0469139,"reason":"broker was unresponsive"},"11679":{"timestamp":1712365574.0481505,"reason":"broker was unresponsive"},"11680":{"timestamp":1712365574.0495498,"reason":"broker was unresponsive"},"11681":{"timestamp":1712365574.0508928,"reason":"broker was unresponsive"},"11682":{"timestamp":1712365574.0521166,"reason":"broker was unresponsive"},"11683":{"timestamp":1712365574.0533624,"reason":"broker was unresponsive"},"11684":{"timestamp":1712365574.0547881,"reason":"broker was unresponsive"},"11685":{"timestamp":1712365574.05601,"reason":"broker was unresponsive"},"11686":{"timestamp":1712365574.0573213,"reason":"broker was unresponsive"},"11687":{"timestamp":1712365574.0585124,"reason":"broker was unresponsive"},"11688":{"timestamp":1712365574.0596926,"reason":"broker was unresponsive"},"11689":{"timestamp":1712365574.0609007,"reason":"broker was unresponsive"},"11690":{"timestamp":1712365574.0622373,"reason":"broker was unresponsive"},"11691":{"timestamp":1712365574.0634422,"reason":"broker was unresponsive"},"11692":{"timestamp":1712365574.0646238,"reason":"broker was unresponsive"},"11693":{"timestamp":1712365574.0658562,"reason":"broker was unresponsive"},"11694":{"timestamp":1712352882.0445571,"reason":"broker was unresponsive"},"11695":{"timestamp":1712365574.0670881,"reason":"broker was unresponsive"},"11696":{"timestamp":1712365574.0683165,"reason":"broker was unresponsive"},"11697":{"timestamp":1712365574.0695369,"reason":"broker was unresponsive"},"11698":{"timestamp":1712365574.0707932,"reason":"broker was unresponsive"},"11699":{"timestamp":1712365574.0721102,"reason":"broker was unresponsive"},"11700":{"timestamp":1712365574.0733862,"reason":"broker was unresponsive"},"11701":{"timestamp":1712365574.0746007,"reason":"broker was unresponsive"},"11702":{"timestamp":1712365574.0758116,"reason":"broker was unresponsive"},"11703":{"timestamp":1712365574.077132,"reason":"broker was unresponsive"},"11704":{"timestamp":1712365574.0784664,"reason":"broker was unresponsive"},"11705":{"timestamp":1712365574.0797985,"reason":"broker was unresponsive"},"11706":{"timestamp":1712365574.0811372,"reason":"broker was unresponsive"},"11707":{"timestamp":1712365574.0824137,"reason":"broker was unresponsive"},"11708":{"timestamp":1712365574.0837054,"reason":"broker was unresponsive"},"11709":{"timestamp":1712365574.0849547,"reason":"broker was unresponsive"},"11710":{"timestamp":1712365574.0861518,"reason":"broker was unresponsive"},"11711":{"timestamp":1712365574.0873725,"reason":"broker was unresponsive"},"11712":{"timestamp":1712365574.0885665,"reason":"broker was unresponsive"},"11713":{"timestamp":1712365574.0897558,"reason":"broker was unresponsive"},"11714":{"timestamp":1712365574.0909441,"reason":"broker was unresponsive"},"11715":{"timestamp":1712365574.092135,"reason":"broker was unresponsive"},"11716":{"timestamp":1712365574.0933781,"reason":"broker was unresponsive"},"11717":{"timestamp":1712365574.0949488,"reason":"broker was unresponsive"},"11718":{"timestamp":1712365574.0961504,"reason":"broker was unresponsive"},"11719":{"timestamp":1712365574.0973582,"reason":"broker was unresponsive"},"11720":{"timestamp":1712365574.0985529,"reason":"broker was unresponsive"},"11721":{"timestamp":1712365574.0997965,"reason":"broker was unresponsive"},"11722":{"timestamp":1712365574.1010034,"reason":"broker was unresponsive"},"11723":{"timestamp":1712365574.1021972,"reason":"broker was unresponsive"},"11724":{"timestamp":1712365574.1034052,"reason":"broker was unresponsive"},"11725":{"timestamp":1712365574.1047161,"reason":"broker was unresponsive"},"11726":{"timestamp":1712365574.1060042,"reason":"broker was unresponsive"},"11727":{"timestamp":1712365574.107301,"reason":"broker was unresponsive"},"11728":{"timestamp":1712365574.108511,"reason":"broker was unresponsive"},"11729":{"timestamp":1712365574.1097074,"reason":"broker was unresponsive"},"11730":{"timestamp":1712365574.1109064,"reason":"broker was unresponsive"},"11731":{"timestamp":1712365574.1121089,"reason":"broker was unresponsive"},"11732":{"timestamp":1712365574.1133173,"reason":"broker was unresponsive"},"11733":{"timestamp":1712365574.1145139,"reason":"broker was unresponsive"},"11734":{"timestamp":1712365574.1157134,"reason":"broker was unresponsive"},"11735":{"timestamp":1712365574.116914,"reason":"broker was unresponsive"},"11736":{"timestamp":1712365574.118113,"reason":"broker was unresponsive"},"11737":{"timestamp":1712365574.1193531,"reason":"broker was unresponsive"},"11738":{"timestamp":1712365574.1206181,"reason":"broker was unresponsive"},"11739":{"timestamp":1712365574.1218214,"reason":"broker was unresponsive"},"11740":{"timestamp":1712365574.1230252,"reason":"broker was unresponsive"},"11741":{"timestamp":1712365574.1243427,"reason":"broker was unresponsive"},"11742":{"timestamp":1712365574.125711,"reason":"broker was unresponsive"},"11743":{"timestamp":1712365574.1270363,"reason":"broker was unresponsive"},"11744":{"timestamp":1712365574.1282978,"reason":"broker was unresponsive"},"11745":{"timestamp":1712365574.1295116,"reason":"broker was unresponsive"},"11746":{"timestamp":1712365574.1307399,"reason":"broker was unresponsive"},"11747":{"timestamp":1712365574.1319807,"reason":"broker was unresponsive"},"11748":{"timestamp":1712365574.1332083,"reason":"broker was unresponsive"},"11749":{"timestamp":1712365574.134516,"reason":"broker was unresponsive"},"11750":{"timestamp":1712365574.1357708,"reason":"broker was unresponsive"},"11751":{"timestamp":1712365574.1370347,"reason":"broker was unresponsive"},"11752":{"timestamp":1712365574.1382954,"reason":"broker was unresponsive"},"11753":{"timestamp":1712365574.1395168,"reason":"broker was unresponsive"},"11754":{"timestamp":1712365574.1407387,"reason":"broker was unresponsive"},"11755":{"timestamp":1712365574.1419568,"reason":"broker was unresponsive"},"11756":{"timestamp":1712365574.143183,"reason":"broker was unresponsive"},"11757":{"timestamp":1712365574.1444252,"reason":"broker was unresponsive"},"11758":{"timestamp":1712365574.1456571,"reason":"broker was unresponsive"},"11759":{"timestamp":1712365574.1468844,"reason":"broker was unresponsive"},"11760":{"timestamp":1712365574.1481116,"reason":"broker was unresponsive"},"11761":{"timestamp":1712365574.1493554,"reason":"broker was unresponsive"},"11762":{"timestamp":1712365574.1505969,"reason":"broker was unresponsive"},"11763":{"timestamp":1712365574.1518533,"reason":"broker was unresponsive"},"11764":{"timestamp":1712365574.1532352,"reason":"broker was unresponsive"},"11777":{"timestamp":1712349643.3440766,"reason":"broker was unresponsive"},"11778":{"timestamp":1712349640.237411,"reason":"broker was unresponsive"},"11765-11776,11779-11780":{"timestamp":1711993308.7730825,"reason":"Diags Running"},"11784":{"timestamp":1711725755.0632687,"reason":"Running Diags - Rabbits Off"},"11790":{"timestamp":1712004602.484952,"reason":"bad partner node - JRG"},"11789,11793":{"timestamp":1712004602.484952,"reason":"kernel panics - JRG"},"11794":{"timestamp":1711725767.0656931,"reason":"bad partner node - JRG"},"11781-11783,11785-11788,11791-11792,11795-11796":{"timestamp":1712004602.484952,"reason":"Running Diags - Rabbits Off"},"11802":{"timestamp":1712365574.1757526,"reason":"broker was unresponsive"},"11803":{"timestamp":1711668961.0618374,"reason":"Running Diags - Rabbits Off"},"11797-11801,11804-11811":{"timestamp":1712004859.9543238,"reason":"Running Diags - Rabbits Off"},"11812":{"timestamp":1712004514.6037207,"reason":"Running Diags - Rabbits Off"},"11813-11828":{"timestamp":1712005088.7732615,"reason":"Running Diags - Rabbits Off"},"11829-11844":{"timestamp":1712068080.9443812,"reason":"Running Diags - Rabbits Off"},"11845-11859":{"timestamp":1712068141.3627269,"reason":"Running Diags - Rabbits Off"},"11860":{"timestamp":1711727735.0622723,"reason":"Running Diags - Rabbits Off"},"11861-11876":{"timestamp":1712068186.5559592,"reason":"Running Diags - Rabbits Off"},"11877":{"timestamp":1712365574.2050045,"reason":"broker was unresponsive"},"11878":{"timestamp":1712365574.2062364,"reason":"broker was unresponsive"},"11879":{"timestamp":1712365574.2074754,"reason":"broker was unresponsive"},"11880":{"timestamp":1712365574.2086878,"reason":"broker was unresponsive"},"11881":{"timestamp":1712365574.209893,"reason":"broker was unresponsive"},"11882":{"timestamp":1712365574.2110989,"reason":"broker was unresponsive"},"11883":{"timestamp":1712365574.2123373,"reason":"broker was unresponsive"},"11884":{"timestamp":1712101849.9139225,"reason":"broker was unresponsive"},"11885":{"timestamp":1712365574.2135563,"reason":"broker was unresponsive"},"11886":{"timestamp":1712365574.2147608,"reason":"broker was unresponsive"},"11887":{"timestamp":1712365574.2159693,"reason":"broker was unresponsive"},"11888":{"timestamp":1712365574.2171705,"reason":"broker was unresponsive"},"11889":{"timestamp":1712365574.2183867,"reason":"broker was unresponsive"},"11890":{"timestamp":1712365574.2195978,"reason":"broker was unresponsive"},"11892":{"timestamp":1712097564.0118735,"reason":"broker was unresponsive"}},"online":"","exclude":"0"}}
+{"timestamp":1712372191.2298894,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1712372299.1416905,"name":"online","context":{"idset":"0"}}
+{"timestamp":1712372299.19066,"name":"online","context":{"idset":"9213,9231,9252,9302,9325,9387,9626,9664,9672,9678,9699,9705,11031,11053,11377,11465,11545,11580"}}
+{"timestamp":1712372299.1950526,"name":"online","context":{"idset":"9212,9219-9220,9223,9230,9236,9242-9243,9267,9270,9273,9280,9285,9289,9292,9298,9310-9311,9313,9319,9321,9326,9330,9348,9363,9367,9372,9375-9376,9379-9380,9383,9390,9404,9408-9409,9412,9415,9418-9419,9421,9423,9438,9449,9589,9591-9593,9597,9608,9627,9636,9643,9645,9652,9654,9657-9658,9677,9679,9692,9702,10359,11014,11025,11036-11037,11048-11049,11052,11057-11058,11061,11064,11071,11074,11077,11089-11090,11093,11264,11274,11276,11285,11289,11303,11306,11319,11338,11340,11347,11350-11353,11360-11362,11382,11386,11401,11410,11427,11430,11432,11435,11447,11450,11459,11462,11470,11472,11478,11483,11493-11494,11500,11509,11511-11513,11546,11549,11593-11594,11788,11796,11800"}}
+{"timestamp":1712372299.1965024,"name":"online","context":{"idset":"952,9205-9210,9215-9216,9221-9222,9224-9227,9229,9232-9235,9237-9238,9241,9244-9249,9251,9253-9254,9256-9259,9261-9262,9264-9266,9268-9269,9271,9274-9279,9281-9284,9286-9288,9290,9293-9295,9297,9299,9301,9303-9304,9306-9309,9312,9314-9315,9317,9320,9322-9324,9328,9331-9332,9335,9337-9347,9349-9352,9354,9356-9358,9360-9362,9364,9366,9368,9370-9371,9374,9381-9382,9384,9386,9388,9391-9396,9398-9401,9403,9405-9406,9410-9411,9413-9414,9416-9417,9424-9425,9427,9429-9432,9434-9437,9439-9446,9450-9454,9456-9458,9460,9590,9594,9596,9598,9600-9603,9605-9607,9609,9611,9613,9615-9622,9625,9628,9630-9632,9635,9637-9642,9644,9646-9651,9655-9656,9659-9663,9665-9671,9673,9675-9676,9680-9685,9687-9689,9691,9693,9695-9698,9700-9701,9703-9704,11008-11009,11013,11015-11019,11022-11024,11027-11028,11033-11035,11038-11047,11051,11055-11056,11062-11063,11066-11068,11070,11072-11073,11075-11076,11078-11086,11088,11091-11092,11094-11103,11105-11106,11253-11259,11261-11263,11265,11267-11271,11273,11275,11277-11281,11284,11288,11290-11291,11293-11294,11296-11297,11299-11300,11304-11305,11307,11309-11310,11312,11315-11316,11321,11323,11335,11337,11339,11341,11343-11346,11349,11354,11356-11357,11359,11363-11364,11379-11381,11384-11385,11400,11402-11404,11406-11407,11412,11415-11421,11423,11425,11428,11433-11434,11436,11443-11444,11446,11448-11449,11454,11456-11458,11461,11463,11467-11469,11473-11477,11479-11482,11484-11490,11492,11495-11499,11502-11503,11505-11508,11510,11514,11516,11518-11521,11523-11526,11529-11530,11533,11535-11539,11541-11544,11547-11548,11551,11553,11555-11556,11558-11562,11564-11566,11568-11571,11573-11576,11578,11582-11586,11589-11590,11592,11595-11598,11600-11602,11781-11783,11791-11792,11795,11797-11799,11801,11805,11810-11811,11814,11817-11820,11822-11828"}}
+{"timestamp":1712372299.2009573,"name":"online","context":{"idset":"109,117,123,254,431,951,9211,9217-9218,9228,9250,9255,9260,9263,9272,9291,9296,9300,9305,9327,9329,9333-9334,9336,9353,9355,9359,9365,9369,9373,9377-9378,9385,9389,9397,9402,9407,9420,9422,9426,9428,9433,9455,9459,9595,9599,9604,9610,9612,9614,9623-9624,9629,9653,9674,9686,9690,9694,11010-11011,11020-11021,11026,11029-11030,11032,11050,11054,11059-11060,11065,11069,11087,11104,11107-11108,11260,11266,11272,11282-11283,11286,11292,11295,11298,11301-11302,11308,11311,11313-11314,11317-11318,11320,11322,11336,11342,11348,11355,11365,11378,11383,11405,11408-11409,11411,11413-11414,11422,11424,11437-11438,11445,11451-11453,11460,11471,11491,11501,11504,11515,11517,11528,11531-11532,11534,11540,11550,11554,11557,11567,11572,11577,11579,11581,11591,11599,11785-11787,11803-11804,11806,11808-11809,11812-11813,11821"}}
+{"timestamp":1712372299.2080591,"name":"online","context":{"idset":"89,113,124,128,130,132,134-135,141,147,149,154,157,165,167-168,174,188,216,222,233,238,242-243,250,262-263,273,284,295-298,306,310-311,322-323,325,331,333,347,352-353,359,363,366,368,370,373,383,386,390,400,414,421,425,428,430,434,438,473-474,477,484,490,498,500,503,510,522-524,570,585,841,975-976,11455,11802"}}
+{"timestamp":1712372299.2148223,"name":"online","context":{"idset":"85-88,90-94,96-108,110-112,114-116,118-120,122,125-127,129,131,133,136-140,142-146,148,150-153,155-156,158-162,164,169-173,175-176,178-181,184-187,189-207,209-215,218-221,224-232,234-237,239-241,244-249,251-253,255-261,264-272,274-279,281-283,285-286,288-294,299-305,307-309,312-321,324,326-330,332,334-346,349-351,354-358,361-362,364-365,367,369,371-372,374-376,379-382,384-385,387-389,391-399,401-413,415-419,422-423,426-427,429,432-433,435-437,439-444,469-472,475-476,478,480-483,485-489,491-497,499,501-502,504-506,508-509,511-521,525-533,535-540,566-569,571-584,586-588,629,631-636,842,871-872,10632,10952,10988,11287,11333-11334,11398-11399,11603-11604"}}
+{"timestamp":1712372299.2241945,"name":"online","context":{"idset":"95,163,166,177,182-183,208,223,280,287,360,377-378,420,424,479,507,534,565,630,834,963-964,10953,10957,10992,11003,11327,11392,11397"}}
+{"timestamp":1712372299.2372296,"name":"online","context":{"idset":"829,949-950,953,10064,10137,10158,10349,10353,10635,10654,10669,10720,10737,10882,10937,10963,11005,11012,11123,11125,11132,11325,11759"}}
+{"timestamp":1712372299.2489285,"name":"online","context":{"idset":"832-833,954,9488,9527,9713-9714,9982,10043,10103,10150,10179,10233,10246,10269,10338,10370,10398,10509,10525,10537,10565,10585,10624,10631,10633,10636,10640-10641,10666,10696,10704,10761,10804,10815,10826,10836,10938,10954,10958,10983,10989,11004,11224,11389,11393,11396"}}
+{"timestamp":1712372299.2551453,"name":"online","context":{"idset":"25,830-831,9491,9712,9722,9743,9798,10023,10059,10100,10106,10133,10142,10194,10217,10236,10267,10294,10346,10522,10566,10599,10659,10706,10713,10724,10867,10875,10897,10910,10913-10914,10918,10956,10968,10970,10972-10973,10977-10979,10981,10987,10991,10998,11110,11332,11369,11394"}}
+{"timestamp":1712372299.2633035,"name":"online","context":{"idset":"26,33,53,827-828,9468,9487,9495,9547,9559,9568,9710-9711,9715,9764,9794,9799,9803,9811,9836,9841,9975,9995,10000,10006-10007,10010,10016-10017,10022,10026,10031,10037,10045,10058,10098-10099,10136,10141,10151,10162,10164,10174,10190,10205,10224,10263,10287-10288,10298,10300,10304,10317,10323,10341-10342,10347-10348,10354,10357,10360,10365,10382,10385,10393,10396,10402,10410,10413,10420,10423,10427,10429,10446,10449,10456,10463,10491-10492,10494,10505,10524,10528,10536,10538-10539,10544,10555,10593,10604,10607,10617,10630,10645,10647,10651,10665,10679-10680,10689,10691-10692,10697,10700,10705,10708,10710,10714-10715,10722,10726,10728,10730,10747-10748,10751,10758,10779,10786,10789,10845,10849,10852,10862,10869,10876,10880,10900,10905,10919,10935,10940,10945,10962,10966,10975-10976,10986,10994,10997,11001-11002,11006-11007,11113,11124,11128,11225,11228-11229,11324,11326,11331,11370,11750"}}
+{"timestamp":1712372299.2673571,"name":"online","context":{"idset":"22,36,45,9466,9484-9485,9504,9536,9569,9716,9727,9753,9790,9805,9821,10021,10025,10034,10044,10063,10078,10080-10081,10095,10097,10108,10120,10125,10128,10134,10139,10146,10148,10157,10159,10170,10172,10176,10182,10185,10187,10191,10198,10202,10207,10209,10218,10230,10244,10248,10250,10252,10259,10266,10271,10274,10278,10296,10310,10314,10322,10333,10345,10351,10361,10363,10366,10369,10384,10400,10406,10408,10428,10438,10443,10447,10451,10468,10495,10500,10502-10504,10519,10521,10523,10529,10531-10532,10546,10558-10559,10571,10575,10583,10587,10600,10603,10612,10649,10663,10668,10671-10672,10674-10675,10683,10685-10686,10695,10699,10701,10703,10725,10727,10729,10732,10735,10738,10755,10777-10778,10780,10791,10796,10806,10814,10822,10824,10828,10833,10835,10850,10873,10877-10878,10881,10883,10921,10928,10948,10955,10959,10961,10967,10971,10984,10990,10996,11112,11120-11122,11223,11230,11368,11371,11375,11391,11761"}}
+{"timestamp":1712372299.2713175,"name":"online","context":{"idset":"1,4-10,12,14-18,21,23-24,27-32,34-35,38-39,41-44,46-49,51,57,59,9479,9482,9520,9540,9706,9708,9723,9788,9827,9999,10019,10040,10049-10051,10056,10060,10062,10090,10101,10109,10113-10114,10135,10140,10144,10149,10152,10160-10161,10166,10169,10171,10188-10189,10193,10195-10196,10211,10215,10221,10223,10225,10228-10229,10232,10234,10239,10242,10254,10258,10268,10279,10286,10289-10290,10293,10295,10301-10302,10305-10309,10311-10312,10316,10319-10320,10325-10326,10339,10350,10352,10367,10377,10379,10381,10388-10389,10391,10395,10403,10414,10418,10425,10433,10439,10448,10450,10452,10465,10467,10490,10493,10497,10507,10515,10520,10530,10533-10534,10540-10541,10552,10560,10564,10568,10570,10572,10574,10577-10581,10584,10592,10595,10597-10598,10605-10606,10608-10610,10613,10615,10621-10623,10646,10650,10653,10660-10661,10670,10684,10688,10698,10711,10716,10719,10721,10731,10734,10736,10742,10746,10759,10764,10767-10770,10785,10790,10793,10795,10816,10839,10841,10843-10844,10848,10855,10859,10864,10870,10874,10879,10888-10889,10896,10901-10903,10906-10908,10916-10917,10926,10933,10939,10946-10947,10950,10982,10985,10993,10995,11109,11111,11175,11221-11222,11227,11231,11244,11756"}}
+{"timestamp":1712372299.2755497,"name":"online","context":{"idset":"2-3,11,13,19-20,37,40,50,52,56,58,9474,9476,9486,9489,9494,9510,9512,9516,9518,9522,9528,9532,9545,9554,9557-9558,9574,9579,9724,9747,9750,9772,9774,9791,9793,9800,9802,9809,9823,9832,9837,9973,9978,9990,9996,9998,10009,10012,10018,10020,10028-10030,10032-10033,10035-10036,10038-10039,10042,10046-10047,10052-10053,10055,10068-10070,10072,10075-10076,10079,10085-10088,10092-10093,10096,10104,10107,10110-10112,10116-10118,10122-10124,10127,10129,10132,10147,10153-10154,10163,10168,10178,10181,10183,10192,10199-10200,10204,10208,10214,10216,10219-10220,10222,10226-10227,10231,10235,10237,10240-10241,10243,10245,10249,10256-10257,10264-10265,10275,10277,10280,10283-10285,10291-10292,10297,10318,10321,10327-10331,10334-10335,10340,10358,10362,10364,10368,10371,10373-10375,10380,10383,10386-10387,10390,10392,10397,10399,10401,10416,10424,10426,10431,10434,10437,10440,10445,10454-10455,10457-10458,10460-10462,10464,10466,10485,10487-10489,10496,10498-10499,10506,10511-10514,10516,10526-10527,10542-10543,10545,10547-10549,10551,10553-10554,10556-10557,10562-10563,10567,10569,10573,10576,10582,10586,10588-10589,10591,10596,10602,10611,10614,10616,10618-10620,10625,10627-10628,10648,10652,10656,10658,10662,10664,10673,10676-10678,10682,10690,10707,10709,10717,10733,10744,10749,10753,10757,10762-10763,10766,10772,10774,10776,10781-10782,10787-10788,10792,10794,10797-10800,10803,10805,10807-10812,10817,10820,10823,10830,10837-10838,10846-10847,10857-10858,10860,10863,10865-10866,10868,10884-10887,10891,10895,10898-10899,10915,10920,10922-10924,10927,10929-10930,10934,10936,10941-10942,10949,10960,10964,10969,10980,11114,11117-11119,11167,11174,11214,11232,11234,11238,11367,11372,11745,11751,11755"}}
+{"timestamp":1712372299.2799253,"name":"online","context":{"idset":"54-55,60,9462,9477,9490,9492,9499,9502,9506,9508,9529,9537,9555,9560,9570-9572,9576,9588,9707,9731,9738,9748,9776,9782-9783,9786,9789,9797,9817,9829,9840,9974,9981,9983-9985,9988,10003-10005,10011,10013,10041,10048,10054,10057,10065,10074,10082-10083,10094,10102,10105,10115,10119,10121,10126,10130-10131,10167,10177,10180,10184,10186,10201,10213,10238,10247,10251,10253,10260-10262,10270,10272-10273,10276,10281-10282,10299,10313,10315,10324,10337,10372,10376,10378,10394,10404-10405,10407,10409,10411-10412,10415,10417,10419,10421-10422,10430,10432,10435-10436,10441-10442,10444,10453,10459,10486,10501,10508,10510,10535,10561,10590,10594,10601,10626,10655,10667,10681,10687,10723,10743,10745,10750,10752,10754,10756,10760,10765,10771,10773,10775,10783,10801-10802,10813,10818-10819,10821,10825,10832,10840,10842,10853-10854,10856,10861,10890,10893-10894,10904,10909,10925,10931-10932,10951,10965,10974,11148,11209,11240,11376,11749,11753-11754,11758,11760,11762,11764,11888"}}
+{"timestamp":1712372299.2831914,"name":"online","context":{"idset":"9464,9467,9469-9470,9478,9480-9481,9483,9497,9513-9514,9519,9541,9543,9548-9550,9553,9564,9566,9575,9580,9582,9733-9734,9740,9745,9767,9770-9771,9775,9796,9808,9810,9812,9818,9825,9831,9977,9979,9993,9997,10008,10073,10084,10091,10165,10173,10175,10197,10203,10206,10210,10212,10255,10332,10336,10550,10784,10829,10831,10834,10851,10892,11162,11197,11251,11328,11366,11727,11752,11757"}}
+{"timestamp":1712372299.2870655,"name":"online","context":{"idset":"9500,9515,9521,9526,9544,9563,9565,9577,9581,9586-9587,9717,9719,9721,9728,9736-9737,9742,9744,9749,9751-9752,9754,9756,9766,9768,9777,9779-9781,9784-9785,9787,9795,9804,9806-9807,9833-9835,9839,9843,9991,10002,10014,10024,10061,10089,10741,10827,11151,11159-11160,11181,11185,11196,11241,11245,11613,11700,11763"}}
+{"timestamp":1712372299.2897749,"name":"online","context":{"idset":"9461,9471,9493,9498,9501,9503,9505,9507,9517,9530,9539,9551-9552,9556,9567,9573,9578,9585,9720,9730,9732,9755,9758-9760,9792,9801,9813,9819,9824,9838,9842,9976,9986,9994,10071,11144,11149,11161,11165,11171,11177,11179,11183,11190,11193,11237,11248,11250,11621,11630,11687-11688,11720,11882,11890"}}
+{"timestamp":1712372299.2923007,"name":"online","context":{"idset":"9472-9473,9496,9511,9525,9531,9542,9546,9583-9584,9709,9729,9735,9739,9741,9757,9765,9769,9778,9814-9816,9822,9826,9828,9980,10001,10015,10027,10066,10077,11146,11152,11195,11198-11199,11205,11207-11208,11211,11215,11217,11219,11242-11243,11624,11681,11690,11695,11711,11744,11886,11889"}}
+{"timestamp":1712372299.295074,"name":"online","context":{"idset":"9465,9538,9718,9773,9820,9844,9987,9992,10067,11126,11141,11147,11153-11156,11163-11164,11166,11180,11182,11189,11191-11192,11194,11200-11201,11203-11204,11206,11212-11213,11216,11218,11246,11615,11692,11716,11718,11722,11878,11885"}}
+{"timestamp":1712372299.297425,"name":"online","context":{"idset":"9475,9524,9746,9762-9763,9989,11142,11150,11157-11158,11168-11170,11172-11173,11178,11188,11202,11210,11239,11612,11622,11628,11662,11667,11686,11689,11693,11696,11698-11699,11715,11724,11728,11734,11736,11743,11879-11880,11883,11887"}}
+{"timestamp":1712372299.2993755,"name":"online","context":{"idset":"9463,9509,9523,9761,9830,11138,11143,11176,11184,11186,11220,11252,11614,11616,11618,11626,11636,11658-11659,11663,11676,11678,11684-11685,11703,11706,11712,11737-11738,11741,11747,11877,11881"}}
+{"timestamp":1712372299.3012252,"name":"online","context":{"idset":"11145,11606,11608-11609,11627,11629,11631,11633,11635,11661,11664-11666,11668,11671-11673,11675,11680,11697,11705,11707-11710,11726"}}
+{"timestamp":1712372299.3028924,"name":"online","context":{"idset":"11135,11187,11605,11611,11617,11619,11623,11625,11653,11655-11656,11670,11674,11679,11682-11683,11691,11701-11702,11704,11717,11725,11729-11730,11732,11735"}}
+{"timestamp":1712372299.3046107,"name":"online","context":{"idset":"11607,11620,11634,11657,11660,11669,11677,11713-11714,11731,11739-11740,11742,11746"}}
+{"timestamp":1712372299.306396,"name":"online","context":{"idset":"11127,11129,11131,11133,11136-11137,11139,11610,11654,11719,11721,11748"}}
+{"timestamp":1712372299.3081355,"name":"online","context":{"idset":"11130,11733"}}
+{"timestamp":1712372299.3098986,"name":"online","context":{"idset":"11723"}}
+{"timestamp":1712372422.7196672,"name":"undrain","context":{"idset":"1-68,77-120,122-216,218-347,349-430,432-444,469-540,565-596,605-612,629-644,653-685,687-689,691-693,695-705,707,718-747,749-756,796,799-816,825-836,841-844,870-872,874-875,881-882,949-954,963-980,9205-9213,9215-9238,9241-9315,9317,9319-9446,9449-9532,9534,9536-9561,9563-9632,9635-9749,9751-9844,9973-10137,10139-10142,10144,10146-10154,10157-10260,10262-10302,10304-10354,10357-10516,10519-10628,10630-10633,10635-10636,10640-10641,10645-10656,10658-10692,10695-10701,10703-10711,10713-10717,10719-10738,10741-10870,10872-10910,10913-10942,10944-10998,11000-11114,11117-11225,11227-11232,11234,11237-11246,11248,11250-11329,11331-11357,11359-11372,11374-11386,11388-11399,11429-11438,11440-11465,11467-11586,11589-11636,11653-11764,11777-11778,11802,11877-11890,11892"}}
+{"timestamp":1712373180.855062,"name":"offline","context":{"idset":"11683"}}
+{"timestamp":1712373342.1589975,"name":"drain","context":{"idset":"11683","reason":"epilog failed for jobid fnq4wwxaXR1","overwrite":0}}
+{"timestamp":1712413954.1778886,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.1859577,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9493761,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9567733,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9568844,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9600046,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9628189,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9629068,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9662435,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9689093,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9689927,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9715741,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9716561,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9743707,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.977143,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9776394,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9816084,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9849157,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9878399,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9906921,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1712413954.9993374,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.012423,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0184104,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0185142,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0245759,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0246723,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0378547,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0407164,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0407658,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0432477,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0468969,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0469487,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0487621,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0488,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0514281,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0574765,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0593119,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0664997,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0666907,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0697684,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0930371,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0930982,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0961754,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.09954,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.0996039,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1023271,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1023793,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1050544,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1077654,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1078217,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1103852,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1313756,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1470435,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1501212,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1503692,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1532962,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1565177,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1574855,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.181011,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1812818,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1847427,"name":"drain","context":{"idset":"85","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1876802,"name":"drain","context":{"idset":"86","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.1879106,"name":"drain","context":{"idset":"87","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3257771,"name":"drain","context":{"idset":"88","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3311079,"name":"drain","context":{"idset":"89","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3331878,"name":"drain","context":{"idset":"90","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3382437,"name":"drain","context":{"idset":"91","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3383148,"name":"drain","context":{"idset":"92","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3436854,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3484459,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3485053,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3485672,"name":"drain","context":{"idset":"96","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3486335,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3486831,"name":"drain","context":{"idset":"98","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3487327,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3487773,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3488243,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3488824,"name":"drain","context":{"idset":"102","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3489327,"name":"drain","context":{"idset":"103","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3554597,"name":"drain","context":{"idset":"104","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3555329,"name":"drain","context":{"idset":"105","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3555896,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3556435,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.355701,"name":"drain","context":{"idset":"108","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3557582,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3558135,"name":"drain","context":{"idset":"110","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3558745,"name":"drain","context":{"idset":"111","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3559256,"name":"drain","context":{"idset":"112","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3559818,"name":"drain","context":{"idset":"113","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3560393,"name":"drain","context":{"idset":"114","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3560915,"name":"drain","context":{"idset":"115","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3561599,"name":"drain","context":{"idset":"116","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3562171,"name":"drain","context":{"idset":"117","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.356293,"name":"drain","context":{"idset":"118","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3563757,"name":"drain","context":{"idset":"119","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3564332,"name":"drain","context":{"idset":"120","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.356492,"name":"drain","context":{"idset":"122","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3565776,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3566432,"name":"drain","context":{"idset":"124","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3567717,"name":"drain","context":{"idset":"125","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3568337,"name":"drain","context":{"idset":"126","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3568974,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3569601,"name":"drain","context":{"idset":"128","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.357024,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3570962,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3571594,"name":"drain","context":{"idset":"131","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3572211,"name":"drain","context":{"idset":"132","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.357316,"name":"drain","context":{"idset":"133","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.357378,"name":"drain","context":{"idset":"134","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3574407,"name":"drain","context":{"idset":"135","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3575022,"name":"drain","context":{"idset":"136","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3575673,"name":"drain","context":{"idset":"137","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3576305,"name":"drain","context":{"idset":"138","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3577785,"name":"drain","context":{"idset":"139","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3578467,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3579087,"name":"drain","context":{"idset":"141","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3579779,"name":"drain","context":{"idset":"142","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3581061,"name":"drain","context":{"idset":"143","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3581729,"name":"drain","context":{"idset":"144","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.358237,"name":"drain","context":{"idset":"145","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3583744,"name":"drain","context":{"idset":"146","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3584504,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3585172,"name":"drain","context":{"idset":"148","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3586454,"name":"drain","context":{"idset":"149","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3587196,"name":"drain","context":{"idset":"150","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3587887,"name":"drain","context":{"idset":"151","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3588784,"name":"drain","context":{"idset":"152","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3589461,"name":"drain","context":{"idset":"153","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3590114,"name":"drain","context":{"idset":"154","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3590817,"name":"drain","context":{"idset":"155","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3591461,"name":"drain","context":{"idset":"156","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3592155,"name":"drain","context":{"idset":"157","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3593564,"name":"drain","context":{"idset":"158","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3594241,"name":"drain","context":{"idset":"159","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3594916,"name":"drain","context":{"idset":"160","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3595567,"name":"drain","context":{"idset":"161","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3596239,"name":"drain","context":{"idset":"162","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3596954,"name":"drain","context":{"idset":"163","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3597648,"name":"drain","context":{"idset":"164","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3598371,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3599076,"name":"drain","context":{"idset":"166","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3600366,"name":"drain","context":{"idset":"167","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.36011,"name":"drain","context":{"idset":"168","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3601809,"name":"drain","context":{"idset":"169","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3602574,"name":"drain","context":{"idset":"170","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3603427,"name":"drain","context":{"idset":"171","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3604205,"name":"drain","context":{"idset":"172","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3604915,"name":"drain","context":{"idset":"173","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3605597,"name":"drain","context":{"idset":"174","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3606422,"name":"drain","context":{"idset":"175","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3607147,"name":"drain","context":{"idset":"176","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3607857,"name":"drain","context":{"idset":"177","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3608806,"name":"drain","context":{"idset":"178","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3609569,"name":"drain","context":{"idset":"179","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.361032,"name":"drain","context":{"idset":"180","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3611066,"name":"drain","context":{"idset":"181","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3612392,"name":"drain","context":{"idset":"182","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3615527,"name":"drain","context":{"idset":"183","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3616309,"name":"drain","context":{"idset":"184","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3617024,"name":"drain","context":{"idset":"185","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3618383,"name":"drain","context":{"idset":"186","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3619158,"name":"drain","context":{"idset":"187","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3619938,"name":"drain","context":{"idset":"188","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3620687,"name":"drain","context":{"idset":"189","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3621378,"name":"drain","context":{"idset":"190","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3622139,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3623631,"name":"drain","context":{"idset":"192","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3624494,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3625247,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3626003,"name":"drain","context":{"idset":"195","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3626721,"name":"drain","context":{"idset":"196","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3627594,"name":"drain","context":{"idset":"197","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3628371,"name":"drain","context":{"idset":"198","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3629124,"name":"drain","context":{"idset":"199","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3629875,"name":"drain","context":{"idset":"200","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3631194,"name":"drain","context":{"idset":"201","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3632011,"name":"drain","context":{"idset":"202","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3632922,"name":"drain","context":{"idset":"203","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3634286,"name":"drain","context":{"idset":"204","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3636942,"name":"drain","context":{"idset":"205","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3637805,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3638587,"name":"drain","context":{"idset":"207","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3639364,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3640122,"name":"drain","context":{"idset":"209","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3640881,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3642955,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3643811,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3644602,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3645406,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3646226,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3647029,"name":"drain","context":{"idset":"216","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3647869,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3648865,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3649704,"name":"drain","context":{"idset":"220","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3650572,"name":"drain","context":{"idset":"221","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3651385,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3652248,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3653197,"name":"drain","context":{"idset":"224","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3654065,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3654861,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3655663,"name":"drain","context":{"idset":"227","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3656495,"name":"drain","context":{"idset":"228","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3657324,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3658156,"name":"drain","context":{"idset":"230","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3659,"name":"drain","context":{"idset":"231","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3659844,"name":"drain","context":{"idset":"232","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.366086,"name":"drain","context":{"idset":"233","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3661714,"name":"drain","context":{"idset":"234","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3662562,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.366492,"name":"drain","context":{"idset":"236","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3665812,"name":"drain","context":{"idset":"237","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3666792,"name":"drain","context":{"idset":"238","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.366766,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3668551,"name":"drain","context":{"idset":"240","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3669465,"name":"drain","context":{"idset":"241","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3673925,"name":"drain","context":{"idset":"242","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3675432,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3676353,"name":"drain","context":{"idset":"244","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3678277,"name":"drain","context":{"idset":"245","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3680725,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3681617,"name":"drain","context":{"idset":"247","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3682516,"name":"drain","context":{"idset":"248","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3683903,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.368489,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3702369,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3703599,"name":"drain","context":{"idset":"252","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3705285,"name":"drain","context":{"idset":"253","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3706217,"name":"drain","context":{"idset":"254","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3707125,"name":"drain","context":{"idset":"255","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3708186,"name":"drain","context":{"idset":"256","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3709183,"name":"drain","context":{"idset":"257","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3710198,"name":"drain","context":{"idset":"258","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3711157,"name":"drain","context":{"idset":"259","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3712308,"name":"drain","context":{"idset":"260","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3713434,"name":"drain","context":{"idset":"261","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3714409,"name":"drain","context":{"idset":"262","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3715396,"name":"drain","context":{"idset":"263","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3716388,"name":"drain","context":{"idset":"264","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3717446,"name":"drain","context":{"idset":"265","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3718441,"name":"drain","context":{"idset":"266","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3719571,"name":"drain","context":{"idset":"267","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3720591,"name":"drain","context":{"idset":"268","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3721516,"name":"drain","context":{"idset":"269","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3723276,"name":"drain","context":{"idset":"270","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3725293,"name":"drain","context":{"idset":"271","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3727269,"name":"drain","context":{"idset":"272","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3734934,"name":"drain","context":{"idset":"273","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3736005,"name":"drain","context":{"idset":"274","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3737206,"name":"drain","context":{"idset":"275","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3738251,"name":"drain","context":{"idset":"276","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3740218,"name":"drain","context":{"idset":"277","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3741305,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3748367,"name":"drain","context":{"idset":"279","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3749485,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3751631,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3754067,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3755188,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3756242,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.376332,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.376442,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3767722,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3769443,"name":"drain","context":{"idset":"288","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3772502,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3774204,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3775234,"name":"drain","context":{"idset":"291","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3784328,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3785856,"name":"drain","context":{"idset":"293","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3786943,"name":"drain","context":{"idset":"294","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3788502,"name":"drain","context":{"idset":"295","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3789589,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3790648,"name":"drain","context":{"idset":"297","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3795495,"name":"drain","context":{"idset":"298","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3801312,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3802474,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3803704,"name":"drain","context":{"idset":"301","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3804827,"name":"drain","context":{"idset":"302","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3805907,"name":"drain","context":{"idset":"303","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3807027,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3821697,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3823001,"name":"drain","context":{"idset":"306","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3824103,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3825159,"name":"drain","context":{"idset":"308","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3827252,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.382838,"name":"drain","context":{"idset":"310","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3829453,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3830507,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3831701,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3832905,"name":"drain","context":{"idset":"314","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3833997,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3835294,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3836422,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3837578,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3838687,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3839769,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3840876,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.384198,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3843417,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3845139,"name":"drain","context":{"idset":"324","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3846357,"name":"drain","context":{"idset":"325","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3847442,"name":"drain","context":{"idset":"326","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3850257,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3851407,"name":"drain","context":{"idset":"328","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3852549,"name":"drain","context":{"idset":"329","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3853924,"name":"drain","context":{"idset":"330","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3857095,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3858283,"name":"drain","context":{"idset":"332","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3859391,"name":"drain","context":{"idset":"333","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3860474,"name":"drain","context":{"idset":"334","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3861618,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3862889,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3865139,"name":"drain","context":{"idset":"337","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3866291,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3867421,"name":"drain","context":{"idset":"339","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.386857,"name":"drain","context":{"idset":"340","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3869743,"name":"drain","context":{"idset":"341","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3870857,"name":"drain","context":{"idset":"342","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3872004,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3874812,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3876011,"name":"drain","context":{"idset":"345","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3877213,"name":"drain","context":{"idset":"346","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3880732,"name":"drain","context":{"idset":"347","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3881953,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3883412,"name":"drain","context":{"idset":"350","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3884656,"name":"drain","context":{"idset":"351","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3885903,"name":"drain","context":{"idset":"352","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3887188,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3888385,"name":"drain","context":{"idset":"354","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3889594,"name":"drain","context":{"idset":"355","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3890853,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.389204,"name":"drain","context":{"idset":"357","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3894796,"name":"drain","context":{"idset":"358","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3896389,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3898699,"name":"drain","context":{"idset":"360","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3899925,"name":"drain","context":{"idset":"361","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3901126,"name":"drain","context":{"idset":"362","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3902588,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3904555,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3905795,"name":"drain","context":{"idset":"365","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3906991,"name":"drain","context":{"idset":"366","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3908398,"name":"drain","context":{"idset":"367","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3910198,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3911433,"name":"drain","context":{"idset":"369","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3912816,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3914096,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3915346,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3916583,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3917921,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3919151,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3920376,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3921711,"name":"drain","context":{"idset":"377","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3923061,"name":"drain","context":{"idset":"378","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3924701,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3926032,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3927271,"name":"drain","context":{"idset":"381","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3928466,"name":"drain","context":{"idset":"382","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3930178,"name":"drain","context":{"idset":"383","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3931563,"name":"drain","context":{"idset":"384","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3932972,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3934765,"name":"drain","context":{"idset":"386","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3936257,"name":"drain","context":{"idset":"387","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3937497,"name":"drain","context":{"idset":"388","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3938711,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3939965,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3941269,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.394253,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3944776,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3946083,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3947327,"name":"drain","context":{"idset":"395","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3948634,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3949952,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3951302,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3952875,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3954196,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3955522,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3957,"name":"drain","context":{"idset":"402","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3958354,"name":"drain","context":{"idset":"403","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3959789,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3961117,"name":"drain","context":{"idset":"405","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3963618,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3964965,"name":"drain","context":{"idset":"407","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3966272,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3967566,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.396888,"name":"drain","context":{"idset":"410","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3970215,"name":"drain","context":{"idset":"411","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3971529,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.397305,"name":"drain","context":{"idset":"413","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3974411,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3975768,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3977108,"name":"drain","context":{"idset":"416","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3978446,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3979857,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3981512,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3983009,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3985221,"name":"drain","context":{"idset":"421","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3986566,"name":"drain","context":{"idset":"422","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3988054,"name":"drain","context":{"idset":"423","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3989687,"name":"drain","context":{"idset":"424","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3991258,"name":"drain","context":{"idset":"425","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3992994,"name":"drain","context":{"idset":"426","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3994527,"name":"drain","context":{"idset":"427","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3995926,"name":"drain","context":{"idset":"428","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3997445,"name":"drain","context":{"idset":"429","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.3998909,"name":"drain","context":{"idset":"430","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4001434,"name":"drain","context":{"idset":"432","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4002929,"name":"drain","context":{"idset":"433","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4004917,"name":"drain","context":{"idset":"434","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4006295,"name":"drain","context":{"idset":"435","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4007719,"name":"drain","context":{"idset":"436","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4009671,"name":"drain","context":{"idset":"437","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4011073,"name":"drain","context":{"idset":"438","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.401253,"name":"drain","context":{"idset":"439","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4014714,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4016173,"name":"drain","context":{"idset":"441","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4018276,"name":"drain","context":{"idset":"442","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4019666,"name":"drain","context":{"idset":"443","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4021204,"name":"drain","context":{"idset":"444","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.402282,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4024298,"name":"drain","context":{"idset":"470","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4025736,"name":"drain","context":{"idset":"471","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4027195,"name":"drain","context":{"idset":"472","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4028645,"name":"drain","context":{"idset":"473","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4030776,"name":"drain","context":{"idset":"474","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4032259,"name":"drain","context":{"idset":"475","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4033997,"name":"drain","context":{"idset":"476","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4035721,"name":"drain","context":{"idset":"477","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4037168,"name":"drain","context":{"idset":"478","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4038727,"name":"drain","context":{"idset":"479","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4040208,"name":"drain","context":{"idset":"480","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4042041,"name":"drain","context":{"idset":"481","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.404381,"name":"drain","context":{"idset":"482","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.40453,"name":"drain","context":{"idset":"483","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4046922,"name":"drain","context":{"idset":"484","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.404839,"name":"drain","context":{"idset":"485","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4049852,"name":"drain","context":{"idset":"486","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4051323,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4053049,"name":"drain","context":{"idset":"488","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.405457,"name":"drain","context":{"idset":"489","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4056122,"name":"drain","context":{"idset":"490","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4057639,"name":"drain","context":{"idset":"491","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4059141,"name":"drain","context":{"idset":"492","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4060652,"name":"drain","context":{"idset":"493","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4062445,"name":"drain","context":{"idset":"494","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4064295,"name":"drain","context":{"idset":"495","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4065814,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4067311,"name":"drain","context":{"idset":"497","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4068837,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4070315,"name":"drain","context":{"idset":"499","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4071879,"name":"drain","context":{"idset":"500","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4073882,"name":"drain","context":{"idset":"501","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4075494,"name":"drain","context":{"idset":"502","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4077542,"name":"drain","context":{"idset":"503","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4079185,"name":"drain","context":{"idset":"504","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4080729,"name":"drain","context":{"idset":"505","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4082301,"name":"drain","context":{"idset":"506","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4083982,"name":"drain","context":{"idset":"507","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4085517,"name":"drain","context":{"idset":"508","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4087133,"name":"drain","context":{"idset":"509","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4088769,"name":"drain","context":{"idset":"510","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4090452,"name":"drain","context":{"idset":"511","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.409205,"name":"drain","context":{"idset":"512","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4094014,"name":"drain","context":{"idset":"513","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4095597,"name":"drain","context":{"idset":"514","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4097145,"name":"drain","context":{"idset":"515","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.409888,"name":"drain","context":{"idset":"516","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4100475,"name":"drain","context":{"idset":"517","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4102008,"name":"drain","context":{"idset":"518","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4103744,"name":"drain","context":{"idset":"519","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.410562,"name":"drain","context":{"idset":"520","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4107194,"name":"drain","context":{"idset":"521","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.410881,"name":"drain","context":{"idset":"522","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4110355,"name":"drain","context":{"idset":"523","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4111915,"name":"drain","context":{"idset":"524","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.411365,"name":"drain","context":{"idset":"525","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4114976,"name":"drain","context":{"idset":"526","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4116533,"name":"drain","context":{"idset":"527","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4117956,"name":"drain","context":{"idset":"528","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4119246,"name":"drain","context":{"idset":"529","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4120481,"name":"drain","context":{"idset":"530","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4124744,"name":"drain","context":{"idset":"531","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4126642,"name":"drain","context":{"idset":"532","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4128041,"name":"drain","context":{"idset":"533","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4129481,"name":"drain","context":{"idset":"534","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.413084,"name":"drain","context":{"idset":"535","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4132168,"name":"drain","context":{"idset":"536","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4135113,"name":"drain","context":{"idset":"537","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4136524,"name":"drain","context":{"idset":"538","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4137995,"name":"drain","context":{"idset":"539","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4139585,"name":"drain","context":{"idset":"540","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4141085,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4142485,"name":"drain","context":{"idset":"566","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4144087,"name":"drain","context":{"idset":"567","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4146285,"name":"drain","context":{"idset":"568","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.414783,"name":"drain","context":{"idset":"569","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4149103,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4150462,"name":"drain","context":{"idset":"571","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4151864,"name":"drain","context":{"idset":"572","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4153454,"name":"drain","context":{"idset":"573","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4155881,"name":"drain","context":{"idset":"574","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4157362,"name":"drain","context":{"idset":"575","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4158909,"name":"drain","context":{"idset":"576","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4160161,"name":"drain","context":{"idset":"577","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4161532,"name":"drain","context":{"idset":"578","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4163041,"name":"drain","context":{"idset":"579","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4164567,"name":"drain","context":{"idset":"580","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4166019,"name":"drain","context":{"idset":"581","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4167452,"name":"drain","context":{"idset":"582","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4168854,"name":"drain","context":{"idset":"583","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.417007,"name":"drain","context":{"idset":"584","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4171913,"name":"drain","context":{"idset":"585","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4173791,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4177256,"name":"drain","context":{"idset":"587","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4178805,"name":"drain","context":{"idset":"588","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4180377,"name":"drain","context":{"idset":"629","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4183135,"name":"drain","context":{"idset":"630","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4186101,"name":"drain","context":{"idset":"631","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.418788,"name":"drain","context":{"idset":"632","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4189606,"name":"drain","context":{"idset":"633","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4191895,"name":"drain","context":{"idset":"634","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4193783,"name":"drain","context":{"idset":"635","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.419553,"name":"drain","context":{"idset":"636","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.419734,"name":"drain","context":{"idset":"827","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4199116,"name":"drain","context":{"idset":"828","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4200871,"name":"drain","context":{"idset":"829","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.420258,"name":"drain","context":{"idset":"830","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4204803,"name":"drain","context":{"idset":"831","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4206586,"name":"drain","context":{"idset":"832","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4208345,"name":"drain","context":{"idset":"833","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.421032,"name":"drain","context":{"idset":"834","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4212046,"name":"drain","context":{"idset":"841","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4213922,"name":"drain","context":{"idset":"842","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4215784,"name":"drain","context":{"idset":"871","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.421757,"name":"drain","context":{"idset":"872","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4219334,"name":"drain","context":{"idset":"949","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4221148,"name":"drain","context":{"idset":"950","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4223084,"name":"drain","context":{"idset":"951","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4225245,"name":"drain","context":{"idset":"952","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4227569,"name":"drain","context":{"idset":"953","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4229372,"name":"drain","context":{"idset":"954","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4231133,"name":"drain","context":{"idset":"963","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4233217,"name":"drain","context":{"idset":"964","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4235027,"name":"drain","context":{"idset":"975","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4236865,"name":"drain","context":{"idset":"976","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4240711,"name":"drain","context":{"idset":"9205","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4244797,"name":"drain","context":{"idset":"9206","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4248848,"name":"drain","context":{"idset":"9207","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4252806,"name":"drain","context":{"idset":"9208","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4257357,"name":"drain","context":{"idset":"9209","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4262097,"name":"drain","context":{"idset":"9210","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4266963,"name":"drain","context":{"idset":"9211","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4270704,"name":"drain","context":{"idset":"9212","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.427567,"name":"drain","context":{"idset":"9213","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4279916,"name":"drain","context":{"idset":"9215","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4283803,"name":"drain","context":{"idset":"9216","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4287434,"name":"drain","context":{"idset":"9217","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4291246,"name":"drain","context":{"idset":"9218","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4295785,"name":"drain","context":{"idset":"9219","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4299481,"name":"drain","context":{"idset":"9220","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4304125,"name":"drain","context":{"idset":"9221","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4308496,"name":"drain","context":{"idset":"9222","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4312251,"name":"drain","context":{"idset":"9223","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4316187,"name":"drain","context":{"idset":"9224","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.43209,"name":"drain","context":{"idset":"9225","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4325655,"name":"drain","context":{"idset":"9226","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.432936,"name":"drain","context":{"idset":"9227","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4334524,"name":"drain","context":{"idset":"9228","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4338262,"name":"drain","context":{"idset":"9229","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4342551,"name":"drain","context":{"idset":"9230","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4346671,"name":"drain","context":{"idset":"9231","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4350457,"name":"drain","context":{"idset":"9232","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4355042,"name":"drain","context":{"idset":"9233","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4358885,"name":"drain","context":{"idset":"9234","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4362898,"name":"drain","context":{"idset":"9235","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4366608,"name":"drain","context":{"idset":"9236","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4370184,"name":"drain","context":{"idset":"9237","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.437609,"name":"drain","context":{"idset":"9238","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4379809,"name":"drain","context":{"idset":"9241","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4383879,"name":"drain","context":{"idset":"9242","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4387817,"name":"drain","context":{"idset":"9243","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4391663,"name":"drain","context":{"idset":"9244","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4398286,"name":"drain","context":{"idset":"9245","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.440382,"name":"drain","context":{"idset":"9246","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.440794,"name":"drain","context":{"idset":"9247","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4411807,"name":"drain","context":{"idset":"9248","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4415472,"name":"drain","context":{"idset":"9249","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.442044,"name":"drain","context":{"idset":"9250","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4424505,"name":"drain","context":{"idset":"9251","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4427917,"name":"drain","context":{"idset":"9252","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4431643,"name":"drain","context":{"idset":"9253","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4436004,"name":"drain","context":{"idset":"9254","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4440424,"name":"drain","context":{"idset":"9255","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4444661,"name":"drain","context":{"idset":"9256","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4448576,"name":"drain","context":{"idset":"9257","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4452541,"name":"drain","context":{"idset":"9258","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4457576,"name":"drain","context":{"idset":"9259","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4461677,"name":"drain","context":{"idset":"9260","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4466767,"name":"drain","context":{"idset":"9261","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4470756,"name":"drain","context":{"idset":"9262","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4474986,"name":"drain","context":{"idset":"9263","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4479663,"name":"drain","context":{"idset":"9264","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4484727,"name":"drain","context":{"idset":"9265","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.448854,"name":"drain","context":{"idset":"9266","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4491422,"name":"drain","context":{"idset":"9267","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4495013,"name":"drain","context":{"idset":"9268","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4497898,"name":"drain","context":{"idset":"9269","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4500897,"name":"drain","context":{"idset":"9270","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4504585,"name":"drain","context":{"idset":"9271","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4509315,"name":"drain","context":{"idset":"9272","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4515676,"name":"drain","context":{"idset":"9273","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4519928,"name":"drain","context":{"idset":"9274","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4525833,"name":"drain","context":{"idset":"9275","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4530361,"name":"drain","context":{"idset":"9276","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4537003,"name":"drain","context":{"idset":"9277","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4540243,"name":"drain","context":{"idset":"9278","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4544103,"name":"drain","context":{"idset":"9279","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4548154,"name":"drain","context":{"idset":"9280","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4552166,"name":"drain","context":{"idset":"9281","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4556277,"name":"drain","context":{"idset":"9282","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4560907,"name":"drain","context":{"idset":"9283","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4565883,"name":"drain","context":{"idset":"9284","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.457,"name":"drain","context":{"idset":"9285","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4575579,"name":"drain","context":{"idset":"9286","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4579666,"name":"drain","context":{"idset":"9287","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4584653,"name":"drain","context":{"idset":"9288","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4587929,"name":"drain","context":{"idset":"9289","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4591343,"name":"drain","context":{"idset":"9290","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4595463,"name":"drain","context":{"idset":"9291","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4602227,"name":"drain","context":{"idset":"9292","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4605727,"name":"drain","context":{"idset":"9293","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4609063,"name":"drain","context":{"idset":"9294","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4612339,"name":"drain","context":{"idset":"9295","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4618273,"name":"drain","context":{"idset":"9296","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4621232,"name":"drain","context":{"idset":"9297","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4626327,"name":"drain","context":{"idset":"9298","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4630525,"name":"drain","context":{"idset":"9299","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4634125,"name":"drain","context":{"idset":"9300","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4638762,"name":"drain","context":{"idset":"9301","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4643002,"name":"drain","context":{"idset":"9302","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4647658,"name":"drain","context":{"idset":"9303","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4651732,"name":"drain","context":{"idset":"9304","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4656982,"name":"drain","context":{"idset":"9305","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4661369,"name":"drain","context":{"idset":"9306","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4668345,"name":"drain","context":{"idset":"9307","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4672384,"name":"drain","context":{"idset":"9308","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4677656,"name":"drain","context":{"idset":"9309","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4681835,"name":"drain","context":{"idset":"9310","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4688013,"name":"drain","context":{"idset":"9311","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4692225,"name":"drain","context":{"idset":"9312","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4697118,"name":"drain","context":{"idset":"9313","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4701681,"name":"drain","context":{"idset":"9314","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.47068,"name":"drain","context":{"idset":"9315","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4711282,"name":"drain","context":{"idset":"9317","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4716065,"name":"drain","context":{"idset":"9319","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4720688,"name":"drain","context":{"idset":"9320","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4726772,"name":"drain","context":{"idset":"9321","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4730909,"name":"drain","context":{"idset":"9322","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4736769,"name":"drain","context":{"idset":"9323","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4740834,"name":"drain","context":{"idset":"9324","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.47451,"name":"drain","context":{"idset":"9325","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.474982,"name":"drain","context":{"idset":"9326","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4754097,"name":"drain","context":{"idset":"9327","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4758375,"name":"drain","context":{"idset":"9328","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4762502,"name":"drain","context":{"idset":"9329","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4767287,"name":"drain","context":{"idset":"9330","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4771926,"name":"drain","context":{"idset":"9331","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4776211,"name":"drain","context":{"idset":"9332","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4780984,"name":"drain","context":{"idset":"9333","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4786747,"name":"drain","context":{"idset":"9334","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4791863,"name":"drain","context":{"idset":"9335","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4801905,"name":"drain","context":{"idset":"9336","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4809048,"name":"drain","context":{"idset":"9337","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4814665,"name":"drain","context":{"idset":"9338","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4820144,"name":"drain","context":{"idset":"9339","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4826171,"name":"drain","context":{"idset":"9340","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4830315,"name":"drain","context":{"idset":"9341","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4836674,"name":"drain","context":{"idset":"9342","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4840889,"name":"drain","context":{"idset":"9343","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4845426,"name":"drain","context":{"idset":"9344","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4850006,"name":"drain","context":{"idset":"9345","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4858665,"name":"drain","context":{"idset":"9346","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4862971,"name":"drain","context":{"idset":"9347","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4867523,"name":"drain","context":{"idset":"9348","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4871736,"name":"drain","context":{"idset":"9349","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4877055,"name":"drain","context":{"idset":"9350","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4881728,"name":"drain","context":{"idset":"9351","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4888408,"name":"drain","context":{"idset":"9352","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4893146,"name":"drain","context":{"idset":"9353","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4897242,"name":"drain","context":{"idset":"9354","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.490144,"name":"drain","context":{"idset":"9355","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4906251,"name":"drain","context":{"idset":"9356","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4910419,"name":"drain","context":{"idset":"9357","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4914837,"name":"drain","context":{"idset":"9358","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4918933,"name":"drain","context":{"idset":"9359","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.49246,"name":"drain","context":{"idset":"9360","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4928703,"name":"drain","context":{"idset":"9361","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4935155,"name":"drain","context":{"idset":"9362","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4940257,"name":"drain","context":{"idset":"9363","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4944518,"name":"drain","context":{"idset":"9364","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.49491,"name":"drain","context":{"idset":"9365","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4954171,"name":"drain","context":{"idset":"9366","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.495743,"name":"drain","context":{"idset":"9367","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4960709,"name":"drain","context":{"idset":"9368","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4964483,"name":"drain","context":{"idset":"9369","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4971969,"name":"drain","context":{"idset":"9370","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.497663,"name":"drain","context":{"idset":"9371","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4980841,"name":"drain","context":{"idset":"9372","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4985516,"name":"drain","context":{"idset":"9373","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.498924,"name":"drain","context":{"idset":"9374","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4992588,"name":"drain","context":{"idset":"9375","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.4998584,"name":"drain","context":{"idset":"9376","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.500319,"name":"drain","context":{"idset":"9377","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.500819,"name":"drain","context":{"idset":"9378","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5012109,"name":"drain","context":{"idset":"9379","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5016508,"name":"drain","context":{"idset":"9380","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5020139,"name":"drain","context":{"idset":"9381","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5024059,"name":"drain","context":{"idset":"9382","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5028613,"name":"drain","context":{"idset":"9383","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5032423,"name":"drain","context":{"idset":"9384","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5037513,"name":"drain","context":{"idset":"9385","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5040801,"name":"drain","context":{"idset":"9386","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.504662,"name":"drain","context":{"idset":"9387","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.505127,"name":"drain","context":{"idset":"9388","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5057149,"name":"drain","context":{"idset":"9389","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5078933,"name":"drain","context":{"idset":"9390","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5082381,"name":"drain","context":{"idset":"9391","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5086193,"name":"drain","context":{"idset":"9392","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5089734,"name":"drain","context":{"idset":"9393","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5093369,"name":"drain","context":{"idset":"9394","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5110004,"name":"drain","context":{"idset":"9395","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5132403,"name":"drain","context":{"idset":"9396","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5136113,"name":"drain","context":{"idset":"9397","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5140085,"name":"drain","context":{"idset":"9398","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.515357,"name":"drain","context":{"idset":"9399","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5157962,"name":"drain","context":{"idset":"9400","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5162015,"name":"drain","context":{"idset":"9401","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5179944,"name":"drain","context":{"idset":"9402","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5184159,"name":"drain","context":{"idset":"9403","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.518997,"name":"drain","context":{"idset":"9404","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5203855,"name":"drain","context":{"idset":"9405","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5208035,"name":"drain","context":{"idset":"9406","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.521184,"name":"drain","context":{"idset":"9407","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5225856,"name":"drain","context":{"idset":"9408","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5230222,"name":"drain","context":{"idset":"9409","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5249023,"name":"drain","context":{"idset":"9410","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5259497,"name":"drain","context":{"idset":"9411","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5277131,"name":"drain","context":{"idset":"9412","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5284894,"name":"drain","context":{"idset":"9413","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5289538,"name":"drain","context":{"idset":"9414","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5305369,"name":"drain","context":{"idset":"9415","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5328362,"name":"drain","context":{"idset":"9416","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5333581,"name":"drain","context":{"idset":"9417","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5338469,"name":"drain","context":{"idset":"9418","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5356762,"name":"drain","context":{"idset":"9419","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5361302,"name":"drain","context":{"idset":"9420","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5366476,"name":"drain","context":{"idset":"9421","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5371513,"name":"drain","context":{"idset":"9422","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5376091,"name":"drain","context":{"idset":"9423","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5380669,"name":"drain","context":{"idset":"9424","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5385683,"name":"drain","context":{"idset":"9425","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5390239,"name":"drain","context":{"idset":"9426","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5395031,"name":"drain","context":{"idset":"9427","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5399678,"name":"drain","context":{"idset":"9428","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5405467,"name":"drain","context":{"idset":"9429","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5410104,"name":"drain","context":{"idset":"9430","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5415626,"name":"drain","context":{"idset":"9431","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5420296,"name":"drain","context":{"idset":"9432","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5425594,"name":"drain","context":{"idset":"9433","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5430665,"name":"drain","context":{"idset":"9434","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5436206,"name":"drain","context":{"idset":"9435","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5441587,"name":"drain","context":{"idset":"9436","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5447078,"name":"drain","context":{"idset":"9437","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5451872,"name":"drain","context":{"idset":"9438","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5456619,"name":"drain","context":{"idset":"9439","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5461817,"name":"drain","context":{"idset":"9440","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5467134,"name":"drain","context":{"idset":"9441","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5475354,"name":"drain","context":{"idset":"9442","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5479963,"name":"drain","context":{"idset":"9443","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5486171,"name":"drain","context":{"idset":"9444","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5491457,"name":"drain","context":{"idset":"9445","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5496306,"name":"drain","context":{"idset":"9446","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5501211,"name":"drain","context":{"idset":"9449","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5505986,"name":"drain","context":{"idset":"9450","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5510817,"name":"drain","context":{"idset":"9451","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5517776,"name":"drain","context":{"idset":"9452","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.552258,"name":"drain","context":{"idset":"9453","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5528102,"name":"drain","context":{"idset":"9454","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5534549,"name":"drain","context":{"idset":"9455","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5539355,"name":"drain","context":{"idset":"9456","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5544696,"name":"drain","context":{"idset":"9457","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5549793,"name":"drain","context":{"idset":"9458","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5556476,"name":"drain","context":{"idset":"9459","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5561316,"name":"drain","context":{"idset":"9460","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5567434,"name":"drain","context":{"idset":"9461","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5572147,"name":"drain","context":{"idset":"9462","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5577908,"name":"drain","context":{"idset":"9463","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.558306,"name":"drain","context":{"idset":"9464","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5588448,"name":"drain","context":{"idset":"9465","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5593591,"name":"drain","context":{"idset":"9466","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5598423,"name":"drain","context":{"idset":"9467","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5603659,"name":"drain","context":{"idset":"9468","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5609405,"name":"drain","context":{"idset":"9469","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5615442,"name":"drain","context":{"idset":"9470","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5620303,"name":"drain","context":{"idset":"9471","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.562511,"name":"drain","context":{"idset":"9472","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5629992,"name":"drain","context":{"idset":"9473","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5634956,"name":"drain","context":{"idset":"9474","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5639746,"name":"drain","context":{"idset":"9475","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5646012,"name":"drain","context":{"idset":"9476","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5650754,"name":"drain","context":{"idset":"9477","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.565546,"name":"drain","context":{"idset":"9478","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5660093,"name":"drain","context":{"idset":"9479","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5665591,"name":"drain","context":{"idset":"9480","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5670388,"name":"drain","context":{"idset":"9481","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5675557,"name":"drain","context":{"idset":"9482","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5680144,"name":"drain","context":{"idset":"9483","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5684915,"name":"drain","context":{"idset":"9484","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5689502,"name":"drain","context":{"idset":"9485","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5694935,"name":"drain","context":{"idset":"9486","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5700195,"name":"drain","context":{"idset":"9487","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5705104,"name":"drain","context":{"idset":"9488","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5709865,"name":"drain","context":{"idset":"9489","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5714815,"name":"drain","context":{"idset":"9490","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5719678,"name":"drain","context":{"idset":"9491","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5724492,"name":"drain","context":{"idset":"9492","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5735285,"name":"drain","context":{"idset":"9493","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5739934,"name":"drain","context":{"idset":"9494","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5745039,"name":"drain","context":{"idset":"9495","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5749669,"name":"drain","context":{"idset":"9496","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.575578,"name":"drain","context":{"idset":"9497","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5759895,"name":"drain","context":{"idset":"9498","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5764229,"name":"drain","context":{"idset":"9499","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5768054,"name":"drain","context":{"idset":"9500","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5773063,"name":"drain","context":{"idset":"9501","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5778768,"name":"drain","context":{"idset":"9502","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5782549,"name":"drain","context":{"idset":"9503","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5788879,"name":"drain","context":{"idset":"9504","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5794947,"name":"drain","context":{"idset":"9505","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5799835,"name":"drain","context":{"idset":"9506","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5805779,"name":"drain","context":{"idset":"9507","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5810776,"name":"drain","context":{"idset":"9508","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5815926,"name":"drain","context":{"idset":"9509","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5820904,"name":"drain","context":{"idset":"9510","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5826688,"name":"drain","context":{"idset":"9511","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5831735,"name":"drain","context":{"idset":"9512","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5836816,"name":"drain","context":{"idset":"9513","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5841696,"name":"drain","context":{"idset":"9514","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5846608,"name":"drain","context":{"idset":"9515","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5851548,"name":"drain","context":{"idset":"9516","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5857217,"name":"drain","context":{"idset":"9517","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5861421,"name":"drain","context":{"idset":"9518","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5865221,"name":"drain","context":{"idset":"9519","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5870118,"name":"drain","context":{"idset":"9520","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5875173,"name":"drain","context":{"idset":"9521","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5879161,"name":"drain","context":{"idset":"9522","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5883632,"name":"drain","context":{"idset":"9523","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5887747,"name":"drain","context":{"idset":"9524","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5892105,"name":"drain","context":{"idset":"9525","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5896997,"name":"drain","context":{"idset":"9526","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5901771,"name":"drain","context":{"idset":"9527","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5907214,"name":"drain","context":{"idset":"9528","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5910633,"name":"drain","context":{"idset":"9529","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5916431,"name":"drain","context":{"idset":"9530","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5920808,"name":"drain","context":{"idset":"9531","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5925179,"name":"drain","context":{"idset":"9532","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5929933,"name":"drain","context":{"idset":"9536","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.593456,"name":"drain","context":{"idset":"9537","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5939155,"name":"drain","context":{"idset":"9538","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5944057,"name":"drain","context":{"idset":"9539","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5947878,"name":"drain","context":{"idset":"9540","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5952289,"name":"drain","context":{"idset":"9541","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5956557,"name":"drain","context":{"idset":"9542","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.596036,"name":"drain","context":{"idset":"9543","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5964518,"name":"drain","context":{"idset":"9544","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.596833,"name":"drain","context":{"idset":"9545","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5973616,"name":"drain","context":{"idset":"9546","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.59798,"name":"drain","context":{"idset":"9547","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5985184,"name":"drain","context":{"idset":"9548","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5990689,"name":"drain","context":{"idset":"9549","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5995681,"name":"drain","context":{"idset":"9550","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.5999689,"name":"drain","context":{"idset":"9551","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6003947,"name":"drain","context":{"idset":"9552","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6009047,"name":"drain","context":{"idset":"9553","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6014125,"name":"drain","context":{"idset":"9554","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6019197,"name":"drain","context":{"idset":"9555","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6025891,"name":"drain","context":{"idset":"9556","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6031196,"name":"drain","context":{"idset":"9557","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6037018,"name":"drain","context":{"idset":"9558","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6042554,"name":"drain","context":{"idset":"9559","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6047721,"name":"drain","context":{"idset":"9560","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.605412,"name":"drain","context":{"idset":"9563","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6059382,"name":"drain","context":{"idset":"9564","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.606468,"name":"drain","context":{"idset":"9565","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6069913,"name":"drain","context":{"idset":"9566","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6075294,"name":"drain","context":{"idset":"9567","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6080773,"name":"drain","context":{"idset":"9568","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.608799,"name":"drain","context":{"idset":"9569","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6091816,"name":"drain","context":{"idset":"9570","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6097584,"name":"drain","context":{"idset":"9571","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6101575,"name":"drain","context":{"idset":"9572","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6106851,"name":"drain","context":{"idset":"9573","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6111288,"name":"drain","context":{"idset":"9574","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6116297,"name":"drain","context":{"idset":"9575","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6121957,"name":"drain","context":{"idset":"9576","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6127989,"name":"drain","context":{"idset":"9577","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6133068,"name":"drain","context":{"idset":"9578","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6140416,"name":"drain","context":{"idset":"9579","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6146111,"name":"drain","context":{"idset":"9580","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6151164,"name":"drain","context":{"idset":"9581","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6158824,"name":"drain","context":{"idset":"9582","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6164138,"name":"drain","context":{"idset":"9583","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6169333,"name":"drain","context":{"idset":"9584","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6174521,"name":"drain","context":{"idset":"9585","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6180077,"name":"drain","context":{"idset":"9586","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.618578,"name":"drain","context":{"idset":"9587","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6191862,"name":"drain","context":{"idset":"9588","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6198902,"name":"drain","context":{"idset":"9589","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6204967,"name":"drain","context":{"idset":"9590","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6211293,"name":"drain","context":{"idset":"9591","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6217844,"name":"drain","context":{"idset":"9592","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6225753,"name":"drain","context":{"idset":"9593","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6231692,"name":"drain","context":{"idset":"9594","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6236565,"name":"drain","context":{"idset":"9595","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.624058,"name":"drain","context":{"idset":"9596","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6245377,"name":"drain","context":{"idset":"9597","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6249378,"name":"drain","context":{"idset":"9598","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6254313,"name":"drain","context":{"idset":"9599","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6260147,"name":"drain","context":{"idset":"9600","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6268964,"name":"drain","context":{"idset":"9601","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6276631,"name":"drain","context":{"idset":"9602","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6282496,"name":"drain","context":{"idset":"9603","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6286542,"name":"drain","context":{"idset":"9604","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6291559,"name":"drain","context":{"idset":"9605","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6297154,"name":"drain","context":{"idset":"9606","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6302342,"name":"drain","context":{"idset":"9607","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6310441,"name":"drain","context":{"idset":"9608","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6317873,"name":"drain","context":{"idset":"9609","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6324377,"name":"drain","context":{"idset":"9610","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6334026,"name":"drain","context":{"idset":"9611","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6340184,"name":"drain","context":{"idset":"9612","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6347301,"name":"drain","context":{"idset":"9613","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6355727,"name":"drain","context":{"idset":"9614","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6365232,"name":"drain","context":{"idset":"9615","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6376364,"name":"drain","context":{"idset":"9616","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6384926,"name":"drain","context":{"idset":"9617","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6393318,"name":"drain","context":{"idset":"9618","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6401289,"name":"drain","context":{"idset":"9619","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6409531,"name":"drain","context":{"idset":"9620","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.641696,"name":"drain","context":{"idset":"9621","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6424415,"name":"drain","context":{"idset":"9622","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.643436,"name":"drain","context":{"idset":"9623","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6441123,"name":"drain","context":{"idset":"9624","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6450303,"name":"drain","context":{"idset":"9625","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6458962,"name":"drain","context":{"idset":"9626","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6467097,"name":"drain","context":{"idset":"9627","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.647675,"name":"drain","context":{"idset":"9628","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6485553,"name":"drain","context":{"idset":"9629","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6493628,"name":"drain","context":{"idset":"9630","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6500392,"name":"drain","context":{"idset":"9631","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6510346,"name":"drain","context":{"idset":"9632","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6517828,"name":"drain","context":{"idset":"9635","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6524794,"name":"drain","context":{"idset":"9636","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.65326,"name":"drain","context":{"idset":"9637","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6542301,"name":"drain","context":{"idset":"9638","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6556139,"name":"drain","context":{"idset":"9639","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6565633,"name":"drain","context":{"idset":"9640","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6575172,"name":"drain","context":{"idset":"9641","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6584578,"name":"drain","context":{"idset":"9642","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6594031,"name":"drain","context":{"idset":"9643","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6601059,"name":"drain","context":{"idset":"9644","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6608541,"name":"drain","context":{"idset":"9645","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6617565,"name":"drain","context":{"idset":"9646","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6627574,"name":"drain","context":{"idset":"9647","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6635478,"name":"drain","context":{"idset":"9648","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6641722,"name":"drain","context":{"idset":"9649","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6649704,"name":"drain","context":{"idset":"9650","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.665534,"name":"drain","context":{"idset":"9651","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6661282,"name":"drain","context":{"idset":"9652","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6669075,"name":"drain","context":{"idset":"9653","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6677008,"name":"drain","context":{"idset":"9654","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6686871,"name":"drain","context":{"idset":"9655","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6695714,"name":"drain","context":{"idset":"9656","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6700115,"name":"drain","context":{"idset":"9657","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6706882,"name":"drain","context":{"idset":"9658","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6714149,"name":"drain","context":{"idset":"9659","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6718888,"name":"drain","context":{"idset":"9660","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6727097,"name":"drain","context":{"idset":"9661","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6734591,"name":"drain","context":{"idset":"9662","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.674196,"name":"drain","context":{"idset":"9663","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6751981,"name":"drain","context":{"idset":"9664","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6759763,"name":"drain","context":{"idset":"9665","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6768224,"name":"drain","context":{"idset":"9666","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6775379,"name":"drain","context":{"idset":"9667","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6784806,"name":"drain","context":{"idset":"9668","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6791935,"name":"drain","context":{"idset":"9669","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6798768,"name":"drain","context":{"idset":"9670","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6806815,"name":"drain","context":{"idset":"9671","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6815331,"name":"drain","context":{"idset":"9672","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6825919,"name":"drain","context":{"idset":"9673","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6837525,"name":"drain","context":{"idset":"9674","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6848657,"name":"drain","context":{"idset":"9675","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6859012,"name":"drain","context":{"idset":"9676","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6868856,"name":"drain","context":{"idset":"9677","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6879621,"name":"drain","context":{"idset":"9678","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6890056,"name":"drain","context":{"idset":"9679","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6898997,"name":"drain","context":{"idset":"9680","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6907516,"name":"drain","context":{"idset":"9681","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6919243,"name":"drain","context":{"idset":"9682","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6930156,"name":"drain","context":{"idset":"9683","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6939566,"name":"drain","context":{"idset":"9684","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6948395,"name":"drain","context":{"idset":"9685","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6959884,"name":"drain","context":{"idset":"9686","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6968734,"name":"drain","context":{"idset":"9687","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6983235,"name":"drain","context":{"idset":"9688","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.6995218,"name":"drain","context":{"idset":"9689","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7006547,"name":"drain","context":{"idset":"9690","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7015717,"name":"drain","context":{"idset":"9691","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7026448,"name":"drain","context":{"idset":"9692","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7039857,"name":"drain","context":{"idset":"9693","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7047949,"name":"drain","context":{"idset":"9694","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7057507,"name":"drain","context":{"idset":"9695","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.706737,"name":"drain","context":{"idset":"9696","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7075799,"name":"drain","context":{"idset":"9697","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7087471,"name":"drain","context":{"idset":"9698","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7095466,"name":"drain","context":{"idset":"9699","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7105594,"name":"drain","context":{"idset":"9700","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7116566,"name":"drain","context":{"idset":"9701","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7127407,"name":"drain","context":{"idset":"9702","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7135301,"name":"drain","context":{"idset":"9703","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7145512,"name":"drain","context":{"idset":"9704","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7157209,"name":"drain","context":{"idset":"9705","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7166522,"name":"drain","context":{"idset":"9706","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7177105,"name":"drain","context":{"idset":"9707","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7186487,"name":"drain","context":{"idset":"9708","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7196293,"name":"drain","context":{"idset":"9709","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7207906,"name":"drain","context":{"idset":"9710","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7218137,"name":"drain","context":{"idset":"9711","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7228389,"name":"drain","context":{"idset":"9712","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7237203,"name":"drain","context":{"idset":"9713","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7245407,"name":"drain","context":{"idset":"9714","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.72559,"name":"drain","context":{"idset":"9715","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7265925,"name":"drain","context":{"idset":"9716","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7276943,"name":"drain","context":{"idset":"9717","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7287116,"name":"drain","context":{"idset":"9718","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.729661,"name":"drain","context":{"idset":"9719","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7306712,"name":"drain","context":{"idset":"9720","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7318008,"name":"drain","context":{"idset":"9721","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7328563,"name":"drain","context":{"idset":"9722","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7339807,"name":"drain","context":{"idset":"9723","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7350657,"name":"drain","context":{"idset":"9724","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7360044,"name":"drain","context":{"idset":"9727","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7369852,"name":"drain","context":{"idset":"9728","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7380934,"name":"drain","context":{"idset":"9729","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7392356,"name":"drain","context":{"idset":"9730","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7403445,"name":"drain","context":{"idset":"9731","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7411788,"name":"drain","context":{"idset":"9732","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7422028,"name":"drain","context":{"idset":"9733","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7431667,"name":"drain","context":{"idset":"9734","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7438679,"name":"drain","context":{"idset":"9735","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7445436,"name":"drain","context":{"idset":"9736","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7454035,"name":"drain","context":{"idset":"9737","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7462902,"name":"drain","context":{"idset":"9738","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7470479,"name":"drain","context":{"idset":"9739","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.747931,"name":"drain","context":{"idset":"9740","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7490399,"name":"drain","context":{"idset":"9741","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7499862,"name":"drain","context":{"idset":"9742","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7511427,"name":"drain","context":{"idset":"9743","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7520266,"name":"drain","context":{"idset":"9744","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7532513,"name":"drain","context":{"idset":"9745","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7542973,"name":"drain","context":{"idset":"9746","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7552178,"name":"drain","context":{"idset":"9747","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7560949,"name":"drain","context":{"idset":"9748","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7570271,"name":"drain","context":{"idset":"9749","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.759093,"name":"drain","context":{"idset":"9751","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7600777,"name":"drain","context":{"idset":"9752","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7611661,"name":"drain","context":{"idset":"9753","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7620285,"name":"drain","context":{"idset":"9754","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7630563,"name":"drain","context":{"idset":"9755","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7642231,"name":"drain","context":{"idset":"9756","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7650092,"name":"drain","context":{"idset":"9757","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7660897,"name":"drain","context":{"idset":"9758","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7669017,"name":"drain","context":{"idset":"9759","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7678516,"name":"drain","context":{"idset":"9760","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7687263,"name":"drain","context":{"idset":"9761","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7695758,"name":"drain","context":{"idset":"9762","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7705801,"name":"drain","context":{"idset":"9763","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.771498,"name":"drain","context":{"idset":"9764","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7722557,"name":"drain","context":{"idset":"9765","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7732019,"name":"drain","context":{"idset":"9766","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7745476,"name":"drain","context":{"idset":"9767","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7756236,"name":"drain","context":{"idset":"9768","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7766349,"name":"drain","context":{"idset":"9769","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7776141,"name":"drain","context":{"idset":"9770","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7785385,"name":"drain","context":{"idset":"9771","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7796903,"name":"drain","context":{"idset":"9772","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7808321,"name":"drain","context":{"idset":"9773","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7818723,"name":"drain","context":{"idset":"9774","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7828486,"name":"drain","context":{"idset":"9775","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.783947,"name":"drain","context":{"idset":"9776","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7848368,"name":"drain","context":{"idset":"9777","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7859111,"name":"drain","context":{"idset":"9778","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7870233,"name":"drain","context":{"idset":"9779","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7880661,"name":"drain","context":{"idset":"9780","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7892089,"name":"drain","context":{"idset":"9781","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7903476,"name":"drain","context":{"idset":"9782","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7911375,"name":"drain","context":{"idset":"9783","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.792048,"name":"drain","context":{"idset":"9784","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.793205,"name":"drain","context":{"idset":"9785","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7942462,"name":"drain","context":{"idset":"9786","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7955399,"name":"drain","context":{"idset":"9787","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7969432,"name":"drain","context":{"idset":"9788","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7980368,"name":"drain","context":{"idset":"9789","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.7994246,"name":"drain","context":{"idset":"9790","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8006232,"name":"drain","context":{"idset":"9791","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8016462,"name":"drain","context":{"idset":"9792","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8027174,"name":"drain","context":{"idset":"9793","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8036475,"name":"drain","context":{"idset":"9794","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8046682,"name":"drain","context":{"idset":"9795","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8056872,"name":"drain","context":{"idset":"9796","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.806854,"name":"drain","context":{"idset":"9797","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8082078,"name":"drain","context":{"idset":"9798","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8092299,"name":"drain","context":{"idset":"9799","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8101943,"name":"drain","context":{"idset":"9800","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8113866,"name":"drain","context":{"idset":"9801","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8124416,"name":"drain","context":{"idset":"9802","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8133936,"name":"drain","context":{"idset":"9803","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8146818,"name":"drain","context":{"idset":"9804","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8159313,"name":"drain","context":{"idset":"9805","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8169971,"name":"drain","context":{"idset":"9806","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8178236,"name":"drain","context":{"idset":"9807","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8189671,"name":"drain","context":{"idset":"9808","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8198214,"name":"drain","context":{"idset":"9809","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8210785,"name":"drain","context":{"idset":"9810","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8223081,"name":"drain","context":{"idset":"9811","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8232138,"name":"drain","context":{"idset":"9812","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8241668,"name":"drain","context":{"idset":"9813","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8249693,"name":"drain","context":{"idset":"9814","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8260744,"name":"drain","context":{"idset":"9815","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8271914,"name":"drain","context":{"idset":"9816","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8281174,"name":"drain","context":{"idset":"9817","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8292093,"name":"drain","context":{"idset":"9818","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8303163,"name":"drain","context":{"idset":"9819","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8317347,"name":"drain","context":{"idset":"9820","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8328853,"name":"drain","context":{"idset":"9821","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8337545,"name":"drain","context":{"idset":"9822","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8348076,"name":"drain","context":{"idset":"9823","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8356807,"name":"drain","context":{"idset":"9824","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8365993,"name":"drain","context":{"idset":"9825","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8375859,"name":"drain","context":{"idset":"9826","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8385766,"name":"drain","context":{"idset":"9827","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.839519,"name":"drain","context":{"idset":"9828","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.84045,"name":"drain","context":{"idset":"9829","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8413393,"name":"drain","context":{"idset":"9830","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8422489,"name":"drain","context":{"idset":"9831","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8433137,"name":"drain","context":{"idset":"9832","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8445044,"name":"drain","context":{"idset":"9833","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8459709,"name":"drain","context":{"idset":"9834","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8468399,"name":"drain","context":{"idset":"9835","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8479373,"name":"drain","context":{"idset":"9836","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8494787,"name":"drain","context":{"idset":"9837","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8504515,"name":"drain","context":{"idset":"9838","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8514302,"name":"drain","context":{"idset":"9839","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8519793,"name":"drain","context":{"idset":"9840","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8527477,"name":"drain","context":{"idset":"9841","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8537295,"name":"drain","context":{"idset":"9842","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8548348,"name":"drain","context":{"idset":"9843","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8557465,"name":"drain","context":{"idset":"9844","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8566556,"name":"drain","context":{"idset":"9973","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8573885,"name":"drain","context":{"idset":"9974","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8583882,"name":"drain","context":{"idset":"9975","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8590829,"name":"drain","context":{"idset":"9976","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8602943,"name":"drain","context":{"idset":"9977","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.86133,"name":"drain","context":{"idset":"9978","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8623087,"name":"drain","context":{"idset":"9979","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8634999,"name":"drain","context":{"idset":"9980","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.864393,"name":"drain","context":{"idset":"9981","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8655305,"name":"drain","context":{"idset":"9982","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8665183,"name":"drain","context":{"idset":"9983","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8677766,"name":"drain","context":{"idset":"9984","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8688221,"name":"drain","context":{"idset":"9985","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.870075,"name":"drain","context":{"idset":"9986","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8710656,"name":"drain","context":{"idset":"9987","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8721669,"name":"drain","context":{"idset":"9988","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.873168,"name":"drain","context":{"idset":"9989","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8740544,"name":"drain","context":{"idset":"9990","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8751256,"name":"drain","context":{"idset":"9991","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8761213,"name":"drain","context":{"idset":"9992","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.877131,"name":"drain","context":{"idset":"9993","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8779891,"name":"drain","context":{"idset":"9994","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8791318,"name":"drain","context":{"idset":"9995","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8801911,"name":"drain","context":{"idset":"9996","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8811104,"name":"drain","context":{"idset":"9997","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8821785,"name":"drain","context":{"idset":"9998","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8832076,"name":"drain","context":{"idset":"9999","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8842978,"name":"drain","context":{"idset":"10000","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8854077,"name":"drain","context":{"idset":"10001","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.886184,"name":"drain","context":{"idset":"10002","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8871567,"name":"drain","context":{"idset":"10003","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8880069,"name":"drain","context":{"idset":"10004","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8891339,"name":"drain","context":{"idset":"10005","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8899682,"name":"drain","context":{"idset":"10006","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8910799,"name":"drain","context":{"idset":"10007","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8921547,"name":"drain","context":{"idset":"10008","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8933442,"name":"drain","context":{"idset":"10009","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8943903,"name":"drain","context":{"idset":"10010","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8955195,"name":"drain","context":{"idset":"10011","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8968134,"name":"drain","context":{"idset":"10012","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8977315,"name":"drain","context":{"idset":"10013","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8987007,"name":"drain","context":{"idset":"10014","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.8996112,"name":"drain","context":{"idset":"10015","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9009814,"name":"drain","context":{"idset":"10016","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9024131,"name":"drain","context":{"idset":"10017","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9038877,"name":"drain","context":{"idset":"10018","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9051716,"name":"drain","context":{"idset":"10019","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.906059,"name":"drain","context":{"idset":"10020","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9070358,"name":"drain","context":{"idset":"10021","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9084191,"name":"drain","context":{"idset":"10022","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9097288,"name":"drain","context":{"idset":"10023","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9107914,"name":"drain","context":{"idset":"10024","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9117928,"name":"drain","context":{"idset":"10025","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9129114,"name":"drain","context":{"idset":"10026","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9140759,"name":"drain","context":{"idset":"10027","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9151354,"name":"drain","context":{"idset":"10028","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9161718,"name":"drain","context":{"idset":"10029","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9172194,"name":"drain","context":{"idset":"10030","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9182987,"name":"drain","context":{"idset":"10031","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9194458,"name":"drain","context":{"idset":"10032","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9205503,"name":"drain","context":{"idset":"10033","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9216831,"name":"drain","context":{"idset":"10034","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9228806,"name":"drain","context":{"idset":"10035","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9238689,"name":"drain","context":{"idset":"10036","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9250596,"name":"drain","context":{"idset":"10037","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9265134,"name":"drain","context":{"idset":"10038","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9277198,"name":"drain","context":{"idset":"10039","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9290502,"name":"drain","context":{"idset":"10040","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9306262,"name":"drain","context":{"idset":"10041","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9316778,"name":"drain","context":{"idset":"10042","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9328973,"name":"drain","context":{"idset":"10043","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9345572,"name":"drain","context":{"idset":"10044","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9356456,"name":"drain","context":{"idset":"10045","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9363213,"name":"drain","context":{"idset":"10046","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9371505,"name":"drain","context":{"idset":"10047","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9378629,"name":"drain","context":{"idset":"10048","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.938406,"name":"drain","context":{"idset":"10049","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9389594,"name":"drain","context":{"idset":"10050","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9396987,"name":"drain","context":{"idset":"10051","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9404955,"name":"drain","context":{"idset":"10052","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9410958,"name":"drain","context":{"idset":"10053","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9419446,"name":"drain","context":{"idset":"10054","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9425573,"name":"drain","context":{"idset":"10055","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9434118,"name":"drain","context":{"idset":"10056","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9438851,"name":"drain","context":{"idset":"10057","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.944519,"name":"drain","context":{"idset":"10058","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9451532,"name":"drain","context":{"idset":"10059","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9458354,"name":"drain","context":{"idset":"10060","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9467926,"name":"drain","context":{"idset":"10061","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9475174,"name":"drain","context":{"idset":"10062","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9480557,"name":"drain","context":{"idset":"10063","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9489646,"name":"drain","context":{"idset":"10064","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9498603,"name":"drain","context":{"idset":"10065","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9506395,"name":"drain","context":{"idset":"10066","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9514656,"name":"drain","context":{"idset":"10067","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.952316,"name":"drain","context":{"idset":"10068","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9531081,"name":"drain","context":{"idset":"10069","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9542174,"name":"drain","context":{"idset":"10070","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9555624,"name":"drain","context":{"idset":"10071","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9561057,"name":"drain","context":{"idset":"10072","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9568722,"name":"drain","context":{"idset":"10073","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.957562,"name":"drain","context":{"idset":"10074","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9581864,"name":"drain","context":{"idset":"10075","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9588959,"name":"drain","context":{"idset":"10076","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9598947,"name":"drain","context":{"idset":"10077","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9605246,"name":"drain","context":{"idset":"10078","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9615626,"name":"drain","context":{"idset":"10079","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9624827,"name":"drain","context":{"idset":"10080","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9630976,"name":"drain","context":{"idset":"10081","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9638448,"name":"drain","context":{"idset":"10082","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9647019,"name":"drain","context":{"idset":"10083","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9657893,"name":"drain","context":{"idset":"10084","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9664171,"name":"drain","context":{"idset":"10085","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9670589,"name":"drain","context":{"idset":"10086","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9680266,"name":"drain","context":{"idset":"10087","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9689329,"name":"drain","context":{"idset":"10088","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.970037,"name":"drain","context":{"idset":"10089","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9708836,"name":"drain","context":{"idset":"10090","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9719584,"name":"drain","context":{"idset":"10091","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9728203,"name":"drain","context":{"idset":"10092","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9735074,"name":"drain","context":{"idset":"10093","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9741051,"name":"drain","context":{"idset":"10094","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9747226,"name":"drain","context":{"idset":"10095","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9755068,"name":"drain","context":{"idset":"10096","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9764218,"name":"drain","context":{"idset":"10097","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9770203,"name":"drain","context":{"idset":"10098","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.977823,"name":"drain","context":{"idset":"10099","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9785967,"name":"drain","context":{"idset":"10100","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9795489,"name":"drain","context":{"idset":"10101","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9808071,"name":"drain","context":{"idset":"10102","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9817986,"name":"drain","context":{"idset":"10103","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9827497,"name":"drain","context":{"idset":"10104","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9836264,"name":"drain","context":{"idset":"10105","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9847028,"name":"drain","context":{"idset":"10106","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9859974,"name":"drain","context":{"idset":"10107","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9869545,"name":"drain","context":{"idset":"10108","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9879236,"name":"drain","context":{"idset":"10109","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.988744,"name":"drain","context":{"idset":"10110","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9896548,"name":"drain","context":{"idset":"10111","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9905825,"name":"drain","context":{"idset":"10112","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.991622,"name":"drain","context":{"idset":"10113","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.992408,"name":"drain","context":{"idset":"10114","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9932921,"name":"drain","context":{"idset":"10115","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9941645,"name":"drain","context":{"idset":"10116","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9951134,"name":"drain","context":{"idset":"10117","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9960837,"name":"drain","context":{"idset":"10118","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9973841,"name":"drain","context":{"idset":"10119","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9983525,"name":"drain","context":{"idset":"10120","reason":"broker was unresponsive"}}
+{"timestamp":1712413955.9992239,"name":"drain","context":{"idset":"10121","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0002158,"name":"drain","context":{"idset":"10122","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0013289,"name":"drain","context":{"idset":"10123","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0027711,"name":"drain","context":{"idset":"10124","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.00367,"name":"drain","context":{"idset":"10125","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0047255,"name":"drain","context":{"idset":"10126","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0055668,"name":"drain","context":{"idset":"10127","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0064976,"name":"drain","context":{"idset":"10128","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0074799,"name":"drain","context":{"idset":"10129","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0083606,"name":"drain","context":{"idset":"10130","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0093174,"name":"drain","context":{"idset":"10131","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0101504,"name":"drain","context":{"idset":"10132","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0110497,"name":"drain","context":{"idset":"10133","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0119946,"name":"drain","context":{"idset":"10134","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0128207,"name":"drain","context":{"idset":"10135","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0137162,"name":"drain","context":{"idset":"10136","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0148597,"name":"drain","context":{"idset":"10137","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0160458,"name":"drain","context":{"idset":"10139","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.016902,"name":"drain","context":{"idset":"10140","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0178854,"name":"drain","context":{"idset":"10141","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.01893,"name":"drain","context":{"idset":"10142","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0197406,"name":"drain","context":{"idset":"10144","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0206289,"name":"drain","context":{"idset":"10146","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0215476,"name":"drain","context":{"idset":"10147","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0225115,"name":"drain","context":{"idset":"10148","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0234864,"name":"drain","context":{"idset":"10149","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0245538,"name":"drain","context":{"idset":"10150","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0254734,"name":"drain","context":{"idset":"10151","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.026587,"name":"drain","context":{"idset":"10152","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0274282,"name":"drain","context":{"idset":"10153","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0283949,"name":"drain","context":{"idset":"10154","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0295501,"name":"drain","context":{"idset":"10157","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0305603,"name":"drain","context":{"idset":"10158","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0318549,"name":"drain","context":{"idset":"10159","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0328722,"name":"drain","context":{"idset":"10160","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0339105,"name":"drain","context":{"idset":"10161","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0348403,"name":"drain","context":{"idset":"10162","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.036124,"name":"drain","context":{"idset":"10163","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0375249,"name":"drain","context":{"idset":"10164","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0385513,"name":"drain","context":{"idset":"10165","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.039562,"name":"drain","context":{"idset":"10166","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0404177,"name":"drain","context":{"idset":"10167","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0413752,"name":"drain","context":{"idset":"10168","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0426099,"name":"drain","context":{"idset":"10169","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0432057,"name":"drain","context":{"idset":"10170","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0438097,"name":"drain","context":{"idset":"10171","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.044733,"name":"drain","context":{"idset":"10172","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.045553,"name":"drain","context":{"idset":"10173","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0466862,"name":"drain","context":{"idset":"10174","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0476258,"name":"drain","context":{"idset":"10175","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0485518,"name":"drain","context":{"idset":"10176","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0498095,"name":"drain","context":{"idset":"10177","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0507693,"name":"drain","context":{"idset":"10178","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.051945,"name":"drain","context":{"idset":"10179","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0530975,"name":"drain","context":{"idset":"10180","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0540516,"name":"drain","context":{"idset":"10181","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0552261,"name":"drain","context":{"idset":"10182","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0563922,"name":"drain","context":{"idset":"10183","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0571845,"name":"drain","context":{"idset":"10184","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0581093,"name":"drain","context":{"idset":"10185","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0593235,"name":"drain","context":{"idset":"10186","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0602403,"name":"drain","context":{"idset":"10187","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0611291,"name":"drain","context":{"idset":"10188","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0620809,"name":"drain","context":{"idset":"10189","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0628891,"name":"drain","context":{"idset":"10190","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.063714,"name":"drain","context":{"idset":"10191","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0645351,"name":"drain","context":{"idset":"10192","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0653865,"name":"drain","context":{"idset":"10193","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0659988,"name":"drain","context":{"idset":"10194","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0666246,"name":"drain","context":{"idset":"10195","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0672412,"name":"drain","context":{"idset":"10196","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0681345,"name":"drain","context":{"idset":"10197","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0688796,"name":"drain","context":{"idset":"10198","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.069567,"name":"drain","context":{"idset":"10199","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0702517,"name":"drain","context":{"idset":"10200","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0710881,"name":"drain","context":{"idset":"10201","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0716054,"name":"drain","context":{"idset":"10202","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.072289,"name":"drain","context":{"idset":"10203","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0729668,"name":"drain","context":{"idset":"10204","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0737021,"name":"drain","context":{"idset":"10205","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.07426,"name":"drain","context":{"idset":"10206","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0752385,"name":"drain","context":{"idset":"10207","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0759306,"name":"drain","context":{"idset":"10208","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0768106,"name":"drain","context":{"idset":"10209","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.077482,"name":"drain","context":{"idset":"10210","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.078198,"name":"drain","context":{"idset":"10211","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0787778,"name":"drain","context":{"idset":"10212","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0793927,"name":"drain","context":{"idset":"10213","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0799499,"name":"drain","context":{"idset":"10214","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0805795,"name":"drain","context":{"idset":"10215","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0811827,"name":"drain","context":{"idset":"10216","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0818965,"name":"drain","context":{"idset":"10217","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0825911,"name":"drain","context":{"idset":"10218","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0833771,"name":"drain","context":{"idset":"10219","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0839181,"name":"drain","context":{"idset":"10220","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0846865,"name":"drain","context":{"idset":"10221","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.085258,"name":"drain","context":{"idset":"10222","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.085856,"name":"drain","context":{"idset":"10223","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0865369,"name":"drain","context":{"idset":"10224","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0871153,"name":"drain","context":{"idset":"10225","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0877223,"name":"drain","context":{"idset":"10226","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0884075,"name":"drain","context":{"idset":"10227","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.089047,"name":"drain","context":{"idset":"10228","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0898986,"name":"drain","context":{"idset":"10229","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0905836,"name":"drain","context":{"idset":"10230","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0911846,"name":"drain","context":{"idset":"10231","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0918918,"name":"drain","context":{"idset":"10232","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0924969,"name":"drain","context":{"idset":"10233","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0929911,"name":"drain","context":{"idset":"10234","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0939088,"name":"drain","context":{"idset":"10235","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0945749,"name":"drain","context":{"idset":"10236","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.095257,"name":"drain","context":{"idset":"10237","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0960791,"name":"drain","context":{"idset":"10238","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0967045,"name":"drain","context":{"idset":"10239","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0974779,"name":"drain","context":{"idset":"10240","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0982325,"name":"drain","context":{"idset":"10241","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0989175,"name":"drain","context":{"idset":"10242","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.0997424,"name":"drain","context":{"idset":"10243","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1003497,"name":"drain","context":{"idset":"10244","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1009753,"name":"drain","context":{"idset":"10245","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1016233,"name":"drain","context":{"idset":"10246","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1023083,"name":"drain","context":{"idset":"10247","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1030028,"name":"drain","context":{"idset":"10248","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1036258,"name":"drain","context":{"idset":"10249","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1041577,"name":"drain","context":{"idset":"10250","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1048353,"name":"drain","context":{"idset":"10251","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.105598,"name":"drain","context":{"idset":"10252","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1062136,"name":"drain","context":{"idset":"10253","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1069074,"name":"drain","context":{"idset":"10254","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1077557,"name":"drain","context":{"idset":"10255","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1084414,"name":"drain","context":{"idset":"10256","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1092038,"name":"drain","context":{"idset":"10257","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1103511,"name":"drain","context":{"idset":"10258","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1112137,"name":"drain","context":{"idset":"10259","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1119747,"name":"drain","context":{"idset":"10260","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1135283,"name":"drain","context":{"idset":"10262","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1145861,"name":"drain","context":{"idset":"10263","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1154375,"name":"drain","context":{"idset":"10264","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1160784,"name":"drain","context":{"idset":"10265","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.116852,"name":"drain","context":{"idset":"10266","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.117743,"name":"drain","context":{"idset":"10267","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1185734,"name":"drain","context":{"idset":"10268","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1195831,"name":"drain","context":{"idset":"10269","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1204345,"name":"drain","context":{"idset":"10270","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1212926,"name":"drain","context":{"idset":"10271","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1221175,"name":"drain","context":{"idset":"10272","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1228685,"name":"drain","context":{"idset":"10273","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1235962,"name":"drain","context":{"idset":"10274","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1244967,"name":"drain","context":{"idset":"10275","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1253912,"name":"drain","context":{"idset":"10276","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1262393,"name":"drain","context":{"idset":"10277","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1270823,"name":"drain","context":{"idset":"10278","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1280286,"name":"drain","context":{"idset":"10279","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1292119,"name":"drain","context":{"idset":"10280","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1301448,"name":"drain","context":{"idset":"10281","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1312404,"name":"drain","context":{"idset":"10282","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1320965,"name":"drain","context":{"idset":"10283","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1329043,"name":"drain","context":{"idset":"10284","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1337528,"name":"drain","context":{"idset":"10285","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1347654,"name":"drain","context":{"idset":"10286","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1357656,"name":"drain","context":{"idset":"10287","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1366301,"name":"drain","context":{"idset":"10288","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1375074,"name":"drain","context":{"idset":"10289","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1385047,"name":"drain","context":{"idset":"10290","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1393597,"name":"drain","context":{"idset":"10291","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1403005,"name":"drain","context":{"idset":"10292","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1412079,"name":"drain","context":{"idset":"10293","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1420648,"name":"drain","context":{"idset":"10294","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1432064,"name":"drain","context":{"idset":"10295","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1443384,"name":"drain","context":{"idset":"10296","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1452878,"name":"drain","context":{"idset":"10297","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1462128,"name":"drain","context":{"idset":"10298","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1471324,"name":"drain","context":{"idset":"10299","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1480842,"name":"drain","context":{"idset":"10300","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1489277,"name":"drain","context":{"idset":"10301","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1500142,"name":"drain","context":{"idset":"10302","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1510935,"name":"drain","context":{"idset":"10304","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1519601,"name":"drain","context":{"idset":"10305","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1529963,"name":"drain","context":{"idset":"10306","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1539161,"name":"drain","context":{"idset":"10307","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1549878,"name":"drain","context":{"idset":"10308","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1559405,"name":"drain","context":{"idset":"10309","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1569772,"name":"drain","context":{"idset":"10310","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1578162,"name":"drain","context":{"idset":"10311","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1586623,"name":"drain","context":{"idset":"10312","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.159687,"name":"drain","context":{"idset":"10313","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1606383,"name":"drain","context":{"idset":"10314","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1616077,"name":"drain","context":{"idset":"10315","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1625154,"name":"drain","context":{"idset":"10316","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.163507,"name":"drain","context":{"idset":"10317","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1645443,"name":"drain","context":{"idset":"10318","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1657503,"name":"drain","context":{"idset":"10319","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1668317,"name":"drain","context":{"idset":"10320","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1676817,"name":"drain","context":{"idset":"10321","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1685612,"name":"drain","context":{"idset":"10322","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1693261,"name":"drain","context":{"idset":"10323","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1702034,"name":"drain","context":{"idset":"10324","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1712213,"name":"drain","context":{"idset":"10325","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1722863,"name":"drain","context":{"idset":"10326","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.173254,"name":"drain","context":{"idset":"10327","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1740181,"name":"drain","context":{"idset":"10328","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1749704,"name":"drain","context":{"idset":"10329","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1759396,"name":"drain","context":{"idset":"10330","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1770334,"name":"drain","context":{"idset":"10331","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1780832,"name":"drain","context":{"idset":"10332","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1792183,"name":"drain","context":{"idset":"10333","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1803088,"name":"drain","context":{"idset":"10334","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1811719,"name":"drain","context":{"idset":"10335","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1820364,"name":"drain","context":{"idset":"10336","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1830552,"name":"drain","context":{"idset":"10337","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.183933,"name":"drain","context":{"idset":"10338","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1849146,"name":"drain","context":{"idset":"10339","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1857853,"name":"drain","context":{"idset":"10340","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1873419,"name":"drain","context":{"idset":"10341","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1886709,"name":"drain","context":{"idset":"10342","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1903486,"name":"drain","context":{"idset":"10345","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1924117,"name":"drain","context":{"idset":"10346","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1938117,"name":"drain","context":{"idset":"10347","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1950011,"name":"drain","context":{"idset":"10348","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1967144,"name":"drain","context":{"idset":"10349","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.1983304,"name":"drain","context":{"idset":"10350","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2005825,"name":"drain","context":{"idset":"10351","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2022088,"name":"drain","context":{"idset":"10352","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2036388,"name":"drain","context":{"idset":"10353","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2056882,"name":"drain","context":{"idset":"10354","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2070742,"name":"drain","context":{"idset":"10357","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2085934,"name":"drain","context":{"idset":"10358","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2096112,"name":"drain","context":{"idset":"10359","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2109532,"name":"drain","context":{"idset":"10360","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2126935,"name":"drain","context":{"idset":"10361","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2141037,"name":"drain","context":{"idset":"10362","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2154863,"name":"drain","context":{"idset":"10363","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2168329,"name":"drain","context":{"idset":"10364","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2182212,"name":"drain","context":{"idset":"10365","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.219806,"name":"drain","context":{"idset":"10366","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2212923,"name":"drain","context":{"idset":"10367","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2225699,"name":"drain","context":{"idset":"10368","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2240124,"name":"drain","context":{"idset":"10369","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2251813,"name":"drain","context":{"idset":"10370","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2268093,"name":"drain","context":{"idset":"10371","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2283115,"name":"drain","context":{"idset":"10372","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2297609,"name":"drain","context":{"idset":"10373","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2318203,"name":"drain","context":{"idset":"10374","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2332942,"name":"drain","context":{"idset":"10375","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2347724,"name":"drain","context":{"idset":"10376","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2363186,"name":"drain","context":{"idset":"10377","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2379293,"name":"drain","context":{"idset":"10378","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2395697,"name":"drain","context":{"idset":"10379","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2413924,"name":"drain","context":{"idset":"10380","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.242933,"name":"drain","context":{"idset":"10381","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2446883,"name":"drain","context":{"idset":"10382","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2463086,"name":"drain","context":{"idset":"10383","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2471087,"name":"drain","context":{"idset":"10384","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2483919,"name":"drain","context":{"idset":"10385","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2496612,"name":"drain","context":{"idset":"10386","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2506678,"name":"drain","context":{"idset":"10387","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2522366,"name":"drain","context":{"idset":"10388","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2537656,"name":"drain","context":{"idset":"10389","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2554727,"name":"drain","context":{"idset":"10390","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2570231,"name":"drain","context":{"idset":"10391","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2592192,"name":"drain","context":{"idset":"10392","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2607892,"name":"drain","context":{"idset":"10393","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2621386,"name":"drain","context":{"idset":"10394","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2634847,"name":"drain","context":{"idset":"10395","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2651386,"name":"drain","context":{"idset":"10396","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2667472,"name":"drain","context":{"idset":"10397","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2681818,"name":"drain","context":{"idset":"10398","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2695484,"name":"drain","context":{"idset":"10399","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2708819,"name":"drain","context":{"idset":"10400","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2723584,"name":"drain","context":{"idset":"10401","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2735195,"name":"drain","context":{"idset":"10402","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2741809,"name":"drain","context":{"idset":"10403","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2757397,"name":"drain","context":{"idset":"10404","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2767437,"name":"drain","context":{"idset":"10405","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2778037,"name":"drain","context":{"idset":"10406","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2790298,"name":"drain","context":{"idset":"10407","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2799392,"name":"drain","context":{"idset":"10408","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2809598,"name":"drain","context":{"idset":"10409","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2819412,"name":"drain","context":{"idset":"10410","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2830749,"name":"drain","context":{"idset":"10411","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2841685,"name":"drain","context":{"idset":"10412","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2848287,"name":"drain","context":{"idset":"10413","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2855861,"name":"drain","context":{"idset":"10414","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2864475,"name":"drain","context":{"idset":"10415","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2881811,"name":"drain","context":{"idset":"10416","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.290381,"name":"drain","context":{"idset":"10417","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2918174,"name":"drain","context":{"idset":"10418","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2932804,"name":"drain","context":{"idset":"10419","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2944272,"name":"drain","context":{"idset":"10420","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2957289,"name":"drain","context":{"idset":"10421","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2972093,"name":"drain","context":{"idset":"10422","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2982991,"name":"drain","context":{"idset":"10423","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.2996166,"name":"drain","context":{"idset":"10424","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3007624,"name":"drain","context":{"idset":"10425","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3020213,"name":"drain","context":{"idset":"10426","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3033798,"name":"drain","context":{"idset":"10427","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3050554,"name":"drain","context":{"idset":"10428","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3073072,"name":"drain","context":{"idset":"10429","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3088925,"name":"drain","context":{"idset":"10430","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3102863,"name":"drain","context":{"idset":"10431","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3121121,"name":"drain","context":{"idset":"10432","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3136756,"name":"drain","context":{"idset":"10433","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3152137,"name":"drain","context":{"idset":"10434","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3163979,"name":"drain","context":{"idset":"10744","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3180211,"name":"drain","context":{"idset":"10745","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.319459,"name":"drain","context":{"idset":"10746","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3207042,"name":"drain","context":{"idset":"10754","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.32182,"name":"drain","context":{"idset":"11024","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3228323,"name":"drain","context":{"idset":"11026","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3240402,"name":"drain","context":{"idset":"11027","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3249412,"name":"drain","context":{"idset":"11029","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3256667,"name":"drain","context":{"idset":"11031","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3264005,"name":"drain","context":{"idset":"11033","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3271623,"name":"drain","context":{"idset":"11034","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3281944,"name":"drain","context":{"idset":"11036","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3288798,"name":"drain","context":{"idset":"11038","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.329772,"name":"drain","context":{"idset":"11040","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3304856,"name":"drain","context":{"idset":"11041","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3312192,"name":"drain","context":{"idset":"11042","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3319581,"name":"drain","context":{"idset":"11043","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.332938,"name":"drain","context":{"idset":"11044","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3339615,"name":"drain","context":{"idset":"11046","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3350055,"name":"drain","context":{"idset":"11047","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3359058,"name":"drain","context":{"idset":"11048","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3369021,"name":"drain","context":{"idset":"11050","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3378906,"name":"drain","context":{"idset":"11052","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3386481,"name":"drain","context":{"idset":"11054","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3395011,"name":"drain","context":{"idset":"11058","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3405855,"name":"drain","context":{"idset":"11061","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3414886,"name":"drain","context":{"idset":"11064","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3422086,"name":"drain","context":{"idset":"11065","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3429613,"name":"drain","context":{"idset":"11070","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3439572,"name":"drain","context":{"idset":"11073","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3448005,"name":"drain","context":{"idset":"11078","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3458107,"name":"drain","context":{"idset":"11079","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3466394,"name":"drain","context":{"idset":"11084","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3477273,"name":"drain","context":{"idset":"11092","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3487744,"name":"drain","context":{"idset":"11096","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3497891,"name":"drain","context":{"idset":"11101","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3506508,"name":"drain","context":{"idset":"11104","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3514631,"name":"drain","context":{"idset":"11106","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3524609,"name":"drain","context":{"idset":"11110","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3532758,"name":"drain","context":{"idset":"11118","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.354177,"name":"drain","context":{"idset":"11120","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3550808,"name":"drain","context":{"idset":"11121","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3558724,"name":"drain","context":{"idset":"11125","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3567343,"name":"drain","context":{"idset":"11137","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3575432,"name":"drain","context":{"idset":"11139","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3584905,"name":"drain","context":{"idset":"11142","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3590562,"name":"drain","context":{"idset":"11145","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3597624,"name":"drain","context":{"idset":"11151","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3607197,"name":"drain","context":{"idset":"11155","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3616736,"name":"drain","context":{"idset":"11158","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3624606,"name":"drain","context":{"idset":"11164","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.363148,"name":"drain","context":{"idset":"11165","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3639915,"name":"drain","context":{"idset":"11166","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.364934,"name":"drain","context":{"idset":"11167","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3655927,"name":"drain","context":{"idset":"11168","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3662412,"name":"drain","context":{"idset":"11169","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.367043,"name":"drain","context":{"idset":"11170","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3677511,"name":"drain","context":{"idset":"11171","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3690746,"name":"drain","context":{"idset":"11172","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3700395,"name":"drain","context":{"idset":"11173","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3709219,"name":"drain","context":{"idset":"11174","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3719144,"name":"drain","context":{"idset":"11175","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3730924,"name":"drain","context":{"idset":"11176","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3741851,"name":"drain","context":{"idset":"11177","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3758013,"name":"drain","context":{"idset":"11178","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3774848,"name":"drain","context":{"idset":"11179","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3790243,"name":"drain","context":{"idset":"11180","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3804221,"name":"drain","context":{"idset":"11181","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3816657,"name":"drain","context":{"idset":"11182","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3833992,"name":"drain","context":{"idset":"11183","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3849368,"name":"drain","context":{"idset":"11184","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3863878,"name":"drain","context":{"idset":"11185","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.387677,"name":"drain","context":{"idset":"11186","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3888066,"name":"drain","context":{"idset":"11187","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3896708,"name":"drain","context":{"idset":"11188","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3904605,"name":"drain","context":{"idset":"11189","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3914933,"name":"drain","context":{"idset":"11190","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3923855,"name":"drain","context":{"idset":"11191","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3936584,"name":"drain","context":{"idset":"11192","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3951688,"name":"drain","context":{"idset":"11193","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3968639,"name":"drain","context":{"idset":"11194","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3984263,"name":"drain","context":{"idset":"11195","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.3998101,"name":"drain","context":{"idset":"11196","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4011676,"name":"drain","context":{"idset":"11197","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4027689,"name":"drain","context":{"idset":"11198","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.404253,"name":"drain","context":{"idset":"11199","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4056628,"name":"drain","context":{"idset":"11200","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.407181,"name":"drain","context":{"idset":"11201","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4081368,"name":"drain","context":{"idset":"11202","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4091649,"name":"drain","context":{"idset":"11203","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4099283,"name":"drain","context":{"idset":"11204","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4107878,"name":"drain","context":{"idset":"11205","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4115379,"name":"drain","context":{"idset":"11206","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4122536,"name":"drain","context":{"idset":"11207","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4136693,"name":"drain","context":{"idset":"11208","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4145002,"name":"drain","context":{"idset":"11209","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4156942,"name":"drain","context":{"idset":"11210","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4170673,"name":"drain","context":{"idset":"11211","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4181499,"name":"drain","context":{"idset":"11212","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.41926,"name":"drain","context":{"idset":"11213","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4200697,"name":"drain","context":{"idset":"11214","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4211419,"name":"drain","context":{"idset":"11215","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4222233,"name":"drain","context":{"idset":"11216","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4234991,"name":"drain","context":{"idset":"11217","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4247267,"name":"drain","context":{"idset":"11218","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4257116,"name":"drain","context":{"idset":"11219","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4272118,"name":"drain","context":{"idset":"11220","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4283681,"name":"drain","context":{"idset":"11221","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.429877,"name":"drain","context":{"idset":"11222","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4307063,"name":"drain","context":{"idset":"11223","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4316208,"name":"drain","context":{"idset":"11224","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4324353,"name":"drain","context":{"idset":"11225","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4331279,"name":"drain","context":{"idset":"11227","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4341009,"name":"drain","context":{"idset":"11228","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4352198,"name":"drain","context":{"idset":"11229","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4363034,"name":"drain","context":{"idset":"11230","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4370821,"name":"drain","context":{"idset":"11231","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4378419,"name":"drain","context":{"idset":"11232","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4388821,"name":"drain","context":{"idset":"11234","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4397116,"name":"drain","context":{"idset":"11237","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.44066,"name":"drain","context":{"idset":"11238","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4414134,"name":"drain","context":{"idset":"11239","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4420524,"name":"drain","context":{"idset":"11240","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4431989,"name":"drain","context":{"idset":"11241","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4443204,"name":"drain","context":{"idset":"11242","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4456155,"name":"drain","context":{"idset":"11243","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4468527,"name":"drain","context":{"idset":"11244","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4484189,"name":"drain","context":{"idset":"11245","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4495392,"name":"drain","context":{"idset":"11246","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4509811,"name":"drain","context":{"idset":"11248","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4520071,"name":"drain","context":{"idset":"11250","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4528093,"name":"drain","context":{"idset":"11251","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4536695,"name":"drain","context":{"idset":"11252","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4545097,"name":"drain","context":{"idset":"11253","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.455389,"name":"drain","context":{"idset":"11254","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4566364,"name":"drain","context":{"idset":"11255","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4576566,"name":"drain","context":{"idset":"11256","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4589312,"name":"drain","context":{"idset":"11257","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4601786,"name":"drain","context":{"idset":"11258","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4613771,"name":"drain","context":{"idset":"11259","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4627421,"name":"drain","context":{"idset":"11260","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.463866,"name":"drain","context":{"idset":"11261","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4650459,"name":"drain","context":{"idset":"11262","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4660089,"name":"drain","context":{"idset":"11263","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4669647,"name":"drain","context":{"idset":"11264","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4680121,"name":"drain","context":{"idset":"11265","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4689422,"name":"drain","context":{"idset":"11266","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4700181,"name":"drain","context":{"idset":"11267","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4707849,"name":"drain","context":{"idset":"11268","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4721069,"name":"drain","context":{"idset":"11269","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.473047,"name":"drain","context":{"idset":"11270","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4741287,"name":"drain","context":{"idset":"11271","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4750032,"name":"drain","context":{"idset":"11272","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4759486,"name":"drain","context":{"idset":"11273","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4768739,"name":"drain","context":{"idset":"11274","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4777243,"name":"drain","context":{"idset":"11275","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4788027,"name":"drain","context":{"idset":"11276","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4795692,"name":"drain","context":{"idset":"11277","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4803958,"name":"drain","context":{"idset":"11278","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4812357,"name":"drain","context":{"idset":"11279","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.481952,"name":"drain","context":{"idset":"11280","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4829364,"name":"drain","context":{"idset":"11281","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4837964,"name":"drain","context":{"idset":"11282","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4848838,"name":"drain","context":{"idset":"11283","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4860334,"name":"drain","context":{"idset":"11284","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4872608,"name":"drain","context":{"idset":"11285","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4883995,"name":"drain","context":{"idset":"11286","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4895141,"name":"drain","context":{"idset":"11287","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4907043,"name":"drain","context":{"idset":"11288","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4919336,"name":"drain","context":{"idset":"11289","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4930496,"name":"drain","context":{"idset":"11290","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4941146,"name":"drain","context":{"idset":"11291","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4950817,"name":"drain","context":{"idset":"11292","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4958427,"name":"drain","context":{"idset":"11293","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4966092,"name":"drain","context":{"idset":"11294","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4978487,"name":"drain","context":{"idset":"11295","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4986989,"name":"drain","context":{"idset":"11296","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.4999673,"name":"drain","context":{"idset":"11297","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5013888,"name":"drain","context":{"idset":"11298","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5029881,"name":"drain","context":{"idset":"11299","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5041785,"name":"drain","context":{"idset":"11300","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5051901,"name":"drain","context":{"idset":"11301","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5060546,"name":"drain","context":{"idset":"11302","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5067887,"name":"drain","context":{"idset":"11303","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5075459,"name":"drain","context":{"idset":"11304","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5085418,"name":"drain","context":{"idset":"11305","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5098083,"name":"drain","context":{"idset":"11306","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5109444,"name":"drain","context":{"idset":"11307","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5118318,"name":"drain","context":{"idset":"11308","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5126197,"name":"drain","context":{"idset":"11309","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5135486,"name":"drain","context":{"idset":"11310","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5147061,"name":"drain","context":{"idset":"11311","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5156627,"name":"drain","context":{"idset":"11312","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5166895,"name":"drain","context":{"idset":"11313","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.517669,"name":"drain","context":{"idset":"11314","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5185885,"name":"drain","context":{"idset":"11315","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5198874,"name":"drain","context":{"idset":"11316","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5206332,"name":"drain","context":{"idset":"11317","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5217161,"name":"drain","context":{"idset":"11318","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5236351,"name":"drain","context":{"idset":"11319","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.525089,"name":"drain","context":{"idset":"11320","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5266047,"name":"drain","context":{"idset":"11321","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5277162,"name":"drain","context":{"idset":"11322","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.528512,"name":"drain","context":{"idset":"11323","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.529551,"name":"drain","context":{"idset":"11324","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5302138,"name":"drain","context":{"idset":"11325","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5309753,"name":"drain","context":{"idset":"11326","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5317233,"name":"drain","context":{"idset":"11327","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5323873,"name":"drain","context":{"idset":"11328","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5331683,"name":"drain","context":{"idset":"11331","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5339701,"name":"drain","context":{"idset":"11332","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5348382,"name":"drain","context":{"idset":"11333","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5357885,"name":"drain","context":{"idset":"11334","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5367348,"name":"drain","context":{"idset":"11335","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5377343,"name":"drain","context":{"idset":"11336","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5386801,"name":"drain","context":{"idset":"11337","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.539607,"name":"drain","context":{"idset":"11338","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.540493,"name":"drain","context":{"idset":"11339","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5412939,"name":"drain","context":{"idset":"11340","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5421553,"name":"drain","context":{"idset":"11341","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5430958,"name":"drain","context":{"idset":"11342","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5440762,"name":"drain","context":{"idset":"11343","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5449326,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5457275,"name":"drain","context":{"idset":"11345","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5466955,"name":"drain","context":{"idset":"11346","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5476315,"name":"drain","context":{"idset":"11347","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5488679,"name":"drain","context":{"idset":"11348","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5497892,"name":"drain","context":{"idset":"11349","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5507221,"name":"drain","context":{"idset":"11350","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5516891,"name":"drain","context":{"idset":"11351","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5526278,"name":"drain","context":{"idset":"11352","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5535431,"name":"drain","context":{"idset":"11353","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5544925,"name":"drain","context":{"idset":"11354","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5554614,"name":"drain","context":{"idset":"11355","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5563698,"name":"drain","context":{"idset":"11356","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5572858,"name":"drain","context":{"idset":"11357","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5581326,"name":"drain","context":{"idset":"11359","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5589957,"name":"drain","context":{"idset":"11360","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5598977,"name":"drain","context":{"idset":"11361","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5607803,"name":"drain","context":{"idset":"11362","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5616984,"name":"drain","context":{"idset":"11363","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.562587,"name":"drain","context":{"idset":"11364","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5634902,"name":"drain","context":{"idset":"11365","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5643861,"name":"drain","context":{"idset":"11366","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5652761,"name":"drain","context":{"idset":"11367","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5661335,"name":"drain","context":{"idset":"11368","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5670059,"name":"drain","context":{"idset":"11369","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5679097,"name":"drain","context":{"idset":"11370","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5695963,"name":"drain","context":{"idset":"11371","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5705564,"name":"drain","context":{"idset":"11372","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5715389,"name":"drain","context":{"idset":"11375","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.572499,"name":"drain","context":{"idset":"11376","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5740185,"name":"drain","context":{"idset":"11377","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5749705,"name":"drain","context":{"idset":"11378","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5759234,"name":"drain","context":{"idset":"11379","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.57687,"name":"drain","context":{"idset":"11380","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.577831,"name":"drain","context":{"idset":"11381","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5787942,"name":"drain","context":{"idset":"11382","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5797734,"name":"drain","context":{"idset":"11383","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5807629,"name":"drain","context":{"idset":"11384","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5817451,"name":"drain","context":{"idset":"11385","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.582721,"name":"drain","context":{"idset":"11386","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5836833,"name":"drain","context":{"idset":"11389","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5846455,"name":"drain","context":{"idset":"11391","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5856104,"name":"drain","context":{"idset":"11392","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5865698,"name":"drain","context":{"idset":"11393","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5875285,"name":"drain","context":{"idset":"11394","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5884643,"name":"drain","context":{"idset":"11396","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5893631,"name":"drain","context":{"idset":"11397","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5908744,"name":"drain","context":{"idset":"11398","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.5918038,"name":"drain","context":{"idset":"11399","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.619801,"name":"drain","context":{"idset":"11430","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.620811,"name":"drain","context":{"idset":"11432","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6218138,"name":"drain","context":{"idset":"11433","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6227984,"name":"drain","context":{"idset":"11434","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.623929,"name":"drain","context":{"idset":"11435","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6249151,"name":"drain","context":{"idset":"11436","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6259573,"name":"drain","context":{"idset":"11437","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6269777,"name":"drain","context":{"idset":"11438","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.627995,"name":"drain","context":{"idset":"11443","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6289723,"name":"drain","context":{"idset":"11444","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6299684,"name":"drain","context":{"idset":"11445","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6309938,"name":"drain","context":{"idset":"11446","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6319728,"name":"drain","context":{"idset":"11447","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6329689,"name":"drain","context":{"idset":"11448","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6339791,"name":"drain","context":{"idset":"11449","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6349752,"name":"drain","context":{"idset":"11450","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6359675,"name":"drain","context":{"idset":"11451","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6369967,"name":"drain","context":{"idset":"11452","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6380577,"name":"drain","context":{"idset":"11453","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6391137,"name":"drain","context":{"idset":"11454","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6401558,"name":"drain","context":{"idset":"11455","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6423788,"name":"drain","context":{"idset":"11456","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6434791,"name":"drain","context":{"idset":"11457","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6445775,"name":"drain","context":{"idset":"11458","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6456745,"name":"drain","context":{"idset":"11459","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6466856,"name":"drain","context":{"idset":"11460","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6476619,"name":"drain","context":{"idset":"11461","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6487288,"name":"drain","context":{"idset":"11462","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6497557,"name":"drain","context":{"idset":"11463","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.650835,"name":"drain","context":{"idset":"11465","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6519263,"name":"drain","context":{"idset":"11467","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6529574,"name":"drain","context":{"idset":"11468","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6539984,"name":"drain","context":{"idset":"11469","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6551235,"name":"drain","context":{"idset":"11470","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6562514,"name":"drain","context":{"idset":"11471","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6573184,"name":"drain","context":{"idset":"11472","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6583333,"name":"drain","context":{"idset":"11473","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6593249,"name":"drain","context":{"idset":"11474","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6603827,"name":"drain","context":{"idset":"11475","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6614795,"name":"drain","context":{"idset":"11476","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6625659,"name":"drain","context":{"idset":"11477","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6637094,"name":"drain","context":{"idset":"11478","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6648183,"name":"drain","context":{"idset":"11479","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6658862,"name":"drain","context":{"idset":"11480","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6670008,"name":"drain","context":{"idset":"11481","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6681018,"name":"drain","context":{"idset":"11482","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6691771,"name":"drain","context":{"idset":"11483","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6701696,"name":"drain","context":{"idset":"11484","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6712048,"name":"drain","context":{"idset":"11485","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6723514,"name":"drain","context":{"idset":"11486","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6733637,"name":"drain","context":{"idset":"11487","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.67436,"name":"drain","context":{"idset":"11488","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6753047,"name":"drain","context":{"idset":"11489","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6762471,"name":"drain","context":{"idset":"11490","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6772416,"name":"drain","context":{"idset":"11491","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6782398,"name":"drain","context":{"idset":"11492","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6792417,"name":"drain","context":{"idset":"11493","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6802478,"name":"drain","context":{"idset":"11494","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6812489,"name":"drain","context":{"idset":"11495","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6822543,"name":"drain","context":{"idset":"11496","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.683284,"name":"drain","context":{"idset":"11497","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6842909,"name":"drain","context":{"idset":"11498","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6853597,"name":"drain","context":{"idset":"11499","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6863563,"name":"drain","context":{"idset":"11500","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6873667,"name":"drain","context":{"idset":"11501","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6883879,"name":"drain","context":{"idset":"11502","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6893904,"name":"drain","context":{"idset":"11503","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6904073,"name":"drain","context":{"idset":"11504","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6914427,"name":"drain","context":{"idset":"11505","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6924622,"name":"drain","context":{"idset":"11506","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6934962,"name":"drain","context":{"idset":"11507","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6945338,"name":"drain","context":{"idset":"11508","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6955097,"name":"drain","context":{"idset":"11509","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6964622,"name":"drain","context":{"idset":"11510","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6974761,"name":"drain","context":{"idset":"11511","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6984782,"name":"drain","context":{"idset":"11512","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.6995201,"name":"drain","context":{"idset":"11513","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7004983,"name":"drain","context":{"idset":"11514","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7014742,"name":"drain","context":{"idset":"11515","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7024622,"name":"drain","context":{"idset":"11516","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7034523,"name":"drain","context":{"idset":"11517","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7044606,"name":"drain","context":{"idset":"11518","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7054884,"name":"drain","context":{"idset":"11519","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.706569,"name":"drain","context":{"idset":"11520","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7076018,"name":"drain","context":{"idset":"11521","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7086227,"name":"drain","context":{"idset":"11523","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7096338,"name":"drain","context":{"idset":"11524","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7106504,"name":"drain","context":{"idset":"11525","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.711679,"name":"drain","context":{"idset":"11526","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7127213,"name":"drain","context":{"idset":"11528","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.713758,"name":"drain","context":{"idset":"11529","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7147951,"name":"drain","context":{"idset":"11530","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7158113,"name":"drain","context":{"idset":"11531","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7168231,"name":"drain","context":{"idset":"11532","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7178502,"name":"drain","context":{"idset":"11533","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.71889,"name":"drain","context":{"idset":"11534","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7198899,"name":"drain","context":{"idset":"11535","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7209425,"name":"drain","context":{"idset":"11536","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7219729,"name":"drain","context":{"idset":"11537","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7230203,"name":"drain","context":{"idset":"11538","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7241218,"name":"drain","context":{"idset":"11539","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7251754,"name":"drain","context":{"idset":"11540","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7262266,"name":"drain","context":{"idset":"11541","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7272828,"name":"drain","context":{"idset":"11542","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7282875,"name":"drain","context":{"idset":"11543","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.729306,"name":"drain","context":{"idset":"11544","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7303402,"name":"drain","context":{"idset":"11545","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7313831,"name":"drain","context":{"idset":"11546","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7324061,"name":"drain","context":{"idset":"11547","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7334528,"name":"drain","context":{"idset":"11548","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7344644,"name":"drain","context":{"idset":"11549","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.735919,"name":"drain","context":{"idset":"11550","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7369454,"name":"drain","context":{"idset":"11551","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7380071,"name":"drain","context":{"idset":"11553","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7390156,"name":"drain","context":{"idset":"11554","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7400498,"name":"drain","context":{"idset":"11555","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7411008,"name":"drain","context":{"idset":"11556","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7421365,"name":"drain","context":{"idset":"11557","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7432008,"name":"drain","context":{"idset":"11558","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7442591,"name":"drain","context":{"idset":"11559","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7455654,"name":"drain","context":{"idset":"11560","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7465994,"name":"drain","context":{"idset":"11561","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7476275,"name":"drain","context":{"idset":"11562","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7486644,"name":"drain","context":{"idset":"11564","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7497709,"name":"drain","context":{"idset":"11565","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7508197,"name":"drain","context":{"idset":"11566","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7519052,"name":"drain","context":{"idset":"11567","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7529488,"name":"drain","context":{"idset":"11568","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7540109,"name":"drain","context":{"idset":"11569","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7550669,"name":"drain","context":{"idset":"11570","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7561433,"name":"drain","context":{"idset":"11571","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7572026,"name":"drain","context":{"idset":"11572","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.75824,"name":"drain","context":{"idset":"11573","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7593303,"name":"drain","context":{"idset":"11574","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.760391,"name":"drain","context":{"idset":"11575","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7614269,"name":"drain","context":{"idset":"11576","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7624824,"name":"drain","context":{"idset":"11577","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7635524,"name":"drain","context":{"idset":"11578","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7646165,"name":"drain","context":{"idset":"11579","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7656653,"name":"drain","context":{"idset":"11580","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7667081,"name":"drain","context":{"idset":"11581","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7677341,"name":"drain","context":{"idset":"11582","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7688022,"name":"drain","context":{"idset":"11583","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7721972,"name":"drain","context":{"idset":"11584","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7732434,"name":"drain","context":{"idset":"11585","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7745845,"name":"drain","context":{"idset":"11586","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7756276,"name":"drain","context":{"idset":"11589","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7766716,"name":"drain","context":{"idset":"11590","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7777195,"name":"drain","context":{"idset":"11591","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7787395,"name":"drain","context":{"idset":"11592","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7797785,"name":"drain","context":{"idset":"11593","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7808645,"name":"drain","context":{"idset":"11594","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7819405,"name":"drain","context":{"idset":"11595","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7830269,"name":"drain","context":{"idset":"11596","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7840664,"name":"drain","context":{"idset":"11597","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7851157,"name":"drain","context":{"idset":"11598","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7861936,"name":"drain","context":{"idset":"11599","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7872815,"name":"drain","context":{"idset":"11600","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7883785,"name":"drain","context":{"idset":"11601","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7894239,"name":"drain","context":{"idset":"11602","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7907357,"name":"drain","context":{"idset":"11603","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7918065,"name":"drain","context":{"idset":"11604","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7928958,"name":"drain","context":{"idset":"11605","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7939947,"name":"drain","context":{"idset":"11606","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7950892,"name":"drain","context":{"idset":"11607","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7961564,"name":"drain","context":{"idset":"11608","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.7972531,"name":"drain","context":{"idset":"11609","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.798346,"name":"drain","context":{"idset":"11610","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.799494,"name":"drain","context":{"idset":"11611","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.80056,"name":"drain","context":{"idset":"11612","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8016531,"name":"drain","context":{"idset":"11613","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8027163,"name":"drain","context":{"idset":"11614","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8038087,"name":"drain","context":{"idset":"11615","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8048787,"name":"drain","context":{"idset":"11616","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8059235,"name":"drain","context":{"idset":"11617","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8070018,"name":"drain","context":{"idset":"11618","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8081379,"name":"drain","context":{"idset":"11619","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8092253,"name":"drain","context":{"idset":"11620","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.810339,"name":"drain","context":{"idset":"11621","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8113234,"name":"drain","context":{"idset":"11622","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8123212,"name":"drain","context":{"idset":"11623","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8133581,"name":"drain","context":{"idset":"11624","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8143461,"name":"drain","context":{"idset":"11625","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8153689,"name":"drain","context":{"idset":"11626","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8164358,"name":"drain","context":{"idset":"11627","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8175085,"name":"drain","context":{"idset":"11628","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8185525,"name":"drain","context":{"idset":"11629","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8196299,"name":"drain","context":{"idset":"11630","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8207102,"name":"drain","context":{"idset":"11631","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.821804,"name":"drain","context":{"idset":"11633","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8228455,"name":"drain","context":{"idset":"11634","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8238857,"name":"drain","context":{"idset":"11635","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8249886,"name":"drain","context":{"idset":"11636","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8260832,"name":"drain","context":{"idset":"11653","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8271043,"name":"drain","context":{"idset":"11654","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8280926,"name":"drain","context":{"idset":"11655","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8291442,"name":"drain","context":{"idset":"11656","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.830229,"name":"drain","context":{"idset":"11657","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8312893,"name":"drain","context":{"idset":"11658","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8323667,"name":"drain","context":{"idset":"11659","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8334332,"name":"drain","context":{"idset":"11660","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8344781,"name":"drain","context":{"idset":"11661","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8355396,"name":"drain","context":{"idset":"11662","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8366048,"name":"drain","context":{"idset":"11663","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8377037,"name":"drain","context":{"idset":"11664","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8387706,"name":"drain","context":{"idset":"11665","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8398547,"name":"drain","context":{"idset":"11666","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8409517,"name":"drain","context":{"idset":"11667","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8420506,"name":"drain","context":{"idset":"11668","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8431304,"name":"drain","context":{"idset":"11669","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8442893,"name":"drain","context":{"idset":"11670","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8453968,"name":"drain","context":{"idset":"11671","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8464961,"name":"drain","context":{"idset":"11672","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8476021,"name":"drain","context":{"idset":"11673","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8486967,"name":"drain","context":{"idset":"11674","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8498282,"name":"drain","context":{"idset":"11675","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8509209,"name":"drain","context":{"idset":"11676","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8520157,"name":"drain","context":{"idset":"11677","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8531172,"name":"drain","context":{"idset":"11678","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8542261,"name":"drain","context":{"idset":"11679","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8552921,"name":"drain","context":{"idset":"11680","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8563745,"name":"drain","context":{"idset":"11681","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8574641,"name":"drain","context":{"idset":"11682","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8585413,"name":"drain","context":{"idset":"11684","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8596292,"name":"drain","context":{"idset":"11685","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8607285,"name":"drain","context":{"idset":"11686","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8618145,"name":"drain","context":{"idset":"11687","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8629122,"name":"drain","context":{"idset":"11688","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.864027,"name":"drain","context":{"idset":"11689","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8651252,"name":"drain","context":{"idset":"11690","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8662117,"name":"drain","context":{"idset":"11691","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8672872,"name":"drain","context":{"idset":"11692","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8683574,"name":"drain","context":{"idset":"11693","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.869504,"name":"drain","context":{"idset":"11695","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8706028,"name":"drain","context":{"idset":"11696","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8716407,"name":"drain","context":{"idset":"11697","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8727391,"name":"drain","context":{"idset":"11698","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8738463,"name":"drain","context":{"idset":"11699","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8750083,"name":"drain","context":{"idset":"11700","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8761363,"name":"drain","context":{"idset":"11701","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8773122,"name":"drain","context":{"idset":"11702","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8784258,"name":"drain","context":{"idset":"11703","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8795505,"name":"drain","context":{"idset":"11704","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8806832,"name":"drain","context":{"idset":"11705","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8817959,"name":"drain","context":{"idset":"11706","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.882901,"name":"drain","context":{"idset":"11707","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.884002,"name":"drain","context":{"idset":"11708","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8850667,"name":"drain","context":{"idset":"11709","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8861532,"name":"drain","context":{"idset":"11710","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8871171,"name":"drain","context":{"idset":"11711","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8881314,"name":"drain","context":{"idset":"11712","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8891046,"name":"drain","context":{"idset":"11713","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8900981,"name":"drain","context":{"idset":"11714","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8911991,"name":"drain","context":{"idset":"11715","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8923209,"name":"drain","context":{"idset":"11716","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8934078,"name":"drain","context":{"idset":"11717","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8945096,"name":"drain","context":{"idset":"11718","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8956299,"name":"drain","context":{"idset":"11719","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8967543,"name":"drain","context":{"idset":"11720","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8978651,"name":"drain","context":{"idset":"11721","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.8989878,"name":"drain","context":{"idset":"11722","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9001255,"name":"drain","context":{"idset":"11723","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.901217,"name":"drain","context":{"idset":"11724","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9023435,"name":"drain","context":{"idset":"11725","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9034524,"name":"drain","context":{"idset":"11726","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9045327,"name":"drain","context":{"idset":"11727","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9056172,"name":"drain","context":{"idset":"11728","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9067485,"name":"drain","context":{"idset":"11729","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9078591,"name":"drain","context":{"idset":"11730","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9090402,"name":"drain","context":{"idset":"11731","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9101136,"name":"drain","context":{"idset":"11732","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9112258,"name":"drain","context":{"idset":"11733","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9123421,"name":"drain","context":{"idset":"11734","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9133706,"name":"drain","context":{"idset":"11735","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9144828,"name":"drain","context":{"idset":"11736","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9156036,"name":"drain","context":{"idset":"11737","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9167304,"name":"drain","context":{"idset":"11738","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9178555,"name":"drain","context":{"idset":"11739","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9190016,"name":"drain","context":{"idset":"11740","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9201324,"name":"drain","context":{"idset":"11741","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9212496,"name":"drain","context":{"idset":"11742","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9223831,"name":"drain","context":{"idset":"11743","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9235225,"name":"drain","context":{"idset":"11744","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9246914,"name":"drain","context":{"idset":"11745","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9258549,"name":"drain","context":{"idset":"11746","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9269896,"name":"drain","context":{"idset":"11747","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9281104,"name":"drain","context":{"idset":"11748","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9292867,"name":"drain","context":{"idset":"11749","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9303946,"name":"drain","context":{"idset":"11750","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9315183,"name":"drain","context":{"idset":"11751","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9326401,"name":"drain","context":{"idset":"11752","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9337518,"name":"drain","context":{"idset":"11753","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9349232,"name":"drain","context":{"idset":"11754","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9360416,"name":"drain","context":{"idset":"11755","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9371829,"name":"drain","context":{"idset":"11756","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9383428,"name":"drain","context":{"idset":"11757","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9395032,"name":"drain","context":{"idset":"11758","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9406102,"name":"drain","context":{"idset":"11759","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9417002,"name":"drain","context":{"idset":"11760","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9427733,"name":"drain","context":{"idset":"11761","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9438474,"name":"drain","context":{"idset":"11762","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9449925,"name":"drain","context":{"idset":"11763","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9461181,"name":"drain","context":{"idset":"11764","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9649324,"name":"drain","context":{"idset":"11802","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9910486,"name":"drain","context":{"idset":"11877","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9922044,"name":"drain","context":{"idset":"11878","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9933789,"name":"drain","context":{"idset":"11879","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9945431,"name":"drain","context":{"idset":"11880","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9956915,"name":"drain","context":{"idset":"11881","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9968631,"name":"drain","context":{"idset":"11882","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9980168,"name":"drain","context":{"idset":"11883","reason":"broker was unresponsive"}}
+{"timestamp":1712413956.9991553,"name":"drain","context":{"idset":"11885","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0043457,"name":"drain","context":{"idset":"11886","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0055072,"name":"drain","context":{"idset":"11887","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.006645,"name":"drain","context":{"idset":"11888","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0077968,"name":"drain","context":{"idset":"11889","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0089545,"name":"drain","context":{"idset":"11890","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.011353,"name":"drain","context":{"idset":"10435","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0126021,"name":"drain","context":{"idset":"10436","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0138545,"name":"drain","context":{"idset":"10437","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0151057,"name":"drain","context":{"idset":"10438","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0163176,"name":"drain","context":{"idset":"10439","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0175149,"name":"drain","context":{"idset":"10440","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0208285,"name":"drain","context":{"idset":"10441","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0220628,"name":"drain","context":{"idset":"10442","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0232887,"name":"drain","context":{"idset":"10443","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0245101,"name":"drain","context":{"idset":"10444","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0264814,"name":"drain","context":{"idset":"10445","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.027694,"name":"drain","context":{"idset":"10446","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0289168,"name":"drain","context":{"idset":"10447","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0301414,"name":"drain","context":{"idset":"10448","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0313733,"name":"drain","context":{"idset":"10449","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0326002,"name":"drain","context":{"idset":"10450","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0337737,"name":"drain","context":{"idset":"10451","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0350163,"name":"drain","context":{"idset":"10452","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0361969,"name":"drain","context":{"idset":"10453","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0371864,"name":"drain","context":{"idset":"10454","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0381839,"name":"drain","context":{"idset":"10455","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0434313,"name":"drain","context":{"idset":"10456","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0446541,"name":"drain","context":{"idset":"10457","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0458732,"name":"drain","context":{"idset":"10458","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0471237,"name":"drain","context":{"idset":"10459","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.048363,"name":"drain","context":{"idset":"10460","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0495849,"name":"drain","context":{"idset":"10461","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0508389,"name":"drain","context":{"idset":"10462","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.052067,"name":"drain","context":{"idset":"10463","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0532923,"name":"drain","context":{"idset":"10464","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0545185,"name":"drain","context":{"idset":"10465","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0557528,"name":"drain","context":{"idset":"10466","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.056977,"name":"drain","context":{"idset":"10467","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0582006,"name":"drain","context":{"idset":"10468","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0594275,"name":"drain","context":{"idset":"10485","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0606472,"name":"drain","context":{"idset":"10486","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0618768,"name":"drain","context":{"idset":"10487","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0630696,"name":"drain","context":{"idset":"10488","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0642431,"name":"drain","context":{"idset":"10489","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.067704,"name":"drain","context":{"idset":"10490","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0688567,"name":"drain","context":{"idset":"10491","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0699775,"name":"drain","context":{"idset":"10492","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0711169,"name":"drain","context":{"idset":"10493","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0722785,"name":"drain","context":{"idset":"10494","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0734034,"name":"drain","context":{"idset":"10495","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0752816,"name":"drain","context":{"idset":"10496","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0764198,"name":"drain","context":{"idset":"10497","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0775535,"name":"drain","context":{"idset":"10498","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0786877,"name":"drain","context":{"idset":"10499","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.079814,"name":"drain","context":{"idset":"10500","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0809467,"name":"drain","context":{"idset":"10501","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0820792,"name":"drain","context":{"idset":"10502","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.084516,"name":"drain","context":{"idset":"10503","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0857632,"name":"drain","context":{"idset":"10504","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0871053,"name":"drain","context":{"idset":"10505","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0883663,"name":"drain","context":{"idset":"10506","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0907516,"name":"drain","context":{"idset":"10507","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0919859,"name":"drain","context":{"idset":"10508","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0932348,"name":"drain","context":{"idset":"10509","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0946014,"name":"drain","context":{"idset":"10510","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0959237,"name":"drain","context":{"idset":"10511","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0973835,"name":"drain","context":{"idset":"10512","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.0988679,"name":"drain","context":{"idset":"10513","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1003928,"name":"drain","context":{"idset":"10514","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1017499,"name":"drain","context":{"idset":"10515","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1035187,"name":"drain","context":{"idset":"10516","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.104883,"name":"drain","context":{"idset":"10519","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1062777,"name":"drain","context":{"idset":"10520","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1076024,"name":"drain","context":{"idset":"10521","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.109386,"name":"drain","context":{"idset":"10522","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1107264,"name":"drain","context":{"idset":"10523","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.112082,"name":"drain","context":{"idset":"10524","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1134474,"name":"drain","context":{"idset":"10525","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1148095,"name":"drain","context":{"idset":"10526","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1165373,"name":"drain","context":{"idset":"10527","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1179144,"name":"drain","context":{"idset":"10528","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1193247,"name":"drain","context":{"idset":"10529","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1219752,"name":"drain","context":{"idset":"10530","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1233871,"name":"drain","context":{"idset":"10531","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1247387,"name":"drain","context":{"idset":"10532","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1261454,"name":"drain","context":{"idset":"10533","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1271884,"name":"drain","context":{"idset":"10534","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1285276,"name":"drain","context":{"idset":"10535","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1300011,"name":"drain","context":{"idset":"10536","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1313689,"name":"drain","context":{"idset":"10537","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1327407,"name":"drain","context":{"idset":"10538","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1342874,"name":"drain","context":{"idset":"10539","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1356485,"name":"drain","context":{"idset":"10540","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1371181,"name":"drain","context":{"idset":"10541","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1385236,"name":"drain","context":{"idset":"10542","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.139905,"name":"drain","context":{"idset":"10543","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1413028,"name":"drain","context":{"idset":"10544","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1428258,"name":"drain","context":{"idset":"10545","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1442266,"name":"drain","context":{"idset":"10546","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1456349,"name":"drain","context":{"idset":"10547","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1470184,"name":"drain","context":{"idset":"10548","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1488123,"name":"drain","context":{"idset":"10549","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1501887,"name":"drain","context":{"idset":"10550","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1516659,"name":"drain","context":{"idset":"10551","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1530664,"name":"drain","context":{"idset":"10552","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1546583,"name":"drain","context":{"idset":"10553","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1563311,"name":"drain","context":{"idset":"10554","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1576934,"name":"drain","context":{"idset":"10555","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1588137,"name":"drain","context":{"idset":"10556","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1598573,"name":"drain","context":{"idset":"10557","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1609113,"name":"drain","context":{"idset":"10558","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1620231,"name":"drain","context":{"idset":"10559","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1644859,"name":"drain","context":{"idset":"10560","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1657617,"name":"drain","context":{"idset":"10561","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.167011,"name":"drain","context":{"idset":"10562","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1682913,"name":"drain","context":{"idset":"10563","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1696637,"name":"drain","context":{"idset":"10564","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.171031,"name":"drain","context":{"idset":"10565","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1724203,"name":"drain","context":{"idset":"10566","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1737912,"name":"drain","context":{"idset":"10567","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1755435,"name":"drain","context":{"idset":"10568","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1769121,"name":"drain","context":{"idset":"10569","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1783066,"name":"drain","context":{"idset":"10570","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1797056,"name":"drain","context":{"idset":"10571","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1810873,"name":"drain","context":{"idset":"10572","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1824934,"name":"drain","context":{"idset":"10573","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1837749,"name":"drain","context":{"idset":"10574","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1850677,"name":"drain","context":{"idset":"10575","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1863353,"name":"drain","context":{"idset":"10576","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1876011,"name":"drain","context":{"idset":"10577","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1888373,"name":"drain","context":{"idset":"10578","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1901062,"name":"drain","context":{"idset":"10579","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.191503,"name":"drain","context":{"idset":"10580","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.192807,"name":"drain","context":{"idset":"10581","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1942604,"name":"drain","context":{"idset":"10582","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1956263,"name":"drain","context":{"idset":"10583","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1969209,"name":"drain","context":{"idset":"10584","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.1985509,"name":"drain","context":{"idset":"10585","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2000768,"name":"drain","context":{"idset":"10586","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2014368,"name":"drain","context":{"idset":"10587","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2027321,"name":"drain","context":{"idset":"10588","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.203995,"name":"drain","context":{"idset":"10589","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2054217,"name":"drain","context":{"idset":"10590","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2067325,"name":"drain","context":{"idset":"10591","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2080231,"name":"drain","context":{"idset":"10592","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2097282,"name":"drain","context":{"idset":"10593","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2110062,"name":"drain","context":{"idset":"10594","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2124255,"name":"drain","context":{"idset":"10595","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2137225,"name":"drain","context":{"idset":"10596","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2150233,"name":"drain","context":{"idset":"10597","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2167416,"name":"drain","context":{"idset":"10598","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2180417,"name":"drain","context":{"idset":"10599","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2193565,"name":"drain","context":{"idset":"10600","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2206268,"name":"drain","context":{"idset":"10601","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2217128,"name":"drain","context":{"idset":"10602","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2227242,"name":"drain","context":{"idset":"10603","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2237258,"name":"drain","context":{"idset":"10604","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2250662,"name":"drain","context":{"idset":"10605","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2264566,"name":"drain","context":{"idset":"10606","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2276716,"name":"drain","context":{"idset":"10607","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2288873,"name":"drain","context":{"idset":"10608","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2309613,"name":"drain","context":{"idset":"10609","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2321894,"name":"drain","context":{"idset":"10610","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2335398,"name":"drain","context":{"idset":"10611","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2347693,"name":"drain","context":{"idset":"10612","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.236016,"name":"drain","context":{"idset":"10613","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2371938,"name":"drain","context":{"idset":"10614","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2385774,"name":"drain","context":{"idset":"10615","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2398384,"name":"drain","context":{"idset":"10616","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2411053,"name":"drain","context":{"idset":"10617","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2425115,"name":"drain","context":{"idset":"10618","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2437899,"name":"drain","context":{"idset":"10619","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2450819,"name":"drain","context":{"idset":"10620","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2464967,"name":"drain","context":{"idset":"10621","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2477784,"name":"drain","context":{"idset":"10622","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.249053,"name":"drain","context":{"idset":"10623","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2503381,"name":"drain","context":{"idset":"10624","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2517633,"name":"drain","context":{"idset":"10625","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2530327,"name":"drain","context":{"idset":"10626","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2543235,"name":"drain","context":{"idset":"10627","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2559566,"name":"drain","context":{"idset":"10628","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2572274,"name":"drain","context":{"idset":"10630","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2585235,"name":"drain","context":{"idset":"10631","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2597854,"name":"drain","context":{"idset":"10632","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2611711,"name":"drain","context":{"idset":"10633","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.26248,"name":"drain","context":{"idset":"10635","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2645473,"name":"drain","context":{"idset":"10636","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2658372,"name":"drain","context":{"idset":"10640","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2671137,"name":"drain","context":{"idset":"10641","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.268487,"name":"drain","context":{"idset":"10645","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.269767,"name":"drain","context":{"idset":"10646","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2710519,"name":"drain","context":{"idset":"10647","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2727184,"name":"drain","context":{"idset":"10648","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.273809,"name":"drain","context":{"idset":"10649","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2750096,"name":"drain","context":{"idset":"10650","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2764575,"name":"drain","context":{"idset":"10651","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2777562,"name":"drain","context":{"idset":"10652","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2790372,"name":"drain","context":{"idset":"10653","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2804511,"name":"drain","context":{"idset":"10654","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2817066,"name":"drain","context":{"idset":"10655","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2828426,"name":"drain","context":{"idset":"10656","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2840657,"name":"drain","context":{"idset":"10658","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2857282,"name":"drain","context":{"idset":"10659","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2870018,"name":"drain","context":{"idset":"10660","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2884183,"name":"drain","context":{"idset":"10661","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2897034,"name":"drain","context":{"idset":"10662","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2909837,"name":"drain","context":{"idset":"10663","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2922437,"name":"drain","context":{"idset":"10664","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2936757,"name":"drain","context":{"idset":"10665","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2949741,"name":"drain","context":{"idset":"10666","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2966583,"name":"drain","context":{"idset":"10667","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2979755,"name":"drain","context":{"idset":"10668","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.2993207,"name":"drain","context":{"idset":"10669","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.300801,"name":"drain","context":{"idset":"10670","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3021288,"name":"drain","context":{"idset":"10671","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3034627,"name":"drain","context":{"idset":"10672","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3048615,"name":"drain","context":{"idset":"10673","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.306077,"name":"drain","context":{"idset":"10674","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.307337,"name":"drain","context":{"idset":"10675","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3087218,"name":"drain","context":{"idset":"10676","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3100066,"name":"drain","context":{"idset":"10677","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.311321,"name":"drain","context":{"idset":"10678","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3126853,"name":"drain","context":{"idset":"10679","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3138845,"name":"drain","context":{"idset":"10680","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3149865,"name":"drain","context":{"idset":"10681","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3160565,"name":"drain","context":{"idset":"10682","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.317584,"name":"drain","context":{"idset":"10683","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3189881,"name":"drain","context":{"idset":"10684","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3204019,"name":"drain","context":{"idset":"10685","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3217695,"name":"drain","context":{"idset":"10686","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.323149,"name":"drain","context":{"idset":"10687","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3249636,"name":"drain","context":{"idset":"10688","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3263431,"name":"drain","context":{"idset":"10689","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3276243,"name":"drain","context":{"idset":"10690","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3288624,"name":"drain","context":{"idset":"10691","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3300877,"name":"drain","context":{"idset":"10692","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3314385,"name":"drain","context":{"idset":"10695","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3326116,"name":"drain","context":{"idset":"10696","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3338604,"name":"drain","context":{"idset":"10697","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3351135,"name":"drain","context":{"idset":"10698","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3363543,"name":"drain","context":{"idset":"10699","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.337688,"name":"drain","context":{"idset":"10700","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3393273,"name":"drain","context":{"idset":"10701","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3407218,"name":"drain","context":{"idset":"10703","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3421848,"name":"drain","context":{"idset":"10704","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.343555,"name":"drain","context":{"idset":"10705","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3448203,"name":"drain","context":{"idset":"10706","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3462119,"name":"drain","context":{"idset":"10707","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3476429,"name":"drain","context":{"idset":"10708","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.349159,"name":"drain","context":{"idset":"10709","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3505797,"name":"drain","context":{"idset":"10710","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3520191,"name":"drain","context":{"idset":"10711","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3535442,"name":"drain","context":{"idset":"10713","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3550763,"name":"drain","context":{"idset":"10714","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3565066,"name":"drain","context":{"idset":"10715","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3579061,"name":"drain","context":{"idset":"10716","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3594527,"name":"drain","context":{"idset":"10717","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3608482,"name":"drain","context":{"idset":"10719","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3622849,"name":"drain","context":{"idset":"10720","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3637414,"name":"drain","context":{"idset":"10721","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3653302,"name":"drain","context":{"idset":"10722","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3667889,"name":"drain","context":{"idset":"10723","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3686144,"name":"drain","context":{"idset":"10724","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3700507,"name":"drain","context":{"idset":"10725","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3714828,"name":"drain","context":{"idset":"10726","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3729296,"name":"drain","context":{"idset":"10727","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3744781,"name":"drain","context":{"idset":"10728","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3759916,"name":"drain","context":{"idset":"10729","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3774569,"name":"drain","context":{"idset":"10730","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3788064,"name":"drain","context":{"idset":"10731","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3801992,"name":"drain","context":{"idset":"10732","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.381675,"name":"drain","context":{"idset":"10733","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3830748,"name":"drain","context":{"idset":"10734","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.384522,"name":"drain","context":{"idset":"10735","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3859527,"name":"drain","context":{"idset":"10736","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3874946,"name":"drain","context":{"idset":"10737","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3889225,"name":"drain","context":{"idset":"10738","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3903553,"name":"drain","context":{"idset":"10741","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3918684,"name":"drain","context":{"idset":"10742","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3932011,"name":"drain","context":{"idset":"10743","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3944595,"name":"drain","context":{"idset":"10747","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3957071,"name":"drain","context":{"idset":"10748","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3969407,"name":"drain","context":{"idset":"10749","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3983572,"name":"drain","context":{"idset":"10750","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.3996542,"name":"drain","context":{"idset":"10751","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4011154,"name":"drain","context":{"idset":"10752","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4025383,"name":"drain","context":{"idset":"10753","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4039295,"name":"drain","context":{"idset":"10755","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4053562,"name":"drain","context":{"idset":"10756","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4066024,"name":"drain","context":{"idset":"10757","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4081187,"name":"drain","context":{"idset":"10758","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4095581,"name":"drain","context":{"idset":"10759","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4109802,"name":"drain","context":{"idset":"10760","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4124117,"name":"drain","context":{"idset":"10761","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4142241,"name":"drain","context":{"idset":"10762","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4156668,"name":"drain","context":{"idset":"10763","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.417079,"name":"drain","context":{"idset":"10764","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4186192,"name":"drain","context":{"idset":"10765","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4203119,"name":"drain","context":{"idset":"10766","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4216306,"name":"drain","context":{"idset":"10767","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4230642,"name":"drain","context":{"idset":"10768","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4246368,"name":"drain","context":{"idset":"10769","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.426048,"name":"drain","context":{"idset":"10770","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4273868,"name":"drain","context":{"idset":"10771","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4286878,"name":"drain","context":{"idset":"10772","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4299524,"name":"drain","context":{"idset":"10773","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4313509,"name":"drain","context":{"idset":"10774","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4326408,"name":"drain","context":{"idset":"10775","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4340701,"name":"drain","context":{"idset":"10776","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4355087,"name":"drain","context":{"idset":"10777","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4370434,"name":"drain","context":{"idset":"10778","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.438324,"name":"drain","context":{"idset":"10779","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4395952,"name":"drain","context":{"idset":"10780","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4409604,"name":"drain","context":{"idset":"10781","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4422889,"name":"drain","context":{"idset":"10782","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4437256,"name":"drain","context":{"idset":"10783","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4452364,"name":"drain","context":{"idset":"10784","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4467032,"name":"drain","context":{"idset":"10785","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4481344,"name":"drain","context":{"idset":"10786","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4495811,"name":"drain","context":{"idset":"10787","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.451344,"name":"drain","context":{"idset":"10788","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4527833,"name":"drain","context":{"idset":"10789","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4542189,"name":"drain","context":{"idset":"10790","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4557712,"name":"drain","context":{"idset":"10791","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4572055,"name":"drain","context":{"idset":"10792","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4586508,"name":"drain","context":{"idset":"10793","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4600863,"name":"drain","context":{"idset":"10794","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4618645,"name":"drain","context":{"idset":"10795","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.463145,"name":"drain","context":{"idset":"10796","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4644418,"name":"drain","context":{"idset":"10797","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4656925,"name":"drain","context":{"idset":"10798","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4672234,"name":"drain","context":{"idset":"10799","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4685183,"name":"drain","context":{"idset":"10800","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4698527,"name":"drain","context":{"idset":"10801","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4711239,"name":"drain","context":{"idset":"10802","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4724224,"name":"drain","context":{"idset":"10803","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4737899,"name":"drain","context":{"idset":"10804","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4751818,"name":"drain","context":{"idset":"10805","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4766843,"name":"drain","context":{"idset":"10806","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4781287,"name":"drain","context":{"idset":"10807","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.479588,"name":"drain","context":{"idset":"10808","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4811537,"name":"drain","context":{"idset":"10809","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.482619,"name":"drain","context":{"idset":"10810","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4840636,"name":"drain","context":{"idset":"10811","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4855251,"name":"drain","context":{"idset":"10812","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4870958,"name":"drain","context":{"idset":"10813","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4887302,"name":"drain","context":{"idset":"10814","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4901927,"name":"drain","context":{"idset":"10815","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4917588,"name":"drain","context":{"idset":"10816","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4931858,"name":"drain","context":{"idset":"10817","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4946108,"name":"drain","context":{"idset":"10818","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4960117,"name":"drain","context":{"idset":"10819","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.4975493,"name":"drain","context":{"idset":"10820","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.499007,"name":"drain","context":{"idset":"10821","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5003867,"name":"drain","context":{"idset":"10822","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5017667,"name":"drain","context":{"idset":"10823","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5033407,"name":"drain","context":{"idset":"10824","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5047057,"name":"drain","context":{"idset":"10825","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5059626,"name":"drain","context":{"idset":"10826","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5072393,"name":"drain","context":{"idset":"10827","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5085294,"name":"drain","context":{"idset":"10828","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.509939,"name":"drain","context":{"idset":"10829","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5115242,"name":"drain","context":{"idset":"10830","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5129724,"name":"drain","context":{"idset":"10831","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5144451,"name":"drain","context":{"idset":"10832","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5160236,"name":"drain","context":{"idset":"10833","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5174994,"name":"drain","context":{"idset":"10834","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5189657,"name":"drain","context":{"idset":"10835","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5204804,"name":"drain","context":{"idset":"10836","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5219486,"name":"drain","context":{"idset":"10837","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5234137,"name":"drain","context":{"idset":"10838","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.524878,"name":"drain","context":{"idset":"10839","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5267119,"name":"drain","context":{"idset":"10840","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5279751,"name":"drain","context":{"idset":"10841","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5291958,"name":"drain","context":{"idset":"10842","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5306213,"name":"drain","context":{"idset":"10843","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5319729,"name":"drain","context":{"idset":"10844","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5332179,"name":"drain","context":{"idset":"10845","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5345714,"name":"drain","context":{"idset":"10846","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5359735,"name":"drain","context":{"idset":"10847","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5373683,"name":"drain","context":{"idset":"10848","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5386767,"name":"drain","context":{"idset":"10849","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5400937,"name":"drain","context":{"idset":"10850","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.541626,"name":"drain","context":{"idset":"10851","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5430517,"name":"drain","context":{"idset":"10852","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5445378,"name":"drain","context":{"idset":"10853","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5458968,"name":"drain","context":{"idset":"10854","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5473864,"name":"drain","context":{"idset":"10855","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.548732,"name":"drain","context":{"idset":"10856","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5500634,"name":"drain","context":{"idset":"10857","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5515354,"name":"drain","context":{"idset":"10858","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5531802,"name":"drain","context":{"idset":"10859","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5546746,"name":"drain","context":{"idset":"10860","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5561459,"name":"drain","context":{"idset":"10861","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.557729,"name":"drain","context":{"idset":"10862","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5590324,"name":"drain","context":{"idset":"10863","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5602348,"name":"drain","context":{"idset":"10864","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5614958,"name":"drain","context":{"idset":"10865","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5627012,"name":"drain","context":{"idset":"10866","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5641937,"name":"drain","context":{"idset":"10867","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5654664,"name":"drain","context":{"idset":"10868","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5668678,"name":"drain","context":{"idset":"10869","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5683241,"name":"drain","context":{"idset":"10870","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5696204,"name":"drain","context":{"idset":"10873","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5708966,"name":"drain","context":{"idset":"10874","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5721862,"name":"drain","context":{"idset":"10875","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5734069,"name":"drain","context":{"idset":"10876","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5749142,"name":"drain","context":{"idset":"10877","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5765309,"name":"drain","context":{"idset":"10878","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5780449,"name":"drain","context":{"idset":"10879","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5795424,"name":"drain","context":{"idset":"10880","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.581147,"name":"drain","context":{"idset":"10881","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5826521,"name":"drain","context":{"idset":"10882","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5841231,"name":"drain","context":{"idset":"10883","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5857389,"name":"drain","context":{"idset":"10884","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5872529,"name":"drain","context":{"idset":"10885","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5887418,"name":"drain","context":{"idset":"10886","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5902388,"name":"drain","context":{"idset":"10887","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5916567,"name":"drain","context":{"idset":"10888","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5930724,"name":"drain","context":{"idset":"10889","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5945077,"name":"drain","context":{"idset":"10890","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5959349,"name":"drain","context":{"idset":"10891","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.5973527,"name":"drain","context":{"idset":"10892","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.598789,"name":"drain","context":{"idset":"10893","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6002047,"name":"drain","context":{"idset":"10894","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.601598,"name":"drain","context":{"idset":"10895","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6030266,"name":"drain","context":{"idset":"10896","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6044648,"name":"drain","context":{"idset":"10897","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6058545,"name":"drain","context":{"idset":"10898","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6072285,"name":"drain","context":{"idset":"10899","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.608619,"name":"drain","context":{"idset":"10900","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6099668,"name":"drain","context":{"idset":"10901","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6113555,"name":"drain","context":{"idset":"10902","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6127388,"name":"drain","context":{"idset":"10903","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6141219,"name":"drain","context":{"idset":"10904","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6154957,"name":"drain","context":{"idset":"10905","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6168463,"name":"drain","context":{"idset":"10906","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6182103,"name":"drain","context":{"idset":"10907","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6194403,"name":"drain","context":{"idset":"10908","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6208265,"name":"drain","context":{"idset":"10909","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6221991,"name":"drain","context":{"idset":"10910","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6235487,"name":"drain","context":{"idset":"10913","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6247911,"name":"drain","context":{"idset":"10914","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6260934,"name":"drain","context":{"idset":"10915","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.628031,"name":"drain","context":{"idset":"10916","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6293964,"name":"drain","context":{"idset":"10917","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6307621,"name":"drain","context":{"idset":"10918","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6321368,"name":"drain","context":{"idset":"10919","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6335115,"name":"drain","context":{"idset":"10920","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.634846,"name":"drain","context":{"idset":"10921","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6361876,"name":"drain","context":{"idset":"10922","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6374958,"name":"drain","context":{"idset":"10923","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6388276,"name":"drain","context":{"idset":"10924","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6402225,"name":"drain","context":{"idset":"10925","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6416175,"name":"drain","context":{"idset":"10926","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6430187,"name":"drain","context":{"idset":"10927","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6444137,"name":"drain","context":{"idset":"10928","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6457117,"name":"drain","context":{"idset":"10929","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6470022,"name":"drain","context":{"idset":"10930","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6482887,"name":"drain","context":{"idset":"10931","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6496804,"name":"drain","context":{"idset":"10932","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6510775,"name":"drain","context":{"idset":"10933","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6524987,"name":"drain","context":{"idset":"10934","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6536975,"name":"drain","context":{"idset":"10935","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6549351,"name":"drain","context":{"idset":"10936","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6562226,"name":"drain","context":{"idset":"10937","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6575868,"name":"drain","context":{"idset":"10938","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6588869,"name":"drain","context":{"idset":"10939","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6600273,"name":"drain","context":{"idset":"10940","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6612535,"name":"drain","context":{"idset":"10941","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6624756,"name":"drain","context":{"idset":"10942","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6636028,"name":"drain","context":{"idset":"10945","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6648605,"name":"drain","context":{"idset":"10946","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6662395,"name":"drain","context":{"idset":"10947","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6675792,"name":"drain","context":{"idset":"10948","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6688147,"name":"drain","context":{"idset":"10949","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6699927,"name":"drain","context":{"idset":"10950","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6712177,"name":"drain","context":{"idset":"10951","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6726081,"name":"drain","context":{"idset":"10952","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6739976,"name":"drain","context":{"idset":"10953","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6752384,"name":"drain","context":{"idset":"10954","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6766272,"name":"drain","context":{"idset":"10955","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6778798,"name":"drain","context":{"idset":"10956","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6792941,"name":"drain","context":{"idset":"10957","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6806967,"name":"drain","context":{"idset":"10958","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6821017,"name":"drain","context":{"idset":"10959","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6835053,"name":"drain","context":{"idset":"10960","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.684891,"name":"drain","context":{"idset":"10961","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6862524,"name":"drain","context":{"idset":"10962","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.687644,"name":"drain","context":{"idset":"10963","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6889157,"name":"drain","context":{"idset":"10964","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6902483,"name":"drain","context":{"idset":"10965","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6915576,"name":"drain","context":{"idset":"10966","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6928701,"name":"drain","context":{"idset":"10967","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6942816,"name":"drain","context":{"idset":"10968","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6956689,"name":"drain","context":{"idset":"10969","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.696857,"name":"drain","context":{"idset":"10970","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6981361,"name":"drain","context":{"idset":"10971","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.6995471,"name":"drain","context":{"idset":"10972","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7009287,"name":"drain","context":{"idset":"10973","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7023945,"name":"drain","context":{"idset":"10974","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7038093,"name":"drain","context":{"idset":"10975","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7052426,"name":"drain","context":{"idset":"10976","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7064202,"name":"drain","context":{"idset":"10977","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7077522,"name":"drain","context":{"idset":"10978","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7089591,"name":"drain","context":{"idset":"10979","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7102869,"name":"drain","context":{"idset":"10980","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7115014,"name":"drain","context":{"idset":"10981","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7129159,"name":"drain","context":{"idset":"10982","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.714323,"name":"drain","context":{"idset":"10983","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7157362,"name":"drain","context":{"idset":"10984","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7171493,"name":"drain","context":{"idset":"10985","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7185755,"name":"drain","context":{"idset":"10986","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7198606,"name":"drain","context":{"idset":"10987","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.721257,"name":"drain","context":{"idset":"10988","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7226937,"name":"drain","context":{"idset":"10989","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7240527,"name":"drain","context":{"idset":"10990","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7253597,"name":"drain","context":{"idset":"10991","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7265878,"name":"drain","context":{"idset":"10992","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7278929,"name":"drain","context":{"idset":"10993","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7292132,"name":"drain","context":{"idset":"10994","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7305598,"name":"drain","context":{"idset":"10995","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7319136,"name":"drain","context":{"idset":"10996","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7332609,"name":"drain","context":{"idset":"10997","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7344804,"name":"drain","context":{"idset":"10998","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7356379,"name":"drain","context":{"idset":"11001","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7368405,"name":"drain","context":{"idset":"11002","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7381883,"name":"drain","context":{"idset":"11003","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7394762,"name":"drain","context":{"idset":"11004","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7408423,"name":"drain","context":{"idset":"11005","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7421768,"name":"drain","context":{"idset":"11006","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7436144,"name":"drain","context":{"idset":"11007","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7450109,"name":"drain","context":{"idset":"11008","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7464333,"name":"drain","context":{"idset":"11009","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7478471,"name":"drain","context":{"idset":"11010","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7492621,"name":"drain","context":{"idset":"11011","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7506542,"name":"drain","context":{"idset":"11012","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7520661,"name":"drain","context":{"idset":"11013","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7535188,"name":"drain","context":{"idset":"11014","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7549002,"name":"drain","context":{"idset":"11015","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7563231,"name":"drain","context":{"idset":"11016","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7577384,"name":"drain","context":{"idset":"11017","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7591608,"name":"drain","context":{"idset":"11018","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7604985,"name":"drain","context":{"idset":"11019","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.761796,"name":"drain","context":{"idset":"11020","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7631791,"name":"drain","context":{"idset":"11021","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7646089,"name":"drain","context":{"idset":"11022","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7659755,"name":"drain","context":{"idset":"11023","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7671754,"name":"drain","context":{"idset":"11025","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7682438,"name":"drain","context":{"idset":"11028","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7695935,"name":"drain","context":{"idset":"11030","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7708933,"name":"drain","context":{"idset":"11032","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7721975,"name":"drain","context":{"idset":"11035","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7734838,"name":"drain","context":{"idset":"11037","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7746801,"name":"drain","context":{"idset":"11039","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7759223,"name":"drain","context":{"idset":"11045","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7772565,"name":"drain","context":{"idset":"11049","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7787285,"name":"drain","context":{"idset":"11051","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7801542,"name":"drain","context":{"idset":"11053","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7813971,"name":"drain","context":{"idset":"11055","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7827647,"name":"drain","context":{"idset":"11056","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7840767,"name":"drain","context":{"idset":"11057","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7855053,"name":"drain","context":{"idset":"11059","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7870526,"name":"drain","context":{"idset":"11060","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7883496,"name":"drain","context":{"idset":"11062","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7896044,"name":"drain","context":{"idset":"11063","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7910349,"name":"drain","context":{"idset":"11066","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7923901,"name":"drain","context":{"idset":"11067","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7935739,"name":"drain","context":{"idset":"11068","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7947004,"name":"drain","context":{"idset":"11069","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7958555,"name":"drain","context":{"idset":"11071","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7970934,"name":"drain","context":{"idset":"11072","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7984312,"name":"drain","context":{"idset":"11074","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.7996171,"name":"drain","context":{"idset":"11075","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8008478,"name":"drain","context":{"idset":"11076","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8021166,"name":"drain","context":{"idset":"11077","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8034725,"name":"drain","context":{"idset":"11080","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8048832,"name":"drain","context":{"idset":"11081","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8060248,"name":"drain","context":{"idset":"11082","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8072615,"name":"drain","context":{"idset":"11083","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8084774,"name":"drain","context":{"idset":"11085","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8097854,"name":"drain","context":{"idset":"11086","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8112106,"name":"drain","context":{"idset":"11087","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.812464,"name":"drain","context":{"idset":"11088","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8137383,"name":"drain","context":{"idset":"11089","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8150618,"name":"drain","context":{"idset":"11090","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8164814,"name":"drain","context":{"idset":"11091","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8178902,"name":"drain","context":{"idset":"11093","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8192811,"name":"drain","context":{"idset":"11094","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8206275,"name":"drain","context":{"idset":"11095","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8219941,"name":"drain","context":{"idset":"11097","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8234344,"name":"drain","context":{"idset":"11098","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8248715,"name":"drain","context":{"idset":"11099","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8263247,"name":"drain","context":{"idset":"11100","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8276126,"name":"drain","context":{"idset":"11102","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8290334,"name":"drain","context":{"idset":"11103","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8304687,"name":"drain","context":{"idset":"11105","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8319046,"name":"drain","context":{"idset":"11107","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8333621,"name":"drain","context":{"idset":"11108","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8347414,"name":"drain","context":{"idset":"11109","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8361681,"name":"drain","context":{"idset":"11111","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8375587,"name":"drain","context":{"idset":"11112","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8388634,"name":"drain","context":{"idset":"11113","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8399701,"name":"drain","context":{"idset":"11114","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8411157,"name":"drain","context":{"idset":"11117","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8425481,"name":"drain","context":{"idset":"11119","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8439481,"name":"drain","context":{"idset":"11122","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.845279,"name":"drain","context":{"idset":"11123","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8466804,"name":"drain","context":{"idset":"11124","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8481138,"name":"drain","context":{"idset":"11126","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8496313,"name":"drain","context":{"idset":"11127","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8509765,"name":"drain","context":{"idset":"11128","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8523879,"name":"drain","context":{"idset":"11129","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8538392,"name":"drain","context":{"idset":"11130","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8551555,"name":"drain","context":{"idset":"11131","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8565974,"name":"drain","context":{"idset":"11132","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8580229,"name":"drain","context":{"idset":"11133","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8594742,"name":"drain","context":{"idset":"11135","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8607345,"name":"drain","context":{"idset":"11136","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8619211,"name":"drain","context":{"idset":"11138","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8633106,"name":"drain","context":{"idset":"11141","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8647346,"name":"drain","context":{"idset":"11143","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8661613,"name":"drain","context":{"idset":"11144","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8674996,"name":"drain","context":{"idset":"11146","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8689346,"name":"drain","context":{"idset":"11147","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8703446,"name":"drain","context":{"idset":"11148","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8717644,"name":"drain","context":{"idset":"11149","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8732047,"name":"drain","context":{"idset":"11150","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.874619,"name":"drain","context":{"idset":"11152","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8759778,"name":"drain","context":{"idset":"11153","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8777635,"name":"drain","context":{"idset":"11154","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8792026,"name":"drain","context":{"idset":"11156","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8806477,"name":"drain","context":{"idset":"11157","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8820834,"name":"drain","context":{"idset":"11159","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8835189,"name":"drain","context":{"idset":"11160","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8849633,"name":"drain","context":{"idset":"11161","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8864169,"name":"drain","context":{"idset":"11162","reason":"broker was unresponsive"}}
+{"timestamp":1712413957.8878465,"name":"drain","context":{"idset":"11163","reason":"broker was unresponsive"}}
+{"timestamp":1712444558.5799844,"name":"online","context":{"idset":"11587"}}
+{"timestamp":1712444558.8907423,"name":"online","context":{"idset":"11588"}}
+{"timestamp":1712457566.3562357,"name":"offline","context":{"idset":"10396"}}
+{"timestamp":1712502916.9239972,"name":"undrain","context":{"idset":"10920"}}
+{"timestamp":1712503300.6401894,"name":"undrain","context":{"idset":"10925"}}
+{"timestamp":1712503301.0853434,"name":"undrain","context":{"idset":"10930"}}
+{"timestamp":1712503301.5320129,"name":"undrain","context":{"idset":"10935"}}
+{"timestamp":1712503301.9758856,"name":"undrain","context":{"idset":"10940"}}
+{"timestamp":1712503303.1917796,"name":"undrain","context":{"idset":"10945"}}
+{"timestamp":1712503303.627337,"name":"undrain","context":{"idset":"10950"}}
+{"timestamp":1712503304.0760038,"name":"undrain","context":{"idset":"10955"}}
+{"timestamp":1712503304.5613346,"name":"undrain","context":{"idset":"10960"}}
+{"timestamp":1712503304.9953768,"name":"undrain","context":{"idset":"10965"}}
+{"timestamp":1712503305.5881369,"name":"undrain","context":{"idset":"10970"}}
+{"timestamp":1712503395.368453,"name":"undrain","context":{"idset":"10975"}}
+{"timestamp":1712503571.0097067,"name":"undrain","context":{"idset":"10980"}}
+{"timestamp":1712504082.7305484,"name":"undrain","context":{"idset":"10985"}}
+{"timestamp":1712504083.1612532,"name":"undrain","context":{"idset":"10990"}}
+{"timestamp":1712504083.5802038,"name":"undrain","context":{"idset":"10993"}}
+{"timestamp":1712504084.0462198,"name":"undrain","context":{"idset":"10994"}}
+{"timestamp":1712504084.4673645,"name":"undrain","context":{"idset":"10995"}}
+{"timestamp":1712504084.8892269,"name":"undrain","context":{"idset":"10996"}}
+{"timestamp":1712507305.3461049,"name":"drain","context":{"idset":"10920,10925,10930,10935,10940,10945,10950,10955,10960,10965","overwrite":0}}
+{"timestamp":1712509491.5233767,"name":"drain","context":{"idset":"10970,10975,10980,10985,10990,10993-10996","overwrite":0}}
+{"timestamp":1712523855.472235,"name":"drain","context":{"idset":"10871-10872,10911-10912,10943-10944","overwrite":0}}
+{"timestamp":1712524755.9559476,"name":"drain","context":{"idset":"11522,11527,11552,11563,11632","overwrite":0}}
+{"timestamp":1712525320.4711599,"name":"offline","context":{"idset":"11781"}}
+{"timestamp":1712525320.4894431,"name":"offline","context":{"idset":"11782"}}
+{"timestamp":1712525320.5589392,"name":"offline","context":{"idset":"11783"}}
+{"timestamp":1712525322.4681697,"name":"offline","context":{"idset":"11785"}}
+{"timestamp":1712525322.4814742,"name":"offline","context":{"idset":"11786"}}
+{"timestamp":1712525322.5585265,"name":"offline","context":{"idset":"11787"}}
+{"timestamp":1712525324.4708083,"name":"offline","context":{"idset":"11788"}}
+{"timestamp":1712525324.4873214,"name":"offline","context":{"idset":"11791"}}
+{"timestamp":1712525324.5626979,"name":"offline","context":{"idset":"11792"}}
+{"timestamp":1712525326.4664011,"name":"offline","context":{"idset":"11795"}}
+{"timestamp":1712525326.5596037,"name":"offline","context":{"idset":"11796"}}
+{"timestamp":1712525596.4672961,"name":"offline","context":{"idset":"11813"}}
+{"timestamp":1712525596.5612075,"name":"offline","context":{"idset":"11814"}}
+{"timestamp":1712525598.4691138,"name":"offline","context":{"idset":"11817"}}
+{"timestamp":1712525598.4839416,"name":"offline","context":{"idset":"11818"}}
+{"timestamp":1712525598.5611122,"name":"offline","context":{"idset":"11819"}}
+{"timestamp":1712525600.4792542,"name":"offline","context":{"idset":"11820"}}
+{"timestamp":1712525600.4988232,"name":"offline","context":{"idset":"11821"}}
+{"timestamp":1712525600.5179188,"name":"offline","context":{"idset":"11822"}}
+{"timestamp":1712525600.5369394,"name":"offline","context":{"idset":"11823"}}
+{"timestamp":1712525600.5630081,"name":"offline","context":{"idset":"11824"}}
+{"timestamp":1712525602.4678988,"name":"offline","context":{"idset":"11825"}}
+{"timestamp":1712525602.4786549,"name":"offline","context":{"idset":"11826"}}
+{"timestamp":1712525602.4889057,"name":"offline","context":{"idset":"11827"}}
+{"timestamp":1712525602.5649004,"name":"offline","context":{"idset":"11828"}}
+{"timestamp":1712525956.4770203,"name":"offline","context":{"idset":"11877"}}
+{"timestamp":1712525956.4980161,"name":"offline","context":{"idset":"11878"}}
+{"timestamp":1712525956.5189037,"name":"offline","context":{"idset":"11879"}}
+{"timestamp":1712525956.6808462,"name":"offline","context":{"idset":"11880"}}
+{"timestamp":1712526049.2944343,"name":"offline","context":{"idset":"11881"}}
+{"timestamp":1712526049.2960753,"name":"offline","context":{"idset":"11882"}}
+{"timestamp":1712526049.2976894,"name":"offline","context":{"idset":"11883"}}
+{"timestamp":1712526049.2992876,"name":"offline","context":{"idset":"11885"}}
+{"timestamp":1712526049.3008831,"name":"offline","context":{"idset":"11886"}}
+{"timestamp":1712526049.3024859,"name":"offline","context":{"idset":"11887"}}
+{"timestamp":1712526049.304069,"name":"offline","context":{"idset":"11888"}}
+{"timestamp":1712526049.3056676,"name":"offline","context":{"idset":"11889"}}
+{"timestamp":1712526049.3072515,"name":"offline","context":{"idset":"11890"}}
+{"timestamp":1712531072.5602005,"name":"offline","context":{"idset":"10501"}}
+{"timestamp":1712531680.5577691,"name":"offline","context":{"idset":"9787"}}
+{"timestamp":1712535842.2940314,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1712535842.5251949,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1712535842.5474243,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1712535842.5753081,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1712535842.6095843,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1712535842.6301787,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1712535842.6609144,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1712535842.7410305,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1712535842.772213,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1712535842.793412,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1712535842.8305926,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1712535842.835984,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1712535842.8542051,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1712535842.9278245,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1712535843.0657921,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1712535843.0907822,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1712535843.2262323,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1712535843.2445121,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1712535843.2605557,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1712535843.2872207,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1712535843.3106411,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1712535843.3420987,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1712535843.3594456,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1712535843.3756561,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1712535843.4019587,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1712535843.4158828,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1712535843.4343567,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1712535843.4752977,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1712535843.4785416,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1712535843.4817724,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1712535843.4895132,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1712535843.5043318,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1712535843.5180361,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1712535843.5430522,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1712535843.5612152,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1712535843.5787144,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1712535843.6075585,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1712535843.643796,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1712535843.6454492,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1712535843.6471963,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1712535843.6566975,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1712535843.6597543,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1712535843.6788573,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1712535843.6904275,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1712535843.6920154,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1712535843.718034,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1712535843.7330339,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1712535843.7614734,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1712535843.764173,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1712535843.785151,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1712535843.7993684,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1712535843.8387988,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1712535843.8523602,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1712535844.1770983,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1712535844.3181334,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1712535844.4160497,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1712535844.4397795,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1712535844.465003,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1712535844.6215966,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1712535844.6349919,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1712535844.6630087,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1712535844.8413243,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1712535844.8571563,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1712535844.8727283,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1712535844.8970206,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1712535844.9028535,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1712535844.9179046,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1712535844.9327817,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1712535844.9481506,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1712535844.9878614,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1712535845.0031996,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1712535845.0186327,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1712535845.0339136,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1712535845.0529683,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1712535845.2262125,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1712535845.4287522,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1712535845.4425304,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1712535845.4595616,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1712535845.476707,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1712535845.4919245,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1712535845.6846845,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1712535845.7034369,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1712535845.7195506,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1712535845.7357092,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1712536557.2626669,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1712537894.4522297,"name":"online","context":{"idset":"11558"}}
+{"timestamp":1712537894.7075739,"name":"online","context":{"idset":"11555,11562"}}
+{"timestamp":1712537895.0269437,"name":"online","context":{"idset":"11547"}}
+{"timestamp":1712537895.0449347,"name":"online","context":{"idset":"11566"}}
+{"timestamp":1712537895.1759861,"name":"online","context":{"idset":"11550,11561"}}
+{"timestamp":1712537895.3714767,"name":"online","context":{"idset":"11574"}}
+{"timestamp":1712537895.4890625,"name":"online","context":{"idset":"11570"}}
+{"timestamp":1712537895.6011019,"name":"online","context":{"idset":"11549"}}
+{"timestamp":1712537895.7121453,"name":"online","context":{"idset":"11548"}}
+{"timestamp":1712537895.9219599,"name":"online","context":{"idset":"11551,11559,11565,11581,11594,11601"}}
+{"timestamp":1712537896.1169946,"name":"online","context":{"idset":"11571,11573,11580,11589"}}
+{"timestamp":1712537896.2327719,"name":"online","context":{"idset":"11556,11576"}}
+{"timestamp":1712537896.343199,"name":"online","context":{"idset":"11553,11557,11572,11595"}}
+{"timestamp":1712537896.4534717,"name":"online","context":{"idset":"11577,11579,11593"}}
+{"timestamp":1712537896.5685585,"name":"online","context":{"idset":"11567,11582,11586,11596,11600"}}
+{"timestamp":1712537896.7623444,"name":"online","context":{"idset":"11554,11560,11564,11599"}}
+{"timestamp":1712537896.9652455,"name":"online","context":{"idset":"11568-11569,11585,11590"}}
+{"timestamp":1712537897.0915058,"name":"online","context":{"idset":"11575,11598"}}
+{"timestamp":1712537897.3026221,"name":"online","context":{"idset":"11578,11583-11584,11591-11592,11603,11605-11606,11608,11610,11612-11614"}}
+{"timestamp":1712537897.5157173,"name":"online","context":{"idset":"11597,11607,11611,11615"}}
+{"timestamp":1712537897.6425848,"name":"online","context":{"idset":"11602,11604"}}
+{"timestamp":1712537897.8460956,"name":"online","context":{"idset":"11609"}}
+{"timestamp":1712537904.413765,"name":"online","context":{"idset":"11619"}}
+{"timestamp":1712537904.8215244,"name":"online","context":{"idset":"11617"}}
+{"timestamp":1712537906.7009499,"name":"online","context":{"idset":"11618"}}
+{"timestamp":1712537907.4300761,"name":"online","context":{"idset":"11620"}}
+{"timestamp":1712537907.6588278,"name":"online","context":{"idset":"11616"}}
+{"timestamp":1712537914.3271339,"name":"online","context":{"idset":"11630"}}
+{"timestamp":1712537915.9822142,"name":"online","context":{"idset":"11622"}}
+{"timestamp":1712537916.5224242,"name":"online","context":{"idset":"11636"}}
+{"timestamp":1712537917.8709099,"name":"online","context":{"idset":"11626,11634"}}
+{"timestamp":1712537918.3954699,"name":"online","context":{"idset":"11624,11631"}}
+{"timestamp":1712537918.7745113,"name":"online","context":{"idset":"11628"}}
+{"timestamp":1712537919.5052793,"name":"online","context":{"idset":"11629"}}
+{"timestamp":1712537920.6185875,"name":"online","context":{"idset":"11625"}}
+{"timestamp":1712537921.0444524,"name":"online","context":{"idset":"11621"}}
+{"timestamp":1712537921.4424613,"name":"online","context":{"idset":"11627"}}
+{"timestamp":1712537921.5668638,"name":"online","context":{"idset":"11623,11633,11635"}}
+{"timestamp":1712542372.1907511,"name":"online","context":{"idset":"11527"}}
+{"timestamp":1712542372.1923499,"name":"online","context":{"idset":"11632"}}
+{"timestamp":1712543708.0058632,"name":"undrain","context":{"idset":"11527,11632"}}
+{"timestamp":1712549509.9401901,"name":"undrain","context":{"idset":"1-60,85-120,122-216,218-347,349-430,432-444,469-540,565-588,629-636,827-834,841-842,871-872,949-954,963-964,975-976,9205-9213,9215-9238,9241-9315,9317,9319-9446,9449-9532,9536-9560,9563-9632,9635-9724,9727-9749,9751-9786,9788-9844,9973-10137,10139-10142,10144,10146-10154,10157-10260,10262-10302,10304-10342,10345-10354,10357-10395,10397-10468,10485-10500,10502-10516,10519-10628,10630-10633,10635-10636,10640-10641,10645-10656,10658-10692,10695-10701,10703-10711,10713-10717,10719-10738,10741-10870,10873-10910,10913-10919,10921-10924,10926-10929,10931-10934,10936-10939,10941-10942,10946-10949,10951-10954,10956-10959,10961-10964,10966-10969,10971-10974,10976-10979,10981-10984,10986-10989,10991-10992,10997-10998,11001-11114,11117-11133,11135-11139,11141-11225,11227-11232,11234,11237-11246,11248,11250-11308,11443-11452,11505-11521,11523-11526,11528-11551,11553-11562,11564-11586,11589-11631,11633-11636,11653-11657,11678-11682,11684-11687,11699-11708,11739-11748,11759-11764,11802"}}
+{"timestamp":1712549588.2296007,"name":"undrain","context":{"idset":"11309-11328,11331-11357,11359-11372,11375-11386,11389,11391-11394,11396-11399,11430,11432-11438,11453-11463,11465,11467-11504,11658-11677,11688-11693,11695-11698,11709-11738,11749-11758"}}
+{"timestamp":1712571356.5604539,"name":"drain","context":{"idset":"10952","reason":"broker was unresponsive"}}
+{"timestamp":1712571420.5611153,"name":"offline","context":{"idset":"10952"}}
+{"timestamp":1712571624.5618145,"name":"drain","context":{"idset":"9472","reason":"broker was unresponsive"}}
+{"timestamp":1712571690.5597398,"name":"offline","context":{"idset":"9472"}}
+{"timestamp":1712577912.559926,"name":"drain","context":{"idset":"9790","reason":"broker was unresponsive"}}
+{"timestamp":1712577978.5580997,"name":"offline","context":{"idset":"9790"}}
+{"timestamp":1712584145.1700065,"name":"online","context":{"idset":"9634"}}
+{"timestamp":1712584145.8059225,"name":"online","context":{"idset":"9633"}}
+{"timestamp":1712584200.258918,"name":"undrain","context":{"idset":"9633-9634"}}
+{"timestamp":1712584544.4606605,"name":"drain","context":{"idset":"11125","reason":"broker was unresponsive"}}
+{"timestamp":1712584544.4607496,"name":"drain","context":{"idset":"11126","reason":"broker was unresponsive"}}
+{"timestamp":1712584544.4608061,"name":"drain","context":{"idset":"11127","reason":"broker was unresponsive"}}
+{"timestamp":1712584544.4608607,"name":"drain","context":{"idset":"11128","reason":"broker was unresponsive"}}
+{"timestamp":1712584544.4609113,"name":"drain","context":{"idset":"11129","reason":"broker was unresponsive"}}
+{"timestamp":1712584544.4609685,"name":"drain","context":{"idset":"11130","reason":"broker was unresponsive"}}
+{"timestamp":1712584544.5684049,"name":"drain","context":{"idset":"11131","reason":"broker was unresponsive"}}
+{"timestamp":1712584550.4608645,"name":"drain","context":{"idset":"11132","reason":"broker was unresponsive"}}
+{"timestamp":1712584550.4609644,"name":"drain","context":{"idset":"11133","reason":"broker was unresponsive"}}
+{"timestamp":1712584550.4610317,"name":"drain","context":{"idset":"11135","reason":"broker was unresponsive"}}
+{"timestamp":1712584550.4610937,"name":"drain","context":{"idset":"11136","reason":"broker was unresponsive"}}
+{"timestamp":1712584550.4611452,"name":"drain","context":{"idset":"11137","reason":"broker was unresponsive"}}
+{"timestamp":1712584550.4611986,"name":"drain","context":{"idset":"11138","reason":"broker was unresponsive"}}
+{"timestamp":1712584550.4612503,"name":"drain","context":{"idset":"11139","reason":"broker was unresponsive"}}
+{"timestamp":1712584550.4613204,"name":"drain","context":{"idset":"11141","reason":"broker was unresponsive"}}
+{"timestamp":1712584550.4613762,"name":"drain","context":{"idset":"11142","reason":"broker was unresponsive"}}
+{"timestamp":1712584550.5946293,"name":"drain","context":{"idset":"11143","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4619329,"name":"drain","context":{"idset":"11144","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4620414,"name":"drain","context":{"idset":"11145","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4621158,"name":"drain","context":{"idset":"11146","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4621851,"name":"drain","context":{"idset":"11147","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4622436,"name":"drain","context":{"idset":"11148","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4623184,"name":"drain","context":{"idset":"11149","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4623771,"name":"drain","context":{"idset":"11150","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4624424,"name":"drain","context":{"idset":"11151","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.462501,"name":"drain","context":{"idset":"11152","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4625692,"name":"drain","context":{"idset":"11153","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4626262,"name":"drain","context":{"idset":"11154","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4626832,"name":"drain","context":{"idset":"11155","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.4627428,"name":"drain","context":{"idset":"11156","reason":"broker was unresponsive"}}
+{"timestamp":1712584556.6427221,"name":"drain","context":{"idset":"11157","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.4638653,"name":"drain","context":{"idset":"11158","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.46399,"name":"drain","context":{"idset":"11159","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.4640834,"name":"drain","context":{"idset":"11160","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.4641702,"name":"drain","context":{"idset":"11161","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.4642599,"name":"drain","context":{"idset":"11162","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.4643643,"name":"drain","context":{"idset":"11163","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.4644504,"name":"drain","context":{"idset":"11164","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.4645391,"name":"drain","context":{"idset":"11165","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.4646227,"name":"drain","context":{"idset":"11166","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.4647171,"name":"drain","context":{"idset":"11167","reason":"broker was unresponsive"}}
+{"timestamp":1712584562.6404088,"name":"drain","context":{"idset":"11168","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.460901,"name":"drain","context":{"idset":"11169","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.4611142,"name":"drain","context":{"idset":"11170","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.4612603,"name":"drain","context":{"idset":"11171","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.4614315,"name":"drain","context":{"idset":"11172","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.461576,"name":"drain","context":{"idset":"11173","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.4617193,"name":"drain","context":{"idset":"11174","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.4618721,"name":"drain","context":{"idset":"11175","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.4620132,"name":"drain","context":{"idset":"11176","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.4621615,"name":"drain","context":{"idset":"11177","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.4623215,"name":"drain","context":{"idset":"11178","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.4624627,"name":"drain","context":{"idset":"11179","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.4626174,"name":"drain","context":{"idset":"11180","reason":"broker was unresponsive"}}
+{"timestamp":1712584568.6661704,"name":"drain","context":{"idset":"11181","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.4617302,"name":"drain","context":{"idset":"11182","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.4618704,"name":"drain","context":{"idset":"11183","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.4619682,"name":"drain","context":{"idset":"11184","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.4620626,"name":"drain","context":{"idset":"11185","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.4621515,"name":"drain","context":{"idset":"11186","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.4622865,"name":"drain","context":{"idset":"11187","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.4623814,"name":"drain","context":{"idset":"11188","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.4624729,"name":"drain","context":{"idset":"11189","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.4625604,"name":"drain","context":{"idset":"11190","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.4626536,"name":"drain","context":{"idset":"11191","reason":"broker was unresponsive"}}
+{"timestamp":1712584574.6380556,"name":"drain","context":{"idset":"11192","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.4611859,"name":"drain","context":{"idset":"11193","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.4613452,"name":"drain","context":{"idset":"11194","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.4614484,"name":"drain","context":{"idset":"11195","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.4615536,"name":"drain","context":{"idset":"11196","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.461669,"name":"drain","context":{"idset":"11197","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.4617867,"name":"drain","context":{"idset":"11198","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.4618855,"name":"drain","context":{"idset":"11199","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.4619849,"name":"drain","context":{"idset":"11200","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.4620826,"name":"drain","context":{"idset":"11201","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.462188,"name":"drain","context":{"idset":"11202","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.4622986,"name":"drain","context":{"idset":"11203","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.4624033,"name":"drain","context":{"idset":"11204","reason":"broker was unresponsive"}}
+{"timestamp":1712584580.6464179,"name":"drain","context":{"idset":"11205","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.462642,"name":"drain","context":{"idset":"11206","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.462795,"name":"drain","context":{"idset":"11207","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.4629083,"name":"drain","context":{"idset":"11208","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.463022,"name":"drain","context":{"idset":"11209","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.4631221,"name":"drain","context":{"idset":"11210","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.4632416,"name":"drain","context":{"idset":"11211","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.4633689,"name":"drain","context":{"idset":"11212","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.4634895,"name":"drain","context":{"idset":"11213","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.4635878,"name":"drain","context":{"idset":"11214","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.4636836,"name":"drain","context":{"idset":"11215","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.4637792,"name":"drain","context":{"idset":"11216","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.4638743,"name":"drain","context":{"idset":"11217","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.4639854,"name":"drain","context":{"idset":"11218","reason":"broker was unresponsive"}}
+{"timestamp":1712584586.6562564,"name":"drain","context":{"idset":"11219","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4631968,"name":"drain","context":{"idset":"11220","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4633706,"name":"drain","context":{"idset":"11221","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4634843,"name":"drain","context":{"idset":"11222","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4635954,"name":"drain","context":{"idset":"11223","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4637067,"name":"drain","context":{"idset":"11224","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4638193,"name":"drain","context":{"idset":"11225","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4639299,"name":"drain","context":{"idset":"11227","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4640379,"name":"drain","context":{"idset":"11228","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4641502,"name":"drain","context":{"idset":"11229","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4642861,"name":"drain","context":{"idset":"11230","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.4643979,"name":"drain","context":{"idset":"11231","reason":"broker was unresponsive"}}
+{"timestamp":1712584592.6617379,"name":"drain","context":{"idset":"11232","reason":"broker was unresponsive"}}
+{"timestamp":1712584598.4620857,"name":"drain","context":{"idset":"11234","reason":"broker was unresponsive"}}
+{"timestamp":1712584598.4622524,"name":"drain","context":{"idset":"11237","reason":"broker was unresponsive"}}
+{"timestamp":1712584598.462405,"name":"drain","context":{"idset":"11238","reason":"broker was unresponsive"}}
+{"timestamp":1712584598.4625425,"name":"drain","context":{"idset":"11239","reason":"broker was unresponsive"}}
+{"timestamp":1712584598.462666,"name":"drain","context":{"idset":"11240","reason":"broker was unresponsive"}}
+{"timestamp":1712584598.4627736,"name":"drain","context":{"idset":"11241","reason":"broker was unresponsive"}}
+{"timestamp":1712584598.4628928,"name":"drain","context":{"idset":"11242","reason":"broker was unresponsive"}}
+{"timestamp":1712584598.4630089,"name":"drain","context":{"idset":"11243","reason":"broker was unresponsive"}}
+{"timestamp":1712584598.6036515,"name":"drain","context":{"idset":"11244","reason":"broker was unresponsive"}}
+{"timestamp":1712584604.4626477,"name":"drain","context":{"idset":"11245","reason":"broker was unresponsive"}}
+{"timestamp":1712584604.4628263,"name":"drain","context":{"idset":"11246","reason":"broker was unresponsive"}}
+{"timestamp":1712584604.4629474,"name":"drain","context":{"idset":"11248","reason":"broker was unresponsive"}}
+{"timestamp":1712584604.4630589,"name":"drain","context":{"idset":"11250","reason":"broker was unresponsive"}}
+{"timestamp":1712584604.4631765,"name":"drain","context":{"idset":"11251","reason":"broker was unresponsive"}}
+{"timestamp":1712584604.5780971,"name":"drain","context":{"idset":"11252","reason":"broker was unresponsive"}}
+{"timestamp":1712584608.4695711,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1712584608.484525,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1712584608.5701072,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1712584610.4699023,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1712584610.4845309,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1712584610.4952841,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1712584610.5666912,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1712584612.4669418,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1712584612.4780092,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1712584612.5583436,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1712584614.4718711,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1712584614.4826481,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1712584614.4949346,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1712584614.5730982,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1712584616.4687197,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1712584616.4799671,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1712584616.567373,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1712584618.490114,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1712584618.5039756,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1712584618.5216122,"name":"offline","context":{"idset":"11146"}}
+{"timestamp":1712584618.5436599,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1712584618.5574241,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1712584618.5749595,"name":"offline","context":{"idset":"11150"}}
+{"timestamp":1712584620.4734583,"name":"offline","context":{"idset":"11149"}}
+{"timestamp":1712584620.4921818,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1712584620.5698647,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1712584622.4787421,"name":"offline","context":{"idset":"11153"}}
+{"timestamp":1712584622.4962189,"name":"offline","context":{"idset":"11154"}}
+{"timestamp":1712584622.5135212,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1712584622.5297611,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1712584622.5597653,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1712584624.4783092,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1712584624.4947886,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1712584624.5130827,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1712584624.5620942,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1712584626.4755816,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1712584626.4963048,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1712584626.5116324,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1712584626.5597599,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1712584628.4672213,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1712584628.4769788,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1712584628.5785642,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1712584630.4740622,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1712584630.488157,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1712584630.4994755,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1712584630.560297,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1712584632.4737053,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1712584632.4886665,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1712584632.5022194,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1712584632.5789309,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1712584634.4743419,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1712584634.4860661,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1712584634.4978898,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1712584634.506005,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1712584634.5751734,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1712584636.469347,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1712584636.4796901,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1712584636.4893882,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1712584636.565608,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1712584638.4691434,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1712584638.5592802,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1712584640.4775987,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1712584640.4969132,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1712584640.5155702,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1712584640.5598674,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1712584642.4679949,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1712584642.4779766,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1712584642.4871421,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1712584642.5638363,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1712584644.4815369,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1712584644.5014005,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1712584644.5185211,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1712584644.5310748,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1712584644.5617716,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1712584646.47084,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1712584646.4817817,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1712584646.4920106,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1712584646.5662398,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1712584648.4959898,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1712584648.5271552,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1712584648.5558064,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1712584648.5863481,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1712584648.6091936,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1712584650.4798563,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1712584650.4982409,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1712584650.5164938,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1712584650.5639303,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1712584652.4768841,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1712584652.5014286,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1712584652.5131307,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1712584652.5284357,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1712584652.5605998,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1712584654.4755356,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1712584654.4917562,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1712584654.5107768,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1712584654.5636444,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1712584656.4769993,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1712584656.4900839,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1712584656.5013056,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1712584656.5126061,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1712584656.5724452,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1712584658.4814284,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1712584658.5056744,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1712584658.5209274,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1712584658.5587854,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1712584660.4689765,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1712584660.5598669,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1712584662.4732411,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1712584662.4896994,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1712584662.5059898,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1712584662.5795126,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1712584664.4645851,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1712584664.5613844,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1712584666.4649968,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1712584666.4744642,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1712584666.5582995,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1712584668.4659884,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1712584668.4756472,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1712584668.5593846,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1712584712.5592253,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1712586051.3557179,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1712586051.4577417,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1712586176.561116,"name":"drain","context":{"idset":"10262","reason":"broker was unresponsive"}}
+{"timestamp":1712586242.4688134,"name":"offline","context":{"idset":"10261"}}
+{"timestamp":1712586242.5607903,"name":"offline","context":{"idset":"10262"}}
+{"timestamp":1712586553.6810555,"name":"drain","context":{"idset":"11021","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712586728.5546196,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.3908463,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.3910499,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.3959017,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.3959472,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.3990037,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4016831,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.401727,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4042249,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4068358,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4068794,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4133084,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4133496,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4133859,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4154608,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4155042,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4187224,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4223042,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4223461,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4263096,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4263494,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4303148,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4303589,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4303973,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4327524,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4327991,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4363122,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4373734,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.437417,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4413123,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4453137,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4453657,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.445406,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4454465,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4487605,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.451467,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4515185,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4530489,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4530957,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4573214,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4583163,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.458365,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4646709,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4683135,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4683626,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4703577,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4736776,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4774101,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4813223,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4813752,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4853158,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4853716,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4854178,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4893193,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4893746,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4924185,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4963186,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.4963698,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5013316,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5013847,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5014296,"name":"drain","context":{"idset":"85","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5053136,"name":"drain","context":{"idset":"86","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5053678,"name":"drain","context":{"idset":"87","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5054154,"name":"drain","context":{"idset":"88","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5054622,"name":"drain","context":{"idset":"89","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5055094,"name":"drain","context":{"idset":"90","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5055525,"name":"drain","context":{"idset":"91","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5055978,"name":"drain","context":{"idset":"92","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5056434,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5056884,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5057352,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5057828,"name":"drain","context":{"idset":"96","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5058298,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.505878,"name":"drain","context":{"idset":"98","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5059266,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5059729,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.506022,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5060761,"name":"drain","context":{"idset":"102","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5061281,"name":"drain","context":{"idset":"103","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5061767,"name":"drain","context":{"idset":"104","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5062263,"name":"drain","context":{"idset":"105","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5062926,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5063429,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5063939,"name":"drain","context":{"idset":"108","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5064447,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5064938,"name":"drain","context":{"idset":"110","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5065453,"name":"drain","context":{"idset":"111","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5065958,"name":"drain","context":{"idset":"112","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5066466,"name":"drain","context":{"idset":"113","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5066993,"name":"drain","context":{"idset":"114","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.506752,"name":"drain","context":{"idset":"115","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5068045,"name":"drain","context":{"idset":"116","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5068581,"name":"drain","context":{"idset":"117","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5069141,"name":"drain","context":{"idset":"118","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5069666,"name":"drain","context":{"idset":"119","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5070202,"name":"drain","context":{"idset":"120","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5070741,"name":"drain","context":{"idset":"122","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5071273,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5071826,"name":"drain","context":{"idset":"124","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5072351,"name":"drain","context":{"idset":"125","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5073063,"name":"drain","context":{"idset":"126","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5073638,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5074203,"name":"drain","context":{"idset":"128","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5074763,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5075326,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5075917,"name":"drain","context":{"idset":"131","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5076475,"name":"drain","context":{"idset":"132","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.507704,"name":"drain","context":{"idset":"133","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5077631,"name":"drain","context":{"idset":"134","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5078204,"name":"drain","context":{"idset":"135","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5078797,"name":"drain","context":{"idset":"136","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5079381,"name":"drain","context":{"idset":"137","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5079982,"name":"drain","context":{"idset":"138","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5080588,"name":"drain","context":{"idset":"139","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5081184,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5081794,"name":"drain","context":{"idset":"141","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5082409,"name":"drain","context":{"idset":"142","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5083148,"name":"drain","context":{"idset":"143","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5083759,"name":"drain","context":{"idset":"144","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5084383,"name":"drain","context":{"idset":"145","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5084994,"name":"drain","context":{"idset":"146","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5085628,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5086257,"name":"drain","context":{"idset":"148","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5086849,"name":"drain","context":{"idset":"149","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.508749,"name":"drain","context":{"idset":"150","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5088134,"name":"drain","context":{"idset":"151","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5088768,"name":"drain","context":{"idset":"152","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5089393,"name":"drain","context":{"idset":"153","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5090039,"name":"drain","context":{"idset":"154","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5090697,"name":"drain","context":{"idset":"155","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5091341,"name":"drain","context":{"idset":"156","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5091994,"name":"drain","context":{"idset":"157","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5092845,"name":"drain","context":{"idset":"158","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5093484,"name":"drain","context":{"idset":"159","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5094142,"name":"drain","context":{"idset":"160","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.509479,"name":"drain","context":{"idset":"161","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5095408,"name":"drain","context":{"idset":"162","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5096064,"name":"drain","context":{"idset":"163","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5096726,"name":"drain","context":{"idset":"164","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5097394,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5098078,"name":"drain","context":{"idset":"166","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5098755,"name":"drain","context":{"idset":"167","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5099418,"name":"drain","context":{"idset":"168","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5100102,"name":"drain","context":{"idset":"169","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5100787,"name":"drain","context":{"idset":"170","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5101483,"name":"drain","context":{"idset":"171","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5102162,"name":"drain","context":{"idset":"172","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5103004,"name":"drain","context":{"idset":"173","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.510371,"name":"drain","context":{"idset":"174","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5104425,"name":"drain","context":{"idset":"175","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5105121,"name":"drain","context":{"idset":"176","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5105822,"name":"drain","context":{"idset":"177","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5106535,"name":"drain","context":{"idset":"178","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5107241,"name":"drain","context":{"idset":"179","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5107944,"name":"drain","context":{"idset":"180","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5108681,"name":"drain","context":{"idset":"181","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5109386,"name":"drain","context":{"idset":"182","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5110135,"name":"drain","context":{"idset":"183","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5110869,"name":"drain","context":{"idset":"184","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5111609,"name":"drain","context":{"idset":"185","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5112352,"name":"drain","context":{"idset":"186","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5113196,"name":"drain","context":{"idset":"187","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5113938,"name":"drain","context":{"idset":"188","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.511467,"name":"drain","context":{"idset":"189","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.511539,"name":"drain","context":{"idset":"190","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5116122,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.511687,"name":"drain","context":{"idset":"192","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5117686,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5118437,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5119197,"name":"drain","context":{"idset":"195","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5119979,"name":"drain","context":{"idset":"196","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5120733,"name":"drain","context":{"idset":"197","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5121496,"name":"drain","context":{"idset":"198","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5122259,"name":"drain","context":{"idset":"199","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5123289,"name":"drain","context":{"idset":"200","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5124094,"name":"drain","context":{"idset":"201","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5124886,"name":"drain","context":{"idset":"202","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5125668,"name":"drain","context":{"idset":"203","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.512646,"name":"drain","context":{"idset":"204","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5127239,"name":"drain","context":{"idset":"205","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5128002,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5128767,"name":"drain","context":{"idset":"207","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5129588,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5130367,"name":"drain","context":{"idset":"209","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5131154,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5131974,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5132921,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5133767,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5134578,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5135386,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5136211,"name":"drain","context":{"idset":"216","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5137024,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5137846,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5138683,"name":"drain","context":{"idset":"220","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5139511,"name":"drain","context":{"idset":"221","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5140369,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.514118,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5142031,"name":"drain","context":{"idset":"224","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.514302,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5143888,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5144746,"name":"drain","context":{"idset":"227","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5145588,"name":"drain","context":{"idset":"228","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5146439,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5147307,"name":"drain","context":{"idset":"230","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5148177,"name":"drain","context":{"idset":"231","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5149035,"name":"drain","context":{"idset":"232","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5149922,"name":"drain","context":{"idset":"233","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5150809,"name":"drain","context":{"idset":"234","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5151687,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5152552,"name":"drain","context":{"idset":"236","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5153589,"name":"drain","context":{"idset":"237","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5154476,"name":"drain","context":{"idset":"238","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5155358,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.515626,"name":"drain","context":{"idset":"240","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5157146,"name":"drain","context":{"idset":"241","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5158031,"name":"drain","context":{"idset":"242","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5158918,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5159819,"name":"drain","context":{"idset":"244","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.516073,"name":"drain","context":{"idset":"245","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5161636,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5162535,"name":"drain","context":{"idset":"247","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5163596,"name":"drain","context":{"idset":"248","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5164521,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5165441,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5166371,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5167298,"name":"drain","context":{"idset":"252","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5168247,"name":"drain","context":{"idset":"253","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5169184,"name":"drain","context":{"idset":"254","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5170121,"name":"drain","context":{"idset":"255","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5171065,"name":"drain","context":{"idset":"256","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.517199,"name":"drain","context":{"idset":"257","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5173061,"name":"drain","context":{"idset":"258","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5174,"name":"drain","context":{"idset":"259","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5174952,"name":"drain","context":{"idset":"260","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5175891,"name":"drain","context":{"idset":"261","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5176847,"name":"drain","context":{"idset":"262","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5177791,"name":"drain","context":{"idset":"263","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.517875,"name":"drain","context":{"idset":"264","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5179715,"name":"drain","context":{"idset":"265","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5180719,"name":"drain","context":{"idset":"266","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5181675,"name":"drain","context":{"idset":"267","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5182865,"name":"drain","context":{"idset":"268","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5183833,"name":"drain","context":{"idset":"269","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5184782,"name":"drain","context":{"idset":"270","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5185769,"name":"drain","context":{"idset":"271","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5186722,"name":"drain","context":{"idset":"272","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5187757,"name":"drain","context":{"idset":"273","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5188761,"name":"drain","context":{"idset":"274","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5189764,"name":"drain","context":{"idset":"275","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5190744,"name":"drain","context":{"idset":"276","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5191741,"name":"drain","context":{"idset":"277","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5192909,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5193951,"name":"drain","context":{"idset":"279","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.519495,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5195909,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5196939,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5197952,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5198967,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5199997,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5201018,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5202029,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5203171,"name":"drain","context":{"idset":"288","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5204232,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5205245,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5206268,"name":"drain","context":{"idset":"291","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5207303,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5208356,"name":"drain","context":{"idset":"293","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5209389,"name":"drain","context":{"idset":"294","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5210447,"name":"drain","context":{"idset":"295","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5211527,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5212603,"name":"drain","context":{"idset":"297","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5213819,"name":"drain","context":{"idset":"298","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5214908,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5215957,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5217032,"name":"drain","context":{"idset":"301","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5218101,"name":"drain","context":{"idset":"302","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5219195,"name":"drain","context":{"idset":"303","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5220265,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5221355,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5222447,"name":"drain","context":{"idset":"306","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.522372,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.522481,"name":"drain","context":{"idset":"308","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5225878,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5227017,"name":"drain","context":{"idset":"310","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5228095,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5229194,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5230312,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5231414,"name":"drain","context":{"idset":"314","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5232532,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5233741,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5234854,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5235956,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5237091,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5238185,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5239308,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5240397,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.524152,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5242875,"name":"drain","context":{"idset":"324","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5244019,"name":"drain","context":{"idset":"325","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5245168,"name":"drain","context":{"idset":"326","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5246301,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.524744,"name":"drain","context":{"idset":"328","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.524857,"name":"drain","context":{"idset":"329","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5249679,"name":"drain","context":{"idset":"330","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5250797,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5251932,"name":"drain","context":{"idset":"332","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5253205,"name":"drain","context":{"idset":"333","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5254352,"name":"drain","context":{"idset":"334","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5255504,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5256636,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5257795,"name":"drain","context":{"idset":"337","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5258949,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5260103,"name":"drain","context":{"idset":"339","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.526125,"name":"drain","context":{"idset":"340","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5262396,"name":"drain","context":{"idset":"341","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5263708,"name":"drain","context":{"idset":"342","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5264912,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5266128,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5267329,"name":"drain","context":{"idset":"345","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5268521,"name":"drain","context":{"idset":"346","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5269699,"name":"drain","context":{"idset":"347","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.527092,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5272138,"name":"drain","context":{"idset":"350","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5273447,"name":"drain","context":{"idset":"351","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5274661,"name":"drain","context":{"idset":"352","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5275877,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5277092,"name":"drain","context":{"idset":"354","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5278308,"name":"drain","context":{"idset":"355","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.527952,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5280771,"name":"drain","context":{"idset":"357","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5281994,"name":"drain","context":{"idset":"358","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5283377,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5284607,"name":"drain","context":{"idset":"360","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5285838,"name":"drain","context":{"idset":"361","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5287094,"name":"drain","context":{"idset":"362","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5288327,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5289567,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5290828,"name":"drain","context":{"idset":"365","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5292101,"name":"drain","context":{"idset":"366","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5293479,"name":"drain","context":{"idset":"367","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5294757,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5296047,"name":"drain","context":{"idset":"369","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5297325,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5298584,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5299861,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5301118,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5302393,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5303836,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5305145,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5306451,"name":"drain","context":{"idset":"377","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5307746,"name":"drain","context":{"idset":"378","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5309026,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5310323,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5311615,"name":"drain","context":{"idset":"381","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5313044,"name":"drain","context":{"idset":"382","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5314333,"name":"drain","context":{"idset":"383","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5315642,"name":"drain","context":{"idset":"384","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5316949,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5318258,"name":"drain","context":{"idset":"386","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5319562,"name":"drain","context":{"idset":"387","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5320873,"name":"drain","context":{"idset":"388","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5322194,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5323651,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.532496,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.532629,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5327637,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5328956,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5330288,"name":"drain","context":{"idset":"395","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5331631,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5333109,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5334463,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5335803,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5337207,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5338566,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.533994,"name":"drain","context":{"idset":"402","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.534128,"name":"drain","context":{"idset":"403","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5342767,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5344155,"name":"drain","context":{"idset":"405","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5345521,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5346885,"name":"drain","context":{"idset":"407","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.534826,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5349641,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5351019,"name":"drain","context":{"idset":"410","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5352371,"name":"drain","context":{"idset":"411","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5353906,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5355291,"name":"drain","context":{"idset":"413","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5356712,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5358083,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5359442,"name":"drain","context":{"idset":"416","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5360787,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5362172,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5363731,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5365174,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5366561,"name":"drain","context":{"idset":"421","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5367947,"name":"drain","context":{"idset":"422","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.536936,"name":"drain","context":{"idset":"423","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5370774,"name":"drain","context":{"idset":"424","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5372167,"name":"drain","context":{"idset":"425","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.53737,"name":"drain","context":{"idset":"426","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5375171,"name":"drain","context":{"idset":"427","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5376511,"name":"drain","context":{"idset":"428","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5377963,"name":"drain","context":{"idset":"429","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5379412,"name":"drain","context":{"idset":"430","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5382125,"name":"drain","context":{"idset":"432","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5383711,"name":"drain","context":{"idset":"433","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5385222,"name":"drain","context":{"idset":"434","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5386655,"name":"drain","context":{"idset":"435","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5388098,"name":"drain","context":{"idset":"436","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5389552,"name":"drain","context":{"idset":"437","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5391018,"name":"drain","context":{"idset":"438","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5392461,"name":"drain","context":{"idset":"439","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5394084,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5395544,"name":"drain","context":{"idset":"441","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5397017,"name":"drain","context":{"idset":"442","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5398493,"name":"drain","context":{"idset":"443","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5399978,"name":"drain","context":{"idset":"444","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5401454,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5403123,"name":"drain","context":{"idset":"470","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5404615,"name":"drain","context":{"idset":"471","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5406113,"name":"drain","context":{"idset":"472","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5407596,"name":"drain","context":{"idset":"473","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5409091,"name":"drain","context":{"idset":"474","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5410604,"name":"drain","context":{"idset":"475","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5412102,"name":"drain","context":{"idset":"476","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5413725,"name":"drain","context":{"idset":"477","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5415237,"name":"drain","context":{"idset":"478","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5416763,"name":"drain","context":{"idset":"479","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5418274,"name":"drain","context":{"idset":"480","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.54198,"name":"drain","context":{"idset":"481","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5421307,"name":"drain","context":{"idset":"482","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5424519,"name":"drain","context":{"idset":"9210","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5427632,"name":"drain","context":{"idset":"9211","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5430794,"name":"drain","context":{"idset":"9212","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5434036,"name":"drain","context":{"idset":"9213","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5437191,"name":"drain","context":{"idset":"9215","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5440345,"name":"drain","context":{"idset":"9216","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5443664,"name":"drain","context":{"idset":"9217","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5446844,"name":"drain","context":{"idset":"9218","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.545001,"name":"drain","context":{"idset":"9219","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5465975,"name":"drain","context":{"idset":"9220","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5469306,"name":"drain","context":{"idset":"9221","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.547256,"name":"drain","context":{"idset":"9222","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.547591,"name":"drain","context":{"idset":"9223","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5479138,"name":"drain","context":{"idset":"9224","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5482337,"name":"drain","context":{"idset":"9225","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5485654,"name":"drain","context":{"idset":"9226","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5488858,"name":"drain","context":{"idset":"9227","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5492105,"name":"drain","context":{"idset":"9228","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5495391,"name":"drain","context":{"idset":"9229","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5498631,"name":"drain","context":{"idset":"9230","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5501878,"name":"drain","context":{"idset":"9231","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5505185,"name":"drain","context":{"idset":"9232","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5508385,"name":"drain","context":{"idset":"9233","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5511596,"name":"drain","context":{"idset":"9234","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5514994,"name":"drain","context":{"idset":"9235","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5518229,"name":"drain","context":{"idset":"9236","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5521483,"name":"drain","context":{"idset":"9237","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5524857,"name":"drain","context":{"idset":"9238","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.552809,"name":"drain","context":{"idset":"9241","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5531337,"name":"drain","context":{"idset":"9242","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5534744,"name":"drain","context":{"idset":"9243","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5538023,"name":"drain","context":{"idset":"9244","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5541322,"name":"drain","context":{"idset":"9245","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5544722,"name":"drain","context":{"idset":"9246","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5548015,"name":"drain","context":{"idset":"9247","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5551298,"name":"drain","context":{"idset":"9248","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5554714,"name":"drain","context":{"idset":"9249","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5557983,"name":"drain","context":{"idset":"9250","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5561292,"name":"drain","context":{"idset":"9251","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5564671,"name":"drain","context":{"idset":"9252","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5567935,"name":"drain","context":{"idset":"9253","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5571239,"name":"drain","context":{"idset":"9254","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5574636,"name":"drain","context":{"idset":"9255","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5577931,"name":"drain","context":{"idset":"9256","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.558121,"name":"drain","context":{"idset":"9257","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5584497,"name":"drain","context":{"idset":"9258","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5587893,"name":"drain","context":{"idset":"9259","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5591125,"name":"drain","context":{"idset":"9260","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5594542,"name":"drain","context":{"idset":"9261","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5598104,"name":"drain","context":{"idset":"9262","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5601666,"name":"drain","context":{"idset":"9263","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5605292,"name":"drain","context":{"idset":"9264","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5608807,"name":"drain","context":{"idset":"9265","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5612245,"name":"drain","context":{"idset":"9266","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5615838,"name":"drain","context":{"idset":"9267","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5619314,"name":"drain","context":{"idset":"9268","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5622573,"name":"drain","context":{"idset":"9269","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5625794,"name":"drain","context":{"idset":"9270","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5629272,"name":"drain","context":{"idset":"9271","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5632946,"name":"drain","context":{"idset":"9272","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5636365,"name":"drain","context":{"idset":"9273","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5639822,"name":"drain","context":{"idset":"9274","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5643473,"name":"drain","context":{"idset":"9275","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5646944,"name":"drain","context":{"idset":"9276","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5650315,"name":"drain","context":{"idset":"9277","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5653961,"name":"drain","context":{"idset":"9278","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5657499,"name":"drain","context":{"idset":"9279","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5661042,"name":"drain","context":{"idset":"9280","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5664792,"name":"drain","context":{"idset":"9281","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5668271,"name":"drain","context":{"idset":"9282","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5671787,"name":"drain","context":{"idset":"9283","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.567548,"name":"drain","context":{"idset":"9284","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5678902,"name":"drain","context":{"idset":"9285","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5682285,"name":"drain","context":{"idset":"9286","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5685878,"name":"drain","context":{"idset":"9287","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.568927,"name":"drain","context":{"idset":"9288","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5692818,"name":"drain","context":{"idset":"9289","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5696285,"name":"drain","context":{"idset":"9290","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5699704,"name":"drain","context":{"idset":"9291","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5703239,"name":"drain","context":{"idset":"9292","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5706391,"name":"drain","context":{"idset":"9293","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5709562,"name":"drain","context":{"idset":"9294","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5712969,"name":"drain","context":{"idset":"9295","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5716288,"name":"drain","context":{"idset":"9296","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5719528,"name":"drain","context":{"idset":"9297","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.572299,"name":"drain","context":{"idset":"9298","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5726321,"name":"drain","context":{"idset":"9299","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5729644,"name":"drain","context":{"idset":"9300","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5732858,"name":"drain","context":{"idset":"9301","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5736091,"name":"drain","context":{"idset":"9302","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5739458,"name":"drain","context":{"idset":"9303","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.574307,"name":"drain","context":{"idset":"9304","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5746469,"name":"drain","context":{"idset":"9305","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5749912,"name":"drain","context":{"idset":"9306","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5753472,"name":"drain","context":{"idset":"9307","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5756953,"name":"drain","context":{"idset":"9308","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5760458,"name":"drain","context":{"idset":"9309","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.576411,"name":"drain","context":{"idset":"9310","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5767593,"name":"drain","context":{"idset":"9311","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.577107,"name":"drain","context":{"idset":"9312","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.577467,"name":"drain","context":{"idset":"9313","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5778158,"name":"drain","context":{"idset":"9314","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.578172,"name":"drain","context":{"idset":"9315","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5785418,"name":"drain","context":{"idset":"9317","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5788937,"name":"drain","context":{"idset":"9319","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5792491,"name":"drain","context":{"idset":"9320","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5796199,"name":"drain","context":{"idset":"9321","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5799797,"name":"drain","context":{"idset":"9322","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5803456,"name":"drain","context":{"idset":"9323","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5806992,"name":"drain","context":{"idset":"9324","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.581058,"name":"drain","context":{"idset":"9325","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5814264,"name":"drain","context":{"idset":"9326","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5817819,"name":"drain","context":{"idset":"9327","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5821393,"name":"drain","context":{"idset":"9328","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5829122,"name":"drain","context":{"idset":"9329","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5832899,"name":"drain","context":{"idset":"9330","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5836513,"name":"drain","context":{"idset":"9331","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.584017,"name":"drain","context":{"idset":"9332","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5844026,"name":"drain","context":{"idset":"9333","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5847645,"name":"drain","context":{"idset":"9334","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5851386,"name":"drain","context":{"idset":"9335","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5855157,"name":"drain","context":{"idset":"9336","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5858822,"name":"drain","context":{"idset":"9337","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.586251,"name":"drain","context":{"idset":"9803","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5866308,"name":"drain","context":{"idset":"9804","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5869935,"name":"drain","context":{"idset":"9805","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5873713,"name":"drain","context":{"idset":"9806","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5877419,"name":"drain","context":{"idset":"9807","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5881081,"name":"drain","context":{"idset":"9808","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5884867,"name":"drain","context":{"idset":"9809","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5888498,"name":"drain","context":{"idset":"9810","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5892155,"name":"drain","context":{"idset":"9811","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5895889,"name":"drain","context":{"idset":"9812","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5899582,"name":"drain","context":{"idset":"9813","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5903397,"name":"drain","context":{"idset":"9814","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.590709,"name":"drain","context":{"idset":"9815","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5910754,"name":"drain","context":{"idset":"9816","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5914567,"name":"drain","context":{"idset":"9817","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5918288,"name":"drain","context":{"idset":"9818","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5921998,"name":"drain","context":{"idset":"9819","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5925908,"name":"drain","context":{"idset":"9820","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5929542,"name":"drain","context":{"idset":"9821","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5933347,"name":"drain","context":{"idset":"9822","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5937121,"name":"drain","context":{"idset":"9823","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5940876,"name":"drain","context":{"idset":"9824","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5944743,"name":"drain","context":{"idset":"9825","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.594831,"name":"drain","context":{"idset":"9826","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5952113,"name":"drain","context":{"idset":"9827","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5956304,"name":"drain","context":{"idset":"9828","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5960085,"name":"drain","context":{"idset":"9829","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5964019,"name":"drain","context":{"idset":"9830","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5967765,"name":"drain","context":{"idset":"9831","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5971522,"name":"drain","context":{"idset":"9832","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5975409,"name":"drain","context":{"idset":"9833","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5979555,"name":"drain","context":{"idset":"9834","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5983856,"name":"drain","context":{"idset":"9835","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5988026,"name":"drain","context":{"idset":"9836","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.5992157,"name":"drain","context":{"idset":"9837","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.599649,"name":"drain","context":{"idset":"9838","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6000617,"name":"drain","context":{"idset":"9839","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6004915,"name":"drain","context":{"idset":"9840","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6009102,"name":"drain","context":{"idset":"9841","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6013367,"name":"drain","context":{"idset":"9842","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.601728,"name":"drain","context":{"idset":"9843","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.602113,"name":"drain","context":{"idset":"9844","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6025484,"name":"drain","context":{"idset":"9973","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6029687,"name":"drain","context":{"idset":"9974","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6033981,"name":"drain","context":{"idset":"9975","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6038151,"name":"drain","context":{"idset":"9976","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6042283,"name":"drain","context":{"idset":"9977","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.604666,"name":"drain","context":{"idset":"9978","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6050828,"name":"drain","context":{"idset":"9979","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6054921,"name":"drain","context":{"idset":"9980","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6058848,"name":"drain","context":{"idset":"9981","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.606286,"name":"drain","context":{"idset":"9982","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6066706,"name":"drain","context":{"idset":"9983","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6070578,"name":"drain","context":{"idset":"9984","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6074595,"name":"drain","context":{"idset":"9985","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6078477,"name":"drain","context":{"idset":"9986","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6082325,"name":"drain","context":{"idset":"9987","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6086323,"name":"drain","context":{"idset":"9988","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6090205,"name":"drain","context":{"idset":"9989","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6094282,"name":"drain","context":{"idset":"9990","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6098175,"name":"drain","context":{"idset":"9991","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6102078,"name":"drain","context":{"idset":"9992","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.610615,"name":"drain","context":{"idset":"9993","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6110103,"name":"drain","context":{"idset":"9994","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6114202,"name":"drain","context":{"idset":"9995","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6118145,"name":"drain","context":{"idset":"9996","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6122053,"name":"drain","context":{"idset":"9997","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6126151,"name":"drain","context":{"idset":"9998","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6130147,"name":"drain","context":{"idset":"9999","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6139631,"name":"drain","context":{"idset":"10000","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.614393,"name":"drain","context":{"idset":"10001","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6147952,"name":"drain","context":{"idset":"10002","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6151931,"name":"drain","context":{"idset":"10003","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6156144,"name":"drain","context":{"idset":"10004","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6160192,"name":"drain","context":{"idset":"10005","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6164317,"name":"drain","context":{"idset":"10006","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6168301,"name":"drain","context":{"idset":"10007","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6172266,"name":"drain","context":{"idset":"10008","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6176438,"name":"drain","context":{"idset":"10009","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6180427,"name":"drain","context":{"idset":"10010","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6184533,"name":"drain","context":{"idset":"10011","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6188517,"name":"drain","context":{"idset":"10012","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6192517,"name":"drain","context":{"idset":"10013","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6196666,"name":"drain","context":{"idset":"10014","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6200719,"name":"drain","context":{"idset":"10015","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6204822,"name":"drain","context":{"idset":"10016","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6208825,"name":"drain","context":{"idset":"10017","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6212976,"name":"drain","context":{"idset":"10018","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6216974,"name":"drain","context":{"idset":"10019","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6220984,"name":"drain","context":{"idset":"10020","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6225176,"name":"drain","context":{"idset":"10021","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6229217,"name":"drain","context":{"idset":"10022","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6233366,"name":"drain","context":{"idset":"10023","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6237435,"name":"drain","context":{"idset":"10024","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6241484,"name":"drain","context":{"idset":"10025","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6245668,"name":"drain","context":{"idset":"10026","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.624974,"name":"drain","context":{"idset":"10027","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6253951,"name":"drain","context":{"idset":"10028","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6258025,"name":"drain","context":{"idset":"10029","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6262112,"name":"drain","context":{"idset":"10030","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6266327,"name":"drain","context":{"idset":"10031","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6270599,"name":"drain","context":{"idset":"10032","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6274965,"name":"drain","context":{"idset":"10033","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6279078,"name":"drain","context":{"idset":"10034","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6289117,"name":"drain","context":{"idset":"10035","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6293607,"name":"drain","context":{"idset":"10036","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6297841,"name":"drain","context":{"idset":"10037","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6301978,"name":"drain","context":{"idset":"10038","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6306324,"name":"drain","context":{"idset":"10039","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6310537,"name":"drain","context":{"idset":"10040","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6314793,"name":"drain","context":{"idset":"10041","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6318932,"name":"drain","context":{"idset":"10042","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6323202,"name":"drain","context":{"idset":"10043","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6327317,"name":"drain","context":{"idset":"10044","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6331453,"name":"drain","context":{"idset":"10045","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6335764,"name":"drain","context":{"idset":"10046","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6339931,"name":"drain","context":{"idset":"10047","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6344178,"name":"drain","context":{"idset":"10048","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6348336,"name":"drain","context":{"idset":"10049","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6352496,"name":"drain","context":{"idset":"10050","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6356893,"name":"drain","context":{"idset":"10051","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6361098,"name":"drain","context":{"idset":"10052","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6365414,"name":"drain","context":{"idset":"10053","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6369588,"name":"drain","context":{"idset":"10054","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6373858,"name":"drain","context":{"idset":"10055","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6378036,"name":"drain","context":{"idset":"10056","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6382213,"name":"drain","context":{"idset":"10057","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6386518,"name":"drain","context":{"idset":"10058","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6390741,"name":"drain","context":{"idset":"10059","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6394982,"name":"drain","context":{"idset":"10060","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.639919,"name":"drain","context":{"idset":"10061","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6403909,"name":"drain","context":{"idset":"10062","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6408136,"name":"drain","context":{"idset":"10063","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6412382,"name":"drain","context":{"idset":"10064","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6416736,"name":"drain","context":{"idset":"10065","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6420975,"name":"drain","context":{"idset":"10066","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6425276,"name":"drain","context":{"idset":"10067","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6429517,"name":"drain","context":{"idset":"10068","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6433873,"name":"drain","context":{"idset":"10069","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6438136,"name":"drain","context":{"idset":"10070","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6442454,"name":"drain","context":{"idset":"10071","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6446838,"name":"drain","context":{"idset":"10072","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6451077,"name":"drain","context":{"idset":"10073","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.645555,"name":"drain","context":{"idset":"10074","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6459813,"name":"drain","context":{"idset":"10075","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6464276,"name":"drain","context":{"idset":"10076","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6468556,"name":"drain","context":{"idset":"10077","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6472993,"name":"drain","context":{"idset":"10078","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6477265,"name":"drain","context":{"idset":"10079","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6481502,"name":"drain","context":{"idset":"10080","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6485918,"name":"drain","context":{"idset":"10081","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6490209,"name":"drain","context":{"idset":"10082","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6494679,"name":"drain","context":{"idset":"10083","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6498976,"name":"drain","context":{"idset":"10084","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6503384,"name":"drain","context":{"idset":"10085","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6507692,"name":"drain","context":{"idset":"10086","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6511993,"name":"drain","context":{"idset":"10087","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6516581,"name":"drain","context":{"idset":"10088","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6520905,"name":"drain","context":{"idset":"10089","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6525209,"name":"drain","context":{"idset":"10090","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6529109,"name":"drain","context":{"idset":"10091","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6533151,"name":"drain","context":{"idset":"10092","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6537123,"name":"drain","context":{"idset":"10093","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6541409,"name":"drain","context":{"idset":"10094","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6545496,"name":"drain","context":{"idset":"10095","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6549408,"name":"drain","context":{"idset":"10096","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6553612,"name":"drain","context":{"idset":"10097","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6557994,"name":"drain","context":{"idset":"10098","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6562357,"name":"drain","context":{"idset":"10099","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.656693,"name":"drain","context":{"idset":"10100","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6571305,"name":"drain","context":{"idset":"10101","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6575823,"name":"drain","context":{"idset":"10102","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6579895,"name":"drain","context":{"idset":"10103","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6584375,"name":"drain","context":{"idset":"10104","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6588819,"name":"drain","context":{"idset":"10105","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6593418,"name":"drain","context":{"idset":"10106","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6597817,"name":"drain","context":{"idset":"10107","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6602185,"name":"drain","context":{"idset":"10108","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6606672,"name":"drain","context":{"idset":"10109","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6611071,"name":"drain","context":{"idset":"10110","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6615558,"name":"drain","context":{"idset":"10111","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6619995,"name":"drain","context":{"idset":"10112","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6624572,"name":"drain","context":{"idset":"10113","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6628983,"name":"drain","context":{"idset":"10114","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6633511,"name":"drain","context":{"idset":"10115","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6637995,"name":"drain","context":{"idset":"10116","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6642411,"name":"drain","context":{"idset":"10117","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6646979,"name":"drain","context":{"idset":"10118","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6651464,"name":"drain","context":{"idset":"10119","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6656065,"name":"drain","context":{"idset":"10120","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6660552,"name":"drain","context":{"idset":"10121","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6665165,"name":"drain","context":{"idset":"10122","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6669641,"name":"drain","context":{"idset":"10123","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6674275,"name":"drain","context":{"idset":"10124","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6678796,"name":"drain","context":{"idset":"10125","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6683455,"name":"drain","context":{"idset":"10126","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6687956,"name":"drain","context":{"idset":"10127","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6692455,"name":"drain","context":{"idset":"10128","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6697102,"name":"drain","context":{"idset":"10129","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6701601,"name":"drain","context":{"idset":"10130","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6706297,"name":"drain","context":{"idset":"10131","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6710827,"name":"drain","context":{"idset":"10132","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6715448,"name":"drain","context":{"idset":"10133","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6719956,"name":"drain","context":{"idset":"10134","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6724648,"name":"drain","context":{"idset":"10135","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6729136,"name":"drain","context":{"idset":"10136","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6733775,"name":"drain","context":{"idset":"10137","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6738343,"name":"drain","context":{"idset":"10139","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6743021,"name":"drain","context":{"idset":"10140","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6747553,"name":"drain","context":{"idset":"10141","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6752114,"name":"drain","context":{"idset":"10142","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6756837,"name":"drain","context":{"idset":"10144","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6761391,"name":"drain","context":{"idset":"10146","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6766078,"name":"drain","context":{"idset":"10147","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6770647,"name":"drain","context":{"idset":"10148","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6775432,"name":"drain","context":{"idset":"10149","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6780005,"name":"drain","context":{"idset":"10150","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6784687,"name":"drain","context":{"idset":"10151","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6789274,"name":"drain","context":{"idset":"10152","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.679399,"name":"drain","context":{"idset":"10153","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6798556,"name":"drain","context":{"idset":"10154","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6803298,"name":"drain","context":{"idset":"10157","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6807847,"name":"drain","context":{"idset":"10158","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.681241,"name":"drain","context":{"idset":"10159","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6817167,"name":"drain","context":{"idset":"10160","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6821754,"name":"drain","context":{"idset":"10161","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6826308,"name":"drain","context":{"idset":"10162","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6830928,"name":"drain","context":{"idset":"10163","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6835754,"name":"drain","context":{"idset":"10164","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6840334,"name":"drain","context":{"idset":"10165","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.68451,"name":"drain","context":{"idset":"10166","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6849742,"name":"drain","context":{"idset":"10167","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6854491,"name":"drain","context":{"idset":"10168","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.68591,"name":"drain","context":{"idset":"10169","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6863899,"name":"drain","context":{"idset":"10170","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.686856,"name":"drain","context":{"idset":"10171","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6873364,"name":"drain","context":{"idset":"10172","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6878049,"name":"drain","context":{"idset":"10173","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6882915,"name":"drain","context":{"idset":"10174","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6887579,"name":"drain","context":{"idset":"10175","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6892271,"name":"drain","context":{"idset":"10176","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6897116,"name":"drain","context":{"idset":"10177","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6901844,"name":"drain","context":{"idset":"10178","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.690666,"name":"drain","context":{"idset":"10179","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6911399,"name":"drain","context":{"idset":"10180","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6922328,"name":"drain","context":{"idset":"10181","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6927319,"name":"drain","context":{"idset":"10182","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6932182,"name":"drain","context":{"idset":"10183","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6937087,"name":"drain","context":{"idset":"10184","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6941893,"name":"drain","context":{"idset":"10185","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6946754,"name":"drain","context":{"idset":"10186","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6951489,"name":"drain","context":{"idset":"10187","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6956377,"name":"drain","context":{"idset":"10188","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6961129,"name":"drain","context":{"idset":"10189","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6966033,"name":"drain","context":{"idset":"10190","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6970463,"name":"drain","context":{"idset":"10191","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.69754,"name":"drain","context":{"idset":"10192","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6980183,"name":"drain","context":{"idset":"10193","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6985066,"name":"drain","context":{"idset":"10194","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6989799,"name":"drain","context":{"idset":"10195","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6994677,"name":"drain","context":{"idset":"10196","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.6999395,"name":"drain","context":{"idset":"10197","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7004318,"name":"drain","context":{"idset":"10198","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7009079,"name":"drain","context":{"idset":"10199","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7013969,"name":"drain","context":{"idset":"10200","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7018728,"name":"drain","context":{"idset":"10201","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7023618,"name":"drain","context":{"idset":"10202","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7028384,"name":"drain","context":{"idset":"10203","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7033305,"name":"drain","context":{"idset":"10204","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7038078,"name":"drain","context":{"idset":"10205","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7042999,"name":"drain","context":{"idset":"10206","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7047801,"name":"drain","context":{"idset":"10207","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7052591,"name":"drain","context":{"idset":"10208","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7057574,"name":"drain","context":{"idset":"10209","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7062535,"name":"drain","context":{"idset":"10210","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7067528,"name":"drain","context":{"idset":"10211","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7072382,"name":"drain","context":{"idset":"10212","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.708308,"name":"drain","context":{"idset":"10213","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7088044,"name":"drain","context":{"idset":"10214","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7093077,"name":"drain","context":{"idset":"10215","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7097974,"name":"drain","context":{"idset":"10216","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7102942,"name":"drain","context":{"idset":"10217","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7107835,"name":"drain","context":{"idset":"10218","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7112877,"name":"drain","context":{"idset":"10219","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7117736,"name":"drain","context":{"idset":"10220","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7122889,"name":"drain","context":{"idset":"10221","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7128019,"name":"drain","context":{"idset":"10222","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7133203,"name":"drain","context":{"idset":"10223","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7138119,"name":"drain","context":{"idset":"10224","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7143171,"name":"drain","context":{"idset":"10225","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7148092,"name":"drain","context":{"idset":"10226","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7153108,"name":"drain","context":{"idset":"10227","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7158036,"name":"drain","context":{"idset":"10228","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7163076,"name":"drain","context":{"idset":"10229","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7168026,"name":"drain","context":{"idset":"10230","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7173121,"name":"drain","context":{"idset":"10231","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7178128,"name":"drain","context":{"idset":"10232","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7183242,"name":"drain","context":{"idset":"10233","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7188106,"name":"drain","context":{"idset":"10234","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7193217,"name":"drain","context":{"idset":"10235","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7198174,"name":"drain","context":{"idset":"10236","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7203276,"name":"drain","context":{"idset":"10237","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.720825,"name":"drain","context":{"idset":"10238","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7213387,"name":"drain","context":{"idset":"10239","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7218339,"name":"drain","context":{"idset":"10240","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7223506,"name":"drain","context":{"idset":"10241","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7228494,"name":"drain","context":{"idset":"10242","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7233629,"name":"drain","context":{"idset":"10243","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7238605,"name":"drain","context":{"idset":"10244","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.724771,"name":"drain","context":{"idset":"10245","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7253087,"name":"drain","context":{"idset":"10246","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7258093,"name":"drain","context":{"idset":"10247","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7263203,"name":"drain","context":{"idset":"10248","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7268238,"name":"drain","context":{"idset":"10249","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7273371,"name":"drain","context":{"idset":"10250","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7278409,"name":"drain","context":{"idset":"10251","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7283392,"name":"drain","context":{"idset":"10252","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7287889,"name":"drain","context":{"idset":"10253","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7292325,"name":"drain","context":{"idset":"10254","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7296946,"name":"drain","context":{"idset":"10255","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7301819,"name":"drain","context":{"idset":"10256","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7306533,"name":"drain","context":{"idset":"10257","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7311106,"name":"drain","context":{"idset":"10258","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7316201,"name":"drain","context":{"idset":"10259","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7321312,"name":"drain","context":{"idset":"10260","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7326567,"name":"drain","context":{"idset":"10263","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7331727,"name":"drain","context":{"idset":"10264","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7337053,"name":"drain","context":{"idset":"10265","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.734216,"name":"drain","context":{"idset":"10266","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7347491,"name":"drain","context":{"idset":"10267","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7352583,"name":"drain","context":{"idset":"10268","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7357845,"name":"drain","context":{"idset":"10269","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7363117,"name":"drain","context":{"idset":"10270","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7368257,"name":"drain","context":{"idset":"10271","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7373495,"name":"drain","context":{"idset":"10272","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7378473,"name":"drain","context":{"idset":"10273","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7383184,"name":"drain","context":{"idset":"10274","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7388027,"name":"drain","context":{"idset":"10275","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7406063,"name":"drain","context":{"idset":"10276","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7411273,"name":"drain","context":{"idset":"10277","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7416584,"name":"drain","context":{"idset":"10278","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7421703,"name":"drain","context":{"idset":"10279","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7426755,"name":"drain","context":{"idset":"10280","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.743109,"name":"drain","context":{"idset":"10281","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7435744,"name":"drain","context":{"idset":"10282","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7440925,"name":"drain","context":{"idset":"10283","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7446256,"name":"drain","context":{"idset":"10284","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7451408,"name":"drain","context":{"idset":"10285","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7456596,"name":"drain","context":{"idset":"10286","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7461843,"name":"drain","context":{"idset":"10287","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7467141,"name":"drain","context":{"idset":"10288","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7472312,"name":"drain","context":{"idset":"10289","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7477601,"name":"drain","context":{"idset":"10290","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7482977,"name":"drain","context":{"idset":"10291","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7488167,"name":"drain","context":{"idset":"10292","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7493513,"name":"drain","context":{"idset":"10293","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7498717,"name":"drain","context":{"idset":"10294","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7504029,"name":"drain","context":{"idset":"10295","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7509208,"name":"drain","context":{"idset":"10296","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7514546,"name":"drain","context":{"idset":"10297","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7519753,"name":"drain","context":{"idset":"10298","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7546835,"name":"drain","context":{"idset":"10299","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7552211,"name":"drain","context":{"idset":"10300","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7557607,"name":"drain","context":{"idset":"10301","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7563004,"name":"drain","context":{"idset":"10302","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7568097,"name":"drain","context":{"idset":"10304","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7573688,"name":"drain","context":{"idset":"10305","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7578964,"name":"drain","context":{"idset":"10306","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7584286,"name":"drain","context":{"idset":"10307","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7589595,"name":"drain","context":{"idset":"10308","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7594974,"name":"drain","context":{"idset":"10309","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7600329,"name":"drain","context":{"idset":"10310","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7605767,"name":"drain","context":{"idset":"10311","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7611074,"name":"drain","context":{"idset":"10312","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7616501,"name":"drain","context":{"idset":"10313","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7621789,"name":"drain","context":{"idset":"10314","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7627165,"name":"drain","context":{"idset":"10315","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7632504,"name":"drain","context":{"idset":"10316","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7637987,"name":"drain","context":{"idset":"10317","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7643411,"name":"drain","context":{"idset":"10318","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.764873,"name":"drain","context":{"idset":"10319","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.765419,"name":"drain","context":{"idset":"10320","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7659488,"name":"drain","context":{"idset":"10321","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7664936,"name":"drain","context":{"idset":"10322","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7670238,"name":"drain","context":{"idset":"10323","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7683253,"name":"drain","context":{"idset":"10324","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7688742,"name":"drain","context":{"idset":"10325","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7694161,"name":"drain","context":{"idset":"10326","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7699544,"name":"drain","context":{"idset":"10327","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7705016,"name":"drain","context":{"idset":"10328","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7710357,"name":"drain","context":{"idset":"10329","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7715809,"name":"drain","context":{"idset":"10330","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7720914,"name":"drain","context":{"idset":"10331","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.772646,"name":"drain","context":{"idset":"10332","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7731864,"name":"drain","context":{"idset":"10333","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7737436,"name":"drain","context":{"idset":"10334","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7742906,"name":"drain","context":{"idset":"10335","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7748289,"name":"drain","context":{"idset":"10336","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7753868,"name":"drain","context":{"idset":"10337","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7759302,"name":"drain","context":{"idset":"10338","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7777786,"name":"drain","context":{"idset":"10339","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7783461,"name":"drain","context":{"idset":"10340","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7788889,"name":"drain","context":{"idset":"10341","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7794406,"name":"drain","context":{"idset":"10342","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7799833,"name":"drain","context":{"idset":"10345","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7805405,"name":"drain","context":{"idset":"10346","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7810862,"name":"drain","context":{"idset":"10347","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7816565,"name":"drain","context":{"idset":"10348","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7821996,"name":"drain","context":{"idset":"10349","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7827535,"name":"drain","context":{"idset":"10350","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7833104,"name":"drain","context":{"idset":"10351","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7838526,"name":"drain","context":{"idset":"10352","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7844098,"name":"drain","context":{"idset":"10353","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7849572,"name":"drain","context":{"idset":"10354","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.785512,"name":"drain","context":{"idset":"10357","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7860579,"name":"drain","context":{"idset":"10358","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7866266,"name":"drain","context":{"idset":"10359","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.787174,"name":"drain","context":{"idset":"10360","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7877436,"name":"drain","context":{"idset":"10361","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7883034,"name":"drain","context":{"idset":"10362","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7888515,"name":"drain","context":{"idset":"10363","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7894137,"name":"drain","context":{"idset":"10364","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7899628,"name":"drain","context":{"idset":"10365","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7905819,"name":"drain","context":{"idset":"10366","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7911351,"name":"drain","context":{"idset":"10367","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7917056,"name":"drain","context":{"idset":"10368","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7922578,"name":"drain","context":{"idset":"10369","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7928224,"name":"drain","context":{"idset":"10370","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7949808,"name":"drain","context":{"idset":"10371","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7956009,"name":"drain","context":{"idset":"10372","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7961504,"name":"drain","context":{"idset":"10373","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7967303,"name":"drain","context":{"idset":"10374","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7973092,"name":"drain","context":{"idset":"10375","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7978654,"name":"drain","context":{"idset":"10376","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7984133,"name":"drain","context":{"idset":"10377","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7989631,"name":"drain","context":{"idset":"10379","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.7995579,"name":"drain","context":{"idset":"10380","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8001261,"name":"drain","context":{"idset":"10381","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8006992,"name":"drain","context":{"idset":"10382","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8012576,"name":"drain","context":{"idset":"10383","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8018355,"name":"drain","context":{"idset":"10384","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8024175,"name":"drain","context":{"idset":"10385","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8029778,"name":"drain","context":{"idset":"10386","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8035641,"name":"drain","context":{"idset":"10387","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8041205,"name":"drain","context":{"idset":"10388","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8047054,"name":"drain","context":{"idset":"10389","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8052964,"name":"drain","context":{"idset":"10390","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8058615,"name":"drain","context":{"idset":"10391","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8064358,"name":"drain","context":{"idset":"10392","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8069971,"name":"drain","context":{"idset":"10393","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.807575,"name":"drain","context":{"idset":"10394","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8081365,"name":"drain","context":{"idset":"10395","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8087103,"name":"drain","context":{"idset":"10397","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8093295,"name":"drain","context":{"idset":"10398","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.809891,"name":"drain","context":{"idset":"10399","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8104711,"name":"drain","context":{"idset":"10400","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8110359,"name":"drain","context":{"idset":"10401","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8116155,"name":"drain","context":{"idset":"10402","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8121848,"name":"drain","context":{"idset":"10403","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8128002,"name":"drain","context":{"idset":"10404","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8134112,"name":"drain","context":{"idset":"10405","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8139827,"name":"drain","context":{"idset":"10406","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8145635,"name":"drain","context":{"idset":"10407","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8151345,"name":"drain","context":{"idset":"10408","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8157129,"name":"drain","context":{"idset":"10409","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8170531,"name":"drain","context":{"idset":"10410","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8176594,"name":"drain","context":{"idset":"10411","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8182385,"name":"drain","context":{"idset":"10412","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8188384,"name":"drain","context":{"idset":"10413","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8194222,"name":"drain","context":{"idset":"10414","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8199897,"name":"drain","context":{"idset":"10415","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8206065,"name":"drain","context":{"idset":"10416","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.821187,"name":"drain","context":{"idset":"10417","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8217669,"name":"drain","context":{"idset":"10418","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8223567,"name":"drain","context":{"idset":"10419","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8229251,"name":"drain","context":{"idset":"10420","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8235075,"name":"drain","context":{"idset":"10421","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8240783,"name":"drain","context":{"idset":"10422","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8246417,"name":"drain","context":{"idset":"10423","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.825217,"name":"drain","context":{"idset":"10424","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8258047,"name":"drain","context":{"idset":"10425","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8263943,"name":"drain","context":{"idset":"10426","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8269694,"name":"drain","context":{"idset":"10427","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8275547,"name":"drain","context":{"idset":"10428","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8281255,"name":"drain","context":{"idset":"10429","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8292584,"name":"drain","context":{"idset":"10430","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8298624,"name":"drain","context":{"idset":"10431","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8304527,"name":"drain","context":{"idset":"10432","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8310299,"name":"drain","context":{"idset":"10433","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8316171,"name":"drain","context":{"idset":"10434","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8321884,"name":"drain","context":{"idset":"10435","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8327739,"name":"drain","context":{"idset":"10436","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8333583,"name":"drain","context":{"idset":"10437","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8339403,"name":"drain","context":{"idset":"10438","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8345239,"name":"drain","context":{"idset":"10439","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8350677,"name":"drain","context":{"idset":"10440","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8356552,"name":"drain","context":{"idset":"10441","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8362365,"name":"drain","context":{"idset":"10442","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8368275,"name":"drain","context":{"idset":"10443","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.837419,"name":"drain","context":{"idset":"10444","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.838001,"name":"drain","context":{"idset":"10445","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8385913,"name":"drain","context":{"idset":"10446","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8402145,"name":"drain","context":{"idset":"10447","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8408067,"name":"drain","context":{"idset":"10448","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8414047,"name":"drain","context":{"idset":"10449","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8419731,"name":"drain","context":{"idset":"10450","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8425689,"name":"drain","context":{"idset":"10451","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8431573,"name":"drain","context":{"idset":"10452","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8437548,"name":"drain","context":{"idset":"10453","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8443525,"name":"drain","context":{"idset":"10454","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8449361,"name":"drain","context":{"idset":"10455","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8459799,"name":"drain","context":{"idset":"10456","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8465879,"name":"drain","context":{"idset":"10457","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8471763,"name":"drain","context":{"idset":"10458","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.847779,"name":"drain","context":{"idset":"10459","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8483837,"name":"drain","context":{"idset":"10460","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8489714,"name":"drain","context":{"idset":"10461","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8495717,"name":"drain","context":{"idset":"10462","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8501604,"name":"drain","context":{"idset":"10463","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.85076,"name":"drain","context":{"idset":"10464","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8513627,"name":"drain","context":{"idset":"10465","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8519509,"name":"drain","context":{"idset":"10466","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8525517,"name":"drain","context":{"idset":"10467","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8531401,"name":"drain","context":{"idset":"10468","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.854126,"name":"drain","context":{"idset":"10485","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8547072,"name":"drain","context":{"idset":"10486","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8553243,"name":"drain","context":{"idset":"10487","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8559277,"name":"drain","context":{"idset":"10488","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8565328,"name":"drain","context":{"idset":"10489","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8571026,"name":"drain","context":{"idset":"10490","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.857729,"name":"drain","context":{"idset":"10491","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8583527,"name":"drain","context":{"idset":"10492","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8589344,"name":"drain","context":{"idset":"10493","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.859581,"name":"drain","context":{"idset":"10494","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8601742,"name":"drain","context":{"idset":"10495","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8607864,"name":"drain","context":{"idset":"10496","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8614063,"name":"drain","context":{"idset":"10497","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.862016,"name":"drain","context":{"idset":"10498","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.862627,"name":"drain","context":{"idset":"10499","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8632059,"name":"drain","context":{"idset":"10500","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8638289,"name":"drain","context":{"idset":"10502","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8644516,"name":"drain","context":{"idset":"10503","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8650537,"name":"drain","context":{"idset":"10504","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8657191,"name":"drain","context":{"idset":"10505","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8663347,"name":"drain","context":{"idset":"10506","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8669367,"name":"drain","context":{"idset":"10507","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8675482,"name":"drain","context":{"idset":"10508","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8681495,"name":"drain","context":{"idset":"10509","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8687625,"name":"drain","context":{"idset":"10510","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8693707,"name":"drain","context":{"idset":"10511","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8699951,"name":"drain","context":{"idset":"10512","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8706181,"name":"drain","context":{"idset":"10513","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.871212,"name":"drain","context":{"idset":"10514","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8718712,"name":"drain","context":{"idset":"10515","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8725102,"name":"drain","context":{"idset":"10516","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8731244,"name":"drain","context":{"idset":"10519","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.873749,"name":"drain","context":{"idset":"10520","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8743668,"name":"drain","context":{"idset":"10521","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8749743,"name":"drain","context":{"idset":"10522","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8755939,"name":"drain","context":{"idset":"10523","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8762009,"name":"drain","context":{"idset":"10524","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8773134,"name":"drain","context":{"idset":"10525","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8779325,"name":"drain","context":{"idset":"10526","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.878561,"name":"drain","context":{"idset":"10527","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8791699,"name":"drain","context":{"idset":"10528","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.879792,"name":"drain","context":{"idset":"10529","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8803945,"name":"drain","context":{"idset":"10530","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.881007,"name":"drain","context":{"idset":"10531","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8816423,"name":"drain","context":{"idset":"10532","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8822541,"name":"drain","context":{"idset":"10533","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8828793,"name":"drain","context":{"idset":"10534","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8835046,"name":"drain","context":{"idset":"10535","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.884115,"name":"drain","context":{"idset":"10536","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8847458,"name":"drain","context":{"idset":"10537","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8853745,"name":"drain","context":{"idset":"10538","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.885989,"name":"drain","context":{"idset":"10539","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8866148,"name":"drain","context":{"idset":"10540","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8872313,"name":"drain","context":{"idset":"10541","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8878782,"name":"drain","context":{"idset":"10542","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8885164,"name":"drain","context":{"idset":"10543","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8891327,"name":"drain","context":{"idset":"10544","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8897579,"name":"drain","context":{"idset":"10545","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8903894,"name":"drain","context":{"idset":"10546","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8910067,"name":"drain","context":{"idset":"10547","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8916423,"name":"drain","context":{"idset":"10548","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8922763,"name":"drain","context":{"idset":"10549","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8928943,"name":"drain","context":{"idset":"10550","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8935254,"name":"drain","context":{"idset":"10551","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.89414,"name":"drain","context":{"idset":"10552","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.894774,"name":"drain","context":{"idset":"10553","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8954101,"name":"drain","context":{"idset":"10554","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8960328,"name":"drain","context":{"idset":"10555","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.896667,"name":"drain","context":{"idset":"10556","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8981438,"name":"drain","context":{"idset":"10557","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8987947,"name":"drain","context":{"idset":"10558","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.8994393,"name":"drain","context":{"idset":"10559","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.900069,"name":"drain","context":{"idset":"10560","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9007123,"name":"drain","context":{"idset":"10561","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9013488,"name":"drain","context":{"idset":"10562","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9019725,"name":"drain","context":{"idset":"10563","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9026079,"name":"drain","context":{"idset":"10564","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9032338,"name":"drain","context":{"idset":"10565","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9038777,"name":"drain","context":{"idset":"10566","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9045229,"name":"drain","context":{"idset":"10567","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9051595,"name":"drain","context":{"idset":"10568","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9058013,"name":"drain","context":{"idset":"10569","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9064455,"name":"drain","context":{"idset":"10570","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9070795,"name":"drain","context":{"idset":"10571","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9077208,"name":"drain","context":{"idset":"10572","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9083633,"name":"drain","context":{"idset":"10573","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9089959,"name":"drain","context":{"idset":"10574","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9096422,"name":"drain","context":{"idset":"10575","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9102941,"name":"drain","context":{"idset":"10576","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9109132,"name":"drain","context":{"idset":"10577","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9115505,"name":"drain","context":{"idset":"10578","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9121923,"name":"drain","context":{"idset":"10579","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9128389,"name":"drain","context":{"idset":"10580","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9134924,"name":"drain","context":{"idset":"10581","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9141266,"name":"drain","context":{"idset":"10582","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9147737,"name":"drain","context":{"idset":"10583","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9154232,"name":"drain","context":{"idset":"10584","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9160588,"name":"drain","context":{"idset":"10585","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9167104,"name":"drain","context":{"idset":"10586","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9173901,"name":"drain","context":{"idset":"10587","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9180703,"name":"drain","context":{"idset":"10588","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9187284,"name":"drain","context":{"idset":"10589","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9193842,"name":"drain","context":{"idset":"10590","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9200246,"name":"drain","context":{"idset":"10591","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9206874,"name":"drain","context":{"idset":"10592","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.92134,"name":"drain","context":{"idset":"10593","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9219739,"name":"drain","context":{"idset":"10594","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9226279,"name":"drain","context":{"idset":"10595","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9232838,"name":"drain","context":{"idset":"10596","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9239259,"name":"drain","context":{"idset":"10597","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9245603,"name":"drain","context":{"idset":"10598","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9252036,"name":"drain","context":{"idset":"10599","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9258575,"name":"drain","context":{"idset":"10600","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.926518,"name":"drain","context":{"idset":"10601","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9271657,"name":"drain","context":{"idset":"10602","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9278259,"name":"drain","context":{"idset":"10603","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9284847,"name":"drain","context":{"idset":"10604","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9291329,"name":"drain","context":{"idset":"10605","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9297945,"name":"drain","context":{"idset":"10606","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9304538,"name":"drain","context":{"idset":"10607","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9311035,"name":"drain","context":{"idset":"10608","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9317656,"name":"drain","context":{"idset":"10609","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9324262,"name":"drain","context":{"idset":"10610","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9330742,"name":"drain","context":{"idset":"10611","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9337378,"name":"drain","context":{"idset":"10612","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9384012,"name":"drain","context":{"idset":"10613","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9390643,"name":"drain","context":{"idset":"10614","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9397256,"name":"drain","context":{"idset":"10615","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9403901,"name":"drain","context":{"idset":"10616","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9410374,"name":"drain","context":{"idset":"10617","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9417028,"name":"drain","context":{"idset":"10618","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9423676,"name":"drain","context":{"idset":"10619","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9430192,"name":"drain","context":{"idset":"10620","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9436779,"name":"drain","context":{"idset":"10621","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.944346,"name":"drain","context":{"idset":"10622","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9450006,"name":"drain","context":{"idset":"10623","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.945663,"name":"drain","context":{"idset":"10624","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9463291,"name":"drain","context":{"idset":"10625","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9469812,"name":"drain","context":{"idset":"10626","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9476433,"name":"drain","context":{"idset":"10627","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9483123,"name":"drain","context":{"idset":"10628","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9489684,"name":"drain","context":{"idset":"10630","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9496441,"name":"drain","context":{"idset":"10631","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9503181,"name":"drain","context":{"idset":"10632","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9509833,"name":"drain","context":{"idset":"10633","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9516582,"name":"drain","context":{"idset":"10635","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9523251,"name":"drain","context":{"idset":"10636","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.952985,"name":"drain","context":{"idset":"10640","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9536617,"name":"drain","context":{"idset":"10641","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.954339,"name":"drain","context":{"idset":"10645","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9549987,"name":"drain","context":{"idset":"10646","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9556708,"name":"drain","context":{"idset":"10647","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9563468,"name":"drain","context":{"idset":"10648","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9570093,"name":"drain","context":{"idset":"10649","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9576864,"name":"drain","context":{"idset":"10650","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9583662,"name":"drain","context":{"idset":"10651","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9590294,"name":"drain","context":{"idset":"10652","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9597032,"name":"drain","context":{"idset":"10653","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.960386,"name":"drain","context":{"idset":"10654","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9610512,"name":"drain","context":{"idset":"10655","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9617321,"name":"drain","context":{"idset":"10656","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9624083,"name":"drain","context":{"idset":"10658","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9630475,"name":"drain","context":{"idset":"10659","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9637303,"name":"drain","context":{"idset":"10660","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9644082,"name":"drain","context":{"idset":"10661","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9650755,"name":"drain","context":{"idset":"10662","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9657593,"name":"drain","context":{"idset":"10663","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9664462,"name":"drain","context":{"idset":"10664","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9671168,"name":"drain","context":{"idset":"10665","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9678028,"name":"drain","context":{"idset":"10666","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9684904,"name":"drain","context":{"idset":"10667","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9691629,"name":"drain","context":{"idset":"10668","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9698458,"name":"drain","context":{"idset":"10669","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.970535,"name":"drain","context":{"idset":"10670","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9712081,"name":"drain","context":{"idset":"10671","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9718914,"name":"drain","context":{"idset":"10672","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9725807,"name":"drain","context":{"idset":"10673","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9732575,"name":"drain","context":{"idset":"10674","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9739485,"name":"drain","context":{"idset":"10675","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9746332,"name":"drain","context":{"idset":"10676","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9753218,"name":"drain","context":{"idset":"10677","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9759951,"name":"drain","context":{"idset":"10678","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9766798,"name":"drain","context":{"idset":"10679","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9773707,"name":"drain","context":{"idset":"10680","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9780483,"name":"drain","context":{"idset":"10681","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9787369,"name":"drain","context":{"idset":"10682","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9794252,"name":"drain","context":{"idset":"10683","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9801066,"name":"drain","context":{"idset":"10684","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9807997,"name":"drain","context":{"idset":"10685","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9814916,"name":"drain","context":{"idset":"10686","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9821699,"name":"drain","context":{"idset":"10687","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.982866,"name":"drain","context":{"idset":"10688","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9835577,"name":"drain","context":{"idset":"10689","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.984236,"name":"drain","context":{"idset":"10690","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9849284,"name":"drain","context":{"idset":"10691","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9856224,"name":"drain","context":{"idset":"10692","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.986316,"name":"drain","context":{"idset":"10695","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9870019,"name":"drain","context":{"idset":"10696","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9876971,"name":"drain","context":{"idset":"10697","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9883957,"name":"drain","context":{"idset":"10698","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9890828,"name":"drain","context":{"idset":"10699","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9897785,"name":"drain","context":{"idset":"10700","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9904838,"name":"drain","context":{"idset":"10701","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9911695,"name":"drain","context":{"idset":"10703","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9918766,"name":"drain","context":{"idset":"10704","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9925759,"name":"drain","context":{"idset":"10705","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9932585,"name":"drain","context":{"idset":"10706","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9939585,"name":"drain","context":{"idset":"10707","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9946609,"name":"drain","context":{"idset":"10708","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9953613,"name":"drain","context":{"idset":"10709","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9960492,"name":"drain","context":{"idset":"10710","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9967499,"name":"drain","context":{"idset":"10711","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9974494,"name":"drain","context":{"idset":"10713","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9981399,"name":"drain","context":{"idset":"10714","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9988456,"name":"drain","context":{"idset":"10715","reason":"broker was unresponsive"}}
+{"timestamp":1712586729.9995472,"name":"drain","context":{"idset":"10716","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0002451,"name":"drain","context":{"idset":"10717","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0009351,"name":"drain","context":{"idset":"10719","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0016501,"name":"drain","context":{"idset":"10720","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0023677,"name":"drain","context":{"idset":"10721","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0030608,"name":"drain","context":{"idset":"10722","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0037658,"name":"drain","context":{"idset":"10723","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0061176,"name":"drain","context":{"idset":"10724","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0068429,"name":"drain","context":{"idset":"10725","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0075555,"name":"drain","context":{"idset":"10726","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0082519,"name":"drain","context":{"idset":"10727","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0089664,"name":"drain","context":{"idset":"10728","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0097313,"name":"drain","context":{"idset":"10729","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0104461,"name":"drain","context":{"idset":"10730","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0111475,"name":"drain","context":{"idset":"10731","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0118568,"name":"drain","context":{"idset":"10732","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.012568,"name":"drain","context":{"idset":"10733","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0132799,"name":"drain","context":{"idset":"10734","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0139809,"name":"drain","context":{"idset":"10735","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0146956,"name":"drain","context":{"idset":"10736","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0154107,"name":"drain","context":{"idset":"10737","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0161116,"name":"drain","context":{"idset":"10738","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0168262,"name":"drain","context":{"idset":"10741","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0175431,"name":"drain","context":{"idset":"10742","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0182457,"name":"drain","context":{"idset":"10743","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0189657,"name":"drain","context":{"idset":"10744","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0196798,"name":"drain","context":{"idset":"10745","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0203986,"name":"drain","context":{"idset":"10746","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0211012,"name":"drain","context":{"idset":"10747","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0218189,"name":"drain","context":{"idset":"10748","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0225396,"name":"drain","context":{"idset":"10749","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0232465,"name":"drain","context":{"idset":"10750","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.023968,"name":"drain","context":{"idset":"10751","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0246878,"name":"drain","context":{"idset":"10752","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0254097,"name":"drain","context":{"idset":"10753","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0261192,"name":"drain","context":{"idset":"10754","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0268397,"name":"drain","context":{"idset":"10755","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0275624,"name":"drain","context":{"idset":"10756","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0282874,"name":"drain","context":{"idset":"10757","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0290027,"name":"drain","context":{"idset":"10758","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0297265,"name":"drain","context":{"idset":"10759","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0304518,"name":"drain","context":{"idset":"10760","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0311618,"name":"drain","context":{"idset":"10761","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0318952,"name":"drain","context":{"idset":"10762","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0326202,"name":"drain","context":{"idset":"10763","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0333416,"name":"drain","context":{"idset":"10764","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0340569,"name":"drain","context":{"idset":"10765","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0347795,"name":"drain","context":{"idset":"10766","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0355017,"name":"drain","context":{"idset":"10767","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0362082,"name":"drain","context":{"idset":"10768","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0369375,"name":"drain","context":{"idset":"10769","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0376675,"name":"drain","context":{"idset":"10770","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0383642,"name":"drain","context":{"idset":"10771","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0390813,"name":"drain","context":{"idset":"10772","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0398061,"name":"drain","context":{"idset":"10773","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0405431,"name":"drain","context":{"idset":"10774","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0412765,"name":"drain","context":{"idset":"10775","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0419953,"name":"drain","context":{"idset":"10776","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0427361,"name":"drain","context":{"idset":"10777","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0434711,"name":"drain","context":{"idset":"10778","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0441885,"name":"drain","context":{"idset":"10779","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.044915,"name":"drain","context":{"idset":"10780","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0456445,"name":"drain","context":{"idset":"10781","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0463805,"name":"drain","context":{"idset":"10782","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0470991,"name":"drain","context":{"idset":"10783","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0478263,"name":"drain","context":{"idset":"10784","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0485594,"name":"drain","context":{"idset":"10785","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0492909,"name":"drain","context":{"idset":"10786","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0500119,"name":"drain","context":{"idset":"10787","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0507464,"name":"drain","context":{"idset":"10788","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0514832,"name":"drain","context":{"idset":"10789","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0522017,"name":"drain","context":{"idset":"10790","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0529327,"name":"drain","context":{"idset":"10791","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0536695,"name":"drain","context":{"idset":"10792","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0544112,"name":"drain","context":{"idset":"10793","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0551353,"name":"drain","context":{"idset":"10794","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0558693,"name":"drain","context":{"idset":"10795","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0566015,"name":"drain","context":{"idset":"10796","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0573413,"name":"drain","context":{"idset":"10797","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0580673,"name":"drain","context":{"idset":"10798","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0588036,"name":"drain","context":{"idset":"10799","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0595448,"name":"drain","context":{"idset":"10800","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0602863,"name":"drain","context":{"idset":"10801","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0610089,"name":"drain","context":{"idset":"10802","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.061754,"name":"drain","context":{"idset":"10803","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.062495,"name":"drain","context":{"idset":"10804","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.063225,"name":"drain","context":{"idset":"10805","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0639656,"name":"drain","context":{"idset":"10806","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.064708,"name":"drain","context":{"idset":"10807","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0654497,"name":"drain","context":{"idset":"10808","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0661778,"name":"drain","context":{"idset":"10809","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0669262,"name":"drain","context":{"idset":"10810","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0676672,"name":"drain","context":{"idset":"10811","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0684099,"name":"drain","context":{"idset":"10812","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0691414,"name":"drain","context":{"idset":"10813","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0698907,"name":"drain","context":{"idset":"10814","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0706484,"name":"drain","context":{"idset":"10815","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0713942,"name":"drain","context":{"idset":"10816","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0721283,"name":"drain","context":{"idset":"10817","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0728791,"name":"drain","context":{"idset":"10818","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0736234,"name":"drain","context":{"idset":"10819","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.074369,"name":"drain","context":{"idset":"10820","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0751014,"name":"drain","context":{"idset":"10821","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0758491,"name":"drain","context":{"idset":"10822","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0766039,"name":"drain","context":{"idset":"10823","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0773475,"name":"drain","context":{"idset":"10824","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0780895,"name":"drain","context":{"idset":"10825","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0788381,"name":"drain","context":{"idset":"10826","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0795918,"name":"drain","context":{"idset":"10827","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0803499,"name":"drain","context":{"idset":"10828","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0810907,"name":"drain","context":{"idset":"10829","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0818517,"name":"drain","context":{"idset":"10830","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.082602,"name":"drain","context":{"idset":"10831","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0833611,"name":"drain","context":{"idset":"10832","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0841022,"name":"drain","context":{"idset":"10833","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0848575,"name":"drain","context":{"idset":"10834","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0856113,"name":"drain","context":{"idset":"10835","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0863626,"name":"drain","context":{"idset":"10836","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0871065,"name":"drain","context":{"idset":"10837","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0878675,"name":"drain","context":{"idset":"10838","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0886238,"name":"drain","context":{"idset":"10839","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0893807,"name":"drain","context":{"idset":"10840","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.090126,"name":"drain","context":{"idset":"10841","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0908861,"name":"drain","context":{"idset":"10842","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0916429,"name":"drain","context":{"idset":"10843","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.092401,"name":"drain","context":{"idset":"10844","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.093147,"name":"drain","context":{"idset":"10845","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.093904,"name":"drain","context":{"idset":"10846","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0946648,"name":"drain","context":{"idset":"10847","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0954287,"name":"drain","context":{"idset":"10848","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0961738,"name":"drain","context":{"idset":"10849","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0969365,"name":"drain","context":{"idset":"10850","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0977061,"name":"drain","context":{"idset":"10851","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0984728,"name":"drain","context":{"idset":"10852","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.099215,"name":"drain","context":{"idset":"10853","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.0999856,"name":"drain","context":{"idset":"10854","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1007521,"name":"drain","context":{"idset":"10855","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1015208,"name":"drain","context":{"idset":"10856","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1022823,"name":"drain","context":{"idset":"10857","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1030324,"name":"drain","context":{"idset":"10858","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1037993,"name":"drain","context":{"idset":"10859","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1045601,"name":"drain","context":{"idset":"10860","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1053317,"name":"drain","context":{"idset":"10861","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1060853,"name":"drain","context":{"idset":"10862","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1068504,"name":"drain","context":{"idset":"10863","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1076145,"name":"drain","context":{"idset":"10864","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1083839,"name":"drain","context":{"idset":"10865","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1091378,"name":"drain","context":{"idset":"10866","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1099076,"name":"drain","context":{"idset":"10867","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1106794,"name":"drain","context":{"idset":"10868","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1114559,"name":"drain","context":{"idset":"10869","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1122112,"name":"drain","context":{"idset":"10870","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1129901,"name":"drain","context":{"idset":"10873","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1137638,"name":"drain","context":{"idset":"10874","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1145341,"name":"drain","context":{"idset":"10875","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1152847,"name":"drain","context":{"idset":"10876","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1160452,"name":"drain","context":{"idset":"10877","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1168158,"name":"drain","context":{"idset":"10878","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1175933,"name":"drain","context":{"idset":"10879","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1183784,"name":"drain","context":{"idset":"10880","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1191421,"name":"drain","context":{"idset":"10881","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1199148,"name":"drain","context":{"idset":"10882","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1206884,"name":"drain","context":{"idset":"10883","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1214144,"name":"drain","context":{"idset":"10884","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1222856,"name":"drain","context":{"idset":"10885","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1229928,"name":"drain","context":{"idset":"10886","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1237235,"name":"drain","context":{"idset":"10887","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1244638,"name":"drain","context":{"idset":"10888","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1251903,"name":"drain","context":{"idset":"10889","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1259255,"name":"drain","context":{"idset":"10890","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1266656,"name":"drain","context":{"idset":"10891","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1274018,"name":"drain","context":{"idset":"10892","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1281295,"name":"drain","context":{"idset":"10893","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1288707,"name":"drain","context":{"idset":"10894","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1296191,"name":"drain","context":{"idset":"10895","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1303594,"name":"drain","context":{"idset":"10896","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1310918,"name":"drain","context":{"idset":"10897","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1318393,"name":"drain","context":{"idset":"10898","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1325855,"name":"drain","context":{"idset":"10899","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1333282,"name":"drain","context":{"idset":"10900","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1340587,"name":"drain","context":{"idset":"10901","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1347938,"name":"drain","context":{"idset":"10902","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1355681,"name":"drain","context":{"idset":"10903","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1363313,"name":"drain","context":{"idset":"10904","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1370683,"name":"drain","context":{"idset":"10905","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.137821,"name":"drain","context":{"idset":"10906","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1385705,"name":"drain","context":{"idset":"10907","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1393213,"name":"drain","context":{"idset":"10908","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1400599,"name":"drain","context":{"idset":"10909","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1408105,"name":"drain","context":{"idset":"10910","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1415596,"name":"drain","context":{"idset":"10913","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.142312,"name":"drain","context":{"idset":"10914","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1430511,"name":"drain","context":{"idset":"10915","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1438079,"name":"drain","context":{"idset":"10916","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1477892,"name":"drain","context":{"idset":"10917","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1486168,"name":"drain","context":{"idset":"10918","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1493907,"name":"drain","context":{"idset":"10919","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1510487,"name":"drain","context":{"idset":"10921","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1518538,"name":"drain","context":{"idset":"10922","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1527429,"name":"drain","context":{"idset":"10923","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1535602,"name":"drain","context":{"idset":"10924","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1551995,"name":"drain","context":{"idset":"10926","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1560402,"name":"drain","context":{"idset":"10927","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1568894,"name":"drain","context":{"idset":"10928","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1577225,"name":"drain","context":{"idset":"10929","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.159318,"name":"drain","context":{"idset":"10931","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1601274,"name":"drain","context":{"idset":"10932","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1609173,"name":"drain","context":{"idset":"10933","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1618023,"name":"drain","context":{"idset":"10934","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1633887,"name":"drain","context":{"idset":"10936","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.164149,"name":"drain","context":{"idset":"10937","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1649587,"name":"drain","context":{"idset":"10938","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1657495,"name":"drain","context":{"idset":"10939","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1673145,"name":"drain","context":{"idset":"10941","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1681118,"name":"drain","context":{"idset":"10942","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1696222,"name":"drain","context":{"idset":"10946","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1704392,"name":"drain","context":{"idset":"10947","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.171277,"name":"drain","context":{"idset":"10948","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1721876,"name":"drain","context":{"idset":"10949","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.173986,"name":"drain","context":{"idset":"10951","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1748338,"name":"drain","context":{"idset":"10953","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1756604,"name":"drain","context":{"idset":"10954","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1773174,"name":"drain","context":{"idset":"10956","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1781948,"name":"drain","context":{"idset":"10957","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1790569,"name":"drain","context":{"idset":"10958","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1799095,"name":"drain","context":{"idset":"10959","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1816118,"name":"drain","context":{"idset":"10961","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1824946,"name":"drain","context":{"idset":"10962","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1836169,"name":"drain","context":{"idset":"10963","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1844623,"name":"drain","context":{"idset":"10964","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1861386,"name":"drain","context":{"idset":"10966","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1869469,"name":"drain","context":{"idset":"10967","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1877575,"name":"drain","context":{"idset":"10968","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1885448,"name":"drain","context":{"idset":"10969","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1897891,"name":"drain","context":{"idset":"10971","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1904087,"name":"drain","context":{"idset":"10972","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1909568,"name":"drain","context":{"idset":"10973","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1915638,"name":"drain","context":{"idset":"10974","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.192745,"name":"drain","context":{"idset":"10976","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1933122,"name":"drain","context":{"idset":"10977","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1940937,"name":"drain","context":{"idset":"10978","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1948147,"name":"drain","context":{"idset":"10979","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1963716,"name":"drain","context":{"idset":"10981","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1971822,"name":"drain","context":{"idset":"10982","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.197983,"name":"drain","context":{"idset":"10983","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.1988099,"name":"drain","context":{"idset":"10984","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2004442,"name":"drain","context":{"idset":"10986","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.201257,"name":"drain","context":{"idset":"10987","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.20208,"name":"drain","context":{"idset":"10988","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2031476,"name":"drain","context":{"idset":"10989","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2047698,"name":"drain","context":{"idset":"10991","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2055717,"name":"drain","context":{"idset":"10992","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2097118,"name":"drain","context":{"idset":"10997","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2105908,"name":"drain","context":{"idset":"10998","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2114873,"name":"drain","context":{"idset":"11001","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2123318,"name":"drain","context":{"idset":"11002","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.213145,"name":"drain","context":{"idset":"11003","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2139823,"name":"drain","context":{"idset":"11004","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2148178,"name":"drain","context":{"idset":"11005","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2157021,"name":"drain","context":{"idset":"11006","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2165871,"name":"drain","context":{"idset":"11007","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2174289,"name":"drain","context":{"idset":"11008","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2182865,"name":"drain","context":{"idset":"11009","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2190502,"name":"drain","context":{"idset":"11010","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2198188,"name":"drain","context":{"idset":"11011","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2206457,"name":"drain","context":{"idset":"11012","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2214029,"name":"drain","context":{"idset":"11013","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2221653,"name":"drain","context":{"idset":"11014","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2229571,"name":"drain","context":{"idset":"11015","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2238092,"name":"drain","context":{"idset":"11016","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.224581,"name":"drain","context":{"idset":"11017","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2254059,"name":"drain","context":{"idset":"11018","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2261283,"name":"drain","context":{"idset":"11019","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2271187,"name":"drain","context":{"idset":"11020","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2290194,"name":"drain","context":{"idset":"11022","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2299685,"name":"drain","context":{"idset":"11023","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2308791,"name":"drain","context":{"idset":"11024","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.231756,"name":"drain","context":{"idset":"11025","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2326181,"name":"drain","context":{"idset":"11026","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2334869,"name":"drain","context":{"idset":"11027","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.234427,"name":"drain","context":{"idset":"11028","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.235357,"name":"drain","context":{"idset":"11029","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2363288,"name":"drain","context":{"idset":"11030","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2372029,"name":"drain","context":{"idset":"11031","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2381244,"name":"drain","context":{"idset":"11032","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2390435,"name":"drain","context":{"idset":"11033","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2399664,"name":"drain","context":{"idset":"11034","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2408648,"name":"drain","context":{"idset":"11035","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2417736,"name":"drain","context":{"idset":"11036","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2427022,"name":"drain","context":{"idset":"11037","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2436416,"name":"drain","context":{"idset":"11038","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2444947,"name":"drain","context":{"idset":"11039","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2452922,"name":"drain","context":{"idset":"11040","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2461562,"name":"drain","context":{"idset":"11041","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2469206,"name":"drain","context":{"idset":"11042","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2477703,"name":"drain","context":{"idset":"11043","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2487102,"name":"drain","context":{"idset":"11044","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2495499,"name":"drain","context":{"idset":"11045","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2504549,"name":"drain","context":{"idset":"11046","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2512844,"name":"drain","context":{"idset":"11047","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2520957,"name":"drain","context":{"idset":"11048","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2529182,"name":"drain","context":{"idset":"11049","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2537689,"name":"drain","context":{"idset":"11050","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2546539,"name":"drain","context":{"idset":"11051","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2554779,"name":"drain","context":{"idset":"11052","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2563386,"name":"drain","context":{"idset":"11053","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2571454,"name":"drain","context":{"idset":"11054","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2581172,"name":"drain","context":{"idset":"11055","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2589848,"name":"drain","context":{"idset":"11056","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2597923,"name":"drain","context":{"idset":"11057","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2606328,"name":"drain","context":{"idset":"11058","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2615047,"name":"drain","context":{"idset":"11059","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2623994,"name":"drain","context":{"idset":"11060","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2632189,"name":"drain","context":{"idset":"11061","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2640638,"name":"drain","context":{"idset":"11062","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2648697,"name":"drain","context":{"idset":"11063","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.265738,"name":"drain","context":{"idset":"11064","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2665806,"name":"drain","context":{"idset":"11065","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2671752,"name":"drain","context":{"idset":"11066","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2680254,"name":"drain","context":{"idset":"11067","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.268791,"name":"drain","context":{"idset":"11068","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2694745,"name":"drain","context":{"idset":"11069","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2700584,"name":"drain","context":{"idset":"11070","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2708108,"name":"drain","context":{"idset":"11071","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2716668,"name":"drain","context":{"idset":"11072","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2725344,"name":"drain","context":{"idset":"11073","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2733746,"name":"drain","context":{"idset":"11074","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2740924,"name":"drain","context":{"idset":"11075","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2748172,"name":"drain","context":{"idset":"11076","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2756732,"name":"drain","context":{"idset":"11077","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2765133,"name":"drain","context":{"idset":"11078","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2773809,"name":"drain","context":{"idset":"11079","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2781878,"name":"drain","context":{"idset":"11080","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2789061,"name":"drain","context":{"idset":"11081","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2796862,"name":"drain","context":{"idset":"11082","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.280401,"name":"drain","context":{"idset":"11083","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2811818,"name":"drain","context":{"idset":"11084","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2820334,"name":"drain","context":{"idset":"11085","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2827263,"name":"drain","context":{"idset":"11086","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2832153,"name":"drain","context":{"idset":"11087","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2839735,"name":"drain","context":{"idset":"11088","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2847669,"name":"drain","context":{"idset":"11089","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2855725,"name":"drain","context":{"idset":"11090","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2863326,"name":"drain","context":{"idset":"11091","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2870779,"name":"drain","context":{"idset":"11092","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2878857,"name":"drain","context":{"idset":"11093","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2886693,"name":"drain","context":{"idset":"11094","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2894821,"name":"drain","context":{"idset":"11095","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2901852,"name":"drain","context":{"idset":"11096","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2909927,"name":"drain","context":{"idset":"11097","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.291822,"name":"drain","context":{"idset":"11098","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2926071,"name":"drain","context":{"idset":"11099","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.293117,"name":"drain","context":{"idset":"11100","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2939143,"name":"drain","context":{"idset":"11101","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2947795,"name":"drain","context":{"idset":"11102","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2956493,"name":"drain","context":{"idset":"11103","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2964497,"name":"drain","context":{"idset":"11104","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.297003,"name":"drain","context":{"idset":"11105","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2976348,"name":"drain","context":{"idset":"11106","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2984884,"name":"drain","context":{"idset":"11107","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2990029,"name":"drain","context":{"idset":"11108","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.2996545,"name":"drain","context":{"idset":"11109","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3003585,"name":"drain","context":{"idset":"11110","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3011329,"name":"drain","context":{"idset":"11111","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.301666,"name":"drain","context":{"idset":"11112","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3021491,"name":"drain","context":{"idset":"11113","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3029151,"name":"drain","context":{"idset":"11114","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.303776,"name":"drain","context":{"idset":"11117","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3044288,"name":"drain","context":{"idset":"11118","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.305233,"name":"drain","context":{"idset":"11119","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3060081,"name":"drain","context":{"idset":"11120","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3065984,"name":"drain","context":{"idset":"11121","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.307178,"name":"drain","context":{"idset":"11122","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3078945,"name":"drain","context":{"idset":"11123","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3086877,"name":"drain","context":{"idset":"11124","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3094378,"name":"drain","context":{"idset":"11253","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3103173,"name":"drain","context":{"idset":"11254","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3111868,"name":"drain","context":{"idset":"11255","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3120561,"name":"drain","context":{"idset":"11256","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3129277,"name":"drain","context":{"idset":"11257","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3135803,"name":"drain","context":{"idset":"11258","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3144333,"name":"drain","context":{"idset":"11259","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3153038,"name":"drain","context":{"idset":"11260","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3161738,"name":"drain","context":{"idset":"11261","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3170178,"name":"drain","context":{"idset":"11262","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3176901,"name":"drain","context":{"idset":"11263","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3185828,"name":"drain","context":{"idset":"11264","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3192801,"name":"drain","context":{"idset":"11265","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3199911,"name":"drain","context":{"idset":"11266","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3208184,"name":"drain","context":{"idset":"11267","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3214822,"name":"drain","context":{"idset":"11268","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3223529,"name":"drain","context":{"idset":"11269","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3232236,"name":"drain","context":{"idset":"11270","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3241799,"name":"drain","context":{"idset":"11271","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3249426,"name":"drain","context":{"idset":"11272","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3257644,"name":"drain","context":{"idset":"11273","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3266084,"name":"drain","context":{"idset":"11274","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3274918,"name":"drain","context":{"idset":"11275","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3283896,"name":"drain","context":{"idset":"11276","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3291388,"name":"drain","context":{"idset":"11277","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3299322,"name":"drain","context":{"idset":"11278","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3306968,"name":"drain","context":{"idset":"11279","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3313055,"name":"drain","context":{"idset":"11280","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3319557,"name":"drain","context":{"idset":"11281","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3326077,"name":"drain","context":{"idset":"11282","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3332946,"name":"drain","context":{"idset":"11283","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.33412,"name":"drain","context":{"idset":"11284","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3348804,"name":"drain","context":{"idset":"11285","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3357017,"name":"drain","context":{"idset":"11286","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3365741,"name":"drain","context":{"idset":"11287","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3372564,"name":"drain","context":{"idset":"11288","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3381317,"name":"drain","context":{"idset":"11289","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3389525,"name":"drain","context":{"idset":"11290","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3396008,"name":"drain","context":{"idset":"11291","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3402054,"name":"drain","context":{"idset":"11292","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3408799,"name":"drain","context":{"idset":"11293","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3417299,"name":"drain","context":{"idset":"11294","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3426425,"name":"drain","context":{"idset":"11295","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3434927,"name":"drain","context":{"idset":"11296","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3443563,"name":"drain","context":{"idset":"11297","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3452356,"name":"drain","context":{"idset":"11298","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3461256,"name":"drain","context":{"idset":"11299","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3470054,"name":"drain","context":{"idset":"11300","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3479106,"name":"drain","context":{"idset":"11301","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3488164,"name":"drain","context":{"idset":"11302","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3498938,"name":"drain","context":{"idset":"11303","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3508298,"name":"drain","context":{"idset":"11304","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3517463,"name":"drain","context":{"idset":"11305","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3526762,"name":"drain","context":{"idset":"11306","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3535187,"name":"drain","context":{"idset":"11307","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3544044,"name":"drain","context":{"idset":"11308","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3551991,"name":"drain","context":{"idset":"11309","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3560393,"name":"drain","context":{"idset":"11310","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3568573,"name":"drain","context":{"idset":"11311","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3576519,"name":"drain","context":{"idset":"11312","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3585615,"name":"drain","context":{"idset":"11313","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3594751,"name":"drain","context":{"idset":"11314","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3603632,"name":"drain","context":{"idset":"11315","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3609359,"name":"drain","context":{"idset":"11316","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3615558,"name":"drain","context":{"idset":"11317","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3621497,"name":"drain","context":{"idset":"11318","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3630621,"name":"drain","context":{"idset":"11319","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3638308,"name":"drain","context":{"idset":"11320","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3644555,"name":"drain","context":{"idset":"11321","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3652065,"name":"drain","context":{"idset":"11322","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3661005,"name":"drain","context":{"idset":"11323","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3669832,"name":"drain","context":{"idset":"11324","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3678229,"name":"drain","context":{"idset":"11325","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.368607,"name":"drain","context":{"idset":"11326","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3692305,"name":"drain","context":{"idset":"11327","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.369848,"name":"drain","context":{"idset":"11328","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3704431,"name":"drain","context":{"idset":"11331","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3712914,"name":"drain","context":{"idset":"11332","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3722007,"name":"drain","context":{"idset":"11333","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3731141,"name":"drain","context":{"idset":"11334","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3740175,"name":"drain","context":{"idset":"11335","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3749199,"name":"drain","context":{"idset":"11336","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3757911,"name":"drain","context":{"idset":"11337","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3765516,"name":"drain","context":{"idset":"11338","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3773844,"name":"drain","context":{"idset":"11339","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3780937,"name":"drain","context":{"idset":"11340","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.378999,"name":"drain","context":{"idset":"11341","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3796902,"name":"drain","context":{"idset":"11342","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3804071,"name":"drain","context":{"idset":"11343","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3812001,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3820944,"name":"drain","context":{"idset":"11345","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3828323,"name":"drain","context":{"idset":"11346","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3835852,"name":"drain","context":{"idset":"11347","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3844664,"name":"drain","context":{"idset":"11348","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3852272,"name":"drain","context":{"idset":"11349","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3858683,"name":"drain","context":{"idset":"11350","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3867068,"name":"drain","context":{"idset":"11351","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3876026,"name":"drain","context":{"idset":"11352","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3884199,"name":"drain","context":{"idset":"11353","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.389241,"name":"drain","context":{"idset":"11354","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.390152,"name":"drain","context":{"idset":"11355","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3910763,"name":"drain","context":{"idset":"11356","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3919828,"name":"drain","context":{"idset":"11357","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3929205,"name":"drain","context":{"idset":"11359","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3937068,"name":"drain","context":{"idset":"11360","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3946249,"name":"drain","context":{"idset":"11361","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3953283,"name":"drain","context":{"idset":"11362","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3960295,"name":"drain","context":{"idset":"11363","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3968246,"name":"drain","context":{"idset":"11364","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3976076,"name":"drain","context":{"idset":"11365","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3982239,"name":"drain","context":{"idset":"11366","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3988559,"name":"drain","context":{"idset":"11367","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.3994796,"name":"drain","context":{"idset":"11368","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.400115,"name":"drain","context":{"idset":"11370","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4007277,"name":"drain","context":{"idset":"11371","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4013898,"name":"drain","context":{"idset":"11372","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4020572,"name":"drain","context":{"idset":"11375","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4026713,"name":"drain","context":{"idset":"11376","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4033256,"name":"drain","context":{"idset":"11377","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4039688,"name":"drain","context":{"idset":"11378","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4046078,"name":"drain","context":{"idset":"11379","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4052176,"name":"drain","context":{"idset":"11380","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4058444,"name":"drain","context":{"idset":"11381","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4065104,"name":"drain","context":{"idset":"11382","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4073603,"name":"drain","context":{"idset":"11383","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4082365,"name":"drain","context":{"idset":"11384","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4091465,"name":"drain","context":{"idset":"11385","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4099174,"name":"drain","context":{"idset":"11386","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4106967,"name":"drain","context":{"idset":"11389","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4115818,"name":"drain","context":{"idset":"11391","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4122086,"name":"drain","context":{"idset":"11392","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4130838,"name":"drain","context":{"idset":"11393","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4140134,"name":"drain","context":{"idset":"11394","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4148858,"name":"drain","context":{"idset":"11396","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4155593,"name":"drain","context":{"idset":"11397","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4161973,"name":"drain","context":{"idset":"11398","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4168212,"name":"drain","context":{"idset":"11399","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4397001,"name":"drain","context":{"idset":"11430","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4407041,"name":"drain","context":{"idset":"11432","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4417124,"name":"drain","context":{"idset":"11433","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4427187,"name":"drain","context":{"idset":"11434","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.443707,"name":"drain","context":{"idset":"11435","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4446914,"name":"drain","context":{"idset":"11436","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.445693,"name":"drain","context":{"idset":"11437","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4466808,"name":"drain","context":{"idset":"11438","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4476676,"name":"drain","context":{"idset":"11443","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.448653,"name":"drain","context":{"idset":"11444","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4496469,"name":"drain","context":{"idset":"11445","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4506195,"name":"drain","context":{"idset":"11446","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4516277,"name":"drain","context":{"idset":"11447","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4525969,"name":"drain","context":{"idset":"11448","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4534168,"name":"drain","context":{"idset":"11449","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.45437,"name":"drain","context":{"idset":"11450","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.455327,"name":"drain","context":{"idset":"11451","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4562852,"name":"drain","context":{"idset":"11452","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4572334,"name":"drain","context":{"idset":"11453","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4581912,"name":"drain","context":{"idset":"11454","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4591448,"name":"drain","context":{"idset":"11455","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4601142,"name":"drain","context":{"idset":"11456","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4610586,"name":"drain","context":{"idset":"11457","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4620132,"name":"drain","context":{"idset":"11458","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4629591,"name":"drain","context":{"idset":"11459","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4639132,"name":"drain","context":{"idset":"11460","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4648573,"name":"drain","context":{"idset":"11461","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4658141,"name":"drain","context":{"idset":"11462","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.466769,"name":"drain","context":{"idset":"11463","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4677193,"name":"drain","context":{"idset":"11465","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4686761,"name":"drain","context":{"idset":"11467","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4696126,"name":"drain","context":{"idset":"11468","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4705722,"name":"drain","context":{"idset":"11469","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4714065,"name":"drain","context":{"idset":"11470","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4722161,"name":"drain","context":{"idset":"11471","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4729528,"name":"drain","context":{"idset":"11472","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4737167,"name":"drain","context":{"idset":"11473","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4746644,"name":"drain","context":{"idset":"11474","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4756198,"name":"drain","context":{"idset":"11475","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4765544,"name":"drain","context":{"idset":"11476","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4775076,"name":"drain","context":{"idset":"11477","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4784706,"name":"drain","context":{"idset":"11478","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.479455,"name":"drain","context":{"idset":"11479","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4804332,"name":"drain","context":{"idset":"11480","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4813943,"name":"drain","context":{"idset":"11481","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.482362,"name":"drain","context":{"idset":"11482","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4833326,"name":"drain","context":{"idset":"11483","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4842455,"name":"drain","context":{"idset":"11484","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4851637,"name":"drain","context":{"idset":"11485","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4860923,"name":"drain","context":{"idset":"11486","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4870527,"name":"drain","context":{"idset":"11487","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4880021,"name":"drain","context":{"idset":"11488","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4889383,"name":"drain","context":{"idset":"11489","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4896843,"name":"drain","context":{"idset":"11490","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4904377,"name":"drain","context":{"idset":"11491","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4914227,"name":"drain","context":{"idset":"11492","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.492398,"name":"drain","context":{"idset":"11493","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4933691,"name":"drain","context":{"idset":"11494","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4943397,"name":"drain","context":{"idset":"11495","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4953141,"name":"drain","context":{"idset":"11496","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4962792,"name":"drain","context":{"idset":"11497","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4972353,"name":"drain","context":{"idset":"11498","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4982007,"name":"drain","context":{"idset":"11499","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.4991639,"name":"drain","context":{"idset":"11500","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5001326,"name":"drain","context":{"idset":"11501","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5015862,"name":"drain","context":{"idset":"11502","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5025635,"name":"drain","context":{"idset":"11503","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5035203,"name":"drain","context":{"idset":"11504","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5044968,"name":"drain","context":{"idset":"11505","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5053713,"name":"drain","context":{"idset":"11506","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5063107,"name":"drain","context":{"idset":"11507","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5073085,"name":"drain","context":{"idset":"11508","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.508312,"name":"drain","context":{"idset":"11509","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5093341,"name":"drain","context":{"idset":"11510","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5103524,"name":"drain","context":{"idset":"11511","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5113707,"name":"drain","context":{"idset":"11512","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5123765,"name":"drain","context":{"idset":"11513","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5133872,"name":"drain","context":{"idset":"11514","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5144064,"name":"drain","context":{"idset":"11515","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5154202,"name":"drain","context":{"idset":"11516","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5164368,"name":"drain","context":{"idset":"11517","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5174608,"name":"drain","context":{"idset":"11518","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5184526,"name":"drain","context":{"idset":"11519","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5194833,"name":"drain","context":{"idset":"11520","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5205128,"name":"drain","context":{"idset":"11521","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5215275,"name":"drain","context":{"idset":"11523","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5225425,"name":"drain","context":{"idset":"11524","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5235567,"name":"drain","context":{"idset":"11525","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5245631,"name":"drain","context":{"idset":"11526","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5265641,"name":"drain","context":{"idset":"11527","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5284719,"name":"drain","context":{"idset":"11528","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5301688,"name":"drain","context":{"idset":"11529","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5320296,"name":"drain","context":{"idset":"11530","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.533988,"name":"drain","context":{"idset":"11531","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5358996,"name":"drain","context":{"idset":"11532","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5369804,"name":"drain","context":{"idset":"11533","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5379636,"name":"drain","context":{"idset":"11534","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5394933,"name":"drain","context":{"idset":"11535","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5404372,"name":"drain","context":{"idset":"11536","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5414178,"name":"drain","context":{"idset":"11537","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.542459,"name":"drain","context":{"idset":"11538","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5434253,"name":"drain","context":{"idset":"11539","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5446434,"name":"drain","context":{"idset":"11540","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5456192,"name":"drain","context":{"idset":"11541","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5466065,"name":"drain","context":{"idset":"11542","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5476038,"name":"drain","context":{"idset":"11543","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5485737,"name":"drain","context":{"idset":"11544","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5495541,"name":"drain","context":{"idset":"11545","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5505438,"name":"drain","context":{"idset":"11546","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5515203,"name":"drain","context":{"idset":"11547","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5525012,"name":"drain","context":{"idset":"11548","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5535195,"name":"drain","context":{"idset":"11549","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.554534,"name":"drain","context":{"idset":"11550","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5555971,"name":"drain","context":{"idset":"11551","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5566483,"name":"drain","context":{"idset":"11553","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.557689,"name":"drain","context":{"idset":"11554","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5587449,"name":"drain","context":{"idset":"11555","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5598109,"name":"drain","context":{"idset":"11556","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5608745,"name":"drain","context":{"idset":"11557","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.561938,"name":"drain","context":{"idset":"11558","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5629847,"name":"drain","context":{"idset":"11559","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5640345,"name":"drain","context":{"idset":"11560","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5650966,"name":"drain","context":{"idset":"11561","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5661449,"name":"drain","context":{"idset":"11562","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5672135,"name":"drain","context":{"idset":"11564","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.568295,"name":"drain","context":{"idset":"11565","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5693491,"name":"drain","context":{"idset":"11566","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5704165,"name":"drain","context":{"idset":"11567","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5714865,"name":"drain","context":{"idset":"11568","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5725384,"name":"drain","context":{"idset":"11569","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5736067,"name":"drain","context":{"idset":"11570","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5746326,"name":"drain","context":{"idset":"11571","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5757039,"name":"drain","context":{"idset":"11572","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5767765,"name":"drain","context":{"idset":"11573","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5777447,"name":"drain","context":{"idset":"11574","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5787716,"name":"drain","context":{"idset":"11581","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5797558,"name":"drain","context":{"idset":"11589","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5807455,"name":"drain","context":{"idset":"11594","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5816984,"name":"drain","context":{"idset":"11601","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5826383,"name":"drain","context":{"idset":"11616","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5835977,"name":"drain","context":{"idset":"11618","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5846083,"name":"drain","context":{"idset":"11620","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.585525,"name":"drain","context":{"idset":"11624","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5866878,"name":"drain","context":{"idset":"11628","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5872605,"name":"drain","context":{"idset":"11629","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5882607,"name":"drain","context":{"idset":"11631","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.589278,"name":"drain","context":{"idset":"11632","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5902858,"name":"drain","context":{"idset":"11654","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5913074,"name":"drain","context":{"idset":"11657","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5923483,"name":"drain","context":{"idset":"11755","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5933681,"name":"drain","context":{"idset":"11757","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5943902,"name":"drain","context":{"idset":"11758","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5954595,"name":"drain","context":{"idset":"11759","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5964973,"name":"drain","context":{"idset":"11761","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5975161,"name":"drain","context":{"idset":"11762","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5985172,"name":"drain","context":{"idset":"11763","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.5995295,"name":"drain","context":{"idset":"11764","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6054478,"name":"drain","context":{"idset":"11802","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6146882,"name":"drain","context":{"idset":"483","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6159282,"name":"drain","context":{"idset":"484","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6171887,"name":"drain","context":{"idset":"485","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6184542,"name":"drain","context":{"idset":"486","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6196148,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6210134,"name":"drain","context":{"idset":"488","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.622432,"name":"drain","context":{"idset":"489","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6238303,"name":"drain","context":{"idset":"490","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6252255,"name":"drain","context":{"idset":"491","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6266243,"name":"drain","context":{"idset":"492","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6278784,"name":"drain","context":{"idset":"493","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6289988,"name":"drain","context":{"idset":"494","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6301939,"name":"drain","context":{"idset":"495","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.631479,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6327732,"name":"drain","context":{"idset":"497","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6341372,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6354682,"name":"drain","context":{"idset":"499","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6367741,"name":"drain","context":{"idset":"500","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6380944,"name":"drain","context":{"idset":"501","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6394088,"name":"drain","context":{"idset":"502","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6407008,"name":"drain","context":{"idset":"503","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6419485,"name":"drain","context":{"idset":"504","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6431699,"name":"drain","context":{"idset":"505","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.645102,"name":"drain","context":{"idset":"506","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6462781,"name":"drain","context":{"idset":"507","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6474295,"name":"drain","context":{"idset":"508","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6486495,"name":"drain","context":{"idset":"509","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6498075,"name":"drain","context":{"idset":"510","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6510365,"name":"drain","context":{"idset":"511","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.652169,"name":"drain","context":{"idset":"512","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6530814,"name":"drain","context":{"idset":"513","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6539876,"name":"drain","context":{"idset":"514","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.654978,"name":"drain","context":{"idset":"515","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6562407,"name":"drain","context":{"idset":"516","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6570463,"name":"drain","context":{"idset":"517","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6577995,"name":"drain","context":{"idset":"518","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6585124,"name":"drain","context":{"idset":"519","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6592066,"name":"drain","context":{"idset":"520","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6599123,"name":"drain","context":{"idset":"521","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6606555,"name":"drain","context":{"idset":"522","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6613655,"name":"drain","context":{"idset":"523","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6620586,"name":"drain","context":{"idset":"524","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6628017,"name":"drain","context":{"idset":"525","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.664046,"name":"drain","context":{"idset":"526","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6650977,"name":"drain","context":{"idset":"527","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6662331,"name":"drain","context":{"idset":"528","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6674724,"name":"drain","context":{"idset":"529","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6684136,"name":"drain","context":{"idset":"530","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6691208,"name":"drain","context":{"idset":"531","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6698539,"name":"drain","context":{"idset":"532","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6708779,"name":"drain","context":{"idset":"533","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6721766,"name":"drain","context":{"idset":"534","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.673485,"name":"drain","context":{"idset":"535","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6747704,"name":"drain","context":{"idset":"536","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6759999,"name":"drain","context":{"idset":"537","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6773059,"name":"drain","context":{"idset":"538","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.678545,"name":"drain","context":{"idset":"539","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6799171,"name":"drain","context":{"idset":"540","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6811414,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6824076,"name":"drain","context":{"idset":"566","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6838059,"name":"drain","context":{"idset":"567","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6851976,"name":"drain","context":{"idset":"568","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6866074,"name":"drain","context":{"idset":"569","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6882858,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6918766,"name":"drain","context":{"idset":"571","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6930072,"name":"drain","context":{"idset":"572","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.694258,"name":"drain","context":{"idset":"573","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6955202,"name":"drain","context":{"idset":"574","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6967566,"name":"drain","context":{"idset":"575","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6979799,"name":"drain","context":{"idset":"576","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.6990914,"name":"drain","context":{"idset":"577","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7003398,"name":"drain","context":{"idset":"578","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7016132,"name":"drain","context":{"idset":"579","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7028675,"name":"drain","context":{"idset":"580","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7041607,"name":"drain","context":{"idset":"581","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7054741,"name":"drain","context":{"idset":"582","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7068131,"name":"drain","context":{"idset":"583","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7081194,"name":"drain","context":{"idset":"584","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7094848,"name":"drain","context":{"idset":"585","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7108483,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7121921,"name":"drain","context":{"idset":"587","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7135489,"name":"drain","context":{"idset":"588","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7149026,"name":"drain","context":{"idset":"629","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7162554,"name":"drain","context":{"idset":"630","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7176178,"name":"drain","context":{"idset":"631","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7187121,"name":"drain","context":{"idset":"632","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7198381,"name":"drain","context":{"idset":"633","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7210834,"name":"drain","context":{"idset":"634","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7223864,"name":"drain","context":{"idset":"635","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7232435,"name":"drain","context":{"idset":"636","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7240987,"name":"drain","context":{"idset":"827","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7253728,"name":"drain","context":{"idset":"828","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7265873,"name":"drain","context":{"idset":"829","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7277544,"name":"drain","context":{"idset":"830","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7287307,"name":"drain","context":{"idset":"831","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7299066,"name":"drain","context":{"idset":"832","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7311819,"name":"drain","context":{"idset":"833","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7323065,"name":"drain","context":{"idset":"834","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7334619,"name":"drain","context":{"idset":"841","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7346184,"name":"drain","context":{"idset":"842","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7353709,"name":"drain","context":{"idset":"871","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.736171,"name":"drain","context":{"idset":"872","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7371733,"name":"drain","context":{"idset":"949","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.73806,"name":"drain","context":{"idset":"950","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7392089,"name":"drain","context":{"idset":"951","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7404337,"name":"drain","context":{"idset":"952","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7415655,"name":"drain","context":{"idset":"953","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7427738,"name":"drain","context":{"idset":"954","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.743994,"name":"drain","context":{"idset":"963","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7452772,"name":"drain","context":{"idset":"964","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7465694,"name":"drain","context":{"idset":"975","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7479019,"name":"drain","context":{"idset":"976","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7492931,"name":"drain","context":{"idset":"9205","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7506368,"name":"drain","context":{"idset":"9206","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7519641,"name":"drain","context":{"idset":"9207","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7532561,"name":"drain","context":{"idset":"9208","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7545531,"name":"drain","context":{"idset":"9209","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7558532,"name":"drain","context":{"idset":"9338","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7571604,"name":"drain","context":{"idset":"9339","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7584763,"name":"drain","context":{"idset":"9340","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7597511,"name":"drain","context":{"idset":"9341","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7610266,"name":"drain","context":{"idset":"9342","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7622592,"name":"drain","context":{"idset":"9343","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7635112,"name":"drain","context":{"idset":"9344","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7647598,"name":"drain","context":{"idset":"9345","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7661479,"name":"drain","context":{"idset":"9346","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7673874,"name":"drain","context":{"idset":"9347","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7687037,"name":"drain","context":{"idset":"9348","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7701216,"name":"drain","context":{"idset":"9349","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7715483,"name":"drain","context":{"idset":"9350","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.772969,"name":"drain","context":{"idset":"9351","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7744024,"name":"drain","context":{"idset":"9352","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7756157,"name":"drain","context":{"idset":"9353","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7768681,"name":"drain","context":{"idset":"9354","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7781751,"name":"drain","context":{"idset":"9355","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7794275,"name":"drain","context":{"idset":"9356","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7807004,"name":"drain","context":{"idset":"9357","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7819154,"name":"drain","context":{"idset":"9358","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7830265,"name":"drain","context":{"idset":"9359","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7841432,"name":"drain","context":{"idset":"9360","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7854519,"name":"drain","context":{"idset":"9361","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7867577,"name":"drain","context":{"idset":"9362","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7880163,"name":"drain","context":{"idset":"9363","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7900097,"name":"drain","context":{"idset":"9364","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.791317,"name":"drain","context":{"idset":"9365","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7925973,"name":"drain","context":{"idset":"9366","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7938597,"name":"drain","context":{"idset":"9367","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7951596,"name":"drain","context":{"idset":"9368","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7965391,"name":"drain","context":{"idset":"9369","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7978685,"name":"drain","context":{"idset":"9370","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.7992058,"name":"drain","context":{"idset":"9371","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8005903,"name":"drain","context":{"idset":"9372","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8019416,"name":"drain","context":{"idset":"9373","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8033357,"name":"drain","context":{"idset":"9374","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8046565,"name":"drain","context":{"idset":"9375","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.805979,"name":"drain","context":{"idset":"9376","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8072953,"name":"drain","context":{"idset":"9377","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.808635,"name":"drain","context":{"idset":"9378","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8099215,"name":"drain","context":{"idset":"9379","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8112173,"name":"drain","context":{"idset":"9380","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8124888,"name":"drain","context":{"idset":"9381","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8138006,"name":"drain","context":{"idset":"9382","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8150802,"name":"drain","context":{"idset":"9383","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8162837,"name":"drain","context":{"idset":"9384","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8174987,"name":"drain","context":{"idset":"9385","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8187146,"name":"drain","context":{"idset":"9386","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8200555,"name":"drain","context":{"idset":"9387","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8213971,"name":"drain","context":{"idset":"9388","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.822742,"name":"drain","context":{"idset":"9389","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8240573,"name":"drain","context":{"idset":"9390","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8253939,"name":"drain","context":{"idset":"9391","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8267012,"name":"drain","context":{"idset":"9392","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.827997,"name":"drain","context":{"idset":"9393","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8293247,"name":"drain","context":{"idset":"9394","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8306069,"name":"drain","context":{"idset":"9395","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8318851,"name":"drain","context":{"idset":"9396","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.833189,"name":"drain","context":{"idset":"9397","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8345163,"name":"drain","context":{"idset":"9398","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8357699,"name":"drain","context":{"idset":"9399","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8370271,"name":"drain","context":{"idset":"9400","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8383422,"name":"drain","context":{"idset":"9401","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8396027,"name":"drain","context":{"idset":"9402","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8409209,"name":"drain","context":{"idset":"9403","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8422599,"name":"drain","context":{"idset":"9404","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8435977,"name":"drain","context":{"idset":"9405","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8448873,"name":"drain","context":{"idset":"9406","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8461578,"name":"drain","context":{"idset":"9407","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8475049,"name":"drain","context":{"idset":"9408","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8488591,"name":"drain","context":{"idset":"9409","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8501613,"name":"drain","context":{"idset":"9410","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.851475,"name":"drain","context":{"idset":"9411","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8527582,"name":"drain","context":{"idset":"9412","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8539002,"name":"drain","context":{"idset":"9413","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8550642,"name":"drain","context":{"idset":"9414","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8561993,"name":"drain","context":{"idset":"9415","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8573587,"name":"drain","context":{"idset":"9416","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8586502,"name":"drain","context":{"idset":"9417","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8599646,"name":"drain","context":{"idset":"9418","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8612936,"name":"drain","context":{"idset":"9419","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8625422,"name":"drain","context":{"idset":"9420","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8638999,"name":"drain","context":{"idset":"9421","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8653309,"name":"drain","context":{"idset":"9422","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8666701,"name":"drain","context":{"idset":"9423","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8679972,"name":"drain","context":{"idset":"9424","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8692935,"name":"drain","context":{"idset":"9425","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8705821,"name":"drain","context":{"idset":"9426","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8718925,"name":"drain","context":{"idset":"9427","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8731501,"name":"drain","context":{"idset":"9428","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8743992,"name":"drain","context":{"idset":"9429","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8757217,"name":"drain","context":{"idset":"9430","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8769753,"name":"drain","context":{"idset":"9431","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8782859,"name":"drain","context":{"idset":"9432","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8795714,"name":"drain","context":{"idset":"9433","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8809032,"name":"drain","context":{"idset":"9434","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8823123,"name":"drain","context":{"idset":"9435","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8836727,"name":"drain","context":{"idset":"9436","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.88502,"name":"drain","context":{"idset":"9437","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8864274,"name":"drain","context":{"idset":"9438","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.887711,"name":"drain","context":{"idset":"9439","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8890312,"name":"drain","context":{"idset":"9440","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8904033,"name":"drain","context":{"idset":"9441","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8917661,"name":"drain","context":{"idset":"9442","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8931262,"name":"drain","context":{"idset":"9443","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8944912,"name":"drain","context":{"idset":"9444","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8958454,"name":"drain","context":{"idset":"9445","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8971846,"name":"drain","context":{"idset":"9446","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8985612,"name":"drain","context":{"idset":"9449","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.8999102,"name":"drain","context":{"idset":"9450","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9012258,"name":"drain","context":{"idset":"9451","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9025667,"name":"drain","context":{"idset":"9452","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9039218,"name":"drain","context":{"idset":"9453","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9052913,"name":"drain","context":{"idset":"9454","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9065897,"name":"drain","context":{"idset":"9455","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.907954,"name":"drain","context":{"idset":"9456","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9093082,"name":"drain","context":{"idset":"9457","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9106171,"name":"drain","context":{"idset":"9458","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9119177,"name":"drain","context":{"idset":"9459","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9132237,"name":"drain","context":{"idset":"9460","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9145815,"name":"drain","context":{"idset":"9461","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9159203,"name":"drain","context":{"idset":"9462","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9173031,"name":"drain","context":{"idset":"9463","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9186444,"name":"drain","context":{"idset":"9464","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9200182,"name":"drain","context":{"idset":"9465","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.921423,"name":"drain","context":{"idset":"9466","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9227405,"name":"drain","context":{"idset":"9467","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9240398,"name":"drain","context":{"idset":"9468","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9253635,"name":"drain","context":{"idset":"9469","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9266675,"name":"drain","context":{"idset":"9470","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9279585,"name":"drain","context":{"idset":"9471","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9292617,"name":"drain","context":{"idset":"9473","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9306107,"name":"drain","context":{"idset":"9474","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9319048,"name":"drain","context":{"idset":"9475","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9331765,"name":"drain","context":{"idset":"9476","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.934531,"name":"drain","context":{"idset":"9477","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9359145,"name":"drain","context":{"idset":"9478","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9372377,"name":"drain","context":{"idset":"9479","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9386327,"name":"drain","context":{"idset":"9480","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9399829,"name":"drain","context":{"idset":"9481","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9413223,"name":"drain","context":{"idset":"9482","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9426286,"name":"drain","context":{"idset":"9483","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9437947,"name":"drain","context":{"idset":"9484","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9453857,"name":"drain","context":{"idset":"9485","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9466512,"name":"drain","context":{"idset":"9486","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9479125,"name":"drain","context":{"idset":"9487","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9491878,"name":"drain","context":{"idset":"9488","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9505107,"name":"drain","context":{"idset":"9489","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9517739,"name":"drain","context":{"idset":"9490","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9531012,"name":"drain","context":{"idset":"9491","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9544032,"name":"drain","context":{"idset":"9492","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9556775,"name":"drain","context":{"idset":"9493","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9569576,"name":"drain","context":{"idset":"9494","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9583254,"name":"drain","context":{"idset":"9495","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9597213,"name":"drain","context":{"idset":"9496","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9610569,"name":"drain","context":{"idset":"9497","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9623821,"name":"drain","context":{"idset":"9498","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9637091,"name":"drain","context":{"idset":"9499","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9650335,"name":"drain","context":{"idset":"9500","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9664228,"name":"drain","context":{"idset":"9501","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9676697,"name":"drain","context":{"idset":"9502","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.969059,"name":"drain","context":{"idset":"9503","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9704745,"name":"drain","context":{"idset":"9504","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9718606,"name":"drain","context":{"idset":"9505","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9732795,"name":"drain","context":{"idset":"9506","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9746549,"name":"drain","context":{"idset":"9507","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.976032,"name":"drain","context":{"idset":"9508","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9774306,"name":"drain","context":{"idset":"9509","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.978677,"name":"drain","context":{"idset":"9510","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9799979,"name":"drain","context":{"idset":"9511","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9813137,"name":"drain","context":{"idset":"9512","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9826455,"name":"drain","context":{"idset":"9513","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9839659,"name":"drain","context":{"idset":"9514","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9854116,"name":"drain","context":{"idset":"9515","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9868162,"name":"drain","context":{"idset":"9516","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9882317,"name":"drain","context":{"idset":"9517","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9896622,"name":"drain","context":{"idset":"9518","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9911244,"name":"drain","context":{"idset":"9519","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9924536,"name":"drain","context":{"idset":"9520","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.993933,"name":"drain","context":{"idset":"9521","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9954226,"name":"drain","context":{"idset":"9522","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9969645,"name":"drain","context":{"idset":"9523","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9983146,"name":"drain","context":{"idset":"9524","reason":"broker was unresponsive"}}
+{"timestamp":1712586730.9995811,"name":"drain","context":{"idset":"9525","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0008485,"name":"drain","context":{"idset":"9526","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0021081,"name":"drain","context":{"idset":"9527","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.003412,"name":"drain","context":{"idset":"9528","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0045972,"name":"drain","context":{"idset":"9529","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0059278,"name":"drain","context":{"idset":"9530","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0068145,"name":"drain","context":{"idset":"9531","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0080519,"name":"drain","context":{"idset":"9532","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0094049,"name":"drain","context":{"idset":"9536","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0110285,"name":"drain","context":{"idset":"9537","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0123601,"name":"drain","context":{"idset":"9538","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0137167,"name":"drain","context":{"idset":"9539","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0150702,"name":"drain","context":{"idset":"9540","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0164077,"name":"drain","context":{"idset":"9541","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0175955,"name":"drain","context":{"idset":"9542","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0186102,"name":"drain","context":{"idset":"9543","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0195086,"name":"drain","context":{"idset":"9544","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0207624,"name":"drain","context":{"idset":"9545","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0220137,"name":"drain","context":{"idset":"9546","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0230381,"name":"drain","context":{"idset":"9547","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0240905,"name":"drain","context":{"idset":"9548","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0254774,"name":"drain","context":{"idset":"9549","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0268221,"name":"drain","context":{"idset":"9550","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0282035,"name":"drain","context":{"idset":"9551","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0295727,"name":"drain","context":{"idset":"9552","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0309281,"name":"drain","context":{"idset":"9553","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0321844,"name":"drain","context":{"idset":"9554","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.033385,"name":"drain","context":{"idset":"9555","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0345161,"name":"drain","context":{"idset":"9556","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0356946,"name":"drain","context":{"idset":"9557","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0365717,"name":"drain","context":{"idset":"9558","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0379314,"name":"drain","context":{"idset":"9559","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0389864,"name":"drain","context":{"idset":"9560","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0404754,"name":"drain","context":{"idset":"9563","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.041822,"name":"drain","context":{"idset":"9564","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0430522,"name":"drain","context":{"idset":"9565","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.044436,"name":"drain","context":{"idset":"9566","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0458765,"name":"drain","context":{"idset":"9567","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0472441,"name":"drain","context":{"idset":"9568","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0506737,"name":"drain","context":{"idset":"9569","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0520508,"name":"drain","context":{"idset":"9570","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0535779,"name":"drain","context":{"idset":"9571","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0549929,"name":"drain","context":{"idset":"9572","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0564137,"name":"drain","context":{"idset":"9573","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0578239,"name":"drain","context":{"idset":"9574","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.059221,"name":"drain","context":{"idset":"9575","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0606625,"name":"drain","context":{"idset":"9576","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0620582,"name":"drain","context":{"idset":"9577","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0634239,"name":"drain","context":{"idset":"9578","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0647357,"name":"drain","context":{"idset":"9579","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0664151,"name":"drain","context":{"idset":"9580","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0678539,"name":"drain","context":{"idset":"9581","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0693984,"name":"drain","context":{"idset":"9582","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0709171,"name":"drain","context":{"idset":"9583","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0724394,"name":"drain","context":{"idset":"9584","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.073946,"name":"drain","context":{"idset":"9585","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0754597,"name":"drain","context":{"idset":"9586","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0767381,"name":"drain","context":{"idset":"9587","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0780528,"name":"drain","context":{"idset":"9588","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0800631,"name":"drain","context":{"idset":"9589","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0814612,"name":"drain","context":{"idset":"9590","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0827935,"name":"drain","context":{"idset":"9591","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0841148,"name":"drain","context":{"idset":"9592","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0854874,"name":"drain","context":{"idset":"9593","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0868764,"name":"drain","context":{"idset":"9594","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.088392,"name":"drain","context":{"idset":"9595","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0899005,"name":"drain","context":{"idset":"9596","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0912588,"name":"drain","context":{"idset":"9597","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.092736,"name":"drain","context":{"idset":"9598","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.094125,"name":"drain","context":{"idset":"9599","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0955634,"name":"drain","context":{"idset":"9600","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0969145,"name":"drain","context":{"idset":"9601","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0982244,"name":"drain","context":{"idset":"9602","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.0995467,"name":"drain","context":{"idset":"9603","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1009421,"name":"drain","context":{"idset":"9604","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1023443,"name":"drain","context":{"idset":"9605","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1037345,"name":"drain","context":{"idset":"9606","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1050808,"name":"drain","context":{"idset":"9607","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1063883,"name":"drain","context":{"idset":"9608","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1077094,"name":"drain","context":{"idset":"9609","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1090407,"name":"drain","context":{"idset":"9610","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1104028,"name":"drain","context":{"idset":"9611","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1117165,"name":"drain","context":{"idset":"9612","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1130414,"name":"drain","context":{"idset":"9613","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1143489,"name":"drain","context":{"idset":"9614","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1159048,"name":"drain","context":{"idset":"9615","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1172321,"name":"drain","context":{"idset":"9616","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1185851,"name":"drain","context":{"idset":"9617","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1199605,"name":"drain","context":{"idset":"9618","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1213291,"name":"drain","context":{"idset":"9619","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1227307,"name":"drain","context":{"idset":"9620","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1240304,"name":"drain","context":{"idset":"9621","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1255157,"name":"drain","context":{"idset":"9622","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1270933,"name":"drain","context":{"idset":"9623","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1284347,"name":"drain","context":{"idset":"9624","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1299784,"name":"drain","context":{"idset":"9625","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1315482,"name":"drain","context":{"idset":"9626","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1329589,"name":"drain","context":{"idset":"9627","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1343358,"name":"drain","context":{"idset":"9628","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1357288,"name":"drain","context":{"idset":"9629","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1371298,"name":"drain","context":{"idset":"9630","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1386001,"name":"drain","context":{"idset":"9631","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1399465,"name":"drain","context":{"idset":"9632","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1413038,"name":"drain","context":{"idset":"9633","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1426706,"name":"drain","context":{"idset":"9634","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1440675,"name":"drain","context":{"idset":"9635","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1455095,"name":"drain","context":{"idset":"9636","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1469035,"name":"drain","context":{"idset":"9637","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1483045,"name":"drain","context":{"idset":"9638","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1497006,"name":"drain","context":{"idset":"9639","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1511226,"name":"drain","context":{"idset":"9640","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.152554,"name":"drain","context":{"idset":"9641","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.154036,"name":"drain","context":{"idset":"9642","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1554885,"name":"drain","context":{"idset":"9643","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1569386,"name":"drain","context":{"idset":"9644","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.158391,"name":"drain","context":{"idset":"9645","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1598122,"name":"drain","context":{"idset":"9646","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1612196,"name":"drain","context":{"idset":"9647","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1625776,"name":"drain","context":{"idset":"9648","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1639426,"name":"drain","context":{"idset":"9649","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1653149,"name":"drain","context":{"idset":"9650","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1666522,"name":"drain","context":{"idset":"9651","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1680095,"name":"drain","context":{"idset":"9652","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.169399,"name":"drain","context":{"idset":"9653","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1709225,"name":"drain","context":{"idset":"9654","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1724873,"name":"drain","context":{"idset":"9655","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1740427,"name":"drain","context":{"idset":"9656","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1756005,"name":"drain","context":{"idset":"9657","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1771376,"name":"drain","context":{"idset":"9658","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1785624,"name":"drain","context":{"idset":"9659","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.179929,"name":"drain","context":{"idset":"9660","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1810019,"name":"drain","context":{"idset":"9661","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1824293,"name":"drain","context":{"idset":"9662","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1840382,"name":"drain","context":{"idset":"9663","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1856475,"name":"drain","context":{"idset":"9664","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1872303,"name":"drain","context":{"idset":"9665","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1885777,"name":"drain","context":{"idset":"9666","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1899714,"name":"drain","context":{"idset":"9667","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1912115,"name":"drain","context":{"idset":"9668","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1925235,"name":"drain","context":{"idset":"9669","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1939445,"name":"drain","context":{"idset":"9670","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.195401,"name":"drain","context":{"idset":"9671","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1968889,"name":"drain","context":{"idset":"9672","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1983616,"name":"drain","context":{"idset":"9673","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.1998191,"name":"drain","context":{"idset":"9674","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.201297,"name":"drain","context":{"idset":"9675","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2027433,"name":"drain","context":{"idset":"9676","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2041993,"name":"drain","context":{"idset":"9677","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2055821,"name":"drain","context":{"idset":"9678","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2069318,"name":"drain","context":{"idset":"9679","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2083673,"name":"drain","context":{"idset":"9680","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2096829,"name":"drain","context":{"idset":"9681","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2110612,"name":"drain","context":{"idset":"9682","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2124543,"name":"drain","context":{"idset":"9683","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2139235,"name":"drain","context":{"idset":"9684","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.215363,"name":"drain","context":{"idset":"9685","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2167358,"name":"drain","context":{"idset":"9686","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2181995,"name":"drain","context":{"idset":"9687","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2196226,"name":"drain","context":{"idset":"9688","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2210205,"name":"drain","context":{"idset":"9689","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2223382,"name":"drain","context":{"idset":"9690","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2239156,"name":"drain","context":{"idset":"9691","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2254496,"name":"drain","context":{"idset":"9692","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2270193,"name":"drain","context":{"idset":"9693","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2285888,"name":"drain","context":{"idset":"9694","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2301528,"name":"drain","context":{"idset":"9695","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2315118,"name":"drain","context":{"idset":"9696","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.232724,"name":"drain","context":{"idset":"9697","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2341723,"name":"drain","context":{"idset":"9698","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2356479,"name":"drain","context":{"idset":"9699","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2370973,"name":"drain","context":{"idset":"9700","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2384171,"name":"drain","context":{"idset":"9701","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2397342,"name":"drain","context":{"idset":"9702","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2411878,"name":"drain","context":{"idset":"9703","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2425573,"name":"drain","context":{"idset":"9704","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2440014,"name":"drain","context":{"idset":"9705","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2455068,"name":"drain","context":{"idset":"9706","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2468436,"name":"drain","context":{"idset":"9707","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2483521,"name":"drain","context":{"idset":"9708","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2497513,"name":"drain","context":{"idset":"9709","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2513542,"name":"drain","context":{"idset":"9710","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.252732,"name":"drain","context":{"idset":"9711","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2541671,"name":"drain","context":{"idset":"9712","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2555456,"name":"drain","context":{"idset":"9713","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2569635,"name":"drain","context":{"idset":"9714","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2584455,"name":"drain","context":{"idset":"9715","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.259861,"name":"drain","context":{"idset":"9716","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2613938,"name":"drain","context":{"idset":"9717","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2628231,"name":"drain","context":{"idset":"9718","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.264323,"name":"drain","context":{"idset":"9719","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2657416,"name":"drain","context":{"idset":"9720","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2673178,"name":"drain","context":{"idset":"9721","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2688537,"name":"drain","context":{"idset":"9722","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2707977,"name":"drain","context":{"idset":"9723","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2721746,"name":"drain","context":{"idset":"9724","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2736673,"name":"drain","context":{"idset":"9727","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2751961,"name":"drain","context":{"idset":"9728","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2765727,"name":"drain","context":{"idset":"9729","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2779958,"name":"drain","context":{"idset":"9730","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2795026,"name":"drain","context":{"idset":"9731","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2808943,"name":"drain","context":{"idset":"9732","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2823794,"name":"drain","context":{"idset":"9733","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2838845,"name":"drain","context":{"idset":"9734","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2854795,"name":"drain","context":{"idset":"9735","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2869167,"name":"drain","context":{"idset":"9736","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2883151,"name":"drain","context":{"idset":"9737","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2897637,"name":"drain","context":{"idset":"9738","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2912157,"name":"drain","context":{"idset":"9739","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2927086,"name":"drain","context":{"idset":"9740","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2942264,"name":"drain","context":{"idset":"9741","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.2955945,"name":"drain","context":{"idset":"9742","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.297049,"name":"drain","context":{"idset":"9743","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.298492,"name":"drain","context":{"idset":"9744","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.299958,"name":"drain","context":{"idset":"9745","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3013802,"name":"drain","context":{"idset":"9746","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3028605,"name":"drain","context":{"idset":"9747","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3043869,"name":"drain","context":{"idset":"9748","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3058908,"name":"drain","context":{"idset":"9749","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3086255,"name":"drain","context":{"idset":"9751","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3100102,"name":"drain","context":{"idset":"9752","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.311435,"name":"drain","context":{"idset":"9753","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3128843,"name":"drain","context":{"idset":"9754","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3143153,"name":"drain","context":{"idset":"9755","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3157122,"name":"drain","context":{"idset":"9756","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3172483,"name":"drain","context":{"idset":"9757","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3187242,"name":"drain","context":{"idset":"9758","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3201716,"name":"drain","context":{"idset":"9759","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3215575,"name":"drain","context":{"idset":"9760","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3229427,"name":"drain","context":{"idset":"9761","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3244977,"name":"drain","context":{"idset":"9762","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3259728,"name":"drain","context":{"idset":"9763","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3275709,"name":"drain","context":{"idset":"9764","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3290741,"name":"drain","context":{"idset":"9765","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3305688,"name":"drain","context":{"idset":"9766","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3320045,"name":"drain","context":{"idset":"9767","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3334634,"name":"drain","context":{"idset":"9768","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3349509,"name":"drain","context":{"idset":"9769","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3363357,"name":"drain","context":{"idset":"9770","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3377023,"name":"drain","context":{"idset":"9771","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3391356,"name":"drain","context":{"idset":"9772","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3402741,"name":"drain","context":{"idset":"9773","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3415582,"name":"drain","context":{"idset":"9774","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3429182,"name":"drain","context":{"idset":"9775","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3443661,"name":"drain","context":{"idset":"9776","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3458397,"name":"drain","context":{"idset":"9777","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3473687,"name":"drain","context":{"idset":"9778","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3488913,"name":"drain","context":{"idset":"9779","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.350405,"name":"drain","context":{"idset":"9780","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3518555,"name":"drain","context":{"idset":"9781","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3532581,"name":"drain","context":{"idset":"9782","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3547437,"name":"drain","context":{"idset":"9783","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3562531,"name":"drain","context":{"idset":"9784","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3577552,"name":"drain","context":{"idset":"9785","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3592403,"name":"drain","context":{"idset":"9786","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3606677,"name":"drain","context":{"idset":"9788","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3621101,"name":"drain","context":{"idset":"9789","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3636181,"name":"drain","context":{"idset":"9791","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3651929,"name":"drain","context":{"idset":"9792","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3668282,"name":"drain","context":{"idset":"9793","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3684528,"name":"drain","context":{"idset":"9794","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3700638,"name":"drain","context":{"idset":"9795","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3715749,"name":"drain","context":{"idset":"9796","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3730192,"name":"drain","context":{"idset":"9797","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.374373,"name":"drain","context":{"idset":"9798","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3757887,"name":"drain","context":{"idset":"9799","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3772466,"name":"drain","context":{"idset":"9800","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3786511,"name":"drain","context":{"idset":"9801","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3812482,"name":"drain","context":{"idset":"10378","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3824382,"name":"drain","context":{"idset":"11369","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3837466,"name":"drain","context":{"idset":"11575","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3848727,"name":"drain","context":{"idset":"11576","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3861785,"name":"drain","context":{"idset":"11577","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3873477,"name":"drain","context":{"idset":"11578","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3885801,"name":"drain","context":{"idset":"11579","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3898339,"name":"drain","context":{"idset":"11580","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3911273,"name":"drain","context":{"idset":"11582","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3919322,"name":"drain","context":{"idset":"11583","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3929214,"name":"drain","context":{"idset":"11584","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3942013,"name":"drain","context":{"idset":"11585","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3954933,"name":"drain","context":{"idset":"11586","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3967469,"name":"drain","context":{"idset":"11590","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3980088,"name":"drain","context":{"idset":"11591","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.3992879,"name":"drain","context":{"idset":"11592","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.400444,"name":"drain","context":{"idset":"11593","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4016156,"name":"drain","context":{"idset":"11595","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4029176,"name":"drain","context":{"idset":"11596","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4041886,"name":"drain","context":{"idset":"11597","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4054463,"name":"drain","context":{"idset":"11598","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4066567,"name":"drain","context":{"idset":"11599","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4078336,"name":"drain","context":{"idset":"11600","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4091022,"name":"drain","context":{"idset":"11602","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4103758,"name":"drain","context":{"idset":"11603","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4116251,"name":"drain","context":{"idset":"11604","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4129028,"name":"drain","context":{"idset":"11605","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4141951,"name":"drain","context":{"idset":"11606","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4154611,"name":"drain","context":{"idset":"11607","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4167063,"name":"drain","context":{"idset":"11608","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4178975,"name":"drain","context":{"idset":"11609","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4191647,"name":"drain","context":{"idset":"11610","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4204166,"name":"drain","context":{"idset":"11611","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.42167,"name":"drain","context":{"idset":"11612","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4227903,"name":"drain","context":{"idset":"11613","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4237223,"name":"drain","context":{"idset":"11614","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.424948,"name":"drain","context":{"idset":"11615","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4261835,"name":"drain","context":{"idset":"11617","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4269402,"name":"drain","context":{"idset":"11619","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4278378,"name":"drain","context":{"idset":"11621","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.429049,"name":"drain","context":{"idset":"11622","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4301631,"name":"drain","context":{"idset":"11623","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.431427,"name":"drain","context":{"idset":"11625","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.432673,"name":"drain","context":{"idset":"11626","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4339521,"name":"drain","context":{"idset":"11627","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4361567,"name":"drain","context":{"idset":"11630","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4374361,"name":"drain","context":{"idset":"11633","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4385822,"name":"drain","context":{"idset":"11634","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4397025,"name":"drain","context":{"idset":"11635","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4408185,"name":"drain","context":{"idset":"11636","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4420106,"name":"drain","context":{"idset":"11653","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4431634,"name":"drain","context":{"idset":"11655","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4444284,"name":"drain","context":{"idset":"11656","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4455707,"name":"drain","context":{"idset":"11658","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.446718,"name":"drain","context":{"idset":"11659","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4478559,"name":"drain","context":{"idset":"11660","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4489849,"name":"drain","context":{"idset":"11661","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4501173,"name":"drain","context":{"idset":"11662","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4512577,"name":"drain","context":{"idset":"11663","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4525442,"name":"drain","context":{"idset":"11664","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4537938,"name":"drain","context":{"idset":"11665","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4551029,"name":"drain","context":{"idset":"11666","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4562542,"name":"drain","context":{"idset":"11667","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4573169,"name":"drain","context":{"idset":"11668","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4584458,"name":"drain","context":{"idset":"11669","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4591877,"name":"drain","context":{"idset":"11670","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4603968,"name":"drain","context":{"idset":"11671","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4617095,"name":"drain","context":{"idset":"11672","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4630427,"name":"drain","context":{"idset":"11673","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4644046,"name":"drain","context":{"idset":"11674","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4657671,"name":"drain","context":{"idset":"11675","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4670997,"name":"drain","context":{"idset":"11676","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4684491,"name":"drain","context":{"idset":"11677","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4697711,"name":"drain","context":{"idset":"11678","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4710908,"name":"drain","context":{"idset":"11679","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.472429,"name":"drain","context":{"idset":"11680","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4736683,"name":"drain","context":{"idset":"11681","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4749012,"name":"drain","context":{"idset":"11682","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4762011,"name":"drain","context":{"idset":"11684","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4775982,"name":"drain","context":{"idset":"11685","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4790306,"name":"drain","context":{"idset":"11686","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4804721,"name":"drain","context":{"idset":"11687","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4819062,"name":"drain","context":{"idset":"11688","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4835217,"name":"drain","context":{"idset":"11689","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4849577,"name":"drain","context":{"idset":"11690","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4864006,"name":"drain","context":{"idset":"11691","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4878304,"name":"drain","context":{"idset":"11692","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4890785,"name":"drain","context":{"idset":"11693","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4902854,"name":"drain","context":{"idset":"11695","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4914629,"name":"drain","context":{"idset":"11696","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4926374,"name":"drain","context":{"idset":"11697","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4938061,"name":"drain","context":{"idset":"11698","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4949739,"name":"drain","context":{"idset":"11699","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.496141,"name":"drain","context":{"idset":"11700","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4973183,"name":"drain","context":{"idset":"11701","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4984868,"name":"drain","context":{"idset":"11702","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.4996507,"name":"drain","context":{"idset":"11703","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5008116,"name":"drain","context":{"idset":"11704","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5019672,"name":"drain","context":{"idset":"11705","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.503124,"name":"drain","context":{"idset":"11706","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.504293,"name":"drain","context":{"idset":"11707","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5054519,"name":"drain","context":{"idset":"11708","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5065949,"name":"drain","context":{"idset":"11709","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5077891,"name":"drain","context":{"idset":"11710","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5092213,"name":"drain","context":{"idset":"11711","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5105162,"name":"drain","context":{"idset":"11712","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5119863,"name":"drain","context":{"idset":"11713","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5133579,"name":"drain","context":{"idset":"11714","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5146155,"name":"drain","context":{"idset":"11715","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5159168,"name":"drain","context":{"idset":"11716","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5172074,"name":"drain","context":{"idset":"11717","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5185118,"name":"drain","context":{"idset":"11718","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5198126,"name":"drain","context":{"idset":"11719","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5209129,"name":"drain","context":{"idset":"11720","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5221124,"name":"drain","context":{"idset":"11721","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5232999,"name":"drain","context":{"idset":"11722","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5246542,"name":"drain","context":{"idset":"11723","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5258479,"name":"drain","context":{"idset":"11724","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5271235,"name":"drain","context":{"idset":"11725","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5285861,"name":"drain","context":{"idset":"11726","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5299277,"name":"drain","context":{"idset":"11727","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5312006,"name":"drain","context":{"idset":"11728","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5320883,"name":"drain","context":{"idset":"11729","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5328805,"name":"drain","context":{"idset":"11730","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5336502,"name":"drain","context":{"idset":"11731","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5344319,"name":"drain","context":{"idset":"11732","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.53567,"name":"drain","context":{"idset":"11733","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5368178,"name":"drain","context":{"idset":"11734","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5381246,"name":"drain","context":{"idset":"11735","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5391765,"name":"drain","context":{"idset":"11736","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5402188,"name":"drain","context":{"idset":"11737","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5415945,"name":"drain","context":{"idset":"11738","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5429542,"name":"drain","context":{"idset":"11739","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5442996,"name":"drain","context":{"idset":"11740","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5456009,"name":"drain","context":{"idset":"11741","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5466692,"name":"drain","context":{"idset":"11742","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5480316,"name":"drain","context":{"idset":"11743","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5493815,"name":"drain","context":{"idset":"11744","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5504146,"name":"drain","context":{"idset":"11745","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5516157,"name":"drain","context":{"idset":"11746","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5529034,"name":"drain","context":{"idset":"11747","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5541804,"name":"drain","context":{"idset":"11748","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5556314,"name":"drain","context":{"idset":"11749","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5571163,"name":"drain","context":{"idset":"11750","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5586212,"name":"drain","context":{"idset":"11751","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5601068,"name":"drain","context":{"idset":"11752","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5616019,"name":"drain","context":{"idset":"11753","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5630894,"name":"drain","context":{"idset":"11754","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5645778,"name":"drain","context":{"idset":"11756","reason":"broker was unresponsive"}}
+{"timestamp":1712586731.5659981,"name":"drain","context":{"idset":"11760","reason":"broker was unresponsive"}}
+{"timestamp":1712586938.2263329,"name":"offline","context":{"idset":"11112"}}
+{"timestamp":1712586944.2269521,"name":"offline","context":{"idset":"11111"}}
+{"timestamp":1712587810.0439785,"name":"online","context":{"idset":"9725"}}
+{"timestamp":1712588131.7204521,"name":"online","context":{"idset":"9726"}}
+{"timestamp":1712590333.48072,"name":"undrain","context":{"idset":"1-60,85-120,122-216,218-347,349-430,432-444,469-540,565-588,629-636,827-834,841-842,871-872,949-954,963-964,975-976,9205-9213,9215-9238,9241-9315,9317,9319-9446,9449-9471,9473-9532,9536-9560,9563-9724,9727-9749,9751-9786,9788-9789,9791-9801,9803-9844,9973-10137,10139-10142,10144,10146-10154,10157-10260,10263-10302,10304-10342,10345-10354,10357-10395,10397-10468,10485-10500,10502-10516,10519-10628,10630-10633,10635-10636,10640-10641,10645-10656,10658-10692,10695-10701,10703-10711,10713-10717,10719-10738,10741-10818,10859-10870,10873-10880,10953-10954,10956-10959,10961-10963,11016-11020,11022-11110,11113-11114,11117-11124,11253-11308,11443-11452,11505-11521,11523-11551,11553-11562,11564-11586,11589-11636,11653-11657,11678-11682,11684-11687,11699-11708,11739-11748,11759-11764,11802"}}
+{"timestamp":1712590345.2272494,"name":"undrain","context":{"idset":"10819-10858,10881-10910,10913-10919,10921-10924,10926-10929,10931-10934,10936-10939,10941-10942,10946-10949,10951,10964,10966-10969,10971-10974,10976-10979,10981-10984,10986-10989,10991-10992,10997-10998,11001-11015,11309-11328,11331-11357,11359-11372,11375-11386,11389,11391-11394,11396-11399,11430,11432-11438,11453-11463,11465,11467-11504,11658-11677,11688-11693,11695-11698,11709-11738,11749-11758"}}
+{"timestamp":1712590359.7660367,"name":"undrain","context":{"idset":"9472,9787,9790,10262,10396,10501,10952,11111-11112,11125-11133,11135-11139,11141-11225,11227-11232,11234,11237-11246,11248,11250-11252,11877-11883,11885-11890"}}
+{"timestamp":1712590448.9743986,"name":"undrain","context":{"idset":"69-76"}}
+{"timestamp":1712590463.0333052,"name":"undrain","context":{"idset":"445-468,541-564"}}
+{"timestamp":1712590474.0578129,"name":"undrain","context":{"idset":"597-604,613-620"}}
+{"timestamp":1712590485.1451368,"name":"undrain","context":{"idset":"621-628"}}
+{"timestamp":1712590496.2881594,"name":"undrain","context":{"idset":"646-652"}}
+{"timestamp":1712590507.0003569,"name":"undrain","context":{"idset":"686,690,694,706,708"}}
+{"timestamp":1712590517.9689429,"name":"undrain","context":{"idset":"709-716"}}
+{"timestamp":1712590526.8316596,"name":"online","context":{"idset":"11522"}}
+{"timestamp":1712590536.8635631,"name":"online","context":{"idset":"10693"}}
+{"timestamp":1712590538.8181703,"name":"online","context":{"idset":"10694"}}
+{"timestamp":1712590769.0396078,"name":"undrain","context":{"idset":"11522"}}
+{"timestamp":1712590844.8453834,"name":"online","context":{"idset":"11588"}}
+{"timestamp":1712590844.8472097,"name":"online","context":{"idset":"11587"}}
+{"timestamp":1712590845.1587965,"name":"online","context":{"idset":"11552"}}
+{"timestamp":1712590847.0792916,"name":"online","context":{"idset":"11563"}}
+{"timestamp":1712591276.0872543,"name":"undrain","context":{"idset":"11552,11563,11587-11588"}}
+{"timestamp":1712592163.4425364,"name":"online","context":{"idset":"9239"}}
+{"timestamp":1712592332.8953078,"name":"online","context":{"idset":"9240"}}
+{"timestamp":1712592515.8810503,"name":"drain","context":{"idset":"10952","overwrite":0}}
+{"timestamp":1712594249.0262253,"name":"drain","context":{"idset":"10869-10996","overwrite":0}}
+{"timestamp":1712596642.1290939,"name":"drain","context":{"idset":"11117","reason":"broker was unresponsive"}}
+{"timestamp":1712596642.9024158,"name":"drain","context":{"idset":"11118","reason":"broker was unresponsive"}}
+{"timestamp":1712596708.1332099,"name":"offline","context":{"idset":"11117"}}
+{"timestamp":1712596709.0182195,"name":"offline","context":{"idset":"11118"}}
+{"timestamp":1712597522.1291575,"name":"drain","context":{"idset":"11381","reason":"broker was unresponsive"}}
+{"timestamp":1712597522.9041014,"name":"drain","context":{"idset":"11382","reason":"broker was unresponsive"}}
+{"timestamp":1712597528.1294539,"name":"drain","context":{"idset":"11383","reason":"broker was unresponsive"}}
+{"timestamp":1712597528.1296108,"name":"drain","context":{"idset":"11384","reason":"broker was unresponsive"}}
+{"timestamp":1712597528.1297331,"name":"drain","context":{"idset":"11385","reason":"broker was unresponsive"}}
+{"timestamp":1712597528.1298556,"name":"drain","context":{"idset":"11386","reason":"broker was unresponsive"}}
+{"timestamp":1712597528.1299732,"name":"drain","context":{"idset":"11389","reason":"broker was unresponsive"}}
+{"timestamp":1712597528.1300926,"name":"drain","context":{"idset":"11391","reason":"broker was unresponsive"}}
+{"timestamp":1712597528.1302114,"name":"drain","context":{"idset":"11392","reason":"broker was unresponsive"}}
+{"timestamp":1712597528.1303468,"name":"drain","context":{"idset":"11393","reason":"broker was unresponsive"}}
+{"timestamp":1712597528.1304727,"name":"drain","context":{"idset":"11394","reason":"broker was unresponsive"}}
+{"timestamp":1712597528.9531279,"name":"drain","context":{"idset":"11396","reason":"broker was unresponsive"}}
+{"timestamp":1712597588.1322308,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1712597588.8655243,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1712597588.9253771,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1712597588.9279716,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1712597588.9371529,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1712597589.108212,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1712597592.1393442,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1712597592.8615355,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1712597592.9332585,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1712597592.9411175,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1712597592.9580355,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1712597593.0061321,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1712598484.8393898,"name":"drain","context":{"idset":"10658","reason":"broker was unresponsive"}}
+{"timestamp":1712598548.8970084,"name":"offline","context":{"idset":"10658"}}
+{"timestamp":1712599496.8345466,"name":"online","context":{"idset":"11392"}}
+{"timestamp":1712599496.8375757,"name":"online","context":{"idset":"11383,11385"}}
+{"timestamp":1712599496.8730733,"name":"online","context":{"idset":"11388,11390-11391"}}
+{"timestamp":1712599497.0848498,"name":"online","context":{"idset":"11382,11384,11393-11396"}}
+{"timestamp":1712599497.2233834,"name":"online","context":{"idset":"11386-11387,11389"}}
+{"timestamp":1712599497.4517977,"name":"online","context":{"idset":"11381"}}
+{"timestamp":1712599525.6223402,"name":"online","context":{"idset":"9214"}}
+{"timestamp":1712599699.0118685,"name":"undrain","context":{"idset":"9214"}}
+{"timestamp":1712601895.9786191,"name":"online","context":{"idset":"11329"}}
+{"timestamp":1712602024.9359465,"name":"online","context":{"idset":"11330"}}
+{"timestamp":1712602153.7117822,"name":"drain","context":{"idset":"11317-11332","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712602439.6739988,"name":"online","context":{"idset":"11429"}}
+{"timestamp":1712602441.0330741,"name":"online","context":{"idset":"11439"}}
+{"timestamp":1712602441.1381528,"name":"online","context":{"idset":"11441"}}
+{"timestamp":1712602441.7814064,"name":"online","context":{"idset":"11442"}}
+{"timestamp":1712602454.8971624,"name":"online","context":{"idset":"11466"}}
+{"timestamp":1712603015.940799,"name":"online","context":{"idset":"9318"}}
+{"timestamp":1712603456.1319213,"name":"drain","context":{"idset":"11802","reason":"broker was unresponsive"}}
+{"timestamp":1712603514.918541,"name":"offline","context":{"idset":"11797"}}
+{"timestamp":1712603516.1494477,"name":"offline","context":{"idset":"11798"}}
+{"timestamp":1712603516.1681037,"name":"offline","context":{"idset":"11799"}}
+{"timestamp":1712603516.1853032,"name":"offline","context":{"idset":"11800"}}
+{"timestamp":1712603517.0280423,"name":"offline","context":{"idset":"11801"}}
+{"timestamp":1712603517.0306258,"name":"offline","context":{"idset":"11802"}}
+{"timestamp":1712603517.0326178,"name":"offline","context":{"idset":"11803"}}
+{"timestamp":1712603518.1398795,"name":"offline","context":{"idset":"11804"}}
+{"timestamp":1712603518.1541414,"name":"offline","context":{"idset":"11805"}}
+{"timestamp":1712603518.9715469,"name":"offline","context":{"idset":"11806"}}
+{"timestamp":1712603519.0358922,"name":"offline","context":{"idset":"11808"}}
+{"timestamp":1712603519.0668757,"name":"offline","context":{"idset":"11809"}}
+{"timestamp":1712603519.1190705,"name":"offline","context":{"idset":"11810"}}
+{"timestamp":1712603522.1345639,"name":"offline","context":{"idset":"11811"}}
+{"timestamp":1712603523.0079908,"name":"offline","context":{"idset":"11812"}}
+{"timestamp":1712603913.9030306,"name":"drain","context":{"idset":"11637-11652","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712603935.0562558,"name":"drain","context":{"idset":"11653-11668","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712603975.9504275,"name":"drain","context":{"idset":"11685-11700","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712603997.0013914,"name":"drain","context":{"idset":"11701-11716","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712604017.4526958,"name":"drain","context":{"idset":"11717-11732","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712604037.9048123,"name":"drain","context":{"idset":"11733-11748","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712604058.9765675,"name":"drain","context":{"idset":"11749-11764","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712604151.0310593,"name":"drain","context":{"idset":"11669-11682,11684","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712604382.8392358,"name":"online","context":{"idset":"11772"}}
+{"timestamp":1712604382.8420825,"name":"online","context":{"idset":"11776"}}
+{"timestamp":1712604382.8447542,"name":"online","context":{"idset":"11765-11766,11769"}}
+{"timestamp":1712604382.8472469,"name":"online","context":{"idset":"11771"}}
+{"timestamp":1712604382.9528461,"name":"online","context":{"idset":"11774,11777,11779"}}
+{"timestamp":1712604383.0828912,"name":"online","context":{"idset":"11770,11778,11780"}}
+{"timestamp":1712604383.1999009,"name":"online","context":{"idset":"11767,11773,11775"}}
+{"timestamp":1712604383.4063001,"name":"online","context":{"idset":"11768"}}
+{"timestamp":1712604582.9783416,"name":"online","context":{"idset":"11784"}}
+{"timestamp":1712604583.6160045,"name":"online","context":{"idset":"11785"}}
+{"timestamp":1712604583.8487937,"name":"online","context":{"idset":"11781,11786,11791-11792,11794-11795"}}
+{"timestamp":1712604583.9631519,"name":"online","context":{"idset":"11789-11790"}}
+{"timestamp":1712604584.0742335,"name":"online","context":{"idset":"11782,11793,11796"}}
+{"timestamp":1712604584.1298776,"name":"online","context":{"idset":"11788"}}
+{"timestamp":1712604584.8507373,"name":"online","context":{"idset":"11787"}}
+{"timestamp":1712604584.8552217,"name":"online","context":{"idset":"11783"}}
+{"timestamp":1712604702.8860257,"name":"online","context":{"idset":"11440"}}
+{"timestamp":1712605835.5471108,"name":"offline","context":{"idset":"9315"}}
+{"timestamp":1712606139.4587028,"name":"online","context":{"idset":"11823,11828"}}
+{"timestamp":1712606139.683445,"name":"online","context":{"idset":"11822,11826"}}
+{"timestamp":1712606139.887177,"name":"online","context":{"idset":"11814,11825"}}
+{"timestamp":1712606140.0897059,"name":"online","context":{"idset":"11819,11827"}}
+{"timestamp":1712606140.2292838,"name":"online","context":{"idset":"11816,11818,11821"}}
+{"timestamp":1712606140.4257035,"name":"online","context":{"idset":"11813,11817,11820,11824"}}
+{"timestamp":1712606140.6343143,"name":"online","context":{"idset":"11815"}}
+{"timestamp":1712606186.1312208,"name":"drain","context":{"idset":"9569","reason":"broker was unresponsive"}}
+{"timestamp":1712606186.2319283,"name":"drain","context":{"idset":"9570","reason":"broker was unresponsive"}}
+{"timestamp":1712606248.1357067,"name":"offline","context":{"idset":"9569"}}
+{"timestamp":1712606248.2279842,"name":"offline","context":{"idset":"9570"}}
+{"timestamp":1712607406.5064204,"name":"online","context":{"idset":"11116"}}
+{"timestamp":1712608124.5300698,"name":"undrain","context":{"idset":"9318"}}
+{"timestamp":1712608487.6653569,"name":"undrain","context":{"idset":"11637-11764"}}
+{"timestamp":1712608723.4447157,"name":"online","context":{"idset":"820"}}
+{"timestamp":1712609309.229135,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1712609369.1195641,"name":"online","context":{"idset":"11512"}}
+{"timestamp":1712609675.0720088,"name":"online","context":{"idset":"876"}}
+{"timestamp":1712611037.0315862,"name":"offline","context":{"idset":"10560"}}
+{"timestamp":1712611037.0543501,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1712611037.0739963,"name":"offline","context":{"idset":"10081"}}
+{"timestamp":1712611037.0934422,"name":"offline","context":{"idset":"10692"}}
+{"timestamp":1712611037.1197963,"name":"offline","context":{"idset":"10539"}}
+{"timestamp":1712611037.1232469,"name":"offline","context":{"idset":"10007"}}
+{"timestamp":1712611037.1499338,"name":"offline","context":{"idset":"9488"}}
+{"timestamp":1712611037.1741271,"name":"offline","context":{"idset":"9630"}}
+{"timestamp":1712611041.4441493,"name":"offline","context":{"idset":"10070"}}
+{"timestamp":1712611041.4468899,"name":"offline","context":{"idset":"10289"}}
+{"timestamp":1712611041.4597909,"name":"offline","context":{"idset":"10659"}}
+{"timestamp":1712611041.4626138,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1712611041.4984004,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1712611041.5173554,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1712611041.532927,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1712611041.5355606,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1712611041.551753,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1712611041.5744238,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1712611041.6246619,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1712611041.6386266,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1712611041.6414285,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1712611041.6455252,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1712611041.6484532,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1712611041.6622939,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1712611041.6790524,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1712611041.7003155,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1712611041.7251627,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1712611041.772227,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1712611041.7755117,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1712611041.7815287,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1712611041.7971618,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1712611041.8002081,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1712611041.8144653,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1712611041.8315644,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1712611041.8501251,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1712611041.8834207,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1712611041.9177463,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1712611041.9550281,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1712611041.9982376,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1712611042.0011535,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1712611042.0039854,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1712611042.0071585,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1712611042.0182133,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1712611042.0210025,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1712611042.0532568,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1712611042.0742354,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1712611042.1084156,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1712611042.1455994,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1712611042.1860178,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1712611042.2104018,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1712611042.2132468,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1712611042.2154865,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1712611042.218163,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1712611042.220789,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1712611042.2230797,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1712611042.2450562,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1712611042.2608194,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1712611042.2754107,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1712611042.3027642,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1712611042.352386,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1712611042.401314,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1712611042.4163473,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1712611042.4193029,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1712611042.4225278,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1712611042.4257209,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1712611042.4290123,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1712611042.432163,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1712611042.4356043,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1712611042.4390984,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1712611042.4531977,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1712611042.5002406,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1712611042.5475941,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1712611042.5824668,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1712611042.5856357,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1712611042.5884225,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1712611042.5915976,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1712611042.5947423,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1712611042.5978439,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1712611042.6001203,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1712611042.6120517,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1712611042.643719,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1712611042.6898704,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1712611042.7357166,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1712611042.7496696,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1712611042.7523685,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1712611042.755085,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1712611042.7577605,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1712611042.7604871,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1712611042.7631993,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1712611042.7770886,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1712611042.8217077,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1712611042.8651607,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1712611042.887816,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1712611042.8905189,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1712611042.8933351,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1712611042.8961375,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1712611042.8989623,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1712611042.9018066,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1712611042.9046118,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1712611042.9071898,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1712611042.9520652,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1712611042.9963789,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1712611043.0208633,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1712611043.0235488,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1712611043.0262163,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1712611043.0292239,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1712611043.032124,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1712611043.0350428,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1712611043.0379705,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1712611043.0468113,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1712611043.0496709,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1712611043.1033502,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1712611043.1470821,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1712611043.1595743,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1712611043.1622376,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1712611043.1648881,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1712611043.1675689,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1712611043.170382,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1712611043.1734931,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1712611043.1861405,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1712611043.2024212,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1712611043.236547,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1712611043.2826834,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1712611043.3184714,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1712611043.3214943,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1712611043.3244863,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1712611043.3275819,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1712611043.331687,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1712611043.3345072,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1712611043.337327,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1712611043.3511169,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1712611043.3671405,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1712611043.3825502,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1712611043.4274862,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1712611043.4708028,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1712611043.5043063,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1712611043.5069511,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1712611043.5096011,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1712611043.5122168,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1712611043.5148108,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1712611043.5175209,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1712611043.520246,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1712611043.5365093,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1712611043.5828593,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1712611043.6275156,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1712611043.6475055,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1712611043.6502862,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1712611043.6529779,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1712611043.65573,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1712611043.6585193,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1712611043.6611903,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1712611043.6639452,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1712611043.6666737,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1712611043.6758862,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1712611043.7180808,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1712611043.7618978,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1712611043.8109853,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1712611043.8139429,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1712611043.8167884,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1712611043.8197496,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1712611043.8228469,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1712611043.8259964,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1712611043.8292291,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1712611043.8322062,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1712611043.8458657,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1712611043.8919706,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1712611043.9376655,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1712611043.961551,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1712611043.9641533,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1712611043.9669425,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1712611043.969615,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1712611043.9718535,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1712611043.9743505,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1712611043.9770555,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1712611043.9795964,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1712611044.0261722,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1712611044.0701399,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1712611044.0934041,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1712611044.0958412,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1712611044.0982966,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1712611044.1010408,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1712611044.1037679,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1712611044.1065581,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1712611044.1098051,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1712611044.1127348,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1712611044.1228356,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1712611044.1379759,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1712611044.1613452,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1712611044.2059624,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1712611044.2388659,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1712611044.2467182,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1712611044.2484243,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1712611044.250015,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1712611044.2516165,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1712611044.2532568,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1712611044.2549205,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1712611044.2565863,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1712611044.2582228,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1712611044.2723444,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1712611044.2950201,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1712611044.296627,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1712611044.3339741,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1712611044.3591695,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1712611044.378377,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1712611044.3799067,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1712611044.3814111,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1712611044.382921,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1712611044.384444,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1712611044.3859532,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1712611044.3874717,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1712611044.3889892,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1712611044.3960102,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1712611044.4228127,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1712611044.4480097,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1712611044.4736106,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1712611044.4872255,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1712611044.4889448,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1712611044.4905066,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1712611044.4921062,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1712611044.4937038,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1712611044.4953604,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1712611044.4969835,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1712611044.4985065,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1712611044.5176511,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1712611044.5424924,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1712611044.5673614,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1712611044.5805323,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1712611044.5820272,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1712611044.5835342,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1712611044.5850134,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1712611044.5865152,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1712611044.588002,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1712611044.5953069,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1712611044.6202042,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1712611044.6453636,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1712611044.6588192,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1712611044.6603658,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1712611044.661881,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1712611044.6633859,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1712611044.6649728,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1712611044.6664777,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1712611044.6679649,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1712611044.6694741,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1712611044.6945684,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1712611044.7195492,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1712611044.7385905,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1712611044.7400787,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1712611044.7415938,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1712611044.7430649,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1712611044.7445374,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1712611044.746012,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1712611044.7474947,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1712611044.7492609,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1712611044.7682517,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1712611044.7932136,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1712611044.8060791,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1712611044.8075476,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1712611044.8090019,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1712611044.8104882,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1712611044.8119321,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1712611044.8133848,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1712611044.8148232,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1712611044.816298,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1712611044.8177702,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1712611044.8231306,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1712611044.8492036,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1712611044.8736069,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1712611044.8865654,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1712611044.8880203,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1712611044.8895073,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1712611044.8909547,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1712611044.8924215,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1712611044.8939042,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1712611044.8953867,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1712611044.9017143,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1712611044.910388,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1712611044.9190917,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1712611044.9278178,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1712611044.9521544,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1712611044.9764793,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1712611044.9950242,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1712611044.9965124,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1712611044.9979506,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1712611044.9994061,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1712611045.0008528,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1712611045.002322,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1712611045.003773,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1712611045.0070391,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1712611045.0156887,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1712611045.0244541,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1712611045.0500789,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1712611045.0743625,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1712611045.0990214,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1712611045.1062491,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1712611045.1076765,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1712611045.109092,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1712611045.1106634,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1712611045.1197097,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1712611045.1340449,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1712611045.14803,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1712611045.1652029,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1712611045.1931751,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1712611045.2184036,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1712611045.2425439,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1712611045.2665226,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1712611045.2848592,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1712611045.2863798,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1712611045.2878902,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1712611045.2894137,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1712611045.2908776,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1712611045.3045971,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1712611045.3214295,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1712611045.3229413,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1712611045.3495941,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1712611045.3751771,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1712611045.3996699,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1712611045.4236948,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1712611045.4364803,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1712611045.4379022,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1712611045.4393318,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1712611045.4407258,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1712611045.4471319,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1712611045.4485807,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1712611045.4650617,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1712611045.4793527,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1712611045.493747,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1712611045.5079336,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1712611045.5418553,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1712611045.5663061,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1712611045.5905845,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1712611045.6097517,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1712611045.6111875,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1712611045.6127546,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1712611045.6141908,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1712611045.6211472,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1712611045.622752,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1712611045.6349311,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1712611045.6488898,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1712611045.6503286,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1712611045.6774457,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1712611045.7034626,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1712611045.7287123,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1712611045.7544999,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1712611045.7560012,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1712611045.7575598,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1712611045.7591434,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1712611045.7609096,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1712611045.7744806,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1712611045.7759888,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1712611045.7881825,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1712611045.8018873,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1712611045.8154254,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1712611045.836015,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1712611045.8624849,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1712611045.888736,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1712611045.9150007,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1712611045.9165304,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1712611045.9180565,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1712611045.9273324,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1712611045.9417114,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1712611045.9433355,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1712611045.9569407,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1712611045.9716523,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1712611045.9732454,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1712611045.9865799,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1712611046.0006342,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1712611046.0277982,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1712611046.0558567,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1712611046.084029,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1712611046.0988717,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1712611046.1005554,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1712611046.10217,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1712611046.1146922,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1712611046.1163692,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1712611046.1292777,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1712611046.1446984,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1712611046.1473017,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1712611046.1613843,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1712611046.1754596,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1712611046.1901672,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1712611046.2037201,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1712611046.2481427,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1712611046.2809048,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1712611046.313472,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1712611046.323102,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1712611046.325012,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1712611046.3269017,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1712611046.3287764,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1712611046.3306584,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1712611046.3325438,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1712611046.3344526,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1712611046.3425674,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1712611046.3444829,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1712611046.370991,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1712611046.38466,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1712611046.4068797,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1712611046.4460812,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1712611046.4829187,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1712611046.5198472,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1712611046.5308211,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1712611046.532949,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1712611046.5351496,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1712611046.5373011,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1712611046.5394204,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1712611046.5415335,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1712611046.5436625,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1712611046.5458117,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1712611046.5479434,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1712611046.5614781,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1712611046.5754237,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1712611046.5979664,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1712611046.6370075,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1712611046.6820931,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1712611046.7283337,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1712611046.7425346,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1712611046.7453165,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1712611046.74804,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1712611046.7507267,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1712611046.7535157,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1712611046.7563412,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1712611046.7592072,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1712611046.7627604,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1712611046.7655957,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1712611046.7759349,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1712611046.7898436,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1712611046.8456364,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1712611046.8967803,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1712611046.935823,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1712611046.9567323,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1712611046.9591975,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1712611046.9614201,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1712611046.963712,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1712611046.9659712,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1712611046.9681702,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1712611046.9708347,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1712611046.9730167,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1712611046.9751852,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1712611046.9773734,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1712611046.9803336,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1712611047.0036581,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1712611047.0435452,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1712611047.0826437,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1712611047.1211746,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1712611047.1412761,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1712611047.1433918,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1712611047.1455238,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1712611047.1476624,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1712611047.1497967,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1712611047.1518924,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1712611047.1540046,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1712611047.156095,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1712611047.1582851,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1712611047.1604352,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1712611047.1887541,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1712611047.2251594,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1712611047.2608986,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1712611047.2876186,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1712611047.2895525,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1712611047.2914946,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1712611047.2934513,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1712611047.2954061,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1712611047.2973313,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1712611047.2992439,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1712611047.3011515,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1712611047.3030827,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1712611047.3234165,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1712611047.3611073,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1712611047.3951206,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1712611047.4284108,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1712611047.437999,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1712611047.4397902,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1712611047.4416454,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1712611047.4434576,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1712611047.4453778,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1712611047.4471536,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1712611047.4489472,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1712611047.4507222,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1712611047.4745419,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1712611047.4949446,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1712611047.5276928,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1712611047.5599999,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1712611047.5912566,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1712611047.6002972,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1712611047.6020172,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1712611047.6037121,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1712611047.6054168,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1712611047.6071005,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1712611047.6087894,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1712611047.610477,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1712611047.6121681,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1712611047.6138859,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1712611047.618377,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1712611047.6497335,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1712611047.680337,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1712611047.7101879,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1712611047.7187059,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1712611047.7202995,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1712611047.7222114,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1712611047.7238622,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1712611047.725472,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1712611047.7353308,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1712611047.7373223,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1712611047.7500219,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1712611047.7649505,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1712611047.7929025,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1712611047.8229976,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1712611047.8526597,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1712611047.8679142,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1712611047.8701656,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1712611047.8719561,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1712611047.8735151,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1712611047.8750665,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1712611047.877954,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1712611047.8916757,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1712611047.8932168,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1712611047.9205301,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1712611047.9345334,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1712611047.9361105,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1712611047.9628081,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1712611048.0026405,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1712611048.0423801,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1712611048.0725477,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1712611048.0746586,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1712611048.0767722,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1712611048.0788667,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1712611048.0809669,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1712611048.0830712,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1712611048.0851634,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1712611048.0964782,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1712611048.0985758,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1712611048.1280987,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1712611048.1665111,"name":"offline","context":{"idset":"820"}}
+{"timestamp":1712611048.2052891,"name":"offline","context":{"idset":"827"}}
+{"timestamp":1712611048.2355051,"name":"offline","context":{"idset":"828"}}
+{"timestamp":1712611048.2376058,"name":"offline","context":{"idset":"829"}}
+{"timestamp":1712611048.2397287,"name":"offline","context":{"idset":"830"}}
+{"timestamp":1712611048.2419043,"name":"offline","context":{"idset":"831"}}
+{"timestamp":1712611048.2440627,"name":"offline","context":{"idset":"832"}}
+{"timestamp":1712611048.2462313,"name":"offline","context":{"idset":"833"}}
+{"timestamp":1712611048.2483923,"name":"offline","context":{"idset":"834"}}
+{"timestamp":1712611048.2504997,"name":"offline","context":{"idset":"841"}}
+{"timestamp":1712611048.2526495,"name":"offline","context":{"idset":"842"}}
+{"timestamp":1712611048.2547643,"name":"offline","context":{"idset":"871"}}
+{"timestamp":1712611048.2668359,"name":"offline","context":{"idset":"872"}}
+{"timestamp":1712611048.3066506,"name":"offline","context":{"idset":"876"}}
+{"timestamp":1712611048.3458476,"name":"offline","context":{"idset":"949"}}
+{"timestamp":1712611048.3853693,"name":"offline","context":{"idset":"950"}}
+{"timestamp":1712611048.3874967,"name":"offline","context":{"idset":"951"}}
+{"timestamp":1712611048.389596,"name":"offline","context":{"idset":"952"}}
+{"timestamp":1712611048.3917577,"name":"offline","context":{"idset":"953"}}
+{"timestamp":1712611048.3939574,"name":"offline","context":{"idset":"954"}}
+{"timestamp":1712611048.396054,"name":"offline","context":{"idset":"963"}}
+{"timestamp":1712611048.3982413,"name":"offline","context":{"idset":"964"}}
+{"timestamp":1712611048.40042,"name":"offline","context":{"idset":"975"}}
+{"timestamp":1712611048.4025192,"name":"offline","context":{"idset":"976"}}
+{"timestamp":1712611048.4132817,"name":"offline","context":{"idset":"9205"}}
+{"timestamp":1712611048.4516926,"name":"offline","context":{"idset":"9206"}}
+{"timestamp":1712611048.4880972,"name":"offline","context":{"idset":"9207"}}
+{"timestamp":1712611048.5160289,"name":"offline","context":{"idset":"9208"}}
+{"timestamp":1712611048.5182486,"name":"offline","context":{"idset":"9209"}}
+{"timestamp":1712611048.5203421,"name":"offline","context":{"idset":"9210"}}
+{"timestamp":1712611048.5225158,"name":"offline","context":{"idset":"9211"}}
+{"timestamp":1712611048.5246322,"name":"offline","context":{"idset":"9212"}}
+{"timestamp":1712611048.526758,"name":"offline","context":{"idset":"9213"}}
+{"timestamp":1712611048.5288975,"name":"offline","context":{"idset":"9214"}}
+{"timestamp":1712611048.5397496,"name":"offline","context":{"idset":"9215"}}
+{"timestamp":1712611048.5418162,"name":"offline","context":{"idset":"9216"}}
+{"timestamp":1712611048.5782998,"name":"offline","context":{"idset":"9217"}}
+{"timestamp":1712611048.6148114,"name":"offline","context":{"idset":"9218"}}
+{"timestamp":1712611048.6518199,"name":"offline","context":{"idset":"9219"}}
+{"timestamp":1712611048.6625068,"name":"offline","context":{"idset":"9220"}}
+{"timestamp":1712611048.6646228,"name":"offline","context":{"idset":"9221"}}
+{"timestamp":1712611048.6667638,"name":"offline","context":{"idset":"9222"}}
+{"timestamp":1712611048.6688743,"name":"offline","context":{"idset":"9223"}}
+{"timestamp":1712611048.67102,"name":"offline","context":{"idset":"9224"}}
+{"timestamp":1712611048.6731346,"name":"offline","context":{"idset":"9225"}}
+{"timestamp":1712611048.6751804,"name":"offline","context":{"idset":"9226"}}
+{"timestamp":1712611048.6772542,"name":"offline","context":{"idset":"9227"}}
+{"timestamp":1712611048.680033,"name":"offline","context":{"idset":"9228"}}
+{"timestamp":1712611048.6926823,"name":"offline","context":{"idset":"9229"}}
+{"timestamp":1712611048.7140324,"name":"offline","context":{"idset":"9230"}}
+{"timestamp":1712611048.750546,"name":"offline","context":{"idset":"9231"}}
+{"timestamp":1712611048.7871552,"name":"offline","context":{"idset":"9232"}}
+{"timestamp":1712611048.8063772,"name":"offline","context":{"idset":"9233"}}
+{"timestamp":1712611048.8084834,"name":"offline","context":{"idset":"9234"}}
+{"timestamp":1712611048.8106003,"name":"offline","context":{"idset":"9235"}}
+{"timestamp":1712611048.8126941,"name":"offline","context":{"idset":"9236"}}
+{"timestamp":1712611048.8148522,"name":"offline","context":{"idset":"9237"}}
+{"timestamp":1712611048.8169827,"name":"offline","context":{"idset":"9238"}}
+{"timestamp":1712611048.8190684,"name":"offline","context":{"idset":"9239"}}
+{"timestamp":1712611048.8211579,"name":"offline","context":{"idset":"9240"}}
+{"timestamp":1712611048.8233318,"name":"offline","context":{"idset":"9241"}}
+{"timestamp":1712611048.8320184,"name":"offline","context":{"idset":"9242"}}
+{"timestamp":1712611048.8790007,"name":"offline","context":{"idset":"9243"}}
+{"timestamp":1712611048.9156606,"name":"offline","context":{"idset":"9244"}}
+{"timestamp":1712611048.9435725,"name":"offline","context":{"idset":"9245"}}
+{"timestamp":1712611048.9456596,"name":"offline","context":{"idset":"9246"}}
+{"timestamp":1712611048.9477754,"name":"offline","context":{"idset":"9247"}}
+{"timestamp":1712611048.9499729,"name":"offline","context":{"idset":"9248"}}
+{"timestamp":1712611048.952069,"name":"offline","context":{"idset":"9249"}}
+{"timestamp":1712611048.9542081,"name":"offline","context":{"idset":"9250"}}
+{"timestamp":1712611048.9563053,"name":"offline","context":{"idset":"9251"}}
+{"timestamp":1712611048.9583962,"name":"offline","context":{"idset":"9252"}}
+{"timestamp":1712611048.9606373,"name":"offline","context":{"idset":"9253"}}
+{"timestamp":1712611048.9731574,"name":"offline","context":{"idset":"9254"}}
+{"timestamp":1712611048.9855733,"name":"offline","context":{"idset":"9255"}}
+{"timestamp":1712611048.9980397,"name":"offline","context":{"idset":"9256"}}
+{"timestamp":1712611049.0357046,"name":"offline","context":{"idset":"9257"}}
+{"timestamp":1712611049.0719502,"name":"offline","context":{"idset":"9258"}}
+{"timestamp":1712611049.1085072,"name":"offline","context":{"idset":"9259"}}
+{"timestamp":1712611049.1278942,"name":"offline","context":{"idset":"9260"}}
+{"timestamp":1712611049.129987,"name":"offline","context":{"idset":"9261"}}
+{"timestamp":1712611049.1320767,"name":"offline","context":{"idset":"9262"}}
+{"timestamp":1712611049.1343431,"name":"offline","context":{"idset":"9263"}}
+{"timestamp":1712611049.1364298,"name":"offline","context":{"idset":"9264"}}
+{"timestamp":1712611049.1385136,"name":"offline","context":{"idset":"9265"}}
+{"timestamp":1712611049.1406369,"name":"offline","context":{"idset":"9266"}}
+{"timestamp":1712611049.1427255,"name":"offline","context":{"idset":"9267"}}
+{"timestamp":1712611049.1448057,"name":"offline","context":{"idset":"9268"}}
+{"timestamp":1712611049.1475048,"name":"offline","context":{"idset":"9269"}}
+{"timestamp":1712611049.1766725,"name":"offline","context":{"idset":"9270"}}
+{"timestamp":1712611049.2128592,"name":"offline","context":{"idset":"9271"}}
+{"timestamp":1712611049.2489402,"name":"offline","context":{"idset":"9272"}}
+{"timestamp":1712611049.2765043,"name":"offline","context":{"idset":"9273"}}
+{"timestamp":1712611049.278626,"name":"offline","context":{"idset":"9274"}}
+{"timestamp":1712611049.2806854,"name":"offline","context":{"idset":"9275"}}
+{"timestamp":1712611049.2827735,"name":"offline","context":{"idset":"9276"}}
+{"timestamp":1712611049.2848396,"name":"offline","context":{"idset":"9277"}}
+{"timestamp":1712611049.2868891,"name":"offline","context":{"idset":"9278"}}
+{"timestamp":1712611049.2889991,"name":"offline","context":{"idset":"9279"}}
+{"timestamp":1712611049.2910662,"name":"offline","context":{"idset":"9280"}}
+{"timestamp":1712611049.2931285,"name":"offline","context":{"idset":"9281"}}
+{"timestamp":1712611049.2952938,"name":"offline","context":{"idset":"9282"}}
+{"timestamp":1712611049.3027387,"name":"offline","context":{"idset":"9283"}}
+{"timestamp":1712611049.3268516,"name":"offline","context":{"idset":"9284"}}
+{"timestamp":1712611049.3633125,"name":"offline","context":{"idset":"9285"}}
+{"timestamp":1712611049.3992357,"name":"offline","context":{"idset":"9286"}}
+{"timestamp":1712611049.4352179,"name":"offline","context":{"idset":"9287"}}
+{"timestamp":1712611049.4543812,"name":"offline","context":{"idset":"9288"}}
+{"timestamp":1712611049.4564302,"name":"offline","context":{"idset":"9289"}}
+{"timestamp":1712611049.4584627,"name":"offline","context":{"idset":"9290"}}
+{"timestamp":1712611049.4604793,"name":"offline","context":{"idset":"9291"}}
+{"timestamp":1712611049.4625559,"name":"offline","context":{"idset":"9292"}}
+{"timestamp":1712611049.4645777,"name":"offline","context":{"idset":"9293"}}
+{"timestamp":1712611049.4665768,"name":"offline","context":{"idset":"9294"}}
+{"timestamp":1712611049.4686086,"name":"offline","context":{"idset":"9295"}}
+{"timestamp":1712611049.4706488,"name":"offline","context":{"idset":"9296"}}
+{"timestamp":1712611049.4727173,"name":"offline","context":{"idset":"9297"}}
+{"timestamp":1712611049.5004396,"name":"offline","context":{"idset":"9298"}}
+{"timestamp":1712611049.5365841,"name":"offline","context":{"idset":"9299"}}
+{"timestamp":1712611049.5728123,"name":"offline","context":{"idset":"9300"}}
+{"timestamp":1712611049.6004145,"name":"offline","context":{"idset":"9301"}}
+{"timestamp":1712611049.6024489,"name":"offline","context":{"idset":"9302"}}
+{"timestamp":1712611049.6045091,"name":"offline","context":{"idset":"9303"}}
+{"timestamp":1712611049.6065609,"name":"offline","context":{"idset":"9304"}}
+{"timestamp":1712611049.6086199,"name":"offline","context":{"idset":"9305"}}
+{"timestamp":1712611049.6106257,"name":"offline","context":{"idset":"9306"}}
+{"timestamp":1712611049.6126773,"name":"offline","context":{"idset":"9307"}}
+{"timestamp":1712611049.6147037,"name":"offline","context":{"idset":"9308"}}
+{"timestamp":1712611049.6167014,"name":"offline","context":{"idset":"9309"}}
+{"timestamp":1712611049.618722,"name":"offline","context":{"idset":"9310"}}
+{"timestamp":1712611049.6460993,"name":"offline","context":{"idset":"9311"}}
+{"timestamp":1712611049.6850362,"name":"offline","context":{"idset":"9312"}}
+{"timestamp":1712611049.7218318,"name":"offline","context":{"idset":"9313"}}
+{"timestamp":1712611049.7410128,"name":"offline","context":{"idset":"9314"}}
+{"timestamp":1712611049.743063,"name":"offline","context":{"idset":"9317"}}
+{"timestamp":1712611049.7450888,"name":"offline","context":{"idset":"9318"}}
+{"timestamp":1712611049.7471194,"name":"offline","context":{"idset":"9319"}}
+{"timestamp":1712611049.7491305,"name":"offline","context":{"idset":"9320"}}
+{"timestamp":1712611049.7511606,"name":"offline","context":{"idset":"9321"}}
+{"timestamp":1712611049.7532308,"name":"offline","context":{"idset":"9322"}}
+{"timestamp":1712611049.7553515,"name":"offline","context":{"idset":"9323"}}
+{"timestamp":1712611049.7573509,"name":"offline","context":{"idset":"9324"}}
+{"timestamp":1712611049.7593393,"name":"offline","context":{"idset":"9325"}}
+{"timestamp":1712611049.7906077,"name":"offline","context":{"idset":"9326"}}
+{"timestamp":1712611049.8270676,"name":"offline","context":{"idset":"9327"}}
+{"timestamp":1712611049.8637533,"name":"offline","context":{"idset":"9328"}}
+{"timestamp":1712611049.891784,"name":"offline","context":{"idset":"9329"}}
+{"timestamp":1712611049.8938215,"name":"offline","context":{"idset":"9330"}}
+{"timestamp":1712611049.8957875,"name":"offline","context":{"idset":"9331"}}
+{"timestamp":1712611049.8978648,"name":"offline","context":{"idset":"9332"}}
+{"timestamp":1712611049.8999701,"name":"offline","context":{"idset":"9333"}}
+{"timestamp":1712611049.9019659,"name":"offline","context":{"idset":"9334"}}
+{"timestamp":1712611049.9040039,"name":"offline","context":{"idset":"9335"}}
+{"timestamp":1712611049.9059792,"name":"offline","context":{"idset":"9336"}}
+{"timestamp":1712611049.9079776,"name":"offline","context":{"idset":"9337"}}
+{"timestamp":1712611049.9099884,"name":"offline","context":{"idset":"9338"}}
+{"timestamp":1712611049.9291182,"name":"offline","context":{"idset":"9339"}}
+{"timestamp":1712611049.9656429,"name":"offline","context":{"idset":"9340"}}
+{"timestamp":1712611050.0020165,"name":"offline","context":{"idset":"9341"}}
+{"timestamp":1712611050.0300016,"name":"offline","context":{"idset":"9342"}}
+{"timestamp":1712611050.0319512,"name":"offline","context":{"idset":"9343"}}
+{"timestamp":1712611050.033896,"name":"offline","context":{"idset":"9344"}}
+{"timestamp":1712611050.0358107,"name":"offline","context":{"idset":"9345"}}
+{"timestamp":1712611050.0377333,"name":"offline","context":{"idset":"9346"}}
+{"timestamp":1712611050.0396564,"name":"offline","context":{"idset":"9347"}}
+{"timestamp":1712611050.04161,"name":"offline","context":{"idset":"9348"}}
+{"timestamp":1712611050.0435243,"name":"offline","context":{"idset":"9349"}}
+{"timestamp":1712611050.045428,"name":"offline","context":{"idset":"9350"}}
+{"timestamp":1712611050.0622082,"name":"offline","context":{"idset":"9351"}}
+{"timestamp":1712611050.0964823,"name":"offline","context":{"idset":"9352"}}
+{"timestamp":1712611050.1300485,"name":"offline","context":{"idset":"9353"}}
+{"timestamp":1712611050.163595,"name":"offline","context":{"idset":"9354"}}
+{"timestamp":1712611050.1658251,"name":"offline","context":{"idset":"9355"}}
+{"timestamp":1712611050.1680942,"name":"offline","context":{"idset":"9356"}}
+{"timestamp":1712611050.1699986,"name":"offline","context":{"idset":"9357"}}
+{"timestamp":1712611050.1719484,"name":"offline","context":{"idset":"9358"}}
+{"timestamp":1712611050.1740289,"name":"offline","context":{"idset":"9359"}}
+{"timestamp":1712611050.1765196,"name":"offline","context":{"idset":"9360"}}
+{"timestamp":1712611050.1788287,"name":"offline","context":{"idset":"9361"}}
+{"timestamp":1712611050.181226,"name":"offline","context":{"idset":"9362"}}
+{"timestamp":1712611050.2010372,"name":"offline","context":{"idset":"9363"}}
+{"timestamp":1712611050.2319617,"name":"offline","context":{"idset":"9364"}}
+{"timestamp":1712611050.2614498,"name":"offline","context":{"idset":"9365"}}
+{"timestamp":1712611050.2835889,"name":"offline","context":{"idset":"9366"}}
+{"timestamp":1712611050.2851748,"name":"offline","context":{"idset":"9367"}}
+{"timestamp":1712611050.2867374,"name":"offline","context":{"idset":"9368"}}
+{"timestamp":1712611050.288296,"name":"offline","context":{"idset":"9369"}}
+{"timestamp":1712611050.289829,"name":"offline","context":{"idset":"9370"}}
+{"timestamp":1712611050.2914095,"name":"offline","context":{"idset":"9371"}}
+{"timestamp":1712611050.2929397,"name":"offline","context":{"idset":"9372"}}
+{"timestamp":1712611050.2945023,"name":"offline","context":{"idset":"9373"}}
+{"timestamp":1712611050.3053639,"name":"offline","context":{"idset":"9374"}}
+{"timestamp":1712611050.3238795,"name":"offline","context":{"idset":"9375"}}
+{"timestamp":1712611050.3520014,"name":"offline","context":{"idset":"9376"}}
+{"timestamp":1712611050.3797455,"name":"offline","context":{"idset":"9377"}}
+{"timestamp":1712611050.4007142,"name":"offline","context":{"idset":"9378"}}
+{"timestamp":1712611050.40222,"name":"offline","context":{"idset":"9379"}}
+{"timestamp":1712611050.4036942,"name":"offline","context":{"idset":"9380"}}
+{"timestamp":1712611050.4051316,"name":"offline","context":{"idset":"9381"}}
+{"timestamp":1712611050.4065945,"name":"offline","context":{"idset":"9382"}}
+{"timestamp":1712611050.4080391,"name":"offline","context":{"idset":"9383"}}
+{"timestamp":1712611050.4095154,"name":"offline","context":{"idset":"9384"}}
+{"timestamp":1712611050.4124982,"name":"offline","context":{"idset":"9385"}}
+{"timestamp":1712611050.4257362,"name":"offline","context":{"idset":"9386"}}
+{"timestamp":1712611050.4408433,"name":"offline","context":{"idset":"9387"}}
+{"timestamp":1712611050.4773068,"name":"offline","context":{"idset":"9388"}}
+{"timestamp":1712611050.4974611,"name":"offline","context":{"idset":"9389"}}
+{"timestamp":1712611050.5327463,"name":"offline","context":{"idset":"9390"}}
+{"timestamp":1712611050.5691655,"name":"offline","context":{"idset":"9391"}}
+{"timestamp":1712611050.5896194,"name":"offline","context":{"idset":"9392"}}
+{"timestamp":1712611050.5917022,"name":"offline","context":{"idset":"9393"}}
+{"timestamp":1712611050.5938435,"name":"offline","context":{"idset":"9394"}}
+{"timestamp":1712611050.595933,"name":"offline","context":{"idset":"9395"}}
+{"timestamp":1712611050.5980587,"name":"offline","context":{"idset":"9396"}}
+{"timestamp":1712611050.6001487,"name":"offline","context":{"idset":"9397"}}
+{"timestamp":1712611050.6021965,"name":"offline","context":{"idset":"9398"}}
+{"timestamp":1712611050.6049461,"name":"offline","context":{"idset":"9399"}}
+{"timestamp":1712611050.6170626,"name":"offline","context":{"idset":"9400"}}
+{"timestamp":1712611050.6391337,"name":"offline","context":{"idset":"9401"}}
+{"timestamp":1712611050.6810563,"name":"offline","context":{"idset":"9402"}}
+{"timestamp":1712611050.7211139,"name":"offline","context":{"idset":"9403"}}
+{"timestamp":1712611050.7403147,"name":"offline","context":{"idset":"9404"}}
+{"timestamp":1712611050.7422535,"name":"offline","context":{"idset":"9405"}}
+{"timestamp":1712611050.744231,"name":"offline","context":{"idset":"9406"}}
+{"timestamp":1712611050.7461779,"name":"offline","context":{"idset":"9407"}}
+{"timestamp":1712611050.7481825,"name":"offline","context":{"idset":"9408"}}
+{"timestamp":1712611050.7501106,"name":"offline","context":{"idset":"9409"}}
+{"timestamp":1712611050.7521391,"name":"offline","context":{"idset":"9410"}}
+{"timestamp":1712611050.754133,"name":"offline","context":{"idset":"9411"}}
+{"timestamp":1712611050.7561114,"name":"offline","context":{"idset":"9412"}}
+{"timestamp":1712611050.7583754,"name":"offline","context":{"idset":"9413"}}
+{"timestamp":1712611050.7699492,"name":"offline","context":{"idset":"9414"}}
+{"timestamp":1712611050.7899377,"name":"offline","context":{"idset":"9415"}}
+{"timestamp":1712611050.8272748,"name":"offline","context":{"idset":"9416"}}
+{"timestamp":1712611050.863456,"name":"offline","context":{"idset":"9417"}}
+{"timestamp":1712611050.8997526,"name":"offline","context":{"idset":"9418"}}
+{"timestamp":1712611050.9102385,"name":"offline","context":{"idset":"9419"}}
+{"timestamp":1712611050.9121902,"name":"offline","context":{"idset":"9420"}}
+{"timestamp":1712611050.9141183,"name":"offline","context":{"idset":"9421"}}
+{"timestamp":1712611050.9160423,"name":"offline","context":{"idset":"9422"}}
+{"timestamp":1712611050.9179869,"name":"offline","context":{"idset":"9423"}}
+{"timestamp":1712611050.9200017,"name":"offline","context":{"idset":"9424"}}
+{"timestamp":1712611050.9220016,"name":"offline","context":{"idset":"9425"}}
+{"timestamp":1712611050.9239163,"name":"offline","context":{"idset":"9426"}}
+{"timestamp":1712611050.925916,"name":"offline","context":{"idset":"9427"}}
+{"timestamp":1712611050.9279327,"name":"offline","context":{"idset":"9428"}}
+{"timestamp":1712611050.9568939,"name":"offline","context":{"idset":"9429"}}
+{"timestamp":1712611050.9929504,"name":"offline","context":{"idset":"9430"}}
+{"timestamp":1712611051.0289755,"name":"offline","context":{"idset":"9431"}}
+{"timestamp":1712611051.0480406,"name":"offline","context":{"idset":"9432"}}
+{"timestamp":1712611051.0499637,"name":"offline","context":{"idset":"9433"}}
+{"timestamp":1712611051.0518851,"name":"offline","context":{"idset":"9434"}}
+{"timestamp":1712611051.0539472,"name":"offline","context":{"idset":"9435"}}
+{"timestamp":1712611051.0561023,"name":"offline","context":{"idset":"9436"}}
+{"timestamp":1712611051.0581212,"name":"offline","context":{"idset":"9437"}}
+{"timestamp":1712611051.0600772,"name":"offline","context":{"idset":"9438"}}
+{"timestamp":1712611051.0621185,"name":"offline","context":{"idset":"9439"}}
+{"timestamp":1712611051.0641127,"name":"offline","context":{"idset":"9440"}}
+{"timestamp":1712611051.0660322,"name":"offline","context":{"idset":"9441"}}
+{"timestamp":1712611051.0679579,"name":"offline","context":{"idset":"9442"}}
+{"timestamp":1712611051.1043899,"name":"offline","context":{"idset":"9443"}}
+{"timestamp":1712611051.1415799,"name":"offline","context":{"idset":"9444"}}
+{"timestamp":1712611051.1775274,"name":"offline","context":{"idset":"9445"}}
+{"timestamp":1712611051.1880121,"name":"offline","context":{"idset":"9446"}}
+{"timestamp":1712611051.1899633,"name":"offline","context":{"idset":"9449"}}
+{"timestamp":1712611051.192009,"name":"offline","context":{"idset":"9450"}}
+{"timestamp":1712611051.1939595,"name":"offline","context":{"idset":"9451"}}
+{"timestamp":1712611051.1961074,"name":"offline","context":{"idset":"9452"}}
+{"timestamp":1712611051.1981592,"name":"offline","context":{"idset":"9453"}}
+{"timestamp":1712611051.2001388,"name":"offline","context":{"idset":"9454"}}
+{"timestamp":1712611051.2020204,"name":"offline","context":{"idset":"9455"}}
+{"timestamp":1712611051.2039018,"name":"offline","context":{"idset":"9456"}}
+{"timestamp":1712611051.2058163,"name":"offline","context":{"idset":"9457"}}
+{"timestamp":1712611051.2077045,"name":"offline","context":{"idset":"9458"}}
+{"timestamp":1712611051.2267683,"name":"offline","context":{"idset":"9459"}}
+{"timestamp":1712611051.2631288,"name":"offline","context":{"idset":"9460"}}
+{"timestamp":1712611051.2991643,"name":"offline","context":{"idset":"9461"}}
+{"timestamp":1712611051.3353159,"name":"offline","context":{"idset":"9462"}}
+{"timestamp":1712611051.3372436,"name":"offline","context":{"idset":"9463"}}
+{"timestamp":1712611051.3391793,"name":"offline","context":{"idset":"9464"}}
+{"timestamp":1712611051.3411176,"name":"offline","context":{"idset":"9465"}}
+{"timestamp":1712611051.343009,"name":"offline","context":{"idset":"9466"}}
+{"timestamp":1712611051.3449469,"name":"offline","context":{"idset":"9467"}}
+{"timestamp":1712611051.3469014,"name":"offline","context":{"idset":"9468"}}
+{"timestamp":1712611051.3488665,"name":"offline","context":{"idset":"9469"}}
+{"timestamp":1712611051.3507292,"name":"offline","context":{"idset":"9470"}}
+{"timestamp":1712611051.3526788,"name":"offline","context":{"idset":"9471"}}
+{"timestamp":1712611051.3546021,"name":"offline","context":{"idset":"9473"}}
+{"timestamp":1712611051.3564925,"name":"offline","context":{"idset":"9474"}}
+{"timestamp":1712611051.3584228,"name":"offline","context":{"idset":"9475"}}
+{"timestamp":1712611051.3603768,"name":"offline","context":{"idset":"9476"}}
+{"timestamp":1712611051.3962314,"name":"offline","context":{"idset":"9477"}}
+{"timestamp":1712611051.4326928,"name":"offline","context":{"idset":"9478"}}
+{"timestamp":1712611051.4688516,"name":"offline","context":{"idset":"9479"}}
+{"timestamp":1712611051.4793437,"name":"offline","context":{"idset":"9480"}}
+{"timestamp":1712611051.4811764,"name":"offline","context":{"idset":"9481"}}
+{"timestamp":1712611051.4831057,"name":"offline","context":{"idset":"9482"}}
+{"timestamp":1712611051.4850001,"name":"offline","context":{"idset":"9483"}}
+{"timestamp":1712611051.4868629,"name":"offline","context":{"idset":"9484"}}
+{"timestamp":1712611051.4888,"name":"offline","context":{"idset":"9485"}}
+{"timestamp":1712611051.4906845,"name":"offline","context":{"idset":"9486"}}
+{"timestamp":1712611051.4925463,"name":"offline","context":{"idset":"9487"}}
+{"timestamp":1712611051.4945016,"name":"offline","context":{"idset":"9489"}}
+{"timestamp":1712611051.4963999,"name":"offline","context":{"idset":"9490"}}
+{"timestamp":1712611051.4982369,"name":"offline","context":{"idset":"9491"}}
+{"timestamp":1712611051.5001161,"name":"offline","context":{"idset":"9492"}}
+{"timestamp":1712611051.5019944,"name":"offline","context":{"idset":"9493"}}
+{"timestamp":1712611051.5208974,"name":"offline","context":{"idset":"9494"}}
+{"timestamp":1712611051.5576539,"name":"offline","context":{"idset":"9495"}}
+{"timestamp":1712611051.5939541,"name":"offline","context":{"idset":"9496"}}
+{"timestamp":1712611051.6300848,"name":"offline","context":{"idset":"9497"}}
+{"timestamp":1712611051.6404939,"name":"offline","context":{"idset":"9498"}}
+{"timestamp":1712611051.6423705,"name":"offline","context":{"idset":"9499"}}
+{"timestamp":1712611051.6442118,"name":"offline","context":{"idset":"9500"}}
+{"timestamp":1712611051.6460881,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1712611051.6479003,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1712611051.6497841,"name":"offline","context":{"idset":"9503"}}
+{"timestamp":1712611051.6516607,"name":"offline","context":{"idset":"9504"}}
+{"timestamp":1712611051.6535203,"name":"offline","context":{"idset":"9505"}}
+{"timestamp":1712611051.655376,"name":"offline","context":{"idset":"9506"}}
+{"timestamp":1712611051.6572294,"name":"offline","context":{"idset":"9507"}}
+{"timestamp":1712611051.6590769,"name":"offline","context":{"idset":"9508"}}
+{"timestamp":1712611051.6609416,"name":"offline","context":{"idset":"9509"}}
+{"timestamp":1712611051.6628659,"name":"offline","context":{"idset":"9510"}}
+{"timestamp":1712611051.690356,"name":"offline","context":{"idset":"9511"}}
+{"timestamp":1712611051.7315562,"name":"offline","context":{"idset":"9512"}}
+{"timestamp":1712611051.7681327,"name":"offline","context":{"idset":"9513"}}
+{"timestamp":1712611051.804812,"name":"offline","context":{"idset":"9514"}}
+{"timestamp":1712611051.8154662,"name":"offline","context":{"idset":"9515"}}
+{"timestamp":1712611051.8174508,"name":"offline","context":{"idset":"9516"}}
+{"timestamp":1712611051.8193145,"name":"offline","context":{"idset":"9517"}}
+{"timestamp":1712611051.8211951,"name":"offline","context":{"idset":"9518"}}
+{"timestamp":1712611051.8230257,"name":"offline","context":{"idset":"9519"}}
+{"timestamp":1712611051.8248713,"name":"offline","context":{"idset":"9520"}}
+{"timestamp":1712611051.8267722,"name":"offline","context":{"idset":"9521"}}
+{"timestamp":1712611051.8286242,"name":"offline","context":{"idset":"9522"}}
+{"timestamp":1712611051.8305755,"name":"offline","context":{"idset":"9523"}}
+{"timestamp":1712611051.8725286,"name":"offline","context":{"idset":"9524"}}
+{"timestamp":1712611051.9083431,"name":"offline","context":{"idset":"9525"}}
+{"timestamp":1712611051.9451442,"name":"offline","context":{"idset":"9526"}}
+{"timestamp":1712611051.9641788,"name":"offline","context":{"idset":"9527"}}
+{"timestamp":1712611051.9660366,"name":"offline","context":{"idset":"9528"}}
+{"timestamp":1712611051.968014,"name":"offline","context":{"idset":"9529"}}
+{"timestamp":1712611051.9698803,"name":"offline","context":{"idset":"9530"}}
+{"timestamp":1712611051.9717395,"name":"offline","context":{"idset":"9531"}}
+{"timestamp":1712611051.9736431,"name":"offline","context":{"idset":"9532"}}
+{"timestamp":1712611051.9755418,"name":"offline","context":{"idset":"9536"}}
+{"timestamp":1712611051.9773486,"name":"offline","context":{"idset":"9537"}}
+{"timestamp":1712611051.9791145,"name":"offline","context":{"idset":"9538"}}
+{"timestamp":1712611051.9810009,"name":"offline","context":{"idset":"9539"}}
+{"timestamp":1712611051.9828866,"name":"offline","context":{"idset":"9540"}}
+{"timestamp":1712611051.9846931,"name":"offline","context":{"idset":"9541"}}
+{"timestamp":1712611052.0041056,"name":"offline","context":{"idset":"9542"}}
+{"timestamp":1712611052.031919,"name":"offline","context":{"idset":"9543"}}
+{"timestamp":1712611052.0339766,"name":"offline","context":{"idset":"9544"}}
+{"timestamp":1712611052.0358658,"name":"offline","context":{"idset":"9545"}}
+{"timestamp":1712611052.0376902,"name":"offline","context":{"idset":"9546"}}
+{"timestamp":1712611052.0396202,"name":"offline","context":{"idset":"9547"}}
+{"timestamp":1712611052.0415165,"name":"offline","context":{"idset":"9548"}}
+{"timestamp":1712611052.0434613,"name":"offline","context":{"idset":"9549"}}
+{"timestamp":1712611052.0452845,"name":"offline","context":{"idset":"9550"}}
+{"timestamp":1712611052.0472143,"name":"offline","context":{"idset":"9551"}}
+{"timestamp":1712611052.0490506,"name":"offline","context":{"idset":"9552"}}
+{"timestamp":1712611052.0509241,"name":"offline","context":{"idset":"9553"}}
+{"timestamp":1712611052.0880833,"name":"offline","context":{"idset":"9554"}}
+{"timestamp":1712611052.1245055,"name":"offline","context":{"idset":"9555"}}
+{"timestamp":1712611052.1612563,"name":"offline","context":{"idset":"9556"}}
+{"timestamp":1712611052.1807325,"name":"offline","context":{"idset":"9557"}}
+{"timestamp":1712611052.18256,"name":"offline","context":{"idset":"9558"}}
+{"timestamp":1712611052.1843615,"name":"offline","context":{"idset":"9559"}}
+{"timestamp":1712611052.1861875,"name":"offline","context":{"idset":"9560"}}
+{"timestamp":1712611052.1880007,"name":"offline","context":{"idset":"9563"}}
+{"timestamp":1712611052.1899381,"name":"offline","context":{"idset":"9564"}}
+{"timestamp":1712611052.1917827,"name":"offline","context":{"idset":"9565"}}
+{"timestamp":1712611052.1935904,"name":"offline","context":{"idset":"9566"}}
+{"timestamp":1712611052.1954,"name":"offline","context":{"idset":"9567"}}
+{"timestamp":1712611052.2058611,"name":"offline","context":{"idset":"9568"}}
+{"timestamp":1712611052.225219,"name":"offline","context":{"idset":"9571"}}
+{"timestamp":1712611052.2270381,"name":"offline","context":{"idset":"9572"}}
+{"timestamp":1712611052.2288597,"name":"offline","context":{"idset":"9573"}}
+{"timestamp":1712611052.230731,"name":"offline","context":{"idset":"9574"}}
+{"timestamp":1712611052.232645,"name":"offline","context":{"idset":"9575"}}
+{"timestamp":1712611052.2344892,"name":"offline","context":{"idset":"9576"}}
+{"timestamp":1712611052.2362881,"name":"offline","context":{"idset":"9577"}}
+{"timestamp":1712611052.2380676,"name":"offline","context":{"idset":"9578"}}
+{"timestamp":1712611052.2399492,"name":"offline","context":{"idset":"9579"}}
+{"timestamp":1712611052.2417607,"name":"offline","context":{"idset":"9580"}}
+{"timestamp":1712611052.2435603,"name":"offline","context":{"idset":"9581"}}
+{"timestamp":1712611052.2453651,"name":"offline","context":{"idset":"9582"}}
+{"timestamp":1712611052.2732353,"name":"offline","context":{"idset":"9583"}}
+{"timestamp":1712611052.3102331,"name":"offline","context":{"idset":"9584"}}
+{"timestamp":1712611052.3376429,"name":"offline","context":{"idset":"9585"}}
+{"timestamp":1712611052.3394172,"name":"offline","context":{"idset":"9586"}}
+{"timestamp":1712611052.3411782,"name":"offline","context":{"idset":"9587"}}
+{"timestamp":1712611052.3429983,"name":"offline","context":{"idset":"9588"}}
+{"timestamp":1712611052.3447509,"name":"offline","context":{"idset":"9589"}}
+{"timestamp":1712611052.3465524,"name":"offline","context":{"idset":"9590"}}
+{"timestamp":1712611052.3482943,"name":"offline","context":{"idset":"9591"}}
+{"timestamp":1712611052.3500998,"name":"offline","context":{"idset":"9592"}}
+{"timestamp":1712611052.3518736,"name":"offline","context":{"idset":"9593"}}
+{"timestamp":1712611052.3536465,"name":"offline","context":{"idset":"9594"}}
+{"timestamp":1712611052.3554394,"name":"offline","context":{"idset":"9595"}}
+{"timestamp":1712611052.3571646,"name":"offline","context":{"idset":"9596"}}
+{"timestamp":1712611052.3837183,"name":"offline","context":{"idset":"9597"}}
+{"timestamp":1712611052.4177747,"name":"offline","context":{"idset":"9598"}}
+{"timestamp":1712611052.4506037,"name":"offline","context":{"idset":"9599"}}
+{"timestamp":1712611052.4601755,"name":"offline","context":{"idset":"9600"}}
+{"timestamp":1712611052.4618144,"name":"offline","context":{"idset":"9601"}}
+{"timestamp":1712611052.4634542,"name":"offline","context":{"idset":"9602"}}
+{"timestamp":1712611052.4650068,"name":"offline","context":{"idset":"9603"}}
+{"timestamp":1712611052.4665935,"name":"offline","context":{"idset":"9604"}}
+{"timestamp":1712611052.4681923,"name":"offline","context":{"idset":"9605"}}
+{"timestamp":1712611052.4698036,"name":"offline","context":{"idset":"9606"}}
+{"timestamp":1712611052.471406,"name":"offline","context":{"idset":"9607"}}
+{"timestamp":1712611052.4729919,"name":"offline","context":{"idset":"9608"}}
+{"timestamp":1712611052.4745729,"name":"offline","context":{"idset":"9609"}}
+{"timestamp":1712611052.4761603,"name":"offline","context":{"idset":"9610"}}
+{"timestamp":1712611052.4776926,"name":"offline","context":{"idset":"9611"}}
+{"timestamp":1712611052.4792933,"name":"offline","context":{"idset":"9612"}}
+{"timestamp":1712611052.4808419,"name":"offline","context":{"idset":"9613"}}
+{"timestamp":1712611052.4824367,"name":"offline","context":{"idset":"9614"}}
+{"timestamp":1712611052.4860399,"name":"offline","context":{"idset":"9615"}}
+{"timestamp":1712611052.5121899,"name":"offline","context":{"idset":"9616"}}
+{"timestamp":1712611052.5434296,"name":"offline","context":{"idset":"9617"}}
+{"timestamp":1712611052.5735931,"name":"offline","context":{"idset":"9618"}}
+{"timestamp":1712611052.6030531,"name":"offline","context":{"idset":"9619"}}
+{"timestamp":1712611052.6182957,"name":"offline","context":{"idset":"9620"}}
+{"timestamp":1712611052.6197071,"name":"offline","context":{"idset":"9621"}}
+{"timestamp":1712611052.6211298,"name":"offline","context":{"idset":"9622"}}
+{"timestamp":1712611052.6226203,"name":"offline","context":{"idset":"9623"}}
+{"timestamp":1712611052.6240356,"name":"offline","context":{"idset":"9624"}}
+{"timestamp":1712611052.6254692,"name":"offline","context":{"idset":"9625"}}
+{"timestamp":1712611052.6268258,"name":"offline","context":{"idset":"9626"}}
+{"timestamp":1712611052.6282024,"name":"offline","context":{"idset":"9627"}}
+{"timestamp":1712611052.6360645,"name":"offline","context":{"idset":"9628"}}
+{"timestamp":1712611052.6472168,"name":"offline","context":{"idset":"9629"}}
+{"timestamp":1712611052.6585925,"name":"offline","context":{"idset":"9631"}}
+{"timestamp":1712611052.6810939,"name":"offline","context":{"idset":"9632"}}
+{"timestamp":1712611052.6825967,"name":"offline","context":{"idset":"9633"}}
+{"timestamp":1712611052.6926055,"name":"offline","context":{"idset":"9634"}}
+{"timestamp":1712611052.7038014,"name":"offline","context":{"idset":"9635"}}
+{"timestamp":1712611052.7156267,"name":"offline","context":{"idset":"9636"}}
+{"timestamp":1712611052.7274835,"name":"offline","context":{"idset":"9637"}}
+{"timestamp":1712611052.7388313,"name":"offline","context":{"idset":"9638"}}
+{"timestamp":1712611052.7507291,"name":"offline","context":{"idset":"9639"}}
+{"timestamp":1712611052.7623906,"name":"offline","context":{"idset":"9640"}}
+{"timestamp":1712611052.7740862,"name":"offline","context":{"idset":"9641"}}
+{"timestamp":1712611052.7854097,"name":"offline","context":{"idset":"9642"}}
+{"timestamp":1712611052.7971759,"name":"offline","context":{"idset":"9643"}}
+{"timestamp":1712611052.8195589,"name":"offline","context":{"idset":"9644"}}
+{"timestamp":1712611052.860815,"name":"offline","context":{"idset":"9645"}}
+{"timestamp":1712611052.8972104,"name":"offline","context":{"idset":"9646"}}
+{"timestamp":1712611052.9339695,"name":"offline","context":{"idset":"9647"}}
+{"timestamp":1712611052.9691598,"name":"offline","context":{"idset":"9648"}}
+{"timestamp":1712611053.004658,"name":"offline","context":{"idset":"9649"}}
+{"timestamp":1712611053.0316496,"name":"offline","context":{"idset":"9650"}}
+{"timestamp":1712611053.0334404,"name":"offline","context":{"idset":"9651"}}
+{"timestamp":1712611053.0351279,"name":"offline","context":{"idset":"9652"}}
+{"timestamp":1712611053.0368731,"name":"offline","context":{"idset":"9653"}}
+{"timestamp":1712611053.038604,"name":"offline","context":{"idset":"9654"}}
+{"timestamp":1712611053.0403395,"name":"offline","context":{"idset":"9655"}}
+{"timestamp":1712611053.0420508,"name":"offline","context":{"idset":"9656"}}
+{"timestamp":1712611053.0437949,"name":"offline","context":{"idset":"9657"}}
+{"timestamp":1712611053.0455561,"name":"offline","context":{"idset":"9658"}}
+{"timestamp":1712611053.0472219,"name":"offline","context":{"idset":"9659"}}
+{"timestamp":1712611053.0489738,"name":"offline","context":{"idset":"9660"}}
+{"timestamp":1712611053.0506833,"name":"offline","context":{"idset":"9661"}}
+{"timestamp":1712611053.052422,"name":"offline","context":{"idset":"9662"}}
+{"timestamp":1712611053.0541389,"name":"offline","context":{"idset":"9663"}}
+{"timestamp":1712611053.0558443,"name":"offline","context":{"idset":"9664"}}
+{"timestamp":1712611053.0575168,"name":"offline","context":{"idset":"9665"}}
+{"timestamp":1712611053.0591969,"name":"offline","context":{"idset":"9666"}}
+{"timestamp":1712611053.0609357,"name":"offline","context":{"idset":"9667"}}
+{"timestamp":1712611053.0627182,"name":"offline","context":{"idset":"9668"}}
+{"timestamp":1712611053.0754669,"name":"offline","context":{"idset":"9669"}}
+{"timestamp":1712611053.1045935,"name":"offline","context":{"idset":"9670"}}
+{"timestamp":1712611053.1400807,"name":"offline","context":{"idset":"9671"}}
+{"timestamp":1712611053.175586,"name":"offline","context":{"idset":"9672"}}
+{"timestamp":1712611053.2114253,"name":"offline","context":{"idset":"9673"}}
+{"timestamp":1712611053.247189,"name":"offline","context":{"idset":"9674"}}
+{"timestamp":1712611053.282594,"name":"offline","context":{"idset":"9675"}}
+{"timestamp":1712611053.31849,"name":"offline","context":{"idset":"9676"}}
+{"timestamp":1712611053.3539262,"name":"offline","context":{"idset":"9677"}}
+{"timestamp":1712611053.3900254,"name":"offline","context":{"idset":"9678"}}
+{"timestamp":1712611053.3917251,"name":"offline","context":{"idset":"9679"}}
+{"timestamp":1712611053.3935072,"name":"offline","context":{"idset":"9680"}}
+{"timestamp":1712611053.3951964,"name":"offline","context":{"idset":"9681"}}
+{"timestamp":1712611053.3969123,"name":"offline","context":{"idset":"9682"}}
+{"timestamp":1712611053.3986204,"name":"offline","context":{"idset":"9683"}}
+{"timestamp":1712611053.4003165,"name":"offline","context":{"idset":"9684"}}
+{"timestamp":1712611053.4020004,"name":"offline","context":{"idset":"9685"}}
+{"timestamp":1712611053.4037151,"name":"offline","context":{"idset":"9686"}}
+{"timestamp":1712611053.4054317,"name":"offline","context":{"idset":"9687"}}
+{"timestamp":1712611053.4072015,"name":"offline","context":{"idset":"9688"}}
+{"timestamp":1712611053.40887,"name":"offline","context":{"idset":"9689"}}
+{"timestamp":1712611053.4105473,"name":"offline","context":{"idset":"9690"}}
+{"timestamp":1712611053.4207158,"name":"offline","context":{"idset":"9691"}}
+{"timestamp":1712611053.4565389,"name":"offline","context":{"idset":"9692"}}
+{"timestamp":1712611053.4940095,"name":"offline","context":{"idset":"9693"}}
+{"timestamp":1712611053.5299895,"name":"offline","context":{"idset":"9694"}}
+{"timestamp":1712611053.5655622,"name":"offline","context":{"idset":"9695"}}
+{"timestamp":1712611053.601088,"name":"offline","context":{"idset":"9696"}}
+{"timestamp":1712611053.6364586,"name":"offline","context":{"idset":"9697"}}
+{"timestamp":1712611053.6555007,"name":"offline","context":{"idset":"9698"}}
+{"timestamp":1712611053.6571832,"name":"offline","context":{"idset":"9699"}}
+{"timestamp":1712611053.6588545,"name":"offline","context":{"idset":"9700"}}
+{"timestamp":1712611053.6605763,"name":"offline","context":{"idset":"9701"}}
+{"timestamp":1712611053.6622934,"name":"offline","context":{"idset":"9702"}}
+{"timestamp":1712611053.6639631,"name":"offline","context":{"idset":"9703"}}
+{"timestamp":1712611053.6656916,"name":"offline","context":{"idset":"9704"}}
+{"timestamp":1712611053.6674249,"name":"offline","context":{"idset":"9705"}}
+{"timestamp":1712611053.6691089,"name":"offline","context":{"idset":"9706"}}
+{"timestamp":1712611053.6964722,"name":"offline","context":{"idset":"9707"}}
+{"timestamp":1712611053.732775,"name":"offline","context":{"idset":"9708"}}
+{"timestamp":1712611053.7685697,"name":"offline","context":{"idset":"9709"}}
+{"timestamp":1712611053.804239,"name":"offline","context":{"idset":"9710"}}
+{"timestamp":1712611053.8397231,"name":"offline","context":{"idset":"9711"}}
+{"timestamp":1712611053.8751721,"name":"offline","context":{"idset":"9712"}}
+{"timestamp":1712611053.8936796,"name":"offline","context":{"idset":"9713"}}
+{"timestamp":1712611053.8953462,"name":"offline","context":{"idset":"9714"}}
+{"timestamp":1712611053.8970852,"name":"offline","context":{"idset":"9715"}}
+{"timestamp":1712611053.8987403,"name":"offline","context":{"idset":"9716"}}
+{"timestamp":1712611053.9004173,"name":"offline","context":{"idset":"9717"}}
+{"timestamp":1712611053.935802,"name":"offline","context":{"idset":"9718"}}
+{"timestamp":1712611053.9544013,"name":"offline","context":{"idset":"9719"}}
+{"timestamp":1712611053.9560692,"name":"offline","context":{"idset":"9720"}}
+{"timestamp":1712611053.9577837,"name":"offline","context":{"idset":"9721"}}
+{"timestamp":1712611053.9594429,"name":"offline","context":{"idset":"9722"}}
+{"timestamp":1712611053.9610946,"name":"offline","context":{"idset":"9723"}}
+{"timestamp":1712611053.9627364,"name":"offline","context":{"idset":"9724"}}
+{"timestamp":1712611053.9644053,"name":"offline","context":{"idset":"9725"}}
+{"timestamp":1712611053.9660661,"name":"offline","context":{"idset":"9726"}}
+{"timestamp":1712611053.9677482,"name":"offline","context":{"idset":"9727"}}
+{"timestamp":1712611053.9694018,"name":"offline","context":{"idset":"9728"}}
+{"timestamp":1712611053.9794714,"name":"offline","context":{"idset":"9729"}}
+{"timestamp":1712611054.0151606,"name":"offline","context":{"idset":"9730"}}
+{"timestamp":1712611054.0511937,"name":"offline","context":{"idset":"9731"}}
+{"timestamp":1712611054.0868766,"name":"offline","context":{"idset":"9732"}}
+{"timestamp":1712611054.0970466,"name":"offline","context":{"idset":"9733"}}
+{"timestamp":1712611054.0987628,"name":"offline","context":{"idset":"9734"}}
+{"timestamp":1712611054.100395,"name":"offline","context":{"idset":"9735"}}
+{"timestamp":1712611054.1020613,"name":"offline","context":{"idset":"9736"}}
+{"timestamp":1712611054.1036992,"name":"offline","context":{"idset":"9737"}}
+{"timestamp":1712611054.1053574,"name":"offline","context":{"idset":"9738"}}
+{"timestamp":1712611054.1070673,"name":"offline","context":{"idset":"9739"}}
+{"timestamp":1712611054.1087277,"name":"offline","context":{"idset":"9740"}}
+{"timestamp":1712611054.110333,"name":"offline","context":{"idset":"9741"}}
+{"timestamp":1712611054.1291769,"name":"offline","context":{"idset":"9742"}}
+{"timestamp":1712611054.156548,"name":"offline","context":{"idset":"9743"}}
+{"timestamp":1712611054.1581831,"name":"offline","context":{"idset":"9744"}}
+{"timestamp":1712611054.1598766,"name":"offline","context":{"idset":"9745"}}
+{"timestamp":1712611054.1615593,"name":"offline","context":{"idset":"9746"}}
+{"timestamp":1712611054.1631987,"name":"offline","context":{"idset":"9747"}}
+{"timestamp":1712611054.1648273,"name":"offline","context":{"idset":"9748"}}
+{"timestamp":1712611054.1665325,"name":"offline","context":{"idset":"9749"}}
+{"timestamp":1712611054.168175,"name":"offline","context":{"idset":"9750"}}
+{"timestamp":1712611054.1784105,"name":"offline","context":{"idset":"9751"}}
+{"timestamp":1712611054.2144301,"name":"offline","context":{"idset":"9752"}}
+{"timestamp":1712611054.256093,"name":"offline","context":{"idset":"9753"}}
+{"timestamp":1712611054.2666972,"name":"offline","context":{"idset":"9754"}}
+{"timestamp":1712611054.268445,"name":"offline","context":{"idset":"9755"}}
+{"timestamp":1712611054.2701814,"name":"offline","context":{"idset":"9756"}}
+{"timestamp":1712611054.27192,"name":"offline","context":{"idset":"9757"}}
+{"timestamp":1712611054.2736609,"name":"offline","context":{"idset":"9758"}}
+{"timestamp":1712611054.2753668,"name":"offline","context":{"idset":"9759"}}
+{"timestamp":1712611054.2858636,"name":"offline","context":{"idset":"9760"}}
+{"timestamp":1712611054.3157194,"name":"offline","context":{"idset":"9761"}}
+{"timestamp":1712611054.3174207,"name":"offline","context":{"idset":"9762"}}
+{"timestamp":1712611054.319056,"name":"offline","context":{"idset":"9763"}}
+{"timestamp":1712611054.3207448,"name":"offline","context":{"idset":"9764"}}
+{"timestamp":1712611054.322432,"name":"offline","context":{"idset":"9765"}}
+{"timestamp":1712611054.3240402,"name":"offline","context":{"idset":"9766"}}
+{"timestamp":1712611054.3257151,"name":"offline","context":{"idset":"9767"}}
+{"timestamp":1712611054.327395,"name":"offline","context":{"idset":"9768"}}
+{"timestamp":1712611054.3290889,"name":"offline","context":{"idset":"9769"}}
+{"timestamp":1712611054.3307757,"name":"offline","context":{"idset":"9770"}}
+{"timestamp":1712611054.3330731,"name":"offline","context":{"idset":"9771"}}
+{"timestamp":1712611054.3355656,"name":"offline","context":{"idset":"9772"}}
+{"timestamp":1712611054.3575594,"name":"offline","context":{"idset":"9773"}}
+{"timestamp":1712611054.3767316,"name":"offline","context":{"idset":"9774"}}
+{"timestamp":1712611054.3785472,"name":"offline","context":{"idset":"9775"}}
+{"timestamp":1712611054.3801887,"name":"offline","context":{"idset":"9776"}}
+{"timestamp":1712611054.3818059,"name":"offline","context":{"idset":"9777"}}
+{"timestamp":1712611054.3834951,"name":"offline","context":{"idset":"9778"}}
+{"timestamp":1712611054.3851666,"name":"offline","context":{"idset":"9779"}}
+{"timestamp":1712611054.3868637,"name":"offline","context":{"idset":"9780"}}
+{"timestamp":1712611054.3886185,"name":"offline","context":{"idset":"9781"}}
+{"timestamp":1712611054.3903179,"name":"offline","context":{"idset":"9782"}}
+{"timestamp":1712611054.3922079,"name":"offline","context":{"idset":"9783"}}
+{"timestamp":1712611054.3938746,"name":"offline","context":{"idset":"9784"}}
+{"timestamp":1712611054.3955612,"name":"offline","context":{"idset":"9785"}}
+{"timestamp":1712611054.397198,"name":"offline","context":{"idset":"9786"}}
+{"timestamp":1712611054.3989191,"name":"offline","context":{"idset":"9788"}}
+{"timestamp":1712611054.4006872,"name":"offline","context":{"idset":"9789"}}
+{"timestamp":1712611054.4023359,"name":"offline","context":{"idset":"9791"}}
+{"timestamp":1712611054.4300363,"name":"offline","context":{"idset":"9792"}}
+{"timestamp":1712611054.4658339,"name":"offline","context":{"idset":"9793"}}
+{"timestamp":1712611054.5017984,"name":"offline","context":{"idset":"9794"}}
+{"timestamp":1712611054.5289361,"name":"offline","context":{"idset":"9795"}}
+{"timestamp":1712611054.530551,"name":"offline","context":{"idset":"9796"}}
+{"timestamp":1712611054.5322185,"name":"offline","context":{"idset":"9797"}}
+{"timestamp":1712611054.5338247,"name":"offline","context":{"idset":"9798"}}
+{"timestamp":1712611054.5354066,"name":"offline","context":{"idset":"9799"}}
+{"timestamp":1712611054.5369639,"name":"offline","context":{"idset":"9800"}}
+{"timestamp":1712611054.5386198,"name":"offline","context":{"idset":"9801"}}
+{"timestamp":1712611054.5402212,"name":"offline","context":{"idset":"9802"}}
+{"timestamp":1712611054.5418553,"name":"offline","context":{"idset":"9803"}}
+{"timestamp":1712611054.5435588,"name":"offline","context":{"idset":"9804"}}
+{"timestamp":1712611054.5451281,"name":"offline","context":{"idset":"9805"}}
+{"timestamp":1712611054.5466738,"name":"offline","context":{"idset":"9806"}}
+{"timestamp":1712611054.5482137,"name":"offline","context":{"idset":"9807"}}
+{"timestamp":1712611054.5498228,"name":"offline","context":{"idset":"9808"}}
+{"timestamp":1712611054.5514884,"name":"offline","context":{"idset":"9809"}}
+{"timestamp":1712611054.5615661,"name":"offline","context":{"idset":"9810"}}
+{"timestamp":1712611054.5971563,"name":"offline","context":{"idset":"9811"}}
+{"timestamp":1712611054.6327167,"name":"offline","context":{"idset":"9812"}}
+{"timestamp":1712611054.6683135,"name":"offline","context":{"idset":"9813"}}
+{"timestamp":1712611054.6698782,"name":"offline","context":{"idset":"9814"}}
+{"timestamp":1712611054.6715496,"name":"offline","context":{"idset":"9815"}}
+{"timestamp":1712611054.6730924,"name":"offline","context":{"idset":"9816"}}
+{"timestamp":1712611054.6746812,"name":"offline","context":{"idset":"9817"}}
+{"timestamp":1712611054.6763227,"name":"offline","context":{"idset":"9818"}}
+{"timestamp":1712611054.6779647,"name":"offline","context":{"idset":"9819"}}
+{"timestamp":1712611054.6795127,"name":"offline","context":{"idset":"9820"}}
+{"timestamp":1712611054.6810322,"name":"offline","context":{"idset":"9821"}}
+{"timestamp":1712611054.6826217,"name":"offline","context":{"idset":"9822"}}
+{"timestamp":1712611054.6842489,"name":"offline","context":{"idset":"9823"}}
+{"timestamp":1712611054.6858523,"name":"offline","context":{"idset":"9824"}}
+{"timestamp":1712611054.6874487,"name":"offline","context":{"idset":"9825"}}
+{"timestamp":1712611054.6890254,"name":"offline","context":{"idset":"9826"}}
+{"timestamp":1712611054.6905713,"name":"offline","context":{"idset":"9827"}}
+{"timestamp":1712611054.6921594,"name":"offline","context":{"idset":"9828"}}
+{"timestamp":1712611054.6937954,"name":"offline","context":{"idset":"9829"}}
+{"timestamp":1712611054.6954117,"name":"offline","context":{"idset":"9830"}}
+{"timestamp":1712611054.6970322,"name":"offline","context":{"idset":"9831"}}
+{"timestamp":1712611054.6986222,"name":"offline","context":{"idset":"9832"}}
+{"timestamp":1712611054.7001729,"name":"offline","context":{"idset":"9833"}}
+{"timestamp":1712611054.7017667,"name":"offline","context":{"idset":"9834"}}
+{"timestamp":1712611054.7203598,"name":"offline","context":{"idset":"9835"}}
+{"timestamp":1712611054.7558808,"name":"offline","context":{"idset":"9836"}}
+{"timestamp":1712611054.7971439,"name":"offline","context":{"idset":"9837"}}
+{"timestamp":1712611054.8323221,"name":"offline","context":{"idset":"9838"}}
+{"timestamp":1712611054.8680294,"name":"offline","context":{"idset":"9839"}}
+{"timestamp":1712611054.8781729,"name":"offline","context":{"idset":"9840"}}
+{"timestamp":1712611054.879796,"name":"offline","context":{"idset":"9841"}}
+{"timestamp":1712611054.8814223,"name":"offline","context":{"idset":"9842"}}
+{"timestamp":1712611054.882972,"name":"offline","context":{"idset":"9843"}}
+{"timestamp":1712611054.8845663,"name":"offline","context":{"idset":"9844"}}
+{"timestamp":1712611054.8861272,"name":"offline","context":{"idset":"9973"}}
+{"timestamp":1712611054.8876908,"name":"offline","context":{"idset":"9974"}}
+{"timestamp":1712611054.8893044,"name":"offline","context":{"idset":"9975"}}
+{"timestamp":1712611054.8909018,"name":"offline","context":{"idset":"9976"}}
+{"timestamp":1712611054.8925502,"name":"offline","context":{"idset":"9977"}}
+{"timestamp":1712611054.8940639,"name":"offline","context":{"idset":"9978"}}
+{"timestamp":1712611054.8956699,"name":"offline","context":{"idset":"9979"}}
+{"timestamp":1712611054.8972499,"name":"offline","context":{"idset":"9980"}}
+{"timestamp":1712611054.8987739,"name":"offline","context":{"idset":"9981"}}
+{"timestamp":1712611054.900311,"name":"offline","context":{"idset":"9982"}}
+{"timestamp":1712611054.9018893,"name":"offline","context":{"idset":"9983"}}
+{"timestamp":1712611054.9034824,"name":"offline","context":{"idset":"9984"}}
+{"timestamp":1712611054.9050531,"name":"offline","context":{"idset":"9985"}}
+{"timestamp":1712611054.9065948,"name":"offline","context":{"idset":"9986"}}
+{"timestamp":1712611054.908134,"name":"offline","context":{"idset":"9987"}}
+{"timestamp":1712611054.9096682,"name":"offline","context":{"idset":"9988"}}
+{"timestamp":1712611054.9113111,"name":"offline","context":{"idset":"9989"}}
+{"timestamp":1712611054.9129205,"name":"offline","context":{"idset":"9990"}}
+{"timestamp":1712611054.9145126,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1712611054.916048,"name":"offline","context":{"idset":"9992"}}
+{"timestamp":1712611054.9175577,"name":"offline","context":{"idset":"9993"}}
+{"timestamp":1712611054.9191101,"name":"offline","context":{"idset":"9994"}}
+{"timestamp":1712611054.9378211,"name":"offline","context":{"idset":"9995"}}
+{"timestamp":1712611054.9739571,"name":"offline","context":{"idset":"9996"}}
+{"timestamp":1712611055.0141518,"name":"offline","context":{"idset":"9997"}}
+{"timestamp":1712611055.0496705,"name":"offline","context":{"idset":"9998"}}
+{"timestamp":1712611055.0856657,"name":"offline","context":{"idset":"9999"}}
+{"timestamp":1712611055.1226058,"name":"offline","context":{"idset":"10000"}}
+{"timestamp":1712611055.1605268,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1712611055.1969628,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1712611055.1986611,"name":"offline","context":{"idset":"10003"}}
+{"timestamp":1712611055.2001948,"name":"offline","context":{"idset":"10004"}}
+{"timestamp":1712611055.2017784,"name":"offline","context":{"idset":"10005"}}
+{"timestamp":1712611055.2033746,"name":"offline","context":{"idset":"10006"}}
+{"timestamp":1712611055.2049515,"name":"offline","context":{"idset":"10008"}}
+{"timestamp":1712611055.206533,"name":"offline","context":{"idset":"10009"}}
+{"timestamp":1712611055.208097,"name":"offline","context":{"idset":"10010"}}
+{"timestamp":1712611055.2096288,"name":"offline","context":{"idset":"10011"}}
+{"timestamp":1712611055.2111075,"name":"offline","context":{"idset":"10012"}}
+{"timestamp":1712611055.2126522,"name":"offline","context":{"idset":"10013"}}
+{"timestamp":1712611055.2141545,"name":"offline","context":{"idset":"10014"}}
+{"timestamp":1712611055.2157202,"name":"offline","context":{"idset":"10015"}}
+{"timestamp":1712611055.217293,"name":"offline","context":{"idset":"10016"}}
+{"timestamp":1712611055.2187722,"name":"offline","context":{"idset":"10017"}}
+{"timestamp":1712611055.2202778,"name":"offline","context":{"idset":"10018"}}
+{"timestamp":1712611055.2218218,"name":"offline","context":{"idset":"10019"}}
+{"timestamp":1712611055.2235134,"name":"offline","context":{"idset":"10020"}}
+{"timestamp":1712611055.2250197,"name":"offline","context":{"idset":"10021"}}
+{"timestamp":1712611055.2265551,"name":"offline","context":{"idset":"10022"}}
+{"timestamp":1712611055.2280502,"name":"offline","context":{"idset":"10023"}}
+{"timestamp":1712611055.2295721,"name":"offline","context":{"idset":"10024"}}
+{"timestamp":1712611055.2310913,"name":"offline","context":{"idset":"10025"}}
+{"timestamp":1712611055.2326262,"name":"offline","context":{"idset":"10026"}}
+{"timestamp":1712611055.2342279,"name":"offline","context":{"idset":"10027"}}
+{"timestamp":1712611055.2358122,"name":"offline","context":{"idset":"10028"}}
+{"timestamp":1712611055.2373703,"name":"offline","context":{"idset":"10029"}}
+{"timestamp":1712611055.2388961,"name":"offline","context":{"idset":"10030"}}
+{"timestamp":1712611055.2403994,"name":"offline","context":{"idset":"10031"}}
+{"timestamp":1712611055.2419682,"name":"offline","context":{"idset":"10032"}}
+{"timestamp":1712611055.243474,"name":"offline","context":{"idset":"10033"}}
+{"timestamp":1712611055.2449806,"name":"offline","context":{"idset":"10034"}}
+{"timestamp":1712611055.2465403,"name":"offline","context":{"idset":"10035"}}
+{"timestamp":1712611055.2480659,"name":"offline","context":{"idset":"10036"}}
+{"timestamp":1712611055.2580409,"name":"offline","context":{"idset":"10037"}}
+{"timestamp":1712611055.2939076,"name":"offline","context":{"idset":"10038"}}
+{"timestamp":1712611055.3299572,"name":"offline","context":{"idset":"10039"}}
+{"timestamp":1712611055.3666782,"name":"offline","context":{"idset":"10040"}}
+{"timestamp":1712611055.4022913,"name":"offline","context":{"idset":"10041"}}
+{"timestamp":1712611055.4377124,"name":"offline","context":{"idset":"10042"}}
+{"timestamp":1712611055.4733784,"name":"offline","context":{"idset":"10043"}}
+{"timestamp":1712611055.508853,"name":"offline","context":{"idset":"10044"}}
+{"timestamp":1712611055.54478,"name":"offline","context":{"idset":"10045"}}
+{"timestamp":1712611055.5741446,"name":"offline","context":{"idset":"10046"}}
+{"timestamp":1712611055.5756674,"name":"offline","context":{"idset":"10047"}}
+{"timestamp":1712611055.5772288,"name":"offline","context":{"idset":"10048"}}
+{"timestamp":1712611055.5789344,"name":"offline","context":{"idset":"10049"}}
+{"timestamp":1712611055.5804708,"name":"offline","context":{"idset":"10050"}}
+{"timestamp":1712611055.5819147,"name":"offline","context":{"idset":"10051"}}
+{"timestamp":1712611055.5834024,"name":"offline","context":{"idset":"10052"}}
+{"timestamp":1712611055.5849154,"name":"offline","context":{"idset":"10053"}}
+{"timestamp":1712611055.5868871,"name":"offline","context":{"idset":"10054"}}
+{"timestamp":1712611055.5888307,"name":"offline","context":{"idset":"10055"}}
+{"timestamp":1712611055.5907476,"name":"offline","context":{"idset":"10056"}}
+{"timestamp":1712611055.5926192,"name":"offline","context":{"idset":"10057"}}
+{"timestamp":1712611055.5946023,"name":"offline","context":{"idset":"10058"}}
+{"timestamp":1712611055.5964754,"name":"offline","context":{"idset":"10059"}}
+{"timestamp":1712611055.5983336,"name":"offline","context":{"idset":"10060"}}
+{"timestamp":1712611055.6005893,"name":"offline","context":{"idset":"10061"}}
+{"timestamp":1712611055.6031647,"name":"offline","context":{"idset":"10062"}}
+{"timestamp":1712611055.6057663,"name":"offline","context":{"idset":"10063"}}
+{"timestamp":1712611055.6081045,"name":"offline","context":{"idset":"10064"}}
+{"timestamp":1712611055.6099687,"name":"offline","context":{"idset":"10065"}}
+{"timestamp":1712611055.6115849,"name":"offline","context":{"idset":"10066"}}
+{"timestamp":1712611055.6243899,"name":"offline","context":{"idset":"10067"}}
+{"timestamp":1712611055.6596298,"name":"offline","context":{"idset":"10068"}}
+{"timestamp":1712611055.6984224,"name":"offline","context":{"idset":"10069"}}
+{"timestamp":1712611055.7364063,"name":"offline","context":{"idset":"10071"}}
+{"timestamp":1712611055.773068,"name":"offline","context":{"idset":"10072"}}
+{"timestamp":1712611055.8098524,"name":"offline","context":{"idset":"10073"}}
+{"timestamp":1712611055.8458788,"name":"offline","context":{"idset":"10074"}}
+{"timestamp":1712611055.8820617,"name":"offline","context":{"idset":"10075"}}
+{"timestamp":1712611055.9178402,"name":"offline","context":{"idset":"10076"}}
+{"timestamp":1712611055.953166,"name":"offline","context":{"idset":"10077"}}
+{"timestamp":1712611055.9713535,"name":"offline","context":{"idset":"10078"}}
+{"timestamp":1712611055.9727697,"name":"offline","context":{"idset":"10079"}}
+{"timestamp":1712611055.9741139,"name":"offline","context":{"idset":"10080"}}
+{"timestamp":1712611055.975477,"name":"offline","context":{"idset":"10082"}}
+{"timestamp":1712611055.9768524,"name":"offline","context":{"idset":"10083"}}
+{"timestamp":1712611055.9782345,"name":"offline","context":{"idset":"10084"}}
+{"timestamp":1712611055.9796293,"name":"offline","context":{"idset":"10085"}}
+{"timestamp":1712611055.980979,"name":"offline","context":{"idset":"10086"}}
+{"timestamp":1712611055.9823372,"name":"offline","context":{"idset":"10087"}}
+{"timestamp":1712611055.9836698,"name":"offline","context":{"idset":"10088"}}
+{"timestamp":1712611055.9850287,"name":"offline","context":{"idset":"10089"}}
+{"timestamp":1712611055.9863777,"name":"offline","context":{"idset":"10090"}}
+{"timestamp":1712611055.9877093,"name":"offline","context":{"idset":"10091"}}
+{"timestamp":1712611055.9890838,"name":"offline","context":{"idset":"10092"}}
+{"timestamp":1712611055.9905002,"name":"offline","context":{"idset":"10093"}}
+{"timestamp":1712611055.9918706,"name":"offline","context":{"idset":"10094"}}
+{"timestamp":1712611055.9931819,"name":"offline","context":{"idset":"10095"}}
+{"timestamp":1712611056.0176055,"name":"offline","context":{"idset":"10096"}}
+{"timestamp":1712611056.0491238,"name":"offline","context":{"idset":"10097"}}
+{"timestamp":1712611056.0792804,"name":"offline","context":{"idset":"10098"}}
+{"timestamp":1712611056.1089115,"name":"offline","context":{"idset":"10099"}}
+{"timestamp":1712611056.138968,"name":"offline","context":{"idset":"10100"}}
+{"timestamp":1712611056.1406522,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1712611056.1419361,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1712611056.1436701,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1712611056.145463,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1712611056.1471486,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1712611056.1488891,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1712611056.150357,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1712611056.1516063,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1712611056.1527913,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1712611056.1539252,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1712611056.1551323,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1712611056.163383,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1712611056.1916494,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1712611056.2188222,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1712611056.2448342,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1712611056.2702153,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1712611056.2953238,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1712611056.3144965,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1712611056.3155696,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1712611056.3168523,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1712611056.3183124,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1712611056.3196983,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1712611056.3207445,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1712611056.3219521,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1712611056.3231626,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1712611056.3241537,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1712611056.3479571,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1712611056.3546524,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1712611056.3556027,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1712611056.3565423,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1712611056.3575087,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1712611056.3584554,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1712611056.3594189,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1712611056.3603806,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1712611056.3613117,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1712611056.3622468,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1712611056.3631809,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1712611056.3641181,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1712611056.3650517,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1712611056.3826079,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1712611056.4056554,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1712611056.4282954,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1712611056.4504545,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1712611056.4513359,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1712611056.4521883,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1712611056.4530363,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1712611056.4538872,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1712611056.4599745,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1712611056.4813807,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1712611056.5021749,"name":"offline","context":{"idset":"10153"}}
+{"timestamp":1712611056.5080686,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1712611056.5089052,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1712611056.5097935,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1712611056.5106218,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1712611056.511472,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1712611056.5273993,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1712611056.5332768,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1712611056.5341055,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1712611056.5349956,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1712611056.5358369,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1712611056.5366645,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1712611056.5374711,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1712611056.5383313,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1712611056.5443244,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1712611056.5502136,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1712611056.5510714,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1712611056.5519176,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1712611056.5527718,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1712611056.5535808,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1712611056.554405,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1712611056.5552042,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1712611056.5560708,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1712611056.5568838,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1712611056.5577202,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1712611056.5585334,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1712611056.5593526,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1712611056.5651932,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1712611056.5860884,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1712611056.6070671,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1712611056.612993,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1712611056.613795,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1712611056.614634,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1712611056.6154513,"name":"offline","context":{"idset":"10188"}}
+{"timestamp":1712611056.6162777,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1712611056.6170902,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1712611056.6179168,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1712611056.6188178,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1712611056.6196434,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1712611056.6205051,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1712611056.6213465,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1712611056.6425006,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1712611056.6635728,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1712611056.6845021,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1712611056.6853449,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1712611056.686156,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1712611056.6869595,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1712611056.6877916,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1712611056.6886435,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1712611056.6894686,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1712611056.6902857,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1712611056.6910863,"name":"offline","context":{"idset":"10206"}}
+{"timestamp":1712611056.6918862,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1712611056.6927118,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1712611056.6935236,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1712611056.6943271,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1712611056.6951153,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1712611056.6959317,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1712611056.7068958,"name":"offline","context":{"idset":"10213"}}
+{"timestamp":1712611056.7278185,"name":"offline","context":{"idset":"10214"}}
+{"timestamp":1712611056.7488022,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1712611056.7699287,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1712611056.775867,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1712611056.7767162,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1712611056.7775493,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1712611056.7783811,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1712611056.7792034,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1712611056.7800722,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1712611056.7809262,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1712611056.7817597,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1712611056.7825918,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1712611056.7834249,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1712611056.7843196,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1712611056.7851505,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1712611056.7860746,"name":"offline","context":{"idset":"10229"}}
+{"timestamp":1712611056.7869096,"name":"offline","context":{"idset":"10230"}}
+{"timestamp":1712611056.7877316,"name":"offline","context":{"idset":"10231"}}
+{"timestamp":1712611056.788584,"name":"offline","context":{"idset":"10232"}}
+{"timestamp":1712611056.7894132,"name":"offline","context":{"idset":"10233"}}
+{"timestamp":1712611056.7902334,"name":"offline","context":{"idset":"10234"}}
+{"timestamp":1712611056.7910972,"name":"offline","context":{"idset":"10235"}}
+{"timestamp":1712611056.7919204,"name":"offline","context":{"idset":"10236"}}
+{"timestamp":1712611056.7927322,"name":"offline","context":{"idset":"10237"}}
+{"timestamp":1712611056.8137848,"name":"offline","context":{"idset":"10238"}}
+{"timestamp":1712611056.8357196,"name":"offline","context":{"idset":"10239"}}
+{"timestamp":1712611056.8571682,"name":"offline","context":{"idset":"10240"}}
+{"timestamp":1712611056.8785105,"name":"offline","context":{"idset":"10241"}}
+{"timestamp":1712611056.899971,"name":"offline","context":{"idset":"10242"}}
+{"timestamp":1712611056.9162884,"name":"offline","context":{"idset":"10243"}}
+{"timestamp":1712611056.917094,"name":"offline","context":{"idset":"10244"}}
+{"timestamp":1712611056.9179397,"name":"offline","context":{"idset":"10245"}}
+{"timestamp":1712611056.9187787,"name":"offline","context":{"idset":"10246"}}
+{"timestamp":1712611056.9195809,"name":"offline","context":{"idset":"10247"}}
+{"timestamp":1712611056.9204252,"name":"offline","context":{"idset":"10248"}}
+{"timestamp":1712611056.921243,"name":"offline","context":{"idset":"10249"}}
+{"timestamp":1712611056.9221056,"name":"offline","context":{"idset":"10250"}}
+{"timestamp":1712611056.922905,"name":"offline","context":{"idset":"10251"}}
+{"timestamp":1712611056.923708,"name":"offline","context":{"idset":"10252"}}
+{"timestamp":1712611056.9244928,"name":"offline","context":{"idset":"10253"}}
+{"timestamp":1712611056.925344,"name":"offline","context":{"idset":"10254"}}
+{"timestamp":1712611056.9261782,"name":"offline","context":{"idset":"10255"}}
+{"timestamp":1712611056.9269738,"name":"offline","context":{"idset":"10256"}}
+{"timestamp":1712611056.9277968,"name":"offline","context":{"idset":"10257"}}
+{"timestamp":1712611056.9285855,"name":"offline","context":{"idset":"10258"}}
+{"timestamp":1712611056.9293742,"name":"offline","context":{"idset":"10259"}}
+{"timestamp":1712611056.930145,"name":"offline","context":{"idset":"10260"}}
+{"timestamp":1712611056.9309227,"name":"offline","context":{"idset":"10263"}}
+{"timestamp":1712611056.9317441,"name":"offline","context":{"idset":"10264"}}
+{"timestamp":1712611056.9326022,"name":"offline","context":{"idset":"10265"}}
+{"timestamp":1712611056.9334011,"name":"offline","context":{"idset":"10266"}}
+{"timestamp":1712611056.9341714,"name":"offline","context":{"idset":"10267"}}
+{"timestamp":1712611056.9349763,"name":"offline","context":{"idset":"10268"}}
+{"timestamp":1712611056.9358284,"name":"offline","context":{"idset":"10269"}}
+{"timestamp":1712611056.9366353,"name":"offline","context":{"idset":"10270"}}
+{"timestamp":1712611056.9374559,"name":"offline","context":{"idset":"10271"}}
+{"timestamp":1712611056.9389124,"name":"offline","context":{"idset":"10272"}}
+{"timestamp":1712611056.9397056,"name":"offline","context":{"idset":"10273"}}
+{"timestamp":1712611056.9474597,"name":"offline","context":{"idset":"10274"}}
+{"timestamp":1712611056.9555669,"name":"offline","context":{"idset":"10275"}}
+{"timestamp":1712611056.956382,"name":"offline","context":{"idset":"10276"}}
+{"timestamp":1712611056.9840848,"name":"offline","context":{"idset":"10277"}}
+{"timestamp":1712611057.0050738,"name":"offline","context":{"idset":"10278"}}
+{"timestamp":1712611057.0259848,"name":"offline","context":{"idset":"10279"}}
+{"timestamp":1712611057.0468321,"name":"offline","context":{"idset":"10280"}}
+{"timestamp":1712611057.0678725,"name":"offline","context":{"idset":"10281"}}
+{"timestamp":1712611057.0888867,"name":"offline","context":{"idset":"10282"}}
+{"timestamp":1712611057.1050751,"name":"offline","context":{"idset":"10283"}}
+{"timestamp":1712611057.1058631,"name":"offline","context":{"idset":"10284"}}
+{"timestamp":1712611057.1066434,"name":"offline","context":{"idset":"10285"}}
+{"timestamp":1712611057.1074045,"name":"offline","context":{"idset":"10286"}}
+{"timestamp":1712611057.1081524,"name":"offline","context":{"idset":"10287"}}
+{"timestamp":1712611057.1089416,"name":"offline","context":{"idset":"10288"}}
+{"timestamp":1712611057.1097162,"name":"offline","context":{"idset":"10290"}}
+{"timestamp":1712611057.1104839,"name":"offline","context":{"idset":"10291"}}
+{"timestamp":1712611057.1112463,"name":"offline","context":{"idset":"10292"}}
+{"timestamp":1712611057.112042,"name":"offline","context":{"idset":"10293"}}
+{"timestamp":1712611057.1127973,"name":"offline","context":{"idset":"10294"}}
+{"timestamp":1712611057.1135695,"name":"offline","context":{"idset":"10295"}}
+{"timestamp":1712611057.1143277,"name":"offline","context":{"idset":"10296"}}
+{"timestamp":1712611057.1150806,"name":"offline","context":{"idset":"10297"}}
+{"timestamp":1712611057.1158681,"name":"offline","context":{"idset":"10298"}}
+{"timestamp":1712611057.1166332,"name":"offline","context":{"idset":"10299"}}
+{"timestamp":1712611057.1174014,"name":"offline","context":{"idset":"10300"}}
+{"timestamp":1712611057.124944,"name":"offline","context":{"idset":"10301"}}
+{"timestamp":1712611057.1257095,"name":"offline","context":{"idset":"10302"}}
+{"timestamp":1712611057.1334596,"name":"offline","context":{"idset":"10304"}}
+{"timestamp":1712611057.1418366,"name":"offline","context":{"idset":"10305"}}
+{"timestamp":1712611057.1425965,"name":"offline","context":{"idset":"10306"}}
+{"timestamp":1712611057.1505196,"name":"offline","context":{"idset":"10307"}}
+{"timestamp":1712611057.1515002,"name":"offline","context":{"idset":"10308"}}
+{"timestamp":1712611057.1665525,"name":"offline","context":{"idset":"10309"}}
+{"timestamp":1712611057.1747615,"name":"offline","context":{"idset":"10310"}}
+{"timestamp":1712611057.1755354,"name":"offline","context":{"idset":"10311"}}
+{"timestamp":1712611057.1832051,"name":"offline","context":{"idset":"10312"}}
+{"timestamp":1712611057.1922917,"name":"offline","context":{"idset":"10313"}}
+{"timestamp":1712611057.1930428,"name":"offline","context":{"idset":"10314"}}
+{"timestamp":1712611057.2035005,"name":"offline","context":{"idset":"10315"}}
+{"timestamp":1712611057.2135465,"name":"offline","context":{"idset":"10316"}}
+{"timestamp":1712611057.2235849,"name":"offline","context":{"idset":"10317"}}
+{"timestamp":1712611057.2335806,"name":"offline","context":{"idset":"10318"}}
+{"timestamp":1712611057.2436111,"name":"offline","context":{"idset":"10319"}}
+{"timestamp":1712611057.2443767,"name":"offline","context":{"idset":"10320"}}
+{"timestamp":1712611057.2541752,"name":"offline","context":{"idset":"10321"}}
+{"timestamp":1712611057.2642038,"name":"offline","context":{"idset":"10322"}}
+{"timestamp":1712611057.2745254,"name":"offline","context":{"idset":"10323"}}
+{"timestamp":1712611057.2841651,"name":"offline","context":{"idset":"10324"}}
+{"timestamp":1712611057.2940474,"name":"offline","context":{"idset":"10325"}}
+{"timestamp":1712611057.304213,"name":"offline","context":{"idset":"10326"}}
+{"timestamp":1712611057.3141069,"name":"offline","context":{"idset":"10327"}}
+{"timestamp":1712611057.3242579,"name":"offline","context":{"idset":"10328"}}
+{"timestamp":1712611057.3250809,"name":"offline","context":{"idset":"10329"}}
+{"timestamp":1712611057.3349259,"name":"offline","context":{"idset":"10330"}}
+{"timestamp":1712611057.3450117,"name":"offline","context":{"idset":"10331"}}
+{"timestamp":1712611057.3570263,"name":"offline","context":{"idset":"10332"}}
+{"timestamp":1712611057.3799233,"name":"offline","context":{"idset":"10333"}}
+{"timestamp":1712611057.4028063,"name":"offline","context":{"idset":"10334"}}
+{"timestamp":1712611057.4254217,"name":"offline","context":{"idset":"10335"}}
+{"timestamp":1712611057.4478738,"name":"offline","context":{"idset":"10336"}}
+{"timestamp":1712611057.4700701,"name":"offline","context":{"idset":"10337"}}
+{"timestamp":1712611057.4923925,"name":"offline","context":{"idset":"10338"}}
+{"timestamp":1712611057.5143569,"name":"offline","context":{"idset":"10339"}}
+{"timestamp":1712611057.5359266,"name":"offline","context":{"idset":"10340"}}
+{"timestamp":1712611057.557256,"name":"offline","context":{"idset":"10341"}}
+{"timestamp":1712611057.5787737,"name":"offline","context":{"idset":"10342"}}
+{"timestamp":1712611057.6001191,"name":"offline","context":{"idset":"10345"}}
+{"timestamp":1712611057.6008773,"name":"offline","context":{"idset":"10346"}}
+{"timestamp":1712611057.6016774,"name":"offline","context":{"idset":"10347"}}
+{"timestamp":1712611057.6024518,"name":"offline","context":{"idset":"10348"}}
+{"timestamp":1712611057.6031938,"name":"offline","context":{"idset":"10349"}}
+{"timestamp":1712611057.6039252,"name":"offline","context":{"idset":"10350"}}
+{"timestamp":1712611057.6047032,"name":"offline","context":{"idset":"10351"}}
+{"timestamp":1712611057.605469,"name":"offline","context":{"idset":"10352"}}
+{"timestamp":1712611057.6062038,"name":"offline","context":{"idset":"10353"}}
+{"timestamp":1712611057.6069586,"name":"offline","context":{"idset":"10354"}}
+{"timestamp":1712611057.6077161,"name":"offline","context":{"idset":"10357"}}
+{"timestamp":1712611057.6084871,"name":"offline","context":{"idset":"10358"}}
+{"timestamp":1712611057.6092172,"name":"offline","context":{"idset":"10359"}}
+{"timestamp":1712611057.6099544,"name":"offline","context":{"idset":"10360"}}
+{"timestamp":1712611057.6107085,"name":"offline","context":{"idset":"10361"}}
+{"timestamp":1712611057.6114609,"name":"offline","context":{"idset":"10362"}}
+{"timestamp":1712611057.6122117,"name":"offline","context":{"idset":"10363"}}
+{"timestamp":1712611057.6129985,"name":"offline","context":{"idset":"10364"}}
+{"timestamp":1712611057.6137452,"name":"offline","context":{"idset":"10365"}}
+{"timestamp":1712611057.6151824,"name":"offline","context":{"idset":"10366"}}
+{"timestamp":1712611057.6248417,"name":"offline","context":{"idset":"10367"}}
+{"timestamp":1712611057.6345565,"name":"offline","context":{"idset":"10368"}}
+{"timestamp":1712611057.6441689,"name":"offline","context":{"idset":"10369"}}
+{"timestamp":1712611057.6449506,"name":"offline","context":{"idset":"10370"}}
+{"timestamp":1712611057.6542382,"name":"offline","context":{"idset":"10371"}}
+{"timestamp":1712611057.6638606,"name":"offline","context":{"idset":"10372"}}
+{"timestamp":1712611057.6646476,"name":"offline","context":{"idset":"10373"}}
+{"timestamp":1712611057.6739109,"name":"offline","context":{"idset":"10374"}}
+{"timestamp":1712611057.6835752,"name":"offline","context":{"idset":"10375"}}
+{"timestamp":1712611057.6936393,"name":"offline","context":{"idset":"10376"}}
+{"timestamp":1712611057.7037241,"name":"offline","context":{"idset":"10377"}}
+{"timestamp":1712611057.7139678,"name":"offline","context":{"idset":"10378"}}
+{"timestamp":1712611057.7244129,"name":"offline","context":{"idset":"10379"}}
+{"timestamp":1712611057.7252085,"name":"offline","context":{"idset":"10380"}}
+{"timestamp":1712611057.7347858,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1712611057.7498696,"name":"offline","context":{"idset":"10382"}}
+{"timestamp":1712611057.7732985,"name":"offline","context":{"idset":"10383"}}
+{"timestamp":1712611057.7965548,"name":"offline","context":{"idset":"10384"}}
+{"timestamp":1712611057.8194118,"name":"offline","context":{"idset":"10385"}}
+{"timestamp":1712611057.8420675,"name":"offline","context":{"idset":"10386"}}
+{"timestamp":1712611057.8648157,"name":"offline","context":{"idset":"10387"}}
+{"timestamp":1712611057.8876388,"name":"offline","context":{"idset":"10388"}}
+{"timestamp":1712611057.910444,"name":"offline","context":{"idset":"10389"}}
+{"timestamp":1712611057.9330866,"name":"offline","context":{"idset":"10390"}}
+{"timestamp":1712611057.9556749,"name":"offline","context":{"idset":"10391"}}
+{"timestamp":1712611057.9779332,"name":"offline","context":{"idset":"10392"}}
+{"timestamp":1712611058.0001569,"name":"offline","context":{"idset":"10393"}}
+{"timestamp":1712611058.0225394,"name":"offline","context":{"idset":"10394"}}
+{"timestamp":1712611058.0443892,"name":"offline","context":{"idset":"10395"}}
+{"timestamp":1712611058.0661087,"name":"offline","context":{"idset":"10397"}}
+{"timestamp":1712611058.0879033,"name":"offline","context":{"idset":"10398"}}
+{"timestamp":1712611058.0968087,"name":"offline","context":{"idset":"10399"}}
+{"timestamp":1712611058.0975459,"name":"offline","context":{"idset":"10400"}}
+{"timestamp":1712611058.0983188,"name":"offline","context":{"idset":"10401"}}
+{"timestamp":1712611058.0991077,"name":"offline","context":{"idset":"10402"}}
+{"timestamp":1712611058.1002152,"name":"offline","context":{"idset":"10403"}}
+{"timestamp":1712611058.1013379,"name":"offline","context":{"idset":"10404"}}
+{"timestamp":1712611058.1024067,"name":"offline","context":{"idset":"10405"}}
+{"timestamp":1712611058.1035526,"name":"offline","context":{"idset":"10406"}}
+{"timestamp":1712611058.1046565,"name":"offline","context":{"idset":"10407"}}
+{"timestamp":1712611058.1054401,"name":"offline","context":{"idset":"10408"}}
+{"timestamp":1712611058.106174,"name":"offline","context":{"idset":"10409"}}
+{"timestamp":1712611058.106894,"name":"offline","context":{"idset":"10410"}}
+{"timestamp":1712611058.1076307,"name":"offline","context":{"idset":"10411"}}
+{"timestamp":1712611058.1084139,"name":"offline","context":{"idset":"10412"}}
+{"timestamp":1712611058.109112,"name":"offline","context":{"idset":"10413"}}
+{"timestamp":1712611058.1098554,"name":"offline","context":{"idset":"10414"}}
+{"timestamp":1712611058.1105535,"name":"offline","context":{"idset":"10415"}}
+{"timestamp":1712611058.1112332,"name":"offline","context":{"idset":"10416"}}
+{"timestamp":1712611058.111933,"name":"offline","context":{"idset":"10417"}}
+{"timestamp":1712611058.1126373,"name":"offline","context":{"idset":"10418"}}
+{"timestamp":1712611058.1133544,"name":"offline","context":{"idset":"10419"}}
+{"timestamp":1712611058.1140847,"name":"offline","context":{"idset":"10420"}}
+{"timestamp":1712611058.1147993,"name":"offline","context":{"idset":"10421"}}
+{"timestamp":1712611058.1180005,"name":"offline","context":{"idset":"10422"}}
+{"timestamp":1712611058.1271865,"name":"offline","context":{"idset":"10423"}}
+{"timestamp":1712611058.1450088,"name":"offline","context":{"idset":"10424"}}
+{"timestamp":1712611058.1540005,"name":"offline","context":{"idset":"10425"}}
+{"timestamp":1712611058.178746,"name":"offline","context":{"idset":"10426"}}
+{"timestamp":1712611058.2007172,"name":"offline","context":{"idset":"10427"}}
+{"timestamp":1712611058.2225614,"name":"offline","context":{"idset":"10428"}}
+{"timestamp":1712611058.2439935,"name":"offline","context":{"idset":"10429"}}
+{"timestamp":1712611058.2651281,"name":"offline","context":{"idset":"10430"}}
+{"timestamp":1712611058.2862201,"name":"offline","context":{"idset":"10431"}}
+{"timestamp":1712611058.3072429,"name":"offline","context":{"idset":"10432"}}
+{"timestamp":1712611058.3291731,"name":"offline","context":{"idset":"10433"}}
+{"timestamp":1712611058.3504982,"name":"offline","context":{"idset":"10434"}}
+{"timestamp":1712611058.3716304,"name":"offline","context":{"idset":"10435"}}
+{"timestamp":1712611058.3926914,"name":"offline","context":{"idset":"10436"}}
+{"timestamp":1712611058.4136472,"name":"offline","context":{"idset":"10437"}}
+{"timestamp":1712611058.4351363,"name":"offline","context":{"idset":"10438"}}
+{"timestamp":1712611058.4563496,"name":"offline","context":{"idset":"10439"}}
+{"timestamp":1712611058.4673586,"name":"offline","context":{"idset":"10440"}}
+{"timestamp":1712611058.4680598,"name":"offline","context":{"idset":"10441"}}
+{"timestamp":1712611058.468791,"name":"offline","context":{"idset":"10442"}}
+{"timestamp":1712611058.4695058,"name":"offline","context":{"idset":"10443"}}
+{"timestamp":1712611058.4701803,"name":"offline","context":{"idset":"10444"}}
+{"timestamp":1712611058.4708667,"name":"offline","context":{"idset":"10445"}}
+{"timestamp":1712611058.4715457,"name":"offline","context":{"idset":"10446"}}
+{"timestamp":1712611058.472204,"name":"offline","context":{"idset":"10447"}}
+{"timestamp":1712611058.4728985,"name":"offline","context":{"idset":"10448"}}
+{"timestamp":1712611058.4736423,"name":"offline","context":{"idset":"10449"}}
+{"timestamp":1712611058.474345,"name":"offline","context":{"idset":"10450"}}
+{"timestamp":1712611058.4750071,"name":"offline","context":{"idset":"10451"}}
+{"timestamp":1712611058.4756927,"name":"offline","context":{"idset":"10452"}}
+{"timestamp":1712611058.4763732,"name":"offline","context":{"idset":"10453"}}
+{"timestamp":1712611058.4771082,"name":"offline","context":{"idset":"10454"}}
+{"timestamp":1712611058.4778223,"name":"offline","context":{"idset":"10455"}}
+{"timestamp":1712611058.4785247,"name":"offline","context":{"idset":"10456"}}
+{"timestamp":1712611058.4792342,"name":"offline","context":{"idset":"10457"}}
+{"timestamp":1712611058.4799314,"name":"offline","context":{"idset":"10458"}}
+{"timestamp":1712611058.4806046,"name":"offline","context":{"idset":"10459"}}
+{"timestamp":1712611058.4812584,"name":"offline","context":{"idset":"10460"}}
+{"timestamp":1712611058.4819846,"name":"offline","context":{"idset":"10461"}}
+{"timestamp":1712611058.4899452,"name":"offline","context":{"idset":"10462"}}
+{"timestamp":1712611058.4989071,"name":"offline","context":{"idset":"10463"}}
+{"timestamp":1712611058.5128746,"name":"offline","context":{"idset":"10464"}}
+{"timestamp":1712611058.5338104,"name":"offline","context":{"idset":"10465"}}
+{"timestamp":1712611058.5557983,"name":"offline","context":{"idset":"10466"}}
+{"timestamp":1712611058.5768373,"name":"offline","context":{"idset":"10467"}}
+{"timestamp":1712611058.5978551,"name":"offline","context":{"idset":"10468"}}
+{"timestamp":1712611058.6186793,"name":"offline","context":{"idset":"10485"}}
+{"timestamp":1712611058.6398726,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1712611058.6606812,"name":"offline","context":{"idset":"10487"}}
+{"timestamp":1712611058.6814599,"name":"offline","context":{"idset":"10488"}}
+{"timestamp":1712611058.7020926,"name":"offline","context":{"idset":"10489"}}
+{"timestamp":1712611058.722826,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1712611058.7434728,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1712611058.749162,"name":"offline","context":{"idset":"10492"}}
+{"timestamp":1712611058.7498307,"name":"offline","context":{"idset":"10493"}}
+{"timestamp":1712611058.7505178,"name":"offline","context":{"idset":"10494"}}
+{"timestamp":1712611058.7511671,"name":"offline","context":{"idset":"10495"}}
+{"timestamp":1712611058.7518358,"name":"offline","context":{"idset":"10496"}}
+{"timestamp":1712611058.7525125,"name":"offline","context":{"idset":"10497"}}
+{"timestamp":1712611058.7531633,"name":"offline","context":{"idset":"10498"}}
+{"timestamp":1712611058.75384,"name":"offline","context":{"idset":"10499"}}
+{"timestamp":1712611058.7545047,"name":"offline","context":{"idset":"10500"}}
+{"timestamp":1712611058.7551737,"name":"offline","context":{"idset":"10502"}}
+{"timestamp":1712611058.7558343,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1712611058.7565088,"name":"offline","context":{"idset":"10504"}}
+{"timestamp":1712611058.7571688,"name":"offline","context":{"idset":"10505"}}
+{"timestamp":1712611058.7578301,"name":"offline","context":{"idset":"10506"}}
+{"timestamp":1712611058.758498,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1712611058.7591505,"name":"offline","context":{"idset":"10508"}}
+{"timestamp":1712611058.7642779,"name":"offline","context":{"idset":"10509"}}
+{"timestamp":1712611058.7729824,"name":"offline","context":{"idset":"10510"}}
+{"timestamp":1712611058.7914774,"name":"offline","context":{"idset":"10511"}}
+{"timestamp":1712611058.8120697,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1712611058.832818,"name":"offline","context":{"idset":"10513"}}
+{"timestamp":1712611058.8538797,"name":"offline","context":{"idset":"10514"}}
+{"timestamp":1712611058.8753479,"name":"offline","context":{"idset":"10515"}}
+{"timestamp":1712611058.8962994,"name":"offline","context":{"idset":"10516"}}
+{"timestamp":1712611058.9170692,"name":"offline","context":{"idset":"10519"}}
+{"timestamp":1712611058.9380527,"name":"offline","context":{"idset":"10520"}}
+{"timestamp":1712611058.9589553,"name":"offline","context":{"idset":"10521"}}
+{"timestamp":1712611058.9796512,"name":"offline","context":{"idset":"10522"}}
+{"timestamp":1712611059.0004656,"name":"offline","context":{"idset":"10523"}}
+{"timestamp":1712611059.0060904,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1712611059.0067375,"name":"offline","context":{"idset":"10525"}}
+{"timestamp":1712611059.007396,"name":"offline","context":{"idset":"10526"}}
+{"timestamp":1712611059.0080488,"name":"offline","context":{"idset":"10527"}}
+{"timestamp":1712611059.0087256,"name":"offline","context":{"idset":"10528"}}
+{"timestamp":1712611059.0093801,"name":"offline","context":{"idset":"10529"}}
+{"timestamp":1712611059.0100257,"name":"offline","context":{"idset":"10530"}}
+{"timestamp":1712611059.0107038,"name":"offline","context":{"idset":"10531"}}
+{"timestamp":1712611059.0113702,"name":"offline","context":{"idset":"10532"}}
+{"timestamp":1712611059.0120165,"name":"offline","context":{"idset":"10533"}}
+{"timestamp":1712611059.0126634,"name":"offline","context":{"idset":"10534"}}
+{"timestamp":1712611059.0133042,"name":"offline","context":{"idset":"10535"}}
+{"timestamp":1712611059.0139303,"name":"offline","context":{"idset":"10536"}}
+{"timestamp":1712611059.0145824,"name":"offline","context":{"idset":"10537"}}
+{"timestamp":1712611059.0152259,"name":"offline","context":{"idset":"10538"}}
+{"timestamp":1712611059.0158954,"name":"offline","context":{"idset":"10540"}}
+{"timestamp":1712611059.0165482,"name":"offline","context":{"idset":"10541"}}
+{"timestamp":1712611059.0171714,"name":"offline","context":{"idset":"10542"}}
+{"timestamp":1712611059.0178211,"name":"offline","context":{"idset":"10543"}}
+{"timestamp":1712611059.0184882,"name":"offline","context":{"idset":"10544"}}
+{"timestamp":1712611059.0241258,"name":"offline","context":{"idset":"10545"}}
+{"timestamp":1712611059.0451047,"name":"offline","context":{"idset":"10546"}}
+{"timestamp":1712611059.0663095,"name":"offline","context":{"idset":"10547"}}
+{"timestamp":1712611059.0873344,"name":"offline","context":{"idset":"10548"}}
+{"timestamp":1712611059.1083941,"name":"offline","context":{"idset":"10549"}}
+{"timestamp":1712611059.1290326,"name":"offline","context":{"idset":"10550"}}
+{"timestamp":1712611059.1501606,"name":"offline","context":{"idset":"10551"}}
+{"timestamp":1712611059.1715229,"name":"offline","context":{"idset":"10552"}}
+{"timestamp":1712611059.1922216,"name":"offline","context":{"idset":"10553"}}
+{"timestamp":1712611059.2029979,"name":"offline","context":{"idset":"10554"}}
+{"timestamp":1712611059.2036464,"name":"offline","context":{"idset":"10555"}}
+{"timestamp":1712611059.204304,"name":"offline","context":{"idset":"10556"}}
+{"timestamp":1712611059.2049246,"name":"offline","context":{"idset":"10557"}}
+{"timestamp":1712611059.2055485,"name":"offline","context":{"idset":"10558"}}
+{"timestamp":1712611059.2061596,"name":"offline","context":{"idset":"10559"}}
+{"timestamp":1712611059.2068038,"name":"offline","context":{"idset":"10561"}}
+{"timestamp":1712611059.2074375,"name":"offline","context":{"idset":"10562"}}
+{"timestamp":1712611059.2080872,"name":"offline","context":{"idset":"10563"}}
+{"timestamp":1712611059.2087617,"name":"offline","context":{"idset":"10564"}}
+{"timestamp":1712611059.2093842,"name":"offline","context":{"idset":"10565"}}
+{"timestamp":1712611059.2100158,"name":"offline","context":{"idset":"10566"}}
+{"timestamp":1712611059.21065,"name":"offline","context":{"idset":"10567"}}
+{"timestamp":1712611059.2113101,"name":"offline","context":{"idset":"10568"}}
+{"timestamp":1712611059.227088,"name":"offline","context":{"idset":"10569"}}
+{"timestamp":1712611059.2479541,"name":"offline","context":{"idset":"10570"}}
+{"timestamp":1712611059.2688382,"name":"offline","context":{"idset":"10571"}}
+{"timestamp":1712611059.2896249,"name":"offline","context":{"idset":"10572"}}
+{"timestamp":1712611059.3106065,"name":"offline","context":{"idset":"10573"}}
+{"timestamp":1712611059.331321,"name":"offline","context":{"idset":"10574"}}
+{"timestamp":1712611059.3520253,"name":"offline","context":{"idset":"10575"}}
+{"timestamp":1712611059.3628511,"name":"offline","context":{"idset":"10576"}}
+{"timestamp":1712611059.363483,"name":"offline","context":{"idset":"10577"}}
+{"timestamp":1712611059.3640943,"name":"offline","context":{"idset":"10578"}}
+{"timestamp":1712611059.3647552,"name":"offline","context":{"idset":"10579"}}
+{"timestamp":1712611059.3653851,"name":"offline","context":{"idset":"10580"}}
+{"timestamp":1712611059.3660159,"name":"offline","context":{"idset":"10581"}}
+{"timestamp":1712611059.3666513,"name":"offline","context":{"idset":"10582"}}
+{"timestamp":1712611059.3672569,"name":"offline","context":{"idset":"10583"}}
+{"timestamp":1712611059.3678734,"name":"offline","context":{"idset":"10584"}}
+{"timestamp":1712611059.3684916,"name":"offline","context":{"idset":"10585"}}
+{"timestamp":1712611059.3690968,"name":"offline","context":{"idset":"10586"}}
+{"timestamp":1712611059.3697083,"name":"offline","context":{"idset":"10587"}}
+{"timestamp":1712611059.3703163,"name":"offline","context":{"idset":"10588"}}
+{"timestamp":1712611059.3709433,"name":"offline","context":{"idset":"10589"}}
+{"timestamp":1712611059.3715975,"name":"offline","context":{"idset":"10590"}}
+{"timestamp":1712611059.3722215,"name":"offline","context":{"idset":"10591"}}
+{"timestamp":1712611059.3728304,"name":"offline","context":{"idset":"10592"}}
+{"timestamp":1712611059.3734462,"name":"offline","context":{"idset":"10593"}}
+{"timestamp":1712611059.3741822,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1712611059.3748856,"name":"offline","context":{"idset":"10595"}}
+{"timestamp":1712611059.3807437,"name":"offline","context":{"idset":"10596"}}
+{"timestamp":1712611059.401475,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1712611059.4223533,"name":"offline","context":{"idset":"10598"}}
+{"timestamp":1712611059.4432254,"name":"offline","context":{"idset":"10599"}}
+{"timestamp":1712611059.4540665,"name":"offline","context":{"idset":"10600"}}
+{"timestamp":1712611059.4546847,"name":"offline","context":{"idset":"10601"}}
+{"timestamp":1712611059.4553266,"name":"offline","context":{"idset":"10602"}}
+{"timestamp":1712611059.4559507,"name":"offline","context":{"idset":"10603"}}
+{"timestamp":1712611059.4565632,"name":"offline","context":{"idset":"10604"}}
+{"timestamp":1712611059.4571552,"name":"offline","context":{"idset":"10605"}}
+{"timestamp":1712611059.4577613,"name":"offline","context":{"idset":"10606"}}
+{"timestamp":1712611059.4583704,"name":"offline","context":{"idset":"10607"}}
+{"timestamp":1712611059.4590015,"name":"offline","context":{"idset":"10608"}}
+{"timestamp":1712611059.4596379,"name":"offline","context":{"idset":"10609"}}
+{"timestamp":1712611059.4602497,"name":"offline","context":{"idset":"10610"}}
+{"timestamp":1712611059.4608867,"name":"offline","context":{"idset":"10611"}}
+{"timestamp":1712611059.4614949,"name":"offline","context":{"idset":"10612"}}
+{"timestamp":1712611059.4621069,"name":"offline","context":{"idset":"10613"}}
+{"timestamp":1712611059.4627578,"name":"offline","context":{"idset":"10614"}}
+{"timestamp":1712611059.4633687,"name":"offline","context":{"idset":"10615"}}
+{"timestamp":1712611059.4639664,"name":"offline","context":{"idset":"10616"}}
+{"timestamp":1712611059.4645817,"name":"offline","context":{"idset":"10617"}}
+{"timestamp":1712611059.4651949,"name":"offline","context":{"idset":"10618"}}
+{"timestamp":1712611059.4658058,"name":"offline","context":{"idset":"10619"}}
+{"timestamp":1712611059.466408,"name":"offline","context":{"idset":"10620"}}
+{"timestamp":1712611059.4670146,"name":"offline","context":{"idset":"10621"}}
+{"timestamp":1712611059.472739,"name":"offline","context":{"idset":"10622"}}
+{"timestamp":1712611059.4935141,"name":"offline","context":{"idset":"10623"}}
+{"timestamp":1712611059.5148504,"name":"offline","context":{"idset":"10624"}}
+{"timestamp":1712611059.5359528,"name":"offline","context":{"idset":"10625"}}
+{"timestamp":1712611059.5569611,"name":"offline","context":{"idset":"10626"}}
+{"timestamp":1712611059.5626853,"name":"offline","context":{"idset":"10627"}}
+{"timestamp":1712611059.5632794,"name":"offline","context":{"idset":"10628"}}
+{"timestamp":1712611059.5638881,"name":"offline","context":{"idset":"10630"}}
+{"timestamp":1712611059.5645077,"name":"offline","context":{"idset":"10631"}}
+{"timestamp":1712611059.5650878,"name":"offline","context":{"idset":"10632"}}
+{"timestamp":1712611059.5657012,"name":"offline","context":{"idset":"10633"}}
+{"timestamp":1712611059.5663009,"name":"offline","context":{"idset":"10635"}}
+{"timestamp":1712611059.5669529,"name":"offline","context":{"idset":"10636"}}
+{"timestamp":1712611059.5677056,"name":"offline","context":{"idset":"10640"}}
+{"timestamp":1712611059.5683115,"name":"offline","context":{"idset":"10641"}}
+{"timestamp":1712611059.5688846,"name":"offline","context":{"idset":"10645"}}
+{"timestamp":1712611059.5694749,"name":"offline","context":{"idset":"10646"}}
+{"timestamp":1712611059.5700469,"name":"offline","context":{"idset":"10647"}}
+{"timestamp":1712611059.570684,"name":"offline","context":{"idset":"10648"}}
+{"timestamp":1712611059.5712912,"name":"offline","context":{"idset":"10649"}}
+{"timestamp":1712611059.5718849,"name":"offline","context":{"idset":"10650"}}
+{"timestamp":1712611059.5724814,"name":"offline","context":{"idset":"10651"}}
+{"timestamp":1712611059.5730827,"name":"offline","context":{"idset":"10652"}}
+{"timestamp":1712611059.5737007,"name":"offline","context":{"idset":"10653"}}
+{"timestamp":1712611059.5743132,"name":"offline","context":{"idset":"10654"}}
+{"timestamp":1712611059.5799322,"name":"offline","context":{"idset":"10655"}}
+{"timestamp":1712611059.6005428,"name":"offline","context":{"idset":"10656"}}
+{"timestamp":1712611059.6213617,"name":"offline","context":{"idset":"10660"}}
+{"timestamp":1712611059.6423008,"name":"offline","context":{"idset":"10661"}}
+{"timestamp":1712611059.6631064,"name":"offline","context":{"idset":"10662"}}
+{"timestamp":1712611059.6739585,"name":"offline","context":{"idset":"10663"}}
+{"timestamp":1712611059.6745698,"name":"offline","context":{"idset":"10664"}}
+{"timestamp":1712611059.6751595,"name":"offline","context":{"idset":"10665"}}
+{"timestamp":1712611059.6757338,"name":"offline","context":{"idset":"10666"}}
+{"timestamp":1712611059.6763322,"name":"offline","context":{"idset":"10667"}}
+{"timestamp":1712611059.6769133,"name":"offline","context":{"idset":"10668"}}
+{"timestamp":1712611059.6774876,"name":"offline","context":{"idset":"10669"}}
+{"timestamp":1712611059.6780701,"name":"offline","context":{"idset":"10670"}}
+{"timestamp":1712611059.6786485,"name":"offline","context":{"idset":"10671"}}
+{"timestamp":1712611059.67923,"name":"offline","context":{"idset":"10672"}}
+{"timestamp":1712611059.6798501,"name":"offline","context":{"idset":"10673"}}
+{"timestamp":1712611059.6804469,"name":"offline","context":{"idset":"10674"}}
+{"timestamp":1712611059.6810458,"name":"offline","context":{"idset":"10675"}}
+{"timestamp":1712611059.6816146,"name":"offline","context":{"idset":"10676"}}
+{"timestamp":1712611059.6821692,"name":"offline","context":{"idset":"10677"}}
+{"timestamp":1712611059.68276,"name":"offline","context":{"idset":"10678"}}
+{"timestamp":1712611059.683346,"name":"offline","context":{"idset":"10679"}}
+{"timestamp":1712611059.683903,"name":"offline","context":{"idset":"10680"}}
+{"timestamp":1712611059.6844959,"name":"offline","context":{"idset":"10681"}}
+{"timestamp":1712611059.6850762,"name":"offline","context":{"idset":"10682"}}
+{"timestamp":1712611059.6856434,"name":"offline","context":{"idset":"10683"}}
+{"timestamp":1712611059.6862166,"name":"offline","context":{"idset":"10684"}}
+{"timestamp":1712611059.6868198,"name":"offline","context":{"idset":"10685"}}
+{"timestamp":1712611059.6874382,"name":"offline","context":{"idset":"10686"}}
+{"timestamp":1712611059.6879933,"name":"offline","context":{"idset":"10687"}}
+{"timestamp":1712611059.6886086,"name":"offline","context":{"idset":"10688"}}
+{"timestamp":1712611059.6891651,"name":"offline","context":{"idset":"10689"}}
+{"timestamp":1712611059.6897473,"name":"offline","context":{"idset":"10690"}}
+{"timestamp":1712611059.6903548,"name":"offline","context":{"idset":"10691"}}
+{"timestamp":1712611059.690933,"name":"offline","context":{"idset":"10693"}}
+{"timestamp":1712611059.7016141,"name":"offline","context":{"idset":"10694"}}
+{"timestamp":1712611059.7176549,"name":"offline","context":{"idset":"10695"}}
+{"timestamp":1712611059.7182262,"name":"offline","context":{"idset":"10696"}}
+{"timestamp":1712611059.7188101,"name":"offline","context":{"idset":"10697"}}
+{"timestamp":1712611059.7193713,"name":"offline","context":{"idset":"10698"}}
+{"timestamp":1712611059.719938,"name":"offline","context":{"idset":"10699"}}
+{"timestamp":1712611059.7205017,"name":"offline","context":{"idset":"10700"}}
+{"timestamp":1712611059.7210743,"name":"offline","context":{"idset":"10701"}}
+{"timestamp":1712611059.7216456,"name":"offline","context":{"idset":"10703"}}
+{"timestamp":1712611059.7221985,"name":"offline","context":{"idset":"10704"}}
+{"timestamp":1712611059.7228117,"name":"offline","context":{"idset":"10705"}}
+{"timestamp":1712611059.7233727,"name":"offline","context":{"idset":"10706"}}
+{"timestamp":1712611059.7239249,"name":"offline","context":{"idset":"10707"}}
+{"timestamp":1712611059.72451,"name":"offline","context":{"idset":"10708"}}
+{"timestamp":1712611059.7250526,"name":"offline","context":{"idset":"10709"}}
+{"timestamp":1712611059.7256198,"name":"offline","context":{"idset":"10710"}}
+{"timestamp":1712611059.7261729,"name":"offline","context":{"idset":"10711"}}
+{"timestamp":1712611059.7267478,"name":"offline","context":{"idset":"10713"}}
+{"timestamp":1712611059.7273319,"name":"offline","context":{"idset":"10714"}}
+{"timestamp":1712611059.7278814,"name":"offline","context":{"idset":"10715"}}
+{"timestamp":1712611059.7284343,"name":"offline","context":{"idset":"10716"}}
+{"timestamp":1712611059.7289934,"name":"offline","context":{"idset":"10717"}}
+{"timestamp":1712611059.7295477,"name":"offline","context":{"idset":"10719"}}
+{"timestamp":1712611059.7300875,"name":"offline","context":{"idset":"10720"}}
+{"timestamp":1712611059.7306633,"name":"offline","context":{"idset":"10721"}}
+{"timestamp":1712611059.7312059,"name":"offline","context":{"idset":"10722"}}
+{"timestamp":1712611059.7317672,"name":"offline","context":{"idset":"10723"}}
+{"timestamp":1712611059.7323341,"name":"offline","context":{"idset":"10724"}}
+{"timestamp":1712611059.7328794,"name":"offline","context":{"idset":"10725"}}
+{"timestamp":1712611059.733438,"name":"offline","context":{"idset":"10726"}}
+{"timestamp":1712611059.7339675,"name":"offline","context":{"idset":"10727"}}
+{"timestamp":1712611059.7345281,"name":"offline","context":{"idset":"10728"}}
+{"timestamp":1712611059.7350724,"name":"offline","context":{"idset":"10729"}}
+{"timestamp":1712611059.7356164,"name":"offline","context":{"idset":"10730"}}
+{"timestamp":1712611059.7361708,"name":"offline","context":{"idset":"10731"}}
+{"timestamp":1712611059.7367439,"name":"offline","context":{"idset":"10732"}}
+{"timestamp":1712611059.7372897,"name":"offline","context":{"idset":"10733"}}
+{"timestamp":1712611059.7378175,"name":"offline","context":{"idset":"10734"}}
+{"timestamp":1712611059.7383602,"name":"offline","context":{"idset":"10735"}}
+{"timestamp":1712611059.7388971,"name":"offline","context":{"idset":"10736"}}
+{"timestamp":1712611059.7394447,"name":"offline","context":{"idset":"10737"}}
+{"timestamp":1712611059.7600193,"name":"offline","context":{"idset":"10738"}}
+{"timestamp":1712611059.7805831,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1712611059.8011322,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1712611059.8217862,"name":"offline","context":{"idset":"10743"}}
+{"timestamp":1712611059.8424556,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1712611059.8581717,"name":"offline","context":{"idset":"10745"}}
+{"timestamp":1712611059.8587432,"name":"offline","context":{"idset":"10746"}}
+{"timestamp":1712611059.8593678,"name":"offline","context":{"idset":"10747"}}
+{"timestamp":1712611059.8599465,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1712611059.8604956,"name":"offline","context":{"idset":"10749"}}
+{"timestamp":1712611059.8610187,"name":"offline","context":{"idset":"10750"}}
+{"timestamp":1712611059.8615663,"name":"offline","context":{"idset":"10751"}}
+{"timestamp":1712611059.8621409,"name":"offline","context":{"idset":"10752"}}
+{"timestamp":1712611059.8627086,"name":"offline","context":{"idset":"10753"}}
+{"timestamp":1712611059.8632374,"name":"offline","context":{"idset":"10754"}}
+{"timestamp":1712611059.8637831,"name":"offline","context":{"idset":"10755"}}
+{"timestamp":1712611059.8643415,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1712611059.8649428,"name":"offline","context":{"idset":"10757"}}
+{"timestamp":1712611059.865515,"name":"offline","context":{"idset":"10758"}}
+{"timestamp":1712611059.8660383,"name":"offline","context":{"idset":"10759"}}
+{"timestamp":1712611059.8666043,"name":"offline","context":{"idset":"10760"}}
+{"timestamp":1712611059.867172,"name":"offline","context":{"idset":"10761"}}
+{"timestamp":1712611059.8677552,"name":"offline","context":{"idset":"10762"}}
+{"timestamp":1712611059.8682852,"name":"offline","context":{"idset":"10763"}}
+{"timestamp":1712611059.8688235,"name":"offline","context":{"idset":"10764"}}
+{"timestamp":1712611059.8693717,"name":"offline","context":{"idset":"10765"}}
+{"timestamp":1712611059.8698933,"name":"offline","context":{"idset":"10766"}}
+{"timestamp":1712611059.8704176,"name":"offline","context":{"idset":"10767"}}
+{"timestamp":1712611059.8709211,"name":"offline","context":{"idset":"10768"}}
+{"timestamp":1712611059.8714511,"name":"offline","context":{"idset":"10769"}}
+{"timestamp":1712611059.8719752,"name":"offline","context":{"idset":"10770"}}
+{"timestamp":1712611059.8725147,"name":"offline","context":{"idset":"10771"}}
+{"timestamp":1712611059.8730202,"name":"offline","context":{"idset":"10772"}}
+{"timestamp":1712611059.8735392,"name":"offline","context":{"idset":"10773"}}
+{"timestamp":1712611059.8740592,"name":"offline","context":{"idset":"10774"}}
+{"timestamp":1712611059.8745778,"name":"offline","context":{"idset":"10775"}}
+{"timestamp":1712611059.8751235,"name":"offline","context":{"idset":"10776"}}
+{"timestamp":1712611059.875674,"name":"offline","context":{"idset":"10777"}}
+{"timestamp":1712611059.8761952,"name":"offline","context":{"idset":"10778"}}
+{"timestamp":1712611059.8767173,"name":"offline","context":{"idset":"10779"}}
+{"timestamp":1712611059.8772192,"name":"offline","context":{"idset":"10780"}}
+{"timestamp":1712611059.8777516,"name":"offline","context":{"idset":"10781"}}
+{"timestamp":1712611059.8782568,"name":"offline","context":{"idset":"10782"}}
+{"timestamp":1712611059.8787758,"name":"offline","context":{"idset":"10783"}}
+{"timestamp":1712611059.8792903,"name":"offline","context":{"idset":"10784"}}
+{"timestamp":1712611059.8798165,"name":"offline","context":{"idset":"10785"}}
+{"timestamp":1712611059.8803482,"name":"offline","context":{"idset":"10786"}}
+{"timestamp":1712611059.8808477,"name":"offline","context":{"idset":"10787"}}
+{"timestamp":1712611059.8813574,"name":"offline","context":{"idset":"10788"}}
+{"timestamp":1712611059.8818822,"name":"offline","context":{"idset":"10789"}}
+{"timestamp":1712611059.8823953,"name":"offline","context":{"idset":"10790"}}
+{"timestamp":1712611059.88289,"name":"offline","context":{"idset":"10791"}}
+{"timestamp":1712611059.8834071,"name":"offline","context":{"idset":"10792"}}
+{"timestamp":1712611059.8839238,"name":"offline","context":{"idset":"10793"}}
+{"timestamp":1712611059.8872068,"name":"offline","context":{"idset":"10794"}}
+{"timestamp":1712611059.8907902,"name":"offline","context":{"idset":"10795"}}
+{"timestamp":1712611059.8944187,"name":"offline","context":{"idset":"10796"}}
+{"timestamp":1712611059.8979936,"name":"offline","context":{"idset":"10797"}}
+{"timestamp":1712611059.901592,"name":"offline","context":{"idset":"10798"}}
+{"timestamp":1712611059.9051619,"name":"offline","context":{"idset":"10799"}}
+{"timestamp":1712611059.908802,"name":"offline","context":{"idset":"10800"}}
+{"timestamp":1712611059.9093125,"name":"offline","context":{"idset":"10801"}}
+{"timestamp":1712611059.9322176,"name":"offline","context":{"idset":"10802"}}
+{"timestamp":1712611059.9526618,"name":"offline","context":{"idset":"10803"}}
+{"timestamp":1712611059.973001,"name":"offline","context":{"idset":"10804"}}
+{"timestamp":1712611059.9935322,"name":"offline","context":{"idset":"10805"}}
+{"timestamp":1712611060.0138938,"name":"offline","context":{"idset":"10806"}}
+{"timestamp":1712611060.0343063,"name":"offline","context":{"idset":"10807"}}
+{"timestamp":1712611060.0550416,"name":"offline","context":{"idset":"10808"}}
+{"timestamp":1712611060.0606818,"name":"offline","context":{"idset":"10809"}}
+{"timestamp":1712611060.0611832,"name":"offline","context":{"idset":"10810"}}
+{"timestamp":1712611060.0617168,"name":"offline","context":{"idset":"10811"}}
+{"timestamp":1712611060.0622067,"name":"offline","context":{"idset":"10812"}}
+{"timestamp":1712611060.0627151,"name":"offline","context":{"idset":"10813"}}
+{"timestamp":1712611060.0632041,"name":"offline","context":{"idset":"10814"}}
+{"timestamp":1712611060.0637391,"name":"offline","context":{"idset":"10815"}}
+{"timestamp":1712611060.0642543,"name":"offline","context":{"idset":"10816"}}
+{"timestamp":1712611060.0647726,"name":"offline","context":{"idset":"10817"}}
+{"timestamp":1712611060.0652559,"name":"offline","context":{"idset":"10818"}}
+{"timestamp":1712611060.0658123,"name":"offline","context":{"idset":"10819"}}
+{"timestamp":1712611060.0663075,"name":"offline","context":{"idset":"10820"}}
+{"timestamp":1712611060.0668736,"name":"offline","context":{"idset":"10821"}}
+{"timestamp":1712611060.0674198,"name":"offline","context":{"idset":"10822"}}
+{"timestamp":1712611060.067914,"name":"offline","context":{"idset":"10823"}}
+{"timestamp":1712611060.0684195,"name":"offline","context":{"idset":"10824"}}
+{"timestamp":1712611060.0689218,"name":"offline","context":{"idset":"10825"}}
+{"timestamp":1712611060.0694163,"name":"offline","context":{"idset":"10826"}}
+{"timestamp":1712611060.0699358,"name":"offline","context":{"idset":"10827"}}
+{"timestamp":1712611060.070436,"name":"offline","context":{"idset":"10828"}}
+{"timestamp":1712611060.070914,"name":"offline","context":{"idset":"10829"}}
+{"timestamp":1712611060.0714176,"name":"offline","context":{"idset":"10830"}}
+{"timestamp":1712611060.0719008,"name":"offline","context":{"idset":"10831"}}
+{"timestamp":1712611060.0723915,"name":"offline","context":{"idset":"10832"}}
+{"timestamp":1712611060.0728729,"name":"offline","context":{"idset":"10833"}}
+{"timestamp":1712611060.0734184,"name":"offline","context":{"idset":"10834"}}
+{"timestamp":1712611060.0738969,"name":"offline","context":{"idset":"10835"}}
+{"timestamp":1712611060.074398,"name":"offline","context":{"idset":"10836"}}
+{"timestamp":1712611060.0748966,"name":"offline","context":{"idset":"10837"}}
+{"timestamp":1712611060.0754015,"name":"offline","context":{"idset":"10838"}}
+{"timestamp":1712611060.0758767,"name":"offline","context":{"idset":"10839"}}
+{"timestamp":1712611060.0763769,"name":"offline","context":{"idset":"10840"}}
+{"timestamp":1712611060.0768509,"name":"offline","context":{"idset":"10841"}}
+{"timestamp":1712611060.0773406,"name":"offline","context":{"idset":"10842"}}
+{"timestamp":1712611060.077837,"name":"offline","context":{"idset":"10843"}}
+{"timestamp":1712611060.0783324,"name":"offline","context":{"idset":"10844"}}
+{"timestamp":1712611060.0788004,"name":"offline","context":{"idset":"10845"}}
+{"timestamp":1712611060.079289,"name":"offline","context":{"idset":"10846"}}
+{"timestamp":1712611060.0797908,"name":"offline","context":{"idset":"10847"}}
+{"timestamp":1712611060.0803294,"name":"offline","context":{"idset":"10848"}}
+{"timestamp":1712611060.083359,"name":"offline","context":{"idset":"10849"}}
+{"timestamp":1712611060.0866868,"name":"offline","context":{"idset":"10850"}}
+{"timestamp":1712611060.0900383,"name":"offline","context":{"idset":"10851"}}
+{"timestamp":1712611060.0933807,"name":"offline","context":{"idset":"10852"}}
+{"timestamp":1712611060.0966606,"name":"offline","context":{"idset":"10853"}}
+{"timestamp":1712611060.099956,"name":"offline","context":{"idset":"10854"}}
+{"timestamp":1712611060.1032445,"name":"offline","context":{"idset":"10855"}}
+{"timestamp":1712611060.1064487,"name":"offline","context":{"idset":"10856"}}
+{"timestamp":1712611060.1097107,"name":"offline","context":{"idset":"10857"}}
+{"timestamp":1712611060.1130202,"name":"offline","context":{"idset":"10858"}}
+{"timestamp":1712611060.1161273,"name":"offline","context":{"idset":"10859"}}
+{"timestamp":1712611060.1194253,"name":"offline","context":{"idset":"10860"}}
+{"timestamp":1712611060.1227167,"name":"offline","context":{"idset":"10861"}}
+{"timestamp":1712611060.1259964,"name":"offline","context":{"idset":"10862"}}
+{"timestamp":1712611060.129283,"name":"offline","context":{"idset":"10863"}}
+{"timestamp":1712611060.1326644,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1712611060.1359282,"name":"offline","context":{"idset":"10865"}}
+{"timestamp":1712611060.1391218,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1712611060.1423962,"name":"offline","context":{"idset":"10867"}}
+{"timestamp":1712611060.1456707,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1712611060.1488714,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1712611060.1521311,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1712611060.1584439,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1712611060.161679,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1712611060.1649752,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1712611060.1712255,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1712611060.179394,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1712611060.1801167,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1712611060.1860218,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1712611060.1882234,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1712611060.1914217,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1712611060.1919947,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1712611060.1948056,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1712611060.2009921,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1712611060.2041862,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1712611060.2074184,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1712611060.2135961,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1712611060.216794,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1712611060.2200344,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1712611060.2262464,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1712611060.2296383,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1712611060.2328258,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1712611060.2334068,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1712611060.2364514,"name":"offline","context":{"idset":"10894"}}
+{"timestamp":1712611060.2396386,"name":"offline","context":{"idset":"10895"}}
+{"timestamp":1712611060.2428062,"name":"offline","context":{"idset":"10896"}}
+{"timestamp":1712611060.2460983,"name":"offline","context":{"idset":"10897"}}
+{"timestamp":1712611060.2493353,"name":"offline","context":{"idset":"10898"}}
+{"timestamp":1712611060.2525403,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1712611060.2555649,"name":"offline","context":{"idset":"10900"}}
+{"timestamp":1712611060.2587397,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1712611060.2620633,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1712611060.2651377,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1712611060.268281,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1712611060.2743764,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1712611060.2775044,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1712611060.2780995,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1712611060.2808549,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1712611060.2868974,"name":"offline","context":{"idset":"10909"}}
+{"timestamp":1712611060.2899921,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1712611060.3101594,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1712611060.3376107,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1712611060.3651271,"name":"offline","context":{"idset":"10915"}}
+{"timestamp":1712611060.3932235,"name":"offline","context":{"idset":"10916"}}
+{"timestamp":1712611060.4222884,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1712611060.453007,"name":"offline","context":{"idset":"10918"}}
+{"timestamp":1712611060.4844801,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1712611060.5172842,"name":"offline","context":{"idset":"10920"}}
+{"timestamp":1712611060.5524325,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1712611060.5959871,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1712611060.638319,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1712611060.6758564,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1712611060.7106972,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1712611060.7443111,"name":"offline","context":{"idset":"10926"}}
+{"timestamp":1712611060.7532921,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1712611060.7540286,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1712611060.7547786,"name":"offline","context":{"idset":"10929"}}
+{"timestamp":1712611060.7555268,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1712611060.7562342,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1712611060.7569773,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1712611060.7577658,"name":"offline","context":{"idset":"10933"}}
+{"timestamp":1712611060.7585075,"name":"offline","context":{"idset":"10934"}}
+{"timestamp":1712611060.759197,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1712611060.759902,"name":"offline","context":{"idset":"10936"}}
+{"timestamp":1712611060.7606521,"name":"offline","context":{"idset":"10937"}}
+{"timestamp":1712611060.7613673,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1712611060.7620835,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1712611060.7628465,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1712611060.763586,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1712611060.7642927,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1712611060.765034,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1712611060.7657714,"name":"offline","context":{"idset":"10946"}}
+{"timestamp":1712611060.7664678,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1712611060.7671807,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1712611060.7679033,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1712611060.7686388,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1712611060.7693698,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1712611060.7700765,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1712611060.770786,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1712611060.7715344,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1712611060.772296,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1712611060.773077,"name":"offline","context":{"idset":"10957"}}
+{"timestamp":1712611060.77387,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1712611060.7746189,"name":"offline","context":{"idset":"10959"}}
+{"timestamp":1712611060.7753558,"name":"offline","context":{"idset":"10960"}}
+{"timestamp":1712611060.7760518,"name":"offline","context":{"idset":"10961"}}
+{"timestamp":1712611060.776741,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1712611060.77742,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1712611060.7780929,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1712611060.7788267,"name":"offline","context":{"idset":"10965"}}
+{"timestamp":1712611060.7795472,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1712611060.7802098,"name":"offline","context":{"idset":"10967"}}
+{"timestamp":1712611060.7808969,"name":"offline","context":{"idset":"10968"}}
+{"timestamp":1712611060.7815652,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1712611060.7822559,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1712611060.7829621,"name":"offline","context":{"idset":"10971"}}
+{"timestamp":1712611060.7836361,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1712611060.7843218,"name":"offline","context":{"idset":"10973"}}
+{"timestamp":1712611060.7849829,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1712611060.7857008,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1712611060.786377,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1712611060.7870641,"name":"offline","context":{"idset":"10977"}}
+{"timestamp":1712611060.7877836,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1712611060.7885442,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1712611060.7892137,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1712611060.7899735,"name":"offline","context":{"idset":"10981"}}
+{"timestamp":1712611060.7907228,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1712611060.7913878,"name":"offline","context":{"idset":"10983"}}
+{"timestamp":1712611060.7920294,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1712611060.792752,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1712611060.7934449,"name":"offline","context":{"idset":"10986"}}
+{"timestamp":1712611060.7940879,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1712611060.7947695,"name":"offline","context":{"idset":"10988"}}
+{"timestamp":1712611060.79545,"name":"offline","context":{"idset":"10989"}}
+{"timestamp":1712611060.7961214,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1712611060.796771,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1712611060.7974224,"name":"offline","context":{"idset":"10992"}}
+{"timestamp":1712611060.7980561,"name":"offline","context":{"idset":"10993"}}
+{"timestamp":1712611060.7987039,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1712611060.7993603,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1712611060.8000267,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1712611060.8006868,"name":"offline","context":{"idset":"10997"}}
+{"timestamp":1712611060.8013377,"name":"offline","context":{"idset":"10998"}}
+{"timestamp":1712611060.8020153,"name":"offline","context":{"idset":"11001"}}
+{"timestamp":1712611060.8027453,"name":"offline","context":{"idset":"11002"}}
+{"timestamp":1712611060.8034198,"name":"offline","context":{"idset":"11003"}}
+{"timestamp":1712611060.8040559,"name":"offline","context":{"idset":"11004"}}
+{"timestamp":1712611060.8047063,"name":"offline","context":{"idset":"11005"}}
+{"timestamp":1712611060.8053455,"name":"offline","context":{"idset":"11006"}}
+{"timestamp":1712611060.805969,"name":"offline","context":{"idset":"11007"}}
+{"timestamp":1712611060.8066337,"name":"offline","context":{"idset":"11008"}}
+{"timestamp":1712611060.8072577,"name":"offline","context":{"idset":"11009"}}
+{"timestamp":1712611060.8079865,"name":"offline","context":{"idset":"11010"}}
+{"timestamp":1712611060.8086488,"name":"offline","context":{"idset":"11011"}}
+{"timestamp":1712611060.8093972,"name":"offline","context":{"idset":"11012"}}
+{"timestamp":1712611060.8100507,"name":"offline","context":{"idset":"11013"}}
+{"timestamp":1712611060.8107393,"name":"offline","context":{"idset":"11014"}}
+{"timestamp":1712611060.8113747,"name":"offline","context":{"idset":"11015"}}
+{"timestamp":1712611060.812072,"name":"offline","context":{"idset":"11016"}}
+{"timestamp":1712611060.8127487,"name":"offline","context":{"idset":"11017"}}
+{"timestamp":1712611060.8133841,"name":"offline","context":{"idset":"11018"}}
+{"timestamp":1712611060.814024,"name":"offline","context":{"idset":"11019"}}
+{"timestamp":1712611060.8147032,"name":"offline","context":{"idset":"11020"}}
+{"timestamp":1712611060.8153739,"name":"offline","context":{"idset":"11021"}}
+{"timestamp":1712611060.816016,"name":"offline","context":{"idset":"11022"}}
+{"timestamp":1712611060.8166435,"name":"offline","context":{"idset":"11023"}}
+{"timestamp":1712611060.8173022,"name":"offline","context":{"idset":"11024"}}
+{"timestamp":1712611060.8179193,"name":"offline","context":{"idset":"11025"}}
+{"timestamp":1712611060.8185482,"name":"offline","context":{"idset":"11026"}}
+{"timestamp":1712611060.8191562,"name":"offline","context":{"idset":"11027"}}
+{"timestamp":1712611060.8197794,"name":"offline","context":{"idset":"11028"}}
+{"timestamp":1712611060.8204694,"name":"offline","context":{"idset":"11029"}}
+{"timestamp":1712611060.8211198,"name":"offline","context":{"idset":"11030"}}
+{"timestamp":1712611060.8217437,"name":"offline","context":{"idset":"11031"}}
+{"timestamp":1712611060.8223715,"name":"offline","context":{"idset":"11032"}}
+{"timestamp":1712611060.8230071,"name":"offline","context":{"idset":"11033"}}
+{"timestamp":1712611060.8236306,"name":"offline","context":{"idset":"11034"}}
+{"timestamp":1712611060.8242598,"name":"offline","context":{"idset":"11035"}}
+{"timestamp":1712611060.8248796,"name":"offline","context":{"idset":"11036"}}
+{"timestamp":1712611060.8254921,"name":"offline","context":{"idset":"11037"}}
+{"timestamp":1712611060.8260934,"name":"offline","context":{"idset":"11038"}}
+{"timestamp":1712611060.826756,"name":"offline","context":{"idset":"11039"}}
+{"timestamp":1712611060.8274169,"name":"offline","context":{"idset":"11040"}}
+{"timestamp":1712611060.8280475,"name":"offline","context":{"idset":"11041"}}
+{"timestamp":1712611060.8286717,"name":"offline","context":{"idset":"11042"}}
+{"timestamp":1712611060.8293157,"name":"offline","context":{"idset":"11043"}}
+{"timestamp":1712611060.8299396,"name":"offline","context":{"idset":"11044"}}
+{"timestamp":1712611060.8305953,"name":"offline","context":{"idset":"11045"}}
+{"timestamp":1712611060.8311951,"name":"offline","context":{"idset":"11046"}}
+{"timestamp":1712611060.8318362,"name":"offline","context":{"idset":"11047"}}
+{"timestamp":1712611060.8489408,"name":"offline","context":{"idset":"11048"}}
+{"timestamp":1712611060.8828743,"name":"offline","context":{"idset":"11049"}}
+{"timestamp":1712611060.9165392,"name":"offline","context":{"idset":"11050"}}
+{"timestamp":1712611060.9500737,"name":"offline","context":{"idset":"11051"}}
+{"timestamp":1712611060.9836383,"name":"offline","context":{"idset":"11052"}}
+{"timestamp":1712611061.0173798,"name":"offline","context":{"idset":"11053"}}
+{"timestamp":1712611061.0507159,"name":"offline","context":{"idset":"11054"}}
+{"timestamp":1712611061.0857267,"name":"offline","context":{"idset":"11055"}}
+{"timestamp":1712611061.1198301,"name":"offline","context":{"idset":"11056"}}
+{"timestamp":1712611061.1533921,"name":"offline","context":{"idset":"11057"}}
+{"timestamp":1712611061.1869378,"name":"offline","context":{"idset":"11058"}}
+{"timestamp":1712611061.2204542,"name":"offline","context":{"idset":"11059"}}
+{"timestamp":1712611061.2539678,"name":"offline","context":{"idset":"11060"}}
+{"timestamp":1712611061.2877493,"name":"offline","context":{"idset":"11061"}}
+{"timestamp":1712611061.3212776,"name":"offline","context":{"idset":"11062"}}
+{"timestamp":1712611061.3550704,"name":"offline","context":{"idset":"11063"}}
+{"timestamp":1712611061.3889852,"name":"offline","context":{"idset":"11064"}}
+{"timestamp":1712611061.4224639,"name":"offline","context":{"idset":"11065"}}
+{"timestamp":1712611061.4572229,"name":"offline","context":{"idset":"11066"}}
+{"timestamp":1712611061.4905713,"name":"offline","context":{"idset":"11067"}}
+{"timestamp":1712611061.5233605,"name":"offline","context":{"idset":"11068"}}
+{"timestamp":1712611061.5561957,"name":"offline","context":{"idset":"11069"}}
+{"timestamp":1712611061.5896289,"name":"offline","context":{"idset":"11070"}}
+{"timestamp":1712611061.6222298,"name":"offline","context":{"idset":"11071"}}
+{"timestamp":1712611061.6549673,"name":"offline","context":{"idset":"11072"}}
+{"timestamp":1712611061.6878989,"name":"offline","context":{"idset":"11073"}}
+{"timestamp":1712611061.7049615,"name":"offline","context":{"idset":"11074"}}
+{"timestamp":1712611061.705565,"name":"offline","context":{"idset":"11075"}}
+{"timestamp":1712611061.7061424,"name":"offline","context":{"idset":"11076"}}
+{"timestamp":1712611061.7067294,"name":"offline","context":{"idset":"11077"}}
+{"timestamp":1712611061.7073538,"name":"offline","context":{"idset":"11078"}}
+{"timestamp":1712611061.7082262,"name":"offline","context":{"idset":"11079"}}
+{"timestamp":1712611061.7088757,"name":"offline","context":{"idset":"11080"}}
+{"timestamp":1712611061.7094767,"name":"offline","context":{"idset":"11081"}}
+{"timestamp":1712611061.7100441,"name":"offline","context":{"idset":"11082"}}
+{"timestamp":1712611061.7106247,"name":"offline","context":{"idset":"11083"}}
+{"timestamp":1712611061.7111912,"name":"offline","context":{"idset":"11084"}}
+{"timestamp":1712611061.7117729,"name":"offline","context":{"idset":"11085"}}
+{"timestamp":1712611061.7123473,"name":"offline","context":{"idset":"11086"}}
+{"timestamp":1712611061.7129099,"name":"offline","context":{"idset":"11087"}}
+{"timestamp":1712611061.7134826,"name":"offline","context":{"idset":"11088"}}
+{"timestamp":1712611061.7140436,"name":"offline","context":{"idset":"11089"}}
+{"timestamp":1712611061.7146244,"name":"offline","context":{"idset":"11090"}}
+{"timestamp":1712611061.7154782,"name":"offline","context":{"idset":"11091"}}
+{"timestamp":1712611061.7163103,"name":"offline","context":{"idset":"11092"}}
+{"timestamp":1712611061.7169418,"name":"offline","context":{"idset":"11093"}}
+{"timestamp":1712611061.7175233,"name":"offline","context":{"idset":"11094"}}
+{"timestamp":1712611061.7180791,"name":"offline","context":{"idset":"11095"}}
+{"timestamp":1712611061.7186518,"name":"offline","context":{"idset":"11096"}}
+{"timestamp":1712611061.719209,"name":"offline","context":{"idset":"11097"}}
+{"timestamp":1712611061.7197833,"name":"offline","context":{"idset":"11098"}}
+{"timestamp":1712611061.7203615,"name":"offline","context":{"idset":"11099"}}
+{"timestamp":1712611061.7209134,"name":"offline","context":{"idset":"11100"}}
+{"timestamp":1712611061.7215018,"name":"offline","context":{"idset":"11101"}}
+{"timestamp":1712611061.7220638,"name":"offline","context":{"idset":"11102"}}
+{"timestamp":1712611061.7226367,"name":"offline","context":{"idset":"11103"}}
+{"timestamp":1712611061.7231839,"name":"offline","context":{"idset":"11104"}}
+{"timestamp":1712611061.7237456,"name":"offline","context":{"idset":"11105"}}
+{"timestamp":1712611061.7243147,"name":"offline","context":{"idset":"11106"}}
+{"timestamp":1712611061.7248588,"name":"offline","context":{"idset":"11107"}}
+{"timestamp":1712611061.7254241,"name":"offline","context":{"idset":"11108"}}
+{"timestamp":1712611061.7259653,"name":"offline","context":{"idset":"11109"}}
+{"timestamp":1712611061.726541,"name":"offline","context":{"idset":"11110"}}
+{"timestamp":1712611061.7270837,"name":"offline","context":{"idset":"11113"}}
+{"timestamp":1712611061.7277431,"name":"offline","context":{"idset":"11114"}}
+{"timestamp":1712611061.7285953,"name":"offline","context":{"idset":"11116"}}
+{"timestamp":1712611061.7291436,"name":"offline","context":{"idset":"11119"}}
+{"timestamp":1712611061.7297008,"name":"offline","context":{"idset":"11120"}}
+{"timestamp":1712611061.7302365,"name":"offline","context":{"idset":"11121"}}
+{"timestamp":1712611061.7307968,"name":"offline","context":{"idset":"11122"}}
+{"timestamp":1712611061.7313552,"name":"offline","context":{"idset":"11123"}}
+{"timestamp":1712611061.7318854,"name":"offline","context":{"idset":"11124"}}
+{"timestamp":1712611061.7324362,"name":"offline","context":{"idset":"11253"}}
+{"timestamp":1712611061.7329705,"name":"offline","context":{"idset":"11254"}}
+{"timestamp":1712611061.7335255,"name":"offline","context":{"idset":"11255"}}
+{"timestamp":1712611061.7340517,"name":"offline","context":{"idset":"11256"}}
+{"timestamp":1712611061.7346134,"name":"offline","context":{"idset":"11257"}}
+{"timestamp":1712611061.7351415,"name":"offline","context":{"idset":"11258"}}
+{"timestamp":1712611061.7356796,"name":"offline","context":{"idset":"11259"}}
+{"timestamp":1712611061.7362041,"name":"offline","context":{"idset":"11260"}}
+{"timestamp":1712611061.7367461,"name":"offline","context":{"idset":"11261"}}
+{"timestamp":1712611061.7372863,"name":"offline","context":{"idset":"11262"}}
+{"timestamp":1712611061.7378085,"name":"offline","context":{"idset":"11263"}}
+{"timestamp":1712611061.7383428,"name":"offline","context":{"idset":"11264"}}
+{"timestamp":1712611061.7388597,"name":"offline","context":{"idset":"11265"}}
+{"timestamp":1712611061.739397,"name":"offline","context":{"idset":"11266"}}
+{"timestamp":1712611061.7399139,"name":"offline","context":{"idset":"11267"}}
+{"timestamp":1712611061.7404492,"name":"offline","context":{"idset":"11268"}}
+{"timestamp":1712611061.7409642,"name":"offline","context":{"idset":"11269"}}
+{"timestamp":1712611061.7414999,"name":"offline","context":{"idset":"11270"}}
+{"timestamp":1712611061.7420106,"name":"offline","context":{"idset":"11271"}}
+{"timestamp":1712611061.7425649,"name":"offline","context":{"idset":"11272"}}
+{"timestamp":1712611061.7430899,"name":"offline","context":{"idset":"11273"}}
+{"timestamp":1712611061.7436311,"name":"offline","context":{"idset":"11274"}}
+{"timestamp":1712611061.7441454,"name":"offline","context":{"idset":"11275"}}
+{"timestamp":1712611061.7446778,"name":"offline","context":{"idset":"11276"}}
+{"timestamp":1712611061.7451856,"name":"offline","context":{"idset":"11277"}}
+{"timestamp":1712611061.7457166,"name":"offline","context":{"idset":"11278"}}
+{"timestamp":1712611061.7462234,"name":"offline","context":{"idset":"11279"}}
+{"timestamp":1712611061.746743,"name":"offline","context":{"idset":"11280"}}
+{"timestamp":1712611061.7472494,"name":"offline","context":{"idset":"11281"}}
+{"timestamp":1712611061.7477736,"name":"offline","context":{"idset":"11282"}}
+{"timestamp":1712611061.7482915,"name":"offline","context":{"idset":"11283"}}
+{"timestamp":1712611061.7487924,"name":"offline","context":{"idset":"11284"}}
+{"timestamp":1712611061.7493129,"name":"offline","context":{"idset":"11285"}}
+{"timestamp":1712611061.7498133,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1712611061.750329,"name":"offline","context":{"idset":"11287"}}
+{"timestamp":1712611061.7508278,"name":"offline","context":{"idset":"11288"}}
+{"timestamp":1712611061.751344,"name":"offline","context":{"idset":"11289"}}
+{"timestamp":1712611061.7518401,"name":"offline","context":{"idset":"11290"}}
+{"timestamp":1712611061.7523465,"name":"offline","context":{"idset":"11291"}}
+{"timestamp":1712611061.7528417,"name":"offline","context":{"idset":"11292"}}
+{"timestamp":1712611061.7533538,"name":"offline","context":{"idset":"11293"}}
+{"timestamp":1712611061.7538462,"name":"offline","context":{"idset":"11294"}}
+{"timestamp":1712611061.7543545,"name":"offline","context":{"idset":"11295"}}
+{"timestamp":1712611061.754847,"name":"offline","context":{"idset":"11296"}}
+{"timestamp":1712611061.7553563,"name":"offline","context":{"idset":"11297"}}
+{"timestamp":1712611061.7558477,"name":"offline","context":{"idset":"11298"}}
+{"timestamp":1712611061.7563639,"name":"offline","context":{"idset":"11299"}}
+{"timestamp":1712611061.7568538,"name":"offline","context":{"idset":"11300"}}
+{"timestamp":1712611061.7573597,"name":"offline","context":{"idset":"11301"}}
+{"timestamp":1712611061.7578478,"name":"offline","context":{"idset":"11302"}}
+{"timestamp":1712611061.7583518,"name":"offline","context":{"idset":"11303"}}
+{"timestamp":1712611061.7588379,"name":"offline","context":{"idset":"11304"}}
+{"timestamp":1712611061.7593446,"name":"offline","context":{"idset":"11305"}}
+{"timestamp":1712611061.7598255,"name":"offline","context":{"idset":"11306"}}
+{"timestamp":1712611061.760402,"name":"offline","context":{"idset":"11307"}}
+{"timestamp":1712611061.7609489,"name":"offline","context":{"idset":"11308"}}
+{"timestamp":1712611061.7615533,"name":"offline","context":{"idset":"11309"}}
+{"timestamp":1712611061.7620873,"name":"offline","context":{"idset":"11310"}}
+{"timestamp":1712611061.7626741,"name":"offline","context":{"idset":"11311"}}
+{"timestamp":1712611061.7632086,"name":"offline","context":{"idset":"11312"}}
+{"timestamp":1712611061.7637649,"name":"offline","context":{"idset":"11313"}}
+{"timestamp":1712611061.7643387,"name":"offline","context":{"idset":"11314"}}
+{"timestamp":1712611061.7648544,"name":"offline","context":{"idset":"11315"}}
+{"timestamp":1712611061.7653639,"name":"offline","context":{"idset":"11316"}}
+{"timestamp":1712611061.7659214,"name":"offline","context":{"idset":"11317"}}
+{"timestamp":1712611061.7664888,"name":"offline","context":{"idset":"11318"}}
+{"timestamp":1712611061.7670448,"name":"offline","context":{"idset":"11319"}}
+{"timestamp":1712611061.7675648,"name":"offline","context":{"idset":"11320"}}
+{"timestamp":1712611061.7680697,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1712611061.7685854,"name":"offline","context":{"idset":"11322"}}
+{"timestamp":1712611061.7691028,"name":"offline","context":{"idset":"11323"}}
+{"timestamp":1712611061.7695937,"name":"offline","context":{"idset":"11324"}}
+{"timestamp":1712611061.7701108,"name":"offline","context":{"idset":"11325"}}
+{"timestamp":1712611061.7876978,"name":"offline","context":{"idset":"11326"}}
+{"timestamp":1712611061.8204901,"name":"offline","context":{"idset":"11327"}}
+{"timestamp":1712611061.8537815,"name":"offline","context":{"idset":"11328"}}
+{"timestamp":1712611061.8893807,"name":"offline","context":{"idset":"11329"}}
+{"timestamp":1712611061.9246948,"name":"offline","context":{"idset":"11330"}}
+{"timestamp":1712611061.9584756,"name":"offline","context":{"idset":"11331"}}
+{"timestamp":1712611061.9908438,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1712611062.0230002,"name":"offline","context":{"idset":"11333"}}
+{"timestamp":1712611062.0554273,"name":"offline","context":{"idset":"11334"}}
+{"timestamp":1712611062.0935826,"name":"offline","context":{"idset":"11335"}}
+{"timestamp":1712611062.1279795,"name":"offline","context":{"idset":"11336"}}
+{"timestamp":1712611062.1612153,"name":"offline","context":{"idset":"11337"}}
+{"timestamp":1712611062.194315,"name":"offline","context":{"idset":"11338"}}
+{"timestamp":1712611062.227349,"name":"offline","context":{"idset":"11339"}}
+{"timestamp":1712611062.2604339,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1712611062.2939506,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1712611062.327575,"name":"offline","context":{"idset":"11342"}}
+{"timestamp":1712611062.361021,"name":"offline","context":{"idset":"11343"}}
+{"timestamp":1712611062.3946147,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1712611062.4277523,"name":"offline","context":{"idset":"11345"}}
+{"timestamp":1712611062.4609368,"name":"offline","context":{"idset":"11346"}}
+{"timestamp":1712611062.4946535,"name":"offline","context":{"idset":"11347"}}
+{"timestamp":1712611062.5270588,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1712611062.5593894,"name":"offline","context":{"idset":"11349"}}
+{"timestamp":1712611062.5917838,"name":"offline","context":{"idset":"11350"}}
+{"timestamp":1712611062.6242349,"name":"offline","context":{"idset":"11351"}}
+{"timestamp":1712611062.6566858,"name":"offline","context":{"idset":"11352"}}
+{"timestamp":1712611062.6731327,"name":"offline","context":{"idset":"11353"}}
+{"timestamp":1712611062.6735926,"name":"offline","context":{"idset":"11354"}}
+{"timestamp":1712611062.6740386,"name":"offline","context":{"idset":"11355"}}
+{"timestamp":1712611062.6744947,"name":"offline","context":{"idset":"11356"}}
+{"timestamp":1712611062.6749387,"name":"offline","context":{"idset":"11357"}}
+{"timestamp":1712611062.6753924,"name":"offline","context":{"idset":"11359"}}
+{"timestamp":1712611062.6758335,"name":"offline","context":{"idset":"11360"}}
+{"timestamp":1712611062.6762874,"name":"offline","context":{"idset":"11361"}}
+{"timestamp":1712611062.6767292,"name":"offline","context":{"idset":"11362"}}
+{"timestamp":1712611062.6771641,"name":"offline","context":{"idset":"11363"}}
+{"timestamp":1712611062.6776164,"name":"offline","context":{"idset":"11364"}}
+{"timestamp":1712611062.6780503,"name":"offline","context":{"idset":"11365"}}
+{"timestamp":1712611062.6785014,"name":"offline","context":{"idset":"11366"}}
+{"timestamp":1712611062.6789351,"name":"offline","context":{"idset":"11367"}}
+{"timestamp":1712611062.6793742,"name":"offline","context":{"idset":"11368"}}
+{"timestamp":1712611062.6798069,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1712611062.6802359,"name":"offline","context":{"idset":"11370"}}
+{"timestamp":1712611062.6806803,"name":"offline","context":{"idset":"11371"}}
+{"timestamp":1712611062.681107,"name":"offline","context":{"idset":"11372"}}
+{"timestamp":1712611062.6815472,"name":"offline","context":{"idset":"11375"}}
+{"timestamp":1712611062.6819725,"name":"offline","context":{"idset":"11376"}}
+{"timestamp":1712611062.6824069,"name":"offline","context":{"idset":"11377"}}
+{"timestamp":1712611062.6828308,"name":"offline","context":{"idset":"11378"}}
+{"timestamp":1712611062.6832523,"name":"offline","context":{"idset":"11379"}}
+{"timestamp":1712611062.6836886,"name":"offline","context":{"idset":"11380"}}
+{"timestamp":1712611062.6841102,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1712611062.6845436,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1712611062.6849637,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1712611062.6853957,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1712611062.6858106,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1712611062.6862271,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1712611062.6866841,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1712611062.687099,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1712611062.6875322,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1712611062.6879461,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1712611062.6883707,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1712611062.6887813,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1712611062.6891918,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1712611062.6896229,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1712611062.6900308,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1712611062.6904516,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1712611062.6908584,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1712611062.6912613,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1712611062.6916769,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1712611062.6920822,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1712611062.6924989,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1712611062.6929009,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1712611062.6933143,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1712611062.6937158,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1712611062.6941166,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1712611062.6945333,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1712611062.6949315,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1712611062.6953559,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1712611062.6957538,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1712611062.6961484,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1712611062.6965733,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1712611062.6969719,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1712611062.6973748,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1712611062.7057908,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1712611062.7380519,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1712611062.7704785,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1712611062.8026793,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1712611062.8348248,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1712611062.8670278,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1712611062.8990874,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1712611062.9312527,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1712611062.9633932,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1712611062.9956491,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1712611063.0277412,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1712611063.0599158,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1712611063.0922451,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1712611063.1244833,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1712611063.1566889,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1712611063.1890726,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1712611063.2216489,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1712611063.2546866,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1712611063.2883635,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1712611063.3205068,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1712611063.3512275,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1712611063.3806057,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1712611063.4089923,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1712611063.4367745,"name":"offline","context":{"idset":"11439"}}
+{"timestamp":1712611063.4638498,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1712611063.4903753,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1712611063.5098724,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1712611063.5101717,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1712611063.5104787,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1712611063.5107753,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1712611063.5110707,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1712611063.5113814,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1712611063.5116725,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1712611063.5119619,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1712611063.5122523,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1712611063.5125539,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1712611063.5128436,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1712611063.5195048,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1712611063.5450161,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1712611063.5700898,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1712611063.594866,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1712611063.6190138,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1712611063.6426144,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1712611063.6661625,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1712611063.6903651,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1712611063.7130301,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1712611063.718657,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1712611063.7188969,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1712611063.719131,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1712611063.7193773,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1712611063.7196097,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1712611063.7198408,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1712611063.7200718,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1712611063.720315,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1712611063.7205548,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1712611063.720901,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1712611063.7211339,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1712611063.7213728,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1712611063.721601,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1712611063.7218268,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1712611063.7220531,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1712611063.7223144,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1712611063.738765,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1712611063.7497358,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1712611063.7499611,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1712611063.7501822,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1712611063.750442,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1712611063.7507555,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1712611063.7509739,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1712611063.7511904,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1712611063.7514253,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1712611063.751642,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1712611063.7518559,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1712611063.7520707,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1712611063.7522941,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1712611063.7525082,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1712611063.7527447,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1712611063.7531552,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1712611063.7534506,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1712611063.7536697,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1712611063.7538805,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1712611063.7540905,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1712611063.7543132,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1712611063.7545209,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1712611063.7547286,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1712611063.7549336,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1712611063.7551401,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1712611063.7553592,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1712611063.7555654,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1712611063.7609522,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1712611063.7611568,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1712611063.7613733,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1712611063.7615757,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1712611063.761776,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1712611063.761977,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1712611063.762176,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1712611063.7624173,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1712611063.7626173,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1712611063.7628145,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1712611063.7630122,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1712611063.7632086,"name":"offline","context":{"idset":"11517"}}
+{"timestamp":1712611063.7634194,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1712611063.7636175,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1712611063.7638135,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1712611063.7640071,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1712611063.7641988,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1712611063.7644041,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1712611063.7645965,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1712611063.7647893,"name":"offline","context":{"idset":"11525"}}
+{"timestamp":1712611063.7649806,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1712611063.7653096,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1712611063.7655072,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1712611063.765698,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1712611063.7658877,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1712611063.7660763,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1712611063.7662778,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1712611063.7771013,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1712611063.7773132,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1712611063.7775016,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1712611063.7776864,"name":"offline","context":{"idset":"11536"}}
+{"timestamp":1712611063.7778721,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1712611063.7780566,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1712611063.7782388,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1712611063.7784326,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1712611063.7786133,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1712611063.778795,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1712611063.7789752,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1712611063.7791533,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1712611063.7793553,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1712611063.7795362,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1712611063.7797148,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1712611063.7798936,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1712611063.7800698,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1712611063.7802472,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1712611063.7804627,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1712611063.7806394,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1712611063.7808139,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1712611063.7809882,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1712611063.7811615,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1712611063.781352,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1712611063.7815256,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1712611063.781698,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1712611063.7818689,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1712611063.7820408,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1712611063.7822118,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1712611063.7875993,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1712611063.7877717,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1712611063.7879388,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1712611063.7881067,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1712611063.7882848,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1712611063.7884524,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1712611063.7886164,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1712611063.7887819,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1712611063.7889483,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1712611063.789113,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1712611063.7893181,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1712611063.7894821,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1712611063.7896442,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1712611063.7898068,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1712611063.7899683,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1712611063.7901294,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1712611063.7903018,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1712611063.7904656,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1712611063.7906249,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1712611063.7907851,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1712611063.7909443,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1712611063.7911015,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1712611063.7912602,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1712611063.791429,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1712611063.7915854,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1712611063.7917433,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1712611063.7918983,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1712611063.7920504,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1712611063.7922032,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1712611063.7923639,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1712611063.7925155,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1712611063.7926674,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1712611063.7928178,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1712611063.7980747,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1712611063.7982295,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1712611063.7984192,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1712611063.798569,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1712611063.798717,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1712611063.7988651,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1712611063.7990124,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1712611063.7991588,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1712611063.7993512,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1712611063.7995017,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1712611063.799648,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1712611063.7997918,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1712611063.7999344,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1712611063.8000772,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1712611063.8002181,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1712611063.8003829,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1712611063.8005257,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1712611063.8006647,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1712611063.8008034,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1712611063.8009412,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1712611063.801079,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1712611063.8012171,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1712611063.8013644,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1712611063.8015022,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1712611063.8016388,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1712611063.8017769,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1712611063.8019109,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1712611063.8020473,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1712611063.802181,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1712611063.8023257,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1712611063.8024595,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1712611063.8025925,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1712611063.8027234,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1712611063.802856,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1712611063.8029854,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1712611063.8031151,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1712611063.8032432,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1712611063.8033836,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1712611063.8035135,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1712611063.8088646,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1712611063.8090005,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1712611063.8091273,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1712611063.8092525,"name":"offline","context":{"idset":"11653"}}
+{"timestamp":1712611063.8093956,"name":"offline","context":{"idset":"11654"}}
+{"timestamp":1712611063.8095226,"name":"offline","context":{"idset":"11655"}}
+{"timestamp":1712611063.8096461,"name":"offline","context":{"idset":"11656"}}
+{"timestamp":1712611063.8097694,"name":"offline","context":{"idset":"11657"}}
+{"timestamp":1712611063.8098924,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1712611063.8100142,"name":"offline","context":{"idset":"11659"}}
+{"timestamp":1712611063.8101361,"name":"offline","context":{"idset":"11660"}}
+{"timestamp":1712611063.8102572,"name":"offline","context":{"idset":"11661"}}
+{"timestamp":1712611063.8104022,"name":"offline","context":{"idset":"11662"}}
+{"timestamp":1712611063.810524,"name":"offline","context":{"idset":"11663"}}
+{"timestamp":1712611063.8106439,"name":"offline","context":{"idset":"11664"}}
+{"timestamp":1712611063.8107631,"name":"offline","context":{"idset":"11665"}}
+{"timestamp":1712611063.8108835,"name":"offline","context":{"idset":"11666"}}
+{"timestamp":1712611063.8110011,"name":"offline","context":{"idset":"11667"}}
+{"timestamp":1712611063.8111167,"name":"offline","context":{"idset":"11668"}}
+{"timestamp":1712611063.8112335,"name":"offline","context":{"idset":"11669"}}
+{"timestamp":1712611063.8113642,"name":"offline","context":{"idset":"11670"}}
+{"timestamp":1712611063.8114817,"name":"offline","context":{"idset":"11671"}}
+{"timestamp":1712611063.8115964,"name":"offline","context":{"idset":"11672"}}
+{"timestamp":1712611063.8117142,"name":"offline","context":{"idset":"11673"}}
+{"timestamp":1712611063.8118281,"name":"offline","context":{"idset":"11674"}}
+{"timestamp":1712611063.8119428,"name":"offline","context":{"idset":"11675"}}
+{"timestamp":1712611063.8120553,"name":"offline","context":{"idset":"11676"}}
+{"timestamp":1712611063.8121691,"name":"offline","context":{"idset":"11677"}}
+{"timestamp":1712611063.8123014,"name":"offline","context":{"idset":"11678"}}
+{"timestamp":1712611063.8124161,"name":"offline","context":{"idset":"11679"}}
+{"timestamp":1712611063.8125274,"name":"offline","context":{"idset":"11680"}}
+{"timestamp":1712611063.8126376,"name":"offline","context":{"idset":"11681"}}
+{"timestamp":1712611063.8127458,"name":"offline","context":{"idset":"11682"}}
+{"timestamp":1712611063.8128541,"name":"offline","context":{"idset":"11684"}}
+{"timestamp":1712611063.8129616,"name":"offline","context":{"idset":"11685"}}
+{"timestamp":1712611063.8130698,"name":"offline","context":{"idset":"11686"}}
+{"timestamp":1712611063.8131764,"name":"offline","context":{"idset":"11687"}}
+{"timestamp":1712611063.8132963,"name":"offline","context":{"idset":"11688"}}
+{"timestamp":1712611063.8134041,"name":"offline","context":{"idset":"11689"}}
+{"timestamp":1712611063.813509,"name":"offline","context":{"idset":"11690"}}
+{"timestamp":1712611063.8136144,"name":"offline","context":{"idset":"11691"}}
+{"timestamp":1712611063.8137171,"name":"offline","context":{"idset":"11692"}}
+{"timestamp":1712611063.8138199,"name":"offline","context":{"idset":"11693"}}
+{"timestamp":1712611063.8139205,"name":"offline","context":{"idset":"11695"}}
+{"timestamp":1712611063.8140204,"name":"offline","context":{"idset":"11696"}}
+{"timestamp":1712611063.8190658,"name":"offline","context":{"idset":"11697"}}
+{"timestamp":1712611063.8241525,"name":"offline","context":{"idset":"11698"}}
+{"timestamp":1712611063.824259,"name":"offline","context":{"idset":"11699"}}
+{"timestamp":1712611063.8243773,"name":"offline","context":{"idset":"11700"}}
+{"timestamp":1712611063.824477,"name":"offline","context":{"idset":"11701"}}
+{"timestamp":1712611063.8245747,"name":"offline","context":{"idset":"11702"}}
+{"timestamp":1712611063.8246722,"name":"offline","context":{"idset":"11703"}}
+{"timestamp":1712611063.8247695,"name":"offline","context":{"idset":"11704"}}
+{"timestamp":1712611063.8248658,"name":"offline","context":{"idset":"11705"}}
+{"timestamp":1712611063.8249612,"name":"offline","context":{"idset":"11706"}}
+{"timestamp":1712611063.8250546,"name":"offline","context":{"idset":"11707"}}
+{"timestamp":1712611063.8251503,"name":"offline","context":{"idset":"11708"}}
+{"timestamp":1712611063.8252451,"name":"offline","context":{"idset":"11709"}}
+{"timestamp":1712611063.8254175,"name":"offline","context":{"idset":"11710"}}
+{"timestamp":1712611063.8256621,"name":"offline","context":{"idset":"11711"}}
+{"timestamp":1712611063.8258162,"name":"offline","context":{"idset":"11712"}}
+{"timestamp":1712611063.8259642,"name":"offline","context":{"idset":"11713"}}
+{"timestamp":1712611063.8261101,"name":"offline","context":{"idset":"11714"}}
+{"timestamp":1712611063.8263273,"name":"offline","context":{"idset":"11715"}}
+{"timestamp":1712611063.8264911,"name":"offline","context":{"idset":"11716"}}
+{"timestamp":1712611063.8266366,"name":"offline","context":{"idset":"11717"}}
+{"timestamp":1712611063.8267791,"name":"offline","context":{"idset":"11718"}}
+{"timestamp":1712611063.8269198,"name":"offline","context":{"idset":"11719"}}
+{"timestamp":1712611063.8270593,"name":"offline","context":{"idset":"11720"}}
+{"timestamp":1712611063.8271973,"name":"offline","context":{"idset":"11721"}}
+{"timestamp":1712611063.8273518,"name":"offline","context":{"idset":"11722"}}
+{"timestamp":1712611063.8274922,"name":"offline","context":{"idset":"11723"}}
+{"timestamp":1712611063.8276296,"name":"offline","context":{"idset":"11724"}}
+{"timestamp":1712611063.8277647,"name":"offline","context":{"idset":"11725"}}
+{"timestamp":1712611063.8279002,"name":"offline","context":{"idset":"11726"}}
+{"timestamp":1712611063.8280337,"name":"offline","context":{"idset":"11727"}}
+{"timestamp":1712611063.8281665,"name":"offline","context":{"idset":"11728"}}
+{"timestamp":1712611063.8283417,"name":"offline","context":{"idset":"11729"}}
+{"timestamp":1712611063.8284743,"name":"offline","context":{"idset":"11730"}}
+{"timestamp":1712611063.828604,"name":"offline","context":{"idset":"11731"}}
+{"timestamp":1712611063.8287337,"name":"offline","context":{"idset":"11732"}}
+{"timestamp":1712611063.8288617,"name":"offline","context":{"idset":"11733"}}
+{"timestamp":1712611063.8289895,"name":"offline","context":{"idset":"11734"}}
+{"timestamp":1712611063.8291178,"name":"offline","context":{"idset":"11735"}}
+{"timestamp":1712611063.8292539,"name":"offline","context":{"idset":"11736"}}
+{"timestamp":1712611063.8294096,"name":"offline","context":{"idset":"11737"}}
+{"timestamp":1712611063.8295364,"name":"offline","context":{"idset":"11738"}}
+{"timestamp":1712611063.8296595,"name":"offline","context":{"idset":"11739"}}
+{"timestamp":1712611063.8297815,"name":"offline","context":{"idset":"11740"}}
+{"timestamp":1712611063.8299167,"name":"offline","context":{"idset":"11741"}}
+{"timestamp":1712611063.8300819,"name":"offline","context":{"idset":"11742"}}
+{"timestamp":1712611063.8303316,"name":"offline","context":{"idset":"11743"}}
+{"timestamp":1712611063.8304679,"name":"offline","context":{"idset":"11744"}}
+{"timestamp":1712611063.8305876,"name":"offline","context":{"idset":"11745"}}
+{"timestamp":1712611063.8307233,"name":"offline","context":{"idset":"11746"}}
+{"timestamp":1712611063.8309174,"name":"offline","context":{"idset":"11747"}}
+{"timestamp":1712611063.831049,"name":"offline","context":{"idset":"11748"}}
+{"timestamp":1712611063.8311832,"name":"offline","context":{"idset":"11749"}}
+{"timestamp":1712611063.8313203,"name":"offline","context":{"idset":"11750"}}
+{"timestamp":1712611063.8314352,"name":"offline","context":{"idset":"11751"}}
+{"timestamp":1712611063.8315501,"name":"offline","context":{"idset":"11752"}}
+{"timestamp":1712611063.8316612,"name":"offline","context":{"idset":"11753"}}
+{"timestamp":1712611063.8317719,"name":"offline","context":{"idset":"11754"}}
+{"timestamp":1712611063.8318813,"name":"offline","context":{"idset":"11755"}}
+{"timestamp":1712611063.8319905,"name":"offline","context":{"idset":"11756"}}
+{"timestamp":1712611063.8321044,"name":"offline","context":{"idset":"11757"}}
+{"timestamp":1712611063.8322132,"name":"offline","context":{"idset":"11758"}}
+{"timestamp":1712611063.8323357,"name":"offline","context":{"idset":"11759"}}
+{"timestamp":1712611063.8324416,"name":"offline","context":{"idset":"11760"}}
+{"timestamp":1712611063.8325484,"name":"offline","context":{"idset":"11761"}}
+{"timestamp":1712611063.8326509,"name":"offline","context":{"idset":"11762"}}
+{"timestamp":1712611063.8327558,"name":"offline","context":{"idset":"11763"}}
+{"timestamp":1712611063.8328574,"name":"offline","context":{"idset":"11764"}}
+{"timestamp":1712611063.8329592,"name":"offline","context":{"idset":"11765"}}
+{"timestamp":1712611063.8330603,"name":"offline","context":{"idset":"11766"}}
+{"timestamp":1712611063.8331604,"name":"offline","context":{"idset":"11767"}}
+{"timestamp":1712611063.8332603,"name":"offline","context":{"idset":"11768"}}
+{"timestamp":1712611063.8333795,"name":"offline","context":{"idset":"11769"}}
+{"timestamp":1712611063.8334787,"name":"offline","context":{"idset":"11770"}}
+{"timestamp":1712611063.8418295,"name":"offline","context":{"idset":"11771"}}
+{"timestamp":1712611063.8419387,"name":"offline","context":{"idset":"11772"}}
+{"timestamp":1712611063.8420353,"name":"offline","context":{"idset":"11773"}}
+{"timestamp":1712611063.8421292,"name":"offline","context":{"idset":"11774"}}
+{"timestamp":1712611063.8422225,"name":"offline","context":{"idset":"11775"}}
+{"timestamp":1712611063.8423274,"name":"offline","context":{"idset":"11776"}}
+{"timestamp":1712611063.8424208,"name":"offline","context":{"idset":"11777"}}
+{"timestamp":1712611063.8425131,"name":"offline","context":{"idset":"11778"}}
+{"timestamp":1712611063.8426032,"name":"offline","context":{"idset":"11779"}}
+{"timestamp":1712611063.8426933,"name":"offline","context":{"idset":"11780"}}
+{"timestamp":1712611063.842782,"name":"offline","context":{"idset":"11781"}}
+{"timestamp":1712611063.8428702,"name":"offline","context":{"idset":"11782"}}
+{"timestamp":1712611063.8429585,"name":"offline","context":{"idset":"11783"}}
+{"timestamp":1712611063.8430448,"name":"offline","context":{"idset":"11784"}}
+{"timestamp":1712611063.8431289,"name":"offline","context":{"idset":"11785"}}
+{"timestamp":1712611063.8432133,"name":"offline","context":{"idset":"11786"}}
+{"timestamp":1712611063.8433123,"name":"offline","context":{"idset":"11787"}}
+{"timestamp":1712611063.8433976,"name":"offline","context":{"idset":"11788"}}
+{"timestamp":1712611063.8434787,"name":"offline","context":{"idset":"11789"}}
+{"timestamp":1712611063.8435588,"name":"offline","context":{"idset":"11790"}}
+{"timestamp":1712611063.8436375,"name":"offline","context":{"idset":"11791"}}
+{"timestamp":1712611063.843715,"name":"offline","context":{"idset":"11792"}}
+{"timestamp":1712611063.8438001,"name":"offline","context":{"idset":"11793"}}
+{"timestamp":1712611063.8438842,"name":"offline","context":{"idset":"11794"}}
+{"timestamp":1712611063.8439658,"name":"offline","context":{"idset":"11795"}}
+{"timestamp":1712611063.8440402,"name":"offline","context":{"idset":"11796"}}
+{"timestamp":1712611063.8441141,"name":"offline","context":{"idset":"11813"}}
+{"timestamp":1712611063.844187,"name":"offline","context":{"idset":"11814"}}
+{"timestamp":1712611063.84426,"name":"offline","context":{"idset":"11815"}}
+{"timestamp":1712611063.8443441,"name":"offline","context":{"idset":"11816"}}
+{"timestamp":1712611063.8444159,"name":"offline","context":{"idset":"11817"}}
+{"timestamp":1712611063.844486,"name":"offline","context":{"idset":"11818"}}
+{"timestamp":1712611063.8445542,"name":"offline","context":{"idset":"11819"}}
+{"timestamp":1712611063.8446221,"name":"offline","context":{"idset":"11820"}}
+{"timestamp":1712611063.8446879,"name":"offline","context":{"idset":"11821"}}
+{"timestamp":1712611063.8447547,"name":"offline","context":{"idset":"11822"}}
+{"timestamp":1712611063.8448193,"name":"offline","context":{"idset":"11823"}}
+{"timestamp":1712611063.844883,"name":"offline","context":{"idset":"11824"}}
+{"timestamp":1712611063.8449454,"name":"offline","context":{"idset":"11825"}}
+{"timestamp":1712611063.8450069,"name":"offline","context":{"idset":"11826"}}
+{"timestamp":1712611063.8450704,"name":"offline","context":{"idset":"11827"}}
+{"timestamp":1712611063.8450968,"name":"offline","context":{"idset":"11828"}}
+{"timestamp":1712611163.1942134,"name":"resource-init","context":{"restart":true,"drain":{"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"348":{"timestamp":1711576490.2548447,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"431":{"timestamp":1711096934.358676,"reason":"nodediag failed pci"},"748":{"timestamp":1710804566.3281977,"reason":"cxi: IP ping fails"},"789-794":{"timestamp":1711461777.1299369,"reason":""},"797-798":{"timestamp":1712261862.602201,"reason":"--reason swapping blades into x1900 - wendy"},"819":{"timestamp":1712342273.5977426,"reason":""},"820":{"timestamp":1712342217.7655268,"reason":""},"821-824":{"timestamp":1712333755.1207211,"reason":""},"883-884":{"timestamp":1711734121.1560712,"reason":""},"9316":{"timestamp":1712350549.0126185,"reason":"Fails loopback test --JRG"},"9447":{"timestamp":1712272276.4058383,"reason":"CXI Stuck at Starting --JRG"},"9448":{"timestamp":1712272297.6105912,"reason":"bad partner node --JRG"},"9569":{"timestamp":1712606186.1312208,"reason":"broker was unresponsive"},"9570":{"timestamp":1712606186.2319283,"reason":"broker was unresponsive"},"9750":{"timestamp":1712336418.2480905,"reason":"nodediag failed amdapu"},"10261":{"timestamp":1712361648.1654882,"reason":"nodediag failed dgemm_perf"},"10303":{"timestamp":1712236781.0769484,"reason":"bad partner node --JRG"},"10355":{"timestamp":1712337664.7496204,"reason":"bad partner node --JRG"},"10517":{"timestamp":1712252115.930419,"reason":"bad partner node"},"10518":{"timestamp":1712086791.4119098,"reason":"Node wont power on --JRG"},"10658":{"timestamp":1712598484.8393898,"reason":"broker was unresponsive"},"10871-10872,10911-10912,10943-10944":{"timestamp":1712523855.472235,"reason":""},"10952":{"timestamp":1712592515.8810503,"reason":""},"10920,10925,10930,10935,10940,10945,10950,10955,10960,10965":{"timestamp":1712507305.3461049,"reason":""},"10869-10870,10873-10910,10913-10919,10921-10924,10926-10929,10931-10934,10936-10939,10941-10942,10946-10949,10951,10953-10954,10956-10959,10961-10964,10966-10969,10971-10974,10976-10979,10981-10984,10986-10989,10991-10992":{"timestamp":1712594249.0262253,"reason":""},"10970,10975,10980,10985,10990,10993-10996":{"timestamp":1712509491.5233767,"reason":""},"11021":{"timestamp":1712586553.6810555,"reason":"nodediag failed dgemm_perf"},"11117":{"timestamp":1712596642.1290939,"reason":"broker was unresponsive"},"11118":{"timestamp":1712596642.9024158,"reason":"broker was unresponsive"},"11317-11332":{"timestamp":1712602153.7117822,"reason":"draining to run on-node HPE diags - KPN"},"11381":{"timestamp":1712597522.1291575,"reason":"broker was unresponsive"},"11382":{"timestamp":1712597522.9041014,"reason":"broker was unresponsive"},"11383":{"timestamp":1712597528.1294539,"reason":"broker was unresponsive"},"11384":{"timestamp":1712597528.1296108,"reason":"broker was unresponsive"},"11385":{"timestamp":1712597528.1297331,"reason":"broker was unresponsive"},"11386":{"timestamp":1712597528.1298556,"reason":"broker was unresponsive"},"11389":{"timestamp":1712597528.1299732,"reason":"broker was unresponsive"},"11391":{"timestamp":1712597528.1300926,"reason":"broker was unresponsive"},"11392":{"timestamp":1712597528.1302114,"reason":"broker was unresponsive"},"11393":{"timestamp":1712597528.1303468,"reason":"broker was unresponsive"},"11394":{"timestamp":1712597528.1304727,"reason":"broker was unresponsive"},"11396":{"timestamp":1712597528.9531279,"reason":"broker was unresponsive"},"11400-11412":{"timestamp":1712270907.4221985,"reason":""},"11413-11428":{"timestamp":1712271808.3093069,"reason":""},"11466":{"timestamp":1712329120.7360339,"reason":""},"11765-11776,11779-11780":{"timestamp":1711993308.7730825,"reason":"Diags Running"},"11784":{"timestamp":1711725755.0632687,"reason":"Running Diags - Rabbits Off"},"11790":{"timestamp":1712004602.484952,"reason":"bad partner node - JRG"},"11789,11793":{"timestamp":1712004602.484952,"reason":"kernel panics - JRG"},"11794":{"timestamp":1711725767.0656931,"reason":"bad partner node - JRG"},"11781-11783,11785-11788,11791-11792,11795-11796":{"timestamp":1712004602.484952,"reason":"Running Diags - Rabbits Off"},"11802":{"timestamp":1712603456.1319213,"reason":"broker was unresponsive"},"11803":{"timestamp":1711668961.0618374,"reason":"Running Diags - Rabbits Off"},"11797-11801,11804-11811":{"timestamp":1712004859.9543238,"reason":"Running Diags - Rabbits Off"},"11812":{"timestamp":1712004514.6037207,"reason":"Running Diags - Rabbits Off"},"11813-11828":{"timestamp":1712005088.7732615,"reason":"Running Diags - Rabbits Off"},"11829-11844":{"timestamp":1712068080.9443812,"reason":"Running Diags - Rabbits Off"},"11845-11859":{"timestamp":1712068141.3627269,"reason":"Running Diags - Rabbits Off"},"11860":{"timestamp":1711727735.0622723,"reason":"Running Diags - Rabbits Off"},"11861-11876":{"timestamp":1712068186.5559592,"reason":"Running Diags - Rabbits Off"}},"online":"","exclude":"0"}}
+{"timestamp":1712611163.221462,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1712611183.740927,"name":"online","context":{"idset":"0"}}
+{"timestamp":1712611185.4958355,"name":"online","context":{"idset":"9206,9251,9333,9352,9366,9553,9636,9647,9676,9700,9702,9750,10000,10282,10489,10524-10525,10563,10633,10777,10800,11419,11423,11598,11600"}}
+{"timestamp":1712611185.6212356,"name":"online","context":{"idset":"820,872,951-952,976,9209,9211,9214-9215,9227,9231,9235,9238,9245,9250,9252,9256-9257,9260,9263,9273,9276,9290-9291,9293-9294,9296-9297,9308,9329,9331,9342,9353,9355,9361-9362,9369,9379,9381,9385,9390-9391,9398-9399,9411,9415,9421,9426-9427,9438,9446,9451,9456,9458,9466,9473,9480,9486,9490,9497,9507,9509,9512,9522,9524,9536,9542,9545,9549,9563,9572-9573,9575,9578,9581,9589,9592,9595-9597,9601,9603,9606,9614-9615,9617-9618,9634-9635,9637,9643,9646,9648-9650,9652,9657,9660,9662,9664,9668-9671,9677,9682,9693-9694,9701,9735,9742,9746,9754,9756,9760,9763,9776,9778,9795,9799,9803,9805,9809,9813-9814,9817,9820,9822,9842,9975-9977,9979-9980,9987,9989,10004,10010,10027-10030,10036-10037,10040,10043,10047,10049,10062,10069,10073,10083,10086,10088,10090,10094,10097,10099,10101-10102,10111,10116-10117,10129,10134,10137,10154,10158,10163,10165,10214,10217,10225-10226,10237,10245,10247,10250,10263,10271,10273,10280-10281,10302,10316,10324,10337,10352,10354,10364,10373,10391-10392,10401,10418,10429,10487,10493,10500,10507,10514,10529,10551,10559,10567,10571,10575,10579-10580,10585,10587,10589,10603,10631,10651,10665,10672,10676,10678,10686,10706,10714,10732,10757,10759,10765,10787,10791,10795,10799,10802,11389,11407,11409-11410,11554-11555,11557,11561,11564,11570,11572-11573,11595,11597,11769,11774,11780-11781,11786,11789,11795,11814,11823,11826"}}
+{"timestamp":1712611185.7320116,"name":"online","context":{"idset":"830,842,954,963,9216,9226,9232-9233,9241,9253,9255,9269-9271,9274,9278,9282,9284-9286,9302-9303,9306,9317,9319,9332,9337,9340,9349-9351,9364,9368,9380,9387,9395,9401,9403,9405,9412,9419-9420,9428,9430,9433,9435,9445,9453,9478-9479,9487-9488,9492-9493,9495,9508,9510,9514,9526-9527,9537-9539,9544,9550,9577,9579,9590,9609,9611,9616,9620,9622,9625,9628,9632,9639-9640,9663,9687-9688,9704,9710-9711,9713,9715,9733,9758,9764,9773,9775,9781,9788,9800,9804,9806,9815,9825-9826,9828,9830,9832,9835,9973,9983,9991,10013,10018,10041,10046,10050,10067,10078-10079,10081,10092,10107-10108,10118,10132,10147,10153,10159,10162,10164,10169,10176,10188-10189,10198,10201,10205,10210-10212,10223,10228,10243-10244,10246,10251,10253,10258,10265,10268-10269,10278-10279,10290,10293,10299,10312,10318,10325,10340,10348,10350,10358,10361,10365,10369,10371,10377,10379,10389,10411,10414,10422,10424,10430-10431,10436,10492,10495,10497-10498,10506,10533,10535,10538,10543,10546,10556,10560,10570,10574,10598,10604,10611,10613,10616,10621,10628,10632,10647,10659,10663,10668,10673-10675,10677,10679,10682,10685,10690-10691,10696,10703,10705,10711,10719-10720,10728,10731,10733,10736,10767,10779,10781,10784-10785,10806,11383,11392,11400,11403,11408,11412,11416,11420-11421,11425,11512,11547,11560,11562,11565,11569,11571,11575,11582-11583,11585,11593-11594,11596,11601,11604,11767,11775,11784,11791,11793,11818,11822,11825,11828"}}
+{"timestamp":1712611185.8909588,"name":"online","context":{"idset":"431,828-829,831-832,834,841,871,876,949-950,953,975,9207-9208,9210,9213,9217-9223,9225,9228-9230,9234,9236-9237,9239-9240,9242-9243,9247-9249,9254,9258-9259,9262,9264-9268,9277,9279-9280,9283,9287-9289,9292,9295,9299-9301,9304-9305,9307,9309-9314,9320-9326,9328,9330,9334-9336,9338-9339,9341,9343-9348,9354,9356,9358-9360,9363,9365,9367,9370-9377,9383,9386,9388-9389,9392,9394,9396-9397,9400,9402,9407-9409,9414,9416-9418,9422-9425,9431-9432,9434,9436-9437,9439,9441,9443,9449-9450,9452,9455,9457,9459-9463,9465,9467-9471,9474-9477,9481-9482,9484-9485,9489,9491,9494,9496,9498-9503,9505-9506,9511,9513,9515,9517-9521,9523,9528-9530,9541,9543,9546-9548,9551-9552,9554-9560,9564,9567-9568,9574,9576,9580,9582,9585-9586,9588,9591,9593-9594,9598-9599,9605,9607-9608,9610,9612-9613,9619,9621,9623-9624,9626,9630,9638,9641-9642,9644-9645,9651,9653-9656,9658-9659,9661,9666-9667,9673-9675,9678,9680-9681,9683,9686,9689-9692,9695-9699,9703,9705,9707-9709,9712,9714,9716,9734,9736-9739,9741,9743-9745,9747,9749,9751-9753,9755,9759,9761-9762,9765-9772,9774,9777,9779-9780,9782-9785,9789,9791-9794,9796,9798,9802,9808,9810-9811,9816,9818-9819,9821,9823-9824,9827,9829,9831,9833-9834,9836-9841,9843-9844,9974,9978,9982,9984,9986,9988,9990,9992-9995,9997-9999,10001-10002,10005,10007,10009,10011-10012,10016-10017,10019-10022,10025-10026,10032-10033,10038-10039,10042,10044-10045,10048,10051-10059,10061,10063-10065,10068,10070-10072,10074,10076-10077,10080,10082,10084-10085,10087,10089,10091,10093,10095-10096,10098,10100,10103-10106,10109-10110,10112-10113,10119-10122,10124-10128,10130-10131,10133,10135-10136,10139,10141-10142,10144,10146,10148-10152,10157,10160-10161,10166-10168,10172-10175,10177-10187,10191-10196,10200,10202-10204,10206-10209,10213,10215-10216,10218-10222,10224,10227,10229-10236,10238-10242,10248-10249,10252,10254-10257,10259-10260,10264,10266-10267,10270,10272,10274-10276,10283-10284,10286-10289,10291-10292,10294-10298,10300-10301,10304-10311,10313-10315,10317,10319-10323,10328-10332,10334,10336,10338,10341-10342,10345,10347,10349,10351,10359-10360,10362,10366-10368,10370,10372,10374-10376,10381-10383,10385-10388,10390,10393-10395,10397-10400,10402-10403,10405-10407,10409-10410,10412-10413,10415-10417,10419-10421,10423,10425-10428,10432-10435,10485,10488,10490-10491,10494,10496,10502,10504,10508,10510-10513,10515-10516,10519-10523,10526-10528,10531-10532,10534,10536-10537,10539-10542,10544-10545,10547-10549,10552-10555,10557-10558,10561-10562,10564,10566,10568-10569,10573,10576-10578,10581-10584,10586,10588,10590-10593,10595-10597,10599,10601-10602,10605-10609,10612,10614-10615,10617,10619,10622,10624-10627,10630,10636,10640-10641,10645-10646,10648-10650,10652-10653,10655-10656,10660-10662,10664,10666-10667,10669,10671,10680-10681,10683-10684,10687-10689,10692-10693,10695,10697-10701,10704,10707-10710,10713,10715-10717,10721-10723,10725-10727,10729-10730,10734-10735,10737-10738,10758,10760-10764,10766,10768-10772,10774-10776,10778,10780,10782-10783,10786,10788-10790,10792-10794,10796-10798,10801,10803-10805,10807,11329,11381-11382,11384-11386,11391,11393-11394,11396,11402,11405-11406,11411,11413-11415,11417-11418,11422,11424,11427-11428,11466,11548-11549,11551,11553,11556,11558-11559,11566-11568,11574,11576-11580,11584,11586,11589,11591-11592,11599,11602-11603,11765-11766,11768,11770-11773,11776,11779,11782-11783,11785,11788,11792,11794,11813,11815,11820-11821,11824,11827"}}
+{"timestamp":1712611186.0310397,"name":"online","context":{"idset":"109,117,123,254,585,827,833,964,9205,9212,9224,9244,9246,9261,9272,9275,9281,9298,9318,9327,9357,9378,9382,9384,9393,9404,9406,9410,9413,9429,9440,9442,9444,9454,9464,9483,9504,9516,9525,9531-9532,9540,9565-9566,9571,9583-9584,9587,9600,9602,9604,9627,9629,9631,9633,9665,9672,9679,9684-9685,9706,9740,9748,9757,9786,9797,9801,9807,9812,9981,9985,9996,10003,10006,10008,10014-10015,10023-10024,10031,10034-10035,10060,10066,10075,10114-10115,10123,10140,10170-10171,10190,10197,10199,10277,10285,10326-10327,10333,10335,10339,10346,10353,10357,10363,10378,10380,10384,10404,10408,10486,10499,10503,10505,10509,10530,10550,10565,10572,10594,10600,10610,10618,10620,10623,10635,10654,10670,10694,10724,10773,11330,11401,11404,11550,11563,11581,11590,11787,11790,11796,11816-11817,11819"}}
+{"timestamp":1712611186.2151613,"name":"online","context":{"idset":"85-108,110-116,118-120,122,124-216,218-253,255-296,298-310,312-338,341-347,349-355,357-369,372-391,393-404,406-412,414-420,422-423,426-427,429-430,432-444,469,471-476,478-482,484-491,493-497,499-503,505-507,509-512,514-516,518-524,526-540,565-571,573-584,586-588,629-636,11113,11388,11390,11395,11397-11398,11587-11588,11777-11778"}}
+{"timestamp":1712611186.3288701,"name":"online","context":{"idset":"297,311,339-340,356,370-371,392,405,413,421,424-425,428,470,477,483,492,498,504,508,513,517,525,572,10814,10970,11122,11124,11387,11399,11449,11541-11543,11545-11546,11552,11678"}}
+{"timestamp":1712611186.5120764,"name":"online","context":{"idset":"7,55,10742,10747,10751,10811,10815,10842,10855,10873,10879,10884,10888,10897,10903,10934-10935,10945,10956,10975,11016,11060,11070,11072,11078,11107-11108,11110,11114,11255,11266,11286-11287,11316,11318,11325,11328,11336,11359,11362,11375,11379-11380,11443,11446,11469-11470,11520,11524,11539,11544,11616,11655,11679,11686-11687,11708,11720,11738,11755"}}
+{"timestamp":1712611186.6364813,"name":"online","context":{"idset":"16,11036,11057,11339,11439,11725"}}
+{"timestamp":1712611186.7567828,"name":"online","context":{"idset":"11,40,51,56,10748,10753,10808-10810,10812-10813,10817,10819,10824,10828,10834,10837,10841,10854,10860-10861,10869,10874,10878,10880-10883,10887,10901-10902,10905,10907-10910,10913,10915-10916,10930-10931,10933,10938-10939,10946-10947,10950,10953,10958-10959,10962,10968-10969,10973,10984-10985,10995,11002-11003,11008,11011-11012,11015,11018,11024-11025,11034,11041-11043,11045,11064-11066,11068,11074,11076,11081,11089,11102-11103,11109,11116,11119-11121,11123,11254,11257,11261,11276,11279,11293,11301,11307,11317,11320,11327,11331,11351-11352,11361,11366,11371-11372,11429,11433,11435,11441,11445,11457,11462,11468,11472,11474,11476,11487,11489,11506-11507,11510,11516,11518-11519,11523,11526,11531,11537,11606,11611,11613,11624,11626-11629,11634,11656,11661-11662,11669,11681,11690,11696,11700,11721,11726,11734-11735,11739,11754,11756"}}
+{"timestamp":1712611186.8781533,"name":"online","context":{"idset":"1-6,8-10,12,14-15,17-31,33-39,41-42,44-50,52-54,57-60,10437,10458,10744,10755,10822,10830,10838,10865,10868,10876,10893,10896,10922,10928,10936-10937,10941,10954,10960-10961,10986,10991,10997,11006,11010,11023,11027-11028,11031,11048,11055,11062,11071,11085,11094,11260,11275,11281,11283,11295-11296,11302,11309,11313,11338,11342,11347,11350,11354,11364,11442,11444,11475,11494,11525,11529,11533,11609,11620,11622,11664,11675,11709,11722,11724,11741-11743,11747"}}
+{"timestamp":1712611187.0028324,"name":"online","context":{"idset":"13,32,43,9732,10438,10453-10454,10464,10466,10745,10749,10752,10756,10818,10820,10823,10826,10833,10835,10843,10847-10848,10852-10853,10858-10859,10864,10866-10867,10870,10875,10877,10885-10886,10890-10892,10894-10895,10898-10900,10906,10914,10917,10919-10920,10923,10925,10927,10932,10940,10957,10963,10967,10979,10981,10983,10988,10990,10992,10998,11004,11007,11013-11014,11019-11020,11022,11026,11029,11032,11035,11039-11040,11044,11051,11054,11058-11059,11069,11077,11079-11080,11084,11086-11087,11092,11098-11100,11104,11106,11253,11268,11272,11274,11280,11284,11292,11297,11300,11305,11312,11319,11326,11335,11346,11363,11367,11376,11378,11451,11460,11484-11485,11492,11495,11497,11505,11508,11608,11615,11671,11677,11684,11692,11698-11699,11706,11712,11715-11716,11731,11763"}}
+{"timestamp":1712611187.1121945,"name":"online","context":{"idset":"9717,9723-9724,10444-10445,10448-10450,10459,10462,10467,10741,10743,10746,10754,10816,10821,10827,10831,10836,10840,10844,10846,10849-10851,10856-10857,10862-10863,10889,10904,10924,10926,10929,10948-10949,10955,10964-10966,10974,10977,10980,10982,10987,10989,10993-10994,10996,11001,11005,11017,11021,11030,11033,11038,11046-11047,11049-11050,11052-11053,11056,11063,11067,11073,11075,11082-11083,11088,11090-11091,11093,11095,11097,11101,11105,11258,11264-11265,11269-11271,11273,11277-11278,11285,11289,11291,11294,11298-11299,11306,11310,11314-11315,11321,11323-11324,11332,11337,11343-11345,11348,11353,11360,11368-11370,11377,11430,11434,11436-11438,11440,11448,11450,11452-11456,11458,11463,11465,11467,11471,11473,11477-11479,11481-11483,11486,11488,11490-11491,11493,11496,11498-11501,11504,11514-11515,11521,11528,11530,11532,11535,11540,11607,11614,11617,11619,11630-11633,11636,11653-11654,11657-11658,11663,11665-11668,11670,11672-11673,11676,11682,11685,11689,11691,11693,11695,11701,11703-11705,11707,11710-11711,11728,11733,11736,11745-11746,11749-11750,11752-11753,11757-11762"}}
+{"timestamp":1712611187.2161977,"name":"online","context":{"idset":"9722,10439,10442-10443,10446-10447,10451,10455,10461,10750,10832,10839,10845,10918,10921,10942,10951,10971-10972,10976,10978,11009,11096,11256,11263,11267,11282,11288,11290,11303-11304,11308,11311,11322,11334,11341,11355,11357,11365,11432,11447,11459,11480,11502-11503,11511,11517,11522,11527,11536,11605,11610,11612,11618,11621,11660,11680,11688,11697,11713,11717-11718,11727,11732,11737,11764"}}
+{"timestamp":1712611187.3339529,"name":"online","context":{"idset":"9725-9727,9730-9731,10456,10460,10463,10465,10468,10825,10829,11037,11061,11259,11262,11333,11340,11349,11356,11461,11513,11538,11623,11625,11674,11702,11714,11719,11723,11729-11730,11740,11744,11748,11751"}}
+{"timestamp":1712611187.5129492,"name":"online","context":{"idset":"9718,9729,10440-10441,10452,10457,11509,11534,11635,11659"}}
+{"timestamp":1712611187.7041302,"name":"online","context":{"idset":"9719-9721,9728"}}
+{"timestamp":1712611543.362499,"name":"online","context":{"idset":"9315"}}
+{"timestamp":1712611561.4819038,"name":"online","context":{"idset":"9316"}}
+{"timestamp":1712611854.5621848,"name":"undrain","context":{"idset":"9316"}}
+{"timestamp":1712611962.0351307,"name":"online","context":{"idset":"10952"}}
+{"timestamp":1712612015.2636912,"name":"drain","context":{"idset":"9316","reason":"reason=testing","overwrite":0}}
+{"timestamp":1712612030.2348673,"name":"offline","context":{"idset":"9316"}}
+{"timestamp":1712613480.3596385,"name":"online","context":{"idset":"9316"}}
+{"timestamp":1712613515.9511073,"name":"undrain","context":{"idset":"9316"}}
+{"timestamp":1712614671.7589865,"name":"online","context":{"idset":"9534"}}
+{"timestamp":1712614699.2640204,"name":"online","context":{"idset":"9533"}}
+{"timestamp":1712614779.473305,"name":"online","context":{"idset":"9535"}}
+{"timestamp":1712615374.4278059,"name":"undrain","context":{"idset":"11317"}}
+{"timestamp":1712615380.6516314,"name":"undrain","context":{"idset":"11318"}}
+{"timestamp":1712615411.0023139,"name":"undrain","context":{"idset":"11319-11332"}}
+{"timestamp":1712615627.077076,"name":"online","context":{"idset":"9569"}}
+{"timestamp":1712615653.4169459,"name":"online","context":{"idset":"9570"}}
+{"timestamp":1712615699.1728806,"name":"undrain","context":{"idset":"9569-9570"}}
+{"timestamp":1712615968.8489521,"name":"drain","context":{"idset":"9472,9561-9562","reason":"reason=Bring up test TB","overwrite":0}}
+{"timestamp":1712616385.1573424,"name":"drain","context":{"idset":"10869-10996","reason":"drained to run diags outside of flux","overwrite":0}}
+{"timestamp":1712616575.6775534,"name":"online","context":{"idset":"797-798"}}
+{"timestamp":1712618232.538229,"name":"drain","context":{"idset":"11684-11685","reason":"drain 4 HPE-diag - KPN","overwrite":0}}
+{"timestamp":1712618449.0411735,"name":"drain","context":{"idset":"11669-11683","reason":"drain 4 HPE-diag - KPN","overwrite":0}}
+{"timestamp":1712618581.7755964,"name":"drain","context":{"idset":"11686-11700","reason":"drain 4 HPE-diag - KPN","overwrite":0}}
+{"timestamp":1712618654.2039006,"name":"drain","context":{"idset":"11733-11748","reason":"drain 4 HPE-diag - KPN","overwrite":0}}
+{"timestamp":1712620439.6120069,"name":"offline","context":{"idset":"10571"}}
+{"timestamp":1712620456.2950494,"name":"undrain","context":{"idset":"11686-11693,11695-11700,11733-11748"}}
+{"timestamp":1712620466.2674189,"name":"undrain","context":{"idset":"11669-11682,11684-11685"}}
+{"timestamp":1712620475.0900786,"name":"undrain","context":{"idset":"11683,11694"}}
+{"timestamp":1712623267.1771698,"name":"online","context":{"idset":"815-816"}}
+{"timestamp":1712628294.123502,"name":"undrain","context":{"idset":"11765-11776,11779-11780"}}
+{"timestamp":1712628869.2215502,"name":"undrain","context":{"idset":"11781-11796"}}
+{"timestamp":1712629182.2684243,"name":"online","context":{"idset":"11801,11808-11809"}}
+{"timestamp":1712629182.4314687,"name":"online","context":{"idset":"11800,11803,11807"}}
+{"timestamp":1712629182.7484875,"name":"online","context":{"idset":"11798,11802,11805,11811"}}
+{"timestamp":1712629182.9419587,"name":"online","context":{"idset":"11804,11806"}}
+{"timestamp":1712629183.1250696,"name":"online","context":{"idset":"11797,11799,11810,11812"}}
+{"timestamp":1712629377.0407686,"name":"undrain","context":{"idset":"11797-11812"}}
+{"timestamp":1712629829.9034624,"name":"undrain","context":{"idset":"11813-11828"}}
+{"timestamp":1712629987.1750803,"name":"online","context":{"idset":"11834"}}
+{"timestamp":1712629987.4093742,"name":"online","context":{"idset":"11832-11833"}}
+{"timestamp":1712629987.672467,"name":"online","context":{"idset":"11836"}}
+{"timestamp":1712629987.8616984,"name":"online","context":{"idset":"11831,11835"}}
+{"timestamp":1712629987.9813564,"name":"online","context":{"idset":"11840-11841"}}
+{"timestamp":1712629988.1704564,"name":"online","context":{"idset":"11829-11830,11837,11839,11844"}}
+{"timestamp":1712629988.355283,"name":"online","context":{"idset":"11838,11842-11843"}}
+{"timestamp":1712630143.6997788,"name":"undrain","context":{"idset":"11829-11844"}}
+{"timestamp":1712630480.4239113,"name":"online","context":{"idset":"11845"}}
+{"timestamp":1712630480.7135,"name":"online","context":{"idset":"11849-11851,11858"}}
+{"timestamp":1712630481.0155394,"name":"online","context":{"idset":"11846-11847,11854,11856-11857,11860"}}
+{"timestamp":1712630481.1984854,"name":"online","context":{"idset":"11848,11853"}}
+{"timestamp":1712630481.420001,"name":"online","context":{"idset":"11855"}}
+{"timestamp":1712630481.6047266,"name":"online","context":{"idset":"11852,11859"}}
+{"timestamp":1712630659.8127651,"name":"online","context":{"idset":"11868-11869"}}
+{"timestamp":1712630660.0344841,"name":"online","context":{"idset":"11861,11863,11871"}}
+{"timestamp":1712630660.3411267,"name":"online","context":{"idset":"11865,11867,11873-11875"}}
+{"timestamp":1712630660.5145538,"name":"online","context":{"idset":"11864,11870,11876"}}
+{"timestamp":1712630660.6316633,"name":"online","context":{"idset":"11862"}}
+{"timestamp":1712630660.9035335,"name":"online","context":{"idset":"11866"}}
+{"timestamp":1712630661.0870464,"name":"online","context":{"idset":"11872"}}
+{"timestamp":1712630824.1167796,"name":"online","context":{"idset":"11373"}}
+{"timestamp":1712630825.9680505,"name":"online","context":{"idset":"11881"}}
+{"timestamp":1712630826.3053949,"name":"online","context":{"idset":"11880,11889"}}
+{"timestamp":1712630826.4930978,"name":"online","context":{"idset":"11877,11879,11884,11891-11892"}}
+{"timestamp":1712630826.732255,"name":"online","context":{"idset":"11878,11882-11883,11887-11888"}}
+{"timestamp":1712630826.9267557,"name":"online","context":{"idset":"11885,11890"}}
+{"timestamp":1712630827.0287664,"name":"online","context":{"idset":"11886"}}
+{"timestamp":1712630844.5050437,"name":"online","context":{"idset":"11374"}}
+{"timestamp":1712634151.7549515,"name":"online","context":{"idset":"11358"}}
+{"timestamp":1712634696.371449,"name":"online","context":{"idset":"11683"}}
+{"timestamp":1712638018.6617815,"name":"drain","context":{"idset":"10614","reason":"broker was unresponsive"}}
+{"timestamp":1712638080.6597142,"name":"offline","context":{"idset":"10614"}}
+{"timestamp":1712647316.5623701,"name":"drain","context":{"idset":"253","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5624616,"name":"drain","context":{"idset":"254","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5624981,"name":"drain","context":{"idset":"255","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5625334,"name":"drain","context":{"idset":"256","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5625677,"name":"drain","context":{"idset":"257","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5626025,"name":"drain","context":{"idset":"258","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5626373,"name":"drain","context":{"idset":"259","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5626674,"name":"drain","context":{"idset":"260","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5626984,"name":"drain","context":{"idset":"261","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5627339,"name":"drain","context":{"idset":"262","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5627668,"name":"drain","context":{"idset":"263","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5628033,"name":"drain","context":{"idset":"264","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5628355,"name":"drain","context":{"idset":"265","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5628669,"name":"drain","context":{"idset":"266","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5629017,"name":"drain","context":{"idset":"267","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5629332,"name":"drain","context":{"idset":"268","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5629661,"name":"drain","context":{"idset":"269","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.563,"name":"drain","context":{"idset":"270","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5630362,"name":"drain","context":{"idset":"271","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5630696,"name":"drain","context":{"idset":"272","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5631044,"name":"drain","context":{"idset":"273","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5631373,"name":"drain","context":{"idset":"274","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.5631759,"name":"drain","context":{"idset":"275","reason":"broker was unresponsive"}}
+{"timestamp":1712647316.7936532,"name":"drain","context":{"idset":"276","reason":"broker was unresponsive"}}
+{"timestamp":1712647378.6611938,"name":"offline","context":{"idset":"253"}}
+{"timestamp":1712647378.6873314,"name":"offline","context":{"idset":"254"}}
+{"timestamp":1712647378.7060962,"name":"offline","context":{"idset":"255"}}
+{"timestamp":1712647378.7214401,"name":"offline","context":{"idset":"256"}}
+{"timestamp":1712647378.7611575,"name":"offline","context":{"idset":"257"}}
+{"timestamp":1712647378.773241,"name":"offline","context":{"idset":"258"}}
+{"timestamp":1712647378.7955356,"name":"offline","context":{"idset":"259"}}
+{"timestamp":1712647378.8155174,"name":"offline","context":{"idset":"260"}}
+{"timestamp":1712647378.8299994,"name":"offline","context":{"idset":"261"}}
+{"timestamp":1712647378.8427787,"name":"offline","context":{"idset":"262"}}
+{"timestamp":1712647378.856391,"name":"offline","context":{"idset":"263"}}
+{"timestamp":1712647378.8697824,"name":"offline","context":{"idset":"264"}}
+{"timestamp":1712647378.8828943,"name":"offline","context":{"idset":"265"}}
+{"timestamp":1712647378.8961747,"name":"offline","context":{"idset":"266"}}
+{"timestamp":1712647378.9098349,"name":"offline","context":{"idset":"267"}}
+{"timestamp":1712647378.9315739,"name":"offline","context":{"idset":"268"}}
+{"timestamp":1712647378.9431548,"name":"offline","context":{"idset":"269"}}
+{"timestamp":1712647378.9573615,"name":"offline","context":{"idset":"270"}}
+{"timestamp":1712647378.9844949,"name":"offline","context":{"idset":"271"}}
+{"timestamp":1712647378.9985995,"name":"offline","context":{"idset":"272"}}
+{"timestamp":1712647379.001446,"name":"offline","context":{"idset":"273"}}
+{"timestamp":1712647379.0117939,"name":"offline","context":{"idset":"274"}}
+{"timestamp":1712647379.032167,"name":"offline","context":{"idset":"275"}}
+{"timestamp":1712647379.0460491,"name":"offline","context":{"idset":"276"}}
+{"timestamp":1712657366.6613636,"name":"drain","context":{"idset":"11569","reason":"broker was unresponsive"}}
+{"timestamp":1712657432.6599851,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1712663698.6636891,"name":"drain","context":{"idset":"9535","reason":"broker was unresponsive"}}
+{"timestamp":1712663760.6631038,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1712668660.602679,"name":"drain","context":{"idset":"11397-11399","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712668697.1828396,"name":"drain","context":{"idset":"11429-11430","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712668712.6965783,"name":"drain","context":{"idset":"11432-11436","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712668726.5783393,"name":"drain","context":{"idset":"11437-11446","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712668746.2213664,"name":"drain","context":{"idset":"11457-11463","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712668755.7139494,"name":"drain","context":{"idset":"11465","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712668765.720181,"name":"drain","context":{"idset":"11467-11468","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712668776.0577521,"name":"drain","context":{"idset":"11469-11478","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712668796.4473162,"name":"drain","context":{"idset":"11479-11488","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712668809.1845181,"name":"drain","context":{"idset":"11489-11498","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712668827.0311522,"name":"drain","context":{"idset":"11499-11508","reason":"Eddie - HPL run","overwrite":0}}
+{"timestamp":1712670059.5767689,"name":"drain","context":{"idset":"11387-11388,11390,11395,11447-11456","overwrite":0}}
+{"timestamp":1712670082.1762195,"name":"drain","context":{"idset":"11431,11464","overwrite":0}}
+{"timestamp":1712672904.2689505,"name":"drain","context":{"idset":"877","reason":"reason=drain for NMC0 swap","overwrite":0}}
+{"timestamp":1712673073.4164507,"name":"drain","context":{"idset":"10405","reason":"broker was unresponsive"}}
+{"timestamp":1712673073.4165792,"name":"drain","context":{"idset":"10406","reason":"broker was unresponsive"}}
+{"timestamp":1712673131.2581394,"name":"drain","context":{"idset":"10407","reason":"broker was unresponsive"}}
+{"timestamp":1712673138.4931045,"name":"drain","context":{"idset":"10408","reason":"broker was unresponsive"}}
+{"timestamp":1712673138.503448,"name":"drain","context":{"idset":"10409","reason":"broker was unresponsive"}}
+{"timestamp":1712673186.9164166,"name":"drain","context":{"idset":"10410","reason":"broker was unresponsive"}}
+{"timestamp":1712673186.9236104,"name":"drain","context":{"idset":"10411","reason":"broker was unresponsive"}}
+{"timestamp":1712673186.9374835,"name":"drain","context":{"idset":"10412","reason":"broker was unresponsive"}}
+{"timestamp":1712673186.9462585,"name":"drain","context":{"idset":"10413","reason":"broker was unresponsive"}}
+{"timestamp":1712673186.9486573,"name":"drain","context":{"idset":"10414","reason":"broker was unresponsive"}}
+{"timestamp":1712673186.9779792,"name":"drain","context":{"idset":"10415","reason":"broker was unresponsive"}}
+{"timestamp":1712673186.9961631,"name":"drain","context":{"idset":"10416","reason":"broker was unresponsive"}}
+{"timestamp":1712673186.9976466,"name":"drain","context":{"idset":"10417","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.0223413,"name":"drain","context":{"idset":"10418","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.0237606,"name":"drain","context":{"idset":"10419","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.0425768,"name":"drain","context":{"idset":"10420","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.0603483,"name":"drain","context":{"idset":"10421","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.0616834,"name":"drain","context":{"idset":"10422","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.0814636,"name":"drain","context":{"idset":"10423","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1369836,"name":"drain","context":{"idset":"10424","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1372092,"name":"drain","context":{"idset":"10425","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1375444,"name":"drain","context":{"idset":"10426","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1380024,"name":"drain","context":{"idset":"10427","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1382096,"name":"drain","context":{"idset":"10428","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1385283,"name":"drain","context":{"idset":"10429","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1387804,"name":"drain","context":{"idset":"10430","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1392365,"name":"drain","context":{"idset":"10431","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1398091,"name":"drain","context":{"idset":"10432","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1456363,"name":"drain","context":{"idset":"10433","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1499197,"name":"drain","context":{"idset":"10434","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1503613,"name":"drain","context":{"idset":"10435","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.168874,"name":"drain","context":{"idset":"10436","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1718247,"name":"drain","context":{"idset":"10437","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1720786,"name":"drain","context":{"idset":"10438","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1751401,"name":"drain","context":{"idset":"10439","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1756122,"name":"drain","context":{"idset":"10440","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1785431,"name":"drain","context":{"idset":"10441","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1788204,"name":"drain","context":{"idset":"10442","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1849263,"name":"drain","context":{"idset":"10443","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1851711,"name":"drain","context":{"idset":"10444","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1879611,"name":"drain","context":{"idset":"10445","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1882398,"name":"drain","context":{"idset":"10446","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.1910388,"name":"drain","context":{"idset":"10447","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2017524,"name":"drain","context":{"idset":"10448","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2132037,"name":"drain","context":{"idset":"10449","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2250705,"name":"drain","context":{"idset":"10450","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2382221,"name":"drain","context":{"idset":"10451","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2519894,"name":"drain","context":{"idset":"10452","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2672317,"name":"drain","context":{"idset":"10453","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2673459,"name":"drain","context":{"idset":"10454","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2674415,"name":"drain","context":{"idset":"10455","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2675354,"name":"drain","context":{"idset":"10456","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.267628,"name":"drain","context":{"idset":"10457","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.26772,"name":"drain","context":{"idset":"10458","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2678146,"name":"drain","context":{"idset":"10459","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2679064,"name":"drain","context":{"idset":"10460","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2679985,"name":"drain","context":{"idset":"10461","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2680962,"name":"drain","context":{"idset":"10462","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2681878,"name":"drain","context":{"idset":"10463","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2682924,"name":"drain","context":{"idset":"10464","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2683885,"name":"drain","context":{"idset":"10465","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2684815,"name":"drain","context":{"idset":"10466","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2685766,"name":"drain","context":{"idset":"10467","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2686734,"name":"drain","context":{"idset":"10468","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2687683,"name":"drain","context":{"idset":"10485","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.268867,"name":"drain","context":{"idset":"10486","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2689645,"name":"drain","context":{"idset":"10487","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2690632,"name":"drain","context":{"idset":"10488","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2691612,"name":"drain","context":{"idset":"10489","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2692599,"name":"drain","context":{"idset":"10490","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2693818,"name":"drain","context":{"idset":"10491","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2694786,"name":"drain","context":{"idset":"10492","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2695777,"name":"drain","context":{"idset":"10493","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2696745,"name":"drain","context":{"idset":"10494","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2697632,"name":"drain","context":{"idset":"10495","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2698522,"name":"drain","context":{"idset":"10496","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2715356,"name":"drain","context":{"idset":"10497","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.274956,"name":"drain","context":{"idset":"10498","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.275624,"name":"drain","context":{"idset":"10499","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2757342,"name":"drain","context":{"idset":"10500","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2815697,"name":"drain","context":{"idset":"10502","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2843421,"name":"drain","context":{"idset":"10503","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2846754,"name":"drain","context":{"idset":"10504","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2896378,"name":"drain","context":{"idset":"10505","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2911656,"name":"drain","context":{"idset":"10506","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2927647,"name":"drain","context":{"idset":"10507","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.2928817,"name":"drain","context":{"idset":"10508","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.3067541,"name":"drain","context":{"idset":"10509","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.3179176,"name":"drain","context":{"idset":"10510","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.3250124,"name":"drain","context":{"idset":"10511","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.3389199,"name":"drain","context":{"idset":"10512","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.3526289,"name":"drain","context":{"idset":"10513","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.364783,"name":"drain","context":{"idset":"10514","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.3755984,"name":"drain","context":{"idset":"10515","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.3863204,"name":"drain","context":{"idset":"10516","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.3972993,"name":"drain","context":{"idset":"10519","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4076667,"name":"drain","context":{"idset":"10520","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4186797,"name":"drain","context":{"idset":"10521","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4293807,"name":"drain","context":{"idset":"10522","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4294798,"name":"drain","context":{"idset":"10523","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4295824,"name":"drain","context":{"idset":"10524","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4296775,"name":"drain","context":{"idset":"10525","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4297719,"name":"drain","context":{"idset":"10526","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4298687,"name":"drain","context":{"idset":"10527","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4299681,"name":"drain","context":{"idset":"10528","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4300666,"name":"drain","context":{"idset":"10529","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4301646,"name":"drain","context":{"idset":"10530","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4302816,"name":"drain","context":{"idset":"10531","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4303846,"name":"drain","context":{"idset":"10532","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4304876,"name":"drain","context":{"idset":"10533","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4305887,"name":"drain","context":{"idset":"10534","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4306901,"name":"drain","context":{"idset":"10535","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4307904,"name":"drain","context":{"idset":"10536","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4308929,"name":"drain","context":{"idset":"10537","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4309969,"name":"drain","context":{"idset":"10538","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4310999,"name":"drain","context":{"idset":"10539","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4312,"name":"drain","context":{"idset":"10540","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4313211,"name":"drain","context":{"idset":"10541","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4314299,"name":"drain","context":{"idset":"10542","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4315393,"name":"drain","context":{"idset":"10543","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4316471,"name":"drain","context":{"idset":"10544","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4317548,"name":"drain","context":{"idset":"10545","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4318647,"name":"drain","context":{"idset":"10546","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.431972,"name":"drain","context":{"idset":"10547","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.43208,"name":"drain","context":{"idset":"10548","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4321897,"name":"drain","context":{"idset":"10549","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4323132,"name":"drain","context":{"idset":"10550","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4324241,"name":"drain","context":{"idset":"10551","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4325352,"name":"drain","context":{"idset":"10552","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.432642,"name":"drain","context":{"idset":"10553","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4327493,"name":"drain","context":{"idset":"10554","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4328582,"name":"drain","context":{"idset":"10555","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4329686,"name":"drain","context":{"idset":"10556","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4330797,"name":"drain","context":{"idset":"10557","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4331927,"name":"drain","context":{"idset":"10558","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4333153,"name":"drain","context":{"idset":"10559","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4334295,"name":"drain","context":{"idset":"10560","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4335408,"name":"drain","context":{"idset":"10561","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4336536,"name":"drain","context":{"idset":"10562","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4337668,"name":"drain","context":{"idset":"10563","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.433876,"name":"drain","context":{"idset":"10564","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4339926,"name":"drain","context":{"idset":"10565","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4341075,"name":"drain","context":{"idset":"10566","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4342206,"name":"drain","context":{"idset":"10567","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4343472,"name":"drain","context":{"idset":"10568","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.434464,"name":"drain","context":{"idset":"10569","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4345796,"name":"drain","context":{"idset":"10570","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4346929,"name":"drain","context":{"idset":"10572","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4348087,"name":"drain","context":{"idset":"10573","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4349279,"name":"drain","context":{"idset":"10574","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4350476,"name":"drain","context":{"idset":"10575","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4351606,"name":"drain","context":{"idset":"10576","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4353039,"name":"drain","context":{"idset":"10577","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4354322,"name":"drain","context":{"idset":"10578","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4355533,"name":"drain","context":{"idset":"10579","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.435673,"name":"drain","context":{"idset":"10580","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4357946,"name":"drain","context":{"idset":"10581","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4359167,"name":"drain","context":{"idset":"10582","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4360387,"name":"drain","context":{"idset":"10583","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4361608,"name":"drain","context":{"idset":"10584","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4362977,"name":"drain","context":{"idset":"10585","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4364145,"name":"drain","context":{"idset":"10586","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4365296,"name":"drain","context":{"idset":"10615","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4366484,"name":"drain","context":{"idset":"10616","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4367647,"name":"drain","context":{"idset":"10617","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.436883,"name":"drain","context":{"idset":"10618","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4370356,"name":"drain","context":{"idset":"10619","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4371519,"name":"drain","context":{"idset":"10620","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4372802,"name":"drain","context":{"idset":"10621","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4374046,"name":"drain","context":{"idset":"10622","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4376137,"name":"drain","context":{"idset":"10623","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4378431,"name":"drain","context":{"idset":"10624","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4379728,"name":"drain","context":{"idset":"10625","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.438102,"name":"drain","context":{"idset":"10626","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4388344,"name":"drain","context":{"idset":"10627","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4390147,"name":"drain","context":{"idset":"9494","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4392333,"name":"drain","context":{"idset":"9495","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4394104,"name":"drain","context":{"idset":"9496","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4395733,"name":"drain","context":{"idset":"9497","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4397299,"name":"drain","context":{"idset":"9498","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4399118,"name":"drain","context":{"idset":"9499","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4400678,"name":"drain","context":{"idset":"9500","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4402244,"name":"drain","context":{"idset":"9501","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4403956,"name":"drain","context":{"idset":"9502","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4405527,"name":"drain","context":{"idset":"9503","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4407063,"name":"drain","context":{"idset":"9504","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4408631,"name":"drain","context":{"idset":"9505","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4410167,"name":"drain","context":{"idset":"9506","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4411678,"name":"drain","context":{"idset":"9507","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4413495,"name":"drain","context":{"idset":"9508","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4415076,"name":"drain","context":{"idset":"9509","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.441663,"name":"drain","context":{"idset":"9510","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4418242,"name":"drain","context":{"idset":"9511","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4419827,"name":"drain","context":{"idset":"9512","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.442142,"name":"drain","context":{"idset":"9513","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4423163,"name":"drain","context":{"idset":"9514","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4424765,"name":"drain","context":{"idset":"9515","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4426394,"name":"drain","context":{"idset":"9516","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4428005,"name":"drain","context":{"idset":"9517","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4429603,"name":"drain","context":{"idset":"9518","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4431241,"name":"drain","context":{"idset":"9519","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4513061,"name":"drain","context":{"idset":"9520","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4514799,"name":"drain","context":{"idset":"9521","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4516432,"name":"drain","context":{"idset":"9522","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4517958,"name":"drain","context":{"idset":"9523","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4519489,"name":"drain","context":{"idset":"9524","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4521062,"name":"drain","context":{"idset":"9525","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4522798,"name":"drain","context":{"idset":"9526","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.452436,"name":"drain","context":{"idset":"9527","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4525902,"name":"drain","context":{"idset":"9528","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4527471,"name":"drain","context":{"idset":"9529","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4529018,"name":"drain","context":{"idset":"9530","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4530578,"name":"drain","context":{"idset":"9531","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4532187,"name":"drain","context":{"idset":"9532","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4533875,"name":"drain","context":{"idset":"9533","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4535458,"name":"drain","context":{"idset":"9534","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4537015,"name":"drain","context":{"idset":"9536","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4538598,"name":"drain","context":{"idset":"9537","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4540164,"name":"drain","context":{"idset":"9538","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4541738,"name":"drain","context":{"idset":"9539","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4543502,"name":"drain","context":{"idset":"9540","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4545126,"name":"drain","context":{"idset":"9541","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4546702,"name":"drain","context":{"idset":"9542","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4548359,"name":"drain","context":{"idset":"9543","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4549956,"name":"drain","context":{"idset":"9544","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4551561,"name":"drain","context":{"idset":"9545","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4553378,"name":"drain","context":{"idset":"9546","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4555171,"name":"drain","context":{"idset":"9547","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4556749,"name":"drain","context":{"idset":"9548","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4558342,"name":"drain","context":{"idset":"9549","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4559968,"name":"drain","context":{"idset":"9550","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4561689,"name":"drain","context":{"idset":"9551","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4563541,"name":"drain","context":{"idset":"9552","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4565165,"name":"drain","context":{"idset":"9553","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4566832,"name":"drain","context":{"idset":"9554","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4568493,"name":"drain","context":{"idset":"9555","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4570172,"name":"drain","context":{"idset":"9556","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.45719,"name":"drain","context":{"idset":"9557","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4573772,"name":"drain","context":{"idset":"9558","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4575596,"name":"drain","context":{"idset":"9559","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4577467,"name":"drain","context":{"idset":"9560","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4579329,"name":"drain","context":{"idset":"9563","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4581177,"name":"drain","context":{"idset":"9564","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4637358,"name":"drain","context":{"idset":"9565","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4639318,"name":"drain","context":{"idset":"9566","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4641159,"name":"drain","context":{"idset":"9567","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4643121,"name":"drain","context":{"idset":"9568","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4644938,"name":"drain","context":{"idset":"9569","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4646726,"name":"drain","context":{"idset":"9570","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4648519,"name":"drain","context":{"idset":"9571","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4650276,"name":"drain","context":{"idset":"9572","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4652066,"name":"drain","context":{"idset":"9573","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4654047,"name":"drain","context":{"idset":"9574","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4655902,"name":"drain","context":{"idset":"9575","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4657712,"name":"drain","context":{"idset":"9576","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4659498,"name":"drain","context":{"idset":"9577","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4661336,"name":"drain","context":{"idset":"9578","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4663324,"name":"drain","context":{"idset":"9579","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4665194,"name":"drain","context":{"idset":"9580","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4667091,"name":"drain","context":{"idset":"9581","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4668994,"name":"drain","context":{"idset":"9582","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.467087,"name":"drain","context":{"idset":"9583","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4672892,"name":"drain","context":{"idset":"9584","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4674814,"name":"drain","context":{"idset":"9585","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4676683,"name":"drain","context":{"idset":"9586","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4678559,"name":"drain","context":{"idset":"9587","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4680407,"name":"drain","context":{"idset":"9588","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4682331,"name":"drain","context":{"idset":"9589","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4684319,"name":"drain","context":{"idset":"9590","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4686251,"name":"drain","context":{"idset":"9591","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4688158,"name":"drain","context":{"idset":"9592","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4690146,"name":"drain","context":{"idset":"9593","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4692085,"name":"drain","context":{"idset":"9594","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4694293,"name":"drain","context":{"idset":"9595","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4696252,"name":"drain","context":{"idset":"9596","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4698255,"name":"drain","context":{"idset":"9597","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4700222,"name":"drain","context":{"idset":"9598","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4702168,"name":"drain","context":{"idset":"9599","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.470427,"name":"drain","context":{"idset":"9600","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4706261,"name":"drain","context":{"idset":"9601","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.470818,"name":"drain","context":{"idset":"9602","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4710114,"name":"drain","context":{"idset":"9603","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4712148,"name":"drain","context":{"idset":"9604","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4794261,"name":"drain","context":{"idset":"9605","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4796293,"name":"drain","context":{"idset":"9606","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4798193,"name":"drain","context":{"idset":"9607","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4800088,"name":"drain","context":{"idset":"9608","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4802051,"name":"drain","context":{"idset":"9609","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4804125,"name":"drain","context":{"idset":"9610","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.480607,"name":"drain","context":{"idset":"9611","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.480798,"name":"drain","context":{"idset":"9612","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4809968,"name":"drain","context":{"idset":"9613","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4811919,"name":"drain","context":{"idset":"9614","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4814217,"name":"drain","context":{"idset":"9615","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4816461,"name":"drain","context":{"idset":"9616","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4818437,"name":"drain","context":{"idset":"9617","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4820642,"name":"drain","context":{"idset":"9618","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4823062,"name":"drain","context":{"idset":"9619","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4825335,"name":"drain","context":{"idset":"9620","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4827559,"name":"drain","context":{"idset":"9621","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.482971,"name":"drain","context":{"idset":"9622","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4831998,"name":"drain","context":{"idset":"9623","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4834406,"name":"drain","context":{"idset":"9624","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4836669,"name":"drain","context":{"idset":"9625","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4838884,"name":"drain","context":{"idset":"9626","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4841113,"name":"drain","context":{"idset":"9627","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4859586,"name":"drain","context":{"idset":"9628","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4861927,"name":"drain","context":{"idset":"9629","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4864292,"name":"drain","context":{"idset":"9630","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4866524,"name":"drain","context":{"idset":"9631","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4868879,"name":"drain","context":{"idset":"9632","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4871147,"name":"drain","context":{"idset":"9633","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4873579,"name":"drain","context":{"idset":"9634","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4875879,"name":"drain","context":{"idset":"9635","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4878147,"name":"drain","context":{"idset":"9636","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4880509,"name":"drain","context":{"idset":"9637","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4882908,"name":"drain","context":{"idset":"9638","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4885199,"name":"drain","context":{"idset":"9639","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4887524,"name":"drain","context":{"idset":"9640","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4889815,"name":"drain","context":{"idset":"9641","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.4892147,"name":"drain","context":{"idset":"9642","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.5043473,"name":"drain","context":{"idset":"9643","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.514941,"name":"drain","context":{"idset":"9644","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.5357158,"name":"drain","context":{"idset":"9645","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.5609932,"name":"drain","context":{"idset":"9646","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.603791,"name":"drain","context":{"idset":"9647","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.6545982,"name":"drain","context":{"idset":"9648","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.7009032,"name":"drain","context":{"idset":"9649","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.7456408,"name":"drain","context":{"idset":"9650","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.7736619,"name":"drain","context":{"idset":"9651","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.7934024,"name":"drain","context":{"idset":"9652","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.8165154,"name":"drain","context":{"idset":"9653","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.8376658,"name":"drain","context":{"idset":"9654","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.8502388,"name":"drain","context":{"idset":"9655","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.8608699,"name":"drain","context":{"idset":"9656","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.8713658,"name":"drain","context":{"idset":"9657","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.8823721,"name":"drain","context":{"idset":"9658","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.8979886,"name":"drain","context":{"idset":"9659","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.9087377,"name":"drain","context":{"idset":"9660","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.919189,"name":"drain","context":{"idset":"9661","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.9356155,"name":"drain","context":{"idset":"9662","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.9521682,"name":"drain","context":{"idset":"9663","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.9801257,"name":"drain","context":{"idset":"9664","reason":"broker was unresponsive"}}
+{"timestamp":1712673187.9996488,"name":"drain","context":{"idset":"9665","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0132742,"name":"drain","context":{"idset":"9666","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0410855,"name":"drain","context":{"idset":"9667","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0413601,"name":"drain","context":{"idset":"9668","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0416174,"name":"drain","context":{"idset":"9669","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0418663,"name":"drain","context":{"idset":"9670","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0421138,"name":"drain","context":{"idset":"9671","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0423841,"name":"drain","context":{"idset":"9672","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0426354,"name":"drain","context":{"idset":"9673","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0428808,"name":"drain","context":{"idset":"9674","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0431387,"name":"drain","context":{"idset":"9675","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0434215,"name":"drain","context":{"idset":"9676","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0436869,"name":"drain","context":{"idset":"9677","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0439453,"name":"drain","context":{"idset":"9678","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0442085,"name":"drain","context":{"idset":"9679","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0444851,"name":"drain","context":{"idset":"9680","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0447404,"name":"drain","context":{"idset":"9681","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0449867,"name":"drain","context":{"idset":"9682","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0452316,"name":"drain","context":{"idset":"9683","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0514135,"name":"drain","context":{"idset":"9684","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0516865,"name":"drain","context":{"idset":"9685","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0519366,"name":"drain","context":{"idset":"9686","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0521834,"name":"drain","context":{"idset":"9687","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0524452,"name":"drain","context":{"idset":"9688","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0526896,"name":"drain","context":{"idset":"9689","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0529344,"name":"drain","context":{"idset":"9690","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.053179,"name":"drain","context":{"idset":"9691","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0534415,"name":"drain","context":{"idset":"9692","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.053688,"name":"drain","context":{"idset":"9693","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0539358,"name":"drain","context":{"idset":"9694","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0541837,"name":"drain","context":{"idset":"9695","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0544386,"name":"drain","context":{"idset":"9696","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0546782,"name":"drain","context":{"idset":"9697","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0549212,"name":"drain","context":{"idset":"9698","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0551651,"name":"drain","context":{"idset":"9699","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0659735,"name":"drain","context":{"idset":"9700","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0662277,"name":"drain","context":{"idset":"9701","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0664842,"name":"drain","context":{"idset":"9702","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0667183,"name":"drain","context":{"idset":"9703","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0670402,"name":"drain","context":{"idset":"9704","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0673089,"name":"drain","context":{"idset":"9705","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0675514,"name":"drain","context":{"idset":"9706","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0677919,"name":"drain","context":{"idset":"9707","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0680318,"name":"drain","context":{"idset":"9708","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0682914,"name":"drain","context":{"idset":"9709","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0685365,"name":"drain","context":{"idset":"9710","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0687807,"name":"drain","context":{"idset":"9711","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0690267,"name":"drain","context":{"idset":"9712","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0692868,"name":"drain","context":{"idset":"9713","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0695338,"name":"drain","context":{"idset":"9714","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0697792,"name":"drain","context":{"idset":"9715","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0700262,"name":"drain","context":{"idset":"9716","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0702882,"name":"drain","context":{"idset":"9717","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0705335,"name":"drain","context":{"idset":"9718","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0707793,"name":"drain","context":{"idset":"9719","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.0710337,"name":"drain","context":{"idset":"9720","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.1111856,"name":"drain","context":{"idset":"9721","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.136977,"name":"drain","context":{"idset":"9722","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.1668456,"name":"drain","context":{"idset":"9723","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.1886599,"name":"drain","context":{"idset":"9724","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.2037947,"name":"drain","context":{"idset":"9725","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.2147844,"name":"drain","context":{"idset":"9726","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.2314355,"name":"drain","context":{"idset":"9727","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.2317233,"name":"drain","context":{"idset":"9728","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.2517877,"name":"drain","context":{"idset":"9729","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.2520754,"name":"drain","context":{"idset":"9730","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.2523718,"name":"drain","context":{"idset":"9731","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.2526567,"name":"drain","context":{"idset":"9732","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.278327,"name":"drain","context":{"idset":"9733","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.2786229,"name":"drain","context":{"idset":"9734","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.2985928,"name":"drain","context":{"idset":"9735","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.3271143,"name":"drain","context":{"idset":"9736","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.3519537,"name":"drain","context":{"idset":"9737","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.3701563,"name":"drain","context":{"idset":"9738","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.3809755,"name":"drain","context":{"idset":"9739","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.395659,"name":"drain","context":{"idset":"9740","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.4063122,"name":"drain","context":{"idset":"9741","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.4169524,"name":"drain","context":{"idset":"9742","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.4326389,"name":"drain","context":{"idset":"9743","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.448477,"name":"drain","context":{"idset":"9744","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.4594104,"name":"drain","context":{"idset":"9745","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.470377,"name":"drain","context":{"idset":"9746","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.4810047,"name":"drain","context":{"idset":"9747","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.4916291,"name":"drain","context":{"idset":"9748","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.5022981,"name":"drain","context":{"idset":"9749","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.5529654,"name":"drain","context":{"idset":"9751","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.5701625,"name":"drain","context":{"idset":"9752","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.5919967,"name":"drain","context":{"idset":"9753","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.6078873,"name":"drain","context":{"idset":"9754","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.6233108,"name":"drain","context":{"idset":"9755","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.639493,"name":"drain","context":{"idset":"9756","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.6582382,"name":"drain","context":{"idset":"9757","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.6791062,"name":"drain","context":{"idset":"9758","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.691988,"name":"drain","context":{"idset":"9759","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7046432,"name":"drain","context":{"idset":"9760","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7154582,"name":"drain","context":{"idset":"9761","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7267795,"name":"drain","context":{"idset":"9762","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.737236,"name":"drain","context":{"idset":"9763","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7487214,"name":"drain","context":{"idset":"9764","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.759191,"name":"drain","context":{"idset":"9765","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7699566,"name":"drain","context":{"idset":"9766","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7702341,"name":"drain","context":{"idset":"9767","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7705219,"name":"drain","context":{"idset":"9768","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7707968,"name":"drain","context":{"idset":"9769","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7710717,"name":"drain","context":{"idset":"9770","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7713614,"name":"drain","context":{"idset":"9771","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.7826231,"name":"drain","context":{"idset":"9772","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.795202,"name":"drain","context":{"idset":"9773","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.805923,"name":"drain","context":{"idset":"9774","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8170295,"name":"drain","context":{"idset":"9775","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8281469,"name":"drain","context":{"idset":"9776","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8284898,"name":"drain","context":{"idset":"9777","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8288176,"name":"drain","context":{"idset":"9778","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8290925,"name":"drain","context":{"idset":"9779","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8294263,"name":"drain","context":{"idset":"9780","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8303545,"name":"drain","context":{"idset":"9781","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8306365,"name":"drain","context":{"idset":"9782","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8426301,"name":"drain","context":{"idset":"9783","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.842963,"name":"drain","context":{"idset":"9784","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8432472,"name":"drain","context":{"idset":"9785","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8435984,"name":"drain","context":{"idset":"9786","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.84394,"name":"drain","context":{"idset":"9788","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8442361,"name":"drain","context":{"idset":"9789","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8445961,"name":"drain","context":{"idset":"9791","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8449297,"name":"drain","context":{"idset":"9792","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8452258,"name":"drain","context":{"idset":"9793","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.845597,"name":"drain","context":{"idset":"9794","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8459518,"name":"drain","context":{"idset":"9795","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.846257,"name":"drain","context":{"idset":"9796","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8466332,"name":"drain","context":{"idset":"9797","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8470135,"name":"drain","context":{"idset":"9798","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8578031,"name":"drain","context":{"idset":"9799","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8580904,"name":"drain","context":{"idset":"9800","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8584549,"name":"drain","context":{"idset":"9801","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.858741,"name":"drain","context":{"idset":"9802","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8590274,"name":"drain","context":{"idset":"9803","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8593276,"name":"drain","context":{"idset":"9804","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8596163,"name":"drain","context":{"idset":"9805","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8599057,"name":"drain","context":{"idset":"9806","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8601878,"name":"drain","context":{"idset":"9807","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.860491,"name":"drain","context":{"idset":"9808","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8607798,"name":"drain","context":{"idset":"9809","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.871805,"name":"drain","context":{"idset":"9810","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8721054,"name":"drain","context":{"idset":"9811","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8724151,"name":"drain","context":{"idset":"9812","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8727069,"name":"drain","context":{"idset":"9813","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8729966,"name":"drain","context":{"idset":"9814","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8733025,"name":"drain","context":{"idset":"9815","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8735974,"name":"drain","context":{"idset":"9816","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8738933,"name":"drain","context":{"idset":"9817","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8741891,"name":"drain","context":{"idset":"9818","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8745058,"name":"drain","context":{"idset":"9819","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8748021,"name":"drain","context":{"idset":"9820","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8751004,"name":"drain","context":{"idset":"9821","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8754101,"name":"drain","context":{"idset":"9822","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8757064,"name":"drain","context":{"idset":"9823","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8760047,"name":"drain","context":{"idset":"9824","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.876313,"name":"drain","context":{"idset":"9825","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8766112,"name":"drain","context":{"idset":"9826","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8769097,"name":"drain","context":{"idset":"9827","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8772099,"name":"drain","context":{"idset":"9828","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.877522,"name":"drain","context":{"idset":"9829","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8778203,"name":"drain","context":{"idset":"9830","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8885117,"name":"drain","context":{"idset":"9831","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8888135,"name":"drain","context":{"idset":"9832","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8891168,"name":"drain","context":{"idset":"9833","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8894408,"name":"drain","context":{"idset":"9834","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8897464,"name":"drain","context":{"idset":"9835","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8900402,"name":"drain","context":{"idset":"9836","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.890347,"name":"drain","context":{"idset":"9837","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8906398,"name":"drain","context":{"idset":"9838","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8909335,"name":"drain","context":{"idset":"9839","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8912292,"name":"drain","context":{"idset":"9840","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8915358,"name":"drain","context":{"idset":"9841","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8918319,"name":"drain","context":{"idset":"9842","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.8921278,"name":"drain","context":{"idset":"9843","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9138148,"name":"drain","context":{"idset":"9844","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9141293,"name":"drain","context":{"idset":"9973","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9144466,"name":"drain","context":{"idset":"9974","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9147468,"name":"drain","context":{"idset":"9975","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.915051,"name":"drain","context":{"idset":"9976","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9153655,"name":"drain","context":{"idset":"9977","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9156697,"name":"drain","context":{"idset":"9978","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9159732,"name":"drain","context":{"idset":"9979","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9162903,"name":"drain","context":{"idset":"9980","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9165902,"name":"drain","context":{"idset":"9981","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9168932,"name":"drain","context":{"idset":"9982","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9171979,"name":"drain","context":{"idset":"9983","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9265685,"name":"drain","context":{"idset":"9984","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9268825,"name":"drain","context":{"idset":"9985","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9271934,"name":"drain","context":{"idset":"9986","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9275172,"name":"drain","context":{"idset":"9987","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9278247,"name":"drain","context":{"idset":"9988","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9503684,"name":"drain","context":{"idset":"9989","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9843214,"name":"drain","context":{"idset":"9990","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9848783,"name":"drain","context":{"idset":"9991","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9855342,"name":"drain","context":{"idset":"9992","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9860342,"name":"drain","context":{"idset":"9993","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9863787,"name":"drain","context":{"idset":"9994","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9866936,"name":"drain","context":{"idset":"9995","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9870057,"name":"drain","context":{"idset":"9996","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9873369,"name":"drain","context":{"idset":"9997","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9876518,"name":"drain","context":{"idset":"9998","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9879658,"name":"drain","context":{"idset":"9999","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9882927,"name":"drain","context":{"idset":"10000","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9886081,"name":"drain","context":{"idset":"10001","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9889221,"name":"drain","context":{"idset":"10002","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9892378,"name":"drain","context":{"idset":"10003","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.991441,"name":"drain","context":{"idset":"10004","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9931879,"name":"drain","context":{"idset":"10005","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9935293,"name":"drain","context":{"idset":"10006","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.993855,"name":"drain","context":{"idset":"10007","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.994179,"name":"drain","context":{"idset":"10008","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9945118,"name":"drain","context":{"idset":"10009","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9948289,"name":"drain","context":{"idset":"10010","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9951458,"name":"drain","context":{"idset":"10011","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9954791,"name":"drain","context":{"idset":"10012","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9957962,"name":"drain","context":{"idset":"10013","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9961128,"name":"drain","context":{"idset":"10014","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9964423,"name":"drain","context":{"idset":"10015","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9967611,"name":"drain","context":{"idset":"10016","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9970767,"name":"drain","context":{"idset":"10017","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9974093,"name":"drain","context":{"idset":"10018","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9977264,"name":"drain","context":{"idset":"10019","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9980462,"name":"drain","context":{"idset":"10020","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9983871,"name":"drain","context":{"idset":"10021","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9987094,"name":"drain","context":{"idset":"10022","reason":"broker was unresponsive"}}
+{"timestamp":1712673188.9990306,"name":"drain","context":{"idset":"10023","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0101292,"name":"drain","context":{"idset":"10024","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.027432,"name":"drain","context":{"idset":"10025","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0397232,"name":"drain","context":{"idset":"10026","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0548575,"name":"drain","context":{"idset":"10027","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0696535,"name":"drain","context":{"idset":"10028","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0859301,"name":"drain","context":{"idset":"10029","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0862613,"name":"drain","context":{"idset":"10030","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0866091,"name":"drain","context":{"idset":"10031","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0869319,"name":"drain","context":{"idset":"10032","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0872548,"name":"drain","context":{"idset":"10033","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0882869,"name":"drain","context":{"idset":"10034","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0886159,"name":"drain","context":{"idset":"10035","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0889511,"name":"drain","context":{"idset":"10036","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0892987,"name":"drain","context":{"idset":"10037","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.089628,"name":"drain","context":{"idset":"10038","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0899508,"name":"drain","context":{"idset":"10039","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0903158,"name":"drain","context":{"idset":"10040","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0906448,"name":"drain","context":{"idset":"10041","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0909724,"name":"drain","context":{"idset":"10042","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0922842,"name":"drain","context":{"idset":"10043","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0926387,"name":"drain","context":{"idset":"10044","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0929804,"name":"drain","context":{"idset":"10045","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0933321,"name":"drain","context":{"idset":"10046","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0936613,"name":"drain","context":{"idset":"10047","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0939839,"name":"drain","context":{"idset":"10048","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0942984,"name":"drain","context":{"idset":"10049","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0945995,"name":"drain","context":{"idset":"10050","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0949116,"name":"drain","context":{"idset":"10051","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0952299,"name":"drain","context":{"idset":"10052","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.095736,"name":"drain","context":{"idset":"10053","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0960867,"name":"drain","context":{"idset":"10054","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0964522,"name":"drain","context":{"idset":"10055","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.096802,"name":"drain","context":{"idset":"10056","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0971637,"name":"drain","context":{"idset":"10057","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.097513,"name":"drain","context":{"idset":"10058","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0978274,"name":"drain","context":{"idset":"10059","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0981491,"name":"drain","context":{"idset":"10060","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0984769,"name":"drain","context":{"idset":"10061","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0987883,"name":"drain","context":{"idset":"10062","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.0991018,"name":"drain","context":{"idset":"10063","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1000528,"name":"drain","context":{"idset":"10064","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1004343,"name":"drain","context":{"idset":"10065","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1007965,"name":"drain","context":{"idset":"10066","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1013155,"name":"drain","context":{"idset":"10067","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.101671,"name":"drain","context":{"idset":"10068","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1020265,"name":"drain","context":{"idset":"10069","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1023965,"name":"drain","context":{"idset":"10070","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.102756,"name":"drain","context":{"idset":"10071","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1031139,"name":"drain","context":{"idset":"10072","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1034684,"name":"drain","context":{"idset":"10073","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1037891,"name":"drain","context":{"idset":"10074","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1146557,"name":"drain","context":{"idset":"10075","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1150274,"name":"drain","context":{"idset":"10076","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.115411,"name":"drain","context":{"idset":"10077","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1157746,"name":"drain","context":{"idset":"10078","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1161335,"name":"drain","context":{"idset":"10079","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1165135,"name":"drain","context":{"idset":"10080","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1168749,"name":"drain","context":{"idset":"10081","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1172359,"name":"drain","context":{"idset":"10082","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1176205,"name":"drain","context":{"idset":"10083","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1179833,"name":"drain","context":{"idset":"10084","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1185341,"name":"drain","context":{"idset":"10085","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1189029,"name":"drain","context":{"idset":"10086","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1192839,"name":"drain","context":{"idset":"10087","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1196558,"name":"drain","context":{"idset":"10088","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1200185,"name":"drain","context":{"idset":"10089","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1203935,"name":"drain","context":{"idset":"10090","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1207654,"name":"drain","context":{"idset":"10091","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1212363,"name":"drain","context":{"idset":"10092","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1216037,"name":"drain","context":{"idset":"10093","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1219537,"name":"drain","context":{"idset":"10094","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1223073,"name":"drain","context":{"idset":"10095","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1226306,"name":"drain","context":{"idset":"10096","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1229599,"name":"drain","context":{"idset":"10097","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1232994,"name":"drain","context":{"idset":"10098","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1236379,"name":"drain","context":{"idset":"10099","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1240172,"name":"drain","context":{"idset":"10100","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.124558,"name":"drain","context":{"idset":"10101","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1249418,"name":"drain","context":{"idset":"10102","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1253324,"name":"drain","context":{"idset":"10103","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1378074,"name":"drain","context":{"idset":"10104","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1506774,"name":"drain","context":{"idset":"10105","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1618457,"name":"drain","context":{"idset":"10106","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1733844,"name":"drain","context":{"idset":"10107","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.1856842,"name":"drain","context":{"idset":"10108","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.2003362,"name":"drain","context":{"idset":"10109","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.2125413,"name":"drain","context":{"idset":"10110","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.2241094,"name":"drain","context":{"idset":"10111","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.235893,"name":"drain","context":{"idset":"10112","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.2483044,"name":"drain","context":{"idset":"10113","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.2818401,"name":"drain","context":{"idset":"10114","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.2989559,"name":"drain","context":{"idset":"10115","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.3107448,"name":"drain","context":{"idset":"10116","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.3217165,"name":"drain","context":{"idset":"10117","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.3334379,"name":"drain","context":{"idset":"10118","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.3448274,"name":"drain","context":{"idset":"10119","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.3566682,"name":"drain","context":{"idset":"10120","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.3679132,"name":"drain","context":{"idset":"10121","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.3789015,"name":"drain","context":{"idset":"10122","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.3906319,"name":"drain","context":{"idset":"10123","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4046099,"name":"drain","context":{"idset":"10124","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4050376,"name":"drain","context":{"idset":"10125","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4055357,"name":"drain","context":{"idset":"10126","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4059241,"name":"drain","context":{"idset":"10127","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4063549,"name":"drain","context":{"idset":"10128","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.40678,"name":"drain","context":{"idset":"10129","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4073784,"name":"drain","context":{"idset":"10130","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4077981,"name":"drain","context":{"idset":"10131","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4082189,"name":"drain","context":{"idset":"10132","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4086411,"name":"drain","context":{"idset":"10133","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.409059,"name":"drain","context":{"idset":"10134","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4096708,"name":"drain","context":{"idset":"10135","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4100959,"name":"drain","context":{"idset":"10136","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4105253,"name":"drain","context":{"idset":"10137","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4109435,"name":"drain","context":{"idset":"10139","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4113541,"name":"drain","context":{"idset":"10140","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4119904,"name":"drain","context":{"idset":"10141","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4124272,"name":"drain","context":{"idset":"10142","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4128489,"name":"drain","context":{"idset":"10144","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4132843,"name":"drain","context":{"idset":"10146","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.413707,"name":"drain","context":{"idset":"10147","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4141247,"name":"drain","context":{"idset":"10148","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4147205,"name":"drain","context":{"idset":"10149","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4151096,"name":"drain","context":{"idset":"10150","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4155042,"name":"drain","context":{"idset":"10151","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4158852,"name":"drain","context":{"idset":"10152","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4162767,"name":"drain","context":{"idset":"10153","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4166577,"name":"drain","context":{"idset":"10154","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4172921,"name":"drain","context":{"idset":"10157","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4176776,"name":"drain","context":{"idset":"10158","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4180615,"name":"drain","context":{"idset":"10159","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4184649,"name":"drain","context":{"idset":"10160","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4188533,"name":"drain","context":{"idset":"10161","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4192276,"name":"drain","context":{"idset":"10162","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4195383,"name":"drain","context":{"idset":"10163","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4198425,"name":"drain","context":{"idset":"10164","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.420162,"name":"drain","context":{"idset":"10165","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4206343,"name":"drain","context":{"idset":"10166","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4209521,"name":"drain","context":{"idset":"10167","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4212885,"name":"drain","context":{"idset":"10168","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4215991,"name":"drain","context":{"idset":"10169","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.421901,"name":"drain","context":{"idset":"10170","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4222229,"name":"drain","context":{"idset":"10171","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4225624,"name":"drain","context":{"idset":"10172","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4228644,"name":"drain","context":{"idset":"10173","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4231565,"name":"drain","context":{"idset":"10174","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4234204,"name":"drain","context":{"idset":"10175","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4239566,"name":"drain","context":{"idset":"10176","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4242806,"name":"drain","context":{"idset":"10177","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4246588,"name":"drain","context":{"idset":"10178","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4250522,"name":"drain","context":{"idset":"10179","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4254348,"name":"drain","context":{"idset":"10180","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4257224,"name":"drain","context":{"idset":"10181","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4260366,"name":"drain","context":{"idset":"10182","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4263725,"name":"drain","context":{"idset":"10183","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4267263,"name":"drain","context":{"idset":"10184","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4274333,"name":"drain","context":{"idset":"10185","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4278347,"name":"drain","context":{"idset":"10186","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4282277,"name":"drain","context":{"idset":"10187","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.428638,"name":"drain","context":{"idset":"10188","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4290309,"name":"drain","context":{"idset":"10189","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4294403,"name":"drain","context":{"idset":"10190","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4298346,"name":"drain","context":{"idset":"10191","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4302278,"name":"drain","context":{"idset":"10192","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4306359,"name":"drain","context":{"idset":"10193","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4310322,"name":"drain","context":{"idset":"10194","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4317591,"name":"drain","context":{"idset":"10195","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4321306,"name":"drain","context":{"idset":"10196","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4325054,"name":"drain","context":{"idset":"10197","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4328451,"name":"drain","context":{"idset":"10198","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4332228,"name":"drain","context":{"idset":"10199","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4336522,"name":"drain","context":{"idset":"10200","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4340622,"name":"drain","context":{"idset":"10201","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4344878,"name":"drain","context":{"idset":"10202","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4348903,"name":"drain","context":{"idset":"10203","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.435298,"name":"drain","context":{"idset":"10204","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4358597,"name":"drain","context":{"idset":"10205","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4362764,"name":"drain","context":{"idset":"10206","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.436677,"name":"drain","context":{"idset":"10207","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4370775,"name":"drain","context":{"idset":"10208","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4374962,"name":"drain","context":{"idset":"10209","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.437902,"name":"drain","context":{"idset":"10210","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4383278,"name":"drain","context":{"idset":"10211","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4387338,"name":"drain","context":{"idset":"10212","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4391413,"name":"drain","context":{"idset":"10213","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4395611,"name":"drain","context":{"idset":"10214","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4399674,"name":"drain","context":{"idset":"10215","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4403899,"name":"drain","context":{"idset":"10216","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4408031,"name":"drain","context":{"idset":"10217","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4412189,"name":"drain","context":{"idset":"10218","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4416449,"name":"drain","context":{"idset":"10219","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4420586,"name":"drain","context":{"idset":"10220","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4424882,"name":"drain","context":{"idset":"10221","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4429064,"name":"drain","context":{"idset":"10222","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4433351,"name":"drain","context":{"idset":"10223","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4437449,"name":"drain","context":{"idset":"10224","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4441514,"name":"drain","context":{"idset":"10225","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4445686,"name":"drain","context":{"idset":"10226","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4449785,"name":"drain","context":{"idset":"10227","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4453995,"name":"drain","context":{"idset":"10228","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4458072,"name":"drain","context":{"idset":"10229","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4462125,"name":"drain","context":{"idset":"10230","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4466372,"name":"drain","context":{"idset":"10231","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4470456,"name":"drain","context":{"idset":"10232","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4474657,"name":"drain","context":{"idset":"10233","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.447871,"name":"drain","context":{"idset":"10234","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4482923,"name":"drain","context":{"idset":"10235","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4487035,"name":"drain","context":{"idset":"10236","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4498041,"name":"drain","context":{"idset":"10237","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.450294,"name":"drain","context":{"idset":"10238","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4507165,"name":"drain","context":{"idset":"10239","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4511333,"name":"drain","context":{"idset":"10240","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4515719,"name":"drain","context":{"idset":"10241","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4519928,"name":"drain","context":{"idset":"10242","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.452426,"name":"drain","context":{"idset":"10243","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.452848,"name":"drain","context":{"idset":"10244","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4532824,"name":"drain","context":{"idset":"10245","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4537032,"name":"drain","context":{"idset":"10246","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.454123,"name":"drain","context":{"idset":"10247","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4545574,"name":"drain","context":{"idset":"10248","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4549761,"name":"drain","context":{"idset":"10249","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4554117,"name":"drain","context":{"idset":"10250","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4558334,"name":"drain","context":{"idset":"10251","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4562519,"name":"drain","context":{"idset":"10252","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4566882,"name":"drain","context":{"idset":"10253","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4571064,"name":"drain","context":{"idset":"10254","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4575338,"name":"drain","context":{"idset":"10255","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.457953,"name":"drain","context":{"idset":"10256","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4583824,"name":"drain","context":{"idset":"10257","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4587967,"name":"drain","context":{"idset":"10258","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4592133,"name":"drain","context":{"idset":"10259","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4596407,"name":"drain","context":{"idset":"10260","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4600587,"name":"drain","context":{"idset":"10263","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4604981,"name":"drain","context":{"idset":"10264","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4609194,"name":"drain","context":{"idset":"10265","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4613273,"name":"drain","context":{"idset":"10266","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4617527,"name":"drain","context":{"idset":"10267","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4621773,"name":"drain","context":{"idset":"10268","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4626138,"name":"drain","context":{"idset":"10269","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4630418,"name":"drain","context":{"idset":"10270","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4634817,"name":"drain","context":{"idset":"10271","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4639101,"name":"drain","context":{"idset":"10272","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4643502,"name":"drain","context":{"idset":"10273","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4647765,"name":"drain","context":{"idset":"10274","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4652016,"name":"drain","context":{"idset":"10275","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4656479,"name":"drain","context":{"idset":"10276","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4660749,"name":"drain","context":{"idset":"10277","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4665213,"name":"drain","context":{"idset":"10278","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4669499,"name":"drain","context":{"idset":"10279","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4673922,"name":"drain","context":{"idset":"10280","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4678195,"name":"drain","context":{"idset":"10281","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4682345,"name":"drain","context":{"idset":"10282","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.4686685,"name":"drain","context":{"idset":"10283","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.480298,"name":"drain","context":{"idset":"10284","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.493669,"name":"drain","context":{"idset":"10285","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.5050936,"name":"drain","context":{"idset":"10286","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.5160105,"name":"drain","context":{"idset":"10287","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.5231731,"name":"drain","context":{"idset":"10288","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.5329926,"name":"drain","context":{"idset":"10289","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.5441844,"name":"drain","context":{"idset":"10290","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.5564156,"name":"drain","context":{"idset":"10291","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.5680075,"name":"drain","context":{"idset":"10292","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.5789557,"name":"drain","context":{"idset":"10293","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.5863636,"name":"drain","context":{"idset":"10294","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.5931323,"name":"drain","context":{"idset":"10295","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.6023521,"name":"drain","context":{"idset":"10296","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.6134033,"name":"drain","context":{"idset":"10297","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.6246214,"name":"drain","context":{"idset":"10298","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.6355507,"name":"drain","context":{"idset":"10299","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.6467335,"name":"drain","context":{"idset":"10300","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.6584516,"name":"drain","context":{"idset":"10301","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.6690867,"name":"drain","context":{"idset":"10302","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.6799793,"name":"drain","context":{"idset":"10304","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.6922877,"name":"drain","context":{"idset":"10305","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.7038753,"name":"drain","context":{"idset":"10306","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.7157204,"name":"drain","context":{"idset":"10307","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.7268548,"name":"drain","context":{"idset":"10308","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.7380748,"name":"drain","context":{"idset":"10309","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.7494447,"name":"drain","context":{"idset":"10310","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.7608714,"name":"drain","context":{"idset":"10311","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.7717428,"name":"drain","context":{"idset":"10312","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.782958,"name":"drain","context":{"idset":"10313","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.7948337,"name":"drain","context":{"idset":"10314","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.806325,"name":"drain","context":{"idset":"10315","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.8176315,"name":"drain","context":{"idset":"10316","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.8290296,"name":"drain","context":{"idset":"10317","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.8401353,"name":"drain","context":{"idset":"10318","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.8517928,"name":"drain","context":{"idset":"10319","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.8655891,"name":"drain","context":{"idset":"10320","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.8763943,"name":"drain","context":{"idset":"10321","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.8867176,"name":"drain","context":{"idset":"10322","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.8978491,"name":"drain","context":{"idset":"10323","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.9081526,"name":"drain","context":{"idset":"10324","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.9184084,"name":"drain","context":{"idset":"10325","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.9292498,"name":"drain","context":{"idset":"10326","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.9380939,"name":"drain","context":{"idset":"10327","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.94625,"name":"drain","context":{"idset":"10328","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.9578955,"name":"drain","context":{"idset":"10329","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.9682839,"name":"drain","context":{"idset":"10330","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.9769497,"name":"drain","context":{"idset":"10331","reason":"broker was unresponsive"}}
+{"timestamp":1712673189.9885278,"name":"drain","context":{"idset":"10332","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0000489,"name":"drain","context":{"idset":"10333","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0115144,"name":"drain","context":{"idset":"10334","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0120096,"name":"drain","context":{"idset":"10335","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0124843,"name":"drain","context":{"idset":"10336","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0129218,"name":"drain","context":{"idset":"10337","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0133829,"name":"drain","context":{"idset":"10338","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0138395,"name":"drain","context":{"idset":"10339","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0143445,"name":"drain","context":{"idset":"10340","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0148377,"name":"drain","context":{"idset":"10341","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0153506,"name":"drain","context":{"idset":"10342","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0158365,"name":"drain","context":{"idset":"10345","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0163465,"name":"drain","context":{"idset":"10346","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0168228,"name":"drain","context":{"idset":"10347","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0172582,"name":"drain","context":{"idset":"10348","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0177212,"name":"drain","context":{"idset":"10349","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0181594,"name":"drain","context":{"idset":"10350","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0186269,"name":"drain","context":{"idset":"10351","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0191243,"name":"drain","context":{"idset":"10352","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0196419,"name":"drain","context":{"idset":"10353","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0201428,"name":"drain","context":{"idset":"10354","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0206547,"name":"drain","context":{"idset":"10357","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.021157,"name":"drain","context":{"idset":"10358","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0216708,"name":"drain","context":{"idset":"10359","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0221753,"name":"drain","context":{"idset":"10360","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0226829,"name":"drain","context":{"idset":"10361","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0231857,"name":"drain","context":{"idset":"10362","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0236773,"name":"drain","context":{"idset":"10363","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0241826,"name":"drain","context":{"idset":"10364","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0247009,"name":"drain","context":{"idset":"10365","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.025208,"name":"drain","context":{"idset":"10366","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0257211,"name":"drain","context":{"idset":"10367","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0262275,"name":"drain","context":{"idset":"10368","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0267477,"name":"drain","context":{"idset":"10369","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0272567,"name":"drain","context":{"idset":"10370","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0277772,"name":"drain","context":{"idset":"10371","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0282547,"name":"drain","context":{"idset":"10372","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0287271,"name":"drain","context":{"idset":"10373","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0291839,"name":"drain","context":{"idset":"10374","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0295827,"name":"drain","context":{"idset":"10375","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0299027,"name":"drain","context":{"idset":"10376","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0303898,"name":"drain","context":{"idset":"10377","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0310314,"name":"drain","context":{"idset":"10378","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0315015,"name":"drain","context":{"idset":"10379","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0319581,"name":"drain","context":{"idset":"10380","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0324528,"name":"drain","context":{"idset":"10381","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0329149,"name":"drain","context":{"idset":"10382","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0333953,"name":"drain","context":{"idset":"10383","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0338671,"name":"drain","context":{"idset":"10384","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.034445,"name":"drain","context":{"idset":"10385","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0349815,"name":"drain","context":{"idset":"10386","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0355248,"name":"drain","context":{"idset":"10387","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0360594,"name":"drain","context":{"idset":"10388","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.036607,"name":"drain","context":{"idset":"10389","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0371382,"name":"drain","context":{"idset":"10390","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0376792,"name":"drain","context":{"idset":"10391","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0382118,"name":"drain","context":{"idset":"10392","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0387146,"name":"drain","context":{"idset":"10393","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0392506,"name":"drain","context":{"idset":"10394","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0397568,"name":"drain","context":{"idset":"10395","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0402536,"name":"drain","context":{"idset":"10397","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0407996,"name":"drain","context":{"idset":"10398","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0413587,"name":"drain","context":{"idset":"10399","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0419054,"name":"drain","context":{"idset":"10400","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0424767,"name":"drain","context":{"idset":"10401","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0430059,"name":"drain","context":{"idset":"10402","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0435436,"name":"drain","context":{"idset":"10403","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0440323,"name":"drain","context":{"idset":"10404","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0445056,"name":"drain","context":{"idset":"10587","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0449619,"name":"drain","context":{"idset":"10588","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0454309,"name":"drain","context":{"idset":"10589","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0458889,"name":"drain","context":{"idset":"10590","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0463734,"name":"drain","context":{"idset":"10591","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0468709,"name":"drain","context":{"idset":"10592","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0473545,"name":"drain","context":{"idset":"10593","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0478373,"name":"drain","context":{"idset":"10594","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0482924,"name":"drain","context":{"idset":"10595","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0487206,"name":"drain","context":{"idset":"10596","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0492563,"name":"drain","context":{"idset":"10597","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0497305,"name":"drain","context":{"idset":"10598","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0502899,"name":"drain","context":{"idset":"10599","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0507374,"name":"drain","context":{"idset":"10600","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0511749,"name":"drain","context":{"idset":"10601","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0516214,"name":"drain","context":{"idset":"10602","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0520296,"name":"drain","context":{"idset":"10603","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0524852,"name":"drain","context":{"idset":"10604","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0529354,"name":"drain","context":{"idset":"10605","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0534093,"name":"drain","context":{"idset":"10606","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0538607,"name":"drain","context":{"idset":"10607","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0543289,"name":"drain","context":{"idset":"10608","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0548813,"name":"drain","context":{"idset":"10609","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0553751,"name":"drain","context":{"idset":"10610","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0559831,"name":"drain","context":{"idset":"10611","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0564923,"name":"drain","context":{"idset":"10612","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0569971,"name":"drain","context":{"idset":"10613","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0575156,"name":"drain","context":{"idset":"10628","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0580375,"name":"drain","context":{"idset":"10630","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0585783,"name":"drain","context":{"idset":"10631","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.059108,"name":"drain","context":{"idset":"10632","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0597153,"name":"drain","context":{"idset":"10633","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0603142,"name":"drain","context":{"idset":"10635","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0608993,"name":"drain","context":{"idset":"10636","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0614967,"name":"drain","context":{"idset":"10640","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0620344,"name":"drain","context":{"idset":"10641","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0625443,"name":"drain","context":{"idset":"10645","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.063082,"name":"drain","context":{"idset":"10646","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.063611,"name":"drain","context":{"idset":"10647","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0641477,"name":"drain","context":{"idset":"10648","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0647042,"name":"drain","context":{"idset":"10649","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0652263,"name":"drain","context":{"idset":"10650","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.065722,"name":"drain","context":{"idset":"10651","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0662172,"name":"drain","context":{"idset":"10652","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0667641,"name":"drain","context":{"idset":"10653","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0673027,"name":"drain","context":{"idset":"10654","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0678039,"name":"drain","context":{"idset":"10655","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.06831,"name":"drain","context":{"idset":"10656","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0688269,"name":"drain","context":{"idset":"10659","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0693765,"name":"drain","context":{"idset":"10660","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0698805,"name":"drain","context":{"idset":"10661","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0703886,"name":"drain","context":{"idset":"10662","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0708735,"name":"drain","context":{"idset":"10663","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0826809,"name":"drain","context":{"idset":"10664","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.0933414,"name":"drain","context":{"idset":"10665","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.1046257,"name":"drain","context":{"idset":"10666","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.1168847,"name":"drain","context":{"idset":"10667","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.1281319,"name":"drain","context":{"idset":"10668","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.1408031,"name":"drain","context":{"idset":"10669","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.157227,"name":"drain","context":{"idset":"10670","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.1688976,"name":"drain","context":{"idset":"10671","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.1795461,"name":"drain","context":{"idset":"10672","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.1904655,"name":"drain","context":{"idset":"10673","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.2010763,"name":"drain","context":{"idset":"10674","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.2133234,"name":"drain","context":{"idset":"10675","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.2257514,"name":"drain","context":{"idset":"10676","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.2383039,"name":"drain","context":{"idset":"10677","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.2506993,"name":"drain","context":{"idset":"10678","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.2622385,"name":"drain","context":{"idset":"10679","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.2742982,"name":"drain","context":{"idset":"10680","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.2866914,"name":"drain","context":{"idset":"10681","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.2979875,"name":"drain","context":{"idset":"10682","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.3081207,"name":"drain","context":{"idset":"10683","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.3190441,"name":"drain","context":{"idset":"10684","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.3300655,"name":"drain","context":{"idset":"10685","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.3384809,"name":"drain","context":{"idset":"10686","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.3485501,"name":"drain","context":{"idset":"10687","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.3578904,"name":"drain","context":{"idset":"10688","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.3689098,"name":"drain","context":{"idset":"10689","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.3788764,"name":"drain","context":{"idset":"10690","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.3876739,"name":"drain","context":{"idset":"10691","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.3988991,"name":"drain","context":{"idset":"10692","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.4105155,"name":"drain","context":{"idset":"10693","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.4187057,"name":"drain","context":{"idset":"10694","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.4299302,"name":"drain","context":{"idset":"10695","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.4404867,"name":"drain","context":{"idset":"10696","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.4515572,"name":"drain","context":{"idset":"10697","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.4630671,"name":"drain","context":{"idset":"10698","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.4748564,"name":"drain","context":{"idset":"10699","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.4862106,"name":"drain","context":{"idset":"10700","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.4976523,"name":"drain","context":{"idset":"10701","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.5079718,"name":"drain","context":{"idset":"10703","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.5198233,"name":"drain","context":{"idset":"10704","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.5277412,"name":"drain","context":{"idset":"10705","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.5394363,"name":"drain","context":{"idset":"10706","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.5501041,"name":"drain","context":{"idset":"10707","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.5583353,"name":"drain","context":{"idset":"10708","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.5695319,"name":"drain","context":{"idset":"10709","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.5810158,"name":"drain","context":{"idset":"10710","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.5922134,"name":"drain","context":{"idset":"10711","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.6028831,"name":"drain","context":{"idset":"10713","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.6114924,"name":"drain","context":{"idset":"10714","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.620425,"name":"drain","context":{"idset":"10715","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.6301751,"name":"drain","context":{"idset":"10716","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.6413507,"name":"drain","context":{"idset":"10717","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.6521719,"name":"drain","context":{"idset":"10719","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.6616082,"name":"drain","context":{"idset":"10720","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.6703031,"name":"drain","context":{"idset":"10721","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.6838028,"name":"drain","context":{"idset":"10722","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.6959367,"name":"drain","context":{"idset":"10723","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.7058861,"name":"drain","context":{"idset":"10724","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.7186675,"name":"drain","context":{"idset":"10725","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.7290552,"name":"drain","context":{"idset":"10726","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.7415495,"name":"drain","context":{"idset":"10727","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.7567799,"name":"drain","context":{"idset":"10728","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.7717843,"name":"drain","context":{"idset":"10729","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.7871127,"name":"drain","context":{"idset":"10730","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.8024747,"name":"drain","context":{"idset":"10731","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.8136859,"name":"drain","context":{"idset":"10732","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.8291836,"name":"drain","context":{"idset":"10733","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.8405497,"name":"drain","context":{"idset":"10734","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.8531463,"name":"drain","context":{"idset":"10735","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.8636301,"name":"drain","context":{"idset":"10736","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.8791463,"name":"drain","context":{"idset":"10737","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.8934906,"name":"drain","context":{"idset":"10738","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.9039724,"name":"drain","context":{"idset":"10741","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.9151495,"name":"drain","context":{"idset":"10742","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.9318209,"name":"drain","context":{"idset":"10743","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.9443135,"name":"drain","context":{"idset":"10744","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.9570763,"name":"drain","context":{"idset":"10745","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.9686728,"name":"drain","context":{"idset":"10746","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.9812162,"name":"drain","context":{"idset":"10747","reason":"broker was unresponsive"}}
+{"timestamp":1712673190.9916005,"name":"drain","context":{"idset":"10748","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.0024738,"name":"drain","context":{"idset":"10749","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.0138233,"name":"drain","context":{"idset":"10750","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.0246227,"name":"drain","context":{"idset":"10751","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.0354204,"name":"drain","context":{"idset":"10752","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.0470479,"name":"drain","context":{"idset":"10753","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.060483,"name":"drain","context":{"idset":"10754","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.0759838,"name":"drain","context":{"idset":"10755","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.0877318,"name":"drain","context":{"idset":"10756","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.0993922,"name":"drain","context":{"idset":"10757","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.1107683,"name":"drain","context":{"idset":"10758","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.1224017,"name":"drain","context":{"idset":"10759","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.1342888,"name":"drain","context":{"idset":"10760","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.1460345,"name":"drain","context":{"idset":"10761","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.1579051,"name":"drain","context":{"idset":"10762","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.1694417,"name":"drain","context":{"idset":"10763","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.1807709,"name":"drain","context":{"idset":"10764","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.1923859,"name":"drain","context":{"idset":"10765","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.202625,"name":"drain","context":{"idset":"10766","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.2127848,"name":"drain","context":{"idset":"10767","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.2239757,"name":"drain","context":{"idset":"10768","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.2353215,"name":"drain","context":{"idset":"10769","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.2462294,"name":"drain","context":{"idset":"10770","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.2576067,"name":"drain","context":{"idset":"10771","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.2698085,"name":"drain","context":{"idset":"10772","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.2815006,"name":"drain","context":{"idset":"10773","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.2927392,"name":"drain","context":{"idset":"10774","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3036156,"name":"drain","context":{"idset":"10775","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3148839,"name":"drain","context":{"idset":"10776","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3154407,"name":"drain","context":{"idset":"10777","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3159876,"name":"drain","context":{"idset":"10778","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3165553,"name":"drain","context":{"idset":"10779","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3171017,"name":"drain","context":{"idset":"10780","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3176613,"name":"drain","context":{"idset":"10781","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3285501,"name":"drain","context":{"idset":"10782","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3422332,"name":"drain","context":{"idset":"10783","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3539631,"name":"drain","context":{"idset":"10784","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3665237,"name":"drain","context":{"idset":"10785","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3773928,"name":"drain","context":{"idset":"10786","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3877766,"name":"drain","context":{"idset":"10787","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.3984289,"name":"drain","context":{"idset":"10788","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.4100897,"name":"drain","context":{"idset":"10789","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.4217474,"name":"drain","context":{"idset":"10790","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.432374,"name":"drain","context":{"idset":"10791","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.4439082,"name":"drain","context":{"idset":"10792","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.4554777,"name":"drain","context":{"idset":"10793","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.4673116,"name":"drain","context":{"idset":"10794","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.478781,"name":"drain","context":{"idset":"10795","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.4904225,"name":"drain","context":{"idset":"10796","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.502162,"name":"drain","context":{"idset":"10797","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.5133879,"name":"drain","context":{"idset":"10798","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.5256777,"name":"drain","context":{"idset":"10799","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.5391197,"name":"drain","context":{"idset":"10800","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.5520544,"name":"drain","context":{"idset":"10801","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.5631435,"name":"drain","context":{"idset":"10802","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.5735328,"name":"drain","context":{"idset":"10803","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.5858469,"name":"drain","context":{"idset":"10804","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.5950959,"name":"drain","context":{"idset":"10805","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.60642,"name":"drain","context":{"idset":"10806","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.6165786,"name":"drain","context":{"idset":"10807","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.6306663,"name":"drain","context":{"idset":"10808","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.6450653,"name":"drain","context":{"idset":"10809","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.6564023,"name":"drain","context":{"idset":"10810","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.6680198,"name":"drain","context":{"idset":"10811","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.6798654,"name":"drain","context":{"idset":"10812","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.6918688,"name":"drain","context":{"idset":"10813","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.7034183,"name":"drain","context":{"idset":"10814","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.7151222,"name":"drain","context":{"idset":"10815","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.7271781,"name":"drain","context":{"idset":"10816","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.7382905,"name":"drain","context":{"idset":"10817","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.7495592,"name":"drain","context":{"idset":"10818","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.7615452,"name":"drain","context":{"idset":"10819","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.771452,"name":"drain","context":{"idset":"10820","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.7829611,"name":"drain","context":{"idset":"10821","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.7943029,"name":"drain","context":{"idset":"10822","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8054988,"name":"drain","context":{"idset":"10823","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8173132,"name":"drain","context":{"idset":"10824","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8294606,"name":"drain","context":{"idset":"10825","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8300841,"name":"drain","context":{"idset":"10826","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8420181,"name":"drain","context":{"idset":"10827","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8426261,"name":"drain","context":{"idset":"10828","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8431976,"name":"drain","context":{"idset":"10829","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.843735,"name":"drain","context":{"idset":"10830","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.844296,"name":"drain","context":{"idset":"10831","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8448591,"name":"drain","context":{"idset":"10832","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8454483,"name":"drain","context":{"idset":"10833","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8460236,"name":"drain","context":{"idset":"10834","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8466208,"name":"drain","context":{"idset":"10835","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8472011,"name":"drain","context":{"idset":"10836","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8477824,"name":"drain","context":{"idset":"10837","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8483591,"name":"drain","context":{"idset":"10838","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8489327,"name":"drain","context":{"idset":"10839","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8495197,"name":"drain","context":{"idset":"10840","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8500648,"name":"drain","context":{"idset":"10841","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8506231,"name":"drain","context":{"idset":"10842","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8511677,"name":"drain","context":{"idset":"10843","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8517761,"name":"drain","context":{"idset":"10844","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8523588,"name":"drain","context":{"idset":"10845","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8529379,"name":"drain","context":{"idset":"10846","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.853502,"name":"drain","context":{"idset":"10847","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8540859,"name":"drain","context":{"idset":"10848","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8546722,"name":"drain","context":{"idset":"10849","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8552506,"name":"drain","context":{"idset":"10850","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8558927,"name":"drain","context":{"idset":"10851","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8565629,"name":"drain","context":{"idset":"10852","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8571992,"name":"drain","context":{"idset":"10853","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8578315,"name":"drain","context":{"idset":"10854","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8584771,"name":"drain","context":{"idset":"10855","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8590806,"name":"drain","context":{"idset":"10856","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8597212,"name":"drain","context":{"idset":"10857","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8603127,"name":"drain","context":{"idset":"10858","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8608449,"name":"drain","context":{"idset":"10859","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8613834,"name":"drain","context":{"idset":"10860","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8619442,"name":"drain","context":{"idset":"10861","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8624799,"name":"drain","context":{"idset":"10862","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.863004,"name":"drain","context":{"idset":"10863","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.863606,"name":"drain","context":{"idset":"10864","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8642466,"name":"drain","context":{"idset":"10865","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8649051,"name":"drain","context":{"idset":"10866","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.8655596,"name":"drain","context":{"idset":"10867","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.866209,"name":"drain","context":{"idset":"10868","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9384642,"name":"drain","context":{"idset":"10997","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9390898,"name":"drain","context":{"idset":"10998","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9397445,"name":"drain","context":{"idset":"11001","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9403949,"name":"drain","context":{"idset":"11002","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9407713,"name":"drain","context":{"idset":"11003","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9412234,"name":"drain","context":{"idset":"11004","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9416897,"name":"drain","context":{"idset":"11005","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9420626,"name":"drain","context":{"idset":"11006","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9424453,"name":"drain","context":{"idset":"11007","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9430599,"name":"drain","context":{"idset":"11008","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9436276,"name":"drain","context":{"idset":"11009","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9442599,"name":"drain","context":{"idset":"11010","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9449131,"name":"drain","context":{"idset":"11011","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9455664,"name":"drain","context":{"idset":"11012","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9462087,"name":"drain","context":{"idset":"11013","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9469047,"name":"drain","context":{"idset":"11014","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9475877,"name":"drain","context":{"idset":"11015","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9482439,"name":"drain","context":{"idset":"11016","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9489059,"name":"drain","context":{"idset":"11017","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9495697,"name":"drain","context":{"idset":"11018","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9502327,"name":"drain","context":{"idset":"11019","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9509096,"name":"drain","context":{"idset":"11020","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9521956,"name":"drain","context":{"idset":"11022","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9528575,"name":"drain","context":{"idset":"11023","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9535608,"name":"drain","context":{"idset":"11024","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9542818,"name":"drain","context":{"idset":"11025","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9549847,"name":"drain","context":{"idset":"11026","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9557023,"name":"drain","context":{"idset":"11027","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9564037,"name":"drain","context":{"idset":"11028","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9570909,"name":"drain","context":{"idset":"11029","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9578238,"name":"drain","context":{"idset":"11030","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9585543,"name":"drain","context":{"idset":"11031","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9592845,"name":"drain","context":{"idset":"11032","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9599936,"name":"drain","context":{"idset":"11033","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9607141,"name":"drain","context":{"idset":"11034","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9614208,"name":"drain","context":{"idset":"11035","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9621136,"name":"drain","context":{"idset":"11036","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9627683,"name":"drain","context":{"idset":"11037","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9633827,"name":"drain","context":{"idset":"11038","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9640005,"name":"drain","context":{"idset":"11039","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9645777,"name":"drain","context":{"idset":"11040","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9652994,"name":"drain","context":{"idset":"11041","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9660215,"name":"drain","context":{"idset":"11042","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9667373,"name":"drain","context":{"idset":"11043","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9674828,"name":"drain","context":{"idset":"11044","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9681966,"name":"drain","context":{"idset":"11045","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.968924,"name":"drain","context":{"idset":"11046","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9696434,"name":"drain","context":{"idset":"11047","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9703677,"name":"drain","context":{"idset":"11048","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9710743,"name":"drain","context":{"idset":"11049","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9718034,"name":"drain","context":{"idset":"11050","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9725261,"name":"drain","context":{"idset":"11051","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9732378,"name":"drain","context":{"idset":"11052","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9739473,"name":"drain","context":{"idset":"11053","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9746771,"name":"drain","context":{"idset":"11054","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9759252,"name":"drain","context":{"idset":"11055","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9764533,"name":"drain","context":{"idset":"11056","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9771085,"name":"drain","context":{"idset":"11057","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.977773,"name":"drain","context":{"idset":"11058","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9785397,"name":"drain","context":{"idset":"11059","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9792931,"name":"drain","context":{"idset":"11060","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9800835,"name":"drain","context":{"idset":"11061","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9808578,"name":"drain","context":{"idset":"11062","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9815943,"name":"drain","context":{"idset":"11063","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9823236,"name":"drain","context":{"idset":"11064","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9830441,"name":"drain","context":{"idset":"11065","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9837072,"name":"drain","context":{"idset":"11066","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9843209,"name":"drain","context":{"idset":"11067","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9849465,"name":"drain","context":{"idset":"11068","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9855998,"name":"drain","context":{"idset":"11069","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9862969,"name":"drain","context":{"idset":"11070","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9869425,"name":"drain","context":{"idset":"11071","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9876468,"name":"drain","context":{"idset":"11072","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.988343,"name":"drain","context":{"idset":"11073","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9890273,"name":"drain","context":{"idset":"11074","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9897234,"name":"drain","context":{"idset":"11075","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9904146,"name":"drain","context":{"idset":"11076","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9910839,"name":"drain","context":{"idset":"11077","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.991785,"name":"drain","context":{"idset":"11078","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9924874,"name":"drain","context":{"idset":"11079","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9931653,"name":"drain","context":{"idset":"11080","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9938538,"name":"drain","context":{"idset":"11081","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9945343,"name":"drain","context":{"idset":"11082","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9953148,"name":"drain","context":{"idset":"11083","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9959705,"name":"drain","context":{"idset":"11084","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.996769,"name":"drain","context":{"idset":"11085","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9975142,"name":"drain","context":{"idset":"11086","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9981682,"name":"drain","context":{"idset":"11087","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9988408,"name":"drain","context":{"idset":"11088","reason":"broker was unresponsive"}}
+{"timestamp":1712673191.9995062,"name":"drain","context":{"idset":"11089","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0001588,"name":"drain","context":{"idset":"11090","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0010514,"name":"drain","context":{"idset":"11091","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0016768,"name":"drain","context":{"idset":"11092","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0023932,"name":"drain","context":{"idset":"11093","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0031288,"name":"drain","context":{"idset":"11094","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0038834,"name":"drain","context":{"idset":"11095","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0046415,"name":"drain","context":{"idset":"11096","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0053792,"name":"drain","context":{"idset":"11097","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0060797,"name":"drain","context":{"idset":"11098","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0068467,"name":"drain","context":{"idset":"11099","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0075998,"name":"drain","context":{"idset":"11100","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.008357,"name":"drain","context":{"idset":"11101","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0090978,"name":"drain","context":{"idset":"11102","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0098507,"name":"drain","context":{"idset":"11103","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0106106,"name":"drain","context":{"idset":"11104","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0113564,"name":"drain","context":{"idset":"11105","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0120144,"name":"drain","context":{"idset":"11106","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0124333,"name":"drain","context":{"idset":"11107","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0128171,"name":"drain","context":{"idset":"11108","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0132325,"name":"drain","context":{"idset":"11109","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0136378,"name":"drain","context":{"idset":"11110","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0140474,"name":"drain","context":{"idset":"11113","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0144691,"name":"drain","context":{"idset":"11114","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0148811,"name":"drain","context":{"idset":"11116","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0152891,"name":"drain","context":{"idset":"11119","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0157006,"name":"drain","context":{"idset":"11120","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0161192,"name":"drain","context":{"idset":"11121","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.016546,"name":"drain","context":{"idset":"11122","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0169413,"name":"drain","context":{"idset":"11123","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0173666,"name":"drain","context":{"idset":"11124","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0177794,"name":"drain","context":{"idset":"11253","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0181904,"name":"drain","context":{"idset":"11254","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0187066,"name":"drain","context":{"idset":"11255","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0191054,"name":"drain","context":{"idset":"11256","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0196519,"name":"drain","context":{"idset":"11257","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0201139,"name":"drain","context":{"idset":"11258","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0206969,"name":"drain","context":{"idset":"11259","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.021158,"name":"drain","context":{"idset":"11260","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0217464,"name":"drain","context":{"idset":"11261","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0221701,"name":"drain","context":{"idset":"11262","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0226004,"name":"drain","context":{"idset":"11263","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0230052,"name":"drain","context":{"idset":"11264","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0234356,"name":"drain","context":{"idset":"11265","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0238307,"name":"drain","context":{"idset":"11266","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0242419,"name":"drain","context":{"idset":"11267","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0249202,"name":"drain","context":{"idset":"11268","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0254316,"name":"drain","context":{"idset":"11269","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0258646,"name":"drain","context":{"idset":"11270","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0262585,"name":"drain","context":{"idset":"11271","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0266929,"name":"drain","context":{"idset":"11272","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0270886,"name":"drain","context":{"idset":"11273","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0275209,"name":"drain","context":{"idset":"11274","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0279412,"name":"drain","context":{"idset":"11275","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0284009,"name":"drain","context":{"idset":"11276","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0289075,"name":"drain","context":{"idset":"11277","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0295999,"name":"drain","context":{"idset":"11278","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0303185,"name":"drain","context":{"idset":"11279","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.031028,"name":"drain","context":{"idset":"11280","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0317488,"name":"drain","context":{"idset":"11281","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0325065,"name":"drain","context":{"idset":"11282","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0332358,"name":"drain","context":{"idset":"11283","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0339551,"name":"drain","context":{"idset":"11284","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0346749,"name":"drain","context":{"idset":"11285","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0353894,"name":"drain","context":{"idset":"11286","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0360999,"name":"drain","context":{"idset":"11287","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0368321,"name":"drain","context":{"idset":"11288","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0375462,"name":"drain","context":{"idset":"11289","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0383549,"name":"drain","context":{"idset":"11290","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0390551,"name":"drain","context":{"idset":"11291","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0397787,"name":"drain","context":{"idset":"11292","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0404787,"name":"drain","context":{"idset":"11293","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0411284,"name":"drain","context":{"idset":"11294","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0418141,"name":"drain","context":{"idset":"11295","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0424812,"name":"drain","context":{"idset":"11296","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0431397,"name":"drain","context":{"idset":"11297","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0438051,"name":"drain","context":{"idset":"11298","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0444944,"name":"drain","context":{"idset":"11299","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0451684,"name":"drain","context":{"idset":"11300","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.045857,"name":"drain","context":{"idset":"11301","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0465765,"name":"drain","context":{"idset":"11302","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0473206,"name":"drain","context":{"idset":"11303","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0479243,"name":"drain","context":{"idset":"11304","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0487258,"name":"drain","context":{"idset":"11305","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0495186,"name":"drain","context":{"idset":"11306","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0503099,"name":"drain","context":{"idset":"11307","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0510981,"name":"drain","context":{"idset":"11308","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0517917,"name":"drain","context":{"idset":"11309","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0524447,"name":"drain","context":{"idset":"11310","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0531268,"name":"drain","context":{"idset":"11311","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0539143,"name":"drain","context":{"idset":"11312","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0547187,"name":"drain","context":{"idset":"11313","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0555151,"name":"drain","context":{"idset":"11314","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0563054,"name":"drain","context":{"idset":"11315","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0570781,"name":"drain","context":{"idset":"11316","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0578775,"name":"drain","context":{"idset":"11317","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0586693,"name":"drain","context":{"idset":"11318","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0594606,"name":"drain","context":{"idset":"11319","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0602508,"name":"drain","context":{"idset":"11320","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0610447,"name":"drain","context":{"idset":"11321","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0618424,"name":"drain","context":{"idset":"11322","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0626404,"name":"drain","context":{"idset":"11323","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0634332,"name":"drain","context":{"idset":"11324","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0642364,"name":"drain","context":{"idset":"11325","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0650053,"name":"drain","context":{"idset":"11326","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.06582,"name":"drain","context":{"idset":"11327","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0666118,"name":"drain","context":{"idset":"11328","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0671542,"name":"drain","context":{"idset":"11329","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0675969,"name":"drain","context":{"idset":"11330","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0680082,"name":"drain","context":{"idset":"11331","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.068434,"name":"drain","context":{"idset":"11332","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0688415,"name":"drain","context":{"idset":"11333","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.069247,"name":"drain","context":{"idset":"11334","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0696697,"name":"drain","context":{"idset":"11335","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0700808,"name":"drain","context":{"idset":"11336","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0705023,"name":"drain","context":{"idset":"11337","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0709088,"name":"drain","context":{"idset":"11338","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0713348,"name":"drain","context":{"idset":"11339","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.071744,"name":"drain","context":{"idset":"11340","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0721521,"name":"drain","context":{"idset":"11341","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0725772,"name":"drain","context":{"idset":"11342","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.072984,"name":"drain","context":{"idset":"11343","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0735252,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0743344,"name":"drain","context":{"idset":"11345","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0751262,"name":"drain","context":{"idset":"11346","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0757172,"name":"drain","context":{"idset":"11347","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.076138,"name":"drain","context":{"idset":"11348","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0765622,"name":"drain","context":{"idset":"11349","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0769699,"name":"drain","context":{"idset":"11350","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0773973,"name":"drain","context":{"idset":"11351","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0778167,"name":"drain","context":{"idset":"11352","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0782437,"name":"drain","context":{"idset":"11353","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.078917,"name":"drain","context":{"idset":"11354","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0796697,"name":"drain","context":{"idset":"11355","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0804203,"name":"drain","context":{"idset":"11356","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0811629,"name":"drain","context":{"idset":"11357","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0819092,"name":"drain","context":{"idset":"11358","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0826538,"name":"drain","context":{"idset":"11359","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0833721,"name":"drain","context":{"idset":"11360","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0840671,"name":"drain","context":{"idset":"11361","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0847807,"name":"drain","context":{"idset":"11362","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0855255,"name":"drain","context":{"idset":"11363","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0862818,"name":"drain","context":{"idset":"11364","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0869746,"name":"drain","context":{"idset":"11365","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0877151,"name":"drain","context":{"idset":"11366","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0884826,"name":"drain","context":{"idset":"11367","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.089222,"name":"drain","context":{"idset":"11368","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0899806,"name":"drain","context":{"idset":"11369","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0907114,"name":"drain","context":{"idset":"11370","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0914752,"name":"drain","context":{"idset":"11371","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0922563,"name":"drain","context":{"idset":"11372","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.093091,"name":"drain","context":{"idset":"11373","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0938864,"name":"drain","context":{"idset":"11374","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0946841,"name":"drain","context":{"idset":"11375","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0955029,"name":"drain","context":{"idset":"11376","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0963228,"name":"drain","context":{"idset":"11377","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0971429,"name":"drain","context":{"idset":"11378","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0979669,"name":"drain","context":{"idset":"11379","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.0987329,"name":"drain","context":{"idset":"11380","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.1944716,"name":"drain","context":{"idset":"11509","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.1953523,"name":"drain","context":{"idset":"11510","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.1961718,"name":"drain","context":{"idset":"11511","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.1969814,"name":"drain","context":{"idset":"11512","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.1978025,"name":"drain","context":{"idset":"11513","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.1987429,"name":"drain","context":{"idset":"11514","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.1994464,"name":"drain","context":{"idset":"11515","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2000265,"name":"drain","context":{"idset":"11516","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2006373,"name":"drain","context":{"idset":"11517","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2011924,"name":"drain","context":{"idset":"11518","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2018948,"name":"drain","context":{"idset":"11519","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2026296,"name":"drain","context":{"idset":"11520","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2033029,"name":"drain","context":{"idset":"11521","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2040787,"name":"drain","context":{"idset":"11522","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2048626,"name":"drain","context":{"idset":"11523","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2055993,"name":"drain","context":{"idset":"11524","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2063448,"name":"drain","context":{"idset":"11525","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2070823,"name":"drain","context":{"idset":"11526","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.207783,"name":"drain","context":{"idset":"11527","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2084389,"name":"drain","context":{"idset":"11528","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2089,"name":"drain","context":{"idset":"11529","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2094457,"name":"drain","context":{"idset":"11530","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2099905,"name":"drain","context":{"idset":"11531","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2105877,"name":"drain","context":{"idset":"11532","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2111864,"name":"drain","context":{"idset":"11533","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2116871,"name":"drain","context":{"idset":"11534","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2122269,"name":"drain","context":{"idset":"11535","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2127769,"name":"drain","context":{"idset":"11536","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2133439,"name":"drain","context":{"idset":"11537","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2140794,"name":"drain","context":{"idset":"11538","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2148621,"name":"drain","context":{"idset":"11539","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2154298,"name":"drain","context":{"idset":"11540","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2163076,"name":"drain","context":{"idset":"11541","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2171683,"name":"drain","context":{"idset":"11542","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2180629,"name":"drain","context":{"idset":"11543","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.218955,"name":"drain","context":{"idset":"11544","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2198281,"name":"drain","context":{"idset":"11545","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2207031,"name":"drain","context":{"idset":"11546","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2226481,"name":"drain","context":{"idset":"11547","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2235332,"name":"drain","context":{"idset":"11548","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2244363,"name":"drain","context":{"idset":"11549","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2253332,"name":"drain","context":{"idset":"11550","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.225935,"name":"drain","context":{"idset":"11551","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2265751,"name":"drain","context":{"idset":"11552","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2271574,"name":"drain","context":{"idset":"11553","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2277951,"name":"drain","context":{"idset":"11554","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2284055,"name":"drain","context":{"idset":"11555","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2290137,"name":"drain","context":{"idset":"11556","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2296567,"name":"drain","context":{"idset":"11557","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2302384,"name":"drain","context":{"idset":"11558","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2308121,"name":"drain","context":{"idset":"11559","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2314119,"name":"drain","context":{"idset":"11560","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2319562,"name":"drain","context":{"idset":"11561","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2325411,"name":"drain","context":{"idset":"11562","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2333629,"name":"drain","context":{"idset":"11563","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2341957,"name":"drain","context":{"idset":"11564","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2350428,"name":"drain","context":{"idset":"11565","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2359111,"name":"drain","context":{"idset":"11566","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2367752,"name":"drain","context":{"idset":"11567","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2376163,"name":"drain","context":{"idset":"11568","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2384636,"name":"drain","context":{"idset":"11570","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2393355,"name":"drain","context":{"idset":"11571","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2401788,"name":"drain","context":{"idset":"11572","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2410114,"name":"drain","context":{"idset":"11573","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2418389,"name":"drain","context":{"idset":"11574","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2426679,"name":"drain","context":{"idset":"11575","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2435217,"name":"drain","context":{"idset":"11576","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2443264,"name":"drain","context":{"idset":"11577","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2451127,"name":"drain","context":{"idset":"11578","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2459297,"name":"drain","context":{"idset":"11579","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2467718,"name":"drain","context":{"idset":"11580","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2475939,"name":"drain","context":{"idset":"11581","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2483966,"name":"drain","context":{"idset":"11582","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2491901,"name":"drain","context":{"idset":"11583","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2500377,"name":"drain","context":{"idset":"11584","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2508512,"name":"drain","context":{"idset":"11585","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.251709,"name":"drain","context":{"idset":"11586","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2526386,"name":"drain","context":{"idset":"11587","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2535563,"name":"drain","context":{"idset":"11588","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2544715,"name":"drain","context":{"idset":"11589","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2553954,"name":"drain","context":{"idset":"11590","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2563031,"name":"drain","context":{"idset":"11591","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2572129,"name":"drain","context":{"idset":"11592","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2581384,"name":"drain","context":{"idset":"11593","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2590606,"name":"drain","context":{"idset":"11594","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2599339,"name":"drain","context":{"idset":"11595","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2607796,"name":"drain","context":{"idset":"11596","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2615807,"name":"drain","context":{"idset":"11597","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.262368,"name":"drain","context":{"idset":"11598","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2631459,"name":"drain","context":{"idset":"11599","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2639408,"name":"drain","context":{"idset":"11600","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2647495,"name":"drain","context":{"idset":"11601","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2655733,"name":"drain","context":{"idset":"11602","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2664053,"name":"drain","context":{"idset":"11603","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2672462,"name":"drain","context":{"idset":"11604","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2678449,"name":"drain","context":{"idset":"11605","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2683895,"name":"drain","context":{"idset":"11606","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.268868,"name":"drain","context":{"idset":"11607","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2693584,"name":"drain","context":{"idset":"11608","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2698314,"name":"drain","context":{"idset":"11609","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.270324,"name":"drain","context":{"idset":"11610","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2707949,"name":"drain","context":{"idset":"11611","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2712827,"name":"drain","context":{"idset":"11612","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2717569,"name":"drain","context":{"idset":"11613","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2722452,"name":"drain","context":{"idset":"11614","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2730265,"name":"drain","context":{"idset":"11615","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2738726,"name":"drain","context":{"idset":"11616","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2747386,"name":"drain","context":{"idset":"11617","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2755747,"name":"drain","context":{"idset":"11618","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2763877,"name":"drain","context":{"idset":"11619","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2772217,"name":"drain","context":{"idset":"11620","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2780421,"name":"drain","context":{"idset":"11621","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2788413,"name":"drain","context":{"idset":"11622","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2796495,"name":"drain","context":{"idset":"11623","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2804604,"name":"drain","context":{"idset":"11624","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2812874,"name":"drain","context":{"idset":"11625","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.28211,"name":"drain","context":{"idset":"11626","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2829754,"name":"drain","context":{"idset":"11627","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2837751,"name":"drain","context":{"idset":"11628","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2845817,"name":"drain","context":{"idset":"11629","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2853932,"name":"drain","context":{"idset":"11630","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2862322,"name":"drain","context":{"idset":"11631","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2870789,"name":"drain","context":{"idset":"11632","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2879064,"name":"drain","context":{"idset":"11633","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2887692,"name":"drain","context":{"idset":"11634","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2896519,"name":"drain","context":{"idset":"11635","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2905214,"name":"drain","context":{"idset":"11636","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2914281,"name":"drain","context":{"idset":"11653","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.292351,"name":"drain","context":{"idset":"11654","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2932198,"name":"drain","context":{"idset":"11655","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2941723,"name":"drain","context":{"idset":"11656","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2951226,"name":"drain","context":{"idset":"11657","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.296067,"name":"drain","context":{"idset":"11658","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.297019,"name":"drain","context":{"idset":"11659","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.297972,"name":"drain","context":{"idset":"11660","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2989523,"name":"drain","context":{"idset":"11661","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.2998276,"name":"drain","context":{"idset":"11662","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3007803,"name":"drain","context":{"idset":"11663","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3016598,"name":"drain","context":{"idset":"11664","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3024623,"name":"drain","context":{"idset":"11665","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3033035,"name":"drain","context":{"idset":"11666","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3040848,"name":"drain","context":{"idset":"11667","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.30496,"name":"drain","context":{"idset":"11668","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3058925,"name":"drain","context":{"idset":"11669","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3068416,"name":"drain","context":{"idset":"11670","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3077433,"name":"drain","context":{"idset":"11671","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3085952,"name":"drain","context":{"idset":"11672","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3094437,"name":"drain","context":{"idset":"11673","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3103006,"name":"drain","context":{"idset":"11674","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3111475,"name":"drain","context":{"idset":"11675","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3120425,"name":"drain","context":{"idset":"11676","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3129101,"name":"drain","context":{"idset":"11677","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3137934,"name":"drain","context":{"idset":"11678","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3146954,"name":"drain","context":{"idset":"11679","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3155532,"name":"drain","context":{"idset":"11680","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3164225,"name":"drain","context":{"idset":"11681","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3172972,"name":"drain","context":{"idset":"11682","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3181787,"name":"drain","context":{"idset":"11683","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3190579,"name":"drain","context":{"idset":"11684","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3199799,"name":"drain","context":{"idset":"11685","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3208976,"name":"drain","context":{"idset":"11686","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3217881,"name":"drain","context":{"idset":"11687","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3226817,"name":"drain","context":{"idset":"11688","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3235819,"name":"drain","context":{"idset":"11689","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3244791,"name":"drain","context":{"idset":"11690","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3253622,"name":"drain","context":{"idset":"11691","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3262227,"name":"drain","context":{"idset":"11692","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3270974,"name":"drain","context":{"idset":"11693","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3279626,"name":"drain","context":{"idset":"11695","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3287969,"name":"drain","context":{"idset":"11696","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3296201,"name":"drain","context":{"idset":"11697","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3304536,"name":"drain","context":{"idset":"11698","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3313506,"name":"drain","context":{"idset":"11699","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3321676,"name":"drain","context":{"idset":"11700","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3330405,"name":"drain","context":{"idset":"11701","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3338578,"name":"drain","context":{"idset":"11702","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3346479,"name":"drain","context":{"idset":"11703","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3355532,"name":"drain","context":{"idset":"11704","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.33652,"name":"drain","context":{"idset":"11705","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3374815,"name":"drain","context":{"idset":"11706","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3384304,"name":"drain","context":{"idset":"11707","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3392384,"name":"drain","context":{"idset":"11708","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3441925,"name":"drain","context":{"idset":"11709","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3451424,"name":"drain","context":{"idset":"11710","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3460841,"name":"drain","context":{"idset":"11711","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3469658,"name":"drain","context":{"idset":"11712","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3478804,"name":"drain","context":{"idset":"11713","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3487554,"name":"drain","context":{"idset":"11714","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3495767,"name":"drain","context":{"idset":"11715","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3504395,"name":"drain","context":{"idset":"11716","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3513424,"name":"drain","context":{"idset":"11717","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3523037,"name":"drain","context":{"idset":"11718","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3532274,"name":"drain","context":{"idset":"11719","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3540604,"name":"drain","context":{"idset":"11720","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.354985,"name":"drain","context":{"idset":"11721","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3559062,"name":"drain","context":{"idset":"11722","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3568664,"name":"drain","context":{"idset":"11723","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3578413,"name":"drain","context":{"idset":"11724","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3587372,"name":"drain","context":{"idset":"11725","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.359683,"name":"drain","context":{"idset":"11726","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3606403,"name":"drain","context":{"idset":"11727","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3616207,"name":"drain","context":{"idset":"11728","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3626029,"name":"drain","context":{"idset":"11729","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3635719,"name":"drain","context":{"idset":"11730","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3645296,"name":"drain","context":{"idset":"11731","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3654978,"name":"drain","context":{"idset":"11732","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3664412,"name":"drain","context":{"idset":"11733","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3674028,"name":"drain","context":{"idset":"11734","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3683207,"name":"drain","context":{"idset":"11735","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3692939,"name":"drain","context":{"idset":"11736","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3702254,"name":"drain","context":{"idset":"11737","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3711994,"name":"drain","context":{"idset":"11738","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3721075,"name":"drain","context":{"idset":"11739","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3730671,"name":"drain","context":{"idset":"11740","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3740463,"name":"drain","context":{"idset":"11741","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3750036,"name":"drain","context":{"idset":"11742","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3759601,"name":"drain","context":{"idset":"11743","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.376925,"name":"drain","context":{"idset":"11744","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3778698,"name":"drain","context":{"idset":"11745","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3787572,"name":"drain","context":{"idset":"11746","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3797398,"name":"drain","context":{"idset":"11747","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.380728,"name":"drain","context":{"idset":"11748","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3817163,"name":"drain","context":{"idset":"11749","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3827083,"name":"drain","context":{"idset":"11750","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3836946,"name":"drain","context":{"idset":"11751","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3847024,"name":"drain","context":{"idset":"11752","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3856797,"name":"drain","context":{"idset":"11753","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3866556,"name":"drain","context":{"idset":"11754","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3876336,"name":"drain","context":{"idset":"11755","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3885961,"name":"drain","context":{"idset":"11756","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3895991,"name":"drain","context":{"idset":"11757","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3907027,"name":"drain","context":{"idset":"11758","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3916962,"name":"drain","context":{"idset":"11759","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3926725,"name":"drain","context":{"idset":"11760","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3936181,"name":"drain","context":{"idset":"11761","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3945673,"name":"drain","context":{"idset":"11762","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3953876,"name":"drain","context":{"idset":"11763","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.396244,"name":"drain","context":{"idset":"11764","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3971958,"name":"drain","context":{"idset":"11765","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.3981867,"name":"drain","context":{"idset":"11766","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.399163,"name":"drain","context":{"idset":"11767","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4000783,"name":"drain","context":{"idset":"11768","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.401082,"name":"drain","context":{"idset":"11769","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4020371,"name":"drain","context":{"idset":"11770","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4029851,"name":"drain","context":{"idset":"11771","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4039507,"name":"drain","context":{"idset":"11772","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4049282,"name":"drain","context":{"idset":"11773","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4059222,"name":"drain","context":{"idset":"11774","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4072559,"name":"drain","context":{"idset":"11775","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4082549,"name":"drain","context":{"idset":"11776","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4092095,"name":"drain","context":{"idset":"11777","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.410207,"name":"drain","context":{"idset":"11778","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4111588,"name":"drain","context":{"idset":"11779","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4121709,"name":"drain","context":{"idset":"11780","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4131649,"name":"drain","context":{"idset":"11781","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4141598,"name":"drain","context":{"idset":"11782","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4151473,"name":"drain","context":{"idset":"11783","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4161334,"name":"drain","context":{"idset":"11784","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4169552,"name":"drain","context":{"idset":"11785","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.417666,"name":"drain","context":{"idset":"11786","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4182186,"name":"drain","context":{"idset":"11787","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4191144,"name":"drain","context":{"idset":"11788","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4200487,"name":"drain","context":{"idset":"11789","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4209747,"name":"drain","context":{"idset":"11790","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4219172,"name":"drain","context":{"idset":"11791","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4228542,"name":"drain","context":{"idset":"11792","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4237974,"name":"drain","context":{"idset":"11793","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4247339,"name":"drain","context":{"idset":"11794","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4256783,"name":"drain","context":{"idset":"11795","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4266021,"name":"drain","context":{"idset":"11796","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4275157,"name":"drain","context":{"idset":"11797","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4284441,"name":"drain","context":{"idset":"11798","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.429368,"name":"drain","context":{"idset":"11799","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4301195,"name":"drain","context":{"idset":"11800","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4309907,"name":"drain","context":{"idset":"11801","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.431931,"name":"drain","context":{"idset":"11802","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4328797,"name":"drain","context":{"idset":"11803","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4336557,"name":"drain","context":{"idset":"11804","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4341762,"name":"drain","context":{"idset":"11805","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4351077,"name":"drain","context":{"idset":"11806","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4361274,"name":"drain","context":{"idset":"11807","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4370854,"name":"drain","context":{"idset":"11808","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4377527,"name":"drain","context":{"idset":"11809","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4383004,"name":"drain","context":{"idset":"11810","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4388263,"name":"drain","context":{"idset":"11811","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.439362,"name":"drain","context":{"idset":"11812","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4398828,"name":"drain","context":{"idset":"11813","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4405029,"name":"drain","context":{"idset":"11814","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4414198,"name":"drain","context":{"idset":"11815","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4423242,"name":"drain","context":{"idset":"11816","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4431715,"name":"drain","context":{"idset":"11817","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4440799,"name":"drain","context":{"idset":"11818","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4451094,"name":"drain","context":{"idset":"11819","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4461353,"name":"drain","context":{"idset":"11820","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4471374,"name":"drain","context":{"idset":"11821","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4480748,"name":"drain","context":{"idset":"11822","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4489045,"name":"drain","context":{"idset":"11823","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4494672,"name":"drain","context":{"idset":"11824","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4501417,"name":"drain","context":{"idset":"11825","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4510541,"name":"drain","context":{"idset":"11826","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4519,"name":"drain","context":{"idset":"11827","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4524524,"name":"drain","context":{"idset":"11828","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4529779,"name":"drain","context":{"idset":"11829","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4535377,"name":"drain","context":{"idset":"11830","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4543369,"name":"drain","context":{"idset":"11831","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4551063,"name":"drain","context":{"idset":"11832","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.45595,"name":"drain","context":{"idset":"11833","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4567003,"name":"drain","context":{"idset":"11834","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4576564,"name":"drain","context":{"idset":"11835","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4586897,"name":"drain","context":{"idset":"11836","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.459712,"name":"drain","context":{"idset":"11837","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4607368,"name":"drain","context":{"idset":"11838","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4617636,"name":"drain","context":{"idset":"11839","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4627886,"name":"drain","context":{"idset":"11840","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4638135,"name":"drain","context":{"idset":"11841","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4648438,"name":"drain","context":{"idset":"11842","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4656591,"name":"drain","context":{"idset":"11843","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4666126,"name":"drain","context":{"idset":"11844","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4970746,"name":"drain","context":{"idset":"11877","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4981236,"name":"drain","context":{"idset":"11878","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.4991329,"name":"drain","context":{"idset":"11879","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.500144,"name":"drain","context":{"idset":"11880","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5011415,"name":"drain","context":{"idset":"11881","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5021091,"name":"drain","context":{"idset":"11882","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5030265,"name":"drain","context":{"idset":"11883","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5039885,"name":"drain","context":{"idset":"11884","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5049415,"name":"drain","context":{"idset":"11885","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5059345,"name":"drain","context":{"idset":"11886","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5068946,"name":"drain","context":{"idset":"11887","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5078499,"name":"drain","context":{"idset":"11888","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5088201,"name":"drain","context":{"idset":"11889","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5097368,"name":"drain","context":{"idset":"11890","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5106466,"name":"drain","context":{"idset":"11891","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5115631,"name":"drain","context":{"idset":"11892","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5127966,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5138535,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5150645,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5163076,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5175807,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5188489,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5201538,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5214422,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5227242,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5243497,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.52563,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5268879,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.528096,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5293324,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5305095,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5317042,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.532872,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5340676,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5353076,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5366488,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5380197,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.539407,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5408089,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5421972,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5434439,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5445163,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5457659,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5470092,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5482883,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5494921,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5507388,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5519276,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.558557,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5598619,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5611446,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5624063,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5637002,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5649109,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5661459,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5674357,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5688031,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5701079,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5714326,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5727479,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5740352,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5753338,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5765595,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5777731,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.579057,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.580457,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5818408,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5832093,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5845814,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5858741,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5872543,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5886753,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5900655,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5914707,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5930412,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5943942,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5958385,"name":"drain","context":{"idset":"85","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5971925,"name":"drain","context":{"idset":"86","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5985227,"name":"drain","context":{"idset":"87","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.5998292,"name":"drain","context":{"idset":"88","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6011407,"name":"drain","context":{"idset":"89","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6024401,"name":"drain","context":{"idset":"90","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6037405,"name":"drain","context":{"idset":"91","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.604974,"name":"drain","context":{"idset":"92","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6061902,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6074667,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6087859,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6101067,"name":"drain","context":{"idset":"96","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.611439,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6127417,"name":"drain","context":{"idset":"98","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6140318,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6153316,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6165755,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6178348,"name":"drain","context":{"idset":"102","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6191542,"name":"drain","context":{"idset":"103","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6205082,"name":"drain","context":{"idset":"104","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6218076,"name":"drain","context":{"idset":"105","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6231251,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6244802,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6258109,"name":"drain","context":{"idset":"108","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6270826,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6284003,"name":"drain","context":{"idset":"110","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6296444,"name":"drain","context":{"idset":"111","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6308043,"name":"drain","context":{"idset":"112","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6320424,"name":"drain","context":{"idset":"113","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6364853,"name":"drain","context":{"idset":"114","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6376815,"name":"drain","context":{"idset":"115","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.638911,"name":"drain","context":{"idset":"116","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6402013,"name":"drain","context":{"idset":"117","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6416438,"name":"drain","context":{"idset":"118","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6430659,"name":"drain","context":{"idset":"119","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.644418,"name":"drain","context":{"idset":"120","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6456668,"name":"drain","context":{"idset":"122","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6469688,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6482489,"name":"drain","context":{"idset":"124","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6495004,"name":"drain","context":{"idset":"125","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6507905,"name":"drain","context":{"idset":"126","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.652056,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6533401,"name":"drain","context":{"idset":"128","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6545751,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6558619,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6572185,"name":"drain","context":{"idset":"131","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6585822,"name":"drain","context":{"idset":"132","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6599202,"name":"drain","context":{"idset":"133","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6612601,"name":"drain","context":{"idset":"134","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6625934,"name":"drain","context":{"idset":"135","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6638906,"name":"drain","context":{"idset":"136","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6652136,"name":"drain","context":{"idset":"137","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6665623,"name":"drain","context":{"idset":"138","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.667937,"name":"drain","context":{"idset":"139","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6692853,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6706264,"name":"drain","context":{"idset":"141","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6726277,"name":"drain","context":{"idset":"142","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6748366,"name":"drain","context":{"idset":"143","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6761932,"name":"drain","context":{"idset":"144","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6775882,"name":"drain","context":{"idset":"145","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6788707,"name":"drain","context":{"idset":"146","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6801689,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6816108,"name":"drain","context":{"idset":"148","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6829674,"name":"drain","context":{"idset":"149","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6843171,"name":"drain","context":{"idset":"150","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6856563,"name":"drain","context":{"idset":"151","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6869767,"name":"drain","context":{"idset":"152","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.688282,"name":"drain","context":{"idset":"153","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6896322,"name":"drain","context":{"idset":"154","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6909862,"name":"drain","context":{"idset":"155","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6923432,"name":"drain","context":{"idset":"156","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6937149,"name":"drain","context":{"idset":"157","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6950483,"name":"drain","context":{"idset":"158","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6963725,"name":"drain","context":{"idset":"159","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6976988,"name":"drain","context":{"idset":"160","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.6995842,"name":"drain","context":{"idset":"161","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.700887,"name":"drain","context":{"idset":"162","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7022314,"name":"drain","context":{"idset":"163","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7036054,"name":"drain","context":{"idset":"164","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7049253,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7062225,"name":"drain","context":{"idset":"166","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7075846,"name":"drain","context":{"idset":"167","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7089438,"name":"drain","context":{"idset":"168","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7103052,"name":"drain","context":{"idset":"169","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7116621,"name":"drain","context":{"idset":"170","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7129445,"name":"drain","context":{"idset":"171","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7142079,"name":"drain","context":{"idset":"172","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7155147,"name":"drain","context":{"idset":"173","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7168195,"name":"drain","context":{"idset":"174","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7180545,"name":"drain","context":{"idset":"175","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7193215,"name":"drain","context":{"idset":"176","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7206187,"name":"drain","context":{"idset":"177","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7218478,"name":"drain","context":{"idset":"178","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.723088,"name":"drain","context":{"idset":"179","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7244787,"name":"drain","context":{"idset":"180","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7259493,"name":"drain","context":{"idset":"181","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7274034,"name":"drain","context":{"idset":"182","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7287531,"name":"drain","context":{"idset":"183","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7302251,"name":"drain","context":{"idset":"184","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7316742,"name":"drain","context":{"idset":"185","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7332304,"name":"drain","context":{"idset":"186","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7346189,"name":"drain","context":{"idset":"187","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.735985,"name":"drain","context":{"idset":"188","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7373598,"name":"drain","context":{"idset":"189","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7387004,"name":"drain","context":{"idset":"190","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7400432,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7413926,"name":"drain","context":{"idset":"192","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7428672,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7443476,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.745805,"name":"drain","context":{"idset":"195","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7471035,"name":"drain","context":{"idset":"196","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7483947,"name":"drain","context":{"idset":"197","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7496886,"name":"drain","context":{"idset":"198","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.750983,"name":"drain","context":{"idset":"199","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7530868,"name":"drain","context":{"idset":"200","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7545366,"name":"drain","context":{"idset":"201","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7559168,"name":"drain","context":{"idset":"202","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.757309,"name":"drain","context":{"idset":"203","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.758682,"name":"drain","context":{"idset":"204","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7600474,"name":"drain","context":{"idset":"205","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7614391,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7627563,"name":"drain","context":{"idset":"207","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7640536,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7654107,"name":"drain","context":{"idset":"209","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7667327,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7680891,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7694106,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7707555,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.772192,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7736566,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.775089,"name":"drain","context":{"idset":"216","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7764816,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7778707,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7792428,"name":"drain","context":{"idset":"220","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7805746,"name":"drain","context":{"idset":"221","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7820771,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7835329,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7849052,"name":"drain","context":{"idset":"224","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7864044,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7878554,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7891898,"name":"drain","context":{"idset":"227","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7905607,"name":"drain","context":{"idset":"228","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.791904,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7931557,"name":"drain","context":{"idset":"230","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7944925,"name":"drain","context":{"idset":"231","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7961504,"name":"drain","context":{"idset":"232","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7975507,"name":"drain","context":{"idset":"233","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.7989218,"name":"drain","context":{"idset":"234","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8003066,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8016052,"name":"drain","context":{"idset":"236","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8029218,"name":"drain","context":{"idset":"237","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8042164,"name":"drain","context":{"idset":"238","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8055744,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8069415,"name":"drain","context":{"idset":"240","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8083713,"name":"drain","context":{"idset":"241","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8097792,"name":"drain","context":{"idset":"242","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8111877,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8125885,"name":"drain","context":{"idset":"244","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8138883,"name":"drain","context":{"idset":"245","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8151984,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8170905,"name":"drain","context":{"idset":"247","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8185089,"name":"drain","context":{"idset":"248","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8201139,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8217044,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8231516,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8245685,"name":"drain","context":{"idset":"252","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8259518,"name":"drain","context":{"idset":"277","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.827342,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8287549,"name":"drain","context":{"idset":"279","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8301356,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.831701,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8330417,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.834429,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8358951,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8373337,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8387582,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8400891,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8414903,"name":"drain","context":{"idset":"288","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8428164,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8441582,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8455575,"name":"drain","context":{"idset":"291","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8469086,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.849133,"name":"drain","context":{"idset":"293","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.851007,"name":"drain","context":{"idset":"294","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8523018,"name":"drain","context":{"idset":"295","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8537261,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8551459,"name":"drain","context":{"idset":"297","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8567636,"name":"drain","context":{"idset":"298","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8581896,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8595953,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8610218,"name":"drain","context":{"idset":"301","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8629045,"name":"drain","context":{"idset":"302","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8643484,"name":"drain","context":{"idset":"303","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8657601,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8671606,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8685517,"name":"drain","context":{"idset":"306","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8699701,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8718586,"name":"drain","context":{"idset":"308","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8732548,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8786848,"name":"drain","context":{"idset":"310","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.880105,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8815308,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8829358,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8842835,"name":"drain","context":{"idset":"314","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.885617,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8870091,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8885617,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8900826,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8914783,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8928554,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.894285,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.895633,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8969662,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8983397,"name":"drain","context":{"idset":"324","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.8997219,"name":"drain","context":{"idset":"325","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.901103,"name":"drain","context":{"idset":"326","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.902549,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9041853,"name":"drain","context":{"idset":"328","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9055474,"name":"drain","context":{"idset":"329","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9069147,"name":"drain","context":{"idset":"330","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9083986,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9097655,"name":"drain","context":{"idset":"332","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9111669,"name":"drain","context":{"idset":"333","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9126101,"name":"drain","context":{"idset":"334","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9140534,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9154623,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9169099,"name":"drain","context":{"idset":"337","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.918376,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9198081,"name":"drain","context":{"idset":"339","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9212344,"name":"drain","context":{"idset":"340","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9226229,"name":"drain","context":{"idset":"341","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.923995,"name":"drain","context":{"idset":"342","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.925386,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9267828,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9285376,"name":"drain","context":{"idset":"345","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9299049,"name":"drain","context":{"idset":"346","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9318676,"name":"drain","context":{"idset":"9452","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9333589,"name":"drain","context":{"idset":"9453","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9347408,"name":"drain","context":{"idset":"9454","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9361367,"name":"drain","context":{"idset":"9455","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9375253,"name":"drain","context":{"idset":"9456","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9389155,"name":"drain","context":{"idset":"9457","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9402893,"name":"drain","context":{"idset":"9458","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9440637,"name":"drain","context":{"idset":"9459","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9454713,"name":"drain","context":{"idset":"9460","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9469111,"name":"drain","context":{"idset":"9461","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9483495,"name":"drain","context":{"idset":"9462","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9498258,"name":"drain","context":{"idset":"9463","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9512837,"name":"drain","context":{"idset":"9464","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.952667,"name":"drain","context":{"idset":"9465","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9542286,"name":"drain","context":{"idset":"9466","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9557574,"name":"drain","context":{"idset":"9467","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9572611,"name":"drain","context":{"idset":"9468","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9588211,"name":"drain","context":{"idset":"9469","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9603012,"name":"drain","context":{"idset":"9470","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.961792,"name":"drain","context":{"idset":"9471","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9633808,"name":"drain","context":{"idset":"9473","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9648519,"name":"drain","context":{"idset":"9474","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9663823,"name":"drain","context":{"idset":"9475","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.967834,"name":"drain","context":{"idset":"9476","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9693372,"name":"drain","context":{"idset":"9477","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9707923,"name":"drain","context":{"idset":"9478","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9722536,"name":"drain","context":{"idset":"9479","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9737339,"name":"drain","context":{"idset":"9480","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9751806,"name":"drain","context":{"idset":"9481","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9766262,"name":"drain","context":{"idset":"9482","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.977998,"name":"drain","context":{"idset":"9483","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9793806,"name":"drain","context":{"idset":"9484","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.980799,"name":"drain","context":{"idset":"9485","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9822261,"name":"drain","context":{"idset":"9486","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9836555,"name":"drain","context":{"idset":"9487","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9851129,"name":"drain","context":{"idset":"9488","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9865358,"name":"drain","context":{"idset":"9489","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.988095,"name":"drain","context":{"idset":"9490","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9896531,"name":"drain","context":{"idset":"9491","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9911573,"name":"drain","context":{"idset":"9492","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9925735,"name":"drain","context":{"idset":"9493","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9939125,"name":"drain","context":{"idset":"347","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9953284,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9967277,"name":"drain","context":{"idset":"350","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9980762,"name":"drain","context":{"idset":"351","reason":"broker was unresponsive"}}
+{"timestamp":1712673192.9994493,"name":"drain","context":{"idset":"352","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0008478,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0021815,"name":"drain","context":{"idset":"354","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0037577,"name":"drain","context":{"idset":"355","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0053391,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0067348,"name":"drain","context":{"idset":"357","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0082343,"name":"drain","context":{"idset":"358","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0096993,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0111833,"name":"drain","context":{"idset":"360","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0126626,"name":"drain","context":{"idset":"361","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0141442,"name":"drain","context":{"idset":"362","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0157373,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0171738,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0185456,"name":"drain","context":{"idset":"365","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0200095,"name":"drain","context":{"idset":"366","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0214934,"name":"drain","context":{"idset":"367","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0229535,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0244167,"name":"drain","context":{"idset":"369","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0260003,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0275912,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0290751,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0304959,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0319669,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0334065,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0347855,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0362101,"name":"drain","context":{"idset":"377","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0377066,"name":"drain","context":{"idset":"378","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0391862,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0406733,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0421207,"name":"drain","context":{"idset":"381","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0436919,"name":"drain","context":{"idset":"382","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.045203,"name":"drain","context":{"idset":"383","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0466933,"name":"drain","context":{"idset":"384","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0485106,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0501266,"name":"drain","context":{"idset":"386","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0516818,"name":"drain","context":{"idset":"387","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0534892,"name":"drain","context":{"idset":"388","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0549574,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0564232,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0579655,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0594752,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0610833,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0626619,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0640988,"name":"drain","context":{"idset":"395","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0655205,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0669878,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0684063,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0698211,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0712264,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0727243,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0743098,"name":"drain","context":{"idset":"402","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0757651,"name":"drain","context":{"idset":"403","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0771766,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.078651,"name":"drain","context":{"idset":"405","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0800529,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0815113,"name":"drain","context":{"idset":"407","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0830848,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0846264,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0862253,"name":"drain","context":{"idset":"410","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0884058,"name":"drain","context":{"idset":"411","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0902548,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0918615,"name":"drain","context":{"idset":"413","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0932386,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0946939,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0962877,"name":"drain","context":{"idset":"416","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0978293,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.0993021,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1006927,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1020725,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1035259,"name":"drain","context":{"idset":"421","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1050227,"name":"drain","context":{"idset":"422","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1064906,"name":"drain","context":{"idset":"423","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1079361,"name":"drain","context":{"idset":"424","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1093452,"name":"drain","context":{"idset":"425","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.110723,"name":"drain","context":{"idset":"426","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1121702,"name":"drain","context":{"idset":"427","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1136966,"name":"drain","context":{"idset":"428","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1152399,"name":"drain","context":{"idset":"429","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.11674,"name":"drain","context":{"idset":"430","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1191962,"name":"drain","context":{"idset":"432","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1206043,"name":"drain","context":{"idset":"433","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1219699,"name":"drain","context":{"idset":"434","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1234694,"name":"drain","context":{"idset":"435","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1250105,"name":"drain","context":{"idset":"436","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.126476,"name":"drain","context":{"idset":"437","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1278689,"name":"drain","context":{"idset":"438","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1292198,"name":"drain","context":{"idset":"439","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1306126,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1319838,"name":"drain","context":{"idset":"441","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1334178,"name":"drain","context":{"idset":"442","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1348042,"name":"drain","context":{"idset":"443","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1362081,"name":"drain","context":{"idset":"444","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1376758,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1392045,"name":"drain","context":{"idset":"470","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1406324,"name":"drain","context":{"idset":"471","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1420536,"name":"drain","context":{"idset":"472","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1435361,"name":"drain","context":{"idset":"473","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1449907,"name":"drain","context":{"idset":"474","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1464574,"name":"drain","context":{"idset":"475","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.147963,"name":"drain","context":{"idset":"476","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1494746,"name":"drain","context":{"idset":"477","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1510403,"name":"drain","context":{"idset":"478","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1525509,"name":"drain","context":{"idset":"479","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1540227,"name":"drain","context":{"idset":"480","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1554127,"name":"drain","context":{"idset":"481","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1567473,"name":"drain","context":{"idset":"482","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1581721,"name":"drain","context":{"idset":"483","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1597347,"name":"drain","context":{"idset":"484","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1612859,"name":"drain","context":{"idset":"485","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.162858,"name":"drain","context":{"idset":"486","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1644056,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1659312,"name":"drain","context":{"idset":"488","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1679194,"name":"drain","context":{"idset":"489","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1694803,"name":"drain","context":{"idset":"490","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1710019,"name":"drain","context":{"idset":"491","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1725223,"name":"drain","context":{"idset":"492","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.174063,"name":"drain","context":{"idset":"493","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1755257,"name":"drain","context":{"idset":"494","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1770153,"name":"drain","context":{"idset":"495","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1784804,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1799481,"name":"drain","context":{"idset":"497","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1814675,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.18295,"name":"drain","context":{"idset":"499","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1844852,"name":"drain","context":{"idset":"500","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1860082,"name":"drain","context":{"idset":"501","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1874979,"name":"drain","context":{"idset":"502","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1890092,"name":"drain","context":{"idset":"503","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.190546,"name":"drain","context":{"idset":"504","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1920991,"name":"drain","context":{"idset":"505","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.19368,"name":"drain","context":{"idset":"506","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1951685,"name":"drain","context":{"idset":"507","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1966877,"name":"drain","context":{"idset":"508","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1982174,"name":"drain","context":{"idset":"509","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.1997387,"name":"drain","context":{"idset":"510","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.201148,"name":"drain","context":{"idset":"511","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2026041,"name":"drain","context":{"idset":"512","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2040813,"name":"drain","context":{"idset":"513","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2056677,"name":"drain","context":{"idset":"514","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2070892,"name":"drain","context":{"idset":"515","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2086623,"name":"drain","context":{"idset":"516","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2102196,"name":"drain","context":{"idset":"517","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2130396,"name":"drain","context":{"idset":"518","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2145548,"name":"drain","context":{"idset":"519","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2160378,"name":"drain","context":{"idset":"520","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2176077,"name":"drain","context":{"idset":"521","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2190447,"name":"drain","context":{"idset":"522","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2205756,"name":"drain","context":{"idset":"523","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2221618,"name":"drain","context":{"idset":"524","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2237117,"name":"drain","context":{"idset":"525","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.22526,"name":"drain","context":{"idset":"526","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.22685,"name":"drain","context":{"idset":"527","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2284341,"name":"drain","context":{"idset":"528","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2299857,"name":"drain","context":{"idset":"529","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2315323,"name":"drain","context":{"idset":"530","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2330549,"name":"drain","context":{"idset":"531","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2346392,"name":"drain","context":{"idset":"532","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2362027,"name":"drain","context":{"idset":"533","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2377594,"name":"drain","context":{"idset":"534","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2392128,"name":"drain","context":{"idset":"535","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2407227,"name":"drain","context":{"idset":"536","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2422915,"name":"drain","context":{"idset":"537","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2438424,"name":"drain","context":{"idset":"538","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2453897,"name":"drain","context":{"idset":"539","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2468588,"name":"drain","context":{"idset":"540","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2483568,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2498574,"name":"drain","context":{"idset":"566","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2513101,"name":"drain","context":{"idset":"567","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.252846,"name":"drain","context":{"idset":"568","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2544565,"name":"drain","context":{"idset":"569","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2560346,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2576144,"name":"drain","context":{"idset":"571","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2591703,"name":"drain","context":{"idset":"572","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2607384,"name":"drain","context":{"idset":"573","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2657204,"name":"drain","context":{"idset":"574","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2671323,"name":"drain","context":{"idset":"575","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.268656,"name":"drain","context":{"idset":"576","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2702169,"name":"drain","context":{"idset":"577","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2717397,"name":"drain","context":{"idset":"578","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2731328,"name":"drain","context":{"idset":"579","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2745576,"name":"drain","context":{"idset":"580","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.276123,"name":"drain","context":{"idset":"581","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2777073,"name":"drain","context":{"idset":"582","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2792411,"name":"drain","context":{"idset":"583","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2807708,"name":"drain","context":{"idset":"584","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2823508,"name":"drain","context":{"idset":"585","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2839396,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.285511,"name":"drain","context":{"idset":"587","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2870197,"name":"drain","context":{"idset":"588","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2885456,"name":"drain","context":{"idset":"629","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2900922,"name":"drain","context":{"idset":"630","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2915955,"name":"drain","context":{"idset":"631","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2931111,"name":"drain","context":{"idset":"632","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2946763,"name":"drain","context":{"idset":"633","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2962339,"name":"drain","context":{"idset":"634","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2977347,"name":"drain","context":{"idset":"635","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.2992361,"name":"drain","context":{"idset":"636","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3032,"name":"drain","context":{"idset":"815","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3046753,"name":"drain","context":{"idset":"816","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3074379,"name":"drain","context":{"idset":"827","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3089201,"name":"drain","context":{"idset":"828","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3104701,"name":"drain","context":{"idset":"829","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3119645,"name":"drain","context":{"idset":"830","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3135138,"name":"drain","context":{"idset":"831","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3151371,"name":"drain","context":{"idset":"832","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3168569,"name":"drain","context":{"idset":"833","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3184543,"name":"drain","context":{"idset":"834","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3199897,"name":"drain","context":{"idset":"841","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3217916,"name":"drain","context":{"idset":"842","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3234386,"name":"drain","context":{"idset":"871","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3250365,"name":"drain","context":{"idset":"872","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3266439,"name":"drain","context":{"idset":"876","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3282249,"name":"drain","context":{"idset":"949","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3297763,"name":"drain","context":{"idset":"950","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3313556,"name":"drain","context":{"idset":"951","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3328989,"name":"drain","context":{"idset":"952","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3344324,"name":"drain","context":{"idset":"953","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3359449,"name":"drain","context":{"idset":"954","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3374965,"name":"drain","context":{"idset":"963","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3390396,"name":"drain","context":{"idset":"964","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3405616,"name":"drain","context":{"idset":"975","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3421016,"name":"drain","context":{"idset":"976","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3437119,"name":"drain","context":{"idset":"9205","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.345479,"name":"drain","context":{"idset":"9206","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3472092,"name":"drain","context":{"idset":"9207","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3528588,"name":"drain","context":{"idset":"9208","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3544958,"name":"drain","context":{"idset":"9209","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3561661,"name":"drain","context":{"idset":"9210","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3578162,"name":"drain","context":{"idset":"9211","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3618371,"name":"drain","context":{"idset":"9212","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3634839,"name":"drain","context":{"idset":"9213","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3652215,"name":"drain","context":{"idset":"9214","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3669505,"name":"drain","context":{"idset":"9215","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3684809,"name":"drain","context":{"idset":"9216","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3700159,"name":"drain","context":{"idset":"9217","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3716679,"name":"drain","context":{"idset":"9218","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3732824,"name":"drain","context":{"idset":"9219","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3748147,"name":"drain","context":{"idset":"9220","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3765602,"name":"drain","context":{"idset":"9221","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3783162,"name":"drain","context":{"idset":"9222","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3799336,"name":"drain","context":{"idset":"9223","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3815064,"name":"drain","context":{"idset":"9224","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3830709,"name":"drain","context":{"idset":"9225","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3846686,"name":"drain","context":{"idset":"9226","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3862853,"name":"drain","context":{"idset":"9227","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3879156,"name":"drain","context":{"idset":"9228","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3895512,"name":"drain","context":{"idset":"9229","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3911881,"name":"drain","context":{"idset":"9230","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.392787,"name":"drain","context":{"idset":"9231","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3943462,"name":"drain","context":{"idset":"9232","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.395992,"name":"drain","context":{"idset":"9233","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3976717,"name":"drain","context":{"idset":"9234","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.3993142,"name":"drain","context":{"idset":"9235","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4009452,"name":"drain","context":{"idset":"9236","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4025862,"name":"drain","context":{"idset":"9237","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4042091,"name":"drain","context":{"idset":"9238","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4058383,"name":"drain","context":{"idset":"9239","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4074414,"name":"drain","context":{"idset":"9240","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4090819,"name":"drain","context":{"idset":"9241","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4109793,"name":"drain","context":{"idset":"9242","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4126043,"name":"drain","context":{"idset":"9243","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4142165,"name":"drain","context":{"idset":"9244","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4158385,"name":"drain","context":{"idset":"9245","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4174571,"name":"drain","context":{"idset":"9246","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4190249,"name":"drain","context":{"idset":"9247","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4206421,"name":"drain","context":{"idset":"9248","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4221568,"name":"drain","context":{"idset":"9249","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4237478,"name":"drain","context":{"idset":"9250","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4261255,"name":"drain","context":{"idset":"9251","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4277236,"name":"drain","context":{"idset":"9252","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4293492,"name":"drain","context":{"idset":"9253","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4309196,"name":"drain","context":{"idset":"9254","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4325292,"name":"drain","context":{"idset":"9255","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4341056,"name":"drain","context":{"idset":"9256","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4364645,"name":"drain","context":{"idset":"9257","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4380102,"name":"drain","context":{"idset":"9258","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.439564,"name":"drain","context":{"idset":"9259","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4412315,"name":"drain","context":{"idset":"9260","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4428737,"name":"drain","context":{"idset":"9261","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.44451,"name":"drain","context":{"idset":"9262","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4460716,"name":"drain","context":{"idset":"9263","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4477515,"name":"drain","context":{"idset":"9264","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4493849,"name":"drain","context":{"idset":"9265","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4509039,"name":"drain","context":{"idset":"9266","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4524548,"name":"drain","context":{"idset":"9267","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4540045,"name":"drain","context":{"idset":"9268","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4555433,"name":"drain","context":{"idset":"9269","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4571111,"name":"drain","context":{"idset":"9270","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4587467,"name":"drain","context":{"idset":"9271","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4603844,"name":"drain","context":{"idset":"9272","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4619904,"name":"drain","context":{"idset":"9273","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4637737,"name":"drain","context":{"idset":"9274","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4652536,"name":"drain","context":{"idset":"9275","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.466861,"name":"drain","context":{"idset":"9276","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4685338,"name":"drain","context":{"idset":"9277","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4700456,"name":"drain","context":{"idset":"9278","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4716129,"name":"drain","context":{"idset":"9279","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4732158,"name":"drain","context":{"idset":"9280","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4748995,"name":"drain","context":{"idset":"9281","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4766591,"name":"drain","context":{"idset":"9282","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4783902,"name":"drain","context":{"idset":"9283","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4800887,"name":"drain","context":{"idset":"9284","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4818325,"name":"drain","context":{"idset":"9285","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4835765,"name":"drain","context":{"idset":"9286","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4853687,"name":"drain","context":{"idset":"9287","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4871719,"name":"drain","context":{"idset":"9288","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4896488,"name":"drain","context":{"idset":"9289","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4913161,"name":"drain","context":{"idset":"9290","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4930923,"name":"drain","context":{"idset":"9291","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4949141,"name":"drain","context":{"idset":"9292","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4966297,"name":"drain","context":{"idset":"9293","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.4984155,"name":"drain","context":{"idset":"9294","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5002859,"name":"drain","context":{"idset":"9295","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5021253,"name":"drain","context":{"idset":"9296","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.504065,"name":"drain","context":{"idset":"9297","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5058994,"name":"drain","context":{"idset":"9298","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.507683,"name":"drain","context":{"idset":"9299","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5094447,"name":"drain","context":{"idset":"9300","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5111558,"name":"drain","context":{"idset":"9301","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5129437,"name":"drain","context":{"idset":"9302","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5146651,"name":"drain","context":{"idset":"9303","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5164304,"name":"drain","context":{"idset":"9304","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5183022,"name":"drain","context":{"idset":"9305","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5198641,"name":"drain","context":{"idset":"9306","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5215573,"name":"drain","context":{"idset":"9307","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5231562,"name":"drain","context":{"idset":"9308","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.524919,"name":"drain","context":{"idset":"9309","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5266891,"name":"drain","context":{"idset":"9310","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5284011,"name":"drain","context":{"idset":"9311","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5299668,"name":"drain","context":{"idset":"9312","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5316184,"name":"drain","context":{"idset":"9313","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5331335,"name":"drain","context":{"idset":"9314","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5346994,"name":"drain","context":{"idset":"9315","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5361769,"name":"drain","context":{"idset":"9316","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5377378,"name":"drain","context":{"idset":"9317","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5396757,"name":"drain","context":{"idset":"9318","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5412905,"name":"drain","context":{"idset":"9319","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.542937,"name":"drain","context":{"idset":"9320","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5445833,"name":"drain","context":{"idset":"9321","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5462434,"name":"drain","context":{"idset":"9322","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5477703,"name":"drain","context":{"idset":"9323","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5493841,"name":"drain","context":{"idset":"9324","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5510452,"name":"drain","context":{"idset":"9325","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5526342,"name":"drain","context":{"idset":"9326","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5542948,"name":"drain","context":{"idset":"9327","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5565345,"name":"drain","context":{"idset":"9328","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5581362,"name":"drain","context":{"idset":"9329","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5597553,"name":"drain","context":{"idset":"9330","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5614116,"name":"drain","context":{"idset":"9331","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5628843,"name":"drain","context":{"idset":"9332","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5644047,"name":"drain","context":{"idset":"9333","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5660579,"name":"drain","context":{"idset":"9334","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5676918,"name":"drain","context":{"idset":"9335","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5693908,"name":"drain","context":{"idset":"9336","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5711627,"name":"drain","context":{"idset":"9337","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5728626,"name":"drain","context":{"idset":"9338","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5745275,"name":"drain","context":{"idset":"9339","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5762067,"name":"drain","context":{"idset":"9340","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5778663,"name":"drain","context":{"idset":"9341","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5795634,"name":"drain","context":{"idset":"9342","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5812213,"name":"drain","context":{"idset":"9343","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5828905,"name":"drain","context":{"idset":"9344","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5845542,"name":"drain","context":{"idset":"9345","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5861433,"name":"drain","context":{"idset":"9346","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5877988,"name":"drain","context":{"idset":"9347","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5893795,"name":"drain","context":{"idset":"9348","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5910547,"name":"drain","context":{"idset":"9349","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5926805,"name":"drain","context":{"idset":"9350","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5945132,"name":"drain","context":{"idset":"9351","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5960908,"name":"drain","context":{"idset":"9352","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5977604,"name":"drain","context":{"idset":"9353","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.5994122,"name":"drain","context":{"idset":"9354","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6010981,"name":"drain","context":{"idset":"9355","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6027975,"name":"drain","context":{"idset":"9356","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6044588,"name":"drain","context":{"idset":"9357","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6060936,"name":"drain","context":{"idset":"9358","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6076963,"name":"drain","context":{"idset":"9359","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6092551,"name":"drain","context":{"idset":"9360","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6109309,"name":"drain","context":{"idset":"9361","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6127081,"name":"drain","context":{"idset":"9362","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6143842,"name":"drain","context":{"idset":"9363","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6159794,"name":"drain","context":{"idset":"9364","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.617625,"name":"drain","context":{"idset":"9365","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6192901,"name":"drain","context":{"idset":"9366","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6209397,"name":"drain","context":{"idset":"9367","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6226084,"name":"drain","context":{"idset":"9368","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6242561,"name":"drain","context":{"idset":"9369","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6258664,"name":"drain","context":{"idset":"9370","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6275976,"name":"drain","context":{"idset":"9371","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6292484,"name":"drain","context":{"idset":"9372","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6348464,"name":"drain","context":{"idset":"9373","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6364543,"name":"drain","context":{"idset":"9374","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6381044,"name":"drain","context":{"idset":"9375","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6396999,"name":"drain","context":{"idset":"9376","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6413205,"name":"drain","context":{"idset":"9377","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6428907,"name":"drain","context":{"idset":"9378","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6445034,"name":"drain","context":{"idset":"9379","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6460857,"name":"drain","context":{"idset":"9380","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6476591,"name":"drain","context":{"idset":"9381","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6492894,"name":"drain","context":{"idset":"9382","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6510079,"name":"drain","context":{"idset":"9383","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6539145,"name":"drain","context":{"idset":"9384","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6557081,"name":"drain","context":{"idset":"9385","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6574831,"name":"drain","context":{"idset":"9386","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6590457,"name":"drain","context":{"idset":"9387","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.663928,"name":"drain","context":{"idset":"9388","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.665581,"name":"drain","context":{"idset":"9389","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6672156,"name":"drain","context":{"idset":"9390","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6688602,"name":"drain","context":{"idset":"9391","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6704502,"name":"drain","context":{"idset":"9392","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6720645,"name":"drain","context":{"idset":"9393","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6738515,"name":"drain","context":{"idset":"9394","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6756425,"name":"drain","context":{"idset":"9395","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6773968,"name":"drain","context":{"idset":"9396","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6789296,"name":"drain","context":{"idset":"9397","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6806557,"name":"drain","context":{"idset":"9398","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6824152,"name":"drain","context":{"idset":"9399","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6841226,"name":"drain","context":{"idset":"9400","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6859243,"name":"drain","context":{"idset":"9401","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.687736,"name":"drain","context":{"idset":"9402","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6895289,"name":"drain","context":{"idset":"9403","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6912489,"name":"drain","context":{"idset":"9404","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6930616,"name":"drain","context":{"idset":"9405","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6948478,"name":"drain","context":{"idset":"9406","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6966696,"name":"drain","context":{"idset":"9407","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.6984775,"name":"drain","context":{"idset":"9408","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7004468,"name":"drain","context":{"idset":"9409","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7022209,"name":"drain","context":{"idset":"9410","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7038465,"name":"drain","context":{"idset":"9411","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7054706,"name":"drain","context":{"idset":"9412","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7070131,"name":"drain","context":{"idset":"9413","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7085869,"name":"drain","context":{"idset":"9414","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7102854,"name":"drain","context":{"idset":"9415","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.712034,"name":"drain","context":{"idset":"9416","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7138052,"name":"drain","context":{"idset":"9417","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7158575,"name":"drain","context":{"idset":"9418","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7174335,"name":"drain","context":{"idset":"9419","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7189374,"name":"drain","context":{"idset":"9420","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.720505,"name":"drain","context":{"idset":"9421","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7223139,"name":"drain","context":{"idset":"9422","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7240934,"name":"drain","context":{"idset":"9423","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7257357,"name":"drain","context":{"idset":"9424","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7273266,"name":"drain","context":{"idset":"9425","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7289023,"name":"drain","context":{"idset":"9426","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7325144,"name":"drain","context":{"idset":"9427","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7341912,"name":"drain","context":{"idset":"9428","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7358685,"name":"drain","context":{"idset":"9429","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7375495,"name":"drain","context":{"idset":"9430","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7391601,"name":"drain","context":{"idset":"9431","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7407885,"name":"drain","context":{"idset":"9432","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7424328,"name":"drain","context":{"idset":"9433","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.744118,"name":"drain","context":{"idset":"9434","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7457526,"name":"drain","context":{"idset":"9435","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7472956,"name":"drain","context":{"idset":"9436","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7488236,"name":"drain","context":{"idset":"9437","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7504377,"name":"drain","context":{"idset":"9438","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7521231,"name":"drain","context":{"idset":"9439","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7531531,"name":"drain","context":{"idset":"9440","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7547002,"name":"drain","context":{"idset":"9441","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7564409,"name":"drain","context":{"idset":"9442","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.758127,"name":"drain","context":{"idset":"9443","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7598834,"name":"drain","context":{"idset":"9444","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7616391,"name":"drain","context":{"idset":"9445","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7633553,"name":"drain","context":{"idset":"9446","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7650328,"name":"drain","context":{"idset":"9449","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7667909,"name":"drain","context":{"idset":"9450","reason":"broker was unresponsive"}}
+{"timestamp":1712673193.7683959,"name":"drain","context":{"idset":"9451","reason":"broker was unresponsive"}}
+{"timestamp":1712673503.4092155,"name":"undrain","context":{"idset":"10147-10148,10153-10154"}}
+{"timestamp":1712674590.0308008,"name":"online","context":{"idset":"795"}}
+{"timestamp":1712674938.3657725,"name":"online","context":{"idset":"796"}}
+{"timestamp":1712675754.142652,"name":"online","context":{"idset":"847"}}
+{"timestamp":1712676165.5360467,"name":"drain","context":{"idset":"10517","reason":"bad partner node --JRG","overwrite":1}}
+{"timestamp":1712677124.7561991,"name":"drain","context":{"idset":"847-848","overwrite":0}}
+{"timestamp":1712677312.1270239,"name":"drain","context":{"idset":"847-848","reason":"rebooting the node","overwrite":0}}
+{"timestamp":1712677469.0798635,"name":"drain","context":{"idset":"795-796","reason":"rebooting the nodes","overwrite":0}}
+{"timestamp":1712677857.6151648,"name":"offline","context":{"idset":"847"}}
+{"timestamp":1712678000.6382906,"name":"offline","context":{"idset":"795"}}
+{"timestamp":1712678216.1254599,"name":"offline","context":{"idset":"796"}}
+{"timestamp":1712678348.9901698,"name":"undrain","context":{"idset":"11845-11860"}}
+{"timestamp":1712678684.9730842,"name":"drain","context":{"idset":"11431","overwrite":0}}
+{"timestamp":1712678691.313581,"name":"drain","context":{"idset":"11464","overwrite":0}}
+{"timestamp":1712679592.1366844,"name":"undrain","context":{"idset":"11685"}}
+{"timestamp":1712679603.7890365,"name":"undrain","context":{"idset":"11686"}}
+{"timestamp":1712679606.3928151,"name":"undrain","context":{"idset":"11687"}}
+{"timestamp":1712679610.2754433,"name":"undrain","context":{"idset":"11688"}}
+{"timestamp":1712679613.6953149,"name":"undrain","context":{"idset":"11689"}}
+{"timestamp":1712679616.5262163,"name":"undrain","context":{"idset":"11690"}}
+{"timestamp":1712679619.43066,"name":"undrain","context":{"idset":"11691"}}
+{"timestamp":1712679625.4308405,"name":"undrain","context":{"idset":"11692"}}
+{"timestamp":1712679630.7119176,"name":"undrain","context":{"idset":"11693"}}
+{"timestamp":1712679639.1329031,"name":"undrain","context":{"idset":"11695"}}
+{"timestamp":1712679643.3880975,"name":"undrain","context":{"idset":"11696"}}
+{"timestamp":1712679673.4598687,"name":"online","context":{"idset":"10138"}}
+{"timestamp":1712679674.0938473,"name":"online","context":{"idset":"10143"}}
+{"timestamp":1712679850.6833072,"name":"undrain","context":{"idset":"11697"}}
+{"timestamp":1712679884.2676826,"name":"undrain","context":{"idset":"11698"}}
+{"timestamp":1712679888.5006177,"name":"undrain","context":{"idset":"11699"}}
+{"timestamp":1712679891.3506572,"name":"undrain","context":{"idset":"11700"}}
+{"timestamp":1712679933.3881025,"name":"undrain","context":{"idset":"10133-10137,10139-10142,10144,10146,10149-10152,10157-10164"}}
+{"timestamp":1712680181.9986107,"name":"online","context":{"idset":"11426"}}
+{"timestamp":1712680204.0492356,"name":"online","context":{"idset":"11431"}}
+{"timestamp":1712680261.5309985,"name":"online","context":{"idset":"11464"}}
+{"timestamp":1712680986.369189,"name":"online","context":{"idset":"847"}}
+{"timestamp":1712681265.0041916,"name":"online","context":{"idset":"848"}}
+{"timestamp":1712681792.2461185,"name":"online","context":{"idset":"10155-10156"}}
+{"timestamp":1712681938.8059394,"name":"undrain","context":{"idset":"795-796"}}
+{"timestamp":1712682014.1628468,"name":"online","context":{"idset":"10145"}}
+{"timestamp":1712682275.9848099,"name":"undrain","context":{"idset":"847-848"}}
+{"timestamp":1712683165.0266848,"name":"online","context":{"idset":"11694"}}
+{"timestamp":1712683674.6412404,"name":"drain","context":{"idset":"10134","reason":"broker was unresponsive"}}
+{"timestamp":1712683791.6526902,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1712683868.55037,"name":"online","context":{"idset":"795"}}
+{"timestamp":1712684017.6910372,"name":"undrain","context":{"idset":"1-60,85-120,122-216,218-252,277-347,349-430,432-444,469-540,565-588,629-636,815-816,827-834,841-842,871-872,876,949-954,963-964,975-976,9205-9446,9449-9471,9473-9534,9536-9560,9563-9749,9751-9786,9788-9789,9791-9844,9973-10132,10165-10260,10263-10302,10304-10342,10345-10354,10357-10395,10397-10468,10485-10500,10502-10516,10519-10570,10572-10613,10615-10628,10630-10633,10635-10636,10640-10641,10645-10656,10659-10701,10703-10711,10713-10717,10719-10738,10741-10868,10997-10998,11001-11020,11022-11072,11116,11119-11124,11253-11386,11389,11391-11394,11396,11570-11578,11665-11684,11727-11844,11877-11892"}}
+{"timestamp":1712684017.7078054,"name":"online","context":{"idset":"796"}}
+{"timestamp":1712684209.1663833,"name":"undrain","context":{"idset":"11861-11876"}}
+{"timestamp":1712684333.1108673,"name":"undrain","context":{"idset":"11073-11110,11113-11114,11509-11568,11579-11636,11653-11664,11701-11726"}}
+{"timestamp":1712684339.5982096,"name":"undrain","context":{"idset":"10134"}}
+{"timestamp":1712684349.4331396,"name":"undrain","context":{"idset":"253-276,9535,10614,10658,11117-11118,11569"}}
+{"timestamp":1712685126.4766591,"name":"drain","context":{"idset":"10122","overwrite":0}}
+{"timestamp":1712685311.6791396,"name":"online","context":{"idset":"11569"}}
+{"timestamp":1712685842.6411266,"name":"drain","context":{"idset":"11694","reason":"broker was unresponsive"}}
+{"timestamp":1712685904.6361566,"name":"offline","context":{"idset":"11694"}}
+{"timestamp":1712686370.2581851,"name":"undrain","context":{"idset":"10122"}}
+{"timestamp":1712686598.3547509,"name":"online","context":{"idset":"10134"}}
+{"timestamp":1712687338.8562014,"name":"drain","context":{"idset":"11509-11540","reason":"drained to perform node diags","overwrite":0}}
+{"timestamp":1712687358.4124846,"name":"online","context":{"idset":"838"}}
+{"timestamp":1712687910.6403966,"name":"drain","context":{"idset":"10795","reason":"broker was unresponsive"}}
+{"timestamp":1712687972.8112063,"name":"offline","context":{"idset":"10795"}}
+{"timestamp":1712689015.7347977,"name":"undrain","context":{"idset":"11387"}}
+{"timestamp":1712689022.0959234,"name":"undrain","context":{"idset":"11388"}}
+{"timestamp":1712689211.4218154,"name":"undrain","context":{"idset":"11390"}}
+{"timestamp":1712689216.4228041,"name":"undrain","context":{"idset":"11395"}}
+{"timestamp":1712689420.5465758,"name":"undrain","context":{"idset":"11397-11404"}}
+{"timestamp":1712689549.5106301,"name":"undrain","context":{"idset":"11405-11412"}}
+{"timestamp":1712689700.8777125,"name":"undrain","context":{"idset":"11413-11420"}}
+{"timestamp":1712689752.6292975,"name":"drain","context":{"idset":"827-828,10121-10122","overwrite":0}}
+{"timestamp":1712689762.0420313,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1712689762.1398034,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1712689768.205025,"name":"offline","context":{"idset":"827"}}
+{"timestamp":1712689768.3052936,"name":"offline","context":{"idset":"828"}}
+{"timestamp":1712689828.8059154,"name":"undrain","context":{"idset":"11421-11428"}}
+{"timestamp":1712690012.0426533,"name":"offline","context":{"idset":"838"}}
+{"timestamp":1712690028.2031574,"name":"online","context":{"idset":"838"}}
+{"timestamp":1712690045.0594604,"name":"undrain","context":{"idset":"11429-11508"}}
+{"timestamp":1712690158.0257664,"name":"online","context":{"idset":"11160"}}
+{"timestamp":1712690158.3746831,"name":"online","context":{"idset":"11126"}}
+{"timestamp":1712690158.4941401,"name":"online","context":{"idset":"11156,11159"}}
+{"timestamp":1712690158.5143743,"name":"online","context":{"idset":"11154"}}
+{"timestamp":1712690158.6717691,"name":"online","context":{"idset":"11131,11136,11140,11164,11192"}}
+{"timestamp":1712690158.7040718,"name":"online","context":{"idset":"11137"}}
+{"timestamp":1712690158.848161,"name":"online","context":{"idset":"11148"}}
+{"timestamp":1712690158.9611778,"name":"online","context":{"idset":"11128,11130,11132,11134,11176"}}
+{"timestamp":1712690159.0715296,"name":"online","context":{"idset":"11127,11153,11157,11161,11166"}}
+{"timestamp":1712690159.1012547,"name":"online","context":{"idset":"11135"}}
+{"timestamp":1712690159.2273452,"name":"online","context":{"idset":"11169"}}
+{"timestamp":1712690159.250591,"name":"online","context":{"idset":"11146,11151,11162"}}
+{"timestamp":1712690159.3560538,"name":"online","context":{"idset":"11189"}}
+{"timestamp":1712690159.3766589,"name":"online","context":{"idset":"11144"}}
+{"timestamp":1712690159.5425851,"name":"online","context":{"idset":"11129,11133,11145,11170,11173,11178"}}
+{"timestamp":1712690159.7618365,"name":"online","context":{"idset":"11149,11155,11158,11172,11180,11182,11204"}}
+{"timestamp":1712690159.8830209,"name":"online","context":{"idset":"11168,11190,11197"}}
+{"timestamp":1712690160.091311,"name":"online","context":{"idset":"11201,11206"}}
+{"timestamp":1712690160.3006902,"name":"online","context":{"idset":"11125,11138-11139,11150,11152,11167,11177,11184,11200,11205,11209"}}
+{"timestamp":1712690160.5133567,"name":"online","context":{"idset":"11142,11163,11165,11183,11188,11242,11248"}}
+{"timestamp":1712690160.6373067,"name":"online","context":{"idset":"11239"}}
+{"timestamp":1712690160.7665288,"name":"online","context":{"idset":"11141,11143,11171,11174-11175,11194,11199,11203,11228,11232-11233"}}
+{"timestamp":1712690160.8894522,"name":"online","context":{"idset":"11147,11193,11196"}}
+{"timestamp":1712690161.1077189,"name":"online","context":{"idset":"11179,11186,11191,11202,11211,11215,11219,11222,11224,11229,11245,11251"}}
+{"timestamp":1712690161.3084459,"name":"online","context":{"idset":"11185,11187,11195,11198,11212,11218,11231,11235,11240-11241,11243-11244,11246,11249"}}
+{"timestamp":1712690161.4283292,"name":"online","context":{"idset":"11181,11213,11238,11252"}}
+{"timestamp":1712690161.5494134,"name":"online","context":{"idset":"11207,11210,11214,11216-11217,11220,11225-11226,11234,11236,11250"}}
+{"timestamp":1712690161.6534164,"name":"online","context":{"idset":"11221,11223,11227,11237,11247"}}
+{"timestamp":1712690161.8512392,"name":"online","context":{"idset":"11208,11230"}}
+{"timestamp":1712691722.6389272,"name":"drain","context":{"idset":"11249","reason":"broker was unresponsive"}}
+{"timestamp":1712691784.6358633,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1712692619.8550365,"name":"online","context":{"idset":"10944"}}
+{"timestamp":1712692620.2019119,"name":"online","context":{"idset":"10943"}}
+{"timestamp":1712693394.1953471,"name":"online","context":{"idset":"837"}}
+{"timestamp":1712693698.6133246,"name":"offline","context":{"idset":"838"}}
+{"timestamp":1712693830.5809126,"name":"drain","context":{"idset":"838","reason":"cxi link speed issue","overwrite":0}}
+{"timestamp":1712693887.2493701,"name":"online","context":{"idset":"838"}}
+{"timestamp":1712693921.3229034,"name":"undrain","context":{"idset":"838"}}
+{"timestamp":1712694215.836597,"name":"online","context":{"idset":"973"}}
+{"timestamp":1712694234.6251481,"name":"online","context":{"idset":"11249"}}
+{"timestamp":1712694242.641341,"name":"drain","context":{"idset":"10100","reason":"broker was unresponsive"}}
+{"timestamp":1712694288.7930882,"name":"online","context":{"idset":"974"}}
+{"timestamp":1712694306.6372118,"name":"offline","context":{"idset":"10100"}}
+{"timestamp":1712694395.3879912,"name":"undrain","context":{"idset":"11249"}}
+{"timestamp":1712695019.519383,"name":"online","context":{"idset":"977"}}
+{"timestamp":1712695509.5686409,"name":"online","context":{"idset":"978"}}
+{"timestamp":1712695879.6981814,"name":"offline","context":{"idset":"838"}}
+{"timestamp":1712695955.4683938,"name":"drain","context":{"idset":"838","reason":"NMC swap","overwrite":0}}
+{"timestamp":1712696019.2403119,"name":"offline","context":{"idset":"837"}}
+{"timestamp":1712696055.4608181,"name":"drain","context":{"idset":"837","reason":"NMC swap","overwrite":0}}
+{"timestamp":1712696216.5419521,"name":"drain","context":{"idset":"847","reason":"broker was unresponsive"}}
+{"timestamp":1712696217.7286751,"name":"drain","context":{"idset":"848","reason":"broker was unresponsive"}}
+{"timestamp":1712696278.5438983,"name":"offline","context":{"idset":"847"}}
+{"timestamp":1712696279.3782973,"name":"offline","context":{"idset":"848"}}
+{"timestamp":1712696364.0966945,"name":"drain","context":{"idset":"629-636","reason":"New Blade Installation --JRG","overwrite":0}}
+{"timestamp":1712696392.5183744,"name":"offline","context":{"idset":"636"}}
+{"timestamp":1712696392.5639312,"name":"offline","context":{"idset":"630"}}
+{"timestamp":1712696392.5691736,"name":"offline","context":{"idset":"629"}}
+{"timestamp":1712696392.6030624,"name":"offline","context":{"idset":"631"}}
+{"timestamp":1712696392.6186485,"name":"offline","context":{"idset":"632"}}
+{"timestamp":1712696393.3455753,"name":"offline","context":{"idset":"633"}}
+{"timestamp":1712696393.3484352,"name":"offline","context":{"idset":"635"}}
+{"timestamp":1712696393.4000976,"name":"offline","context":{"idset":"634"}}
+{"timestamp":1712696847.4075968,"name":"online","context":{"idset":"757"}}
+{"timestamp":1712700686.5422509,"name":"drain","context":{"idset":"831","reason":"broker was unresponsive"}}
+{"timestamp":1712700686.5424805,"name":"drain","context":{"idset":"832","reason":"broker was unresponsive"}}
+{"timestamp":1712700686.5426056,"name":"drain","context":{"idset":"11799","reason":"broker was unresponsive"}}
+{"timestamp":1712700686.6443911,"name":"drain","context":{"idset":"11800","reason":"broker was unresponsive"}}
+{"timestamp":1712700750.5479693,"name":"offline","context":{"idset":"831"}}
+{"timestamp":1712700750.6365728,"name":"offline","context":{"idset":"832"}}
+{"timestamp":1712700752.5484662,"name":"offline","context":{"idset":"11799"}}
+{"timestamp":1712700752.6366472,"name":"offline","context":{"idset":"11800"}}
+{"timestamp":1712700944.7348719,"name":"drain","context":{"idset":"829-830,11249-11250","overwrite":0}}
+{"timestamp":1712700953.1094453,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1712700953.6328826,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1712700959.8697572,"name":"offline","context":{"idset":"829"}}
+{"timestamp":1712700959.9696975,"name":"offline","context":{"idset":"830"}}
+{"timestamp":1712701049.4153893,"name":"online","context":{"idset":"875"}}
+{"timestamp":1712701543.4290242,"name":"online","context":{"idset":"9902"}}
+{"timestamp":1712701544.1351111,"name":"online","context":{"idset":"9850,9877"}}
+{"timestamp":1712701544.3634853,"name":"online","context":{"idset":"9900,9906"}}
+{"timestamp":1712701544.4625533,"name":"online","context":{"idset":"9878,9890"}}
+{"timestamp":1712701544.6238518,"name":"online","context":{"idset":"9846,9857,9885"}}
+{"timestamp":1712701544.7532058,"name":"online","context":{"idset":"9894"}}
+{"timestamp":1712701544.9627531,"name":"online","context":{"idset":"9849,9853,9855,9858,9863,9867"}}
+{"timestamp":1712701545.0602818,"name":"online","context":{"idset":"9845,9861,9865,9914"}}
+{"timestamp":1712701545.1536086,"name":"online","context":{"idset":"9848,9947,9957"}}
+{"timestamp":1712701545.2469234,"name":"online","context":{"idset":"9872,9901"}}
+{"timestamp":1712701545.3973007,"name":"online","context":{"idset":"9932"}}
+{"timestamp":1712701545.6408896,"name":"online","context":{"idset":"9862,9893"}}
+{"timestamp":1712701545.6617851,"name":"online","context":{"idset":"9933"}}
+{"timestamp":1712701545.6829317,"name":"online","context":{"idset":"9852"}}
+{"timestamp":1712701545.8091938,"name":"online","context":{"idset":"9873,9886"}}
+{"timestamp":1712701545.9688981,"name":"online","context":{"idset":"9895"}}
+{"timestamp":1712701546.0702527,"name":"online","context":{"idset":"9930"}}
+{"timestamp":1712701546.175523,"name":"online","context":{"idset":"9854,9860"}}
+{"timestamp":1712701546.3767331,"name":"online","context":{"idset":"9899"}}
+{"timestamp":1712701546.6075268,"name":"online","context":{"idset":"9941,9948"}}
+{"timestamp":1712701546.715924,"name":"online","context":{"idset":"9897,9908"}}
+{"timestamp":1712701546.8227899,"name":"online","context":{"idset":"9880,9920,9970"}}
+{"timestamp":1712701547.0319529,"name":"online","context":{"idset":"9859,9892,9912,9928,9943,9953"}}
+{"timestamp":1712701547.2199922,"name":"online","context":{"idset":"9960,9972"}}
+{"timestamp":1712701547.3402567,"name":"online","context":{"idset":"9856,9924,9940,9946"}}
+{"timestamp":1712701547.5307839,"name":"online","context":{"idset":"9851,9868,9882,9896,9909,9918,9922,9956,9965"}}
+{"timestamp":1712701547.7189004,"name":"online","context":{"idset":"9904,9915,9959"}}
+{"timestamp":1712701547.8409352,"name":"online","context":{"idset":"9871,9875-9876"}}
+{"timestamp":1712701548.0323906,"name":"online","context":{"idset":"9847,9869,9898,9903,9916,9934,9958,9963,9967-9968"}}
+{"timestamp":1712701548.2367079,"name":"online","context":{"idset":"9874,9887,9911,9919,9935,9942,9945,9955,9966"}}
+{"timestamp":1712701548.3607163,"name":"online","context":{"idset":"9910,9926,9936,9938,9950,9952,9962"}}
+{"timestamp":1712701548.5619764,"name":"online","context":{"idset":"9864,9870,9879,9883,9888-9889,9891,9905,9907,9913,9927,9939,9961,9964,9969"}}
+{"timestamp":1712701548.7116206,"name":"online","context":{"idset":"9866,9884,9917,9925,9949,9951"}}
+{"timestamp":1712701548.9134159,"name":"online","context":{"idset":"9881,9929,9954"}}
+{"timestamp":1712701549.0370371,"name":"online","context":{"idset":"9921,9923,9931,9937,9944"}}
+{"timestamp":1712701549.3313186,"name":"online","context":{"idset":"9971"}}
+{"timestamp":1712701679.2107792,"name":"drain","context":{"idset":"878","reason":"troubshooting H/W","overwrite":0}}
+{"timestamp":1712702762.7232027,"name":"online","context":{"idset":"9535"}}
+{"timestamp":1712702994.9218936,"name":"drain","context":{"idset":"9295","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712703798.9192524,"name":"drain","context":{"idset":"10099","reason":"broker was unresponsive"}}
+{"timestamp":1712703864.6369345,"name":"offline","context":{"idset":"10099"}}
+{"timestamp":1712704250.6394441,"name":"drain","context":{"idset":"9992","reason":"broker was unresponsive"}}
+{"timestamp":1712704312.6380508,"name":"offline","context":{"idset":"9992"}}
+{"timestamp":1712704712.6371667,"name":"offline","context":{"idset":"9295"}}
+{"timestamp":1712705218.6465671,"name":"drain","context":{"idset":"9535","reason":"broker was unresponsive"}}
+{"timestamp":1712705282.6381428,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1712705553.1276381,"name":"online","context":{"idset":"10396"}}
+{"timestamp":1712705553.821378,"name":"online","context":{"idset":"10470"}}
+{"timestamp":1712705554.3363607,"name":"online","context":{"idset":"10469,10473,10480-10481"}}
+{"timestamp":1712705554.5500302,"name":"online","context":{"idset":"10471,10475-10476,10484"}}
+{"timestamp":1712705554.7707939,"name":"online","context":{"idset":"10472,10474,10477,10483"}}
+{"timestamp":1712705555.0672302,"name":"online","context":{"idset":"10478,10482"}}
+{"timestamp":1712705555.3677182,"name":"online","context":{"idset":"10479"}}
+{"timestamp":1712706078.404104,"name":"online","context":{"idset":"9295"}}
+{"timestamp":1712706119.4617889,"name":"undrain","context":{"idset":"9295"}}
+{"timestamp":1712706558.2548637,"name":"drain","context":{"idset":"9269-9332","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712706610.3852332,"name":"undrain","context":{"idset":"9269-9332"}}
+{"timestamp":1712708584.7386613,"name":"drain","context":{"idset":"9269-9332","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712709152.8111782,"name":"undrain","context":{"idset":"9269-9332"}}
+{"timestamp":1712709220.7222512,"name":"drain","context":{"idset":"9269-9332","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712709821.3537636,"name":"undrain","context":{"idset":"9269-9332"}}
+{"timestamp":1712710006.770658,"name":"drain","context":{"idset":"9269-9332","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712710144.0153348,"name":"drain","context":{"idset":"9788","reason":"bad partner node --JRG","overwrite":0}}
+{"timestamp":1712710156.6409111,"name":"drain","context":{"idset":"10251","reason":"broker was unresponsive"}}
+{"timestamp":1712710161.8355932,"name":"offline","context":{"idset":"9788"}}
+{"timestamp":1712710220.6365254,"name":"offline","context":{"idset":"10251"}}
+{"timestamp":1712710356.9176273,"name":"undrain","context":{"idset":"9269-9332"}}
+{"timestamp":1712711094.53824,"name":"drain","context":{"idset":"9789","reason":"bad partner node --JRG","overwrite":0}}
+{"timestamp":1712711113.7427719,"name":"offline","context":{"idset":"9789"}}
+{"timestamp":1712711292.6745179,"name":"drain","context":{"idset":"9269-9332","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712711833.4081793,"name":"undrain","context":{"idset":"9269-9332"}}
+{"timestamp":1712712700.6430008,"name":"drain","context":{"idset":"10401","reason":"broker was unresponsive"}}
+{"timestamp":1712712731.7304468,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1712712764.8649487,"name":"offline","context":{"idset":"10401"}}
+{"timestamp":1712713403.9947059,"name":"drain","context":{"idset":"10360","reason":"nodediag failed tapwrap","overwrite":0}}
+{"timestamp":1712713412.3814816,"name":"drain","context":{"idset":"10357","reason":"nodediag failed tapwrap","overwrite":0}}
+{"timestamp":1712714256.6386673,"name":"drain","context":{"idset":"9295","reason":"broker was unresponsive"}}
+{"timestamp":1712714320.6367924,"name":"offline","context":{"idset":"9295"}}
+{"timestamp":1712719596.4397154,"name":"undrain","context":{"idset":"10357,10360"}}
+{"timestamp":1712759587.8826554,"name":"drain","context":{"idset":"10191","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5207119,"name":"drain","context":{"idset":"10192","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.524488,"name":"drain","context":{"idset":"10193","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.525614,"name":"drain","context":{"idset":"10194","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5329671,"name":"drain","context":{"idset":"10195","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5330544,"name":"drain","context":{"idset":"10196","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5370257,"name":"drain","context":{"idset":"10197","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5408795,"name":"drain","context":{"idset":"10198","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5409653,"name":"drain","context":{"idset":"10199","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5453472,"name":"drain","context":{"idset":"10200","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5493572,"name":"drain","context":{"idset":"10201","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5494432,"name":"drain","context":{"idset":"10202","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5530808,"name":"drain","context":{"idset":"10203","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.553169,"name":"drain","context":{"idset":"10204","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5566199,"name":"drain","context":{"idset":"10205","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5651779,"name":"drain","context":{"idset":"10206","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5652866,"name":"drain","context":{"idset":"10207","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5687048,"name":"drain","context":{"idset":"10208","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.568927,"name":"drain","context":{"idset":"10209","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5723712,"name":"drain","context":{"idset":"10210","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5764072,"name":"drain","context":{"idset":"10211","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5767405,"name":"drain","context":{"idset":"10212","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5816553,"name":"drain","context":{"idset":"10213","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5853043,"name":"drain","context":{"idset":"10214","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5855868,"name":"drain","context":{"idset":"10215","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5891681,"name":"drain","context":{"idset":"10216","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5894332,"name":"drain","context":{"idset":"10217","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5931337,"name":"drain","context":{"idset":"10218","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5969877,"name":"drain","context":{"idset":"10219","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.5973172,"name":"drain","context":{"idset":"10220","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.613472,"name":"drain","context":{"idset":"10221","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.617523,"name":"drain","context":{"idset":"10222","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6178267,"name":"drain","context":{"idset":"10223","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6221282,"name":"drain","context":{"idset":"10224","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6223979,"name":"drain","context":{"idset":"10225","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6265631,"name":"drain","context":{"idset":"10226","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6307979,"name":"drain","context":{"idset":"10227","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6322062,"name":"drain","context":{"idset":"10228","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6367564,"name":"drain","context":{"idset":"10229","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6370239,"name":"drain","context":{"idset":"10230","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6430783,"name":"drain","context":{"idset":"10231","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6472411,"name":"drain","context":{"idset":"10232","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6475379,"name":"drain","context":{"idset":"10233","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6721225,"name":"drain","context":{"idset":"10234","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6818414,"name":"drain","context":{"idset":"10235","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6886294,"name":"drain","context":{"idset":"10236","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6928706,"name":"drain","context":{"idset":"10237","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.6940141,"name":"drain","context":{"idset":"10238","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7009754,"name":"drain","context":{"idset":"10239","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7051897,"name":"drain","context":{"idset":"10240","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7055609,"name":"drain","context":{"idset":"10241","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7091672,"name":"drain","context":{"idset":"10242","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7094936,"name":"drain","context":{"idset":"10243","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7146709,"name":"drain","context":{"idset":"10244","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7187068,"name":"drain","context":{"idset":"10245","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.719034,"name":"drain","context":{"idset":"10246","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7193615,"name":"drain","context":{"idset":"10247","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7231214,"name":"drain","context":{"idset":"10248","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7275474,"name":"drain","context":{"idset":"10249","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7278504,"name":"drain","context":{"idset":"10250","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7321105,"name":"drain","context":{"idset":"10252","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7324491,"name":"drain","context":{"idset":"10253","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7327454,"name":"drain","context":{"idset":"10254","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7330747,"name":"drain","context":{"idset":"10255","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7333732,"name":"drain","context":{"idset":"10256","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7336726,"name":"drain","context":{"idset":"10257","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7339718,"name":"drain","context":{"idset":"10258","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7343123,"name":"drain","context":{"idset":"10259","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7349064,"name":"drain","context":{"idset":"10260","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.735188,"name":"drain","context":{"idset":"10263","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7354927,"name":"drain","context":{"idset":"10264","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7358007,"name":"drain","context":{"idset":"10265","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7360792,"name":"drain","context":{"idset":"10266","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7363381,"name":"drain","context":{"idset":"10267","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7366631,"name":"drain","context":{"idset":"10268","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7369492,"name":"drain","context":{"idset":"10269","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7479627,"name":"drain","context":{"idset":"10270","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7783222,"name":"drain","context":{"idset":"10271","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.7928207,"name":"drain","context":{"idset":"10272","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.806725,"name":"drain","context":{"idset":"10273","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8223398,"name":"drain","context":{"idset":"10274","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8371627,"name":"drain","context":{"idset":"10275","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8476532,"name":"drain","context":{"idset":"10276","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8588185,"name":"drain","context":{"idset":"10277","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8694305,"name":"drain","context":{"idset":"10278","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8804567,"name":"drain","context":{"idset":"10279","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8805707,"name":"drain","context":{"idset":"10280","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8806691,"name":"drain","context":{"idset":"10281","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8807766,"name":"drain","context":{"idset":"10282","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8808835,"name":"drain","context":{"idset":"10283","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8809896,"name":"drain","context":{"idset":"10284","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8810966,"name":"drain","context":{"idset":"10285","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8812077,"name":"drain","context":{"idset":"10286","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8813379,"name":"drain","context":{"idset":"10287","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8814528,"name":"drain","context":{"idset":"10288","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8815668,"name":"drain","context":{"idset":"10289","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8816795,"name":"drain","context":{"idset":"10290","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8817928,"name":"drain","context":{"idset":"10291","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8818989,"name":"drain","context":{"idset":"10292","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8820119,"name":"drain","context":{"idset":"10293","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8821228,"name":"drain","context":{"idset":"10294","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8822362,"name":"drain","context":{"idset":"10295","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.882359,"name":"drain","context":{"idset":"10296","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8824761,"name":"drain","context":{"idset":"10297","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8825891,"name":"drain","context":{"idset":"10298","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8827016,"name":"drain","context":{"idset":"10299","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8828177,"name":"drain","context":{"idset":"10300","reason":"broker was unresponsive"}}
+{"timestamp":1712759589.8829348,"name":"drain","context":{"idset":"10301","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7378001,"name":"drain","context":{"idset":"10302","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7379339,"name":"drain","context":{"idset":"10304","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7380397,"name":"drain","context":{"idset":"10305","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7381473,"name":"drain","context":{"idset":"10306","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7382863,"name":"drain","context":{"idset":"10307","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7383926,"name":"drain","context":{"idset":"10308","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7384913,"name":"drain","context":{"idset":"10309","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7385886,"name":"drain","context":{"idset":"10310","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7386878,"name":"drain","context":{"idset":"10311","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7387881,"name":"drain","context":{"idset":"10312","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7388906,"name":"drain","context":{"idset":"10313","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7389889,"name":"drain","context":{"idset":"10314","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7390893,"name":"drain","context":{"idset":"10315","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7391891,"name":"drain","context":{"idset":"10316","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7393036,"name":"drain","context":{"idset":"10317","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.739408,"name":"drain","context":{"idset":"10318","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7395165,"name":"drain","context":{"idset":"10319","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7396221,"name":"drain","context":{"idset":"10320","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7397285,"name":"drain","context":{"idset":"10321","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7398362,"name":"drain","context":{"idset":"10322","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.739944,"name":"drain","context":{"idset":"10323","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.740052,"name":"drain","context":{"idset":"10324","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7401605,"name":"drain","context":{"idset":"10325","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7402849,"name":"drain","context":{"idset":"10326","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7403972,"name":"drain","context":{"idset":"10327","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7405055,"name":"drain","context":{"idset":"10328","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7406151,"name":"drain","context":{"idset":"10329","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7407339,"name":"drain","context":{"idset":"10330","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7408412,"name":"drain","context":{"idset":"10331","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7409475,"name":"drain","context":{"idset":"10332","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7410557,"name":"drain","context":{"idset":"10333","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7411644,"name":"drain","context":{"idset":"10334","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7412889,"name":"drain","context":{"idset":"10335","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7413998,"name":"drain","context":{"idset":"10336","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7415128,"name":"drain","context":{"idset":"10337","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.741626,"name":"drain","context":{"idset":"10338","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7417402,"name":"drain","context":{"idset":"10339","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7418528,"name":"drain","context":{"idset":"10340","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7419682,"name":"drain","context":{"idset":"10341","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7420826,"name":"drain","context":{"idset":"10342","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7421966,"name":"drain","context":{"idset":"10345","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7423279,"name":"drain","context":{"idset":"10346","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7424471,"name":"drain","context":{"idset":"10347","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7425625,"name":"drain","context":{"idset":"10348","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7426786,"name":"drain","context":{"idset":"10349","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7427933,"name":"drain","context":{"idset":"10350","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7429085,"name":"drain","context":{"idset":"10351","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7430208,"name":"drain","context":{"idset":"10352","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7431381,"name":"drain","context":{"idset":"10353","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7432613,"name":"drain","context":{"idset":"10354","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7433891,"name":"drain","context":{"idset":"10357","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7435112,"name":"drain","context":{"idset":"10358","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7436323,"name":"drain","context":{"idset":"10359","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.743753,"name":"drain","context":{"idset":"10360","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7438722,"name":"drain","context":{"idset":"10361","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7439878,"name":"drain","context":{"idset":"10362","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7441072,"name":"drain","context":{"idset":"10363","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7442274,"name":"drain","context":{"idset":"10364","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7443619,"name":"drain","context":{"idset":"10365","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7490962,"name":"drain","context":{"idset":"10366","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7492883,"name":"drain","context":{"idset":"10367","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7494414,"name":"drain","context":{"idset":"10368","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7495909,"name":"drain","context":{"idset":"10369","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7497363,"name":"drain","context":{"idset":"10370","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7498815,"name":"drain","context":{"idset":"10371","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7500286,"name":"drain","context":{"idset":"10372","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7501767,"name":"drain","context":{"idset":"10373","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.750351,"name":"drain","context":{"idset":"10374","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7504988,"name":"drain","context":{"idset":"10375","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7506478,"name":"drain","context":{"idset":"10376","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7507961,"name":"drain","context":{"idset":"10377","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7509463,"name":"drain","context":{"idset":"10378","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7510972,"name":"drain","context":{"idset":"10379","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7512503,"name":"drain","context":{"idset":"10380","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7514205,"name":"drain","context":{"idset":"10381","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7515712,"name":"drain","context":{"idset":"10382","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7517242,"name":"drain","context":{"idset":"10383","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7518821,"name":"drain","context":{"idset":"10384","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.752039,"name":"drain","context":{"idset":"10385","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7521946,"name":"drain","context":{"idset":"10386","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7523658,"name":"drain","context":{"idset":"10387","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7525244,"name":"drain","context":{"idset":"10388","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7526844,"name":"drain","context":{"idset":"10389","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7528422,"name":"drain","context":{"idset":"10390","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7530015,"name":"drain","context":{"idset":"10391","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.753161,"name":"drain","context":{"idset":"10392","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7533307,"name":"drain","context":{"idset":"10393","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7534881,"name":"drain","context":{"idset":"10394","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7536449,"name":"drain","context":{"idset":"10395","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7538013,"name":"drain","context":{"idset":"10396","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7539597,"name":"drain","context":{"idset":"10397","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.754122,"name":"drain","context":{"idset":"10398","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7542963,"name":"drain","context":{"idset":"10399","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7544591,"name":"drain","context":{"idset":"10400","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7546189,"name":"drain","context":{"idset":"10402","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.754781,"name":"drain","context":{"idset":"10403","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7549422,"name":"drain","context":{"idset":"10404","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7551064,"name":"drain","context":{"idset":"10405","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7583058,"name":"drain","context":{"idset":"10406","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7584889,"name":"drain","context":{"idset":"10407","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.758651,"name":"drain","context":{"idset":"10408","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7588165,"name":"drain","context":{"idset":"10409","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7589772,"name":"drain","context":{"idset":"10410","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7591403,"name":"drain","context":{"idset":"10411","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7593176,"name":"drain","context":{"idset":"10412","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7594824,"name":"drain","context":{"idset":"10413","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7596462,"name":"drain","context":{"idset":"10414","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7598085,"name":"drain","context":{"idset":"10415","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.759974,"name":"drain","context":{"idset":"10416","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.760139,"name":"drain","context":{"idset":"10417","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7603202,"name":"drain","context":{"idset":"10418","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7604873,"name":"drain","context":{"idset":"10419","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7606523,"name":"drain","context":{"idset":"10420","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7608199,"name":"drain","context":{"idset":"10421","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7609899,"name":"drain","context":{"idset":"10422","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7611601,"name":"drain","context":{"idset":"10423","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7613435,"name":"drain","context":{"idset":"10424","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7615128,"name":"drain","context":{"idset":"10425","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7616799,"name":"drain","context":{"idset":"10426","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7618511,"name":"drain","context":{"idset":"10427","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7620206,"name":"drain","context":{"idset":"10428","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7621925,"name":"drain","context":{"idset":"10429","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7623818,"name":"drain","context":{"idset":"10430","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7625539,"name":"drain","context":{"idset":"10431","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7627265,"name":"drain","context":{"idset":"10432","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7629004,"name":"drain","context":{"idset":"10433","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7630732,"name":"drain","context":{"idset":"10434","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7632458,"name":"drain","context":{"idset":"10435","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7634325,"name":"drain","context":{"idset":"10436","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7636077,"name":"drain","context":{"idset":"10437","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.763782,"name":"drain","context":{"idset":"10438","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7639592,"name":"drain","context":{"idset":"10439","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7641349,"name":"drain","context":{"idset":"10440","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7643256,"name":"drain","context":{"idset":"10441","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7645028,"name":"drain","context":{"idset":"10442","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7646718,"name":"drain","context":{"idset":"10443","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7648513,"name":"drain","context":{"idset":"10444","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7650299,"name":"drain","context":{"idset":"10445","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7652082,"name":"drain","context":{"idset":"10446","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7654057,"name":"drain","context":{"idset":"10447","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7655845,"name":"drain","context":{"idset":"10448","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7657642,"name":"drain","context":{"idset":"10449","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7659433,"name":"drain","context":{"idset":"10450","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.766124,"name":"drain","context":{"idset":"10451","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7663233,"name":"drain","context":{"idset":"10452","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7665093,"name":"drain","context":{"idset":"10453","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7666919,"name":"drain","context":{"idset":"10454","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7668769,"name":"drain","context":{"idset":"10455","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7670598,"name":"drain","context":{"idset":"10456","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7672451,"name":"drain","context":{"idset":"10457","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7674503,"name":"drain","context":{"idset":"10458","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7676363,"name":"drain","context":{"idset":"10459","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7678263,"name":"drain","context":{"idset":"10460","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.768014,"name":"drain","context":{"idset":"10461","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7682021,"name":"drain","context":{"idset":"10462","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7684062,"name":"drain","context":{"idset":"10463","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7685955,"name":"drain","context":{"idset":"10464","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7687862,"name":"drain","context":{"idset":"10465","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7689776,"name":"drain","context":{"idset":"10466","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7691708,"name":"drain","context":{"idset":"10467","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7693729,"name":"drain","context":{"idset":"10468","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7695413,"name":"drain","context":{"idset":"10469","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7697144,"name":"drain","context":{"idset":"10470","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.769897,"name":"drain","context":{"idset":"10471","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.770083,"name":"drain","context":{"idset":"10472","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7702599,"name":"drain","context":{"idset":"10473","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7705054,"name":"drain","context":{"idset":"10474","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7707045,"name":"drain","context":{"idset":"10475","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7709072,"name":"drain","context":{"idset":"10476","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7711053,"name":"drain","context":{"idset":"10477","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7713201,"name":"drain","context":{"idset":"10478","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7715197,"name":"drain","context":{"idset":"10479","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7717216,"name":"drain","context":{"idset":"10480","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7719219,"name":"drain","context":{"idset":"10481","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7721207,"name":"drain","context":{"idset":"10482","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7723339,"name":"drain","context":{"idset":"10483","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.772536,"name":"drain","context":{"idset":"10484","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7727416,"name":"drain","context":{"idset":"10485","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7729435,"name":"drain","context":{"idset":"10486","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7731471,"name":"drain","context":{"idset":"10487","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7733638,"name":"drain","context":{"idset":"10488","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7735667,"name":"drain","context":{"idset":"10489","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7737689,"name":"drain","context":{"idset":"10490","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7739747,"name":"drain","context":{"idset":"10491","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7741802,"name":"drain","context":{"idset":"10492","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7744019,"name":"drain","context":{"idset":"10493","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7746086,"name":"drain","context":{"idset":"10494","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7748177,"name":"drain","context":{"idset":"10495","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7750235,"name":"drain","context":{"idset":"10496","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7752304,"name":"drain","context":{"idset":"10497","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7754562,"name":"drain","context":{"idset":"10498","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7756698,"name":"drain","context":{"idset":"10499","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7758775,"name":"drain","context":{"idset":"10500","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7760975,"name":"drain","context":{"idset":"10502","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7763231,"name":"drain","context":{"idset":"10503","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7765336,"name":"drain","context":{"idset":"10504","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7767456,"name":"drain","context":{"idset":"10505","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7769597,"name":"drain","context":{"idset":"10506","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7771754,"name":"drain","context":{"idset":"10507","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7774477,"name":"drain","context":{"idset":"10508","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.777663,"name":"drain","context":{"idset":"10509","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7778735,"name":"drain","context":{"idset":"10510","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7780859,"name":"drain","context":{"idset":"10511","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7783127,"name":"drain","context":{"idset":"10512","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7785256,"name":"drain","context":{"idset":"10513","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.778738,"name":"drain","context":{"idset":"10514","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7789509,"name":"drain","context":{"idset":"10515","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7791638,"name":"drain","context":{"idset":"10516","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7793932,"name":"drain","context":{"idset":"10519","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.779608,"name":"drain","context":{"idset":"10520","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7798221,"name":"drain","context":{"idset":"10521","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7800398,"name":"drain","context":{"idset":"10522","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.780257,"name":"drain","context":{"idset":"10523","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7804906,"name":"drain","context":{"idset":"10524","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7807088,"name":"drain","context":{"idset":"10525","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7809286,"name":"drain","context":{"idset":"10526","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7811475,"name":"drain","context":{"idset":"10527","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7813835,"name":"drain","context":{"idset":"10528","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7816038,"name":"drain","context":{"idset":"10529","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7818277,"name":"drain","context":{"idset":"10530","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7820501,"name":"drain","context":{"idset":"10531","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.782289,"name":"drain","context":{"idset":"10532","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7825131,"name":"drain","context":{"idset":"10533","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7827337,"name":"drain","context":{"idset":"10534","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7829542,"name":"drain","context":{"idset":"10535","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7831752,"name":"drain","context":{"idset":"10536","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7834148,"name":"drain","context":{"idset":"10537","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7836411,"name":"drain","context":{"idset":"10538","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7838635,"name":"drain","context":{"idset":"10539","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7840877,"name":"drain","context":{"idset":"10540","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7843301,"name":"drain","context":{"idset":"10541","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7845542,"name":"drain","context":{"idset":"10542","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7847791,"name":"drain","context":{"idset":"10543","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7850022,"name":"drain","context":{"idset":"10544","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.785229,"name":"drain","context":{"idset":"10545","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7854724,"name":"drain","context":{"idset":"10546","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7857018,"name":"drain","context":{"idset":"10547","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7859352,"name":"drain","context":{"idset":"10548","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7861657,"name":"drain","context":{"idset":"10549","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7864134,"name":"drain","context":{"idset":"10550","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7866449,"name":"drain","context":{"idset":"10551","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7868776,"name":"drain","context":{"idset":"10552","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7871127,"name":"drain","context":{"idset":"10553","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7873573,"name":"drain","context":{"idset":"10554","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7875876,"name":"drain","context":{"idset":"10555","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7877889,"name":"drain","context":{"idset":"10556","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7880242,"name":"drain","context":{"idset":"10557","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7882583,"name":"drain","context":{"idset":"10558","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7885039,"name":"drain","context":{"idset":"10559","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7887383,"name":"drain","context":{"idset":"10560","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7889719,"name":"drain","context":{"idset":"10561","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7892067,"name":"drain","context":{"idset":"10562","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7894549,"name":"drain","context":{"idset":"10563","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7896893,"name":"drain","context":{"idset":"10564","reason":"broker was unresponsive"}}
+{"timestamp":1712759590.7899251,"name":"drain","context":{"idset":"10565","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6170011,"name":"drain","context":{"idset":"10566","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6172853,"name":"drain","context":{"idset":"10567","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6175139,"name":"drain","context":{"idset":"10568","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.617758,"name":"drain","context":{"idset":"10569","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6179988,"name":"drain","context":{"idset":"10570","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.618242,"name":"drain","context":{"idset":"10572","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6185071,"name":"drain","context":{"idset":"10573","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6187572,"name":"drain","context":{"idset":"10574","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6190104,"name":"drain","context":{"idset":"10575","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6192541,"name":"drain","context":{"idset":"10576","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6195188,"name":"drain","context":{"idset":"10577","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6197562,"name":"drain","context":{"idset":"10578","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6200006,"name":"drain","context":{"idset":"10579","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6202395,"name":"drain","context":{"idset":"10580","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6204941,"name":"drain","context":{"idset":"10581","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6207414,"name":"drain","context":{"idset":"10582","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6209915,"name":"drain","context":{"idset":"10583","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6212404,"name":"drain","context":{"idset":"10584","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6215036,"name":"drain","context":{"idset":"10585","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6217513,"name":"drain","context":{"idset":"10586","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.622005,"name":"drain","context":{"idset":"10587","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6222384,"name":"drain","context":{"idset":"10588","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6224802,"name":"drain","context":{"idset":"10589","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.622689,"name":"drain","context":{"idset":"10590","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6229105,"name":"drain","context":{"idset":"10591","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6231389,"name":"drain","context":{"idset":"10592","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6233966,"name":"drain","context":{"idset":"10593","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6236444,"name":"drain","context":{"idset":"10594","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6238918,"name":"drain","context":{"idset":"10595","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6241348,"name":"drain","context":{"idset":"10596","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6243804,"name":"drain","context":{"idset":"10597","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.624608,"name":"drain","context":{"idset":"10598","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6248357,"name":"drain","context":{"idset":"10599","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6250653,"name":"drain","context":{"idset":"10600","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6253107,"name":"drain","context":{"idset":"10601","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6255419,"name":"drain","context":{"idset":"10602","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6257734,"name":"drain","context":{"idset":"10603","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6260028,"name":"drain","context":{"idset":"10604","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6262333,"name":"drain","context":{"idset":"10605","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6264822,"name":"drain","context":{"idset":"10606","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6267161,"name":"drain","context":{"idset":"10607","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6269495,"name":"drain","context":{"idset":"10608","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6271906,"name":"drain","context":{"idset":"10609","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6274619,"name":"drain","context":{"idset":"10610","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.627794,"name":"drain","context":{"idset":"10611","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6280475,"name":"drain","context":{"idset":"10612","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6283109,"name":"drain","context":{"idset":"10613","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.628556,"name":"drain","context":{"idset":"10615","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6288161,"name":"drain","context":{"idset":"10616","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6290357,"name":"drain","context":{"idset":"10617","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6293092,"name":"drain","context":{"idset":"10618","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6295762,"name":"drain","context":{"idset":"10619","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6298468,"name":"drain","context":{"idset":"10620","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6301177,"name":"drain","context":{"idset":"10621","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6304042,"name":"drain","context":{"idset":"10622","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6306784,"name":"drain","context":{"idset":"10623","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6309485,"name":"drain","context":{"idset":"10624","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6312165,"name":"drain","context":{"idset":"10625","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6315117,"name":"drain","context":{"idset":"10626","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6317878,"name":"drain","context":{"idset":"10627","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6320624,"name":"drain","context":{"idset":"10628","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.632349,"name":"drain","context":{"idset":"10630","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6326239,"name":"drain","context":{"idset":"10631","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6328955,"name":"drain","context":{"idset":"10632","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6331742,"name":"drain","context":{"idset":"10633","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6334639,"name":"drain","context":{"idset":"10635","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6337435,"name":"drain","context":{"idset":"10636","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.634016,"name":"drain","context":{"idset":"10640","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.63431,"name":"drain","context":{"idset":"10641","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6345878,"name":"drain","context":{"idset":"10645","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6348662,"name":"drain","context":{"idset":"10646","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6351428,"name":"drain","context":{"idset":"10647","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6354375,"name":"drain","context":{"idset":"10648","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6357148,"name":"drain","context":{"idset":"10649","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6359866,"name":"drain","context":{"idset":"10650","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6362877,"name":"drain","context":{"idset":"10651","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6365671,"name":"drain","context":{"idset":"10652","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6368427,"name":"drain","context":{"idset":"10653","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.637121,"name":"drain","context":{"idset":"10654","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6373854,"name":"drain","context":{"idset":"10655","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6376398,"name":"drain","context":{"idset":"10656","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6378894,"name":"drain","context":{"idset":"10659","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6381364,"name":"drain","context":{"idset":"10660","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6383367,"name":"drain","context":{"idset":"10661","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6385076,"name":"drain","context":{"idset":"10662","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6386616,"name":"drain","context":{"idset":"10663","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6388144,"name":"drain","context":{"idset":"10664","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6389647,"name":"drain","context":{"idset":"10665","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6391158,"name":"drain","context":{"idset":"10666","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6392887,"name":"drain","context":{"idset":"10667","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6394405,"name":"drain","context":{"idset":"10668","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6395926,"name":"drain","context":{"idset":"10669","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6397476,"name":"drain","context":{"idset":"10670","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6399167,"name":"drain","context":{"idset":"10671","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6400697,"name":"drain","context":{"idset":"10672","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6402235,"name":"drain","context":{"idset":"10673","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6403952,"name":"drain","context":{"idset":"10674","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6406384,"name":"drain","context":{"idset":"10675","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6409173,"name":"drain","context":{"idset":"10676","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.641191,"name":"drain","context":{"idset":"10677","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6414649,"name":"drain","context":{"idset":"10678","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6417365,"name":"drain","context":{"idset":"10679","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6420133,"name":"drain","context":{"idset":"10680","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6423123,"name":"drain","context":{"idset":"10681","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6426125,"name":"drain","context":{"idset":"10682","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.642895,"name":"drain","context":{"idset":"10683","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6431749,"name":"drain","context":{"idset":"10684","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6434639,"name":"drain","context":{"idset":"10685","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6437461,"name":"drain","context":{"idset":"10686","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6440251,"name":"drain","context":{"idset":"10687","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6443186,"name":"drain","context":{"idset":"10688","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6445732,"name":"drain","context":{"idset":"10689","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6448247,"name":"drain","context":{"idset":"10690","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6450729,"name":"drain","context":{"idset":"10691","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6453378,"name":"drain","context":{"idset":"10692","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6455872,"name":"drain","context":{"idset":"10693","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6458454,"name":"drain","context":{"idset":"10694","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6461017,"name":"drain","context":{"idset":"10695","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6463859,"name":"drain","context":{"idset":"10696","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6466644,"name":"drain","context":{"idset":"10697","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6469276,"name":"drain","context":{"idset":"10698","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6471934,"name":"drain","context":{"idset":"10699","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6475604,"name":"drain","context":{"idset":"10700","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6478255,"name":"drain","context":{"idset":"10701","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6480868,"name":"drain","context":{"idset":"10703","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6483743,"name":"drain","context":{"idset":"10704","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.648664,"name":"drain","context":{"idset":"10705","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6489458,"name":"drain","context":{"idset":"10706","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6492264,"name":"drain","context":{"idset":"10707","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6495278,"name":"drain","context":{"idset":"10708","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6498148,"name":"drain","context":{"idset":"10709","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6501009,"name":"drain","context":{"idset":"10710","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6504071,"name":"drain","context":{"idset":"10711","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6506953,"name":"drain","context":{"idset":"10713","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6509717,"name":"drain","context":{"idset":"10714","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6512384,"name":"drain","context":{"idset":"10715","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6515265,"name":"drain","context":{"idset":"10716","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6517832,"name":"drain","context":{"idset":"10717","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6520615,"name":"drain","context":{"idset":"10719","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6523683,"name":"drain","context":{"idset":"10720","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6526592,"name":"drain","context":{"idset":"10721","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6529498,"name":"drain","context":{"idset":"10722","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6532412,"name":"drain","context":{"idset":"10723","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6535475,"name":"drain","context":{"idset":"10724","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6538393,"name":"drain","context":{"idset":"10725","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6541278,"name":"drain","context":{"idset":"10726","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6544297,"name":"drain","context":{"idset":"10727","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6547189,"name":"drain","context":{"idset":"10728","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6550212,"name":"drain","context":{"idset":"10729","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6553154,"name":"drain","context":{"idset":"10730","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6556063,"name":"drain","context":{"idset":"10731","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6558969,"name":"drain","context":{"idset":"10732","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6561801,"name":"drain","context":{"idset":"10733","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6564729,"name":"drain","context":{"idset":"10734","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6567545,"name":"drain","context":{"idset":"10735","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6570449,"name":"drain","context":{"idset":"10736","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6573379,"name":"drain","context":{"idset":"10737","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6576056,"name":"drain","context":{"idset":"10738","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6581228,"name":"drain","context":{"idset":"10741","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6584115,"name":"drain","context":{"idset":"10742","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6586819,"name":"drain","context":{"idset":"10743","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6589804,"name":"drain","context":{"idset":"10744","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6592407,"name":"drain","context":{"idset":"10745","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6595368,"name":"drain","context":{"idset":"10746","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6598179,"name":"drain","context":{"idset":"10747","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6600904,"name":"drain","context":{"idset":"10748","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6603761,"name":"drain","context":{"idset":"10749","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.660646,"name":"drain","context":{"idset":"10750","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6609111,"name":"drain","context":{"idset":"10751","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6611888,"name":"drain","context":{"idset":"10752","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6614745,"name":"drain","context":{"idset":"10753","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.661747,"name":"drain","context":{"idset":"10754","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6620319,"name":"drain","context":{"idset":"10755","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6623178,"name":"drain","context":{"idset":"10756","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6625981,"name":"drain","context":{"idset":"10757","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6628823,"name":"drain","context":{"idset":"10758","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.663167,"name":"drain","context":{"idset":"10759","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6634698,"name":"drain","context":{"idset":"10760","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6637547,"name":"drain","context":{"idset":"10761","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6640406,"name":"drain","context":{"idset":"10762","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6643434,"name":"drain","context":{"idset":"10763","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.664628,"name":"drain","context":{"idset":"10764","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6649413,"name":"drain","context":{"idset":"10765","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6652467,"name":"drain","context":{"idset":"10766","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6655667,"name":"drain","context":{"idset":"10767","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.665889,"name":"drain","context":{"idset":"10768","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6662078,"name":"drain","context":{"idset":"10769","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.666523,"name":"drain","context":{"idset":"10770","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6668515,"name":"drain","context":{"idset":"10771","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.667187,"name":"drain","context":{"idset":"10772","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6675327,"name":"drain","context":{"idset":"10773","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6678648,"name":"drain","context":{"idset":"10774","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6682031,"name":"drain","context":{"idset":"10775","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6685596,"name":"drain","context":{"idset":"10776","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6688941,"name":"drain","context":{"idset":"10777","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6692324,"name":"drain","context":{"idset":"10778","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6695864,"name":"drain","context":{"idset":"10779","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6699219,"name":"drain","context":{"idset":"10780","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6702597,"name":"drain","context":{"idset":"10781","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6706121,"name":"drain","context":{"idset":"10782","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6709526,"name":"drain","context":{"idset":"10783","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6713066,"name":"drain","context":{"idset":"10784","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6716402,"name":"drain","context":{"idset":"10785","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6719809,"name":"drain","context":{"idset":"10786","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6723325,"name":"drain","context":{"idset":"10787","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6726775,"name":"drain","context":{"idset":"10788","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6730177,"name":"drain","context":{"idset":"10789","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6733725,"name":"drain","context":{"idset":"10790","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6737139,"name":"drain","context":{"idset":"10791","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6740541,"name":"drain","context":{"idset":"10792","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6744089,"name":"drain","context":{"idset":"10793","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.674726,"name":"drain","context":{"idset":"10794","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6750274,"name":"drain","context":{"idset":"10796","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6753371,"name":"drain","context":{"idset":"10797","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6756306,"name":"drain","context":{"idset":"10798","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6759179,"name":"drain","context":{"idset":"10799","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6762147,"name":"drain","context":{"idset":"10800","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.676537,"name":"drain","context":{"idset":"10801","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6768298,"name":"drain","context":{"idset":"10802","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6771181,"name":"drain","context":{"idset":"10803","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6774189,"name":"drain","context":{"idset":"10804","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6777101,"name":"drain","context":{"idset":"10805","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.67802,"name":"drain","context":{"idset":"10806","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6783528,"name":"drain","context":{"idset":"10807","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6786397,"name":"drain","context":{"idset":"10808","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6789377,"name":"drain","context":{"idset":"10809","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6792412,"name":"drain","context":{"idset":"10810","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.679554,"name":"drain","context":{"idset":"10811","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6798365,"name":"drain","context":{"idset":"10812","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6801395,"name":"drain","context":{"idset":"10813","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6804609,"name":"drain","context":{"idset":"10814","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6807647,"name":"drain","context":{"idset":"10815","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6810613,"name":"drain","context":{"idset":"10816","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6813772,"name":"drain","context":{"idset":"10817","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6816847,"name":"drain","context":{"idset":"10818","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6819994,"name":"drain","context":{"idset":"10819","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6823449,"name":"drain","context":{"idset":"10820","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.682672,"name":"drain","context":{"idset":"10821","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.683013,"name":"drain","context":{"idset":"10822","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.683346,"name":"drain","context":{"idset":"10823","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6836467,"name":"drain","context":{"idset":"10824","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6839497,"name":"drain","context":{"idset":"10825","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6842763,"name":"drain","context":{"idset":"10826","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6846058,"name":"drain","context":{"idset":"10827","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6849337,"name":"drain","context":{"idset":"10828","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6852462,"name":"drain","context":{"idset":"10829","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6855786,"name":"drain","context":{"idset":"10830","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6858962,"name":"drain","context":{"idset":"10831","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6862061,"name":"drain","context":{"idset":"10832","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6865392,"name":"drain","context":{"idset":"10833","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6868746,"name":"drain","context":{"idset":"10834","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6872044,"name":"drain","context":{"idset":"10835","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6875408,"name":"drain","context":{"idset":"10836","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6878715,"name":"drain","context":{"idset":"10837","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.688221,"name":"drain","context":{"idset":"10838","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.688575,"name":"drain","context":{"idset":"10839","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6889174,"name":"drain","context":{"idset":"10840","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6892495,"name":"drain","context":{"idset":"10841","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6896098,"name":"drain","context":{"idset":"10842","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.689961,"name":"drain","context":{"idset":"10843","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6903386,"name":"drain","context":{"idset":"10844","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6907053,"name":"drain","context":{"idset":"10845","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6910613,"name":"drain","context":{"idset":"10846","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6914213,"name":"drain","context":{"idset":"10847","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6917696,"name":"drain","context":{"idset":"10848","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.692111,"name":"drain","context":{"idset":"10849","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6924729,"name":"drain","context":{"idset":"10850","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6928153,"name":"drain","context":{"idset":"10851","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6931584,"name":"drain","context":{"idset":"10852","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6935213,"name":"drain","context":{"idset":"10853","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6938698,"name":"drain","context":{"idset":"10854","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6942201,"name":"drain","context":{"idset":"10855","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6945853,"name":"drain","context":{"idset":"10856","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6949329,"name":"drain","context":{"idset":"10857","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6952944,"name":"drain","context":{"idset":"10858","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6956275,"name":"drain","context":{"idset":"10859","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6959455,"name":"drain","context":{"idset":"10860","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6962545,"name":"drain","context":{"idset":"10861","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6965852,"name":"drain","context":{"idset":"10862","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6969151,"name":"drain","context":{"idset":"10863","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6972473,"name":"drain","context":{"idset":"10864","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6975973,"name":"drain","context":{"idset":"10865","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6979597,"name":"drain","context":{"idset":"10866","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6983202,"name":"drain","context":{"idset":"10867","reason":"broker was unresponsive"}}
+{"timestamp":1712759591.6986685,"name":"drain","context":{"idset":"10868","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4221883,"name":"drain","context":{"idset":"10997","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4226396,"name":"drain","context":{"idset":"10998","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4230587,"name":"drain","context":{"idset":"11001","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4234798,"name":"drain","context":{"idset":"11002","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.423888,"name":"drain","context":{"idset":"11003","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4243228,"name":"drain","context":{"idset":"11004","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4247401,"name":"drain","context":{"idset":"11005","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4251459,"name":"drain","context":{"idset":"11006","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4255552,"name":"drain","context":{"idset":"11007","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4259539,"name":"drain","context":{"idset":"11008","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4263663,"name":"drain","context":{"idset":"11009","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4267662,"name":"drain","context":{"idset":"11010","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4271679,"name":"drain","context":{"idset":"11011","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4275813,"name":"drain","context":{"idset":"11012","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4279919,"name":"drain","context":{"idset":"11013","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.428401,"name":"drain","context":{"idset":"11014","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4287958,"name":"drain","context":{"idset":"11015","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4291911,"name":"drain","context":{"idset":"11016","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4296048,"name":"drain","context":{"idset":"11017","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4300027,"name":"drain","context":{"idset":"11018","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4304178,"name":"drain","context":{"idset":"11019","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4308178,"name":"drain","context":{"idset":"11020","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4316301,"name":"drain","context":{"idset":"11022","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4320414,"name":"drain","context":{"idset":"11023","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4324777,"name":"drain","context":{"idset":"11024","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4328902,"name":"drain","context":{"idset":"11025","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4332855,"name":"drain","context":{"idset":"11026","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4337115,"name":"drain","context":{"idset":"11027","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4341214,"name":"drain","context":{"idset":"11028","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4345419,"name":"drain","context":{"idset":"11029","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4349661,"name":"drain","context":{"idset":"11030","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4354115,"name":"drain","context":{"idset":"11031","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4358392,"name":"drain","context":{"idset":"11032","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4362876,"name":"drain","context":{"idset":"11033","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.436718,"name":"drain","context":{"idset":"11034","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4371614,"name":"drain","context":{"idset":"11035","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4376137,"name":"drain","context":{"idset":"11036","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4380553,"name":"drain","context":{"idset":"11037","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4385159,"name":"drain","context":{"idset":"11038","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4389508,"name":"drain","context":{"idset":"11039","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.439393,"name":"drain","context":{"idset":"11040","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4398444,"name":"drain","context":{"idset":"11041","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4402997,"name":"drain","context":{"idset":"11042","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4407642,"name":"drain","context":{"idset":"11043","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4412272,"name":"drain","context":{"idset":"11044","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4416962,"name":"drain","context":{"idset":"11045","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4421592,"name":"drain","context":{"idset":"11046","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4426372,"name":"drain","context":{"idset":"11047","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4431002,"name":"drain","context":{"idset":"11048","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4435737,"name":"drain","context":{"idset":"11049","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4440386,"name":"drain","context":{"idset":"11050","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4445159,"name":"drain","context":{"idset":"11051","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4449763,"name":"drain","context":{"idset":"11052","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4454505,"name":"drain","context":{"idset":"11053","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.445909,"name":"drain","context":{"idset":"11054","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4463484,"name":"drain","context":{"idset":"11055","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4467673,"name":"drain","context":{"idset":"11056","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4471474,"name":"drain","context":{"idset":"11057","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4475448,"name":"drain","context":{"idset":"11058","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4479365,"name":"drain","context":{"idset":"11059","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4483397,"name":"drain","context":{"idset":"11060","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4487088,"name":"drain","context":{"idset":"11061","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4491057,"name":"drain","context":{"idset":"11062","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4494214,"name":"drain","context":{"idset":"11063","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4497766,"name":"drain","context":{"idset":"11064","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4501758,"name":"drain","context":{"idset":"11065","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4505141,"name":"drain","context":{"idset":"11066","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4509254,"name":"drain","context":{"idset":"11067","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4513671,"name":"drain","context":{"idset":"11068","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4517906,"name":"drain","context":{"idset":"11069","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4522178,"name":"drain","context":{"idset":"11070","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4526494,"name":"drain","context":{"idset":"11071","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4530621,"name":"drain","context":{"idset":"11072","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4535003,"name":"drain","context":{"idset":"11073","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4539268,"name":"drain","context":{"idset":"11074","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4543817,"name":"drain","context":{"idset":"11075","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4548256,"name":"drain","context":{"idset":"11076","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.455286,"name":"drain","context":{"idset":"11077","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4557343,"name":"drain","context":{"idset":"11078","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4561732,"name":"drain","context":{"idset":"11079","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4566464,"name":"drain","context":{"idset":"11080","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4571233,"name":"drain","context":{"idset":"11081","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4575796,"name":"drain","context":{"idset":"11082","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4580224,"name":"drain","context":{"idset":"11083","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4584782,"name":"drain","context":{"idset":"11084","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4589188,"name":"drain","context":{"idset":"11085","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4593711,"name":"drain","context":{"idset":"11086","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4598162,"name":"drain","context":{"idset":"11087","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4602616,"name":"drain","context":{"idset":"11088","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4607284,"name":"drain","context":{"idset":"11089","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4611747,"name":"drain","context":{"idset":"11090","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4616323,"name":"drain","context":{"idset":"11091","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4620717,"name":"drain","context":{"idset":"11092","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4625261,"name":"drain","context":{"idset":"11093","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4629669,"name":"drain","context":{"idset":"11094","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4634185,"name":"drain","context":{"idset":"11095","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4638629,"name":"drain","context":{"idset":"11096","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4643376,"name":"drain","context":{"idset":"11097","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4647629,"name":"drain","context":{"idset":"11098","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4651833,"name":"drain","context":{"idset":"11099","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4656129,"name":"drain","context":{"idset":"11100","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4660335,"name":"drain","context":{"idset":"11101","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4664681,"name":"drain","context":{"idset":"11102","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4668918,"name":"drain","context":{"idset":"11103","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.467351,"name":"drain","context":{"idset":"11104","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4678152,"name":"drain","context":{"idset":"11105","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4693718,"name":"drain","context":{"idset":"11106","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4708183,"name":"drain","context":{"idset":"11107","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4728539,"name":"drain","context":{"idset":"11108","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4749556,"name":"drain","context":{"idset":"11109","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4771202,"name":"drain","context":{"idset":"11110","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4789248,"name":"drain","context":{"idset":"11113","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4804122,"name":"drain","context":{"idset":"11114","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4814508,"name":"drain","context":{"idset":"11116","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4825857,"name":"drain","context":{"idset":"11119","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4837184,"name":"drain","context":{"idset":"11120","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4846714,"name":"drain","context":{"idset":"11121","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4859455,"name":"drain","context":{"idset":"11122","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.487283,"name":"drain","context":{"idset":"11123","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4882228,"name":"drain","context":{"idset":"11124","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4892173,"name":"drain","context":{"idset":"11125","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4905972,"name":"drain","context":{"idset":"11126","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4918332,"name":"drain","context":{"idset":"11127","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4934547,"name":"drain","context":{"idset":"11128","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4944172,"name":"drain","context":{"idset":"11129","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4949284,"name":"drain","context":{"idset":"11130","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4954829,"name":"drain","context":{"idset":"11131","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.49599,"name":"drain","context":{"idset":"11132","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4965138,"name":"drain","context":{"idset":"11133","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4970212,"name":"drain","context":{"idset":"11134","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4975932,"name":"drain","context":{"idset":"11135","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4981239,"name":"drain","context":{"idset":"11136","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4986386,"name":"drain","context":{"idset":"11137","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4991193,"name":"drain","context":{"idset":"11138","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.4997928,"name":"drain","context":{"idset":"11139","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5003836,"name":"drain","context":{"idset":"11140","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5008974,"name":"drain","context":{"idset":"11141","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5015595,"name":"drain","context":{"idset":"11142","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5020499,"name":"drain","context":{"idset":"11143","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5025945,"name":"drain","context":{"idset":"11144","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5031092,"name":"drain","context":{"idset":"11145","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5036352,"name":"drain","context":{"idset":"11146","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5041413,"name":"drain","context":{"idset":"11147","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.504674,"name":"drain","context":{"idset":"11148","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5051973,"name":"drain","context":{"idset":"11149","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5057647,"name":"drain","context":{"idset":"11150","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5062833,"name":"drain","context":{"idset":"11151","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5068278,"name":"drain","context":{"idset":"11152","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5074098,"name":"drain","context":{"idset":"11153","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5079701,"name":"drain","context":{"idset":"11154","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5085733,"name":"drain","context":{"idset":"11155","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5091298,"name":"drain","context":{"idset":"11156","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5096922,"name":"drain","context":{"idset":"11157","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5102472,"name":"drain","context":{"idset":"11158","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5108066,"name":"drain","context":{"idset":"11159","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5113437,"name":"drain","context":{"idset":"11160","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5117924,"name":"drain","context":{"idset":"11161","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5123165,"name":"drain","context":{"idset":"11162","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5128188,"name":"drain","context":{"idset":"11163","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5133598,"name":"drain","context":{"idset":"11164","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5138958,"name":"drain","context":{"idset":"11165","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5144396,"name":"drain","context":{"idset":"11166","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5149746,"name":"drain","context":{"idset":"11167","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5154393,"name":"drain","context":{"idset":"11168","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5158777,"name":"drain","context":{"idset":"11169","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5163407,"name":"drain","context":{"idset":"11170","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5168383,"name":"drain","context":{"idset":"11171","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5174196,"name":"drain","context":{"idset":"11172","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5180302,"name":"drain","context":{"idset":"11173","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5186138,"name":"drain","context":{"idset":"11174","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.519165,"name":"drain","context":{"idset":"11175","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5196774,"name":"drain","context":{"idset":"11176","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5201116,"name":"drain","context":{"idset":"11177","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5205894,"name":"drain","context":{"idset":"11178","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5210712,"name":"drain","context":{"idset":"11179","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.521651,"name":"drain","context":{"idset":"11180","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.52212,"name":"drain","context":{"idset":"11181","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.522603,"name":"drain","context":{"idset":"11182","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5230992,"name":"drain","context":{"idset":"11183","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5235598,"name":"drain","context":{"idset":"11184","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.524065,"name":"drain","context":{"idset":"11185","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5246119,"name":"drain","context":{"idset":"11186","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5251348,"name":"drain","context":{"idset":"11187","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5259411,"name":"drain","context":{"idset":"11188","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5264928,"name":"drain","context":{"idset":"11189","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5269737,"name":"drain","context":{"idset":"11190","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5275307,"name":"drain","context":{"idset":"11191","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5280221,"name":"drain","context":{"idset":"11192","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5284953,"name":"drain","context":{"idset":"11193","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5289545,"name":"drain","context":{"idset":"11194","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5294893,"name":"drain","context":{"idset":"11195","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5299883,"name":"drain","context":{"idset":"11196","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5305665,"name":"drain","context":{"idset":"11197","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5311172,"name":"drain","context":{"idset":"11198","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5316379,"name":"drain","context":{"idset":"11199","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5321474,"name":"drain","context":{"idset":"11200","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5324674,"name":"drain","context":{"idset":"11201","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5328109,"name":"drain","context":{"idset":"11202","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5333056,"name":"drain","context":{"idset":"11203","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5336587,"name":"drain","context":{"idset":"11204","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5339539,"name":"drain","context":{"idset":"11205","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5342469,"name":"drain","context":{"idset":"11206","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5347102,"name":"drain","context":{"idset":"11207","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5451548,"name":"drain","context":{"idset":"11208","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5569015,"name":"drain","context":{"idset":"11209","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5574014,"name":"drain","context":{"idset":"11210","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5578575,"name":"drain","context":{"idset":"11211","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.558331,"name":"drain","context":{"idset":"11212","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5588038,"name":"drain","context":{"idset":"11213","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.559305,"name":"drain","context":{"idset":"11214","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.559756,"name":"drain","context":{"idset":"11215","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5602388,"name":"drain","context":{"idset":"11216","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.571389,"name":"drain","context":{"idset":"11217","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5823009,"name":"drain","context":{"idset":"11218","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5828159,"name":"drain","context":{"idset":"11219","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5833187,"name":"drain","context":{"idset":"11220","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5837984,"name":"drain","context":{"idset":"11221","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5843213,"name":"drain","context":{"idset":"11222","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5847986,"name":"drain","context":{"idset":"11223","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5853114,"name":"drain","context":{"idset":"11224","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5856829,"name":"drain","context":{"idset":"11225","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5859828,"name":"drain","context":{"idset":"11226","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.586313,"name":"drain","context":{"idset":"11227","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5868366,"name":"drain","context":{"idset":"11228","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5873494,"name":"drain","context":{"idset":"11229","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.587851,"name":"drain","context":{"idset":"11230","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5883515,"name":"drain","context":{"idset":"11231","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5888464,"name":"drain","context":{"idset":"11232","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5894256,"name":"drain","context":{"idset":"11233","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5898349,"name":"drain","context":{"idset":"11234","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5904071,"name":"drain","context":{"idset":"11235","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5909672,"name":"drain","context":{"idset":"11236","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.591568,"name":"drain","context":{"idset":"11237","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5921402,"name":"drain","context":{"idset":"11238","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.592813,"name":"drain","context":{"idset":"11239","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5934839,"name":"drain","context":{"idset":"11240","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5941086,"name":"drain","context":{"idset":"11241","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5947335,"name":"drain","context":{"idset":"11242","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5953617,"name":"drain","context":{"idset":"11243","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.595999,"name":"drain","context":{"idset":"11244","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5966108,"name":"drain","context":{"idset":"11245","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5972507,"name":"drain","context":{"idset":"11246","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5978236,"name":"drain","context":{"idset":"11247","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5983727,"name":"drain","context":{"idset":"11248","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5989211,"name":"drain","context":{"idset":"11251","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.5994968,"name":"drain","context":{"idset":"11252","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6000359,"name":"drain","context":{"idset":"11253","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.600625,"name":"drain","context":{"idset":"11254","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6012387,"name":"drain","context":{"idset":"11255","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6018629,"name":"drain","context":{"idset":"11256","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6025267,"name":"drain","context":{"idset":"11257","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.603147,"name":"drain","context":{"idset":"11258","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6036983,"name":"drain","context":{"idset":"11259","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6043353,"name":"drain","context":{"idset":"11260","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6048725,"name":"drain","context":{"idset":"11261","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6054664,"name":"drain","context":{"idset":"11262","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6060574,"name":"drain","context":{"idset":"11263","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.60657,"name":"drain","context":{"idset":"11264","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6071658,"name":"drain","context":{"idset":"11265","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6077182,"name":"drain","context":{"idset":"11266","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6082981,"name":"drain","context":{"idset":"11267","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6087723,"name":"drain","context":{"idset":"11268","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6092563,"name":"drain","context":{"idset":"11269","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6098185,"name":"drain","context":{"idset":"11270","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.610369,"name":"drain","context":{"idset":"11271","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6108594,"name":"drain","context":{"idset":"11272","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.611433,"name":"drain","context":{"idset":"11273","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6119974,"name":"drain","context":{"idset":"11274","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6126034,"name":"drain","context":{"idset":"11275","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6131597,"name":"drain","context":{"idset":"11276","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6136589,"name":"drain","context":{"idset":"11277","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6141768,"name":"drain","context":{"idset":"11278","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6147459,"name":"drain","context":{"idset":"11279","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.615303,"name":"drain","context":{"idset":"11280","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6158764,"name":"drain","context":{"idset":"11281","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6165156,"name":"drain","context":{"idset":"11282","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.617085,"name":"drain","context":{"idset":"11283","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6176744,"name":"drain","context":{"idset":"11284","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6182034,"name":"drain","context":{"idset":"11285","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6187479,"name":"drain","context":{"idset":"11286","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6193075,"name":"drain","context":{"idset":"11287","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6198559,"name":"drain","context":{"idset":"11288","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6204252,"name":"drain","context":{"idset":"11289","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6209991,"name":"drain","context":{"idset":"11290","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6215954,"name":"drain","context":{"idset":"11291","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6228149,"name":"drain","context":{"idset":"11292","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6233673,"name":"drain","context":{"idset":"11293","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6239967,"name":"drain","context":{"idset":"11294","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6246061,"name":"drain","context":{"idset":"11295","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.625072,"name":"drain","context":{"idset":"11296","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6255867,"name":"drain","context":{"idset":"11297","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6260233,"name":"drain","context":{"idset":"11298","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6264982,"name":"drain","context":{"idset":"11299","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6270432,"name":"drain","context":{"idset":"11300","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6276486,"name":"drain","context":{"idset":"11301","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6283281,"name":"drain","context":{"idset":"11302","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6288171,"name":"drain","context":{"idset":"11303","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6291609,"name":"drain","context":{"idset":"11304","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6295159,"name":"drain","context":{"idset":"11305","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6298625,"name":"drain","context":{"idset":"11306","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6302063,"name":"drain","context":{"idset":"11307","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6307297,"name":"drain","context":{"idset":"11308","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6313987,"name":"drain","context":{"idset":"11309","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6318154,"name":"drain","context":{"idset":"11310","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6321394,"name":"drain","context":{"idset":"11311","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6325295,"name":"drain","context":{"idset":"11312","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6330853,"name":"drain","context":{"idset":"11313","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6336694,"name":"drain","context":{"idset":"11314","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6342957,"name":"drain","context":{"idset":"11315","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6348772,"name":"drain","context":{"idset":"11316","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6354795,"name":"drain","context":{"idset":"11317","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6360359,"name":"drain","context":{"idset":"11318","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6366224,"name":"drain","context":{"idset":"11319","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6371562,"name":"drain","context":{"idset":"11320","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6377304,"name":"drain","context":{"idset":"11321","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6383197,"name":"drain","context":{"idset":"11322","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6388981,"name":"drain","context":{"idset":"11323","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6394873,"name":"drain","context":{"idset":"11324","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6400793,"name":"drain","context":{"idset":"11325","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6407011,"name":"drain","context":{"idset":"11326","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6413341,"name":"drain","context":{"idset":"11327","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6419675,"name":"drain","context":{"idset":"11328","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6425838,"name":"drain","context":{"idset":"11329","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6431479,"name":"drain","context":{"idset":"11330","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6437755,"name":"drain","context":{"idset":"11331","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6443784,"name":"drain","context":{"idset":"11332","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.644985,"name":"drain","context":{"idset":"11333","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6455872,"name":"drain","context":{"idset":"11334","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6462171,"name":"drain","context":{"idset":"11335","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6469111,"name":"drain","context":{"idset":"11336","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6474674,"name":"drain","context":{"idset":"11337","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6480186,"name":"drain","context":{"idset":"11338","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6485491,"name":"drain","context":{"idset":"11339","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6490805,"name":"drain","context":{"idset":"11340","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.649657,"name":"drain","context":{"idset":"11341","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.650171,"name":"drain","context":{"idset":"11342","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.650722,"name":"drain","context":{"idset":"11343","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6512802,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6518152,"name":"drain","context":{"idset":"11345","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6523798,"name":"drain","context":{"idset":"11346","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.652972,"name":"drain","context":{"idset":"11347","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.653578,"name":"drain","context":{"idset":"11348","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6541684,"name":"drain","context":{"idset":"11349","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6547911,"name":"drain","context":{"idset":"11350","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6553931,"name":"drain","context":{"idset":"11351","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6559784,"name":"drain","context":{"idset":"11352","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6565921,"name":"drain","context":{"idset":"11353","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6571681,"name":"drain","context":{"idset":"11354","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6577628,"name":"drain","context":{"idset":"11355","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6583338,"name":"drain","context":{"idset":"11356","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6588879,"name":"drain","context":{"idset":"11357","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6594598,"name":"drain","context":{"idset":"11358","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6600397,"name":"drain","context":{"idset":"11359","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.660671,"name":"drain","context":{"idset":"11360","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6612129,"name":"drain","context":{"idset":"11361","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6618903,"name":"drain","context":{"idset":"11362","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6626096,"name":"drain","context":{"idset":"11363","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6632881,"name":"drain","context":{"idset":"11364","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6639597,"name":"drain","context":{"idset":"11365","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6646211,"name":"drain","context":{"idset":"11366","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6652834,"name":"drain","context":{"idset":"11367","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6659229,"name":"drain","context":{"idset":"11368","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6666136,"name":"drain","context":{"idset":"11369","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6672907,"name":"drain","context":{"idset":"11370","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6679862,"name":"drain","context":{"idset":"11371","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6686978,"name":"drain","context":{"idset":"11372","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6693611,"name":"drain","context":{"idset":"11373","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6700089,"name":"drain","context":{"idset":"11374","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6706519,"name":"drain","context":{"idset":"11375","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6713467,"name":"drain","context":{"idset":"11376","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6719704,"name":"drain","context":{"idset":"11377","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6726489,"name":"drain","context":{"idset":"11378","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6734133,"name":"drain","context":{"idset":"11379","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.674077,"name":"drain","context":{"idset":"11380","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.674767,"name":"drain","context":{"idset":"11381","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6754725,"name":"drain","context":{"idset":"11382","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6761081,"name":"drain","context":{"idset":"11383","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.676765,"name":"drain","context":{"idset":"11384","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.67751,"name":"drain","context":{"idset":"11385","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.678215,"name":"drain","context":{"idset":"11386","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.678916,"name":"drain","context":{"idset":"11387","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.679601,"name":"drain","context":{"idset":"11388","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6802769,"name":"drain","context":{"idset":"11389","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6808803,"name":"drain","context":{"idset":"11390","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6815279,"name":"drain","context":{"idset":"11391","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6821511,"name":"drain","context":{"idset":"11392","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6828098,"name":"drain","context":{"idset":"11393","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6834009,"name":"drain","context":{"idset":"11394","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6840727,"name":"drain","context":{"idset":"11395","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6846976,"name":"drain","context":{"idset":"11396","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6853449,"name":"drain","context":{"idset":"11397","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6859448,"name":"drain","context":{"idset":"11398","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6865573,"name":"drain","context":{"idset":"11399","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6871595,"name":"drain","context":{"idset":"11400","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6877646,"name":"drain","context":{"idset":"11401","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6883903,"name":"drain","context":{"idset":"11402","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6890039,"name":"drain","context":{"idset":"11403","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6895649,"name":"drain","context":{"idset":"11404","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6901498,"name":"drain","context":{"idset":"11405","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6907525,"name":"drain","context":{"idset":"11406","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6913528,"name":"drain","context":{"idset":"11407","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6919451,"name":"drain","context":{"idset":"11408","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6925559,"name":"drain","context":{"idset":"11409","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6930673,"name":"drain","context":{"idset":"11410","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6936266,"name":"drain","context":{"idset":"11411","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.6942849,"name":"drain","context":{"idset":"11412","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7064137,"name":"drain","context":{"idset":"11413","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7070096,"name":"drain","context":{"idset":"11414","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7075849,"name":"drain","context":{"idset":"11415","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7081778,"name":"drain","context":{"idset":"11416","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7088006,"name":"drain","context":{"idset":"11417","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7094195,"name":"drain","context":{"idset":"11418","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7100053,"name":"drain","context":{"idset":"11419","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7106287,"name":"drain","context":{"idset":"11420","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7112229,"name":"drain","context":{"idset":"11421","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.711725,"name":"drain","context":{"idset":"11422","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7120786,"name":"drain","context":{"idset":"11423","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7124937,"name":"drain","context":{"idset":"11424","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7128701,"name":"drain","context":{"idset":"11425","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7132549,"name":"drain","context":{"idset":"11426","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.71382,"name":"drain","context":{"idset":"11427","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7144599,"name":"drain","context":{"idset":"11428","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7150624,"name":"drain","context":{"idset":"11429","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7156656,"name":"drain","context":{"idset":"11430","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7163172,"name":"drain","context":{"idset":"11431","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7169476,"name":"drain","context":{"idset":"11432","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7175741,"name":"drain","context":{"idset":"11433","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7181799,"name":"drain","context":{"idset":"11434","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7188287,"name":"drain","context":{"idset":"11435","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7194612,"name":"drain","context":{"idset":"11436","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7200503,"name":"drain","context":{"idset":"11437","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7206528,"name":"drain","context":{"idset":"11438","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7212598,"name":"drain","context":{"idset":"11439","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7218609,"name":"drain","context":{"idset":"11440","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7225015,"name":"drain","context":{"idset":"11441","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7230957,"name":"drain","context":{"idset":"11442","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7236865,"name":"drain","context":{"idset":"11443","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7242932,"name":"drain","context":{"idset":"11444","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7248557,"name":"drain","context":{"idset":"11445","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7255063,"name":"drain","context":{"idset":"11446","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7260852,"name":"drain","context":{"idset":"11447","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7266648,"name":"drain","context":{"idset":"11448","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7272384,"name":"drain","context":{"idset":"11449","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.727838,"name":"drain","context":{"idset":"11450","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7284334,"name":"drain","context":{"idset":"11451","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7289982,"name":"drain","context":{"idset":"11452","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7295976,"name":"drain","context":{"idset":"11453","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7301738,"name":"drain","context":{"idset":"11454","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.730813,"name":"drain","context":{"idset":"11455","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7315104,"name":"drain","context":{"idset":"11456","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7321408,"name":"drain","context":{"idset":"11457","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7327461,"name":"drain","context":{"idset":"11458","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7334085,"name":"drain","context":{"idset":"11459","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7340064,"name":"drain","context":{"idset":"11460","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7345457,"name":"drain","context":{"idset":"11461","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7350743,"name":"drain","context":{"idset":"11462","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7357497,"name":"drain","context":{"idset":"11463","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7363746,"name":"drain","context":{"idset":"11464","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7370389,"name":"drain","context":{"idset":"11465","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7377217,"name":"drain","context":{"idset":"11466","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7383697,"name":"drain","context":{"idset":"11467","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7390258,"name":"drain","context":{"idset":"11468","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7396922,"name":"drain","context":{"idset":"11469","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7403805,"name":"drain","context":{"idset":"11470","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7410457,"name":"drain","context":{"idset":"11471","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7417355,"name":"drain","context":{"idset":"11472","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7423985,"name":"drain","context":{"idset":"11473","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7430742,"name":"drain","context":{"idset":"11474","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7436876,"name":"drain","context":{"idset":"11475","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7443109,"name":"drain","context":{"idset":"11476","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7449317,"name":"drain","context":{"idset":"11477","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.745615,"name":"drain","context":{"idset":"11478","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7462878,"name":"drain","context":{"idset":"11479","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7470083,"name":"drain","context":{"idset":"11480","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7476995,"name":"drain","context":{"idset":"11481","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7483664,"name":"drain","context":{"idset":"11482","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7490761,"name":"drain","context":{"idset":"11483","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7497611,"name":"drain","context":{"idset":"11484","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7504163,"name":"drain","context":{"idset":"11485","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7510793,"name":"drain","context":{"idset":"11486","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7517576,"name":"drain","context":{"idset":"11487","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7523887,"name":"drain","context":{"idset":"11488","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7530017,"name":"drain","context":{"idset":"11489","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7536528,"name":"drain","context":{"idset":"11490","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7543101,"name":"drain","context":{"idset":"11491","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7548883,"name":"drain","context":{"idset":"11492","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7555923,"name":"drain","context":{"idset":"11493","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7562273,"name":"drain","context":{"idset":"11494","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7568157,"name":"drain","context":{"idset":"11495","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7574139,"name":"drain","context":{"idset":"11496","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7579937,"name":"drain","context":{"idset":"11497","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7585475,"name":"drain","context":{"idset":"11498","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7591336,"name":"drain","context":{"idset":"11499","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7597299,"name":"drain","context":{"idset":"11500","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7603214,"name":"drain","context":{"idset":"11501","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7609522,"name":"drain","context":{"idset":"11502","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7619376,"name":"drain","context":{"idset":"11503","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7625978,"name":"drain","context":{"idset":"11504","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7632568,"name":"drain","context":{"idset":"11505","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7638781,"name":"drain","context":{"idset":"11506","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7645004,"name":"drain","context":{"idset":"11507","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7651138,"name":"drain","context":{"idset":"11508","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7847648,"name":"drain","context":{"idset":"11541","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7852886,"name":"drain","context":{"idset":"11542","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.785984,"name":"drain","context":{"idset":"11543","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7866147,"name":"drain","context":{"idset":"11544","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.787297,"name":"drain","context":{"idset":"11545","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7879021,"name":"drain","context":{"idset":"11546","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7885287,"name":"drain","context":{"idset":"11547","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7892823,"name":"drain","context":{"idset":"11548","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7899983,"name":"drain","context":{"idset":"11549","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.790652,"name":"drain","context":{"idset":"11550","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7913098,"name":"drain","context":{"idset":"11551","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7919211,"name":"drain","context":{"idset":"11552","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7924542,"name":"drain","context":{"idset":"11553","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7929587,"name":"drain","context":{"idset":"11554","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7936351,"name":"drain","context":{"idset":"11555","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7941349,"name":"drain","context":{"idset":"11556","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7946458,"name":"drain","context":{"idset":"11557","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7950449,"name":"drain","context":{"idset":"11558","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7957051,"name":"drain","context":{"idset":"11559","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7964203,"name":"drain","context":{"idset":"11560","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7969058,"name":"drain","context":{"idset":"11561","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7974412,"name":"drain","context":{"idset":"11562","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7980356,"name":"drain","context":{"idset":"11563","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.798758,"name":"drain","context":{"idset":"11564","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.7995231,"name":"drain","context":{"idset":"11565","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8003275,"name":"drain","context":{"idset":"11566","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8010836,"name":"drain","context":{"idset":"11567","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8020079,"name":"drain","context":{"idset":"11568","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8028038,"name":"drain","context":{"idset":"11569","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8035731,"name":"drain","context":{"idset":"11570","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.804374,"name":"drain","context":{"idset":"11571","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8051133,"name":"drain","context":{"idset":"11572","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8057694,"name":"drain","context":{"idset":"11573","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8064082,"name":"drain","context":{"idset":"11574","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8071251,"name":"drain","context":{"idset":"11575","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8078654,"name":"drain","context":{"idset":"11576","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.808533,"name":"drain","context":{"idset":"11577","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.809166,"name":"drain","context":{"idset":"11578","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8098047,"name":"drain","context":{"idset":"11579","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8105085,"name":"drain","context":{"idset":"11580","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8109479,"name":"drain","context":{"idset":"11581","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8114259,"name":"drain","context":{"idset":"11582","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8118861,"name":"drain","context":{"idset":"11583","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8127072,"name":"drain","context":{"idset":"11584","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8134422,"name":"drain","context":{"idset":"11585","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8140738,"name":"drain","context":{"idset":"11586","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.814703,"name":"drain","context":{"idset":"11587","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8154101,"name":"drain","context":{"idset":"11588","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8160288,"name":"drain","context":{"idset":"11589","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8164778,"name":"drain","context":{"idset":"11590","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8168905,"name":"drain","context":{"idset":"11591","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8175614,"name":"drain","context":{"idset":"11592","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8182518,"name":"drain","context":{"idset":"11593","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8190131,"name":"drain","context":{"idset":"11594","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8196847,"name":"drain","context":{"idset":"11595","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8203144,"name":"drain","context":{"idset":"11596","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8207932,"name":"drain","context":{"idset":"11597","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8211808,"name":"drain","context":{"idset":"11598","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8216333,"name":"drain","context":{"idset":"11599","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8222816,"name":"drain","context":{"idset":"11600","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8230252,"name":"drain","context":{"idset":"11601","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8236783,"name":"drain","context":{"idset":"11602","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8243864,"name":"drain","context":{"idset":"11603","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8250616,"name":"drain","context":{"idset":"11604","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8258314,"name":"drain","context":{"idset":"11605","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8265154,"name":"drain","context":{"idset":"11606","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8272941,"name":"drain","context":{"idset":"11607","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8279274,"name":"drain","context":{"idset":"11608","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.828635,"name":"drain","context":{"idset":"11609","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8293607,"name":"drain","context":{"idset":"11610","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8300767,"name":"drain","context":{"idset":"11611","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8308134,"name":"drain","context":{"idset":"11612","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8315594,"name":"drain","context":{"idset":"11613","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8323851,"name":"drain","context":{"idset":"11614","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8332434,"name":"drain","context":{"idset":"11615","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8340166,"name":"drain","context":{"idset":"11616","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8348129,"name":"drain","context":{"idset":"11617","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8355825,"name":"drain","context":{"idset":"11618","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8363597,"name":"drain","context":{"idset":"11619","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8370376,"name":"drain","context":{"idset":"11620","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8377256,"name":"drain","context":{"idset":"11621","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8384416,"name":"drain","context":{"idset":"11622","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8391223,"name":"drain","context":{"idset":"11623","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8396561,"name":"drain","context":{"idset":"11624","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8400762,"name":"drain","context":{"idset":"11625","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8406317,"name":"drain","context":{"idset":"11626","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8413472,"name":"drain","context":{"idset":"11627","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8420429,"name":"drain","context":{"idset":"11628","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8427763,"name":"drain","context":{"idset":"11629","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8434517,"name":"drain","context":{"idset":"11630","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8441305,"name":"drain","context":{"idset":"11631","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8448517,"name":"drain","context":{"idset":"11632","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8455849,"name":"drain","context":{"idset":"11633","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8462396,"name":"drain","context":{"idset":"11634","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8469799,"name":"drain","context":{"idset":"11635","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8476946,"name":"drain","context":{"idset":"11636","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8483911,"name":"drain","context":{"idset":"11653","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8491185,"name":"drain","context":{"idset":"11654","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8498912,"name":"drain","context":{"idset":"11655","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8504291,"name":"drain","context":{"idset":"11656","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8510737,"name":"drain","context":{"idset":"11657","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8517802,"name":"drain","context":{"idset":"11658","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8525436,"name":"drain","context":{"idset":"11659","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8532577,"name":"drain","context":{"idset":"11660","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8539503,"name":"drain","context":{"idset":"11661","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.854672,"name":"drain","context":{"idset":"11662","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8553531,"name":"drain","context":{"idset":"11663","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8560736,"name":"drain","context":{"idset":"11664","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.856806,"name":"drain","context":{"idset":"11665","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.857475,"name":"drain","context":{"idset":"11666","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8581674,"name":"drain","context":{"idset":"11667","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8590043,"name":"drain","context":{"idset":"11668","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8597703,"name":"drain","context":{"idset":"11669","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8605366,"name":"drain","context":{"idset":"11670","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8613465,"name":"drain","context":{"idset":"11671","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8620915,"name":"drain","context":{"idset":"11672","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.862844,"name":"drain","context":{"idset":"11673","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8636801,"name":"drain","context":{"idset":"11674","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8644814,"name":"drain","context":{"idset":"11675","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8652782,"name":"drain","context":{"idset":"11676","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8660903,"name":"drain","context":{"idset":"11677","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8668916,"name":"drain","context":{"idset":"11678","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8676465,"name":"drain","context":{"idset":"11679","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8684747,"name":"drain","context":{"idset":"11680","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.869283,"name":"drain","context":{"idset":"11681","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.870034,"name":"drain","context":{"idset":"11682","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8707652,"name":"drain","context":{"idset":"11683","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8715239,"name":"drain","context":{"idset":"11684","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8722773,"name":"drain","context":{"idset":"11685","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8730693,"name":"drain","context":{"idset":"11686","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8738332,"name":"drain","context":{"idset":"11687","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.874594,"name":"drain","context":{"idset":"11688","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8753893,"name":"drain","context":{"idset":"11689","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8761528,"name":"drain","context":{"idset":"11690","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8768914,"name":"drain","context":{"idset":"11691","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8776491,"name":"drain","context":{"idset":"11692","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8784049,"name":"drain","context":{"idset":"11693","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8791955,"name":"drain","context":{"idset":"11695","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8798947,"name":"drain","context":{"idset":"11696","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8805575,"name":"drain","context":{"idset":"11697","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8813884,"name":"drain","context":{"idset":"11698","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8821106,"name":"drain","context":{"idset":"11699","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8829286,"name":"drain","context":{"idset":"11700","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8836601,"name":"drain","context":{"idset":"11701","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8843515,"name":"drain","context":{"idset":"11702","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8851068,"name":"drain","context":{"idset":"11703","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8858562,"name":"drain","context":{"idset":"11704","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8865588,"name":"drain","context":{"idset":"11705","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8872516,"name":"drain","context":{"idset":"11706","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8879845,"name":"drain","context":{"idset":"11707","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8887591,"name":"drain","context":{"idset":"11708","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8895111,"name":"drain","context":{"idset":"11709","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.890245,"name":"drain","context":{"idset":"11710","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8909535,"name":"drain","context":{"idset":"11711","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8916831,"name":"drain","context":{"idset":"11712","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8924255,"name":"drain","context":{"idset":"11713","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8930974,"name":"drain","context":{"idset":"11714","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8938589,"name":"drain","context":{"idset":"11715","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8946512,"name":"drain","context":{"idset":"11716","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8953803,"name":"drain","context":{"idset":"11717","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8961377,"name":"drain","context":{"idset":"11718","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.896889,"name":"drain","context":{"idset":"11719","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8976429,"name":"drain","context":{"idset":"11720","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8984988,"name":"drain","context":{"idset":"11721","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.8993022,"name":"drain","context":{"idset":"11722","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9000895,"name":"drain","context":{"idset":"11723","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9008667,"name":"drain","context":{"idset":"11724","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9016199,"name":"drain","context":{"idset":"11725","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.902514,"name":"drain","context":{"idset":"11726","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9033194,"name":"drain","context":{"idset":"11727","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9040565,"name":"drain","context":{"idset":"11728","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9047596,"name":"drain","context":{"idset":"11729","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9055343,"name":"drain","context":{"idset":"11730","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9062543,"name":"drain","context":{"idset":"11731","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9070032,"name":"drain","context":{"idset":"11732","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.907738,"name":"drain","context":{"idset":"11733","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9085023,"name":"drain","context":{"idset":"11734","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9092324,"name":"drain","context":{"idset":"11735","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9099665,"name":"drain","context":{"idset":"11736","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9105008,"name":"drain","context":{"idset":"11737","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9113085,"name":"drain","context":{"idset":"11738","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9120483,"name":"drain","context":{"idset":"11739","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9126413,"name":"drain","context":{"idset":"11740","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9134097,"name":"drain","context":{"idset":"11741","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9142399,"name":"drain","context":{"idset":"11742","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9149935,"name":"drain","context":{"idset":"11743","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9157424,"name":"drain","context":{"idset":"11744","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9165704,"name":"drain","context":{"idset":"11745","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.917218,"name":"drain","context":{"idset":"11746","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9179676,"name":"drain","context":{"idset":"11747","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9187732,"name":"drain","context":{"idset":"11748","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.919502,"name":"drain","context":{"idset":"11749","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9201849,"name":"drain","context":{"idset":"11750","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9209697,"name":"drain","context":{"idset":"11751","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9217486,"name":"drain","context":{"idset":"11752","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.922467,"name":"drain","context":{"idset":"11753","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9232292,"name":"drain","context":{"idset":"11754","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9240091,"name":"drain","context":{"idset":"11755","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9248128,"name":"drain","context":{"idset":"11756","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9256554,"name":"drain","context":{"idset":"11757","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9265263,"name":"drain","context":{"idset":"11758","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9273319,"name":"drain","context":{"idset":"11759","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9280987,"name":"drain","context":{"idset":"11760","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9289086,"name":"drain","context":{"idset":"11761","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9297152,"name":"drain","context":{"idset":"11762","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9304798,"name":"drain","context":{"idset":"11763","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9313128,"name":"drain","context":{"idset":"11764","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9320669,"name":"drain","context":{"idset":"11765","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9328105,"name":"drain","context":{"idset":"11766","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9335394,"name":"drain","context":{"idset":"11767","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9342132,"name":"drain","context":{"idset":"11768","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9350295,"name":"drain","context":{"idset":"11769","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9359004,"name":"drain","context":{"idset":"11770","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9368262,"name":"drain","context":{"idset":"11771","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9376516,"name":"drain","context":{"idset":"11772","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9384513,"name":"drain","context":{"idset":"11773","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9392185,"name":"drain","context":{"idset":"11774","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9401209,"name":"drain","context":{"idset":"11775","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9409797,"name":"drain","context":{"idset":"11776","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9418163,"name":"drain","context":{"idset":"11777","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9426997,"name":"drain","context":{"idset":"11778","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9435503,"name":"drain","context":{"idset":"11837","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9443879,"name":"drain","context":{"idset":"11838","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9451773,"name":"drain","context":{"idset":"11839","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9461071,"name":"drain","context":{"idset":"11840","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9468963,"name":"drain","context":{"idset":"11841","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9477103,"name":"drain","context":{"idset":"11842","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9484317,"name":"drain","context":{"idset":"11843","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9491653,"name":"drain","context":{"idset":"11844","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9499123,"name":"drain","context":{"idset":"11845","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9506671,"name":"drain","context":{"idset":"11846","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9514117,"name":"drain","context":{"idset":"11847","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9635985,"name":"drain","context":{"idset":"11848","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9761219,"name":"drain","context":{"idset":"11849","reason":"broker was unresponsive"}}
+{"timestamp":1712759592.9890785,"name":"drain","context":{"idset":"11850","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0020802,"name":"drain","context":{"idset":"11851","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0135503,"name":"drain","context":{"idset":"11852","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0259607,"name":"drain","context":{"idset":"11853","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0266213,"name":"drain","context":{"idset":"11854","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0274146,"name":"drain","context":{"idset":"11855","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0281169,"name":"drain","context":{"idset":"11856","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0287523,"name":"drain","context":{"idset":"11857","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0293744,"name":"drain","context":{"idset":"11858","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0299644,"name":"drain","context":{"idset":"11859","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0305252,"name":"drain","context":{"idset":"11860","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0313504,"name":"drain","context":{"idset":"11861","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0322173,"name":"drain","context":{"idset":"11862","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0330961,"name":"drain","context":{"idset":"11866","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0339451,"name":"drain","context":{"idset":"11867","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0350542,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0361776,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0371211,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0379546,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0387392,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.039681,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0406315,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0416827,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0427454,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0437975,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0448949,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0455978,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0464425,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0474527,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0484881,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.049545,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0505929,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0518239,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0530922,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0542479,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0551283,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0562358,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0573137,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0584519,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.059422,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0605109,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0615366,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0625634,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0635641,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0645857,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0657382,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0667183,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0676875,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0685661,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0694158,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0704622,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0715427,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.072691,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0737579,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0747676,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0757704,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0767736,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0777776,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0787954,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0798049,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0807812,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0817854,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.082809,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0838566,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.085012,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0862384,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0875115,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0887952,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0900016,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0911794,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0923069,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0933442,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0943658,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.095371,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0963695,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0969999,"name":"drain","context":{"idset":"85","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0978651,"name":"drain","context":{"idset":"86","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.098618,"name":"drain","context":{"idset":"87","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.0996385,"name":"drain","context":{"idset":"88","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1007106,"name":"drain","context":{"idset":"89","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1017134,"name":"drain","context":{"idset":"91","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1027448,"name":"drain","context":{"idset":"92","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1035404,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.104419,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.105252,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1061618,"name":"drain","context":{"idset":"96","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1070538,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1080422,"name":"drain","context":{"idset":"98","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1089892,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1100414,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1110268,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1121771,"name":"drain","context":{"idset":"102","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.113332,"name":"drain","context":{"idset":"103","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1144764,"name":"drain","context":{"idset":"104","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1156452,"name":"drain","context":{"idset":"105","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.128257,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.140301,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1534228,"name":"drain","context":{"idset":"108","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1655073,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1766667,"name":"drain","context":{"idset":"110","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.1861985,"name":"drain","context":{"idset":"111","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.196022,"name":"drain","context":{"idset":"112","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.2075677,"name":"drain","context":{"idset":"113","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.2181072,"name":"drain","context":{"idset":"114","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.2309413,"name":"drain","context":{"idset":"115","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.2427409,"name":"drain","context":{"idset":"116","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.2558777,"name":"drain","context":{"idset":"117","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.2685637,"name":"drain","context":{"idset":"118","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.2818356,"name":"drain","context":{"idset":"119","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.2953477,"name":"drain","context":{"idset":"120","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3060675,"name":"drain","context":{"idset":"122","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3164256,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3290629,"name":"drain","context":{"idset":"124","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3300517,"name":"drain","context":{"idset":"125","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3310301,"name":"drain","context":{"idset":"126","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3320653,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3330956,"name":"drain","context":{"idset":"128","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.334125,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3351209,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3361821,"name":"drain","context":{"idset":"131","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3372118,"name":"drain","context":{"idset":"132","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3382409,"name":"drain","context":{"idset":"133","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3393064,"name":"drain","context":{"idset":"134","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3402984,"name":"drain","context":{"idset":"135","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3413055,"name":"drain","context":{"idset":"136","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3424149,"name":"drain","context":{"idset":"137","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3435202,"name":"drain","context":{"idset":"138","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3446941,"name":"drain","context":{"idset":"139","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3457818,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3468077,"name":"drain","context":{"idset":"141","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3479805,"name":"drain","context":{"idset":"142","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3490756,"name":"drain","context":{"idset":"143","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3502915,"name":"drain","context":{"idset":"144","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3515272,"name":"drain","context":{"idset":"145","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3527265,"name":"drain","context":{"idset":"146","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3579257,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3590732,"name":"drain","context":{"idset":"148","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3602366,"name":"drain","context":{"idset":"149","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3615966,"name":"drain","context":{"idset":"150","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3628197,"name":"drain","context":{"idset":"151","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3640516,"name":"drain","context":{"idset":"152","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.365284,"name":"drain","context":{"idset":"153","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3665962,"name":"drain","context":{"idset":"154","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3679073,"name":"drain","context":{"idset":"155","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3690426,"name":"drain","context":{"idset":"156","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3699889,"name":"drain","context":{"idset":"157","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3710387,"name":"drain","context":{"idset":"158","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.37184,"name":"drain","context":{"idset":"159","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3726392,"name":"drain","context":{"idset":"160","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3737943,"name":"drain","context":{"idset":"161","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3746176,"name":"drain","context":{"idset":"162","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3752549,"name":"drain","context":{"idset":"163","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.376152,"name":"drain","context":{"idset":"164","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3770111,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.377732,"name":"drain","context":{"idset":"166","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.378561,"name":"drain","context":{"idset":"167","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3792379,"name":"drain","context":{"idset":"168","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3799968,"name":"drain","context":{"idset":"169","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3809438,"name":"drain","context":{"idset":"170","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3819463,"name":"drain","context":{"idset":"171","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3829725,"name":"drain","context":{"idset":"172","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3840015,"name":"drain","context":{"idset":"173","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3846669,"name":"drain","context":{"idset":"174","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3853056,"name":"drain","context":{"idset":"175","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3860478,"name":"drain","context":{"idset":"176","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3871093,"name":"drain","context":{"idset":"177","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3881099,"name":"drain","context":{"idset":"178","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3889749,"name":"drain","context":{"idset":"179","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3899708,"name":"drain","context":{"idset":"180","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3911073,"name":"drain","context":{"idset":"181","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.392345,"name":"drain","context":{"idset":"182","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3934178,"name":"drain","context":{"idset":"183","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3945928,"name":"drain","context":{"idset":"184","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3956759,"name":"drain","context":{"idset":"185","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3967571,"name":"drain","context":{"idset":"186","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3978817,"name":"drain","context":{"idset":"187","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.3989916,"name":"drain","context":{"idset":"188","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4001131,"name":"drain","context":{"idset":"189","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4012835,"name":"drain","context":{"idset":"190","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4024396,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4041855,"name":"drain","context":{"idset":"192","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4067633,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4094679,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4131553,"name":"drain","context":{"idset":"195","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4155102,"name":"drain","context":{"idset":"196","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4167511,"name":"drain","context":{"idset":"197","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4178691,"name":"drain","context":{"idset":"198","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4190118,"name":"drain","context":{"idset":"199","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4201474,"name":"drain","context":{"idset":"200","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.421313,"name":"drain","context":{"idset":"201","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4224529,"name":"drain","context":{"idset":"202","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4235845,"name":"drain","context":{"idset":"203","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4247248,"name":"drain","context":{"idset":"204","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4258389,"name":"drain","context":{"idset":"205","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4269652,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4280784,"name":"drain","context":{"idset":"207","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4290659,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4300385,"name":"drain","context":{"idset":"209","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4311044,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4319415,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4326818,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4337072,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4344933,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4354506,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4361954,"name":"drain","context":{"idset":"216","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4368889,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4375787,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4382358,"name":"drain","context":{"idset":"220","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4393182,"name":"drain","context":{"idset":"221","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4404173,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4414973,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4426019,"name":"drain","context":{"idset":"224","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4436836,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4447677,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4458532,"name":"drain","context":{"idset":"227","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4469402,"name":"drain","context":{"idset":"228","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4479811,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4490159,"name":"drain","context":{"idset":"230","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4500496,"name":"drain","context":{"idset":"231","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4511294,"name":"drain","context":{"idset":"232","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.452152,"name":"drain","context":{"idset":"233","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4533334,"name":"drain","context":{"idset":"234","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4544923,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4556706,"name":"drain","context":{"idset":"236","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4568255,"name":"drain","context":{"idset":"237","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4579873,"name":"drain","context":{"idset":"238","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4591503,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4603188,"name":"drain","context":{"idset":"240","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4614592,"name":"drain","context":{"idset":"241","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4623249,"name":"drain","context":{"idset":"242","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4631848,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4642315,"name":"drain","context":{"idset":"244","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4650087,"name":"drain","context":{"idset":"245","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4660485,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4671209,"name":"drain","context":{"idset":"247","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4681971,"name":"drain","context":{"idset":"248","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4692988,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4704185,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4715316,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4726746,"name":"drain","context":{"idset":"252","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4737902,"name":"drain","context":{"idset":"277","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4749115,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4760361,"name":"drain","context":{"idset":"279","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4771504,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4782522,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4793701,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4804862,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4815412,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4825909,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4836662,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4846249,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4857743,"name":"drain","context":{"idset":"288","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4869246,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4881046,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.489301,"name":"drain","context":{"idset":"291","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4904857,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4916611,"name":"drain","context":{"idset":"293","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4928389,"name":"drain","context":{"idset":"294","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.49403,"name":"drain","context":{"idset":"295","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4952068,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4958856,"name":"drain","context":{"idset":"297","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4965422,"name":"drain","context":{"idset":"298","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4971783,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4980845,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4987538,"name":"drain","context":{"idset":"301","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.4994514,"name":"drain","context":{"idset":"302","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5001128,"name":"drain","context":{"idset":"303","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5007753,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.50142,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5020618,"name":"drain","context":{"idset":"306","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5027244,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5034513,"name":"drain","context":{"idset":"308","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5045819,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5057132,"name":"drain","context":{"idset":"310","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5068393,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5079701,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5091102,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5102248,"name":"drain","context":{"idset":"314","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5113556,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5124738,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5135939,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5146613,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5157304,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5168421,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5178864,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5191069,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.520344,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5215585,"name":"drain","context":{"idset":"324","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5227649,"name":"drain","context":{"idset":"325","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5239694,"name":"drain","context":{"idset":"326","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5251646,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5263896,"name":"drain","context":{"idset":"328","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5273025,"name":"drain","context":{"idset":"329","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5279558,"name":"drain","context":{"idset":"330","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5286145,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.529289,"name":"drain","context":{"idset":"332","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5299325,"name":"drain","context":{"idset":"333","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5305624,"name":"drain","context":{"idset":"334","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5312033,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5318654,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5325615,"name":"drain","context":{"idset":"337","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5332439,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5339253,"name":"drain","context":{"idset":"339","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.534585,"name":"drain","context":{"idset":"340","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5354235,"name":"drain","context":{"idset":"341","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5365231,"name":"drain","context":{"idset":"342","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5373976,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5381792,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5388625,"name":"drain","context":{"idset":"345","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5395396,"name":"drain","context":{"idset":"346","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5402277,"name":"drain","context":{"idset":"347","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5408866,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5415709,"name":"drain","context":{"idset":"350","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5423129,"name":"drain","context":{"idset":"351","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5431297,"name":"drain","context":{"idset":"352","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.544153,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5451663,"name":"drain","context":{"idset":"354","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5459971,"name":"drain","context":{"idset":"355","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5470154,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5480978,"name":"drain","context":{"idset":"357","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5491598,"name":"drain","context":{"idset":"358","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.55019,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5513024,"name":"drain","context":{"idset":"360","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5523906,"name":"drain","context":{"idset":"361","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5534723,"name":"drain","context":{"idset":"362","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5545645,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5556922,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.556468,"name":"drain","context":{"idset":"365","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5575397,"name":"drain","context":{"idset":"366","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5586004,"name":"drain","context":{"idset":"367","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.559428,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5603237,"name":"drain","context":{"idset":"369","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.561069,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5618603,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5627902,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5639205,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.565042,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5661821,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5673263,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5684822,"name":"drain","context":{"idset":"377","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5696545,"name":"drain","context":{"idset":"378","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5708354,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5719407,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5731246,"name":"drain","context":{"idset":"381","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5743132,"name":"drain","context":{"idset":"382","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5754972,"name":"drain","context":{"idset":"383","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5766578,"name":"drain","context":{"idset":"384","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5778117,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5789692,"name":"drain","context":{"idset":"386","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5800719,"name":"drain","context":{"idset":"387","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5811687,"name":"drain","context":{"idset":"388","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5823309,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.583544,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5848134,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.586071,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5873404,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5885994,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5898514,"name":"drain","context":{"idset":"395","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.591069,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5920887,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5931857,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5943074,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5954015,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5965688,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5977213,"name":"drain","context":{"idset":"402","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.5988748,"name":"drain","context":{"idset":"403","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6000323,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6012268,"name":"drain","context":{"idset":"405","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6024663,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6036344,"name":"drain","context":{"idset":"407","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6048009,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6059625,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6071467,"name":"drain","context":{"idset":"410","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6084034,"name":"drain","context":{"idset":"411","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6095867,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6107712,"name":"drain","context":{"idset":"413","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6119564,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6131523,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6143589,"name":"drain","context":{"idset":"416","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6155405,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.616719,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6178792,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.618999,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6201239,"name":"drain","context":{"idset":"421","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6212504,"name":"drain","context":{"idset":"422","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6225066,"name":"drain","context":{"idset":"423","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6237774,"name":"drain","context":{"idset":"424","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6250527,"name":"drain","context":{"idset":"425","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6263511,"name":"drain","context":{"idset":"426","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6276295,"name":"drain","context":{"idset":"427","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.628911,"name":"drain","context":{"idset":"428","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6300027,"name":"drain","context":{"idset":"429","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6311324,"name":"drain","context":{"idset":"430","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6332603,"name":"drain","context":{"idset":"432","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.634481,"name":"drain","context":{"idset":"433","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6357238,"name":"drain","context":{"idset":"434","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6369333,"name":"drain","context":{"idset":"435","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6380947,"name":"drain","context":{"idset":"436","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6391692,"name":"drain","context":{"idset":"437","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6402962,"name":"drain","context":{"idset":"438","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6413784,"name":"drain","context":{"idset":"439","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6424713,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6436148,"name":"drain","context":{"idset":"441","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6446829,"name":"drain","context":{"idset":"442","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6457665,"name":"drain","context":{"idset":"443","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6469476,"name":"drain","context":{"idset":"444","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6482246,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.648982,"name":"drain","context":{"idset":"470","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6499779,"name":"drain","context":{"idset":"471","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6512005,"name":"drain","context":{"idset":"472","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6524704,"name":"drain","context":{"idset":"473","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6537161,"name":"drain","context":{"idset":"474","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6550155,"name":"drain","context":{"idset":"475","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.656117,"name":"drain","context":{"idset":"476","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.657357,"name":"drain","context":{"idset":"477","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6586564,"name":"drain","context":{"idset":"478","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6599596,"name":"drain","context":{"idset":"479","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6612067,"name":"drain","context":{"idset":"480","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.662519,"name":"drain","context":{"idset":"481","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6638186,"name":"drain","context":{"idset":"482","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6649849,"name":"drain","context":{"idset":"483","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.666189,"name":"drain","context":{"idset":"484","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.667433,"name":"drain","context":{"idset":"485","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6687188,"name":"drain","context":{"idset":"486","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6699502,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.671176,"name":"drain","context":{"idset":"488","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6724093,"name":"drain","context":{"idset":"489","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6736176,"name":"drain","context":{"idset":"490","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6748359,"name":"drain","context":{"idset":"491","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6760194,"name":"drain","context":{"idset":"492","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6773207,"name":"drain","context":{"idset":"493","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6785398,"name":"drain","context":{"idset":"494","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6797144,"name":"drain","context":{"idset":"495","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6808832,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6820619,"name":"drain","context":{"idset":"497","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6831317,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6844015,"name":"drain","context":{"idset":"499","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6856878,"name":"drain","context":{"idset":"500","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6868777,"name":"drain","context":{"idset":"501","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6881695,"name":"drain","context":{"idset":"502","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.689481,"name":"drain","context":{"idset":"503","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.690773,"name":"drain","context":{"idset":"504","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6921616,"name":"drain","context":{"idset":"505","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6934929,"name":"drain","context":{"idset":"506","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6947446,"name":"drain","context":{"idset":"507","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6960478,"name":"drain","context":{"idset":"508","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6973135,"name":"drain","context":{"idset":"509","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6986384,"name":"drain","context":{"idset":"510","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.6998131,"name":"drain","context":{"idset":"511","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7010789,"name":"drain","context":{"idset":"512","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7023945,"name":"drain","context":{"idset":"513","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7034752,"name":"drain","context":{"idset":"514","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7045965,"name":"drain","context":{"idset":"515","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7057087,"name":"drain","context":{"idset":"516","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7068093,"name":"drain","context":{"idset":"517","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7079339,"name":"drain","context":{"idset":"518","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.709079,"name":"drain","context":{"idset":"519","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7102289,"name":"drain","context":{"idset":"520","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7113988,"name":"drain","context":{"idset":"521","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7124901,"name":"drain","context":{"idset":"522","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7138093,"name":"drain","context":{"idset":"523","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7151024,"name":"drain","context":{"idset":"524","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7164006,"name":"drain","context":{"idset":"525","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7176726,"name":"drain","context":{"idset":"526","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7190087,"name":"drain","context":{"idset":"527","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7203488,"name":"drain","context":{"idset":"528","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7216754,"name":"drain","context":{"idset":"529","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7230082,"name":"drain","context":{"idset":"530","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7242186,"name":"drain","context":{"idset":"531","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7254941,"name":"drain","context":{"idset":"532","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7267582,"name":"drain","context":{"idset":"533","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7280757,"name":"drain","context":{"idset":"534","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7293236,"name":"drain","context":{"idset":"535","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7305739,"name":"drain","context":{"idset":"536","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7318401,"name":"drain","context":{"idset":"537","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7330952,"name":"drain","context":{"idset":"538","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7343524,"name":"drain","context":{"idset":"539","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7355304,"name":"drain","context":{"idset":"540","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7366977,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7379491,"name":"drain","context":{"idset":"566","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7391269,"name":"drain","context":{"idset":"567","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7403324,"name":"drain","context":{"idset":"568","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7415051,"name":"drain","context":{"idset":"569","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.742703,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7439213,"name":"drain","context":{"idset":"571","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7450745,"name":"drain","context":{"idset":"572","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7462406,"name":"drain","context":{"idset":"573","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7475586,"name":"drain","context":{"idset":"574","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7488201,"name":"drain","context":{"idset":"575","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7500858,"name":"drain","context":{"idset":"576","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7513628,"name":"drain","context":{"idset":"577","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7524936,"name":"drain","context":{"idset":"578","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7536638,"name":"drain","context":{"idset":"579","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7558451,"name":"drain","context":{"idset":"580","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7570727,"name":"drain","context":{"idset":"581","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7582784,"name":"drain","context":{"idset":"582","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7594929,"name":"drain","context":{"idset":"583","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7607639,"name":"drain","context":{"idset":"584","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7621121,"name":"drain","context":{"idset":"585","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7634597,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7646883,"name":"drain","context":{"idset":"587","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7659011,"name":"drain","context":{"idset":"588","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7672513,"name":"drain","context":{"idset":"757","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7686422,"name":"drain","context":{"idset":"795","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7699728,"name":"drain","context":{"idset":"796","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7732935,"name":"drain","context":{"idset":"815","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7746093,"name":"drain","context":{"idset":"816","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7769077,"name":"drain","context":{"idset":"833","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7781959,"name":"drain","context":{"idset":"834","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7794733,"name":"drain","context":{"idset":"841","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7807305,"name":"drain","context":{"idset":"842","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7819908,"name":"drain","context":{"idset":"871","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7831533,"name":"drain","context":{"idset":"872","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7843587,"name":"drain","context":{"idset":"875","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7855587,"name":"drain","context":{"idset":"876","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7869306,"name":"drain","context":{"idset":"949","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7881563,"name":"drain","context":{"idset":"950","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7894225,"name":"drain","context":{"idset":"951","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7907717,"name":"drain","context":{"idset":"952","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7921422,"name":"drain","context":{"idset":"953","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7935216,"name":"drain","context":{"idset":"954","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7948883,"name":"drain","context":{"idset":"963","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7962596,"name":"drain","context":{"idset":"964","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.797621,"name":"drain","context":{"idset":"973","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.7989511,"name":"drain","context":{"idset":"974","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8003523,"name":"drain","context":{"idset":"975","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8017106,"name":"drain","context":{"idset":"976","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8030772,"name":"drain","context":{"idset":"977","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8044603,"name":"drain","context":{"idset":"978","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8057117,"name":"drain","context":{"idset":"9205","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8070283,"name":"drain","context":{"idset":"9206","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8084259,"name":"drain","context":{"idset":"9207","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8097253,"name":"drain","context":{"idset":"9208","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8110461,"name":"drain","context":{"idset":"9209","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8123362,"name":"drain","context":{"idset":"9210","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8137231,"name":"drain","context":{"idset":"9211","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8151016,"name":"drain","context":{"idset":"9212","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8164577,"name":"drain","context":{"idset":"9213","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8177259,"name":"drain","context":{"idset":"9214","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8189702,"name":"drain","context":{"idset":"9215","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8203135,"name":"drain","context":{"idset":"9216","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8216908,"name":"drain","context":{"idset":"9217","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8230724,"name":"drain","context":{"idset":"9218","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8244693,"name":"drain","context":{"idset":"9219","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8258429,"name":"drain","context":{"idset":"9220","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8272252,"name":"drain","context":{"idset":"9221","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8281548,"name":"drain","context":{"idset":"9222","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8290222,"name":"drain","context":{"idset":"9223","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8297491,"name":"drain","context":{"idset":"9224","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8305154,"name":"drain","context":{"idset":"9225","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8317361,"name":"drain","context":{"idset":"9226","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8329933,"name":"drain","context":{"idset":"9227","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8343253,"name":"drain","context":{"idset":"9228","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8356292,"name":"drain","context":{"idset":"9229","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8369732,"name":"drain","context":{"idset":"9230","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8382995,"name":"drain","context":{"idset":"9231","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8396041,"name":"drain","context":{"idset":"9232","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8409154,"name":"drain","context":{"idset":"9233","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8421977,"name":"drain","context":{"idset":"9234","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8434913,"name":"drain","context":{"idset":"9235","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8447876,"name":"drain","context":{"idset":"9236","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8460104,"name":"drain","context":{"idset":"9237","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8472328,"name":"drain","context":{"idset":"9238","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8484964,"name":"drain","context":{"idset":"9239","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.849575,"name":"drain","context":{"idset":"9240","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8508472,"name":"drain","context":{"idset":"9241","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8521354,"name":"drain","context":{"idset":"9242","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8533933,"name":"drain","context":{"idset":"9243","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8545723,"name":"drain","context":{"idset":"9244","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8557596,"name":"drain","context":{"idset":"9245","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8569193,"name":"drain","context":{"idset":"9246","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8580897,"name":"drain","context":{"idset":"9247","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8592758,"name":"drain","context":{"idset":"9248","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8605103,"name":"drain","context":{"idset":"9249","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8617232,"name":"drain","context":{"idset":"9250","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8629198,"name":"drain","context":{"idset":"9251","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8641169,"name":"drain","context":{"idset":"9252","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8653028,"name":"drain","context":{"idset":"9253","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8664708,"name":"drain","context":{"idset":"9254","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8677573,"name":"drain","context":{"idset":"9255","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8690593,"name":"drain","context":{"idset":"9256","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8703465,"name":"drain","context":{"idset":"9257","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.871629,"name":"drain","context":{"idset":"9258","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8728943,"name":"drain","context":{"idset":"9259","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8741,"name":"drain","context":{"idset":"9260","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8753612,"name":"drain","context":{"idset":"9261","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8765926,"name":"drain","context":{"idset":"9262","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8777947,"name":"drain","context":{"idset":"9263","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8789604,"name":"drain","context":{"idset":"9264","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8802967,"name":"drain","context":{"idset":"9265","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8815222,"name":"drain","context":{"idset":"9266","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8826528,"name":"drain","context":{"idset":"9267","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8838546,"name":"drain","context":{"idset":"9268","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8849549,"name":"drain","context":{"idset":"9269","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8861947,"name":"drain","context":{"idset":"9270","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8874762,"name":"drain","context":{"idset":"9271","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8887105,"name":"drain","context":{"idset":"9272","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8899593,"name":"drain","context":{"idset":"9273","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8911641,"name":"drain","context":{"idset":"9274","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8923934,"name":"drain","context":{"idset":"9275","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8936889,"name":"drain","context":{"idset":"9276","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8950202,"name":"drain","context":{"idset":"9277","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8962917,"name":"drain","context":{"idset":"9278","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8974926,"name":"drain","context":{"idset":"9279","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8987274,"name":"drain","context":{"idset":"9280","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.8999739,"name":"drain","context":{"idset":"9281","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9012105,"name":"drain","context":{"idset":"9282","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9025431,"name":"drain","context":{"idset":"9283","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9038293,"name":"drain","context":{"idset":"9284","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9050748,"name":"drain","context":{"idset":"9285","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9062881,"name":"drain","context":{"idset":"9286","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9075451,"name":"drain","context":{"idset":"9287","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9084971,"name":"drain","context":{"idset":"9288","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9097805,"name":"drain","context":{"idset":"9289","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9110701,"name":"drain","context":{"idset":"9290","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9123416,"name":"drain","context":{"idset":"9291","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9136226,"name":"drain","context":{"idset":"9292","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9148629,"name":"drain","context":{"idset":"9293","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9161615,"name":"drain","context":{"idset":"9294","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9174497,"name":"drain","context":{"idset":"9296","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9187171,"name":"drain","context":{"idset":"9297","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.919992,"name":"drain","context":{"idset":"9298","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9208214,"name":"drain","context":{"idset":"9299","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9220517,"name":"drain","context":{"idset":"9300","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9234061,"name":"drain","context":{"idset":"9301","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9246757,"name":"drain","context":{"idset":"9302","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9258785,"name":"drain","context":{"idset":"9303","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9272242,"name":"drain","context":{"idset":"9304","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9285831,"name":"drain","context":{"idset":"9305","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9299464,"name":"drain","context":{"idset":"9306","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9313061,"name":"drain","context":{"idset":"9307","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9326508,"name":"drain","context":{"idset":"9308","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9339967,"name":"drain","context":{"idset":"9309","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.935365,"name":"drain","context":{"idset":"9310","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9367154,"name":"drain","context":{"idset":"9311","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9380558,"name":"drain","context":{"idset":"9312","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9394062,"name":"drain","context":{"idset":"9313","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9407618,"name":"drain","context":{"idset":"9314","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9421215,"name":"drain","context":{"idset":"9315","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.94347,"name":"drain","context":{"idset":"9316","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9448688,"name":"drain","context":{"idset":"9317","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9461646,"name":"drain","context":{"idset":"9318","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9474702,"name":"drain","context":{"idset":"9319","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9487677,"name":"drain","context":{"idset":"9320","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9500651,"name":"drain","context":{"idset":"9321","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9513509,"name":"drain","context":{"idset":"9322","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9526556,"name":"drain","context":{"idset":"9323","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9539464,"name":"drain","context":{"idset":"9324","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9551134,"name":"drain","context":{"idset":"9325","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9563267,"name":"drain","context":{"idset":"9326","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9575033,"name":"drain","context":{"idset":"9327","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9586999,"name":"drain","context":{"idset":"9328","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9598527,"name":"drain","context":{"idset":"9329","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9610136,"name":"drain","context":{"idset":"9330","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9622366,"name":"drain","context":{"idset":"9331","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9634457,"name":"drain","context":{"idset":"9332","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9646707,"name":"drain","context":{"idset":"9333","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9658952,"name":"drain","context":{"idset":"9334","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.967114,"name":"drain","context":{"idset":"9335","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9683497,"name":"drain","context":{"idset":"9336","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9695935,"name":"drain","context":{"idset":"9337","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9708095,"name":"drain","context":{"idset":"9338","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9720242,"name":"drain","context":{"idset":"9339","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9732289,"name":"drain","context":{"idset":"9340","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9744477,"name":"drain","context":{"idset":"9341","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9756341,"name":"drain","context":{"idset":"9342","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9768231,"name":"drain","context":{"idset":"9343","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9780281,"name":"drain","context":{"idset":"9344","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9793439,"name":"drain","context":{"idset":"9345","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9806435,"name":"drain","context":{"idset":"9346","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9819071,"name":"drain","context":{"idset":"9347","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9831536,"name":"drain","context":{"idset":"9348","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9843864,"name":"drain","context":{"idset":"9349","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9856465,"name":"drain","context":{"idset":"9350","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9869621,"name":"drain","context":{"idset":"9351","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9884734,"name":"drain","context":{"idset":"9352","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9899814,"name":"drain","context":{"idset":"9353","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9914412,"name":"drain","context":{"idset":"9354","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9927108,"name":"drain","context":{"idset":"9355","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9941614,"name":"drain","context":{"idset":"9356","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9955478,"name":"drain","context":{"idset":"9357","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9969366,"name":"drain","context":{"idset":"9358","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9983487,"name":"drain","context":{"idset":"9359","reason":"broker was unresponsive"}}
+{"timestamp":1712759593.9997442,"name":"drain","context":{"idset":"9360","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0010943,"name":"drain","context":{"idset":"9361","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0024252,"name":"drain","context":{"idset":"9362","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0035851,"name":"drain","context":{"idset":"9363","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0049493,"name":"drain","context":{"idset":"9364","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0063221,"name":"drain","context":{"idset":"9365","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0076401,"name":"drain","context":{"idset":"9366","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0089953,"name":"drain","context":{"idset":"9367","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0103743,"name":"drain","context":{"idset":"9368","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0117121,"name":"drain","context":{"idset":"9369","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0129719,"name":"drain","context":{"idset":"9370","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.014236,"name":"drain","context":{"idset":"9371","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0155485,"name":"drain","context":{"idset":"9372","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0168815,"name":"drain","context":{"idset":"9373","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.018157,"name":"drain","context":{"idset":"9374","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0194628,"name":"drain","context":{"idset":"9375","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0207739,"name":"drain","context":{"idset":"9376","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0220459,"name":"drain","context":{"idset":"9377","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0233514,"name":"drain","context":{"idset":"9378","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0246911,"name":"drain","context":{"idset":"9379","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.025948,"name":"drain","context":{"idset":"9380","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0272605,"name":"drain","context":{"idset":"9381","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0285852,"name":"drain","context":{"idset":"9382","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.029865,"name":"drain","context":{"idset":"9383","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0311592,"name":"drain","context":{"idset":"9384","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0324841,"name":"drain","context":{"idset":"9385","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0338256,"name":"drain","context":{"idset":"9386","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0351467,"name":"drain","context":{"idset":"9387","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0364754,"name":"drain","context":{"idset":"9388","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0378065,"name":"drain","context":{"idset":"9389","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0390999,"name":"drain","context":{"idset":"9390","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0404434,"name":"drain","context":{"idset":"9391","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.041744,"name":"drain","context":{"idset":"9392","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0430532,"name":"drain","context":{"idset":"9393","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0443659,"name":"drain","context":{"idset":"9394","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0457032,"name":"drain","context":{"idset":"9395","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0470006,"name":"drain","context":{"idset":"9396","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0483601,"name":"drain","context":{"idset":"9397","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0497274,"name":"drain","context":{"idset":"9398","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0510066,"name":"drain","context":{"idset":"9399","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.052299,"name":"drain","context":{"idset":"9400","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0535619,"name":"drain","context":{"idset":"9401","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0548935,"name":"drain","context":{"idset":"9402","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.056222,"name":"drain","context":{"idset":"9403","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0575569,"name":"drain","context":{"idset":"9404","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0589125,"name":"drain","context":{"idset":"9405","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.060401,"name":"drain","context":{"idset":"9406","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0618634,"name":"drain","context":{"idset":"9407","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0631924,"name":"drain","context":{"idset":"9408","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0646591,"name":"drain","context":{"idset":"9409","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0660362,"name":"drain","context":{"idset":"9410","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0672886,"name":"drain","context":{"idset":"9411","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0685775,"name":"drain","context":{"idset":"9412","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0698643,"name":"drain","context":{"idset":"9413","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0711906,"name":"drain","context":{"idset":"9414","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.072515,"name":"drain","context":{"idset":"9415","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0738511,"name":"drain","context":{"idset":"9416","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0749919,"name":"drain","context":{"idset":"9417","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0758126,"name":"drain","context":{"idset":"9418","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0771871,"name":"drain","context":{"idset":"9419","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0786784,"name":"drain","context":{"idset":"9420","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0801468,"name":"drain","context":{"idset":"9421","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0816207,"name":"drain","context":{"idset":"9422","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.082823,"name":"drain","context":{"idset":"9423","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0840893,"name":"drain","context":{"idset":"9424","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0853801,"name":"drain","context":{"idset":"9425","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0865819,"name":"drain","context":{"idset":"9426","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0878916,"name":"drain","context":{"idset":"9427","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0891969,"name":"drain","context":{"idset":"9428","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0904756,"name":"drain","context":{"idset":"9429","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0917954,"name":"drain","context":{"idset":"9430","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0930872,"name":"drain","context":{"idset":"9431","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.094383,"name":"drain","context":{"idset":"9432","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0958209,"name":"drain","context":{"idset":"9433","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.0973725,"name":"drain","context":{"idset":"9434","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.098902,"name":"drain","context":{"idset":"9435","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1003966,"name":"drain","context":{"idset":"9436","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1017311,"name":"drain","context":{"idset":"9437","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1030772,"name":"drain","context":{"idset":"9438","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1043344,"name":"drain","context":{"idset":"9439","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1056058,"name":"drain","context":{"idset":"9440","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1068809,"name":"drain","context":{"idset":"9441","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1081138,"name":"drain","context":{"idset":"9442","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1090744,"name":"drain","context":{"idset":"9443","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1102378,"name":"drain","context":{"idset":"9444","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1116142,"name":"drain","context":{"idset":"9445","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1129467,"name":"drain","context":{"idset":"9446","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1144159,"name":"drain","context":{"idset":"9449","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1159334,"name":"drain","context":{"idset":"9450","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1173103,"name":"drain","context":{"idset":"9451","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1186368,"name":"drain","context":{"idset":"9452","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1199517,"name":"drain","context":{"idset":"9453","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.121285,"name":"drain","context":{"idset":"9454","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1223495,"name":"drain","context":{"idset":"9455","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1235366,"name":"drain","context":{"idset":"9456","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1248519,"name":"drain","context":{"idset":"9457","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.125901,"name":"drain","context":{"idset":"9458","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.127141,"name":"drain","context":{"idset":"9459","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1280234,"name":"drain","context":{"idset":"9460","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1288967,"name":"drain","context":{"idset":"9461","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1299999,"name":"drain","context":{"idset":"9462","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.131156,"name":"drain","context":{"idset":"9463","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1320579,"name":"drain","context":{"idset":"9464","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1329162,"name":"drain","context":{"idset":"9465","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1340797,"name":"drain","context":{"idset":"9466","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1351943,"name":"drain","context":{"idset":"9467","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1365674,"name":"drain","context":{"idset":"9468","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1378124,"name":"drain","context":{"idset":"9469","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1391287,"name":"drain","context":{"idset":"9470","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1405184,"name":"drain","context":{"idset":"9471","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1417966,"name":"drain","context":{"idset":"9473","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1427488,"name":"drain","context":{"idset":"9474","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1436734,"name":"drain","context":{"idset":"9475","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1448946,"name":"drain","context":{"idset":"9476","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1458082,"name":"drain","context":{"idset":"9477","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1467197,"name":"drain","context":{"idset":"9478","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1477125,"name":"drain","context":{"idset":"9479","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1490746,"name":"drain","context":{"idset":"9480","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1504462,"name":"drain","context":{"idset":"9481","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1518574,"name":"drain","context":{"idset":"9482","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1532786,"name":"drain","context":{"idset":"9483","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1546841,"name":"drain","context":{"idset":"9484","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1560638,"name":"drain","context":{"idset":"9485","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1574559,"name":"drain","context":{"idset":"9486","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1588411,"name":"drain","context":{"idset":"9487","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1602237,"name":"drain","context":{"idset":"9488","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1616144,"name":"drain","context":{"idset":"9489","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1629846,"name":"drain","context":{"idset":"9490","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1643198,"name":"drain","context":{"idset":"9491","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1656256,"name":"drain","context":{"idset":"9492","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1669908,"name":"drain","context":{"idset":"9493","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1682882,"name":"drain","context":{"idset":"9494","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1695573,"name":"drain","context":{"idset":"9495","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.170526,"name":"drain","context":{"idset":"9496","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1719906,"name":"drain","context":{"idset":"9497","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1734498,"name":"drain","context":{"idset":"9498","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.174911,"name":"drain","context":{"idset":"9499","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1763778,"name":"drain","context":{"idset":"9500","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1778471,"name":"drain","context":{"idset":"9501","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1793225,"name":"drain","context":{"idset":"9502","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1807957,"name":"drain","context":{"idset":"9503","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1817563,"name":"drain","context":{"idset":"9504","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1826882,"name":"drain","context":{"idset":"9505","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1836004,"name":"drain","context":{"idset":"9506","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1845891,"name":"drain","context":{"idset":"9507","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.18589,"name":"drain","context":{"idset":"9508","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1872535,"name":"drain","context":{"idset":"9509","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1886368,"name":"drain","context":{"idset":"9510","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1899941,"name":"drain","context":{"idset":"9511","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1913388,"name":"drain","context":{"idset":"9512","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.192766,"name":"drain","context":{"idset":"9513","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1941631,"name":"drain","context":{"idset":"9514","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1955636,"name":"drain","context":{"idset":"9515","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1969705,"name":"drain","context":{"idset":"9516","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1983578,"name":"drain","context":{"idset":"9517","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.1997442,"name":"drain","context":{"idset":"9518","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2011306,"name":"drain","context":{"idset":"9519","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.202517,"name":"drain","context":{"idset":"9520","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2038374,"name":"drain","context":{"idset":"9521","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2051494,"name":"drain","context":{"idset":"9522","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.206512,"name":"drain","context":{"idset":"9523","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2078671,"name":"drain","context":{"idset":"9524","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.209348,"name":"drain","context":{"idset":"9525","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2108092,"name":"drain","context":{"idset":"9526","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2122445,"name":"drain","context":{"idset":"9527","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.213728,"name":"drain","context":{"idset":"9528","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2151766,"name":"drain","context":{"idset":"9529","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2166383,"name":"drain","context":{"idset":"9530","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2181189,"name":"drain","context":{"idset":"9531","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.219599,"name":"drain","context":{"idset":"9532","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.220876,"name":"drain","context":{"idset":"9533","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2218468,"name":"drain","context":{"idset":"9534","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2231047,"name":"drain","context":{"idset":"9536","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2241268,"name":"drain","context":{"idset":"9537","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2254629,"name":"drain","context":{"idset":"9538","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2268329,"name":"drain","context":{"idset":"9539","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2282271,"name":"drain","context":{"idset":"9540","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2296472,"name":"drain","context":{"idset":"9541","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2311034,"name":"drain","context":{"idset":"9542","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2325149,"name":"drain","context":{"idset":"9543","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.233911,"name":"drain","context":{"idset":"9544","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2353418,"name":"drain","context":{"idset":"9545","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2367551,"name":"drain","context":{"idset":"9546","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2381597,"name":"drain","context":{"idset":"9547","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2395802,"name":"drain","context":{"idset":"9548","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2409732,"name":"drain","context":{"idset":"9549","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2423308,"name":"drain","context":{"idset":"9550","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2436593,"name":"drain","context":{"idset":"9551","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2450337,"name":"drain","context":{"idset":"9552","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2463212,"name":"drain","context":{"idset":"9553","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2478008,"name":"drain","context":{"idset":"9554","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2493026,"name":"drain","context":{"idset":"9555","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.250797,"name":"drain","context":{"idset":"9556","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2523124,"name":"drain","context":{"idset":"9557","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2537971,"name":"drain","context":{"idset":"9558","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2552831,"name":"drain","context":{"idset":"9559","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2567642,"name":"drain","context":{"idset":"9560","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2580993,"name":"drain","context":{"idset":"9563","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2594483,"name":"drain","context":{"idset":"9564","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2607524,"name":"drain","context":{"idset":"9565","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2621102,"name":"drain","context":{"idset":"9566","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2631369,"name":"drain","context":{"idset":"9567","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.264128,"name":"drain","context":{"idset":"9568","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2651615,"name":"drain","context":{"idset":"9569","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2660494,"name":"drain","context":{"idset":"9570","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2669208,"name":"drain","context":{"idset":"9571","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2678108,"name":"drain","context":{"idset":"9572","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2686639,"name":"drain","context":{"idset":"9573","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2697551,"name":"drain","context":{"idset":"9574","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2709417,"name":"drain","context":{"idset":"9575","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2718394,"name":"drain","context":{"idset":"9576","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2727215,"name":"drain","context":{"idset":"9577","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2735932,"name":"drain","context":{"idset":"9578","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2744403,"name":"drain","context":{"idset":"9579","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2753456,"name":"drain","context":{"idset":"9580","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2761705,"name":"drain","context":{"idset":"9581","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2770915,"name":"drain","context":{"idset":"9582","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2784448,"name":"drain","context":{"idset":"9583","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.27983,"name":"drain","context":{"idset":"9584","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2808073,"name":"drain","context":{"idset":"9585","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2817714,"name":"drain","context":{"idset":"9586","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2826765,"name":"drain","context":{"idset":"9587","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2835693,"name":"drain","context":{"idset":"9588","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2844617,"name":"drain","context":{"idset":"9589","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2853241,"name":"drain","context":{"idset":"9590","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2861938,"name":"drain","context":{"idset":"9591","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2871182,"name":"drain","context":{"idset":"9592","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2881408,"name":"drain","context":{"idset":"9593","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2890732,"name":"drain","context":{"idset":"9594","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2899377,"name":"drain","context":{"idset":"9595","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2907951,"name":"drain","context":{"idset":"9596","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2916656,"name":"drain","context":{"idset":"9597","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2928948,"name":"drain","context":{"idset":"9598","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.29426,"name":"drain","context":{"idset":"9599","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.295536,"name":"drain","context":{"idset":"9600","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2968626,"name":"drain","context":{"idset":"9601","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2982428,"name":"drain","context":{"idset":"9602","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.2996528,"name":"drain","context":{"idset":"9603","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3009019,"name":"drain","context":{"idset":"9604","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.301774,"name":"drain","context":{"idset":"9605","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3028276,"name":"drain","context":{"idset":"9606","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3041649,"name":"drain","context":{"idset":"9607","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3054132,"name":"drain","context":{"idset":"9608","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3067851,"name":"drain","context":{"idset":"9609","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3081789,"name":"drain","context":{"idset":"9610","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3095803,"name":"drain","context":{"idset":"9611","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3109055,"name":"drain","context":{"idset":"9612","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3123486,"name":"drain","context":{"idset":"9613","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.313745,"name":"drain","context":{"idset":"9614","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3151722,"name":"drain","context":{"idset":"9615","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.316638,"name":"drain","context":{"idset":"9616","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3181074,"name":"drain","context":{"idset":"9617","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3195596,"name":"drain","context":{"idset":"9618","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3209941,"name":"drain","context":{"idset":"9619","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3224289,"name":"drain","context":{"idset":"9620","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3238554,"name":"drain","context":{"idset":"9621","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3253045,"name":"drain","context":{"idset":"9622","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3270228,"name":"drain","context":{"idset":"9623","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3283989,"name":"drain","context":{"idset":"9624","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3297541,"name":"drain","context":{"idset":"9625","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3311539,"name":"drain","context":{"idset":"9626","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3325496,"name":"drain","context":{"idset":"9627","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3339388,"name":"drain","context":{"idset":"9628","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3353498,"name":"drain","context":{"idset":"9629","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3367047,"name":"drain","context":{"idset":"9630","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3378117,"name":"drain","context":{"idset":"9631","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3389573,"name":"drain","context":{"idset":"9632","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3399036,"name":"drain","context":{"idset":"9633","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3408895,"name":"drain","context":{"idset":"9634","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3418694,"name":"drain","context":{"idset":"9635","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3427472,"name":"drain","context":{"idset":"9636","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3438632,"name":"drain","context":{"idset":"9637","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3451748,"name":"drain","context":{"idset":"9638","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3465171,"name":"drain","context":{"idset":"9639","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3478549,"name":"drain","context":{"idset":"9640","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3491654,"name":"drain","context":{"idset":"9641","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3504934,"name":"drain","context":{"idset":"9642","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3518126,"name":"drain","context":{"idset":"9643","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3531373,"name":"drain","context":{"idset":"9644","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3544781,"name":"drain","context":{"idset":"9645","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3558691,"name":"drain","context":{"idset":"9646","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3572867,"name":"drain","context":{"idset":"9647","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3586493,"name":"drain","context":{"idset":"9648","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3599479,"name":"drain","context":{"idset":"9649","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.361378,"name":"drain","context":{"idset":"9650","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3627994,"name":"drain","context":{"idset":"9651","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3642559,"name":"drain","context":{"idset":"9652","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3657386,"name":"drain","context":{"idset":"9653","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3673043,"name":"drain","context":{"idset":"9654","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3688459,"name":"drain","context":{"idset":"9655","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3704097,"name":"drain","context":{"idset":"9656","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3719611,"name":"drain","context":{"idset":"9657","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3735094,"name":"drain","context":{"idset":"9658","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3748667,"name":"drain","context":{"idset":"9659","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3762867,"name":"drain","context":{"idset":"9660","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3774991,"name":"drain","context":{"idset":"9661","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3788733,"name":"drain","context":{"idset":"9662","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3801672,"name":"drain","context":{"idset":"9663","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.381233,"name":"drain","context":{"idset":"9664","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3826294,"name":"drain","context":{"idset":"9665","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.384166,"name":"drain","context":{"idset":"9666","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3857386,"name":"drain","context":{"idset":"9667","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3873212,"name":"drain","context":{"idset":"9668","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3888552,"name":"drain","context":{"idset":"9669","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.390434,"name":"drain","context":{"idset":"9670","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3916862,"name":"drain","context":{"idset":"9671","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3931358,"name":"drain","context":{"idset":"9672","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3946662,"name":"drain","context":{"idset":"9673","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3962045,"name":"drain","context":{"idset":"9674","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.3977554,"name":"drain","context":{"idset":"9675","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.399297,"name":"drain","context":{"idset":"9676","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.400811,"name":"drain","context":{"idset":"9677","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4023457,"name":"drain","context":{"idset":"9678","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4038711,"name":"drain","context":{"idset":"9679","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4052966,"name":"drain","context":{"idset":"9680","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4063709,"name":"drain","context":{"idset":"9681","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4077003,"name":"drain","context":{"idset":"9682","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4091246,"name":"drain","context":{"idset":"9683","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4102352,"name":"drain","context":{"idset":"9684","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4112947,"name":"drain","context":{"idset":"9685","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4122598,"name":"drain","context":{"idset":"9686","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.413311,"name":"drain","context":{"idset":"9687","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4143202,"name":"drain","context":{"idset":"9688","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4152527,"name":"drain","context":{"idset":"9689","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4162316,"name":"drain","context":{"idset":"9690","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4173112,"name":"drain","context":{"idset":"9691","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.418649,"name":"drain","context":{"idset":"9692","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4199803,"name":"drain","context":{"idset":"9693","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4213297,"name":"drain","context":{"idset":"9694","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4226561,"name":"drain","context":{"idset":"9695","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4239948,"name":"drain","context":{"idset":"9696","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4253428,"name":"drain","context":{"idset":"9697","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4266722,"name":"drain","context":{"idset":"9698","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4280014,"name":"drain","context":{"idset":"9699","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4292941,"name":"drain","context":{"idset":"9700","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4306839,"name":"drain","context":{"idset":"9701","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4320002,"name":"drain","context":{"idset":"9702","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4333074,"name":"drain","context":{"idset":"9703","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.434623,"name":"drain","context":{"idset":"9704","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4359326,"name":"drain","context":{"idset":"9705","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4372289,"name":"drain","context":{"idset":"9706","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.438545,"name":"drain","context":{"idset":"9707","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4398644,"name":"drain","context":{"idset":"9708","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4411743,"name":"drain","context":{"idset":"9709","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4425249,"name":"drain","context":{"idset":"9710","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4438605,"name":"drain","context":{"idset":"9711","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4453015,"name":"drain","context":{"idset":"9712","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4468019,"name":"drain","context":{"idset":"9713","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4481833,"name":"drain","context":{"idset":"9714","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4496584,"name":"drain","context":{"idset":"9715","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.451056,"name":"drain","context":{"idset":"9716","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.452471,"name":"drain","context":{"idset":"9717","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4543688,"name":"drain","context":{"idset":"9718","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4556954,"name":"drain","context":{"idset":"9719","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4568341,"name":"drain","context":{"idset":"9720","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4583001,"name":"drain","context":{"idset":"9721","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.459667,"name":"drain","context":{"idset":"9722","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4610481,"name":"drain","context":{"idset":"9723","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4623296,"name":"drain","context":{"idset":"9724","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4636166,"name":"drain","context":{"idset":"9725","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4649987,"name":"drain","context":{"idset":"9726","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4664109,"name":"drain","context":{"idset":"9727","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4675195,"name":"drain","context":{"idset":"9728","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4689679,"name":"drain","context":{"idset":"9729","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4703486,"name":"drain","context":{"idset":"9730","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4717336,"name":"drain","context":{"idset":"9731","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4731467,"name":"drain","context":{"idset":"9732","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4745893,"name":"drain","context":{"idset":"9733","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4759631,"name":"drain","context":{"idset":"9734","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4808307,"name":"drain","context":{"idset":"9735","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4823718,"name":"drain","context":{"idset":"9736","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4838932,"name":"drain","context":{"idset":"9737","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4853935,"name":"drain","context":{"idset":"9738","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4868853,"name":"drain","context":{"idset":"9739","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4884131,"name":"drain","context":{"idset":"9740","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4899018,"name":"drain","context":{"idset":"9741","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4913867,"name":"drain","context":{"idset":"9742","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4928901,"name":"drain","context":{"idset":"9743","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.494329,"name":"drain","context":{"idset":"9744","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4957445,"name":"drain","context":{"idset":"9745","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4971883,"name":"drain","context":{"idset":"9746","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.4985969,"name":"drain","context":{"idset":"9747","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5001714,"name":"drain","context":{"idset":"9748","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5017579,"name":"drain","context":{"idset":"9749","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5046687,"name":"drain","context":{"idset":"9751","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5062327,"name":"drain","context":{"idset":"9752","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5077951,"name":"drain","context":{"idset":"9753","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5093834,"name":"drain","context":{"idset":"9754","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5109048,"name":"drain","context":{"idset":"9755","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5124204,"name":"drain","context":{"idset":"9756","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5137289,"name":"drain","context":{"idset":"9757","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5150414,"name":"drain","context":{"idset":"9758","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5161445,"name":"drain","context":{"idset":"9759","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5175855,"name":"drain","context":{"idset":"9760","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5188029,"name":"drain","context":{"idset":"9761","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5202286,"name":"drain","context":{"idset":"9762","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.521699,"name":"drain","context":{"idset":"9763","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5231721,"name":"drain","context":{"idset":"9764","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5246594,"name":"drain","context":{"idset":"9765","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5259926,"name":"drain","context":{"idset":"9766","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5274374,"name":"drain","context":{"idset":"9767","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5288889,"name":"drain","context":{"idset":"9768","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5303822,"name":"drain","context":{"idset":"9769","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5318458,"name":"drain","context":{"idset":"9770","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5333221,"name":"drain","context":{"idset":"9771","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5348418,"name":"drain","context":{"idset":"9772","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5363722,"name":"drain","context":{"idset":"9773","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.537925,"name":"drain","context":{"idset":"9774","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5394695,"name":"drain","context":{"idset":"9775","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5410142,"name":"drain","context":{"idset":"9776","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5425534,"name":"drain","context":{"idset":"9777","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5440674,"name":"drain","context":{"idset":"9778","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5456192,"name":"drain","context":{"idset":"9779","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5471575,"name":"drain","context":{"idset":"9780","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5487061,"name":"drain","context":{"idset":"9781","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5502441,"name":"drain","context":{"idset":"9782","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5519111,"name":"drain","context":{"idset":"9783","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5535254,"name":"drain","context":{"idset":"9784","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5550325,"name":"drain","context":{"idset":"9785","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5565982,"name":"drain","context":{"idset":"9786","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5579562,"name":"drain","context":{"idset":"9791","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5596559,"name":"drain","context":{"idset":"9792","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5609877,"name":"drain","context":{"idset":"9793","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5623441,"name":"drain","context":{"idset":"9794","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5636694,"name":"drain","context":{"idset":"9795","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5650041,"name":"drain","context":{"idset":"9796","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5663552,"name":"drain","context":{"idset":"9797","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5677702,"name":"drain","context":{"idset":"9798","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5690992,"name":"drain","context":{"idset":"9799","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5704403,"name":"drain","context":{"idset":"9800","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5717487,"name":"drain","context":{"idset":"9801","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5730648,"name":"drain","context":{"idset":"9802","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5743928,"name":"drain","context":{"idset":"9803","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5756922,"name":"drain","context":{"idset":"9804","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5773003,"name":"drain","context":{"idset":"9805","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.578944,"name":"drain","context":{"idset":"9806","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5804281,"name":"drain","context":{"idset":"9807","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5817688,"name":"drain","context":{"idset":"9808","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5832562,"name":"drain","context":{"idset":"9809","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5849478,"name":"drain","context":{"idset":"9810","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5866859,"name":"drain","context":{"idset":"9811","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5884044,"name":"drain","context":{"idset":"9812","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.589951,"name":"drain","context":{"idset":"9813","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5914521,"name":"drain","context":{"idset":"9814","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5926573,"name":"drain","context":{"idset":"9815","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5939775,"name":"drain","context":{"idset":"9816","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5951426,"name":"drain","context":{"idset":"9817","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5964446,"name":"drain","context":{"idset":"9818","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5976529,"name":"drain","context":{"idset":"9819","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.5988312,"name":"drain","context":{"idset":"9820","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6000175,"name":"drain","context":{"idset":"9821","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6011624,"name":"drain","context":{"idset":"9822","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6025059,"name":"drain","context":{"idset":"9823","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6038942,"name":"drain","context":{"idset":"9824","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6053329,"name":"drain","context":{"idset":"9825","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.606796,"name":"drain","context":{"idset":"9826","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6083038,"name":"drain","context":{"idset":"9827","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6097817,"name":"drain","context":{"idset":"9828","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6112599,"name":"drain","context":{"idset":"9829","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6125901,"name":"drain","context":{"idset":"9830","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6139085,"name":"drain","context":{"idset":"9831","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6153982,"name":"drain","context":{"idset":"9832","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6168711,"name":"drain","context":{"idset":"9833","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6182573,"name":"drain","context":{"idset":"9834","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6196454,"name":"drain","context":{"idset":"9835","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6209209,"name":"drain","context":{"idset":"9836","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.622401,"name":"drain","context":{"idset":"9837","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6237268,"name":"drain","context":{"idset":"9838","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6251984,"name":"drain","context":{"idset":"9839","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6294169,"name":"drain","context":{"idset":"9840","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6306396,"name":"drain","context":{"idset":"9841","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6318319,"name":"drain","context":{"idset":"9842","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6332417,"name":"drain","context":{"idset":"9843","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6346552,"name":"drain","context":{"idset":"9844","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6361458,"name":"drain","context":{"idset":"9845","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6376872,"name":"drain","context":{"idset":"9846","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6392009,"name":"drain","context":{"idset":"9847","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6407912,"name":"drain","context":{"idset":"9848","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6422999,"name":"drain","context":{"idset":"9849","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6438084,"name":"drain","context":{"idset":"9850","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6453056,"name":"drain","context":{"idset":"9851","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.646791,"name":"drain","context":{"idset":"9852","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6483028,"name":"drain","context":{"idset":"9853","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6498239,"name":"drain","context":{"idset":"9854","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.651377,"name":"drain","context":{"idset":"9855","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.652884,"name":"drain","context":{"idset":"9856","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6544549,"name":"drain","context":{"idset":"9857","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6559608,"name":"drain","context":{"idset":"9858","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6574907,"name":"drain","context":{"idset":"9859","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6589899,"name":"drain","context":{"idset":"9860","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6605155,"name":"drain","context":{"idset":"9861","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6620295,"name":"drain","context":{"idset":"9862","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.663553,"name":"drain","context":{"idset":"9863","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6650574,"name":"drain","context":{"idset":"9864","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6665623,"name":"drain","context":{"idset":"9865","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6680679,"name":"drain","context":{"idset":"9866","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6696043,"name":"drain","context":{"idset":"9867","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6710923,"name":"drain","context":{"idset":"9868","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6725874,"name":"drain","context":{"idset":"9869","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6741276,"name":"drain","context":{"idset":"9870","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6756444,"name":"drain","context":{"idset":"9871","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6771531,"name":"drain","context":{"idset":"9872","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6787083,"name":"drain","context":{"idset":"9873","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6801941,"name":"drain","context":{"idset":"9874","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6815374,"name":"drain","context":{"idset":"9875","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6828327,"name":"drain","context":{"idset":"9876","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6840208,"name":"drain","context":{"idset":"9877","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6853037,"name":"drain","context":{"idset":"9878","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6866927,"name":"drain","context":{"idset":"9879","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6879272,"name":"drain","context":{"idset":"9880","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6891813,"name":"drain","context":{"idset":"9881","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6908,"name":"drain","context":{"idset":"9882","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6923883,"name":"drain","context":{"idset":"9883","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6939399,"name":"drain","context":{"idset":"9884","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6955078,"name":"drain","context":{"idset":"9885","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6969419,"name":"drain","context":{"idset":"9886","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.6985164,"name":"drain","context":{"idset":"9887","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7000761,"name":"drain","context":{"idset":"9888","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7015786,"name":"drain","context":{"idset":"9889","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7028451,"name":"drain","context":{"idset":"9890","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7041051,"name":"drain","context":{"idset":"9891","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7054183,"name":"drain","context":{"idset":"9892","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7069693,"name":"drain","context":{"idset":"9893","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.70855,"name":"drain","context":{"idset":"9894","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7101035,"name":"drain","context":{"idset":"9895","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7116349,"name":"drain","context":{"idset":"9896","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7131498,"name":"drain","context":{"idset":"9897","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7145405,"name":"drain","context":{"idset":"9898","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7157233,"name":"drain","context":{"idset":"9899","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7169528,"name":"drain","context":{"idset":"9900","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7184954,"name":"drain","context":{"idset":"9901","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7200634,"name":"drain","context":{"idset":"9902","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7215741,"name":"drain","context":{"idset":"9903","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7228711,"name":"drain","context":{"idset":"9904","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7244177,"name":"drain","context":{"idset":"9905","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7258129,"name":"drain","context":{"idset":"9906","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7273254,"name":"drain","context":{"idset":"9907","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7288959,"name":"drain","context":{"idset":"9908","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7304497,"name":"drain","context":{"idset":"9909","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.731977,"name":"drain","context":{"idset":"9910","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7335541,"name":"drain","context":{"idset":"9911","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7350354,"name":"drain","context":{"idset":"9912","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.736613,"name":"drain","context":{"idset":"9913","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7380929,"name":"drain","context":{"idset":"9914","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7396069,"name":"drain","context":{"idset":"9915","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7409987,"name":"drain","context":{"idset":"9916","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7423234,"name":"drain","context":{"idset":"9917","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.743732,"name":"drain","context":{"idset":"9918","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7448657,"name":"drain","context":{"idset":"9919","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7460756,"name":"drain","context":{"idset":"9920","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7473276,"name":"drain","context":{"idset":"9921","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7488849,"name":"drain","context":{"idset":"9922","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7503393,"name":"drain","context":{"idset":"9923","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7515872,"name":"drain","context":{"idset":"9924","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7527611,"name":"drain","context":{"idset":"9925","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7539601,"name":"drain","context":{"idset":"9926","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7551396,"name":"drain","context":{"idset":"9927","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7563608,"name":"drain","context":{"idset":"9928","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7576237,"name":"drain","context":{"idset":"9929","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7589562,"name":"drain","context":{"idset":"9930","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7603765,"name":"drain","context":{"idset":"9931","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7619119,"name":"drain","context":{"idset":"9932","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.763494,"name":"drain","context":{"idset":"9933","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7649267,"name":"drain","context":{"idset":"9934","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7666657,"name":"drain","context":{"idset":"9935","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7676907,"name":"drain","context":{"idset":"9936","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7685761,"name":"drain","context":{"idset":"9937","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7694929,"name":"drain","context":{"idset":"9938","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7703884,"name":"drain","context":{"idset":"9939","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7720215,"name":"drain","context":{"idset":"9940","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7736416,"name":"drain","context":{"idset":"9941","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7753978,"name":"drain","context":{"idset":"9942","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7771132,"name":"drain","context":{"idset":"9943","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7787609,"name":"drain","context":{"idset":"9944","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7796915,"name":"drain","context":{"idset":"9945","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7808428,"name":"drain","context":{"idset":"9946","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.78251,"name":"drain","context":{"idset":"9947","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7842293,"name":"drain","context":{"idset":"9948","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7859554,"name":"drain","context":{"idset":"9949","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7872281,"name":"drain","context":{"idset":"9950","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.788255,"name":"drain","context":{"idset":"9951","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7897074,"name":"drain","context":{"idset":"9952","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7914515,"name":"drain","context":{"idset":"9953","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7931695,"name":"drain","context":{"idset":"9954","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7949173,"name":"drain","context":{"idset":"9955","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7966447,"name":"drain","context":{"idset":"9956","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7983787,"name":"drain","context":{"idset":"9957","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.7999685,"name":"drain","context":{"idset":"9958","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8014565,"name":"drain","context":{"idset":"9959","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8031878,"name":"drain","context":{"idset":"9960","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8049281,"name":"drain","context":{"idset":"9961","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8066719,"name":"drain","context":{"idset":"9962","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8084128,"name":"drain","context":{"idset":"9963","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8101447,"name":"drain","context":{"idset":"9964","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.811686,"name":"drain","context":{"idset":"9965","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8128774,"name":"drain","context":{"idset":"9966","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8137789,"name":"drain","context":{"idset":"9967","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8146677,"name":"drain","context":{"idset":"9968","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8155527,"name":"drain","context":{"idset":"9969","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8164451,"name":"drain","context":{"idset":"9970","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8173375,"name":"drain","context":{"idset":"9971","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8182933,"name":"drain","context":{"idset":"9972","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8194685,"name":"drain","context":{"idset":"9973","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8210299,"name":"drain","context":{"idset":"9974","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8223994,"name":"drain","context":{"idset":"9975","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8235574,"name":"drain","context":{"idset":"9976","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8251655,"name":"drain","context":{"idset":"9977","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8268688,"name":"drain","context":{"idset":"9978","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8285882,"name":"drain","context":{"idset":"9979","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8385029,"name":"drain","context":{"idset":"9980","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8401458,"name":"drain","context":{"idset":"9981","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.841717,"name":"drain","context":{"idset":"9982","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8432584,"name":"drain","context":{"idset":"9983","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8449936,"name":"drain","context":{"idset":"9984","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8467798,"name":"drain","context":{"idset":"9985","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8485584,"name":"drain","context":{"idset":"9986","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8501098,"name":"drain","context":{"idset":"9987","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8518193,"name":"drain","context":{"idset":"9988","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8536119,"name":"drain","context":{"idset":"9989","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8553736,"name":"drain","context":{"idset":"9990","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8570695,"name":"drain","context":{"idset":"9991","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8586316,"name":"drain","context":{"idset":"9993","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8601785,"name":"drain","context":{"idset":"9994","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8614388,"name":"drain","context":{"idset":"9995","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8628697,"name":"drain","context":{"idset":"9996","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8646216,"name":"drain","context":{"idset":"9997","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8663597,"name":"drain","context":{"idset":"9998","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8680894,"name":"drain","context":{"idset":"9999","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8696973,"name":"drain","context":{"idset":"10000","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8714621,"name":"drain","context":{"idset":"10001","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8734577,"name":"drain","context":{"idset":"10002","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8829639,"name":"drain","context":{"idset":"10003","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8847454,"name":"drain","context":{"idset":"10004","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8863003,"name":"drain","context":{"idset":"10005","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8878458,"name":"drain","context":{"idset":"10006","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8893754,"name":"drain","context":{"idset":"10007","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8909421,"name":"drain","context":{"idset":"10008","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8919599,"name":"drain","context":{"idset":"10009","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.893389,"name":"drain","context":{"idset":"10010","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8949959,"name":"drain","context":{"idset":"10011","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8965588,"name":"drain","context":{"idset":"10012","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8977075,"name":"drain","context":{"idset":"10013","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.8995011,"name":"drain","context":{"idset":"10014","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9012964,"name":"drain","context":{"idset":"10015","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9030755,"name":"drain","context":{"idset":"10016","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9048891,"name":"drain","context":{"idset":"10017","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.906693,"name":"drain","context":{"idset":"10018","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9085877,"name":"drain","context":{"idset":"10019","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9102182,"name":"drain","context":{"idset":"10020","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9118214,"name":"drain","context":{"idset":"10021","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9130666,"name":"drain","context":{"idset":"10022","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9147441,"name":"drain","context":{"idset":"10023","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9162879,"name":"drain","context":{"idset":"10024","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9178352,"name":"drain","context":{"idset":"10025","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9195828,"name":"drain","context":{"idset":"10026","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9213896,"name":"drain","context":{"idset":"10027","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9230905,"name":"drain","context":{"idset":"10028","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9246795,"name":"drain","context":{"idset":"10029","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9263344,"name":"drain","context":{"idset":"10030","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9280176,"name":"drain","context":{"idset":"10031","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.929678,"name":"drain","context":{"idset":"10032","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.93132,"name":"drain","context":{"idset":"10033","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9329488,"name":"drain","context":{"idset":"10034","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9345677,"name":"drain","context":{"idset":"10035","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9361398,"name":"drain","context":{"idset":"10036","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9377298,"name":"drain","context":{"idset":"10037","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9393134,"name":"drain","context":{"idset":"10038","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9408844,"name":"drain","context":{"idset":"10039","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9426522,"name":"drain","context":{"idset":"10040","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9444611,"name":"drain","context":{"idset":"10041","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9461632,"name":"drain","context":{"idset":"10042","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9477434,"name":"drain","context":{"idset":"10043","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9492846,"name":"drain","context":{"idset":"10044","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9510138,"name":"drain","context":{"idset":"10045","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9529333,"name":"drain","context":{"idset":"10046","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9547517,"name":"drain","context":{"idset":"10047","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9565799,"name":"drain","context":{"idset":"10048","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9583936,"name":"drain","context":{"idset":"10049","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9601982,"name":"drain","context":{"idset":"10050","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9620137,"name":"drain","context":{"idset":"10051","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9633341,"name":"drain","context":{"idset":"10052","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9642351,"name":"drain","context":{"idset":"10053","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9658046,"name":"drain","context":{"idset":"10054","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9673603,"name":"drain","context":{"idset":"10055","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9690816,"name":"drain","context":{"idset":"10056","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9709101,"name":"drain","context":{"idset":"10057","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9727411,"name":"drain","context":{"idset":"10058","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9745829,"name":"drain","context":{"idset":"10059","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9764278,"name":"drain","context":{"idset":"10060","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9781904,"name":"drain","context":{"idset":"10061","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9799144,"name":"drain","context":{"idset":"10062","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9817109,"name":"drain","context":{"idset":"10063","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9833307,"name":"drain","context":{"idset":"10064","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9849212,"name":"drain","context":{"idset":"10065","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9866796,"name":"drain","context":{"idset":"10066","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9884307,"name":"drain","context":{"idset":"10067","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9900713,"name":"drain","context":{"idset":"10068","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9917305,"name":"drain","context":{"idset":"10069","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.993392,"name":"drain","context":{"idset":"10070","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9952312,"name":"drain","context":{"idset":"10071","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.997056,"name":"drain","context":{"idset":"10072","reason":"broker was unresponsive"}}
+{"timestamp":1712759594.9989004,"name":"drain","context":{"idset":"10073","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.000742,"name":"drain","context":{"idset":"10074","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.00248,"name":"drain","context":{"idset":"10075","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0041807,"name":"drain","context":{"idset":"10076","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0059414,"name":"drain","context":{"idset":"10077","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0075924,"name":"drain","context":{"idset":"10078","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0087686,"name":"drain","context":{"idset":"10079","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0105169,"name":"drain","context":{"idset":"10080","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0123742,"name":"drain","context":{"idset":"10081","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0142121,"name":"drain","context":{"idset":"10082","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0160635,"name":"drain","context":{"idset":"10083","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0179179,"name":"drain","context":{"idset":"10084","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0197663,"name":"drain","context":{"idset":"10085","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0216095,"name":"drain","context":{"idset":"10086","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0233533,"name":"drain","context":{"idset":"10087","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0250711,"name":"drain","context":{"idset":"10088","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0269041,"name":"drain","context":{"idset":"10089","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0287471,"name":"drain","context":{"idset":"10090","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0306029,"name":"drain","context":{"idset":"10091","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0324447,"name":"drain","context":{"idset":"10092","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0341778,"name":"drain","context":{"idset":"10093","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0357552,"name":"drain","context":{"idset":"10094","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0373552,"name":"drain","context":{"idset":"10095","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0389404,"name":"drain","context":{"idset":"10096","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0406613,"name":"drain","context":{"idset":"10097","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0425253,"name":"drain","context":{"idset":"10098","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0443742,"name":"drain","context":{"idset":"10101","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0462558,"name":"drain","context":{"idset":"10102","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0480795,"name":"drain","context":{"idset":"10103","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0498211,"name":"drain","context":{"idset":"10104","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0515099,"name":"drain","context":{"idset":"10105","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0530851,"name":"drain","context":{"idset":"10106","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.054769,"name":"drain","context":{"idset":"10107","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0565667,"name":"drain","context":{"idset":"10108","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0584087,"name":"drain","context":{"idset":"10109","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0601764,"name":"drain","context":{"idset":"10110","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0618098,"name":"drain","context":{"idset":"10111","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0635262,"name":"drain","context":{"idset":"10112","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0652299,"name":"drain","context":{"idset":"10113","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0669148,"name":"drain","context":{"idset":"10114","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.068578,"name":"drain","context":{"idset":"10115","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0701966,"name":"drain","context":{"idset":"10116","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0717492,"name":"drain","context":{"idset":"10117","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0733387,"name":"drain","context":{"idset":"10118","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0750182,"name":"drain","context":{"idset":"10119","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0768013,"name":"drain","context":{"idset":"10120","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0786512,"name":"drain","context":{"idset":"10123","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0804827,"name":"drain","context":{"idset":"10124","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0823159,"name":"drain","context":{"idset":"10125","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0839987,"name":"drain","context":{"idset":"10126","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.085669,"name":"drain","context":{"idset":"10127","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0872786,"name":"drain","context":{"idset":"10128","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0889347,"name":"drain","context":{"idset":"10129","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0906553,"name":"drain","context":{"idset":"10130","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.092437,"name":"drain","context":{"idset":"10131","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0941551,"name":"drain","context":{"idset":"10132","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0956864,"name":"drain","context":{"idset":"10133","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.0973582,"name":"drain","context":{"idset":"10134","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.099035,"name":"drain","context":{"idset":"10135","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1007617,"name":"drain","context":{"idset":"10136","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1024747,"name":"drain","context":{"idset":"10137","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1041989,"name":"drain","context":{"idset":"10138","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1060655,"name":"drain","context":{"idset":"10139","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1078088,"name":"drain","context":{"idset":"10140","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1095221,"name":"drain","context":{"idset":"10141","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1110671,"name":"drain","context":{"idset":"10142","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1127825,"name":"drain","context":{"idset":"10143","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1145165,"name":"drain","context":{"idset":"10144","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1161277,"name":"drain","context":{"idset":"10145","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1178565,"name":"drain","context":{"idset":"10146","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1195846,"name":"drain","context":{"idset":"10147","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1212595,"name":"drain","context":{"idset":"10148","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1229248,"name":"drain","context":{"idset":"10149","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1246212,"name":"drain","context":{"idset":"10150","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1262908,"name":"drain","context":{"idset":"10151","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.127877,"name":"drain","context":{"idset":"10152","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1295893,"name":"drain","context":{"idset":"10153","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1312497,"name":"drain","context":{"idset":"10154","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1329825,"name":"drain","context":{"idset":"10155","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1347749,"name":"drain","context":{"idset":"10156","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1366873,"name":"drain","context":{"idset":"10157","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1385806,"name":"drain","context":{"idset":"10158","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1403449,"name":"drain","context":{"idset":"10159","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1421599,"name":"drain","context":{"idset":"10160","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1435313,"name":"drain","context":{"idset":"10161","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.144959,"name":"drain","context":{"idset":"10162","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1463697,"name":"drain","context":{"idset":"10163","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1477709,"name":"drain","context":{"idset":"10164","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.149168,"name":"drain","context":{"idset":"10165","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1505883,"name":"drain","context":{"idset":"10166","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1519942,"name":"drain","context":{"idset":"10167","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1534126,"name":"drain","context":{"idset":"10168","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1548164,"name":"drain","context":{"idset":"10169","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1562192,"name":"drain","context":{"idset":"10170","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1576359,"name":"drain","context":{"idset":"10171","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1590409,"name":"drain","context":{"idset":"10172","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1604578,"name":"drain","context":{"idset":"10173","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1618602,"name":"drain","context":{"idset":"10174","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1632962,"name":"drain","context":{"idset":"10175","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1648324,"name":"drain","context":{"idset":"10176","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1666133,"name":"drain","context":{"idset":"10177","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1682987,"name":"drain","context":{"idset":"10178","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1699822,"name":"drain","context":{"idset":"10179","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.171701,"name":"drain","context":{"idset":"10180","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1734092,"name":"drain","context":{"idset":"10181","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1751761,"name":"drain","context":{"idset":"10182","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.177104,"name":"drain","context":{"idset":"10183","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1790149,"name":"drain","context":{"idset":"10184","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1807108,"name":"drain","context":{"idset":"10185","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1821225,"name":"drain","context":{"idset":"10186","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1838639,"name":"drain","context":{"idset":"10187","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1855831,"name":"drain","context":{"idset":"10188","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.187243,"name":"drain","context":{"idset":"10189","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1891305,"name":"drain","context":{"idset":"10190","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1908345,"name":"drain","context":{"idset":"11779","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.192523,"name":"drain","context":{"idset":"11780","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1941903,"name":"drain","context":{"idset":"11781","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.195878,"name":"drain","context":{"idset":"11782","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1975551,"name":"drain","context":{"idset":"11783","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.1992264,"name":"drain","context":{"idset":"11784","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2008021,"name":"drain","context":{"idset":"11785","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.202318,"name":"drain","context":{"idset":"11786","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2039287,"name":"drain","context":{"idset":"11787","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2056236,"name":"drain","context":{"idset":"11788","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.207381,"name":"drain","context":{"idset":"11789","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2090251,"name":"drain","context":{"idset":"11790","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.210773,"name":"drain","context":{"idset":"11791","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.212451,"name":"drain","context":{"idset":"11792","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2138386,"name":"drain","context":{"idset":"11793","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2153728,"name":"drain","context":{"idset":"11794","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2170169,"name":"drain","context":{"idset":"11795","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2186816,"name":"drain","context":{"idset":"11796","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2202978,"name":"drain","context":{"idset":"11797","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2218812,"name":"drain","context":{"idset":"11798","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2235162,"name":"drain","context":{"idset":"11801","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2251294,"name":"drain","context":{"idset":"11802","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2268019,"name":"drain","context":{"idset":"11803","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2284615,"name":"drain","context":{"idset":"11804","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2301087,"name":"drain","context":{"idset":"11805","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2317662,"name":"drain","context":{"idset":"11806","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2334158,"name":"drain","context":{"idset":"11807","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2350612,"name":"drain","context":{"idset":"11808","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2366636,"name":"drain","context":{"idset":"11809","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2382855,"name":"drain","context":{"idset":"11810","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2399502,"name":"drain","context":{"idset":"11811","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2416213,"name":"drain","context":{"idset":"11812","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2432811,"name":"drain","context":{"idset":"11813","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2449343,"name":"drain","context":{"idset":"11814","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2465651,"name":"drain","context":{"idset":"11815","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.24822,"name":"drain","context":{"idset":"11816","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2498796,"name":"drain","context":{"idset":"11817","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2515366,"name":"drain","context":{"idset":"11818","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2531807,"name":"drain","context":{"idset":"11819","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2548494,"name":"drain","context":{"idset":"11820","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2564924,"name":"drain","context":{"idset":"11821","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.257906,"name":"drain","context":{"idset":"11822","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2594087,"name":"drain","context":{"idset":"11823","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2610343,"name":"drain","context":{"idset":"11824","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2627096,"name":"drain","context":{"idset":"11825","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2643721,"name":"drain","context":{"idset":"11826","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2660332,"name":"drain","context":{"idset":"11827","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2677112,"name":"drain","context":{"idset":"11828","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2693825,"name":"drain","context":{"idset":"11829","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2710207,"name":"drain","context":{"idset":"11830","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2724769,"name":"drain","context":{"idset":"11831","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2733572,"name":"drain","context":{"idset":"11832","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2745974,"name":"drain","context":{"idset":"11833","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2759848,"name":"drain","context":{"idset":"11834","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.277597,"name":"drain","context":{"idset":"11835","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2790189,"name":"drain","context":{"idset":"11836","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2804186,"name":"drain","context":{"idset":"11863","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2819834,"name":"drain","context":{"idset":"11864","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2836056,"name":"drain","context":{"idset":"11865","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2852445,"name":"drain","context":{"idset":"11868","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2868869,"name":"drain","context":{"idset":"11869","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2885611,"name":"drain","context":{"idset":"11870","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2902291,"name":"drain","context":{"idset":"11871","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2919037,"name":"drain","context":{"idset":"11872","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2935686,"name":"drain","context":{"idset":"11873","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2952342,"name":"drain","context":{"idset":"11874","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2968745,"name":"drain","context":{"idset":"11875","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.2985454,"name":"drain","context":{"idset":"11876","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3001201,"name":"drain","context":{"idset":"11877","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3017993,"name":"drain","context":{"idset":"11878","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3034372,"name":"drain","context":{"idset":"11879","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3048999,"name":"drain","context":{"idset":"11880","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3065991,"name":"drain","context":{"idset":"11881","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3082776,"name":"drain","context":{"idset":"11882","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3099411,"name":"drain","context":{"idset":"11883","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3112814,"name":"drain","context":{"idset":"11884","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.312942,"name":"drain","context":{"idset":"11885","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3146737,"name":"drain","context":{"idset":"11886","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3163812,"name":"drain","context":{"idset":"11887","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3180873,"name":"drain","context":{"idset":"11888","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3198073,"name":"drain","context":{"idset":"11889","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3215282,"name":"drain","context":{"idset":"11890","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3232319,"name":"drain","context":{"idset":"11891","reason":"broker was unresponsive"}}
+{"timestamp":1712759595.3248541,"name":"drain","context":{"idset":"11892","reason":"broker was unresponsive"}}
+{"timestamp":1712764407.8388875,"name":"undrain","context":{"idset":"1-60,91-92,757,795-796,815-816,833-834,841-842,871-872,875-876,949-954,963-964,973-978,9205-9294,9296-9446,9449-9471,9473-9534,9536-9560,9563-9749,9751-9786,9791-9991,9993-10098,10101-10120,10123-10250,10252-10260,10263-10302,10304-10342,10345-10354,10357-10400,10402-10500,10502-10516,10519-10543,10564-10570,10572-10574,10641,10645-10653,10741-10794,10796-10868,11060-11072,11116,11119-11124,11141-11248,11251-11508,11541-11548,11569-11578,11665-11693,11695-11706,11727-11798,11801-11892"}}
+{"timestamp":1712764763.4017813,"name":"offline","context":{"idset":"9471"}}
+{"timestamp":1712765149.7120404,"name":"offline","context":{"idset":"757"}}
+{"timestamp":1712765289.1808684,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1712765289.6775393,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1712765291.7703505,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1712765292.4020314,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1712765292.8026168,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1712765292.9035709,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1712765293.0029385,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1712765293.1225443,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1712765293.1423721,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1712765293.1596801,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1712765293.1913288,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1712765293.2797031,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1712765293.3102565,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1712765293.331425,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1712765293.3442318,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1712765293.3728721,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1712765293.4021394,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1712765293.4077837,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1712765293.4408503,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1712765293.4515488,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1712765293.4620616,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1712765293.5574691,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1712765293.5644577,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1712765293.5809762,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1712765293.6224432,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1712765293.6330793,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1712765293.6383176,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1712765293.6522501,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1712765293.684612,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1712765293.6868193,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1712765293.7219012,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1712765293.8066287,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1712765293.9462514,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1712765293.9718051,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1712765294.0007293,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1712765294.0389783,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1712765294.1068468,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1712765294.1434133,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1712765294.1565878,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1712765294.1860921,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1712765294.1900856,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1712765294.2168889,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1712765294.2286465,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1712765294.2374263,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1712765294.2517841,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1712765294.2704833,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1712765294.2731397,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1712765294.3079026,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1712765294.3105812,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1712765294.3266799,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1712765294.4573395,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1712765294.4810691,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1712765294.4847109,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1712765294.4877629,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1712765294.5273619,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1712765294.5703967,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1712765294.5928011,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1712765294.6235509,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1712765294.6417916,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1712765294.669971,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1712765294.6879187,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1712765294.697866,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1712765294.7075844,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1712765294.7098114,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1712765294.7416394,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1712765294.7948308,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1712765294.8092535,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1712765294.8284416,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1712765294.8531978,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1712765294.8828912,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1712765294.8939247,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1712765294.9904852,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1712765295.0069916,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1712765295.0095162,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1712765295.0254104,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1712765295.0696242,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1712765295.1039412,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1712765295.1261227,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1712765295.1301632,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1712765295.2032807,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1712765295.2078767,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1712765295.2386031,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1712765295.3048275,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1712765295.324393,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1712765295.3275347,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1712765295.342113,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1712765295.379575,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1712765295.411705,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1712765295.4147894,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1712765295.4714122,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1712765295.4992602,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1712765295.5310721,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1712765295.5531158,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1712765295.560189,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1712765295.6059682,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1712765295.6236281,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1712765295.6514785,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1712765295.6827283,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1712765295.6951966,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1712765295.7313006,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1712765295.7341454,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1712765295.7493603,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1712765295.8169363,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1712765295.8625016,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1712765295.8806965,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1712765295.8835237,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1712765295.9271431,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1712765295.9609504,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1712765295.9637423,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1712765296.0004604,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1712765296.0032442,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1712765296.0285337,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1712765296.0669491,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1712765296.0712545,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1712765296.0747068,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1712765296.1128125,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1712765296.1299942,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1712765296.1628582,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1712765296.1819611,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1712765296.2211037,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1712765296.2442214,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1712765296.2472041,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1712765296.2501616,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1712765296.2919123,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1712765296.3445768,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1712765296.3746464,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1712765296.3773949,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1712765296.3801117,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1712765296.3885806,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1712765296.3924646,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1712765296.4254513,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1712765296.4538052,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1712765296.4646602,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1712765296.484164,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1712765296.502667,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1712765296.5207436,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1712765296.5384879,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1712765296.5825076,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1712765296.5941415,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1712765296.5968766,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1712765296.6069775,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1712765296.6967967,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1712765296.7148678,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1712765296.7524779,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1712765296.7868679,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1712765296.8165092,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1712765296.8372517,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1712765296.8399222,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1712765296.8424554,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1712765296.9010496,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1712765296.9040914,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1712765296.9477863,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1712765296.9848893,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1712765297.0173514,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1712765297.039736,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1712765297.042577,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1712765297.0594168,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1712765297.0622261,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1712765297.1155744,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1712765297.1388767,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1712765297.1650407,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1712765297.1832089,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1712765297.2011411,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1712765297.2382507,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1712765297.2705784,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1712765297.2930906,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1712765297.2959311,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1712765297.3213222,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1712765297.3381672,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1712765297.3415959,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1712765297.3912475,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1712765297.432687,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1712765297.4356937,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1712765297.4537568,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1712765297.4567826,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1712765297.4732587,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1712765297.5289965,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1712765297.5618362,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1712765297.5743635,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1712765297.5776048,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1712765297.580626,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1712765297.6411455,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1712765297.6596837,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1712765297.6829152,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1712765297.7128966,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1712765297.7813461,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1712765297.7845352,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1712765297.787468,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1712765297.806246,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1712765297.8231232,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1712765297.8489752,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1712765297.8859282,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1712765297.9095545,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1712765297.9126418,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1712765297.9235651,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1712765297.9407003,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1712765297.9435685,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1712765298.020025,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1712765298.0245347,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1712765298.0299792,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1712765298.0433495,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1712765298.0640452,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1712765298.1138515,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1712765298.1449156,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1712765298.1476705,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1712765298.1545956,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1712765298.1743433,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1712765298.1985111,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1712765298.2335179,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1712765298.3010416,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1712765298.3037894,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1712765298.3064978,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1712765298.309181,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1712765298.3353281,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1712765298.3718531,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1712765298.3940854,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1712765298.3968296,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1712765298.4508049,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1712765298.4539733,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1712765298.5221474,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1712765298.5556369,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1712765298.5586717,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1712765298.5687103,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1712765298.5868239,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1712765298.6185496,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1712765298.6403549,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1712765298.6430402,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1712765298.6618817,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1712765298.7005038,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1712765298.7321839,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1712765298.7350936,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1712765298.7378185,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1712765298.7530386,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1712765298.7777517,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1712765298.8229456,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1712765298.8820901,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1712765298.9228311,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1712765298.927587,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1712765298.9399767,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1712765298.9713767,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1712765299.0023301,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1712765299.110487,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1712765299.1148336,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1712765299.1187255,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1712765299.1304078,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1712765299.1587045,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1712765299.1910934,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1712765299.2526181,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1712765299.3118389,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1712765299.332921,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1712765299.3380203,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1712765299.3430898,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1712765299.3734407,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1712765299.4361496,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1712765299.4941182,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1712765299.5161278,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1712765299.5212336,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1712765299.5260303,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1712765299.5515039,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1712765299.5815995,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1712765299.6111047,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1712765299.6419189,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1712765299.7217805,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1712765299.7807288,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1712765299.803242,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1712765299.8084784,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1712765299.8134809,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1712765299.8180563,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1712765299.8523521,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1712765300.0206695,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1712765300.0265696,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1712765300.0324595,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1712765300.0380604,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1712765300.0435913,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1712765300.0492356,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1712765300.054857,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1712765300.1259804,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1712765300.1818213,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1712765300.2214718,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1712765300.2259686,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1712765300.2303023,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1712765300.2347078,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1712765300.2510962,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1712765300.2556612,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1712765300.2766049,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1712765300.3505728,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1712765300.3965719,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1712765300.4005833,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1712765300.4043965,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1712765300.4080582,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1712765300.4351735,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1712765300.5023396,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1712765300.5488155,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1712765300.5860379,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1712765300.5891075,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1712765300.5921652,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1712765300.5953,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1712765300.5983794,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1712765300.6084924,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1712765300.6392851,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1712765300.6822808,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1712765300.7260895,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1712765300.7435405,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1712765300.7474198,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1712765300.7511716,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1712765300.7549391,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1712765300.7586255,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1712765300.8062224,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1712765300.8523636,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1712765300.8556328,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1712765300.8594639,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1712765300.8631022,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1712765300.870805,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1712765300.892204,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1712765300.9423664,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1712765300.9773691,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1712765301.0008113,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1712765301.0035596,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1712765301.0062568,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1712765301.0091863,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1712765301.0121818,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1712765301.062938,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1712765301.0989625,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1712765301.111686,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1712765301.1145487,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1712765301.1174471,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1712765301.120635,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1712765301.1352479,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1712765301.1516893,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1712765301.1699753,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1712765301.210448,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1712765301.2449827,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1712765301.2831316,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1712765301.2859414,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1712765301.2890751,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1712765301.2921023,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1712765301.3100083,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1712765301.3273387,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1712765301.3624742,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1712765301.3979459,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1712765301.4223101,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1712765301.4250941,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1712765301.4280469,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1712765301.4311154,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1712765301.4339056,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1712765301.4628313,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1712765301.5434706,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1712765301.5790944,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1712765301.5820668,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1712765301.5849986,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1712765301.5879438,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1712765301.5909293,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1712765301.5938849,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1712765301.5966625,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1712765301.6394372,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1712765301.6956162,"name":"offline","context":{"idset":"566"}}
+{"timestamp":1712765301.6985681,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1712765301.7015462,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1712765301.7045004,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1712765301.7166266,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1712765301.7333906,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1712765301.771533,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1712765301.8054538,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1712765301.8278799,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1712765301.8303959,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1712765301.8329561,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1712765301.8359215,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1712765301.8439407,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1712765301.8605039,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1712765301.8792131,"name":"offline","context":{"idset":"581"}}
+{"timestamp":1712765301.9287946,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1712765301.9696019,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1712765301.9714563,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1712765301.9732251,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1712765301.9749963,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1712765301.976861,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1712765301.9787989,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1712765302.0497642,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1712765302.0668578,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1712765302.0687754,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1712765302.0707657,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1712765302.072916,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1712765302.0748432,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1712765302.1213834,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1712765302.1440942,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1712765302.1459961,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1712765302.1478217,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1712765302.1499588,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1712765302.1520243,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1712765302.168997,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1712765302.2068806,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1712765302.2088563,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1712765302.2104785,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1712765302.2188416,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1712765302.2652307,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1712765302.2671704,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1712765302.2772274,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1712765302.3027151,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1712765302.3205361,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1712765302.3231359,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1712765302.325897,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1712765302.3358986,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1712765302.3378127,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1712765302.3400888,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1712765302.3420684,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1712765302.3544207,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1712765302.3661785,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1712765302.3996508,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1712765302.4173479,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1712765302.4238617,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1712765302.4457371,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1712765302.4576342,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1712765302.4704418,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1712765302.4884703,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1712765302.5355015,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1712765302.5374751,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1712765302.5400496,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1712765302.5709739,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1712765302.6008904,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1712765302.6339633,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1712765302.6578405,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1712765302.6605277,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1712765302.692462,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1712765302.7247646,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1712765302.7677064,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1712765302.770474,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1712765302.7732265,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1712765302.7987394,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1712765302.8293905,"name":"drain","context":{"idset":"85-89,93-120,122-216,218-252,277-347,349-430,432-444,469-540,565-588","reason":"epilog failed for jobid foaMyfmfGZu","overwrite":0}}
+{"timestamp":1712765504.2735741,"name":"drain","context":{"idset":"9634","reason":"broker was unresponsive"}}
+{"timestamp":1712765570.2683578,"name":"offline","context":{"idset":"9634"}}
+{"timestamp":1712767117.3857222,"name":"online","context":{"idset":"9634"}}
+{"timestamp":1712768184.2675853,"name":"offline","context":{"idset":"9634"}}
+{"timestamp":1712769000.1727512,"name":"drain","context":{"idset":"9831","reason":"broker was unresponsive"}}
+{"timestamp":1712769000.2726119,"name":"drain","context":{"idset":"9832","reason":"broker was unresponsive"}}
+{"timestamp":1712769064.1771486,"name":"offline","context":{"idset":"9831"}}
+{"timestamp":1712769064.2686136,"name":"offline","context":{"idset":"9832"}}
+{"timestamp":1712769153.0442808,"name":"undrain","context":{"idset":"85-89,93-120,122-216,218-252,277-347,349-430,432-444,469-540,565-588,831-832,847-848,9295,9535,9634,9831-9832,9992,10099-10100,10251,10401,10795,11694,11799-11800"}}
+{"timestamp":1712769165.4134147,"name":"undrain","context":{"idset":"10544-10563,10575-10613,10615-10628,10630-10633,10635-10636,10640,10654-10656,10659-10701,10703-10711,10713-10717,10719-10738,10997-10998,11001-11020,11022-11059,11073-11110,11113-11114,11125-11140,11549-11568,11579-11636,11653-11664,11707-11726"}}
+{"timestamp":1712769733.7238729,"name":"online","context":{"idset":"10121"}}
+{"timestamp":1712769734.2407994,"name":"online","context":{"idset":"10122"}}
+{"timestamp":1712770113.0569081,"name":"undrain","context":{"idset":"827-828"}}
+{"timestamp":1712770175.403811,"name":"drain","context":{"idset":"827-828","overwrite":0}}
+{"timestamp":1712770464.27321,"name":"online","context":{"idset":"9832"}}
+{"timestamp":1712770464.8328047,"name":"online","context":{"idset":"9831"}}
+{"timestamp":1712770544.6203129,"name":"undrain","context":{"idset":"10121-10122"}}
+{"timestamp":1712771510.2727382,"name":"drain","context":{"idset":"11136","reason":"broker was unresponsive"}}
+{"timestamp":1712771572.1759999,"name":"drain","context":{"idset":"11136","reason":"epilog failed for jobid foZdcGvYqNw","overwrite":0}}
+{"timestamp":1712771572.2701766,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1712771612.2722294,"name":"drain","context":{"idset":"11135","reason":"broker was unresponsive"}}
+{"timestamp":1712771676.2723017,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1712771742.2719834,"name":"drain","context":{"idset":"9633","reason":"broker was unresponsive"}}
+{"timestamp":1712771808.2679689,"name":"offline","context":{"idset":"9633"}}
+{"timestamp":1712772186.2797091,"name":"offline","context":{"idset":"834"}}
+{"timestamp":1712772186.3796091,"name":"offline","context":{"idset":"833"}}
+{"timestamp":1712772452.7661858,"name":"online","context":{"idset":"11250"}}
+{"timestamp":1712772452.9614801,"name":"online","context":{"idset":"11249"}}
+{"timestamp":1712772509.0683105,"name":"online","context":{"idset":"461"}}
+{"timestamp":1712773634.2761142,"name":"online","context":{"idset":"11799"}}
+{"timestamp":1712773634.8033538,"name":"online","context":{"idset":"11800"}}
+{"timestamp":1712773680.2712946,"name":"drain","context":{"idset":"9296","reason":"broker was unresponsive"}}
+{"timestamp":1712773744.273629,"name":"offline","context":{"idset":"9296"}}
+{"timestamp":1712774140.6730404,"name":"online","context":{"idset":"161"}}
+{"timestamp":1712774154.7240837,"name":"online","context":{"idset":"104"}}
+{"timestamp":1712774154.9189651,"name":"online","context":{"idset":"88,92-93"}}
+{"timestamp":1712774155.2431178,"name":"online","context":{"idset":"89,105-106"}}
+{"timestamp":1712774155.4353051,"name":"online","context":{"idset":"90"}}
+{"timestamp":1712774155.5483375,"name":"online","context":{"idset":"113"}}
+{"timestamp":1712774155.5862987,"name":"online","context":{"idset":"119"}}
+{"timestamp":1712774155.7075768,"name":"online","context":{"idset":"101"}}
+{"timestamp":1712774155.9231853,"name":"online","context":{"idset":"107"}}
+{"timestamp":1712774156.2278399,"name":"online","context":{"idset":"108,111"}}
+{"timestamp":1712774156.3507857,"name":"online","context":{"idset":"94"}}
+{"timestamp":1712774156.8699346,"name":"online","context":{"idset":"98"}}
+{"timestamp":1712774157.0632546,"name":"online","context":{"idset":"97"}}
+{"timestamp":1712774157.2293026,"name":"online","context":{"idset":"91"}}
+{"timestamp":1712774157.5916679,"name":"online","context":{"idset":"96,112"}}
+{"timestamp":1712774158.029376,"name":"online","context":{"idset":"85,87,182"}}
+{"timestamp":1712774158.2501757,"name":"online","context":{"idset":"95,125"}}
+{"timestamp":1712774158.3871315,"name":"online","context":{"idset":"86"}}
+{"timestamp":1712774158.5283334,"name":"online","context":{"idset":"116,126"}}
+{"timestamp":1712774158.6799262,"name":"online","context":{"idset":"109,120,122"}}
+{"timestamp":1712774158.9599416,"name":"online","context":{"idset":"133,136"}}
+{"timestamp":1712774159.0977833,"name":"online","context":{"idset":"146"}}
+{"timestamp":1712774159.2282233,"name":"online","context":{"idset":"181,184"}}
+{"timestamp":1712774159.5395727,"name":"online","context":{"idset":"124,158,163,177"}}
+{"timestamp":1712774159.750674,"name":"online","context":{"idset":"110,118,138,141-142,147-148,152,165"}}
+{"timestamp":1712774159.9455855,"name":"online","context":{"idset":"115,127,154,160,170"}}
+{"timestamp":1712774160.0701842,"name":"online","context":{"idset":"128,155"}}
+{"timestamp":1712774160.1746497,"name":"online","context":{"idset":"134,174,187"}}
+{"timestamp":1712774160.3620424,"name":"online","context":{"idset":"99-100,102,123,129,143,145,149,172-173,185"}}
+{"timestamp":1712774160.4648397,"name":"online","context":{"idset":"114,157,183"}}
+{"timestamp":1712774160.5835133,"name":"online","context":{"idset":"162"}}
+{"timestamp":1712774160.7126441,"name":"online","context":{"idset":"103,131,153,159,166,168,176,179,186"}}
+{"timestamp":1712774160.9189236,"name":"online","context":{"idset":"117,130,132,135,137,139-140,144,150-151,156,164,167,169,171,175"}}
+{"timestamp":1712774161.1203194,"name":"online","context":{"idset":"178,188,190"}}
+{"timestamp":1712774161.2300267,"name":"online","context":{"idset":"180,189,191-195,197-198,201-203"}}
+{"timestamp":1712774161.3501608,"name":"online","context":{"idset":"196,199-200,204-205"}}
+{"timestamp":1712774161.8853102,"name":"online","context":{"idset":"206"}}
+{"timestamp":1712774163.800648,"name":"online","context":{"idset":"207"}}
+{"timestamp":1712774163.9035082,"name":"online","context":{"idset":"208"}}
+{"timestamp":1712774164.0228133,"name":"online","context":{"idset":"209-210"}}
+{"timestamp":1712774164.1237154,"name":"online","context":{"idset":"211"}}
+{"timestamp":1712774164.4242315,"name":"online","context":{"idset":"212,214"}}
+{"timestamp":1712774164.5626435,"name":"online","context":{"idset":"213"}}
+{"timestamp":1712774164.5877864,"name":"online","context":{"idset":"215"}}
+{"timestamp":1712774164.7195058,"name":"online","context":{"idset":"216,218"}}
+{"timestamp":1712774164.8596745,"name":"online","context":{"idset":"219-223"}}
+{"timestamp":1712774164.977886,"name":"online","context":{"idset":"224"}}
+{"timestamp":1712774165.0928807,"name":"online","context":{"idset":"225"}}
+{"timestamp":1712774165.2155304,"name":"online","context":{"idset":"226-227"}}
+{"timestamp":1712774165.4161785,"name":"online","context":{"idset":"228"}}
+{"timestamp":1712774165.9779341,"name":"online","context":{"idset":"229"}}
+{"timestamp":1712774166.1791019,"name":"online","context":{"idset":"230-231"}}
+{"timestamp":1712774166.7050874,"name":"online","context":{"idset":"232-233"}}
+{"timestamp":1712774166.9651382,"name":"online","context":{"idset":"234"}}
+{"timestamp":1712774167.1283522,"name":"online","context":{"idset":"235"}}
+{"timestamp":1712774167.3257742,"name":"online","context":{"idset":"236-237,239"}}
+{"timestamp":1712774167.4470477,"name":"online","context":{"idset":"238"}}
+{"timestamp":1712774167.4840424,"name":"online","context":{"idset":"240"}}
+{"timestamp":1712774167.7053175,"name":"online","context":{"idset":"241"}}
+{"timestamp":1712774167.844763,"name":"online","context":{"idset":"242-245"}}
+{"timestamp":1712774167.9754322,"name":"online","context":{"idset":"246-247"}}
+{"timestamp":1712774168.1036785,"name":"online","context":{"idset":"248-250"}}
+{"timestamp":1712774168.3788867,"name":"online","context":{"idset":"252"}}
+{"timestamp":1712774168.5532084,"name":"online","context":{"idset":"251"}}
+{"timestamp":1712774169.4568782,"name":"online","context":{"idset":"278"}}
+{"timestamp":1712774169.5892584,"name":"online","context":{"idset":"281"}}
+{"timestamp":1712774170.1033337,"name":"online","context":{"idset":"279-280"}}
+{"timestamp":1712774170.4824131,"name":"online","context":{"idset":"277"}}
+{"timestamp":1712774170.6131086,"name":"online","context":{"idset":"282"}}
+{"timestamp":1712774170.7478557,"name":"online","context":{"idset":"287"}}
+{"timestamp":1712774170.9211931,"name":"online","context":{"idset":"284,286"}}
+{"timestamp":1712774171.0604794,"name":"online","context":{"idset":"285"}}
+{"timestamp":1712774171.309382,"name":"online","context":{"idset":"283"}}
+{"timestamp":1712774171.8600674,"name":"online","context":{"idset":"289"}}
+{"timestamp":1712774171.9770756,"name":"online","context":{"idset":"288"}}
+{"timestamp":1712774172.0858169,"name":"online","context":{"idset":"292"}}
+{"timestamp":1712774172.2553282,"name":"online","context":{"idset":"290-291"}}
+{"timestamp":1712774172.3628666,"name":"online","context":{"idset":"294"}}
+{"timestamp":1712774172.4885406,"name":"online","context":{"idset":"296"}}
+{"timestamp":1712774172.7470737,"name":"online","context":{"idset":"298-299"}}
+{"timestamp":1712774172.8934765,"name":"online","context":{"idset":"293,295,297,300,302,304,306-307,309"}}
+{"timestamp":1712774173.0323749,"name":"online","context":{"idset":"301,303"}}
+{"timestamp":1712774173.1518385,"name":"online","context":{"idset":"305,308,312"}}
+{"timestamp":1712774173.3537076,"name":"online","context":{"idset":"310-311,313-326,328-330,333-335"}}
+{"timestamp":1712774173.4404168,"name":"online","context":{"idset":"331-332"}}
+{"timestamp":1712774173.6673265,"name":"online","context":{"idset":"327,336-339"}}
+{"timestamp":1712774173.954803,"name":"online","context":{"idset":"340-342"}}
+{"timestamp":1712774174.2284682,"name":"online","context":{"idset":"343-344"}}
+{"timestamp":1712774174.3163218,"name":"online","context":{"idset":"345"}}
+{"timestamp":1712774174.4779069,"name":"online","context":{"idset":"346-347,349-350"}}
+{"timestamp":1712774174.5789375,"name":"online","context":{"idset":"351-352"}}
+{"timestamp":1712774174.7784851,"name":"online","context":{"idset":"353-356,358"}}
+{"timestamp":1712774174.9828422,"name":"online","context":{"idset":"357,359-361"}}
+{"timestamp":1712774175.0846784,"name":"online","context":{"idset":"362,364"}}
+{"timestamp":1712774175.2013512,"name":"online","context":{"idset":"363,365-367,369"}}
+{"timestamp":1712774175.307338,"name":"online","context":{"idset":"368,370"}}
+{"timestamp":1712774175.6035707,"name":"online","context":{"idset":"371"}}
+{"timestamp":1712774175.7252836,"name":"online","context":{"idset":"372"}}
+{"timestamp":1712774175.9915719,"name":"online","context":{"idset":"373"}}
+{"timestamp":1712774176.1241918,"name":"online","context":{"idset":"374"}}
+{"timestamp":1712774176.1396246,"name":"online","context":{"idset":"375"}}
+{"timestamp":1712774176.2642057,"name":"online","context":{"idset":"376,378"}}
+{"timestamp":1712774176.356739,"name":"online","context":{"idset":"377"}}
+{"timestamp":1712774176.4538245,"name":"online","context":{"idset":"379"}}
+{"timestamp":1712774176.6594672,"name":"online","context":{"idset":"380"}}
+{"timestamp":1712774176.7880979,"name":"online","context":{"idset":"381-385"}}
+{"timestamp":1712774177.0440307,"name":"online","context":{"idset":"387"}}
+{"timestamp":1712774177.1567612,"name":"online","context":{"idset":"386,390"}}
+{"timestamp":1712774177.2768626,"name":"online","context":{"idset":"388-389,391"}}
+{"timestamp":1712774177.4233997,"name":"online","context":{"idset":"392"}}
+{"timestamp":1712774177.9518366,"name":"online","context":{"idset":"393"}}
+{"timestamp":1712774178.5608051,"name":"online","context":{"idset":"394"}}
+{"timestamp":1712774178.7143555,"name":"online","context":{"idset":"395"}}
+{"timestamp":1712774178.8362832,"name":"online","context":{"idset":"396"}}
+{"timestamp":1712774178.8560874,"name":"online","context":{"idset":"397"}}
+{"timestamp":1712774179.00069,"name":"online","context":{"idset":"398,400"}}
+{"timestamp":1712774179.1198711,"name":"online","context":{"idset":"402"}}
+{"timestamp":1712774179.2651401,"name":"online","context":{"idset":"399"}}
+{"timestamp":1712774179.3977137,"name":"online","context":{"idset":"401,404"}}
+{"timestamp":1712774179.7801332,"name":"online","context":{"idset":"403,405-406"}}
+{"timestamp":1712774179.9280453,"name":"online","context":{"idset":"407-410"}}
+{"timestamp":1712774180.0670569,"name":"online","context":{"idset":"411"}}
+{"timestamp":1712774180.0949447,"name":"online","context":{"idset":"412"}}
+{"timestamp":1712774180.2668419,"name":"online","context":{"idset":"413-415"}}
+{"timestamp":1712774180.7929795,"name":"online","context":{"idset":"416"}}
+{"timestamp":1712774180.9233496,"name":"online","context":{"idset":"417-418"}}
+{"timestamp":1712774181.3277235,"name":"online","context":{"idset":"419"}}
+{"timestamp":1712774181.4304514,"name":"online","context":{"idset":"420-421"}}
+{"timestamp":1712774181.4491146,"name":"online","context":{"idset":"422"}}
+{"timestamp":1712774181.930398,"name":"online","context":{"idset":"423,426"}}
+{"timestamp":1712774182.0684927,"name":"online","context":{"idset":"424-425,428"}}
+{"timestamp":1712774182.2440491,"name":"online","context":{"idset":"427"}}
+{"timestamp":1712774182.359297,"name":"online","context":{"idset":"429-432"}}
+{"timestamp":1712774182.4777884,"name":"online","context":{"idset":"433-434"}}
+{"timestamp":1712774182.5882206,"name":"online","context":{"idset":"435-438,440"}}
+{"timestamp":1712774182.8841403,"name":"online","context":{"idset":"439,441-444"}}
+{"timestamp":1712774182.9326389,"name":"online","context":{"idset":"469"}}
+{"timestamp":1712774183.075943,"name":"online","context":{"idset":"470"}}
+{"timestamp":1712774183.2410343,"name":"online","context":{"idset":"472"}}
+{"timestamp":1712774183.4114816,"name":"online","context":{"idset":"473"}}
+{"timestamp":1712774183.4663136,"name":"online","context":{"idset":"471,476"}}
+{"timestamp":1712774183.6784015,"name":"online","context":{"idset":"474-475"}}
+{"timestamp":1712774184.0205495,"name":"online","context":{"idset":"477"}}
+{"timestamp":1712774184.0827119,"name":"online","context":{"idset":"480"}}
+{"timestamp":1712774184.3331282,"name":"online","context":{"idset":"479,481-482"}}
+{"timestamp":1712774184.4787936,"name":"online","context":{"idset":"478,483"}}
+{"timestamp":1712774184.7260256,"name":"online","context":{"idset":"484-486"}}
+{"timestamp":1712774184.9368598,"name":"online","context":{"idset":"487-492"}}
+{"timestamp":1712774185.1639948,"name":"online","context":{"idset":"494-497"}}
+{"timestamp":1712774185.5994587,"name":"online","context":{"idset":"498"}}
+{"timestamp":1712774185.8623831,"name":"online","context":{"idset":"499,501,503"}}
+{"timestamp":1712774186.0718231,"name":"online","context":{"idset":"500,505,507-508,513"}}
+{"timestamp":1712774186.2663007,"name":"online","context":{"idset":"502,504,506,509-512,515-516,518,520"}}
+{"timestamp":1712774186.4625983,"name":"online","context":{"idset":"514,517,519,521-529"}}
+{"timestamp":1712774186.5808949,"name":"online","context":{"idset":"530-532,534,537"}}
+{"timestamp":1712774186.7000151,"name":"online","context":{"idset":"533"}}
+{"timestamp":1712774186.7373135,"name":"online","context":{"idset":"535,538"}}
+{"timestamp":1712774186.8835361,"name":"online","context":{"idset":"536,539-540"}}
+{"timestamp":1712774189.0914793,"name":"online","context":{"idset":"565"}}
+{"timestamp":1712774189.2267201,"name":"online","context":{"idset":"568"}}
+{"timestamp":1712774189.4405537,"name":"online","context":{"idset":"567,569-570"}}
+{"timestamp":1712774189.7605784,"name":"online","context":{"idset":"571"}}
+{"timestamp":1712774189.9845974,"name":"online","context":{"idset":"572-573,576"}}
+{"timestamp":1712774190.1256611,"name":"online","context":{"idset":"575"}}
+{"timestamp":1712774190.215956,"name":"online","context":{"idset":"574"}}
+{"timestamp":1712774190.3738909,"name":"online","context":{"idset":"577-579"}}
+{"timestamp":1712774190.5845659,"name":"online","context":{"idset":"580"}}
+{"timestamp":1712774190.9016919,"name":"online","context":{"idset":"582-583"}}
+{"timestamp":1712774191.005496,"name":"online","context":{"idset":"584"}}
+{"timestamp":1712774191.1331346,"name":"online","context":{"idset":"585-588"}}
+{"timestamp":1712774306.2720413,"name":"drain","context":{"idset":"11693","reason":"broker was unresponsive"}}
+{"timestamp":1712774370.2693677,"name":"offline","context":{"idset":"11693"}}
+{"timestamp":1712774736.2741404,"name":"drain","context":{"idset":"570","reason":"broker was unresponsive"}}
+{"timestamp":1712774769.8573737,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1712774770.0710881,"name":"online","context":{"idset":"10871"}}
+{"timestamp":1712774800.2685883,"name":"offline","context":{"idset":"570"}}
+{"timestamp":1712774876.2733092,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1712774882.2727616,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1712774936.2728894,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1712774940.2688291,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1712774948.2728655,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1712775002.2687302,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1712775129.0089099,"name":"undrain","context":{"idset":"431"}}
+{"timestamp":1712775619.1332941,"name":"undrain","context":{"idset":"9633"}}
+{"timestamp":1712775668.1795921,"name":"drain","context":{"idset":"10997","reason":"broker was unresponsive"}}
+{"timestamp":1712775668.1797345,"name":"drain","context":{"idset":"10998","reason":"broker was unresponsive"}}
+{"timestamp":1712775668.1798193,"name":"drain","context":{"idset":"11001","reason":"broker was unresponsive"}}
+{"timestamp":1712775668.1799252,"name":"drain","context":{"idset":"11002","reason":"broker was unresponsive"}}
+{"timestamp":1712775668.1800196,"name":"drain","context":{"idset":"11003","reason":"broker was unresponsive"}}
+{"timestamp":1712775668.1801109,"name":"drain","context":{"idset":"11004","reason":"broker was unresponsive"}}
+{"timestamp":1712775668.3181269,"name":"drain","context":{"idset":"11005","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.3917646,"name":"drain","context":{"idset":"11006","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.3919408,"name":"drain","context":{"idset":"11007","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.3920252,"name":"drain","context":{"idset":"11008","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.3921072,"name":"drain","context":{"idset":"11009","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.3921888,"name":"drain","context":{"idset":"11010","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.3922842,"name":"drain","context":{"idset":"11011","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.3923743,"name":"drain","context":{"idset":"11012","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.3924594,"name":"drain","context":{"idset":"11013","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.3925328,"name":"drain","context":{"idset":"11014","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.3926077,"name":"drain","context":{"idset":"11015","reason":"broker was unresponsive"}}
+{"timestamp":1712775674.5825222,"name":"drain","context":{"idset":"11016","reason":"broker was unresponsive"}}
+{"timestamp":1712775679.1627071,"name":"online","context":{"idset":"9633"}}
+{"timestamp":1712775680.4555504,"name":"drain","context":{"idset":"11017","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.4557374,"name":"drain","context":{"idset":"11018","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.4558284,"name":"drain","context":{"idset":"11019","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.4559169,"name":"drain","context":{"idset":"11020","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.4560475,"name":"drain","context":{"idset":"11022","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.4561219,"name":"drain","context":{"idset":"11023","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.4561987,"name":"drain","context":{"idset":"11024","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.4562895,"name":"drain","context":{"idset":"11025","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.4563642,"name":"drain","context":{"idset":"11026","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.4564443,"name":"drain","context":{"idset":"11027","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.4565291,"name":"drain","context":{"idset":"11028","reason":"broker was unresponsive"}}
+{"timestamp":1712775680.6274686,"name":"drain","context":{"idset":"11029","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.1764457,"name":"drain","context":{"idset":"11030","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.1765893,"name":"drain","context":{"idset":"11031","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.1766727,"name":"drain","context":{"idset":"11032","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.1767504,"name":"drain","context":{"idset":"11033","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.1768272,"name":"drain","context":{"idset":"11034","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.1769016,"name":"drain","context":{"idset":"11035","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.1769772,"name":"drain","context":{"idset":"11036","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.1770535,"name":"drain","context":{"idset":"11037","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.177129,"name":"drain","context":{"idset":"11038","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.1772041,"name":"drain","context":{"idset":"11039","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.1773028,"name":"drain","context":{"idset":"11040","reason":"broker was unresponsive"}}
+{"timestamp":1712775686.3241224,"name":"drain","context":{"idset":"11041","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1793866,"name":"drain","context":{"idset":"11042","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1795506,"name":"drain","context":{"idset":"11043","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1796758,"name":"drain","context":{"idset":"11044","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1797979,"name":"drain","context":{"idset":"11045","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1799212,"name":"drain","context":{"idset":"11046","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1800494,"name":"drain","context":{"idset":"11047","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1801827,"name":"drain","context":{"idset":"11048","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1803365,"name":"drain","context":{"idset":"11049","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1804783,"name":"drain","context":{"idset":"11050","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1806185,"name":"drain","context":{"idset":"11051","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1807458,"name":"drain","context":{"idset":"11052","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1808619,"name":"drain","context":{"idset":"11053","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.1809785,"name":"drain","context":{"idset":"11054","reason":"broker was unresponsive"}}
+{"timestamp":1712775692.3661623,"name":"drain","context":{"idset":"11055","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.175498,"name":"drain","context":{"idset":"11056","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.1756396,"name":"drain","context":{"idset":"11057","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.1757417,"name":"drain","context":{"idset":"11058","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.1758344,"name":"drain","context":{"idset":"11059","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.1759305,"name":"drain","context":{"idset":"11060","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.176023,"name":"drain","context":{"idset":"11061","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.1761136,"name":"drain","context":{"idset":"11062","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.1762064,"name":"drain","context":{"idset":"11063","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.1763418,"name":"drain","context":{"idset":"11064","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.1764717,"name":"drain","context":{"idset":"11065","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.1765752,"name":"drain","context":{"idset":"11066","reason":"broker was unresponsive"}}
+{"timestamp":1712775698.3449585,"name":"drain","context":{"idset":"11067","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.1758199,"name":"drain","context":{"idset":"11068","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.1760488,"name":"drain","context":{"idset":"11069","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.1762266,"name":"drain","context":{"idset":"11070","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.1764228,"name":"drain","context":{"idset":"11071","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.1765909,"name":"drain","context":{"idset":"11072","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.1767645,"name":"drain","context":{"idset":"11073","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.1769369,"name":"drain","context":{"idset":"11074","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.1771042,"name":"drain","context":{"idset":"11075","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.1772857,"name":"drain","context":{"idset":"11076","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.177454,"name":"drain","context":{"idset":"11077","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.1776268,"name":"drain","context":{"idset":"11078","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.177804,"name":"drain","context":{"idset":"11079","reason":"broker was unresponsive"}}
+{"timestamp":1712775704.3606093,"name":"drain","context":{"idset":"11080","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1790006,"name":"drain","context":{"idset":"11081","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1792057,"name":"drain","context":{"idset":"11082","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1793966,"name":"drain","context":{"idset":"11083","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1795621,"name":"drain","context":{"idset":"11084","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1797225,"name":"drain","context":{"idset":"11085","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1798854,"name":"drain","context":{"idset":"11086","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1800499,"name":"drain","context":{"idset":"11087","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1802163,"name":"drain","context":{"idset":"11088","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1803987,"name":"drain","context":{"idset":"11089","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1805675,"name":"drain","context":{"idset":"11090","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1807342,"name":"drain","context":{"idset":"11091","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.1809027,"name":"drain","context":{"idset":"11092","reason":"broker was unresponsive"}}
+{"timestamp":1712775710.3600688,"name":"drain","context":{"idset":"11093","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1760926,"name":"drain","context":{"idset":"11094","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1762519,"name":"drain","context":{"idset":"11095","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1764271,"name":"drain","context":{"idset":"11096","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1765528,"name":"drain","context":{"idset":"11097","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1766815,"name":"drain","context":{"idset":"11098","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1768076,"name":"drain","context":{"idset":"11099","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1769297,"name":"drain","context":{"idset":"11100","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1770661,"name":"drain","context":{"idset":"11101","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1771901,"name":"drain","context":{"idset":"11102","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1773233,"name":"drain","context":{"idset":"11103","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1774414,"name":"drain","context":{"idset":"11104","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1775587,"name":"drain","context":{"idset":"11105","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.1776783,"name":"drain","context":{"idset":"11106","reason":"broker was unresponsive"}}
+{"timestamp":1712775716.3881655,"name":"drain","context":{"idset":"11107","reason":"broker was unresponsive"}}
+{"timestamp":1712775722.1748443,"name":"drain","context":{"idset":"11108","reason":"broker was unresponsive"}}
+{"timestamp":1712775722.1750472,"name":"drain","context":{"idset":"11109","reason":"broker was unresponsive"}}
+{"timestamp":1712775722.1752193,"name":"drain","context":{"idset":"11110","reason":"broker was unresponsive"}}
+{"timestamp":1712775722.1754007,"name":"drain","context":{"idset":"11113","reason":"broker was unresponsive"}}
+{"timestamp":1712775722.175566,"name":"drain","context":{"idset":"11114","reason":"broker was unresponsive"}}
+{"timestamp":1712775722.1757271,"name":"drain","context":{"idset":"11116","reason":"broker was unresponsive"}}
+{"timestamp":1712775722.1759071,"name":"drain","context":{"idset":"11119","reason":"broker was unresponsive"}}
+{"timestamp":1712775722.176074,"name":"drain","context":{"idset":"11120","reason":"broker was unresponsive"}}
+{"timestamp":1712775722.1762502,"name":"drain","context":{"idset":"11121","reason":"broker was unresponsive"}}
+{"timestamp":1712775722.3547852,"name":"drain","context":{"idset":"11122","reason":"broker was unresponsive"}}
+{"timestamp":1712775728.1740193,"name":"drain","context":{"idset":"11123","reason":"broker was unresponsive"}}
+{"timestamp":1712775728.2734411,"name":"drain","context":{"idset":"11124","reason":"broker was unresponsive"}}
+{"timestamp":1712775732.1861482,"name":"offline","context":{"idset":"10997"}}
+{"timestamp":1712775732.2044296,"name":"offline","context":{"idset":"10998"}}
+{"timestamp":1712775732.2770941,"name":"offline","context":{"idset":"11001"}}
+{"timestamp":1712775734.1881285,"name":"offline","context":{"idset":"11002"}}
+{"timestamp":1712775734.2059112,"name":"offline","context":{"idset":"11003"}}
+{"timestamp":1712775734.2229888,"name":"offline","context":{"idset":"11004"}}
+{"timestamp":1712775734.2696397,"name":"offline","context":{"idset":"11005"}}
+{"timestamp":1712775736.1945143,"name":"offline","context":{"idset":"11006"}}
+{"timestamp":1712775736.2136116,"name":"offline","context":{"idset":"11008"}}
+{"timestamp":1712775736.2356026,"name":"offline","context":{"idset":"11009"}}
+{"timestamp":1712775736.268101,"name":"offline","context":{"idset":"11010"}}
+{"timestamp":1712775737.210778,"name":"offline","context":{"idset":"11011"}}
+{"timestamp":1712775737.2289863,"name":"offline","context":{"idset":"11012"}}
+{"timestamp":1712775737.2932158,"name":"offline","context":{"idset":"11013"}}
+{"timestamp":1712775740.1913035,"name":"offline","context":{"idset":"11007"}}
+{"timestamp":1712775740.2081437,"name":"offline","context":{"idset":"11014"}}
+{"timestamp":1712775740.4452837,"name":"offline","context":{"idset":"11015"}}
+{"timestamp":1712775740.4646218,"name":"offline","context":{"idset":"11016"}}
+{"timestamp":1712775740.506444,"name":"offline","context":{"idset":"11017"}}
+{"timestamp":1712775741.4431033,"name":"offline","context":{"idset":"11018"}}
+{"timestamp":1712775741.4460585,"name":"offline","context":{"idset":"11019"}}
+{"timestamp":1712775741.5000658,"name":"offline","context":{"idset":"11020"}}
+{"timestamp":1712775742.6960363,"name":"offline","context":{"idset":"11021"}}
+{"timestamp":1712775742.7105222,"name":"offline","context":{"idset":"11022"}}
+{"timestamp":1712775742.7239816,"name":"offline","context":{"idset":"11023"}}
+{"timestamp":1712775742.7870805,"name":"offline","context":{"idset":"11024"}}
+{"timestamp":1712775746.1901016,"name":"offline","context":{"idset":"11025"}}
+{"timestamp":1712775746.2079427,"name":"offline","context":{"idset":"11026"}}
+{"timestamp":1712775746.2243626,"name":"offline","context":{"idset":"11027"}}
+{"timestamp":1712775746.2401221,"name":"offline","context":{"idset":"11028"}}
+{"timestamp":1712775746.2707033,"name":"offline","context":{"idset":"11029"}}
+{"timestamp":1712775747.5034952,"name":"offline","context":{"idset":"11030"}}
+{"timestamp":1712775747.5241337,"name":"offline","context":{"idset":"11031"}}
+{"timestamp":1712775747.5876658,"name":"offline","context":{"idset":"11032"}}
+{"timestamp":1712775750.1991415,"name":"offline","context":{"idset":"11033"}}
+{"timestamp":1712775750.2242537,"name":"offline","context":{"idset":"11034"}}
+{"timestamp":1712775750.2463169,"name":"offline","context":{"idset":"11035"}}
+{"timestamp":1712775750.2650545,"name":"offline","context":{"idset":"11036"}}
+{"timestamp":1712775750.292681,"name":"offline","context":{"idset":"11037"}}
+{"timestamp":1712775752.2014563,"name":"offline","context":{"idset":"11038"}}
+{"timestamp":1712775752.2325296,"name":"offline","context":{"idset":"11039"}}
+{"timestamp":1712775752.2573581,"name":"offline","context":{"idset":"11040"}}
+{"timestamp":1712775752.5608311,"name":"offline","context":{"idset":"11041"}}
+{"timestamp":1712775753.5936775,"name":"offline","context":{"idset":"11042"}}
+{"timestamp":1712775753.6154757,"name":"offline","context":{"idset":"11043"}}
+{"timestamp":1712775753.6426306,"name":"offline","context":{"idset":"11044"}}
+{"timestamp":1712775753.6641471,"name":"offline","context":{"idset":"11045"}}
+{"timestamp":1712775753.7051971,"name":"offline","context":{"idset":"11046"}}
+{"timestamp":1712775756.1890635,"name":"offline","context":{"idset":"11047"}}
+{"timestamp":1712775756.2097178,"name":"offline","context":{"idset":"11048"}}
+{"timestamp":1712775756.2315199,"name":"offline","context":{"idset":"11049"}}
+{"timestamp":1712775756.2472577,"name":"offline","context":{"idset":"11050"}}
+{"timestamp":1712775756.5037379,"name":"offline","context":{"idset":"11051"}}
+{"timestamp":1712775757.5056944,"name":"offline","context":{"idset":"11052"}}
+{"timestamp":1712775757.5261064,"name":"offline","context":{"idset":"11054"}}
+{"timestamp":1712775757.6036098,"name":"offline","context":{"idset":"11055"}}
+{"timestamp":1712775760.1993372,"name":"offline","context":{"idset":"11056"}}
+{"timestamp":1712775760.22065,"name":"offline","context":{"idset":"11057"}}
+{"timestamp":1712775760.2451596,"name":"offline","context":{"idset":"11058"}}
+{"timestamp":1712775760.2741916,"name":"offline","context":{"idset":"11059"}}
+{"timestamp":1712775760.2963147,"name":"offline","context":{"idset":"11060"}}
+{"timestamp":1712775761.258204,"name":"offline","context":{"idset":"11061"}}
+{"timestamp":1712775761.2788844,"name":"offline","context":{"idset":"11062"}}
+{"timestamp":1712775761.2986984,"name":"offline","context":{"idset":"11063"}}
+{"timestamp":1712775761.3385084,"name":"offline","context":{"idset":"11064"}}
+{"timestamp":1712775764.1856585,"name":"offline","context":{"idset":"11065"}}
+{"timestamp":1712775764.2050519,"name":"offline","context":{"idset":"11066"}}
+{"timestamp":1712775764.2734663,"name":"offline","context":{"idset":"11067"}}
+{"timestamp":1712775766.1850359,"name":"offline","context":{"idset":"11068"}}
+{"timestamp":1712775766.2027647,"name":"offline","context":{"idset":"11069"}}
+{"timestamp":1712775766.2781811,"name":"offline","context":{"idset":"11070"}}
+{"timestamp":1712775768.1824286,"name":"offline","context":{"idset":"11071"}}
+{"timestamp":1712775768.1943414,"name":"offline","context":{"idset":"11072"}}
+{"timestamp":1712775768.2065783,"name":"offline","context":{"idset":"11073"}}
+{"timestamp":1712775768.2215095,"name":"offline","context":{"idset":"11074"}}
+{"timestamp":1712775768.2666292,"name":"offline","context":{"idset":"11075"}}
+{"timestamp":1712775770.1869495,"name":"offline","context":{"idset":"11077"}}
+{"timestamp":1712775770.2032964,"name":"offline","context":{"idset":"11078"}}
+{"timestamp":1712775770.2210922,"name":"offline","context":{"idset":"11079"}}
+{"timestamp":1712775770.3065276,"name":"offline","context":{"idset":"11080"}}
+{"timestamp":1712775772.1911826,"name":"offline","context":{"idset":"11081"}}
+{"timestamp":1712775772.2094851,"name":"offline","context":{"idset":"11082"}}
+{"timestamp":1712775772.2264082,"name":"offline","context":{"idset":"11083"}}
+{"timestamp":1712775772.2716126,"name":"offline","context":{"idset":"11084"}}
+{"timestamp":1712775773.2400126,"name":"offline","context":{"idset":"11076"}}
+{"timestamp":1712775773.2610171,"name":"offline","context":{"idset":"11085"}}
+{"timestamp":1712775773.5417085,"name":"offline","context":{"idset":"11086"}}
+{"timestamp":1712775773.5856333,"name":"offline","context":{"idset":"11087"}}
+{"timestamp":1712775773.6039898,"name":"offline","context":{"idset":"11088"}}
+{"timestamp":1712775773.6316361,"name":"offline","context":{"idset":"11089"}}
+{"timestamp":1712775776.1898613,"name":"offline","context":{"idset":"11053"}}
+{"timestamp":1712775776.2035677,"name":"offline","context":{"idset":"11090"}}
+{"timestamp":1712775776.411572,"name":"offline","context":{"idset":"11091"}}
+{"timestamp":1712775776.4229758,"name":"offline","context":{"idset":"11092"}}
+{"timestamp":1712775776.4673462,"name":"offline","context":{"idset":"11093"}}
+{"timestamp":1712775778.2100868,"name":"offline","context":{"idset":"11095"}}
+{"timestamp":1712775778.212893,"name":"offline","context":{"idset":"11096"}}
+{"timestamp":1712775778.2155607,"name":"offline","context":{"idset":"11097"}}
+{"timestamp":1712775778.2181346,"name":"offline","context":{"idset":"11098"}}
+{"timestamp":1712775780.2051201,"name":"offline","context":{"idset":"11099"}}
+{"timestamp":1712775780.2277222,"name":"offline","context":{"idset":"11100"}}
+{"timestamp":1712775780.247859,"name":"offline","context":{"idset":"11101"}}
+{"timestamp":1712775780.2670674,"name":"offline","context":{"idset":"11102"}}
+{"timestamp":1712775780.2976215,"name":"offline","context":{"idset":"11103"}}
+{"timestamp":1712775781.4533134,"name":"offline","context":{"idset":"11104"}}
+{"timestamp":1712775781.4801531,"name":"offline","context":{"idset":"11105"}}
+{"timestamp":1712775781.50348,"name":"offline","context":{"idset":"11106"}}
+{"timestamp":1712775781.5418451,"name":"offline","context":{"idset":"11107"}}
+{"timestamp":1712775784.1918151,"name":"offline","context":{"idset":"11108"}}
+{"timestamp":1712775784.2277598,"name":"offline","context":{"idset":"11109"}}
+{"timestamp":1712775784.2720878,"name":"offline","context":{"idset":"11110"}}
+{"timestamp":1712775785.2877057,"name":"offline","context":{"idset":"11113"}}
+{"timestamp":1712775785.3131533,"name":"offline","context":{"idset":"11114"}}
+{"timestamp":1712775785.5574069,"name":"offline","context":{"idset":"11116"}}
+{"timestamp":1712775788.2090859,"name":"offline","context":{"idset":"11119"}}
+{"timestamp":1712775788.2440343,"name":"offline","context":{"idset":"11120"}}
+{"timestamp":1712775788.2828844,"name":"offline","context":{"idset":"11121"}}
+{"timestamp":1712775788.3189096,"name":"offline","context":{"idset":"11122"}}
+{"timestamp":1712775788.350605,"name":"offline","context":{"idset":"11123"}}
+{"timestamp":1712775788.3939409,"name":"offline","context":{"idset":"11124"}}
+{"timestamp":1712775793.7546606,"name":"offline","context":{"idset":"11094"}}
+{"timestamp":1712776180.1749156,"name":"drain","context":{"idset":"795","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1750479,"name":"drain","context":{"idset":"796","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1751392,"name":"drain","context":{"idset":"815","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1751883,"name":"drain","context":{"idset":"816","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1753199,"name":"drain","context":{"idset":"842","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.175375,"name":"drain","context":{"idset":"871","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1754277,"name":"drain","context":{"idset":"872","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1754768,"name":"drain","context":{"idset":"875","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1755378,"name":"drain","context":{"idset":"876","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1755862,"name":"drain","context":{"idset":"949","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1756411,"name":"drain","context":{"idset":"950","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1756907,"name":"drain","context":{"idset":"952","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1757445,"name":"drain","context":{"idset":"954","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1758006,"name":"drain","context":{"idset":"963","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.1758513,"name":"drain","context":{"idset":"973","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.175905,"name":"drain","context":{"idset":"975","reason":"broker was unresponsive"}}
+{"timestamp":1712776180.4014995,"name":"drain","context":{"idset":"977","reason":"broker was unresponsive"}}
+{"timestamp":1712776540.0526371,"name":"online","context":{"idset":"9634"}}
+{"timestamp":1712777120.1732297,"name":"drain","context":{"idset":"11469","reason":"broker was unresponsive"}}
+{"timestamp":1712777120.2739613,"name":"drain","context":{"idset":"11470","reason":"broker was unresponsive"}}
+{"timestamp":1712777182.1861796,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1712777182.2685766,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1712777216.194092,"name":"offline","context":{"idset":"875"}}
+{"timestamp":1712777216.2696841,"name":"offline","context":{"idset":"876"}}
+{"timestamp":1712778184.786752,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1712778184.8100827,"name":"offline","context":{"idset":"10765"}}
+{"timestamp":1712778184.8293126,"name":"offline","context":{"idset":"10750"}}
+{"timestamp":1712778184.8619957,"name":"offline","context":{"idset":"10745"}}
+{"timestamp":1712778184.9491174,"name":"offline","context":{"idset":"10749"}}
+{"timestamp":1712778184.9932497,"name":"offline","context":{"idset":"10791"}}
+{"timestamp":1712778185.0631599,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1712778185.0814617,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1712778185.1089466,"name":"offline","context":{"idset":"10746"}}
+{"timestamp":1712778185.1175852,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1712778185.1563227,"name":"offline","context":{"idset":"10751"}}
+{"timestamp":1712778185.1761868,"name":"offline","context":{"idset":"10753"}}
+{"timestamp":1712778185.1908426,"name":"offline","context":{"idset":"10754"}}
+{"timestamp":1712778185.1936731,"name":"offline","context":{"idset":"10761"}}
+{"timestamp":1712778185.2085068,"name":"offline","context":{"idset":"10762"}}
+{"timestamp":1712778185.2248325,"name":"offline","context":{"idset":"10769"}}
+{"timestamp":1712778185.2279665,"name":"offline","context":{"idset":"10832"}}
+{"timestamp":1712778185.3353972,"name":"offline","context":{"idset":"10747"}}
+{"timestamp":1712778185.3391664,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1712778185.3558691,"name":"offline","context":{"idset":"10758"}}
+{"timestamp":1712778185.3728361,"name":"offline","context":{"idset":"10759"}}
+{"timestamp":1712778185.3756194,"name":"offline","context":{"idset":"10772"}}
+{"timestamp":1712778185.3987372,"name":"offline","context":{"idset":"10774"}}
+{"timestamp":1712778185.401654,"name":"offline","context":{"idset":"10794"}}
+{"timestamp":1712778185.4063833,"name":"offline","context":{"idset":"10796"}}
+{"timestamp":1712778185.4227839,"name":"offline","context":{"idset":"10812"}}
+{"timestamp":1712778185.4255588,"name":"offline","context":{"idset":"10743"}}
+{"timestamp":1712778185.5622323,"name":"offline","context":{"idset":"10752"}}
+{"timestamp":1712778185.5812483,"name":"offline","context":{"idset":"10757"}}
+{"timestamp":1712778185.5840299,"name":"offline","context":{"idset":"10766"}}
+{"timestamp":1712778185.5980542,"name":"offline","context":{"idset":"10767"}}
+{"timestamp":1712778185.6145422,"name":"offline","context":{"idset":"10771"}}
+{"timestamp":1712778185.6172764,"name":"offline","context":{"idset":"10773"}}
+{"timestamp":1712778185.6310976,"name":"offline","context":{"idset":"10782"}}
+{"timestamp":1712778185.6575739,"name":"offline","context":{"idset":"10783"}}
+{"timestamp":1712778185.6803386,"name":"offline","context":{"idset":"10786"}}
+{"timestamp":1712778185.7030804,"name":"offline","context":{"idset":"10788"}}
+{"timestamp":1712778185.7157683,"name":"offline","context":{"idset":"10792"}}
+{"timestamp":1712778185.7193761,"name":"offline","context":{"idset":"10797"}}
+{"timestamp":1712778185.7220979,"name":"offline","context":{"idset":"10798"}}
+{"timestamp":1712778185.7248647,"name":"offline","context":{"idset":"10799"}}
+{"timestamp":1712778185.8548977,"name":"offline","context":{"idset":"10805"}}
+{"timestamp":1712778185.8859754,"name":"offline","context":{"idset":"10806"}}
+{"timestamp":1712778185.9103763,"name":"offline","context":{"idset":"10807"}}
+{"timestamp":1712778185.9309092,"name":"offline","context":{"idset":"10809"}}
+{"timestamp":1712778185.9477561,"name":"offline","context":{"idset":"10813"}}
+{"timestamp":1712778185.9649115,"name":"offline","context":{"idset":"10819"}}
+{"timestamp":1712778185.9824934,"name":"offline","context":{"idset":"10820"}}
+{"timestamp":1712778186.0176387,"name":"offline","context":{"idset":"10823"}}
+{"timestamp":1712778186.0604138,"name":"offline","context":{"idset":"10826"}}
+{"timestamp":1712778186.1004069,"name":"offline","context":{"idset":"10829"}}
+{"timestamp":1712778186.1407733,"name":"offline","context":{"idset":"10841"}}
+{"timestamp":1712778186.1806464,"name":"offline","context":{"idset":"10855"}}
+{"timestamp":1712778186.2028584,"name":"offline","context":{"idset":"10857"}}
+{"timestamp":1712778186.2078564,"name":"offline","context":{"idset":"10862"}}
+{"timestamp":1712778186.2127936,"name":"offline","context":{"idset":"10863"}}
+{"timestamp":1712778186.217772,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1712778186.2226787,"name":"offline","context":{"idset":"10755"}}
+{"timestamp":1712778186.2275882,"name":"offline","context":{"idset":"10760"}}
+{"timestamp":1712778186.2594123,"name":"offline","context":{"idset":"10764"}}
+{"timestamp":1712778186.3147492,"name":"offline","context":{"idset":"10776"}}
+{"timestamp":1712778186.3429713,"name":"offline","context":{"idset":"10779"}}
+{"timestamp":1712778186.3699691,"name":"offline","context":{"idset":"10780"}}
+{"timestamp":1712778186.3942809,"name":"offline","context":{"idset":"10781"}}
+{"timestamp":1712778186.3989837,"name":"offline","context":{"idset":"10787"}}
+{"timestamp":1712778186.462105,"name":"offline","context":{"idset":"10793"}}
+{"timestamp":1712778186.5052054,"name":"offline","context":{"idset":"10800"}}
+{"timestamp":1712778186.5437226,"name":"offline","context":{"idset":"10801"}}
+{"timestamp":1712778186.5857904,"name":"offline","context":{"idset":"10804"}}
+{"timestamp":1712778186.6299078,"name":"offline","context":{"idset":"10810"}}
+{"timestamp":1712778186.6525791,"name":"offline","context":{"idset":"10811"}}
+{"timestamp":1712778186.6582382,"name":"offline","context":{"idset":"10814"}}
+{"timestamp":1712778186.6637356,"name":"offline","context":{"idset":"10815"}}
+{"timestamp":1712778186.6687167,"name":"offline","context":{"idset":"10816"}}
+{"timestamp":1712778186.6734867,"name":"offline","context":{"idset":"10818"}}
+{"timestamp":1712778186.6781814,"name":"offline","context":{"idset":"10821"}}
+{"timestamp":1712778186.6895182,"name":"offline","context":{"idset":"10822"}}
+{"timestamp":1712778186.7448664,"name":"offline","context":{"idset":"10827"}}
+{"timestamp":1712778186.791152,"name":"offline","context":{"idset":"10831"}}
+{"timestamp":1712778186.8374176,"name":"offline","context":{"idset":"10833"}}
+{"timestamp":1712778186.8642948,"name":"offline","context":{"idset":"10837"}}
+{"timestamp":1712778186.8694296,"name":"offline","context":{"idset":"10838"}}
+{"timestamp":1712778186.8905549,"name":"offline","context":{"idset":"10839"}}
+{"timestamp":1712778186.9170864,"name":"offline","context":{"idset":"10840"}}
+{"timestamp":1712778186.9218466,"name":"offline","context":{"idset":"10845"}}
+{"timestamp":1712778186.9610178,"name":"offline","context":{"idset":"10846"}}
+{"timestamp":1712778187.0034113,"name":"offline","context":{"idset":"10848"}}
+{"timestamp":1712778187.0432415,"name":"offline","context":{"idset":"10849"}}
+{"timestamp":1712778187.0827599,"name":"offline","context":{"idset":"10850"}}
+{"timestamp":1712778187.1218851,"name":"offline","context":{"idset":"10851"}}
+{"timestamp":1712778187.1581705,"name":"offline","context":{"idset":"10852"}}
+{"timestamp":1712778187.194597,"name":"offline","context":{"idset":"10853"}}
+{"timestamp":1712778187.2214227,"name":"offline","context":{"idset":"10856"}}
+{"timestamp":1712778187.2270272,"name":"offline","context":{"idset":"10858"}}
+{"timestamp":1712778187.2324369,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1712778187.2381842,"name":"offline","context":{"idset":"10865"}}
+{"timestamp":1712778187.2550702,"name":"offline","context":{"idset":"10867"}}
+{"timestamp":1712778187.3084984,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1712778187.3378913,"name":"offline","context":{"idset":"10763"}}
+{"timestamp":1712778187.3906596,"name":"offline","context":{"idset":"10768"}}
+{"timestamp":1712778187.3959191,"name":"offline","context":{"idset":"10770"}}
+{"timestamp":1712778187.4228888,"name":"offline","context":{"idset":"10775"}}
+{"timestamp":1712778187.4540291,"name":"offline","context":{"idset":"10777"}}
+{"timestamp":1712778187.4858053,"name":"offline","context":{"idset":"10778"}}
+{"timestamp":1712778187.4904573,"name":"offline","context":{"idset":"10784"}}
+{"timestamp":1712778187.5153601,"name":"offline","context":{"idset":"10785"}}
+{"timestamp":1712778187.5441823,"name":"offline","context":{"idset":"10789"}}
+{"timestamp":1712778187.5726926,"name":"offline","context":{"idset":"10790"}}
+{"timestamp":1712778187.599777,"name":"offline","context":{"idset":"10802"}}
+{"timestamp":1712778187.6317947,"name":"offline","context":{"idset":"10803"}}
+{"timestamp":1712778187.663146,"name":"offline","context":{"idset":"10808"}}
+{"timestamp":1712778187.6963832,"name":"offline","context":{"idset":"10817"}}
+{"timestamp":1712778187.7281466,"name":"offline","context":{"idset":"10824"}}
+{"timestamp":1712778187.7577851,"name":"offline","context":{"idset":"10825"}}
+{"timestamp":1712778187.7880981,"name":"offline","context":{"idset":"10828"}}
+{"timestamp":1712778187.818433,"name":"offline","context":{"idset":"10830"}}
+{"timestamp":1712778187.8505812,"name":"offline","context":{"idset":"10834"}}
+{"timestamp":1712778187.8680041,"name":"offline","context":{"idset":"10835"}}
+{"timestamp":1712778187.8718145,"name":"offline","context":{"idset":"10836"}}
+{"timestamp":1712778187.8758333,"name":"offline","context":{"idset":"10842"}}
+{"timestamp":1712778187.9093521,"name":"offline","context":{"idset":"10843"}}
+{"timestamp":1712778187.9489601,"name":"offline","context":{"idset":"10844"}}
+{"timestamp":1712778187.9821038,"name":"offline","context":{"idset":"10847"}}
+{"timestamp":1712778188.0047383,"name":"offline","context":{"idset":"10854"}}
+{"timestamp":1712778188.0084362,"name":"offline","context":{"idset":"10859"}}
+{"timestamp":1712778188.0271058,"name":"offline","context":{"idset":"10860"}}
+{"timestamp":1712778188.0438921,"name":"offline","context":{"idset":"10861"}}
+{"timestamp":1712779278.1735909,"name":"drain","context":{"idset":"951","reason":"broker was unresponsive"}}
+{"timestamp":1712779278.1737723,"name":"drain","context":{"idset":"953","reason":"broker was unresponsive"}}
+{"timestamp":1712779282.9884412,"name":"online","context":{"idset":"496,498,586"}}
+{"timestamp":1712779307.1635306,"name":"undrain","context":{"idset":"496,498,586"}}
+{"timestamp":1712779350.6440535,"name":"online","context":{"idset":"10998"}}
+{"timestamp":1712779350.7557399,"name":"online","context":{"idset":"10997,11003,11005,11037"}}
+{"timestamp":1712779350.845221,"name":"online","context":{"idset":"11001"}}
+{"timestamp":1712779351.0218296,"name":"online","context":{"idset":"11012,11038"}}
+{"timestamp":1712779351.2133851,"name":"online","context":{"idset":"11002,11034"}}
+{"timestamp":1712779351.737998,"name":"online","context":{"idset":"11009"}}
+{"timestamp":1712779351.8978653,"name":"online","context":{"idset":"11043"}}
+{"timestamp":1712779352.0072865,"name":"online","context":{"idset":"11062"}}
+{"timestamp":1712779352.4141574,"name":"online","context":{"idset":"11069"}}
+{"timestamp":1712779352.999078,"name":"online","context":{"idset":"11028"}}
+{"timestamp":1712779353.1905148,"name":"online","context":{"idset":"11042"}}
+{"timestamp":1712779353.4330339,"name":"online","context":{"idset":"11052"}}
+{"timestamp":1712779353.5306914,"name":"online","context":{"idset":"11105"}}
+{"timestamp":1712779353.6434841,"name":"online","context":{"idset":"11094,11102"}}
+{"timestamp":1712779354.038444,"name":"online","context":{"idset":"11056,11084"}}
+{"timestamp":1712779354.157182,"name":"online","context":{"idset":"11019,11048,11053"}}
+{"timestamp":1712779354.2758813,"name":"online","context":{"idset":"11045,11063"}}
+{"timestamp":1712779354.3946974,"name":"online","context":{"idset":"11066,11101,11106"}}
+{"timestamp":1712779354.6051815,"name":"online","context":{"idset":"11006,11023,11027,11030,11093,11109"}}
+{"timestamp":1712779354.7075732,"name":"online","context":{"idset":"11086"}}
+{"timestamp":1712779354.8269055,"name":"online","context":{"idset":"11014,11016,11047,11049,11071,11081"}}
+{"timestamp":1712779354.9487846,"name":"online","context":{"idset":"11007,11039,11072,11075,11122"}}
+{"timestamp":1712779355.1563959,"name":"online","context":{"idset":"11004,11010-11011,11018,11025,11036,11040,11050,11070,11074,11113,11123"}}
+{"timestamp":1712779355.4087696,"name":"online","context":{"idset":"11013,11015,11017,11024,11026,11031,11033,11051,11055,11057,11060,11073,11076,11078,11082,11085,11087-11088,11090,11099,11104,11111,11116"}}
+{"timestamp":1712779355.5123649,"name":"online","context":{"idset":"11008,11029,11064-11065,11080,11089,11118,11121"}}
+{"timestamp":1712779355.6645513,"name":"online","context":{"idset":"11020,11092,11120"}}
+{"timestamp":1712779355.8054557,"name":"online","context":{"idset":"11035,11041,11044,11046,11054,11058-11059,11067,11077,11079,11083,11091,11095-11097,11100,11103,11107,11110,11112,11115"}}
+{"timestamp":1712779355.9509659,"name":"online","context":{"idset":"11032,11068,11098,11119"}}
+{"timestamp":1712779356.0579698,"name":"online","context":{"idset":"11108,11114,11117,11124"}}
+{"timestamp":1712779356.6931107,"name":"online","context":{"idset":"11061"}}
+{"timestamp":1712779927.6715689,"name":"online","context":{"idset":"10749"}}
+{"timestamp":1712779927.802525,"name":"online","context":{"idset":"10745,10759"}}
+{"timestamp":1712779927.9213221,"name":"online","context":{"idset":"10747"}}
+{"timestamp":1712779928.1571784,"name":"online","context":{"idset":"10744,10761"}}
+{"timestamp":1712779928.2785261,"name":"online","context":{"idset":"10755"}}
+{"timestamp":1712779928.3994861,"name":"online","context":{"idset":"10741,10743,10750,10756,10785"}}
+{"timestamp":1712779928.6193311,"name":"online","context":{"idset":"10769"}}
+{"timestamp":1712779928.75879,"name":"online","context":{"idset":"10762"}}
+{"timestamp":1712779928.8910322,"name":"online","context":{"idset":"10748,10805"}}
+{"timestamp":1712779929.0536838,"name":"online","context":{"idset":"10764"}}
+{"timestamp":1712779929.0847347,"name":"online","context":{"idset":"10791"}}
+{"timestamp":1712779929.3211565,"name":"online","context":{"idset":"10776"}}
+{"timestamp":1712779929.4586554,"name":"online","context":{"idset":"10773"}}
+{"timestamp":1712779929.7086239,"name":"online","context":{"idset":"10753"}}
+{"timestamp":1712779929.838094,"name":"online","context":{"idset":"10746"}}
+{"timestamp":1712779930.0345469,"name":"online","context":{"idset":"10763"}}
+{"timestamp":1712779930.2696474,"name":"online","context":{"idset":"10758"}}
+{"timestamp":1712779930.2889035,"name":"online","context":{"idset":"10767,10786"}}
+{"timestamp":1712779930.4421475,"name":"online","context":{"idset":"10752"}}
+{"timestamp":1712779930.6194379,"name":"online","context":{"idset":"10751,10757,10760,10778,10792,10794,10808"}}
+{"timestamp":1712779930.7315366,"name":"online","context":{"idset":"10783,10793,10865"}}
+{"timestamp":1712779930.8329728,"name":"online","context":{"idset":"10768,10770,10775,10784,10803"}}
+{"timestamp":1712779930.9027195,"name":"online","context":{"idset":"10766"}}
+{"timestamp":1712779931.0254481,"name":"online","context":{"idset":"10754,10772,10799"}}
+{"timestamp":1712779931.1413715,"name":"online","context":{"idset":"10771,10825,10861"}}
+{"timestamp":1712779931.3302915,"name":"online","context":{"idset":"10788,10797,10800,10816,10826,10828"}}
+{"timestamp":1712779931.5539896,"name":"online","context":{"idset":"10777,10789,10822,10839,10842,10850-10851"}}
+{"timestamp":1712779931.6751397,"name":"online","context":{"idset":"10790,10806,10864"}}
+{"timestamp":1712779931.8729267,"name":"online","context":{"idset":"10779-10782,10802,10809,10814,10817,10831-10832,10836,10847,10855,10858-10859"}}
+{"timestamp":1712779931.9920847,"name":"online","context":{"idset":"10765,10774,10796,10798,10807,10815,10835,10849,10860,10862,10866,10868"}}
+{"timestamp":1712779932.193507,"name":"online","context":{"idset":"10827,10830,10846,10863"}}
+{"timestamp":1712779932.3934934,"name":"online","context":{"idset":"10787,10801,10804,10811-10813,10819,10829,10834,10841,10845,10852-10853,10857"}}
+{"timestamp":1712779932.5992212,"name":"online","context":{"idset":"10824,10867"}}
+{"timestamp":1712779932.7030368,"name":"online","context":{"idset":"10810,10818,10821,10823,10833,10837-10838,10843-10844,10856"}}
+{"timestamp":1712779932.828053,"name":"online","context":{"idset":"10840,10848"}}
+{"timestamp":1712779933.028055,"name":"online","context":{"idset":"10820"}}
+{"timestamp":1712779933.313787,"name":"online","context":{"idset":"10854"}}
+{"timestamp":1712780400.5334163,"name":"online","context":{"idset":"814"}}
+{"timestamp":1712780413.3028424,"name":"drain","context":{"idset":"769-770,9875-9876","overwrite":0}}
+{"timestamp":1712780424.7866147,"name":"offline","context":{"idset":"9875"}}
+{"timestamp":1712780424.9201832,"name":"offline","context":{"idset":"9876"}}
+{"timestamp":1712780966.6532512,"name":"online","context":{"idset":"10501"}}
+{"timestamp":1712781574.9398363,"name":"online","context":{"idset":"828"}}
+{"timestamp":1712781575.3078046,"name":"online","context":{"idset":"827"}}
+{"timestamp":1712781651.213799,"name":"undrain","context":{"idset":"827-828"}}
+{"timestamp":1712782123.953275,"name":"undrain","context":{"idset":"795-796,815-816,842,871-872,949-954,963,973,975,977,10997-10998,11001-11020,11023-11110,11113-11114,11116,11119-11124"}}
+{"timestamp":1712782908.8304446,"name":"online","context":{"idset":"805"}}
+{"timestamp":1712783150.2728264,"name":"drain","context":{"idset":"580","reason":"broker was unresponsive"}}
+{"timestamp":1712783156.2739537,"name":"drain","context":{"idset":"501","reason":"broker was unresponsive"}}
+{"timestamp":1712783164.2746041,"name":"drain","context":{"idset":"506","reason":"broker was unresponsive"}}
+{"timestamp":1712783214.2687814,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1712783218.2686911,"name":"offline","context":{"idset":"501"}}
+{"timestamp":1712783228.269321,"name":"offline","context":{"idset":"506"}}
+{"timestamp":1712783278.273731,"name":"drain","context":{"idset":"508","reason":"broker was unresponsive"}}
+{"timestamp":1712783332.1390359,"name":"drain","context":{"idset":"814","overwrite":0}}
+{"timestamp":1712783343.7366958,"name":"offline","context":{"idset":"508"}}
+{"timestamp":1712783468.1806631,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1712783468.2694292,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1712783526.2686629,"name":"offline","context":{"idset":"814"}}
+{"timestamp":1712783582.2739837,"name":"drain","context":{"idset":"10572","reason":"broker was unresponsive"}}
+{"timestamp":1712783648.2694247,"name":"offline","context":{"idset":"10572"}}
+{"timestamp":1712783976.5270588,"name":"undrain","context":{"idset":"11509-11540"}}
+{"timestamp":1712784048.6162529,"name":"drain","context":{"idset":"11541-11572","overwrite":0}}
+{"timestamp":1712784155.8177819,"name":"online","context":{"idset":"9471"}}
+{"timestamp":1712784156.2442129,"name":"online","context":{"idset":"9472"}}
+{"timestamp":1712784156.5561304,"name":"online","context":{"idset":"806"}}
+{"timestamp":1712784752.3245814,"name":"drain","context":{"idset":"11509-11540","overwrite":0}}
+{"timestamp":1712785602.350831,"name":"undrain","context":{"idset":"9472"}}
+{"timestamp":1712785925.3118219,"name":"drain","context":{"idset":"11095","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712788908.2083852,"name":"undrain","context":{"idset":"821-824"}}
+{"timestamp":1712788937.3791847,"name":"undrain","context":{"idset":"819-820"}}
+{"timestamp":1712788958.4918723,"name":"undrain","context":{"idset":"883-884"}}
+{"timestamp":1712789932.1736479,"name":"drain","context":{"idset":"11015","reason":"broker was unresponsive"}}
+{"timestamp":1712789932.2734027,"name":"drain","context":{"idset":"11016","reason":"broker was unresponsive"}}
+{"timestamp":1712789936.2717652,"name":"drain","context":{"idset":"11005","reason":"broker was unresponsive"}}
+{"timestamp":1712789938.2751291,"name":"drain","context":{"idset":"11006","reason":"broker was unresponsive"}}
+{"timestamp":1712789994.1862273,"name":"offline","context":{"idset":"11015"}}
+{"timestamp":1712789994.2683895,"name":"offline","context":{"idset":"11016"}}
+{"timestamp":1712790002.1800253,"name":"offline","context":{"idset":"11005"}}
+{"timestamp":1712790002.2698097,"name":"offline","context":{"idset":"11006"}}
+{"timestamp":1712790364.1851747,"name":"undrain","context":{"idset":"11095"}}
+{"timestamp":1712791238.8350906,"name":"drain","context":{"idset":"773-774,11063-11064","overwrite":0}}
+{"timestamp":1712791246.1454072,"name":"offline","context":{"idset":"11063"}}
+{"timestamp":1712791246.3670788,"name":"offline","context":{"idset":"11064"}}
+{"timestamp":1712791436.2748356,"name":"drain","context":{"idset":"11062","reason":"broker was unresponsive"}}
+{"timestamp":1712791438.2759089,"name":"drain","context":{"idset":"11061","reason":"broker was unresponsive"}}
+{"timestamp":1712791500.1773376,"name":"offline","context":{"idset":"11061"}}
+{"timestamp":1712791500.2694912,"name":"offline","context":{"idset":"11062"}}
+{"timestamp":1712792380.1267438,"name":"drain","context":{"idset":"775-776,11095-11096","overwrite":0}}
+{"timestamp":1712792496.1737318,"name":"drain","context":{"idset":"11093","reason":"broker was unresponsive"}}
+{"timestamp":1712792496.2739158,"name":"drain","context":{"idset":"11094","reason":"broker was unresponsive"}}
+{"timestamp":1712792542.1813791,"name":"offline","context":{"idset":"11095"}}
+{"timestamp":1712792542.2695339,"name":"offline","context":{"idset":"11096"}}
+{"timestamp":1712792558.2691004,"name":"offline","context":{"idset":"11093"}}
+{"timestamp":1712792576.2695811,"name":"offline","context":{"idset":"11094"}}
+{"timestamp":1712792785.4481058,"name":"drain","context":{"idset":"11093-11096","reason":"epilog failed for jobid fomGU6HnX75","overwrite":0}}
+{"timestamp":1712793286.2737594,"name":"drain","context":{"idset":"502","reason":"broker was unresponsive"}}
+{"timestamp":1712793348.2683566,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1712793582.1446042,"name":"drain","context":{"idset":"10485-10516,10519-10570,10573-10612","overwrite":0}}
+{"timestamp":1712793741.6598742,"name":"drain","context":{"idset":"506","reason":"Rabbit crashed due to node bringup --JRG","overwrite":1}}
+{"timestamp":1712793790.1774707,"name":"drain","context":{"idset":"10542","reason":"epilog failed for jobid fokxKvhDHUb","overwrite":0}}
+{"timestamp":1712793790.2699566,"name":"offline","context":{"idset":"10542"}}
+{"timestamp":1712795494.9011507,"name":"drain","context":{"idset":"9333-9460","reason":"draining to run on-node HPE diags - KPN","overwrite":1}}
+{"timestamp":1712809889.0399728,"name":"undrain","context":{"idset":"9333-9460"}}
+{"timestamp":1712810409.7507937,"name":"drain","context":{"idset":"9359","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712810487.4417636,"name":"undrain","context":{"idset":"9359"}}
+{"timestamp":1712810556.2721529,"name":"drain","context":{"idset":"10085","reason":"broker was unresponsive"}}
+{"timestamp":1712810618.2710571,"name":"offline","context":{"idset":"10085"}}
+{"timestamp":1712820708.2740927,"name":"drain","context":{"idset":"9781","reason":"broker was unresponsive"}}
+{"timestamp":1712820774.2698095,"name":"offline","context":{"idset":"9781"}}
+{"timestamp":1712832930.2723477,"name":"drain","context":{"idset":"10396","reason":"broker was unresponsive"}}
+{"timestamp":1712832996.2691779,"name":"offline","context":{"idset":"10396"}}
+{"timestamp":1712838856.271975,"name":"drain","context":{"idset":"9616","reason":"broker was unresponsive"}}
+{"timestamp":1712838922.2681372,"name":"offline","context":{"idset":"9616"}}
+{"timestamp":1712840395.1714897,"name":"undrain","context":{"idset":"9781"}}
+{"timestamp":1712840457.3466792,"name":"undrain","context":{"idset":"9750"}}
+{"timestamp":1712845152.1719842,"name":"drain","context":{"idset":"11069","reason":"broker was unresponsive"}}
+{"timestamp":1712845152.2719579,"name":"drain","context":{"idset":"11070","reason":"broker was unresponsive"}}
+{"timestamp":1712845218.1797824,"name":"offline","context":{"idset":"11069"}}
+{"timestamp":1712845218.2703841,"name":"offline","context":{"idset":"11070"}}
+{"timestamp":1712845914.0093741,"name":"drain","context":{"idset":"88","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.1661949,"name":"drain","context":{"idset":"89","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.1754177,"name":"drain","context":{"idset":"90","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.1943781,"name":"drain","context":{"idset":"91","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2034197,"name":"drain","context":{"idset":"92","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.203512,"name":"drain","context":{"idset":"93","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2085533,"name":"drain","context":{"idset":"94","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2127266,"name":"drain","context":{"idset":"96","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2129905,"name":"drain","context":{"idset":"97","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2171111,"name":"drain","context":{"idset":"98","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2172067,"name":"drain","context":{"idset":"99","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2219527,"name":"drain","context":{"idset":"100","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2262118,"name":"drain","context":{"idset":"101","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2264829,"name":"drain","context":{"idset":"102","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2314146,"name":"drain","context":{"idset":"103","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2363729,"name":"drain","context":{"idset":"104","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2364719,"name":"drain","context":{"idset":"105","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2409718,"name":"drain","context":{"idset":"106","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2411032,"name":"drain","context":{"idset":"107","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2456834,"name":"drain","context":{"idset":"108","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2507477,"name":"drain","context":{"idset":"111","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.250906,"name":"drain","context":{"idset":"112","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2550921,"name":"drain","context":{"idset":"113","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2592022,"name":"drain","context":{"idset":"114","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.265218,"name":"drain","context":{"idset":"117","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2690482,"name":"drain","context":{"idset":"119","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2691579,"name":"drain","context":{"idset":"123","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2734921,"name":"drain","context":{"idset":"128","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2787094,"name":"drain","context":{"idset":"129","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2789078,"name":"drain","context":{"idset":"130","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2842114,"name":"drain","context":{"idset":"131","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2843924,"name":"drain","context":{"idset":"132","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2897997,"name":"drain","context":{"idset":"134","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.2949591,"name":"drain","context":{"idset":"135","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.295058,"name":"drain","context":{"idset":"137","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3015127,"name":"drain","context":{"idset":"139","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.306284,"name":"drain","context":{"idset":"140","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3063982,"name":"drain","context":{"idset":"143","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3113506,"name":"drain","context":{"idset":"144","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3116784,"name":"drain","context":{"idset":"145","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3168788,"name":"drain","context":{"idset":"149","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3275857,"name":"drain","context":{"idset":"150","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3276412,"name":"drain","context":{"idset":"151","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3358259,"name":"drain","context":{"idset":"153","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3362963,"name":"drain","context":{"idset":"207","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.352447,"name":"drain","context":{"idset":"208","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3605998,"name":"drain","context":{"idset":"209","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3606603,"name":"drain","context":{"idset":"210","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3648744,"name":"drain","context":{"idset":"225","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3702519,"name":"drain","context":{"idset":"249","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3703289,"name":"drain","context":{"idset":"250","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3763139,"name":"drain","context":{"idset":"251","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3763702,"name":"drain","context":{"idset":"252","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3813214,"name":"drain","context":{"idset":"278","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3862982,"name":"drain","context":{"idset":"279","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.386364,"name":"drain","context":{"idset":"280","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.390795,"name":"drain","context":{"idset":"281","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3955495,"name":"drain","context":{"idset":"343","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.3956294,"name":"drain","context":{"idset":"344","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4001682,"name":"drain","context":{"idset":"345","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4004056,"name":"drain","context":{"idset":"346","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4086807,"name":"drain","context":{"idset":"347","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4134715,"name":"drain","context":{"idset":"349","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.413902,"name":"drain","context":{"idset":"350","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4203632,"name":"drain","context":{"idset":"351","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4206138,"name":"drain","context":{"idset":"352","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4251676,"name":"drain","context":{"idset":"353","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4497268,"name":"drain","context":{"idset":"354","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4498293,"name":"drain","context":{"idset":"355","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4499118,"name":"drain","context":{"idset":"356","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4499776,"name":"drain","context":{"idset":"357","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.450027,"name":"drain","context":{"idset":"358","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4503345,"name":"drain","context":{"idset":"359","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4504051,"name":"drain","context":{"idset":"360","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4507809,"name":"drain","context":{"idset":"361","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.450933,"name":"drain","context":{"idset":"362","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.450994,"name":"drain","context":{"idset":"363","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4510682,"name":"drain","context":{"idset":"364","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4511361,"name":"drain","context":{"idset":"365","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4512088,"name":"drain","context":{"idset":"366","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4515636,"name":"drain","context":{"idset":"367","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4516547,"name":"drain","context":{"idset":"368","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4517157,"name":"drain","context":{"idset":"369","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4521458,"name":"drain","context":{"idset":"370","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4523478,"name":"drain","context":{"idset":"371","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4530768,"name":"drain","context":{"idset":"372","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4535162,"name":"drain","context":{"idset":"373","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4538672,"name":"drain","context":{"idset":"411","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4540384,"name":"drain","context":{"idset":"412","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4541948,"name":"drain","context":{"idset":"413","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4546962,"name":"drain","context":{"idset":"414","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4548593,"name":"drain","context":{"idset":"415","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4550929,"name":"drain","context":{"idset":"416","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4552941,"name":"drain","context":{"idset":"417","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4555488,"name":"drain","context":{"idset":"418","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4556832,"name":"drain","context":{"idset":"419","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4560292,"name":"drain","context":{"idset":"420","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4563804,"name":"drain","context":{"idset":"421","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4564786,"name":"drain","context":{"idset":"422","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4566157,"name":"drain","context":{"idset":"423","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4567575,"name":"drain","context":{"idset":"425","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4569631,"name":"drain","context":{"idset":"426","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4571004,"name":"drain","context":{"idset":"428","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4573584,"name":"drain","context":{"idset":"431","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4575474,"name":"drain","context":{"idset":"496","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4579751,"name":"drain","context":{"idset":"498","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4581406,"name":"drain","context":{"idset":"500","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4585905,"name":"drain","context":{"idset":"503","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4588947,"name":"drain","context":{"idset":"504","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4589872,"name":"drain","context":{"idset":"505","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4596813,"name":"drain","context":{"idset":"507","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4599235,"name":"drain","context":{"idset":"509","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4600823,"name":"drain","context":{"idset":"510","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4601471,"name":"drain","context":{"idset":"511","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4601967,"name":"drain","context":{"idset":"512","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4602816,"name":"drain","context":{"idset":"513","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4603479,"name":"drain","context":{"idset":"514","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4604118,"name":"drain","context":{"idset":"515","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4604764,"name":"drain","context":{"idset":"516","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4605451,"name":"drain","context":{"idset":"517","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4606128,"name":"drain","context":{"idset":"518","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4606795,"name":"drain","context":{"idset":"519","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4607363,"name":"drain","context":{"idset":"520","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4607873,"name":"drain","context":{"idset":"521","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4608386,"name":"drain","context":{"idset":"522","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4608836,"name":"drain","context":{"idset":"523","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4609451,"name":"drain","context":{"idset":"524","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4609921,"name":"drain","context":{"idset":"525","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4610546,"name":"drain","context":{"idset":"526","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4611244,"name":"drain","context":{"idset":"527","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4611912,"name":"drain","context":{"idset":"528","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4612584,"name":"drain","context":{"idset":"529","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4613426,"name":"drain","context":{"idset":"530","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4614005,"name":"drain","context":{"idset":"531","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4614437,"name":"drain","context":{"idset":"532","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4614959,"name":"drain","context":{"idset":"533","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4615529,"name":"drain","context":{"idset":"534","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4616063,"name":"drain","context":{"idset":"535","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4616582,"name":"drain","context":{"idset":"536","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4617207,"name":"drain","context":{"idset":"537","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4617684,"name":"drain","context":{"idset":"538","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4618189,"name":"drain","context":{"idset":"539","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4618762,"name":"drain","context":{"idset":"540","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4619448,"name":"drain","context":{"idset":"565","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4620156,"name":"drain","context":{"idset":"567","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4620805,"name":"drain","context":{"idset":"568","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4621396,"name":"drain","context":{"idset":"569","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4621851,"name":"drain","context":{"idset":"571","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4622304,"name":"drain","context":{"idset":"572","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4623091,"name":"drain","context":{"idset":"573","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4623792,"name":"drain","context":{"idset":"574","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4624326,"name":"drain","context":{"idset":"575","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.462486,"name":"drain","context":{"idset":"576","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4625537,"name":"drain","context":{"idset":"577","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4626098,"name":"drain","context":{"idset":"578","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4626806,"name":"drain","context":{"idset":"579","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4627376,"name":"drain","context":{"idset":"582","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.462791,"name":"drain","context":{"idset":"583","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4628439,"name":"drain","context":{"idset":"584","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4628968,"name":"drain","context":{"idset":"585","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4629483,"name":"drain","context":{"idset":"586","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4630244,"name":"drain","context":{"idset":"587","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4630959,"name":"drain","context":{"idset":"588","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4631608,"name":"drain","context":{"idset":"795","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4632492,"name":"drain","context":{"idset":"796","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4634273,"name":"drain","context":{"idset":"805","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4635084,"name":"drain","context":{"idset":"806","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.463577,"name":"drain","context":{"idset":"815","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4636338,"name":"drain","context":{"idset":"816","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4637196,"name":"drain","context":{"idset":"820","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4638064,"name":"drain","context":{"idset":"827","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4638875,"name":"drain","context":{"idset":"828","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4639726,"name":"drain","context":{"idset":"841","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4640341,"name":"drain","context":{"idset":"842","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4641094,"name":"drain","context":{"idset":"871","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4641633,"name":"drain","context":{"idset":"872","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4642344,"name":"drain","context":{"idset":"949","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4643381,"name":"drain","context":{"idset":"950","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4644246,"name":"drain","context":{"idset":"951","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4645035,"name":"drain","context":{"idset":"952","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4645877,"name":"drain","context":{"idset":"953","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4646659,"name":"drain","context":{"idset":"954","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4647284,"name":"drain","context":{"idset":"963","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4648166,"name":"drain","context":{"idset":"964","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4649124,"name":"drain","context":{"idset":"973","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4650059,"name":"drain","context":{"idset":"974","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.465091,"name":"drain","context":{"idset":"975","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4651816,"name":"drain","context":{"idset":"976","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4653754,"name":"drain","context":{"idset":"9333","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4655683,"name":"drain","context":{"idset":"9335","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4657571,"name":"drain","context":{"idset":"9337","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4659636,"name":"drain","context":{"idset":"9338","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4661603,"name":"drain","context":{"idset":"9339","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4663649,"name":"drain","context":{"idset":"9340","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.466552,"name":"drain","context":{"idset":"9342","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4667277,"name":"drain","context":{"idset":"9343","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4668987,"name":"drain","context":{"idset":"9345","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.467052,"name":"drain","context":{"idset":"9346","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4672496,"name":"drain","context":{"idset":"9348","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4674623,"name":"drain","context":{"idset":"9349","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4676385,"name":"drain","context":{"idset":"9350","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4677861,"name":"drain","context":{"idset":"9351","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4679167,"name":"drain","context":{"idset":"9352","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4680777,"name":"drain","context":{"idset":"9353","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4682016,"name":"drain","context":{"idset":"9354","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4683714,"name":"drain","context":{"idset":"9355","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4685254,"name":"drain","context":{"idset":"9358","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.468684,"name":"drain","context":{"idset":"9359","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4688706,"name":"drain","context":{"idset":"9360","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4690289,"name":"drain","context":{"idset":"9363","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4691651,"name":"drain","context":{"idset":"9364","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4693592,"name":"drain","context":{"idset":"9366","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4695077,"name":"drain","context":{"idset":"9367","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4696605,"name":"drain","context":{"idset":"9368","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4698124,"name":"drain","context":{"idset":"9369","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4699681,"name":"drain","context":{"idset":"9374","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4701307,"name":"drain","context":{"idset":"9375","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4702964,"name":"drain","context":{"idset":"9376","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4704752,"name":"drain","context":{"idset":"9377","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4706059,"name":"drain","context":{"idset":"9379","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4707704,"name":"drain","context":{"idset":"9380","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.470921,"name":"drain","context":{"idset":"9381","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4710937,"name":"drain","context":{"idset":"9382","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4712174,"name":"drain","context":{"idset":"9383","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4713922,"name":"drain","context":{"idset":"9385","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4715493,"name":"drain","context":{"idset":"9386","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4717119,"name":"drain","context":{"idset":"9387","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4718764,"name":"drain","context":{"idset":"9388","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4720423,"name":"drain","context":{"idset":"9389","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.472209,"name":"drain","context":{"idset":"9391","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4724,"name":"drain","context":{"idset":"9394","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4725866,"name":"drain","context":{"idset":"9395","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4727407,"name":"drain","context":{"idset":"9396","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4729161,"name":"drain","context":{"idset":"9397","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4730849,"name":"drain","context":{"idset":"9399","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4732561,"name":"drain","context":{"idset":"9400","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4734166,"name":"drain","context":{"idset":"9401","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4735992,"name":"drain","context":{"idset":"9403","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4737885,"name":"drain","context":{"idset":"9404","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4739354,"name":"drain","context":{"idset":"9405","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4740853,"name":"drain","context":{"idset":"9407","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4742413,"name":"drain","context":{"idset":"9408","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4744573,"name":"drain","context":{"idset":"9409","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4746034,"name":"drain","context":{"idset":"9411","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4747744,"name":"drain","context":{"idset":"9412","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4749517,"name":"drain","context":{"idset":"9414","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4751534,"name":"drain","context":{"idset":"9415","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4753268,"name":"drain","context":{"idset":"9417","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4754889,"name":"drain","context":{"idset":"9418","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4756565,"name":"drain","context":{"idset":"9419","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4758337,"name":"drain","context":{"idset":"9420","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4759777,"name":"drain","context":{"idset":"9421","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4761496,"name":"drain","context":{"idset":"9424","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4763565,"name":"drain","context":{"idset":"9425","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4765577,"name":"drain","context":{"idset":"9427","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4767303,"name":"drain","context":{"idset":"9430","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4768839,"name":"drain","context":{"idset":"9431","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4770637,"name":"drain","context":{"idset":"9432","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4772203,"name":"drain","context":{"idset":"9433","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4778934,"name":"drain","context":{"idset":"9434","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4780734,"name":"drain","context":{"idset":"9435","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4782901,"name":"drain","context":{"idset":"9436","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4784698,"name":"drain","context":{"idset":"9437","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4786658,"name":"drain","context":{"idset":"9443","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4788454,"name":"drain","context":{"idset":"9444","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4790366,"name":"drain","context":{"idset":"9589","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4792068,"name":"drain","context":{"idset":"9590","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4794343,"name":"drain","context":{"idset":"9591","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.479605,"name":"drain","context":{"idset":"9592","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4797728,"name":"drain","context":{"idset":"9593","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4799504,"name":"drain","context":{"idset":"9594","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4801199,"name":"drain","context":{"idset":"9595","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4803195,"name":"drain","context":{"idset":"9596","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4804873,"name":"drain","context":{"idset":"9597","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4806774,"name":"drain","context":{"idset":"9598","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4808836,"name":"drain","context":{"idset":"9599","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4810669,"name":"drain","context":{"idset":"9602","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4812446,"name":"drain","context":{"idset":"9603","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4814756,"name":"drain","context":{"idset":"9605","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4817159,"name":"drain","context":{"idset":"9606","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4819458,"name":"drain","context":{"idset":"9607","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4821749,"name":"drain","context":{"idset":"9608","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4824352,"name":"drain","context":{"idset":"9609","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4836004,"name":"drain","context":{"idset":"9610","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4838381,"name":"drain","context":{"idset":"9611","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4840682,"name":"drain","context":{"idset":"9612","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4843125,"name":"drain","context":{"idset":"9613","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4845552,"name":"drain","context":{"idset":"9614","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4847217,"name":"drain","context":{"idset":"9615","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4848905,"name":"drain","context":{"idset":"9617","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4850833,"name":"drain","context":{"idset":"9618","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4852564,"name":"drain","context":{"idset":"9620","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.485472,"name":"drain","context":{"idset":"9621","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4856586,"name":"drain","context":{"idset":"9626","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.485858,"name":"drain","context":{"idset":"9627","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4860454,"name":"drain","context":{"idset":"9628","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4862156,"name":"drain","context":{"idset":"9629","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4864082,"name":"drain","context":{"idset":"9630","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4866176,"name":"drain","context":{"idset":"9631","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4868031,"name":"drain","context":{"idset":"9633","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4869819,"name":"drain","context":{"idset":"9635","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4872127,"name":"drain","context":{"idset":"9636","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4874341,"name":"drain","context":{"idset":"9637","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4876382,"name":"drain","context":{"idset":"9638","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4878139,"name":"drain","context":{"idset":"9640","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4880288,"name":"drain","context":{"idset":"9643","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.488245,"name":"drain","context":{"idset":"9644","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4884901,"name":"drain","context":{"idset":"9645","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4887156,"name":"drain","context":{"idset":"9648","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4889092,"name":"drain","context":{"idset":"9650","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4890862,"name":"drain","context":{"idset":"9651","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.48928,"name":"drain","context":{"idset":"9652","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4894753,"name":"drain","context":{"idset":"9653","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4896841,"name":"drain","context":{"idset":"9655","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4898977,"name":"drain","context":{"idset":"9656","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4900997,"name":"drain","context":{"idset":"9657","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4903097,"name":"drain","context":{"idset":"9658","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4905589,"name":"drain","context":{"idset":"9659","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4908109,"name":"drain","context":{"idset":"9660","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4910645,"name":"drain","context":{"idset":"9661","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.491327,"name":"drain","context":{"idset":"9662","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4915895,"name":"drain","context":{"idset":"9663","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4918377,"name":"drain","context":{"idset":"9664","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4920964,"name":"drain","context":{"idset":"9665","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4923697,"name":"drain","context":{"idset":"9667","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4926255,"name":"drain","context":{"idset":"9668","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4928827,"name":"drain","context":{"idset":"9669","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4931352,"name":"drain","context":{"idset":"9670","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4934065,"name":"drain","context":{"idset":"9672","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4969051,"name":"drain","context":{"idset":"9673","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4971604,"name":"drain","context":{"idset":"9674","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4974291,"name":"drain","context":{"idset":"9675","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4976895,"name":"drain","context":{"idset":"9676","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.497952,"name":"drain","context":{"idset":"9677","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.498209,"name":"drain","context":{"idset":"9678","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4984789,"name":"drain","context":{"idset":"9681","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4987342,"name":"drain","context":{"idset":"9682","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4989913,"name":"drain","context":{"idset":"9683","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4992316,"name":"drain","context":{"idset":"9684","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4994895,"name":"drain","context":{"idset":"9686","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4997454,"name":"drain","context":{"idset":"9687","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.4999995,"name":"drain","context":{"idset":"9690","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5002837,"name":"drain","context":{"idset":"9691","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5005448,"name":"drain","context":{"idset":"9693","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5007946,"name":"drain","context":{"idset":"9694","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5010536,"name":"drain","context":{"idset":"9696","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5013235,"name":"drain","context":{"idset":"9697","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5015924,"name":"drain","context":{"idset":"9698","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5018611,"name":"drain","context":{"idset":"9699","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5021253,"name":"drain","context":{"idset":"9702","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5024052,"name":"drain","context":{"idset":"9703","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5029418,"name":"drain","context":{"idset":"9704","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5094166,"name":"drain","context":{"idset":"9705","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.50969,"name":"drain","context":{"idset":"9706","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5099528,"name":"drain","context":{"idset":"9707","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5102277,"name":"drain","context":{"idset":"9708","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5105007,"name":"drain","context":{"idset":"9709","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5107794,"name":"drain","context":{"idset":"9711","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5110452,"name":"drain","context":{"idset":"9712","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5113189,"name":"drain","context":{"idset":"9713","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5115895,"name":"drain","context":{"idset":"9714","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5118625,"name":"drain","context":{"idset":"9715","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5121293,"name":"drain","context":{"idset":"9719","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5124531,"name":"drain","context":{"idset":"9720","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5127251,"name":"drain","context":{"idset":"9722","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5129969,"name":"drain","context":{"idset":"9723","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5132844,"name":"drain","context":{"idset":"9725","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5135562,"name":"drain","context":{"idset":"9726","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5138388,"name":"drain","context":{"idset":"9727","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.514112,"name":"drain","context":{"idset":"9728","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5143995,"name":"drain","context":{"idset":"9729","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5146737,"name":"drain","context":{"idset":"9730","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5149543,"name":"drain","context":{"idset":"9731","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5152287,"name":"drain","context":{"idset":"9732","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5155239,"name":"drain","context":{"idset":"9733","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5158062,"name":"drain","context":{"idset":"9734","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5160887,"name":"drain","context":{"idset":"9735","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5163994,"name":"drain","context":{"idset":"9737","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5166907,"name":"drain","context":{"idset":"9738","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5169737,"name":"drain","context":{"idset":"9740","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5172508,"name":"drain","context":{"idset":"9742","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5175636,"name":"drain","context":{"idset":"9743","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5178387,"name":"drain","context":{"idset":"9744","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5181184,"name":"drain","context":{"idset":"9745","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5184047,"name":"drain","context":{"idset":"9746","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5186813,"name":"drain","context":{"idset":"9747","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5189579,"name":"drain","context":{"idset":"9752","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5192397,"name":"drain","context":{"idset":"9753","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5195315,"name":"drain","context":{"idset":"9754","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5198131,"name":"drain","context":{"idset":"9755","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5200882,"name":"drain","context":{"idset":"9756","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5203776,"name":"drain","context":{"idset":"9757","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5206554,"name":"drain","context":{"idset":"9759","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5209315,"name":"drain","context":{"idset":"9760","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5212095,"name":"drain","context":{"idset":"9761","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5215037,"name":"drain","context":{"idset":"9762","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5217812,"name":"drain","context":{"idset":"9763","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5220566,"name":"drain","context":{"idset":"9766","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5223684,"name":"drain","context":{"idset":"9768","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5226483,"name":"drain","context":{"idset":"9769","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.522927,"name":"drain","context":{"idset":"9770","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.52321,"name":"drain","context":{"idset":"9771","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5235145,"name":"drain","context":{"idset":"9772","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5237958,"name":"drain","context":{"idset":"9773","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5240822,"name":"drain","context":{"idset":"9775","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.524394,"name":"drain","context":{"idset":"9776","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5246816,"name":"drain","context":{"idset":"9777","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5249982,"name":"drain","context":{"idset":"9778","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5253055,"name":"drain","context":{"idset":"9780","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5256028,"name":"drain","context":{"idset":"9797","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5259023,"name":"drain","context":{"idset":"9798","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5262048,"name":"drain","context":{"idset":"9799","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5265172,"name":"drain","context":{"idset":"9800","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5267653,"name":"drain","context":{"idset":"9801","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5270631,"name":"drain","context":{"idset":"9803","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5273929,"name":"drain","context":{"idset":"9805","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5276973,"name":"drain","context":{"idset":"9808","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5279965,"name":"drain","context":{"idset":"9809","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5283003,"name":"drain","context":{"idset":"9810","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5285914,"name":"drain","context":{"idset":"9811","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5288839,"name":"drain","context":{"idset":"9812","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5291984,"name":"drain","context":{"idset":"9813","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5295234,"name":"drain","context":{"idset":"9814","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5298445,"name":"drain","context":{"idset":"9815","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5301661,"name":"drain","context":{"idset":"9817","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5304978,"name":"drain","context":{"idset":"9818","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5308211,"name":"drain","context":{"idset":"9821","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5311344,"name":"drain","context":{"idset":"9822","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5314631,"name":"drain","context":{"idset":"9823","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.531785,"name":"drain","context":{"idset":"9825","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5321171,"name":"drain","context":{"idset":"9826","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5324438,"name":"drain","context":{"idset":"9830","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5327594,"name":"drain","context":{"idset":"9831","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5330806,"name":"drain","context":{"idset":"9832","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.533406,"name":"drain","context":{"idset":"9833","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5337267,"name":"drain","context":{"idset":"9834","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5340519,"name":"drain","context":{"idset":"9835","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.534389,"name":"drain","context":{"idset":"9836","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5347381,"name":"drain","context":{"idset":"9838","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.535074,"name":"drain","context":{"idset":"9839","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5354078,"name":"drain","context":{"idset":"9841","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5357471,"name":"drain","context":{"idset":"9842","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5360692,"name":"drain","context":{"idset":"9843","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5364079,"name":"drain","context":{"idset":"9844","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5367362,"name":"drain","context":{"idset":"11097","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5370629,"name":"drain","context":{"idset":"11101","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5373921,"name":"drain","context":{"idset":"11105","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5377254,"name":"drain","context":{"idset":"11107","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.538404,"name":"drain","context":{"idset":"11765","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5387418,"name":"drain","context":{"idset":"11766","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5390735,"name":"drain","context":{"idset":"11767","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5394299,"name":"drain","context":{"idset":"11769","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5397661,"name":"drain","context":{"idset":"11770","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5401089,"name":"drain","context":{"idset":"11771","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5404501,"name":"drain","context":{"idset":"11773","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.540781,"name":"drain","context":{"idset":"11774","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5411248,"name":"drain","context":{"idset":"11775","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5414655,"name":"drain","context":{"idset":"11776","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5418036,"name":"drain","context":{"idset":"11777","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5421953,"name":"drain","context":{"idset":"11779","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5425324,"name":"drain","context":{"idset":"11780","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5428684,"name":"drain","context":{"idset":"11781","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5432115,"name":"drain","context":{"idset":"11782","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5435421,"name":"drain","context":{"idset":"11783","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5438824,"name":"drain","context":{"idset":"11784","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5442128,"name":"drain","context":{"idset":"11785","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5445647,"name":"drain","context":{"idset":"11786","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5449183,"name":"drain","context":{"idset":"11787","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5452592,"name":"drain","context":{"idset":"11788","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.545624,"name":"drain","context":{"idset":"11789","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5459647,"name":"drain","context":{"idset":"11790","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5463345,"name":"drain","context":{"idset":"11791","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5466795,"name":"drain","context":{"idset":"11794","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5470111,"name":"drain","context":{"idset":"11795","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5473671,"name":"drain","context":{"idset":"11796","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5477166,"name":"drain","context":{"idset":"11799","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5480585,"name":"drain","context":{"idset":"11800","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5483937,"name":"drain","context":{"idset":"11801","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5487235,"name":"drain","context":{"idset":"11803","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5490797,"name":"drain","context":{"idset":"11804","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5494318,"name":"drain","context":{"idset":"11805","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5497684,"name":"drain","context":{"idset":"11806","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5501194,"name":"drain","context":{"idset":"11807","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5504777,"name":"drain","context":{"idset":"11808","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5508296,"name":"drain","context":{"idset":"11809","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5511835,"name":"drain","context":{"idset":"11811","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5515356,"name":"drain","context":{"idset":"11812","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5518754,"name":"drain","context":{"idset":"11813","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.552218,"name":"drain","context":{"idset":"11814","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5525634,"name":"drain","context":{"idset":"11815","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.552912,"name":"drain","context":{"idset":"11816","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5531871,"name":"drain","context":{"idset":"11818","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5535038,"name":"drain","context":{"idset":"11821","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5538125,"name":"drain","context":{"idset":"11822","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5540628,"name":"drain","context":{"idset":"11823","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5543754,"name":"drain","context":{"idset":"11824","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5547135,"name":"drain","context":{"idset":"11825","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5550461,"name":"drain","context":{"idset":"11826","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5553908,"name":"drain","context":{"idset":"11827","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5557318,"name":"drain","context":{"idset":"11828","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5560706,"name":"drain","context":{"idset":"11829","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.556422,"name":"drain","context":{"idset":"11833","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5567548,"name":"drain","context":{"idset":"11835","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.557075,"name":"drain","context":{"idset":"11838","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5574186,"name":"drain","context":{"idset":"11839","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5577536,"name":"drain","context":{"idset":"11840","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5580757,"name":"drain","context":{"idset":"11841","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5584188,"name":"drain","context":{"idset":"11843","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5587585,"name":"drain","context":{"idset":"11844","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5590844,"name":"drain","context":{"idset":"11846","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5594258,"name":"drain","context":{"idset":"11847","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5597577,"name":"drain","context":{"idset":"11848","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5600944,"name":"drain","context":{"idset":"11850","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5604451,"name":"drain","context":{"idset":"11851","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5607975,"name":"drain","context":{"idset":"11852","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5611522,"name":"drain","context":{"idset":"11853","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5615053,"name":"drain","context":{"idset":"11856","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5618424,"name":"drain","context":{"idset":"11857","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5621703,"name":"drain","context":{"idset":"11860","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.56252,"name":"drain","context":{"idset":"11862","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5628555,"name":"drain","context":{"idset":"11863","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5631976,"name":"drain","context":{"idset":"11865","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5635529,"name":"drain","context":{"idset":"11866","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5638909,"name":"drain","context":{"idset":"11867","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5642292,"name":"drain","context":{"idset":"11868","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5645766,"name":"drain","context":{"idset":"11869","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5648859,"name":"drain","context":{"idset":"11870","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.565196,"name":"drain","context":{"idset":"11872","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5655394,"name":"drain","context":{"idset":"11873","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5658884,"name":"drain","context":{"idset":"11875","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5662401,"name":"drain","context":{"idset":"11876","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.566601,"name":"drain","context":{"idset":"11877","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5669448,"name":"drain","context":{"idset":"11880","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5672503,"name":"drain","context":{"idset":"11881","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.567574,"name":"drain","context":{"idset":"11882","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5678573,"name":"drain","context":{"idset":"11883","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5681162,"name":"drain","context":{"idset":"11884","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5684412,"name":"drain","context":{"idset":"11886","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5686877,"name":"drain","context":{"idset":"11889","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5689576,"name":"drain","context":{"idset":"11892","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5692811,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5696385,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5699768,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5703351,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5707159,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5710967,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5715299,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.57199,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5723989,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5728209,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5732892,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.574744,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5751522,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5755258,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5758975,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5762401,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5765927,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.57693,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5773129,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5777106,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5780771,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5784433,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5787895,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.579143,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5795565,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5799031,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5802381,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.580601,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5810351,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5814924,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5819471,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5824318,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5828869,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5833619,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5838175,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5842836,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5847409,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5852022,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5856719,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5861287,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5865898,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5870476,"name":"drain","context":{"idset":"43","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5875115,"name":"drain","context":{"idset":"44","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5879645,"name":"drain","context":{"idset":"45","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5884244,"name":"drain","context":{"idset":"46","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5888813,"name":"drain","context":{"idset":"47","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5893469,"name":"drain","context":{"idset":"48","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5898235,"name":"drain","context":{"idset":"49","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5902874,"name":"drain","context":{"idset":"50","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5907507,"name":"drain","context":{"idset":"51","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5912077,"name":"drain","context":{"idset":"52","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5916784,"name":"drain","context":{"idset":"53","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5921364,"name":"drain","context":{"idset":"54","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5926108,"name":"drain","context":{"idset":"55","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5930762,"name":"drain","context":{"idset":"56","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5935488,"name":"drain","context":{"idset":"57","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5940235,"name":"drain","context":{"idset":"58","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5944963,"name":"drain","context":{"idset":"59","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5949655,"name":"drain","context":{"idset":"60","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5954812,"name":"drain","context":{"idset":"85","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5959699,"name":"drain","context":{"idset":"86","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5964477,"name":"drain","context":{"idset":"87","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5969315,"name":"drain","context":{"idset":"95","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5974159,"name":"drain","context":{"idset":"109","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5978887,"name":"drain","context":{"idset":"110","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.598376,"name":"drain","context":{"idset":"115","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5988503,"name":"drain","context":{"idset":"116","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5993366,"name":"drain","context":{"idset":"118","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.5998187,"name":"drain","context":{"idset":"120","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6003053,"name":"drain","context":{"idset":"122","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6007781,"name":"drain","context":{"idset":"124","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6012483,"name":"drain","context":{"idset":"125","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6017351,"name":"drain","context":{"idset":"126","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6022227,"name":"drain","context":{"idset":"127","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.602711,"name":"drain","context":{"idset":"133","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6031928,"name":"drain","context":{"idset":"136","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6036847,"name":"drain","context":{"idset":"138","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6041765,"name":"drain","context":{"idset":"141","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6045833,"name":"drain","context":{"idset":"142","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6049497,"name":"drain","context":{"idset":"146","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6052909,"name":"drain","context":{"idset":"147","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6056368,"name":"drain","context":{"idset":"148","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6059737,"name":"drain","context":{"idset":"152","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6063545,"name":"drain","context":{"idset":"154","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.60672,"name":"drain","context":{"idset":"155","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6070955,"name":"drain","context":{"idset":"156","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6074653,"name":"drain","context":{"idset":"157","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6078231,"name":"drain","context":{"idset":"158","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6082423,"name":"drain","context":{"idset":"159","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6086364,"name":"drain","context":{"idset":"160","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6089828,"name":"drain","context":{"idset":"161","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6093454,"name":"drain","context":{"idset":"162","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6098258,"name":"drain","context":{"idset":"163","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6103256,"name":"drain","context":{"idset":"164","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6108136,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6113029,"name":"drain","context":{"idset":"166","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6117842,"name":"drain","context":{"idset":"167","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6122763,"name":"drain","context":{"idset":"168","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6127605,"name":"drain","context":{"idset":"169","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6132498,"name":"drain","context":{"idset":"170","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6137516,"name":"drain","context":{"idset":"171","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6142423,"name":"drain","context":{"idset":"172","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6165178,"name":"drain","context":{"idset":"173","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6173911,"name":"drain","context":{"idset":"174","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6190596,"name":"drain","context":{"idset":"175","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6207657,"name":"drain","context":{"idset":"176","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6218681,"name":"drain","context":{"idset":"177","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6231978,"name":"drain","context":{"idset":"178","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.624306,"name":"drain","context":{"idset":"179","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.625592,"name":"drain","context":{"idset":"180","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6272755,"name":"drain","context":{"idset":"181","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6286316,"name":"drain","context":{"idset":"182","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6299176,"name":"drain","context":{"idset":"183","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6310167,"name":"drain","context":{"idset":"184","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6327896,"name":"drain","context":{"idset":"185","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.633847,"name":"drain","context":{"idset":"186","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6348901,"name":"drain","context":{"idset":"187","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6367922,"name":"drain","context":{"idset":"188","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6373355,"name":"drain","context":{"idset":"189","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6378605,"name":"drain","context":{"idset":"190","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.638392,"name":"drain","context":{"idset":"191","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6389146,"name":"drain","context":{"idset":"192","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6394455,"name":"drain","context":{"idset":"193","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6399639,"name":"drain","context":{"idset":"194","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.640502,"name":"drain","context":{"idset":"195","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6410294,"name":"drain","context":{"idset":"196","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6415713,"name":"drain","context":{"idset":"197","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6420958,"name":"drain","context":{"idset":"198","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6426783,"name":"drain","context":{"idset":"199","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6432018,"name":"drain","context":{"idset":"200","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6440072,"name":"drain","context":{"idset":"201","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6445615,"name":"drain","context":{"idset":"202","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6450915,"name":"drain","context":{"idset":"203","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6456304,"name":"drain","context":{"idset":"204","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6461535,"name":"drain","context":{"idset":"205","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6466935,"name":"drain","context":{"idset":"206","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6472199,"name":"drain","context":{"idset":"211","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6503036,"name":"drain","context":{"idset":"212","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6508467,"name":"drain","context":{"idset":"213","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6513929,"name":"drain","context":{"idset":"214","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6519215,"name":"drain","context":{"idset":"215","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6524622,"name":"drain","context":{"idset":"216","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6529915,"name":"drain","context":{"idset":"218","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.653538,"name":"drain","context":{"idset":"219","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6540709,"name":"drain","context":{"idset":"220","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6546228,"name":"drain","context":{"idset":"221","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6551597,"name":"drain","context":{"idset":"222","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6557105,"name":"drain","context":{"idset":"223","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.656249,"name":"drain","context":{"idset":"224","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6568465,"name":"drain","context":{"idset":"226","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6573966,"name":"drain","context":{"idset":"227","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.657933,"name":"drain","context":{"idset":"228","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.658489,"name":"drain","context":{"idset":"229","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.65903,"name":"drain","context":{"idset":"230","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6595783,"name":"drain","context":{"idset":"231","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6601188,"name":"drain","context":{"idset":"232","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6606939,"name":"drain","context":{"idset":"233","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.661237,"name":"drain","context":{"idset":"234","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6617973,"name":"drain","context":{"idset":"235","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6623514,"name":"drain","context":{"idset":"236","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6628904,"name":"drain","context":{"idset":"237","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6634448,"name":"drain","context":{"idset":"238","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6639836,"name":"drain","context":{"idset":"239","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6645393,"name":"drain","context":{"idset":"240","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6650846,"name":"drain","context":{"idset":"241","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6656482,"name":"drain","context":{"idset":"242","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6661966,"name":"drain","context":{"idset":"243","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6667588,"name":"drain","context":{"idset":"244","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6673205,"name":"drain","context":{"idset":"245","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.667872,"name":"drain","context":{"idset":"246","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.668426,"name":"drain","context":{"idset":"247","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6689322,"name":"drain","context":{"idset":"248","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6694705,"name":"drain","context":{"idset":"277","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6700206,"name":"drain","context":{"idset":"282","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6705914,"name":"drain","context":{"idset":"283","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6711462,"name":"drain","context":{"idset":"284","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6717136,"name":"drain","context":{"idset":"285","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6723082,"name":"drain","context":{"idset":"286","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6728587,"name":"drain","context":{"idset":"287","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6734233,"name":"drain","context":{"idset":"288","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6739783,"name":"drain","context":{"idset":"289","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6745391,"name":"drain","context":{"idset":"290","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6750944,"name":"drain","context":{"idset":"291","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6756599,"name":"drain","context":{"idset":"292","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6762156,"name":"drain","context":{"idset":"293","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6767862,"name":"drain","context":{"idset":"294","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6773527,"name":"drain","context":{"idset":"295","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6779096,"name":"drain","context":{"idset":"296","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6784801,"name":"drain","context":{"idset":"297","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6790349,"name":"drain","context":{"idset":"298","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.679601,"name":"drain","context":{"idset":"299","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6801615,"name":"drain","context":{"idset":"300","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6807311,"name":"drain","context":{"idset":"301","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6813066,"name":"drain","context":{"idset":"302","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.681874,"name":"drain","context":{"idset":"303","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.682445,"name":"drain","context":{"idset":"304","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6830072,"name":"drain","context":{"idset":"305","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6835818,"name":"drain","context":{"idset":"306","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6841457,"name":"drain","context":{"idset":"307","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6847231,"name":"drain","context":{"idset":"308","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6853015,"name":"drain","context":{"idset":"309","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6858659,"name":"drain","context":{"idset":"310","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6864352,"name":"drain","context":{"idset":"311","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6869972,"name":"drain","context":{"idset":"312","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6875694,"name":"drain","context":{"idset":"313","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6881337,"name":"drain","context":{"idset":"314","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6887109,"name":"drain","context":{"idset":"315","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6893632,"name":"drain","context":{"idset":"316","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6899359,"name":"drain","context":{"idset":"317","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6905172,"name":"drain","context":{"idset":"318","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6910884,"name":"drain","context":{"idset":"319","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6916687,"name":"drain","context":{"idset":"320","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6922436,"name":"drain","context":{"idset":"321","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6928318,"name":"drain","context":{"idset":"322","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6934249,"name":"drain","context":{"idset":"323","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6939993,"name":"drain","context":{"idset":"324","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6945887,"name":"drain","context":{"idset":"325","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6951654,"name":"drain","context":{"idset":"326","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6957514,"name":"drain","context":{"idset":"327","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6963387,"name":"drain","context":{"idset":"328","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6969197,"name":"drain","context":{"idset":"329","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.69751,"name":"drain","context":{"idset":"330","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.6980915,"name":"drain","context":{"idset":"331","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.702698,"name":"drain","context":{"idset":"332","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.703305,"name":"drain","context":{"idset":"333","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7038856,"name":"drain","context":{"idset":"334","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7044766,"name":"drain","context":{"idset":"335","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7050564,"name":"drain","context":{"idset":"336","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7056515,"name":"drain","context":{"idset":"337","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7062316,"name":"drain","context":{"idset":"338","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7068744,"name":"drain","context":{"idset":"339","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7074702,"name":"drain","context":{"idset":"340","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7080545,"name":"drain","context":{"idset":"341","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7086592,"name":"drain","context":{"idset":"342","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7092395,"name":"drain","context":{"idset":"374","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.709836,"name":"drain","context":{"idset":"375","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.710433,"name":"drain","context":{"idset":"376","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7110174,"name":"drain","context":{"idset":"377","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7116129,"name":"drain","context":{"idset":"378","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7121966,"name":"drain","context":{"idset":"379","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7127945,"name":"drain","context":{"idset":"380","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7133961,"name":"drain","context":{"idset":"381","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7139826,"name":"drain","context":{"idset":"382","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7145867,"name":"drain","context":{"idset":"383","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7151749,"name":"drain","context":{"idset":"384","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7157757,"name":"drain","context":{"idset":"385","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7163732,"name":"drain","context":{"idset":"386","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7169635,"name":"drain","context":{"idset":"387","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.717577,"name":"drain","context":{"idset":"388","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7181697,"name":"drain","context":{"idset":"389","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7187777,"name":"drain","context":{"idset":"390","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7193856,"name":"drain","context":{"idset":"391","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7199752,"name":"drain","context":{"idset":"392","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7205756,"name":"drain","context":{"idset":"393","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7211683,"name":"drain","context":{"idset":"394","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7217722,"name":"drain","context":{"idset":"395","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7223816,"name":"drain","context":{"idset":"396","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7229729,"name":"drain","context":{"idset":"397","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7235742,"name":"drain","context":{"idset":"398","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7241683,"name":"drain","context":{"idset":"399","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7247837,"name":"drain","context":{"idset":"400","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7253854,"name":"drain","context":{"idset":"401","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.725976,"name":"drain","context":{"idset":"402","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7265778,"name":"drain","context":{"idset":"403","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7271731,"name":"drain","context":{"idset":"404","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7277846,"name":"drain","context":{"idset":"405","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7283931,"name":"drain","context":{"idset":"406","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7289903,"name":"drain","context":{"idset":"407","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7295976,"name":"drain","context":{"idset":"408","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7301981,"name":"drain","context":{"idset":"409","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7308185,"name":"drain","context":{"idset":"410","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7314327,"name":"drain","context":{"idset":"424","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7320297,"name":"drain","context":{"idset":"427","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7326357,"name":"drain","context":{"idset":"429","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7332332,"name":"drain","context":{"idset":"430","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7338481,"name":"drain","context":{"idset":"432","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.734463,"name":"drain","context":{"idset":"433","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7350647,"name":"drain","context":{"idset":"434","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7356791,"name":"drain","context":{"idset":"435","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7363007,"name":"drain","context":{"idset":"436","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7369051,"name":"drain","context":{"idset":"437","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7375221,"name":"drain","context":{"idset":"438","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7381268,"name":"drain","context":{"idset":"439","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7387457,"name":"drain","context":{"idset":"440","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7393661,"name":"drain","context":{"idset":"441","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7399712,"name":"drain","context":{"idset":"442","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7405879,"name":"drain","context":{"idset":"443","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7411976,"name":"drain","context":{"idset":"444","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7418177,"name":"drain","context":{"idset":"461","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7424402,"name":"drain","context":{"idset":"469","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7430527,"name":"drain","context":{"idset":"470","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7436705,"name":"drain","context":{"idset":"471","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7442999,"name":"drain","context":{"idset":"472","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7449119,"name":"drain","context":{"idset":"473","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7455325,"name":"drain","context":{"idset":"474","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7461441,"name":"drain","context":{"idset":"475","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7467582,"name":"drain","context":{"idset":"476","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7473724,"name":"drain","context":{"idset":"477","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7479706,"name":"drain","context":{"idset":"478","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7486117,"name":"drain","context":{"idset":"479","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7492299,"name":"drain","context":{"idset":"480","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7498598,"name":"drain","context":{"idset":"481","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7504826,"name":"drain","context":{"idset":"482","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7510967,"name":"drain","context":{"idset":"483","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7517319,"name":"drain","context":{"idset":"484","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7523692,"name":"drain","context":{"idset":"485","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7529883,"name":"drain","context":{"idset":"486","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7536259,"name":"drain","context":{"idset":"487","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7542455,"name":"drain","context":{"idset":"488","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7548788,"name":"drain","context":{"idset":"489","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7555139,"name":"drain","context":{"idset":"490","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7561355,"name":"drain","context":{"idset":"491","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7567804,"name":"drain","context":{"idset":"492","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7574692,"name":"drain","context":{"idset":"494","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7580926,"name":"drain","context":{"idset":"495","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7587297,"name":"drain","context":{"idset":"497","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7593663,"name":"drain","context":{"idset":"499","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7599797,"name":"drain","context":{"idset":"977","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7606099,"name":"drain","context":{"idset":"978","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7612216,"name":"drain","context":{"idset":"9205","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7618554,"name":"drain","context":{"idset":"9206","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7624843,"name":"drain","context":{"idset":"9207","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7631001,"name":"drain","context":{"idset":"9208","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7637253,"name":"drain","context":{"idset":"9209","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7643538,"name":"drain","context":{"idset":"9210","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7649708,"name":"drain","context":{"idset":"9211","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7656031,"name":"drain","context":{"idset":"9212","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7662213,"name":"drain","context":{"idset":"9213","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7668476,"name":"drain","context":{"idset":"9214","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7674785,"name":"drain","context":{"idset":"9215","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7680981,"name":"drain","context":{"idset":"9216","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7687271,"name":"drain","context":{"idset":"9217","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7693529,"name":"drain","context":{"idset":"9218","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7699687,"name":"drain","context":{"idset":"9219","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7705967,"name":"drain","context":{"idset":"9220","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.771215,"name":"drain","context":{"idset":"9221","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7718492,"name":"drain","context":{"idset":"9222","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7724776,"name":"drain","context":{"idset":"9223","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7731035,"name":"drain","context":{"idset":"9224","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7737837,"name":"drain","context":{"idset":"9225","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7744195,"name":"drain","context":{"idset":"9226","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.775044,"name":"drain","context":{"idset":"9227","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7756848,"name":"drain","context":{"idset":"9228","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7763205,"name":"drain","context":{"idset":"9229","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7769444,"name":"drain","context":{"idset":"9230","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7775879,"name":"drain","context":{"idset":"9231","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7782114,"name":"drain","context":{"idset":"9232","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7788482,"name":"drain","context":{"idset":"9233","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.779485,"name":"drain","context":{"idset":"9234","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7801147,"name":"drain","context":{"idset":"9235","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7807555,"name":"drain","context":{"idset":"9236","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7813869,"name":"drain","context":{"idset":"9237","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7820189,"name":"drain","context":{"idset":"9238","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7830448,"name":"drain","context":{"idset":"9239","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7836983,"name":"drain","context":{"idset":"9240","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7843397,"name":"drain","context":{"idset":"9241","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7849696,"name":"drain","context":{"idset":"9242","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7856116,"name":"drain","context":{"idset":"9243","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7862465,"name":"drain","context":{"idset":"9244","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7868934,"name":"drain","context":{"idset":"9245","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7875342,"name":"drain","context":{"idset":"9246","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.788161,"name":"drain","context":{"idset":"9247","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7888026,"name":"drain","context":{"idset":"9248","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7894583,"name":"drain","context":{"idset":"9249","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7900932,"name":"drain","context":{"idset":"9250","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7907317,"name":"drain","context":{"idset":"9251","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7913849,"name":"drain","context":{"idset":"9252","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7920191,"name":"drain","context":{"idset":"9253","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7935393,"name":"drain","context":{"idset":"9254","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7941873,"name":"drain","context":{"idset":"9255","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7948377,"name":"drain","context":{"idset":"9256","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7955177,"name":"drain","context":{"idset":"9257","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7961881,"name":"drain","context":{"idset":"9258","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7968497,"name":"drain","context":{"idset":"9259","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7975099,"name":"drain","context":{"idset":"9260","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7981553,"name":"drain","context":{"idset":"9261","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.7988243,"name":"drain","context":{"idset":"9262","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.799485,"name":"drain","context":{"idset":"9263","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8001385,"name":"drain","context":{"idset":"9264","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8007879,"name":"drain","context":{"idset":"9265","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.801456,"name":"drain","context":{"idset":"9266","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8021011,"name":"drain","context":{"idset":"9267","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8026917,"name":"drain","context":{"idset":"9268","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8033342,"name":"drain","context":{"idset":"9269","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8039591,"name":"drain","context":{"idset":"9270","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8045967,"name":"drain","context":{"idset":"9271","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8052232,"name":"drain","context":{"idset":"9272","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.805871,"name":"drain","context":{"idset":"9273","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8065193,"name":"drain","context":{"idset":"9274","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8072476,"name":"drain","context":{"idset":"9275","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8081543,"name":"drain","context":{"idset":"9276","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8088076,"name":"drain","context":{"idset":"9277","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8094552,"name":"drain","context":{"idset":"9278","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8100832,"name":"drain","context":{"idset":"9279","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8107307,"name":"drain","context":{"idset":"9280","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8113821,"name":"drain","context":{"idset":"9281","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8120139,"name":"drain","context":{"idset":"9282","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8126364,"name":"drain","context":{"idset":"9283","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8132515,"name":"drain","context":{"idset":"9284","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8138742,"name":"drain","context":{"idset":"9285","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8145261,"name":"drain","context":{"idset":"9286","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8151243,"name":"drain","context":{"idset":"9287","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.815697,"name":"drain","context":{"idset":"9288","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8179901,"name":"drain","context":{"idset":"9289","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8186665,"name":"drain","context":{"idset":"9290","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.819339,"name":"drain","context":{"idset":"9291","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8199897,"name":"drain","context":{"idset":"9292","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8206561,"name":"drain","context":{"idset":"9293","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8213243,"name":"drain","context":{"idset":"9294","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8219786,"name":"drain","context":{"idset":"9297","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.822648,"name":"drain","context":{"idset":"9298","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.823324,"name":"drain","context":{"idset":"9299","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8240392,"name":"drain","context":{"idset":"9300","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8247128,"name":"drain","context":{"idset":"9301","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8253977,"name":"drain","context":{"idset":"9302","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.826107,"name":"drain","context":{"idset":"9303","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8267481,"name":"drain","context":{"idset":"9304","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8274086,"name":"drain","context":{"idset":"9305","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8280444,"name":"drain","context":{"idset":"9306","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8287051,"name":"drain","context":{"idset":"9307","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8293719,"name":"drain","context":{"idset":"9308","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8300226,"name":"drain","context":{"idset":"9309","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.830688,"name":"drain","context":{"idset":"9310","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8313878,"name":"drain","context":{"idset":"9311","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.832839,"name":"drain","context":{"idset":"9312","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8342319,"name":"drain","context":{"idset":"9313","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8350239,"name":"drain","context":{"idset":"9314","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8357477,"name":"drain","context":{"idset":"9315","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8364942,"name":"drain","context":{"idset":"9316","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8371735,"name":"drain","context":{"idset":"9317","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8378675,"name":"drain","context":{"idset":"9318","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8385572,"name":"drain","context":{"idset":"9319","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8392344,"name":"drain","context":{"idset":"9320","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8399997,"name":"drain","context":{"idset":"9321","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8406882,"name":"drain","context":{"idset":"9322","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8413754,"name":"drain","context":{"idset":"9323","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8424249,"name":"drain","context":{"idset":"9324","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8430898,"name":"drain","context":{"idset":"9325","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8441191,"name":"drain","context":{"idset":"9326","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8458064,"name":"drain","context":{"idset":"9327","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8465033,"name":"drain","context":{"idset":"9328","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8471794,"name":"drain","context":{"idset":"9329","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8479407,"name":"drain","context":{"idset":"9330","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8486633,"name":"drain","context":{"idset":"9331","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8494918,"name":"drain","context":{"idset":"9332","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8501918,"name":"drain","context":{"idset":"9334","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8508945,"name":"drain","context":{"idset":"9336","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8515818,"name":"drain","context":{"idset":"9341","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8522778,"name":"drain","context":{"idset":"9344","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.853106,"name":"drain","context":{"idset":"9347","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8539245,"name":"drain","context":{"idset":"9356","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8546281,"name":"drain","context":{"idset":"9357","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.855377,"name":"drain","context":{"idset":"9361","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8560679,"name":"drain","context":{"idset":"9362","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8571434,"name":"drain","context":{"idset":"9365","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8578308,"name":"drain","context":{"idset":"9370","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8585196,"name":"drain","context":{"idset":"9371","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8592072,"name":"drain","context":{"idset":"9372","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8599174,"name":"drain","context":{"idset":"9373","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.86061,"name":"drain","context":{"idset":"9378","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8612859,"name":"drain","context":{"idset":"9384","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8619616,"name":"drain","context":{"idset":"9390","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8627181,"name":"drain","context":{"idset":"9392","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.863487,"name":"drain","context":{"idset":"9393","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.864166,"name":"drain","context":{"idset":"9398","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8651636,"name":"drain","context":{"idset":"9402","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8659523,"name":"drain","context":{"idset":"9406","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8667989,"name":"drain","context":{"idset":"9410","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8676133,"name":"drain","context":{"idset":"9413","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8683319,"name":"drain","context":{"idset":"9416","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8690314,"name":"drain","context":{"idset":"9422","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8697433,"name":"drain","context":{"idset":"9423","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8704343,"name":"drain","context":{"idset":"9426","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8710527,"name":"drain","context":{"idset":"9428","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8717132,"name":"drain","context":{"idset":"9429","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8724322,"name":"drain","context":{"idset":"9438","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8731303,"name":"drain","context":{"idset":"9439","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8738437,"name":"drain","context":{"idset":"9440","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8745499,"name":"drain","context":{"idset":"9441","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8750212,"name":"drain","context":{"idset":"9442","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8756921,"name":"drain","context":{"idset":"9445","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8760676,"name":"drain","context":{"idset":"9446","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8764472,"name":"drain","context":{"idset":"9449","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8768151,"name":"drain","context":{"idset":"9450","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8771818,"name":"drain","context":{"idset":"9451","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.877569,"name":"drain","context":{"idset":"9452","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8779318,"name":"drain","context":{"idset":"9453","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8783109,"name":"drain","context":{"idset":"9454","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8786726,"name":"drain","context":{"idset":"9455","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.879035,"name":"drain","context":{"idset":"9456","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8794124,"name":"drain","context":{"idset":"9457","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8797772,"name":"drain","context":{"idset":"9458","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.880197,"name":"drain","context":{"idset":"9459","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.880837,"name":"drain","context":{"idset":"9460","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8816476,"name":"drain","context":{"idset":"9461","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.882247,"name":"drain","context":{"idset":"9462","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8828855,"name":"drain","context":{"idset":"9463","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8835294,"name":"drain","context":{"idset":"9464","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8841972,"name":"drain","context":{"idset":"9465","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8849039,"name":"drain","context":{"idset":"9466","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8855624,"name":"drain","context":{"idset":"9467","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8861809,"name":"drain","context":{"idset":"9468","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8868213,"name":"drain","context":{"idset":"9469","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8875022,"name":"drain","context":{"idset":"9470","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8883362,"name":"drain","context":{"idset":"9471","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8890111,"name":"drain","context":{"idset":"9472","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8899789,"name":"drain","context":{"idset":"9473","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8906796,"name":"drain","context":{"idset":"9474","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8913562,"name":"drain","context":{"idset":"9475","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8920124,"name":"drain","context":{"idset":"9476","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.892683,"name":"drain","context":{"idset":"9477","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.893347,"name":"drain","context":{"idset":"9478","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8940134,"name":"drain","context":{"idset":"9479","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8948519,"name":"drain","context":{"idset":"9480","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8955028,"name":"drain","context":{"idset":"9481","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8961277,"name":"drain","context":{"idset":"9482","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8967621,"name":"drain","context":{"idset":"9483","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8976104,"name":"drain","context":{"idset":"9484","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8983431,"name":"drain","context":{"idset":"9485","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8990009,"name":"drain","context":{"idset":"9486","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.8997054,"name":"drain","context":{"idset":"9487","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.900362,"name":"drain","context":{"idset":"9488","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9010036,"name":"drain","context":{"idset":"9489","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9016807,"name":"drain","context":{"idset":"9490","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9023402,"name":"drain","context":{"idset":"9491","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9030001,"name":"drain","context":{"idset":"9492","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9036968,"name":"drain","context":{"idset":"9493","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9043577,"name":"drain","context":{"idset":"9494","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9049981,"name":"drain","context":{"idset":"9495","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9056675,"name":"drain","context":{"idset":"9496","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.906343,"name":"drain","context":{"idset":"9497","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9069953,"name":"drain","context":{"idset":"9498","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9076877,"name":"drain","context":{"idset":"9499","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9083536,"name":"drain","context":{"idset":"9500","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9090173,"name":"drain","context":{"idset":"9501","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9097157,"name":"drain","context":{"idset":"9502","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9104075,"name":"drain","context":{"idset":"9503","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9110506,"name":"drain","context":{"idset":"9504","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9117234,"name":"drain","context":{"idset":"9505","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9123182,"name":"drain","context":{"idset":"9506","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9129755,"name":"drain","context":{"idset":"9507","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9136548,"name":"drain","context":{"idset":"9508","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9143426,"name":"drain","context":{"idset":"9509","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9149985,"name":"drain","context":{"idset":"9510","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9156699,"name":"drain","context":{"idset":"9511","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9163198,"name":"drain","context":{"idset":"9512","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9170785,"name":"drain","context":{"idset":"9513","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9177437,"name":"drain","context":{"idset":"9514","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9184113,"name":"drain","context":{"idset":"9515","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9190753,"name":"drain","context":{"idset":"9516","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9197464,"name":"drain","context":{"idset":"9517","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.920409,"name":"drain","context":{"idset":"9518","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9210579,"name":"drain","context":{"idset":"9519","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9215834,"name":"drain","context":{"idset":"9520","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9221482,"name":"drain","context":{"idset":"9521","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9231572,"name":"drain","context":{"idset":"9522","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9238381,"name":"drain","context":{"idset":"9523","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9247234,"name":"drain","context":{"idset":"9524","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.925441,"name":"drain","context":{"idset":"9525","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9261136,"name":"drain","context":{"idset":"9526","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9267917,"name":"drain","context":{"idset":"9527","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9274712,"name":"drain","context":{"idset":"9528","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9281237,"name":"drain","context":{"idset":"9529","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9288108,"name":"drain","context":{"idset":"9530","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9295206,"name":"drain","context":{"idset":"9531","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.930202,"name":"drain","context":{"idset":"9532","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9308903,"name":"drain","context":{"idset":"9533","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9315805,"name":"drain","context":{"idset":"9534","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9322529,"name":"drain","context":{"idset":"9536","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9329438,"name":"drain","context":{"idset":"9537","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.933634,"name":"drain","context":{"idset":"9538","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9349124,"name":"drain","context":{"idset":"9539","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9356112,"name":"drain","context":{"idset":"9540","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9363139,"name":"drain","context":{"idset":"9541","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9370339,"name":"drain","context":{"idset":"9542","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9377465,"name":"drain","context":{"idset":"9543","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9384644,"name":"drain","context":{"idset":"9544","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.939158,"name":"drain","context":{"idset":"9545","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.939867,"name":"drain","context":{"idset":"9546","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.940572,"name":"drain","context":{"idset":"9547","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9412496,"name":"drain","context":{"idset":"9548","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.941967,"name":"drain","context":{"idset":"9549","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9427187,"name":"drain","context":{"idset":"9550","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9434366,"name":"drain","context":{"idset":"9551","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9441309,"name":"drain","context":{"idset":"9552","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9448473,"name":"drain","context":{"idset":"9553","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9455559,"name":"drain","context":{"idset":"9554","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9462569,"name":"drain","context":{"idset":"9555","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9474277,"name":"drain","context":{"idset":"9556","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9481096,"name":"drain","context":{"idset":"9557","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9488006,"name":"drain","context":{"idset":"9558","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9494951,"name":"drain","context":{"idset":"9559","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9502034,"name":"drain","context":{"idset":"9560","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9508858,"name":"drain","context":{"idset":"9563","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9515662,"name":"drain","context":{"idset":"9564","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9522524,"name":"drain","context":{"idset":"9565","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9529653,"name":"drain","context":{"idset":"9566","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9536698,"name":"drain","context":{"idset":"9567","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9543698,"name":"drain","context":{"idset":"9568","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9550481,"name":"drain","context":{"idset":"9569","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9557533,"name":"drain","context":{"idset":"9570","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9564641,"name":"drain","context":{"idset":"9571","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9571671,"name":"drain","context":{"idset":"9572","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9578981,"name":"drain","context":{"idset":"9573","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9585981,"name":"drain","context":{"idset":"9574","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9593086,"name":"drain","context":{"idset":"9575","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9601982,"name":"drain","context":{"idset":"9576","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.960902,"name":"drain","context":{"idset":"9577","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9621563,"name":"drain","context":{"idset":"9578","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9628837,"name":"drain","context":{"idset":"9579","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9636395,"name":"drain","context":{"idset":"9580","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9644148,"name":"drain","context":{"idset":"9581","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9651268,"name":"drain","context":{"idset":"9582","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9658537,"name":"drain","context":{"idset":"9583","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9665616,"name":"drain","context":{"idset":"9584","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9672914,"name":"drain","context":{"idset":"9585","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9679911,"name":"drain","context":{"idset":"9586","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9686933,"name":"drain","context":{"idset":"9587","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9693995,"name":"drain","context":{"idset":"9588","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9700909,"name":"drain","context":{"idset":"9600","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9707804,"name":"drain","context":{"idset":"9601","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.971489,"name":"drain","context":{"idset":"9604","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9721963,"name":"drain","context":{"idset":"9619","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9729266,"name":"drain","context":{"idset":"9622","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9736595,"name":"drain","context":{"idset":"9623","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9743845,"name":"drain","context":{"idset":"9624","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9751492,"name":"drain","context":{"idset":"9625","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9758856,"name":"drain","context":{"idset":"9632","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9766138,"name":"drain","context":{"idset":"9634","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9773376,"name":"drain","context":{"idset":"9639","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9780524,"name":"drain","context":{"idset":"9641","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9787836,"name":"drain","context":{"idset":"9642","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9795048,"name":"drain","context":{"idset":"9646","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9802067,"name":"drain","context":{"idset":"9647","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9809201,"name":"drain","context":{"idset":"9649","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9817843,"name":"drain","context":{"idset":"9654","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9825079,"name":"drain","context":{"idset":"9666","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9831734,"name":"drain","context":{"idset":"9671","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9838476,"name":"drain","context":{"idset":"9679","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.984515,"name":"drain","context":{"idset":"9680","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9851785,"name":"drain","context":{"idset":"9685","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9859726,"name":"drain","context":{"idset":"9688","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9867191,"name":"drain","context":{"idset":"9689","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9874995,"name":"drain","context":{"idset":"9692","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9882574,"name":"drain","context":{"idset":"9695","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9890275,"name":"drain","context":{"idset":"9700","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9897268,"name":"drain","context":{"idset":"9701","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9904368,"name":"drain","context":{"idset":"9710","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9911728,"name":"drain","context":{"idset":"9716","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.991905,"name":"drain","context":{"idset":"9717","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9926212,"name":"drain","context":{"idset":"9718","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9933357,"name":"drain","context":{"idset":"9721","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.994035,"name":"drain","context":{"idset":"9724","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.99472,"name":"drain","context":{"idset":"9736","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9953902,"name":"drain","context":{"idset":"9739","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9960835,"name":"drain","context":{"idset":"9741","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9967895,"name":"drain","context":{"idset":"9748","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9976468,"name":"drain","context":{"idset":"9749","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9983721,"name":"drain","context":{"idset":"9750","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9990492,"name":"drain","context":{"idset":"9751","reason":"broker was unresponsive"}}
+{"timestamp":1712845932.9997272,"name":"drain","context":{"idset":"9758","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0004108,"name":"drain","context":{"idset":"9764","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0011082,"name":"drain","context":{"idset":"9765","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0018382,"name":"drain","context":{"idset":"9767","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0025489,"name":"drain","context":{"idset":"9774","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0032558,"name":"drain","context":{"idset":"9779","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0039158,"name":"drain","context":{"idset":"9782","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0046065,"name":"drain","context":{"idset":"9783","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0053263,"name":"drain","context":{"idset":"9784","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0064595,"name":"drain","context":{"idset":"9785","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0071273,"name":"drain","context":{"idset":"9786","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.007796,"name":"drain","context":{"idset":"9791","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0084741,"name":"drain","context":{"idset":"9792","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0091686,"name":"drain","context":{"idset":"9793","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0098794,"name":"drain","context":{"idset":"9794","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0106177,"name":"drain","context":{"idset":"9795","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0115778,"name":"drain","context":{"idset":"9796","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0123651,"name":"drain","context":{"idset":"9802","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0131209,"name":"drain","context":{"idset":"9804","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0138772,"name":"drain","context":{"idset":"9806","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0146499,"name":"drain","context":{"idset":"9807","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0154276,"name":"drain","context":{"idset":"9816","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0161903,"name":"drain","context":{"idset":"9819","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0169692,"name":"drain","context":{"idset":"9820","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0177438,"name":"drain","context":{"idset":"9824","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0184827,"name":"drain","context":{"idset":"9827","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0191483,"name":"drain","context":{"idset":"9828","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0198185,"name":"drain","context":{"idset":"9829","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0205436,"name":"drain","context":{"idset":"9837","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0212047,"name":"drain","context":{"idset":"9840","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0219138,"name":"drain","context":{"idset":"9845","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0226066,"name":"drain","context":{"idset":"9846","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.023334,"name":"drain","context":{"idset":"9847","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0239904,"name":"drain","context":{"idset":"9848","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0246387,"name":"drain","context":{"idset":"9849","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0251493,"name":"drain","context":{"idset":"9850","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0255759,"name":"drain","context":{"idset":"9851","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0261323,"name":"drain","context":{"idset":"9852","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0268447,"name":"drain","context":{"idset":"9853","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0275249,"name":"drain","context":{"idset":"9854","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0281899,"name":"drain","context":{"idset":"9855","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0288615,"name":"drain","context":{"idset":"9856","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0295637,"name":"drain","context":{"idset":"9857","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0304894,"name":"drain","context":{"idset":"9858","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0312035,"name":"drain","context":{"idset":"9859","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0319297,"name":"drain","context":{"idset":"9860","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0326686,"name":"drain","context":{"idset":"9861","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0333922,"name":"drain","context":{"idset":"9862","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0341024,"name":"drain","context":{"idset":"9863","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0349741,"name":"drain","context":{"idset":"9864","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0356879,"name":"drain","context":{"idset":"9865","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0364161,"name":"drain","context":{"idset":"9866","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.037127,"name":"drain","context":{"idset":"9867","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0378177,"name":"drain","context":{"idset":"9868","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0385003,"name":"drain","context":{"idset":"9869","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0391812,"name":"drain","context":{"idset":"9870","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0404387,"name":"drain","context":{"idset":"9871","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0411451,"name":"drain","context":{"idset":"9872","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0418515,"name":"drain","context":{"idset":"9873","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.042593,"name":"drain","context":{"idset":"9874","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0433354,"name":"drain","context":{"idset":"9877","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0440576,"name":"drain","context":{"idset":"9878","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0447962,"name":"drain","context":{"idset":"9879","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0455163,"name":"drain","context":{"idset":"9880","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0461981,"name":"drain","context":{"idset":"9881","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0469153,"name":"drain","context":{"idset":"9882","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0477231,"name":"drain","context":{"idset":"9883","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0483825,"name":"drain","context":{"idset":"9884","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0490205,"name":"drain","context":{"idset":"9885","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0496676,"name":"drain","context":{"idset":"9886","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.050354,"name":"drain","context":{"idset":"9887","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0510027,"name":"drain","context":{"idset":"9888","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0516696,"name":"drain","context":{"idset":"9889","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.052352,"name":"drain","context":{"idset":"9890","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.053067,"name":"drain","context":{"idset":"9891","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0537961,"name":"drain","context":{"idset":"9892","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0545466,"name":"drain","context":{"idset":"9893","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0552819,"name":"drain","context":{"idset":"9894","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0560584,"name":"drain","context":{"idset":"9895","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0568118,"name":"drain","context":{"idset":"9896","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0575542,"name":"drain","context":{"idset":"9897","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0583043,"name":"drain","context":{"idset":"9898","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0590382,"name":"drain","context":{"idset":"9899","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0603206,"name":"drain","context":{"idset":"9900","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0610323,"name":"drain","context":{"idset":"9901","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0617685,"name":"drain","context":{"idset":"9902","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0625205,"name":"drain","context":{"idset":"9903","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0632479,"name":"drain","context":{"idset":"9904","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0640044,"name":"drain","context":{"idset":"9905","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0647914,"name":"drain","context":{"idset":"9906","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0655406,"name":"drain","context":{"idset":"9907","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0662806,"name":"drain","context":{"idset":"9908","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0670042,"name":"drain","context":{"idset":"9909","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0677562,"name":"drain","context":{"idset":"9910","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0685086,"name":"drain","context":{"idset":"9911","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0692456,"name":"drain","context":{"idset":"9912","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0699811,"name":"drain","context":{"idset":"9913","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0707161,"name":"drain","context":{"idset":"9914","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0714464,"name":"drain","context":{"idset":"9915","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0721893,"name":"drain","context":{"idset":"9916","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0729008,"name":"drain","context":{"idset":"9917","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0736048,"name":"drain","context":{"idset":"9918","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0743055,"name":"drain","context":{"idset":"9919","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0749953,"name":"drain","context":{"idset":"9920","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0757048,"name":"drain","context":{"idset":"9921","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0804505,"name":"drain","context":{"idset":"9922","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0811825,"name":"drain","context":{"idset":"9923","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.082093,"name":"drain","context":{"idset":"9924","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0828445,"name":"drain","context":{"idset":"9925","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0836325,"name":"drain","context":{"idset":"9926","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0844061,"name":"drain","context":{"idset":"9927","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0850701,"name":"drain","context":{"idset":"9928","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0857449,"name":"drain","context":{"idset":"9929","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0864592,"name":"drain","context":{"idset":"9930","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0871994,"name":"drain","context":{"idset":"9931","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0878963,"name":"drain","context":{"idset":"9932","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.08866,"name":"drain","context":{"idset":"9933","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0894244,"name":"drain","context":{"idset":"9934","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0901854,"name":"drain","context":{"idset":"9935","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0909438,"name":"drain","context":{"idset":"9936","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0917051,"name":"drain","context":{"idset":"9937","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.092459,"name":"drain","context":{"idset":"9938","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0932012,"name":"drain","context":{"idset":"9939","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0939746,"name":"drain","context":{"idset":"9940","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0947413,"name":"drain","context":{"idset":"9941","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.09554,"name":"drain","context":{"idset":"9942","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0962877,"name":"drain","context":{"idset":"9943","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0969863,"name":"drain","context":{"idset":"9944","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.097785,"name":"drain","context":{"idset":"9945","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0985496,"name":"drain","context":{"idset":"9946","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.0992963,"name":"drain","context":{"idset":"9947","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1000156,"name":"drain","context":{"idset":"9948","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1007845,"name":"drain","context":{"idset":"9949","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.101546,"name":"drain","context":{"idset":"9950","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1023371,"name":"drain","context":{"idset":"9951","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1031194,"name":"drain","context":{"idset":"9952","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1038752,"name":"drain","context":{"idset":"9953","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1046612,"name":"drain","context":{"idset":"9954","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1054227,"name":"drain","context":{"idset":"9955","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1062219,"name":"drain","context":{"idset":"9956","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1069868,"name":"drain","context":{"idset":"9957","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1077595,"name":"drain","context":{"idset":"9958","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1085315,"name":"drain","context":{"idset":"9959","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1093245,"name":"drain","context":{"idset":"9960","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1101074,"name":"drain","context":{"idset":"9961","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1108797,"name":"drain","context":{"idset":"9962","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.111629,"name":"drain","context":{"idset":"9963","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1123548,"name":"drain","context":{"idset":"9964","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1131117,"name":"drain","context":{"idset":"9965","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.113874,"name":"drain","context":{"idset":"9966","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1146441,"name":"drain","context":{"idset":"9967","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1154213,"name":"drain","context":{"idset":"9968","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1161857,"name":"drain","context":{"idset":"9969","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.116945,"name":"drain","context":{"idset":"9970","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1177006,"name":"drain","context":{"idset":"9971","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1184297,"name":"drain","context":{"idset":"9972","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1191671,"name":"drain","context":{"idset":"9973","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1199234,"name":"drain","context":{"idset":"9974","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1206911,"name":"drain","context":{"idset":"9975","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1214797,"name":"drain","context":{"idset":"9976","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1222043,"name":"drain","context":{"idset":"9977","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1229324,"name":"drain","context":{"idset":"9978","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.123662,"name":"drain","context":{"idset":"9979","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1243963,"name":"drain","context":{"idset":"9980","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1251478,"name":"drain","context":{"idset":"9981","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1259048,"name":"drain","context":{"idset":"9982","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1264942,"name":"drain","context":{"idset":"9983","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1270909,"name":"drain","context":{"idset":"9984","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1276829,"name":"drain","context":{"idset":"9985","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1282876,"name":"drain","context":{"idset":"9986","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1289165,"name":"drain","context":{"idset":"9987","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1296561,"name":"drain","context":{"idset":"9988","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1302917,"name":"drain","context":{"idset":"9989","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1309936,"name":"drain","context":{"idset":"9990","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1316133,"name":"drain","context":{"idset":"9991","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1322458,"name":"drain","context":{"idset":"9993","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1329255,"name":"drain","context":{"idset":"9994","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1336393,"name":"drain","context":{"idset":"9995","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1342981,"name":"drain","context":{"idset":"9996","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1349885,"name":"drain","context":{"idset":"9997","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1357222,"name":"drain","context":{"idset":"9998","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1364582,"name":"drain","context":{"idset":"9999","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.137182,"name":"drain","context":{"idset":"10000","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1379011,"name":"drain","context":{"idset":"10001","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1384385,"name":"drain","context":{"idset":"10002","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1391599,"name":"drain","context":{"idset":"10003","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1398385,"name":"drain","context":{"idset":"10004","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1403589,"name":"drain","context":{"idset":"10005","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1410947,"name":"drain","context":{"idset":"10006","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1418633,"name":"drain","context":{"idset":"10007","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1424565,"name":"drain","context":{"idset":"10008","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1431761,"name":"drain","context":{"idset":"10009","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.14394,"name":"drain","context":{"idset":"10010","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1447246,"name":"drain","context":{"idset":"10011","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1454935,"name":"drain","context":{"idset":"10012","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1461477,"name":"drain","context":{"idset":"10013","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1466947,"name":"drain","context":{"idset":"10014","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1472335,"name":"drain","context":{"idset":"10015","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1477275,"name":"drain","context":{"idset":"10016","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1484632,"name":"drain","context":{"idset":"10017","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1492252,"name":"drain","context":{"idset":"10018","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1497149,"name":"drain","context":{"idset":"10019","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1502287,"name":"drain","context":{"idset":"10020","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1508071,"name":"drain","context":{"idset":"10021","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1516054,"name":"drain","context":{"idset":"10022","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1521349,"name":"drain","context":{"idset":"10023","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1526315,"name":"drain","context":{"idset":"10024","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1532378,"name":"drain","context":{"idset":"10025","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1537237,"name":"drain","context":{"idset":"10026","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.154238,"name":"drain","context":{"idset":"10027","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1547012,"name":"drain","context":{"idset":"10028","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1553502,"name":"drain","context":{"idset":"10029","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1558025,"name":"drain","context":{"idset":"10030","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1564257,"name":"drain","context":{"idset":"10031","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1572828,"name":"drain","context":{"idset":"10032","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1581271,"name":"drain","context":{"idset":"10033","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1589689,"name":"drain","context":{"idset":"10034","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1598158,"name":"drain","context":{"idset":"10035","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1608553,"name":"drain","context":{"idset":"10036","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1616764,"name":"drain","context":{"idset":"10037","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1621628,"name":"drain","context":{"idset":"10038","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.162704,"name":"drain","context":{"idset":"10039","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.163168,"name":"drain","context":{"idset":"10040","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1637557,"name":"drain","context":{"idset":"10041","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1644247,"name":"drain","context":{"idset":"10042","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.165215,"name":"drain","context":{"idset":"10043","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1660058,"name":"drain","context":{"idset":"10044","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1667995,"name":"drain","context":{"idset":"10045","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1675911,"name":"drain","context":{"idset":"10046","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1683841,"name":"drain","context":{"idset":"10047","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1691771,"name":"drain","context":{"idset":"10048","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1699362,"name":"drain","context":{"idset":"10049","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.170692,"name":"drain","context":{"idset":"10050","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1714447,"name":"drain","context":{"idset":"10051","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1721857,"name":"drain","context":{"idset":"10052","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1729755,"name":"drain","context":{"idset":"10053","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1737115,"name":"drain","context":{"idset":"10054","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1745303,"name":"drain","context":{"idset":"10055","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1753883,"name":"drain","context":{"idset":"10056","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1762321,"name":"drain","context":{"idset":"10057","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1770797,"name":"drain","context":{"idset":"10058","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.177942,"name":"drain","context":{"idset":"10059","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1787777,"name":"drain","context":{"idset":"10060","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1796246,"name":"drain","context":{"idset":"10061","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1804895,"name":"drain","context":{"idset":"10062","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1813698,"name":"drain","context":{"idset":"10063","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1823914,"name":"drain","context":{"idset":"10064","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1832507,"name":"drain","context":{"idset":"10065","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1841161,"name":"drain","context":{"idset":"10066","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.18489,"name":"drain","context":{"idset":"10067","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1853828,"name":"drain","context":{"idset":"10068","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1860132,"name":"drain","context":{"idset":"10069","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1867216,"name":"drain","context":{"idset":"10070","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1872437,"name":"drain","context":{"idset":"10071","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1877606,"name":"drain","context":{"idset":"10072","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1883621,"name":"drain","context":{"idset":"10073","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1888325,"name":"drain","context":{"idset":"10074","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.189328,"name":"drain","context":{"idset":"10075","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1898191,"name":"drain","context":{"idset":"10076","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.19032,"name":"drain","context":{"idset":"10077","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1910055,"name":"drain","context":{"idset":"10078","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1915324,"name":"drain","context":{"idset":"10079","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1920812,"name":"drain","context":{"idset":"10080","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1928735,"name":"drain","context":{"idset":"10081","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1934831,"name":"drain","context":{"idset":"10082","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1939752,"name":"drain","context":{"idset":"10083","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1945176,"name":"drain","context":{"idset":"10084","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1950622,"name":"drain","context":{"idset":"10086","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1958454,"name":"drain","context":{"idset":"10087","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1967378,"name":"drain","context":{"idset":"10088","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1977365,"name":"drain","context":{"idset":"10089","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1985402,"name":"drain","context":{"idset":"10090","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.1993303,"name":"drain","context":{"idset":"10091","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2001424,"name":"drain","context":{"idset":"10092","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2009616,"name":"drain","context":{"idset":"10093","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2018189,"name":"drain","context":{"idset":"10094","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2026594,"name":"drain","context":{"idset":"10095","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2034645,"name":"drain","context":{"idset":"10096","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2042916,"name":"drain","context":{"idset":"10097","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2051027,"name":"drain","context":{"idset":"10098","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2059424,"name":"drain","context":{"idset":"10101","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2067759,"name":"drain","context":{"idset":"10102","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2075975,"name":"drain","context":{"idset":"10103","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2084169,"name":"drain","context":{"idset":"10104","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2092113,"name":"drain","context":{"idset":"10105","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2100356,"name":"drain","context":{"idset":"10106","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2108593,"name":"drain","context":{"idset":"10107","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2116487,"name":"drain","context":{"idset":"10108","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2124345,"name":"drain","context":{"idset":"10109","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2131968,"name":"drain","context":{"idset":"10110","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2139871,"name":"drain","context":{"idset":"10111","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2148228,"name":"drain","context":{"idset":"10112","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2155123,"name":"drain","context":{"idset":"10113","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.216388,"name":"drain","context":{"idset":"10114","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2172918,"name":"drain","context":{"idset":"10115","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2181747,"name":"drain","context":{"idset":"10116","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2190597,"name":"drain","context":{"idset":"10117","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2199438,"name":"drain","context":{"idset":"10118","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2208366,"name":"drain","context":{"idset":"10119","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2217338,"name":"drain","context":{"idset":"10120","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2226307,"name":"drain","context":{"idset":"10121","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.223527,"name":"drain","context":{"idset":"10122","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2242064,"name":"drain","context":{"idset":"10123","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.22469,"name":"drain","context":{"idset":"10124","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2251513,"name":"drain","context":{"idset":"10125","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2256374,"name":"drain","context":{"idset":"10126","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2260933,"name":"drain","context":{"idset":"10127","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.226563,"name":"drain","context":{"idset":"10128","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2270296,"name":"drain","context":{"idset":"10129","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2275012,"name":"drain","context":{"idset":"10130","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2279713,"name":"drain","context":{"idset":"10131","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2284436,"name":"drain","context":{"idset":"10132","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2290335,"name":"drain","context":{"idset":"10133","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.229696,"name":"drain","context":{"idset":"10134","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2304664,"name":"drain","context":{"idset":"10135","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2312486,"name":"drain","context":{"idset":"10136","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2318642,"name":"drain","context":{"idset":"10137","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2323589,"name":"drain","context":{"idset":"10138","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2328222,"name":"drain","context":{"idset":"10139","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2332969,"name":"drain","context":{"idset":"10140","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.233803,"name":"drain","context":{"idset":"10141","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2344904,"name":"drain","context":{"idset":"10142","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2356832,"name":"drain","context":{"idset":"10143","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2365832,"name":"drain","context":{"idset":"10144","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2374368,"name":"drain","context":{"idset":"10145","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2382784,"name":"drain","context":{"idset":"10146","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2390752,"name":"drain","context":{"idset":"10147","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2399354,"name":"drain","context":{"idset":"10148","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2407365,"name":"drain","context":{"idset":"10149","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2415602,"name":"drain","context":{"idset":"10150","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2423868,"name":"drain","context":{"idset":"10151","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2432001,"name":"drain","context":{"idset":"10152","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2440636,"name":"drain","context":{"idset":"10153","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2449181,"name":"drain","context":{"idset":"10154","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2457716,"name":"drain","context":{"idset":"10155","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2466338,"name":"drain","context":{"idset":"10156","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2477396,"name":"drain","context":{"idset":"10157","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2486508,"name":"drain","context":{"idset":"10158","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2494862,"name":"drain","context":{"idset":"10159","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2503109,"name":"drain","context":{"idset":"10160","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2511048,"name":"drain","context":{"idset":"10161","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2519164,"name":"drain","context":{"idset":"10162","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2527549,"name":"drain","context":{"idset":"10163","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2536163,"name":"drain","context":{"idset":"10164","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2545128,"name":"drain","context":{"idset":"10165","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2554328,"name":"drain","context":{"idset":"10166","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2564433,"name":"drain","context":{"idset":"10167","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2573631,"name":"drain","context":{"idset":"10168","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2582915,"name":"drain","context":{"idset":"10169","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2591956,"name":"drain","context":{"idset":"10170","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2601161,"name":"drain","context":{"idset":"10171","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2610395,"name":"drain","context":{"idset":"10172","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2619221,"name":"drain","context":{"idset":"10173","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2625096,"name":"drain","context":{"idset":"10174","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2630415,"name":"drain","context":{"idset":"10175","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2636213,"name":"drain","context":{"idset":"10176","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.264436,"name":"drain","context":{"idset":"10177","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2652829,"name":"drain","context":{"idset":"10178","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2661347,"name":"drain","context":{"idset":"10179","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2670002,"name":"drain","context":{"idset":"10180","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2678576,"name":"drain","context":{"idset":"10181","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2686386,"name":"drain","context":{"idset":"10182","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2694986,"name":"drain","context":{"idset":"10183","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2703257,"name":"drain","context":{"idset":"10184","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2711236,"name":"drain","context":{"idset":"10185","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.271966,"name":"drain","context":{"idset":"10186","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2728252,"name":"drain","context":{"idset":"10187","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2736912,"name":"drain","context":{"idset":"10188","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2745106,"name":"drain","context":{"idset":"10189","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2753208,"name":"drain","context":{"idset":"10190","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2761159,"name":"drain","context":{"idset":"10191","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2769654,"name":"drain","context":{"idset":"10192","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2778213,"name":"drain","context":{"idset":"10193","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2787466,"name":"drain","context":{"idset":"10194","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2796791,"name":"drain","context":{"idset":"10195","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2806134,"name":"drain","context":{"idset":"10196","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2815399,"name":"drain","context":{"idset":"10197","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2829635,"name":"drain","context":{"idset":"10198","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2839015,"name":"drain","context":{"idset":"10199","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2848339,"name":"drain","context":{"idset":"10200","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2857556,"name":"drain","context":{"idset":"10201","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2866073,"name":"drain","context":{"idset":"10202","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2874761,"name":"drain","context":{"idset":"10203","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2884121,"name":"drain","context":{"idset":"10204","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2893474,"name":"drain","context":{"idset":"10205","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2902837,"name":"drain","context":{"idset":"10206","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2911994,"name":"drain","context":{"idset":"10207","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2921181,"name":"drain","context":{"idset":"10208","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2930534,"name":"drain","context":{"idset":"10209","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2939825,"name":"drain","context":{"idset":"10210","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2949169,"name":"drain","context":{"idset":"10211","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2957783,"name":"drain","context":{"idset":"10212","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2966514,"name":"drain","context":{"idset":"10213","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2975128,"name":"drain","context":{"idset":"10214","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.2984991,"name":"drain","context":{"idset":"10215","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.299377,"name":"drain","context":{"idset":"10216","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3001962,"name":"drain","context":{"idset":"10217","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3010166,"name":"drain","context":{"idset":"10218","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3018365,"name":"drain","context":{"idset":"10219","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3027012,"name":"drain","context":{"idset":"10220","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3034804,"name":"drain","context":{"idset":"10221","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3044016,"name":"drain","context":{"idset":"10222","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3053405,"name":"drain","context":{"idset":"10223","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3062949,"name":"drain","context":{"idset":"10224","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3072205,"name":"drain","context":{"idset":"10225","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3081627,"name":"drain","context":{"idset":"10226","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3090062,"name":"drain","context":{"idset":"10227","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3099408,"name":"drain","context":{"idset":"10228","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3108904,"name":"drain","context":{"idset":"10229","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3118386,"name":"drain","context":{"idset":"10230","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3127725,"name":"drain","context":{"idset":"10231","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3136253,"name":"drain","context":{"idset":"10232","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3145175,"name":"drain","context":{"idset":"10233","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3154018,"name":"drain","context":{"idset":"10234","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3162458,"name":"drain","context":{"idset":"10235","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3167775,"name":"drain","context":{"idset":"10236","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3173118,"name":"drain","context":{"idset":"10237","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.317898,"name":"drain","context":{"idset":"10238","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3185031,"name":"drain","context":{"idset":"10239","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3190677,"name":"drain","context":{"idset":"10240","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3195984,"name":"drain","context":{"idset":"10241","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3203807,"name":"drain","context":{"idset":"10242","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3208945,"name":"drain","context":{"idset":"10243","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3213954,"name":"drain","context":{"idset":"10244","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.32195,"name":"drain","context":{"idset":"10245","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3227999,"name":"drain","context":{"idset":"10246","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3236973,"name":"drain","context":{"idset":"10247","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3245661,"name":"drain","context":{"idset":"10248","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3254662,"name":"drain","context":{"idset":"10249","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3263485,"name":"drain","context":{"idset":"10250","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3272579,"name":"drain","context":{"idset":"10252","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3288,"name":"drain","context":{"idset":"10253","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3296928,"name":"drain","context":{"idset":"10254","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.33059,"name":"drain","context":{"idset":"10255","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3314865,"name":"drain","context":{"idset":"10256","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3323748,"name":"drain","context":{"idset":"10257","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3332558,"name":"drain","context":{"idset":"10258","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3341382,"name":"drain","context":{"idset":"10259","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3350329,"name":"drain","context":{"idset":"10260","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3358912,"name":"drain","context":{"idset":"10263","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3367059,"name":"drain","context":{"idset":"10264","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3375401,"name":"drain","context":{"idset":"10265","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3383498,"name":"drain","context":{"idset":"10266","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3391652,"name":"drain","context":{"idset":"10267","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3400118,"name":"drain","context":{"idset":"10268","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.340853,"name":"drain","context":{"idset":"10269","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3417225,"name":"drain","context":{"idset":"10270","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3425908,"name":"drain","context":{"idset":"10271","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3434694,"name":"drain","context":{"idset":"10272","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3444214,"name":"drain","context":{"idset":"10273","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3453977,"name":"drain","context":{"idset":"10274","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.34637,"name":"drain","context":{"idset":"10275","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3473432,"name":"drain","context":{"idset":"10276","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3483119,"name":"drain","context":{"idset":"10277","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3492782,"name":"drain","context":{"idset":"10278","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3501194,"name":"drain","context":{"idset":"10279","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3510695,"name":"drain","context":{"idset":"10280","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3520114,"name":"drain","context":{"idset":"10281","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3528788,"name":"drain","context":{"idset":"10282","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3538475,"name":"drain","context":{"idset":"10283","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3547344,"name":"drain","context":{"idset":"10284","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3556223,"name":"drain","context":{"idset":"10285","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3565178,"name":"drain","context":{"idset":"10286","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3574212,"name":"drain","context":{"idset":"10287","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3583348,"name":"drain","context":{"idset":"10288","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3592577,"name":"drain","context":{"idset":"10289","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3603866,"name":"drain","context":{"idset":"10290","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3612881,"name":"drain","context":{"idset":"10291","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3621895,"name":"drain","context":{"idset":"10292","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.363013,"name":"drain","context":{"idset":"10293","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3639047,"name":"drain","context":{"idset":"10294","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3648272,"name":"drain","context":{"idset":"10295","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3657329,"name":"drain","context":{"idset":"10296","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3666315,"name":"drain","context":{"idset":"10297","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3675251,"name":"drain","context":{"idset":"10298","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3684304,"name":"drain","context":{"idset":"10299","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3692951,"name":"drain","context":{"idset":"10300","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3701344,"name":"drain","context":{"idset":"10301","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3709869,"name":"drain","context":{"idset":"10302","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3718903,"name":"drain","context":{"idset":"10304","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3728151,"name":"drain","context":{"idset":"10305","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3737972,"name":"drain","context":{"idset":"10306","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3747714,"name":"drain","context":{"idset":"10307","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3757546,"name":"drain","context":{"idset":"10308","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3767378,"name":"drain","context":{"idset":"10309","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3777251,"name":"drain","context":{"idset":"10310","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.378695,"name":"drain","context":{"idset":"10311","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3796852,"name":"drain","context":{"idset":"10312","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3806238,"name":"drain","context":{"idset":"10313","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3815045,"name":"drain","context":{"idset":"10314","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3822601,"name":"drain","context":{"idset":"10315","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3833203,"name":"drain","context":{"idset":"10316","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.384217,"name":"drain","context":{"idset":"10317","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3851199,"name":"drain","context":{"idset":"10318","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3860338,"name":"drain","context":{"idset":"10319","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.386941,"name":"drain","context":{"idset":"10320","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.387861,"name":"drain","context":{"idset":"10321","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3887544,"name":"drain","context":{"idset":"10322","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3896883,"name":"drain","context":{"idset":"10323","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.39061,"name":"drain","context":{"idset":"10324","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3915625,"name":"drain","context":{"idset":"10325","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3925145,"name":"drain","context":{"idset":"10326","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3934562,"name":"drain","context":{"idset":"10327","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3944793,"name":"drain","context":{"idset":"10328","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.395458,"name":"drain","context":{"idset":"10329","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3963976,"name":"drain","context":{"idset":"10330","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.39732,"name":"drain","context":{"idset":"10331","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3983986,"name":"drain","context":{"idset":"10332","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.3993239,"name":"drain","context":{"idset":"10333","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4002273,"name":"drain","context":{"idset":"10334","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4012995,"name":"drain","context":{"idset":"10335","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4022305,"name":"drain","context":{"idset":"10336","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4031458,"name":"drain","context":{"idset":"10337","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4040623,"name":"drain","context":{"idset":"10338","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4049709,"name":"drain","context":{"idset":"10339","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4058821,"name":"drain","context":{"idset":"10340","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4067898,"name":"drain","context":{"idset":"10341","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4076965,"name":"drain","context":{"idset":"10342","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4085639,"name":"drain","context":{"idset":"10345","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4094255,"name":"drain","context":{"idset":"10346","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4103231,"name":"drain","context":{"idset":"10347","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4112232,"name":"drain","context":{"idset":"10348","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.412169,"name":"drain","context":{"idset":"10349","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4130554,"name":"drain","context":{"idset":"10350","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.413923,"name":"drain","context":{"idset":"10351","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4148021,"name":"drain","context":{"idset":"10352","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4157202,"name":"drain","context":{"idset":"10353","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.416682,"name":"drain","context":{"idset":"10354","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4176393,"name":"drain","context":{"idset":"10357","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4186428,"name":"drain","context":{"idset":"10358","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.419651,"name":"drain","context":{"idset":"10359","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4206586,"name":"drain","context":{"idset":"10360","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4217973,"name":"drain","context":{"idset":"10361","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4227424,"name":"drain","context":{"idset":"10362","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.423681,"name":"drain","context":{"idset":"10363","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4246998,"name":"drain","context":{"idset":"10364","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4257109,"name":"drain","context":{"idset":"10365","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4267206,"name":"drain","context":{"idset":"10366","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4277308,"name":"drain","context":{"idset":"10367","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4287381,"name":"drain","context":{"idset":"10368","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4297416,"name":"drain","context":{"idset":"10369","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4305305,"name":"drain","context":{"idset":"10370","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4310544,"name":"drain","context":{"idset":"10371","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4318469,"name":"drain","context":{"idset":"10372","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.432745,"name":"drain","context":{"idset":"10373","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4336593,"name":"drain","context":{"idset":"10374","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4345357,"name":"drain","context":{"idset":"10375","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4354475,"name":"drain","context":{"idset":"10376","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.436398,"name":"drain","context":{"idset":"10377","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4373493,"name":"drain","context":{"idset":"10378","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4383421,"name":"drain","context":{"idset":"10379","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4392893,"name":"drain","context":{"idset":"10380","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4402134,"name":"drain","context":{"idset":"10381","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4411674,"name":"drain","context":{"idset":"10382","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4421039,"name":"drain","context":{"idset":"10383","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4429901,"name":"drain","context":{"idset":"10384","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4439352,"name":"drain","context":{"idset":"10385","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4448624,"name":"drain","context":{"idset":"10386","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4457865,"name":"drain","context":{"idset":"10387","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4468472,"name":"drain","context":{"idset":"10388","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.447767,"name":"drain","context":{"idset":"10389","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4486589,"name":"drain","context":{"idset":"10390","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4495776,"name":"drain","context":{"idset":"10391","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.450491,"name":"drain","context":{"idset":"10392","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4514215,"name":"drain","context":{"idset":"10393","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4523613,"name":"drain","context":{"idset":"10394","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4533279,"name":"drain","context":{"idset":"10395","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4543481,"name":"drain","context":{"idset":"10397","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.455369,"name":"drain","context":{"idset":"10398","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4563875,"name":"drain","context":{"idset":"10399","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4574001,"name":"drain","context":{"idset":"10400","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4583063,"name":"drain","context":{"idset":"10402","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4592786,"name":"drain","context":{"idset":"10403","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4603093,"name":"drain","context":{"idset":"10404","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4613245,"name":"drain","context":{"idset":"10405","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4623423,"name":"drain","context":{"idset":"10406","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4632776,"name":"drain","context":{"idset":"10407","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4641867,"name":"drain","context":{"idset":"10408","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4651041,"name":"drain","context":{"idset":"10409","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4660554,"name":"drain","context":{"idset":"10410","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4669573,"name":"drain","context":{"idset":"10411","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4678874,"name":"drain","context":{"idset":"10412","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4688394,"name":"drain","context":{"idset":"10413","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4697497,"name":"drain","context":{"idset":"10414","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4703805,"name":"drain","context":{"idset":"10415","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.470912,"name":"drain","context":{"idset":"10416","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4717071,"name":"drain","context":{"idset":"10417","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4725392,"name":"drain","context":{"idset":"10418","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.473258,"name":"drain","context":{"idset":"10419","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4741216,"name":"drain","context":{"idset":"10420","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4750192,"name":"drain","context":{"idset":"10421","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4759052,"name":"drain","context":{"idset":"10422","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4767797,"name":"drain","context":{"idset":"10423","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4776678,"name":"drain","context":{"idset":"10424","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4785395,"name":"drain","context":{"idset":"10425","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4794002,"name":"drain","context":{"idset":"10426","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4799914,"name":"drain","context":{"idset":"10427","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4808898,"name":"drain","context":{"idset":"10428","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4816463,"name":"drain","context":{"idset":"10429","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4822912,"name":"drain","context":{"idset":"10430","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4831777,"name":"drain","context":{"idset":"10431","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4839253,"name":"drain","context":{"idset":"10432","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4846056,"name":"drain","context":{"idset":"10433","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4852097,"name":"drain","context":{"idset":"10434","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4858093,"name":"drain","context":{"idset":"10435","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4866066,"name":"drain","context":{"idset":"10436","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4875481,"name":"drain","context":{"idset":"10437","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4884994,"name":"drain","context":{"idset":"10438","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4893119,"name":"drain","context":{"idset":"10439","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4898512,"name":"drain","context":{"idset":"10440","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4904714,"name":"drain","context":{"idset":"10441","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4910033,"name":"drain","context":{"idset":"10442","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4915962,"name":"drain","context":{"idset":"10443","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4923172,"name":"drain","context":{"idset":"10444","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4932199,"name":"drain","context":{"idset":"10445","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.494144,"name":"drain","context":{"idset":"10446","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4948118,"name":"drain","context":{"idset":"10447","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4956446,"name":"drain","context":{"idset":"10448","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4964719,"name":"drain","context":{"idset":"10449","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4971316,"name":"drain","context":{"idset":"10450","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4976954,"name":"drain","context":{"idset":"10451","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4984159,"name":"drain","context":{"idset":"10452","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.4989703,"name":"drain","context":{"idset":"10453","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.499517,"name":"drain","context":{"idset":"10454","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5000894,"name":"drain","context":{"idset":"10455","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5006402,"name":"drain","context":{"idset":"10456","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5011694,"name":"drain","context":{"idset":"10457","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5020132,"name":"drain","context":{"idset":"10458","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5028977,"name":"drain","context":{"idset":"10459","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5037894,"name":"drain","context":{"idset":"10460","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5046711,"name":"drain","context":{"idset":"10461","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5056396,"name":"drain","context":{"idset":"10462","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.506619,"name":"drain","context":{"idset":"10463","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5075884,"name":"drain","context":{"idset":"10464","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5085633,"name":"drain","context":{"idset":"10465","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5095437,"name":"drain","context":{"idset":"10466","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5105104,"name":"drain","context":{"idset":"10467","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5114703,"name":"drain","context":{"idset":"10468","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5124485,"name":"drain","context":{"idset":"10469","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5133631,"name":"drain","context":{"idset":"10470","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5142539,"name":"drain","context":{"idset":"10471","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.515167,"name":"drain","context":{"idset":"10472","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5161295,"name":"drain","context":{"idset":"10473","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5170877,"name":"drain","context":{"idset":"10474","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5181284,"name":"drain","context":{"idset":"10475","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5191717,"name":"drain","context":{"idset":"10476","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5202301,"name":"drain","context":{"idset":"10477","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5212955,"name":"drain","context":{"idset":"10478","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5223467,"name":"drain","context":{"idset":"10479","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5234046,"name":"drain","context":{"idset":"10480","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.524457,"name":"drain","context":{"idset":"10481","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.525496,"name":"drain","context":{"idset":"10482","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5264387,"name":"drain","context":{"idset":"10483","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.5273781,"name":"drain","context":{"idset":"10484","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6434138,"name":"drain","context":{"idset":"10613","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6445158,"name":"drain","context":{"idset":"10615","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6455505,"name":"drain","context":{"idset":"10616","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6465633,"name":"drain","context":{"idset":"10617","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6475992,"name":"drain","context":{"idset":"10618","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6486373,"name":"drain","context":{"idset":"10619","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6496279,"name":"drain","context":{"idset":"10620","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6506028,"name":"drain","context":{"idset":"10621","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6515744,"name":"drain","context":{"idset":"10622","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6526008,"name":"drain","context":{"idset":"10623","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6535573,"name":"drain","context":{"idset":"10624","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6544957,"name":"drain","context":{"idset":"10625","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6555173,"name":"drain","context":{"idset":"10626","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6566181,"name":"drain","context":{"idset":"10627","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6577346,"name":"drain","context":{"idset":"10628","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6588595,"name":"drain","context":{"idset":"10630","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6599293,"name":"drain","context":{"idset":"10631","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6610606,"name":"drain","context":{"idset":"10632","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.662178,"name":"drain","context":{"idset":"10633","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6632302,"name":"drain","context":{"idset":"10635","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6642098,"name":"drain","context":{"idset":"10636","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6652033,"name":"drain","context":{"idset":"10640","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6662111,"name":"drain","context":{"idset":"10641","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.667217,"name":"drain","context":{"idset":"10645","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6683316,"name":"drain","context":{"idset":"10646","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6694396,"name":"drain","context":{"idset":"10647","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6700785,"name":"drain","context":{"idset":"10648","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6709125,"name":"drain","context":{"idset":"10649","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6715262,"name":"drain","context":{"idset":"10650","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6721032,"name":"drain","context":{"idset":"10651","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6730015,"name":"drain","context":{"idset":"10652","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6740446,"name":"drain","context":{"idset":"10653","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6750476,"name":"drain","context":{"idset":"10654","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.676101,"name":"drain","context":{"idset":"10655","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6771734,"name":"drain","context":{"idset":"10656","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6782365,"name":"drain","context":{"idset":"10659","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6793013,"name":"drain","context":{"idset":"10660","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6803446,"name":"drain","context":{"idset":"10661","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6813879,"name":"drain","context":{"idset":"10662","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6824338,"name":"drain","context":{"idset":"10663","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6835742,"name":"drain","context":{"idset":"10664","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6845624,"name":"drain","context":{"idset":"10665","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6855779,"name":"drain","context":{"idset":"10666","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6866143,"name":"drain","context":{"idset":"10667","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6876717,"name":"drain","context":{"idset":"10668","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.688731,"name":"drain","context":{"idset":"10669","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6897974,"name":"drain","context":{"idset":"10670","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6909277,"name":"drain","context":{"idset":"10671","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6920669,"name":"drain","context":{"idset":"10672","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6931565,"name":"drain","context":{"idset":"10673","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6943285,"name":"drain","context":{"idset":"10674","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6954746,"name":"drain","context":{"idset":"10675","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6966364,"name":"drain","context":{"idset":"10676","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6977715,"name":"drain","context":{"idset":"10677","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.6990278,"name":"drain","context":{"idset":"10678","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7001162,"name":"drain","context":{"idset":"10679","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7011807,"name":"drain","context":{"idset":"10680","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7023365,"name":"drain","context":{"idset":"10681","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7034216,"name":"drain","context":{"idset":"10682","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7044587,"name":"drain","context":{"idset":"10683","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7055821,"name":"drain","context":{"idset":"10684","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7065818,"name":"drain","context":{"idset":"10685","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7076643,"name":"drain","context":{"idset":"10686","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7088192,"name":"drain","context":{"idset":"10687","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7099719,"name":"drain","context":{"idset":"10688","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.711123,"name":"drain","context":{"idset":"10689","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7122867,"name":"drain","context":{"idset":"10690","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7133417,"name":"drain","context":{"idset":"10691","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7143795,"name":"drain","context":{"idset":"10692","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7151375,"name":"drain","context":{"idset":"10693","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7161629,"name":"drain","context":{"idset":"10694","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.71733,"name":"drain","context":{"idset":"10695","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7184694,"name":"drain","context":{"idset":"10696","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7196119,"name":"drain","context":{"idset":"10697","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7205293,"name":"drain","context":{"idset":"10698","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7214265,"name":"drain","context":{"idset":"10699","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7220726,"name":"drain","context":{"idset":"10700","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7227066,"name":"drain","context":{"idset":"10701","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7233205,"name":"drain","context":{"idset":"10703","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7239091,"name":"drain","context":{"idset":"10704","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7246451,"name":"drain","context":{"idset":"10705","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7256789,"name":"drain","context":{"idset":"10706","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7266235,"name":"drain","context":{"idset":"10707","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7275639,"name":"drain","context":{"idset":"10708","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7284942,"name":"drain","context":{"idset":"10709","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7294812,"name":"drain","context":{"idset":"10710","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7304814,"name":"drain","context":{"idset":"10711","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7314563,"name":"drain","context":{"idset":"10713","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7324941,"name":"drain","context":{"idset":"10714","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7335355,"name":"drain","context":{"idset":"10715","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7345986,"name":"drain","context":{"idset":"10716","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7356503,"name":"drain","context":{"idset":"10717","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7369633,"name":"drain","context":{"idset":"10719","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7380323,"name":"drain","context":{"idset":"10720","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7391055,"name":"drain","context":{"idset":"10721","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7402189,"name":"drain","context":{"idset":"10722","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7409058,"name":"drain","context":{"idset":"10723","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7415416,"name":"drain","context":{"idset":"10724","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7421875,"name":"drain","context":{"idset":"10725","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7431045,"name":"drain","context":{"idset":"10726","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7437348,"name":"drain","context":{"idset":"10727","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7443752,"name":"drain","context":{"idset":"10728","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7451272,"name":"drain","context":{"idset":"10729","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7457569,"name":"drain","context":{"idset":"10730","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7467966,"name":"drain","context":{"idset":"10731","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7478545,"name":"drain","context":{"idset":"10732","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.748621,"name":"drain","context":{"idset":"10733","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7495999,"name":"drain","context":{"idset":"10734","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7505767,"name":"drain","context":{"idset":"10735","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7512107,"name":"drain","context":{"idset":"10736","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7520468,"name":"drain","context":{"idset":"10737","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7531056,"name":"drain","context":{"idset":"10738","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7541547,"name":"drain","context":{"idset":"10741","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.754786,"name":"drain","context":{"idset":"10743","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.755501,"name":"drain","context":{"idset":"10744","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7561066,"name":"drain","context":{"idset":"10745","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7568791,"name":"drain","context":{"idset":"10746","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7576954,"name":"drain","context":{"idset":"10747","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.758415,"name":"drain","context":{"idset":"10748","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7590234,"name":"drain","context":{"idset":"10749","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7599142,"name":"drain","context":{"idset":"10750","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7646432,"name":"drain","context":{"idset":"10751","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7655747,"name":"drain","context":{"idset":"10752","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7664957,"name":"drain","context":{"idset":"10753","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7674189,"name":"drain","context":{"idset":"10754","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.768368,"name":"drain","context":{"idset":"10755","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7694457,"name":"drain","context":{"idset":"10756","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7701552,"name":"drain","context":{"idset":"10757","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7710435,"name":"drain","context":{"idset":"10758","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7720115,"name":"drain","context":{"idset":"10759","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7729907,"name":"drain","context":{"idset":"10760","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7740214,"name":"drain","context":{"idset":"10761","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7751064,"name":"drain","context":{"idset":"10762","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7761657,"name":"drain","context":{"idset":"10763","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7772236,"name":"drain","context":{"idset":"10764","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7783103,"name":"drain","context":{"idset":"10765","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7793868,"name":"drain","context":{"idset":"10766","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7804654,"name":"drain","context":{"idset":"10767","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.781616,"name":"drain","context":{"idset":"10768","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7827287,"name":"drain","context":{"idset":"10769","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7839653,"name":"drain","context":{"idset":"10770","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7850716,"name":"drain","context":{"idset":"10771","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7861896,"name":"drain","context":{"idset":"10772","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7873008,"name":"drain","context":{"idset":"10773","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7884302,"name":"drain","context":{"idset":"10774","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7895226,"name":"drain","context":{"idset":"10775","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7905698,"name":"drain","context":{"idset":"10776","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7916145,"name":"drain","context":{"idset":"10777","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7926517,"name":"drain","context":{"idset":"10778","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7937148,"name":"drain","context":{"idset":"10779","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7947571,"name":"drain","context":{"idset":"10780","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7959383,"name":"drain","context":{"idset":"10781","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7971225,"name":"drain","context":{"idset":"10782","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.798377,"name":"drain","context":{"idset":"10783","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.7996857,"name":"drain","context":{"idset":"10784","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8008773,"name":"drain","context":{"idset":"10785","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8020625,"name":"drain","context":{"idset":"10786","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8028848,"name":"drain","context":{"idset":"10787","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8037231,"name":"drain","context":{"idset":"10788","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8048811,"name":"drain","context":{"idset":"10789","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8060522,"name":"drain","context":{"idset":"10790","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8072455,"name":"drain","context":{"idset":"10791","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.80845,"name":"drain","context":{"idset":"10792","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8096416,"name":"drain","context":{"idset":"10793","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8108387,"name":"drain","context":{"idset":"10794","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8118351,"name":"drain","context":{"idset":"10796","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8124859,"name":"drain","context":{"idset":"10797","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8131001,"name":"drain","context":{"idset":"10798","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8137696,"name":"drain","context":{"idset":"10799","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8144746,"name":"drain","context":{"idset":"10800","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8155577,"name":"drain","context":{"idset":"10801","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8166547,"name":"drain","context":{"idset":"10802","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8173053,"name":"drain","context":{"idset":"10803","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8182223,"name":"drain","context":{"idset":"10804","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8192468,"name":"drain","context":{"idset":"10805","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8203351,"name":"drain","context":{"idset":"10806","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8214428,"name":"drain","context":{"idset":"10807","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8225718,"name":"drain","context":{"idset":"10808","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8236799,"name":"drain","context":{"idset":"10809","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8248081,"name":"drain","context":{"idset":"10810","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8259616,"name":"drain","context":{"idset":"10811","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8270757,"name":"drain","context":{"idset":"10812","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8281956,"name":"drain","context":{"idset":"10813","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8293302,"name":"drain","context":{"idset":"10814","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8304365,"name":"drain","context":{"idset":"10815","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8315368,"name":"drain","context":{"idset":"10816","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8326581,"name":"drain","context":{"idset":"10817","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8337083,"name":"drain","context":{"idset":"10818","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8347585,"name":"drain","context":{"idset":"10819","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8358345,"name":"drain","context":{"idset":"10820","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8368571,"name":"drain","context":{"idset":"10821","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8380439,"name":"drain","context":{"idset":"10822","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8392551,"name":"drain","context":{"idset":"10823","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8404839,"name":"drain","context":{"idset":"10824","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8416998,"name":"drain","context":{"idset":"10825","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8429105,"name":"drain","context":{"idset":"10826","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8440757,"name":"drain","context":{"idset":"10827","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8453059,"name":"drain","context":{"idset":"10828","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8463557,"name":"drain","context":{"idset":"10829","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8474455,"name":"drain","context":{"idset":"10830","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8485289,"name":"drain","context":{"idset":"10831","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8496156,"name":"drain","context":{"idset":"10832","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8506839,"name":"drain","context":{"idset":"10833","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8517666,"name":"drain","context":{"idset":"10834","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8528581,"name":"drain","context":{"idset":"10835","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8539581,"name":"drain","context":{"idset":"10836","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8550336,"name":"drain","context":{"idset":"10837","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8561172,"name":"drain","context":{"idset":"10838","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.857198,"name":"drain","context":{"idset":"10839","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8582821,"name":"drain","context":{"idset":"10840","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8593628,"name":"drain","context":{"idset":"10841","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8604503,"name":"drain","context":{"idset":"10842","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8615158,"name":"drain","context":{"idset":"10843","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8626008,"name":"drain","context":{"idset":"10844","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.863677,"name":"drain","context":{"idset":"10845","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8647382,"name":"drain","context":{"idset":"10846","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.865804,"name":"drain","context":{"idset":"10847","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8668656,"name":"drain","context":{"idset":"10848","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8679085,"name":"drain","context":{"idset":"10849","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8690054,"name":"drain","context":{"idset":"10850","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8700967,"name":"drain","context":{"idset":"10851","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8711991,"name":"drain","context":{"idset":"10852","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8722997,"name":"drain","context":{"idset":"10853","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8734012,"name":"drain","context":{"idset":"10854","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8745189,"name":"drain","context":{"idset":"10855","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8756156,"name":"drain","context":{"idset":"10856","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8767169,"name":"drain","context":{"idset":"10857","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8778448,"name":"drain","context":{"idset":"10858","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8789561,"name":"drain","context":{"idset":"10859","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8800704,"name":"drain","context":{"idset":"10860","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8811665,"name":"drain","context":{"idset":"10861","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8822219,"name":"drain","context":{"idset":"10862","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8833585,"name":"drain","context":{"idset":"10863","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8843205,"name":"drain","context":{"idset":"10864","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8852332,"name":"drain","context":{"idset":"10865","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8862154,"name":"drain","context":{"idset":"10866","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8871846,"name":"drain","context":{"idset":"10867","reason":"broker was unresponsive"}}
+{"timestamp":1712845933.8881881,"name":"drain","context":{"idset":"10868","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0224776,"name":"drain","context":{"idset":"10997","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0237131,"name":"drain","context":{"idset":"10998","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0249317,"name":"drain","context":{"idset":"11001","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0261111,"name":"drain","context":{"idset":"11002","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0273309,"name":"drain","context":{"idset":"11003","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.028533,"name":"drain","context":{"idset":"11004","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0297487,"name":"drain","context":{"idset":"11007","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0309393,"name":"drain","context":{"idset":"11008","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.032083,"name":"drain","context":{"idset":"11009","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0330055,"name":"drain","context":{"idset":"11010","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0339682,"name":"drain","context":{"idset":"11011","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0349038,"name":"drain","context":{"idset":"11012","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0358729,"name":"drain","context":{"idset":"11013","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0368047,"name":"drain","context":{"idset":"11014","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.037787,"name":"drain","context":{"idset":"11017","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0387127,"name":"drain","context":{"idset":"11018","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0396833,"name":"drain","context":{"idset":"11019","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.040704,"name":"drain","context":{"idset":"11020","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0418048,"name":"drain","context":{"idset":"11023","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0429072,"name":"drain","context":{"idset":"11024","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0436783,"name":"drain","context":{"idset":"11025","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0443726,"name":"drain","context":{"idset":"11026","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.045037,"name":"drain","context":{"idset":"11027","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0457273,"name":"drain","context":{"idset":"11028","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0465534,"name":"drain","context":{"idset":"11029","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0476451,"name":"drain","context":{"idset":"11030","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0486944,"name":"drain","context":{"idset":"11031","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.050771,"name":"drain","context":{"idset":"11032","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0528333,"name":"drain","context":{"idset":"11033","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0540605,"name":"drain","context":{"idset":"11034","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0552189,"name":"drain","context":{"idset":"11035","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0565155,"name":"drain","context":{"idset":"11036","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0578585,"name":"drain","context":{"idset":"11037","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0591667,"name":"drain","context":{"idset":"11038","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0604873,"name":"drain","context":{"idset":"11039","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0618343,"name":"drain","context":{"idset":"11040","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.063143,"name":"drain","context":{"idset":"11041","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.064328,"name":"drain","context":{"idset":"11042","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0650802,"name":"drain","context":{"idset":"11043","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0660126,"name":"drain","context":{"idset":"11044","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0668957,"name":"drain","context":{"idset":"11045","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0678058,"name":"drain","context":{"idset":"11046","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0686486,"name":"drain","context":{"idset":"11047","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0693707,"name":"drain","context":{"idset":"11048","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0700555,"name":"drain","context":{"idset":"11049","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0707414,"name":"drain","context":{"idset":"11050","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0715766,"name":"drain","context":{"idset":"11051","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.072396,"name":"drain","context":{"idset":"11052","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0730786,"name":"drain","context":{"idset":"11053","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0741243,"name":"drain","context":{"idset":"11054","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0752122,"name":"drain","context":{"idset":"11055","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0765688,"name":"drain","context":{"idset":"11056","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0777693,"name":"drain","context":{"idset":"11057","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0788977,"name":"drain","context":{"idset":"11058","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0804994,"name":"drain","context":{"idset":"11059","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0818202,"name":"drain","context":{"idset":"11060","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0831139,"name":"drain","context":{"idset":"11065","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0844069,"name":"drain","context":{"idset":"11066","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0857832,"name":"drain","context":{"idset":"11067","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0866611,"name":"drain","context":{"idset":"11068","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.087358,"name":"drain","context":{"idset":"11071","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0880346,"name":"drain","context":{"idset":"11072","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0888777,"name":"drain","context":{"idset":"11073","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0897884,"name":"drain","context":{"idset":"11074","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0904901,"name":"drain","context":{"idset":"11075","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0911665,"name":"drain","context":{"idset":"11076","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0918565,"name":"drain","context":{"idset":"11077","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0927331,"name":"drain","context":{"idset":"11078","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.093853,"name":"drain","context":{"idset":"11079","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0949833,"name":"drain","context":{"idset":"11080","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0961399,"name":"drain","context":{"idset":"11081","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0973525,"name":"drain","context":{"idset":"11082","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0984869,"name":"drain","context":{"idset":"11083","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.0996761,"name":"drain","context":{"idset":"11084","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1007957,"name":"drain","context":{"idset":"11085","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1019409,"name":"drain","context":{"idset":"11086","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1030555,"name":"drain","context":{"idset":"11087","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1041558,"name":"drain","context":{"idset":"11088","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1053796,"name":"drain","context":{"idset":"11089","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.106499,"name":"drain","context":{"idset":"11090","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1076186,"name":"drain","context":{"idset":"11091","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1087568,"name":"drain","context":{"idset":"11092","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1098943,"name":"drain","context":{"idset":"11098","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1110151,"name":"drain","context":{"idset":"11099","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1121562,"name":"drain","context":{"idset":"11100","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1130133,"name":"drain","context":{"idset":"11102","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1140201,"name":"drain","context":{"idset":"11103","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1152153,"name":"drain","context":{"idset":"11104","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1164525,"name":"drain","context":{"idset":"11106","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1175976,"name":"drain","context":{"idset":"11108","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.118793,"name":"drain","context":{"idset":"11109","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1199963,"name":"drain","context":{"idset":"11110","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1212068,"name":"drain","context":{"idset":"11111","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1224787,"name":"drain","context":{"idset":"11112","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1237614,"name":"drain","context":{"idset":"11113","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1250005,"name":"drain","context":{"idset":"11114","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1262269,"name":"drain","context":{"idset":"11115","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1274939,"name":"drain","context":{"idset":"11116","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1287503,"name":"drain","context":{"idset":"11117","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1299584,"name":"drain","context":{"idset":"11118","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1312456,"name":"drain","context":{"idset":"11119","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1324883,"name":"drain","context":{"idset":"11120","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1339912,"name":"drain","context":{"idset":"11121","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1351562,"name":"drain","context":{"idset":"11122","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1363375,"name":"drain","context":{"idset":"11123","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1375427,"name":"drain","context":{"idset":"11124","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1387482,"name":"drain","context":{"idset":"11125","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1399457,"name":"drain","context":{"idset":"11126","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1412177,"name":"drain","context":{"idset":"11127","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1425776,"name":"drain","context":{"idset":"11128","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1439111,"name":"drain","context":{"idset":"11129","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1451838,"name":"drain","context":{"idset":"11130","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1463885,"name":"drain","context":{"idset":"11131","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1475048,"name":"drain","context":{"idset":"11132","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1486578,"name":"drain","context":{"idset":"11133","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1498168,"name":"drain","context":{"idset":"11134","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1508961,"name":"drain","context":{"idset":"11137","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1521137,"name":"drain","context":{"idset":"11138","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1533527,"name":"drain","context":{"idset":"11139","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1545942,"name":"drain","context":{"idset":"11140","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1558502,"name":"drain","context":{"idset":"11141","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1571612,"name":"drain","context":{"idset":"11142","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1584122,"name":"drain","context":{"idset":"11143","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.159652,"name":"drain","context":{"idset":"11144","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1608977,"name":"drain","context":{"idset":"11145","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1621628,"name":"drain","context":{"idset":"11146","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1634307,"name":"drain","context":{"idset":"11147","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1646631,"name":"drain","context":{"idset":"11148","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.16591,"name":"drain","context":{"idset":"11149","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1671383,"name":"drain","context":{"idset":"11150","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1683917,"name":"drain","context":{"idset":"11151","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1696613,"name":"drain","context":{"idset":"11152","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1708562,"name":"drain","context":{"idset":"11153","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1720328,"name":"drain","context":{"idset":"11154","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1732357,"name":"drain","context":{"idset":"11155","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1743541,"name":"drain","context":{"idset":"11156","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1753707,"name":"drain","context":{"idset":"11157","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1762302,"name":"drain","context":{"idset":"11158","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1771743,"name":"drain","context":{"idset":"11159","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1778998,"name":"drain","context":{"idset":"11160","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1787393,"name":"drain","context":{"idset":"11161","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1794598,"name":"drain","context":{"idset":"11162","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1803257,"name":"drain","context":{"idset":"11163","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1810944,"name":"drain","context":{"idset":"11164","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1819332,"name":"drain","context":{"idset":"11165","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1827106,"name":"drain","context":{"idset":"11166","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1834424,"name":"drain","context":{"idset":"11167","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1841974,"name":"drain","context":{"idset":"11168","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1849208,"name":"drain","context":{"idset":"11169","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1857281,"name":"drain","context":{"idset":"11170","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1864498,"name":"drain","context":{"idset":"11171","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1871431,"name":"drain","context":{"idset":"11172","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1879385,"name":"drain","context":{"idset":"11173","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.188808,"name":"drain","context":{"idset":"11174","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1895623,"name":"drain","context":{"idset":"11175","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1902587,"name":"drain","context":{"idset":"11176","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1913252,"name":"drain","context":{"idset":"11177","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1925025,"name":"drain","context":{"idset":"11178","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1936853,"name":"drain","context":{"idset":"11179","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1947165,"name":"drain","context":{"idset":"11180","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1957915,"name":"drain","context":{"idset":"11181","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1965551,"name":"drain","context":{"idset":"11182","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1976244,"name":"drain","context":{"idset":"11183","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1983891,"name":"drain","context":{"idset":"11184","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.1990919,"name":"drain","context":{"idset":"11185","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2001009,"name":"drain","context":{"idset":"11186","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2009447,"name":"drain","context":{"idset":"11187","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2017133,"name":"drain","context":{"idset":"11188","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2029469,"name":"drain","context":{"idset":"11189","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2041888,"name":"drain","context":{"idset":"11190","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2054517,"name":"drain","context":{"idset":"11191","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2067242,"name":"drain","context":{"idset":"11192","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2080173,"name":"drain","context":{"idset":"11193","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2092993,"name":"drain","context":{"idset":"11194","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.210577,"name":"drain","context":{"idset":"11195","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2118416,"name":"drain","context":{"idset":"11196","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.213012,"name":"drain","context":{"idset":"11197","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2141614,"name":"drain","context":{"idset":"11198","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2153296,"name":"drain","context":{"idset":"11199","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2164879,"name":"drain","context":{"idset":"11200","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.217731,"name":"drain","context":{"idset":"11201","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2189453,"name":"drain","context":{"idset":"11202","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2196839,"name":"drain","context":{"idset":"11203","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2204056,"name":"drain","context":{"idset":"11204","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2214458,"name":"drain","context":{"idset":"11205","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2226911,"name":"drain","context":{"idset":"11206","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2240598,"name":"drain","context":{"idset":"11207","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2253401,"name":"drain","context":{"idset":"11208","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2265713,"name":"drain","context":{"idset":"11209","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2277901,"name":"drain","context":{"idset":"11210","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2290089,"name":"drain","context":{"idset":"11211","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2303338,"name":"drain","context":{"idset":"11212","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2316167,"name":"drain","context":{"idset":"11213","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2331846,"name":"drain","context":{"idset":"11214","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2347596,"name":"drain","context":{"idset":"11215","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2363086,"name":"drain","context":{"idset":"11216","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.237664,"name":"drain","context":{"idset":"11217","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.239053,"name":"drain","context":{"idset":"11218","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2404509,"name":"drain","context":{"idset":"11219","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2416241,"name":"drain","context":{"idset":"11220","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2423804,"name":"drain","context":{"idset":"11221","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2434196,"name":"drain","context":{"idset":"11222","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2446079,"name":"drain","context":{"idset":"11223","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2457848,"name":"drain","context":{"idset":"11224","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2470648,"name":"drain","context":{"idset":"11225","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2482891,"name":"drain","context":{"idset":"11226","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2495198,"name":"drain","context":{"idset":"11227","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2507763,"name":"drain","context":{"idset":"11228","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2521062,"name":"drain","context":{"idset":"11229","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.253459,"name":"drain","context":{"idset":"11230","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2546341,"name":"drain","context":{"idset":"11231","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2558279,"name":"drain","context":{"idset":"11232","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2570724,"name":"drain","context":{"idset":"11233","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2583704,"name":"drain","context":{"idset":"11234","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2596562,"name":"drain","context":{"idset":"11235","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2609279,"name":"drain","context":{"idset":"11236","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2622268,"name":"drain","context":{"idset":"11237","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2635303,"name":"drain","context":{"idset":"11238","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2647729,"name":"drain","context":{"idset":"11239","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2659917,"name":"drain","context":{"idset":"11240","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2673569,"name":"drain","context":{"idset":"11241","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2686851,"name":"drain","context":{"idset":"11242","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2699947,"name":"drain","context":{"idset":"11243","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2713079,"name":"drain","context":{"idset":"11244","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2726083,"name":"drain","context":{"idset":"11245","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2739022,"name":"drain","context":{"idset":"11246","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.275177,"name":"drain","context":{"idset":"11247","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2764628,"name":"drain","context":{"idset":"11248","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2777686,"name":"drain","context":{"idset":"11251","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2790246,"name":"drain","context":{"idset":"11252","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2803118,"name":"drain","context":{"idset":"11253","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2815511,"name":"drain","context":{"idset":"11254","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2828119,"name":"drain","context":{"idset":"11255","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.2961729,"name":"drain","context":{"idset":"11256","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.3081331,"name":"drain","context":{"idset":"11257","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.3207738,"name":"drain","context":{"idset":"11258","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.3329015,"name":"drain","context":{"idset":"11259","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.3448751,"name":"drain","context":{"idset":"11260","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.3590636,"name":"drain","context":{"idset":"11261","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.371505,"name":"drain","context":{"idset":"11262","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.3835194,"name":"drain","context":{"idset":"11263","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.3955834,"name":"drain","context":{"idset":"11264","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.4074891,"name":"drain","context":{"idset":"11265","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.4195375,"name":"drain","context":{"idset":"11266","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.4322214,"name":"drain","context":{"idset":"11267","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.445189,"name":"drain","context":{"idset":"11268","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.4579177,"name":"drain","context":{"idset":"11269","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.4686639,"name":"drain","context":{"idset":"11270","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.4804661,"name":"drain","context":{"idset":"11271","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.4925919,"name":"drain","context":{"idset":"11272","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5040338,"name":"drain","context":{"idset":"11273","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5159237,"name":"drain","context":{"idset":"11274","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5278161,"name":"drain","context":{"idset":"11275","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5401609,"name":"drain","context":{"idset":"11276","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5414681,"name":"drain","context":{"idset":"11277","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5427852,"name":"drain","context":{"idset":"11278","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5441234,"name":"drain","context":{"idset":"11279","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5454905,"name":"drain","context":{"idset":"11280","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5468197,"name":"drain","context":{"idset":"11281","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5481424,"name":"drain","context":{"idset":"11282","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5494967,"name":"drain","context":{"idset":"11283","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5507929,"name":"drain","context":{"idset":"11284","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5520561,"name":"drain","context":{"idset":"11285","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5533588,"name":"drain","context":{"idset":"11286","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5546708,"name":"drain","context":{"idset":"11287","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5559683,"name":"drain","context":{"idset":"11288","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5572858,"name":"drain","context":{"idset":"11289","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.558629,"name":"drain","context":{"idset":"11290","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5599456,"name":"drain","context":{"idset":"11291","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5612209,"name":"drain","context":{"idset":"11292","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5626462,"name":"drain","context":{"idset":"11293","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5640609,"name":"drain","context":{"idset":"11294","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5653989,"name":"drain","context":{"idset":"11295","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5667515,"name":"drain","context":{"idset":"11296","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.568141,"name":"drain","context":{"idset":"11297","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5694425,"name":"drain","context":{"idset":"11298","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5706584,"name":"drain","context":{"idset":"11299","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5718851,"name":"drain","context":{"idset":"11300","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5731463,"name":"drain","context":{"idset":"11301","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5750635,"name":"drain","context":{"idset":"11302","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5764043,"name":"drain","context":{"idset":"11303","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5777166,"name":"drain","context":{"idset":"11304","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5790479,"name":"drain","context":{"idset":"11305","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5803881,"name":"drain","context":{"idset":"11306","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5816953,"name":"drain","context":{"idset":"11307","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5830162,"name":"drain","context":{"idset":"11308","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5843933,"name":"drain","context":{"idset":"11309","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5857403,"name":"drain","context":{"idset":"11310","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5870695,"name":"drain","context":{"idset":"11311","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5883994,"name":"drain","context":{"idset":"11312","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5897245,"name":"drain","context":{"idset":"11313","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5910358,"name":"drain","context":{"idset":"11314","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5923481,"name":"drain","context":{"idset":"11315","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5936649,"name":"drain","context":{"idset":"11316","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.594928,"name":"drain","context":{"idset":"11317","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5962205,"name":"drain","context":{"idset":"11318","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5975618,"name":"drain","context":{"idset":"11319","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.5988655,"name":"drain","context":{"idset":"11320","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6001511,"name":"drain","context":{"idset":"11321","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.601505,"name":"drain","context":{"idset":"11322","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6028385,"name":"drain","context":{"idset":"11323","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.604152,"name":"drain","context":{"idset":"11324","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6054361,"name":"drain","context":{"idset":"11325","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6067021,"name":"drain","context":{"idset":"11326","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6081076,"name":"drain","context":{"idset":"11327","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6094429,"name":"drain","context":{"idset":"11328","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6107898,"name":"drain","context":{"idset":"11329","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6121454,"name":"drain","context":{"idset":"11330","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6134942,"name":"drain","context":{"idset":"11331","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6148303,"name":"drain","context":{"idset":"11332","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6161613,"name":"drain","context":{"idset":"11333","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6174672,"name":"drain","context":{"idset":"11334","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6187885,"name":"drain","context":{"idset":"11335","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6201086,"name":"drain","context":{"idset":"11336","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6213923,"name":"drain","context":{"idset":"11337","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6227052,"name":"drain","context":{"idset":"11338","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6239805,"name":"drain","context":{"idset":"11339","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6253192,"name":"drain","context":{"idset":"11340","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.626662,"name":"drain","context":{"idset":"11341","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6281023,"name":"drain","context":{"idset":"11342","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6295288,"name":"drain","context":{"idset":"11343","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6308494,"name":"drain","context":{"idset":"11344","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6320953,"name":"drain","context":{"idset":"11345","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6334054,"name":"drain","context":{"idset":"11346","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.634732,"name":"drain","context":{"idset":"11347","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6360264,"name":"drain","context":{"idset":"11348","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6373487,"name":"drain","context":{"idset":"11349","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6386683,"name":"drain","context":{"idset":"11350","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6399307,"name":"drain","context":{"idset":"11351","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6412067,"name":"drain","context":{"idset":"11352","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6425331,"name":"drain","context":{"idset":"11353","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6438856,"name":"drain","context":{"idset":"11354","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6451504,"name":"drain","context":{"idset":"11355","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6464899,"name":"drain","context":{"idset":"11356","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6478217,"name":"drain","context":{"idset":"11357","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6491385,"name":"drain","context":{"idset":"11358","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.650378,"name":"drain","context":{"idset":"11359","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6516237,"name":"drain","context":{"idset":"11360","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6528466,"name":"drain","context":{"idset":"11361","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6540837,"name":"drain","context":{"idset":"11362","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6553769,"name":"drain","context":{"idset":"11363","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6566978,"name":"drain","context":{"idset":"11364","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6580439,"name":"drain","context":{"idset":"11365","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6593351,"name":"drain","context":{"idset":"11366","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6607039,"name":"drain","context":{"idset":"11367","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.662092,"name":"drain","context":{"idset":"11368","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6633768,"name":"drain","context":{"idset":"11369","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6646619,"name":"drain","context":{"idset":"11370","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6659539,"name":"drain","context":{"idset":"11371","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6672943,"name":"drain","context":{"idset":"11372","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6686101,"name":"drain","context":{"idset":"11373","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6699393,"name":"drain","context":{"idset":"11374","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6712401,"name":"drain","context":{"idset":"11375","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6725764,"name":"drain","context":{"idset":"11376","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6738651,"name":"drain","context":{"idset":"11377","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6751921,"name":"drain","context":{"idset":"11378","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6766365,"name":"drain","context":{"idset":"11379","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6780169,"name":"drain","context":{"idset":"11380","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6793664,"name":"drain","context":{"idset":"11381","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6806695,"name":"drain","context":{"idset":"11382","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.681973,"name":"drain","context":{"idset":"11383","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6832814,"name":"drain","context":{"idset":"11384","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6845837,"name":"drain","context":{"idset":"11385","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6858795,"name":"drain","context":{"idset":"11386","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6871302,"name":"drain","context":{"idset":"11387","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6884363,"name":"drain","context":{"idset":"11388","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6897378,"name":"drain","context":{"idset":"11389","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6910582,"name":"drain","context":{"idset":"11390","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6923218,"name":"drain","context":{"idset":"11391","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6936855,"name":"drain","context":{"idset":"11392","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6949492,"name":"drain","context":{"idset":"11393","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6962435,"name":"drain","context":{"idset":"11394","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6975677,"name":"drain","context":{"idset":"11395","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.6989033,"name":"drain","context":{"idset":"11396","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.700284,"name":"drain","context":{"idset":"11397","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.701628,"name":"drain","context":{"idset":"11398","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7029624,"name":"drain","context":{"idset":"11399","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7043397,"name":"drain","context":{"idset":"11400","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7056992,"name":"drain","context":{"idset":"11401","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7070374,"name":"drain","context":{"idset":"11402","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7084329,"name":"drain","context":{"idset":"11403","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7098248,"name":"drain","context":{"idset":"11404","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7111747,"name":"drain","context":{"idset":"11405","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7125633,"name":"drain","context":{"idset":"11406","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7139103,"name":"drain","context":{"idset":"11407","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7152829,"name":"drain","context":{"idset":"11408","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7166228,"name":"drain","context":{"idset":"11409","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7180011,"name":"drain","context":{"idset":"11410","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7207725,"name":"drain","context":{"idset":"11411","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7221878,"name":"drain","context":{"idset":"11412","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7235658,"name":"drain","context":{"idset":"11413","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7248986,"name":"drain","context":{"idset":"11414","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7262223,"name":"drain","context":{"idset":"11415","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7276073,"name":"drain","context":{"idset":"11416","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.729023,"name":"drain","context":{"idset":"11417","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7303834,"name":"drain","context":{"idset":"11418","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7317603,"name":"drain","context":{"idset":"11419","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7331264,"name":"drain","context":{"idset":"11420","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7344966,"name":"drain","context":{"idset":"11421","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7358539,"name":"drain","context":{"idset":"11422","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7371521,"name":"drain","context":{"idset":"11423","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7385416,"name":"drain","context":{"idset":"11424","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7400181,"name":"drain","context":{"idset":"11425","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7415383,"name":"drain","context":{"idset":"11426","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7429512,"name":"drain","context":{"idset":"11427","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7443378,"name":"drain","context":{"idset":"11428","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7456923,"name":"drain","context":{"idset":"11429","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7470658,"name":"drain","context":{"idset":"11430","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7485416,"name":"drain","context":{"idset":"11431","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7498472,"name":"drain","context":{"idset":"11432","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7512028,"name":"drain","context":{"idset":"11433","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7525551,"name":"drain","context":{"idset":"11434","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.753907,"name":"drain","context":{"idset":"11435","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7554257,"name":"drain","context":{"idset":"11436","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7567813,"name":"drain","context":{"idset":"11437","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.758116,"name":"drain","context":{"idset":"11438","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7595122,"name":"drain","context":{"idset":"11439","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7608464,"name":"drain","context":{"idset":"11440","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.762337,"name":"drain","context":{"idset":"11441","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.763756,"name":"drain","context":{"idset":"11442","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7650511,"name":"drain","context":{"idset":"11443","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7664132,"name":"drain","context":{"idset":"11444","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7678428,"name":"drain","context":{"idset":"11445","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.769443,"name":"drain","context":{"idset":"11446","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7709389,"name":"drain","context":{"idset":"11447","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7723968,"name":"drain","context":{"idset":"11448","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.773762,"name":"drain","context":{"idset":"11449","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7752378,"name":"drain","context":{"idset":"11450","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7767851,"name":"drain","context":{"idset":"11451","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7782564,"name":"drain","context":{"idset":"11452","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7797477,"name":"drain","context":{"idset":"11453","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7812355,"name":"drain","context":{"idset":"11454","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7827356,"name":"drain","context":{"idset":"11455","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7841887,"name":"drain","context":{"idset":"11456","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7855678,"name":"drain","context":{"idset":"11457","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.786871,"name":"drain","context":{"idset":"11458","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7881854,"name":"drain","context":{"idset":"11459","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7895911,"name":"drain","context":{"idset":"11460","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7910144,"name":"drain","context":{"idset":"11461","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.792412,"name":"drain","context":{"idset":"11462","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7937896,"name":"drain","context":{"idset":"11463","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.795099,"name":"drain","context":{"idset":"11464","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7964325,"name":"drain","context":{"idset":"11465","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7977982,"name":"drain","context":{"idset":"11466","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.7993968,"name":"drain","context":{"idset":"11467","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.800739,"name":"drain","context":{"idset":"11468","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8021364,"name":"drain","context":{"idset":"11471","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8036232,"name":"drain","context":{"idset":"11472","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.805052,"name":"drain","context":{"idset":"11473","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.806437,"name":"drain","context":{"idset":"11474","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8078218,"name":"drain","context":{"idset":"11475","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8091841,"name":"drain","context":{"idset":"11476","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8105628,"name":"drain","context":{"idset":"11477","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8119287,"name":"drain","context":{"idset":"11478","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8132889,"name":"drain","context":{"idset":"11479","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8146536,"name":"drain","context":{"idset":"11480","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8160086,"name":"drain","context":{"idset":"11481","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8174336,"name":"drain","context":{"idset":"11482","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8188105,"name":"drain","context":{"idset":"11483","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8201785,"name":"drain","context":{"idset":"11484","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8215499,"name":"drain","context":{"idset":"11485","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8228815,"name":"drain","context":{"idset":"11486","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8273387,"name":"drain","context":{"idset":"11487","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8286304,"name":"drain","context":{"idset":"11488","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8299012,"name":"drain","context":{"idset":"11489","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8311739,"name":"drain","context":{"idset":"11490","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8324754,"name":"drain","context":{"idset":"11491","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8338525,"name":"drain","context":{"idset":"11492","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8352151,"name":"drain","context":{"idset":"11493","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8366396,"name":"drain","context":{"idset":"11494","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8380628,"name":"drain","context":{"idset":"11495","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.839468,"name":"drain","context":{"idset":"11496","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8407881,"name":"drain","context":{"idset":"11497","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8421929,"name":"drain","context":{"idset":"11498","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8436553,"name":"drain","context":{"idset":"11499","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8450561,"name":"drain","context":{"idset":"11500","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.846442,"name":"drain","context":{"idset":"11501","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8478127,"name":"drain","context":{"idset":"11502","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8492117,"name":"drain","context":{"idset":"11503","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8506114,"name":"drain","context":{"idset":"11504","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8519902,"name":"drain","context":{"idset":"11505","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8532896,"name":"drain","context":{"idset":"11506","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8545721,"name":"drain","context":{"idset":"11507","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.8558977,"name":"drain","context":{"idset":"11508","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.944066,"name":"drain","context":{"idset":"11573","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9455504,"name":"drain","context":{"idset":"11574","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9468989,"name":"drain","context":{"idset":"11575","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9482899,"name":"drain","context":{"idset":"11576","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9497747,"name":"drain","context":{"idset":"11577","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9513392,"name":"drain","context":{"idset":"11578","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9528551,"name":"drain","context":{"idset":"11579","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9541945,"name":"drain","context":{"idset":"11580","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9555399,"name":"drain","context":{"idset":"11581","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9568357,"name":"drain","context":{"idset":"11582","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9581695,"name":"drain","context":{"idset":"11583","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9594965,"name":"drain","context":{"idset":"11584","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9608436,"name":"drain","context":{"idset":"11585","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.962157,"name":"drain","context":{"idset":"11586","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.963342,"name":"drain","context":{"idset":"11587","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9647386,"name":"drain","context":{"idset":"11588","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9661462,"name":"drain","context":{"idset":"11589","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.967602,"name":"drain","context":{"idset":"11590","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9693635,"name":"drain","context":{"idset":"11591","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9708047,"name":"drain","context":{"idset":"11592","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9722533,"name":"drain","context":{"idset":"11593","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9737198,"name":"drain","context":{"idset":"11594","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9751658,"name":"drain","context":{"idset":"11595","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9766045,"name":"drain","context":{"idset":"11596","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9780402,"name":"drain","context":{"idset":"11597","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9794235,"name":"drain","context":{"idset":"11598","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9807806,"name":"drain","context":{"idset":"11599","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9820542,"name":"drain","context":{"idset":"11600","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9833801,"name":"drain","context":{"idset":"11601","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9843409,"name":"drain","context":{"idset":"11602","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9855685,"name":"drain","context":{"idset":"11603","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.986501,"name":"drain","context":{"idset":"11604","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9876742,"name":"drain","context":{"idset":"11605","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9888196,"name":"drain","context":{"idset":"11606","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9897113,"name":"drain","context":{"idset":"11607","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9909651,"name":"drain","context":{"idset":"11608","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9918408,"name":"drain","context":{"idset":"11609","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9927375,"name":"drain","context":{"idset":"11610","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9941032,"name":"drain","context":{"idset":"11611","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9954479,"name":"drain","context":{"idset":"11612","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9967792,"name":"drain","context":{"idset":"11613","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9977667,"name":"drain","context":{"idset":"11614","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9986656,"name":"drain","context":{"idset":"11615","reason":"broker was unresponsive"}}
+{"timestamp":1712845934.9996357,"name":"drain","context":{"idset":"11616","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0008788,"name":"drain","context":{"idset":"11617","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0022526,"name":"drain","context":{"idset":"11618","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0035186,"name":"drain","context":{"idset":"11619","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0049436,"name":"drain","context":{"idset":"11620","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0063157,"name":"drain","context":{"idset":"11621","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0078678,"name":"drain","context":{"idset":"11622","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0094495,"name":"drain","context":{"idset":"11623","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0110085,"name":"drain","context":{"idset":"11624","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0125589,"name":"drain","context":{"idset":"11625","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0138535,"name":"drain","context":{"idset":"11626","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0148957,"name":"drain","context":{"idset":"11627","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0162091,"name":"drain","context":{"idset":"11628","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0176668,"name":"drain","context":{"idset":"11629","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0187457,"name":"drain","context":{"idset":"11630","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0196528,"name":"drain","context":{"idset":"11631","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0210707,"name":"drain","context":{"idset":"11632","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.022536,"name":"drain","context":{"idset":"11633","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0240269,"name":"drain","context":{"idset":"11634","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0255613,"name":"drain","context":{"idset":"11635","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0270183,"name":"drain","context":{"idset":"11636","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0284832,"name":"drain","context":{"idset":"11653","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0299592,"name":"drain","context":{"idset":"11654","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.03141,"name":"drain","context":{"idset":"11655","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0328496,"name":"drain","context":{"idset":"11656","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0343063,"name":"drain","context":{"idset":"11657","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.035687,"name":"drain","context":{"idset":"11658","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0371063,"name":"drain","context":{"idset":"11659","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0385041,"name":"drain","context":{"idset":"11660","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0400741,"name":"drain","context":{"idset":"11661","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0416734,"name":"drain","context":{"idset":"11662","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0432265,"name":"drain","context":{"idset":"11663","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0448005,"name":"drain","context":{"idset":"11664","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0463345,"name":"drain","context":{"idset":"11665","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0476141,"name":"drain","context":{"idset":"11666","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0484731,"name":"drain","context":{"idset":"11667","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0498488,"name":"drain","context":{"idset":"11668","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0512204,"name":"drain","context":{"idset":"11669","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0524862,"name":"drain","context":{"idset":"11670","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0539222,"name":"drain","context":{"idset":"11671","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0554037,"name":"drain","context":{"idset":"11672","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.056895,"name":"drain","context":{"idset":"11673","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0584166,"name":"drain","context":{"idset":"11674","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0598724,"name":"drain","context":{"idset":"11675","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0613618,"name":"drain","context":{"idset":"11676","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0628119,"name":"drain","context":{"idset":"11677","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0642767,"name":"drain","context":{"idset":"11678","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.065742,"name":"drain","context":{"idset":"11679","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0671651,"name":"drain","context":{"idset":"11680","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0685692,"name":"drain","context":{"idset":"11681","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0699413,"name":"drain","context":{"idset":"11682","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0715079,"name":"drain","context":{"idset":"11683","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0730929,"name":"drain","context":{"idset":"11684","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.074687,"name":"drain","context":{"idset":"11685","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0762837,"name":"drain","context":{"idset":"11686","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0778596,"name":"drain","context":{"idset":"11687","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0791979,"name":"drain","context":{"idset":"11688","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0800824,"name":"drain","context":{"idset":"11689","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0809252,"name":"drain","context":{"idset":"11690","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0818002,"name":"drain","context":{"idset":"11691","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0826421,"name":"drain","context":{"idset":"11692","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0835812,"name":"drain","context":{"idset":"11695","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0845459,"name":"drain","context":{"idset":"11696","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0855532,"name":"drain","context":{"idset":"11697","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0865631,"name":"drain","context":{"idset":"11698","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0875354,"name":"drain","context":{"idset":"11699","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0884264,"name":"drain","context":{"idset":"11700","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0896196,"name":"drain","context":{"idset":"11701","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0905697,"name":"drain","context":{"idset":"11702","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0916677,"name":"drain","context":{"idset":"11703","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0926466,"name":"drain","context":{"idset":"11704","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0935018,"name":"drain","context":{"idset":"11705","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0943704,"name":"drain","context":{"idset":"11706","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0956867,"name":"drain","context":{"idset":"11707","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.097156,"name":"drain","context":{"idset":"11708","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0986114,"name":"drain","context":{"idset":"11709","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.0999401,"name":"drain","context":{"idset":"11710","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1010225,"name":"drain","context":{"idset":"11711","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1021373,"name":"drain","context":{"idset":"11712","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1033278,"name":"drain","context":{"idset":"11713","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1042287,"name":"drain","context":{"idset":"11714","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1050985,"name":"drain","context":{"idset":"11715","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1059833,"name":"drain","context":{"idset":"11716","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1069131,"name":"drain","context":{"idset":"11717","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1079934,"name":"drain","context":{"idset":"11718","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1091483,"name":"drain","context":{"idset":"11719","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1100309,"name":"drain","context":{"idset":"11720","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1111526,"name":"drain","context":{"idset":"11721","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1121743,"name":"drain","context":{"idset":"11722","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1131465,"name":"drain","context":{"idset":"11723","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1144705,"name":"drain","context":{"idset":"11724","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1154385,"name":"drain","context":{"idset":"11725","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1168296,"name":"drain","context":{"idset":"11726","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1177156,"name":"drain","context":{"idset":"11727","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1190033,"name":"drain","context":{"idset":"11728","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1203618,"name":"drain","context":{"idset":"11729","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1217306,"name":"drain","context":{"idset":"11730","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1230838,"name":"drain","context":{"idset":"11731","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1241779,"name":"drain","context":{"idset":"11732","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.125232,"name":"drain","context":{"idset":"11733","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1261368,"name":"drain","context":{"idset":"11734","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1270683,"name":"drain","context":{"idset":"11735","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1279423,"name":"drain","context":{"idset":"11736","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.128829,"name":"drain","context":{"idset":"11737","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1296806,"name":"drain","context":{"idset":"11738","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1305494,"name":"drain","context":{"idset":"11739","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1314075,"name":"drain","context":{"idset":"11740","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1322989,"name":"drain","context":{"idset":"11741","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1331427,"name":"drain","context":{"idset":"11742","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1340201,"name":"drain","context":{"idset":"11743","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1349201,"name":"drain","context":{"idset":"11744","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1358395,"name":"drain","context":{"idset":"11745","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1369338,"name":"drain","context":{"idset":"11746","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1383152,"name":"drain","context":{"idset":"11747","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1396987,"name":"drain","context":{"idset":"11748","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1411943,"name":"drain","context":{"idset":"11749","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1428628,"name":"drain","context":{"idset":"11750","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1444559,"name":"drain","context":{"idset":"11751","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1460185,"name":"drain","context":{"idset":"11752","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1491973,"name":"drain","context":{"idset":"11753","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1508794,"name":"drain","context":{"idset":"11754","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1524863,"name":"drain","context":{"idset":"11755","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1541328,"name":"drain","context":{"idset":"11756","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1556859,"name":"drain","context":{"idset":"11757","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1572502,"name":"drain","context":{"idset":"11758","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1587975,"name":"drain","context":{"idset":"11759","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1602902,"name":"drain","context":{"idset":"11760","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1617785,"name":"drain","context":{"idset":"11761","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1631413,"name":"drain","context":{"idset":"11762","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1644373,"name":"drain","context":{"idset":"11763","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.165673,"name":"drain","context":{"idset":"11764","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.166908,"name":"drain","context":{"idset":"11768","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.168149,"name":"drain","context":{"idset":"11772","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.169451,"name":"drain","context":{"idset":"11778","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1706889,"name":"drain","context":{"idset":"11792","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1719248,"name":"drain","context":{"idset":"11793","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1731594,"name":"drain","context":{"idset":"11797","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1744056,"name":"drain","context":{"idset":"11798","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1756682,"name":"drain","context":{"idset":"11802","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1769021,"name":"drain","context":{"idset":"11810","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1781359,"name":"drain","context":{"idset":"11817","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1793897,"name":"drain","context":{"idset":"11819","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1808174,"name":"drain","context":{"idset":"11820","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1817954,"name":"drain","context":{"idset":"11830","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1826589,"name":"drain","context":{"idset":"11831","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1835563,"name":"drain","context":{"idset":"11832","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.18484,"name":"drain","context":{"idset":"11834","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.186244,"name":"drain","context":{"idset":"11836","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1871283,"name":"drain","context":{"idset":"11837","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1879749,"name":"drain","context":{"idset":"11842","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1888616,"name":"drain","context":{"idset":"11845","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1897027,"name":"drain","context":{"idset":"11849","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1907337,"name":"drain","context":{"idset":"11854","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1921487,"name":"drain","context":{"idset":"11855","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1935658,"name":"drain","context":{"idset":"11858","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1950772,"name":"drain","context":{"idset":"11859","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1965785,"name":"drain","context":{"idset":"11861","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1979442,"name":"drain","context":{"idset":"11864","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.1993964,"name":"drain","context":{"idset":"11871","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.2009041,"name":"drain","context":{"idset":"11874","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.2024789,"name":"drain","context":{"idset":"11878","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.2039888,"name":"drain","context":{"idset":"11879","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.205472,"name":"drain","context":{"idset":"11885","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.207092,"name":"drain","context":{"idset":"11887","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.2086861,"name":"drain","context":{"idset":"11888","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.2101955,"name":"drain","context":{"idset":"11890","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.2117004,"name":"drain","context":{"idset":"11891","reason":"broker was unresponsive"}}
+{"timestamp":1712845935.2138448,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1712846421.3112073,"name":"online","context":{"idset":"10911"}}
+{"timestamp":1712846692.8397684,"name":"online","context":{"idset":"9876"}}
+{"timestamp":1712846692.8430586,"name":"online","context":{"idset":"9875"}}
+{"timestamp":1712846796.4688616,"name":"online","context":{"idset":"11249-11250"}}
+{"timestamp":1712846833.6484904,"name":"undrain","context":{"idset":"1-60,85-120,122-216,218-252,277-347,349-444,461,469-492,494-500,503-505,507,509-540,565,567-569,571-579,582-588,795-796,805-806,815-816,820,827-828,841-842,871-872,949-954,963-964,973-978,9205-9294,9297-9446,9449-9534,9536-9560,9563-9615,9617-9780,9782-9786,9791-9874,9877-9991,9993-10084,10086-10098,10101-10250,10252-10260,10263-10302,10304-10342,10345-10354,10357-10395,10397-10400,10402-10484,10613,10615,10641,10645-10653,10997-10998,11001-11004,11007-11014,11017-11020,11023-11060,11065-11068,11071-11092,11097-11134,11137-11248,11251-11468,11471-11508,11573-11578,11665-11692,11695-11706,11727-11892"}}
+{"timestamp":1712846876.8293252,"name":"online","context":{"idset":"11469"}}
+{"timestamp":1712846936.3089187,"name":"undrain","context":{"idset":"501-502,508,570,580,875-876,9296,9616,10085,10396,10572,10616-10628,10630-10633,10635-10636,10640,10654-10656,10659-10701,10703-10711,10713-10717,10719-10738,10741,10743-10794,10796-10868,11005-11006,11015-11016,11022,11061-11062,11069-11070,11093-11094,11135-11136,11469-11470,11579-11636,11653-11664,11693,11707-11726"}}
+{"timestamp":1712847885.1546335,"name":"online","context":{"idset":"11069,11093"}}
+{"timestamp":1712847885.3731153,"name":"online","context":{"idset":"11094"}}
+{"timestamp":1712847886.0368247,"name":"online","context":{"idset":"11070"}}
+{"timestamp":1712848492.1756489,"name":"online","context":{"idset":"838"}}
+{"timestamp":1712848502.3360834,"name":"online","context":{"idset":"837"}}
+{"timestamp":1712849060.1516485,"name":"online","context":{"idset":"9295"}}
+{"timestamp":1712849060.7178788,"name":"online","context":{"idset":"9296"}}
+{"timestamp":1712849612.3305345,"name":"drain","context":{"idset":"509-516","reason":"New Blade Installation","overwrite":0}}
+{"timestamp":1712849729.4545047,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1712849729.4982963,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1712849729.5180082,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1712849729.6106806,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1712849729.680619,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1712849729.6942401,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1712849729.715004,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1712849729.7417898,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1712849729.75582,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1712849729.7797494,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1712849729.8145831,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1712849729.832629,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1712849729.8498971,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1712849729.8683572,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1712849729.9017701,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1712849731.4270198,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1712849731.4890234,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1712849731.5222902,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1712849731.5446243,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1712849731.5611534,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1712849731.577924,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1712849731.6458545,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1712849731.6513081,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1712849731.6637144,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1712849731.6986828,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1712849731.7040932,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1712849731.7088151,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1712849731.7222466,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1712849731.7573709,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1712849731.8120866,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1712849731.8295884,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1712849731.8463738,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1712849731.8625119,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1712849731.8797317,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1712849731.9148176,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1712849731.9363461,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1712849731.939184,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1712849731.9431815,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1712849731.958993,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1712849731.9618351,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1712849731.985553,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1712849732.0027208,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1712849732.0199234,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1712849732.0373166,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1712849732.1353095,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1712849732.3586099,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1712849733.2873447,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1712849733.3304813,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1712849733.3502998,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1712849733.4025016,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1712849733.4423754,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1712849733.4592366,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1712849733.4649887,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1712849733.4699268,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1712849733.4902377,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1712849733.5113358,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1712849733.5154848,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1712849733.6420715,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1712849733.6452432,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1712849733.6476421,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1712849733.6499829,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1712849733.6675622,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1712849733.6777399,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1712849733.7110097,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1712849733.744796,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1712849733.7613761,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1712849733.7768993,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1712849733.7929466,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1712849733.8368685,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1712849733.8546965,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1712849733.8728905,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1712849733.8903482,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1712849733.9063232,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1712849733.9295597,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1712849733.9542079,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1712849733.9709547,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1712849734.2040691,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1712849734.4455547,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1712849735.1616864,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1712849735.1884573,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1712849735.2079506,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1712849735.3094308,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1712849735.3568211,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1712849735.4472396,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1712849735.478699,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1712849735.5089374,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1712849735.5281844,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1712849735.5553048,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1712849735.558152,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1712849735.5779517,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1712849735.5943651,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1712849735.6130779,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1712849735.6159632,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1712849735.6470754,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1712849735.6933928,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1712849735.7120454,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1712849735.747143,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1712849735.7627723,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1712849735.78005,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1712849735.8146238,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1712849735.8174868,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1712849735.8273399,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1712849735.8430536,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1712849735.8588154,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1712849735.8854694,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1712849735.9048114,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1712849735.9261248,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1712849735.9455118,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1712849736.197525,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1712849736.2791069,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1712849737.0067747,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1712849737.0312347,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1712849737.1280453,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1712849737.2346914,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1712849737.248142,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1712849737.2589626,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1712849737.271106,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1712849737.2826905,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1712849737.2959812,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1712849737.3322396,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1712849737.3429458,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1712849737.3512988,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1712849737.3717127,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1712849737.3745525,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1712849737.3892205,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1712849737.3920693,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1712849737.3999836,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1712849737.4106004,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1712849737.4264534,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1712849737.4393413,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1712849737.4615762,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1712849737.4729028,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1712849737.4835207,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1712849737.5310926,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1712849737.5338981,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1712849737.5367069,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1712849737.539525,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1712849737.7325656,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1712849737.7353928,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1712849737.7493563,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1712849738.1340735,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1712849738.8898246,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1712849738.9152255,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1712849738.931813,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1712849739.0302999,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1712849739.131706,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1712849739.1531417,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1712849739.1743114,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1712849739.1956699,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1712849739.2140772,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1712849739.2326918,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1712849739.2825868,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1712849739.2999551,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1712849739.3173449,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1712849739.334554,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1712849739.3601651,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1712849739.3659778,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1712849739.3886023,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1712849739.4054677,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1712849739.4215059,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1712849739.4435456,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1712849739.4610369,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1712849739.477392,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1712849739.4936984,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1712849739.5096178,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1712849739.5260293,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1712849740.970067,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1712849740.9896557,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1712849741.0077083,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1712849741.0258427,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1712849741.0537314,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1712849741.1434948,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1712849741.1689031,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1712849741.2679098,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1712849741.811126,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1712849741.8300245,"name":"offline","context":{"idset":"286"}}
+{"timestamp":1712849741.8485422,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1712849741.9500279,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1712849741.9862554,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1712849742.0114317,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1712849742.0270877,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1712849742.1119781,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1712849742.7982903,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1712849742.814846,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1712849742.8339639,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1712849742.8478489,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1712849742.9500649,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1712849742.9664752,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1712849742.9783697,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1712849743.0660481,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1712849743.6365397,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1712849743.6555471,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1712849743.7562482,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1712849743.7900286,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1712849743.8102174,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1712849743.8252308,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1712849743.8426604,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1712849743.8678353,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1712849743.984561,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1712849743.9872863,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1712849744.0865402,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1712849744.6125278,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1712849744.657876,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1712849744.6738656,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1712849744.703088,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1712849744.7936854,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1712849744.8160617,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1712849744.8311043,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1712849744.9151232,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1712849745.4696994,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1712849745.4908962,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1712849745.5065105,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1712849745.5925779,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1712849745.8329136,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1712849745.8357978,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1712849745.8384902,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1712849745.8505223,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1712849745.8532071,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1712849745.8559966,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1712849745.8587091,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1712849745.8768504,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1712849746.7371228,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1712849746.7398093,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1712849746.7424722,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1712849746.7451468,"name":"offline","context":{"idset":"336"}}
+{"timestamp":1712849746.7478371,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1712849746.750597,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1712849746.753324,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1712849746.7587702,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1712849747.1279511,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1712849747.3024457,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1712849747.3401477,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1712849747.3676906,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1712849747.413197,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1712849747.4158857,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1712849747.4401751,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1712849747.4833252,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1712849747.523365,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1712849747.5319948,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1712849747.647084,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1712849747.6587255,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1712849747.756505,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1712849748.2333326,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1712849748.3330398,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1712849748.3577487,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1712849748.3734677,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1712849748.3919661,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1712849748.4916153,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1712849748.512599,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1712849748.6141675,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1712849748.9022241,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1712849749.1349194,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1712849749.1682339,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1712849749.1831768,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1712849749.268508,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1712849749.3030708,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1712849749.3265042,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1712849749.3492839,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1712849749.3646488,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1712849749.4004452,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1712849749.4034402,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1712849749.4297307,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1712849749.4469652,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1712849749.4910457,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1712849749.5399137,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1712849749.5530677,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1712849749.5826135,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1712849749.6645641,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1712849750.1309652,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1712849750.1660926,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1712849750.2021356,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1712849750.2271512,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1712849750.276432,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1712849750.3040349,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1712849750.3626919,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1712849750.5455399,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1712849750.6488907,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1712849750.8263092,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1712849750.9963374,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1712849751.0443366,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1712849751.0943201,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1712849751.120188,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1712849751.1439946,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1712849751.2051656,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1712849751.2236438,"name":"offline","context":{"idset":"397"}}
+{"timestamp":1712849751.2446527,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1712849751.250293,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1712849751.2890849,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1712849751.2944613,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1712849751.3073475,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1712849751.3980558,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1712849751.424619,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1712849751.447063,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1712849751.4734192,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1712849751.4983497,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1712849751.5566275,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1712849751.597811,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1712849751.6018176,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1712849751.6058915,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1712849751.6096272,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1712849752.0166583,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1712849752.0429666,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1712849752.0594356,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1712849752.0950212,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1712849752.1212544,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1712849752.1953952,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1712849752.3768263,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1712849752.4791329,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1712849752.9143949,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1712849753.0144291,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1712849753.2329748,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1712849753.3339102,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1712849753.3745108,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1712849753.4751315,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1712849753.6167765,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1712849753.6340406,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1712849753.66837,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1712849753.7024035,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1712849753.7909703,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1712849753.9244056,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1712849753.9455123,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1712849753.9642789,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1712849753.9853463,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1712849753.9883687,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1712849754.0559602,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1712849754.0751724,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1712849754.0820878,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1712849754.0991209,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1712849754.1021118,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1712849754.1051199,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1712849754.1530461,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1712849754.3207934,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1712849754.399893,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1712849756.0474296,"name":"offline","context":{"idset":"461"}}
+{"timestamp":1712849756.0676954,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1712849756.1232603,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1712849756.1382847,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1712849756.1984625,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1712849756.219816,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1712849756.2729564,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1712849756.3377829,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1712849756.4261594,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1712849756.4659498,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1712849756.4930768,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1712849756.5753403,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1712849756.6977584,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1712849756.8001873,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1712849757.0164459,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1712849757.0681367,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1712849757.0853741,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1712849757.170186,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1712849757.3022366,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1712849757.6299632,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1712849757.6773005,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1712849757.7669692,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1712849757.891166,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1712849757.9901378,"name":"offline","context":{"idset":"491"}}
+{"timestamp":1712849758.0678644,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1712849758.0811844,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1712849758.1667876,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1712849758.2892394,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1712849758.3043506,"name":"offline","context":{"idset":"498"}}
+{"timestamp":1712849758.3256207,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1712849758.3472378,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1712849758.3881884,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1712849758.4041066,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1712849758.4188211,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1712849758.4349866,"name":"offline","context":{"idset":"509"}}
+{"timestamp":1712849758.4609134,"name":"offline","context":{"idset":"510"}}
+{"timestamp":1712849758.4707322,"name":"offline","context":{"idset":"512"}}
+{"timestamp":1712849758.4937103,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1712849758.6298933,"name":"offline","context":{"idset":"511"}}
+{"timestamp":1712849758.737839,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1712849758.861227,"name":"offline","context":{"idset":"514"}}
+{"timestamp":1712849758.9885805,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1712849759.0209403,"name":"offline","context":{"idset":"516"}}
+{"timestamp":1712849759.0768631,"name":"offline","context":{"idset":"513"}}
+{"timestamp":1712849759.1684721,"name":"offline","context":{"idset":"515"}}
+{"timestamp":1712849759.3401897,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1712849759.3853991,"name":"offline","context":{"idset":"518"}}
+{"timestamp":1712849759.4107766,"name":"offline","context":{"idset":"519"}}
+{"timestamp":1712849759.5099087,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1712849759.8122394,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1712849759.9508085,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1712849760.1271977,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1712849760.2331135,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1712849760.2712464,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1712849760.2931848,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1712849760.3107307,"name":"offline","context":{"idset":"537"}}
+{"timestamp":1712849760.3276315,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1712849760.3657341,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1712849760.3691719,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1712849760.3972919,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1712849760.426748,"name":"offline","context":{"idset":"530"}}
+{"timestamp":1712849760.4338403,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1712849760.5774443,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1712849760.6078506,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1712849760.6259255,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1712849760.7249455,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1712849760.9359043,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1712849761.0487194,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1712849762.3315651,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1712849762.4465117,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1712849762.5457022,"name":"offline","context":{"idset":"568"}}
+{"timestamp":1712849762.6599293,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1712849762.7643652,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1712849762.8077641,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1712849762.9070253,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1712849762.9368472,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1712849763.0373409,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1712849763.0532842,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1712849763.0643299,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1712849763.0770788,"name":"offline","context":{"idset":"583"}}
+{"timestamp":1712849763.1768963,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1712849763.2919655,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1712849763.3908265,"name":"offline","context":{"idset":"586"}}
+{"timestamp":1712849763.4906971,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1712849763.620976,"name":"offline","context":{"idset":"588"}}
+{"timestamp":1712849789.3946834,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1712849789.4090278,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1712849789.5002348,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1712849880.8798809,"name":"undrain","context":{"idset":"838"}}
+{"timestamp":1712849903.437361,"name":"undrain","context":{"idset":"837"}}
+{"timestamp":1712850673.0354795,"name":"drain","context":{"idset":"837-838","reason":"--reason Runnning diags - estella2","overwrite":0}}
+{"timestamp":1712850871.7051232,"name":"online","context":{"idset":"86,89,91"}}
+{"timestamp":1712850871.9102931,"name":"online","context":{"idset":"87-88,90"}}
+{"timestamp":1712850872.1188862,"name":"online","context":{"idset":"92"}}
+{"timestamp":1712850872.1632495,"name":"online","context":{"idset":"85"}}
+{"timestamp":1712850874.1123004,"name":"online","context":{"idset":"94"}}
+{"timestamp":1712850874.7230315,"name":"online","context":{"idset":"93"}}
+{"timestamp":1712850874.8519402,"name":"online","context":{"idset":"96,100"}}
+{"timestamp":1712850874.9838197,"name":"online","context":{"idset":"109"}}
+{"timestamp":1712850875.1018229,"name":"online","context":{"idset":"97,101"}}
+{"timestamp":1712850875.2411947,"name":"online","context":{"idset":"103,111"}}
+{"timestamp":1712850875.3600562,"name":"online","context":{"idset":"98,102,105,107,113"}}
+{"timestamp":1712850875.5606937,"name":"online","context":{"idset":"95,99,106,108,115"}}
+{"timestamp":1712850875.6633599,"name":"online","context":{"idset":"104,110,112,114,116"}}
+{"timestamp":1712850880.6946528,"name":"online","context":{"idset":"117"}}
+{"timestamp":1712850880.8183246,"name":"online","context":{"idset":"118-120"}}
+{"timestamp":1712850881.0762377,"name":"online","context":{"idset":"122"}}
+{"timestamp":1712850881.1845796,"name":"online","context":{"idset":"123-124"}}
+{"timestamp":1712850883.0323348,"name":"online","context":{"idset":"125"}}
+{"timestamp":1712850883.3949542,"name":"online","context":{"idset":"126"}}
+{"timestamp":1712850883.7477217,"name":"online","context":{"idset":"127"}}
+{"timestamp":1712850883.9538634,"name":"online","context":{"idset":"128"}}
+{"timestamp":1712850884.1647639,"name":"online","context":{"idset":"129-130"}}
+{"timestamp":1712850884.5359788,"name":"online","context":{"idset":"131,133"}}
+{"timestamp":1712850884.8485529,"name":"online","context":{"idset":"132,134"}}
+{"timestamp":1712850885.1519432,"name":"online","context":{"idset":"135-136,138"}}
+{"timestamp":1712850885.3645043,"name":"online","context":{"idset":"137,139-141"}}
+{"timestamp":1712850885.5980825,"name":"online","context":{"idset":"142-146,149"}}
+{"timestamp":1712850885.7020335,"name":"online","context":{"idset":"147-148"}}
+{"timestamp":1712850889.7084887,"name":"online","context":{"idset":"150"}}
+{"timestamp":1712850889.8610933,"name":"online","context":{"idset":"152-153"}}
+{"timestamp":1712850890.0601141,"name":"online","context":{"idset":"151,154"}}
+{"timestamp":1712850890.1625705,"name":"online","context":{"idset":"155-156"}}
+{"timestamp":1712850892.0482187,"name":"online","context":{"idset":"157"}}
+{"timestamp":1712850892.3381581,"name":"online","context":{"idset":"158"}}
+{"timestamp":1712850892.7514389,"name":"online","context":{"idset":"159"}}
+{"timestamp":1712850892.9743066,"name":"online","context":{"idset":"160"}}
+{"timestamp":1712850893.1869783,"name":"online","context":{"idset":"161-162"}}
+{"timestamp":1712850893.5500631,"name":"online","context":{"idset":"163-164"}}
+{"timestamp":1712850893.8991723,"name":"online","context":{"idset":"166"}}
+{"timestamp":1712850894.0937347,"name":"online","context":{"idset":"165"}}
+{"timestamp":1712850894.2995183,"name":"online","context":{"idset":"169"}}
+{"timestamp":1712850894.5184853,"name":"online","context":{"idset":"167-168,170-171"}}
+{"timestamp":1712850894.6586742,"name":"online","context":{"idset":"172-173"}}
+{"timestamp":1712850894.8667686,"name":"online","context":{"idset":"174-176"}}
+{"timestamp":1712850905.2779796,"name":"online","context":{"idset":"493,502"}}
+{"timestamp":1712850905.5512745,"name":"online","context":{"idset":"528"}}
+{"timestamp":1712850905.6550958,"name":"online","context":{"idset":"529"}}
+{"timestamp":1712850905.7875147,"name":"online","context":{"idset":"527,530-533"}}
+{"timestamp":1712850906.0958953,"name":"online","context":{"idset":"534-535"}}
+{"timestamp":1712850906.3167429,"name":"online","context":{"idset":"536"}}
+{"timestamp":1712850906.6700563,"name":"online","context":{"idset":"537"}}
+{"timestamp":1712850907.2479014,"name":"online","context":{"idset":"538,540"}}
+{"timestamp":1712850907.3513591,"name":"online","context":{"idset":"539"}}
+{"timestamp":1712850914.8306355,"name":"online","context":{"idset":"565,568"}}
+{"timestamp":1712850914.9503517,"name":"online","context":{"idset":"567,569"}}
+{"timestamp":1712850915.3806615,"name":"online","context":{"idset":"571"}}
+{"timestamp":1712850915.6756358,"name":"online","context":{"idset":"572,574-575,577,582"}}
+{"timestamp":1712850915.8794045,"name":"online","context":{"idset":"578-579,583"}}
+{"timestamp":1712850916.1033931,"name":"online","context":{"idset":"584"}}
+{"timestamp":1712850916.3097689,"name":"online","context":{"idset":"585,587"}}
+{"timestamp":1712850916.414144,"name":"online","context":{"idset":"586,588"}}
+{"timestamp":1712851419.3917129,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1712851419.4831491,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1712851544.6178129,"name":"undrain","context":{"idset":"11249-11250"}}
+{"timestamp":1712851545.7746203,"name":"undrain","context":{"idset":"9875-9876"}}
+{"timestamp":1712851949.4889591,"name":"drain","context":{"idset":"9782","reason":"broker was unresponsive"}}
+{"timestamp":1712852011.4844966,"name":"offline","context":{"idset":"9782"}}
+{"timestamp":1712852650.0254178,"name":"undrain","context":{"idset":"837-838"}}
+{"timestamp":1712853317.4912872,"name":"drain","context":{"idset":"9991","reason":"broker was unresponsive"}}
+{"timestamp":1712853379.4843078,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1712853393.1330025,"name":"online","context":{"idset":"11470"}}
+{"timestamp":1712853578.3560216,"name":"drain","context":{"idset":"10795-10796","reason":"draining to process swap --JRG","overwrite":0}}
+{"timestamp":1712854277.3872395,"name":"drain","context":{"idset":"949","reason":"broker was unresponsive"}}
+{"timestamp":1712854277.4874694,"name":"drain","context":{"idset":"950","reason":"broker was unresponsive"}}
+{"timestamp":1712854341.4050813,"name":"offline","context":{"idset":"949"}}
+{"timestamp":1712854341.5108874,"name":"offline","context":{"idset":"950"}}
+{"timestamp":1712854844.2730172,"name":"online","context":{"idset":"11000"}}
+{"timestamp":1712854844.5757592,"name":"online","context":{"idset":"11064"}}
+{"timestamp":1712854844.6818528,"name":"online","context":{"idset":"11096"}}
+{"timestamp":1712854845.4153557,"name":"online","context":{"idset":"11095"}}
+{"timestamp":1712854846.3324692,"name":"online","context":{"idset":"11063"}}
+{"timestamp":1712855728.3936656,"name":"online","context":{"idset":"177-179"}}
+{"timestamp":1712855728.9405811,"name":"online","context":{"idset":"181"}}
+{"timestamp":1712855731.6938198,"name":"online","context":{"idset":"182,184"}}
+{"timestamp":1712855731.9075897,"name":"online","context":{"idset":"185-188"}}
+{"timestamp":1712855734.5886326,"name":"online","context":{"idset":"192,202"}}
+{"timestamp":1712855734.7131524,"name":"online","context":{"idset":"193,201"}}
+{"timestamp":1712855734.8154509,"name":"online","context":{"idset":"189"}}
+{"timestamp":1712855734.9393752,"name":"online","context":{"idset":"204-205"}}
+{"timestamp":1712855734.962095,"name":"online","context":{"idset":"191,198"}}
+{"timestamp":1712855735.0930297,"name":"online","context":{"idset":"206"}}
+{"timestamp":1712855735.3099053,"name":"online","context":{"idset":"195,197,200,203,207-208"}}
+{"timestamp":1712855735.5706728,"name":"online","context":{"idset":"209"}}
+{"timestamp":1712855737.6144831,"name":"online","context":{"idset":"211-212"}}
+{"timestamp":1712855740.3080337,"name":"online","context":{"idset":"213"}}
+{"timestamp":1712855740.7302051,"name":"online","context":{"idset":"215"}}
+{"timestamp":1712855741.4705229,"name":"online","context":{"idset":"218"}}
+{"timestamp":1712855741.4737799,"name":"online","context":{"idset":"216,221"}}
+{"timestamp":1712855743.1756504,"name":"online","context":{"idset":"219"}}
+{"timestamp":1712855744.1512587,"name":"online","context":{"idset":"214,223-224"}}
+{"timestamp":1712855744.1547935,"name":"online","context":{"idset":"226,228"}}
+{"timestamp":1712855744.1582251,"name":"online","context":{"idset":"230"}}
+{"timestamp":1712855744.1612997,"name":"online","context":{"idset":"232"}}
+{"timestamp":1712855744.1785843,"name":"online","context":{"idset":"233"}}
+{"timestamp":1712855744.3273509,"name":"online","context":{"idset":"234-235"}}
+{"timestamp":1712855744.5945508,"name":"online","context":{"idset":"236-237,239-241"}}
+{"timestamp":1712855744.9099808,"name":"online","context":{"idset":"242"}}
+{"timestamp":1712855745.9552128,"name":"online","context":{"idset":"220"}}
+{"timestamp":1712855746.4671664,"name":"online","context":{"idset":"244"}}
+{"timestamp":1712855746.678005,"name":"online","context":{"idset":"243,245"}}
+{"timestamp":1712855746.9067819,"name":"online","context":{"idset":"246"}}
+{"timestamp":1712855750.2424464,"name":"online","context":{"idset":"247"}}
+{"timestamp":1712855750.2457972,"name":"online","context":{"idset":"248"}}
+{"timestamp":1712855750.3439131,"name":"online","context":{"idset":"250"}}
+{"timestamp":1712855750.5907674,"name":"online","context":{"idset":"249"}}
+{"timestamp":1712855751.3138516,"name":"online","context":{"idset":"251"}}
+{"timestamp":1712855752.3705394,"name":"online","context":{"idset":"252"}}
+{"timestamp":1712855755.3532891,"name":"online","context":{"idset":"277"}}
+{"timestamp":1712855756.2505438,"name":"online","context":{"idset":"278-281"}}
+{"timestamp":1712855756.2529082,"name":"online","context":{"idset":"283"}}
+{"timestamp":1712855756.2561426,"name":"online","context":{"idset":"282,284-285"}}
+{"timestamp":1712855756.2586367,"name":"online","context":{"idset":"286"}}
+{"timestamp":1712855756.453516,"name":"online","context":{"idset":"287"}}
+{"timestamp":1712855756.857898,"name":"online","context":{"idset":"288"}}
+{"timestamp":1712855758.2493005,"name":"online","context":{"idset":"289"}}
+{"timestamp":1712855758.494581,"name":"online","context":{"idset":"290-291"}}
+{"timestamp":1712855759.0340149,"name":"online","context":{"idset":"292"}}
+{"timestamp":1712855759.1819158,"name":"online","context":{"idset":"293-294"}}
+{"timestamp":1712855760.5052154,"name":"online","context":{"idset":"296"}}
+{"timestamp":1712855761.0878818,"name":"online","context":{"idset":"297"}}
+{"timestamp":1712855762.2836797,"name":"online","context":{"idset":"298"}}
+{"timestamp":1712855763.3399844,"name":"online","context":{"idset":"300"}}
+{"timestamp":1712855763.8243299,"name":"online","context":{"idset":"303"}}
+{"timestamp":1712855764.086704,"name":"online","context":{"idset":"302"}}
+{"timestamp":1712855764.6109078,"name":"online","context":{"idset":"301,306,308"}}
+{"timestamp":1712855764.8558536,"name":"online","context":{"idset":"309"}}
+{"timestamp":1712855765.0759661,"name":"online","context":{"idset":"307"}}
+{"timestamp":1712855765.3496273,"name":"online","context":{"idset":"304"}}
+{"timestamp":1712855765.589642,"name":"online","context":{"idset":"310-311"}}
+{"timestamp":1712855765.6943495,"name":"online","context":{"idset":"312"}}
+{"timestamp":1712855766.116092,"name":"online","context":{"idset":"305,315,319"}}
+{"timestamp":1712855766.4228272,"name":"online","context":{"idset":"314,316"}}
+{"timestamp":1712855766.6665022,"name":"online","context":{"idset":"317-318,321"}}
+{"timestamp":1712855766.7988625,"name":"online","context":{"idset":"320"}}
+{"timestamp":1712855767.9123378,"name":"online","context":{"idset":"322"}}
+{"timestamp":1712855769.0196993,"name":"online","context":{"idset":"326"}}
+{"timestamp":1712855769.021687,"name":"online","context":{"idset":"324-325"}}
+{"timestamp":1712855769.023591,"name":"online","context":{"idset":"323"}}
+{"timestamp":1712855769.5585439,"name":"online","context":{"idset":"327-328"}}
+{"timestamp":1712855771.593049,"name":"online","context":{"idset":"330"}}
+{"timestamp":1712855772.345787,"name":"online","context":{"idset":"331"}}
+{"timestamp":1712855773.2495511,"name":"online","context":{"idset":"333"}}
+{"timestamp":1712855773.6795516,"name":"online","context":{"idset":"334"}}
+{"timestamp":1712855773.9876645,"name":"online","context":{"idset":"336-337"}}
+{"timestamp":1712855774.3192396,"name":"online","context":{"idset":"338"}}
+{"timestamp":1712855774.5434809,"name":"online","context":{"idset":"339,341"}}
+{"timestamp":1712855774.6705308,"name":"online","context":{"idset":"340"}}
+{"timestamp":1712855774.7745731,"name":"online","context":{"idset":"343"}}
+{"timestamp":1712855775.0006433,"name":"online","context":{"idset":"344-345"}}
+{"timestamp":1712855775.1205783,"name":"online","context":{"idset":"346"}}
+{"timestamp":1712855775.3633578,"name":"online","context":{"idset":"347"}}
+{"timestamp":1712855775.5827713,"name":"online","context":{"idset":"349,351-352"}}
+{"timestamp":1712855776.6794505,"name":"online","context":{"idset":"353"}}
+{"timestamp":1712855777.3598351,"name":"online","context":{"idset":"354"}}
+{"timestamp":1712855777.4930527,"name":"online","context":{"idset":"357"}}
+{"timestamp":1712855777.7040081,"name":"online","context":{"idset":"358"}}
+{"timestamp":1712855778.3660648,"name":"online","context":{"idset":"359"}}
+{"timestamp":1712855780.0548759,"name":"online","context":{"idset":"361"}}
+{"timestamp":1712855780.4344485,"name":"online","context":{"idset":"362"}}
+{"timestamp":1712855781.2222562,"name":"online","context":{"idset":"363"}}
+{"timestamp":1712855782.2108619,"name":"online","context":{"idset":"364"}}
+{"timestamp":1712855782.7678201,"name":"online","context":{"idset":"366-367"}}
+{"timestamp":1712855783.3154404,"name":"online","context":{"idset":"369-370"}}
+{"timestamp":1712855783.58074,"name":"online","context":{"idset":"371-374"}}
+{"timestamp":1712855783.9905143,"name":"online","context":{"idset":"375-376"}}
+{"timestamp":1712855784.1516008,"name":"online","context":{"idset":"377-378"}}
+{"timestamp":1712855784.2704391,"name":"online","context":{"idset":"379"}}
+{"timestamp":1712855784.314606,"name":"online","context":{"idset":"380"}}
+{"timestamp":1712855784.5177212,"name":"online","context":{"idset":"381-382"}}
+{"timestamp":1712855785.5733206,"name":"online","context":{"idset":"383"}}
+{"timestamp":1712855786.5104964,"name":"online","context":{"idset":"386-387"}}
+{"timestamp":1712855786.8736799,"name":"online","context":{"idset":"388"}}
+{"timestamp":1712855787.099854,"name":"online","context":{"idset":"389"}}
+{"timestamp":1712855787.2043567,"name":"online","context":{"idset":"390"}}
+{"timestamp":1712855787.8921885,"name":"online","context":{"idset":"391"}}
+{"timestamp":1712855789.0933208,"name":"online","context":{"idset":"392"}}
+{"timestamp":1712855789.3220627,"name":"online","context":{"idset":"393"}}
+{"timestamp":1712855790.0887043,"name":"online","context":{"idset":"394"}}
+{"timestamp":1712855791.0955977,"name":"online","context":{"idset":"395"}}
+{"timestamp":1712855791.5898228,"name":"online","context":{"idset":"396-397"}}
+{"timestamp":1712855792.1439917,"name":"online","context":{"idset":"398"}}
+{"timestamp":1712855792.7804811,"name":"online","context":{"idset":"402"}}
+{"timestamp":1712855793.01949,"name":"online","context":{"idset":"401"}}
+{"timestamp":1712855793.3392556,"name":"online","context":{"idset":"407-408"}}
+{"timestamp":1712855793.6724131,"name":"online","context":{"idset":"405"}}
+{"timestamp":1712855793.8841372,"name":"online","context":{"idset":"410"}}
+{"timestamp":1712855794.0255141,"name":"online","context":{"idset":"404,412"}}
+{"timestamp":1712855794.126812,"name":"online","context":{"idset":"406"}}
+{"timestamp":1712855794.2310054,"name":"online","context":{"idset":"411"}}
+{"timestamp":1712855794.8592479,"name":"online","context":{"idset":"413"}}
+{"timestamp":1712855796.2486432,"name":"online","context":{"idset":"416-417"}}
+{"timestamp":1712855796.457721,"name":"online","context":{"idset":"418"}}
+{"timestamp":1712855797.1422112,"name":"online","context":{"idset":"414,420-421"}}
+{"timestamp":1712855799.8578143,"name":"online","context":{"idset":"423"}}
+{"timestamp":1712855799.9716041,"name":"online","context":{"idset":"424"}}
+{"timestamp":1712855800.7337515,"name":"online","context":{"idset":"425"}}
+{"timestamp":1712855801.5790298,"name":"online","context":{"idset":"426,428"}}
+{"timestamp":1712855802.0263519,"name":"online","context":{"idset":"429-431"}}
+{"timestamp":1712855802.2596533,"name":"online","context":{"idset":"427"}}
+{"timestamp":1712855802.4711645,"name":"online","context":{"idset":"433-434"}}
+{"timestamp":1712855803.1029608,"name":"online","context":{"idset":"436"}}
+{"timestamp":1712855803.4578168,"name":"online","context":{"idset":"437"}}
+{"timestamp":1712855803.660296,"name":"online","context":{"idset":"439,441"}}
+{"timestamp":1712855804.2255006,"name":"online","context":{"idset":"442"}}
+{"timestamp":1712855805.1258788,"name":"online","context":{"idset":"444"}}
+{"timestamp":1712855811.1089425,"name":"online","context":{"idset":"469-470"}}
+{"timestamp":1712855811.3071384,"name":"online","context":{"idset":"471-472"}}
+{"timestamp":1712855812.0231645,"name":"online","context":{"idset":"473-474"}}
+{"timestamp":1712855812.3228393,"name":"online","context":{"idset":"476-477"}}
+{"timestamp":1712855812.5771754,"name":"online","context":{"idset":"480"}}
+{"timestamp":1712855812.8024056,"name":"online","context":{"idset":"481-483"}}
+{"timestamp":1712855813.1212139,"name":"online","context":{"idset":"485-486"}}
+{"timestamp":1712855813.3706639,"name":"online","context":{"idset":"488-492"}}
+{"timestamp":1712855816.099839,"name":"online","context":{"idset":"494"}}
+{"timestamp":1712855819.7860272,"name":"online","context":{"idset":"495"}}
+{"timestamp":1712855820.0771656,"name":"online","context":{"idset":"496-497"}}
+{"timestamp":1712855820.2143254,"name":"online","context":{"idset":"499"}}
+{"timestamp":1712855820.9839265,"name":"online","context":{"idset":"500"}}
+{"timestamp":1712855821.4504278,"name":"online","context":{"idset":"503"}}
+{"timestamp":1712855821.783457,"name":"online","context":{"idset":"507"}}
+{"timestamp":1712855821.8866525,"name":"online","context":{"idset":"504"}}
+{"timestamp":1712855822.3512065,"name":"online","context":{"idset":"517"}}
+{"timestamp":1712855822.7248676,"name":"online","context":{"idset":"519"}}
+{"timestamp":1712855822.9463904,"name":"online","context":{"idset":"505"}}
+{"timestamp":1712855825.3060622,"name":"online","context":{"idset":"521"}}
+{"timestamp":1712855825.6512585,"name":"online","context":{"idset":"524"}}
+{"timestamp":1712855826.0516961,"name":"online","context":{"idset":"522,525"}}
+{"timestamp":1712855826.2474077,"name":"online","context":{"idset":"523"}}
+{"timestamp":1712857052.9419241,"name":"online","context":{"idset":"937,939"}}
+{"timestamp":1712857053.3056974,"name":"online","context":{"idset":"943"}}
+{"timestamp":1712857053.5539336,"name":"online","context":{"idset":"941"}}
+{"timestamp":1712857053.6801834,"name":"online","context":{"idset":"947"}}
+{"timestamp":1712857053.898509,"name":"online","context":{"idset":"945"}}
+{"timestamp":1712857302.5880837,"name":"online","context":{"idset":"183"}}
+{"timestamp":1712857302.8368313,"name":"online","context":{"idset":"199,225,231"}}
+{"timestamp":1712857302.9612944,"name":"online","context":{"idset":"196"}}
+{"timestamp":1712857303.1897473,"name":"online","context":{"idset":"229,238"}}
+{"timestamp":1712857303.4177756,"name":"online","context":{"idset":"299"}}
+{"timestamp":1712857303.5546789,"name":"online","context":{"idset":"295,313"}}
+{"timestamp":1712857303.7906849,"name":"online","context":{"idset":"329,335"}}
+{"timestamp":1712857304.0312576,"name":"online","context":{"idset":"342"}}
+{"timestamp":1712857307.8757582,"name":"online","context":{"idset":"350"}}
+{"timestamp":1712857307.8786845,"name":"online","context":{"idset":"355"}}
+{"timestamp":1712857307.8817813,"name":"online","context":{"idset":"356"}}
+{"timestamp":1712857307.8847406,"name":"online","context":{"idset":"360"}}
+{"timestamp":1712857308.118618,"name":"online","context":{"idset":"368"}}
+{"timestamp":1712857308.5028591,"name":"online","context":{"idset":"384"}}
+{"timestamp":1712857308.7436669,"name":"online","context":{"idset":"400"}}
+{"timestamp":1712857309.1069174,"name":"online","context":{"idset":"415"}}
+{"timestamp":1712857309.3273249,"name":"online","context":{"idset":"438"}}
+{"timestamp":1712857309.4650762,"name":"online","context":{"idset":"422"}}
+{"timestamp":1712857309.7007551,"name":"online","context":{"idset":"435,440,475"}}
+{"timestamp":1712857309.9136457,"name":"online","context":{"idset":"479"}}
+{"timestamp":1712857310.016716,"name":"online","context":{"idset":"487"}}
+{"timestamp":1712857310.1197512,"name":"online","context":{"idset":"478,518,520,526"}}
+{"timestamp":1712857311.4899836,"name":"drain","context":{"idset":"11424","reason":"broker was unresponsive"}}
+{"timestamp":1712857327.9831634,"name":"offline","context":{"idset":"10796"}}
+{"timestamp":1712857375.4854393,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1712857566.9584422,"name":"online","context":{"idset":"9535"}}
+{"timestamp":1712857956.8554637,"name":"online","context":{"idset":"761"}}
+{"timestamp":1712857957.0778463,"name":"online","context":{"idset":"762"}}
+{"timestamp":1712858079.4850941,"name":"drain","context":{"idset":"9533","reason":"broker was unresponsive"}}
+{"timestamp":1712858143.4845164,"name":"offline","context":{"idset":"9533"}}
+{"timestamp":1712858384.0348849,"name":"drain","context":{"idset":"573-580","reason":"New Blade Install --JRG","overwrite":0}}
+{"timestamp":1712858408.0883121,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1712858408.111026,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1712858408.1324942,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1712858408.1511683,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1712858408.2516379,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1712858512.7494123,"name":"drain","context":{"idset":"9534","reason":"broker was unresponsive"}}
+{"timestamp":1712858544.3368163,"name":"online","context":{"idset":"10903"}}
+{"timestamp":1712858544.4916186,"name":"online","context":{"idset":"10904"}}
+{"timestamp":1712858574.3327796,"name":"offline","context":{"idset":"9534"}}
+{"timestamp":1712859028.2233922,"name":"drain","context":{"idset":"828","reason":"broker was unresponsive"}}
+{"timestamp":1712859030.3444552,"name":"drain","context":{"idset":"827","reason":"broker was unresponsive"}}
+{"timestamp":1712859093.4011233,"name":"offline","context":{"idset":"827"}}
+{"timestamp":1712859094.2848749,"name":"offline","context":{"idset":"828"}}
+{"timestamp":1712859820.4433031,"name":"online","context":{"idset":"11005"}}
+{"timestamp":1712859822.1334462,"name":"online","context":{"idset":"11006"}}
+{"timestamp":1712859822.1881545,"name":"online","context":{"idset":"11061"}}
+{"timestamp":1712859822.8913608,"name":"online","context":{"idset":"11062"}}
+{"timestamp":1712859913.0536332,"name":"online","context":{"idset":"11136"}}
+{"timestamp":1712859913.260911,"name":"online","context":{"idset":"11135"}}
+{"timestamp":1712859931.1503253,"name":"drain","context":{"idset":"11135-11136","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1712860088.277787,"name":"undrain","context":{"idset":"10485-10516,10519-10541,10543-10570,10573-10612"}}
+{"timestamp":1712860250.2877991,"name":"undrain","context":{"idset":"11063-11064,11095-11096"}}
+{"timestamp":1712860558.2800913,"name":"drain","context":{"idset":"10613","reason":"broker was unresponsive"}}
+{"timestamp":1712860620.3064969,"name":"offline","context":{"idset":"10613"}}
+{"timestamp":1712861118.9619558,"name":"drain","context":{"idset":"10255","reason":"nodediag failed cxi pci","overwrite":0}}
+{"timestamp":1712861186.2315538,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1712861186.2346272,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1712861186.2377107,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1712861186.2407598,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1712861186.2439959,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1712861186.2472084,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1712861186.2503691,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1712861186.2533848,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1712861186.2566798,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1712861186.2599597,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1712861186.2634139,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1712861186.2665913,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1712861186.2696414,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1712861186.2727425,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1712861186.2758665,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1712861190.7613707,"name":"drain","context":{"idset":"11240","reason":"epilog failed for jobid fouxi1Dc6wR","overwrite":0}}
+{"timestamp":1712861190.8613815,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1712861252.2806678,"name":"drain","context":{"idset":"765-766","reason":"--reason Performing nodediag-estella2","overwrite":0}}
+{"timestamp":1712861494.1055927,"name":"online","context":{"idset":"934"}}
+{"timestamp":1712861688.5244303,"name":"online","context":{"idset":"9059"}}
+{"timestamp":1712861688.5277488,"name":"online","context":{"idset":"9031,9036"}}
+{"timestamp":1712861688.5308812,"name":"online","context":{"idset":"9037"}}
+{"timestamp":1712861688.6439869,"name":"online","context":{"idset":"9005,9055"}}
+{"timestamp":1712861688.9320698,"name":"online","context":{"idset":"9004"}}
+{"timestamp":1712861690.0970132,"name":"online","context":{"idset":"9011"}}
+{"timestamp":1712861690.1012402,"name":"online","context":{"idset":"9007,9066"}}
+{"timestamp":1712861690.1057284,"name":"online","context":{"idset":"9057"}}
+{"timestamp":1712861690.6560597,"name":"online","context":{"idset":"9074"}}
+{"timestamp":1712861690.785924,"name":"online","context":{"idset":"9054"}}
+{"timestamp":1712861690.8345699,"name":"online","context":{"idset":"9039"}}
+{"timestamp":1712861691.1531303,"name":"online","context":{"idset":"9070"}}
+{"timestamp":1712861691.2836392,"name":"online","context":{"idset":"8982"}}
+{"timestamp":1712861692.0835505,"name":"online","context":{"idset":"9014,9019,9030"}}
+{"timestamp":1712861692.0868042,"name":"online","context":{"idset":"9008"}}
+{"timestamp":1712861692.0904264,"name":"online","context":{"idset":"9025,9034,9043"}}
+{"timestamp":1712861692.0939741,"name":"online","context":{"idset":"9006,9013,9035,9072"}}
+{"timestamp":1712861692.1070747,"name":"online","context":{"idset":"9058"}}
+{"timestamp":1712861692.4004025,"name":"online","context":{"idset":"8959,8962,8971,9016,9045,9048"}}
+{"timestamp":1712861692.6917522,"name":"online","context":{"idset":"8958,8966-8968,8973,8984,8989,9001,9009,9012,9015,9020,9029,9032-9033,9049,9052,9061,9073"}}
+{"timestamp":1712861692.8319135,"name":"online","context":{"idset":"8988,8993,8998,9010,9021,9044,9046,9050"}}
+{"timestamp":1712861692.976939,"name":"online","context":{"idset":"8960-8961,8969,8990,8999,9017,9023,9026,9038,9041,9063-9064"}}
+{"timestamp":1712861693.1274421,"name":"online","context":{"idset":"8954,8972,8980,9000,9024,9027,9040,9069,9075"}}
+{"timestamp":1712861693.2491331,"name":"online","context":{"idset":"8957,8986,8997,9022,9042,9047,9051,9056,9065,9068,9076"}}
+{"timestamp":1712861693.4138696,"name":"online","context":{"idset":"8951"}}
+{"timestamp":1712861694.3500352,"name":"online","context":{"idset":"8949,8952,8956,8964-8965,8974,8976-8978,8981,8983,8985,8987,8995,9002,9060,9067,9071"}}
+{"timestamp":1712861694.353801,"name":"online","context":{"idset":"8970,8991,8996"}}
+{"timestamp":1712861694.3574648,"name":"online","context":{"idset":"8975,8979,8992,8994,9028"}}
+{"timestamp":1712861694.3610935,"name":"online","context":{"idset":"8950,8955,8963,9062"}}
+{"timestamp":1712861694.364677,"name":"online","context":{"idset":"8953"}}
+{"timestamp":1712861980.3612182,"name":"online","context":{"idset":"11015-11016,11021"}}
+{"timestamp":1712861980.583462,"name":"online","context":{"idset":"11022"}}
+{"timestamp":1712862032.7360361,"name":"drain","context":{"idset":"762","reason":"nodediag failed amdapu dgemm_perf","overwrite":0}}
+{"timestamp":1712862214.4574487,"name":"undrain","context":{"idset":"11021"}}
+{"timestamp":1712862852.1190321,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1712863022.1354616,"name":"drain","context":{"idset":"9535","reason":"broker was unresponsive"}}
+{"timestamp":1712863086.1282196,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1712863358.1213357,"name":"drain","context":{"idset":"10402","reason":"broker was unresponsive"}}
+{"timestamp":1712863420.1092064,"name":"offline","context":{"idset":"10402"}}
+{"timestamp":1712863959.0793853,"name":"undrain","context":{"idset":"748"}}
+{"timestamp":1712864116.1275845,"name":"drain","context":{"idset":"9054","reason":"broker was unresponsive"}}
+{"timestamp":1712864130.4486458,"name":"drain","context":{"idset":"9075","reason":"broker was unresponsive"}}
+{"timestamp":1712864180.0952775,"name":"offline","context":{"idset":"9054"}}
+{"timestamp":1712864194.112236,"name":"offline","context":{"idset":"9075"}}
+{"timestamp":1712864318.5868478,"name":"online","context":{"idset":"9447"}}
+{"timestamp":1712864318.8126888,"name":"online","context":{"idset":"9448"}}
+{"timestamp":1712864412.1188557,"name":"online","context":{"idset":"8009"}}
+{"timestamp":1712864412.1221268,"name":"online","context":{"idset":"7926"}}
+{"timestamp":1712864412.13726,"name":"online","context":{"idset":"7929"}}
+{"timestamp":1712864412.260452,"name":"online","context":{"idset":"8022"}}
+{"timestamp":1712864412.2954283,"name":"online","context":{"idset":"7932,8012,8016,8032"}}
+{"timestamp":1712864412.4357882,"name":"online","context":{"idset":"8044"}}
+{"timestamp":1712864412.7351248,"name":"online","context":{"idset":"8023"}}
+{"timestamp":1712864412.9341664,"name":"online","context":{"idset":"8021,8039"}}
+{"timestamp":1712864413.0754092,"name":"online","context":{"idset":"8024,8038"}}
+{"timestamp":1712864413.2049775,"name":"online","context":{"idset":"8011,8026"}}
+{"timestamp":1712864413.3246753,"name":"online","context":{"idset":"8013"}}
+{"timestamp":1712864414.1884432,"name":"online","context":{"idset":"7931,8015"}}
+{"timestamp":1712864414.1922803,"name":"online","context":{"idset":"8010"}}
+{"timestamp":1712864414.1957133,"name":"online","context":{"idset":"7936"}}
+{"timestamp":1712864414.199789,"name":"online","context":{"idset":"8029"}}
+{"timestamp":1712864414.2040312,"name":"online","context":{"idset":"8008"}}
+{"timestamp":1712864414.2086344,"name":"online","context":{"idset":"7980"}}
+{"timestamp":1712864414.3543694,"name":"online","context":{"idset":"7958"}}
+{"timestamp":1712864414.4849653,"name":"online","context":{"idset":"8033"}}
+{"timestamp":1712864414.6441135,"name":"online","context":{"idset":"8045"}}
+{"timestamp":1712864414.6607068,"name":"online","context":{"idset":"8014"}}
+{"timestamp":1712864414.7996223,"name":"online","context":{"idset":"7928,7983"}}
+{"timestamp":1712864414.8195634,"name":"online","context":{"idset":"8006"}}
+{"timestamp":1712864414.9385712,"name":"online","context":{"idset":"8017-8018"}}
+{"timestamp":1712864414.9755123,"name":"online","context":{"idset":"7961"}}
+{"timestamp":1712864415.0988882,"name":"online","context":{"idset":"7930,8036"}}
+{"timestamp":1712864415.1227036,"name":"online","context":{"idset":"8020"}}
+{"timestamp":1712864415.296783,"name":"online","context":{"idset":"7960,8050"}}
+{"timestamp":1712864416.1209192,"name":"online","context":{"idset":"7927,8031,8046,8051"}}
+{"timestamp":1712864416.1245639,"name":"online","context":{"idset":"7925,7956,8019,8028"}}
+{"timestamp":1712864416.1281579,"name":"online","context":{"idset":"7933,7981-7982,8025,8034,8042,8052"}}
+{"timestamp":1712864416.1317828,"name":"online","context":{"idset":"7937,7940,8030,8043"}}
+{"timestamp":1712864416.1848361,"name":"online","context":{"idset":"7934,7942,7957,8007,8035"}}
+{"timestamp":1712864416.4208021,"name":"online","context":{"idset":"7959,7962,7966,7984,7989-7990,8040-8041"}}
+{"timestamp":1712864416.5419676,"name":"online","context":{"idset":"7985"}}
+{"timestamp":1712864416.7580335,"name":"online","context":{"idset":"7935,7939,7967,7978,7986,7992,7995,8027,8037,8049"}}
+{"timestamp":1712864416.8818736,"name":"online","context":{"idset":"7938,7941,7951-7952,7964-7965,7969,7971,7973,7976-7977,7979,8005"}}
+{"timestamp":1712864417.0031953,"name":"online","context":{"idset":"7945,7954,7970,7974,7993,8001-8002"}}
+{"timestamp":1712864417.1254871,"name":"online","context":{"idset":"7946,7950,7953,7972,7991,7996"}}
+{"timestamp":1712864417.2686307,"name":"online","context":{"idset":"7948-7949,7963,7968,7975,7997"}}
+{"timestamp":1712864417.3707552,"name":"online","context":{"idset":"7944,7955,7994,7998-8000,8004"}}
+{"timestamp":1712864418.1521845,"name":"online","context":{"idset":"7947,8003"}}
+{"timestamp":1712864418.1567085,"name":"online","context":{"idset":"7943"}}
+{"timestamp":1712864426.4722612,"name":"drain","context":{"idset":"65","reason":"NC unresponsive","overwrite":0}}
+{"timestamp":1712864539.3291147,"name":"drain","context":{"idset":"253-276","reason":"Leak investigation","overwrite":0}}
+{"timestamp":1712864642.1122453,"name":"drain","context":{"idset":"9588","reason":"broker was unresponsive"}}
+{"timestamp":1712864644.1386249,"name":"drain","context":{"idset":"9587","reason":"broker was unresponsive"}}
+{"timestamp":1712864705.3284414,"name":"offline","context":{"idset":"9587"}}
+{"timestamp":1712864705.4158721,"name":"offline","context":{"idset":"9588"}}
+{"timestamp":1712864756.4313865,"name":"drain","context":{"idset":"217","reason":"Drops to UEFI on boot","overwrite":0}}
+{"timestamp":1712864871.3503184,"name":"drain","context":{"idset":"501","reason":"ASSERT_EFI_ERROR on boot","overwrite":0}}
+{"timestamp":1712864932.9871686,"name":"drain","context":{"idset":"508","reason":"Drops to UEFI on boot","overwrite":0}}
+{"timestamp":1712864992.8868096,"name":"drain","context":{"idset":"570","reason":"ASSERT_EFI_ERROR on boot","overwrite":0}}
+{"timestamp":1712865047.1073606,"name":"undrain","context":{"idset":"11135-11136"}}
+{"timestamp":1712865159.0108924,"name":"online","context":{"idset":"8047"}}
+{"timestamp":1712865316.1076317,"name":"online","context":{"idset":"9003"}}
+{"timestamp":1712866322.2417347,"name":"online","context":{"idset":"7987"}}
+{"timestamp":1712866350.7723167,"name":"drain","context":{"idset":"937-938,11095-11096","overwrite":0}}
+{"timestamp":1712866366.2025063,"name":"offline","context":{"idset":"11095"}}
+{"timestamp":1712866366.301544,"name":"offline","context":{"idset":"11096"}}
+{"timestamp":1712866372.4825892,"name":"offline","context":{"idset":"937"}}
+{"timestamp":1712867664.4398384,"name":"drain","context":{"idset":"11509-11588","overwrite":0}}
+{"timestamp":1712867758.8799312,"name":"undrain","context":{"idset":"11509-11572"}}
+{"timestamp":1712868053.3882167,"name":"drain","context":{"idset":"795","reason":"broker was unresponsive"}}
+{"timestamp":1712868054.1067045,"name":"drain","context":{"idset":"796","reason":"broker was unresponsive"}}
+{"timestamp":1712868117.3941269,"name":"offline","context":{"idset":"795"}}
+{"timestamp":1712868118.1737876,"name":"offline","context":{"idset":"796"}}
+{"timestamp":1712869067.3880043,"name":"drain","context":{"idset":"841","reason":"broker was unresponsive"}}
+{"timestamp":1712869067.488508,"name":"drain","context":{"idset":"842","reason":"broker was unresponsive"}}
+{"timestamp":1712869107.2075434,"name":"online","context":{"idset":"8048"}}
+{"timestamp":1712869133.3976536,"name":"offline","context":{"idset":"841"}}
+{"timestamp":1712869133.4847431,"name":"offline","context":{"idset":"842"}}
+{"timestamp":1712870255.3889601,"name":"drain","context":{"idset":"9557","reason":"broker was unresponsive"}}
+{"timestamp":1712870255.4897308,"name":"drain","context":{"idset":"9558","reason":"broker was unresponsive"}}
+{"timestamp":1712870261.3887694,"name":"drain","context":{"idset":"9559","reason":"broker was unresponsive"}}
+{"timestamp":1712870261.4890063,"name":"drain","context":{"idset":"9560","reason":"broker was unresponsive"}}
+{"timestamp":1712870267.3898699,"name":"drain","context":{"idset":"9563","reason":"broker was unresponsive"}}
+{"timestamp":1712870267.3899961,"name":"drain","context":{"idset":"9564","reason":"broker was unresponsive"}}
+{"timestamp":1712870267.3900747,"name":"drain","context":{"idset":"9565","reason":"broker was unresponsive"}}
+{"timestamp":1712870267.5055051,"name":"drain","context":{"idset":"9566","reason":"broker was unresponsive"}}
+{"timestamp":1712870273.3867383,"name":"drain","context":{"idset":"9567","reason":"broker was unresponsive"}}
+{"timestamp":1712870273.4878447,"name":"drain","context":{"idset":"9568","reason":"broker was unresponsive"}}
+{"timestamp":1712870279.391638,"name":"drain","context":{"idset":"9569","reason":"broker was unresponsive"}}
+{"timestamp":1712870279.3917589,"name":"drain","context":{"idset":"9570","reason":"broker was unresponsive"}}
+{"timestamp":1712870279.3918495,"name":"drain","context":{"idset":"9571","reason":"broker was unresponsive"}}
+{"timestamp":1712870279.4914186,"name":"drain","context":{"idset":"9572","reason":"broker was unresponsive"}}
+{"timestamp":1712870317.3926826,"name":"offline","context":{"idset":"9557"}}
+{"timestamp":1712870317.4849288,"name":"offline","context":{"idset":"9558"}}
+{"timestamp":1712870323.4849627,"name":"offline","context":{"idset":"9560"}}
+{"timestamp":1712870329.3971667,"name":"offline","context":{"idset":"9563"}}
+{"timestamp":1712870329.485806,"name":"offline","context":{"idset":"9564"}}
+{"timestamp":1712870333.4026871,"name":"offline","context":{"idset":"9565"}}
+{"timestamp":1712870333.4932437,"name":"offline","context":{"idset":"9566"}}
+{"timestamp":1712870337.3993294,"name":"offline","context":{"idset":"9567"}}
+{"timestamp":1712870337.4870796,"name":"offline","context":{"idset":"9568"}}
+{"timestamp":1712870339.4898522,"name":"offline","context":{"idset":"9559"}}
+{"timestamp":1712870341.4019239,"name":"offline","context":{"idset":"9569"}}
+{"timestamp":1712870341.4845798,"name":"offline","context":{"idset":"9570"}}
+{"timestamp":1712870345.414578,"name":"offline","context":{"idset":"9571"}}
+{"timestamp":1712870345.4938126,"name":"offline","context":{"idset":"9572"}}
+{"timestamp":1712870603.5196011,"name":"drain","context":{"idset":"9557-9560,9563-9571","reason":"epilog failed for jobid fox7vqB5xWs","overwrite":0}}
+{"timestamp":1712870761.8975849,"name":"undrain","context":{"idset":"765-766"}}
+{"timestamp":1712870827.2805638,"name":"online","context":{"idset":"766"}}
+{"timestamp":1712870919.0362146,"name":"online","context":{"idset":"11096"}}
+{"timestamp":1712870919.3378727,"name":"online","context":{"idset":"11095"}}
+{"timestamp":1712870971.3283691,"name":"drain","context":{"idset":"11095-11096","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1712871145.8685236,"name":"online","context":{"idset":"841-842"}}
+{"timestamp":1712871261.0428114,"name":"drain","context":{"idset":"7925-7987,7989-8047,8049-8052","reason":"diags","overwrite":0}}
+{"timestamp":1712871444.1933727,"name":"drain","context":{"idset":"767-768","overwrite":0}}
+{"timestamp":1712871754.7317705,"name":"undrain","context":{"idset":"7925-7987,7989-8047,8049-8052"}}
+{"timestamp":1712871794.6748843,"name":"online","context":{"idset":"10796"}}
+{"timestamp":1712871794.7774827,"name":"online","context":{"idset":"10795"}}
+{"timestamp":1712871836.3195343,"name":"undrain","context":{"idset":"10795-10796"}}
+{"timestamp":1712871868.1973295,"name":"drain","context":{"idset":"767-768","reason":"--reason drain to resolve a cxi issue ","overwrite":0}}
+{"timestamp":1712872483.4850714,"name":"online","context":{"idset":"765"}}
+{"timestamp":1712872628.3843384,"name":"drain","context":{"idset":"765","reason":"--reason resolve error:rank 765-estella2","overwrite":0}}
+{"timestamp":1712872676.0627444,"name":"offline","context":{"idset":"765"}}
+{"timestamp":1712872855.1486433,"name":"online","context":{"idset":"765"}}
+{"timestamp":1712872881.9549704,"name":"undrain","context":{"idset":"765"}}
+{"timestamp":1712873721.4897759,"name":"drain","context":{"idset":"10541","reason":"broker was unresponsive"}}
+{"timestamp":1712873785.4861882,"name":"offline","context":{"idset":"10541"}}
+{"timestamp":1712873795.4896183,"name":"drain","context":{"idset":"8048","reason":"broker was unresponsive"}}
+{"timestamp":1712873857.4871547,"name":"offline","context":{"idset":"8048"}}
+{"timestamp":1712873964.3604848,"name":"drain","context":{"idset":"9003","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712874088.0913939,"name":"online","context":{"idset":"10517"}}
+{"timestamp":1712874088.2484071,"name":"online","context":{"idset":"10402"}}
+{"timestamp":1712874088.3508554,"name":"online","context":{"idset":"10401"}}
+{"timestamp":1712874088.797569,"name":"online","context":{"idset":"10518"}}
+{"timestamp":1712874174.3357418,"name":"undrain","context":{"idset":"10517"}}
+{"timestamp":1712874178.0765035,"name":"undrain","context":{"idset":"10518"}}
+{"timestamp":1712874190.4907138,"name":"undrain","context":{"idset":"10402"}}
+{"timestamp":1712874256.1546974,"name":"undrain","context":{"idset":"11095-11096"}}
+{"timestamp":1712874369.335789,"name":"online","context":{"idset":"10571"}}
+{"timestamp":1712874369.7637856,"name":"online","context":{"idset":"10572"}}
+{"timestamp":1712874510.8769896,"name":"undrain","context":{"idset":"10541-10542"}}
+{"timestamp":1712874524.4095085,"name":"drain","context":{"idset":"10541-10542","reason":"HP Blade swap cxi issue","overwrite":0}}
+{"timestamp":1712874626.5917356,"name":"online","context":{"idset":"9054"}}
+{"timestamp":1712874626.8736567,"name":"online","context":{"idset":"9053"}}
+{"timestamp":1712875192.965554,"name":"undrain","context":{"idset":"841-842"}}
+{"timestamp":1712875473.468318,"name":"offline","context":{"idset":"505"}}
+{"timestamp":1712876177.5721767,"name":"online","context":{"idset":"8048"}}
+{"timestamp":1712876217.1334486,"name":"undrain","context":{"idset":"8048"}}
+{"timestamp":1712876380.0871921,"name":"drain","context":{"idset":"493-500,502-505,507","reason":"New blade install --JRG","overwrite":0}}
+{"timestamp":1712876404.7154319,"name":"offline","context":{"idset":"497"}}
+{"timestamp":1712876404.7730157,"name":"offline","context":{"idset":"502"}}
+{"timestamp":1712876404.8568246,"name":"offline","context":{"idset":"493"}}
+{"timestamp":1712876405.2516005,"name":"offline","context":{"idset":"499"}}
+{"timestamp":1712876405.302778,"name":"offline","context":{"idset":"503"}}
+{"timestamp":1712876405.403933,"name":"offline","context":{"idset":"496"}}
+{"timestamp":1712876405.4788687,"name":"offline","context":{"idset":"504"}}
+{"timestamp":1712876405.5018353,"name":"offline","context":{"idset":"495"}}
+{"timestamp":1712876405.6021852,"name":"offline","context":{"idset":"494"}}
+{"timestamp":1712876405.6392303,"name":"offline","context":{"idset":"500"}}
+{"timestamp":1712876405.7384276,"name":"offline","context":{"idset":"507"}}
+{"timestamp":1712877964.7837136,"name":"online","context":{"idset":"11240"}}
+{"timestamp":1712878172.6309819,"name":"undrain","context":{"idset":"11240"}}
+{"timestamp":1712878259.3287642,"name":"online","context":{"idset":"11248"}}
+{"timestamp":1712878259.6138761,"name":"online","context":{"idset":"11241,11243,11250"}}
+{"timestamp":1712878259.8467622,"name":"online","context":{"idset":"11237-11238,11242,11244"}}
+{"timestamp":1712878259.9890335,"name":"online","context":{"idset":"11246"}}
+{"timestamp":1712878260.1165051,"name":"online","context":{"idset":"11239,11247,11251"}}
+{"timestamp":1712878260.2635717,"name":"online","context":{"idset":"11252"}}
+{"timestamp":1712878260.411365,"name":"online","context":{"idset":"11245"}}
+{"timestamp":1712878260.5142822,"name":"online","context":{"idset":"11249"}}
+{"timestamp":1712880214.2451546,"name":"drain","context":{"idset":"10086","reason":"nodediag failed","overwrite":0}}
+{"timestamp":1712880684.8637371,"name":"drain","context":{"idset":"9572","reason":"epilog failed for jobid fox6jXfP92P","overwrite":0}}
+{"timestamp":1712880687.8568184,"name":"drain","context":{"idset":"8048","reason":"epilog failed for jobid foxZW969KtT","overwrite":0}}
+{"timestamp":1712886626.3204839,"name":"online","context":{"idset":"10637"}}
+{"timestamp":1712886626.476568,"name":"online","context":{"idset":"10634,10639,10644"}}
+{"timestamp":1712886626.8587697,"name":"online","context":{"idset":"10629,10638"}}
+{"timestamp":1712886627.1219535,"name":"online","context":{"idset":"10642-10643"}}
+{"timestamp":1712890690.2298319,"name":"resource-init","context":{"restart":true,"drain":{"65":{"timestamp":1712864426.4722612,"reason":"NC unresponsive"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"217":{"timestamp":1712864756.4313865,"reason":"Drops to UEFI on boot"},"253-276":{"timestamp":1712864539.3291147,"reason":"Leak investigation"},"348":{"timestamp":1711576490.2548447,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"501":{"timestamp":1712864871.3503184,"reason":"ASSERT_EFI_ERROR on boot"},"506":{"timestamp":1712783164.2746041,"reason":"Rabbit crashed due to node bringup --JRG"},"493-500,502-505,507":{"timestamp":1712876380.0871921,"reason":"New blade install --JRG"},"508":{"timestamp":1712864932.9871686,"reason":"Drops to UEFI on boot"},"509-516":{"timestamp":1712849612.3305345,"reason":"New Blade Installation"},"570":{"timestamp":1712864992.8868096,"reason":"ASSERT_EFI_ERROR on boot"},"573-580":{"timestamp":1712858384.0348849,"reason":"New Blade Install --JRG"},"629-636":{"timestamp":1712696364.0966945,"reason":"New Blade Installation --JRG"},"762":{"timestamp":1712862032.7360361,"reason":"nodediag failed amdapu dgemm_perf"},"767-768":{"timestamp":1712871444.1933727,"reason":"--reason drain to resolve a cxi issue "},"769-770":{"timestamp":1712780413.3028424,"reason":""},"773-774":{"timestamp":1712791238.8350906,"reason":""},"775-776":{"timestamp":1712792380.1267438,"reason":""},"789-794":{"timestamp":1711461777.1299369,"reason":""},"795":{"timestamp":1712868053.3882167,"reason":"broker was unresponsive"},"796":{"timestamp":1712868054.1067045,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261862.602201,"reason":"--reason swapping blades into x1900 - wendy"},"814":{"timestamp":1712783332.1390359,"reason":""},"827":{"timestamp":1712859030.3444552,"reason":"broker was unresponsive"},"828":{"timestamp":1712859028.2233922,"reason":"broker was unresponsive"},"829-830":{"timestamp":1712700944.7348719,"reason":""},"877":{"timestamp":1712672904.2689505,"reason":"reason=drain for NMC0 swap"},"878":{"timestamp":1712701679.2107792,"reason":"troubshooting H/W"},"937-938":{"timestamp":1712866350.7723167,"reason":""},"949":{"timestamp":1712854277.3872395,"reason":"broker was unresponsive"},"950":{"timestamp":1712854277.4874694,"reason":"broker was unresponsive"},"8048":{"timestamp":1712880687.8568184,"reason":"epilog failed for jobid foxZW969KtT"},"9003":{"timestamp":1712873964.3604848,"reason":"nodediag failed dgemm_perf"},"9054":{"timestamp":1712864116.1275845,"reason":"broker was unresponsive"},"9075":{"timestamp":1712864130.4486458,"reason":"broker was unresponsive"},"9533":{"timestamp":1712858079.4850941,"reason":"broker was unresponsive"},"9534":{"timestamp":1712858512.7494123,"reason":"broker was unresponsive"},"9535":{"timestamp":1712863022.1354616,"reason":"broker was unresponsive"},"9557":{"timestamp":1712870255.3889601,"reason":"broker was unresponsive"},"9558":{"timestamp":1712870255.4897308,"reason":"broker was unresponsive"},"9559":{"timestamp":1712870261.3887694,"reason":"broker was unresponsive"},"9560":{"timestamp":1712870261.4890063,"reason":"broker was unresponsive"},"9561-9562":{"timestamp":1712615968.8489521,"reason":"reason=Bring up test TB"},"9563":{"timestamp":1712870267.3898699,"reason":"broker was unresponsive"},"9564":{"timestamp":1712870267.3899961,"reason":"broker was unresponsive"},"9565":{"timestamp":1712870267.3900747,"reason":"broker was unresponsive"},"9566":{"timestamp":1712870267.5055051,"reason":"broker was unresponsive"},"9567":{"timestamp":1712870273.3867383,"reason":"broker was unresponsive"},"9568":{"timestamp":1712870273.4878447,"reason":"broker was unresponsive"},"9569":{"timestamp":1712870279.391638,"reason":"broker was unresponsive"},"9570":{"timestamp":1712870279.3917589,"reason":"broker was unresponsive"},"9571":{"timestamp":1712870279.3918495,"reason":"broker was unresponsive"},"9572":{"timestamp":1712870279.4914186,"reason":"broker was unresponsive"},"9587":{"timestamp":1712864644.1386249,"reason":"broker was unresponsive"},"9588":{"timestamp":1712864642.1122453,"reason":"broker was unresponsive"},"9782":{"timestamp":1712851949.4889591,"reason":"broker was unresponsive"},"9788":{"timestamp":1712710144.0153348,"reason":"bad partner node --JRG"},"9789":{"timestamp":1712711094.53824,"reason":"bad partner node --JRG"},"9991":{"timestamp":1712853317.4912872,"reason":"broker was unresponsive"},"10086":{"timestamp":1712880214.2451546,"reason":"nodediag failed"},"10255":{"timestamp":1712861118.9619558,"reason":"nodediag failed cxi pci"},"10261":{"timestamp":1712361648.1654882,"reason":"nodediag failed dgemm_perf"},"10303":{"timestamp":1712236781.0769484,"reason":"bad partner node --JRG"},"10355":{"timestamp":1712337664.7496204,"reason":"bad partner node --JRG"},"10541-10542":{"timestamp":1712874524.4095085,"reason":"HP Blade swap cxi issue"},"10613":{"timestamp":1712860558.2800913,"reason":"broker was unresponsive"},"10871-10872,10911-10912,10943-10944":{"timestamp":1712523855.472235,"reason":""},"10952":{"timestamp":1712592515.8810503,"reason":""},"10920,10925,10930,10935,10940,10945,10950,10955,10960,10965":{"timestamp":1712507305.3461049,"reason":""},"10869-10870,10873-10910,10913-10919,10921-10924,10926-10929,10931-10934,10936-10939,10941-10942,10946-10949,10951,10953-10954,10956-10959,10961-10964,10966-10969,10971-10974,10976-10979,10981-10984,10986-10989,10991-10992":{"timestamp":1712594249.0262253,"reason":""},"10970,10975,10980,10985,10990,10993-10996":{"timestamp":1712509491.5233767,"reason":""},"11424":{"timestamp":1712857311.4899836,"reason":"broker was unresponsive"},"11573-11588":{"timestamp":1712867664.4398384,"reason":""}},"online":"","exclude":"0"}}
+{"timestamp":1712890690.2841005,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1712890721.59041,"name":"online","context":{"idset":"0"}}
+{"timestamp":1712890723.4009519,"name":"online","context":{"idset":"10634,10878,10956,10958,10989,11067,11069,11077,11079-11080,11083,11110,11142,11146,11175,11212,11219,11236,11264,11299"}}
+{"timestamp":1712890723.4906185,"name":"online","context":{"idset":"798,10361,10629,10869,10872,10875,10879,10881,10883-10884,10913,10936,10944,10952,10973-10975,10994-10995,11001-11002,11005,11010,11024,11044,11047,11050,11068,11095-11096,11112-11113,11130-11131,11147,11159,11161,11163,11168,11173,11176,11180,11182,11215,11221,11225,11229,11237,11247,11256,11269,11289,11293,11295,11303,11315,11323,11325,11328-11329,11333"}}
+{"timestamp":1712890723.6391788,"name":"online","context":{"idset":"10402,10639,10876-10877,10888,10890-10891,10895,10916-10918,10921-10929,10931,10938-10939,10941-10943,10945-10947,10954-10955,10959,10962-10964,10969-10970,10972,10980,10984-10986,10998,11003-11004,11006,11008,11011,11013-11014,11016-11018,11021,11029,11031,11033-11034,11036,11039,11041,11043,11048,11051-11052,11056-11057,11061-11062,11064-11066,11070-11071,11078,11081-11082,11084,11087-11091,11093,11100-11102,11105,11107-11109,11111,11116-11117,11119-11120,11122,11125-11127,11135,11137,11139-11141,11143-11144,11150-11153,11155-11156,11158,11160,11165,11174,11179,11184,11194-11195,11198-11201,11204,11206,11208-11211,11213-11214,11217,11231,11235,11240,11245,11251-11252,11260,11262,11265,11267,11270,11274-11275,11277,11280,11287,11290,11292,11294,11296-11298,11305,11307,11309-11312,11321,11326"}}
+{"timestamp":1712890723.7651005,"name":"online","context":{"idset":"797,8949,9053-9054,10354,10357-10359,10364-10367,10401,10637,10642,10644,10870-10871,10880,10882,10885-10887,10889,10892,10897,10899-10902,10904,10906-10907,10910,10914-10915,10919,10930,10932-10933,10935,10937,10940,10949-10951,10953,10957,10960-10961,10965-10968,10971,10976-10979,10981,10983,10987-10988,10990,10992-10993,10996-10997,11000,11007,11009,11012,11015,11019-11020,11022-11023,11027-11028,11030,11032,11037,11040,11042,11045-11046,11049,11054-11055,11058-11060,11063,11072-11076,11085-11086,11092,11094,11097-11099,11103-11104,11106,11114-11115,11118,11121,11128-11129,11132-11134,11136,11138,11145,11149,11157,11162,11164,11166-11167,11170-11171,11177,11183,11185-11189,11191-11192,11196-11197,11202-11203,11205,11218,11220,11222-11224,11226-11228,11230,11234,11238-11239,11241-11244,11246,11248,11250,11253-11255,11257-11259,11261,11263,11266,11272-11273,11276,11279,11281-11286,11288,11291,11300-11302,11304,11306,11308,11313-11314,11316,11320,11322,11324,11327,11332,11334-11335,11569"}}
+{"timestamp":1712890723.9005063,"name":"online","context":{"idset":"10360,10362-10363,10638,10643,10894,10896,10898,10903,10905,10908-10909,10920,10991,11025,11123-11124,11172,11178,11193,11207,11271,11278,11319"}}
+{"timestamp":1712890724.0107903,"name":"online","context":{"idset":"86,88"}}
+{"timestamp":1712890724.1448183,"name":"online","context":{"idset":"85,87,871,9713,9725,9831,10064,11216"}}
+{"timestamp":1712890724.2481983,"name":"online","context":{"idset":"10097-10098,10572,10813,11317,11510"}}
+{"timestamp":1712890724.5312045,"name":"online","context":{"idset":"872,953,963,976,9004,9243,9493,9895,9952,10368,10414,10443,10508,10536,11342,11593,11728"}}
+{"timestamp":1712890724.6888103,"name":"online","context":{"idset":"805,815,820,7987,10370,10568,10631,10649,10700,11414,11462,11518"}}
+{"timestamp":1712890724.8026035,"name":"online","context":{"idset":"10532,10540,10630,10652,10654"}}
+{"timestamp":1712890724.9112163,"name":"online","context":{"idset":"49,155,377,415,951-952,7977,9355,9437,9446,9479,9513,9710,9869,9873,9881,9911,9930,9944,10037,10236,10318,10392,10435,10454,10551,10585,10661,10666,10689,10737,10753,10781,10783,11344,11524,11571,11677,11701,11704,11722,11788,11833,11845,11849"}}
+{"timestamp":1712890725.108376,"name":"online","context":{"idset":"2,4,8,10-11,15,21,28-29,31,35-36,38,40-41,45,48,51-52,125,159,183,245,247,287,347,375,964,978,7929,7945,8010,8022,8034,8046-8047,8051,8962,8964,8980,8986,9213,9215,9312,9337,9347,9373,9383,9387,9397,9401,9425,9443,9449,9464,9599,9633,9715-9716,9783,9865,9879,9892,9962,9970,10163,10243,10268,10273,10290,10315,10371,10384,10388,10400,10420,10425,10457,10461,10529,10567,10603,10608,10620-10621,10640,10660,10691,10693,10719,10723,10741,10758,10767,10801-10802,10821,10843,11374,11390,11392,11422,11428-11429,11457,11470,11485,11497,11502,11546,11548,11608,11625,11631,11655,11662,11666,11668,11670,11681,11689,11702,11720,11772,11784,11789,11807-11808,11825-11827,11848,11883,11887,11891"}}
+{"timestamp":1712890725.295655,"name":"online","context":{"idset":"1,3,5-7,9,12-14,16-20,22-27,30,32-34,37,39,42-44,46-47,50,91,111,151,163,189,199,211,215,231,239,343,357,359,373,381,393,395,427,431,441,471,479,481,523,525,527,529,531,533-535,539,565,582,584-585,587,766,816,934,974,7927,7935-7936,7942,7950,7959,7967,7971,7975,8000,8004-8005,8008-8009,8013-8014,8025,8028,8030,8040,8042,8048-8049,8951,8955,8959-8961,8965,8969,8972,8974,8977,8982,8985,8990,8993,8999-9000,9002,9005-9006,9045,9209,9211,9221,9224-9225,9230,9239,9241,9244,9246,9258,9262,9264,9267,9274,9277,9284,9294,9300,9306,9316,9320,9328,9339,9343,9345-9346,9354,9358,9361,9363,9369,9375-9376,9379,9393,9402,9405,9407,9411,9413,9415,9424,9427,9429-9431,9441-9442,9447,9451-9453,9455-9456,9459,9466-9467,9469,9473-9475,9478,9480,9483,9485,9488-9489,9495,9498,9501,9512,9519,9521,9526,9597,9610,9618,9622,9698,9711-9712,9726-9727,9747,9786,9821,9823,9832,9847,9849,9851,9854-9856,9868,9871,9883-9884,9889,9893,9922,9928-9929,9932,9938,9942,9958,9964,9968,10032,10059,10101-10102,10108,10114,10123-10124,10127,10153,10161,10170,10184,10195,10202,10211,10213,10216,10218,10223,10238,10241,10246,10270,10278,10284,10293,10297,10308,10316,10324,10334-10336,10341,10348,10352,10372,10376,10382,10391,10393-10394,10411,10416,10423,10428,10432,10437,10453,10468,10475,10479,10483,10485,10487,10498,10502,10517,10530,10537,10547-10548,10554-10556,10576-10577,10584,10587,10591,10602,10604-10605,10610,10619,10628,10632,10636,10641,10645-10646,10659,10662,10672,10675,10678,10683,10685,10696,10709-10710,10722,10725,10736,10746-10747,10757,10779-10780,10788,10797-10798,10803,10805,10815,10819,10822,10829,10853-10854,10862,10865,11343,11346-11347,11351,11354,11359,11361,11368,11375,11378,11380-11381,11384,11393-11394,11399,11401-11403,11406,11411-11412,11418,11420,11430,11433-11434,11444,11451,11455,11458,11476,11480,11499,11503,11516,11526,11531,11539,11544,11547,11549,11554,11559,11586,11594,11602,11615-11616,11619,11630,11632-11633,11663,11678,11685,11688,11696,11698,11700,11705,11727,11739,11741-11742,11748,11771,11774,11780,11794,11797,11803,11819,11822,11829,11841,11843,11847,11864-11865,11867,11873,11880,11882,11888"}}
+{"timestamp":1712890725.4366407,"name":"online","context":{"idset":"55,117,119,122,139,145,165,177,185,201,206-207,233,238,244,277,283,298,301,313,323,341,353,405,426,437,521,532,538,569,571,765,947,954,7926,7932,7934,7943,7990-7991,8001-8002,8006-8007,8012,8015-8019,8023,8029,8031,8035-8037,8045,8052,8950,8952,8954,8956-8958,8966-8967,8975-8976,8983-8984,8987,8991-8992,8994,8996-8998,9001,9027,9205-9206,9208,9212,9216-9217,9223,9228,9232-9233,9236-9237,9242,9248,9259,9266,9270,9278,9288,9293,9304,9325,9330,9333,9341,9353,9357,9360,9362,9367,9377,9380-9381,9385,9388,9395,9406,9408-9410,9417,9420,9423,9439,9445,9448,9457,9461,9463,9465,9468,9470,9476,9487,9491-9492,9494,9497,9499-9500,9503-9504,9507,9509-9510,9520,9553,9626,9670,9687,9714,9791,9845-9846,9852,9857,9859-9862,9867,9870,9875,9878,9880,9882,9885-9886,9891,9894,9896-9897,9899,9901-9902,9904-9905,9913,9915-9917,9919-9920,9927,9931,9936-9937,9939,9943,9945-9950,9954-9956,9959,9961,9963,9965-9966,9971-9972,9974,10044,10058,10078,10083,10090,10103,10105,10107,10111,10113,10115-10116,10119,10121-10122,10126,10128-10131,10133,10135,10137,10140-10141,10145,10147,10150-10152,10155-10156,10158-10160,10164,10166,10172,10174-10175,10179,10186-10187,10189,10191-10193,10200-10201,10204-10208,10210,10215,10220-10222,10224,10226,10228,10232,10234,10239-10240,10248,10252,10254,10256,10258,10263,10265-10267,10272,10274-10276,10280,10282,10292,10295-10296,10298,10300-10302,10305-10307,10309-10311,10314,10320,10322-10323,10326-10327,10333,10345-10347,10349-10351,10353,10373-10374,10377-10380,10383,10385-10387,10389-10390,10395,10397-10399,10403-10408,10410,10417-10419,10421-10422,10424,10427,10429-10431,10436,10438-10440,10444-10445,10447,10449,10451-10452,10455,10458-10459,10463,10465,10467,10469-10470,10472-10474,10478,10481-10482,10484,10488-10489,10491-10492,10494-10497,10499-10500,10503-10504,10506,10509-10512,10514,10516,10518-10522,10524,10526,10534-10535,10538-10539,10545-10546,10549-10550,10558-10560,10562,10564,10566,10570-10571,10575,10578,10580-10581,10583,10588-10589,10593-10596,10598,10600-10601,10606-10607,10616-10618,10622-10627,10633,10635,10648,10650-10651,10653,10656,10663-10664,10667-10668,10670,10673-10674,10676,10679-10680,10682,10684,10686,10688,10690,10692,10695,10697,10699,10701,10703-10707,10714-10716,10721,10724,10726,10728-10735,10738,10743,10745,10750,10752,10755,10759-10763,10765,10768,10770-10771,10773,10775-10776,10784-10786,10789,10791-10792,10795-10796,10799-10800,10806,10810,10812,10814,10820,10823-10828,10831,10833-10836,10838-10839,10841,10845,10848,10851,10856,10859-10860,10863-10864,10866,10868,11336,11338-11339,11341,11348,11350,11356-11358,11360,11362-11363,11367,11370-11371,11376-11377,11379,11382-11383,11385-11387,11389,11396,11398,11400,11405,11410,11413,11416,11421,11423,11426,11431-11432,11435-11436,11440-11443,11446,11448,11450,11454,11456,11460,11463-11469,11474-11475,11477-11478,11482-11484,11487-11496,11498,11501,11504,11507,11509,11511-11512,11517,11520,11522,11525,11528-11530,11533-11538,11540-11541,11543,11550-11553,11556,11562,11589,11591-11592,11595,11597,11599-11601,11603-11605,11607,11609-11613,11617-11618,11620,11622-11624,11626-11629,11634-11636,11654,11656-11658,11664-11665,11671-11674,11676,11679,11683-11684,11686,11690-11692,11706-11708,11713-11718,11724,11726,11729-11730,11733-11734,11736-11738,11743,11745-11746,11749-11755,11758-11761,11765-11767,11769-11770,11776,11778-11779,11783,11785,11787,11790,11792-11793,11795,11798-11799,11802,11805-11806,11809,11811-11813,11815-11818,11821,11823-11824,11830,11832,11834-11835,11837-11838,11840,11842,11844,11846,11853-11854,11857-11861,11863,11866,11869-11870,11874-11878,11881,11885-11886,11889-11890,11892"}}
+{"timestamp":1712890725.5757074,"name":"online","context":{"idset":"53,59,89,93-95,99,103,105,112,120,123,127,132-133,138,140,146,152-154,160,167,170,172-174,181,184,195-196,202-203,212-214,220,230,232,237,249-250,278-279,284,288-290,292,294,296-297,300,302-303,305,308-309,312,314,317-318,321-322,324,327,333,337-338,342,355-356,361,364,366,376,378,380,386,388,390-391,406,408,410,412,417,421-422,425,428,436,442,470,472,474,477-478,488,490-491,522,528,536,540,567,572,761-762,837,841,973,975,7931,7933,7937,7939,7944,7949,7951-7957,7963,7965-7966,7968-7969,7973-7974,7976,7978,7981-7983,7985-7986,7989,7993-7997,8003,8011,8020-8021,8024,8026-8027,8032-8033,8038-8039,8044,8050,8953,8963,8968,8970-8971,8973,8978-8979,8988-8989,8995,9003,9011,9029,9043,9064,9072,9207,9210,9214,9218-9220,9222,9226-9227,9229,9231,9234-9235,9238,9240,9245,9247,9249,9252-9254,9261,9268,9272,9275,9279-9282,9285-9287,9290-9291,9296,9298,9301-9303,9307,9309-9311,9313-9314,9318-9319,9322,9324,9326,9332,9334,9336,9338,9342,9344,9348-9351,9359,9365-9366,9368,9370-9372,9374,9378,9382,9384,9386,9389-9392,9394,9396,9398-9399,9403-9404,9412,9416,9418-9419,9421-9422,9426,9432-9433,9435-9436,9438,9440,9444,9450,9454,9458,9462,9481-9482,9484,9486,9490,9496,9505-9506,9508,9514-9517,9522-9523,9530,9554-9555,9576,9628-9629,9643,9651-9652,9660,9669,9678,9682-9685,9699-9700,9706,9737,9739,9751,9757,9759,9774,9794,9799,9805,9808,9835,9848,9850,9858,9863-9864,9866,9872,9874,9876,9887,9898,9900,9903,9906-9910,9914,9918,9921,9923-9926,9934-9935,9940-9941,9951,9953,9957,9969,9984,10022,10039,10047,10056,10071-10072,10092,10104,10106,10112,10117-10118,10120,10132,10134,10136,10138-10139,10142-10144,10146,10148-10149,10154,10157,10162,10165,10167-10169,10176-10178,10181,10183,10188,10190,10194,10196-10199,10203,10209,10214,10217,10219,10225,10227,10229-10231,10235,10237,10242,10244-10245,10247,10250,10253,10257,10259-10260,10269,10271,10277,10279,10283,10289,10291,10294,10299,10304,10313,10317,10321,10325,10328-10332,10337,10339-10340,10342,10375,10412-10413,10415,10441-10442,10448,10450,10460,10462,10464,10466,10471,10476-10477,10480,10486,10490,10493,10501,10505,10507,10515,10523,10525,10527,10533,10543-10544,10552-10553,10557,10561,10563,10565,10569,10573,10579,10586,10590,10592,10597,10599,10609,10611-10612,10615,10647,10655,10665,10677,10681,10687,10698,10708,10711,10713,10720,10727,10744,10748-10749,10754,10756,10766,10772,10774,10778,10782,10790,10793-10794,10804,10807-10809,10811,10816-10818,10830,10832,10837,10840,10842,10844,10846-10847,10850,10852,10855,10857-10858,10861,10867,11337,11340,11345,11349,11353,11355,11364-11365,11369,11372-11373,11391,11407-11408,11415,11417,11419,11425,11427,11438,11445,11449,11452-11453,11459,11461,11471,11479,11481,11486,11500,11505-11506,11508,11513,11515,11519,11521,11523,11527,11532,11545,11555,11558,11560,11564,11566-11567,11573-11575,11577,11580-11581,11583,11585,11588,11590,11596,11598,11606,11614,11653,11659-11660,11667,11669,11675,11680,11682,11687,11695,11697,11699,11703,11709-11712,11721,11723,11725,11732,11735,11744,11747,11757,11762,11764,11773,11775,11777,11781,11786,11791,11796,11800-11801,11804,11810,11814,11820,11828,11836,11839,11850,11852,11856,11862,11868,11871-11872,11884"}}
+{"timestamp":1712890725.7000446,"name":"online","context":{"idset":"58,90,96-98,100-102,104,107-108,114,118,126,128-131,134-135,142-143,147-148,150,156-158,161-162,166,168-169,171,175,178,182,186-187,193,198,200,209,216,218,223-224,228-229,236,248,251,282,307,310,315-316,326,331,334-336,345-346,349-351,354,362,367-368,370-371,374,379,383-384,387,396-398,402,411,413-414,420,423-424,430,434-435,438-439,444,469,475-476,483,489,492,517,520,524,526,806,842,939,941,943,945,977,7925,7928,7930,7938,7940-7941,7946-7948,7960-7962,7972,7979-7980,7984,7992,7998-7999,8041,8043,9013,9015,9030,9033,9035,9039,9046,9048,9050,9055,9076,9250-9251,9255-9257,9260,9263,9265,9269,9271,9273,9276,9289,9292,9299,9305,9315,9317,9321,9323,9327,9329,9331,9335,9340,9352,9364,9400,9428,9460,9502,9532,9537,9545-9546,9549,9556,9589-9590,9595,9600,9602-9603,9613,9620,9624,9627,9642,9654,9662,9664,9666,9673,9680,9686,9691-9692,9694,9696,9708,9723,9728,9731,9740,9744,9772,9777,9779-9780,9792,9796,9802,9804,9806,9810-9811,9824,9836-9837,9973,9975,9985,9996,9999,10002,10006,10009,10011,10024,10026,10033,10041,10043,10045,10052-10053,10055,10060,10062,10065,10067-10068,10070,10082,10084,10094-10095,10171,10173,10180,10182,10185,10233,10249,10255,10264,10281,10285,10287,10319,10338,10409,10446,10456,10513,10531,10694,10764,10849,11395,11561,11563,11565,11570,11572,11578-11579,11582,11584,11587,11661,11719,11763"}}
+{"timestamp":1712890725.8199503,"name":"online","context":{"idset":"92,110,197,208,243,281,285,291,319,329,392,480,486,7958,7970,9009,9019,9022,9025,9036-9037,9060,9062,9066,9524,9529,9538,9573,9591,9601,9606,9614-9615,9649,9653,9656,9674,9718,9750,9752-9753,9768,9800,9807,9812,9815,9822,9977,10005,10007,10051,10061,10069,10074,10093,11557"}}
+{"timestamp":1712890725.95367,"name":"online","context":{"idset":"54,106,109,124,137,149,164,179,241,293,306,339,363,369,407,429,485,9008,9010,9012,9017,9021,9024,9026,9028,9040-9041,9047,9049,9051,9058-9059,9061,9308,9527,9539,9550,9552,9578,9592-9594,9609,9631,9638,9640,9646,9650,9658,9679,9702,9736,9741-9742,9749,9755,9761,9775-9776,9778,9798,9801,9818,9829,9834,9838,9842,9986,9990,9994,9998,10017,10019,10089,10096"}}
+{"timestamp":1712890726.0598288,"name":"online","context":{"idset":"113,116,191,219,226,280,320,325,330,352,360,372,404,418,9023,9038,9044,9056-9057,9065,9068-9069,9074,9525,9540,9544,9547,9581-9582,9596,9604-9605,9623,9630,9635-9637,9655,9668,9676,9689,9705,9707,9720,9722,9730,9732,9734,9738,9743,9748,9754,9756,9760,9763,9765,9769-9770,9793,9795,9803,9813,9816,9826-9828,9839,9841,9843-9844,9978,9980,10001,10003-10004,10010,10016,10018,10020-10021,10031,10034,10038,10040,10048-10050,10054,10057,10073,10076-10077,10080,10087"}}
+{"timestamp":1712890726.1808219,"name":"online","context":{"idset":"56-57,60,9007,9032,9471,9531,9536,9579,9617,9619,9625,9644,9647-9648,9657,9661,9677,9693,9695,9701,9704,9719,9729,9733,9764,9766-9767,9773,9809,9819,9833,9840,9981,10008,10012-10013,10015,10023,10025,10035-10036,10042,10046,10066,10075,10079,10088,10091"}}
+{"timestamp":1712890726.3079088,"name":"online","context":{"idset":"9034,9052,9063,9070,9295,9472,9528,9548,9639,9681,9709,9758,9784,9817,9987-9988,10027"}}
+{"timestamp":1712890726.4453192,"name":"online","context":{"idset":"9014,9031,9071,9541,9577,9724,9735,9745,9762,9785,9814,10030"}}
+{"timestamp":1712890726.5699162,"name":"online","context":{"idset":"9020,9042,9543,9574,9580,9584-9585,9612,9621,9632,9659,9671,9675,9690,9697,9797,9820,9825,9976,9982,9989,9997,10028,10063,10081"}}
+{"timestamp":1712890726.6927462,"name":"online","context":{"idset":"9073,9575,9583,9607-9608,9641,9645,9663,9667,9688,9746,9771,9830"}}
+{"timestamp":1712890726.8241017,"name":"online","context":{"idset":"9551,9665"}}
+{"timestamp":1712890812.1243975,"name":"online","context":{"idset":"10086"}}
+{"timestamp":1712891288.5170541,"name":"online","context":{"idset":"10751,11851"}}
+{"timestamp":1712891288.6390669,"name":"online","context":{"idset":"11831,11879"}}
+{"timestamp":1712891288.8952506,"name":"online","context":{"idset":"10769,10777,10787,11768,11782,11855"}}
+{"timestamp":1712892043.6406412,"name":"offline","context":{"idset":"11765"}}
+{"timestamp":1712892043.9105759,"name":"offline","context":{"idset":"11771"}}
+{"timestamp":1712892043.9212465,"name":"offline","context":{"idset":"11779"}}
+{"timestamp":1712892043.957118,"name":"offline","context":{"idset":"11770"}}
+{"timestamp":1712892043.9698319,"name":"offline","context":{"idset":"11776"}}
+{"timestamp":1712892044.0486567,"name":"offline","context":{"idset":"11791"}}
+{"timestamp":1712892044.1695707,"name":"offline","context":{"idset":"11772"}}
+{"timestamp":1712892044.1999516,"name":"offline","context":{"idset":"11788"}}
+{"timestamp":1712892044.215101,"name":"offline","context":{"idset":"11802"}}
+{"timestamp":1712892044.231339,"name":"offline","context":{"idset":"11767"}}
+{"timestamp":1712892044.2900484,"name":"offline","context":{"idset":"11832"}}
+{"timestamp":1712892044.3062239,"name":"offline","context":{"idset":"11766"}}
+{"timestamp":1712892044.3200986,"name":"offline","context":{"idset":"11768"}}
+{"timestamp":1712892044.329668,"name":"offline","context":{"idset":"11769"}}
+{"timestamp":1712892044.3346167,"name":"offline","context":{"idset":"11777"}}
+{"timestamp":1712892044.3509626,"name":"offline","context":{"idset":"11780"}}
+{"timestamp":1712892044.378752,"name":"offline","context":{"idset":"11784"}}
+{"timestamp":1712892044.400506,"name":"offline","context":{"idset":"11785"}}
+{"timestamp":1712892044.4036145,"name":"offline","context":{"idset":"11792"}}
+{"timestamp":1712892044.4151194,"name":"offline","context":{"idset":"11793"}}
+{"timestamp":1712892044.4175334,"name":"offline","context":{"idset":"11797"}}
+{"timestamp":1712892044.4427547,"name":"offline","context":{"idset":"11801"}}
+{"timestamp":1712892044.450892,"name":"offline","context":{"idset":"11803"}}
+{"timestamp":1712892044.4593065,"name":"offline","context":{"idset":"11814"}}
+{"timestamp":1712892044.4611163,"name":"offline","context":{"idset":"11816"}}
+{"timestamp":1712892044.4766707,"name":"offline","context":{"idset":"11854"}}
+{"timestamp":1712892044.5212538,"name":"offline","context":{"idset":"11774"}}
+{"timestamp":1712892044.5360746,"name":"offline","context":{"idset":"11778"}}
+{"timestamp":1712892044.5507858,"name":"offline","context":{"idset":"11781"}}
+{"timestamp":1712892044.5530496,"name":"offline","context":{"idset":"11783"}}
+{"timestamp":1712892044.5652285,"name":"offline","context":{"idset":"11789"}}
+{"timestamp":1712892044.5739026,"name":"offline","context":{"idset":"11794"}}
+{"timestamp":1712892044.587126,"name":"offline","context":{"idset":"11812"}}
+{"timestamp":1712892044.6004508,"name":"offline","context":{"idset":"11813"}}
+{"timestamp":1712892044.6022732,"name":"offline","context":{"idset":"11825"}}
+{"timestamp":1712892044.6040797,"name":"offline","context":{"idset":"11829"}}
+{"timestamp":1712892044.6077816,"name":"offline","context":{"idset":"11775"}}
+{"timestamp":1712892044.6877272,"name":"offline","context":{"idset":"11782"}}
+{"timestamp":1712892044.6895955,"name":"offline","context":{"idset":"11786"}}
+{"timestamp":1712892044.7020214,"name":"offline","context":{"idset":"11790"}}
+{"timestamp":1712892044.7163138,"name":"offline","context":{"idset":"11795"}}
+{"timestamp":1712892044.7182889,"name":"offline","context":{"idset":"11809"}}
+{"timestamp":1712892044.7324126,"name":"offline","context":{"idset":"11820"}}
+{"timestamp":1712892044.7485189,"name":"offline","context":{"idset":"11841"}}
+{"timestamp":1712892044.7719097,"name":"offline","context":{"idset":"11847"}}
+{"timestamp":1712892044.791527,"name":"offline","context":{"idset":"11849"}}
+{"timestamp":1712892044.8279879,"name":"offline","context":{"idset":"11773"}}
+{"timestamp":1712892044.8640101,"name":"offline","context":{"idset":"11798"}}
+{"timestamp":1712892044.8670337,"name":"offline","context":{"idset":"11800"}}
+{"timestamp":1712892044.8700516,"name":"offline","context":{"idset":"11804"}}
+{"timestamp":1712892044.8730354,"name":"offline","context":{"idset":"11805"}}
+{"timestamp":1712892044.9653637,"name":"offline","context":{"idset":"11806"}}
+{"timestamp":1712892044.9876914,"name":"offline","context":{"idset":"11810"}}
+{"timestamp":1712892044.99123,"name":"offline","context":{"idset":"11811"}}
+{"timestamp":1712892045.0284777,"name":"offline","context":{"idset":"11817"}}
+{"timestamp":1712892045.0657139,"name":"offline","context":{"idset":"11821"}}
+{"timestamp":1712892045.1020315,"name":"offline","context":{"idset":"11823"}}
+{"timestamp":1712892045.1378927,"name":"offline","context":{"idset":"11828"}}
+{"timestamp":1712892045.1558969,"name":"offline","context":{"idset":"11833"}}
+{"timestamp":1712892045.1828022,"name":"offline","context":{"idset":"11835"}}
+{"timestamp":1712892045.2046359,"name":"offline","context":{"idset":"11837"}}
+{"timestamp":1712892045.2264483,"name":"offline","context":{"idset":"11848"}}
+{"timestamp":1712892045.24822,"name":"offline","context":{"idset":"11850"}}
+{"timestamp":1712892045.2705519,"name":"offline","context":{"idset":"11853"}}
+{"timestamp":1712892045.2738237,"name":"offline","context":{"idset":"11858"}}
+{"timestamp":1712892045.2768216,"name":"offline","context":{"idset":"11860"}}
+{"timestamp":1712892045.2799096,"name":"offline","context":{"idset":"11865"}}
+{"timestamp":1712892045.2948098,"name":"offline","context":{"idset":"11866"}}
+{"timestamp":1712892045.3403261,"name":"offline","context":{"idset":"11787"}}
+{"timestamp":1712892045.3761985,"name":"offline","context":{"idset":"11796"}}
+{"timestamp":1712892045.4155002,"name":"offline","context":{"idset":"11807"}}
+{"timestamp":1712892045.4499526,"name":"offline","context":{"idset":"11808"}}
+{"timestamp":1712892045.4668958,"name":"offline","context":{"idset":"11818"}}
+{"timestamp":1712892045.4698532,"name":"offline","context":{"idset":"11819"}}
+{"timestamp":1712892045.4840322,"name":"offline","context":{"idset":"11822"}}
+{"timestamp":1712892045.5013266,"name":"offline","context":{"idset":"11824"}}
+{"timestamp":1712892045.5184829,"name":"offline","context":{"idset":"11827"}}
+{"timestamp":1712892045.5349956,"name":"offline","context":{"idset":"11830"}}
+{"timestamp":1712892045.5620515,"name":"offline","context":{"idset":"11834"}}
+{"timestamp":1712892045.5931251,"name":"offline","context":{"idset":"11836"}}
+{"timestamp":1712892045.6256266,"name":"offline","context":{"idset":"11839"}}
+{"timestamp":1712892045.6520441,"name":"offline","context":{"idset":"11842"}}
+{"timestamp":1712892045.6776938,"name":"offline","context":{"idset":"11843"}}
+{"timestamp":1712892045.702668,"name":"offline","context":{"idset":"11844"}}
+{"timestamp":1712892045.7271698,"name":"offline","context":{"idset":"11846"}}
+{"timestamp":1712892045.7305033,"name":"offline","context":{"idset":"11851"}}
+{"timestamp":1712892045.7338018,"name":"offline","context":{"idset":"11852"}}
+{"timestamp":1712892045.7369821,"name":"offline","context":{"idset":"11856"}}
+{"timestamp":1712892045.7640743,"name":"offline","context":{"idset":"11857"}}
+{"timestamp":1712892045.7865939,"name":"offline","context":{"idset":"11859"}}
+{"timestamp":1712892045.8167608,"name":"offline","context":{"idset":"11861"}}
+{"timestamp":1712892045.8198047,"name":"offline","context":{"idset":"11862"}}
+{"timestamp":1712892045.8317804,"name":"offline","context":{"idset":"11863"}}
+{"timestamp":1712892045.8464916,"name":"offline","context":{"idset":"11867"}}
+{"timestamp":1712892045.8599994,"name":"offline","context":{"idset":"11868"}}
+{"timestamp":1712892045.8747506,"name":"offline","context":{"idset":"11870"}}
+{"timestamp":1712892045.8879311,"name":"offline","context":{"idset":"11871"}}
+{"timestamp":1712892045.9008791,"name":"offline","context":{"idset":"11872"}}
+{"timestamp":1712892045.9037175,"name":"offline","context":{"idset":"11873"}}
+{"timestamp":1712892045.9139373,"name":"offline","context":{"idset":"11874"}}
+{"timestamp":1712892045.9355521,"name":"offline","context":{"idset":"11875"}}
+{"timestamp":1712892045.9563937,"name":"offline","context":{"idset":"11876"}}
+{"timestamp":1712892045.976367,"name":"offline","context":{"idset":"11877"}}
+{"timestamp":1712892045.9961472,"name":"offline","context":{"idset":"11880"}}
+{"timestamp":1712892046.0155699,"name":"offline","context":{"idset":"11881"}}
+{"timestamp":1712892046.0347517,"name":"offline","context":{"idset":"11882"}}
+{"timestamp":1712892046.0532773,"name":"offline","context":{"idset":"11883"}}
+{"timestamp":1712892046.0714791,"name":"offline","context":{"idset":"11884"}}
+{"timestamp":1712892046.0894382,"name":"offline","context":{"idset":"11885"}}
+{"timestamp":1712892046.0918188,"name":"offline","context":{"idset":"11886"}}
+{"timestamp":1712892046.0942287,"name":"offline","context":{"idset":"11887"}}
+{"timestamp":1712892046.0966368,"name":"offline","context":{"idset":"11888"}}
+{"timestamp":1712892046.0990138,"name":"offline","context":{"idset":"11889"}}
+{"timestamp":1712892046.107259,"name":"offline","context":{"idset":"11890"}}
+{"timestamp":1712892046.1186256,"name":"offline","context":{"idset":"11891"}}
+{"timestamp":1712892046.1209309,"name":"offline","context":{"idset":"11799"}}
+{"timestamp":1712892046.1300447,"name":"offline","context":{"idset":"11815"}}
+{"timestamp":1712892046.1413541,"name":"offline","context":{"idset":"11826"}}
+{"timestamp":1712892046.1526392,"name":"offline","context":{"idset":"11831"}}
+{"timestamp":1712892046.1631987,"name":"offline","context":{"idset":"11838"}}
+{"timestamp":1712892046.1734426,"name":"offline","context":{"idset":"11840"}}
+{"timestamp":1712892046.1836169,"name":"offline","context":{"idset":"11845"}}
+{"timestamp":1712892046.1858225,"name":"offline","context":{"idset":"11855"}}
+{"timestamp":1712892046.1938715,"name":"offline","context":{"idset":"11864"}}
+{"timestamp":1712892046.2039189,"name":"offline","context":{"idset":"11869"}}
+{"timestamp":1712892046.2061086,"name":"offline","context":{"idset":"11878"}}
+{"timestamp":1712892046.2139771,"name":"offline","context":{"idset":"11879"}}
+{"timestamp":1712892046.2239451,"name":"offline","context":{"idset":"11892"}}
+{"timestamp":1712893667.272445,"name":"online","context":{"idset":"10742"}}
+{"timestamp":1712894036.4446051,"name":"online","context":{"idset":"11786,11788"}}
+{"timestamp":1712894036.5882487,"name":"online","context":{"idset":"11765"}}
+{"timestamp":1712894036.7359669,"name":"online","context":{"idset":"11766,11771"}}
+{"timestamp":1712894036.9270499,"name":"online","context":{"idset":"11774,11778"}}
+{"timestamp":1712894037.0897801,"name":"online","context":{"idset":"11791,11810"}}
+{"timestamp":1712894037.2281969,"name":"online","context":{"idset":"11767,11777,11781,11783,11801"}}
+{"timestamp":1712894037.4340751,"name":"online","context":{"idset":"11785"}}
+{"timestamp":1712894037.5362983,"name":"online","context":{"idset":"11787"}}
+{"timestamp":1712894037.6838953,"name":"online","context":{"idset":"11780,11799"}}
+{"timestamp":1712894037.9114687,"name":"online","context":{"idset":"11798"}}
+{"timestamp":1712894038.1740556,"name":"online","context":{"idset":"11832"}}
+{"timestamp":1712894038.3392978,"name":"drain","context":{"idset":"11812","reason":"broker was unresponsive"}}
+{"timestamp":1712894038.3394063,"name":"drain","context":{"idset":"11834","reason":"broker was unresponsive"}}
+{"timestamp":1712894038.3394823,"name":"drain","context":{"idset":"11842","reason":"broker was unresponsive"}}
+{"timestamp":1712894038.4422846,"name":"drain","context":{"idset":"11887","reason":"broker was unresponsive"}}
+{"timestamp":1712894038.4457455,"name":"online","context":{"idset":"11779"}}
+{"timestamp":1712894038.6176107,"name":"online","context":{"idset":"11822"}}
+{"timestamp":1712894038.8270147,"name":"online","context":{"idset":"11875"}}
+{"timestamp":1712894038.8562953,"name":"online","context":{"idset":"11843"}}
+{"timestamp":1712894038.8779509,"name":"online","context":{"idset":"11769"}}
+{"timestamp":1712894039.0437016,"name":"online","context":{"idset":"11782,11790"}}
+{"timestamp":1712894039.3299584,"name":"online","context":{"idset":"11824"}}
+{"timestamp":1712894039.3628249,"name":"online","context":{"idset":"11773"}}
+{"timestamp":1712894039.5223255,"name":"online","context":{"idset":"11772,11804,11840"}}
+{"timestamp":1712894039.5543883,"name":"online","context":{"idset":"11818"}}
+{"timestamp":1712894039.7427926,"name":"online","context":{"idset":"11830"}}
+{"timestamp":1712894039.8639898,"name":"online","context":{"idset":"11793,11797,11821"}}
+{"timestamp":1712894039.9875333,"name":"online","context":{"idset":"11776,11795,11816,11829"}}
+{"timestamp":1712894040.1081722,"name":"online","context":{"idset":"11858,11867,11880,11888"}}
+{"timestamp":1712894040.2115948,"name":"online","context":{"idset":"11784"}}
+{"timestamp":1712894040.3473594,"name":"online","context":{"idset":"11770,11796,11819"}}
+{"timestamp":1712894040.584641,"name":"online","context":{"idset":"11768,11775,11792,11794,11800,11809,11817,11825,11839,11842,11844,11855,11860,11868,11879,11887"}}
+{"timestamp":1712894040.7324433,"name":"online","context":{"idset":"11850"}}
+{"timestamp":1712894040.9618223,"name":"online","context":{"idset":"11802-11803,11805-11806,11812,11814-11815,11820,11827-11828,11831,11833-11834,11836,11846,11853,11857,11864,11871,11881,11886,11892"}}
+{"timestamp":1712894041.1133864,"name":"online","context":{"idset":"11807-11808,11826,11835,11837,11847,11859,11869,11872,11878,11882"}}
+{"timestamp":1712894041.249728,"name":"online","context":{"idset":"11811,11851-11852,11863,11865-11866"}}
+{"timestamp":1712894041.3799455,"name":"online","context":{"idset":"11813,11845,11854,11856,11870,11889"}}
+{"timestamp":1712894041.4835017,"name":"online","context":{"idset":"11849,11861-11862,11873-11874,11876,11883,11885"}}
+{"timestamp":1712894041.6063693,"name":"online","context":{"idset":"11848"}}
+{"timestamp":1712894041.7202885,"name":"online","context":{"idset":"11838,11841,11877,11891"}}
+{"timestamp":1712894042.0018647,"name":"online","context":{"idset":"11884,11890"}}
+{"timestamp":1712894222.2665682,"name":"online","context":{"idset":"11789"}}
+{"timestamp":1712894285.1276972,"name":"online","context":{"idset":"11823"}}
+{"timestamp":1712894353.7952852,"name":"undrain","context":{"idset":"11812,11834,11842,11887"}}
+{"timestamp":1712912800.4405386,"name":"offline","context":{"idset":"7928"}}
+{"timestamp":1712929218.4364753,"name":"offline","context":{"idset":"9615"}}
+{"timestamp":1712930998.0441911,"name":"online","context":{"idset":"10396"}}
+{"timestamp":1712931203.1639795,"name":"online","context":{"idset":"10658"}}
+{"timestamp":1712931204.5058937,"name":"online","context":{"idset":"10657"}}
+{"timestamp":1712931240.3506067,"name":"offline","context":{"idset":"797"}}
+{"timestamp":1712931240.454906,"name":"offline","context":{"idset":"798"}}
+{"timestamp":1712931382.9980457,"name":"online","context":{"idset":"10702"}}
+{"timestamp":1712931412.4241462,"name":"online","context":{"idset":"9564"}}
+{"timestamp":1712931412.6234269,"name":"online","context":{"idset":"9569"}}
+{"timestamp":1712931412.8331361,"name":"online","context":{"idset":"9560,9563,9565,9567"}}
+{"timestamp":1712931413.0299804,"name":"online","context":{"idset":"9557,9566,9571"}}
+{"timestamp":1712931413.1531873,"name":"online","context":{"idset":"9558-9559,9561-9562"}}
+{"timestamp":1712931413.3616512,"name":"online","context":{"idset":"9568,9570,9572"}}
+{"timestamp":1712931420.7055428,"name":"online","context":{"idset":"10712"}}
+{"timestamp":1712931421.2159173,"name":"online","context":{"idset":"10718"}}
+{"timestamp":1712931458.703382,"name":"online","context":{"idset":"10739"}}
+{"timestamp":1712931459.3637524,"name":"online","context":{"idset":"10740"}}
+{"timestamp":1712931464.4360983,"name":"offline","context":{"idset":"7927"}}
+{"timestamp":1712931493.4967601,"name":"drain","context":{"idset":"7927","reason":"Parter node failure","overwrite":0}}
+{"timestamp":1712931722.3486252,"name":"offline","context":{"idset":"7925"}}
+{"timestamp":1712931722.4361744,"name":"offline","context":{"idset":"7926"}}
+{"timestamp":1712931886.0126631,"name":"online","context":{"idset":"9535"}}
+{"timestamp":1712932087.9197545,"name":"online","context":{"idset":"9781"}}
+{"timestamp":1712932088.1433012,"name":"online","context":{"idset":"9782"}}
+{"timestamp":1712932878.1914799,"name":"online","context":{"idset":"227"}}
+{"timestamp":1712932904.6138883,"name":"undrain","context":{"idset":"10869-10872,10875-10892,10894-10910,10913-10933,10935-10947,10949-10981,10983-10996,11573-11575,11577-11588"}}
+{"timestamp":1712932916.5126393,"name":"online","context":{"idset":"399"}}
+{"timestamp":1712932928.211103,"name":"online","context":{"idset":"409"}}
+{"timestamp":1712932947.1588495,"name":"online","context":{"idset":"484"}}
+{"timestamp":1712932955.0749447,"name":"online","context":{"idset":"332"}}
+{"timestamp":1712932970.7510486,"name":"online","context":{"idset":"210"}}
+{"timestamp":1712932974.2511892,"name":"online","context":{"idset":"180"}}
+{"timestamp":1712932983.8654547,"name":"online","context":{"idset":"385"}}
+{"timestamp":1712932984.1526551,"name":"online","context":{"idset":"190"}}
+{"timestamp":1712933020.6351442,"name":"online","context":{"idset":"194"}}
+{"timestamp":1712933230.4367785,"name":"offline","context":{"idset":"8048"}}
+{"timestamp":1712933581.0146523,"name":"online","context":{"idset":"10099"}}
+{"timestamp":1712933581.2659774,"name":"online","context":{"idset":"10100"}}
+{"timestamp":1712933674.6849163,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1712933741.3686047,"name":"drain","context":{"idset":"10669,10671,10717","overwrite":0}}
+{"timestamp":1712933746.5863693,"name":"undrain","context":{"idset":"10669,10671,10717"}}
+{"timestamp":1712933982.9094241,"name":"online","context":{"idset":"7964"}}
+{"timestamp":1712934229.1663623,"name":"drain","context":{"idset":"10706","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712934236.817574,"name":"online","context":{"idset":"768"}}
+{"timestamp":1712934246.443306,"name":"drain","context":{"idset":"10669,10671,10717","overwrite":0}}
+{"timestamp":1712934265.4968991,"name":"undrain","context":{"idset":"10669,10671,10717"}}
+{"timestamp":1712934269.7108614,"name":"drain","context":{"idset":"10738","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712934272.9475808,"name":"drain","context":{"idset":"10734","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712934290.4371474,"name":"offline","context":{"idset":"9017"}}
+{"timestamp":1712934354.750489,"name":"undrain","context":{"idset":"767-768"}}
+{"timestamp":1712934400.1105566,"name":"online","context":{"idset":"8048"}}
+{"timestamp":1712934497.3982587,"name":"undrain","context":{"idset":"8048"}}
+{"timestamp":1712934666.3478515,"name":"offline","context":{"idset":"841"}}
+{"timestamp":1712934666.4377928,"name":"offline","context":{"idset":"842"}}
+{"timestamp":1712934698.2307673,"name":"online","context":{"idset":"767"}}
+{"timestamp":1712934947.9139359,"name":"offline","context":{"idset":"767"}}
+{"timestamp":1712935037.7961144,"name":"online","context":{"idset":"767"}}
+{"timestamp":1712935710.9566383,"name":"online","context":{"idset":"843"}}
+{"timestamp":1712935740.6774743,"name":"online","context":{"idset":"844"}}
+{"timestamp":1712935801.321161,"name":"online","context":{"idset":"7925"}}
+{"timestamp":1712935801.5435617,"name":"online","context":{"idset":"7926"}}
+{"timestamp":1712936128.4346333,"name":"offline","context":{"idset":"872"}}
+{"timestamp":1712936178.4358041,"name":"offline","context":{"idset":"871"}}
+{"timestamp":1712937130.5036509,"name":"online","context":{"idset":"873"}}
+{"timestamp":1712937150.445019,"name":"online","context":{"idset":"874"}}
+{"timestamp":1712937334.3470705,"name":"offline","context":{"idset":"873"}}
+{"timestamp":1712937334.4377003,"name":"offline","context":{"idset":"874"}}
+{"timestamp":1712937640.4346528,"name":"offline","context":{"idset":"10997"}}
+{"timestamp":1712937644.4371164,"name":"offline","context":{"idset":"11000"}}
+{"timestamp":1712937648.341697,"name":"offline","context":{"idset":"11001"}}
+{"timestamp":1712937648.434911,"name":"offline","context":{"idset":"11002"}}
+{"timestamp":1712937649.4000404,"name":"offline","context":{"idset":"11004"}}
+{"timestamp":1712937649.5342183,"name":"offline","context":{"idset":"11003"}}
+{"timestamp":1712937654.3554332,"name":"offline","context":{"idset":"11005"}}
+{"timestamp":1712937654.4439454,"name":"offline","context":{"idset":"11006"}}
+{"timestamp":1712937658.3488057,"name":"offline","context":{"idset":"11007"}}
+{"timestamp":1712937658.4391358,"name":"offline","context":{"idset":"11008"}}
+{"timestamp":1712937660.3459318,"name":"offline","context":{"idset":"11009"}}
+{"timestamp":1712937660.4361966,"name":"offline","context":{"idset":"11010"}}
+{"timestamp":1712937662.3455145,"name":"offline","context":{"idset":"11011"}}
+{"timestamp":1712937662.4360192,"name":"offline","context":{"idset":"11012"}}
+{"timestamp":1712937690.4356599,"name":"offline","context":{"idset":"10998"}}
+{"timestamp":1712938506.1929417,"name":"online","context":{"idset":"874"}}
+{"timestamp":1712938506.318944,"name":"online","context":{"idset":"873"}}
+{"timestamp":1712938662.4360197,"name":"offline","context":{"idset":"11319"}}
+{"timestamp":1712938700.3446288,"name":"offline","context":{"idset":"10631"}}
+{"timestamp":1712938700.4358239,"name":"offline","context":{"idset":"10632"}}
+{"timestamp":1712938848.3441279,"name":"offline","context":{"idset":"761"}}
+{"timestamp":1712938848.4351685,"name":"offline","context":{"idset":"762"}}
+{"timestamp":1712938922.4370465,"name":"offline","context":{"idset":"11317"}}
+{"timestamp":1712939015.2720578,"name":"drain","context":{"idset":"481","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712939033.9870858,"name":"drain","context":{"idset":"478","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712939037.9081478,"name":"drain","context":{"idset":"477","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712939042.0713556,"name":"drain","context":{"idset":"480","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712939056.9337916,"name":"drain","context":{"idset":"479","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712939468.8921432,"name":"online","context":{"idset":"11514"}}
+{"timestamp":1712940064.4361103,"name":"offline","context":{"idset":"9253"}}
+{"timestamp":1712940508.2201235,"name":"drain","context":{"idset":"7988","overwrite":0}}
+{"timestamp":1712940532.8652604,"name":"drain","context":{"idset":"7987","reason":"parter node error","overwrite":0}}
+{"timestamp":1712940563.0709505,"name":"offline","context":{"idset":"7987"}}
+{"timestamp":1712941604.4370754,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1712941615.6237082,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1712941678.7218478,"name":"offline","context":{"idset":"11329"}}
+{"timestamp":1712941978.2994287,"name":"undrain","context":{"idset":"10541-10542"}}
+{"timestamp":1712942103.0553768,"name":"drain","context":{"idset":"361,420","reason":"epilog failed for jobid fp6az3kDFZy","overwrite":0}}
+{"timestamp":1712942376.9929025,"name":"online","context":{"idset":"10669"}}
+{"timestamp":1712942565.3305595,"name":"online","context":{"idset":"881-882"}}
+{"timestamp":1712942652.2362139,"name":"online","context":{"idset":"11424"}}
+{"timestamp":1712942657.4291172,"name":"online","context":{"idset":"11423"}}
+{"timestamp":1712942807.3987277,"name":"online","context":{"idset":"11621"}}
+{"timestamp":1712942807.6968157,"name":"online","context":{"idset":"11542"}}
+{"timestamp":1712942810.3757529,"name":"online","context":{"idset":"11568"}}
+{"timestamp":1712943014.1144056,"name":"online","context":{"idset":"10671"}}
+{"timestamp":1712943045.1758368,"name":"online","context":{"idset":"10717"}}
+{"timestamp":1712943540.0513546,"name":"online","context":{"idset":"10631"}}
+{"timestamp":1712943783.7637284,"name":"undrain","context":{"idset":"11576"}}
+{"timestamp":1712943923.583056,"name":"online","context":{"idset":"10632"}}
+{"timestamp":1712945030.4349699,"name":"offline","context":{"idset":"10086"}}
+{"timestamp":1712945258.6365204,"name":"online","context":{"idset":"971"}}
+{"timestamp":1712945258.8389013,"name":"online","context":{"idset":"851-852,972"}}
+{"timestamp":1712945259.0678244,"name":"online","context":{"idset":"850"}}
+{"timestamp":1712945259.2019374,"name":"online","context":{"idset":"849"}}
+{"timestamp":1712945395.4889505,"name":"drain","context":{"idset":"854,10706,10734","reason":"Debugging downed nodes - vls","overwrite":1}}
+{"timestamp":1712945459.5096622,"name":"undrain","context":{"idset":"10706,10734,10738"}}
+{"timestamp":1712945625.0302892,"name":"online","context":{"idset":"9992"}}
+{"timestamp":1712945625.6319995,"name":"online","context":{"idset":"9991"}}
+{"timestamp":1712946110.744539,"name":"undrain","context":{"idset":"9557-9572"}}
+{"timestamp":1712946284.4381227,"name":"offline","context":{"idset":"815"}}
+{"timestamp":1712946310.4399183,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1712946332.440007,"name":"offline","context":{"idset":"816"}}
+{"timestamp":1712946518.0625284,"name":"drain","context":{"idset":"11562","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946521.7504351,"name":"drain","context":{"idset":"11563","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946531.9203229,"name":"drain","context":{"idset":"11572","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946536.8803725,"name":"drain","context":{"idset":"11557","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946537.3792932,"name":"drain","context":{"idset":"11558","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946541.4257057,"name":"drain","context":{"idset":"11560","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946545.3991642,"name":"drain","context":{"idset":"11564","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946546.6530943,"name":"drain","context":{"idset":"11570","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946553.0778487,"name":"drain","context":{"idset":"11567","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946555.6685538,"name":"drain","context":{"idset":"11565","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946563.4115796,"name":"drain","context":{"idset":"11566","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712946568.8431597,"name":"drain","context":{"idset":"11561","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947180.32444,"name":"online","context":{"idset":"9615"}}
+{"timestamp":1712947214.3441076,"name":"online","context":{"idset":"9616"}}
+{"timestamp":1712947381.1300602,"name":"undrain","context":{"idset":"11557-11558,11560-11567,11570,11572"}}
+{"timestamp":1712947625.9596472,"name":"drain","context":{"idset":"11562","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947626.3782065,"name":"drain","context":{"idset":"11557","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947631.3930502,"name":"drain","context":{"idset":"11563","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947639.0167303,"name":"drain","context":{"idset":"11558","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947640.7394433,"name":"drain","context":{"idset":"11560","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947643.924407,"name":"drain","context":{"idset":"11565","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947644.4983435,"name":"drain","context":{"idset":"11572","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947644.647804,"name":"drain","context":{"idset":"11564","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947644.7980826,"name":"drain","context":{"idset":"11570","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947644.9674342,"name":"drain","context":{"idset":"11567","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947647.9040687,"name":"online","context":{"idset":"10356"}}
+{"timestamp":1712947649.0531714,"name":"drain","context":{"idset":"11566","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947651.6311378,"name":"drain","context":{"idset":"11561","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712947653.0740192,"name":"online","context":{"idset":"10355"}}
+{"timestamp":1712947690.2701061,"name":"online","context":{"idset":"769"}}
+{"timestamp":1712947801.4375491,"name":"undrain","context":{"idset":"10355"}}
+{"timestamp":1712948305.0965281,"name":"undrain","context":{"idset":"11557-11558,11560-11567,11570,11572"}}
+{"timestamp":1712948388.895031,"name":"drain","context":{"idset":"11577","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948396.5798891,"name":"online","context":{"idset":"8981,9067,9414,9634"}}
+{"timestamp":1712948396.7913983,"name":"online","context":{"idset":"9297,9356,9511,9518,9586,9598,9721"}}
+{"timestamp":1712948396.9046845,"name":"online","context":{"idset":"9283,9434,9477,9611,9717"}}
+{"timestamp":1712948397.0258391,"name":"online","context":{"idset":"9016,9542,9672,9703"}}
+{"timestamp":1712948397.4809082,"name":"drain","context":{"idset":"11580","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948400.8566289,"name":"drain","context":{"idset":"11582","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948408.5236993,"name":"drain","context":{"idset":"11585","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948409.4576666,"name":"drain","context":{"idset":"11583","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948410.4020586,"name":"drain","context":{"idset":"11581","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948413.1531005,"name":"drain","context":{"idset":"11587","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948413.3445218,"name":"drain","context":{"idset":"11584","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948416.6440544,"name":"drain","context":{"idset":"11579","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948417.081764,"name":"drain","context":{"idset":"11574","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948423.5872767,"name":"drain","context":{"idset":"11578","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948423.7451758,"name":"drain","context":{"idset":"11588","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948427.0906992,"name":"drain","context":{"idset":"11568","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948448.6510158,"name":"drain","context":{"idset":"11573","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712948478.3470473,"name":"offline","context":{"idset":"765"}}
+{"timestamp":1712948478.4388936,"name":"offline","context":{"idset":"766"}}
+{"timestamp":1712948656.4378603,"name":"offline","context":{"idset":"882"}}
+{"timestamp":1712949112.4297297,"name":"online","context":{"idset":"10934"}}
+{"timestamp":1712949112.669378,"name":"online","context":{"idset":"10893,10948"}}
+{"timestamp":1712949112.7914369,"name":"online","context":{"idset":"10873-10874"}}
+{"timestamp":1712949113.3363888,"name":"online","context":{"idset":"10982"}}
+{"timestamp":1712949214.2159472,"name":"undrain","context":{"idset":"10873-10874,10893,10934,10948,10982"}}
+{"timestamp":1712950220.1828775,"name":"online","context":{"idset":"11640,11644"}}
+{"timestamp":1712950220.5150523,"name":"online","context":{"idset":"11643,11649"}}
+{"timestamp":1712950220.7406564,"name":"online","context":{"idset":"11638,11641,11651"}}
+{"timestamp":1712950220.8784792,"name":"online","context":{"idset":"11639,11648,11650"}}
+{"timestamp":1712950221.0913999,"name":"online","context":{"idset":"11642,11647"}}
+{"timestamp":1712950221.3023062,"name":"online","context":{"idset":"11645-11646"}}
+{"timestamp":1712950221.4197576,"name":"online","context":{"idset":"11652"}}
+{"timestamp":1712950221.7954736,"name":"online","context":{"idset":"11637"}}
+{"timestamp":1712950327.9362206,"name":"online","context":{"idset":"10344"}}
+{"timestamp":1712950331.5369172,"name":"online","context":{"idset":"10343"}}
+{"timestamp":1712950843.2379835,"name":"online","context":{"idset":"9588"}}
+{"timestamp":1712950843.376554,"name":"online","context":{"idset":"9587"}}
+{"timestamp":1712951213.144124,"name":"undrain","context":{"idset":"9587-9588"}}
+{"timestamp":1712951637.6165602,"name":"drain","context":{"idset":"11637-11652","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1712951786.0432136,"name":"undrain","context":{"idset":"11637-11652"}}
+{"timestamp":1712952446.9003825,"name":"undrain","context":{"idset":"11568,11573-11574,11577-11585,11587-11588"}}
+{"timestamp":1712953157.4248369,"name":"online","context":{"idset":"770"}}
+{"timestamp":1712953215.5812068,"name":"undrain","context":{"idset":"769-770"}}
+{"timestamp":1712953496.4339502,"name":"drain","context":{"idset":"11562","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953500.7439325,"name":"drain","context":{"idset":"11557","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953510.0802593,"name":"drain","context":{"idset":"11558","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953510.5810142,"name":"drain","context":{"idset":"11572","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953512.7092385,"name":"drain","context":{"idset":"11563","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953522.1002409,"name":"drain","context":{"idset":"11564","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953525.9290047,"name":"drain","context":{"idset":"11567","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953526.0856488,"name":"drain","context":{"idset":"11570","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953529.117943,"name":"drain","context":{"idset":"11565","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953529.9858813,"name":"drain","context":{"idset":"11566","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953530.8933868,"name":"drain","context":{"idset":"11581","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953531.1781602,"name":"drain","context":{"idset":"11583","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953531.3784466,"name":"drain","context":{"idset":"11584","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953533.4664888,"name":"drain","context":{"idset":"11561","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953536.2212386,"name":"drain","context":{"idset":"11568","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953538.9321792,"name":"drain","context":{"idset":"11580","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953539.7299294,"name":"drain","context":{"idset":"11588","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953539.913888,"name":"drain","context":{"idset":"11560","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953541.2107675,"name":"drain","context":{"idset":"11579","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953541.8604009,"name":"drain","context":{"idset":"11587","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953542.2388783,"name":"drain","context":{"idset":"11574","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953542.7601724,"name":"drain","context":{"idset":"11585","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953545.9583356,"name":"drain","context":{"idset":"11573","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953546.4172895,"name":"drain","context":{"idset":"11578","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953548.6102457,"name":"drain","context":{"idset":"11582","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953550.0592394,"name":"drain","context":{"idset":"11577","reason":"nodediag failed dgemm_perf","overwrite":0}}
+{"timestamp":1712953820.3512211,"name":"offline","context":{"idset":"767"}}
+{"timestamp":1712953820.4417677,"name":"offline","context":{"idset":"768"}}
+{"timestamp":1712954882.3432314,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1712954882.4362848,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1712955344.3522274,"name":"offline","context":{"idset":"10619"}}
+{"timestamp":1712955344.4454703,"name":"offline","context":{"idset":"10620"}}
+{"timestamp":1712955604.4363623,"name":"offline","context":{"idset":"881"}}
+{"timestamp":1712958170.3847344,"name":"offline","context":{"idset":"9526"}}
+{"timestamp":1712958170.4060009,"name":"offline","context":{"idset":"9527"}}
+{"timestamp":1712958170.4278681,"name":"offline","context":{"idset":"9528"}}
+{"timestamp":1712958170.4470124,"name":"offline","context":{"idset":"9529"}}
+{"timestamp":1712958170.469069,"name":"offline","context":{"idset":"9530"}}
+{"timestamp":1712958170.4859354,"name":"offline","context":{"idset":"9531"}}
+{"timestamp":1712958170.507489,"name":"offline","context":{"idset":"9532"}}
+{"timestamp":1712958170.5312002,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1712958170.5514412,"name":"offline","context":{"idset":"9536"}}
+{"timestamp":1712958170.5834444,"name":"offline","context":{"idset":"9537"}}
+{"timestamp":1712958170.6088722,"name":"offline","context":{"idset":"9538"}}
+{"timestamp":1712958170.6292305,"name":"offline","context":{"idset":"9539"}}
+{"timestamp":1712958170.6513281,"name":"offline","context":{"idset":"9540"}}
+{"timestamp":1712958218.4358263,"name":"offline","context":{"idset":"9525"}}
+{"timestamp":1712958258.4079885,"name":"undrain","context":{"idset":"11424"}}
+{"timestamp":1712959979.9354806,"name":"online","context":{"idset":"771"}}
+{"timestamp":1712960279.0864685,"name":"online","context":{"idset":"11575"}}
+{"timestamp":1712960279.2093303,"name":"online","context":{"idset":"11576"}}
+{"timestamp":1712960354.6816046,"name":"undrain","context":{"idset":"11557-11558,11560-11568"}}
+{"timestamp":1712960862.3457475,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1712960862.4379666,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1712960888.3410218,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1712960888.4390202,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1712962470.2024174,"name":"online","context":{"idset":"9912,9933"}}
+{"timestamp":1712962470.3218763,"name":"online","context":{"idset":"9853,9967"}}
+{"timestamp":1712962470.447613,"name":"online","context":{"idset":"9960"}}
+{"timestamp":1712962470.5696571,"name":"online","context":{"idset":"9877,9890"}}
+{"timestamp":1712962470.6984458,"name":"online","context":{"idset":"9888"}}
+{"timestamp":1712963278.3469195,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1712963278.3593991,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1712963278.3755608,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1712963278.454958,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1712963280.3554597,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1712963280.3670223,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1712963280.38293,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1712963280.39517,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1712963280.4065175,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1712963280.4183342,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1712963280.428802,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1712963280.4443173,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1712963378.34726,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1712963378.3594203,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1712963378.370476,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1712963378.4358211,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1712963380.3533087,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1712963380.3719726,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1712963380.3950155,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1712963380.4362962,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1712963398.3540845,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1712963398.3659973,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1712963398.3780124,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1712963398.3901799,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1712963398.4014211,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1712963398.4123042,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1712963398.4232512,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1712963398.4425282,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1712963439.0982606,"name":"online","context":{"idset":"10542"}}
+{"timestamp":1712963439.4148479,"name":"online","context":{"idset":"10541"}}
+{"timestamp":1712963549.6503761,"name":"online","context":{"idset":"11233"}}
+{"timestamp":1712963549.78175,"name":"online","context":{"idset":"11169,11232"}}
+{"timestamp":1712963549.9063108,"name":"online","context":{"idset":"11148,11154,11181,11190,11249"}}
+{"timestamp":1712964494.311043,"name":"online","context":{"idset":"7987"}}
+{"timestamp":1712964494.4286246,"name":"online","context":{"idset":"7988"}}
+{"timestamp":1712964550.9056358,"name":"undrain","context":{"idset":"7987-7988"}}
+{"timestamp":1712964732.6836481,"name":"undrain","context":{"idset":"10086"}}
+{"timestamp":1712964795.9575379,"name":"online","context":{"idset":"10085"}}
+{"timestamp":1712964837.7879162,"name":"online","context":{"idset":"10086"}}
+{"timestamp":1712966168.3090708,"name":"online","context":{"idset":"10001-10002"}}
+{"timestamp":1712966688.3467865,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1712966688.4375901,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1712966724.4387631,"name":"offline","context":{"idset":"805"}}
+{"timestamp":1712966772.4364457,"name":"offline","context":{"idset":"806"}}
+{"timestamp":1712972080.4370613,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1712972924.0058954,"name":"online","context":{"idset":"10212"}}
+{"timestamp":1712972924.1202528,"name":"online","context":{"idset":"10125"}}
+{"timestamp":1712972924.2525365,"name":"online","context":{"idset":"10109-10110"}}
+{"timestamp":1712976392.9553857,"name":"online","context":{"idset":"9253"}}
+{"timestamp":1712977177.914119,"name":"online","context":{"idset":"10369,10381,10433"}}
+{"timestamp":1712977178.2271538,"name":"online","context":{"idset":"10426,10434"}}
+{"timestamp":1712978621.7672369,"name":"online","context":{"idset":"10251"}}
+{"timestamp":1712978749.5854318,"name":"online","context":{"idset":"10261"}}
+{"timestamp":1712978749.8583479,"name":"online","context":{"idset":"10303"}}
+{"timestamp":1712978750.5321596,"name":"online","context":{"idset":"10262"}}
+{"timestamp":1712978849.5344789,"name":"undrain","context":{"idset":"10255,10261,10303"}}
+{"timestamp":1712978935.0892186,"name":"online","context":{"idset":"10288,10312"}}
+{"timestamp":1712978935.2155659,"name":"online","context":{"idset":"10286"}}
+{"timestamp":1712979110.437582,"name":"offline","context":{"idset":"10305"}}
+{"timestamp":1712983249.9450934,"name":"online","context":{"idset":"10528,10574,10582"}}
+{"timestamp":1712983583.5848818,"name":"online","context":{"idset":"11053"}}
+{"timestamp":1712983583.7066488,"name":"online","context":{"idset":"11026"}}
+{"timestamp":1712983583.9394825,"name":"online","context":{"idset":"11035,11038"}}
+{"timestamp":1712983773.3254657,"name":"online","context":{"idset":"11557-11558"}}
+{"timestamp":1712983773.5532546,"name":"online","context":{"idset":"11572"}}
+{"timestamp":1712983773.8480535,"name":"online","context":{"idset":"11563-11564,11567"}}
+{"timestamp":1712983773.9824023,"name":"online","context":{"idset":"11560,11566,11570"}}
+{"timestamp":1712983774.0984135,"name":"online","context":{"idset":"11559,11561,11568"}}
+{"timestamp":1712983774.2240677,"name":"online","context":{"idset":"11562,11565,11571"}}
+{"timestamp":1712983774.3591795,"name":"online","context":{"idset":"11569"}}
+{"timestamp":1712983779.0835974,"name":"offline","context":{"idset":"9205"}}
+{"timestamp":1712983779.097959,"name":"offline","context":{"idset":"9206"}}
+{"timestamp":1712983779.1099319,"name":"offline","context":{"idset":"9207"}}
+{"timestamp":1712983779.1216125,"name":"offline","context":{"idset":"9208"}}
+{"timestamp":1712983779.1311102,"name":"offline","context":{"idset":"9209"}}
+{"timestamp":1712983779.141876,"name":"offline","context":{"idset":"9210"}}
+{"timestamp":1712983779.1534066,"name":"offline","context":{"idset":"9211"}}
+{"timestamp":1712983779.9485536,"name":"offline","context":{"idset":"9212"}}
+{"timestamp":1712983779.9518962,"name":"offline","context":{"idset":"9213"}}
+{"timestamp":1712983779.955344,"name":"offline","context":{"idset":"9214"}}
+{"timestamp":1712983779.9586501,"name":"offline","context":{"idset":"9215"}}
+{"timestamp":1712983779.9618626,"name":"offline","context":{"idset":"9216"}}
+{"timestamp":1712983779.9752362,"name":"offline","context":{"idset":"9217"}}
+{"timestamp":1712983779.9787552,"name":"offline","context":{"idset":"9218"}}
+{"timestamp":1712983779.9821405,"name":"offline","context":{"idset":"9219"}}
+{"timestamp":1712983779.9853942,"name":"offline","context":{"idset":"9220"}}
+{"timestamp":1712983779.9888649,"name":"offline","context":{"idset":"9221"}}
+{"timestamp":1712983780.0222056,"name":"offline","context":{"idset":"9222"}}
+{"timestamp":1712983780.0578287,"name":"offline","context":{"idset":"9223"}}
+{"timestamp":1712983780.0616989,"name":"offline","context":{"idset":"9224"}}
+{"timestamp":1712983780.0653555,"name":"offline","context":{"idset":"9225"}}
+{"timestamp":1712983780.0693374,"name":"offline","context":{"idset":"9226"}}
+{"timestamp":1712983780.0730588,"name":"offline","context":{"idset":"9227"}}
+{"timestamp":1712983780.0767944,"name":"offline","context":{"idset":"9228"}}
+{"timestamp":1712983780.080739,"name":"offline","context":{"idset":"9229"}}
+{"timestamp":1712983780.0843973,"name":"offline","context":{"idset":"9230"}}
+{"timestamp":1712983780.0880034,"name":"offline","context":{"idset":"9231"}}
+{"timestamp":1712983780.0916264,"name":"offline","context":{"idset":"9232"}}
+{"timestamp":1712983780.0953016,"name":"offline","context":{"idset":"9233"}}
+{"timestamp":1712983780.0984445,"name":"offline","context":{"idset":"9234"}}
+{"timestamp":1712983780.1016414,"name":"offline","context":{"idset":"9235"}}
+{"timestamp":1712983780.105159,"name":"offline","context":{"idset":"9236"}}
+{"timestamp":1712983780.1085901,"name":"offline","context":{"idset":"9237"}}
+{"timestamp":1712983780.1119049,"name":"offline","context":{"idset":"9238"}}
+{"timestamp":1712983780.1153879,"name":"offline","context":{"idset":"9239"}}
+{"timestamp":1712983780.1189637,"name":"offline","context":{"idset":"9240"}}
+{"timestamp":1712983780.1226842,"name":"offline","context":{"idset":"9242"}}
+{"timestamp":1712983780.1262507,"name":"offline","context":{"idset":"9243"}}
+{"timestamp":1712983780.1302044,"name":"offline","context":{"idset":"9244"}}
+{"timestamp":1712983780.1336854,"name":"offline","context":{"idset":"9245"}}
+{"timestamp":1712983780.1372871,"name":"offline","context":{"idset":"9246"}}
+{"timestamp":1712983780.1409431,"name":"offline","context":{"idset":"9247"}}
+{"timestamp":1712983780.1443818,"name":"offline","context":{"idset":"9248"}}
+{"timestamp":1712983780.147985,"name":"offline","context":{"idset":"9249"}}
+{"timestamp":1712983780.1514299,"name":"offline","context":{"idset":"9250"}}
+{"timestamp":1712983780.1550281,"name":"offline","context":{"idset":"9251"}}
+{"timestamp":1712983780.1586964,"name":"offline","context":{"idset":"9252"}}
+{"timestamp":1712983780.1621594,"name":"offline","context":{"idset":"9253"}}
+{"timestamp":1712983780.1659708,"name":"offline","context":{"idset":"9254"}}
+{"timestamp":1712983780.1693642,"name":"offline","context":{"idset":"9255"}}
+{"timestamp":1712983780.1727836,"name":"offline","context":{"idset":"9256"}}
+{"timestamp":1712983780.1763234,"name":"offline","context":{"idset":"9257"}}
+{"timestamp":1712983780.179986,"name":"offline","context":{"idset":"9258"}}
+{"timestamp":1712983780.1832752,"name":"offline","context":{"idset":"9259"}}
+{"timestamp":1712983780.1864393,"name":"offline","context":{"idset":"9260"}}
+{"timestamp":1712983780.1898611,"name":"offline","context":{"idset":"9261"}}
+{"timestamp":1712983780.1930885,"name":"offline","context":{"idset":"9262"}}
+{"timestamp":1712983780.1963308,"name":"offline","context":{"idset":"9263"}}
+{"timestamp":1712983780.1995251,"name":"offline","context":{"idset":"9264"}}
+{"timestamp":1712983780.2030914,"name":"offline","context":{"idset":"9265"}}
+{"timestamp":1712983780.2062252,"name":"offline","context":{"idset":"9266"}}
+{"timestamp":1712983780.2093797,"name":"offline","context":{"idset":"9267"}}
+{"timestamp":1712983780.2132175,"name":"offline","context":{"idset":"9268"}}
+{"timestamp":1712983780.2165112,"name":"offline","context":{"idset":"9269"}}
+{"timestamp":1712983780.219703,"name":"offline","context":{"idset":"9270"}}
+{"timestamp":1712983780.2230997,"name":"offline","context":{"idset":"9271"}}
+{"timestamp":1712983780.2263138,"name":"offline","context":{"idset":"9272"}}
+{"timestamp":1712983780.2294898,"name":"offline","context":{"idset":"9274"}}
+{"timestamp":1712983780.2330565,"name":"offline","context":{"idset":"9275"}}
+{"timestamp":1712983780.236244,"name":"offline","context":{"idset":"9276"}}
+{"timestamp":1712983780.2394812,"name":"offline","context":{"idset":"9277"}}
+{"timestamp":1712983780.2428069,"name":"offline","context":{"idset":"9278"}}
+{"timestamp":1712983780.2465193,"name":"offline","context":{"idset":"9279"}}
+{"timestamp":1712983780.2499449,"name":"offline","context":{"idset":"9280"}}
+{"timestamp":1712983780.2536237,"name":"offline","context":{"idset":"9282"}}
+{"timestamp":1712983780.2574599,"name":"offline","context":{"idset":"9283"}}
+{"timestamp":1712983780.2611272,"name":"offline","context":{"idset":"9284"}}
+{"timestamp":1712983780.2646794,"name":"offline","context":{"idset":"9285"}}
+{"timestamp":1712983780.2682991,"name":"offline","context":{"idset":"9286"}}
+{"timestamp":1712983780.2717464,"name":"offline","context":{"idset":"9287"}}
+{"timestamp":1712983780.2753041,"name":"offline","context":{"idset":"9288"}}
+{"timestamp":1712983780.2793376,"name":"offline","context":{"idset":"9289"}}
+{"timestamp":1712983780.2940536,"name":"offline","context":{"idset":"9291"}}
+{"timestamp":1712983780.2975202,"name":"offline","context":{"idset":"9292"}}
+{"timestamp":1712983780.301188,"name":"offline","context":{"idset":"9293"}}
+{"timestamp":1712983780.3047209,"name":"offline","context":{"idset":"9294"}}
+{"timestamp":1712983780.308429,"name":"offline","context":{"idset":"9295"}}
+{"timestamp":1712983780.3115892,"name":"offline","context":{"idset":"9296"}}
+{"timestamp":1712983780.3147178,"name":"offline","context":{"idset":"9297"}}
+{"timestamp":1712983780.3294091,"name":"offline","context":{"idset":"9298"}}
+{"timestamp":1712983780.3610129,"name":"offline","context":{"idset":"9299"}}
+{"timestamp":1712983780.3834012,"name":"offline","context":{"idset":"9300"}}
+{"timestamp":1712983780.4033084,"name":"offline","context":{"idset":"9301"}}
+{"timestamp":1712983780.4219627,"name":"offline","context":{"idset":"9302"}}
+{"timestamp":1712983780.4405158,"name":"offline","context":{"idset":"9303"}}
+{"timestamp":1712983780.4685862,"name":"offline","context":{"idset":"9304"}}
+{"timestamp":1712983780.50685,"name":"offline","context":{"idset":"9305"}}
+{"timestamp":1712983780.5392332,"name":"offline","context":{"idset":"9306"}}
+{"timestamp":1712983780.5616279,"name":"offline","context":{"idset":"9307"}}
+{"timestamp":1712983780.5648263,"name":"offline","context":{"idset":"9308"}}
+{"timestamp":1712983780.5681531,"name":"offline","context":{"idset":"9309"}}
+{"timestamp":1712983780.5713644,"name":"offline","context":{"idset":"9310"}}
+{"timestamp":1712983780.5874398,"name":"offline","context":{"idset":"9311"}}
+{"timestamp":1712983780.6056623,"name":"offline","context":{"idset":"9312"}}
+{"timestamp":1712983780.632293,"name":"offline","context":{"idset":"9313"}}
+{"timestamp":1712983780.6537488,"name":"offline","context":{"idset":"9314"}}
+{"timestamp":1712983780.6765573,"name":"offline","context":{"idset":"9316"}}
+{"timestamp":1712983780.6958587,"name":"offline","context":{"idset":"9317"}}
+{"timestamp":1712983780.7141056,"name":"offline","context":{"idset":"9318"}}
+{"timestamp":1712983780.717334,"name":"offline","context":{"idset":"9319"}}
+{"timestamp":1712983780.7325723,"name":"offline","context":{"idset":"9321"}}
+{"timestamp":1712983780.7357712,"name":"offline","context":{"idset":"9322"}}
+{"timestamp":1712983780.7508557,"name":"offline","context":{"idset":"9323"}}
+{"timestamp":1712983780.7873225,"name":"offline","context":{"idset":"9324"}}
+{"timestamp":1712983780.8056359,"name":"offline","context":{"idset":"9325"}}
+{"timestamp":1712983780.8238466,"name":"offline","context":{"idset":"9326"}}
+{"timestamp":1712983780.860455,"name":"offline","context":{"idset":"9327"}}
+{"timestamp":1712983780.8785291,"name":"offline","context":{"idset":"9328"}}
+{"timestamp":1712983780.8968904,"name":"offline","context":{"idset":"9329"}}
+{"timestamp":1712983780.915029,"name":"offline","context":{"idset":"9330"}}
+{"timestamp":1712983780.9330449,"name":"offline","context":{"idset":"9331"}}
+{"timestamp":1712983780.9511373,"name":"offline","context":{"idset":"9332"}}
+{"timestamp":1712983780.9694469,"name":"offline","context":{"idset":"9333"}}
+{"timestamp":1712983780.9876316,"name":"offline","context":{"idset":"9334"}}
+{"timestamp":1712983781.0058727,"name":"offline","context":{"idset":"9335"}}
+{"timestamp":1712983781.0239496,"name":"offline","context":{"idset":"9336"}}
+{"timestamp":1712983781.0419779,"name":"offline","context":{"idset":"9337"}}
+{"timestamp":1712983781.0602438,"name":"offline","context":{"idset":"9338"}}
+{"timestamp":1712983781.0786021,"name":"offline","context":{"idset":"9339"}}
+{"timestamp":1712983781.0968263,"name":"offline","context":{"idset":"9340"}}
+{"timestamp":1712983781.1150146,"name":"offline","context":{"idset":"9341"}}
+{"timestamp":1712983781.1330945,"name":"offline","context":{"idset":"9342"}}
+{"timestamp":1712983781.1706069,"name":"offline","context":{"idset":"9343"}}
+{"timestamp":1712983781.2028279,"name":"offline","context":{"idset":"9344"}}
+{"timestamp":1712983781.2351611,"name":"offline","context":{"idset":"9346"}}
+{"timestamp":1712983781.2669213,"name":"offline","context":{"idset":"9347"}}
+{"timestamp":1712983781.2895935,"name":"offline","context":{"idset":"9348"}}
+{"timestamp":1712983781.3113205,"name":"offline","context":{"idset":"9349"}}
+{"timestamp":1712983781.33306,"name":"offline","context":{"idset":"9350"}}
+{"timestamp":1712983781.3549643,"name":"offline","context":{"idset":"9351"}}
+{"timestamp":1712983781.3766549,"name":"offline","context":{"idset":"9352"}}
+{"timestamp":1712983781.3796852,"name":"offline","context":{"idset":"9353"}}
+{"timestamp":1712983781.3827252,"name":"offline","context":{"idset":"9354"}}
+{"timestamp":1712983781.3857532,"name":"offline","context":{"idset":"9355"}}
+{"timestamp":1712983781.3887863,"name":"offline","context":{"idset":"9356"}}
+{"timestamp":1712983781.3918097,"name":"offline","context":{"idset":"9357"}}
+{"timestamp":1712983781.3948479,"name":"offline","context":{"idset":"9358"}}
+{"timestamp":1712983781.3978693,"name":"offline","context":{"idset":"9359"}}
+{"timestamp":1712983781.4008837,"name":"offline","context":{"idset":"9360"}}
+{"timestamp":1712983781.4039021,"name":"offline","context":{"idset":"9361"}}
+{"timestamp":1712983781.4069242,"name":"offline","context":{"idset":"9362"}}
+{"timestamp":1712983781.4099467,"name":"offline","context":{"idset":"9363"}}
+{"timestamp":1712983781.4129713,"name":"offline","context":{"idset":"9364"}}
+{"timestamp":1712983781.4414294,"name":"offline","context":{"idset":"9365"}}
+{"timestamp":1712983781.4444432,"name":"offline","context":{"idset":"9366"}}
+{"timestamp":1712983781.4774785,"name":"offline","context":{"idset":"9367"}}
+{"timestamp":1712983781.4805362,"name":"offline","context":{"idset":"9368"}}
+{"timestamp":1712983781.5137691,"name":"offline","context":{"idset":"9369"}}
+{"timestamp":1712983781.5320928,"name":"offline","context":{"idset":"9370"}}
+{"timestamp":1712983781.5500493,"name":"offline","context":{"idset":"9371"}}
+{"timestamp":1712983781.5860896,"name":"offline","context":{"idset":"9372"}}
+{"timestamp":1712983781.6222191,"name":"offline","context":{"idset":"9373"}}
+{"timestamp":1712983781.6677504,"name":"offline","context":{"idset":"9374"}}
+{"timestamp":1712983781.6899836,"name":"offline","context":{"idset":"9375"}}
+{"timestamp":1712983781.7125487,"name":"offline","context":{"idset":"9376"}}
+{"timestamp":1712983781.7499447,"name":"offline","context":{"idset":"9377"}}
+{"timestamp":1712983781.7859154,"name":"offline","context":{"idset":"9378"}}
+{"timestamp":1712983781.8044734,"name":"offline","context":{"idset":"9379"}}
+{"timestamp":1712983781.8221347,"name":"offline","context":{"idset":"9380"}}
+{"timestamp":1712983781.8403413,"name":"offline","context":{"idset":"9381"}}
+{"timestamp":1712983781.8585517,"name":"offline","context":{"idset":"9382"}}
+{"timestamp":1712983781.8767166,"name":"offline","context":{"idset":"9383"}}
+{"timestamp":1712983781.8947511,"name":"offline","context":{"idset":"9384"}}
+{"timestamp":1712983781.9127252,"name":"offline","context":{"idset":"9385"}}
+{"timestamp":1712983781.930918,"name":"offline","context":{"idset":"9386"}}
+{"timestamp":1712983781.9489315,"name":"offline","context":{"idset":"9388"}}
+{"timestamp":1712983781.9666107,"name":"offline","context":{"idset":"9389"}}
+{"timestamp":1712983781.9850955,"name":"offline","context":{"idset":"9390"}}
+{"timestamp":1712983782.0022292,"name":"offline","context":{"idset":"9391"}}
+{"timestamp":1712983782.0191519,"name":"offline","context":{"idset":"9392"}}
+{"timestamp":1712983782.0357554,"name":"offline","context":{"idset":"9393"}}
+{"timestamp":1712983782.052207,"name":"offline","context":{"idset":"9394"}}
+{"timestamp":1712983782.0684044,"name":"offline","context":{"idset":"9395"}}
+{"timestamp":1712983782.0843198,"name":"offline","context":{"idset":"9396"}}
+{"timestamp":1712983782.1000242,"name":"offline","context":{"idset":"9397"}}
+{"timestamp":1712983782.1155219,"name":"offline","context":{"idset":"9398"}}
+{"timestamp":1712983782.1307201,"name":"offline","context":{"idset":"9399"}}
+{"timestamp":1712983782.1457772,"name":"offline","context":{"idset":"9400"}}
+{"timestamp":1712983782.1605773,"name":"offline","context":{"idset":"9401"}}
+{"timestamp":1712983782.17523,"name":"offline","context":{"idset":"9402"}}
+{"timestamp":1712983782.1898184,"name":"offline","context":{"idset":"9403"}}
+{"timestamp":1712983782.2041507,"name":"offline","context":{"idset":"9404"}}
+{"timestamp":1712983782.2183645,"name":"offline","context":{"idset":"9405"}}
+{"timestamp":1712983782.2323811,"name":"offline","context":{"idset":"9407"}}
+{"timestamp":1712983782.2460983,"name":"offline","context":{"idset":"9408"}}
+{"timestamp":1712983782.2596695,"name":"offline","context":{"idset":"9409"}}
+{"timestamp":1712983782.2733271,"name":"offline","context":{"idset":"9410"}}
+{"timestamp":1712983782.2866325,"name":"offline","context":{"idset":"9411"}}
+{"timestamp":1712983782.2999096,"name":"offline","context":{"idset":"9412"}}
+{"timestamp":1712983782.31285,"name":"offline","context":{"idset":"9413"}}
+{"timestamp":1712983782.3257086,"name":"offline","context":{"idset":"9414"}}
+{"timestamp":1712983782.3438611,"name":"offline","context":{"idset":"9415"}}
+{"timestamp":1712983782.3613648,"name":"offline","context":{"idset":"9416"}}
+{"timestamp":1712983782.3742347,"name":"offline","context":{"idset":"9417"}}
+{"timestamp":1712983782.3896427,"name":"offline","context":{"idset":"9418"}}
+{"timestamp":1712983782.4290226,"name":"offline","context":{"idset":"9419"}}
+{"timestamp":1712983782.4559226,"name":"offline","context":{"idset":"9420"}}
+{"timestamp":1712983782.4877162,"name":"offline","context":{"idset":"9421"}}
+{"timestamp":1712983782.6275656,"name":"offline","context":{"idset":"9422"}}
+{"timestamp":1712983782.6675861,"name":"offline","context":{"idset":"9423"}}
+{"timestamp":1712983782.6903851,"name":"offline","context":{"idset":"9424"}}
+{"timestamp":1712983782.7226851,"name":"offline","context":{"idset":"9425"}}
+{"timestamp":1712983782.7454073,"name":"offline","context":{"idset":"9426"}}
+{"timestamp":1712983782.7685525,"name":"offline","context":{"idset":"9427"}}
+{"timestamp":1712983782.8767278,"name":"offline","context":{"idset":"9428"}}
+{"timestamp":1712983782.8991852,"name":"offline","context":{"idset":"9429"}}
+{"timestamp":1712983782.9211688,"name":"offline","context":{"idset":"9430"}}
+{"timestamp":1712983782.9523189,"name":"offline","context":{"idset":"9431"}}
+{"timestamp":1712983782.9738927,"name":"offline","context":{"idset":"9432"}}
+{"timestamp":1712983783.0048351,"name":"offline","context":{"idset":"9433"}}
+{"timestamp":1712983783.0264072,"name":"offline","context":{"idset":"9434"}}
+{"timestamp":1712983783.0479686,"name":"offline","context":{"idset":"9435"}}
+{"timestamp":1712983783.0788124,"name":"offline","context":{"idset":"9436"}}
+{"timestamp":1712983783.1003335,"name":"offline","context":{"idset":"9437"}}
+{"timestamp":1712983783.1218123,"name":"offline","context":{"idset":"9438"}}
+{"timestamp":1712983783.1525438,"name":"offline","context":{"idset":"9439"}}
+{"timestamp":1712983783.1648149,"name":"offline","context":{"idset":"9440"}}
+{"timestamp":1712983783.1677644,"name":"offline","context":{"idset":"9441"}}
+{"timestamp":1712983783.1707206,"name":"offline","context":{"idset":"9442"}}
+{"timestamp":1712983783.1736619,"name":"offline","context":{"idset":"9443"}}
+{"timestamp":1712983783.176625,"name":"offline","context":{"idset":"9444"}}
+{"timestamp":1712983783.17957,"name":"offline","context":{"idset":"9445"}}
+{"timestamp":1712983783.1825209,"name":"offline","context":{"idset":"9446"}}
+{"timestamp":1712983783.1854718,"name":"offline","context":{"idset":"9447"}}
+{"timestamp":1712983783.1884089,"name":"offline","context":{"idset":"9448"}}
+{"timestamp":1712983783.1913736,"name":"offline","context":{"idset":"9449"}}
+{"timestamp":1712983783.1943212,"name":"offline","context":{"idset":"9450"}}
+{"timestamp":1712983783.1972535,"name":"offline","context":{"idset":"9451"}}
+{"timestamp":1712983783.2001979,"name":"offline","context":{"idset":"9452"}}
+{"timestamp":1712983783.2031381,"name":"offline","context":{"idset":"9453"}}
+{"timestamp":1712983783.2060828,"name":"offline","context":{"idset":"9454"}}
+{"timestamp":1712983783.2163506,"name":"offline","context":{"idset":"9455"}}
+{"timestamp":1712983783.2341061,"name":"offline","context":{"idset":"9456"}}
+{"timestamp":1712983783.2370536,"name":"offline","context":{"idset":"9457"}}
+{"timestamp":1712983783.2520323,"name":"offline","context":{"idset":"9458"}}
+{"timestamp":1712983783.2700143,"name":"offline","context":{"idset":"9459"}}
+{"timestamp":1712983783.2729635,"name":"offline","context":{"idset":"9460"}}
+{"timestamp":1712983783.3058426,"name":"offline","context":{"idset":"9463"}}
+{"timestamp":1712983783.3087752,"name":"offline","context":{"idset":"9465"}}
+{"timestamp":1712983783.3239293,"name":"offline","context":{"idset":"9466"}}
+{"timestamp":1712983783.3686326,"name":"offline","context":{"idset":"9468"}}
+{"timestamp":1712983783.3878121,"name":"offline","context":{"idset":"9469"}}
+{"timestamp":1712983783.4267297,"name":"offline","context":{"idset":"9470"}}
+{"timestamp":1712983783.4447474,"name":"offline","context":{"idset":"9471"}}
+{"timestamp":1712983783.4626365,"name":"offline","context":{"idset":"9472"}}
+{"timestamp":1712983783.4801416,"name":"offline","context":{"idset":"9474"}}
+{"timestamp":1712983783.515238,"name":"offline","context":{"idset":"9475"}}
+{"timestamp":1712983783.5685441,"name":"offline","context":{"idset":"9476"}}
+{"timestamp":1712983783.594332,"name":"offline","context":{"idset":"9477"}}
+{"timestamp":1712983783.6297715,"name":"offline","context":{"idset":"9478"}}
+{"timestamp":1712983783.6646276,"name":"offline","context":{"idset":"9479"}}
+{"timestamp":1712983783.6912882,"name":"offline","context":{"idset":"9481"}}
+{"timestamp":1712983783.7127593,"name":"offline","context":{"idset":"9482"}}
+{"timestamp":1712983783.7342341,"name":"offline","context":{"idset":"9484"}}
+{"timestamp":1712983783.7649517,"name":"offline","context":{"idset":"9485"}}
+{"timestamp":1712983783.7864721,"name":"offline","context":{"idset":"9488"}}
+{"timestamp":1712983783.808475,"name":"offline","context":{"idset":"9489"}}
+{"timestamp":1712983783.8409092,"name":"offline","context":{"idset":"9490"}}
+{"timestamp":1712983783.8624504,"name":"offline","context":{"idset":"9491"}}
+{"timestamp":1712983783.8839157,"name":"offline","context":{"idset":"9492"}}
+{"timestamp":1712983783.9053588,"name":"offline","context":{"idset":"9494"}}
+{"timestamp":1712983783.9268804,"name":"offline","context":{"idset":"9495"}}
+{"timestamp":1712983783.9391446,"name":"offline","context":{"idset":"9496"}}
+{"timestamp":1712983783.9606373,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1712983783.9877913,"name":"offline","context":{"idset":"9503"}}
+{"timestamp":1712983783.9999971,"name":"offline","context":{"idset":"9505"}}
+{"timestamp":1712983784.0214796,"name":"offline","context":{"idset":"9506"}}
+{"timestamp":1712983784.0429497,"name":"offline","context":{"idset":"9507"}}
+{"timestamp":1712983784.0551767,"name":"offline","context":{"idset":"9509"}}
+{"timestamp":1712983784.0674412,"name":"offline","context":{"idset":"9512"}}
+{"timestamp":1712983784.079653,"name":"offline","context":{"idset":"9513"}}
+{"timestamp":1712983784.1011143,"name":"offline","context":{"idset":"9514"}}
+{"timestamp":1712983784.1132996,"name":"offline","context":{"idset":"9515"}}
+{"timestamp":1712983784.2744863,"name":"offline","context":{"idset":"9516"}}
+{"timestamp":1712983784.296062,"name":"offline","context":{"idset":"9517"}}
+{"timestamp":1712983784.3268454,"name":"offline","context":{"idset":"9519"}}
+{"timestamp":1712983784.3484063,"name":"offline","context":{"idset":"9521"}}
+{"timestamp":1712983784.3698719,"name":"offline","context":{"idset":"9522"}}
+{"timestamp":1712983784.39133,"name":"offline","context":{"idset":"9523"}}
+{"timestamp":1712983784.4250894,"name":"offline","context":{"idset":"9524"}}
+{"timestamp":1712983784.4473934,"name":"offline","context":{"idset":"9558"}}
+{"timestamp":1712983784.4688613,"name":"offline","context":{"idset":"9559"}}
+{"timestamp":1712983784.5203316,"name":"offline","context":{"idset":"9560"}}
+{"timestamp":1712983784.532517,"name":"offline","context":{"idset":"9561"}}
+{"timestamp":1712983784.5425589,"name":"offline","context":{"idset":"9562"}}
+{"timestamp":1712983784.5879986,"name":"offline","context":{"idset":"9563"}}
+{"timestamp":1712983784.6089637,"name":"offline","context":{"idset":"9565"}}
+{"timestamp":1712983784.6284556,"name":"offline","context":{"idset":"9566"}}
+{"timestamp":1712983784.6462898,"name":"offline","context":{"idset":"9567"}}
+{"timestamp":1712983784.663959,"name":"offline","context":{"idset":"9568"}}
+{"timestamp":1712983784.6814988,"name":"offline","context":{"idset":"9569"}}
+{"timestamp":1712983784.6990178,"name":"offline","context":{"idset":"9570"}}
+{"timestamp":1712983784.7174225,"name":"offline","context":{"idset":"9571"}}
+{"timestamp":1712983784.7434485,"name":"offline","context":{"idset":"9572"}}
+{"timestamp":1712983784.7612824,"name":"offline","context":{"idset":"9573"}}
+{"timestamp":1712983784.7641599,"name":"offline","context":{"idset":"9574"}}
+{"timestamp":1712983784.7792392,"name":"offline","context":{"idset":"9576"}}
+{"timestamp":1712983784.7968431,"name":"offline","context":{"idset":"9579"}}
+{"timestamp":1712983784.8141708,"name":"offline","context":{"idset":"9580"}}
+{"timestamp":1712983784.8170571,"name":"offline","context":{"idset":"9581"}}
+{"timestamp":1712983784.8317909,"name":"offline","context":{"idset":"9582"}}
+{"timestamp":1712983784.8498831,"name":"offline","context":{"idset":"9583"}}
+{"timestamp":1712983784.8527677,"name":"offline","context":{"idset":"9584"}}
+{"timestamp":1712983784.8872921,"name":"offline","context":{"idset":"9586"}}
+{"timestamp":1712983784.890157,"name":"offline","context":{"idset":"9587"}}
+{"timestamp":1712983784.9045877,"name":"offline","context":{"idset":"9588"}}
+{"timestamp":1712983784.9074712,"name":"offline","context":{"idset":"10357"}}
+{"timestamp":1712983784.9220135,"name":"offline","context":{"idset":"10358"}}
+{"timestamp":1712983784.9393642,"name":"offline","context":{"idset":"10359"}}
+{"timestamp":1712983784.9422042,"name":"offline","context":{"idset":"10360"}}
+{"timestamp":1712983784.9566646,"name":"offline","context":{"idset":"10361"}}
+{"timestamp":1712983785.3598969,"name":"offline","context":{"idset":"10362"}}
+{"timestamp":1712983785.3814054,"name":"offline","context":{"idset":"10363"}}
+{"timestamp":1712983785.3849478,"name":"offline","context":{"idset":"10364"}}
+{"timestamp":1712983785.4009018,"name":"offline","context":{"idset":"10365"}}
+{"timestamp":1712983785.4558535,"name":"offline","context":{"idset":"10366"}}
+{"timestamp":1712983785.5093589,"name":"offline","context":{"idset":"10367"}}
+{"timestamp":1712983785.5449889,"name":"offline","context":{"idset":"10368"}}
+{"timestamp":1712983785.5804935,"name":"offline","context":{"idset":"10369"}}
+{"timestamp":1712983785.5980678,"name":"offline","context":{"idset":"10370"}}
+{"timestamp":1712983785.6337473,"name":"offline","context":{"idset":"10371"}}
+{"timestamp":1712983785.6691613,"name":"offline","context":{"idset":"10372"}}
+{"timestamp":1712983785.7092826,"name":"offline","context":{"idset":"10373"}}
+{"timestamp":1712983785.7263691,"name":"offline","context":{"idset":"10374"}}
+{"timestamp":1712983785.7802322,"name":"offline","context":{"idset":"10375"}}
+{"timestamp":1712983785.8155744,"name":"offline","context":{"idset":"10376"}}
+{"timestamp":1712983785.8690026,"name":"offline","context":{"idset":"10377"}}
+{"timestamp":1712983785.9046514,"name":"offline","context":{"idset":"10378"}}
+{"timestamp":1712983785.9219539,"name":"offline","context":{"idset":"10379"}}
+{"timestamp":1712983785.9571204,"name":"offline","context":{"idset":"10380"}}
+{"timestamp":1712983785.9745631,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1712983785.9921188,"name":"offline","context":{"idset":"10382"}}
+{"timestamp":1712983786.0100162,"name":"offline","context":{"idset":"10383"}}
+{"timestamp":1712983786.0276275,"name":"offline","context":{"idset":"10384"}}
+{"timestamp":1712983786.0453057,"name":"offline","context":{"idset":"10385"}}
+{"timestamp":1712983786.0632496,"name":"offline","context":{"idset":"10386"}}
+{"timestamp":1712983786.0813172,"name":"offline","context":{"idset":"10387"}}
+{"timestamp":1712983786.0989571,"name":"offline","context":{"idset":"10388"}}
+{"timestamp":1712983786.1166439,"name":"offline","context":{"idset":"10389"}}
+{"timestamp":1712983786.1342814,"name":"offline","context":{"idset":"10390"}}
+{"timestamp":1712983786.1517947,"name":"offline","context":{"idset":"10391"}}
+{"timestamp":1712983786.1695056,"name":"offline","context":{"idset":"10392"}}
+{"timestamp":1712983786.1874244,"name":"offline","context":{"idset":"10393"}}
+{"timestamp":1712983786.2051206,"name":"offline","context":{"idset":"10394"}}
+{"timestamp":1712983786.2230616,"name":"offline","context":{"idset":"10395"}}
+{"timestamp":1712983786.2679894,"name":"offline","context":{"idset":"10396"}}
+{"timestamp":1712983786.287894,"name":"offline","context":{"idset":"10397"}}
+{"timestamp":1712983786.3083856,"name":"offline","context":{"idset":"10398"}}
+{"timestamp":1712983786.3283758,"name":"offline","context":{"idset":"10399"}}
+{"timestamp":1712983786.3558886,"name":"offline","context":{"idset":"10400"}}
+{"timestamp":1712983786.376148,"name":"offline","context":{"idset":"10401"}}
+{"timestamp":1712983786.3955934,"name":"offline","context":{"idset":"10402"}}
+{"timestamp":1712983786.4129369,"name":"offline","context":{"idset":"10403"}}
+{"timestamp":1712983786.4302881,"name":"offline","context":{"idset":"10404"}}
+{"timestamp":1712983786.4478106,"name":"offline","context":{"idset":"10405"}}
+{"timestamp":1712983786.465498,"name":"offline","context":{"idset":"10406"}}
+{"timestamp":1712983786.482796,"name":"offline","context":{"idset":"10407"}}
+{"timestamp":1712983786.5003469,"name":"offline","context":{"idset":"10408"}}
+{"timestamp":1712983786.5178118,"name":"offline","context":{"idset":"10409"}}
+{"timestamp":1712983786.5350361,"name":"offline","context":{"idset":"10410"}}
+{"timestamp":1712983786.5526509,"name":"offline","context":{"idset":"10411"}}
+{"timestamp":1712983786.5699921,"name":"offline","context":{"idset":"10412"}}
+{"timestamp":1712983786.587455,"name":"offline","context":{"idset":"10413"}}
+{"timestamp":1712983786.6051438,"name":"offline","context":{"idset":"10414"}}
+{"timestamp":1712983786.6233959,"name":"offline","context":{"idset":"10415"}}
+{"timestamp":1712983786.6408343,"name":"offline","context":{"idset":"10416"}}
+{"timestamp":1712983786.6583326,"name":"offline","context":{"idset":"10417"}}
+{"timestamp":1712983786.6757896,"name":"offline","context":{"idset":"10418"}}
+{"timestamp":1712983786.6937449,"name":"offline","context":{"idset":"10419"}}
+{"timestamp":1712983786.719358,"name":"offline","context":{"idset":"10420"}}
+{"timestamp":1712983786.7380805,"name":"offline","context":{"idset":"10421"}}
+{"timestamp":1712983786.7576673,"name":"offline","context":{"idset":"10422"}}
+{"timestamp":1712983786.7753687,"name":"offline","context":{"idset":"10423"}}
+{"timestamp":1712983786.7947524,"name":"offline","context":{"idset":"10424"}}
+{"timestamp":1712983786.8097064,"name":"offline","context":{"idset":"10425"}}
+{"timestamp":1712983786.8274162,"name":"offline","context":{"idset":"10427"}}
+{"timestamp":1712983786.8449435,"name":"offline","context":{"idset":"10429"}}
+{"timestamp":1712983786.8641965,"name":"offline","context":{"idset":"10430"}}
+{"timestamp":1712983786.8793056,"name":"offline","context":{"idset":"10431"}}
+{"timestamp":1712983786.8977141,"name":"offline","context":{"idset":"10432"}}
+{"timestamp":1712983786.914686,"name":"offline","context":{"idset":"10433"}}
+{"timestamp":1712983786.9336615,"name":"offline","context":{"idset":"10434"}}
+{"timestamp":1712983786.9487154,"name":"offline","context":{"idset":"10435"}}
+{"timestamp":1712983786.9654291,"name":"offline","context":{"idset":"10436"}}
+{"timestamp":1712983786.9822283,"name":"offline","context":{"idset":"10437"}}
+{"timestamp":1712983787.0006742,"name":"offline","context":{"idset":"10438"}}
+{"timestamp":1712983787.0155859,"name":"offline","context":{"idset":"10440"}}
+{"timestamp":1712983787.031785,"name":"offline","context":{"idset":"10441"}}
+{"timestamp":1712983787.0484459,"name":"offline","context":{"idset":"10442"}}
+{"timestamp":1712983787.0672596,"name":"offline","context":{"idset":"10443"}}
+{"timestamp":1712983787.0818143,"name":"offline","context":{"idset":"10444"}}
+{"timestamp":1712983787.1135712,"name":"offline","context":{"idset":"10445"}}
+{"timestamp":1712983787.1335382,"name":"offline","context":{"idset":"10446"}}
+{"timestamp":1712983787.151068,"name":"offline","context":{"idset":"10448"}}
+{"timestamp":1712983787.1686683,"name":"offline","context":{"idset":"10449"}}
+{"timestamp":1712983787.1862278,"name":"offline","context":{"idset":"10450"}}
+{"timestamp":1712983787.202904,"name":"offline","context":{"idset":"10451"}}
+{"timestamp":1712983787.2166553,"name":"offline","context":{"idset":"10452"}}
+{"timestamp":1712983787.2338443,"name":"offline","context":{"idset":"10453"}}
+{"timestamp":1712983787.2472181,"name":"offline","context":{"idset":"10454"}}
+{"timestamp":1712983787.264159,"name":"offline","context":{"idset":"10455"}}
+{"timestamp":1712983787.2775018,"name":"offline","context":{"idset":"10456"}}
+{"timestamp":1712983787.2917721,"name":"offline","context":{"idset":"10457"}}
+{"timestamp":1712983787.3055727,"name":"offline","context":{"idset":"10458"}}
+{"timestamp":1712983787.3218348,"name":"offline","context":{"idset":"10459"}}
+{"timestamp":1712983787.3358731,"name":"offline","context":{"idset":"10460"}}
+{"timestamp":1712983787.3481808,"name":"offline","context":{"idset":"10461"}}
+{"timestamp":1712983787.3635497,"name":"offline","context":{"idset":"10462"}}
+{"timestamp":1712983787.3747325,"name":"offline","context":{"idset":"10463"}}
+{"timestamp":1712983787.389194,"name":"offline","context":{"idset":"10464"}}
+{"timestamp":1712983787.4033847,"name":"offline","context":{"idset":"10465"}}
+{"timestamp":1712983787.4148426,"name":"offline","context":{"idset":"10466"}}
+{"timestamp":1712983787.4297221,"name":"offline","context":{"idset":"10467"}}
+{"timestamp":1712983787.4415858,"name":"offline","context":{"idset":"10468"}}
+{"timestamp":1712983787.4538393,"name":"offline","context":{"idset":"10469"}}
+{"timestamp":1712983787.4955115,"name":"offline","context":{"idset":"10470"}}
+{"timestamp":1712983787.5176201,"name":"offline","context":{"idset":"10471"}}
+{"timestamp":1712983787.5390017,"name":"offline","context":{"idset":"10472"}}
+{"timestamp":1712983787.551115,"name":"offline","context":{"idset":"10473"}}
+{"timestamp":1712983787.5725794,"name":"offline","context":{"idset":"10474"}}
+{"timestamp":1712983787.5940359,"name":"offline","context":{"idset":"10475"}}
+{"timestamp":1712983787.6253557,"name":"offline","context":{"idset":"10476"}}
+{"timestamp":1712983787.6579597,"name":"offline","context":{"idset":"10477"}}
+{"timestamp":1712983787.6913774,"name":"offline","context":{"idset":"10478"}}
+{"timestamp":1712983787.7146485,"name":"offline","context":{"idset":"10479"}}
+{"timestamp":1712983787.7381308,"name":"offline","context":{"idset":"10480"}}
+{"timestamp":1712983787.7510016,"name":"offline","context":{"idset":"10481"}}
+{"timestamp":1712983787.7736664,"name":"offline","context":{"idset":"10482"}}
+{"timestamp":1712983787.797152,"name":"offline","context":{"idset":"10484"}}
+{"timestamp":1712983795.3041089,"name":"offline","context":{"idset":"9290"}}
+{"timestamp":1712983795.3069744,"name":"offline","context":{"idset":"9487"}}
+{"timestamp":1712983795.3097904,"name":"offline","context":{"idset":"10447"}}
+{"timestamp":1712983795.3126025,"name":"offline","context":{"idset":"10483"}}
+{"timestamp":1712983795.3153992,"name":"offline","context":{"idset":"10428"}}
+{"timestamp":1712983795.3597977,"name":"offline","context":{"idset":"9387"}}
+{"timestamp":1712983795.7660673,"name":"offline","context":{"idset":"9493"}}
+{"timestamp":1712983820.9626493,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1712983821.0616274,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1712983822.9435413,"name":"online","context":{"idset":"11570"}}
+{"timestamp":1712983823.1410792,"name":"online","context":{"idset":"11572"}}
+{"timestamp":1712983826.3575404,"name":"offline","context":{"idset":"9241"}}
+{"timestamp":1712983826.3681965,"name":"offline","context":{"idset":"9273"}}
+{"timestamp":1712983826.3786843,"name":"offline","context":{"idset":"9281"}}
+{"timestamp":1712983826.3891029,"name":"offline","context":{"idset":"9315"}}
+{"timestamp":1712983826.3995411,"name":"offline","context":{"idset":"9320"}}
+{"timestamp":1712983826.4099901,"name":"offline","context":{"idset":"9345"}}
+{"timestamp":1712983826.8056064,"name":"offline","context":{"idset":"9406"}}
+{"timestamp":1712983827.2220964,"name":"offline","context":{"idset":"9473"}}
+{"timestamp":1712983828.9343047,"name":"offline","context":{"idset":"9577"}}
+{"timestamp":1712983828.9370389,"name":"offline","context":{"idset":"10426"}}
+{"timestamp":1712983828.9397681,"name":"offline","context":{"idset":"10439"}}
+{"timestamp":1712983946.6513424,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1712983959.9513338,"name":"online","context":{"idset":"11570"}}
+{"timestamp":1712984066.8113656,"name":"undrain","context":{"idset":"11570,11572"}}
+{"timestamp":1712984383.7661326,"name":"online","context":{"idset":"11731"}}
+{"timestamp":1712984383.8996317,"name":"online","context":{"idset":"11740,11756"}}
+{"timestamp":1713018924.4391418,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1713018949.2828331,"name":"online","context":{"idset":"283"}}
+{"timestamp":1713025456.4340198,"name":"offline","context":{"idset":"851"}}
+{"timestamp":1713038378.4375775,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1713060806.4370995,"name":"offline","context":{"idset":"9614"}}
+{"timestamp":1713089808.4357972,"name":"offline","context":{"idset":"10066"}}
+{"timestamp":1713105252.8636901,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1713105268.0938206,"name":"online","context":{"idset":"283"}}
+{"timestamp":1713132630.8628211,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1713149681.4467285,"name":"drain","context":{"idset":"11790","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1713149804.7670074,"name":"drain","context":{"idset":"11790","reason":"epilog failed for jobid fpPyxf8HeV5","overwrite":0}}
+{"timestamp":1713149804.8604658,"name":"offline","context":{"idset":"11790"}}
+{"timestamp":1713190128.6283119,"name":"drain","context":{"idset":"565,567,569,571-572","reason":"New Blade insallation","overwrite":0}}
+{"timestamp":1713190152.8885422,"name":"drain","context":{"idset":"566,568","reason":"New Blade insallation","overwrite":1}}
+{"timestamp":1713190190.9373114,"name":"offline","context":{"idset":"565"}}
+{"timestamp":1713190190.9660075,"name":"offline","context":{"idset":"569"}}
+{"timestamp":1713190190.987524,"name":"offline","context":{"idset":"572"}}
+{"timestamp":1713190190.9991436,"name":"offline","context":{"idset":"567"}}
+{"timestamp":1713190191.0788772,"name":"offline","context":{"idset":"571"}}
+{"timestamp":1713191262.0554755,"name":"drain","context":{"idset":"581-588","reason":"New Blade Installation --JRG","overwrite":1}}
+{"timestamp":1713191277.2446589,"name":"offline","context":{"idset":"584"}}
+{"timestamp":1713191277.2627885,"name":"offline","context":{"idset":"585"}}
+{"timestamp":1713191277.2766004,"name":"offline","context":{"idset":"582"}}
+{"timestamp":1713191277.3760464,"name":"offline","context":{"idset":"587"}}
+{"timestamp":1713191742.788661,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1713191767.0884998,"name":"online","context":{"idset":"283"}}
+{"timestamp":1713193904.7869143,"name":"offline","context":{"idset":"939"}}
+{"timestamp":1713193944.7014494,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1713193944.7902081,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1713194952.78794,"name":"offline","context":{"idset":"9613"}}
+{"timestamp":1713195918.4511232,"name":"offline","context":{"idset":"771"}}
+{"timestamp":1713195978.7231426,"name":"drain","context":{"idset":"771-772","overwrite":0}}
+{"timestamp":1713196398.6946731,"name":"offline","context":{"idset":"11641"}}
+{"timestamp":1713196398.788048,"name":"offline","context":{"idset":"11642"}}
+{"timestamp":1713196417.5460756,"name":"drain","context":{"idset":"963-964","reason":"moving blade","overwrite":0}}
+{"timestamp":1713196451.3415651,"name":"offline","context":{"idset":"964"}}
+{"timestamp":1713196451.4407511,"name":"offline","context":{"idset":"963"}}
+{"timestamp":1713196606.7871914,"name":"offline","context":{"idset":"10306"}}
+{"timestamp":1713197673.6156301,"name":"drain","context":{"idset":"961-962","overwrite":0}}
+{"timestamp":1713199073.1936011,"name":"online","context":{"idset":"10001"}}
+{"timestamp":1713199073.3527796,"name":"online","context":{"idset":"10002"}}
+{"timestamp":1713199138.7879179,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1713199709.6937907,"name":"online","context":{"idset":"10613-10614"}}
+{"timestamp":1713200782.7875481,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1713200784.7878244,"name":"offline","context":{"idset":"11322"}}
+{"timestamp":1713201266.6981466,"name":"offline","context":{"idset":"769"}}
+{"timestamp":1713201266.78917,"name":"offline","context":{"idset":"770"}}
+{"timestamp":1713202708.6991644,"name":"offline","context":{"idset":"11323"}}
+{"timestamp":1713202708.79161,"name":"offline","context":{"idset":"11324"}}
+{"timestamp":1713204246.7004237,"name":"offline","context":{"idset":"11325"}}
+{"timestamp":1713204246.8005993,"name":"offline","context":{"idset":"11326"}}
+{"timestamp":1713205982.6971049,"name":"offline","context":{"idset":"11691"}}
+{"timestamp":1713205982.787833,"name":"offline","context":{"idset":"11692"}}
+{"timestamp":1713207260.7876954,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1713208091.1719108,"name":"online","context":{"idset":"10620"}}
+{"timestamp":1713208091.3827643,"name":"online","context":{"idset":"10619"}}
+{"timestamp":1713208253.9507182,"name":"undrain","context":{"idset":"10613"}}
+{"timestamp":1713208592.6959651,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1713208592.7880456,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1713208594.6964304,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1713208594.7072816,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1713208594.7873275,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1713208596.7000313,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1713208596.7175043,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1713208596.7985597,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1713208598.7033367,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1713208598.7210107,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1713208598.7386086,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1713208598.7882252,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1713208600.7064068,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1713208600.7274351,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1713208600.7477384,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1713208600.7883389,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1713208602.7064533,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1713208602.7225938,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1713208602.7400539,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1713208602.8239436,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1713208604.7076752,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1713208604.7291791,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1713208604.747833,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1713208604.7645392,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1713208604.7906528,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1713208606.6971176,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1713208606.7089858,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1713208606.7947831,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1713208608.700536,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1713208608.7129779,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1713208608.7243831,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1713208608.7356625,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1713208608.8192363,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1713208610.7115269,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1713208610.7337883,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1713208610.7532289,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1713208610.7778423,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1713208610.797035,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1713208612.707372,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1713208612.7231967,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1713208612.7403469,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1713208612.7885926,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1713208614.7038703,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1713208614.7248187,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1713208614.7445893,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1713208614.788126,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1713208616.7086718,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1713208616.7249155,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1713208616.7445505,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1713208616.76403,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1713208616.7944863,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1713208618.6967833,"name":"offline","context":{"idset":"10153"}}
+{"timestamp":1713208618.7089124,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1713208618.7211266,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1713208618.7852566,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1713208620.6961005,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1713208620.7083099,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1713208620.7869391,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1713208622.696398,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1713208622.7078478,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1713208622.7362363,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1713208622.7859383,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1713208624.698077,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1713208624.7138026,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1713208624.790664,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1713208626.6949208,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1713208626.7050502,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1713208626.7871072,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1713208628.7052596,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1713208628.7234015,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1713208628.7418978,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1713208628.7583318,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1713208628.7877669,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1713208630.7069385,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1713208630.7299528,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1713208630.753572,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1713208630.7905629,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1713208632.7040789,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1713208632.7203431,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1713208632.7356188,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1713208632.7502036,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1713208632.7865949,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1713208634.6962211,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1713208634.7095981,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1713208634.7866044,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1713208636.7015018,"name":"offline","context":{"idset":"10188"}}
+{"timestamp":1713208636.7178123,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1713208636.7339008,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1713208636.8278151,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1713208638.7023845,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1713208638.718761,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1713208638.7349331,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1713208638.8106856,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1713208640.7087421,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1713208640.7263014,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1713208640.7450716,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1713208640.7926023,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1713208641.7951384,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1713208641.8107352,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1713208641.8298075,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1713208641.8783669,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1713208644.703881,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1713208644.7221198,"name":"offline","context":{"idset":"10206"}}
+{"timestamp":1713208644.7391553,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1713208644.7899945,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1713208646.7199075,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1713208646.7415175,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1713208646.7574492,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1713208646.7972283,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1713208648.6971874,"name":"offline","context":{"idset":"10213"}}
+{"timestamp":1713208648.7077324,"name":"offline","context":{"idset":"10214"}}
+{"timestamp":1713208648.7177696,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1713208648.7280562,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1713208648.7861166,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1713208650.7018692,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1713208650.7218223,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1713208650.80128,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1713208652.6990941,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1713208652.7093055,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1713208652.7191706,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1713208652.7292099,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1713208652.8389347,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1713208654.6975918,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1713208654.7068996,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1713208654.8048835,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1713208688.6928356,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1713208688.7864754,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1713218745.8955915,"name":"drain","context":{"idset":"10101-10228","overwrite":0}}
+{"timestamp":1713218750.2113497,"name":"undrain","context":{"idset":"10101-10228"}}
+{"timestamp":1713219302.4457872,"name":"drain","context":{"idset":"787-788","overwrite":0}}
+{"timestamp":1713222080.6978369,"name":"offline","context":{"idset":"10485"}}
+{"timestamp":1713222080.7885532,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1713222082.7087533,"name":"offline","context":{"idset":"10487"}}
+{"timestamp":1713222082.7279277,"name":"offline","context":{"idset":"10488"}}
+{"timestamp":1713222082.7488739,"name":"offline","context":{"idset":"10489"}}
+{"timestamp":1713222082.7909896,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1713222084.7006104,"name":"offline","context":{"idset":"10492"}}
+{"timestamp":1713222084.7150824,"name":"offline","context":{"idset":"10493"}}
+{"timestamp":1713222084.7299619,"name":"offline","context":{"idset":"10494"}}
+{"timestamp":1713222084.8074996,"name":"offline","context":{"idset":"10495"}}
+{"timestamp":1713222086.7128079,"name":"offline","context":{"idset":"10496"}}
+{"timestamp":1713222086.7301121,"name":"offline","context":{"idset":"10497"}}
+{"timestamp":1713222086.7513359,"name":"offline","context":{"idset":"10498"}}
+{"timestamp":1713222086.7639108,"name":"offline","context":{"idset":"10499"}}
+{"timestamp":1713222086.7782619,"name":"offline","context":{"idset":"10500"}}
+{"timestamp":1713222086.7957015,"name":"offline","context":{"idset":"10501"}}
+{"timestamp":1713222088.7066953,"name":"offline","context":{"idset":"10502"}}
+{"timestamp":1713222088.7253253,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1713222088.7397776,"name":"offline","context":{"idset":"10504"}}
+{"timestamp":1713222088.7558923,"name":"offline","context":{"idset":"10505"}}
+{"timestamp":1713222088.7890861,"name":"offline","context":{"idset":"10506"}}
+{"timestamp":1713222090.6966281,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1713222090.7073317,"name":"offline","context":{"idset":"10508"}}
+{"timestamp":1713222090.717104,"name":"offline","context":{"idset":"10509"}}
+{"timestamp":1713222090.7897089,"name":"offline","context":{"idset":"10510"}}
+{"timestamp":1713222092.7055669,"name":"offline","context":{"idset":"10511"}}
+{"timestamp":1713222092.7158859,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1713222092.7255738,"name":"offline","context":{"idset":"10513"}}
+{"timestamp":1713222092.7350252,"name":"offline","context":{"idset":"10514"}}
+{"timestamp":1713222092.7445188,"name":"offline","context":{"idset":"10515"}}
+{"timestamp":1713222092.7862477,"name":"offline","context":{"idset":"10516"}}
+{"timestamp":1713222095.0235152,"name":"offline","context":{"idset":"10517"}}
+{"timestamp":1713222096.7006571,"name":"offline","context":{"idset":"10518"}}
+{"timestamp":1713222096.7175648,"name":"offline","context":{"idset":"10519"}}
+{"timestamp":1713222096.7336709,"name":"offline","context":{"idset":"10520"}}
+{"timestamp":1713222096.8131857,"name":"offline","context":{"idset":"10521"}}
+{"timestamp":1713222098.6995864,"name":"offline","context":{"idset":"10522"}}
+{"timestamp":1713222098.7184844,"name":"offline","context":{"idset":"10523"}}
+{"timestamp":1713222098.7947016,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1713222100.7022073,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1713222100.7136569,"name":"offline","context":{"idset":"10525"}}
+{"timestamp":1713222100.7236149,"name":"offline","context":{"idset":"10526"}}
+{"timestamp":1713222100.7344911,"name":"offline","context":{"idset":"10527"}}
+{"timestamp":1713222100.7871535,"name":"offline","context":{"idset":"10528"}}
+{"timestamp":1713222102.6947651,"name":"offline","context":{"idset":"10529"}}
+{"timestamp":1713222102.7065349,"name":"offline","context":{"idset":"10532"}}
+{"timestamp":1713222102.7892408,"name":"offline","context":{"idset":"10533"}}
+{"timestamp":1713222104.6994896,"name":"offline","context":{"idset":"10534"}}
+{"timestamp":1713222104.7157886,"name":"offline","context":{"idset":"10535"}}
+{"timestamp":1713222104.7958076,"name":"offline","context":{"idset":"10537"}}
+{"timestamp":1713222106.6998894,"name":"offline","context":{"idset":"10538"}}
+{"timestamp":1713222106.7149117,"name":"offline","context":{"idset":"10540"}}
+{"timestamp":1713222106.7971666,"name":"offline","context":{"idset":"10541"}}
+{"timestamp":1713222108.701515,"name":"offline","context":{"idset":"10542"}}
+{"timestamp":1713222108.7131953,"name":"offline","context":{"idset":"10543"}}
+{"timestamp":1713222108.7240946,"name":"offline","context":{"idset":"10544"}}
+{"timestamp":1713222108.7347424,"name":"offline","context":{"idset":"10545"}}
+{"timestamp":1713222108.8117826,"name":"offline","context":{"idset":"10546"}}
+{"timestamp":1713222110.6990042,"name":"offline","context":{"idset":"10547"}}
+{"timestamp":1713222110.7105005,"name":"offline","context":{"idset":"10548"}}
+{"timestamp":1713222110.7208357,"name":"offline","context":{"idset":"10549"}}
+{"timestamp":1713222110.7879243,"name":"offline","context":{"idset":"10550"}}
+{"timestamp":1713222112.7142289,"name":"offline","context":{"idset":"10551"}}
+{"timestamp":1713222112.7298446,"name":"offline","context":{"idset":"10552"}}
+{"timestamp":1713222112.7533221,"name":"offline","context":{"idset":"10553"}}
+{"timestamp":1713222112.7899175,"name":"offline","context":{"idset":"10554"}}
+{"timestamp":1713222114.7072098,"name":"offline","context":{"idset":"10555"}}
+{"timestamp":1713222114.7238371,"name":"offline","context":{"idset":"10556"}}
+{"timestamp":1713222114.7411597,"name":"offline","context":{"idset":"10557"}}
+{"timestamp":1713222114.7888503,"name":"offline","context":{"idset":"10558"}}
+{"timestamp":1713222116.7049994,"name":"offline","context":{"idset":"10559"}}
+{"timestamp":1713222116.7205977,"name":"offline","context":{"idset":"10560"}}
+{"timestamp":1713222116.7374408,"name":"offline","context":{"idset":"10561"}}
+{"timestamp":1713222116.7895448,"name":"offline","context":{"idset":"10562"}}
+{"timestamp":1713222118.701668,"name":"offline","context":{"idset":"10563"}}
+{"timestamp":1713222118.7161126,"name":"offline","context":{"idset":"10564"}}
+{"timestamp":1713222118.7953088,"name":"offline","context":{"idset":"10565"}}
+{"timestamp":1713222120.69876,"name":"offline","context":{"idset":"10567"}}
+{"timestamp":1713222120.7102056,"name":"offline","context":{"idset":"10568"}}
+{"timestamp":1713222120.7209973,"name":"offline","context":{"idset":"10569"}}
+{"timestamp":1713222120.7867324,"name":"offline","context":{"idset":"10570"}}
+{"timestamp":1713222122.7084465,"name":"offline","context":{"idset":"10539"}}
+{"timestamp":1713222122.7287018,"name":"offline","context":{"idset":"10571"}}
+{"timestamp":1713222122.747159,"name":"offline","context":{"idset":"10572"}}
+{"timestamp":1713222122.7659276,"name":"offline","context":{"idset":"10573"}}
+{"timestamp":1713222122.7846451,"name":"offline","context":{"idset":"10574"}}
+{"timestamp":1713222122.8047466,"name":"offline","context":{"idset":"10575"}}
+{"timestamp":1713222124.6991138,"name":"offline","context":{"idset":"10576"}}
+{"timestamp":1713222124.7088857,"name":"offline","context":{"idset":"10577"}}
+{"timestamp":1713222124.7183502,"name":"offline","context":{"idset":"10578"}}
+{"timestamp":1713222124.8108611,"name":"offline","context":{"idset":"10579"}}
+{"timestamp":1713222126.6995294,"name":"offline","context":{"idset":"10580"}}
+{"timestamp":1713222126.717041,"name":"offline","context":{"idset":"10581"}}
+{"timestamp":1713222126.806046,"name":"offline","context":{"idset":"10582"}}
+{"timestamp":1713222128.6956589,"name":"offline","context":{"idset":"10583"}}
+{"timestamp":1713222128.7064583,"name":"offline","context":{"idset":"10584"}}
+{"timestamp":1713222128.7871833,"name":"offline","context":{"idset":"10585"}}
+{"timestamp":1713222130.7049451,"name":"offline","context":{"idset":"10586"}}
+{"timestamp":1713222130.7203,"name":"offline","context":{"idset":"10587"}}
+{"timestamp":1713222130.7355757,"name":"offline","context":{"idset":"10588"}}
+{"timestamp":1713222130.7508729,"name":"offline","context":{"idset":"10589"}}
+{"timestamp":1713222130.7879422,"name":"offline","context":{"idset":"10590"}}
+{"timestamp":1713222132.7037239,"name":"offline","context":{"idset":"10591"}}
+{"timestamp":1713222132.7195024,"name":"offline","context":{"idset":"10592"}}
+{"timestamp":1713222132.7350218,"name":"offline","context":{"idset":"10593"}}
+{"timestamp":1713222132.7505581,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1713222132.7875745,"name":"offline","context":{"idset":"10595"}}
+{"timestamp":1713222134.6955452,"name":"offline","context":{"idset":"10596"}}
+{"timestamp":1713222134.7052846,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1713222134.7157302,"name":"offline","context":{"idset":"10598"}}
+{"timestamp":1713222134.8173819,"name":"offline","context":{"idset":"10599"}}
+{"timestamp":1713222136.6956723,"name":"offline","context":{"idset":"10601"}}
+{"timestamp":1713222136.7056997,"name":"offline","context":{"idset":"10602"}}
+{"timestamp":1713222136.7873054,"name":"offline","context":{"idset":"10603"}}
+{"timestamp":1713222138.7057793,"name":"offline","context":{"idset":"10604"}}
+{"timestamp":1713222138.7228363,"name":"offline","context":{"idset":"10605"}}
+{"timestamp":1713222138.7391737,"name":"offline","context":{"idset":"10606"}}
+{"timestamp":1713222138.754401,"name":"offline","context":{"idset":"10607"}}
+{"timestamp":1713222138.7875175,"name":"offline","context":{"idset":"10608"}}
+{"timestamp":1713222140.697201,"name":"offline","context":{"idset":"10609"}}
+{"timestamp":1713222140.7076678,"name":"offline","context":{"idset":"10610"}}
+{"timestamp":1713222140.7173672,"name":"offline","context":{"idset":"10611"}}
+{"timestamp":1713222140.8030865,"name":"offline","context":{"idset":"10612"}}
+{"timestamp":1713222148.7029831,"name":"offline","context":{"idset":"10530"}}
+{"timestamp":1713222148.7898204,"name":"offline","context":{"idset":"10531"}}
+{"timestamp":1713222152.7866592,"name":"offline","context":{"idset":"10536"}}
+{"timestamp":1713222168.789351,"name":"offline","context":{"idset":"10566"}}
+{"timestamp":1713222180.7861524,"name":"offline","context":{"idset":"10600"}}
+{"timestamp":1713222816.4705586,"name":"online","context":{"idset":"10107,10115,10135,10148"}}
+{"timestamp":1713222816.607281,"name":"online","context":{"idset":"10116"}}
+{"timestamp":1713222816.7777011,"name":"online","context":{"idset":"10143,10154,10159"}}
+{"timestamp":1713222816.8147368,"name":"online","context":{"idset":"10136"}}
+{"timestamp":1713222816.9762154,"name":"online","context":{"idset":"10104,10132"}}
+{"timestamp":1713222817.1100984,"name":"online","context":{"idset":"10103,10113,10166"}}
+{"timestamp":1713222817.2890556,"name":"online","context":{"idset":"10112"}}
+{"timestamp":1713222817.4540646,"name":"online","context":{"idset":"10222"}}
+{"timestamp":1713222817.5744328,"name":"online","context":{"idset":"10101,10108"}}
+{"timestamp":1713222817.9470158,"name":"online","context":{"idset":"10111,10145"}}
+{"timestamp":1713222818.0730171,"name":"online","context":{"idset":"10133"}}
+{"timestamp":1713222818.327879,"name":"online","context":{"idset":"10156"}}
+{"timestamp":1713222818.4497087,"name":"online","context":{"idset":"10221"}}
+{"timestamp":1713222818.5791788,"name":"online","context":{"idset":"10146,10157"}}
+{"timestamp":1713222818.7835729,"name":"online","context":{"idset":"10172"}}
+{"timestamp":1713222818.9079268,"name":"online","context":{"idset":"10106,10126,10207"}}
+{"timestamp":1713222819.1250484,"name":"online","context":{"idset":"10142,10153,10186,10215"}}
+{"timestamp":1713222819.3425665,"name":"online","context":{"idset":"10167,10169,10225"}}
+{"timestamp":1713222819.561295,"name":"online","context":{"idset":"10127,10131,10137,10139,10141,10196,10202,10206,10220"}}
+{"timestamp":1713222819.6785407,"name":"online","context":{"idset":"10102,10125,10150,10168"}}
+{"timestamp":1713222819.7981963,"name":"online","context":{"idset":"10204,10210,10219"}}
+{"timestamp":1713222819.917171,"name":"online","context":{"idset":"10117,10121-10122,10124,10128,10134,10160,10164,10179"}}
+{"timestamp":1713222820.1279473,"name":"online","context":{"idset":"10105,10110,10118,10120,10140,10170,10178,10209,10226"}}
+{"timestamp":1713222820.2627826,"name":"online","context":{"idset":"10144,10185"}}
+{"timestamp":1713222820.4805007,"name":"online","context":{"idset":"10114,10129,10171,10175-10176,10180,10182,10184,10190-10191,10208"}}
+{"timestamp":1713222820.6025918,"name":"online","context":{"idset":"10149,10163,10165,10177,10192,10194,10200"}}
+{"timestamp":1713222820.7296133,"name":"online","context":{"idset":"10109,10147,10197"}}
+{"timestamp":1713222820.8587356,"name":"online","context":{"idset":"10119,10123,10130,10151-10152,10155,10158,10173-10174,10181,10193,10198,10203,10205,10211-10214,10227"}}
+{"timestamp":1713222821.0746195,"name":"online","context":{"idset":"10138,10189,10199"}}
+{"timestamp":1713222821.1990902,"name":"online","context":{"idset":"10183,10187-10188,10195,10217,10223-10224,10228"}}
+{"timestamp":1713222821.3374469,"name":"online","context":{"idset":"10218"}}
+{"timestamp":1713222821.4581409,"name":"online","context":{"idset":"10201"}}
+{"timestamp":1713222821.7161663,"name":"online","context":{"idset":"10216"}}
+{"timestamp":1713229804.464401,"name":"drain","context":{"idset":"397","reason":"prolog failed for jobid fpkPrjR8F59","overwrite":0}}
+{"timestamp":1713230523.6110218,"name":"drain","context":{"idset":"283,336,491","reason":"Down","overwrite":0}}
+{"timestamp":1713230540.7870066,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1713230747.9881301,"name":"drain","context":{"idset":"283","reason":"epilog failed for jobid fpkRNQ316CF","overwrite":0}}
+{"timestamp":1713234090.0715036,"name":"online","context":{"idset":"10873"}}
+{"timestamp":1713234327.5047722,"name":"online","context":{"idset":"10874"}}
+{"timestamp":1713235914.0270033,"name":"online","context":{"idset":"10358"}}
+{"timestamp":1713235915.7173162,"name":"online","context":{"idset":"10406"}}
+{"timestamp":1713235918.8598199,"name":"online","context":{"idset":"10433"}}
+{"timestamp":1713235919.3778546,"name":"online","context":{"idset":"10364,10371"}}
+{"timestamp":1713235919.52121,"name":"online","context":{"idset":"10458"}}
+{"timestamp":1713235919.9443808,"name":"online","context":{"idset":"10465"}}
+{"timestamp":1713235920.4512925,"name":"online","context":{"idset":"10401"}}
+{"timestamp":1713235921.3077729,"name":"online","context":{"idset":"10373"}}
+{"timestamp":1713235921.4350352,"name":"online","context":{"idset":"10410"}}
+{"timestamp":1713235922.5327115,"name":"online","context":{"idset":"10383"}}
+{"timestamp":1713235923.0601161,"name":"online","context":{"idset":"10362"}}
+{"timestamp":1713235924.0361395,"name":"online","context":{"idset":"10409,10412,10466"}}
+{"timestamp":1713235924.3639169,"name":"online","context":{"idset":"10377,10390"}}
+{"timestamp":1713235924.6396182,"name":"online","context":{"idset":"10376"}}
+{"timestamp":1713235924.855969,"name":"online","context":{"idset":"10366,10378"}}
+{"timestamp":1713235925.0230045,"name":"online","context":{"idset":"10443"}}
+{"timestamp":1713235925.1594968,"name":"online","context":{"idset":"10431"}}
+{"timestamp":1713235925.6532869,"name":"online","context":{"idset":"10368"}}
+{"timestamp":1713235926.2662578,"name":"online","context":{"idset":"10361"}}
+{"timestamp":1713235927.173106,"name":"online","context":{"idset":"10367,10478"}}
+{"timestamp":1713235927.4203207,"name":"online","context":{"idset":"10425"}}
+{"timestamp":1713235927.4692876,"name":"online","context":{"idset":"10452,10482"}}
+{"timestamp":1713235928.5090477,"name":"online","context":{"idset":"10359,10387"}}
+{"timestamp":1713235928.6719005,"name":"online","context":{"idset":"10404,10408"}}
+{"timestamp":1713235928.9744046,"name":"online","context":{"idset":"10357,10375,10379-10380,10429,10449,10454,10472"}}
+{"timestamp":1713235929.110919,"name":"online","context":{"idset":"10360,10453"}}
+{"timestamp":1713235929.2395096,"name":"online","context":{"idset":"10395,10397,10434,10467"}}
+{"timestamp":1713235929.367877,"name":"online","context":{"idset":"10437,10457"}}
+{"timestamp":1713235929.8074148,"name":"online","context":{"idset":"10400,10405,10407,10439,10469,10473,10475,10484"}}
+{"timestamp":1713235929.9329453,"name":"online","context":{"idset":"10382,10411,10463"}}
+{"timestamp":1713235930.0645549,"name":"online","context":{"idset":"10388,10424"}}
+{"timestamp":1713235930.1898735,"name":"online","context":{"idset":"10422"}}
+{"timestamp":1713235930.6859093,"name":"online","context":{"idset":"10398,10427,10451"}}
+{"timestamp":1713235931.3613505,"name":"drain","context":{"idset":"10374","reason":"broker was unresponsive"}}
+{"timestamp":1713235931.5169892,"name":"online","context":{"idset":"10384,10428,10436"}}
+{"timestamp":1713235931.9042094,"name":"online","context":{"idset":"10455,10476"}}
+{"timestamp":1713235932.2120869,"name":"online","context":{"idset":"10389"}}
+{"timestamp":1713235932.4184723,"name":"online","context":{"idset":"10399"}}
+{"timestamp":1713235932.8808997,"name":"online","context":{"idset":"10374"}}
+{"timestamp":1713235933.7972512,"name":"online","context":{"idset":"10438"}}
+{"timestamp":1713235934.3060462,"name":"online","context":{"idset":"10372"}}
+{"timestamp":1713235934.4375196,"name":"online","context":{"idset":"10414"}}
+{"timestamp":1713235934.7312238,"name":"online","context":{"idset":"10363,10402,10435,10459,10471,10474"}}
+{"timestamp":1713235934.8766539,"name":"online","context":{"idset":"10369"}}
+{"timestamp":1713235935.0335641,"name":"online","context":{"idset":"10370"}}
+{"timestamp":1713235935.1799657,"name":"online","context":{"idset":"10440,10481"}}
+{"timestamp":1713235935.3151758,"name":"online","context":{"idset":"10423,10462"}}
+{"timestamp":1713235935.4568429,"name":"online","context":{"idset":"10441"}}
+{"timestamp":1713235935.7307141,"name":"online","context":{"idset":"10445"}}
+{"timestamp":1713235935.8083451,"name":"online","context":{"idset":"10444,10477"}}
+{"timestamp":1713235935.9767797,"name":"online","context":{"idset":"10392"}}
+{"timestamp":1713235936.6261516,"name":"online","context":{"idset":"10483"}}
+{"timestamp":1713235937.3236332,"name":"online","context":{"idset":"10460"}}
+{"timestamp":1713235938.1231468,"name":"online","context":{"idset":"10470"}}
+{"timestamp":1713235938.2979565,"name":"online","context":{"idset":"10381"}}
+{"timestamp":1713235939.2611177,"name":"online","context":{"idset":"10426"}}
+{"timestamp":1713235939.5227718,"name":"online","context":{"idset":"10446"}}
+{"timestamp":1713235939.7537346,"name":"online","context":{"idset":"10448"}}
+{"timestamp":1713235940.0603297,"name":"online","context":{"idset":"10447"}}
+{"timestamp":1713235940.3734987,"name":"online","context":{"idset":"10394"}}
+{"timestamp":1713235940.6001225,"name":"online","context":{"idset":"10396,10415"}}
+{"timestamp":1713235941.3050191,"name":"online","context":{"idset":"10393"}}
+{"timestamp":1713235942.4218574,"name":"online","context":{"idset":"10403"}}
+{"timestamp":1713235943.1760502,"name":"online","context":{"idset":"10416"}}
+{"timestamp":1713235943.4590065,"name":"online","context":{"idset":"10418"}}
+{"timestamp":1713235944.4560456,"name":"online","context":{"idset":"10385,10391"}}
+{"timestamp":1713235945.2519176,"name":"online","context":{"idset":"10479"}}
+{"timestamp":1713235947.6984301,"name":"online","context":{"idset":"10420"}}
+{"timestamp":1713235949.5730362,"name":"online","context":{"idset":"10365,10461"}}
+{"timestamp":1713235950.4910698,"name":"online","context":{"idset":"10468"}}
+{"timestamp":1713235950.6216397,"name":"online","context":{"idset":"10419"}}
+{"timestamp":1713235950.7664557,"name":"online","context":{"idset":"10386,10442"}}
+{"timestamp":1713235951.7179866,"name":"online","context":{"idset":"10456"}}
+{"timestamp":1713235953.3071344,"name":"online","context":{"idset":"10464,10480"}}
+{"timestamp":1713235955.8134151,"name":"online","context":{"idset":"10413"}}
+{"timestamp":1713235955.9457717,"name":"online","context":{"idset":"10417"}}
+{"timestamp":1713235956.1465514,"name":"online","context":{"idset":"10430"}}
+{"timestamp":1713235956.7970247,"name":"online","context":{"idset":"10432"}}
+{"timestamp":1713235957.2580485,"name":"online","context":{"idset":"10450"}}
+{"timestamp":1713235959.5101643,"name":"online","context":{"idset":"10421"}}
+{"timestamp":1713236307.0546622,"name":"online","context":{"idset":"9226"}}
+{"timestamp":1713236307.3238149,"name":"online","context":{"idset":"9207-9208,9215"}}
+{"timestamp":1713236307.542999,"name":"online","context":{"idset":"9205,9232"}}
+{"timestamp":1713236307.6756375,"name":"online","context":{"idset":"9212,9221-9222"}}
+{"timestamp":1713236307.8018303,"name":"online","context":{"idset":"9209"}}
+{"timestamp":1713236308.071043,"name":"online","context":{"idset":"9211"}}
+{"timestamp":1713236308.2527053,"name":"online","context":{"idset":"9210"}}
+{"timestamp":1713236308.394032,"name":"online","context":{"idset":"9206"}}
+{"timestamp":1713236308.5284162,"name":"online","context":{"idset":"9219"}}
+{"timestamp":1713236308.6784647,"name":"online","context":{"idset":"9223"}}
+{"timestamp":1713236309.1869228,"name":"online","context":{"idset":"9227,9230"}}
+{"timestamp":1713236309.7803018,"name":"online","context":{"idset":"9237"}}
+{"timestamp":1713236310.1356728,"name":"online","context":{"idset":"9220"}}
+{"timestamp":1713236310.2977531,"name":"online","context":{"idset":"9248"}}
+{"timestamp":1713236310.3327105,"name":"online","context":{"idset":"9253"}}
+{"timestamp":1713236310.7944098,"name":"online","context":{"idset":"9268"}}
+{"timestamp":1713236310.8243093,"name":"online","context":{"idset":"9254"}}
+{"timestamp":1713236310.9915309,"name":"online","context":{"idset":"9298"}}
+{"timestamp":1713236311.1530809,"name":"online","context":{"idset":"9242,9244,9249"}}
+{"timestamp":1713236311.3368471,"name":"online","context":{"idset":"9276"}}
+{"timestamp":1713236311.3775713,"name":"online","context":{"idset":"9217"}}
+{"timestamp":1713236311.5447445,"name":"online","context":{"idset":"9218,9269"}}
+{"timestamp":1713236311.6885705,"name":"online","context":{"idset":"9240,9280"}}
+{"timestamp":1713236311.9301982,"name":"online","context":{"idset":"9231,9233,9236,9257,9304"}}
+{"timestamp":1713236312.0647099,"name":"online","context":{"idset":"9225,9300"}}
+{"timestamp":1713236312.2975554,"name":"online","context":{"idset":"9243,9265,9288,9297"}}
+{"timestamp":1713236312.4277644,"name":"online","context":{"idset":"9252,9273"}}
+{"timestamp":1713236312.5555956,"name":"online","context":{"idset":"9228,9278"}}
+{"timestamp":1713236312.6834309,"name":"online","context":{"idset":"9239,9274"}}
+{"timestamp":1713236312.8254409,"name":"online","context":{"idset":"9216,9238,9245,9250,9255-9256,9267,9270,9285,9289-9290,9308,9328"}}
+{"timestamp":1713236312.9813108,"name":"online","context":{"idset":"9234-9235,9261,9266,9271-9272,9281,9292,9296,9307,9311,9331"}}
+{"timestamp":1713236313.2116773,"name":"online","context":{"idset":"9229,9247,9251,9258,9263,9277,9286,9293,9295,9309,9318,9332"}}
+{"timestamp":1713236313.3492012,"name":"online","context":{"idset":"9224,9259,9262,9264,9275,9279,9282,9287,9294,9301,9310,9312,9322-9323,9326"}}
+{"timestamp":1713236313.4724748,"name":"online","context":{"idset":"9246,9260,9284,9291,9302,9306,9313-9315,9321,9327,9329-9330"}}
+{"timestamp":1713236313.593884,"name":"online","context":{"idset":"9303"}}
+{"timestamp":1713236313.7803447,"name":"online","context":{"idset":"9283,9305,9316,9319-9320,9324"}}
+{"timestamp":1713236313.9005494,"name":"online","context":{"idset":"9299,9317,9325"}}
+{"timestamp":1713236369.6882522,"name":"offline","context":{"idset":"10374"}}
+{"timestamp":1713236371.784416,"name":"online","context":{"idset":"10374"}}
+{"timestamp":1713236427.6804388,"name":"undrain","context":{"idset":"10374"}}
+{"timestamp":1713236521.5845647,"name":"online","context":{"idset":"9241"}}
+{"timestamp":1713236522.4884591,"name":"online","context":{"idset":"9213-9214"}}
+{"timestamp":1713236752.6985657,"name":"offline","context":{"idset":"11685"}}
+{"timestamp":1713236752.7882245,"name":"offline","context":{"idset":"11686"}}
+{"timestamp":1713237184.7869039,"name":"offline","context":{"idset":"10412"}}
+{"timestamp":1713237704.0144269,"name":"online","context":{"idset":"11588"}}
+{"timestamp":1713238632.879544,"name":"online","context":{"idset":"11587"}}
+{"timestamp":1713273658.6276512,"name":"online","context":{"idset":"9613"}}
+{"timestamp":1713273681.4700348,"name":"online","context":{"idset":"9614"}}
+{"timestamp":1713274327.528789,"name":"online","context":{"idset":"11203"}}
+{"timestamp":1713274360.5061352,"name":"online","context":{"idset":"11204"}}
+{"timestamp":1713275343.7651885,"name":"offline","context":{"idset":"9989"}}
+{"timestamp":1713275343.7931952,"name":"offline","context":{"idset":"9990"}}
+{"timestamp":1713275343.8177149,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1713275343.8435619,"name":"offline","context":{"idset":"9992"}}
+{"timestamp":1713275343.8661287,"name":"offline","context":{"idset":"9994"}}
+{"timestamp":1713275343.8891621,"name":"offline","context":{"idset":"9996"}}
+{"timestamp":1713275343.9145384,"name":"offline","context":{"idset":"9997"}}
+{"timestamp":1713275343.9379833,"name":"offline","context":{"idset":"9998"}}
+{"timestamp":1713275343.9717369,"name":"offline","context":{"idset":"9999"}}
+{"timestamp":1713275344.0026064,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1713275344.0260799,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1713275344.0494466,"name":"offline","context":{"idset":"10003"}}
+{"timestamp":1713275344.0746555,"name":"offline","context":{"idset":"10004"}}
+{"timestamp":1713276350.1326573,"name":"online","context":{"idset":"9999"}}
+{"timestamp":1713276350.2841234,"name":"online","context":{"idset":"9994"}}
+{"timestamp":1713276350.4106977,"name":"online","context":{"idset":"9990,9997,10004"}}
+{"timestamp":1713276350.5556195,"name":"online","context":{"idset":"9995,10002-10003"}}
+{"timestamp":1713276350.7718077,"name":"online","context":{"idset":"9998"}}
+{"timestamp":1713276407.9998384,"name":"online","context":{"idset":"10001"}}
+{"timestamp":1713278622.9572425,"name":"resource-init","context":{"restart":true,"drain":{"65":{"timestamp":1712864426.4722612,"reason":"NC unresponsive"},"121":{"timestamp":1710136167.4201007,"reason":"nodediag failed pci"},"217":{"timestamp":1712864756.4313865,"reason":"Drops to UEFI on boot"},"253-276":{"timestamp":1712864539.3291147,"reason":"Leak investigation"},"348":{"timestamp":1711576490.2548447,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"397":{"timestamp":1713229804.464401,"reason":"prolog failed for jobid fpkPrjR8F59"},"361,420":{"timestamp":1712942103.0553768,"reason":"epilog failed for jobid fp6az3kDFZy"},"477":{"timestamp":1712939037.9081478,"reason":"nodediag failed cxi"},"478":{"timestamp":1712939033.9870858,"reason":"nodediag failed cxi"},"479":{"timestamp":1712939056.9337916,"reason":"nodediag failed cxi"},"480":{"timestamp":1712939042.0713556,"reason":"nodediag failed cxi"},"481":{"timestamp":1712939015.2720578,"reason":"nodediag failed cxi"},"283,336,491":{"timestamp":1713230523.6110218,"reason":"Down"},"501":{"timestamp":1712864871.3503184,"reason":"ASSERT_EFI_ERROR on boot"},"506":{"timestamp":1712783164.2746041,"reason":"Rabbit crashed due to node bringup --JRG"},"493-500,502-505,507":{"timestamp":1712876380.0871921,"reason":"New blade install --JRG"},"508":{"timestamp":1712864932.9871686,"reason":"Drops to UEFI on boot"},"509-516":{"timestamp":1712849612.3305345,"reason":"New Blade Installation"},"566,568":{"timestamp":1713190152.8885422,"reason":"New Blade insallation"},"570":{"timestamp":1712864992.8868096,"reason":"ASSERT_EFI_ERROR on boot"},"565,567,569,571-572":{"timestamp":1713190128.6283119,"reason":"New Blade insallation"},"573-580":{"timestamp":1712858384.0348849,"reason":"New Blade Install --JRG"},"581-588":{"timestamp":1713191262.0554755,"reason":"New Blade Installation --JRG"},"629-636":{"timestamp":1712696364.0966945,"reason":"New Blade Installation --JRG"},"762":{"timestamp":1712862032.7360361,"reason":"nodediag failed amdapu dgemm_perf"},"771-772":{"timestamp":1713195978.7231426,"reason":""},"773-774":{"timestamp":1712791238.8350906,"reason":""},"775-776":{"timestamp":1712792380.1267438,"reason":""},"787-788":{"timestamp":1713219302.4457872,"reason":""},"789-794":{"timestamp":1711461777.1299369,"reason":""},"795":{"timestamp":1712868053.3882167,"reason":"broker was unresponsive"},"796":{"timestamp":1712868054.1067045,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261862.602201,"reason":"--reason swapping blades into x1900 - wendy"},"814":{"timestamp":1712783332.1390359,"reason":""},"827":{"timestamp":1712859030.3444552,"reason":"broker was unresponsive"},"828":{"timestamp":1712859028.2233922,"reason":"broker was unresponsive"},"829-830":{"timestamp":1712700944.7348719,"reason":""},"854":{"timestamp":1712945395.4889505,"reason":"Debugging downed nodes - vls"},"877":{"timestamp":1712672904.2689505,"reason":"reason=drain for NMC0 swap"},"878":{"timestamp":1712701679.2107792,"reason":"troubshooting H/W"},"937-938":{"timestamp":1712866350.7723167,"reason":""},"949":{"timestamp":1712854277.3872395,"reason":"broker was unresponsive"},"950":{"timestamp":1712854277.4874694,"reason":"broker was unresponsive"},"961-962":{"timestamp":1713197673.6156301,"reason":""},"963-964":{"timestamp":1713196417.5460756,"reason":"moving blade"},"7927":{"timestamp":1712931493.4967601,"reason":"Parter node failure"},"9003":{"timestamp":1712873964.3604848,"reason":"nodediag failed dgemm_perf"},"9054":{"timestamp":1712864116.1275845,"reason":"broker was unresponsive"},"9075":{"timestamp":1712864130.4486458,"reason":"broker was unresponsive"},"9533":{"timestamp":1712858079.4850941,"reason":"broker was unresponsive"},"9534":{"timestamp":1712858512.7494123,"reason":"broker was unresponsive"},"9535":{"timestamp":1712863022.1354616,"reason":"broker was unresponsive"},"9782":{"timestamp":1712851949.4889591,"reason":"broker was unresponsive"},"9788":{"timestamp":1712710144.0153348,"reason":"bad partner node --JRG"},"9789":{"timestamp":1712711094.53824,"reason":"bad partner node --JRG"},"9991":{"timestamp":1712853317.4912872,"reason":"broker was unresponsive"},"10911-10912":{"timestamp":1712523855.472235,"reason":""},"11573":{"timestamp":1712953545.9583356,"reason":"nodediag failed dgemm_perf"},"11574":{"timestamp":1712953542.2388783,"reason":"nodediag failed dgemm_perf"},"11577":{"timestamp":1712953550.0592394,"reason":"nodediag failed dgemm_perf"},"11578":{"timestamp":1712953546.4172895,"reason":"nodediag failed dgemm_perf"},"11579":{"timestamp":1712953541.2107675,"reason":"nodediag failed dgemm_perf"},"11580":{"timestamp":1712953538.9321792,"reason":"nodediag failed dgemm_perf"},"11581":{"timestamp":1712953530.8933868,"reason":"nodediag failed dgemm_perf"},"11582":{"timestamp":1712953548.6102457,"reason":"nodediag failed dgemm_perf"},"11583":{"timestamp":1712953531.1781602,"reason":"nodediag failed dgemm_perf"},"11584":{"timestamp":1712953531.3784466,"reason":"nodediag failed dgemm_perf"},"11585":{"timestamp":1712953542.7601724,"reason":"nodediag failed dgemm_perf"},"11587":{"timestamp":1712953541.8604009,"reason":"nodediag failed dgemm_perf"},"11588":{"timestamp":1712953539.7299294,"reason":"nodediag failed dgemm_perf"},"11790":{"timestamp":1713149681.4467285,"reason":"Epilog failed to complete"}},"online":"","exclude":"0"}}
+{"timestamp":1713278623.0165277,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1713278651.4980116,"name":"online","context":{"idset":"0"}}
+{"timestamp":1713278653.603349,"name":"online","context":{"idset":"820,951,7934,7936,7938,7940-7941,7950,7960-7962,7967-7968,7975,7979-7981,7987,7989-7991,7993-7994,7996,7998,8000,8002,8006,8008,8014-8017,8019,8022-8023,8026-8028,8031,8034-8036,8038-8042,8048,8951,8953,8961,8965,8972,8976,8979,8981,8984,8990,8995-8996,8999,9002,9004-9005,9007,9011-9012,9036,9040,9048,9050,9052-9053,9056,9059,9063-9065,9072,9613-9614,9719,9724,9728-9729,9733,9736,9742,9748,9758,9762-9765,9768-9769,9773,9793,9795-9796,9798-9799,9808,9818-9819,9822,9827,9829,9831,9837-9838,9842-9843,9977-9978,9981,9985,9994,10003,10005,10009,10012,10021,10032,10035,10038-10042,10045,10053,10055,10059,10062,10065,10068,10074,10077,10080,10082,10159,10164,11254-11256,11260,11262-11265,11270-11271,11277,11280,11283,11288,11290,11292-11295,11297,11299-11300,11302-11303,11328,11424,11587,11639,11643-11644,11650,11652,11687-11688,11696,11698-11700"}}
+{"timestamp":1713278654.0709975,"name":"online","context":{"idset":"86,952-954,973-976,7925-7926,7929-7933,7935,7937,7939,7942-7949,7951-7959,7963-7966,7969-7974,7976-7978,7982-7986,7988,7992,7995,7997,7999,8001,8003-8005,8007,8009-8013,8018,8020-8021,8024-8025,8029-8030,8032-8033,8037,8043-8047,8049-8052,8949-8950,8952,8954-8960,8962-8964,8966-8971,8973-8975,8977-8978,8980,8982-8983,8985-8989,8991-8994,8997-8998,9000-9001,9003,9006,9008,9010,9013-9016,9019-9035,9037-9039,9041-9047,9049,9051,9054-9055,9057-9058,9060-9062,9066-9071,9073-9074,9076,9717-9718,9720-9723,9725-9727,9730-9732,9734-9735,9737-9741,9743-9747,9749-9757,9759-9761,9766-9767,9770-9772,9774-9786,9791-9792,9794,9797,9800-9807,9809-9817,9820-9821,9823-9826,9828,9830,9832-9836,9839-9841,9844,9973-9976,9980,9982,9984,9986-9988,9990-9993,9995-10002,10004,10006-10008,10010-10011,10013,10015-10020,10022-10028,10030-10031,10033-10034,10036-10037,10043-10044,10046-10052,10054,10056-10058,10060-10061,10063-10064,10067,10069-10073,10075-10076,10078-10079,10081,10083-10084,10153-10158,10160,10163,10872,11253,11257-11259,11261,11266-11267,11269,11272-11276,11278-11279,11281-11282,11284-11287,11289,11291,11296,11298,11301,11320,11327,11423,11588,11637-11638,11640,11645-11649,11651,11689-11690,11695,11697"}}
+{"timestamp":1713278654.2000144,"name":"online","context":{"idset":"972,977,9009,9575,9578,9989,10351,11332"}}
+{"timestamp":1713278654.3372118,"name":"online","context":{"idset":"85,978,9910,9926"}}
+{"timestamp":1713278654.4893949,"name":"online","context":{"idset":"874,934,9461,9486"}}
+{"timestamp":1713278654.6026084,"name":"online","context":{"idset":"527-529,531,533,535,539,873,971,9462,9467,9483,9497,9499,9511,9585,9605,9691,9713,9716,10170,10286,10376,10746,10888,10963,11136,11350"}}
+{"timestamp":1713278654.738353,"name":"online","context":{"idset":"111,183,313,345,373,415,532,534,536,538,837,843-844,852,945,9208,9210,9219,9236-9237,9259,9265,9281-9282,9290,9294,9308,9315,9464,9480,9498,9500,9510,9518,9520,9551,9595,9603,9607,9609-9610,9612,9615,9633-9634,9643,9664,9666,9668,9678,9687,9696,9699,9714-9715,9846,9869,9873,9895,9897,9904,9908-9909,9918,9966,10092,10097-10098,10110,10142,10150,10176,10194,10198,10218-10219,10223,10225,10231,10237,10250,10253,10255,10262,10267,10274,10300,10302,10308,10310,10314,10316,10323,10327,10334,10346,10358,10368,10382,10385,10397-10398,10406,10425,10434,10438,10456,10460,10464,10468,10621,10636,10639,10657,10661-10664,10683,10693,10696,10742-10743,10753,10763,10773,10776,10780,10797,10815,10819,10829,10841,10859,10865,10874,10884,10886,10892,10904,10909,10921,10927,10944-10945,10962,10964,10972,10979,10981,10987,10992,11030,11036,11072,11074-11075,11086,11089,11092,11104,11121,11127-11128,11177,11203-11204,11209,11219,11227,11231,11233,11237-11238,11240-11242,11334,11339,11342,11347,11362,11368,11386,11393,11400,11405-11406,11410,11412,11414,11416,11421,11433,11438,11441,11455-11456,11462,11464,11470,11474,11479,11504,11513,11516,11518,11520,11531,11534-11535,11544,11553,11556,11559,11563,11567,11572,11597,11609,11612,11621,11630,11634,11659,11663-11664,11717-11718,11724,11731,11734,11738,11743,11752,11762-11763,11768,11771,11773,11779,11785-11786,11791,11793-11795,11800,11805,11823,11834-11835,11840,11843,11845-11846,11854,11866,11874,11881"}}
+{"timestamp":1713278655.0091679,"name":"online","context":{"idset":"147,347,540,9218,9232,9240,9272,9274,9284,9298,9320,9328,9502,9504,9608,9628,9670,9680,9700,9712,9866,9882,9887,9902,9914,9952,9970,10085-10086,10093,10197,10226,10246,10258,10261,10272,10280,10290,10341,10352,10360,10400,10415,10471,10475,10623,10721,10736,10751,10758,10774,10785,10791,10843,10847,10863-10864,10870,10908,10913,10918,10928,10958,10965,10967,10970,10978,10993,11055,11084,11093,11095,11114,11122,11126,11135,11161,11179,11224-11225,11351,11354,11363,11375,11390,11394,11420,11422,11429,11432,11449,11459,11461,11468,11486,11489,11491,11502,11533,11590,11594,11608,11617,11627,11671,11682,11705-11706,11721,11737,11764,11767,11770,11796,11801,11803-11804,11807,11813,11831,11851,11856-11857,11859,11865,11872-11873,11875,11877,11891"}}
+{"timestamp":1713278655.1526058,"name":"online","context":{"idset":"9890,9931,9967,10096,11653"}}
+{"timestamp":1713278655.2598977,"name":"online","context":{"idset":"10230,10333,10350,10416,10465,10470,10482,10640,10767,10771,10783,10799,10871,10905,10915-10916,10919,10960,10968,10984,11096,11111-11112,11151,11396,11415,11720"}}
+{"timestamp":1713278655.3958263,"name":"online","context":{"idset":"118-119,123,139,149,152,166,175,203,211,281,302,329,341,386,398,413,429,475,478,489,941,943,947,9205-9206,9214-9216,9222,9224,9226,9229-9230,9233,9242,9246,9250,9252,9256-9258,9261-9264,9266-9270,9273,9275,9277,9280,9283,9285-9287,9291-9293,9295-9296,9299,9301-9307,9309-9311,9313-9314,9316,9318,9321-9327,9329-9331,9508,9541,9543-9550,9552-9553,9555-9557,9564,9589-9590,9592-9594,9596-9602,9604,9606,9611,9618,9626-9627,9635,9654,9658,9662,9671,9673,9686,9706,10109,10118,10283,10399,10649,10653,10658,10854,10862,10885,10897,10946,11028,11167,11181,11191,11435,11493,11498,11501,11509,11619,11626,11748,11758"}}
+{"timestamp":1713278655.5238605,"name":"online","context":{"idset":"109,126,169,182,209,9636,9642,9644,9646,9648,9650-9651,9653,9660,9669,9672,9674,9676,9682-9684,9692-9694,9698,9704-9705,9710-9711,9845,9847-9848,9850-9854,9856-9861,9868,9870,9872,9879,9886,9889,9905,9911-9912,9915,9922,9924,9928,9946-9948,9954-9955,9962,9964,9968,10094-10095,10117,10147,10172,10174,10190,10203,10217,10221,10232-10233,10236,10238,10242,10248,10257,10275-10276,10282,10295,10303-10304,10312,10324,10329-10330,10332,10336,10348,10357,10372-10373,10375,10378,10381,10387,10394,10402,10404,10411,10413,10417,10422,10424,10428,10430,10432,10435,10443-10444,10448,10450,10458,10466-10467,10469,10474,10477-10481,10484,10614,10627,10631-10635,10641,10648,10652,10654,10656,10660,10666,10668-10669,10679,10688,10691,10702,10705,10720,10723,10730,10732,10737,10741,10744-10745,10748,10750,10755,10760,10764,10766,10775,10787,10789,10795,10801,10804-10810,10814,10817,10820,10822,10831,10833-10836,10842,10851,10856,10858,10868,10878,10880-10881,10883,10898,10900,10902,10914,10917,10922-10924,10926,10933,10937,10939,10949-10950,10953,10966,10977,10986,10988,10991,10994-10995,11039,11070,11076-11079,11099,11105-11107,11110,11115-11116,11125,11141,11159,11163,11168,11175,11192,11194,11198,11207,11212,11228,11232,11235,11245,11248-11249,11252,11308,11333,11337,11343,11346,11353,11356-11357,11361,11365,11370-11371,11373-11374,11376,11380,11384,11408,11426-11427,11440,11443,11445-11446,11451-11454,11457,11460,11466,11469,11476,11478,11481,11508,11510,11514,11517,11525-11526,11537,11541,11543,11547,11551-11552,11555,11561-11562,11565,11570,11592-11593,11595-11596,11603,11606,11613-11614,11622,11633,11636,11658,11665,11668-11670,11673,11675-11678,11703,11709-11711,11716,11719,11723,11728,11736,11740-11742,11753,11755,11757,11774,11780,11783,11788,11797,11802,11809-11811,11815-11817,11819-11821,11827,11832,11839,11841,11850,11858,11862-11863,11869,11871,11878-11880,11882,11884,11889,11892"}}
+{"timestamp":1713278655.6720009,"name":"online","context":{"idset":"41,114,116,132,135,137,145,168,170,180,190,198,200,208,213,223,226,238,244,248,280,282,290,300-301,312,315,318,335,342,350-354,359,362,371,374,378,381,383,385,402,404,406,409,414,423,426,442,469,480,488,490,492,517,521,9207,9211-9212,9217,9227,9231,9234-9235,9238,9243-9245,9247-9249,9251,9254-9255,9260,9271,9278-9279,9288-9289,9300,9312,9319,9332,9542,9591,9616-9617,9620,9622-9625,9629,9631,9640,9645,9649,9652,9661,9675,9708-9709,9862,9864-9865,9871,9875,9880,9883,9885,9888,9891-9893,9896,9898-9901,9906-9907,9913,9916-9917,9923,9925,9930,9932-9934,9936-9939,9941,9944,9949-9951,9953,9958-9961,9969,10089-10090,10099,10112,10114,10119,10124,10126-10127,10132,10141,10146,10152,10171,10179,10185,10189,10199,10211,10215-10216,10220,10227-10229,10234-10235,10240-10241,10244-10245,10247,10249,10251-10252,10254,10256,10259-10260,10266,10268-10271,10273,10278-10279,10284-10285,10287-10288,10293-10294,10296-10299,10307,10313,10317,10319-10320,10326,10328,10331,10335,10338,10340,10343,10345,10347,10354,10356,10359,10361-10362,10364-10365,10367,10370-10371,10374,10377,10379-10380,10383-10384,10386,10388,10390-10392,10396,10405,10408-10410,10414,10418-10420,10426-10427,10429,10431,10433,10436-10437,10439-10440,10442,10445-10447,10449,10451-10453,10455,10457,10459,10461-10463,10476,10483,10613,10615-10620,10622,10624-10626,10628-10629,10642-10645,10647,10651,10655,10659,10665,10671,10674,10692,10694-10695,10700-10701,10706,10718-10719,10724-10725,10728,10739-10740,10747,10749,10754,10756-10757,10761-10762,10765,10768-10770,10772,10779,10781-10782,10786,10788,10790,10793-10794,10796,10798,10800,10802,10811-10812,10816,10818,10821,10823-10824,10826-10828,10830,10832,10837-10840,10844-10846,10848-10850,10852-10853,10855,10857,10860-10861,10866-10867,10869,10873,10875-10877,10879,10887,10890-10891,10895,10899,10903,10906-10907,10920,10925,10930-10932,10934-10936,10938,10940-10942,10947-10948,10951,10954-10956,10959,10961,10969,10973-10975,10982-10983,10985,10989-10990,10996,11017-11018,11021-11022,11027,11032,11034,11042,11046,11050,11052,11060,11062,11064,11067-11068,11081-11083,11085,11087-11088,11090,11097-11098,11100-11103,11108-11109,11113,11117-11120,11123-11124,11130,11138-11140,11144-11145,11148,11150,11152-11153,11155,11162,11166,11170-11171,11176,11180,11186,11189,11195,11199,11202,11208,11213,11216-11217,11226,11229-11230,11234,11236,11239,11244,11246-11247,11250-11251,11305-11307,11309,11313-11316,11336,11338,11340-11341,11344-11345,11348-11349,11355,11358-11360,11364,11367,11369,11377-11379,11381-11383,11385,11389,11391-11392,11398-11399,11401-11403,11407,11411,11413,11417-11418,11425,11428,11430-11431,11436,11442,11444,11450,11458,11463,11465,11475,11477,11480,11483-11485,11487,11490,11492,11494-11495,11497,11499-11500,11503,11506-11507,11512,11515,11519,11522-11524,11527-11530,11532,11536,11539,11542,11545-11546,11548-11550,11554,11558,11560,11564,11571,11589,11591,11598,11600-11602,11604-11605,11610-11611,11615-11616,11623-11625,11628-11629,11654,11656-11657,11662,11666-11667,11672,11674,11679-11681,11683-11684,11701-11702,11704,11707-11708,11712-11715,11722,11726-11727,11730,11735,11739,11745-11747,11749-11751,11754,11756,11759,11761,11765-11766,11769,11772,11775,11778,11781-11782,11784,11787,11792,11798-11799,11806,11808,11818,11824-11826,11828-11830,11836-11838,11844,11847,11849,11852,11855,11860-11861,11868,11870,11876,11883,11885-11888,11890"}}
+{"timestamp":1713278655.807452,"name":"online","context":{"idset":"37,39,44-46,52,87,93-94,99-101,108,117,120,122,128-129,131,138,148,157-158,160,167,177-178,191,193-196,202,206-207,214-215,218,220,231-232,237,249-250,303,305,307,310,314,319,322,324,334,337,339,346,360,367,369,372,377,390,392,395-396,405,407,410-411,417,427,431,435-436,441,470-472,477,483,485-486,520,522,9209,9213,9220-9221,9223,9225,9228,9239,9241,9253,9276,9297,9317,9554,9619,9630,9632,9637-9639,9641,9655-9657,9659,9677,9679,9681,9685,9688-9690,9695,9697,9701-9703,9707,9849,9855,9863,9867,9874,9876-9878,9881,9884,9894,9903,9919-9921,9927,9929,9935,9940,9942,9945,9956-9957,9963,9965,9971-9972,10087-10088,10091,10101-10108,10111,10113,10115-10116,10121-10123,10125,10128-10129,10131,10135,10137-10140,10143-10145,10149,10151,10165-10169,10173,10175,10178,10180-10184,10186,10188,10193,10195-10196,10200-10202,10204-10210,10212-10214,10222,10224,10239,10243,10263-10265,10277,10281,10289,10291-10292,10301,10309,10311,10315,10318,10321-10322,10325,10337,10339,10342,10344,10349,10353,10355,10363,10366,10369,10389,10393,10395,10401,10403,10407,10421,10423,10441,10454,10472-10473,10630,10637,10646,10650,10667,10672-10673,10675-10677,10680-10682,10684-10687,10689-10690,10697-10698,10704,10708-10713,10715-10717,10722,10726-10727,10729,10731,10733-10735,10752,10759,10777-10778,10784,10792,10803,10813,10825,10882,10889,10893-10894,10896,10901,10910,10929,10943,10952,10957,10971,10976,10980,11013-11016,11019-11020,11023-11026,11029,11031,11033,11035,11037-11038,11040-11041,11043-11045,11047-11049,11051,11053-11054,11056-11059,11061,11063,11065-11066,11069,11071,11073,11080,11091,11094,11129,11131,11134,11142-11143,11146-11147,11149,11154,11156-11158,11165,11169,11172-11174,11178,11182,11184-11185,11188,11190,11193,11196,11201,11205-11206,11210-11211,11214-11215,11218,11221-11223,11243,11304,11310-11312,11335,11372,11387,11395,11419,11434,11448,11467,11471,11482,11488,11496,11505,11511,11521,11538,11540,11566,11568,11599,11607,11618,11620,11631-11632,11635,11655,11660-11661,11725,11729,11732-11733,11744,11760,11776-11777,11789,11812,11814,11822,11833,11842,11848,11853,11864,11867"}}
+{"timestamp":1713278655.9536591,"name":"online","context":{"idset":"23,34,51,96,107,112-113,127,130,146,172,174,181,184,186,210,212,224,227,236,289,292,320-321,332,343,349,370,376,384,388,393,408,422,424,434,438,444,476,479,523,525-526,9621,9663,9943,10100,10120,10130,10133-10134,10136,10148,10191-10192,10638,10670,10699,10703,10714,10738,11132-11133,11137,11183,11187,11197,11200,11220"}}
+{"timestamp":1713278656.0714815,"name":"online","context":{"idset":"1,3-4,10-11,16,18,24,30,38,88-92,95,97-98,102-106,110,124-125,133-134,140,142-143,151,153-156,161-165,171,179,185,187,189,199,219,228,233,239,241,243,245,247,277,279,284-285,287-288,293-294,296-297,306,308-309,316-317,323,325-327,330-331,333,338,355-357,363-364,366,368,379-380,387,391,399,418,421,425,428,439,474,481,484,524,9647,9665,9667,10177,10187,10678,10707,11160,11164"}}
+{"timestamp":1713278656.2088354,"name":"online","context":{"idset":"6,9,14,17,19-20,22,25-26,29,33,36,43,47,49-50,53-54,57-59,159,173,197,216,229-230,251,298,375,412,430,437"}}
+{"timestamp":1713278656.3385129,"name":"online","context":{"idset":"5,13,15,21,27,35,40,55-56,150,201,278,291"}}
+{"timestamp":1713278656.4609592,"name":"online","context":{"idset":"2,7,12,28,31-32,42,48"}}
+{"timestamp":1713278656.5948308,"name":"online","context":{"idset":"8,60"}}
+{"timestamp":1713279355.6963561,"name":"offline","context":{"idset":"10411"}}
+{"timestamp":1713280105.6877241,"name":"offline","context":{"idset":"852"}}
+{"timestamp":1713280987.596895,"name":"offline","context":{"idset":"9213"}}
+{"timestamp":1713280987.6894116,"name":"offline","context":{"idset":"9214"}}
+{"timestamp":1713280989.6891427,"name":"offline","context":{"idset":"9241"}}
+{"timestamp":1713283817.6899226,"name":"offline","context":{"idset":"837"}}
+{"timestamp":1713284765.004673,"name":"undrain","context":{"idset":"9991"}}
+{"timestamp":1713285041.6903281,"name":"offline","context":{"idset":"941"}}
+{"timestamp":1713285188.0521011,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1713285259.6082146,"name":"offline","context":{"idset":"8949"}}
+{"timestamp":1713285259.6233268,"name":"offline","context":{"idset":"8950"}}
+{"timestamp":1713285259.6395657,"name":"offline","context":{"idset":"8965"}}
+{"timestamp":1713285259.6900148,"name":"offline","context":{"idset":"8966"}}
+{"timestamp":1713285261.6267807,"name":"offline","context":{"idset":"8981"}}
+{"timestamp":1713285261.6454723,"name":"offline","context":{"idset":"8982"}}
+{"timestamp":1713285261.6641426,"name":"offline","context":{"idset":"8998"}}
+{"timestamp":1713285261.6820083,"name":"offline","context":{"idset":"9013"}}
+{"timestamp":1713285261.6988196,"name":"offline","context":{"idset":"9014"}}
+{"timestamp":1713285261.7167814,"name":"offline","context":{"idset":"9029"}}
+{"timestamp":1713285261.7333167,"name":"offline","context":{"idset":"9030"}}
+{"timestamp":1713285261.7564662,"name":"offline","context":{"idset":"9045"}}
+{"timestamp":1713285261.774826,"name":"offline","context":{"idset":"9046"}}
+{"timestamp":1713285261.7931645,"name":"offline","context":{"idset":"9061"}}
+{"timestamp":1713285261.8115561,"name":"offline","context":{"idset":"9062"}}
+{"timestamp":1713285309.6884046,"name":"offline","context":{"idset":"8997"}}
+{"timestamp":1713285365.6224272,"name":"offline","context":{"idset":"8951"}}
+{"timestamp":1713285365.6339786,"name":"offline","context":{"idset":"8952"}}
+{"timestamp":1713285365.6452134,"name":"offline","context":{"idset":"8967"}}
+{"timestamp":1713285365.6566253,"name":"offline","context":{"idset":"8968"}}
+{"timestamp":1713285365.6677945,"name":"offline","context":{"idset":"8983"}}
+{"timestamp":1713285365.6791072,"name":"offline","context":{"idset":"8984"}}
+{"timestamp":1713285365.6901608,"name":"offline","context":{"idset":"8999"}}
+{"timestamp":1713285365.7013044,"name":"offline","context":{"idset":"9000"}}
+{"timestamp":1713285365.7117262,"name":"offline","context":{"idset":"9015"}}
+{"timestamp":1713285365.7237606,"name":"offline","context":{"idset":"9016"}}
+{"timestamp":1713285365.7354784,"name":"offline","context":{"idset":"9031"}}
+{"timestamp":1713285365.7515123,"name":"offline","context":{"idset":"9032"}}
+{"timestamp":1713285365.762399,"name":"offline","context":{"idset":"9047"}}
+{"timestamp":1713285365.7734704,"name":"offline","context":{"idset":"9048"}}
+{"timestamp":1713285365.7843173,"name":"offline","context":{"idset":"9063"}}
+{"timestamp":1713285365.7950156,"name":"offline","context":{"idset":"9064"}}
+{"timestamp":1713285585.690129,"name":"offline","context":{"idset":"10307"}}
+{"timestamp":1713285625.6124587,"name":"offline","context":{"idset":"8953"}}
+{"timestamp":1713285625.622395,"name":"offline","context":{"idset":"8969"}}
+{"timestamp":1713285625.6322622,"name":"offline","context":{"idset":"8970"}}
+{"timestamp":1713285625.6426375,"name":"offline","context":{"idset":"8985"}}
+{"timestamp":1713285625.651737,"name":"offline","context":{"idset":"8986"}}
+{"timestamp":1713285625.6615539,"name":"offline","context":{"idset":"9001"}}
+{"timestamp":1713285625.6712558,"name":"offline","context":{"idset":"9002"}}
+{"timestamp":1713285625.6809671,"name":"offline","context":{"idset":"9033"}}
+{"timestamp":1713285625.6906083,"name":"offline","context":{"idset":"9049"}}
+{"timestamp":1713285625.7010911,"name":"offline","context":{"idset":"9050"}}
+{"timestamp":1713285625.7107692,"name":"offline","context":{"idset":"9065"}}
+{"timestamp":1713285625.7258852,"name":"offline","context":{"idset":"9066"}}
+{"timestamp":1713285648.6670551,"name":"offline","context":{"idset":"8955"}}
+{"timestamp":1713285648.684602,"name":"offline","context":{"idset":"8956"}}
+{"timestamp":1713285648.7027116,"name":"offline","context":{"idset":"8971"}}
+{"timestamp":1713285648.7282062,"name":"offline","context":{"idset":"8972"}}
+{"timestamp":1713285648.7428317,"name":"offline","context":{"idset":"8987"}}
+{"timestamp":1713285648.7605145,"name":"offline","context":{"idset":"8988"}}
+{"timestamp":1713285648.7741525,"name":"offline","context":{"idset":"9003"}}
+{"timestamp":1713285648.7904348,"name":"offline","context":{"idset":"9004"}}
+{"timestamp":1713285648.8078148,"name":"offline","context":{"idset":"9019"}}
+{"timestamp":1713285648.8244207,"name":"offline","context":{"idset":"9020"}}
+{"timestamp":1713285648.8397222,"name":"offline","context":{"idset":"9035"}}
+{"timestamp":1713285648.8566883,"name":"offline","context":{"idset":"9036"}}
+{"timestamp":1713285648.872205,"name":"offline","context":{"idset":"9051"}}
+{"timestamp":1713285648.90097,"name":"offline","context":{"idset":"9052"}}
+{"timestamp":1713285648.9170787,"name":"offline","context":{"idset":"9067"}}
+{"timestamp":1713285648.9340901,"name":"offline","context":{"idset":"9068"}}
+{"timestamp":1713285673.5980699,"name":"offline","context":{"idset":"8954"}}
+{"timestamp":1713285673.6882229,"name":"offline","context":{"idset":"9034"}}
+{"timestamp":1713285715.6162267,"name":"offline","context":{"idset":"8957"}}
+{"timestamp":1713285715.6270018,"name":"offline","context":{"idset":"8958"}}
+{"timestamp":1713285715.6381869,"name":"offline","context":{"idset":"8973"}}
+{"timestamp":1713285715.6493969,"name":"offline","context":{"idset":"8974"}}
+{"timestamp":1713285715.6604595,"name":"offline","context":{"idset":"8989"}}
+{"timestamp":1713285715.6715891,"name":"offline","context":{"idset":"8990"}}
+{"timestamp":1713285715.6817298,"name":"offline","context":{"idset":"9006"}}
+{"timestamp":1713285715.6914856,"name":"offline","context":{"idset":"9021"}}
+{"timestamp":1713285715.7074232,"name":"offline","context":{"idset":"9022"}}
+{"timestamp":1713285715.7269988,"name":"offline","context":{"idset":"9037"}}
+{"timestamp":1713285715.7381048,"name":"offline","context":{"idset":"9038"}}
+{"timestamp":1713285715.7484286,"name":"offline","context":{"idset":"9053"}}
+{"timestamp":1713285715.7585716,"name":"offline","context":{"idset":"9054"}}
+{"timestamp":1713285715.7689965,"name":"offline","context":{"idset":"9069"}}
+{"timestamp":1713285715.7815199,"name":"offline","context":{"idset":"9070"}}
+{"timestamp":1713285721.6175053,"name":"offline","context":{"idset":"8959"}}
+{"timestamp":1713285721.6272264,"name":"offline","context":{"idset":"8960"}}
+{"timestamp":1713285721.6369367,"name":"offline","context":{"idset":"8975"}}
+{"timestamp":1713285721.6466258,"name":"offline","context":{"idset":"8976"}}
+{"timestamp":1713285721.6563122,"name":"offline","context":{"idset":"8991"}}
+{"timestamp":1713285721.6659029,"name":"offline","context":{"idset":"8992"}}
+{"timestamp":1713285721.6769907,"name":"offline","context":{"idset":"9007"}}
+{"timestamp":1713285721.6848056,"name":"offline","context":{"idset":"9008"}}
+{"timestamp":1713285721.6942194,"name":"offline","context":{"idset":"9023"}}
+{"timestamp":1713285721.7097056,"name":"offline","context":{"idset":"9024"}}
+{"timestamp":1713285721.7200971,"name":"offline","context":{"idset":"9039"}}
+{"timestamp":1713285721.7304845,"name":"offline","context":{"idset":"9040"}}
+{"timestamp":1713285721.7408669,"name":"offline","context":{"idset":"9055"}}
+{"timestamp":1713285721.7512505,"name":"offline","context":{"idset":"9056"}}
+{"timestamp":1713285721.7628324,"name":"offline","context":{"idset":"9071"}}
+{"timestamp":1713285721.7723465,"name":"offline","context":{"idset":"9072"}}
+{"timestamp":1713285727.6165261,"name":"offline","context":{"idset":"8961"}}
+{"timestamp":1713285727.6259668,"name":"offline","context":{"idset":"8962"}}
+{"timestamp":1713285727.6353374,"name":"offline","context":{"idset":"8977"}}
+{"timestamp":1713285727.6446459,"name":"offline","context":{"idset":"8978"}}
+{"timestamp":1713285727.6539822,"name":"offline","context":{"idset":"8993"}}
+{"timestamp":1713285727.6633325,"name":"offline","context":{"idset":"8994"}}
+{"timestamp":1713285727.6725042,"name":"offline","context":{"idset":"9009"}}
+{"timestamp":1713285727.6817908,"name":"offline","context":{"idset":"9010"}}
+{"timestamp":1713285727.6960435,"name":"offline","context":{"idset":"9025"}}
+{"timestamp":1713285727.7067719,"name":"offline","context":{"idset":"9026"}}
+{"timestamp":1713285727.7166514,"name":"offline","context":{"idset":"9041"}}
+{"timestamp":1713285727.7265074,"name":"offline","context":{"idset":"9042"}}
+{"timestamp":1713285727.7363274,"name":"offline","context":{"idset":"9057"}}
+{"timestamp":1713285727.745997,"name":"offline","context":{"idset":"9058"}}
+{"timestamp":1713285727.7551358,"name":"offline","context":{"idset":"9073"}}
+{"timestamp":1713285727.7651446,"name":"offline","context":{"idset":"9074"}}
+{"timestamp":1713285731.6406732,"name":"offline","context":{"idset":"8963"}}
+{"timestamp":1713285731.6573937,"name":"offline","context":{"idset":"8964"}}
+{"timestamp":1713285731.6713173,"name":"offline","context":{"idset":"8979"}}
+{"timestamp":1713285731.68607,"name":"offline","context":{"idset":"8980"}}
+{"timestamp":1713285731.7046068,"name":"offline","context":{"idset":"8995"}}
+{"timestamp":1713285731.7195172,"name":"offline","context":{"idset":"8996"}}
+{"timestamp":1713285731.7360623,"name":"offline","context":{"idset":"9011"}}
+{"timestamp":1713285731.754324,"name":"offline","context":{"idset":"9012"}}
+{"timestamp":1713285731.768913,"name":"offline","context":{"idset":"9027"}}
+{"timestamp":1713285731.7938986,"name":"offline","context":{"idset":"9028"}}
+{"timestamp":1713285731.8124738,"name":"offline","context":{"idset":"9043"}}
+{"timestamp":1713285731.8317952,"name":"offline","context":{"idset":"9044"}}
+{"timestamp":1713285731.8511999,"name":"offline","context":{"idset":"9059"}}
+{"timestamp":1713285731.8640139,"name":"offline","context":{"idset":"9060"}}
+{"timestamp":1713285731.8803499,"name":"offline","context":{"idset":"9076"}}
+{"timestamp":1713285763.6883268,"name":"offline","context":{"idset":"9005"}}
+{"timestamp":1713286271.691222,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1713287290.4177001,"name":"online","context":{"idset":"9991"}}
+{"timestamp":1713288465.5971017,"name":"drain","context":{"idset":"9991","reason":"epilog failed for jobid fpsoSfRBccP","overwrite":0}}
+{"timestamp":1713288465.6927204,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1713288469.6908736,"name":"offline","context":{"idset":"9992"}}
+{"timestamp":1713290620.3120332,"name":"drain","context":{"idset":"525-532","overwrite":0}}
+{"timestamp":1713290671.6290429,"name":"online","context":{"idset":"11008"}}
+{"timestamp":1713290671.7836831,"name":"online","context":{"idset":"10997,10999,11007"}}
+{"timestamp":1713290671.8232009,"name":"online","context":{"idset":"11012"}}
+{"timestamp":1713290672.1215045,"name":"online","context":{"idset":"11001,11005,11010"}}
+{"timestamp":1713290672.3258436,"name":"online","context":{"idset":"10998,11006"}}
+{"timestamp":1713290672.7675321,"name":"online","context":{"idset":"11009,11011"}}
+{"timestamp":1713290672.8958163,"name":"online","context":{"idset":"11002"}}
+{"timestamp":1713290673.1282461,"name":"online","context":{"idset":"11003"}}
+{"timestamp":1713290673.3811133,"name":"online","context":{"idset":"11000"}}
+{"timestamp":1713290674.1341057,"name":"online","context":{"idset":"11004"}}
+{"timestamp":1713290771.9623983,"name":"offline","context":{"idset":"531"}}
+{"timestamp":1713290771.9792838,"name":"offline","context":{"idset":"525"}}
+{"timestamp":1713290772.0037801,"name":"offline","context":{"idset":"532"}}
+{"timestamp":1713290772.0281873,"name":"offline","context":{"idset":"527"}}
+{"timestamp":1713290772.0670502,"name":"offline","context":{"idset":"529"}}
+{"timestamp":1713290772.0941467,"name":"offline","context":{"idset":"526"}}
+{"timestamp":1713290772.1498842,"name":"offline","context":{"idset":"528"}}
+{"timestamp":1713290835.4144058,"name":"drain","context":{"idset":"11125-11252","overwrite":0}}
+{"timestamp":1713290848.3846834,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1713290848.4034641,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1713290848.4140503,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1713290848.4361873,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1713290848.4485836,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1713290848.5336409,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1713290848.6125736,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1713290848.6234121,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1713290848.6444309,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1713290848.6567976,"name":"offline","context":{"idset":"11153"}}
+{"timestamp":1713290848.6660776,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1713290848.6943207,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1713290848.6970563,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1713290848.70473,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1713290848.7154584,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1713290848.7182307,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1713290848.7260082,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1713290848.7481949,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1713290848.7529464,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1713290848.7950156,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1713290848.8058167,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1713290848.8088095,"name":"offline","context":{"idset":"11149"}}
+{"timestamp":1713290848.8115704,"name":"offline","context":{"idset":"11154"}}
+{"timestamp":1713290848.8142061,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1713290848.8262858,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1713290848.854135,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1713290848.8715167,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1713290848.8741603,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1713290848.8767869,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1713290848.8798604,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1713290848.883822,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1713290848.8872211,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1713290848.9175589,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1713290848.9284403,"name":"offline","context":{"idset":"11146"}}
+{"timestamp":1713290848.9313955,"name":"offline","context":{"idset":"11150"}}
+{"timestamp":1713290848.941808,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1713290848.9522069,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1713290848.9562135,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1713290848.978538,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1713290849.0139358,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1713290849.0535607,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1713290849.0941548,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1713290849.0991373,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1713290849.1041248,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1713290849.1091075,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1713290849.1140733,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1713290849.119072,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1713290849.1240752,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1713290849.1291533,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1713290849.1341166,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1713290849.1391003,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1713290849.1441038,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1713290849.1492021,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1713290849.1542115,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1713290849.1929176,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1713290849.232764,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1713290849.2734673,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1713290849.3145437,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1713290849.3456552,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1713290849.3545649,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1713290849.411011,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1713290849.4374771,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1713290849.4426191,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1713290849.4592392,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1713290849.4642611,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1713290849.4812524,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1713290849.4862876,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1713290849.5021791,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1713290849.5113046,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1713290849.526046,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1713290849.5568001,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1713290849.5975332,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1713290849.6382139,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1713290849.6790535,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1713290849.7196751,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1713290849.7604635,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1713290849.7654855,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1713290849.7704632,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1713290849.7754397,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1713290849.7803745,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1713290849.785383,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1713290849.8063293,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1713290849.8343172,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1713290849.8543317,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1713290849.8593824,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1713290849.8747296,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1713290849.882751,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1713290849.8917885,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1713290849.8967264,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1713290849.9177132,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1713290849.9375706,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1713290850.0094481,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1713290850.0503361,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1713290850.0913286,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1713290850.1324434,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1713290850.17203,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1713290850.2177045,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1713290850.2673793,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1713290850.3164701,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1713290850.3625474,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1713290850.3872542,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1713290850.392441,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1713290850.397645,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1713290850.402817,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1713290850.4232972,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1713290850.4461901,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1713290850.4651117,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1713290850.4860623,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1713290850.5069506,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1713290850.5480182,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1713290850.5903409,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1713290850.6323979,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1713290850.6533678,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1713290850.6722519,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1713290850.6935182,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1713290850.714391,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1713290850.7352386,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1713290850.7703383,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1713290850.8025746,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1713290850.8349297,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1713290850.8666961,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1713290850.8984923,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1713290850.9299343,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1713290850.9581029,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1713290850.9828861,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1713290850.9972215,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1713290851.0004601,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1713290851.0035164,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1713291066.955596,"name":"online","context":{"idset":"7928"}}
+{"timestamp":1713291067.1617649,"name":"online","context":{"idset":"7927"}}
+{"timestamp":1713291329.4621787,"name":"online","context":{"idset":"763,765"}}
+{"timestamp":1713291329.7445819,"name":"online","context":{"idset":"766"}}
+{"timestamp":1713291329.96313,"name":"online","context":{"idset":"764"}}
+{"timestamp":1713291623.6807625,"name":"online","context":{"idset":"573"}}
+{"timestamp":1713291625.0494134,"name":"online","context":{"idset":"574-576,579"}}
+{"timestamp":1713291625.1619153,"name":"online","context":{"idset":"577-578,580"}}
+{"timestamp":1713291934.4875407,"name":"online","context":{"idset":"787"}}
+{"timestamp":1713291969.0208213,"name":"online","context":{"idset":"788"}}
+{"timestamp":1713292272.7649019,"name":"undrain","context":{"idset":"9533-9535"}}
+{"timestamp":1713293268.1101277,"name":"online","context":{"idset":"9534"}}
+{"timestamp":1713293268.6504278,"name":"online","context":{"idset":"9525,9530,9536"}}
+{"timestamp":1713293268.7886372,"name":"online","context":{"idset":"9532"}}
+{"timestamp":1713293268.9096439,"name":"online","context":{"idset":"9527,9539"}}
+{"timestamp":1713293269.0271075,"name":"online","context":{"idset":"9526,9535,9538"}}
+{"timestamp":1713293269.1529703,"name":"online","context":{"idset":"9528,9537"}}
+{"timestamp":1713293269.277638,"name":"online","context":{"idset":"9529,9531,9533,9540"}}
+{"timestamp":1713293412.2741399,"name":"drain","context":{"idset":"10997-11124","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1713293461.6900241,"name":"offline","context":{"idset":"764"}}
+{"timestamp":1713293962.1473153,"name":"online","context":{"idset":"11128"}}
+{"timestamp":1713293962.6052392,"name":"online","context":{"idset":"11148,11158,11186"}}
+{"timestamp":1713293962.7660203,"name":"online","context":{"idset":"11131,11144"}}
+{"timestamp":1713293962.8934541,"name":"online","context":{"idset":"11125"}}
+{"timestamp":1713293962.9161634,"name":"online","context":{"idset":"11127"}}
+{"timestamp":1713293963.0911078,"name":"online","context":{"idset":"11154"}}
+{"timestamp":1713293963.341584,"name":"online","context":{"idset":"11139"}}
+{"timestamp":1713293963.5592263,"name":"online","context":{"idset":"11162,11196"}}
+{"timestamp":1713293963.7339365,"name":"online","context":{"idset":"11223"}}
+{"timestamp":1713293963.8868544,"name":"online","context":{"idset":"11136,11173"}}
+{"timestamp":1713293964.0355489,"name":"online","context":{"idset":"11126,11209,11224"}}
+{"timestamp":1713293964.0607524,"name":"online","context":{"idset":"11155,11195"}}
+{"timestamp":1713293964.3445103,"name":"online","context":{"idset":"11208"}}
+{"timestamp":1713293964.4854465,"name":"online","context":{"idset":"11141,11187"}}
+{"timestamp":1713293964.6248102,"name":"online","context":{"idset":"11207"}}
+{"timestamp":1713293964.6526325,"name":"online","context":{"idset":"11130"}}
+{"timestamp":1713293964.6743007,"name":"online","context":{"idset":"11220"}}
+{"timestamp":1713293964.8034563,"name":"online","context":{"idset":"11234"}}
+{"timestamp":1713293964.8334551,"name":"online","context":{"idset":"11161,11182,11237"}}
+{"timestamp":1713293965.1083121,"name":"online","context":{"idset":"11152,11218"}}
+{"timestamp":1713293965.1430387,"name":"online","context":{"idset":"11252"}}
+{"timestamp":1713293965.2746739,"name":"online","context":{"idset":"11177,11192,11206,11231"}}
+{"timestamp":1713293965.301122,"name":"online","context":{"idset":"11190,11244"}}
+{"timestamp":1713293965.4076691,"name":"online","context":{"idset":"11200,11221,11228,11236,11241"}}
+{"timestamp":1713293965.5195096,"name":"online","context":{"idset":"11205,11212"}}
+{"timestamp":1713293965.6429305,"name":"online","context":{"idset":"11143,11145,11176,11185,11240"}}
+{"timestamp":1713293965.7652452,"name":"online","context":{"idset":"11163,11170,11181,11184,11226"}}
+{"timestamp":1713293966.0307169,"name":"online","context":{"idset":"11142,11174,11210"}}
+{"timestamp":1713293966.1465173,"name":"online","context":{"idset":"11135,11140,11149"}}
+{"timestamp":1713293966.2699649,"name":"online","context":{"idset":"11129,11133,11183,11188,11233"}}
+{"timestamp":1713293966.3851926,"name":"online","context":{"idset":"11153,11169,11180,11198,11246,11250"}}
+{"timestamp":1713293966.4981098,"name":"online","context":{"idset":"11179,11211"}}
+{"timestamp":1713293966.6129305,"name":"online","context":{"idset":"11132,11138,11168,11194,11197,11230,11238"}}
+{"timestamp":1713293966.8139217,"name":"online","context":{"idset":"11160,11172,11199,11201,11215,11225,11242,11249"}}
+{"timestamp":1713293966.9284904,"name":"online","context":{"idset":"11137,11146,11157,11191,11202-11203,11216-11217,11229"}}
+{"timestamp":1713293967.1367378,"name":"online","context":{"idset":"11134,11150,11164,11166,11178,11189,11193,11213,11227,11232,11251"}}
+{"timestamp":1713293967.2498875,"name":"online","context":{"idset":"11151,11219,11245"}}
+{"timestamp":1713293967.3506947,"name":"online","context":{"idset":"11156,11159,11165,11171,11204,11235,11239,11243"}}
+{"timestamp":1713293967.6506364,"name":"online","context":{"idset":"11167,11247"}}
+{"timestamp":1713293967.7678676,"name":"online","context":{"idset":"11214,11248"}}
+{"timestamp":1713293967.9673643,"name":"online","context":{"idset":"11147,11175"}}
+{"timestamp":1713293968.0809808,"name":"online","context":{"idset":"11222"}}
+{"timestamp":1713294368.2583389,"name":"undrain","context":{"idset":"11125-11252"}}
+{"timestamp":1713294503.6063724,"name":"offline","context":{"idset":"9461"}}
+{"timestamp":1713294503.6270809,"name":"offline","context":{"idset":"9462"}}
+{"timestamp":1713294503.6457593,"name":"offline","context":{"idset":"9464"}}
+{"timestamp":1713294503.6930535,"name":"offline","context":{"idset":"9467"}}
+{"timestamp":1713294983.6000521,"name":"offline","context":{"idset":"9483"}}
+{"timestamp":1713294983.7097898,"name":"offline","context":{"idset":"9486"}}
+{"timestamp":1713294987.766917,"name":"offline","context":{"idset":"9480"}}
+{"timestamp":1713295047.0072503,"name":"online","context":{"idset":"10488,10495,10548"}}
+{"timestamp":1713295047.2086167,"name":"online","context":{"idset":"10502"}}
+{"timestamp":1713295047.3510766,"name":"online","context":{"idset":"10497,10553"}}
+{"timestamp":1713295047.4889112,"name":"online","context":{"idset":"10519,10543"}}
+{"timestamp":1713295047.6929924,"name":"online","context":{"idset":"10516,10527,10538"}}
+{"timestamp":1713295047.8368027,"name":"online","context":{"idset":"10547"}}
+{"timestamp":1713295048.1759918,"name":"online","context":{"idset":"10513"}}
+{"timestamp":1713295048.3277519,"name":"online","context":{"idset":"10525,10541"}}
+{"timestamp":1713295048.8059292,"name":"online","context":{"idset":"10503,10507,10524"}}
+{"timestamp":1713295048.8340037,"name":"online","context":{"idset":"10573"}}
+{"timestamp":1713295048.9967084,"name":"online","context":{"idset":"10492,10508"}}
+{"timestamp":1713295049.0188496,"name":"online","context":{"idset":"10490"}}
+{"timestamp":1713295049.1951363,"name":"online","context":{"idset":"10531"}}
+{"timestamp":1713295049.3351736,"name":"online","context":{"idset":"10496,10505"}}
+{"timestamp":1713295049.4805448,"name":"online","context":{"idset":"10504,10563"}}
+{"timestamp":1713295049.5020645,"name":"online","context":{"idset":"10486"}}
+{"timestamp":1713295049.7183962,"name":"online","context":{"idset":"10570"}}
+{"timestamp":1713295049.7428923,"name":"online","context":{"idset":"10554"}}
+{"timestamp":1713295049.8582261,"name":"online","context":{"idset":"10493"}}
+{"timestamp":1713295049.8863878,"name":"online","context":{"idset":"10509,10556,10606"}}
+{"timestamp":1713295050.0467646,"name":"online","context":{"idset":"10494,10511,10560"}}
+{"timestamp":1713295050.1650226,"name":"online","context":{"idset":"10597"}}
+{"timestamp":1713295050.2795455,"name":"online","context":{"idset":"10498,10551,10569"}}
+{"timestamp":1713295050.3949649,"name":"online","context":{"idset":"10518,10539,10565,10579"}}
+{"timestamp":1713295050.5101557,"name":"online","context":{"idset":"10523,10533,10555"}}
+{"timestamp":1713295050.6252241,"name":"online","context":{"idset":"10491,10529,10535-10536,10564,10568,10576,10591"}}
+{"timestamp":1713295050.852741,"name":"online","context":{"idset":"10485,10489,10501,10512,10514,10517,10520,10522,10526,10581,10584,10593,10599,10607,10612"}}
+{"timestamp":1713295051.0737491,"name":"online","context":{"idset":"10487,10530,10542,10561,10575,10598,10601,10608"}}
+{"timestamp":1713295051.1891992,"name":"online","context":{"idset":"10499,10545,10589"}}
+{"timestamp":1713295051.303339,"name":"online","context":{"idset":"10500,10506,10574,10596,10611"}}
+{"timestamp":1713295051.4174986,"name":"online","context":{"idset":"10510,10521,10534,10558-10559,10595,10605"}}
+{"timestamp":1713295051.5346701,"name":"online","context":{"idset":"10515,10532,10549,10566,10588,10600"}}
+{"timestamp":1713295051.6490228,"name":"online","context":{"idset":"10544,10557,10562,10567,10572,10577,10580,10585,10609"}}
+{"timestamp":1713295051.7677474,"name":"online","context":{"idset":"10540,10550,10552,10571,10578,10582-10583,10586-10587,10594,10602-10603"}}
+{"timestamp":1713295052.0009277,"name":"online","context":{"idset":"10528,10590,10592,10610"}}
+{"timestamp":1713295052.2220063,"name":"online","context":{"idset":"10604"}}
+{"timestamp":1713295052.3368695,"name":"online","context":{"idset":"10537"}}
+{"timestamp":1713295077.6895783,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1713295584.6612587,"name":"undrain","context":{"idset":"9788-9789"}}
+{"timestamp":1713295703.6110506,"name":"online","context":{"idset":"10307"}}
+{"timestamp":1713295704.3825655,"name":"online","context":{"idset":"10306"}}
+{"timestamp":1713295704.9099152,"name":"online","context":{"idset":"10305"}}
+{"timestamp":1713295789.6055441,"name":"offline","context":{"idset":"9781"}}
+{"timestamp":1713295789.621115,"name":"offline","context":{"idset":"9782"}}
+{"timestamp":1713295789.6427317,"name":"offline","context":{"idset":"9783"}}
+{"timestamp":1713295789.6895993,"name":"offline","context":{"idset":"9784"}}
+{"timestamp":1713295791.6245475,"name":"offline","context":{"idset":"9785"}}
+{"timestamp":1713295791.6445446,"name":"offline","context":{"idset":"9786"}}
+{"timestamp":1713295791.6708722,"name":"offline","context":{"idset":"9791"}}
+{"timestamp":1713295791.690834,"name":"offline","context":{"idset":"9792"}}
+{"timestamp":1713295791.7143433,"name":"offline","context":{"idset":"9793"}}
+{"timestamp":1713295791.7342517,"name":"offline","context":{"idset":"9794"}}
+{"timestamp":1713295791.7541602,"name":"offline","context":{"idset":"9795"}}
+{"timestamp":1713295791.775259,"name":"offline","context":{"idset":"9796"}}
+{"timestamp":1713295869.6884511,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1713296233.30774,"name":"online","context":{"idset":"903"}}
+{"timestamp":1713296313.0669224,"name":"online","context":{"idset":"11203"}}
+{"timestamp":1713298399.8146825,"name":"online","context":{"idset":"833"}}
+{"timestamp":1713298400.2463253,"name":"online","context":{"idset":"834"}}
+{"timestamp":1713298424.7476943,"name":"online","context":{"idset":"851-852"}}
+{"timestamp":1713298500.3204994,"name":"online","context":{"idset":"9786"}}
+{"timestamp":1713298500.4506373,"name":"online","context":{"idset":"9788"}}
+{"timestamp":1713298500.669508,"name":"online","context":{"idset":"9787,9791,9793-9794"}}
+{"timestamp":1713298500.7843473,"name":"online","context":{"idset":"9781"}}
+{"timestamp":1713298501.0004203,"name":"online","context":{"idset":"9782-9784,9795-9796"}}
+{"timestamp":1713298501.2153997,"name":"online","context":{"idset":"9785,9792"}}
+{"timestamp":1713298848.836632,"name":"online","context":{"idset":"10546"}}
+{"timestamp":1713300453.9318366,"name":"drain","context":{"idset":"69","reason":"New Blade Installation","overwrite":0}}
+{"timestamp":1713301655.7741578,"name":"online","context":{"idset":"10162"}}
+{"timestamp":1713301655.9764774,"name":"online","context":{"idset":"10161"}}
+{"timestamp":1713301704.3912609,"name":"online","context":{"idset":"11641"}}
+{"timestamp":1713301704.6711714,"name":"online","context":{"idset":"11642"}}
+{"timestamp":1713302209.5982623,"name":"offline","context":{"idset":"843"}}
+{"timestamp":1713302209.6912262,"name":"offline","context":{"idset":"844"}}
+{"timestamp":1713302214.6664815,"name":"drain","context":{"idset":"11637-11764","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1713302316.5490866,"name":"undrain","context":{"idset":"11637-11764"}}
+{"timestamp":1713302549.7014897,"name":"drain","context":{"idset":"11637-11764","reason":"draining to run on-node HPE diags - KPN\n","overwrite":0}}
+{"timestamp":1713302660.4036105,"name":"undrain","context":{"idset":"11637-11764"}}
+{"timestamp":1713302661.0644612,"name":"online","context":{"idset":"9465"}}
+{"timestamp":1713302661.2683942,"name":"online","context":{"idset":"9461"}}
+{"timestamp":1713302661.4948254,"name":"online","context":{"idset":"9464,9469,9473"}}
+{"timestamp":1713302661.6397593,"name":"online","context":{"idset":"9470"}}
+{"timestamp":1713302661.8226583,"name":"online","context":{"idset":"9468,9471"}}
+{"timestamp":1713302661.9498072,"name":"online","context":{"idset":"9467,9472"}}
+{"timestamp":1713302662.1698356,"name":"online","context":{"idset":"9462-9463,9466,9474-9476"}}
+{"timestamp":1713302764.9184775,"name":"online","context":{"idset":"9478"}}
+{"timestamp":1713302765.1519382,"name":"online","context":{"idset":"9480"}}
+{"timestamp":1713302765.3085904,"name":"online","context":{"idset":"9477,9479"}}
+{"timestamp":1713302765.4562137,"name":"online","context":{"idset":"9484-9485,9489"}}
+{"timestamp":1713302795.7318168,"name":"undrain","context":{"idset":"10997-11012"}}
+{"timestamp":1713302818.6329286,"name":"drain","context":{"idset":"533-540","reason":"New Blade Install --JRG","overwrite":0}}
+{"timestamp":1713302823.0647831,"name":"drain","context":{"idset":"11637-11652","reason":"draining to run on-node HPE diags - KPN\n","overwrite":0}}
+{"timestamp":1713302847.452316,"name":"offline","context":{"idset":"540"}}
+{"timestamp":1713302847.5609937,"name":"offline","context":{"idset":"534"}}
+{"timestamp":1713302847.5840552,"name":"offline","context":{"idset":"535"}}
+{"timestamp":1713302847.6526537,"name":"offline","context":{"idset":"539"}}
+{"timestamp":1713302847.6861076,"name":"offline","context":{"idset":"533"}}
+{"timestamp":1713302847.71942,"name":"offline","context":{"idset":"536"}}
+{"timestamp":1713302847.756304,"name":"offline","context":{"idset":"538"}}
+{"timestamp":1713302915.9419322,"name":"undrain","context":{"idset":"11637-11652"}}
+{"timestamp":1713303012.0250208,"name":"drain","context":{"idset":"11637-11652","reason":"draining to run on-node HPE diags - KPN\n","overwrite":0}}
+{"timestamp":1713303075.1291461,"name":"undrain","context":{"idset":"11637-11652"}}
+{"timestamp":1713303082.7746553,"name":"undrain","context":{"idset":"11013-11124"}}
+{"timestamp":1713303120.2577186,"name":"drain","context":{"idset":"11637-11652","reason":"draining to run on-node HPE diags - KPN\n","overwrite":0}}
+{"timestamp":1713303305.5422003,"name":"undrain","context":{"idset":"11637-11652"}}
+{"timestamp":1713303468.1921523,"name":"offline","context":{"idset":"11641"}}
+{"timestamp":1713303468.2911742,"name":"offline","context":{"idset":"11642"}}
+{"timestamp":1713303469.1005991,"name":"offline","context":{"idset":"11637"}}
+{"timestamp":1713303469.5959868,"name":"offline","context":{"idset":"11638"}}
+{"timestamp":1713303469.6901441,"name":"offline","context":{"idset":"11640"}}
+{"timestamp":1713303469.7725575,"name":"offline","context":{"idset":"11639"}}
+{"timestamp":1713303469.7793028,"name":"offline","context":{"idset":"11645"}}
+{"timestamp":1713303469.7953463,"name":"offline","context":{"idset":"11647"}}
+{"timestamp":1713303469.8100619,"name":"offline","context":{"idset":"11648"}}
+{"timestamp":1713303469.8583753,"name":"offline","context":{"idset":"11652"}}
+{"timestamp":1713303469.8743503,"name":"offline","context":{"idset":"11643"}}
+{"timestamp":1713303469.8904383,"name":"offline","context":{"idset":"11644"}}
+{"timestamp":1713303469.9071896,"name":"offline","context":{"idset":"11646"}}
+{"timestamp":1713303469.9320004,"name":"offline","context":{"idset":"11649"}}
+{"timestamp":1713303469.9354806,"name":"offline","context":{"idset":"11650"}}
+{"timestamp":1713303469.9488873,"name":"offline","context":{"idset":"11651"}}
+{"timestamp":1713303525.3898511,"name":"online","context":{"idset":"11641"}}
+{"timestamp":1713303526.0784543,"name":"online","context":{"idset":"11642"}}
+{"timestamp":1713303546.1168609,"name":"online","context":{"idset":"11639,11648"}}
+{"timestamp":1713303547.0437472,"name":"online","context":{"idset":"11637"}}
+{"timestamp":1713303547.1586783,"name":"online","context":{"idset":"11650"}}
+{"timestamp":1713303547.4227254,"name":"online","context":{"idset":"11638,11647"}}
+{"timestamp":1713303547.9779277,"name":"online","context":{"idset":"11652"}}
+{"timestamp":1713303548.2078471,"name":"online","context":{"idset":"11640,11649"}}
+{"timestamp":1713303548.3238208,"name":"online","context":{"idset":"11644"}}
+{"timestamp":1713303548.5355558,"name":"online","context":{"idset":"11645-11646,11651"}}
+{"timestamp":1713303548.7588027,"name":"online","context":{"idset":"11643"}}
+{"timestamp":1713303787.0957325,"name":"undrain","context":{"idset":"9991"}}
+{"timestamp":1713303942.6862326,"name":"online","context":{"idset":"9991"}}
+{"timestamp":1713303943.2729583,"name":"online","context":{"idset":"9992"}}
+{"timestamp":1713304012.3797235,"name":"drain","context":{"idset":"11641","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713304150.9483166,"name":"undrain","context":{"idset":"7927"}}
+{"timestamp":1713305180.8829193,"name":"drain","context":{"idset":"517-524","reason":"New Blade Installation --JRG","overwrite":0}}
+{"timestamp":1713305347.9201958,"name":"offline","context":{"idset":"517"}}
+{"timestamp":1713305347.9491959,"name":"offline","context":{"idset":"520"}}
+{"timestamp":1713305347.9688454,"name":"offline","context":{"idset":"521"}}
+{"timestamp":1713305347.9876199,"name":"offline","context":{"idset":"522"}}
+{"timestamp":1713305348.0053604,"name":"offline","context":{"idset":"523"}}
+{"timestamp":1713305348.0287352,"name":"offline","context":{"idset":"524"}}
+{"timestamp":1713305564.2167971,"name":"online","context":{"idset":"11557"}}
+{"timestamp":1713305564.9346542,"name":"online","context":{"idset":"11558"}}
+{"timestamp":1713305769.688833,"name":"offline","context":{"idset":"10396"}}
+{"timestamp":1713305858.1988056,"name":"drain","context":{"idset":"11642","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713306375.5980546,"name":"offline","context":{"idset":"9589"}}
+{"timestamp":1713306375.6108816,"name":"offline","context":{"idset":"9590"}}
+{"timestamp":1713306375.6896865,"name":"offline","context":{"idset":"9591"}}
+{"timestamp":1713306377.626364,"name":"offline","context":{"idset":"9593"}}
+{"timestamp":1713306377.6663048,"name":"offline","context":{"idset":"9594"}}
+{"timestamp":1713306377.6937485,"name":"offline","context":{"idset":"9595"}}
+{"timestamp":1713306377.7312322,"name":"offline","context":{"idset":"9596"}}
+{"timestamp":1713306377.7700739,"name":"offline","context":{"idset":"9597"}}
+{"timestamp":1713306379.6058502,"name":"offline","context":{"idset":"9598"}}
+{"timestamp":1713306379.626502,"name":"offline","context":{"idset":"9599"}}
+{"timestamp":1713306379.6467531,"name":"offline","context":{"idset":"9600"}}
+{"timestamp":1713306379.6882408,"name":"offline","context":{"idset":"9601"}}
+{"timestamp":1713306381.6037731,"name":"offline","context":{"idset":"9602"}}
+{"timestamp":1713306381.6191289,"name":"offline","context":{"idset":"9603"}}
+{"timestamp":1713306381.6331735,"name":"offline","context":{"idset":"9604"}}
+{"timestamp":1713306381.7220447,"name":"offline","context":{"idset":"9605"}}
+{"timestamp":1713306383.6022849,"name":"offline","context":{"idset":"9606"}}
+{"timestamp":1713306383.6148431,"name":"offline","context":{"idset":"9607"}}
+{"timestamp":1713306383.6269987,"name":"offline","context":{"idset":"9608"}}
+{"timestamp":1713306383.7218831,"name":"offline","context":{"idset":"9609"}}
+{"timestamp":1713306385.6034176,"name":"offline","context":{"idset":"9610"}}
+{"timestamp":1713306385.6171381,"name":"offline","context":{"idset":"9611"}}
+{"timestamp":1713306385.6328177,"name":"offline","context":{"idset":"9612"}}
+{"timestamp":1713306385.64662,"name":"offline","context":{"idset":"9613"}}
+{"timestamp":1713306385.6887243,"name":"offline","context":{"idset":"9614"}}
+{"timestamp":1713306387.6085234,"name":"offline","context":{"idset":"9615"}}
+{"timestamp":1713306387.6279705,"name":"offline","context":{"idset":"9616"}}
+{"timestamp":1713306387.6466422,"name":"offline","context":{"idset":"9617"}}
+{"timestamp":1713306387.7010407,"name":"offline","context":{"idset":"9618"}}
+{"timestamp":1713306389.6015537,"name":"offline","context":{"idset":"9620"}}
+{"timestamp":1713306389.6214647,"name":"offline","context":{"idset":"9621"}}
+{"timestamp":1713306389.6905894,"name":"offline","context":{"idset":"9622"}}
+{"timestamp":1713306391.6103785,"name":"offline","context":{"idset":"9623"}}
+{"timestamp":1713306391.6280122,"name":"offline","context":{"idset":"9624"}}
+{"timestamp":1713306391.6454291,"name":"offline","context":{"idset":"9625"}}
+{"timestamp":1713306391.6645811,"name":"offline","context":{"idset":"9626"}}
+{"timestamp":1713306391.6828997,"name":"offline","context":{"idset":"9627"}}
+{"timestamp":1713306391.713897,"name":"offline","context":{"idset":"9628"}}
+{"timestamp":1713306393.6055541,"name":"offline","context":{"idset":"9592"}}
+{"timestamp":1713306393.6244586,"name":"offline","context":{"idset":"9629"}}
+{"timestamp":1713306393.6436148,"name":"offline","context":{"idset":"9630"}}
+{"timestamp":1713306393.6897624,"name":"offline","context":{"idset":"9631"}}
+{"timestamp":1713306395.6016083,"name":"offline","context":{"idset":"9632"}}
+{"timestamp":1713306395.6142311,"name":"offline","context":{"idset":"9634"}}
+{"timestamp":1713306395.6278718,"name":"offline","context":{"idset":"9635"}}
+{"timestamp":1713306395.6398818,"name":"offline","context":{"idset":"9636"}}
+{"timestamp":1713306395.7028723,"name":"offline","context":{"idset":"9637"}}
+{"timestamp":1713306397.5993528,"name":"offline","context":{"idset":"9638"}}
+{"timestamp":1713306397.6116543,"name":"offline","context":{"idset":"9639"}}
+{"timestamp":1713306397.6979287,"name":"offline","context":{"idset":"9640"}}
+{"timestamp":1713306399.5986199,"name":"offline","context":{"idset":"9641"}}
+{"timestamp":1713306399.6118677,"name":"offline","context":{"idset":"9642"}}
+{"timestamp":1713306399.6251292,"name":"offline","context":{"idset":"9643"}}
+{"timestamp":1713306399.7097394,"name":"offline","context":{"idset":"9644"}}
+{"timestamp":1713306401.6005299,"name":"offline","context":{"idset":"9645"}}
+{"timestamp":1713306401.6126852,"name":"offline","context":{"idset":"9646"}}
+{"timestamp":1713306401.6255729,"name":"offline","context":{"idset":"9647"}}
+{"timestamp":1713306401.7080598,"name":"offline","context":{"idset":"9648"}}
+{"timestamp":1713306403.6002262,"name":"offline","context":{"idset":"9649"}}
+{"timestamp":1713306403.6141193,"name":"offline","context":{"idset":"9650"}}
+{"timestamp":1713306403.6281192,"name":"offline","context":{"idset":"9651"}}
+{"timestamp":1713306403.6957386,"name":"offline","context":{"idset":"9652"}}
+{"timestamp":1713306405.600198,"name":"offline","context":{"idset":"9653"}}
+{"timestamp":1713306405.6133318,"name":"offline","context":{"idset":"9654"}}
+{"timestamp":1713306405.6261542,"name":"offline","context":{"idset":"9655"}}
+{"timestamp":1713306405.6885986,"name":"offline","context":{"idset":"9656"}}
+{"timestamp":1713306407.5980046,"name":"offline","context":{"idset":"9657"}}
+{"timestamp":1713306407.6107798,"name":"offline","context":{"idset":"9658"}}
+{"timestamp":1713306407.6951795,"name":"offline","context":{"idset":"9659"}}
+{"timestamp":1713306409.5990448,"name":"offline","context":{"idset":"9660"}}
+{"timestamp":1713306409.6112533,"name":"offline","context":{"idset":"9661"}}
+{"timestamp":1713306409.6967471,"name":"offline","context":{"idset":"9662"}}
+{"timestamp":1713306411.6083519,"name":"offline","context":{"idset":"9663"}}
+{"timestamp":1713306411.6337221,"name":"offline","context":{"idset":"9664"}}
+{"timestamp":1713306411.6589129,"name":"offline","context":{"idset":"9665"}}
+{"timestamp":1713306411.6890492,"name":"offline","context":{"idset":"9666"}}
+{"timestamp":1713306413.60449,"name":"offline","context":{"idset":"9667"}}
+{"timestamp":1713306413.6238062,"name":"offline","context":{"idset":"9668"}}
+{"timestamp":1713306413.642544,"name":"offline","context":{"idset":"9669"}}
+{"timestamp":1713306413.6904595,"name":"offline","context":{"idset":"9670"}}
+{"timestamp":1713306415.6006832,"name":"offline","context":{"idset":"9671"}}
+{"timestamp":1713306415.6137877,"name":"offline","context":{"idset":"9672"}}
+{"timestamp":1713306415.6235244,"name":"offline","context":{"idset":"9673"}}
+{"timestamp":1713306415.63586,"name":"offline","context":{"idset":"9674"}}
+{"timestamp":1713306415.6912398,"name":"offline","context":{"idset":"9675"}}
+{"timestamp":1713306417.6075485,"name":"offline","context":{"idset":"9676"}}
+{"timestamp":1713306417.6279175,"name":"offline","context":{"idset":"9677"}}
+{"timestamp":1713306417.6475015,"name":"offline","context":{"idset":"9679"}}
+{"timestamp":1713306417.6953382,"name":"offline","context":{"idset":"9680"}}
+{"timestamp":1713306419.5999146,"name":"offline","context":{"idset":"9682"}}
+{"timestamp":1713306419.6914849,"name":"offline","context":{"idset":"9683"}}
+{"timestamp":1713306421.6095371,"name":"offline","context":{"idset":"9684"}}
+{"timestamp":1713306421.6296794,"name":"offline","context":{"idset":"9685"}}
+{"timestamp":1713306421.6484663,"name":"offline","context":{"idset":"9686"}}
+{"timestamp":1713306421.6702199,"name":"offline","context":{"idset":"9687"}}
+{"timestamp":1713306421.6961055,"name":"offline","context":{"idset":"9688"}}
+{"timestamp":1713306423.6051853,"name":"offline","context":{"idset":"9689"}}
+{"timestamp":1713306423.6235359,"name":"offline","context":{"idset":"9690"}}
+{"timestamp":1713306423.6426377,"name":"offline","context":{"idset":"9691"}}
+{"timestamp":1713306423.6896341,"name":"offline","context":{"idset":"9692"}}
+{"timestamp":1713306425.5995355,"name":"offline","context":{"idset":"9693"}}
+{"timestamp":1713306425.6130395,"name":"offline","context":{"idset":"9694"}}
+{"timestamp":1713306425.6249101,"name":"offline","context":{"idset":"9695"}}
+{"timestamp":1713306425.6880298,"name":"offline","context":{"idset":"9696"}}
+{"timestamp":1713306427.5969636,"name":"offline","context":{"idset":"9698"}}
+{"timestamp":1713306427.6089234,"name":"offline","context":{"idset":"9699"}}
+{"timestamp":1713306427.699368,"name":"offline","context":{"idset":"9700"}}
+{"timestamp":1713306429.595907,"name":"offline","context":{"idset":"9701"}}
+{"timestamp":1713306429.6890655,"name":"offline","context":{"idset":"9702"}}
+{"timestamp":1713306431.5970192,"name":"offline","context":{"idset":"9703"}}
+{"timestamp":1713306431.6090097,"name":"offline","context":{"idset":"9704"}}
+{"timestamp":1713306431.688818,"name":"offline","context":{"idset":"9705"}}
+{"timestamp":1713306433.5972157,"name":"offline","context":{"idset":"9708"}}
+{"timestamp":1713306433.6115825,"name":"offline","context":{"idset":"9709"}}
+{"timestamp":1713306433.6997883,"name":"offline","context":{"idset":"9710"}}
+{"timestamp":1713306435.5971422,"name":"offline","context":{"idset":"9711"}}
+{"timestamp":1713306435.6084688,"name":"offline","context":{"idset":"9712"}}
+{"timestamp":1713306435.7027061,"name":"offline","context":{"idset":"9713"}}
+{"timestamp":1713306437.5976145,"name":"offline","context":{"idset":"9714"}}
+{"timestamp":1713306437.6090591,"name":"offline","context":{"idset":"9715"}}
+{"timestamp":1713306437.6970642,"name":"offline","context":{"idset":"9716"}}
+{"timestamp":1713306439.6886656,"name":"offline","context":{"idset":"9619"}}
+{"timestamp":1713306443.6887031,"name":"offline","context":{"idset":"9633"}}
+{"timestamp":1713306449.6905298,"name":"offline","context":{"idset":"9706"}}
+{"timestamp":1713306465.6909814,"name":"offline","context":{"idset":"9678"}}
+{"timestamp":1713306469.6903954,"name":"offline","context":{"idset":"9681"}}
+{"timestamp":1713306473.6889815,"name":"offline","context":{"idset":"9697"}}
+{"timestamp":1713306481.6036296,"name":"drain","context":{"idset":"9589-9716","reason":"epilog failed for jobid fpurPhZYu4j","overwrite":0}}
+{"timestamp":1713306481.6920171,"name":"offline","context":{"idset":"9707"}}
+{"timestamp":1713308040.4089501,"name":"online","context":{"idset":"10911-10912"}}
+{"timestamp":1713308543.2296302,"name":"undrain","context":{"idset":"10911-10912"}}
+{"timestamp":1713309221.1271937,"name":"drain","context":{"idset":"557-564","reason":"New blade installation -KK","overwrite":0}}
+{"timestamp":1713310715.4476476,"name":"online","context":{"idset":"10396,10412"}}
+{"timestamp":1713310716.0912104,"name":"online","context":{"idset":"10411"}}
+{"timestamp":1713310943.6884749,"name":"offline","context":{"idset":"10402"}}
+{"timestamp":1713311105.6891286,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1713311121.9665644,"name":"online","context":{"idset":"11569,11575-11576,11586"}}
+{"timestamp":1713311239.1563106,"name":"online","context":{"idset":"9339"}}
+{"timestamp":1713311239.5817049,"name":"online","context":{"idset":"9435"}}
+{"timestamp":1713311239.7270334,"name":"online","context":{"idset":"9346,9365"}}
+{"timestamp":1713311239.8939745,"name":"online","context":{"idset":"9375,9414"}}
+{"timestamp":1713311240.1176367,"name":"online","context":{"idset":"9402"}}
+{"timestamp":1713311240.2433162,"name":"online","context":{"idset":"9366,9380,9411"}}
+{"timestamp":1713311240.3647964,"name":"online","context":{"idset":"9418,9446"}}
+{"timestamp":1713311240.4904885,"name":"online","context":{"idset":"9335"}}
+{"timestamp":1713311240.6189477,"name":"online","context":{"idset":"9432"}}
+{"timestamp":1713311240.7419062,"name":"online","context":{"idset":"9334,9429"}}
+{"timestamp":1713311240.9078841,"name":"online","context":{"idset":"9434"}}
+{"timestamp":1713311241.041389,"name":"online","context":{"idset":"9350,9352"}}
+{"timestamp":1713311241.0665405,"name":"online","context":{"idset":"9351"}}
+{"timestamp":1713311241.2557921,"name":"online","context":{"idset":"9343,9392"}}
+{"timestamp":1713311241.5028405,"name":"online","context":{"idset":"9361"}}
+{"timestamp":1713311241.6297348,"name":"online","context":{"idset":"9374"}}
+{"timestamp":1713311241.7690496,"name":"online","context":{"idset":"9360,9362"}}
+{"timestamp":1713311241.8860686,"name":"online","context":{"idset":"9412"}}
+{"timestamp":1713311242.0087299,"name":"online","context":{"idset":"9436"}}
+{"timestamp":1713311242.2335365,"name":"online","context":{"idset":"9413"}}
+{"timestamp":1713311242.3600595,"name":"online","context":{"idset":"9407"}}
+{"timestamp":1713311242.8321223,"name":"online","context":{"idset":"9333,9379,9430,9438"}}
+{"timestamp":1713311242.8956649,"name":"online","context":{"idset":"9450"}}
+{"timestamp":1713311243.0218718,"name":"online","context":{"idset":"9376,9456"}}
+{"timestamp":1713311243.1452885,"name":"online","context":{"idset":"9341,9372-9373,9458"}}
+{"timestamp":1713311243.3084688,"name":"online","context":{"idset":"9355"}}
+{"timestamp":1713311243.3281817,"name":"online","context":{"idset":"9421"}}
+{"timestamp":1713311243.4658208,"name":"online","context":{"idset":"9385-9386,9437"}}
+{"timestamp":1713311243.6797535,"name":"online","context":{"idset":"9344,9367,9397,9426,9442-9443"}}
+{"timestamp":1713311243.8128307,"name":"online","context":{"idset":"9390"}}
+{"timestamp":1713311244.0331345,"name":"online","context":{"idset":"9338,9404"}}
+{"timestamp":1713311244.1623826,"name":"online","context":{"idset":"9370,9378,9424,9441"}}
+{"timestamp":1713311244.2876279,"name":"online","context":{"idset":"9377,9433"}}
+{"timestamp":1713311244.3051357,"name":"online","context":{"idset":"9395"}}
+{"timestamp":1713311244.3188643,"name":"online","context":{"idset":"9369"}}
+{"timestamp":1713311244.4816096,"name":"online","context":{"idset":"9340,9417,9444"}}
+{"timestamp":1713311244.505568,"name":"online","context":{"idset":"9400"}}
+{"timestamp":1713311244.6568291,"name":"online","context":{"idset":"9353,9399,9415-9416"}}
+{"timestamp":1713311244.7715197,"name":"online","context":{"idset":"9345,9398,9428,9439"}}
+{"timestamp":1713311245.0330305,"name":"online","context":{"idset":"9371,9381,9401,9405,9425"}}
+{"timestamp":1713311245.192858,"name":"online","context":{"idset":"9358,9368,9420,9423,9427"}}
+{"timestamp":1713311245.3108318,"name":"online","context":{"idset":"9336,9342,9348,9357,9403,9431"}}
+{"timestamp":1713311245.4337962,"name":"online","context":{"idset":"9408,9422,9440"}}
+{"timestamp":1713311245.5531871,"name":"online","context":{"idset":"9406,9409-9410"}}
+{"timestamp":1713311245.7749991,"name":"online","context":{"idset":"9347,9349,9419,9448,9452"}}
+{"timestamp":1713311245.9908111,"name":"online","context":{"idset":"9337,9359,9363"}}
+{"timestamp":1713311246.1316097,"name":"online","context":{"idset":"9354,9387,9389,9445"}}
+{"timestamp":1713311246.3558588,"name":"online","context":{"idset":"9356,9384,9388,9391,9393,9396,9451"}}
+{"timestamp":1713311246.6089492,"name":"online","context":{"idset":"9364,9394,9449,9457,9460"}}
+{"timestamp":1713311246.7354681,"name":"online","context":{"idset":"9382"}}
+{"timestamp":1713311246.9066741,"name":"online","context":{"idset":"9383,9447,9453-9454,9459"}}
+{"timestamp":1713311247.3398011,"name":"online","context":{"idset":"9455"}}
+{"timestamp":1713311261.3849187,"name":"online","context":{"idset":"11580"}}
+{"timestamp":1713311261.5139825,"name":"online","context":{"idset":"11573,11579"}}
+{"timestamp":1713311261.788511,"name":"online","context":{"idset":"11578,11581,11583"}}
+{"timestamp":1713311262.145427,"name":"online","context":{"idset":"11584-11585"}}
+{"timestamp":1713311262.3781796,"name":"online","context":{"idset":"11582"}}
+{"timestamp":1713311262.7200065,"name":"online","context":{"idset":"11574,11577"}}
+{"timestamp":1713311351.7738247,"name":"undrain","context":{"idset":"11573-11574,11577-11585,11587-11588"}}
+{"timestamp":1713311657.0348125,"name":"online","context":{"idset":"9789"}}
+{"timestamp":1713311768.2106616,"name":"online","context":{"idset":"9790"}}
+{"timestamp":1713311772.8536842,"name":"drain","context":{"idset":"10358","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713311906.0588176,"name":"drain","context":{"idset":"10360","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713316379.7168951,"name":"drain","context":{"idset":"9333-9460","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1713316570.8752124,"name":"undrain","context":{"idset":"9333-9460"}}
+{"timestamp":1713316791.1306329,"name":"drain","context":{"idset":"9333-9460","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1713317313.9242415,"name":"undrain","context":{"idset":"9333-9460"}}
+{"timestamp":1713319512.3102863,"name":"drain","context":{"idset":"10901-10916","overwrite":0}}
+{"timestamp":1713319538.0471337,"name":"undrain","context":{"idset":"10901-10916"}}
+{"timestamp":1713323365.02279,"name":"online","context":{"idset":"10402"}}
+{"timestamp":1713324005.4785674,"name":"undrain","context":{"idset":"10358,10360"}}
+{"timestamp":1713325565.6891487,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1713326076.6416039,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1713326076.7406814,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1713326076.8891952,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1713326076.9151487,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1713326076.9537203,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1713326076.9707568,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1713326077.0456564,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1713326077.2034681,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1713326077.3033304,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1713326077.4937236,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1713326077.5939121,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1713326077.6860769,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1713326077.9700878,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1713326078.0714681,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1713326078.1518519,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1713326078.2391286,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1713326078.3130717,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713326078.3959126,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1713326078.4689806,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1713326078.6354449,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1713326078.6839929,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1713326078.7833493,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1713326079.4949567,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1713326079.5890477,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1713326079.6102986,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1713326079.6132534,"name":"offline","context":{"idset":"10896"}}
+{"timestamp":1713326079.6622212,"name":"offline","context":{"idset":"10898"}}
+{"timestamp":1713326079.7882419,"name":"offline","context":{"idset":"10895"}}
+{"timestamp":1713326079.888041,"name":"offline","context":{"idset":"10897"}}
+{"timestamp":1713326080.1385088,"name":"offline","context":{"idset":"10894"}}
+{"timestamp":1713326080.161303,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1713326080.2354157,"name":"offline","context":{"idset":"10900"}}
+{"timestamp":1713326080.5369425,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1713326080.7029469,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1713326080.7459197,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1713326080.8472774,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1713326080.9470959,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1713326081.025619,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1713326081.3876221,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1713326081.5971913,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1713326081.6894481,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1713326081.7710536,"name":"offline","context":{"idset":"10909"}}
+{"timestamp":1713326081.7761855,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1713326081.8447886,"name":"offline","context":{"idset":"10912"}}
+{"timestamp":1713326081.8586736,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1713326081.939661,"name":"offline","context":{"idset":"10915"}}
+{"timestamp":1713326082.1406002,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1713326082.1806576,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1713326082.2804644,"name":"offline","context":{"idset":"10918"}}
+{"timestamp":1713326082.4428365,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1713326082.685097,"name":"offline","context":{"idset":"10916"}}
+{"timestamp":1713326082.8253219,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1713326083.0419316,"name":"offline","context":{"idset":"10920"}}
+{"timestamp":1713326083.1410387,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1713326083.688957,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1713326083.7709301,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1713326083.8527524,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1713326084.0251937,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1713326084.0876169,"name":"offline","context":{"idset":"10929"}}
+{"timestamp":1713326084.1691747,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1713326084.2418845,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1713326084.4473858,"name":"offline","context":{"idset":"10926"}}
+{"timestamp":1713326084.5465169,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1713326085.4916458,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1713327744.2595797,"name":"online","context":{"idset":"10871"}}
+{"timestamp":1713327744.486999,"name":"online","context":{"idset":"10876"}}
+{"timestamp":1713327745.2215858,"name":"online","context":{"idset":"10874"}}
+{"timestamp":1713327745.4781775,"name":"online","context":{"idset":"10877-10878,10882-10883"}}
+{"timestamp":1713327745.6440096,"name":"online","context":{"idset":"10872-10873"}}
+{"timestamp":1713327745.876292,"name":"online","context":{"idset":"10869-10870,10875,10879-10881,10884"}}
+{"timestamp":1713327758.8789344,"name":"online","context":{"idset":"10886,10888-10890"}}
+{"timestamp":1713327759.1282163,"name":"online","context":{"idset":"10885,10887,10891-10900"}}
+{"timestamp":1713327780.6554337,"name":"online","context":{"idset":"10901-10907,10909,10913,10918"}}
+{"timestamp":1713327780.8800509,"name":"online","context":{"idset":"10908,10910-10912,10914-10916,10919"}}
+{"timestamp":1713327782.7323074,"name":"online","context":{"idset":"10921"}}
+{"timestamp":1713327783.0266366,"name":"online","context":{"idset":"10917,10923-10924,10926-10928,10931"}}
+{"timestamp":1713327783.165298,"name":"online","context":{"idset":"10920,10932"}}
+{"timestamp":1713327783.2906268,"name":"online","context":{"idset":"10922,10925,10929-10930"}}
+{"timestamp":1713362133.1570115,"name":"undrain","context":{"idset":"9782"}}
+{"timestamp":1713363117.8197739,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1713363118.154789,"name":"offline","context":{"idset":"10934"}}
+{"timestamp":1713363118.7628736,"name":"offline","context":{"idset":"10946"}}
+{"timestamp":1713363118.8623378,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1713363119.0668049,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1713363119.4343584,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1713363119.5997965,"name":"offline","context":{"idset":"10937"}}
+{"timestamp":1713363119.6143918,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1713363119.6330245,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1713363119.6912165,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1713363119.9215107,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1713363119.950521,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1713363120.0499136,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1713363120.1914148,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1713363120.2046766,"name":"offline","context":{"idset":"10936"}}
+{"timestamp":1713363120.2163453,"name":"offline","context":{"idset":"10933"}}
+{"timestamp":1713363120.2327995,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1713363120.3323116,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1713363120.6322386,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1713363120.7319982,"name":"offline","context":{"idset":"10952"}}
+{"timestamp":1713363120.7985971,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1713363120.8857379,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1713363121.0758076,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1713363121.1757867,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1713363121.3146303,"name":"offline","context":{"idset":"10960"}}
+{"timestamp":1713363121.4671278,"name":"offline","context":{"idset":"10965"}}
+{"timestamp":1713363121.5653107,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1713363121.6324115,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1713363121.6469183,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1713363121.7309055,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1713363122.0926967,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1713363122.4077511,"name":"offline","context":{"idset":"10959"}}
+{"timestamp":1713363122.659632,"name":"offline","context":{"idset":"10968"}}
+{"timestamp":1713363122.9052243,"name":"offline","context":{"idset":"10957"}}
+{"timestamp":1713363123.4251401,"name":"offline","context":{"idset":"10977"}}
+{"timestamp":1713363123.5247838,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1713363123.6649833,"name":"offline","context":{"idset":"10981"}}
+{"timestamp":1713363123.7950296,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1713363123.9756124,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1713363124.001847,"name":"offline","context":{"idset":"10983"}}
+{"timestamp":1713363124.0351846,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1713363124.1276178,"name":"offline","context":{"idset":"10961"}}
+{"timestamp":1713363124.1964083,"name":"offline","context":{"idset":"10973"}}
+{"timestamp":1713363124.271744,"name":"offline","context":{"idset":"10986"}}
+{"timestamp":1713363124.3710732,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1713363124.8307312,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1713363124.9082999,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1713363125.0324395,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1713363125.2752113,"name":"offline","context":{"idset":"10971"}}
+{"timestamp":1713363125.2956276,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1713363125.3258574,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1713363125.3457718,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1713363125.3847942,"name":"offline","context":{"idset":"10967"}}
+{"timestamp":1713363125.4744186,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1713363125.7527304,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1713363125.8471146,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1713363125.8955336,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1713363125.9059763,"name":"offline","context":{"idset":"10992"}}
+{"timestamp":1713363125.9309082,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1713363126.2066019,"name":"offline","context":{"idset":"10993"}}
+{"timestamp":1713363126.3046901,"name":"offline","context":{"idset":"10989"}}
+{"timestamp":1713363126.5255601,"name":"offline","context":{"idset":"10988"}}
+{"timestamp":1713363126.6864972,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1713363126.9722068,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1713363481.8497827,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1713363481.8813989,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1713363481.9010677,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1713363481.9218352,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1713363481.9453723,"name":"offline","context":{"idset":"11517"}}
+{"timestamp":1713363482.0008163,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1713363482.0250416,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1713363482.0434728,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1713363482.077688,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1713363482.098557,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1713363482.1248162,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1713363482.1456733,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1713363482.1666663,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1713363482.1865845,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1713363483.2058196,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1713363483.3331633,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1713363483.5997267,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1713363483.6862831,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1713363484.1478927,"name":"offline","context":{"idset":"11525"}}
+{"timestamp":1713363485.081398,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1713363485.127996,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1713363485.1691096,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1713363485.2414155,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1713363485.2586443,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1713363485.3404069,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1713363485.3680761,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1713363485.4577966,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1713363485.4623685,"name":"offline","context":{"idset":"11536"}}
+{"timestamp":1713363485.4926007,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1713363485.5315475,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1713363485.5983067,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1713363485.634129,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1713363487.3623872,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1713363487.4054332,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1713363487.4785283,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1713363487.4864492,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1713363487.5188422,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1713363487.5564818,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1713363487.5713294,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1713363487.574785,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1713363487.592392,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1713363487.595521,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1713363487.6628528,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1713363488.5983789,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1713363488.6265874,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1713363488.6485696,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1713363488.6715934,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1713363488.6864047,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1713363488.7048025,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1713363488.7222903,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1713363488.7388775,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1713363488.7565143,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1713363488.78618,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1713363488.8301854,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1713363489.0225074,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1713363489.415184,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1713363489.5896504,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1713363489.8694243,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1713363489.9683995,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1713363490.1273069,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1713363490.2669103,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1713363490.4222379,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1713363490.4883428,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1713363490.5860376,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1713363490.856046,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1713363490.9255652,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1713363491.0177999,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1713363491.2001057,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1713363491.2993286,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1713363491.599632,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1713363491.6914039,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1713363492.0062189,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1713363492.0465267,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1713363492.1417019,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1713363492.3231633,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1713363492.3646419,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1713363492.4095104,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1713363492.5005832,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1713363492.6242259,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1713363492.646898,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1713363492.665678,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1713363492.7038243,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1713363492.8197193,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1713363492.9760985,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1713363493.1558681,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1713363493.1750019,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1713363493.2727828,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1713363493.5205948,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1713363493.5402935,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1713363493.6061862,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1713363493.9010832,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1713363493.9250622,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1713363493.9478045,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1713363494.0404058,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1713363494.8336387,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1713363494.8809826,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1713363494.8982439,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1713363494.9192858,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1713363494.9451718,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1713363494.9641032,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1713363495.3372824,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1713363495.3973677,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1713363495.4250994,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1713363495.446696,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1713363495.4915707,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1713363495.7080266,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1713363495.7290809,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1713363496.4475305,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1713363496.5763185,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1713363497.0255375,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1713363497.0498281,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1713363497.0715969,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1713363497.1053064,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1713363497.1275151,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1713363497.1479745,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1713363497.1751103,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1713363497.1970501,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1713363497.5041041,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1713363498.394197,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1713363498.4140377,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1713363498.4337041,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1713363498.4530933,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1713363498.4722314,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1713363498.5017667,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1713363498.5235369,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1713363498.5564609,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1713363498.5658398,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1713363498.5850964,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1713364374.5606024,"name":"offline","context":{"idset":"9555"}}
+{"timestamp":1713364790.5836914,"name":"drain","context":{"idset":"9213-9214","reason":"Debugging downed nodes - vls","overwrite":0}}
+{"timestamp":1713364829.8574481,"name":"undrain","context":{"idset":"9213-9214"}}
+{"timestamp":1713364975.1127479,"name":"online","context":{"idset":"9213-9214"}}
+{"timestamp":1713365185.3513255,"name":"undrain","context":{"idset":"9589-9716"}}
+{"timestamp":1713366071.2510142,"name":"online","context":{"idset":"9620"}}
+{"timestamp":1713366071.8295779,"name":"online","context":{"idset":"9594,9598,9610,9631"}}
+{"timestamp":1713366072.0521305,"name":"online","context":{"idset":"9590"}}
+{"timestamp":1713366072.3449085,"name":"online","context":{"idset":"9602"}}
+{"timestamp":1713366072.9134171,"name":"online","context":{"idset":"9603"}}
+{"timestamp":1713366073.0513484,"name":"online","context":{"idset":"9640"}}
+{"timestamp":1713366074.2346237,"name":"online","context":{"idset":"9592,9615"}}
+{"timestamp":1713366074.3609567,"name":"online","context":{"idset":"9638"}}
+{"timestamp":1713366074.6521509,"name":"online","context":{"idset":"9643"}}
+{"timestamp":1713366074.7581809,"name":"online","context":{"idset":"9597"}}
+{"timestamp":1713366074.8525603,"name":"online","context":{"idset":"9613"}}
+{"timestamp":1713366075.0277739,"name":"online","context":{"idset":"9600,9651"}}
+{"timestamp":1713366075.0642035,"name":"online","context":{"idset":"9589"}}
+{"timestamp":1713366075.1784058,"name":"online","context":{"idset":"9642,9649"}}
+{"timestamp":1713366075.281311,"name":"online","context":{"idset":"9697"}}
+{"timestamp":1713366075.2968264,"name":"online","context":{"idset":"9629,9656"}}
+{"timestamp":1713366075.4373548,"name":"online","context":{"idset":"9618"}}
+{"timestamp":1713366075.4530513,"name":"drain","context":{"idset":"9655","reason":"broker was unresponsive"}}
+{"timestamp":1713366075.453156,"name":"drain","context":{"idset":"9676","reason":"broker was unresponsive"}}
+{"timestamp":1713366075.4532764,"name":"drain","context":{"idset":"9694","reason":"broker was unresponsive"}}
+{"timestamp":1713366075.5678089,"name":"drain","context":{"idset":"9700","reason":"broker was unresponsive"}}
+{"timestamp":1713366075.6792312,"name":"online","context":{"idset":"9617,9637"}}
+{"timestamp":1713366075.8210742,"name":"online","context":{"idset":"9647"}}
+{"timestamp":1713366076.0365055,"name":"online","context":{"idset":"9596,9601,9623-9624,9698"}}
+{"timestamp":1713366076.2315533,"name":"online","context":{"idset":"9632,9641,9674,9695,9714"}}
+{"timestamp":1713366076.3487387,"name":"online","context":{"idset":"9626"}}
+{"timestamp":1713366076.3797209,"name":"online","context":{"idset":"9628,9681"}}
+{"timestamp":1713366076.4948151,"name":"online","context":{"idset":"9605,9696"}}
+{"timestamp":1713366076.6092553,"name":"online","context":{"idset":"9591,9614,9635,9650,9662,9673,9680,9710"}}
+{"timestamp":1713366076.7482157,"name":"online","context":{"idset":"9608,9616,9621-9622,9630,9636,9667-9668,9677,9688,9716"}}
+{"timestamp":1713366076.9490838,"name":"online","context":{"idset":"9606,9639,9652-9653,9658,9672,9685-9686,9702,9707,9709"}}
+{"timestamp":1713366077.0632131,"name":"online","context":{"idset":"9612,9645,9671,9704"}}
+{"timestamp":1713366077.1765018,"name":"online","context":{"idset":"9619,9654,9657,9663,9675,9689,9706"}}
+{"timestamp":1713366077.2886577,"name":"online","context":{"idset":"9604,9627"}}
+{"timestamp":1713366077.4027197,"name":"online","context":{"idset":"9607,9609,9625,9644,9664,9678-9679,9683,9693,9711"}}
+{"timestamp":1713366077.4617252,"name":"online","context":{"idset":"9661,9669"}}
+{"timestamp":1713366077.6649017,"name":"online","context":{"idset":"9593,9595,9655,9659-9660,9665-9666,9670,9699,9701,9703,9708"}}
+{"timestamp":1713366077.7973843,"name":"online","context":{"idset":"9611,9648,9676,9687,9700,9712"}}
+{"timestamp":1713366078.0051954,"name":"online","context":{"idset":"9599,9646,9682,9684,9690-9691,9694,9713,9715"}}
+{"timestamp":1713366078.2231586,"name":"online","context":{"idset":"9705"}}
+{"timestamp":1713366078.4483199,"name":"online","context":{"idset":"9692"}}
+{"timestamp":1713366627.6254237,"name":"online","context":{"idset":"9535"}}
+{"timestamp":1713367982.3108435,"name":"undrain","context":{"idset":"9655,9676,9694,9700"}}
+{"timestamp":1713368302.8131859,"name":"online","context":{"idset":"892"}}
+{"timestamp":1713368303.0409958,"name":"online","context":{"idset":"898"}}
+{"timestamp":1713368303.1772454,"name":"online","context":{"idset":"888,890"}}
+{"timestamp":1713368303.377821,"name":"online","context":{"idset":"889,891,893,896-897"}}
+{"timestamp":1713368303.6102338,"name":"online","context":{"idset":"887,894-895"}}
+{"timestamp":1713368455.5125699,"name":"drain","context":{"idset":"10978","reason":"broker was unresponsive"}}
+{"timestamp":1713368455.5155201,"name":"online","context":{"idset":"10939"}}
+{"timestamp":1713368455.8695319,"name":"online","context":{"idset":"10934"}}
+{"timestamp":1713368456.0894318,"name":"online","context":{"idset":"10948"}}
+{"timestamp":1713368456.2205749,"name":"online","context":{"idset":"10950"}}
+{"timestamp":1713368456.3468213,"name":"online","context":{"idset":"10967"}}
+{"timestamp":1713368456.5933588,"name":"online","context":{"idset":"10942"}}
+{"timestamp":1713368456.707499,"name":"online","context":{"idset":"10941,10945"}}
+{"timestamp":1713368456.8852937,"name":"online","context":{"idset":"10943"}}
+{"timestamp":1713368456.9044077,"name":"online","context":{"idset":"10979"}}
+{"timestamp":1713368457.0270581,"name":"online","context":{"idset":"10984"}}
+{"timestamp":1713368457.1609581,"name":"online","context":{"idset":"10975"}}
+{"timestamp":1713368457.4497485,"name":"online","context":{"idset":"10976"}}
+{"timestamp":1713368457.563097,"name":"drain","context":{"idset":"10993","reason":"broker was unresponsive"}}
+{"timestamp":1713368457.6809707,"name":"online","context":{"idset":"10977,10991,10995"}}
+{"timestamp":1713368457.7971399,"name":"online","context":{"idset":"10935,10978"}}
+{"timestamp":1713368458.0354443,"name":"online","context":{"idset":"10936,10949,10951-10952,10954"}}
+{"timestamp":1713368458.2462242,"name":"online","context":{"idset":"10933,10955,10973,10980,10988"}}
+{"timestamp":1713368458.3826478,"name":"online","context":{"idset":"10946,10961,10987"}}
+{"timestamp":1713368458.5949836,"name":"online","context":{"idset":"10940,10944,10956,10959,10968,10989,10994"}}
+{"timestamp":1713368458.8149707,"name":"online","context":{"idset":"10960,10963,10985"}}
+{"timestamp":1713368458.965122,"name":"online","context":{"idset":"10938,10947,10953,10962,10966,10971,10992"}}
+{"timestamp":1713368459.0848951,"name":"online","context":{"idset":"10958,10964-10965,10969,10974,10981-10983"}}
+{"timestamp":1713368459.2846487,"name":"online","context":{"idset":"10957,10972,10986,10990,10996"}}
+{"timestamp":1713368459.7608526,"name":"online","context":{"idset":"10993"}}
+{"timestamp":1713368459.9643707,"name":"online","context":{"idset":"10970"}}
+{"timestamp":1713368620.3614013,"name":"undrain","context":{"idset":"10978,10993"}}
+{"timestamp":1713369411.0043466,"name":"online","context":{"idset":"780"}}
+{"timestamp":1713370047.5536687,"name":"offline","context":{"idset":"9533"}}
+{"timestamp":1713370579.9708996,"name":"online","context":{"idset":"9241"}}
+{"timestamp":1713370845.5525215,"name":"offline","context":{"idset":"9359"}}
+{"timestamp":1713371179.5120952,"name":"online","context":{"idset":"11514"}}
+{"timestamp":1713371179.6484203,"name":"online","context":{"idset":"11519"}}
+{"timestamp":1713371179.9379914,"name":"online","context":{"idset":"11509,11515,11518"}}
+{"timestamp":1713371180.1519494,"name":"online","context":{"idset":"11513,11523"}}
+{"timestamp":1713371180.3508465,"name":"online","context":{"idset":"11512,11521"}}
+{"timestamp":1713371180.4867318,"name":"online","context":{"idset":"11510-11511,11516,11524"}}
+{"timestamp":1713371180.6983011,"name":"online","context":{"idset":"11517,11520,11522"}}
+{"timestamp":1713371191.2112074,"name":"online","context":{"idset":"11525"}}
+{"timestamp":1713371191.8089933,"name":"online","context":{"idset":"11527"}}
+{"timestamp":1713371192.1780705,"name":"online","context":{"idset":"11530-11531,11533-11534"}}
+{"timestamp":1713371192.4470756,"name":"online","context":{"idset":"11526,11529,11532,11537-11538,11541"}}
+{"timestamp":1713371192.7247829,"name":"online","context":{"idset":"11535-11536,11539"}}
+{"timestamp":1713371192.8710759,"name":"online","context":{"idset":"11540"}}
+{"timestamp":1713371203.3210671,"name":"online","context":{"idset":"11542"}}
+{"timestamp":1713371203.3761544,"name":"online","context":{"idset":"11543"}}
+{"timestamp":1713371203.993257,"name":"online","context":{"idset":"11545"}}
+{"timestamp":1713371204.2549338,"name":"online","context":{"idset":"11546"}}
+{"timestamp":1713371204.6684926,"name":"online","context":{"idset":"11544,11548"}}
+{"timestamp":1713371204.7921412,"name":"online","context":{"idset":"11557"}}
+{"timestamp":1713371204.9169817,"name":"online","context":{"idset":"11555"}}
+{"timestamp":1713371205.1351881,"name":"online","context":{"idset":"11549,11553-11554,11556"}}
+{"timestamp":1713371205.2647789,"name":"online","context":{"idset":"11547"}}
+{"timestamp":1713371205.4121892,"name":"online","context":{"idset":"11550,11552"}}
+{"timestamp":1713371205.6905274,"name":"online","context":{"idset":"11551"}}
+{"timestamp":1713371215.7317669,"name":"online","context":{"idset":"11558-11559"}}
+{"timestamp":1713371215.9904635,"name":"online","context":{"idset":"11560-11561"}}
+{"timestamp":1713371216.307821,"name":"online","context":{"idset":"11562-11563"}}
+{"timestamp":1713371216.531317,"name":"online","context":{"idset":"11566"}}
+{"timestamp":1713371216.6742492,"name":"online","context":{"idset":"11569"}}
+{"timestamp":1713371216.8697751,"name":"online","context":{"idset":"11571"}}
+{"timestamp":1713371217.0946712,"name":"online","context":{"idset":"11565,11568,11570,11573"}}
+{"timestamp":1713371217.4313035,"name":"online","context":{"idset":"11572"}}
+{"timestamp":1713371217.66872,"name":"online","context":{"idset":"11564,11567"}}
+{"timestamp":1713371227.6364765,"name":"online","context":{"idset":"11574-11575"}}
+{"timestamp":1713371227.9599819,"name":"online","context":{"idset":"11576"}}
+{"timestamp":1713371228.3076768,"name":"online","context":{"idset":"11577"}}
+{"timestamp":1713371228.5357854,"name":"online","context":{"idset":"11578-11579,11581"}}
+{"timestamp":1713371228.8757885,"name":"online","context":{"idset":"11580,11582"}}
+{"timestamp":1713371229.0967891,"name":"online","context":{"idset":"11584,11586,11588-11589"}}
+{"timestamp":1713371229.2249634,"name":"online","context":{"idset":"11583"}}
+{"timestamp":1713371229.4471304,"name":"online","context":{"idset":"11585,11587"}}
+{"timestamp":1713371239.3517013,"name":"online","context":{"idset":"11590"}}
+{"timestamp":1713371239.4786251,"name":"online","context":{"idset":"11591-11592"}}
+{"timestamp":1713371239.775955,"name":"online","context":{"idset":"11593"}}
+{"timestamp":1713371240.4306207,"name":"online","context":{"idset":"11595"}}
+{"timestamp":1713371240.5573153,"name":"online","context":{"idset":"11597"}}
+{"timestamp":1713371240.7985196,"name":"online","context":{"idset":"11594,11598"}}
+{"timestamp":1713371241.0483496,"name":"online","context":{"idset":"11596,11600"}}
+{"timestamp":1713371241.1695416,"name":"online","context":{"idset":"11601,11605"}}
+{"timestamp":1713371241.4150751,"name":"online","context":{"idset":"11599,11603"}}
+{"timestamp":1713371241.6625886,"name":"online","context":{"idset":"11604"}}
+{"timestamp":1713371241.9719062,"name":"online","context":{"idset":"11602"}}
+{"timestamp":1713371251.0091655,"name":"online","context":{"idset":"11606-11607"}}
+{"timestamp":1713371251.5489314,"name":"online","context":{"idset":"11608-11609"}}
+{"timestamp":1713371252.1781282,"name":"online","context":{"idset":"11611"}}
+{"timestamp":1713371252.4799356,"name":"online","context":{"idset":"11610"}}
+{"timestamp":1713371252.7783306,"name":"online","context":{"idset":"11613"}}
+{"timestamp":1713371252.9144895,"name":"online","context":{"idset":"11615"}}
+{"timestamp":1713371253.0438914,"name":"online","context":{"idset":"11612"}}
+{"timestamp":1713371253.3746421,"name":"online","context":{"idset":"11616-11617,11620"}}
+{"timestamp":1713371253.5007577,"name":"online","context":{"idset":"11614"}}
+{"timestamp":1713371253.6456139,"name":"online","context":{"idset":"11618-11619,11621"}}
+{"timestamp":1713371256.897656,"name":"online","context":{"idset":"9533"}}
+{"timestamp":1713371262.7664921,"name":"online","context":{"idset":"11622"}}
+{"timestamp":1713371263.433778,"name":"online","context":{"idset":"11623-11625"}}
+{"timestamp":1713371264.1666238,"name":"online","context":{"idset":"11627"}}
+{"timestamp":1713371264.4126425,"name":"online","context":{"idset":"11626"}}
+{"timestamp":1713371264.6241572,"name":"online","context":{"idset":"11628"}}
+{"timestamp":1713371264.9781535,"name":"online","context":{"idset":"11630"}}
+{"timestamp":1713371265.1986387,"name":"online","context":{"idset":"11629,11631-11634"}}
+{"timestamp":1713371265.4181724,"name":"online","context":{"idset":"11635"}}
+{"timestamp":1713371265.9469576,"name":"online","context":{"idset":"11636"}}
+{"timestamp":1713371949.0642457,"name":"online","context":{"idset":"11528"}}
+{"timestamp":1713371997.4938538,"name":"offline","context":{"idset":"10085"}}
+{"timestamp":1713371997.5101666,"name":"offline","context":{"idset":"10087"}}
+{"timestamp":1713371997.5261025,"name":"offline","context":{"idset":"10088"}}
+{"timestamp":1713371997.5411174,"name":"offline","context":{"idset":"10089"}}
+{"timestamp":1713371997.5590436,"name":"offline","context":{"idset":"10090"}}
+{"timestamp":1713371997.577507,"name":"offline","context":{"idset":"10091"}}
+{"timestamp":1713371997.5940173,"name":"offline","context":{"idset":"10092"}}
+{"timestamp":1713371997.6103861,"name":"offline","context":{"idset":"10093"}}
+{"timestamp":1713371997.6260195,"name":"offline","context":{"idset":"10094"}}
+{"timestamp":1713371997.6402791,"name":"offline","context":{"idset":"10095"}}
+{"timestamp":1713371997.656492,"name":"offline","context":{"idset":"10096"}}
+{"timestamp":1713371997.6740315,"name":"offline","context":{"idset":"10097"}}
+{"timestamp":1713371997.6895263,"name":"offline","context":{"idset":"10098"}}
+{"timestamp":1713371997.7039142,"name":"offline","context":{"idset":"10099"}}
+{"timestamp":1713371997.714709,"name":"offline","context":{"idset":"10100"}}
+{"timestamp":1713372045.5524743,"name":"offline","context":{"idset":"10086"}}
+{"timestamp":1713372077.4696012,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1713372077.4814095,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1713372077.4931035,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1713372077.5047007,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1713372077.5162144,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1713372077.5275986,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1713372077.5534673,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1713373222.1638317,"name":"online","context":{"idset":"10937"}}
+{"timestamp":1713373405.5527096,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1713374073.53883,"name":"drain","context":{"idset":"10088","reason":"broker was unresponsive"}}
+{"timestamp":1713374074.8195567,"name":"online","context":{"idset":"10095"}}
+{"timestamp":1713374075.0421484,"name":"online","context":{"idset":"10087,10093,10096"}}
+{"timestamp":1713374075.159349,"name":"online","context":{"idset":"10092"}}
+{"timestamp":1713374075.3834236,"name":"online","context":{"idset":"10090"}}
+{"timestamp":1713374075.5567043,"name":"online","context":{"idset":"10085,10098-10099"}}
+{"timestamp":1713374075.6911173,"name":"online","context":{"idset":"10086,10091,10097"}}
+{"timestamp":1713374075.9436786,"name":"online","context":{"idset":"10088,10094,10100"}}
+{"timestamp":1713374076.3539968,"name":"online","context":{"idset":"10089"}}
+{"timestamp":1713374175.218868,"name":"undrain","context":{"idset":"10088"}}
+{"timestamp":1713374631.4771509,"name":"offline","context":{"idset":"9477"}}
+{"timestamp":1713374631.4894512,"name":"offline","context":{"idset":"9478"}}
+{"timestamp":1713374631.5019753,"name":"offline","context":{"idset":"9479"}}
+{"timestamp":1713374631.5146847,"name":"offline","context":{"idset":"9480"}}
+{"timestamp":1713374631.5267277,"name":"offline","context":{"idset":"9484"}}
+{"timestamp":1713374631.5386491,"name":"offline","context":{"idset":"9485"}}
+{"timestamp":1713374631.5553286,"name":"offline","context":{"idset":"9489"}}
+{"timestamp":1713376016.909178,"name":"drain","context":{"idset":"807-810","overwrite":0}}
+{"timestamp":1713376119.5532806,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1713376423.4632502,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1713376423.5535455,"name":"offline","context":{"idset":"9992"}}
+{"timestamp":1713377015.5535262,"name":"offline","context":{"idset":"9533"}}
+{"timestamp":1713377027.5569856,"name":"offline","context":{"idset":"10409"}}
+{"timestamp":1713377032.5715587,"name":"offline","context":{"idset":"9534"}}
+{"timestamp":1713377043.4665012,"name":"offline","context":{"idset":"787"}}
+{"timestamp":1713377043.554414,"name":"offline","context":{"idset":"788"}}
+{"timestamp":1713377059.5548282,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1713377231.0942838,"name":"online","context":{"idset":"9490"}}
+{"timestamp":1713377232.1767945,"name":"online","context":{"idset":"9485"}}
+{"timestamp":1713377232.4775398,"name":"online","context":{"idset":"9482"}}
+{"timestamp":1713377233.0545413,"name":"online","context":{"idset":"9477,9483"}}
+{"timestamp":1713377233.2560194,"name":"online","context":{"idset":"9489,9492"}}
+{"timestamp":1713377233.5939629,"name":"online","context":{"idset":"9480"}}
+{"timestamp":1713377234.2369182,"name":"online","context":{"idset":"9491"}}
+{"timestamp":1713377234.8489921,"name":"online","context":{"idset":"9481"}}
+{"timestamp":1713377235.5458,"name":"online","context":{"idset":"9486,9488"}}
+{"timestamp":1713377235.9824617,"name":"online","context":{"idset":"9478,9484"}}
+{"timestamp":1713377236.895997,"name":"online","context":{"idset":"9487"}}
+{"timestamp":1713377237.5332375,"name":"online","context":{"idset":"9479"}}
+{"timestamp":1713377431.4844365,"name":"drain","context":{"idset":"9533","reason":"epilog failed for jobid fq56fLyse87","overwrite":0}}
+{"timestamp":1713377565.4699917,"name":"offline","context":{"idset":"9497"}}
+{"timestamp":1713377565.4827554,"name":"offline","context":{"idset":"9498"}}
+{"timestamp":1713377565.4950576,"name":"offline","context":{"idset":"9499"}}
+{"timestamp":1713377565.5078728,"name":"offline","context":{"idset":"9500"}}
+{"timestamp":1713377565.5199959,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1713377565.5322988,"name":"offline","context":{"idset":"9504"}}
+{"timestamp":1713377565.5543168,"name":"offline","context":{"idset":"9508"}}
+{"timestamp":1713378633.4618547,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1713378633.5525274,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1713378635.4600163,"name":"drain","context":{"idset":"162,409-410","reason":"prolog failed for jobid fq5jtQQSQnT","overwrite":0}}
+{"timestamp":1713378635.5522115,"name":"offline","context":{"idset":"410"}}
+{"timestamp":1713379407.1507421,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1713379488.2240586,"name":"online","context":{"idset":"9494,9507"}}
+{"timestamp":1713379488.3872545,"name":"online","context":{"idset":"9495,9503-9504"}}
+{"timestamp":1713379488.5761435,"name":"online","context":{"idset":"9493,9497-9499,9501-9502,9505-9506,9508"}}
+{"timestamp":1713379489.4915643,"name":"online","context":{"idset":"9496,9500"}}
+{"timestamp":1713379680.6256909,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1713379696.4423754,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1713379961.9607015,"name":"drain","context":{"idset":"475-476,483-484","reason":"epilog failed for jobid fq5kLbezSHd","overwrite":0}}
+{"timestamp":1713379961.9610527,"name":"drain","context":{"idset":"162,409-410,483-484","reason":"epilog failed for jobid fq5fNwJCsrP","overwrite":0}}
+{"timestamp":1713379962.0626366,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1713379970.4466753,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1713380020.4504061,"name":"online","context":{"idset":"809-810"}}
+{"timestamp":1713380076.8624363,"name":"drain","context":{"idset":"474,483","reason":"epilog failed for jobid fq5rJJ1FQF9","overwrite":0}}
+{"timestamp":1713380094.4413602,"name":"undrain","context":{"idset":"809-810"}}
+{"timestamp":1713380179.4648933,"name":"offline","context":{"idset":"9510"}}
+{"timestamp":1713380179.478231,"name":"offline","context":{"idset":"9511"}}
+{"timestamp":1713380179.49138,"name":"offline","context":{"idset":"9518"}}
+{"timestamp":1713380179.556823,"name":"offline","context":{"idset":"9520"}}
+{"timestamp":1713382531.5518823,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1713382638.9328206,"name":"online","context":{"idset":"9523"}}
+{"timestamp":1713382639.2005916,"name":"online","context":{"idset":"9509"}}
+{"timestamp":1713382639.4521706,"name":"online","context":{"idset":"9517,9519,9522"}}
+{"timestamp":1713382639.7186072,"name":"online","context":{"idset":"9510-9512,9520-9521"}}
+{"timestamp":1713382639.9741037,"name":"online","context":{"idset":"9513-9514,9516,9518"}}
+{"timestamp":1713382640.1945426,"name":"online","context":{"idset":"9515,9524"}}
+{"timestamp":1713382661.335629,"name":"drain","context":{"idset":"404","reason":"epilog failed for jobid fq6AWWH4x9V","overwrite":0}}
+{"timestamp":1713382774.8277168,"name":"online","context":{"idset":"573-579"}}
+{"timestamp":1713382775.0808651,"name":"online","context":{"idset":"580"}}
+{"timestamp":1713385091.5231142,"name":"undrain","context":{"idset":"877-878"}}
+{"timestamp":1713385202.9542856,"name":"online","context":{"idset":"877"}}
+{"timestamp":1713385203.3938427,"name":"online","context":{"idset":"878"}}
+{"timestamp":1713385929.3193638,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1713385929.3487427,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1713385929.3735104,"name":"offline","context":{"idset":"90"}}
+{"timestamp":1713385929.3983083,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1713385929.4231782,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1713385929.4477446,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1713385929.4732795,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1713385929.4978824,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1713385929.5224173,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1713385929.5696652,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1713385929.5770111,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1713385929.5980895,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1713385929.6227801,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1713385929.643512,"name":"offline","context":{"idset":"117"}}
+{"timestamp":1713385929.680141,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1713385929.744184,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1713385929.7482231,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1713385929.7522762,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1713385929.7569695,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1713385929.7705817,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1713385929.8361409,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1713385929.8774276,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1713385929.8814793,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1713385929.8855147,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1713385929.8951082,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1713385929.9188972,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1713385929.9670727,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1713385929.9834802,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1713385929.9903557,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1713385930.0232418,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1713385930.0453739,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1713385930.0491333,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1713385930.0931957,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1713385930.1457288,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1713385930.1764126,"name":"offline","context":{"idset":"165"}}
+{"timestamp":1713385930.1803606,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1713385930.1893849,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1713385930.1931601,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1713385930.2133818,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1713385930.2371378,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1713385930.2960644,"name":"offline","context":{"idset":"172"}}
+{"timestamp":1713385930.3356729,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1713385930.3408051,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1713385930.3457446,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1713385930.3603489,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1713385930.3841949,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1713385930.3880789,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1713385930.4229994,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1713385930.5083005,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1713385930.5245485,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1713385930.5344341,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1713385930.5564969,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1713385930.5613754,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1713385930.5763505,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1713385930.5968125,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1713385930.6155622,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1713385930.6348193,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1713385930.6579123,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1713385930.7187619,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1713385930.7746785,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1713385930.7919955,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1713385930.7957416,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1713385930.7994707,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1713385930.80318,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1713385930.8166811,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1713385930.8499732,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1713385930.8756459,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1713385930.9011357,"name":"offline","context":{"idset":"206"}}
+{"timestamp":1713385930.961148,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1713385931.014797,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1713385931.0464206,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1713385931.0518465,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1713385931.0565429,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1713385931.0607245,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1713385931.0642257,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1713385931.0704896,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1713385931.1146553,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1713385931.1717467,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1713385931.239033,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1713385931.271318,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1713385931.2800083,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1713385931.2879791,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1713385931.2931261,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1713385931.2987046,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1713385931.3043721,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1713385931.3098941,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1713385931.315335,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1713385931.3479302,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1713385931.4045954,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1713385931.4578979,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1713385931.4744413,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1713385931.4783013,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1713385931.4821718,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1713385931.4860775,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1713385931.4900756,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1713385931.4941399,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1713385931.50126,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1713385931.5206418,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1713385931.5401363,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1713385931.5809605,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1713385931.6364291,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1713385931.6649232,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1713385931.668704,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1713385931.6726222,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1713385931.6765018,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1713385931.6803865,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1713385931.6989224,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1713385931.7177038,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1713385931.738234,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1713385931.7602556,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1713385931.8003323,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1713385931.8400822,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1713385931.8522751,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1713385931.8551729,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1713385931.8581066,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1713385931.86114,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1713385931.8935115,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1713385931.8965476,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1713385931.9133668,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1713385931.9620111,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1713385932.0093095,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1713385932.0480602,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1713385932.0511096,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1713385932.0541654,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1713385932.0572307,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1713385932.0603769,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1713385932.0634372,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1713385932.0731883,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1713385932.1152377,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1713385932.156569,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1713385932.1788473,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1713385932.1818109,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1713385932.1848626,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1713385932.1878171,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1713385932.1907845,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1713385932.2071164,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1713385932.2390604,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1713385932.2842448,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1713385932.3065286,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1713385932.3096018,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1713385932.3126798,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1713385932.3156672,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1713385932.3187058,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1713385932.3219819,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1713385932.3426774,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1713385932.3624249,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1713385932.3910222,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1713385932.4312882,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1713385932.4619346,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1713385932.4648094,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1713385932.4676778,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1713385932.4767597,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1713385932.4995277,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1713385932.5224578,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1713385932.5452135,"name":"offline","context":{"idset":"335"}}
+{"timestamp":1713385932.5664713,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1713385932.6061752,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1713385932.6366417,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1713385932.6395171,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1713385932.6596522,"name":"offline","context":{"idset":"342"}}
+{"timestamp":1713385932.6823859,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1713385932.6852579,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1713385932.7055271,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1713385932.7297332,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1713385932.732708,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1713385932.7768052,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1713385932.8178844,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1713385932.8589158,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1713385932.861871,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1713385932.8709793,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1713385932.8958631,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1713385932.8988378,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1713385932.9188881,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1713385932.9218938,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1713385932.9419501,"name":"offline","context":{"idset":"360"}}
+{"timestamp":1713385932.9927912,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1713385933.0443451,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1713385933.0756204,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1713385933.0785489,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1713385933.081538,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1713385933.1161749,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1713385933.119416,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1713385933.1392162,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1713385933.1618581,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1713385933.1647737,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1713385933.1848304,"name":"offline","context":{"idset":"373"}}
+{"timestamp":1713385933.2171192,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1713385933.267873,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1713385933.3091142,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1713385933.3216004,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1713385933.3244941,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1713385933.3436725,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1713385933.3465331,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1713385933.3666728,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1713385933.3891921,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1713385933.3921382,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1713385933.4124947,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1713385933.4373107,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1713385933.4953146,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1713385933.5363863,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1713385933.5679207,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1713385933.5707958,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1713385933.5736537,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1713385933.5931323,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1713385933.6157894,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1713385933.6186259,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1713385933.6396358,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1713385933.6611755,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1713385933.6640828,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1713385933.7160842,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1713385933.7619572,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1713385933.8031337,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1713385933.8060071,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1713385933.8181593,"name":"offline","context":{"idset":"411"}}
+{"timestamp":1713385933.8211145,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1713385933.8424463,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1713385933.8649812,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1713385933.8679147,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1713385933.8886013,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1713385933.9125237,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1713385933.9537241,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1713385933.9948854,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1713385934.0420084,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1713385934.0461776,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1713385934.0501785,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1713385934.0530827,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1713385934.0607634,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1713385934.0799322,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1713385934.099133,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1713385934.1184821,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1713385934.1629124,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1713385934.2137265,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1713385934.2451999,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1713385934.2480779,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1713385934.2509911,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1713385934.2537837,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1713385934.2566068,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1713385934.2594936,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1713385934.2790251,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1713385934.3315442,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1713385934.4006119,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1713385934.4546323,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1713385934.458137,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1713385934.461586,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1713385934.4649982,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1713385934.4684179,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1713385934.4718254,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1713385934.4753213,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1713385934.478828,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1713385934.4822409,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1713385934.5362918,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1713385934.5858891,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1713385934.6124604,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1713385934.6159651,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1713385934.6194558,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1713385934.6229017,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1713385934.6263316,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1713385934.6297331,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1713385934.6331496,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1713385934.6711783,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1713385934.7309496,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1713385934.7761037,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1713385934.7804768,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1713385934.7849674,"name":"offline","context":{"idset":"763"}}
+{"timestamp":1713385934.789315,"name":"offline","context":{"idset":"765"}}
+{"timestamp":1713385934.7937195,"name":"offline","context":{"idset":"766"}}
+{"timestamp":1713385934.7982142,"name":"offline","context":{"idset":"780"}}
+{"timestamp":1713385934.8164728,"name":"offline","context":{"idset":"809"}}
+{"timestamp":1713385934.8539932,"name":"offline","context":{"idset":"810"}}
+{"timestamp":1713385934.8959868,"name":"offline","context":{"idset":"820"}}
+{"timestamp":1713385934.9374032,"name":"offline","context":{"idset":"833"}}
+{"timestamp":1713385934.9402378,"name":"offline","context":{"idset":"834"}}
+{"timestamp":1713385934.9430957,"name":"offline","context":{"idset":"851"}}
+{"timestamp":1713385934.9458883,"name":"offline","context":{"idset":"852"}}
+{"timestamp":1713385934.9486969,"name":"offline","context":{"idset":"873"}}
+{"timestamp":1713385934.9515224,"name":"offline","context":{"idset":"874"}}
+{"timestamp":1713385934.9990277,"name":"offline","context":{"idset":"877"}}
+{"timestamp":1713385935.0399163,"name":"offline","context":{"idset":"878"}}
+{"timestamp":1713385935.0711527,"name":"offline","context":{"idset":"887"}}
+{"timestamp":1713385935.0739894,"name":"offline","context":{"idset":"888"}}
+{"timestamp":1713385935.0767508,"name":"offline","context":{"idset":"889"}}
+{"timestamp":1713385935.0795512,"name":"offline","context":{"idset":"890"}}
+{"timestamp":1713385935.0843041,"name":"offline","context":{"idset":"891"}}
+{"timestamp":1713385935.1031914,"name":"offline","context":{"idset":"892"}}
+{"timestamp":1713385935.106004,"name":"offline","context":{"idset":"893"}}
+{"timestamp":1713385935.1650851,"name":"offline","context":{"idset":"894"}}
+{"timestamp":1713385935.2059,"name":"offline","context":{"idset":"895"}}
+{"timestamp":1713385935.218097,"name":"offline","context":{"idset":"896"}}
+{"timestamp":1713385935.2208831,"name":"offline","context":{"idset":"897"}}
+{"timestamp":1713385935.2237184,"name":"offline","context":{"idset":"898"}}
+{"timestamp":1713385935.2265062,"name":"offline","context":{"idset":"903"}}
+{"timestamp":1713385935.242974,"name":"offline","context":{"idset":"934"}}
+{"timestamp":1713385935.2615867,"name":"offline","context":{"idset":"943"}}
+{"timestamp":1713385935.3006124,"name":"offline","context":{"idset":"945"}}
+{"timestamp":1713385935.3415132,"name":"offline","context":{"idset":"947"}}
+{"timestamp":1713385935.3538303,"name":"offline","context":{"idset":"951"}}
+{"timestamp":1713385935.356637,"name":"offline","context":{"idset":"952"}}
+{"timestamp":1713385935.3593838,"name":"offline","context":{"idset":"953"}}
+{"timestamp":1713385935.3621337,"name":"offline","context":{"idset":"954"}}
+{"timestamp":1713385935.3648987,"name":"offline","context":{"idset":"971"}}
+{"timestamp":1713385935.3799345,"name":"offline","context":{"idset":"972"}}
+{"timestamp":1713385935.41047,"name":"offline","context":{"idset":"973"}}
+{"timestamp":1713385935.4573479,"name":"offline","context":{"idset":"974"}}
+{"timestamp":1713385935.4890516,"name":"offline","context":{"idset":"975"}}
+{"timestamp":1713385935.4918323,"name":"offline","context":{"idset":"976"}}
+{"timestamp":1713385935.4947066,"name":"offline","context":{"idset":"977"}}
+{"timestamp":1713385935.4974501,"name":"offline","context":{"idset":"978"}}
+{"timestamp":1713385935.5002394,"name":"offline","context":{"idset":"7925"}}
+{"timestamp":1713385935.5159259,"name":"offline","context":{"idset":"7926"}}
+{"timestamp":1713385935.5373659,"name":"offline","context":{"idset":"7927"}}
+{"timestamp":1713385935.5561845,"name":"offline","context":{"idset":"7928"}}
+{"timestamp":1713385935.6036184,"name":"offline","context":{"idset":"7929"}}
+{"timestamp":1713385935.6472766,"name":"offline","context":{"idset":"7930"}}
+{"timestamp":1713385935.6500635,"name":"offline","context":{"idset":"7931"}}
+{"timestamp":1713385935.6528411,"name":"offline","context":{"idset":"7932"}}
+{"timestamp":1713385935.663924,"name":"offline","context":{"idset":"7933"}}
+{"timestamp":1713385935.6666725,"name":"offline","context":{"idset":"7934"}}
+{"timestamp":1713385935.6832416,"name":"offline","context":{"idset":"7935"}}
+{"timestamp":1713385935.702163,"name":"offline","context":{"idset":"7936"}}
+{"timestamp":1713385935.7210503,"name":"offline","context":{"idset":"7937"}}
+{"timestamp":1713385935.7410946,"name":"offline","context":{"idset":"7938"}}
+{"timestamp":1713385935.7842865,"name":"offline","context":{"idset":"7939"}}
+{"timestamp":1713385935.8253345,"name":"offline","context":{"idset":"7940"}}
+{"timestamp":1713385935.8567965,"name":"offline","context":{"idset":"7941"}}
+{"timestamp":1713385935.8595397,"name":"offline","context":{"idset":"7942"}}
+{"timestamp":1713385935.8623364,"name":"offline","context":{"idset":"7943"}}
+{"timestamp":1713385935.8650186,"name":"offline","context":{"idset":"7944"}}
+{"timestamp":1713385935.8677533,"name":"offline","context":{"idset":"7945"}}
+{"timestamp":1713385935.8718555,"name":"offline","context":{"idset":"7946"}}
+{"timestamp":1713385935.8886652,"name":"offline","context":{"idset":"7947"}}
+{"timestamp":1713385935.9050798,"name":"offline","context":{"idset":"7948"}}
+{"timestamp":1713385935.940546,"name":"offline","context":{"idset":"7949"}}
+{"timestamp":1713385935.9853916,"name":"offline","context":{"idset":"7950"}}
+{"timestamp":1713385936.0266063,"name":"offline","context":{"idset":"7951"}}
+{"timestamp":1713385936.0292768,"name":"offline","context":{"idset":"7952"}}
+{"timestamp":1713385936.0320747,"name":"offline","context":{"idset":"7953"}}
+{"timestamp":1713385936.0348072,"name":"offline","context":{"idset":"7954"}}
+{"timestamp":1713385936.0375373,"name":"offline","context":{"idset":"7955"}}
+{"timestamp":1713385936.0402267,"name":"offline","context":{"idset":"7956"}}
+{"timestamp":1713385936.0429258,"name":"offline","context":{"idset":"7957"}}
+{"timestamp":1713385936.0456903,"name":"offline","context":{"idset":"7958"}}
+{"timestamp":1713385936.0696948,"name":"offline","context":{"idset":"7959"}}
+{"timestamp":1713385936.1114087,"name":"offline","context":{"idset":"7960"}}
+{"timestamp":1713385936.1574235,"name":"offline","context":{"idset":"7961"}}
+{"timestamp":1713385936.1698802,"name":"offline","context":{"idset":"7962"}}
+{"timestamp":1713385936.1726177,"name":"offline","context":{"idset":"7963"}}
+{"timestamp":1713385936.1753614,"name":"offline","context":{"idset":"7964"}}
+{"timestamp":1713385936.178026,"name":"offline","context":{"idset":"7965"}}
+{"timestamp":1713385936.1807497,"name":"offline","context":{"idset":"7966"}}
+{"timestamp":1713385936.1834302,"name":"offline","context":{"idset":"7967"}}
+{"timestamp":1713385936.1870837,"name":"offline","context":{"idset":"7968"}}
+{"timestamp":1713385936.229984,"name":"offline","context":{"idset":"7969"}}
+{"timestamp":1713385936.2714889,"name":"offline","context":{"idset":"7970"}}
+{"timestamp":1713385936.3055782,"name":"offline","context":{"idset":"7971"}}
+{"timestamp":1713385936.3083496,"name":"offline","context":{"idset":"7972"}}
+{"timestamp":1713385936.3111007,"name":"offline","context":{"idset":"7973"}}
+{"timestamp":1713385936.3138037,"name":"offline","context":{"idset":"7974"}}
+{"timestamp":1713385936.3164871,"name":"offline","context":{"idset":"7975"}}
+{"timestamp":1713385936.3192253,"name":"offline","context":{"idset":"7976"}}
+{"timestamp":1713385936.3219335,"name":"offline","context":{"idset":"7977"}}
+{"timestamp":1713385936.3245785,"name":"offline","context":{"idset":"7978"}}
+{"timestamp":1713385936.3561354,"name":"offline","context":{"idset":"7979"}}
+{"timestamp":1713385936.3969469,"name":"offline","context":{"idset":"7980"}}
+{"timestamp":1713385936.4300885,"name":"offline","context":{"idset":"7981"}}
+{"timestamp":1713385936.4327595,"name":"offline","context":{"idset":"7982"}}
+{"timestamp":1713385936.4354408,"name":"offline","context":{"idset":"7983"}}
+{"timestamp":1713385936.4380488,"name":"offline","context":{"idset":"7984"}}
+{"timestamp":1713385936.4407103,"name":"offline","context":{"idset":"7985"}}
+{"timestamp":1713385936.4433451,"name":"offline","context":{"idset":"7986"}}
+{"timestamp":1713385936.4459891,"name":"offline","context":{"idset":"7987"}}
+{"timestamp":1713385936.4486821,"name":"offline","context":{"idset":"7988"}}
+{"timestamp":1713385936.4513772,"name":"offline","context":{"idset":"7989"}}
+{"timestamp":1713385936.454073,"name":"offline","context":{"idset":"7990"}}
+{"timestamp":1713385936.4980762,"name":"offline","context":{"idset":"7991"}}
+{"timestamp":1713385936.5390832,"name":"offline","context":{"idset":"7992"}}
+{"timestamp":1713385936.5512877,"name":"offline","context":{"idset":"7993"}}
+{"timestamp":1713385936.553911,"name":"offline","context":{"idset":"7994"}}
+{"timestamp":1713385936.5566278,"name":"offline","context":{"idset":"7995"}}
+{"timestamp":1713385936.5592597,"name":"offline","context":{"idset":"7996"}}
+{"timestamp":1713385936.561898,"name":"offline","context":{"idset":"7997"}}
+{"timestamp":1713385936.564544,"name":"offline","context":{"idset":"7998"}}
+{"timestamp":1713385936.5672245,"name":"offline","context":{"idset":"7999"}}
+{"timestamp":1713385936.5699139,"name":"offline","context":{"idset":"8000"}}
+{"timestamp":1713385936.5725102,"name":"offline","context":{"idset":"8001"}}
+{"timestamp":1713385936.5752473,"name":"offline","context":{"idset":"8002"}}
+{"timestamp":1713385936.5779591,"name":"offline","context":{"idset":"8003"}}
+{"timestamp":1713385936.6097174,"name":"offline","context":{"idset":"8004"}}
+{"timestamp":1713385936.6508253,"name":"offline","context":{"idset":"8005"}}
+{"timestamp":1713385936.6825984,"name":"offline","context":{"idset":"8006"}}
+{"timestamp":1713385936.685384,"name":"offline","context":{"idset":"8007"}}
+{"timestamp":1713385936.6880171,"name":"offline","context":{"idset":"8008"}}
+{"timestamp":1713385936.6906824,"name":"offline","context":{"idset":"8009"}}
+{"timestamp":1713385936.693362,"name":"offline","context":{"idset":"8010"}}
+{"timestamp":1713385936.6960065,"name":"offline","context":{"idset":"8011"}}
+{"timestamp":1713385936.6986237,"name":"offline","context":{"idset":"8012"}}
+{"timestamp":1713385936.7012529,"name":"offline","context":{"idset":"8013"}}
+{"timestamp":1713385936.7039759,"name":"offline","context":{"idset":"8014"}}
+{"timestamp":1713385936.7066119,"name":"offline","context":{"idset":"8015"}}
+{"timestamp":1713385936.7092459,"name":"offline","context":{"idset":"8016"}}
+{"timestamp":1713385936.7407935,"name":"offline","context":{"idset":"8017"}}
+{"timestamp":1713385936.7846944,"name":"offline","context":{"idset":"8018"}}
+{"timestamp":1713385936.8257699,"name":"offline","context":{"idset":"8019"}}
+{"timestamp":1713385936.8379505,"name":"offline","context":{"idset":"8020"}}
+{"timestamp":1713385936.8406329,"name":"offline","context":{"idset":"8021"}}
+{"timestamp":1713385936.843245,"name":"offline","context":{"idset":"8022"}}
+{"timestamp":1713385936.8459055,"name":"offline","context":{"idset":"8023"}}
+{"timestamp":1713385936.8486013,"name":"offline","context":{"idset":"8024"}}
+{"timestamp":1713385936.8512292,"name":"offline","context":{"idset":"8025"}}
+{"timestamp":1713385936.8538799,"name":"offline","context":{"idset":"8026"}}
+{"timestamp":1713385936.8565538,"name":"offline","context":{"idset":"8027"}}
+{"timestamp":1713385936.8592169,"name":"offline","context":{"idset":"8028"}}
+{"timestamp":1713385936.8618946,"name":"offline","context":{"idset":"8029"}}
+{"timestamp":1713385936.8838315,"name":"offline","context":{"idset":"8030"}}
+{"timestamp":1713385936.9245427,"name":"offline","context":{"idset":"8031"}}
+{"timestamp":1713385936.9638588,"name":"offline","context":{"idset":"8032"}}
+{"timestamp":1713385936.9929996,"name":"offline","context":{"idset":"8033"}}
+{"timestamp":1713385936.9955001,"name":"offline","context":{"idset":"8034"}}
+{"timestamp":1713385936.9979527,"name":"offline","context":{"idset":"8035"}}
+{"timestamp":1713385937.0003693,"name":"offline","context":{"idset":"8036"}}
+{"timestamp":1713385937.0028386,"name":"offline","context":{"idset":"8037"}}
+{"timestamp":1713385937.0052938,"name":"offline","context":{"idset":"8038"}}
+{"timestamp":1713385937.0077665,"name":"offline","context":{"idset":"8039"}}
+{"timestamp":1713385937.0102136,"name":"offline","context":{"idset":"8040"}}
+{"timestamp":1713385937.0125794,"name":"offline","context":{"idset":"8041"}}
+{"timestamp":1713385937.0235569,"name":"offline","context":{"idset":"8042"}}
+{"timestamp":1713385937.0593469,"name":"offline","context":{"idset":"8043"}}
+{"timestamp":1713385937.0942397,"name":"offline","context":{"idset":"8044"}}
+{"timestamp":1713385937.128988,"name":"offline","context":{"idset":"8045"}}
+{"timestamp":1713385937.1311123,"name":"offline","context":{"idset":"8046"}}
+{"timestamp":1713385937.1332049,"name":"offline","context":{"idset":"8047"}}
+{"timestamp":1713385937.1353474,"name":"offline","context":{"idset":"8048"}}
+{"timestamp":1713385937.1375012,"name":"offline","context":{"idset":"8049"}}
+{"timestamp":1713385937.1396189,"name":"offline","context":{"idset":"8050"}}
+{"timestamp":1713385937.1417682,"name":"offline","context":{"idset":"8051"}}
+{"timestamp":1713385937.1439216,"name":"offline","context":{"idset":"8052"}}
+{"timestamp":1713385937.161588,"name":"offline","context":{"idset":"9205"}}
+{"timestamp":1713385937.1936991,"name":"offline","context":{"idset":"9206"}}
+{"timestamp":1713385937.22492,"name":"offline","context":{"idset":"9207"}}
+{"timestamp":1713385937.2485373,"name":"offline","context":{"idset":"9208"}}
+{"timestamp":1713385937.2505007,"name":"offline","context":{"idset":"9209"}}
+{"timestamp":1713385937.2524369,"name":"offline","context":{"idset":"9210"}}
+{"timestamp":1713385937.2543962,"name":"offline","context":{"idset":"9211"}}
+{"timestamp":1713385937.2562563,"name":"offline","context":{"idset":"9212"}}
+{"timestamp":1713385937.2581449,"name":"offline","context":{"idset":"9213"}}
+{"timestamp":1713385937.2599912,"name":"offline","context":{"idset":"9214"}}
+{"timestamp":1713385937.2759793,"name":"offline","context":{"idset":"9215"}}
+{"timestamp":1713385937.3054271,"name":"offline","context":{"idset":"9216"}}
+{"timestamp":1713385937.3072336,"name":"offline","context":{"idset":"9217"}}
+{"timestamp":1713385937.3090539,"name":"offline","context":{"idset":"9218"}}
+{"timestamp":1713385937.3108902,"name":"offline","context":{"idset":"9219"}}
+{"timestamp":1713385937.3126786,"name":"offline","context":{"idset":"9220"}}
+{"timestamp":1713385937.3145084,"name":"offline","context":{"idset":"9221"}}
+{"timestamp":1713385937.3162711,"name":"offline","context":{"idset":"9222"}}
+{"timestamp":1713385937.3180234,"name":"offline","context":{"idset":"9223"}}
+{"timestamp":1713385937.3198223,"name":"offline","context":{"idset":"9224"}}
+{"timestamp":1713385937.3215797,"name":"offline","context":{"idset":"9225"}}
+{"timestamp":1713385937.3433881,"name":"offline","context":{"idset":"9226"}}
+{"timestamp":1713385937.3711431,"name":"offline","context":{"idset":"9227"}}
+{"timestamp":1713385937.3862493,"name":"offline","context":{"idset":"9228"}}
+{"timestamp":1713385937.3886471,"name":"offline","context":{"idset":"9229"}}
+{"timestamp":1713385937.3904395,"name":"offline","context":{"idset":"9230"}}
+{"timestamp":1713385937.3921103,"name":"offline","context":{"idset":"9231"}}
+{"timestamp":1713385937.3937678,"name":"offline","context":{"idset":"9232"}}
+{"timestamp":1713385937.3954835,"name":"offline","context":{"idset":"9233"}}
+{"timestamp":1713385937.397135,"name":"offline","context":{"idset":"9234"}}
+{"timestamp":1713385937.3988678,"name":"offline","context":{"idset":"9235"}}
+{"timestamp":1713385937.4069889,"name":"offline","context":{"idset":"9236"}}
+{"timestamp":1713385937.4087062,"name":"offline","context":{"idset":"9237"}}
+{"timestamp":1713385937.4178011,"name":"offline","context":{"idset":"9238"}}
+{"timestamp":1713385937.4347043,"name":"offline","context":{"idset":"9239"}}
+{"timestamp":1713385937.4614639,"name":"offline","context":{"idset":"9240"}}
+{"timestamp":1713385937.4875486,"name":"offline","context":{"idset":"9241"}}
+{"timestamp":1713385937.4952371,"name":"offline","context":{"idset":"9242"}}
+{"timestamp":1713385937.4968634,"name":"offline","context":{"idset":"9243"}}
+{"timestamp":1713385937.4984183,"name":"offline","context":{"idset":"9244"}}
+{"timestamp":1713385937.5000088,"name":"offline","context":{"idset":"9245"}}
+{"timestamp":1713385937.50159,"name":"offline","context":{"idset":"9246"}}
+{"timestamp":1713385937.5044117,"name":"offline","context":{"idset":"9247"}}
+{"timestamp":1713385937.5151119,"name":"offline","context":{"idset":"9248"}}
+{"timestamp":1713385937.5256577,"name":"offline","context":{"idset":"9249"}}
+{"timestamp":1713385937.5364528,"name":"offline","context":{"idset":"9250"}}
+{"timestamp":1713385937.5473006,"name":"offline","context":{"idset":"9251"}}
+{"timestamp":1713385937.5578334,"name":"offline","context":{"idset":"9252"}}
+{"timestamp":1713385937.5686061,"name":"offline","context":{"idset":"9253"}}
+{"timestamp":1713385937.5703845,"name":"offline","context":{"idset":"9254"}}
+{"timestamp":1713385937.5792162,"name":"offline","context":{"idset":"9255"}}
+{"timestamp":1713385937.6020999,"name":"offline","context":{"idset":"9256"}}
+{"timestamp":1713385937.627991,"name":"offline","context":{"idset":"9257"}}
+{"timestamp":1713385937.6534894,"name":"offline","context":{"idset":"9258"}}
+{"timestamp":1713385937.6785662,"name":"offline","context":{"idset":"9259"}}
+{"timestamp":1713385937.6801333,"name":"offline","context":{"idset":"9260"}}
+{"timestamp":1713385937.681689,"name":"offline","context":{"idset":"9261"}}
+{"timestamp":1713385937.6832583,"name":"offline","context":{"idset":"9262"}}
+{"timestamp":1713385937.6847854,"name":"offline","context":{"idset":"9263"}}
+{"timestamp":1713385937.6863463,"name":"offline","context":{"idset":"9264"}}
+{"timestamp":1713385937.687849,"name":"offline","context":{"idset":"9265"}}
+{"timestamp":1713385937.6893482,"name":"offline","context":{"idset":"9266"}}
+{"timestamp":1713385937.6959245,"name":"offline","context":{"idset":"9267"}}
+{"timestamp":1713385937.706394,"name":"offline","context":{"idset":"9268"}}
+{"timestamp":1713385937.7171166,"name":"offline","context":{"idset":"9269"}}
+{"timestamp":1713385937.7274785,"name":"offline","context":{"idset":"9270"}}
+{"timestamp":1713385937.737952,"name":"offline","context":{"idset":"9271"}}
+{"timestamp":1713385937.7394919,"name":"offline","context":{"idset":"9272"}}
+{"timestamp":1713385937.7485898,"name":"offline","context":{"idset":"9273"}}
+{"timestamp":1713385937.7740948,"name":"offline","context":{"idset":"9274"}}
+{"timestamp":1713385937.7990749,"name":"offline","context":{"idset":"9275"}}
+{"timestamp":1713385937.8236508,"name":"offline","context":{"idset":"9276"}}
+{"timestamp":1713385937.8485734,"name":"offline","context":{"idset":"9277"}}
+{"timestamp":1713385937.8733609,"name":"offline","context":{"idset":"9278"}}
+{"timestamp":1713385937.874851,"name":"offline","context":{"idset":"9279"}}
+{"timestamp":1713385937.8763731,"name":"offline","context":{"idset":"9280"}}
+{"timestamp":1713385937.8778329,"name":"offline","context":{"idset":"9281"}}
+{"timestamp":1713385937.8793542,"name":"offline","context":{"idset":"9282"}}
+{"timestamp":1713385937.8808761,"name":"offline","context":{"idset":"9283"}}
+{"timestamp":1713385937.8824031,"name":"offline","context":{"idset":"9284"}}
+{"timestamp":1713385937.8839073,"name":"offline","context":{"idset":"9285"}}
+{"timestamp":1713385937.8854287,"name":"offline","context":{"idset":"9286"}}
+{"timestamp":1713385937.8869071,"name":"offline","context":{"idset":"9287"}}
+{"timestamp":1713385937.8949201,"name":"offline","context":{"idset":"9288"}}
+{"timestamp":1713385937.9112725,"name":"offline","context":{"idset":"9289"}}
+{"timestamp":1713385937.9384739,"name":"offline","context":{"idset":"9290"}}
+{"timestamp":1713385937.9633429,"name":"offline","context":{"idset":"9291"}}
+{"timestamp":1713385937.9882159,"name":"offline","context":{"idset":"9292"}}
+{"timestamp":1713385938.0132461,"name":"offline","context":{"idset":"9293"}}
+{"timestamp":1713385938.01478,"name":"offline","context":{"idset":"9294"}}
+{"timestamp":1713385938.0162795,"name":"offline","context":{"idset":"9295"}}
+{"timestamp":1713385938.0177941,"name":"offline","context":{"idset":"9296"}}
+{"timestamp":1713385938.0192583,"name":"offline","context":{"idset":"9297"}}
+{"timestamp":1713385938.020808,"name":"offline","context":{"idset":"9298"}}
+{"timestamp":1713385938.0223594,"name":"offline","context":{"idset":"9299"}}
+{"timestamp":1713385938.0315163,"name":"offline","context":{"idset":"9300"}}
+{"timestamp":1713385938.041795,"name":"offline","context":{"idset":"9301"}}
+{"timestamp":1713385938.0522313,"name":"offline","context":{"idset":"9302"}}
+{"timestamp":1713385938.0537536,"name":"offline","context":{"idset":"9303"}}
+{"timestamp":1713385938.0629482,"name":"offline","context":{"idset":"9304"}}
+{"timestamp":1713385938.0878918,"name":"offline","context":{"idset":"9305"}}
+{"timestamp":1713385938.1128087,"name":"offline","context":{"idset":"9306"}}
+{"timestamp":1713385938.1374981,"name":"offline","context":{"idset":"9307"}}
+{"timestamp":1713385938.1619911,"name":"offline","context":{"idset":"9308"}}
+{"timestamp":1713385938.1728427,"name":"offline","context":{"idset":"9309"}}
+{"timestamp":1713385938.1753762,"name":"offline","context":{"idset":"9310"}}
+{"timestamp":1713385938.1778502,"name":"offline","context":{"idset":"9311"}}
+{"timestamp":1713385938.1803739,"name":"offline","context":{"idset":"9312"}}
+{"timestamp":1713385938.1828508,"name":"offline","context":{"idset":"9313"}}
+{"timestamp":1713385938.1853423,"name":"offline","context":{"idset":"9314"}}
+{"timestamp":1713385938.1888695,"name":"offline","context":{"idset":"9315"}}
+{"timestamp":1713385938.1993229,"name":"offline","context":{"idset":"9316"}}
+{"timestamp":1713385938.2096624,"name":"offline","context":{"idset":"9317"}}
+{"timestamp":1713385938.2201061,"name":"offline","context":{"idset":"9318"}}
+{"timestamp":1713385938.2227983,"name":"offline","context":{"idset":"9319"}}
+{"timestamp":1713385938.2501724,"name":"offline","context":{"idset":"9320"}}
+{"timestamp":1713385938.2940397,"name":"offline","context":{"idset":"9321"}}
+{"timestamp":1713385938.3409595,"name":"offline","context":{"idset":"9322"}}
+{"timestamp":1713385938.3822095,"name":"offline","context":{"idset":"9323"}}
+{"timestamp":1713385938.4135826,"name":"offline","context":{"idset":"9324"}}
+{"timestamp":1713385938.4160204,"name":"offline","context":{"idset":"9325"}}
+{"timestamp":1713385938.4185355,"name":"offline","context":{"idset":"9326"}}
+{"timestamp":1713385938.4209671,"name":"offline","context":{"idset":"9327"}}
+{"timestamp":1713385938.423481,"name":"offline","context":{"idset":"9328"}}
+{"timestamp":1713385938.4259055,"name":"offline","context":{"idset":"9329"}}
+{"timestamp":1713385938.4283652,"name":"offline","context":{"idset":"9330"}}
+{"timestamp":1713385938.4307771,"name":"offline","context":{"idset":"9331"}}
+{"timestamp":1713385938.4331641,"name":"offline","context":{"idset":"9332"}}
+{"timestamp":1713385938.4356234,"name":"offline","context":{"idset":"9333"}}
+{"timestamp":1713385938.4380784,"name":"offline","context":{"idset":"9334"}}
+{"timestamp":1713385938.4405227,"name":"offline","context":{"idset":"9335"}}
+{"timestamp":1713385938.4717047,"name":"offline","context":{"idset":"9336"}}
+{"timestamp":1713385938.513335,"name":"offline","context":{"idset":"9337"}}
+{"timestamp":1713385938.558527,"name":"offline","context":{"idset":"9338"}}
+{"timestamp":1713385938.5994804,"name":"offline","context":{"idset":"9339"}}
+{"timestamp":1713385938.6019886,"name":"offline","context":{"idset":"9340"}}
+{"timestamp":1713385938.6045241,"name":"offline","context":{"idset":"9341"}}
+{"timestamp":1713385938.606952,"name":"offline","context":{"idset":"9342"}}
+{"timestamp":1713385938.6094024,"name":"offline","context":{"idset":"9343"}}
+{"timestamp":1713385938.6118333,"name":"offline","context":{"idset":"9344"}}
+{"timestamp":1713385938.6143737,"name":"offline","context":{"idset":"9345"}}
+{"timestamp":1713385938.6168694,"name":"offline","context":{"idset":"9346"}}
+{"timestamp":1713385938.6391079,"name":"offline","context":{"idset":"9347"}}
+{"timestamp":1713385938.6806061,"name":"offline","context":{"idset":"9348"}}
+{"timestamp":1713385938.7216625,"name":"offline","context":{"idset":"9349"}}
+{"timestamp":1713385938.7631052,"name":"offline","context":{"idset":"9350"}}
+{"timestamp":1713385938.7656467,"name":"offline","context":{"idset":"9351"}}
+{"timestamp":1713385938.7680876,"name":"offline","context":{"idset":"9352"}}
+{"timestamp":1713385938.7704864,"name":"offline","context":{"idset":"9353"}}
+{"timestamp":1713385938.7729418,"name":"offline","context":{"idset":"9354"}}
+{"timestamp":1713385938.7754285,"name":"offline","context":{"idset":"9355"}}
+{"timestamp":1713385938.7778163,"name":"offline","context":{"idset":"9356"}}
+{"timestamp":1713385938.7802572,"name":"offline","context":{"idset":"9357"}}
+{"timestamp":1713385938.8119485,"name":"offline","context":{"idset":"9358"}}
+{"timestamp":1713385938.8144805,"name":"offline","context":{"idset":"9360"}}
+{"timestamp":1713385938.8168771,"name":"offline","context":{"idset":"9361"}}
+{"timestamp":1713385938.8193426,"name":"offline","context":{"idset":"9362"}}
+{"timestamp":1713385938.8218248,"name":"offline","context":{"idset":"9363"}}
+{"timestamp":1713385938.8243573,"name":"offline","context":{"idset":"9364"}}
+{"timestamp":1713385938.8267884,"name":"offline","context":{"idset":"9365"}}
+{"timestamp":1713385938.8292589,"name":"offline","context":{"idset":"9366"}}
+{"timestamp":1713385938.8318057,"name":"offline","context":{"idset":"9367"}}
+{"timestamp":1713385938.8343222,"name":"offline","context":{"idset":"9368"}}
+{"timestamp":1713385938.8763776,"name":"offline","context":{"idset":"9369"}}
+{"timestamp":1713385938.9172413,"name":"offline","context":{"idset":"9370"}}
+{"timestamp":1713385938.9579568,"name":"offline","context":{"idset":"9371"}}
+{"timestamp":1713385938.9700003,"name":"offline","context":{"idset":"9372"}}
+{"timestamp":1713385938.9724934,"name":"offline","context":{"idset":"9373"}}
+{"timestamp":1713385938.9749167,"name":"offline","context":{"idset":"9374"}}
+{"timestamp":1713385938.9774079,"name":"offline","context":{"idset":"9375"}}
+{"timestamp":1713385938.9798071,"name":"offline","context":{"idset":"9376"}}
+{"timestamp":1713385938.9821873,"name":"offline","context":{"idset":"9377"}}
+{"timestamp":1713385939.0037689,"name":"offline","context":{"idset":"9378"}}
+{"timestamp":1713385939.0353532,"name":"offline","context":{"idset":"9379"}}
+{"timestamp":1713385939.0377824,"name":"offline","context":{"idset":"9380"}}
+{"timestamp":1713385939.0401268,"name":"offline","context":{"idset":"9381"}}
+{"timestamp":1713385939.0426221,"name":"offline","context":{"idset":"9382"}}
+{"timestamp":1713385939.0450001,"name":"offline","context":{"idset":"9383"}}
+{"timestamp":1713385939.0474613,"name":"offline","context":{"idset":"9384"}}
+{"timestamp":1713385939.0498617,"name":"offline","context":{"idset":"9385"}}
+{"timestamp":1713385939.0522962,"name":"offline","context":{"idset":"9386"}}
+{"timestamp":1713385939.0546794,"name":"offline","context":{"idset":"9387"}}
+{"timestamp":1713385939.057128,"name":"offline","context":{"idset":"9388"}}
+{"timestamp":1713385939.0882769,"name":"offline","context":{"idset":"9389"}}
+{"timestamp":1713385939.1295128,"name":"offline","context":{"idset":"9390"}}
+{"timestamp":1713385939.1319306,"name":"offline","context":{"idset":"9391"}}
+{"timestamp":1713385939.1343603,"name":"offline","context":{"idset":"9392"}}
+{"timestamp":1713385939.1367517,"name":"offline","context":{"idset":"9393"}}
+{"timestamp":1713385939.1391213,"name":"offline","context":{"idset":"9394"}}
+{"timestamp":1713385939.1414924,"name":"offline","context":{"idset":"9395"}}
+{"timestamp":1713385939.1438446,"name":"offline","context":{"idset":"9396"}}
+{"timestamp":1713385939.1461864,"name":"offline","context":{"idset":"9397"}}
+{"timestamp":1713385939.1486218,"name":"offline","context":{"idset":"9398"}}
+{"timestamp":1713385939.151077,"name":"offline","context":{"idset":"9399"}}
+{"timestamp":1713385939.1534462,"name":"offline","context":{"idset":"9400"}}
+{"timestamp":1713385939.1557829,"name":"offline","context":{"idset":"9401"}}
+{"timestamp":1713385939.1966221,"name":"offline","context":{"idset":"9402"}}
+{"timestamp":1713385939.2421174,"name":"offline","context":{"idset":"9403"}}
+{"timestamp":1713385939.2541685,"name":"offline","context":{"idset":"9404"}}
+{"timestamp":1713385939.2566257,"name":"offline","context":{"idset":"9405"}}
+{"timestamp":1713385939.2590258,"name":"offline","context":{"idset":"9406"}}
+{"timestamp":1713385939.2614625,"name":"offline","context":{"idset":"9407"}}
+{"timestamp":1713385939.2638764,"name":"offline","context":{"idset":"9408"}}
+{"timestamp":1713385939.2663555,"name":"offline","context":{"idset":"9409"}}
+{"timestamp":1713385939.2687449,"name":"offline","context":{"idset":"9410"}}
+{"timestamp":1713385939.2710946,"name":"offline","context":{"idset":"9411"}}
+{"timestamp":1713385939.2968609,"name":"offline","context":{"idset":"9412"}}
+{"timestamp":1713385939.3389919,"name":"offline","context":{"idset":"9413"}}
+{"timestamp":1713385939.3802195,"name":"offline","context":{"idset":"9414"}}
+{"timestamp":1713385939.3922007,"name":"offline","context":{"idset":"9415"}}
+{"timestamp":1713385939.3946183,"name":"offline","context":{"idset":"9416"}}
+{"timestamp":1713385939.3969979,"name":"offline","context":{"idset":"9417"}}
+{"timestamp":1713385939.3995001,"name":"offline","context":{"idset":"9418"}}
+{"timestamp":1713385939.402473,"name":"offline","context":{"idset":"9419"}}
+{"timestamp":1713385939.4059684,"name":"offline","context":{"idset":"9420"}}
+{"timestamp":1713385939.4090672,"name":"offline","context":{"idset":"9421"}}
+{"timestamp":1713385939.4114645,"name":"offline","context":{"idset":"9422"}}
+{"timestamp":1713385939.4137998,"name":"offline","context":{"idset":"9423"}}
+{"timestamp":1713385939.416132,"name":"offline","context":{"idset":"9424"}}
+{"timestamp":1713385939.4280529,"name":"offline","context":{"idset":"9425"}}
+{"timestamp":1713385939.4685879,"name":"offline","context":{"idset":"9426"}}
+{"timestamp":1713385939.5092955,"name":"offline","context":{"idset":"9427"}}
+{"timestamp":1713385939.5116556,"name":"offline","context":{"idset":"9428"}}
+{"timestamp":1713385939.5139844,"name":"offline","context":{"idset":"9429"}}
+{"timestamp":1713385939.5163562,"name":"offline","context":{"idset":"9430"}}
+{"timestamp":1713385939.5186739,"name":"offline","context":{"idset":"9431"}}
+{"timestamp":1713385939.5210552,"name":"offline","context":{"idset":"9432"}}
+{"timestamp":1713385939.5233862,"name":"offline","context":{"idset":"9433"}}
+{"timestamp":1713385939.525707,"name":"offline","context":{"idset":"9434"}}
+{"timestamp":1713385939.5280535,"name":"offline","context":{"idset":"9435"}}
+{"timestamp":1713385939.5303993,"name":"offline","context":{"idset":"9436"}}
+{"timestamp":1713385939.5327575,"name":"offline","context":{"idset":"9437"}}
+{"timestamp":1713385939.5350869,"name":"offline","context":{"idset":"9438"}}
+{"timestamp":1713385939.5374863,"name":"offline","context":{"idset":"9439"}}
+{"timestamp":1713385939.5397894,"name":"offline","context":{"idset":"9440"}}
+{"timestamp":1713385939.5615461,"name":"offline","context":{"idset":"9441"}}
+{"timestamp":1713385939.6024051,"name":"offline","context":{"idset":"9442"}}
+{"timestamp":1713385939.6434026,"name":"offline","context":{"idset":"9443"}}
+{"timestamp":1713385939.6843812,"name":"offline","context":{"idset":"9444"}}
+{"timestamp":1713385939.6867366,"name":"offline","context":{"idset":"9445"}}
+{"timestamp":1713385939.6890695,"name":"offline","context":{"idset":"9446"}}
+{"timestamp":1713385939.6914408,"name":"offline","context":{"idset":"9447"}}
+{"timestamp":1713385939.6937625,"name":"offline","context":{"idset":"9448"}}
+{"timestamp":1713385939.696111,"name":"offline","context":{"idset":"9449"}}
+{"timestamp":1713385939.6984873,"name":"offline","context":{"idset":"9450"}}
+{"timestamp":1713385939.700767,"name":"offline","context":{"idset":"9451"}}
+{"timestamp":1713385939.7031133,"name":"offline","context":{"idset":"9452"}}
+{"timestamp":1713385939.7054489,"name":"offline","context":{"idset":"9453"}}
+{"timestamp":1713385939.7078149,"name":"offline","context":{"idset":"9454"}}
+{"timestamp":1713385939.7101331,"name":"offline","context":{"idset":"9455"}}
+{"timestamp":1713385939.7124634,"name":"offline","context":{"idset":"9456"}}
+{"timestamp":1713385939.7535934,"name":"offline","context":{"idset":"9457"}}
+{"timestamp":1713385939.7978001,"name":"offline","context":{"idset":"9458"}}
+{"timestamp":1713385939.8384988,"name":"offline","context":{"idset":"9459"}}
+{"timestamp":1713385939.87974,"name":"offline","context":{"idset":"9460"}}
+{"timestamp":1713385939.9110193,"name":"offline","context":{"idset":"9461"}}
+{"timestamp":1713385939.9133945,"name":"offline","context":{"idset":"9462"}}
+{"timestamp":1713385939.915781,"name":"offline","context":{"idset":"9463"}}
+{"timestamp":1713385939.9182382,"name":"offline","context":{"idset":"9464"}}
+{"timestamp":1713385939.92062,"name":"offline","context":{"idset":"9465"}}
+{"timestamp":1713385939.9228902,"name":"offline","context":{"idset":"9466"}}
+{"timestamp":1713385939.9252994,"name":"offline","context":{"idset":"9467"}}
+{"timestamp":1713385939.9275751,"name":"offline","context":{"idset":"9468"}}
+{"timestamp":1713385939.9299567,"name":"offline","context":{"idset":"9469"}}
+{"timestamp":1713385939.9322374,"name":"offline","context":{"idset":"9470"}}
+{"timestamp":1713385939.9345753,"name":"offline","context":{"idset":"9471"}}
+{"timestamp":1713385939.956202,"name":"offline","context":{"idset":"9472"}}
+{"timestamp":1713385939.9970708,"name":"offline","context":{"idset":"9473"}}
+{"timestamp":1713385940.0379262,"name":"offline","context":{"idset":"9474"}}
+{"timestamp":1713385940.0789464,"name":"offline","context":{"idset":"9475"}}
+{"timestamp":1713385940.081315,"name":"offline","context":{"idset":"9476"}}
+{"timestamp":1713385940.0836666,"name":"offline","context":{"idset":"9477"}}
+{"timestamp":1713385940.0860324,"name":"offline","context":{"idset":"9478"}}
+{"timestamp":1713385940.0884163,"name":"offline","context":{"idset":"9479"}}
+{"timestamp":1713385940.0907133,"name":"offline","context":{"idset":"9480"}}
+{"timestamp":1713385940.0930827,"name":"offline","context":{"idset":"9481"}}
+{"timestamp":1713385940.095489,"name":"offline","context":{"idset":"9482"}}
+{"timestamp":1713385940.0978112,"name":"offline","context":{"idset":"9483"}}
+{"timestamp":1713385940.1001823,"name":"offline","context":{"idset":"9484"}}
+{"timestamp":1713385940.1025984,"name":"offline","context":{"idset":"9485"}}
+{"timestamp":1713385940.1049087,"name":"offline","context":{"idset":"9486"}}
+{"timestamp":1713385940.1072354,"name":"offline","context":{"idset":"9487"}}
+{"timestamp":1713385940.1415596,"name":"offline","context":{"idset":"9488"}}
+{"timestamp":1713385940.1825242,"name":"offline","context":{"idset":"9489"}}
+{"timestamp":1713385940.223403,"name":"offline","context":{"idset":"9490"}}
+{"timestamp":1713385940.264394,"name":"offline","context":{"idset":"9491"}}
+{"timestamp":1713385940.2764649,"name":"offline","context":{"idset":"9492"}}
+{"timestamp":1713385940.2788546,"name":"offline","context":{"idset":"9493"}}
+{"timestamp":1713385940.2826838,"name":"offline","context":{"idset":"9494"}}
+{"timestamp":1713385940.2877138,"name":"offline","context":{"idset":"9495"}}
+{"timestamp":1713385940.2923822,"name":"offline","context":{"idset":"9496"}}
+{"timestamp":1713385940.2971635,"name":"offline","context":{"idset":"9497"}}
+{"timestamp":1713385940.301976,"name":"offline","context":{"idset":"9498"}}
+{"timestamp":1713385940.3063476,"name":"offline","context":{"idset":"9499"}}
+{"timestamp":1713385940.3849995,"name":"offline","context":{"idset":"9500"}}
+{"timestamp":1713385940.4599631,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1713385940.534389,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1713385940.5906742,"name":"offline","context":{"idset":"9503"}}
+{"timestamp":1713385940.5942228,"name":"offline","context":{"idset":"9504"}}
+{"timestamp":1713385940.5976982,"name":"offline","context":{"idset":"9505"}}
+{"timestamp":1713385940.6011591,"name":"offline","context":{"idset":"9506"}}
+{"timestamp":1713385940.6046166,"name":"offline","context":{"idset":"9507"}}
+{"timestamp":1713385940.6080618,"name":"offline","context":{"idset":"9508"}}
+{"timestamp":1713385940.6115789,"name":"offline","context":{"idset":"9509"}}
+{"timestamp":1713385940.6655092,"name":"offline","context":{"idset":"9510"}}
+{"timestamp":1713385940.6884229,"name":"offline","context":{"idset":"9511"}}
+{"timestamp":1713385940.692909,"name":"offline","context":{"idset":"9512"}}
+{"timestamp":1713385940.6974208,"name":"offline","context":{"idset":"9513"}}
+{"timestamp":1713385940.701889,"name":"offline","context":{"idset":"9514"}}
+{"timestamp":1713385940.7060218,"name":"offline","context":{"idset":"9515"}}
+{"timestamp":1713385940.7100449,"name":"offline","context":{"idset":"9516"}}
+{"timestamp":1713385940.7140236,"name":"offline","context":{"idset":"9517"}}
+{"timestamp":1713385940.7179019,"name":"offline","context":{"idset":"9518"}}
+{"timestamp":1713385940.7224131,"name":"offline","context":{"idset":"9519"}}
+{"timestamp":1713385940.7268202,"name":"offline","context":{"idset":"9520"}}
+{"timestamp":1713385940.7313142,"name":"offline","context":{"idset":"9521"}}
+{"timestamp":1713385940.7357342,"name":"offline","context":{"idset":"9522"}}
+{"timestamp":1713385940.7576559,"name":"offline","context":{"idset":"9523"}}
+{"timestamp":1713385940.8331912,"name":"offline","context":{"idset":"9524"}}
+{"timestamp":1713385940.898381,"name":"offline","context":{"idset":"9525"}}
+{"timestamp":1713385940.9539938,"name":"offline","context":{"idset":"9526"}}
+{"timestamp":1713385940.9578154,"name":"offline","context":{"idset":"9527"}}
+{"timestamp":1713385940.9615119,"name":"offline","context":{"idset":"9528"}}
+{"timestamp":1713385940.9654267,"name":"offline","context":{"idset":"9529"}}
+{"timestamp":1713385940.9698803,"name":"offline","context":{"idset":"9530"}}
+{"timestamp":1713385940.9742813,"name":"offline","context":{"idset":"9531"}}
+{"timestamp":1713385940.9787195,"name":"offline","context":{"idset":"9532"}}
+{"timestamp":1713385940.9831448,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1713385941.0040238,"name":"offline","context":{"idset":"9536"}}
+{"timestamp":1713385941.0790062,"name":"offline","context":{"idset":"9537"}}
+{"timestamp":1713385941.1537037,"name":"offline","context":{"idset":"9538"}}
+{"timestamp":1713385941.2290623,"name":"offline","context":{"idset":"9539"}}
+{"timestamp":1713385941.2334571,"name":"offline","context":{"idset":"9540"}}
+{"timestamp":1713385941.2378097,"name":"offline","context":{"idset":"9541"}}
+{"timestamp":1713385941.2421925,"name":"offline","context":{"idset":"9542"}}
+{"timestamp":1713385941.2463953,"name":"offline","context":{"idset":"9543"}}
+{"timestamp":1713385941.2506807,"name":"offline","context":{"idset":"9544"}}
+{"timestamp":1713385941.2549429,"name":"offline","context":{"idset":"9545"}}
+{"timestamp":1713385941.2589676,"name":"offline","context":{"idset":"9546"}}
+{"timestamp":1713385941.3083208,"name":"offline","context":{"idset":"9547"}}
+{"timestamp":1713385941.3477688,"name":"offline","context":{"idset":"9548"}}
+{"timestamp":1713385941.3525574,"name":"offline","context":{"idset":"9549"}}
+{"timestamp":1713385941.3570468,"name":"offline","context":{"idset":"9550"}}
+{"timestamp":1713385941.3615017,"name":"offline","context":{"idset":"9551"}}
+{"timestamp":1713385941.3659286,"name":"offline","context":{"idset":"9552"}}
+{"timestamp":1713385941.3702533,"name":"offline","context":{"idset":"9553"}}
+{"timestamp":1713385941.3743021,"name":"offline","context":{"idset":"9554"}}
+{"timestamp":1713385941.3786523,"name":"offline","context":{"idset":"9556"}}
+{"timestamp":1713385941.4379013,"name":"offline","context":{"idset":"9557"}}
+{"timestamp":1713385941.5270998,"name":"offline","context":{"idset":"9564"}}
+{"timestamp":1713385941.5717552,"name":"offline","context":{"idset":"9575"}}
+{"timestamp":1713385941.5765812,"name":"offline","context":{"idset":"9578"}}
+{"timestamp":1713385941.5806608,"name":"offline","context":{"idset":"9585"}}
+{"timestamp":1713385941.5853343,"name":"offline","context":{"idset":"9589"}}
+{"timestamp":1713385941.5896051,"name":"offline","context":{"idset":"9590"}}
+{"timestamp":1713385941.5936973,"name":"offline","context":{"idset":"9591"}}
+{"timestamp":1713385941.5978875,"name":"offline","context":{"idset":"9592"}}
+{"timestamp":1713385941.604001,"name":"offline","context":{"idset":"9593"}}
+{"timestamp":1713385941.6086061,"name":"offline","context":{"idset":"9594"}}
+{"timestamp":1713385941.6700108,"name":"offline","context":{"idset":"9595"}}
+{"timestamp":1713385941.6729,"name":"offline","context":{"idset":"9596"}}
+{"timestamp":1713385941.6764953,"name":"offline","context":{"idset":"9597"}}
+{"timestamp":1713385941.6804187,"name":"offline","context":{"idset":"9598"}}
+{"timestamp":1713385941.6846936,"name":"offline","context":{"idset":"9599"}}
+{"timestamp":1713385941.688936,"name":"offline","context":{"idset":"9600"}}
+{"timestamp":1713385941.6930676,"name":"offline","context":{"idset":"9601"}}
+{"timestamp":1713385941.6977837,"name":"offline","context":{"idset":"9602"}}
+{"timestamp":1713385941.7016163,"name":"offline","context":{"idset":"9603"}}
+{"timestamp":1713385941.7052128,"name":"offline","context":{"idset":"9604"}}
+{"timestamp":1713385941.7089272,"name":"offline","context":{"idset":"9605"}}
+{"timestamp":1713385941.7126169,"name":"offline","context":{"idset":"9606"}}
+{"timestamp":1713385941.7330709,"name":"offline","context":{"idset":"9607"}}
+{"timestamp":1713385941.800339,"name":"offline","context":{"idset":"9608"}}
+{"timestamp":1713385941.8350534,"name":"offline","context":{"idset":"9609"}}
+{"timestamp":1713385941.838599,"name":"offline","context":{"idset":"9610"}}
+{"timestamp":1713385941.8419652,"name":"offline","context":{"idset":"9611"}}
+{"timestamp":1713385941.8453107,"name":"offline","context":{"idset":"9612"}}
+{"timestamp":1713385941.8487403,"name":"offline","context":{"idset":"9613"}}
+{"timestamp":1713385941.8521464,"name":"offline","context":{"idset":"9614"}}
+{"timestamp":1713385941.8555641,"name":"offline","context":{"idset":"9615"}}
+{"timestamp":1713385941.8589511,"name":"offline","context":{"idset":"9616"}}
+{"timestamp":1713385941.8624098,"name":"offline","context":{"idset":"9617"}}
+{"timestamp":1713385941.8657749,"name":"offline","context":{"idset":"9618"}}
+{"timestamp":1713385941.8690865,"name":"offline","context":{"idset":"9619"}}
+{"timestamp":1713385941.8724248,"name":"offline","context":{"idset":"9620"}}
+{"timestamp":1713385941.9338696,"name":"offline","context":{"idset":"9621"}}
+{"timestamp":1713385941.9936182,"name":"offline","context":{"idset":"9622"}}
+{"timestamp":1713385942.0495169,"name":"offline","context":{"idset":"9623"}}
+{"timestamp":1713385942.0940371,"name":"offline","context":{"idset":"9624"}}
+{"timestamp":1713385942.0971458,"name":"offline","context":{"idset":"9625"}}
+{"timestamp":1713385942.1003134,"name":"offline","context":{"idset":"9626"}}
+{"timestamp":1713385942.1034155,"name":"offline","context":{"idset":"9627"}}
+{"timestamp":1713385942.1065109,"name":"offline","context":{"idset":"9628"}}
+{"timestamp":1713385942.1095538,"name":"offline","context":{"idset":"9629"}}
+{"timestamp":1713385942.112612,"name":"offline","context":{"idset":"9630"}}
+{"timestamp":1713385942.1156633,"name":"offline","context":{"idset":"9631"}}
+{"timestamp":1713385942.1753631,"name":"offline","context":{"idset":"9632"}}
+{"timestamp":1713385942.2295768,"name":"offline","context":{"idset":"9635"}}
+{"timestamp":1713385942.2547841,"name":"offline","context":{"idset":"9636"}}
+{"timestamp":1713385942.2573233,"name":"offline","context":{"idset":"9637"}}
+{"timestamp":1713385942.2603276,"name":"offline","context":{"idset":"9638"}}
+{"timestamp":1713385942.2632828,"name":"offline","context":{"idset":"9639"}}
+{"timestamp":1713385942.266233,"name":"offline","context":{"idset":"9640"}}
+{"timestamp":1713385942.2691331,"name":"offline","context":{"idset":"9641"}}
+{"timestamp":1713385942.2717791,"name":"offline","context":{"idset":"9642"}}
+{"timestamp":1713385942.2744434,"name":"offline","context":{"idset":"9643"}}
+{"timestamp":1713385942.277503,"name":"offline","context":{"idset":"9644"}}
+{"timestamp":1713385942.2800756,"name":"offline","context":{"idset":"9645"}}
+{"timestamp":1713385942.3376939,"name":"offline","context":{"idset":"9646"}}
+{"timestamp":1713385942.3940713,"name":"offline","context":{"idset":"9647"}}
+{"timestamp":1713385942.4218726,"name":"offline","context":{"idset":"9648"}}
+{"timestamp":1713385942.4250958,"name":"offline","context":{"idset":"9649"}}
+{"timestamp":1713385942.4281569,"name":"offline","context":{"idset":"9650"}}
+{"timestamp":1713385942.4312756,"name":"offline","context":{"idset":"9651"}}
+{"timestamp":1713385942.4344065,"name":"offline","context":{"idset":"9652"}}
+{"timestamp":1713385942.437465,"name":"offline","context":{"idset":"9653"}}
+{"timestamp":1713385942.440522,"name":"offline","context":{"idset":"9654"}}
+{"timestamp":1713385942.4435692,"name":"offline","context":{"idset":"9655"}}
+{"timestamp":1713385942.4464288,"name":"offline","context":{"idset":"9656"}}
+{"timestamp":1713385942.49031,"name":"offline","context":{"idset":"9657"}}
+{"timestamp":1713385942.5431159,"name":"offline","context":{"idset":"9658"}}
+{"timestamp":1713385942.5869575,"name":"offline","context":{"idset":"9659"}}
+{"timestamp":1713385942.589839,"name":"offline","context":{"idset":"9660"}}
+{"timestamp":1713385942.5928214,"name":"offline","context":{"idset":"9661"}}
+{"timestamp":1713385942.595793,"name":"offline","context":{"idset":"9662"}}
+{"timestamp":1713385942.598635,"name":"offline","context":{"idset":"9663"}}
+{"timestamp":1713385942.6015735,"name":"offline","context":{"idset":"9664"}}
+{"timestamp":1713385942.6034122,"name":"offline","context":{"idset":"9665"}}
+{"timestamp":1713385942.6063359,"name":"offline","context":{"idset":"9666"}}
+{"timestamp":1713385942.6090808,"name":"offline","context":{"idset":"9667"}}
+{"timestamp":1713385942.611836,"name":"offline","context":{"idset":"9668"}}
+{"timestamp":1713385942.6144981,"name":"offline","context":{"idset":"9669"}}
+{"timestamp":1713385942.6169858,"name":"offline","context":{"idset":"9670"}}
+{"timestamp":1713385942.6194568,"name":"offline","context":{"idset":"9671"}}
+{"timestamp":1713385942.6442535,"name":"offline","context":{"idset":"9672"}}
+{"timestamp":1713385942.6903527,"name":"offline","context":{"idset":"9673"}}
+{"timestamp":1713385942.7352448,"name":"offline","context":{"idset":"9674"}}
+{"timestamp":1713385942.7377334,"name":"offline","context":{"idset":"9675"}}
+{"timestamp":1713385942.7402484,"name":"offline","context":{"idset":"9676"}}
+{"timestamp":1713385942.7425308,"name":"offline","context":{"idset":"9677"}}
+{"timestamp":1713385942.7448556,"name":"offline","context":{"idset":"9678"}}
+{"timestamp":1713385942.7472475,"name":"offline","context":{"idset":"9679"}}
+{"timestamp":1713385942.7497211,"name":"offline","context":{"idset":"9680"}}
+{"timestamp":1713385942.752121,"name":"offline","context":{"idset":"9681"}}
+{"timestamp":1713385942.7545075,"name":"offline","context":{"idset":"9682"}}
+{"timestamp":1713385942.7568099,"name":"offline","context":{"idset":"9683"}}
+{"timestamp":1713385942.7591162,"name":"offline","context":{"idset":"9684"}}
+{"timestamp":1713385942.7614346,"name":"offline","context":{"idset":"9685"}}
+{"timestamp":1713385942.8066449,"name":"offline","context":{"idset":"9686"}}
+{"timestamp":1713385942.8513689,"name":"offline","context":{"idset":"9687"}}
+{"timestamp":1713385942.8940122,"name":"offline","context":{"idset":"9688"}}
+{"timestamp":1713385942.9371893,"name":"offline","context":{"idset":"9689"}}
+{"timestamp":1713385942.9573743,"name":"offline","context":{"idset":"9690"}}
+{"timestamp":1713385942.9596689,"name":"offline","context":{"idset":"9691"}}
+{"timestamp":1713385942.9619272,"name":"offline","context":{"idset":"9692"}}
+{"timestamp":1713385942.9641039,"name":"offline","context":{"idset":"9693"}}
+{"timestamp":1713385942.9664209,"name":"offline","context":{"idset":"9694"}}
+{"timestamp":1713385942.9683819,"name":"offline","context":{"idset":"9695"}}
+{"timestamp":1713385942.9746699,"name":"offline","context":{"idset":"9696"}}
+{"timestamp":1713385942.9768305,"name":"offline","context":{"idset":"9697"}}
+{"timestamp":1713385942.9790254,"name":"offline","context":{"idset":"9698"}}
+{"timestamp":1713385942.9814389,"name":"offline","context":{"idset":"9699"}}
+{"timestamp":1713385943.020664,"name":"offline","context":{"idset":"9700"}}
+{"timestamp":1713385943.0605137,"name":"offline","context":{"idset":"9701"}}
+{"timestamp":1713385943.1033659,"name":"offline","context":{"idset":"9702"}}
+{"timestamp":1713385943.1276948,"name":"offline","context":{"idset":"9703"}}
+{"timestamp":1713385943.1300623,"name":"offline","context":{"idset":"9704"}}
+{"timestamp":1713385943.1324677,"name":"offline","context":{"idset":"9705"}}
+{"timestamp":1713385943.1348515,"name":"offline","context":{"idset":"9706"}}
+{"timestamp":1713385943.1370814,"name":"offline","context":{"idset":"9707"}}
+{"timestamp":1713385943.1391459,"name":"offline","context":{"idset":"9708"}}
+{"timestamp":1713385943.1413996,"name":"offline","context":{"idset":"9709"}}
+{"timestamp":1713385943.1439607,"name":"offline","context":{"idset":"9710"}}
+{"timestamp":1713385943.1465178,"name":"offline","context":{"idset":"9711"}}
+{"timestamp":1713385943.1489296,"name":"offline","context":{"idset":"9712"}}
+{"timestamp":1713385943.184238,"name":"offline","context":{"idset":"9713"}}
+{"timestamp":1713385943.2297912,"name":"offline","context":{"idset":"9714"}}
+{"timestamp":1713385943.2730293,"name":"offline","context":{"idset":"9715"}}
+{"timestamp":1713385943.3033214,"name":"offline","context":{"idset":"9716"}}
+{"timestamp":1713385943.3174236,"name":"offline","context":{"idset":"9717"}}
+{"timestamp":1713385943.3187921,"name":"offline","context":{"idset":"9718"}}
+{"timestamp":1713385943.3201141,"name":"offline","context":{"idset":"9719"}}
+{"timestamp":1713385943.3214624,"name":"offline","context":{"idset":"9720"}}
+{"timestamp":1713385943.3227675,"name":"offline","context":{"idset":"9721"}}
+{"timestamp":1713385943.3243601,"name":"offline","context":{"idset":"9722"}}
+{"timestamp":1713385943.3256867,"name":"offline","context":{"idset":"9723"}}
+{"timestamp":1713385943.3271246,"name":"offline","context":{"idset":"9724"}}
+{"timestamp":1713385943.3287108,"name":"offline","context":{"idset":"9725"}}
+{"timestamp":1713385943.3306022,"name":"offline","context":{"idset":"9726"}}
+{"timestamp":1713385943.332094,"name":"offline","context":{"idset":"9727"}}
+{"timestamp":1713385943.3424284,"name":"offline","context":{"idset":"9728"}}
+{"timestamp":1713385943.3700528,"name":"offline","context":{"idset":"9729"}}
+{"timestamp":1713385943.3968713,"name":"offline","context":{"idset":"9730"}}
+{"timestamp":1713385943.424763,"name":"offline","context":{"idset":"9731"}}
+{"timestamp":1713385943.4266832,"name":"offline","context":{"idset":"9732"}}
+{"timestamp":1713385943.4280107,"name":"offline","context":{"idset":"9733"}}
+{"timestamp":1713385943.4293315,"name":"offline","context":{"idset":"9734"}}
+{"timestamp":1713385943.4307218,"name":"offline","context":{"idset":"9735"}}
+{"timestamp":1713385943.4321275,"name":"offline","context":{"idset":"9736"}}
+{"timestamp":1713385943.4335635,"name":"offline","context":{"idset":"9737"}}
+{"timestamp":1713385943.4348817,"name":"offline","context":{"idset":"9738"}}
+{"timestamp":1713385943.4361751,"name":"offline","context":{"idset":"9739"}}
+{"timestamp":1713385943.4374955,"name":"offline","context":{"idset":"9740"}}
+{"timestamp":1713385943.4388309,"name":"offline","context":{"idset":"9741"}}
+{"timestamp":1713385943.4402442,"name":"offline","context":{"idset":"9742"}}
+{"timestamp":1713385943.4415584,"name":"offline","context":{"idset":"9743"}}
+{"timestamp":1713385943.4429989,"name":"offline","context":{"idset":"9744"}}
+{"timestamp":1713385943.4444218,"name":"offline","context":{"idset":"9745"}}
+{"timestamp":1713385943.4642067,"name":"offline","context":{"idset":"9746"}}
+{"timestamp":1713385943.4912033,"name":"offline","context":{"idset":"9747"}}
+{"timestamp":1713385943.5178585,"name":"offline","context":{"idset":"9748"}}
+{"timestamp":1713385943.5449789,"name":"offline","context":{"idset":"9749"}}
+{"timestamp":1713385943.5741675,"name":"offline","context":{"idset":"9750"}}
+{"timestamp":1713385943.5825443,"name":"offline","context":{"idset":"9751"}}
+{"timestamp":1713385943.5839655,"name":"offline","context":{"idset":"9752"}}
+{"timestamp":1713385943.5854459,"name":"offline","context":{"idset":"9753"}}
+{"timestamp":1713385943.58675,"name":"offline","context":{"idset":"9754"}}
+{"timestamp":1713385943.5880437,"name":"offline","context":{"idset":"9755"}}
+{"timestamp":1713385943.5893431,"name":"offline","context":{"idset":"9756"}}
+{"timestamp":1713385943.5906334,"name":"offline","context":{"idset":"9757"}}
+{"timestamp":1713385943.5924442,"name":"offline","context":{"idset":"9758"}}
+{"timestamp":1713385943.5939734,"name":"offline","context":{"idset":"9759"}}
+{"timestamp":1713385943.595324,"name":"offline","context":{"idset":"9760"}}
+{"timestamp":1713385943.5967305,"name":"offline","context":{"idset":"9761"}}
+{"timestamp":1713385943.6189141,"name":"offline","context":{"idset":"9762"}}
+{"timestamp":1713385943.6509433,"name":"offline","context":{"idset":"9763"}}
+{"timestamp":1713385943.6803105,"name":"offline","context":{"idset":"9764"}}
+{"timestamp":1713385943.7080028,"name":"offline","context":{"idset":"9765"}}
+{"timestamp":1713385943.7374668,"name":"offline","context":{"idset":"9766"}}
+{"timestamp":1713385943.7514746,"name":"offline","context":{"idset":"9767"}}
+{"timestamp":1713385943.7531466,"name":"offline","context":{"idset":"9768"}}
+{"timestamp":1713385943.754581,"name":"offline","context":{"idset":"9769"}}
+{"timestamp":1713385943.7559199,"name":"offline","context":{"idset":"9770"}}
+{"timestamp":1713385943.7574177,"name":"offline","context":{"idset":"9771"}}
+{"timestamp":1713385943.7588296,"name":"offline","context":{"idset":"9772"}}
+{"timestamp":1713385943.7602813,"name":"offline","context":{"idset":"9773"}}
+{"timestamp":1713385943.7615666,"name":"offline","context":{"idset":"9774"}}
+{"timestamp":1713385943.7629678,"name":"offline","context":{"idset":"9775"}}
+{"timestamp":1713385943.7643332,"name":"offline","context":{"idset":"9776"}}
+{"timestamp":1713385943.7657528,"name":"offline","context":{"idset":"9777"}}
+{"timestamp":1713385943.7930181,"name":"offline","context":{"idset":"9778"}}
+{"timestamp":1713385943.8233078,"name":"offline","context":{"idset":"9779"}}
+{"timestamp":1713385943.8535452,"name":"offline","context":{"idset":"9780"}}
+{"timestamp":1713385943.8767667,"name":"offline","context":{"idset":"9781"}}
+{"timestamp":1713385943.8781626,"name":"offline","context":{"idset":"9782"}}
+{"timestamp":1713385943.8799469,"name":"offline","context":{"idset":"9783"}}
+{"timestamp":1713385943.881253,"name":"offline","context":{"idset":"9784"}}
+{"timestamp":1713385943.8825495,"name":"offline","context":{"idset":"9785"}}
+{"timestamp":1713385943.8838868,"name":"offline","context":{"idset":"9786"}}
+{"timestamp":1713385943.8851688,"name":"offline","context":{"idset":"9787"}}
+{"timestamp":1713385943.8865972,"name":"offline","context":{"idset":"9788"}}
+{"timestamp":1713385943.8878825,"name":"offline","context":{"idset":"9789"}}
+{"timestamp":1713385943.8891821,"name":"offline","context":{"idset":"9790"}}
+{"timestamp":1713385943.8908093,"name":"offline","context":{"idset":"9791"}}
+{"timestamp":1713385943.8920977,"name":"offline","context":{"idset":"9792"}}
+{"timestamp":1713385943.8937235,"name":"offline","context":{"idset":"9793"}}
+{"timestamp":1713385943.89571,"name":"offline","context":{"idset":"9794"}}
+{"timestamp":1713385943.8971219,"name":"offline","context":{"idset":"9795"}}
+{"timestamp":1713385943.9267557,"name":"offline","context":{"idset":"9796"}}
+{"timestamp":1713385943.9656103,"name":"offline","context":{"idset":"9797"}}
+{"timestamp":1713385943.9943049,"name":"offline","context":{"idset":"9798"}}
+{"timestamp":1713385944.022296,"name":"offline","context":{"idset":"9799"}}
+{"timestamp":1713385944.0245919,"name":"offline","context":{"idset":"9800"}}
+{"timestamp":1713385944.0268152,"name":"offline","context":{"idset":"9801"}}
+{"timestamp":1713385944.029011,"name":"offline","context":{"idset":"9802"}}
+{"timestamp":1713385944.0312819,"name":"offline","context":{"idset":"9803"}}
+{"timestamp":1713385944.0335822,"name":"offline","context":{"idset":"9804"}}
+{"timestamp":1713385944.0359356,"name":"offline","context":{"idset":"9805"}}
+{"timestamp":1713385944.0383685,"name":"offline","context":{"idset":"9806"}}
+{"timestamp":1713385944.0407777,"name":"offline","context":{"idset":"9807"}}
+{"timestamp":1713385944.042536,"name":"offline","context":{"idset":"9808"}}
+{"timestamp":1713385944.0447528,"name":"offline","context":{"idset":"9809"}}
+{"timestamp":1713385944.0578139,"name":"offline","context":{"idset":"9810"}}
+{"timestamp":1713385944.1023648,"name":"offline","context":{"idset":"9811"}}
+{"timestamp":1713385944.1477509,"name":"offline","context":{"idset":"9812"}}
+{"timestamp":1713385944.189914,"name":"offline","context":{"idset":"9813"}}
+{"timestamp":1713385944.2329903,"name":"offline","context":{"idset":"9814"}}
+{"timestamp":1713385944.2530282,"name":"offline","context":{"idset":"9815"}}
+{"timestamp":1713385944.255152,"name":"offline","context":{"idset":"9816"}}
+{"timestamp":1713385944.257333,"name":"offline","context":{"idset":"9817"}}
+{"timestamp":1713385944.2597389,"name":"offline","context":{"idset":"9818"}}
+{"timestamp":1713385944.2621849,"name":"offline","context":{"idset":"9819"}}
+{"timestamp":1713385944.264611,"name":"offline","context":{"idset":"9820"}}
+{"timestamp":1713385944.2669277,"name":"offline","context":{"idset":"9821"}}
+{"timestamp":1713385944.268683,"name":"offline","context":{"idset":"9822"}}
+{"timestamp":1713385944.2709758,"name":"offline","context":{"idset":"9823"}}
+{"timestamp":1713385944.2842774,"name":"offline","context":{"idset":"9824"}}
+{"timestamp":1713385944.3265169,"name":"offline","context":{"idset":"9825"}}
+{"timestamp":1713385944.3689165,"name":"offline","context":{"idset":"9826"}}
+{"timestamp":1713385944.4127469,"name":"offline","context":{"idset":"9827"}}
+{"timestamp":1713385944.4565108,"name":"offline","context":{"idset":"9828"}}
+{"timestamp":1713385944.4587862,"name":"offline","context":{"idset":"9829"}}
+{"timestamp":1713385944.4611142,"name":"offline","context":{"idset":"9830"}}
+{"timestamp":1713385944.4633846,"name":"offline","context":{"idset":"9831"}}
+{"timestamp":1713385944.4654994,"name":"offline","context":{"idset":"9832"}}
+{"timestamp":1713385944.4675398,"name":"offline","context":{"idset":"9833"}}
+{"timestamp":1713385944.4695859,"name":"offline","context":{"idset":"9834"}}
+{"timestamp":1713385944.4717622,"name":"offline","context":{"idset":"9835"}}
+{"timestamp":1713385944.4740214,"name":"offline","context":{"idset":"9836"}}
+{"timestamp":1713385944.4762228,"name":"offline","context":{"idset":"9837"}}
+{"timestamp":1713385944.4783242,"name":"offline","context":{"idset":"9838"}}
+{"timestamp":1713385944.4804947,"name":"offline","context":{"idset":"9839"}}
+{"timestamp":1713385944.4826126,"name":"offline","context":{"idset":"9840"}}
+{"timestamp":1713385944.4849112,"name":"offline","context":{"idset":"9841"}}
+{"timestamp":1713385944.5179365,"name":"offline","context":{"idset":"9842"}}
+{"timestamp":1713385944.5585055,"name":"offline","context":{"idset":"9843"}}
+{"timestamp":1713385944.6031249,"name":"offline","context":{"idset":"9844"}}
+{"timestamp":1713385944.6471815,"name":"offline","context":{"idset":"9845"}}
+{"timestamp":1713385944.6494865,"name":"offline","context":{"idset":"9846"}}
+{"timestamp":1713385944.651741,"name":"offline","context":{"idset":"9847"}}
+{"timestamp":1713385944.6539547,"name":"offline","context":{"idset":"9848"}}
+{"timestamp":1713385944.6561031,"name":"offline","context":{"idset":"9849"}}
+{"timestamp":1713385944.6584127,"name":"offline","context":{"idset":"9850"}}
+{"timestamp":1713385944.6608706,"name":"offline","context":{"idset":"9851"}}
+{"timestamp":1713385944.6632979,"name":"offline","context":{"idset":"9852"}}
+{"timestamp":1713385944.6657028,"name":"offline","context":{"idset":"9853"}}
+{"timestamp":1713385944.6678362,"name":"offline","context":{"idset":"9854"}}
+{"timestamp":1713385944.6702096,"name":"offline","context":{"idset":"9855"}}
+{"timestamp":1713385944.6724346,"name":"offline","context":{"idset":"9856"}}
+{"timestamp":1713385944.6964998,"name":"offline","context":{"idset":"9857"}}
+{"timestamp":1713385944.7448883,"name":"offline","context":{"idset":"9858"}}
+{"timestamp":1713385944.7861271,"name":"offline","context":{"idset":"9859"}}
+{"timestamp":1713385944.8279183,"name":"offline","context":{"idset":"9860"}}
+{"timestamp":1713385944.8736541,"name":"offline","context":{"idset":"9861"}}
+{"timestamp":1713385944.8789196,"name":"offline","context":{"idset":"9862"}}
+{"timestamp":1713385944.8811817,"name":"offline","context":{"idset":"9863"}}
+{"timestamp":1713385944.8836062,"name":"offline","context":{"idset":"9864"}}
+{"timestamp":1713385944.8858223,"name":"offline","context":{"idset":"9865"}}
+{"timestamp":1713385944.8880267,"name":"offline","context":{"idset":"9866"}}
+{"timestamp":1713385944.8904247,"name":"offline","context":{"idset":"9867"}}
+{"timestamp":1713385944.8926625,"name":"offline","context":{"idset":"9868"}}
+{"timestamp":1713385944.8948512,"name":"offline","context":{"idset":"9869"}}
+{"timestamp":1713385944.8970251,"name":"offline","context":{"idset":"9870"}}
+{"timestamp":1713385944.9201808,"name":"offline","context":{"idset":"9871"}}
+{"timestamp":1713385944.9659648,"name":"offline","context":{"idset":"9872"}}
+{"timestamp":1713385945.0106137,"name":"offline","context":{"idset":"9873"}}
+{"timestamp":1713385945.0556691,"name":"offline","context":{"idset":"9874"}}
+{"timestamp":1713385945.0688901,"name":"offline","context":{"idset":"9875"}}
+{"timestamp":1713385945.0713842,"name":"offline","context":{"idset":"9876"}}
+{"timestamp":1713385945.073446,"name":"offline","context":{"idset":"9877"}}
+{"timestamp":1713385945.075572,"name":"offline","context":{"idset":"9878"}}
+{"timestamp":1713385945.0777194,"name":"offline","context":{"idset":"9879"}}
+{"timestamp":1713385945.0799069,"name":"offline","context":{"idset":"9880"}}
+{"timestamp":1713385945.0823245,"name":"offline","context":{"idset":"9881"}}
+{"timestamp":1713385945.0844395,"name":"offline","context":{"idset":"9882"}}
+{"timestamp":1713385945.0864928,"name":"offline","context":{"idset":"9883"}}
+{"timestamp":1713385945.088517,"name":"offline","context":{"idset":"9884"}}
+{"timestamp":1713385945.0905488,"name":"offline","context":{"idset":"9885"}}
+{"timestamp":1713385945.1351542,"name":"offline","context":{"idset":"9886"}}
+{"timestamp":1713385945.180861,"name":"offline","context":{"idset":"9887"}}
+{"timestamp":1713385945.2240484,"name":"offline","context":{"idset":"9888"}}
+{"timestamp":1713385945.2263174,"name":"offline","context":{"idset":"9889"}}
+{"timestamp":1713385945.2285209,"name":"offline","context":{"idset":"9890"}}
+{"timestamp":1713385945.2306554,"name":"offline","context":{"idset":"9891"}}
+{"timestamp":1713385945.2327173,"name":"offline","context":{"idset":"9892"}}
+{"timestamp":1713385945.2349432,"name":"offline","context":{"idset":"9893"}}
+{"timestamp":1713385945.2373283,"name":"offline","context":{"idset":"9894"}}
+{"timestamp":1713385945.2396107,"name":"offline","context":{"idset":"9895"}}
+{"timestamp":1713385945.2418556,"name":"offline","context":{"idset":"9896"}}
+{"timestamp":1713385945.2440715,"name":"offline","context":{"idset":"9897"}}
+{"timestamp":1713385945.246172,"name":"offline","context":{"idset":"9898"}}
+{"timestamp":1713385945.259239,"name":"offline","context":{"idset":"9899"}}
+{"timestamp":1713385945.3025029,"name":"offline","context":{"idset":"9900"}}
+{"timestamp":1713385945.3451993,"name":"offline","context":{"idset":"9901"}}
+{"timestamp":1713385945.3931201,"name":"offline","context":{"idset":"9902"}}
+{"timestamp":1713385945.4275839,"name":"offline","context":{"idset":"9903"}}
+{"timestamp":1713385945.4298193,"name":"offline","context":{"idset":"9904"}}
+{"timestamp":1713385945.431905,"name":"offline","context":{"idset":"9905"}}
+{"timestamp":1713385945.4339626,"name":"offline","context":{"idset":"9906"}}
+{"timestamp":1713385945.4360416,"name":"offline","context":{"idset":"9907"}}
+{"timestamp":1713385945.4383342,"name":"offline","context":{"idset":"9908"}}
+{"timestamp":1713385945.4722924,"name":"offline","context":{"idset":"9909"}}
+{"timestamp":1713385945.5174153,"name":"offline","context":{"idset":"9910"}}
+{"timestamp":1713385945.5621595,"name":"offline","context":{"idset":"9911"}}
+{"timestamp":1713385945.607697,"name":"offline","context":{"idset":"9912"}}
+{"timestamp":1713385945.6098971,"name":"offline","context":{"idset":"9913"}}
+{"timestamp":1713385945.6120665,"name":"offline","context":{"idset":"9914"}}
+{"timestamp":1713385945.6141317,"name":"offline","context":{"idset":"9915"}}
+{"timestamp":1713385945.6163027,"name":"offline","context":{"idset":"9916"}}
+{"timestamp":1713385945.6184323,"name":"offline","context":{"idset":"9917"}}
+{"timestamp":1713385945.6205339,"name":"offline","context":{"idset":"9918"}}
+{"timestamp":1713385945.6225495,"name":"offline","context":{"idset":"9919"}}
+{"timestamp":1713385945.6687775,"name":"offline","context":{"idset":"9920"}}
+{"timestamp":1713385945.6710172,"name":"offline","context":{"idset":"9921"}}
+{"timestamp":1713385945.6732392,"name":"offline","context":{"idset":"9922"}}
+{"timestamp":1713385945.675477,"name":"offline","context":{"idset":"9923"}}
+{"timestamp":1713385945.6775181,"name":"offline","context":{"idset":"9924"}}
+{"timestamp":1713385945.6796558,"name":"offline","context":{"idset":"9925"}}
+{"timestamp":1713385945.6823049,"name":"offline","context":{"idset":"9926"}}
+{"timestamp":1713385945.6847558,"name":"offline","context":{"idset":"9927"}}
+{"timestamp":1713385945.6869154,"name":"offline","context":{"idset":"9928"}}
+{"timestamp":1713385945.68911,"name":"offline","context":{"idset":"9929"}}
+{"timestamp":1713385945.7257133,"name":"offline","context":{"idset":"9930"}}
+{"timestamp":1713385945.7737076,"name":"offline","context":{"idset":"9931"}}
+{"timestamp":1713385945.819803,"name":"offline","context":{"idset":"9932"}}
+{"timestamp":1713385945.821986,"name":"offline","context":{"idset":"9933"}}
+{"timestamp":1713385945.8240664,"name":"offline","context":{"idset":"9934"}}
+{"timestamp":1713385945.8261807,"name":"offline","context":{"idset":"9935"}}
+{"timestamp":1713385945.82813,"name":"offline","context":{"idset":"9936"}}
+{"timestamp":1713385945.8300228,"name":"offline","context":{"idset":"9937"}}
+{"timestamp":1713385945.8319879,"name":"offline","context":{"idset":"9938"}}
+{"timestamp":1713385945.8340693,"name":"offline","context":{"idset":"9939"}}
+{"timestamp":1713385945.836277,"name":"offline","context":{"idset":"9940"}}
+{"timestamp":1713385945.8509071,"name":"offline","context":{"idset":"9941"}}
+{"timestamp":1713385945.8964174,"name":"offline","context":{"idset":"9942"}}
+{"timestamp":1713385945.9071662,"name":"offline","context":{"idset":"9943"}}
+{"timestamp":1713385945.9090195,"name":"offline","context":{"idset":"9944"}}
+{"timestamp":1713385945.9108527,"name":"offline","context":{"idset":"9945"}}
+{"timestamp":1713385945.9126801,"name":"offline","context":{"idset":"9946"}}
+{"timestamp":1713385945.9144866,"name":"offline","context":{"idset":"9947"}}
+{"timestamp":1713385945.916281,"name":"offline","context":{"idset":"9948"}}
+{"timestamp":1713385945.9182942,"name":"offline","context":{"idset":"9949"}}
+{"timestamp":1713385945.9204214,"name":"offline","context":{"idset":"9950"}}
+{"timestamp":1713385945.9222176,"name":"offline","context":{"idset":"9951"}}
+{"timestamp":1713385945.9239891,"name":"offline","context":{"idset":"9952"}}
+{"timestamp":1713385945.9257646,"name":"offline","context":{"idset":"9953"}}
+{"timestamp":1713385945.9706504,"name":"offline","context":{"idset":"9954"}}
+{"timestamp":1713385946.016021,"name":"offline","context":{"idset":"9955"}}
+{"timestamp":1713385946.0401487,"name":"offline","context":{"idset":"9956"}}
+{"timestamp":1713385946.0422862,"name":"offline","context":{"idset":"9957"}}
+{"timestamp":1713385946.0440326,"name":"offline","context":{"idset":"9958"}}
+{"timestamp":1713385946.0459893,"name":"offline","context":{"idset":"9959"}}
+{"timestamp":1713385946.0480165,"name":"offline","context":{"idset":"9960"}}
+{"timestamp":1713385946.0500932,"name":"offline","context":{"idset":"9961"}}
+{"timestamp":1713385946.0522997,"name":"offline","context":{"idset":"9962"}}
+{"timestamp":1713385946.0544708,"name":"offline","context":{"idset":"9963"}}
+{"timestamp":1713385946.0566299,"name":"offline","context":{"idset":"9964"}}
+{"timestamp":1713385946.0692801,"name":"offline","context":{"idset":"9965"}}
+{"timestamp":1713385946.1138659,"name":"offline","context":{"idset":"9966"}}
+{"timestamp":1713385946.1597154,"name":"offline","context":{"idset":"9967"}}
+{"timestamp":1713385946.2040629,"name":"offline","context":{"idset":"9968"}}
+{"timestamp":1713385946.2062304,"name":"offline","context":{"idset":"9969"}}
+{"timestamp":1713385946.208391,"name":"offline","context":{"idset":"9970"}}
+{"timestamp":1713385946.2105272,"name":"offline","context":{"idset":"9971"}}
+{"timestamp":1713385946.212641,"name":"offline","context":{"idset":"9972"}}
+{"timestamp":1713385946.2147219,"name":"offline","context":{"idset":"9973"}}
+{"timestamp":1713385946.2167428,"name":"offline","context":{"idset":"9974"}}
+{"timestamp":1713385946.2188752,"name":"offline","context":{"idset":"9975"}}
+{"timestamp":1713385946.2208889,"name":"offline","context":{"idset":"9976"}}
+{"timestamp":1713385946.233357,"name":"offline","context":{"idset":"9977"}}
+{"timestamp":1713385946.2787037,"name":"offline","context":{"idset":"9978"}}
+{"timestamp":1713385946.325695,"name":"offline","context":{"idset":"9980"}}
+{"timestamp":1713385946.3278916,"name":"offline","context":{"idset":"9981"}}
+{"timestamp":1713385946.3300128,"name":"offline","context":{"idset":"9982"}}
+{"timestamp":1713385946.3321843,"name":"offline","context":{"idset":"9984"}}
+{"timestamp":1713385946.3342113,"name":"offline","context":{"idset":"9985"}}
+{"timestamp":1713385946.3362465,"name":"offline","context":{"idset":"9986"}}
+{"timestamp":1713385946.3383734,"name":"offline","context":{"idset":"9987"}}
+{"timestamp":1713385946.3403633,"name":"offline","context":{"idset":"9988"}}
+{"timestamp":1713385946.3424225,"name":"offline","context":{"idset":"9989"}}
+{"timestamp":1713385946.3444262,"name":"offline","context":{"idset":"9990"}}
+{"timestamp":1713385946.3465409,"name":"offline","context":{"idset":"9993"}}
+{"timestamp":1713385946.3799884,"name":"offline","context":{"idset":"9994"}}
+{"timestamp":1713385946.4260519,"name":"offline","context":{"idset":"9995"}}
+{"timestamp":1713385946.4609671,"name":"offline","context":{"idset":"9996"}}
+{"timestamp":1713385946.4629815,"name":"offline","context":{"idset":"9997"}}
+{"timestamp":1713385946.4649081,"name":"offline","context":{"idset":"9998"}}
+{"timestamp":1713385946.4670041,"name":"offline","context":{"idset":"9999"}}
+{"timestamp":1713385946.469291,"name":"offline","context":{"idset":"10000"}}
+{"timestamp":1713385946.4715178,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1713385946.4737077,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1713385946.4758639,"name":"offline","context":{"idset":"10003"}}
+{"timestamp":1713385946.4777739,"name":"offline","context":{"idset":"10004"}}
+{"timestamp":1713385946.4797478,"name":"offline","context":{"idset":"10005"}}
+{"timestamp":1713385946.4817934,"name":"offline","context":{"idset":"10006"}}
+{"timestamp":1713385946.5253351,"name":"offline","context":{"idset":"10007"}}
+{"timestamp":1713385946.570087,"name":"offline","context":{"idset":"10008"}}
+{"timestamp":1713385946.6052251,"name":"offline","context":{"idset":"10009"}}
+{"timestamp":1713385946.6071866,"name":"offline","context":{"idset":"10010"}}
+{"timestamp":1713385946.6093194,"name":"offline","context":{"idset":"10011"}}
+{"timestamp":1713385946.6113646,"name":"offline","context":{"idset":"10012"}}
+{"timestamp":1713385946.6135094,"name":"offline","context":{"idset":"10013"}}
+{"timestamp":1713385946.6156182,"name":"offline","context":{"idset":"10015"}}
+{"timestamp":1713385946.6176455,"name":"offline","context":{"idset":"10016"}}
+{"timestamp":1713385946.619581,"name":"offline","context":{"idset":"10017"}}
+{"timestamp":1713385946.6215713,"name":"offline","context":{"idset":"10018"}}
+{"timestamp":1713385946.623524,"name":"offline","context":{"idset":"10019"}}
+{"timestamp":1713385946.6254239,"name":"offline","context":{"idset":"10020"}}
+{"timestamp":1713385946.6275263,"name":"offline","context":{"idset":"10021"}}
+{"timestamp":1713385946.6295865,"name":"offline","context":{"idset":"10022"}}
+{"timestamp":1713385946.6316102,"name":"offline","context":{"idset":"10023"}}
+{"timestamp":1713385946.6659136,"name":"offline","context":{"idset":"10024"}}
+{"timestamp":1713385946.7116683,"name":"offline","context":{"idset":"10025"}}
+{"timestamp":1713385946.7573204,"name":"offline","context":{"idset":"10026"}}
+{"timestamp":1713385946.7701626,"name":"offline","context":{"idset":"10027"}}
+{"timestamp":1713385946.7723713,"name":"offline","context":{"idset":"10028"}}
+{"timestamp":1713385946.7743907,"name":"offline","context":{"idset":"10030"}}
+{"timestamp":1713385946.7763855,"name":"offline","context":{"idset":"10031"}}
+{"timestamp":1713385946.7785838,"name":"offline","context":{"idset":"10032"}}
+{"timestamp":1713385946.7808185,"name":"offline","context":{"idset":"10033"}}
+{"timestamp":1713385946.7830496,"name":"offline","context":{"idset":"10034"}}
+{"timestamp":1713385946.7851748,"name":"offline","context":{"idset":"10035"}}
+{"timestamp":1713385946.7870634,"name":"offline","context":{"idset":"10036"}}
+{"timestamp":1713385946.7889557,"name":"offline","context":{"idset":"10037"}}
+{"timestamp":1713385946.7908919,"name":"offline","context":{"idset":"10038"}}
+{"timestamp":1713385946.7929146,"name":"offline","context":{"idset":"10039"}}
+{"timestamp":1713385946.7949257,"name":"offline","context":{"idset":"10040"}}
+{"timestamp":1713385946.8387718,"name":"offline","context":{"idset":"10041"}}
+{"timestamp":1713385946.8762817,"name":"offline","context":{"idset":"10042"}}
+{"timestamp":1713385946.9207504,"name":"offline","context":{"idset":"10043"}}
+{"timestamp":1713385946.966033,"name":"offline","context":{"idset":"10044"}}
+{"timestamp":1713385947.0115824,"name":"offline","context":{"idset":"10045"}}
+{"timestamp":1713385947.0138178,"name":"offline","context":{"idset":"10046"}}
+{"timestamp":1713385947.0159359,"name":"offline","context":{"idset":"10047"}}
+{"timestamp":1713385947.0180631,"name":"offline","context":{"idset":"10048"}}
+{"timestamp":1713385947.0200064,"name":"offline","context":{"idset":"10049"}}
+{"timestamp":1713385947.0219915,"name":"offline","context":{"idset":"10050"}}
+{"timestamp":1713385947.0239499,"name":"offline","context":{"idset":"10051"}}
+{"timestamp":1713385947.0259907,"name":"offline","context":{"idset":"10052"}}
+{"timestamp":1713385947.0282123,"name":"offline","context":{"idset":"10053"}}
+{"timestamp":1713385947.0303972,"name":"offline","context":{"idset":"10054"}}
+{"timestamp":1713385947.0538697,"name":"offline","context":{"idset":"10055"}}
+{"timestamp":1713385947.0985169,"name":"offline","context":{"idset":"10056"}}
+{"timestamp":1713385947.1425626,"name":"offline","context":{"idset":"10057"}}
+{"timestamp":1713385947.1806018,"name":"offline","context":{"idset":"10058"}}
+{"timestamp":1713385947.192672,"name":"offline","context":{"idset":"10059"}}
+{"timestamp":1713385947.1945479,"name":"offline","context":{"idset":"10060"}}
+{"timestamp":1713385947.1963241,"name":"offline","context":{"idset":"10061"}}
+{"timestamp":1713385947.1980793,"name":"offline","context":{"idset":"10062"}}
+{"timestamp":1713385947.1998606,"name":"offline","context":{"idset":"10063"}}
+{"timestamp":1713385947.2017846,"name":"offline","context":{"idset":"10064"}}
+{"timestamp":1713385947.20382,"name":"offline","context":{"idset":"10065"}}
+{"timestamp":1713385947.2056599,"name":"offline","context":{"idset":"10067"}}
+{"timestamp":1713385947.207418,"name":"offline","context":{"idset":"10068"}}
+{"timestamp":1713385947.2195046,"name":"offline","context":{"idset":"10069"}}
+{"timestamp":1713385947.2634904,"name":"offline","context":{"idset":"10070"}}
+{"timestamp":1713385947.308254,"name":"offline","context":{"idset":"10071"}}
+{"timestamp":1713385947.3532515,"name":"offline","context":{"idset":"10072"}}
+{"timestamp":1713385947.3648334,"name":"offline","context":{"idset":"10073"}}
+{"timestamp":1713385947.3666885,"name":"offline","context":{"idset":"10074"}}
+{"timestamp":1713385947.3686001,"name":"offline","context":{"idset":"10075"}}
+{"timestamp":1713385947.3705149,"name":"offline","context":{"idset":"10076"}}
+{"timestamp":1713385947.3724208,"name":"offline","context":{"idset":"10077"}}
+{"timestamp":1713385947.3743665,"name":"offline","context":{"idset":"10078"}}
+{"timestamp":1713385947.3763473,"name":"offline","context":{"idset":"10079"}}
+{"timestamp":1713385947.378319,"name":"offline","context":{"idset":"10080"}}
+{"timestamp":1713385947.3801985,"name":"offline","context":{"idset":"10081"}}
+{"timestamp":1713385947.3821774,"name":"offline","context":{"idset":"10082"}}
+{"timestamp":1713385947.4157133,"name":"offline","context":{"idset":"10083"}}
+{"timestamp":1713385947.4603984,"name":"offline","context":{"idset":"10084"}}
+{"timestamp":1713385947.5067554,"name":"offline","context":{"idset":"10085"}}
+{"timestamp":1713385947.5304794,"name":"offline","context":{"idset":"10086"}}
+{"timestamp":1713385947.5321474,"name":"offline","context":{"idset":"10087"}}
+{"timestamp":1713385947.5337903,"name":"offline","context":{"idset":"10088"}}
+{"timestamp":1713385947.5354409,"name":"offline","context":{"idset":"10089"}}
+{"timestamp":1713385947.5370727,"name":"offline","context":{"idset":"10090"}}
+{"timestamp":1713385947.538753,"name":"offline","context":{"idset":"10091"}}
+{"timestamp":1713385947.5404258,"name":"offline","context":{"idset":"10092"}}
+{"timestamp":1713385947.5423889,"name":"offline","context":{"idset":"10093"}}
+{"timestamp":1713385947.5443282,"name":"offline","context":{"idset":"10094"}}
+{"timestamp":1713385947.5462089,"name":"offline","context":{"idset":"10095"}}
+{"timestamp":1713385947.5481565,"name":"offline","context":{"idset":"10096"}}
+{"timestamp":1713385947.5707431,"name":"offline","context":{"idset":"10097"}}
+{"timestamp":1713385947.6145523,"name":"offline","context":{"idset":"10098"}}
+{"timestamp":1713385947.6531229,"name":"offline","context":{"idset":"10099"}}
+{"timestamp":1713385947.6945467,"name":"offline","context":{"idset":"10100"}}
+{"timestamp":1713385947.7175753,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1713385947.7194674,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1713385947.721549,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1713385947.7233901,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1713385947.725354,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1713385947.7273052,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1713385947.7292521,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1713385947.731293,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1713385947.7332847,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1713385947.7353246,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1713385947.7586126,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1713385947.8038535,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1713385947.8486195,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1713385947.891609,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1713385947.9147205,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1713385947.9165728,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1713385947.9185169,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1713385947.9204347,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1713385947.9223828,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1713385947.9243631,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1713385947.9264808,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1713385947.928582,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1713385947.9306886,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1713385947.9327064,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1713385947.9346895,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1713385947.9561574,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1713385947.9973145,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1713385948.0391517,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1713385948.0714388,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1713385948.0733902,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1713385948.0752828,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1713385948.0770755,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1713385948.0789635,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1713385948.0808482,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1713385948.0829444,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1713385948.0850494,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1713385948.0871494,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1713385948.0892093,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1713385948.0911922,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1713385948.124336,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1713385948.1690516,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1713385948.2125771,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1713385948.2352421,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1713385948.2371235,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1713385948.2389576,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1713385948.2406967,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1713385948.242573,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1713385948.244473,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1713385948.2465179,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1713385948.2485278,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1713385948.2504768,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1713385948.2524211,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1713385948.2541769,"name":"offline","context":{"idset":"10153"}}
+{"timestamp":1713385948.2560799,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1713385948.2580049,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1713385948.2915449,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1713385948.3337224,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1713385948.3783858,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1713385948.4121528,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1713385948.4141498,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1713385948.4162383,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1713385948.4183335,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1713385948.4202592,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1713385948.4221308,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1713385948.4239988,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1713385948.425868,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1713385948.4277782,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1713385948.4297354,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1713385948.4316003,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1713385948.4743545,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1713385948.5185392,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1713385948.5620344,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1713385948.5846589,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1713385948.5863261,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1713385948.5880949,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1713385948.5902421,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1713385948.5924318,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1713385948.5945127,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1713385948.5961983,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1713385948.5978508,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1713385948.5994897,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1713385948.6317837,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1713385948.6741848,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1713385948.7162366,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1713385948.7510884,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1713385948.7531631,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1713385948.7551615,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1713385948.7571626,"name":"offline","context":{"idset":"10188"}}
+{"timestamp":1713385948.759151,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1713385948.7611337,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1713385948.7631235,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1713385948.7763398,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1713385948.8185387,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1713385948.8509977,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1713385948.8526821,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1713385948.8543887,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1713385948.8560643,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1713385948.8577144,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1713385948.8593674,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1713385948.8609889,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1713385948.8626311,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1713385948.8642566,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1713385948.8660965,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1713385948.8679373,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1713385948.8943565,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1713385948.9192979,"name":"offline","context":{"idset":"10206"}}
+{"timestamp":1713385948.9263222,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1713385948.9273744,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1713385948.9284246,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1713385948.9294629,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1713385948.9307013,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1713385948.932019,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1713385948.9336102,"name":"offline","context":{"idset":"10213"}}
+{"timestamp":1713385948.9351351,"name":"offline","context":{"idset":"10214"}}
+{"timestamp":1713385948.9366794,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1713385948.938242,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1713385948.939822,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1713385948.9413905,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1713385948.9603384,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1713385948.9850581,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1713385949.0104358,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1713385949.0173681,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1713385949.0183926,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1713385949.0194142,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1713385949.0204329,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1713385949.0214489,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1713385949.0224676,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1713385949.0234911,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1713385949.0245242,"name":"offline","context":{"idset":"10229"}}
+{"timestamp":1713385949.0255468,"name":"offline","context":{"idset":"10230"}}
+{"timestamp":1713385949.0265694,"name":"offline","context":{"idset":"10231"}}
+{"timestamp":1713385949.0275919,"name":"offline","context":{"idset":"10232"}}
+{"timestamp":1713385949.0286114,"name":"offline","context":{"idset":"10233"}}
+{"timestamp":1713385949.0529158,"name":"offline","context":{"idset":"10234"}}
+{"timestamp":1713385949.0777185,"name":"offline","context":{"idset":"10235"}}
+{"timestamp":1713385949.1019037,"name":"offline","context":{"idset":"10236"}}
+{"timestamp":1713385949.1260743,"name":"offline","context":{"idset":"10237"}}
+{"timestamp":1713385949.1270819,"name":"offline","context":{"idset":"10238"}}
+{"timestamp":1713385949.1280792,"name":"offline","context":{"idset":"10239"}}
+{"timestamp":1713385949.1290784,"name":"offline","context":{"idset":"10240"}}
+{"timestamp":1713385949.1300709,"name":"offline","context":{"idset":"10241"}}
+{"timestamp":1713385949.1310682,"name":"offline","context":{"idset":"10242"}}
+{"timestamp":1713385949.1320717,"name":"offline","context":{"idset":"10243"}}
+{"timestamp":1713385949.1330731,"name":"offline","context":{"idset":"10244"}}
+{"timestamp":1713385949.1340723,"name":"offline","context":{"idset":"10245"}}
+{"timestamp":1713385949.1350734,"name":"offline","context":{"idset":"10246"}}
+{"timestamp":1713385949.1360645,"name":"offline","context":{"idset":"10247"}}
+{"timestamp":1713385949.1485891,"name":"offline","context":{"idset":"10248"}}
+{"timestamp":1713385949.1725469,"name":"offline","context":{"idset":"10249"}}
+{"timestamp":1713385949.1965487,"name":"offline","context":{"idset":"10250"}}
+{"timestamp":1713385949.2166574,"name":"offline","context":{"idset":"10251"}}
+{"timestamp":1713385949.217658,"name":"offline","context":{"idset":"10252"}}
+{"timestamp":1713385949.2186532,"name":"offline","context":{"idset":"10253"}}
+{"timestamp":1713385949.2196434,"name":"offline","context":{"idset":"10254"}}
+{"timestamp":1713385949.2206311,"name":"offline","context":{"idset":"10255"}}
+{"timestamp":1713385949.2216198,"name":"offline","context":{"idset":"10256"}}
+{"timestamp":1713385949.243557,"name":"offline","context":{"idset":"10257"}}
+{"timestamp":1713385949.2676976,"name":"offline","context":{"idset":"10258"}}
+{"timestamp":1713385949.2915814,"name":"offline","context":{"idset":"10259"}}
+{"timestamp":1713385949.2983646,"name":"offline","context":{"idset":"10260"}}
+{"timestamp":1713385949.2993648,"name":"offline","context":{"idset":"10261"}}
+{"timestamp":1713385949.3003607,"name":"offline","context":{"idset":"10262"}}
+{"timestamp":1713385949.3013499,"name":"offline","context":{"idset":"10263"}}
+{"timestamp":1713385949.3023295,"name":"offline","context":{"idset":"10264"}}
+{"timestamp":1713385949.3033121,"name":"offline","context":{"idset":"10265"}}
+{"timestamp":1713385949.3042946,"name":"offline","context":{"idset":"10266"}}
+{"timestamp":1713385949.3283079,"name":"offline","context":{"idset":"10267"}}
+{"timestamp":1713385949.335058,"name":"offline","context":{"idset":"10268"}}
+{"timestamp":1713385949.3360362,"name":"offline","context":{"idset":"10269"}}
+{"timestamp":1713385949.337014,"name":"offline","context":{"idset":"10270"}}
+{"timestamp":1713385949.3379955,"name":"offline","context":{"idset":"10271"}}
+{"timestamp":1713385949.3389895,"name":"offline","context":{"idset":"10272"}}
+{"timestamp":1713385949.3399634,"name":"offline","context":{"idset":"10273"}}
+{"timestamp":1713385949.3409538,"name":"offline","context":{"idset":"10274"}}
+{"timestamp":1713385949.3419354,"name":"offline","context":{"idset":"10275"}}
+{"timestamp":1713385949.3429244,"name":"offline","context":{"idset":"10276"}}
+{"timestamp":1713385949.3439112,"name":"offline","context":{"idset":"10277"}}
+{"timestamp":1713385949.3448808,"name":"offline","context":{"idset":"10278"}}
+{"timestamp":1713385949.3689363,"name":"offline","context":{"idset":"10279"}}
+{"timestamp":1713385949.3871493,"name":"offline","context":{"idset":"10280"}}
+{"timestamp":1713385949.3881235,"name":"offline","context":{"idset":"10281"}}
+{"timestamp":1713385949.3890941,"name":"offline","context":{"idset":"10282"}}
+{"timestamp":1713385949.390064,"name":"offline","context":{"idset":"10283"}}
+{"timestamp":1713385949.3910313,"name":"offline","context":{"idset":"10284"}}
+{"timestamp":1713385949.3920012,"name":"offline","context":{"idset":"10285"}}
+{"timestamp":1713385949.3929679,"name":"offline","context":{"idset":"10286"}}
+{"timestamp":1713385949.3939357,"name":"offline","context":{"idset":"10287"}}
+{"timestamp":1713385949.3949108,"name":"offline","context":{"idset":"10288"}}
+{"timestamp":1713385949.3958762,"name":"offline","context":{"idset":"10289"}}
+{"timestamp":1713385949.3968365,"name":"offline","context":{"idset":"10290"}}
+{"timestamp":1713385949.3977935,"name":"offline","context":{"idset":"10291"}}
+{"timestamp":1713385949.398767,"name":"offline","context":{"idset":"10292"}}
+{"timestamp":1713385949.4169524,"name":"offline","context":{"idset":"10293"}}
+{"timestamp":1713385949.440948,"name":"offline","context":{"idset":"10294"}}
+{"timestamp":1713385949.4672334,"name":"offline","context":{"idset":"10295"}}
+{"timestamp":1713385949.4797466,"name":"offline","context":{"idset":"10296"}}
+{"timestamp":1713385949.4807167,"name":"offline","context":{"idset":"10297"}}
+{"timestamp":1713385949.4816754,"name":"offline","context":{"idset":"10298"}}
+{"timestamp":1713385949.4826362,"name":"offline","context":{"idset":"10299"}}
+{"timestamp":1713385949.4835911,"name":"offline","context":{"idset":"10300"}}
+{"timestamp":1713385949.4845493,"name":"offline","context":{"idset":"10301"}}
+{"timestamp":1713385949.4855127,"name":"offline","context":{"idset":"10302"}}
+{"timestamp":1713385949.4864781,"name":"offline","context":{"idset":"10303"}}
+{"timestamp":1713385949.4874425,"name":"offline","context":{"idset":"10304"}}
+{"timestamp":1713385949.4884067,"name":"offline","context":{"idset":"10305"}}
+{"timestamp":1713385949.4893639,"name":"offline","context":{"idset":"10306"}}
+{"timestamp":1713385949.501842,"name":"offline","context":{"idset":"10307"}}
+{"timestamp":1713385949.5257823,"name":"offline","context":{"idset":"10308"}}
+{"timestamp":1713385949.5497277,"name":"offline","context":{"idset":"10309"}}
+{"timestamp":1713385949.5737698,"name":"offline","context":{"idset":"10310"}}
+{"timestamp":1713385949.5804658,"name":"offline","context":{"idset":"10311"}}
+{"timestamp":1713385949.5814269,"name":"offline","context":{"idset":"10312"}}
+{"timestamp":1713385949.5823865,"name":"offline","context":{"idset":"10313"}}
+{"timestamp":1713385949.5833404,"name":"offline","context":{"idset":"10314"}}
+{"timestamp":1713385949.5842934,"name":"offline","context":{"idset":"10315"}}
+{"timestamp":1713385949.5852332,"name":"offline","context":{"idset":"10316"}}
+{"timestamp":1713385949.5861862,"name":"offline","context":{"idset":"10317"}}
+{"timestamp":1713385949.5871389,"name":"offline","context":{"idset":"10318"}}
+{"timestamp":1713385949.5880842,"name":"offline","context":{"idset":"10319"}}
+{"timestamp":1713385949.5948715,"name":"offline","context":{"idset":"10320"}}
+{"timestamp":1713385949.6187079,"name":"offline","context":{"idset":"10321"}}
+{"timestamp":1713385949.6426115,"name":"offline","context":{"idset":"10322"}}
+{"timestamp":1713385949.6690257,"name":"offline","context":{"idset":"10323"}}
+{"timestamp":1713385949.6814256,"name":"offline","context":{"idset":"10324"}}
+{"timestamp":1713385949.6823921,"name":"offline","context":{"idset":"10325"}}
+{"timestamp":1713385949.6833537,"name":"offline","context":{"idset":"10326"}}
+{"timestamp":1713385949.6843009,"name":"offline","context":{"idset":"10327"}}
+{"timestamp":1713385949.6852317,"name":"offline","context":{"idset":"10328"}}
+{"timestamp":1713385949.6861758,"name":"offline","context":{"idset":"10329"}}
+{"timestamp":1713385949.6871171,"name":"offline","context":{"idset":"10330"}}
+{"timestamp":1713385949.6880567,"name":"offline","context":{"idset":"10331"}}
+{"timestamp":1713385949.6889942,"name":"offline","context":{"idset":"10332"}}
+{"timestamp":1713385949.7071619,"name":"offline","context":{"idset":"10333"}}
+{"timestamp":1713385949.7310138,"name":"offline","context":{"idset":"10334"}}
+{"timestamp":1713385949.754698,"name":"offline","context":{"idset":"10335"}}
+{"timestamp":1713385949.7726331,"name":"offline","context":{"idset":"10336"}}
+{"timestamp":1713385949.773571,"name":"offline","context":{"idset":"10337"}}
+{"timestamp":1713385949.774509,"name":"offline","context":{"idset":"10338"}}
+{"timestamp":1713385949.7754486,"name":"offline","context":{"idset":"10339"}}
+{"timestamp":1713385949.7763877,"name":"offline","context":{"idset":"10340"}}
+{"timestamp":1713385949.7773216,"name":"offline","context":{"idset":"10341"}}
+{"timestamp":1713385949.7782433,"name":"offline","context":{"idset":"10342"}}
+{"timestamp":1713385949.7791772,"name":"offline","context":{"idset":"10343"}}
+{"timestamp":1713385949.780107,"name":"offline","context":{"idset":"10344"}}
+{"timestamp":1713385949.78104,"name":"offline","context":{"idset":"10345"}}
+{"timestamp":1713385949.7819693,"name":"offline","context":{"idset":"10346"}}
+{"timestamp":1713385949.7829044,"name":"offline","context":{"idset":"10347"}}
+{"timestamp":1713385949.8009942,"name":"offline","context":{"idset":"10348"}}
+{"timestamp":1713385949.8248632,"name":"offline","context":{"idset":"10349"}}
+{"timestamp":1713385949.8371465,"name":"offline","context":{"idset":"10350"}}
+{"timestamp":1713385949.8380771,"name":"offline","context":{"idset":"10351"}}
+{"timestamp":1713385949.8390164,"name":"offline","context":{"idset":"10352"}}
+{"timestamp":1713385949.8399408,"name":"offline","context":{"idset":"10353"}}
+{"timestamp":1713385949.8408654,"name":"offline","context":{"idset":"10354"}}
+{"timestamp":1713385949.8417892,"name":"offline","context":{"idset":"10355"}}
+{"timestamp":1713385949.8427103,"name":"offline","context":{"idset":"10356"}}
+{"timestamp":1713385949.8436375,"name":"offline","context":{"idset":"10357"}}
+{"timestamp":1713385949.8445604,"name":"offline","context":{"idset":"10358"}}
+{"timestamp":1713385949.8454876,"name":"offline","context":{"idset":"10359"}}
+{"timestamp":1713385949.8464034,"name":"offline","context":{"idset":"10360"}}
+{"timestamp":1713385949.847321,"name":"offline","context":{"idset":"10361"}}
+{"timestamp":1713385949.8482287,"name":"offline","context":{"idset":"10362"}}
+{"timestamp":1713385949.8491452,"name":"offline","context":{"idset":"10363"}}
+{"timestamp":1713385949.8500695,"name":"offline","context":{"idset":"10364"}}
+{"timestamp":1713385949.8509851,"name":"offline","context":{"idset":"10365"}}
+{"timestamp":1713385949.8518977,"name":"offline","context":{"idset":"10366"}}
+{"timestamp":1713385949.8528128,"name":"offline","context":{"idset":"10367"}}
+{"timestamp":1713385949.8650811,"name":"offline","context":{"idset":"10368"}}
+{"timestamp":1713385949.8887296,"name":"offline","context":{"idset":"10369"}}
+{"timestamp":1713385949.9123812,"name":"offline","context":{"idset":"10370"}}
+{"timestamp":1713385949.9359813,"name":"offline","context":{"idset":"10371"}}
+{"timestamp":1713385949.9597287,"name":"offline","context":{"idset":"10372"}}
+{"timestamp":1713385949.9663577,"name":"offline","context":{"idset":"10373"}}
+{"timestamp":1713385949.9672585,"name":"offline","context":{"idset":"10374"}}
+{"timestamp":1713385949.9681766,"name":"offline","context":{"idset":"10375"}}
+{"timestamp":1713385949.9691045,"name":"offline","context":{"idset":"10376"}}
+{"timestamp":1713385949.9700356,"name":"offline","context":{"idset":"10377"}}
+{"timestamp":1713385949.9709489,"name":"offline","context":{"idset":"10378"}}
+{"timestamp":1713385949.9718609,"name":"offline","context":{"idset":"10379"}}
+{"timestamp":1713385949.9727781,"name":"offline","context":{"idset":"10380"}}
+{"timestamp":1713385949.9736884,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1713385949.974597,"name":"offline","context":{"idset":"10382"}}
+{"timestamp":1713385949.975502,"name":"offline","context":{"idset":"10383"}}
+{"timestamp":1713385949.9764221,"name":"offline","context":{"idset":"10384"}}
+{"timestamp":1713385949.9773276,"name":"offline","context":{"idset":"10385"}}
+{"timestamp":1713385949.9782197,"name":"offline","context":{"idset":"10386"}}
+{"timestamp":1713385949.9791219,"name":"offline","context":{"idset":"10387"}}
+{"timestamp":1713385949.9800324,"name":"offline","context":{"idset":"10388"}}
+{"timestamp":1713385949.9809525,"name":"offline","context":{"idset":"10389"}}
+{"timestamp":1713385949.9818671,"name":"offline","context":{"idset":"10390"}}
+{"timestamp":1713385949.9827774,"name":"offline","context":{"idset":"10391"}}
+{"timestamp":1713385949.9836991,"name":"offline","context":{"idset":"10392"}}
+{"timestamp":1713385949.9903097,"name":"offline","context":{"idset":"10393"}}
+{"timestamp":1713385950.0139699,"name":"offline","context":{"idset":"10394"}}
+{"timestamp":1713385950.0375457,"name":"offline","context":{"idset":"10395"}}
+{"timestamp":1713385950.0611298,"name":"offline","context":{"idset":"10396"}}
+{"timestamp":1713385950.084723,"name":"offline","context":{"idset":"10397"}}
+{"timestamp":1713385950.102634,"name":"offline","context":{"idset":"10398"}}
+{"timestamp":1713385950.1035383,"name":"offline","context":{"idset":"10399"}}
+{"timestamp":1713385950.1044405,"name":"offline","context":{"idset":"10400"}}
+{"timestamp":1713385950.1053383,"name":"offline","context":{"idset":"10401"}}
+{"timestamp":1713385950.1062274,"name":"offline","context":{"idset":"10402"}}
+{"timestamp":1713385950.1071374,"name":"offline","context":{"idset":"10403"}}
+{"timestamp":1713385950.108037,"name":"offline","context":{"idset":"10404"}}
+{"timestamp":1713385950.1089506,"name":"offline","context":{"idset":"10405"}}
+{"timestamp":1713385950.1098652,"name":"offline","context":{"idset":"10406"}}
+{"timestamp":1713385950.1107657,"name":"offline","context":{"idset":"10407"}}
+{"timestamp":1713385950.1116481,"name":"offline","context":{"idset":"10408"}}
+{"timestamp":1713385950.1125319,"name":"offline","context":{"idset":"10410"}}
+{"timestamp":1713385950.1134706,"name":"offline","context":{"idset":"10411"}}
+{"timestamp":1713385950.1144161,"name":"offline","context":{"idset":"10412"}}
+{"timestamp":1713385950.1153572,"name":"offline","context":{"idset":"10413"}}
+{"timestamp":1713385950.1162901,"name":"offline","context":{"idset":"10414"}}
+{"timestamp":1713385950.1332428,"name":"offline","context":{"idset":"10415"}}
+{"timestamp":1713385950.1600685,"name":"offline","context":{"idset":"10416"}}
+{"timestamp":1713385950.1906238,"name":"offline","context":{"idset":"10417"}}
+{"timestamp":1713385950.2143359,"name":"offline","context":{"idset":"10418"}}
+{"timestamp":1713385950.2379341,"name":"offline","context":{"idset":"10419"}}
+{"timestamp":1713385950.2615259,"name":"offline","context":{"idset":"10420"}}
+{"timestamp":1713385950.2624125,"name":"offline","context":{"idset":"10421"}}
+{"timestamp":1713385950.2632909,"name":"offline","context":{"idset":"10422"}}
+{"timestamp":1713385950.2641582,"name":"offline","context":{"idset":"10423"}}
+{"timestamp":1713385950.2650471,"name":"offline","context":{"idset":"10424"}}
+{"timestamp":1713385950.2659292,"name":"offline","context":{"idset":"10425"}}
+{"timestamp":1713385950.2668059,"name":"offline","context":{"idset":"10426"}}
+{"timestamp":1713385950.2676866,"name":"offline","context":{"idset":"10427"}}
+{"timestamp":1713385950.2685664,"name":"offline","context":{"idset":"10428"}}
+{"timestamp":1713385950.2694371,"name":"offline","context":{"idset":"10429"}}
+{"timestamp":1713385950.2703092,"name":"offline","context":{"idset":"10430"}}
+{"timestamp":1713385950.2711675,"name":"offline","context":{"idset":"10431"}}
+{"timestamp":1713385950.2720366,"name":"offline","context":{"idset":"10432"}}
+{"timestamp":1713385950.2729065,"name":"offline","context":{"idset":"10433"}}
+{"timestamp":1713385950.2737744,"name":"offline","context":{"idset":"10434"}}
+{"timestamp":1713385950.2747579,"name":"offline","context":{"idset":"10435"}}
+{"timestamp":1713385950.2756376,"name":"offline","context":{"idset":"10436"}}
+{"timestamp":1713385950.2765167,"name":"offline","context":{"idset":"10437"}}
+{"timestamp":1713385950.2773838,"name":"offline","context":{"idset":"10438"}}
+{"timestamp":1713385950.2782364,"name":"offline","context":{"idset":"10439"}}
+{"timestamp":1713385950.2791002,"name":"offline","context":{"idset":"10440"}}
+{"timestamp":1713385950.2799726,"name":"offline","context":{"idset":"10441"}}
+{"timestamp":1713385950.280843,"name":"offline","context":{"idset":"10442"}}
+{"timestamp":1713385950.2817106,"name":"offline","context":{"idset":"10443"}}
+{"timestamp":1713385950.3052845,"name":"offline","context":{"idset":"10444"}}
+{"timestamp":1713385950.3288672,"name":"offline","context":{"idset":"10445"}}
+{"timestamp":1713385950.3525953,"name":"offline","context":{"idset":"10446"}}
+{"timestamp":1713385950.3762562,"name":"offline","context":{"idset":"10447"}}
+{"timestamp":1713385950.3828154,"name":"offline","context":{"idset":"10448"}}
+{"timestamp":1713385950.383677,"name":"offline","context":{"idset":"10449"}}
+{"timestamp":1713385950.3845367,"name":"offline","context":{"idset":"10450"}}
+{"timestamp":1713385950.3854024,"name":"offline","context":{"idset":"10451"}}
+{"timestamp":1713385950.3862493,"name":"offline","context":{"idset":"10452"}}
+{"timestamp":1713385950.3871262,"name":"offline","context":{"idset":"10453"}}
+{"timestamp":1713385950.3879883,"name":"offline","context":{"idset":"10454"}}
+{"timestamp":1713385950.3888521,"name":"offline","context":{"idset":"10455"}}
+{"timestamp":1713385950.3897078,"name":"offline","context":{"idset":"10456"}}
+{"timestamp":1713385950.3905838,"name":"offline","context":{"idset":"10457"}}
+{"timestamp":1713385950.3914497,"name":"offline","context":{"idset":"10458"}}
+{"timestamp":1713385950.3923068,"name":"offline","context":{"idset":"10459"}}
+{"timestamp":1713385950.3931508,"name":"offline","context":{"idset":"10460"}}
+{"timestamp":1713385950.394007,"name":"offline","context":{"idset":"10461"}}
+{"timestamp":1713385950.394861,"name":"offline","context":{"idset":"10462"}}
+{"timestamp":1713385950.3957155,"name":"offline","context":{"idset":"10463"}}
+{"timestamp":1713385950.3965697,"name":"offline","context":{"idset":"10464"}}
+{"timestamp":1713385950.3974373,"name":"offline","context":{"idset":"10465"}}
+{"timestamp":1713385950.3982885,"name":"offline","context":{"idset":"10466"}}
+{"timestamp":1713385950.3991239,"name":"offline","context":{"idset":"10467"}}
+{"timestamp":1713385950.3999751,"name":"offline","context":{"idset":"10468"}}
+{"timestamp":1713385950.4008226,"name":"offline","context":{"idset":"10469"}}
+{"timestamp":1713385950.4016731,"name":"offline","context":{"idset":"10470"}}
+{"timestamp":1713385950.4025259,"name":"offline","context":{"idset":"10471"}}
+{"timestamp":1713385950.4033921,"name":"offline","context":{"idset":"10472"}}
+{"timestamp":1713385950.4042394,"name":"offline","context":{"idset":"10473"}}
+{"timestamp":1713385950.4277797,"name":"offline","context":{"idset":"10474"}}
+{"timestamp":1713385950.4516377,"name":"offline","context":{"idset":"10475"}}
+{"timestamp":1713385950.4773803,"name":"offline","context":{"idset":"10476"}}
+{"timestamp":1713385950.5008357,"name":"offline","context":{"idset":"10477"}}
+{"timestamp":1713385950.5242889,"name":"offline","context":{"idset":"10478"}}
+{"timestamp":1713385950.547761,"name":"offline","context":{"idset":"10479"}}
+{"timestamp":1713385950.5715525,"name":"offline","context":{"idset":"10480"}}
+{"timestamp":1713385950.5950763,"name":"offline","context":{"idset":"10481"}}
+{"timestamp":1713385950.5959334,"name":"offline","context":{"idset":"10482"}}
+{"timestamp":1713385950.5967731,"name":"offline","context":{"idset":"10483"}}
+{"timestamp":1713385950.5976105,"name":"offline","context":{"idset":"10484"}}
+{"timestamp":1713385950.5984457,"name":"offline","context":{"idset":"10485"}}
+{"timestamp":1713385950.5992925,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1713385950.6001177,"name":"offline","context":{"idset":"10487"}}
+{"timestamp":1713385950.600949,"name":"offline","context":{"idset":"10488"}}
+{"timestamp":1713385950.6017845,"name":"offline","context":{"idset":"10489"}}
+{"timestamp":1713385950.6026189,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1713385950.6034548,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1713385950.6042891,"name":"offline","context":{"idset":"10492"}}
+{"timestamp":1713385950.605114,"name":"offline","context":{"idset":"10493"}}
+{"timestamp":1713385950.6059568,"name":"offline","context":{"idset":"10494"}}
+{"timestamp":1713385950.6067946,"name":"offline","context":{"idset":"10495"}}
+{"timestamp":1713385950.6076336,"name":"offline","context":{"idset":"10496"}}
+{"timestamp":1713385950.6084719,"name":"offline","context":{"idset":"10497"}}
+{"timestamp":1713385950.6093006,"name":"offline","context":{"idset":"10498"}}
+{"timestamp":1713385950.6214566,"name":"offline","context":{"idset":"10499"}}
+{"timestamp":1713385950.6448765,"name":"offline","context":{"idset":"10500"}}
+{"timestamp":1713385950.6683216,"name":"offline","context":{"idset":"10501"}}
+{"timestamp":1713385950.6918042,"name":"offline","context":{"idset":"10502"}}
+{"timestamp":1713385950.7152438,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1713385950.74143,"name":"offline","context":{"idset":"10504"}}
+{"timestamp":1713385950.7648876,"name":"offline","context":{"idset":"10505"}}
+{"timestamp":1713385950.7770586,"name":"offline","context":{"idset":"10506"}}
+{"timestamp":1713385950.7778962,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1713385950.7787263,"name":"offline","context":{"idset":"10508"}}
+{"timestamp":1713385950.7795537,"name":"offline","context":{"idset":"10509"}}
+{"timestamp":1713385950.7803857,"name":"offline","context":{"idset":"10510"}}
+{"timestamp":1713385950.7812066,"name":"offline","context":{"idset":"10511"}}
+{"timestamp":1713385950.7820385,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1713385950.7828777,"name":"offline","context":{"idset":"10513"}}
+{"timestamp":1713385950.7837033,"name":"offline","context":{"idset":"10514"}}
+{"timestamp":1713385950.784523,"name":"offline","context":{"idset":"10515"}}
+{"timestamp":1713385950.7853446,"name":"offline","context":{"idset":"10516"}}
+{"timestamp":1713385950.7861965,"name":"offline","context":{"idset":"10517"}}
+{"timestamp":1713385950.7870224,"name":"offline","context":{"idset":"10518"}}
+{"timestamp":1713385950.7878656,"name":"offline","context":{"idset":"10519"}}
+{"timestamp":1713385950.7886901,"name":"offline","context":{"idset":"10520"}}
+{"timestamp":1713385950.7895269,"name":"offline","context":{"idset":"10521"}}
+{"timestamp":1713385950.7903483,"name":"offline","context":{"idset":"10522"}}
+{"timestamp":1713385950.7911515,"name":"offline","context":{"idset":"10523"}}
+{"timestamp":1713385950.7919738,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1713385950.7927907,"name":"offline","context":{"idset":"10525"}}
+{"timestamp":1713385950.7936049,"name":"offline","context":{"idset":"10526"}}
+{"timestamp":1713385950.7944195,"name":"offline","context":{"idset":"10527"}}
+{"timestamp":1713385950.7952204,"name":"offline","context":{"idset":"10528"}}
+{"timestamp":1713385950.7960584,"name":"offline","context":{"idset":"10529"}}
+{"timestamp":1713385950.796874,"name":"offline","context":{"idset":"10530"}}
+{"timestamp":1713385950.7976854,"name":"offline","context":{"idset":"10531"}}
+{"timestamp":1713385950.7984951,"name":"offline","context":{"idset":"10532"}}
+{"timestamp":1713385950.7993071,"name":"offline","context":{"idset":"10533"}}
+{"timestamp":1713385950.8001068,"name":"offline","context":{"idset":"10534"}}
+{"timestamp":1713385950.8009567,"name":"offline","context":{"idset":"10535"}}
+{"timestamp":1713385950.8017714,"name":"offline","context":{"idset":"10536"}}
+{"timestamp":1713385950.8025854,"name":"offline","context":{"idset":"10537"}}
+{"timestamp":1713385950.8033938,"name":"offline","context":{"idset":"10538"}}
+{"timestamp":1713385950.8268166,"name":"offline","context":{"idset":"10539"}}
+{"timestamp":1713385950.8502009,"name":"offline","context":{"idset":"10540"}}
+{"timestamp":1713385950.8734634,"name":"offline","context":{"idset":"10541"}}
+{"timestamp":1713385950.8968015,"name":"offline","context":{"idset":"10542"}}
+{"timestamp":1713385950.920188,"name":"offline","context":{"idset":"10543"}}
+{"timestamp":1713385950.92665,"name":"offline","context":{"idset":"10544"}}
+{"timestamp":1713385950.9274654,"name":"offline","context":{"idset":"10545"}}
+{"timestamp":1713385950.9282606,"name":"offline","context":{"idset":"10546"}}
+{"timestamp":1713385950.9290771,"name":"offline","context":{"idset":"10547"}}
+{"timestamp":1713385950.9298859,"name":"offline","context":{"idset":"10548"}}
+{"timestamp":1713385950.9306943,"name":"offline","context":{"idset":"10549"}}
+{"timestamp":1713385950.9315145,"name":"offline","context":{"idset":"10550"}}
+{"timestamp":1713385950.9323189,"name":"offline","context":{"idset":"10551"}}
+{"timestamp":1713385950.9331117,"name":"offline","context":{"idset":"10552"}}
+{"timestamp":1713385950.9339137,"name":"offline","context":{"idset":"10553"}}
+{"timestamp":1713385950.9347179,"name":"offline","context":{"idset":"10554"}}
+{"timestamp":1713385950.9355314,"name":"offline","context":{"idset":"10555"}}
+{"timestamp":1713385950.9363377,"name":"offline","context":{"idset":"10556"}}
+{"timestamp":1713385950.9371252,"name":"offline","context":{"idset":"10557"}}
+{"timestamp":1713385950.9379234,"name":"offline","context":{"idset":"10558"}}
+{"timestamp":1713385950.9387283,"name":"offline","context":{"idset":"10559"}}
+{"timestamp":1713385950.9395342,"name":"offline","context":{"idset":"10560"}}
+{"timestamp":1713385950.9403327,"name":"offline","context":{"idset":"10561"}}
+{"timestamp":1713385950.9411175,"name":"offline","context":{"idset":"10562"}}
+{"timestamp":1713385950.9419134,"name":"offline","context":{"idset":"10563"}}
+{"timestamp":1713385950.9427135,"name":"offline","context":{"idset":"10564"}}
+{"timestamp":1713385950.9435101,"name":"offline","context":{"idset":"10565"}}
+{"timestamp":1713385950.9443054,"name":"offline","context":{"idset":"10566"}}
+{"timestamp":1713385950.9450908,"name":"offline","context":{"idset":"10567"}}
+{"timestamp":1713385950.9458876,"name":"offline","context":{"idset":"10568"}}
+{"timestamp":1713385950.9466827,"name":"offline","context":{"idset":"10569"}}
+{"timestamp":1713385950.9474781,"name":"offline","context":{"idset":"10570"}}
+{"timestamp":1713385950.9482801,"name":"offline","context":{"idset":"10571"}}
+{"timestamp":1713385950.949059,"name":"offline","context":{"idset":"10572"}}
+{"timestamp":1713385950.9498544,"name":"offline","context":{"idset":"10573"}}
+{"timestamp":1713385950.9506495,"name":"offline","context":{"idset":"10574"}}
+{"timestamp":1713385950.9514415,"name":"offline","context":{"idset":"10575"}}
+{"timestamp":1713385950.9522183,"name":"offline","context":{"idset":"10576"}}
+{"timestamp":1713385950.9530075,"name":"offline","context":{"idset":"10577"}}
+{"timestamp":1713385950.9538031,"name":"offline","context":{"idset":"10578"}}
+{"timestamp":1713385950.9546046,"name":"offline","context":{"idset":"10579"}}
+{"timestamp":1713385950.9554002,"name":"offline","context":{"idset":"10580"}}
+{"timestamp":1713385950.956176,"name":"offline","context":{"idset":"10581"}}
+{"timestamp":1713385950.9569628,"name":"offline","context":{"idset":"10582"}}
+{"timestamp":1713385950.9748147,"name":"offline","context":{"idset":"10583"}}
+{"timestamp":1713385950.9981968,"name":"offline","context":{"idset":"10584"}}
+{"timestamp":1713385951.0215266,"name":"offline","context":{"idset":"10585"}}
+{"timestamp":1713385951.0448201,"name":"offline","context":{"idset":"10586"}}
+{"timestamp":1713385951.0684571,"name":"offline","context":{"idset":"10587"}}
+{"timestamp":1713385951.092762,"name":"offline","context":{"idset":"10588"}}
+{"timestamp":1713385951.1218491,"name":"offline","context":{"idset":"10589"}}
+{"timestamp":1713385951.1475565,"name":"offline","context":{"idset":"10590"}}
+{"timestamp":1713385951.1707516,"name":"offline","context":{"idset":"10591"}}
+{"timestamp":1713385951.1939383,"name":"offline","context":{"idset":"10592"}}
+{"timestamp":1713385951.2116945,"name":"offline","context":{"idset":"10593"}}
+{"timestamp":1713385951.212481,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1713385951.2132497,"name":"offline","context":{"idset":"10595"}}
+{"timestamp":1713385951.2140317,"name":"offline","context":{"idset":"10596"}}
+{"timestamp":1713385951.2148111,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1713385951.2155964,"name":"offline","context":{"idset":"10598"}}
+{"timestamp":1713385951.2163885,"name":"offline","context":{"idset":"10599"}}
+{"timestamp":1713385951.2171559,"name":"offline","context":{"idset":"10600"}}
+{"timestamp":1713385951.217937,"name":"offline","context":{"idset":"10601"}}
+{"timestamp":1713385951.2187176,"name":"offline","context":{"idset":"10602"}}
+{"timestamp":1713385951.2195024,"name":"offline","context":{"idset":"10603"}}
+{"timestamp":1713385951.2202811,"name":"offline","context":{"idset":"10604"}}
+{"timestamp":1713385951.2210479,"name":"offline","context":{"idset":"10605"}}
+{"timestamp":1713385951.2218239,"name":"offline","context":{"idset":"10606"}}
+{"timestamp":1713385951.2225959,"name":"offline","context":{"idset":"10607"}}
+{"timestamp":1713385951.2233717,"name":"offline","context":{"idset":"10608"}}
+{"timestamp":1713385951.2241988,"name":"offline","context":{"idset":"10609"}}
+{"timestamp":1713385951.2249753,"name":"offline","context":{"idset":"10610"}}
+{"timestamp":1713385951.2257528,"name":"offline","context":{"idset":"10611"}}
+{"timestamp":1713385951.2265255,"name":"offline","context":{"idset":"10612"}}
+{"timestamp":1713385951.2273042,"name":"offline","context":{"idset":"10613"}}
+{"timestamp":1713385951.2280667,"name":"offline","context":{"idset":"10614"}}
+{"timestamp":1713385951.2288396,"name":"offline","context":{"idset":"10615"}}
+{"timestamp":1713385951.2296219,"name":"offline","context":{"idset":"10616"}}
+{"timestamp":1713385951.2303991,"name":"offline","context":{"idset":"10617"}}
+{"timestamp":1713385951.2311618,"name":"offline","context":{"idset":"10618"}}
+{"timestamp":1713385951.2319317,"name":"offline","context":{"idset":"10619"}}
+{"timestamp":1713385951.232702,"name":"offline","context":{"idset":"10620"}}
+{"timestamp":1713385951.2334688,"name":"offline","context":{"idset":"10621"}}
+{"timestamp":1713385951.2511106,"name":"offline","context":{"idset":"10622"}}
+{"timestamp":1713385951.2743998,"name":"offline","context":{"idset":"10623"}}
+{"timestamp":1713385951.2979791,"name":"offline","context":{"idset":"10624"}}
+{"timestamp":1713385951.3213363,"name":"offline","context":{"idset":"10625"}}
+{"timestamp":1713385951.3446152,"name":"offline","context":{"idset":"10626"}}
+{"timestamp":1713385951.3679924,"name":"offline","context":{"idset":"10627"}}
+{"timestamp":1713385951.3912902,"name":"offline","context":{"idset":"10628"}}
+{"timestamp":1713385951.4144537,"name":"offline","context":{"idset":"10629"}}
+{"timestamp":1713385951.4376323,"name":"offline","context":{"idset":"10630"}}
+{"timestamp":1713385951.4783115,"name":"offline","context":{"idset":"10631"}}
+{"timestamp":1713385951.4878054,"name":"offline","context":{"idset":"10632"}}
+{"timestamp":1713385951.4890594,"name":"offline","context":{"idset":"10633"}}
+{"timestamp":1713385951.4903808,"name":"offline","context":{"idset":"10634"}}
+{"timestamp":1713385951.4919662,"name":"offline","context":{"idset":"10635"}}
+{"timestamp":1713385951.4933765,"name":"offline","context":{"idset":"10636"}}
+{"timestamp":1713385951.4943721,"name":"offline","context":{"idset":"10637"}}
+{"timestamp":1713385951.495754,"name":"offline","context":{"idset":"10638"}}
+{"timestamp":1713385951.4972379,"name":"offline","context":{"idset":"10639"}}
+{"timestamp":1713385951.4987695,"name":"offline","context":{"idset":"10640"}}
+{"timestamp":1713385951.5002303,"name":"offline","context":{"idset":"10641"}}
+{"timestamp":1713385951.5012419,"name":"offline","context":{"idset":"10642"}}
+{"timestamp":1713385951.5022154,"name":"offline","context":{"idset":"10643"}}
+{"timestamp":1713385951.5032039,"name":"offline","context":{"idset":"10644"}}
+{"timestamp":1713385951.5042071,"name":"offline","context":{"idset":"10645"}}
+{"timestamp":1713385951.5051911,"name":"offline","context":{"idset":"10646"}}
+{"timestamp":1713385951.5064745,"name":"offline","context":{"idset":"10647"}}
+{"timestamp":1713385951.5077446,"name":"offline","context":{"idset":"10648"}}
+{"timestamp":1713385951.5090883,"name":"offline","context":{"idset":"10649"}}
+{"timestamp":1713385951.5103393,"name":"offline","context":{"idset":"10650"}}
+{"timestamp":1713385951.5113349,"name":"offline","context":{"idset":"10651"}}
+{"timestamp":1713385951.5125096,"name":"offline","context":{"idset":"10652"}}
+{"timestamp":1713385951.5135353,"name":"offline","context":{"idset":"10653"}}
+{"timestamp":1713385951.5144954,"name":"offline","context":{"idset":"10654"}}
+{"timestamp":1713385951.5155485,"name":"offline","context":{"idset":"10655"}}
+{"timestamp":1713385951.5167992,"name":"offline","context":{"idset":"10656"}}
+{"timestamp":1713385951.5180457,"name":"offline","context":{"idset":"10657"}}
+{"timestamp":1713385951.5192895,"name":"offline","context":{"idset":"10658"}}
+{"timestamp":1713385951.5202174,"name":"offline","context":{"idset":"10659"}}
+{"timestamp":1713385951.5211737,"name":"offline","context":{"idset":"10660"}}
+{"timestamp":1713385951.5221314,"name":"offline","context":{"idset":"10661"}}
+{"timestamp":1713385951.5230801,"name":"offline","context":{"idset":"10662"}}
+{"timestamp":1713385951.5240366,"name":"offline","context":{"idset":"10663"}}
+{"timestamp":1713385951.5249748,"name":"offline","context":{"idset":"10664"}}
+{"timestamp":1713385951.5259149,"name":"offline","context":{"idset":"10665"}}
+{"timestamp":1713385951.5268986,"name":"offline","context":{"idset":"10666"}}
+{"timestamp":1713385951.5278611,"name":"offline","context":{"idset":"10667"}}
+{"timestamp":1713385951.5288751,"name":"offline","context":{"idset":"10668"}}
+{"timestamp":1713385951.5298367,"name":"offline","context":{"idset":"10669"}}
+{"timestamp":1713385951.5307605,"name":"offline","context":{"idset":"10670"}}
+{"timestamp":1713385951.5541978,"name":"offline","context":{"idset":"10671"}}
+{"timestamp":1713385951.5879381,"name":"offline","context":{"idset":"10672"}}
+{"timestamp":1713385951.6191006,"name":"offline","context":{"idset":"10673"}}
+{"timestamp":1713385951.6603098,"name":"offline","context":{"idset":"10674"}}
+{"timestamp":1713385951.6838055,"name":"offline","context":{"idset":"10675"}}
+{"timestamp":1713385951.7073402,"name":"offline","context":{"idset":"10676"}}
+{"timestamp":1713385951.7309172,"name":"offline","context":{"idset":"10677"}}
+{"timestamp":1713385951.7552872,"name":"offline","context":{"idset":"10678"}}
+{"timestamp":1713385951.7786412,"name":"offline","context":{"idset":"10679"}}
+{"timestamp":1713385951.8030546,"name":"offline","context":{"idset":"10680"}}
+{"timestamp":1713385951.8207259,"name":"offline","context":{"idset":"10681"}}
+{"timestamp":1713385951.8214815,"name":"offline","context":{"idset":"10682"}}
+{"timestamp":1713385951.8222094,"name":"offline","context":{"idset":"10683"}}
+{"timestamp":1713385951.8229623,"name":"offline","context":{"idset":"10684"}}
+{"timestamp":1713385951.82371,"name":"offline","context":{"idset":"10685"}}
+{"timestamp":1713385951.8244443,"name":"offline","context":{"idset":"10686"}}
+{"timestamp":1713385951.825175,"name":"offline","context":{"idset":"10687"}}
+{"timestamp":1713385951.825912,"name":"offline","context":{"idset":"10688"}}
+{"timestamp":1713385951.8266459,"name":"offline","context":{"idset":"10689"}}
+{"timestamp":1713385951.8273828,"name":"offline","context":{"idset":"10690"}}
+{"timestamp":1713385951.8281062,"name":"offline","context":{"idset":"10691"}}
+{"timestamp":1713385951.8288448,"name":"offline","context":{"idset":"10692"}}
+{"timestamp":1713385951.829582,"name":"offline","context":{"idset":"10693"}}
+{"timestamp":1713385951.8303213,"name":"offline","context":{"idset":"10694"}}
+{"timestamp":1713385951.831044,"name":"offline","context":{"idset":"10695"}}
+{"timestamp":1713385951.8317771,"name":"offline","context":{"idset":"10696"}}
+{"timestamp":1713385951.832505,"name":"offline","context":{"idset":"10697"}}
+{"timestamp":1713385951.8332245,"name":"offline","context":{"idset":"10698"}}
+{"timestamp":1713385951.8339677,"name":"offline","context":{"idset":"10699"}}
+{"timestamp":1713385951.8346972,"name":"offline","context":{"idset":"10700"}}
+{"timestamp":1713385951.8354228,"name":"offline","context":{"idset":"10701"}}
+{"timestamp":1713385951.8361409,"name":"offline","context":{"idset":"10702"}}
+{"timestamp":1713385951.8368688,"name":"offline","context":{"idset":"10703"}}
+{"timestamp":1713385951.8375957,"name":"offline","context":{"idset":"10704"}}
+{"timestamp":1713385951.8383224,"name":"offline","context":{"idset":"10705"}}
+{"timestamp":1713385951.8390393,"name":"offline","context":{"idset":"10706"}}
+{"timestamp":1713385951.8397696,"name":"offline","context":{"idset":"10707"}}
+{"timestamp":1713385951.8404944,"name":"offline","context":{"idset":"10708"}}
+{"timestamp":1713385951.8412089,"name":"offline","context":{"idset":"10709"}}
+{"timestamp":1713385951.8419654,"name":"offline","context":{"idset":"10710"}}
+{"timestamp":1713385951.8426921,"name":"offline","context":{"idset":"10711"}}
+{"timestamp":1713385951.8434129,"name":"offline","context":{"idset":"10712"}}
+{"timestamp":1713385951.8441219,"name":"offline","context":{"idset":"10713"}}
+{"timestamp":1713385951.8448565,"name":"offline","context":{"idset":"10714"}}
+{"timestamp":1713385951.8455939,"name":"offline","context":{"idset":"10715"}}
+{"timestamp":1713385951.846314,"name":"offline","context":{"idset":"10716"}}
+{"timestamp":1713385951.8470223,"name":"offline","context":{"idset":"10717"}}
+{"timestamp":1713385951.8589914,"name":"offline","context":{"idset":"10718"}}
+{"timestamp":1713385951.8820233,"name":"offline","context":{"idset":"10719"}}
+{"timestamp":1713385951.9051361,"name":"offline","context":{"idset":"10720"}}
+{"timestamp":1713385951.9285748,"name":"offline","context":{"idset":"10721"}}
+{"timestamp":1713385951.9519255,"name":"offline","context":{"idset":"10722"}}
+{"timestamp":1713385951.9752023,"name":"offline","context":{"idset":"10723"}}
+{"timestamp":1713385951.9985089,"name":"offline","context":{"idset":"10724"}}
+{"timestamp":1713385952.0218272,"name":"offline","context":{"idset":"10725"}}
+{"timestamp":1713385952.0468528,"name":"offline","context":{"idset":"10726"}}
+{"timestamp":1713385952.0700338,"name":"offline","context":{"idset":"10727"}}
+{"timestamp":1713385952.0932393,"name":"offline","context":{"idset":"10728"}}
+{"timestamp":1713385952.1166253,"name":"offline","context":{"idset":"10729"}}
+{"timestamp":1713385952.1229982,"name":"offline","context":{"idset":"10730"}}
+{"timestamp":1713385952.1237156,"name":"offline","context":{"idset":"10731"}}
+{"timestamp":1713385952.1244309,"name":"offline","context":{"idset":"10732"}}
+{"timestamp":1713385952.1251314,"name":"offline","context":{"idset":"10733"}}
+{"timestamp":1713385952.1258433,"name":"offline","context":{"idset":"10734"}}
+{"timestamp":1713385952.1265583,"name":"offline","context":{"idset":"10735"}}
+{"timestamp":1713385952.1272576,"name":"offline","context":{"idset":"10736"}}
+{"timestamp":1713385952.12797,"name":"offline","context":{"idset":"10737"}}
+{"timestamp":1713385952.1286857,"name":"offline","context":{"idset":"10738"}}
+{"timestamp":1713385952.1294,"name":"offline","context":{"idset":"10739"}}
+{"timestamp":1713385952.1300981,"name":"offline","context":{"idset":"10740"}}
+{"timestamp":1713385952.1308081,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1713385952.1315219,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1713385952.1322181,"name":"offline","context":{"idset":"10743"}}
+{"timestamp":1713385952.1329286,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1713385952.1337056,"name":"offline","context":{"idset":"10745"}}
+{"timestamp":1713385952.1344318,"name":"offline","context":{"idset":"10746"}}
+{"timestamp":1713385952.1351287,"name":"offline","context":{"idset":"10747"}}
+{"timestamp":1713385952.1358387,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1713385952.1365499,"name":"offline","context":{"idset":"10749"}}
+{"timestamp":1713385952.137243,"name":"offline","context":{"idset":"10750"}}
+{"timestamp":1713385952.1379528,"name":"offline","context":{"idset":"10751"}}
+{"timestamp":1713385952.1386619,"name":"offline","context":{"idset":"10752"}}
+{"timestamp":1713385952.1393666,"name":"offline","context":{"idset":"10753"}}
+{"timestamp":1713385952.1400585,"name":"offline","context":{"idset":"10754"}}
+{"timestamp":1713385952.1407599,"name":"offline","context":{"idset":"10755"}}
+{"timestamp":1713385952.1414955,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1713385952.1421938,"name":"offline","context":{"idset":"10757"}}
+{"timestamp":1713385952.1428995,"name":"offline","context":{"idset":"10758"}}
+{"timestamp":1713385952.1436114,"name":"offline","context":{"idset":"10759"}}
+{"timestamp":1713385952.1443138,"name":"offline","context":{"idset":"10760"}}
+{"timestamp":1713385952.1450191,"name":"offline","context":{"idset":"10761"}}
+{"timestamp":1713385952.1457891,"name":"offline","context":{"idset":"10762"}}
+{"timestamp":1713385952.1537814,"name":"offline","context":{"idset":"10763"}}
+{"timestamp":1713385952.1770785,"name":"offline","context":{"idset":"10764"}}
+{"timestamp":1713385952.2003539,"name":"offline","context":{"idset":"10765"}}
+{"timestamp":1713385952.2237785,"name":"offline","context":{"idset":"10766"}}
+{"timestamp":1713385952.2470572,"name":"offline","context":{"idset":"10767"}}
+{"timestamp":1713385952.2703638,"name":"offline","context":{"idset":"10768"}}
+{"timestamp":1713385952.293771,"name":"offline","context":{"idset":"10769"}}
+{"timestamp":1713385952.3181891,"name":"offline","context":{"idset":"10770"}}
+{"timestamp":1713385952.342869,"name":"offline","context":{"idset":"10771"}}
+{"timestamp":1713385952.3663962,"name":"offline","context":{"idset":"10772"}}
+{"timestamp":1713385952.3784995,"name":"offline","context":{"idset":"10773"}}
+{"timestamp":1713385952.3791866,"name":"offline","context":{"idset":"10774"}}
+{"timestamp":1713385952.3798811,"name":"offline","context":{"idset":"10775"}}
+{"timestamp":1713385952.3805749,"name":"offline","context":{"idset":"10776"}}
+{"timestamp":1713385952.3812563,"name":"offline","context":{"idset":"10777"}}
+{"timestamp":1713385952.3819458,"name":"offline","context":{"idset":"10778"}}
+{"timestamp":1713385952.3826444,"name":"offline","context":{"idset":"10779"}}
+{"timestamp":1713385952.3833349,"name":"offline","context":{"idset":"10780"}}
+{"timestamp":1713385952.384011,"name":"offline","context":{"idset":"10781"}}
+{"timestamp":1713385952.3847094,"name":"offline","context":{"idset":"10782"}}
+{"timestamp":1713385952.3854036,"name":"offline","context":{"idset":"10783"}}
+{"timestamp":1713385952.3860805,"name":"offline","context":{"idset":"10784"}}
+{"timestamp":1713385952.38677,"name":"offline","context":{"idset":"10785"}}
+{"timestamp":1713385952.3874643,"name":"offline","context":{"idset":"10786"}}
+{"timestamp":1713385952.3881683,"name":"offline","context":{"idset":"10787"}}
+{"timestamp":1713385952.3888659,"name":"offline","context":{"idset":"10788"}}
+{"timestamp":1713385952.3895514,"name":"offline","context":{"idset":"10789"}}
+{"timestamp":1713385952.3902247,"name":"offline","context":{"idset":"10790"}}
+{"timestamp":1713385952.390907,"name":"offline","context":{"idset":"10791"}}
+{"timestamp":1713385952.3915901,"name":"offline","context":{"idset":"10792"}}
+{"timestamp":1713385952.3922603,"name":"offline","context":{"idset":"10793"}}
+{"timestamp":1713385952.3929439,"name":"offline","context":{"idset":"10794"}}
+{"timestamp":1713385952.3936265,"name":"offline","context":{"idset":"10795"}}
+{"timestamp":1713385952.3943489,"name":"offline","context":{"idset":"10796"}}
+{"timestamp":1713385952.3950462,"name":"offline","context":{"idset":"10797"}}
+{"timestamp":1713385952.3957283,"name":"offline","context":{"idset":"10798"}}
+{"timestamp":1713385952.3964169,"name":"offline","context":{"idset":"10799"}}
+{"timestamp":1713385952.3970897,"name":"offline","context":{"idset":"10800"}}
+{"timestamp":1713385952.3977754,"name":"offline","context":{"idset":"10801"}}
+{"timestamp":1713385952.3984599,"name":"offline","context":{"idset":"10802"}}
+{"timestamp":1713385952.399158,"name":"offline","context":{"idset":"10803"}}
+{"timestamp":1713385952.3998477,"name":"offline","context":{"idset":"10804"}}
+{"timestamp":1713385952.4005282,"name":"offline","context":{"idset":"10805"}}
+{"timestamp":1713385952.4011934,"name":"offline","context":{"idset":"10806"}}
+{"timestamp":1713385952.4018717,"name":"offline","context":{"idset":"10807"}}
+{"timestamp":1713385952.4025705,"name":"offline","context":{"idset":"10808"}}
+{"timestamp":1713385952.4032423,"name":"offline","context":{"idset":"10809"}}
+{"timestamp":1713385952.4096782,"name":"offline","context":{"idset":"10810"}}
+{"timestamp":1713385952.4352801,"name":"offline","context":{"idset":"10811"}}
+{"timestamp":1713385952.4606097,"name":"offline","context":{"idset":"10812"}}
+{"timestamp":1713385952.4860549,"name":"offline","context":{"idset":"10813"}}
+{"timestamp":1713385952.5095856,"name":"offline","context":{"idset":"10814"}}
+{"timestamp":1713385952.5366781,"name":"offline","context":{"idset":"10815"}}
+{"timestamp":1713385952.5612173,"name":"offline","context":{"idset":"10816"}}
+{"timestamp":1713385952.5855372,"name":"offline","context":{"idset":"10817"}}
+{"timestamp":1713385952.6105199,"name":"offline","context":{"idset":"10818"}}
+{"timestamp":1713385952.6339042,"name":"offline","context":{"idset":"10819"}}
+{"timestamp":1713385952.6571119,"name":"offline","context":{"idset":"10820"}}
+{"timestamp":1713385952.6803424,"name":"offline","context":{"idset":"10821"}}
+{"timestamp":1713385952.681004,"name":"offline","context":{"idset":"10822"}}
+{"timestamp":1713385952.6816759,"name":"offline","context":{"idset":"10823"}}
+{"timestamp":1713385952.6823499,"name":"offline","context":{"idset":"10824"}}
+{"timestamp":1713385952.6830049,"name":"offline","context":{"idset":"10825"}}
+{"timestamp":1713385952.683671,"name":"offline","context":{"idset":"10826"}}
+{"timestamp":1713385952.6843398,"name":"offline","context":{"idset":"10827"}}
+{"timestamp":1713385952.6849928,"name":"offline","context":{"idset":"10828"}}
+{"timestamp":1713385952.6856589,"name":"offline","context":{"idset":"10829"}}
+{"timestamp":1713385952.6863239,"name":"offline","context":{"idset":"10830"}}
+{"timestamp":1713385952.6869745,"name":"offline","context":{"idset":"10831"}}
+{"timestamp":1713385952.6876392,"name":"offline","context":{"idset":"10832"}}
+{"timestamp":1713385952.6883039,"name":"offline","context":{"idset":"10833"}}
+{"timestamp":1713385952.6889558,"name":"offline","context":{"idset":"10834"}}
+{"timestamp":1713385952.6896193,"name":"offline","context":{"idset":"10835"}}
+{"timestamp":1713385952.6902812,"name":"offline","context":{"idset":"10836"}}
+{"timestamp":1713385952.6909306,"name":"offline","context":{"idset":"10837"}}
+{"timestamp":1713385952.6915896,"name":"offline","context":{"idset":"10838"}}
+{"timestamp":1713385952.6922355,"name":"offline","context":{"idset":"10839"}}
+{"timestamp":1713385952.6928971,"name":"offline","context":{"idset":"10840"}}
+{"timestamp":1713385952.6935608,"name":"offline","context":{"idset":"10841"}}
+{"timestamp":1713385952.6942079,"name":"offline","context":{"idset":"10842"}}
+{"timestamp":1713385952.6948671,"name":"offline","context":{"idset":"10843"}}
+{"timestamp":1713385952.6955206,"name":"offline","context":{"idset":"10844"}}
+{"timestamp":1713385952.6961637,"name":"offline","context":{"idset":"10845"}}
+{"timestamp":1713385952.6968174,"name":"offline","context":{"idset":"10846"}}
+{"timestamp":1713385952.697473,"name":"offline","context":{"idset":"10847"}}
+{"timestamp":1713385952.6981137,"name":"offline","context":{"idset":"10848"}}
+{"timestamp":1713385952.6987679,"name":"offline","context":{"idset":"10849"}}
+{"timestamp":1713385952.6994216,"name":"offline","context":{"idset":"10850"}}
+{"timestamp":1713385952.7000599,"name":"offline","context":{"idset":"10851"}}
+{"timestamp":1713385952.7007239,"name":"offline","context":{"idset":"10852"}}
+{"timestamp":1713385952.7013898,"name":"offline","context":{"idset":"10853"}}
+{"timestamp":1713385952.7190502,"name":"offline","context":{"idset":"10854"}}
+{"timestamp":1713385952.7421942,"name":"offline","context":{"idset":"10855"}}
+{"timestamp":1713385952.7655029,"name":"offline","context":{"idset":"10856"}}
+{"timestamp":1713385952.7889969,"name":"offline","context":{"idset":"10857"}}
+{"timestamp":1713385952.8122883,"name":"offline","context":{"idset":"10858"}}
+{"timestamp":1713385952.8354959,"name":"offline","context":{"idset":"10859"}}
+{"timestamp":1713385952.8586597,"name":"offline","context":{"idset":"10860"}}
+{"timestamp":1713385952.8837976,"name":"offline","context":{"idset":"10861"}}
+{"timestamp":1713385952.9071248,"name":"offline","context":{"idset":"10862"}}
+{"timestamp":1713385952.9303281,"name":"offline","context":{"idset":"10863"}}
+{"timestamp":1713385952.9542489,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1713385952.97838,"name":"offline","context":{"idset":"10865"}}
+{"timestamp":1713385952.9791815,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1713385952.9800103,"name":"offline","context":{"idset":"10867"}}
+{"timestamp":1713385952.9808459,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1713385952.9816451,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1713385952.9824512,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1713385952.9833024,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1713385952.9840424,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713385952.9849675,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1713385952.986059,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1713385952.9871457,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1713385952.9881699,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1713385952.9889877,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1713385952.9897771,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1713385952.9905491,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1713385952.9913487,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1713385952.9999559,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1713385953.0244138,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1713385953.0493772,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1713385953.0726988,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1713385953.0962863,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1713385953.1243107,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1713385953.1477301,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1713385953.1733356,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1713385953.2006288,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1713385953.2185354,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1713385953.2191668,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1713385953.2198002,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1713385953.2204471,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1713385953.2210751,"name":"offline","context":{"idset":"10894"}}
+{"timestamp":1713385953.2217069,"name":"offline","context":{"idset":"10895"}}
+{"timestamp":1713385953.2223365,"name":"offline","context":{"idset":"10896"}}
+{"timestamp":1713385953.2229543,"name":"offline","context":{"idset":"10897"}}
+{"timestamp":1713385953.2235897,"name":"offline","context":{"idset":"10898"}}
+{"timestamp":1713385953.2243285,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1713385953.2249568,"name":"offline","context":{"idset":"10900"}}
+{"timestamp":1713385953.2255859,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1713385953.2261984,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1713385953.2268264,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1713385953.2274597,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1713385953.2280698,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1713385953.2286973,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1713385953.2293227,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1713385953.2299323,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1713385953.2305586,"name":"offline","context":{"idset":"10909"}}
+{"timestamp":1713385953.2311747,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1713385953.2318017,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1713385953.2324266,"name":"offline","context":{"idset":"10912"}}
+{"timestamp":1713385953.2330334,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1713385953.2336576,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1713385953.2342834,"name":"offline","context":{"idset":"10915"}}
+{"timestamp":1713385953.2348921,"name":"offline","context":{"idset":"10916"}}
+{"timestamp":1713385953.2355149,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1713385953.236119,"name":"offline","context":{"idset":"10918"}}
+{"timestamp":1713385953.2367368,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1713385953.237354,"name":"offline","context":{"idset":"10920"}}
+{"timestamp":1713385953.2379584,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1713385953.2442157,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1713385953.2674205,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1713385953.2905519,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1713385953.3137293,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1713385953.3312106,"name":"offline","context":{"idset":"10926"}}
+{"timestamp":1713385953.3318276,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1713385953.3324392,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1713385953.3330383,"name":"offline","context":{"idset":"10929"}}
+{"timestamp":1713385953.3336461,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1713385953.3342469,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1713385953.3348646,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1713385953.3354776,"name":"offline","context":{"idset":"10933"}}
+{"timestamp":1713385953.3360929,"name":"offline","context":{"idset":"10934"}}
+{"timestamp":1713385953.3367095,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1713385953.3373253,"name":"offline","context":{"idset":"10936"}}
+{"timestamp":1713385953.3379228,"name":"offline","context":{"idset":"10937"}}
+{"timestamp":1713385953.3385284,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1713385953.3391242,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1713385953.3397272,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1713385953.3403308,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1713385953.3409328,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1713385953.3415389,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1713385953.3421319,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1713385953.3427339,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1713385953.3433337,"name":"offline","context":{"idset":"10946"}}
+{"timestamp":1713385953.3439238,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1713385953.3445601,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1713385953.345165,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1713385953.3458188,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1713385953.3464456,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1713385953.3470595,"name":"offline","context":{"idset":"10952"}}
+{"timestamp":1713385953.3476655,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1713385953.3482516,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1713385953.3488522,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1713385953.3494465,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1713385953.3500321,"name":"offline","context":{"idset":"10957"}}
+{"timestamp":1713385953.3506255,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1713385953.3512096,"name":"offline","context":{"idset":"10959"}}
+{"timestamp":1713385953.3518052,"name":"offline","context":{"idset":"10960"}}
+{"timestamp":1713385953.3523993,"name":"offline","context":{"idset":"10961"}}
+{"timestamp":1713385953.3529823,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1713385953.3535738,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1713385953.3767323,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1713385953.3998098,"name":"offline","context":{"idset":"10965"}}
+{"timestamp":1713385953.423008,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1713385953.4461412,"name":"offline","context":{"idset":"10967"}}
+{"timestamp":1713385953.4709654,"name":"offline","context":{"idset":"10968"}}
+{"timestamp":1713385953.4940722,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1713385953.5172837,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1713385953.540463,"name":"offline","context":{"idset":"10971"}}
+{"timestamp":1713385953.5638647,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1713385953.5870271,"name":"offline","context":{"idset":"10973"}}
+{"timestamp":1713385953.5954711,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1713385953.5960751,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1713385953.5966694,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1713385953.5972457,"name":"offline","context":{"idset":"10977"}}
+{"timestamp":1713385953.5978441,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1713385953.5984385,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1713385953.5990126,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1713385953.5996015,"name":"offline","context":{"idset":"10981"}}
+{"timestamp":1713385953.6001766,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1713385953.6007659,"name":"offline","context":{"idset":"10983"}}
+{"timestamp":1713385953.6013546,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1713385953.6019232,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1713385953.6025243,"name":"offline","context":{"idset":"10986"}}
+{"timestamp":1713385953.6030943,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1713385953.6036806,"name":"offline","context":{"idset":"10988"}}
+{"timestamp":1713385953.604249,"name":"offline","context":{"idset":"10989"}}
+{"timestamp":1713385953.6218112,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1713385953.6448851,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1713385953.6700161,"name":"offline","context":{"idset":"10992"}}
+{"timestamp":1713385953.697376,"name":"offline","context":{"idset":"10993"}}
+{"timestamp":1713385953.7263539,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1713385953.7510297,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1713385953.7746022,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1713385953.7979858,"name":"offline","context":{"idset":"10997"}}
+{"timestamp":1713385953.8155861,"name":"offline","context":{"idset":"10998"}}
+{"timestamp":1713385953.8161635,"name":"offline","context":{"idset":"10999"}}
+{"timestamp":1713385953.8167486,"name":"offline","context":{"idset":"11000"}}
+{"timestamp":1713385953.8173354,"name":"offline","context":{"idset":"11001"}}
+{"timestamp":1713385953.8179035,"name":"offline","context":{"idset":"11002"}}
+{"timestamp":1713385953.8184865,"name":"offline","context":{"idset":"11003"}}
+{"timestamp":1713385953.819051,"name":"offline","context":{"idset":"11004"}}
+{"timestamp":1713385953.8196292,"name":"offline","context":{"idset":"11005"}}
+{"timestamp":1713385953.8201938,"name":"offline","context":{"idset":"11006"}}
+{"timestamp":1713385953.8207757,"name":"offline","context":{"idset":"11007"}}
+{"timestamp":1713385953.8213625,"name":"offline","context":{"idset":"11008"}}
+{"timestamp":1713385953.821929,"name":"offline","context":{"idset":"11009"}}
+{"timestamp":1713385953.822516,"name":"offline","context":{"idset":"11010"}}
+{"timestamp":1713385953.8230772,"name":"offline","context":{"idset":"11011"}}
+{"timestamp":1713385953.823694,"name":"offline","context":{"idset":"11012"}}
+{"timestamp":1713385953.8242848,"name":"offline","context":{"idset":"11013"}}
+{"timestamp":1713385953.8248491,"name":"offline","context":{"idset":"11014"}}
+{"timestamp":1713385953.836792,"name":"offline","context":{"idset":"11015"}}
+{"timestamp":1713385953.8600221,"name":"offline","context":{"idset":"11016"}}
+{"timestamp":1713385953.8852022,"name":"offline","context":{"idset":"11017"}}
+{"timestamp":1713385953.9088194,"name":"offline","context":{"idset":"11018"}}
+{"timestamp":1713385953.9093988,"name":"offline","context":{"idset":"11019"}}
+{"timestamp":1713385953.9099607,"name":"offline","context":{"idset":"11020"}}
+{"timestamp":1713385953.9105363,"name":"offline","context":{"idset":"11021"}}
+{"timestamp":1713385953.9110925,"name":"offline","context":{"idset":"11022"}}
+{"timestamp":1713385953.9116676,"name":"offline","context":{"idset":"11023"}}
+{"timestamp":1713385953.9122221,"name":"offline","context":{"idset":"11024"}}
+{"timestamp":1713385953.9127934,"name":"offline","context":{"idset":"11025"}}
+{"timestamp":1713385953.9133654,"name":"offline","context":{"idset":"11026"}}
+{"timestamp":1713385953.9139199,"name":"offline","context":{"idset":"11027"}}
+{"timestamp":1713385953.9144871,"name":"offline","context":{"idset":"11028"}}
+{"timestamp":1713385953.9150395,"name":"offline","context":{"idset":"11029"}}
+{"timestamp":1713385953.9156218,"name":"offline","context":{"idset":"11030"}}
+{"timestamp":1713385953.9162009,"name":"offline","context":{"idset":"11031"}}
+{"timestamp":1713385953.9167645,"name":"offline","context":{"idset":"11032"}}
+{"timestamp":1713385953.9173274,"name":"offline","context":{"idset":"11033"}}
+{"timestamp":1713385953.9178758,"name":"offline","context":{"idset":"11034"}}
+{"timestamp":1713385953.9184334,"name":"offline","context":{"idset":"11035"}}
+{"timestamp":1713385953.9189794,"name":"offline","context":{"idset":"11036"}}
+{"timestamp":1713385953.9422047,"name":"offline","context":{"idset":"11037"}}
+{"timestamp":1713385953.9655645,"name":"offline","context":{"idset":"11038"}}
+{"timestamp":1713385953.9887776,"name":"offline","context":{"idset":"11039"}}
+{"timestamp":1713385953.9893413,"name":"offline","context":{"idset":"11040"}}
+{"timestamp":1713385953.9898851,"name":"offline","context":{"idset":"11041"}}
+{"timestamp":1713385953.9904523,"name":"offline","context":{"idset":"11042"}}
+{"timestamp":1713385953.9910033,"name":"offline","context":{"idset":"11043"}}
+{"timestamp":1713385953.9915593,"name":"offline","context":{"idset":"11044"}}
+{"timestamp":1713385953.992101,"name":"offline","context":{"idset":"11045"}}
+{"timestamp":1713385953.9926529,"name":"offline","context":{"idset":"11046"}}
+{"timestamp":1713385953.9931922,"name":"offline","context":{"idset":"11047"}}
+{"timestamp":1713385953.9937446,"name":"offline","context":{"idset":"11048"}}
+{"timestamp":1713385953.9943011,"name":"offline","context":{"idset":"11049"}}
+{"timestamp":1713385953.9948397,"name":"offline","context":{"idset":"11050"}}
+{"timestamp":1713385954.0010784,"name":"offline","context":{"idset":"11051"}}
+{"timestamp":1713385954.0243316,"name":"offline","context":{"idset":"11052"}}
+{"timestamp":1713385954.0474858,"name":"offline","context":{"idset":"11053"}}
+{"timestamp":1713385954.0607989,"name":"offline","context":{"idset":"11054"}}
+{"timestamp":1713385954.0613778,"name":"offline","context":{"idset":"11055"}}
+{"timestamp":1713385954.0619142,"name":"offline","context":{"idset":"11056"}}
+{"timestamp":1713385954.0624766,"name":"offline","context":{"idset":"11057"}}
+{"timestamp":1713385954.0630224,"name":"offline","context":{"idset":"11058"}}
+{"timestamp":1713385954.0635786,"name":"offline","context":{"idset":"11059"}}
+{"timestamp":1713385954.0641162,"name":"offline","context":{"idset":"11060"}}
+{"timestamp":1713385954.0646656,"name":"offline","context":{"idset":"11061"}}
+{"timestamp":1713385954.0651996,"name":"offline","context":{"idset":"11062"}}
+{"timestamp":1713385954.0657539,"name":"offline","context":{"idset":"11063"}}
+{"timestamp":1713385954.0663011,"name":"offline","context":{"idset":"11064"}}
+{"timestamp":1713385954.0668361,"name":"offline","context":{"idset":"11065"}}
+{"timestamp":1713385954.0673842,"name":"offline","context":{"idset":"11066"}}
+{"timestamp":1713385954.0679255,"name":"offline","context":{"idset":"11067"}}
+{"timestamp":1713385954.0684724,"name":"offline","context":{"idset":"11068"}}
+{"timestamp":1713385954.0690026,"name":"offline","context":{"idset":"11069"}}
+{"timestamp":1713385954.0695481,"name":"offline","context":{"idset":"11070"}}
+{"timestamp":1713385954.0700729,"name":"offline","context":{"idset":"11071"}}
+{"timestamp":1713385954.0706131,"name":"offline","context":{"idset":"11072"}}
+{"timestamp":1713385954.0711379,"name":"offline","context":{"idset":"11073"}}
+{"timestamp":1713385954.0716765,"name":"offline","context":{"idset":"11074"}}
+{"timestamp":1713385954.0722008,"name":"offline","context":{"idset":"11075"}}
+{"timestamp":1713385954.0727377,"name":"offline","context":{"idset":"11076"}}
+{"timestamp":1713385954.0732594,"name":"offline","context":{"idset":"11077"}}
+{"timestamp":1713385954.0737965,"name":"offline","context":{"idset":"11078"}}
+{"timestamp":1713385954.0743339,"name":"offline","context":{"idset":"11079"}}
+{"timestamp":1713385954.0748558,"name":"offline","context":{"idset":"11080"}}
+{"timestamp":1713385954.0979574,"name":"offline","context":{"idset":"11081"}}
+{"timestamp":1713385954.0985029,"name":"offline","context":{"idset":"11082"}}
+{"timestamp":1713385954.0990434,"name":"offline","context":{"idset":"11083"}}
+{"timestamp":1713385954.0995803,"name":"offline","context":{"idset":"11084"}}
+{"timestamp":1713385954.1001325,"name":"offline","context":{"idset":"11085"}}
+{"timestamp":1713385954.1006739,"name":"offline","context":{"idset":"11086"}}
+{"timestamp":1713385954.1011913,"name":"offline","context":{"idset":"11087"}}
+{"timestamp":1713385954.1017208,"name":"offline","context":{"idset":"11088"}}
+{"timestamp":1713385954.102237,"name":"offline","context":{"idset":"11089"}}
+{"timestamp":1713385954.1027811,"name":"offline","context":{"idset":"11090"}}
+{"timestamp":1713385954.1033168,"name":"offline","context":{"idset":"11091"}}
+{"timestamp":1713385954.1038394,"name":"offline","context":{"idset":"11092"}}
+{"timestamp":1713385954.1043696,"name":"offline","context":{"idset":"11093"}}
+{"timestamp":1713385954.1048899,"name":"offline","context":{"idset":"11094"}}
+{"timestamp":1713385954.1054168,"name":"offline","context":{"idset":"11095"}}
+{"timestamp":1713385954.105937,"name":"offline","context":{"idset":"11096"}}
+{"timestamp":1713385954.1064632,"name":"offline","context":{"idset":"11097"}}
+{"timestamp":1713385954.1069763,"name":"offline","context":{"idset":"11098"}}
+{"timestamp":1713385954.1074982,"name":"offline","context":{"idset":"11099"}}
+{"timestamp":1713385954.1080084,"name":"offline","context":{"idset":"11100"}}
+{"timestamp":1713385954.1085315,"name":"offline","context":{"idset":"11101"}}
+{"timestamp":1713385954.10904,"name":"offline","context":{"idset":"11102"}}
+{"timestamp":1713385954.1095636,"name":"offline","context":{"idset":"11103"}}
+{"timestamp":1713385954.1100724,"name":"offline","context":{"idset":"11104"}}
+{"timestamp":1713385954.1105907,"name":"offline","context":{"idset":"11105"}}
+{"timestamp":1713385954.1110983,"name":"offline","context":{"idset":"11106"}}
+{"timestamp":1713385954.1116145,"name":"offline","context":{"idset":"11107"}}
+{"timestamp":1713385954.1121209,"name":"offline","context":{"idset":"11108"}}
+{"timestamp":1713385954.1126552,"name":"offline","context":{"idset":"11109"}}
+{"timestamp":1713385954.1131608,"name":"offline","context":{"idset":"11110"}}
+{"timestamp":1713385954.1136773,"name":"offline","context":{"idset":"11111"}}
+{"timestamp":1713385954.1141813,"name":"offline","context":{"idset":"11112"}}
+{"timestamp":1713385954.1147199,"name":"offline","context":{"idset":"11113"}}
+{"timestamp":1713385954.1152258,"name":"offline","context":{"idset":"11114"}}
+{"timestamp":1713385954.1157467,"name":"offline","context":{"idset":"11115"}}
+{"timestamp":1713385954.1162519,"name":"offline","context":{"idset":"11116"}}
+{"timestamp":1713385954.1168032,"name":"offline","context":{"idset":"11117"}}
+{"timestamp":1713385954.1173201,"name":"offline","context":{"idset":"11118"}}
+{"timestamp":1713385954.1178246,"name":"offline","context":{"idset":"11119"}}
+{"timestamp":1713385954.118336,"name":"offline","context":{"idset":"11120"}}
+{"timestamp":1713385954.1188369,"name":"offline","context":{"idset":"11121"}}
+{"timestamp":1713385954.1193492,"name":"offline","context":{"idset":"11122"}}
+{"timestamp":1713385954.1198466,"name":"offline","context":{"idset":"11123"}}
+{"timestamp":1713385954.1203547,"name":"offline","context":{"idset":"11124"}}
+{"timestamp":1713385954.1208515,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1713385954.121361,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1713385954.1218555,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1713385954.1223617,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1713385954.1228569,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1713385954.1233649,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1713385954.1238596,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1713385954.1470661,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1713385954.1701074,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1713385954.1931894,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1713385954.2164614,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1713385954.2395916,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1713385954.2676873,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1713385954.2681935,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1713385954.2687037,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1713385954.2691998,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1713385954.2697053,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1713385954.2701955,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1713385954.2707,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1713385954.271193,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1713385954.2717147,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1713385954.2722046,"name":"offline","context":{"idset":"11146"}}
+{"timestamp":1713385954.2727087,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1713385954.2731972,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1713385954.2736979,"name":"offline","context":{"idset":"11149"}}
+{"timestamp":1713385954.2741868,"name":"offline","context":{"idset":"11150"}}
+{"timestamp":1713385954.274688,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1713385954.2751749,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1713385954.2756703,"name":"offline","context":{"idset":"11153"}}
+{"timestamp":1713385954.2761567,"name":"offline","context":{"idset":"11154"}}
+{"timestamp":1713385954.2766581,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1713385954.277142,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1713385954.277642,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1713385954.2781255,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1713385954.278625,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1713385954.2791054,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1713385954.2795975,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1713385954.2800786,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1713385954.2805772,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1713385954.2810564,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1713385954.281548,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1713385954.2820256,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1713385954.2825189,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1713385954.2829955,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1713385954.2834849,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1713385954.2839627,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1713385954.2844551,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1713385954.28493,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1713385954.285418,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1713385954.2858908,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1713385954.3041492,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1713385954.3274825,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1713385954.3505137,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1713385954.3736119,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1713385954.3967659,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1713385954.4201784,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1713385954.4435508,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1713385954.4711723,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1713385954.4943171,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1713385954.5173845,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1713385954.5403817,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1713385954.5634298,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1713385954.5863905,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1713385954.5925329,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1713385954.5930004,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1713385954.5934818,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1713385954.5939467,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1713385954.5944331,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1713385954.5949159,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1713385954.5954022,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1713385954.5958652,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1713385954.6189346,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1713385954.6418743,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1713385954.6648173,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1713385954.6877391,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1713385954.7125845,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1713385954.7365768,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1713385954.7503326,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1713385954.7508039,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1713385954.751297,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1713385954.7517593,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1713385954.7522273,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1713385954.752723,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1713385954.75319,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1713385954.7536685,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1713385954.7541358,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1713385954.754611,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1713385954.7551043,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1713385954.7555809,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1713385954.7560377,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1713385954.7565119,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1713385954.7569611,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1713385954.7574291,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1713385954.7578809,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1713385954.7583535,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1713385954.7587998,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1713385954.7592442,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1713385954.7597263,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1713385954.7601738,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1713385954.7776506,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1713385954.7998688,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1713385954.8003483,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1713385954.8007963,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1713385954.8012388,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1713385954.8016975,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1713385954.8021421,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1713385954.802603,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1713385954.8030465,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1713385954.8035021,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1713385954.8039432,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1713385954.8043985,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1713385954.8048415,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1713385954.8052959,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1713385954.8057384,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1713385954.8061905,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1713385954.8066633,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1713385954.8071053,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1713385954.8075852,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1713385954.8080304,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1713385954.8084919,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1713385954.8089316,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1713385954.8093851,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1713385954.809824,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1713385954.810286,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1713385954.8107245,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1713385954.811161,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1713385954.8116117,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1713385954.812048,"name":"offline","context":{"idset":"11253"}}
+{"timestamp":1713385954.8125029,"name":"offline","context":{"idset":"11254"}}
+{"timestamp":1713385954.8129377,"name":"offline","context":{"idset":"11255"}}
+{"timestamp":1713385954.8133991,"name":"offline","context":{"idset":"11256"}}
+{"timestamp":1713385954.8138337,"name":"offline","context":{"idset":"11257"}}
+{"timestamp":1713385954.8142862,"name":"offline","context":{"idset":"11258"}}
+{"timestamp":1713385954.8260758,"name":"offline","context":{"idset":"11259"}}
+{"timestamp":1713385954.8503666,"name":"offline","context":{"idset":"11260"}}
+{"timestamp":1713385954.8734639,"name":"offline","context":{"idset":"11261"}}
+{"timestamp":1713385954.8964858,"name":"offline","context":{"idset":"11262"}}
+{"timestamp":1713385954.9216392,"name":"offline","context":{"idset":"11263"}}
+{"timestamp":1713385954.927738,"name":"offline","context":{"idset":"11264"}}
+{"timestamp":1713385954.9281673,"name":"offline","context":{"idset":"11265"}}
+{"timestamp":1713385954.928628,"name":"offline","context":{"idset":"11266"}}
+{"timestamp":1713385954.9290574,"name":"offline","context":{"idset":"11267"}}
+{"timestamp":1713385954.9294996,"name":"offline","context":{"idset":"11269"}}
+{"timestamp":1713385954.929925,"name":"offline","context":{"idset":"11270"}}
+{"timestamp":1713385954.9303622,"name":"offline","context":{"idset":"11271"}}
+{"timestamp":1713385954.9307842,"name":"offline","context":{"idset":"11272"}}
+{"timestamp":1713385954.9312062,"name":"offline","context":{"idset":"11273"}}
+{"timestamp":1713385954.931649,"name":"offline","context":{"idset":"11274"}}
+{"timestamp":1713385954.9320726,"name":"offline","context":{"idset":"11275"}}
+{"timestamp":1713385954.9325151,"name":"offline","context":{"idset":"11276"}}
+{"timestamp":1713385954.9329457,"name":"offline","context":{"idset":"11277"}}
+{"timestamp":1713385954.9333794,"name":"offline","context":{"idset":"11278"}}
+{"timestamp":1713385954.9337986,"name":"offline","context":{"idset":"11279"}}
+{"timestamp":1713385954.934217,"name":"offline","context":{"idset":"11280"}}
+{"timestamp":1713385954.9346499,"name":"offline","context":{"idset":"11281"}}
+{"timestamp":1713385954.9350681,"name":"offline","context":{"idset":"11282"}}
+{"timestamp":1713385954.9355109,"name":"offline","context":{"idset":"11283"}}
+{"timestamp":1713385954.9359295,"name":"offline","context":{"idset":"11284"}}
+{"timestamp":1713385954.9363604,"name":"offline","context":{"idset":"11285"}}
+{"timestamp":1713385954.9367766,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1713385954.9371991,"name":"offline","context":{"idset":"11287"}}
+{"timestamp":1713385954.9376438,"name":"offline","context":{"idset":"11288"}}
+{"timestamp":1713385954.9380612,"name":"offline","context":{"idset":"11289"}}
+{"timestamp":1713385954.938498,"name":"offline","context":{"idset":"11290"}}
+{"timestamp":1713385954.9389124,"name":"offline","context":{"idset":"11291"}}
+{"timestamp":1713385954.9393411,"name":"offline","context":{"idset":"11292"}}
+{"timestamp":1713385954.9397516,"name":"offline","context":{"idset":"11293"}}
+{"timestamp":1713385954.9401619,"name":"offline","context":{"idset":"11294"}}
+{"timestamp":1713385954.9405885,"name":"offline","context":{"idset":"11295"}}
+{"timestamp":1713385954.9409974,"name":"offline","context":{"idset":"11296"}}
+{"timestamp":1713385954.9414225,"name":"offline","context":{"idset":"11297"}}
+{"timestamp":1713385954.9418316,"name":"offline","context":{"idset":"11298"}}
+{"timestamp":1713385954.9422388,"name":"offline","context":{"idset":"11299"}}
+{"timestamp":1713385954.9426618,"name":"offline","context":{"idset":"11300"}}
+{"timestamp":1713385954.9430711,"name":"offline","context":{"idset":"11301"}}
+{"timestamp":1713385954.9435003,"name":"offline","context":{"idset":"11302"}}
+{"timestamp":1713385954.9439063,"name":"offline","context":{"idset":"11303"}}
+{"timestamp":1713385954.9443288,"name":"offline","context":{"idset":"11304"}}
+{"timestamp":1713385954.9447351,"name":"offline","context":{"idset":"11305"}}
+{"timestamp":1713385954.9451387,"name":"offline","context":{"idset":"11306"}}
+{"timestamp":1713385954.9455631,"name":"offline","context":{"idset":"11307"}}
+{"timestamp":1713385954.9459665,"name":"offline","context":{"idset":"11308"}}
+{"timestamp":1713385954.9463885,"name":"offline","context":{"idset":"11309"}}
+{"timestamp":1713385954.9467907,"name":"offline","context":{"idset":"11310"}}
+{"timestamp":1713385954.947192,"name":"offline","context":{"idset":"11311"}}
+{"timestamp":1713385954.9591117,"name":"offline","context":{"idset":"11312"}}
+{"timestamp":1713385954.9822116,"name":"offline","context":{"idset":"11313"}}
+{"timestamp":1713385955.0054963,"name":"offline","context":{"idset":"11314"}}
+{"timestamp":1713385955.0306573,"name":"offline","context":{"idset":"11315"}}
+{"timestamp":1713385955.0536354,"name":"offline","context":{"idset":"11316"}}
+{"timestamp":1713385955.0764771,"name":"offline","context":{"idset":"11320"}}
+{"timestamp":1713385955.0825136,"name":"offline","context":{"idset":"11327"}}
+{"timestamp":1713385955.0829201,"name":"offline","context":{"idset":"11328"}}
+{"timestamp":1713385955.0833428,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1713385955.0837443,"name":"offline","context":{"idset":"11333"}}
+{"timestamp":1713385955.0841424,"name":"offline","context":{"idset":"11334"}}
+{"timestamp":1713385955.0845532,"name":"offline","context":{"idset":"11335"}}
+{"timestamp":1713385955.0849485,"name":"offline","context":{"idset":"11336"}}
+{"timestamp":1713385955.0853543,"name":"offline","context":{"idset":"11337"}}
+{"timestamp":1713385955.0857494,"name":"offline","context":{"idset":"11338"}}
+{"timestamp":1713385955.086144,"name":"offline","context":{"idset":"11339"}}
+{"timestamp":1713385955.0865517,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1713385955.086946,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1713385955.0873616,"name":"offline","context":{"idset":"11342"}}
+{"timestamp":1713385955.0877559,"name":"offline","context":{"idset":"11343"}}
+{"timestamp":1713385955.0881455,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1713385955.088547,"name":"offline","context":{"idset":"11345"}}
+{"timestamp":1713385955.0889382,"name":"offline","context":{"idset":"11346"}}
+{"timestamp":1713385955.0893416,"name":"offline","context":{"idset":"11347"}}
+{"timestamp":1713385955.0897303,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1713385955.0901172,"name":"offline","context":{"idset":"11349"}}
+{"timestamp":1713385955.0905209,"name":"offline","context":{"idset":"11350"}}
+{"timestamp":1713385955.0909059,"name":"offline","context":{"idset":"11351"}}
+{"timestamp":1713385955.0913079,"name":"offline","context":{"idset":"11353"}}
+{"timestamp":1713385955.0916953,"name":"offline","context":{"idset":"11354"}}
+{"timestamp":1713385955.0920789,"name":"offline","context":{"idset":"11355"}}
+{"timestamp":1713385955.0924792,"name":"offline","context":{"idset":"11356"}}
+{"timestamp":1713385955.0928609,"name":"offline","context":{"idset":"11357"}}
+{"timestamp":1713385955.0932446,"name":"offline","context":{"idset":"11358"}}
+{"timestamp":1713385955.0936415,"name":"offline","context":{"idset":"11359"}}
+{"timestamp":1713385955.0940218,"name":"offline","context":{"idset":"11360"}}
+{"timestamp":1713385955.0944147,"name":"offline","context":{"idset":"11361"}}
+{"timestamp":1713385955.0947928,"name":"offline","context":{"idset":"11362"}}
+{"timestamp":1713385955.09517,"name":"offline","context":{"idset":"11363"}}
+{"timestamp":1713385955.0955603,"name":"offline","context":{"idset":"11364"}}
+{"timestamp":1713385955.095938,"name":"offline","context":{"idset":"11365"}}
+{"timestamp":1713385955.0963275,"name":"offline","context":{"idset":"11367"}}
+{"timestamp":1713385955.0967035,"name":"offline","context":{"idset":"11368"}}
+{"timestamp":1713385955.0970805,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1713385955.0974667,"name":"offline","context":{"idset":"11370"}}
+{"timestamp":1713385955.0978401,"name":"offline","context":{"idset":"11371"}}
+{"timestamp":1713385955.0982141,"name":"offline","context":{"idset":"11372"}}
+{"timestamp":1713385955.0986052,"name":"offline","context":{"idset":"11373"}}
+{"timestamp":1713385955.0989804,"name":"offline","context":{"idset":"11374"}}
+{"timestamp":1713385955.099369,"name":"offline","context":{"idset":"11375"}}
+{"timestamp":1713385955.0997436,"name":"offline","context":{"idset":"11376"}}
+{"timestamp":1713385955.1001153,"name":"offline","context":{"idset":"11377"}}
+{"timestamp":1713385955.1005018,"name":"offline","context":{"idset":"11378"}}
+{"timestamp":1713385955.1008716,"name":"offline","context":{"idset":"11379"}}
+{"timestamp":1713385955.1012423,"name":"offline","context":{"idset":"11380"}}
+{"timestamp":1713385955.1016276,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1713385955.1019962,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1713385955.1024008,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1713385955.1027699,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1713385955.1031392,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1713385955.1035335,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1713385955.1039023,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1713385955.1042824,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1713385955.1046481,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1713385955.1050138,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1713385955.1053936,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1713385955.105757,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1713385955.1061203,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1713385955.1064999,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1713385955.1068654,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1713385955.1072276,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1713385955.1075978,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1713385955.1079571,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1713385955.1083286,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1713385955.1086869,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1713385955.109046,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1713385955.1094131,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1713385955.1097684,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1713385955.1101239,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1713385955.1105051,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1713385955.1108634,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1713385955.1112273,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1713385955.1116011,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1713385955.1119561,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1713385955.1123323,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1713385955.1126857,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1713385955.1130378,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1713385955.1134198,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1713385955.1137745,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1713385955.1141257,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1713385955.1144896,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1713385955.1148422,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1713385955.1152122,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1713385955.1156266,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1713385955.1218374,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1713385955.1470354,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1713385955.1723793,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1713385955.1956611,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1713385955.2209182,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1713385955.246459,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1713385955.2694752,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1713385955.2932379,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1713385955.3163145,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1713385955.3393276,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1713385955.362395,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1713385955.3740306,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1713385955.3743975,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1713385955.3747416,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1713385955.3750837,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1713385955.3754387,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1713385955.3757827,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1713385955.3761232,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1713385955.3764861,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1713385955.3768299,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1713385955.3771772,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1713385955.3775353,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1713385955.377871,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1713385955.3782098,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1713385955.3785603,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1713385955.3788972,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1713385955.3792362,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1713385955.3795862,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1713385955.3799231,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1713385955.3802564,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1713385955.3806095,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1713385955.3809447,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1713385955.381295,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1713385955.3816273,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1713385955.3819568,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1713385955.3823125,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1713385955.3826444,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1713385955.3829727,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1713385955.3833148,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1713385955.3836417,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1713385955.383966,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1713385955.3843069,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1713385955.3846352,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1713385955.3849607,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1713385955.3853004,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1713385955.385632,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1713385955.385958,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1713385955.3862967,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1713385955.3866167,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1713385955.3869548,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1713385955.3872864,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1713385955.3876064,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1713385955.3879251,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1713385955.3882451,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1713385955.3885784,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1713385955.3888953,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1713385955.3892121,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1713385955.3895447,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1713385955.3898609,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1713385955.390177,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1713385955.3905168,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1713385955.4138474,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1713385955.4366407,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1713385955.4593749,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1713385955.4834232,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1713385955.5076761,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1713385955.5311806,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1713385955.5540144,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1713385955.5811,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1713385955.6060355,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1713385955.6309278,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1713385955.6566346,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1713385955.680371,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1713385955.7047548,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1713385955.7287152,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1713385955.7523122,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1713385955.7826483,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1713385955.8180156,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1713385955.8442781,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1713385955.8672867,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1713385955.892437,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1713385955.9185717,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1713385955.9413388,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1713385955.9416542,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1713385955.9419661,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1713385955.942287,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1713385955.9425976,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1713385955.9429071,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1713385955.9432151,"name":"offline","context":{"idset":"11517"}}
+{"timestamp":1713385955.9435365,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1713385955.9438415,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1713385955.9441476,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1713385955.944464,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1713385955.9447682,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1713385955.9450738,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1713385955.9453905,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1713385955.9456937,"name":"offline","context":{"idset":"11525"}}
+{"timestamp":1713385955.9459965,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1713385955.9463096,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1713385955.9466102,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1713385955.9469078,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1713385955.9472046,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1713385955.9475129,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1713385955.9478073,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1713385955.9481001,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1713385955.9484415,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1713385955.9544046,"name":"offline","context":{"idset":"11536"}}
+{"timestamp":1713385955.9773135,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1713385956.0000904,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1713385956.0230114,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1713385956.0458763,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1713385956.0716949,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1713385956.0956833,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1713385956.1072967,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1713385956.1075952,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1713385956.1078863,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1713385956.1081772,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1713385956.1084797,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1713385956.1087728,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1713385956.1090624,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1713385956.1093638,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1713385956.1096537,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1713385956.1099424,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1713385956.110229,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1713385956.1105382,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1713385956.1108236,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1713385956.1111081,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1713385956.1114066,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1713385956.1116939,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1713385956.1119783,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1713385956.1122727,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1713385956.1125574,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1713385956.1128371,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1713385956.113117,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1713385956.113482,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1713385956.1138265,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1713385956.1141517,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1713385956.1145394,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1713385956.1148703,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1713385956.1152053,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1713385956.1155887,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1713385956.1160059,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1713385956.1163392,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1713385956.1166766,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1713385956.1170006,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1713385956.1173408,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1713385956.1176686,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1713385956.1179981,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1713385956.118341,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1713385956.1186604,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1713385956.1189919,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1713385956.1193438,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1713385956.1196673,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1713385956.1199877,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1713385956.1203244,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1713385956.1206546,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1713385956.1209795,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1713385956.1213417,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1713385956.1216688,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1713385956.1219506,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1713385956.1222179,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1713385956.1225009,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1713385956.1227643,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1713385956.132015,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1713385956.1600957,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1713385956.1891317,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1713385956.2129443,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1713385956.2451193,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1713385956.2696776,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1713385956.2908483,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1713385956.2912374,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1713385956.2916374,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1713385956.2920232,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1713385956.2924283,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1713385956.2928073,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1713385956.2931848,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1713385956.2935727,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1713385956.2939496,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1713385956.294328,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1713385956.2947028,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1713385956.2950683,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1713385956.2954664,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1713385956.2958503,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1713385956.2962341,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1713385956.2966127,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1713385956.2969964,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1713385956.2973588,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1713385956.297682,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1713385956.2979374,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1713385956.298188,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1713385956.2984588,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1713385956.2987075,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1713385956.2989566,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1713385956.2992048,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1713385956.2994721,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1713385956.2997217,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1713385956.2999699,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1713385956.3002174,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1713385956.3005304,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1713385956.3007829,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1713385956.3010292,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1713385956.3013067,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1713385956.3015532,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1713385956.3017943,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1713385956.302036,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1713385956.3022895,"name":"offline","context":{"idset":"11637"}}
+{"timestamp":1713385956.3025386,"name":"offline","context":{"idset":"11638"}}
+{"timestamp":1713385956.3027763,"name":"offline","context":{"idset":"11639"}}
+{"timestamp":1713385956.3030148,"name":"offline","context":{"idset":"11640"}}
+{"timestamp":1713385956.303252,"name":"offline","context":{"idset":"11641"}}
+{"timestamp":1713385956.3035479,"name":"offline","context":{"idset":"11642"}}
+{"timestamp":1713385956.3037856,"name":"offline","context":{"idset":"11643"}}
+{"timestamp":1713385956.3040216,"name":"offline","context":{"idset":"11644"}}
+{"timestamp":1713385956.3042586,"name":"offline","context":{"idset":"11645"}}
+{"timestamp":1713385956.3045101,"name":"offline","context":{"idset":"11646"}}
+{"timestamp":1713385956.3047454,"name":"offline","context":{"idset":"11647"}}
+{"timestamp":1713385956.3049798,"name":"offline","context":{"idset":"11648"}}
+{"timestamp":1713385956.3052132,"name":"offline","context":{"idset":"11649"}}
+{"timestamp":1713385956.305459,"name":"offline","context":{"idset":"11650"}}
+{"timestamp":1713385956.3056962,"name":"offline","context":{"idset":"11651"}}
+{"timestamp":1713385956.3059304,"name":"offline","context":{"idset":"11652"}}
+{"timestamp":1713385956.3061631,"name":"offline","context":{"idset":"11653"}}
+{"timestamp":1713385956.3064039,"name":"offline","context":{"idset":"11654"}}
+{"timestamp":1713385956.306632,"name":"offline","context":{"idset":"11655"}}
+{"timestamp":1713385956.3068612,"name":"offline","context":{"idset":"11656"}}
+{"timestamp":1713385956.3070879,"name":"offline","context":{"idset":"11657"}}
+{"timestamp":1713385956.307327,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1713385956.3075533,"name":"offline","context":{"idset":"11659"}}
+{"timestamp":1713385956.3077796,"name":"offline","context":{"idset":"11660"}}
+{"timestamp":1713385956.3080075,"name":"offline","context":{"idset":"11661"}}
+{"timestamp":1713385956.3082333,"name":"offline","context":{"idset":"11662"}}
+{"timestamp":1713385956.3084671,"name":"offline","context":{"idset":"11663"}}
+{"timestamp":1713385956.3086894,"name":"offline","context":{"idset":"11664"}}
+{"timestamp":1713385956.3089135,"name":"offline","context":{"idset":"11665"}}
+{"timestamp":1713385956.3091354,"name":"offline","context":{"idset":"11666"}}
+{"timestamp":1713385956.3093681,"name":"offline","context":{"idset":"11667"}}
+{"timestamp":1713385956.3095894,"name":"offline","context":{"idset":"11668"}}
+{"timestamp":1713385956.3098106,"name":"offline","context":{"idset":"11669"}}
+{"timestamp":1713385956.3100281,"name":"offline","context":{"idset":"11670"}}
+{"timestamp":1713385956.3102486,"name":"offline","context":{"idset":"11671"}}
+{"timestamp":1713385956.3104784,"name":"offline","context":{"idset":"11672"}}
+{"timestamp":1713385956.3106964,"name":"offline","context":{"idset":"11673"}}
+{"timestamp":1713385956.3109341,"name":"offline","context":{"idset":"11674"}}
+{"timestamp":1713385956.3111658,"name":"offline","context":{"idset":"11675"}}
+{"timestamp":1713385956.3113959,"name":"offline","context":{"idset":"11676"}}
+{"timestamp":1713385956.3116124,"name":"offline","context":{"idset":"11677"}}
+{"timestamp":1713385956.3118286,"name":"offline","context":{"idset":"11678"}}
+{"timestamp":1713385956.3120439,"name":"offline","context":{"idset":"11679"}}
+{"timestamp":1713385956.312259,"name":"offline","context":{"idset":"11680"}}
+{"timestamp":1713385956.3124857,"name":"offline","context":{"idset":"11681"}}
+{"timestamp":1713385956.3126991,"name":"offline","context":{"idset":"11682"}}
+{"timestamp":1713385956.3129132,"name":"offline","context":{"idset":"11683"}}
+{"timestamp":1713385956.3131261,"name":"offline","context":{"idset":"11684"}}
+{"timestamp":1713385956.3133545,"name":"offline","context":{"idset":"11687"}}
+{"timestamp":1713385956.3135679,"name":"offline","context":{"idset":"11688"}}
+{"timestamp":1713385956.3137803,"name":"offline","context":{"idset":"11689"}}
+{"timestamp":1713385956.3139901,"name":"offline","context":{"idset":"11690"}}
+{"timestamp":1713385956.3142011,"name":"offline","context":{"idset":"11695"}}
+{"timestamp":1713385956.3144288,"name":"offline","context":{"idset":"11696"}}
+{"timestamp":1713385956.3146381,"name":"offline","context":{"idset":"11697"}}
+{"timestamp":1713385956.3149016,"name":"offline","context":{"idset":"11698"}}
+{"timestamp":1713385956.3151174,"name":"offline","context":{"idset":"11699"}}
+{"timestamp":1713385956.3153424,"name":"offline","context":{"idset":"11700"}}
+{"timestamp":1713385956.3155506,"name":"offline","context":{"idset":"11701"}}
+{"timestamp":1713385956.3383689,"name":"offline","context":{"idset":"11702"}}
+{"timestamp":1713385956.3614681,"name":"offline","context":{"idset":"11703"}}
+{"timestamp":1713385956.3842957,"name":"offline","context":{"idset":"11704"}}
+{"timestamp":1713385956.4071681,"name":"offline","context":{"idset":"11705"}}
+{"timestamp":1713385956.4299779,"name":"offline","context":{"idset":"11706"}}
+{"timestamp":1713385956.4527209,"name":"offline","context":{"idset":"11707"}}
+{"timestamp":1713385956.4753475,"name":"offline","context":{"idset":"11708"}}
+{"timestamp":1713385956.497957,"name":"offline","context":{"idset":"11709"}}
+{"timestamp":1713385956.5205445,"name":"offline","context":{"idset":"11710"}}
+{"timestamp":1713385956.5431321,"name":"offline","context":{"idset":"11711"}}
+{"timestamp":1713385956.5433514,"name":"offline","context":{"idset":"11712"}}
+{"timestamp":1713385956.5436091,"name":"offline","context":{"idset":"11713"}}
+{"timestamp":1713385956.5439088,"name":"offline","context":{"idset":"11714"}}
+{"timestamp":1713385956.544163,"name":"offline","context":{"idset":"11715"}}
+{"timestamp":1713385956.5444293,"name":"offline","context":{"idset":"11716"}}
+{"timestamp":1713385956.5447164,"name":"offline","context":{"idset":"11717"}}
+{"timestamp":1713385956.5449731,"name":"offline","context":{"idset":"11718"}}
+{"timestamp":1713385956.5451694,"name":"offline","context":{"idset":"11719"}}
+{"timestamp":1713385956.5453894,"name":"offline","context":{"idset":"11720"}}
+{"timestamp":1713385956.5455849,"name":"offline","context":{"idset":"11721"}}
+{"timestamp":1713385956.5457776,"name":"offline","context":{"idset":"11722"}}
+{"timestamp":1713385956.5460329,"name":"offline","context":{"idset":"11723"}}
+{"timestamp":1713385956.5463367,"name":"offline","context":{"idset":"11724"}}
+{"timestamp":1713385956.5466263,"name":"offline","context":{"idset":"11725"}}
+{"timestamp":1713385956.5469146,"name":"offline","context":{"idset":"11726"}}
+{"timestamp":1713385956.5472009,"name":"offline","context":{"idset":"11727"}}
+{"timestamp":1713385956.5475004,"name":"offline","context":{"idset":"11728"}}
+{"timestamp":1713385956.5477529,"name":"offline","context":{"idset":"11729"}}
+{"timestamp":1713385956.5480442,"name":"offline","context":{"idset":"11730"}}
+{"timestamp":1713385956.5483613,"name":"offline","context":{"idset":"11731"}}
+{"timestamp":1713385956.5486658,"name":"offline","context":{"idset":"11732"}}
+{"timestamp":1713385956.5489597,"name":"offline","context":{"idset":"11733"}}
+{"timestamp":1713385956.5492585,"name":"offline","context":{"idset":"11734"}}
+{"timestamp":1713385956.5495698,"name":"offline","context":{"idset":"11735"}}
+{"timestamp":1713385956.5498719,"name":"offline","context":{"idset":"11736"}}
+{"timestamp":1713385956.5501692,"name":"offline","context":{"idset":"11737"}}
+{"timestamp":1713385956.5504732,"name":"offline","context":{"idset":"11738"}}
+{"timestamp":1713385956.5507545,"name":"offline","context":{"idset":"11739"}}
+{"timestamp":1713385956.5510252,"name":"offline","context":{"idset":"11740"}}
+{"timestamp":1713385956.5512185,"name":"offline","context":{"idset":"11741"}}
+{"timestamp":1713385956.5514166,"name":"offline","context":{"idset":"11742"}}
+{"timestamp":1713385956.5516014,"name":"offline","context":{"idset":"11743"}}
+{"timestamp":1713385956.5517955,"name":"offline","context":{"idset":"11744"}}
+{"timestamp":1713385956.5519781,"name":"offline","context":{"idset":"11745"}}
+{"timestamp":1713385956.552161,"name":"offline","context":{"idset":"11746"}}
+{"timestamp":1713385956.5523713,"name":"offline","context":{"idset":"11747"}}
+{"timestamp":1713385956.5525522,"name":"offline","context":{"idset":"11748"}}
+{"timestamp":1713385956.5527341,"name":"offline","context":{"idset":"11749"}}
+{"timestamp":1713385956.5529137,"name":"offline","context":{"idset":"11750"}}
+{"timestamp":1713385956.5531142,"name":"offline","context":{"idset":"11751"}}
+{"timestamp":1713385956.553309,"name":"offline","context":{"idset":"11752"}}
+{"timestamp":1713385956.5534885,"name":"offline","context":{"idset":"11753"}}
+{"timestamp":1713385956.5536687,"name":"offline","context":{"idset":"11754"}}
+{"timestamp":1713385956.553844,"name":"offline","context":{"idset":"11755"}}
+{"timestamp":1713385956.5540185,"name":"offline","context":{"idset":"11756"}}
+{"timestamp":1713385956.5541921,"name":"offline","context":{"idset":"11757"}}
+{"timestamp":1713385956.5543799,"name":"offline","context":{"idset":"11758"}}
+{"timestamp":1713385956.5545545,"name":"offline","context":{"idset":"11759"}}
+{"timestamp":1713385956.554728,"name":"offline","context":{"idset":"11760"}}
+{"timestamp":1713385956.5549023,"name":"offline","context":{"idset":"11761"}}
+{"timestamp":1713385956.5550754,"name":"offline","context":{"idset":"11762"}}
+{"timestamp":1713385956.5552466,"name":"offline","context":{"idset":"11763"}}
+{"timestamp":1713385956.5554295,"name":"offline","context":{"idset":"11764"}}
+{"timestamp":1713385956.5556023,"name":"offline","context":{"idset":"11765"}}
+{"timestamp":1713385956.5557745,"name":"offline","context":{"idset":"11766"}}
+{"timestamp":1713385956.5559435,"name":"offline","context":{"idset":"11767"}}
+{"timestamp":1713385956.5561128,"name":"offline","context":{"idset":"11768"}}
+{"timestamp":1713385956.556293,"name":"offline","context":{"idset":"11769"}}
+{"timestamp":1713385956.5564637,"name":"offline","context":{"idset":"11770"}}
+{"timestamp":1713385956.5566316,"name":"offline","context":{"idset":"11771"}}
+{"timestamp":1713385956.5567975,"name":"offline","context":{"idset":"11772"}}
+{"timestamp":1713385956.5569656,"name":"offline","context":{"idset":"11773"}}
+{"timestamp":1713385956.5572474,"name":"offline","context":{"idset":"11774"}}
+{"timestamp":1713385956.557447,"name":"offline","context":{"idset":"11775"}}
+{"timestamp":1713385956.5576148,"name":"offline","context":{"idset":"11776"}}
+{"timestamp":1713385956.5577798,"name":"offline","context":{"idset":"11777"}}
+{"timestamp":1713385956.5579455,"name":"offline","context":{"idset":"11778"}}
+{"timestamp":1713385956.5581117,"name":"offline","context":{"idset":"11779"}}
+{"timestamp":1713385956.5582871,"name":"offline","context":{"idset":"11780"}}
+{"timestamp":1713385956.5584705,"name":"offline","context":{"idset":"11781"}}
+{"timestamp":1713385956.5586336,"name":"offline","context":{"idset":"11782"}}
+{"timestamp":1713385956.5587955,"name":"offline","context":{"idset":"11783"}}
+{"timestamp":1713385956.5589557,"name":"offline","context":{"idset":"11784"}}
+{"timestamp":1713385956.5591171,"name":"offline","context":{"idset":"11785"}}
+{"timestamp":1713385956.559299,"name":"offline","context":{"idset":"11786"}}
+{"timestamp":1713385956.5594661,"name":"offline","context":{"idset":"11787"}}
+{"timestamp":1713385956.5596247,"name":"offline","context":{"idset":"11788"}}
+{"timestamp":1713385956.5597813,"name":"offline","context":{"idset":"11789"}}
+{"timestamp":1713385956.5599387,"name":"offline","context":{"idset":"11791"}}
+{"timestamp":1713385956.5601072,"name":"offline","context":{"idset":"11792"}}
+{"timestamp":1713385956.5602775,"name":"offline","context":{"idset":"11793"}}
+{"timestamp":1713385956.5604346,"name":"offline","context":{"idset":"11794"}}
+{"timestamp":1713385956.5605896,"name":"offline","context":{"idset":"11795"}}
+{"timestamp":1713385956.5607436,"name":"offline","context":{"idset":"11796"}}
+{"timestamp":1713385956.5608971,"name":"offline","context":{"idset":"11797"}}
+{"timestamp":1713385956.5610492,"name":"offline","context":{"idset":"11798"}}
+{"timestamp":1713385956.5611999,"name":"offline","context":{"idset":"11799"}}
+{"timestamp":1713385956.5614061,"name":"offline","context":{"idset":"11800"}}
+{"timestamp":1713385956.561563,"name":"offline","context":{"idset":"11801"}}
+{"timestamp":1713385956.5617127,"name":"offline","context":{"idset":"11802"}}
+{"timestamp":1713385956.5618625,"name":"offline","context":{"idset":"11803"}}
+{"timestamp":1713385956.5620131,"name":"offline","context":{"idset":"11804"}}
+{"timestamp":1713385956.5621636,"name":"offline","context":{"idset":"11805"}}
+{"timestamp":1713385956.5623455,"name":"offline","context":{"idset":"11806"}}
+{"timestamp":1713385956.5624964,"name":"offline","context":{"idset":"11807"}}
+{"timestamp":1713385956.5626457,"name":"offline","context":{"idset":"11808"}}
+{"timestamp":1713385956.5627925,"name":"offline","context":{"idset":"11809"}}
+{"timestamp":1713385956.5629392,"name":"offline","context":{"idset":"11810"}}
+{"timestamp":1713385956.5630846,"name":"offline","context":{"idset":"11811"}}
+{"timestamp":1713385956.5632334,"name":"offline","context":{"idset":"11812"}}
+{"timestamp":1713385956.5633924,"name":"offline","context":{"idset":"11813"}}
+{"timestamp":1713385956.5635381,"name":"offline","context":{"idset":"11814"}}
+{"timestamp":1713385956.5636811,"name":"offline","context":{"idset":"11815"}}
+{"timestamp":1713385956.5638256,"name":"offline","context":{"idset":"11816"}}
+{"timestamp":1713385956.5639677,"name":"offline","context":{"idset":"11817"}}
+{"timestamp":1713385956.5641093,"name":"offline","context":{"idset":"11818"}}
+{"timestamp":1713385956.5642581,"name":"offline","context":{"idset":"11819"}}
+{"timestamp":1713385956.5644414,"name":"offline","context":{"idset":"11820"}}
+{"timestamp":1713385956.564585,"name":"offline","context":{"idset":"11821"}}
+{"timestamp":1713385956.5647252,"name":"offline","context":{"idset":"11822"}}
+{"timestamp":1713385956.5648656,"name":"offline","context":{"idset":"11823"}}
+{"timestamp":1713385956.5650041,"name":"offline","context":{"idset":"11824"}}
+{"timestamp":1713385956.5651429,"name":"offline","context":{"idset":"11825"}}
+{"timestamp":1713385956.565331,"name":"offline","context":{"idset":"11826"}}
+{"timestamp":1713385956.5654752,"name":"offline","context":{"idset":"11827"}}
+{"timestamp":1713385956.5656133,"name":"offline","context":{"idset":"11828"}}
+{"timestamp":1713385956.5657482,"name":"offline","context":{"idset":"11829"}}
+{"timestamp":1713385956.5658882,"name":"offline","context":{"idset":"11830"}}
+{"timestamp":1713385956.5660233,"name":"offline","context":{"idset":"11831"}}
+{"timestamp":1713385956.5661583,"name":"offline","context":{"idset":"11832"}}
+{"timestamp":1713385956.56633,"name":"offline","context":{"idset":"11833"}}
+{"timestamp":1713385956.5664685,"name":"offline","context":{"idset":"11834"}}
+{"timestamp":1713385956.5666022,"name":"offline","context":{"idset":"11835"}}
+{"timestamp":1713385956.5667589,"name":"offline","context":{"idset":"11836"}}
+{"timestamp":1713385956.5669632,"name":"offline","context":{"idset":"11837"}}
+{"timestamp":1713385956.5671828,"name":"offline","context":{"idset":"11838"}}
+{"timestamp":1713385956.5674844,"name":"offline","context":{"idset":"11839"}}
+{"timestamp":1713385956.5677271,"name":"offline","context":{"idset":"11840"}}
+{"timestamp":1713385956.567874,"name":"offline","context":{"idset":"11841"}}
+{"timestamp":1713385956.5680094,"name":"offline","context":{"idset":"11842"}}
+{"timestamp":1713385956.5681396,"name":"offline","context":{"idset":"11843"}}
+{"timestamp":1713385956.568285,"name":"offline","context":{"idset":"11844"}}
+{"timestamp":1713385956.5684805,"name":"offline","context":{"idset":"11845"}}
+{"timestamp":1713385956.568692,"name":"offline","context":{"idset":"11846"}}
+{"timestamp":1713385956.5688598,"name":"offline","context":{"idset":"11847"}}
+{"timestamp":1713385956.5689924,"name":"offline","context":{"idset":"11848"}}
+{"timestamp":1713385956.5691185,"name":"offline","context":{"idset":"11849"}}
+{"timestamp":1713385956.5692425,"name":"offline","context":{"idset":"11850"}}
+{"timestamp":1713385956.5693867,"name":"offline","context":{"idset":"11851"}}
+{"timestamp":1713385956.5695121,"name":"offline","context":{"idset":"11852"}}
+{"timestamp":1713385956.5696363,"name":"offline","context":{"idset":"11853"}}
+{"timestamp":1713385956.5697603,"name":"offline","context":{"idset":"11854"}}
+{"timestamp":1713385956.5698838,"name":"offline","context":{"idset":"11855"}}
+{"timestamp":1713385956.5700057,"name":"offline","context":{"idset":"11856"}}
+{"timestamp":1713385956.5701275,"name":"offline","context":{"idset":"11857"}}
+{"timestamp":1713385956.5702517,"name":"offline","context":{"idset":"11858"}}
+{"timestamp":1713385956.5704021,"name":"offline","context":{"idset":"11859"}}
+{"timestamp":1713385956.5705249,"name":"offline","context":{"idset":"11860"}}
+{"timestamp":1713385956.5706465,"name":"offline","context":{"idset":"11861"}}
+{"timestamp":1713385956.5707662,"name":"offline","context":{"idset":"11862"}}
+{"timestamp":1713385956.5708849,"name":"offline","context":{"idset":"11863"}}
+{"timestamp":1713385956.5710025,"name":"offline","context":{"idset":"11864"}}
+{"timestamp":1713385956.57112,"name":"offline","context":{"idset":"11865"}}
+{"timestamp":1713385956.5712361,"name":"offline","context":{"idset":"11866"}}
+{"timestamp":1713385956.5713701,"name":"offline","context":{"idset":"11867"}}
+{"timestamp":1713385956.571486,"name":"offline","context":{"idset":"11868"}}
+{"timestamp":1713385956.5716007,"name":"offline","context":{"idset":"11869"}}
+{"timestamp":1713385956.5717154,"name":"offline","context":{"idset":"11870"}}
+{"timestamp":1713385956.5718284,"name":"offline","context":{"idset":"11871"}}
+{"timestamp":1713385956.5719423,"name":"offline","context":{"idset":"11872"}}
+{"timestamp":1713385956.5720551,"name":"offline","context":{"idset":"11873"}}
+{"timestamp":1713385956.5721698,"name":"offline","context":{"idset":"11874"}}
+{"timestamp":1713385956.5722983,"name":"offline","context":{"idset":"11875"}}
+{"timestamp":1713385956.5724103,"name":"offline","context":{"idset":"11876"}}
+{"timestamp":1713385956.583652,"name":"offline","context":{"idset":"11877"}}
+{"timestamp":1713385956.6063204,"name":"offline","context":{"idset":"11878"}}
+{"timestamp":1713385956.6286602,"name":"offline","context":{"idset":"11879"}}
+{"timestamp":1713385956.6507571,"name":"offline","context":{"idset":"11880"}}
+{"timestamp":1713385956.6727748,"name":"offline","context":{"idset":"11881"}}
+{"timestamp":1713385956.6948111,"name":"offline","context":{"idset":"11882"}}
+{"timestamp":1713385956.7169006,"name":"offline","context":{"idset":"11883"}}
+{"timestamp":1713385956.7390676,"name":"offline","context":{"idset":"11884"}}
+{"timestamp":1713385956.7612374,"name":"offline","context":{"idset":"11885"}}
+{"timestamp":1713385956.7833009,"name":"offline","context":{"idset":"11886"}}
+{"timestamp":1713385956.8053067,"name":"offline","context":{"idset":"11887"}}
+{"timestamp":1713385956.827323,"name":"offline","context":{"idset":"11888"}}
+{"timestamp":1713385956.8493679,"name":"offline","context":{"idset":"11889"}}
+{"timestamp":1713385956.865972,"name":"offline","context":{"idset":"11890"}}
+{"timestamp":1713385956.8660824,"name":"offline","context":{"idset":"11891"}}
+{"timestamp":1713385956.8661485,"name":"offline","context":{"idset":"11892"}}
+{"timestamp":1713385956.8662088,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1713385956.8662846,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1713385956.8663473,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1713385956.8664074,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1713385956.8664622,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1713385956.8665185,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1713385956.8665745,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1713385956.8666313,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1713385956.8666894,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1713385956.8667459,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1713385956.8668022,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1713385956.8668559,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1713385956.8669102,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1713385956.8669641,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1713385956.8670156,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1713385956.8670681,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1713385956.8671196,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1713385956.8671706,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1713385956.8672206,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1713385956.8672855,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1713385956.8673391,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1713385956.8673899,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1713385956.8674393,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1713385956.8674886,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1713385956.867537,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1713385956.8675849,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1713385956.867631,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1713385956.8676791,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1713385956.8677244,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1713385956.8677714,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1713385956.8678155,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1713385956.867862,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1713385956.8679113,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1713385956.8679576,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1713385956.8680024,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1713385956.8680484,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1713385956.8680949,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1713385956.8681393,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1713385956.868185,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1713385956.8682303,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1713385956.8682847,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1713385956.8683281,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1713385956.8683686,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1713385956.8684099,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1713385956.8684511,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1713385956.8684907,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1713385956.8685315,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1713385956.8685703,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1713385956.8686099,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1713385956.8686483,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1713385956.8686855,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1713385956.8687232,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1713385956.8687613,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1713385956.8687975,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1713385956.8688352,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1713385956.8688729,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1713385956.8689089,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1713385956.8689435,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1713385956.8689775,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1713385956.8690114,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1713385956.8690455,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1713385956.8690798,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1713385956.8691127,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1713385956.8691528,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1713385956.8691866,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1713385956.86922,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1713385956.869251,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1713385956.8693039,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1713385956.8693459,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1713385956.8693769,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1713385956.8694062,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1713385956.8694365,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1713385956.8694689,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1713385956.8694978,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1713385956.8695271,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1713385956.8695557,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1713385956.8695827,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1713385956.8696103,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1713385956.8696368,"name":"offline","context":{"idset":"120"}}
+{"timestamp":1713385956.8696635,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1713385956.8696892,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1713385956.8697135,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1713385956.8697386,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1713385956.8697631,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1713385956.8697872,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1713385956.8698115,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1713385956.8698344,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1713385956.8698568,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1713385956.8698797,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1713385956.8699012,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1713385956.8699222,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1713385956.8699422,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1713385956.8699639,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1713385956.8699846,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1713385956.8700044,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1713385956.8700237,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1713385956.8700421,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1713385956.87006,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1713385956.870079,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1713385956.8700979,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1713385956.8701153,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1713385956.8701322,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1713385956.8701491,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1713385956.8701651,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1713395465.6419158,"name":"resource-init","context":{"restart":true,"drain":{"789-794":{"timestamp":1711461777.0,"reason":""},"283,336,491":{"timestamp":1713230524.0,"reason":"Down"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"9075":{"timestamp":1712864130.0,"reason":"broker was unresponsive"},"773-774":{"timestamp":1712791239.0,"reason":""},"361,420":{"timestamp":1712942103.0,"reason":"epilog failed for jobid fp6az3kDFZy"},"775-776":{"timestamp":1712792380.0,"reason":""},"404":{"timestamp":1713382661.0,"reason":"epilog failed for jobid fq6AWWH4x9V"},"963-964":{"timestamp":1713196418.0,"reason":"moving blade"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"11790":{"timestamp":1713149681.0,"reason":"Epilog failed to complete"},"11641":{"timestamp":1713304012.0,"reason":"nodediag failed cxi"},"9533":{"timestamp":1713377431.0,"reason":"epilog failed for jobid fq56fLyse87"},"814":{"timestamp":1712783332.0,"reason":""},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"397":{"timestamp":1713229804.0,"reason":"prolog failed for jobid fpkPrjR8F59"},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"961-962":{"timestamp":1713197674.0,"reason":""},"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"573-580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"829-830":{"timestamp":1712700945.0,"reason":""},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"787-788":{"timestamp":1713219302.0,"reason":""},"795":{"timestamp":1712868053.0,"reason":"broker was unresponsive"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"9003":{"timestamp":1712873964.0,"reason":"nodediag failed dgemm_perf"},"474":{"timestamp":1713380077.0,"reason":"epilog failed for jobid fq5rJJ1FQF9"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"828":{"timestamp":1712859028.0,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261863.0,"reason":"--reason swapping blades into x1900 - wendy"},"807-808":{"timestamp":1713376017.0,"reason":""},"827":{"timestamp":1712859030.0,"reason":"broker was unresponsive"},"475-476,483-484":{"timestamp":1713379962.0,"reason":"epilog failed for jobid fq5kLbezSHd"},"11642":{"timestamp":1713305858.0,"reason":"nodediag failed amdapu"},"796":{"timestamp":1712868054.0,"reason":"broker was unresponsive"},"162,409-410":{"timestamp":1713378635.0,"reason":"prolog failed for jobid fq5jtQQSQnT"},"937-938":{"timestamp":1712866351.0,"reason":""},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"771-772":{"timestamp":1713195979.0,"reason":""},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"9054":{"timestamp":1712864116.0,"reason":"broker was unresponsive"},"525-532":{"timestamp":1713290620.0,"reason":""},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1713395465.6427844,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1713395474.9559255,"name":"online","context":{"idset":"0"}}
+{"timestamp":1713395476.6997695,"name":"online","context":{"idset":"10621,10625,10635,10645,10652-10653,10661"}}
+{"timestamp":1713395476.8086178,"name":"online","context":{"idset":"10616-10617,10640,10646,10654,10660,10662"}}
+{"timestamp":1713395476.9113379,"name":"online","context":{"idset":"10624,10626-10628,10630,10633,10636,10641,10647-10648,10650-10651,10655-10656,10659,10663-10666"}}
+{"timestamp":1713395477.014591,"name":"online","context":{"idset":"10615,10618,10622-10623,10649"}}
+{"timestamp":1713395477.2688589,"name":"online","context":{"idset":"477-481,573-580"}}
+{"timestamp":1713395546.764771,"name":"online","context":{"idset":"809"}}
+{"timestamp":1713396057.8446317,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1713396057.931139,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1713396058.0146482,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1713396058.1145916,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1713396058.2587972,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1713396058.8453298,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1713396058.9450386,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1713396058.9916136,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1713396059.0073845,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1713396059.0881879,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1713396059.1011386,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1713396059.1980927,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1713396059.306818,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1713396059.3226945,"name":"offline","context":{"idset":"10626"}}
+{"timestamp":1713396059.3229344,"name":"offline","context":{"idset":"10618"}}
+{"timestamp":1713396059.3922386,"name":"offline","context":{"idset":"10617"}}
+{"timestamp":1713396059.4045663,"name":"offline","context":{"idset":"10640"}}
+{"timestamp":1713396059.4261327,"name":"offline","context":{"idset":"10661"}}
+{"timestamp":1713396059.4376926,"name":"offline","context":{"idset":"10624"}}
+{"timestamp":1713396059.4378731,"name":"offline","context":{"idset":"10616"}}
+{"timestamp":1713396059.4602075,"name":"offline","context":{"idset":"10664"}}
+{"timestamp":1713396059.4603686,"name":"offline","context":{"idset":"10622"}}
+{"timestamp":1713396059.4605196,"name":"offline","context":{"idset":"10625"}}
+{"timestamp":1713396059.4897556,"name":"offline","context":{"idset":"10654"}}
+{"timestamp":1713396059.4898899,"name":"offline","context":{"idset":"10621"}}
+{"timestamp":1713396059.4899819,"name":"offline","context":{"idset":"10635"}}
+{"timestamp":1713396059.4994051,"name":"offline","context":{"idset":"10649"}}
+{"timestamp":1713396059.5251546,"name":"offline","context":{"idset":"10630"}}
+{"timestamp":1713396059.5253198,"name":"offline","context":{"idset":"10627"}}
+{"timestamp":1713396059.5254066,"name":"offline","context":{"idset":"10633"}}
+{"timestamp":1713396059.5523548,"name":"offline","context":{"idset":"10623"}}
+{"timestamp":1713396059.5524473,"name":"offline","context":{"idset":"10647"}}
+{"timestamp":1713396059.5525341,"name":"offline","context":{"idset":"10628"}}
+{"timestamp":1713396059.552659,"name":"offline","context":{"idset":"10636"}}
+{"timestamp":1713396059.5911767,"name":"offline","context":{"idset":"10615"}}
+{"timestamp":1713396059.5912683,"name":"offline","context":{"idset":"10650"}}
+{"timestamp":1713396059.5913439,"name":"offline","context":{"idset":"10655"}}
+{"timestamp":1713396059.5914133,"name":"offline","context":{"idset":"10659"}}
+{"timestamp":1713396059.5914853,"name":"offline","context":{"idset":"10645"}}
+{"timestamp":1713396059.5920861,"name":"offline","context":{"idset":"10662"}}
+{"timestamp":1713396059.5932183,"name":"offline","context":{"idset":"10663"}}
+{"timestamp":1713396059.6582532,"name":"offline","context":{"idset":"10651"}}
+{"timestamp":1713396059.6583357,"name":"offline","context":{"idset":"10641"}}
+{"timestamp":1713396059.6584058,"name":"offline","context":{"idset":"10648"}}
+{"timestamp":1713396059.6584716,"name":"offline","context":{"idset":"10656"}}
+{"timestamp":1713396059.6585352,"name":"offline","context":{"idset":"10652"}}
+{"timestamp":1713396059.6585965,"name":"offline","context":{"idset":"10665"}}
+{"timestamp":1713396059.6586556,"name":"offline","context":{"idset":"10646"}}
+{"timestamp":1713396059.6587179,"name":"offline","context":{"idset":"10660"}}
+{"timestamp":1713396059.7383609,"name":"offline","context":{"idset":"10666"}}
+{"timestamp":1713396059.8210428,"name":"offline","context":{"idset":"10653"}}
+{"timestamp":1713396284.4506865,"name":"online","context":{"idset":"766,819"}}
+{"timestamp":1713396284.7214427,"name":"online","context":{"idset":"821,832"}}
+{"timestamp":1713396284.844032,"name":"online","context":{"idset":"834"}}
+{"timestamp":1713396284.981039,"name":"online","context":{"idset":"774,780"}}
+{"timestamp":1713396285.2505953,"name":"online","context":{"idset":"801,805,815"}}
+{"timestamp":1713396285.4152584,"name":"online","context":{"idset":"797,802,851,878"}}
+{"timestamp":1713396285.6452711,"name":"online","context":{"idset":"882"}}
+{"timestamp":1713396286.0313408,"name":"online","context":{"idset":"852"}}
+{"timestamp":1713396287.7727737,"name":"online","context":{"idset":"890"}}
+{"timestamp":1713396289.8493664,"name":"online","context":{"idset":"896"}}
+{"timestamp":1713396293.9849427,"name":"online","context":{"idset":"906"}}
+{"timestamp":1713396294.1031876,"name":"online","context":{"idset":"904"}}
+{"timestamp":1713396295.0436063,"name":"online","context":{"idset":"912"}}
+{"timestamp":1713396295.5823495,"name":"online","context":{"idset":"924,948,952"}}
+{"timestamp":1713396295.7623773,"name":"online","context":{"idset":"926,930,932,946"}}
+{"timestamp":1713396295.8772779,"name":"online","context":{"idset":"960"}}
+{"timestamp":1713396295.9886346,"name":"online","context":{"idset":"944,958"}}
+{"timestamp":1713396296.2494953,"name":"online","context":{"idset":"934,956"}}
+{"timestamp":1713396297.577704,"name":"online","context":{"idset":"970"}}
+{"timestamp":1713396297.7585616,"name":"online","context":{"idset":"968,976"}}
+{"timestamp":1713396297.8880906,"name":"online","context":{"idset":"974,980"}}
+{"timestamp":1713396780.3639348,"name":"online","context":{"idset":"816"}}
+{"timestamp":1713396974.9449353,"name":"online","context":{"idset":"7672"}}
+{"timestamp":1713396975.8550582,"name":"online","context":{"idset":"7690"}}
+{"timestamp":1713396975.8650098,"name":"online","context":{"idset":"7670"}}
+{"timestamp":1713396976.3242893,"name":"online","context":{"idset":"7698"}}
+{"timestamp":1713396976.4755781,"name":"online","context":{"idset":"7671,7673"}}
+{"timestamp":1713396976.8445642,"name":"online","context":{"idset":"7683,7685,7699"}}
+{"timestamp":1713396977.0487254,"name":"online","context":{"idset":"7675,7684,7687,7727"}}
+{"timestamp":1713396977.1751702,"name":"online","context":{"idset":"7680,7682,7686,7711"}}
+{"timestamp":1713396977.4660037,"name":"online","context":{"idset":"7705,7710"}}
+{"timestamp":1713396977.6167231,"name":"online","context":{"idset":"7681,7694,7707,7715,7719"}}
+{"timestamp":1713396977.7563388,"name":"online","context":{"idset":"7674,7677,7691,7696-7697,7703,7724,7733,7746"}}
+{"timestamp":1713396977.8721676,"name":"online","context":{"idset":"7695,7721,7732,7744"}}
+{"timestamp":1713396977.9848571,"name":"online","context":{"idset":"7669,7688,7706,7708,7712,7716,7742"}}
+{"timestamp":1713396978.1255925,"name":"online","context":{"idset":"7704,7726,7728,7730,7737,7740,7743,7747"}}
+{"timestamp":1713396978.2478263,"name":"online","context":{"idset":"7717,7739,7741"}}
+{"timestamp":1713396978.364887,"name":"online","context":{"idset":"7709,7720,7738,7748"}}
+{"timestamp":1713396978.6221232,"name":"online","context":{"idset":"7729,7734,7745"}}
+{"timestamp":1713396981.2559035,"name":"online","context":{"idset":"7749"}}
+{"timestamp":1713396981.542731,"name":"online","context":{"idset":"7756"}}
+{"timestamp":1713396981.8626149,"name":"online","context":{"idset":"7751,7753,7755"}}
+{"timestamp":1713396981.9737413,"name":"online","context":{"idset":"7757,7761-7762"}}
+{"timestamp":1713396982.08744,"name":"online","context":{"idset":"7752,7758-7759,7765-7766"}}
+{"timestamp":1713396982.2271333,"name":"online","context":{"idset":"7760"}}
+{"timestamp":1713396982.3370852,"name":"online","context":{"idset":"7768"}}
+{"timestamp":1713396982.4428236,"name":"online","context":{"idset":"7767,7769"}}
+{"timestamp":1713396984.7481489,"name":"online","context":{"idset":"7770"}}
+{"timestamp":1713396984.964396,"name":"online","context":{"idset":"7779"}}
+{"timestamp":1713396985.2292571,"name":"online","context":{"idset":"7777"}}
+{"timestamp":1713396985.9729867,"name":"online","context":{"idset":"7772-7773,7781"}}
+{"timestamp":1713396985.9733956,"name":"online","context":{"idset":"7788"}}
+{"timestamp":1713396985.9737689,"name":"online","context":{"idset":"7771,7787,7789"}}
+{"timestamp":1713396985.973938,"name":"online","context":{"idset":"7774,7790"}}
+{"timestamp":1713396985.9741566,"name":"online","context":{"idset":"7776,7792,7796"}}
+{"timestamp":1713396985.9744322,"name":"online","context":{"idset":"7795"}}
+{"timestamp":1713396986.0199776,"name":"online","context":{"idset":"7782,7784,7793,7802"}}
+{"timestamp":1713396986.1551433,"name":"online","context":{"idset":"7780,7794,7797,7801"}}
+{"timestamp":1713396986.3324847,"name":"online","context":{"idset":"7785"}}
+{"timestamp":1713396986.5118904,"name":"online","context":{"idset":"7799,7803"}}
+{"timestamp":1713396986.6361606,"name":"online","context":{"idset":"7791,7804"}}
+{"timestamp":1713396987.1120615,"name":"online","context":{"idset":"7805"}}
+{"timestamp":1713396987.8604457,"name":"online","context":{"idset":"7807"}}
+{"timestamp":1713396988.0419953,"name":"online","context":{"idset":"7810,7812"}}
+{"timestamp":1713396988.2472227,"name":"online","context":{"idset":"7808,7813"}}
+{"timestamp":1713396988.4807513,"name":"online","context":{"idset":"7814"}}
+{"timestamp":1713396988.6302869,"name":"online","context":{"idset":"7811"}}
+{"timestamp":1713396988.8331544,"name":"online","context":{"idset":"7816"}}
+{"timestamp":1713396988.9378133,"name":"online","context":{"idset":"7815"}}
+{"timestamp":1713396989.680232,"name":"online","context":{"idset":"7820"}}
+{"timestamp":1713396989.6810241,"name":"online","context":{"idset":"7825"}}
+{"timestamp":1713396989.6813164,"name":"online","context":{"idset":"7818"}}
+{"timestamp":1713396989.6816919,"name":"online","context":{"idset":"7822,7827,7831"}}
+{"timestamp":1713396989.7337451,"name":"online","context":{"idset":"7817"}}
+{"timestamp":1713396989.8481774,"name":"online","context":{"idset":"7821"}}
+{"timestamp":1713396990.0740895,"name":"online","context":{"idset":"7830"}}
+{"timestamp":1713396990.6142173,"name":"online","context":{"idset":"7829"}}
+{"timestamp":1713396990.7878561,"name":"online","context":{"idset":"7835"}}
+{"timestamp":1713396991.0240843,"name":"online","context":{"idset":"7826,7833,7851"}}
+{"timestamp":1713396991.2144749,"name":"online","context":{"idset":"7832,7838"}}
+{"timestamp":1713396991.3290424,"name":"online","context":{"idset":"7842"}}
+{"timestamp":1713396991.4823618,"name":"online","context":{"idset":"7843"}}
+{"timestamp":1713396991.6395583,"name":"online","context":{"idset":"7834,7836-7837,7847"}}
+{"timestamp":1713396991.7472062,"name":"online","context":{"idset":"7828"}}
+{"timestamp":1713396991.7676442,"name":"online","context":{"idset":"7872"}}
+{"timestamp":1713396991.9390652,"name":"online","context":{"idset":"7839,7859"}}
+{"timestamp":1713396991.9869449,"name":"online","context":{"idset":"7844,7880"}}
+{"timestamp":1713396992.1077783,"name":"online","context":{"idset":"7840,7846,7852,7863,7877-7878"}}
+{"timestamp":1713396992.1385543,"name":"online","context":{"idset":"7853,7879"}}
+{"timestamp":1713396992.2691464,"name":"online","context":{"idset":"7845,7850,7856,7860,7862,7864"}}
+{"timestamp":1713396992.3762047,"name":"online","context":{"idset":"7865,7867,7870,7873-7875,7881-7882"}}
+{"timestamp":1713396992.4676688,"name":"online","context":{"idset":"7866,7876,7883,7885"}}
+{"timestamp":1713396992.5255167,"name":"online","context":{"idset":"7848,7869"}}
+{"timestamp":1713396992.7924421,"name":"online","context":{"idset":"7849"}}
+{"timestamp":1713396992.9346373,"name":"online","context":{"idset":"7884,7886"}}
+{"timestamp":1713396993.4957061,"name":"online","context":{"idset":"7888"}}
+{"timestamp":1713396993.9736238,"name":"online","context":{"idset":"7889,7893,7896"}}
+{"timestamp":1713396994.1683464,"name":"online","context":{"idset":"7887,7891,7894-7895"}}
+{"timestamp":1713396994.3478923,"name":"online","context":{"idset":"7890"}}
+{"timestamp":1713396994.6005571,"name":"online","context":{"idset":"7897,7903-7904"}}
+{"timestamp":1713396994.7025073,"name":"online","context":{"idset":"7899,7901"}}
+{"timestamp":1713396994.8431981,"name":"online","context":{"idset":"7892,7898"}}
+{"timestamp":1713396997.2247109,"name":"online","context":{"idset":"7907"}}
+{"timestamp":1713396997.3516841,"name":"online","context":{"idset":"7905,7910"}}
+{"timestamp":1713396997.4852388,"name":"online","context":{"idset":"7906,7908,7915"}}
+{"timestamp":1713396997.6973166,"name":"online","context":{"idset":"7912,7921"}}
+{"timestamp":1713396997.8858833,"name":"online","context":{"idset":"7913-7914"}}
+{"timestamp":1713396997.9731889,"name":"online","context":{"idset":"7917,7919"}}
+{"timestamp":1713396998.1251593,"name":"online","context":{"idset":"7920"}}
+{"timestamp":1713396998.3190851,"name":"online","context":{"idset":"7922"}}
+{"timestamp":1713396998.5200264,"name":"online","context":{"idset":"7923"}}
+{"timestamp":1713396998.6367688,"name":"online","context":{"idset":"7926,7931"}}
+{"timestamp":1713396999.3528457,"name":"online","context":{"idset":"7932,7942"}}
+{"timestamp":1713396999.3533566,"name":"online","context":{"idset":"7924,7927-7929,7934"}}
+{"timestamp":1713396999.3536484,"name":"online","context":{"idset":"7935,7937,7940"}}
+{"timestamp":1713396999.3539269,"name":"online","context":{"idset":"7933"}}
+{"timestamp":1713396999.35426,"name":"online","context":{"idset":"7938"}}
+{"timestamp":1713396999.3545871,"name":"online","context":{"idset":"7943"}}
+{"timestamp":1713396999.7653124,"name":"online","context":{"idset":"7945"}}
+{"timestamp":1713396999.8400257,"name":"online","context":{"idset":"7946"}}
+{"timestamp":1713397000.2251861,"name":"online","context":{"idset":"7947"}}
+{"timestamp":1713397000.3356128,"name":"online","context":{"idset":"7955"}}
+{"timestamp":1713397000.4980798,"name":"online","context":{"idset":"7951,7956"}}
+{"timestamp":1713397000.7756135,"name":"online","context":{"idset":"7954,7957"}}
+{"timestamp":1713397000.8764536,"name":"online","context":{"idset":"7958"}}
+{"timestamp":1713397001.1367064,"name":"online","context":{"idset":"7959"}}
+{"timestamp":1713397001.5329478,"name":"online","context":{"idset":"7963-7964"}}
+{"timestamp":1713397001.6411908,"name":"online","context":{"idset":"7960,7962"}}
+{"timestamp":1713397002.0248258,"name":"online","context":{"idset":"7965,7967-7968,7970"}}
+{"timestamp":1713397002.2186985,"name":"online","context":{"idset":"7973"}}
+{"timestamp":1713397002.3861291,"name":"online","context":{"idset":"7972"}}
+{"timestamp":1713397002.4992893,"name":"online","context":{"idset":"7969"}}
+{"timestamp":1713397002.8014784,"name":"online","context":{"idset":"7977"}}
+{"timestamp":1713397002.8184879,"name":"online","context":{"idset":"7974"}}
+{"timestamp":1713397003.0100133,"name":"online","context":{"idset":"7976"}}
+{"timestamp":1713397003.2526941,"name":"online","context":{"idset":"7982"}}
+{"timestamp":1713397003.3845732,"name":"online","context":{"idset":"7980,7984"}}
+{"timestamp":1713397003.4536574,"name":"online","context":{"idset":"7985"}}
+{"timestamp":1713397003.75899,"name":"online","context":{"idset":"7986"}}
+{"timestamp":1713397004.610997,"name":"online","context":{"idset":"7991,7994"}}
+{"timestamp":1713397004.6115942,"name":"online","context":{"idset":"7989"}}
+{"timestamp":1713397004.6122904,"name":"online","context":{"idset":"7993"}}
+{"timestamp":1713397004.612977,"name":"online","context":{"idset":"7990"}}
+{"timestamp":1713397004.6136553,"name":"online","context":{"idset":"7996"}}
+{"timestamp":1713397004.6627855,"name":"online","context":{"idset":"7995"}}
+{"timestamp":1713397004.9167256,"name":"online","context":{"idset":"7997"}}
+{"timestamp":1713397005.202985,"name":"online","context":{"idset":"7992"}}
+{"timestamp":1713397005.3298714,"name":"online","context":{"idset":"7998,8004"}}
+{"timestamp":1713397005.3722742,"name":"online","context":{"idset":"8003"}}
+{"timestamp":1713397005.6475315,"name":"online","context":{"idset":"7999"}}
+{"timestamp":1713397005.8502533,"name":"online","context":{"idset":"8000,8005,8008,8011"}}
+{"timestamp":1713397005.9761028,"name":"online","context":{"idset":"8012,8015"}}
+{"timestamp":1713397006.2258203,"name":"online","context":{"idset":"8010,8019,8039"}}
+{"timestamp":1713397006.3373716,"name":"online","context":{"idset":"8025,8028,8040"}}
+{"timestamp":1713397006.4851305,"name":"online","context":{"idset":"8016,8021,8024,8031,8042"}}
+{"timestamp":1713397006.6011822,"name":"online","context":{"idset":"8018,8023,8026,8033-8034,8043"}}
+{"timestamp":1713397006.8177962,"name":"online","context":{"idset":"8017,8029,8038,8047-8048,8185,8198"}}
+{"timestamp":1713397006.8782294,"name":"online","context":{"idset":"8035"}}
+{"timestamp":1713397007.0674236,"name":"online","context":{"idset":"8030,8046,8050,8052,8182,8189,8196-8197,8200"}}
+{"timestamp":1713397007.2022054,"name":"online","context":{"idset":"8186"}}
+{"timestamp":1713397007.303328,"name":"online","context":{"idset":"8188"}}
+{"timestamp":1713397007.4166431,"name":"online","context":{"idset":"8032,8184,8201"}}
+{"timestamp":1713397007.6559637,"name":"online","context":{"idset":"8190"}}
+{"timestamp":1713397008.1148689,"name":"online","context":{"idset":"8199"}}
+{"timestamp":1713397009.1052442,"name":"online","context":{"idset":"8202-8203"}}
+{"timestamp":1713397009.2885401,"name":"online","context":{"idset":"8204"}}
+{"timestamp":1713397009.5725129,"name":"online","context":{"idset":"8206"}}
+{"timestamp":1713397009.6074531,"name":"online","context":{"idset":"8213"}}
+{"timestamp":1713397009.8802934,"name":"online","context":{"idset":"8211"}}
+{"timestamp":1713397010.0505919,"name":"online","context":{"idset":"8208"}}
+{"timestamp":1713397010.0628994,"name":"online","context":{"idset":"8219"}}
+{"timestamp":1713397010.2395086,"name":"online","context":{"idset":"8207,8216"}}
+{"timestamp":1713397010.4165232,"name":"online","context":{"idset":"8212,8215"}}
+{"timestamp":1713397010.4683516,"name":"online","context":{"idset":"8218"}}
+{"timestamp":1713397010.679652,"name":"online","context":{"idset":"8221,8224"}}
+{"timestamp":1713397010.8505185,"name":"online","context":{"idset":"8226-8227"}}
+{"timestamp":1713397010.9301863,"name":"online","context":{"idset":"8225"}}
+{"timestamp":1713397011.0435894,"name":"online","context":{"idset":"8209"}}
+{"timestamp":1713397011.1652043,"name":"online","context":{"idset":"8222-8223,8229,8231"}}
+{"timestamp":1713397011.4388652,"name":"online","context":{"idset":"8234-8235"}}
+{"timestamp":1713397011.5568135,"name":"online","context":{"idset":"8230"}}
+{"timestamp":1713397011.7027721,"name":"online","context":{"idset":"8233,8238,8240"}}
+{"timestamp":1713397011.8689432,"name":"online","context":{"idset":"8241"}}
+{"timestamp":1713397012.0076146,"name":"online","context":{"idset":"8243"}}
+{"timestamp":1713397012.0950453,"name":"online","context":{"idset":"8239"}}
+{"timestamp":1713397012.1305938,"name":"online","context":{"idset":"8242"}}
+{"timestamp":1713397012.3669386,"name":"online","context":{"idset":"8244"}}
+{"timestamp":1713397012.4818578,"name":"online","context":{"idset":"8245,8248"}}
+{"timestamp":1713397012.6084118,"name":"online","context":{"idset":"8249"}}
+{"timestamp":1713397013.0858815,"name":"online","context":{"idset":"8252"}}
+{"timestamp":1713397013.1882763,"name":"online","context":{"idset":"8250"}}
+{"timestamp":1713397013.301213,"name":"online","context":{"idset":"8253"}}
+{"timestamp":1713397013.6065757,"name":"online","context":{"idset":"8251,8257"}}
+{"timestamp":1713397014.0533566,"name":"online","context":{"idset":"8258,8264"}}
+{"timestamp":1713397014.2675261,"name":"online","context":{"idset":"8255,8261-8262,8266"}}
+{"timestamp":1713397014.2978864,"name":"online","context":{"idset":"8263"}}
+{"timestamp":1713397015.1518495,"name":"online","context":{"idset":"8265"}}
+{"timestamp":1713397015.1527297,"name":"online","context":{"idset":"8267-8269,8272"}}
+{"timestamp":1713397015.2006884,"name":"online","context":{"idset":"8271"}}
+{"timestamp":1713397015.4677994,"name":"online","context":{"idset":"8270,8275"}}
+{"timestamp":1713397015.6656091,"name":"online","context":{"idset":"8274"}}
+{"timestamp":1713397015.7348349,"name":"online","context":{"idset":"8279"}}
+{"timestamp":1713397015.7944841,"name":"online","context":{"idset":"8282-8283"}}
+{"timestamp":1713397016.0232375,"name":"online","context":{"idset":"8280"}}
+{"timestamp":1713397016.2352581,"name":"online","context":{"idset":"8276,8278"}}
+{"timestamp":1713397016.4490669,"name":"online","context":{"idset":"8286"}}
+{"timestamp":1713397016.5073891,"name":"online","context":{"idset":"8285"}}
+{"timestamp":1713397016.7387521,"name":"online","context":{"idset":"8287"}}
+{"timestamp":1713397017.1660485,"name":"online","context":{"idset":"8289-8290"}}
+{"timestamp":1713397017.3925359,"name":"online","context":{"idset":"8288"}}
+{"timestamp":1713397017.6533141,"name":"online","context":{"idset":"8293"}}
+{"timestamp":1713397018.5571947,"name":"online","context":{"idset":"8295-8296,8300"}}
+{"timestamp":1713397018.5580876,"name":"online","context":{"idset":"8294"}}
+{"timestamp":1713397018.5587411,"name":"online","context":{"idset":"8297,8303"}}
+{"timestamp":1713397018.5594287,"name":"online","context":{"idset":"8302,8307"}}
+{"timestamp":1713397019.3215952,"name":"online","context":{"idset":"8305-8306,8308"}}
+{"timestamp":1713397019.3576078,"name":"online","context":{"idset":"8826"}}
+{"timestamp":1713397019.3583336,"name":"online","context":{"idset":"8299"}}
+{"timestamp":1713397019.359179,"name":"online","context":{"idset":"8822,8827"}}
+{"timestamp":1713397019.4248271,"name":"online","context":{"idset":"8821"}}
+{"timestamp":1713397019.5865688,"name":"online","context":{"idset":"8823-8824,8832,8837"}}
+{"timestamp":1713397019.7998226,"name":"online","context":{"idset":"8828"}}
+{"timestamp":1713397019.9193373,"name":"online","context":{"idset":"8839"}}
+{"timestamp":1713397020.0785336,"name":"online","context":{"idset":"8833,8835"}}
+{"timestamp":1713397020.20699,"name":"online","context":{"idset":"8840"}}
+{"timestamp":1713397020.2477715,"name":"online","context":{"idset":"8844,8848"}}
+{"timestamp":1713397020.435674,"name":"online","context":{"idset":"8829,8836,8847"}}
+{"timestamp":1713397020.6534722,"name":"online","context":{"idset":"8841,8845,8851"}}
+{"timestamp":1713397020.7222202,"name":"online","context":{"idset":"8834,8843,8846"}}
+{"timestamp":1713397021.3032632,"name":"online","context":{"idset":"8852,8854"}}
+{"timestamp":1713397021.5287418,"name":"online","context":{"idset":"8856"}}
+{"timestamp":1713397021.7347453,"name":"online","context":{"idset":"8858-8859"}}
+{"timestamp":1713397021.8752315,"name":"online","context":{"idset":"8855"}}
+{"timestamp":1713397021.950896,"name":"online","context":{"idset":"8853,8860"}}
+{"timestamp":1713397022.480052,"name":"online","context":{"idset":"8857,8862,8865"}}
+{"timestamp":1713397022.5683126,"name":"online","context":{"idset":"8867"}}
+{"timestamp":1713397022.6218841,"name":"online","context":{"idset":"8866"}}
+{"timestamp":1713397022.8053799,"name":"online","context":{"idset":"8873"}}
+{"timestamp":1713397022.8262794,"name":"online","context":{"idset":"8864"}}
+{"timestamp":1713397023.1759753,"name":"online","context":{"idset":"8861,8868,8874-8876"}}
+{"timestamp":1713397023.2195582,"name":"online","context":{"idset":"8870"}}
+{"timestamp":1713397023.3063908,"name":"online","context":{"idset":"8863,8879"}}
+{"timestamp":1713397023.4577608,"name":"online","context":{"idset":"8878"}}
+{"timestamp":1713397023.5133071,"name":"online","context":{"idset":"8884"}}
+{"timestamp":1713397023.6330175,"name":"online","context":{"idset":"8877,8891"}}
+{"timestamp":1713397023.8220513,"name":"online","context":{"idset":"8869,8880"}}
+{"timestamp":1713397023.8774817,"name":"online","context":{"idset":"8887"}}
+{"timestamp":1713397024.0804849,"name":"online","context":{"idset":"8881,8888"}}
+{"timestamp":1713397024.2553699,"name":"online","context":{"idset":"8890,8892,8895"}}
+{"timestamp":1713397024.3858993,"name":"online","context":{"idset":"8893-8894"}}
+{"timestamp":1713397024.546278,"name":"online","context":{"idset":"8885"}}
+{"timestamp":1713397024.5635431,"name":"online","context":{"idset":"8903"}}
+{"timestamp":1713397024.6414561,"name":"online","context":{"idset":"8897,8899,8901"}}
+{"timestamp":1713397024.8738639,"name":"online","context":{"idset":"8898,8904,8906,8911"}}
+{"timestamp":1713397024.8941002,"name":"online","context":{"idset":"8912"}}
+{"timestamp":1713397025.0285091,"name":"online","context":{"idset":"8910,8914"}}
+{"timestamp":1713397025.3552246,"name":"online","context":{"idset":"8907,8916"}}
+{"timestamp":1713397025.6259046,"name":"online","context":{"idset":"8913,8918"}}
+{"timestamp":1713397025.7411137,"name":"online","context":{"idset":"8915"}}
+{"timestamp":1713397025.925091,"name":"online","context":{"idset":"8921"}}
+{"timestamp":1713397026.0042777,"name":"online","context":{"idset":"8920"}}
+{"timestamp":1713397026.5092831,"name":"online","context":{"idset":"8925"}}
+{"timestamp":1713397026.6357601,"name":"online","context":{"idset":"8924,8926-8927,8929-8930"}}
+{"timestamp":1713397026.7871318,"name":"online","context":{"idset":"8928"}}
+{"timestamp":1713397027.0041606,"name":"online","context":{"idset":"8931,8935"}}
+{"timestamp":1713397027.1720591,"name":"online","context":{"idset":"8934,8936"}}
+{"timestamp":1713397027.2888584,"name":"online","context":{"idset":"8937-8938"}}
+{"timestamp":1713397027.4444902,"name":"online","context":{"idset":"8932"}}
+{"timestamp":1713397027.579329,"name":"online","context":{"idset":"8940"}}
+{"timestamp":1713397027.724364,"name":"online","context":{"idset":"8939"}}
+{"timestamp":1713397028.1254323,"name":"online","context":{"idset":"8946"}}
+{"timestamp":1713397028.2157807,"name":"online","context":{"idset":"8942"}}
+{"timestamp":1713397028.344465,"name":"online","context":{"idset":"8947"}}
+{"timestamp":1713397028.4697616,"name":"online","context":{"idset":"8943-8944,9077-9078"}}
+{"timestamp":1713397029.1882706,"name":"online","context":{"idset":"9080"}}
+{"timestamp":1713397029.7471559,"name":"online","context":{"idset":"9081,9084"}}
+{"timestamp":1713397029.8271189,"name":"online","context":{"idset":"9083,9086"}}
+{"timestamp":1713397029.8811111,"name":"online","context":{"idset":"9079"}}
+{"timestamp":1713397030.2774091,"name":"online","context":{"idset":"9085,9090"}}
+{"timestamp":1713397030.3912413,"name":"online","context":{"idset":"9087,9089"}}
+{"timestamp":1713397030.4625003,"name":"online","context":{"idset":"9082"}}
+{"timestamp":1713397030.6424594,"name":"online","context":{"idset":"9093-9095"}}
+{"timestamp":1713397030.7105169,"name":"online","context":{"idset":"9091"}}
+{"timestamp":1713397030.871489,"name":"online","context":{"idset":"9092,9098"}}
+{"timestamp":1713397031.1760712,"name":"online","context":{"idset":"9096"}}
+{"timestamp":1713397031.2777219,"name":"online","context":{"idset":"9099-9100"}}
+{"timestamp":1713397031.5973008,"name":"online","context":{"idset":"9102"}}
+{"timestamp":1713397031.7116768,"name":"online","context":{"idset":"9101"}}
+{"timestamp":1713397031.9652436,"name":"online","context":{"idset":"9103"}}
+{"timestamp":1713397032.0801895,"name":"online","context":{"idset":"9105,9107"}}
+{"timestamp":1713397032.2004795,"name":"online","context":{"idset":"9109"}}
+{"timestamp":1713397032.3294418,"name":"online","context":{"idset":"9108"}}
+{"timestamp":1713397032.3846023,"name":"online","context":{"idset":"9104"}}
+{"timestamp":1713397032.4801803,"name":"online","context":{"idset":"9106"}}
+{"timestamp":1713397032.6148548,"name":"online","context":{"idset":"9111,9114"}}
+{"timestamp":1713397032.9867866,"name":"online","context":{"idset":"9112-9113,9119,9125"}}
+{"timestamp":1713397033.0220852,"name":"online","context":{"idset":"9116"}}
+{"timestamp":1713397033.123148,"name":"online","context":{"idset":"9115,9121-9122"}}
+{"timestamp":1713397033.3111246,"name":"online","context":{"idset":"9110,9123"}}
+{"timestamp":1713397033.3366067,"name":"online","context":{"idset":"9120"}}
+{"timestamp":1713397033.518023,"name":"online","context":{"idset":"9124"}}
+{"timestamp":1713397033.6089265,"name":"online","context":{"idset":"9126"}}
+{"timestamp":1713397033.726213,"name":"online","context":{"idset":"9129"}}
+{"timestamp":1713397034.0003839,"name":"online","context":{"idset":"9132"}}
+{"timestamp":1713397034.056057,"name":"online","context":{"idset":"9127"}}
+{"timestamp":1713397034.1183414,"name":"online","context":{"idset":"9128"}}
+{"timestamp":1713397034.2536359,"name":"online","context":{"idset":"9136"}}
+{"timestamp":1713397034.4135542,"name":"online","context":{"idset":"9134"}}
+{"timestamp":1713397034.5171907,"name":"online","context":{"idset":"9133,9137"}}
+{"timestamp":1713397034.8153484,"name":"online","context":{"idset":"9135,9141"}}
+{"timestamp":1713397035.0274227,"name":"online","context":{"idset":"9139"}}
+{"timestamp":1713397035.1525047,"name":"online","context":{"idset":"9148"}}
+{"timestamp":1713397035.5046539,"name":"online","context":{"idset":"9138,9143"}}
+{"timestamp":1713397035.5438986,"name":"online","context":{"idset":"9146"}}
+{"timestamp":1713397035.6391268,"name":"online","context":{"idset":"9150-9151"}}
+{"timestamp":1713397035.7637289,"name":"online","context":{"idset":"9149,9158"}}
+{"timestamp":1713397035.794879,"name":"online","context":{"idset":"9156"}}
+{"timestamp":1713397035.8906798,"name":"online","context":{"idset":"9147"}}
+{"timestamp":1713397036.0292435,"name":"online","context":{"idset":"9153"}}
+{"timestamp":1713397036.1923664,"name":"online","context":{"idset":"9152,9155,9159-9160"}}
+{"timestamp":1713397036.2214344,"name":"online","context":{"idset":"9157"}}
+{"timestamp":1713397036.5937519,"name":"online","context":{"idset":"9162,9164-9165"}}
+{"timestamp":1713397036.7142656,"name":"online","context":{"idset":"9154,9163,9166"}}
+{"timestamp":1713397037.0099514,"name":"online","context":{"idset":"9167,9169"}}
+{"timestamp":1713397037.1887598,"name":"online","context":{"idset":"9171,9175"}}
+{"timestamp":1713397037.3595889,"name":"online","context":{"idset":"9172-9174,9183"}}
+{"timestamp":1713397037.4210205,"name":"online","context":{"idset":"9179"}}
+{"timestamp":1713397037.5574608,"name":"online","context":{"idset":"9180"}}
+{"timestamp":1713397037.6642108,"name":"online","context":{"idset":"9177"}}
+{"timestamp":1713397037.8249824,"name":"online","context":{"idset":"9182"}}
+{"timestamp":1713397038.0180371,"name":"online","context":{"idset":"9176,9178"}}
+{"timestamp":1713397038.0400615,"name":"online","context":{"idset":"9184"}}
+{"timestamp":1713397038.1578462,"name":"online","context":{"idset":"9186"}}
+{"timestamp":1713397038.3325901,"name":"online","context":{"idset":"9187,9190,9192"}}
+{"timestamp":1713397038.6601779,"name":"online","context":{"idset":"9189,9196"}}
+{"timestamp":1713397038.8765187,"name":"online","context":{"idset":"9195"}}
+{"timestamp":1713397038.9519339,"name":"online","context":{"idset":"9188"}}
+{"timestamp":1713397039.075974,"name":"online","context":{"idset":"9191,9199"}}
+{"timestamp":1713397039.1297021,"name":"online","context":{"idset":"9207"}}
+{"timestamp":1713397039.2695277,"name":"online","context":{"idset":"9204"}}
+{"timestamp":1713397039.5094621,"name":"online","context":{"idset":"9197"}}
+{"timestamp":1713397039.5550354,"name":"online","context":{"idset":"9205"}}
+{"timestamp":1713397039.6794338,"name":"online","context":{"idset":"9201,9209,9211"}}
+{"timestamp":1713397039.9372625,"name":"online","context":{"idset":"9210"}}
+{"timestamp":1713397040.241781,"name":"online","context":{"idset":"9212,9216"}}
+{"timestamp":1713397040.382843,"name":"online","context":{"idset":"9215"}}
+{"timestamp":1713397040.5219581,"name":"online","context":{"idset":"9214"}}
+{"timestamp":1713397040.5523827,"name":"online","context":{"idset":"9220"}}
+{"timestamp":1713397040.7229927,"name":"online","context":{"idset":"9218"}}
+{"timestamp":1713397041.0776381,"name":"online","context":{"idset":"9222-9223,9225"}}
+{"timestamp":1713397041.0990789,"name":"online","context":{"idset":"9224"}}
+{"timestamp":1713397041.539542,"name":"online","context":{"idset":"9221,9226"}}
+{"timestamp":1713397041.7661769,"name":"online","context":{"idset":"9227,9229,9233"}}
+{"timestamp":1713397041.8473794,"name":"online","context":{"idset":"9230"}}
+{"timestamp":1713397042.0124867,"name":"online","context":{"idset":"9235"}}
+{"timestamp":1713397042.1827633,"name":"online","context":{"idset":"9236-9237"}}
+{"timestamp":1713397042.3095756,"name":"online","context":{"idset":"9244"}}
+{"timestamp":1713397042.3848333,"name":"online","context":{"idset":"9234"}}
+{"timestamp":1713397042.5507658,"name":"online","context":{"idset":"9242"}}
+{"timestamp":1713397042.7639143,"name":"online","context":{"idset":"9239,9241,9243"}}
+{"timestamp":1713397043.015995,"name":"online","context":{"idset":"9248"}}
+{"timestamp":1713397043.3069634,"name":"online","context":{"idset":"9249"}}
+{"timestamp":1713397043.3896215,"name":"online","context":{"idset":"9247,9250"}}
+{"timestamp":1713397043.5571594,"name":"online","context":{"idset":"9245"}}
+{"timestamp":1713397043.7537549,"name":"online","context":{"idset":"9251-9252"}}
+{"timestamp":1713397043.7883584,"name":"online","context":{"idset":"9254"}}
+{"timestamp":1713397044.0722156,"name":"online","context":{"idset":"9253,9257"}}
+{"timestamp":1713397044.1237354,"name":"online","context":{"idset":"9255,9258"}}
+{"timestamp":1713397044.3616972,"name":"online","context":{"idset":"9256"}}
+{"timestamp":1713397044.514117,"name":"online","context":{"idset":"9259-9260"}}
+{"timestamp":1713397044.6559556,"name":"online","context":{"idset":"9263"}}
+{"timestamp":1713397044.7030792,"name":"online","context":{"idset":"9269"}}
+{"timestamp":1713397044.931577,"name":"online","context":{"idset":"9267"}}
+{"timestamp":1713397045.1119957,"name":"online","context":{"idset":"9261,9270-9271,9276"}}
+{"timestamp":1713397045.2102008,"name":"online","context":{"idset":"9264"}}
+{"timestamp":1713397045.3430278,"name":"online","context":{"idset":"9274-9275"}}
+{"timestamp":1713397045.4304311,"name":"online","context":{"idset":"9272"}}
+{"timestamp":1713397045.5245025,"name":"online","context":{"idset":"9262,9268,9278"}}
+{"timestamp":1713397045.9313724,"name":"online","context":{"idset":"9279,9285-9286"}}
+{"timestamp":1713397045.9896028,"name":"online","context":{"idset":"9287"}}
+{"timestamp":1713397046.105078,"name":"online","context":{"idset":"9283"}}
+{"timestamp":1713397046.1514637,"name":"online","context":{"idset":"9281"}}
+{"timestamp":1713397046.4815946,"name":"online","context":{"idset":"9292"}}
+{"timestamp":1713397046.6019001,"name":"online","context":{"idset":"9291"}}
+{"timestamp":1713397046.6703162,"name":"online","context":{"idset":"9289-9290"}}
+{"timestamp":1713397046.8599479,"name":"online","context":{"idset":"9294"}}
+{"timestamp":1713397046.9607959,"name":"online","context":{"idset":"9293"}}
+{"timestamp":1713397047.2728925,"name":"online","context":{"idset":"9297"}}
+{"timestamp":1713397047.326566,"name":"online","context":{"idset":"9300"}}
+{"timestamp":1713397047.438544,"name":"online","context":{"idset":"9304"}}
+{"timestamp":1713397047.6134672,"name":"online","context":{"idset":"9295,9308"}}
+{"timestamp":1713397047.9405222,"name":"online","context":{"idset":"9307"}}
+{"timestamp":1713397048.0495005,"name":"online","context":{"idset":"9296,9305"}}
+{"timestamp":1713397048.2233706,"name":"online","context":{"idset":"9309"}}
+{"timestamp":1713397048.2713869,"name":"online","context":{"idset":"9302,9315"}}
+{"timestamp":1713397048.4124985,"name":"online","context":{"idset":"9306,9310,9312,9314,9316,9318"}}
+{"timestamp":1713397048.6481774,"name":"online","context":{"idset":"9319"}}
+{"timestamp":1713397048.7797751,"name":"online","context":{"idset":"9320,9322"}}
+{"timestamp":1713397048.8043473,"name":"online","context":{"idset":"9323"}}
+{"timestamp":1713397048.9598761,"name":"online","context":{"idset":"9324"}}
+{"timestamp":1713397049.0062997,"name":"online","context":{"idset":"9321"}}
+{"timestamp":1713397049.1139221,"name":"online","context":{"idset":"9329"}}
+{"timestamp":1713397049.2503192,"name":"online","context":{"idset":"9326,9333"}}
+{"timestamp":1713397049.3521538,"name":"online","context":{"idset":"9325,9327,9336,9339"}}
+{"timestamp":1713397049.4169092,"name":"online","context":{"idset":"9344"}}
+{"timestamp":1713397049.5336215,"name":"online","context":{"idset":"9337,9349"}}
+{"timestamp":1713397049.6297879,"name":"online","context":{"idset":"9341"}}
+{"timestamp":1713397049.7617784,"name":"online","context":{"idset":"9345-9346,9350"}}
+{"timestamp":1713397050.0836494,"name":"online","context":{"idset":"9347"}}
+{"timestamp":1713397050.2523198,"name":"online","context":{"idset":"9335,9353-9354"}}
+{"timestamp":1713397050.5670457,"name":"online","context":{"idset":"9351,9357"}}
+{"timestamp":1713397050.6913893,"name":"online","context":{"idset":"9359"}}
+{"timestamp":1713397050.810132,"name":"online","context":{"idset":"9362"}}
+{"timestamp":1713397050.9122505,"name":"online","context":{"idset":"9360-9361,9366"}}
+{"timestamp":1713397050.9957528,"name":"online","context":{"idset":"9365"}}
+{"timestamp":1713397051.0885189,"name":"online","context":{"idset":"9363"}}
+{"timestamp":1713397051.1431243,"name":"online","context":{"idset":"9367"}}
+{"timestamp":1713397051.190021,"name":"online","context":{"idset":"9369"}}
+{"timestamp":1713397051.3714135,"name":"online","context":{"idset":"9364"}}
+{"timestamp":1713397051.6220484,"name":"online","context":{"idset":"9371"}}
+{"timestamp":1713397051.7756205,"name":"online","context":{"idset":"9370"}}
+{"timestamp":1713397052.0716391,"name":"online","context":{"idset":"9374"}}
+{"timestamp":1713397052.2213402,"name":"online","context":{"idset":"9377,9379,9382"}}
+{"timestamp":1713397052.3420107,"name":"online","context":{"idset":"9375"}}
+{"timestamp":1713397052.5736821,"name":"online","context":{"idset":"9372,9383"}}
+{"timestamp":1713397052.89622,"name":"online","context":{"idset":"9385"}}
+{"timestamp":1713397053.1204469,"name":"online","context":{"idset":"9386"}}
+{"timestamp":1713397053.2590475,"name":"online","context":{"idset":"9384,9387,9389"}}
+{"timestamp":1713397053.5819452,"name":"online","context":{"idset":"9390"}}
+{"timestamp":1713397053.7301011,"name":"online","context":{"idset":"9391"}}
+{"timestamp":1713397053.9469085,"name":"online","context":{"idset":"9392-9393,9395"}}
+{"timestamp":1713397054.0754867,"name":"online","context":{"idset":"9394"}}
+{"timestamp":1713397054.3911018,"name":"online","context":{"idset":"9399"}}
+{"timestamp":1713397054.4776421,"name":"online","context":{"idset":"9396,9402"}}
+{"timestamp":1713397054.868422,"name":"online","context":{"idset":"9398,9403-9404,9407"}}
+{"timestamp":1713397055.0627525,"name":"online","context":{"idset":"9401,9405"}}
+{"timestamp":1713397055.1918776,"name":"online","context":{"idset":"9400,9410"}}
+{"timestamp":1713397055.2718694,"name":"online","context":{"idset":"9406"}}
+{"timestamp":1713397055.4557517,"name":"online","context":{"idset":"9409"}}
+{"timestamp":1713397055.5512059,"name":"online","context":{"idset":"9415"}}
+{"timestamp":1713397055.6820827,"name":"online","context":{"idset":"9411,9414"}}
+{"timestamp":1713397055.900562,"name":"online","context":{"idset":"9412,9419,9422"}}
+{"timestamp":1713397056.1596625,"name":"online","context":{"idset":"9417"}}
+{"timestamp":1713397056.6282389,"name":"online","context":{"idset":"9416,9420,9425"}}
+{"timestamp":1713397056.8052447,"name":"online","context":{"idset":"9429"}}
+{"timestamp":1713397056.9254112,"name":"online","context":{"idset":"9424,9428,9431-9432"}}
+{"timestamp":1713397057.1258779,"name":"online","context":{"idset":"9426"}}
+{"timestamp":1713397057.2203968,"name":"online","context":{"idset":"9430,9433"}}
+{"timestamp":1713397057.4317102,"name":"online","context":{"idset":"9427,9434"}}
+{"timestamp":1713397057.4533594,"name":"online","context":{"idset":"9435"}}
+{"timestamp":1713397057.5468574,"name":"online","context":{"idset":"9436"}}
+{"timestamp":1713397057.6333628,"name":"online","context":{"idset":"9438-9439"}}
+{"timestamp":1713397057.7273962,"name":"online","context":{"idset":"9437,9441"}}
+{"timestamp":1713397058.1190755,"name":"online","context":{"idset":"9440,9443-9444"}}
+{"timestamp":1713397058.396404,"name":"online","context":{"idset":"9445-9447"}}
+{"timestamp":1713397059.4261284,"name":"online","context":{"idset":"9448"}}
+{"timestamp":1713397059.4275448,"name":"online","context":{"idset":"9449"}}
+{"timestamp":1713397059.4288797,"name":"online","context":{"idset":"9456"}}
+{"timestamp":1713397059.4300377,"name":"online","context":{"idset":"9450,9453"}}
+{"timestamp":1713397059.431186,"name":"online","context":{"idset":"9451"}}
+{"timestamp":1713397059.7268579,"name":"online","context":{"idset":"9455,9457,9459"}}
+{"timestamp":1713397059.7955608,"name":"online","context":{"idset":"9461,9472"}}
+{"timestamp":1713397059.9066117,"name":"online","context":{"idset":"9458"}}
+{"timestamp":1713397060.0688708,"name":"online","context":{"idset":"9464"}}
+{"timestamp":1713397060.3057072,"name":"online","context":{"idset":"9477"}}
+{"timestamp":1713397060.4085393,"name":"online","context":{"idset":"9465,9467,9469,9479"}}
+{"timestamp":1713397060.5410206,"name":"online","context":{"idset":"9466,9476"}}
+{"timestamp":1713397060.605958,"name":"online","context":{"idset":"9473,9475"}}
+{"timestamp":1713397060.646214,"name":"online","context":{"idset":"9471"}}
+{"timestamp":1713397060.7769287,"name":"online","context":{"idset":"9478"}}
+{"timestamp":1713397060.9562881,"name":"online","context":{"idset":"9483,9485-9486"}}
+{"timestamp":1713397061.0524063,"name":"online","context":{"idset":"9482"}}
+{"timestamp":1713397061.3357651,"name":"online","context":{"idset":"9484"}}
+{"timestamp":1713397061.3610024,"name":"online","context":{"idset":"9497"}}
+{"timestamp":1713397061.4806945,"name":"online","context":{"idset":"9488-9489,9492-9493"}}
+{"timestamp":1713397061.6344123,"name":"online","context":{"idset":"9495-9496,9500"}}
+{"timestamp":1713397061.9130948,"name":"online","context":{"idset":"9499"}}
+{"timestamp":1713397061.9927249,"name":"online","context":{"idset":"9503"}}
+{"timestamp":1713397062.1359773,"name":"online","context":{"idset":"9487,9498,9504"}}
+{"timestamp":1713397062.2877939,"name":"online","context":{"idset":"9508"}}
+{"timestamp":1713397062.3897293,"name":"online","context":{"idset":"9509"}}
+{"timestamp":1713397062.5122933,"name":"online","context":{"idset":"9506"}}
+{"timestamp":1713397062.6357906,"name":"online","context":{"idset":"9510-9512"}}
+{"timestamp":1713397062.7561564,"name":"online","context":{"idset":"9502,9513"}}
+{"timestamp":1713397062.7821522,"name":"online","context":{"idset":"9514"}}
+{"timestamp":1713397063.0583148,"name":"online","context":{"idset":"9515,9518"}}
+{"timestamp":1713397063.184289,"name":"online","context":{"idset":"9517,9519"}}
+{"timestamp":1713397063.2944579,"name":"online","context":{"idset":"9521"}}
+{"timestamp":1713397063.5354605,"name":"online","context":{"idset":"9520"}}
+{"timestamp":1713397063.6027744,"name":"online","context":{"idset":"9522"}}
+{"timestamp":1713397063.7455876,"name":"online","context":{"idset":"9523"}}
+{"timestamp":1713397064.1461306,"name":"online","context":{"idset":"9524,9528"}}
+{"timestamp":1713397064.3821936,"name":"online","context":{"idset":"9525,9530"}}
+{"timestamp":1713397064.4595149,"name":"online","context":{"idset":"9531"}}
+{"timestamp":1713397064.9344938,"name":"online","context":{"idset":"9526"}}
+{"timestamp":1713397065.058136,"name":"online","context":{"idset":"9532,9535,9540"}}
+{"timestamp":1713397065.2453716,"name":"online","context":{"idset":"9537"}}
+{"timestamp":1713397065.3296499,"name":"online","context":{"idset":"9536"}}
+{"timestamp":1713397065.4311419,"name":"online","context":{"idset":"9545,9548"}}
+{"timestamp":1713397065.6847482,"name":"online","context":{"idset":"9549"}}
+{"timestamp":1713397065.853456,"name":"online","context":{"idset":"9550"}}
+{"timestamp":1713397065.957432,"name":"online","context":{"idset":"9547,9552-9553"}}
+{"timestamp":1713397066.2822542,"name":"online","context":{"idset":"9554,9564"}}
+{"timestamp":1713397066.5668681,"name":"online","context":{"idset":"9557"}}
+{"timestamp":1713397066.717479,"name":"online","context":{"idset":"9575,9592-9593"}}
+{"timestamp":1713397066.8782938,"name":"online","context":{"idset":"9590"}}
+{"timestamp":1713397066.8811016,"name":"online","context":{"idset":"9597"}}
+{"timestamp":1713397067.1100535,"name":"online","context":{"idset":"9598"}}
+{"timestamp":1713397067.3851442,"name":"online","context":{"idset":"9555,9594,9596,9601"}}
+{"timestamp":1713397067.5106294,"name":"online","context":{"idset":"9600"}}
+{"timestamp":1713397067.8754141,"name":"online","context":{"idset":"9604,9606"}}
+{"timestamp":1713397067.9051642,"name":"online","context":{"idset":"9602"}}
+{"timestamp":1713397068.17558,"name":"online","context":{"idset":"9605"}}
+{"timestamp":1713397068.3051472,"name":"online","context":{"idset":"9603,9610"}}
+{"timestamp":1713397068.4383552,"name":"online","context":{"idset":"9609"}}
+{"timestamp":1713397068.6167176,"name":"online","context":{"idset":"9611-9613,9616"}}
+{"timestamp":1713397068.7936146,"name":"online","context":{"idset":"9614-9615"}}
+{"timestamp":1713397068.8634336,"name":"online","context":{"idset":"9630"}}
+{"timestamp":1713397068.9651978,"name":"online","context":{"idset":"9618-9619,9621"}}
+{"timestamp":1713397069.1523018,"name":"online","context":{"idset":"9617"}}
+{"timestamp":1713397069.330554,"name":"online","context":{"idset":"9620,9623,9628"}}
+{"timestamp":1713397069.6771095,"name":"online","context":{"idset":"9632,9635-9637"}}
+{"timestamp":1713397069.9266424,"name":"online","context":{"idset":"9642"}}
+{"timestamp":1713397070.0654709,"name":"online","context":{"idset":"9629,9643"}}
+{"timestamp":1713397070.2343891,"name":"online","context":{"idset":"9645"}}
+{"timestamp":1713397070.5842721,"name":"online","context":{"idset":"9648"}}
+{"timestamp":1713397070.6488886,"name":"online","context":{"idset":"9644"}}
+{"timestamp":1713397070.7734599,"name":"online","context":{"idset":"9646-9647"}}
+{"timestamp":1713397070.9437103,"name":"online","context":{"idset":"9649"}}
+{"timestamp":1713397071.0599892,"name":"online","context":{"idset":"9650"}}
+{"timestamp":1713397071.2793829,"name":"online","context":{"idset":"9653"}}
+{"timestamp":1713397071.5196049,"name":"online","context":{"idset":"9652,9657"}}
+{"timestamp":1713397071.7137601,"name":"online","context":{"idset":"9654,9656,9658"}}
+{"timestamp":1713397072.0007515,"name":"online","context":{"idset":"9651"}}
+{"timestamp":1713397072.1734684,"name":"online","context":{"idset":"9659"}}
+{"timestamp":1713397072.2656581,"name":"online","context":{"idset":"9661,9663"}}
+{"timestamp":1713397072.4086652,"name":"online","context":{"idset":"9662,9671-9672"}}
+{"timestamp":1713397072.6014199,"name":"online","context":{"idset":"9665-9667"}}
+{"timestamp":1713397072.6449492,"name":"online","context":{"idset":"9674"}}
+{"timestamp":1713397072.7676339,"name":"online","context":{"idset":"9668,9673,9675"}}
+{"timestamp":1713397072.943099,"name":"online","context":{"idset":"9827"}}
+{"timestamp":1713397073.119643,"name":"online","context":{"idset":"9676,9678,9681,9684,9686"}}
+{"timestamp":1713397073.2237639,"name":"online","context":{"idset":"9680"}}
+{"timestamp":1713397073.4000273,"name":"online","context":{"idset":"9683,9826"}}
+{"timestamp":1713397073.8285606,"name":"online","context":{"idset":"9831-9832"}}
+{"timestamp":1713397073.9541202,"name":"online","context":{"idset":"9829,9835"}}
+{"timestamp":1713397074.2230263,"name":"online","context":{"idset":"9838"}}
+{"timestamp":1713397074.3730474,"name":"online","context":{"idset":"9828,9834,9846"}}
+{"timestamp":1713397074.6366799,"name":"online","context":{"idset":"9841-9843"}}
+{"timestamp":1713397074.7921948,"name":"online","context":{"idset":"9848,9850-9851"}}
+{"timestamp":1713397074.8193896,"name":"online","context":{"idset":"9837"}}
+{"timestamp":1713397074.8908513,"name":"online","context":{"idset":"9833"}}
+{"timestamp":1713397075.0110769,"name":"online","context":{"idset":"9839,9854"}}
+{"timestamp":1713397075.1890569,"name":"online","context":{"idset":"9852,9855"}}
+{"timestamp":1713397075.5948367,"name":"online","context":{"idset":"9856,9858-9859"}}
+{"timestamp":1713397076.1215775,"name":"online","context":{"idset":"9865"}}
+{"timestamp":1713397076.1885109,"name":"online","context":{"idset":"9860"}}
+{"timestamp":1713397076.3937027,"name":"online","context":{"idset":"9863,9866"}}
+{"timestamp":1713397076.4867439,"name":"online","context":{"idset":"9861,9868-9869"}}
+{"timestamp":1713397076.8219008,"name":"online","context":{"idset":"9871"}}
+{"timestamp":1713397076.8885102,"name":"online","context":{"idset":"9862"}}
+{"timestamp":1713397076.9181616,"name":"online","context":{"idset":"9870"}}
+{"timestamp":1713397076.9584239,"name":"online","context":{"idset":"9873"}}
+{"timestamp":1713397077.1070528,"name":"online","context":{"idset":"9874"}}
+{"timestamp":1713397077.522311,"name":"online","context":{"idset":"9877"}}
+{"timestamp":1713397077.8011956,"name":"online","context":{"idset":"9878-9879,9881-9882"}}
+{"timestamp":1713397077.8836839,"name":"online","context":{"idset":"9875-9876"}}
+{"timestamp":1713397077.9858654,"name":"online","context":{"idset":"9880"}}
+{"timestamp":1713397078.3306053,"name":"online","context":{"idset":"9883,9886"}}
+{"timestamp":1713397078.6136451,"name":"online","context":{"idset":"9887"}}
+{"timestamp":1713397078.8269053,"name":"online","context":{"idset":"9891,9894"}}
+{"timestamp":1713397078.9748373,"name":"online","context":{"idset":"9885,9890,9895"}}
+{"timestamp":1713397079.114043,"name":"online","context":{"idset":"9893,9896-9897"}}
+{"timestamp":1713397079.3410258,"name":"online","context":{"idset":"9901"}}
+{"timestamp":1713397079.3846595,"name":"online","context":{"idset":"9889"}}
+{"timestamp":1713397079.5345035,"name":"online","context":{"idset":"9898,9900"}}
+{"timestamp":1713397079.6587083,"name":"online","context":{"idset":"9902"}}
+{"timestamp":1713397079.9007239,"name":"online","context":{"idset":"9905"}}
+{"timestamp":1713397079.9954698,"name":"online","context":{"idset":"9903,9908"}}
+{"timestamp":1713397080.2741189,"name":"online","context":{"idset":"9906,9910,9914"}}
+{"timestamp":1713397080.5439699,"name":"online","context":{"idset":"9911,9917"}}
+{"timestamp":1713397080.7527273,"name":"online","context":{"idset":"9915,9919,9921"}}
+{"timestamp":1713397080.9954555,"name":"online","context":{"idset":"9922,9928"}}
+{"timestamp":1713397081.1687498,"name":"online","context":{"idset":"9912,9924,9927"}}
+{"timestamp":1713397081.3001943,"name":"online","context":{"idset":"9926,9929"}}
+{"timestamp":1713397081.51987,"name":"online","context":{"idset":"9934"}}
+{"timestamp":1713397081.8275607,"name":"online","context":{"idset":"9933"}}
+{"timestamp":1713397081.9560468,"name":"online","context":{"idset":"9936"}}
+{"timestamp":1713397082.0202608,"name":"online","context":{"idset":"9931,9941"}}
+{"timestamp":1713397082.1205714,"name":"online","context":{"idset":"9930,9932,9935"}}
+{"timestamp":1713397082.5080571,"name":"online","context":{"idset":"9944"}}
+{"timestamp":1713397082.5404792,"name":"online","context":{"idset":"9937,9943"}}
+{"timestamp":1713397082.5712967,"name":"online","context":{"idset":"9939"}}
+{"timestamp":1713397083.0141318,"name":"online","context":{"idset":"9945"}}
+{"timestamp":1713397083.0682507,"name":"online","context":{"idset":"9952"}}
+{"timestamp":1713397083.1620238,"name":"online","context":{"idset":"9947"}}
+{"timestamp":1713397083.5057025,"name":"online","context":{"idset":"9951,9956"}}
+{"timestamp":1713397083.5437949,"name":"online","context":{"idset":"9948"}}
+{"timestamp":1713397083.6760976,"name":"online","context":{"idset":"9949"}}
+{"timestamp":1713397083.7796271,"name":"online","context":{"idset":"9957-9958"}}
+{"timestamp":1713397084.031198,"name":"online","context":{"idset":"9959,9961"}}
+{"timestamp":1713397084.4506705,"name":"online","context":{"idset":"9953,9963"}}
+{"timestamp":1713397084.6451755,"name":"online","context":{"idset":"9967,9972,9975"}}
+{"timestamp":1713397084.7900085,"name":"online","context":{"idset":"9960,9965,9969,9973"}}
+{"timestamp":1713397084.9269228,"name":"online","context":{"idset":"9970"}}
+{"timestamp":1713397085.0614879,"name":"online","context":{"idset":"9974"}}
+{"timestamp":1713397085.160114,"name":"online","context":{"idset":"9977,9982"}}
+{"timestamp":1713397085.3512113,"name":"online","context":{"idset":"9978,9980,9986"}}
+{"timestamp":1713397085.4998214,"name":"online","context":{"idset":"9981"}}
+{"timestamp":1713397085.6888328,"name":"online","context":{"idset":"9983,9987"}}
+{"timestamp":1713397087.348042,"name":"online","context":{"idset":"9988"}}
+{"timestamp":1713397087.5656679,"name":"online","context":{"idset":"10003"}}
+{"timestamp":1713397087.9210203,"name":"online","context":{"idset":"10008"}}
+{"timestamp":1713397088.336827,"name":"online","context":{"idset":"10002"}}
+{"timestamp":1713397088.6230168,"name":"online","context":{"idset":"9971"}}
+{"timestamp":1713397088.7870979,"name":"online","context":{"idset":"9996"}}
+{"timestamp":1713397089.3712099,"name":"online","context":{"idset":"9994"}}
+{"timestamp":1713397091.6618876,"name":"online","context":{"idset":"10007"}}
+{"timestamp":1713397092.9454341,"name":"online","context":{"idset":"10032"}}
+{"timestamp":1713397093.1233954,"name":"online","context":{"idset":"10047"}}
+{"timestamp":1713397093.9259105,"name":"online","context":{"idset":"10013"}}
+{"timestamp":1713397094.4240046,"name":"online","context":{"idset":"10006"}}
+{"timestamp":1713397094.6283588,"name":"online","context":{"idset":"9995,10005"}}
+{"timestamp":1713397095.6021461,"name":"online","context":{"idset":"10011"}}
+{"timestamp":1713397096.1735468,"name":"online","context":{"idset":"9997,10038"}}
+{"timestamp":1713397098.194979,"name":"online","context":{"idset":"10009"}}
+{"timestamp":1713397099.1404197,"name":"online","context":{"idset":"10015,10023"}}
+{"timestamp":1713397099.3272381,"name":"online","context":{"idset":"10076,10095"}}
+{"timestamp":1713397099.3854668,"name":"online","context":{"idset":"10012"}}
+{"timestamp":1713397099.5472493,"name":"online","context":{"idset":"10099"}}
+{"timestamp":1713397099.9733384,"name":"online","context":{"idset":"10035"}}
+{"timestamp":1713397100.1454866,"name":"online","context":{"idset":"10072"}}
+{"timestamp":1713397100.3372602,"name":"online","context":{"idset":"10068"}}
+{"timestamp":1713397100.8805342,"name":"online","context":{"idset":"10031"}}
+{"timestamp":1713397101.0479195,"name":"online","context":{"idset":"10053"}}
+{"timestamp":1713397101.2999346,"name":"online","context":{"idset":"9989,10024"}}
+{"timestamp":1713397101.447742,"name":"online","context":{"idset":"10018,10045"}}
+{"timestamp":1713397101.7370329,"name":"online","context":{"idset":"10027,10040"}}
+{"timestamp":1713397102.0147474,"name":"online","context":{"idset":"10067"}}
+{"timestamp":1713397102.2196448,"name":"online","context":{"idset":"10043"}}
+{"timestamp":1713397102.6552458,"name":"online","context":{"idset":"10056,10059"}}
+{"timestamp":1713397103.0335937,"name":"online","context":{"idset":"10033"}}
+{"timestamp":1713397103.517776,"name":"online","context":{"idset":"10014,10054"}}
+{"timestamp":1713397103.8709278,"name":"online","context":{"idset":"10022"}}
+{"timestamp":1713397104.0137942,"name":"online","context":{"idset":"10039"}}
+{"timestamp":1713397104.4471765,"name":"online","context":{"idset":"10120"}}
+{"timestamp":1713397104.6888974,"name":"online","context":{"idset":"10049"}}
+{"timestamp":1713397105.3597348,"name":"online","context":{"idset":"10145"}}
+{"timestamp":1713397105.5279517,"name":"online","context":{"idset":"10079"}}
+{"timestamp":1713397105.7275684,"name":"online","context":{"idset":"10078"}}
+{"timestamp":1713397105.8191867,"name":"online","context":{"idset":"10051,10058"}}
+{"timestamp":1713397106.1308129,"name":"online","context":{"idset":"10086,10125"}}
+{"timestamp":1713397106.3205736,"name":"online","context":{"idset":"10025,10091"}}
+{"timestamp":1713397106.5066843,"name":"online","context":{"idset":"10048,10063,10066"}}
+{"timestamp":1713397107.3005347,"name":"online","context":{"idset":"10028"}}
+{"timestamp":1713397107.4288924,"name":"online","context":{"idset":"10037"}}
+{"timestamp":1713397107.6102729,"name":"online","context":{"idset":"9999,10041,10087"}}
+{"timestamp":1713397107.9368849,"name":"online","context":{"idset":"10057"}}
+{"timestamp":1713397108.0755041,"name":"online","context":{"idset":"10097"}}
+{"timestamp":1713397108.2218659,"name":"online","context":{"idset":"10084"}}
+{"timestamp":1713397108.3718297,"name":"online","context":{"idset":"10085,10153"}}
+{"timestamp":1713397109.1604557,"name":"online","context":{"idset":"10026"}}
+{"timestamp":1713397109.428715,"name":"online","context":{"idset":"10042,10100"}}
+{"timestamp":1713397109.5940833,"name":"online","context":{"idset":"10062"}}
+{"timestamp":1713397109.7672353,"name":"online","context":{"idset":"10052"}}
+{"timestamp":1713397110.1000557,"name":"online","context":{"idset":"10075"}}
+{"timestamp":1713397110.3246651,"name":"online","context":{"idset":"10050"}}
+{"timestamp":1713397111.0707433,"name":"online","context":{"idset":"10044,10073-10074"}}
+{"timestamp":1713397111.78426,"name":"online","context":{"idset":"10239"}}
+{"timestamp":1713397112.4255133,"name":"online","context":{"idset":"10077,10240"}}
+{"timestamp":1713397112.6674628,"name":"online","context":{"idset":"10081,10168"}}
+{"timestamp":1713397112.8937488,"name":"online","context":{"idset":"10094"}}
+{"timestamp":1713397112.996011,"name":"online","context":{"idset":"10082"}}
+{"timestamp":1713397113.2037697,"name":"online","context":{"idset":"10136"}}
+{"timestamp":1713397113.6690357,"name":"online","context":{"idset":"10070"}}
+{"timestamp":1713397113.961211,"name":"online","context":{"idset":"10065"}}
+{"timestamp":1713397114.1480899,"name":"online","context":{"idset":"10096"}}
+{"timestamp":1713397114.5910606,"name":"online","context":{"idset":"10080"}}
+{"timestamp":1713397115.2303267,"name":"online","context":{"idset":"10161"}}
+{"timestamp":1713397115.6627109,"name":"online","context":{"idset":"10069"}}
+{"timestamp":1713397115.9007967,"name":"online","context":{"idset":"10108,10217"}}
+{"timestamp":1713397116.2116532,"name":"online","context":{"idset":"10083,10223,10243"}}
+{"timestamp":1713397116.4038689,"name":"online","context":{"idset":"10144"}}
+{"timestamp":1713397116.6940579,"name":"online","context":{"idset":"10106"}}
+{"timestamp":1713397117.1262429,"name":"online","context":{"idset":"10139"}}
+{"timestamp":1713397117.2130599,"name":"online","context":{"idset":"10098"}}
+{"timestamp":1713397117.5583327,"name":"online","context":{"idset":"10093"}}
+{"timestamp":1713397117.9393456,"name":"online","context":{"idset":"10121"}}
+{"timestamp":1713397118.2296369,"name":"online","context":{"idset":"10101,10137"}}
+{"timestamp":1713397118.7415843,"name":"online","context":{"idset":"10141"}}
+{"timestamp":1713397119.3916788,"name":"online","context":{"idset":"10176"}}
+{"timestamp":1713397119.8155713,"name":"online","context":{"idset":"10229,10244"}}
+{"timestamp":1713397120.6860476,"name":"online","context":{"idset":"10122"}}
+{"timestamp":1713397121.0487967,"name":"online","context":{"idset":"10222"}}
+{"timestamp":1713397121.6375237,"name":"online","context":{"idset":"10228"}}
+{"timestamp":1713397121.682369,"name":"online","context":{"idset":"10127"}}
+{"timestamp":1713397121.85584,"name":"online","context":{"idset":"10200"}}
+{"timestamp":1713397121.9709883,"name":"online","context":{"idset":"10114"}}
+{"timestamp":1713397122.1845312,"name":"online","context":{"idset":"10036"}}
+{"timestamp":1713397122.4910009,"name":"online","context":{"idset":"10246"}}
+{"timestamp":1713397122.7500885,"name":"online","context":{"idset":"10250,10264"}}
+{"timestamp":1713397123.0220954,"name":"online","context":{"idset":"10135"}}
+{"timestamp":1713397123.1523006,"name":"online","context":{"idset":"10172,10260"}}
+{"timestamp":1713397123.3222554,"name":"online","context":{"idset":"10258"}}
+{"timestamp":1713397123.5356402,"name":"online","context":{"idset":"10138"}}
+{"timestamp":1713397123.5829732,"name":"online","context":{"idset":"10286"}}
+{"timestamp":1713397123.6886353,"name":"online","context":{"idset":"10226,10273,10308"}}
+{"timestamp":1713397123.7778893,"name":"online","context":{"idset":"10170,10184,10207"}}
+{"timestamp":1713397123.8242357,"name":"online","context":{"idset":"10254,10285,10330"}}
+{"timestamp":1713397123.8655934,"name":"online","context":{"idset":"10178"}}
+{"timestamp":1713397123.9683297,"name":"online","context":{"idset":"10188,10262,10297,10321"}}
+{"timestamp":1713397124.1057298,"name":"online","context":{"idset":"10245,10272,10295,10300,10307,10316"}}
+{"timestamp":1713397124.2411261,"name":"online","context":{"idset":"10182,10288,10298,10313-10314,10328"}}
+{"timestamp":1713397124.3470657,"name":"online","context":{"idset":"10195,10267,10278,10323,10339"}}
+{"timestamp":1713397124.4522552,"name":"online","context":{"idset":"10191,10210,10248,10274,10287,10304,10306,10338,10342"}}
+{"timestamp":1713397124.5532272,"name":"online","context":{"idset":"10186,10257,10282,10290,10303,10309,10340-10341"}}
+{"timestamp":1713397124.6858239,"name":"online","context":{"idset":"10194,10270,10294,10310-10311,10326,10337"}}
+{"timestamp":1713397124.7882214,"name":"online","context":{"idset":"10266,10289,10317,10343-10345"}}
+{"timestamp":1713397124.8899748,"name":"online","context":{"idset":"10279,10291,10302,10305,10333,10346,10348"}}
+{"timestamp":1713397125.016278,"name":"online","context":{"idset":"10192,10247,10249,10253"}}
+{"timestamp":1713397125.1042855,"name":"online","context":{"idset":"10320"}}
+{"timestamp":1713397125.3552651,"name":"online","context":{"idset":"10284,10299,10334,10349"}}
+{"timestamp":1713397125.4922948,"name":"online","context":{"idset":"10269,10324"}}
+{"timestamp":1713397125.6190932,"name":"online","context":{"idset":"10263,10301,10322,10336"}}
+{"timestamp":1713397125.6615095,"name":"online","context":{"idset":"10318"}}
+{"timestamp":1713397125.6896479,"name":"online","context":{"idset":"10174"}}
+{"timestamp":1713397125.7915463,"name":"online","context":{"idset":"10140,10218,10292"}}
+{"timestamp":1713397125.830622,"name":"online","context":{"idset":"10350"}}
+{"timestamp":1713397125.9771979,"name":"online","context":{"idset":"10276,10296,10351"}}
+{"timestamp":1713397126.2640886,"name":"online","context":{"idset":"10352"}}
+{"timestamp":1713397127.3099692,"name":"online","context":{"idset":"10353"}}
+{"timestamp":1713397127.6153138,"name":"online","context":{"idset":"10354-10355"}}
+{"timestamp":1713397127.864398,"name":"online","context":{"idset":"10356"}}
+{"timestamp":1713397128.0610807,"name":"online","context":{"idset":"10358"}}
+{"timestamp":1713397128.25372,"name":"online","context":{"idset":"10359"}}
+{"timestamp":1713397128.47453,"name":"online","context":{"idset":"10361"}}
+{"timestamp":1713397128.6733928,"name":"online","context":{"idset":"10363"}}
+{"timestamp":1713397128.8790267,"name":"online","context":{"idset":"10357,10362"}}
+{"timestamp":1713397129.1349275,"name":"online","context":{"idset":"10364"}}
+{"timestamp":1713397129.4970045,"name":"online","context":{"idset":"10366"}}
+{"timestamp":1713397129.7482166,"name":"online","context":{"idset":"10368"}}
+{"timestamp":1713397130.1963246,"name":"online","context":{"idset":"10369"}}
+{"timestamp":1713397130.4469934,"name":"online","context":{"idset":"10370-10371"}}
+{"timestamp":1713397131.4085751,"name":"online","context":{"idset":"10374"}}
+{"timestamp":1713397131.7434902,"name":"online","context":{"idset":"10375"}}
+{"timestamp":1713397132.0652881,"name":"online","context":{"idset":"10376"}}
+{"timestamp":1713397132.7213402,"name":"online","context":{"idset":"10377"}}
+{"timestamp":1713397133.0610955,"name":"online","context":{"idset":"10379"}}
+{"timestamp":1713397133.8500009,"name":"online","context":{"idset":"10385,10387"}}
+{"timestamp":1713397133.8933806,"name":"online","context":{"idset":"10384"}}
+{"timestamp":1713397133.9773397,"name":"online","context":{"idset":"10383"}}
+{"timestamp":1713397134.3361654,"name":"online","context":{"idset":"10389"}}
+{"timestamp":1713397134.4801826,"name":"online","context":{"idset":"10388"}}
+{"timestamp":1713397134.5472338,"name":"online","context":{"idset":"10381"}}
+{"timestamp":1713397134.916749,"name":"online","context":{"idset":"10393"}}
+{"timestamp":1713397135.0796614,"name":"online","context":{"idset":"10400"}}
+{"timestamp":1713397135.1131804,"name":"online","context":{"idset":"10398"}}
+{"timestamp":1713397135.1803892,"name":"online","context":{"idset":"10406"}}
+{"timestamp":1713397135.2400606,"name":"online","context":{"idset":"10392"}}
+{"timestamp":1713397135.3279297,"name":"online","context":{"idset":"10403"}}
+{"timestamp":1713397135.5185144,"name":"online","context":{"idset":"10402"}}
+{"timestamp":1713397135.6130679,"name":"online","context":{"idset":"10394,10404,10408"}}
+{"timestamp":1713397135.6581423,"name":"online","context":{"idset":"10397,10407"}}
+{"timestamp":1713397135.7612989,"name":"online","context":{"idset":"10416"}}
+{"timestamp":1713397135.8051033,"name":"online","context":{"idset":"10414"}}
+{"timestamp":1713397135.8529682,"name":"online","context":{"idset":"10426"}}
+{"timestamp":1713397135.9237247,"name":"online","context":{"idset":"10411"}}
+{"timestamp":1713397135.9708507,"name":"online","context":{"idset":"10412"}}
+{"timestamp":1713397136.03056,"name":"online","context":{"idset":"10419"}}
+{"timestamp":1713397136.1095629,"name":"online","context":{"idset":"10415"}}
+{"timestamp":1713397136.2606862,"name":"online","context":{"idset":"10423"}}
+{"timestamp":1713397136.2924509,"name":"online","context":{"idset":"10417"}}
+{"timestamp":1713397136.5343821,"name":"online","context":{"idset":"10429,10431,10437"}}
+{"timestamp":1713397136.5703011,"name":"online","context":{"idset":"10405,10421,10434"}}
+{"timestamp":1713397136.6241143,"name":"online","context":{"idset":"10428"}}
+{"timestamp":1713397136.7546699,"name":"online","context":{"idset":"10427,10430,10433,10436,10438,10441"}}
+{"timestamp":1713397136.8288634,"name":"online","context":{"idset":"10432,10443"}}
+{"timestamp":1713397136.9988174,"name":"online","context":{"idset":"10442,10455"}}
+{"timestamp":1713397137.191169,"name":"online","context":{"idset":"10435,10445,10447,10458"}}
+{"timestamp":1713397137.4350054,"name":"online","context":{"idset":"10440,10449,10461-10462"}}
+{"timestamp":1713397137.4799988,"name":"online","context":{"idset":"10439,10450-10451,10467"}}
+{"timestamp":1713397137.6752443,"name":"online","context":{"idset":"10446,10466"}}
+{"timestamp":1713397137.829695,"name":"online","context":{"idset":"10476,10488"}}
+{"timestamp":1713397138.0027235,"name":"online","context":{"idset":"10453,10463,10468,10487"}}
+{"timestamp":1713397138.0437794,"name":"online","context":{"idset":"10465,10473-10474,10483"}}
+{"timestamp":1713397138.0938435,"name":"online","context":{"idset":"10475,10493"}}
+{"timestamp":1713397138.2289305,"name":"online","context":{"idset":"10459,10478-10479,10492,10495,10501,10519"}}
+{"timestamp":1713397138.2982118,"name":"online","context":{"idset":"10469,10505,10520"}}
+{"timestamp":1713397138.4079545,"name":"online","context":{"idset":"10457,10470,10486,10490,10499,10506,10513"}}
+{"timestamp":1713397138.5383539,"name":"online","context":{"idset":"10472,10480,10482,10489,10497,10500,10502,10511,10516,10521"}}
+{"timestamp":1713397138.6105766,"name":"online","context":{"idset":"10504"}}
+{"timestamp":1713397138.6443152,"name":"online","context":{"idset":"10496,10510"}}
+{"timestamp":1713397138.7883162,"name":"online","context":{"idset":"10484,10498,10508,10512"}}
+{"timestamp":1713397138.8918769,"name":"online","context":{"idset":"10514"}}
+{"timestamp":1713397138.9947855,"name":"online","context":{"idset":"10523"}}
+{"timestamp":1713397139.2074521,"name":"online","context":{"idset":"10509,10524"}}
+{"timestamp":1713397139.5370443,"name":"online","context":{"idset":"10525"}}
+{"timestamp":1713397139.9699469,"name":"online","context":{"idset":"10526,10528"}}
+{"timestamp":1713397140.2212787,"name":"online","context":{"idset":"10530,10533-10534"}}
+{"timestamp":1713397140.6056817,"name":"online","context":{"idset":"10527"}}
+{"timestamp":1713397140.8169844,"name":"online","context":{"idset":"10535-10536"}}
+{"timestamp":1713397141.7127161,"name":"online","context":{"idset":"10539"}}
+{"timestamp":1713397142.0173078,"name":"online","context":{"idset":"10542"}}
+{"timestamp":1713397142.1722186,"name":"online","context":{"idset":"10538"}}
+{"timestamp":1713397142.3066998,"name":"online","context":{"idset":"10540-10541"}}
+{"timestamp":1713397143.1991699,"name":"online","context":{"idset":"10543"}}
+{"timestamp":1713397144.2148774,"name":"online","context":{"idset":"10547,10550"}}
+{"timestamp":1713397145.8673086,"name":"online","context":{"idset":"10553-10554"}}
+{"timestamp":1713397146.557425,"name":"online","context":{"idset":"10559"}}
+{"timestamp":1713397146.6062839,"name":"online","context":{"idset":"10545"}}
+{"timestamp":1713397146.830883,"name":"online","context":{"idset":"10556"}}
+{"timestamp":1713397146.9418938,"name":"online","context":{"idset":"10561"}}
+{"timestamp":1713397146.9925644,"name":"online","context":{"idset":"10560,10567"}}
+{"timestamp":1713397147.0306652,"name":"online","context":{"idset":"10565"}}
+{"timestamp":1713397147.3600216,"name":"online","context":{"idset":"10573"}}
+{"timestamp":1713397147.433547,"name":"online","context":{"idset":"10562,10564"}}
+{"timestamp":1713397147.4738348,"name":"online","context":{"idset":"10576,10581"}}
+{"timestamp":1713397147.6499572,"name":"online","context":{"idset":"10558,10577"}}
+{"timestamp":1713397147.70226,"name":"online","context":{"idset":"10586"}}
+{"timestamp":1713397147.9682035,"name":"online","context":{"idset":"10570,10575"}}
+{"timestamp":1713397148.1146803,"name":"online","context":{"idset":"10568,10579,10583-10585,10591"}}
+{"timestamp":1713397148.1633804,"name":"online","context":{"idset":"10589"}}
+{"timestamp":1713397148.2674584,"name":"online","context":{"idset":"10587,10601-10602"}}
+{"timestamp":1713397148.6849432,"name":"online","context":{"idset":"10555,10557,10590,10592-10593,10595,10599,10604,10606"}}
+{"timestamp":1713397148.8350792,"name":"online","context":{"idset":"10582,10588,10597,10605,10619"}}
+{"timestamp":1713397148.9287956,"name":"online","context":{"idset":"10596,10613"}}
+{"timestamp":1713397149.0017743,"name":"online","context":{"idset":"10603,10629,10631"}}
+{"timestamp":1713397149.0642934,"name":"online","context":{"idset":"10609,10627"}}
+{"timestamp":1713397149.1893218,"name":"online","context":{"idset":"10614,10635"}}
+{"timestamp":1713397149.3446171,"name":"online","context":{"idset":"10634"}}
+{"timestamp":1713397149.5340776,"name":"online","context":{"idset":"10632,10657,10667"}}
+{"timestamp":1713397149.6356394,"name":"online","context":{"idset":"10639"}}
+{"timestamp":1713397149.7424777,"name":"online","context":{"idset":"10658"}}
+{"timestamp":1713397149.8011549,"name":"online","context":{"idset":"10637"}}
+{"timestamp":1713397149.8924763,"name":"online","context":{"idset":"10643"}}
+{"timestamp":1713397149.9494717,"name":"online","context":{"idset":"10662"}}
+{"timestamp":1713397150.0025547,"name":"online","context":{"idset":"10668"}}
+{"timestamp":1713397150.3274574,"name":"online","context":{"idset":"10672"}}
+{"timestamp":1713397150.5208936,"name":"online","context":{"idset":"10666,10670"}}
+{"timestamp":1713397150.5727248,"name":"online","context":{"idset":"10669,10673,10676"}}
+{"timestamp":1713397150.9820645,"name":"online","context":{"idset":"10678"}}
+{"timestamp":1713397151.0380478,"name":"online","context":{"idset":"10671"}}
+{"timestamp":1713397151.1770172,"name":"online","context":{"idset":"10620,10677,10682,10685"}}
+{"timestamp":1713397151.3291547,"name":"online","context":{"idset":"10674-10675"}}
+{"timestamp":1713397151.4886262,"name":"online","context":{"idset":"10686,10694,10697"}}
+{"timestamp":1713397151.5317926,"name":"online","context":{"idset":"10693"}}
+{"timestamp":1713397151.7784595,"name":"online","context":{"idset":"10692,10713"}}
+{"timestamp":1713397151.8544483,"name":"online","context":{"idset":"10699,10706"}}
+{"timestamp":1713397151.9891553,"name":"online","context":{"idset":"10680,10689,10691,10705,10709"}}
+{"timestamp":1713397152.1223845,"name":"online","context":{"idset":"10679,10681,10687,10707,10712"}}
+{"timestamp":1713397152.2800405,"name":"online","context":{"idset":"10698,10708,10711,10715-10716"}}
+{"timestamp":1713397152.4720259,"name":"online","context":{"idset":"10700,10702,10704,10710,10718"}}
+{"timestamp":1713397152.5254965,"name":"online","context":{"idset":"10721,10726,10732"}}
+{"timestamp":1713397152.6733966,"name":"online","context":{"idset":"10719,10723"}}
+{"timestamp":1713397152.8054833,"name":"online","context":{"idset":"10714,10720,10724"}}
+{"timestamp":1713397152.9380949,"name":"online","context":{"idset":"10722,10731,10733"}}
+{"timestamp":1713397153.0951934,"name":"online","context":{"idset":"10735"}}
+{"timestamp":1713397153.2370517,"name":"online","context":{"idset":"10725,10727-10728,10734"}}
+{"timestamp":1713397153.6541123,"name":"online","context":{"idset":"10736-10737"}}
+{"timestamp":1713397153.9427776,"name":"online","context":{"idset":"10739"}}
+{"timestamp":1713397154.1405714,"name":"online","context":{"idset":"10738"}}
+{"timestamp":1713397154.5687206,"name":"online","context":{"idset":"10742"}}
+{"timestamp":1713397155.7843516,"name":"online","context":{"idset":"10744"}}
+{"timestamp":1713397156.3280377,"name":"online","context":{"idset":"10746"}}
+{"timestamp":1713397157.8558738,"name":"online","context":{"idset":"10748"}}
+{"timestamp":1713397158.2494154,"name":"online","context":{"idset":"10749"}}
+{"timestamp":1713397158.5262964,"name":"online","context":{"idset":"10750"}}
+{"timestamp":1713397158.7921484,"name":"online","context":{"idset":"10752,10754"}}
+{"timestamp":1713397158.8554022,"name":"online","context":{"idset":"10756,10758-10759"}}
+{"timestamp":1713397159.204052,"name":"online","context":{"idset":"10761,10763"}}
+{"timestamp":1713397159.2845895,"name":"online","context":{"idset":"10766"}}
+{"timestamp":1713397159.3326719,"name":"online","context":{"idset":"10768"}}
+{"timestamp":1713397159.5048389,"name":"online","context":{"idset":"10773"}}
+{"timestamp":1713397159.6524098,"name":"online","context":{"idset":"10769,10775"}}
+{"timestamp":1713397159.7460217,"name":"online","context":{"idset":"10762"}}
+{"timestamp":1713397159.8054824,"name":"online","context":{"idset":"10774,10776-10777"}}
+{"timestamp":1713397159.9467947,"name":"online","context":{"idset":"10771,10778"}}
+{"timestamp":1713397160.0103047,"name":"online","context":{"idset":"10780,10783"}}
+{"timestamp":1713397160.1478813,"name":"online","context":{"idset":"10779,10784"}}
+{"timestamp":1713397160.1941323,"name":"online","context":{"idset":"10782"}}
+{"timestamp":1713397160.2614539,"name":"online","context":{"idset":"10789"}}
+{"timestamp":1713397160.3596189,"name":"online","context":{"idset":"10785,10787,10794"}}
+{"timestamp":1713397160.5437341,"name":"online","context":{"idset":"10792"}}
+{"timestamp":1713397160.6892242,"name":"online","context":{"idset":"10791,10798"}}
+{"timestamp":1713397160.8326583,"name":"online","context":{"idset":"10788,10797,10802,10806"}}
+{"timestamp":1713397160.8931854,"name":"online","context":{"idset":"10795,10803,10807"}}
+{"timestamp":1713397161.0782096,"name":"online","context":{"idset":"10786,10799,10804"}}
+{"timestamp":1713397161.2686932,"name":"online","context":{"idset":"10800"}}
+{"timestamp":1713397161.4219441,"name":"online","context":{"idset":"10809,10813-10814,10819,10823"}}
+{"timestamp":1713397161.6959238,"name":"online","context":{"idset":"10808,10810,10816-10817,10828,10831"}}
+{"timestamp":1713397161.7540278,"name":"online","context":{"idset":"10801,10829-10830"}}
+{"timestamp":1713397161.9662976,"name":"online","context":{"idset":"10818,10821,10827,10834"}}
+{"timestamp":1713397162.0926926,"name":"online","context":{"idset":"10836"}}
+{"timestamp":1713397162.1319287,"name":"online","context":{"idset":"10833,10837"}}
+{"timestamp":1713397162.3420863,"name":"online","context":{"idset":"10842"}}
+{"timestamp":1713397162.4280772,"name":"online","context":{"idset":"10838"}}
+{"timestamp":1713397162.4914076,"name":"online","context":{"idset":"10839"}}
+{"timestamp":1713397162.8663311,"name":"online","context":{"idset":"10844"}}
+{"timestamp":1713397163.0352447,"name":"online","context":{"idset":"10843"}}
+{"timestamp":1713397163.0816109,"name":"online","context":{"idset":"10848"}}
+{"timestamp":1713397163.1229808,"name":"online","context":{"idset":"10846"}}
+{"timestamp":1713397163.3012164,"name":"online","context":{"idset":"10845,10847,10849,10853-10854"}}
+{"timestamp":1713397163.4874072,"name":"online","context":{"idset":"10851"}}
+{"timestamp":1713397163.5407581,"name":"online","context":{"idset":"10852"}}
+{"timestamp":1713397163.5832715,"name":"online","context":{"idset":"10866"}}
+{"timestamp":1713397163.6228426,"name":"online","context":{"idset":"10850,10868"}}
+{"timestamp":1713397163.7460132,"name":"online","context":{"idset":"10857,10865"}}
+{"timestamp":1713397163.8914053,"name":"online","context":{"idset":"10855,10864"}}
+{"timestamp":1713397164.0736845,"name":"online","context":{"idset":"10863"}}
+{"timestamp":1713397164.2800839,"name":"online","context":{"idset":"10859,10867,10869,10874"}}
+{"timestamp":1713397164.4252932,"name":"online","context":{"idset":"10875,10877"}}
+{"timestamp":1713397164.5241592,"name":"online","context":{"idset":"10871,10880-10881"}}
+{"timestamp":1713397164.6587727,"name":"online","context":{"idset":"10872,10878,10886,10888"}}
+{"timestamp":1713397164.8568141,"name":"online","context":{"idset":"10883"}}
+{"timestamp":1713397164.9599874,"name":"online","context":{"idset":"10885,10907"}}
+{"timestamp":1713397165.0600288,"name":"online","context":{"idset":"10890,10892-10893,10899,10904"}}
+{"timestamp":1713397165.1977687,"name":"online","context":{"idset":"10889"}}
+{"timestamp":1713397165.3536794,"name":"online","context":{"idset":"10882,10901,10905,10910-10911"}}
+{"timestamp":1713397165.5638242,"name":"online","context":{"idset":"10908"}}
+{"timestamp":1713397165.7180016,"name":"online","context":{"idset":"10902,10912,10914,10919,10921-10922"}}
+{"timestamp":1713397165.8957429,"name":"online","context":{"idset":"10913,10917"}}
+{"timestamp":1713397166.254051,"name":"online","context":{"idset":"10923"}}
+{"timestamp":1713397166.7368481,"name":"online","context":{"idset":"10924"}}
+{"timestamp":1713397167.1861968,"name":"online","context":{"idset":"10925"}}
+{"timestamp":1713397168.0320587,"name":"online","context":{"idset":"10927"}}
+{"timestamp":1713397168.1354928,"name":"online","context":{"idset":"10928"}}
+{"timestamp":1713397170.0459056,"name":"online","context":{"idset":"10931"}}
+{"timestamp":1713397170.5140126,"name":"online","context":{"idset":"10932"}}
+{"timestamp":1713397170.6053114,"name":"online","context":{"idset":"10935"}}
+{"timestamp":1713397170.7558129,"name":"online","context":{"idset":"10936,10939"}}
+{"timestamp":1713397170.8741908,"name":"online","context":{"idset":"10930"}}
+{"timestamp":1713397170.944411,"name":"online","context":{"idset":"10940-10941"}}
+{"timestamp":1713397170.9868932,"name":"online","context":{"idset":"10938,10942"}}
+{"timestamp":1713397171.1841536,"name":"online","context":{"idset":"10944-10945"}}
+{"timestamp":1713397171.4460227,"name":"online","context":{"idset":"10951"}}
+{"timestamp":1713397171.5026255,"name":"online","context":{"idset":"10948"}}
+{"timestamp":1713397171.6107922,"name":"online","context":{"idset":"10943"}}
+{"timestamp":1713397171.6779473,"name":"online","context":{"idset":"10953"}}
+{"timestamp":1713397171.7599564,"name":"online","context":{"idset":"10947,10949"}}
+{"timestamp":1713397172.1027818,"name":"online","context":{"idset":"10952"}}
+{"timestamp":1713397172.1563497,"name":"online","context":{"idset":"10950,10970"}}
+{"timestamp":1713397172.2203484,"name":"online","context":{"idset":"10969"}}
+{"timestamp":1713397172.3259923,"name":"online","context":{"idset":"10958,10963"}}
+{"timestamp":1713397172.3695953,"name":"online","context":{"idset":"10971"}}
+{"timestamp":1713397172.5233779,"name":"online","context":{"idset":"10954,10956,10974"}}
+{"timestamp":1713397172.6016777,"name":"online","context":{"idset":"10962,10966"}}
+{"timestamp":1713397172.7535365,"name":"online","context":{"idset":"10964,10975,10987"}}
+{"timestamp":1713397173.0284164,"name":"online","context":{"idset":"10976,10979-10980,10982,10985"}}
+{"timestamp":1713397173.1694086,"name":"online","context":{"idset":"10978"}}
+{"timestamp":1713397173.4150474,"name":"online","context":{"idset":"10984,10990,11001,11003"}}
+{"timestamp":1713397173.494556,"name":"online","context":{"idset":"11000"}}
+{"timestamp":1713397173.549294,"name":"online","context":{"idset":"10988,10996"}}
+{"timestamp":1713397173.6914785,"name":"online","context":{"idset":"10994,11006"}}
+{"timestamp":1713397173.7567501,"name":"online","context":{"idset":"11005"}}
+{"timestamp":1713397173.8531427,"name":"online","context":{"idset":"10991"}}
+{"timestamp":1713397173.9472792,"name":"online","context":{"idset":"10995,11008,11013"}}
+{"timestamp":1713397174.0567658,"name":"online","context":{"idset":"11018"}}
+{"timestamp":1713397174.1353557,"name":"online","context":{"idset":"10997,10999,11004,11012,11015"}}
+{"timestamp":1713397174.333209,"name":"online","context":{"idset":"11007,11010,11021"}}
+{"timestamp":1713397174.3985777,"name":"online","context":{"idset":"11014"}}
+{"timestamp":1713397174.4554067,"name":"online","context":{"idset":"11009"}}
+{"timestamp":1713397174.5258443,"name":"online","context":{"idset":"11011,11020"}}
+{"timestamp":1713397174.6999974,"name":"online","context":{"idset":"11022"}}
+{"timestamp":1713397174.8484929,"name":"online","context":{"idset":"11024,11026"}}
+{"timestamp":1713397174.9557796,"name":"online","context":{"idset":"11025,11027"}}
+{"timestamp":1713397175.1123586,"name":"online","context":{"idset":"11029"}}
+{"timestamp":1713397175.2377284,"name":"online","context":{"idset":"11032"}}
+{"timestamp":1713397175.3379874,"name":"online","context":{"idset":"11035"}}
+{"timestamp":1713397175.4015477,"name":"online","context":{"idset":"11037"}}
+{"timestamp":1713397175.4567418,"name":"online","context":{"idset":"11030"}}
+{"timestamp":1713397175.594542,"name":"online","context":{"idset":"11038,11043"}}
+{"timestamp":1713397175.7504435,"name":"online","context":{"idset":"11040"}}
+{"timestamp":1713397175.9719954,"name":"online","context":{"idset":"11031,11042,11047"}}
+{"timestamp":1713397176.0777035,"name":"online","context":{"idset":"11039,11045,11058"}}
+{"timestamp":1713397176.2193348,"name":"online","context":{"idset":"11034,11044,11051,11055"}}
+{"timestamp":1713397176.3705058,"name":"online","context":{"idset":"11049,11053,11059,11061"}}
+{"timestamp":1713397176.4971361,"name":"online","context":{"idset":"11063"}}
+{"timestamp":1713397176.6657741,"name":"online","context":{"idset":"11054,11060,11062,11065"}}
+{"timestamp":1713397176.8912895,"name":"online","context":{"idset":"11067"}}
+{"timestamp":1713397177.0540562,"name":"online","context":{"idset":"11066"}}
+{"timestamp":1713397177.20717,"name":"online","context":{"idset":"11068-11069,11073,11093"}}
+{"timestamp":1713397177.2912858,"name":"online","context":{"idset":"11072,11079,11081"}}
+{"timestamp":1713397177.428221,"name":"online","context":{"idset":"11064,11071,11074,11077"}}
+{"timestamp":1713397177.5307052,"name":"online","context":{"idset":"11075,11083"}}
+{"timestamp":1713397177.671587,"name":"online","context":{"idset":"11078,11082,11086-11088"}}
+{"timestamp":1713397177.7762029,"name":"online","context":{"idset":"11084-11085,11094"}}
+{"timestamp":1713397178.0526192,"name":"online","context":{"idset":"11091"}}
+{"timestamp":1713397178.2874527,"name":"online","context":{"idset":"11096"}}
+{"timestamp":1713397178.5142014,"name":"online","context":{"idset":"11089"}}
+{"timestamp":1713397178.751761,"name":"online","context":{"idset":"11098"}}
+{"timestamp":1713397179.0593665,"name":"online","context":{"idset":"11100"}}
+{"timestamp":1713397179.403435,"name":"online","context":{"idset":"11102"}}
+{"timestamp":1713397180.1940422,"name":"online","context":{"idset":"11103"}}
+{"timestamp":1713397182.1102004,"name":"online","context":{"idset":"11105"}}
+{"timestamp":1713397182.5537927,"name":"online","context":{"idset":"11104,11109-11110"}}
+{"timestamp":1713397182.8060777,"name":"online","context":{"idset":"11108"}}
+{"timestamp":1713397183.0023787,"name":"online","context":{"idset":"11113"}}
+{"timestamp":1713397183.0567548,"name":"online","context":{"idset":"11115"}}
+{"timestamp":1713397183.2603607,"name":"online","context":{"idset":"11111,11114"}}
+{"timestamp":1713397183.3085256,"name":"online","context":{"idset":"11118"}}
+{"timestamp":1713397183.4010146,"name":"online","context":{"idset":"11112"}}
+{"timestamp":1713397183.5311401,"name":"online","context":{"idset":"11116"}}
+{"timestamp":1713397183.6241148,"name":"online","context":{"idset":"11122"}}
+{"timestamp":1713397183.6810317,"name":"online","context":{"idset":"11121"}}
+{"timestamp":1713397183.8184481,"name":"online","context":{"idset":"11119-11120"}}
+{"timestamp":1713397183.9197257,"name":"online","context":{"idset":"11117"}}
+{"timestamp":1713397184.0600262,"name":"online","context":{"idset":"11123-11124,11132"}}
+{"timestamp":1713397184.1128454,"name":"online","context":{"idset":"11126"}}
+{"timestamp":1713397184.2023699,"name":"online","context":{"idset":"11139"}}
+{"timestamp":1713397184.3618338,"name":"online","context":{"idset":"11127,11131,11142"}}
+{"timestamp":1713397184.5556421,"name":"online","context":{"idset":"11133,11143"}}
+{"timestamp":1713397184.6014633,"name":"online","context":{"idset":"11125,11135,11141"}}
+{"timestamp":1713397184.7347796,"name":"online","context":{"idset":"11137"}}
+{"timestamp":1713397184.8204401,"name":"online","context":{"idset":"11145,11151"}}
+{"timestamp":1713397184.9074693,"name":"online","context":{"idset":"11147-11148,11152"}}
+{"timestamp":1713397185.0005221,"name":"online","context":{"idset":"11154"}}
+{"timestamp":1713397185.0869174,"name":"online","context":{"idset":"11155"}}
+{"timestamp":1713397185.7615955,"name":"online","context":{"idset":"11156"}}
+{"timestamp":1713397185.7643609,"name":"online","context":{"idset":"11149,11157"}}
+{"timestamp":1713397185.7672496,"name":"online","context":{"idset":"11159"}}
+{"timestamp":1713397185.7692769,"name":"online","context":{"idset":"11160"}}
+{"timestamp":1713397185.7711828,"name":"online","context":{"idset":"11163-11164,11166-11167"}}
+{"timestamp":1713397185.7735023,"name":"online","context":{"idset":"11170,11176"}}
+{"timestamp":1713397185.8662398,"name":"online","context":{"idset":"11165"}}
+{"timestamp":1713397185.9060152,"name":"online","context":{"idset":"11174"}}
+{"timestamp":1713397185.9417293,"name":"online","context":{"idset":"11169"}}
+{"timestamp":1713397185.9933395,"name":"online","context":{"idset":"11162,11168,11178"}}
+{"timestamp":1713397186.0703316,"name":"online","context":{"idset":"11172-11173"}}
+{"timestamp":1713397186.2529008,"name":"online","context":{"idset":"11171,11179"}}
+{"timestamp":1713397186.5304196,"name":"online","context":{"idset":"11175,11177,11181"}}
+{"timestamp":1713397186.6481268,"name":"online","context":{"idset":"11186"}}
+{"timestamp":1713397186.8290424,"name":"online","context":{"idset":"11183,11189-11190"}}
+{"timestamp":1713397186.9033375,"name":"online","context":{"idset":"11187"}}
+{"timestamp":1713397187.0650194,"name":"online","context":{"idset":"11180,11185"}}
+{"timestamp":1713397187.2109933,"name":"online","context":{"idset":"11191"}}
+{"timestamp":1713397187.2720373,"name":"online","context":{"idset":"11193-11194,11198"}}
+{"timestamp":1713397187.486342,"name":"online","context":{"idset":"11196-11197,11203"}}
+{"timestamp":1713397187.601728,"name":"online","context":{"idset":"11213"}}
+{"timestamp":1713397187.6475701,"name":"online","context":{"idset":"11200"}}
+{"timestamp":1713397187.6986716,"name":"online","context":{"idset":"11205"}}
+{"timestamp":1713397187.7626719,"name":"online","context":{"idset":"11206"}}
+{"timestamp":1713397187.9408219,"name":"online","context":{"idset":"11216,11219"}}
+{"timestamp":1713397188.0844786,"name":"online","context":{"idset":"11204,11214"}}
+{"timestamp":1713397188.215414,"name":"online","context":{"idset":"11207,11211,11220"}}
+{"timestamp":1713397188.261734,"name":"online","context":{"idset":"11221"}}
+{"timestamp":1713397188.3180711,"name":"online","context":{"idset":"11199,11224"}}
+{"timestamp":1713397188.4472497,"name":"online","context":{"idset":"11218"}}
+{"timestamp":1713397188.5910947,"name":"online","context":{"idset":"11217,11226,11229"}}
+{"timestamp":1713397188.7039382,"name":"online","context":{"idset":"11234,11236"}}
+{"timestamp":1713397188.978828,"name":"online","context":{"idset":"11231,11233,11237"}}
+{"timestamp":1713397189.1466763,"name":"online","context":{"idset":"11227,11235"}}
+{"timestamp":1713397189.2300386,"name":"online","context":{"idset":"11244"}}
+{"timestamp":1713397189.3701985,"name":"online","context":{"idset":"11240,11245"}}
+{"timestamp":1713397189.5125175,"name":"online","context":{"idset":"11239,11246,11253-11254"}}
+{"timestamp":1713397189.7051611,"name":"online","context":{"idset":"11242,11247-11248"}}
+{"timestamp":1713397189.8288751,"name":"online","context":{"idset":"11241,11255"}}
+{"timestamp":1713397189.9768548,"name":"online","context":{"idset":"11249,11251,11259,11264-11265"}}
+{"timestamp":1713397190.3383818,"name":"online","context":{"idset":"11243,11252,11266"}}
+{"timestamp":1713397190.4436042,"name":"online","context":{"idset":"11263"}}
+{"timestamp":1713397190.6052637,"name":"online","context":{"idset":"11261"}}
+{"timestamp":1713397190.8866394,"name":"online","context":{"idset":"11267-11268"}}
+{"timestamp":1713397191.4076512,"name":"online","context":{"idset":"11269"}}
+{"timestamp":1713397192.0382411,"name":"online","context":{"idset":"11272"}}
+{"timestamp":1713397192.6429546,"name":"online","context":{"idset":"11273"}}
+{"timestamp":1713397194.3944607,"name":"online","context":{"idset":"11278-11279"}}
+{"timestamp":1713397194.7625093,"name":"online","context":{"idset":"11283"}}
+{"timestamp":1713397194.8153708,"name":"online","context":{"idset":"11282"}}
+{"timestamp":1713397194.927274,"name":"online","context":{"idset":"11275-11276,11280-11281"}}
+{"timestamp":1713397195.0891087,"name":"online","context":{"idset":"11287"}}
+{"timestamp":1713397195.1807268,"name":"online","context":{"idset":"11286"}}
+{"timestamp":1713397195.3557155,"name":"online","context":{"idset":"11285"}}
+{"timestamp":1713397195.4096773,"name":"online","context":{"idset":"11289"}}
+{"timestamp":1713397195.5574474,"name":"online","context":{"idset":"11288,11291,11296"}}
+{"timestamp":1713397195.7117374,"name":"online","context":{"idset":"11290"}}
+{"timestamp":1713397195.967485,"name":"online","context":{"idset":"11293"}}
+{"timestamp":1713397196.1153753,"name":"online","context":{"idset":"11300,11304-11305,11308"}}
+{"timestamp":1713397196.2957923,"name":"online","context":{"idset":"11298,11301"}}
+{"timestamp":1713397196.4607389,"name":"online","context":{"idset":"11294"}}
+{"timestamp":1713397196.5128772,"name":"online","context":{"idset":"11292,11297"}}
+{"timestamp":1713397196.5607984,"name":"online","context":{"idset":"11317"}}
+{"timestamp":1713397196.6216502,"name":"online","context":{"idset":"11310"}}
+{"timestamp":1713397196.6731749,"name":"online","context":{"idset":"11309,11311,11313"}}
+{"timestamp":1713397196.864114,"name":"online","context":{"idset":"11307,11318-11319"}}
+{"timestamp":1713397197.0192413,"name":"online","context":{"idset":"11312,11314,11327"}}
+{"timestamp":1713397197.304395,"name":"online","context":{"idset":"11328,11331,11333,11338"}}
+{"timestamp":1713397197.580313,"name":"online","context":{"idset":"11335-11336"}}
+{"timestamp":1713397197.6658299,"name":"online","context":{"idset":"11345"}}
+{"timestamp":1713397197.7577398,"name":"online","context":{"idset":"11315,11329"}}
+{"timestamp":1713397197.9605315,"name":"online","context":{"idset":"11342"}}
+{"timestamp":1713397198.0320392,"name":"online","context":{"idset":"11330,11332,11339"}}
+{"timestamp":1713397198.1060085,"name":"online","context":{"idset":"11350"}}
+{"timestamp":1713397198.2093437,"name":"online","context":{"idset":"11340-11341,11351,11354"}}
+{"timestamp":1713397198.2662566,"name":"online","context":{"idset":"11343,11347"}}
+{"timestamp":1713397198.4149199,"name":"online","context":{"idset":"11344,11346,11348-11349"}}
+{"timestamp":1713397198.6163998,"name":"online","context":{"idset":"11353,11356,11368"}}
+{"timestamp":1713397198.6696177,"name":"online","context":{"idset":"11359"}}
+{"timestamp":1713397198.8512144,"name":"online","context":{"idset":"11355,11360,11363,11366"}}
+{"timestamp":1713397198.9558859,"name":"online","context":{"idset":"11358,11362"}}
+{"timestamp":1713397199.1859701,"name":"online","context":{"idset":"11371"}}
+{"timestamp":1713397199.3473995,"name":"online","context":{"idset":"11367,11370"}}
+{"timestamp":1713397199.4267609,"name":"online","context":{"idset":"11369"}}
+{"timestamp":1713397199.6191518,"name":"online","context":{"idset":"11372"}}
+{"timestamp":1713397199.7620206,"name":"online","context":{"idset":"11373,11379"}}
+{"timestamp":1713397199.8845689,"name":"online","context":{"idset":"11376-11378,11382"}}
+{"timestamp":1713397200.0622585,"name":"online","context":{"idset":"11384"}}
+{"timestamp":1713397200.2332172,"name":"online","context":{"idset":"11380"}}
+{"timestamp":1713397200.2893388,"name":"online","context":{"idset":"11381"}}
+{"timestamp":1713397200.3449135,"name":"online","context":{"idset":"11374,11386"}}
+{"timestamp":1713397200.4762578,"name":"online","context":{"idset":"11383"}}
+{"timestamp":1713397200.5596023,"name":"online","context":{"idset":"11390"}}
+{"timestamp":1713397200.6985867,"name":"online","context":{"idset":"11387"}}
+{"timestamp":1713397200.8485074,"name":"online","context":{"idset":"11388-11389,11391,11393-11395"}}
+{"timestamp":1713397200.950022,"name":"online","context":{"idset":"11392"}}
+{"timestamp":1713397201.0022144,"name":"online","context":{"idset":"11400"}}
+{"timestamp":1713397201.0684462,"name":"online","context":{"idset":"11398"}}
+{"timestamp":1713397201.2374907,"name":"online","context":{"idset":"11399,11401-11402,11406"}}
+{"timestamp":1713397201.4543116,"name":"online","context":{"idset":"11403,11412"}}
+{"timestamp":1713397201.614011,"name":"online","context":{"idset":"11413"}}
+{"timestamp":1713397201.7830586,"name":"online","context":{"idset":"11407,11420"}}
+{"timestamp":1713397201.9817529,"name":"online","context":{"idset":"11404,11408"}}
+{"timestamp":1713397202.247951,"name":"online","context":{"idset":"11414,11417,11421,11423"}}
+{"timestamp":1713397202.425221,"name":"online","context":{"idset":"11416"}}
+{"timestamp":1713397202.6954498,"name":"online","context":{"idset":"11418,11422,11424,11426,11430"}}
+{"timestamp":1713397202.7807887,"name":"online","context":{"idset":"11431"}}
+{"timestamp":1713397203.2190163,"name":"online","context":{"idset":"11397,11427,11432"}}
+{"timestamp":1713397204.0004556,"name":"online","context":{"idset":"11411,11415"}}
+{"timestamp":1713397204.6005197,"name":"online","context":{"idset":"11434"}}
+{"timestamp":1713397204.9852571,"name":"online","context":{"idset":"11433"}}
+{"timestamp":1713397206.3446267,"name":"online","context":{"idset":"11435,11437"}}
+{"timestamp":1713397206.4524505,"name":"online","context":{"idset":"11436"}}
+{"timestamp":1713397206.6691973,"name":"online","context":{"idset":"11440"}}
+{"timestamp":1713397206.7780061,"name":"online","context":{"idset":"11438"}}
+{"timestamp":1713397206.8499019,"name":"online","context":{"idset":"11441"}}
+{"timestamp":1713397206.917738,"name":"online","context":{"idset":"11444"}}
+{"timestamp":1713397207.2241006,"name":"online","context":{"idset":"11445,11449"}}
+{"timestamp":1713397207.3477037,"name":"online","context":{"idset":"11448,11451"}}
+{"timestamp":1713397207.4129434,"name":"online","context":{"idset":"11446,11450"}}
+{"timestamp":1713397207.5825219,"name":"online","context":{"idset":"11447,11454"}}
+{"timestamp":1713397207.7881515,"name":"online","context":{"idset":"11455"}}
+{"timestamp":1713397207.9429226,"name":"online","context":{"idset":"11453,11456"}}
+{"timestamp":1713397208.0647671,"name":"online","context":{"idset":"11459"}}
+{"timestamp":1713397208.1987922,"name":"online","context":{"idset":"11457,11464"}}
+{"timestamp":1713397208.4144611,"name":"online","context":{"idset":"11461"}}
+{"timestamp":1713397208.5513971,"name":"online","context":{"idset":"11466"}}
+{"timestamp":1713397208.737643,"name":"online","context":{"idset":"11463,11468"}}
+{"timestamp":1713397208.8942556,"name":"online","context":{"idset":"11460,11462,11465,11472"}}
+{"timestamp":1713397208.9674025,"name":"online","context":{"idset":"11473,11478"}}
+{"timestamp":1713397209.0236325,"name":"online","context":{"idset":"11471,11482"}}
+{"timestamp":1713397209.1068609,"name":"online","context":{"idset":"11470,11475"}}
+{"timestamp":1713397209.1601107,"name":"online","context":{"idset":"11474"}}
+{"timestamp":1713397209.3354592,"name":"online","context":{"idset":"11467,11484-11485"}}
+{"timestamp":1713397209.4835405,"name":"online","context":{"idset":"11479-11480,11483,11486,11491"}}
+{"timestamp":1713397209.6306794,"name":"online","context":{"idset":"11494"}}
+{"timestamp":1713397209.7018094,"name":"online","context":{"idset":"11487"}}
+{"timestamp":1713397209.7878282,"name":"online","context":{"idset":"11493,11500"}}
+{"timestamp":1713397209.9044323,"name":"online","context":{"idset":"11497"}}
+{"timestamp":1713397210.0621481,"name":"online","context":{"idset":"11505"}}
+{"timestamp":1713397210.1633501,"name":"online","context":{"idset":"11490,11495,11506,11508"}}
+{"timestamp":1713397210.2347896,"name":"online","context":{"idset":"11488,11503"}}
+{"timestamp":1713397210.3419373,"name":"online","context":{"idset":"11510"}}
+{"timestamp":1713397210.5847847,"name":"online","context":{"idset":"11501,11507"}}
+{"timestamp":1713397210.6675172,"name":"online","context":{"idset":"11513"}}
+{"timestamp":1713397210.7134051,"name":"online","context":{"idset":"11514"}}
+{"timestamp":1713397210.7921798,"name":"online","context":{"idset":"11511,11515,11520"}}
+{"timestamp":1713397210.945936,"name":"online","context":{"idset":"11518,11522"}}
+{"timestamp":1713397211.1404288,"name":"online","context":{"idset":"11524"}}
+{"timestamp":1713397211.2316554,"name":"online","context":{"idset":"11509,11517,11523,11527"}}
+{"timestamp":1713397211.2850523,"name":"online","context":{"idset":"11530"}}
+{"timestamp":1713397211.4276009,"name":"online","context":{"idset":"11519,11521,11529"}}
+{"timestamp":1713397211.5742874,"name":"online","context":{"idset":"11526"}}
+{"timestamp":1713397211.7633078,"name":"online","context":{"idset":"11537"}}
+{"timestamp":1713397211.824084,"name":"online","context":{"idset":"11531,11535"}}
+{"timestamp":1713397212.0026944,"name":"online","context":{"idset":"11533"}}
+{"timestamp":1713397212.0590243,"name":"online","context":{"idset":"11546"}}
+{"timestamp":1713397212.2014883,"name":"online","context":{"idset":"11532,11534"}}
+{"timestamp":1713397212.2744164,"name":"online","context":{"idset":"11536,11538,11540"}}
+{"timestamp":1713397212.3394802,"name":"online","context":{"idset":"11547"}}
+{"timestamp":1713397212.4476683,"name":"online","context":{"idset":"11544-11545"}}
+{"timestamp":1713397212.5888813,"name":"online","context":{"idset":"11542,11551"}}
+{"timestamp":1713397212.6532257,"name":"online","context":{"idset":"11550"}}
+{"timestamp":1713397212.7417822,"name":"online","context":{"idset":"11539"}}
+{"timestamp":1713397212.8264854,"name":"online","context":{"idset":"11541,11552"}}
+{"timestamp":1713397212.9798403,"name":"online","context":{"idset":"11548"}}
+{"timestamp":1713397213.2089841,"name":"online","context":{"idset":"11554-11555,11557"}}
+{"timestamp":1713397213.4795434,"name":"online","context":{"idset":"11558,11560-11561,11563"}}
+{"timestamp":1713397213.6977651,"name":"online","context":{"idset":"11559,11562"}}
+{"timestamp":1713397213.9259887,"name":"online","context":{"idset":"11556,11564,11572"}}
+{"timestamp":1713397214.0379748,"name":"online","context":{"idset":"11567,11570-11571,11574"}}
+{"timestamp":1713397214.2213552,"name":"online","context":{"idset":"11566"}}
+{"timestamp":1713397214.46527,"name":"online","context":{"idset":"11573,11575-11576"}}
+{"timestamp":1713397214.6190195,"name":"online","context":{"idset":"11577-11578,11586"}}
+{"timestamp":1713397214.8582344,"name":"online","context":{"idset":"11581,11583"}}
+{"timestamp":1713397215.064405,"name":"online","context":{"idset":"11579,11584"}}
+{"timestamp":1713397215.2148814,"name":"online","context":{"idset":"11588"}}
+{"timestamp":1713397215.8573825,"name":"online","context":{"idset":"11591-11592"}}
+{"timestamp":1713397216.3346114,"name":"online","context":{"idset":"11593-11594"}}
+{"timestamp":1713397218.0935543,"name":"online","context":{"idset":"11595"}}
+{"timestamp":1713397218.2372594,"name":"online","context":{"idset":"11596"}}
+{"timestamp":1713397218.4756901,"name":"online","context":{"idset":"11597"}}
+{"timestamp":1713397218.6748974,"name":"online","context":{"idset":"11598,11606"}}
+{"timestamp":1713397218.9146745,"name":"online","context":{"idset":"11603"}}
+{"timestamp":1713397219.0186894,"name":"online","context":{"idset":"11607"}}
+{"timestamp":1713397219.1641879,"name":"online","context":{"idset":"11605,11609-11610"}}
+{"timestamp":1713397219.419858,"name":"online","context":{"idset":"11611"}}
+{"timestamp":1713397219.5660901,"name":"online","context":{"idset":"11612-11613"}}
+{"timestamp":1713397219.7671108,"name":"online","context":{"idset":"11615-11618,11620-11621"}}
+{"timestamp":1713397220.2104754,"name":"online","context":{"idset":"11622-11623"}}
+{"timestamp":1713397220.3642845,"name":"online","context":{"idset":"11626"}}
+{"timestamp":1713397220.5451441,"name":"online","context":{"idset":"11631"}}
+{"timestamp":1713397220.694947,"name":"online","context":{"idset":"11625,11627-11628,11632"}}
+{"timestamp":1713397220.9891503,"name":"online","context":{"idset":"11633"}}
+{"timestamp":1713397222.4742858,"name":"online","context":{"idset":"11653"}}
+{"timestamp":1713397223.2834303,"name":"online","context":{"idset":"11677"}}
+{"timestamp":1713397223.3498909,"name":"online","context":{"idset":"11674"}}
+{"timestamp":1713397223.9089425,"name":"online","context":{"idset":"11652"}}
+{"timestamp":1713397224.6358783,"name":"online","context":{"idset":"11638,11661,11678"}}
+{"timestamp":1713397224.6932144,"name":"online","context":{"idset":"11667"}}
+{"timestamp":1713397224.7530949,"name":"online","context":{"idset":"11679"}}
+{"timestamp":1713397224.8556192,"name":"online","context":{"idset":"11675"}}
+{"timestamp":1713397225.001338,"name":"online","context":{"idset":"11640,11668"}}
+{"timestamp":1713397225.2066541,"name":"online","context":{"idset":"11655,11666,11697"}}
+{"timestamp":1713397225.3612905,"name":"online","context":{"idset":"11671"}}
+{"timestamp":1713397225.4275794,"name":"online","context":{"idset":"11665"}}
+{"timestamp":1713397225.6630599,"name":"online","context":{"idset":"11737"}}
+{"timestamp":1713397225.756041,"name":"online","context":{"idset":"11694"}}
+{"timestamp":1713397225.8055496,"name":"online","context":{"idset":"11657,11690"}}
+{"timestamp":1713397226.0994277,"name":"online","context":{"idset":"11681"}}
+{"timestamp":1713397226.310647,"name":"online","context":{"idset":"11651,11660,11685-11686,11728"}}
+{"timestamp":1713397226.4050157,"name":"online","context":{"idset":"11684"}}
+{"timestamp":1713397226.6575584,"name":"online","context":{"idset":"11701"}}
+{"timestamp":1713397226.8513229,"name":"online","context":{"idset":"11670,11698"}}
+{"timestamp":1713397226.9500971,"name":"online","context":{"idset":"11643"}}
+{"timestamp":1713397227.2379448,"name":"online","context":{"idset":"11726"}}
+{"timestamp":1713397227.3703382,"name":"online","context":{"idset":"11732"}}
+{"timestamp":1713397227.5279627,"name":"online","context":{"idset":"11641,11673,11720,11754"}}
+{"timestamp":1713397227.7468696,"name":"online","context":{"idset":"11710"}}
+{"timestamp":1713397227.8045037,"name":"online","context":{"idset":"11707,11711"}}
+{"timestamp":1713397227.9581838,"name":"online","context":{"idset":"11650,11700,11731"}}
+{"timestamp":1713397228.1515501,"name":"online","context":{"idset":"11687"}}
+{"timestamp":1713397228.2108483,"name":"online","context":{"idset":"11680,11683"}}
+{"timestamp":1713397228.3345103,"name":"online","context":{"idset":"11730"}}
+{"timestamp":1713397228.430402,"name":"online","context":{"idset":"11646"}}
+{"timestamp":1713397228.5148206,"name":"online","context":{"idset":"11637"}}
+{"timestamp":1713397228.6389585,"name":"online","context":{"idset":"11693,11703"}}
+{"timestamp":1713397228.7945371,"name":"online","context":{"idset":"11662,11739"}}
+{"timestamp":1713397228.9563379,"name":"online","context":{"idset":"11713"}}
+{"timestamp":1713397229.2075555,"name":"online","context":{"idset":"11757"}}
+{"timestamp":1713397229.354553,"name":"online","context":{"idset":"11642,11689"}}
+{"timestamp":1713397229.4176788,"name":"online","context":{"idset":"11648"}}
+{"timestamp":1713397229.6517515,"name":"online","context":{"idset":"11708,11745"}}
+{"timestamp":1713397229.7603354,"name":"online","context":{"idset":"11645,11709"}}
+{"timestamp":1713397229.9546471,"name":"online","context":{"idset":"11733,11747"}}
+{"timestamp":1713397230.0120456,"name":"online","context":{"idset":"11719"}}
+{"timestamp":1713397230.16608,"name":"online","context":{"idset":"11705,11714,11764"}}
+{"timestamp":1713397230.2498507,"name":"online","context":{"idset":"11699"}}
+{"timestamp":1713397230.3966739,"name":"online","context":{"idset":"11743"}}
+{"timestamp":1713397230.4551721,"name":"online","context":{"idset":"11664"}}
+{"timestamp":1713397230.5122731,"name":"online","context":{"idset":"11676"}}
+{"timestamp":1713397230.6723964,"name":"online","context":{"idset":"11721,11755,11765"}}
+{"timestamp":1713397230.8916807,"name":"online","context":{"idset":"11766,11771"}}
+{"timestamp":1713397231.0021091,"name":"online","context":{"idset":"11748,11768"}}
+{"timestamp":1713397231.1049595,"name":"online","context":{"idset":"11742,11759,11774"}}
+{"timestamp":1713397231.2844055,"name":"online","context":{"idset":"11659,11663,11738,11744,11769"}}
+{"timestamp":1713397231.4322548,"name":"online","context":{"idset":"11727,11770,11778,11781"}}
+{"timestamp":1713397231.5814826,"name":"online","context":{"idset":"11735,11775,11780,11782"}}
+{"timestamp":1713397231.7562115,"name":"online","context":{"idset":"11776,11779"}}
+{"timestamp":1713397231.9046931,"name":"online","context":{"idset":"11789"}}
+{"timestamp":1713397232.0656474,"name":"online","context":{"idset":"11647,11783"}}
+{"timestamp":1713397232.1151223,"name":"online","context":{"idset":"11740"}}
+{"timestamp":1713397232.3491752,"name":"online","context":{"idset":"11723,11793"}}
+{"timestamp":1713397232.5739174,"name":"online","context":{"idset":"11696,11715,11788"}}
+{"timestamp":1713397232.7997425,"name":"online","context":{"idset":"11750,11799"}}
+{"timestamp":1713397232.9538603,"name":"online","context":{"idset":"11734,11746"}}
+{"timestamp":1713397233.0762124,"name":"online","context":{"idset":"11798"}}
+{"timestamp":1713397233.1816964,"name":"online","context":{"idset":"11729"}}
+{"timestamp":1713397233.8759515,"name":"online","context":{"idset":"11706,11718"}}
+{"timestamp":1713397234.1291592,"name":"online","context":{"idset":"11749"}}
+{"timestamp":1713397234.5766473,"name":"online","context":{"idset":"11800"}}
+{"timestamp":1713397235.0821331,"name":"online","context":{"idset":"11717"}}
+{"timestamp":1713397235.2401485,"name":"online","context":{"idset":"11741"}}
+{"timestamp":1713397235.3521628,"name":"online","context":{"idset":"11802,11804-11805"}}
+{"timestamp":1713397235.4252908,"name":"online","context":{"idset":"11806"}}
+{"timestamp":1713397235.5276194,"name":"online","context":{"idset":"11807"}}
+{"timestamp":1713397235.6231797,"name":"online","context":{"idset":"11803"}}
+{"timestamp":1713397235.8389237,"name":"online","context":{"idset":"11808"}}
+{"timestamp":1713397236.1823349,"name":"online","context":{"idset":"11758"}}
+{"timestamp":1713397236.4299064,"name":"online","context":{"idset":"11811"}}
+{"timestamp":1713397236.8217618,"name":"online","context":{"idset":"11812"}}
+{"timestamp":1713397236.9261243,"name":"online","context":{"idset":"11809,11817"}}
+{"timestamp":1713397237.0261893,"name":"online","context":{"idset":"11821"}}
+{"timestamp":1713397237.1320667,"name":"online","context":{"idset":"11814,11819"}}
+{"timestamp":1713397237.4177432,"name":"online","context":{"idset":"11813,11822"}}
+{"timestamp":1713397237.4996803,"name":"online","context":{"idset":"11818,11820"}}
+{"timestamp":1713397237.6017859,"name":"online","context":{"idset":"11761"}}
+{"timestamp":1713397237.6724718,"name":"online","context":{"idset":"11825,11828"}}
+{"timestamp":1713397237.7751191,"name":"online","context":{"idset":"11824,11826"}}
+{"timestamp":1713397237.942266,"name":"online","context":{"idset":"11830,11832"}}
+{"timestamp":1713397238.0024686,"name":"online","context":{"idset":"11827"}}
+{"timestamp":1713397238.0867047,"name":"online","context":{"idset":"11829,11835"}}
+{"timestamp":1713397238.304822,"name":"online","context":{"idset":"11716,11763,11837"}}
+{"timestamp":1713397238.4078932,"name":"online","context":{"idset":"11840"}}
+{"timestamp":1713397238.5024898,"name":"online","context":{"idset":"11839"}}
+{"timestamp":1713397238.5572011,"name":"online","context":{"idset":"11756,11841"}}
+{"timestamp":1713397238.6606302,"name":"online","context":{"idset":"11843"}}
+{"timestamp":1713397238.7252152,"name":"online","context":{"idset":"11844"}}
+{"timestamp":1713397238.9336619,"name":"online","context":{"idset":"11847"}}
+{"timestamp":1713397239.2139068,"name":"online","context":{"idset":"11842,11848"}}
+{"timestamp":1713397239.3841012,"name":"online","context":{"idset":"11849"}}
+{"timestamp":1713397239.7290537,"name":"online","context":{"idset":"11845"}}
+{"timestamp":1713397239.8036306,"name":"online","context":{"idset":"11846,11854,11857"}}
+{"timestamp":1713397239.9635901,"name":"online","context":{"idset":"11850,11855,11861"}}
+{"timestamp":1713397240.1265118,"name":"online","context":{"idset":"11853,11860,11862"}}
+{"timestamp":1713397240.2004166,"name":"online","context":{"idset":"11863"}}
+{"timestamp":1713397240.2597578,"name":"online","context":{"idset":"11852,11859"}}
+{"timestamp":1713397240.5904605,"name":"online","context":{"idset":"11865,11867-11868,11872"}}
+{"timestamp":1713397240.7523894,"name":"online","context":{"idset":"11869"}}
+{"timestamp":1713397240.9141307,"name":"online","context":{"idset":"11874-11875"}}
+{"timestamp":1713397241.0783789,"name":"online","context":{"idset":"11877"}}
+{"timestamp":1713397241.306685,"name":"online","context":{"idset":"11880"}}
+{"timestamp":1713397241.5176466,"name":"online","context":{"idset":"11876,11883-11884"}}
+{"timestamp":1713397241.6655054,"name":"online","context":{"idset":"11878,11882"}}
+{"timestamp":1713397241.7679317,"name":"online","context":{"idset":"11881"}}
+{"timestamp":1713397241.9682107,"name":"online","context":{"idset":"11885"}}
+{"timestamp":1713397242.2553711,"name":"online","context":{"idset":"11879,11889-11890"}}
+{"timestamp":1713397242.4906726,"name":"online","context":{"idset":"11888,11892"}}
+{"timestamp":1713397242.7014949,"name":"online","context":{"idset":"11887"}}
+{"timestamp":1713397243.6593244,"name":"online","context":{"idset":"11704"}}
+{"timestamp":1713398649.4037375,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713398649.4073839,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1713398649.4736974,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1713398649.4806764,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1713398649.4861529,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1713398649.5567026,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1713398649.559901,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1713398649.5630293,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1713398649.6429064,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1713398649.6462343,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1713398649.7451868,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1713398651.3081944,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1713398651.3163352,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1713398651.3838432,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1713398651.4415443,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1713398651.4914486,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1713398651.4943893,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1713398651.5953522,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1713398653.4236808,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1713398653.4845419,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1713398653.5416441,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1713398653.6410973,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1713398655.4333365,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1713398655.5298703,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1713398655.6779706,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1713398657.3952816,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1713398657.7074521,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1713398659.3778083,"name":"offline","context":{"idset":"10912"}}
+{"timestamp":1713398659.4679439,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1713398659.5672648,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1713398661.4377158,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1713398661.5390971,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1713398661.593518,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1713398661.5965228,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1713398661.6654623,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1713398661.6676791,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1713398661.7679737,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1713398663.3430226,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1713398663.4424484,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1713398663.5507195,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1713398663.6053457,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1713398663.660764,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1713398663.7609298,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1713398665.2936406,"name":"offline","context":{"idset":"10936"}}
+{"timestamp":1713398665.482702,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1713398665.5410652,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1713398665.5465209,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1713398665.6111901,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1713398665.7136269,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1713398667.3939443,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1713398667.4902561,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1713398667.5435538,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1713398667.5497367,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1713398667.6266966,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1713398667.6305382,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1713398667.7293856,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1713398669.283262,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1713398669.4494486,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1713398669.5475597,"name":"offline","context":{"idset":"10952"}}
+{"timestamp":1713398669.6212859,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1713398669.6680033,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1713398669.766608,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1713398671.5132043,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1713398671.5690708,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1713398671.6294661,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1713398671.7280266,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1713398673.5240099,"name":"offline","context":{"idset":"10971"}}
+{"timestamp":1713398673.5799987,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1713398673.651366,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1713398673.7019353,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1713398673.8011088,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1713398675.5580535,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1713398675.6118872,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1713398675.6826608,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1713398675.7828171,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1713398677.5862575,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1713398677.6857417,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1713398677.8680542,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1713399185.1471207,"name":"online","context":{"idset":"763"}}
+{"timestamp":1713399236.1614487,"name":"online","context":{"idset":"9789-9790"}}
+{"timestamp":1713399242.4408481,"name":"online","context":{"idset":"10001"}}
+{"timestamp":1713399280.479655,"name":"online","context":{"idset":"10646,10661,10665"}}
+{"timestamp":1713399281.0671434,"name":"online","context":{"idset":"10616,10621-10626,10628,10630,10636,10645,10649,10654-10656,10660,10664"}}
+{"timestamp":1713399296.3491716,"name":"online","context":{"idset":"10652"}}
+{"timestamp":1713399296.3624804,"name":"online","context":{"idset":"10647,10650-10651,10659"}}
+{"timestamp":1713400950.4230845,"name":"online","context":{"idset":"773"}}
+{"timestamp":1713400950.4997718,"name":"online","context":{"idset":"765"}}
+{"timestamp":1713400950.6132283,"name":"online","context":{"idset":"764"}}
+{"timestamp":1713400950.8289857,"name":"online","context":{"idset":"772"}}
+{"timestamp":1713400951.1082265,"name":"online","context":{"idset":"800"}}
+{"timestamp":1713400951.3039501,"name":"online","context":{"idset":"785,808,820,828"}}
+{"timestamp":1713400951.6053357,"name":"online","context":{"idset":"836,894,7701"}}
+{"timestamp":1713400951.7856195,"name":"online","context":{"idset":"771,781,806"}}
+{"timestamp":1713400951.9449091,"name":"online","context":{"idset":"7702"}}
+{"timestamp":1713400952.110775,"name":"online","context":{"idset":"940,7676,7678,7692"}}
+{"timestamp":1713400952.2699559,"name":"online","context":{"idset":"892,978,7689"}}
+{"timestamp":1713400952.4328325,"name":"online","context":{"idset":"810,7693"}}
+{"timestamp":1713400952.5910251,"name":"online","context":{"idset":"830,939,943,7679,7700"}}
+{"timestamp":1713400952.8902919,"name":"online","context":{"idset":"818"}}
+{"timestamp":1713400957.0834696,"name":"online","context":{"idset":"7713"}}
+{"timestamp":1713400957.2543049,"name":"online","context":{"idset":"7718,7735"}}
+{"timestamp":1713400957.4506106,"name":"online","context":{"idset":"7714,7723"}}
+{"timestamp":1713400957.5519772,"name":"online","context":{"idset":"7725"}}
+{"timestamp":1713400957.7151544,"name":"online","context":{"idset":"7736,7764"}}
+{"timestamp":1713400957.9025559,"name":"online","context":{"idset":"7722,7731"}}
+{"timestamp":1713400958.0680625,"name":"online","context":{"idset":"7750,7763"}}
+{"timestamp":1713400958.2459688,"name":"online","context":{"idset":"7775"}}
+{"timestamp":1713400960.0330577,"name":"online","context":{"idset":"7778,7786"}}
+{"timestamp":1713400960.4133883,"name":"online","context":{"idset":"7800"}}
+{"timestamp":1713400960.6206563,"name":"online","context":{"idset":"7783"}}
+{"timestamp":1713400961.0882485,"name":"online","context":{"idset":"7798"}}
+{"timestamp":1713400962.6358914,"name":"online","context":{"idset":"7806"}}
+{"timestamp":1713400962.7550128,"name":"online","context":{"idset":"7809"}}
+{"timestamp":1713400963.9681709,"name":"online","context":{"idset":"7819,7823"}}
+{"timestamp":1713400964.5332918,"name":"online","context":{"idset":"7858,7868"}}
+{"timestamp":1713400964.7010813,"name":"online","context":{"idset":"7841,7855"}}
+{"timestamp":1713400964.8410718,"name":"online","context":{"idset":"7871"}}
+{"timestamp":1713400965.0196218,"name":"online","context":{"idset":"7857,7861"}}
+{"timestamp":1713400965.1853487,"name":"online","context":{"idset":"7854"}}
+{"timestamp":1713400966.8861985,"name":"online","context":{"idset":"7900"}}
+{"timestamp":1713400967.4189885,"name":"online","context":{"idset":"7902"}}
+{"timestamp":1713400968.9770989,"name":"online","context":{"idset":"7911"}}
+{"timestamp":1713400969.0918822,"name":"online","context":{"idset":"7909"}}
+{"timestamp":1713400969.5765259,"name":"online","context":{"idset":"7916,7918"}}
+{"timestamp":1713400969.7504516,"name":"online","context":{"idset":"7925"}}
+{"timestamp":1713400970.2204766,"name":"online","context":{"idset":"7930"}}
+{"timestamp":1713400970.7452276,"name":"online","context":{"idset":"7941"}}
+{"timestamp":1713400970.906265,"name":"online","context":{"idset":"7936,7939"}}
+{"timestamp":1713400971.2475452,"name":"online","context":{"idset":"7952"}}
+{"timestamp":1713400971.400748,"name":"online","context":{"idset":"7948,7950,7953,7966"}}
+{"timestamp":1713400971.4723008,"name":"online","context":{"idset":"7971"}}
+{"timestamp":1713400971.7597661,"name":"online","context":{"idset":"7949"}}
+{"timestamp":1713400971.9678204,"name":"online","context":{"idset":"7975,7981"}}
+{"timestamp":1713400972.0680397,"name":"online","context":{"idset":"7983,7988"}}
+{"timestamp":1713400972.1598198,"name":"online","context":{"idset":"7961,7979"}}
+{"timestamp":1713400973.11724,"name":"online","context":{"idset":"8001"}}
+{"timestamp":1713400973.5350592,"name":"online","context":{"idset":"8009"}}
+{"timestamp":1713400973.703063,"name":"online","context":{"idset":"8007"}}
+{"timestamp":1713400973.9890168,"name":"online","context":{"idset":"8002"}}
+{"timestamp":1713400974.1849096,"name":"online","context":{"idset":"8006,8014"}}
+{"timestamp":1713400974.5444314,"name":"online","context":{"idset":"8013"}}
+{"timestamp":1713400975.0468385,"name":"online","context":{"idset":"8022"}}
+{"timestamp":1713400975.5728252,"name":"online","context":{"idset":"8027"}}
+{"timestamp":1713400976.4549861,"name":"online","context":{"idset":"8037"}}
+{"timestamp":1713400976.6394484,"name":"online","context":{"idset":"8041,8044"}}
+{"timestamp":1713400976.8115885,"name":"online","context":{"idset":"8045"}}
+{"timestamp":1713400977.0309081,"name":"online","context":{"idset":"8049"}}
+{"timestamp":1713400977.738785,"name":"online","context":{"idset":"8051"}}
+{"timestamp":1713400982.650866,"name":"online","context":{"idset":"8181"}}
+{"timestamp":1713400982.975246,"name":"online","context":{"idset":"8187,8192"}}
+{"timestamp":1713400983.321733,"name":"online","context":{"idset":"8191"}}
+{"timestamp":1713400983.4079304,"name":"online","context":{"idset":"8193,8214,8217"}}
+{"timestamp":1713400983.5127773,"name":"online","context":{"idset":"8195"}}
+{"timestamp":1713400983.5716898,"name":"online","context":{"idset":"8210"}}
+{"timestamp":1713400983.6629045,"name":"online","context":{"idset":"8228"}}
+{"timestamp":1713400983.8603237,"name":"online","context":{"idset":"8205,8220"}}
+{"timestamp":1713400984.1548352,"name":"online","context":{"idset":"8194"}}
+{"timestamp":1713400984.2766378,"name":"online","context":{"idset":"8236"}}
+{"timestamp":1713400984.6358283,"name":"online","context":{"idset":"8232"}}
+{"timestamp":1713400985.0074096,"name":"online","context":{"idset":"8246-8247"}}
+{"timestamp":1713400985.1637053,"name":"online","context":{"idset":"8259"}}
+{"timestamp":1713400985.6253829,"name":"online","context":{"idset":"8254,8256,8260"}}
+{"timestamp":1713400985.790247,"name":"online","context":{"idset":"8273,8277"}}
+{"timestamp":1713400986.2939973,"name":"online","context":{"idset":"8284"}}
+{"timestamp":1713400986.4647291,"name":"online","context":{"idset":"8281"}}
+{"timestamp":1713400987.1483634,"name":"online","context":{"idset":"8292"}}
+{"timestamp":1713400987.7432752,"name":"online","context":{"idset":"8298"}}
+{"timestamp":1713400989.3906422,"name":"online","context":{"idset":"8301"}}
+{"timestamp":1713400994.4639277,"name":"online","context":{"idset":"8825,8831"}}
+{"timestamp":1713400994.8415375,"name":"online","context":{"idset":"8830"}}
+{"timestamp":1713400995.0593793,"name":"online","context":{"idset":"8850"}}
+{"timestamp":1713400995.2654426,"name":"online","context":{"idset":"8838"}}
+{"timestamp":1713400995.8750358,"name":"online","context":{"idset":"8872"}}
+{"timestamp":1713400996.0300162,"name":"online","context":{"idset":"8849,8883"}}
+{"timestamp":1713400996.2735257,"name":"online","context":{"idset":"8871,8882,8889"}}
+{"timestamp":1713400997.0440319,"name":"online","context":{"idset":"8909"}}
+{"timestamp":1713400997.2060101,"name":"online","context":{"idset":"8905,8922"}}
+{"timestamp":1713400997.312299,"name":"online","context":{"idset":"8919"}}
+{"timestamp":1713400997.5034924,"name":"online","context":{"idset":"8908,8917,8923"}}
+{"timestamp":1713400997.6729069,"name":"online","context":{"idset":"8900"}}
+{"timestamp":1713400997.8635993,"name":"online","context":{"idset":"8933"}}
+{"timestamp":1713400999.850966,"name":"online","context":{"idset":"8945"}}
+{"timestamp":1713401000.9593701,"name":"online","context":{"idset":"8948"}}
+{"timestamp":1713401001.3640025,"name":"online","context":{"idset":"8959,8961,8975"}}
+{"timestamp":1713401001.4406316,"name":"online","context":{"idset":"8951"}}
+{"timestamp":1713401001.5597301,"name":"online","context":{"idset":"8950"}}
+{"timestamp":1713401001.7691581,"name":"online","context":{"idset":"8952"}}
+{"timestamp":1713401001.892863,"name":"online","context":{"idset":"8978"}}
+{"timestamp":1713401002.1580021,"name":"online","context":{"idset":"8988"}}
+{"timestamp":1713401002.2220044,"name":"online","context":{"idset":"8958"}}
+{"timestamp":1713401002.2866776,"name":"online","context":{"idset":"8991,9001"}}
+{"timestamp":1713401002.3865514,"name":"online","context":{"idset":"8992"}}
+{"timestamp":1713401002.5216861,"name":"online","context":{"idset":"9017"}}
+{"timestamp":1713401003.2254853,"name":"online","context":{"idset":"9037"}}
+{"timestamp":1713401003.4019032,"name":"online","context":{"idset":"9004"}}
+{"timestamp":1713401003.5662439,"name":"online","context":{"idset":"9009"}}
+{"timestamp":1713401003.8943694,"name":"online","context":{"idset":"8981,9023"}}
+{"timestamp":1713401004.0778167,"name":"online","context":{"idset":"9003,9048"}}
+{"timestamp":1713401004.1395085,"name":"online","context":{"idset":"9002,9053"}}
+{"timestamp":1713401004.2920361,"name":"online","context":{"idset":"8998,9007,9088,9097"}}
+{"timestamp":1713401004.464797,"name":"online","context":{"idset":"8997,9011"}}
+{"timestamp":1713401004.6259992,"name":"online","context":{"idset":"9028,9033"}}
+{"timestamp":1713401004.7851272,"name":"online","context":{"idset":"9013-9014,9050"}}
+{"timestamp":1713401004.9518487,"name":"online","context":{"idset":"9012,9021,9042"}}
+{"timestamp":1713401005.1087961,"name":"online","context":{"idset":"9026,9055"}}
+{"timestamp":1713401005.2660651,"name":"online","context":{"idset":"9016,9034"}}
+{"timestamp":1713401005.417757,"name":"online","context":{"idset":"9020,9022,9039"}}
+{"timestamp":1713401005.607667,"name":"online","context":{"idset":"9040,9044"}}
+{"timestamp":1713401005.7828369,"name":"online","context":{"idset":"9038"}}
+{"timestamp":1713401005.977983,"name":"online","context":{"idset":"9024,9047,9051"}}
+{"timestamp":1713401006.1381063,"name":"online","context":{"idset":"9045,9056"}}
+{"timestamp":1713401006.2961142,"name":"online","context":{"idset":"9054,9118"}}
+{"timestamp":1713401006.366081,"name":"online","context":{"idset":"9060"}}
+{"timestamp":1713401007.0761006,"name":"online","context":{"idset":"9066"}}
+{"timestamp":1713401007.3123589,"name":"online","context":{"idset":"9064-9065,9071"}}
+{"timestamp":1713401007.6922901,"name":"online","context":{"idset":"9069,9073-9074"}}
+{"timestamp":1713401008.5303407,"name":"online","context":{"idset":"9130"}}
+{"timestamp":1713401008.7123721,"name":"online","context":{"idset":"9131"}}
+{"timestamp":1713401009.8599002,"name":"online","context":{"idset":"9144"}}
+{"timestamp":1713401010.0892448,"name":"online","context":{"idset":"9145"}}
+{"timestamp":1713401010.2552779,"name":"online","context":{"idset":"9142"}}
+{"timestamp":1713401010.8585615,"name":"online","context":{"idset":"9140"}}
+{"timestamp":1713401011.3352931,"name":"online","context":{"idset":"9168,9170"}}
+{"timestamp":1713401012.8674612,"name":"online","context":{"idset":"9181"}}
+{"timestamp":1713401013.3013115,"name":"online","context":{"idset":"9185"}}
+{"timestamp":1713401013.7624934,"name":"online","context":{"idset":"9198"}}
+{"timestamp":1713401014.3363142,"name":"online","context":{"idset":"9200,9202-9203"}}
+{"timestamp":1713401015.0851851,"name":"online","context":{"idset":"9213"}}
+{"timestamp":1713401015.0877218,"name":"online","context":{"idset":"9206"}}
+{"timestamp":1713401015.5504336,"name":"online","context":{"idset":"9219"}}
+{"timestamp":1713401015.6196952,"name":"online","context":{"idset":"9228"}}
+{"timestamp":1713401015.9550531,"name":"online","context":{"idset":"9231"}}
+{"timestamp":1713401016.0650425,"name":"online","context":{"idset":"9217,9238"}}
+{"timestamp":1713401016.2438242,"name":"online","context":{"idset":"9232,9240"}}
+{"timestamp":1713401016.549089,"name":"online","context":{"idset":"9246"}}
+{"timestamp":1713401016.7109103,"name":"online","context":{"idset":"9265-9266"}}
+{"timestamp":1713401017.454689,"name":"online","context":{"idset":"9273"}}
+{"timestamp":1713401017.6212306,"name":"online","context":{"idset":"9277"}}
+{"timestamp":1713401017.6955349,"name":"online","context":{"idset":"9282"}}
+{"timestamp":1713401017.8223429,"name":"online","context":{"idset":"9284"}}
+{"timestamp":1713401018.2706468,"name":"online","context":{"idset":"9288"}}
+{"timestamp":1713401019.8995821,"name":"online","context":{"idset":"9301"}}
+{"timestamp":1713401019.970015,"name":"online","context":{"idset":"9299"}}
+{"timestamp":1713401020.1936276,"name":"online","context":{"idset":"9303,9311"}}
+{"timestamp":1713401020.555654,"name":"online","context":{"idset":"9313"}}
+{"timestamp":1713401021.0616684,"name":"online","context":{"idset":"9317"}}
+{"timestamp":1713401022.0977495,"name":"online","context":{"idset":"9328,9330"}}
+{"timestamp":1713401022.2336369,"name":"online","context":{"idset":"9338"}}
+{"timestamp":1713401022.5519228,"name":"online","context":{"idset":"9331,9340,9343"}}
+{"timestamp":1713401022.716743,"name":"online","context":{"idset":"9332,9334,9342"}}
+{"timestamp":1713401022.9337959,"name":"online","context":{"idset":"9348,9352"}}
+{"timestamp":1713401023.1034093,"name":"online","context":{"idset":"9355"}}
+{"timestamp":1713401023.3954558,"name":"online","context":{"idset":"9368"}}
+{"timestamp":1713401023.6634426,"name":"online","context":{"idset":"9373,9380"}}
+{"timestamp":1713401023.8701406,"name":"online","context":{"idset":"9358,9376,9381"}}
+{"timestamp":1713401024.363364,"name":"online","context":{"idset":"9388"}}
+{"timestamp":1713401025.3594599,"name":"online","context":{"idset":"9397"}}
+{"timestamp":1713401026.3789775,"name":"online","context":{"idset":"9408"}}
+{"timestamp":1713401026.8377628,"name":"online","context":{"idset":"9413"}}
+{"timestamp":1713401027.356168,"name":"online","context":{"idset":"9418,9421"}}
+{"timestamp":1713401027.4746346,"name":"online","context":{"idset":"9423"}}
+{"timestamp":1713401029.1181753,"name":"online","context":{"idset":"9452,9454"}}
+{"timestamp":1713401029.1862626,"name":"online","context":{"idset":"9442"}}
+{"timestamp":1713401029.5817878,"name":"online","context":{"idset":"9462"}}
+{"timestamp":1713401029.6746845,"name":"online","context":{"idset":"9463"}}
+{"timestamp":1713401029.8645158,"name":"online","context":{"idset":"9460"}}
+{"timestamp":1713401030.0431318,"name":"online","context":{"idset":"9468"}}
+{"timestamp":1713401030.8562603,"name":"online","context":{"idset":"9474"}}
+{"timestamp":1713401030.8608925,"name":"online","context":{"idset":"9470"}}
+{"timestamp":1713401030.8654044,"name":"online","context":{"idset":"9481"}}
+{"timestamp":1713401031.3594639,"name":"online","context":{"idset":"9480"}}
+{"timestamp":1713401031.8531406,"name":"online","context":{"idset":"9490-9491"}}
+{"timestamp":1713401032.6126566,"name":"online","context":{"idset":"9505"}}
+{"timestamp":1713401032.8594482,"name":"online","context":{"idset":"9501"}}
+{"timestamp":1713401033.6184928,"name":"online","context":{"idset":"9516"}}
+{"timestamp":1713401034.2409842,"name":"online","context":{"idset":"9529"}}
+{"timestamp":1713401034.3929589,"name":"online","context":{"idset":"9538"}}
+{"timestamp":1713401034.6094112,"name":"online","context":{"idset":"9541"}}
+{"timestamp":1713401034.6924386,"name":"online","context":{"idset":"9527"}}
+{"timestamp":1713401034.7791317,"name":"online","context":{"idset":"9544"}}
+{"timestamp":1713401034.9265194,"name":"online","context":{"idset":"9533,9543"}}
+{"timestamp":1713401035.0265996,"name":"online","context":{"idset":"9542"}}
+{"timestamp":1713401035.0999157,"name":"online","context":{"idset":"9546"}}
+{"timestamp":1713401035.2465959,"name":"online","context":{"idset":"9534,9539,9551"}}
+{"timestamp":1713401035.4222441,"name":"online","context":{"idset":"9556"}}
+{"timestamp":1713401035.761337,"name":"online","context":{"idset":"9560"}}
+{"timestamp":1713401035.948642,"name":"online","context":{"idset":"9559,9561,9563,9565"}}
+{"timestamp":1713401036.1762381,"name":"online","context":{"idset":"9562"}}
+{"timestamp":1713401036.4220102,"name":"online","context":{"idset":"9566-9567,9569,9572,9576"}}
+{"timestamp":1713401036.6449692,"name":"online","context":{"idset":"9558,9568,9570-9571,9577"}}
+{"timestamp":1713401036.8965721,"name":"online","context":{"idset":"9573,9579"}}
+{"timestamp":1713401037.1036682,"name":"online","context":{"idset":"9574,9578,9580,9583"}}
+{"timestamp":1713401037.264185,"name":"online","context":{"idset":"9581,9585,9588"}}
+{"timestamp":1713401037.4323432,"name":"online","context":{"idset":"9582,9584,9586"}}
+{"timestamp":1713401037.7320182,"name":"online","context":{"idset":"9589"}}
+{"timestamp":1713401038.017848,"name":"online","context":{"idset":"9591,9595"}}
+{"timestamp":1713401038.4341455,"name":"online","context":{"idset":"9599"}}
+{"timestamp":1713401039.0225182,"name":"online","context":{"idset":"9607-9608"}}
+{"timestamp":1713401039.8829622,"name":"online","context":{"idset":"9622"}}
+{"timestamp":1713401040.1594949,"name":"online","context":{"idset":"9624-9625"}}
+{"timestamp":1713401040.516593,"name":"online","context":{"idset":"9626,9631"}}
+{"timestamp":1713401040.6777382,"name":"online","context":{"idset":"9627"}}
+{"timestamp":1713401040.8606725,"name":"online","context":{"idset":"9640"}}
+{"timestamp":1713401040.9646358,"name":"online","context":{"idset":"9638"}}
+{"timestamp":1713401041.1254399,"name":"online","context":{"idset":"9639,9641"}}
+{"timestamp":1713401041.6270633,"name":"online","context":{"idset":"9655"}}
+{"timestamp":1713401042.1019306,"name":"online","context":{"idset":"9660"}}
+{"timestamp":1713401042.6037869,"name":"online","context":{"idset":"9664"}}
+{"timestamp":1713401044.1936016,"name":"online","context":{"idset":"9669"}}
+{"timestamp":1713401044.9619112,"name":"online","context":{"idset":"9679"}}
+{"timestamp":1713401045.1761374,"name":"online","context":{"idset":"9677"}}
+{"timestamp":1713401045.6598389,"name":"online","context":{"idset":"9687"}}
+{"timestamp":1713401045.8445089,"name":"online","context":{"idset":"9691"}}
+{"timestamp":1713401045.9489419,"name":"online","context":{"idset":"9682,9693"}}
+{"timestamp":1713401046.0242162,"name":"online","context":{"idset":"9688"}}
+{"timestamp":1713401046.1089153,"name":"online","context":{"idset":"9689"}}
+{"timestamp":1713401046.2486961,"name":"online","context":{"idset":"9690"}}
+{"timestamp":1713401046.3850014,"name":"online","context":{"idset":"9695"}}
+{"timestamp":1713401046.4654229,"name":"online","context":{"idset":"9692"}}
+{"timestamp":1713401046.528465,"name":"online","context":{"idset":"9697"}}
+{"timestamp":1713401046.6321776,"name":"online","context":{"idset":"9694,9696,9698-9700,9704"}}
+{"timestamp":1713401046.7890651,"name":"online","context":{"idset":"9701-9702"}}
+{"timestamp":1713401046.8600047,"name":"online","context":{"idset":"9703"}}
+{"timestamp":1713401046.9477515,"name":"online","context":{"idset":"9705"}}
+{"timestamp":1713401047.222182,"name":"online","context":{"idset":"9706,9708-9711"}}
+{"timestamp":1713401047.3846519,"name":"online","context":{"idset":"9717"}}
+{"timestamp":1713401047.4553716,"name":"online","context":{"idset":"9707"}}
+{"timestamp":1713401047.5243354,"name":"online","context":{"idset":"9712-9713,9718"}}
+{"timestamp":1713401047.5905213,"name":"online","context":{"idset":"9722"}}
+{"timestamp":1713401047.656136,"name":"online","context":{"idset":"9715,9720-9721"}}
+{"timestamp":1713401047.7666326,"name":"online","context":{"idset":"9714"}}
+{"timestamp":1713401047.9078417,"name":"online","context":{"idset":"9723,9725"}}
+{"timestamp":1713401048.1139765,"name":"online","context":{"idset":"9730"}}
+{"timestamp":1713401048.2217741,"name":"online","context":{"idset":"9731"}}
+{"timestamp":1713401048.3347242,"name":"online","context":{"idset":"9726,9736"}}
+{"timestamp":1713401048.4383645,"name":"online","context":{"idset":"9716,9732"}}
+{"timestamp":1713401048.5982466,"name":"online","context":{"idset":"9724,9727-9729,9734,9739-9740"}}
+{"timestamp":1713401049.0557992,"name":"online","context":{"idset":"9735"}}
+{"timestamp":1713401049.1375999,"name":"online","context":{"idset":"9741-9742,9746"}}
+{"timestamp":1713401049.2039182,"name":"online","context":{"idset":"9737,9751"}}
+{"timestamp":1713401049.3068774,"name":"online","context":{"idset":"9745"}}
+{"timestamp":1713401049.4673858,"name":"online","context":{"idset":"9744,9750,9760"}}
+{"timestamp":1713401049.6835358,"name":"online","context":{"idset":"9738,9747-9748,9752"}}
+{"timestamp":1713401049.7747078,"name":"online","context":{"idset":"9757,9766"}}
+{"timestamp":1713401049.9420025,"name":"online","context":{"idset":"9759"}}
+{"timestamp":1713401050.0076032,"name":"online","context":{"idset":"9754,9761,9767"}}
+{"timestamp":1713401050.0766547,"name":"online","context":{"idset":"9749,9753,9765,9769,9775"}}
+{"timestamp":1713401050.0822468,"name":"online","context":{"idset":"9770"}}
+{"timestamp":1713401050.1621821,"name":"online","context":{"idset":"9755,9773"}}
+{"timestamp":1713401050.2297266,"name":"online","context":{"idset":"9756,9758,9772,9778-9779"}}
+{"timestamp":1713401050.3335896,"name":"online","context":{"idset":"9763-9764,9768,9774"}}
+{"timestamp":1713401050.4289303,"name":"online","context":{"idset":"9762,9776"}}
+{"timestamp":1713401050.5862072,"name":"online","context":{"idset":"9780-9781"}}
+{"timestamp":1713401050.759809,"name":"online","context":{"idset":"9784"}}
+{"timestamp":1713401050.9235001,"name":"online","context":{"idset":"9783"}}
+{"timestamp":1713401051.0032511,"name":"online","context":{"idset":"9782"}}
+{"timestamp":1713401051.1652036,"name":"online","context":{"idset":"9786,9788"}}
+{"timestamp":1713401051.3262513,"name":"online","context":{"idset":"9785,9791"}}
+{"timestamp":1713401051.8757842,"name":"online","context":{"idset":"9792"}}
+{"timestamp":1713401052.1093209,"name":"online","context":{"idset":"9793"}}
+{"timestamp":1713401052.2146008,"name":"online","context":{"idset":"9787,9796,9798,9800"}}
+{"timestamp":1713401052.3805985,"name":"online","context":{"idset":"9797,9799"}}
+{"timestamp":1713401052.5707812,"name":"online","context":{"idset":"9802"}}
+{"timestamp":1713401052.84462,"name":"online","context":{"idset":"9794,9803,9805"}}
+{"timestamp":1713401053.0363522,"name":"online","context":{"idset":"9801,9804,9808"}}
+{"timestamp":1713401053.2105682,"name":"online","context":{"idset":"9806"}}
+{"timestamp":1713401053.6073654,"name":"online","context":{"idset":"9809"}}
+{"timestamp":1713401054.0104914,"name":"online","context":{"idset":"9810"}}
+{"timestamp":1713401054.572778,"name":"online","context":{"idset":"9811"}}
+{"timestamp":1713401055.9141221,"name":"online","context":{"idset":"9812"}}
+{"timestamp":1713401056.3886931,"name":"online","context":{"idset":"9813"}}
+{"timestamp":1713401057.1579037,"name":"online","context":{"idset":"9814"}}
+{"timestamp":1713401057.4188383,"name":"online","context":{"idset":"9815"}}
+{"timestamp":1713401057.5969603,"name":"online","context":{"idset":"9818"}}
+{"timestamp":1713401057.7859848,"name":"online","context":{"idset":"9816"}}
+{"timestamp":1713401057.8685787,"name":"online","context":{"idset":"9817,9822"}}
+{"timestamp":1713401058.0755558,"name":"online","context":{"idset":"9824"}}
+{"timestamp":1713401058.2372911,"name":"online","context":{"idset":"9821,9823,9825,9836"}}
+{"timestamp":1713401059.0361223,"name":"online","context":{"idset":"9830,9840,9845"}}
+{"timestamp":1713401059.6247616,"name":"online","context":{"idset":"9849"}}
+{"timestamp":1713401059.797684,"name":"online","context":{"idset":"9844"}}
+{"timestamp":1713401059.8753102,"name":"online","context":{"idset":"9853"}}
+{"timestamp":1713401059.9477105,"name":"online","context":{"idset":"9857"}}
+{"timestamp":1713401060.8242867,"name":"online","context":{"idset":"9867"}}
+{"timestamp":1713401061.3800941,"name":"online","context":{"idset":"9872,9884"}}
+{"timestamp":1713401062.0485754,"name":"online","context":{"idset":"9899"}}
+{"timestamp":1713401062.2164702,"name":"online","context":{"idset":"9907,9913"}}
+{"timestamp":1713401062.404063,"name":"online","context":{"idset":"9904"}}
+{"timestamp":1713401062.4716899,"name":"online","context":{"idset":"9918"}}
+{"timestamp":1713401063.1599627,"name":"online","context":{"idset":"9920"}}
+{"timestamp":1713401063.9833622,"name":"online","context":{"idset":"9923"}}
+{"timestamp":1713401064.6228912,"name":"online","context":{"idset":"9938,9946"}}
+{"timestamp":1713401065.1150901,"name":"online","context":{"idset":"9942,9950"}}
+{"timestamp":1713401065.5142691,"name":"online","context":{"idset":"9954"}}
+{"timestamp":1713401066.0969987,"name":"online","context":{"idset":"9964"}}
+{"timestamp":1713401066.2146676,"name":"online","context":{"idset":"9962"}}
+{"timestamp":1713401066.6163187,"name":"online","context":{"idset":"9968"}}
+{"timestamp":1713401066.7087319,"name":"online","context":{"idset":"9966"}}
+{"timestamp":1713401067.5106249,"name":"online","context":{"idset":"9976"}}
+{"timestamp":1713401067.6096518,"name":"online","context":{"idset":"9985,9990"}}
+{"timestamp":1713401067.7907536,"name":"online","context":{"idset":"9979,9984"}}
+{"timestamp":1713401068.4495151,"name":"online","context":{"idset":"9998"}}
+{"timestamp":1713401068.5543723,"name":"online","context":{"idset":"10000,10010"}}
+{"timestamp":1713401068.8348329,"name":"online","context":{"idset":"9993,10017,10019"}}
+{"timestamp":1713401069.0118892,"name":"online","context":{"idset":"10021"}}
+{"timestamp":1713401069.1736739,"name":"online","context":{"idset":"10016,10020"}}
+{"timestamp":1713401069.5392139,"name":"online","context":{"idset":"10030"}}
+{"timestamp":1713401069.7643714,"name":"online","context":{"idset":"10029"}}
+{"timestamp":1713401070.3242915,"name":"online","context":{"idset":"10034"}}
+{"timestamp":1713401071.446408,"name":"online","context":{"idset":"10061"}}
+{"timestamp":1713401071.944458,"name":"online","context":{"idset":"10060"}}
+{"timestamp":1713401072.222559,"name":"online","context":{"idset":"10055"}}
+{"timestamp":1713401072.4754536,"name":"online","context":{"idset":"10071"}}
+{"timestamp":1713401073.8407147,"name":"online","context":{"idset":"10090"}}
+{"timestamp":1713401074.0772855,"name":"online","context":{"idset":"10089,10092"}}
+{"timestamp":1713401074.6203249,"name":"online","context":{"idset":"10104"}}
+{"timestamp":1713401074.8371973,"name":"online","context":{"idset":"10088"}}
+{"timestamp":1713401074.9380887,"name":"online","context":{"idset":"10105"}}
+{"timestamp":1713401075.0999177,"name":"online","context":{"idset":"10102-10103,10107,10115"}}
+{"timestamp":1713401075.3767931,"name":"online","context":{"idset":"10116,10118,10123-10124"}}
+{"timestamp":1713401075.4678965,"name":"online","context":{"idset":"10109,10117,10119"}}
+{"timestamp":1713401075.6285968,"name":"online","context":{"idset":"10126"}}
+{"timestamp":1713401075.8906345,"name":"online","context":{"idset":"10131,10133"}}
+{"timestamp":1713401075.9670763,"name":"online","context":{"idset":"10134"}}
+{"timestamp":1713401076.0747614,"name":"online","context":{"idset":"10132"}}
+{"timestamp":1713401076.0809855,"name":"online","context":{"idset":"10128"}}
+{"timestamp":1713401076.8780413,"name":"online","context":{"idset":"10143"}}
+{"timestamp":1713401077.467684,"name":"online","context":{"idset":"10159"}}
+{"timestamp":1713401077.5858488,"name":"online","context":{"idset":"10152"}}
+{"timestamp":1713401077.7669239,"name":"online","context":{"idset":"10156"}}
+{"timestamp":1713401077.8898497,"name":"online","context":{"idset":"10158,10164,10166-10167"}}
+{"timestamp":1713401077.9907267,"name":"online","context":{"idset":"10162"}}
+{"timestamp":1713401078.0976496,"name":"online","context":{"idset":"10146,10160,10165"}}
+{"timestamp":1713401078.1612077,"name":"online","context":{"idset":"10171"}}
+{"timestamp":1713401078.2167904,"name":"online","context":{"idset":"10180"}}
+{"timestamp":1713401078.273778,"name":"online","context":{"idset":"10151,10183"}}
+{"timestamp":1713401078.3580294,"name":"online","context":{"idset":"10169"}}
+{"timestamp":1713401078.5490053,"name":"online","context":{"idset":"10173,10179"}}
+{"timestamp":1713401078.620858,"name":"online","context":{"idset":"10181"}}
+{"timestamp":1713401078.8072317,"name":"online","context":{"idset":"10177"}}
+{"timestamp":1713401079.0575559,"name":"online","context":{"idset":"10190"}}
+{"timestamp":1713401079.2971759,"name":"online","context":{"idset":"10185"}}
+{"timestamp":1713401079.4930587,"name":"online","context":{"idset":"10201"}}
+{"timestamp":1713401079.8743997,"name":"online","context":{"idset":"10198,10202"}}
+{"timestamp":1713401080.1649926,"name":"online","context":{"idset":"10205,10209,10211,10213,10216"}}
+{"timestamp":1713401080.3792939,"name":"online","context":{"idset":"10204,10206,10224"}}
+{"timestamp":1713401080.4805679,"name":"online","context":{"idset":"10197,10232"}}
+{"timestamp":1713401080.6377325,"name":"online","context":{"idset":"10214"}}
+{"timestamp":1713401080.8271639,"name":"online","context":{"idset":"10231"}}
+{"timestamp":1713401081.125756,"name":"online","context":{"idset":"10212,10237"}}
+{"timestamp":1713401081.5959549,"name":"online","context":{"idset":"10251-10252"}}
+{"timestamp":1713401082.1462576,"name":"online","context":{"idset":"10255-10256"}}
+{"timestamp":1713401082.3126459,"name":"online","context":{"idset":"10259"}}
+{"timestamp":1713401084.0258145,"name":"online","context":{"idset":"10275"}}
+{"timestamp":1713401084.3135595,"name":"online","context":{"idset":"10277"}}
+{"timestamp":1713401084.6835084,"name":"online","context":{"idset":"10271,10280"}}
+{"timestamp":1713401085.9885499,"name":"online","context":{"idset":"10293"}}
+{"timestamp":1713401089.7873166,"name":"online","context":{"idset":"10347"}}
+{"timestamp":1713401089.9548454,"name":"online","context":{"idset":"10365"}}
+{"timestamp":1713401091.1890745,"name":"online","context":{"idset":"10386,10391"}}
+{"timestamp":1713401091.7677729,"name":"online","context":{"idset":"10390,10399"}}
+{"timestamp":1713401091.9107404,"name":"online","context":{"idset":"10410"}}
+{"timestamp":1713401092.2680938,"name":"online","context":{"idset":"10425"}}
+{"timestamp":1713401093.6135736,"name":"online","context":{"idset":"10444"}}
+{"timestamp":1713401094.070133,"name":"online","context":{"idset":"10454,10460"}}
+{"timestamp":1713401094.535531,"name":"online","context":{"idset":"10481"}}
+{"timestamp":1713401095.0356643,"name":"online","context":{"idset":"10471"}}
+{"timestamp":1713401096.0801327,"name":"online","context":{"idset":"10494"}}
+{"timestamp":1713401096.9154096,"name":"online","context":{"idset":"10515,10517-10518"}}
+{"timestamp":1713401098.8169651,"name":"online","context":{"idset":"10551"}}
+{"timestamp":1713401099.8671732,"name":"online","context":{"idset":"10571,10574"}}
+{"timestamp":1713401100.0501857,"name":"online","context":{"idset":"10569"}}
+{"timestamp":1713401100.2359591,"name":"online","context":{"idset":"10578"}}
+{"timestamp":1713401100.8928919,"name":"online","context":{"idset":"10598"}}
+{"timestamp":1713401101.3540061,"name":"online","context":{"idset":"10594,10608"}}
+{"timestamp":1713401101.823329,"name":"online","context":{"idset":"10615"}}
+{"timestamp":1713401103.2408152,"name":"online","context":{"idset":"10641"}}
+{"timestamp":1713401103.5182898,"name":"online","context":{"idset":"10648"}}
+{"timestamp":1713401103.9893343,"name":"online","context":{"idset":"10633"}}
+{"timestamp":1713401105.6558938,"name":"online","context":{"idset":"10688"}}
+{"timestamp":1713401105.8787656,"name":"online","context":{"idset":"10695"}}
+{"timestamp":1713401109.6252885,"name":"online","context":{"idset":"10760,10767"}}
+{"timestamp":1713401109.7320449,"name":"online","context":{"idset":"10765,10772"}}
+{"timestamp":1713401113.3608444,"name":"online","context":{"idset":"10870"}}
+{"timestamp":1713401113.4363809,"name":"online","context":{"idset":"10858"}}
+{"timestamp":1713401113.6043675,"name":"online","context":{"idset":"10869"}}
+{"timestamp":1713401113.8155413,"name":"online","context":{"idset":"10871"}}
+{"timestamp":1713401114.0348492,"name":"online","context":{"idset":"10875"}}
+{"timestamp":1713401114.1812477,"name":"online","context":{"idset":"10873,10877,10882,10886"}}
+{"timestamp":1713401114.2880752,"name":"online","context":{"idset":"10883"}}
+{"timestamp":1713401114.3710155,"name":"online","context":{"idset":"10893,10899"}}
+{"timestamp":1713401114.4543509,"name":"online","context":{"idset":"10888"}}
+{"timestamp":1713401114.5271616,"name":"online","context":{"idset":"10907"}}
+{"timestamp":1713401114.5996618,"name":"online","context":{"idset":"10891,10901,10904"}}
+{"timestamp":1713401114.994421,"name":"online","context":{"idset":"10908,10910"}}
+{"timestamp":1713401115.1138685,"name":"online","context":{"idset":"10917"}}
+{"timestamp":1713401115.1916721,"name":"online","context":{"idset":"10890,10913"}}
+{"timestamp":1713401115.650681,"name":"online","context":{"idset":"10922"}}
+{"timestamp":1713401115.773941,"name":"online","context":{"idset":"10914,10928"}}
+{"timestamp":1713401115.8904805,"name":"online","context":{"idset":"10924"}}
+{"timestamp":1713401115.962028,"name":"online","context":{"idset":"10925"}}
+{"timestamp":1713401116.0721266,"name":"online","context":{"idset":"10931,10935"}}
+{"timestamp":1713401116.1426914,"name":"online","context":{"idset":"10919,10923"}}
+{"timestamp":1713401116.2481787,"name":"online","context":{"idset":"10927,10930,10932"}}
+{"timestamp":1713401116.428175,"name":"online","context":{"idset":"10940"}}
+{"timestamp":1713401116.6019127,"name":"online","context":{"idset":"10939,10944"}}
+{"timestamp":1713401116.7881155,"name":"online","context":{"idset":"10938,10941-10942"}}
+{"timestamp":1713401116.893615,"name":"online","context":{"idset":"10921,10943,10945,10947,10953"}}
+{"timestamp":1713401117.1193206,"name":"online","context":{"idset":"10949,10951"}}
+{"timestamp":1713401117.2906301,"name":"online","context":{"idset":"10954-10955,10958,10966"}}
+{"timestamp":1713401117.4521339,"name":"online","context":{"idset":"10950,10963"}}
+{"timestamp":1713401117.6136222,"name":"online","context":{"idset":"10948"}}
+{"timestamp":1713401117.7770984,"name":"online","context":{"idset":"10964"}}
+{"timestamp":1713401117.843091,"name":"online","context":{"idset":"10978"}}
+{"timestamp":1713401118.0713637,"name":"online","context":{"idset":"10962,10980"}}
+{"timestamp":1713401118.239779,"name":"online","context":{"idset":"10956,10969-10970,10972,10975-10976,10979,10982,10984-10985"}}
+{"timestamp":1713401118.5389397,"name":"online","context":{"idset":"10974"}}
+{"timestamp":1713401119.9860151,"name":"online","context":{"idset":"11046"}}
+{"timestamp":1713401126.1696291,"name":"online","context":{"idset":"11107"}}
+{"timestamp":1713401127.0340371,"name":"online","context":{"idset":"11138"}}
+{"timestamp":1713401127.2103934,"name":"online","context":{"idset":"11134"}}
+{"timestamp":1713401127.420357,"name":"online","context":{"idset":"11128"}}
+{"timestamp":1713401127.7744119,"name":"online","context":{"idset":"11144"}}
+{"timestamp":1713401128.3783977,"name":"online","context":{"idset":"11150"}}
+{"timestamp":1713401129.5362954,"name":"online","context":{"idset":"11184"}}
+{"timestamp":1713401129.9153678,"name":"online","context":{"idset":"11195"}}
+{"timestamp":1713401130.7891831,"name":"online","context":{"idset":"11201"}}
+{"timestamp":1713401131.7220576,"name":"online","context":{"idset":"11210"}}
+{"timestamp":1713401133.5254672,"name":"online","context":{"idset":"11277"}}
+{"timestamp":1713401135.2643702,"name":"online","context":{"idset":"11322"}}
+{"timestamp":1713401138.7181938,"name":"online","context":{"idset":"11365"}}
+{"timestamp":1713401139.0109603,"name":"online","context":{"idset":"11385"}}
+{"timestamp":1713401140.5203071,"name":"online","context":{"idset":"11425"}}
+{"timestamp":1713401140.9928675,"name":"online","context":{"idset":"11410"}}
+{"timestamp":1713401145.1815243,"name":"online","context":{"idset":"11516"}}
+{"timestamp":1713401145.3776915,"name":"online","context":{"idset":"11512"}}
+{"timestamp":1713401146.6333616,"name":"online","context":{"idset":"11549"}}
+{"timestamp":1713401146.802237,"name":"online","context":{"idset":"11543"}}
+{"timestamp":1713401147.3304183,"name":"online","context":{"idset":"11565"}}
+{"timestamp":1713401148.314537,"name":"online","context":{"idset":"11599"}}
+{"timestamp":1713401150.808111,"name":"online","context":{"idset":"11639"}}
+{"timestamp":1713401152.6597264,"name":"online","context":{"idset":"11688"}}
+{"timestamp":1713401155.1402352,"name":"online","context":{"idset":"11736"}}
+{"timestamp":1713401871.7596416,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1713401871.7938945,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1713401871.7996917,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1713401871.8029828,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1713401871.8057847,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1713401871.808634,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1713401871.9036305,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1713401872.0099108,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1713401873.7974367,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1713401873.8752248,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1713401873.8863118,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1713401873.9791341,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1713401874.0772622,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1713401876.6383219,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1713401881.7016609,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1713401881.7993014,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1713401881.8993473,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1713401883.7780087,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1713401883.8782132,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1713401884.0547919,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1713401885.731066,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1713401885.886405,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1713401885.9578724,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1713401886.0255735,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1713401886.1255169,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1713401887.7788754,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1713401887.952173,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1713401887.9573762,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1713401888.0580337,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1713401888.6364789,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1713401889.849329,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1713401889.9274864,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1713401890.0272369,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1713401891.877562,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1713401891.9652631,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1713401892.0621483,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1713401893.8239846,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1713401893.9089525,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1713401893.9141827,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1713401893.9207022,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1713401894.0175786,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1713401894.1883662,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1713401895.7744935,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1713401895.8479409,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1713401895.8550937,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1713401895.8643186,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1713401895.961679,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1713401895.9682069,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1713401896.0446615,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1713401896.1443305,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1713401896.3218629,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1713401897.8096054,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1713401897.9799101,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1713401898.0603721,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1713401898.1574156,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1713401898.3259537,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1713401900.0288928,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1713401900.1284776,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1713401900.36044,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1713401900.7065234,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1713401902.0056815,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1713401902.0819211,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1713401902.1801007,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1713401902.4068949,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1713401902.8274434,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1713401904.1072607,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1713401904.2795606,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1713401904.5419321,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1713401906.0560548,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1713401906.1549444,"name":"offline","context":{"idset":"10988"}}
+{"timestamp":1713401906.5813062,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1713401907.8605871,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1713401908.0237837,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1713401908.0948982,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1713401908.1937013,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1713404665.7728183,"name":"drain","context":{"idset":"10875","reason":"broker was unresponsive"}}
+{"timestamp":1713404667.4836411,"name":"online","context":{"idset":"10871,10873,10910,10921"}}
+{"timestamp":1713404667.616446,"name":"online","context":{"idset":"10922"}}
+{"timestamp":1713404667.7148523,"name":"online","context":{"idset":"10701,10901"}}
+{"timestamp":1713404667.9479907,"name":"online","context":{"idset":"10875,10913"}}
+{"timestamp":1713404668.1081235,"name":"online","context":{"idset":"10869,10891,10907,10919,10924,10928"}}
+{"timestamp":1713404668.2702236,"name":"online","context":{"idset":"10893,10908,10942"}}
+{"timestamp":1713404668.4314044,"name":"online","context":{"idset":"10870,10877,10882-10883,10888,10904,10923,10935,10939"}}
+{"timestamp":1713404668.6002169,"name":"online","context":{"idset":"10884,10886,10914,10931,10940,10943,10945"}}
+{"timestamp":1713404668.7666497,"name":"online","context":{"idset":"10917,10930,10944"}}
+{"timestamp":1713404668.930258,"name":"online","context":{"idset":"10899,10925,10927,10941"}}
+{"timestamp":1713404669.128484,"name":"online","context":{"idset":"10932"}}
+{"timestamp":1713404669.6178644,"name":"online","context":{"idset":"10938"}}
+{"timestamp":1713404669.7788715,"name":"online","context":{"idset":"10890"}}
+{"timestamp":1713404675.6986289,"name":"online","context":{"idset":"11528"}}
+{"timestamp":1713404676.4043584,"name":"online","context":{"idset":"10987"}}
+{"timestamp":1713404676.9910846,"name":"online","context":{"idset":"10951"}}
+{"timestamp":1713404677.1179383,"name":"online","context":{"idset":"10983,10985"}}
+{"timestamp":1713404677.3138514,"name":"online","context":{"idset":"10953,10958,10966"}}
+{"timestamp":1713404677.4765959,"name":"online","context":{"idset":"10976,10984"}}
+{"timestamp":1713404677.5854895,"name":"online","context":{"idset":"10948"}}
+{"timestamp":1713404677.769424,"name":"online","context":{"idset":"10975,10979,10991"}}
+{"timestamp":1713404677.9309211,"name":"online","context":{"idset":"10947,10963,10980,10982"}}
+{"timestamp":1713404678.0400736,"name":"online","context":{"idset":"10949,10956,10978,10994-10995"}}
+{"timestamp":1713404678.199034,"name":"online","context":{"idset":"10950,10955,10957,10962,10996"}}
+{"timestamp":1713404678.3055325,"name":"online","context":{"idset":"10964"}}
+{"timestamp":1713404678.7622583,"name":"online","context":{"idset":"10954,10969,10974,10990"}}
+{"timestamp":1713404679.0322266,"name":"online","context":{"idset":"10972"}}
+{"timestamp":1713404679.2810121,"name":"online","context":{"idset":"10970"}}
+{"timestamp":1713405573.8108659,"name":"offline","context":{"idset":"9533"}}
+{"timestamp":1713405595.8106289,"name":"offline","context":{"idset":"10957"}}
+{"timestamp":1713405601.8136528,"name":"offline","context":{"idset":"10983"}}
+{"timestamp":1713406173.7537327,"name":"online","context":{"idset":"9670,9685"}}
+{"timestamp":1713406221.2155991,"name":"online","context":{"idset":"9634"}}
+{"timestamp":1713406235.7256389,"name":"offline","context":{"idset":"952"}}
+{"timestamp":1713406235.7284992,"name":"offline","context":{"idset":"956"}}
+{"timestamp":1713406235.731312,"name":"offline","context":{"idset":"958"}}
+{"timestamp":1713406235.8119054,"name":"offline","context":{"idset":"960"}}
+{"timestamp":1713406273.7702346,"name":"online","context":{"idset":"9633"}}
+{"timestamp":1713406422.3519244,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1713406422.3672159,"name":"offline","context":{"idset":"11517"}}
+{"timestamp":1713406422.4049575,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1713406422.4719486,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1713406422.5514145,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1713406422.5578394,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1713406422.5657001,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1713406422.5719094,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1713406422.5753551,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1713406422.5789478,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1713406422.5950205,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1713406422.5985465,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1713406422.6022799,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1713406422.6182337,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1713406422.6235521,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1713406422.7451732,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1713406424.4013021,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1713406424.4756422,"name":"offline","context":{"idset":"11536"}}
+{"timestamp":1713406424.4861486,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1713406424.4889431,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1713406424.5173364,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1713406424.5206277,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1713406424.5923545,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1713406424.6667259,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1713406424.6724424,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1713406424.675612,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1713406424.6783562,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1713406424.6810343,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1713406424.6838875,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1713406424.6974978,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1713406424.7028401,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1713406424.7854726,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1713406426.4634035,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1713406426.4732919,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1713406426.4759691,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1713406426.5716305,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1713406426.5831435,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1713406426.5859222,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1713406426.6216457,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1713406426.6246676,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1713406426.6273973,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1713406426.6304166,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1713406426.6896181,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1713406426.7869544,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1713406426.7982795,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1713406426.8052955,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1713406426.8100259,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1713406426.8964655,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1713406428.4713111,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1713406428.5450606,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1713406428.5497005,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1713406428.555706,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1713406428.5584247,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1713406428.5632827,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1713406428.6609666,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1713406428.6703155,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1713406428.6738508,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1713406428.677387,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1713406428.7776465,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1713406428.7848883,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1713406428.7882764,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1713406428.793153,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1713406428.7965021,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1713406428.8931658,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1713406430.5103405,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1713406430.5150516,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1713406430.61166,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1713406430.6172416,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1713406430.6980333,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1713406430.7052033,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1713406430.7078381,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1713406430.7135036,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1713406430.7204676,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1713406430.7230749,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1713406430.7308502,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1713406430.8207893,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1713406430.8294628,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1713406430.8366692,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1713406430.8392885,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1713406430.9365201,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1713406432.5226326,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1713406432.6012721,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1713406432.6756108,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1713406432.6789947,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1713406432.6886623,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1713406432.7863925,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1713406432.7983711,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1713406432.8053031,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1713406432.8105254,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1713406432.8160589,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1713406432.8187602,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1713406432.8249366,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1713406432.8276658,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1713406432.8303342,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1713406432.9167974,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1713406433.0372167,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1713406434.5742331,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1713406434.6484463,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1713406434.6547449,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1713406434.6578143,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1713406434.7514505,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1713406434.7547965,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1713406434.8313999,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1713406434.8345037,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1713406434.933574,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1713406510.2932949,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1713406510.3616343,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1713406510.3706353,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1713406510.3756208,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1713406510.3807077,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1713406510.4750414,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1713406510.480181,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1713406510.4834807,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1713406510.4891093,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1713406510.4945641,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1713406510.4978242,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1713406510.5007033,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1713406510.5134218,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1713406510.5164509,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1713406510.519243,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1713406510.610532,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1713406512.2653694,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1713406512.3466847,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1713406512.4162567,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1713406512.4239068,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1713406512.4272633,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1713406512.5231373,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1713406512.528621,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1713406512.5335946,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1713406512.5386465,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1713406512.54338,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1713406512.5468314,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1713406512.5506003,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1713406512.5534554,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1713406512.564877,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1713406512.639003,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1713406512.801898,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1713406514.3400548,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1713406514.3492029,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1713406514.4302201,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1713406514.4968152,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1713406514.5077159,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1713406514.5136395,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1713406514.5163565,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1713406514.6079254,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1713406514.6179712,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1713406514.6241665,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1713406514.6296396,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1713406514.6322882,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1713406514.6350346,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1713406514.6387079,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1713406514.7312973,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1713406514.8465559,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1713406516.3559923,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1713406516.4234385,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1713406516.4823158,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1713406516.4887304,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1713406516.5626318,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1713406516.5766504,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1713406516.5844917,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1713406516.5894468,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1713406516.6800916,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1713406516.6881433,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1713406516.6950092,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1713406516.7017641,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1713406516.7043197,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1713406516.7068791,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1713406516.7999318,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1713406516.896697,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1713406518.3356853,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1713406518.4187474,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1713406518.4220021,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1713406518.4252098,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1713406518.5161119,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1713406518.5864532,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1713406518.5896521,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1713406518.5950594,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1713406518.5982497,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1713406518.6947725,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1713407025.61168,"name":"online","context":{"idset":"10905,10911"}}
+{"timestamp":1713407025.8610773,"name":"online","context":{"idset":"10880,10915"}}
+{"timestamp":1713407026.0960433,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1713407026.2684803,"name":"online","context":{"idset":"10918"}}
+{"timestamp":1713407026.3716643,"name":"online","context":{"idset":"10874,10881,10916,10926"}}
+{"timestamp":1713407026.4737341,"name":"online","context":{"idset":"10898"}}
+{"timestamp":1713407026.6328723,"name":"online","context":{"idset":"10936,10957"}}
+{"timestamp":1713407026.8278511,"name":"online","context":{"idset":"10892,10896,10961"}}
+{"timestamp":1713407026.9872696,"name":"online","context":{"idset":"10879,10887,10934,10971,10977,10986"}}
+{"timestamp":1713407027.0885906,"name":"online","context":{"idset":"10897,10903,10912,10952"}}
+{"timestamp":1713407027.275125,"name":"online","context":{"idset":"10933,10973"}}
+{"timestamp":1713407027.4348159,"name":"online","context":{"idset":"10900"}}
+{"timestamp":1713407027.8867805,"name":"online","context":{"idset":"10889"}}
+{"timestamp":1713407093.4844646,"name":"undrain","context":{"idset":"10875"}}
+{"timestamp":1713408233.9861102,"name":"online","context":{"idset":"9494,9507"}}
+{"timestamp":1713408234.4599204,"name":"online","context":{"idset":"9587"}}
+{"timestamp":1713408234.7320359,"name":"online","context":{"idset":"9533"}}
+{"timestamp":1713408359.1457269,"name":"undrain","context":{"idset":"9533"}}
+{"timestamp":1713409484.4097424,"name":"online","context":{"idset":"9378"}}
+{"timestamp":1713409484.5787418,"name":"online","context":{"idset":"9356"}}
+{"timestamp":1713409728.0042088,"name":"online","context":{"idset":"9280,9298"}}
+{"timestamp":1713409783.402338,"name":"online","context":{"idset":"9208"}}
+{"timestamp":1713409944.8326414,"name":"online","context":{"idset":"9916"}}
+{"timestamp":1713409945.0035007,"name":"online","context":{"idset":"9925,9940"}}
+{"timestamp":1713409945.1696267,"name":"online","context":{"idset":"9847,9955"}}
+{"timestamp":1713410322.1699243,"name":"online","context":{"idset":"11097"}}
+{"timestamp":1713410322.3430598,"name":"online","context":{"idset":"11002,11033,11036,11050,11070,11106"}}
+{"timestamp":1713410322.5049,"name":"online","context":{"idset":"10998,11016"}}
+{"timestamp":1713410322.6630988,"name":"online","context":{"idset":"11017,11028"}}
+{"timestamp":1713410322.8289814,"name":"online","context":{"idset":"11056"}}
+{"timestamp":1713410322.9874649,"name":"online","context":{"idset":"11076,11080,11092,11095,11099,11101"}}
+{"timestamp":1713410323.1457176,"name":"online","context":{"idset":"11023"}}
+{"timestamp":1713410323.3131309,"name":"online","context":{"idset":"11052,11090"}}
+{"timestamp":1713410673.6292644,"name":"online","context":{"idset":"15"}}
+{"timestamp":1713410673.8573027,"name":"online","context":{"idset":"34,43"}}
+{"timestamp":1713410674.0211196,"name":"online","context":{"idset":"2,13"}}
+{"timestamp":1713410674.3121998,"name":"online","context":{"idset":"6,8,35"}}
+{"timestamp":1713410674.5588906,"name":"online","context":{"idset":"10,24"}}
+{"timestamp":1713410674.7529526,"name":"online","context":{"idset":"1,4"}}
+{"timestamp":1713410674.8602483,"name":"online","context":{"idset":"18,32"}}
+{"timestamp":1713410675.0327585,"name":"online","context":{"idset":"3,26,30,46,58"}}
+{"timestamp":1713410675.1900895,"name":"online","context":{"idset":"7,11,16,22,36,55,59"}}
+{"timestamp":1713410676.1179676,"name":"online","context":{"idset":"31,60"}}
+{"timestamp":1713410676.1214194,"name":"online","context":{"idset":"17,54"}}
+{"timestamp":1713410728.1625345,"name":"online","context":{"idset":"10618,10663,10696,10717"}}
+{"timestamp":1713410728.7817693,"name":"online","context":{"idset":"10690"}}
+{"timestamp":1713410779.6581578,"name":"online","context":{"idset":"88"}}
+{"timestamp":1713410779.9472134,"name":"online","context":{"idset":"86"}}
+{"timestamp":1713410780.2733481,"name":"online","context":{"idset":"85,91-92,95"}}
+{"timestamp":1713410780.8735361,"name":"online","context":{"idset":"97,108"}}
+{"timestamp":1713410780.9763589,"name":"online","context":{"idset":"103"}}
+{"timestamp":1713410781.2571957,"name":"online","context":{"idset":"101"}}
+{"timestamp":1713410781.6093519,"name":"online","context":{"idset":"93,114"}}
+{"timestamp":1713410781.7851105,"name":"online","context":{"idset":"89-90,104"}}
+{"timestamp":1713410783.5232577,"name":"online","context":{"idset":"116"}}
+{"timestamp":1713410783.6345415,"name":"online","context":{"idset":"111"}}
+{"timestamp":1713410783.8427513,"name":"online","context":{"idset":"87"}}
+{"timestamp":1713410784.5458169,"name":"online","context":{"idset":"115"}}
+{"timestamp":1713410785.0258288,"name":"online","context":{"idset":"94,110"}}
+{"timestamp":1713410785.7031107,"name":"online","context":{"idset":"98"}}
+{"timestamp":1713410785.8383656,"name":"online","context":{"idset":"136"}}
+{"timestamp":1713410786.3240647,"name":"online","context":{"idset":"138"}}
+{"timestamp":1713410786.40645,"name":"online","context":{"idset":"119"}}
+{"timestamp":1713410786.484581,"name":"online","context":{"idset":"96"}}
+{"timestamp":1713410786.6216276,"name":"online","context":{"idset":"146"}}
+{"timestamp":1713410787.120265,"name":"online","context":{"idset":"102,120"}}
+{"timestamp":1713410787.1271045,"name":"online","context":{"idset":"152"}}
+{"timestamp":1713410787.2230866,"name":"online","context":{"idset":"137"}}
+{"timestamp":1713410787.4259031,"name":"online","context":{"idset":"162"}}
+{"timestamp":1713410787.7175274,"name":"online","context":{"idset":"134,170"}}
+{"timestamp":1713410787.8300622,"name":"online","context":{"idset":"160,172"}}
+{"timestamp":1713410787.9227862,"name":"online","context":{"idset":"131"}}
+{"timestamp":1713410788.1534052,"name":"online","context":{"idset":"113,130,147,153,231"}}
+{"timestamp":1713410788.2384539,"name":"online","context":{"idset":"123,228"}}
+{"timestamp":1713410788.3428252,"name":"online","context":{"idset":"126"}}
+{"timestamp":1713410788.3483324,"name":"online","context":{"idset":"179"}}
+{"timestamp":1713410788.4483063,"name":"online","context":{"idset":"232"}}
+{"timestamp":1713410788.5484824,"name":"online","context":{"idset":"156,203,243"}}
+{"timestamp":1713410788.6501007,"name":"online","context":{"idset":"133"}}
+{"timestamp":1713410788.7490859,"name":"online","context":{"idset":"100"}}
+{"timestamp":1713410788.8258288,"name":"online","context":{"idset":"201,206"}}
+{"timestamp":1713410788.9091895,"name":"online","context":{"idset":"105,107,184"}}
+{"timestamp":1713410788.9958699,"name":"online","context":{"idset":"118,127,208"}}
+{"timestamp":1713410789.0824828,"name":"online","context":{"idset":"180,294"}}
+{"timestamp":1713410789.1610844,"name":"online","context":{"idset":"168"}}
+{"timestamp":1713410789.2356088,"name":"online","context":{"idset":"289"}}
+{"timestamp":1713410789.3034899,"name":"online","context":{"idset":"157"}}
+{"timestamp":1713410789.393513,"name":"online","context":{"idset":"154,165,199"}}
+{"timestamp":1713410789.4683273,"name":"online","context":{"idset":"186"}}
+{"timestamp":1713410789.5608542,"name":"online","context":{"idset":"183"}}
+{"timestamp":1713410789.6375275,"name":"online","context":{"idset":"178,282"}}
+{"timestamp":1713410789.8001666,"name":"online","context":{"idset":"143,188,237,299,302,304,319"}}
+{"timestamp":1713410789.8694296,"name":"online","context":{"idset":"317"}}
+{"timestamp":1713410790.0420151,"name":"online","context":{"idset":"158,169,223,318,320,323"}}
+{"timestamp":1713410790.2122114,"name":"online","context":{"idset":"324"}}
+{"timestamp":1713410790.3063231,"name":"online","context":{"idset":"125,129,197,216,229,316,332"}}
+{"timestamp":1713410790.3961849,"name":"online","context":{"idset":"159,193,234,249"}}
+{"timestamp":1713410790.4823511,"name":"online","context":{"idset":"117,144,213-214,225,306"}}
+{"timestamp":1713410790.5704036,"name":"online","context":{"idset":"106"}}
+{"timestamp":1713410790.6715908,"name":"online","context":{"idset":"190"}}
+{"timestamp":1713410790.7588513,"name":"online","context":{"idset":"155,174,198,285,298,301,312"}}
+{"timestamp":1713410790.9142177,"name":"online","context":{"idset":"163,175,235,308"}}
+{"timestamp":1713410790.9875994,"name":"online","context":{"idset":"122,145,149,167,173,191,202,236,278,326"}}
+{"timestamp":1713410791.0629444,"name":"online","context":{"idset":"148,209,250,330"}}
+{"timestamp":1713410791.1829653,"name":"online","context":{"idset":"212"}}
+{"timestamp":1713410791.2655563,"name":"online","context":{"idset":"189"}}
+{"timestamp":1713410791.342397,"name":"online","context":{"idset":"166,244"}}
+{"timestamp":1713410791.4252958,"name":"online","context":{"idset":"150"}}
+{"timestamp":1713410791.528523,"name":"online","context":{"idset":"195,286,288,329"}}
+{"timestamp":1713410791.6776237,"name":"online","context":{"idset":"132,284,310"}}
+{"timestamp":1713410791.7908871,"name":"online","context":{"idset":"239"}}
+{"timestamp":1713410791.8712416,"name":"online","context":{"idset":"287"}}
+{"timestamp":1713410791.9744108,"name":"online","context":{"idset":"171,226,240,245,322"}}
+{"timestamp":1713410792.1357636,"name":"online","context":{"idset":"161,196,338"}}
+{"timestamp":1713410792.3566203,"name":"online","context":{"idset":"141,187,224,238,247,297"}}
+{"timestamp":1713410792.5221145,"name":"online","context":{"idset":"337"}}
+{"timestamp":1713410792.6069002,"name":"online","context":{"idset":"182,210"}}
+{"timestamp":1713410792.7149134,"name":"online","context":{"idset":"99"}}
+{"timestamp":1713410792.8852062,"name":"online","context":{"idset":"124,192,242,307"}}
+{"timestamp":1713410792.9597452,"name":"online","context":{"idset":"283,321"}}
+{"timestamp":1713410793.126559,"name":"online","context":{"idset":"151,164"}}
+{"timestamp":1713410793.2934773,"name":"online","context":{"idset":"295"}}
+{"timestamp":1713410793.4544148,"name":"online","context":{"idset":"194,215,246"}}
+{"timestamp":1713410793.5590529,"name":"online","context":{"idset":"204,328,339"}}
+{"timestamp":1713410793.6341412,"name":"online","context":{"idset":"300"}}
+{"timestamp":1713410793.7060823,"name":"online","context":{"idset":"128,222"}}
+{"timestamp":1713410793.7939508,"name":"online","context":{"idset":"109,200,291"}}
+{"timestamp":1713410793.8872628,"name":"online","context":{"idset":"176"}}
+{"timestamp":1713410793.9983511,"name":"online","context":{"idset":"311"}}
+{"timestamp":1713410794.1338317,"name":"online","context":{"idset":"135"}}
+{"timestamp":1713410794.2078509,"name":"online","context":{"idset":"331"}}
+{"timestamp":1713410794.2851226,"name":"online","context":{"idset":"333"}}
+{"timestamp":1713410794.3817992,"name":"online","context":{"idset":"248"}}
+{"timestamp":1713410794.8471622,"name":"online","context":{"idset":"112,205"}}
+{"timestamp":1713410794.9333942,"name":"online","context":{"idset":"140"}}
+{"timestamp":1713410795.0122857,"name":"online","context":{"idset":"181"}}
+{"timestamp":1713410795.3311982,"name":"online","context":{"idset":"342"}}
+{"timestamp":1713410795.4162714,"name":"online","context":{"idset":"185,207"}}
+{"timestamp":1713410796.0222716,"name":"online","context":{"idset":"230,309"}}
+{"timestamp":1713410796.2011328,"name":"online","context":{"idset":"211,218,241,251-252,281,290,305"}}
+{"timestamp":1713410796.3127785,"name":"online","context":{"idset":"280,293"}}
+{"timestamp":1713410796.4639442,"name":"online","context":{"idset":"177,345"}}
+{"timestamp":1713410796.5762784,"name":"online","context":{"idset":"292,343-344"}}
+{"timestamp":1713410796.6480587,"name":"online","context":{"idset":"142,325,346"}}
+{"timestamp":1713410796.7492907,"name":"online","context":{"idset":"227,340,347"}}
+{"timestamp":1713410796.852129,"name":"online","context":{"idset":"221,341"}}
+{"timestamp":1713410796.9581022,"name":"online","context":{"idset":"314,327,334,349"}}
+{"timestamp":1713410797.1249003,"name":"online","context":{"idset":"353"}}
+{"timestamp":1713410797.230303,"name":"online","context":{"idset":"350,352"}}
+{"timestamp":1713410797.3410437,"name":"online","context":{"idset":"139,351"}}
+{"timestamp":1713410797.4186957,"name":"online","context":{"idset":"315,335"}}
+{"timestamp":1713410797.5292227,"name":"online","context":{"idset":"354-356,368"}}
+{"timestamp":1713410797.6380999,"name":"online","context":{"idset":"357-358,360,365"}}
+{"timestamp":1713410797.8077157,"name":"online","context":{"idset":"359,361-362,364,367,371"}}
+{"timestamp":1713410797.9769957,"name":"online","context":{"idset":"363,366,369-370,372-376"}}
+{"timestamp":1713410798.1553435,"name":"online","context":{"idset":"279,377,379-380"}}
+{"timestamp":1713410798.2331817,"name":"online","context":{"idset":"233,378,382"}}
+{"timestamp":1713410798.4039567,"name":"online","context":{"idset":"219-220,277,296,303,313"}}
+{"timestamp":1713410798.5159857,"name":"online","context":{"idset":"381"}}
+{"timestamp":1713410798.8033917,"name":"online","context":{"idset":"383"}}
+{"timestamp":1713410798.9272101,"name":"online","context":{"idset":"384"}}
+{"timestamp":1713410799.1375639,"name":"online","context":{"idset":"385"}}
+{"timestamp":1713410799.2101321,"name":"online","context":{"idset":"386"}}
+{"timestamp":1713410799.5149803,"name":"online","context":{"idset":"387"}}
+{"timestamp":1713410799.7648761,"name":"online","context":{"idset":"388"}}
+{"timestamp":1713410799.9261758,"name":"online","context":{"idset":"389"}}
+{"timestamp":1713410800.5892994,"name":"online","context":{"idset":"390"}}
+{"timestamp":1713410800.682605,"name":"online","context":{"idset":"391"}}
+{"timestamp":1713410801.0972543,"name":"online","context":{"idset":"392"}}
+{"timestamp":1713410801.3545487,"name":"online","context":{"idset":"393"}}
+{"timestamp":1713410801.6389699,"name":"online","context":{"idset":"395"}}
+{"timestamp":1713410801.8925939,"name":"online","context":{"idset":"396,399"}}
+{"timestamp":1713410801.9654543,"name":"online","context":{"idset":"398"}}
+{"timestamp":1713410802.0393326,"name":"online","context":{"idset":"394"}}
+{"timestamp":1713410802.1733909,"name":"online","context":{"idset":"400"}}
+{"timestamp":1713410802.4235921,"name":"online","context":{"idset":"402"}}
+{"timestamp":1713410802.5145812,"name":"online","context":{"idset":"401"}}
+{"timestamp":1713410802.602845,"name":"online","context":{"idset":"403-405"}}
+{"timestamp":1713410802.7609806,"name":"online","context":{"idset":"406"}}
+{"timestamp":1713410802.8377225,"name":"online","context":{"idset":"407-408"}}
+{"timestamp":1713410802.9133465,"name":"online","context":{"idset":"409,414"}}
+{"timestamp":1713410803.0479898,"name":"online","context":{"idset":"411-412"}}
+{"timestamp":1713410803.2161968,"name":"online","context":{"idset":"413,415"}}
+{"timestamp":1713410803.3183663,"name":"online","context":{"idset":"416-421"}}
+{"timestamp":1713410803.5264568,"name":"online","context":{"idset":"422-425"}}
+{"timestamp":1713410803.7266257,"name":"online","context":{"idset":"426,428"}}
+{"timestamp":1713410803.8974986,"name":"online","context":{"idset":"429"}}
+{"timestamp":1713410804.0038486,"name":"online","context":{"idset":"427,430-431"}}
+{"timestamp":1713410804.2173798,"name":"online","context":{"idset":"432,435-437"}}
+{"timestamp":1713410804.3793991,"name":"online","context":{"idset":"433-434,439,441"}}
+{"timestamp":1713410804.4818223,"name":"online","context":{"idset":"438,440,442"}}
+{"timestamp":1713410804.6440563,"name":"online","context":{"idset":"443-444"}}
+{"timestamp":1713410804.9536581,"name":"online","context":{"idset":"469-470,473,475,477"}}
+{"timestamp":1713410805.1257098,"name":"online","context":{"idset":"471-472,474,476,478-487,489"}}
+{"timestamp":1713410805.295718,"name":"online","context":{"idset":"488,490,492,573,577"}}
+{"timestamp":1713410805.4683805,"name":"online","context":{"idset":"576,578-579"}}
+{"timestamp":1713410805.6392708,"name":"online","context":{"idset":"575,580"}}
+{"timestamp":1713410983.0884507,"name":"online","context":{"idset":"574"}}
+{"timestamp":1713411537.8113711,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1713411862.8593152,"name":"online","context":{"idset":"9"}}
+{"timestamp":1713411863.0477486,"name":"online","context":{"idset":"37"}}
+{"timestamp":1713411863.2881808,"name":"online","context":{"idset":"20,29"}}
+{"timestamp":1713411863.4595461,"name":"online","context":{"idset":"51"}}
+{"timestamp":1713411863.5674393,"name":"online","context":{"idset":"28"}}
+{"timestamp":1713411863.7476373,"name":"online","context":{"idset":"25,53"}}
+{"timestamp":1713411863.9133811,"name":"online","context":{"idset":"33"}}
+{"timestamp":1713411864.0861819,"name":"online","context":{"idset":"23,38,42,44"}}
+{"timestamp":1713411864.2583921,"name":"online","context":{"idset":"21,41"}}
+{"timestamp":1713411864.4345326,"name":"online","context":{"idset":"5,14,27,45,49"}}
+{"timestamp":1713411864.6079831,"name":"online","context":{"idset":"40,47,52"}}
+{"timestamp":1713411864.7140446,"name":"online","context":{"idset":"50,57"}}
+{"timestamp":1713411864.8958795,"name":"online","context":{"idset":"39"}}
+{"timestamp":1713412540.7011521,"name":"online","context":{"idset":"11522"}}
+{"timestamp":1713412541.1886022,"name":"online","context":{"idset":"11515,11519"}}
+{"timestamp":1713412541.5128207,"name":"online","context":{"idset":"11510,11517"}}
+{"timestamp":1713412541.6905148,"name":"online","context":{"idset":"11512-11514,11523"}}
+{"timestamp":1713412541.7950766,"name":"online","context":{"idset":"11509,11511"}}
+{"timestamp":1713412541.9045908,"name":"online","context":{"idset":"11518,11521,11524"}}
+{"timestamp":1713412542.0703104,"name":"online","context":{"idset":"11516,11520"}}
+{"timestamp":1713412551.7795322,"name":"drain","context":{"idset":"11531","reason":"broker was unresponsive"}}
+{"timestamp":1713412552.6996467,"name":"online","context":{"idset":"11525-11527"}}
+{"timestamp":1713412553.1987872,"name":"online","context":{"idset":"11538"}}
+{"timestamp":1713412553.3694921,"name":"online","context":{"idset":"11528-11529"}}
+{"timestamp":1713412553.818716,"name":"online","context":{"idset":"11532-11533,11536"}}
+{"timestamp":1713412553.9308915,"name":"online","context":{"idset":"11530-11531,11534"}}
+{"timestamp":1713412554.0351417,"name":"online","context":{"idset":"11539"}}
+{"timestamp":1713412554.2034197,"name":"online","context":{"idset":"11535,11537,11540"}}
+{"timestamp":1713412564.2990706,"name":"online","context":{"idset":"11541"}}
+{"timestamp":1713412564.4029839,"name":"online","context":{"idset":"11542"}}
+{"timestamp":1713412565.2145693,"name":"online","context":{"idset":"11543,11545-11546"}}
+{"timestamp":1713412565.3209543,"name":"online","context":{"idset":"11544"}}
+{"timestamp":1713412565.4916344,"name":"online","context":{"idset":"11547,11549"}}
+{"timestamp":1713412565.689852,"name":"online","context":{"idset":"11548,11550-11551"}}
+{"timestamp":1713412565.9838557,"name":"online","context":{"idset":"11553,11555-11556"}}
+{"timestamp":1713412566.2032037,"name":"online","context":{"idset":"11552,11554"}}
+{"timestamp":1713412575.6491411,"name":"online","context":{"idset":"11557"}}
+{"timestamp":1713412576.4991841,"name":"online","context":{"idset":"11558-11559"}}
+{"timestamp":1713412577.0176463,"name":"online","context":{"idset":"11560"}}
+{"timestamp":1713412577.4038172,"name":"online","context":{"idset":"11563"}}
+{"timestamp":1713412577.5887094,"name":"online","context":{"idset":"11561-11562,11564,11568"}}
+{"timestamp":1713412577.7721932,"name":"online","context":{"idset":"11565"}}
+{"timestamp":1713412577.9580605,"name":"online","context":{"idset":"11570-11572"}}
+{"timestamp":1713412578.1415362,"name":"online","context":{"idset":"11569"}}
+{"timestamp":1713412578.3467481,"name":"online","context":{"idset":"11566-11567"}}
+{"timestamp":1713412587.6428642,"name":"online","context":{"idset":"11573"}}
+{"timestamp":1713412588.3803494,"name":"online","context":{"idset":"11574"}}
+{"timestamp":1713412588.9478652,"name":"online","context":{"idset":"11575-11576"}}
+{"timestamp":1713412589.1300104,"name":"online","context":{"idset":"11579"}}
+{"timestamp":1713412589.3046923,"name":"online","context":{"idset":"11577"}}
+{"timestamp":1713412589.4740853,"name":"online","context":{"idset":"11580,11585"}}
+{"timestamp":1713412589.648459,"name":"online","context":{"idset":"11578,11583-11584"}}
+{"timestamp":1713412589.7617326,"name":"online","context":{"idset":"11581"}}
+{"timestamp":1713412589.8860743,"name":"online","context":{"idset":"11582,11586,11588"}}
+{"timestamp":1713412589.9982789,"name":"online","context":{"idset":"11587"}}
+{"timestamp":1713412599.1230021,"name":"online","context":{"idset":"11589"}}
+{"timestamp":1713412600.3149135,"name":"online","context":{"idset":"11590"}}
+{"timestamp":1713412600.7091715,"name":"online","context":{"idset":"11592"}}
+{"timestamp":1713412600.9724085,"name":"online","context":{"idset":"11591"}}
+{"timestamp":1713412601.1897743,"name":"online","context":{"idset":"11593,11597"}}
+{"timestamp":1713412601.4321046,"name":"online","context":{"idset":"11594,11596,11598-11599"}}
+{"timestamp":1713412601.6009028,"name":"online","context":{"idset":"11600,11603"}}
+{"timestamp":1713412601.7733839,"name":"online","context":{"idset":"11601-11602"}}
+{"timestamp":1713412601.9390008,"name":"online","context":{"idset":"11595,11604"}}
+{"timestamp":1713412610.9073329,"name":"online","context":{"idset":"11605"}}
+{"timestamp":1713412612.0709054,"name":"online","context":{"idset":"11606"}}
+{"timestamp":1713412612.2566984,"name":"online","context":{"idset":"11607"}}
+{"timestamp":1713412612.7408845,"name":"online","context":{"idset":"11608-11609"}}
+{"timestamp":1713412613.1331651,"name":"online","context":{"idset":"11612"}}
+{"timestamp":1713412613.318928,"name":"online","context":{"idset":"11611,11613"}}
+{"timestamp":1713412613.4240963,"name":"online","context":{"idset":"11610,11614,11616,11618"}}
+{"timestamp":1713412613.5315893,"name":"online","context":{"idset":"11619-11620"}}
+{"timestamp":1713412613.7181201,"name":"online","context":{"idset":"11617"}}
+{"timestamp":1713412613.9024866,"name":"online","context":{"idset":"11615"}}
+{"timestamp":1713412622.910831,"name":"online","context":{"idset":"11621"}}
+{"timestamp":1713412623.9238427,"name":"online","context":{"idset":"11622"}}
+{"timestamp":1713412624.595612,"name":"online","context":{"idset":"11623,11625"}}
+{"timestamp":1713412624.8625875,"name":"online","context":{"idset":"11624"}}
+{"timestamp":1713412625.1587973,"name":"online","context":{"idset":"11627,11631"}}
+{"timestamp":1713412625.3699522,"name":"online","context":{"idset":"11626,11629,11634-11635"}}
+{"timestamp":1713412625.5473001,"name":"online","context":{"idset":"11628,11630,11632-11633"}}
+{"timestamp":1713412625.8743019,"name":"online","context":{"idset":"11636"}}
+{"timestamp":1713412669.7276893,"name":"online","context":{"idset":"12"}}
+{"timestamp":1713412696.0859149,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1713412724.1026051,"name":"online","context":{"idset":"19"}}
+{"timestamp":1713412744.2246263,"name":"online","context":{"idset":"56"}}
+{"timestamp":1713412744.8953753,"name":"online","context":{"idset":"48"}}
+{"timestamp":1713412747.3699594,"name":"online","context":{"idset":"11531"}}
+{"timestamp":1713412842.2057154,"name":"undrain","context":{"idset":"11531"}}
+{"timestamp":1713413651.6391521,"name":"online","context":{"idset":"10261,10265,10315,10332"}}
+{"timestamp":1713413651.8888772,"name":"online","context":{"idset":"10230,10238,10241,10268"}}
+{"timestamp":1713413652.0790653,"name":"online","context":{"idset":"10283"}}
+{"timestamp":1713413652.2622361,"name":"online","context":{"idset":"10234,10281,10319"}}
+{"timestamp":1713413823.4811769,"name":"online","context":{"idset":"10373,10448"}}
+{"timestamp":1713413823.5893321,"name":"online","context":{"idset":"10420"}}
+{"timestamp":1713413823.868253,"name":"online","context":{"idset":"10372"}}
+{"timestamp":1713414733.7847099,"name":"online","context":{"idset":"10607,10611"}}
+{"timestamp":1713414733.9651086,"name":"online","context":{"idset":"10491,10503,10537,10549,10552,10563"}}
+{"timestamp":1713414734.1476774,"name":"online","context":{"idset":"10507,10522,10548,10566,10610,10612"}}
+{"timestamp":1713414734.3309119,"name":"online","context":{"idset":"10544,10572"}}
+{"timestamp":1713414734.5102997,"name":"online","context":{"idset":"10580"}}
+{"timestamp":1713414848.2643099,"name":"online","context":{"idset":"10885"}}
+{"timestamp":1713414848.5896184,"name":"online","context":{"idset":"10960"}}
+{"timestamp":1713414848.7820697,"name":"online","context":{"idset":"10877-10878,10913,10923-10924,10931"}}
+{"timestamp":1713414848.9540555,"name":"online","context":{"idset":"10871,10891,10893,10994"}}
+{"timestamp":1713414849.1279829,"name":"online","context":{"idset":"10876,10954"}}
+{"timestamp":1713414849.2386923,"name":"online","context":{"idset":"10870,10873,10882,10894,10944"}}
+{"timestamp":1713414849.4054921,"name":"online","context":{"idset":"10886,10906,10940,10996"}}
+{"timestamp":1713414849.5685713,"name":"online","context":{"idset":"10908,10910"}}
+{"timestamp":1713414849.7336199,"name":"online","context":{"idset":"10869,10938,10951"}}
+{"timestamp":1713414849.9085348,"name":"online","context":{"idset":"10917"}}
+{"timestamp":1713415129.2426405,"name":"online","context":{"idset":"9888"}}
+{"timestamp":1713415129.5360804,"name":"online","context":{"idset":"9892"}}
+{"timestamp":1713415944.870934,"name":"online","context":{"idset":"11321"}}
+{"timestamp":1713415949.9225776,"name":"online","context":{"idset":"11357"}}
+{"timestamp":1713416573.5180233,"name":"undrain","context":{"idset":"11641-11642"}}
+{"timestamp":1713416826.9806678,"name":"online","context":{"idset":"9719,9819"}}
+{"timestamp":1713416827.5169804,"name":"online","context":{"idset":"9771,9777"}}
+{"timestamp":1713416827.8039474,"name":"online","context":{"idset":"9807"}}
+{"timestamp":1713416828.027787,"name":"online","context":{"idset":"9795"}}
+{"timestamp":1713416828.5216882,"name":"online","context":{"idset":"9743"}}
+{"timestamp":1713417315.9422941,"name":"online","context":{"idset":"10004,11041"}}
+{"timestamp":1713417316.1865027,"name":"online","context":{"idset":"11212,11258"}}
+{"timestamp":1713417316.3713627,"name":"online","context":{"idset":"11019,11203"}}
+{"timestamp":1713417316.5643859,"name":"online","context":{"idset":"11048,11361"}}
+{"timestamp":1713417317.0440142,"name":"online","context":{"idset":"11334"}}
+{"timestamp":1713417317.1569397,"name":"online","context":{"idset":"11209"}}
+{"timestamp":1713417317.4198954,"name":"online","context":{"idset":"11352"}}
+{"timestamp":1713417551.8119483,"name":"offline","context":{"idset":"9678"}}
+{"timestamp":1713417577.8113742,"name":"offline","context":{"idset":"9476"}}
+{"timestamp":1713419499.8124251,"name":"offline","context":{"idset":"9788"}}
+{"timestamp":1713421017.8116601,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1713422693.5790069,"name":"online","context":{"idset":"11533"}}
+{"timestamp":1713425875.8126538,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1713427023.8116119,"name":"offline","context":{"idset":"9934"}}
+{"timestamp":1713449281.7233465,"name":"offline","context":{"idset":"10229"}}
+{"timestamp":1713449281.8114541,"name":"offline","context":{"idset":"10230"}}
+{"timestamp":1713449283.7283113,"name":"offline","context":{"idset":"10231"}}
+{"timestamp":1713449283.7318828,"name":"offline","context":{"idset":"10232"}}
+{"timestamp":1713449283.8133519,"name":"offline","context":{"idset":"10234"}}
+{"timestamp":1713449285.7280734,"name":"offline","context":{"idset":"10237"}}
+{"timestamp":1713449285.7314947,"name":"offline","context":{"idset":"10238"}}
+{"timestamp":1713449285.8120065,"name":"offline","context":{"idset":"10239"}}
+{"timestamp":1713449287.7251778,"name":"offline","context":{"idset":"10240"}}
+{"timestamp":1713449287.7287269,"name":"offline","context":{"idset":"10241"}}
+{"timestamp":1713449287.8121281,"name":"offline","context":{"idset":"10243"}}
+{"timestamp":1713449289.8130679,"name":"offline","context":{"idset":"10244"}}
+{"timestamp":1713449787.8122694,"name":"offline","context":{"idset":"10245"}}
+{"timestamp":1713449789.729732,"name":"offline","context":{"idset":"10246"}}
+{"timestamp":1713449789.7332559,"name":"offline","context":{"idset":"10247"}}
+{"timestamp":1713449789.7366133,"name":"offline","context":{"idset":"10248"}}
+{"timestamp":1713449789.8123989,"name":"offline","context":{"idset":"10249"}}
+{"timestamp":1713449791.7359767,"name":"offline","context":{"idset":"10250"}}
+{"timestamp":1713449791.739363,"name":"offline","context":{"idset":"10251"}}
+{"timestamp":1713449791.7427864,"name":"offline","context":{"idset":"10252"}}
+{"timestamp":1713449791.7461648,"name":"offline","context":{"idset":"10253"}}
+{"timestamp":1713449791.8122699,"name":"offline","context":{"idset":"10254"}}
+{"timestamp":1713449793.7320073,"name":"offline","context":{"idset":"10255"}}
+{"timestamp":1713449793.737606,"name":"offline","context":{"idset":"10256"}}
+{"timestamp":1713449793.7431226,"name":"offline","context":{"idset":"10257"}}
+{"timestamp":1713449793.8152425,"name":"offline","context":{"idset":"10258"}}
+{"timestamp":1713449795.7307446,"name":"offline","context":{"idset":"10259"}}
+{"timestamp":1713449795.8128262,"name":"offline","context":{"idset":"10260"}}
+{"timestamp":1713449819.7311516,"name":"offline","context":{"idset":"10261"}}
+{"timestamp":1713449819.8135788,"name":"offline","context":{"idset":"10262"}}
+{"timestamp":1713449821.7273195,"name":"offline","context":{"idset":"10263"}}
+{"timestamp":1713449821.7333808,"name":"offline","context":{"idset":"10264"}}
+{"timestamp":1713449821.8115232,"name":"offline","context":{"idset":"10265"}}
+{"timestamp":1713449823.7240188,"name":"offline","context":{"idset":"10266"}}
+{"timestamp":1713449823.8113272,"name":"offline","context":{"idset":"10268"}}
+{"timestamp":1713449825.7297089,"name":"offline","context":{"idset":"10270"}}
+{"timestamp":1713449825.7374017,"name":"offline","context":{"idset":"10271"}}
+{"timestamp":1713449825.7448685,"name":"offline","context":{"idset":"10272"}}
+{"timestamp":1713449825.8124712,"name":"offline","context":{"idset":"10273"}}
+{"timestamp":1713449827.7332165,"name":"offline","context":{"idset":"10269"}}
+{"timestamp":1713449827.7372861,"name":"offline","context":{"idset":"10274"}}
+{"timestamp":1713449827.7412703,"name":"offline","context":{"idset":"10275"}}
+{"timestamp":1713449827.7452466,"name":"offline","context":{"idset":"10276"}}
+{"timestamp":1713449827.7493122,"name":"offline","context":{"idset":"10277"}}
+{"timestamp":1713449827.810694,"name":"offline","context":{"idset":"10278"}}
+{"timestamp":1713449829.7327876,"name":"offline","context":{"idset":"10279"}}
+{"timestamp":1713449829.7368131,"name":"offline","context":{"idset":"10280"}}
+{"timestamp":1713449829.7410109,"name":"offline","context":{"idset":"10281"}}
+{"timestamp":1713449829.744997,"name":"offline","context":{"idset":"10282"}}
+{"timestamp":1713449829.8120446,"name":"offline","context":{"idset":"10283"}}
+{"timestamp":1713449831.7384717,"name":"offline","context":{"idset":"10284"}}
+{"timestamp":1713449831.7443984,"name":"offline","context":{"idset":"10285"}}
+{"timestamp":1713449831.7501411,"name":"offline","context":{"idset":"10286"}}
+{"timestamp":1713449831.8112104,"name":"offline","context":{"idset":"10287"}}
+{"timestamp":1713449833.7288132,"name":"offline","context":{"idset":"10288"}}
+{"timestamp":1713449833.7362349,"name":"offline","context":{"idset":"10289"}}
+{"timestamp":1713449833.7434371,"name":"offline","context":{"idset":"10290"}}
+{"timestamp":1713449833.8111613,"name":"offline","context":{"idset":"10291"}}
+{"timestamp":1713449835.7300503,"name":"offline","context":{"idset":"10292"}}
+{"timestamp":1713449835.7362998,"name":"offline","context":{"idset":"10293"}}
+{"timestamp":1713449835.7419498,"name":"offline","context":{"idset":"10294"}}
+{"timestamp":1713449835.8115461,"name":"offline","context":{"idset":"10295"}}
+{"timestamp":1713449837.7281325,"name":"offline","context":{"idset":"10296"}}
+{"timestamp":1713449837.7357857,"name":"offline","context":{"idset":"10297"}}
+{"timestamp":1713449837.7430067,"name":"offline","context":{"idset":"10298"}}
+{"timestamp":1713449837.810848,"name":"offline","context":{"idset":"10299"}}
+{"timestamp":1713449839.7304082,"name":"offline","context":{"idset":"10300"}}
+{"timestamp":1713449839.7380152,"name":"offline","context":{"idset":"10301"}}
+{"timestamp":1713449839.7452328,"name":"offline","context":{"idset":"10302"}}
+{"timestamp":1713449839.8121819,"name":"offline","context":{"idset":"10303"}}
+{"timestamp":1713449841.7335603,"name":"offline","context":{"idset":"10267"}}
+{"timestamp":1713449841.7367749,"name":"offline","context":{"idset":"10304"}}
+{"timestamp":1713449841.7399313,"name":"offline","context":{"idset":"10305"}}
+{"timestamp":1713449841.7431099,"name":"offline","context":{"idset":"10306"}}
+{"timestamp":1713449841.7462733,"name":"offline","context":{"idset":"10307"}}
+{"timestamp":1713449841.8113821,"name":"offline","context":{"idset":"10308"}}
+{"timestamp":1713449843.7232082,"name":"offline","context":{"idset":"10309"}}
+{"timestamp":1713449843.8114765,"name":"offline","context":{"idset":"10313"}}
+{"timestamp":1713449845.7315979,"name":"offline","context":{"idset":"10314"}}
+{"timestamp":1713449845.7355814,"name":"offline","context":{"idset":"10315"}}
+{"timestamp":1713449845.7394204,"name":"offline","context":{"idset":"10316"}}
+{"timestamp":1713449845.8126049,"name":"offline","context":{"idset":"10317"}}
+{"timestamp":1713449847.7275841,"name":"offline","context":{"idset":"10318"}}
+{"timestamp":1713449847.7308233,"name":"offline","context":{"idset":"10319"}}
+{"timestamp":1713449847.7339916,"name":"offline","context":{"idset":"10320"}}
+{"timestamp":1713449847.8119347,"name":"offline","context":{"idset":"10321"}}
+{"timestamp":1713449849.7305167,"name":"offline","context":{"idset":"10322"}}
+{"timestamp":1713449849.7345426,"name":"offline","context":{"idset":"10323"}}
+{"timestamp":1713449849.7385323,"name":"offline","context":{"idset":"10324"}}
+{"timestamp":1713449849.8122306,"name":"offline","context":{"idset":"10326"}}
+{"timestamp":1713449851.7299986,"name":"offline","context":{"idset":"10328"}}
+{"timestamp":1713449851.8151846,"name":"offline","context":{"idset":"10330"}}
+{"timestamp":1713449853.7262993,"name":"offline","context":{"idset":"10332"}}
+{"timestamp":1713449853.7300096,"name":"offline","context":{"idset":"10333"}}
+{"timestamp":1713449853.8109,"name":"offline","context":{"idset":"10334"}}
+{"timestamp":1713449855.7224116,"name":"offline","context":{"idset":"10336"}}
+{"timestamp":1713449855.8108225,"name":"offline","context":{"idset":"10337"}}
+{"timestamp":1713449857.7298124,"name":"offline","context":{"idset":"10339"}}
+{"timestamp":1713449857.7336516,"name":"offline","context":{"idset":"10340"}}
+{"timestamp":1713449857.7373521,"name":"offline","context":{"idset":"10341"}}
+{"timestamp":1713449857.8114741,"name":"offline","context":{"idset":"10342"}}
+{"timestamp":1713449859.733598,"name":"offline","context":{"idset":"10343"}}
+{"timestamp":1713449859.7364249,"name":"offline","context":{"idset":"10344"}}
+{"timestamp":1713449859.7391746,"name":"offline","context":{"idset":"10345"}}
+{"timestamp":1713449859.8116095,"name":"offline","context":{"idset":"10346"}}
+{"timestamp":1713449861.7286282,"name":"offline","context":{"idset":"10347"}}
+{"timestamp":1713449861.8098984,"name":"offline","context":{"idset":"10348"}}
+{"timestamp":1713449863.7305524,"name":"offline","context":{"idset":"10349"}}
+{"timestamp":1713449863.7343743,"name":"offline","context":{"idset":"10350"}}
+{"timestamp":1713449863.8116107,"name":"offline","context":{"idset":"10352"}}
+{"timestamp":1713449865.7328193,"name":"offline","context":{"idset":"10353"}}
+{"timestamp":1713449865.7364805,"name":"offline","context":{"idset":"10354"}}
+{"timestamp":1713449865.7416794,"name":"offline","context":{"idset":"10355"}}
+{"timestamp":1713449865.8117952,"name":"offline","context":{"idset":"10356"}}
+{"timestamp":1713449875.8121023,"name":"offline","context":{"idset":"10338"}}
+{"timestamp":1713449891.8114245,"name":"offline","context":{"idset":"10310"}}
+{"timestamp":1713449913.8126478,"name":"offline","context":{"idset":"10351"}}
+{"timestamp":1713450323.7256496,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1713450323.7289238,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1713450323.7321415,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1713450323.8127377,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1713450325.7265038,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1713450325.7296329,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1713450325.7328866,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1713450325.8113401,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1713450327.730854,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1713450327.7347455,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1713450327.7392223,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1713450327.7431142,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1713450327.8120081,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1713450329.7251749,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1713450329.8142974,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1713450331.7323895,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1713450331.7361789,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1713450331.739742,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1713450331.8133254,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1713450333.7329292,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1713450333.7369127,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1713450333.7407031,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1713450333.8118339,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1713450335.7265685,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1713450335.7300785,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1713450335.814281,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1713450337.7296622,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1713450337.7346637,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1713450337.7384937,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1713450337.8115478,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1713450339.7311389,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1713450339.7349701,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1713450339.7387209,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1713450339.8118925,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1713450341.726079,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1713450341.7296095,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1713450341.8116732,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1713450343.731667,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1713450343.7351377,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1713450343.7386646,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1713450343.8140593,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1713450345.721916,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1713450345.8106012,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1713450347.7375443,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1713450347.7409201,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1713450347.7444048,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1713450347.7479491,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1713450347.8113327,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1713450349.7311075,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1713450349.7347441,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1713450349.7383709,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1713450349.8121912,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1713450351.7238529,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1713450351.8105047,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1713450353.7295077,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1713450353.7326496,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1713450353.7357571,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1713450353.8115356,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1713450355.7337329,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1713450355.7374778,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1713450355.7410939,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1713450355.8103156,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1713450357.7434874,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1713450357.7495742,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1713450357.7557342,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1713450357.836864,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1713450359.7182064,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1713450359.8121397,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1713450361.7322614,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1713450361.7362578,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1713450361.7400887,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1713450361.7443833,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1713450361.8122184,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1713450363.7305121,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1713450363.7353418,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1713450363.739599,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1713450363.8123496,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1713450365.7247376,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1713450365.7302551,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1713450365.8116505,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1713450367.729212,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1713450367.7331014,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1713450367.8114638,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1713450369.729816,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1713450369.7329462,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1713450369.736021,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1713450369.8109002,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1713450371.7289584,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1713450371.7347577,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1713450371.8098311,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1713450373.7293613,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1713450373.7324467,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1713450373.8116655,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1713450375.7259288,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1713450375.7290096,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1713450375.8116462,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1713450377.7255831,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1713450377.8136775,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1713450379.7245686,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1713450379.7283518,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1713450379.8120811,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1713450381.7254033,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1713450381.7285142,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1713450381.7316387,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1713450381.8110836,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1713450421.8131778,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1713451563.7309625,"name":"undrain","context":{"idset":"162,409"}}
+{"timestamp":1713451572.765372,"name":"undrain","context":{"idset":"404"}}
+{"timestamp":1713451584.2158923,"name":"undrain","context":{"idset":"474"}}
+{"timestamp":1713451594.6398733,"name":"undrain","context":{"idset":"475-476,483-484"}}
+{"timestamp":1713451600.8194523,"name":"undrain","context":{"idset":"808"}}
+{"timestamp":1713452592.5791361,"name":"offline","context":{"idset":"9819"}}
+{"timestamp":1713452621.8450394,"name":"online","context":{"idset":"10064"}}
+{"timestamp":1713452806.9638608,"name":"online","context":{"idset":"10883"}}
+{"timestamp":1713452807.0790236,"name":"online","context":{"idset":"10921"}}
+{"timestamp":1713452807.283915,"name":"online","context":{"idset":"10919"}}
+{"timestamp":1713453743.5035467,"name":"online","context":{"idset":"10890"}}
+{"timestamp":1713453743.9852524,"name":"online","context":{"idset":"10895"}}
+{"timestamp":1713453744.1754291,"name":"online","context":{"idset":"10888,10899"}}
+{"timestamp":1713453744.3536727,"name":"online","context":{"idset":"10884,10901,10920,10928"}}
+{"timestamp":1713453744.4587104,"name":"online","context":{"idset":"10904,10909,10925,10927"}}
+{"timestamp":1713453744.6558692,"name":"online","context":{"idset":"10907"}}
+{"timestamp":1713453744.9668438,"name":"online","context":{"idset":"10914"}}
+{"timestamp":1713453745.1542609,"name":"online","context":{"idset":"10902"}}
+{"timestamp":1713453745.365202,"name":"online","context":{"idset":"10922"}}
+{"timestamp":1713453755.2658997,"name":"online","context":{"idset":"10929"}}
+{"timestamp":1713453755.7462552,"name":"online","context":{"idset":"10932"}}
+{"timestamp":1713453756.0689626,"name":"online","context":{"idset":"10930,10935"}}
+{"timestamp":1713453756.2593763,"name":"online","context":{"idset":"10939,10948"}}
+{"timestamp":1713453756.4595611,"name":"online","context":{"idset":"10941-10942,10945-10946,10950"}}
+{"timestamp":1713453756.5621934,"name":"online","context":{"idset":"10949"}}
+{"timestamp":1713453756.6946096,"name":"online","context":{"idset":"10953"}}
+{"timestamp":1713453757.0291369,"name":"online","context":{"idset":"10937,10943,10947"}}
+{"timestamp":1713453767.3606181,"name":"online","context":{"idset":"10956"}}
+{"timestamp":1713453767.5553887,"name":"online","context":{"idset":"10955"}}
+{"timestamp":1713453768.031738,"name":"online","context":{"idset":"10958-10959,10963"}}
+{"timestamp":1713453768.5199792,"name":"online","context":{"idset":"10972"}}
+{"timestamp":1713453768.7174327,"name":"online","context":{"idset":"10962,10967-10969,10975"}}
+{"timestamp":1713453769.0034559,"name":"online","context":{"idset":"10964"}}
+{"timestamp":1713453769.2114756,"name":"online","context":{"idset":"10966,10974"}}
+{"timestamp":1713453769.3970633,"name":"online","context":{"idset":"10965"}}
+{"timestamp":1713453769.6230395,"name":"online","context":{"idset":"10970"}}
+{"timestamp":1713453779.1316404,"name":"online","context":{"idset":"10976"}}
+{"timestamp":1713453779.3417838,"name":"online","context":{"idset":"10978"}}
+{"timestamp":1713453779.4499183,"name":"online","context":{"idset":"10979"}}
+{"timestamp":1713453779.9377315,"name":"online","context":{"idset":"10980"}}
+{"timestamp":1713453780.1289239,"name":"online","context":{"idset":"10981,10984"}}
+{"timestamp":1713453780.4070272,"name":"online","context":{"idset":"10982-10983"}}
+{"timestamp":1713453780.6408987,"name":"online","context":{"idset":"10985,10987"}}
+{"timestamp":1713453780.942724,"name":"online","context":{"idset":"10988,10991"}}
+{"timestamp":1713453781.1313264,"name":"online","context":{"idset":"10989-10990"}}
+{"timestamp":1713453781.3160226,"name":"online","context":{"idset":"10992-10993"}}
+{"timestamp":1713453791.313632,"name":"online","context":{"idset":"10995"}}
+{"timestamp":1713454238.5807977,"name":"offline","context":{"idset":"9734"}}
+{"timestamp":1713454762.4728858,"name":"online","context":{"idset":"11634"}}
+{"timestamp":1713455939.2789052,"name":"drain","context":{"idset":"807-808","overwrite":0}}
+{"timestamp":1713456157.8321154,"name":"online","context":{"idset":"9678"}}
+{"timestamp":1713456367.2062066,"name":"offline","context":{"idset":"808"}}
+{"timestamp":1713457797.7911055,"name":"online","context":{"idset":"10360"}}
+{"timestamp":1713459998.3310711,"name":"undrain","context":{"idset":"775-776"}}
+{"timestamp":1713461272.5784795,"name":"offline","context":{"idset":"8234"}}
+{"timestamp":1713461987.9776838,"name":"online","context":{"idset":"10529,10546,10600"}}
+{"timestamp":1713461988.1539631,"name":"online","context":{"idset":"10531"}}
+{"timestamp":1713461988.8137369,"name":"online","context":{"idset":"10485"}}
+{"timestamp":1713462121.1861684,"name":"online","context":{"idset":"10378,10422"}}
+{"timestamp":1713462121.3707523,"name":"online","context":{"idset":"10395"}}
+{"timestamp":1713462121.5568109,"name":"online","context":{"idset":"10477"}}
+{"timestamp":1713463446.9589553,"name":"online","context":{"idset":"10532"}}
+{"timestamp":1713465322.4252298,"name":"drain","context":{"idset":"775-776","reason":"--reason fixing rank 765 not drained","overwrite":0}}
+{"timestamp":1713465436.0146596,"name":"undrain","context":{"idset":"775-776"}}
+{"timestamp":1713465656.583967,"name":"offline","context":{"idset":"11058"}}
+{"timestamp":1713465908.0313659,"name":"online","context":{"idset":"10401"}}
+{"timestamp":1713465908.2957454,"name":"online","context":{"idset":"10413"}}
+{"timestamp":1713465908.475193,"name":"online","context":{"idset":"10452"}}
+{"timestamp":1713466488.5802281,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1713466490.5003307,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1713466490.5037544,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1713466490.5793102,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1713466492.5019088,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1713466492.5056832,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1713466492.5094063,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1713466492.5778692,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1713466494.5018215,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1713466494.5060806,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1713466494.579443,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1713466496.4994359,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1713466496.5054932,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1713466496.5097568,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1713466496.579504,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1713466498.4953022,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1713466498.4985495,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1713466498.5793965,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1713466500.4977422,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1713466500.5007856,"name":"offline","context":{"idset":"11149"}}
+{"timestamp":1713466500.5038342,"name":"offline","context":{"idset":"11150"}}
+{"timestamp":1713466500.5792966,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1713466502.4953661,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1713466502.4990175,"name":"offline","context":{"idset":"11154"}}
+{"timestamp":1713466502.5793517,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1713466504.5026143,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1713466504.506485,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1713466504.5106864,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1713466504.5797799,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1713466506.4975934,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1713466506.5006654,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1713466506.5037103,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1713466506.5786247,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1713466508.4989464,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1713466508.5027659,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1713466508.5070188,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1713466508.5818303,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1713466510.4986567,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1713466510.5058503,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1713466510.5131927,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1713466510.5185888,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1713466510.5791266,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1713466512.5095546,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1713466512.5172868,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1713466512.5239108,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1713466512.5301304,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1713466512.6141167,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1713466514.4982734,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1713466514.5028176,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1713466514.5824137,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1713466516.4941928,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1713466516.4984677,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1713466516.5822484,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1713466518.4975052,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1713466518.5006757,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1713466518.5053635,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1713466518.5791903,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1713466520.4977143,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1713466520.501513,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1713466520.5788951,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1713466522.4998593,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1713466522.5056548,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1713466522.5113773,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1713466522.5171626,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1713466522.5776632,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1713466524.4926314,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1713466524.4956572,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1713466524.5787654,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1713466526.4943535,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1713466526.4974937,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1713466526.5006089,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1713466526.5773177,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1713466528.4957683,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1713466528.4995422,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1713466528.5794559,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1713466530.5065625,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1713466530.5101023,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1713466530.5136893,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1713466530.5791795,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1713466532.4962518,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1713466532.4992621,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1713466532.579397,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1713466534.5780253,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1713466536.4918725,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1713466536.4953604,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1713466536.5785582,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1713466538.5786045,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1713466540.4985437,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1713466540.5049636,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1713466540.5114033,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1713466540.5795515,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1713466542.4995637,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1713466542.5028806,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1713466542.5069265,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1713466542.5787971,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1713466544.500494,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1713466544.5060918,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1713466544.5118992,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1713466544.5165524,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1713466544.5790746,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1713466546.4932685,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1713466546.4970415,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1713466546.5805986,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1713466548.5773573,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1713466576.5784369,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1713466588.5789397,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1713466760.0955582,"name":"online","context":{"idset":"9476"}}
+{"timestamp":1713468219.5826445,"name":"online","context":{"idset":"10424"}}
+{"timestamp":1713468219.6852436,"name":"online","context":{"idset":"10367"}}
+{"timestamp":1713468219.7949286,"name":"online","context":{"idset":"10380,10418,10464"}}
+{"timestamp":1713468710.4936922,"name":"offline","context":{"idset":"10465"}}
+{"timestamp":1713468710.5804594,"name":"offline","context":{"idset":"10466"}}
+{"timestamp":1713468730.5777819,"name":"offline","context":{"idset":"10455"}}
+{"timestamp":1713468768.5781021,"name":"offline","context":{"idset":"10410"}}
+{"timestamp":1713468800.5762827,"name":"offline","context":{"idset":"10395"}}
+{"timestamp":1713468827.8978612,"name":"offline","context":{"idset":"314"}}
+{"timestamp":1713468828.0143082,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1713468828.4736161,"name":"offline","context":{"idset":"313"}}
+{"timestamp":1713468828.7959061,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1713469536.5785279,"name":"offline","context":{"idset":"948"}}
+{"timestamp":1713470753.6389699,"name":"online","context":{"idset":"11139"}}
+{"timestamp":1713470753.7992833,"name":"online","context":{"idset":"11134"}}
+{"timestamp":1713470753.8988087,"name":"online","context":{"idset":"11147,11172"}}
+{"timestamp":1713470754.0558116,"name":"online","context":{"idset":"11141"}}
+{"timestamp":1713470754.1917019,"name":"online","context":{"idset":"11135,11215"}}
+{"timestamp":1713470754.3195891,"name":"online","context":{"idset":"11174"}}
+{"timestamp":1713470754.5910802,"name":"online","context":{"idset":"11207,11213"}}
+{"timestamp":1713470754.7086189,"name":"online","context":{"idset":"11125"}}
+{"timestamp":1713470754.7986476,"name":"online","context":{"idset":"11133,11138,11165"}}
+{"timestamp":1713470754.9974372,"name":"online","context":{"idset":"11127,11176"}}
+{"timestamp":1713470755.2054753,"name":"online","context":{"idset":"11132"}}
+{"timestamp":1713470755.326051,"name":"online","context":{"idset":"11178"}}
+{"timestamp":1713470755.4261222,"name":"online","context":{"idset":"11151"}}
+{"timestamp":1713470755.7021663,"name":"online","context":{"idset":"11148,11171,11194"}}
+{"timestamp":1713470755.9420595,"name":"online","context":{"idset":"11150,11189,11223"}}
+{"timestamp":1713470756.0568464,"name":"online","context":{"idset":"11130,11217"}}
+{"timestamp":1713470756.2459455,"name":"online","context":{"idset":"11131,11164,11167,11184,11190,11209,11219"}}
+{"timestamp":1713470756.4405897,"name":"online","context":{"idset":"11137,11140,11144,11161,11175,11181,11186"}}
+{"timestamp":1713470756.5434244,"name":"online","context":{"idset":"11154-11155,11201,11228"}}
+{"timestamp":1713470756.6572127,"name":"online","context":{"idset":"11126,11145,11156,11211-11212"}}
+{"timestamp":1713470756.7637763,"name":"online","context":{"idset":"11142,11177,11180"}}
+{"timestamp":1713470756.9579067,"name":"online","context":{"idset":"11128,11157-11158,11160,11166,11169,11185,11199,11204-11206,11222,11231,11244-11245"}}
+{"timestamp":1713470757.066519,"name":"online","context":{"idset":"11216,11218,11221,11224,11229"}}
+{"timestamp":1713470757.1701488,"name":"online","context":{"idset":"11159,11163,11193,11200,11240"}}
+{"timestamp":1713470757.2772632,"name":"online","context":{"idset":"11149,11170,11188,11191,11197,11225,11238,11248"}}
+{"timestamp":1713470757.4677474,"name":"online","context":{"idset":"11143,11153,11168,11173,11187,11195-11196,11198,11203,11208,11214,11220,11226,11232-11237,11239,11242,11246"}}
+{"timestamp":1713470757.6430945,"name":"online","context":{"idset":"11162,11227,11230,11241,11243,11250"}}
+{"timestamp":1713470757.8173845,"name":"online","context":{"idset":"11152,11179,11183,11210,11247,11252"}}
+{"timestamp":1713470757.9200149,"name":"online","context":{"idset":"11249,11251"}}
+{"timestamp":1713471450.5778744,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1713471785.6043551,"name":"offline","context":{"idset":"578"}}
+{"timestamp":1713472093.5923898,"name":"online","context":{"idset":"775"}}
+{"timestamp":1713472126.5584836,"name":"offline","context":{"idset":"279"}}
+{"timestamp":1713472141.4122648,"name":"online","context":{"idset":"776"}}
+{"timestamp":1713472190.0959013,"name":"undrain","context":{"idset":"773-774"}}
+{"timestamp":1713472284.6963391,"name":"online","context":{"idset":"9991"}}
+{"timestamp":1713472327.6820242,"name":"online","context":{"idset":"9992"}}
+{"timestamp":1713472359.9133053,"name":"online","context":{"idset":"877"}}
+{"timestamp":1713472528.5782285,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1713472678.5823879,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1713472710.5787158,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1713472958.5791121,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1713473002.5826461,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1713473154.7696865,"name":"online","context":{"idset":"10046"}}
+{"timestamp":1713473494.4950309,"name":"offline","context":{"idset":"11253"}}
+{"timestamp":1713473494.4984517,"name":"offline","context":{"idset":"11254"}}
+{"timestamp":1713473494.578773,"name":"offline","context":{"idset":"11255"}}
+{"timestamp":1713473496.4957719,"name":"offline","context":{"idset":"11258"}}
+{"timestamp":1713473496.4989357,"name":"offline","context":{"idset":"11259"}}
+{"timestamp":1713473496.5782442,"name":"offline","context":{"idset":"11261"}}
+{"timestamp":1713473498.5084386,"name":"offline","context":{"idset":"11263"}}
+{"timestamp":1713473498.5115082,"name":"offline","context":{"idset":"11264"}}
+{"timestamp":1713473498.5145783,"name":"offline","context":{"idset":"11265"}}
+{"timestamp":1713473498.5809188,"name":"offline","context":{"idset":"11266"}}
+{"timestamp":1713473500.4987442,"name":"offline","context":{"idset":"11267"}}
+{"timestamp":1713473500.5044615,"name":"offline","context":{"idset":"11268"}}
+{"timestamp":1713473500.578773,"name":"offline","context":{"idset":"11269"}}
+{"timestamp":1713473502.496321,"name":"offline","context":{"idset":"11272"}}
+{"timestamp":1713473502.5004907,"name":"offline","context":{"idset":"11273"}}
+{"timestamp":1713473502.5785296,"name":"offline","context":{"idset":"11275"}}
+{"timestamp":1713473504.4977922,"name":"offline","context":{"idset":"11276"}}
+{"timestamp":1713473504.5017774,"name":"offline","context":{"idset":"11277"}}
+{"timestamp":1713473504.5050933,"name":"offline","context":{"idset":"11278"}}
+{"timestamp":1713473504.5782812,"name":"offline","context":{"idset":"11279"}}
+{"timestamp":1713473506.5071664,"name":"offline","context":{"idset":"11280"}}
+{"timestamp":1713473506.5105312,"name":"offline","context":{"idset":"11281"}}
+{"timestamp":1713473506.5137498,"name":"offline","context":{"idset":"11282"}}
+{"timestamp":1713473506.5171609,"name":"offline","context":{"idset":"11283"}}
+{"timestamp":1713473506.579349,"name":"offline","context":{"idset":"11285"}}
+{"timestamp":1713473508.4997718,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1713473508.5048602,"name":"offline","context":{"idset":"11287"}}
+{"timestamp":1713473508.5085685,"name":"offline","context":{"idset":"11288"}}
+{"timestamp":1713473508.5781643,"name":"offline","context":{"idset":"11289"}}
+{"timestamp":1713473510.50471,"name":"offline","context":{"idset":"11290"}}
+{"timestamp":1713473510.5092189,"name":"offline","context":{"idset":"11291"}}
+{"timestamp":1713473510.5127509,"name":"offline","context":{"idset":"11292"}}
+{"timestamp":1713473510.5799193,"name":"offline","context":{"idset":"11293"}}
+{"timestamp":1713473512.5079684,"name":"offline","context":{"idset":"11296"}}
+{"timestamp":1713473512.5177147,"name":"offline","context":{"idset":"11297"}}
+{"timestamp":1713473512.583667,"name":"offline","context":{"idset":"11298"}}
+{"timestamp":1713473514.4955883,"name":"offline","context":{"idset":"11300"}}
+{"timestamp":1713473514.4992204,"name":"offline","context":{"idset":"11301"}}
+{"timestamp":1713473514.5794239,"name":"offline","context":{"idset":"11304"}}
+{"timestamp":1713473516.5275137,"name":"offline","context":{"idset":"11305"}}
+{"timestamp":1713473516.5410199,"name":"offline","context":{"idset":"11307"}}
+{"timestamp":1713473516.5875015,"name":"offline","context":{"idset":"11308"}}
+{"timestamp":1713473518.5190444,"name":"offline","context":{"idset":"11309"}}
+{"timestamp":1713473518.52246,"name":"offline","context":{"idset":"11310"}}
+{"timestamp":1713473518.535018,"name":"offline","context":{"idset":"11311"}}
+{"timestamp":1713473518.538409,"name":"offline","context":{"idset":"11312"}}
+{"timestamp":1713473518.6056056,"name":"offline","context":{"idset":"11313"}}
+{"timestamp":1713473520.4923139,"name":"offline","context":{"idset":"11314"}}
+{"timestamp":1713473520.4957104,"name":"offline","context":{"idset":"11315"}}
+{"timestamp":1713473520.499511,"name":"offline","context":{"idset":"11317"}}
+{"timestamp":1713473520.5781569,"name":"offline","context":{"idset":"11318"}}
+{"timestamp":1713473522.4949605,"name":"offline","context":{"idset":"11319"}}
+{"timestamp":1713473522.5006666,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1713473522.5008171,"name":"drain","context":{"idset":"11321-11322","reason":"prolog failed for jobid fqGgXCNjEVM","overwrite":0}}
+{"timestamp":1713473522.5802319,"name":"offline","context":{"idset":"11322"}}
+{"timestamp":1713473530.4921398,"name":"offline","context":{"idset":"11327"}}
+{"timestamp":1713473530.5774434,"name":"offline","context":{"idset":"11328"}}
+{"timestamp":1713473532.4986877,"name":"offline","context":{"idset":"11329"}}
+{"timestamp":1713473532.5017884,"name":"offline","context":{"idset":"11330"}}
+{"timestamp":1713473532.504853,"name":"offline","context":{"idset":"11331"}}
+{"timestamp":1713473532.5079114,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1713473532.5781324,"name":"offline","context":{"idset":"11333"}}
+{"timestamp":1713473533.2632308,"name":"offline","context":{"idset":"11334"}}
+{"timestamp":1713473533.3189664,"name":"offline","context":{"idset":"11335"}}
+{"timestamp":1713473533.3470483,"name":"offline","context":{"idset":"11336"}}
+{"timestamp":1713473536.4947507,"name":"offline","context":{"idset":"11338"}}
+{"timestamp":1713473536.4981492,"name":"offline","context":{"idset":"11339"}}
+{"timestamp":1713473536.5013659,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1713473536.5797095,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1713473538.5051103,"name":"offline","context":{"idset":"11342"}}
+{"timestamp":1713473538.5083187,"name":"offline","context":{"idset":"11343"}}
+{"timestamp":1713473538.5115919,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1713473538.5148427,"name":"offline","context":{"idset":"11345"}}
+{"timestamp":1713473538.5802784,"name":"offline","context":{"idset":"11346"}}
+{"timestamp":1713473540.492342,"name":"offline","context":{"idset":"11347"}}
+{"timestamp":1713473540.4961126,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1713473540.5798936,"name":"offline","context":{"idset":"11349"}}
+{"timestamp":1713473542.499223,"name":"offline","context":{"idset":"11350"}}
+{"timestamp":1713473542.5049119,"name":"offline","context":{"idset":"11351"}}
+{"timestamp":1713473542.5109506,"name":"offline","context":{"idset":"11352"}}
+{"timestamp":1713473542.5810745,"name":"offline","context":{"idset":"11353"}}
+{"timestamp":1713473544.5033054,"name":"offline","context":{"idset":"11354"}}
+{"timestamp":1713473544.5065165,"name":"offline","context":{"idset":"11355"}}
+{"timestamp":1713473544.5805812,"name":"offline","context":{"idset":"11356"}}
+{"timestamp":1713473546.5104663,"name":"offline","context":{"idset":"11357"}}
+{"timestamp":1713473546.5139439,"name":"offline","context":{"idset":"11358"}}
+{"timestamp":1713473546.5169444,"name":"offline","context":{"idset":"11359"}}
+{"timestamp":1713473546.5201681,"name":"offline","context":{"idset":"11360"}}
+{"timestamp":1713473546.5816414,"name":"offline","context":{"idset":"11361"}}
+{"timestamp":1713473548.5090733,"name":"offline","context":{"idset":"11362"}}
+{"timestamp":1713473548.5149701,"name":"offline","context":{"idset":"11363"}}
+{"timestamp":1713473548.5205991,"name":"offline","context":{"idset":"11365"}}
+{"timestamp":1713473548.5817046,"name":"offline","context":{"idset":"11366"}}
+{"timestamp":1713473550.4935377,"name":"offline","context":{"idset":"11367"}}
+{"timestamp":1713473550.4965324,"name":"offline","context":{"idset":"11368"}}
+{"timestamp":1713473550.5777493,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1713473552.5085371,"name":"offline","context":{"idset":"11370"}}
+{"timestamp":1713473552.5116265,"name":"offline","context":{"idset":"11371"}}
+{"timestamp":1713473552.5147278,"name":"offline","context":{"idset":"11372"}}
+{"timestamp":1713473552.5177474,"name":"offline","context":{"idset":"11373"}}
+{"timestamp":1713473552.5790815,"name":"offline","context":{"idset":"11374"}}
+{"timestamp":1713473554.4979522,"name":"offline","context":{"idset":"11376"}}
+{"timestamp":1713473554.5009725,"name":"offline","context":{"idset":"11377"}}
+{"timestamp":1713473554.578599,"name":"offline","context":{"idset":"11378"}}
+{"timestamp":1713473556.4910076,"name":"offline","context":{"idset":"11379"}}
+{"timestamp":1713473556.5791273,"name":"offline","context":{"idset":"11380"}}
+{"timestamp":1713473558.5784152,"name":"offline","context":{"idset":"11294"}}
+{"timestamp":1713474598.5429752,"name":"offline","context":{"idset":"9413"}}
+{"timestamp":1713474598.5470781,"name":"offline","context":{"idset":"9414"}}
+{"timestamp":1713474598.5509682,"name":"offline","context":{"idset":"9415"}}
+{"timestamp":1713474598.5549495,"name":"offline","context":{"idset":"9416"}}
+{"timestamp":1713474598.5589826,"name":"offline","context":{"idset":"9417"}}
+{"timestamp":1713474598.5629895,"name":"offline","context":{"idset":"9418"}}
+{"timestamp":1713474598.5668368,"name":"offline","context":{"idset":"9419"}}
+{"timestamp":1713474598.5706494,"name":"offline","context":{"idset":"9420"}}
+{"timestamp":1713474598.574513,"name":"offline","context":{"idset":"9421"}}
+{"timestamp":1713474598.5793316,"name":"offline","context":{"idset":"9422"}}
+{"timestamp":1713474598.5871553,"name":"offline","context":{"idset":"9423"}}
+{"timestamp":1713474598.5914536,"name":"offline","context":{"idset":"9424"}}
+{"timestamp":1713474598.5953534,"name":"offline","context":{"idset":"9425"}}
+{"timestamp":1713474598.5992579,"name":"offline","context":{"idset":"9426"}}
+{"timestamp":1713474598.6032326,"name":"offline","context":{"idset":"9427"}}
+{"timestamp":1713474598.6191523,"name":"offline","context":{"idset":"9428"}}
+{"timestamp":1713476125.1152384,"name":"drain","context":{"idset":"773-774","reason":"--reason Troubleshoot nodes","overwrite":0}}
+{"timestamp":1713478936.6509023,"name":"online","context":{"idset":"11181"}}
+{"timestamp":1713478936.8324609,"name":"online","context":{"idset":"11135"}}
+{"timestamp":1713478936.9459457,"name":"online","context":{"idset":"11130,11136"}}
+{"timestamp":1713478937.1280639,"name":"online","context":{"idset":"11129"}}
+{"timestamp":1713478937.4352283,"name":"online","context":{"idset":"11145-11146"}}
+{"timestamp":1713479195.0285699,"name":"online","context":{"idset":"11465"}}
+{"timestamp":1713479195.1688216,"name":"online","context":{"idset":"11394"}}
+{"timestamp":1713479195.2644806,"name":"online","context":{"idset":"11384,11405"}}
+{"timestamp":1713479195.468215,"name":"online","context":{"idset":"11462"}}
+{"timestamp":1713479195.6312492,"name":"online","context":{"idset":"11432"}}
+{"timestamp":1713479195.718241,"name":"online","context":{"idset":"11402"}}
+{"timestamp":1713479195.8799624,"name":"online","context":{"idset":"11416"}}
+{"timestamp":1713479196.0571914,"name":"online","context":{"idset":"11382,11393"}}
+{"timestamp":1713479196.1620677,"name":"online","context":{"idset":"11385,11390,11408"}}
+{"timestamp":1713479196.4118674,"name":"online","context":{"idset":"11414,11431,11456"}}
+{"timestamp":1713479196.5515053,"name":"drain","context":{"idset":"11461","reason":"broker was unresponsive"}}
+{"timestamp":1713479196.5547001,"name":"online","context":{"idset":"11381,11433,11443"}}
+{"timestamp":1713479196.6729178,"name":"online","context":{"idset":"11388,11406"}}
+{"timestamp":1713479196.6880465,"name":"online","context":{"idset":"11417"}}
+{"timestamp":1713479197.0129764,"name":"online","context":{"idset":"11395,11448"}}
+{"timestamp":1713479197.1259308,"name":"online","context":{"idset":"11441"}}
+{"timestamp":1713479197.2302494,"name":"online","context":{"idset":"11449"}}
+{"timestamp":1713479197.3368249,"name":"online","context":{"idset":"11418,11505"}}
+{"timestamp":1713479197.4270597,"name":"online","context":{"idset":"11436"}}
+{"timestamp":1713479197.5230298,"name":"online","context":{"idset":"11424,11434"}}
+{"timestamp":1713479197.6259871,"name":"online","context":{"idset":"11392,11442"}}
+{"timestamp":1713479197.6316633,"name":"online","context":{"idset":"11504"}}
+{"timestamp":1713479197.7423773,"name":"online","context":{"idset":"11407"}}
+{"timestamp":1713479197.7555821,"name":"online","context":{"idset":"11438,11457,11485"}}
+{"timestamp":1713479197.8702767,"name":"online","context":{"idset":"11421,11439,11453,11472,11497"}}
+{"timestamp":1713479197.9682157,"name":"online","context":{"idset":"11413,11427,11437,11483,11489"}}
+{"timestamp":1713479198.1581233,"name":"online","context":{"idset":"11389,11412,11422,11468,11500"}}
+{"timestamp":1713479198.2620883,"name":"online","context":{"idset":"11428,11430,11460,11466,11476,11482,11487,11498"}}
+{"timestamp":1713479198.3745272,"name":"online","context":{"idset":"11467,11481,11484"}}
+{"timestamp":1713479198.4845016,"name":"online","context":{"idset":"11491"}}
+{"timestamp":1713479198.7450631,"name":"online","context":{"idset":"11496"}}
+{"timestamp":1713479198.8545012,"name":"online","context":{"idset":"11409,11461"}}
+{"timestamp":1713479198.9665356,"name":"online","context":{"idset":"11391,11458-11459,11463-11464,11477,11479-11480,11494-11495"}}
+{"timestamp":1713479199.0803041,"name":"online","context":{"idset":"11404,11454,11470,11506"}}
+{"timestamp":1713479199.2597322,"name":"online","context":{"idset":"11399,11403,11425,11429,11450,11474-11475,11486,11502"}}
+{"timestamp":1713479199.4467416,"name":"online","context":{"idset":"11383,11398,11400,11410,11415,11445,11447"}}
+{"timestamp":1713479199.5576689,"name":"online","context":{"idset":"11401,11435,11469,11478,11488,11503,11508"}}
+{"timestamp":1713479199.7407691,"name":"online","context":{"idset":"11411,11423,11426,11440,11451-11452,11473,11490,11499"}}
+{"timestamp":1713479199.9212365,"name":"online","context":{"idset":"11396,11492"}}
+{"timestamp":1713479200.0229979,"name":"online","context":{"idset":"11386-11387,11444,11446,11455,11471,11493,11501,11507"}}
+{"timestamp":1713479237.617275,"name":"undrain","context":{"idset":"11461"}}
+{"timestamp":1713479733.2996147,"name":"drain","context":{"idset":"803-804","reason":"status","overwrite":0}}
+{"timestamp":1713480060.0849979,"name":"drain","context":{"idset":"44","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480060.3161159,"name":"drain","context":{"idset":"29","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480060.5016932,"name":"drain","context":{"idset":"27","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480060.7056046,"name":"drain","context":{"idset":"8","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480060.895999,"name":"drain","context":{"idset":"9","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480061.0843661,"name":"drain","context":{"idset":"6","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480061.2725441,"name":"drain","context":{"idset":"26","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480061.453572,"name":"drain","context":{"idset":"5","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480061.6399705,"name":"drain","context":{"idset":"52","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480061.7838337,"name":"drain","context":{"idset":"28","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480061.9053893,"name":"drain","context":{"idset":"4","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480062.0192513,"name":"drain","context":{"idset":"50","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480062.116195,"name":"drain","context":{"idset":"17","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480062.2171288,"name":"drain","context":{"idset":"24","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480062.3177383,"name":"drain","context":{"idset":"21","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480062.4063392,"name":"drain","context":{"idset":"22","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480062.5092621,"name":"drain","context":{"idset":"2","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480062.6110375,"name":"drain","context":{"idset":"12","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480062.7126806,"name":"drain","context":{"idset":"18","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480062.8103864,"name":"drain","context":{"idset":"20","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480062.9011104,"name":"drain","context":{"idset":"32","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.0032811,"name":"drain","context":{"idset":"33","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.1000807,"name":"drain","context":{"idset":"38","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.1922977,"name":"drain","context":{"idset":"56","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.2832365,"name":"drain","context":{"idset":"31","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.3857143,"name":"drain","context":{"idset":"14","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.4797401,"name":"drain","context":{"idset":"57","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.5775566,"name":"drain","context":{"idset":"10","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.6694856,"name":"drain","context":{"idset":"40","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.7637284,"name":"drain","context":{"idset":"55","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.8555751,"name":"drain","context":{"idset":"51","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480063.9484205,"name":"drain","context":{"idset":"16","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480064.0402603,"name":"drain","context":{"idset":"39","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480064.1372237,"name":"drain","context":{"idset":"41","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480064.2345219,"name":"drain","context":{"idset":"36","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480064.3345563,"name":"drain","context":{"idset":"45","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480064.4297328,"name":"drain","context":{"idset":"60","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480064.5181532,"name":"drain","context":{"idset":"43","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480064.6051624,"name":"drain","context":{"idset":"53","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713480064.696537,"name":"drain","context":{"idset":"48","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713481186.5783827,"name":"offline","context":{"idset":"924"}}
+{"timestamp":1713481320.5779638,"name":"offline","context":{"idset":"9359"}}
+{"timestamp":1713481933.4896398,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1713482196.6912057,"name":"undrain","context":{"idset":"2,4-6,8-10,12,14,16-18,20-22,24,26-29,31-33,36,38-39,41,43-45,48,50-53,55-57,60"}}
+{"timestamp":1713482490.6762002,"name":"drain","context":{"idset":"11891","reason":"drain node to notify scheduler","overwrite":0}}
+{"timestamp":1713482748.4928634,"name":"online","context":{"idset":"11191"}}
+{"timestamp":1713482777.4201055,"name":"online","context":{"idset":"11202"}}
+{"timestamp":1713482777.9633648,"name":"online","context":{"idset":"11201"}}
+{"timestamp":1713482882.578311,"name":"offline","context":{"idset":"797"}}
+{"timestamp":1713483018.5787749,"name":"offline","context":{"idset":"828"}}
+{"timestamp":1713483030.579011,"name":"offline","context":{"idset":"830"}}
+{"timestamp":1713483038.5811503,"name":"offline","context":{"idset":"832"}}
+{"timestamp":1713483042.578779,"name":"offline","context":{"idset":"821"}}
+{"timestamp":1713483048.577765,"name":"offline","context":{"idset":"834"}}
+{"timestamp":1713483216.5786312,"name":"offline","context":{"idset":"926"}}
+{"timestamp":1713484572.4899328,"name":"offline","context":{"idset":"773"}}
+{"timestamp":1713484572.5789244,"name":"offline","context":{"idset":"774"}}
+{"timestamp":1713484590.492965,"name":"offline","context":{"idset":"775"}}
+{"timestamp":1713484590.5804477,"name":"offline","context":{"idset":"776"}}
+{"timestamp":1713484618.5776761,"name":"offline","context":{"idset":"780"}}
+{"timestamp":1713484633.4217517,"name":"offline","context":{"idset":"781"}}
+{"timestamp":1713484642.5791118,"name":"offline","context":{"idset":"785"}}
+{"timestamp":1713484752.5783231,"name":"offline","context":{"idset":"906"}}
+{"timestamp":1713484794.578007,"name":"offline","context":{"idset":"904"}}
+{"timestamp":1713484826.5821128,"name":"offline","context":{"idset":"912"}}
+{"timestamp":1713484835.5257077,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1713484837.9880791,"name":"online","context":{"idset":"40"}}
+{"timestamp":1713485688.5777297,"name":"offline","context":{"idset":"968"}}
+{"timestamp":1713485702.5798674,"name":"offline","context":{"idset":"970"}}
+{"timestamp":1713485736.5792129,"name":"offline","context":{"idset":"974"}}
+{"timestamp":1713490290.2797379,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1713490290.3808079,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1713490290.3994668,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1713490290.4961226,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1713490290.5919189,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1713490290.6659617,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1713490290.6698215,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1713490290.7700918,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1713490290.7799854,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1713490290.8795185,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1713490290.9554057,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1713490291.0308912,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1713490291.0450704,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1713490291.1449699,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1713490291.2214298,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1713490291.2251046,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1713490291.3128436,"name":"offline","context":{"idset":"11517"}}
+{"timestamp":1713490291.3193038,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1713490291.3223484,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1713490291.3255994,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1713490291.4252176,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1713490291.5091231,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1713490291.5150626,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1713490291.5185719,"name":"offline","context":{"idset":"11525"}}
+{"timestamp":1713490291.6181948,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1713490291.6261992,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1713490291.7112606,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1713490291.7176139,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1713490291.7206302,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1713490291.8173223,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1713490291.8979974,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1713490291.9048507,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1713490291.9078956,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1713490291.9922011,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1713490291.9967902,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1713490292.0006423,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1713490292.0847309,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1713490292.1570244,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1713490292.2244036,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1713490292.2378364,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1713490292.3231723,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1713490292.3274367,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1713490292.333694,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1713490292.3369238,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1713490292.4341941,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1713490292.4650965,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1713490292.4683843,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1713490292.5636613,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1713490292.6624341,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1713490292.665911,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1713490292.6721017,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1713490292.676429,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1713490292.771857,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1713490292.7883322,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1713490292.7916396,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1713490292.8074667,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1713490292.8291209,"name":"offline","context":{"idset":"11536"}}
+{"timestamp":1713490292.8322554,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1713490292.8920095,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1713490292.9776816,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1713490292.9816585,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1713490292.9859247,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1713490292.9903874,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1713490292.9942253,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1713490293.005698,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1713490293.0966723,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1713490293.1836586,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1713490293.1903527,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1713490293.1934476,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1713490293.1995873,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1713490293.2064202,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1713490293.2106702,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1713490293.2499321,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1713490293.2536702,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1713490293.2573948,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1713490293.2716796,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1713490293.2754855,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1713490293.4054086,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1713490293.4104161,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1713490293.4150267,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1713490293.4189191,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1713490293.4226737,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1713490293.5102708,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1713490293.5889776,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1713490293.5930157,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1713490293.5996573,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1713490293.6033916,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1713490293.6984365,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1713490293.7093337,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1713490293.7159662,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1713490293.7190948,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1713490293.81457,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1713490293.8206267,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1713490293.82426,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1713490293.8279202,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1713490293.9255638,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1713490293.929745,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1713490293.9330151,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1713490293.936743,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1713490294.0361319,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1713490294.1151497,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1713490294.1188641,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1713490294.1226096,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1713490294.2093637,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1713490294.2163234,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1713490294.2195439,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1713490294.2225554,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1713490294.2293282,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1713490294.2358146,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1713490294.2389717,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1713490294.3320396,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1713490294.4041061,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1713490294.4079115,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1713490294.5106883,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1713490294.6019073,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1713490294.613359,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1713490294.6191263,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1713490294.6245649,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1713490294.6307042,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1713490294.6368263,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1713490294.6427972,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1713490294.6485596,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1713490294.7272828,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1713490294.8087108,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1713490294.8230131,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1713490294.9232116,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1713490295.1461339,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1713490295.2451646,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1713492700.5119128,"name":"online","context":{"idset":"11515"}}
+{"timestamp":1713492701.0058377,"name":"online","context":{"idset":"11510,11514,11519,11521"}}
+{"timestamp":1713492701.482836,"name":"online","context":{"idset":"11509,11511,11513,11518,11522"}}
+{"timestamp":1713492701.6855423,"name":"online","context":{"idset":"11516,11520"}}
+{"timestamp":1713492701.9800956,"name":"online","context":{"idset":"11512"}}
+{"timestamp":1713492702.3469071,"name":"online","context":{"idset":"11524"}}
+{"timestamp":1713492702.5424955,"name":"online","context":{"idset":"11517,11523"}}
+{"timestamp":1713492712.5603657,"name":"online","context":{"idset":"11525"}}
+{"timestamp":1713492712.7808139,"name":"online","context":{"idset":"11527"}}
+{"timestamp":1713492712.9715562,"name":"online","context":{"idset":"11526"}}
+{"timestamp":1713492713.1588485,"name":"online","context":{"idset":"11528,11533"}}
+{"timestamp":1713492713.3447268,"name":"online","context":{"idset":"11530"}}
+{"timestamp":1713492713.4598551,"name":"online","context":{"idset":"11529,11531-11532"}}
+{"timestamp":1713492713.6643996,"name":"online","context":{"idset":"11535"}}
+{"timestamp":1713492713.8514841,"name":"online","context":{"idset":"11534,11536-11538"}}
+{"timestamp":1713492714.3157792,"name":"online","context":{"idset":"11539-11540"}}
+{"timestamp":1713492724.4590564,"name":"online","context":{"idset":"11541,11543"}}
+{"timestamp":1713492724.6874843,"name":"online","context":{"idset":"11542"}}
+{"timestamp":1713492724.9663641,"name":"online","context":{"idset":"11544-11546"}}
+{"timestamp":1713492725.187238,"name":"online","context":{"idset":"11547"}}
+{"timestamp":1713492725.6813066,"name":"online","context":{"idset":"11548-11551,11553"}}
+{"timestamp":1713492725.8612154,"name":"online","context":{"idset":"11552,11554"}}
+{"timestamp":1713492726.468262,"name":"online","context":{"idset":"11556"}}
+{"timestamp":1713492727.0422428,"name":"online","context":{"idset":"11555"}}
+{"timestamp":1713492736.7021632,"name":"online","context":{"idset":"11558-11559"}}
+{"timestamp":1713492736.9442682,"name":"online","context":{"idset":"11563"}}
+{"timestamp":1713492737.1829703,"name":"online","context":{"idset":"11557,11560-11561"}}
+{"timestamp":1713492737.4460902,"name":"online","context":{"idset":"11565"}}
+{"timestamp":1713492737.6584001,"name":"online","context":{"idset":"11562,11566,11569-11570"}}
+{"timestamp":1713492737.8300941,"name":"online","context":{"idset":"11567-11568,11571"}}
+{"timestamp":1713492738.1237667,"name":"online","context":{"idset":"11564,11572"}}
+{"timestamp":1713492748.0692413,"name":"online","context":{"idset":"11574"}}
+{"timestamp":1713492748.651144,"name":"online","context":{"idset":"11573"}}
+{"timestamp":1713492748.8393896,"name":"online","context":{"idset":"11577,11580"}}
+{"timestamp":1713492748.9492676,"name":"online","context":{"idset":"11576,11578"}}
+{"timestamp":1713492749.1333408,"name":"online","context":{"idset":"11575,11579"}}
+{"timestamp":1713492749.5246012,"name":"online","context":{"idset":"11581-11582,11585"}}
+{"timestamp":1713492749.7007821,"name":"online","context":{"idset":"11583,11588"}}
+{"timestamp":1713492749.8858855,"name":"online","context":{"idset":"11589"}}
+{"timestamp":1713492750.1724806,"name":"online","context":{"idset":"11587"}}
+{"timestamp":1713492750.3612735,"name":"online","context":{"idset":"11584"}}
+{"timestamp":1713492759.6450408,"name":"online","context":{"idset":"11590"}}
+{"timestamp":1713492760.3337305,"name":"online","context":{"idset":"11591"}}
+{"timestamp":1713492760.5631723,"name":"online","context":{"idset":"11592,11596"}}
+{"timestamp":1713492760.8166709,"name":"online","context":{"idset":"11593-11595"}}
+{"timestamp":1713492761.1188254,"name":"online","context":{"idset":"11598-11599"}}
+{"timestamp":1713492761.352144,"name":"online","context":{"idset":"11601,11603"}}
+{"timestamp":1713492761.5340011,"name":"online","context":{"idset":"11597,11602,11605"}}
+{"timestamp":1713492761.867501,"name":"online","context":{"idset":"11600"}}
+{"timestamp":1713492762.0439234,"name":"online","context":{"idset":"11604"}}
+{"timestamp":1713492771.5076532,"name":"online","context":{"idset":"11606"}}
+{"timestamp":1713492772.148881,"name":"online","context":{"idset":"11607"}}
+{"timestamp":1713492772.4357722,"name":"online","context":{"idset":"11608,11610"}}
+{"timestamp":1713492772.6715143,"name":"online","context":{"idset":"11609,11611"}}
+{"timestamp":1713492772.7798414,"name":"online","context":{"idset":"11614"}}
+{"timestamp":1713492772.9915633,"name":"online","context":{"idset":"11613,11615"}}
+{"timestamp":1713492773.2803552,"name":"online","context":{"idset":"11612,11616,11619"}}
+{"timestamp":1713492773.4796443,"name":"online","context":{"idset":"11617"}}
+{"timestamp":1713492773.5819819,"name":"online","context":{"idset":"11618"}}
+{"timestamp":1713492773.7649653,"name":"online","context":{"idset":"11620-11621"}}
+{"timestamp":1713492784.1108091,"name":"online","context":{"idset":"11623"}}
+{"timestamp":1713492784.2791698,"name":"online","context":{"idset":"11622,11624"}}
+{"timestamp":1713492784.4600112,"name":"online","context":{"idset":"11625"}}
+{"timestamp":1713492784.6317232,"name":"online","context":{"idset":"11626-11627"}}
+{"timestamp":1713492784.7968936,"name":"online","context":{"idset":"11628,11634"}}
+{"timestamp":1713492784.9576328,"name":"online","context":{"idset":"11629-11630"}}
+{"timestamp":1713492785.2403383,"name":"online","context":{"idset":"11633"}}
+{"timestamp":1713492785.4054606,"name":"online","context":{"idset":"11636"}}
+{"timestamp":1713492785.5676212,"name":"online","context":{"idset":"11632"}}
+{"timestamp":1713492785.7586358,"name":"online","context":{"idset":"11635"}}
+{"timestamp":1713493245.6092141,"name":"online","context":{"idset":"11586,11631"}}
+{"timestamp":1713493363.936765,"name":"online","context":{"idset":"11691"}}
+{"timestamp":1713493364.2720356,"name":"online","context":{"idset":"11692"}}
+{"timestamp":1713493373.3446527,"name":"online","context":{"idset":"11695"}}
+{"timestamp":1713493554.6570697,"name":"online","context":{"idset":"11644,11649"}}
+{"timestamp":1713494938.5779295,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1713495404.1245708,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1713495404.1324301,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1713495404.2308714,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1713495404.3358958,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1713495802.5790691,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1713497151.2269104,"name":"online","context":{"idset":"10880"}}
+{"timestamp":1713497829.1461442,"name":"online","context":{"idset":"10883"}}
+{"timestamp":1713497829.2314231,"name":"online","context":{"idset":"10919"}}
+{"timestamp":1713497829.7169895,"name":"online","context":{"idset":"10921"}}
+{"timestamp":1713514159.5175042,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1713531589.1321206,"name":"offline","context":{"idset":"10519"}}
+{"timestamp":1713536916.9833832,"name":"online","context":{"idset":"11533"}}
+{"timestamp":1713536917.0871136,"name":"online","context":{"idset":"11634"}}
+{"timestamp":1713537688.6182065,"name":"drain","context":{"idset":"9863-9864,9909-9910,11057-11058,11181-11182,11201-11202","overwrite":0}}
+{"timestamp":1713537698.1688504,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1713537698.1733706,"name":"offline","context":{"idset":"9863"}}
+{"timestamp":1713537698.1777499,"name":"offline","context":{"idset":"9910"}}
+{"timestamp":1713537698.2770472,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1713537698.3807387,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1713537909.4185259,"name":"online","context":{"idset":"10519"}}
+{"timestamp":1713538406.7760348,"name":"offline","context":{"idset":"766"}}
+{"timestamp":1713538408.773417,"name":"offline","context":{"idset":"11723"}}
+{"timestamp":1713538456.7734694,"name":"offline","context":{"idset":"765"}}
+{"timestamp":1713538473.7555804,"name":"online","context":{"idset":"10241-10242"}}
+{"timestamp":1713538473.8501782,"name":"online","context":{"idset":"10276"}}
+{"timestamp":1713538474.0313447,"name":"online","context":{"idset":"10307"}}
+{"timestamp":1713538474.3169403,"name":"online","context":{"idset":"10232"}}
+{"timestamp":1713538474.4071221,"name":"online","context":{"idset":"10281"}}
+{"timestamp":1713538474.6467307,"name":"online","context":{"idset":"10266"}}
+{"timestamp":1713538474.7726512,"name":"online","context":{"idset":"10239"}}
+{"timestamp":1713538475.4400387,"name":"online","context":{"idset":"10238"}}
+{"timestamp":1713538475.8803287,"name":"online","context":{"idset":"10267"}}
+{"timestamp":1713538476.0900667,"name":"online","context":{"idset":"10292"}}
+{"timestamp":1713538476.1845357,"name":"online","context":{"idset":"10316,10341"}}
+{"timestamp":1713538476.1932726,"name":"online","context":{"idset":"10250"}}
+{"timestamp":1713538476.2972641,"name":"online","context":{"idset":"10270"}}
+{"timestamp":1713538476.3010948,"name":"online","context":{"idset":"10246"}}
+{"timestamp":1713538476.3063982,"name":"online","context":{"idset":"10298"}}
+{"timestamp":1713538476.5742309,"name":"online","context":{"idset":"10261,10263"}}
+{"timestamp":1713538476.7347322,"name":"online","context":{"idset":"10252,10315"}}
+{"timestamp":1713538476.9019949,"name":"online","context":{"idset":"10230,10234,10245,10249,10255,10260,10286"}}
+{"timestamp":1713538477.094691,"name":"online","context":{"idset":"10237,10251,10274,10280"}}
+{"timestamp":1713538477.1972764,"name":"online","context":{"idset":"10244,10277,10297,10300,10310,10328"}}
+{"timestamp":1713538477.3040023,"name":"online","context":{"idset":"10240,10271,10288,10321"}}
+{"timestamp":1713538477.4161243,"name":"online","context":{"idset":"10231,10306,10309,10334"}}
+{"timestamp":1713538477.5975556,"name":"online","context":{"idset":"10229,10247,10253-10254,10256,10273,10279,10287,10290,10302,10312,10317,10322,10342"}}
+{"timestamp":1713538477.7791951,"name":"online","context":{"idset":"10264,10295,10299,10324,10336,10347,10349"}}
+{"timestamp":1713538477.8854918,"name":"online","context":{"idset":"10243,10257,10262,10265,10272,10275,10285,10318-10320,10338-10339,10348"}}
+{"timestamp":1713538478.0610192,"name":"online","context":{"idset":"10233,10248,10258-10259,10278,10283,10294,10355"}}
+{"timestamp":1713538478.1676576,"name":"online","context":{"idset":"10268,10332,10340,10343-10344,10350,10354"}}
+{"timestamp":1713538478.2733562,"name":"online","context":{"idset":"10282,10284,10291,10293,10296,10301,10304-10305,10333,10335,10351"}}
+{"timestamp":1713538478.4563251,"name":"online","context":{"idset":"10269,10289,10308,10337,10356"}}
+{"timestamp":1713538478.6403942,"name":"online","context":{"idset":"10314,10323,10326,10345-10346,10352-10353"}}
+{"timestamp":1713538478.8501735,"name":"online","context":{"idset":"10303,10330"}}
+{"timestamp":1713538479.1878223,"name":"online","context":{"idset":"10313"}}
+{"timestamp":1713538538.0759425,"name":"online","context":{"idset":"11397"}}
+{"timestamp":1713538538.280915,"name":"online","context":{"idset":"11419"}}
+{"timestamp":1713538538.5772257,"name":"online","context":{"idset":"11420"}}
+{"timestamp":1713538984.3960185,"name":"undrain","context":{"idset":"773-774"}}
+{"timestamp":1713539070.421061,"name":"online","context":{"idset":"773"}}
+{"timestamp":1713539077.8954771,"name":"online","context":{"idset":"545,547"}}
+{"timestamp":1713539077.997694,"name":"online","context":{"idset":"541-543"}}
+{"timestamp":1713539078.1144221,"name":"online","context":{"idset":"544,546,548"}}
+{"timestamp":1713539091.3794832,"name":"online","context":{"idset":"774"}}
+{"timestamp":1713539312.7670095,"name":"drain","context":{"idset":"809-810,877-878","overwrite":0}}
+{"timestamp":1713539341.732945,"name":"offline","context":{"idset":"809"}}
+{"timestamp":1713539341.9189961,"name":"offline","context":{"idset":"878"}}
+{"timestamp":1713539341.9235032,"name":"offline","context":{"idset":"810"}}
+{"timestamp":1713539342.0229168,"name":"offline","context":{"idset":"877"}}
+{"timestamp":1713540456.6893053,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1713540456.6934435,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1713540456.7732759,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1713540640.3570411,"name":"drain","context":{"idset":"925-926,9863-9864","overwrite":0}}
+{"timestamp":1713545187.5182343,"name":"offline","context":{"idset":"7676"}}
+{"timestamp":1713545187.6046653,"name":"offline","context":{"idset":"7680"}}
+{"timestamp":1713545187.6119664,"name":"offline","context":{"idset":"7670"}}
+{"timestamp":1713545187.615839,"name":"offline","context":{"idset":"7678"}}
+{"timestamp":1713545187.6190848,"name":"offline","context":{"idset":"7672"}}
+{"timestamp":1713545187.6223178,"name":"offline","context":{"idset":"7677"}}
+{"timestamp":1713545187.6255391,"name":"offline","context":{"idset":"7682"}}
+{"timestamp":1713545187.7227895,"name":"offline","context":{"idset":"7684"}}
+{"timestamp":1713545187.8006275,"name":"offline","context":{"idset":"7681"}}
+{"timestamp":1713545187.8070171,"name":"offline","context":{"idset":"7683"}}
+{"timestamp":1713545187.8131564,"name":"offline","context":{"idset":"7669"}}
+{"timestamp":1713545187.8166988,"name":"offline","context":{"idset":"7671"}}
+{"timestamp":1713545187.8199613,"name":"offline","context":{"idset":"7673"}}
+{"timestamp":1713545187.8232081,"name":"offline","context":{"idset":"7674"}}
+{"timestamp":1713545187.8264513,"name":"offline","context":{"idset":"7675"}}
+{"timestamp":1713545187.9156272,"name":"offline","context":{"idset":"7679"}}
+{"timestamp":1713545189.5487163,"name":"offline","context":{"idset":"7685"}}
+{"timestamp":1713545189.560045,"name":"offline","context":{"idset":"7691"}}
+{"timestamp":1713545189.5682468,"name":"offline","context":{"idset":"7695"}}
+{"timestamp":1713545189.5743737,"name":"offline","context":{"idset":"7688"}}
+{"timestamp":1713545189.5874164,"name":"offline","context":{"idset":"7687"}}
+{"timestamp":1713545189.6776447,"name":"offline","context":{"idset":"7689"}}
+{"timestamp":1713545189.7630847,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1713545189.7673736,"name":"offline","context":{"idset":"7699"}}
+{"timestamp":1713545189.7735558,"name":"offline","context":{"idset":"7694"}}
+{"timestamp":1713545189.7771256,"name":"offline","context":{"idset":"7692"}}
+{"timestamp":1713545189.780443,"name":"offline","context":{"idset":"7686"}}
+{"timestamp":1713545189.7838557,"name":"offline","context":{"idset":"7690"}}
+{"timestamp":1713545189.8820419,"name":"offline","context":{"idset":"7697"}}
+{"timestamp":1713545189.8966126,"name":"offline","context":{"idset":"7698"}}
+{"timestamp":1713545189.8999982,"name":"offline","context":{"idset":"7700"}}
+{"timestamp":1713545189.9973834,"name":"offline","context":{"idset":"7696"}}
+{"timestamp":1713545191.4070833,"name":"offline","context":{"idset":"7701"}}
+{"timestamp":1713545191.5481682,"name":"offline","context":{"idset":"7710"}}
+{"timestamp":1713545191.6433306,"name":"offline","context":{"idset":"7705"}}
+{"timestamp":1713545191.6648099,"name":"offline","context":{"idset":"7703"}}
+{"timestamp":1713545191.6680582,"name":"offline","context":{"idset":"7704"}}
+{"timestamp":1713545191.7162571,"name":"offline","context":{"idset":"7706"}}
+{"timestamp":1713545191.7197049,"name":"offline","context":{"idset":"7711"}}
+{"timestamp":1713545191.7322528,"name":"offline","context":{"idset":"7702"}}
+{"timestamp":1713545191.7357121,"name":"offline","context":{"idset":"7707"}}
+{"timestamp":1713545191.7390151,"name":"offline","context":{"idset":"7708"}}
+{"timestamp":1713545191.8199019,"name":"offline","context":{"idset":"7709"}}
+{"timestamp":1713545191.9299974,"name":"offline","context":{"idset":"7713"}}
+{"timestamp":1713545191.9361069,"name":"offline","context":{"idset":"7712"}}
+{"timestamp":1713545191.9422297,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1713545192.0336871,"name":"offline","context":{"idset":"7714"}}
+{"timestamp":1713545192.1396065,"name":"offline","context":{"idset":"7716"}}
+{"timestamp":1713545193.3805845,"name":"offline","context":{"idset":"7717"}}
+{"timestamp":1713545193.5100474,"name":"offline","context":{"idset":"7718"}}
+{"timestamp":1713545193.6056285,"name":"offline","context":{"idset":"7720"}}
+{"timestamp":1713545193.615392,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1713545193.6196489,"name":"offline","context":{"idset":"7725"}}
+{"timestamp":1713545193.7126167,"name":"offline","context":{"idset":"7726"}}
+{"timestamp":1713545193.8004692,"name":"offline","context":{"idset":"7722"}}
+{"timestamp":1713545193.8068891,"name":"offline","context":{"idset":"7724"}}
+{"timestamp":1713545193.8101571,"name":"offline","context":{"idset":"7721"}}
+{"timestamp":1713545193.8145702,"name":"offline","context":{"idset":"7723"}}
+{"timestamp":1713545193.8189805,"name":"offline","context":{"idset":"7728"}}
+{"timestamp":1713545193.9156597,"name":"offline","context":{"idset":"7727"}}
+{"timestamp":1713545194.0204141,"name":"offline","context":{"idset":"7729"}}
+{"timestamp":1713545194.0266566,"name":"offline","context":{"idset":"7731"}}
+{"timestamp":1713545194.1173403,"name":"offline","context":{"idset":"7730"}}
+{"timestamp":1713545194.2987747,"name":"offline","context":{"idset":"7732"}}
+{"timestamp":1713545195.3676479,"name":"offline","context":{"idset":"7734"}}
+{"timestamp":1713545195.4662621,"name":"offline","context":{"idset":"7733"}}
+{"timestamp":1713545195.4787481,"name":"offline","context":{"idset":"7737"}}
+{"timestamp":1713545195.570205,"name":"offline","context":{"idset":"7738"}}
+{"timestamp":1713545195.5852599,"name":"offline","context":{"idset":"7735"}}
+{"timestamp":1713545195.5887196,"name":"offline","context":{"idset":"7740"}}
+{"timestamp":1713545195.6902487,"name":"offline","context":{"idset":"7742"}}
+{"timestamp":1713545195.7056556,"name":"offline","context":{"idset":"7736"}}
+{"timestamp":1713545195.7145317,"name":"offline","context":{"idset":"7746"}}
+{"timestamp":1713545195.8071618,"name":"offline","context":{"idset":"7744"}}
+{"timestamp":1713545195.8265157,"name":"offline","context":{"idset":"7741"}}
+{"timestamp":1713545195.8297849,"name":"offline","context":{"idset":"7747"}}
+{"timestamp":1713545195.9270854,"name":"offline","context":{"idset":"7743"}}
+{"timestamp":1713545196.0384934,"name":"offline","context":{"idset":"7745"}}
+{"timestamp":1713545196.1500981,"name":"offline","context":{"idset":"7748"}}
+{"timestamp":1713545197.33289,"name":"offline","context":{"idset":"7749"}}
+{"timestamp":1713545197.4363623,"name":"offline","context":{"idset":"7750"}}
+{"timestamp":1713545197.5207863,"name":"offline","context":{"idset":"7751"}}
+{"timestamp":1713545197.5296936,"name":"offline","context":{"idset":"7753"}}
+{"timestamp":1713545197.6227112,"name":"offline","context":{"idset":"7752"}}
+{"timestamp":1713545197.6265197,"name":"offline","context":{"idset":"7756"}}
+{"timestamp":1713545197.7266173,"name":"offline","context":{"idset":"7757"}}
+{"timestamp":1713545197.7328372,"name":"offline","context":{"idset":"7758"}}
+{"timestamp":1713545197.7389853,"name":"offline","context":{"idset":"7759"}}
+{"timestamp":1713545197.7426901,"name":"offline","context":{"idset":"7755"}}
+{"timestamp":1713545197.8354404,"name":"offline","context":{"idset":"7762"}}
+{"timestamp":1713545197.9096367,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1713545197.9161177,"name":"offline","context":{"idset":"7760"}}
+{"timestamp":1713545198.0126185,"name":"offline","context":{"idset":"7763"}}
+{"timestamp":1713545199.2670588,"name":"offline","context":{"idset":"7764"}}
+{"timestamp":1713545199.4329183,"name":"offline","context":{"idset":"7765"}}
+{"timestamp":1713545199.5139186,"name":"offline","context":{"idset":"7766"}}
+{"timestamp":1713545199.5170836,"name":"offline","context":{"idset":"7767"}}
+{"timestamp":1713545199.6095617,"name":"offline","context":{"idset":"7770"}}
+{"timestamp":1713545199.6128271,"name":"offline","context":{"idset":"7768"}}
+{"timestamp":1713545199.7039471,"name":"offline","context":{"idset":"7772"}}
+{"timestamp":1713545199.7093263,"name":"offline","context":{"idset":"7769"}}
+{"timestamp":1713545199.713717,"name":"offline","context":{"idset":"7774"}}
+{"timestamp":1713545199.7182875,"name":"offline","context":{"idset":"7773"}}
+{"timestamp":1713545199.722218,"name":"offline","context":{"idset":"7775"}}
+{"timestamp":1713545199.8206418,"name":"offline","context":{"idset":"7771"}}
+{"timestamp":1713545199.9091387,"name":"offline","context":{"idset":"7777"}}
+{"timestamp":1713545200.0078914,"name":"offline","context":{"idset":"7776"}}
+{"timestamp":1713545201.3156376,"name":"offline","context":{"idset":"7778"}}
+{"timestamp":1713545201.4166086,"name":"offline","context":{"idset":"7779"}}
+{"timestamp":1713545201.4215429,"name":"offline","context":{"idset":"7781"}}
+{"timestamp":1713545201.5232568,"name":"offline","context":{"idset":"7784"}}
+{"timestamp":1713545201.5635102,"name":"offline","context":{"idset":"7780"}}
+{"timestamp":1713545201.5671506,"name":"offline","context":{"idset":"7785"}}
+{"timestamp":1713545201.6625607,"name":"offline","context":{"idset":"7783"}}
+{"timestamp":1713545201.6767051,"name":"offline","context":{"idset":"7782"}}
+{"timestamp":1713545201.6842117,"name":"offline","context":{"idset":"7786"}}
+{"timestamp":1713545201.6906896,"name":"offline","context":{"idset":"7789"}}
+{"timestamp":1713545201.7834506,"name":"offline","context":{"idset":"7787"}}
+{"timestamp":1713545201.8657663,"name":"offline","context":{"idset":"7790"}}
+{"timestamp":1713545201.965276,"name":"offline","context":{"idset":"7788"}}
+{"timestamp":1713545202.0689025,"name":"offline","context":{"idset":"7791"}}
+{"timestamp":1713545203.2247946,"name":"offline","context":{"idset":"7792"}}
+{"timestamp":1713545203.3363698,"name":"offline","context":{"idset":"7795"}}
+{"timestamp":1713545203.416223,"name":"offline","context":{"idset":"7793"}}
+{"timestamp":1713545203.513459,"name":"offline","context":{"idset":"7794"}}
+{"timestamp":1713545203.6126547,"name":"offline","context":{"idset":"7796"}}
+{"timestamp":1713545597.3777514,"name":"drain","context":{"idset":"141-148","reason":"ARP Storm Problem testing","overwrite":0}}
+{"timestamp":1713545616.9937403,"name":"drain","context":{"idset":"133-140","reason":"ARP Storm Problem testing","overwrite":0}}
+{"timestamp":1713545634.1806197,"name":"drain","context":{"idset":"149-156","reason":"ARP Storm Problem testing","overwrite":0}}
+{"timestamp":1713545743.1527946,"name":"offline","context":{"idset":"151"}}
+{"timestamp":1713545743.1643431,"name":"offline","context":{"idset":"148"}}
+{"timestamp":1713545743.174396,"name":"offline","context":{"idset":"150"}}
+{"timestamp":1713545743.1830835,"name":"offline","context":{"idset":"141"}}
+{"timestamp":1713545743.2814322,"name":"offline","context":{"idset":"142"}}
+{"timestamp":1713545743.4356506,"name":"offline","context":{"idset":"155"}}
+{"timestamp":1713545743.4486427,"name":"offline","context":{"idset":"156"}}
+{"timestamp":1713545743.4602327,"name":"offline","context":{"idset":"143"}}
+{"timestamp":1713545743.4771471,"name":"offline","context":{"idset":"144"}}
+{"timestamp":1713545743.4900351,"name":"offline","context":{"idset":"145"}}
+{"timestamp":1713545743.4963977,"name":"offline","context":{"idset":"146"}}
+{"timestamp":1713545743.5023165,"name":"offline","context":{"idset":"147"}}
+{"timestamp":1713545743.5103855,"name":"offline","context":{"idset":"149"}}
+{"timestamp":1713545743.5167263,"name":"offline","context":{"idset":"152"}}
+{"timestamp":1713545743.5264363,"name":"offline","context":{"idset":"153"}}
+{"timestamp":1713545743.6821649,"name":"offline","context":{"idset":"154"}}
+{"timestamp":1713546521.0611145,"name":"online","context":{"idset":"9819"}}
+{"timestamp":1713546646.8298302,"name":"offline","context":{"idset":"140"}}
+{"timestamp":1713546647.5353482,"name":"offline","context":{"idset":"136"}}
+{"timestamp":1713546648.3614697,"name":"offline","context":{"idset":"134"}}
+{"timestamp":1713546648.6016009,"name":"offline","context":{"idset":"138"}}
+{"timestamp":1713546648.8480959,"name":"offline","context":{"idset":"135"}}
+{"timestamp":1713546649.4249516,"name":"offline","context":{"idset":"137"}}
+{"timestamp":1713546650.1922951,"name":"offline","context":{"idset":"133"}}
+{"timestamp":1713546650.8815455,"name":"offline","context":{"idset":"139"}}
+{"timestamp":1713546977.8032594,"name":"online","context":{"idset":"9820"}}
+{"timestamp":1713547199.0512249,"name":"online","context":{"idset":"9734"}}
+{"timestamp":1713547205.2869322,"name":"online","context":{"idset":"9788"}}
+{"timestamp":1713547441.3668644,"name":"drain","context":{"idset":"811-812","overwrite":0}}
+{"timestamp":1713549544.772959,"name":"offline","context":{"idset":"9734"}}
+{"timestamp":1713550828.1542823,"name":"online","context":{"idset":"9734"}}
+{"timestamp":1713551324.684099,"name":"offline","context":{"idset":"8950"}}
+{"timestamp":1713551324.7736273,"name":"offline","context":{"idset":"8951"}}
+{"timestamp":1713551326.7718318,"name":"offline","context":{"idset":"8952"}}
+{"timestamp":1713551328.6860232,"name":"offline","context":{"idset":"8958"}}
+{"timestamp":1713551328.772464,"name":"offline","context":{"idset":"8959"}}
+{"timestamp":1713551330.7722573,"name":"offline","context":{"idset":"8961"}}
+{"timestamp":1713551336.7760813,"name":"offline","context":{"idset":"8975"}}
+{"timestamp":1713551338.7710452,"name":"offline","context":{"idset":"8981"}}
+{"timestamp":1713551342.7730179,"name":"offline","context":{"idset":"8988"}}
+{"timestamp":1713551344.7724714,"name":"offline","context":{"idset":"8991"}}
+{"timestamp":1713551346.7721827,"name":"offline","context":{"idset":"8992"}}
+{"timestamp":1713551348.6849103,"name":"offline","context":{"idset":"8997"}}
+{"timestamp":1713551348.7764909,"name":"offline","context":{"idset":"8998"}}
+{"timestamp":1713551350.6948259,"name":"offline","context":{"idset":"9001"}}
+{"timestamp":1713551350.6980841,"name":"offline","context":{"idset":"9002"}}
+{"timestamp":1713551350.7733862,"name":"offline","context":{"idset":"9003"}}
+{"timestamp":1713551352.6924112,"name":"offline","context":{"idset":"9004"}}
+{"timestamp":1713551352.7745707,"name":"offline","context":{"idset":"9007"}}
+{"timestamp":1713551354.7728953,"name":"offline","context":{"idset":"9009"}}
+{"timestamp":1713551356.6920006,"name":"offline","context":{"idset":"9012"}}
+{"timestamp":1713551356.6954601,"name":"offline","context":{"idset":"9013"}}
+{"timestamp":1713551356.6989739,"name":"offline","context":{"idset":"9014"}}
+{"timestamp":1713551356.7724535,"name":"offline","context":{"idset":"9016"}}
+{"timestamp":1713551358.6892715,"name":"offline","context":{"idset":"9017"}}
+{"timestamp":1713551358.6930711,"name":"offline","context":{"idset":"9020"}}
+{"timestamp":1713551358.7731397,"name":"offline","context":{"idset":"9021"}}
+{"timestamp":1713551360.6904008,"name":"offline","context":{"idset":"9022"}}
+{"timestamp":1713551360.6942415,"name":"offline","context":{"idset":"9023"}}
+{"timestamp":1713551360.7721629,"name":"offline","context":{"idset":"9024"}}
+{"timestamp":1713551362.7735236,"name":"offline","context":{"idset":"9026"}}
+{"timestamp":1713551364.7728434,"name":"offline","context":{"idset":"9028"}}
+{"timestamp":1713551366.6905718,"name":"offline","context":{"idset":"9033"}}
+{"timestamp":1713551366.7722855,"name":"offline","context":{"idset":"9034"}}
+{"timestamp":1713551368.6934087,"name":"offline","context":{"idset":"9037"}}
+{"timestamp":1713551368.6968608,"name":"offline","context":{"idset":"9038"}}
+{"timestamp":1713551368.700285,"name":"offline","context":{"idset":"9039"}}
+{"timestamp":1713551368.775635,"name":"offline","context":{"idset":"9040"}}
+{"timestamp":1713551370.6865804,"name":"offline","context":{"idset":"9042"}}
+{"timestamp":1713551370.7731392,"name":"offline","context":{"idset":"9044"}}
+{"timestamp":1713551372.690629,"name":"offline","context":{"idset":"9045"}}
+{"timestamp":1713551372.6937416,"name":"offline","context":{"idset":"9047"}}
+{"timestamp":1713551372.6967659,"name":"offline","context":{"idset":"9048"}}
+{"timestamp":1713551372.7730086,"name":"offline","context":{"idset":"9050"}}
+{"timestamp":1713551374.6904294,"name":"offline","context":{"idset":"9051"}}
+{"timestamp":1713551374.6960144,"name":"offline","context":{"idset":"9053"}}
+{"timestamp":1713551374.7732697,"name":"offline","context":{"idset":"9054"}}
+{"timestamp":1713551376.6886578,"name":"offline","context":{"idset":"9055"}}
+{"timestamp":1713551376.6917288,"name":"offline","context":{"idset":"9056"}}
+{"timestamp":1713551376.7713785,"name":"offline","context":{"idset":"9060"}}
+{"timestamp":1713551386.7730787,"name":"offline","context":{"idset":"8978"}}
+{"timestamp":1713551404.7723973,"name":"offline","context":{"idset":"9011"}}
+{"timestamp":1713551741.0513191,"name":"offline","context":{"idset":"10618"}}
+{"timestamp":1713552018.4983103,"name":"drain","context":{"idset":"929-930,11181-11182","overwrite":0}}
+{"timestamp":1713552035.8384056,"name":"offline","context":{"idset":"930"}}
+{"timestamp":1713552215.5809484,"name":"undrain","context":{"idset":"11790,11891"}}
+{"timestamp":1713552578.9948723,"name":"undrain","context":{"idset":"9863-9864,9909-9910,11057-11058,11201-11202"}}
+{"timestamp":1713553068.8825023,"name":"online","context":{"idset":"9863"}}
+{"timestamp":1713553069.1269288,"name":"online","context":{"idset":"9910,11202"}}
+{"timestamp":1713553069.4181061,"name":"online","context":{"idset":"9864,11057,11201"}}
+{"timestamp":1713553069.6528404,"name":"online","context":{"idset":"9909,11058"}}
+{"timestamp":1713553184.4056239,"name":"online","context":{"idset":"11203"}}
+{"timestamp":1713553320.2757006,"name":"drain","context":{"idset":"11191-11192","overwrite":0}}
+{"timestamp":1713553343.3642437,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1713553591.5898242,"name":"drain","context":{"idset":"549-556","reason":"New Blade Installation","overwrite":0}}
+{"timestamp":1713553719.6063812,"name":"online","context":{"idset":"9733"}}
+{"timestamp":1713554788.6511211,"name":"online","context":{"idset":"11192"}}
+{"timestamp":1713554788.922183,"name":"online","context":{"idset":"11191"}}
+{"timestamp":1713554805.7546613,"name":"undrain","context":{"idset":"11191-11192"}}
+{"timestamp":1713554956.772121,"name":"offline","context":{"idset":"10639"}}
+{"timestamp":1713555171.001754,"name":"online","context":{"idset":"10110"}}
+{"timestamp":1713556321.597549,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1713556323.4890714,"name":"online","context":{"idset":"10110"}}
+{"timestamp":1713556323.9558935,"name":"online","context":{"idset":"10157,10163,10221"}}
+{"timestamp":1713556324.1357729,"name":"online","context":{"idset":"10112-10113,10812,10826"}}
+{"timestamp":1713556324.3535595,"name":"online","context":{"idset":"10753,11792"}}
+{"timestamp":1713556324.4693999,"name":"online","context":{"idset":"10815,11838"}}
+{"timestamp":1713556325.0009251,"name":"online","context":{"idset":"10142,10824,11767,11787,11834"}}
+{"timestamp":1713556325.2027574,"name":"online","context":{"idset":"10215,10225,10770,10793,10820,10825,10840,11786,11791,11796,11801,11831,11836,11858,11870"}}
+{"timestamp":1713556325.3054907,"name":"online","context":{"idset":"10148,10154,10790,10796,10811,10841,10856,11810,11866"}}
+{"timestamp":1713556325.4923224,"name":"online","context":{"idset":"10111,10147,10150,10155,10175,10189,10227,10757,10764,10781,10832,11797,11816,11851,11873,11886,11891"}}
+{"timestamp":1713556325.5945985,"name":"online","context":{"idset":"10193,10208,10219,10805,11773,11784,11823,11871"}}
+{"timestamp":1713556325.6982095,"name":"online","context":{"idset":"10129,10149,11794"}}
+{"timestamp":1713556325.862478,"name":"online","context":{"idset":"10187,10861,11772,11795"}}
+{"timestamp":1713556462.5846817,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1713556463.7297308,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1713556463.9343677,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1713556464.033663,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1713556464.3907034,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1713556826.7189801,"name":"offline","context":{"idset":"9399"}}
+{"timestamp":1713556826.7725446,"name":"offline","context":{"idset":"9400"}}
+{"timestamp":1713556914.6885417,"name":"offline","context":{"idset":"9405"}}
+{"timestamp":1713556914.7725732,"name":"offline","context":{"idset":"9406"}}
+{"timestamp":1713557511.7452393,"name":"undrain","context":{"idset":"811-812"}}
+{"timestamp":1713557752.921227,"name":"drain","context":{"idset":"811-812","reason":"--reason Running hpe diags - JM","overwrite":0}}
+{"timestamp":1713558274.6643858,"name":"drain","context":{"idset":"777-778","overwrite":0}}
+{"timestamp":1713558317.3296719,"name":"online","context":{"idset":"777"}}
+{"timestamp":1713558317.7165959,"name":"online","context":{"idset":"778"}}
+{"timestamp":1713558342.4242108,"name":"undrain","context":{"idset":"777-778"}}
+{"timestamp":1713558370.8417406,"name":"online","context":{"idset":"8036"}}
+{"timestamp":1713558482.0323198,"name":"online","context":{"idset":"8020"}}
+{"timestamp":1713558927.0583436,"name":"online","context":{"idset":"7987"}}
+{"timestamp":1713560378.9793062,"name":"online","context":{"idset":"541"}}
+{"timestamp":1713560379.1559694,"name":"online","context":{"idset":"545"}}
+{"timestamp":1713560379.5484431,"name":"online","context":{"idset":"546"}}
+{"timestamp":1713560577.8496423,"name":"online","context":{"idset":"542-543,548"}}
+{"timestamp":1713560578.0458794,"name":"online","context":{"idset":"544"}}
+{"timestamp":1713560628.3021281,"name":"online","context":{"idset":"547"}}
+{"timestamp":1713561919.0674427,"name":"drain","context":{"idset":"11173-11188","overwrite":0}}
+{"timestamp":1713561936.796062,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1713561936.8861101,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1713561936.8937881,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1713561936.8994892,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1713561936.9052083,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1713561936.995327,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1713561937.0287578,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1713561937.0334229,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1713561937.0368516,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1713561937.0398252,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1713561937.0428102,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1713561937.1347985,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1713561937.1417732,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1713561937.240896,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1713562064.7712579,"name":"offline","context":{"idset":"10704"}}
+{"timestamp":1713562156.7728195,"name":"offline","context":{"idset":"976"}}
+{"timestamp":1713562389.5699332,"name":"online","context":{"idset":"7676"}}
+{"timestamp":1713562389.7520385,"name":"online","context":{"idset":"7670,7672"}}
+{"timestamp":1713562389.9585123,"name":"online","context":{"idset":"7679"}}
+{"timestamp":1713562390.1335518,"name":"online","context":{"idset":"7682"}}
+{"timestamp":1713562390.2367489,"name":"online","context":{"idset":"7673,7677,7687,7694"}}
+{"timestamp":1713562390.3447711,"name":"online","context":{"idset":"7685,7690,7698"}}
+{"timestamp":1713562390.4585946,"name":"online","context":{"idset":"7669,7708"}}
+{"timestamp":1713562390.541394,"name":"online","context":{"idset":"7680,7684"}}
+{"timestamp":1713562390.7118859,"name":"online","context":{"idset":"7671"}}
+{"timestamp":1713562390.8205092,"name":"online","context":{"idset":"7678,7689"}}
+{"timestamp":1713562390.9018667,"name":"online","context":{"idset":"7675"}}
+{"timestamp":1713562391.0059271,"name":"online","context":{"idset":"7681,7710"}}
+{"timestamp":1713562391.101409,"name":"online","context":{"idset":"7692,7709"}}
+{"timestamp":1713562391.4429719,"name":"online","context":{"idset":"7718"}}
+{"timestamp":1713562392.3778214,"name":"online","context":{"idset":"7717"}}
+{"timestamp":1713562392.4727244,"name":"online","context":{"idset":"7763"}}
+{"timestamp":1713562392.5665545,"name":"online","context":{"idset":"7691,7765"}}
+{"timestamp":1713562392.7544165,"name":"online","context":{"idset":"7715"}}
+{"timestamp":1713562392.849504,"name":"online","context":{"idset":"7722"}}
+{"timestamp":1713562392.93701,"name":"online","context":{"idset":"7706"}}
+{"timestamp":1713562393.245074,"name":"online","context":{"idset":"7771"}}
+{"timestamp":1713562393.3460896,"name":"online","context":{"idset":"7686,7726,7746"}}
+{"timestamp":1713562393.4306228,"name":"online","context":{"idset":"7786"}}
+{"timestamp":1713562393.5089691,"name":"online","context":{"idset":"7703"}}
+{"timestamp":1713562393.6203191,"name":"online","context":{"idset":"7742"}}
+{"timestamp":1713562393.7344275,"name":"online","context":{"idset":"7674,7688,7693,7749,7777"}}
+{"timestamp":1713562393.8433974,"name":"online","context":{"idset":"7713,7762,7793"}}
+{"timestamp":1713562393.9667456,"name":"online","context":{"idset":"7770,7787"}}
+{"timestamp":1713562394.1506188,"name":"online","context":{"idset":"7683,7700,7740"}}
+{"timestamp":1713562394.3381991,"name":"online","context":{"idset":"7699,7732,7743"}}
+{"timestamp":1713562394.4405985,"name":"online","context":{"idset":"7696,7702,7705,7721,7731,7741,7761,7772,7781"}}
+{"timestamp":1713562394.5439553,"name":"online","context":{"idset":"7723,7730,7764,7780"}}
+{"timestamp":1713562394.6448779,"name":"online","context":{"idset":"7707,7756,7788"}}
+{"timestamp":1713562394.8539493,"name":"online","context":{"idset":"7704,7711-7712,7725,7736,7752,7757,7775,7785,7796"}}
+{"timestamp":1713562395.0486958,"name":"online","context":{"idset":"7727"}}
+{"timestamp":1713562395.1646457,"name":"online","context":{"idset":"7733,7735,7737,7748,7753,7760,7767,7769,7776,7784"}}
+{"timestamp":1713562395.2704585,"name":"online","context":{"idset":"7744"}}
+{"timestamp":1713562395.3740747,"name":"online","context":{"idset":"7697,7701,7714,7719,7729,7734,7738,7747,7754-7755,7759,7768,7782-7783,7789-7790,7792"}}
+{"timestamp":1713562395.5583,"name":"online","context":{"idset":"7724,7758,7778,7791,7795"}}
+{"timestamp":1713562395.6597457,"name":"online","context":{"idset":"7750,7766,7779,7794"}}
+{"timestamp":1713562395.761771,"name":"online","context":{"idset":"7720,7728,7751,7773"}}
+{"timestamp":1713562396.2491753,"name":"online","context":{"idset":"7774"}}
+{"timestamp":1713562405.7749097,"name":"online","context":{"idset":"10381"}}
+{"timestamp":1713562405.8851097,"name":"online","context":{"idset":"10456,10465"}}
+{"timestamp":1713562406.0651658,"name":"online","context":{"idset":"10409-10410,10455"}}
+{"timestamp":1713562406.3396177,"name":"online","context":{"idset":"10466"}}
+{"timestamp":1713563202.7734632,"name":"offline","context":{"idset":"800"}}
+{"timestamp":1713563212.7751646,"name":"offline","context":{"idset":"802"}}
+{"timestamp":1713563296.8325214,"name":"undrain","context":{"idset":"283"}}
+{"timestamp":1713563384.7720706,"name":"offline","context":{"idset":"801"}}
+{"timestamp":1713563470.485939,"name":"undrain","context":{"idset":"361,420"}}
+{"timestamp":1713563908.4569855,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1713563909.3426812,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1713563909.710947,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1713563909.9348323,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1713563910.02002,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1713563910.1183922,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1713563915.0718594,"name":"drain","context":{"idset":"548","reason":"epilog failed for jobid fqUWkS8j8hM","overwrite":0}}
+{"timestamp":1713563915.1716456,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1713563916.1489913,"name":"drain","context":{"idset":"547","reason":"epilog failed for jobid fqUWmieM3AX","overwrite":0}}
+{"timestamp":1713563916.295085,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1713564589.6583929,"name":"online","context":{"idset":"11174,11176"}}
+{"timestamp":1713564589.8754463,"name":"online","context":{"idset":"11179,11184"}}
+{"timestamp":1713564589.9779947,"name":"online","context":{"idset":"11175,11180,11182,11186"}}
+{"timestamp":1713564590.1672428,"name":"online","context":{"idset":"11173,11178,11181"}}
+{"timestamp":1713564590.377651,"name":"online","context":{"idset":"11177,11187-11188"}}
+{"timestamp":1713564590.5633109,"name":"online","context":{"idset":"11185"}}
+{"timestamp":1713564590.8570824,"name":"online","context":{"idset":"11183"}}
+{"timestamp":1713564603.9912984,"name":"undrain","context":{"idset":"11173-11188"}}
+{"timestamp":1713564748.7720599,"name":"offline","context":{"idset":"9636"}}
+{"timestamp":1713564980.5596151,"name":"drain","context":{"idset":"90","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713564985.2120481,"name":"drain","context":{"idset":"206","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713564989.1392832,"name":"drain","context":{"idset":"411","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713565634.7137001,"name":"offline","context":{"idset":"11717"}}
+{"timestamp":1713565634.717521,"name":"offline","context":{"idset":"11718"}}
+{"timestamp":1713565634.7210822,"name":"offline","context":{"idset":"11719"}}
+{"timestamp":1713565634.7247028,"name":"offline","context":{"idset":"11721"}}
+{"timestamp":1713565634.7284472,"name":"offline","context":{"idset":"11726"}}
+{"timestamp":1713565634.7316241,"name":"offline","context":{"idset":"11727"}}
+{"timestamp":1713565634.7347577,"name":"offline","context":{"idset":"11728"}}
+{"timestamp":1713565634.7378504,"name":"offline","context":{"idset":"11729"}}
+{"timestamp":1713565634.740977,"name":"offline","context":{"idset":"11730"}}
+{"timestamp":1713565634.7441397,"name":"offline","context":{"idset":"11731"}}
+{"timestamp":1713565634.8459275,"name":"offline","context":{"idset":"11732"}}
+{"timestamp":1713565654.7732344,"name":"offline","context":{"idset":"11720"}}
+{"timestamp":1713566798.8961368,"name":"online","context":{"idset":"9934"}}
+{"timestamp":1713566924.6849473,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1713566924.7723825,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1713567260.7747169,"name":"offline","context":{"idset":"7823"}}
+{"timestamp":1713567306.7718253,"name":"offline","context":{"idset":"943"}}
+{"timestamp":1713567356.7743475,"name":"offline","context":{"idset":"944"}}
+{"timestamp":1713567745.3793852,"name":"online","context":{"idset":"7716"}}
+{"timestamp":1713567745.4888098,"name":"online","context":{"idset":"7745"}}
+{"timestamp":1713567745.7491789,"name":"online","context":{"idset":"7695"}}
+{"timestamp":1713570161.7870798,"name":"drain","context":{"idset":"11365-11380","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1713571380.7716217,"name":"offline","context":{"idset":"8883"}}
+{"timestamp":1713572052.7723989,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1713574216.7374074,"name":"drain","context":{"idset":"7739","reason":"epilog failed for jobid fqVLQvJZBvf","overwrite":0}}
+{"timestamp":1713574216.8376219,"name":"offline","context":{"idset":"7739"}}
+{"timestamp":1713583453.1396449,"name":"online","context":{"idset":"11203"}}
+{"timestamp":1713587280.7724402,"name":"offline","context":{"idset":"7921"}}
+{"timestamp":1713631909.3869057,"name":"online","context":{"idset":"7978"}}
+{"timestamp":1713643650.8064463,"name":"offline","context":{"idset":"7909"}}
+{"timestamp":1713644731.6399281,"name":"offline","context":{"idset":"7911"}}
+{"timestamp":1713644732.6572785,"name":"offline","context":{"idset":"7923"}}
+{"timestamp":1713644732.8520575,"name":"offline","context":{"idset":"7910"}}
+{"timestamp":1713644732.9350705,"name":"offline","context":{"idset":"7913"}}
+{"timestamp":1713644732.9389021,"name":"offline","context":{"idset":"7915"}}
+{"timestamp":1713644733.0381103,"name":"offline","context":{"idset":"7916"}}
+{"timestamp":1713644733.213618,"name":"offline","context":{"idset":"7920"}}
+{"timestamp":1713644733.3171616,"name":"offline","context":{"idset":"7918"}}
+{"timestamp":1713644733.4930074,"name":"offline","context":{"idset":"7914"}}
+{"timestamp":1713644733.8479257,"name":"offline","context":{"idset":"7912"}}
+{"timestamp":1713644734.2745037,"name":"offline","context":{"idset":"7917"}}
+{"timestamp":1713644734.462775,"name":"offline","context":{"idset":"7922"}}
+{"timestamp":1713644734.6465824,"name":"offline","context":{"idset":"7924"}}
+{"timestamp":1713644734.8477175,"name":"offline","context":{"idset":"7919"}}
+{"timestamp":1713647151.3990211,"name":"online","context":{"idset":"9161"}}
+{"timestamp":1713649568.8373826,"name":"online","context":{"idset":"11654,11658"}}
+{"timestamp":1713649569.7590079,"name":"online","context":{"idset":"11656"}}
+{"timestamp":1713649699.2307615,"name":"online","context":{"idset":"11672,11682"}}
+{"timestamp":1713649699.4418514,"name":"online","context":{"idset":"11669"}}
+{"timestamp":1713649863.9317214,"name":"offline","context":{"idset":"7701"}}
+{"timestamp":1713649864.5734076,"name":"offline","context":{"idset":"7706"}}
+{"timestamp":1713649864.7449777,"name":"offline","context":{"idset":"7707"}}
+{"timestamp":1713649864.8267531,"name":"offline","context":{"idset":"7711"}}
+{"timestamp":1713649864.8426952,"name":"offline","context":{"idset":"7710"}}
+{"timestamp":1713649864.9416835,"name":"offline","context":{"idset":"7714"}}
+{"timestamp":1713649865.1775339,"name":"offline","context":{"idset":"7708"}}
+{"timestamp":1713649865.189606,"name":"offline","context":{"idset":"7704"}}
+{"timestamp":1713649865.2869642,"name":"offline","context":{"idset":"7716"}}
+{"timestamp":1713649865.4638231,"name":"offline","context":{"idset":"7703"}}
+{"timestamp":1713649865.7505322,"name":"offline","context":{"idset":"7713"}}
+{"timestamp":1713649865.832093,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1713649865.9272413,"name":"offline","context":{"idset":"7702"}}
+{"timestamp":1713649866.1846516,"name":"offline","context":{"idset":"7709"}}
+{"timestamp":1713649866.7306566,"name":"offline","context":{"idset":"7712"}}
+{"timestamp":1713649867.2118654,"name":"offline","context":{"idset":"7705"}}
+{"timestamp":1713649868.124264,"name":"online","context":{"idset":"11702"}}
+{"timestamp":1713649869.0178533,"name":"online","context":{"idset":"11712"}}
+{"timestamp":1713650544.6471829,"name":"online","context":{"idset":"11751"}}
+{"timestamp":1713650544.8757741,"name":"online","context":{"idset":"11760"}}
+{"timestamp":1713650544.9913194,"name":"online","context":{"idset":"11752-11753"}}
+{"timestamp":1713650545.1638308,"name":"online","context":{"idset":"11762"}}
+{"timestamp":1713650891.6115005,"name":"online","context":{"idset":"11279"}}
+{"timestamp":1713650891.8024464,"name":"online","context":{"idset":"11270"}}
+{"timestamp":1713650892.0174918,"name":"online","context":{"idset":"11277"}}
+{"timestamp":1713650892.120621,"name":"online","context":{"idset":"11275,11284"}}
+{"timestamp":1713650892.4005933,"name":"online","context":{"idset":"11271,11282"}}
+{"timestamp":1713650892.5243394,"name":"online","context":{"idset":"11269,11278,11281,11283"}}
+{"timestamp":1713650892.7099516,"name":"online","context":{"idset":"11273,11276,11280"}}
+{"timestamp":1713650892.8125105,"name":"online","context":{"idset":"11272,11274"}}
+{"timestamp":1713651102.7319114,"name":"online","context":{"idset":"11290,11296"}}
+{"timestamp":1713651102.8344331,"name":"online","context":{"idset":"11286,11293,11295,11299"}}
+{"timestamp":1713651103.0132468,"name":"online","context":{"idset":"11289"}}
+{"timestamp":1713651103.1289995,"name":"online","context":{"idset":"11294,11297"}}
+{"timestamp":1713651103.3215458,"name":"online","context":{"idset":"11287,11291-11292,11300"}}
+{"timestamp":1713651103.5072181,"name":"online","context":{"idset":"11288,11298"}}
+{"timestamp":1713651301.9249821,"name":"online","context":{"idset":"11309-11310"}}
+{"timestamp":1713651302.1334653,"name":"online","context":{"idset":"11312,11314"}}
+{"timestamp":1713651302.4248545,"name":"online","context":{"idset":"11303-11304,11306,11311"}}
+{"timestamp":1713651302.6631835,"name":"online","context":{"idset":"11308,11316"}}
+{"timestamp":1713651302.8768685,"name":"online","context":{"idset":"11302,11305,11313,11315"}}
+{"timestamp":1713651303.197969,"name":"online","context":{"idset":"11307"}}
+{"timestamp":1713651453.4860187,"name":"online","context":{"idset":"11342"}}
+{"timestamp":1713651453.8232901,"name":"online","context":{"idset":"11334"}}
+{"timestamp":1713651454.0165584,"name":"online","context":{"idset":"11335"}}
+{"timestamp":1713651454.3446972,"name":"online","context":{"idset":"11346-11347"}}
+{"timestamp":1713651454.5377877,"name":"online","context":{"idset":"11333,11337,11340,11344-11345"}}
+{"timestamp":1713651454.8526268,"name":"online","context":{"idset":"11339,11341,11348"}}
+{"timestamp":1713651455.0403171,"name":"online","context":{"idset":"11336,11343"}}
+{"timestamp":1713651518.6896143,"name":"online","context":{"idset":"7711"}}
+{"timestamp":1713651695.7226424,"name":"online","context":{"idset":"11350,11354"}}
+{"timestamp":1713651695.9195716,"name":"online","context":{"idset":"11351,11358,11361,11363"}}
+{"timestamp":1713651696.2969964,"name":"online","context":{"idset":"11356,11364"}}
+{"timestamp":1713651696.4867425,"name":"online","context":{"idset":"11352-11353,11357"}}
+{"timestamp":1713651696.6002357,"name":"online","context":{"idset":"11359-11360,11362"}}
+{"timestamp":1713651696.8324771,"name":"online","context":{"idset":"11349"}}
+{"timestamp":1713651697.0268617,"name":"online","context":{"idset":"11355"}}
+{"timestamp":1713652234.8960235,"name":"online","context":{"idset":"9406"}}
+{"timestamp":1713652235.114296,"name":"online","context":{"idset":"9405"}}
+{"timestamp":1713652235.3020372,"name":"online","context":{"idset":"9400"}}
+{"timestamp":1713654706.5886865,"name":"online","context":{"idset":"7704"}}
+{"timestamp":1713654706.8183289,"name":"online","context":{"idset":"7707"}}
+{"timestamp":1713654706.9213245,"name":"online","context":{"idset":"7708"}}
+{"timestamp":1713654707.1175969,"name":"online","context":{"idset":"7705,7713"}}
+{"timestamp":1713654707.4057391,"name":"online","context":{"idset":"7701,7709-7710"}}
+{"timestamp":1713654707.5949237,"name":"online","context":{"idset":"7703,7706,7716"}}
+{"timestamp":1713654707.7073107,"name":"online","context":{"idset":"7714"}}
+{"timestamp":1713654707.8986154,"name":"online","context":{"idset":"7702,7712"}}
+{"timestamp":1713657257.2023306,"name":"online","context":{"idset":"9416,9422"}}
+{"timestamp":1713657257.6026285,"name":"online","context":{"idset":"9418,9423"}}
+{"timestamp":1713657257.7091849,"name":"online","context":{"idset":"9421,9424,9427"}}
+{"timestamp":1713657257.8972485,"name":"online","context":{"idset":"9413-9415,9425,9428"}}
+{"timestamp":1713657258.104516,"name":"online","context":{"idset":"9420"}}
+{"timestamp":1713657258.293771,"name":"online","context":{"idset":"9419"}}
+{"timestamp":1713657258.6292293,"name":"online","context":{"idset":"9417,9426"}}
+{"timestamp":1713657722.7545595,"name":"offline","context":{"idset":"7716"}}
+{"timestamp":1713657766.7490594,"name":"offline","context":{"idset":"946"}}
+{"timestamp":1713657840.5773649,"name":"online","context":{"idset":"11266"}}
+{"timestamp":1713657840.8268466,"name":"online","context":{"idset":"11253,11257"}}
+{"timestamp":1713657841.1580994,"name":"online","context":{"idset":"11258,11260-11261"}}
+{"timestamp":1713657841.3429232,"name":"online","context":{"idset":"11254-11255,11259,11268"}}
+{"timestamp":1713657841.527796,"name":"online","context":{"idset":"11262,11265,11267"}}
+{"timestamp":1713657841.6420145,"name":"online","context":{"idset":"11263-11264"}}
+{"timestamp":1713658137.4025214,"name":"online","context":{"idset":"11318,11330"}}
+{"timestamp":1713658137.5107937,"name":"online","context":{"idset":"11323"}}
+{"timestamp":1713658137.7125685,"name":"online","context":{"idset":"11317,11320,11322,11331"}}
+{"timestamp":1713658137.9014838,"name":"online","context":{"idset":"11324,11329"}}
+{"timestamp":1713658138.0074043,"name":"online","context":{"idset":"11319"}}
+{"timestamp":1713658138.1972933,"name":"online","context":{"idset":"11321,11328,11332"}}
+{"timestamp":1713658138.4902565,"name":"online","context":{"idset":"11327"}}
+{"timestamp":1713660282.8509884,"name":"offline","context":{"idset":"7670"}}
+{"timestamp":1713660283.6747682,"name":"offline","context":{"idset":"7669"}}
+{"timestamp":1713660283.7709138,"name":"offline","context":{"idset":"7676"}}
+{"timestamp":1713660283.8681066,"name":"offline","context":{"idset":"7677"}}
+{"timestamp":1713660284.0710762,"name":"offline","context":{"idset":"7673"}}
+{"timestamp":1713660284.1704416,"name":"offline","context":{"idset":"7671"}}
+{"timestamp":1713660284.1749544,"name":"offline","context":{"idset":"7678"}}
+{"timestamp":1713660284.1795409,"name":"offline","context":{"idset":"7682"}}
+{"timestamp":1713660284.272202,"name":"offline","context":{"idset":"7680"}}
+{"timestamp":1713660284.3833351,"name":"offline","context":{"idset":"7684"}}
+{"timestamp":1713660284.4826841,"name":"offline","context":{"idset":"7683"}}
+{"timestamp":1713660284.6785195,"name":"offline","context":{"idset":"7679"}}
+{"timestamp":1713660284.7998729,"name":"offline","context":{"idset":"7681"}}
+{"timestamp":1713660284.9018664,"name":"offline","context":{"idset":"7675"}}
+{"timestamp":1713660285.0942323,"name":"offline","context":{"idset":"7685"}}
+{"timestamp":1713660285.5589592,"name":"offline","context":{"idset":"7674"}}
+{"timestamp":1713660285.7482502,"name":"offline","context":{"idset":"7672"}}
+{"timestamp":1713660285.9862967,"name":"offline","context":{"idset":"7686"}}
+{"timestamp":1713660286.0818677,"name":"offline","context":{"idset":"7688"}}
+{"timestamp":1713660286.3899734,"name":"offline","context":{"idset":"7687"}}
+{"timestamp":1713660286.9427361,"name":"offline","context":{"idset":"7689"}}
+{"timestamp":1713660287.2123952,"name":"offline","context":{"idset":"7699"}}
+{"timestamp":1713660287.3525391,"name":"offline","context":{"idset":"7692"}}
+{"timestamp":1713660287.45315,"name":"offline","context":{"idset":"7700"}}
+{"timestamp":1713660287.5890009,"name":"offline","context":{"idset":"7690"}}
+{"timestamp":1713660287.682961,"name":"offline","context":{"idset":"7691"}}
+{"timestamp":1713660287.7844744,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1713660287.7930622,"name":"offline","context":{"idset":"7695"}}
+{"timestamp":1713660287.8897374,"name":"offline","context":{"idset":"7698"}}
+{"timestamp":1713660288.0953987,"name":"offline","context":{"idset":"7717"}}
+{"timestamp":1713660288.4054039,"name":"offline","context":{"idset":"7718"}}
+{"timestamp":1713660288.6045735,"name":"offline","context":{"idset":"7720"}}
+{"timestamp":1713660288.9180512,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1713660288.9883347,"name":"offline","context":{"idset":"7697"}}
+{"timestamp":1713660289.0871148,"name":"offline","context":{"idset":"7696"}}
+{"timestamp":1713660289.2693815,"name":"offline","context":{"idset":"7721"}}
+{"timestamp":1713660289.4624941,"name":"offline","context":{"idset":"7694"}}
+{"timestamp":1713660289.8810418,"name":"offline","context":{"idset":"7723"}}
+{"timestamp":1713660290.0593088,"name":"offline","context":{"idset":"7724"}}
+{"timestamp":1713660290.3227229,"name":"offline","context":{"idset":"7749"}}
+{"timestamp":1713660290.8212438,"name":"offline","context":{"idset":"7750"}}
+{"timestamp":1713660291.1456134,"name":"offline","context":{"idset":"7751"}}
+{"timestamp":1713660291.150388,"name":"offline","context":{"idset":"7728"}}
+{"timestamp":1713660291.2509453,"name":"offline","context":{"idset":"7725"}}
+{"timestamp":1713660291.2645113,"name":"offline","context":{"idset":"7726"}}
+{"timestamp":1713660291.3639905,"name":"offline","context":{"idset":"7729"}}
+{"timestamp":1713660291.4668999,"name":"offline","context":{"idset":"7732"}}
+{"timestamp":1713660291.6682911,"name":"offline","context":{"idset":"7722"}}
+{"timestamp":1713660291.7679477,"name":"offline","context":{"idset":"7752"}}
+{"timestamp":1713660291.8835626,"name":"offline","context":{"idset":"7753"}}
+{"timestamp":1713660291.9825346,"name":"offline","context":{"idset":"7730"}}
+{"timestamp":1713660292.5432811,"name":"offline","context":{"idset":"7754"}}
+{"timestamp":1713660292.7313228,"name":"offline","context":{"idset":"7727"}}
+{"timestamp":1713660292.9689844,"name":"offline","context":{"idset":"7765"}}
+{"timestamp":1713660293.069133,"name":"offline","context":{"idset":"7766"}}
+{"timestamp":1713660293.2726636,"name":"offline","context":{"idset":"7767"}}
+{"timestamp":1713660293.5017831,"name":"offline","context":{"idset":"7768"}}
+{"timestamp":1713660293.6997285,"name":"offline","context":{"idset":"7731"}}
+{"timestamp":1713660293.8836405,"name":"offline","context":{"idset":"7755"}}
+{"timestamp":1713660294.1821771,"name":"offline","context":{"idset":"7756"}}
+{"timestamp":1713660294.4935648,"name":"offline","context":{"idset":"7758"}}
+{"timestamp":1713660294.5805426,"name":"offline","context":{"idset":"7769"}}
+{"timestamp":1713660294.5872862,"name":"offline","context":{"idset":"7757"}}
+{"timestamp":1713660294.6883287,"name":"offline","context":{"idset":"7770"}}
+{"timestamp":1713660295.3120871,"name":"offline","context":{"idset":"7760"}}
+{"timestamp":1713660295.7378836,"name":"offline","context":{"idset":"7759"}}
+{"timestamp":1713660296.2619922,"name":"offline","context":{"idset":"7762"}}
+{"timestamp":1713660296.3365359,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1713660296.4326103,"name":"offline","context":{"idset":"7772"}}
+{"timestamp":1713660296.731576,"name":"offline","context":{"idset":"7777"}}
+{"timestamp":1713660296.8172932,"name":"offline","context":{"idset":"7774"}}
+{"timestamp":1713660296.9167407,"name":"offline","context":{"idset":"7771"}}
+{"timestamp":1713660297.1474915,"name":"offline","context":{"idset":"7763"}}
+{"timestamp":1713660297.3962269,"name":"offline","context":{"idset":"7781"}}
+{"timestamp":1713660297.4912391,"name":"offline","context":{"idset":"7783"}}
+{"timestamp":1713660297.4988375,"name":"offline","context":{"idset":"7784"}}
+{"timestamp":1713660297.5997689,"name":"offline","context":{"idset":"7782"}}
+{"timestamp":1713660297.7262628,"name":"offline","context":{"idset":"7778"}}
+{"timestamp":1713660297.8590198,"name":"offline","context":{"idset":"7780"}}
+{"timestamp":1713660297.957701,"name":"offline","context":{"idset":"7785"}}
+{"timestamp":1713660298.3379076,"name":"offline","context":{"idset":"7775"}}
+{"timestamp":1713660298.4726536,"name":"offline","context":{"idset":"7786"}}
+{"timestamp":1713660298.5567782,"name":"offline","context":{"idset":"7764"}}
+{"timestamp":1713660298.5609374,"name":"offline","context":{"idset":"7773"}}
+{"timestamp":1713660298.6593957,"name":"offline","context":{"idset":"7788"}}
+{"timestamp":1713660298.8536942,"name":"offline","context":{"idset":"7787"}}
+{"timestamp":1713660299.0497017,"name":"offline","context":{"idset":"7779"}}
+{"timestamp":1713660299.8292005,"name":"offline","context":{"idset":"7776"}}
+{"timestamp":1713660300.2278214,"name":"offline","context":{"idset":"7789"}}
+{"timestamp":1713660300.4022908,"name":"offline","context":{"idset":"7792"}}
+{"timestamp":1713660300.4071805,"name":"offline","context":{"idset":"7791"}}
+{"timestamp":1713660300.5068233,"name":"offline","context":{"idset":"7793"}}
+{"timestamp":1713660301.074074,"name":"offline","context":{"idset":"7790"}}
+{"timestamp":1713660301.2614515,"name":"offline","context":{"idset":"7794"}}
+{"timestamp":1713660301.551698,"name":"offline","context":{"idset":"7796"}}
+{"timestamp":1713660303.1715505,"name":"offline","context":{"idset":"7795"}}
+{"timestamp":1713667690.6661515,"name":"offline","context":{"idset":"9083"}}
+{"timestamp":1713667690.7498283,"name":"offline","context":{"idset":"9084"}}
+{"timestamp":1713667836.7529275,"name":"offline","context":{"idset":"978"}}
+{"timestamp":1713668000.7503049,"name":"offline","context":{"idset":"9118"}}
+{"timestamp":1713671986.7501247,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1713727839.1302004,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1713727840.4469712,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1713727840.7924457,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1713727840.8918917,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1713727841.075331,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1713727841.3328974,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1713727841.6641829,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1713727841.8266962,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1713727841.9125049,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1713727841.9944828,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1713727842.0761039,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1713727842.1595197,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1713727842.2656465,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1713727842.4680479,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713727842.5682361,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1713727843.2897673,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1713727905.6523235,"name":"offline","context":{"idset":"10895"}}
+{"timestamp":1713727906.2223537,"name":"offline","context":{"idset":"10896"}}
+{"timestamp":1713727906.3214073,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1713727906.5013216,"name":"offline","context":{"idset":"10894"}}
+{"timestamp":1713727906.9701893,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1713727907.0547161,"name":"offline","context":{"idset":"10897"}}
+{"timestamp":1713727907.1538818,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1713727907.3310668,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1713727907.7401726,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1713727907.8375962,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1713727908.0324841,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1713727908.038717,"name":"offline","context":{"idset":"10900"}}
+{"timestamp":1713727908.1349511,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1713727908.2347019,"name":"offline","context":{"idset":"10898"}}
+{"timestamp":1713727909.3353598,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1713727909.7269714,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1713727941.7716939,"name":"offline","context":{"idset":"10915"}}
+{"timestamp":1713727942.8306236,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1713727942.9212828,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1713727943.0185671,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1713727943.206984,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1713727943.3080208,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1713727943.9316976,"name":"offline","context":{"idset":"10909"}}
+{"timestamp":1713727944.0275493,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1713727944.0312104,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1713727944.131588,"name":"offline","context":{"idset":"10912"}}
+{"timestamp":1713727944.3151207,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1713727944.4045177,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1713727944.4084642,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1713727944.5075085,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1713727944.6104822,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1713727944.7852583,"name":"offline","context":{"idset":"10916"}}
+{"timestamp":1713730352.8133547,"name":"online","context":{"idset":"10870,10877"}}
+{"timestamp":1713730353.0161135,"name":"online","context":{"idset":"10873,10875-10876"}}
+{"timestamp":1713730353.2581351,"name":"online","context":{"idset":"10869,10872,10880"}}
+{"timestamp":1713730353.4364436,"name":"online","context":{"idset":"10881"}}
+{"timestamp":1713730353.6315799,"name":"online","context":{"idset":"10874,10878,10884"}}
+{"timestamp":1713730353.9207933,"name":"online","context":{"idset":"10882-10883"}}
+{"timestamp":1713730362.6047876,"name":"online","context":{"idset":"10885"}}
+{"timestamp":1713730362.7275269,"name":"online","context":{"idset":"10886"}}
+{"timestamp":1713730364.6019764,"name":"online","context":{"idset":"10887"}}
+{"timestamp":1713730364.8071895,"name":"online","context":{"idset":"10888-10890,10893"}}
+{"timestamp":1713730365.1118758,"name":"online","context":{"idset":"10891-10892,10897"}}
+{"timestamp":1713730365.3216603,"name":"online","context":{"idset":"10894,10898"}}
+{"timestamp":1713730365.5179822,"name":"online","context":{"idset":"10895,10900"}}
+{"timestamp":1713730365.7194328,"name":"online","context":{"idset":"10896,10899"}}
+{"timestamp":1713730374.5628371,"name":"online","context":{"idset":"10902"}}
+{"timestamp":1713730376.7261848,"name":"online","context":{"idset":"10903,10906-10907"}}
+{"timestamp":1713730376.9717231,"name":"online","context":{"idset":"10904,10908"}}
+{"timestamp":1713730377.2328939,"name":"online","context":{"idset":"10905,10909"}}
+{"timestamp":1713730377.8005233,"name":"online","context":{"idset":"10910,10912"}}
+{"timestamp":1713730377.9989424,"name":"online","context":{"idset":"10915-10916"}}
+{"timestamp":1713730378.2166409,"name":"online","context":{"idset":"10911,10913"}}
+{"timestamp":1713737040.7420592,"name":"online","context":{"idset":"9118"}}
+{"timestamp":1713737040.9906998,"name":"online","context":{"idset":"9117"}}
+{"timestamp":1713737606.2251313,"name":"online","context":{"idset":"10914"}}
+{"timestamp":1713737608.9199464,"name":"online","context":{"idset":"10879"}}
+{"timestamp":1713738176.3586812,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713738258.3566682,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1713740576.2775769,"name":"online","context":{"idset":"10871"}}
+{"timestamp":1713740577.1658609,"name":"online","context":{"idset":"10902"}}
+{"timestamp":1713741367.0311785,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1713741788.2713282,"name":"offline","context":{"idset":"10229"}}
+{"timestamp":1713741788.3580177,"name":"offline","context":{"idset":"10230"}}
+{"timestamp":1713741790.2791214,"name":"offline","context":{"idset":"10231"}}
+{"timestamp":1713741790.2830496,"name":"offline","context":{"idset":"10232"}}
+{"timestamp":1713741790.286922,"name":"offline","context":{"idset":"10233"}}
+{"timestamp":1713741790.3594875,"name":"offline","context":{"idset":"10234"}}
+{"timestamp":1713741792.2720745,"name":"offline","context":{"idset":"10237"}}
+{"timestamp":1713741792.3577888,"name":"offline","context":{"idset":"10238"}}
+{"timestamp":1713741794.281533,"name":"offline","context":{"idset":"10239"}}
+{"timestamp":1713741794.2856815,"name":"offline","context":{"idset":"10240"}}
+{"timestamp":1713741794.2898159,"name":"offline","context":{"idset":"10241"}}
+{"timestamp":1713741794.2939696,"name":"offline","context":{"idset":"10242"}}
+{"timestamp":1713741794.3580117,"name":"offline","context":{"idset":"10243"}}
+{"timestamp":1713741796.296792,"name":"offline","context":{"idset":"10244"}}
+{"timestamp":1713741796.30198,"name":"offline","context":{"idset":"10245"}}
+{"timestamp":1713741796.3070853,"name":"offline","context":{"idset":"10246"}}
+{"timestamp":1713741796.3962722,"name":"offline","context":{"idset":"10247"}}
+{"timestamp":1713741798.2804992,"name":"offline","context":{"idset":"10248"}}
+{"timestamp":1713741798.2842536,"name":"offline","context":{"idset":"10249"}}
+{"timestamp":1713741798.2878702,"name":"offline","context":{"idset":"10250"}}
+{"timestamp":1713741798.2915797,"name":"offline","context":{"idset":"10251"}}
+{"timestamp":1713741798.3579662,"name":"offline","context":{"idset":"10252"}}
+{"timestamp":1713741800.2735436,"name":"offline","context":{"idset":"10253"}}
+{"timestamp":1713741800.3586178,"name":"offline","context":{"idset":"10254"}}
+{"timestamp":1713741802.276747,"name":"offline","context":{"idset":"10255"}}
+{"timestamp":1713741802.3624928,"name":"offline","context":{"idset":"10256"}}
+{"timestamp":1713741804.2779939,"name":"offline","context":{"idset":"10257"}}
+{"timestamp":1713741804.281935,"name":"offline","context":{"idset":"10258"}}
+{"timestamp":1713741804.3585663,"name":"offline","context":{"idset":"10259"}}
+{"timestamp":1713741806.2761953,"name":"offline","context":{"idset":"10260"}}
+{"timestamp":1713741806.2822113,"name":"offline","context":{"idset":"10261"}}
+{"timestamp":1713741806.2882729,"name":"offline","context":{"idset":"10262"}}
+{"timestamp":1713741806.3573217,"name":"offline","context":{"idset":"10263"}}
+{"timestamp":1713741808.2740073,"name":"offline","context":{"idset":"10264"}}
+{"timestamp":1713741808.2775044,"name":"offline","context":{"idset":"10265"}}
+{"timestamp":1713741808.2810504,"name":"offline","context":{"idset":"10266"}}
+{"timestamp":1713741808.3568234,"name":"offline","context":{"idset":"10267"}}
+{"timestamp":1713741810.2851622,"name":"offline","context":{"idset":"10268"}}
+{"timestamp":1713741810.2885542,"name":"offline","context":{"idset":"10269"}}
+{"timestamp":1713741810.2921743,"name":"offline","context":{"idset":"10270"}}
+{"timestamp":1713741810.3573992,"name":"offline","context":{"idset":"10271"}}
+{"timestamp":1713741812.2752852,"name":"offline","context":{"idset":"10272"}}
+{"timestamp":1713741812.2828178,"name":"offline","context":{"idset":"10273"}}
+{"timestamp":1713741812.3560288,"name":"offline","context":{"idset":"10274"}}
+{"timestamp":1713741814.2767451,"name":"offline","context":{"idset":"10275"}}
+{"timestamp":1713741814.2830732,"name":"offline","context":{"idset":"10277"}}
+{"timestamp":1713741814.2903166,"name":"offline","context":{"idset":"10278"}}
+{"timestamp":1713741814.3594472,"name":"offline","context":{"idset":"10279"}}
+{"timestamp":1713741816.2791562,"name":"offline","context":{"idset":"10280"}}
+{"timestamp":1713741816.2833934,"name":"offline","context":{"idset":"10281"}}
+{"timestamp":1713741816.358629,"name":"offline","context":{"idset":"10282"}}
+{"timestamp":1713741818.2733388,"name":"offline","context":{"idset":"10283"}}
+{"timestamp":1713741818.2779899,"name":"offline","context":{"idset":"10284"}}
+{"timestamp":1713741818.3590195,"name":"offline","context":{"idset":"10287"}}
+{"timestamp":1713741820.2783909,"name":"offline","context":{"idset":"10288"}}
+{"timestamp":1713741820.2822928,"name":"offline","context":{"idset":"10289"}}
+{"timestamp":1713741820.2859328,"name":"offline","context":{"idset":"10290"}}
+{"timestamp":1713741820.2892704,"name":"offline","context":{"idset":"10291"}}
+{"timestamp":1713741820.3583033,"name":"offline","context":{"idset":"10292"}}
+{"timestamp":1713741822.2825296,"name":"offline","context":{"idset":"10293"}}
+{"timestamp":1713741822.2858109,"name":"offline","context":{"idset":"10294"}}
+{"timestamp":1713741822.2892261,"name":"offline","context":{"idset":"10295"}}
+{"timestamp":1713741822.3569922,"name":"offline","context":{"idset":"10296"}}
+{"timestamp":1713741824.2806852,"name":"offline","context":{"idset":"10297"}}
+{"timestamp":1713741824.2845616,"name":"offline","context":{"idset":"10298"}}
+{"timestamp":1713741824.2885487,"name":"offline","context":{"idset":"10299"}}
+{"timestamp":1713741824.3582847,"name":"offline","context":{"idset":"10300"}}
+{"timestamp":1713741826.2790689,"name":"offline","context":{"idset":"10302"}}
+{"timestamp":1713741826.2822883,"name":"offline","context":{"idset":"10303"}}
+{"timestamp":1713741826.3593955,"name":"offline","context":{"idset":"10304"}}
+{"timestamp":1713741828.2720377,"name":"offline","context":{"idset":"10305"}}
+{"timestamp":1713741828.2751741,"name":"offline","context":{"idset":"10307"}}
+{"timestamp":1713741828.357754,"name":"offline","context":{"idset":"10308"}}
+{"timestamp":1713741830.2706563,"name":"offline","context":{"idset":"10309"}}
+{"timestamp":1713741830.2737966,"name":"offline","context":{"idset":"10310"}}
+{"timestamp":1713741830.2770288,"name":"offline","context":{"idset":"10312"}}
+{"timestamp":1713741830.3599622,"name":"offline","context":{"idset":"10313"}}
+{"timestamp":1713741832.2799857,"name":"offline","context":{"idset":"10314"}}
+{"timestamp":1713741832.283309,"name":"offline","context":{"idset":"10315"}}
+{"timestamp":1713741832.2864199,"name":"offline","context":{"idset":"10316"}}
+{"timestamp":1713741832.2894864,"name":"offline","context":{"idset":"10317"}}
+{"timestamp":1713741832.3579276,"name":"offline","context":{"idset":"10318"}}
+{"timestamp":1713741834.2881634,"name":"offline","context":{"idset":"10285"}}
+{"timestamp":1713741834.2919316,"name":"offline","context":{"idset":"10319"}}
+{"timestamp":1713741834.2950139,"name":"offline","context":{"idset":"10320"}}
+{"timestamp":1713741834.298162,"name":"offline","context":{"idset":"10321"}}
+{"timestamp":1713741834.3583639,"name":"offline","context":{"idset":"10322"}}
+{"timestamp":1713741836.2747545,"name":"offline","context":{"idset":"10323"}}
+{"timestamp":1713741836.2787967,"name":"offline","context":{"idset":"10324"}}
+{"timestamp":1713741836.3586881,"name":"offline","context":{"idset":"10326"}}
+{"timestamp":1713741838.2857769,"name":"offline","context":{"idset":"10328"}}
+{"timestamp":1713741838.3711872,"name":"offline","context":{"idset":"10330"}}
+{"timestamp":1713741840.2694061,"name":"offline","context":{"idset":"10301"}}
+{"timestamp":1713741840.2717094,"name":"offline","context":{"idset":"10332"}}
+{"timestamp":1713741840.2738709,"name":"offline","context":{"idset":"10333"}}
+{"timestamp":1713741840.3567507,"name":"offline","context":{"idset":"10334"}}
+{"timestamp":1713741842.2723982,"name":"offline","context":{"idset":"10335"}}
+{"timestamp":1713741842.3609288,"name":"offline","context":{"idset":"10336"}}
+{"timestamp":1713741844.2826385,"name":"offline","context":{"idset":"10306"}}
+{"timestamp":1713741844.2859488,"name":"offline","context":{"idset":"10337"}}
+{"timestamp":1713741844.2895215,"name":"offline","context":{"idset":"10338"}}
+{"timestamp":1713741844.2959843,"name":"offline","context":{"idset":"10339"}}
+{"timestamp":1713741844.3001418,"name":"offline","context":{"idset":"10340"}}
+{"timestamp":1713741844.3825223,"name":"offline","context":{"idset":"10341"}}
+{"timestamp":1713741846.2741973,"name":"offline","context":{"idset":"10342"}}
+{"timestamp":1713741846.2780328,"name":"offline","context":{"idset":"10343"}}
+{"timestamp":1713741846.2817779,"name":"offline","context":{"idset":"10344"}}
+{"timestamp":1713741846.2861543,"name":"offline","context":{"idset":"10345"}}
+{"timestamp":1713741846.3572464,"name":"offline","context":{"idset":"10346"}}
+{"timestamp":1713741848.2725704,"name":"offline","context":{"idset":"10347"}}
+{"timestamp":1713741848.2759993,"name":"offline","context":{"idset":"10348"}}
+{"timestamp":1713741848.2801096,"name":"offline","context":{"idset":"10349"}}
+{"timestamp":1713741848.3596032,"name":"offline","context":{"idset":"10350"}}
+{"timestamp":1713741850.2786467,"name":"offline","context":{"idset":"10351"}}
+{"timestamp":1713741850.2822444,"name":"offline","context":{"idset":"10352"}}
+{"timestamp":1713741850.2855885,"name":"offline","context":{"idset":"10353"}}
+{"timestamp":1713741850.2891164,"name":"offline","context":{"idset":"10354"}}
+{"timestamp":1713741850.3575709,"name":"offline","context":{"idset":"10355"}}
+{"timestamp":1713741852.3601542,"name":"offline","context":{"idset":"10356"}}
+{"timestamp":1713741856.3597918,"name":"offline","context":{"idset":"10276"}}
+{"timestamp":1713741864.3566155,"name":"offline","context":{"idset":"10286"}}
+{"timestamp":1713742790.8363504,"name":"online","context":{"idset":"10901"}}
+{"timestamp":1713742854.3613675,"name":"offline","context":{"idset":"10311"}}
+{"timestamp":1713746450.2607749,"name":"drain","context":{"idset":"10233","reason":"broker was unresponsive"}}
+{"timestamp":1713746450.3606398,"name":"drain","context":{"idset":"10248","reason":"broker was unresponsive"}}
+{"timestamp":1713746451.116972,"name":"online","context":{"idset":"10235"}}
+{"timestamp":1713746451.2801464,"name":"online","context":{"idset":"10230,10254"}}
+{"timestamp":1713746451.7262318,"name":"online","context":{"idset":"10232,10242"}}
+{"timestamp":1713746451.888607,"name":"online","context":{"idset":"10234"}}
+{"timestamp":1713746451.9732344,"name":"online","context":{"idset":"10231"}}
+{"timestamp":1713746451.9870508,"name":"online","context":{"idset":"10279"}}
+{"timestamp":1713746452.0916259,"name":"online","context":{"idset":"10237"}}
+{"timestamp":1713746452.1799078,"name":"online","context":{"idset":"10238-10239"}}
+{"timestamp":1713746452.2679296,"name":"online","context":{"idset":"10241"}}
+{"timestamp":1713746452.3677752,"name":"online","context":{"idset":"10243"}}
+{"timestamp":1713746452.3733773,"name":"online","context":{"idset":"10229"}}
+{"timestamp":1713746452.4804771,"name":"online","context":{"idset":"10261"}}
+{"timestamp":1713746452.6970422,"name":"online","context":{"idset":"10233,10246,10248,10293"}}
+{"timestamp":1713746452.7910373,"name":"online","context":{"idset":"10236,10266,10272-10273"}}
+{"timestamp":1713746452.8821108,"name":"online","context":{"idset":"10262"}}
+{"timestamp":1713746452.9741983,"name":"online","context":{"idset":"10275"}}
+{"timestamp":1713746453.2072308,"name":"online","context":{"idset":"10252"}}
+{"timestamp":1713746453.3006461,"name":"online","context":{"idset":"10253"}}
+{"timestamp":1713746453.3925555,"name":"online","context":{"idset":"10255,10267,10274"}}
+{"timestamp":1713746453.4796131,"name":"online","context":{"idset":"10250,10263"}}
+{"timestamp":1713746453.6114914,"name":"online","context":{"idset":"10287"}}
+{"timestamp":1713746453.7057588,"name":"online","context":{"idset":"10245,10257"}}
+{"timestamp":1713746453.8859477,"name":"online","context":{"idset":"10247,10251,10270,10280,10317"}}
+{"timestamp":1713746454.0800536,"name":"online","context":{"idset":"10271,10291,10328"}}
+{"timestamp":1713746454.1875374,"name":"online","context":{"idset":"10258,10314,10349"}}
+{"timestamp":1713746454.3345881,"name":"online","context":{"idset":"10269,10337"}}
+{"timestamp":1713746454.4191284,"name":"online","context":{"idset":"10264,10301,10339"}}
+{"timestamp":1713746454.5069549,"name":"online","context":{"idset":"10325"}}
+{"timestamp":1713746454.7201464,"name":"online","context":{"idset":"10259,10268,10294,10322,10338,10353"}}
+{"timestamp":1713746454.8244078,"name":"online","context":{"idset":"10327"}}
+{"timestamp":1713746455.0076237,"name":"online","context":{"idset":"10354"}}
+{"timestamp":1713746455.1186302,"name":"online","context":{"idset":"10277,10298,10300,10321,10342,10345"}}
+{"timestamp":1713746455.3072786,"name":"online","context":{"idset":"10265,10278,10282-10283,10302,10347"}}
+{"timestamp":1713746455.5028756,"name":"online","context":{"idset":"10256,10289,10303,10326,10330"}}
+{"timestamp":1713746455.6069155,"name":"online","context":{"idset":"10285-10286,10290,10333"}}
+{"timestamp":1713746455.8002582,"name":"online","context":{"idset":"10295-10297,10299,10304,10306,10308-10309,10312,10318,10323-10324,10331,10334,10343,10351"}}
+{"timestamp":1713746455.9853888,"name":"online","context":{"idset":"10281,10305,10307,10311,10313,10319,10341,10350,10355"}}
+{"timestamp":1713746456.0886025,"name":"online","context":{"idset":"10335,10346"}}
+{"timestamp":1713746456.5738332,"name":"online","context":{"idset":"10329"}}
+{"timestamp":1713746456.7550073,"name":"online","context":{"idset":"10310,10315"}}
+{"timestamp":1713763818.3548493,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1713771626.268322,"name":"offline","context":{"idset":"10485"}}
+{"timestamp":1713771626.3574588,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1713772520.2691307,"name":"offline","context":{"idset":"10487"}}
+{"timestamp":1713772520.3577473,"name":"offline","context":{"idset":"10488"}}
+{"timestamp":1713772696.2699053,"name":"offline","context":{"idset":"10489"}}
+{"timestamp":1713772696.3569045,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1713772888.3569474,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1713773202.3567295,"name":"offline","context":{"idset":"10493"}}
+{"timestamp":1713773204.3574867,"name":"offline","context":{"idset":"10494"}}
+{"timestamp":1713773644.3569469,"name":"offline","context":{"idset":"10495"}}
+{"timestamp":1713773646.3578563,"name":"offline","context":{"idset":"10496"}}
+{"timestamp":1713773814.3578587,"name":"offline","context":{"idset":"10497"}}
+{"timestamp":1713773816.3588398,"name":"offline","context":{"idset":"10498"}}
+{"timestamp":1713774016.3546894,"name":"offline","context":{"idset":"10499"}}
+{"timestamp":1713774018.3555722,"name":"offline","context":{"idset":"10500"}}
+{"timestamp":1713774406.3571165,"name":"offline","context":{"idset":"10501"}}
+{"timestamp":1713774408.3570039,"name":"offline","context":{"idset":"10502"}}
+{"timestamp":1713774652.2685757,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1713774652.3566484,"name":"offline","context":{"idset":"10504"}}
+{"timestamp":1713775076.2710795,"name":"offline","context":{"idset":"10505"}}
+{"timestamp":1713775076.3580575,"name":"offline","context":{"idset":"10506"}}
+{"timestamp":1713775738.3568997,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1713775740.3571625,"name":"offline","context":{"idset":"10508"}}
+{"timestamp":1713775976.3573611,"name":"offline","context":{"idset":"10509"}}
+{"timestamp":1713775978.3581076,"name":"offline","context":{"idset":"10510"}}
+{"timestamp":1713776246.3562779,"name":"offline","context":{"idset":"10511"}}
+{"timestamp":1713776248.3577306,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1713776626.267504,"name":"offline","context":{"idset":"10513"}}
+{"timestamp":1713776626.3609123,"name":"offline","context":{"idset":"10514"}}
+{"timestamp":1713776824.3564887,"name":"offline","context":{"idset":"10515"}}
+{"timestamp":1713776826.3570275,"name":"offline","context":{"idset":"10516"}}
+{"timestamp":1713777506.3570328,"name":"offline","context":{"idset":"10518"}}
+{"timestamp":1713777508.3566353,"name":"offline","context":{"idset":"10517"}}
+{"timestamp":1713777684.2679067,"name":"offline","context":{"idset":"10519"}}
+{"timestamp":1713777684.3574634,"name":"offline","context":{"idset":"10520"}}
+{"timestamp":1713794410.1037405,"name":"drain","context":{"idset":"421-460","reason":"New Blade Installation --JRG","overwrite":1}}
+{"timestamp":1713794492.1167364,"name":"offline","context":{"idset":"423"}}
+{"timestamp":1713794492.1203439,"name":"offline","context":{"idset":"424"}}
+{"timestamp":1713794492.1326959,"name":"offline","context":{"idset":"431"}}
+{"timestamp":1713794492.2278636,"name":"offline","context":{"idset":"426"}}
+{"timestamp":1713794492.2323987,"name":"offline","context":{"idset":"421"}}
+{"timestamp":1713794492.2362154,"name":"offline","context":{"idset":"422"}}
+{"timestamp":1713794492.2424481,"name":"offline","context":{"idset":"425"}}
+{"timestamp":1713794492.245961,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1713794492.2521117,"name":"offline","context":{"idset":"436"}}
+{"timestamp":1713794492.2580299,"name":"offline","context":{"idset":"427"}}
+{"timestamp":1713794492.2777474,"name":"offline","context":{"idset":"428"}}
+{"timestamp":1713794492.2891934,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1713794492.292495,"name":"offline","context":{"idset":"430"}}
+{"timestamp":1713794492.2960572,"name":"offline","context":{"idset":"432"}}
+{"timestamp":1713794492.2993789,"name":"offline","context":{"idset":"433"}}
+{"timestamp":1713794492.3026528,"name":"offline","context":{"idset":"434"}}
+{"timestamp":1713794492.3059461,"name":"offline","context":{"idset":"435"}}
+{"timestamp":1713794492.3182228,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1713794492.3216045,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1713794492.3248823,"name":"offline","context":{"idset":"429"}}
+{"timestamp":1713794492.4745862,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1713794492.4778175,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1713794492.5610204,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1713794797.2329798,"name":"drain","context":{"idset":"461-476","reason":"New Blade Installation --JRG","overwrite":1}}
+{"timestamp":1713794815.8261898,"name":"offline","context":{"idset":"471"}}
+{"timestamp":1713794815.8293829,"name":"offline","context":{"idset":"473"}}
+{"timestamp":1713794815.9258316,"name":"offline","context":{"idset":"469"}}
+{"timestamp":1713794816.0118475,"name":"offline","context":{"idset":"474"}}
+{"timestamp":1713794816.017822,"name":"offline","context":{"idset":"476"}}
+{"timestamp":1713794816.02105,"name":"offline","context":{"idset":"470"}}
+{"timestamp":1713794816.0244036,"name":"offline","context":{"idset":"472"}}
+{"timestamp":1713794816.1178107,"name":"offline","context":{"idset":"475"}}
+{"timestamp":1713795612.861063,"name":"drain","context":{"idset":"77-84","reason":"New blade installation","overwrite":0}}
+{"timestamp":1713796603.4477985,"name":"offline","context":{"idset":"10521"}}
+{"timestamp":1713796603.5319877,"name":"offline","context":{"idset":"10522"}}
+{"timestamp":1713796821.7190468,"name":"online","context":{"idset":"10130"}}
+{"timestamp":1713797021.5347903,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1713797023.5331824,"name":"offline","context":{"idset":"10523"}}
+{"timestamp":1713797094.9849277,"name":"online","context":{"idset":"10134"}}
+{"timestamp":1713797310.9056966,"name":"online","context":{"idset":"10196,10199,10203"}}
+{"timestamp":1713797491.1096981,"name":"online","context":{"idset":"10835"}}
+{"timestamp":1713797491.3082895,"name":"online","context":{"idset":"11777,11815"}}
+{"timestamp":1713797491.7069533,"name":"online","context":{"idset":"10220,10822,11833,11864"}}
+{"timestamp":1713797491.8229141,"name":"online","context":{"idset":"10751,10862,11785"}}
+{"timestamp":1713797495.4500387,"name":"offline","context":{"idset":"8181"}}
+{"timestamp":1713797495.5322163,"name":"offline","context":{"idset":"8182"}}
+{"timestamp":1713800937.5324368,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1713800976.9578936,"name":"online","context":{"idset":"7682"}}
+{"timestamp":1713800977.2397916,"name":"online","context":{"idset":"7669,7681"}}
+{"timestamp":1713800977.3476923,"name":"online","context":{"idset":"7679"}}
+{"timestamp":1713800977.4558644,"name":"online","context":{"idset":"7671,7676,7678"}}
+{"timestamp":1713800977.6462803,"name":"online","context":{"idset":"7670,7673-7675"}}
+{"timestamp":1713800977.7478874,"name":"online","context":{"idset":"7680,7683-7684"}}
+{"timestamp":1713800977.9467287,"name":"online","context":{"idset":"7672"}}
+{"timestamp":1713800978.1436238,"name":"online","context":{"idset":"7677"}}
+{"timestamp":1713800988.8971541,"name":"online","context":{"idset":"7685,7687"}}
+{"timestamp":1713800989.2034159,"name":"online","context":{"idset":"7692"}}
+{"timestamp":1713800989.4311814,"name":"online","context":{"idset":"7686,7688"}}
+{"timestamp":1713800989.6362519,"name":"online","context":{"idset":"7689,7691,7695,7697-7698,7700"}}
+{"timestamp":1713800989.8437216,"name":"online","context":{"idset":"7690,7694"}}
+{"timestamp":1713800989.9444003,"name":"online","context":{"idset":"7699"}}
+{"timestamp":1713801001.0888517,"name":"online","context":{"idset":"7716-7718"}}
+{"timestamp":1713801001.5931175,"name":"online","context":{"idset":"7720-7721"}}
+{"timestamp":1713801001.7835305,"name":"online","context":{"idset":"7723"}}
+{"timestamp":1713801001.8966625,"name":"online","context":{"idset":"7724-7726"}}
+{"timestamp":1713801002.1424301,"name":"online","context":{"idset":"7722"}}
+{"timestamp":1713801002.6386089,"name":"online","context":{"idset":"7727-7728"}}
+{"timestamp":1713801012.8626873,"name":"online","context":{"idset":"7729,7731"}}
+{"timestamp":1713801013.0877347,"name":"online","context":{"idset":"7730,7732"}}
+{"timestamp":1713801013.3797746,"name":"online","context":{"idset":"7749,7751-7752"}}
+{"timestamp":1713801013.9058146,"name":"online","context":{"idset":"7750,7753,7755"}}
+{"timestamp":1713801014.1348147,"name":"online","context":{"idset":"7756"}}
+{"timestamp":1713801024.3853431,"name":"online","context":{"idset":"7757"}}
+{"timestamp":1713801024.7751362,"name":"online","context":{"idset":"7759"}}
+{"timestamp":1713801025.0707753,"name":"online","context":{"idset":"7758,7760"}}
+{"timestamp":1713801025.2978578,"name":"online","context":{"idset":"7766"}}
+{"timestamp":1713801025.3991048,"name":"online","context":{"idset":"7764"}}
+{"timestamp":1713801025.5899339,"name":"online","context":{"idset":"7762,7765"}}
+{"timestamp":1713801026.0714335,"name":"online","context":{"idset":"7767-7768"}}
+{"timestamp":1713801026.2481303,"name":"online","context":{"idset":"7763"}}
+{"timestamp":1713801029.5581136,"name":"drain","context":{"idset":"7693","reason":"broker was unresponsive"}}
+{"timestamp":1713801036.438633,"name":"online","context":{"idset":"7769"}}
+{"timestamp":1713801036.9434631,"name":"online","context":{"idset":"7770"}}
+{"timestamp":1713801037.1575089,"name":"online","context":{"idset":"7772"}}
+{"timestamp":1713801037.2722275,"name":"online","context":{"idset":"7771,7773-7774,7777"}}
+{"timestamp":1713801037.4660635,"name":"online","context":{"idset":"7776"}}
+{"timestamp":1713801037.9697452,"name":"online","context":{"idset":"7778-7779"}}
+{"timestamp":1713801038.1939309,"name":"online","context":{"idset":"7775"}}
+{"timestamp":1713801045.2582104,"name":"online","context":{"idset":"7780"}}
+{"timestamp":1713801048.8086944,"name":"online","context":{"idset":"7782"}}
+{"timestamp":1713801049.0029588,"name":"online","context":{"idset":"7783,7787"}}
+{"timestamp":1713801049.2006505,"name":"online","context":{"idset":"7784"}}
+{"timestamp":1713801049.5948505,"name":"online","context":{"idset":"7789"}}
+{"timestamp":1713801049.8944924,"name":"online","context":{"idset":"7788,7790"}}
+{"timestamp":1713801050.0854149,"name":"online","context":{"idset":"7696,7786,7791"}}
+{"timestamp":1713801050.4042439,"name":"online","context":{"idset":"7785"}}
+{"timestamp":1713801053.5359766,"name":"drain","context":{"idset":"7754","reason":"broker was unresponsive"}}
+{"timestamp":1713801056.9815669,"name":"online","context":{"idset":"7792"}}
+{"timestamp":1713801060.7883334,"name":"online","context":{"idset":"7793-7794"}}
+{"timestamp":1713801060.9818513,"name":"online","context":{"idset":"7795-7796"}}
+{"timestamp":1713801080.2345369,"name":"online","context":{"idset":"7754"}}
+{"timestamp":1713801085.5375087,"name":"drain","context":{"idset":"7715","reason":"broker was unresponsive"}}
+{"timestamp":1713801091.5379665,"name":"drain","context":{"idset":"7719","reason":"broker was unresponsive"}}
+{"timestamp":1713801117.9909158,"name":"online","context":{"idset":"7693"}}
+{"timestamp":1713801129.5375788,"name":"drain","context":{"idset":"7761","reason":"broker was unresponsive"}}
+{"timestamp":1713801149.5373271,"name":"drain","context":{"idset":"7781","reason":"broker was unresponsive"}}
+{"timestamp":1713801178.5001318,"name":"online","context":{"idset":"7715"}}
+{"timestamp":1713801186.1323519,"name":"online","context":{"idset":"7719"}}
+{"timestamp":1713801207.4683917,"name":"online","context":{"idset":"7761"}}
+{"timestamp":1713801219.596,"name":"online","context":{"idset":"7781"}}
+{"timestamp":1713801374.5603013,"name":"undrain","context":{"idset":"7693,7715,7719,7754,7761,7781"}}
+{"timestamp":1713802367.533736,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1713802379.532717,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1713802381.5334291,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1713802805.0172668,"name":"drain","context":{"idset":"773-774,11203-11204","overwrite":0}}
+{"timestamp":1713802808.7689307,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1713802814.3270783,"name":"offline","context":{"idset":"773"}}
+{"timestamp":1713802818.8375137,"name":"offline","context":{"idset":"774"}}
+{"timestamp":1713803565.3719475,"name":"offline","context":{"idset":"772"}}
+{"timestamp":1713803581.6441727,"name":"offline","context":{"idset":"771"}}
+{"timestamp":1713803764.2434633,"name":"online","context":{"idset":"10742"}}
+{"timestamp":1713804185.5332654,"name":"offline","context":{"idset":"10307"}}
+{"timestamp":1713804615.5337794,"name":"offline","context":{"idset":"764"}}
+{"timestamp":1713804966.5525055,"name":"online","context":{"idset":"9399"}}
+{"timestamp":1713805480.570219,"name":"online","context":{"idset":"8440"}}
+{"timestamp":1713805481.7557247,"name":"online","context":{"idset":"8443"}}
+{"timestamp":1713805481.9891319,"name":"online","context":{"idset":"8451,8453"}}
+{"timestamp":1713805482.52668,"name":"online","context":{"idset":"8461"}}
+{"timestamp":1713805483.0514588,"name":"online","context":{"idset":"8445"}}
+{"timestamp":1713805483.1523523,"name":"online","context":{"idset":"8437"}}
+{"timestamp":1713805483.5238626,"name":"online","context":{"idset":"8448"}}
+{"timestamp":1713805483.6218398,"name":"online","context":{"idset":"8438,8455"}}
+{"timestamp":1713805484.0173149,"name":"online","context":{"idset":"8465"}}
+{"timestamp":1713805484.0240848,"name":"online","context":{"idset":"8477"}}
+{"timestamp":1713805484.478595,"name":"online","context":{"idset":"8447,8460"}}
+{"timestamp":1713805484.5799785,"name":"online","context":{"idset":"8446,8466"}}
+{"timestamp":1713805484.6739848,"name":"online","context":{"idset":"8459,8464"}}
+{"timestamp":1713805484.7789927,"name":"online","context":{"idset":"8454"}}
+{"timestamp":1713805484.9709458,"name":"online","context":{"idset":"8441"}}
+{"timestamp":1713805485.0857258,"name":"online","context":{"idset":"8449"}}
+{"timestamp":1713805485.2091045,"name":"online","context":{"idset":"8442,8450,8471,8490"}}
+{"timestamp":1713805485.3179076,"name":"online","context":{"idset":"8439"}}
+{"timestamp":1713805485.418716,"name":"online","context":{"idset":"8478"}}
+{"timestamp":1713805485.626498,"name":"online","context":{"idset":"8444,8487,8502,8506"}}
+{"timestamp":1713805485.7280288,"name":"online","context":{"idset":"8462,8503-8504"}}
+{"timestamp":1713805485.9317803,"name":"online","context":{"idset":"8457-8458,8467-8468,8472,8474-8475,8491-8492,8496,8510,8522,8525,8535,8545,8556"}}
+{"timestamp":1713805486.0363388,"name":"online","context":{"idset":"8456,8473,8482-8484,8486,8544"}}
+{"timestamp":1713805486.2235775,"name":"online","context":{"idset":"8463,8469,8497,8501,8509,8511-8512,8514,8516-8517,8524,8526,8541,8554,8558,8563"}}
+{"timestamp":1713805486.3286448,"name":"online","context":{"idset":"8499,8507,8550"}}
+{"timestamp":1713805486.4353237,"name":"online","context":{"idset":"8476,8488-8489,8508,8527,8531,8533,8539,8552,8557"}}
+{"timestamp":1713805486.5535855,"name":"online","context":{"idset":"8493-8494,8500,8505,8521,8523,8530,8540,8546,8560"}}
+{"timestamp":1713805486.7408557,"name":"online","context":{"idset":"8470,8480-8481,8485,8515,8532,8548-8549,8553"}}
+{"timestamp":1713805486.9338379,"name":"online","context":{"idset":"8498,8519-8520,8536,8555"}}
+{"timestamp":1713805487.1243203,"name":"online","context":{"idset":"8534,8538,8559"}}
+{"timestamp":1713805487.2317815,"name":"online","context":{"idset":"8479,8513,8518,8529,8537,8542,8564"}}
+{"timestamp":1713805487.3370647,"name":"online","context":{"idset":"8528,8543,8547,8551"}}
+{"timestamp":1713805487.5304215,"name":"online","context":{"idset":"8561-8562"}}
+{"timestamp":1713805487.7253304,"name":"online","context":{"idset":"8452,8495"}}
+{"timestamp":1713805594.4572792,"name":"online","context":{"idset":"9359"}}
+{"timestamp":1713805608.199333,"name":"online","context":{"idset":"7944"}}
+{"timestamp":1713806347.1282685,"name":"online","context":{"idset":"10155"}}
+{"timestamp":1713806481.1396823,"name":"online","context":{"idset":"10156"}}
+{"timestamp":1713808051.5325186,"name":"offline","context":{"idset":"818"}}
+{"timestamp":1713808489.6690357,"name":"offline","context":{"idset":"8437"}}
+{"timestamp":1713808489.6735048,"name":"offline","context":{"idset":"8438"}}
+{"timestamp":1713808489.6759362,"name":"offline","context":{"idset":"8439"}}
+{"timestamp":1713808489.6782329,"name":"offline","context":{"idset":"8440"}}
+{"timestamp":1713808489.6806893,"name":"offline","context":{"idset":"8441"}}
+{"timestamp":1713808489.6829624,"name":"offline","context":{"idset":"8442"}}
+{"timestamp":1713808489.6852508,"name":"offline","context":{"idset":"8443"}}
+{"timestamp":1713808489.6877155,"name":"offline","context":{"idset":"8444"}}
+{"timestamp":1713808489.6905296,"name":"offline","context":{"idset":"8445"}}
+{"timestamp":1713808489.694711,"name":"offline","context":{"idset":"8446"}}
+{"timestamp":1713808489.6991007,"name":"offline","context":{"idset":"8448"}}
+{"timestamp":1713808489.7030015,"name":"offline","context":{"idset":"8449"}}
+{"timestamp":1713808489.7054188,"name":"offline","context":{"idset":"8450"}}
+{"timestamp":1713808489.707854,"name":"offline","context":{"idset":"8451"}}
+{"timestamp":1713808489.710144,"name":"offline","context":{"idset":"8452"}}
+{"timestamp":1713808489.712491,"name":"offline","context":{"idset":"8453"}}
+{"timestamp":1713808489.7148113,"name":"offline","context":{"idset":"8454"}}
+{"timestamp":1713808489.7169776,"name":"offline","context":{"idset":"8455"}}
+{"timestamp":1713808489.7191896,"name":"offline","context":{"idset":"8456"}}
+{"timestamp":1713808489.7214744,"name":"offline","context":{"idset":"8457"}}
+{"timestamp":1713808489.7238171,"name":"offline","context":{"idset":"8460"}}
+{"timestamp":1713808489.7264066,"name":"offline","context":{"idset":"8462"}}
+{"timestamp":1713808489.7287793,"name":"offline","context":{"idset":"8463"}}
+{"timestamp":1713808489.7319093,"name":"offline","context":{"idset":"8464"}}
+{"timestamp":1713808489.7361829,"name":"offline","context":{"idset":"8466"}}
+{"timestamp":1713808489.7403781,"name":"offline","context":{"idset":"8467"}}
+{"timestamp":1713808489.744523,"name":"offline","context":{"idset":"8468"}}
+{"timestamp":1713808489.7487285,"name":"offline","context":{"idset":"8469"}}
+{"timestamp":1713808489.7528858,"name":"offline","context":{"idset":"8470"}}
+{"timestamp":1713808489.757055,"name":"offline","context":{"idset":"8471"}}
+{"timestamp":1713808489.7613382,"name":"offline","context":{"idset":"8472"}}
+{"timestamp":1713808489.7655187,"name":"offline","context":{"idset":"8473"}}
+{"timestamp":1713808489.7697802,"name":"offline","context":{"idset":"8474"}}
+{"timestamp":1713808489.7739844,"name":"offline","context":{"idset":"8476"}}
+{"timestamp":1713808489.7783287,"name":"offline","context":{"idset":"8478"}}
+{"timestamp":1713808489.7824962,"name":"offline","context":{"idset":"8484"}}
+{"timestamp":1713808489.7866237,"name":"offline","context":{"idset":"8487"}}
+{"timestamp":1713808489.8021836,"name":"offline","context":{"idset":"8489"}}
+{"timestamp":1713808489.8044357,"name":"offline","context":{"idset":"8493"}}
+{"timestamp":1713808489.8065777,"name":"offline","context":{"idset":"8495"}}
+{"timestamp":1713808489.8090189,"name":"offline","context":{"idset":"8496"}}
+{"timestamp":1713808489.8111699,"name":"offline","context":{"idset":"8497"}}
+{"timestamp":1713808489.813293,"name":"offline","context":{"idset":"8498"}}
+{"timestamp":1713808489.8154614,"name":"offline","context":{"idset":"8499"}}
+{"timestamp":1713808489.8175879,"name":"offline","context":{"idset":"8500"}}
+{"timestamp":1713808489.8197813,"name":"offline","context":{"idset":"8501"}}
+{"timestamp":1713808489.8219364,"name":"offline","context":{"idset":"8502"}}
+{"timestamp":1713808489.8240793,"name":"offline","context":{"idset":"8503"}}
+{"timestamp":1713808489.8262126,"name":"offline","context":{"idset":"8504"}}
+{"timestamp":1713808489.8285096,"name":"offline","context":{"idset":"8505"}}
+{"timestamp":1713808489.8323948,"name":"offline","context":{"idset":"8506"}}
+{"timestamp":1713808489.858603,"name":"offline","context":{"idset":"8507"}}
+{"timestamp":1713808489.8840654,"name":"offline","context":{"idset":"8508"}}
+{"timestamp":1713808489.8920622,"name":"offline","context":{"idset":"8509"}}
+{"timestamp":1713808489.894206,"name":"offline","context":{"idset":"8511"}}
+{"timestamp":1713808489.8963752,"name":"offline","context":{"idset":"8513"}}
+{"timestamp":1713808489.8985851,"name":"offline","context":{"idset":"8514"}}
+{"timestamp":1713808489.9007571,"name":"offline","context":{"idset":"8515"}}
+{"timestamp":1713808489.9029593,"name":"offline","context":{"idset":"8517"}}
+{"timestamp":1713808489.9051127,"name":"offline","context":{"idset":"8518"}}
+{"timestamp":1713808489.9073856,"name":"offline","context":{"idset":"8519"}}
+{"timestamp":1713808489.9099555,"name":"offline","context":{"idset":"8520"}}
+{"timestamp":1713808489.9124122,"name":"offline","context":{"idset":"8521"}}
+{"timestamp":1713808489.9145942,"name":"offline","context":{"idset":"8525"}}
+{"timestamp":1713808489.9169595,"name":"offline","context":{"idset":"8526"}}
+{"timestamp":1713808489.9191711,"name":"offline","context":{"idset":"8527"}}
+{"timestamp":1713808489.9224279,"name":"offline","context":{"idset":"8528"}}
+{"timestamp":1713808489.944226,"name":"offline","context":{"idset":"8530"}}
+{"timestamp":1713808489.9650884,"name":"offline","context":{"idset":"8531"}}
+{"timestamp":1713808489.9869175,"name":"offline","context":{"idset":"8532"}}
+{"timestamp":1713808489.9906774,"name":"offline","context":{"idset":"8533"}}
+{"timestamp":1713808489.994328,"name":"offline","context":{"idset":"8538"}}
+{"timestamp":1713808489.9969854,"name":"offline","context":{"idset":"8540"}}
+{"timestamp":1713808489.9995763,"name":"offline","context":{"idset":"8541"}}
+{"timestamp":1713808490.0019369,"name":"offline","context":{"idset":"8542"}}
+{"timestamp":1713808490.0046518,"name":"offline","context":{"idset":"8544"}}
+{"timestamp":1713808490.0092592,"name":"offline","context":{"idset":"8545"}}
+{"timestamp":1713808490.0135672,"name":"offline","context":{"idset":"8547"}}
+{"timestamp":1713808490.01791,"name":"offline","context":{"idset":"8548"}}
+{"timestamp":1713808490.0203605,"name":"offline","context":{"idset":"8549"}}
+{"timestamp":1713808490.042136,"name":"offline","context":{"idset":"8550"}}
+{"timestamp":1713808490.0562279,"name":"offline","context":{"idset":"8552"}}
+{"timestamp":1713808490.0735571,"name":"offline","context":{"idset":"8553"}}
+{"timestamp":1713808490.0871146,"name":"offline","context":{"idset":"8554"}}
+{"timestamp":1713808490.1006355,"name":"offline","context":{"idset":"8555"}}
+{"timestamp":1713808490.1028485,"name":"offline","context":{"idset":"8556"}}
+{"timestamp":1713808490.1050217,"name":"offline","context":{"idset":"8557"}}
+{"timestamp":1713808490.109195,"name":"offline","context":{"idset":"8558"}}
+{"timestamp":1713808490.1134546,"name":"offline","context":{"idset":"8559"}}
+{"timestamp":1713808490.1176515,"name":"offline","context":{"idset":"8560"}}
+{"timestamp":1713808490.1220982,"name":"offline","context":{"idset":"8561"}}
+{"timestamp":1713808490.1258717,"name":"offline","context":{"idset":"8562"}}
+{"timestamp":1713808490.1296489,"name":"offline","context":{"idset":"8563"}}
+{"timestamp":1713808490.1599231,"name":"offline","context":{"idset":"8564"}}
+{"timestamp":1713808491.5210907,"name":"offline","context":{"idset":"8459"}}
+{"timestamp":1713808491.5239785,"name":"offline","context":{"idset":"8461"}}
+{"timestamp":1713808491.5274472,"name":"offline","context":{"idset":"8465"}}
+{"timestamp":1713808491.530653,"name":"offline","context":{"idset":"8475"}}
+{"timestamp":1713808491.5337517,"name":"offline","context":{"idset":"8477"}}
+{"timestamp":1713808491.5370545,"name":"offline","context":{"idset":"8479"}}
+{"timestamp":1713808491.5395787,"name":"offline","context":{"idset":"8481"}}
+{"timestamp":1713808491.5417085,"name":"offline","context":{"idset":"8482"}}
+{"timestamp":1713808491.5438082,"name":"offline","context":{"idset":"8483"}}
+{"timestamp":1713808491.5458789,"name":"offline","context":{"idset":"8485"}}
+{"timestamp":1713808491.5479574,"name":"offline","context":{"idset":"8486"}}
+{"timestamp":1713808491.5501506,"name":"offline","context":{"idset":"8488"}}
+{"timestamp":1713808491.5543175,"name":"offline","context":{"idset":"8490"}}
+{"timestamp":1713808491.5582962,"name":"offline","context":{"idset":"8491"}}
+{"timestamp":1713808491.5622146,"name":"offline","context":{"idset":"8492"}}
+{"timestamp":1713808491.5663307,"name":"offline","context":{"idset":"8494"}}
+{"timestamp":1713808491.5702381,"name":"offline","context":{"idset":"8510"}}
+{"timestamp":1713808491.5741482,"name":"offline","context":{"idset":"8512"}}
+{"timestamp":1713808491.5780337,"name":"offline","context":{"idset":"8516"}}
+{"timestamp":1713808491.5820017,"name":"offline","context":{"idset":"8522"}}
+{"timestamp":1713808491.5860181,"name":"offline","context":{"idset":"8523"}}
+{"timestamp":1713808491.5899837,"name":"offline","context":{"idset":"8529"}}
+{"timestamp":1713808491.5939281,"name":"offline","context":{"idset":"8534"}}
+{"timestamp":1713808491.6085064,"name":"offline","context":{"idset":"8535"}}
+{"timestamp":1713808491.6124928,"name":"offline","context":{"idset":"8537"}}
+{"timestamp":1713808491.616442,"name":"offline","context":{"idset":"8539"}}
+{"timestamp":1713808491.6208749,"name":"offline","context":{"idset":"8543"}}
+{"timestamp":1713808491.6248453,"name":"offline","context":{"idset":"8546"}}
+{"timestamp":1713808491.6288087,"name":"offline","context":{"idset":"8551"}}
+{"timestamp":1713808493.5327964,"name":"offline","context":{"idset":"8458"}}
+{"timestamp":1713808511.532069,"name":"offline","context":{"idset":"8480"}}
+{"timestamp":1713808537.4461186,"name":"offline","context":{"idset":"8447"}}
+{"timestamp":1713808537.5328212,"name":"offline","context":{"idset":"8524"}}
+{"timestamp":1713808541.5334947,"name":"offline","context":{"idset":"8536"}}
+{"timestamp":1713808871.6596701,"name":"online","context":{"idset":"10756"}}
+{"timestamp":1713809189.9029732,"name":"online","context":{"idset":"11285"}}
+{"timestamp":1713809858.844974,"name":"drain","context":{"idset":"7696","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713809864.8432317,"name":"drain","context":{"idset":"7754","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713810084.8220437,"name":"drain","context":{"idset":"8181-8308","reason":"testing -kk","overwrite":0}}
+{"timestamp":1713810424.344018,"name":"online","context":{"idset":"11203"}}
+{"timestamp":1713810424.6321635,"name":"online","context":{"idset":"11204"}}
+{"timestamp":1713810491.2805736,"name":"drain","context":{"idset":"11203-11204","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1713811472.194345,"name":"online","context":{"idset":"767-768"}}
+{"timestamp":1713811796.4110909,"name":"online","context":{"idset":"8182"}}
+{"timestamp":1713811980.7147315,"name":"offline","context":{"idset":"763"}}
+{"timestamp":1713813557.6578379,"name":"online","context":{"idset":"7739"}}
+{"timestamp":1713815287.5328524,"name":"offline","context":{"idset":"10308"}}
+{"timestamp":1713815925.5349715,"name":"offline","context":{"idset":"10324"}}
+{"timestamp":1713815927.5347753,"name":"offline","context":{"idset":"10323"}}
+{"timestamp":1713816302.0100281,"name":"online","context":{"idset":"7914"}}
+{"timestamp":1713816302.2085207,"name":"online","context":{"idset":"7912"}}
+{"timestamp":1713816302.4981,"name":"online","context":{"idset":"7823,7913,7916-7917"}}
+{"timestamp":1713816302.8070199,"name":"online","context":{"idset":"7910,7920"}}
+{"timestamp":1713816303.00172,"name":"online","context":{"idset":"7909,7921-7922"}}
+{"timestamp":1713816303.1967008,"name":"online","context":{"idset":"7915,7918-7919"}}
+{"timestamp":1713816303.3173652,"name":"online","context":{"idset":"7911"}}
+{"timestamp":1713816313.9503052,"name":"online","context":{"idset":"7923-7924"}}
+{"timestamp":1713816346.9312205,"name":"online","context":{"idset":"7824"}}
+{"timestamp":1713816481.5322735,"name":"offline","context":{"idset":"8833"}}
+{"timestamp":1713816483.5307369,"name":"offline","context":{"idset":"8834"}}
+{"timestamp":1713816779.4464228,"name":"offline","context":{"idset":"11285"}}
+{"timestamp":1713816779.5327702,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1713816813.5325537,"name":"offline","context":{"idset":"851"}}
+{"timestamp":1713816815.5333796,"name":"offline","context":{"idset":"852"}}
+{"timestamp":1713817217.2820065,"name":"undrain","context":{"idset":"11203-11204"}}
+{"timestamp":1713817629.537564,"name":"offline","context":{"idset":"11302"}}
+{"timestamp":1713818315.5325456,"name":"offline","context":{"idset":"11337"}}
+{"timestamp":1713819021.5310721,"name":"offline","context":{"idset":"7921"}}
+{"timestamp":1713821745.702383,"name":"offline","context":{"idset":"896"}}
+{"timestamp":1713821912.9111559,"name":"online","context":{"idset":"9083-9084"}}
+{"timestamp":1713822904.04284,"name":"offline","context":{"idset":"9359"}}
+{"timestamp":1713823546.1683242,"name":"online","context":{"idset":"10491,10500,10504"}}
+{"timestamp":1713823546.1717467,"name":"online","context":{"idset":"10505,10521"}}
+{"timestamp":1713823546.3899751,"name":"online","context":{"idset":"10494,10516,10519"}}
+{"timestamp":1713823546.578413,"name":"online","context":{"idset":"10511"}}
+{"timestamp":1713823546.6853752,"name":"online","context":{"idset":"10485,10510,10513"}}
+{"timestamp":1713823546.8684387,"name":"online","context":{"idset":"10514"}}
+{"timestamp":1713823547.0756507,"name":"online","context":{"idset":"10499,10506,10520,10522-10524"}}
+{"timestamp":1713823547.2637579,"name":"online","context":{"idset":"10487-10488,10490,10493,10495,10501,10503,10507-10508,10517"}}
+{"timestamp":1713823547.3682325,"name":"online","context":{"idset":"10486,10498,10502,10509,10512"}}
+{"timestamp":1713823548.0743804,"name":"online","context":{"idset":"10515"}}
+{"timestamp":1713823548.0777051,"name":"online","context":{"idset":"10497"}}
+{"timestamp":1713823548.0808935,"name":"online","context":{"idset":"10496"}}
+{"timestamp":1713823548.1598885,"name":"online","context":{"idset":"10518"}}
+{"timestamp":1713823644.4066112,"name":"online","context":{"idset":"9636"}}
+{"timestamp":1713825258.074883,"name":"online","context":{"idset":"10744"}}
+{"timestamp":1713825258.2797751,"name":"online","context":{"idset":"10748"}}
+{"timestamp":1713826355.0925379,"name":"offline","context":{"idset":"820"}}
+{"timestamp":1713827243.5359695,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1713829392.9965711,"name":"offline","context":{"idset":"7754"}}
+{"timestamp":1713830114.2914922,"name":"offline","context":{"idset":"7696"}}
+{"timestamp":1713830144.2749681,"name":"offline","context":{"idset":"7739"}}
+{"timestamp":1713830639.5323849,"name":"offline","context":{"idset":"7922"}}
+{"timestamp":1713833229.1155198,"name":"online","context":{"idset":"7739"}}
+{"timestamp":1713833229.6125014,"name":"online","context":{"idset":"7754"}}
+{"timestamp":1713839331.9803264,"name":"online","context":{"idset":"11301"}}
+{"timestamp":1713839332.0990901,"name":"online","context":{"idset":"11302"}}
+{"timestamp":1713839442.5850832,"name":"online","context":{"idset":"11325"}}
+{"timestamp":1713839443.0753276,"name":"online","context":{"idset":"11326"}}
+{"timestamp":1713839520.3506086,"name":"online","context":{"idset":"11338"}}
+{"timestamp":1713839520.5785947,"name":"online","context":{"idset":"11337"}}
+{"timestamp":1713839622.1805317,"name":"online","context":{"idset":"11367"}}
+{"timestamp":1713839624.111043,"name":"online","context":{"idset":"11371"}}
+{"timestamp":1713839624.6508918,"name":"online","context":{"idset":"11365,11375,11377"}}
+{"timestamp":1713839624.8440976,"name":"online","context":{"idset":"11369,11373"}}
+{"timestamp":1713839625.2378597,"name":"online","context":{"idset":"11378"}}
+{"timestamp":1713839626.4241021,"name":"online","context":{"idset":"11380"}}
+{"timestamp":1713839627.3097703,"name":"online","context":{"idset":"11376"}}
+{"timestamp":1713839627.8066916,"name":"online","context":{"idset":"11374"}}
+{"timestamp":1713839627.9883382,"name":"online","context":{"idset":"11366"}}
+{"timestamp":1713839628.3076317,"name":"online","context":{"idset":"11368,11370,11372"}}
+{"timestamp":1713843627.3924549,"name":"online","context":{"idset":"7696"}}
+{"timestamp":1713843665.1706376,"name":"offline","context":{"idset":"7696"}}
+{"timestamp":1713843914.1199784,"name":"online","context":{"idset":"7696"}}
+{"timestamp":1713843971.6216497,"name":"offline","context":{"idset":"7754"}}
+{"timestamp":1713844130.7305102,"name":"online","context":{"idset":"7754"}}
+{"timestamp":1713844176.6717978,"name":"undrain","context":{"idset":"7696,7754"}}
+{"timestamp":1713844248.163595,"name":"offline","context":{"idset":"7739"}}
+{"timestamp":1713844439.6494093,"name":"online","context":{"idset":"7739"}}
+{"timestamp":1713844511.5880141,"name":"undrain","context":{"idset":"7739"}}
+{"timestamp":1713852604.9808753,"name":"offline","context":{"idset":"7802"}}
+{"timestamp":1713852605.0810714,"name":"offline","context":{"idset":"7803"}}
+{"timestamp":1713852605.1929913,"name":"offline","context":{"idset":"7798"}}
+{"timestamp":1713852605.2921207,"name":"offline","context":{"idset":"7800"}}
+{"timestamp":1713852605.519371,"name":"offline","context":{"idset":"7797"}}
+{"timestamp":1713852606.241483,"name":"offline","context":{"idset":"7799"}}
+{"timestamp":1713852606.552063,"name":"offline","context":{"idset":"7804"}}
+{"timestamp":1713852606.6682053,"name":"offline","context":{"idset":"7801"}}
+{"timestamp":1713853809.5254121,"name":"offline","context":{"idset":"7805"}}
+{"timestamp":1713853809.6523225,"name":"offline","context":{"idset":"7810"}}
+{"timestamp":1713853809.7760563,"name":"offline","context":{"idset":"7812"}}
+{"timestamp":1713853809.8858547,"name":"offline","context":{"idset":"7809"}}
+{"timestamp":1713853810.1154432,"name":"offline","context":{"idset":"7806"}}
+{"timestamp":1713853810.4991927,"name":"offline","context":{"idset":"7808"}}
+{"timestamp":1713853811.0323389,"name":"offline","context":{"idset":"7811"}}
+{"timestamp":1713853811.6045175,"name":"offline","context":{"idset":"7807"}}
+{"timestamp":1713854049.9256146,"name":"drain","context":{"idset":"7719","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713854098.7562912,"name":"drain","context":{"idset":"7761","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713854116.4974015,"name":"drain","context":{"idset":"7693","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713854117.7418802,"name":"drain","context":{"idset":"7715","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713856214.6046338,"name":"offline","context":{"idset":"7826"}}
+{"timestamp":1713856214.6938465,"name":"offline","context":{"idset":"7828"}}
+{"timestamp":1713856214.7002542,"name":"offline","context":{"idset":"7821"}}
+{"timestamp":1713856214.8004799,"name":"offline","context":{"idset":"7822"}}
+{"timestamp":1713856215.0556049,"name":"offline","context":{"idset":"7825"}}
+{"timestamp":1713856215.4024801,"name":"offline","context":{"idset":"7827"}}
+{"timestamp":1713856215.9716516,"name":"offline","context":{"idset":"7823"}}
+{"timestamp":1713856321.8052132,"name":"offline","context":{"idset":"7824"}}
+{"timestamp":1713857462.0388978,"name":"offline","context":{"idset":"7830"}}
+{"timestamp":1713857462.3032317,"name":"offline","context":{"idset":"7834"}}
+{"timestamp":1713857462.3069239,"name":"offline","context":{"idset":"7829"}}
+{"timestamp":1713857462.4074485,"name":"offline","context":{"idset":"7831"}}
+{"timestamp":1713857462.6051369,"name":"offline","context":{"idset":"7835"}}
+{"timestamp":1713857463.2928288,"name":"offline","context":{"idset":"7833"}}
+{"timestamp":1713857463.6784892,"name":"offline","context":{"idset":"7836"}}
+{"timestamp":1713857463.7777605,"name":"offline","context":{"idset":"7832"}}
+{"timestamp":1713858666.8670499,"name":"offline","context":{"idset":"7838"}}
+{"timestamp":1713858667.1333885,"name":"offline","context":{"idset":"7842"}}
+{"timestamp":1713858667.6200271,"name":"offline","context":{"idset":"7841"}}
+{"timestamp":1713858667.8376229,"name":"offline","context":{"idset":"7843"}}
+{"timestamp":1713858668.0350156,"name":"offline","context":{"idset":"7840"}}
+{"timestamp":1713858668.1925488,"name":"offline","context":{"idset":"7839"}}
+{"timestamp":1713858668.2909853,"name":"offline","context":{"idset":"7844"}}
+{"timestamp":1713858668.5953431,"name":"offline","context":{"idset":"7837"}}
+{"timestamp":1713859871.5950122,"name":"offline","context":{"idset":"7847"}}
+{"timestamp":1713859871.6940696,"name":"offline","context":{"idset":"7851"}}
+{"timestamp":1713859871.8025582,"name":"offline","context":{"idset":"7850"}}
+{"timestamp":1713859871.8938627,"name":"offline","context":{"idset":"7848"}}
+{"timestamp":1713859872.2699344,"name":"offline","context":{"idset":"7852"}}
+{"timestamp":1713859872.4727492,"name":"offline","context":{"idset":"7849"}}
+{"timestamp":1713859873.2040462,"name":"offline","context":{"idset":"7845"}}
+{"timestamp":1713859873.789449,"name":"offline","context":{"idset":"7846"}}
+{"timestamp":1713861076.4445021,"name":"offline","context":{"idset":"7856"}}
+{"timestamp":1713861076.5441632,"name":"offline","context":{"idset":"7857"}}
+{"timestamp":1713861077.0830052,"name":"offline","context":{"idset":"7860"}}
+{"timestamp":1713861077.1786132,"name":"offline","context":{"idset":"7854"}}
+{"timestamp":1713861077.2817333,"name":"offline","context":{"idset":"7853"}}
+{"timestamp":1713861077.7179942,"name":"offline","context":{"idset":"7859"}}
+{"timestamp":1713861077.8167827,"name":"offline","context":{"idset":"7858"}}
+{"timestamp":1713861077.9981837,"name":"offline","context":{"idset":"7855"}}
+{"timestamp":1713862280.5363736,"name":"offline","context":{"idset":"7865"}}
+{"timestamp":1713862280.6342728,"name":"offline","context":{"idset":"7864"}}
+{"timestamp":1713862280.8643153,"name":"offline","context":{"idset":"7862"}}
+{"timestamp":1713862281.4630227,"name":"offline","context":{"idset":"7867"}}
+{"timestamp":1713862281.889169,"name":"offline","context":{"idset":"7863"}}
+{"timestamp":1713862282.1965263,"name":"offline","context":{"idset":"7861"}}
+{"timestamp":1713862282.5153825,"name":"offline","context":{"idset":"7868"}}
+{"timestamp":1713862283.1583159,"name":"offline","context":{"idset":"7866"}}
+{"timestamp":1713863485.9573531,"name":"offline","context":{"idset":"7870"}}
+{"timestamp":1713863486.4354398,"name":"offline","context":{"idset":"7875"}}
+{"timestamp":1713863487.0522103,"name":"offline","context":{"idset":"7872"}}
+{"timestamp":1713863487.1506455,"name":"offline","context":{"idset":"7869"}}
+{"timestamp":1713863487.431967,"name":"offline","context":{"idset":"7873"}}
+{"timestamp":1713863487.6517825,"name":"offline","context":{"idset":"7871"}}
+{"timestamp":1713863487.9400802,"name":"offline","context":{"idset":"7876"}}
+{"timestamp":1713863488.1401713,"name":"offline","context":{"idset":"7874"}}
+{"timestamp":1713864690.5772123,"name":"offline","context":{"idset":"7878"}}
+{"timestamp":1713864690.7759109,"name":"offline","context":{"idset":"7883"}}
+{"timestamp":1713864691.1729796,"name":"offline","context":{"idset":"7879"}}
+{"timestamp":1713864691.2724869,"name":"offline","context":{"idset":"7882"}}
+{"timestamp":1713864691.5987113,"name":"offline","context":{"idset":"7880"}}
+{"timestamp":1713864691.6146502,"name":"offline","context":{"idset":"7884"}}
+{"timestamp":1713864691.7123387,"name":"offline","context":{"idset":"7881"}}
+{"timestamp":1713864692.8140655,"name":"offline","context":{"idset":"7877"}}
+{"timestamp":1713865894.829427,"name":"offline","context":{"idset":"7892"}}
+{"timestamp":1713865894.9338636,"name":"offline","context":{"idset":"7885"}}
+{"timestamp":1713865895.0337682,"name":"offline","context":{"idset":"7886"}}
+{"timestamp":1713865895.2055743,"name":"offline","context":{"idset":"7889"}}
+{"timestamp":1713865895.3053424,"name":"offline","context":{"idset":"7891"}}
+{"timestamp":1713865895.7512853,"name":"offline","context":{"idset":"7887"}}
+{"timestamp":1713865896.4930382,"name":"offline","context":{"idset":"7888"}}
+{"timestamp":1713865897.6773741,"name":"offline","context":{"idset":"7890"}}
+{"timestamp":1713867099.5894427,"name":"offline","context":{"idset":"7895"}}
+{"timestamp":1713867099.690671,"name":"offline","context":{"idset":"7894"}}
+{"timestamp":1713867099.7874825,"name":"offline","context":{"idset":"7896"}}
+{"timestamp":1713867100.0538561,"name":"offline","context":{"idset":"7898"}}
+{"timestamp":1713867100.1512017,"name":"offline","context":{"idset":"7899"}}
+{"timestamp":1713867101.1708951,"name":"offline","context":{"idset":"7897"}}
+{"timestamp":1713867101.6159277,"name":"offline","context":{"idset":"7900"}}
+{"timestamp":1713867101.9111845,"name":"offline","context":{"idset":"7893"}}
+{"timestamp":1713868304.2487438,"name":"offline","context":{"idset":"7901"}}
+{"timestamp":1713868304.3484819,"name":"offline","context":{"idset":"7903"}}
+{"timestamp":1713868304.5429533,"name":"offline","context":{"idset":"7905"}}
+{"timestamp":1713868304.6577554,"name":"offline","context":{"idset":"7908"}}
+{"timestamp":1713868304.8415928,"name":"offline","context":{"idset":"7902"}}
+{"timestamp":1713868304.9413192,"name":"offline","context":{"idset":"7906"}}
+{"timestamp":1713868305.2099402,"name":"offline","context":{"idset":"7904"}}
+{"timestamp":1713868305.3258986,"name":"offline","context":{"idset":"7907"}}
+{"timestamp":1713869507.8094127,"name":"offline","context":{"idset":"7915"}}
+{"timestamp":1713869507.9830055,"name":"offline","context":{"idset":"7912"}}
+{"timestamp":1713869508.0822873,"name":"offline","context":{"idset":"7914"}}
+{"timestamp":1713869509.1010954,"name":"offline","context":{"idset":"7909"}}
+{"timestamp":1713869509.1995451,"name":"offline","context":{"idset":"7916"}}
+{"timestamp":1713869509.527806,"name":"offline","context":{"idset":"7913"}}
+{"timestamp":1713869509.6274588,"name":"offline","context":{"idset":"7910"}}
+{"timestamp":1713869509.8582458,"name":"offline","context":{"idset":"7911"}}
+{"timestamp":1713870711.2641375,"name":"offline","context":{"idset":"7919"}}
+{"timestamp":1713870711.3976285,"name":"offline","context":{"idset":"7917"}}
+{"timestamp":1713870711.4158802,"name":"offline","context":{"idset":"7920"}}
+{"timestamp":1713870711.5151455,"name":"offline","context":{"idset":"7918"}}
+{"timestamp":1713871914.2167847,"name":"offline","context":{"idset":"7924"}}
+{"timestamp":1713871914.3163419,"name":"offline","context":{"idset":"7923"}}
+{"timestamp":1713873133.5369599,"name":"offline","context":{"idset":"7980"}}
+{"timestamp":1713879893.1167343,"name":"drain","context":{"idset":"11125-11252","overwrite":0}}
+{"timestamp":1713880279.4495969,"name":"offline","context":{"idset":"9635"}}
+{"timestamp":1713880279.5333433,"name":"offline","context":{"idset":"9636"}}
+{"timestamp":1713880313.5335999,"name":"offline","context":{"idset":"894"}}
+{"timestamp":1713880799.6263103,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1713880799.7180846,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1713880799.7263272,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1713880799.7363746,"name":"offline","context":{"idset":"11154"}}
+{"timestamp":1713880799.7424216,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1713880799.7478347,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1713880799.7613239,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1713880799.8467901,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1713880799.9497008,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1713880799.9576805,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1713880799.9672036,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1713880799.9730906,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1713880800.0448554,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1713880800.0481989,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1713880800.0609853,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1713880800.0649395,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1713880800.0689545,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1713880800.0733943,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1713880800.0766718,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1713880800.0799356,"name":"offline","context":{"idset":"11146"}}
+{"timestamp":1713880800.0988367,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1713880800.1180344,"name":"offline","context":{"idset":"11149"}}
+{"timestamp":1713880800.1219835,"name":"offline","context":{"idset":"11150"}}
+{"timestamp":1713880800.1256173,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1713880800.1305308,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1713880800.1358712,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1713880800.1398149,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1713880800.1432476,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1713880800.1486435,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1713880800.154016,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1713880800.1587923,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1713880800.1625023,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1713880800.1670396,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1713880800.170516,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1713880800.1738944,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1713880800.1772616,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1713880800.1804957,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1713880800.2746344,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1713880800.4473851,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1713880800.4507077,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1713880800.4539859,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1713880800.4572432,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1713880800.4606135,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1713880800.4640481,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1713880800.4675529,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1713880800.4711518,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1713880800.4746141,"name":"offline","context":{"idset":"11153"}}
+{"timestamp":1713880800.4777732,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1713880800.4809139,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1713880800.4841731,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1713880800.4874399,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1713880800.4906626,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1713880800.4938929,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1713880800.497067,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1713880800.5002124,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1713880800.6475532,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1713880800.6507988,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1713880800.6540902,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1713880800.6572993,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1713880800.6604908,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1713880800.6729045,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1713880800.6760755,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1713880800.6792943,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1713880800.6824958,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1713880800.6856875,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1713880800.7030122,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1713880800.7061853,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1713880800.7093072,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1713880800.7125642,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1713880800.7157788,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1713880800.7189605,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1713880800.7221425,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1713880800.7252858,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1713880800.7468212,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1713880800.7676699,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1713880800.7884662,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1713880800.7917337,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1713880800.7948444,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1713880800.7979116,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1713880800.8010032,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1713880800.8041446,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1713880800.8074279,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1713880800.810591,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1713880800.8136504,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1713880800.8168206,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1713880800.8201468,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1713880800.8237052,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1713880800.8284059,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1713880800.8315749,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1713880800.8346715,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1713880800.8377385,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1713880800.8407943,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1713880800.8438711,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1713880800.8468823,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1713880800.849895,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1713880800.8618183,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1713880800.8823709,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1713880800.90311,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1713880800.9241872,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1713880800.9453497,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1713880800.9657173,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1713880800.9775321,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1713880800.9805779,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1713880800.9836092,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1713880800.9866436,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1713880800.989677,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1713880800.9926734,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1713880800.9958367,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1713880800.9988739,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1713880801.0237114,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1713880801.0439723,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1713880801.0644412,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1713880801.0849049,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1713880801.1055079,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1713880801.1258969,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1713880801.1375647,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1713880801.140564,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1713880801.1435285,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1713880801.1637743,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1713880801.1845329,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1713880801.1875045,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1713880801.1904991,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1713880801.1935217,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1713880801.1967931,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1713880801.6411395,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1713880801.6441617,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1713880801.6472492,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1713880805.3635886,"name":"drain","context":{"idset":"11240","reason":"epilog failed for jobid fr4tRX4xPSw","overwrite":0}}
+{"timestamp":1713880805.4645185,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1713881466.2238214,"name":"offline","context":{"idset":"7818"}}
+{"timestamp":1713881466.4854627,"name":"offline","context":{"idset":"7815"}}
+{"timestamp":1713881466.5912256,"name":"offline","context":{"idset":"7820"}}
+{"timestamp":1713881466.7753394,"name":"offline","context":{"idset":"7814"}}
+{"timestamp":1713881466.8684332,"name":"offline","context":{"idset":"7813"}}
+{"timestamp":1713881466.965337,"name":"offline","context":{"idset":"7816"}}
+{"timestamp":1713881467.3015893,"name":"offline","context":{"idset":"7819"}}
+{"timestamp":1713881468.3218434,"name":"offline","context":{"idset":"7817"}}
+{"timestamp":1713882829.8525572,"name":"online","context":{"idset":"11152"}}
+{"timestamp":1713882830.010046,"name":"online","context":{"idset":"11137"}}
+{"timestamp":1713882830.1511805,"name":"online","context":{"idset":"11173"}}
+{"timestamp":1713882830.3114433,"name":"online","context":{"idset":"11135"}}
+{"timestamp":1713882830.4337397,"name":"online","context":{"idset":"11130"}}
+{"timestamp":1713882830.5307889,"name":"online","context":{"idset":"11131"}}
+{"timestamp":1713882830.6572723,"name":"online","context":{"idset":"11128,11138"}}
+{"timestamp":1713882830.7593551,"name":"online","context":{"idset":"11167"}}
+{"timestamp":1713882831.0292172,"name":"online","context":{"idset":"11133,11172,11189"}}
+{"timestamp":1713882831.1197462,"name":"online","context":{"idset":"11145"}}
+{"timestamp":1713882831.1388459,"name":"online","context":{"idset":"11170"}}
+{"timestamp":1713882831.2651272,"name":"online","context":{"idset":"11182,11206,11229"}}
+{"timestamp":1713882831.4594223,"name":"online","context":{"idset":"11150,11154-11155,11226"}}
+{"timestamp":1713882831.6687443,"name":"online","context":{"idset":"11171,11177,11195,11228"}}
+{"timestamp":1713882831.8795364,"name":"online","context":{"idset":"11188,11224"}}
+{"timestamp":1713882832.090483,"name":"online","context":{"idset":"11139,11158,11162"}}
+{"timestamp":1713882832.1925285,"name":"online","context":{"idset":"11140,11147,11178,11210,11213"}}
+{"timestamp":1713882832.2945559,"name":"online","context":{"idset":"11146"}}
+{"timestamp":1713882832.4811497,"name":"online","context":{"idset":"11142,11242"}}
+{"timestamp":1713882832.6005495,"name":"online","context":{"idset":"11129,11134,11180,11232"}}
+{"timestamp":1713882832.7963274,"name":"online","context":{"idset":"11136,11175,11185,11196,11207,11214,11238"}}
+{"timestamp":1713882832.942766,"name":"online","context":{"idset":"11199,11204,11218"}}
+{"timestamp":1713882833.1497257,"name":"online","context":{"idset":"11156,11166,11190,11211,11231,11252"}}
+{"timestamp":1713882833.3618515,"name":"online","context":{"idset":"11125-11127,11141,11144,11151,11157,11161,11186,11193,11241"}}
+{"timestamp":1713882833.464371,"name":"online","context":{"idset":"11132,11187,11205,11220,11222,11244"}}
+{"timestamp":1713882833.5780346,"name":"online","context":{"idset":"11143,11153,11160,11184,11230"}}
+{"timestamp":1713882833.678977,"name":"online","context":{"idset":"11148,11168,11174,11179,11191,11215-11216,11219,11221,11223,11247-11248"}}
+{"timestamp":1713882833.8744218,"name":"online","context":{"idset":"11149,11159,11164-11165,11176,11183,11192,11197,11203,11208,11212,11217,11234"}}
+{"timestamp":1713882833.9847145,"name":"online","context":{"idset":"11198,11201-11202,11209,11227,11233,11236,11246,11250"}}
+{"timestamp":1713882834.19031,"name":"online","context":{"idset":"11163,11169,11181,11200,11225,11235,11251"}}
+{"timestamp":1713882834.4050586,"name":"online","context":{"idset":"11194,11237,11239,11245,11249"}}
+{"timestamp":1713882834.5074782,"name":"online","context":{"idset":"11243"}}
+{"timestamp":1713882847.6946902,"name":"undrain","context":{"idset":"11125-11252"}}
+{"timestamp":1713883106.6111057,"name":"online","context":{"idset":"11240"}}
+{"timestamp":1713884596.2981601,"name":"drain","context":{"idset":"7781","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713886122.5392351,"name":"online","context":{"idset":"884"}}
+{"timestamp":1713886123.2637897,"name":"online","context":{"idset":"883"}}
+{"timestamp":1713886407.6455867,"name":"drain","context":{"idset":"11240","reason":"epilog failed for jobid frBvC8p1PwV","overwrite":0}}
+{"timestamp":1713886407.7453802,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1713887988.5150731,"name":"undrain","context":{"idset":"11321-11322"}}
+{"timestamp":1713888661.4989727,"name":"online","context":{"idset":"11285"}}
+{"timestamp":1713888661.7126811,"name":"online","context":{"idset":"11286"}}
+{"timestamp":1713888881.0341513,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1713889125.7568984,"name":"online","context":{"idset":"11723,11730"}}
+{"timestamp":1713889125.9639916,"name":"online","context":{"idset":"11717,11720"}}
+{"timestamp":1713889126.2509904,"name":"online","context":{"idset":"11719,11726,11728"}}
+{"timestamp":1713889126.3491466,"name":"online","context":{"idset":"11718,11724,11732"}}
+{"timestamp":1713889126.5406768,"name":"online","context":{"idset":"11722"}}
+{"timestamp":1713889142.5689669,"name":"online","context":{"idset":"11729"}}
+{"timestamp":1713889142.9786162,"name":"online","context":{"idset":"11721"}}
+{"timestamp":1713889305.039695,"name":"drain","context":{"idset":"11731","reason":"broker was unresponsive"}}
+{"timestamp":1713889385.6696639,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1713889396.9439874,"name":"online","context":{"idset":"11731"}}
+{"timestamp":1713889419.9341111,"name":"online","context":{"idset":"11725"}}
+{"timestamp":1713889468.8825791,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713889527.780674,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1713889563.0989952,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1713889576.131928,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1713889770.2835245,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1713889837.4495072,"name":"online","context":{"idset":"9535"}}
+{"timestamp":1713890581.7938142,"name":"offline","context":{"idset":"8884"}}
+{"timestamp":1713891239.8387475,"name":"offline","context":{"idset":"9360"}}
+{"timestamp":1713892043.0335002,"name":"offline","context":{"idset":"7979"}}
+{"timestamp":1713892049.4817402,"name":"online","context":{"idset":"10134"}}
+{"timestamp":1713892051.6760716,"name":"drain","context":{"idset":"11725","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713892743.0351655,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1713893468.9014587,"name":"online","context":{"idset":"10489"}}
+{"timestamp":1713894327.0343726,"name":"offline","context":{"idset":"11255"}}
+{"timestamp":1713894359.0343637,"name":"offline","context":{"idset":"882"}}
+{"timestamp":1713896997.095875,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1713897773.0864758,"name":"online","context":{"idset":"779"}}
+{"timestamp":1713897876.4346669,"name":"offline","context":{"idset":"779"}}
+{"timestamp":1713899697.2090824,"name":"online","context":{"idset":"9360"}}
+{"timestamp":1713899697.4123225,"name":"online","context":{"idset":"9359"}}
+{"timestamp":1713899942.9472032,"name":"offline","context":{"idset":"10261"}}
+{"timestamp":1713899943.0360498,"name":"offline","context":{"idset":"10262"}}
+{"timestamp":1713899979.0333824,"name":"offline","context":{"idset":"890"}}
+{"timestamp":1713900202.7599516,"name":"online","context":{"idset":"10919"}}
+{"timestamp":1713900202.9681814,"name":"online","context":{"idset":"10880,10883"}}
+{"timestamp":1713900203.1568687,"name":"online","context":{"idset":"10921"}}
+{"timestamp":1713900203.3417366,"name":"online","context":{"idset":"10922"}}
+{"timestamp":1713900203.4588647,"name":"online","context":{"idset":"10879"}}
+{"timestamp":1713900203.665235,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1713900681.702395,"name":"online","context":{"idset":"11856"}}
+{"timestamp":1713901226.5996945,"name":"drain","context":{"idset":"9735-9736","overwrite":0}}
+{"timestamp":1713901331.4475005,"name":"undrain","context":{"idset":"9735-9736"}}
+{"timestamp":1713902338.5955629,"name":"drain","context":{"idset":"11324","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713902389.0332687,"name":"offline","context":{"idset":"11380"}}
+{"timestamp":1713902903.0357025,"name":"offline","context":{"idset":"9359"}}
+{"timestamp":1713903109.0362761,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1713903135.4180164,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1713903135.5147588,"name":"offline","context":{"idset":"7781"}}
+{"timestamp":1713903135.520509,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1713903135.6192141,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1713903136.6415429,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1713903502.9490073,"name":"offline","context":{"idset":"9651"}}
+{"timestamp":1713903503.034976,"name":"offline","context":{"idset":"9652"}}
+{"timestamp":1713904194.3411636,"name":"drain","context":{"idset":"825-826","reason":"--reason re-flashing BIOS","overwrite":0}}
+{"timestamp":1713904309.2164865,"name":"online","context":{"idset":"9359"}}
+{"timestamp":1713904328.4521334,"name":"online","context":{"idset":"542-543,545,548"}}
+{"timestamp":1713904328.6463509,"name":"online","context":{"idset":"541"}}
+{"timestamp":1713905315.3075962,"name":"online","context":{"idset":"10743"}}
+{"timestamp":1713905315.5483844,"name":"online","context":{"idset":"10741"}}
+{"timestamp":1713905685.9132683,"name":"online","context":{"idset":"10747"}}
+{"timestamp":1713905687.4127743,"name":"online","context":{"idset":"10745"}}
+{"timestamp":1713905704.9012313,"name":"online","context":{"idset":"10755"}}
+{"timestamp":1713906470.3445983,"name":"online","context":{"idset":"7693"}}
+{"timestamp":1713906541.3943419,"name":"online","context":{"idset":"9636"}}
+{"timestamp":1713906541.3975515,"name":"online","context":{"idset":"9635"}}
+{"timestamp":1713906555.1475914,"name":"online","context":{"idset":"7715"}}
+{"timestamp":1713906588.945132,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1713906589.3220472,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1713906589.4218349,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1713906589.5232437,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1713906589.5316908,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1713906589.6301787,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1713906589.6443486,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1713906589.7423611,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1713906589.7548134,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1713906589.8556306,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1713906590.1787124,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1713906590.3675485,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1713906590.4590731,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1713906590.5588322,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713906590.6772249,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1713906590.7764475,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1713906667.9717803,"name":"online","context":{"idset":"7719"}}
+{"timestamp":1713906701.2841039,"name":"online","context":{"idset":"7761"}}
+{"timestamp":1713906900.8020496,"name":"offline","context":{"idset":"892"}}
+{"timestamp":1713907627.8736429,"name":"undrain","context":{"idset":"7693,7715,7719,7761"}}
+{"timestamp":1713907641.8584683,"name":"drain","context":{"idset":"779-780","reason":"--reason Troubleshoot BIOS issue","overwrite":0}}
+{"timestamp":1713907877.0338695,"name":"offline","context":{"idset":"11685"}}
+{"timestamp":1713907879.0346074,"name":"offline","context":{"idset":"11686"}}
+{"timestamp":1713908313.5783226,"name":"online","context":{"idset":"10860"}}
+{"timestamp":1713908399.1415987,"name":"online","context":{"idset":"10395"}}
+{"timestamp":1713908889.3952434,"name":"online","context":{"idset":"803"}}
+{"timestamp":1713909105.1252248,"name":"online","context":{"idset":"11686"}}
+{"timestamp":1713909938.5714874,"name":"online","context":{"idset":"9651-9652"}}
+{"timestamp":1713909939.4765239,"name":"online","context":{"idset":"10381"}}
+{"timestamp":1713910724.4596641,"name":"undrain","context":{"idset":"8182,8184-8233,8235-8236,8238-8290,8292-8303,8305-8308"}}
+{"timestamp":1713910767.1200578,"name":"drain","context":{"idset":"11789","overwrite":0}}
+{"timestamp":1713910773.4701493,"name":"drain","context":{"idset":"11789","reason":"testing -kk","overwrite":0}}
+{"timestamp":1713911879.6324205,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1713911879.7310426,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1713911879.7381558,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1713911879.7501118,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1713911879.7566614,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1713911879.7622752,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1713911879.7660646,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1713911879.7814126,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1713911879.7858398,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1713911879.7897415,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1713911879.8920882,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1713911879.9347136,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1713911879.9385211,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1713911880.0311193,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1713912227.0344191,"name":"offline","context":{"idset":"10395"}}
+{"timestamp":1713912381.0354578,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1713913603.5905368,"name":"online","context":{"idset":"10382"}}
+{"timestamp":1713913603.6999075,"name":"online","context":{"idset":"10381"}}
+{"timestamp":1713913603.8902435,"name":"online","context":{"idset":"10396"}}
+{"timestamp":1713913604.4358728,"name":"online","context":{"idset":"10395"}}
+{"timestamp":1713914148.398994,"name":"online","context":{"idset":"8181"}}
+{"timestamp":1713914148.6199334,"name":"online","context":{"idset":"8183"}}
+{"timestamp":1713914195.7272837,"name":"offline","context":{"idset":"8201"}}
+{"timestamp":1713914587.7858119,"name":"online","context":{"idset":"11242"}}
+{"timestamp":1713914587.8976436,"name":"online","context":{"idset":"11239"}}
+{"timestamp":1713914588.3740196,"name":"online","context":{"idset":"11244"}}
+{"timestamp":1713914588.5628664,"name":"online","context":{"idset":"11240"}}
+{"timestamp":1713914588.7496598,"name":"online","context":{"idset":"11237,11241,11250"}}
+{"timestamp":1713914588.8562376,"name":"online","context":{"idset":"11245-11246,11248-11249,11252"}}
+{"timestamp":1713914588.9596539,"name":"online","context":{"idset":"11243"}}
+{"timestamp":1713914589.1496003,"name":"online","context":{"idset":"11238,11247,11251"}}
+{"timestamp":1713914677.0351253,"name":"offline","context":{"idset":"8233"}}
+{"timestamp":1713915346.2637968,"name":"drain","context":{"idset":"482-490,492","reason":"New Blade Installation --JRG","overwrite":1}}
+{"timestamp":1713915402.1051688,"name":"online","context":{"idset":"8291"}}
+{"timestamp":1713915404.0652561,"name":"offline","context":{"idset":"481"}}
+{"timestamp":1713915404.0736415,"name":"offline","context":{"idset":"480"}}
+{"timestamp":1713915404.0776551,"name":"offline","context":{"idset":"477"}}
+{"timestamp":1713915404.1756842,"name":"offline","context":{"idset":"478"}}
+{"timestamp":1713915404.1848409,"name":"offline","context":{"idset":"479"}}
+{"timestamp":1713915404.190593,"name":"offline","context":{"idset":"492"}}
+{"timestamp":1713915404.1965075,"name":"offline","context":{"idset":"482"}}
+{"timestamp":1713915404.202692,"name":"offline","context":{"idset":"483"}}
+{"timestamp":1713915404.2066791,"name":"offline","context":{"idset":"484"}}
+{"timestamp":1713915404.2106335,"name":"offline","context":{"idset":"485"}}
+{"timestamp":1713915404.2190027,"name":"offline","context":{"idset":"487"}}
+{"timestamp":1713915404.2229679,"name":"offline","context":{"idset":"488"}}
+{"timestamp":1713915404.2270069,"name":"offline","context":{"idset":"490"}}
+{"timestamp":1713915404.2309432,"name":"offline","context":{"idset":"486"}}
+{"timestamp":1713915404.3580496,"name":"offline","context":{"idset":"489"}}
+{"timestamp":1713916197.7234499,"name":"online","context":{"idset":"10878-10879,10881"}}
+{"timestamp":1713916197.931987,"name":"online","context":{"idset":"10877,10880,10882"}}
+{"timestamp":1713916332.8560202,"name":"online","context":{"idset":"10870"}}
+{"timestamp":1713916333.119097,"name":"online","context":{"idset":"10871,10873-10876"}}
+{"timestamp":1713916333.4188621,"name":"online","context":{"idset":"10869"}}
+{"timestamp":1713916337.4237685,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1713917073.0357461,"name":"offline","context":{"idset":"10188"}}
+{"timestamp":1713918148.5827897,"name":"online","context":{"idset":"11379-11380"}}
+{"timestamp":1713918183.589632,"name":"undrain","context":{"idset":"11365-11380"}}
+{"timestamp":1713918468.5224636,"name":"online","context":{"idset":"7979"}}
+{"timestamp":1713918551.726064,"name":"online","context":{"idset":"7980"}}
+{"timestamp":1713919059.3909171,"name":"drain","context":{"idset":"11857","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1713919531.4682245,"name":"online","context":{"idset":"10639"}}
+{"timestamp":1713919531.8164318,"name":"online","context":{"idset":"10703"}}
+{"timestamp":1713919532.0154204,"name":"online","context":{"idset":"10704"}}
+{"timestamp":1713919532.3054776,"name":"online","context":{"idset":"10683-10684,10730"}}
+{"timestamp":1713919532.5022469,"name":"online","context":{"idset":"10618"}}
+{"timestamp":1713919532.7949154,"name":"online","context":{"idset":"10617,10640"}}
+{"timestamp":1713919532.9884644,"name":"online","context":{"idset":"10729"}}
+{"timestamp":1713919533.0943823,"name":"online","context":{"idset":"10653"}}
+{"timestamp":1713919734.1418276,"name":"online","context":{"idset":"10740"}}
+{"timestamp":1713919920.1015887,"name":"undrain","context":{"idset":"11240"}}
+{"timestamp":1713921050.4636536,"name":"online","context":{"idset":"10642"}}
+{"timestamp":1713921050.6742382,"name":"online","context":{"idset":"10638,10644"}}
+{"timestamp":1713921995.8073151,"name":"online","context":{"idset":"11685"}}
+{"timestamp":1713922121.0346429,"name":"offline","context":{"idset":"9359"}}
+{"timestamp":1713923744.779,"name":"drain","context":{"idset":"9216","reason":"epilog failed for jobid frDgn5yhD6o","overwrite":0}}
+{"timestamp":1713923744.878186,"name":"offline","context":{"idset":"9216"}}
+{"timestamp":1713926647.0349302,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1713926689.2551956,"name":"online","context":{"idset":"11658"}}
+{"timestamp":1713926800.7762866,"name":"drain","context":{"idset":"11374","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713927555.0346265,"name":"offline","context":{"idset":"9365"}}
+{"timestamp":1713927557.0346851,"name":"offline","context":{"idset":"9366"}}
+{"timestamp":1713927609.0356939,"name":"offline","context":{"idset":"932"}}
+{"timestamp":1713928460.2976611,"name":"online","context":{"idset":"8677"}}
+{"timestamp":1713928475.4222779,"name":"online","context":{"idset":"8683"}}
+{"timestamp":1713928477.5962451,"name":"online","context":{"idset":"8687"}}
+{"timestamp":1713928478.0916307,"name":"online","context":{"idset":"8680"}}
+{"timestamp":1713928478.4093816,"name":"online","context":{"idset":"8689"}}
+{"timestamp":1713928478.599118,"name":"online","context":{"idset":"8685"}}
+{"timestamp":1713928479.132056,"name":"online","context":{"idset":"8679"}}
+{"timestamp":1713928479.3936338,"name":"online","context":{"idset":"8681"}}
+{"timestamp":1713928480.3192317,"name":"online","context":{"idset":"8691"}}
+{"timestamp":1713928480.4187737,"name":"online","context":{"idset":"8688"}}
+{"timestamp":1713928481.2458394,"name":"online","context":{"idset":"8682"}}
+{"timestamp":1713928481.4659791,"name":"online","context":{"idset":"8678,8684"}}
+{"timestamp":1713928482.2307971,"name":"online","context":{"idset":"8686"}}
+{"timestamp":1713928482.4263783,"name":"online","context":{"idset":"8692"}}
+{"timestamp":1713928482.6255283,"name":"online","context":{"idset":"8690"}}
+{"timestamp":1713928484.5051858,"name":"drain","context":{"idset":"11373","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713928487.2428775,"name":"drain","context":{"idset":"10341","reason":"nodediag failed bogomips","overwrite":0}}
+{"timestamp":1713928672.2596905,"name":"drain","context":{"idset":"11377","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713928725.7356539,"name":"drain","context":{"idset":"11369","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713929876.9478488,"name":"offline","context":{"idset":"11373"}}
+{"timestamp":1713929877.0346301,"name":"offline","context":{"idset":"11374"}}
+{"timestamp":1713930153.7663767,"name":"drain","context":{"idset":"11370","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713930206.9490738,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1713930207.0346057,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1713930246.2773032,"name":"online","context":{"idset":"7693"}}
+{"timestamp":1713930248.5336852,"name":"online","context":{"idset":"7761"}}
+{"timestamp":1713930635.0345428,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1713930674.8324726,"name":"online","context":{"idset":"7715"}}
+{"timestamp":1713931867.0357049,"name":"offline","context":{"idset":"11729"}}
+{"timestamp":1713931907.5243344,"name":"online","context":{"idset":"11729"}}
+{"timestamp":1713932859.035202,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1713932898.592154,"name":"online","context":{"idset":"7715"}}
+{"timestamp":1713937235.0351923,"name":"offline","context":{"idset":"934"}}
+{"timestamp":1713937313.0348175,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713937352.7266705,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1713937726.767272,"name":"online","context":{"idset":"934"}}
+{"timestamp":1713943873.0358531,"name":"offline","context":{"idset":"11825"}}
+{"timestamp":1713944433.0352366,"name":"offline","context":{"idset":"11370"}}
+{"timestamp":1713944498.6400285,"name":"online","context":{"idset":"11370"}}
+{"timestamp":1713944527.0351849,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1713944549.0346608,"name":"offline","context":{"idset":"11731"}}
+{"timestamp":1713944566.4956234,"name":"online","context":{"idset":"7719"}}
+{"timestamp":1713944592.0482678,"name":"online","context":{"idset":"11731"}}
+{"timestamp":1713944774.9511373,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1713944774.9551311,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1713944775.0351572,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1713944815.5405972,"name":"online","context":{"idset":"7761"}}
+{"timestamp":1713944816.3550992,"name":"online","context":{"idset":"7719"}}
+{"timestamp":1713944841.6127179,"name":"online","context":{"idset":"7715"}}
+{"timestamp":1713947986.9556475,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1713947987.0361314,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1713948026.6634412,"name":"online","context":{"idset":"11369"}}
+{"timestamp":1713948027.7114763,"name":"online","context":{"idset":"7715"}}
+{"timestamp":1713951581.0341721,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1713951621.2971277,"name":"online","context":{"idset":"7719"}}
+{"timestamp":1713951714.2512305,"name":"drain","context":{"idset":"11729","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713951748.2914326,"name":"drain","context":{"idset":"11721","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713951804.6859686,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1713951850.1040041,"name":"online","context":{"idset":"11369"}}
+{"timestamp":1713952201.0359488,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1713953621.2605143,"name":"drain","context":{"idset":"10310","reason":"nodediag failed bogomips","overwrite":0}}
+{"timestamp":1713954012.942399,"name":"drain","context":{"idset":"11658","reason":"epilog failed for jobid frLfYxGth6w","overwrite":0}}
+{"timestamp":1713954013.0349252,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1713954057.4994619,"name":"online","context":{"idset":"11658"}}
+{"timestamp":1713956230.5261266,"name":"drain","context":{"idset":"10872","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713957152.9506073,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1713957152.9539366,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1713957153.035064,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1713957192.2384608,"name":"online","context":{"idset":"7719"}}
+{"timestamp":1713957192.6048737,"name":"online","context":{"idset":"7715"}}
+{"timestamp":1713957194.5267334,"name":"online","context":{"idset":"7761"}}
+{"timestamp":1713959973.0349975,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1713960015.9256239,"name":"online","context":{"idset":"11658"}}
+{"timestamp":1713962505.0348277,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713962545.9807911,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1713963884.7211974,"name":"drain","context":{"idset":"939","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713965050.1786172,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1713965050.5320919,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1713965051.1189053,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1713965051.8776727,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1713965052.532285,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1713967540.7116601,"name":"online","context":{"idset":"1061-1062"}}
+{"timestamp":1713967834.9437568,"name":"drain","context":{"idset":"934","reason":"epilog failed for jobid frMYzSv7G6b","overwrite":0}}
+{"timestamp":1713967835.0337162,"name":"offline","context":{"idset":"934"}}
+{"timestamp":1713968370.9462767,"name":"offline","context":{"idset":"805"}}
+{"timestamp":1713968371.0345936,"name":"offline","context":{"idset":"806"}}
+{"timestamp":1713969828.0249221,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1713969874.4260335,"name":"online","context":{"idset":"11369"}}
+{"timestamp":1713970330.020931,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1713970354.0214965,"name":"offline","context":{"idset":"11731"}}
+{"timestamp":1713970373.9929743,"name":"online","context":{"idset":"7719"}}
+{"timestamp":1713970396.178256,"name":"online","context":{"idset":"11731"}}
+{"timestamp":1713970760.0219743,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1713970786.0210292,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1713970839.9377592,"name":"online","context":{"idset":"7693"}}
+{"timestamp":1713970968.6421256,"name":"online","context":{"idset":"8671"}}
+{"timestamp":1713970969.0044565,"name":"online","context":{"idset":"8668"}}
+{"timestamp":1713970969.1306806,"name":"online","context":{"idset":"8670,8672"}}
+{"timestamp":1713970969.3338912,"name":"online","context":{"idset":"8669,8676"}}
+{"timestamp":1713970969.5407424,"name":"online","context":{"idset":"8666"}}
+{"timestamp":1713970969.6477129,"name":"online","context":{"idset":"8673,8675"}}
+{"timestamp":1713970969.7545655,"name":"online","context":{"idset":"8661-8662,8665,8667"}}
+{"timestamp":1713970969.9474878,"name":"online","context":{"idset":"8663"}}
+{"timestamp":1713970970.1516016,"name":"online","context":{"idset":"8664"}}
+{"timestamp":1713970970.2548888,"name":"online","context":{"idset":"8674"}}
+{"timestamp":1713971241.934397,"name":"offline","context":{"idset":"11323"}}
+{"timestamp":1713971242.0212889,"name":"offline","context":{"idset":"11324"}}
+{"timestamp":1713971448.0212944,"name":"offline","context":{"idset":"819"}}
+{"timestamp":1713972414.0213256,"name":"offline","context":{"idset":"11721"}}
+{"timestamp":1713972454.2940562,"name":"online","context":{"idset":"11721"}}
+{"timestamp":1713972657.5031843,"name":"drain","context":{"idset":"11253-11268","reason":"draining KPN","overwrite":0}}
+{"timestamp":1713972687.6942582,"name":"offline","context":{"idset":"11258"}}
+{"timestamp":1713972688.0136502,"name":"offline","context":{"idset":"11253"}}
+{"timestamp":1713972688.0168214,"name":"offline","context":{"idset":"11261"}}
+{"timestamp":1713972688.0225289,"name":"offline","context":{"idset":"11260"}}
+{"timestamp":1713972688.0273397,"name":"offline","context":{"idset":"11259"}}
+{"timestamp":1713972688.0336587,"name":"offline","context":{"idset":"11266"}}
+{"timestamp":1713972688.134583,"name":"offline","context":{"idset":"11268"}}
+{"timestamp":1713972688.1589563,"name":"offline","context":{"idset":"11257"}}
+{"timestamp":1713972688.1630721,"name":"offline","context":{"idset":"11267"}}
+{"timestamp":1713972688.1671898,"name":"offline","context":{"idset":"11264"}}
+{"timestamp":1713972688.2631638,"name":"offline","context":{"idset":"11254"}}
+{"timestamp":1713972688.3560383,"name":"offline","context":{"idset":"11263"}}
+{"timestamp":1713972688.3599122,"name":"offline","context":{"idset":"11265"}}
+{"timestamp":1713972688.4590225,"name":"offline","context":{"idset":"11262"}}
+{"timestamp":1713973190.021915,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1713973378.6588573,"name":"online","context":{"idset":"934"}}
+{"timestamp":1713974318.0242918,"name":"offline","context":{"idset":"9413"}}
+{"timestamp":1713974319.9566655,"name":"offline","context":{"idset":"9414"}}
+{"timestamp":1713974319.9623053,"name":"offline","context":{"idset":"9415"}}
+{"timestamp":1713974319.9676921,"name":"offline","context":{"idset":"9416"}}
+{"timestamp":1713974320.0556583,"name":"offline","context":{"idset":"9417"}}
+{"timestamp":1713974321.9434838,"name":"offline","context":{"idset":"9418"}}
+{"timestamp":1713974321.9511869,"name":"offline","context":{"idset":"9419"}}
+{"timestamp":1713974321.9588473,"name":"offline","context":{"idset":"9420"}}
+{"timestamp":1713974322.0221767,"name":"offline","context":{"idset":"9421"}}
+{"timestamp":1713974323.938602,"name":"offline","context":{"idset":"9422"}}
+{"timestamp":1713974323.9418662,"name":"offline","context":{"idset":"9423"}}
+{"timestamp":1713974323.9451573,"name":"offline","context":{"idset":"9424"}}
+{"timestamp":1713974324.021533,"name":"offline","context":{"idset":"9425"}}
+{"timestamp":1713974325.941406,"name":"offline","context":{"idset":"9426"}}
+{"timestamp":1713974325.9500229,"name":"offline","context":{"idset":"9427"}}
+{"timestamp":1713974326.021764,"name":"offline","context":{"idset":"9428"}}
+{"timestamp":1713974339.5721209,"name":"drain","context":{"idset":"9413-9428","reason":"--reason rebooting switches - INIT -kpn","overwrite":0}}
+{"timestamp":1713975164.2883806,"name":"drain","context":{"idset":"11285-11300","reason":"--reason rebooting switches - INIT -kpn","overwrite":0}}
+{"timestamp":1713975177.9340787,"name":"offline","context":{"idset":"11285"}}
+{"timestamp":1713975178.1111755,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1713975178.1267347,"name":"offline","context":{"idset":"11290"}}
+{"timestamp":1713975178.135241,"name":"offline","context":{"idset":"11295"}}
+{"timestamp":1713975178.1388106,"name":"offline","context":{"idset":"11298"}}
+{"timestamp":1713975178.2383845,"name":"offline","context":{"idset":"11289"}}
+{"timestamp":1713975178.3285162,"name":"offline","context":{"idset":"11299"}}
+{"timestamp":1713975178.3317068,"name":"offline","context":{"idset":"11297"}}
+{"timestamp":1713975178.344059,"name":"offline","context":{"idset":"11296"}}
+{"timestamp":1713975178.3522704,"name":"offline","context":{"idset":"11300"}}
+{"timestamp":1713975178.3623395,"name":"offline","context":{"idset":"11292"}}
+{"timestamp":1713975178.3667362,"name":"offline","context":{"idset":"11291"}}
+{"timestamp":1713975178.3710923,"name":"offline","context":{"idset":"11287"}}
+{"timestamp":1713975178.3745468,"name":"offline","context":{"idset":"11288"}}
+{"timestamp":1713975178.383333,"name":"offline","context":{"idset":"11294"}}
+{"timestamp":1713975178.4922512,"name":"offline","context":{"idset":"11293"}}
+{"timestamp":1713975221.3440952,"name":"online","context":{"idset":"9194"}}
+{"timestamp":1713975221.8378694,"name":"online","context":{"idset":"9193"}}
+{"timestamp":1713975394.7897098,"name":"online","context":{"idset":"833"}}
+{"timestamp":1713975438.5571983,"name":"online","context":{"idset":"834"}}
+{"timestamp":1713975457.3852818,"name":"online","context":{"idset":"9216"}}
+{"timestamp":1713975496.758235,"name":"online","context":{"idset":"809"}}
+{"timestamp":1713975497.1405277,"name":"online","context":{"idset":"810"}}
+{"timestamp":1713975562.1897571,"name":"undrain","context":{"idset":"809"}}
+{"timestamp":1713975572.965703,"name":"undrain","context":{"idset":"810"}}
+{"timestamp":1713976896.0372002,"name":"drain","context":{"idset":"9216","reason":"Debugging downed nodes - vls","overwrite":1}}
+{"timestamp":1713976922.4203992,"name":"undrain","context":{"idset":"9216"}}
+{"timestamp":1713977008.0212169,"name":"offline","context":{"idset":"11377"}}
+{"timestamp":1713977047.8286633,"name":"online","context":{"idset":"11377"}}
+{"timestamp":1713977545.1555369,"name":"online","context":{"idset":"826"}}
+{"timestamp":1713977546.0826054,"name":"online","context":{"idset":"825"}}
+{"timestamp":1713977639.0777535,"name":"undrain","context":{"idset":"825"}}
+{"timestamp":1713977653.7177241,"name":"undrain","context":{"idset":"826"}}
+{"timestamp":1713978657.501276,"name":"online","context":{"idset":"8574,8579"}}
+{"timestamp":1713978658.0446281,"name":"online","context":{"idset":"8567"}}
+{"timestamp":1713978658.440691,"name":"online","context":{"idset":"8571"}}
+{"timestamp":1713978658.697633,"name":"online","context":{"idset":"8595"}}
+{"timestamp":1713978658.8147733,"name":"online","context":{"idset":"8591"}}
+{"timestamp":1713978658.9351673,"name":"online","context":{"idset":"8570,8573,8575,8587,8593"}}
+{"timestamp":1713978659.0426149,"name":"online","context":{"idset":"8588,8596"}}
+{"timestamp":1713978659.1669784,"name":"online","context":{"idset":"8568,8586"}}
+{"timestamp":1713978659.2887769,"name":"online","context":{"idset":"8565,8572,8583,8590"}}
+{"timestamp":1713978659.4766622,"name":"online","context":{"idset":"8569,8576,8584,8592"}}
+{"timestamp":1713978659.6607347,"name":"online","context":{"idset":"8577-8578,8582"}}
+{"timestamp":1713978659.7620244,"name":"online","context":{"idset":"8581,8594"}}
+{"timestamp":1713978659.9656911,"name":"online","context":{"idset":"8566"}}
+{"timestamp":1713978660.1424978,"name":"online","context":{"idset":"8585"}}
+{"timestamp":1713978660.7629235,"name":"online","context":{"idset":"8597"}}
+{"timestamp":1713978661.2492189,"name":"online","context":{"idset":"8598"}}
+{"timestamp":1713978669.5068998,"name":"online","context":{"idset":"8599"}}
+{"timestamp":1713978669.6655071,"name":"online","context":{"idset":"8600"}}
+{"timestamp":1713978670.4300725,"name":"online","context":{"idset":"8603"}}
+{"timestamp":1713978670.7268929,"name":"online","context":{"idset":"8607"}}
+{"timestamp":1713978671.0848429,"name":"online","context":{"idset":"8604,8606"}}
+{"timestamp":1713978671.2859828,"name":"online","context":{"idset":"8602"}}
+{"timestamp":1713978671.3975432,"name":"online","context":{"idset":"8608,8611"}}
+{"timestamp":1713978671.5844254,"name":"online","context":{"idset":"8601,8609-8610,8612"}}
+{"timestamp":1713978671.8736696,"name":"online","context":{"idset":"8605"}}
+{"timestamp":1713978780.8831599,"name":"offline","context":{"idset":"10378"}}
+{"timestamp":1713978822.8428228,"name":"online","context":{"idset":"10378"}}
+{"timestamp":1713978932.1737111,"name":"online","context":{"idset":"8633"}}
+{"timestamp":1713978933.2484009,"name":"online","context":{"idset":"8629,8632"}}
+{"timestamp":1713978933.7352252,"name":"online","context":{"idset":"8643"}}
+{"timestamp":1713978933.9966359,"name":"online","context":{"idset":"8644"}}
+{"timestamp":1713978934.1026227,"name":"online","context":{"idset":"8640"}}
+{"timestamp":1713978934.2997806,"name":"online","context":{"idset":"8638-8639,8647-8648,8655"}}
+{"timestamp":1713978934.487184,"name":"online","context":{"idset":"8637,8656"}}
+{"timestamp":1713978934.592139,"name":"online","context":{"idset":"8630-8631,8634,8641,8646,8649,8654,8659"}}
+{"timestamp":1713978934.7830412,"name":"online","context":{"idset":"8636,8650-8651,8657"}}
+{"timestamp":1713978934.9759786,"name":"online","context":{"idset":"8645,8658,8660"}}
+{"timestamp":1713978935.2715011,"name":"online","context":{"idset":"8653"}}
+{"timestamp":1713979977.049118,"name":"drain","context":{"idset":"11826","overwrite":0}}
+{"timestamp":1713980277.3085213,"name":"offline","context":{"idset":"11857"}}
+{"timestamp":1713980343.7119722,"name":"drain","context":{"idset":"10187","overwrite":0}}
+{"timestamp":1713980603.5726736,"name":"online","context":{"idset":"7821"}}
+{"timestamp":1713980603.7700684,"name":"online","context":{"idset":"7814,7828"}}
+{"timestamp":1713980603.9643183,"name":"online","context":{"idset":"7822-7823"}}
+{"timestamp":1713980604.1057124,"name":"online","context":{"idset":"7816-7819,7826"}}
+{"timestamp":1713980604.4687009,"name":"online","context":{"idset":"7815,7827"}}
+{"timestamp":1713980604.9638717,"name":"online","context":{"idset":"7813,7820,7825"}}
+{"timestamp":1713981218.0206583,"name":"offline","context":{"idset":"11729"}}
+{"timestamp":1713981258.1133544,"name":"online","context":{"idset":"11729"}}
+{"timestamp":1713981320.4010706,"name":"online","context":{"idset":"11857"}}
+{"timestamp":1713981428.4746592,"name":"undrain","context":{"idset":"11857"}}
+{"timestamp":1713981434.0254135,"name":"drain","context":{"idset":"165","reason":"broker was unresponsive"}}
+{"timestamp":1713981537.6113958,"name":"drain","context":{"idset":"117","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713981544.4127698,"name":"drain","context":{"idset":"172","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713981548.3638871,"name":"drain","context":{"idset":"120","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713981552.7912447,"name":"drain","context":{"idset":"286","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713981556.5384898,"name":"drain","context":{"idset":"335","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713981560.2453289,"name":"drain","context":{"idset":"4538","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713981565.5230312,"name":"drain","context":{"idset":"342","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713981572.6249416,"name":"undrain","context":{"idset":"4538"}}
+{"timestamp":1713981578.6009247,"name":"drain","context":{"idset":"360","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713981581.3898735,"name":"drain","context":{"idset":"372","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713981697.619077,"name":"drain","context":{"idset":"373","reason":"Unreachable","overwrite":0}}
+{"timestamp":1713981808.1331735,"name":"online","context":{"idset":"8201"}}
+{"timestamp":1713981956.0230765,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1713981996.464642,"name":"online","context":{"idset":"7693"}}
+{"timestamp":1713982120.0826187,"name":"online","context":{"idset":"7808,7835,7875"}}
+{"timestamp":1713982120.3910141,"name":"online","context":{"idset":"7839"}}
+{"timestamp":1713982120.5857799,"name":"online","context":{"idset":"7805,7842"}}
+{"timestamp":1713982120.9220755,"name":"online","context":{"idset":"7830,7845"}}
+{"timestamp":1713982122.3753452,"name":"online","context":{"idset":"7856"}}
+{"timestamp":1713982123.0969205,"name":"online","context":{"idset":"7850"}}
+{"timestamp":1713982123.2600441,"name":"online","context":{"idset":"7810,7834"}}
+{"timestamp":1713982123.3661463,"name":"online","context":{"idset":"7807"}}
+{"timestamp":1713982123.687793,"name":"online","context":{"idset":"7800"}}
+{"timestamp":1713982123.7857769,"name":"online","context":{"idset":"7812"}}
+{"timestamp":1713982123.8848345,"name":"online","context":{"idset":"7877"}}
+{"timestamp":1713982123.9970539,"name":"online","context":{"idset":"7885"}}
+{"timestamp":1713982124.5114341,"name":"online","context":{"idset":"7811"}}
+{"timestamp":1713982124.624084,"name":"online","context":{"idset":"7804"}}
+{"timestamp":1713982124.730252,"name":"online","context":{"idset":"7854"}}
+{"timestamp":1713982124.8451977,"name":"online","context":{"idset":"7832"}}
+{"timestamp":1713982124.9556952,"name":"online","context":{"idset":"7846"}}
+{"timestamp":1713982125.1468124,"name":"online","context":{"idset":"7864"}}
+{"timestamp":1713982125.5778708,"name":"online","context":{"idset":"7798,7841,7869"}}
+{"timestamp":1713982125.5983331,"name":"online","context":{"idset":"7799"}}
+{"timestamp":1713982125.7298334,"name":"online","context":{"idset":"7844"}}
+{"timestamp":1713982125.8518212,"name":"online","context":{"idset":"7797"}}
+{"timestamp":1713982125.962779,"name":"online","context":{"idset":"7886,7917"}}
+{"timestamp":1713982126.086844,"name":"online","context":{"idset":"7904"}}
+{"timestamp":1713982126.2741504,"name":"online","context":{"idset":"7840,7876,7901"}}
+{"timestamp":1713982126.4791913,"name":"online","context":{"idset":"7801,7831,7848,7853,7857,7865,7874,7878"}}
+{"timestamp":1713982126.5830142,"name":"online","context":{"idset":"7849,7871,7880,7887,7900"}}
+{"timestamp":1713982126.7820101,"name":"online","context":{"idset":"7802,7866,7895,7911,7914"}}
+{"timestamp":1713982126.9763165,"name":"online","context":{"idset":"7803,7843,7847,7852,7863,7889,7903,7913,7922"}}
+{"timestamp":1713982127.0906911,"name":"online","context":{"idset":"7806,7861,7873,7881,7897,7908"}}
+{"timestamp":1713982127.2832866,"name":"online","context":{"idset":"7809,7833,7837,7855,7890,7894,7899,7912"}}
+{"timestamp":1713982127.3864319,"name":"online","context":{"idset":"7836,7870,7896,7898"}}
+{"timestamp":1713982127.4941757,"name":"online","context":{"idset":"7868"}}
+{"timestamp":1713982127.5970886,"name":"online","context":{"idset":"7838,7858,7872,7882-7883,7907,7916,7921"}}
+{"timestamp":1713982127.7812579,"name":"online","context":{"idset":"7879,7888,7891-7893,7910,7918"}}
+{"timestamp":1713982127.884145,"name":"online","context":{"idset":"7867,7905-7906,7915,7924"}}
+{"timestamp":1713982128.0951302,"name":"online","context":{"idset":"7829,7851,7862,7919,7923"}}
+{"timestamp":1713982128.3945735,"name":"online","context":{"idset":"7859-7860,7884,7902,7909,7920"}}
+{"timestamp":1713982738.0258427,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713982740.0206497,"name":"offline","context":{"idset":"11731"}}
+{"timestamp":1713982779.9535737,"name":"online","context":{"idset":"11731"}}
+{"timestamp":1713982780.788698,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1713983852.2899179,"name":"drain","context":{"idset":"11717-11720,11722-11724,11726,11728,11730,11732","reason":"--force --reason rebooting switches - INIT -kpn","overwrite":0}}
+{"timestamp":1713983875.5440009,"name":"offline","context":{"idset":"11718"}}
+{"timestamp":1713983875.6376917,"name":"offline","context":{"idset":"11724"}}
+{"timestamp":1713983875.6446459,"name":"offline","context":{"idset":"11729"}}
+{"timestamp":1713983875.6483469,"name":"offline","context":{"idset":"11726"}}
+{"timestamp":1713983875.651587,"name":"offline","context":{"idset":"11722"}}
+{"timestamp":1713983875.654819,"name":"offline","context":{"idset":"11723"}}
+{"timestamp":1713983875.75246,"name":"offline","context":{"idset":"11717"}}
+{"timestamp":1713983875.8495436,"name":"offline","context":{"idset":"11732"}}
+{"timestamp":1713983875.8527844,"name":"offline","context":{"idset":"11721"}}
+{"timestamp":1713983875.8560247,"name":"offline","context":{"idset":"11728"}}
+{"timestamp":1713983875.8608286,"name":"offline","context":{"idset":"11719"}}
+{"timestamp":1713983875.8640766,"name":"offline","context":{"idset":"11720"}}
+{"timestamp":1713983875.8812191,"name":"offline","context":{"idset":"11730"}}
+{"timestamp":1713983875.9707811,"name":"offline","context":{"idset":"11731"}}
+{"timestamp":1713983888.6762266,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1713983888.6877182,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1713983888.7873702,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1713983888.7915592,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1713983888.7989557,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1713983888.8022375,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1713983888.8950973,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1713983888.8991163,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1713983888.9028335,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1713983888.9119093,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1713983889.007658,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1713983889.0141535,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1713983889.0174496,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1713983889.113152,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1713984022.0223844,"name":"offline","context":{"idset":"11725"}}
+{"timestamp":1713984539.9351609,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1713984540.0219829,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1713984581.7881403,"name":"online","context":{"idset":"7693"}}
+{"timestamp":1713984604.8172634,"name":"online","context":{"idset":"7761"}}
+{"timestamp":1713984672.021672,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1713984711.4005969,"name":"online","context":{"idset":"11369"}}
+{"timestamp":1713986608.0214212,"name":"offline","context":{"idset":"11826"}}
+{"timestamp":1713986692.025279,"name":"offline","context":{"idset":"7823"}}
+{"timestamp":1713986816.1609867,"name":"offline","context":{"idset":"9541"}}
+{"timestamp":1713986816.1658697,"name":"offline","context":{"idset":"9542"}}
+{"timestamp":1713986839.934917,"name":"offline","context":{"idset":"767"}}
+{"timestamp":1713986840.0226574,"name":"offline","context":{"idset":"768"}}
+{"timestamp":1713988319.0342031,"name":"online","context":{"idset":"8833"}}
+{"timestamp":1713988437.1831586,"name":"online","context":{"idset":"8834"}}
+{"timestamp":1713988470.8125949,"name":"online","context":{"idset":"8842"}}
+{"timestamp":1713988548.0222647,"name":"offline","context":{"idset":"8201"}}
+{"timestamp":1713988590.1685307,"name":"online","context":{"idset":"8883"}}
+{"timestamp":1713988655.7870061,"name":"online","context":{"idset":"8884"}}
+{"timestamp":1713988810.1959062,"name":"online","context":{"idset":"8886"}}
+{"timestamp":1713988868.6186252,"name":"online","context":{"idset":"8896"}}
+{"timestamp":1713988915.2490942,"name":"online","context":{"idset":"8902"}}
+{"timestamp":1713988954.1058891,"name":"online","context":{"idset":"8941"}}
+{"timestamp":1713990896.8189778,"name":"online","context":{"idset":"7823"}}
+{"timestamp":1713990947.6505928,"name":"online","context":{"idset":"7824"}}
+{"timestamp":1713991536.020798,"name":"offline","context":{"idset":"980"}}
+{"timestamp":1713991775.946907,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1713991775.9534266,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1713991776.023838,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1713991800.0205011,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1713991815.611542,"name":"online","context":{"idset":"7715"}}
+{"timestamp":1713991816.6356783,"name":"online","context":{"idset":"7693"}}
+{"timestamp":1713991820.4532008,"name":"online","context":{"idset":"7719"}}
+{"timestamp":1713991869.0561922,"name":"drain","context":{"idset":"8833","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1713991872.6267755,"name":"online","context":{"idset":"11369"}}
+{"timestamp":1713991886.020638,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1713991926.2828381,"name":"online","context":{"idset":"11658"}}
+{"timestamp":1713991978.0245416,"name":"offline","context":{"idset":"7815"}}
+{"timestamp":1713992004.6642532,"name":"offline","context":{"idset":"10378"}}
+{"timestamp":1713992045.8539214,"name":"online","context":{"idset":"10378"}}
+{"timestamp":1713992322.756424,"name":"offline","context":{"idset":"8836"}}
+{"timestamp":1713992322.8590682,"name":"offline","context":{"idset":"8832"}}
+{"timestamp":1713992322.9510598,"name":"offline","context":{"idset":"8835"}}
+{"timestamp":1713992322.9607227,"name":"offline","context":{"idset":"8830"}}
+{"timestamp":1713992322.9673085,"name":"offline","context":{"idset":"8834"}}
+{"timestamp":1713992323.0650914,"name":"offline","context":{"idset":"8829"}}
+{"timestamp":1713992323.1958506,"name":"offline","context":{"idset":"8831"}}
+{"timestamp":1713992323.5321946,"name":"offline","context":{"idset":"8833"}}
+{"timestamp":1713992372.6008196,"name":"online","context":{"idset":"8830"}}
+{"timestamp":1713992372.6042724,"name":"online","context":{"idset":"8832,8836"}}
+{"timestamp":1713992372.6076913,"name":"online","context":{"idset":"8835"}}
+{"timestamp":1713992373.1479688,"name":"online","context":{"idset":"8829,8831"}}
+{"timestamp":1713992378.6342278,"name":"online","context":{"idset":"8834"}}
+{"timestamp":1713992475.40552,"name":"online","context":{"idset":"837"}}
+{"timestamp":1713992554.5801034,"name":"online","context":{"idset":"838"}}
+{"timestamp":1713992566.0220287,"name":"offline","context":{"idset":"934"}}
+{"timestamp":1713992571.93839,"name":"offline","context":{"idset":"939"}}
+{"timestamp":1713992572.0227051,"name":"offline","context":{"idset":"940"}}
+{"timestamp":1713992598.0215328,"name":"offline","context":{"idset":"10378"}}
+{"timestamp":1713992641.3771513,"name":"online","context":{"idset":"10378"}}
+{"timestamp":1713993007.9351523,"name":"offline","context":{"idset":"825"}}
+{"timestamp":1713993008.0233638,"name":"offline","context":{"idset":"826"}}
+{"timestamp":1713993025.9341683,"name":"offline","context":{"idset":"833"}}
+{"timestamp":1713993026.0222838,"name":"offline","context":{"idset":"834"}}
+{"timestamp":1713993030.0218511,"name":"offline","context":{"idset":"836"}}
+{"timestamp":1713993472.9415767,"name":"online","context":{"idset":"8960"}}
+{"timestamp":1713993473.1258521,"name":"online","context":{"idset":"8952,8968"}}
+{"timestamp":1713993473.4129937,"name":"online","context":{"idset":"8949"}}
+{"timestamp":1713993473.5108697,"name":"online","context":{"idset":"8955,8963"}}
+{"timestamp":1713993474.1195138,"name":"online","context":{"idset":"8961"}}
+{"timestamp":1713993474.3867786,"name":"online","context":{"idset":"8956,8964,8966"}}
+{"timestamp":1713993474.6903262,"name":"online","context":{"idset":"8983"}}
+{"timestamp":1713993474.850328,"name":"online","context":{"idset":"8970"}}
+{"timestamp":1713993475.3923712,"name":"online","context":{"idset":"8967"}}
+{"timestamp":1713993475.5473998,"name":"online","context":{"idset":"8959"}}
+{"timestamp":1713993475.6393192,"name":"online","context":{"idset":"8971"}}
+{"timestamp":1713993475.8538814,"name":"online","context":{"idset":"8950"}}
+{"timestamp":1713993476.1632924,"name":"online","context":{"idset":"8953"}}
+{"timestamp":1713993476.2760305,"name":"online","context":{"idset":"8958"}}
+{"timestamp":1713993476.4486639,"name":"online","context":{"idset":"8954,8985"}}
+{"timestamp":1713993476.7196367,"name":"online","context":{"idset":"8951,9022"}}
+{"timestamp":1713993477.0235732,"name":"online","context":{"idset":"8993"}}
+{"timestamp":1713993477.1368895,"name":"online","context":{"idset":"8990,9006"}}
+{"timestamp":1713993477.334492,"name":"online","context":{"idset":"9018"}}
+{"timestamp":1713993477.6242876,"name":"online","context":{"idset":"8980,9046,9058"}}
+{"timestamp":1713993477.8446331,"name":"online","context":{"idset":"8974,8977,8979,9001,9033,9038,9049,9076"}}
+{"timestamp":1713993477.9470098,"name":"online","context":{"idset":"8978"}}
+{"timestamp":1713993478.1403334,"name":"online","context":{"idset":"8957,8962,8972-8973,8975,9017,9051,9059-9060"}}
+{"timestamp":1713993478.4402971,"name":"online","context":{"idset":"8965,8969,8986,9009,9045,9052"}}
+{"timestamp":1713993478.6362131,"name":"online","context":{"idset":"8988,9012,9050,9053"}}
+{"timestamp":1713993478.7379227,"name":"online","context":{"idset":"9005,9041,9048"}}
+{"timestamp":1713993478.9285569,"name":"online","context":{"idset":"8976,8992,9010,9020,9026"}}
+{"timestamp":1713993479.0308917,"name":"online","context":{"idset":"8994,9013,9047,9061"}}
+{"timestamp":1713993479.1493492,"name":"online","context":{"idset":"8991,9055"}}
+{"timestamp":1713993479.2571232,"name":"online","context":{"idset":"8981,9019,9023,9029,9056-9057,9072"}}
+{"timestamp":1713993479.4475536,"name":"online","context":{"idset":"8984,9002,9007-9008,9014,9016,9021,9028,9031-9032,9063,9067,9070"}}
+{"timestamp":1713993479.5508432,"name":"online","context":{"idset":"9062"}}
+{"timestamp":1713993479.6601841,"name":"online","context":{"idset":"8987,8989,8995-8997,9015,9024-9025,9068"}}
+{"timestamp":1713993479.7643774,"name":"online","context":{"idset":"9004,9027,9034,9037,9039"}}
+{"timestamp":1713993479.9474058,"name":"online","context":{"idset":"8998,9011,9030,9044"}}
+{"timestamp":1713993480.1446569,"name":"online","context":{"idset":"8999-9000,9035-9036,9040,9042"}}
+{"timestamp":1713993563.9462037,"name":"offline","context":{"idset":"11269"}}
+{"timestamp":1713993563.9542603,"name":"offline","context":{"idset":"11270"}}
+{"timestamp":1713993563.962261,"name":"offline","context":{"idset":"11271"}}
+{"timestamp":1713993564.0217187,"name":"offline","context":{"idset":"11272"}}
+{"timestamp":1713993565.9432962,"name":"offline","context":{"idset":"11273"}}
+{"timestamp":1713993565.9472847,"name":"offline","context":{"idset":"11274"}}
+{"timestamp":1713993565.950778,"name":"offline","context":{"idset":"11275"}}
+{"timestamp":1713993566.0223799,"name":"offline","context":{"idset":"11276"}}
+{"timestamp":1713993567.9437575,"name":"offline","context":{"idset":"11277"}}
+{"timestamp":1713993567.9474831,"name":"offline","context":{"idset":"11278"}}
+{"timestamp":1713993567.9512198,"name":"offline","context":{"idset":"11279"}}
+{"timestamp":1713993567.9548595,"name":"offline","context":{"idset":"11280"}}
+{"timestamp":1713993568.021486,"name":"offline","context":{"idset":"11281"}}
+{"timestamp":1713993569.9397008,"name":"offline","context":{"idset":"11282"}}
+{"timestamp":1713993569.9441853,"name":"offline","context":{"idset":"11283"}}
+{"timestamp":1713993570.022629,"name":"offline","context":{"idset":"11284"}}
+{"timestamp":1713993577.9491537,"name":"offline","context":{"idset":"11301"}}
+{"timestamp":1713993577.9524925,"name":"offline","context":{"idset":"11302"}}
+{"timestamp":1713993577.9557757,"name":"offline","context":{"idset":"11303"}}
+{"timestamp":1713993577.9594371,"name":"offline","context":{"idset":"11304"}}
+{"timestamp":1713993578.0290244,"name":"offline","context":{"idset":"11305"}}
+{"timestamp":1713993580.2538478,"name":"offline","context":{"idset":"11306"}}
+{"timestamp":1713993580.2571867,"name":"offline","context":{"idset":"11307"}}
+{"timestamp":1713993580.2605181,"name":"offline","context":{"idset":"11308"}}
+{"timestamp":1713993581.9387426,"name":"offline","context":{"idset":"11309"}}
+{"timestamp":1713993581.9422724,"name":"offline","context":{"idset":"11310"}}
+{"timestamp":1713993582.0223184,"name":"offline","context":{"idset":"11312"}}
+{"timestamp":1713993583.936981,"name":"offline","context":{"idset":"11313"}}
+{"timestamp":1713993583.940846,"name":"offline","context":{"idset":"11314"}}
+{"timestamp":1713993583.9449294,"name":"offline","context":{"idset":"11315"}}
+{"timestamp":1713993584.02163,"name":"offline","context":{"idset":"11316"}}
+{"timestamp":1713993585.9460227,"name":"offline","context":{"idset":"11317"}}
+{"timestamp":1713993585.9519994,"name":"offline","context":{"idset":"11318"}}
+{"timestamp":1713993585.9576542,"name":"offline","context":{"idset":"11319"}}
+{"timestamp":1713993586.0723126,"name":"offline","context":{"idset":"11320"}}
+{"timestamp":1713993587.940871,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1713993588.0233738,"name":"offline","context":{"idset":"11322"}}
+{"timestamp":1713993611.9360685,"name":"offline","context":{"idset":"777"}}
+{"timestamp":1713993612.0242357,"name":"offline","context":{"idset":"778"}}
+{"timestamp":1713993632.0214117,"name":"offline","context":{"idset":"11311"}}
+{"timestamp":1713993641.9419343,"name":"offline","context":{"idset":"11325"}}
+{"timestamp":1713993641.9459324,"name":"offline","context":{"idset":"11326"}}
+{"timestamp":1713993641.9498258,"name":"offline","context":{"idset":"11327"}}
+{"timestamp":1713993642.0219126,"name":"offline","context":{"idset":"11328"}}
+{"timestamp":1713993643.9590311,"name":"offline","context":{"idset":"11329"}}
+{"timestamp":1713993643.9630802,"name":"offline","context":{"idset":"11330"}}
+{"timestamp":1713993644.0321801,"name":"offline","context":{"idset":"11331"}}
+{"timestamp":1713993645.9563775,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1713993645.962214,"name":"offline","context":{"idset":"11333"}}
+{"timestamp":1713993645.9661722,"name":"offline","context":{"idset":"11334"}}
+{"timestamp":1713993645.9700496,"name":"offline","context":{"idset":"11335"}}
+{"timestamp":1713993646.0541339,"name":"offline","context":{"idset":"11336"}}
+{"timestamp":1713993647.9409633,"name":"offline","context":{"idset":"11337"}}
+{"timestamp":1713993647.9438298,"name":"offline","context":{"idset":"11338"}}
+{"timestamp":1713993647.9463897,"name":"offline","context":{"idset":"11339"}}
+{"timestamp":1713993648.0196812,"name":"offline","context":{"idset":"11340"}}
+{"timestamp":1713993649.9525785,"name":"offline","context":{"idset":"11341"}}
+{"timestamp":1713993649.9563239,"name":"offline","context":{"idset":"11342"}}
+{"timestamp":1713993649.9597909,"name":"offline","context":{"idset":"11343"}}
+{"timestamp":1713993650.0225289,"name":"offline","context":{"idset":"11344"}}
+{"timestamp":1713993651.9404719,"name":"offline","context":{"idset":"11345"}}
+{"timestamp":1713993651.9442756,"name":"offline","context":{"idset":"11346"}}
+{"timestamp":1713993652.0215542,"name":"offline","context":{"idset":"11347"}}
+{"timestamp":1713993653.9469819,"name":"offline","context":{"idset":"11348"}}
+{"timestamp":1713993653.9536459,"name":"offline","context":{"idset":"11349"}}
+{"timestamp":1713993653.9602568,"name":"offline","context":{"idset":"11350"}}
+{"timestamp":1713993653.9668665,"name":"offline","context":{"idset":"11351"}}
+{"timestamp":1713993654.040447,"name":"offline","context":{"idset":"11352"}}
+{"timestamp":1713993655.9427114,"name":"offline","context":{"idset":"11353"}}
+{"timestamp":1713993655.946825,"name":"offline","context":{"idset":"11354"}}
+{"timestamp":1713993655.9508929,"name":"offline","context":{"idset":"11355"}}
+{"timestamp":1713993656.0218565,"name":"offline","context":{"idset":"11356"}}
+{"timestamp":1713993657.94403,"name":"offline","context":{"idset":"11357"}}
+{"timestamp":1713993657.9473381,"name":"offline","context":{"idset":"11358"}}
+{"timestamp":1713993657.9506247,"name":"offline","context":{"idset":"11359"}}
+{"timestamp":1713993657.9538939,"name":"offline","context":{"idset":"11360"}}
+{"timestamp":1713993658.021183,"name":"offline","context":{"idset":"11361"}}
+{"timestamp":1713993659.9513166,"name":"offline","context":{"idset":"11362"}}
+{"timestamp":1713993659.958396,"name":"offline","context":{"idset":"11363"}}
+{"timestamp":1713993659.9654036,"name":"offline","context":{"idset":"11364"}}
+{"timestamp":1713993659.9721684,"name":"offline","context":{"idset":"11365"}}
+{"timestamp":1713993660.0477126,"name":"offline","context":{"idset":"11366"}}
+{"timestamp":1713993661.9393077,"name":"offline","context":{"idset":"11367"}}
+{"timestamp":1713993661.9429047,"name":"offline","context":{"idset":"11368"}}
+{"timestamp":1713993661.9465783,"name":"offline","context":{"idset":"11369"}}
+{"timestamp":1713993662.0255322,"name":"offline","context":{"idset":"11371"}}
+{"timestamp":1713993663.9347157,"name":"offline","context":{"idset":"11372"}}
+{"timestamp":1713993664.0226808,"name":"offline","context":{"idset":"11375"}}
+{"timestamp":1713993665.9407351,"name":"offline","context":{"idset":"11376"}}
+{"timestamp":1713993665.9442654,"name":"offline","context":{"idset":"11377"}}
+{"timestamp":1713993665.9476969,"name":"offline","context":{"idset":"11378"}}
+{"timestamp":1713993665.9510465,"name":"offline","context":{"idset":"11379"}}
+{"timestamp":1713993666.0217502,"name":"offline","context":{"idset":"11380"}}
+{"timestamp":1713993708.0226529,"name":"offline","context":{"idset":"803"}}
+{"timestamp":1713993714.0214324,"name":"offline","context":{"idset":"11370"}}
+{"timestamp":1713993875.9410369,"name":"offline","context":{"idset":"809"}}
+{"timestamp":1713993875.9449081,"name":"offline","context":{"idset":"810"}}
+{"timestamp":1713993875.9531479,"name":"offline","context":{"idset":"815"}}
+{"timestamp":1713993876.022655,"name":"offline","context":{"idset":"816"}}
+{"timestamp":1713994092.0220313,"name":"offline","context":{"idset":"838"}}
+{"timestamp":1713994140.0219221,"name":"offline","context":{"idset":"837"}}
+{"timestamp":1713994247.9313717,"name":"offline","context":{"idset":"883"}}
+{"timestamp":1713994248.0200818,"name":"offline","context":{"idset":"884"}}
+{"timestamp":1713994773.9400942,"name":"offline","context":{"idset":"1061"}}
+{"timestamp":1713994774.0224853,"name":"offline","context":{"idset":"1062"}}
+{"timestamp":1713996208.0264785,"name":"drain","context":{"idset":"8982","reason":"broker was unresponsive"}}
+{"timestamp":1713996476.3225846,"name":"online","context":{"idset":"8982"}}
+{"timestamp":1713996857.0682576,"name":"offline","context":{"idset":"7669"}}
+{"timestamp":1713997024.0217688,"name":"offline","context":{"idset":"8982"}}
+{"timestamp":1713997056.8277807,"name":"online","context":{"idset":"8982"}}
+{"timestamp":1713997411.8487368,"name":"online","context":{"idset":"8635"}}
+{"timestamp":1713997412.426276,"name":"online","context":{"idset":"8652"}}
+{"timestamp":1713997747.9993882,"name":"online","context":{"idset":"7669"}}
+{"timestamp":1713997921.6065016,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1713998456.7942622,"name":"online","context":{"idset":"9541"}}
+{"timestamp":1713998532.7441065,"name":"online","context":{"idset":"9542"}}
+{"timestamp":1713999080.0497062,"name":"undrain","context":{"idset":"10872"}}
+{"timestamp":1713999306.7902467,"name":"online","context":{"idset":"10870"}}
+{"timestamp":1713999306.7944481,"name":"online","context":{"idset":"10876,10883"}}
+{"timestamp":1713999306.798332,"name":"online","context":{"idset":"10878"}}
+{"timestamp":1713999306.8020718,"name":"online","context":{"idset":"10874,10884"}}
+{"timestamp":1713999306.8058574,"name":"online","context":{"idset":"10873,10875"}}
+{"timestamp":1713999306.8902674,"name":"online","context":{"idset":"10871,10877,10882"}}
+{"timestamp":1713999307.0886064,"name":"online","context":{"idset":"10869,10879-10880"}}
+{"timestamp":1713999307.3896823,"name":"online","context":{"idset":"10881"}}
+{"timestamp":1713999850.6839352,"name":"offline","context":{"idset":"7816"}}
+{"timestamp":1713999993.5726602,"name":"online","context":{"idset":"11537-11538"}}
+{"timestamp":1714002555.3519673,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1714002594.3442559,"name":"online","context":{"idset":"7761"}}
+{"timestamp":1714003302.0237002,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1714003929.666765,"name":"online","context":{"idset":"91"}}
+{"timestamp":1714004710.4968286,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1714004711.4024286,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1714004761.7710586,"name":"online","context":{"idset":"7761"}}
+{"timestamp":1714004761.9082077,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1714004762.5349841,"name":"drain","context":{"idset":"10740","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1714004765.3008771,"name":"online","context":{"idset":"7715"}}
+{"timestamp":1714004796.2721143,"name":"online","context":{"idset":"7693"}}
+{"timestamp":1714005927.2861414,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1714005927.4147067,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1714005964.3157346,"name":"online","context":{"idset":"9365"}}
+{"timestamp":1714005965.5543749,"name":"online","context":{"idset":"9415,9424"}}
+{"timestamp":1714005965.5585687,"name":"online","context":{"idset":"9366"}}
+{"timestamp":1714005965.5626848,"name":"online","context":{"idset":"9359,9427-9428"}}
+{"timestamp":1714005965.5666385,"name":"online","context":{"idset":"9418-9419,9421,9423"}}
+{"timestamp":1714005965.5704122,"name":"online","context":{"idset":"9416-9417,9420"}}
+{"timestamp":1714005965.5741699,"name":"online","context":{"idset":"9413,9422,9426"}}
+{"timestamp":1714005965.8080964,"name":"online","context":{"idset":"9414,9425"}}
+{"timestamp":1714006013.7895868,"name":"undrain","context":{"idset":"9413-9428"}}
+{"timestamp":1714012868.7307026,"name":"resource-init","context":{"restart":true,"drain":{"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"11377":{"timestamp":1713928672.0,"reason":"nodediag failed amdapu"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"796":{"timestamp":1712868054.0,"reason":"broker was unresponsive"},"372":{"timestamp":1713981581.0,"reason":"Unreachable"},"877-878":{"timestamp":1713539313.0,"reason":""},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"},"795":{"timestamp":1712868053.0,"reason":"broker was unresponsive"},"336,491":{"timestamp":1713230524.0,"reason":"Down"},"90":{"timestamp":1713564981.0,"reason":"Unreachable"},"929-930":{"timestamp":1713552018.0,"reason":""},"773-774":{"timestamp":1713802805.0,"reason":""},"373":{"timestamp":1713981698.0,"reason":"Unreachable"},"77-84":{"timestamp":1713795613.0,"reason":"New blade installation"},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"11826":{"timestamp":1713979977.0,"reason":""},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"117":{"timestamp":1713981538.0,"reason":"Unreachable"},"411":{"timestamp":1713564989.0,"reason":"Unreachable"},"11253-11268":{"timestamp":1713972658.0,"reason":"draining KPN"},"807":{"timestamp":1713376017.0,"reason":""},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"811-812":{"timestamp":1713557753.0,"reason":"--reason Running hpe diags - JM"},"10310":{"timestamp":1713953621.0,"reason":"nodediag failed bogomips"},"10341":{"timestamp":1713928487.0,"reason":"nodediag failed bogomips"},"779-780":{"timestamp":1713907642.0,"reason":"--reason Troubleshoot BIOS issue"},"11725":{"timestamp":1713892052.0,"reason":"nodediag failed amdapu"},"11285-11300":{"timestamp":1713975164.0,"reason":"--reason rebooting switches - INIT -kpn"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"11658":{"timestamp":1713954013.0,"reason":"epilog failed for jobid frLfYxGth6w"},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"828":{"timestamp":1712859028.0,"reason":"broker was unresponsive"},"286":{"timestamp":1713981553.0,"reason":"Unreachable"},"808":{"timestamp":1713455939.0,"reason":""},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"10740":{"timestamp":1714004763.0,"reason":"nodediag failed amdapu"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"11374":{"timestamp":1713926801.0,"reason":"nodediag failed amdapu"},"206":{"timestamp":1713564985.0,"reason":"Unreachable"},"8181,8183,8234,8237,8291,8304":{"timestamp":1713810085.0,"reason":"testing -kk"},"11373":{"timestamp":1713928485.0,"reason":"nodediag failed amdapu"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"141-148":{"timestamp":1713545597.0,"reason":"ARP Storm Problem testing"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"9075":{"timestamp":1712864130.0,"reason":"broker was unresponsive"},"11370":{"timestamp":1713930154.0,"reason":"nodediag failed amdapu"},"573-580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"149-156":{"timestamp":1713545634.0,"reason":"ARP Storm Problem testing"},"165":{"timestamp":1713981434.0,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261863.0,"reason":"--reason swapping blades into x1900 - wendy"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"11717-11720,11722-11724,11726,11728,11730,11732":{"timestamp":1713983852.0,"reason":"--force --reason rebooting switches - INIT -kpn"},"172":{"timestamp":1713981544.0,"reason":"Unreachable"},"11324":{"timestamp":1713902339.0,"reason":"nodediag failed amdapu"},"11789":{"timestamp":1713910767.0,"reason":"testing -kk"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"787-788":{"timestamp":1713219302.0,"reason":""},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"10233,10248":{"timestamp":1713746450.0,"reason":"broker was unresponsive"},"7781":{"timestamp":1713884596.0,"reason":"nodediag failed amdapu"},"963-964":{"timestamp":1713196418.0,"reason":"moving blade"},"397":{"timestamp":1713229804.0,"reason":"prolog failed for jobid fpkPrjR8F59"},"937-938":{"timestamp":1712866351.0,"reason":""},"10187":{"timestamp":1713980344.0,"reason":""},"360":{"timestamp":1713981579.0,"reason":"Unreachable"},"925-926":{"timestamp":1713540640.0,"reason":""},"461-476":{"timestamp":1713794797.0,"reason":"New Blade Installation --JRG"},"9003":{"timestamp":1712873964.0,"reason":"nodediag failed dgemm_perf"},"421-460":{"timestamp":1713794410.0,"reason":"New Blade Installation --JRG"},"410":{"timestamp":1713378635.0,"reason":"prolog failed for jobid fq5jtQQSQnT"},"120":{"timestamp":1713981548.0,"reason":"Unreachable"},"803-804":{"timestamp":1713479733.0,"reason":"status"},"789-794":{"timestamp":1711461777.0,"reason":""},"11729":{"timestamp":1713951714.0,"reason":"nodediag failed amdapu"},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"961-962":{"timestamp":1713197674.0,"reason":""},"827":{"timestamp":1712859030.0,"reason":"broker was unresponsive"},"482-490,492":{"timestamp":1713915346.0,"reason":"New Blade Installation --JRG"},"11369":{"timestamp":1713928726.0,"reason":"nodediag failed amdapu"},"342":{"timestamp":1713981566.0,"reason":"Unreachable"},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"549-556":{"timestamp":1713553592.0,"reason":"New Blade Installation"},"8833":{"timestamp":1713991869.0,"reason":"nodediag failed amdapu"},"11721":{"timestamp":1713951748.0,"reason":"nodediag failed amdapu"},"9054":{"timestamp":1712864116.0,"reason":"broker was unresponsive"},"771-772":{"timestamp":1713195979.0,"reason":""},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"8982":{"timestamp":1713996208.0,"reason":"broker was unresponsive"},"939":{"timestamp":1713963885.0,"reason":"nodediag failed amdapu"},"547":{"timestamp":1713563916.0,"reason":"epilog failed for jobid fqUWmieM3AX"},"814":{"timestamp":1712783332.0,"reason":""},"133-140":{"timestamp":1713545617.0,"reason":"ARP Storm Problem testing"},"829-830":{"timestamp":1712700945.0,"reason":""},"525-532":{"timestamp":1713290620.0,"reason":""},"548":{"timestamp":1713563915.0,"reason":"epilog failed for jobid fqUWkS8j8hM"},"11731":{"timestamp":1713889305.0,"reason":"broker was unresponsive"},"335":{"timestamp":1713981557.0,"reason":"Unreachable"},"934":{"timestamp":1713967835.0,"reason":"epilog failed for jobid frMYzSv7G6b"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1714012868.7489319,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1714012876.305932,"name":"online","context":{"idset":"0"}}
+{"timestamp":1714012878.0931406,"name":"online","context":{"idset":"8181,8566,8568,8574,8577-8578,8629-8630,8632,8830,8969,8972,9348,9372"}}
+{"timestamp":1714012878.1948316,"name":"online","context":{"idset":"7715,8183,8205,8565,8569-8573,8576,8580,8589,8631,8633,8635,8652,8832,8836,8965-8968,8970-8971,9346,9357,9359,9378,9396,9429,9432,9447,9456,10248"}}
+{"timestamp":1714012878.3777521,"name":"online","context":{"idset":"7669,7693,7719,7761,7781,8197-8200,8202-8204,8206-8212,8291,8575,8829,8831,8834-8835,8957-8958,8961,8973,8975-8976,8978,9045,9048,9051-9052,9333,9342,9344-9345,9350,9352-9353,9363,9370,9376,9380,9406,9415,9421,9423,9431,9435,9444-9446,9448-9450,9452,9458-9459,10233,10378,11658"}}
+{"timestamp":1714012878.4795666,"name":"online","context":{"idset":"8959-8960,8963-8964,8974,8977,8980,9047,9049-9050,9385,9405"}}
+{"timestamp":1714012878.5818982,"name":"online","context":{"idset":"8962,9339,9351,9358,9373,9375,9392,9400,9403,9408,9411,9414,9419,9425,9430,9451,9460"}}
+{"timestamp":1714012878.7649658,"name":"online","context":{"idset":"85-86,9334-9338,9341,9349,9354,9356,9360,9362,9366-9368,9371,9374,9377,9379,9381-9384,9386,9388,9390,9393,9395,9397,9399,9401-9402,9412-9413,9416-9418,9420,9422,9424,9426-9427,9434,9436-9437,9439,9441-9442,9453,9455,9457"}}
+{"timestamp":1714012878.9506409,"name":"online","context":{"idset":"9340,9347,9364,9387,9389,9391,9394,9398,9407,9410,9428,9433,9438,9440,9454"}}
+{"timestamp":1714012879.1452641,"name":"online","context":{"idset":"169,239,9055,9060,10136,10333,10801"}}
+{"timestamp":1714012879.3388772,"name":"online","context":{"idset":"45,109,111,115,119,125,163,167,175,181,187,207,225,249,277,311,327,333,579,884,8949,8952-8953,8956,9053,9056,9058,9076,10363,11601,11627,11707,11761"}}
+{"timestamp":1714012879.5404978,"name":"online","context":{"idset":"7,89,131,157,177,195,197,201,219,227,229,241,285,287,289,293,315,319,325,573,8639,8950-8951,8954,9070,9207,9630,10538,10865,10870,10936,11608,11668,11695"}}
+{"timestamp":1714012879.6449747,"name":"online","context":{"idset":"91,95,100-101,103,105,107,123,127,129,179,183,185,191,193,203,209,213,215,221,223,233,243,245,247,251,283,291,295,299,301,309,321,331,883,8267,8593,8890,9072,9223,9248,9267,9559,9596,9624,9707,9829,9914,10290,10379,10456,10513,10524,10531,10589,10609,10648,10685,11157,11417,11431,11478,11532,11596,11602,11667,11677"}}
+{"timestamp":1714012879.7544868,"name":"online","context":{"idset":"9,15,33,39,49,53,161,237,281,8605,9842,10268,10754,11133,11204,11580"}}
+{"timestamp":1714012879.8520155,"name":"online","context":{"idset":"1,29,43,93,99,171,199,205,211,231,305,307,323,329,8213,8240-8241,8255,8653,8669,8685,8821,8854,8903,8929,8941,9077,9120,9137,9144,9157-9158,9205,9224,9261,9275,9404,9489,9494,9541,9560,9623,9666,9687,9693,9729-9730,9742,9755,9801,9902,9905,9993,10013,10075,10236,10246,10250,10302,10310,10334,10355,10394,10402,10424,10497,10580,10683,10689,10701,10748,10750,10803,10820,10841,10916,10929,10971,10977,11025,11053,11074,11086,11098,11119,11145,11180,11184,11224,11408,11454,11505,11517,11538,11556,11569,11579,11587,11692,11740,11763,11769,11782,11787,11835,11890"}}
+{"timestamp":1714012879.9550223,"name":"online","context":{"idset":"3,5,11,17,19,21,25,27,31,35,37,41,51,56-57,59,8195,8223,8252,9579,9604,9660,9711,9759,9830,9910,9913,10119,10228-10229,10232,10282,10373,10453,10484,10501,10616,10642,10739,10741,10811,10945,10964,11063,11143,11175,11208,11612,11649,11743,11818,11844,11871"}}
+{"timestamp":1714012880.0771279,"name":"online","context":{"idset":"13,47,55,58,113,173,189,412,415,8192,8236,8263,8579,8663,8692,8822,8850,8885,8895,9063,9081,9093,9115,9149,9173,9176,9187,9235,9237,9245,9325,9490,9512,9551,9606,9610,9648,9668,9734,9781,9783,9803,9857,9863,9876,9901,9934,9973,9977,10003,10116,10122,10127,10135,10159,10198,10221,10235,10256,10298,10313,10350,10358,10367,10375,10426,10450,10472,10543,10549,10571,10655,10661,10673,10705,10745,10788,10790,10825,10832,10873,10883,10912,10925,10973,10983,11017,11057,11067,11093,11118,11130,11149,11209,11220-11221,11400,11463,11467,11476,11498,11501,11512,11528,11564,11674,11687,11714-11715,11762,11811,11823,11838-11839,11851-11852,11870"}}
+{"timestamp":1714012880.1850016,"name":"online","context":{"idset":"399,8191,8305,8594,8604,8684,8688,8899,8931,9061,9067,9145,9150,9160,9198,9212,9263,9271,9300,9328,9502,9535,10031,10510,11153,11183,11410,11554,11568,11659,11683,11694,11819"}}
+{"timestamp":1714012880.2869501,"name":"online","context":{"idset":"353,359,9578,9583,9587,9601,9605,9629,9644,9674,9676,9680,9692,9731,9756,9782,9793,9795,9807,9811,9822,9873,9893,9959,9986-9987,9990,10010,10077,10090,10121,10138,10160,10170,10190,10230,10252,10266,10289,10318,10561"}}
+{"timestamp":1714012880.3924086,"name":"online","context":{"idset":"361,380,384,387,396,403,417,419,9269,9555,9570,9799,9827,10019,10054,10060-10061,10097,10099,10111,10118,10132,10143,10166,10187,10191,10212-10213,10227,10247,10274,10285,10300-10301,10304,10309,10312,10326,10328,10353,10406-10408,10416,10427,10430,10436-10437,10458-10459,10468,10470,10491,10493,10500,10503,10508-10509,10525,10532,10569,10581,10588,10596,10598,10620,10624,10626,10632,10644,10652,10681,10742,10769,10830,10853,10855,10872,10985,11041,11051,11088,11104,11155,11185,11391,11493,11592"}}
+{"timestamp":1714012880.5043764,"name":"online","context":{"idset":"10,24,28,30,38,42,44,46,48,50,60,87,97,174,218,345,355,357,367,409,580,8194,8245,8257,8277,8296,8643,8919,9018,9133,9202,9209,9241,9257-9258,9468,9488,9510,9539,9550,9592,9616,9620,9636,9640,9650,9678,9684,9691,9695,9719,9735,9743,9754,9767,9771,9775,9815,9824,9843,9847,9855,9883,9929,9949,9955,9970,9975,9979,9991,10007,10015,10022,10025,10027,10035,10039,10043,10047,10059,10079,10083,10087,10095,10113,10134,10210-10211,10217,10222,10238,10245,10254,10293,10296,10314,10359,10364,10386,10390,10393,10412,10418,10428,10434,10442,10464,10474,10478,10487,10492,10505,10523,10533,10537,10547,10551,10559,10577,10587,10597,10604,10610,10618-10619,10636,10647,10657,10663,10665,10675,10677,10679,10684,10691,10697,10703-10704,10713,10717-10718,10723,10733,10758,10763-10764,10772-10774,10776,10780,10786,10792-10793,10799,10807,10809,10813,10819,10821,10851,10866,10884,10890,10893,10896-10897,10900,10907-10909,10917,10922-10924,10938,10940,10944,10946,10949,10952,10975,10981,10984,10989,11005,11013,11019,11027,11045,11065,11080,11091,11094,11096,11100,11106,11110,11112,11122,11124,11132,11135,11141,11147,11162,11168,11173-11174,11177,11193-11194,11197,11212,11216,11222,11228,11240-11241,11243,11252,11406,11412,11421,11438,11442,11446,11457,11460,11470,11474,11477,11485,11491,11494-11495,11509-11510,11516,11520,11541-11542,11544,11557,11572,11584,11599-11600,11610-11611,11621,11623,11630,11633,11642,11665,11691,11693,11703,11712,11750,11759,11779,11805,11817,11831,11873,11887"}}
+{"timestamp":1714012880.6189959,"name":"online","context":{"idset":"2,6,14,20,22,26,32,34,36,208,326,337,340-341,343,351,354,365,369-370,375,377,381,385,389,391,405,408,413,8049,8188-8189,8193,8219,8228-8229,8239,8243,8253,8259,8261,8265,8268,8282,8284,8288-8289,8292,8294,8306,8585,8599,8601,8609-8610,8637,8641,8651,8665,8671,8681,8689,8824,8826,8828,8838,8846,8860,8865,8873,8877,8879,8881,8909,8913,8916-8917,8944,9038,9068,9079,9099,9105,9118,9123,9126,9129,9138,9146,9151,9159,9168,9170,9174,9178-9179,9182,9200,9204,9210,9215,9217,9225,9227,9233-9234,9236,9243-9244,9249,9251,9264-9265,9273,9276,9278,9284,9288,9290,9297,9308,9313,9326,9332,9443,9461-9462,9474,9476,9503,9509,9514,9516,9526,9530,9537,9554,9557,9561,9565,9567,9575,9581,9600,9603,9608,9617,9628,9637,9643,9653,9656-9657,9663,9672-9673,9682,9686,9698,9703,9705,9713,9723,9733,9737,9739,9746,9753,9763,9766,9779,9785,9787,9802,9804,9813,9820,9836-9837,9840,9853,9867,9875,9877,9884-9885,9887-9889,9891,9897-9899,9915,9917,9919,9923,9927,9931,9933,9935,9941,9943,9947,9964,9968,9989,9994,9997,10011,10023,10032,10037,10042,10048,10064-10065,10067,10071,10091,10102,10104,10110,10117,10139,10152,10154,10158,10202,10218,10264,10272,10287,10343,10371,10382,10414,10438,10440,10454,10506,10511,10517,10521,10526,10555,10557,10560,10565,10575,10579,10586,10592,10612,10625,10628,10650,10731,10740,10746,10782-10784,10797,10826,10828-10829,10831,10833,10838-10839,10843,10849,10861,10877,10881,10901,10905,10921,10928,10954,10956,10960,10967-10968,10970,10993,11009,11011,11032,11043,11049,11052,11055,11058,11076-11077,11090,11102,11105,11125,11127,11129,11136-11137,11161,11166,11169,11179,11195,11205,11223,11233,11248,11382,11386,11388,11393-11394,11396,11402,11418,11435,11462,11466,11469,11473,11483,11487,11524,11533,11536,11545-11546,11548-11549,11552,11560,11574,11576,11581-11583,11588,11590,11606,11609,11616,11620,11629,11631,11636-11637,11639,11641,11643,11645,11647,11651,11653,11663-11664,11675-11676,11679,11689,11697,11704,11711,11713,11735,11739,11741-11742,11746,11755,11758,11760,11764,11766-11767,11773,11775-11776,11778,11783,11789,11795,11799,11803,11807,11809,11813,11820,11822,11824,11847,11855,11857,11859,11867,11869,11883,11885"}}
+{"timestamp":1714012880.7613952,"name":"online","context":{"idset":"40,54,104,132,164,166,180,198,230,234,298,338,344,383,390,8047,8246,8290,8297,8591,8660,8668,8674,8679,8840,8844,8848,8852,8872,8875,8884,8889,8891,8896,8902,8907,8939,8947,9015,9019,9021-9022,9024,9026,9028,9030,9032,9135,9140,9155-9156,9175,9177,9183,9188,9211,9219,9239,9260,9282,9286,9294,9298,9306,9310,9322,9492,9500,9508,9548,9569,9602,9612-9613,9632,9652,9661-9662,9665,9670-9671,9681,9685,9712,9714,9716,9724,9726,9751,9769,9774,9776,9778,9789-9790,9792,9810,9816,9819,9839,9848-9849,9859,9861-9862,9936-9937,9958,9965,9999,10002,10005,10014,10029,10044-10045,10049,10055,10057-10058,10069,10096,10112,10125-10126,10130,10141,10145,10153,10172,10177,10203,10205,10207,10237,10275,10277-10278,10297,10325,10327,10330,10335,10338,10347,10351,10354,10357,10369,10374,10377,10391,10398,10433,10447,10460,10465,10469,10471,10475,10482,10485-10486,10488-10489,10498,10516,10553,10568,10574,10584,10590,10594,10599,10605,10613,10617,10622,10631,10637,10640-10641,10643,10664,10693,10699,10706,10712,10720,10728-10729,10732,10734,10737,10751,10753,10756,10762,10767,10770,10775,10787,10805,10812,10815,10823,10827,10836-10837,10847,10856,10885,10898-10899,10927,10933,10941,10943,10951,10959,10965,10969,10976,10982,10997,11003-11004,11020,11026,11031,11044,11060,11078-11079,11082,11085,11092,11114,11128,11152,11156,11165,11176,11188,11210,11214,11217,11234,11239,11245,11383,11399,11414-11415,11424,11426,11428,11430,11432,11436,11464,11481,11499,11503-11504,11518,11550,11553,11558,11563,11566-11567,11595,11614,11626,11628,11638,11652,11661,11670,11672,11681,11699,11706,11709-11710,11733-11734,11737-11738,11752,11765,11768,11784-11785,11800,11810,11814,11816,11829-11830,11837,11853,11874"}}
+{"timestamp":1714012880.9311609,"name":"online","context":{"idset":"8,12,16,18,52,88,92,94,108,110,112,124,128,160,188,190,194,196,200,214,216,222,226,228,236,292,296,310,316,334,339,346-347,349-350,356,358,362-363,368,371-372,378-379,382,392-393,395,398,401-402,406,418,576,8050-8052,8185,8187,8214-8217,8220-8222,8225,8227,8231-8232,8247,8249,8251,8254,8260,8264,8269-8271,8273,8275,8279-8281,8283,8285-8286,8293,8295,8298,8300,8302,8581,8583-8584,8588,8592,8598,8603,8607,8640,8645,8647-8648,8657-8659,8661,8666-8667,8673,8677,8683,8691,8837,8839,8842,8855,8858,8864,8867,8869,8871,8878,8882-8883,8892-8893,8897,8901,8904-8905,8911,8915,8918,8921,8923,8925,8927-8928,8930,8933,8936-8937,8940,8945,8948,8986-8987,8989,8993,8996,8998-9002,9004-9005,9008-9009,9013-9014,9016-9017,9020,9023,9025,9027,9031,9033-9035,9037,9039-9042,9044,9082,9087-9090,9095,9097,9101,9107,9109,9111,9113,9117,9119,9121,9125,9127,9131,9139,9142,9148,9152,9154,9162-9164,9166,9172,9180-9181,9190-9192,9194,9196-9197,9199,9201,9208,9213,9216,9220-9221,9226,9229-9231,9247,9255,9259,9272,9280-9281,9285,9292,9295-9296,9304,9309,9312,9318-9321,9324,9330,9355,9365,9464,9466,9470,9472,9475,9478-9480,9482,9484,9487,9493,9496,9518-9519,9522-9523,9528,9534,9536,9538,9540,9544,9546,9553,9556,9558,9562,9566,9571-9572,9574,9585,9590-9591,9594,9598,9611,9614,9618-9619,9622,9625-9627,9634,9638,9641-9642,9646,9654,9664,9667,9669,9675,9677,9679,9689,9697,9699-9702,9709,9715,9717,9725,9740-9741,9745,9747,9757,9761-9762,9768,9770,9773,9777,9780,9786,9791,9794,9805,9809,9814,9828,9831-9835,9841,9844-9845,9851-9852,9858,9865,9869,9871,9878-9879,9881,9894,9903,9906-9907,9909,9911-9912,9921-9922,9925,9928,9939,9942,9945,9951,9953,9956,9961,9963,9966,9969,9971-9972,9976,9978,9980-9982,9985,9988,9995,10001,10009,10012,10017,10021,10024,10028,10030,10033-10034,10041,10046,10050,10053,10062-10063,10066,10070,10072-10074,10076,10078,10080-10082,10086,10089,10092-10094,10101,10103,10106-10109,10123,10128,10131,10149-10150,10156-10157,10161-10164,10169,10171,10174,10178,10180-10181,10185-10186,10193-10194,10196,10206,10209,10214-10216,10220,10223-10225,10231,10234,10239,10242,10258,10265,10267,10269-10270,10273,10279-10280,10286,10291,10295,10305-10306,10315,10331,10337,10342,10345,10360-10361,10365,10370,10380-10381,10383-10384,10387,10389,10392,10396-10397,10399,10401,10403,10409-10411,10415,10419-10420,10422,10429,10431-10432,10443,10445,10448-10449,10451-10452,10462-10463,10476-10477,10479-10480,10483,10490,10495,10499,10502,10504,10507,10512,10515,10519,10527,10534-10535,10539-10541,10544-10545,10548,10556,10563,10567,10570,10572-10573,10576,10582-10583,10585,10593,10602,10608,10621,10623,10629-10630,10634-10635,10638,10646,10649,10653-10654,10659-10660,10666-10667,10669-10672,10676,10690,10692,10694-10695,10700,10702,10707,10709-10711,10714-10715,10719,10721,10724-10727,10735-10736,10738,10743-10744,10747,10749,10755,10757,10759-10760,10766,10768,10771,10777-10779,10785,10794-10796,10798,10800,10802,10808,10810,10816-10818,10822,10834-10835,10840,10844-10845,10848,10852,10854,10858-10859,10862-10864,10867,10869,10871,10874-10876,10882,10886-10887,10892,10894,10902,10906,10911,10913-10915,10918-10919,10931-10932,10934-10935,10939,10948,10950,10953,10957-10958,10961-10962,10966,10978,10986,10990,10992,10995-10996,10998-11001,11007,11010,11014-11016,11023-11024,11029,11033-11035,11037-11040,11042,11046-11048,11056,11061,11066,11069,11071-11073,11075,11083-11084,11087,11095,11099,11107-11108,11111,11115,11117,11120-11121,11123,11126,11131,11142,11144,11148,11151,11154,11158-11160,11164,11167,11170,11172,11178,11186-11187,11191,11196,11198-11202,11206-11207,11211,11213,11215,11218,11226-11227,11229-11230,11232,11235-11236,11238,11244,11246-11247,11250-11251,11381,11384-11385,11387,11390,11392,11395,11397-11398,11403-11404,11409,11411,11416,11419-11420,11422,11427,11429,11437,11441,11444,11447,11450,11456,11459,11465,11468,11471-11472,11482,11486,11489-11490,11500,11502,11507-11508,11514,11521-11522,11525-11527,11530,11534,11537,11539-11540,11547,11555,11559,11565,11570,11573,11577-11578,11586,11594,11597-11598,11604-11605,11613,11619,11625,11634-11635,11655,11657,11660,11662,11666,11669,11671,11673,11698,11701,11744,11749,11753-11754,11757,11771,11774,11777,11781,11793,11797-11798,11801-11802,11815,11821,11832-11833,11836,11841,11843,11849,11858,11863,11865-11866,11875-11876,11879,11881,11884,11889,11891"}}
+{"timestamp":1714012881.1007216,"name":"online","context":{"idset":"96,98,102,106,114,116,118,122,126,130,158,168,170,176,178,182,184,192,202,204,210,212,220,224,232,238,240,242,244,246,248,250,252,280,282,284,288,290,294,300,302,304,306,308,312,318,320,322,324,328,330,332,352,364,366,374,376,386,388,394,404,407,414,416,420,575,577,8031,8046,8048,8182,8184,8186,8190,8196,8224,8226,8230,8235,8238,8242,8244,8248,8250,8256,8258,8262,8266,8274,8276,8278,8287,8299,8301,8303,8307-8308,8586-8587,8590,8595-8596,8602,8606,8608,8611-8612,8636,8638,8644,8646,8649-8650,8654,8656,8662,8664,8670,8672,8675-8676,8678,8680,8682,8686-8687,8690,8825,8827,8841,8843,8845,8847,8849,8853,8856-8857,8859,8861-8863,8866,8870,8874,8876,8880,8886-8888,8894,8898,8900,8906,8908,8910,8912,8914,8920,8922,8924,8926,8932,8934,8938,8942-8943,8981-8985,8988,8990-8992,8994-8995,8997,9006-9007,9010-9012,9029,9036,9078,9080,9083-9084,9086,9091-9092,9094,9096,9098,9100,9102-9104,9106,9108,9110,9112,9114,9116,9122,9124,9128,9130,9132,9134,9136,9141,9143,9147,9153,9161,9165,9167,9169,9171,9185,9189,9193,9195,9203,9206,9214,9218,9222,9228,9232,9238,9240,9242,9246,9250,9252,9254,9256,9262,9266,9268,9270,9274,9277,9279,9283,9287,9289,9291,9299,9301-9303,9305,9307,9311,9315,9317,9323,9327,9329,9331,9343,9463,9465,9467,9469,9471,9473,9477,9481,9485-9486,9495,9497,9499,9504-9505,9507,9511,9513,9515,9517,9520-9521,9524-9525,9527,9529,9531-9533,9542-9543,9545,9547,9549,9552,9568,9576-9577,9580,9582,9584,9586,9588-9589,9593,9597,9599,9607,9609,9615,9621,9631,9633,9635,9645,9647,9649,9655,9658-9659,9683,9688,9690,9696,9704,9706,9708,9710,9718,9721-9722,9728,9732,9736,9738,9744,9748,9750,9752,9758,9760,9764-9765,9772,9784,9788,9796-9798,9800,9806,9808,9812,9818,9821,9823,9825-9826,9838,9846,9850,9854,9856,9860,9864,9866,9868,9870,9872,9874,9880,9882,9886,9890,9892,9896,9900,9904,9908,9916,9918,9920,9924,9926,9930,9932,9938,9940,9944,9946,9948,9950,9952,9954,9957,9960,9962,9967,9974,9984,9992,9996,9998,10000,10004,10006,10008,10016,10018,10020,10026,10036,10038,10040,10052,10056,10068,10084-10085,10088,10100,10105,10114-10115,10120,10124,10129,10133,10137,10140,10142,10144,10146-10148,10151,10155,10165,10167-10168,10173,10175-10176,10179,10182-10184,10189,10192,10197,10199-10201,10204,10208,10226,10241,10243,10251,10255,10257,10259,10263,10271,10281,10283,10294,10299,10303,10311,10317,10319,10321,10329,10339,10341,10362,10366,10368,10372,10385,10388,10395,10405,10413,10421,10423,10425,10435,10439,10441,10446,10455,10457,10461,10467,10473,10481,10494,10496,10518,10520,10522,10528-10530,10536,10542,10546,10550,10552,10554,10558,10562,10564,10566,10591,10595,10601,10603,10607,10611,10614-10615,10633,10645,10651,10656,10658,10662,10668,10674,10678,10680,10682,10686,10688,10696,10698,10708,10716,10722,10730,10761,10765,10781,10789,10791,10804,10806,10814,10824,10842,10846,10860,10868,10878,10888,10895,10904,10910,10920,10926,10930,10947,10955,10972,10974,10979,10988,10991,10994,11002,11006,11008,11012,11018,11022,11028,11030,11036,11050,11059,11062,11064,11081,11097,11101,11109,11113,11116,11134,11138-11139,11146,11150,11171,11182,11190,11192,11203,11219,11225,11231,11237,11242,11389,11401,11405,11407,11413,11423,11425,11433,11439-11440,11443,11445,11448-11449,11451-11453,11455,11461,11475,11479-11480,11484,11488,11492,11496-11497,11506,11513,11515,11519,11523,11529,11531,11543,11551,11561-11562,11571,11575,11585,11589,11591,11603,11607,11615,11617-11618,11622,11624,11632,11640,11644,11646,11648,11650,11656,11678,11680,11682,11684-11686,11688,11696,11700,11702,11705,11708,11716,11736,11745,11747,11751,11756,11770,11772,11780,11786,11788,11791-11792,11794,11796,11804,11806,11808,11812,11828,11840,11842,11846,11848,11854,11860-11862,11864,11868,11872,11877-11878,11880,11882,11886,11888,11892"}}
+{"timestamp":1714012881.3144474,"name":"online","context":{"idset":"162,7738,7802,7874,7901,7903,8021,8600,9065,9069,9071,9073-9074,10349"}}
+{"timestamp":1714012881.5014422,"name":"online","context":{"idset":"7754,7870,7919,7937,9064"}}
+{"timestamp":1714012881.6712027,"name":"online","context":{"idset":"7690,7716,7743,7752-7753,7768,7804,7832,7850,7865,7889,7894,7896,7936,7967,7981,7985,7991,9066"}}
+{"timestamp":1714012881.8501763,"name":"online","context":{"idset":"7670,7672,7740,7747,7750,7793,7810,7814,7818,7826,7834,7836,7848,7868,7884,7886,7898,7904,7912-7913,7940-7942,8010,8022"}}
+{"timestamp":1714012882.0326507,"name":"online","context":{"idset":"7679-7680,7710,7718,7746,7828,7833,7854-7855,7878,7880,7902,7909,7920,7926,7957,7965,7977,7980,7989,7995,8024,8037,8044"}}
+{"timestamp":1714012882.2264054,"name":"online","context":{"idset":"7676,7688,7704,7713,7720,7724,7726,7728,7732-7733,7764,7770,7776,7782,7785,7798,7806,7808,7820,7822,7829,7844,7856,7862,7888,7890,7906,7914,7918,7960,7969,7986,7997,8001,8004-8007,8014"}}
+{"timestamp":1714012882.3304119,"name":"online","context":{"idset":"7684,7691,7709,7727,7729,7735,7744,7763,7767,7771,7778,7789,7809,7825,7845,7852-7853,7857,7866,7893,7905,7911,7928-7929,7953,7975,7979,8013,8026,8029"}}
+{"timestamp":1714012882.4333751,"name":"online","context":{"idset":"7678,7685,7696,7769,7773,7779,7807,7837,7851,7863,7867,7916,7945,7994,8036"}}
+{"timestamp":1714012882.5386572,"name":"online","context":{"idset":"7681,7698,7731,7749,7762,7766,7787,7797,7801,7830,7859,7861,7927,7933,7939,7949-7950,7972,8009,8012,8045"}}
+{"timestamp":1714012882.7148151,"name":"online","context":{"idset":"7677,7686,7702,7730,7734,7741-7742,7758-7759,7765,7777,7791,7799-7800,7812,7824,7835,7839,7841,7846,7871-7872,7876,7879,7882-7883,7897,7915,7921,7955,7966,7971,7973,7987,7993,8002,8017,8019,8028,8035,8039-8040"}}
+{"timestamp":1714012882.817024,"name":"online","context":{"idset":"7745,7756,7790,7813,7819,7823,7827,7831,7860,7885,7910,7917,7934-7935,7947-7948,7961-7962,7970,7976,7988,7990,7996,7998,8011,8016,8018,8032-8034,8042"}}
+{"timestamp":1714012882.91995,"name":"online","context":{"idset":"7692,7694,7705-7706,7714,7736,7748,7751,7774,7784,7842,7858,7864,7887,7908,7924,7951,7983,7999,8015,8023,8041"}}
+{"timestamp":1714012883.025219,"name":"online","context":{"idset":"7671,7683,7689,7700-7701,7712,7717,7721,7723,7725,7755,7760,7772,7775,7780,7795-7796,7811,7817,7843,7847,7869,7873,7881,7899,7923,7925,7932,7946,7954,7964,7968,7984,8003,8027,8030,8038"}}
+{"timestamp":1714012883.1955705,"name":"online","context":{"idset":"7675,7697,7707,7737,7739,7788,7794,7805,7821,7838,7891-7892,7956,7978,7982"}}
+{"timestamp":1714012883.307791,"name":"online","context":{"idset":"7673,7703,7708,7722,7783,7895,7958,8025"}}
+{"timestamp":1714012883.49176,"name":"online","context":{"idset":"7695,7699,7711,7757,7803,7877,7900,7930-7931,7944,7963,8000,8008"}}
+{"timestamp":1714012883.6988449,"name":"online","context":{"idset":"7938,8020"}}
+{"timestamp":1714012888.4008701,"name":"online","context":{"idset":"8634"}}
+{"timestamp":1714012888.5917401,"name":"online","context":{"idset":"8567,8979,9046"}}
+{"timestamp":1714012888.7705293,"name":"online","context":{"idset":"8955,9564,11827"}}
+{"timestamp":1714012888.9427166,"name":"online","context":{"idset":"7907,7943,7952,7974,8043,8272,8597,8655,8868,9057,9059,9062,9293,9316,9409,9563,9595,9651,9694,9720,9749,10051,10098,10322,10346,10444,10578,10687,10752,10857,10937,10963,10980,10987,11054,11070,11163,11189,11458,11511,11535,11593,11654,11690,11834,11845"}}
+{"timestamp":1714012889.1268222,"name":"online","context":{"idset":"7682,7687,7792,7840,7849,7875,7922,7959,7992,8218,8582,8823,8851,8935,8946,9085,9184,9186,9253,9369,9491,9506,9573,9639,9727,9817,9895,10195,10219,10253,10376,10404,10417,10466,10514,10600,10627,10639,10850,10891,10903,10942,11021,11089,11103,11181,11249,11434,11748,11850,11856"}}
+{"timestamp":1714012889.3057175,"name":"online","context":{"idset":"159,186,278,303,7674,7786,9314,9361,9483,9498,9983,10400,10606,10889,11068,11140"}}
+{"timestamp":1714012889.5015321,"name":"online","context":{"idset":"235,297,317,400"}}
+{"timestamp":1714012889.7685404,"name":"online","context":{"idset":"4"}}
+{"timestamp":1714012889.9644222,"name":"online","context":{"idset":"23"}}
+{"timestamp":1714013056.4711175,"name":"online","context":{"idset":"574"}}
+{"timestamp":1714013473.6341405,"name":"online","context":{"idset":"8613"}}
+{"timestamp":1714013477.5999424,"name":"online","context":{"idset":"8619"}}
+{"timestamp":1714013478.8953941,"name":"online","context":{"idset":"8614"}}
+{"timestamp":1714013479.6201096,"name":"online","context":{"idset":"8624"}}
+{"timestamp":1714013481.0020769,"name":"online","context":{"idset":"8620-8621"}}
+{"timestamp":1714013481.2814436,"name":"online","context":{"idset":"8616"}}
+{"timestamp":1714013481.4766383,"name":"online","context":{"idset":"8625,8628"}}
+{"timestamp":1714013481.9412267,"name":"online","context":{"idset":"8615,8618,8623,8627"}}
+{"timestamp":1714013482.1149824,"name":"online","context":{"idset":"8617,8622,8626"}}
+{"timestamp":1714013713.6316857,"name":"online","context":{"idset":"8642"}}
+{"timestamp":1714017605.8718774,"name":"offline","context":{"idset":"11660"}}
+{"timestamp":1714020724.7983506,"name":"drain","context":{"idset":"9333-9348","reason":"--reason back plane fix -KPN","overwrite":0}}
+{"timestamp":1714020739.7009661,"name":"offline","context":{"idset":"9338"}}
+{"timestamp":1714020739.7971578,"name":"offline","context":{"idset":"9333"}}
+{"timestamp":1714020739.9407973,"name":"offline","context":{"idset":"9345"}}
+{"timestamp":1714020739.948736,"name":"offline","context":{"idset":"9336"}}
+{"timestamp":1714020739.9565516,"name":"offline","context":{"idset":"9343"}}
+{"timestamp":1714020740.0556529,"name":"offline","context":{"idset":"9346"}}
+{"timestamp":1714020740.0623107,"name":"offline","context":{"idset":"9334"}}
+{"timestamp":1714020740.068481,"name":"offline","context":{"idset":"9335"}}
+{"timestamp":1714020740.0748758,"name":"offline","context":{"idset":"9337"}}
+{"timestamp":1714020740.0812266,"name":"offline","context":{"idset":"9339"}}
+{"timestamp":1714020740.0875518,"name":"offline","context":{"idset":"9340"}}
+{"timestamp":1714020740.0938725,"name":"offline","context":{"idset":"9341"}}
+{"timestamp":1714020740.1175244,"name":"offline","context":{"idset":"9342"}}
+{"timestamp":1714020740.1234169,"name":"offline","context":{"idset":"9344"}}
+{"timestamp":1714020740.1295867,"name":"offline","context":{"idset":"9347"}}
+{"timestamp":1714020740.1352491,"name":"offline","context":{"idset":"9348"}}
+{"timestamp":1714021088.8921282,"name":"drain","context":{"idset":"9349-9364","reason":"--reason back plane fix -KPN","overwrite":0}}
+{"timestamp":1714021148.2072444,"name":"offline","context":{"idset":"9349"}}
+{"timestamp":1714021148.2909093,"name":"offline","context":{"idset":"9351"}}
+{"timestamp":1714021148.2999954,"name":"offline","context":{"idset":"9354"}}
+{"timestamp":1714021148.3078794,"name":"offline","context":{"idset":"9364"}}
+{"timestamp":1714021148.315798,"name":"offline","context":{"idset":"9357"}}
+{"timestamp":1714021148.3200119,"name":"offline","context":{"idset":"9363"}}
+{"timestamp":1714021148.3304651,"name":"offline","context":{"idset":"9350"}}
+{"timestamp":1714021148.338124,"name":"offline","context":{"idset":"9353"}}
+{"timestamp":1714021148.4231303,"name":"offline","context":{"idset":"9355"}}
+{"timestamp":1714021148.51511,"name":"offline","context":{"idset":"9358"}}
+{"timestamp":1714021148.5231655,"name":"offline","context":{"idset":"9360"}}
+{"timestamp":1714021148.5277467,"name":"offline","context":{"idset":"9361"}}
+{"timestamp":1714021148.5322127,"name":"offline","context":{"idset":"9352"}}
+{"timestamp":1714021148.5503335,"name":"offline","context":{"idset":"9359"}}
+{"timestamp":1714021148.5710809,"name":"offline","context":{"idset":"9356"}}
+{"timestamp":1714021148.6317418,"name":"offline","context":{"idset":"9362"}}
+{"timestamp":1714021375.5272179,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1714021375.5326836,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1714021375.5360494,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1714021375.5393913,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1714021375.5427294,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1714021375.5460572,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1714021375.5493803,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1714021375.553057,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1714021375.556416,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1714021375.5597529,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1714021375.5630836,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1714021375.5663846,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1714021375.570092,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1714021375.573432,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1714021375.5767536,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1714021375.5908418,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1714021375.5975184,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1714021375.601928,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1714021375.6139235,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1714021375.6172483,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1714021375.6209064,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1714021375.6242223,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1714021375.6275387,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1714021375.6308861,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1714021375.6428685,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1714021375.6547227,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1714021375.6580489,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1714021375.6613719,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1714021375.6646702,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1714021375.676856,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1714021375.6801562,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1714021375.6834769,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1714021375.6868868,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1714021375.7049479,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1714021375.7082555,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1714021375.7115614,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1714021375.71492,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1714021375.7182381,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1714021375.730602,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1714021375.7339354,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1714021375.7372561,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1714021375.7498343,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1714021375.7531188,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1714021375.7564304,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1714021375.7618258,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1714021375.7773955,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1714021375.7807713,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1714021375.7840943,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1714021375.7873797,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1714021375.799895,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1714021375.803261,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1714021375.8065581,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1714021375.8099418,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1714021375.8224912,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1714021375.8257718,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1714021375.8290918,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1714021375.8323705,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1714021375.8443024,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1714021375.847559,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1714021375.8508542,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1714021375.8541114,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1714021375.8573797,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1714021375.869211,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1714021375.8724782,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1714021375.8757329,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1714021375.8877006,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1714021375.8909764,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1714021375.8942771,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1714021375.8975344,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1714021375.9093449,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1714021375.9126499,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1714021375.916008,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1714021375.9192753,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1714021375.9225652,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1714021375.9258537,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1714021375.9291017,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1714021375.9323459,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1714021375.9355969,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1714021375.9388151,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1714021375.9420507,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1714021375.9453385,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1714021375.9486451,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1714021375.9519036,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1714021375.9551685,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1714021375.9584053,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1714021375.9617667,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1714021375.9650409,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1714021375.9771218,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1714021375.9804192,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1714021375.9836349,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1714021375.9869108,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1714021375.9912941,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1714021375.9952071,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1714021375.9986434,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1714021376.0019376,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1714021376.0137899,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1714021376.0170393,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1714021376.0203772,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1714021376.0236292,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1714021376.0353882,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1714021376.038738,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1714021376.0419939,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1714021376.0454583,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1714021376.0487051,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1714021376.0519269,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1714021376.0552163,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1714021376.0584896,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1714021376.0617142,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1714021376.0649381,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1714021376.0681541,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1714021376.071348,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1714021376.0745385,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1714021376.0778892,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1714021376.0811188,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1714021376.0843244,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1714021376.0960593,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1714021376.099257,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1714021376.1024628,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1714021376.1146386,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1714021376.1178691,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1714021376.1211658,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1714021376.1328838,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1714021376.1361604,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1714021376.1393638,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1714021376.1511767,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1714021376.1543748,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1714021376.1662445,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1714021376.1694987,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1714021376.1726961,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1714021376.1859808,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1714021376.1904297,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1714021376.1936383,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1714021376.1968665,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1714021376.2000949,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1714021376.2032897,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1714021376.2065034,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1714021376.2096844,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1714021376.2128971,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1714021376.2161953,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1714021376.2193744,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1714021376.2225356,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1714021376.2257621,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1714021376.2289534,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1714021376.232131,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1714021376.2353575,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1714021376.2505112,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1714021376.2550571,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1714021376.2583184,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1714021376.261503,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1714021376.2646608,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1714021376.2678299,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1714021376.2710247,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1714021376.2741892,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1714021376.2775218,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1714021376.280822,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1714021376.2840245,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1714021376.2871828,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1714021376.2905722,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1714021376.2937403,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1714021376.2969151,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1714021376.308965,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1714021376.3121362,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1714021376.3152974,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1714021376.3270032,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1714021376.3302076,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1714021376.3333697,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1714021376.3365061,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1714021376.3396935,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1714021376.3513813,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1714021376.3545125,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1714021376.3580446,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1714021376.3697929,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1714021376.3729548,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1714021376.3761914,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1714021376.3793492,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1714021376.3824701,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1714021376.3855941,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1714021376.389236,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1714021376.3924112,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1714021376.3956606,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1714021376.3987849,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1714021376.403048,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1714021376.4063354,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1714021376.4094734,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1714021376.4125843,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1714021376.4157019,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1714021376.4188313,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1714021376.4219599,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1714021376.4335608,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1714021376.4367294,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1714021376.4398558,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1714021376.4429748,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1714021376.4545963,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1714021376.4576986,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1714021376.4608178,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1714021376.4639375,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1714021376.4755216,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1714021376.4786747,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1714021376.4818738,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1714021376.4849834,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1714021376.4880929,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1714021376.4998672,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1714021376.5029781,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1714021376.5061691,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1714021376.509315,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1714021376.5124121,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1714021376.5155761,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1714021376.5187049,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1714021376.521812,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1714021376.52495,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1714021376.5280352,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1714021376.5311964,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1714021376.534292,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1714021376.5373695,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1714021376.5404639,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1714021376.5435464,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1714021376.5466125,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1714021376.5497248,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1714021376.5529237,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1714021376.5560706,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1714021376.5592167,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1714021376.5623131,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1714021376.5660572,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1714021376.5800974,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1714021376.5831919,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1714021376.5862746,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1714021376.5981154,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1714021376.6011972,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1714021376.6129577,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1714021376.6160383,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1714021376.6191084,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1714021376.6221867,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1714021376.6252592,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1714021376.6368859,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1714021376.6485357,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1714021376.6516755,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1714021376.6548333,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1714021376.6579142,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1714021376.6610112,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1714021376.6642103,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1714021376.6672761,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1714021376.6703644,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1714021376.6734371,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1714021376.6766367,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1714021376.679791,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1714021376.6829164,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1714021376.6860394,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1714021376.6892126,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1714021376.6923361,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1714021376.6953969,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1714021376.6984632,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1714021376.7100656,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1714021376.7131019,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1714021376.7161341,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1714021376.7191873,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1714021376.7307668,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1714021376.7338479,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1714021376.7369127,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1714021376.7399385,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1714021376.7431023,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1714021376.7546358,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1714021376.7576842,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1714021376.7607441,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1714021376.763788,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1714021376.766861,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1714021376.769891,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1714021376.7729588,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1714021376.7759895,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1714021376.7790263,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1714021376.7820776,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1714021376.7851923,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1714021376.7882154,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1714021376.7912946,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1714021376.7943976,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1714021376.8060069,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1714021376.8090353,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1714021376.8120487,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1714021376.8150914,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1714021376.8353691,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1714021376.8383915,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1714021376.8498611,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1714021376.8529058,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1714021376.8559189,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1714021376.8589621,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1714021376.8620338,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1714021376.8650284,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1714021376.8681326,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1714021376.8711517,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1714021376.8741395,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1714021376.877193,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1714021376.8802249,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1714021376.8832211,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1714021376.886229,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1714021376.8892267,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1714021376.8922458,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1714021376.8953602,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1714021376.8984063,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1714021376.9014053,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1714021376.9044061,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1714021376.9159827,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1714021376.9208589,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1714021376.9268334,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1714021376.932158,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1714021376.9351614,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1714021376.9466727,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1714021376.9496698,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1714021376.9529939,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1714021376.9559982,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1714021376.9674654,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1714021376.9704528,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1714021376.9734399,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1714021376.9764442,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1714021376.9794242,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1714021376.9912519,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1714021376.9942536,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1714021376.9972892,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1714021377.0002861,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1714021377.003268,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1714021377.0062799,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1714021377.0092688,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1714021377.0122716,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1714021377.0153086,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1714021377.0183032,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1714021377.0212929,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1714021377.0242794,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1714021377.0276675,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1714021377.0322261,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1714021377.0365722,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1714021377.0409777,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1714021377.0439818,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1714021377.0469522,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1714021377.0499182,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1714021377.0609789,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1714021377.0639195,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1714021377.0668564,"name":"offline","context":{"idset":"883"}}
+{"timestamp":1714021377.0778093,"name":"offline","context":{"idset":"884"}}
+{"timestamp":1714021377.0817938,"name":"offline","context":{"idset":"7669"}}
+{"timestamp":1714021377.0854273,"name":"offline","context":{"idset":"7670"}}
+{"timestamp":1714021377.0883904,"name":"offline","context":{"idset":"7671"}}
+{"timestamp":1714021377.0912759,"name":"offline","context":{"idset":"7672"}}
+{"timestamp":1714021377.09411,"name":"offline","context":{"idset":"7673"}}
+{"timestamp":1714021377.1045084,"name":"offline","context":{"idset":"7674"}}
+{"timestamp":1714021377.1152313,"name":"offline","context":{"idset":"7675"}}
+{"timestamp":1714021377.1180427,"name":"offline","context":{"idset":"7676"}}
+{"timestamp":1714021377.1208465,"name":"offline","context":{"idset":"7677"}}
+{"timestamp":1714021377.123625,"name":"offline","context":{"idset":"7678"}}
+{"timestamp":1714021377.126405,"name":"offline","context":{"idset":"7679"}}
+{"timestamp":1714021377.1291759,"name":"offline","context":{"idset":"7680"}}
+{"timestamp":1714021377.1319149,"name":"offline","context":{"idset":"7681"}}
+{"timestamp":1714021377.134671,"name":"offline","context":{"idset":"7682"}}
+{"timestamp":1714021377.1374004,"name":"offline","context":{"idset":"7683"}}
+{"timestamp":1714021377.1401324,"name":"offline","context":{"idset":"7684"}}
+{"timestamp":1714021377.1428599,"name":"offline","context":{"idset":"7685"}}
+{"timestamp":1714021377.1455698,"name":"offline","context":{"idset":"7686"}}
+{"timestamp":1714021377.1483161,"name":"offline","context":{"idset":"7687"}}
+{"timestamp":1714021377.1510186,"name":"offline","context":{"idset":"7688"}}
+{"timestamp":1714021377.1609831,"name":"offline","context":{"idset":"7689"}}
+{"timestamp":1714021377.163646,"name":"offline","context":{"idset":"7690"}}
+{"timestamp":1714021377.1663601,"name":"offline","context":{"idset":"7691"}}
+{"timestamp":1714021377.1690233,"name":"offline","context":{"idset":"7692"}}
+{"timestamp":1714021377.1790648,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1714021377.1817007,"name":"offline","context":{"idset":"7694"}}
+{"timestamp":1714021377.1843224,"name":"offline","context":{"idset":"7695"}}
+{"timestamp":1714021377.1869426,"name":"offline","context":{"idset":"7696"}}
+{"timestamp":1714021377.1965055,"name":"offline","context":{"idset":"7697"}}
+{"timestamp":1714021377.1991036,"name":"offline","context":{"idset":"7698"}}
+{"timestamp":1714021377.2017121,"name":"offline","context":{"idset":"7699"}}
+{"timestamp":1714021377.2043161,"name":"offline","context":{"idset":"7700"}}
+{"timestamp":1714021377.213819,"name":"offline","context":{"idset":"7701"}}
+{"timestamp":1714021377.2209458,"name":"offline","context":{"idset":"7702"}}
+{"timestamp":1714021377.2239361,"name":"offline","context":{"idset":"7703"}}
+{"timestamp":1714021377.2271571,"name":"offline","context":{"idset":"7704"}}
+{"timestamp":1714021377.2309239,"name":"offline","context":{"idset":"7705"}}
+{"timestamp":1714021377.2337139,"name":"offline","context":{"idset":"7706"}}
+{"timestamp":1714021377.2365174,"name":"offline","context":{"idset":"7707"}}
+{"timestamp":1714021377.2391145,"name":"offline","context":{"idset":"7708"}}
+{"timestamp":1714021377.2416208,"name":"offline","context":{"idset":"7709"}}
+{"timestamp":1714021377.2459462,"name":"offline","context":{"idset":"7710"}}
+{"timestamp":1714021377.2484305,"name":"offline","context":{"idset":"7711"}}
+{"timestamp":1714021377.2509089,"name":"offline","context":{"idset":"7712"}}
+{"timestamp":1714021377.2541108,"name":"offline","context":{"idset":"7713"}}
+{"timestamp":1714021377.2586355,"name":"offline","context":{"idset":"7714"}}
+{"timestamp":1714021377.2635226,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1714021377.266901,"name":"offline","context":{"idset":"7716"}}
+{"timestamp":1714021377.2693563,"name":"offline","context":{"idset":"7717"}}
+{"timestamp":1714021377.2828689,"name":"offline","context":{"idset":"7718"}}
+{"timestamp":1714021377.2867224,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1714021377.2891448,"name":"offline","context":{"idset":"7720"}}
+{"timestamp":1714021377.2915561,"name":"offline","context":{"idset":"7721"}}
+{"timestamp":1714021377.2939355,"name":"offline","context":{"idset":"7722"}}
+{"timestamp":1714021377.2963247,"name":"offline","context":{"idset":"7723"}}
+{"timestamp":1714021377.2987106,"name":"offline","context":{"idset":"7724"}}
+{"timestamp":1714021377.3085113,"name":"offline","context":{"idset":"7725"}}
+{"timestamp":1714021377.3108976,"name":"offline","context":{"idset":"7726"}}
+{"timestamp":1714021377.3132374,"name":"offline","context":{"idset":"7727"}}
+{"timestamp":1714021377.3220134,"name":"offline","context":{"idset":"7728"}}
+{"timestamp":1714021377.3243425,"name":"offline","context":{"idset":"7729"}}
+{"timestamp":1714021377.3267663,"name":"offline","context":{"idset":"7730"}}
+{"timestamp":1714021377.3310311,"name":"offline","context":{"idset":"7731"}}
+{"timestamp":1714021377.3428123,"name":"offline","context":{"idset":"7732"}}
+{"timestamp":1714021377.3451471,"name":"offline","context":{"idset":"7733"}}
+{"timestamp":1714021377.3474538,"name":"offline","context":{"idset":"7734"}}
+{"timestamp":1714021377.3497403,"name":"offline","context":{"idset":"7735"}}
+{"timestamp":1714021377.3520255,"name":"offline","context":{"idset":"7736"}}
+{"timestamp":1714021377.3606737,"name":"offline","context":{"idset":"7737"}}
+{"timestamp":1714021377.3632886,"name":"offline","context":{"idset":"7738"}}
+{"timestamp":1714021377.3661044,"name":"offline","context":{"idset":"7739"}}
+{"timestamp":1714021377.3693299,"name":"offline","context":{"idset":"7740"}}
+{"timestamp":1714021377.3718314,"name":"offline","context":{"idset":"7741"}}
+{"timestamp":1714021377.3740728,"name":"offline","context":{"idset":"7742"}}
+{"timestamp":1714021377.3763638,"name":"offline","context":{"idset":"7743"}}
+{"timestamp":1714021377.3785975,"name":"offline","context":{"idset":"7744"}}
+{"timestamp":1714021377.3813779,"name":"offline","context":{"idset":"7745"}}
+{"timestamp":1714021377.3836002,"name":"offline","context":{"idset":"7746"}}
+{"timestamp":1714021377.3857963,"name":"offline","context":{"idset":"7747"}}
+{"timestamp":1714021377.3881104,"name":"offline","context":{"idset":"7748"}}
+{"timestamp":1714021377.3903644,"name":"offline","context":{"idset":"7749"}}
+{"timestamp":1714021377.3925614,"name":"offline","context":{"idset":"7750"}}
+{"timestamp":1714021377.3947432,"name":"offline","context":{"idset":"7751"}}
+{"timestamp":1714021377.3969431,"name":"offline","context":{"idset":"7752"}}
+{"timestamp":1714021377.3991206,"name":"offline","context":{"idset":"7753"}}
+{"timestamp":1714021377.4012928,"name":"offline","context":{"idset":"7754"}}
+{"timestamp":1714021377.4093852,"name":"offline","context":{"idset":"7755"}}
+{"timestamp":1714021377.4115319,"name":"offline","context":{"idset":"7756"}}
+{"timestamp":1714021377.4136949,"name":"offline","context":{"idset":"7757"}}
+{"timestamp":1714021377.4158525,"name":"offline","context":{"idset":"7758"}}
+{"timestamp":1714021377.4239826,"name":"offline","context":{"idset":"7759"}}
+{"timestamp":1714021377.4261231,"name":"offline","context":{"idset":"7760"}}
+{"timestamp":1714021377.4282646,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1714021377.4303832,"name":"offline","context":{"idset":"7762"}}
+{"timestamp":1714021377.4324896,"name":"offline","context":{"idset":"7763"}}
+{"timestamp":1714021377.4403152,"name":"offline","context":{"idset":"7764"}}
+{"timestamp":1714021377.4424028,"name":"offline","context":{"idset":"7765"}}
+{"timestamp":1714021377.4445291,"name":"offline","context":{"idset":"7766"}}
+{"timestamp":1714021377.446614,"name":"offline","context":{"idset":"7767"}}
+{"timestamp":1714021377.4487019,"name":"offline","context":{"idset":"7768"}}
+{"timestamp":1714021377.4507632,"name":"offline","context":{"idset":"7769"}}
+{"timestamp":1714021377.452848,"name":"offline","context":{"idset":"7770"}}
+{"timestamp":1714021377.4549496,"name":"offline","context":{"idset":"7771"}}
+{"timestamp":1714021377.4570277,"name":"offline","context":{"idset":"7772"}}
+{"timestamp":1714021377.4590895,"name":"offline","context":{"idset":"7773"}}
+{"timestamp":1714021377.4611318,"name":"offline","context":{"idset":"7774"}}
+{"timestamp":1714021377.4631748,"name":"offline","context":{"idset":"7775"}}
+{"timestamp":1714021377.4652243,"name":"offline","context":{"idset":"7776"}}
+{"timestamp":1714021377.4672768,"name":"offline","context":{"idset":"7777"}}
+{"timestamp":1714021377.4693112,"name":"offline","context":{"idset":"7778"}}
+{"timestamp":1714021377.4713678,"name":"offline","context":{"idset":"7779"}}
+{"timestamp":1714021377.479058,"name":"offline","context":{"idset":"7780"}}
+{"timestamp":1714021377.4811854,"name":"offline","context":{"idset":"7781"}}
+{"timestamp":1714021377.4832036,"name":"offline","context":{"idset":"7782"}}
+{"timestamp":1714021377.485213,"name":"offline","context":{"idset":"7783"}}
+{"timestamp":1714021377.4927859,"name":"offline","context":{"idset":"7784"}}
+{"timestamp":1714021377.4948196,"name":"offline","context":{"idset":"7785"}}
+{"timestamp":1714021377.4968164,"name":"offline","context":{"idset":"7786"}}
+{"timestamp":1714021377.4987991,"name":"offline","context":{"idset":"7787"}}
+{"timestamp":1714021377.5007756,"name":"offline","context":{"idset":"7788"}}
+{"timestamp":1714021377.5081899,"name":"offline","context":{"idset":"7789"}}
+{"timestamp":1714021377.5101542,"name":"offline","context":{"idset":"7790"}}
+{"timestamp":1714021377.5121515,"name":"offline","context":{"idset":"7791"}}
+{"timestamp":1714021377.5141661,"name":"offline","context":{"idset":"7792"}}
+{"timestamp":1714021377.516129,"name":"offline","context":{"idset":"7793"}}
+{"timestamp":1714021377.5180802,"name":"offline","context":{"idset":"7794"}}
+{"timestamp":1714021377.5254889,"name":"offline","context":{"idset":"7795"}}
+{"timestamp":1714021377.527482,"name":"offline","context":{"idset":"7796"}}
+{"timestamp":1714021377.5294993,"name":"offline","context":{"idset":"7797"}}
+{"timestamp":1714021377.5314572,"name":"offline","context":{"idset":"7798"}}
+{"timestamp":1714021377.5334427,"name":"offline","context":{"idset":"7799"}}
+{"timestamp":1714021377.5353861,"name":"offline","context":{"idset":"7800"}}
+{"timestamp":1714021377.5372987,"name":"offline","context":{"idset":"7801"}}
+{"timestamp":1714021377.5392237,"name":"offline","context":{"idset":"7802"}}
+{"timestamp":1714021377.5411363,"name":"offline","context":{"idset":"7803"}}
+{"timestamp":1714021377.5430403,"name":"offline","context":{"idset":"7804"}}
+{"timestamp":1714021377.5449393,"name":"offline","context":{"idset":"7805"}}
+{"timestamp":1714021377.5468309,"name":"offline","context":{"idset":"7806"}}
+{"timestamp":1714021377.5487187,"name":"offline","context":{"idset":"7807"}}
+{"timestamp":1714021377.5506065,"name":"offline","context":{"idset":"7808"}}
+{"timestamp":1714021377.5524938,"name":"offline","context":{"idset":"7809"}}
+{"timestamp":1714021377.5543714,"name":"offline","context":{"idset":"7810"}}
+{"timestamp":1714021377.5562682,"name":"offline","context":{"idset":"7811"}}
+{"timestamp":1714021377.5634141,"name":"offline","context":{"idset":"7812"}}
+{"timestamp":1714021377.565289,"name":"offline","context":{"idset":"7813"}}
+{"timestamp":1714021377.5671303,"name":"offline","context":{"idset":"7814"}}
+{"timestamp":1714021377.5689845,"name":"offline","context":{"idset":"7817"}}
+{"timestamp":1714021377.5708337,"name":"offline","context":{"idset":"7818"}}
+{"timestamp":1714021377.5777988,"name":"offline","context":{"idset":"7819"}}
+{"timestamp":1714021377.5796676,"name":"offline","context":{"idset":"7820"}}
+{"timestamp":1714021377.5815272,"name":"offline","context":{"idset":"7821"}}
+{"timestamp":1714021377.5833502,"name":"offline","context":{"idset":"7822"}}
+{"timestamp":1714021377.5851774,"name":"offline","context":{"idset":"7823"}}
+{"timestamp":1714021377.5920827,"name":"offline","context":{"idset":"7824"}}
+{"timestamp":1714021377.5939019,"name":"offline","context":{"idset":"7825"}}
+{"timestamp":1714021377.5957019,"name":"offline","context":{"idset":"7826"}}
+{"timestamp":1714021377.5974858,"name":"offline","context":{"idset":"7827"}}
+{"timestamp":1714021377.5992842,"name":"offline","context":{"idset":"7828"}}
+{"timestamp":1714021377.6010685,"name":"offline","context":{"idset":"7829"}}
+{"timestamp":1714021377.6078486,"name":"offline","context":{"idset":"7830"}}
+{"timestamp":1714021377.6096249,"name":"offline","context":{"idset":"7831"}}
+{"timestamp":1714021377.6113908,"name":"offline","context":{"idset":"7832"}}
+{"timestamp":1714021377.6181128,"name":"offline","context":{"idset":"7833"}}
+{"timestamp":1714021377.6198821,"name":"offline","context":{"idset":"7834"}}
+{"timestamp":1714021377.6216302,"name":"offline","context":{"idset":"7835"}}
+{"timestamp":1714021377.6233945,"name":"offline","context":{"idset":"7836"}}
+{"timestamp":1714021377.6251407,"name":"offline","context":{"idset":"7837"}}
+{"timestamp":1714021377.6318054,"name":"offline","context":{"idset":"7838"}}
+{"timestamp":1714021377.6335409,"name":"offline","context":{"idset":"7839"}}
+{"timestamp":1714021377.6352947,"name":"offline","context":{"idset":"7840"}}
+{"timestamp":1714021377.6370444,"name":"offline","context":{"idset":"7841"}}
+{"timestamp":1714021377.638773,"name":"offline","context":{"idset":"7842"}}
+{"timestamp":1714021377.6405101,"name":"offline","context":{"idset":"7843"}}
+{"timestamp":1714021377.6422253,"name":"offline","context":{"idset":"7844"}}
+{"timestamp":1714021377.6487861,"name":"offline","context":{"idset":"7845"}}
+{"timestamp":1714021377.6504996,"name":"offline","context":{"idset":"7846"}}
+{"timestamp":1714021377.6522069,"name":"offline","context":{"idset":"7847"}}
+{"timestamp":1714021377.6539199,"name":"offline","context":{"idset":"7848"}}
+{"timestamp":1714021377.6604142,"name":"offline","context":{"idset":"7849"}}
+{"timestamp":1714021377.662112,"name":"offline","context":{"idset":"7850"}}
+{"timestamp":1714021377.6637988,"name":"offline","context":{"idset":"7851"}}
+{"timestamp":1714021377.6654851,"name":"offline","context":{"idset":"7852"}}
+{"timestamp":1714021377.6719072,"name":"offline","context":{"idset":"7853"}}
+{"timestamp":1714021377.6735833,"name":"offline","context":{"idset":"7854"}}
+{"timestamp":1714021377.6752546,"name":"offline","context":{"idset":"7855"}}
+{"timestamp":1714021377.6769295,"name":"offline","context":{"idset":"7856"}}
+{"timestamp":1714021377.6785989,"name":"offline","context":{"idset":"7857"}}
+{"timestamp":1714021377.6849511,"name":"offline","context":{"idset":"7858"}}
+{"timestamp":1714021377.6866086,"name":"offline","context":{"idset":"7859"}}
+{"timestamp":1714021377.688272,"name":"offline","context":{"idset":"7860"}}
+{"timestamp":1714021377.6899495,"name":"offline","context":{"idset":"7861"}}
+{"timestamp":1714021377.6916461,"name":"offline","context":{"idset":"7862"}}
+{"timestamp":1714021377.6980174,"name":"offline","context":{"idset":"7863"}}
+{"timestamp":1714021377.6996732,"name":"offline","context":{"idset":"7864"}}
+{"timestamp":1714021377.7013505,"name":"offline","context":{"idset":"7865"}}
+{"timestamp":1714021377.7030191,"name":"offline","context":{"idset":"7866"}}
+{"timestamp":1714021377.7046807,"name":"offline","context":{"idset":"7867"}}
+{"timestamp":1714021377.7110267,"name":"offline","context":{"idset":"7868"}}
+{"timestamp":1714021377.712729,"name":"offline","context":{"idset":"7869"}}
+{"timestamp":1714021377.7143836,"name":"offline","context":{"idset":"7870"}}
+{"timestamp":1714021377.7160366,"name":"offline","context":{"idset":"7871"}}
+{"timestamp":1714021377.7176938,"name":"offline","context":{"idset":"7872"}}
+{"timestamp":1714021377.7240348,"name":"offline","context":{"idset":"7873"}}
+{"timestamp":1714021377.7256899,"name":"offline","context":{"idset":"7874"}}
+{"timestamp":1714021377.7273283,"name":"offline","context":{"idset":"7875"}}
+{"timestamp":1714021377.7289972,"name":"offline","context":{"idset":"7876"}}
+{"timestamp":1714021377.7306392,"name":"offline","context":{"idset":"7877"}}
+{"timestamp":1714021377.7322941,"name":"offline","context":{"idset":"7878"}}
+{"timestamp":1714021377.7386522,"name":"offline","context":{"idset":"7879"}}
+{"timestamp":1714021377.7402952,"name":"offline","context":{"idset":"7880"}}
+{"timestamp":1714021377.7419484,"name":"offline","context":{"idset":"7881"}}
+{"timestamp":1714021377.7435896,"name":"offline","context":{"idset":"7882"}}
+{"timestamp":1714021377.7452416,"name":"offline","context":{"idset":"7883"}}
+{"timestamp":1714021377.7515731,"name":"offline","context":{"idset":"7884"}}
+{"timestamp":1714021377.7532222,"name":"offline","context":{"idset":"7885"}}
+{"timestamp":1714021377.7548766,"name":"offline","context":{"idset":"7886"}}
+{"timestamp":1714021377.7565215,"name":"offline","context":{"idset":"7887"}}
+{"timestamp":1714021377.7581694,"name":"offline","context":{"idset":"7888"}}
+{"timestamp":1714021377.7645049,"name":"offline","context":{"idset":"7889"}}
+{"timestamp":1714021377.7661614,"name":"offline","context":{"idset":"7890"}}
+{"timestamp":1714021377.7678101,"name":"offline","context":{"idset":"7891"}}
+{"timestamp":1714021377.7694554,"name":"offline","context":{"idset":"7892"}}
+{"timestamp":1714021377.7710891,"name":"offline","context":{"idset":"7893"}}
+{"timestamp":1714021377.7727318,"name":"offline","context":{"idset":"7894"}}
+{"timestamp":1714021377.7791431,"name":"offline","context":{"idset":"7895"}}
+{"timestamp":1714021377.7810848,"name":"offline","context":{"idset":"7896"}}
+{"timestamp":1714021377.7828398,"name":"offline","context":{"idset":"7897"}}
+{"timestamp":1714021377.7850807,"name":"offline","context":{"idset":"7898"}}
+{"timestamp":1714021377.7876813,"name":"offline","context":{"idset":"7899"}}
+{"timestamp":1714021377.7893751,"name":"offline","context":{"idset":"7900"}}
+{"timestamp":1714021377.7910056,"name":"offline","context":{"idset":"7901"}}
+{"timestamp":1714021377.7974057,"name":"offline","context":{"idset":"7902"}}
+{"timestamp":1714021377.7990603,"name":"offline","context":{"idset":"7903"}}
+{"timestamp":1714021377.8007414,"name":"offline","context":{"idset":"7904"}}
+{"timestamp":1714021377.802392,"name":"offline","context":{"idset":"7905"}}
+{"timestamp":1714021377.8040388,"name":"offline","context":{"idset":"7906"}}
+{"timestamp":1714021377.8056741,"name":"offline","context":{"idset":"7907"}}
+{"timestamp":1714021377.8073049,"name":"offline","context":{"idset":"7908"}}
+{"timestamp":1714021377.8089454,"name":"offline","context":{"idset":"7909"}}
+{"timestamp":1714021377.8105879,"name":"offline","context":{"idset":"7910"}}
+{"timestamp":1714021377.8122234,"name":"offline","context":{"idset":"7911"}}
+{"timestamp":1714021377.8138549,"name":"offline","context":{"idset":"7912"}}
+{"timestamp":1714021377.815491,"name":"offline","context":{"idset":"7913"}}
+{"timestamp":1714021377.8171227,"name":"offline","context":{"idset":"7914"}}
+{"timestamp":1714021377.8187537,"name":"offline","context":{"idset":"7915"}}
+{"timestamp":1714021377.8203909,"name":"offline","context":{"idset":"7916"}}
+{"timestamp":1714021377.8220141,"name":"offline","context":{"idset":"7917"}}
+{"timestamp":1714021377.8236403,"name":"offline","context":{"idset":"7918"}}
+{"timestamp":1714021377.8252826,"name":"offline","context":{"idset":"7919"}}
+{"timestamp":1714021377.82692,"name":"offline","context":{"idset":"7920"}}
+{"timestamp":1714021377.8285522,"name":"offline","context":{"idset":"7921"}}
+{"timestamp":1714021377.8301966,"name":"offline","context":{"idset":"7922"}}
+{"timestamp":1714021377.831841,"name":"offline","context":{"idset":"7923"}}
+{"timestamp":1714021377.8334751,"name":"offline","context":{"idset":"7924"}}
+{"timestamp":1714021377.8399577,"name":"offline","context":{"idset":"7925"}}
+{"timestamp":1714021377.8416059,"name":"offline","context":{"idset":"7926"}}
+{"timestamp":1714021377.8480124,"name":"offline","context":{"idset":"7927"}}
+{"timestamp":1714021377.8496573,"name":"offline","context":{"idset":"7928"}}
+{"timestamp":1714021377.85131,"name":"offline","context":{"idset":"7929"}}
+{"timestamp":1714021377.852957,"name":"offline","context":{"idset":"7930"}}
+{"timestamp":1714021377.8546,"name":"offline","context":{"idset":"7931"}}
+{"timestamp":1714021377.8562448,"name":"offline","context":{"idset":"7932"}}
+{"timestamp":1714021377.8626425,"name":"offline","context":{"idset":"7933"}}
+{"timestamp":1714021377.8642747,"name":"offline","context":{"idset":"7934"}}
+{"timestamp":1714021377.8659179,"name":"offline","context":{"idset":"7935"}}
+{"timestamp":1714021377.8675652,"name":"offline","context":{"idset":"7936"}}
+{"timestamp":1714021377.8739235,"name":"offline","context":{"idset":"7937"}}
+{"timestamp":1714021377.8755329,"name":"offline","context":{"idset":"7938"}}
+{"timestamp":1714021377.8771493,"name":"offline","context":{"idset":"7939"}}
+{"timestamp":1714021377.8787553,"name":"offline","context":{"idset":"7940"}}
+{"timestamp":1714021377.8803756,"name":"offline","context":{"idset":"7941"}}
+{"timestamp":1714021377.8819938,"name":"offline","context":{"idset":"7942"}}
+{"timestamp":1714021377.8836069,"name":"offline","context":{"idset":"7943"}}
+{"timestamp":1714021377.8852282,"name":"offline","context":{"idset":"7944"}}
+{"timestamp":1714021377.8915427,"name":"offline","context":{"idset":"7945"}}
+{"timestamp":1714021377.8931689,"name":"offline","context":{"idset":"7946"}}
+{"timestamp":1714021377.8947885,"name":"offline","context":{"idset":"7947"}}
+{"timestamp":1714021377.9010878,"name":"offline","context":{"idset":"7948"}}
+{"timestamp":1714021377.9027014,"name":"offline","context":{"idset":"7949"}}
+{"timestamp":1714021377.9043155,"name":"offline","context":{"idset":"7950"}}
+{"timestamp":1714021377.9106116,"name":"offline","context":{"idset":"7951"}}
+{"timestamp":1714021377.9122214,"name":"offline","context":{"idset":"7952"}}
+{"timestamp":1714021377.9185121,"name":"offline","context":{"idset":"7953"}}
+{"timestamp":1714021377.920114,"name":"offline","context":{"idset":"7954"}}
+{"timestamp":1714021377.9217892,"name":"offline","context":{"idset":"7955"}}
+{"timestamp":1714021377.9234157,"name":"offline","context":{"idset":"7956"}}
+{"timestamp":1714021377.9250398,"name":"offline","context":{"idset":"7957"}}
+{"timestamp":1714021377.926651,"name":"offline","context":{"idset":"7958"}}
+{"timestamp":1714021377.9282722,"name":"offline","context":{"idset":"7959"}}
+{"timestamp":1714021377.9298916,"name":"offline","context":{"idset":"7960"}}
+{"timestamp":1714021377.9314976,"name":"offline","context":{"idset":"7961"}}
+{"timestamp":1714021377.9331121,"name":"offline","context":{"idset":"7962"}}
+{"timestamp":1714021377.9347286,"name":"offline","context":{"idset":"7963"}}
+{"timestamp":1714021377.936343,"name":"offline","context":{"idset":"7964"}}
+{"timestamp":1714021377.9379587,"name":"offline","context":{"idset":"7965"}}
+{"timestamp":1714021377.9395788,"name":"offline","context":{"idset":"7966"}}
+{"timestamp":1714021377.9411924,"name":"offline","context":{"idset":"7967"}}
+{"timestamp":1714021377.9428082,"name":"offline","context":{"idset":"7968"}}
+{"timestamp":1714021377.9444389,"name":"offline","context":{"idset":"7969"}}
+{"timestamp":1714021377.9508209,"name":"offline","context":{"idset":"7970"}}
+{"timestamp":1714021377.9524443,"name":"offline","context":{"idset":"7971"}}
+{"timestamp":1714021377.9540718,"name":"offline","context":{"idset":"7972"}}
+{"timestamp":1714021377.9556868,"name":"offline","context":{"idset":"7973"}}
+{"timestamp":1714021377.9572935,"name":"offline","context":{"idset":"7974"}}
+{"timestamp":1714021377.9636779,"name":"offline","context":{"idset":"7975"}}
+{"timestamp":1714021377.9652989,"name":"offline","context":{"idset":"7976"}}
+{"timestamp":1714021377.9669185,"name":"offline","context":{"idset":"7977"}}
+{"timestamp":1714021377.9685388,"name":"offline","context":{"idset":"7978"}}
+{"timestamp":1714021377.9701593,"name":"offline","context":{"idset":"7979"}}
+{"timestamp":1714021377.9717803,"name":"offline","context":{"idset":"7980"}}
+{"timestamp":1714021377.9781327,"name":"offline","context":{"idset":"7981"}}
+{"timestamp":1714021377.979744,"name":"offline","context":{"idset":"7982"}}
+{"timestamp":1714021377.9813514,"name":"offline","context":{"idset":"7983"}}
+{"timestamp":1714021377.9829664,"name":"offline","context":{"idset":"7984"}}
+{"timestamp":1714021377.9893315,"name":"offline","context":{"idset":"7985"}}
+{"timestamp":1714021377.9909365,"name":"offline","context":{"idset":"7986"}}
+{"timestamp":1714021377.9925425,"name":"offline","context":{"idset":"7987"}}
+{"timestamp":1714021377.994144,"name":"offline","context":{"idset":"7988"}}
+{"timestamp":1714021377.9957492,"name":"offline","context":{"idset":"7989"}}
+{"timestamp":1714021377.9973576,"name":"offline","context":{"idset":"7990"}}
+{"timestamp":1714021377.9989583,"name":"offline","context":{"idset":"7991"}}
+{"timestamp":1714021378.0005703,"name":"offline","context":{"idset":"7992"}}
+{"timestamp":1714021378.0021672,"name":"offline","context":{"idset":"7993"}}
+{"timestamp":1714021378.0037642,"name":"offline","context":{"idset":"7994"}}
+{"timestamp":1714021378.0053718,"name":"offline","context":{"idset":"7995"}}
+{"timestamp":1714021378.0069773,"name":"offline","context":{"idset":"7996"}}
+{"timestamp":1714021378.008574,"name":"offline","context":{"idset":"7997"}}
+{"timestamp":1714021378.0101702,"name":"offline","context":{"idset":"7998"}}
+{"timestamp":1714021378.0117707,"name":"offline","context":{"idset":"7999"}}
+{"timestamp":1714021378.0133712,"name":"offline","context":{"idset":"8000"}}
+{"timestamp":1714021378.0149744,"name":"offline","context":{"idset":"8001"}}
+{"timestamp":1714021378.016567,"name":"offline","context":{"idset":"8002"}}
+{"timestamp":1714021378.0181663,"name":"offline","context":{"idset":"8003"}}
+{"timestamp":1714021378.0197577,"name":"offline","context":{"idset":"8004"}}
+{"timestamp":1714021378.0213628,"name":"offline","context":{"idset":"8005"}}
+{"timestamp":1714021378.0229647,"name":"offline","context":{"idset":"8006"}}
+{"timestamp":1714021378.0293236,"name":"offline","context":{"idset":"8007"}}
+{"timestamp":1714021378.0309262,"name":"offline","context":{"idset":"8008"}}
+{"timestamp":1714021378.032516,"name":"offline","context":{"idset":"8009"}}
+{"timestamp":1714021378.0341158,"name":"offline","context":{"idset":"8010"}}
+{"timestamp":1714021378.0357084,"name":"offline","context":{"idset":"8011"}}
+{"timestamp":1714021378.0429943,"name":"offline","context":{"idset":"8012"}}
+{"timestamp":1714021378.0453756,"name":"offline","context":{"idset":"8013"}}
+{"timestamp":1714021378.0469761,"name":"offline","context":{"idset":"8014"}}
+{"timestamp":1714021378.0485573,"name":"offline","context":{"idset":"8015"}}
+{"timestamp":1714021378.0501428,"name":"offline","context":{"idset":"8016"}}
+{"timestamp":1714021378.051734,"name":"offline","context":{"idset":"8017"}}
+{"timestamp":1714021378.0580804,"name":"offline","context":{"idset":"8018"}}
+{"timestamp":1714021378.0596638,"name":"offline","context":{"idset":"8019"}}
+{"timestamp":1714021378.0612569,"name":"offline","context":{"idset":"8020"}}
+{"timestamp":1714021378.0628395,"name":"offline","context":{"idset":"8021"}}
+{"timestamp":1714021378.0644257,"name":"offline","context":{"idset":"8022"}}
+{"timestamp":1714021378.0660093,"name":"offline","context":{"idset":"8023"}}
+{"timestamp":1714021378.0676031,"name":"offline","context":{"idset":"8024"}}
+{"timestamp":1714021378.0739217,"name":"offline","context":{"idset":"8025"}}
+{"timestamp":1714021378.0755217,"name":"offline","context":{"idset":"8026"}}
+{"timestamp":1714021378.0771129,"name":"offline","context":{"idset":"8027"}}
+{"timestamp":1714021378.0834193,"name":"offline","context":{"idset":"8028"}}
+{"timestamp":1714021378.0850027,"name":"offline","context":{"idset":"8029"}}
+{"timestamp":1714021378.0865827,"name":"offline","context":{"idset":"8030"}}
+{"timestamp":1714021378.0929294,"name":"offline","context":{"idset":"8031"}}
+{"timestamp":1714021378.0945163,"name":"offline","context":{"idset":"8032"}}
+{"timestamp":1714021378.0961189,"name":"offline","context":{"idset":"8033"}}
+{"timestamp":1714021378.0976882,"name":"offline","context":{"idset":"8034"}}
+{"timestamp":1714021378.1040003,"name":"offline","context":{"idset":"8035"}}
+{"timestamp":1714021378.105577,"name":"offline","context":{"idset":"8036"}}
+{"timestamp":1714021378.1118896,"name":"offline","context":{"idset":"8037"}}
+{"timestamp":1714021378.1134694,"name":"offline","context":{"idset":"8038"}}
+{"timestamp":1714021378.1150475,"name":"offline","context":{"idset":"8039"}}
+{"timestamp":1714021378.1166213,"name":"offline","context":{"idset":"8040"}}
+{"timestamp":1714021378.1182079,"name":"offline","context":{"idset":"8041"}}
+{"timestamp":1714021378.1245186,"name":"offline","context":{"idset":"8042"}}
+{"timestamp":1714021378.126101,"name":"offline","context":{"idset":"8043"}}
+{"timestamp":1714021378.1276734,"name":"offline","context":{"idset":"8044"}}
+{"timestamp":1714021378.1292515,"name":"offline","context":{"idset":"8045"}}
+{"timestamp":1714021378.1308351,"name":"offline","context":{"idset":"8046"}}
+{"timestamp":1714021378.1371493,"name":"offline","context":{"idset":"8047"}}
+{"timestamp":1714021378.1387093,"name":"offline","context":{"idset":"8048"}}
+{"timestamp":1714021378.1402738,"name":"offline","context":{"idset":"8049"}}
+{"timestamp":1714021378.141835,"name":"offline","context":{"idset":"8050"}}
+{"timestamp":1714021378.1433947,"name":"offline","context":{"idset":"8051"}}
+{"timestamp":1714021378.1496706,"name":"offline","context":{"idset":"8052"}}
+{"timestamp":1714021378.1512189,"name":"offline","context":{"idset":"8181"}}
+{"timestamp":1714021378.1527631,"name":"offline","context":{"idset":"8182"}}
+{"timestamp":1714021378.1543217,"name":"offline","context":{"idset":"8183"}}
+{"timestamp":1714021378.1558774,"name":"offline","context":{"idset":"8184"}}
+{"timestamp":1714021378.1574306,"name":"offline","context":{"idset":"8185"}}
+{"timestamp":1714021378.1637197,"name":"offline","context":{"idset":"8186"}}
+{"timestamp":1714021378.1652732,"name":"offline","context":{"idset":"8187"}}
+{"timestamp":1714021378.1668327,"name":"offline","context":{"idset":"8188"}}
+{"timestamp":1714021378.1683934,"name":"offline","context":{"idset":"8189"}}
+{"timestamp":1714021378.1706498,"name":"offline","context":{"idset":"8190"}}
+{"timestamp":1714021378.1722093,"name":"offline","context":{"idset":"8191"}}
+{"timestamp":1714021378.1785042,"name":"offline","context":{"idset":"8192"}}
+{"timestamp":1714021378.1800516,"name":"offline","context":{"idset":"8193"}}
+{"timestamp":1714021378.1815994,"name":"offline","context":{"idset":"8194"}}
+{"timestamp":1714021378.1831493,"name":"offline","context":{"idset":"8195"}}
+{"timestamp":1714021378.184701,"name":"offline","context":{"idset":"8196"}}
+{"timestamp":1714021378.1862488,"name":"offline","context":{"idset":"8197"}}
+{"timestamp":1714021378.1925375,"name":"offline","context":{"idset":"8198"}}
+{"timestamp":1714021378.1940811,"name":"offline","context":{"idset":"8199"}}
+{"timestamp":1714021378.1956303,"name":"offline","context":{"idset":"8200"}}
+{"timestamp":1714021378.197185,"name":"offline","context":{"idset":"8202"}}
+{"timestamp":1714021378.1987262,"name":"offline","context":{"idset":"8203"}}
+{"timestamp":1714021378.2050164,"name":"offline","context":{"idset":"8204"}}
+{"timestamp":1714021378.2065754,"name":"offline","context":{"idset":"8205"}}
+{"timestamp":1714021378.2081306,"name":"offline","context":{"idset":"8206"}}
+{"timestamp":1714021378.2096753,"name":"offline","context":{"idset":"8207"}}
+{"timestamp":1714021378.2112215,"name":"offline","context":{"idset":"8208"}}
+{"timestamp":1714021378.2127554,"name":"offline","context":{"idset":"8209"}}
+{"timestamp":1714021378.2142873,"name":"offline","context":{"idset":"8210"}}
+{"timestamp":1714021378.2206295,"name":"offline","context":{"idset":"8211"}}
+{"timestamp":1714021378.2221766,"name":"offline","context":{"idset":"8212"}}
+{"timestamp":1714021378.223717,"name":"offline","context":{"idset":"8213"}}
+{"timestamp":1714021378.2252522,"name":"offline","context":{"idset":"8214"}}
+{"timestamp":1714021378.2267928,"name":"offline","context":{"idset":"8215"}}
+{"timestamp":1714021378.2283368,"name":"offline","context":{"idset":"8216"}}
+{"timestamp":1714021378.2298758,"name":"offline","context":{"idset":"8217"}}
+{"timestamp":1714021378.2314098,"name":"offline","context":{"idset":"8218"}}
+{"timestamp":1714021378.2329526,"name":"offline","context":{"idset":"8219"}}
+{"timestamp":1714021378.2344906,"name":"offline","context":{"idset":"8220"}}
+{"timestamp":1714021378.2360766,"name":"offline","context":{"idset":"8221"}}
+{"timestamp":1714021378.2376325,"name":"offline","context":{"idset":"8222"}}
+{"timestamp":1714021378.239187,"name":"offline","context":{"idset":"8223"}}
+{"timestamp":1714021378.2407184,"name":"offline","context":{"idset":"8224"}}
+{"timestamp":1714021378.2422609,"name":"offline","context":{"idset":"8225"}}
+{"timestamp":1714021378.243788,"name":"offline","context":{"idset":"8226"}}
+{"timestamp":1714021378.2453313,"name":"offline","context":{"idset":"8227"}}
+{"timestamp":1714021378.2468729,"name":"offline","context":{"idset":"8228"}}
+{"timestamp":1714021378.2484057,"name":"offline","context":{"idset":"8229"}}
+{"timestamp":1714021378.2499526,"name":"offline","context":{"idset":"8230"}}
+{"timestamp":1714021378.2514884,"name":"offline","context":{"idset":"8231"}}
+{"timestamp":1714021378.2577889,"name":"offline","context":{"idset":"8232"}}
+{"timestamp":1714021378.2593293,"name":"offline","context":{"idset":"8235"}}
+{"timestamp":1714021378.2608697,"name":"offline","context":{"idset":"8236"}}
+{"timestamp":1714021378.2624021,"name":"offline","context":{"idset":"8238"}}
+{"timestamp":1714021378.263936,"name":"offline","context":{"idset":"8239"}}
+{"timestamp":1714021378.2654591,"name":"offline","context":{"idset":"8240"}}
+{"timestamp":1714021378.2717438,"name":"offline","context":{"idset":"8241"}}
+{"timestamp":1714021378.2732744,"name":"offline","context":{"idset":"8242"}}
+{"timestamp":1714021378.2747884,"name":"offline","context":{"idset":"8243"}}
+{"timestamp":1714021378.2763221,"name":"offline","context":{"idset":"8244"}}
+{"timestamp":1714021378.2778504,"name":"offline","context":{"idset":"8245"}}
+{"timestamp":1714021378.2793779,"name":"offline","context":{"idset":"8246"}}
+{"timestamp":1714021378.285646,"name":"offline","context":{"idset":"8247"}}
+{"timestamp":1714021378.2871695,"name":"offline","context":{"idset":"8248"}}
+{"timestamp":1714021378.2886932,"name":"offline","context":{"idset":"8249"}}
+{"timestamp":1714021378.2902238,"name":"offline","context":{"idset":"8250"}}
+{"timestamp":1714021378.2917461,"name":"offline","context":{"idset":"8251"}}
+{"timestamp":1714021378.2932763,"name":"offline","context":{"idset":"8252"}}
+{"timestamp":1714021378.2947967,"name":"offline","context":{"idset":"8253"}}
+{"timestamp":1714021378.2963226,"name":"offline","context":{"idset":"8254"}}
+{"timestamp":1714021378.2978504,"name":"offline","context":{"idset":"8255"}}
+{"timestamp":1714021378.2993686,"name":"offline","context":{"idset":"8256"}}
+{"timestamp":1714021378.3008959,"name":"offline","context":{"idset":"8257"}}
+{"timestamp":1714021378.3024144,"name":"offline","context":{"idset":"8258"}}
+{"timestamp":1714021378.3039398,"name":"offline","context":{"idset":"8259"}}
+{"timestamp":1714021378.305459,"name":"offline","context":{"idset":"8260"}}
+{"timestamp":1714021378.3069794,"name":"offline","context":{"idset":"8261"}}
+{"timestamp":1714021378.3085294,"name":"offline","context":{"idset":"8262"}}
+{"timestamp":1714021378.3100498,"name":"offline","context":{"idset":"8263"}}
+{"timestamp":1714021378.3115692,"name":"offline","context":{"idset":"8264"}}
+{"timestamp":1714021378.3130949,"name":"offline","context":{"idset":"8265"}}
+{"timestamp":1714021378.3146117,"name":"offline","context":{"idset":"8266"}}
+{"timestamp":1714021378.3161304,"name":"offline","context":{"idset":"8267"}}
+{"timestamp":1714021378.3176496,"name":"offline","context":{"idset":"8268"}}
+{"timestamp":1714021378.3191671,"name":"offline","context":{"idset":"8269"}}
+{"timestamp":1714021378.3206782,"name":"offline","context":{"idset":"8270"}}
+{"timestamp":1714021378.3221974,"name":"offline","context":{"idset":"8271"}}
+{"timestamp":1714021378.3284395,"name":"offline","context":{"idset":"8272"}}
+{"timestamp":1714021378.3299441,"name":"offline","context":{"idset":"8273"}}
+{"timestamp":1714021378.3314567,"name":"offline","context":{"idset":"8274"}}
+{"timestamp":1714021378.3329728,"name":"offline","context":{"idset":"8275"}}
+{"timestamp":1714021378.3344851,"name":"offline","context":{"idset":"8276"}}
+{"timestamp":1714021378.3362589,"name":"offline","context":{"idset":"8277"}}
+{"timestamp":1714021378.3445194,"name":"offline","context":{"idset":"8278"}}
+{"timestamp":1714021378.3460863,"name":"offline","context":{"idset":"8279"}}
+{"timestamp":1714021378.3476014,"name":"offline","context":{"idset":"8280"}}
+{"timestamp":1714021378.3491213,"name":"offline","context":{"idset":"8281"}}
+{"timestamp":1714021378.3506327,"name":"offline","context":{"idset":"8282"}}
+{"timestamp":1714021378.3521492,"name":"offline","context":{"idset":"8283"}}
+{"timestamp":1714021378.3583848,"name":"offline","context":{"idset":"8284"}}
+{"timestamp":1714021378.3598962,"name":"offline","context":{"idset":"8285"}}
+{"timestamp":1714021378.3613963,"name":"offline","context":{"idset":"8286"}}
+{"timestamp":1714021378.3629081,"name":"offline","context":{"idset":"8287"}}
+{"timestamp":1714021378.3644145,"name":"offline","context":{"idset":"8288"}}
+{"timestamp":1714021378.365921,"name":"offline","context":{"idset":"8289"}}
+{"timestamp":1714021378.3674264,"name":"offline","context":{"idset":"8290"}}
+{"timestamp":1714021378.3689454,"name":"offline","context":{"idset":"8291"}}
+{"timestamp":1714021378.3704851,"name":"offline","context":{"idset":"8292"}}
+{"timestamp":1714021378.3720076,"name":"offline","context":{"idset":"8293"}}
+{"timestamp":1714021378.3735111,"name":"offline","context":{"idset":"8294"}}
+{"timestamp":1714021378.3750122,"name":"offline","context":{"idset":"8295"}}
+{"timestamp":1714021378.3765328,"name":"offline","context":{"idset":"8296"}}
+{"timestamp":1714021378.3780746,"name":"offline","context":{"idset":"8297"}}
+{"timestamp":1714021378.3795702,"name":"offline","context":{"idset":"8298"}}
+{"timestamp":1714021378.3810699,"name":"offline","context":{"idset":"8299"}}
+{"timestamp":1714021378.3825698,"name":"offline","context":{"idset":"8300"}}
+{"timestamp":1714021378.3840683,"name":"offline","context":{"idset":"8301"}}
+{"timestamp":1714021378.3855748,"name":"offline","context":{"idset":"8302"}}
+{"timestamp":1714021378.3870804,"name":"offline","context":{"idset":"8303"}}
+{"timestamp":1714021378.3885803,"name":"offline","context":{"idset":"8305"}}
+{"timestamp":1714021378.3901169,"name":"offline","context":{"idset":"8306"}}
+{"timestamp":1714021378.3916309,"name":"offline","context":{"idset":"8307"}}
+{"timestamp":1714021378.3979056,"name":"offline","context":{"idset":"8308"}}
+{"timestamp":1714021378.3994088,"name":"offline","context":{"idset":"8565"}}
+{"timestamp":1714021378.400912,"name":"offline","context":{"idset":"8566"}}
+{"timestamp":1714021378.4024143,"name":"offline","context":{"idset":"8567"}}
+{"timestamp":1714021378.4039197,"name":"offline","context":{"idset":"8568"}}
+{"timestamp":1714021378.4102185,"name":"offline","context":{"idset":"8569"}}
+{"timestamp":1714021378.4117236,"name":"offline","context":{"idset":"8570"}}
+{"timestamp":1714021378.413228,"name":"offline","context":{"idset":"8571"}}
+{"timestamp":1714021378.4147108,"name":"offline","context":{"idset":"8572"}}
+{"timestamp":1714021378.4162014,"name":"offline","context":{"idset":"8573"}}
+{"timestamp":1714021378.4176977,"name":"offline","context":{"idset":"8574"}}
+{"timestamp":1714021378.4242353,"name":"offline","context":{"idset":"8575"}}
+{"timestamp":1714021378.4257431,"name":"offline","context":{"idset":"8576"}}
+{"timestamp":1714021378.4272368,"name":"offline","context":{"idset":"8577"}}
+{"timestamp":1714021378.4287312,"name":"offline","context":{"idset":"8578"}}
+{"timestamp":1714021378.4302354,"name":"offline","context":{"idset":"8579"}}
+{"timestamp":1714021378.4317346,"name":"offline","context":{"idset":"8580"}}
+{"timestamp":1714021378.4332218,"name":"offline","context":{"idset":"8581"}}
+{"timestamp":1714021378.4347155,"name":"offline","context":{"idset":"8582"}}
+{"timestamp":1714021378.4362109,"name":"offline","context":{"idset":"8583"}}
+{"timestamp":1714021378.4377007,"name":"offline","context":{"idset":"8584"}}
+{"timestamp":1714021378.4391804,"name":"offline","context":{"idset":"8585"}}
+{"timestamp":1714021378.4406598,"name":"offline","context":{"idset":"8586"}}
+{"timestamp":1714021378.4421489,"name":"offline","context":{"idset":"8587"}}
+{"timestamp":1714021378.4436262,"name":"offline","context":{"idset":"8588"}}
+{"timestamp":1714021378.4451139,"name":"offline","context":{"idset":"8589"}}
+{"timestamp":1714021378.4465892,"name":"offline","context":{"idset":"8590"}}
+{"timestamp":1714021378.448077,"name":"offline","context":{"idset":"8591"}}
+{"timestamp":1714021378.4495583,"name":"offline","context":{"idset":"8592"}}
+{"timestamp":1714021378.4510343,"name":"offline","context":{"idset":"8593"}}
+{"timestamp":1714021378.4525485,"name":"offline","context":{"idset":"8594"}}
+{"timestamp":1714021378.4540491,"name":"offline","context":{"idset":"8595"}}
+{"timestamp":1714021378.4555321,"name":"offline","context":{"idset":"8596"}}
+{"timestamp":1714021378.4570372,"name":"offline","context":{"idset":"8597"}}
+{"timestamp":1714021378.4585247,"name":"offline","context":{"idset":"8598"}}
+{"timestamp":1714021378.4647503,"name":"offline","context":{"idset":"8599"}}
+{"timestamp":1714021378.4662428,"name":"offline","context":{"idset":"8600"}}
+{"timestamp":1714021378.4677243,"name":"offline","context":{"idset":"8601"}}
+{"timestamp":1714021378.4691935,"name":"offline","context":{"idset":"8602"}}
+{"timestamp":1714021378.4706681,"name":"offline","context":{"idset":"8603"}}
+{"timestamp":1714021378.4768748,"name":"offline","context":{"idset":"8604"}}
+{"timestamp":1714021378.4783347,"name":"offline","context":{"idset":"8605"}}
+{"timestamp":1714021378.4797986,"name":"offline","context":{"idset":"8606"}}
+{"timestamp":1714021378.4813428,"name":"offline","context":{"idset":"8607"}}
+{"timestamp":1714021378.4828577,"name":"offline","context":{"idset":"8608"}}
+{"timestamp":1714021378.4843206,"name":"offline","context":{"idset":"8609"}}
+{"timestamp":1714021378.4857783,"name":"offline","context":{"idset":"8610"}}
+{"timestamp":1714021378.4872603,"name":"offline","context":{"idset":"8611"}}
+{"timestamp":1714021378.4887221,"name":"offline","context":{"idset":"8612"}}
+{"timestamp":1714021378.4901905,"name":"offline","context":{"idset":"8613"}}
+{"timestamp":1714021378.4916453,"name":"offline","context":{"idset":"8614"}}
+{"timestamp":1714021378.4931138,"name":"offline","context":{"idset":"8615"}}
+{"timestamp":1714021378.4946027,"name":"offline","context":{"idset":"8616"}}
+{"timestamp":1714021378.4960682,"name":"offline","context":{"idset":"8617"}}
+{"timestamp":1714021378.4975219,"name":"offline","context":{"idset":"8618"}}
+{"timestamp":1714021378.4989712,"name":"offline","context":{"idset":"8619"}}
+{"timestamp":1714021378.5004408,"name":"offline","context":{"idset":"8620"}}
+{"timestamp":1714021378.5019009,"name":"offline","context":{"idset":"8621"}}
+{"timestamp":1714021378.5033517,"name":"offline","context":{"idset":"8622"}}
+{"timestamp":1714021378.5048153,"name":"offline","context":{"idset":"8623"}}
+{"timestamp":1714021378.5062702,"name":"offline","context":{"idset":"8624"}}
+{"timestamp":1714021378.5077381,"name":"offline","context":{"idset":"8625"}}
+{"timestamp":1714021378.5091994,"name":"offline","context":{"idset":"8626"}}
+{"timestamp":1714021378.5106518,"name":"offline","context":{"idset":"8627"}}
+{"timestamp":1714021378.5121074,"name":"offline","context":{"idset":"8628"}}
+{"timestamp":1714021378.5182674,"name":"offline","context":{"idset":"8629"}}
+{"timestamp":1714021378.5244443,"name":"offline","context":{"idset":"8630"}}
+{"timestamp":1714021378.5259144,"name":"offline","context":{"idset":"8631"}}
+{"timestamp":1714021378.5273645,"name":"offline","context":{"idset":"8632"}}
+{"timestamp":1714021378.5335023,"name":"offline","context":{"idset":"8633"}}
+{"timestamp":1714021378.5349603,"name":"offline","context":{"idset":"8634"}}
+{"timestamp":1714021378.5364048,"name":"offline","context":{"idset":"8635"}}
+{"timestamp":1714021378.537854,"name":"offline","context":{"idset":"8636"}}
+{"timestamp":1714021378.5393035,"name":"offline","context":{"idset":"8637"}}
+{"timestamp":1714021378.5407474,"name":"offline","context":{"idset":"8638"}}
+{"timestamp":1714021378.5421939,"name":"offline","context":{"idset":"8639"}}
+{"timestamp":1714021378.543637,"name":"offline","context":{"idset":"8640"}}
+{"timestamp":1714021378.5450866,"name":"offline","context":{"idset":"8641"}}
+{"timestamp":1714021378.5465376,"name":"offline","context":{"idset":"8642"}}
+{"timestamp":1714021378.547977,"name":"offline","context":{"idset":"8643"}}
+{"timestamp":1714021378.5494206,"name":"offline","context":{"idset":"8644"}}
+{"timestamp":1714021378.5508583,"name":"offline","context":{"idset":"8645"}}
+{"timestamp":1714021378.5522985,"name":"offline","context":{"idset":"8646"}}
+{"timestamp":1714021378.553745,"name":"offline","context":{"idset":"8647"}}
+{"timestamp":1714021378.555181,"name":"offline","context":{"idset":"8648"}}
+{"timestamp":1714021378.5566192,"name":"offline","context":{"idset":"8649"}}
+{"timestamp":1714021378.5580676,"name":"offline","context":{"idset":"8650"}}
+{"timestamp":1714021378.5595057,"name":"offline","context":{"idset":"8651"}}
+{"timestamp":1714021378.5609539,"name":"offline","context":{"idset":"8652"}}
+{"timestamp":1714021378.5626009,"name":"offline","context":{"idset":"8653"}}
+{"timestamp":1714021378.5640483,"name":"offline","context":{"idset":"8654"}}
+{"timestamp":1714021378.5709634,"name":"offline","context":{"idset":"8655"}}
+{"timestamp":1714021378.5724177,"name":"offline","context":{"idset":"8656"}}
+{"timestamp":1714021378.573869,"name":"offline","context":{"idset":"8657"}}
+{"timestamp":1714021378.5753026,"name":"offline","context":{"idset":"8658"}}
+{"timestamp":1714021378.5767443,"name":"offline","context":{"idset":"8659"}}
+{"timestamp":1714021378.5781868,"name":"offline","context":{"idset":"8660"}}
+{"timestamp":1714021378.5796185,"name":"offline","context":{"idset":"8661"}}
+{"timestamp":1714021378.5810618,"name":"offline","context":{"idset":"8662"}}
+{"timestamp":1714021378.5872021,"name":"offline","context":{"idset":"8663"}}
+{"timestamp":1714021378.5933111,"name":"offline","context":{"idset":"8664"}}
+{"timestamp":1714021378.594743,"name":"offline","context":{"idset":"8665"}}
+{"timestamp":1714021378.5961714,"name":"offline","context":{"idset":"8666"}}
+{"timestamp":1714021378.5976,"name":"offline","context":{"idset":"8667"}}
+{"timestamp":1714021378.5990388,"name":"offline","context":{"idset":"8668"}}
+{"timestamp":1714021378.6051521,"name":"offline","context":{"idset":"8669"}}
+{"timestamp":1714021378.6065915,"name":"offline","context":{"idset":"8670"}}
+{"timestamp":1714021378.6080365,"name":"offline","context":{"idset":"8671"}}
+{"timestamp":1714021378.6094732,"name":"offline","context":{"idset":"8672"}}
+{"timestamp":1714021378.6109178,"name":"offline","context":{"idset":"8673"}}
+{"timestamp":1714021378.6170654,"name":"offline","context":{"idset":"8674"}}
+{"timestamp":1714021378.6185102,"name":"offline","context":{"idset":"8675"}}
+{"timestamp":1714021378.6199367,"name":"offline","context":{"idset":"8676"}}
+{"timestamp":1714021378.6213582,"name":"offline","context":{"idset":"8677"}}
+{"timestamp":1714021378.6227884,"name":"offline","context":{"idset":"8678"}}
+{"timestamp":1714021378.6242192,"name":"offline","context":{"idset":"8679"}}
+{"timestamp":1714021378.6303425,"name":"offline","context":{"idset":"8680"}}
+{"timestamp":1714021378.631784,"name":"offline","context":{"idset":"8681"}}
+{"timestamp":1714021378.6332211,"name":"offline","context":{"idset":"8682"}}
+{"timestamp":1714021378.6346464,"name":"offline","context":{"idset":"8683"}}
+{"timestamp":1714021378.6360745,"name":"offline","context":{"idset":"8684"}}
+{"timestamp":1714021378.64218,"name":"offline","context":{"idset":"8685"}}
+{"timestamp":1714021378.6436145,"name":"offline","context":{"idset":"8686"}}
+{"timestamp":1714021378.6450593,"name":"offline","context":{"idset":"8687"}}
+{"timestamp":1714021378.646487,"name":"offline","context":{"idset":"8688"}}
+{"timestamp":1714021378.6479256,"name":"offline","context":{"idset":"8689"}}
+{"timestamp":1714021378.6493566,"name":"offline","context":{"idset":"8690"}}
+{"timestamp":1714021378.6507926,"name":"offline","context":{"idset":"8691"}}
+{"timestamp":1714021378.656965,"name":"offline","context":{"idset":"8692"}}
+{"timestamp":1714021378.6583984,"name":"offline","context":{"idset":"8821"}}
+{"timestamp":1714021378.6598256,"name":"offline","context":{"idset":"8822"}}
+{"timestamp":1714021378.6612504,"name":"offline","context":{"idset":"8823"}}
+{"timestamp":1714021378.6626794,"name":"offline","context":{"idset":"8824"}}
+{"timestamp":1714021378.6641119,"name":"offline","context":{"idset":"8825"}}
+{"timestamp":1714021378.6702707,"name":"offline","context":{"idset":"8826"}}
+{"timestamp":1714021378.6716959,"name":"offline","context":{"idset":"8827"}}
+{"timestamp":1714021378.6731098,"name":"offline","context":{"idset":"8828"}}
+{"timestamp":1714021378.67454,"name":"offline","context":{"idset":"8829"}}
+{"timestamp":1714021378.6759617,"name":"offline","context":{"idset":"8830"}}
+{"timestamp":1714021378.6773911,"name":"offline","context":{"idset":"8831"}}
+{"timestamp":1714021378.6788223,"name":"offline","context":{"idset":"8832"}}
+{"timestamp":1714021378.6849575,"name":"offline","context":{"idset":"8834"}}
+{"timestamp":1714021378.6863806,"name":"offline","context":{"idset":"8835"}}
+{"timestamp":1714021378.6877887,"name":"offline","context":{"idset":"8836"}}
+{"timestamp":1714021378.6892159,"name":"offline","context":{"idset":"8837"}}
+{"timestamp":1714021378.690629,"name":"offline","context":{"idset":"8838"}}
+{"timestamp":1714021378.6920514,"name":"offline","context":{"idset":"8839"}}
+{"timestamp":1714021378.6981869,"name":"offline","context":{"idset":"8840"}}
+{"timestamp":1714021378.6996086,"name":"offline","context":{"idset":"8841"}}
+{"timestamp":1714021378.7010274,"name":"offline","context":{"idset":"8842"}}
+{"timestamp":1714021378.7024415,"name":"offline","context":{"idset":"8843"}}
+{"timestamp":1714021378.7038591,"name":"offline","context":{"idset":"8844"}}
+{"timestamp":1714021378.7052712,"name":"offline","context":{"idset":"8845"}}
+{"timestamp":1714021378.7066848,"name":"offline","context":{"idset":"8846"}}
+{"timestamp":1714021378.7128491,"name":"offline","context":{"idset":"8847"}}
+{"timestamp":1714021378.7142742,"name":"offline","context":{"idset":"8848"}}
+{"timestamp":1714021378.7156909,"name":"offline","context":{"idset":"8849"}}
+{"timestamp":1714021378.7171142,"name":"offline","context":{"idset":"8850"}}
+{"timestamp":1714021378.7185295,"name":"offline","context":{"idset":"8851"}}
+{"timestamp":1714021378.7199521,"name":"offline","context":{"idset":"8852"}}
+{"timestamp":1714021378.7213683,"name":"offline","context":{"idset":"8853"}}
+{"timestamp":1714021378.7274883,"name":"offline","context":{"idset":"8854"}}
+{"timestamp":1714021378.7289028,"name":"offline","context":{"idset":"8855"}}
+{"timestamp":1714021378.7303143,"name":"offline","context":{"idset":"8856"}}
+{"timestamp":1714021378.7317259,"name":"offline","context":{"idset":"8857"}}
+{"timestamp":1714021378.7331352,"name":"offline","context":{"idset":"8858"}}
+{"timestamp":1714021378.7345514,"name":"offline","context":{"idset":"8859"}}
+{"timestamp":1714021378.7359731,"name":"offline","context":{"idset":"8860"}}
+{"timestamp":1714021378.7420864,"name":"offline","context":{"idset":"8861"}}
+{"timestamp":1714021378.7435062,"name":"offline","context":{"idset":"8862"}}
+{"timestamp":1714021378.7449198,"name":"offline","context":{"idset":"8863"}}
+{"timestamp":1714021378.7463257,"name":"offline","context":{"idset":"8864"}}
+{"timestamp":1714021378.747776,"name":"offline","context":{"idset":"8865"}}
+{"timestamp":1714021378.7491965,"name":"offline","context":{"idset":"8866"}}
+{"timestamp":1714021378.7554367,"name":"offline","context":{"idset":"8867"}}
+{"timestamp":1714021378.756866,"name":"offline","context":{"idset":"8868"}}
+{"timestamp":1714021378.7582767,"name":"offline","context":{"idset":"8869"}}
+{"timestamp":1714021378.759686,"name":"offline","context":{"idset":"8870"}}
+{"timestamp":1714021378.7610881,"name":"offline","context":{"idset":"8871"}}
+{"timestamp":1714021378.7672193,"name":"offline","context":{"idset":"8872"}}
+{"timestamp":1714021378.7686236,"name":"offline","context":{"idset":"8873"}}
+{"timestamp":1714021378.7700343,"name":"offline","context":{"idset":"8874"}}
+{"timestamp":1714021378.7714319,"name":"offline","context":{"idset":"8875"}}
+{"timestamp":1714021378.7728338,"name":"offline","context":{"idset":"8876"}}
+{"timestamp":1714021378.7742271,"name":"offline","context":{"idset":"8877"}}
+{"timestamp":1714021378.7756224,"name":"offline","context":{"idset":"8878"}}
+{"timestamp":1714021378.7770109,"name":"offline","context":{"idset":"8879"}}
+{"timestamp":1714021378.7784123,"name":"offline","context":{"idset":"8880"}}
+{"timestamp":1714021378.7800779,"name":"offline","context":{"idset":"8881"}}
+{"timestamp":1714021378.7825029,"name":"offline","context":{"idset":"8882"}}
+{"timestamp":1714021378.7848322,"name":"offline","context":{"idset":"8883"}}
+{"timestamp":1714021378.7871506,"name":"offline","context":{"idset":"8884"}}
+{"timestamp":1714021378.7894468,"name":"offline","context":{"idset":"8885"}}
+{"timestamp":1714021378.7908888,"name":"offline","context":{"idset":"8886"}}
+{"timestamp":1714021378.7922871,"name":"offline","context":{"idset":"8887"}}
+{"timestamp":1714021378.7937112,"name":"offline","context":{"idset":"8888"}}
+{"timestamp":1714021378.7951658,"name":"offline","context":{"idset":"8889"}}
+{"timestamp":1714021378.7965741,"name":"offline","context":{"idset":"8890"}}
+{"timestamp":1714021378.7979703,"name":"offline","context":{"idset":"8891"}}
+{"timestamp":1714021378.7993665,"name":"offline","context":{"idset":"8892"}}
+{"timestamp":1714021378.8007462,"name":"offline","context":{"idset":"8893"}}
+{"timestamp":1714021378.8021586,"name":"offline","context":{"idset":"8894"}}
+{"timestamp":1714021378.8083098,"name":"offline","context":{"idset":"8895"}}
+{"timestamp":1714021378.8097169,"name":"offline","context":{"idset":"8896"}}
+{"timestamp":1714021378.8111081,"name":"offline","context":{"idset":"8897"}}
+{"timestamp":1714021378.8125017,"name":"offline","context":{"idset":"8898"}}
+{"timestamp":1714021378.8139002,"name":"offline","context":{"idset":"8899"}}
+{"timestamp":1714021378.8200028,"name":"offline","context":{"idset":"8900"}}
+{"timestamp":1714021378.8213985,"name":"offline","context":{"idset":"8901"}}
+{"timestamp":1714021378.8227856,"name":"offline","context":{"idset":"8902"}}
+{"timestamp":1714021378.8243275,"name":"offline","context":{"idset":"8903"}}
+{"timestamp":1714021378.8257272,"name":"offline","context":{"idset":"8904"}}
+{"timestamp":1714021378.8271282,"name":"offline","context":{"idset":"8905"}}
+{"timestamp":1714021378.8285263,"name":"offline","context":{"idset":"8906"}}
+{"timestamp":1714021378.8299205,"name":"offline","context":{"idset":"8907"}}
+{"timestamp":1714021378.836031,"name":"offline","context":{"idset":"8908"}}
+{"timestamp":1714021378.8374224,"name":"offline","context":{"idset":"8909"}}
+{"timestamp":1714021378.8388267,"name":"offline","context":{"idset":"8910"}}
+{"timestamp":1714021378.8402328,"name":"offline","context":{"idset":"8911"}}
+{"timestamp":1714021378.8416264,"name":"offline","context":{"idset":"8912"}}
+{"timestamp":1714021378.8477187,"name":"offline","context":{"idset":"8913"}}
+{"timestamp":1714021378.8491073,"name":"offline","context":{"idset":"8914"}}
+{"timestamp":1714021378.8504951,"name":"offline","context":{"idset":"8915"}}
+{"timestamp":1714021378.8518882,"name":"offline","context":{"idset":"8916"}}
+{"timestamp":1714021378.8532927,"name":"offline","context":{"idset":"8917"}}
+{"timestamp":1714021378.8551719,"name":"offline","context":{"idset":"8918"}}
+{"timestamp":1714021378.8567243,"name":"offline","context":{"idset":"8919"}}
+{"timestamp":1714021378.8644171,"name":"offline","context":{"idset":"8920"}}
+{"timestamp":1714021378.8658082,"name":"offline","context":{"idset":"8921"}}
+{"timestamp":1714021378.8671815,"name":"offline","context":{"idset":"8922"}}
+{"timestamp":1714021378.8685799,"name":"offline","context":{"idset":"8923"}}
+{"timestamp":1714021378.8699644,"name":"offline","context":{"idset":"8924"}}
+{"timestamp":1714021378.8713396,"name":"offline","context":{"idset":"8925"}}
+{"timestamp":1714021378.8773816,"name":"offline","context":{"idset":"8926"}}
+{"timestamp":1714021378.8787689,"name":"offline","context":{"idset":"8927"}}
+{"timestamp":1714021378.8801584,"name":"offline","context":{"idset":"8928"}}
+{"timestamp":1714021378.881536,"name":"offline","context":{"idset":"8929"}}
+{"timestamp":1714021378.882921,"name":"offline","context":{"idset":"8930"}}
+{"timestamp":1714021378.8843009,"name":"offline","context":{"idset":"8931"}}
+{"timestamp":1714021378.8856814,"name":"offline","context":{"idset":"8932"}}
+{"timestamp":1714021378.8870559,"name":"offline","context":{"idset":"8933"}}
+{"timestamp":1714021378.8931444,"name":"offline","context":{"idset":"8934"}}
+{"timestamp":1714021378.894526,"name":"offline","context":{"idset":"8935"}}
+{"timestamp":1714021378.8959007,"name":"offline","context":{"idset":"8936"}}
+{"timestamp":1714021378.8972774,"name":"offline","context":{"idset":"8937"}}
+{"timestamp":1714021378.8986564,"name":"offline","context":{"idset":"8938"}}
+{"timestamp":1714021378.9000351,"name":"offline","context":{"idset":"8939"}}
+{"timestamp":1714021378.9061396,"name":"offline","context":{"idset":"8940"}}
+{"timestamp":1714021378.9075184,"name":"offline","context":{"idset":"8941"}}
+{"timestamp":1714021378.9089141,"name":"offline","context":{"idset":"8942"}}
+{"timestamp":1714021378.9102919,"name":"offline","context":{"idset":"8943"}}
+{"timestamp":1714021378.9116633,"name":"offline","context":{"idset":"8944"}}
+{"timestamp":1714021378.9130406,"name":"offline","context":{"idset":"8945"}}
+{"timestamp":1714021378.9191196,"name":"offline","context":{"idset":"8946"}}
+{"timestamp":1714021378.9204991,"name":"offline","context":{"idset":"8947"}}
+{"timestamp":1714021378.921869,"name":"offline","context":{"idset":"8948"}}
+{"timestamp":1714021378.9232554,"name":"offline","context":{"idset":"8949"}}
+{"timestamp":1714021378.9246304,"name":"offline","context":{"idset":"8950"}}
+{"timestamp":1714021378.9260051,"name":"offline","context":{"idset":"8951"}}
+{"timestamp":1714021378.9273753,"name":"offline","context":{"idset":"8952"}}
+{"timestamp":1714021378.933439,"name":"offline","context":{"idset":"8953"}}
+{"timestamp":1714021378.9348176,"name":"offline","context":{"idset":"8954"}}
+{"timestamp":1714021378.9361861,"name":"offline","context":{"idset":"8955"}}
+{"timestamp":1714021378.9375491,"name":"offline","context":{"idset":"8956"}}
+{"timestamp":1714021378.9389267,"name":"offline","context":{"idset":"8957"}}
+{"timestamp":1714021378.9402914,"name":"offline","context":{"idset":"8958"}}
+{"timestamp":1714021378.9463933,"name":"offline","context":{"idset":"8959"}}
+{"timestamp":1714021378.9477766,"name":"offline","context":{"idset":"8960"}}
+{"timestamp":1714021378.9491537,"name":"offline","context":{"idset":"8961"}}
+{"timestamp":1714021378.9505277,"name":"offline","context":{"idset":"8962"}}
+{"timestamp":1714021378.9519045,"name":"offline","context":{"idset":"8963"}}
+{"timestamp":1714021378.953275,"name":"offline","context":{"idset":"8964"}}
+{"timestamp":1714021378.9593697,"name":"offline","context":{"idset":"8965"}}
+{"timestamp":1714021378.9607418,"name":"offline","context":{"idset":"8966"}}
+{"timestamp":1714021378.9621131,"name":"offline","context":{"idset":"8967"}}
+{"timestamp":1714021378.9634819,"name":"offline","context":{"idset":"8968"}}
+{"timestamp":1714021378.9648461,"name":"offline","context":{"idset":"8969"}}
+{"timestamp":1714021378.9662151,"name":"offline","context":{"idset":"8970"}}
+{"timestamp":1714021378.9675798,"name":"offline","context":{"idset":"8971"}}
+{"timestamp":1714021378.9736664,"name":"offline","context":{"idset":"8972"}}
+{"timestamp":1714021378.9750338,"name":"offline","context":{"idset":"8973"}}
+{"timestamp":1714021378.9763901,"name":"offline","context":{"idset":"8974"}}
+{"timestamp":1714021378.977746,"name":"offline","context":{"idset":"8975"}}
+{"timestamp":1714021378.9791095,"name":"offline","context":{"idset":"8976"}}
+{"timestamp":1714021378.9804676,"name":"offline","context":{"idset":"8977"}}
+{"timestamp":1714021378.9865146,"name":"offline","context":{"idset":"8978"}}
+{"timestamp":1714021378.9878712,"name":"offline","context":{"idset":"8979"}}
+{"timestamp":1714021378.989228,"name":"offline","context":{"idset":"8980"}}
+{"timestamp":1714021378.990577,"name":"offline","context":{"idset":"8981"}}
+{"timestamp":1714021378.9919398,"name":"offline","context":{"idset":"8982"}}
+{"timestamp":1714021378.9932909,"name":"offline","context":{"idset":"8983"}}
+{"timestamp":1714021378.9946783,"name":"offline","context":{"idset":"8984"}}
+{"timestamp":1714021378.9960434,"name":"offline","context":{"idset":"8985"}}
+{"timestamp":1714021379.0021031,"name":"offline","context":{"idset":"8986"}}
+{"timestamp":1714021379.0034609,"name":"offline","context":{"idset":"8987"}}
+{"timestamp":1714021379.0048096,"name":"offline","context":{"idset":"8988"}}
+{"timestamp":1714021379.0061605,"name":"offline","context":{"idset":"8989"}}
+{"timestamp":1714021379.0075114,"name":"offline","context":{"idset":"8990"}}
+{"timestamp":1714021379.0088851,"name":"offline","context":{"idset":"8991"}}
+{"timestamp":1714021379.0102437,"name":"offline","context":{"idset":"8992"}}
+{"timestamp":1714021379.0115879,"name":"offline","context":{"idset":"8993"}}
+{"timestamp":1714021379.012958,"name":"offline","context":{"idset":"8994"}}
+{"timestamp":1714021379.0143082,"name":"offline","context":{"idset":"8995"}}
+{"timestamp":1714021379.0157259,"name":"offline","context":{"idset":"8996"}}
+{"timestamp":1714021379.0170839,"name":"offline","context":{"idset":"8997"}}
+{"timestamp":1714021379.018441,"name":"offline","context":{"idset":"8998"}}
+{"timestamp":1714021379.0197847,"name":"offline","context":{"idset":"8999"}}
+{"timestamp":1714021379.0211456,"name":"offline","context":{"idset":"9000"}}
+{"timestamp":1714021379.0224881,"name":"offline","context":{"idset":"9001"}}
+{"timestamp":1714021379.0238483,"name":"offline","context":{"idset":"9002"}}
+{"timestamp":1714021379.0252163,"name":"offline","context":{"idset":"9004"}}
+{"timestamp":1714021379.0267763,"name":"offline","context":{"idset":"9005"}}
+{"timestamp":1714021379.0281425,"name":"offline","context":{"idset":"9006"}}
+{"timestamp":1714021379.0295255,"name":"offline","context":{"idset":"9007"}}
+{"timestamp":1714021379.0308924,"name":"offline","context":{"idset":"9008"}}
+{"timestamp":1714021379.0322559,"name":"offline","context":{"idset":"9009"}}
+{"timestamp":1714021379.0336666,"name":"offline","context":{"idset":"9010"}}
+{"timestamp":1714021379.039783,"name":"offline","context":{"idset":"9011"}}
+{"timestamp":1714021379.0412488,"name":"offline","context":{"idset":"9012"}}
+{"timestamp":1714021379.0425999,"name":"offline","context":{"idset":"9013"}}
+{"timestamp":1714021379.0439594,"name":"offline","context":{"idset":"9014"}}
+{"timestamp":1714021379.0453241,"name":"offline","context":{"idset":"9015"}}
+{"timestamp":1714021379.0466657,"name":"offline","context":{"idset":"9016"}}
+{"timestamp":1714021379.054296,"name":"offline","context":{"idset":"9017"}}
+{"timestamp":1714021379.0556381,"name":"offline","context":{"idset":"9018"}}
+{"timestamp":1714021379.0569911,"name":"offline","context":{"idset":"9019"}}
+{"timestamp":1714021379.0583384,"name":"offline","context":{"idset":"9020"}}
+{"timestamp":1714021379.059684,"name":"offline","context":{"idset":"9021"}}
+{"timestamp":1714021379.0610249,"name":"offline","context":{"idset":"9022"}}
+{"timestamp":1714021379.0623798,"name":"offline","context":{"idset":"9023"}}
+{"timestamp":1714021379.0637281,"name":"offline","context":{"idset":"9024"}}
+{"timestamp":1714021379.0698843,"name":"offline","context":{"idset":"9025"}}
+{"timestamp":1714021379.0712526,"name":"offline","context":{"idset":"9026"}}
+{"timestamp":1714021379.0726004,"name":"offline","context":{"idset":"9027"}}
+{"timestamp":1714021379.0739534,"name":"offline","context":{"idset":"9028"}}
+{"timestamp":1714021379.0752971,"name":"offline","context":{"idset":"9029"}}
+{"timestamp":1714021379.0766404,"name":"offline","context":{"idset":"9030"}}
+{"timestamp":1714021379.0779757,"name":"offline","context":{"idset":"9031"}}
+{"timestamp":1714021379.0793164,"name":"offline","context":{"idset":"9032"}}
+{"timestamp":1714021379.0806592,"name":"offline","context":{"idset":"9033"}}
+{"timestamp":1714021379.0820127,"name":"offline","context":{"idset":"9034"}}
+{"timestamp":1714021379.0833561,"name":"offline","context":{"idset":"9035"}}
+{"timestamp":1714021379.0846949,"name":"offline","context":{"idset":"9036"}}
+{"timestamp":1714021379.08603,"name":"offline","context":{"idset":"9037"}}
+{"timestamp":1714021379.0873632,"name":"offline","context":{"idset":"9038"}}
+{"timestamp":1714021379.0887764,"name":"offline","context":{"idset":"9039"}}
+{"timestamp":1714021379.0901229,"name":"offline","context":{"idset":"9040"}}
+{"timestamp":1714021379.0914578,"name":"offline","context":{"idset":"9041"}}
+{"timestamp":1714021379.0927975,"name":"offline","context":{"idset":"9042"}}
+{"timestamp":1714021379.0941403,"name":"offline","context":{"idset":"9044"}}
+{"timestamp":1714021379.0954702,"name":"offline","context":{"idset":"9045"}}
+{"timestamp":1714021379.0968263,"name":"offline","context":{"idset":"9046"}}
+{"timestamp":1714021379.0981586,"name":"offline","context":{"idset":"9047"}}
+{"timestamp":1714021379.099488,"name":"offline","context":{"idset":"9048"}}
+{"timestamp":1714021379.1008317,"name":"offline","context":{"idset":"9049"}}
+{"timestamp":1714021379.1070018,"name":"offline","context":{"idset":"9050"}}
+{"timestamp":1714021379.1083376,"name":"offline","context":{"idset":"9051"}}
+{"timestamp":1714021379.1096776,"name":"offline","context":{"idset":"9052"}}
+{"timestamp":1714021379.1110151,"name":"offline","context":{"idset":"9053"}}
+{"timestamp":1714021379.112442,"name":"offline","context":{"idset":"9055"}}
+{"timestamp":1714021379.1137793,"name":"offline","context":{"idset":"9056"}}
+{"timestamp":1714021379.1151209,"name":"offline","context":{"idset":"9057"}}
+{"timestamp":1714021379.1211767,"name":"offline","context":{"idset":"9058"}}
+{"timestamp":1714021379.122515,"name":"offline","context":{"idset":"9059"}}
+{"timestamp":1714021379.1238422,"name":"offline","context":{"idset":"9060"}}
+{"timestamp":1714021379.1251671,"name":"offline","context":{"idset":"9061"}}
+{"timestamp":1714021379.1265163,"name":"offline","context":{"idset":"9062"}}
+{"timestamp":1714021379.1278515,"name":"offline","context":{"idset":"9063"}}
+{"timestamp":1714021379.1338909,"name":"offline","context":{"idset":"9064"}}
+{"timestamp":1714021379.1352167,"name":"offline","context":{"idset":"9065"}}
+{"timestamp":1714021379.1365395,"name":"offline","context":{"idset":"9066"}}
+{"timestamp":1714021379.1378596,"name":"offline","context":{"idset":"9067"}}
+{"timestamp":1714021379.139183,"name":"offline","context":{"idset":"9068"}}
+{"timestamp":1714021379.1404982,"name":"offline","context":{"idset":"9069"}}
+{"timestamp":1714021379.1418254,"name":"offline","context":{"idset":"9070"}}
+{"timestamp":1714021379.1494548,"name":"offline","context":{"idset":"9071"}}
+{"timestamp":1714021379.1514542,"name":"offline","context":{"idset":"9072"}}
+{"timestamp":1714021379.1533682,"name":"offline","context":{"idset":"9073"}}
+{"timestamp":1714021379.1552656,"name":"offline","context":{"idset":"9074"}}
+{"timestamp":1714021379.1570375,"name":"offline","context":{"idset":"9076"}}
+{"timestamp":1714021379.1584001,"name":"offline","context":{"idset":"9077"}}
+{"timestamp":1714021379.1644738,"name":"offline","context":{"idset":"9078"}}
+{"timestamp":1714021379.1657963,"name":"offline","context":{"idset":"9079"}}
+{"timestamp":1714021379.1671166,"name":"offline","context":{"idset":"9080"}}
+{"timestamp":1714021379.1684294,"name":"offline","context":{"idset":"9081"}}
+{"timestamp":1714021379.1697564,"name":"offline","context":{"idset":"9082"}}
+{"timestamp":1714021379.1710613,"name":"offline","context":{"idset":"9083"}}
+{"timestamp":1714021379.172368,"name":"offline","context":{"idset":"9084"}}
+{"timestamp":1714021379.1736844,"name":"offline","context":{"idset":"9085"}}
+{"timestamp":1714021379.1749976,"name":"offline","context":{"idset":"9086"}}
+{"timestamp":1714021379.1763384,"name":"offline","context":{"idset":"9087"}}
+{"timestamp":1714021379.1776464,"name":"offline","context":{"idset":"9088"}}
+{"timestamp":1714021379.1790214,"name":"offline","context":{"idset":"9089"}}
+{"timestamp":1714021379.1803434,"name":"offline","context":{"idset":"9090"}}
+{"timestamp":1714021379.1816506,"name":"offline","context":{"idset":"9091"}}
+{"timestamp":1714021379.1829569,"name":"offline","context":{"idset":"9092"}}
+{"timestamp":1714021379.1842852,"name":"offline","context":{"idset":"9093"}}
+{"timestamp":1714021379.1856015,"name":"offline","context":{"idset":"9094"}}
+{"timestamp":1714021379.1869097,"name":"offline","context":{"idset":"9095"}}
+{"timestamp":1714021379.1882119,"name":"offline","context":{"idset":"9096"}}
+{"timestamp":1714021379.1895216,"name":"offline","context":{"idset":"9097"}}
+{"timestamp":1714021379.1908433,"name":"offline","context":{"idset":"9098"}}
+{"timestamp":1714021379.1921623,"name":"offline","context":{"idset":"9099"}}
+{"timestamp":1714021379.1934838,"name":"offline","context":{"idset":"9100"}}
+{"timestamp":1714021379.1948111,"name":"offline","context":{"idset":"9101"}}
+{"timestamp":1714021379.1961243,"name":"offline","context":{"idset":"9102"}}
+{"timestamp":1714021379.1974373,"name":"offline","context":{"idset":"9103"}}
+{"timestamp":1714021379.203517,"name":"offline","context":{"idset":"9104"}}
+{"timestamp":1714021379.2048256,"name":"offline","context":{"idset":"9105"}}
+{"timestamp":1714021379.2061374,"name":"offline","context":{"idset":"9106"}}
+{"timestamp":1714021379.2074306,"name":"offline","context":{"idset":"9107"}}
+{"timestamp":1714021379.2087245,"name":"offline","context":{"idset":"9108"}}
+{"timestamp":1714021379.210027,"name":"offline","context":{"idset":"9109"}}
+{"timestamp":1714021379.2113225,"name":"offline","context":{"idset":"9110"}}
+{"timestamp":1714021379.2173152,"name":"offline","context":{"idset":"9111"}}
+{"timestamp":1714021379.2186141,"name":"offline","context":{"idset":"9112"}}
+{"timestamp":1714021379.2199166,"name":"offline","context":{"idset":"9113"}}
+{"timestamp":1714021379.2212048,"name":"offline","context":{"idset":"9114"}}
+{"timestamp":1714021379.2224958,"name":"offline","context":{"idset":"9115"}}
+{"timestamp":1714021379.2238081,"name":"offline","context":{"idset":"9116"}}
+{"timestamp":1714021379.2251151,"name":"offline","context":{"idset":"9117"}}
+{"timestamp":1714021379.2264094,"name":"offline","context":{"idset":"9118"}}
+{"timestamp":1714021379.2324128,"name":"offline","context":{"idset":"9119"}}
+{"timestamp":1714021379.2337003,"name":"offline","context":{"idset":"9120"}}
+{"timestamp":1714021379.2350039,"name":"offline","context":{"idset":"9121"}}
+{"timestamp":1714021379.2362988,"name":"offline","context":{"idset":"9122"}}
+{"timestamp":1714021379.2376208,"name":"offline","context":{"idset":"9123"}}
+{"timestamp":1714021379.2389266,"name":"offline","context":{"idset":"9124"}}
+{"timestamp":1714021379.2402277,"name":"offline","context":{"idset":"9125"}}
+{"timestamp":1714021379.2415147,"name":"offline","context":{"idset":"9126"}}
+{"timestamp":1714021379.2428133,"name":"offline","context":{"idset":"9127"}}
+{"timestamp":1714021379.2441027,"name":"offline","context":{"idset":"9128"}}
+{"timestamp":1714021379.2453911,"name":"offline","context":{"idset":"9129"}}
+{"timestamp":1714021379.2466881,"name":"offline","context":{"idset":"9130"}}
+{"timestamp":1714021379.2479832,"name":"offline","context":{"idset":"9131"}}
+{"timestamp":1714021379.2492723,"name":"offline","context":{"idset":"9132"}}
+{"timestamp":1714021379.2506781,"name":"offline","context":{"idset":"9133"}}
+{"timestamp":1714021379.2519965,"name":"offline","context":{"idset":"9134"}}
+{"timestamp":1714021379.2532895,"name":"offline","context":{"idset":"9135"}}
+{"timestamp":1714021379.254585,"name":"offline","context":{"idset":"9136"}}
+{"timestamp":1714021379.2558992,"name":"offline","context":{"idset":"9137"}}
+{"timestamp":1714021379.2571938,"name":"offline","context":{"idset":"9138"}}
+{"timestamp":1714021379.2584691,"name":"offline","context":{"idset":"9139"}}
+{"timestamp":1714021379.259768,"name":"offline","context":{"idset":"9140"}}
+{"timestamp":1714021379.2611117,"name":"offline","context":{"idset":"9141"}}
+{"timestamp":1714021379.2624199,"name":"offline","context":{"idset":"9142"}}
+{"timestamp":1714021379.263844,"name":"offline","context":{"idset":"9143"}}
+{"timestamp":1714021379.2701867,"name":"offline","context":{"idset":"9144"}}
+{"timestamp":1714021379.2714746,"name":"offline","context":{"idset":"9145"}}
+{"timestamp":1714021379.2727623,"name":"offline","context":{"idset":"9146"}}
+{"timestamp":1714021379.2741237,"name":"offline","context":{"idset":"9147"}}
+{"timestamp":1714021379.2755494,"name":"offline","context":{"idset":"9148"}}
+{"timestamp":1714021379.2768993,"name":"offline","context":{"idset":"9149"}}
+{"timestamp":1714021379.282985,"name":"offline","context":{"idset":"9150"}}
+{"timestamp":1714021379.284878,"name":"offline","context":{"idset":"9151"}}
+{"timestamp":1714021379.2871273,"name":"offline","context":{"idset":"9152"}}
+{"timestamp":1714021379.2896509,"name":"offline","context":{"idset":"9153"}}
+{"timestamp":1714021379.2920122,"name":"offline","context":{"idset":"9154"}}
+{"timestamp":1714021379.2933197,"name":"offline","context":{"idset":"9155"}}
+{"timestamp":1714021379.2945869,"name":"offline","context":{"idset":"9156"}}
+{"timestamp":1714021379.2958546,"name":"offline","context":{"idset":"9157"}}
+{"timestamp":1714021379.297122,"name":"offline","context":{"idset":"9158"}}
+{"timestamp":1714021379.3031278,"name":"offline","context":{"idset":"9159"}}
+{"timestamp":1714021379.3044238,"name":"offline","context":{"idset":"9160"}}
+{"timestamp":1714021379.3057084,"name":"offline","context":{"idset":"9161"}}
+{"timestamp":1714021379.3069837,"name":"offline","context":{"idset":"9162"}}
+{"timestamp":1714021379.3129828,"name":"offline","context":{"idset":"9163"}}
+{"timestamp":1714021379.3142645,"name":"offline","context":{"idset":"9164"}}
+{"timestamp":1714021379.3155384,"name":"offline","context":{"idset":"9165"}}
+{"timestamp":1714021379.3168161,"name":"offline","context":{"idset":"9166"}}
+{"timestamp":1714021379.318089,"name":"offline","context":{"idset":"9167"}}
+{"timestamp":1714021379.3193862,"name":"offline","context":{"idset":"9168"}}
+{"timestamp":1714021379.3275261,"name":"offline","context":{"idset":"9169"}}
+{"timestamp":1714021379.3347325,"name":"offline","context":{"idset":"9170"}}
+{"timestamp":1714021379.3363712,"name":"offline","context":{"idset":"9171"}}
+{"timestamp":1714021379.3380249,"name":"offline","context":{"idset":"9172"}}
+{"timestamp":1714021379.3395574,"name":"offline","context":{"idset":"9173"}}
+{"timestamp":1714021379.3411112,"name":"offline","context":{"idset":"9174"}}
+{"timestamp":1714021379.3426623,"name":"offline","context":{"idset":"9175"}}
+{"timestamp":1714021379.344213,"name":"offline","context":{"idset":"9176"}}
+{"timestamp":1714021379.3515484,"name":"offline","context":{"idset":"9177"}}
+{"timestamp":1714021379.353158,"name":"offline","context":{"idset":"9178"}}
+{"timestamp":1714021379.3547912,"name":"offline","context":{"idset":"9179"}}
+{"timestamp":1714021379.3563108,"name":"offline","context":{"idset":"9180"}}
+{"timestamp":1714021379.3578465,"name":"offline","context":{"idset":"9181"}}
+{"timestamp":1714021379.3593428,"name":"offline","context":{"idset":"9182"}}
+{"timestamp":1714021379.3608356,"name":"offline","context":{"idset":"9183"}}
+{"timestamp":1714021379.3680766,"name":"offline","context":{"idset":"9184"}}
+{"timestamp":1714021379.3696985,"name":"offline","context":{"idset":"9185"}}
+{"timestamp":1714021379.3713751,"name":"offline","context":{"idset":"9186"}}
+{"timestamp":1714021379.3729336,"name":"offline","context":{"idset":"9187"}}
+{"timestamp":1714021379.3744266,"name":"offline","context":{"idset":"9188"}}
+{"timestamp":1714021379.3760052,"name":"offline","context":{"idset":"9189"}}
+{"timestamp":1714021379.3775554,"name":"offline","context":{"idset":"9190"}}
+{"timestamp":1714021379.3791082,"name":"offline","context":{"idset":"9191"}}
+{"timestamp":1714021379.3862796,"name":"offline","context":{"idset":"9192"}}
+{"timestamp":1714021379.3878217,"name":"offline","context":{"idset":"9193"}}
+{"timestamp":1714021379.389328,"name":"offline","context":{"idset":"9194"}}
+{"timestamp":1714021379.3908679,"name":"offline","context":{"idset":"9195"}}
+{"timestamp":1714021379.3923893,"name":"offline","context":{"idset":"9196"}}
+{"timestamp":1714021379.3995762,"name":"offline","context":{"idset":"9197"}}
+{"timestamp":1714021379.4011128,"name":"offline","context":{"idset":"9198"}}
+{"timestamp":1714021379.4025726,"name":"offline","context":{"idset":"9199"}}
+{"timestamp":1714021379.4041197,"name":"offline","context":{"idset":"9200"}}
+{"timestamp":1714021379.405376,"name":"offline","context":{"idset":"9201"}}
+{"timestamp":1714021379.4112988,"name":"offline","context":{"idset":"9202"}}
+{"timestamp":1714021379.4125457,"name":"offline","context":{"idset":"9203"}}
+{"timestamp":1714021379.4138093,"name":"offline","context":{"idset":"9204"}}
+{"timestamp":1714021379.41506,"name":"offline","context":{"idset":"9205"}}
+{"timestamp":1714021379.4163196,"name":"offline","context":{"idset":"9206"}}
+{"timestamp":1714021379.4175725,"name":"offline","context":{"idset":"9207"}}
+{"timestamp":1714021379.4188535,"name":"offline","context":{"idset":"9208"}}
+{"timestamp":1714021379.4201124,"name":"offline","context":{"idset":"9209"}}
+{"timestamp":1714021379.4261773,"name":"offline","context":{"idset":"9210"}}
+{"timestamp":1714021379.4274378,"name":"offline","context":{"idset":"9211"}}
+{"timestamp":1714021379.4286902,"name":"offline","context":{"idset":"9212"}}
+{"timestamp":1714021379.4299474,"name":"offline","context":{"idset":"9213"}}
+{"timestamp":1714021379.4311986,"name":"offline","context":{"idset":"9214"}}
+{"timestamp":1714021379.4324629,"name":"offline","context":{"idset":"9215"}}
+{"timestamp":1714021379.4384451,"name":"offline","context":{"idset":"9216"}}
+{"timestamp":1714021379.4396915,"name":"offline","context":{"idset":"9217"}}
+{"timestamp":1714021379.4409442,"name":"offline","context":{"idset":"9218"}}
+{"timestamp":1714021379.442184,"name":"offline","context":{"idset":"9219"}}
+{"timestamp":1714021379.4434721,"name":"offline","context":{"idset":"9220"}}
+{"timestamp":1714021379.4447169,"name":"offline","context":{"idset":"9221"}}
+{"timestamp":1714021379.4459667,"name":"offline","context":{"idset":"9222"}}
+{"timestamp":1714021379.4518962,"name":"offline","context":{"idset":"9223"}}
+{"timestamp":1714021379.4531457,"name":"offline","context":{"idset":"9224"}}
+{"timestamp":1714021379.4544661,"name":"offline","context":{"idset":"9225"}}
+{"timestamp":1714021379.4557474,"name":"offline","context":{"idset":"9226"}}
+{"timestamp":1714021379.4570119,"name":"offline","context":{"idset":"9227"}}
+{"timestamp":1714021379.4582582,"name":"offline","context":{"idset":"9228"}}
+{"timestamp":1714021379.4642215,"name":"offline","context":{"idset":"9229"}}
+{"timestamp":1714021379.4655499,"name":"offline","context":{"idset":"9230"}}
+{"timestamp":1714021379.4667947,"name":"offline","context":{"idset":"9231"}}
+{"timestamp":1714021379.4680414,"name":"offline","context":{"idset":"9232"}}
+{"timestamp":1714021379.4692829,"name":"offline","context":{"idset":"9233"}}
+{"timestamp":1714021379.4705157,"name":"offline","context":{"idset":"9234"}}
+{"timestamp":1714021379.4717417,"name":"offline","context":{"idset":"9235"}}
+{"timestamp":1714021379.4776809,"name":"offline","context":{"idset":"9236"}}
+{"timestamp":1714021379.4789174,"name":"offline","context":{"idset":"9237"}}
+{"timestamp":1714021379.4801588,"name":"offline","context":{"idset":"9238"}}
+{"timestamp":1714021379.4814303,"name":"offline","context":{"idset":"9239"}}
+{"timestamp":1714021379.4826994,"name":"offline","context":{"idset":"9240"}}
+{"timestamp":1714021379.4839933,"name":"offline","context":{"idset":"9241"}}
+{"timestamp":1714021379.48523,"name":"offline","context":{"idset":"9242"}}
+{"timestamp":1714021379.4911573,"name":"offline","context":{"idset":"9243"}}
+{"timestamp":1714021379.4923968,"name":"offline","context":{"idset":"9244"}}
+{"timestamp":1714021379.4936287,"name":"offline","context":{"idset":"9245"}}
+{"timestamp":1714021379.4949374,"name":"offline","context":{"idset":"9246"}}
+{"timestamp":1714021379.4961658,"name":"offline","context":{"idset":"9247"}}
+{"timestamp":1714021379.4974008,"name":"offline","context":{"idset":"9248"}}
+{"timestamp":1714021379.4986365,"name":"offline","context":{"idset":"9249"}}
+{"timestamp":1714021379.4998784,"name":"offline","context":{"idset":"9250"}}
+{"timestamp":1714021379.5011117,"name":"offline","context":{"idset":"9251"}}
+{"timestamp":1714021379.5023751,"name":"offline","context":{"idset":"9252"}}
+{"timestamp":1714021379.5036135,"name":"offline","context":{"idset":"9253"}}
+{"timestamp":1714021379.5048759,"name":"offline","context":{"idset":"9254"}}
+{"timestamp":1714021379.5061045,"name":"offline","context":{"idset":"9255"}}
+{"timestamp":1714021379.5073307,"name":"offline","context":{"idset":"9256"}}
+{"timestamp":1714021379.5086272,"name":"offline","context":{"idset":"9257"}}
+{"timestamp":1714021379.509866,"name":"offline","context":{"idset":"9258"}}
+{"timestamp":1714021379.5112498,"name":"offline","context":{"idset":"9259"}}
+{"timestamp":1714021379.5125101,"name":"offline","context":{"idset":"9260"}}
+{"timestamp":1714021379.5137374,"name":"offline","context":{"idset":"9261"}}
+{"timestamp":1714021379.5149651,"name":"offline","context":{"idset":"9262"}}
+{"timestamp":1714021379.5163789,"name":"offline","context":{"idset":"9263"}}
+{"timestamp":1714021379.5176151,"name":"offline","context":{"idset":"9264"}}
+{"timestamp":1714021379.5188689,"name":"offline","context":{"idset":"9265"}}
+{"timestamp":1714021379.5200984,"name":"offline","context":{"idset":"9266"}}
+{"timestamp":1714021379.5214539,"name":"offline","context":{"idset":"9267"}}
+{"timestamp":1714021379.5227058,"name":"offline","context":{"idset":"9268"}}
+{"timestamp":1714021379.5239425,"name":"offline","context":{"idset":"9269"}}
+{"timestamp":1714021379.5298665,"name":"offline","context":{"idset":"9270"}}
+{"timestamp":1714021379.5310998,"name":"offline","context":{"idset":"9271"}}
+{"timestamp":1714021379.5323257,"name":"offline","context":{"idset":"9272"}}
+{"timestamp":1714021379.5335534,"name":"offline","context":{"idset":"9273"}}
+{"timestamp":1714021379.5347664,"name":"offline","context":{"idset":"9274"}}
+{"timestamp":1714021379.5360029,"name":"offline","context":{"idset":"9275"}}
+{"timestamp":1714021379.537226,"name":"offline","context":{"idset":"9276"}}
+{"timestamp":1714021379.543149,"name":"offline","context":{"idset":"9277"}}
+{"timestamp":1714021379.5443771,"name":"offline","context":{"idset":"9278"}}
+{"timestamp":1714021379.5456014,"name":"offline","context":{"idset":"9279"}}
+{"timestamp":1714021379.5468152,"name":"offline","context":{"idset":"9280"}}
+{"timestamp":1714021379.5480363,"name":"offline","context":{"idset":"9281"}}
+{"timestamp":1714021379.5493195,"name":"offline","context":{"idset":"9282"}}
+{"timestamp":1714021379.5505466,"name":"offline","context":{"idset":"9283"}}
+{"timestamp":1714021379.556448,"name":"offline","context":{"idset":"9284"}}
+{"timestamp":1714021379.5576653,"name":"offline","context":{"idset":"9285"}}
+{"timestamp":1714021379.5588963,"name":"offline","context":{"idset":"9286"}}
+{"timestamp":1714021379.5601225,"name":"offline","context":{"idset":"9287"}}
+{"timestamp":1714021379.5613499,"name":"offline","context":{"idset":"9288"}}
+{"timestamp":1714021379.5625701,"name":"offline","context":{"idset":"9289"}}
+{"timestamp":1714021379.5637848,"name":"offline","context":{"idset":"9290"}}
+{"timestamp":1714021379.5697119,"name":"offline","context":{"idset":"9291"}}
+{"timestamp":1714021379.570992,"name":"offline","context":{"idset":"9292"}}
+{"timestamp":1714021379.5722117,"name":"offline","context":{"idset":"9293"}}
+{"timestamp":1714021379.5734315,"name":"offline","context":{"idset":"9294"}}
+{"timestamp":1714021379.5748601,"name":"offline","context":{"idset":"9295"}}
+{"timestamp":1714021379.5760815,"name":"offline","context":{"idset":"9296"}}
+{"timestamp":1714021379.5772932,"name":"offline","context":{"idset":"9297"}}
+{"timestamp":1714021379.5785224,"name":"offline","context":{"idset":"9298"}}
+{"timestamp":1714021379.5797353,"name":"offline","context":{"idset":"9299"}}
+{"timestamp":1714021379.5809507,"name":"offline","context":{"idset":"9300"}}
+{"timestamp":1714021379.5821776,"name":"offline","context":{"idset":"9301"}}
+{"timestamp":1714021379.5834124,"name":"offline","context":{"idset":"9302"}}
+{"timestamp":1714021379.5846305,"name":"offline","context":{"idset":"9303"}}
+{"timestamp":1714021379.5858505,"name":"offline","context":{"idset":"9304"}}
+{"timestamp":1714021379.5874329,"name":"offline","context":{"idset":"9305"}}
+{"timestamp":1714021379.5891218,"name":"offline","context":{"idset":"9306"}}
+{"timestamp":1714021379.5909705,"name":"offline","context":{"idset":"9307"}}
+{"timestamp":1714021379.5924401,"name":"offline","context":{"idset":"9308"}}
+{"timestamp":1714021379.5936649,"name":"offline","context":{"idset":"9309"}}
+{"timestamp":1714021379.5948844,"name":"offline","context":{"idset":"9310"}}
+{"timestamp":1714021379.5961599,"name":"offline","context":{"idset":"9311"}}
+{"timestamp":1714021379.5975285,"name":"offline","context":{"idset":"9312"}}
+{"timestamp":1714021379.5987449,"name":"offline","context":{"idset":"9313"}}
+{"timestamp":1714021379.5999596,"name":"offline","context":{"idset":"9314"}}
+{"timestamp":1714021379.6011603,"name":"offline","context":{"idset":"9315"}}
+{"timestamp":1714021379.6023557,"name":"offline","context":{"idset":"9316"}}
+{"timestamp":1714021379.603555,"name":"offline","context":{"idset":"9317"}}
+{"timestamp":1714021379.6047795,"name":"offline","context":{"idset":"9318"}}
+{"timestamp":1714021379.6107368,"name":"offline","context":{"idset":"9319"}}
+{"timestamp":1714021379.6119516,"name":"offline","context":{"idset":"9320"}}
+{"timestamp":1714021379.613251,"name":"offline","context":{"idset":"9321"}}
+{"timestamp":1714021379.6144643,"name":"offline","context":{"idset":"9322"}}
+{"timestamp":1714021379.61566,"name":"offline","context":{"idset":"9323"}}
+{"timestamp":1714021379.6219349,"name":"offline","context":{"idset":"9324"}}
+{"timestamp":1714021379.6231453,"name":"offline","context":{"idset":"9325"}}
+{"timestamp":1714021379.624347,"name":"offline","context":{"idset":"9326"}}
+{"timestamp":1714021379.62555,"name":"offline","context":{"idset":"9327"}}
+{"timestamp":1714021379.6267474,"name":"offline","context":{"idset":"9328"}}
+{"timestamp":1714021379.6279533,"name":"offline","context":{"idset":"9329"}}
+{"timestamp":1714021379.6293659,"name":"offline","context":{"idset":"9330"}}
+{"timestamp":1714021379.6306362,"name":"offline","context":{"idset":"9331"}}
+{"timestamp":1714021379.636544,"name":"offline","context":{"idset":"9332"}}
+{"timestamp":1714021379.6377366,"name":"offline","context":{"idset":"9365"}}
+{"timestamp":1714021379.6389396,"name":"offline","context":{"idset":"9366"}}
+{"timestamp":1714021379.6401312,"name":"offline","context":{"idset":"9367"}}
+{"timestamp":1714021379.6413219,"name":"offline","context":{"idset":"9368"}}
+{"timestamp":1714021379.6425152,"name":"offline","context":{"idset":"9369"}}
+{"timestamp":1714021379.6483901,"name":"offline","context":{"idset":"9370"}}
+{"timestamp":1714021379.6495764,"name":"offline","context":{"idset":"9371"}}
+{"timestamp":1714021379.6507668,"name":"offline","context":{"idset":"9372"}}
+{"timestamp":1714021379.6519997,"name":"offline","context":{"idset":"9373"}}
+{"timestamp":1714021379.6531911,"name":"offline","context":{"idset":"9374"}}
+{"timestamp":1714021379.6543806,"name":"offline","context":{"idset":"9375"}}
+{"timestamp":1714021379.6555681,"name":"offline","context":{"idset":"9376"}}
+{"timestamp":1714021379.6567428,"name":"offline","context":{"idset":"9377"}}
+{"timestamp":1714021379.6579459,"name":"offline","context":{"idset":"9378"}}
+{"timestamp":1714021379.6591358,"name":"offline","context":{"idset":"9379"}}
+{"timestamp":1714021379.6603227,"name":"offline","context":{"idset":"9380"}}
+{"timestamp":1714021379.6615028,"name":"offline","context":{"idset":"9381"}}
+{"timestamp":1714021379.6626854,"name":"offline","context":{"idset":"9382"}}
+{"timestamp":1714021379.6638587,"name":"offline","context":{"idset":"9383"}}
+{"timestamp":1714021379.6650348,"name":"offline","context":{"idset":"9384"}}
+{"timestamp":1714021379.6662073,"name":"offline","context":{"idset":"9385"}}
+{"timestamp":1714021379.6673813,"name":"offline","context":{"idset":"9386"}}
+{"timestamp":1714021379.66855,"name":"offline","context":{"idset":"9387"}}
+{"timestamp":1714021379.6697228,"name":"offline","context":{"idset":"9388"}}
+{"timestamp":1714021379.6708961,"name":"offline","context":{"idset":"9389"}}
+{"timestamp":1714021379.6720777,"name":"offline","context":{"idset":"9390"}}
+{"timestamp":1714021379.6732531,"name":"offline","context":{"idset":"9391"}}
+{"timestamp":1714021379.6744256,"name":"offline","context":{"idset":"9392"}}
+{"timestamp":1714021379.675607,"name":"offline","context":{"idset":"9393"}}
+{"timestamp":1714021379.6767721,"name":"offline","context":{"idset":"9394"}}
+{"timestamp":1714021379.6779485,"name":"offline","context":{"idset":"9395"}}
+{"timestamp":1714021379.6791246,"name":"offline","context":{"idset":"9396"}}
+{"timestamp":1714021379.6802921,"name":"offline","context":{"idset":"9397"}}
+{"timestamp":1714021379.6814642,"name":"offline","context":{"idset":"9398"}}
+{"timestamp":1714021379.6826234,"name":"offline","context":{"idset":"9399"}}
+{"timestamp":1714021379.6837904,"name":"offline","context":{"idset":"9400"}}
+{"timestamp":1714021379.6896582,"name":"offline","context":{"idset":"9401"}}
+{"timestamp":1714021379.6908302,"name":"offline","context":{"idset":"9402"}}
+{"timestamp":1714021379.6920059,"name":"offline","context":{"idset":"9403"}}
+{"timestamp":1714021379.6931767,"name":"offline","context":{"idset":"9404"}}
+{"timestamp":1714021379.6943426,"name":"offline","context":{"idset":"9405"}}
+{"timestamp":1714021379.7004113,"name":"offline","context":{"idset":"9406"}}
+{"timestamp":1714021379.7015865,"name":"offline","context":{"idset":"9407"}}
+{"timestamp":1714021379.7027483,"name":"offline","context":{"idset":"9408"}}
+{"timestamp":1714021379.7039135,"name":"offline","context":{"idset":"9409"}}
+{"timestamp":1714021379.7051153,"name":"offline","context":{"idset":"9410"}}
+{"timestamp":1714021379.7062898,"name":"offline","context":{"idset":"9411"}}
+{"timestamp":1714021379.7121363,"name":"offline","context":{"idset":"9412"}}
+{"timestamp":1714021379.7133086,"name":"offline","context":{"idset":"9413"}}
+{"timestamp":1714021379.7144706,"name":"offline","context":{"idset":"9414"}}
+{"timestamp":1714021379.7156339,"name":"offline","context":{"idset":"9415"}}
+{"timestamp":1714021379.7214386,"name":"offline","context":{"idset":"9416"}}
+{"timestamp":1714021379.7226048,"name":"offline","context":{"idset":"9417"}}
+{"timestamp":1714021379.7237549,"name":"offline","context":{"idset":"9418"}}
+{"timestamp":1714021379.7249165,"name":"offline","context":{"idset":"9419"}}
+{"timestamp":1714021379.7308273,"name":"offline","context":{"idset":"9420"}}
+{"timestamp":1714021379.7319925,"name":"offline","context":{"idset":"9421"}}
+{"timestamp":1714021379.7331572,"name":"offline","context":{"idset":"9422"}}
+{"timestamp":1714021379.7343147,"name":"offline","context":{"idset":"9423"}}
+{"timestamp":1714021379.7354805,"name":"offline","context":{"idset":"9424"}}
+{"timestamp":1714021379.7366405,"name":"offline","context":{"idset":"9425"}}
+{"timestamp":1714021379.7378089,"name":"offline","context":{"idset":"9426"}}
+{"timestamp":1714021379.7389677,"name":"offline","context":{"idset":"9427"}}
+{"timestamp":1714021379.7401214,"name":"offline","context":{"idset":"9428"}}
+{"timestamp":1714021379.741272,"name":"offline","context":{"idset":"9429"}}
+{"timestamp":1714021379.742429,"name":"offline","context":{"idset":"9430"}}
+{"timestamp":1714021379.7435806,"name":"offline","context":{"idset":"9431"}}
+{"timestamp":1714021379.7447264,"name":"offline","context":{"idset":"9432"}}
+{"timestamp":1714021379.7458827,"name":"offline","context":{"idset":"9433"}}
+{"timestamp":1714021379.7470343,"name":"offline","context":{"idset":"9434"}}
+{"timestamp":1714021379.7528188,"name":"offline","context":{"idset":"9435"}}
+{"timestamp":1714021379.7539699,"name":"offline","context":{"idset":"9436"}}
+{"timestamp":1714021379.7551208,"name":"offline","context":{"idset":"9437"}}
+{"timestamp":1714021379.7562673,"name":"offline","context":{"idset":"9438"}}
+{"timestamp":1714021379.7620754,"name":"offline","context":{"idset":"9439"}}
+{"timestamp":1714021379.7633622,"name":"offline","context":{"idset":"9440"}}
+{"timestamp":1714021379.764518,"name":"offline","context":{"idset":"9441"}}
+{"timestamp":1714021379.7656653,"name":"offline","context":{"idset":"9442"}}
+{"timestamp":1714021379.7720387,"name":"offline","context":{"idset":"9443"}}
+{"timestamp":1714021379.7731979,"name":"offline","context":{"idset":"9444"}}
+{"timestamp":1714021379.774395,"name":"offline","context":{"idset":"9445"}}
+{"timestamp":1714021379.7755501,"name":"offline","context":{"idset":"9446"}}
+{"timestamp":1714021379.7766907,"name":"offline","context":{"idset":"9447"}}
+{"timestamp":1714021379.7825322,"name":"offline","context":{"idset":"9448"}}
+{"timestamp":1714021379.7883503,"name":"offline","context":{"idset":"9449"}}
+{"timestamp":1714021379.7894921,"name":"offline","context":{"idset":"9450"}}
+{"timestamp":1714021379.7906342,"name":"offline","context":{"idset":"9451"}}
+{"timestamp":1714021379.7917769,"name":"offline","context":{"idset":"9452"}}
+{"timestamp":1714021379.7929235,"name":"offline","context":{"idset":"9453"}}
+{"timestamp":1714021379.7988172,"name":"offline","context":{"idset":"9454"}}
+{"timestamp":1714021379.7999616,"name":"offline","context":{"idset":"9455"}}
+{"timestamp":1714021379.8011067,"name":"offline","context":{"idset":"9456"}}
+{"timestamp":1714021379.8022482,"name":"offline","context":{"idset":"9457"}}
+{"timestamp":1714021379.8034089,"name":"offline","context":{"idset":"9458"}}
+{"timestamp":1714021379.8045707,"name":"offline","context":{"idset":"9459"}}
+{"timestamp":1714021379.8057106,"name":"offline","context":{"idset":"9460"}}
+{"timestamp":1714021379.811482,"name":"offline","context":{"idset":"9461"}}
+{"timestamp":1714021379.8126283,"name":"offline","context":{"idset":"9462"}}
+{"timestamp":1714021379.8184066,"name":"offline","context":{"idset":"9463"}}
+{"timestamp":1714021379.8195662,"name":"offline","context":{"idset":"9464"}}
+{"timestamp":1714021379.8207045,"name":"offline","context":{"idset":"9465"}}
+{"timestamp":1714021379.8218479,"name":"offline","context":{"idset":"9466"}}
+{"timestamp":1714021379.822988,"name":"offline","context":{"idset":"9467"}}
+{"timestamp":1714021379.8241227,"name":"offline","context":{"idset":"9468"}}
+{"timestamp":1714021379.8298848,"name":"offline","context":{"idset":"9469"}}
+{"timestamp":1714021379.8310235,"name":"offline","context":{"idset":"9470"}}
+{"timestamp":1714021379.8321579,"name":"offline","context":{"idset":"9471"}}
+{"timestamp":1714021379.8332903,"name":"offline","context":{"idset":"9472"}}
+{"timestamp":1714021379.8344276,"name":"offline","context":{"idset":"9473"}}
+{"timestamp":1714021379.8355594,"name":"offline","context":{"idset":"9474"}}
+{"timestamp":1714021379.8367085,"name":"offline","context":{"idset":"9475"}}
+{"timestamp":1714021379.8424969,"name":"offline","context":{"idset":"9476"}}
+{"timestamp":1714021379.8436401,"name":"offline","context":{"idset":"9477"}}
+{"timestamp":1714021379.8447859,"name":"offline","context":{"idset":"9478"}}
+{"timestamp":1714021379.845927,"name":"offline","context":{"idset":"9479"}}
+{"timestamp":1714021379.8470592,"name":"offline","context":{"idset":"9480"}}
+{"timestamp":1714021379.848201,"name":"offline","context":{"idset":"9481"}}
+{"timestamp":1714021379.8493323,"name":"offline","context":{"idset":"9482"}}
+{"timestamp":1714021379.8551404,"name":"offline","context":{"idset":"9483"}}
+{"timestamp":1714021379.856272,"name":"offline","context":{"idset":"9484"}}
+{"timestamp":1714021379.8574047,"name":"offline","context":{"idset":"9485"}}
+{"timestamp":1714021379.8585372,"name":"offline","context":{"idset":"9486"}}
+{"timestamp":1714021379.8596666,"name":"offline","context":{"idset":"9487"}}
+{"timestamp":1714021379.8608108,"name":"offline","context":{"idset":"9488"}}
+{"timestamp":1714021379.8619461,"name":"offline","context":{"idset":"9489"}}
+{"timestamp":1714021379.8630726,"name":"offline","context":{"idset":"9490"}}
+{"timestamp":1714021379.8642015,"name":"offline","context":{"idset":"9491"}}
+{"timestamp":1714021379.8699708,"name":"offline","context":{"idset":"9492"}}
+{"timestamp":1714021379.871103,"name":"offline","context":{"idset":"9493"}}
+{"timestamp":1714021379.8722289,"name":"offline","context":{"idset":"9494"}}
+{"timestamp":1714021379.8733516,"name":"offline","context":{"idset":"9495"}}
+{"timestamp":1714021379.8744786,"name":"offline","context":{"idset":"9496"}}
+{"timestamp":1714021379.8756149,"name":"offline","context":{"idset":"9497"}}
+{"timestamp":1714021379.8767428,"name":"offline","context":{"idset":"9498"}}
+{"timestamp":1714021379.8778574,"name":"offline","context":{"idset":"9499"}}
+{"timestamp":1714021379.8837755,"name":"offline","context":{"idset":"9500"}}
+{"timestamp":1714021379.8849201,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1714021379.8860428,"name":"offline","context":{"idset":"9503"}}
+{"timestamp":1714021379.8871477,"name":"offline","context":{"idset":"9504"}}
+{"timestamp":1714021379.8882492,"name":"offline","context":{"idset":"9505"}}
+{"timestamp":1714021379.8893566,"name":"offline","context":{"idset":"9506"}}
+{"timestamp":1714021379.890465,"name":"offline","context":{"idset":"9507"}}
+{"timestamp":1714021379.8961887,"name":"offline","context":{"idset":"9508"}}
+{"timestamp":1714021379.8972986,"name":"offline","context":{"idset":"9509"}}
+{"timestamp":1714021379.8984106,"name":"offline","context":{"idset":"9510"}}
+{"timestamp":1714021379.8995252,"name":"offline","context":{"idset":"9511"}}
+{"timestamp":1714021379.9006319,"name":"offline","context":{"idset":"9512"}}
+{"timestamp":1714021379.9017365,"name":"offline","context":{"idset":"9513"}}
+{"timestamp":1714021379.9077477,"name":"offline","context":{"idset":"9514"}}
+{"timestamp":1714021379.9090583,"name":"offline","context":{"idset":"9515"}}
+{"timestamp":1714021379.9102621,"name":"offline","context":{"idset":"9516"}}
+{"timestamp":1714021379.9117923,"name":"offline","context":{"idset":"9517"}}
+{"timestamp":1714021379.9133523,"name":"offline","context":{"idset":"9518"}}
+{"timestamp":1714021379.9149835,"name":"offline","context":{"idset":"9519"}}
+{"timestamp":1714021379.9161928,"name":"offline","context":{"idset":"9520"}}
+{"timestamp":1714021379.9219198,"name":"offline","context":{"idset":"9521"}}
+{"timestamp":1714021379.923023,"name":"offline","context":{"idset":"9522"}}
+{"timestamp":1714021379.9241927,"name":"offline","context":{"idset":"9523"}}
+{"timestamp":1714021379.925312,"name":"offline","context":{"idset":"9524"}}
+{"timestamp":1714021379.9264202,"name":"offline","context":{"idset":"9525"}}
+{"timestamp":1714021379.9275255,"name":"offline","context":{"idset":"9526"}}
+{"timestamp":1714021379.9286342,"name":"offline","context":{"idset":"9527"}}
+{"timestamp":1714021379.929729,"name":"offline","context":{"idset":"9528"}}
+{"timestamp":1714021379.9355197,"name":"offline","context":{"idset":"9529"}}
+{"timestamp":1714021379.9366231,"name":"offline","context":{"idset":"9530"}}
+{"timestamp":1714021379.937737,"name":"offline","context":{"idset":"9531"}}
+{"timestamp":1714021379.9388504,"name":"offline","context":{"idset":"9532"}}
+{"timestamp":1714021379.9399595,"name":"offline","context":{"idset":"9533"}}
+{"timestamp":1714021379.9410563,"name":"offline","context":{"idset":"9534"}}
+{"timestamp":1714021379.9421554,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1714021379.9432511,"name":"offline","context":{"idset":"9536"}}
+{"timestamp":1714021379.9490061,"name":"offline","context":{"idset":"9537"}}
+{"timestamp":1714021379.950109,"name":"offline","context":{"idset":"9538"}}
+{"timestamp":1714021379.9512053,"name":"offline","context":{"idset":"9539"}}
+{"timestamp":1714021379.9523051,"name":"offline","context":{"idset":"9540"}}
+{"timestamp":1714021379.9534013,"name":"offline","context":{"idset":"9541"}}
+{"timestamp":1714021379.9548757,"name":"offline","context":{"idset":"9542"}}
+{"timestamp":1714021379.9565115,"name":"offline","context":{"idset":"9543"}}
+{"timestamp":1714021379.9581208,"name":"offline","context":{"idset":"9544"}}
+{"timestamp":1714021379.9662719,"name":"offline","context":{"idset":"9545"}}
+{"timestamp":1714021379.9675238,"name":"offline","context":{"idset":"9546"}}
+{"timestamp":1714021379.9686217,"name":"offline","context":{"idset":"9547"}}
+{"timestamp":1714021379.9697239,"name":"offline","context":{"idset":"9548"}}
+{"timestamp":1714021379.9708223,"name":"offline","context":{"idset":"9549"}}
+{"timestamp":1714021379.9719152,"name":"offline","context":{"idset":"9550"}}
+{"timestamp":1714021379.9730132,"name":"offline","context":{"idset":"9551"}}
+{"timestamp":1714021379.9741099,"name":"offline","context":{"idset":"9552"}}
+{"timestamp":1714021379.9752004,"name":"offline","context":{"idset":"9553"}}
+{"timestamp":1714021379.9809425,"name":"offline","context":{"idset":"9554"}}
+{"timestamp":1714021379.9820352,"name":"offline","context":{"idset":"9555"}}
+{"timestamp":1714021379.983134,"name":"offline","context":{"idset":"9556"}}
+{"timestamp":1714021379.9842212,"name":"offline","context":{"idset":"9557"}}
+{"timestamp":1714021379.9853106,"name":"offline","context":{"idset":"9558"}}
+{"timestamp":1714021379.9864082,"name":"offline","context":{"idset":"9559"}}
+{"timestamp":1714021379.9874916,"name":"offline","context":{"idset":"9560"}}
+{"timestamp":1714021379.9932449,"name":"offline","context":{"idset":"9561"}}
+{"timestamp":1714021379.9943357,"name":"offline","context":{"idset":"9562"}}
+{"timestamp":1714021379.995424,"name":"offline","context":{"idset":"9563"}}
+{"timestamp":1714021379.996505,"name":"offline","context":{"idset":"9564"}}
+{"timestamp":1714021379.997596,"name":"offline","context":{"idset":"9565"}}
+{"timestamp":1714021379.9986725,"name":"offline","context":{"idset":"9566"}}
+{"timestamp":1714021379.9997475,"name":"offline","context":{"idset":"9567"}}
+{"timestamp":1714021380.0008357,"name":"offline","context":{"idset":"9568"}}
+{"timestamp":1714021380.0066066,"name":"offline","context":{"idset":"9569"}}
+{"timestamp":1714021380.0076897,"name":"offline","context":{"idset":"9570"}}
+{"timestamp":1714021380.0087681,"name":"offline","context":{"idset":"9571"}}
+{"timestamp":1714021380.0098507,"name":"offline","context":{"idset":"9572"}}
+{"timestamp":1714021380.0109348,"name":"offline","context":{"idset":"9573"}}
+{"timestamp":1714021380.0120096,"name":"offline","context":{"idset":"9574"}}
+{"timestamp":1714021380.0130911,"name":"offline","context":{"idset":"9575"}}
+{"timestamp":1714021380.0141637,"name":"offline","context":{"idset":"9576"}}
+{"timestamp":1714021380.015238,"name":"offline","context":{"idset":"9577"}}
+{"timestamp":1714021380.0163059,"name":"offline","context":{"idset":"9578"}}
+{"timestamp":1714021380.0173783,"name":"offline","context":{"idset":"9579"}}
+{"timestamp":1714021380.0184469,"name":"offline","context":{"idset":"9580"}}
+{"timestamp":1714021380.0195172,"name":"offline","context":{"idset":"9581"}}
+{"timestamp":1714021380.0205956,"name":"offline","context":{"idset":"9582"}}
+{"timestamp":1714021380.0216656,"name":"offline","context":{"idset":"9583"}}
+{"timestamp":1714021380.0227356,"name":"offline","context":{"idset":"9584"}}
+{"timestamp":1714021380.0238271,"name":"offline","context":{"idset":"9585"}}
+{"timestamp":1714021380.0249023,"name":"offline","context":{"idset":"9586"}}
+{"timestamp":1714021380.0259781,"name":"offline","context":{"idset":"9587"}}
+{"timestamp":1714021380.027055,"name":"offline","context":{"idset":"9588"}}
+{"timestamp":1714021380.0281308,"name":"offline","context":{"idset":"9589"}}
+{"timestamp":1714021380.0292044,"name":"offline","context":{"idset":"9590"}}
+{"timestamp":1714021380.0302839,"name":"offline","context":{"idset":"9591"}}
+{"timestamp":1714021380.0313566,"name":"offline","context":{"idset":"9592"}}
+{"timestamp":1714021380.0324292,"name":"offline","context":{"idset":"9593"}}
+{"timestamp":1714021380.0335002,"name":"offline","context":{"idset":"9594"}}
+{"timestamp":1714021380.0345695,"name":"offline","context":{"idset":"9595"}}
+{"timestamp":1714021380.035671,"name":"offline","context":{"idset":"9596"}}
+{"timestamp":1714021380.036747,"name":"offline","context":{"idset":"9597"}}
+{"timestamp":1714021380.037822,"name":"offline","context":{"idset":"9598"}}
+{"timestamp":1714021380.038892,"name":"offline","context":{"idset":"9599"}}
+{"timestamp":1714021380.0446048,"name":"offline","context":{"idset":"9600"}}
+{"timestamp":1714021380.0456767,"name":"offline","context":{"idset":"9601"}}
+{"timestamp":1714021380.046741,"name":"offline","context":{"idset":"9602"}}
+{"timestamp":1714021380.0478075,"name":"offline","context":{"idset":"9603"}}
+{"timestamp":1714021380.0488639,"name":"offline","context":{"idset":"9604"}}
+{"timestamp":1714021380.0499296,"name":"offline","context":{"idset":"9605"}}
+{"timestamp":1714021380.0509961,"name":"offline","context":{"idset":"9606"}}
+{"timestamp":1714021380.0520627,"name":"offline","context":{"idset":"9607"}}
+{"timestamp":1714021380.0531292,"name":"offline","context":{"idset":"9608"}}
+{"timestamp":1714021380.0589523,"name":"offline","context":{"idset":"9609"}}
+{"timestamp":1714021380.0600238,"name":"offline","context":{"idset":"9610"}}
+{"timestamp":1714021380.061096,"name":"offline","context":{"idset":"9611"}}
+{"timestamp":1714021380.06216,"name":"offline","context":{"idset":"9612"}}
+{"timestamp":1714021380.0632224,"name":"offline","context":{"idset":"9613"}}
+{"timestamp":1714021380.0642877,"name":"offline","context":{"idset":"9614"}}
+{"timestamp":1714021380.0653515,"name":"offline","context":{"idset":"9615"}}
+{"timestamp":1714021380.0711112,"name":"offline","context":{"idset":"9616"}}
+{"timestamp":1714021380.0721848,"name":"offline","context":{"idset":"9617"}}
+{"timestamp":1714021380.0732436,"name":"offline","context":{"idset":"9618"}}
+{"timestamp":1714021380.0743051,"name":"offline","context":{"idset":"9619"}}
+{"timestamp":1714021380.0753632,"name":"offline","context":{"idset":"9620"}}
+{"timestamp":1714021380.0764182,"name":"offline","context":{"idset":"9621"}}
+{"timestamp":1714021380.077477,"name":"offline","context":{"idset":"9622"}}
+{"timestamp":1714021380.0785916,"name":"offline","context":{"idset":"9623"}}
+{"timestamp":1714021380.0796518,"name":"offline","context":{"idset":"9624"}}
+{"timestamp":1714021380.0807052,"name":"offline","context":{"idset":"9625"}}
+{"timestamp":1714021380.081754,"name":"offline","context":{"idset":"9626"}}
+{"timestamp":1714021380.0828083,"name":"offline","context":{"idset":"9627"}}
+{"timestamp":1714021380.0838602,"name":"offline","context":{"idset":"9628"}}
+{"timestamp":1714021380.084908,"name":"offline","context":{"idset":"9629"}}
+{"timestamp":1714021380.0859644,"name":"offline","context":{"idset":"9630"}}
+{"timestamp":1714021380.0870149,"name":"offline","context":{"idset":"9631"}}
+{"timestamp":1714021380.0880764,"name":"offline","context":{"idset":"9632"}}
+{"timestamp":1714021380.0891256,"name":"offline","context":{"idset":"9633"}}
+{"timestamp":1714021380.0901728,"name":"offline","context":{"idset":"9634"}}
+{"timestamp":1714021380.0912149,"name":"offline","context":{"idset":"9635"}}
+{"timestamp":1714021380.0922701,"name":"offline","context":{"idset":"9636"}}
+{"timestamp":1714021380.0933144,"name":"offline","context":{"idset":"9637"}}
+{"timestamp":1714021380.0943611,"name":"offline","context":{"idset":"9638"}}
+{"timestamp":1714021380.0954018,"name":"offline","context":{"idset":"9639"}}
+{"timestamp":1714021380.0964732,"name":"offline","context":{"idset":"9640"}}
+{"timestamp":1714021380.0975254,"name":"offline","context":{"idset":"9641"}}
+{"timestamp":1714021380.0985873,"name":"offline","context":{"idset":"9642"}}
+{"timestamp":1714021380.0996311,"name":"offline","context":{"idset":"9643"}}
+{"timestamp":1714021380.1006711,"name":"offline","context":{"idset":"9644"}}
+{"timestamp":1714021380.1017067,"name":"offline","context":{"idset":"9645"}}
+{"timestamp":1714021380.1074684,"name":"offline","context":{"idset":"9646"}}
+{"timestamp":1714021380.1085472,"name":"offline","context":{"idset":"9647"}}
+{"timestamp":1714021380.1096022,"name":"offline","context":{"idset":"9648"}}
+{"timestamp":1714021380.1111403,"name":"offline","context":{"idset":"9649"}}
+{"timestamp":1714021380.1126223,"name":"offline","context":{"idset":"9650"}}
+{"timestamp":1714021380.114337,"name":"offline","context":{"idset":"9651"}}
+{"timestamp":1714021380.1155193,"name":"offline","context":{"idset":"9652"}}
+{"timestamp":1714021380.117198,"name":"offline","context":{"idset":"9653"}}
+{"timestamp":1714021380.1248314,"name":"offline","context":{"idset":"9654"}}
+{"timestamp":1714021380.1261089,"name":"offline","context":{"idset":"9655"}}
+{"timestamp":1714021380.1278477,"name":"offline","context":{"idset":"9656"}}
+{"timestamp":1714021380.1294265,"name":"offline","context":{"idset":"9657"}}
+{"timestamp":1714021380.1309874,"name":"offline","context":{"idset":"9658"}}
+{"timestamp":1714021380.1327224,"name":"offline","context":{"idset":"9659"}}
+{"timestamp":1714021380.1341276,"name":"offline","context":{"idset":"9660"}}
+{"timestamp":1714021380.1358523,"name":"offline","context":{"idset":"9661"}}
+{"timestamp":1714021380.1376157,"name":"offline","context":{"idset":"9662"}}
+{"timestamp":1714021380.1389947,"name":"offline","context":{"idset":"9663"}}
+{"timestamp":1714021380.1408198,"name":"offline","context":{"idset":"9664"}}
+{"timestamp":1714021380.1426003,"name":"offline","context":{"idset":"9665"}}
+{"timestamp":1714021380.1441038,"name":"offline","context":{"idset":"9666"}}
+{"timestamp":1714021380.1458666,"name":"offline","context":{"idset":"9667"}}
+{"timestamp":1714021380.1476536,"name":"offline","context":{"idset":"9668"}}
+{"timestamp":1714021380.1493416,"name":"offline","context":{"idset":"9669"}}
+{"timestamp":1714021380.151046,"name":"offline","context":{"idset":"9670"}}
+{"timestamp":1714021380.1528244,"name":"offline","context":{"idset":"9671"}}
+{"timestamp":1714021380.1550889,"name":"offline","context":{"idset":"9672"}}
+{"timestamp":1714021380.157217,"name":"offline","context":{"idset":"9673"}}
+{"timestamp":1714021380.1593263,"name":"offline","context":{"idset":"9674"}}
+{"timestamp":1714021380.1614196,"name":"offline","context":{"idset":"9675"}}
+{"timestamp":1714021380.1635516,"name":"offline","context":{"idset":"9676"}}
+{"timestamp":1714021380.165657,"name":"offline","context":{"idset":"9677"}}
+{"timestamp":1714021380.1677563,"name":"offline","context":{"idset":"9678"}}
+{"timestamp":1714021380.169883,"name":"offline","context":{"idset":"9679"}}
+{"timestamp":1714021380.1719804,"name":"offline","context":{"idset":"9680"}}
+{"timestamp":1714021380.1740816,"name":"offline","context":{"idset":"9681"}}
+{"timestamp":1714021380.1761682,"name":"offline","context":{"idset":"9682"}}
+{"timestamp":1714021380.1782663,"name":"offline","context":{"idset":"9683"}}
+{"timestamp":1714021380.1803808,"name":"offline","context":{"idset":"9684"}}
+{"timestamp":1714021380.1920297,"name":"offline","context":{"idset":"9685"}}
+{"timestamp":1714021380.19416,"name":"offline","context":{"idset":"9686"}}
+{"timestamp":1714021380.1962531,"name":"offline","context":{"idset":"9687"}}
+{"timestamp":1714021380.1983697,"name":"offline","context":{"idset":"9688"}}
+{"timestamp":1714021380.2004695,"name":"offline","context":{"idset":"9689"}}
+{"timestamp":1714021380.2025847,"name":"offline","context":{"idset":"9690"}}
+{"timestamp":1714021380.2046907,"name":"offline","context":{"idset":"9691"}}
+{"timestamp":1714021380.2163491,"name":"offline","context":{"idset":"9692"}}
+{"timestamp":1714021380.218473,"name":"offline","context":{"idset":"9693"}}
+{"timestamp":1714021380.220578,"name":"offline","context":{"idset":"9694"}}
+{"timestamp":1714021380.2226653,"name":"offline","context":{"idset":"9695"}}
+{"timestamp":1714021380.2248242,"name":"offline","context":{"idset":"9696"}}
+{"timestamp":1714021380.2269354,"name":"offline","context":{"idset":"9697"}}
+{"timestamp":1714021380.2290223,"name":"offline","context":{"idset":"9698"}}
+{"timestamp":1714021380.2311347,"name":"offline","context":{"idset":"9699"}}
+{"timestamp":1714021380.2429233,"name":"offline","context":{"idset":"9700"}}
+{"timestamp":1714021380.2450194,"name":"offline","context":{"idset":"9701"}}
+{"timestamp":1714021380.2471027,"name":"offline","context":{"idset":"9702"}}
+{"timestamp":1714021380.2491777,"name":"offline","context":{"idset":"9703"}}
+{"timestamp":1714021380.2512479,"name":"offline","context":{"idset":"9704"}}
+{"timestamp":1714021380.2533305,"name":"offline","context":{"idset":"9705"}}
+{"timestamp":1714021380.2554023,"name":"offline","context":{"idset":"9706"}}
+{"timestamp":1714021380.257473,"name":"offline","context":{"idset":"9707"}}
+{"timestamp":1714021380.2595339,"name":"offline","context":{"idset":"9708"}}
+{"timestamp":1714021380.2711887,"name":"offline","context":{"idset":"9709"}}
+{"timestamp":1714021380.2732866,"name":"offline","context":{"idset":"9710"}}
+{"timestamp":1714021380.2753782,"name":"offline","context":{"idset":"9711"}}
+{"timestamp":1714021380.2774589,"name":"offline","context":{"idset":"9712"}}
+{"timestamp":1714021380.2795672,"name":"offline","context":{"idset":"9713"}}
+{"timestamp":1714021380.2816725,"name":"offline","context":{"idset":"9714"}}
+{"timestamp":1714021380.2837598,"name":"offline","context":{"idset":"9715"}}
+{"timestamp":1714021380.2858849,"name":"offline","context":{"idset":"9716"}}
+{"timestamp":1714021380.2975321,"name":"offline","context":{"idset":"9717"}}
+{"timestamp":1714021380.2996206,"name":"offline","context":{"idset":"9718"}}
+{"timestamp":1714021380.3016884,"name":"offline","context":{"idset":"9719"}}
+{"timestamp":1714021380.3037739,"name":"offline","context":{"idset":"9720"}}
+{"timestamp":1714021380.3058658,"name":"offline","context":{"idset":"9721"}}
+{"timestamp":1714021380.3079224,"name":"offline","context":{"idset":"9722"}}
+{"timestamp":1714021380.3100076,"name":"offline","context":{"idset":"9723"}}
+{"timestamp":1714021380.3120718,"name":"offline","context":{"idset":"9724"}}
+{"timestamp":1714021380.3141484,"name":"offline","context":{"idset":"9725"}}
+{"timestamp":1714021380.3162465,"name":"offline","context":{"idset":"9726"}}
+{"timestamp":1714021380.318327,"name":"offline","context":{"idset":"9727"}}
+{"timestamp":1714021380.3204124,"name":"offline","context":{"idset":"9728"}}
+{"timestamp":1714021380.3224785,"name":"offline","context":{"idset":"9729"}}
+{"timestamp":1714021380.3245761,"name":"offline","context":{"idset":"9730"}}
+{"timestamp":1714021380.3267167,"name":"offline","context":{"idset":"9731"}}
+{"timestamp":1714021380.3301947,"name":"offline","context":{"idset":"9732"}}
+{"timestamp":1714021380.332725,"name":"offline","context":{"idset":"9733"}}
+{"timestamp":1714021380.3348134,"name":"offline","context":{"idset":"9734"}}
+{"timestamp":1714021380.3368838,"name":"offline","context":{"idset":"9735"}}
+{"timestamp":1714021380.338918,"name":"offline","context":{"idset":"9736"}}
+{"timestamp":1714021380.3409855,"name":"offline","context":{"idset":"9737"}}
+{"timestamp":1714021380.3431451,"name":"offline","context":{"idset":"9738"}}
+{"timestamp":1714021380.3472373,"name":"offline","context":{"idset":"9739"}}
+{"timestamp":1714021380.3513114,"name":"offline","context":{"idset":"9740"}}
+{"timestamp":1714021380.3553822,"name":"offline","context":{"idset":"9741"}}
+{"timestamp":1714021380.3584504,"name":"offline","context":{"idset":"9742"}}
+{"timestamp":1714021380.3605232,"name":"offline","context":{"idset":"9743"}}
+{"timestamp":1714021380.3625832,"name":"offline","context":{"idset":"9744"}}
+{"timestamp":1714021380.3646455,"name":"offline","context":{"idset":"9745"}}
+{"timestamp":1714021380.3666952,"name":"offline","context":{"idset":"9746"}}
+{"timestamp":1714021380.3687253,"name":"offline","context":{"idset":"9747"}}
+{"timestamp":1714021380.3707578,"name":"offline","context":{"idset":"9748"}}
+{"timestamp":1714021380.3728104,"name":"offline","context":{"idset":"9749"}}
+{"timestamp":1714021380.3844995,"name":"offline","context":{"idset":"9750"}}
+{"timestamp":1714021380.3865595,"name":"offline","context":{"idset":"9751"}}
+{"timestamp":1714021380.3886206,"name":"offline","context":{"idset":"9752"}}
+{"timestamp":1714021380.3906591,"name":"offline","context":{"idset":"9753"}}
+{"timestamp":1714021380.392797,"name":"offline","context":{"idset":"9754"}}
+{"timestamp":1714021380.3948774,"name":"offline","context":{"idset":"9755"}}
+{"timestamp":1714021380.3969059,"name":"offline","context":{"idset":"9756"}}
+{"timestamp":1714021380.3989644,"name":"offline","context":{"idset":"9757"}}
+{"timestamp":1714021380.4009981,"name":"offline","context":{"idset":"9758"}}
+{"timestamp":1714021380.4030459,"name":"offline","context":{"idset":"9759"}}
+{"timestamp":1714021380.4050727,"name":"offline","context":{"idset":"9760"}}
+{"timestamp":1714021380.4071176,"name":"offline","context":{"idset":"9761"}}
+{"timestamp":1714021380.4091487,"name":"offline","context":{"idset":"9762"}}
+{"timestamp":1714021380.4111896,"name":"offline","context":{"idset":"9763"}}
+{"timestamp":1714021380.4132266,"name":"offline","context":{"idset":"9764"}}
+{"timestamp":1714021380.4152577,"name":"offline","context":{"idset":"9765"}}
+{"timestamp":1714021380.417294,"name":"offline","context":{"idset":"9766"}}
+{"timestamp":1714021380.419337,"name":"offline","context":{"idset":"9767"}}
+{"timestamp":1714021380.421365,"name":"offline","context":{"idset":"9768"}}
+{"timestamp":1714021380.4233882,"name":"offline","context":{"idset":"9769"}}
+{"timestamp":1714021380.4254684,"name":"offline","context":{"idset":"9770"}}
+{"timestamp":1714021380.4275188,"name":"offline","context":{"idset":"9771"}}
+{"timestamp":1714021380.4295514,"name":"offline","context":{"idset":"9772"}}
+{"timestamp":1714021380.4315991,"name":"offline","context":{"idset":"9773"}}
+{"timestamp":1714021380.4336398,"name":"offline","context":{"idset":"9774"}}
+{"timestamp":1714021380.4356687,"name":"offline","context":{"idset":"9775"}}
+{"timestamp":1714021380.4377029,"name":"offline","context":{"idset":"9776"}}
+{"timestamp":1714021380.439734,"name":"offline","context":{"idset":"9777"}}
+{"timestamp":1714021380.4417799,"name":"offline","context":{"idset":"9778"}}
+{"timestamp":1714021380.4438274,"name":"offline","context":{"idset":"9779"}}
+{"timestamp":1714021380.4458456,"name":"offline","context":{"idset":"9780"}}
+{"timestamp":1714021380.4479246,"name":"offline","context":{"idset":"9781"}}
+{"timestamp":1714021380.4595201,"name":"offline","context":{"idset":"9782"}}
+{"timestamp":1714021380.4615581,"name":"offline","context":{"idset":"9783"}}
+{"timestamp":1714021380.4636693,"name":"offline","context":{"idset":"9784"}}
+{"timestamp":1714021380.465704,"name":"offline","context":{"idset":"9785"}}
+{"timestamp":1714021380.467762,"name":"offline","context":{"idset":"9786"}}
+{"timestamp":1714021380.4697919,"name":"offline","context":{"idset":"9787"}}
+{"timestamp":1714021380.471813,"name":"offline","context":{"idset":"9788"}}
+{"timestamp":1714021380.473799,"name":"offline","context":{"idset":"9789"}}
+{"timestamp":1714021380.4758928,"name":"offline","context":{"idset":"9790"}}
+{"timestamp":1714021380.4778988,"name":"offline","context":{"idset":"9791"}}
+{"timestamp":1714021380.4798839,"name":"offline","context":{"idset":"9792"}}
+{"timestamp":1714021380.4914913,"name":"offline","context":{"idset":"9793"}}
+{"timestamp":1714021380.4935005,"name":"offline","context":{"idset":"9794"}}
+{"timestamp":1714021380.4955366,"name":"offline","context":{"idset":"9795"}}
+{"timestamp":1714021380.4975777,"name":"offline","context":{"idset":"9796"}}
+{"timestamp":1714021380.499578,"name":"offline","context":{"idset":"9797"}}
+{"timestamp":1714021380.5015717,"name":"offline","context":{"idset":"9798"}}
+{"timestamp":1714021380.5036204,"name":"offline","context":{"idset":"9799"}}
+{"timestamp":1714021380.5056102,"name":"offline","context":{"idset":"9800"}}
+{"timestamp":1714021380.5076015,"name":"offline","context":{"idset":"9801"}}
+{"timestamp":1714021380.5190048,"name":"offline","context":{"idset":"9802"}}
+{"timestamp":1714021380.5210021,"name":"offline","context":{"idset":"9803"}}
+{"timestamp":1714021380.5229964,"name":"offline","context":{"idset":"9804"}}
+{"timestamp":1714021380.5250204,"name":"offline","context":{"idset":"9805"}}
+{"timestamp":1714021380.527035,"name":"offline","context":{"idset":"9806"}}
+{"timestamp":1714021380.5290248,"name":"offline","context":{"idset":"9807"}}
+{"timestamp":1714021380.5310292,"name":"offline","context":{"idset":"9808"}}
+{"timestamp":1714021380.5330975,"name":"offline","context":{"idset":"9809"}}
+{"timestamp":1714021380.5350924,"name":"offline","context":{"idset":"9810"}}
+{"timestamp":1714021380.5465672,"name":"offline","context":{"idset":"9811"}}
+{"timestamp":1714021380.5485461,"name":"offline","context":{"idset":"9812"}}
+{"timestamp":1714021380.5505149,"name":"offline","context":{"idset":"9813"}}
+{"timestamp":1714021380.5524952,"name":"offline","context":{"idset":"9814"}}
+{"timestamp":1714021380.5544603,"name":"offline","context":{"idset":"9815"}}
+{"timestamp":1714021380.5577908,"name":"offline","context":{"idset":"9816"}}
+{"timestamp":1714021380.5743444,"name":"offline","context":{"idset":"9817"}}
+{"timestamp":1714021380.5768554,"name":"offline","context":{"idset":"9818"}}
+{"timestamp":1714021380.5788391,"name":"offline","context":{"idset":"9819"}}
+{"timestamp":1714021380.5808117,"name":"offline","context":{"idset":"9820"}}
+{"timestamp":1714021380.5827839,"name":"offline","context":{"idset":"9821"}}
+{"timestamp":1714021380.5847585,"name":"offline","context":{"idset":"9822"}}
+{"timestamp":1714021380.5867279,"name":"offline","context":{"idset":"9823"}}
+{"timestamp":1714021380.5887012,"name":"offline","context":{"idset":"9824"}}
+{"timestamp":1714021380.5906487,"name":"offline","context":{"idset":"9825"}}
+{"timestamp":1714021380.5925996,"name":"offline","context":{"idset":"9826"}}
+{"timestamp":1714021380.6040547,"name":"offline","context":{"idset":"9827"}}
+{"timestamp":1714021380.6060238,"name":"offline","context":{"idset":"9828"}}
+{"timestamp":1714021380.6079943,"name":"offline","context":{"idset":"9829"}}
+{"timestamp":1714021380.6099622,"name":"offline","context":{"idset":"9830"}}
+{"timestamp":1714021380.6119268,"name":"offline","context":{"idset":"9831"}}
+{"timestamp":1714021380.6138804,"name":"offline","context":{"idset":"9832"}}
+{"timestamp":1714021380.6160882,"name":"offline","context":{"idset":"9833"}}
+{"timestamp":1714021380.6193175,"name":"offline","context":{"idset":"9834"}}
+{"timestamp":1714021380.6308143,"name":"offline","context":{"idset":"9835"}}
+{"timestamp":1714021380.6327856,"name":"offline","context":{"idset":"9836"}}
+{"timestamp":1714021380.6347518,"name":"offline","context":{"idset":"9837"}}
+{"timestamp":1714021380.6367257,"name":"offline","context":{"idset":"9838"}}
+{"timestamp":1714021380.6386847,"name":"offline","context":{"idset":"9839"}}
+{"timestamp":1714021380.6406331,"name":"offline","context":{"idset":"9840"}}
+{"timestamp":1714021380.6425781,"name":"offline","context":{"idset":"9841"}}
+{"timestamp":1714021380.6445162,"name":"offline","context":{"idset":"9842"}}
+{"timestamp":1714021380.6464593,"name":"offline","context":{"idset":"9843"}}
+{"timestamp":1714021380.6579256,"name":"offline","context":{"idset":"9844"}}
+{"timestamp":1714021380.6600082,"name":"offline","context":{"idset":"9845"}}
+{"timestamp":1714021380.662092,"name":"offline","context":{"idset":"9846"}}
+{"timestamp":1714021380.6641657,"name":"offline","context":{"idset":"9847"}}
+{"timestamp":1714021380.6662471,"name":"offline","context":{"idset":"9848"}}
+{"timestamp":1714021380.6683252,"name":"offline","context":{"idset":"9849"}}
+{"timestamp":1714021380.670418,"name":"offline","context":{"idset":"9850"}}
+{"timestamp":1714021380.6724825,"name":"offline","context":{"idset":"9851"}}
+{"timestamp":1714021380.6745458,"name":"offline","context":{"idset":"9852"}}
+{"timestamp":1714021380.6868196,"name":"offline","context":{"idset":"9853"}}
+{"timestamp":1714021380.6889093,"name":"offline","context":{"idset":"9854"}}
+{"timestamp":1714021380.6909914,"name":"offline","context":{"idset":"9855"}}
+{"timestamp":1714021380.6930523,"name":"offline","context":{"idset":"9856"}}
+{"timestamp":1714021380.6951237,"name":"offline","context":{"idset":"9857"}}
+{"timestamp":1714021380.69719,"name":"offline","context":{"idset":"9858"}}
+{"timestamp":1714021380.6992741,"name":"offline","context":{"idset":"9859"}}
+{"timestamp":1714021380.7014136,"name":"offline","context":{"idset":"9860"}}
+{"timestamp":1714021380.7034762,"name":"offline","context":{"idset":"9861"}}
+{"timestamp":1714021380.7156773,"name":"offline","context":{"idset":"9862"}}
+{"timestamp":1714021380.7177446,"name":"offline","context":{"idset":"9863"}}
+{"timestamp":1714021380.7198229,"name":"offline","context":{"idset":"9864"}}
+{"timestamp":1714021380.7218916,"name":"offline","context":{"idset":"9865"}}
+{"timestamp":1714021380.7239563,"name":"offline","context":{"idset":"9866"}}
+{"timestamp":1714021380.7260332,"name":"offline","context":{"idset":"9867"}}
+{"timestamp":1714021380.7280898,"name":"offline","context":{"idset":"9868"}}
+{"timestamp":1714021380.7301495,"name":"offline","context":{"idset":"9869"}}
+{"timestamp":1714021380.7322035,"name":"offline","context":{"idset":"9870"}}
+{"timestamp":1714021380.7443779,"name":"offline","context":{"idset":"9871"}}
+{"timestamp":1714021380.7464581,"name":"offline","context":{"idset":"9872"}}
+{"timestamp":1714021380.7495706,"name":"offline","context":{"idset":"9873"}}
+{"timestamp":1714021380.7516255,"name":"offline","context":{"idset":"9874"}}
+{"timestamp":1714021380.7536509,"name":"offline","context":{"idset":"9875"}}
+{"timestamp":1714021380.7557111,"name":"offline","context":{"idset":"9876"}}
+{"timestamp":1714021380.7577322,"name":"offline","context":{"idset":"9877"}}
+{"timestamp":1714021380.7597644,"name":"offline","context":{"idset":"9878"}}
+{"timestamp":1714021380.7617841,"name":"offline","context":{"idset":"9879"}}
+{"timestamp":1714021380.7638245,"name":"offline","context":{"idset":"9880"}}
+{"timestamp":1714021380.7762675,"name":"offline","context":{"idset":"9881"}}
+{"timestamp":1714021380.7781992,"name":"offline","context":{"idset":"9882"}}
+{"timestamp":1714021380.7801275,"name":"offline","context":{"idset":"9883"}}
+{"timestamp":1714021380.7820346,"name":"offline","context":{"idset":"9884"}}
+{"timestamp":1714021380.7839463,"name":"offline","context":{"idset":"9885"}}
+{"timestamp":1714021380.7859201,"name":"offline","context":{"idset":"9886"}}
+{"timestamp":1714021380.7878315,"name":"offline","context":{"idset":"9887"}}
+{"timestamp":1714021380.7901468,"name":"offline","context":{"idset":"9888"}}
+{"timestamp":1714021380.7923737,"name":"offline","context":{"idset":"9889"}}
+{"timestamp":1714021380.8086448,"name":"offline","context":{"idset":"9890"}}
+{"timestamp":1714021380.8106427,"name":"offline","context":{"idset":"9891"}}
+{"timestamp":1714021380.8126273,"name":"offline","context":{"idset":"9892"}}
+{"timestamp":1714021380.8149927,"name":"offline","context":{"idset":"9893"}}
+{"timestamp":1714021380.8172138,"name":"offline","context":{"idset":"9894"}}
+{"timestamp":1714021380.8191047,"name":"offline","context":{"idset":"9895"}}
+{"timestamp":1714021380.8212759,"name":"offline","context":{"idset":"9896"}}
+{"timestamp":1714021380.823199,"name":"offline","context":{"idset":"9897"}}
+{"timestamp":1714021380.8250918,"name":"offline","context":{"idset":"9898"}}
+{"timestamp":1714021380.8269722,"name":"offline","context":{"idset":"9899"}}
+{"timestamp":1714021380.8382893,"name":"offline","context":{"idset":"9900"}}
+{"timestamp":1714021380.8401842,"name":"offline","context":{"idset":"9901"}}
+{"timestamp":1714021380.8420789,"name":"offline","context":{"idset":"9902"}}
+{"timestamp":1714021380.8439758,"name":"offline","context":{"idset":"9903"}}
+{"timestamp":1714021380.8463333,"name":"offline","context":{"idset":"9904"}}
+{"timestamp":1714021380.8491678,"name":"offline","context":{"idset":"9905"}}
+{"timestamp":1714021380.8520067,"name":"offline","context":{"idset":"9906"}}
+{"timestamp":1714021380.8548105,"name":"offline","context":{"idset":"9907"}}
+{"timestamp":1714021380.8572366,"name":"offline","context":{"idset":"9908"}}
+{"timestamp":1714021380.8591518,"name":"offline","context":{"idset":"9909"}}
+{"timestamp":1714021380.861064,"name":"offline","context":{"idset":"9910"}}
+{"timestamp":1714021380.8629558,"name":"offline","context":{"idset":"9911"}}
+{"timestamp":1714021380.8743212,"name":"offline","context":{"idset":"9912"}}
+{"timestamp":1714021380.8762138,"name":"offline","context":{"idset":"9913"}}
+{"timestamp":1714021380.8781471,"name":"offline","context":{"idset":"9914"}}
+{"timestamp":1714021380.8800302,"name":"offline","context":{"idset":"9915"}}
+{"timestamp":1714021380.881901,"name":"offline","context":{"idset":"9916"}}
+{"timestamp":1714021380.8837943,"name":"offline","context":{"idset":"9917"}}
+{"timestamp":1714021380.8951309,"name":"offline","context":{"idset":"9918"}}
+{"timestamp":1714021380.897002,"name":"offline","context":{"idset":"9919"}}
+{"timestamp":1714021380.898881,"name":"offline","context":{"idset":"9920"}}
+{"timestamp":1714021380.9007587,"name":"offline","context":{"idset":"9921"}}
+{"timestamp":1714021380.9026396,"name":"offline","context":{"idset":"9922"}}
+{"timestamp":1714021380.9045293,"name":"offline","context":{"idset":"9923"}}
+{"timestamp":1714021380.9063995,"name":"offline","context":{"idset":"9924"}}
+{"timestamp":1714021380.9082549,"name":"offline","context":{"idset":"9925"}}
+{"timestamp":1714021380.9101243,"name":"offline","context":{"idset":"9926"}}
+{"timestamp":1714021380.9120028,"name":"offline","context":{"idset":"9927"}}
+{"timestamp":1714021380.9233277,"name":"offline","context":{"idset":"9928"}}
+{"timestamp":1714021380.9251914,"name":"offline","context":{"idset":"9929"}}
+{"timestamp":1714021380.9270504,"name":"offline","context":{"idset":"9930"}}
+{"timestamp":1714021380.9289172,"name":"offline","context":{"idset":"9931"}}
+{"timestamp":1714021380.9307764,"name":"offline","context":{"idset":"9932"}}
+{"timestamp":1714021380.9330385,"name":"offline","context":{"idset":"9933"}}
+{"timestamp":1714021380.9353218,"name":"offline","context":{"idset":"9934"}}
+{"timestamp":1714021380.9380834,"name":"offline","context":{"idset":"9935"}}
+{"timestamp":1714021380.9405417,"name":"offline","context":{"idset":"9936"}}
+{"timestamp":1714021380.9424093,"name":"offline","context":{"idset":"9937"}}
+{"timestamp":1714021380.9442875,"name":"offline","context":{"idset":"9938"}}
+{"timestamp":1714021380.9461472,"name":"offline","context":{"idset":"9939"}}
+{"timestamp":1714021380.9479887,"name":"offline","context":{"idset":"9940"}}
+{"timestamp":1714021380.9498355,"name":"offline","context":{"idset":"9941"}}
+{"timestamp":1714021380.9516649,"name":"offline","context":{"idset":"9942"}}
+{"timestamp":1714021380.9535036,"name":"offline","context":{"idset":"9943"}}
+{"timestamp":1714021380.9553452,"name":"offline","context":{"idset":"9944"}}
+{"timestamp":1714021380.9571495,"name":"offline","context":{"idset":"9945"}}
+{"timestamp":1714021380.9586818,"name":"offline","context":{"idset":"9946"}}
+{"timestamp":1714021380.9601524,"name":"offline","context":{"idset":"9947"}}
+{"timestamp":1714021380.9619286,"name":"offline","context":{"idset":"9948"}}
+{"timestamp":1714021380.963414,"name":"offline","context":{"idset":"9949"}}
+{"timestamp":1714021380.9648817,"name":"offline","context":{"idset":"9950"}}
+{"timestamp":1714021380.9664738,"name":"offline","context":{"idset":"9951"}}
+{"timestamp":1714021380.969085,"name":"offline","context":{"idset":"9952"}}
+{"timestamp":1714021380.9719915,"name":"offline","context":{"idset":"9953"}}
+{"timestamp":1714021380.9739568,"name":"offline","context":{"idset":"9954"}}
+{"timestamp":1714021380.9754248,"name":"offline","context":{"idset":"9955"}}
+{"timestamp":1714021380.9769027,"name":"offline","context":{"idset":"9956"}}
+{"timestamp":1714021380.9783659,"name":"offline","context":{"idset":"9957"}}
+{"timestamp":1714021380.9798362,"name":"offline","context":{"idset":"9958"}}
+{"timestamp":1714021380.9812946,"name":"offline","context":{"idset":"9959"}}
+{"timestamp":1714021380.9827745,"name":"offline","context":{"idset":"9960"}}
+{"timestamp":1714021380.9842381,"name":"offline","context":{"idset":"9961"}}
+{"timestamp":1714021380.9856923,"name":"offline","context":{"idset":"9962"}}
+{"timestamp":1714021380.9871395,"name":"offline","context":{"idset":"9963"}}
+{"timestamp":1714021380.9885833,"name":"offline","context":{"idset":"9964"}}
+{"timestamp":1714021380.9976892,"name":"offline","context":{"idset":"9965"}}
+{"timestamp":1714021380.9991724,"name":"offline","context":{"idset":"9966"}}
+{"timestamp":1714021381.0006342,"name":"offline","context":{"idset":"9967"}}
+{"timestamp":1714021381.0021214,"name":"offline","context":{"idset":"9968"}}
+{"timestamp":1714021381.0037677,"name":"offline","context":{"idset":"9969"}}
+{"timestamp":1714021381.0052636,"name":"offline","context":{"idset":"9970"}}
+{"timestamp":1714021381.0068655,"name":"offline","context":{"idset":"9971"}}
+{"timestamp":1714021381.008595,"name":"offline","context":{"idset":"9972"}}
+{"timestamp":1714021381.0113707,"name":"offline","context":{"idset":"9973"}}
+{"timestamp":1714021381.0215228,"name":"offline","context":{"idset":"9974"}}
+{"timestamp":1714021381.0230405,"name":"offline","context":{"idset":"9975"}}
+{"timestamp":1714021381.0245075,"name":"offline","context":{"idset":"9976"}}
+{"timestamp":1714021381.0259633,"name":"offline","context":{"idset":"9977"}}
+{"timestamp":1714021381.0276871,"name":"offline","context":{"idset":"9978"}}
+{"timestamp":1714021381.0292327,"name":"offline","context":{"idset":"9979"}}
+{"timestamp":1714021381.0306919,"name":"offline","context":{"idset":"9980"}}
+{"timestamp":1714021381.0323112,"name":"offline","context":{"idset":"9981"}}
+{"timestamp":1714021381.033993,"name":"offline","context":{"idset":"9982"}}
+{"timestamp":1714021381.0356765,"name":"offline","context":{"idset":"9983"}}
+{"timestamp":1714021381.0450511,"name":"offline","context":{"idset":"9984"}}
+{"timestamp":1714021381.0465045,"name":"offline","context":{"idset":"9985"}}
+{"timestamp":1714021381.0482128,"name":"offline","context":{"idset":"9986"}}
+{"timestamp":1714021381.0496714,"name":"offline","context":{"idset":"9987"}}
+{"timestamp":1714021381.0511243,"name":"offline","context":{"idset":"9988"}}
+{"timestamp":1714021381.0526426,"name":"offline","context":{"idset":"9989"}}
+{"timestamp":1714021381.054347,"name":"offline","context":{"idset":"9990"}}
+{"timestamp":1714021381.0558875,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1714021381.0573361,"name":"offline","context":{"idset":"9992"}}
+{"timestamp":1714021381.0587687,"name":"offline","context":{"idset":"9993"}}
+{"timestamp":1714021381.0684581,"name":"offline","context":{"idset":"9994"}}
+{"timestamp":1714021381.0699739,"name":"offline","context":{"idset":"9995"}}
+{"timestamp":1714021381.0714345,"name":"offline","context":{"idset":"9996"}}
+{"timestamp":1714021381.0728822,"name":"offline","context":{"idset":"9997"}}
+{"timestamp":1714021381.0745955,"name":"offline","context":{"idset":"9998"}}
+{"timestamp":1714021381.076066,"name":"offline","context":{"idset":"9999"}}
+{"timestamp":1714021381.077498,"name":"offline","context":{"idset":"10000"}}
+{"timestamp":1714021381.0789311,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1714021381.0804002,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1714021381.0818501,"name":"offline","context":{"idset":"10003"}}
+{"timestamp":1714021381.0832827,"name":"offline","context":{"idset":"10004"}}
+{"timestamp":1714021381.0847118,"name":"offline","context":{"idset":"10005"}}
+{"timestamp":1714021381.0861886,"name":"offline","context":{"idset":"10006"}}
+{"timestamp":1714021381.0876265,"name":"offline","context":{"idset":"10007"}}
+{"timestamp":1714021381.0891371,"name":"offline","context":{"idset":"10008"}}
+{"timestamp":1714021381.0905814,"name":"offline","context":{"idset":"10009"}}
+{"timestamp":1714021381.0920224,"name":"offline","context":{"idset":"10010"}}
+{"timestamp":1714021381.0934494,"name":"offline","context":{"idset":"10011"}}
+{"timestamp":1714021381.094897,"name":"offline","context":{"idset":"10012"}}
+{"timestamp":1714021381.0973065,"name":"offline","context":{"idset":"10013"}}
+{"timestamp":1714021381.1002073,"name":"offline","context":{"idset":"10014"}}
+{"timestamp":1714021381.1031189,"name":"offline","context":{"idset":"10015"}}
+{"timestamp":1714021381.1060352,"name":"offline","context":{"idset":"10016"}}
+{"timestamp":1714021381.1089635,"name":"offline","context":{"idset":"10017"}}
+{"timestamp":1714021381.110935,"name":"offline","context":{"idset":"10018"}}
+{"timestamp":1714021381.1123741,"name":"offline","context":{"idset":"10019"}}
+{"timestamp":1714021381.1138413,"name":"offline","context":{"idset":"10020"}}
+{"timestamp":1714021381.1157877,"name":"offline","context":{"idset":"10021"}}
+{"timestamp":1714021381.1181278,"name":"offline","context":{"idset":"10022"}}
+{"timestamp":1714021381.1196489,"name":"offline","context":{"idset":"10023"}}
+{"timestamp":1714021381.1225023,"name":"offline","context":{"idset":"10024"}}
+{"timestamp":1714021381.1253211,"name":"offline","context":{"idset":"10025"}}
+{"timestamp":1714021381.1280649,"name":"offline","context":{"idset":"10026"}}
+{"timestamp":1714021381.1306856,"name":"offline","context":{"idset":"10027"}}
+{"timestamp":1714021381.1335256,"name":"offline","context":{"idset":"10028"}}
+{"timestamp":1714021381.1363709,"name":"offline","context":{"idset":"10029"}}
+{"timestamp":1714021381.1391852,"name":"offline","context":{"idset":"10030"}}
+{"timestamp":1714021381.1421249,"name":"offline","context":{"idset":"10031"}}
+{"timestamp":1714021381.1603031,"name":"offline","context":{"idset":"10032"}}
+{"timestamp":1714021381.1632302,"name":"offline","context":{"idset":"10033"}}
+{"timestamp":1714021381.1661077,"name":"offline","context":{"idset":"10034"}}
+{"timestamp":1714021381.168988,"name":"offline","context":{"idset":"10035"}}
+{"timestamp":1714021381.1720047,"name":"offline","context":{"idset":"10036"}}
+{"timestamp":1714021381.1748784,"name":"offline","context":{"idset":"10037"}}
+{"timestamp":1714021381.1777291,"name":"offline","context":{"idset":"10038"}}
+{"timestamp":1714021381.1806262,"name":"offline","context":{"idset":"10039"}}
+{"timestamp":1714021381.1834869,"name":"offline","context":{"idset":"10040"}}
+{"timestamp":1714021381.1863194,"name":"offline","context":{"idset":"10041"}}
+{"timestamp":1714021381.1891668,"name":"offline","context":{"idset":"10042"}}
+{"timestamp":1714021381.2060864,"name":"offline","context":{"idset":"10043"}}
+{"timestamp":1714021381.2089012,"name":"offline","context":{"idset":"10044"}}
+{"timestamp":1714021381.2117002,"name":"offline","context":{"idset":"10045"}}
+{"timestamp":1714021381.2143791,"name":"offline","context":{"idset":"10046"}}
+{"timestamp":1714021381.215838,"name":"offline","context":{"idset":"10047"}}
+{"timestamp":1714021381.2179234,"name":"offline","context":{"idset":"10048"}}
+{"timestamp":1714021381.2193341,"name":"offline","context":{"idset":"10049"}}
+{"timestamp":1714021381.2207325,"name":"offline","context":{"idset":"10050"}}
+{"timestamp":1714021381.2221289,"name":"offline","context":{"idset":"10051"}}
+{"timestamp":1714021381.2324963,"name":"offline","context":{"idset":"10052"}}
+{"timestamp":1714021381.2339103,"name":"offline","context":{"idset":"10053"}}
+{"timestamp":1714021381.2353055,"name":"offline","context":{"idset":"10054"}}
+{"timestamp":1714021381.2366958,"name":"offline","context":{"idset":"10055"}}
+{"timestamp":1714021381.2381406,"name":"offline","context":{"idset":"10056"}}
+{"timestamp":1714021381.2396181,"name":"offline","context":{"idset":"10057"}}
+{"timestamp":1714021381.2410283,"name":"offline","context":{"idset":"10058"}}
+{"timestamp":1714021381.2424235,"name":"offline","context":{"idset":"10059"}}
+{"timestamp":1714021381.2438426,"name":"offline","context":{"idset":"10060"}}
+{"timestamp":1714021381.2452316,"name":"offline","context":{"idset":"10061"}}
+{"timestamp":1714021381.2541814,"name":"offline","context":{"idset":"10062"}}
+{"timestamp":1714021381.2555645,"name":"offline","context":{"idset":"10063"}}
+{"timestamp":1714021381.2569461,"name":"offline","context":{"idset":"10064"}}
+{"timestamp":1714021381.258353,"name":"offline","context":{"idset":"10065"}}
+{"timestamp":1714021381.2597423,"name":"offline","context":{"idset":"10066"}}
+{"timestamp":1714021381.2611299,"name":"offline","context":{"idset":"10067"}}
+{"timestamp":1714021381.2625329,"name":"offline","context":{"idset":"10068"}}
+{"timestamp":1714021381.264209,"name":"offline","context":{"idset":"10069"}}
+{"timestamp":1714021381.2658944,"name":"offline","context":{"idset":"10070"}}
+{"timestamp":1714021381.2675495,"name":"offline","context":{"idset":"10071"}}
+{"timestamp":1714021381.2690783,"name":"offline","context":{"idset":"10072"}}
+{"timestamp":1714021381.2780631,"name":"offline","context":{"idset":"10073"}}
+{"timestamp":1714021381.2794459,"name":"offline","context":{"idset":"10074"}}
+{"timestamp":1714021381.2808299,"name":"offline","context":{"idset":"10075"}}
+{"timestamp":1714021381.2822046,"name":"offline","context":{"idset":"10076"}}
+{"timestamp":1714021381.2835774,"name":"offline","context":{"idset":"10077"}}
+{"timestamp":1714021381.2849591,"name":"offline","context":{"idset":"10078"}}
+{"timestamp":1714021381.286334,"name":"offline","context":{"idset":"10079"}}
+{"timestamp":1714021381.2876983,"name":"offline","context":{"idset":"10080"}}
+{"timestamp":1714021381.2892618,"name":"offline","context":{"idset":"10081"}}
+{"timestamp":1714021381.2906721,"name":"offline","context":{"idset":"10082"}}
+{"timestamp":1714021381.2920575,"name":"offline","context":{"idset":"10083"}}
+{"timestamp":1714021381.2934279,"name":"offline","context":{"idset":"10084"}}
+{"timestamp":1714021381.2954547,"name":"offline","context":{"idset":"10085"}}
+{"timestamp":1714021381.2981191,"name":"offline","context":{"idset":"10086"}}
+{"timestamp":1714021381.3007674,"name":"offline","context":{"idset":"10087"}}
+{"timestamp":1714021381.3034298,"name":"offline","context":{"idset":"10088"}}
+{"timestamp":1714021381.305944,"name":"offline","context":{"idset":"10089"}}
+{"timestamp":1714021381.3086209,"name":"offline","context":{"idset":"10090"}}
+{"timestamp":1714021381.3112731,"name":"offline","context":{"idset":"10091"}}
+{"timestamp":1714021381.3129122,"name":"offline","context":{"idset":"10092"}}
+{"timestamp":1714021381.3142703,"name":"offline","context":{"idset":"10093"}}
+{"timestamp":1714021381.3156304,"name":"offline","context":{"idset":"10094"}}
+{"timestamp":1714021381.3169844,"name":"offline","context":{"idset":"10095"}}
+{"timestamp":1714021381.318337,"name":"offline","context":{"idset":"10096"}}
+{"timestamp":1714021381.319699,"name":"offline","context":{"idset":"10097"}}
+{"timestamp":1714021381.3210547,"name":"offline","context":{"idset":"10098"}}
+{"timestamp":1714021381.3224077,"name":"offline","context":{"idset":"10099"}}
+{"timestamp":1714021381.3237617,"name":"offline","context":{"idset":"10100"}}
+{"timestamp":1714021381.3251119,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1714021381.3264587,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1714021381.3278189,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1714021381.3291645,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1714021381.3305149,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1714021381.3318703,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1714021381.333209,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1714021381.3345511,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1714021381.3358965,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1714021381.3372352,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1714021381.3461549,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1714021381.3475332,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1714021381.348882,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1714021381.3502288,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1714021381.3515706,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1714021381.3529224,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1714021381.3542712,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1714021381.3556166,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1714021381.3569548,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1714021381.358299,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1714021381.3596344,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1714021381.3685086,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1714021381.3698483,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1714021381.3711853,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1714021381.3725245,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1714021381.3738706,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1714021381.3752081,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1714021381.3765438,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1714021381.3778841,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1714021381.3792658,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1714021381.3882077,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1714021381.3895583,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1714021381.390909,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1714021381.392272,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1714021381.3936465,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1714021381.3950067,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1714021381.3963296,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1714021381.3976471,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1714021381.3989677,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1714021381.4002802,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1714021381.4015903,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1714021381.4029019,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1714021381.4116888,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1714021381.4130149,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1714021381.4143391,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1714021381.4156733,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1714021381.4170036,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1714021381.4183235,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1714021381.4196396,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1714021381.4209621,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1714021381.422276,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1714021381.4235914,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1714021381.4324045,"name":"offline","context":{"idset":"10153"}}
+{"timestamp":1714021381.4337242,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1714021381.4350455,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1714021381.4363596,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1714021381.4376645,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1714021381.4389741,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1714021381.440316,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1714021381.4416349,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1714021381.4429414,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1714021381.4442527,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1714021381.4531536,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1714021381.4544628,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1714021381.4557719,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1714021381.4571223,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1714021381.458446,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1714021381.4597714,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1714021381.4611158,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1714021381.4624228,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1714021381.4637268,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1714021381.4650745,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1714021381.4663754,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1714021381.4676678,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1714021381.476511,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1714021381.4778376,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1714021381.4791319,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1714021381.4804332,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1714021381.4817455,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1714021381.4830954,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1714021381.4844158,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1714021381.485728,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1714021381.4870203,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1714021381.4883132,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1714021381.4895976,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1714021381.4984128,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1714021381.4997082,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1714021381.5010192,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1714021381.5023224,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1714021381.5036314,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1714021381.5049279,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1714021381.5062156,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1714021381.507493,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1714021381.5087821,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1714021381.5100806,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1714021381.5188978,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1714021381.5201826,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1714021381.5214758,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1714021381.5227623,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1714021381.524049,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1714021381.5253415,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1714021381.5266197,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1714021381.5278964,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1714021381.529181,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1714021381.5306776,"name":"offline","context":{"idset":"10206"}}
+{"timestamp":1714021381.5319626,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1714021381.5409932,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1714021381.5422785,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1714021381.5435545,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1714021381.5448411,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1714021381.5461173,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1714021381.5473819,"name":"offline","context":{"idset":"10213"}}
+{"timestamp":1714021381.5486624,"name":"offline","context":{"idset":"10214"}}
+{"timestamp":1714021381.5499535,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1714021381.5512214,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1714021381.5524859,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1714021381.553751,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1714021381.5625865,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1714021381.5640061,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1714021381.5658848,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1714021381.5677526,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1714021381.5696323,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1714021381.5714662,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1714021381.572741,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1714021381.5740173,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1714021381.5752978,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1714021381.576555,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1714021381.5778277,"name":"offline","context":{"idset":"10229"}}
+{"timestamp":1714021381.5867167,"name":"offline","context":{"idset":"10230"}}
+{"timestamp":1714021381.5879838,"name":"offline","context":{"idset":"10231"}}
+{"timestamp":1714021381.5892448,"name":"offline","context":{"idset":"10232"}}
+{"timestamp":1714021381.5904973,"name":"offline","context":{"idset":"10233"}}
+{"timestamp":1714021381.5917635,"name":"offline","context":{"idset":"10234"}}
+{"timestamp":1714021381.5930259,"name":"offline","context":{"idset":"10235"}}
+{"timestamp":1714021381.594281,"name":"offline","context":{"idset":"10236"}}
+{"timestamp":1714021381.5955555,"name":"offline","context":{"idset":"10237"}}
+{"timestamp":1714021381.5968308,"name":"offline","context":{"idset":"10238"}}
+{"timestamp":1714021381.598098,"name":"offline","context":{"idset":"10239"}}
+{"timestamp":1714021381.5993457,"name":"offline","context":{"idset":"10241"}}
+{"timestamp":1714021381.6005783,"name":"offline","context":{"idset":"10242"}}
+{"timestamp":1714021381.6092877,"name":"offline","context":{"idset":"10243"}}
+{"timestamp":1714021381.6105325,"name":"offline","context":{"idset":"10245"}}
+{"timestamp":1714021381.6117876,"name":"offline","context":{"idset":"10246"}}
+{"timestamp":1714021381.6130519,"name":"offline","context":{"idset":"10247"}}
+{"timestamp":1714021381.6143024,"name":"offline","context":{"idset":"10248"}}
+{"timestamp":1714021381.615761,"name":"offline","context":{"idset":"10250"}}
+{"timestamp":1714021381.6171479,"name":"offline","context":{"idset":"10251"}}
+{"timestamp":1714021381.6183872,"name":"offline","context":{"idset":"10252"}}
+{"timestamp":1714021381.6196194,"name":"offline","context":{"idset":"10253"}}
+{"timestamp":1714021381.6208606,"name":"offline","context":{"idset":"10254"}}
+{"timestamp":1714021381.6221218,"name":"offline","context":{"idset":"10255"}}
+{"timestamp":1714021381.630837,"name":"offline","context":{"idset":"10256"}}
+{"timestamp":1714021381.6320693,"name":"offline","context":{"idset":"10257"}}
+{"timestamp":1714021381.6332996,"name":"offline","context":{"idset":"10258"}}
+{"timestamp":1714021381.6345284,"name":"offline","context":{"idset":"10259"}}
+{"timestamp":1714021381.6357462,"name":"offline","context":{"idset":"10263"}}
+{"timestamp":1714021381.6369789,"name":"offline","context":{"idset":"10264"}}
+{"timestamp":1714021381.6382024,"name":"offline","context":{"idset":"10265"}}
+{"timestamp":1714021381.6394372,"name":"offline","context":{"idset":"10266"}}
+{"timestamp":1714021381.6406624,"name":"offline","context":{"idset":"10267"}}
+{"timestamp":1714021381.6418967,"name":"offline","context":{"idset":"10268"}}
+{"timestamp":1714021381.643122,"name":"offline","context":{"idset":"10269"}}
+{"timestamp":1714021381.6443591,"name":"offline","context":{"idset":"10270"}}
+{"timestamp":1714021381.6455784,"name":"offline","context":{"idset":"10271"}}
+{"timestamp":1714021381.6467991,"name":"offline","context":{"idset":"10272"}}
+{"timestamp":1714021381.6480234,"name":"offline","context":{"idset":"10273"}}
+{"timestamp":1714021381.649236,"name":"offline","context":{"idset":"10274"}}
+{"timestamp":1714021381.6504505,"name":"offline","context":{"idset":"10275"}}
+{"timestamp":1714021381.6516664,"name":"offline","context":{"idset":"10277"}}
+{"timestamp":1714021381.6528766,"name":"offline","context":{"idset":"10278"}}
+{"timestamp":1714021381.6540914,"name":"offline","context":{"idset":"10279"}}
+{"timestamp":1714021381.6553128,"name":"offline","context":{"idset":"10280"}}
+{"timestamp":1714021381.6565261,"name":"offline","context":{"idset":"10281"}}
+{"timestamp":1714021381.657742,"name":"offline","context":{"idset":"10282"}}
+{"timestamp":1714021381.658967,"name":"offline","context":{"idset":"10283"}}
+{"timestamp":1714021381.6601787,"name":"offline","context":{"idset":"10285"}}
+{"timestamp":1714021381.6613889,"name":"offline","context":{"idset":"10286"}}
+{"timestamp":1714021381.6625962,"name":"offline","context":{"idset":"10287"}}
+{"timestamp":1714021381.6638215,"name":"offline","context":{"idset":"10289"}}
+{"timestamp":1714021381.6650314,"name":"offline","context":{"idset":"10290"}}
+{"timestamp":1714021381.6662352,"name":"offline","context":{"idset":"10291"}}
+{"timestamp":1714021381.6674335,"name":"offline","context":{"idset":"10293"}}
+{"timestamp":1714021381.6686358,"name":"offline","context":{"idset":"10294"}}
+{"timestamp":1714021381.6698563,"name":"offline","context":{"idset":"10295"}}
+{"timestamp":1714021381.6710598,"name":"offline","context":{"idset":"10296"}}
+{"timestamp":1714021381.6722841,"name":"offline","context":{"idset":"10297"}}
+{"timestamp":1714021381.6734962,"name":"offline","context":{"idset":"10298"}}
+{"timestamp":1714021381.6746974,"name":"offline","context":{"idset":"10299"}}
+{"timestamp":1714021381.6759009,"name":"offline","context":{"idset":"10300"}}
+{"timestamp":1714021381.6770997,"name":"offline","context":{"idset":"10301"}}
+{"timestamp":1714021381.6783001,"name":"offline","context":{"idset":"10302"}}
+{"timestamp":1714021381.6794968,"name":"offline","context":{"idset":"10303"}}
+{"timestamp":1714021381.6806934,"name":"offline","context":{"idset":"10304"}}
+{"timestamp":1714021381.6819034,"name":"offline","context":{"idset":"10305"}}
+{"timestamp":1714021381.6831005,"name":"offline","context":{"idset":"10306"}}
+{"timestamp":1714021381.6842952,"name":"offline","context":{"idset":"10309"}}
+{"timestamp":1714021381.6854868,"name":"offline","context":{"idset":"10310"}}
+{"timestamp":1714021381.6941979,"name":"offline","context":{"idset":"10311"}}
+{"timestamp":1714021381.6954076,"name":"offline","context":{"idset":"10312"}}
+{"timestamp":1714021381.696609,"name":"offline","context":{"idset":"10313"}}
+{"timestamp":1714021381.6978114,"name":"offline","context":{"idset":"10314"}}
+{"timestamp":1714021381.6990097,"name":"offline","context":{"idset":"10315"}}
+{"timestamp":1714021381.700206,"name":"offline","context":{"idset":"10317"}}
+{"timestamp":1714021381.7014041,"name":"offline","context":{"idset":"10318"}}
+{"timestamp":1714021381.7025964,"name":"offline","context":{"idset":"10319"}}
+{"timestamp":1714021381.7037876,"name":"offline","context":{"idset":"10321"}}
+{"timestamp":1714021381.7049949,"name":"offline","context":{"idset":"10322"}}
+{"timestamp":1714021381.7061889,"name":"offline","context":{"idset":"10325"}}
+{"timestamp":1714021381.7149546,"name":"offline","context":{"idset":"10326"}}
+{"timestamp":1714021381.7161543,"name":"offline","context":{"idset":"10327"}}
+{"timestamp":1714021381.7173479,"name":"offline","context":{"idset":"10328"}}
+{"timestamp":1714021381.7185376,"name":"offline","context":{"idset":"10329"}}
+{"timestamp":1714021381.7197232,"name":"offline","context":{"idset":"10330"}}
+{"timestamp":1714021381.7209103,"name":"offline","context":{"idset":"10331"}}
+{"timestamp":1714021381.722095,"name":"offline","context":{"idset":"10333"}}
+{"timestamp":1714021381.7232842,"name":"offline","context":{"idset":"10334"}}
+{"timestamp":1714021381.7244647,"name":"offline","context":{"idset":"10335"}}
+{"timestamp":1714021381.7256484,"name":"offline","context":{"idset":"10337"}}
+{"timestamp":1714021381.7268369,"name":"offline","context":{"idset":"10338"}}
+{"timestamp":1714021381.735498,"name":"offline","context":{"idset":"10339"}}
+{"timestamp":1714021381.7366838,"name":"offline","context":{"idset":"10341"}}
+{"timestamp":1714021381.7378569,"name":"offline","context":{"idset":"10342"}}
+{"timestamp":1714021381.739064,"name":"offline","context":{"idset":"10343"}}
+{"timestamp":1714021381.7402437,"name":"offline","context":{"idset":"10345"}}
+{"timestamp":1714021381.7414186,"name":"offline","context":{"idset":"10346"}}
+{"timestamp":1714021381.7425935,"name":"offline","context":{"idset":"10347"}}
+{"timestamp":1714021381.7437727,"name":"offline","context":{"idset":"10349"}}
+{"timestamp":1714021381.744983,"name":"offline","context":{"idset":"10350"}}
+{"timestamp":1714021381.7461858,"name":"offline","context":{"idset":"10351"}}
+{"timestamp":1714021381.7473652,"name":"offline","context":{"idset":"10353"}}
+{"timestamp":1714021381.7485297,"name":"offline","context":{"idset":"10354"}}
+{"timestamp":1714021381.7496953,"name":"offline","context":{"idset":"10355"}}
+{"timestamp":1714021381.7508602,"name":"offline","context":{"idset":"10357"}}
+{"timestamp":1714021381.7594843,"name":"offline","context":{"idset":"10358"}}
+{"timestamp":1714021381.7606351,"name":"offline","context":{"idset":"10359"}}
+{"timestamp":1714021381.7617912,"name":"offline","context":{"idset":"10360"}}
+{"timestamp":1714021381.7629521,"name":"offline","context":{"idset":"10361"}}
+{"timestamp":1714021381.7641087,"name":"offline","context":{"idset":"10362"}}
+{"timestamp":1714021381.7652569,"name":"offline","context":{"idset":"10363"}}
+{"timestamp":1714021381.7664108,"name":"offline","context":{"idset":"10364"}}
+{"timestamp":1714021381.7675533,"name":"offline","context":{"idset":"10365"}}
+{"timestamp":1714021381.7687013,"name":"offline","context":{"idset":"10366"}}
+{"timestamp":1714021381.7698476,"name":"offline","context":{"idset":"10367"}}
+{"timestamp":1714021381.770997,"name":"offline","context":{"idset":"10368"}}
+{"timestamp":1714021381.7721441,"name":"offline","context":{"idset":"10369"}}
+{"timestamp":1714021381.773294,"name":"offline","context":{"idset":"10370"}}
+{"timestamp":1714021381.7744362,"name":"offline","context":{"idset":"10371"}}
+{"timestamp":1714021381.7755823,"name":"offline","context":{"idset":"10372"}}
+{"timestamp":1714021381.7767172,"name":"offline","context":{"idset":"10373"}}
+{"timestamp":1714021381.7778542,"name":"offline","context":{"idset":"10374"}}
+{"timestamp":1714021381.7790122,"name":"offline","context":{"idset":"10375"}}
+{"timestamp":1714021381.7801549,"name":"offline","context":{"idset":"10376"}}
+{"timestamp":1714021381.781296,"name":"offline","context":{"idset":"10377"}}
+{"timestamp":1714021381.7824342,"name":"offline","context":{"idset":"10378"}}
+{"timestamp":1714021381.783577,"name":"offline","context":{"idset":"10379"}}
+{"timestamp":1714021381.7847195,"name":"offline","context":{"idset":"10380"}}
+{"timestamp":1714021381.7858543,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1714021381.7869885,"name":"offline","context":{"idset":"10382"}}
+{"timestamp":1714021381.7881243,"name":"offline","context":{"idset":"10383"}}
+{"timestamp":1714021381.7892568,"name":"offline","context":{"idset":"10384"}}
+{"timestamp":1714021381.790391,"name":"offline","context":{"idset":"10385"}}
+{"timestamp":1714021381.7915251,"name":"offline","context":{"idset":"10386"}}
+{"timestamp":1714021381.7926581,"name":"offline","context":{"idset":"10387"}}
+{"timestamp":1714021381.7937896,"name":"offline","context":{"idset":"10388"}}
+{"timestamp":1714021381.7949283,"name":"offline","context":{"idset":"10389"}}
+{"timestamp":1714021381.7960579,"name":"offline","context":{"idset":"10390"}}
+{"timestamp":1714021381.7971871,"name":"offline","context":{"idset":"10391"}}
+{"timestamp":1714021381.798321,"name":"offline","context":{"idset":"10392"}}
+{"timestamp":1714021381.799444,"name":"offline","context":{"idset":"10393"}}
+{"timestamp":1714021381.8005641,"name":"offline","context":{"idset":"10394"}}
+{"timestamp":1714021381.8016825,"name":"offline","context":{"idset":"10395"}}
+{"timestamp":1714021381.8028142,"name":"offline","context":{"idset":"10396"}}
+{"timestamp":1714021381.8039465,"name":"offline","context":{"idset":"10397"}}
+{"timestamp":1714021381.8050754,"name":"offline","context":{"idset":"10398"}}
+{"timestamp":1714021381.8061974,"name":"offline","context":{"idset":"10399"}}
+{"timestamp":1714021381.8073189,"name":"offline","context":{"idset":"10400"}}
+{"timestamp":1714021381.8084397,"name":"offline","context":{"idset":"10401"}}
+{"timestamp":1714021381.8095665,"name":"offline","context":{"idset":"10402"}}
+{"timestamp":1714021381.8107533,"name":"offline","context":{"idset":"10403"}}
+{"timestamp":1714021381.8119771,"name":"offline","context":{"idset":"10404"}}
+{"timestamp":1714021381.8131192,"name":"offline","context":{"idset":"10405"}}
+{"timestamp":1714021381.8142302,"name":"offline","context":{"idset":"10406"}}
+{"timestamp":1714021381.8153408,"name":"offline","context":{"idset":"10407"}}
+{"timestamp":1714021381.8164504,"name":"offline","context":{"idset":"10408"}}
+{"timestamp":1714021381.8175561,"name":"offline","context":{"idset":"10409"}}
+{"timestamp":1714021381.8186624,"name":"offline","context":{"idset":"10410"}}
+{"timestamp":1714021381.8197682,"name":"offline","context":{"idset":"10411"}}
+{"timestamp":1714021381.8208787,"name":"offline","context":{"idset":"10412"}}
+{"timestamp":1714021381.8219857,"name":"offline","context":{"idset":"10413"}}
+{"timestamp":1714021381.8230922,"name":"offline","context":{"idset":"10414"}}
+{"timestamp":1714021381.8241935,"name":"offline","context":{"idset":"10415"}}
+{"timestamp":1714021381.8252902,"name":"offline","context":{"idset":"10416"}}
+{"timestamp":1714021381.8263946,"name":"offline","context":{"idset":"10417"}}
+{"timestamp":1714021381.827491,"name":"offline","context":{"idset":"10418"}}
+{"timestamp":1714021381.8285894,"name":"offline","context":{"idset":"10419"}}
+{"timestamp":1714021381.8296814,"name":"offline","context":{"idset":"10420"}}
+{"timestamp":1714021381.8307848,"name":"offline","context":{"idset":"10421"}}
+{"timestamp":1714021381.8318932,"name":"offline","context":{"idset":"10422"}}
+{"timestamp":1714021381.8329895,"name":"offline","context":{"idset":"10423"}}
+{"timestamp":1714021381.834106,"name":"offline","context":{"idset":"10424"}}
+{"timestamp":1714021381.835207,"name":"offline","context":{"idset":"10425"}}
+{"timestamp":1714021381.8363004,"name":"offline","context":{"idset":"10426"}}
+{"timestamp":1714021381.8373923,"name":"offline","context":{"idset":"10427"}}
+{"timestamp":1714021381.838484,"name":"offline","context":{"idset":"10428"}}
+{"timestamp":1714021381.8395693,"name":"offline","context":{"idset":"10429"}}
+{"timestamp":1714021381.8406656,"name":"offline","context":{"idset":"10430"}}
+{"timestamp":1714021381.8417616,"name":"offline","context":{"idset":"10431"}}
+{"timestamp":1714021381.8428597,"name":"offline","context":{"idset":"10432"}}
+{"timestamp":1714021381.8439486,"name":"offline","context":{"idset":"10433"}}
+{"timestamp":1714021381.8450356,"name":"offline","context":{"idset":"10434"}}
+{"timestamp":1714021381.8461239,"name":"offline","context":{"idset":"10435"}}
+{"timestamp":1714021381.8472071,"name":"offline","context":{"idset":"10436"}}
+{"timestamp":1714021381.8482928,"name":"offline","context":{"idset":"10437"}}
+{"timestamp":1714021381.8493769,"name":"offline","context":{"idset":"10438"}}
+{"timestamp":1714021381.8504717,"name":"offline","context":{"idset":"10439"}}
+{"timestamp":1714021381.8515525,"name":"offline","context":{"idset":"10440"}}
+{"timestamp":1714021381.8526371,"name":"offline","context":{"idset":"10441"}}
+{"timestamp":1714021381.8537257,"name":"offline","context":{"idset":"10442"}}
+{"timestamp":1714021381.8547966,"name":"offline","context":{"idset":"10443"}}
+{"timestamp":1714021381.8558955,"name":"offline","context":{"idset":"10444"}}
+{"timestamp":1714021381.8569794,"name":"offline","context":{"idset":"10445"}}
+{"timestamp":1714021381.8580611,"name":"offline","context":{"idset":"10446"}}
+{"timestamp":1714021381.8591416,"name":"offline","context":{"idset":"10447"}}
+{"timestamp":1714021381.8676555,"name":"offline","context":{"idset":"10448"}}
+{"timestamp":1714021381.8687327,"name":"offline","context":{"idset":"10449"}}
+{"timestamp":1714021381.8698268,"name":"offline","context":{"idset":"10450"}}
+{"timestamp":1714021381.8709509,"name":"offline","context":{"idset":"10451"}}
+{"timestamp":1714021381.8720386,"name":"offline","context":{"idset":"10452"}}
+{"timestamp":1714021381.8731236,"name":"offline","context":{"idset":"10453"}}
+{"timestamp":1714021381.8742046,"name":"offline","context":{"idset":"10454"}}
+{"timestamp":1714021381.8752842,"name":"offline","context":{"idset":"10455"}}
+{"timestamp":1714021381.8763671,"name":"offline","context":{"idset":"10456"}}
+{"timestamp":1714021381.8774517,"name":"offline","context":{"idset":"10457"}}
+{"timestamp":1714021381.878536,"name":"offline","context":{"idset":"10458"}}
+{"timestamp":1714021381.8796129,"name":"offline","context":{"idset":"10459"}}
+{"timestamp":1714021381.8806901,"name":"offline","context":{"idset":"10460"}}
+{"timestamp":1714021381.8892212,"name":"offline","context":{"idset":"10461"}}
+{"timestamp":1714021381.8903012,"name":"offline","context":{"idset":"10462"}}
+{"timestamp":1714021381.8913741,"name":"offline","context":{"idset":"10463"}}
+{"timestamp":1714021381.8924465,"name":"offline","context":{"idset":"10464"}}
+{"timestamp":1714021381.8935189,"name":"offline","context":{"idset":"10465"}}
+{"timestamp":1714021381.8945961,"name":"offline","context":{"idset":"10466"}}
+{"timestamp":1714021381.8956635,"name":"offline","context":{"idset":"10467"}}
+{"timestamp":1714021381.8967357,"name":"offline","context":{"idset":"10468"}}
+{"timestamp":1714021381.8977933,"name":"offline","context":{"idset":"10469"}}
+{"timestamp":1714021381.8988726,"name":"offline","context":{"idset":"10470"}}
+{"timestamp":1714021381.8999393,"name":"offline","context":{"idset":"10471"}}
+{"timestamp":1714021381.9010055,"name":"offline","context":{"idset":"10472"}}
+{"timestamp":1714021381.9020717,"name":"offline","context":{"idset":"10473"}}
+{"timestamp":1714021381.9031394,"name":"offline","context":{"idset":"10474"}}
+{"timestamp":1714021381.9042046,"name":"offline","context":{"idset":"10475"}}
+{"timestamp":1714021381.905272,"name":"offline","context":{"idset":"10476"}}
+{"timestamp":1714021381.9063327,"name":"offline","context":{"idset":"10477"}}
+{"timestamp":1714021381.9073951,"name":"offline","context":{"idset":"10478"}}
+{"timestamp":1714021381.9084549,"name":"offline","context":{"idset":"10479"}}
+{"timestamp":1714021381.9095244,"name":"offline","context":{"idset":"10480"}}
+{"timestamp":1714021381.910578,"name":"offline","context":{"idset":"10481"}}
+{"timestamp":1714021381.9116416,"name":"offline","context":{"idset":"10482"}}
+{"timestamp":1714021381.9127169,"name":"offline","context":{"idset":"10483"}}
+{"timestamp":1714021381.9137769,"name":"offline","context":{"idset":"10484"}}
+{"timestamp":1714021381.9148383,"name":"offline","context":{"idset":"10485"}}
+{"timestamp":1714021381.9158962,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1714021381.9169497,"name":"offline","context":{"idset":"10487"}}
+{"timestamp":1714021381.9180038,"name":"offline","context":{"idset":"10488"}}
+{"timestamp":1714021381.9190543,"name":"offline","context":{"idset":"10489"}}
+{"timestamp":1714021381.9201078,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1714021381.9211736,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1714021381.9222319,"name":"offline","context":{"idset":"10492"}}
+{"timestamp":1714021381.9232845,"name":"offline","context":{"idset":"10493"}}
+{"timestamp":1714021381.9243348,"name":"offline","context":{"idset":"10494"}}
+{"timestamp":1714021381.9253831,"name":"offline","context":{"idset":"10495"}}
+{"timestamp":1714021381.9264328,"name":"offline","context":{"idset":"10496"}}
+{"timestamp":1714021381.9274914,"name":"offline","context":{"idset":"10497"}}
+{"timestamp":1714021381.9285297,"name":"offline","context":{"idset":"10498"}}
+{"timestamp":1714021381.9295747,"name":"offline","context":{"idset":"10499"}}
+{"timestamp":1714021381.930613,"name":"offline","context":{"idset":"10500"}}
+{"timestamp":1714021381.9316587,"name":"offline","context":{"idset":"10501"}}
+{"timestamp":1714021381.9327009,"name":"offline","context":{"idset":"10502"}}
+{"timestamp":1714021381.9337444,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1714021381.9348128,"name":"offline","context":{"idset":"10504"}}
+{"timestamp":1714021381.9358611,"name":"offline","context":{"idset":"10505"}}
+{"timestamp":1714021381.9369028,"name":"offline","context":{"idset":"10506"}}
+{"timestamp":1714021381.9379439,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1714021381.9389827,"name":"offline","context":{"idset":"10508"}}
+{"timestamp":1714021381.9400246,"name":"offline","context":{"idset":"10509"}}
+{"timestamp":1714021381.9410684,"name":"offline","context":{"idset":"10510"}}
+{"timestamp":1714021381.9422722,"name":"offline","context":{"idset":"10511"}}
+{"timestamp":1714021381.9439573,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1714021381.9454002,"name":"offline","context":{"idset":"10513"}}
+{"timestamp":1714021381.946444,"name":"offline","context":{"idset":"10514"}}
+{"timestamp":1714021381.9474719,"name":"offline","context":{"idset":"10515"}}
+{"timestamp":1714021381.9559124,"name":"offline","context":{"idset":"10516"}}
+{"timestamp":1714021381.9569781,"name":"offline","context":{"idset":"10517"}}
+{"timestamp":1714021381.9580173,"name":"offline","context":{"idset":"10518"}}
+{"timestamp":1714021381.9590471,"name":"offline","context":{"idset":"10519"}}
+{"timestamp":1714021381.960068,"name":"offline","context":{"idset":"10520"}}
+{"timestamp":1714021381.9611061,"name":"offline","context":{"idset":"10521"}}
+{"timestamp":1714021381.962142,"name":"offline","context":{"idset":"10522"}}
+{"timestamp":1714021381.9631696,"name":"offline","context":{"idset":"10523"}}
+{"timestamp":1714021381.9641919,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1714021381.9652159,"name":"offline","context":{"idset":"10525"}}
+{"timestamp":1714021381.9662433,"name":"offline","context":{"idset":"10526"}}
+{"timestamp":1714021381.9672647,"name":"offline","context":{"idset":"10527"}}
+{"timestamp":1714021381.9682875,"name":"offline","context":{"idset":"10528"}}
+{"timestamp":1714021381.9693139,"name":"offline","context":{"idset":"10529"}}
+{"timestamp":1714021381.9777429,"name":"offline","context":{"idset":"10530"}}
+{"timestamp":1714021381.9787698,"name":"offline","context":{"idset":"10531"}}
+{"timestamp":1714021381.9797926,"name":"offline","context":{"idset":"10532"}}
+{"timestamp":1714021381.9808376,"name":"offline","context":{"idset":"10533"}}
+{"timestamp":1714021381.9818604,"name":"offline","context":{"idset":"10534"}}
+{"timestamp":1714021381.9828794,"name":"offline","context":{"idset":"10535"}}
+{"timestamp":1714021381.983897,"name":"offline","context":{"idset":"10536"}}
+{"timestamp":1714021381.984916,"name":"offline","context":{"idset":"10537"}}
+{"timestamp":1714021381.9859445,"name":"offline","context":{"idset":"10538"}}
+{"timestamp":1714021381.9869606,"name":"offline","context":{"idset":"10539"}}
+{"timestamp":1714021381.9879754,"name":"offline","context":{"idset":"10540"}}
+{"timestamp":1714021381.9889922,"name":"offline","context":{"idset":"10541"}}
+{"timestamp":1714021381.9900019,"name":"offline","context":{"idset":"10542"}}
+{"timestamp":1714021381.9910126,"name":"offline","context":{"idset":"10543"}}
+{"timestamp":1714021381.9994655,"name":"offline","context":{"idset":"10544"}}
+{"timestamp":1714021382.0004797,"name":"offline","context":{"idset":"10545"}}
+{"timestamp":1714021382.0014846,"name":"offline","context":{"idset":"10546"}}
+{"timestamp":1714021382.0024872,"name":"offline","context":{"idset":"10547"}}
+{"timestamp":1714021382.0034997,"name":"offline","context":{"idset":"10548"}}
+{"timestamp":1714021382.0045102,"name":"offline","context":{"idset":"10549"}}
+{"timestamp":1714021382.0055118,"name":"offline","context":{"idset":"10550"}}
+{"timestamp":1714021382.0065169,"name":"offline","context":{"idset":"10551"}}
+{"timestamp":1714021382.0075209,"name":"offline","context":{"idset":"10552"}}
+{"timestamp":1714021382.0085251,"name":"offline","context":{"idset":"10553"}}
+{"timestamp":1714021382.0095229,"name":"offline","context":{"idset":"10554"}}
+{"timestamp":1714021382.0105195,"name":"offline","context":{"idset":"10555"}}
+{"timestamp":1714021382.0115275,"name":"offline","context":{"idset":"10556"}}
+{"timestamp":1714021382.0125353,"name":"offline","context":{"idset":"10557"}}
+{"timestamp":1714021382.0210576,"name":"offline","context":{"idset":"10558"}}
+{"timestamp":1714021382.022094,"name":"offline","context":{"idset":"10559"}}
+{"timestamp":1714021382.0231018,"name":"offline","context":{"idset":"10560"}}
+{"timestamp":1714021382.024106,"name":"offline","context":{"idset":"10561"}}
+{"timestamp":1714021382.0251055,"name":"offline","context":{"idset":"10562"}}
+{"timestamp":1714021382.0261052,"name":"offline","context":{"idset":"10563"}}
+{"timestamp":1714021382.0271001,"name":"offline","context":{"idset":"10564"}}
+{"timestamp":1714021382.0281119,"name":"offline","context":{"idset":"10565"}}
+{"timestamp":1714021382.0291214,"name":"offline","context":{"idset":"10566"}}
+{"timestamp":1714021382.0301163,"name":"offline","context":{"idset":"10567"}}
+{"timestamp":1714021382.0311034,"name":"offline","context":{"idset":"10568"}}
+{"timestamp":1714021382.0320899,"name":"offline","context":{"idset":"10569"}}
+{"timestamp":1714021382.0330725,"name":"offline","context":{"idset":"10570"}}
+{"timestamp":1714021382.0340545,"name":"offline","context":{"idset":"10571"}}
+{"timestamp":1714021382.0350654,"name":"offline","context":{"idset":"10572"}}
+{"timestamp":1714021382.036072,"name":"offline","context":{"idset":"10573"}}
+{"timestamp":1714021382.0445278,"name":"offline","context":{"idset":"10574"}}
+{"timestamp":1714021382.045526,"name":"offline","context":{"idset":"10575"}}
+{"timestamp":1714021382.0465233,"name":"offline","context":{"idset":"10576"}}
+{"timestamp":1714021382.0475185,"name":"offline","context":{"idset":"10577"}}
+{"timestamp":1714021382.0485177,"name":"offline","context":{"idset":"10578"}}
+{"timestamp":1714021382.04951,"name":"offline","context":{"idset":"10579"}}
+{"timestamp":1714021382.0505056,"name":"offline","context":{"idset":"10580"}}
+{"timestamp":1714021382.0514939,"name":"offline","context":{"idset":"10581"}}
+{"timestamp":1714021382.0524774,"name":"offline","context":{"idset":"10582"}}
+{"timestamp":1714021382.0534544,"name":"offline","context":{"idset":"10583"}}
+{"timestamp":1714021382.05444,"name":"offline","context":{"idset":"10584"}}
+{"timestamp":1714021382.0554223,"name":"offline","context":{"idset":"10585"}}
+{"timestamp":1714021382.0564005,"name":"offline","context":{"idset":"10586"}}
+{"timestamp":1714021382.0573926,"name":"offline","context":{"idset":"10587"}}
+{"timestamp":1714021382.0658314,"name":"offline","context":{"idset":"10588"}}
+{"timestamp":1714021382.0668356,"name":"offline","context":{"idset":"10589"}}
+{"timestamp":1714021382.0678177,"name":"offline","context":{"idset":"10590"}}
+{"timestamp":1714021382.0687866,"name":"offline","context":{"idset":"10591"}}
+{"timestamp":1714021382.0697632,"name":"offline","context":{"idset":"10592"}}
+{"timestamp":1714021382.0707409,"name":"offline","context":{"idset":"10593"}}
+{"timestamp":1714021382.071723,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1714021382.0726829,"name":"offline","context":{"idset":"10595"}}
+{"timestamp":1714021382.0736496,"name":"offline","context":{"idset":"10596"}}
+{"timestamp":1714021382.0747263,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1714021382.0757229,"name":"offline","context":{"idset":"10598"}}
+{"timestamp":1714021382.0766921,"name":"offline","context":{"idset":"10599"}}
+{"timestamp":1714021382.077656,"name":"offline","context":{"idset":"10600"}}
+{"timestamp":1714021382.0786178,"name":"offline","context":{"idset":"10601"}}
+{"timestamp":1714021382.0795755,"name":"offline","context":{"idset":"10602"}}
+{"timestamp":1714021382.0805371,"name":"offline","context":{"idset":"10603"}}
+{"timestamp":1714021382.0889618,"name":"offline","context":{"idset":"10604"}}
+{"timestamp":1714021382.0899539,"name":"offline","context":{"idset":"10605"}}
+{"timestamp":1714021382.0909224,"name":"offline","context":{"idset":"10606"}}
+{"timestamp":1714021382.0918868,"name":"offline","context":{"idset":"10607"}}
+{"timestamp":1714021382.092855,"name":"offline","context":{"idset":"10608"}}
+{"timestamp":1714021382.093822,"name":"offline","context":{"idset":"10609"}}
+{"timestamp":1714021382.0947764,"name":"offline","context":{"idset":"10610"}}
+{"timestamp":1714021382.0957401,"name":"offline","context":{"idset":"10611"}}
+{"timestamp":1714021382.0966957,"name":"offline","context":{"idset":"10612"}}
+{"timestamp":1714021382.0976586,"name":"offline","context":{"idset":"10613"}}
+{"timestamp":1714021382.0986154,"name":"offline","context":{"idset":"10614"}}
+{"timestamp":1714021382.0995705,"name":"offline","context":{"idset":"10615"}}
+{"timestamp":1714021382.100522,"name":"offline","context":{"idset":"10616"}}
+{"timestamp":1714021382.1014884,"name":"offline","context":{"idset":"10617"}}
+{"timestamp":1714021382.1099527,"name":"offline","context":{"idset":"10618"}}
+{"timestamp":1714021382.110913,"name":"offline","context":{"idset":"10619"}}
+{"timestamp":1714021382.111866,"name":"offline","context":{"idset":"10620"}}
+{"timestamp":1714021382.1128166,"name":"offline","context":{"idset":"10621"}}
+{"timestamp":1714021382.1137633,"name":"offline","context":{"idset":"10622"}}
+{"timestamp":1714021382.1147187,"name":"offline","context":{"idset":"10623"}}
+{"timestamp":1714021382.1156683,"name":"offline","context":{"idset":"10624"}}
+{"timestamp":1714021382.116617,"name":"offline","context":{"idset":"10625"}}
+{"timestamp":1714021382.117568,"name":"offline","context":{"idset":"10626"}}
+{"timestamp":1714021382.1185172,"name":"offline","context":{"idset":"10627"}}
+{"timestamp":1714021382.1194642,"name":"offline","context":{"idset":"10628"}}
+{"timestamp":1714021382.1204128,"name":"offline","context":{"idset":"10629"}}
+{"timestamp":1714021382.1213801,"name":"offline","context":{"idset":"10630"}}
+{"timestamp":1714021382.1223807,"name":"offline","context":{"idset":"10631"}}
+{"timestamp":1714021382.1233344,"name":"offline","context":{"idset":"10632"}}
+{"timestamp":1714021382.1317058,"name":"offline","context":{"idset":"10633"}}
+{"timestamp":1714021382.1326444,"name":"offline","context":{"idset":"10634"}}
+{"timestamp":1714021382.1335788,"name":"offline","context":{"idset":"10635"}}
+{"timestamp":1714021382.1345191,"name":"offline","context":{"idset":"10636"}}
+{"timestamp":1714021382.1354656,"name":"offline","context":{"idset":"10637"}}
+{"timestamp":1714021382.1364114,"name":"offline","context":{"idset":"10638"}}
+{"timestamp":1714021382.1373503,"name":"offline","context":{"idset":"10639"}}
+{"timestamp":1714021382.1382887,"name":"offline","context":{"idset":"10640"}}
+{"timestamp":1714021382.1392272,"name":"offline","context":{"idset":"10641"}}
+{"timestamp":1714021382.1401637,"name":"offline","context":{"idset":"10642"}}
+{"timestamp":1714021382.1410999,"name":"offline","context":{"idset":"10643"}}
+{"timestamp":1714021382.1420364,"name":"offline","context":{"idset":"10644"}}
+{"timestamp":1714021382.1429684,"name":"offline","context":{"idset":"10645"}}
+{"timestamp":1714021382.1439025,"name":"offline","context":{"idset":"10646"}}
+{"timestamp":1714021382.144835,"name":"offline","context":{"idset":"10647"}}
+{"timestamp":1714021382.1531761,"name":"offline","context":{"idset":"10648"}}
+{"timestamp":1714021382.1541109,"name":"offline","context":{"idset":"10649"}}
+{"timestamp":1714021382.1550422,"name":"offline","context":{"idset":"10650"}}
+{"timestamp":1714021382.1559711,"name":"offline","context":{"idset":"10651"}}
+{"timestamp":1714021382.1569035,"name":"offline","context":{"idset":"10652"}}
+{"timestamp":1714021382.1578321,"name":"offline","context":{"idset":"10653"}}
+{"timestamp":1714021382.1587691,"name":"offline","context":{"idset":"10654"}}
+{"timestamp":1714021382.1596985,"name":"offline","context":{"idset":"10655"}}
+{"timestamp":1714021382.1606119,"name":"offline","context":{"idset":"10656"}}
+{"timestamp":1714021382.1615472,"name":"offline","context":{"idset":"10657"}}
+{"timestamp":1714021382.1624732,"name":"offline","context":{"idset":"10658"}}
+{"timestamp":1714021382.1633875,"name":"offline","context":{"idset":"10659"}}
+{"timestamp":1714021382.1643071,"name":"offline","context":{"idset":"10660"}}
+{"timestamp":1714021382.165221,"name":"offline","context":{"idset":"10661"}}
+{"timestamp":1714021382.166136,"name":"offline","context":{"idset":"10662"}}
+{"timestamp":1714021382.167053,"name":"offline","context":{"idset":"10663"}}
+{"timestamp":1714021382.167968,"name":"offline","context":{"idset":"10664"}}
+{"timestamp":1714021382.1763256,"name":"offline","context":{"idset":"10665"}}
+{"timestamp":1714021382.1772492,"name":"offline","context":{"idset":"10666"}}
+{"timestamp":1714021382.1781685,"name":"offline","context":{"idset":"10667"}}
+{"timestamp":1714021382.1790872,"name":"offline","context":{"idset":"10668"}}
+{"timestamp":1714021382.1800044,"name":"offline","context":{"idset":"10669"}}
+{"timestamp":1714021382.1809211,"name":"offline","context":{"idset":"10670"}}
+{"timestamp":1714021382.1818354,"name":"offline","context":{"idset":"10671"}}
+{"timestamp":1714021382.1827469,"name":"offline","context":{"idset":"10672"}}
+{"timestamp":1714021382.1836648,"name":"offline","context":{"idset":"10673"}}
+{"timestamp":1714021382.1845784,"name":"offline","context":{"idset":"10674"}}
+{"timestamp":1714021382.1854901,"name":"offline","context":{"idset":"10675"}}
+{"timestamp":1714021382.1863959,"name":"offline","context":{"idset":"10676"}}
+{"timestamp":1714021382.1873059,"name":"offline","context":{"idset":"10677"}}
+{"timestamp":1714021382.1882169,"name":"offline","context":{"idset":"10678"}}
+{"timestamp":1714021382.1891217,"name":"offline","context":{"idset":"10679"}}
+{"timestamp":1714021382.1901364,"name":"offline","context":{"idset":"10680"}}
+{"timestamp":1714021382.1915135,"name":"offline","context":{"idset":"10681"}}
+{"timestamp":1714021382.2029922,"name":"offline","context":{"idset":"10682"}}
+{"timestamp":1714021382.203907,"name":"offline","context":{"idset":"10683"}}
+{"timestamp":1714021382.2048275,"name":"offline","context":{"idset":"10684"}}
+{"timestamp":1714021382.2057478,"name":"offline","context":{"idset":"10685"}}
+{"timestamp":1714021382.2066579,"name":"offline","context":{"idset":"10686"}}
+{"timestamp":1714021382.2075605,"name":"offline","context":{"idset":"10687"}}
+{"timestamp":1714021382.208463,"name":"offline","context":{"idset":"10688"}}
+{"timestamp":1714021382.2093639,"name":"offline","context":{"idset":"10689"}}
+{"timestamp":1714021382.210263,"name":"offline","context":{"idset":"10690"}}
+{"timestamp":1714021382.2111652,"name":"offline","context":{"idset":"10691"}}
+{"timestamp":1714021382.2120686,"name":"offline","context":{"idset":"10692"}}
+{"timestamp":1714021382.2129664,"name":"offline","context":{"idset":"10693"}}
+{"timestamp":1714021382.2138624,"name":"offline","context":{"idset":"10694"}}
+{"timestamp":1714021382.2147522,"name":"offline","context":{"idset":"10695"}}
+{"timestamp":1714021382.2156458,"name":"offline","context":{"idset":"10696"}}
+{"timestamp":1714021382.2165403,"name":"offline","context":{"idset":"10697"}}
+{"timestamp":1714021382.2249286,"name":"offline","context":{"idset":"10698"}}
+{"timestamp":1714021382.2258308,"name":"offline","context":{"idset":"10699"}}
+{"timestamp":1714021382.2267084,"name":"offline","context":{"idset":"10700"}}
+{"timestamp":1714021382.2275977,"name":"offline","context":{"idset":"10701"}}
+{"timestamp":1714021382.2284849,"name":"offline","context":{"idset":"10702"}}
+{"timestamp":1714021382.2293949,"name":"offline","context":{"idset":"10703"}}
+{"timestamp":1714021382.2302866,"name":"offline","context":{"idset":"10704"}}
+{"timestamp":1714021382.2311735,"name":"offline","context":{"idset":"10705"}}
+{"timestamp":1714021382.232065,"name":"offline","context":{"idset":"10706"}}
+{"timestamp":1714021382.2329543,"name":"offline","context":{"idset":"10707"}}
+{"timestamp":1714021382.2340372,"name":"offline","context":{"idset":"10708"}}
+{"timestamp":1714021382.2349753,"name":"offline","context":{"idset":"10709"}}
+{"timestamp":1714021382.2358761,"name":"offline","context":{"idset":"10710"}}
+{"timestamp":1714021382.2367542,"name":"offline","context":{"idset":"10711"}}
+{"timestamp":1714021382.2376406,"name":"offline","context":{"idset":"10712"}}
+{"timestamp":1714021382.2460663,"name":"offline","context":{"idset":"10713"}}
+{"timestamp":1714021382.2469704,"name":"offline","context":{"idset":"10714"}}
+{"timestamp":1714021382.248009,"name":"offline","context":{"idset":"10715"}}
+{"timestamp":1714021382.2489247,"name":"offline","context":{"idset":"10716"}}
+{"timestamp":1714021382.2498136,"name":"offline","context":{"idset":"10717"}}
+{"timestamp":1714021382.2506931,"name":"offline","context":{"idset":"10718"}}
+{"timestamp":1714021382.2516158,"name":"offline","context":{"idset":"10719"}}
+{"timestamp":1714021382.2525015,"name":"offline","context":{"idset":"10720"}}
+{"timestamp":1714021382.2533891,"name":"offline","context":{"idset":"10721"}}
+{"timestamp":1714021382.2542887,"name":"offline","context":{"idset":"10722"}}
+{"timestamp":1714021382.2551675,"name":"offline","context":{"idset":"10723"}}
+{"timestamp":1714021382.2560468,"name":"offline","context":{"idset":"10724"}}
+{"timestamp":1714021382.2643864,"name":"offline","context":{"idset":"10725"}}
+{"timestamp":1714021382.2652698,"name":"offline","context":{"idset":"10726"}}
+{"timestamp":1714021382.2661541,"name":"offline","context":{"idset":"10727"}}
+{"timestamp":1714021382.2670257,"name":"offline","context":{"idset":"10728"}}
+{"timestamp":1714021382.2678955,"name":"offline","context":{"idset":"10729"}}
+{"timestamp":1714021382.268827,"name":"offline","context":{"idset":"10730"}}
+{"timestamp":1714021382.2696955,"name":"offline","context":{"idset":"10731"}}
+{"timestamp":1714021382.2705722,"name":"offline","context":{"idset":"10732"}}
+{"timestamp":1714021382.2714784,"name":"offline","context":{"idset":"10733"}}
+{"timestamp":1714021382.2723541,"name":"offline","context":{"idset":"10734"}}
+{"timestamp":1714021382.2732222,"name":"offline","context":{"idset":"10735"}}
+{"timestamp":1714021382.2740986,"name":"offline","context":{"idset":"10736"}}
+{"timestamp":1714021382.2749639,"name":"offline","context":{"idset":"10737"}}
+{"timestamp":1714021382.2758262,"name":"offline","context":{"idset":"10738"}}
+{"timestamp":1714021382.2766809,"name":"offline","context":{"idset":"10739"}}
+{"timestamp":1714021382.2775433,"name":"offline","context":{"idset":"10740"}}
+{"timestamp":1714021382.2784097,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1714021382.2867737,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1714021382.2876709,"name":"offline","context":{"idset":"10743"}}
+{"timestamp":1714021382.2885375,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1714021382.2893896,"name":"offline","context":{"idset":"10745"}}
+{"timestamp":1714021382.2902582,"name":"offline","context":{"idset":"10746"}}
+{"timestamp":1714021382.291116,"name":"offline","context":{"idset":"10747"}}
+{"timestamp":1714021382.2919781,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1714021382.292846,"name":"offline","context":{"idset":"10749"}}
+{"timestamp":1714021382.2936943,"name":"offline","context":{"idset":"10750"}}
+{"timestamp":1714021382.294554,"name":"offline","context":{"idset":"10751"}}
+{"timestamp":1714021382.2954178,"name":"offline","context":{"idset":"10752"}}
+{"timestamp":1714021382.296273,"name":"offline","context":{"idset":"10753"}}
+{"timestamp":1714021382.2971251,"name":"offline","context":{"idset":"10754"}}
+{"timestamp":1714021382.2979848,"name":"offline","context":{"idset":"10755"}}
+{"timestamp":1714021382.2988327,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1714021382.2996752,"name":"offline","context":{"idset":"10757"}}
+{"timestamp":1714021382.3005199,"name":"offline","context":{"idset":"10758"}}
+{"timestamp":1714021382.3087854,"name":"offline","context":{"idset":"10759"}}
+{"timestamp":1714021382.3096383,"name":"offline","context":{"idset":"10760"}}
+{"timestamp":1714021382.3104858,"name":"offline","context":{"idset":"10761"}}
+{"timestamp":1714021382.3113704,"name":"offline","context":{"idset":"10762"}}
+{"timestamp":1714021382.3122261,"name":"offline","context":{"idset":"10763"}}
+{"timestamp":1714021382.3130713,"name":"offline","context":{"idset":"10764"}}
+{"timestamp":1714021382.3139224,"name":"offline","context":{"idset":"10765"}}
+{"timestamp":1714021382.3147576,"name":"offline","context":{"idset":"10766"}}
+{"timestamp":1714021382.3155994,"name":"offline","context":{"idset":"10767"}}
+{"timestamp":1714021382.3164399,"name":"offline","context":{"idset":"10768"}}
+{"timestamp":1714021382.3172908,"name":"offline","context":{"idset":"10769"}}
+{"timestamp":1714021382.3181345,"name":"offline","context":{"idset":"10770"}}
+{"timestamp":1714021382.3189774,"name":"offline","context":{"idset":"10771"}}
+{"timestamp":1714021382.3198142,"name":"offline","context":{"idset":"10772"}}
+{"timestamp":1714021382.3206444,"name":"offline","context":{"idset":"10773"}}
+{"timestamp":1714021382.3214796,"name":"offline","context":{"idset":"10774"}}
+{"timestamp":1714021382.3298109,"name":"offline","context":{"idset":"10775"}}
+{"timestamp":1714021382.3306508,"name":"offline","context":{"idset":"10776"}}
+{"timestamp":1714021382.3314862,"name":"offline","context":{"idset":"10777"}}
+{"timestamp":1714021382.3323195,"name":"offline","context":{"idset":"10778"}}
+{"timestamp":1714021382.3331523,"name":"offline","context":{"idset":"10779"}}
+{"timestamp":1714021382.3340073,"name":"offline","context":{"idset":"10780"}}
+{"timestamp":1714021382.3348415,"name":"offline","context":{"idset":"10781"}}
+{"timestamp":1714021382.3356631,"name":"offline","context":{"idset":"10782"}}
+{"timestamp":1714021382.3364916,"name":"offline","context":{"idset":"10783"}}
+{"timestamp":1714021382.3373163,"name":"offline","context":{"idset":"10784"}}
+{"timestamp":1714021382.3381641,"name":"offline","context":{"idset":"10785"}}
+{"timestamp":1714021382.3389943,"name":"offline","context":{"idset":"10786"}}
+{"timestamp":1714021382.3398185,"name":"offline","context":{"idset":"10787"}}
+{"timestamp":1714021382.340637,"name":"offline","context":{"idset":"10788"}}
+{"timestamp":1714021382.3414652,"name":"offline","context":{"idset":"10789"}}
+{"timestamp":1714021382.3422873,"name":"offline","context":{"idset":"10790"}}
+{"timestamp":1714021382.3505337,"name":"offline","context":{"idset":"10791"}}
+{"timestamp":1714021382.3513606,"name":"offline","context":{"idset":"10792"}}
+{"timestamp":1714021382.3522127,"name":"offline","context":{"idset":"10793"}}
+{"timestamp":1714021382.3530588,"name":"offline","context":{"idset":"10794"}}
+{"timestamp":1714021382.3538976,"name":"offline","context":{"idset":"10795"}}
+{"timestamp":1714021382.3547211,"name":"offline","context":{"idset":"10796"}}
+{"timestamp":1714021382.355535,"name":"offline","context":{"idset":"10797"}}
+{"timestamp":1714021382.356349,"name":"offline","context":{"idset":"10798"}}
+{"timestamp":1714021382.357172,"name":"offline","context":{"idset":"10799"}}
+{"timestamp":1714021382.3579917,"name":"offline","context":{"idset":"10800"}}
+{"timestamp":1714021382.3587999,"name":"offline","context":{"idset":"10801"}}
+{"timestamp":1714021382.3596172,"name":"offline","context":{"idset":"10802"}}
+{"timestamp":1714021382.3604362,"name":"offline","context":{"idset":"10803"}}
+{"timestamp":1714021382.3612483,"name":"offline","context":{"idset":"10804"}}
+{"timestamp":1714021382.3620653,"name":"offline","context":{"idset":"10805"}}
+{"timestamp":1714021382.3628914,"name":"offline","context":{"idset":"10806"}}
+{"timestamp":1714021382.3636959,"name":"offline","context":{"idset":"10807"}}
+{"timestamp":1714021382.3645141,"name":"offline","context":{"idset":"10808"}}
+{"timestamp":1714021382.3653185,"name":"offline","context":{"idset":"10809"}}
+{"timestamp":1714021382.3661242,"name":"offline","context":{"idset":"10810"}}
+{"timestamp":1714021382.3669317,"name":"offline","context":{"idset":"10811"}}
+{"timestamp":1714021382.3677294,"name":"offline","context":{"idset":"10812"}}
+{"timestamp":1714021382.368536,"name":"offline","context":{"idset":"10813"}}
+{"timestamp":1714021382.3693421,"name":"offline","context":{"idset":"10814"}}
+{"timestamp":1714021382.3701441,"name":"offline","context":{"idset":"10815"}}
+{"timestamp":1714021382.37095,"name":"offline","context":{"idset":"10816"}}
+{"timestamp":1714021382.3717489,"name":"offline","context":{"idset":"10817"}}
+{"timestamp":1714021382.3725557,"name":"offline","context":{"idset":"10818"}}
+{"timestamp":1714021382.3734074,"name":"offline","context":{"idset":"10819"}}
+{"timestamp":1714021382.3742137,"name":"offline","context":{"idset":"10820"}}
+{"timestamp":1714021382.3750083,"name":"offline","context":{"idset":"10821"}}
+{"timestamp":1714021382.3757966,"name":"offline","context":{"idset":"10822"}}
+{"timestamp":1714021382.3766029,"name":"offline","context":{"idset":"10823"}}
+{"timestamp":1714021382.3774018,"name":"offline","context":{"idset":"10824"}}
+{"timestamp":1714021382.3781996,"name":"offline","context":{"idset":"10825"}}
+{"timestamp":1714021382.3790057,"name":"offline","context":{"idset":"10826"}}
+{"timestamp":1714021382.3798306,"name":"offline","context":{"idset":"10827"}}
+{"timestamp":1714021382.3806424,"name":"offline","context":{"idset":"10828"}}
+{"timestamp":1714021382.3814576,"name":"offline","context":{"idset":"10829"}}
+{"timestamp":1714021382.3822634,"name":"offline","context":{"idset":"10830"}}
+{"timestamp":1714021382.3830566,"name":"offline","context":{"idset":"10831"}}
+{"timestamp":1714021382.383857,"name":"offline","context":{"idset":"10832"}}
+{"timestamp":1714021382.3846998,"name":"offline","context":{"idset":"10833"}}
+{"timestamp":1714021382.3855054,"name":"offline","context":{"idset":"10834"}}
+{"timestamp":1714021382.3862925,"name":"offline","context":{"idset":"10835"}}
+{"timestamp":1714021382.38708,"name":"offline","context":{"idset":"10836"}}
+{"timestamp":1714021382.3878763,"name":"offline","context":{"idset":"10837"}}
+{"timestamp":1714021382.3886561,"name":"offline","context":{"idset":"10838"}}
+{"timestamp":1714021382.3894415,"name":"offline","context":{"idset":"10839"}}
+{"timestamp":1714021382.3977742,"name":"offline","context":{"idset":"10840"}}
+{"timestamp":1714021382.3985748,"name":"offline","context":{"idset":"10841"}}
+{"timestamp":1714021382.3993587,"name":"offline","context":{"idset":"10842"}}
+{"timestamp":1714021382.4001479,"name":"offline","context":{"idset":"10843"}}
+{"timestamp":1714021382.4009306,"name":"offline","context":{"idset":"10844"}}
+{"timestamp":1714021382.4017022,"name":"offline","context":{"idset":"10845"}}
+{"timestamp":1714021382.4024799,"name":"offline","context":{"idset":"10846"}}
+{"timestamp":1714021382.4032602,"name":"offline","context":{"idset":"10847"}}
+{"timestamp":1714021382.4040444,"name":"offline","context":{"idset":"10848"}}
+{"timestamp":1714021382.4048407,"name":"offline","context":{"idset":"10849"}}
+{"timestamp":1714021382.4056132,"name":"offline","context":{"idset":"10850"}}
+{"timestamp":1714021382.4063816,"name":"offline","context":{"idset":"10851"}}
+{"timestamp":1714021382.4071553,"name":"offline","context":{"idset":"10852"}}
+{"timestamp":1714021382.4079382,"name":"offline","context":{"idset":"10853"}}
+{"timestamp":1714021382.4087157,"name":"offline","context":{"idset":"10854"}}
+{"timestamp":1714021382.4094961,"name":"offline","context":{"idset":"10855"}}
+{"timestamp":1714021382.4102752,"name":"offline","context":{"idset":"10856"}}
+{"timestamp":1714021382.4110472,"name":"offline","context":{"idset":"10857"}}
+{"timestamp":1714021382.4118237,"name":"offline","context":{"idset":"10858"}}
+{"timestamp":1714021382.4126177,"name":"offline","context":{"idset":"10859"}}
+{"timestamp":1714021382.4133918,"name":"offline","context":{"idset":"10860"}}
+{"timestamp":1714021382.4216022,"name":"offline","context":{"idset":"10861"}}
+{"timestamp":1714021382.4223754,"name":"offline","context":{"idset":"10862"}}
+{"timestamp":1714021382.4231424,"name":"offline","context":{"idset":"10863"}}
+{"timestamp":1714021382.4239078,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1714021382.4246633,"name":"offline","context":{"idset":"10865"}}
+{"timestamp":1714021382.425463,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1714021382.4262316,"name":"offline","context":{"idset":"10867"}}
+{"timestamp":1714021382.4269948,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1714021382.4277477,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1714021382.4285095,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1714021382.4292758,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1714021382.4300349,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1714021382.4307847,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1714021382.4315376,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1714021382.4322934,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1714021382.4330702,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1714021382.4338276,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1714021382.4345665,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1714021382.4353101,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1714021382.4360578,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1714021382.436795,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1714021382.4375486,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1714021382.4382925,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1714021382.4390321,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1714021382.4397624,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1714021382.4405065,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1714021382.441247,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1714021382.4419861,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1714021382.442714,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1714021382.443495,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1714021382.4442415,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1714021382.4449768,"name":"offline","context":{"idset":"10894"}}
+{"timestamp":1714021382.4457021,"name":"offline","context":{"idset":"10895"}}
+{"timestamp":1714021382.4464343,"name":"offline","context":{"idset":"10896"}}
+{"timestamp":1714021382.44717,"name":"offline","context":{"idset":"10897"}}
+{"timestamp":1714021382.4479153,"name":"offline","context":{"idset":"10898"}}
+{"timestamp":1714021382.4486411,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1714021382.4568908,"name":"offline","context":{"idset":"10900"}}
+{"timestamp":1714021382.4576206,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1714021382.4583871,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1714021382.4591393,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1714021382.4598939,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1714021382.4606159,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1714021382.4613409,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1714021382.4620917,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1714021382.4628234,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1714021382.4635499,"name":"offline","context":{"idset":"10909"}}
+{"timestamp":1714021382.4643283,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1714021382.4650624,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1714021382.465775,"name":"offline","context":{"idset":"10912"}}
+{"timestamp":1714021382.4665163,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1714021382.4672368,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1714021382.4679632,"name":"offline","context":{"idset":"10915"}}
+{"timestamp":1714021382.4686749,"name":"offline","context":{"idset":"10916"}}
+{"timestamp":1714021382.4694042,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1714021382.4701409,"name":"offline","context":{"idset":"10918"}}
+{"timestamp":1714021382.4708569,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1714021382.4715638,"name":"offline","context":{"idset":"10920"}}
+{"timestamp":1714021382.4722905,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1714021382.4730144,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1714021382.4737208,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1714021382.4744587,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1714021382.4751759,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1714021382.4758902,"name":"offline","context":{"idset":"10926"}}
+{"timestamp":1714021382.4766243,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1714021382.4775977,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1714021382.4786906,"name":"offline","context":{"idset":"10929"}}
+{"timestamp":1714021382.4797351,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1714021382.4807699,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1714021382.4817963,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1714021382.4828725,"name":"offline","context":{"idset":"10933"}}
+{"timestamp":1714021382.4932787,"name":"offline","context":{"idset":"10934"}}
+{"timestamp":1714021382.494019,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1714021382.4947212,"name":"offline","context":{"idset":"10936"}}
+{"timestamp":1714021382.4954278,"name":"offline","context":{"idset":"10937"}}
+{"timestamp":1714021382.4961314,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1714021382.4968309,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1714021382.4975407,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1714021382.4982381,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1714021382.4989467,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1714021382.4996393,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1714021382.5003331,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1714021382.5010307,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1714021382.5017381,"name":"offline","context":{"idset":"10946"}}
+{"timestamp":1714021382.5024374,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1714021382.5031433,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1714021382.5038569,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1714021382.5045609,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1714021382.5052528,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1714021382.5059395,"name":"offline","context":{"idset":"10952"}}
+{"timestamp":1714021382.5141234,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1714021382.5148163,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1714021382.5154994,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1714021382.5161891,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1714021382.5168743,"name":"offline","context":{"idset":"10957"}}
+{"timestamp":1714021382.5175536,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1714021382.5182543,"name":"offline","context":{"idset":"10959"}}
+{"timestamp":1714021382.518939,"name":"offline","context":{"idset":"10960"}}
+{"timestamp":1714021382.5196154,"name":"offline","context":{"idset":"10961"}}
+{"timestamp":1714021382.5202997,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1714021382.5209858,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1714021382.5216832,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1714021382.522367,"name":"offline","context":{"idset":"10965"}}
+{"timestamp":1714021382.5230453,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1714021382.523716,"name":"offline","context":{"idset":"10967"}}
+{"timestamp":1714021382.5243936,"name":"offline","context":{"idset":"10968"}}
+{"timestamp":1714021382.5250702,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1714021382.5257378,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1714021382.5264113,"name":"offline","context":{"idset":"10971"}}
+{"timestamp":1714021382.5270865,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1714021382.5277512,"name":"offline","context":{"idset":"10973"}}
+{"timestamp":1714021382.5284255,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1714021382.5290964,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1714021382.5297642,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1714021382.5304515,"name":"offline","context":{"idset":"10977"}}
+{"timestamp":1714021382.5311239,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1714021382.5318117,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1714021382.5324781,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1714021382.5331428,"name":"offline","context":{"idset":"10981"}}
+{"timestamp":1714021382.5338066,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1714021382.5344658,"name":"offline","context":{"idset":"10983"}}
+{"timestamp":1714021382.5351367,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1714021382.5357924,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1714021382.5364542,"name":"offline","context":{"idset":"10986"}}
+{"timestamp":1714021382.537118,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1714021382.5377917,"name":"offline","context":{"idset":"10988"}}
+{"timestamp":1714021382.5384552,"name":"offline","context":{"idset":"10989"}}
+{"timestamp":1714021382.5391159,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1714021382.5397656,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1714021382.5404234,"name":"offline","context":{"idset":"10992"}}
+{"timestamp":1714021382.5410793,"name":"offline","context":{"idset":"10993"}}
+{"timestamp":1714021382.5417271,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1714021382.5423768,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1714021382.5430317,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1714021382.5436766,"name":"offline","context":{"idset":"10997"}}
+{"timestamp":1714021382.544349,"name":"offline","context":{"idset":"10998"}}
+{"timestamp":1714021382.5450165,"name":"offline","context":{"idset":"10999"}}
+{"timestamp":1714021382.5456634,"name":"offline","context":{"idset":"11000"}}
+{"timestamp":1714021382.5463121,"name":"offline","context":{"idset":"11001"}}
+{"timestamp":1714021382.5469663,"name":"offline","context":{"idset":"11002"}}
+{"timestamp":1714021382.5476093,"name":"offline","context":{"idset":"11003"}}
+{"timestamp":1714021382.5482619,"name":"offline","context":{"idset":"11004"}}
+{"timestamp":1714021382.5489266,"name":"offline","context":{"idset":"11005"}}
+{"timestamp":1714021382.5495701,"name":"offline","context":{"idset":"11006"}}
+{"timestamp":1714021382.5503976,"name":"offline","context":{"idset":"11007"}}
+{"timestamp":1714021382.551065,"name":"offline","context":{"idset":"11008"}}
+{"timestamp":1714021382.551707,"name":"offline","context":{"idset":"11009"}}
+{"timestamp":1714021382.5523629,"name":"offline","context":{"idset":"11010"}}
+{"timestamp":1714021382.5530114,"name":"offline","context":{"idset":"11011"}}
+{"timestamp":1714021382.5536478,"name":"offline","context":{"idset":"11012"}}
+{"timestamp":1714021382.5542905,"name":"offline","context":{"idset":"11013"}}
+{"timestamp":1714021382.5549343,"name":"offline","context":{"idset":"11014"}}
+{"timestamp":1714021382.5555665,"name":"offline","context":{"idset":"11015"}}
+{"timestamp":1714021382.5562046,"name":"offline","context":{"idset":"11016"}}
+{"timestamp":1714021382.5568428,"name":"offline","context":{"idset":"11017"}}
+{"timestamp":1714021382.5574729,"name":"offline","context":{"idset":"11018"}}
+{"timestamp":1714021382.5581114,"name":"offline","context":{"idset":"11019"}}
+{"timestamp":1714021382.5587406,"name":"offline","context":{"idset":"11020"}}
+{"timestamp":1714021382.5593839,"name":"offline","context":{"idset":"11021"}}
+{"timestamp":1714021382.5600207,"name":"offline","context":{"idset":"11022"}}
+{"timestamp":1714021382.5606482,"name":"offline","context":{"idset":"11023"}}
+{"timestamp":1714021382.5612998,"name":"offline","context":{"idset":"11024"}}
+{"timestamp":1714021382.5619521,"name":"offline","context":{"idset":"11025"}}
+{"timestamp":1714021382.5625808,"name":"offline","context":{"idset":"11026"}}
+{"timestamp":1714021382.5632164,"name":"offline","context":{"idset":"11027"}}
+{"timestamp":1714021382.5638475,"name":"offline","context":{"idset":"11028"}}
+{"timestamp":1714021382.5644701,"name":"offline","context":{"idset":"11029"}}
+{"timestamp":1714021382.5651002,"name":"offline","context":{"idset":"11030"}}
+{"timestamp":1714021382.5732167,"name":"offline","context":{"idset":"11031"}}
+{"timestamp":1714021382.5738547,"name":"offline","context":{"idset":"11032"}}
+{"timestamp":1714021382.5744765,"name":"offline","context":{"idset":"11033"}}
+{"timestamp":1714021382.575104,"name":"offline","context":{"idset":"11034"}}
+{"timestamp":1714021382.5757205,"name":"offline","context":{"idset":"11035"}}
+{"timestamp":1714021382.5763452,"name":"offline","context":{"idset":"11036"}}
+{"timestamp":1714021382.5769715,"name":"offline","context":{"idset":"11037"}}
+{"timestamp":1714021382.5775859,"name":"offline","context":{"idset":"11038"}}
+{"timestamp":1714021382.578207,"name":"offline","context":{"idset":"11039"}}
+{"timestamp":1714021382.5788283,"name":"offline","context":{"idset":"11040"}}
+{"timestamp":1714021382.5794439,"name":"offline","context":{"idset":"11041"}}
+{"timestamp":1714021382.5800796,"name":"offline","context":{"idset":"11042"}}
+{"timestamp":1714021382.5806944,"name":"offline","context":{"idset":"11043"}}
+{"timestamp":1714021382.5813136,"name":"offline","context":{"idset":"11044"}}
+{"timestamp":1714021382.5819335,"name":"offline","context":{"idset":"11045"}}
+{"timestamp":1714021382.5825417,"name":"offline","context":{"idset":"11046"}}
+{"timestamp":1714021382.5831594,"name":"offline","context":{"idset":"11047"}}
+{"timestamp":1714021382.5837677,"name":"offline","context":{"idset":"11048"}}
+{"timestamp":1714021382.5843832,"name":"offline","context":{"idset":"11049"}}
+{"timestamp":1714021382.5849981,"name":"offline","context":{"idset":"11050"}}
+{"timestamp":1714021382.5930645,"name":"offline","context":{"idset":"11051"}}
+{"timestamp":1714021382.5936699,"name":"offline","context":{"idset":"11052"}}
+{"timestamp":1714021382.5942822,"name":"offline","context":{"idset":"11053"}}
+{"timestamp":1714021382.5948937,"name":"offline","context":{"idset":"11054"}}
+{"timestamp":1714021382.5955081,"name":"offline","context":{"idset":"11055"}}
+{"timestamp":1714021382.5962307,"name":"offline","context":{"idset":"11056"}}
+{"timestamp":1714021382.5968492,"name":"offline","context":{"idset":"11057"}}
+{"timestamp":1714021382.5974524,"name":"offline","context":{"idset":"11058"}}
+{"timestamp":1714021382.5980675,"name":"offline","context":{"idset":"11059"}}
+{"timestamp":1714021382.5986688,"name":"offline","context":{"idset":"11060"}}
+{"timestamp":1714021382.599278,"name":"offline","context":{"idset":"11061"}}
+{"timestamp":1714021382.5999141,"name":"offline","context":{"idset":"11062"}}
+{"timestamp":1714021382.6005213,"name":"offline","context":{"idset":"11063"}}
+{"timestamp":1714021382.6011901,"name":"offline","context":{"idset":"11064"}}
+{"timestamp":1714021382.6017849,"name":"offline","context":{"idset":"11065"}}
+{"timestamp":1714021382.6023862,"name":"offline","context":{"idset":"11066"}}
+{"timestamp":1714021382.6030138,"name":"offline","context":{"idset":"11067"}}
+{"timestamp":1714021382.6036355,"name":"offline","context":{"idset":"11068"}}
+{"timestamp":1714021382.604284,"name":"offline","context":{"idset":"11069"}}
+{"timestamp":1714021382.6048827,"name":"offline","context":{"idset":"11070"}}
+{"timestamp":1714021382.6054704,"name":"offline","context":{"idset":"11071"}}
+{"timestamp":1714021382.6060688,"name":"offline","context":{"idset":"11072"}}
+{"timestamp":1714021382.6066592,"name":"offline","context":{"idset":"11073"}}
+{"timestamp":1714021382.6072521,"name":"offline","context":{"idset":"11074"}}
+{"timestamp":1714021382.6152825,"name":"offline","context":{"idset":"11075"}}
+{"timestamp":1714021382.6159606,"name":"offline","context":{"idset":"11076"}}
+{"timestamp":1714021382.6165566,"name":"offline","context":{"idset":"11077"}}
+{"timestamp":1714021382.6171489,"name":"offline","context":{"idset":"11078"}}
+{"timestamp":1714021382.617734,"name":"offline","context":{"idset":"11079"}}
+{"timestamp":1714021382.6183317,"name":"offline","context":{"idset":"11080"}}
+{"timestamp":1714021382.6189644,"name":"offline","context":{"idset":"11081"}}
+{"timestamp":1714021382.6195462,"name":"offline","context":{"idset":"11082"}}
+{"timestamp":1714021382.6201386,"name":"offline","context":{"idset":"11083"}}
+{"timestamp":1714021382.6207638,"name":"offline","context":{"idset":"11084"}}
+{"timestamp":1714021382.6214125,"name":"offline","context":{"idset":"11085"}}
+{"timestamp":1714021382.6220038,"name":"offline","context":{"idset":"11086"}}
+{"timestamp":1714021382.6226306,"name":"offline","context":{"idset":"11087"}}
+{"timestamp":1714021382.6232162,"name":"offline","context":{"idset":"11088"}}
+{"timestamp":1714021382.6237917,"name":"offline","context":{"idset":"11089"}}
+{"timestamp":1714021382.6243763,"name":"offline","context":{"idset":"11090"}}
+{"timestamp":1714021382.6249602,"name":"offline","context":{"idset":"11091"}}
+{"timestamp":1714021382.625536,"name":"offline","context":{"idset":"11092"}}
+{"timestamp":1714021382.6261287,"name":"offline","context":{"idset":"11093"}}
+{"timestamp":1714021382.62674,"name":"offline","context":{"idset":"11094"}}
+{"timestamp":1714021382.6347659,"name":"offline","context":{"idset":"11095"}}
+{"timestamp":1714021382.6353481,"name":"offline","context":{"idset":"11096"}}
+{"timestamp":1714021382.6359243,"name":"offline","context":{"idset":"11097"}}
+{"timestamp":1714021382.6364925,"name":"offline","context":{"idset":"11098"}}
+{"timestamp":1714021382.6370676,"name":"offline","context":{"idset":"11099"}}
+{"timestamp":1714021382.6376712,"name":"offline","context":{"idset":"11100"}}
+{"timestamp":1714021382.6382945,"name":"offline","context":{"idset":"11101"}}
+{"timestamp":1714021382.6388967,"name":"offline","context":{"idset":"11102"}}
+{"timestamp":1714021382.6394613,"name":"offline","context":{"idset":"11103"}}
+{"timestamp":1714021382.6400437,"name":"offline","context":{"idset":"11104"}}
+{"timestamp":1714021382.6406238,"name":"offline","context":{"idset":"11105"}}
+{"timestamp":1714021382.6412029,"name":"offline","context":{"idset":"11106"}}
+{"timestamp":1714021382.64183,"name":"offline","context":{"idset":"11107"}}
+{"timestamp":1714021382.642421,"name":"offline","context":{"idset":"11108"}}
+{"timestamp":1714021382.6430252,"name":"offline","context":{"idset":"11109"}}
+{"timestamp":1714021382.6435864,"name":"offline","context":{"idset":"11110"}}
+{"timestamp":1714021382.6441545,"name":"offline","context":{"idset":"11111"}}
+{"timestamp":1714021382.6447115,"name":"offline","context":{"idset":"11112"}}
+{"timestamp":1714021382.6452928,"name":"offline","context":{"idset":"11113"}}
+{"timestamp":1714021382.6458607,"name":"offline","context":{"idset":"11114"}}
+{"timestamp":1714021382.6464162,"name":"offline","context":{"idset":"11115"}}
+{"timestamp":1714021382.6469793,"name":"offline","context":{"idset":"11116"}}
+{"timestamp":1714021382.6475499,"name":"offline","context":{"idset":"11117"}}
+{"timestamp":1714021382.6481106,"name":"offline","context":{"idset":"11118"}}
+{"timestamp":1714021382.6486638,"name":"offline","context":{"idset":"11119"}}
+{"timestamp":1714021382.6492248,"name":"offline","context":{"idset":"11120"}}
+{"timestamp":1714021382.6497753,"name":"offline","context":{"idset":"11121"}}
+{"timestamp":1714021382.6503341,"name":"offline","context":{"idset":"11122"}}
+{"timestamp":1714021382.6508927,"name":"offline","context":{"idset":"11123"}}
+{"timestamp":1714021382.6514409,"name":"offline","context":{"idset":"11124"}}
+{"timestamp":1714021382.6519973,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1714021382.6525586,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1714021382.6531835,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1714021382.6537275,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1714021382.6542766,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1714021382.6548283,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1714021382.6553743,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1714021382.6559262,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1714021382.6564677,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1714021382.6570389,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1714021382.6575837,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1714021382.6581292,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1714021382.6586668,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1714021382.6592078,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1714021382.6597435,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1714021382.6602829,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1714021382.6608291,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1714021382.6613646,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1714021382.6619048,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1714021382.6624382,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1714021382.6629753,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1714021382.6635158,"name":"offline","context":{"idset":"11146"}}
+{"timestamp":1714021382.6640959,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1714021382.6646509,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1714021382.6652503,"name":"offline","context":{"idset":"11149"}}
+{"timestamp":1714021382.6657782,"name":"offline","context":{"idset":"11150"}}
+{"timestamp":1714021382.6663141,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1714021382.6668487,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1714021382.6673734,"name":"offline","context":{"idset":"11153"}}
+{"timestamp":1714021382.6679516,"name":"offline","context":{"idset":"11154"}}
+{"timestamp":1714021382.6684723,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1714021382.6690893,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1714021382.6696076,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1714021382.6701334,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1714021382.6706486,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1714021382.6711679,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1714021382.6717315,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1714021382.672261,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1714021382.6728444,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1714021382.6733904,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1714021382.6739118,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1714021382.6744223,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1714021382.6749432,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1714021382.6754513,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1714021382.675977,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1714021382.6764929,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1714021382.6770172,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1714021382.6775813,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1714021382.6782289,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1714021382.6789153,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1714021382.6794255,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1714021382.6799779,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1714021382.6804807,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1714021382.6810737,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1714021382.6888158,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1714021382.6893167,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1714021382.6898224,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1714021382.6903176,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1714021382.6908176,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1714021382.6913104,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1714021382.691807,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1714021382.692297,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1714021382.6927855,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1714021382.6932745,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1714021382.6937883,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1714021382.6942787,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1714021382.6947999,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1714021382.695327,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1714021382.6958144,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1714021382.6962936,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1714021382.6967719,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1714021382.697257,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1714021382.6977336,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1714021382.6982136,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1714021382.6987152,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1714021382.6992188,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1714021382.7070441,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1714021382.70752,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1714021382.7080488,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1714021382.7085328,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1714021382.7090876,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1714021382.709549,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1714021382.7100229,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1714021382.7104871,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1714021382.7109587,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1714021382.7114215,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1714021382.7119279,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1714021382.7123907,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1714021382.7129171,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1714021382.7133968,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1714021382.7138689,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1714021382.7143264,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1714021382.7147844,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1714021382.7152488,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1714021382.7158144,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1714021382.716352,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1714021382.7169576,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1714021382.7174926,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1714021382.7253895,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1714021382.7258565,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1714021382.7263076,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1714021382.7267554,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1714021382.7272143,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1714021382.7276649,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1714021382.7281225,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1714021382.7285688,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1714021382.7290177,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1714021382.7294574,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1714021382.7299035,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1714021382.7303405,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1714021382.7307768,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1714021382.7312214,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1714021382.7316546,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1714021382.7320924,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1714021382.7325251,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1714021382.7329633,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1714021382.7333946,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1714021382.7338316,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1714021382.7342615,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1714021382.7347085,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1714021382.735152,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1714021382.7355802,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1714021382.7429535,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1714021382.7433808,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1714021382.7438114,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1714021382.7442338,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1714021382.7446563,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1714021382.7450793,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1714021382.7454941,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1714021382.7459123,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1714021382.7463236,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1714021382.7467325,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1714021382.7471464,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1714021382.747555,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1714021382.7479696,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1714021382.7483761,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1714021382.7487826,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1714021382.7491953,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1714021382.7495992,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1714021382.7500281,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1714021382.7504368,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1714021382.7508454,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1714021382.7512474,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1714021382.7516501,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1714021382.7521894,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1714021382.7525856,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1714021382.7597942,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1714021382.7602072,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1714021382.7606061,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1714021382.761009,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1714021382.761404,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1714021382.7617974,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1714021382.7621958,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1714021382.7625864,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1714021382.7629828,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1714021382.7633719,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1714021382.7637618,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1714021382.7641566,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1714021382.7645409,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1714021382.7649782,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1714021382.7653759,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1714021382.7657723,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1714021382.7661736,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1714021382.7665539,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1714021382.7669411,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1714021382.7673192,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1714021382.7676959,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1714021382.7680779,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1714021382.7684538,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1714021382.7688341,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1714021382.7692096,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1714021382.7696311,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1714021382.7700765,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1714021382.7704511,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1714021382.7708302,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1714021382.7712016,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1714021382.7715735,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1714021382.7719789,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1714021382.7723477,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1714021382.772716,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1714021382.7730913,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1714021382.7734578,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1714021382.7738559,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1714021382.7742574,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1714021382.7746215,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1714021382.7750177,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1714021382.7753787,"name":"offline","context":{"idset":"11439"}}
+{"timestamp":1714021382.7757375,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1714021382.7761052,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1714021382.7764723,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1714021382.7768896,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1714021382.7772832,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1714021382.7776411,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1714021382.7780063,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1714021382.7783616,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1714021382.7787187,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1714021382.7791173,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1714021382.7794743,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1714021382.7798491,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1714021382.7802038,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1714021382.7805543,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1714021382.7809103,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1714021382.7812588,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1714021382.7816067,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1714021382.7819595,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1714021382.7823064,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1714021382.782654,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1714021382.7830057,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1714021382.7833512,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1714021382.7836947,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1714021382.7840459,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1714021382.7843895,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1714021382.7847307,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1714021382.7850788,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1714021382.7854195,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1714021382.7857604,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1714021382.7861199,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1714021382.7864614,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1714021382.7867975,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1714021382.7871449,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1714021382.7874777,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1714021382.7878332,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1714021382.7882078,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1714021382.7885385,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1714021382.788902,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1714021382.7892325,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1714021382.7895622,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1714021382.7898962,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1714021382.7902231,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1714021382.7905495,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1714021382.7908826,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1714021382.7912076,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1714021382.7915316,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1714021382.7918618,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1714021382.7921853,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1714021382.7925084,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1714021382.7928374,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1714021382.7931595,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1714021382.7934804,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1714021382.7938204,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1714021382.7941828,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1714021382.7945008,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1714021382.7948418,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1714021382.7951691,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1714021382.795485,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1714021382.7958076,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1714021382.7961242,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1714021382.7964375,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1714021382.7967508,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1714021382.7970681,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1714021382.8049285,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1714021382.8053846,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1714021382.8058462,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1714021382.8062928,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1714021382.8067353,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1714021382.8071849,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1714021382.8075993,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1714021382.8080554,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1714021382.8084943,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1714021382.8089337,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1714021382.8093691,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1714021382.8098142,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1714021382.8102546,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1714021382.8106828,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1714021382.8111868,"name":"offline","context":{"idset":"11517"}}
+{"timestamp":1714021382.8116806,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1714021382.8121533,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1714021382.8125823,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1714021382.813014,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1714021382.813436,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1714021382.8138745,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1714021382.8142917,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1714021382.8145845,"name":"offline","context":{"idset":"11525"}}
+{"timestamp":1714021382.8148909,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1714021382.815181,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1714021382.821857,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1714021382.8221483,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1714021382.822438,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1714021382.8227224,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1714021382.8230164,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1714021382.8233011,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1714021382.8235843,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1714021382.8238816,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1714021382.8241899,"name":"offline","context":{"idset":"11536"}}
+{"timestamp":1714021382.8244925,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1714021382.8247745,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1714021382.8250613,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1714021382.8253415,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1714021382.8256197,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1714021382.8259082,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1714021382.8261859,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1714021382.8264618,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1714021382.8267362,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1714021382.8270209,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1714021382.8272953,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1714021382.8275683,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1714021382.8278537,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1714021382.8281264,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1714021382.8283978,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1714021382.8286686,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1714021382.8289475,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1714021382.8292141,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1714021382.8294802,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1714021382.8361447,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1714021382.8364122,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1714021382.8366761,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1714021382.8369458,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1714021382.8372087,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1714021382.8374708,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1714021382.8377311,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1714021382.8379974,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1714021382.8382576,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1714021382.8385167,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1714021382.8387744,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1714021382.8390417,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1714021382.8392982,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1714021382.8395548,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1714021382.8398182,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1714021382.8400726,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1714021382.8403244,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1714021382.8405752,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1714021382.8408558,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1714021382.8411129,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1714021382.8413622,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1714021382.8416107,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1714021382.8418827,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1714021382.8421474,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1714021382.8423965,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1714021382.8426449,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1714021382.8429019,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1714021382.8431525,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1714021382.8496964,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1714021382.8499753,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1714021382.8502452,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1714021382.8504872,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1714021382.8507271,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1714021382.8509736,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1714021382.8512118,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1714021382.8514485,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1714021382.8516855,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1714021382.8519533,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1714021382.85219,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1714021382.8524251,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1714021382.8526592,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1714021382.8529112,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1714021382.8531616,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1714021382.8533938,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1714021382.8536251,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1714021382.853869,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1714021382.854141,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1714021382.8543711,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1714021382.8546002,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1714021382.8548369,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1714021382.8550658,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1714021382.8552923,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1714021382.8555183,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1714021382.8557436,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1714021382.8559771,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1714021382.8562024,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1714021382.856426,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1714021382.8628187,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1714021382.8630416,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1714021382.863261,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1714021382.8634794,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1714021382.8636982,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1714021382.8639238,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1714021382.8641407,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1714021382.8643575,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1714021382.864573,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1714021382.8647878,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1714021382.8650391,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1714021382.8652537,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1714021382.8654666,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1714021382.8656788,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1714021382.8659036,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1714021382.866154,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1714021382.8663657,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1714021382.8665757,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1714021382.8667848,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1714021382.8669996,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1714021382.8672078,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1714021382.867415,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1714021382.8676221,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1714021382.8678353,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1714021382.868041,"name":"offline","context":{"idset":"11637"}}
+{"timestamp":1714021382.8682458,"name":"offline","context":{"idset":"11638"}}
+{"timestamp":1714021382.8684497,"name":"offline","context":{"idset":"11639"}}
+{"timestamp":1714021382.868654,"name":"offline","context":{"idset":"11640"}}
+{"timestamp":1714021382.8688648,"name":"offline","context":{"idset":"11641"}}
+{"timestamp":1714021382.8690662,"name":"offline","context":{"idset":"11642"}}
+{"timestamp":1714021382.8692675,"name":"offline","context":{"idset":"11643"}}
+{"timestamp":1714021382.8694677,"name":"offline","context":{"idset":"11644"}}
+{"timestamp":1714021382.869668,"name":"offline","context":{"idset":"11645"}}
+{"timestamp":1714021382.8698747,"name":"offline","context":{"idset":"11646"}}
+{"timestamp":1714021382.8700731,"name":"offline","context":{"idset":"11647"}}
+{"timestamp":1714021382.8702719,"name":"offline","context":{"idset":"11648"}}
+{"timestamp":1714021382.8704698,"name":"offline","context":{"idset":"11649"}}
+{"timestamp":1714021382.8769591,"name":"offline","context":{"idset":"11650"}}
+{"timestamp":1714021382.8771734,"name":"offline","context":{"idset":"11651"}}
+{"timestamp":1714021382.8773692,"name":"offline","context":{"idset":"11652"}}
+{"timestamp":1714021382.8775632,"name":"offline","context":{"idset":"11653"}}
+{"timestamp":1714021382.8777568,"name":"offline","context":{"idset":"11654"}}
+{"timestamp":1714021382.8780167,"name":"offline","context":{"idset":"11655"}}
+{"timestamp":1714021382.8782611,"name":"offline","context":{"idset":"11656"}}
+{"timestamp":1714021382.8784893,"name":"offline","context":{"idset":"11657"}}
+{"timestamp":1714021382.8786826,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1714021382.8789163,"name":"offline","context":{"idset":"11659"}}
+{"timestamp":1714021382.8791513,"name":"offline","context":{"idset":"11661"}}
+{"timestamp":1714021382.8793979,"name":"offline","context":{"idset":"11662"}}
+{"timestamp":1714021382.8796136,"name":"offline","context":{"idset":"11663"}}
+{"timestamp":1714021382.8798454,"name":"offline","context":{"idset":"11664"}}
+{"timestamp":1714021382.880048,"name":"offline","context":{"idset":"11665"}}
+{"timestamp":1714021382.8802567,"name":"offline","context":{"idset":"11666"}}
+{"timestamp":1714021382.8804417,"name":"offline","context":{"idset":"11667"}}
+{"timestamp":1714021382.8806245,"name":"offline","context":{"idset":"11668"}}
+{"timestamp":1714021382.8808172,"name":"offline","context":{"idset":"11669"}}
+{"timestamp":1714021382.8809993,"name":"offline","context":{"idset":"11670"}}
+{"timestamp":1714021382.8811786,"name":"offline","context":{"idset":"11671"}}
+{"timestamp":1714021382.8813584,"name":"offline","context":{"idset":"11672"}}
+{"timestamp":1714021382.8815372,"name":"offline","context":{"idset":"11673"}}
+{"timestamp":1714021382.8817141,"name":"offline","context":{"idset":"11674"}}
+{"timestamp":1714021382.8819008,"name":"offline","context":{"idset":"11675"}}
+{"timestamp":1714021382.8820782,"name":"offline","context":{"idset":"11676"}}
+{"timestamp":1714021382.8822546,"name":"offline","context":{"idset":"11677"}}
+{"timestamp":1714021382.8824306,"name":"offline","context":{"idset":"11678"}}
+{"timestamp":1714021382.8826044,"name":"offline","context":{"idset":"11679"}}
+{"timestamp":1714021382.8827772,"name":"offline","context":{"idset":"11680"}}
+{"timestamp":1714021382.8829606,"name":"offline","context":{"idset":"11681"}}
+{"timestamp":1714021382.8831334,"name":"offline","context":{"idset":"11682"}}
+{"timestamp":1714021382.8894293,"name":"offline","context":{"idset":"11683"}}
+{"timestamp":1714021382.8896048,"name":"offline","context":{"idset":"11684"}}
+{"timestamp":1714021382.8897784,"name":"offline","context":{"idset":"11685"}}
+{"timestamp":1714021382.8899598,"name":"offline","context":{"idset":"11686"}}
+{"timestamp":1714021382.8901322,"name":"offline","context":{"idset":"11687"}}
+{"timestamp":1714021382.8903017,"name":"offline","context":{"idset":"11688"}}
+{"timestamp":1714021382.8904722,"name":"offline","context":{"idset":"11689"}}
+{"timestamp":1714021382.8906422,"name":"offline","context":{"idset":"11690"}}
+{"timestamp":1714021382.8908203,"name":"offline","context":{"idset":"11691"}}
+{"timestamp":1714021382.8909893,"name":"offline","context":{"idset":"11692"}}
+{"timestamp":1714021382.8911562,"name":"offline","context":{"idset":"11693"}}
+{"timestamp":1714021382.8913217,"name":"offline","context":{"idset":"11694"}}
+{"timestamp":1714021382.8914871,"name":"offline","context":{"idset":"11695"}}
+{"timestamp":1714021382.89165,"name":"offline","context":{"idset":"11696"}}
+{"timestamp":1714021382.891818,"name":"offline","context":{"idset":"11697"}}
+{"timestamp":1714021382.891993,"name":"offline","context":{"idset":"11698"}}
+{"timestamp":1714021382.8921618,"name":"offline","context":{"idset":"11699"}}
+{"timestamp":1714021382.8923221,"name":"offline","context":{"idset":"11700"}}
+{"timestamp":1714021382.8924809,"name":"offline","context":{"idset":"11701"}}
+{"timestamp":1714021382.8926392,"name":"offline","context":{"idset":"11702"}}
+{"timestamp":1714021382.8927968,"name":"offline","context":{"idset":"11703"}}
+{"timestamp":1714021382.8929603,"name":"offline","context":{"idset":"11704"}}
+{"timestamp":1714021382.8931167,"name":"offline","context":{"idset":"11705"}}
+{"timestamp":1714021382.8932726,"name":"offline","context":{"idset":"11706"}}
+{"timestamp":1714021382.8934271,"name":"offline","context":{"idset":"11707"}}
+{"timestamp":1714021382.8935809,"name":"offline","context":{"idset":"11708"}}
+{"timestamp":1714021382.8937347,"name":"offline","context":{"idset":"11709"}}
+{"timestamp":1714021382.893894,"name":"offline","context":{"idset":"11710"}}
+{"timestamp":1714021382.8940721,"name":"offline","context":{"idset":"11711"}}
+{"timestamp":1714021382.8942306,"name":"offline","context":{"idset":"11712"}}
+{"timestamp":1714021382.8943837,"name":"offline","context":{"idset":"11713"}}
+{"timestamp":1714021382.8945341,"name":"offline","context":{"idset":"11714"}}
+{"timestamp":1714021382.9007175,"name":"offline","context":{"idset":"11715"}}
+{"timestamp":1714021382.9008732,"name":"offline","context":{"idset":"11716"}}
+{"timestamp":1714021382.901022,"name":"offline","context":{"idset":"11733"}}
+{"timestamp":1714021382.9011667,"name":"offline","context":{"idset":"11734"}}
+{"timestamp":1714021382.9013114,"name":"offline","context":{"idset":"11735"}}
+{"timestamp":1714021382.9014556,"name":"offline","context":{"idset":"11736"}}
+{"timestamp":1714021382.9015987,"name":"offline","context":{"idset":"11737"}}
+{"timestamp":1714021382.9017413,"name":"offline","context":{"idset":"11738"}}
+{"timestamp":1714021382.9018915,"name":"offline","context":{"idset":"11739"}}
+{"timestamp":1714021382.9020326,"name":"offline","context":{"idset":"11740"}}
+{"timestamp":1714021382.9021738,"name":"offline","context":{"idset":"11741"}}
+{"timestamp":1714021382.9023142,"name":"offline","context":{"idset":"11742"}}
+{"timestamp":1714021382.9024534,"name":"offline","context":{"idset":"11743"}}
+{"timestamp":1714021382.9025919,"name":"offline","context":{"idset":"11744"}}
+{"timestamp":1714021382.9027307,"name":"offline","context":{"idset":"11745"}}
+{"timestamp":1714021382.902873,"name":"offline","context":{"idset":"11746"}}
+{"timestamp":1714021382.9030247,"name":"offline","context":{"idset":"11747"}}
+{"timestamp":1714021382.9031756,"name":"offline","context":{"idset":"11748"}}
+{"timestamp":1714021382.903311,"name":"offline","context":{"idset":"11749"}}
+{"timestamp":1714021382.903446,"name":"offline","context":{"idset":"11750"}}
+{"timestamp":1714021382.9035802,"name":"offline","context":{"idset":"11751"}}
+{"timestamp":1714021382.903713,"name":"offline","context":{"idset":"11752"}}
+{"timestamp":1714021382.9038525,"name":"offline","context":{"idset":"11753"}}
+{"timestamp":1714021382.9039958,"name":"offline","context":{"idset":"11754"}}
+{"timestamp":1714021382.904139,"name":"offline","context":{"idset":"11755"}}
+{"timestamp":1714021382.9042699,"name":"offline","context":{"idset":"11756"}}
+{"timestamp":1714021382.9044003,"name":"offline","context":{"idset":"11757"}}
+{"timestamp":1714021382.9045291,"name":"offline","context":{"idset":"11758"}}
+{"timestamp":1714021382.9046586,"name":"offline","context":{"idset":"11759"}}
+{"timestamp":1714021382.9047871,"name":"offline","context":{"idset":"11760"}}
+{"timestamp":1714021382.9049208,"name":"offline","context":{"idset":"11761"}}
+{"timestamp":1714021382.9110177,"name":"offline","context":{"idset":"11762"}}
+{"timestamp":1714021382.9111452,"name":"offline","context":{"idset":"11763"}}
+{"timestamp":1714021382.9112716,"name":"offline","context":{"idset":"11764"}}
+{"timestamp":1714021382.9113963,"name":"offline","context":{"idset":"11765"}}
+{"timestamp":1714021382.9115193,"name":"offline","context":{"idset":"11766"}}
+{"timestamp":1714021382.9116423,"name":"offline","context":{"idset":"11767"}}
+{"timestamp":1714021382.9117639,"name":"offline","context":{"idset":"11768"}}
+{"timestamp":1714021382.9119203,"name":"offline","context":{"idset":"11769"}}
+{"timestamp":1714021382.9120443,"name":"offline","context":{"idset":"11770"}}
+{"timestamp":1714021382.9121642,"name":"offline","context":{"idset":"11771"}}
+{"timestamp":1714021382.9123032,"name":"offline","context":{"idset":"11772"}}
+{"timestamp":1714021382.9124272,"name":"offline","context":{"idset":"11773"}}
+{"timestamp":1714021382.9125454,"name":"offline","context":{"idset":"11774"}}
+{"timestamp":1714021382.9126632,"name":"offline","context":{"idset":"11775"}}
+{"timestamp":1714021382.9127812,"name":"offline","context":{"idset":"11776"}}
+{"timestamp":1714021382.9129083,"name":"offline","context":{"idset":"11777"}}
+{"timestamp":1714021382.913039,"name":"offline","context":{"idset":"11778"}}
+{"timestamp":1714021382.9131677,"name":"offline","context":{"idset":"11779"}}
+{"timestamp":1714021382.9132831,"name":"offline","context":{"idset":"11780"}}
+{"timestamp":1714021382.9133968,"name":"offline","context":{"idset":"11781"}}
+{"timestamp":1714021382.9135103,"name":"offline","context":{"idset":"11782"}}
+{"timestamp":1714021382.9136229,"name":"offline","context":{"idset":"11783"}}
+{"timestamp":1714021382.9137347,"name":"offline","context":{"idset":"11784"}}
+{"timestamp":1714021382.9138546,"name":"offline","context":{"idset":"11785"}}
+{"timestamp":1714021382.913969,"name":"offline","context":{"idset":"11786"}}
+{"timestamp":1714021382.9140983,"name":"offline","context":{"idset":"11787"}}
+{"timestamp":1714021382.9200876,"name":"offline","context":{"idset":"11788"}}
+{"timestamp":1714021382.9202003,"name":"offline","context":{"idset":"11789"}}
+{"timestamp":1714021382.9203088,"name":"offline","context":{"idset":"11791"}}
+{"timestamp":1714021382.9204156,"name":"offline","context":{"idset":"11792"}}
+{"timestamp":1714021382.9205222,"name":"offline","context":{"idset":"11793"}}
+{"timestamp":1714021382.9206278,"name":"offline","context":{"idset":"11794"}}
+{"timestamp":1714021382.9207337,"name":"offline","context":{"idset":"11795"}}
+{"timestamp":1714021382.9208448,"name":"offline","context":{"idset":"11796"}}
+{"timestamp":1714021382.9209485,"name":"offline","context":{"idset":"11797"}}
+{"timestamp":1714021382.9210517,"name":"offline","context":{"idset":"11798"}}
+{"timestamp":1714021382.9211538,"name":"offline","context":{"idset":"11799"}}
+{"timestamp":1714021382.9212542,"name":"offline","context":{"idset":"11800"}}
+{"timestamp":1714021382.9213543,"name":"offline","context":{"idset":"11801"}}
+{"timestamp":1714021382.921454,"name":"offline","context":{"idset":"11802"}}
+{"timestamp":1714021382.9215529,"name":"offline","context":{"idset":"11803"}}
+{"timestamp":1714021382.9216506,"name":"offline","context":{"idset":"11804"}}
+{"timestamp":1714021382.9217484,"name":"offline","context":{"idset":"11805"}}
+{"timestamp":1714021382.9218504,"name":"offline","context":{"idset":"11806"}}
+{"timestamp":1714021382.9219477,"name":"offline","context":{"idset":"11807"}}
+{"timestamp":1714021382.9220428,"name":"offline","context":{"idset":"11808"}}
+{"timestamp":1714021382.9221382,"name":"offline","context":{"idset":"11809"}}
+{"timestamp":1714021382.9222338,"name":"offline","context":{"idset":"11810"}}
+{"timestamp":1714021382.9223323,"name":"offline","context":{"idset":"11811"}}
+{"timestamp":1714021382.9224265,"name":"offline","context":{"idset":"11812"}}
+{"timestamp":1714021382.922523,"name":"offline","context":{"idset":"11813"}}
+{"timestamp":1714021382.9226196,"name":"offline","context":{"idset":"11814"}}
+{"timestamp":1714021382.9227111,"name":"offline","context":{"idset":"11815"}}
+{"timestamp":1714021382.9228084,"name":"offline","context":{"idset":"11816"}}
+{"timestamp":1714021382.9229007,"name":"offline","context":{"idset":"11817"}}
+{"timestamp":1714021382.9229937,"name":"offline","context":{"idset":"11818"}}
+{"timestamp":1714021382.9230864,"name":"offline","context":{"idset":"11819"}}
+{"timestamp":1714021382.9231758,"name":"offline","context":{"idset":"11820"}}
+{"timestamp":1714021382.9232671,"name":"offline","context":{"idset":"11821"}}
+{"timestamp":1714021382.9233553,"name":"offline","context":{"idset":"11822"}}
+{"timestamp":1714021382.9234467,"name":"offline","context":{"idset":"11823"}}
+{"timestamp":1714021382.9235349,"name":"offline","context":{"idset":"11824"}}
+{"timestamp":1714021382.92362,"name":"offline","context":{"idset":"11827"}}
+{"timestamp":1714021382.9295726,"name":"offline","context":{"idset":"11828"}}
+{"timestamp":1714021382.9296603,"name":"offline","context":{"idset":"11829"}}
+{"timestamp":1714021382.9297433,"name":"offline","context":{"idset":"11830"}}
+{"timestamp":1714021382.929832,"name":"offline","context":{"idset":"11831"}}
+{"timestamp":1714021382.9299145,"name":"offline","context":{"idset":"11832"}}
+{"timestamp":1714021382.9299953,"name":"offline","context":{"idset":"11833"}}
+{"timestamp":1714021382.9300756,"name":"offline","context":{"idset":"11834"}}
+{"timestamp":1714021382.9301541,"name":"offline","context":{"idset":"11835"}}
+{"timestamp":1714021382.930232,"name":"offline","context":{"idset":"11836"}}
+{"timestamp":1714021382.9303088,"name":"offline","context":{"idset":"11837"}}
+{"timestamp":1714021382.9303851,"name":"offline","context":{"idset":"11838"}}
+{"timestamp":1714021382.9304614,"name":"offline","context":{"idset":"11839"}}
+{"timestamp":1714021382.9305372,"name":"offline","context":{"idset":"11840"}}
+{"timestamp":1714021382.9306118,"name":"offline","context":{"idset":"11841"}}
+{"timestamp":1714021382.930686,"name":"offline","context":{"idset":"11842"}}
+{"timestamp":1714021382.9307599,"name":"offline","context":{"idset":"11843"}}
+{"timestamp":1714021382.9308403,"name":"offline","context":{"idset":"11844"}}
+{"timestamp":1714021382.9309146,"name":"offline","context":{"idset":"11845"}}
+{"timestamp":1714021382.9309862,"name":"offline","context":{"idset":"11846"}}
+{"timestamp":1714021382.9310589,"name":"offline","context":{"idset":"11847"}}
+{"timestamp":1714021382.9311302,"name":"offline","context":{"idset":"11848"}}
+{"timestamp":1714021382.9312024,"name":"offline","context":{"idset":"11849"}}
+{"timestamp":1714021382.937043,"name":"offline","context":{"idset":"11850"}}
+{"timestamp":1714021382.9371142,"name":"offline","context":{"idset":"11851"}}
+{"timestamp":1714021382.9371836,"name":"offline","context":{"idset":"11852"}}
+{"timestamp":1714021382.9372518,"name":"offline","context":{"idset":"11853"}}
+{"timestamp":1714021382.9373243,"name":"offline","context":{"idset":"11854"}}
+{"timestamp":1714021382.937391,"name":"offline","context":{"idset":"11855"}}
+{"timestamp":1714021382.9374585,"name":"offline","context":{"idset":"11856"}}
+{"timestamp":1714021382.9375291,"name":"offline","context":{"idset":"11857"}}
+{"timestamp":1714021382.937598,"name":"offline","context":{"idset":"11858"}}
+{"timestamp":1714021382.9376645,"name":"offline","context":{"idset":"11859"}}
+{"timestamp":1714021382.9377296,"name":"offline","context":{"idset":"11860"}}
+{"timestamp":1714021382.9377959,"name":"offline","context":{"idset":"11861"}}
+{"timestamp":1714021382.9378676,"name":"offline","context":{"idset":"11862"}}
+{"timestamp":1714021382.9379311,"name":"offline","context":{"idset":"11863"}}
+{"timestamp":1714021382.9379919,"name":"offline","context":{"idset":"11864"}}
+{"timestamp":1714021382.938055,"name":"offline","context":{"idset":"11865"}}
+{"timestamp":1714021382.9381168,"name":"offline","context":{"idset":"11866"}}
+{"timestamp":1714021382.9381764,"name":"offline","context":{"idset":"11867"}}
+{"timestamp":1714021382.9382348,"name":"offline","context":{"idset":"11868"}}
+{"timestamp":1714021382.9382913,"name":"offline","context":{"idset":"11869"}}
+{"timestamp":1714021382.9383471,"name":"offline","context":{"idset":"11870"}}
+{"timestamp":1714021382.9384024,"name":"offline","context":{"idset":"11871"}}
+{"timestamp":1714021382.938458,"name":"offline","context":{"idset":"11872"}}
+{"timestamp":1714021382.9385138,"name":"offline","context":{"idset":"11873"}}
+{"timestamp":1714021382.9385688,"name":"offline","context":{"idset":"11874"}}
+{"timestamp":1714021382.9386222,"name":"offline","context":{"idset":"11875"}}
+{"timestamp":1714021382.9386752,"name":"offline","context":{"idset":"11876"}}
+{"timestamp":1714021382.9387274,"name":"offline","context":{"idset":"11877"}}
+{"timestamp":1714021382.938781,"name":"offline","context":{"idset":"11878"}}
+{"timestamp":1714021382.9388425,"name":"offline","context":{"idset":"11879"}}
+{"timestamp":1714021382.9388952,"name":"offline","context":{"idset":"11880"}}
+{"timestamp":1714021382.9389474,"name":"offline","context":{"idset":"11881"}}
+{"timestamp":1714021382.9447212,"name":"offline","context":{"idset":"11882"}}
+{"timestamp":1714021382.944773,"name":"offline","context":{"idset":"11883"}}
+{"timestamp":1714021382.9448295,"name":"offline","context":{"idset":"11884"}}
+{"timestamp":1714021382.9448779,"name":"offline","context":{"idset":"11885"}}
+{"timestamp":1714021382.9449258,"name":"offline","context":{"idset":"11886"}}
+{"timestamp":1714021382.944972,"name":"offline","context":{"idset":"11887"}}
+{"timestamp":1714021382.9450173,"name":"offline","context":{"idset":"11888"}}
+{"timestamp":1714021382.9450629,"name":"offline","context":{"idset":"11889"}}
+{"timestamp":1714021382.945107,"name":"offline","context":{"idset":"11890"}}
+{"timestamp":1714021382.9451501,"name":"offline","context":{"idset":"11891"}}
+{"timestamp":1714021382.9451678,"name":"offline","context":{"idset":"11892"}}
+{"timestamp":1714021383.4091454,"name":"drain","context":{"idset":"9365-9380","reason":"--reason back plane fix -KPN","overwrite":0}}
+{"timestamp":1714022896.8780994,"name":"resource-init","context":{"restart":true,"drain":{"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"11377":{"timestamp":1713928672.0,"reason":"nodediag failed amdapu"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"796":{"timestamp":1712868054.0,"reason":"broker was unresponsive"},"372":{"timestamp":1713981581.0,"reason":"Unreachable"},"877-878":{"timestamp":1713539313.0,"reason":""},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"},"795":{"timestamp":1712868053.0,"reason":"broker was unresponsive"},"336,491":{"timestamp":1713230524.0,"reason":"Down"},"90":{"timestamp":1713564981.0,"reason":"Unreachable"},"929-930":{"timestamp":1713552018.0,"reason":""},"773-774":{"timestamp":1713802805.0,"reason":""},"373":{"timestamp":1713981698.0,"reason":"Unreachable"},"77-84":{"timestamp":1713795613.0,"reason":"New blade installation"},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"11826":{"timestamp":1713979977.0,"reason":""},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"117":{"timestamp":1713981538.0,"reason":"Unreachable"},"411":{"timestamp":1713564989.0,"reason":"Unreachable"},"11253-11268":{"timestamp":1713972658.0,"reason":"draining KPN"},"807":{"timestamp":1713376017.0,"reason":""},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"811-812":{"timestamp":1713557753.0,"reason":"--reason Running hpe diags - JM"},"10310":{"timestamp":1713953621.0,"reason":"nodediag failed bogomips"},"10341":{"timestamp":1713928487.0,"reason":"nodediag failed bogomips"},"779-780":{"timestamp":1713907642.0,"reason":"--reason Troubleshoot BIOS issue"},"11725":{"timestamp":1713892052.0,"reason":"nodediag failed amdapu"},"11285-11300":{"timestamp":1713975164.0,"reason":"--reason rebooting switches - INIT -kpn"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"11658":{"timestamp":1713954013.0,"reason":"epilog failed for jobid frLfYxGth6w"},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"828":{"timestamp":1712859028.0,"reason":"broker was unresponsive"},"286":{"timestamp":1713981553.0,"reason":"Unreachable"},"808":{"timestamp":1713455939.0,"reason":""},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"10740":{"timestamp":1714004763.0,"reason":"nodediag failed amdapu"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"11374":{"timestamp":1713926801.0,"reason":"nodediag failed amdapu"},"206":{"timestamp":1713564985.0,"reason":"Unreachable"},"8181,8183,8234,8237,8291,8304":{"timestamp":1713810085.0,"reason":"testing -kk"},"11373":{"timestamp":1713928485.0,"reason":"nodediag failed amdapu"},"9349-9364":{"timestamp":1714021089.0,"reason":"--reason back plane fix -KPN"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"141-148":{"timestamp":1713545597.0,"reason":"ARP Storm Problem testing"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"9075":{"timestamp":1712864130.0,"reason":"broker was unresponsive"},"11370":{"timestamp":1713930154.0,"reason":"nodediag failed amdapu"},"9333-9348":{"timestamp":1714020725.0,"reason":"--reason back plane fix -KPN"},"573-580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"149-156":{"timestamp":1713545634.0,"reason":"ARP Storm Problem testing"},"165":{"timestamp":1713981434.0,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261863.0,"reason":"--reason swapping blades into x1900 - wendy"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"11717-11720,11722-11724,11726,11728,11730,11732":{"timestamp":1713983852.0,"reason":"--force --reason rebooting switches - INIT -kpn"},"172":{"timestamp":1713981544.0,"reason":"Unreachable"},"11324":{"timestamp":1713902339.0,"reason":"nodediag failed amdapu"},"11789":{"timestamp":1713910767.0,"reason":"testing -kk"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"787-788":{"timestamp":1713219302.0,"reason":""},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"10233,10248":{"timestamp":1713746450.0,"reason":"broker was unresponsive"},"7781":{"timestamp":1713884596.0,"reason":"nodediag failed amdapu"},"963-964":{"timestamp":1713196418.0,"reason":"moving blade"},"397":{"timestamp":1713229804.0,"reason":"prolog failed for jobid fpkPrjR8F59"},"937-938":{"timestamp":1712866351.0,"reason":""},"10187":{"timestamp":1713980344.0,"reason":""},"360":{"timestamp":1713981579.0,"reason":"Unreachable"},"925-926":{"timestamp":1713540640.0,"reason":""},"461-476":{"timestamp":1713794797.0,"reason":"New Blade Installation --JRG"},"9365-9380":{"timestamp":1714021383.0,"reason":"--reason back plane fix -KPN"},"9003":{"timestamp":1712873964.0,"reason":"nodediag failed dgemm_perf"},"421-460":{"timestamp":1713794410.0,"reason":"New Blade Installation --JRG"},"410":{"timestamp":1713378635.0,"reason":"prolog failed for jobid fq5jtQQSQnT"},"120":{"timestamp":1713981548.0,"reason":"Unreachable"},"803-804":{"timestamp":1713479733.0,"reason":"status"},"789-794":{"timestamp":1711461777.0,"reason":""},"11729":{"timestamp":1713951714.0,"reason":"nodediag failed amdapu"},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"961-962":{"timestamp":1713197674.0,"reason":""},"827":{"timestamp":1712859030.0,"reason":"broker was unresponsive"},"482-490,492":{"timestamp":1713915346.0,"reason":"New Blade Installation --JRG"},"11369":{"timestamp":1713928726.0,"reason":"nodediag failed amdapu"},"342":{"timestamp":1713981566.0,"reason":"Unreachable"},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"549-556":{"timestamp":1713553592.0,"reason":"New Blade Installation"},"8833":{"timestamp":1713991869.0,"reason":"nodediag failed amdapu"},"11721":{"timestamp":1713951748.0,"reason":"nodediag failed amdapu"},"9054":{"timestamp":1712864116.0,"reason":"broker was unresponsive"},"771-772":{"timestamp":1713195979.0,"reason":""},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"8982":{"timestamp":1713996208.0,"reason":"broker was unresponsive"},"939":{"timestamp":1713963885.0,"reason":"nodediag failed amdapu"},"547":{"timestamp":1713563916.0,"reason":"epilog failed for jobid fqUWmieM3AX"},"814":{"timestamp":1712783332.0,"reason":""},"133-140":{"timestamp":1713545617.0,"reason":"ARP Storm Problem testing"},"829-830":{"timestamp":1712700945.0,"reason":""},"525-532":{"timestamp":1713290620.0,"reason":""},"548":{"timestamp":1713563915.0,"reason":"epilog failed for jobid fqUWkS8j8hM"},"11731":{"timestamp":1713889305.0,"reason":"broker was unresponsive"},"335":{"timestamp":1713981557.0,"reason":"Unreachable"},"934":{"timestamp":1713967835.0,"reason":"epilog failed for jobid frMYzSv7G6b"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1714022896.8870354,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1714022904.5504813,"name":"online","context":{"idset":"0"}}
+{"timestamp":1714022906.3517046,"name":"online","context":{"idset":"7794,7796-7797,7824,7850,7864,7886,7910,7926,7951,7960,7965-7966,7990,8034,8038-8039,8047,8186,8189,8192,8232,8269,8292,8825,8844,8860,8868,8885,8887,8891,8896,8898,8925,8947-8948,9081,9091,9119,9126,9149,9155,9157,9159,9163,9179,9199-9200,9229,9263,9315,9330,9438,9466,9478,9494,9496,9499,9516,9524,9566,9569,9580,9606,9610,9616,9632,9648,9651,9653,9674,9704,9730,9761,9778,9781,9783,9803,9810,9816,9820,9826-9827,9832,10353,10364,10372,10382,10398,10401,10406,10415,10418-10421,10444,10454,10456,10480,10513,10526,10537,10542,10567,10569,10578,10588,10614,10624,10649,10663,10680,10695,10700,10703-10704,10722,10890,10901,10915,10963,10973,10985,10997,11043,11050,11067,11081,11125-11126,11136,11149,11162,11183,11189,11220,11232,11789"}}
+{"timestamp":1714022906.3685946,"name":"online","context":{"idset":"9615,10400,10511,10517"}}
+{"timestamp":1714022906.4898448,"name":"online","context":{"idset":"7787-7789,7791-7792,7799,7801,7809,7813,7825-7826,7833,7841-7842,7845,7854,7857-7859,7861,7869,7871,7878,7881,7883,7888-7889,7891,7895,7897,7901,7904-7907,7912,7914,7918,7922-7924,7928-7929,7934,7936-7937,7939,7947,7952,7957,7959,7973,7975-7977,7979,7985-7987,7989,7993,7995,7998,8002,8006,8010,8013-8014,8020,8024-8025,8027,8032,8036,8040-8041,8043,8181,8183-8184,8199,8202,8205,8207-8208,8210-8211,8213-8214,8217,8228,8240,8257,8261-8263,8276,8278,8280-8281,8299,8305-8306,8308,8830,8832,8836,8845,8848-8849,8852,8855-8857,8861,8865,8869,8871,8874-8875,8877,8884,8888,8890,8892,8895,8897,8899-8901,8904,8923,8929-8930,8933,8938-8940,8944-8945,8965,8969,8971-8972,9077,9084,9088,9095-9096,9100-9101,9103,9106,9114,9118,9124-9125,9127,9129,9138,9140,9142,9150,9154,9156,9158,9165,9167,9171,9173,9185,9187,9190,9193,9197-9198,9203,9207-9208,9212-9213,9215-9216,9222,9232-9233,9238,9241,9244,9249,9251-9252,9254,9257,9259-9261,9264,9267,9273,9281,9283-9285,9287,9293-9294,9299,9301-9302,9305,9310,9313,9316,9319,9321,9327,9331,9430,9432,9439,9442,9444,9446-9455,9460-9461,9465,9473-9474,9480,9485,9490,9493,9500,9502-9503,9505-9506,9508,9513-9514,9521,9523,9527,9529-9531,9534,9537,9545,9549,9554,9557,9560,9572-9575,9579,9583,9586-9588,9595-9596,9598,9601,9608-9609,9621,9623,9637,9641,9650,9656-9658,9665-9666,9668,9673,9686-9687,9692-9695,9697,9703,9705,9709,9711,9713,9719,9724-9725,9729,9731-9732,9738,9740,9745-9748,9751-9755,9762,9765,9772,9774,9777,9784-9785,9789-9790,9792,9796,9799,9802,9805,9808-9809,9812-9813,9819,9823-9824,9830,10233,10248,10346,10359,10363,10377,10383-10384,10386,10388,10397,10399,10404,10414,10433-10435,10437-10439,10441-10442,10445,10448,10452-10453,10455,10457-10458,10469-10470,10473-10477,10481,10484,10488,10490,10496,10501-10502,10506-10507,10512,10514,10519,10523,10528,10534,10539,10541,10550-10551,10553,10559,10562,10577,10579,10581,10584,10587,10600,10602,10604,10607,10612,10617,10621,10628-10629,10633,10635,10637,10641-10642,10644-10647,10651,10655,10661,10665,10672,10679,10689-10691,10697,10701,10705,10708,10711-10713,10715-10716,10719-10720,10723,10725-10726,10869,10871,10873,10875,10877,10881,10884-10885,10889,10891,10893,10898,10904,10906,10920-10922,10924-10926,10928-10930,10933,10936,10939-10940,10950,10953,10955,10957,10960-10961,10965,10970-10971,10974,10978-10980,10982,10984,10986,10990-10991,10998,11005,11010-11011,11017,11023,11026,11031-11032,11034,11036-11039,11045,11047,11059,11062-11065,11071-11075,11078,11082,11084-11085,11087,11095-11096,11099,11103,11105-11106,11109-11110,11112,11114,11116,11124,11127,11129,11131-11132,11134,11138-11139,11147,11151-11152,11155-11157,11165,11167-11168,11171,11174,11177-11178,11180,11182,11188,11192,11197,11200,11202,11207,11215,11219,11221,11231,11233"}}
+{"timestamp":1714022906.6251576,"name":"online","context":{"idset":"883-884,7782,7785-7786,7793,7795,7800,7802-7803,7805-7806,7808,7811,7818,7821,7823,7827-7830,7832,7835-7838,7844,7846-7849,7851-7853,7855,7860,7862,7866,7870,7873-7876,7879-7880,7882,7884-7885,7887,7893-7894,7896,7899-7900,7902-7903,7908,7911,7913,7915-7916,7919-7920,7925,7927,7930-7932,7935,7940-7941,7946,7948-7950,7953-7954,7958,7961,7963-7964,7967-7968,7970-7972,7974,7978,7980-7981,7983-7984,7988,7991-7992,7994,7996-7997,8003-8004,8007-8009,8012,8015,8017-8019,8022-8023,8026,8028-8029,8031,8033,8035,8037,8042,8044,8046,8048-8052,8182,8185,8187-8188,8191,8193-8198,8200,8203-8204,8206,8209,8212,8215-8216,8218-8220,8222-8223,8225,8227,8229,8231,8236,8238-8239,8241-8247,8249,8251-8253,8255-8256,8259-8260,8266-8268,8271-8273,8275,8277,8279,8282,8284-8291,8294-8298,8300-8303,8307,8822,8824,8826-8829,8831,8834-8835,8837-8843,8846-8847,8850-8851,8853-8854,8858-8859,8862-8864,8866-8867,8872,8876,8878,8880-8882,8889,8893,8902-8903,8906,8908-8917,8919,8921-8922,8924,8926,8928,8931-8932,8934-8937,8941-8942,8946,8966-8968,8970,8973,9079-9080,9082-9083,9085-9086,9089-9090,9092-9094,9097-9099,9102,9104-9105,9107,9109-9113,9115-9117,9120-9121,9123,9128,9130,9132,9134-9135,9139,9141,9143-9145,9147-9148,9151-9152,9160,9162,9164,9168-9169,9172,9175-9176,9178,9180-9184,9186,9191-9192,9194,9196,9205-9206,9209-9211,9214,9217-9219,9223-9225,9227-9228,9230-9231,9234-9237,9239-9240,9242-9243,9245,9247-9248,9250,9255-9256,9258,9265-9266,9268-9270,9272,9276-9280,9286,9288-9290,9292,9295-9296,9298,9303-9304,9306-9309,9311-9312,9318,9322-9326,9328-9329,9429,9431,9433-9437,9440-9441,9443,9456-9459,9462-9463,9470-9472,9476,9479,9481-9484,9486-9489,9491-9492,9495,9498,9504,9507,9509-9512,9515,9517-9519,9522,9526,9528,9532,9535-9536,9540-9544,9546-9548,9550-9553,9555-9556,9558-9559,9564-9565,9568,9570,9576-9578,9581-9582,9584-9585,9589,9591-9592,9599-9600,9602-9603,9605,9611,9613-9614,9617,9619-9620,9622,9624-9625,9627,9629,9631,9633,9635-9636,9638-9640,9642,9645,9647,9649,9652,9654-9655,9659-9661,9663,9667,9670,9675-9676,9678-9683,9688-9689,9691,9696,9698-9701,9706,9708,9714-9718,9721-9723,9726,9728,9733,9735-9737,9741,9743-9744,9749-9750,9756-9758,9763-9764,9766-9770,9775,9780,9782,9787-9788,9791,9793-9794,9797-9798,9801,9804,9807,9811,9814-9815,9817-9818,9822,9825,9828-9829,10187,10310,10345,10347,10350-10351,10354,10357-10358,10360-10362,10365-10366,10369-10370,10373,10376,10378-10379,10381,10385,10389-10391,10393-10394,10402,10407-10409,10411,10413,10422,10424-10426,10428-10429,10431,10440,10447,10449-10451,10459-10461,10463-10464,10466-10468,10472,10478-10479,10482-10483,10485-10487,10489,10491-10492,10494-10495,10497,10499,10505,10508,10510,10515,10518,10520-10522,10525,10529,10531-10533,10535,10538,10543-10544,10546,10548-10549,10552,10554-10557,10560-10561,10564,10566,10573-10576,10580,10582-10583,10585,10590,10593-10597,10601,10609-10611,10613,10616,10618-10620,10622,10625-10627,10631,10638-10640,10643,10648,10652-10653,10656,10658,10660,10662,10664,10666-10671,10673-10675,10677-10678,10681-10687,10693-10694,10696,10699,10702,10706-10707,10710,10717,10721,10724,10727,10740,10876,10882-10883,10886-10888,10892,10894-10897,10899-10900,10902-10903,10905,10908-10909,10913-10914,10917-10918,10923,10931-10932,10934-10935,10937-10938,10941-10949,10951,10954,10958,10968,10972,10975-10977,10981,10987-10989,10994-10996,11000,11002-11004,11006-11007,11009,11013-11014,11016,11019-11022,11024-11025,11027,11029-11030,11033,11040,11042,11044,11048-11049,11051-11054,11056,11058,11060,11070,11076-11077,11079-11080,11083,11086,11088-11094,11097-11098,11100-11102,11107-11108,11111,11113,11115,11117-11119,11121-11123,11128,11133,11135,11137,11140-11141,11144-11146,11148,11153-11154,11158-11160,11164,11169-11170,11172,11175,11179,11181,11184-11187,11190-11191,11193-11196,11198-11199,11201,11203-11205,11209,11211-11214,11216-11218,11222-11230,11234-11237,11239-11240"}}
+{"timestamp":1714022906.7395637,"name":"online","context":{"idset":"7781,7783,7790,7798,7804,7807,7810,7814,7820,7822,7831,7834,7840,7843,7863,7865,7867-7868,7872,7877,7890,7892,7917,7921,7938,7943,7955-7956,7969,7982,7999,8001,8005,8011,8016,8030,8190,8224,8226,8230,8235,8248,8250,8254,8264,8270,8821,8823,8870,8883,8886,8905,8907,8920,8927,8943,8957-8958,8961,8974-8976,8978,8980,9046,9049,9051-9052,9078,9087,9131,9146,9153,9166,9170,9174,9188-9189,9201-9202,9204,9221,9226,9271,9274,9282,9291,9297,9300,9314,9317,9320,9332,9464,9467,9469,9477,9497,9520,9525,9533,9538-9539,9562,9571,9590,9594,9597,9604,9607,9612,9628,9630,9634,9644,9662,9664,9669,9671,9677,9684-9685,9690,9702,9710,9712,9720,9727,9734,9742,9759-9760,9771,9773,9776,9800,9806,9821,10349,10368,10371,10374-10375,10380,10387,10392,10396,10403,10405,10410,10412,10417,10423,10427,10430,10443,10471,10500,10503-10504,10509,10524,10527,10530,10545,10547,10558,10563,10565,10570-10571,10592,10598-10599,10603,10606,10608,10615,10630,10632,10634,10636,10654,10657,10659,10688,10692,10698,10709,10870,10911-10912,10927,10952,10956,10962,10966-10967,10969,10983,10992-10993,10999,11015,11018,11028,11041,11046,11055,11057,11061,11066,11068,11120,11142,11150,11161,11163,11166,11173,11176,11206,11210,11238"}}
+{"timestamp":1714022906.9231997,"name":"online","context":{"idset":"85-87,89,91-92,94-96,98,100,103-108,110,112-113,115-116,118-119,123-126,128-132,157-162,164,167-168,170-171,173-174,177-179,182,186-190,193,198,200-204,207-211,213-216,218-219,221-224,226-227,229-242,244-248,250-251,278,280-281,283-285,287-288,290-291,293-295,297-298,300-301,303-304,308-312,315,317,319-332,334,337-341,343-347,350,352-353,355-356,359,361,363,367-369,371-372,374,377,379,381-382,384-385,387-388,390,394-395,398-399,401-409,412-415,417-420,7933,8265,8959-8960,8962-8964,9047-9048,9050"}}
+{"timestamp":1714022907.0806234,"name":"online","context":{"idset":"88,93,97,99,101-102,109,114,122,127,166,175-176,180-181,184-185,191-192,195-196,199,205,212,220,225,228,249,277,282,289,292,318,349,351,354,358,362,364-366,370,375-376,378,380,386,392-393,396,400"}}
+{"timestamp":1714022907.2438617,"name":"online","context":{"idset":"7693,9937-9938,9993,10026,10029"}}
+{"timestamp":1714022907.400212,"name":"online","context":{"idset":"4,41,48,58,60,7724,7733,8953,9939,9953,9960,9962,10006"}}
+{"timestamp":1714022907.5477767,"name":"online","context":{"idset":"43-45,47,49,7674,7688,7697,7711,7729-7730,7732,7738,7756,7770,7775,8949,8952,8955-8956,9934-9936,9940,9958,9971,9994,10011,10034"}}
+{"timestamp":1714022907.6976633,"name":"online","context":{"idset":"3,5,7-12,14-16,18,24,27,29,32,35-36,40,42,50,52,56-57,573,7679,7685,7687,7689,7695,7710,7717,7719-7720,7726-7727,7731,7746,7749,7755,7759,7776,8951,8954,9950,9961,9967,9989,9997,10022"}}
+{"timestamp":1714022907.799058,"name":"online","context":{"idset":"1-2,6,13,17,19-23,25-26,28,30-31,33-34,37-39,46,51,53-55,59,7670,7683,7690,7692,7703,7708,7714,7716,7718,7723,7735,7737,7745,7751-7752,7757-7758,7763,7779,9943,9966,9969-9970,9987-9988,9996,10002-10003,10008-10009,10019-10020,10030"}}
+{"timestamp":1714022907.9022777,"name":"online","context":{"idset":"7669,7672-7673,7682,7684,7694,7698,7701,7705-7706,7713,7721,7747-7748,7754,7761,7764-7767,7771-7772,7778,7780,9947,9957,9959,9975,9977,9983-9984,9991,9998,10000,10007,10021,10023,10028"}}
+{"timestamp":1714022908.0061829,"name":"online","context":{"idset":"7675,7677,7680-7681,7686,7696,7700,7702,7704,7707,7739-7742,7744,7750,7768,7773-7774,7777,9942,9944-9946,9951,9956,9973-9974,9976,9979,9981-9982,9999,10005,10017-10018,10027,10032"}}
+{"timestamp":1714022908.1654184,"name":"online","context":{"idset":"7671,7676,7678,7709,7712,7715,7734,7736,7743,7760,7762,9941,9949,9952,9954-9955,9963-9964,9972,9978,9980,9986,9992,9995,10004,10010,10013-10014,10024-10025,10031,10033,10035"}}
+{"timestamp":1714022908.2685008,"name":"online","context":{"idset":"7769,9031,9034,9948,9985,9990,10012,10015"}}
+{"timestamp":1714022908.3704731,"name":"online","context":{"idset":"8991,9014,9020,9032-9033,9035"}}
+{"timestamp":1714022908.5198824,"name":"online","context":{"idset":"8992-8993,8995-8996,9013,9015-9019,9021-9029,9036-9042,9044,10016"}}
+{"timestamp":1714022908.6680365,"name":"online","context":{"idset":"8990,8994,8999-9000"}}
+{"timestamp":1714022908.8129163,"name":"online","context":{"idset":"575,8981,8984,8986-8987,8998,9001,9004-9010,9012"}}
+{"timestamp":1714022908.9178836,"name":"online","context":{"idset":"576,8982-8983,8985,8988,8997,9002,9011"}}
+{"timestamp":1714022910.403239,"name":"online","context":{"idset":"9057"}}
+{"timestamp":1714022910.6638722,"name":"online","context":{"idset":"579,9053,9056,9058-9060"}}
+{"timestamp":1714022913.2579129,"name":"online","context":{"idset":"7839,10493,10714"}}
+{"timestamp":1714022913.4254367,"name":"online","context":{"idset":"7856,7962,8274,8293,8873,8894,8918,9133,9136,9275,9468,9779,10355,10465,10498,10586,10589,10623,10650,10916,10964,11012,11658"}}
+{"timestamp":1714022913.5270391,"name":"online","context":{"idset":"7898,7944,8258,9122,9177,9195,9246,9561,9567,9646,9672,10416,10432,10572,10591,10605,10676,10728,10907,10919,11008,11130"}}
+{"timestamp":1714022913.6290994,"name":"online","context":{"idset":"7942,8000,8979,9137,9563,9618,9643,9739,9786,10446,10516,10910,11001,11035,11143"}}
+{"timestamp":1714022913.7315211,"name":"online","context":{"idset":"7753,7812,7817,7909,8021,9030,9045,9055,9108,9220,9253,9445,10462,10540,10874,11208"}}
+{"timestamp":1714022913.8819897,"name":"online","context":{"idset":"302,305-306,577,7725,7728,7784,7819,8221,8283,8879,8950,8977,8989,9161,9262,9475,9593,9626,9707,9795,9831,9965,9968,10341,10367,10395,10436,10536,10568,10872,10959,11069,11104"}}
+{"timestamp":1714022914.0268931,"name":"online","context":{"idset":"111,163,169,243,252,296,299,333,357,383,391,416,7691,7945,8045,10001,10718,10878"}}
+{"timestamp":1714022914.170718,"name":"online","context":{"idset":"194,197,307,389"}}
+{"timestamp":1714022914.3134573,"name":"online","context":{"idset":"183,316,7699,7722,10036"}}
+{"timestamp":1714023084.3194993,"name":"online","context":{"idset":"574"}}
+{"timestamp":1714024391.8303885,"name":"online","context":{"idset":"580"}}
+{"timestamp":1714024559.4516501,"name":"offline","context":{"idset":"580"}}
+{"timestamp":1714052577.4518507,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1714053160.8742225,"name":"drain","context":{"idset":"9429-9444","reason":"--reason down for backplain work -kpn","overwrite":0}}
+{"timestamp":1714053178.2916124,"name":"offline","context":{"idset":"9430"}}
+{"timestamp":1714053178.294661,"name":"offline","context":{"idset":"9429"}}
+{"timestamp":1714053178.3654678,"name":"offline","context":{"idset":"9437"}}
+{"timestamp":1714053178.3682876,"name":"offline","context":{"idset":"9433"}}
+{"timestamp":1714053178.4277611,"name":"offline","context":{"idset":"9441"}}
+{"timestamp":1714053178.4325366,"name":"offline","context":{"idset":"9444"}}
+{"timestamp":1714053178.4360611,"name":"offline","context":{"idset":"9443"}}
+{"timestamp":1714053178.4399185,"name":"offline","context":{"idset":"9439"}}
+{"timestamp":1714053178.4496081,"name":"offline","context":{"idset":"9432"}}
+{"timestamp":1714053178.5336673,"name":"offline","context":{"idset":"9440"}}
+{"timestamp":1714053178.5366688,"name":"offline","context":{"idset":"9435"}}
+{"timestamp":1714053178.561445,"name":"offline","context":{"idset":"9434"}}
+{"timestamp":1714053178.5643246,"name":"offline","context":{"idset":"9431"}}
+{"timestamp":1714053178.5671952,"name":"offline","context":{"idset":"9436"}}
+{"timestamp":1714053178.6408651,"name":"offline","context":{"idset":"9442"}}
+{"timestamp":1714053178.7908208,"name":"offline","context":{"idset":"9438"}}
+{"timestamp":1714053591.7314105,"name":"drain","context":{"idset":"9445-9460","reason":"--reason down for backplain work -kpn","overwrite":0}}
+{"timestamp":1714053789.362848,"name":"offline","context":{"idset":"9445"}}
+{"timestamp":1714053789.4522598,"name":"offline","context":{"idset":"9446"}}
+{"timestamp":1714053791.3621304,"name":"offline","context":{"idset":"9447"}}
+{"timestamp":1714053791.3654397,"name":"offline","context":{"idset":"9448"}}
+{"timestamp":1714053791.4505446,"name":"offline","context":{"idset":"9449"}}
+{"timestamp":1714053793.3687687,"name":"offline","context":{"idset":"9450"}}
+{"timestamp":1714053793.3725016,"name":"offline","context":{"idset":"9451"}}
+{"timestamp":1714053793.3764429,"name":"offline","context":{"idset":"9452"}}
+{"timestamp":1714053793.3791428,"name":"offline","context":{"idset":"9453"}}
+{"timestamp":1714053793.4530919,"name":"offline","context":{"idset":"9454"}}
+{"timestamp":1714053795.3673317,"name":"offline","context":{"idset":"9455"}}
+{"timestamp":1714053795.3711658,"name":"offline","context":{"idset":"9456"}}
+{"timestamp":1714053795.3754759,"name":"offline","context":{"idset":"9457"}}
+{"timestamp":1714053795.4546058,"name":"offline","context":{"idset":"9458"}}
+{"timestamp":1714053797.3622959,"name":"offline","context":{"idset":"9459"}}
+{"timestamp":1714053797.452342,"name":"offline","context":{"idset":"9460"}}
+{"timestamp":1714054119.4510686,"name":"offline","context":{"idset":"9044"}}
+{"timestamp":1714055285.4519277,"name":"offline","context":{"idset":"11789"}}
+{"timestamp":1714055842.2635415,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1714056452.0598805,"name":"undrain","context":{"idset":"10740"}}
+{"timestamp":1714056714.1715112,"name":"offline","context":{"idset":"8999"}}
+{"timestamp":1714056714.2663321,"name":"offline","context":{"idset":"9000"}}
+{"timestamp":1714056814.2625792,"name":"offline","context":{"idset":"9004"}}
+{"timestamp":1714057816.7075603,"name":"drain","context":{"idset":"871-874","overwrite":0}}
+{"timestamp":1714057837.0625563,"name":"drain","context":{"idset":"871-874","overwrite":0}}
+{"timestamp":1714057945.1060159,"name":"online","context":{"idset":"9054"}}
+{"timestamp":1714057995.7579441,"name":"online","context":{"idset":"9075"}}
+{"timestamp":1714058097.6502178,"name":"online","context":{"idset":"874"}}
+{"timestamp":1714058173.9245882,"name":"online","context":{"idset":"873"}}
+{"timestamp":1714058203.9910476,"name":"online","context":{"idset":"8999"}}
+{"timestamp":1714058205.082931,"name":"online","context":{"idset":"9000"}}
+{"timestamp":1714058211.1099019,"name":"online","context":{"idset":"872"}}
+{"timestamp":1714058232.0423753,"name":"online","context":{"idset":"871"}}
+{"timestamp":1714058370.8892736,"name":"online","context":{"idset":"9072"}}
+{"timestamp":1714058371.3415489,"name":"online","context":{"idset":"9070"}}
+{"timestamp":1714058371.5247447,"name":"online","context":{"idset":"9071,9076"}}
+{"timestamp":1714058371.674598,"name":"online","context":{"idset":"9074"}}
+{"timestamp":1714058371.8228266,"name":"online","context":{"idset":"9069"}}
+{"timestamp":1714058371.9716306,"name":"online","context":{"idset":"9073"}}
+{"timestamp":1714058480.070127,"name":"undrain","context":{"idset":"877-878"}}
+{"timestamp":1714058519.2504499,"name":"online","context":{"idset":"877"}}
+{"timestamp":1714058531.0150826,"name":"online","context":{"idset":"878"}}
+{"timestamp":1714058659.64448,"name":"online","context":{"idset":"9065"}}
+{"timestamp":1714058661.0984063,"name":"online","context":{"idset":"9061,9068"}}
+{"timestamp":1714058661.2718406,"name":"online","context":{"idset":"9064"}}
+{"timestamp":1714058661.5651979,"name":"online","context":{"idset":"9062,9066"}}
+{"timestamp":1714058662.8287609,"name":"online","context":{"idset":"9067"}}
+{"timestamp":1714058662.830796,"name":"online","context":{"idset":"9063"}}
+{"timestamp":1714059197.3750143,"name":"undrain","context":{"idset":"7781"}}
+{"timestamp":1714059409.2158391,"name":"online","context":{"idset":"8318"}}
+{"timestamp":1714059410.0249021,"name":"online","context":{"idset":"8316"}}
+{"timestamp":1714059410.7932982,"name":"online","context":{"idset":"8313"}}
+{"timestamp":1714059410.7955248,"name":"online","context":{"idset":"8326"}}
+{"timestamp":1714059410.8232436,"name":"online","context":{"idset":"8343"}}
+{"timestamp":1714059411.447304,"name":"online","context":{"idset":"8371"}}
+{"timestamp":1714059412.9673848,"name":"online","context":{"idset":"8370"}}
+{"timestamp":1714059412.9695535,"name":"online","context":{"idset":"8334"}}
+{"timestamp":1714059413.0591881,"name":"online","context":{"idset":"8315"}}
+{"timestamp":1714059413.5817108,"name":"online","context":{"idset":"8328"}}
+{"timestamp":1714059413.7831373,"name":"online","context":{"idset":"8336,8375,8388"}}
+{"timestamp":1714059414.0181723,"name":"online","context":{"idset":"8325"}}
+{"timestamp":1714059414.2235525,"name":"online","context":{"idset":"8314"}}
+{"timestamp":1714059414.4008145,"name":"online","context":{"idset":"8331,8352"}}
+{"timestamp":1714059414.7683554,"name":"online","context":{"idset":"8361"}}
+{"timestamp":1714059414.845325,"name":"online","context":{"idset":"8384"}}
+{"timestamp":1714059414.925926,"name":"online","context":{"idset":"8310,8359"}}
+{"timestamp":1714059415.2967691,"name":"online","context":{"idset":"8365"}}
+{"timestamp":1714059415.3701732,"name":"online","context":{"idset":"8432"}}
+{"timestamp":1714059415.9266877,"name":"online","context":{"idset":"8333"}}
+{"timestamp":1714059416.0901191,"name":"online","context":{"idset":"8358"}}
+{"timestamp":1714059416.2144983,"name":"online","context":{"idset":"8329,8405"}}
+{"timestamp":1714059416.2160141,"name":"online","context":{"idset":"8356"}}
+{"timestamp":1714059416.2875857,"name":"online","context":{"idset":"8317"}}
+{"timestamp":1714059416.3342309,"name":"online","context":{"idset":"8345,8404"}}
+{"timestamp":1714059416.3749177,"name":"online","context":{"idset":"8320"}}
+{"timestamp":1714059416.4370515,"name":"online","context":{"idset":"8360"}}
+{"timestamp":1714059416.5748894,"name":"online","context":{"idset":"8353,8373,8414,8425"}}
+{"timestamp":1714059416.7211425,"name":"online","context":{"idset":"8332,8346-8347,8354,8369,8379"}}
+{"timestamp":1714059416.7880719,"name":"online","context":{"idset":"8362,8400"}}
+{"timestamp":1714059416.8474767,"name":"online","context":{"idset":"8393"}}
+{"timestamp":1714059416.9935102,"name":"online","context":{"idset":"8322,8327,8376,8407,8411,8416,8423"}}
+{"timestamp":1714059417.1742644,"name":"online","context":{"idset":"8311-8312,8381,8385,8431"}}
+{"timestamp":1714059417.3313818,"name":"online","context":{"idset":"8309,8323,8341,8344,8363,8367,8383,8409,8413,8433"}}
+{"timestamp":1714059417.4919002,"name":"online","context":{"idset":"8319,8339,8351,8357,8391,8422,8434"}}
+{"timestamp":1714059417.6487422,"name":"online","context":{"idset":"8377,8389,8392,8421"}}
+{"timestamp":1714059417.8030205,"name":"online","context":{"idset":"8335,8337,8372,8398,8427"}}
+{"timestamp":1714059417.957665,"name":"online","context":{"idset":"8321,8330,8338,8348-8349,8355,8364,8390,8419-8420"}}
+{"timestamp":1714059418.1160035,"name":"online","context":{"idset":"8342,8374,8378,8394,8399,8401,8417-8418,8429"}}
+{"timestamp":1714059418.3103144,"name":"online","context":{"idset":"8382,8395,8403,8410,8415,8430"}}
+{"timestamp":1714059418.4127614,"name":"online","context":{"idset":"8340,8386,8397,8402"}}
+{"timestamp":1714059418.6074729,"name":"online","context":{"idset":"8324,8350,8368,8380,8406,8412,8424,8426,8428,8435-8436"}}
+{"timestamp":1714059418.7679148,"name":"online","context":{"idset":"8396"}}
+{"timestamp":1714059419.0811603,"name":"online","context":{"idset":"8387,8408"}}
+{"timestamp":1714059419.2353411,"name":"online","context":{"idset":"8366"}}
+{"timestamp":1714059458.6320345,"name":"drain","context":{"idset":"8309-8436","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714060395.4332845,"name":"drain","context":{"idset":"349-404","reason":"New Blade Installation --JRG","overwrite":1}}
+{"timestamp":1714060424.9669254,"name":"offline","context":{"idset":"351"}}
+{"timestamp":1714060425.0521464,"name":"offline","context":{"idset":"352"}}
+{"timestamp":1714060425.0623491,"name":"offline","context":{"idset":"358"}}
+{"timestamp":1714060425.071198,"name":"offline","context":{"idset":"368"}}
+{"timestamp":1714060425.0758598,"name":"offline","context":{"idset":"355"}}
+{"timestamp":1714060425.0812726,"name":"offline","context":{"idset":"349"}}
+{"timestamp":1714060425.0862327,"name":"offline","context":{"idset":"350"}}
+{"timestamp":1714060425.1379294,"name":"offline","context":{"idset":"353"}}
+{"timestamp":1714060425.1430528,"name":"offline","context":{"idset":"354"}}
+{"timestamp":1714060425.1643717,"name":"offline","context":{"idset":"356"}}
+{"timestamp":1714060425.1688125,"name":"offline","context":{"idset":"357"}}
+{"timestamp":1714060425.1737018,"name":"offline","context":{"idset":"361"}}
+{"timestamp":1714060425.1788595,"name":"offline","context":{"idset":"363"}}
+{"timestamp":1714060425.1838133,"name":"offline","context":{"idset":"364"}}
+{"timestamp":1714060425.1889133,"name":"offline","context":{"idset":"366"}}
+{"timestamp":1714060425.1940913,"name":"offline","context":{"idset":"367"}}
+{"timestamp":1714060425.2170906,"name":"offline","context":{"idset":"369"}}
+{"timestamp":1714060425.2218955,"name":"offline","context":{"idset":"370"}}
+{"timestamp":1714060425.2268412,"name":"offline","context":{"idset":"372"}}
+{"timestamp":1714060425.2315493,"name":"offline","context":{"idset":"375"}}
+{"timestamp":1714060425.2366867,"name":"offline","context":{"idset":"376"}}
+{"timestamp":1714060425.2418108,"name":"offline","context":{"idset":"382"}}
+{"timestamp":1714060425.246711,"name":"offline","context":{"idset":"359"}}
+{"timestamp":1714060425.4812951,"name":"offline","context":{"idset":"362"}}
+{"timestamp":1714060425.5907197,"name":"offline","context":{"idset":"383"}}
+{"timestamp":1714060425.5956876,"name":"offline","context":{"idset":"384"}}
+{"timestamp":1714060425.6007726,"name":"offline","context":{"idset":"385"}}
+{"timestamp":1714060425.6057885,"name":"offline","context":{"idset":"386"}}
+{"timestamp":1714060425.6108351,"name":"offline","context":{"idset":"371"}}
+{"timestamp":1714060425.6159852,"name":"offline","context":{"idset":"387"}}
+{"timestamp":1714060425.620446,"name":"offline","context":{"idset":"388"}}
+{"timestamp":1714060425.6252813,"name":"offline","context":{"idset":"389"}}
+{"timestamp":1714060425.6300447,"name":"offline","context":{"idset":"390"}}
+{"timestamp":1714060425.6333084,"name":"offline","context":{"idset":"394"}}
+{"timestamp":1714060425.6372256,"name":"offline","context":{"idset":"395"}}
+{"timestamp":1714060425.6412272,"name":"offline","context":{"idset":"396"}}
+{"timestamp":1714060425.6455507,"name":"offline","context":{"idset":"398"}}
+{"timestamp":1714060425.649703,"name":"offline","context":{"idset":"399"}}
+{"timestamp":1714060425.6694436,"name":"offline","context":{"idset":"400"}}
+{"timestamp":1714060425.6743939,"name":"offline","context":{"idset":"365"}}
+{"timestamp":1714060425.6793449,"name":"offline","context":{"idset":"374"}}
+{"timestamp":1714060425.6834748,"name":"offline","context":{"idset":"377"}}
+{"timestamp":1714060425.6883459,"name":"offline","context":{"idset":"378"}}
+{"timestamp":1714060425.692735,"name":"offline","context":{"idset":"379"}}
+{"timestamp":1714060425.6968982,"name":"offline","context":{"idset":"380"}}
+{"timestamp":1714060425.7012751,"name":"offline","context":{"idset":"381"}}
+{"timestamp":1714060425.7055717,"name":"offline","context":{"idset":"391"}}
+{"timestamp":1714060425.7100937,"name":"offline","context":{"idset":"392"}}
+{"timestamp":1714060425.7150586,"name":"offline","context":{"idset":"393"}}
+{"timestamp":1714060425.7203314,"name":"offline","context":{"idset":"401"}}
+{"timestamp":1714060425.7252355,"name":"offline","context":{"idset":"402"}}
+{"timestamp":1714060425.7302663,"name":"offline","context":{"idset":"403"}}
+{"timestamp":1714060425.7353814,"name":"offline","context":{"idset":"404"}}
+{"timestamp":1714060912.2634273,"name":"offline","context":{"idset":"8309"}}
+{"timestamp":1714060914.2634432,"name":"offline","context":{"idset":"8341"}}
+{"timestamp":1714061244.7841172,"name":"online","context":{"idset":"10731-10733,10738"}}
+{"timestamp":1714061244.9770384,"name":"online","context":{"idset":"10739"}}
+{"timestamp":1714061245.136055,"name":"online","context":{"idset":"10736-10737"}}
+{"timestamp":1714061245.3459268,"name":"online","context":{"idset":"10735"}}
+{"timestamp":1714061245.5118608,"name":"online","context":{"idset":"10730,10734"}}
+{"timestamp":1714061245.6127553,"name":"online","context":{"idset":"10729"}}
+{"timestamp":1714062752.0161371,"name":"online","context":{"idset":"7816"}}
+{"timestamp":1714062752.3079762,"name":"online","context":{"idset":"7815"}}
+{"timestamp":1714063529.5802894,"name":"online","context":{"idset":"8608"}}
+{"timestamp":1714063529.5829723,"name":"online","context":{"idset":"8612"}}
+{"timestamp":1714063529.5857511,"name":"online","context":{"idset":"8597,8599,8604"}}
+{"timestamp":1714063529.588604,"name":"online","context":{"idset":"8598,8601"}}
+{"timestamp":1714063529.5912967,"name":"online","context":{"idset":"8600,8607,8611"}}
+{"timestamp":1714063529.5940096,"name":"online","context":{"idset":"8602-8603,8605-8606"}}
+{"timestamp":1714063529.596678,"name":"online","context":{"idset":"8609-8610"}}
+{"timestamp":1714063953.7524991,"name":"drain","context":{"idset":"11509-11636","overwrite":0}}
+{"timestamp":1714064152.5005522,"name":"online","context":{"idset":"542"}}
+{"timestamp":1714064152.6629276,"name":"online","context":{"idset":"544"}}
+{"timestamp":1714064152.8176928,"name":"online","context":{"idset":"541,545,547"}}
+{"timestamp":1714064153.0122528,"name":"online","context":{"idset":"546"}}
+{"timestamp":1714064153.2122276,"name":"online","context":{"idset":"543"}}
+{"timestamp":1714064486.0294783,"name":"online","context":{"idset":"827"}}
+{"timestamp":1714064540.5719903,"name":"drain","context":{"idset":"923-924","reason":"status","overwrite":0}}
+{"timestamp":1714064604.970964,"name":"online","context":{"idset":"9502"}}
+{"timestamp":1714064631.7892992,"name":"online","context":{"idset":"9501"}}
+{"timestamp":1714064641.9106963,"name":"drain","context":{"idset":"8597-8612","overwrite":0}}
+{"timestamp":1714064666.0914893,"name":"offline","context":{"idset":"8611"}}
+{"timestamp":1714064666.1899574,"name":"offline","context":{"idset":"8598"}}
+{"timestamp":1714064666.4375153,"name":"offline","context":{"idset":"8600"}}
+{"timestamp":1714064666.5054438,"name":"offline","context":{"idset":"8612"}}
+{"timestamp":1714064666.5754845,"name":"offline","context":{"idset":"8604"}}
+{"timestamp":1714064666.5841517,"name":"offline","context":{"idset":"8599"}}
+{"timestamp":1714064666.5874093,"name":"offline","context":{"idset":"8607"}}
+{"timestamp":1714064666.6850536,"name":"offline","context":{"idset":"8608"}}
+{"timestamp":1714064666.761662,"name":"offline","context":{"idset":"8605"}}
+{"timestamp":1714064666.7640417,"name":"offline","context":{"idset":"8610"}}
+{"timestamp":1714064666.7882602,"name":"offline","context":{"idset":"8603"}}
+{"timestamp":1714064666.791872,"name":"offline","context":{"idset":"8602"}}
+{"timestamp":1714064666.8074203,"name":"offline","context":{"idset":"8606"}}
+{"timestamp":1714064666.8719645,"name":"offline","context":{"idset":"8609"}}
+{"timestamp":1714064666.9645405,"name":"offline","context":{"idset":"8597"}}
+{"timestamp":1714064667.0647988,"name":"offline","context":{"idset":"8601"}}
+{"timestamp":1714064717.9596148,"name":"online","context":{"idset":"808"}}
+{"timestamp":1714064842.8236458,"name":"online","context":{"idset":"828"}}
+{"timestamp":1714064867.3331964,"name":"undrain","context":{"idset":"8309-8436"}}
+{"timestamp":1714064976.8753943,"name":"undrain","context":{"idset":"827-828"}}
+{"timestamp":1714065128.2403569,"name":"online","context":{"idset":"10104,10116"}}
+{"timestamp":1714065128.4044678,"name":"online","context":{"idset":"10111,10113,10118"}}
+{"timestamp":1714065128.5623367,"name":"online","context":{"idset":"10101,10120"}}
+{"timestamp":1714065128.6662464,"name":"online","context":{"idset":"10117,10122,10131"}}
+{"timestamp":1714065129.3573217,"name":"online","context":{"idset":"10103,10108,10112,10114,10119,10123-10129,10132"}}
+{"timestamp":1714065129.6419954,"name":"online","context":{"idset":"10102,10105-10107,10109-10110,10115,10121,10130"}}
+{"timestamp":1714065133.7699065,"name":"online","context":{"idset":"10133"}}
+{"timestamp":1714065134.001538,"name":"online","context":{"idset":"10134,10138"}}
+{"timestamp":1714065134.1684384,"name":"online","context":{"idset":"10151"}}
+{"timestamp":1714065134.4800603,"name":"online","context":{"idset":"10140,10152"}}
+{"timestamp":1714065135.1091738,"name":"online","context":{"idset":"10144"}}
+{"timestamp":1714065135.212003,"name":"online","context":{"idset":"10136-10137,10143,10145,10147,10149-10150"}}
+{"timestamp":1714065135.5464485,"name":"online","context":{"idset":"10139,10146,10148,10153-10157,10159,10163"}}
+{"timestamp":1714065135.7023356,"name":"online","context":{"idset":"10135,10141-10142,10160,10162"}}
+{"timestamp":1714065137.3964331,"name":"online","context":{"idset":"10164"}}
+{"timestamp":1714065137.6727965,"name":"online","context":{"idset":"10158,10161"}}
+{"timestamp":1714065140.6483951,"name":"online","context":{"idset":"10170"}}
+{"timestamp":1714065142.6841884,"name":"online","context":{"idset":"10166,10180"}}
+{"timestamp":1714065142.8392012,"name":"online","context":{"idset":"10190-10191,10195-10196"}}
+{"timestamp":1714065143.4831748,"name":"online","context":{"idset":"10171"}}
+{"timestamp":1714065143.5870063,"name":"online","context":{"idset":"10181,10193"}}
+{"timestamp":1714065143.8765128,"name":"online","context":{"idset":"10165,10167,10174,10184-10186,10192,10194"}}
+{"timestamp":1714065144.0519288,"name":"online","context":{"idset":"10172-10173,10182-10183,10189"}}
+{"timestamp":1714065144.2091403,"name":"online","context":{"idset":"10175,10177,10179"}}
+{"timestamp":1714065144.3802247,"name":"online","context":{"idset":"10168-10169,10176,10178"}}
+{"timestamp":1714065149.1137166,"name":"online","context":{"idset":"10198"}}
+{"timestamp":1714065149.2807581,"name":"online","context":{"idset":"10217"}}
+{"timestamp":1714065149.4507191,"name":"online","context":{"idset":"10204,10209,10214"}}
+{"timestamp":1714065149.6265833,"name":"online","context":{"idset":"10200,10202,10205,10221"}}
+{"timestamp":1714065149.8185761,"name":"online","context":{"idset":"10213"}}
+{"timestamp":1714065149.9947181,"name":"online","context":{"idset":"10211-10212,10215,10218,10220,10223,10225"}}
+{"timestamp":1714065150.1826503,"name":"online","context":{"idset":"10199,10206,10210,10222,10224,10226"}}
+{"timestamp":1714065150.3754191,"name":"online","context":{"idset":"10197,10201,10207-10208,10216"}}
+{"timestamp":1714065150.5712752,"name":"online","context":{"idset":"10219"}}
+{"timestamp":1714065150.7518609,"name":"online","context":{"idset":"10203"}}
+{"timestamp":1714065151.1043599,"name":"online","context":{"idset":"10228"}}
+{"timestamp":1714065151.4664922,"name":"online","context":{"idset":"10227"}}
+{"timestamp":1714065156.9076328,"name":"online","context":{"idset":"10751,10758,10760"}}
+{"timestamp":1714065157.0176718,"name":"online","context":{"idset":"10749"}}
+{"timestamp":1714065157.2361746,"name":"online","context":{"idset":"10742,10746,10757,10761-10763,10766"}}
+{"timestamp":1714065157.43662,"name":"online","context":{"idset":"10743"}}
+{"timestamp":1714065157.6081932,"name":"online","context":{"idset":"10750,10754,10767,10769"}}
+{"timestamp":1714065157.7859533,"name":"online","context":{"idset":"10753,10755"}}
+{"timestamp":1714065157.943872,"name":"online","context":{"idset":"10748,10759,10764,10770"}}
+{"timestamp":1714065158.1256993,"name":"online","context":{"idset":"10752"}}
+{"timestamp":1714065158.3194532,"name":"online","context":{"idset":"10741,10745,10747,10756,10765,10768"}}
+{"timestamp":1714065158.4225018,"name":"online","context":{"idset":"10744,10771"}}
+{"timestamp":1714065158.6216559,"name":"online","context":{"idset":"10772"}}
+{"timestamp":1714065163.9839635,"name":"online","context":{"idset":"10773-10774"}}
+{"timestamp":1714065164.1573751,"name":"online","context":{"idset":"10782"}}
+{"timestamp":1714065164.5733638,"name":"online","context":{"idset":"10775,10781"}}
+{"timestamp":1714065164.8045826,"name":"online","context":{"idset":"10776,10790,10794"}}
+{"timestamp":1714065164.9936163,"name":"online","context":{"idset":"10779"}}
+{"timestamp":1714065165.1772177,"name":"online","context":{"idset":"10795"}}
+{"timestamp":1714065165.3366964,"name":"online","context":{"idset":"10777-10778,10787,10793"}}
+{"timestamp":1714065165.4933703,"name":"online","context":{"idset":"10780,10783"}}
+{"timestamp":1714065165.5981612,"name":"online","context":{"idset":"10785-10786,10799,10802"}}
+{"timestamp":1714065165.7517462,"name":"online","context":{"idset":"10788-10789"}}
+{"timestamp":1714065165.915221,"name":"online","context":{"idset":"10784,10792,10796-10797"}}
+{"timestamp":1714065166.0826757,"name":"online","context":{"idset":"10791,10800"}}
+{"timestamp":1714065166.2624178,"name":"online","context":{"idset":"10804"}}
+{"timestamp":1714065166.4377947,"name":"online","context":{"idset":"10803"}}
+{"timestamp":1714065166.5942662,"name":"online","context":{"idset":"10798,10801"}}
+{"timestamp":1714065170.4749861,"name":"online","context":{"idset":"10805"}}
+{"timestamp":1714065170.6746504,"name":"online","context":{"idset":"10807"}}
+{"timestamp":1714065171.0135777,"name":"online","context":{"idset":"10808"}}
+{"timestamp":1714065171.2737172,"name":"online","context":{"idset":"10809"}}
+{"timestamp":1714065171.6504467,"name":"online","context":{"idset":"10810,10813"}}
+{"timestamp":1714065171.7543674,"name":"online","context":{"idset":"10806,10811"}}
+{"timestamp":1714065172.1149952,"name":"online","context":{"idset":"10815,10824"}}
+{"timestamp":1714065172.3369668,"name":"online","context":{"idset":"10812,10814,10816-10817,10819-10820,10822"}}
+{"timestamp":1714065172.502187,"name":"online","context":{"idset":"10821"}}
+{"timestamp":1714065172.6738114,"name":"online","context":{"idset":"10818,10825,10832"}}
+{"timestamp":1714065172.9796526,"name":"online","context":{"idset":"10823,10827,10829,10831,10834"}}
+{"timestamp":1714065173.1460576,"name":"online","context":{"idset":"10828,10833"}}
+{"timestamp":1714065173.3542631,"name":"online","context":{"idset":"10826,10830,10835-10836"}}
+{"timestamp":1714065176.9212775,"name":"online","context":{"idset":"10837"}}
+{"timestamp":1714065177.7780054,"name":"online","context":{"idset":"10838-10839"}}
+{"timestamp":1714065177.8376453,"name":"online","context":{"idset":"10840"}}
+{"timestamp":1714065178.188652,"name":"online","context":{"idset":"10842"}}
+{"timestamp":1714065178.3881087,"name":"online","context":{"idset":"10841,10843"}}
+{"timestamp":1714065178.7036173,"name":"online","context":{"idset":"10844-10845"}}
+{"timestamp":1714065178.8962176,"name":"online","context":{"idset":"10846,10848-10849,10851,10853"}}
+{"timestamp":1714065179.0025089,"name":"online","context":{"idset":"10858"}}
+{"timestamp":1714065179.1996953,"name":"online","context":{"idset":"10847,10850,10852,10857,10860"}}
+{"timestamp":1714065179.5602045,"name":"online","context":{"idset":"10859"}}
+{"timestamp":1714065179.7238839,"name":"online","context":{"idset":"10854-10855,10862"}}
+{"timestamp":1714065179.8876898,"name":"online","context":{"idset":"10856,10865"}}
+{"timestamp":1714065180.0578418,"name":"online","context":{"idset":"10861,10863"}}
+{"timestamp":1714065180.2248459,"name":"online","context":{"idset":"10866-10868"}}
+{"timestamp":1714065180.3949637,"name":"online","context":{"idset":"10864"}}
+{"timestamp":1714065183.4352663,"name":"online","context":{"idset":"11765"}}
+{"timestamp":1714065183.9189413,"name":"online","context":{"idset":"11766"}}
+{"timestamp":1714065184.4712465,"name":"online","context":{"idset":"11768,11770"}}
+{"timestamp":1714065185.0134814,"name":"online","context":{"idset":"11767,11769"}}
+{"timestamp":1714065185.3589153,"name":"online","context":{"idset":"11771,11774"}}
+{"timestamp":1714065185.6139984,"name":"online","context":{"idset":"11772,11777,11780"}}
+{"timestamp":1714065185.819371,"name":"online","context":{"idset":"11775-11776,11778-11779,11788"}}
+{"timestamp":1714065185.9968598,"name":"online","context":{"idset":"11773,11782,11787"}}
+{"timestamp":1714065186.161375,"name":"online","context":{"idset":"11783"}}
+{"timestamp":1714065186.3959355,"name":"online","context":{"idset":"11781,11786,11791-11792"}}
+{"timestamp":1714065186.6240611,"name":"online","context":{"idset":"11784"}}
+{"timestamp":1714065186.79231,"name":"online","context":{"idset":"11785,11795"}}
+{"timestamp":1714065186.960501,"name":"online","context":{"idset":"11794,11796"}}
+{"timestamp":1714065187.1326323,"name":"online","context":{"idset":"11793"}}
+{"timestamp":1714065190.6891475,"name":"online","context":{"idset":"11797-11799"}}
+{"timestamp":1714065191.2374878,"name":"online","context":{"idset":"11800"}}
+{"timestamp":1714065191.5876827,"name":"online","context":{"idset":"11804"}}
+{"timestamp":1714065191.8909001,"name":"online","context":{"idset":"11802-11803"}}
+{"timestamp":1714065192.8224814,"name":"online","context":{"idset":"11801"}}
+{"timestamp":1714065192.8255258,"name":"online","context":{"idset":"11805-11808,11810,11813"}}
+{"timestamp":1714065192.8282883,"name":"online","context":{"idset":"11809,11811,11816"}}
+{"timestamp":1714065192.838007,"name":"online","context":{"idset":"11812,11815,11819"}}
+{"timestamp":1714065192.9101155,"name":"online","context":{"idset":"11817"}}
+{"timestamp":1714065193.7791181,"name":"online","context":{"idset":"11814,11822,11824"}}
+{"timestamp":1714065193.7817659,"name":"online","context":{"idset":"11820-11821"}}
+{"timestamp":1714065193.7844088,"name":"online","context":{"idset":"11818,11823"}}
+{"timestamp":1714065195.6354249,"name":"online","context":{"idset":"11827-11828"}}
+{"timestamp":1714065197.3523576,"name":"online","context":{"idset":"11831"}}
+{"timestamp":1714065197.4692826,"name":"online","context":{"idset":"11830"}}
+{"timestamp":1714065197.5576954,"name":"online","context":{"idset":"11829"}}
+{"timestamp":1714065197.8665404,"name":"online","context":{"idset":"11832"}}
+{"timestamp":1714065198.2636101,"name":"online","context":{"idset":"11835"}}
+{"timestamp":1714065198.5184376,"name":"online","context":{"idset":"11833"}}
+{"timestamp":1714065198.7967381,"name":"online","context":{"idset":"11840"}}
+{"timestamp":1714065199.1214776,"name":"online","context":{"idset":"11839"}}
+{"timestamp":1714065199.2824569,"name":"online","context":{"idset":"11838,11841"}}
+{"timestamp":1714065199.4480085,"name":"online","context":{"idset":"11834"}}
+{"timestamp":1714065199.6190341,"name":"online","context":{"idset":"11837,11842,11847"}}
+{"timestamp":1714065199.830363,"name":"online","context":{"idset":"11843-11844,11846"}}
+{"timestamp":1714065200.0955229,"name":"online","context":{"idset":"11852-11853"}}
+{"timestamp":1714065200.2438931,"name":"online","context":{"idset":"11850"}}
+{"timestamp":1714065200.3662114,"name":"online","context":{"idset":"11836,11851"}}
+{"timestamp":1714065200.599371,"name":"online","context":{"idset":"11845,11854-11855"}}
+{"timestamp":1714065200.733582,"name":"online","context":{"idset":"11849"}}
+{"timestamp":1714065200.9365242,"name":"online","context":{"idset":"11856"}}
+{"timestamp":1714065201.1028755,"name":"online","context":{"idset":"11848"}}
+{"timestamp":1714065202.6424983,"name":"online","context":{"idset":"11857-11858"}}
+{"timestamp":1714065203.1209671,"name":"online","context":{"idset":"11859-11860"}}
+{"timestamp":1714065203.8867643,"name":"online","context":{"idset":"11861"}}
+{"timestamp":1714065204.0789857,"name":"online","context":{"idset":"11862"}}
+{"timestamp":1714065204.6139011,"name":"online","context":{"idset":"11863-11864"}}
+{"timestamp":1714065205.4380665,"name":"online","context":{"idset":"11865"}}
+{"timestamp":1714065205.6456957,"name":"online","context":{"idset":"11868"}}
+{"timestamp":1714065205.8171008,"name":"online","context":{"idset":"11866"}}
+{"timestamp":1714065205.9942343,"name":"online","context":{"idset":"11867"}}
+{"timestamp":1714065206.3270915,"name":"online","context":{"idset":"11869,11871"}}
+{"timestamp":1714065206.5277493,"name":"online","context":{"idset":"11872-11873"}}
+{"timestamp":1714065206.6886754,"name":"online","context":{"idset":"11870,11875"}}
+{"timestamp":1714065206.8068452,"name":"online","context":{"idset":"11880"}}
+{"timestamp":1714065206.9140248,"name":"online","context":{"idset":"11876-11877"}}
+{"timestamp":1714065207.0687408,"name":"online","context":{"idset":"11874,11879,11881-11882"}}
+{"timestamp":1714065207.3678355,"name":"online","context":{"idset":"11883"}}
+{"timestamp":1714065207.5323112,"name":"online","context":{"idset":"11884,11886"}}
+{"timestamp":1714065207.6160924,"name":"online","context":{"idset":"11878,11885"}}
+{"timestamp":1714065207.9075332,"name":"online","context":{"idset":"11887-11888"}}
+{"timestamp":1714065209.0702453,"name":"online","context":{"idset":"11889-11890"}}
+{"timestamp":1714065209.7700193,"name":"online","context":{"idset":"11891"}}
+{"timestamp":1714065209.9729187,"name":"online","context":{"idset":"11892"}}
+{"timestamp":1714065244.6289601,"name":"online","context":{"idset":"8341"}}
+{"timestamp":1714065507.4096789,"name":"online","context":{"idset":"8309"}}
+{"timestamp":1714065712.2163446,"name":"offline","context":{"idset":"9493"}}
+{"timestamp":1714065712.2190895,"name":"offline","context":{"idset":"9494"}}
+{"timestamp":1714065712.2217379,"name":"offline","context":{"idset":"9495"}}
+{"timestamp":1714065712.2243953,"name":"offline","context":{"idset":"9496"}}
+{"timestamp":1714065712.2270854,"name":"offline","context":{"idset":"9497"}}
+{"timestamp":1714065712.2297215,"name":"offline","context":{"idset":"9498"}}
+{"timestamp":1714065712.2324502,"name":"offline","context":{"idset":"9499"}}
+{"timestamp":1714065712.235225,"name":"offline","context":{"idset":"9500"}}
+{"timestamp":1714065712.2381196,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1714065712.2407498,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1714065712.2438686,"name":"offline","context":{"idset":"9503"}}
+{"timestamp":1714065712.2464988,"name":"offline","context":{"idset":"9504"}}
+{"timestamp":1714065712.249109,"name":"offline","context":{"idset":"9505"}}
+{"timestamp":1714065712.2517266,"name":"offline","context":{"idset":"9506"}}
+{"timestamp":1714065712.254374,"name":"offline","context":{"idset":"9507"}}
+{"timestamp":1714065712.2631092,"name":"drain","context":{"idset":"9493-9508","reason":"epilog failed for jobid fratvYNikvT","overwrite":0}}
+{"timestamp":1714065712.3247445,"name":"offline","context":{"idset":"9508"}}
+{"timestamp":1714066190.9512579,"name":"drain","context":{"idset":"877-878","reason":"--reason troubleshoot cxi","overwrite":0}}
+{"timestamp":1714066490.8935003,"name":"online","context":{"idset":"8606"}}
+{"timestamp":1714066491.1615348,"name":"online","context":{"idset":"8598,8602"}}
+{"timestamp":1714066491.402761,"name":"online","context":{"idset":"8603"}}
+{"timestamp":1714066491.6500187,"name":"online","context":{"idset":"8612"}}
+{"timestamp":1714066491.822346,"name":"online","context":{"idset":"8601,8607"}}
+{"timestamp":1714066491.9247024,"name":"online","context":{"idset":"8604,8608"}}
+{"timestamp":1714066492.1385062,"name":"online","context":{"idset":"8599,8610"}}
+{"timestamp":1714066492.3330822,"name":"online","context":{"idset":"8597,8600,8609"}}
+{"timestamp":1714066492.5151429,"name":"online","context":{"idset":"8611"}}
+{"timestamp":1714066492.993609,"name":"online","context":{"idset":"8605"}}
+{"timestamp":1714066495.657001,"name":"undrain","context":{"idset":"8597-8612"}}
+{"timestamp":1714066548.9902277,"name":"undrain","context":{"idset":"11509-11636"}}
+{"timestamp":1714066666.0818803,"name":"drain","context":{"idset":"8309-8436","reason":"--reason rebooting - wendy","overwrite":0}}
+{"timestamp":1714066679.1934192,"name":"offline","context":{"idset":"8320"}}
+{"timestamp":1714066679.2921734,"name":"offline","context":{"idset":"8330"}}
+{"timestamp":1714066679.4148235,"name":"offline","context":{"idset":"8325"}}
+{"timestamp":1714066679.4963896,"name":"offline","context":{"idset":"8322"}}
+{"timestamp":1714066679.5657201,"name":"offline","context":{"idset":"8311"}}
+{"timestamp":1714066679.5691171,"name":"offline","context":{"idset":"8316"}}
+{"timestamp":1714066679.6705806,"name":"offline","context":{"idset":"8358"}}
+{"timestamp":1714066679.677201,"name":"offline","context":{"idset":"8315"}}
+{"timestamp":1714066679.6844511,"name":"offline","context":{"idset":"8321"}}
+{"timestamp":1714066679.7791054,"name":"offline","context":{"idset":"8329"}}
+{"timestamp":1714066679.8677528,"name":"offline","context":{"idset":"8362"}}
+{"timestamp":1714066679.9409595,"name":"offline","context":{"idset":"8396"}}
+{"timestamp":1714066680.0074677,"name":"offline","context":{"idset":"8344"}}
+{"timestamp":1714066680.1007748,"name":"offline","context":{"idset":"8331"}}
+{"timestamp":1714066680.2013309,"name":"offline","context":{"idset":"8309"}}
+{"timestamp":1714066680.2789633,"name":"offline","context":{"idset":"8371"}}
+{"timestamp":1714066680.2814958,"name":"offline","context":{"idset":"8332"}}
+{"timestamp":1714066680.2852931,"name":"offline","context":{"idset":"8349"}}
+{"timestamp":1714066680.3034775,"name":"offline","context":{"idset":"8339"}}
+{"timestamp":1714066680.3068435,"name":"offline","context":{"idset":"8375"}}
+{"timestamp":1714066680.3924477,"name":"offline","context":{"idset":"8409"}}
+{"timestamp":1714066680.4655929,"name":"offline","context":{"idset":"8319"}}
+{"timestamp":1714066680.4701564,"name":"offline","context":{"idset":"8363"}}
+{"timestamp":1714066680.4752514,"name":"offline","context":{"idset":"8323"}}
+{"timestamp":1714066680.484539,"name":"offline","context":{"idset":"8327"}}
+{"timestamp":1714066680.4874804,"name":"offline","context":{"idset":"8366"}}
+{"timestamp":1714066680.5805686,"name":"offline","context":{"idset":"8378"}}
+{"timestamp":1714066680.656136,"name":"offline","context":{"idset":"8403"}}
+{"timestamp":1714066680.6593461,"name":"offline","context":{"idset":"8335"}}
+{"timestamp":1714066680.6621916,"name":"offline","context":{"idset":"8336"}}
+{"timestamp":1714066680.6659744,"name":"offline","context":{"idset":"8391"}}
+{"timestamp":1714066680.6691756,"name":"offline","context":{"idset":"8333"}}
+{"timestamp":1714066680.684535,"name":"offline","context":{"idset":"8317"}}
+{"timestamp":1714066680.7735844,"name":"offline","context":{"idset":"8318"}}
+{"timestamp":1714066680.8590131,"name":"offline","context":{"idset":"8314"}}
+{"timestamp":1714066680.8627827,"name":"offline","context":{"idset":"8341"}}
+{"timestamp":1714066680.8662057,"name":"offline","context":{"idset":"8326"}}
+{"timestamp":1714066680.8699973,"name":"offline","context":{"idset":"8343"}}
+{"timestamp":1714066680.8737772,"name":"offline","context":{"idset":"8351"}}
+{"timestamp":1714066680.8805044,"name":"offline","context":{"idset":"8312"}}
+{"timestamp":1714066680.9093838,"name":"offline","context":{"idset":"8372"}}
+{"timestamp":1714066680.9134221,"name":"offline","context":{"idset":"8354"}}
+{"timestamp":1714066680.9173615,"name":"offline","context":{"idset":"8380"}}
+{"timestamp":1714066681.0206709,"name":"offline","context":{"idset":"8412"}}
+{"timestamp":1714066681.0255506,"name":"offline","context":{"idset":"8313"}}
+{"timestamp":1714066681.0300765,"name":"offline","context":{"idset":"8417"}}
+{"timestamp":1714066681.0331099,"name":"offline","context":{"idset":"8342"}}
+{"timestamp":1714066681.046428,"name":"offline","context":{"idset":"8411"}}
+{"timestamp":1714066681.049391,"name":"offline","context":{"idset":"8386"}}
+{"timestamp":1714066681.1331091,"name":"offline","context":{"idset":"8416"}}
+{"timestamp":1714066681.1744668,"name":"offline","context":{"idset":"8394"}}
+{"timestamp":1714066681.1776998,"name":"offline","context":{"idset":"8368"}}
+{"timestamp":1714066681.180809,"name":"offline","context":{"idset":"8398"}}
+{"timestamp":1714066681.1837571,"name":"offline","context":{"idset":"8345"}}
+{"timestamp":1714066681.1871288,"name":"offline","context":{"idset":"8347"}}
+{"timestamp":1714066681.1901183,"name":"offline","context":{"idset":"8379"}}
+{"timestamp":1714066681.2807107,"name":"offline","context":{"idset":"8350"}}
+{"timestamp":1714066681.3571661,"name":"offline","context":{"idset":"8399"}}
+{"timestamp":1714066681.3627276,"name":"offline","context":{"idset":"8352"}}
+{"timestamp":1714066681.3664577,"name":"offline","context":{"idset":"8393"}}
+{"timestamp":1714066681.3863716,"name":"offline","context":{"idset":"8407"}}
+{"timestamp":1714066681.3892367,"name":"offline","context":{"idset":"8348"}}
+{"timestamp":1714066681.4016244,"name":"offline","context":{"idset":"8357"}}
+{"timestamp":1714066681.4702911,"name":"offline","context":{"idset":"8389"}}
+{"timestamp":1714066681.5566492,"name":"offline","context":{"idset":"8328"}}
+{"timestamp":1714066681.560617,"name":"offline","context":{"idset":"8324"}}
+{"timestamp":1714066681.5636494,"name":"offline","context":{"idset":"8390"}}
+{"timestamp":1714066681.5675726,"name":"offline","context":{"idset":"8337"}}
+{"timestamp":1714066681.6117473,"name":"offline","context":{"idset":"8359"}}
+{"timestamp":1714066681.6147816,"name":"offline","context":{"idset":"8338"}}
+{"timestamp":1714066681.6309476,"name":"offline","context":{"idset":"8355"}}
+{"timestamp":1714066681.6335964,"name":"offline","context":{"idset":"8360"}}
+{"timestamp":1714066681.6365771,"name":"offline","context":{"idset":"8385"}}
+{"timestamp":1714066681.6397867,"name":"offline","context":{"idset":"8401"}}
+{"timestamp":1714066681.6461406,"name":"offline","context":{"idset":"8419"}}
+{"timestamp":1714066681.649174,"name":"offline","context":{"idset":"8424"}}
+{"timestamp":1714066681.6623421,"name":"offline","context":{"idset":"8428"}}
+{"timestamp":1714066681.6758397,"name":"offline","context":{"idset":"8433"}}
+{"timestamp":1714066681.6790259,"name":"offline","context":{"idset":"8340"}}
+{"timestamp":1714066681.6822424,"name":"offline","context":{"idset":"8400"}}
+{"timestamp":1714066681.8192,"name":"offline","context":{"idset":"8420"}}
+{"timestamp":1714066681.8446949,"name":"offline","context":{"idset":"8373"}}
+{"timestamp":1714066681.861129,"name":"offline","context":{"idset":"8405"}}
+{"timestamp":1714066681.8640845,"name":"offline","context":{"idset":"8310"}}
+{"timestamp":1714066681.8672197,"name":"offline","context":{"idset":"8376"}}
+{"timestamp":1714066681.870132,"name":"offline","context":{"idset":"8436"}}
+{"timestamp":1714066681.9610326,"name":"offline","context":{"idset":"8384"}}
+{"timestamp":1714066682.0379753,"name":"offline","context":{"idset":"8414"}}
+{"timestamp":1714066682.0409334,"name":"offline","context":{"idset":"8369"}}
+{"timestamp":1714066682.0459685,"name":"offline","context":{"idset":"8374"}}
+{"timestamp":1714066682.0487366,"name":"offline","context":{"idset":"8406"}}
+{"timestamp":1714066682.0631101,"name":"offline","context":{"idset":"8413"}}
+{"timestamp":1714066682.066293,"name":"offline","context":{"idset":"8387"}}
+{"timestamp":1714066682.1515973,"name":"offline","context":{"idset":"8418"}}
+{"timestamp":1714066682.2478745,"name":"offline","context":{"idset":"8365"}}
+{"timestamp":1714066682.253351,"name":"offline","context":{"idset":"8388"}}
+{"timestamp":1714066682.2883925,"name":"offline","context":{"idset":"8395"}}
+{"timestamp":1714066682.2912967,"name":"offline","context":{"idset":"8361"}}
+{"timestamp":1714066682.2992165,"name":"offline","context":{"idset":"8392"}}
+{"timestamp":1714066682.3567972,"name":"offline","context":{"idset":"8434"}}
+{"timestamp":1714066682.4437945,"name":"offline","context":{"idset":"8334"}}
+{"timestamp":1714066682.4478185,"name":"offline","context":{"idset":"8404"}}
+{"timestamp":1714066682.450948,"name":"offline","context":{"idset":"8377"}}
+{"timestamp":1714066682.5514443,"name":"offline","context":{"idset":"8353"}}
+{"timestamp":1714066682.614701,"name":"offline","context":{"idset":"8382"}}
+{"timestamp":1714066682.624773,"name":"offline","context":{"idset":"8356"}}
+{"timestamp":1714066682.6297197,"name":"offline","context":{"idset":"8425"}}
+{"timestamp":1714066682.7222917,"name":"offline","context":{"idset":"8346"}}
+{"timestamp":1714066682.7466762,"name":"offline","context":{"idset":"8408"}}
+{"timestamp":1714066682.8421347,"name":"offline","context":{"idset":"8421"}}
+{"timestamp":1714066682.9124134,"name":"offline","context":{"idset":"8432"}}
+{"timestamp":1714066682.9790559,"name":"offline","context":{"idset":"8367"}}
+{"timestamp":1714066682.9828293,"name":"offline","context":{"idset":"8402"}}
+{"timestamp":1714066683.069067,"name":"offline","context":{"idset":"8410"}}
+{"timestamp":1714066683.0728714,"name":"offline","context":{"idset":"8397"}}
+{"timestamp":1714066683.1593716,"name":"offline","context":{"idset":"8364"}}
+{"timestamp":1714066683.1627302,"name":"offline","context":{"idset":"8427"}}
+{"timestamp":1714066683.2627876,"name":"offline","context":{"idset":"8381"}}
+{"timestamp":1714066683.3389621,"name":"offline","context":{"idset":"8429"}}
+{"timestamp":1714066683.4119627,"name":"offline","context":{"idset":"8435"}}
+{"timestamp":1714066683.4777009,"name":"offline","context":{"idset":"8415"}}
+{"timestamp":1714066683.5827501,"name":"offline","context":{"idset":"8423"}}
+{"timestamp":1714066683.6609864,"name":"offline","context":{"idset":"8431"}}
+{"timestamp":1714066683.7305477,"name":"offline","context":{"idset":"8383"}}
+{"timestamp":1714066683.8149934,"name":"offline","context":{"idset":"8426"}}
+{"timestamp":1714066683.9130471,"name":"offline","context":{"idset":"8422"}}
+{"timestamp":1714066683.9773903,"name":"offline","context":{"idset":"8430"}}
+{"timestamp":1714066684.0768363,"name":"offline","context":{"idset":"8370"}}
+{"timestamp":1714067110.21977,"name":"offline","context":{"idset":"10213"}}
+{"timestamp":1714067110.2242663,"name":"offline","context":{"idset":"10214"}}
+{"timestamp":1714067110.228797,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1714067110.2332606,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1714067110.2377696,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1714067110.2423182,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1714067110.2466614,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1714067110.2510521,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1714067110.2553453,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1714067110.2597654,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1714067110.2640305,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1714067110.2685094,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1714067110.9703526,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1714067110.9734621,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1714067110.9855831,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1714067111.0197673,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1714067165.044755,"name":"drain","context":{"idset":"8597-8612","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714068042.2647524,"name":"offline","context":{"idset":"878"}}
+{"timestamp":1714068207.508677,"name":"online","context":{"idset":"11511"}}
+{"timestamp":1714068207.8005543,"name":"online","context":{"idset":"11576"}}
+{"timestamp":1714068207.9021127,"name":"online","context":{"idset":"11515"}}
+{"timestamp":1714068208.0175786,"name":"online","context":{"idset":"11585"}}
+{"timestamp":1714068208.0903246,"name":"online","context":{"idset":"11535"}}
+{"timestamp":1714068208.22296,"name":"online","context":{"idset":"11514,11565,11602"}}
+{"timestamp":1714068208.3238842,"name":"online","context":{"idset":"11575"}}
+{"timestamp":1714068208.4114473,"name":"online","context":{"idset":"11529"}}
+{"timestamp":1714068208.4766762,"name":"online","context":{"idset":"11513,11516,11522"}}
+{"timestamp":1714068208.5496776,"name":"online","context":{"idset":"11560,11567,11583"}}
+{"timestamp":1714068208.6916831,"name":"online","context":{"idset":"11512"}}
+{"timestamp":1714068208.8477967,"name":"online","context":{"idset":"11519,11526,11550,11566"}}
+{"timestamp":1714068209.03075,"name":"online","context":{"idset":"11555,11557,11596,11612,11626"}}
+{"timestamp":1714068209.1938472,"name":"online","context":{"idset":"11509-11510,11523,11564,11617"}}
+{"timestamp":1714068209.3618593,"name":"online","context":{"idset":"11521,11527,11554,11561,11587,11593,11633,11635"}}
+{"timestamp":1714068209.5293372,"name":"online","context":{"idset":"11537,11545,11618"}}
+{"timestamp":1714068209.7014899,"name":"online","context":{"idset":"11559,11578,11588,11591,11614"}}
+{"timestamp":1714068209.8744264,"name":"online","context":{"idset":"11534,11548,11551,11568,11636"}}
+{"timestamp":1714068210.0594199,"name":"online","context":{"idset":"11517-11518,11531,11558,11592,11615,11620,11625,11634"}}
+{"timestamp":1714068210.2225685,"name":"online","context":{"idset":"11530,11541,11574,11590,11603,11629"}}
+{"timestamp":1714068210.3291147,"name":"online","context":{"idset":"11525,11539,11570,11572,11580,11605,11616,11622"}}
+{"timestamp":1714068210.5168169,"name":"online","context":{"idset":"11532-11533,11543,11577,11595,11607,11621"}}
+{"timestamp":1714068210.6791227,"name":"online","context":{"idset":"11544,11549,11562,11573,11586,11598,11627,11632"}}
+{"timestamp":1714068210.786468,"name":"online","context":{"idset":"11520,11524,11528,11538,11547,11553,11582,11584,11589,11599,11604,11608,11611,11623"}}
+{"timestamp":1714068210.9608493,"name":"online","context":{"idset":"11540,11542,11546,11556,11579,11594,11597,11600-11601,11606,11609,11628,11630-11631"}}
+{"timestamp":1714068211.1612873,"name":"online","context":{"idset":"11552,11563,11571,11581,11610"}}
+{"timestamp":1714068211.2672508,"name":"online","context":{"idset":"11536,11613,11619,11624"}}
+{"timestamp":1714068211.5084264,"name":"online","context":{"idset":"11569"}}
+{"timestamp":1714068680.3335018,"name":"online","context":{"idset":"9501"}}
+{"timestamp":1714068680.5383356,"name":"online","context":{"idset":"9497"}}
+{"timestamp":1714068680.6956275,"name":"online","context":{"idset":"9495,9503"}}
+{"timestamp":1714068680.9166067,"name":"online","context":{"idset":"9493-9494"}}
+{"timestamp":1714068681.0783317,"name":"online","context":{"idset":"9498-9499"}}
+{"timestamp":1714068681.3030207,"name":"online","context":{"idset":"9496,9502,9505-9506"}}
+{"timestamp":1714068681.5406327,"name":"online","context":{"idset":"9500,9504,9507"}}
+{"timestamp":1714068681.764605,"name":"online","context":{"idset":"9508"}}
+{"timestamp":1714068712.2626119,"name":"offline","context":{"idset":"10233"}}
+{"timestamp":1714068720.263855,"name":"offline","context":{"idset":"10248"}}
+{"timestamp":1714068750.2649541,"name":"offline","context":{"idset":"10310"}}
+{"timestamp":1714068770.2630985,"name":"offline","context":{"idset":"10341"}}
+{"timestamp":1714068772.180644,"name":"offline","context":{"idset":"10345"}}
+{"timestamp":1714068772.1848373,"name":"offline","context":{"idset":"10346"}}
+{"timestamp":1714068772.2680824,"name":"offline","context":{"idset":"10347"}}
+{"timestamp":1714068774.1885819,"name":"offline","context":{"idset":"10349"}}
+{"timestamp":1714068774.1928306,"name":"offline","context":{"idset":"10350"}}
+{"timestamp":1714068774.2660387,"name":"offline","context":{"idset":"10351"}}
+{"timestamp":1714068776.1771898,"name":"offline","context":{"idset":"10353"}}
+{"timestamp":1714068776.1801279,"name":"offline","context":{"idset":"10354"}}
+{"timestamp":1714068776.2629278,"name":"offline","context":{"idset":"10355"}}
+{"timestamp":1714069034.2650013,"name":"offline","context":{"idset":"828"}}
+{"timestamp":1714069132.3940766,"name":"undrain","context":{"idset":"9493-9508"}}
+{"timestamp":1714069472.0908577,"name":"online","context":{"idset":"8170"}}
+{"timestamp":1714069472.6417582,"name":"online","context":{"idset":"8169,8174,8180"}}
+{"timestamp":1714069472.8558717,"name":"online","context":{"idset":"8172-8173,8175,8179,8237"}}
+{"timestamp":1714069473.0321643,"name":"online","context":{"idset":"8176,8178"}}
+{"timestamp":1714069473.3140066,"name":"online","context":{"idset":"8171,8177"}}
+{"timestamp":1714069601.6656423,"name":"online","context":{"idset":"8059"}}
+{"timestamp":1714069603.1642373,"name":"online","context":{"idset":"8055"}}
+{"timestamp":1714069604.122961,"name":"online","context":{"idset":"8057,8062"}}
+{"timestamp":1714069604.1284444,"name":"online","context":{"idset":"8056,8067"}}
+{"timestamp":1714069604.1320434,"name":"online","context":{"idset":"8063,8066"}}
+{"timestamp":1714069604.1355333,"name":"online","context":{"idset":"8064"}}
+{"timestamp":1714069605.109046,"name":"online","context":{"idset":"8053,8061,8065"}}
+{"timestamp":1714069605.1145055,"name":"online","context":{"idset":"8058,8068"}}
+{"timestamp":1714069605.1181738,"name":"online","context":{"idset":"8054,8060"}}
+{"timestamp":1714069679.5291765,"name":"undrain","context":{"idset":"877-878"}}
+{"timestamp":1714069728.9068637,"name":"online","context":{"idset":"878"}}
+{"timestamp":1714069801.5225866,"name":"online","context":{"idset":"8078"}}
+{"timestamp":1714069808.2494626,"name":"online","context":{"idset":"8075"}}
+{"timestamp":1714069809.3896027,"name":"online","context":{"idset":"8074,8076,8080-8081"}}
+{"timestamp":1714069809.88201,"name":"online","context":{"idset":"8069-8070,8083"}}
+{"timestamp":1714069810.039566,"name":"online","context":{"idset":"8071"}}
+{"timestamp":1714069810.6148221,"name":"online","context":{"idset":"8072-8073,8077"}}
+{"timestamp":1714069810.7165976,"name":"online","context":{"idset":"8079,8082,8084"}}
+{"timestamp":1714069873.2921529,"name":"drain","context":{"idset":"546","reason":"nodediag failed cxi pci","overwrite":0}}
+{"timestamp":1714069877.5896714,"name":"drain","context":{"idset":"545","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714069881.1079395,"name":"drain","context":{"idset":"544","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714070492.8618567,"name":"online","context":{"idset":"8313"}}
+{"timestamp":1714070493.098788,"name":"online","context":{"idset":"8312"}}
+{"timestamp":1714070493.631825,"name":"online","context":{"idset":"8311"}}
+{"timestamp":1714070493.8601792,"name":"online","context":{"idset":"8315,8329"}}
+{"timestamp":1714070494.0333302,"name":"online","context":{"idset":"8319,8325"}}
+{"timestamp":1714070494.3539426,"name":"online","context":{"idset":"8314"}}
+{"timestamp":1714070494.4906619,"name":"online","context":{"idset":"8330"}}
+{"timestamp":1714070495.2355216,"name":"online","context":{"idset":"8327"}}
+{"timestamp":1714070495.9804704,"name":"online","context":{"idset":"8310"}}
+{"timestamp":1714070496.7788379,"name":"online","context":{"idset":"8318,8340"}}
+{"timestamp":1714070497.0998764,"name":"online","context":{"idset":"8336"}}
+{"timestamp":1714070497.3450532,"name":"online","context":{"idset":"8320,8359"}}
+{"timestamp":1714070497.6266062,"name":"online","context":{"idset":"8324"}}
+{"timestamp":1714070497.7338653,"name":"online","context":{"idset":"8322"}}
+{"timestamp":1714070498.1946251,"name":"online","context":{"idset":"8342,8347"}}
+{"timestamp":1714070498.2801361,"name":"online","context":{"idset":"8363"}}
+{"timestamp":1714070498.3474455,"name":"online","context":{"idset":"8382"}}
+{"timestamp":1714070498.4190152,"name":"online","context":{"idset":"8339"}}
+{"timestamp":1714070498.6319284,"name":"online","context":{"idset":"8348"}}
+{"timestamp":1714070498.7063763,"name":"online","context":{"idset":"8349"}}
+{"timestamp":1714070498.8358719,"name":"online","context":{"idset":"8395"}}
+{"timestamp":1714070498.9042206,"name":"online","context":{"idset":"8332"}}
+{"timestamp":1714070499.0838921,"name":"online","context":{"idset":"8334,8389"}}
+{"timestamp":1714070499.2238638,"name":"online","context":{"idset":"8309"}}
+{"timestamp":1714070499.2935669,"name":"online","context":{"idset":"8338"}}
+{"timestamp":1714070499.4630115,"name":"online","context":{"idset":"8345"}}
+{"timestamp":1714070499.525429,"name":"online","context":{"idset":"8331"}}
+{"timestamp":1714070499.6735988,"name":"online","context":{"idset":"8341"}}
+{"timestamp":1714070499.7604337,"name":"online","context":{"idset":"8328"}}
+{"timestamp":1714070499.89235,"name":"online","context":{"idset":"8356"}}
+{"timestamp":1714070499.961278,"name":"online","context":{"idset":"8362"}}
+{"timestamp":1714070500.063309,"name":"online","context":{"idset":"8361,8398"}}
+{"timestamp":1714070500.1311064,"name":"online","context":{"idset":"8358"}}
+{"timestamp":1714070500.3463964,"name":"online","context":{"idset":"8317,8326,8374"}}
+{"timestamp":1714070500.6328051,"name":"online","context":{"idset":"8321,8333,8355,8372,8386,8401"}}
+{"timestamp":1714070500.7101288,"name":"online","context":{"idset":"8371,8411"}}
+{"timestamp":1714070500.7799313,"name":"online","context":{"idset":"8354,8364,8406"}}
+{"timestamp":1714070500.945914,"name":"online","context":{"idset":"8343,8378,8383,8393"}}
+{"timestamp":1714070501.1159959,"name":"online","context":{"idset":"8344,8346,8350-8351,8360,8365,8369,8385"}}
+{"timestamp":1714070501.2835615,"name":"online","context":{"idset":"8357,8366,8380,8400,8407"}}
+{"timestamp":1714070501.4641995,"name":"online","context":{"idset":"8316,8323,8368,8373,8390,8396,8408,8410,8424"}}
+{"timestamp":1714070501.5713427,"name":"online","context":{"idset":"8352,8377,8381,8384,8388,8391"}}
+{"timestamp":1714070501.7346659,"name":"online","context":{"idset":"8387,8392,8397,8399,8404,8409,8415"}}
+{"timestamp":1714070501.9176843,"name":"online","context":{"idset":"8353,8376,8379,8421,8425-8426,8432,8435"}}
+{"timestamp":1714070502.0971727,"name":"online","context":{"idset":"8335,8367,8370,8375"}}
+{"timestamp":1714070502.2045867,"name":"online","context":{"idset":"8419"}}
+{"timestamp":1714070502.3843634,"name":"online","context":{"idset":"8337,8394,8402-8403,8405,8412,8414,8417-8418,8420,8422-8423,8428-8430,8434,8436"}}
+{"timestamp":1714070502.6746883,"name":"online","context":{"idset":"8416,8427"}}
+{"timestamp":1714070503.0029104,"name":"online","context":{"idset":"8413"}}
+{"timestamp":1714070503.2209489,"name":"online","context":{"idset":"8433"}}
+{"timestamp":1714070549.6776645,"name":"online","context":{"idset":"852"}}
+{"timestamp":1714070610.6258247,"name":"online","context":{"idset":"851"}}
+{"timestamp":1714070923.5204384,"name":"online","context":{"idset":"8091"}}
+{"timestamp":1714070924.8937833,"name":"online","context":{"idset":"8100"}}
+{"timestamp":1714070927.9449065,"name":"online","context":{"idset":"8094"}}
+{"timestamp":1714070930.9024372,"name":"online","context":{"idset":"8092"}}
+{"timestamp":1714070931.2551641,"name":"online","context":{"idset":"8099"}}
+{"timestamp":1714070931.5684128,"name":"online","context":{"idset":"8089"}}
+{"timestamp":1714070931.8303728,"name":"online","context":{"idset":"8090"}}
+{"timestamp":1714070932.0315197,"name":"online","context":{"idset":"8086,8097"}}
+{"timestamp":1714070932.9130816,"name":"online","context":{"idset":"8087,8096"}}
+{"timestamp":1714070932.916091,"name":"online","context":{"idset":"8085,8088,8093"}}
+{"timestamp":1714070932.9192946,"name":"online","context":{"idset":"8095"}}
+{"timestamp":1714070932.9221532,"name":"online","context":{"idset":"8098"}}
+{"timestamp":1714071513.0555923,"name":"offline","context":{"idset":"7730"}}
+{"timestamp":1714073626.5694263,"name":"offline","context":{"idset":"7711"}}
+{"timestamp":1714073674.4273918,"name":"online","context":{"idset":"7730"}}
+{"timestamp":1714073796.6989737,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1714073926.7882922,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1714074106.4059894,"name":"drain","context":{"idset":"541","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714074108.0625057,"name":"drain","context":{"idset":"543","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714074108.6826661,"name":"drain","context":{"idset":"542","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714074112.3329737,"name":"drain","context":{"idset":"420","reason":"epilog failed for jobid frbYvBLD1gT","overwrite":0}}
+{"timestamp":1714074171.5527387,"name":"offline","context":{"idset":"10421"}}
+{"timestamp":1714074171.6422603,"name":"offline","context":{"idset":"10422"}}
+{"timestamp":1714074204.5976906,"name":"offline","context":{"idset":"873"}}
+{"timestamp":1714074207.6406353,"name":"offline","context":{"idset":"874"}}
+{"timestamp":1714074414.8935785,"name":"online","context":{"idset":"176"}}
+{"timestamp":1714074556.4178386,"name":"online","context":{"idset":"420"}}
+{"timestamp":1714074870.9274633,"name":"online","context":{"idset":"7711"}}
+{"timestamp":1714075497.1219304,"name":"online","context":{"idset":"8119"}}
+{"timestamp":1714075497.4935844,"name":"online","context":{"idset":"8130"}}
+{"timestamp":1714075499.2522581,"name":"online","context":{"idset":"8121,8132"}}
+{"timestamp":1714075499.255214,"name":"online","context":{"idset":"8127"}}
+{"timestamp":1714075499.259464,"name":"online","context":{"idset":"8128"}}
+{"timestamp":1714075499.2625644,"name":"online","context":{"idset":"8122,8125"}}
+{"timestamp":1714075499.265574,"name":"online","context":{"idset":"8117,8123"}}
+{"timestamp":1714075499.2684941,"name":"online","context":{"idset":"8129"}}
+{"timestamp":1714075499.2714458,"name":"online","context":{"idset":"8118,8120,8124,8126,8131"}}
+{"timestamp":1714075523.1783631,"name":"online","context":{"idset":"8136"}}
+{"timestamp":1714075524.7524567,"name":"online","context":{"idset":"8142"}}
+{"timestamp":1714075529.2779803,"name":"online","context":{"idset":"8137"}}
+{"timestamp":1714075531.204565,"name":"online","context":{"idset":"8135"}}
+{"timestamp":1714075531.2081869,"name":"online","context":{"idset":"8133,8145"}}
+{"timestamp":1714075531.2118742,"name":"online","context":{"idset":"8138,8148"}}
+{"timestamp":1714075531.2157722,"name":"online","context":{"idset":"8144"}}
+{"timestamp":1714075531.2195194,"name":"online","context":{"idset":"8134,8140,8143"}}
+{"timestamp":1714075531.2231929,"name":"online","context":{"idset":"8141"}}
+{"timestamp":1714075531.4260905,"name":"online","context":{"idset":"8146-8147"}}
+{"timestamp":1714075532.9645619,"name":"online","context":{"idset":"8139"}}
+{"timestamp":1714076077.2704811,"name":"undrain","context":{"idset":"8309-8430,8432-8436"}}
+{"timestamp":1714076984.6341228,"name":"offline","context":{"idset":"542"}}
+{"timestamp":1714076985.2398739,"name":"offline","context":{"idset":"546"}}
+{"timestamp":1714076985.4872375,"name":"offline","context":{"idset":"547"}}
+{"timestamp":1714076986.3439221,"name":"offline","context":{"idset":"544"}}
+{"timestamp":1714076986.6743994,"name":"offline","context":{"idset":"545"}}
+{"timestamp":1714076987.2005038,"name":"offline","context":{"idset":"543"}}
+{"timestamp":1714076987.4284151,"name":"offline","context":{"idset":"541"}}
+{"timestamp":1714078016.4257438,"name":"offline","context":{"idset":"871"}}
+{"timestamp":1714078019.8789787,"name":"offline","context":{"idset":"872"}}
+{"timestamp":1714078483.5168409,"name":"drain","context":{"idset":"878","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1714078511.9581754,"name":"drain","context":{"idset":"877","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1714078590.9879019,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1714078824.1035175,"name":"drain","context":{"idset":"191","reason":"unreachable","overwrite":0}}
+{"timestamp":1714079045.0188537,"name":"drain","context":{"idset":"85-92","reason":"New Blade installation","overwrite":1}}
+{"timestamp":1714079094.8226483,"name":"offline","context":{"idset":"86"}}
+{"timestamp":1714079094.8370128,"name":"offline","context":{"idset":"85"}}
+{"timestamp":1714079094.8407881,"name":"offline","context":{"idset":"91"}}
+{"timestamp":1714079094.9370513,"name":"offline","context":{"idset":"87"}}
+{"timestamp":1714079095.027436,"name":"offline","context":{"idset":"88"}}
+{"timestamp":1714079095.0338485,"name":"offline","context":{"idset":"89"}}
+{"timestamp":1714079095.1272011,"name":"offline","context":{"idset":"92"}}
+{"timestamp":1714079186.545382,"name":"online","context":{"idset":"8153"}}
+{"timestamp":1714079190.6990929,"name":"online","context":{"idset":"8150"}}
+{"timestamp":1714079191.6031277,"name":"online","context":{"idset":"115"}}
+{"timestamp":1714079192.6787324,"name":"online","context":{"idset":"8156"}}
+{"timestamp":1714079192.9694259,"name":"online","context":{"idset":"8160"}}
+{"timestamp":1714079193.4786594,"name":"online","context":{"idset":"8161"}}
+{"timestamp":1714079194.6842103,"name":"online","context":{"idset":"8154,8164"}}
+{"timestamp":1714079194.6879208,"name":"online","context":{"idset":"8149"}}
+{"timestamp":1714079194.8986681,"name":"online","context":{"idset":"8152,8155,8157-8158"}}
+{"timestamp":1714079195.0968163,"name":"online","context":{"idset":"8162"}}
+{"timestamp":1714079195.2330627,"name":"online","context":{"idset":"8159"}}
+{"timestamp":1714079195.4145908,"name":"online","context":{"idset":"8163"}}
+{"timestamp":1714079196.715126,"name":"online","context":{"idset":"8151"}}
+{"timestamp":1714079382.7167945,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1714079492.7502766,"name":"drain","context":{"idset":"115","reason":"unreachable","overwrite":0}}
+{"timestamp":1714079560.9911604,"name":"drain","context":{"idset":"110","reason":"unreachable","overwrite":0}}
+{"timestamp":1714079564.9202528,"name":"drain","context":{"idset":"119","reason":"unreachable","overwrite":0}}
+{"timestamp":1714079944.1354983,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1714079944.1408315,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1714079944.1458874,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1714079944.1511972,"name":"offline","context":{"idset":"190"}}
+{"timestamp":1714079944.1570604,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1714079944.253521,"name":"offline","context":{"idset":"319"}}
+{"timestamp":1714079946.1572042,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1714079946.1605442,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1714079946.1765499,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1714079946.179646,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1714079946.1914582,"name":"offline","context":{"idset":"184"}}
+{"timestamp":1714079946.1957729,"name":"offline","context":{"idset":"187"}}
+{"timestamp":1714079946.248755,"name":"offline","context":{"idset":"317"}}
+{"timestamp":1714079948.1245351,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1714079948.1279168,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1714079948.131413,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1714079948.2068765,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1714079954.1205485,"name":"offline","context":{"idset":"208"}}
+{"timestamp":1714079954.2084615,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1714079956.1214714,"name":"drain","context":{"idset":"98,100,102,111-112,114,119,123-124,161,184,187,190,208,303-304,310,317,319,330","reason":"epilog failed for jobid frcnKxdu6xP","overwrite":0}}
+{"timestamp":1714079956.2076297,"name":"offline","context":{"idset":"161"}}
+{"timestamp":1714079999.2806783,"name":"drain","context":{"idset":"6517-6644","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714080099.7606182,"name":"offline","context":{"idset":"883"}}
+{"timestamp":1714080104.3648298,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1714080104.3677735,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1714080104.3706903,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1714080104.3735452,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1714080104.3763876,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1714080104.3792191,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1714080104.3820636,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1714080104.3848941,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1714080104.3877349,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1714080104.390574,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1714080104.3934102,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1714080104.3962622,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1714080104.3991022,"name":"offline","context":{"idset":"884"}}
+{"timestamp":1714080104.4019437,"name":"offline","context":{"idset":"7685"}}
+{"timestamp":1714080104.4047749,"name":"offline","context":{"idset":"7733"}}
+{"timestamp":1714080104.4163799,"name":"offline","context":{"idset":"7746"}}
+{"timestamp":1714080104.4199188,"name":"offline","context":{"idset":"7747"}}
+{"timestamp":1714080104.4234214,"name":"offline","context":{"idset":"7979"}}
+{"timestamp":1714080104.4269607,"name":"offline","context":{"idset":"7991"}}
+{"timestamp":1714080104.4305086,"name":"offline","context":{"idset":"7998"}}
+{"timestamp":1714080104.4445803,"name":"offline","context":{"idset":"8012"}}
+{"timestamp":1714080104.448034,"name":"offline","context":{"idset":"8022"}}
+{"timestamp":1714080104.4515016,"name":"offline","context":{"idset":"8032"}}
+{"timestamp":1714080104.4549277,"name":"offline","context":{"idset":"8041"}}
+{"timestamp":1714080104.4583931,"name":"offline","context":{"idset":"8044"}}
+{"timestamp":1714080104.4822593,"name":"offline","context":{"idset":"8046"}}
+{"timestamp":1714080104.4856386,"name":"offline","context":{"idset":"8047"}}
+{"timestamp":1714080104.4889855,"name":"offline","context":{"idset":"8048"}}
+{"timestamp":1714080104.4923468,"name":"offline","context":{"idset":"8050"}}
+{"timestamp":1714080104.49564,"name":"offline","context":{"idset":"8051"}}
+{"timestamp":1714080104.5090754,"name":"offline","context":{"idset":"8052"}}
+{"timestamp":1714080104.5123436,"name":"offline","context":{"idset":"8150"}}
+{"timestamp":1714080104.5156226,"name":"offline","context":{"idset":"8151"}}
+{"timestamp":1714080104.5191031,"name":"offline","context":{"idset":"8152"}}
+{"timestamp":1714080104.5225379,"name":"offline","context":{"idset":"8153"}}
+{"timestamp":1714080104.5459447,"name":"offline","context":{"idset":"8154"}}
+{"timestamp":1714080104.5493834,"name":"offline","context":{"idset":"8155"}}
+{"timestamp":1714080104.5524068,"name":"offline","context":{"idset":"8156"}}
+{"timestamp":1714080104.555403,"name":"offline","context":{"idset":"8157"}}
+{"timestamp":1714080104.558409,"name":"offline","context":{"idset":"8158"}}
+{"timestamp":1714080104.5719037,"name":"offline","context":{"idset":"8159"}}
+{"timestamp":1714080104.5752206,"name":"offline","context":{"idset":"8160"}}
+{"timestamp":1714080104.578784,"name":"offline","context":{"idset":"8161"}}
+{"timestamp":1714080104.5823562,"name":"offline","context":{"idset":"8162"}}
+{"timestamp":1714080104.5858078,"name":"offline","context":{"idset":"8163"}}
+{"timestamp":1714080104.5994925,"name":"offline","context":{"idset":"8164"}}
+{"timestamp":1714080104.6031644,"name":"offline","context":{"idset":"8169"}}
+{"timestamp":1714080104.6068375,"name":"offline","context":{"idset":"8170"}}
+{"timestamp":1714080104.6097438,"name":"offline","context":{"idset":"8171"}}
+{"timestamp":1714080104.6127203,"name":"offline","context":{"idset":"8172"}}
+{"timestamp":1714080104.6332383,"name":"offline","context":{"idset":"8173"}}
+{"timestamp":1714080104.6360648,"name":"offline","context":{"idset":"8174"}}
+{"timestamp":1714080104.6391487,"name":"offline","context":{"idset":"8175"}}
+{"timestamp":1714080104.6421223,"name":"offline","context":{"idset":"8177"}}
+{"timestamp":1714080104.6453116,"name":"offline","context":{"idset":"8178"}}
+{"timestamp":1714080104.6481659,"name":"offline","context":{"idset":"8179"}}
+{"timestamp":1714080104.6597159,"name":"offline","context":{"idset":"8180"}}
+{"timestamp":1714080104.6625509,"name":"offline","context":{"idset":"8181"}}
+{"timestamp":1714080104.6653414,"name":"offline","context":{"idset":"8182"}}
+{"timestamp":1714080104.6681764,"name":"offline","context":{"idset":"8183"}}
+{"timestamp":1714080104.6710339,"name":"offline","context":{"idset":"8184"}}
+{"timestamp":1714080104.6738958,"name":"offline","context":{"idset":"8185"}}
+{"timestamp":1714080104.6941106,"name":"offline","context":{"idset":"8186"}}
+{"timestamp":1714080104.6969399,"name":"offline","context":{"idset":"8187"}}
+{"timestamp":1714080104.7083757,"name":"offline","context":{"idset":"8188"}}
+{"timestamp":1714080104.7111754,"name":"offline","context":{"idset":"8189"}}
+{"timestamp":1714080104.713979,"name":"offline","context":{"idset":"8190"}}
+{"timestamp":1714080104.7167881,"name":"offline","context":{"idset":"8191"}}
+{"timestamp":1714080104.7196302,"name":"offline","context":{"idset":"8192"}}
+{"timestamp":1714080104.7224498,"name":"offline","context":{"idset":"8193"}}
+{"timestamp":1714080104.7252386,"name":"offline","context":{"idset":"8194"}}
+{"timestamp":1714080104.7280326,"name":"offline","context":{"idset":"8195"}}
+{"timestamp":1714080104.7308505,"name":"offline","context":{"idset":"8196"}}
+{"timestamp":1714080104.733696,"name":"offline","context":{"idset":"8197"}}
+{"timestamp":1714080104.7365308,"name":"offline","context":{"idset":"8198"}}
+{"timestamp":1714080104.739326,"name":"offline","context":{"idset":"8199"}}
+{"timestamp":1714080104.7421598,"name":"offline","context":{"idset":"8200"}}
+{"timestamp":1714080104.7449484,"name":"offline","context":{"idset":"8202"}}
+{"timestamp":1714080104.7477193,"name":"offline","context":{"idset":"8203"}}
+{"timestamp":1714080104.761961,"name":"offline","context":{"idset":"8204"}}
+{"timestamp":1714080104.7660275,"name":"offline","context":{"idset":"8205"}}
+{"timestamp":1714080104.7695093,"name":"offline","context":{"idset":"8206"}}
+{"timestamp":1714080104.7729905,"name":"offline","context":{"idset":"8207"}}
+{"timestamp":1714080104.7764597,"name":"offline","context":{"idset":"8208"}}
+{"timestamp":1714080104.7907553,"name":"offline","context":{"idset":"8209"}}
+{"timestamp":1714080104.7942486,"name":"offline","context":{"idset":"8210"}}
+{"timestamp":1714080104.7977176,"name":"offline","context":{"idset":"8211"}}
+{"timestamp":1714080104.8011944,"name":"offline","context":{"idset":"8212"}}
+{"timestamp":1714080104.8046758,"name":"offline","context":{"idset":"8213"}}
+{"timestamp":1714080104.8226411,"name":"offline","context":{"idset":"8214"}}
+{"timestamp":1714080104.828721,"name":"offline","context":{"idset":"8215"}}
+{"timestamp":1714080104.8322241,"name":"offline","context":{"idset":"8216"}}
+{"timestamp":1714080104.8356674,"name":"offline","context":{"idset":"8217"}}
+{"timestamp":1714080104.8391621,"name":"offline","context":{"idset":"8218"}}
+{"timestamp":1714080104.8540421,"name":"offline","context":{"idset":"8219"}}
+{"timestamp":1714080104.8577297,"name":"offline","context":{"idset":"8220"}}
+{"timestamp":1714080104.8724396,"name":"offline","context":{"idset":"8221"}}
+{"timestamp":1714080104.8759763,"name":"offline","context":{"idset":"8222"}}
+{"timestamp":1714080104.8794723,"name":"offline","context":{"idset":"8223"}}
+{"timestamp":1714080104.8830633,"name":"offline","context":{"idset":"8224"}}
+{"timestamp":1714080104.8865409,"name":"offline","context":{"idset":"8225"}}
+{"timestamp":1714080104.8900337,"name":"offline","context":{"idset":"8226"}}
+{"timestamp":1714080104.8942611,"name":"offline","context":{"idset":"8227"}}
+{"timestamp":1714080104.8982766,"name":"offline","context":{"idset":"8228"}}
+{"timestamp":1714080104.9032831,"name":"offline","context":{"idset":"8229"}}
+{"timestamp":1714080104.9084899,"name":"offline","context":{"idset":"8230"}}
+{"timestamp":1714080104.9136875,"name":"offline","context":{"idset":"8231"}}
+{"timestamp":1714080104.9173,"name":"offline","context":{"idset":"8232"}}
+{"timestamp":1714080104.9208651,"name":"offline","context":{"idset":"8235"}}
+{"timestamp":1714080104.9244263,"name":"offline","context":{"idset":"8236"}}
+{"timestamp":1714080104.9394014,"name":"offline","context":{"idset":"8237"}}
+{"timestamp":1714080104.942879,"name":"offline","context":{"idset":"8238"}}
+{"timestamp":1714080104.946557,"name":"offline","context":{"idset":"8239"}}
+{"timestamp":1714080104.9503169,"name":"offline","context":{"idset":"8240"}}
+{"timestamp":1714080104.9538906,"name":"offline","context":{"idset":"8241"}}
+{"timestamp":1714080104.9690294,"name":"offline","context":{"idset":"8242"}}
+{"timestamp":1714080104.9726515,"name":"offline","context":{"idset":"8243"}}
+{"timestamp":1714080104.9761474,"name":"offline","context":{"idset":"8244"}}
+{"timestamp":1714080104.9797924,"name":"offline","context":{"idset":"8245"}}
+{"timestamp":1714080104.9832838,"name":"offline","context":{"idset":"8246"}}
+{"timestamp":1714080104.998117,"name":"offline","context":{"idset":"8247"}}
+{"timestamp":1714080105.0015919,"name":"offline","context":{"idset":"8248"}}
+{"timestamp":1714080105.0052528,"name":"offline","context":{"idset":"8249"}}
+{"timestamp":1714080105.0088899,"name":"offline","context":{"idset":"8250"}}
+{"timestamp":1714080105.0123332,"name":"offline","context":{"idset":"8251"}}
+{"timestamp":1714080105.026962,"name":"offline","context":{"idset":"8252"}}
+{"timestamp":1714080105.0438251,"name":"offline","context":{"idset":"8253"}}
+{"timestamp":1714080105.0473428,"name":"offline","context":{"idset":"8254"}}
+{"timestamp":1714080105.0503263,"name":"offline","context":{"idset":"8255"}}
+{"timestamp":1714080105.0531554,"name":"offline","context":{"idset":"8256"}}
+{"timestamp":1714080105.0560174,"name":"offline","context":{"idset":"8257"}}
+{"timestamp":1714080105.0588648,"name":"offline","context":{"idset":"8258"}}
+{"timestamp":1714080105.0617478,"name":"offline","context":{"idset":"8259"}}
+{"timestamp":1714080105.0645859,"name":"offline","context":{"idset":"8260"}}
+{"timestamp":1714080105.0674343,"name":"offline","context":{"idset":"8261"}}
+{"timestamp":1714080105.0702446,"name":"offline","context":{"idset":"8262"}}
+{"timestamp":1714080105.0730593,"name":"offline","context":{"idset":"8263"}}
+{"timestamp":1714080105.0759654,"name":"offline","context":{"idset":"8264"}}
+{"timestamp":1714080105.0805874,"name":"offline","context":{"idset":"8265"}}
+{"timestamp":1714080105.0835135,"name":"offline","context":{"idset":"8266"}}
+{"timestamp":1714080105.0879645,"name":"offline","context":{"idset":"8267"}}
+{"timestamp":1714080105.092314,"name":"offline","context":{"idset":"8268"}}
+{"timestamp":1714080105.0952265,"name":"offline","context":{"idset":"8269"}}
+{"timestamp":1714080105.0981789,"name":"offline","context":{"idset":"8270"}}
+{"timestamp":1714080105.1011994,"name":"offline","context":{"idset":"8271"}}
+{"timestamp":1714080105.1041632,"name":"offline","context":{"idset":"8272"}}
+{"timestamp":1714080105.1071188,"name":"offline","context":{"idset":"8273"}}
+{"timestamp":1714080105.1101792,"name":"offline","context":{"idset":"8274"}}
+{"timestamp":1714080105.1134033,"name":"offline","context":{"idset":"8275"}}
+{"timestamp":1714080105.1163805,"name":"offline","context":{"idset":"8276"}}
+{"timestamp":1714080105.119426,"name":"offline","context":{"idset":"8277"}}
+{"timestamp":1714080105.1225884,"name":"offline","context":{"idset":"8278"}}
+{"timestamp":1714080105.1256974,"name":"offline","context":{"idset":"8279"}}
+{"timestamp":1714080105.1290026,"name":"offline","context":{"idset":"8280"}}
+{"timestamp":1714080105.1322775,"name":"offline","context":{"idset":"8281"}}
+{"timestamp":1714080105.1355743,"name":"offline","context":{"idset":"8282"}}
+{"timestamp":1714080105.1391857,"name":"offline","context":{"idset":"8283"}}
+{"timestamp":1714080105.1540852,"name":"offline","context":{"idset":"8284"}}
+{"timestamp":1714080105.1576607,"name":"offline","context":{"idset":"8285"}}
+{"timestamp":1714080105.1611283,"name":"offline","context":{"idset":"8286"}}
+{"timestamp":1714080105.1644726,"name":"offline","context":{"idset":"8287"}}
+{"timestamp":1714080105.1678247,"name":"offline","context":{"idset":"8288"}}
+{"timestamp":1714080105.1819358,"name":"offline","context":{"idset":"8289"}}
+{"timestamp":1714080105.1848958,"name":"offline","context":{"idset":"8290"}}
+{"timestamp":1714080105.1878617,"name":"offline","context":{"idset":"8291"}}
+{"timestamp":1714080105.1907437,"name":"offline","context":{"idset":"8292"}}
+{"timestamp":1714080105.1936426,"name":"offline","context":{"idset":"8293"}}
+{"timestamp":1714080105.2057548,"name":"offline","context":{"idset":"8294"}}
+{"timestamp":1714080105.2086148,"name":"offline","context":{"idset":"8295"}}
+{"timestamp":1714080105.2211244,"name":"offline","context":{"idset":"8296"}}
+{"timestamp":1714080105.2339566,"name":"offline","context":{"idset":"8297"}}
+{"timestamp":1714080105.2367496,"name":"offline","context":{"idset":"8298"}}
+{"timestamp":1714080105.2395322,"name":"offline","context":{"idset":"8299"}}
+{"timestamp":1714080105.2423139,"name":"offline","context":{"idset":"8300"}}
+{"timestamp":1714080105.2461269,"name":"offline","context":{"idset":"8301"}}
+{"timestamp":1714080105.2488763,"name":"offline","context":{"idset":"8302"}}
+{"timestamp":1714080105.2515876,"name":"offline","context":{"idset":"8303"}}
+{"timestamp":1714080105.2542875,"name":"offline","context":{"idset":"8305"}}
+{"timestamp":1714080105.2572031,"name":"offline","context":{"idset":"8306"}}
+{"timestamp":1714080105.2601991,"name":"offline","context":{"idset":"8307"}}
+{"timestamp":1714080105.262881,"name":"offline","context":{"idset":"8308"}}
+{"timestamp":1714080105.2655547,"name":"offline","context":{"idset":"8309"}}
+{"timestamp":1714080105.2682343,"name":"offline","context":{"idset":"8310"}}
+{"timestamp":1714080105.2709172,"name":"offline","context":{"idset":"8311"}}
+{"timestamp":1714080105.2735937,"name":"offline","context":{"idset":"8312"}}
+{"timestamp":1714080105.2762954,"name":"offline","context":{"idset":"8313"}}
+{"timestamp":1714080105.2790151,"name":"offline","context":{"idset":"8314"}}
+{"timestamp":1714080105.2817097,"name":"offline","context":{"idset":"8315"}}
+{"timestamp":1714080105.2930708,"name":"offline","context":{"idset":"8316"}}
+{"timestamp":1714080105.295753,"name":"offline","context":{"idset":"8317"}}
+{"timestamp":1714080105.2984643,"name":"offline","context":{"idset":"8318"}}
+{"timestamp":1714080105.3011374,"name":"offline","context":{"idset":"8319"}}
+{"timestamp":1714080105.3038509,"name":"offline","context":{"idset":"8320"}}
+{"timestamp":1714080105.3151987,"name":"offline","context":{"idset":"8321"}}
+{"timestamp":1714080105.3178949,"name":"offline","context":{"idset":"8322"}}
+{"timestamp":1714080105.3206887,"name":"offline","context":{"idset":"8323"}}
+{"timestamp":1714080105.3234684,"name":"offline","context":{"idset":"8324"}}
+{"timestamp":1714080105.32636,"name":"offline","context":{"idset":"8325"}}
+{"timestamp":1714080105.3397667,"name":"offline","context":{"idset":"8326"}}
+{"timestamp":1714080105.3430505,"name":"offline","context":{"idset":"8327"}}
+{"timestamp":1714080105.3459425,"name":"offline","context":{"idset":"8328"}}
+{"timestamp":1714080105.3487291,"name":"offline","context":{"idset":"8329"}}
+{"timestamp":1714080105.3517964,"name":"offline","context":{"idset":"8330"}}
+{"timestamp":1714080105.364784,"name":"offline","context":{"idset":"8331"}}
+{"timestamp":1714080105.3774414,"name":"offline","context":{"idset":"8332"}}
+{"timestamp":1714080105.3804572,"name":"offline","context":{"idset":"8333"}}
+{"timestamp":1714080105.3833983,"name":"offline","context":{"idset":"8334"}}
+{"timestamp":1714080105.3865869,"name":"offline","context":{"idset":"8335"}}
+{"timestamp":1714080105.3892834,"name":"offline","context":{"idset":"8336"}}
+{"timestamp":1714080105.3921573,"name":"offline","context":{"idset":"8337"}}
+{"timestamp":1714080105.3948684,"name":"offline","context":{"idset":"8338"}}
+{"timestamp":1714080105.3975506,"name":"offline","context":{"idset":"8339"}}
+{"timestamp":1714080105.4002235,"name":"offline","context":{"idset":"8340"}}
+{"timestamp":1714080105.4029241,"name":"offline","context":{"idset":"8341"}}
+{"timestamp":1714080105.4056499,"name":"offline","context":{"idset":"8342"}}
+{"timestamp":1714080105.4086096,"name":"offline","context":{"idset":"8343"}}
+{"timestamp":1714080105.4127512,"name":"offline","context":{"idset":"8344"}}
+{"timestamp":1714080105.416748,"name":"offline","context":{"idset":"8345"}}
+{"timestamp":1714080105.4203551,"name":"offline","context":{"idset":"8346"}}
+{"timestamp":1714080105.4244232,"name":"offline","context":{"idset":"8347"}}
+{"timestamp":1714080105.4283507,"name":"offline","context":{"idset":"8348"}}
+{"timestamp":1714080105.4310625,"name":"offline","context":{"idset":"8349"}}
+{"timestamp":1714080105.4337385,"name":"offline","context":{"idset":"8350"}}
+{"timestamp":1714080105.4363911,"name":"offline","context":{"idset":"8351"}}
+{"timestamp":1714080105.4390485,"name":"offline","context":{"idset":"8352"}}
+{"timestamp":1714080105.4416981,"name":"offline","context":{"idset":"8353"}}
+{"timestamp":1714080105.4444544,"name":"offline","context":{"idset":"8354"}}
+{"timestamp":1714080105.4471302,"name":"offline","context":{"idset":"8355"}}
+{"timestamp":1714080105.4498758,"name":"offline","context":{"idset":"8356"}}
+{"timestamp":1714080105.4525371,"name":"offline","context":{"idset":"8357"}}
+{"timestamp":1714080105.4553227,"name":"offline","context":{"idset":"8358"}}
+{"timestamp":1714080105.4580286,"name":"offline","context":{"idset":"8359"}}
+{"timestamp":1714080105.4606869,"name":"offline","context":{"idset":"8360"}}
+{"timestamp":1714080105.4633627,"name":"offline","context":{"idset":"8361"}}
+{"timestamp":1714080105.4660263,"name":"offline","context":{"idset":"8362"}}
+{"timestamp":1714080105.4686604,"name":"offline","context":{"idset":"8363"}}
+{"timestamp":1714080105.4799724,"name":"offline","context":{"idset":"8364"}}
+{"timestamp":1714080105.482626,"name":"offline","context":{"idset":"8365"}}
+{"timestamp":1714080105.485285,"name":"offline","context":{"idset":"8366"}}
+{"timestamp":1714080105.4880641,"name":"offline","context":{"idset":"8367"}}
+{"timestamp":1714080105.491076,"name":"offline","context":{"idset":"8368"}}
+{"timestamp":1714080105.5030153,"name":"offline","context":{"idset":"8369"}}
+{"timestamp":1714080105.5057042,"name":"offline","context":{"idset":"8370"}}
+{"timestamp":1714080105.5083482,"name":"offline","context":{"idset":"8371"}}
+{"timestamp":1714080105.5109844,"name":"offline","context":{"idset":"8372"}}
+{"timestamp":1714080105.5137048,"name":"offline","context":{"idset":"8373"}}
+{"timestamp":1714080105.5164025,"name":"offline","context":{"idset":"8374"}}
+{"timestamp":1714080105.5277593,"name":"offline","context":{"idset":"8375"}}
+{"timestamp":1714080105.5304351,"name":"offline","context":{"idset":"8376"}}
+{"timestamp":1714080105.5420699,"name":"offline","context":{"idset":"8377"}}
+{"timestamp":1714080105.5556788,"name":"offline","context":{"idset":"8378"}}
+{"timestamp":1714080105.560539,"name":"offline","context":{"idset":"8379"}}
+{"timestamp":1714080105.5650184,"name":"offline","context":{"idset":"8380"}}
+{"timestamp":1714080105.5687637,"name":"offline","context":{"idset":"8381"}}
+{"timestamp":1714080105.5730236,"name":"offline","context":{"idset":"8382"}}
+{"timestamp":1714080105.5778527,"name":"offline","context":{"idset":"8383"}}
+{"timestamp":1714080105.5818701,"name":"offline","context":{"idset":"8384"}}
+{"timestamp":1714080105.5859165,"name":"offline","context":{"idset":"8385"}}
+{"timestamp":1714080105.5904791,"name":"offline","context":{"idset":"8386"}}
+{"timestamp":1714080105.5954118,"name":"offline","context":{"idset":"8387"}}
+{"timestamp":1714080105.59904,"name":"offline","context":{"idset":"8388"}}
+{"timestamp":1714080105.603117,"name":"offline","context":{"idset":"8389"}}
+{"timestamp":1714080105.6061246,"name":"offline","context":{"idset":"8390"}}
+{"timestamp":1714080105.6087837,"name":"offline","context":{"idset":"8391"}}
+{"timestamp":1714080105.6114466,"name":"offline","context":{"idset":"8392"}}
+{"timestamp":1714080105.614099,"name":"offline","context":{"idset":"8393"}}
+{"timestamp":1714080105.6167085,"name":"offline","context":{"idset":"8394"}}
+{"timestamp":1714080105.6193557,"name":"offline","context":{"idset":"8395"}}
+{"timestamp":1714080105.6220307,"name":"offline","context":{"idset":"8396"}}
+{"timestamp":1714080105.6343093,"name":"offline","context":{"idset":"8397"}}
+{"timestamp":1714080105.6369879,"name":"offline","context":{"idset":"8398"}}
+{"timestamp":1714080105.6397052,"name":"offline","context":{"idset":"8399"}}
+{"timestamp":1714080105.6423934,"name":"offline","context":{"idset":"8400"}}
+{"timestamp":1714080105.645057,"name":"offline","context":{"idset":"8401"}}
+{"timestamp":1714080105.6565564,"name":"offline","context":{"idset":"8402"}}
+{"timestamp":1714080105.6592264,"name":"offline","context":{"idset":"8403"}}
+{"timestamp":1714080105.6620147,"name":"offline","context":{"idset":"8404"}}
+{"timestamp":1714080105.664664,"name":"offline","context":{"idset":"8405"}}
+{"timestamp":1714080105.6673663,"name":"offline","context":{"idset":"8406"}}
+{"timestamp":1714080105.6803122,"name":"offline","context":{"idset":"8407"}}
+{"timestamp":1714080105.6837134,"name":"offline","context":{"idset":"8408"}}
+{"timestamp":1714080105.6863954,"name":"offline","context":{"idset":"8409"}}
+{"timestamp":1714080105.6891499,"name":"offline","context":{"idset":"8410"}}
+{"timestamp":1714080105.6918409,"name":"offline","context":{"idset":"8411"}}
+{"timestamp":1714080105.703862,"name":"offline","context":{"idset":"8412"}}
+{"timestamp":1714080105.7152545,"name":"offline","context":{"idset":"8413"}}
+{"timestamp":1714080105.7179174,"name":"offline","context":{"idset":"8414"}}
+{"timestamp":1714080105.7206435,"name":"offline","context":{"idset":"8415"}}
+{"timestamp":1714080105.723346,"name":"offline","context":{"idset":"8416"}}
+{"timestamp":1714080105.7260687,"name":"offline","context":{"idset":"8417"}}
+{"timestamp":1714080105.7287204,"name":"offline","context":{"idset":"8418"}}
+{"timestamp":1714080105.7314005,"name":"offline","context":{"idset":"8419"}}
+{"timestamp":1714080105.7341404,"name":"offline","context":{"idset":"8420"}}
+{"timestamp":1714080105.7368391,"name":"offline","context":{"idset":"8421"}}
+{"timestamp":1714080105.7395957,"name":"offline","context":{"idset":"8422"}}
+{"timestamp":1714080105.7424078,"name":"offline","context":{"idset":"8423"}}
+{"timestamp":1714080105.7453027,"name":"offline","context":{"idset":"8424"}}
+{"timestamp":1714080105.7480924,"name":"offline","context":{"idset":"8425"}}
+{"timestamp":1714080105.7508416,"name":"offline","context":{"idset":"8426"}}
+{"timestamp":1714080105.7535679,"name":"offline","context":{"idset":"8427"}}
+{"timestamp":1714080105.756237,"name":"offline","context":{"idset":"8428"}}
+{"timestamp":1714080105.7588911,"name":"offline","context":{"idset":"8429"}}
+{"timestamp":1714080105.761512,"name":"offline","context":{"idset":"8430"}}
+{"timestamp":1714080105.7641721,"name":"offline","context":{"idset":"8432"}}
+{"timestamp":1714080105.7756391,"name":"offline","context":{"idset":"8433"}}
+{"timestamp":1714080105.7783768,"name":"offline","context":{"idset":"8434"}}
+{"timestamp":1714080105.7811942,"name":"offline","context":{"idset":"8435"}}
+{"timestamp":1714080105.7840047,"name":"offline","context":{"idset":"8436"}}
+{"timestamp":1714080105.7867315,"name":"offline","context":{"idset":"8597"}}
+{"timestamp":1714080105.798084,"name":"offline","context":{"idset":"8598"}}
+{"timestamp":1714080105.8007207,"name":"offline","context":{"idset":"8599"}}
+{"timestamp":1714080105.8033421,"name":"offline","context":{"idset":"8600"}}
+{"timestamp":1714080105.8059537,"name":"offline","context":{"idset":"8601"}}
+{"timestamp":1714080105.8086002,"name":"offline","context":{"idset":"8602"}}
+{"timestamp":1714080105.8198719,"name":"offline","context":{"idset":"8603"}}
+{"timestamp":1714080105.8225274,"name":"offline","context":{"idset":"8604"}}
+{"timestamp":1714080105.8251996,"name":"offline","context":{"idset":"8605"}}
+{"timestamp":1714080105.8279047,"name":"offline","context":{"idset":"8606"}}
+{"timestamp":1714080105.8305612,"name":"offline","context":{"idset":"8607"}}
+{"timestamp":1714080105.8419557,"name":"offline","context":{"idset":"8608"}}
+{"timestamp":1714080105.8531883,"name":"offline","context":{"idset":"8609"}}
+{"timestamp":1714080105.8558292,"name":"offline","context":{"idset":"8610"}}
+{"timestamp":1714080105.8585696,"name":"offline","context":{"idset":"8611"}}
+{"timestamp":1714080105.861264,"name":"offline","context":{"idset":"8612"}}
+{"timestamp":1714080105.8640513,"name":"offline","context":{"idset":"8821"}}
+{"timestamp":1714080105.8667214,"name":"offline","context":{"idset":"8822"}}
+{"timestamp":1714080105.8694634,"name":"offline","context":{"idset":"8823"}}
+{"timestamp":1714080105.8721917,"name":"offline","context":{"idset":"8824"}}
+{"timestamp":1714080105.8749959,"name":"offline","context":{"idset":"8825"}}
+{"timestamp":1714080105.8777626,"name":"offline","context":{"idset":"8826"}}
+{"timestamp":1714080105.8804197,"name":"offline","context":{"idset":"8827"}}
+{"timestamp":1714080105.8831506,"name":"offline","context":{"idset":"8828"}}
+{"timestamp":1714080105.8858089,"name":"offline","context":{"idset":"8829"}}
+{"timestamp":1714080105.8884313,"name":"offline","context":{"idset":"8830"}}
+{"timestamp":1714080105.8909936,"name":"offline","context":{"idset":"8831"}}
+{"timestamp":1714080105.8936141,"name":"offline","context":{"idset":"8832"}}
+{"timestamp":1714080105.8963668,"name":"offline","context":{"idset":"8834"}}
+{"timestamp":1714080105.8991003,"name":"offline","context":{"idset":"8835"}}
+{"timestamp":1714080105.901876,"name":"offline","context":{"idset":"8836"}}
+{"timestamp":1714080105.9048512,"name":"offline","context":{"idset":"8837"}}
+{"timestamp":1714080105.9075987,"name":"offline","context":{"idset":"8838"}}
+{"timestamp":1714080105.9103363,"name":"offline","context":{"idset":"8839"}}
+{"timestamp":1714080105.9131463,"name":"offline","context":{"idset":"8840"}}
+{"timestamp":1714080105.9158337,"name":"offline","context":{"idset":"8841"}}
+{"timestamp":1714080105.9184709,"name":"offline","context":{"idset":"8842"}}
+{"timestamp":1714080105.9210646,"name":"offline","context":{"idset":"8843"}}
+{"timestamp":1714080105.9331169,"name":"offline","context":{"idset":"8844"}}
+{"timestamp":1714080105.9359739,"name":"offline","context":{"idset":"8845"}}
+{"timestamp":1714080105.9388177,"name":"offline","context":{"idset":"8846"}}
+{"timestamp":1714080105.941587,"name":"offline","context":{"idset":"8847"}}
+{"timestamp":1714080105.9442689,"name":"offline","context":{"idset":"8848"}}
+{"timestamp":1714080105.9610488,"name":"offline","context":{"idset":"8849"}}
+{"timestamp":1714080105.9650142,"name":"offline","context":{"idset":"8850"}}
+{"timestamp":1714080105.9692278,"name":"offline","context":{"idset":"8851"}}
+{"timestamp":1714080105.9732797,"name":"offline","context":{"idset":"8852"}}
+{"timestamp":1714080105.9772487,"name":"offline","context":{"idset":"8853"}}
+{"timestamp":1714080105.9952776,"name":"offline","context":{"idset":"8854"}}
+{"timestamp":1714080106.0210516,"name":"offline","context":{"idset":"8855"}}
+{"timestamp":1714080106.0253074,"name":"offline","context":{"idset":"8856"}}
+{"timestamp":1714080106.0297582,"name":"offline","context":{"idset":"8857"}}
+{"timestamp":1714080106.0341733,"name":"offline","context":{"idset":"8858"}}
+{"timestamp":1714080106.0493543,"name":"offline","context":{"idset":"8859"}}
+{"timestamp":1714080106.060895,"name":"offline","context":{"idset":"8860"}}
+{"timestamp":1714080106.0635254,"name":"offline","context":{"idset":"8861"}}
+{"timestamp":1714080106.0660927,"name":"offline","context":{"idset":"8862"}}
+{"timestamp":1714080106.0687225,"name":"offline","context":{"idset":"8863"}}
+{"timestamp":1714080106.0724609,"name":"offline","context":{"idset":"8864"}}
+{"timestamp":1714080106.0750973,"name":"offline","context":{"idset":"8865"}}
+{"timestamp":1714080106.0776677,"name":"offline","context":{"idset":"8866"}}
+{"timestamp":1714080106.08025,"name":"offline","context":{"idset":"8867"}}
+{"timestamp":1714080106.0828848,"name":"offline","context":{"idset":"8868"}}
+{"timestamp":1714080106.0854774,"name":"offline","context":{"idset":"8869"}}
+{"timestamp":1714080106.0880787,"name":"offline","context":{"idset":"8870"}}
+{"timestamp":1714080106.0906761,"name":"offline","context":{"idset":"8871"}}
+{"timestamp":1714080106.0932202,"name":"offline","context":{"idset":"8872"}}
+{"timestamp":1714080106.09587,"name":"offline","context":{"idset":"8873"}}
+{"timestamp":1714080106.098418,"name":"offline","context":{"idset":"8874"}}
+{"timestamp":1714080106.1098678,"name":"offline","context":{"idset":"8875"}}
+{"timestamp":1714080106.1124675,"name":"offline","context":{"idset":"8876"}}
+{"timestamp":1714080106.1152279,"name":"offline","context":{"idset":"8877"}}
+{"timestamp":1714080106.1177883,"name":"offline","context":{"idset":"8878"}}
+{"timestamp":1714080106.1204128,"name":"offline","context":{"idset":"8879"}}
+{"timestamp":1714080106.1317289,"name":"offline","context":{"idset":"8880"}}
+{"timestamp":1714080106.134295,"name":"offline","context":{"idset":"8881"}}
+{"timestamp":1714080106.1368365,"name":"offline","context":{"idset":"8882"}}
+{"timestamp":1714080106.1393621,"name":"offline","context":{"idset":"8883"}}
+{"timestamp":1714080106.1419818,"name":"offline","context":{"idset":"8884"}}
+{"timestamp":1714080106.1621501,"name":"offline","context":{"idset":"8885"}}
+{"timestamp":1714080106.1646986,"name":"offline","context":{"idset":"8886"}}
+{"timestamp":1714080106.1672721,"name":"offline","context":{"idset":"8887"}}
+{"timestamp":1714080106.169836,"name":"offline","context":{"idset":"8888"}}
+{"timestamp":1714080106.1724043,"name":"offline","context":{"idset":"8889"}}
+{"timestamp":1714080106.1838319,"name":"offline","context":{"idset":"8890"}}
+{"timestamp":1714080106.186444,"name":"offline","context":{"idset":"8891"}}
+{"timestamp":1714080106.1984711,"name":"offline","context":{"idset":"8892"}}
+{"timestamp":1714080106.2016058,"name":"offline","context":{"idset":"8893"}}
+{"timestamp":1714080106.2055333,"name":"offline","context":{"idset":"8894"}}
+{"timestamp":1714080106.2093132,"name":"offline","context":{"idset":"8895"}}
+{"timestamp":1714080106.2132454,"name":"offline","context":{"idset":"8896"}}
+{"timestamp":1714080106.2170775,"name":"offline","context":{"idset":"8897"}}
+{"timestamp":1714080106.2209055,"name":"offline","context":{"idset":"8898"}}
+{"timestamp":1714080106.2247233,"name":"offline","context":{"idset":"8899"}}
+{"timestamp":1714080106.2273099,"name":"offline","context":{"idset":"8900"}}
+{"timestamp":1714080106.2298713,"name":"offline","context":{"idset":"8901"}}
+{"timestamp":1714080106.2324312,"name":"offline","context":{"idset":"8902"}}
+{"timestamp":1714080106.2350419,"name":"offline","context":{"idset":"8903"}}
+{"timestamp":1714080106.23755,"name":"offline","context":{"idset":"8904"}}
+{"timestamp":1714080106.2400844,"name":"offline","context":{"idset":"8905"}}
+{"timestamp":1714080106.2426548,"name":"offline","context":{"idset":"8906"}}
+{"timestamp":1714080106.2452765,"name":"offline","context":{"idset":"8907"}}
+{"timestamp":1714080106.2478538,"name":"offline","context":{"idset":"8908"}}
+{"timestamp":1714080106.2504261,"name":"offline","context":{"idset":"8909"}}
+{"timestamp":1714080106.252996,"name":"offline","context":{"idset":"8910"}}
+{"timestamp":1714080106.2555137,"name":"offline","context":{"idset":"8911"}}
+{"timestamp":1714080106.2580466,"name":"offline","context":{"idset":"8912"}}
+{"timestamp":1714080106.2605641,"name":"offline","context":{"idset":"8913"}}
+{"timestamp":1714080106.2631118,"name":"offline","context":{"idset":"8914"}}
+{"timestamp":1714080106.2756774,"name":"offline","context":{"idset":"8915"}}
+{"timestamp":1714080106.2873812,"name":"offline","context":{"idset":"8916"}}
+{"timestamp":1714080106.2899241,"name":"offline","context":{"idset":"8917"}}
+{"timestamp":1714080106.2924819,"name":"offline","context":{"idset":"8918"}}
+{"timestamp":1714080106.2950084,"name":"offline","context":{"idset":"8919"}}
+{"timestamp":1714080106.2975678,"name":"offline","context":{"idset":"8920"}}
+{"timestamp":1714080106.3001173,"name":"offline","context":{"idset":"8921"}}
+{"timestamp":1714080106.3027418,"name":"offline","context":{"idset":"8922"}}
+{"timestamp":1714080106.3053508,"name":"offline","context":{"idset":"8923"}}
+{"timestamp":1714080106.3079276,"name":"offline","context":{"idset":"8924"}}
+{"timestamp":1714080106.3105342,"name":"offline","context":{"idset":"8925"}}
+{"timestamp":1714080106.3131034,"name":"offline","context":{"idset":"8926"}}
+{"timestamp":1714080106.3156757,"name":"offline","context":{"idset":"8927"}}
+{"timestamp":1714080106.3182058,"name":"offline","context":{"idset":"8928"}}
+{"timestamp":1714080106.320745,"name":"offline","context":{"idset":"8929"}}
+{"timestamp":1714080106.3232765,"name":"offline","context":{"idset":"8930"}}
+{"timestamp":1714080106.3258166,"name":"offline","context":{"idset":"8931"}}
+{"timestamp":1714080106.3283391,"name":"offline","context":{"idset":"8932"}}
+{"timestamp":1714080106.330853,"name":"offline","context":{"idset":"8933"}}
+{"timestamp":1714080106.3334038,"name":"offline","context":{"idset":"8934"}}
+{"timestamp":1714080106.3448319,"name":"offline","context":{"idset":"8935"}}
+{"timestamp":1714080106.3473661,"name":"offline","context":{"idset":"8936"}}
+{"timestamp":1714080106.3499212,"name":"offline","context":{"idset":"8937"}}
+{"timestamp":1714080106.352459,"name":"offline","context":{"idset":"8938"}}
+{"timestamp":1714080106.3550417,"name":"offline","context":{"idset":"8939"}}
+{"timestamp":1714080106.3575792,"name":"offline","context":{"idset":"8940"}}
+{"timestamp":1714080106.3690095,"name":"offline","context":{"idset":"8941"}}
+{"timestamp":1714080106.3716807,"name":"offline","context":{"idset":"8942"}}
+{"timestamp":1714080106.3742919,"name":"offline","context":{"idset":"8943"}}
+{"timestamp":1714080106.3768184,"name":"offline","context":{"idset":"8944"}}
+{"timestamp":1714080106.379442,"name":"offline","context":{"idset":"8945"}}
+{"timestamp":1714080106.399462,"name":"offline","context":{"idset":"8946"}}
+{"timestamp":1714080106.402045,"name":"offline","context":{"idset":"8947"}}
+{"timestamp":1714080106.4045668,"name":"offline","context":{"idset":"8948"}}
+{"timestamp":1714080106.4071624,"name":"offline","context":{"idset":"8949"}}
+{"timestamp":1714080106.4097133,"name":"offline","context":{"idset":"8950"}}
+{"timestamp":1714080106.4211981,"name":"offline","context":{"idset":"8951"}}
+{"timestamp":1714080106.4237397,"name":"offline","context":{"idset":"8952"}}
+{"timestamp":1714080106.4262793,"name":"offline","context":{"idset":"8953"}}
+{"timestamp":1714080106.428848,"name":"offline","context":{"idset":"8954"}}
+{"timestamp":1714080106.431392,"name":"offline","context":{"idset":"8955"}}
+{"timestamp":1714080106.4514973,"name":"offline","context":{"idset":"8956"}}
+{"timestamp":1714080106.4539561,"name":"offline","context":{"idset":"8957"}}
+{"timestamp":1714080106.4564092,"name":"offline","context":{"idset":"8958"}}
+{"timestamp":1714080106.4588583,"name":"offline","context":{"idset":"8959"}}
+{"timestamp":1714080106.461319,"name":"offline","context":{"idset":"8960"}}
+{"timestamp":1714080106.4805775,"name":"offline","context":{"idset":"8961"}}
+{"timestamp":1714080106.4829769,"name":"offline","context":{"idset":"8962"}}
+{"timestamp":1714080106.485415,"name":"offline","context":{"idset":"8963"}}
+{"timestamp":1714080106.4886429,"name":"offline","context":{"idset":"8964"}}
+{"timestamp":1714080106.4920623,"name":"offline","context":{"idset":"8965"}}
+{"timestamp":1714080106.4959335,"name":"offline","context":{"idset":"8966"}}
+{"timestamp":1714080106.5082653,"name":"offline","context":{"idset":"8967"}}
+{"timestamp":1714080106.5148637,"name":"offline","context":{"idset":"8968"}}
+{"timestamp":1714080106.5191453,"name":"offline","context":{"idset":"8969"}}
+{"timestamp":1714080106.5234933,"name":"offline","context":{"idset":"8970"}}
+{"timestamp":1714080106.5278084,"name":"offline","context":{"idset":"8971"}}
+{"timestamp":1714080106.5510139,"name":"offline","context":{"idset":"8972"}}
+{"timestamp":1714080106.5538125,"name":"offline","context":{"idset":"8973"}}
+{"timestamp":1714080106.5565817,"name":"offline","context":{"idset":"8974"}}
+{"timestamp":1714080106.5607281,"name":"offline","context":{"idset":"8975"}}
+{"timestamp":1714080106.5662265,"name":"offline","context":{"idset":"8976"}}
+{"timestamp":1714080106.5716083,"name":"offline","context":{"idset":"8977"}}
+{"timestamp":1714080106.5874767,"name":"offline","context":{"idset":"8978"}}
+{"timestamp":1714080106.5901983,"name":"offline","context":{"idset":"8979"}}
+{"timestamp":1714080106.6024873,"name":"offline","context":{"idset":"8980"}}
+{"timestamp":1714080106.6146407,"name":"offline","context":{"idset":"8981"}}
+{"timestamp":1714080106.6172874,"name":"offline","context":{"idset":"8982"}}
+{"timestamp":1714080106.6199117,"name":"offline","context":{"idset":"8983"}}
+{"timestamp":1714080106.6225162,"name":"offline","context":{"idset":"8984"}}
+{"timestamp":1714080106.6251113,"name":"offline","context":{"idset":"8985"}}
+{"timestamp":1714080106.6277175,"name":"offline","context":{"idset":"8986"}}
+{"timestamp":1714080106.6303248,"name":"offline","context":{"idset":"8987"}}
+{"timestamp":1714080106.6329427,"name":"offline","context":{"idset":"8988"}}
+{"timestamp":1714080106.6355517,"name":"offline","context":{"idset":"8989"}}
+{"timestamp":1714080106.6382656,"name":"offline","context":{"idset":"8990"}}
+{"timestamp":1714080106.640939,"name":"offline","context":{"idset":"8991"}}
+{"timestamp":1714080106.6435642,"name":"offline","context":{"idset":"8992"}}
+{"timestamp":1714080106.6462176,"name":"offline","context":{"idset":"8993"}}
+{"timestamp":1714080106.6487699,"name":"offline","context":{"idset":"8994"}}
+{"timestamp":1714080106.6513047,"name":"offline","context":{"idset":"8995"}}
+{"timestamp":1714080106.6538363,"name":"offline","context":{"idset":"8996"}}
+{"timestamp":1714080106.6563461,"name":"offline","context":{"idset":"8997"}}
+{"timestamp":1714080106.6588547,"name":"offline","context":{"idset":"8998"}}
+{"timestamp":1714080106.6613576,"name":"offline","context":{"idset":"8999"}}
+{"timestamp":1714080106.6638505,"name":"offline","context":{"idset":"9000"}}
+{"timestamp":1714080106.6663306,"name":"offline","context":{"idset":"9001"}}
+{"timestamp":1714080106.6690276,"name":"offline","context":{"idset":"9002"}}
+{"timestamp":1714080106.6830552,"name":"offline","context":{"idset":"9005"}}
+{"timestamp":1714080106.6866975,"name":"offline","context":{"idset":"9006"}}
+{"timestamp":1714080106.690321,"name":"offline","context":{"idset":"9007"}}
+{"timestamp":1714080106.6939306,"name":"offline","context":{"idset":"9008"}}
+{"timestamp":1714080106.696435,"name":"offline","context":{"idset":"9009"}}
+{"timestamp":1714080106.6989493,"name":"offline","context":{"idset":"9010"}}
+{"timestamp":1714080106.7102571,"name":"offline","context":{"idset":"9011"}}
+{"timestamp":1714080106.7126713,"name":"offline","context":{"idset":"9012"}}
+{"timestamp":1714080106.715085,"name":"offline","context":{"idset":"9013"}}
+{"timestamp":1714080106.7175064,"name":"offline","context":{"idset":"9014"}}
+{"timestamp":1714080106.7199371,"name":"offline","context":{"idset":"9015"}}
+{"timestamp":1714080106.7223957,"name":"offline","context":{"idset":"9016"}}
+{"timestamp":1714080106.7335453,"name":"offline","context":{"idset":"9017"}}
+{"timestamp":1714080106.7359409,"name":"offline","context":{"idset":"9018"}}
+{"timestamp":1714080106.7382944,"name":"offline","context":{"idset":"9019"}}
+{"timestamp":1714080106.7406683,"name":"offline","context":{"idset":"9020"}}
+{"timestamp":1714080106.7430854,"name":"offline","context":{"idset":"9021"}}
+{"timestamp":1714080106.7455108,"name":"offline","context":{"idset":"9022"}}
+{"timestamp":1714080106.7562881,"name":"offline","context":{"idset":"9023"}}
+{"timestamp":1714080106.7695413,"name":"offline","context":{"idset":"9024"}}
+{"timestamp":1714080106.7727375,"name":"offline","context":{"idset":"9025"}}
+{"timestamp":1714080106.7750473,"name":"offline","context":{"idset":"9026"}}
+{"timestamp":1714080106.777329,"name":"offline","context":{"idset":"9027"}}
+{"timestamp":1714080106.7795894,"name":"offline","context":{"idset":"9028"}}
+{"timestamp":1714080106.7819209,"name":"offline","context":{"idset":"9029"}}
+{"timestamp":1714080106.7843812,"name":"offline","context":{"idset":"9030"}}
+{"timestamp":1714080106.786736,"name":"offline","context":{"idset":"9031"}}
+{"timestamp":1714080106.7893023,"name":"offline","context":{"idset":"9032"}}
+{"timestamp":1714080106.7916353,"name":"offline","context":{"idset":"9033"}}
+{"timestamp":1714080106.7938902,"name":"offline","context":{"idset":"9034"}}
+{"timestamp":1714080106.7961283,"name":"offline","context":{"idset":"9035"}}
+{"timestamp":1714080106.7983346,"name":"offline","context":{"idset":"9036"}}
+{"timestamp":1714080106.8005359,"name":"offline","context":{"idset":"9037"}}
+{"timestamp":1714080106.8027911,"name":"offline","context":{"idset":"9038"}}
+{"timestamp":1714080106.8049893,"name":"offline","context":{"idset":"9039"}}
+{"timestamp":1714080106.8071997,"name":"offline","context":{"idset":"9040"}}
+{"timestamp":1714080106.8093939,"name":"offline","context":{"idset":"9041"}}
+{"timestamp":1714080106.811578,"name":"offline","context":{"idset":"9042"}}
+{"timestamp":1714080106.8137653,"name":"offline","context":{"idset":"9045"}}
+{"timestamp":1714080106.8159332,"name":"offline","context":{"idset":"9046"}}
+{"timestamp":1714080106.8181334,"name":"offline","context":{"idset":"9047"}}
+{"timestamp":1714080106.8282464,"name":"offline","context":{"idset":"9048"}}
+{"timestamp":1714080106.8304374,"name":"offline","context":{"idset":"9049"}}
+{"timestamp":1714080106.8325996,"name":"offline","context":{"idset":"9050"}}
+{"timestamp":1714080106.8352149,"name":"offline","context":{"idset":"9051"}}
+{"timestamp":1714080106.8380237,"name":"offline","context":{"idset":"9052"}}
+{"timestamp":1714080106.8409183,"name":"offline","context":{"idset":"9053"}}
+{"timestamp":1714080106.8547833,"name":"offline","context":{"idset":"9054"}}
+{"timestamp":1714080106.8574879,"name":"offline","context":{"idset":"9055"}}
+{"timestamp":1714080106.859653,"name":"offline","context":{"idset":"9056"}}
+{"timestamp":1714080106.8618848,"name":"offline","context":{"idset":"9057"}}
+{"timestamp":1714080106.8640058,"name":"offline","context":{"idset":"9058"}}
+{"timestamp":1714080106.8665428,"name":"offline","context":{"idset":"9059"}}
+{"timestamp":1714080106.8762107,"name":"offline","context":{"idset":"9060"}}
+{"timestamp":1714080106.8782797,"name":"offline","context":{"idset":"9061"}}
+{"timestamp":1714080106.8803411,"name":"offline","context":{"idset":"9062"}}
+{"timestamp":1714080106.8823864,"name":"offline","context":{"idset":"9063"}}
+{"timestamp":1714080106.8844545,"name":"offline","context":{"idset":"9064"}}
+{"timestamp":1714080106.8864849,"name":"offline","context":{"idset":"9065"}}
+{"timestamp":1714080106.8962543,"name":"offline","context":{"idset":"9066"}}
+{"timestamp":1714080106.9057093,"name":"offline","context":{"idset":"9067"}}
+{"timestamp":1714080106.9077704,"name":"offline","context":{"idset":"9068"}}
+{"timestamp":1714080106.9097953,"name":"offline","context":{"idset":"9069"}}
+{"timestamp":1714080106.9117997,"name":"offline","context":{"idset":"9070"}}
+{"timestamp":1714080106.9137874,"name":"offline","context":{"idset":"9071"}}
+{"timestamp":1714080106.91576,"name":"offline","context":{"idset":"9072"}}
+{"timestamp":1714080106.9177601,"name":"offline","context":{"idset":"9073"}}
+{"timestamp":1714080106.9197655,"name":"offline","context":{"idset":"9074"}}
+{"timestamp":1714080106.921757,"name":"offline","context":{"idset":"9075"}}
+{"timestamp":1714080106.9237273,"name":"offline","context":{"idset":"9076"}}
+{"timestamp":1714080106.9256775,"name":"offline","context":{"idset":"9077"}}
+{"timestamp":1714080106.9289558,"name":"offline","context":{"idset":"9078"}}
+{"timestamp":1714080106.9323032,"name":"offline","context":{"idset":"9079"}}
+{"timestamp":1714080106.9358182,"name":"offline","context":{"idset":"9080"}}
+{"timestamp":1714080106.9392498,"name":"offline","context":{"idset":"9081"}}
+{"timestamp":1714080106.9421411,"name":"offline","context":{"idset":"9082"}}
+{"timestamp":1714080106.9566538,"name":"offline","context":{"idset":"9083"}}
+{"timestamp":1714080106.9590702,"name":"offline","context":{"idset":"9084"}}
+{"timestamp":1714080106.9610043,"name":"offline","context":{"idset":"9085"}}
+{"timestamp":1714080106.9629912,"name":"offline","context":{"idset":"9086"}}
+{"timestamp":1714080106.9649279,"name":"offline","context":{"idset":"9087"}}
+{"timestamp":1714080106.9668355,"name":"offline","context":{"idset":"9088"}}
+{"timestamp":1714080106.9687409,"name":"offline","context":{"idset":"9089"}}
+{"timestamp":1714080106.9706316,"name":"offline","context":{"idset":"9090"}}
+{"timestamp":1714080106.9725175,"name":"offline","context":{"idset":"9091"}}
+{"timestamp":1714080106.9744034,"name":"offline","context":{"idset":"9092"}}
+{"timestamp":1714080106.9763112,"name":"offline","context":{"idset":"9093"}}
+{"timestamp":1714080106.978195,"name":"offline","context":{"idset":"9094"}}
+{"timestamp":1714080106.9800649,"name":"offline","context":{"idset":"9095"}}
+{"timestamp":1714080106.9887915,"name":"offline","context":{"idset":"9096"}}
+{"timestamp":1714080106.9906607,"name":"offline","context":{"idset":"9097"}}
+{"timestamp":1714080106.9925425,"name":"offline","context":{"idset":"9098"}}
+{"timestamp":1714080106.9944155,"name":"offline","context":{"idset":"9099"}}
+{"timestamp":1714080106.9962678,"name":"offline","context":{"idset":"9100"}}
+{"timestamp":1714080106.9981141,"name":"offline","context":{"idset":"9101"}}
+{"timestamp":1714080107.0067117,"name":"offline","context":{"idset":"9102"}}
+{"timestamp":1714080107.0085638,"name":"offline","context":{"idset":"9103"}}
+{"timestamp":1714080107.010401,"name":"offline","context":{"idset":"9104"}}
+{"timestamp":1714080107.0122287,"name":"offline","context":{"idset":"9105"}}
+{"timestamp":1714080107.0140541,"name":"offline","context":{"idset":"9106"}}
+{"timestamp":1714080107.0158727,"name":"offline","context":{"idset":"9107"}}
+{"timestamp":1714080107.0243852,"name":"offline","context":{"idset":"9108"}}
+{"timestamp":1714080107.0327132,"name":"offline","context":{"idset":"9109"}}
+{"timestamp":1714080107.0345032,"name":"offline","context":{"idset":"9110"}}
+{"timestamp":1714080107.0362828,"name":"offline","context":{"idset":"9111"}}
+{"timestamp":1714080107.0380406,"name":"offline","context":{"idset":"9112"}}
+{"timestamp":1714080107.0398114,"name":"offline","context":{"idset":"9113"}}
+{"timestamp":1714080107.0415671,"name":"offline","context":{"idset":"9114"}}
+{"timestamp":1714080107.0433354,"name":"offline","context":{"idset":"9115"}}
+{"timestamp":1714080107.0450935,"name":"offline","context":{"idset":"9116"}}
+{"timestamp":1714080107.0468481,"name":"offline","context":{"idset":"9117"}}
+{"timestamp":1714080107.0485933,"name":"offline","context":{"idset":"9118"}}
+{"timestamp":1714080107.050329,"name":"offline","context":{"idset":"9119"}}
+{"timestamp":1714080107.0520833,"name":"offline","context":{"idset":"9120"}}
+{"timestamp":1714080107.0538342,"name":"offline","context":{"idset":"9121"}}
+{"timestamp":1714080107.0555856,"name":"offline","context":{"idset":"9122"}}
+{"timestamp":1714080107.0573213,"name":"offline","context":{"idset":"9123"}}
+{"timestamp":1714080107.0590644,"name":"offline","context":{"idset":"9124"}}
+{"timestamp":1714080107.0608256,"name":"offline","context":{"idset":"9125"}}
+{"timestamp":1714080107.0625763,"name":"offline","context":{"idset":"9126"}}
+{"timestamp":1714080107.0643222,"name":"offline","context":{"idset":"9127"}}
+{"timestamp":1714080107.0660951,"name":"offline","context":{"idset":"9128"}}
+{"timestamp":1714080107.0678692,"name":"offline","context":{"idset":"9129"}}
+{"timestamp":1714080107.0696483,"name":"offline","context":{"idset":"9130"}}
+{"timestamp":1714080107.0714374,"name":"offline","context":{"idset":"9131"}}
+{"timestamp":1714080107.0732274,"name":"offline","context":{"idset":"9132"}}
+{"timestamp":1714080107.0816526,"name":"offline","context":{"idset":"9133"}}
+{"timestamp":1714080107.0834763,"name":"offline","context":{"idset":"9134"}}
+{"timestamp":1714080107.0919933,"name":"offline","context":{"idset":"9135"}}
+{"timestamp":1714080107.093852,"name":"offline","context":{"idset":"9136"}}
+{"timestamp":1714080107.0956764,"name":"offline","context":{"idset":"9137"}}
+{"timestamp":1714080107.0975101,"name":"offline","context":{"idset":"9138"}}
+{"timestamp":1714080107.0993509,"name":"offline","context":{"idset":"9139"}}
+{"timestamp":1714080107.1011817,"name":"offline","context":{"idset":"9140"}}
+{"timestamp":1714080107.103008,"name":"offline","context":{"idset":"9141"}}
+{"timestamp":1714080107.1048365,"name":"offline","context":{"idset":"9142"}}
+{"timestamp":1714080107.1066673,"name":"offline","context":{"idset":"9143"}}
+{"timestamp":1714080107.1084876,"name":"offline","context":{"idset":"9144"}}
+{"timestamp":1714080107.1103103,"name":"offline","context":{"idset":"9145"}}
+{"timestamp":1714080107.1121464,"name":"offline","context":{"idset":"9146"}}
+{"timestamp":1714080107.113971,"name":"offline","context":{"idset":"9147"}}
+{"timestamp":1714080107.1157758,"name":"offline","context":{"idset":"9148"}}
+{"timestamp":1714080107.1175981,"name":"offline","context":{"idset":"9149"}}
+{"timestamp":1714080107.1194234,"name":"offline","context":{"idset":"9150"}}
+{"timestamp":1714080107.1212385,"name":"offline","context":{"idset":"9151"}}
+{"timestamp":1714080107.1230314,"name":"offline","context":{"idset":"9152"}}
+{"timestamp":1714080107.1247938,"name":"offline","context":{"idset":"9153"}}
+{"timestamp":1714080107.1265526,"name":"offline","context":{"idset":"9154"}}
+{"timestamp":1714080107.1283386,"name":"offline","context":{"idset":"9155"}}
+{"timestamp":1714080107.1301141,"name":"offline","context":{"idset":"9156"}}
+{"timestamp":1714080107.1318827,"name":"offline","context":{"idset":"9157"}}
+{"timestamp":1714080107.1336849,"name":"offline","context":{"idset":"9158"}}
+{"timestamp":1714080107.1356554,"name":"offline","context":{"idset":"9159"}}
+{"timestamp":1714080107.1441312,"name":"offline","context":{"idset":"9160"}}
+{"timestamp":1714080107.1459396,"name":"offline","context":{"idset":"9161"}}
+{"timestamp":1714080107.1477311,"name":"offline","context":{"idset":"9162"}}
+{"timestamp":1714080107.1495261,"name":"offline","context":{"idset":"9163"}}
+{"timestamp":1714080107.1513355,"name":"offline","context":{"idset":"9164"}}
+{"timestamp":1714080107.1531353,"name":"offline","context":{"idset":"9165"}}
+{"timestamp":1714080107.1566122,"name":"offline","context":{"idset":"9166"}}
+{"timestamp":1714080107.1591558,"name":"offline","context":{"idset":"9167"}}
+{"timestamp":1714080107.1700242,"name":"offline","context":{"idset":"9168"}}
+{"timestamp":1714080107.1727874,"name":"offline","context":{"idset":"9169"}}
+{"timestamp":1714080107.1745431,"name":"offline","context":{"idset":"9170"}}
+{"timestamp":1714080107.1763086,"name":"offline","context":{"idset":"9171"}}
+{"timestamp":1714080107.1780694,"name":"offline","context":{"idset":"9172"}}
+{"timestamp":1714080107.1798379,"name":"offline","context":{"idset":"9173"}}
+{"timestamp":1714080107.1815877,"name":"offline","context":{"idset":"9174"}}
+{"timestamp":1714080107.1833298,"name":"offline","context":{"idset":"9175"}}
+{"timestamp":1714080107.1982942,"name":"offline","context":{"idset":"9176"}}
+{"timestamp":1714080107.2000744,"name":"offline","context":{"idset":"9177"}}
+{"timestamp":1714080107.2018447,"name":"offline","context":{"idset":"9178"}}
+{"timestamp":1714080107.2039161,"name":"offline","context":{"idset":"9179"}}
+{"timestamp":1714080107.2061698,"name":"offline","context":{"idset":"9180"}}
+{"timestamp":1714080107.2085788,"name":"offline","context":{"idset":"9181"}}
+{"timestamp":1714080107.2262716,"name":"offline","context":{"idset":"9182"}}
+{"timestamp":1714080107.2283542,"name":"offline","context":{"idset":"9183"}}
+{"timestamp":1714080107.2301877,"name":"offline","context":{"idset":"9184"}}
+{"timestamp":1714080107.2326112,"name":"offline","context":{"idset":"9185"}}
+{"timestamp":1714080107.23491,"name":"offline","context":{"idset":"9186"}}
+{"timestamp":1714080107.2370205,"name":"offline","context":{"idset":"9187"}}
+{"timestamp":1714080107.2394848,"name":"offline","context":{"idset":"9188"}}
+{"timestamp":1714080107.2416043,"name":"offline","context":{"idset":"9189"}}
+{"timestamp":1714080107.2439029,"name":"offline","context":{"idset":"9190"}}
+{"timestamp":1714080107.2457821,"name":"offline","context":{"idset":"9191"}}
+{"timestamp":1714080107.2545345,"name":"offline","context":{"idset":"9192"}}
+{"timestamp":1714080107.256417,"name":"offline","context":{"idset":"9193"}}
+{"timestamp":1714080107.2584262,"name":"offline","context":{"idset":"9194"}}
+{"timestamp":1714080107.2602079,"name":"offline","context":{"idset":"9195"}}
+{"timestamp":1714080107.261991,"name":"offline","context":{"idset":"9196"}}
+{"timestamp":1714080107.263777,"name":"offline","context":{"idset":"9197"}}
+{"timestamp":1714080107.2655585,"name":"offline","context":{"idset":"9198"}}
+{"timestamp":1714080107.2673395,"name":"offline","context":{"idset":"9199"}}
+{"timestamp":1714080107.2764592,"name":"offline","context":{"idset":"9200"}}
+{"timestamp":1714080107.2782724,"name":"offline","context":{"idset":"9201"}}
+{"timestamp":1714080107.2802498,"name":"offline","context":{"idset":"9202"}}
+{"timestamp":1714080107.2820868,"name":"offline","context":{"idset":"9203"}}
+{"timestamp":1714080107.2838964,"name":"offline","context":{"idset":"9204"}}
+{"timestamp":1714080107.2857921,"name":"offline","context":{"idset":"9205"}}
+{"timestamp":1714080107.2875905,"name":"offline","context":{"idset":"9206"}}
+{"timestamp":1714080107.2893758,"name":"offline","context":{"idset":"9207"}}
+{"timestamp":1714080107.3058381,"name":"offline","context":{"idset":"9208"}}
+{"timestamp":1714080107.3076925,"name":"offline","context":{"idset":"9209"}}
+{"timestamp":1714080107.30953,"name":"offline","context":{"idset":"9210"}}
+{"timestamp":1714080107.3113699,"name":"offline","context":{"idset":"9211"}}
+{"timestamp":1714080107.3132398,"name":"offline","context":{"idset":"9212"}}
+{"timestamp":1714080107.32321,"name":"offline","context":{"idset":"9213"}}
+{"timestamp":1714080107.3384767,"name":"offline","context":{"idset":"9214"}}
+{"timestamp":1714080107.3420782,"name":"offline","context":{"idset":"9215"}}
+{"timestamp":1714080107.3602395,"name":"offline","context":{"idset":"9216"}}
+{"timestamp":1714080107.3620961,"name":"offline","context":{"idset":"9217"}}
+{"timestamp":1714080107.3639524,"name":"offline","context":{"idset":"9218"}}
+{"timestamp":1714080107.3658235,"name":"offline","context":{"idset":"9219"}}
+{"timestamp":1714080107.3676913,"name":"offline","context":{"idset":"9220"}}
+{"timestamp":1714080107.369561,"name":"offline","context":{"idset":"9221"}}
+{"timestamp":1714080107.3714857,"name":"offline","context":{"idset":"9222"}}
+{"timestamp":1714080107.3734441,"name":"offline","context":{"idset":"9223"}}
+{"timestamp":1714080107.3753462,"name":"offline","context":{"idset":"9224"}}
+{"timestamp":1714080107.3776524,"name":"offline","context":{"idset":"9225"}}
+{"timestamp":1714080107.3795438,"name":"offline","context":{"idset":"9226"}}
+{"timestamp":1714080107.3814073,"name":"offline","context":{"idset":"9227"}}
+{"timestamp":1714080107.3832905,"name":"offline","context":{"idset":"9228"}}
+{"timestamp":1714080107.3860433,"name":"offline","context":{"idset":"9229"}}
+{"timestamp":1714080107.3896103,"name":"offline","context":{"idset":"9230"}}
+{"timestamp":1714080107.3931766,"name":"offline","context":{"idset":"9231"}}
+{"timestamp":1714080107.3957357,"name":"offline","context":{"idset":"9232"}}
+{"timestamp":1714080107.397634,"name":"offline","context":{"idset":"9233"}}
+{"timestamp":1714080107.3995357,"name":"offline","context":{"idset":"9234"}}
+{"timestamp":1714080107.4087145,"name":"offline","context":{"idset":"9235"}}
+{"timestamp":1714080107.4106343,"name":"offline","context":{"idset":"9236"}}
+{"timestamp":1714080107.4126422,"name":"offline","context":{"idset":"9237"}}
+{"timestamp":1714080107.414562,"name":"offline","context":{"idset":"9238"}}
+{"timestamp":1714080107.416502,"name":"offline","context":{"idset":"9239"}}
+{"timestamp":1714080107.4185576,"name":"offline","context":{"idset":"9240"}}
+{"timestamp":1714080107.4295855,"name":"offline","context":{"idset":"9241"}}
+{"timestamp":1714080107.431622,"name":"offline","context":{"idset":"9242"}}
+{"timestamp":1714080107.4344807,"name":"offline","context":{"idset":"9243"}}
+{"timestamp":1714080107.4374261,"name":"offline","context":{"idset":"9244"}}
+{"timestamp":1714080107.4403362,"name":"offline","context":{"idset":"9245"}}
+{"timestamp":1714080107.4575961,"name":"offline","context":{"idset":"9246"}}
+{"timestamp":1714080107.4681649,"name":"offline","context":{"idset":"9247"}}
+{"timestamp":1714080107.4701982,"name":"offline","context":{"idset":"9248"}}
+{"timestamp":1714080107.4723299,"name":"offline","context":{"idset":"9249"}}
+{"timestamp":1714080107.4743507,"name":"offline","context":{"idset":"9250"}}
+{"timestamp":1714080107.4764988,"name":"offline","context":{"idset":"9251"}}
+{"timestamp":1714080107.4785144,"name":"offline","context":{"idset":"9252"}}
+{"timestamp":1714080107.4807293,"name":"offline","context":{"idset":"9253"}}
+{"timestamp":1714080107.4827483,"name":"offline","context":{"idset":"9254"}}
+{"timestamp":1714080107.4850001,"name":"offline","context":{"idset":"9255"}}
+{"timestamp":1714080107.4877775,"name":"offline","context":{"idset":"9256"}}
+{"timestamp":1714080107.4900155,"name":"offline","context":{"idset":"9257"}}
+{"timestamp":1714080107.4920607,"name":"offline","context":{"idset":"9258"}}
+{"timestamp":1714080107.4940999,"name":"offline","context":{"idset":"9259"}}
+{"timestamp":1714080107.4961257,"name":"offline","context":{"idset":"9260"}}
+{"timestamp":1714080107.4981582,"name":"offline","context":{"idset":"9261"}}
+{"timestamp":1714080107.5002055,"name":"offline","context":{"idset":"9262"}}
+{"timestamp":1714080107.5022471,"name":"offline","context":{"idset":"9263"}}
+{"timestamp":1714080107.5042837,"name":"offline","context":{"idset":"9264"}}
+{"timestamp":1714080107.5063474,"name":"offline","context":{"idset":"9265"}}
+{"timestamp":1714080107.5164921,"name":"offline","context":{"idset":"9266"}}
+{"timestamp":1714080107.5185871,"name":"offline","context":{"idset":"9267"}}
+{"timestamp":1714080107.5208309,"name":"offline","context":{"idset":"9268"}}
+{"timestamp":1714080107.5229347,"name":"offline","context":{"idset":"9269"}}
+{"timestamp":1714080107.5251572,"name":"offline","context":{"idset":"9270"}}
+{"timestamp":1714080107.5272601,"name":"offline","context":{"idset":"9271"}}
+{"timestamp":1714080107.5381334,"name":"offline","context":{"idset":"9272"}}
+{"timestamp":1714080107.5402699,"name":"offline","context":{"idset":"9273"}}
+{"timestamp":1714080107.5431087,"name":"offline","context":{"idset":"9274"}}
+{"timestamp":1714080107.5452616,"name":"offline","context":{"idset":"9275"}}
+{"timestamp":1714080107.5475628,"name":"offline","context":{"idset":"9276"}}
+{"timestamp":1714080107.5761099,"name":"offline","context":{"idset":"9277"}}
+{"timestamp":1714080107.5853953,"name":"offline","context":{"idset":"9278"}}
+{"timestamp":1714080107.5875666,"name":"offline","context":{"idset":"9279"}}
+{"timestamp":1714080107.5901628,"name":"offline","context":{"idset":"9280"}}
+{"timestamp":1714080107.592226,"name":"offline","context":{"idset":"9281"}}
+{"timestamp":1714080107.5946739,"name":"offline","context":{"idset":"9282"}}
+{"timestamp":1714080107.5969694,"name":"offline","context":{"idset":"9283"}}
+{"timestamp":1714080107.5990307,"name":"offline","context":{"idset":"9284"}}
+{"timestamp":1714080107.6009591,"name":"offline","context":{"idset":"9285"}}
+{"timestamp":1714080107.603133,"name":"offline","context":{"idset":"9286"}}
+{"timestamp":1714080107.605237,"name":"offline","context":{"idset":"9287"}}
+{"timestamp":1714080107.6073709,"name":"offline","context":{"idset":"9288"}}
+{"timestamp":1714080107.6094141,"name":"offline","context":{"idset":"9289"}}
+{"timestamp":1714080107.6115568,"name":"offline","context":{"idset":"9290"}}
+{"timestamp":1714080107.6134048,"name":"offline","context":{"idset":"9291"}}
+{"timestamp":1714080107.6152248,"name":"offline","context":{"idset":"9292"}}
+{"timestamp":1714080107.6170678,"name":"offline","context":{"idset":"9293"}}
+{"timestamp":1714080107.6188774,"name":"offline","context":{"idset":"9294"}}
+{"timestamp":1714080107.6206775,"name":"offline","context":{"idset":"9295"}}
+{"timestamp":1714080107.6224916,"name":"offline","context":{"idset":"9296"}}
+{"timestamp":1714080107.6313498,"name":"offline","context":{"idset":"9297"}}
+{"timestamp":1714080107.6331697,"name":"offline","context":{"idset":"9298"}}
+{"timestamp":1714080107.6350594,"name":"offline","context":{"idset":"9299"}}
+{"timestamp":1714080107.6369171,"name":"offline","context":{"idset":"9300"}}
+{"timestamp":1714080107.6387205,"name":"offline","context":{"idset":"9301"}}
+{"timestamp":1714080107.6405342,"name":"offline","context":{"idset":"9302"}}
+{"timestamp":1714080107.6494849,"name":"offline","context":{"idset":"9303"}}
+{"timestamp":1714080107.6513369,"name":"offline","context":{"idset":"9304"}}
+{"timestamp":1714080107.6531684,"name":"offline","context":{"idset":"9305"}}
+{"timestamp":1714080107.6549807,"name":"offline","context":{"idset":"9306"}}
+{"timestamp":1714080107.6567926,"name":"offline","context":{"idset":"9307"}}
+{"timestamp":1714080107.6739917,"name":"offline","context":{"idset":"9308"}}
+{"timestamp":1714080107.675822,"name":"offline","context":{"idset":"9309"}}
+{"timestamp":1714080107.6848075,"name":"offline","context":{"idset":"9310"}}
+{"timestamp":1714080107.6866283,"name":"offline","context":{"idset":"9311"}}
+{"timestamp":1714080107.6885734,"name":"offline","context":{"idset":"9312"}}
+{"timestamp":1714080107.6904051,"name":"offline","context":{"idset":"9313"}}
+{"timestamp":1714080107.6922297,"name":"offline","context":{"idset":"9314"}}
+{"timestamp":1714080107.6941621,"name":"offline","context":{"idset":"9315"}}
+{"timestamp":1714080107.6962521,"name":"offline","context":{"idset":"9316"}}
+{"timestamp":1714080107.6981037,"name":"offline","context":{"idset":"9317"}}
+{"timestamp":1714080107.7000902,"name":"offline","context":{"idset":"9318"}}
+{"timestamp":1714080107.7021408,"name":"offline","context":{"idset":"9319"}}
+{"timestamp":1714080107.7039926,"name":"offline","context":{"idset":"9320"}}
+{"timestamp":1714080107.7059796,"name":"offline","context":{"idset":"9321"}}
+{"timestamp":1714080107.7078485,"name":"offline","context":{"idset":"9322"}}
+{"timestamp":1714080107.7171535,"name":"offline","context":{"idset":"9323"}}
+{"timestamp":1714080107.7190819,"name":"offline","context":{"idset":"9324"}}
+{"timestamp":1714080107.7209814,"name":"offline","context":{"idset":"9325"}}
+{"timestamp":1714080107.7228792,"name":"offline","context":{"idset":"9326"}}
+{"timestamp":1714080107.7247655,"name":"offline","context":{"idset":"9327"}}
+{"timestamp":1714080107.7266212,"name":"offline","context":{"idset":"9328"}}
+{"timestamp":1714080107.7284801,"name":"offline","context":{"idset":"9329"}}
+{"timestamp":1714080107.7303851,"name":"offline","context":{"idset":"9330"}}
+{"timestamp":1714080107.7322245,"name":"offline","context":{"idset":"9331"}}
+{"timestamp":1714080107.7341375,"name":"offline","context":{"idset":"9332"}}
+{"timestamp":1714080107.7360065,"name":"offline","context":{"idset":"9461"}}
+{"timestamp":1714080107.7378709,"name":"offline","context":{"idset":"9462"}}
+{"timestamp":1714080107.7396986,"name":"offline","context":{"idset":"9463"}}
+{"timestamp":1714080107.741538,"name":"offline","context":{"idset":"9464"}}
+{"timestamp":1714080107.7434607,"name":"offline","context":{"idset":"9465"}}
+{"timestamp":1714080107.745342,"name":"offline","context":{"idset":"9466"}}
+{"timestamp":1714080107.7472751,"name":"offline","context":{"idset":"9467"}}
+{"timestamp":1714080107.7493181,"name":"offline","context":{"idset":"9468"}}
+{"timestamp":1714080107.7511842,"name":"offline","context":{"idset":"9469"}}
+{"timestamp":1714080107.7536933,"name":"offline","context":{"idset":"9470"}}
+{"timestamp":1714080107.7563581,"name":"offline","context":{"idset":"9471"}}
+{"timestamp":1714080107.7581575,"name":"offline","context":{"idset":"9472"}}
+{"timestamp":1714080107.7599659,"name":"offline","context":{"idset":"9473"}}
+{"timestamp":1714080107.7617764,"name":"offline","context":{"idset":"9474"}}
+{"timestamp":1714080107.7635717,"name":"offline","context":{"idset":"9475"}}
+{"timestamp":1714080107.7725446,"name":"offline","context":{"idset":"9476"}}
+{"timestamp":1714080107.7743595,"name":"offline","context":{"idset":"9477"}}
+{"timestamp":1714080107.7761698,"name":"offline","context":{"idset":"9478"}}
+{"timestamp":1714080107.7779839,"name":"offline","context":{"idset":"9479"}}
+{"timestamp":1714080107.7799404,"name":"offline","context":{"idset":"9480"}}
+{"timestamp":1714080107.7817898,"name":"offline","context":{"idset":"9481"}}
+{"timestamp":1714080107.7836273,"name":"offline","context":{"idset":"9482"}}
+{"timestamp":1714080107.7930582,"name":"offline","context":{"idset":"9483"}}
+{"timestamp":1714080107.7948918,"name":"offline","context":{"idset":"9484"}}
+{"timestamp":1714080107.7967389,"name":"offline","context":{"idset":"9485"}}
+{"timestamp":1714080107.7985618,"name":"offline","context":{"idset":"9486"}}
+{"timestamp":1714080107.8003757,"name":"offline","context":{"idset":"9487"}}
+{"timestamp":1714080107.802242,"name":"offline","context":{"idset":"9488"}}
+{"timestamp":1714080107.8040521,"name":"offline","context":{"idset":"9489"}}
+{"timestamp":1714080107.8132515,"name":"offline","context":{"idset":"9490"}}
+{"timestamp":1714080107.815048,"name":"offline","context":{"idset":"9491"}}
+{"timestamp":1714080107.8168397,"name":"offline","context":{"idset":"9492"}}
+{"timestamp":1714080107.8186338,"name":"offline","context":{"idset":"9493"}}
+{"timestamp":1714080107.8204167,"name":"offline","context":{"idset":"9494"}}
+{"timestamp":1714080107.8222082,"name":"offline","context":{"idset":"9495"}}
+{"timestamp":1714080107.8250449,"name":"offline","context":{"idset":"9496"}}
+{"timestamp":1714080107.8415656,"name":"offline","context":{"idset":"9497"}}
+{"timestamp":1714080107.843364,"name":"offline","context":{"idset":"9498"}}
+{"timestamp":1714080107.8691287,"name":"offline","context":{"idset":"9499"}}
+{"timestamp":1714080107.8721781,"name":"offline","context":{"idset":"9500"}}
+{"timestamp":1714080107.8739862,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1714080107.8758514,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1714080107.8920431,"name":"offline","context":{"idset":"9503"}}
+{"timestamp":1714080107.8939955,"name":"offline","context":{"idset":"9504"}}
+{"timestamp":1714080107.8957655,"name":"offline","context":{"idset":"9505"}}
+{"timestamp":1714080107.8975401,"name":"offline","context":{"idset":"9506"}}
+{"timestamp":1714080107.8992922,"name":"offline","context":{"idset":"9507"}}
+{"timestamp":1714080107.9010572,"name":"offline","context":{"idset":"9508"}}
+{"timestamp":1714080107.9099684,"name":"offline","context":{"idset":"9509"}}
+{"timestamp":1714080107.9117343,"name":"offline","context":{"idset":"9510"}}
+{"timestamp":1714080107.9134936,"name":"offline","context":{"idset":"9511"}}
+{"timestamp":1714080107.9152532,"name":"offline","context":{"idset":"9512"}}
+{"timestamp":1714080107.9170108,"name":"offline","context":{"idset":"9513"}}
+{"timestamp":1714080107.9187589,"name":"offline","context":{"idset":"9514"}}
+{"timestamp":1714080107.9205263,"name":"offline","context":{"idset":"9515"}}
+{"timestamp":1714080107.9375603,"name":"offline","context":{"idset":"9516"}}
+{"timestamp":1714080107.9393857,"name":"offline","context":{"idset":"9517"}}
+{"timestamp":1714080107.9411478,"name":"offline","context":{"idset":"9518"}}
+{"timestamp":1714080107.9429109,"name":"offline","context":{"idset":"9519"}}
+{"timestamp":1714080107.9446404,"name":"offline","context":{"idset":"9520"}}
+{"timestamp":1714080107.9463747,"name":"offline","context":{"idset":"9521"}}
+{"timestamp":1714080107.9620767,"name":"offline","context":{"idset":"9522"}}
+{"timestamp":1714080107.9638119,"name":"offline","context":{"idset":"9523"}}
+{"timestamp":1714080107.9655557,"name":"offline","context":{"idset":"9524"}}
+{"timestamp":1714080107.968385,"name":"offline","context":{"idset":"9525"}}
+{"timestamp":1714080107.9702287,"name":"offline","context":{"idset":"9526"}}
+{"timestamp":1714080107.9719505,"name":"offline","context":{"idset":"9527"}}
+{"timestamp":1714080107.9736619,"name":"offline","context":{"idset":"9528"}}
+{"timestamp":1714080107.9823885,"name":"offline","context":{"idset":"9529"}}
+{"timestamp":1714080107.9841135,"name":"offline","context":{"idset":"9530"}}
+{"timestamp":1714080107.9858189,"name":"offline","context":{"idset":"9531"}}
+{"timestamp":1714080107.9875417,"name":"offline","context":{"idset":"9532"}}
+{"timestamp":1714080107.9892664,"name":"offline","context":{"idset":"9533"}}
+{"timestamp":1714080107.9977844,"name":"offline","context":{"idset":"9534"}}
+{"timestamp":1714080107.9995308,"name":"offline","context":{"idset":"9536"}}
+{"timestamp":1714080108.001245,"name":"offline","context":{"idset":"9537"}}
+{"timestamp":1714080108.0029635,"name":"offline","context":{"idset":"9538"}}
+{"timestamp":1714080108.0046737,"name":"offline","context":{"idset":"9539"}}
+{"timestamp":1714080108.0063956,"name":"offline","context":{"idset":"9540"}}
+{"timestamp":1714080108.0081217,"name":"offline","context":{"idset":"9541"}}
+{"timestamp":1714080108.0176725,"name":"offline","context":{"idset":"9542"}}
+{"timestamp":1714080108.0193927,"name":"offline","context":{"idset":"9543"}}
+{"timestamp":1714080108.0210781,"name":"offline","context":{"idset":"9544"}}
+{"timestamp":1714080108.0227528,"name":"offline","context":{"idset":"9545"}}
+{"timestamp":1714080108.0244343,"name":"offline","context":{"idset":"9546"}}
+{"timestamp":1714080108.0261018,"name":"offline","context":{"idset":"9547"}}
+{"timestamp":1714080108.0427096,"name":"offline","context":{"idset":"9548"}}
+{"timestamp":1714080108.0453196,"name":"offline","context":{"idset":"9549"}}
+{"timestamp":1714080108.0476265,"name":"offline","context":{"idset":"9550"}}
+{"timestamp":1714080108.049329,"name":"offline","context":{"idset":"9551"}}
+{"timestamp":1714080108.0509973,"name":"offline","context":{"idset":"9552"}}
+{"timestamp":1714080108.052644,"name":"offline","context":{"idset":"9553"}}
+{"timestamp":1714080108.0612421,"name":"offline","context":{"idset":"9554"}}
+{"timestamp":1714080108.0629027,"name":"offline","context":{"idset":"9555"}}
+{"timestamp":1714080108.0645473,"name":"offline","context":{"idset":"9556"}}
+{"timestamp":1714080108.0661843,"name":"offline","context":{"idset":"9557"}}
+{"timestamp":1714080108.0678151,"name":"offline","context":{"idset":"9558"}}
+{"timestamp":1714080108.0694513,"name":"offline","context":{"idset":"9559"}}
+{"timestamp":1714080108.0844538,"name":"offline","context":{"idset":"9560"}}
+{"timestamp":1714080108.0860891,"name":"offline","context":{"idset":"9561"}}
+{"timestamp":1714080108.0877066,"name":"offline","context":{"idset":"9562"}}
+{"timestamp":1714080108.0893137,"name":"offline","context":{"idset":"9563"}}
+{"timestamp":1714080108.0909722,"name":"offline","context":{"idset":"9564"}}
+{"timestamp":1714080108.0926318,"name":"offline","context":{"idset":"9565"}}
+{"timestamp":1714080108.1141903,"name":"offline","context":{"idset":"9566"}}
+{"timestamp":1714080108.1164596,"name":"offline","context":{"idset":"9567"}}
+{"timestamp":1714080108.1187189,"name":"offline","context":{"idset":"9568"}}
+{"timestamp":1714080108.120981,"name":"offline","context":{"idset":"9569"}}
+{"timestamp":1714080108.1232388,"name":"offline","context":{"idset":"9570"}}
+{"timestamp":1714080108.1255062,"name":"offline","context":{"idset":"9571"}}
+{"timestamp":1714080108.1462147,"name":"offline","context":{"idset":"9572"}}
+{"timestamp":1714080108.1484857,"name":"offline","context":{"idset":"9573"}}
+{"timestamp":1714080108.1514001,"name":"offline","context":{"idset":"9574"}}
+{"timestamp":1714080108.1553204,"name":"offline","context":{"idset":"9575"}}
+{"timestamp":1714080108.1575809,"name":"offline","context":{"idset":"9576"}}
+{"timestamp":1714080108.1669133,"name":"offline","context":{"idset":"9577"}}
+{"timestamp":1714080108.1864746,"name":"offline","context":{"idset":"9578"}}
+{"timestamp":1714080108.1882722,"name":"offline","context":{"idset":"9579"}}
+{"timestamp":1714080108.1898642,"name":"offline","context":{"idset":"9580"}}
+{"timestamp":1714080108.1914935,"name":"offline","context":{"idset":"9581"}}
+{"timestamp":1714080108.1930995,"name":"offline","context":{"idset":"9582"}}
+{"timestamp":1714080108.1951227,"name":"offline","context":{"idset":"9583"}}
+{"timestamp":1714080108.2120359,"name":"offline","context":{"idset":"9584"}}
+{"timestamp":1714080108.2139084,"name":"offline","context":{"idset":"9585"}}
+{"timestamp":1714080108.2161107,"name":"offline","context":{"idset":"9586"}}
+{"timestamp":1714080108.2178469,"name":"offline","context":{"idset":"9587"}}
+{"timestamp":1714080108.219368,"name":"offline","context":{"idset":"9588"}}
+{"timestamp":1714080108.2208786,"name":"offline","context":{"idset":"9589"}}
+{"timestamp":1714080108.2349427,"name":"offline","context":{"idset":"9590"}}
+{"timestamp":1714080108.2428052,"name":"offline","context":{"idset":"9591"}}
+{"timestamp":1714080108.2443342,"name":"offline","context":{"idset":"9592"}}
+{"timestamp":1714080108.2458532,"name":"offline","context":{"idset":"9593"}}
+{"timestamp":1714080108.2474325,"name":"offline","context":{"idset":"9594"}}
+{"timestamp":1714080108.2490265,"name":"offline","context":{"idset":"9595"}}
+{"timestamp":1714080108.2505226,"name":"offline","context":{"idset":"9596"}}
+{"timestamp":1714080108.2520082,"name":"offline","context":{"idset":"9597"}}
+{"timestamp":1714080108.25383,"name":"offline","context":{"idset":"9598"}}
+{"timestamp":1714080108.255492,"name":"offline","context":{"idset":"9599"}}
+{"timestamp":1714080108.2569993,"name":"offline","context":{"idset":"9600"}}
+{"timestamp":1714080108.2585533,"name":"offline","context":{"idset":"9601"}}
+{"timestamp":1714080108.2601893,"name":"offline","context":{"idset":"9602"}}
+{"timestamp":1714080108.2616546,"name":"offline","context":{"idset":"9603"}}
+{"timestamp":1714080108.2631581,"name":"offline","context":{"idset":"9604"}}
+{"timestamp":1714080108.2647707,"name":"offline","context":{"idset":"9605"}}
+{"timestamp":1714080108.2663379,"name":"offline","context":{"idset":"9606"}}
+{"timestamp":1714080108.2678297,"name":"offline","context":{"idset":"9607"}}
+{"timestamp":1714080108.2693472,"name":"offline","context":{"idset":"9608"}}
+{"timestamp":1714080108.2708249,"name":"offline","context":{"idset":"9609"}}
+{"timestamp":1714080108.2722809,"name":"offline","context":{"idset":"9610"}}
+{"timestamp":1714080108.2737238,"name":"offline","context":{"idset":"9611"}}
+{"timestamp":1714080108.2751579,"name":"offline","context":{"idset":"9612"}}
+{"timestamp":1714080108.2827053,"name":"offline","context":{"idset":"9613"}}
+{"timestamp":1714080108.2841933,"name":"offline","context":{"idset":"9614"}}
+{"timestamp":1714080108.2856572,"name":"offline","context":{"idset":"9615"}}
+{"timestamp":1714080108.2871187,"name":"offline","context":{"idset":"9616"}}
+{"timestamp":1714080108.2885566,"name":"offline","context":{"idset":"9617"}}
+{"timestamp":1714080108.2900105,"name":"offline","context":{"idset":"9618"}}
+{"timestamp":1714080108.2914524,"name":"offline","context":{"idset":"9619"}}
+{"timestamp":1714080108.2989714,"name":"offline","context":{"idset":"9620"}}
+{"timestamp":1714080108.3064058,"name":"offline","context":{"idset":"9621"}}
+{"timestamp":1714080108.3080475,"name":"offline","context":{"idset":"9622"}}
+{"timestamp":1714080108.3096697,"name":"offline","context":{"idset":"9623"}}
+{"timestamp":1714080108.3127165,"name":"offline","context":{"idset":"9624"}}
+{"timestamp":1714080108.315593,"name":"offline","context":{"idset":"9625"}}
+{"timestamp":1714080108.3182821,"name":"offline","context":{"idset":"9626"}}
+{"timestamp":1714080108.3198397,"name":"offline","context":{"idset":"9627"}}
+{"timestamp":1714080108.3213458,"name":"offline","context":{"idset":"9628"}}
+{"timestamp":1714080108.3286695,"name":"offline","context":{"idset":"9629"}}
+{"timestamp":1714080108.3359978,"name":"offline","context":{"idset":"9630"}}
+{"timestamp":1714080108.3382268,"name":"offline","context":{"idset":"9631"}}
+{"timestamp":1714080108.3407075,"name":"offline","context":{"idset":"9632"}}
+{"timestamp":1714080108.3431847,"name":"offline","context":{"idset":"9633"}}
+{"timestamp":1714080108.3456571,"name":"offline","context":{"idset":"9634"}}
+{"timestamp":1714080108.3481333,"name":"offline","context":{"idset":"9635"}}
+{"timestamp":1714080108.3503911,"name":"offline","context":{"idset":"9636"}}
+{"timestamp":1714080108.3519673,"name":"offline","context":{"idset":"9637"}}
+{"timestamp":1714080108.3533411,"name":"offline","context":{"idset":"9638"}}
+{"timestamp":1714080108.3546994,"name":"offline","context":{"idset":"9639"}}
+{"timestamp":1714080108.3560743,"name":"offline","context":{"idset":"9640"}}
+{"timestamp":1714080108.3574731,"name":"offline","context":{"idset":"9641"}}
+{"timestamp":1714080108.3588395,"name":"offline","context":{"idset":"9642"}}
+{"timestamp":1714080108.3602822,"name":"offline","context":{"idset":"9643"}}
+{"timestamp":1714080108.3674908,"name":"offline","context":{"idset":"9644"}}
+{"timestamp":1714080108.3688865,"name":"offline","context":{"idset":"9645"}}
+{"timestamp":1714080108.3702478,"name":"offline","context":{"idset":"9646"}}
+{"timestamp":1714080108.3716075,"name":"offline","context":{"idset":"9647"}}
+{"timestamp":1714080108.3730984,"name":"offline","context":{"idset":"9648"}}
+{"timestamp":1714080108.374512,"name":"offline","context":{"idset":"9649"}}
+{"timestamp":1714080108.3817327,"name":"offline","context":{"idset":"9650"}}
+{"timestamp":1714080108.3831248,"name":"offline","context":{"idset":"9651"}}
+{"timestamp":1714080108.3847241,"name":"offline","context":{"idset":"9652"}}
+{"timestamp":1714080108.3868108,"name":"offline","context":{"idset":"9653"}}
+{"timestamp":1714080108.388813,"name":"offline","context":{"idset":"9654"}}
+{"timestamp":1714080108.390372,"name":"offline","context":{"idset":"9655"}}
+{"timestamp":1714080108.3917236,"name":"offline","context":{"idset":"9656"}}
+{"timestamp":1714080108.404556,"name":"offline","context":{"idset":"9657"}}
+{"timestamp":1714080108.4058845,"name":"offline","context":{"idset":"9658"}}
+{"timestamp":1714080108.4129322,"name":"offline","context":{"idset":"9659"}}
+{"timestamp":1714080108.4142497,"name":"offline","context":{"idset":"9660"}}
+{"timestamp":1714080108.4155607,"name":"offline","context":{"idset":"9661"}}
+{"timestamp":1714080108.4168808,"name":"offline","context":{"idset":"9662"}}
+{"timestamp":1714080108.4182038,"name":"offline","context":{"idset":"9663"}}
+{"timestamp":1714080108.419522,"name":"offline","context":{"idset":"9664"}}
+{"timestamp":1714080108.4208286,"name":"offline","context":{"idset":"9665"}}
+{"timestamp":1714080108.4221425,"name":"offline","context":{"idset":"9666"}}
+{"timestamp":1714080108.4234517,"name":"offline","context":{"idset":"9667"}}
+{"timestamp":1714080108.4247653,"name":"offline","context":{"idset":"9668"}}
+{"timestamp":1714080108.4260793,"name":"offline","context":{"idset":"9669"}}
+{"timestamp":1714080108.4273958,"name":"offline","context":{"idset":"9670"}}
+{"timestamp":1714080108.4287167,"name":"offline","context":{"idset":"9671"}}
+{"timestamp":1714080108.4300334,"name":"offline","context":{"idset":"9672"}}
+{"timestamp":1714080108.4313493,"name":"offline","context":{"idset":"9673"}}
+{"timestamp":1714080108.4326694,"name":"offline","context":{"idset":"9674"}}
+{"timestamp":1714080108.4339893,"name":"offline","context":{"idset":"9675"}}
+{"timestamp":1714080108.4352982,"name":"offline","context":{"idset":"9676"}}
+{"timestamp":1714080108.4366121,"name":"offline","context":{"idset":"9677"}}
+{"timestamp":1714080108.4434047,"name":"offline","context":{"idset":"9678"}}
+{"timestamp":1714080108.4507205,"name":"offline","context":{"idset":"9679"}}
+{"timestamp":1714080108.4521122,"name":"offline","context":{"idset":"9680"}}
+{"timestamp":1714080108.453407,"name":"offline","context":{"idset":"9681"}}
+{"timestamp":1714080108.4546905,"name":"offline","context":{"idset":"9682"}}
+{"timestamp":1714080108.4559751,"name":"offline","context":{"idset":"9683"}}
+{"timestamp":1714080108.4572449,"name":"offline","context":{"idset":"9684"}}
+{"timestamp":1714080108.4585102,"name":"offline","context":{"idset":"9685"}}
+{"timestamp":1714080108.4597764,"name":"offline","context":{"idset":"9686"}}
+{"timestamp":1714080108.4610429,"name":"offline","context":{"idset":"9687"}}
+{"timestamp":1714080108.4623096,"name":"offline","context":{"idset":"9688"}}
+{"timestamp":1714080108.4635739,"name":"offline","context":{"idset":"9689"}}
+{"timestamp":1714080108.4648399,"name":"offline","context":{"idset":"9690"}}
+{"timestamp":1714080108.4661055,"name":"offline","context":{"idset":"9691"}}
+{"timestamp":1714080108.4727747,"name":"offline","context":{"idset":"9692"}}
+{"timestamp":1714080108.4740517,"name":"offline","context":{"idset":"9693"}}
+{"timestamp":1714080108.4753091,"name":"offline","context":{"idset":"9694"}}
+{"timestamp":1714080108.4767871,"name":"offline","context":{"idset":"9695"}}
+{"timestamp":1714080108.4783015,"name":"offline","context":{"idset":"9696"}}
+{"timestamp":1714080108.4808431,"name":"offline","context":{"idset":"9697"}}
+{"timestamp":1714080108.4825413,"name":"offline","context":{"idset":"9698"}}
+{"timestamp":1714080108.4837909,"name":"offline","context":{"idset":"9699"}}
+{"timestamp":1714080108.48506,"name":"offline","context":{"idset":"9700"}}
+{"timestamp":1714080108.4863214,"name":"offline","context":{"idset":"9701"}}
+{"timestamp":1714080108.4875655,"name":"offline","context":{"idset":"9702"}}
+{"timestamp":1714080108.4888115,"name":"offline","context":{"idset":"9703"}}
+{"timestamp":1714080108.4900413,"name":"offline","context":{"idset":"9704"}}
+{"timestamp":1714080108.4912984,"name":"offline","context":{"idset":"9705"}}
+{"timestamp":1714080108.4925418,"name":"offline","context":{"idset":"9706"}}
+{"timestamp":1714080108.4937916,"name":"offline","context":{"idset":"9707"}}
+{"timestamp":1714080108.4950302,"name":"offline","context":{"idset":"9708"}}
+{"timestamp":1714080108.4962573,"name":"offline","context":{"idset":"9709"}}
+{"timestamp":1714080108.4974942,"name":"offline","context":{"idset":"9710"}}
+{"timestamp":1714080108.4987299,"name":"offline","context":{"idset":"9711"}}
+{"timestamp":1714080108.4999771,"name":"offline","context":{"idset":"9712"}}
+{"timestamp":1714080108.50121,"name":"offline","context":{"idset":"9713"}}
+{"timestamp":1714080108.5077903,"name":"offline","context":{"idset":"9714"}}
+{"timestamp":1714080108.5090151,"name":"offline","context":{"idset":"9715"}}
+{"timestamp":1714080108.5102377,"name":"offline","context":{"idset":"9716"}}
+{"timestamp":1714080108.5114462,"name":"offline","context":{"idset":"9717"}}
+{"timestamp":1714080108.5126565,"name":"offline","context":{"idset":"9718"}}
+{"timestamp":1714080108.5138795,"name":"offline","context":{"idset":"9719"}}
+{"timestamp":1714080108.5256698,"name":"offline","context":{"idset":"9720"}}
+{"timestamp":1714080108.5269079,"name":"offline","context":{"idset":"9721"}}
+{"timestamp":1714080108.5281236,"name":"offline","context":{"idset":"9722"}}
+{"timestamp":1714080108.5293365,"name":"offline","context":{"idset":"9723"}}
+{"timestamp":1714080108.5305545,"name":"offline","context":{"idset":"9724"}}
+{"timestamp":1714080108.5317717,"name":"offline","context":{"idset":"9725"}}
+{"timestamp":1714080108.5435464,"name":"offline","context":{"idset":"9726"}}
+{"timestamp":1714080108.5447693,"name":"offline","context":{"idset":"9727"}}
+{"timestamp":1714080108.5459864,"name":"offline","context":{"idset":"9728"}}
+{"timestamp":1714080108.5471966,"name":"offline","context":{"idset":"9729"}}
+{"timestamp":1714080108.5484068,"name":"offline","context":{"idset":"9730"}}
+{"timestamp":1714080108.549619,"name":"offline","context":{"idset":"9731"}}
+{"timestamp":1714080108.5614293,"name":"offline","context":{"idset":"9732"}}
+{"timestamp":1714080108.5626383,"name":"offline","context":{"idset":"9733"}}
+{"timestamp":1714080108.5638614,"name":"offline","context":{"idset":"9734"}}
+{"timestamp":1714080108.5650809,"name":"offline","context":{"idset":"9735"}}
+{"timestamp":1714080108.5662866,"name":"offline","context":{"idset":"9736"}}
+{"timestamp":1714080108.5674958,"name":"offline","context":{"idset":"9737"}}
+{"timestamp":1714080108.5741491,"name":"offline","context":{"idset":"9738"}}
+{"timestamp":1714080108.5807192,"name":"offline","context":{"idset":"9739"}}
+{"timestamp":1714080108.5819321,"name":"offline","context":{"idset":"9740"}}
+{"timestamp":1714080108.5831525,"name":"offline","context":{"idset":"9741"}}
+{"timestamp":1714080108.5843616,"name":"offline","context":{"idset":"9742"}}
+{"timestamp":1714080108.5855613,"name":"offline","context":{"idset":"9743"}}
+{"timestamp":1714080108.5867708,"name":"offline","context":{"idset":"9744"}}
+{"timestamp":1714080108.5986142,"name":"offline","context":{"idset":"9745"}}
+{"timestamp":1714080108.6051517,"name":"offline","context":{"idset":"9746"}}
+{"timestamp":1714080108.6063545,"name":"offline","context":{"idset":"9747"}}
+{"timestamp":1714080108.6075768,"name":"offline","context":{"idset":"9748"}}
+{"timestamp":1714080108.6088102,"name":"offline","context":{"idset":"9749"}}
+{"timestamp":1714080108.6100161,"name":"offline","context":{"idset":"9750"}}
+{"timestamp":1714080108.6165504,"name":"offline","context":{"idset":"9751"}}
+{"timestamp":1714080108.6230881,"name":"offline","context":{"idset":"9752"}}
+{"timestamp":1714080108.6242993,"name":"offline","context":{"idset":"9753"}}
+{"timestamp":1714080108.6255105,"name":"offline","context":{"idset":"9754"}}
+{"timestamp":1714080108.6267192,"name":"offline","context":{"idset":"9755"}}
+{"timestamp":1714080108.627938,"name":"offline","context":{"idset":"9756"}}
+{"timestamp":1714080108.6291504,"name":"offline","context":{"idset":"9757"}}
+{"timestamp":1714080108.6498356,"name":"offline","context":{"idset":"9758"}}
+{"timestamp":1714080108.6613767,"name":"offline","context":{"idset":"9759"}}
+{"timestamp":1714080108.6626134,"name":"offline","context":{"idset":"9760"}}
+{"timestamp":1714080108.6638212,"name":"offline","context":{"idset":"9761"}}
+{"timestamp":1714080108.6657212,"name":"offline","context":{"idset":"9762"}}
+{"timestamp":1714080108.6677856,"name":"offline","context":{"idset":"9763"}}
+{"timestamp":1714080108.6699944,"name":"offline","context":{"idset":"9764"}}
+{"timestamp":1714080108.6723623,"name":"offline","context":{"idset":"9765"}}
+{"timestamp":1714080108.6745884,"name":"offline","context":{"idset":"9766"}}
+{"timestamp":1714080108.6765852,"name":"offline","context":{"idset":"9767"}}
+{"timestamp":1714080108.6788268,"name":"offline","context":{"idset":"9768"}}
+{"timestamp":1714080108.68137,"name":"offline","context":{"idset":"9769"}}
+{"timestamp":1714080108.6835244,"name":"offline","context":{"idset":"9770"}}
+{"timestamp":1714080108.6856461,"name":"offline","context":{"idset":"9771"}}
+{"timestamp":1714080108.6877096,"name":"offline","context":{"idset":"9772"}}
+{"timestamp":1714080108.6898954,"name":"offline","context":{"idset":"9773"}}
+{"timestamp":1714080108.6921773,"name":"offline","context":{"idset":"9774"}}
+{"timestamp":1714080108.6942532,"name":"offline","context":{"idset":"9775"}}
+{"timestamp":1714080108.7062705,"name":"offline","context":{"idset":"9776"}}
+{"timestamp":1714080108.7085423,"name":"offline","context":{"idset":"9777"}}
+{"timestamp":1714080108.7114737,"name":"offline","context":{"idset":"9778"}}
+{"timestamp":1714080108.7138669,"name":"offline","context":{"idset":"9779"}}
+{"timestamp":1714080108.7162476,"name":"offline","context":{"idset":"9780"}}
+{"timestamp":1714080108.7186449,"name":"offline","context":{"idset":"9781"}}
+{"timestamp":1714080108.7208107,"name":"offline","context":{"idset":"9782"}}
+{"timestamp":1714080108.7333918,"name":"offline","context":{"idset":"9783"}}
+{"timestamp":1714080108.7355254,"name":"offline","context":{"idset":"9784"}}
+{"timestamp":1714080108.7379484,"name":"offline","context":{"idset":"9785"}}
+{"timestamp":1714080108.7402904,"name":"offline","context":{"idset":"9786"}}
+{"timestamp":1714080108.7426379,"name":"offline","context":{"idset":"9787"}}
+{"timestamp":1714080108.7447982,"name":"offline","context":{"idset":"9788"}}
+{"timestamp":1714080108.7470493,"name":"offline","context":{"idset":"9789"}}
+{"timestamp":1714080108.7693796,"name":"offline","context":{"idset":"9790"}}
+{"timestamp":1714080108.7821596,"name":"offline","context":{"idset":"9791"}}
+{"timestamp":1714080108.7844453,"name":"offline","context":{"idset":"9792"}}
+{"timestamp":1714080108.7867348,"name":"offline","context":{"idset":"9793"}}
+{"timestamp":1714080108.7889998,"name":"offline","context":{"idset":"9794"}}
+{"timestamp":1714080108.7913013,"name":"offline","context":{"idset":"9795"}}
+{"timestamp":1714080108.7934685,"name":"offline","context":{"idset":"9796"}}
+{"timestamp":1714080108.7958055,"name":"offline","context":{"idset":"9797"}}
+{"timestamp":1714080108.7981353,"name":"offline","context":{"idset":"9798"}}
+{"timestamp":1714080108.8004844,"name":"offline","context":{"idset":"9799"}}
+{"timestamp":1714080108.8028395,"name":"offline","context":{"idset":"9800"}}
+{"timestamp":1714080108.8051345,"name":"offline","context":{"idset":"9801"}}
+{"timestamp":1714080108.8074076,"name":"offline","context":{"idset":"9802"}}
+{"timestamp":1714080108.8094244,"name":"offline","context":{"idset":"9803"}}
+{"timestamp":1714080108.8118122,"name":"offline","context":{"idset":"9804"}}
+{"timestamp":1714080108.8138754,"name":"offline","context":{"idset":"9805"}}
+{"timestamp":1714080108.8166718,"name":"offline","context":{"idset":"9806"}}
+{"timestamp":1714080108.8188641,"name":"offline","context":{"idset":"9807"}}
+{"timestamp":1714080108.820889,"name":"offline","context":{"idset":"9808"}}
+{"timestamp":1714080108.8232248,"name":"offline","context":{"idset":"9809"}}
+{"timestamp":1714080108.8256865,"name":"offline","context":{"idset":"9810"}}
+{"timestamp":1714080108.8280892,"name":"offline","context":{"idset":"9811"}}
+{"timestamp":1714080108.8401608,"name":"offline","context":{"idset":"9812"}}
+{"timestamp":1714080108.8422232,"name":"offline","context":{"idset":"9813"}}
+{"timestamp":1714080108.8443909,"name":"offline","context":{"idset":"9814"}}
+{"timestamp":1714080108.8467228,"name":"offline","context":{"idset":"9815"}}
+{"timestamp":1714080108.8487554,"name":"offline","context":{"idset":"9816"}}
+{"timestamp":1714080108.8508461,"name":"offline","context":{"idset":"9817"}}
+{"timestamp":1714080108.8530295,"name":"offline","context":{"idset":"9818"}}
+{"timestamp":1714080108.8814604,"name":"offline","context":{"idset":"9819"}}
+{"timestamp":1714080108.8837681,"name":"offline","context":{"idset":"9820"}}
+{"timestamp":1714080108.8859944,"name":"offline","context":{"idset":"9821"}}
+{"timestamp":1714080108.8880739,"name":"offline","context":{"idset":"9822"}}
+{"timestamp":1714080108.8902781,"name":"offline","context":{"idset":"9823"}}
+{"timestamp":1714080108.8925669,"name":"offline","context":{"idset":"9824"}}
+{"timestamp":1714080108.8947525,"name":"offline","context":{"idset":"9825"}}
+{"timestamp":1714080108.9074588,"name":"offline","context":{"idset":"9826"}}
+{"timestamp":1714080108.9095738,"name":"offline","context":{"idset":"9827"}}
+{"timestamp":1714080108.9216812,"name":"offline","context":{"idset":"9828"}}
+{"timestamp":1714080108.9239087,"name":"offline","context":{"idset":"9829"}}
+{"timestamp":1714080108.9260044,"name":"offline","context":{"idset":"9830"}}
+{"timestamp":1714080108.9281027,"name":"offline","context":{"idset":"9831"}}
+{"timestamp":1714080108.9304113,"name":"offline","context":{"idset":"9832"}}
+{"timestamp":1714080108.9327648,"name":"offline","context":{"idset":"9934"}}
+{"timestamp":1714080108.9349151,"name":"offline","context":{"idset":"9935"}}
+{"timestamp":1714080108.9369555,"name":"offline","context":{"idset":"9936"}}
+{"timestamp":1714080108.9392138,"name":"offline","context":{"idset":"9937"}}
+{"timestamp":1714080108.9414473,"name":"offline","context":{"idset":"9938"}}
+{"timestamp":1714080108.9435267,"name":"offline","context":{"idset":"9939"}}
+{"timestamp":1714080108.945595,"name":"offline","context":{"idset":"9940"}}
+{"timestamp":1714080108.9476309,"name":"offline","context":{"idset":"9941"}}
+{"timestamp":1714080108.9496834,"name":"offline","context":{"idset":"9942"}}
+{"timestamp":1714080108.9517565,"name":"offline","context":{"idset":"9943"}}
+{"timestamp":1714080108.9539559,"name":"offline","context":{"idset":"9944"}}
+{"timestamp":1714080108.9559932,"name":"offline","context":{"idset":"9945"}}
+{"timestamp":1714080108.9579916,"name":"offline","context":{"idset":"9946"}}
+{"timestamp":1714080108.9602652,"name":"offline","context":{"idset":"9947"}}
+{"timestamp":1714080108.9624572,"name":"offline","context":{"idset":"9948"}}
+{"timestamp":1714080108.9643259,"name":"offline","context":{"idset":"9949"}}
+{"timestamp":1714080108.9776113,"name":"offline","context":{"idset":"9950"}}
+{"timestamp":1714080108.9800608,"name":"offline","context":{"idset":"9951"}}
+{"timestamp":1714080108.9823494,"name":"offline","context":{"idset":"9952"}}
+{"timestamp":1714080108.9848428,"name":"offline","context":{"idset":"9953"}}
+{"timestamp":1714080108.9871864,"name":"offline","context":{"idset":"9954"}}
+{"timestamp":1714080108.9894872,"name":"offline","context":{"idset":"9955"}}
+{"timestamp":1714080108.991668,"name":"offline","context":{"idset":"9956"}}
+{"timestamp":1714080109.0043583,"name":"offline","context":{"idset":"9957"}}
+{"timestamp":1714080109.0065727,"name":"offline","context":{"idset":"9958"}}
+{"timestamp":1714080109.0088198,"name":"offline","context":{"idset":"9959"}}
+{"timestamp":1714080109.0110185,"name":"offline","context":{"idset":"9960"}}
+{"timestamp":1714080109.0130897,"name":"offline","context":{"idset":"9961"}}
+{"timestamp":1714080109.0151689,"name":"offline","context":{"idset":"9962"}}
+{"timestamp":1714080109.0171819,"name":"offline","context":{"idset":"9963"}}
+{"timestamp":1714080109.0292666,"name":"offline","context":{"idset":"9964"}}
+{"timestamp":1714080109.0315835,"name":"offline","context":{"idset":"9965"}}
+{"timestamp":1714080109.0442328,"name":"offline","context":{"idset":"9966"}}
+{"timestamp":1714080109.0576427,"name":"offline","context":{"idset":"9967"}}
+{"timestamp":1714080109.0600021,"name":"offline","context":{"idset":"9968"}}
+{"timestamp":1714080109.0623245,"name":"offline","context":{"idset":"9969"}}
+{"timestamp":1714080109.0646501,"name":"offline","context":{"idset":"9970"}}
+{"timestamp":1714080109.0669897,"name":"offline","context":{"idset":"9971"}}
+{"timestamp":1714080109.0692961,"name":"offline","context":{"idset":"9972"}}
+{"timestamp":1714080109.0713787,"name":"offline","context":{"idset":"9973"}}
+{"timestamp":1714080109.073565,"name":"offline","context":{"idset":"9974"}}
+{"timestamp":1714080109.0763054,"name":"offline","context":{"idset":"9975"}}
+{"timestamp":1714080109.0806289,"name":"offline","context":{"idset":"9976"}}
+{"timestamp":1714080109.0830367,"name":"offline","context":{"idset":"9977"}}
+{"timestamp":1714080109.0854225,"name":"offline","context":{"idset":"9978"}}
+{"timestamp":1714080109.0875628,"name":"offline","context":{"idset":"9979"}}
+{"timestamp":1714080109.0892932,"name":"offline","context":{"idset":"9980"}}
+{"timestamp":1714080109.0914624,"name":"offline","context":{"idset":"9981"}}
+{"timestamp":1714080109.0938125,"name":"offline","context":{"idset":"9982"}}
+{"timestamp":1714080109.1069345,"name":"offline","context":{"idset":"9983"}}
+{"timestamp":1714080109.109405,"name":"offline","context":{"idset":"9984"}}
+{"timestamp":1714080109.1114302,"name":"offline","context":{"idset":"9985"}}
+{"timestamp":1714080109.113471,"name":"offline","context":{"idset":"9986"}}
+{"timestamp":1714080109.1154833,"name":"offline","context":{"idset":"9987"}}
+{"timestamp":1714080109.11763,"name":"offline","context":{"idset":"9988"}}
+{"timestamp":1714080109.1197753,"name":"offline","context":{"idset":"9989"}}
+{"timestamp":1714080109.1218739,"name":"offline","context":{"idset":"9990"}}
+{"timestamp":1714080109.1241143,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1714080109.1264443,"name":"offline","context":{"idset":"9992"}}
+{"timestamp":1714080109.1285183,"name":"offline","context":{"idset":"9993"}}
+{"timestamp":1714080109.1306436,"name":"offline","context":{"idset":"9994"}}
+{"timestamp":1714080109.132844,"name":"offline","context":{"idset":"9995"}}
+{"timestamp":1714080109.1350148,"name":"offline","context":{"idset":"9996"}}
+{"timestamp":1714080109.1371675,"name":"offline","context":{"idset":"9997"}}
+{"timestamp":1714080109.1393423,"name":"offline","context":{"idset":"9998"}}
+{"timestamp":1714080109.1414926,"name":"offline","context":{"idset":"9999"}}
+{"timestamp":1714080109.1437588,"name":"offline","context":{"idset":"10000"}}
+{"timestamp":1714080109.1459441,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1714080109.1482003,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1714080109.1504812,"name":"offline","context":{"idset":"10003"}}
+{"timestamp":1714080109.1526573,"name":"offline","context":{"idset":"10004"}}
+{"timestamp":1714080109.154866,"name":"offline","context":{"idset":"10005"}}
+{"timestamp":1714080109.1570623,"name":"offline","context":{"idset":"10006"}}
+{"timestamp":1714080109.1694839,"name":"offline","context":{"idset":"10007"}}
+{"timestamp":1714080109.1716769,"name":"offline","context":{"idset":"10008"}}
+{"timestamp":1714080109.1736426,"name":"offline","context":{"idset":"10009"}}
+{"timestamp":1714080109.1756358,"name":"offline","context":{"idset":"10010"}}
+{"timestamp":1714080109.1776519,"name":"offline","context":{"idset":"10011"}}
+{"timestamp":1714080109.1799366,"name":"offline","context":{"idset":"10012"}}
+{"timestamp":1714080109.1929283,"name":"offline","context":{"idset":"10013"}}
+{"timestamp":1714080109.195092,"name":"offline","context":{"idset":"10014"}}
+{"timestamp":1714080109.1971595,"name":"offline","context":{"idset":"10015"}}
+{"timestamp":1714080109.1994047,"name":"offline","context":{"idset":"10016"}}
+{"timestamp":1714080109.2014725,"name":"offline","context":{"idset":"10017"}}
+{"timestamp":1714080109.2037177,"name":"offline","context":{"idset":"10018"}}
+{"timestamp":1714080109.2278934,"name":"offline","context":{"idset":"10019"}}
+{"timestamp":1714080109.2302105,"name":"offline","context":{"idset":"10020"}}
+{"timestamp":1714080109.2324116,"name":"offline","context":{"idset":"10021"}}
+{"timestamp":1714080109.2346005,"name":"offline","context":{"idset":"10022"}}
+{"timestamp":1714080109.2369497,"name":"offline","context":{"idset":"10023"}}
+{"timestamp":1714080109.2391214,"name":"offline","context":{"idset":"10024"}}
+{"timestamp":1714080109.2621386,"name":"offline","context":{"idset":"10025"}}
+{"timestamp":1714080109.2643428,"name":"offline","context":{"idset":"10026"}}
+{"timestamp":1714080109.2665949,"name":"offline","context":{"idset":"10027"}}
+{"timestamp":1714080109.268805,"name":"offline","context":{"idset":"10028"}}
+{"timestamp":1714080109.2709239,"name":"offline","context":{"idset":"10029"}}
+{"timestamp":1714080109.2731688,"name":"offline","context":{"idset":"10030"}}
+{"timestamp":1714080109.275358,"name":"offline","context":{"idset":"10031"}}
+{"timestamp":1714080109.2983358,"name":"offline","context":{"idset":"10032"}}
+{"timestamp":1714080109.300643,"name":"offline","context":{"idset":"10033"}}
+{"timestamp":1714080109.3029056,"name":"offline","context":{"idset":"10034"}}
+{"timestamp":1714080109.3051102,"name":"offline","context":{"idset":"10035"}}
+{"timestamp":1714080109.3073006,"name":"offline","context":{"idset":"10036"}}
+{"timestamp":1714080109.3095407,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1714080109.3117077,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1714080109.313668,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1714080109.3158357,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1714080109.3180611,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1714080109.3202591,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1714080109.3433337,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1714080109.3456573,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1714080109.3479118,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1714080109.3500311,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1714080109.3520129,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1714080109.3542697,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1714080109.3564851,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1714080109.3585064,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1714080109.3704841,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1714080109.3826203,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1714080109.3846235,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1714080109.3868568,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1714080109.3890514,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1714080109.3910666,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1714080109.412961,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1714080109.4245744,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1714080109.4264069,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1714080109.428251,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1714080109.4304774,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1714080109.4324543,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1714080109.4343979,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1714080109.4563432,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1714080109.458874,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1714080109.4607837,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1714080109.4627545,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1714080109.4650033,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1714080109.467032,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1714080109.4691346,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1714080109.5028799,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1714080109.5053141,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1714080109.5078402,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1714080109.5101953,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1714080109.512466,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1714080109.5147624,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1714080109.5169871,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1714080109.5408289,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1714080109.5429721,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1714080109.5450816,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1714080109.5471683,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1714080109.5491242,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1714080109.5513644,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1714080109.5559287,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1714080109.5787864,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1714080109.5808306,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1714080109.5830557,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1714080109.5852261,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1714080109.5872314,"name":"offline","context":{"idset":"10153"}}
+{"timestamp":1714080109.5892556,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1714080109.5914774,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1714080109.6041615,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1714080109.6061506,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1714080109.6081455,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1714080109.610069,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1714080109.6120358,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1714080109.6140468,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1714080109.616194,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1714080109.6440918,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1714080109.6465402,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1714080109.6490054,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1714080109.6514602,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1714080109.6543334,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1714080109.6567476,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1714080109.6591332,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1714080109.6752126,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1714080109.6776285,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1714080109.6801302,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1714080109.6825736,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1714080109.6850657,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1714080109.687516,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1714080109.6899104,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1714080109.717452,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1714080109.7223027,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1714080109.7250991,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1714080109.7274909,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1714080109.7298353,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1714080109.7324474,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1714080109.7352808,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1714080109.7648306,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1714080109.7801254,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1714080109.7825153,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1714080109.7852399,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1714080109.788106,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1714080109.7930939,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1714080109.7956727,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1714080109.7980139,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1714080109.8002675,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1714080109.8028188,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1714080109.8054726,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1714080109.8081446,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1714080109.8107228,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1714080109.813055,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1714080109.8153465,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1714080109.8179836,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1714080109.8206105,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1714080109.8232577,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1714080109.8257725,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1714080109.8281834,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1714080109.8306253,"name":"offline","context":{"idset":"10206"}}
+{"timestamp":1714080109.8458779,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1714080109.8485458,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1714080109.8512135,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1714080109.8538461,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1714080109.856436,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1714080109.8590047,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1714080109.8614845,"name":"offline","context":{"idset":"10357"}}
+{"timestamp":1714080109.8756044,"name":"offline","context":{"idset":"10358"}}
+{"timestamp":1714080109.8782165,"name":"offline","context":{"idset":"10359"}}
+{"timestamp":1714080109.8808453,"name":"offline","context":{"idset":"10360"}}
+{"timestamp":1714080109.8834238,"name":"offline","context":{"idset":"10361"}}
+{"timestamp":1714080109.8859901,"name":"offline","context":{"idset":"10362"}}
+{"timestamp":1714080109.8885496,"name":"offline","context":{"idset":"10363"}}
+{"timestamp":1714080109.891108,"name":"offline","context":{"idset":"10364"}}
+{"timestamp":1714080109.9166725,"name":"offline","context":{"idset":"10365"}}
+{"timestamp":1714080109.9317615,"name":"offline","context":{"idset":"10366"}}
+{"timestamp":1714080109.9343472,"name":"offline","context":{"idset":"10367"}}
+{"timestamp":1714080109.9368804,"name":"offline","context":{"idset":"10368"}}
+{"timestamp":1714080109.9393666,"name":"offline","context":{"idset":"10369"}}
+{"timestamp":1714080109.9416904,"name":"offline","context":{"idset":"10370"}}
+{"timestamp":1714080109.9435272,"name":"offline","context":{"idset":"10371"}}
+{"timestamp":1714080109.9458885,"name":"offline","context":{"idset":"10372"}}
+{"timestamp":1714080109.9484854,"name":"offline","context":{"idset":"10373"}}
+{"timestamp":1714080109.9510984,"name":"offline","context":{"idset":"10374"}}
+{"timestamp":1714080109.9537225,"name":"offline","context":{"idset":"10375"}}
+{"timestamp":1714080109.956208,"name":"offline","context":{"idset":"10376"}}
+{"timestamp":1714080109.9588599,"name":"offline","context":{"idset":"10377"}}
+{"timestamp":1714080109.9611154,"name":"offline","context":{"idset":"10378"}}
+{"timestamp":1714080109.9632969,"name":"offline","context":{"idset":"10379"}}
+{"timestamp":1714080109.9654725,"name":"offline","context":{"idset":"10380"}}
+{"timestamp":1714080109.9677157,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1714080109.9700358,"name":"offline","context":{"idset":"10382"}}
+{"timestamp":1714080109.9722321,"name":"offline","context":{"idset":"10383"}}
+{"timestamp":1714080109.9743681,"name":"offline","context":{"idset":"10384"}}
+{"timestamp":1714080109.9765406,"name":"offline","context":{"idset":"10385"}}
+{"timestamp":1714080109.9788654,"name":"offline","context":{"idset":"10386"}}
+{"timestamp":1714080109.9810402,"name":"offline","context":{"idset":"10387"}}
+{"timestamp":1714080109.9851718,"name":"offline","context":{"idset":"10388"}}
+{"timestamp":1714080110.0015085,"name":"offline","context":{"idset":"10389"}}
+{"timestamp":1714080110.0043418,"name":"offline","context":{"idset":"10390"}}
+{"timestamp":1714080110.0070961,"name":"offline","context":{"idset":"10391"}}
+{"timestamp":1714080110.0097792,"name":"offline","context":{"idset":"10392"}}
+{"timestamp":1714080110.0126708,"name":"offline","context":{"idset":"10393"}}
+{"timestamp":1714080110.0278699,"name":"offline","context":{"idset":"10394"}}
+{"timestamp":1714080110.0302689,"name":"offline","context":{"idset":"10395"}}
+{"timestamp":1714080110.0325737,"name":"offline","context":{"idset":"10396"}}
+{"timestamp":1714080110.0351424,"name":"offline","context":{"idset":"10397"}}
+{"timestamp":1714080110.0378358,"name":"offline","context":{"idset":"10398"}}
+{"timestamp":1714080110.0404806,"name":"offline","context":{"idset":"10399"}}
+{"timestamp":1714080110.0431094,"name":"offline","context":{"idset":"10400"}}
+{"timestamp":1714080110.04547,"name":"offline","context":{"idset":"10401"}}
+{"timestamp":1714080110.0722687,"name":"offline","context":{"idset":"10402"}}
+{"timestamp":1714080110.0749114,"name":"offline","context":{"idset":"10403"}}
+{"timestamp":1714080110.0773966,"name":"offline","context":{"idset":"10404"}}
+{"timestamp":1714080110.0800188,"name":"offline","context":{"idset":"10405"}}
+{"timestamp":1714080110.0827825,"name":"offline","context":{"idset":"10406"}}
+{"timestamp":1714080110.0891085,"name":"offline","context":{"idset":"10407"}}
+{"timestamp":1714080110.091737,"name":"offline","context":{"idset":"10408"}}
+{"timestamp":1714080110.1069772,"name":"offline","context":{"idset":"10409"}}
+{"timestamp":1714080110.1219645,"name":"offline","context":{"idset":"10410"}}
+{"timestamp":1714080110.1244648,"name":"offline","context":{"idset":"10411"}}
+{"timestamp":1714080110.127007,"name":"offline","context":{"idset":"10412"}}
+{"timestamp":1714080110.129364,"name":"offline","context":{"idset":"10413"}}
+{"timestamp":1714080110.1315155,"name":"offline","context":{"idset":"10414"}}
+{"timestamp":1714080110.1336765,"name":"offline","context":{"idset":"10415"}}
+{"timestamp":1714080110.1361587,"name":"offline","context":{"idset":"10416"}}
+{"timestamp":1714080110.1385038,"name":"offline","context":{"idset":"10417"}}
+{"timestamp":1714080110.1406996,"name":"offline","context":{"idset":"10418"}}
+{"timestamp":1714080110.1429772,"name":"offline","context":{"idset":"10419"}}
+{"timestamp":1714080110.1454797,"name":"offline","context":{"idset":"10420"}}
+{"timestamp":1714080110.1479354,"name":"offline","context":{"idset":"10423"}}
+{"timestamp":1714080110.1503651,"name":"offline","context":{"idset":"10424"}}
+{"timestamp":1714080110.1525948,"name":"offline","context":{"idset":"10425"}}
+{"timestamp":1714080110.1547709,"name":"offline","context":{"idset":"10426"}}
+{"timestamp":1714080110.1568754,"name":"offline","context":{"idset":"10427"}}
+{"timestamp":1714080110.159061,"name":"offline","context":{"idset":"10428"}}
+{"timestamp":1714080110.1617351,"name":"offline","context":{"idset":"10429"}}
+{"timestamp":1714080110.164619,"name":"offline","context":{"idset":"10430"}}
+{"timestamp":1714080110.182374,"name":"offline","context":{"idset":"10431"}}
+{"timestamp":1714080110.1978424,"name":"offline","context":{"idset":"10432"}}
+{"timestamp":1714080110.200429,"name":"offline","context":{"idset":"10433"}}
+{"timestamp":1714080110.2029598,"name":"offline","context":{"idset":"10434"}}
+{"timestamp":1714080110.205251,"name":"offline","context":{"idset":"10435"}}
+{"timestamp":1714080110.2077153,"name":"offline","context":{"idset":"10436"}}
+{"timestamp":1714080110.2101238,"name":"offline","context":{"idset":"10437"}}
+{"timestamp":1714080110.2126076,"name":"offline","context":{"idset":"10438"}}
+{"timestamp":1714080110.2151663,"name":"offline","context":{"idset":"10439"}}
+{"timestamp":1714080110.2175863,"name":"offline","context":{"idset":"10440"}}
+{"timestamp":1714080110.2195604,"name":"offline","context":{"idset":"10441"}}
+{"timestamp":1714080110.2217028,"name":"offline","context":{"idset":"10442"}}
+{"timestamp":1714080110.2257102,"name":"offline","context":{"idset":"10443"}}
+{"timestamp":1714080110.2279167,"name":"offline","context":{"idset":"10444"}}
+{"timestamp":1714080110.2300808,"name":"offline","context":{"idset":"10445"}}
+{"timestamp":1714080110.232568,"name":"offline","context":{"idset":"10446"}}
+{"timestamp":1714080110.2350349,"name":"offline","context":{"idset":"10447"}}
+{"timestamp":1714080110.2375305,"name":"offline","context":{"idset":"10448"}}
+{"timestamp":1714080110.2396953,"name":"offline","context":{"idset":"10449"}}
+{"timestamp":1714080110.2419963,"name":"offline","context":{"idset":"10450"}}
+{"timestamp":1714080110.2444935,"name":"offline","context":{"idset":"10451"}}
+{"timestamp":1714080110.247175,"name":"offline","context":{"idset":"10452"}}
+{"timestamp":1714080110.2497702,"name":"offline","context":{"idset":"10453"}}
+{"timestamp":1714080110.2654932,"name":"offline","context":{"idset":"10454"}}
+{"timestamp":1714080110.2685266,"name":"offline","context":{"idset":"10455"}}
+{"timestamp":1714080110.2709849,"name":"offline","context":{"idset":"10456"}}
+{"timestamp":1714080110.2731225,"name":"offline","context":{"idset":"10457"}}
+{"timestamp":1714080110.2754529,"name":"offline","context":{"idset":"10458"}}
+{"timestamp":1714080110.2781692,"name":"offline","context":{"idset":"10459"}}
+{"timestamp":1714080110.2806818,"name":"offline","context":{"idset":"10460"}}
+{"timestamp":1714080110.282809,"name":"offline","context":{"idset":"10461"}}
+{"timestamp":1714080110.2849247,"name":"offline","context":{"idset":"10462"}}
+{"timestamp":1714080110.2874951,"name":"offline","context":{"idset":"10463"}}
+{"timestamp":1714080110.289854,"name":"offline","context":{"idset":"10464"}}
+{"timestamp":1714080110.2923858,"name":"offline","context":{"idset":"10465"}}
+{"timestamp":1714080110.2947128,"name":"offline","context":{"idset":"10466"}}
+{"timestamp":1714080110.2968397,"name":"offline","context":{"idset":"10467"}}
+{"timestamp":1714080110.2992382,"name":"offline","context":{"idset":"10468"}}
+{"timestamp":1714080110.3018489,"name":"offline","context":{"idset":"10469"}}
+{"timestamp":1714080110.3043933,"name":"offline","context":{"idset":"10470"}}
+{"timestamp":1714080110.3092239,"name":"offline","context":{"idset":"10471"}}
+{"timestamp":1714080110.3111415,"name":"offline","context":{"idset":"10472"}}
+{"timestamp":1714080110.3134418,"name":"offline","context":{"idset":"10473"}}
+{"timestamp":1714080110.3156657,"name":"offline","context":{"idset":"10474"}}
+{"timestamp":1714080110.3181515,"name":"offline","context":{"idset":"10475"}}
+{"timestamp":1714080110.3207328,"name":"offline","context":{"idset":"10476"}}
+{"timestamp":1714080110.3228731,"name":"offline","context":{"idset":"10477"}}
+{"timestamp":1714080110.3250122,"name":"offline","context":{"idset":"10478"}}
+{"timestamp":1714080110.3275115,"name":"offline","context":{"idset":"10479"}}
+{"timestamp":1714080110.3302343,"name":"offline","context":{"idset":"10480"}}
+{"timestamp":1714080110.3325129,"name":"offline","context":{"idset":"10481"}}
+{"timestamp":1714080110.335402,"name":"offline","context":{"idset":"10482"}}
+{"timestamp":1714080110.3381586,"name":"offline","context":{"idset":"10483"}}
+{"timestamp":1714080110.3404877,"name":"offline","context":{"idset":"10484"}}
+{"timestamp":1714080110.3557971,"name":"offline","context":{"idset":"10485"}}
+{"timestamp":1714080110.357949,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1714080110.3600607,"name":"offline","context":{"idset":"10487"}}
+{"timestamp":1714080110.362215,"name":"offline","context":{"idset":"10488"}}
+{"timestamp":1714080110.3644638,"name":"offline","context":{"idset":"10489"}}
+{"timestamp":1714080110.3667958,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1714080110.3693349,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1714080110.3719618,"name":"offline","context":{"idset":"10492"}}
+{"timestamp":1714080110.3751814,"name":"offline","context":{"idset":"10493"}}
+{"timestamp":1714080110.3776593,"name":"offline","context":{"idset":"10494"}}
+{"timestamp":1714080110.3799605,"name":"offline","context":{"idset":"10495"}}
+{"timestamp":1714080110.3961728,"name":"offline","context":{"idset":"10496"}}
+{"timestamp":1714080110.3986733,"name":"offline","context":{"idset":"10497"}}
+{"timestamp":1714080110.4013913,"name":"offline","context":{"idset":"10498"}}
+{"timestamp":1714080110.4041989,"name":"offline","context":{"idset":"10499"}}
+{"timestamp":1714080110.4068401,"name":"offline","context":{"idset":"10500"}}
+{"timestamp":1714080110.4093823,"name":"offline","context":{"idset":"10501"}}
+{"timestamp":1714080110.4121423,"name":"offline","context":{"idset":"10502"}}
+{"timestamp":1714080110.4147301,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1714080110.4171882,"name":"offline","context":{"idset":"10504"}}
+{"timestamp":1714080110.4197774,"name":"offline","context":{"idset":"10505"}}
+{"timestamp":1714080110.436866,"name":"offline","context":{"idset":"10506"}}
+{"timestamp":1714080110.4389451,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1714080110.4413319,"name":"offline","context":{"idset":"10508"}}
+{"timestamp":1714080110.4436271,"name":"offline","context":{"idset":"10509"}}
+{"timestamp":1714080110.4460714,"name":"offline","context":{"idset":"10510"}}
+{"timestamp":1714080110.4488742,"name":"offline","context":{"idset":"10511"}}
+{"timestamp":1714080110.4514041,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1714080110.4536614,"name":"offline","context":{"idset":"10513"}}
+{"timestamp":1714080110.4557257,"name":"offline","context":{"idset":"10514"}}
+{"timestamp":1714080110.4579849,"name":"offline","context":{"idset":"10515"}}
+{"timestamp":1714080110.4808314,"name":"offline","context":{"idset":"10516"}}
+{"timestamp":1714080110.4824641,"name":"offline","context":{"idset":"10517"}}
+{"timestamp":1714080110.4842153,"name":"offline","context":{"idset":"10518"}}
+{"timestamp":1714080110.4860966,"name":"offline","context":{"idset":"10519"}}
+{"timestamp":1714080110.4879537,"name":"offline","context":{"idset":"10520"}}
+{"timestamp":1714080110.4897463,"name":"offline","context":{"idset":"10521"}}
+{"timestamp":1714080110.4916291,"name":"offline","context":{"idset":"10522"}}
+{"timestamp":1714080110.4937336,"name":"offline","context":{"idset":"10523"}}
+{"timestamp":1714080110.4957457,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1714080110.4975584,"name":"offline","context":{"idset":"10525"}}
+{"timestamp":1714080110.5101781,"name":"offline","context":{"idset":"10526"}}
+{"timestamp":1714080110.511734,"name":"offline","context":{"idset":"10527"}}
+{"timestamp":1714080110.5135369,"name":"offline","context":{"idset":"10528"}}
+{"timestamp":1714080110.5150638,"name":"offline","context":{"idset":"10529"}}
+{"timestamp":1714080110.5165558,"name":"offline","context":{"idset":"10530"}}
+{"timestamp":1714080110.5180593,"name":"offline","context":{"idset":"10531"}}
+{"timestamp":1714080110.5197115,"name":"offline","context":{"idset":"10532"}}
+{"timestamp":1714080110.5218396,"name":"offline","context":{"idset":"10533"}}
+{"timestamp":1714080110.5237241,"name":"offline","context":{"idset":"10534"}}
+{"timestamp":1714080110.5254943,"name":"offline","context":{"idset":"10535"}}
+{"timestamp":1714080110.5270791,"name":"offline","context":{"idset":"10536"}}
+{"timestamp":1714080110.537178,"name":"offline","context":{"idset":"10537"}}
+{"timestamp":1714080110.5389926,"name":"offline","context":{"idset":"10538"}}
+{"timestamp":1714080110.5405948,"name":"offline","context":{"idset":"10539"}}
+{"timestamp":1714080110.5421324,"name":"offline","context":{"idset":"10540"}}
+{"timestamp":1714080110.5436623,"name":"offline","context":{"idset":"10541"}}
+{"timestamp":1714080110.5453846,"name":"offline","context":{"idset":"10542"}}
+{"timestamp":1714080110.5469148,"name":"offline","context":{"idset":"10543"}}
+{"timestamp":1714080110.548631,"name":"offline","context":{"idset":"10544"}}
+{"timestamp":1714080110.5506074,"name":"offline","context":{"idset":"10545"}}
+{"timestamp":1714080110.552258,"name":"offline","context":{"idset":"10546"}}
+{"timestamp":1714080110.5715673,"name":"offline","context":{"idset":"10547"}}
+{"timestamp":1714080110.5734704,"name":"offline","context":{"idset":"10548"}}
+{"timestamp":1714080110.5751405,"name":"offline","context":{"idset":"10549"}}
+{"timestamp":1714080110.5767584,"name":"offline","context":{"idset":"10550"}}
+{"timestamp":1714080110.5786738,"name":"offline","context":{"idset":"10551"}}
+{"timestamp":1714080110.5807452,"name":"offline","context":{"idset":"10552"}}
+{"timestamp":1714080110.5826998,"name":"offline","context":{"idset":"10553"}}
+{"timestamp":1714080110.5845888,"name":"offline","context":{"idset":"10554"}}
+{"timestamp":1714080110.5970399,"name":"offline","context":{"idset":"10555"}}
+{"timestamp":1714080110.5987942,"name":"offline","context":{"idset":"10556"}}
+{"timestamp":1714080110.6005058,"name":"offline","context":{"idset":"10557"}}
+{"timestamp":1714080110.6021504,"name":"offline","context":{"idset":"10558"}}
+{"timestamp":1714080110.6042185,"name":"offline","context":{"idset":"10559"}}
+{"timestamp":1714080110.6060216,"name":"offline","context":{"idset":"10560"}}
+{"timestamp":1714080110.6078992,"name":"offline","context":{"idset":"10561"}}
+{"timestamp":1714080110.6100347,"name":"offline","context":{"idset":"10562"}}
+{"timestamp":1714080110.6220837,"name":"offline","context":{"idset":"10563"}}
+{"timestamp":1714080110.6239457,"name":"offline","context":{"idset":"10564"}}
+{"timestamp":1714080110.625982,"name":"offline","context":{"idset":"10565"}}
+{"timestamp":1714080110.6278143,"name":"offline","context":{"idset":"10566"}}
+{"timestamp":1714080110.6300564,"name":"offline","context":{"idset":"10567"}}
+{"timestamp":1714080110.632055,"name":"offline","context":{"idset":"10568"}}
+{"timestamp":1714080110.6340311,"name":"offline","context":{"idset":"10569"}}
+{"timestamp":1714080110.6364226,"name":"offline","context":{"idset":"10570"}}
+{"timestamp":1714080110.6384823,"name":"offline","context":{"idset":"10571"}}
+{"timestamp":1714080110.6618795,"name":"offline","context":{"idset":"10572"}}
+{"timestamp":1714080110.6638815,"name":"offline","context":{"idset":"10573"}}
+{"timestamp":1714080110.6658158,"name":"offline","context":{"idset":"10574"}}
+{"timestamp":1714080110.6676114,"name":"offline","context":{"idset":"10575"}}
+{"timestamp":1714080110.6695704,"name":"offline","context":{"idset":"10576"}}
+{"timestamp":1714080110.6712139,"name":"offline","context":{"idset":"10577"}}
+{"timestamp":1714080110.6729825,"name":"offline","context":{"idset":"10578"}}
+{"timestamp":1714080110.6751118,"name":"offline","context":{"idset":"10579"}}
+{"timestamp":1714080110.7006502,"name":"offline","context":{"idset":"10580"}}
+{"timestamp":1714080110.7124457,"name":"offline","context":{"idset":"10581"}}
+{"timestamp":1714080110.7205594,"name":"offline","context":{"idset":"10582"}}
+{"timestamp":1714080110.722441,"name":"offline","context":{"idset":"10583"}}
+{"timestamp":1714080110.7243874,"name":"offline","context":{"idset":"10584"}}
+{"timestamp":1714080110.7263179,"name":"offline","context":{"idset":"10585"}}
+{"timestamp":1714080110.7281647,"name":"offline","context":{"idset":"10586"}}
+{"timestamp":1714080110.7595127,"name":"offline","context":{"idset":"10587"}}
+{"timestamp":1714080110.7676373,"name":"offline","context":{"idset":"10588"}}
+{"timestamp":1714080110.7692842,"name":"offline","context":{"idset":"10589"}}
+{"timestamp":1714080110.7708409,"name":"offline","context":{"idset":"10590"}}
+{"timestamp":1714080110.7796779,"name":"offline","context":{"idset":"10591"}}
+{"timestamp":1714080110.7815676,"name":"offline","context":{"idset":"10592"}}
+{"timestamp":1714080110.7832744,"name":"offline","context":{"idset":"10593"}}
+{"timestamp":1714080110.7848079,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1714080110.7942605,"name":"offline","context":{"idset":"10595"}}
+{"timestamp":1714080110.8122311,"name":"offline","context":{"idset":"10596"}}
+{"timestamp":1714080110.813957,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1714080110.8265927,"name":"offline","context":{"idset":"10598"}}
+{"timestamp":1714080110.8284476,"name":"offline","context":{"idset":"10599"}}
+{"timestamp":1714080110.830301,"name":"offline","context":{"idset":"10600"}}
+{"timestamp":1714080110.8318679,"name":"offline","context":{"idset":"10601"}}
+{"timestamp":1714080110.8333964,"name":"offline","context":{"idset":"10602"}}
+{"timestamp":1714080110.8353019,"name":"offline","context":{"idset":"10603"}}
+{"timestamp":1714080110.8371916,"name":"offline","context":{"idset":"10604"}}
+{"timestamp":1714080110.8388493,"name":"offline","context":{"idset":"10605"}}
+{"timestamp":1714080110.8404231,"name":"offline","context":{"idset":"10606"}}
+{"timestamp":1714080110.8420956,"name":"offline","context":{"idset":"10607"}}
+{"timestamp":1714080110.8439605,"name":"offline","context":{"idset":"10608"}}
+{"timestamp":1714080110.8458307,"name":"offline","context":{"idset":"10609"}}
+{"timestamp":1714080110.847537,"name":"offline","context":{"idset":"10610"}}
+{"timestamp":1714080110.8494532,"name":"offline","context":{"idset":"10611"}}
+{"timestamp":1714080110.8512881,"name":"offline","context":{"idset":"10612"}}
+{"timestamp":1714080110.8532343,"name":"offline","context":{"idset":"10613"}}
+{"timestamp":1714080110.8550999,"name":"offline","context":{"idset":"10614"}}
+{"timestamp":1714080110.8568847,"name":"offline","context":{"idset":"10615"}}
+{"timestamp":1714080110.8587856,"name":"offline","context":{"idset":"10616"}}
+{"timestamp":1714080110.8716915,"name":"offline","context":{"idset":"10617"}}
+{"timestamp":1714080110.8733702,"name":"offline","context":{"idset":"10618"}}
+{"timestamp":1714080110.8750403,"name":"offline","context":{"idset":"10619"}}
+{"timestamp":1714080110.8767359,"name":"offline","context":{"idset":"10620"}}
+{"timestamp":1714080110.8786767,"name":"offline","context":{"idset":"10621"}}
+{"timestamp":1714080110.8804691,"name":"offline","context":{"idset":"10622"}}
+{"timestamp":1714080110.8823683,"name":"offline","context":{"idset":"10623"}}
+{"timestamp":1714080110.8949792,"name":"offline","context":{"idset":"10624"}}
+{"timestamp":1714080110.8968549,"name":"offline","context":{"idset":"10625"}}
+{"timestamp":1714080110.898716,"name":"offline","context":{"idset":"10626"}}
+{"timestamp":1714080110.9006002,"name":"offline","context":{"idset":"10627"}}
+{"timestamp":1714080110.902586,"name":"offline","context":{"idset":"10628"}}
+{"timestamp":1714080110.9044948,"name":"offline","context":{"idset":"10629"}}
+{"timestamp":1714080110.9064689,"name":"offline","context":{"idset":"10630"}}
+{"timestamp":1714080110.9297566,"name":"offline","context":{"idset":"10631"}}
+{"timestamp":1714080110.9421043,"name":"offline","context":{"idset":"10632"}}
+{"timestamp":1714080110.9438963,"name":"offline","context":{"idset":"10633"}}
+{"timestamp":1714080110.9456506,"name":"offline","context":{"idset":"10634"}}
+{"timestamp":1714080110.9474719,"name":"offline","context":{"idset":"10635"}}
+{"timestamp":1714080110.9538736,"name":"offline","context":{"idset":"10636"}}
+{"timestamp":1714080110.9557576,"name":"offline","context":{"idset":"10637"}}
+{"timestamp":1714080110.957927,"name":"offline","context":{"idset":"10638"}}
+{"timestamp":1714080110.9596725,"name":"offline","context":{"idset":"10639"}}
+{"timestamp":1714080110.9614556,"name":"offline","context":{"idset":"10640"}}
+{"timestamp":1714080110.9632125,"name":"offline","context":{"idset":"10641"}}
+{"timestamp":1714080110.9649751,"name":"offline","context":{"idset":"10642"}}
+{"timestamp":1714080110.9671538,"name":"offline","context":{"idset":"10643"}}
+{"timestamp":1714080110.9689822,"name":"offline","context":{"idset":"10644"}}
+{"timestamp":1714080110.9707227,"name":"offline","context":{"idset":"10645"}}
+{"timestamp":1714080110.9724805,"name":"offline","context":{"idset":"10646"}}
+{"timestamp":1714080110.9742291,"name":"offline","context":{"idset":"10647"}}
+{"timestamp":1714080110.9759901,"name":"offline","context":{"idset":"10648"}}
+{"timestamp":1714080110.9781799,"name":"offline","context":{"idset":"10649"}}
+{"timestamp":1714080110.9800005,"name":"offline","context":{"idset":"10650"}}
+{"timestamp":1714080110.9818556,"name":"offline","context":{"idset":"10651"}}
+{"timestamp":1714080110.98353,"name":"offline","context":{"idset":"10652"}}
+{"timestamp":1714080110.9853313,"name":"offline","context":{"idset":"10653"}}
+{"timestamp":1714080110.9876833,"name":"offline","context":{"idset":"10654"}}
+{"timestamp":1714080110.9895654,"name":"offline","context":{"idset":"10655"}}
+{"timestamp":1714080111.0004079,"name":"offline","context":{"idset":"10656"}}
+{"timestamp":1714080111.0022008,"name":"offline","context":{"idset":"10657"}}
+{"timestamp":1714080111.0040147,"name":"offline","context":{"idset":"10658"}}
+{"timestamp":1714080111.0057514,"name":"offline","context":{"idset":"10659"}}
+{"timestamp":1714080111.008081,"name":"offline","context":{"idset":"10660"}}
+{"timestamp":1714080111.0099206,"name":"offline","context":{"idset":"10661"}}
+{"timestamp":1714080111.0116832,"name":"offline","context":{"idset":"10662"}}
+{"timestamp":1714080111.0239305,"name":"offline","context":{"idset":"10663"}}
+{"timestamp":1714080111.0257208,"name":"offline","context":{"idset":"10664"}}
+{"timestamp":1714080111.027601,"name":"offline","context":{"idset":"10665"}}
+{"timestamp":1714080111.0294607,"name":"offline","context":{"idset":"10666"}}
+{"timestamp":1714080111.0313017,"name":"offline","context":{"idset":"10667"}}
+{"timestamp":1714080111.0330594,"name":"offline","context":{"idset":"10668"}}
+{"timestamp":1714080111.0348208,"name":"offline","context":{"idset":"10669"}}
+{"timestamp":1714080111.0365825,"name":"offline","context":{"idset":"10670"}}
+{"timestamp":1714080111.0491076,"name":"offline","context":{"idset":"10671"}}
+{"timestamp":1714080111.0509014,"name":"offline","context":{"idset":"10672"}}
+{"timestamp":1714080111.0526843,"name":"offline","context":{"idset":"10673"}}
+{"timestamp":1714080111.0751123,"name":"offline","context":{"idset":"10674"}}
+{"timestamp":1714080111.0768247,"name":"offline","context":{"idset":"10675"}}
+{"timestamp":1714080111.0786057,"name":"offline","context":{"idset":"10676"}}
+{"timestamp":1714080111.0803053,"name":"offline","context":{"idset":"10677"}}
+{"timestamp":1714080111.0820906,"name":"offline","context":{"idset":"10678"}}
+{"timestamp":1714080111.0838563,"name":"offline","context":{"idset":"10679"}}
+{"timestamp":1714080111.0855718,"name":"offline","context":{"idset":"10680"}}
+{"timestamp":1714080111.087611,"name":"offline","context":{"idset":"10681"}}
+{"timestamp":1714080111.0892522,"name":"offline","context":{"idset":"10682"}}
+{"timestamp":1714080111.0909758,"name":"offline","context":{"idset":"10683"}}
+{"timestamp":1714080111.0925913,"name":"offline","context":{"idset":"10684"}}
+{"timestamp":1714080111.0942647,"name":"offline","context":{"idset":"10685"}}
+{"timestamp":1714080111.096123,"name":"offline","context":{"idset":"10686"}}
+{"timestamp":1714080111.097729,"name":"offline","context":{"idset":"10687"}}
+{"timestamp":1714080111.0993586,"name":"offline","context":{"idset":"10688"}}
+{"timestamp":1714080111.1012523,"name":"offline","context":{"idset":"10689"}}
+{"timestamp":1714080111.1030228,"name":"offline","context":{"idset":"10690"}}
+{"timestamp":1714080111.1051309,"name":"offline","context":{"idset":"10691"}}
+{"timestamp":1714080111.1067848,"name":"offline","context":{"idset":"10692"}}
+{"timestamp":1714080111.1086295,"name":"offline","context":{"idset":"10693"}}
+{"timestamp":1714080111.1104374,"name":"offline","context":{"idset":"10694"}}
+{"timestamp":1714080111.1128094,"name":"offline","context":{"idset":"10695"}}
+{"timestamp":1714080111.1145434,"name":"offline","context":{"idset":"10696"}}
+{"timestamp":1714080111.1161864,"name":"offline","context":{"idset":"10697"}}
+{"timestamp":1714080111.1180236,"name":"offline","context":{"idset":"10698"}}
+{"timestamp":1714080111.1201203,"name":"offline","context":{"idset":"10699"}}
+{"timestamp":1714080111.1336298,"name":"offline","context":{"idset":"10700"}}
+{"timestamp":1714080111.1353681,"name":"offline","context":{"idset":"10701"}}
+{"timestamp":1714080111.1372437,"name":"offline","context":{"idset":"10702"}}
+{"timestamp":1714080111.1391635,"name":"offline","context":{"idset":"10703"}}
+{"timestamp":1714080111.1410179,"name":"offline","context":{"idset":"10704"}}
+{"timestamp":1714080111.1437573,"name":"offline","context":{"idset":"10705"}}
+{"timestamp":1714080111.1453867,"name":"offline","context":{"idset":"10706"}}
+{"timestamp":1714080111.1470015,"name":"offline","context":{"idset":"10707"}}
+{"timestamp":1714080111.1684463,"name":"offline","context":{"idset":"10708"}}
+{"timestamp":1714080111.1701477,"name":"offline","context":{"idset":"10709"}}
+{"timestamp":1714080111.1717925,"name":"offline","context":{"idset":"10710"}}
+{"timestamp":1714080111.1734393,"name":"offline","context":{"idset":"10711"}}
+{"timestamp":1714080111.17507,"name":"offline","context":{"idset":"10712"}}
+{"timestamp":1714080111.177129,"name":"offline","context":{"idset":"10713"}}
+{"timestamp":1714080111.1787946,"name":"offline","context":{"idset":"10714"}}
+{"timestamp":1714080111.1804786,"name":"offline","context":{"idset":"10715"}}
+{"timestamp":1714080111.1926317,"name":"offline","context":{"idset":"10716"}}
+{"timestamp":1714080111.1942546,"name":"offline","context":{"idset":"10717"}}
+{"timestamp":1714080111.2062893,"name":"offline","context":{"idset":"10718"}}
+{"timestamp":1714080111.2079391,"name":"offline","context":{"idset":"10719"}}
+{"timestamp":1714080111.2095709,"name":"offline","context":{"idset":"10720"}}
+{"timestamp":1714080111.2114365,"name":"offline","context":{"idset":"10721"}}
+{"timestamp":1714080111.2130661,"name":"offline","context":{"idset":"10722"}}
+{"timestamp":1714080111.2147415,"name":"offline","context":{"idset":"10723"}}
+{"timestamp":1714080111.2163424,"name":"offline","context":{"idset":"10724"}}
+{"timestamp":1714080111.2179401,"name":"offline","context":{"idset":"10725"}}
+{"timestamp":1714080111.2195194,"name":"offline","context":{"idset":"10726"}}
+{"timestamp":1714080111.2211118,"name":"offline","context":{"idset":"10727"}}
+{"timestamp":1714080111.2226963,"name":"offline","context":{"idset":"10728"}}
+{"timestamp":1714080111.2243171,"name":"offline","context":{"idset":"10729"}}
+{"timestamp":1714080111.2259071,"name":"offline","context":{"idset":"10730"}}
+{"timestamp":1714080111.2275019,"name":"offline","context":{"idset":"10731"}}
+{"timestamp":1714080111.238909,"name":"offline","context":{"idset":"10732"}}
+{"timestamp":1714080111.2404957,"name":"offline","context":{"idset":"10733"}}
+{"timestamp":1714080111.2420869,"name":"offline","context":{"idset":"10734"}}
+{"timestamp":1714080111.2436678,"name":"offline","context":{"idset":"10735"}}
+{"timestamp":1714080111.2453539,"name":"offline","context":{"idset":"10736"}}
+{"timestamp":1714080111.246944,"name":"offline","context":{"idset":"10737"}}
+{"timestamp":1714080111.2485175,"name":"offline","context":{"idset":"10738"}}
+{"timestamp":1714080111.2500906,"name":"offline","context":{"idset":"10739"}}
+{"timestamp":1714080111.2516623,"name":"offline","context":{"idset":"10740"}}
+{"timestamp":1714080111.2532358,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1714080111.2548218,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1714080111.2563875,"name":"offline","context":{"idset":"10743"}}
+{"timestamp":1714080111.2579651,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1714080111.2595401,"name":"offline","context":{"idset":"10745"}}
+{"timestamp":1714080111.2611256,"name":"offline","context":{"idset":"10746"}}
+{"timestamp":1714080111.262691,"name":"offline","context":{"idset":"10747"}}
+{"timestamp":1714080111.2642629,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1714080111.2658417,"name":"offline","context":{"idset":"10749"}}
+{"timestamp":1714080111.2674165,"name":"offline","context":{"idset":"10750"}}
+{"timestamp":1714080111.2689829,"name":"offline","context":{"idset":"10751"}}
+{"timestamp":1714080111.2705495,"name":"offline","context":{"idset":"10752"}}
+{"timestamp":1714080111.2721291,"name":"offline","context":{"idset":"10753"}}
+{"timestamp":1714080111.2736912,"name":"offline","context":{"idset":"10754"}}
+{"timestamp":1714080111.2752678,"name":"offline","context":{"idset":"10755"}}
+{"timestamp":1714080111.2917378,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1714080111.2933338,"name":"offline","context":{"idset":"10757"}}
+{"timestamp":1714080111.2948949,"name":"offline","context":{"idset":"10758"}}
+{"timestamp":1714080111.2964723,"name":"offline","context":{"idset":"10759"}}
+{"timestamp":1714080111.3077321,"name":"offline","context":{"idset":"10760"}}
+{"timestamp":1714080111.3092999,"name":"offline","context":{"idset":"10761"}}
+{"timestamp":1714080111.3108876,"name":"offline","context":{"idset":"10762"}}
+{"timestamp":1714080111.3124454,"name":"offline","context":{"idset":"10763"}}
+{"timestamp":1714080111.3140125,"name":"offline","context":{"idset":"10764"}}
+{"timestamp":1714080111.3155701,"name":"offline","context":{"idset":"10765"}}
+{"timestamp":1714080111.3171875,"name":"offline","context":{"idset":"10766"}}
+{"timestamp":1714080111.318759,"name":"offline","context":{"idset":"10767"}}
+{"timestamp":1714080111.330018,"name":"offline","context":{"idset":"10768"}}
+{"timestamp":1714080111.3316789,"name":"offline","context":{"idset":"10769"}}
+{"timestamp":1714080111.3332489,"name":"offline","context":{"idset":"10770"}}
+{"timestamp":1714080111.3347862,"name":"offline","context":{"idset":"10771"}}
+{"timestamp":1714080111.3363612,"name":"offline","context":{"idset":"10772"}}
+{"timestamp":1714080111.3379161,"name":"offline","context":{"idset":"10773"}}
+{"timestamp":1714080111.3394697,"name":"offline","context":{"idset":"10774"}}
+{"timestamp":1714080111.3410971,"name":"offline","context":{"idset":"10775"}}
+{"timestamp":1714080111.3619823,"name":"offline","context":{"idset":"10776"}}
+{"timestamp":1714080111.3636298,"name":"offline","context":{"idset":"10777"}}
+{"timestamp":1714080111.3653078,"name":"offline","context":{"idset":"10778"}}
+{"timestamp":1714080111.3669066,"name":"offline","context":{"idset":"10779"}}
+{"timestamp":1714080111.3684869,"name":"offline","context":{"idset":"10780"}}
+{"timestamp":1714080111.370975,"name":"offline","context":{"idset":"10781"}}
+{"timestamp":1714080111.3724835,"name":"offline","context":{"idset":"10782"}}
+{"timestamp":1714080111.3740351,"name":"offline","context":{"idset":"10783"}}
+{"timestamp":1714080111.3846838,"name":"offline","context":{"idset":"10784"}}
+{"timestamp":1714080111.3863511,"name":"offline","context":{"idset":"10785"}}
+{"timestamp":1714080111.3879774,"name":"offline","context":{"idset":"10786"}}
+{"timestamp":1714080111.389607,"name":"offline","context":{"idset":"10787"}}
+{"timestamp":1714080111.3912544,"name":"offline","context":{"idset":"10788"}}
+{"timestamp":1714080111.3929193,"name":"offline","context":{"idset":"10789"}}
+{"timestamp":1714080111.3944187,"name":"offline","context":{"idset":"10790"}}
+{"timestamp":1714080111.3959336,"name":"offline","context":{"idset":"10791"}}
+{"timestamp":1714080111.4171453,"name":"offline","context":{"idset":"10792"}}
+{"timestamp":1714080111.4188819,"name":"offline","context":{"idset":"10793"}}
+{"timestamp":1714080111.4205573,"name":"offline","context":{"idset":"10794"}}
+{"timestamp":1714080111.4222875,"name":"offline","context":{"idset":"10795"}}
+{"timestamp":1714080111.4239707,"name":"offline","context":{"idset":"10796"}}
+{"timestamp":1714080111.4255304,"name":"offline","context":{"idset":"10797"}}
+{"timestamp":1714080111.4269328,"name":"offline","context":{"idset":"10798"}}
+{"timestamp":1714080111.428313,"name":"offline","context":{"idset":"10799"}}
+{"timestamp":1714080111.4501014,"name":"offline","context":{"idset":"10800"}}
+{"timestamp":1714080111.4517002,"name":"offline","context":{"idset":"10801"}}
+{"timestamp":1714080111.4533319,"name":"offline","context":{"idset":"10802"}}
+{"timestamp":1714080111.4549756,"name":"offline","context":{"idset":"10803"}}
+{"timestamp":1714080111.4564557,"name":"offline","context":{"idset":"10804"}}
+{"timestamp":1714080111.4579356,"name":"offline","context":{"idset":"10805"}}
+{"timestamp":1714080111.4593709,"name":"offline","context":{"idset":"10806"}}
+{"timestamp":1714080111.4610384,"name":"offline","context":{"idset":"10807"}}
+{"timestamp":1714080111.4764862,"name":"offline","context":{"idset":"10808"}}
+{"timestamp":1714080111.4780867,"name":"offline","context":{"idset":"10809"}}
+{"timestamp":1714080111.4797134,"name":"offline","context":{"idset":"10810"}}
+{"timestamp":1714080111.4812417,"name":"offline","context":{"idset":"10811"}}
+{"timestamp":1714080111.4827385,"name":"offline","context":{"idset":"10812"}}
+{"timestamp":1714080111.4843259,"name":"offline","context":{"idset":"10813"}}
+{"timestamp":1714080111.4859648,"name":"offline","context":{"idset":"10814"}}
+{"timestamp":1714080111.5077119,"name":"offline","context":{"idset":"10815"}}
+{"timestamp":1714080111.5092039,"name":"offline","context":{"idset":"10816"}}
+{"timestamp":1714080111.5106039,"name":"offline","context":{"idset":"10817"}}
+{"timestamp":1714080111.511981,"name":"offline","context":{"idset":"10818"}}
+{"timestamp":1714080111.513443,"name":"offline","context":{"idset":"10819"}}
+{"timestamp":1714080111.514837,"name":"offline","context":{"idset":"10820"}}
+{"timestamp":1714080111.5162752,"name":"offline","context":{"idset":"10821"}}
+{"timestamp":1714080111.5177069,"name":"offline","context":{"idset":"10822"}}
+{"timestamp":1714080111.5294273,"name":"offline","context":{"idset":"10823"}}
+{"timestamp":1714080111.531461,"name":"offline","context":{"idset":"10824"}}
+{"timestamp":1714080111.5336838,"name":"offline","context":{"idset":"10825"}}
+{"timestamp":1714080111.535362,"name":"offline","context":{"idset":"10826"}}
+{"timestamp":1714080111.5370085,"name":"offline","context":{"idset":"10827"}}
+{"timestamp":1714080111.5386479,"name":"offline","context":{"idset":"10828"}}
+{"timestamp":1714080111.5401373,"name":"offline","context":{"idset":"10829"}}
+{"timestamp":1714080111.5418169,"name":"offline","context":{"idset":"10830"}}
+{"timestamp":1714080111.5623651,"name":"offline","context":{"idset":"10831"}}
+{"timestamp":1714080111.5637105,"name":"offline","context":{"idset":"10832"}}
+{"timestamp":1714080111.5652103,"name":"offline","context":{"idset":"10833"}}
+{"timestamp":1714080111.5666361,"name":"offline","context":{"idset":"10834"}}
+{"timestamp":1714080111.5680609,"name":"offline","context":{"idset":"10835"}}
+{"timestamp":1714080111.569098,"name":"offline","context":{"idset":"10836"}}
+{"timestamp":1714080111.570528,"name":"offline","context":{"idset":"10837"}}
+{"timestamp":1714080111.5719497,"name":"offline","context":{"idset":"10838"}}
+{"timestamp":1714080111.5892053,"name":"offline","context":{"idset":"10839"}}
+{"timestamp":1714080111.5903828,"name":"offline","context":{"idset":"10840"}}
+{"timestamp":1714080111.5914059,"name":"offline","context":{"idset":"10841"}}
+{"timestamp":1714080111.5925207,"name":"offline","context":{"idset":"10842"}}
+{"timestamp":1714080111.5938704,"name":"offline","context":{"idset":"10843"}}
+{"timestamp":1714080111.5952871,"name":"offline","context":{"idset":"10844"}}
+{"timestamp":1714080111.5966125,"name":"offline","context":{"idset":"10845"}}
+{"timestamp":1714080111.5979805,"name":"offline","context":{"idset":"10846"}}
+{"timestamp":1714080111.6079359,"name":"offline","context":{"idset":"10847"}}
+{"timestamp":1714080111.6185629,"name":"offline","context":{"idset":"10848"}}
+{"timestamp":1714080111.6201713,"name":"offline","context":{"idset":"10849"}}
+{"timestamp":1714080111.6217024,"name":"offline","context":{"idset":"10850"}}
+{"timestamp":1714080111.6232402,"name":"offline","context":{"idset":"10851"}}
+{"timestamp":1714080111.624788,"name":"offline","context":{"idset":"10852"}}
+{"timestamp":1714080111.6264141,"name":"offline","context":{"idset":"10853"}}
+{"timestamp":1714080111.6279852,"name":"offline","context":{"idset":"10854"}}
+{"timestamp":1714080111.6296282,"name":"offline","context":{"idset":"10855"}}
+{"timestamp":1714080111.6311235,"name":"offline","context":{"idset":"10856"}}
+{"timestamp":1714080111.632587,"name":"offline","context":{"idset":"10857"}}
+{"timestamp":1714080111.6340504,"name":"offline","context":{"idset":"10858"}}
+{"timestamp":1714080111.6355171,"name":"offline","context":{"idset":"10859"}}
+{"timestamp":1714080111.6370378,"name":"offline","context":{"idset":"10860"}}
+{"timestamp":1714080111.63849,"name":"offline","context":{"idset":"10861"}}
+{"timestamp":1714080111.6400208,"name":"offline","context":{"idset":"10862"}}
+{"timestamp":1714080111.6415446,"name":"offline","context":{"idset":"10863"}}
+{"timestamp":1714080111.6430144,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1714080111.6444838,"name":"offline","context":{"idset":"10865"}}
+{"timestamp":1714080111.6458778,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1714080111.6472392,"name":"offline","context":{"idset":"10867"}}
+{"timestamp":1714080111.6486278,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1714080111.6593099,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1714080111.6606398,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1714080111.6619942,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1714080111.663388,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1714080111.6644855,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1714080111.6655545,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1714080111.6669583,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1714080111.6684186,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1714080111.6698329,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1714080111.6813903,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1714080111.6829002,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1714080111.6844096,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1714080111.6859231,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1714080111.6876624,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1714080111.6894281,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1714080111.691323,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1714080111.6928608,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1714080111.7157407,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1714080111.7276003,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1714080111.7291389,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1714080111.7306836,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1714080111.7322192,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1714080111.733773,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1714080111.7355034,"name":"offline","context":{"idset":"10894"}}
+{"timestamp":1714080111.7371166,"name":"offline","context":{"idset":"10895"}}
+{"timestamp":1714080111.7388878,"name":"offline","context":{"idset":"10896"}}
+{"timestamp":1714080111.7403979,"name":"offline","context":{"idset":"10897"}}
+{"timestamp":1714080111.7419441,"name":"offline","context":{"idset":"10898"}}
+{"timestamp":1714080111.743453,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1714080111.7449894,"name":"offline","context":{"idset":"10900"}}
+{"timestamp":1714080111.7466245,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1714080111.7481515,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1714080111.7501917,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1714080111.751817,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1714080111.7533224,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1714080111.7548246,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1714080111.7563083,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1714080111.757834,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1714080111.7595501,"name":"offline","context":{"idset":"10909"}}
+{"timestamp":1714080111.7617817,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1714080111.7633033,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1714080111.7648156,"name":"offline","context":{"idset":"10912"}}
+{"timestamp":1714080111.7663093,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1714080111.7678163,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1714080111.7693181,"name":"offline","context":{"idset":"10915"}}
+{"timestamp":1714080111.7822535,"name":"offline","context":{"idset":"10916"}}
+{"timestamp":1714080111.7837539,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1714080111.785459,"name":"offline","context":{"idset":"10918"}}
+{"timestamp":1714080111.7874348,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1714080111.7889555,"name":"offline","context":{"idset":"10920"}}
+{"timestamp":1714080111.7904789,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1714080111.7920041,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1714080111.7938385,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1714080111.7956984,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1714080111.797235,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1714080111.8099771,"name":"offline","context":{"idset":"10926"}}
+{"timestamp":1714080111.8118997,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1714080111.8137684,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1714080111.815274,"name":"offline","context":{"idset":"10929"}}
+{"timestamp":1714080111.8167975,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1714080111.818315,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1714080111.81985,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1714080111.8214729,"name":"offline","context":{"idset":"10933"}}
+{"timestamp":1714080111.8447382,"name":"offline","context":{"idset":"10934"}}
+{"timestamp":1714080111.8462875,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1714080111.8582282,"name":"offline","context":{"idset":"10936"}}
+{"timestamp":1714080111.8597267,"name":"offline","context":{"idset":"10937"}}
+{"timestamp":1714080111.8612266,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1714080111.862767,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1714080111.8643184,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1714080111.8659174,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1714080111.8676727,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1714080111.8695166,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1714080111.8710587,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1714080111.8725615,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1714080111.8740981,"name":"offline","context":{"idset":"10946"}}
+{"timestamp":1714080111.8756549,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1714080111.8778372,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1714080111.8794217,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1714080111.8809555,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1714080111.8824804,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1714080111.8839948,"name":"offline","context":{"idset":"10952"}}
+{"timestamp":1714080111.8855076,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1714080111.8879104,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1714080111.8894224,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1714080111.8909688,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1714080111.8924685,"name":"offline","context":{"idset":"10957"}}
+{"timestamp":1714080111.8940372,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1714080111.9071264,"name":"offline","context":{"idset":"10959"}}
+{"timestamp":1714080111.9087315,"name":"offline","context":{"idset":"10960"}}
+{"timestamp":1714080111.9104919,"name":"offline","context":{"idset":"10961"}}
+{"timestamp":1714080111.9119897,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1714080111.9135063,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1714080111.9150257,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1714080111.9268699,"name":"offline","context":{"idset":"10965"}}
+{"timestamp":1714080111.9283354,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1714080111.9297867,"name":"offline","context":{"idset":"10967"}}
+{"timestamp":1714080111.9312611,"name":"offline","context":{"idset":"10968"}}
+{"timestamp":1714080111.9327369,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1714080111.9341948,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1714080111.9544132,"name":"offline","context":{"idset":"10971"}}
+{"timestamp":1714080111.9611249,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1714080111.9619951,"name":"offline","context":{"idset":"10973"}}
+{"timestamp":1714080111.9627759,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1714080111.9636412,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1714080111.964503,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1714080111.9652989,"name":"offline","context":{"idset":"10977"}}
+{"timestamp":1714080111.9661019,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1714080111.972548,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1714080111.9733431,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1714080111.974128,"name":"offline","context":{"idset":"10981"}}
+{"timestamp":1714080111.9749129,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1714080111.9756863,"name":"offline","context":{"idset":"10983"}}
+{"timestamp":1714080111.9764779,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1714080111.9772611,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1714080111.9780328,"name":"offline","context":{"idset":"10986"}}
+{"timestamp":1714080111.9787955,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1714080111.9795833,"name":"offline","context":{"idset":"10988"}}
+{"timestamp":1714080111.9805024,"name":"offline","context":{"idset":"10989"}}
+{"timestamp":1714080111.9813073,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1714080111.9820743,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1714080111.9829438,"name":"offline","context":{"idset":"10992"}}
+{"timestamp":1714080111.9838061,"name":"offline","context":{"idset":"10993"}}
+{"timestamp":1714080111.9845948,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1714080111.9853959,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1714080111.9862945,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1714080111.9870782,"name":"offline","context":{"idset":"10997"}}
+{"timestamp":1714080111.9878554,"name":"offline","context":{"idset":"10998"}}
+{"timestamp":1714080111.9886842,"name":"offline","context":{"idset":"10999"}}
+{"timestamp":1714080111.9952688,"name":"offline","context":{"idset":"11000"}}
+{"timestamp":1714080111.9960461,"name":"offline","context":{"idset":"11001"}}
+{"timestamp":1714080111.9970219,"name":"offline","context":{"idset":"11002"}}
+{"timestamp":1714080111.9978573,"name":"offline","context":{"idset":"11003"}}
+{"timestamp":1714080111.9986224,"name":"offline","context":{"idset":"11004"}}
+{"timestamp":1714080111.9993823,"name":"offline","context":{"idset":"11005"}}
+{"timestamp":1714080112.0001514,"name":"offline","context":{"idset":"11006"}}
+{"timestamp":1714080112.0009248,"name":"offline","context":{"idset":"11007"}}
+{"timestamp":1714080112.0074174,"name":"offline","context":{"idset":"11008"}}
+{"timestamp":1714080112.0081964,"name":"offline","context":{"idset":"11009"}}
+{"timestamp":1714080112.0089607,"name":"offline","context":{"idset":"11010"}}
+{"timestamp":1714080112.0097299,"name":"offline","context":{"idset":"11011"}}
+{"timestamp":1714080112.0104818,"name":"offline","context":{"idset":"11012"}}
+{"timestamp":1714080112.0112877,"name":"offline","context":{"idset":"11013"}}
+{"timestamp":1714080112.0120685,"name":"offline","context":{"idset":"11014"}}
+{"timestamp":1714080112.0128546,"name":"offline","context":{"idset":"11015"}}
+{"timestamp":1714080112.0253465,"name":"offline","context":{"idset":"11016"}}
+{"timestamp":1714080112.0261354,"name":"offline","context":{"idset":"11017"}}
+{"timestamp":1714080112.0269258,"name":"offline","context":{"idset":"11018"}}
+{"timestamp":1714080112.0278597,"name":"offline","context":{"idset":"11019"}}
+{"timestamp":1714080112.0287397,"name":"offline","context":{"idset":"11020"}}
+{"timestamp":1714080112.0296688,"name":"offline","context":{"idset":"11021"}}
+{"timestamp":1714080112.0304518,"name":"offline","context":{"idset":"11022"}}
+{"timestamp":1714080112.0312593,"name":"offline","context":{"idset":"11023"}}
+{"timestamp":1714080112.037642,"name":"offline","context":{"idset":"11024"}}
+{"timestamp":1714080112.0384779,"name":"offline","context":{"idset":"11025"}}
+{"timestamp":1714080112.0392418,"name":"offline","context":{"idset":"11026"}}
+{"timestamp":1714080112.0399942,"name":"offline","context":{"idset":"11027"}}
+{"timestamp":1714080112.0407386,"name":"offline","context":{"idset":"11028"}}
+{"timestamp":1714080112.0415699,"name":"offline","context":{"idset":"11029"}}
+{"timestamp":1714080112.0423412,"name":"offline","context":{"idset":"11030"}}
+{"timestamp":1714080112.0430851,"name":"offline","context":{"idset":"11031"}}
+{"timestamp":1714080112.0554006,"name":"offline","context":{"idset":"11032"}}
+{"timestamp":1714080112.0561612,"name":"offline","context":{"idset":"11033"}}
+{"timestamp":1714080112.0569835,"name":"offline","context":{"idset":"11034"}}
+{"timestamp":1714080112.0577912,"name":"offline","context":{"idset":"11035"}}
+{"timestamp":1714080112.0585809,"name":"offline","context":{"idset":"11036"}}
+{"timestamp":1714080112.0593641,"name":"offline","context":{"idset":"11037"}}
+{"timestamp":1714080112.0602212,"name":"offline","context":{"idset":"11038"}}
+{"timestamp":1714080112.0609734,"name":"offline","context":{"idset":"11039"}}
+{"timestamp":1714080112.0617962,"name":"offline","context":{"idset":"11040"}}
+{"timestamp":1714080112.0683372,"name":"offline","context":{"idset":"11041"}}
+{"timestamp":1714080112.0690908,"name":"offline","context":{"idset":"11042"}}
+{"timestamp":1714080112.0699167,"name":"offline","context":{"idset":"11043"}}
+{"timestamp":1714080112.0706556,"name":"offline","context":{"idset":"11044"}}
+{"timestamp":1714080112.0714686,"name":"offline","context":{"idset":"11045"}}
+{"timestamp":1714080112.0722303,"name":"offline","context":{"idset":"11046"}}
+{"timestamp":1714080112.0729764,"name":"offline","context":{"idset":"11047"}}
+{"timestamp":1714080112.0737159,"name":"offline","context":{"idset":"11048"}}
+{"timestamp":1714080112.0858276,"name":"offline","context":{"idset":"11049"}}
+{"timestamp":1714080112.0866487,"name":"offline","context":{"idset":"11050"}}
+{"timestamp":1714080112.0873964,"name":"offline","context":{"idset":"11051"}}
+{"timestamp":1714080112.0881371,"name":"offline","context":{"idset":"11052"}}
+{"timestamp":1714080112.0890145,"name":"offline","context":{"idset":"11053"}}
+{"timestamp":1714080112.0897596,"name":"offline","context":{"idset":"11054"}}
+{"timestamp":1714080112.0905664,"name":"offline","context":{"idset":"11055"}}
+{"timestamp":1714080112.0913291,"name":"offline","context":{"idset":"11056"}}
+{"timestamp":1714080112.0979624,"name":"offline","context":{"idset":"11057"}}
+{"timestamp":1714080112.0987158,"name":"offline","context":{"idset":"11058"}}
+{"timestamp":1714080112.0997589,"name":"offline","context":{"idset":"11059"}}
+{"timestamp":1714080112.1007807,"name":"offline","context":{"idset":"11060"}}
+{"timestamp":1714080112.101912,"name":"offline","context":{"idset":"11061"}}
+{"timestamp":1714080112.1030304,"name":"offline","context":{"idset":"11062"}}
+{"timestamp":1714080112.1041687,"name":"offline","context":{"idset":"11063"}}
+{"timestamp":1714080112.1052558,"name":"offline","context":{"idset":"11064"}}
+{"timestamp":1714080112.1063154,"name":"offline","context":{"idset":"11065"}}
+{"timestamp":1714080112.1073594,"name":"offline","context":{"idset":"11066"}}
+{"timestamp":1714080112.1205721,"name":"offline","context":{"idset":"11067"}}
+{"timestamp":1714080112.1214807,"name":"offline","context":{"idset":"11068"}}
+{"timestamp":1714080112.1222556,"name":"offline","context":{"idset":"11069"}}
+{"timestamp":1714080112.1230443,"name":"offline","context":{"idset":"11070"}}
+{"timestamp":1714080112.1238191,"name":"offline","context":{"idset":"11071"}}
+{"timestamp":1714080112.1247294,"name":"offline","context":{"idset":"11072"}}
+{"timestamp":1714080112.1256037,"name":"offline","context":{"idset":"11073"}}
+{"timestamp":1714080112.126379,"name":"offline","context":{"idset":"11074"}}
+{"timestamp":1714080112.141454,"name":"offline","context":{"idset":"11075"}}
+{"timestamp":1714080112.142499,"name":"offline","context":{"idset":"11076"}}
+{"timestamp":1714080112.1434801,"name":"offline","context":{"idset":"11077"}}
+{"timestamp":1714080112.1443632,"name":"offline","context":{"idset":"11078"}}
+{"timestamp":1714080112.1452861,"name":"offline","context":{"idset":"11079"}}
+{"timestamp":1714080112.1461399,"name":"offline","context":{"idset":"11080"}}
+{"timestamp":1714080112.1470296,"name":"offline","context":{"idset":"11081"}}
+{"timestamp":1714080112.1478457,"name":"offline","context":{"idset":"11082"}}
+{"timestamp":1714080112.1568379,"name":"offline","context":{"idset":"11083"}}
+{"timestamp":1714080112.1578958,"name":"offline","context":{"idset":"11084"}}
+{"timestamp":1714080112.1587081,"name":"offline","context":{"idset":"11085"}}
+{"timestamp":1714080112.159858,"name":"offline","context":{"idset":"11086"}}
+{"timestamp":1714080112.1609814,"name":"offline","context":{"idset":"11087"}}
+{"timestamp":1714080112.1619155,"name":"offline","context":{"idset":"11088"}}
+{"timestamp":1714080112.1630521,"name":"offline","context":{"idset":"11089"}}
+{"timestamp":1714080112.1639624,"name":"offline","context":{"idset":"11090"}}
+{"timestamp":1714080112.1776576,"name":"offline","context":{"idset":"11091"}}
+{"timestamp":1714080112.1783984,"name":"offline","context":{"idset":"11092"}}
+{"timestamp":1714080112.1791267,"name":"offline","context":{"idset":"11093"}}
+{"timestamp":1714080112.1799254,"name":"offline","context":{"idset":"11094"}}
+{"timestamp":1714080112.1807849,"name":"offline","context":{"idset":"11095"}}
+{"timestamp":1714080112.1815267,"name":"offline","context":{"idset":"11096"}}
+{"timestamp":1714080112.1823316,"name":"offline","context":{"idset":"11097"}}
+{"timestamp":1714080112.1831257,"name":"offline","context":{"idset":"11098"}}
+{"timestamp":1714080112.1895607,"name":"offline","context":{"idset":"11099"}}
+{"timestamp":1714080112.1903553,"name":"offline","context":{"idset":"11100"}}
+{"timestamp":1714080112.1910999,"name":"offline","context":{"idset":"11101"}}
+{"timestamp":1714080112.1918576,"name":"offline","context":{"idset":"11102"}}
+{"timestamp":1714080112.1926794,"name":"offline","context":{"idset":"11103"}}
+{"timestamp":1714080112.1934955,"name":"offline","context":{"idset":"11104"}}
+{"timestamp":1714080112.1943085,"name":"offline","context":{"idset":"11105"}}
+{"timestamp":1714080112.1950271,"name":"offline","context":{"idset":"11106"}}
+{"timestamp":1714080112.1957264,"name":"offline","context":{"idset":"11107"}}
+{"timestamp":1714080112.20804,"name":"offline","context":{"idset":"11108"}}
+{"timestamp":1714080112.2087569,"name":"offline","context":{"idset":"11109"}}
+{"timestamp":1714080112.2095535,"name":"offline","context":{"idset":"11110"}}
+{"timestamp":1714080112.2103631,"name":"offline","context":{"idset":"11111"}}
+{"timestamp":1714080112.2110877,"name":"offline","context":{"idset":"11112"}}
+{"timestamp":1714080112.2118464,"name":"offline","context":{"idset":"11113"}}
+{"timestamp":1714080112.2127104,"name":"offline","context":{"idset":"11114"}}
+{"timestamp":1714080112.2134871,"name":"offline","context":{"idset":"11115"}}
+{"timestamp":1714080112.2142117,"name":"offline","context":{"idset":"11116"}}
+{"timestamp":1714080112.2209535,"name":"offline","context":{"idset":"11117"}}
+{"timestamp":1714080112.2216728,"name":"offline","context":{"idset":"11118"}}
+{"timestamp":1714080112.2224789,"name":"offline","context":{"idset":"11119"}}
+{"timestamp":1714080112.2232609,"name":"offline","context":{"idset":"11120"}}
+{"timestamp":1714080112.2239761,"name":"offline","context":{"idset":"11121"}}
+{"timestamp":1714080112.2246864,"name":"offline","context":{"idset":"11122"}}
+{"timestamp":1714080112.2255089,"name":"offline","context":{"idset":"11123"}}
+{"timestamp":1714080112.2263019,"name":"offline","context":{"idset":"11124"}}
+{"timestamp":1714080112.2271545,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1714080112.2394481,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1714080112.2402368,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1714080112.2409711,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1714080112.2416756,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1714080112.2424643,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1714080112.2435319,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1714080112.2446258,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1714080112.2456975,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1714080112.2552772,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1714080112.2617373,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1714080112.2624974,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1714080112.2633815,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1714080112.2640853,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1714080112.2647824,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1714080112.2654889,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1714080112.2662497,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1714080112.2671301,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1714080112.2679284,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1714080112.2686436,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1714080112.2695022,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1714080112.2702851,"name":"offline","context":{"idset":"11146"}}
+{"timestamp":1714080112.2710669,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1714080112.2717574,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1714080112.2725346,"name":"offline","context":{"idset":"11149"}}
+{"timestamp":1714080112.2733169,"name":"offline","context":{"idset":"11150"}}
+{"timestamp":1714080112.2740119,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1714080112.2747011,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1714080112.2753882,"name":"offline","context":{"idset":"11153"}}
+{"timestamp":1714080112.2760849,"name":"offline","context":{"idset":"11154"}}
+{"timestamp":1714080112.2768695,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1714080112.2775557,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1714080112.2783287,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1714080112.2847629,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1714080112.2859826,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1714080112.2872057,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1714080112.2883019,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1714080112.2889895,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1714080112.2896609,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1714080112.2905154,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1714080112.2912698,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1714080112.2977588,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1714080112.298542,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1714080112.2993178,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1714080112.3000593,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1714080112.300734,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1714080112.3014131,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1714080112.3021154,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1714080112.3030066,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1714080112.3037746,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1714080112.3045461,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1714080112.3052616,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1714080112.3060563,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1714080112.3067818,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1714080112.307534,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1714080112.313977,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1714080112.3147023,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1714080112.3154249,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1714080112.3161077,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1714080112.3167696,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1714080112.317445,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1714080112.3181989,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1714080112.3189433,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1714080112.3196201,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1714080112.3321457,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1714080112.3328435,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1714080112.3335323,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1714080112.3342004,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1714080112.3349361,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1714080112.3356912,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1714080112.3364272,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1714080112.3371155,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1714080112.3378673,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1714080112.3386004,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1714080112.3392789,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1714080112.3401084,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1714080112.3408284,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1714080112.3415465,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1714080112.342221,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1714080112.3429596,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1714080112.3436425,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1714080112.3443778,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1714080112.3451028,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1714080112.3458352,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1714080112.3464983,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1714080112.3474348,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1714080112.3482504,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1714080112.3489497,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1714080112.3496544,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1714080112.3503404,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1714080112.3510931,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1714080112.351773,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1714080112.3524268,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1714080112.3531868,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1714080112.3539515,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1714080112.3549662,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1714080112.355649,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1714080112.3564928,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1714080112.3572683,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1714080112.3579454,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1714080112.3588691,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1714080112.3674486,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1714080112.3683624,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1714080112.3692021,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1714080112.369936,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1714080112.3709683,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1714080112.3719375,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1714080112.3725846,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1714080112.3733134,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1714080112.3741946,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1714080112.3750248,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1714080112.3756893,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1714080112.37675,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1714080112.3774292,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1714080112.3781652,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1714080112.3902948,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1714080112.3909535,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1714080112.3915908,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1714080112.392225,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1714080112.3928668,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1714080112.3935349,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1714080112.3942051,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1714080112.3949332,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1714080112.3955681,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1714080112.3963118,"name":"offline","context":{"idset":"11517"}}
+{"timestamp":1714080112.3969646,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1714080112.3975976,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1714080112.3982334,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1714080112.398874,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1714080112.399575,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1714080112.400373,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1714080112.4010224,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1714080112.4017239,"name":"offline","context":{"idset":"11525"}}
+{"timestamp":1714080112.4023724,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1714080112.4029982,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1714080112.4036155,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1714080112.4164569,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1714080112.417706,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1714080112.4189844,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1714080112.4202399,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1714080112.421494,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1714080112.422739,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1714080112.4239583,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1714080112.4251971,"name":"offline","context":{"idset":"11536"}}
+{"timestamp":1714080112.426441,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1714080112.4276824,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1714080112.4289434,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1714080112.4411294,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1714080112.4423954,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1714080112.4436588,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1714080112.444917,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1714080112.4460006,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1714080112.4467075,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1714080112.4473381,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1714080112.4479575,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1714080112.4486349,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1714080112.4495606,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1714080112.4502254,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1714080112.4509261,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1714080112.4515502,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1714080112.4522822,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1714080112.4529045,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1714080112.4535272,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1714080112.4541359,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1714080112.4550388,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1714080112.455725,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1714080112.4563498,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1714080112.4570403,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1714080112.4576552,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1714080112.4582632,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1714080112.4588857,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1714080112.4594917,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1714080112.4602034,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1714080112.4608274,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1714080112.4614761,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1714080112.4621329,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1714080112.4627357,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1714080112.4633405,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1714080112.4639421,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1714080112.46456,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1714080112.4651694,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1714080112.4657719,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1714080112.4663837,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1714080112.4670076,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1714080112.4677813,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1714080112.4684095,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1714080112.4693279,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1714080112.4771314,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1714080112.4777701,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1714080112.4784076,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1714080112.4790428,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1714080112.4796379,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1714080112.4802377,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1714080112.4808512,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1714080112.4814491,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1714080112.4820738,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1714080112.4827745,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1714080112.496433,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1714080112.4970577,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1714080112.4976604,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1714080112.4982569,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1714080112.4989612,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1714080112.4996707,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1714080112.5003362,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1714080112.5010509,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1714080112.5017288,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1714080112.5023656,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1714080112.50302,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1714080112.5095556,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1714080112.5176797,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1714080112.5185637,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1714080112.5193591,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1714080112.5203547,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1714080112.521347,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1714080112.5221856,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1714080112.5227916,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1714080112.5234358,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1714080112.5241718,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1714080112.5249801,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1714080112.5256019,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1714080112.5264378,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1714080112.5270512,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1714080112.5280645,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1714080112.5287232,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1714080112.5293877,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1714080112.5302227,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1714080112.5310602,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1714080112.5317292,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1714080112.5325456,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1714080112.5334299,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1714080112.5342271,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1714080112.5349081,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1714080112.5356338,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1714080112.5367475,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1714080112.5379391,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1714080112.5390756,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1714080112.5402021,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1714080112.5516939,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1714080112.5522945,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1714080112.552937,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1714080112.5537119,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1714080112.5544372,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1714080112.5551944,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1714080112.5557983,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1714080112.5563965,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1714080112.5571153,"name":"offline","context":{"idset":"11765"}}
+{"timestamp":1714080112.5673225,"name":"offline","context":{"idset":"11766"}}
+{"timestamp":1714080112.5684412,"name":"offline","context":{"idset":"11767"}}
+{"timestamp":1714080112.5695689,"name":"offline","context":{"idset":"11768"}}
+{"timestamp":1714080112.5706127,"name":"offline","context":{"idset":"11769"}}
+{"timestamp":1714080112.5724702,"name":"offline","context":{"idset":"11770"}}
+{"timestamp":1714080112.5735605,"name":"offline","context":{"idset":"11771"}}
+{"timestamp":1714080112.5747263,"name":"offline","context":{"idset":"11772"}}
+{"timestamp":1714080112.5758302,"name":"offline","context":{"idset":"11773"}}
+{"timestamp":1714080112.6014733,"name":"offline","context":{"idset":"11774"}}
+{"timestamp":1714080112.6026828,"name":"offline","context":{"idset":"11775"}}
+{"timestamp":1714080112.6037588,"name":"offline","context":{"idset":"11776"}}
+{"timestamp":1714080112.6049194,"name":"offline","context":{"idset":"11777"}}
+{"timestamp":1714080112.6060603,"name":"offline","context":{"idset":"11778"}}
+{"timestamp":1714080112.6072054,"name":"offline","context":{"idset":"11779"}}
+{"timestamp":1714080112.6083183,"name":"offline","context":{"idset":"11780"}}
+{"timestamp":1714080112.6094635,"name":"offline","context":{"idset":"11781"}}
+{"timestamp":1714080112.6105735,"name":"offline","context":{"idset":"11782"}}
+{"timestamp":1714080112.622483,"name":"offline","context":{"idset":"11783"}}
+{"timestamp":1714080112.6236012,"name":"offline","context":{"idset":"11784"}}
+{"timestamp":1714080112.6246653,"name":"offline","context":{"idset":"11785"}}
+{"timestamp":1714080112.6258519,"name":"offline","context":{"idset":"11786"}}
+{"timestamp":1714080112.6269219,"name":"offline","context":{"idset":"11787"}}
+{"timestamp":1714080112.6280704,"name":"offline","context":{"idset":"11788"}}
+{"timestamp":1714080112.6292057,"name":"offline","context":{"idset":"11791"}}
+{"timestamp":1714080112.6304209,"name":"offline","context":{"idset":"11792"}}
+{"timestamp":1714080112.6315114,"name":"offline","context":{"idset":"11793"}}
+{"timestamp":1714080112.6568577,"name":"offline","context":{"idset":"11794"}}
+{"timestamp":1714080112.65802,"name":"offline","context":{"idset":"11795"}}
+{"timestamp":1714080112.6590979,"name":"offline","context":{"idset":"11796"}}
+{"timestamp":1714080112.6601734,"name":"offline","context":{"idset":"11797"}}
+{"timestamp":1714080112.6613157,"name":"offline","context":{"idset":"11798"}}
+{"timestamp":1714080112.6624403,"name":"offline","context":{"idset":"11799"}}
+{"timestamp":1714080112.663502,"name":"offline","context":{"idset":"11800"}}
+{"timestamp":1714080112.6645958,"name":"offline","context":{"idset":"11801"}}
+{"timestamp":1714080112.6656408,"name":"offline","context":{"idset":"11802"}}
+{"timestamp":1714080112.67854,"name":"offline","context":{"idset":"11803"}}
+{"timestamp":1714080112.6798561,"name":"offline","context":{"idset":"11804"}}
+{"timestamp":1714080112.6810184,"name":"offline","context":{"idset":"11805"}}
+{"timestamp":1714080112.6821029,"name":"offline","context":{"idset":"11806"}}
+{"timestamp":1714080112.6831763,"name":"offline","context":{"idset":"11807"}}
+{"timestamp":1714080112.6842964,"name":"offline","context":{"idset":"11808"}}
+{"timestamp":1714080112.6853793,"name":"offline","context":{"idset":"11809"}}
+{"timestamp":1714080112.6865168,"name":"offline","context":{"idset":"11810"}}
+{"timestamp":1714080112.6875873,"name":"offline","context":{"idset":"11811"}}
+{"timestamp":1714080112.7118862,"name":"offline","context":{"idset":"11812"}}
+{"timestamp":1714080112.7130268,"name":"offline","context":{"idset":"11813"}}
+{"timestamp":1714080112.7141299,"name":"offline","context":{"idset":"11814"}}
+{"timestamp":1714080112.7162488,"name":"offline","context":{"idset":"11815"}}
+{"timestamp":1714080112.7173631,"name":"offline","context":{"idset":"11816"}}
+{"timestamp":1714080112.7184467,"name":"offline","context":{"idset":"11817"}}
+{"timestamp":1714080112.7195098,"name":"offline","context":{"idset":"11818"}}
+{"timestamp":1714080112.7205775,"name":"offline","context":{"idset":"11819"}}
+{"timestamp":1714080112.7216833,"name":"offline","context":{"idset":"11820"}}
+{"timestamp":1714080112.7340646,"name":"offline","context":{"idset":"11821"}}
+{"timestamp":1714080112.7352171,"name":"offline","context":{"idset":"11822"}}
+{"timestamp":1714080112.736937,"name":"offline","context":{"idset":"11823"}}
+{"timestamp":1714080112.7380502,"name":"offline","context":{"idset":"11824"}}
+{"timestamp":1714080112.7391224,"name":"offline","context":{"idset":"11827"}}
+{"timestamp":1714080112.7401907,"name":"offline","context":{"idset":"11828"}}
+{"timestamp":1714080112.7412605,"name":"offline","context":{"idset":"11829"}}
+{"timestamp":1714080112.7429457,"name":"offline","context":{"idset":"11830"}}
+{"timestamp":1714080112.7442806,"name":"offline","context":{"idset":"11831"}}
+{"timestamp":1714080112.7675302,"name":"offline","context":{"idset":"11832"}}
+{"timestamp":1714080112.7690525,"name":"offline","context":{"idset":"11833"}}
+{"timestamp":1714080112.7809918,"name":"offline","context":{"idset":"11834"}}
+{"timestamp":1714080112.7928329,"name":"offline","context":{"idset":"11835"}}
+{"timestamp":1714080112.793874,"name":"offline","context":{"idset":"11836"}}
+{"timestamp":1714080112.7949519,"name":"offline","context":{"idset":"11837"}}
+{"timestamp":1714080112.7960634,"name":"offline","context":{"idset":"11838"}}
+{"timestamp":1714080112.7971044,"name":"offline","context":{"idset":"11839"}}
+{"timestamp":1714080112.7981367,"name":"offline","context":{"idset":"11840"}}
+{"timestamp":1714080112.7991712,"name":"offline","context":{"idset":"11841"}}
+{"timestamp":1714080112.8001857,"name":"offline","context":{"idset":"11842"}}
+{"timestamp":1714080112.8011796,"name":"offline","context":{"idset":"11843"}}
+{"timestamp":1714080112.8025656,"name":"offline","context":{"idset":"11844"}}
+{"timestamp":1714080112.8035946,"name":"offline","context":{"idset":"11845"}}
+{"timestamp":1714080112.8046756,"name":"offline","context":{"idset":"11846"}}
+{"timestamp":1714080112.8057861,"name":"offline","context":{"idset":"11847"}}
+{"timestamp":1714080112.8068824,"name":"offline","context":{"idset":"11848"}}
+{"timestamp":1714080112.8079247,"name":"offline","context":{"idset":"11849"}}
+{"timestamp":1714080112.8090045,"name":"offline","context":{"idset":"11850"}}
+{"timestamp":1714080112.8100655,"name":"offline","context":{"idset":"11851"}}
+{"timestamp":1714080112.8111241,"name":"offline","context":{"idset":"11852"}}
+{"timestamp":1714080112.8121884,"name":"offline","context":{"idset":"11853"}}
+{"timestamp":1714080112.8132002,"name":"offline","context":{"idset":"11854"}}
+{"timestamp":1714080112.8141971,"name":"offline","context":{"idset":"11855"}}
+{"timestamp":1714080112.8152459,"name":"offline","context":{"idset":"11856"}}
+{"timestamp":1714080112.8164628,"name":"offline","context":{"idset":"11857"}}
+{"timestamp":1714080112.8309321,"name":"offline","context":{"idset":"11858"}}
+{"timestamp":1714080112.8320155,"name":"offline","context":{"idset":"11859"}}
+{"timestamp":1714080112.8330469,"name":"offline","context":{"idset":"11860"}}
+{"timestamp":1714080112.8345211,"name":"offline","context":{"idset":"11861"}}
+{"timestamp":1714080112.8355088,"name":"offline","context":{"idset":"11862"}}
+{"timestamp":1714080112.8365111,"name":"offline","context":{"idset":"11863"}}
+{"timestamp":1714080112.8375823,"name":"offline","context":{"idset":"11864"}}
+{"timestamp":1714080112.8387537,"name":"offline","context":{"idset":"11865"}}
+{"timestamp":1714080112.8397906,"name":"offline","context":{"idset":"11866"}}
+{"timestamp":1714080112.8409863,"name":"offline","context":{"idset":"11867"}}
+{"timestamp":1714080112.8637469,"name":"offline","context":{"idset":"11868"}}
+{"timestamp":1714080112.864877,"name":"offline","context":{"idset":"11869"}}
+{"timestamp":1714080112.8773286,"name":"offline","context":{"idset":"11870"}}
+{"timestamp":1714080112.8898115,"name":"offline","context":{"idset":"11871"}}
+{"timestamp":1714080112.8908486,"name":"offline","context":{"idset":"11872"}}
+{"timestamp":1714080112.8919942,"name":"offline","context":{"idset":"11873"}}
+{"timestamp":1714080112.8930097,"name":"offline","context":{"idset":"11874"}}
+{"timestamp":1714080112.8943288,"name":"offline","context":{"idset":"11875"}}
+{"timestamp":1714080112.8953073,"name":"offline","context":{"idset":"11876"}}
+{"timestamp":1714080112.8963482,"name":"offline","context":{"idset":"11877"}}
+{"timestamp":1714080112.8973515,"name":"offline","context":{"idset":"11878"}}
+{"timestamp":1714080112.8983922,"name":"offline","context":{"idset":"11879"}}
+{"timestamp":1714080112.8993828,"name":"offline","context":{"idset":"11880"}}
+{"timestamp":1714080112.9004309,"name":"offline","context":{"idset":"11881"}}
+{"timestamp":1714080112.9014111,"name":"offline","context":{"idset":"11882"}}
+{"timestamp":1714080112.9024272,"name":"offline","context":{"idset":"11883"}}
+{"timestamp":1714080112.9034257,"name":"offline","context":{"idset":"11884"}}
+{"timestamp":1714080112.9048381,"name":"offline","context":{"idset":"11885"}}
+{"timestamp":1714080112.9058185,"name":"offline","context":{"idset":"11886"}}
+{"timestamp":1714080112.9069326,"name":"offline","context":{"idset":"11887"}}
+{"timestamp":1714080112.9079204,"name":"offline","context":{"idset":"11888"}}
+{"timestamp":1714080112.9090033,"name":"offline","context":{"idset":"11889"}}
+{"timestamp":1714080112.9100294,"name":"offline","context":{"idset":"11890"}}
+{"timestamp":1714080112.9111059,"name":"offline","context":{"idset":"11891"}}
+{"timestamp":1714080112.9119022,"name":"offline","context":{"idset":"11892"}}
+{"timestamp":1714080112.9126749,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1714080112.9135003,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1714080112.9142883,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1714080112.9150875,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1714080112.9158823,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1714080112.916666,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1714080112.9174991,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1714080112.9182811,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1714080112.9190671,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1714080112.919857,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1714080112.9206786,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1714080112.9325995,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1714080112.933424,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1714080112.9342606,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1714080112.9350674,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1714080112.9358554,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1714080112.9366994,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1714080112.9375169,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1714080112.9382865,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1714080112.9390681,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1714080112.9399056,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1714080112.9406681,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1714080112.9414506,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1714080112.9423487,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1714080112.9431148,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1714080112.9438856,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1714080112.9553952,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1714080112.9562006,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1714080112.9570031,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1714080112.9578366,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1714080112.9586074,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1714080112.9593694,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1714080112.96014,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1714080112.9609187,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1714080112.9616876,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1714080112.962497,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1714080112.9632409,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1714080112.9640405,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1714080112.9648523,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1714080112.9656923,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1714080112.9773986,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1714080112.9890702,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1714080113.0010002,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1714080113.0019293,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1714080113.0027587,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1714080113.0036764,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1714080113.0045273,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1714080113.0053968,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1714080113.0062199,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1714080113.0072758,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1714080113.0080824,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1714080113.0088997,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1714080113.0096273,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1714080113.0103817,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1714080113.011236,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1714080113.0120151,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1714080113.0130715,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1714080113.013958,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1714080113.0148363,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1714080113.0156744,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1714080113.0164816,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1714080113.0172884,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1714080113.0181487,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1714080113.0189495,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1714080113.019788,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1714080113.0205629,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1714080113.021337,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1714080113.0221429,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1714080113.0228934,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1714080113.0237026,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1714080113.0244331,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1714080113.0253279,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1714080113.0263038,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1714080113.0271008,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1714080113.0281014,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1714080113.0293353,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1714080113.0306342,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1714080113.0314004,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1714080113.0327291,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1714080113.0335217,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1714080113.034287,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1714080113.0350394,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1714080113.0357363,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1714080113.0364101,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1714080113.0371587,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1714080113.0379164,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1714080113.0386534,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1714080113.0394149,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1714080113.0401199,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1714080113.0409577,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1714080113.0418668,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1714080113.042717,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1714080113.0435984,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1714080113.0444839,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1714080113.0453517,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1714080113.0466659,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1714080113.0477772,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1714080113.0488253,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1714080113.0496781,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1714080113.0510042,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1714080113.0520315,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1714080113.0530269,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1714080113.0539358,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1714080113.0547552,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1714080113.0555992,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1714080113.0566244,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1714080113.0573847,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1714080113.0581903,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1714080113.0590208,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1714080113.059783,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1714080113.0605319,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1714080113.061306,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1714080113.0621657,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1714080113.0629654,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1714080113.063746,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1714080113.0645826,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1714080113.0652943,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1714080113.066097,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1714080113.0668662,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1714080113.0676346,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1714080113.0683849,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1714080113.0694144,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1714080113.0702522,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1714080113.0710263,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1714080113.0717721,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1714080113.0726178,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1714080113.0735183,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1714080113.0746119,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1714080113.0792108,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1714080113.080081,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1714080113.0809662,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1714080113.0818467,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1714080113.0827665,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1714080113.083642,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1714080113.084434,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1714080113.0853133,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1714080113.0861895,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1714080113.0869713,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1714080113.0877676,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1714080113.0888162,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1714080113.0896027,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1714080113.0903447,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1714080113.0911057,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1714080113.092118,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1714080113.1058478,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1714080113.106693,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1714080113.1077366,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1714080113.1084242,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1714080113.1091728,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1714080113.1104028,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1714080113.1112406,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1714080113.1121511,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1714080113.1131332,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1714080113.1136036,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1714080113.1141078,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1714080113.1145251,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1714080113.1149778,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1714080113.1153476,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1714080113.1157146,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1714080113.1161337,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1714080113.1165271,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1714080113.1169424,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1714080113.124872,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1714080113.1253831,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1714080113.1261928,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1714080113.1267505,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1714080113.127264,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1714080113.1277997,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1714080113.128298,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1714080113.1287839,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1714080113.1293218,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1714080113.1298513,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1714080113.1303484,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1714080113.1307588,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1714080113.1313047,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1714080113.1318233,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1714080113.1323054,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1714080113.1327615,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1714080113.13324,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1714080113.1337471,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1714080113.1343255,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1714080113.1347995,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1714080113.1353374,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1714080113.1358316,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1714080113.1363745,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1714080113.1368744,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1714080113.1373749,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1714080113.1379061,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1714080113.1383512,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1714080113.1387405,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1714080113.1391683,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1714080113.1395993,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1714080113.1400635,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1714080113.140594,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1714080113.1646001,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1714080113.1654005,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1714080113.1823232,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1714080113.2048683,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1714080113.2081602,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1714080113.2087293,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1714080113.2092788,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1714080113.209779,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1714080113.2103283,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1714080113.2108788,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1714080113.2114191,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1714080113.2119617,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1714080113.2124867,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1714080113.2130327,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1714080113.2135704,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1714080113.2141054,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1714080113.2146349,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1714080113.2151647,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1714080113.2156889,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1714080113.2161765,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1714080113.2166548,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1714080113.2295113,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1714080113.2300098,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1714080113.2305655,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1714080113.2335203,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1714080113.234144,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1714080113.2371485,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1714080113.2377186,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1714080113.2383056,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1714080113.2388639,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1714080113.2394283,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1714080113.2399762,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1714080113.2405334,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1714080113.2410936,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1714080113.2416396,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1714080113.242167,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1714080113.2427022,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1714080113.2432582,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1714080113.2438173,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1714080113.2443616,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1714080113.2449172,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1714080113.2454565,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1714080113.2459888,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1714080113.2464938,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1714080113.2470551,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1714080113.2475958,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1714080113.2481403,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1714080113.2486656,"name":"offline","context":{"idset":"808"}}
+{"timestamp":1714080113.249203,"name":"offline","context":{"idset":"827"}}
+{"timestamp":1714080113.2497334,"name":"offline","context":{"idset":"851"}}
+{"timestamp":1714080113.2502666,"name":"offline","context":{"idset":"852"}}
+{"timestamp":1714080113.2507937,"name":"offline","context":{"idset":"877"}}
+{"timestamp":1714080113.2512982,"name":"offline","context":{"idset":"878"}}
+{"timestamp":1714080113.2518399,"name":"offline","context":{"idset":"7669"}}
+{"timestamp":1714080113.2523623,"name":"offline","context":{"idset":"7670"}}
+{"timestamp":1714080113.2528856,"name":"offline","context":{"idset":"7671"}}
+{"timestamp":1714080113.2534094,"name":"offline","context":{"idset":"7672"}}
+{"timestamp":1714080113.2539456,"name":"offline","context":{"idset":"7673"}}
+{"timestamp":1714080113.2544906,"name":"offline","context":{"idset":"7674"}}
+{"timestamp":1714080113.2549942,"name":"offline","context":{"idset":"7675"}}
+{"timestamp":1714080113.2555254,"name":"offline","context":{"idset":"7676"}}
+{"timestamp":1714080113.2560525,"name":"offline","context":{"idset":"7677"}}
+{"timestamp":1714080113.256577,"name":"offline","context":{"idset":"7678"}}
+{"timestamp":1714080113.2570939,"name":"offline","context":{"idset":"7679"}}
+{"timestamp":1714080113.2576022,"name":"offline","context":{"idset":"7680"}}
+{"timestamp":1714080113.2581179,"name":"offline","context":{"idset":"7681"}}
+{"timestamp":1714080113.2586296,"name":"offline","context":{"idset":"7682"}}
+{"timestamp":1714080113.2592134,"name":"offline","context":{"idset":"7683"}}
+{"timestamp":1714080113.2670982,"name":"offline","context":{"idset":"7684"}}
+{"timestamp":1714080113.2675583,"name":"offline","context":{"idset":"7686"}}
+{"timestamp":1714080113.2680008,"name":"offline","context":{"idset":"7687"}}
+{"timestamp":1714080113.2684319,"name":"offline","context":{"idset":"7688"}}
+{"timestamp":1714080113.2688708,"name":"offline","context":{"idset":"7689"}}
+{"timestamp":1714080113.269352,"name":"offline","context":{"idset":"7690"}}
+{"timestamp":1714080113.2698364,"name":"offline","context":{"idset":"7691"}}
+{"timestamp":1714080113.2703156,"name":"offline","context":{"idset":"7692"}}
+{"timestamp":1714080113.2707927,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1714080113.2713156,"name":"offline","context":{"idset":"7694"}}
+{"timestamp":1714080113.27179,"name":"offline","context":{"idset":"7695"}}
+{"timestamp":1714080113.2722862,"name":"offline","context":{"idset":"7696"}}
+{"timestamp":1714080113.2727754,"name":"offline","context":{"idset":"7697"}}
+{"timestamp":1714080113.2733765,"name":"offline","context":{"idset":"7698"}}
+{"timestamp":1714080113.2740722,"name":"offline","context":{"idset":"7699"}}
+{"timestamp":1714080113.2745452,"name":"offline","context":{"idset":"7700"}}
+{"timestamp":1714080113.2751141,"name":"offline","context":{"idset":"7701"}}
+{"timestamp":1714080113.275599,"name":"offline","context":{"idset":"7702"}}
+{"timestamp":1714080113.2850804,"name":"offline","context":{"idset":"7703"}}
+{"timestamp":1714080113.285562,"name":"offline","context":{"idset":"7704"}}
+{"timestamp":1714080113.2955554,"name":"offline","context":{"idset":"7705"}}
+{"timestamp":1714080113.296015,"name":"offline","context":{"idset":"7706"}}
+{"timestamp":1714080113.2964714,"name":"offline","context":{"idset":"7707"}}
+{"timestamp":1714080113.2969334,"name":"offline","context":{"idset":"7708"}}
+{"timestamp":1714080113.2973897,"name":"offline","context":{"idset":"7709"}}
+{"timestamp":1714080113.2978454,"name":"offline","context":{"idset":"7710"}}
+{"timestamp":1714080113.2983067,"name":"offline","context":{"idset":"7711"}}
+{"timestamp":1714080113.2987571,"name":"offline","context":{"idset":"7712"}}
+{"timestamp":1714080113.3023458,"name":"offline","context":{"idset":"7713"}}
+{"timestamp":1714080113.302752,"name":"offline","context":{"idset":"7714"}}
+{"timestamp":1714080113.3031592,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1714080113.3035426,"name":"offline","context":{"idset":"7716"}}
+{"timestamp":1714080113.3039384,"name":"offline","context":{"idset":"7717"}}
+{"timestamp":1714080113.3042996,"name":"offline","context":{"idset":"7718"}}
+{"timestamp":1714080113.3046565,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1714080113.3050616,"name":"offline","context":{"idset":"7720"}}
+{"timestamp":1714080113.3054986,"name":"offline","context":{"idset":"7721"}}
+{"timestamp":1714080113.3059387,"name":"offline","context":{"idset":"7722"}}
+{"timestamp":1714080113.3063772,"name":"offline","context":{"idset":"7723"}}
+{"timestamp":1714080113.3068211,"name":"offline","context":{"idset":"7724"}}
+{"timestamp":1714080113.3072543,"name":"offline","context":{"idset":"7725"}}
+{"timestamp":1714080113.3076861,"name":"offline","context":{"idset":"7726"}}
+{"timestamp":1714080113.3081167,"name":"offline","context":{"idset":"7727"}}
+{"timestamp":1714080113.3085485,"name":"offline","context":{"idset":"7728"}}
+{"timestamp":1714080113.3089619,"name":"offline","context":{"idset":"7729"}}
+{"timestamp":1714080113.3093603,"name":"offline","context":{"idset":"7730"}}
+{"timestamp":1714080113.3097498,"name":"offline","context":{"idset":"7731"}}
+{"timestamp":1714080113.3101656,"name":"offline","context":{"idset":"7732"}}
+{"timestamp":1714080113.3105307,"name":"offline","context":{"idset":"7734"}}
+{"timestamp":1714080113.3108914,"name":"offline","context":{"idset":"7735"}}
+{"timestamp":1714080113.3112497,"name":"offline","context":{"idset":"7736"}}
+{"timestamp":1714080113.3116128,"name":"offline","context":{"idset":"7737"}}
+{"timestamp":1714080113.3119721,"name":"offline","context":{"idset":"7738"}}
+{"timestamp":1714080113.3123357,"name":"offline","context":{"idset":"7739"}}
+{"timestamp":1714080113.3127062,"name":"offline","context":{"idset":"7740"}}
+{"timestamp":1714080113.3130889,"name":"offline","context":{"idset":"7741"}}
+{"timestamp":1714080113.3134758,"name":"offline","context":{"idset":"7742"}}
+{"timestamp":1714080113.3138633,"name":"offline","context":{"idset":"7743"}}
+{"timestamp":1714080113.3142264,"name":"offline","context":{"idset":"7744"}}
+{"timestamp":1714080113.3145814,"name":"offline","context":{"idset":"7745"}}
+{"timestamp":1714080113.3149977,"name":"offline","context":{"idset":"7748"}}
+{"timestamp":1714080113.3154058,"name":"offline","context":{"idset":"7749"}}
+{"timestamp":1714080113.31582,"name":"offline","context":{"idset":"7750"}}
+{"timestamp":1714080113.3162239,"name":"offline","context":{"idset":"7751"}}
+{"timestamp":1714080113.3166254,"name":"offline","context":{"idset":"7752"}}
+{"timestamp":1714080113.3193753,"name":"offline","context":{"idset":"7753"}}
+{"timestamp":1714080113.3197958,"name":"offline","context":{"idset":"7754"}}
+{"timestamp":1714080113.3201809,"name":"offline","context":{"idset":"7755"}}
+{"timestamp":1714080113.3205919,"name":"offline","context":{"idset":"7756"}}
+{"timestamp":1714080113.3210051,"name":"offline","context":{"idset":"7757"}}
+{"timestamp":1714080113.3214056,"name":"offline","context":{"idset":"7758"}}
+{"timestamp":1714080113.3218148,"name":"offline","context":{"idset":"7759"}}
+{"timestamp":1714080113.3222139,"name":"offline","context":{"idset":"7760"}}
+{"timestamp":1714080113.3226342,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1714080113.3231542,"name":"offline","context":{"idset":"7762"}}
+{"timestamp":1714080113.3235388,"name":"offline","context":{"idset":"7763"}}
+{"timestamp":1714080113.3239262,"name":"offline","context":{"idset":"7764"}}
+{"timestamp":1714080113.3243246,"name":"offline","context":{"idset":"7765"}}
+{"timestamp":1714080113.3247168,"name":"offline","context":{"idset":"7766"}}
+{"timestamp":1714080113.3250947,"name":"offline","context":{"idset":"7767"}}
+{"timestamp":1714080113.3254669,"name":"offline","context":{"idset":"7768"}}
+{"timestamp":1714080113.3258495,"name":"offline","context":{"idset":"7769"}}
+{"timestamp":1714080113.3262303,"name":"offline","context":{"idset":"7770"}}
+{"timestamp":1714080113.3266013,"name":"offline","context":{"idset":"7771"}}
+{"timestamp":1714080113.3269999,"name":"offline","context":{"idset":"7772"}}
+{"timestamp":1714080113.3273888,"name":"offline","context":{"idset":"7773"}}
+{"timestamp":1714080113.3277693,"name":"offline","context":{"idset":"7774"}}
+{"timestamp":1714080113.3281574,"name":"offline","context":{"idset":"7775"}}
+{"timestamp":1714080113.3285394,"name":"offline","context":{"idset":"7776"}}
+{"timestamp":1714080113.3289211,"name":"offline","context":{"idset":"7777"}}
+{"timestamp":1714080113.3292983,"name":"offline","context":{"idset":"7778"}}
+{"timestamp":1714080113.3296695,"name":"offline","context":{"idset":"7779"}}
+{"timestamp":1714080113.3300467,"name":"offline","context":{"idset":"7780"}}
+{"timestamp":1714080113.3304222,"name":"offline","context":{"idset":"7781"}}
+{"timestamp":1714080113.3391614,"name":"offline","context":{"idset":"7782"}}
+{"timestamp":1714080113.3395414,"name":"offline","context":{"idset":"7783"}}
+{"timestamp":1714080113.339901,"name":"offline","context":{"idset":"7784"}}
+{"timestamp":1714080113.3402293,"name":"offline","context":{"idset":"7785"}}
+{"timestamp":1714080113.3405602,"name":"offline","context":{"idset":"7786"}}
+{"timestamp":1714080113.3408978,"name":"offline","context":{"idset":"7787"}}
+{"timestamp":1714080113.3412347,"name":"offline","context":{"idset":"7788"}}
+{"timestamp":1714080113.3416059,"name":"offline","context":{"idset":"7789"}}
+{"timestamp":1714080113.3419271,"name":"offline","context":{"idset":"7790"}}
+{"timestamp":1714080113.3423131,"name":"offline","context":{"idset":"7791"}}
+{"timestamp":1714080113.3427026,"name":"offline","context":{"idset":"7792"}}
+{"timestamp":1714080113.3430951,"name":"offline","context":{"idset":"7793"}}
+{"timestamp":1714080113.343482,"name":"offline","context":{"idset":"7794"}}
+{"timestamp":1714080113.3438725,"name":"offline","context":{"idset":"7795"}}
+{"timestamp":1714080113.3441949,"name":"offline","context":{"idset":"7796"}}
+{"timestamp":1714080113.3445559,"name":"offline","context":{"idset":"7797"}}
+{"timestamp":1714080113.3449497,"name":"offline","context":{"idset":"7798"}}
+{"timestamp":1714080113.3453326,"name":"offline","context":{"idset":"7799"}}
+{"timestamp":1714080113.3456919,"name":"offline","context":{"idset":"7800"}}
+{"timestamp":1714080113.3460312,"name":"offline","context":{"idset":"7801"}}
+{"timestamp":1714080113.3550186,"name":"offline","context":{"idset":"7802"}}
+{"timestamp":1714080113.3553851,"name":"offline","context":{"idset":"7803"}}
+{"timestamp":1714080113.3557332,"name":"offline","context":{"idset":"7804"}}
+{"timestamp":1714080113.3561649,"name":"offline","context":{"idset":"7805"}}
+{"timestamp":1714080113.3610828,"name":"offline","context":{"idset":"7806"}}
+{"timestamp":1714080113.3615024,"name":"offline","context":{"idset":"7807"}}
+{"timestamp":1714080113.3618276,"name":"offline","context":{"idset":"7808"}}
+{"timestamp":1714080113.3621612,"name":"offline","context":{"idset":"7809"}}
+{"timestamp":1714080113.3624971,"name":"offline","context":{"idset":"7810"}}
+{"timestamp":1714080113.3628469,"name":"offline","context":{"idset":"7811"}}
+{"timestamp":1714080113.3631527,"name":"offline","context":{"idset":"7812"}}
+{"timestamp":1714080113.3634892,"name":"offline","context":{"idset":"7813"}}
+{"timestamp":1714080113.3638244,"name":"offline","context":{"idset":"7814"}}
+{"timestamp":1714080113.3641334,"name":"offline","context":{"idset":"7815"}}
+{"timestamp":1714080113.3644333,"name":"offline","context":{"idset":"7816"}}
+{"timestamp":1714080113.3647213,"name":"offline","context":{"idset":"7817"}}
+{"timestamp":1714080113.3650038,"name":"offline","context":{"idset":"7818"}}
+{"timestamp":1714080113.3652833,"name":"offline","context":{"idset":"7819"}}
+{"timestamp":1714080113.365556,"name":"offline","context":{"idset":"7820"}}
+{"timestamp":1714080113.3658414,"name":"offline","context":{"idset":"7821"}}
+{"timestamp":1714080113.3661728,"name":"offline","context":{"idset":"7822"}}
+{"timestamp":1714080113.3665133,"name":"offline","context":{"idset":"7823"}}
+{"timestamp":1714080113.3668578,"name":"offline","context":{"idset":"7824"}}
+{"timestamp":1714080113.3671935,"name":"offline","context":{"idset":"7825"}}
+{"timestamp":1714080113.3675272,"name":"offline","context":{"idset":"7826"}}
+{"timestamp":1714080113.3678663,"name":"offline","context":{"idset":"7827"}}
+{"timestamp":1714080113.3682017,"name":"offline","context":{"idset":"7828"}}
+{"timestamp":1714080113.3685427,"name":"offline","context":{"idset":"7829"}}
+{"timestamp":1714080113.3688867,"name":"offline","context":{"idset":"7830"}}
+{"timestamp":1714080113.3692117,"name":"offline","context":{"idset":"7831"}}
+{"timestamp":1714080113.3695371,"name":"offline","context":{"idset":"7832"}}
+{"timestamp":1714080113.3698668,"name":"offline","context":{"idset":"7833"}}
+{"timestamp":1714080113.3701935,"name":"offline","context":{"idset":"7834"}}
+{"timestamp":1714080113.3705175,"name":"offline","context":{"idset":"7835"}}
+{"timestamp":1714080113.3708496,"name":"offline","context":{"idset":"7836"}}
+{"timestamp":1714080113.3711746,"name":"offline","context":{"idset":"7837"}}
+{"timestamp":1714080113.371501,"name":"offline","context":{"idset":"7838"}}
+{"timestamp":1714080113.3718505,"name":"offline","context":{"idset":"7839"}}
+{"timestamp":1714080113.3802938,"name":"offline","context":{"idset":"7840"}}
+{"timestamp":1714080113.3805928,"name":"offline","context":{"idset":"7841"}}
+{"timestamp":1714080113.3808937,"name":"offline","context":{"idset":"7842"}}
+{"timestamp":1714080113.3811688,"name":"offline","context":{"idset":"7843"}}
+{"timestamp":1714080113.3814938,"name":"offline","context":{"idset":"7844"}}
+{"timestamp":1714080113.3818171,"name":"offline","context":{"idset":"7845"}}
+{"timestamp":1714080113.3821576,"name":"offline","context":{"idset":"7846"}}
+{"timestamp":1714080113.382504,"name":"offline","context":{"idset":"7847"}}
+{"timestamp":1714080113.3828251,"name":"offline","context":{"idset":"7848"}}
+{"timestamp":1714080113.3831434,"name":"offline","context":{"idset":"7849"}}
+{"timestamp":1714080113.3834331,"name":"offline","context":{"idset":"7850"}}
+{"timestamp":1714080113.3836963,"name":"offline","context":{"idset":"7851"}}
+{"timestamp":1714080113.3843958,"name":"offline","context":{"idset":"7852"}}
+{"timestamp":1714080113.3847141,"name":"offline","context":{"idset":"7853"}}
+{"timestamp":1714080113.3850257,"name":"offline","context":{"idset":"7854"}}
+{"timestamp":1714080113.3853328,"name":"offline","context":{"idset":"7855"}}
+{"timestamp":1714080113.3856339,"name":"offline","context":{"idset":"7856"}}
+{"timestamp":1714080113.385937,"name":"offline","context":{"idset":"7857"}}
+{"timestamp":1714080113.3862321,"name":"offline","context":{"idset":"7858"}}
+{"timestamp":1714080113.3865242,"name":"offline","context":{"idset":"7859"}}
+{"timestamp":1714080113.3868208,"name":"offline","context":{"idset":"7860"}}
+{"timestamp":1714080113.3870823,"name":"offline","context":{"idset":"7861"}}
+{"timestamp":1714080113.3873596,"name":"offline","context":{"idset":"7862"}}
+{"timestamp":1714080113.3876379,"name":"offline","context":{"idset":"7863"}}
+{"timestamp":1714080113.3879249,"name":"offline","context":{"idset":"7864"}}
+{"timestamp":1714080113.3882005,"name":"offline","context":{"idset":"7865"}}
+{"timestamp":1714080113.3884766,"name":"offline","context":{"idset":"7866"}}
+{"timestamp":1714080113.3887498,"name":"offline","context":{"idset":"7867"}}
+{"timestamp":1714080113.3890297,"name":"offline","context":{"idset":"7868"}}
+{"timestamp":1714080113.3892965,"name":"offline","context":{"idset":"7869"}}
+{"timestamp":1714080113.3895552,"name":"offline","context":{"idset":"7870"}}
+{"timestamp":1714080113.3898423,"name":"offline","context":{"idset":"7871"}}
+{"timestamp":1714080113.3901029,"name":"offline","context":{"idset":"7872"}}
+{"timestamp":1714080113.390377,"name":"offline","context":{"idset":"7873"}}
+{"timestamp":1714080113.3906448,"name":"offline","context":{"idset":"7874"}}
+{"timestamp":1714080113.3909035,"name":"offline","context":{"idset":"7875"}}
+{"timestamp":1714080113.3911893,"name":"offline","context":{"idset":"7876"}}
+{"timestamp":1714080113.3914695,"name":"offline","context":{"idset":"7877"}}
+{"timestamp":1714080113.3917513,"name":"offline","context":{"idset":"7878"}}
+{"timestamp":1714080113.3920357,"name":"offline","context":{"idset":"7879"}}
+{"timestamp":1714080113.3923264,"name":"offline","context":{"idset":"7880"}}
+{"timestamp":1714080113.3926122,"name":"offline","context":{"idset":"7881"}}
+{"timestamp":1714080113.392889,"name":"offline","context":{"idset":"7882"}}
+{"timestamp":1714080113.3931458,"name":"offline","context":{"idset":"7883"}}
+{"timestamp":1714080113.4116948,"name":"offline","context":{"idset":"7884"}}
+{"timestamp":1714080113.4120042,"name":"offline","context":{"idset":"7885"}}
+{"timestamp":1714080113.4122932,"name":"offline","context":{"idset":"7886"}}
+{"timestamp":1714080113.41258,"name":"offline","context":{"idset":"7887"}}
+{"timestamp":1714080113.412868,"name":"offline","context":{"idset":"7888"}}
+{"timestamp":1714080113.4131491,"name":"offline","context":{"idset":"7889"}}
+{"timestamp":1714080113.4134269,"name":"offline","context":{"idset":"7890"}}
+{"timestamp":1714080113.4137049,"name":"offline","context":{"idset":"7891"}}
+{"timestamp":1714080113.4139891,"name":"offline","context":{"idset":"7892"}}
+{"timestamp":1714080113.4142659,"name":"offline","context":{"idset":"7893"}}
+{"timestamp":1714080113.4145393,"name":"offline","context":{"idset":"7894"}}
+{"timestamp":1714080113.4147737,"name":"offline","context":{"idset":"7895"}}
+{"timestamp":1714080113.4150212,"name":"offline","context":{"idset":"7896"}}
+{"timestamp":1714080113.4152894,"name":"offline","context":{"idset":"7897"}}
+{"timestamp":1714080113.4155478,"name":"offline","context":{"idset":"7898"}}
+{"timestamp":1714080113.4158175,"name":"offline","context":{"idset":"7899"}}
+{"timestamp":1714080113.4160795,"name":"offline","context":{"idset":"7900"}}
+{"timestamp":1714080113.4163439,"name":"offline","context":{"idset":"7901"}}
+{"timestamp":1714080113.4166036,"name":"offline","context":{"idset":"7902"}}
+{"timestamp":1714080113.4168694,"name":"offline","context":{"idset":"7903"}}
+{"timestamp":1714080113.4171276,"name":"offline","context":{"idset":"7904"}}
+{"timestamp":1714080113.4173803,"name":"offline","context":{"idset":"7905"}}
+{"timestamp":1714080113.4176314,"name":"offline","context":{"idset":"7906"}}
+{"timestamp":1714080113.4183247,"name":"offline","context":{"idset":"7907"}}
+{"timestamp":1714080113.418582,"name":"offline","context":{"idset":"7908"}}
+{"timestamp":1714080113.4188364,"name":"offline","context":{"idset":"7909"}}
+{"timestamp":1714080113.4190867,"name":"offline","context":{"idset":"7910"}}
+{"timestamp":1714080113.419342,"name":"offline","context":{"idset":"7911"}}
+{"timestamp":1714080113.4195902,"name":"offline","context":{"idset":"7912"}}
+{"timestamp":1714080113.419842,"name":"offline","context":{"idset":"7913"}}
+{"timestamp":1714080113.4200854,"name":"offline","context":{"idset":"7914"}}
+{"timestamp":1714080113.4203298,"name":"offline","context":{"idset":"7915"}}
+{"timestamp":1714080113.4205728,"name":"offline","context":{"idset":"7916"}}
+{"timestamp":1714080113.4208226,"name":"offline","context":{"idset":"7917"}}
+{"timestamp":1714080113.4210634,"name":"offline","context":{"idset":"7918"}}
+{"timestamp":1714080113.4213026,"name":"offline","context":{"idset":"7919"}}
+{"timestamp":1714080113.42154,"name":"offline","context":{"idset":"7920"}}
+{"timestamp":1714080113.4217787,"name":"offline","context":{"idset":"7921"}}
+{"timestamp":1714080113.4222603,"name":"offline","context":{"idset":"7922"}}
+{"timestamp":1714080113.4225099,"name":"offline","context":{"idset":"7923"}}
+{"timestamp":1714080113.4310191,"name":"offline","context":{"idset":"7924"}}
+{"timestamp":1714080113.4312663,"name":"offline","context":{"idset":"7925"}}
+{"timestamp":1714080113.4315047,"name":"offline","context":{"idset":"7926"}}
+{"timestamp":1714080113.4317393,"name":"offline","context":{"idset":"7927"}}
+{"timestamp":1714080113.4319627,"name":"offline","context":{"idset":"7928"}}
+{"timestamp":1714080113.4321876,"name":"offline","context":{"idset":"7929"}}
+{"timestamp":1714080113.4324048,"name":"offline","context":{"idset":"7930"}}
+{"timestamp":1714080113.4326262,"name":"offline","context":{"idset":"7931"}}
+{"timestamp":1714080113.4328361,"name":"offline","context":{"idset":"7932"}}
+{"timestamp":1714080113.433028,"name":"offline","context":{"idset":"7933"}}
+{"timestamp":1714080113.4332151,"name":"offline","context":{"idset":"7934"}}
+{"timestamp":1714080113.4333999,"name":"offline","context":{"idset":"7935"}}
+{"timestamp":1714080113.433579,"name":"offline","context":{"idset":"7936"}}
+{"timestamp":1714080113.4337695,"name":"offline","context":{"idset":"7937"}}
+{"timestamp":1714080113.4339771,"name":"offline","context":{"idset":"7938"}}
+{"timestamp":1714080113.4341891,"name":"offline","context":{"idset":"7939"}}
+{"timestamp":1714080113.4343953,"name":"offline","context":{"idset":"7940"}}
+{"timestamp":1714080113.4345958,"name":"offline","context":{"idset":"7941"}}
+{"timestamp":1714080113.4347885,"name":"offline","context":{"idset":"7942"}}
+{"timestamp":1714080113.4349995,"name":"offline","context":{"idset":"7943"}}
+{"timestamp":1714080113.4351981,"name":"offline","context":{"idset":"7944"}}
+{"timestamp":1714080113.4354203,"name":"offline","context":{"idset":"7945"}}
+{"timestamp":1714080113.4356339,"name":"offline","context":{"idset":"7946"}}
+{"timestamp":1714080113.4358499,"name":"offline","context":{"idset":"7947"}}
+{"timestamp":1714080113.4360731,"name":"offline","context":{"idset":"7948"}}
+{"timestamp":1714080113.4362967,"name":"offline","context":{"idset":"7949"}}
+{"timestamp":1714080113.4365225,"name":"offline","context":{"idset":"7950"}}
+{"timestamp":1714080113.436748,"name":"offline","context":{"idset":"7951"}}
+{"timestamp":1714080113.436969,"name":"offline","context":{"idset":"7952"}}
+{"timestamp":1714080113.4371872,"name":"offline","context":{"idset":"7953"}}
+{"timestamp":1714080113.4466736,"name":"offline","context":{"idset":"7954"}}
+{"timestamp":1714080113.4721684,"name":"offline","context":{"idset":"7955"}}
+{"timestamp":1714080113.4724445,"name":"offline","context":{"idset":"7956"}}
+{"timestamp":1714080113.4726348,"name":"offline","context":{"idset":"7957"}}
+{"timestamp":1714080113.472816,"name":"offline","context":{"idset":"7958"}}
+{"timestamp":1714080113.4729986,"name":"offline","context":{"idset":"7959"}}
+{"timestamp":1714080113.4731774,"name":"offline","context":{"idset":"7960"}}
+{"timestamp":1714080113.4733441,"name":"offline","context":{"idset":"7961"}}
+{"timestamp":1714080113.4735205,"name":"offline","context":{"idset":"7962"}}
+{"timestamp":1714080113.4737041,"name":"offline","context":{"idset":"7963"}}
+{"timestamp":1714080113.4738977,"name":"offline","context":{"idset":"7964"}}
+{"timestamp":1714080113.474087,"name":"offline","context":{"idset":"7965"}}
+{"timestamp":1714080113.4742916,"name":"offline","context":{"idset":"7966"}}
+{"timestamp":1714080113.4744925,"name":"offline","context":{"idset":"7967"}}
+{"timestamp":1714080113.4746952,"name":"offline","context":{"idset":"7968"}}
+{"timestamp":1714080113.4748921,"name":"offline","context":{"idset":"7969"}}
+{"timestamp":1714080113.4750891,"name":"offline","context":{"idset":"7970"}}
+{"timestamp":1714080113.4752829,"name":"offline","context":{"idset":"7971"}}
+{"timestamp":1714080113.4754772,"name":"offline","context":{"idset":"7972"}}
+{"timestamp":1714080113.4756718,"name":"offline","context":{"idset":"7973"}}
+{"timestamp":1714080113.4758692,"name":"offline","context":{"idset":"7974"}}
+{"timestamp":1714080113.4760592,"name":"offline","context":{"idset":"7975"}}
+{"timestamp":1714080113.476248,"name":"offline","context":{"idset":"7976"}}
+{"timestamp":1714080113.4764352,"name":"offline","context":{"idset":"7977"}}
+{"timestamp":1714080113.4766212,"name":"offline","context":{"idset":"7978"}}
+{"timestamp":1714080113.4768133,"name":"offline","context":{"idset":"7980"}}
+{"timestamp":1714080113.476999,"name":"offline","context":{"idset":"7981"}}
+{"timestamp":1714080113.4771831,"name":"offline","context":{"idset":"7982"}}
+{"timestamp":1714080113.4773667,"name":"offline","context":{"idset":"7983"}}
+{"timestamp":1714080113.4950111,"name":"offline","context":{"idset":"7984"}}
+{"timestamp":1714080113.495213,"name":"offline","context":{"idset":"7985"}}
+{"timestamp":1714080113.4953959,"name":"offline","context":{"idset":"7986"}}
+{"timestamp":1714080113.4955764,"name":"offline","context":{"idset":"7987"}}
+{"timestamp":1714080113.4957538,"name":"offline","context":{"idset":"7988"}}
+{"timestamp":1714080113.4963393,"name":"offline","context":{"idset":"7989"}}
+{"timestamp":1714080113.4965086,"name":"offline","context":{"idset":"7990"}}
+{"timestamp":1714080113.4967422,"name":"offline","context":{"idset":"7992"}}
+{"timestamp":1714080113.4969304,"name":"offline","context":{"idset":"7993"}}
+{"timestamp":1714080113.4970932,"name":"offline","context":{"idset":"7994"}}
+{"timestamp":1714080113.49733,"name":"offline","context":{"idset":"7995"}}
+{"timestamp":1714080113.4974813,"name":"offline","context":{"idset":"7996"}}
+{"timestamp":1714080113.4976244,"name":"offline","context":{"idset":"7997"}}
+{"timestamp":1714080113.4977708,"name":"offline","context":{"idset":"7999"}}
+{"timestamp":1714080113.4979415,"name":"offline","context":{"idset":"8000"}}
+{"timestamp":1714080113.4981186,"name":"offline","context":{"idset":"8001"}}
+{"timestamp":1714080113.4982855,"name":"offline","context":{"idset":"8002"}}
+{"timestamp":1714080113.4984593,"name":"offline","context":{"idset":"8003"}}
+{"timestamp":1714080113.4986265,"name":"offline","context":{"idset":"8004"}}
+{"timestamp":1714080113.4987884,"name":"offline","context":{"idset":"8005"}}
+{"timestamp":1714080113.4989564,"name":"offline","context":{"idset":"8006"}}
+{"timestamp":1714080113.4991179,"name":"offline","context":{"idset":"8007"}}
+{"timestamp":1714080113.4992778,"name":"offline","context":{"idset":"8008"}}
+{"timestamp":1714080113.4994366,"name":"offline","context":{"idset":"8009"}}
+{"timestamp":1714080113.499594,"name":"offline","context":{"idset":"8010"}}
+{"timestamp":1714080113.4997511,"name":"offline","context":{"idset":"8011"}}
+{"timestamp":1714080113.4999089,"name":"offline","context":{"idset":"8013"}}
+{"timestamp":1714080113.5000637,"name":"offline","context":{"idset":"8014"}}
+{"timestamp":1714080113.5002174,"name":"offline","context":{"idset":"8015"}}
+{"timestamp":1714080113.500371,"name":"offline","context":{"idset":"8016"}}
+{"timestamp":1714080113.5125086,"name":"offline","context":{"idset":"8017"}}
+{"timestamp":1714080113.51267,"name":"offline","context":{"idset":"8018"}}
+{"timestamp":1714080113.5128248,"name":"offline","context":{"idset":"8019"}}
+{"timestamp":1714080113.512964,"name":"offline","context":{"idset":"8020"}}
+{"timestamp":1714080113.5130939,"name":"offline","context":{"idset":"8021"}}
+{"timestamp":1714080113.5132194,"name":"offline","context":{"idset":"8023"}}
+{"timestamp":1714080113.5133376,"name":"offline","context":{"idset":"8024"}}
+{"timestamp":1714080113.5134609,"name":"offline","context":{"idset":"8025"}}
+{"timestamp":1714080113.5135877,"name":"offline","context":{"idset":"8026"}}
+{"timestamp":1714080113.5137117,"name":"offline","context":{"idset":"8027"}}
+{"timestamp":1714080113.5138466,"name":"offline","context":{"idset":"8028"}}
+{"timestamp":1714080113.5139914,"name":"offline","context":{"idset":"8029"}}
+{"timestamp":1714080113.5141306,"name":"offline","context":{"idset":"8030"}}
+{"timestamp":1714080113.5142674,"name":"offline","context":{"idset":"8031"}}
+{"timestamp":1714080113.5144043,"name":"offline","context":{"idset":"8033"}}
+{"timestamp":1714080113.5145407,"name":"offline","context":{"idset":"8034"}}
+{"timestamp":1714080113.5146756,"name":"offline","context":{"idset":"8035"}}
+{"timestamp":1714080113.5148401,"name":"offline","context":{"idset":"8036"}}
+{"timestamp":1714080113.5149858,"name":"offline","context":{"idset":"8037"}}
+{"timestamp":1714080113.5151143,"name":"offline","context":{"idset":"8038"}}
+{"timestamp":1714080113.5152323,"name":"offline","context":{"idset":"8039"}}
+{"timestamp":1714080113.5153527,"name":"offline","context":{"idset":"8040"}}
+{"timestamp":1714080113.5154865,"name":"offline","context":{"idset":"8042"}}
+{"timestamp":1714080113.5156171,"name":"offline","context":{"idset":"8043"}}
+{"timestamp":1714080113.5157471,"name":"offline","context":{"idset":"8045"}}
+{"timestamp":1714080113.5158792,"name":"offline","context":{"idset":"8049"}}
+{"timestamp":1714080113.5160041,"name":"offline","context":{"idset":"8053"}}
+{"timestamp":1714080113.5161273,"name":"offline","context":{"idset":"8054"}}
+{"timestamp":1714080113.5162494,"name":"offline","context":{"idset":"8055"}}
+{"timestamp":1714080113.5163705,"name":"offline","context":{"idset":"8056"}}
+{"timestamp":1714080113.5164895,"name":"offline","context":{"idset":"8057"}}
+{"timestamp":1714080113.5337105,"name":"offline","context":{"idset":"8058"}}
+{"timestamp":1714080113.5339017,"name":"offline","context":{"idset":"8059"}}
+{"timestamp":1714080113.5340514,"name":"offline","context":{"idset":"8060"}}
+{"timestamp":1714080113.5341954,"name":"offline","context":{"idset":"8061"}}
+{"timestamp":1714080113.5343382,"name":"offline","context":{"idset":"8062"}}
+{"timestamp":1714080113.5344782,"name":"offline","context":{"idset":"8063"}}
+{"timestamp":1714080113.5346158,"name":"offline","context":{"idset":"8064"}}
+{"timestamp":1714080113.5347536,"name":"offline","context":{"idset":"8065"}}
+{"timestamp":1714080113.5348954,"name":"offline","context":{"idset":"8066"}}
+{"timestamp":1714080113.5350308,"name":"offline","context":{"idset":"8067"}}
+{"timestamp":1714080113.5351682,"name":"offline","context":{"idset":"8068"}}
+{"timestamp":1714080113.5353007,"name":"offline","context":{"idset":"8069"}}
+{"timestamp":1714080113.5354335,"name":"offline","context":{"idset":"8070"}}
+{"timestamp":1714080113.5355644,"name":"offline","context":{"idset":"8071"}}
+{"timestamp":1714080113.5356944,"name":"offline","context":{"idset":"8072"}}
+{"timestamp":1714080113.5360508,"name":"offline","context":{"idset":"8073"}}
+{"timestamp":1714080113.5361986,"name":"offline","context":{"idset":"8074"}}
+{"timestamp":1714080113.5363286,"name":"offline","context":{"idset":"8075"}}
+{"timestamp":1714080113.5364554,"name":"offline","context":{"idset":"8076"}}
+{"timestamp":1714080113.5365806,"name":"offline","context":{"idset":"8077"}}
+{"timestamp":1714080113.5367055,"name":"offline","context":{"idset":"8078"}}
+{"timestamp":1714080113.5368333,"name":"offline","context":{"idset":"8079"}}
+{"timestamp":1714080113.5369596,"name":"offline","context":{"idset":"8080"}}
+{"timestamp":1714080113.5370812,"name":"offline","context":{"idset":"8081"}}
+{"timestamp":1714080113.5372021,"name":"offline","context":{"idset":"8082"}}
+{"timestamp":1714080113.537322,"name":"offline","context":{"idset":"8083"}}
+{"timestamp":1714080113.5374403,"name":"offline","context":{"idset":"8084"}}
+{"timestamp":1714080113.5375659,"name":"offline","context":{"idset":"8085"}}
+{"timestamp":1714080113.5376883,"name":"offline","context":{"idset":"8086"}}
+{"timestamp":1714080113.537811,"name":"offline","context":{"idset":"8087"}}
+{"timestamp":1714080113.5379276,"name":"offline","context":{"idset":"8088"}}
+{"timestamp":1714080113.5380414,"name":"offline","context":{"idset":"8089"}}
+{"timestamp":1714080113.5381529,"name":"offline","context":{"idset":"8090"}}
+{"timestamp":1714080113.5474336,"name":"offline","context":{"idset":"8091"}}
+{"timestamp":1714080113.5475469,"name":"offline","context":{"idset":"8092"}}
+{"timestamp":1714080113.5476623,"name":"offline","context":{"idset":"8093"}}
+{"timestamp":1714080113.5477798,"name":"offline","context":{"idset":"8094"}}
+{"timestamp":1714080113.5478969,"name":"offline","context":{"idset":"8095"}}
+{"timestamp":1714080113.5480044,"name":"offline","context":{"idset":"8096"}}
+{"timestamp":1714080113.5481136,"name":"offline","context":{"idset":"8097"}}
+{"timestamp":1714080113.5482218,"name":"offline","context":{"idset":"8098"}}
+{"timestamp":1714080113.5483303,"name":"offline","context":{"idset":"8099"}}
+{"timestamp":1714080113.5484331,"name":"offline","context":{"idset":"8100"}}
+{"timestamp":1714080113.5485344,"name":"offline","context":{"idset":"8117"}}
+{"timestamp":1714080113.5486372,"name":"offline","context":{"idset":"8118"}}
+{"timestamp":1714080113.5487392,"name":"offline","context":{"idset":"8119"}}
+{"timestamp":1714080113.548842,"name":"offline","context":{"idset":"8120"}}
+{"timestamp":1714080113.5489411,"name":"offline","context":{"idset":"8121"}}
+{"timestamp":1714080113.5490394,"name":"offline","context":{"idset":"8122"}}
+{"timestamp":1714080113.5491374,"name":"offline","context":{"idset":"8123"}}
+{"timestamp":1714080113.5492346,"name":"offline","context":{"idset":"8124"}}
+{"timestamp":1714080113.54933,"name":"offline","context":{"idset":"8125"}}
+{"timestamp":1714080113.5494244,"name":"offline","context":{"idset":"8126"}}
+{"timestamp":1714080113.5495186,"name":"offline","context":{"idset":"8127"}}
+{"timestamp":1714080113.5496097,"name":"offline","context":{"idset":"8128"}}
+{"timestamp":1714080113.5497007,"name":"offline","context":{"idset":"8129"}}
+{"timestamp":1714080113.5497901,"name":"offline","context":{"idset":"8130"}}
+{"timestamp":1714080113.549891,"name":"offline","context":{"idset":"8131"}}
+{"timestamp":1714080113.5499783,"name":"offline","context":{"idset":"8132"}}
+{"timestamp":1714080113.5500674,"name":"offline","context":{"idset":"8133"}}
+{"timestamp":1714080113.5501552,"name":"offline","context":{"idset":"8134"}}
+{"timestamp":1714080113.5502403,"name":"offline","context":{"idset":"8135"}}
+{"timestamp":1714080113.5503256,"name":"offline","context":{"idset":"8136"}}
+{"timestamp":1714080113.5504086,"name":"offline","context":{"idset":"8137"}}
+{"timestamp":1714080113.5504906,"name":"offline","context":{"idset":"8138"}}
+{"timestamp":1714080113.5505714,"name":"offline","context":{"idset":"8139"}}
+{"timestamp":1714080113.5506518,"name":"offline","context":{"idset":"8140"}}
+{"timestamp":1714080113.5507295,"name":"offline","context":{"idset":"8141"}}
+{"timestamp":1714080113.5589786,"name":"offline","context":{"idset":"8142"}}
+{"timestamp":1714080113.559056,"name":"offline","context":{"idset":"8143"}}
+{"timestamp":1714080113.559124,"name":"offline","context":{"idset":"8144"}}
+{"timestamp":1714080113.5591879,"name":"offline","context":{"idset":"8145"}}
+{"timestamp":1714080113.5592508,"name":"offline","context":{"idset":"8146"}}
+{"timestamp":1714080113.559314,"name":"offline","context":{"idset":"8147"}}
+{"timestamp":1714080113.559376,"name":"offline","context":{"idset":"8148"}}
+{"timestamp":1714080113.5594373,"name":"offline","context":{"idset":"8149"}}
+{"timestamp":1714080113.559479,"name":"offline","context":{"idset":"8176"}}
+{"timestamp":1714080202.8902364,"name":"resource-init","context":{"restart":true,"drain":{"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"98,100,102,111-112,114,123-124,161,184,187,190,208,303-304,310,317,319,330":{"timestamp":1714079956.0,"reason":"epilog failed for jobid frcnKxdu6xP"},"11377":{"timestamp":1713928672.0,"reason":"nodediag failed amdapu"},"543":{"timestamp":1714074108.0,"reason":"nodediag failed cxi"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"8597-8612":{"timestamp":1714067165.0,"reason":"--reason draining to run on-node diags - wendy"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"796":{"timestamp":1712868054.0,"reason":"broker was unresponsive"},"119":{"timestamp":1714079565.0,"reason":"unreachable"},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"},"795":{"timestamp":1712868053.0,"reason":"broker was unresponsive"},"372":{"timestamp":1713981581.0,"reason":"New Blade Installation --JRG"},"336,491":{"timestamp":1713230524.0,"reason":"Down"},"929-930":{"timestamp":1713552018.0,"reason":""},"773-774":{"timestamp":1713802805.0,"reason":""},"77-84":{"timestamp":1713795613.0,"reason":"New blade installation"},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"11826":{"timestamp":1713979977.0,"reason":""},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"115":{"timestamp":1714079493.0,"reason":"unreachable"},"9445-9460":{"timestamp":1714053592.0,"reason":"--reason down for backplain work -kpn"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"117":{"timestamp":1713981538.0,"reason":"Unreachable"},"360":{"timestamp":1713981579.0,"reason":"New Blade Installation --JRG"},"411":{"timestamp":1713564989.0,"reason":"Unreachable"},"11253-11268":{"timestamp":1713972658.0,"reason":"draining KPN"},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"807":{"timestamp":1713376017.0,"reason":""},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"811-812":{"timestamp":1713557753.0,"reason":"--reason Running hpe diags - JM"},"10310":{"timestamp":1713953621.0,"reason":"nodediag failed bogomips"},"8431":{"timestamp":1714066666.0,"reason":"--reason rebooting - wendy"},"10341":{"timestamp":1713928487.0,"reason":"nodediag failed bogomips"},"779-780":{"timestamp":1713907642.0,"reason":"--reason Troubleshoot BIOS issue"},"11725":{"timestamp":1713892052.0,"reason":"nodediag failed amdapu"},"11285-11300":{"timestamp":1713975164.0,"reason":"--reason rebooting switches - INIT -kpn"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"11658":{"timestamp":1713954013.0,"reason":"epilog failed for jobid frLfYxGth6w"},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"286":{"timestamp":1713981553.0,"reason":"Unreachable"},"808":{"timestamp":1713455939.0,"reason":""},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"110":{"timestamp":1714079561.0,"reason":"unreachable"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"544":{"timestamp":1714069881.0,"reason":"nodediag failed cxi"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"11374":{"timestamp":1713926801.0,"reason":"nodediag failed amdapu"},"206":{"timestamp":1713564985.0,"reason":"Unreachable"},"349-359,361-371,374-396,398-404":{"timestamp":1714060395.0,"reason":"New Blade Installation --JRG"},"8181,8183,8234,8237,8291,8304":{"timestamp":1713810085.0,"reason":"testing -kk"},"11373":{"timestamp":1713928485.0,"reason":"nodediag failed amdapu"},"9349-9364":{"timestamp":1714021089.0,"reason":"--reason back plane fix -KPN"},"545":{"timestamp":1714069878.0,"reason":"nodediag failed cxi"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"878":{"timestamp":1714078484.0,"reason":"nodediag failed amdapu"},"141-148":{"timestamp":1713545597.0,"reason":"ARP Storm Problem testing"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"9075":{"timestamp":1712864130.0,"reason":"broker was unresponsive"},"542":{"timestamp":1714074109.0,"reason":"nodediag failed cxi"},"11370":{"timestamp":1713930154.0,"reason":"nodediag failed amdapu"},"9333-9348":{"timestamp":1714020725.0,"reason":"--reason back plane fix -KPN"},"573-580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"149-156":{"timestamp":1713545634.0,"reason":"ARP Storm Problem testing"},"165":{"timestamp":1713981434.0,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261863.0,"reason":"--reason swapping blades into x1900 - wendy"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"11717-11720,11722-11724,11726,11728,11730,11732":{"timestamp":1713983852.0,"reason":"--force --reason rebooting switches - INIT -kpn"},"172":{"timestamp":1713981544.0,"reason":"Unreachable"},"11324":{"timestamp":1713902339.0,"reason":"nodediag failed amdapu"},"11789":{"timestamp":1713910767.0,"reason":"testing -kk"},"373":{"timestamp":1713981698.0,"reason":"New Blade Installation --JRG"},"541":{"timestamp":1714074106.0,"reason":"nodediag failed cxi"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"787-788":{"timestamp":1713219302.0,"reason":""},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"10233,10248":{"timestamp":1713746450.0,"reason":"broker was unresponsive"},"85-89,91-92":{"timestamp":1714079045.0,"reason":"New Blade installation"},"963-964":{"timestamp":1713196418.0,"reason":"moving blade"},"937-938":{"timestamp":1712866351.0,"reason":""},"10187":{"timestamp":1713980344.0,"reason":""},"925-926":{"timestamp":1713540640.0,"reason":""},"461-476":{"timestamp":1713794797.0,"reason":"New Blade Installation --JRG"},"397":{"timestamp":1713229804.0,"reason":"New Blade Installation --JRG"},"9365-9380":{"timestamp":1714021383.0,"reason":"--reason back plane fix -KPN"},"9003":{"timestamp":1712873964.0,"reason":"nodediag failed dgemm_perf"},"421-460":{"timestamp":1713794410.0,"reason":"New Blade Installation --JRG"},"410":{"timestamp":1713378635.0,"reason":"prolog failed for jobid fq5jtQQSQnT"},"120":{"timestamp":1713981548.0,"reason":"Unreachable"},"803-804":{"timestamp":1713479733.0,"reason":"status"},"789-794":{"timestamp":1711461777.0,"reason":""},"191":{"timestamp":1714078824.0,"reason":"unreachable"},"11729":{"timestamp":1713951714.0,"reason":"nodediag failed amdapu"},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"6517-6644":{"timestamp":1714079999.0,"reason":"--reason draining to run on-node diags - wendy"},"961-962":{"timestamp":1713197674.0,"reason":""},"482-490,492":{"timestamp":1713915346.0,"reason":"New Blade Installation --JRG"},"11369":{"timestamp":1713928726.0,"reason":"nodediag failed amdapu"},"342":{"timestamp":1713981566.0,"reason":"Unreachable"},"923-924":{"timestamp":1714064541.0,"reason":"status"},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"546":{"timestamp":1714069873.0,"reason":"nodediag failed cxi pci"},"549-556":{"timestamp":1713553592.0,"reason":"New Blade Installation"},"8833":{"timestamp":1713991869.0,"reason":"nodediag failed amdapu"},"90":{"timestamp":1713564981.0,"reason":"New Blade installation"},"11721":{"timestamp":1713951748.0,"reason":"nodediag failed amdapu"},"9054":{"timestamp":1712864116.0,"reason":"broker was unresponsive"},"771-772":{"timestamp":1713195979.0,"reason":""},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"877":{"timestamp":1714078512.0,"reason":"nodediag failed amdapu"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"8982":{"timestamp":1713996208.0,"reason":"broker was unresponsive"},"939":{"timestamp":1713963885.0,"reason":"nodediag failed amdapu"},"547":{"timestamp":1713563916.0,"reason":"epilog failed for jobid fqUWmieM3AX"},"420":{"timestamp":1714074112.0,"reason":"epilog failed for jobid frbYvBLD1gT"},"9429-9444":{"timestamp":1714053161.0,"reason":"--reason down for backplain work -kpn"},"814":{"timestamp":1712783332.0,"reason":""},"871-874":{"timestamp":1714057817.0,"reason":""},"829-830":{"timestamp":1712700945.0,"reason":""},"133-140":{"timestamp":1713545617.0,"reason":"ARP Storm Problem testing"},"525-532":{"timestamp":1713290620.0,"reason":""},"548":{"timestamp":1713563915.0,"reason":"epilog failed for jobid fqUWkS8j8hM"},"11731":{"timestamp":1713889305.0,"reason":"broker was unresponsive"},"335":{"timestamp":1713981557.0,"reason":"Unreachable"},"934":{"timestamp":1713967835.0,"reason":"epilog failed for jobid frMYzSv7G6b"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1714080202.9209533,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1714080213.9784188,"name":"online","context":{"idset":"0"}}
+{"timestamp":1714080215.820437,"name":"online","context":{"idset":"8153,8181,8186,8192,8214,8228,8261,8269,8280-8281,8291-8292,8313,8435"}}
+{"timestamp":1714080216.0510285,"name":"online","context":{"idset":"884,8149-8152,8154-8157,8159-8164,8178,8182-8185,8187-8191,8193-8200,8202-8208,8211-8213,8215-8226,8229-8232,8236-8242,8244-8248,8250-8260,8262-8265,8268,8270-8279,8282-8286,8288-8290,8293-8296,8309-8312,8314-8316,8318-8324,8421-8422,8424-8425,8427-8430,8432-8434,8597-8598,8600-8606,8608-8612,9075,11658"}}
+{"timestamp":1714080216.0860722,"name":"online","context":{"idset":"8266"}}
+{"timestamp":1714080216.1793406,"name":"online","context":{"idset":"8607"}}
+{"timestamp":1714080216.2553046,"name":"online","context":{"idset":"8176,8235,8249,8287,9054"}}
+{"timestamp":1714080216.4038224,"name":"online","context":{"idset":"9831"}}
+{"timestamp":1714080216.5947011,"name":"online","context":{"idset":"93-94,420,579,808,827,851,8169-8173,8177,8180,11239"}}
+{"timestamp":1714080216.9317245,"name":"online","context":{"idset":"8049,8084,8174-8175,8179,10166,10180,10703,11238"}}
+{"timestamp":1714080217.0431378,"name":"online","context":{"idset":"419,573,7711,7815,8047,8051,8911,8945,8982,9138,9829,10191,10195-10196,10535,10579,10610,10652,10837,11074,11167,11181,11205,11240"}}
+{"timestamp":1714080217.2002487,"name":"online","context":{"idset":"14,56,343,405,409,8844,8867,9939,10663,10723,10881,10917,11057,11203"}}
+{"timestamp":1714080217.2464442,"name":"online","context":{"idset":"109,113,127,129,131,157,159,171,173,179,181,8832,10460,10932"}}
+{"timestamp":1714080217.2468233,"name":"online","context":{"idset":"103,177"}}
+{"timestamp":1714080217.2495496,"name":"online","context":{"idset":"9074,10667,10889,10977,11108"}}
+{"timestamp":1714080217.3768699,"name":"online","context":{"idset":"99,101,115,169,189,191,223,225,229,241,247,283,285,287,291,299,301,311,323,325,331,333,339,341,407,413,418,576,883,7730,8138,8210,8298,8840,8842,8865,8899,8977,8979,9085,9160,9198,9284,9466,9784,9810,9959,9991,10114,10147,10159,10190,10380,10390-10391,10402,10404,10430,10438,10454,10492,10532,10537,10608,10614,10622,10639,10655,10657,10665,10671,10685,10697,10705,10735,10756,10762,10770,10791,10795,10797,10871,10873,10875,10893,10899,10964,10995,11005,11045,11078,11084,11094,11104,11141,11157,11189,11201,11208,11220,11520,11590,11592,11596,11770,11876"}}
+{"timestamp":1714080217.4606347,"name":"online","context":{"idset":"95,105,107,167,8148,8850,8919,9186,9227,10583,10869,10958"}}
+{"timestamp":1714080217.4666297,"name":"online","context":{"idset":"8419,8868,9072,9493,9544,10644,10968,11195"}}
+{"timestamp":1714080217.4935071,"name":"online","context":{"idset":"8895,8912,8959"}}
+{"timestamp":1714080217.5900607,"name":"online","context":{"idset":"7,10,22,37,199,212,243,245,248,293-294,345,347,406,415,878,8046,8125,8131,8134,8308,8385,8599,8824,8852,8856,8858,8863,8866,8877-8878,8881,8887,8891,8929,8936,8940,8947,8960,8963,8978,9046,9055,9076,9079,9091,9103,9105,9115,9118-9119,9125,9133,9142,9152,9154,9158-9159,9161-9162,9170,9180,9182,9184,9189-9190,9194,9196,9200,9205,9212,9216-9217,9229,9243,9258,9291,9311,9319-9320,9322,9324,9327-9328,9330,9465,9490,9501,9527,9551,9566-9567,9571,9588,9595,9605,9615,9619,9628,9681,9688,9724,9746,9762,9782,9821,9832,9937-9938,9958,9964,9966,9972,9980,9994,10003,10113,10119,10138,10163,10181,10202,10359,10375,10384,10396-10397,10403,10432,10450,10467,10470,10482,10484,10486,10513,10547,10565,10571,10577,10585,10620,10630,10638,10689,10707,10709,10772,10863,11055,11112,11158"}}
+{"timestamp":1714080217.7202744,"name":"online","context":{"idset":"1-6,8-9,11-13,15-21,23-26,28-36,38-44,46-50,53-55,57-60,97,104,110,125,130,158,163,168,174-175,183,185-186,188,192-193,195,197-198,200-203,207,209,211,213-216,219,221,227,231,233,235,237,239-240,249,251,277,280,289,295-298,302,306-307,309,312,321,327,329,332,338,340,414,416,577,6531,6556,6565-6566,6569,6609-6610,6636,6638,7816,8050,8091,8099,8133,8136-8137,8142,8146-8147,8209,8243,8297,8302,8306,8317,8338,8350,8390,8423,8822,8825,8830,8836,8848,8851,8854,8862,8873-8875,8879,8893,8898,8917,8921,8923,8928,8930,8944,8961-8962,8973,8975,9071,9077,9080,9089,9095,9101,9108-9109,9111,9113,9137,9146,9156,9179,9252,9257,9270,9278,9282,9290,9304,9310,9470,9472,9481-9482,9485,9505,9507,9511-9512,9518,9522,9533-9534,9536,9553,9570,9622,9631,9635,9665,9694,9700,9732,9738,9752,9758,9778,9790,9794,9804,9812,9816,9823,9830,9935,9960-9962,9968,9970,9974,9996,9998-9999,10004,10011,10020,10026,10030,10034,10101,10111-10112,10122,10131,10133,10139,10149,10157,10183,10205,10209,10357,10361-10365,10367-10371,10373,10377,10382,10386-10387,10392,10400-10401,10406,10408,10412,10414,10416,10419-10420,10423-10424,10426,10428,10433,10436,10440,10442,10445-10446,10448,10452,10458,10461-10462,10464,10468,10472-10474,10476,10478,10485,10487,10489,10493,10495,10501,10505,10507,10511,10515,10517,10519,10523,10525,10529,10539,10541,10543,10545,10549,10553,10555,10557,10563,10569,10587,10592,10596,10600,10606,10612,10618,10634,10642,10654,10659,10661,10675,10677-10679,10681,10683,10687,10701,10711,10714,10717,10719,10721,10725,10727,10729,10733,10737-10739,10741,10744,10746,10750,10752,10757,10759-10760,10764,10766,10768,10774,10777-10778,10780-10781,10785-10788,10790,10796,10800,10803,10807-10809,10811,10813,10815,10819,10821,10823,10825-10826,10833,10835,10839,10841,10844-10845,10847,10851,10854-10855,10857,10859-10861,10867-10868,10874,10877,10882-10883,10887,10891,10895,10900,10902,10905,10907,10909,10912,10914,10922-10925,10927,10933-10934,10940,10944,10946,10948,10950,10952-10954,10960-10961,10966,10973,10975,10979,10983,10985,10991,10993,10997,11001,11004,11006,11009,11013-11015,11017,11019,11027-11029,11031-11032,11036-11037,11039,11041,11043,11046-11047,11049-11051,11053,11060-11061,11063-11065,11068,11070,11085-11086,11090,11092,11095,11098,11100,11106,11115-11116,11118,11120,11124-11128,11133,11135,11139,11145,11147,11149-11150,11152-11153,11155,11159,11164,11166,11171,11173-11174,11177-11179,11182-11183,11197,11204,11206,11212,11214-11215,11219,11221-11222,11224,11229,11232,11234,11236,11510,11514,11518,11524-11525,11533,11539,11541,11547,11555,11563,11565,11591,11593-11594,11603,11784,11802,11820,11828,11832,11852,11886,11892"}}
+{"timestamp":1714080217.7714114,"name":"online","context":{"idset":"238,6600,8305,8331,8382,8416,8846,8901,8905,8909,8915,8927,8937,8941-8943,8957,9129,9148,9172,9176,9187,9204,9232,9299,9463,9471,9480,9502,9537,9714,9982,10018,10197,10398,10429,10431,10456,10588,10704,10734,10865,10984,10992,11067,11114,11161"}}
+{"timestamp":1714080217.9390998,"name":"online","context":{"idset":"106,122,162,166,180,230,232,242,284,328,346,408,6559,6563,6570,6590,6598,6606,6613,6618,6634,8121,8140,8144,8303,8341,8369,8379,8392,8413,8436,8826-8829,8835,8845,8849,8855,8869,8871,8884,8886,8892,8903-8904,8906,8913-8914,8918,8920,8922,8931,8935,8938,8946,8949,8953,8966,8970,8972,9045,9050,9082-9083,9087,9099,9107,9114,9123,9127-9128,9144,9150-9151,9164,9168,9171,9177-9178,9188,9210,9220,9249,9261,9265,9293,9303,9313,9321,9326,9332,9469,9477-9478,9492,9497,9519-9520,9531,9543,9561,9564,9574,9576,9579-9580,9587,9591,9593,9599,9633,9637,9642-9643,9659,9667,9669,9673,9677,9686,9695,9698,9703-9704,9711-9712,9716,9718,9734,9736,9751,9776-9777,9779,9796,9798,9802,9818,9936,9946,9954,9963,9965,9967,9971,9986,9995,9997,10001,10010,10036,10102,10104,10110,10116,10123,10128-10129,10144,10151,10178,10182,10184,10186,10189,10199,10210-10212,10358,10388-10389,10395,10409-10411,10413,10418,10455,10457,10463,10466,10469,10479-10480,10499,10509,10533,10559,10561,10576,10594,10597-10598,10602,10626,10628,10632,10640,10646,10666,10668,10676,10680,10682,10688,10691,10700,10710,10713,10730,10740,10742,10748,10751,10754-10755,10761,10763,10767,10776,10782,10784,10789,10792-10794,10810,10814,10817-10818,10822,10829-10831,10834,10838,10842-10843,10850,10852,10878,10884,10901,10908,10915-10916,10920,10926,10937,10941,10945,10987,10999,11003,11007,11011,11016,11021,11023,11025,11033-11035,11054,11069,11075,11077,11079-11080,11087-11088,11099,11101,11105,11107,11110-11111,11113,11121-11122,11131,11134,11136,11138,11140,11143,11148,11151,11160,11162-11163,11185,11187,11198-11199,11210,11218,11226,11228,11230,11512,11516,11522,11549,11573,11579,11581,11595,11597-11598,11600,11602,11604,11609,11766,11772,11776,11780,11782,11788,11792,11806,11836,11862,11864,11878,11884,11888,11890"}}
+{"timestamp":1714080218.0707691,"name":"online","context":{"idset":"108,116,118,126,128,132,160,164,176,178,182,194,196,204,210,220,224,226,228,234,236,244,246,250,252,282,290,292,300,308,316,318,320,322,324,344,877,6517,6528,6532,6543,6552,6554,6562,6572,6576,6585,6615,6619,6643,8052,8085,8087,8089,8093-8095,8119,8123,8127,8129-8130,8135,8139,8141,8143,8145,8267,8299,8301,8307,8330,8332,8334,8336,8340,8342,8348,8354,8370,8373-8374,8376,8378,8380,8388,8394,8396-8398,8401,8403,8407,8409,8411,8414,8821,8823,8831,8837,8839,8841,8843,8853,8857,8859,8861,8864,8870,8872,8876,8880,8882-8883,8888,8890,8894,8896-8897,8900,8902,8907-8908,8916,8924-8926,8964,8968,8976,9014,9034,9038,9042,9047,9053,9059,9073,9078,9081,9084,9086,9090,9092-9093,9097,9102,9104,9106,9112,9116,9122,9126,9131,9135-9136,9141,9143,9145,9147,9149,9153,9155,9163,9165,9167,9169,9173,9175,9183,9185,9191,9193,9195,9197,9199,9201,9203,9206,9208,9213,9215,9218,9221-9222,9224-9226,9231,9234,9236,9239,9242,9244,9247,9250-9251,9253-9256,9260,9262-9264,9267-9268,9275,9277,9279,9281,9283,9287,9289,9292,9295-9298,9300,9302,9305,9307,9309,9315-9318,9323,9325,9329,9331,9461-9462,9467,9473-9475,9479,9483-9484,9487,9489,9491,9496,9504,9506,9508,9513,9515-9517,9521,9523,9525-9526,9529-9530,9532,9538-9541,9545,9547-9550,9552,9554-9559,9562-9563,9569,9572-9573,9577-9578,9582-9586,9589,9592,9597,9601-9604,9610-9614,9617-9618,9620-9621,9623-9624,9627,9629-9630,9634,9639,9641,9644,9646,9648-9655,9660-9662,9664,9666,9670-9671,9675,9679,9683,9685,9687,9689-9690,9696,9699,9702,9706-9708,9710,9713,9719,9722-9723,9735,9741-9742,9744,9748-9750,9756-9757,9759-9760,9763-9764,9768-9769,9771-9774,9780-9781,9786,9789,9793,9795,9797,9800-9801,9803,9806-9808,9814-9815,9817,9824-9827,9934,9940-9942,9944,9947-9948,9950-9952,9969,9973,9975-9978,9981,9983-9984,9988,9993,10000,10002,10012-10014,10016,10022-10023,10025,10027-10029,10032,10103,10107,10115,10120,10140,10153,10155,10158,10160,10175,10192,10198,10203-10204,10206-10207,10366,10374,10376,10378-10379,10381,10407,10415,10417,10425,10427,10435,10437,10439,10441,10447,10449,10451,10453,10459,10471,10475,10477,10481,10483,10488,10490-10491,10494,10497-10498,10500,10502-10503,10508,10512,10514,10516,10518,10521,10524,10526-10528,10531,10540,10551,10554,10556,10562,10566-10568,10575,10578,10581,10589,10595,10604-10605,10609,10611,10627,10629,10631,10633,10641,10648-10650,10656,10664,10672,10684,10686,10693,10696,10698,10702,10708,10712,10718,10722,10726,10728,10732,10736,10743,10745,10747,10749,10753,10769,10773,10775,10779,10798,10802,10804,10806,10812,10816,10820,10824,10827-10828,10832,10836,10840,10846,10856,10858,10862,10864,10866,10870,10872,10876,10886,10888,10890,10892,10894,10896-10898,10904,10906,10910,10918-10919,10921,10929,10931,10935,10939,10943,10949,10951,10957,10959,10963,10967,10971,10974,10976,10980-10982,10986,10988,10994,10998,11000,11008,11010,11012,11020,11022,11026,11030,11038,11044,11048,11056,11058,11062,11066,11073,11081,11083,11089,11091,11093,11097,11103,11117,11119,11123,11129,11132,11137,11144,11146,11154,11156,11168-11170,11172,11176,11184,11186,11188,11190,11192,11194,11196,11200,11202,11207,11209,11211,11213,11223,11225,11227,11233,11527,11529,11531,11534-11535,11537,11543,11545,11551,11557,11561,11567,11571,11575,11577,11583,11585,11587,11599,11605,11607,11611,11613-11614,11617-11618,11620,11626,11628,11632,11636,11768,11774,11778,11793-11796,11798,11800,11804,11808,11810,11812,11814-11816,11822,11830,11834,11838-11840,11842,11844,11846,11848,11850,11854,11856,11858,11860,11866,11872,11874,11882"}}
+{"timestamp":1714080218.1812508,"name":"online","context":{"idset":"6536,6542,6548,6550,6553,6557,6560,6580,6589,6596,6604,6637,6640,8086,8092,8096,8100,8122,8128,8326,8329,8344,8346,8351,8356,8360-8364,8366-8368,8372,8375,8377,8386,8402,8404,8412,8418,8834,8932,8952,8971,8974,8981,8991,8997,9005,9051-9052,9057,9096,9110,9117,9124,9132,9134,9157,9181,9209,9219,9223,9230,9233,9237,9240,9245,9271-9274,9288,9294,9314,9464,9476,9494,9510,9514,9528,9546,9565,9568,9575,9581,9590,9594,9600,9607-9608,9616,9625-9626,9638,9647,9657-9658,9668,9672,9676,9678,9680,9682,9684,9697,9715,9717,9725-9726,9733,9737,9739,9743,9745,9747,9753-9755,9775,9783,9787,9805,9811,9813,9819-9820,9822,9943,9945,9949,9953,9955,9979,9985,9987,9989,10007-10008,10015,10019,10021,10024,10031,10035,10105,10118,10121,10126-10127,10132,10137,10150,10154,10165,10170,10177,10193,10201,10383,10434,10443,10496,10504,10506,10510,10522,10530,10534,10538,10542,10544,10550,10552,10570,10572,10574,10582,10584,10586,10590-10591,10593,10601,10603,10613,10615-10617,10621,10625,10635,10637,10643,10645,10647,10651,10653,10658,10660,10670,10673-10674,10692,10694,10706,10731,10799,10849,10911,10928,10938,10955,10962,10965,10969-10970,10972,10978,11002,11018,11024,11071-11072,11082,11102,11142,11180,11193,11538,11552-11554,11556,11559,11569,11582,11589,11601,11606,11608,11612,11616,11622,11624,11630-11631,11634,11779,11781,11786,11805,11817-11819,11821,11823,11827,11829,11831,11833,11835,11843,11863,11865,11868"}}
+{"timestamp":1714080218.2914171,"name":"online","context":{"idset":"96,205,337,412,575,6522,6524,6530,6544,6561,6564,6567-6568,6574,6578-6579,6583,6595,6621,6628,6630,6633,8048,8088,8090,8098,8118,8120,8126,8132,8325,8328,8333,8335,8339,8343,8347,8349,8352,8355,8357-8359,8381,8387,8389,8391,8395,8400,8406,8408,8415,8417,8847,8889,8933-8934,8939,8950,8954-8956,8967,8969,8987,9000-9001,9016,9020-9022,9026,9040,9049,9058,9070,9088,9100,9120,9166,9207,9259,9266,9269,9280,9308,9312,9468,9488,9498-9499,9560,9596,9632,9674,9693,9701,9705,9721,9727,9729-9730,9767,9770,9799,9809,9990,10005-10006,10009,10017,10033,10125,10130,10134,10146,10162,10164,10167,10169,10172-10174,10176,10179,10200,10520,10536,10607,10619,10624,10699,10715-10716,10771,10848,10853,10903,10947,10990,11042,11059,11130,11165,11526,11532,11566,11576,11588,11623,11767,11783,11787,11791,11799,11824,11837,11849,11869"}}
+{"timestamp":1714080218.3923709,"name":"online","context":{"idset":"51,9957,10108,10168,10360,10385,10546,10623,10936,11891"}}
+{"timestamp":1714080218.5811405,"name":"online","context":{"idset":"27,45,52,170,218,222,278,281,288,315,326,6518,6520-6521,6540-6541,6555,6558,6571,6573,6575,6584,6586,6599,6603,6605,6607,6611-6612,6614,6616-6617,6620,6625,6635,6644,8327,8345,8353,8365,8384,8393,8410,8420,8838,8910,8958,8980,8986,8992,8998-8999,9011,9017,9030,9032,9048,9056,9098,9121,9140,9174,9202,9235,9238,9248,9276,9286,9301,9306,9486,9500,9503,9509,9524,9609,9645,9663,9709,9740,9766,9788,9791,9828,9992,10109,10124,10135-10136,10141-10142,10145,10156,10161,10171,10399,10465,10548,10558,10599,10662,10690,10805,10913,10942,11040,11076,11109,11175,11191,11231,11235,11509,11528,11536,11540,11546,11548,11558,11560,11562,11564,11568,11570,11572,11584,11586,11615,11619,11621,11625,11627,11629,11633,11635,11765,11769,11771,11773,11777,11785,11801,11807,11809,11811,11813,11847,11851,11853,11855,11857,11859,11870-11871,11873,11875,11877,11879,11883,11885,11889"}}
+{"timestamp":1714080218.7497714,"name":"online","context":{"idset":"852,6526,6546,6588,8371,9024,9037,9039,9041,9130,9228,9241,9598,9640,9728,9731,10106,10185,10724,11096,11511,11515,11517,11521,11542,11550,11580,11845,11887"}}
+{"timestamp":1714080218.9122479,"name":"online","context":{"idset":"417,6523,6533,6608,6622,6624,8117,8337,8383,8399,8426,8983,8985,8989,8996,9008-9009,9013,9019,9023,9025,9028,9033,9036,9069,9094,9192,9246,9285,9656,9765,9785,10148,10194,10560,10573,10720,10765,10783,10989,11513,11519,11523,11578,11610,11775,11797"}}
+{"timestamp":1714080219.0798743,"name":"online","context":{"idset":"6534,6581,6591,7752,8124,8984,8988,8990,8993-8995,9002,9006-9007,9010,9012,9015,9018,9031,9035,9542,9691,10117,10394,10564,11217,11574,11803,11867,11881"}}
+{"timestamp":1714080219.2432244,"name":"online","context":{"idset":"6602,7896,8158,8227,9029,10444"}}
+{"timestamp":1714080219.4082613,"name":"online","context":{"idset":"6641,8097,8860,8951,9027,9214,9495,10801"}}
+{"timestamp":1714080219.5760334,"name":"online","context":{"idset":"334,6519,6538,7738,7747,7793,8075,8405,9060,9606,9761,10208,10636,10885,10930,10956,11216,11880"}}
+{"timestamp":1714080219.7523444,"name":"online","context":{"idset":"6529,7716,7836,7912,7940,7967,7981,7985,7991,8010,8057,8300,9211,10580,10669,10695"}}
+{"timestamp":1714080219.85408,"name":"online","context":{"idset":"6551,6593,7680,7776,7865,7889,7901,7907,7936,7942,8005,8021,8965,9636,9692,9956,10143,10372,10393,10405,11052,11544"}}
+{"timestamp":1714080220.0234039,"name":"online","context":{"idset":"6577,6632,7704,7743,7753,7802,7808,7894,7937,7953,8031,8885,11530"}}
+{"timestamp":1714080220.1265807,"name":"online","context":{"idset":"7740,7834,7852,7965,8004,8948,11841"}}
+{"timestamp":1714080220.2295523,"name":"online","context":{"idset":"6629,7718,7868,7906,7977,7995,9720,9792,10152,10758,11237,11861"}}
+{"timestamp":1714080220.3336222,"name":"online","context":{"idset":"6547,6639,7709,7764,7770,7814,7844,7902,7913,8006,8063,8072,10996"}}
+{"timestamp":1714080220.5126667,"name":"online","context":{"idset":"305,6626,7672,7688,7720,7750,7822,7833,7903-7904,7911,7914,7918,7929,7957,8014,8073,9139"}}
+{"timestamp":1714080220.6185882,"name":"online","context":{"idset":"7676,7785,7886,7943,8007,8037,8044"}}
+{"timestamp":1714080220.7189963,"name":"online","context":{"idset":"6537,6594,6623,7690,7724,7768,7810,7826,7828-7829,7832,7856,7874,7880,7884,7898,7969,7986,8043,8064"}}
+{"timestamp":1714080220.8242855,"name":"online","context":{"idset":"6582,6631,7681,7684,7691,7710,7713,7719,7728,7733,7786,7798,7804,7818,7820,7850-7851,7853-7855,7866,7920,7926,7975,7989,8012,8022,8026,8082"}}
+{"timestamp":1714080221.0134485,"name":"online","context":{"idset":"6527,6601,6627,7670,7705,7735,7746,7769,7789,7870,7922,7941,8076"}}
+{"timestamp":1714080221.1164274,"name":"online","context":{"idset":"6525,7744,7754,7801,7848,7863,7871,7890,7939,7945,7960,8013,8024,8056,8081"}}
+{"timestamp":1714080221.2194633,"name":"online","context":{"idset":"6535,6597,7767,7806,7809,7812,7845,7909,7972,8036"}}
+{"timestamp":1714080221.3244145,"name":"online","context":{"idset":"7729,7734,7771,7781,7859,7861,7935,7994"}}
+{"timestamp":1714080221.4973469,"name":"online","context":{"idset":"6545,7779,7792,7797,7857,7862,7872,7875,7905,7949-7950,7973,7980,8028-8029,8035,8042"}}
+{"timestamp":1714080221.6043916,"name":"online","context":{"idset":"6549,6592,7682,7685,7726,7825,7876,7882,7928,7933,7971,7993,7997,8009,8016,8061,8074"}}
+{"timestamp":1714080221.7117939,"name":"online","context":{"idset":"6642,7696,7698,7727,7741,7759,7846,7908,7921,7955,7988,8053,8065-8066,8077"}}
+{"timestamp":1714080221.8189859,"name":"online","context":{"idset":"7695,7732,7756,7758,7763,7766,7778,7790,7799,7813,7824,7830,7841,7860,7893,7897,7927,7962,7976,7990,8002,8034,8045,8069-8070,8078"}}
+{"timestamp":1714080222.0042922,"name":"online","context":{"idset":"7669,7677,7791,7831,7839,7885,7970,7979,7992,7996,8017-8018,8059,8062,8071,8079"}}
+{"timestamp":1714080222.1114316,"name":"online","context":{"idset":"6539,7674,7693,7748-7749,7762,7765,7772,7800,7823,7835,7837,7849,7879,7883,7888,7916,7934,7948,7998,8011,8032-8033"}}
+{"timestamp":1714080222.2090669,"name":"online","context":{"idset":"7731,7777,7819,8001,8003,8039-8041"}}
+{"timestamp":1714080222.3155913,"name":"online","context":{"idset":"6587,7692,7708,7745,7761,7773,7787,7867,7878,7923,7947,7952,8027,8038,8060"}}
+{"timestamp":1714080222.4225492,"name":"online","context":{"idset":"7706,7717,7721,7723,7840,7858,7951,7954,7958,7983,8023,8030,8054,8068"}}
+{"timestamp":1714080222.5736096,"name":"online","context":{"idset":"7686,7689,7712,7736,7910,7925,7956,7974,7984,8015,8067,8080"}}
+{"timestamp":1714080222.7588778,"name":"online","context":{"idset":"7671,7675,7678-7679,7683,7694,7700,7714,7722,7725,7742,7760,7774,7780,7811,7838,7873,7881,7917,7919,7924,7932,7944,7946,7964,7966,7987,7999,8019"}}
+{"timestamp":1714080222.9397643,"name":"online","context":{"idset":"7707,7739,7775,7783,7796,7843,7864,7869,7877,7892,7895,7915,7930,7959,8008,8025"}}
+{"timestamp":1714080223.129658,"name":"online","context":{"idset":"7702-7703,7715,7751,7782,7794-7795,7842,7887,7931,7961,7963,7978,8000,8020,8055,8058"}}
+{"timestamp":1714080223.2356935,"name":"online","context":{"idset":"7673,7737,7827,7891,7900,7968,7982"}}
+{"timestamp":1714080223.3392982,"name":"online","context":{"idset":"7699,7784,7821,7938,8083"}}
+{"timestamp":1714080223.4447381,"name":"online","context":{"idset":"7805,7899"}}
+{"timestamp":1714080223.5493636,"name":"online","context":{"idset":"7701,7807"}}
+{"timestamp":1714080223.7236693,"name":"online","context":{"idset":"7687,7697,7755,7757,7803,7817,7847"}}
+{"timestamp":1714080224.0706499,"name":"online","context":{"idset":"7788"}}
+{"timestamp":1714080334.0243573,"name":"offline","context":{"idset":"8408"}}
+{"timestamp":1714080338.9905999,"name":"offline","context":{"idset":"8853"}}
+{"timestamp":1714080338.9943519,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1714080338.9980733,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1714080339.0017626,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1714080339.0054386,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1714080339.0091786,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1714080339.0127046,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1714080339.0162811,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1714080339.0200422,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1714080339.0325246,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1714080339.0362065,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1714080339.0398548,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1714080339.0437934,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1714080339.0475652,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1714080339.0513244,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1714080339.0551054,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1714080339.0588915,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1714080339.06743,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1714080339.075213,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1714080339.0869744,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1714080339.1068244,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1714080339.116852,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1714080339.1311963,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1714080339.1346567,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1714080339.1385415,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1714080339.1532705,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1714080339.1565738,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1714080339.1600454,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1714080339.1734772,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1714080339.1868629,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1714080339.1903226,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1714080339.1937537,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1714080339.1971881,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1714080339.2104645,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1714080339.2138736,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1714080339.2171521,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1714080339.2203343,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1714080339.233757,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1714080339.2373769,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1714080339.2408133,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1714080339.2442212,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1714080339.2476146,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1714080339.2613456,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1714080339.2647567,"name":"offline","context":{"idset":"43"}}
+{"timestamp":1714080339.2681811,"name":"offline","context":{"idset":"44"}}
+{"timestamp":1714080339.2716026,"name":"offline","context":{"idset":"45"}}
+{"timestamp":1714080339.2846224,"name":"offline","context":{"idset":"46"}}
+{"timestamp":1714080339.2880149,"name":"offline","context":{"idset":"47"}}
+{"timestamp":1714080339.291419,"name":"offline","context":{"idset":"48"}}
+{"timestamp":1714080339.2948308,"name":"offline","context":{"idset":"49"}}
+{"timestamp":1714080339.2982261,"name":"offline","context":{"idset":"50"}}
+{"timestamp":1714080339.3015916,"name":"offline","context":{"idset":"51"}}
+{"timestamp":1714080339.3156989,"name":"offline","context":{"idset":"52"}}
+{"timestamp":1714080339.3188491,"name":"offline","context":{"idset":"53"}}
+{"timestamp":1714080339.3213086,"name":"offline","context":{"idset":"54"}}
+{"timestamp":1714080339.3237903,"name":"offline","context":{"idset":"55"}}
+{"timestamp":1714080339.3271418,"name":"offline","context":{"idset":"56"}}
+{"timestamp":1714080339.3308465,"name":"offline","context":{"idset":"57"}}
+{"timestamp":1714080339.33429,"name":"offline","context":{"idset":"58"}}
+{"timestamp":1714080339.3371072,"name":"offline","context":{"idset":"59"}}
+{"timestamp":1714080339.3391161,"name":"offline","context":{"idset":"60"}}
+{"timestamp":1714080339.3412299,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1714080339.3442311,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1714080339.346945,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1714080339.3496647,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1714080339.3626473,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1714080339.36619,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1714080339.3697565,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1714080339.3822834,"name":"offline","context":{"idset":"103"}}
+{"timestamp":1714080339.3857784,"name":"offline","context":{"idset":"104"}}
+{"timestamp":1714080339.3978434,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1714080339.4008493,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1714080339.4032931,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1714080339.4066639,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1714080339.418999,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1714080339.422219,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1714080339.4259131,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1714080339.4295988,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1714080339.4329104,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1714080339.43591,"name":"offline","context":{"idset":"118"}}
+{"timestamp":1714080339.4389753,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1714080339.4422784,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1714080339.4554987,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1714080339.4587104,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1714080339.4620237,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1714080339.4653492,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1714080339.4685364,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1714080339.4714975,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1714080339.474566,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1714080339.4874082,"name":"offline","context":{"idset":"157"}}
+{"timestamp":1714080339.500618,"name":"offline","context":{"idset":"158"}}
+{"timestamp":1714080339.5038991,"name":"offline","context":{"idset":"159"}}
+{"timestamp":1714080339.5074077,"name":"offline","context":{"idset":"160"}}
+{"timestamp":1714080339.5108638,"name":"offline","context":{"idset":"162"}}
+{"timestamp":1714080339.5231528,"name":"offline","context":{"idset":"163"}}
+{"timestamp":1714080339.5267873,"name":"offline","context":{"idset":"164"}}
+{"timestamp":1714080339.530463,"name":"offline","context":{"idset":"166"}}
+{"timestamp":1714080339.5393891,"name":"offline","context":{"idset":"167"}}
+{"timestamp":1714080339.5424125,"name":"offline","context":{"idset":"168"}}
+{"timestamp":1714080339.5461385,"name":"offline","context":{"idset":"169"}}
+{"timestamp":1714080339.5496016,"name":"offline","context":{"idset":"170"}}
+{"timestamp":1714080339.5531297,"name":"offline","context":{"idset":"171"}}
+{"timestamp":1714080339.5562468,"name":"offline","context":{"idset":"173"}}
+{"timestamp":1714080339.5591686,"name":"offline","context":{"idset":"174"}}
+{"timestamp":1714080339.5620561,"name":"offline","context":{"idset":"175"}}
+{"timestamp":1714080339.5651286,"name":"offline","context":{"idset":"176"}}
+{"timestamp":1714080339.5683563,"name":"offline","context":{"idset":"177"}}
+{"timestamp":1714080339.581682,"name":"offline","context":{"idset":"178"}}
+{"timestamp":1714080339.5847981,"name":"offline","context":{"idset":"179"}}
+{"timestamp":1714080339.5880818,"name":"offline","context":{"idset":"180"}}
+{"timestamp":1714080339.5915108,"name":"offline","context":{"idset":"181"}}
+{"timestamp":1714080339.5945563,"name":"offline","context":{"idset":"182"}}
+{"timestamp":1714080339.5977972,"name":"offline","context":{"idset":"183"}}
+{"timestamp":1714080339.6011019,"name":"offline","context":{"idset":"185"}}
+{"timestamp":1714080339.6043935,"name":"offline","context":{"idset":"186"}}
+{"timestamp":1714080339.6078978,"name":"offline","context":{"idset":"188"}}
+{"timestamp":1714080339.6113961,"name":"offline","context":{"idset":"189"}}
+{"timestamp":1714080339.6148603,"name":"offline","context":{"idset":"191"}}
+{"timestamp":1714080339.618341,"name":"offline","context":{"idset":"192"}}
+{"timestamp":1714080339.6319184,"name":"offline","context":{"idset":"193"}}
+{"timestamp":1714080339.635597,"name":"offline","context":{"idset":"194"}}
+{"timestamp":1714080339.6388588,"name":"offline","context":{"idset":"195"}}
+{"timestamp":1714080339.6422887,"name":"offline","context":{"idset":"196"}}
+{"timestamp":1714080339.6456809,"name":"offline","context":{"idset":"197"}}
+{"timestamp":1714080339.6491537,"name":"offline","context":{"idset":"198"}}
+{"timestamp":1714080339.6524446,"name":"offline","context":{"idset":"199"}}
+{"timestamp":1714080339.6556809,"name":"offline","context":{"idset":"200"}}
+{"timestamp":1714080339.6589079,"name":"offline","context":{"idset":"201"}}
+{"timestamp":1714080339.6621134,"name":"offline","context":{"idset":"202"}}
+{"timestamp":1714080339.6652844,"name":"offline","context":{"idset":"203"}}
+{"timestamp":1714080339.6688685,"name":"offline","context":{"idset":"204"}}
+{"timestamp":1714080339.6724384,"name":"offline","context":{"idset":"205"}}
+{"timestamp":1714080339.6759162,"name":"offline","context":{"idset":"207"}}
+{"timestamp":1714080339.6795204,"name":"offline","context":{"idset":"209"}}
+{"timestamp":1714080339.6831148,"name":"offline","context":{"idset":"210"}}
+{"timestamp":1714080339.6858115,"name":"offline","context":{"idset":"211"}}
+{"timestamp":1714080339.688421,"name":"offline","context":{"idset":"212"}}
+{"timestamp":1714080339.6911085,"name":"offline","context":{"idset":"213"}}
+{"timestamp":1714080339.7050102,"name":"offline","context":{"idset":"214"}}
+{"timestamp":1714080339.7089443,"name":"offline","context":{"idset":"215"}}
+{"timestamp":1714080339.7119617,"name":"offline","context":{"idset":"216"}}
+{"timestamp":1714080339.7148581,"name":"offline","context":{"idset":"218"}}
+{"timestamp":1714080339.7184184,"name":"offline","context":{"idset":"219"}}
+{"timestamp":1714080339.7215419,"name":"offline","context":{"idset":"220"}}
+{"timestamp":1714080339.7244992,"name":"offline","context":{"idset":"221"}}
+{"timestamp":1714080339.7274833,"name":"offline","context":{"idset":"222"}}
+{"timestamp":1714080339.7306485,"name":"offline","context":{"idset":"223"}}
+{"timestamp":1714080339.7335842,"name":"offline","context":{"idset":"224"}}
+{"timestamp":1714080339.7368288,"name":"offline","context":{"idset":"225"}}
+{"timestamp":1714080339.7509754,"name":"offline","context":{"idset":"226"}}
+{"timestamp":1714080339.7544019,"name":"offline","context":{"idset":"227"}}
+{"timestamp":1714080339.7578235,"name":"offline","context":{"idset":"228"}}
+{"timestamp":1714080339.7613816,"name":"offline","context":{"idset":"229"}}
+{"timestamp":1714080339.7643411,"name":"offline","context":{"idset":"230"}}
+{"timestamp":1714080339.7672095,"name":"offline","context":{"idset":"231"}}
+{"timestamp":1714080339.7701111,"name":"offline","context":{"idset":"232"}}
+{"timestamp":1714080339.7732103,"name":"offline","context":{"idset":"233"}}
+{"timestamp":1714080339.776669,"name":"offline","context":{"idset":"234"}}
+{"timestamp":1714080339.7800651,"name":"offline","context":{"idset":"235"}}
+{"timestamp":1714080339.7830076,"name":"offline","context":{"idset":"236"}}
+{"timestamp":1714080339.7860227,"name":"offline","context":{"idset":"237"}}
+{"timestamp":1714080339.789166,"name":"offline","context":{"idset":"238"}}
+{"timestamp":1714080339.7923369,"name":"offline","context":{"idset":"239"}}
+{"timestamp":1714080339.7952435,"name":"offline","context":{"idset":"240"}}
+{"timestamp":1714080339.7984006,"name":"offline","context":{"idset":"241"}}
+{"timestamp":1714080339.802304,"name":"offline","context":{"idset":"242"}}
+{"timestamp":1714080339.8148999,"name":"offline","context":{"idset":"243"}}
+{"timestamp":1714080339.8168311,"name":"offline","context":{"idset":"244"}}
+{"timestamp":1714080339.818671,"name":"offline","context":{"idset":"245"}}
+{"timestamp":1714080339.8213947,"name":"offline","context":{"idset":"246"}}
+{"timestamp":1714080339.831753,"name":"offline","context":{"idset":"247"}}
+{"timestamp":1714080339.8342459,"name":"offline","context":{"idset":"248"}}
+{"timestamp":1714080339.8374152,"name":"offline","context":{"idset":"249"}}
+{"timestamp":1714080339.85026,"name":"offline","context":{"idset":"250"}}
+{"timestamp":1714080339.853544,"name":"offline","context":{"idset":"251"}}
+{"timestamp":1714080339.8566368,"name":"offline","context":{"idset":"252"}}
+{"timestamp":1714080339.8598778,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1714080339.8633397,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1714080339.8665965,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1714080339.8695521,"name":"offline","context":{"idset":"281"}}
+{"timestamp":1714080339.872874,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1714080339.8762124,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1714080339.8796201,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1714080339.8829715,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1714080339.8864214,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1714080339.8898506,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1714080339.8932703,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1714080339.8966796,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1714080339.9001138,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1714080339.9032586,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1714080339.9061728,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1714080339.9094684,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1714080339.9129303,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1714080339.9164572,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1714080339.9282112,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1714080339.9307899,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1714080339.933378,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1714080339.9439874,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1714080339.9465845,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1714080339.949167,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1714080339.9596336,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1714080339.9622655,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1714080339.9651318,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1714080339.9679544,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1714080339.9708393,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1714080339.9728153,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1714080339.9759436,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1714080339.9797819,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1714080339.983052,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1714080339.9863291,"name":"offline","context":{"idset":"318"}}
+{"timestamp":1714080339.9895608,"name":"offline","context":{"idset":"320"}}
+{"timestamp":1714080339.9923551,"name":"offline","context":{"idset":"321"}}
+{"timestamp":1714080339.9952395,"name":"offline","context":{"idset":"322"}}
+{"timestamp":1714080339.9983799,"name":"offline","context":{"idset":"323"}}
+{"timestamp":1714080340.0013788,"name":"offline","context":{"idset":"324"}}
+{"timestamp":1714080340.0038316,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1714080340.0066156,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1714080340.0094631,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1714080340.0125587,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1714080340.0158615,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1714080340.0186617,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1714080340.021409,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1714080340.0241251,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1714080340.0351434,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1714080340.0370817,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1714080340.0390625,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1714080340.0409403,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1714080340.0427608,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1714080340.0448482,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1714080340.0469756,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1714080340.058419,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1714080340.0611811,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1714080340.0636485,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1714080340.0673459,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1714080340.0711293,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1714080340.0742834,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1714080340.0773618,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1714080340.0926046,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1714080340.0953743,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1714080340.098068,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1714080340.1007962,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1714080340.1034944,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1714080340.1158233,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1714080340.1307328,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1714080340.1344545,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1714080340.137552,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1714080340.1402001,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1714080340.1429362,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1714080340.1457257,"name":"offline","context":{"idset":"573"}}
+{"timestamp":1714080340.1482725,"name":"offline","context":{"idset":"575"}}
+{"timestamp":1714080340.1509681,"name":"offline","context":{"idset":"576"}}
+{"timestamp":1714080340.1535947,"name":"offline","context":{"idset":"577"}}
+{"timestamp":1714080340.1564465,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1714080340.1599643,"name":"offline","context":{"idset":"808"}}
+{"timestamp":1714080340.1627026,"name":"offline","context":{"idset":"827"}}
+{"timestamp":1714080340.1785574,"name":"offline","context":{"idset":"851"}}
+{"timestamp":1714080340.1819205,"name":"offline","context":{"idset":"852"}}
+{"timestamp":1714080340.1854622,"name":"offline","context":{"idset":"877"}}
+{"timestamp":1714080340.188782,"name":"offline","context":{"idset":"878"}}
+{"timestamp":1714080340.192174,"name":"offline","context":{"idset":"883"}}
+{"timestamp":1714080340.1953914,"name":"offline","context":{"idset":"884"}}
+{"timestamp":1714080340.2080295,"name":"offline","context":{"idset":"6517"}}
+{"timestamp":1714080340.2108734,"name":"offline","context":{"idset":"6518"}}
+{"timestamp":1714080340.2140594,"name":"offline","context":{"idset":"6519"}}
+{"timestamp":1714080340.2172463,"name":"offline","context":{"idset":"6520"}}
+{"timestamp":1714080340.2205257,"name":"offline","context":{"idset":"6521"}}
+{"timestamp":1714080340.2237377,"name":"offline","context":{"idset":"6522"}}
+{"timestamp":1714080340.2268894,"name":"offline","context":{"idset":"6523"}}
+{"timestamp":1714080340.2302139,"name":"offline","context":{"idset":"6524"}}
+{"timestamp":1714080340.257731,"name":"offline","context":{"idset":"6525"}}
+{"timestamp":1714080340.2610044,"name":"offline","context":{"idset":"6526"}}
+{"timestamp":1714080340.2641287,"name":"offline","context":{"idset":"6527"}}
+{"timestamp":1714080340.267267,"name":"offline","context":{"idset":"6528"}}
+{"timestamp":1714080340.2703784,"name":"offline","context":{"idset":"6529"}}
+{"timestamp":1714080340.2734952,"name":"offline","context":{"idset":"6530"}}
+{"timestamp":1714080340.2762735,"name":"offline","context":{"idset":"6531"}}
+{"timestamp":1714080340.2794313,"name":"offline","context":{"idset":"6532"}}
+{"timestamp":1714080340.2822781,"name":"offline","context":{"idset":"6533"}}
+{"timestamp":1714080340.2851088,"name":"offline","context":{"idset":"6534"}}
+{"timestamp":1714080340.2878385,"name":"offline","context":{"idset":"6535"}}
+{"timestamp":1714080340.2905519,"name":"offline","context":{"idset":"6536"}}
+{"timestamp":1714080340.2933574,"name":"offline","context":{"idset":"6537"}}
+{"timestamp":1714080340.3054106,"name":"offline","context":{"idset":"6538"}}
+{"timestamp":1714080340.308857,"name":"offline","context":{"idset":"6539"}}
+{"timestamp":1714080340.312134,"name":"offline","context":{"idset":"6540"}}
+{"timestamp":1714080340.3153515,"name":"offline","context":{"idset":"6541"}}
+{"timestamp":1714080340.3186419,"name":"offline","context":{"idset":"6542"}}
+{"timestamp":1714080340.3322802,"name":"offline","context":{"idset":"6543"}}
+{"timestamp":1714080340.3354673,"name":"offline","context":{"idset":"6544"}}
+{"timestamp":1714080340.3387132,"name":"offline","context":{"idset":"6545"}}
+{"timestamp":1714080340.3418913,"name":"offline","context":{"idset":"6546"}}
+{"timestamp":1714080340.3550909,"name":"offline","context":{"idset":"6547"}}
+{"timestamp":1714080340.3583426,"name":"offline","context":{"idset":"6548"}}
+{"timestamp":1714080340.3616312,"name":"offline","context":{"idset":"6549"}}
+{"timestamp":1714080340.3650587,"name":"offline","context":{"idset":"6550"}}
+{"timestamp":1714080340.3684914,"name":"offline","context":{"idset":"6551"}}
+{"timestamp":1714080340.3715091,"name":"offline","context":{"idset":"6552"}}
+{"timestamp":1714080340.3746805,"name":"offline","context":{"idset":"6553"}}
+{"timestamp":1714080340.3775249,"name":"offline","context":{"idset":"6554"}}
+{"timestamp":1714080340.3909433,"name":"offline","context":{"idset":"6555"}}
+{"timestamp":1714080340.394264,"name":"offline","context":{"idset":"6556"}}
+{"timestamp":1714080340.3974004,"name":"offline","context":{"idset":"6557"}}
+{"timestamp":1714080340.4006443,"name":"offline","context":{"idset":"6558"}}
+{"timestamp":1714080340.4037497,"name":"offline","context":{"idset":"6559"}}
+{"timestamp":1714080340.4066164,"name":"offline","context":{"idset":"6560"}}
+{"timestamp":1714080340.4187927,"name":"offline","context":{"idset":"6561"}}
+{"timestamp":1714080340.4216716,"name":"offline","context":{"idset":"6562"}}
+{"timestamp":1714080340.4248843,"name":"offline","context":{"idset":"6563"}}
+{"timestamp":1714080340.4281576,"name":"offline","context":{"idset":"6564"}}
+{"timestamp":1714080340.4315095,"name":"offline","context":{"idset":"6565"}}
+{"timestamp":1714080340.4347925,"name":"offline","context":{"idset":"6566"}}
+{"timestamp":1714080340.4380612,"name":"offline","context":{"idset":"6567"}}
+{"timestamp":1714080340.4413648,"name":"offline","context":{"idset":"6568"}}
+{"timestamp":1714080340.4444358,"name":"offline","context":{"idset":"6569"}}
+{"timestamp":1714080340.4475603,"name":"offline","context":{"idset":"6570"}}
+{"timestamp":1714080340.4508545,"name":"offline","context":{"idset":"6571"}}
+{"timestamp":1714080340.4626355,"name":"offline","context":{"idset":"6572"}}
+{"timestamp":1714080340.4657245,"name":"offline","context":{"idset":"6573"}}
+{"timestamp":1714080340.4689307,"name":"offline","context":{"idset":"6574"}}
+{"timestamp":1714080340.4722466,"name":"offline","context":{"idset":"6575"}}
+{"timestamp":1714080340.4756458,"name":"offline","context":{"idset":"6576"}}
+{"timestamp":1714080340.4887753,"name":"offline","context":{"idset":"6577"}}
+{"timestamp":1714080340.4916697,"name":"offline","context":{"idset":"6578"}}
+{"timestamp":1714080340.4946129,"name":"offline","context":{"idset":"6579"}}
+{"timestamp":1714080340.4972215,"name":"offline","context":{"idset":"6580"}}
+{"timestamp":1714080340.4999504,"name":"offline","context":{"idset":"6581"}}
+{"timestamp":1714080340.5027897,"name":"offline","context":{"idset":"6582"}}
+{"timestamp":1714080340.5199902,"name":"offline","context":{"idset":"6583"}}
+{"timestamp":1714080340.5259562,"name":"offline","context":{"idset":"6584"}}
+{"timestamp":1714080340.5394161,"name":"offline","context":{"idset":"6585"}}
+{"timestamp":1714080340.5427332,"name":"offline","context":{"idset":"6586"}}
+{"timestamp":1714080340.5458221,"name":"offline","context":{"idset":"6587"}}
+{"timestamp":1714080340.5497403,"name":"offline","context":{"idset":"6588"}}
+{"timestamp":1714080340.5527518,"name":"offline","context":{"idset":"6589"}}
+{"timestamp":1714080340.5560522,"name":"offline","context":{"idset":"6590"}}
+{"timestamp":1714080340.5593195,"name":"offline","context":{"idset":"6591"}}
+{"timestamp":1714080340.5621598,"name":"offline","context":{"idset":"6592"}}
+{"timestamp":1714080340.5712886,"name":"offline","context":{"idset":"6593"}}
+{"timestamp":1714080340.5742915,"name":"offline","context":{"idset":"6594"}}
+{"timestamp":1714080340.5812216,"name":"offline","context":{"idset":"6595"}}
+{"timestamp":1714080340.5961473,"name":"offline","context":{"idset":"6596"}}
+{"timestamp":1714080340.5985887,"name":"offline","context":{"idset":"6597"}}
+{"timestamp":1714080340.6010263,"name":"offline","context":{"idset":"6598"}}
+{"timestamp":1714080340.6034577,"name":"offline","context":{"idset":"6599"}}
+{"timestamp":1714080340.6181986,"name":"offline","context":{"idset":"6600"}}
+{"timestamp":1714080340.6249967,"name":"offline","context":{"idset":"6601"}}
+{"timestamp":1714080340.6276052,"name":"offline","context":{"idset":"6602"}}
+{"timestamp":1714080340.630177,"name":"offline","context":{"idset":"6603"}}
+{"timestamp":1714080340.6413114,"name":"offline","context":{"idset":"6604"}}
+{"timestamp":1714080340.6453362,"name":"offline","context":{"idset":"6605"}}
+{"timestamp":1714080340.6482122,"name":"offline","context":{"idset":"6606"}}
+{"timestamp":1714080340.6507823,"name":"offline","context":{"idset":"6607"}}
+{"timestamp":1714080340.663275,"name":"offline","context":{"idset":"6608"}}
+{"timestamp":1714080340.6657808,"name":"offline","context":{"idset":"6609"}}
+{"timestamp":1714080340.6681757,"name":"offline","context":{"idset":"6610"}}
+{"timestamp":1714080340.6698554,"name":"offline","context":{"idset":"6611"}}
+{"timestamp":1714080340.67782,"name":"offline","context":{"idset":"6612"}}
+{"timestamp":1714080340.6807916,"name":"offline","context":{"idset":"6613"}}
+{"timestamp":1714080340.6838439,"name":"offline","context":{"idset":"6614"}}
+{"timestamp":1714080340.6862035,"name":"offline","context":{"idset":"6615"}}
+{"timestamp":1714080340.7001967,"name":"offline","context":{"idset":"6616"}}
+{"timestamp":1714080340.7028463,"name":"offline","context":{"idset":"6617"}}
+{"timestamp":1714080340.7057302,"name":"offline","context":{"idset":"6618"}}
+{"timestamp":1714080340.7084746,"name":"offline","context":{"idset":"6619"}}
+{"timestamp":1714080340.7111444,"name":"offline","context":{"idset":"6620"}}
+{"timestamp":1714080340.7139642,"name":"offline","context":{"idset":"6621"}}
+{"timestamp":1714080340.7160516,"name":"offline","context":{"idset":"6622"}}
+{"timestamp":1714080340.7190359,"name":"offline","context":{"idset":"6623"}}
+{"timestamp":1714080340.7218308,"name":"offline","context":{"idset":"6624"}}
+{"timestamp":1714080340.7246034,"name":"offline","context":{"idset":"6625"}}
+{"timestamp":1714080340.7274275,"name":"offline","context":{"idset":"6626"}}
+{"timestamp":1714080340.7300279,"name":"offline","context":{"idset":"6627"}}
+{"timestamp":1714080340.7327685,"name":"offline","context":{"idset":"6628"}}
+{"timestamp":1714080340.7351944,"name":"offline","context":{"idset":"6629"}}
+{"timestamp":1714080340.7380455,"name":"offline","context":{"idset":"6630"}}
+{"timestamp":1714080340.740387,"name":"offline","context":{"idset":"6631"}}
+{"timestamp":1714080340.7427309,"name":"offline","context":{"idset":"6632"}}
+{"timestamp":1714080340.7452826,"name":"offline","context":{"idset":"6633"}}
+{"timestamp":1714080340.7480524,"name":"offline","context":{"idset":"6634"}}
+{"timestamp":1714080340.7505171,"name":"offline","context":{"idset":"6635"}}
+{"timestamp":1714080340.7619553,"name":"offline","context":{"idset":"6636"}}
+{"timestamp":1714080340.7648222,"name":"offline","context":{"idset":"6637"}}
+{"timestamp":1714080340.7679467,"name":"offline","context":{"idset":"6638"}}
+{"timestamp":1714080340.7710888,"name":"offline","context":{"idset":"6639"}}
+{"timestamp":1714080340.7742095,"name":"offline","context":{"idset":"6640"}}
+{"timestamp":1714080340.7868178,"name":"offline","context":{"idset":"6641"}}
+{"timestamp":1714080340.7892206,"name":"offline","context":{"idset":"6642"}}
+{"timestamp":1714080340.7922099,"name":"offline","context":{"idset":"6643"}}
+{"timestamp":1714080340.8056107,"name":"offline","context":{"idset":"6644"}}
+{"timestamp":1714080340.8084757,"name":"offline","context":{"idset":"7669"}}
+{"timestamp":1714080340.8112149,"name":"offline","context":{"idset":"7670"}}
+{"timestamp":1714080340.8216207,"name":"offline","context":{"idset":"7671"}}
+{"timestamp":1714080340.8239391,"name":"offline","context":{"idset":"7672"}}
+{"timestamp":1714080340.8266382,"name":"offline","context":{"idset":"7673"}}
+{"timestamp":1714080340.8295457,"name":"offline","context":{"idset":"7674"}}
+{"timestamp":1714080340.8323298,"name":"offline","context":{"idset":"7675"}}
+{"timestamp":1714080340.8448265,"name":"offline","context":{"idset":"7676"}}
+{"timestamp":1714080340.8475692,"name":"offline","context":{"idset":"7677"}}
+{"timestamp":1714080340.8502402,"name":"offline","context":{"idset":"7678"}}
+{"timestamp":1714080340.8530173,"name":"offline","context":{"idset":"7679"}}
+{"timestamp":1714080340.8633988,"name":"offline","context":{"idset":"7680"}}
+{"timestamp":1714080340.8659315,"name":"offline","context":{"idset":"7681"}}
+{"timestamp":1714080340.8682933,"name":"offline","context":{"idset":"7682"}}
+{"timestamp":1714080340.87942,"name":"offline","context":{"idset":"7683"}}
+{"timestamp":1714080340.8815992,"name":"offline","context":{"idset":"7684"}}
+{"timestamp":1714080340.8836713,"name":"offline","context":{"idset":"7685"}}
+{"timestamp":1714080340.8928828,"name":"offline","context":{"idset":"7686"}}
+{"timestamp":1714080340.8954413,"name":"offline","context":{"idset":"7687"}}
+{"timestamp":1714080340.8977242,"name":"offline","context":{"idset":"7688"}}
+{"timestamp":1714080340.908371,"name":"offline","context":{"idset":"7689"}}
+{"timestamp":1714080340.9110107,"name":"offline","context":{"idset":"7690"}}
+{"timestamp":1714080340.9138842,"name":"offline","context":{"idset":"7691"}}
+{"timestamp":1714080340.9247568,"name":"offline","context":{"idset":"7692"}}
+{"timestamp":1714080340.9277265,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1714080340.9306941,"name":"offline","context":{"idset":"7694"}}
+{"timestamp":1714080340.933598,"name":"offline","context":{"idset":"7695"}}
+{"timestamp":1714080340.9359155,"name":"offline","context":{"idset":"7696"}}
+{"timestamp":1714080340.9383869,"name":"offline","context":{"idset":"7697"}}
+{"timestamp":1714080340.9493933,"name":"offline","context":{"idset":"7698"}}
+{"timestamp":1714080340.951735,"name":"offline","context":{"idset":"7699"}}
+{"timestamp":1714080340.9538898,"name":"offline","context":{"idset":"7700"}}
+{"timestamp":1714080340.9628985,"name":"offline","context":{"idset":"7701"}}
+{"timestamp":1714080340.9656763,"name":"offline","context":{"idset":"7702"}}
+{"timestamp":1714080340.9686449,"name":"offline","context":{"idset":"7703"}}
+{"timestamp":1714080340.9711378,"name":"offline","context":{"idset":"7704"}}
+{"timestamp":1714080340.9740407,"name":"offline","context":{"idset":"7705"}}
+{"timestamp":1714080340.9876783,"name":"offline","context":{"idset":"7706"}}
+{"timestamp":1714080340.9906127,"name":"offline","context":{"idset":"7707"}}
+{"timestamp":1714080340.993485,"name":"offline","context":{"idset":"7708"}}
+{"timestamp":1714080340.9959879,"name":"offline","context":{"idset":"7709"}}
+{"timestamp":1714080340.9984114,"name":"offline","context":{"idset":"7710"}}
+{"timestamp":1714080341.0006995,"name":"offline","context":{"idset":"7711"}}
+{"timestamp":1714080341.003159,"name":"offline","context":{"idset":"7712"}}
+{"timestamp":1714080341.0056612,"name":"offline","context":{"idset":"7713"}}
+{"timestamp":1714080341.008049,"name":"offline","context":{"idset":"7714"}}
+{"timestamp":1714080341.0103364,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1714080341.0126374,"name":"offline","context":{"idset":"7716"}}
+{"timestamp":1714080341.014555,"name":"offline","context":{"idset":"7717"}}
+{"timestamp":1714080341.0164745,"name":"offline","context":{"idset":"7718"}}
+{"timestamp":1714080341.0187712,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1714080341.0213294,"name":"offline","context":{"idset":"7720"}}
+{"timestamp":1714080341.0238442,"name":"offline","context":{"idset":"7721"}}
+{"timestamp":1714080341.0334237,"name":"offline","context":{"idset":"7722"}}
+{"timestamp":1714080341.0357184,"name":"offline","context":{"idset":"7723"}}
+{"timestamp":1714080341.0377941,"name":"offline","context":{"idset":"7724"}}
+{"timestamp":1714080341.0394976,"name":"offline","context":{"idset":"7725"}}
+{"timestamp":1714080341.0412524,"name":"offline","context":{"idset":"7726"}}
+{"timestamp":1714080341.0430157,"name":"offline","context":{"idset":"7727"}}
+{"timestamp":1714080341.0446575,"name":"offline","context":{"idset":"7728"}}
+{"timestamp":1714080341.0551827,"name":"offline","context":{"idset":"7729"}}
+{"timestamp":1714080341.0656407,"name":"offline","context":{"idset":"7730"}}
+{"timestamp":1714080341.0689662,"name":"offline","context":{"idset":"7731"}}
+{"timestamp":1714080341.0713568,"name":"offline","context":{"idset":"7732"}}
+{"timestamp":1714080341.0737603,"name":"offline","context":{"idset":"7733"}}
+{"timestamp":1714080341.0767832,"name":"offline","context":{"idset":"7734"}}
+{"timestamp":1714080341.079478,"name":"offline","context":{"idset":"7735"}}
+{"timestamp":1714080341.0819969,"name":"offline","context":{"idset":"7736"}}
+{"timestamp":1714080341.0844591,"name":"offline","context":{"idset":"7737"}}
+{"timestamp":1714080341.0869401,"name":"offline","context":{"idset":"7738"}}
+{"timestamp":1714080341.089577,"name":"offline","context":{"idset":"7739"}}
+{"timestamp":1714080341.0923109,"name":"offline","context":{"idset":"7740"}}
+{"timestamp":1714080341.0953121,"name":"offline","context":{"idset":"7741"}}
+{"timestamp":1714080341.0975678,"name":"offline","context":{"idset":"7742"}}
+{"timestamp":1714080341.0998154,"name":"offline","context":{"idset":"7743"}}
+{"timestamp":1714080341.102108,"name":"offline","context":{"idset":"7744"}}
+{"timestamp":1714080341.1051626,"name":"offline","context":{"idset":"7745"}}
+{"timestamp":1714080341.1077688,"name":"offline","context":{"idset":"7746"}}
+{"timestamp":1714080341.1100154,"name":"offline","context":{"idset":"7747"}}
+{"timestamp":1714080341.1122594,"name":"offline","context":{"idset":"7748"}}
+{"timestamp":1714080341.1226702,"name":"offline","context":{"idset":"7749"}}
+{"timestamp":1714080341.1254232,"name":"offline","context":{"idset":"7750"}}
+{"timestamp":1714080341.1278138,"name":"offline","context":{"idset":"7751"}}
+{"timestamp":1714080341.130229,"name":"offline","context":{"idset":"7752"}}
+{"timestamp":1714080341.1454346,"name":"offline","context":{"idset":"7753"}}
+{"timestamp":1714080341.1477487,"name":"offline","context":{"idset":"7754"}}
+{"timestamp":1714080341.1508479,"name":"offline","context":{"idset":"7755"}}
+{"timestamp":1714080341.1538754,"name":"offline","context":{"idset":"7756"}}
+{"timestamp":1714080341.1642551,"name":"offline","context":{"idset":"7757"}}
+{"timestamp":1714080341.1666653,"name":"offline","context":{"idset":"7758"}}
+{"timestamp":1714080341.169548,"name":"offline","context":{"idset":"7759"}}
+{"timestamp":1714080341.1726432,"name":"offline","context":{"idset":"7760"}}
+{"timestamp":1714080341.1830652,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1714080341.1934462,"name":"offline","context":{"idset":"7762"}}
+{"timestamp":1714080341.196317,"name":"offline","context":{"idset":"7763"}}
+{"timestamp":1714080341.198632,"name":"offline","context":{"idset":"7764"}}
+{"timestamp":1714080341.2009528,"name":"offline","context":{"idset":"7765"}}
+{"timestamp":1714080341.2035193,"name":"offline","context":{"idset":"7766"}}
+{"timestamp":1714080341.2060461,"name":"offline","context":{"idset":"7767"}}
+{"timestamp":1714080341.2168503,"name":"offline","context":{"idset":"7768"}}
+{"timestamp":1714080341.2198868,"name":"offline","context":{"idset":"7769"}}
+{"timestamp":1714080341.2221682,"name":"offline","context":{"idset":"7770"}}
+{"timestamp":1714080341.2243884,"name":"offline","context":{"idset":"7771"}}
+{"timestamp":1714080341.2266827,"name":"offline","context":{"idset":"7772"}}
+{"timestamp":1714080341.2293739,"name":"offline","context":{"idset":"7773"}}
+{"timestamp":1714080341.2322981,"name":"offline","context":{"idset":"7774"}}
+{"timestamp":1714080341.2348163,"name":"offline","context":{"idset":"7775"}}
+{"timestamp":1714080341.2374003,"name":"offline","context":{"idset":"7776"}}
+{"timestamp":1714080341.2475898,"name":"offline","context":{"idset":"7777"}}
+{"timestamp":1714080341.2499182,"name":"offline","context":{"idset":"7778"}}
+{"timestamp":1714080341.2522902,"name":"offline","context":{"idset":"7779"}}
+{"timestamp":1714080341.2547283,"name":"offline","context":{"idset":"7780"}}
+{"timestamp":1714080341.2653828,"name":"offline","context":{"idset":"7781"}}
+{"timestamp":1714080341.2677975,"name":"offline","context":{"idset":"7782"}}
+{"timestamp":1714080341.2703855,"name":"offline","context":{"idset":"7783"}}
+{"timestamp":1714080341.2806184,"name":"offline","context":{"idset":"7784"}}
+{"timestamp":1714080341.2829449,"name":"offline","context":{"idset":"7785"}}
+{"timestamp":1714080341.28526,"name":"offline","context":{"idset":"7786"}}
+{"timestamp":1714080341.2875292,"name":"offline","context":{"idset":"7787"}}
+{"timestamp":1714080341.2980695,"name":"offline","context":{"idset":"7788"}}
+{"timestamp":1714080341.3003097,"name":"offline","context":{"idset":"7789"}}
+{"timestamp":1714080341.3027487,"name":"offline","context":{"idset":"7790"}}
+{"timestamp":1714080341.3129289,"name":"offline","context":{"idset":"7791"}}
+{"timestamp":1714080341.3152242,"name":"offline","context":{"idset":"7792"}}
+{"timestamp":1714080341.3273973,"name":"offline","context":{"idset":"7793"}}
+{"timestamp":1714080341.3295875,"name":"offline","context":{"idset":"7794"}}
+{"timestamp":1714080341.3317769,"name":"offline","context":{"idset":"7795"}}
+{"timestamp":1714080341.3341312,"name":"offline","context":{"idset":"7796"}}
+{"timestamp":1714080341.3371055,"name":"offline","context":{"idset":"7797"}}
+{"timestamp":1714080341.3477693,"name":"offline","context":{"idset":"7798"}}
+{"timestamp":1714080341.3506446,"name":"offline","context":{"idset":"7799"}}
+{"timestamp":1714080341.3535063,"name":"offline","context":{"idset":"7800"}}
+{"timestamp":1714080341.3563726,"name":"offline","context":{"idset":"7801"}}
+{"timestamp":1714080341.3592651,"name":"offline","context":{"idset":"7802"}}
+{"timestamp":1714080341.3620899,"name":"offline","context":{"idset":"7803"}}
+{"timestamp":1714080341.3649476,"name":"offline","context":{"idset":"7804"}}
+{"timestamp":1714080341.3863237,"name":"offline","context":{"idset":"7805"}}
+{"timestamp":1714080341.3891847,"name":"offline","context":{"idset":"7806"}}
+{"timestamp":1714080341.3920548,"name":"offline","context":{"idset":"7807"}}
+{"timestamp":1714080341.3950582,"name":"offline","context":{"idset":"7808"}}
+{"timestamp":1714080341.3981447,"name":"offline","context":{"idset":"7809"}}
+{"timestamp":1714080341.4010007,"name":"offline","context":{"idset":"7810"}}
+{"timestamp":1714080341.4038453,"name":"offline","context":{"idset":"7811"}}
+{"timestamp":1714080341.4066849,"name":"offline","context":{"idset":"7812"}}
+{"timestamp":1714080341.4095364,"name":"offline","context":{"idset":"7813"}}
+{"timestamp":1714080341.4204476,"name":"offline","context":{"idset":"7814"}}
+{"timestamp":1714080341.4236531,"name":"offline","context":{"idset":"7815"}}
+{"timestamp":1714080341.4269781,"name":"offline","context":{"idset":"7816"}}
+{"timestamp":1714080341.4298689,"name":"offline","context":{"idset":"7817"}}
+{"timestamp":1714080341.4326789,"name":"offline","context":{"idset":"7818"}}
+{"timestamp":1714080341.4355242,"name":"offline","context":{"idset":"7819"}}
+{"timestamp":1714080341.4383583,"name":"offline","context":{"idset":"7820"}}
+{"timestamp":1714080341.4411809,"name":"offline","context":{"idset":"7821"}}
+{"timestamp":1714080341.4440217,"name":"offline","context":{"idset":"7822"}}
+{"timestamp":1714080341.4562051,"name":"offline","context":{"idset":"7823"}}
+{"timestamp":1714080341.4590507,"name":"offline","context":{"idset":"7824"}}
+{"timestamp":1714080341.4619596,"name":"offline","context":{"idset":"7825"}}
+{"timestamp":1714080341.4648075,"name":"offline","context":{"idset":"7826"}}
+{"timestamp":1714080341.467644,"name":"offline","context":{"idset":"7827"}}
+{"timestamp":1714080341.4704666,"name":"offline","context":{"idset":"7828"}}
+{"timestamp":1714080341.4733098,"name":"offline","context":{"idset":"7829"}}
+{"timestamp":1714080341.4854622,"name":"offline","context":{"idset":"7830"}}
+{"timestamp":1714080341.49756,"name":"offline","context":{"idset":"7831"}}
+{"timestamp":1714080341.5004156,"name":"offline","context":{"idset":"7832"}}
+{"timestamp":1714080341.5032463,"name":"offline","context":{"idset":"7833"}}
+{"timestamp":1714080341.5060687,"name":"offline","context":{"idset":"7834"}}
+{"timestamp":1714080341.5187745,"name":"offline","context":{"idset":"7835"}}
+{"timestamp":1714080341.5311019,"name":"offline","context":{"idset":"7836"}}
+{"timestamp":1714080341.5434632,"name":"offline","context":{"idset":"7837"}}
+{"timestamp":1714080341.5463395,"name":"offline","context":{"idset":"7838"}}
+{"timestamp":1714080341.5492127,"name":"offline","context":{"idset":"7839"}}
+{"timestamp":1714080341.552074,"name":"offline","context":{"idset":"7840"}}
+{"timestamp":1714080341.5549493,"name":"offline","context":{"idset":"7841"}}
+{"timestamp":1714080341.5578473,"name":"offline","context":{"idset":"7842"}}
+{"timestamp":1714080341.5606236,"name":"offline","context":{"idset":"7843"}}
+{"timestamp":1714080341.5635207,"name":"offline","context":{"idset":"7844"}}
+{"timestamp":1714080341.5663071,"name":"offline","context":{"idset":"7845"}}
+{"timestamp":1714080341.5692811,"name":"offline","context":{"idset":"7846"}}
+{"timestamp":1714080341.5722435,"name":"offline","context":{"idset":"7847"}}
+{"timestamp":1714080341.5752027,"name":"offline","context":{"idset":"7848"}}
+{"timestamp":1714080341.5874598,"name":"offline","context":{"idset":"7849"}}
+{"timestamp":1714080341.5903225,"name":"offline","context":{"idset":"7850"}}
+{"timestamp":1714080341.5931802,"name":"offline","context":{"idset":"7851"}}
+{"timestamp":1714080341.596015,"name":"offline","context":{"idset":"7852"}}
+{"timestamp":1714080341.5988696,"name":"offline","context":{"idset":"7853"}}
+{"timestamp":1714080341.6016879,"name":"offline","context":{"idset":"7854"}}
+{"timestamp":1714080341.6046252,"name":"offline","context":{"idset":"7855"}}
+{"timestamp":1714080341.6075077,"name":"offline","context":{"idset":"7856"}}
+{"timestamp":1714080341.6103439,"name":"offline","context":{"idset":"7857"}}
+{"timestamp":1714080341.613344,"name":"offline","context":{"idset":"7858"}}
+{"timestamp":1714080341.6162722,"name":"offline","context":{"idset":"7859"}}
+{"timestamp":1714080341.6275389,"name":"offline","context":{"idset":"7860"}}
+{"timestamp":1714080341.6297045,"name":"offline","context":{"idset":"7861"}}
+{"timestamp":1714080341.63186,"name":"offline","context":{"idset":"7862"}}
+{"timestamp":1714080341.6343572,"name":"offline","context":{"idset":"7863"}}
+{"timestamp":1714080341.6371934,"name":"offline","context":{"idset":"7864"}}
+{"timestamp":1714080341.649462,"name":"offline","context":{"idset":"7865"}}
+{"timestamp":1714080341.6522849,"name":"offline","context":{"idset":"7866"}}
+{"timestamp":1714080341.6551037,"name":"offline","context":{"idset":"7867"}}
+{"timestamp":1714080341.6673713,"name":"offline","context":{"idset":"7868"}}
+{"timestamp":1714080341.6702361,"name":"offline","context":{"idset":"7869"}}
+{"timestamp":1714080341.6731262,"name":"offline","context":{"idset":"7870"}}
+{"timestamp":1714080341.6760166,"name":"offline","context":{"idset":"7871"}}
+{"timestamp":1714080341.6789041,"name":"offline","context":{"idset":"7872"}}
+{"timestamp":1714080341.6816487,"name":"offline","context":{"idset":"7873"}}
+{"timestamp":1714080341.6844521,"name":"offline","context":{"idset":"7874"}}
+{"timestamp":1714080341.6872401,"name":"offline","context":{"idset":"7875"}}
+{"timestamp":1714080341.6900184,"name":"offline","context":{"idset":"7876"}}
+{"timestamp":1714080341.6928296,"name":"offline","context":{"idset":"7877"}}
+{"timestamp":1714080341.7051148,"name":"offline","context":{"idset":"7878"}}
+{"timestamp":1714080341.7079306,"name":"offline","context":{"idset":"7879"}}
+{"timestamp":1714080341.710746,"name":"offline","context":{"idset":"7880"}}
+{"timestamp":1714080341.713625,"name":"offline","context":{"idset":"7881"}}
+{"timestamp":1714080341.7164137,"name":"offline","context":{"idset":"7882"}}
+{"timestamp":1714080341.7192578,"name":"offline","context":{"idset":"7883"}}
+{"timestamp":1714080341.7316501,"name":"offline","context":{"idset":"7884"}}
+{"timestamp":1714080341.7344439,"name":"offline","context":{"idset":"7885"}}
+{"timestamp":1714080341.7372785,"name":"offline","context":{"idset":"7886"}}
+{"timestamp":1714080341.7400451,"name":"offline","context":{"idset":"7887"}}
+{"timestamp":1714080341.7522812,"name":"offline","context":{"idset":"7888"}}
+{"timestamp":1714080341.7551687,"name":"offline","context":{"idset":"7889"}}
+{"timestamp":1714080341.7580535,"name":"offline","context":{"idset":"7890"}}
+{"timestamp":1714080341.7609017,"name":"offline","context":{"idset":"7891"}}
+{"timestamp":1714080341.7638218,"name":"offline","context":{"idset":"7892"}}
+{"timestamp":1714080341.7666113,"name":"offline","context":{"idset":"7893"}}
+{"timestamp":1714080341.7694378,"name":"offline","context":{"idset":"7894"}}
+{"timestamp":1714080341.7721946,"name":"offline","context":{"idset":"7895"}}
+{"timestamp":1714080341.7749088,"name":"offline","context":{"idset":"7896"}}
+{"timestamp":1714080341.7873089,"name":"offline","context":{"idset":"7897"}}
+{"timestamp":1714080341.7901249,"name":"offline","context":{"idset":"7898"}}
+{"timestamp":1714080341.7929299,"name":"offline","context":{"idset":"7899"}}
+{"timestamp":1714080341.7957151,"name":"offline","context":{"idset":"7900"}}
+{"timestamp":1714080341.7984157,"name":"offline","context":{"idset":"7901"}}
+{"timestamp":1714080341.8011816,"name":"offline","context":{"idset":"7902"}}
+{"timestamp":1714080341.8039455,"name":"offline","context":{"idset":"7903"}}
+{"timestamp":1714080341.8159199,"name":"offline","context":{"idset":"7904"}}
+{"timestamp":1714080341.8186786,"name":"offline","context":{"idset":"7905"}}
+{"timestamp":1714080341.8214169,"name":"offline","context":{"idset":"7906"}}
+{"timestamp":1714080341.8241341,"name":"offline","context":{"idset":"7907"}}
+{"timestamp":1714080341.8268714,"name":"offline","context":{"idset":"7908"}}
+{"timestamp":1714080341.8295426,"name":"offline","context":{"idset":"7909"}}
+{"timestamp":1714080341.8413901,"name":"offline","context":{"idset":"7910"}}
+{"timestamp":1714080341.8441725,"name":"offline","context":{"idset":"7911"}}
+{"timestamp":1714080341.8468757,"name":"offline","context":{"idset":"7912"}}
+{"timestamp":1714080341.8497257,"name":"offline","context":{"idset":"7913"}}
+{"timestamp":1714080341.8623798,"name":"offline","context":{"idset":"7914"}}
+{"timestamp":1714080341.8740954,"name":"offline","context":{"idset":"7915"}}
+{"timestamp":1714080341.8766046,"name":"offline","context":{"idset":"7916"}}
+{"timestamp":1714080341.8791399,"name":"offline","context":{"idset":"7917"}}
+{"timestamp":1714080341.881614,"name":"offline","context":{"idset":"7918"}}
+{"timestamp":1714080341.8841774,"name":"offline","context":{"idset":"7919"}}
+{"timestamp":1714080341.9052608,"name":"offline","context":{"idset":"7920"}}
+{"timestamp":1714080341.9077573,"name":"offline","context":{"idset":"7921"}}
+{"timestamp":1714080341.9103982,"name":"offline","context":{"idset":"7922"}}
+{"timestamp":1714080341.9127657,"name":"offline","context":{"idset":"7923"}}
+{"timestamp":1714080341.9152603,"name":"offline","context":{"idset":"7924"}}
+{"timestamp":1714080341.9178309,"name":"offline","context":{"idset":"7925"}}
+{"timestamp":1714080341.9202619,"name":"offline","context":{"idset":"7926"}}
+{"timestamp":1714080341.930898,"name":"offline","context":{"idset":"7927"}}
+{"timestamp":1714080341.9333177,"name":"offline","context":{"idset":"7928"}}
+{"timestamp":1714080341.9359396,"name":"offline","context":{"idset":"7929"}}
+{"timestamp":1714080341.9471567,"name":"offline","context":{"idset":"7930"}}
+{"timestamp":1714080341.9577467,"name":"offline","context":{"idset":"7931"}}
+{"timestamp":1714080341.9601798,"name":"offline","context":{"idset":"7932"}}
+{"timestamp":1714080341.9708154,"name":"offline","context":{"idset":"7933"}}
+{"timestamp":1714080341.9732349,"name":"offline","context":{"idset":"7934"}}
+{"timestamp":1714080341.9756398,"name":"offline","context":{"idset":"7935"}}
+{"timestamp":1714080341.9781981,"name":"offline","context":{"idset":"7936"}}
+{"timestamp":1714080341.980757,"name":"offline","context":{"idset":"7937"}}
+{"timestamp":1714080341.9833241,"name":"offline","context":{"idset":"7938"}}
+{"timestamp":1714080341.9859207,"name":"offline","context":{"idset":"7939"}}
+{"timestamp":1714080341.9884434,"name":"offline","context":{"idset":"7940"}}
+{"timestamp":1714080341.9910364,"name":"offline","context":{"idset":"7941"}}
+{"timestamp":1714080341.9936264,"name":"offline","context":{"idset":"7942"}}
+{"timestamp":1714080341.9960623,"name":"offline","context":{"idset":"7943"}}
+{"timestamp":1714080341.998543,"name":"offline","context":{"idset":"7944"}}
+{"timestamp":1714080342.0011079,"name":"offline","context":{"idset":"7945"}}
+{"timestamp":1714080342.0133798,"name":"offline","context":{"idset":"7946"}}
+{"timestamp":1714080342.0160835,"name":"offline","context":{"idset":"7947"}}
+{"timestamp":1714080342.0191898,"name":"offline","context":{"idset":"7948"}}
+{"timestamp":1714080342.0217965,"name":"offline","context":{"idset":"7949"}}
+{"timestamp":1714080342.0333211,"name":"offline","context":{"idset":"7950"}}
+{"timestamp":1714080342.0358253,"name":"offline","context":{"idset":"7951"}}
+{"timestamp":1714080342.038425,"name":"offline","context":{"idset":"7952"}}
+{"timestamp":1714080342.0410066,"name":"offline","context":{"idset":"7953"}}
+{"timestamp":1714080342.0525343,"name":"offline","context":{"idset":"7954"}}
+{"timestamp":1714080342.0551066,"name":"offline","context":{"idset":"7955"}}
+{"timestamp":1714080342.057718,"name":"offline","context":{"idset":"7956"}}
+{"timestamp":1714080342.0602715,"name":"offline","context":{"idset":"7957"}}
+{"timestamp":1714080342.0720415,"name":"offline","context":{"idset":"7958"}}
+{"timestamp":1714080342.0746267,"name":"offline","context":{"idset":"7959"}}
+{"timestamp":1714080342.0771527,"name":"offline","context":{"idset":"7960"}}
+{"timestamp":1714080342.0797062,"name":"offline","context":{"idset":"7961"}}
+{"timestamp":1714080342.0822661,"name":"offline","context":{"idset":"7962"}}
+{"timestamp":1714080342.084831,"name":"offline","context":{"idset":"7963"}}
+{"timestamp":1714080342.0874755,"name":"offline","context":{"idset":"7964"}}
+{"timestamp":1714080342.1031389,"name":"offline","context":{"idset":"7965"}}
+{"timestamp":1714080342.1056788,"name":"offline","context":{"idset":"7966"}}
+{"timestamp":1714080342.1082199,"name":"offline","context":{"idset":"7967"}}
+{"timestamp":1714080342.1107318,"name":"offline","context":{"idset":"7968"}}
+{"timestamp":1714080342.1132586,"name":"offline","context":{"idset":"7969"}}
+{"timestamp":1714080342.1158173,"name":"offline","context":{"idset":"7970"}}
+{"timestamp":1714080342.1183743,"name":"offline","context":{"idset":"7971"}}
+{"timestamp":1714080342.1209519,"name":"offline","context":{"idset":"7972"}}
+{"timestamp":1714080342.1234956,"name":"offline","context":{"idset":"7973"}}
+{"timestamp":1714080342.135823,"name":"offline","context":{"idset":"7974"}}
+{"timestamp":1714080342.1384158,"name":"offline","context":{"idset":"7975"}}
+{"timestamp":1714080342.1410244,"name":"offline","context":{"idset":"7976"}}
+{"timestamp":1714080342.1503019,"name":"offline","context":{"idset":"7977"}}
+{"timestamp":1714080342.1528149,"name":"offline","context":{"idset":"7978"}}
+{"timestamp":1714080342.1549575,"name":"offline","context":{"idset":"7979"}}
+{"timestamp":1714080342.1572161,"name":"offline","context":{"idset":"7980"}}
+{"timestamp":1714080342.1597781,"name":"offline","context":{"idset":"7981"}}
+{"timestamp":1714080342.1623831,"name":"offline","context":{"idset":"7982"}}
+{"timestamp":1714080342.1650398,"name":"offline","context":{"idset":"7983"}}
+{"timestamp":1714080342.1676207,"name":"offline","context":{"idset":"7984"}}
+{"timestamp":1714080342.17029,"name":"offline","context":{"idset":"7985"}}
+{"timestamp":1714080342.1729796,"name":"offline","context":{"idset":"7986"}}
+{"timestamp":1714080342.1849744,"name":"offline","context":{"idset":"7987"}}
+{"timestamp":1714080342.1970162,"name":"offline","context":{"idset":"7988"}}
+{"timestamp":1714080342.1997049,"name":"offline","context":{"idset":"7989"}}
+{"timestamp":1714080342.2023861,"name":"offline","context":{"idset":"7990"}}
+{"timestamp":1714080342.2050343,"name":"offline","context":{"idset":"7991"}}
+{"timestamp":1714080342.2077537,"name":"offline","context":{"idset":"7992"}}
+{"timestamp":1714080342.2106485,"name":"offline","context":{"idset":"7993"}}
+{"timestamp":1714080342.2134159,"name":"offline","context":{"idset":"7994"}}
+{"timestamp":1714080342.2161577,"name":"offline","context":{"idset":"7995"}}
+{"timestamp":1714080342.2189891,"name":"offline","context":{"idset":"7996"}}
+{"timestamp":1714080342.22172,"name":"offline","context":{"idset":"7997"}}
+{"timestamp":1714080342.2338557,"name":"offline","context":{"idset":"7998"}}
+{"timestamp":1714080342.2556896,"name":"offline","context":{"idset":"7999"}}
+{"timestamp":1714080342.2584987,"name":"offline","context":{"idset":"8000"}}
+{"timestamp":1714080342.2617106,"name":"offline","context":{"idset":"8001"}}
+{"timestamp":1714080342.2649581,"name":"offline","context":{"idset":"8002"}}
+{"timestamp":1714080342.2677839,"name":"offline","context":{"idset":"8003"}}
+{"timestamp":1714080342.270879,"name":"offline","context":{"idset":"8004"}}
+{"timestamp":1714080342.2737403,"name":"offline","context":{"idset":"8005"}}
+{"timestamp":1714080342.2766342,"name":"offline","context":{"idset":"8006"}}
+{"timestamp":1714080342.2795074,"name":"offline","context":{"idset":"8007"}}
+{"timestamp":1714080342.2818103,"name":"offline","context":{"idset":"8008"}}
+{"timestamp":1714080342.2840109,"name":"offline","context":{"idset":"8009"}}
+{"timestamp":1714080342.3007674,"name":"offline","context":{"idset":"8010"}}
+{"timestamp":1714080342.3036296,"name":"offline","context":{"idset":"8011"}}
+{"timestamp":1714080342.3063588,"name":"offline","context":{"idset":"8012"}}
+{"timestamp":1714080342.3096046,"name":"offline","context":{"idset":"8013"}}
+{"timestamp":1714080342.3122182,"name":"offline","context":{"idset":"8014"}}
+{"timestamp":1714080342.3146887,"name":"offline","context":{"idset":"8015"}}
+{"timestamp":1714080342.3171651,"name":"offline","context":{"idset":"8016"}}
+{"timestamp":1714080342.3194985,"name":"offline","context":{"idset":"8017"}}
+{"timestamp":1714080342.32182,"name":"offline","context":{"idset":"8018"}}
+{"timestamp":1714080342.324358,"name":"offline","context":{"idset":"8019"}}
+{"timestamp":1714080342.3269584,"name":"offline","context":{"idset":"8020"}}
+{"timestamp":1714080342.3298461,"name":"offline","context":{"idset":"8021"}}
+{"timestamp":1714080342.3326731,"name":"offline","context":{"idset":"8022"}}
+{"timestamp":1714080342.3353004,"name":"offline","context":{"idset":"8023"}}
+{"timestamp":1714080342.3379929,"name":"offline","context":{"idset":"8024"}}
+{"timestamp":1714080342.3404746,"name":"offline","context":{"idset":"8025"}}
+{"timestamp":1714080342.343257,"name":"offline","context":{"idset":"8026"}}
+{"timestamp":1714080342.3463235,"name":"offline","context":{"idset":"8027"}}
+{"timestamp":1714080342.3494251,"name":"offline","context":{"idset":"8028"}}
+{"timestamp":1714080342.3521438,"name":"offline","context":{"idset":"8029"}}
+{"timestamp":1714080342.3551741,"name":"offline","context":{"idset":"8030"}}
+{"timestamp":1714080342.3574376,"name":"offline","context":{"idset":"8031"}}
+{"timestamp":1714080342.3599138,"name":"offline","context":{"idset":"8032"}}
+{"timestamp":1714080342.3626766,"name":"offline","context":{"idset":"8033"}}
+{"timestamp":1714080342.3650284,"name":"offline","context":{"idset":"8034"}}
+{"timestamp":1714080342.3676167,"name":"offline","context":{"idset":"8035"}}
+{"timestamp":1714080342.3701649,"name":"offline","context":{"idset":"8036"}}
+{"timestamp":1714080342.3728752,"name":"offline","context":{"idset":"8037"}}
+{"timestamp":1714080342.3753417,"name":"offline","context":{"idset":"8038"}}
+{"timestamp":1714080342.3871894,"name":"offline","context":{"idset":"8039"}}
+{"timestamp":1714080342.3894162,"name":"offline","context":{"idset":"8040"}}
+{"timestamp":1714080342.3916357,"name":"offline","context":{"idset":"8041"}}
+{"timestamp":1714080342.4017611,"name":"offline","context":{"idset":"8042"}}
+{"timestamp":1714080342.4128144,"name":"offline","context":{"idset":"8043"}}
+{"timestamp":1714080342.4148297,"name":"offline","context":{"idset":"8044"}}
+{"timestamp":1714080342.4168246,"name":"offline","context":{"idset":"8045"}}
+{"timestamp":1714080342.4191284,"name":"offline","context":{"idset":"8046"}}
+{"timestamp":1714080342.4306281,"name":"offline","context":{"idset":"8047"}}
+{"timestamp":1714080342.4334049,"name":"offline","context":{"idset":"8048"}}
+{"timestamp":1714080342.4359705,"name":"offline","context":{"idset":"8049"}}
+{"timestamp":1714080342.438786,"name":"offline","context":{"idset":"8050"}}
+{"timestamp":1714080342.4417284,"name":"offline","context":{"idset":"8051"}}
+{"timestamp":1714080342.4446449,"name":"offline","context":{"idset":"8052"}}
+{"timestamp":1714080342.4471672,"name":"offline","context":{"idset":"8053"}}
+{"timestamp":1714080342.449585,"name":"offline","context":{"idset":"8054"}}
+{"timestamp":1714080342.460458,"name":"offline","context":{"idset":"8055"}}
+{"timestamp":1714080342.4630058,"name":"offline","context":{"idset":"8056"}}
+{"timestamp":1714080342.4659095,"name":"offline","context":{"idset":"8057"}}
+{"timestamp":1714080342.468926,"name":"offline","context":{"idset":"8058"}}
+{"timestamp":1714080342.4717355,"name":"offline","context":{"idset":"8059"}}
+{"timestamp":1714080342.4746168,"name":"offline","context":{"idset":"8060"}}
+{"timestamp":1714080342.4855382,"name":"offline","context":{"idset":"8061"}}
+{"timestamp":1714080342.4883776,"name":"offline","context":{"idset":"8062"}}
+{"timestamp":1714080342.4912841,"name":"offline","context":{"idset":"8063"}}
+{"timestamp":1714080342.4941251,"name":"offline","context":{"idset":"8064"}}
+{"timestamp":1714080342.4970944,"name":"offline","context":{"idset":"8065"}}
+{"timestamp":1714080342.4999104,"name":"offline","context":{"idset":"8066"}}
+{"timestamp":1714080342.5023222,"name":"offline","context":{"idset":"8067"}}
+{"timestamp":1714080342.50457,"name":"offline","context":{"idset":"8068"}}
+{"timestamp":1714080342.5070744,"name":"offline","context":{"idset":"8069"}}
+{"timestamp":1714080342.5099599,"name":"offline","context":{"idset":"8070"}}
+{"timestamp":1714080342.5128405,"name":"offline","context":{"idset":"8071"}}
+{"timestamp":1714080342.5254548,"name":"offline","context":{"idset":"8072"}}
+{"timestamp":1714080342.5280995,"name":"offline","context":{"idset":"8073"}}
+{"timestamp":1714080342.5305805,"name":"offline","context":{"idset":"8074"}}
+{"timestamp":1714080342.5331535,"name":"offline","context":{"idset":"8075"}}
+{"timestamp":1714080342.5357096,"name":"offline","context":{"idset":"8076"}}
+{"timestamp":1714080342.5383368,"name":"offline","context":{"idset":"8077"}}
+{"timestamp":1714080342.5408893,"name":"offline","context":{"idset":"8078"}}
+{"timestamp":1714080342.5435262,"name":"offline","context":{"idset":"8079"}}
+{"timestamp":1714080342.5493371,"name":"offline","context":{"idset":"8080"}}
+{"timestamp":1714080342.5523789,"name":"offline","context":{"idset":"8081"}}
+{"timestamp":1714080342.5633502,"name":"offline","context":{"idset":"8082"}}
+{"timestamp":1714080342.5654905,"name":"offline","context":{"idset":"8083"}}
+{"timestamp":1714080342.5678146,"name":"offline","context":{"idset":"8084"}}
+{"timestamp":1714080342.5704496,"name":"offline","context":{"idset":"8085"}}
+{"timestamp":1714080342.5729823,"name":"offline","context":{"idset":"8086"}}
+{"timestamp":1714080342.5754626,"name":"offline","context":{"idset":"8087"}}
+{"timestamp":1714080342.578167,"name":"offline","context":{"idset":"8088"}}
+{"timestamp":1714080342.5958953,"name":"offline","context":{"idset":"8089"}}
+{"timestamp":1714080342.597934,"name":"offline","context":{"idset":"8090"}}
+{"timestamp":1714080342.5999207,"name":"offline","context":{"idset":"8091"}}
+{"timestamp":1714080342.6020162,"name":"offline","context":{"idset":"8092"}}
+{"timestamp":1714080342.6039741,"name":"offline","context":{"idset":"8093"}}
+{"timestamp":1714080342.605911,"name":"offline","context":{"idset":"8094"}}
+{"timestamp":1714080342.6079299,"name":"offline","context":{"idset":"8095"}}
+{"timestamp":1714080342.6176579,"name":"offline","context":{"idset":"8096"}}
+{"timestamp":1714080342.6196144,"name":"offline","context":{"idset":"8097"}}
+{"timestamp":1714080342.6215508,"name":"offline","context":{"idset":"8098"}}
+{"timestamp":1714080342.6239884,"name":"offline","context":{"idset":"8099"}}
+{"timestamp":1714080342.6259444,"name":"offline","context":{"idset":"8100"}}
+{"timestamp":1714080342.6279373,"name":"offline","context":{"idset":"8117"}}
+{"timestamp":1714080342.6298897,"name":"offline","context":{"idset":"8118"}}
+{"timestamp":1714080342.632077,"name":"offline","context":{"idset":"8119"}}
+{"timestamp":1714080342.642453,"name":"offline","context":{"idset":"8120"}}
+{"timestamp":1714080342.6439695,"name":"offline","context":{"idset":"8121"}}
+{"timestamp":1714080342.6454546,"name":"offline","context":{"idset":"8122"}}
+{"timestamp":1714080342.6469023,"name":"offline","context":{"idset":"8123"}}
+{"timestamp":1714080342.6483991,"name":"offline","context":{"idset":"8124"}}
+{"timestamp":1714080342.6502533,"name":"offline","context":{"idset":"8125"}}
+{"timestamp":1714080342.6591914,"name":"offline","context":{"idset":"8126"}}
+{"timestamp":1714080342.6611924,"name":"offline","context":{"idset":"8127"}}
+{"timestamp":1714080342.6633997,"name":"offline","context":{"idset":"8128"}}
+{"timestamp":1714080342.6655509,"name":"offline","context":{"idset":"8129"}}
+{"timestamp":1714080342.6675665,"name":"offline","context":{"idset":"8130"}}
+{"timestamp":1714080342.6698835,"name":"offline","context":{"idset":"8131"}}
+{"timestamp":1714080342.6722069,"name":"offline","context":{"idset":"8132"}}
+{"timestamp":1714080342.6747553,"name":"offline","context":{"idset":"8133"}}
+{"timestamp":1714080342.6831522,"name":"offline","context":{"idset":"8134"}}
+{"timestamp":1714080342.6847515,"name":"offline","context":{"idset":"8135"}}
+{"timestamp":1714080342.6871183,"name":"offline","context":{"idset":"8136"}}
+{"timestamp":1714080342.6897011,"name":"offline","context":{"idset":"8137"}}
+{"timestamp":1714080342.6921093,"name":"offline","context":{"idset":"8138"}}
+{"timestamp":1714080342.6941509,"name":"offline","context":{"idset":"8139"}}
+{"timestamp":1714080342.6960602,"name":"offline","context":{"idset":"8140"}}
+{"timestamp":1714080342.6980071,"name":"offline","context":{"idset":"8141"}}
+{"timestamp":1714080342.6999378,"name":"offline","context":{"idset":"8142"}}
+{"timestamp":1714080342.7018363,"name":"offline","context":{"idset":"8143"}}
+{"timestamp":1714080342.708019,"name":"offline","context":{"idset":"8144"}}
+{"timestamp":1714080342.7099514,"name":"offline","context":{"idset":"8145"}}
+{"timestamp":1714080342.7119379,"name":"offline","context":{"idset":"8146"}}
+{"timestamp":1714080342.7211895,"name":"offline","context":{"idset":"8147"}}
+{"timestamp":1714080342.7231731,"name":"offline","context":{"idset":"8148"}}
+{"timestamp":1714080342.7251463,"name":"offline","context":{"idset":"8149"}}
+{"timestamp":1714080342.7344246,"name":"offline","context":{"idset":"8150"}}
+{"timestamp":1714080342.744025,"name":"offline","context":{"idset":"8151"}}
+{"timestamp":1714080342.7460859,"name":"offline","context":{"idset":"8152"}}
+{"timestamp":1714080342.7480609,"name":"offline","context":{"idset":"8153"}}
+{"timestamp":1714080342.7500751,"name":"offline","context":{"idset":"8154"}}
+{"timestamp":1714080342.7520468,"name":"offline","context":{"idset":"8155"}}
+{"timestamp":1714080342.7542872,"name":"offline","context":{"idset":"8156"}}
+{"timestamp":1714080342.7563329,"name":"offline","context":{"idset":"8157"}}
+{"timestamp":1714080342.7585695,"name":"offline","context":{"idset":"8158"}}
+{"timestamp":1714080342.7696815,"name":"offline","context":{"idset":"8159"}}
+{"timestamp":1714080342.7720797,"name":"offline","context":{"idset":"8160"}}
+{"timestamp":1714080342.7742472,"name":"offline","context":{"idset":"8161"}}
+{"timestamp":1714080342.7761614,"name":"offline","context":{"idset":"8162"}}
+{"timestamp":1714080342.7780471,"name":"offline","context":{"idset":"8163"}}
+{"timestamp":1714080342.7799551,"name":"offline","context":{"idset":"8164"}}
+{"timestamp":1714080342.7820086,"name":"offline","context":{"idset":"8169"}}
+{"timestamp":1714080342.7839048,"name":"offline","context":{"idset":"8170"}}
+{"timestamp":1714080342.793129,"name":"offline","context":{"idset":"8171"}}
+{"timestamp":1714080342.79563,"name":"offline","context":{"idset":"8172"}}
+{"timestamp":1714080342.7981615,"name":"offline","context":{"idset":"8173"}}
+{"timestamp":1714080342.8002224,"name":"offline","context":{"idset":"8174"}}
+{"timestamp":1714080342.802676,"name":"offline","context":{"idset":"8175"}}
+{"timestamp":1714080342.8053782,"name":"offline","context":{"idset":"8176"}}
+{"timestamp":1714080342.8081262,"name":"offline","context":{"idset":"8177"}}
+{"timestamp":1714080342.8107607,"name":"offline","context":{"idset":"8178"}}
+{"timestamp":1714080342.8129871,"name":"offline","context":{"idset":"8179"}}
+{"timestamp":1714080342.8151245,"name":"offline","context":{"idset":"8180"}}
+{"timestamp":1714080342.8172019,"name":"offline","context":{"idset":"8181"}}
+{"timestamp":1714080342.8195925,"name":"offline","context":{"idset":"8182"}}
+{"timestamp":1714080342.8219295,"name":"offline","context":{"idset":"8183"}}
+{"timestamp":1714080342.8239179,"name":"offline","context":{"idset":"8184"}}
+{"timestamp":1714080342.8257995,"name":"offline","context":{"idset":"8185"}}
+{"timestamp":1714080342.8279347,"name":"offline","context":{"idset":"8186"}}
+{"timestamp":1714080342.8300498,"name":"offline","context":{"idset":"8187"}}
+{"timestamp":1714080342.8323619,"name":"offline","context":{"idset":"8188"}}
+{"timestamp":1714080342.8345282,"name":"offline","context":{"idset":"8189"}}
+{"timestamp":1714080342.8366985,"name":"offline","context":{"idset":"8190"}}
+{"timestamp":1714080342.839184,"name":"offline","context":{"idset":"8191"}}
+{"timestamp":1714080342.8413725,"name":"offline","context":{"idset":"8192"}}
+{"timestamp":1714080342.8435307,"name":"offline","context":{"idset":"8193"}}
+{"timestamp":1714080342.8456473,"name":"offline","context":{"idset":"8194"}}
+{"timestamp":1714080342.8478618,"name":"offline","context":{"idset":"8195"}}
+{"timestamp":1714080342.85025,"name":"offline","context":{"idset":"8196"}}
+{"timestamp":1714080342.8521643,"name":"offline","context":{"idset":"8197"}}
+{"timestamp":1714080342.8540299,"name":"offline","context":{"idset":"8198"}}
+{"timestamp":1714080342.8561957,"name":"offline","context":{"idset":"8199"}}
+{"timestamp":1714080342.8582966,"name":"offline","context":{"idset":"8200"}}
+{"timestamp":1714080342.860369,"name":"offline","context":{"idset":"8202"}}
+{"timestamp":1714080342.8622141,"name":"offline","context":{"idset":"8203"}}
+{"timestamp":1714080342.8641751,"name":"offline","context":{"idset":"8204"}}
+{"timestamp":1714080342.866261,"name":"offline","context":{"idset":"8205"}}
+{"timestamp":1714080342.8681006,"name":"offline","context":{"idset":"8206"}}
+{"timestamp":1714080342.8702085,"name":"offline","context":{"idset":"8207"}}
+{"timestamp":1714080342.8724082,"name":"offline","context":{"idset":"8208"}}
+{"timestamp":1714080342.8743026,"name":"offline","context":{"idset":"8209"}}
+{"timestamp":1714080342.8842893,"name":"offline","context":{"idset":"8210"}}
+{"timestamp":1714080342.8862376,"name":"offline","context":{"idset":"8211"}}
+{"timestamp":1714080342.8889384,"name":"offline","context":{"idset":"8212"}}
+{"timestamp":1714080342.8916678,"name":"offline","context":{"idset":"8213"}}
+{"timestamp":1714080342.8943164,"name":"offline","context":{"idset":"8214"}}
+{"timestamp":1714080342.897095,"name":"offline","context":{"idset":"8215"}}
+{"timestamp":1714080342.8998818,"name":"offline","context":{"idset":"8216"}}
+{"timestamp":1714080342.9028599,"name":"offline","context":{"idset":"8217"}}
+{"timestamp":1714080342.9058266,"name":"offline","context":{"idset":"8218"}}
+{"timestamp":1714080342.9090152,"name":"offline","context":{"idset":"8219"}}
+{"timestamp":1714080342.9116824,"name":"offline","context":{"idset":"8220"}}
+{"timestamp":1714080342.914454,"name":"offline","context":{"idset":"8221"}}
+{"timestamp":1714080342.9170692,"name":"offline","context":{"idset":"8222"}}
+{"timestamp":1714080342.9365652,"name":"offline","context":{"idset":"8223"}}
+{"timestamp":1714080342.9391706,"name":"offline","context":{"idset":"8224"}}
+{"timestamp":1714080342.9418402,"name":"offline","context":{"idset":"8225"}}
+{"timestamp":1714080342.9444058,"name":"offline","context":{"idset":"8226"}}
+{"timestamp":1714080342.9469693,"name":"offline","context":{"idset":"8227"}}
+{"timestamp":1714080342.9495451,"name":"offline","context":{"idset":"8228"}}
+{"timestamp":1714080342.9518697,"name":"offline","context":{"idset":"8229"}}
+{"timestamp":1714080342.9546318,"name":"offline","context":{"idset":"8230"}}
+{"timestamp":1714080342.9570208,"name":"offline","context":{"idset":"8231"}}
+{"timestamp":1714080342.9719751,"name":"offline","context":{"idset":"8232"}}
+{"timestamp":1714080342.9846561,"name":"offline","context":{"idset":"8235"}}
+{"timestamp":1714080342.9872434,"name":"offline","context":{"idset":"8236"}}
+{"timestamp":1714080342.9905322,"name":"offline","context":{"idset":"8237"}}
+{"timestamp":1714080342.9934866,"name":"offline","context":{"idset":"8238"}}
+{"timestamp":1714080342.9961948,"name":"offline","context":{"idset":"8239"}}
+{"timestamp":1714080342.9987261,"name":"offline","context":{"idset":"8240"}}
+{"timestamp":1714080343.0012815,"name":"offline","context":{"idset":"8241"}}
+{"timestamp":1714080343.0038664,"name":"offline","context":{"idset":"8242"}}
+{"timestamp":1714080343.0063839,"name":"offline","context":{"idset":"8243"}}
+{"timestamp":1714080343.0090184,"name":"offline","context":{"idset":"8244"}}
+{"timestamp":1714080343.0117278,"name":"offline","context":{"idset":"8245"}}
+{"timestamp":1714080343.0142791,"name":"offline","context":{"idset":"8246"}}
+{"timestamp":1714080343.0323927,"name":"offline","context":{"idset":"8247"}}
+{"timestamp":1714080343.0422723,"name":"offline","context":{"idset":"8248"}}
+{"timestamp":1714080343.0447559,"name":"offline","context":{"idset":"8249"}}
+{"timestamp":1714080343.0472319,"name":"offline","context":{"idset":"8250"}}
+{"timestamp":1714080343.0497248,"name":"offline","context":{"idset":"8251"}}
+{"timestamp":1714080343.0684705,"name":"offline","context":{"idset":"8252"}}
+{"timestamp":1714080343.0704701,"name":"offline","context":{"idset":"8253"}}
+{"timestamp":1714080343.0732403,"name":"offline","context":{"idset":"8254"}}
+{"timestamp":1714080343.0762932,"name":"offline","context":{"idset":"8255"}}
+{"timestamp":1714080343.0790634,"name":"offline","context":{"idset":"8256"}}
+{"timestamp":1714080343.0821247,"name":"offline","context":{"idset":"8257"}}
+{"timestamp":1714080343.0848477,"name":"offline","context":{"idset":"8258"}}
+{"timestamp":1714080343.0873168,"name":"offline","context":{"idset":"8259"}}
+{"timestamp":1714080343.0898349,"name":"offline","context":{"idset":"8260"}}
+{"timestamp":1714080343.0927212,"name":"offline","context":{"idset":"8261"}}
+{"timestamp":1714080343.1032734,"name":"offline","context":{"idset":"8262"}}
+{"timestamp":1714080343.1050677,"name":"offline","context":{"idset":"8263"}}
+{"timestamp":1714080343.1068518,"name":"offline","context":{"idset":"8264"}}
+{"timestamp":1714080343.108654,"name":"offline","context":{"idset":"8265"}}
+{"timestamp":1714080343.1104403,"name":"offline","context":{"idset":"8266"}}
+{"timestamp":1714080343.112905,"name":"offline","context":{"idset":"8267"}}
+{"timestamp":1714080343.1156888,"name":"offline","context":{"idset":"8268"}}
+{"timestamp":1714080343.1184263,"name":"offline","context":{"idset":"8269"}}
+{"timestamp":1714080343.1210675,"name":"offline","context":{"idset":"8270"}}
+{"timestamp":1714080343.1236908,"name":"offline","context":{"idset":"8271"}}
+{"timestamp":1714080343.1261504,"name":"offline","context":{"idset":"8272"}}
+{"timestamp":1714080343.1287682,"name":"offline","context":{"idset":"8273"}}
+{"timestamp":1714080343.1315539,"name":"offline","context":{"idset":"8274"}}
+{"timestamp":1714080343.1341603,"name":"offline","context":{"idset":"8275"}}
+{"timestamp":1714080343.1366184,"name":"offline","context":{"idset":"8276"}}
+{"timestamp":1714080343.1390717,"name":"offline","context":{"idset":"8277"}}
+{"timestamp":1714080343.1415288,"name":"offline","context":{"idset":"8278"}}
+{"timestamp":1714080343.1439874,"name":"offline","context":{"idset":"8279"}}
+{"timestamp":1714080343.1464396,"name":"offline","context":{"idset":"8280"}}
+{"timestamp":1714080343.1489017,"name":"offline","context":{"idset":"8281"}}
+{"timestamp":1714080343.1513436,"name":"offline","context":{"idset":"8282"}}
+{"timestamp":1714080343.1604776,"name":"offline","context":{"idset":"8283"}}
+{"timestamp":1714080343.1621296,"name":"offline","context":{"idset":"8284"}}
+{"timestamp":1714080343.1638896,"name":"offline","context":{"idset":"8285"}}
+{"timestamp":1714080343.1653106,"name":"offline","context":{"idset":"8286"}}
+{"timestamp":1714080343.1672268,"name":"offline","context":{"idset":"8287"}}
+{"timestamp":1714080343.169076,"name":"offline","context":{"idset":"8288"}}
+{"timestamp":1714080343.170342,"name":"offline","context":{"idset":"8289"}}
+{"timestamp":1714080343.1716311,"name":"offline","context":{"idset":"8290"}}
+{"timestamp":1714080343.1729021,"name":"offline","context":{"idset":"8291"}}
+{"timestamp":1714080343.1794138,"name":"offline","context":{"idset":"8292"}}
+{"timestamp":1714080343.1817763,"name":"offline","context":{"idset":"8293"}}
+{"timestamp":1714080343.1841855,"name":"offline","context":{"idset":"8294"}}
+{"timestamp":1714080343.1857429,"name":"offline","context":{"idset":"8295"}}
+{"timestamp":1714080343.1869807,"name":"offline","context":{"idset":"8296"}}
+{"timestamp":1714080343.1882293,"name":"offline","context":{"idset":"8297"}}
+{"timestamp":1714080343.1894832,"name":"offline","context":{"idset":"8298"}}
+{"timestamp":1714080343.1907377,"name":"offline","context":{"idset":"8299"}}
+{"timestamp":1714080343.1920002,"name":"offline","context":{"idset":"8300"}}
+{"timestamp":1714080343.1932597,"name":"offline","context":{"idset":"8301"}}
+{"timestamp":1714080343.1945264,"name":"offline","context":{"idset":"8302"}}
+{"timestamp":1714080343.1957667,"name":"offline","context":{"idset":"8303"}}
+{"timestamp":1714080343.202035,"name":"offline","context":{"idset":"8305"}}
+{"timestamp":1714080343.2033608,"name":"offline","context":{"idset":"8306"}}
+{"timestamp":1714080343.2046454,"name":"offline","context":{"idset":"8307"}}
+{"timestamp":1714080343.2059388,"name":"offline","context":{"idset":"8308"}}
+{"timestamp":1714080343.2072084,"name":"offline","context":{"idset":"8309"}}
+{"timestamp":1714080343.208488,"name":"offline","context":{"idset":"8310"}}
+{"timestamp":1714080343.2097631,"name":"offline","context":{"idset":"8311"}}
+{"timestamp":1714080343.2110364,"name":"offline","context":{"idset":"8312"}}
+{"timestamp":1714080343.2123044,"name":"offline","context":{"idset":"8313"}}
+{"timestamp":1714080343.2137175,"name":"offline","context":{"idset":"8314"}}
+{"timestamp":1714080343.214999,"name":"offline","context":{"idset":"8315"}}
+{"timestamp":1714080343.2216835,"name":"offline","context":{"idset":"8316"}}
+{"timestamp":1714080343.2229185,"name":"offline","context":{"idset":"8317"}}
+{"timestamp":1714080343.2242556,"name":"offline","context":{"idset":"8318"}}
+{"timestamp":1714080343.2255144,"name":"offline","context":{"idset":"8319"}}
+{"timestamp":1714080343.2269318,"name":"offline","context":{"idset":"8320"}}
+{"timestamp":1714080343.2283337,"name":"offline","context":{"idset":"8321"}}
+{"timestamp":1714080343.2349749,"name":"offline","context":{"idset":"8322"}}
+{"timestamp":1714080343.2363904,"name":"offline","context":{"idset":"8323"}}
+{"timestamp":1714080343.237958,"name":"offline","context":{"idset":"8324"}}
+{"timestamp":1714080343.2392473,"name":"offline","context":{"idset":"8325"}}
+{"timestamp":1714080343.246237,"name":"offline","context":{"idset":"8326"}}
+{"timestamp":1714080343.247582,"name":"offline","context":{"idset":"8327"}}
+{"timestamp":1714080343.2489018,"name":"offline","context":{"idset":"8328"}}
+{"timestamp":1714080343.2555518,"name":"offline","context":{"idset":"8329"}}
+{"timestamp":1714080343.2569513,"name":"offline","context":{"idset":"8330"}}
+{"timestamp":1714080343.2583828,"name":"offline","context":{"idset":"8331"}}
+{"timestamp":1714080343.2598834,"name":"offline","context":{"idset":"8332"}}
+{"timestamp":1714080343.2612147,"name":"offline","context":{"idset":"8333"}}
+{"timestamp":1714080343.2626615,"name":"offline","context":{"idset":"8334"}}
+{"timestamp":1714080343.2641392,"name":"offline","context":{"idset":"8335"}}
+{"timestamp":1714080343.2654729,"name":"offline","context":{"idset":"8336"}}
+{"timestamp":1714080343.2669752,"name":"offline","context":{"idset":"8337"}}
+{"timestamp":1714080343.2684457,"name":"offline","context":{"idset":"8338"}}
+{"timestamp":1714080343.2698529,"name":"offline","context":{"idset":"8339"}}
+{"timestamp":1714080343.2711668,"name":"offline","context":{"idset":"8340"}}
+{"timestamp":1714080343.2725725,"name":"offline","context":{"idset":"8341"}}
+{"timestamp":1714080343.2739353,"name":"offline","context":{"idset":"8342"}}
+{"timestamp":1714080343.2752972,"name":"offline","context":{"idset":"8343"}}
+{"timestamp":1714080343.2820466,"name":"offline","context":{"idset":"8344"}}
+{"timestamp":1714080343.2833304,"name":"offline","context":{"idset":"8345"}}
+{"timestamp":1714080343.2847919,"name":"offline","context":{"idset":"8346"}}
+{"timestamp":1714080343.2862289,"name":"offline","context":{"idset":"8347"}}
+{"timestamp":1714080343.2875249,"name":"offline","context":{"idset":"8348"}}
+{"timestamp":1714080343.2889407,"name":"offline","context":{"idset":"8349"}}
+{"timestamp":1714080343.2957311,"name":"offline","context":{"idset":"8350"}}
+{"timestamp":1714080343.2970119,"name":"offline","context":{"idset":"8351"}}
+{"timestamp":1714080343.2982895,"name":"offline","context":{"idset":"8352"}}
+{"timestamp":1714080343.2996831,"name":"offline","context":{"idset":"8353"}}
+{"timestamp":1714080343.3009779,"name":"offline","context":{"idset":"8354"}}
+{"timestamp":1714080343.3024507,"name":"offline","context":{"idset":"8355"}}
+{"timestamp":1714080343.3037205,"name":"offline","context":{"idset":"8356"}}
+{"timestamp":1714080343.3121092,"name":"offline","context":{"idset":"8357"}}
+{"timestamp":1714080343.3133919,"name":"offline","context":{"idset":"8358"}}
+{"timestamp":1714080343.3147011,"name":"offline","context":{"idset":"8359"}}
+{"timestamp":1714080343.3159788,"name":"offline","context":{"idset":"8360"}}
+{"timestamp":1714080343.3174171,"name":"offline","context":{"idset":"8361"}}
+{"timestamp":1714080343.3187449,"name":"offline","context":{"idset":"8362"}}
+{"timestamp":1714080343.3201506,"name":"offline","context":{"idset":"8363"}}
+{"timestamp":1714080343.3269796,"name":"offline","context":{"idset":"8364"}}
+{"timestamp":1714080343.3282452,"name":"offline","context":{"idset":"8365"}}
+{"timestamp":1714080343.329618,"name":"offline","context":{"idset":"8366"}}
+{"timestamp":1714080343.330889,"name":"offline","context":{"idset":"8367"}}
+{"timestamp":1714080343.3321731,"name":"offline","context":{"idset":"8368"}}
+{"timestamp":1714080343.3334787,"name":"offline","context":{"idset":"8369"}}
+{"timestamp":1714080343.3349342,"name":"offline","context":{"idset":"8370"}}
+{"timestamp":1714080343.3366477,"name":"offline","context":{"idset":"8371"}}
+{"timestamp":1714080343.3445194,"name":"offline","context":{"idset":"8372"}}
+{"timestamp":1714080343.3457828,"name":"offline","context":{"idset":"8373"}}
+{"timestamp":1714080343.3470609,"name":"offline","context":{"idset":"8374"}}
+{"timestamp":1714080343.3483133,"name":"offline","context":{"idset":"8375"}}
+{"timestamp":1714080343.3496652,"name":"offline","context":{"idset":"8376"}}
+{"timestamp":1714080343.3510423,"name":"offline","context":{"idset":"8377"}}
+{"timestamp":1714080343.3522923,"name":"offline","context":{"idset":"8378"}}
+{"timestamp":1714080343.35356,"name":"offline","context":{"idset":"8379"}}
+{"timestamp":1714080343.3601251,"name":"offline","context":{"idset":"8380"}}
+{"timestamp":1714080343.3613973,"name":"offline","context":{"idset":"8381"}}
+{"timestamp":1714080343.3626945,"name":"offline","context":{"idset":"8382"}}
+{"timestamp":1714080343.3640568,"name":"offline","context":{"idset":"8383"}}
+{"timestamp":1714080343.3653307,"name":"offline","context":{"idset":"8384"}}
+{"timestamp":1714080343.3715689,"name":"offline","context":{"idset":"8385"}}
+{"timestamp":1714080343.373862,"name":"offline","context":{"idset":"8386"}}
+{"timestamp":1714080343.3760686,"name":"offline","context":{"idset":"8387"}}
+{"timestamp":1714080343.3784714,"name":"offline","context":{"idset":"8388"}}
+{"timestamp":1714080343.3807728,"name":"offline","context":{"idset":"8389"}}
+{"timestamp":1714080343.383311,"name":"offline","context":{"idset":"8390"}}
+{"timestamp":1714080343.3855691,"name":"offline","context":{"idset":"8391"}}
+{"timestamp":1714080343.3878539,"name":"offline","context":{"idset":"8392"}}
+{"timestamp":1714080343.39012,"name":"offline","context":{"idset":"8393"}}
+{"timestamp":1714080343.3924139,"name":"offline","context":{"idset":"8394"}}
+{"timestamp":1714080343.4008739,"name":"offline","context":{"idset":"8395"}}
+{"timestamp":1714080343.4031494,"name":"offline","context":{"idset":"8396"}}
+{"timestamp":1714080343.4053831,"name":"offline","context":{"idset":"8397"}}
+{"timestamp":1714080343.4076669,"name":"offline","context":{"idset":"8398"}}
+{"timestamp":1714080343.4098842,"name":"offline","context":{"idset":"8399"}}
+{"timestamp":1714080343.4186399,"name":"offline","context":{"idset":"8400"}}
+{"timestamp":1714080343.4209185,"name":"offline","context":{"idset":"8401"}}
+{"timestamp":1714080343.4231474,"name":"offline","context":{"idset":"8402"}}
+{"timestamp":1714080343.4251471,"name":"offline","context":{"idset":"8403"}}
+{"timestamp":1714080343.4264178,"name":"offline","context":{"idset":"8404"}}
+{"timestamp":1714080343.4276717,"name":"offline","context":{"idset":"8405"}}
+{"timestamp":1714080343.434269,"name":"offline","context":{"idset":"8406"}}
+{"timestamp":1714080343.4355383,"name":"offline","context":{"idset":"8407"}}
+{"timestamp":1714080343.4368763,"name":"offline","context":{"idset":"8409"}}
+{"timestamp":1714080343.4381714,"name":"offline","context":{"idset":"8410"}}
+{"timestamp":1714080343.4395072,"name":"offline","context":{"idset":"8411"}}
+{"timestamp":1714080343.4407401,"name":"offline","context":{"idset":"8412"}}
+{"timestamp":1714080343.4421101,"name":"offline","context":{"idset":"8413"}}
+{"timestamp":1714080343.4434593,"name":"offline","context":{"idset":"8414"}}
+{"timestamp":1714080343.4447176,"name":"offline","context":{"idset":"8415"}}
+{"timestamp":1714080343.4461532,"name":"offline","context":{"idset":"8416"}}
+{"timestamp":1714080343.4474514,"name":"offline","context":{"idset":"8417"}}
+{"timestamp":1714080343.4488499,"name":"offline","context":{"idset":"8418"}}
+{"timestamp":1714080343.4501042,"name":"offline","context":{"idset":"8419"}}
+{"timestamp":1714080343.4515231,"name":"offline","context":{"idset":"8420"}}
+{"timestamp":1714080343.4529395,"name":"offline","context":{"idset":"8421"}}
+{"timestamp":1714080343.4541926,"name":"offline","context":{"idset":"8422"}}
+{"timestamp":1714080343.4555514,"name":"offline","context":{"idset":"8423"}}
+{"timestamp":1714080343.4567914,"name":"offline","context":{"idset":"8424"}}
+{"timestamp":1714080343.4582338,"name":"offline","context":{"idset":"8425"}}
+{"timestamp":1714080343.4595785,"name":"offline","context":{"idset":"8426"}}
+{"timestamp":1714080343.4609373,"name":"offline","context":{"idset":"8427"}}
+{"timestamp":1714080343.4623141,"name":"offline","context":{"idset":"8428"}}
+{"timestamp":1714080343.4690642,"name":"offline","context":{"idset":"8429"}}
+{"timestamp":1714080343.4704051,"name":"offline","context":{"idset":"8430"}}
+{"timestamp":1714080343.4717963,"name":"offline","context":{"idset":"8432"}}
+{"timestamp":1714080343.4730852,"name":"offline","context":{"idset":"8433"}}
+{"timestamp":1714080343.4743972,"name":"offline","context":{"idset":"8434"}}
+{"timestamp":1714080343.4756646,"name":"offline","context":{"idset":"8435"}}
+{"timestamp":1714080343.482703,"name":"offline","context":{"idset":"8436"}}
+{"timestamp":1714080343.4844477,"name":"offline","context":{"idset":"8597"}}
+{"timestamp":1714080343.4861741,"name":"offline","context":{"idset":"8598"}}
+{"timestamp":1714080343.4879155,"name":"offline","context":{"idset":"8599"}}
+{"timestamp":1714080343.4896529,"name":"offline","context":{"idset":"8600"}}
+{"timestamp":1714080343.4985874,"name":"offline","context":{"idset":"8601"}}
+{"timestamp":1714080343.5003307,"name":"offline","context":{"idset":"8602"}}
+{"timestamp":1714080343.5020804,"name":"offline","context":{"idset":"8603"}}
+{"timestamp":1714080343.5038135,"name":"offline","context":{"idset":"8604"}}
+{"timestamp":1714080343.5130742,"name":"offline","context":{"idset":"8605"}}
+{"timestamp":1714080343.5148685,"name":"offline","context":{"idset":"8606"}}
+{"timestamp":1714080343.516602,"name":"offline","context":{"idset":"8607"}}
+{"timestamp":1714080343.5183377,"name":"offline","context":{"idset":"8608"}}
+{"timestamp":1714080343.5200753,"name":"offline","context":{"idset":"8609"}}
+{"timestamp":1714080343.5217791,"name":"offline","context":{"idset":"8610"}}
+{"timestamp":1714080343.5234923,"name":"offline","context":{"idset":"8611"}}
+{"timestamp":1714080343.5255258,"name":"offline","context":{"idset":"8612"}}
+{"timestamp":1714080343.5272253,"name":"offline","context":{"idset":"8821"}}
+{"timestamp":1714080343.5289102,"name":"offline","context":{"idset":"8822"}}
+{"timestamp":1714080343.5305924,"name":"offline","context":{"idset":"8823"}}
+{"timestamp":1714080343.5322845,"name":"offline","context":{"idset":"8824"}}
+{"timestamp":1714080343.5338392,"name":"offline","context":{"idset":"8825"}}
+{"timestamp":1714080343.5355654,"name":"offline","context":{"idset":"8826"}}
+{"timestamp":1714080343.5376954,"name":"offline","context":{"idset":"8827"}}
+{"timestamp":1714080343.5399187,"name":"offline","context":{"idset":"8828"}}
+{"timestamp":1714080343.5419636,"name":"offline","context":{"idset":"8829"}}
+{"timestamp":1714080343.5436904,"name":"offline","context":{"idset":"8830"}}
+{"timestamp":1714080343.5454507,"name":"offline","context":{"idset":"8831"}}
+{"timestamp":1714080343.547179,"name":"offline","context":{"idset":"8832"}}
+{"timestamp":1714080343.5491321,"name":"offline","context":{"idset":"8834"}}
+{"timestamp":1714080343.5513353,"name":"offline","context":{"idset":"8835"}}
+{"timestamp":1714080343.5535336,"name":"offline","context":{"idset":"8836"}}
+{"timestamp":1714080343.5553665,"name":"offline","context":{"idset":"8837"}}
+{"timestamp":1714080343.557209,"name":"offline","context":{"idset":"8838"}}
+{"timestamp":1714080343.559072,"name":"offline","context":{"idset":"8839"}}
+{"timestamp":1714080343.5612643,"name":"offline","context":{"idset":"8840"}}
+{"timestamp":1714080343.5634186,"name":"offline","context":{"idset":"8841"}}
+{"timestamp":1714080343.565335,"name":"offline","context":{"idset":"8842"}}
+{"timestamp":1714080343.5755394,"name":"offline","context":{"idset":"8843"}}
+{"timestamp":1714080343.5775588,"name":"offline","context":{"idset":"8844"}}
+{"timestamp":1714080343.5797715,"name":"offline","context":{"idset":"8845"}}
+{"timestamp":1714080343.5819151,"name":"offline","context":{"idset":"8846"}}
+{"timestamp":1714080343.5841222,"name":"offline","context":{"idset":"8847"}}
+{"timestamp":1714080343.5862832,"name":"offline","context":{"idset":"8848"}}
+{"timestamp":1714080343.5884943,"name":"offline","context":{"idset":"8849"}}
+{"timestamp":1714080343.5972402,"name":"offline","context":{"idset":"8850"}}
+{"timestamp":1714080343.5994501,"name":"offline","context":{"idset":"8851"}}
+{"timestamp":1714080343.6015182,"name":"offline","context":{"idset":"8852"}}
+{"timestamp":1714080343.6037035,"name":"offline","context":{"idset":"8854"}}
+{"timestamp":1714080343.6058559,"name":"offline","context":{"idset":"8855"}}
+{"timestamp":1714080343.6080327,"name":"offline","context":{"idset":"8856"}}
+{"timestamp":1714080343.6101885,"name":"offline","context":{"idset":"8857"}}
+{"timestamp":1714080343.6123793,"name":"offline","context":{"idset":"8858"}}
+{"timestamp":1714080343.6219733,"name":"offline","context":{"idset":"8859"}}
+{"timestamp":1714080343.623662,"name":"offline","context":{"idset":"8860"}}
+{"timestamp":1714080343.6249166,"name":"offline","context":{"idset":"8861"}}
+{"timestamp":1714080343.6261597,"name":"offline","context":{"idset":"8862"}}
+{"timestamp":1714080343.6273997,"name":"offline","context":{"idset":"8863"}}
+{"timestamp":1714080343.628634,"name":"offline","context":{"idset":"8864"}}
+{"timestamp":1714080343.6298733,"name":"offline","context":{"idset":"8865"}}
+{"timestamp":1714080343.6311181,"name":"offline","context":{"idset":"8866"}}
+{"timestamp":1714080343.6323647,"name":"offline","context":{"idset":"8867"}}
+{"timestamp":1714080343.633595,"name":"offline","context":{"idset":"8868"}}
+{"timestamp":1714080343.634819,"name":"offline","context":{"idset":"8869"}}
+{"timestamp":1714080343.6360331,"name":"offline","context":{"idset":"8870"}}
+{"timestamp":1714080343.6373267,"name":"offline","context":{"idset":"8871"}}
+{"timestamp":1714080343.6385729,"name":"offline","context":{"idset":"8872"}}
+{"timestamp":1714080343.6398237,"name":"offline","context":{"idset":"8873"}}
+{"timestamp":1714080343.6410656,"name":"offline","context":{"idset":"8874"}}
+{"timestamp":1714080343.6423049,"name":"offline","context":{"idset":"8875"}}
+{"timestamp":1714080343.6435487,"name":"offline","context":{"idset":"8876"}}
+{"timestamp":1714080343.6447735,"name":"offline","context":{"idset":"8877"}}
+{"timestamp":1714080343.6460395,"name":"offline","context":{"idset":"8878"}}
+{"timestamp":1714080343.6473093,"name":"offline","context":{"idset":"8879"}}
+{"timestamp":1714080343.6485732,"name":"offline","context":{"idset":"8880"}}
+{"timestamp":1714080343.6551735,"name":"offline","context":{"idset":"8881"}}
+{"timestamp":1714080343.6563799,"name":"offline","context":{"idset":"8882"}}
+{"timestamp":1714080343.6576366,"name":"offline","context":{"idset":"8883"}}
+{"timestamp":1714080343.6588721,"name":"offline","context":{"idset":"8884"}}
+{"timestamp":1714080343.6601248,"name":"offline","context":{"idset":"8885"}}
+{"timestamp":1714080343.6613417,"name":"offline","context":{"idset":"8886"}}
+{"timestamp":1714080343.6625454,"name":"offline","context":{"idset":"8887"}}
+{"timestamp":1714080343.6637158,"name":"offline","context":{"idset":"8888"}}
+{"timestamp":1714080343.6649182,"name":"offline","context":{"idset":"8889"}}
+{"timestamp":1714080343.6660907,"name":"offline","context":{"idset":"8890"}}
+{"timestamp":1714080343.6672695,"name":"offline","context":{"idset":"8891"}}
+{"timestamp":1714080343.6685078,"name":"offline","context":{"idset":"8892"}}
+{"timestamp":1714080343.6697526,"name":"offline","context":{"idset":"8893"}}
+{"timestamp":1714080343.6769211,"name":"offline","context":{"idset":"8894"}}
+{"timestamp":1714080343.6782768,"name":"offline","context":{"idset":"8895"}}
+{"timestamp":1714080343.6799662,"name":"offline","context":{"idset":"8896"}}
+{"timestamp":1714080343.6818306,"name":"offline","context":{"idset":"8897"}}
+{"timestamp":1714080343.6835887,"name":"offline","context":{"idset":"8898"}}
+{"timestamp":1714080343.6853282,"name":"offline","context":{"idset":"8899"}}
+{"timestamp":1714080343.687047,"name":"offline","context":{"idset":"8900"}}
+{"timestamp":1714080343.688472,"name":"offline","context":{"idset":"8901"}}
+{"timestamp":1714080343.6897142,"name":"offline","context":{"idset":"8902"}}
+{"timestamp":1714080343.690959,"name":"offline","context":{"idset":"8903"}}
+{"timestamp":1714080343.6975639,"name":"offline","context":{"idset":"8904"}}
+{"timestamp":1714080343.6988485,"name":"offline","context":{"idset":"8905"}}
+{"timestamp":1714080343.7000666,"name":"offline","context":{"idset":"8906"}}
+{"timestamp":1714080343.7012923,"name":"offline","context":{"idset":"8907"}}
+{"timestamp":1714080343.702512,"name":"offline","context":{"idset":"8908"}}
+{"timestamp":1714080343.7037482,"name":"offline","context":{"idset":"8909"}}
+{"timestamp":1714080343.7052045,"name":"offline","context":{"idset":"8910"}}
+{"timestamp":1714080343.7064226,"name":"offline","context":{"idset":"8911"}}
+{"timestamp":1714080343.7076509,"name":"offline","context":{"idset":"8912"}}
+{"timestamp":1714080343.7088928,"name":"offline","context":{"idset":"8913"}}
+{"timestamp":1714080343.7101326,"name":"offline","context":{"idset":"8914"}}
+{"timestamp":1714080343.7220652,"name":"offline","context":{"idset":"8915"}}
+{"timestamp":1714080343.7232957,"name":"offline","context":{"idset":"8916"}}
+{"timestamp":1714080343.7245603,"name":"offline","context":{"idset":"8917"}}
+{"timestamp":1714080343.7257626,"name":"offline","context":{"idset":"8918"}}
+{"timestamp":1714080343.7269394,"name":"offline","context":{"idset":"8919"}}
+{"timestamp":1714080343.7280941,"name":"offline","context":{"idset":"8920"}}
+{"timestamp":1714080343.7343383,"name":"offline","context":{"idset":"8921"}}
+{"timestamp":1714080343.7355688,"name":"offline","context":{"idset":"8922"}}
+{"timestamp":1714080343.7367728,"name":"offline","context":{"idset":"8923"}}
+{"timestamp":1714080343.7379837,"name":"offline","context":{"idset":"8924"}}
+{"timestamp":1714080343.739212,"name":"offline","context":{"idset":"8925"}}
+{"timestamp":1714080343.7404099,"name":"offline","context":{"idset":"8926"}}
+{"timestamp":1714080343.7469425,"name":"offline","context":{"idset":"8927"}}
+{"timestamp":1714080343.7481487,"name":"offline","context":{"idset":"8928"}}
+{"timestamp":1714080343.7493942,"name":"offline","context":{"idset":"8929"}}
+{"timestamp":1714080343.7506437,"name":"offline","context":{"idset":"8930"}}
+{"timestamp":1714080343.7518835,"name":"offline","context":{"idset":"8931"}}
+{"timestamp":1714080343.7531104,"name":"offline","context":{"idset":"8932"}}
+{"timestamp":1714080343.754324,"name":"offline","context":{"idset":"8933"}}
+{"timestamp":1714080343.7609744,"name":"offline","context":{"idset":"8934"}}
+{"timestamp":1714080343.7621789,"name":"offline","context":{"idset":"8935"}}
+{"timestamp":1714080343.7633786,"name":"offline","context":{"idset":"8936"}}
+{"timestamp":1714080343.7645731,"name":"offline","context":{"idset":"8937"}}
+{"timestamp":1714080343.7710726,"name":"offline","context":{"idset":"8938"}}
+{"timestamp":1714080343.7722566,"name":"offline","context":{"idset":"8939"}}
+{"timestamp":1714080343.7734389,"name":"offline","context":{"idset":"8940"}}
+{"timestamp":1714080343.7745872,"name":"offline","context":{"idset":"8941"}}
+{"timestamp":1714080343.7757256,"name":"offline","context":{"idset":"8942"}}
+{"timestamp":1714080343.7768638,"name":"offline","context":{"idset":"8943"}}
+{"timestamp":1714080343.7787354,"name":"offline","context":{"idset":"8944"}}
+{"timestamp":1714080343.7801712,"name":"offline","context":{"idset":"8945"}}
+{"timestamp":1714080343.7815559,"name":"offline","context":{"idset":"8946"}}
+{"timestamp":1714080343.7829475,"name":"offline","context":{"idset":"8947"}}
+{"timestamp":1714080343.7842994,"name":"offline","context":{"idset":"8948"}}
+{"timestamp":1714080343.7856543,"name":"offline","context":{"idset":"8949"}}
+{"timestamp":1714080343.7933395,"name":"offline","context":{"idset":"8950"}}
+{"timestamp":1714080343.7947767,"name":"offline","context":{"idset":"8951"}}
+{"timestamp":1714080343.7962017,"name":"offline","context":{"idset":"8952"}}
+{"timestamp":1714080343.7976286,"name":"offline","context":{"idset":"8953"}}
+{"timestamp":1714080343.7990606,"name":"offline","context":{"idset":"8954"}}
+{"timestamp":1714080343.8004701,"name":"offline","context":{"idset":"8955"}}
+{"timestamp":1714080343.8019035,"name":"offline","context":{"idset":"8956"}}
+{"timestamp":1714080343.8033106,"name":"offline","context":{"idset":"8957"}}
+{"timestamp":1714080343.8048429,"name":"offline","context":{"idset":"8958"}}
+{"timestamp":1714080343.8135154,"name":"offline","context":{"idset":"8959"}}
+{"timestamp":1714080343.8161488,"name":"offline","context":{"idset":"8960"}}
+{"timestamp":1714080343.8187544,"name":"offline","context":{"idset":"8961"}}
+{"timestamp":1714080343.8213682,"name":"offline","context":{"idset":"8962"}}
+{"timestamp":1714080343.8239768,"name":"offline","context":{"idset":"8963"}}
+{"timestamp":1714080343.8265686,"name":"offline","context":{"idset":"8964"}}
+{"timestamp":1714080343.8291759,"name":"offline","context":{"idset":"8965"}}
+{"timestamp":1714080343.8317742,"name":"offline","context":{"idset":"8966"}}
+{"timestamp":1714080343.8461905,"name":"offline","context":{"idset":"8967"}}
+{"timestamp":1714080343.8487964,"name":"offline","context":{"idset":"8968"}}
+{"timestamp":1714080343.8513899,"name":"offline","context":{"idset":"8969"}}
+{"timestamp":1714080343.8540115,"name":"offline","context":{"idset":"8970"}}
+{"timestamp":1714080343.8566022,"name":"offline","context":{"idset":"8971"}}
+{"timestamp":1714080343.8591988,"name":"offline","context":{"idset":"8972"}}
+{"timestamp":1714080343.8617799,"name":"offline","context":{"idset":"8973"}}
+{"timestamp":1714080343.8762677,"name":"offline","context":{"idset":"8974"}}
+{"timestamp":1714080343.8788567,"name":"offline","context":{"idset":"8975"}}
+{"timestamp":1714080343.881434,"name":"offline","context":{"idset":"8976"}}
+{"timestamp":1714080343.8840454,"name":"offline","context":{"idset":"8977"}}
+{"timestamp":1714080343.8866236,"name":"offline","context":{"idset":"8978"}}
+{"timestamp":1714080343.8892016,"name":"offline","context":{"idset":"8979"}}
+{"timestamp":1714080343.8917794,"name":"offline","context":{"idset":"8980"}}
+{"timestamp":1714080343.8943379,"name":"offline","context":{"idset":"8981"}}
+{"timestamp":1714080343.8969183,"name":"offline","context":{"idset":"8982"}}
+{"timestamp":1714080343.8994937,"name":"offline","context":{"idset":"8983"}}
+{"timestamp":1714080343.902056,"name":"offline","context":{"idset":"8984"}}
+{"timestamp":1714080343.9046171,"name":"offline","context":{"idset":"8985"}}
+{"timestamp":1714080343.9073007,"name":"offline","context":{"idset":"8986"}}
+{"timestamp":1714080343.9098797,"name":"offline","context":{"idset":"8987"}}
+{"timestamp":1714080343.9124408,"name":"offline","context":{"idset":"8988"}}
+{"timestamp":1714080343.9150248,"name":"offline","context":{"idset":"8989"}}
+{"timestamp":1714080343.9175997,"name":"offline","context":{"idset":"8990"}}
+{"timestamp":1714080343.9201648,"name":"offline","context":{"idset":"8991"}}
+{"timestamp":1714080343.9227393,"name":"offline","context":{"idset":"8992"}}
+{"timestamp":1714080343.9371705,"name":"offline","context":{"idset":"8993"}}
+{"timestamp":1714080343.9385467,"name":"offline","context":{"idset":"8994"}}
+{"timestamp":1714080343.939918,"name":"offline","context":{"idset":"8995"}}
+{"timestamp":1714080343.9413142,"name":"offline","context":{"idset":"8996"}}
+{"timestamp":1714080343.9427314,"name":"offline","context":{"idset":"8997"}}
+{"timestamp":1714080343.9441004,"name":"offline","context":{"idset":"8998"}}
+{"timestamp":1714080343.9455085,"name":"offline","context":{"idset":"8999"}}
+{"timestamp":1714080343.9469426,"name":"offline","context":{"idset":"9000"}}
+{"timestamp":1714080343.9485023,"name":"offline","context":{"idset":"9001"}}
+{"timestamp":1714080343.9498847,"name":"offline","context":{"idset":"9002"}}
+{"timestamp":1714080343.9512687,"name":"offline","context":{"idset":"9005"}}
+{"timestamp":1714080343.9664237,"name":"offline","context":{"idset":"9006"}}
+{"timestamp":1714080343.9692137,"name":"offline","context":{"idset":"9007"}}
+{"timestamp":1714080343.9714892,"name":"offline","context":{"idset":"9008"}}
+{"timestamp":1714080343.9734895,"name":"offline","context":{"idset":"9009"}}
+{"timestamp":1714080343.9754829,"name":"offline","context":{"idset":"9010"}}
+{"timestamp":1714080343.9774694,"name":"offline","context":{"idset":"9011"}}
+{"timestamp":1714080343.9794371,"name":"offline","context":{"idset":"9012"}}
+{"timestamp":1714080343.9811819,"name":"offline","context":{"idset":"9013"}}
+{"timestamp":1714080343.9831297,"name":"offline","context":{"idset":"9014"}}
+{"timestamp":1714080343.9851913,"name":"offline","context":{"idset":"9015"}}
+{"timestamp":1714080344.0013239,"name":"offline","context":{"idset":"9016"}}
+{"timestamp":1714080344.0026119,"name":"offline","context":{"idset":"9017"}}
+{"timestamp":1714080344.0037997,"name":"offline","context":{"idset":"9018"}}
+{"timestamp":1714080344.0051401,"name":"offline","context":{"idset":"9019"}}
+{"timestamp":1714080344.0065038,"name":"offline","context":{"idset":"9020"}}
+{"timestamp":1714080344.0078635,"name":"offline","context":{"idset":"9021"}}
+{"timestamp":1714080344.0091836,"name":"offline","context":{"idset":"9022"}}
+{"timestamp":1714080344.0160074,"name":"offline","context":{"idset":"9023"}}
+{"timestamp":1714080344.0225489,"name":"offline","context":{"idset":"9024"}}
+{"timestamp":1714080344.0236952,"name":"offline","context":{"idset":"9025"}}
+{"timestamp":1714080344.0249457,"name":"offline","context":{"idset":"9026"}}
+{"timestamp":1714080344.0260594,"name":"offline","context":{"idset":"9027"}}
+{"timestamp":1714080344.0271463,"name":"offline","context":{"idset":"9028"}}
+{"timestamp":1714080344.0284071,"name":"offline","context":{"idset":"9029"}}
+{"timestamp":1714080344.0295262,"name":"offline","context":{"idset":"9030"}}
+{"timestamp":1714080344.030771,"name":"offline","context":{"idset":"9031"}}
+{"timestamp":1714080344.0319588,"name":"offline","context":{"idset":"9032"}}
+{"timestamp":1714080344.0332754,"name":"offline","context":{"idset":"9033"}}
+{"timestamp":1714080344.034431,"name":"offline","context":{"idset":"9034"}}
+{"timestamp":1714080344.0357661,"name":"offline","context":{"idset":"9035"}}
+{"timestamp":1714080344.0369859,"name":"offline","context":{"idset":"9036"}}
+{"timestamp":1714080344.0380857,"name":"offline","context":{"idset":"9037"}}
+{"timestamp":1714080344.0446723,"name":"offline","context":{"idset":"9038"}}
+{"timestamp":1714080344.0458889,"name":"offline","context":{"idset":"9039"}}
+{"timestamp":1714080344.0469606,"name":"offline","context":{"idset":"9040"}}
+{"timestamp":1714080344.048192,"name":"offline","context":{"idset":"9041"}}
+{"timestamp":1714080344.049489,"name":"offline","context":{"idset":"9042"}}
+{"timestamp":1714080344.0507407,"name":"offline","context":{"idset":"9045"}}
+{"timestamp":1714080344.0518901,"name":"offline","context":{"idset":"9046"}}
+{"timestamp":1714080344.053201,"name":"offline","context":{"idset":"9047"}}
+{"timestamp":1714080344.0543344,"name":"offline","context":{"idset":"9048"}}
+{"timestamp":1714080344.0556345,"name":"offline","context":{"idset":"9049"}}
+{"timestamp":1714080344.0568559,"name":"offline","context":{"idset":"9050"}}
+{"timestamp":1714080344.0579305,"name":"offline","context":{"idset":"9051"}}
+{"timestamp":1714080344.0590155,"name":"offline","context":{"idset":"9052"}}
+{"timestamp":1714080344.0656636,"name":"offline","context":{"idset":"9053"}}
+{"timestamp":1714080344.0668716,"name":"offline","context":{"idset":"9054"}}
+{"timestamp":1714080344.0679326,"name":"offline","context":{"idset":"9055"}}
+{"timestamp":1714080344.0691197,"name":"offline","context":{"idset":"9056"}}
+{"timestamp":1714080344.0758772,"name":"offline","context":{"idset":"9057"}}
+{"timestamp":1714080344.0770564,"name":"offline","context":{"idset":"9058"}}
+{"timestamp":1714080344.0782511,"name":"offline","context":{"idset":"9059"}}
+{"timestamp":1714080344.0793788,"name":"offline","context":{"idset":"9060"}}
+{"timestamp":1714080344.0806434,"name":"offline","context":{"idset":"9069"}}
+{"timestamp":1714080344.0819333,"name":"offline","context":{"idset":"9070"}}
+{"timestamp":1714080344.0832376,"name":"offline","context":{"idset":"9071"}}
+{"timestamp":1714080344.0845349,"name":"offline","context":{"idset":"9072"}}
+{"timestamp":1714080344.0858393,"name":"offline","context":{"idset":"9073"}}
+{"timestamp":1714080344.087146,"name":"offline","context":{"idset":"9074"}}
+{"timestamp":1714080344.0884445,"name":"offline","context":{"idset":"9075"}}
+{"timestamp":1714080344.0897286,"name":"offline","context":{"idset":"9076"}}
+{"timestamp":1714080344.0910192,"name":"offline","context":{"idset":"9077"}}
+{"timestamp":1714080344.0922985,"name":"offline","context":{"idset":"9078"}}
+{"timestamp":1714080344.101522,"name":"offline","context":{"idset":"9079"}}
+{"timestamp":1714080344.1036258,"name":"offline","context":{"idset":"9080"}}
+{"timestamp":1714080344.1056304,"name":"offline","context":{"idset":"9081"}}
+{"timestamp":1714080344.1075361,"name":"offline","context":{"idset":"9082"}}
+{"timestamp":1714080344.1094077,"name":"offline","context":{"idset":"9083"}}
+{"timestamp":1714080344.1111941,"name":"offline","context":{"idset":"9084"}}
+{"timestamp":1714080344.1129746,"name":"offline","context":{"idset":"9085"}}
+{"timestamp":1714080344.1149311,"name":"offline","context":{"idset":"9086"}}
+{"timestamp":1714080344.1168959,"name":"offline","context":{"idset":"9087"}}
+{"timestamp":1714080344.118392,"name":"offline","context":{"idset":"9088"}}
+{"timestamp":1714080344.1196856,"name":"offline","context":{"idset":"9089"}}
+{"timestamp":1714080344.1207421,"name":"offline","context":{"idset":"9090"}}
+{"timestamp":1714080344.1217918,"name":"offline","context":{"idset":"9091"}}
+{"timestamp":1714080344.1228435,"name":"offline","context":{"idset":"9092"}}
+{"timestamp":1714080344.1238906,"name":"offline","context":{"idset":"9093"}}
+{"timestamp":1714080344.1249433,"name":"offline","context":{"idset":"9094"}}
+{"timestamp":1714080344.1259844,"name":"offline","context":{"idset":"9095"}}
+{"timestamp":1714080344.1270368,"name":"offline","context":{"idset":"9096"}}
+{"timestamp":1714080344.1329508,"name":"offline","context":{"idset":"9097"}}
+{"timestamp":1714080344.1339743,"name":"offline","context":{"idset":"9098"}}
+{"timestamp":1714080344.1349945,"name":"offline","context":{"idset":"9099"}}
+{"timestamp":1714080344.1360128,"name":"offline","context":{"idset":"9100"}}
+{"timestamp":1714080344.1370409,"name":"offline","context":{"idset":"9101"}}
+{"timestamp":1714080344.1380613,"name":"offline","context":{"idset":"9102"}}
+{"timestamp":1714080344.1390905,"name":"offline","context":{"idset":"9103"}}
+{"timestamp":1714080344.1401038,"name":"offline","context":{"idset":"9104"}}
+{"timestamp":1714080344.145992,"name":"offline","context":{"idset":"9105"}}
+{"timestamp":1714080344.1470103,"name":"offline","context":{"idset":"9106"}}
+{"timestamp":1714080344.1480432,"name":"offline","context":{"idset":"9107"}}
+{"timestamp":1714080344.1490691,"name":"offline","context":{"idset":"9108"}}
+{"timestamp":1714080344.1501102,"name":"offline","context":{"idset":"9109"}}
+{"timestamp":1714080344.1511378,"name":"offline","context":{"idset":"9110"}}
+{"timestamp":1714080344.1521585,"name":"offline","context":{"idset":"9111"}}
+{"timestamp":1714080344.1531839,"name":"offline","context":{"idset":"9112"}}
+{"timestamp":1714080344.1594648,"name":"offline","context":{"idset":"9113"}}
+{"timestamp":1714080344.1604736,"name":"offline","context":{"idset":"9114"}}
+{"timestamp":1714080344.16151,"name":"offline","context":{"idset":"9115"}}
+{"timestamp":1714080344.1625354,"name":"offline","context":{"idset":"9116"}}
+{"timestamp":1714080344.1635711,"name":"offline","context":{"idset":"9117"}}
+{"timestamp":1714080344.1645865,"name":"offline","context":{"idset":"9118"}}
+{"timestamp":1714080344.1656084,"name":"offline","context":{"idset":"9119"}}
+{"timestamp":1714080344.1666181,"name":"offline","context":{"idset":"9120"}}
+{"timestamp":1714080344.1676292,"name":"offline","context":{"idset":"9121"}}
+{"timestamp":1714080344.1686883,"name":"offline","context":{"idset":"9122"}}
+{"timestamp":1714080344.1697073,"name":"offline","context":{"idset":"9123"}}
+{"timestamp":1714080344.1707118,"name":"offline","context":{"idset":"9124"}}
+{"timestamp":1714080344.1717293,"name":"offline","context":{"idset":"9125"}}
+{"timestamp":1714080344.172756,"name":"offline","context":{"idset":"9126"}}
+{"timestamp":1714080344.1737616,"name":"offline","context":{"idset":"9127"}}
+{"timestamp":1714080344.1747618,"name":"offline","context":{"idset":"9128"}}
+{"timestamp":1714080344.1757796,"name":"offline","context":{"idset":"9129"}}
+{"timestamp":1714080344.1767852,"name":"offline","context":{"idset":"9130"}}
+{"timestamp":1714080344.1777864,"name":"offline","context":{"idset":"9131"}}
+{"timestamp":1714080344.178823,"name":"offline","context":{"idset":"9132"}}
+{"timestamp":1714080344.1846678,"name":"offline","context":{"idset":"9133"}}
+{"timestamp":1714080344.185679,"name":"offline","context":{"idset":"9134"}}
+{"timestamp":1714080344.1866727,"name":"offline","context":{"idset":"9135"}}
+{"timestamp":1714080344.1876748,"name":"offline","context":{"idset":"9136"}}
+{"timestamp":1714080344.1886866,"name":"offline","context":{"idset":"9137"}}
+{"timestamp":1714080344.1896882,"name":"offline","context":{"idset":"9138"}}
+{"timestamp":1714080344.1906784,"name":"offline","context":{"idset":"9139"}}
+{"timestamp":1714080344.1916811,"name":"offline","context":{"idset":"9140"}}
+{"timestamp":1714080344.1926703,"name":"offline","context":{"idset":"9141"}}
+{"timestamp":1714080344.1985416,"name":"offline","context":{"idset":"9142"}}
+{"timestamp":1714080344.1995482,"name":"offline","context":{"idset":"9143"}}
+{"timestamp":1714080344.2005415,"name":"offline","context":{"idset":"9144"}}
+{"timestamp":1714080344.2015376,"name":"offline","context":{"idset":"9145"}}
+{"timestamp":1714080344.2025492,"name":"offline","context":{"idset":"9146"}}
+{"timestamp":1714080344.2035441,"name":"offline","context":{"idset":"9147"}}
+{"timestamp":1714080344.2140853,"name":"offline","context":{"idset":"9148"}}
+{"timestamp":1714080344.2150724,"name":"offline","context":{"idset":"9149"}}
+{"timestamp":1714080344.2160594,"name":"offline","context":{"idset":"9150"}}
+{"timestamp":1714080344.2170608,"name":"offline","context":{"idset":"9151"}}
+{"timestamp":1714080344.21806,"name":"offline","context":{"idset":"9152"}}
+{"timestamp":1714080344.219048,"name":"offline","context":{"idset":"9153"}}
+{"timestamp":1714080344.220041,"name":"offline","context":{"idset":"9154"}}
+{"timestamp":1714080344.2210453,"name":"offline","context":{"idset":"9155"}}
+{"timestamp":1714080344.222044,"name":"offline","context":{"idset":"9156"}}
+{"timestamp":1714080344.2230477,"name":"offline","context":{"idset":"9157"}}
+{"timestamp":1714080344.2240419,"name":"offline","context":{"idset":"9158"}}
+{"timestamp":1714080344.2250333,"name":"offline","context":{"idset":"9159"}}
+{"timestamp":1714080344.2356467,"name":"offline","context":{"idset":"9160"}}
+{"timestamp":1714080344.236625,"name":"offline","context":{"idset":"9161"}}
+{"timestamp":1714080344.2376058,"name":"offline","context":{"idset":"9162"}}
+{"timestamp":1714080344.2385848,"name":"offline","context":{"idset":"9163"}}
+{"timestamp":1714080344.2395859,"name":"offline","context":{"idset":"9164"}}
+{"timestamp":1714080344.2405624,"name":"offline","context":{"idset":"9165"}}
+{"timestamp":1714080344.2415433,"name":"offline","context":{"idset":"9166"}}
+{"timestamp":1714080344.2425232,"name":"offline","context":{"idset":"9167"}}
+{"timestamp":1714080344.2434943,"name":"offline","context":{"idset":"9168"}}
+{"timestamp":1714080344.2444773,"name":"offline","context":{"idset":"9169"}}
+{"timestamp":1714080344.2502668,"name":"offline","context":{"idset":"9170"}}
+{"timestamp":1714080344.2512605,"name":"offline","context":{"idset":"9171"}}
+{"timestamp":1714080344.2522376,"name":"offline","context":{"idset":"9172"}}
+{"timestamp":1714080344.2532442,"name":"offline","context":{"idset":"9173"}}
+{"timestamp":1714080344.2542241,"name":"offline","context":{"idset":"9174"}}
+{"timestamp":1714080344.2552187,"name":"offline","context":{"idset":"9175"}}
+{"timestamp":1714080344.2562311,"name":"offline","context":{"idset":"9176"}}
+{"timestamp":1714080344.2572412,"name":"offline","context":{"idset":"9177"}}
+{"timestamp":1714080344.258235,"name":"offline","context":{"idset":"9178"}}
+{"timestamp":1714080344.2592318,"name":"offline","context":{"idset":"9179"}}
+{"timestamp":1714080344.2650726,"name":"offline","context":{"idset":"9180"}}
+{"timestamp":1714080344.2660499,"name":"offline","context":{"idset":"9181"}}
+{"timestamp":1714080344.2670221,"name":"offline","context":{"idset":"9182"}}
+{"timestamp":1714080344.2680173,"name":"offline","context":{"idset":"9183"}}
+{"timestamp":1714080344.2690022,"name":"offline","context":{"idset":"9184"}}
+{"timestamp":1714080344.2699897,"name":"offline","context":{"idset":"9185"}}
+{"timestamp":1714080344.2709801,"name":"offline","context":{"idset":"9186"}}
+{"timestamp":1714080344.2719529,"name":"offline","context":{"idset":"9187"}}
+{"timestamp":1714080344.2729254,"name":"offline","context":{"idset":"9188"}}
+{"timestamp":1714080344.2738943,"name":"offline","context":{"idset":"9189"}}
+{"timestamp":1714080344.2748826,"name":"offline","context":{"idset":"9190"}}
+{"timestamp":1714080344.2758524,"name":"offline","context":{"idset":"9191"}}
+{"timestamp":1714080344.2768226,"name":"offline","context":{"idset":"9192"}}
+{"timestamp":1714080344.2777913,"name":"offline","context":{"idset":"9193"}}
+{"timestamp":1714080344.2787611,"name":"offline","context":{"idset":"9194"}}
+{"timestamp":1714080344.2797465,"name":"offline","context":{"idset":"9195"}}
+{"timestamp":1714080344.2856276,"name":"offline","context":{"idset":"9196"}}
+{"timestamp":1714080344.2915232,"name":"offline","context":{"idset":"9197"}}
+{"timestamp":1714080344.2925186,"name":"offline","context":{"idset":"9198"}}
+{"timestamp":1714080344.2935534,"name":"offline","context":{"idset":"9199"}}
+{"timestamp":1714080344.2945464,"name":"offline","context":{"idset":"9200"}}
+{"timestamp":1714080344.2955477,"name":"offline","context":{"idset":"9201"}}
+{"timestamp":1714080344.2965407,"name":"offline","context":{"idset":"9202"}}
+{"timestamp":1714080344.2975252,"name":"offline","context":{"idset":"9203"}}
+{"timestamp":1714080344.2985032,"name":"offline","context":{"idset":"9204"}}
+{"timestamp":1714080344.2994766,"name":"offline","context":{"idset":"9205"}}
+{"timestamp":1714080344.3004596,"name":"offline","context":{"idset":"9206"}}
+{"timestamp":1714080344.3014441,"name":"offline","context":{"idset":"9207"}}
+{"timestamp":1714080344.3120627,"name":"offline","context":{"idset":"9208"}}
+{"timestamp":1714080344.3130293,"name":"offline","context":{"idset":"9209"}}
+{"timestamp":1714080344.3140025,"name":"offline","context":{"idset":"9210"}}
+{"timestamp":1714080344.314975,"name":"offline","context":{"idset":"9211"}}
+{"timestamp":1714080344.3159425,"name":"offline","context":{"idset":"9212"}}
+{"timestamp":1714080344.3168976,"name":"offline","context":{"idset":"9213"}}
+{"timestamp":1714080344.3227053,"name":"offline","context":{"idset":"9214"}}
+{"timestamp":1714080344.328532,"name":"offline","context":{"idset":"9215"}}
+{"timestamp":1714080344.3294952,"name":"offline","context":{"idset":"9216"}}
+{"timestamp":1714080344.3304465,"name":"offline","context":{"idset":"9217"}}
+{"timestamp":1714080344.3314035,"name":"offline","context":{"idset":"9218"}}
+{"timestamp":1714080344.3419514,"name":"offline","context":{"idset":"9219"}}
+{"timestamp":1714080344.3429244,"name":"offline","context":{"idset":"9220"}}
+{"timestamp":1714080344.3439927,"name":"offline","context":{"idset":"9221"}}
+{"timestamp":1714080344.3450038,"name":"offline","context":{"idset":"9222"}}
+{"timestamp":1714080344.3459775,"name":"offline","context":{"idset":"9223"}}
+{"timestamp":1714080344.3519516,"name":"offline","context":{"idset":"9224"}}
+{"timestamp":1714080344.3529301,"name":"offline","context":{"idset":"9225"}}
+{"timestamp":1714080344.3539178,"name":"offline","context":{"idset":"9226"}}
+{"timestamp":1714080344.3548732,"name":"offline","context":{"idset":"9227"}}
+{"timestamp":1714080344.3558545,"name":"offline","context":{"idset":"9228"}}
+{"timestamp":1714080344.3568065,"name":"offline","context":{"idset":"9229"}}
+{"timestamp":1714080344.3578215,"name":"offline","context":{"idset":"9230"}}
+{"timestamp":1714080344.358798,"name":"offline","context":{"idset":"9231"}}
+{"timestamp":1714080344.3597574,"name":"offline","context":{"idset":"9232"}}
+{"timestamp":1714080344.3607366,"name":"offline","context":{"idset":"9233"}}
+{"timestamp":1714080344.3624861,"name":"offline","context":{"idset":"9234"}}
+{"timestamp":1714080344.3634453,"name":"offline","context":{"idset":"9235"}}
+{"timestamp":1714080344.3643825,"name":"offline","context":{"idset":"9236"}}
+{"timestamp":1714080344.3653183,"name":"offline","context":{"idset":"9237"}}
+{"timestamp":1714080344.3737485,"name":"offline","context":{"idset":"9238"}}
+{"timestamp":1714080344.3751986,"name":"offline","context":{"idset":"9239"}}
+{"timestamp":1714080344.3767202,"name":"offline","context":{"idset":"9240"}}
+{"timestamp":1714080344.3782036,"name":"offline","context":{"idset":"9241"}}
+{"timestamp":1714080344.379317,"name":"offline","context":{"idset":"9242"}}
+{"timestamp":1714080344.3803017,"name":"offline","context":{"idset":"9243"}}
+{"timestamp":1714080344.381263,"name":"offline","context":{"idset":"9244"}}
+{"timestamp":1714080344.3822372,"name":"offline","context":{"idset":"9245"}}
+{"timestamp":1714080344.3832362,"name":"offline","context":{"idset":"9246"}}
+{"timestamp":1714080344.3842144,"name":"offline","context":{"idset":"9247"}}
+{"timestamp":1714080344.3851728,"name":"offline","context":{"idset":"9248"}}
+{"timestamp":1714080344.3861268,"name":"offline","context":{"idset":"9249"}}
+{"timestamp":1714080344.3870673,"name":"offline","context":{"idset":"9250"}}
+{"timestamp":1714080344.3928542,"name":"offline","context":{"idset":"9251"}}
+{"timestamp":1714080344.3938103,"name":"offline","context":{"idset":"9252"}}
+{"timestamp":1714080344.3947554,"name":"offline","context":{"idset":"9253"}}
+{"timestamp":1714080344.3956914,"name":"offline","context":{"idset":"9254"}}
+{"timestamp":1714080344.3966315,"name":"offline","context":{"idset":"9255"}}
+{"timestamp":1714080344.3975947,"name":"offline","context":{"idset":"9256"}}
+{"timestamp":1714080344.3985524,"name":"offline","context":{"idset":"9257"}}
+{"timestamp":1714080344.3995223,"name":"offline","context":{"idset":"9258"}}
+{"timestamp":1714080344.4004903,"name":"offline","context":{"idset":"9259"}}
+{"timestamp":1714080344.4014606,"name":"offline","context":{"idset":"9260"}}
+{"timestamp":1714080344.4024258,"name":"offline","context":{"idset":"9261"}}
+{"timestamp":1714080344.4033771,"name":"offline","context":{"idset":"9262"}}
+{"timestamp":1714080344.414088,"name":"offline","context":{"idset":"9263"}}
+{"timestamp":1714080344.4150367,"name":"offline","context":{"idset":"9264"}}
+{"timestamp":1714080344.4159892,"name":"offline","context":{"idset":"9265"}}
+{"timestamp":1714080344.4169438,"name":"offline","context":{"idset":"9266"}}
+{"timestamp":1714080344.4179211,"name":"offline","context":{"idset":"9267"}}
+{"timestamp":1714080344.4188566,"name":"offline","context":{"idset":"9268"}}
+{"timestamp":1714080344.419785,"name":"offline","context":{"idset":"9269"}}
+{"timestamp":1714080344.4207168,"name":"offline","context":{"idset":"9270"}}
+{"timestamp":1714080344.4216418,"name":"offline","context":{"idset":"9271"}}
+{"timestamp":1714080344.4225888,"name":"offline","context":{"idset":"9272"}}
+{"timestamp":1714080344.4349225,"name":"offline","context":{"idset":"9273"}}
+{"timestamp":1714080344.436379,"name":"offline","context":{"idset":"9274"}}
+{"timestamp":1714080344.4376011,"name":"offline","context":{"idset":"9275"}}
+{"timestamp":1714080344.4385791,"name":"offline","context":{"idset":"9276"}}
+{"timestamp":1714080344.4395256,"name":"offline","context":{"idset":"9277"}}
+{"timestamp":1714080344.4404736,"name":"offline","context":{"idset":"9278"}}
+{"timestamp":1714080344.4414096,"name":"offline","context":{"idset":"9279"}}
+{"timestamp":1714080344.442343,"name":"offline","context":{"idset":"9280"}}
+{"timestamp":1714080344.4432838,"name":"offline","context":{"idset":"9281"}}
+{"timestamp":1714080344.4442337,"name":"offline","context":{"idset":"9282"}}
+{"timestamp":1714080344.4451811,"name":"offline","context":{"idset":"9283"}}
+{"timestamp":1714080344.4461124,"name":"offline","context":{"idset":"9284"}}
+{"timestamp":1714080344.4470491,"name":"offline","context":{"idset":"9285"}}
+{"timestamp":1714080344.4479885,"name":"offline","context":{"idset":"9286"}}
+{"timestamp":1714080344.4489229,"name":"offline","context":{"idset":"9287"}}
+{"timestamp":1714080344.449852,"name":"offline","context":{"idset":"9288"}}
+{"timestamp":1714080344.4507728,"name":"offline","context":{"idset":"9289"}}
+{"timestamp":1714080344.4517064,"name":"offline","context":{"idset":"9290"}}
+{"timestamp":1714080344.4526646,"name":"offline","context":{"idset":"9291"}}
+{"timestamp":1714080344.4536095,"name":"offline","context":{"idset":"9292"}}
+{"timestamp":1714080344.4545672,"name":"offline","context":{"idset":"9293"}}
+{"timestamp":1714080344.4555387,"name":"offline","context":{"idset":"9294"}}
+{"timestamp":1714080344.4565399,"name":"offline","context":{"idset":"9295"}}
+{"timestamp":1714080344.457505,"name":"offline","context":{"idset":"9296"}}
+{"timestamp":1714080344.4584601,"name":"offline","context":{"idset":"9297"}}
+{"timestamp":1714080344.4594002,"name":"offline","context":{"idset":"9298"}}
+{"timestamp":1714080344.4603925,"name":"offline","context":{"idset":"9299"}}
+{"timestamp":1714080344.4613564,"name":"offline","context":{"idset":"9300"}}
+{"timestamp":1714080344.4622939,"name":"offline","context":{"idset":"9301"}}
+{"timestamp":1714080344.4632134,"name":"offline","context":{"idset":"9302"}}
+{"timestamp":1714080344.4641697,"name":"offline","context":{"idset":"9303"}}
+{"timestamp":1714080344.4651625,"name":"offline","context":{"idset":"9304"}}
+{"timestamp":1714080344.4661627,"name":"offline","context":{"idset":"9305"}}
+{"timestamp":1714080344.4671052,"name":"offline","context":{"idset":"9306"}}
+{"timestamp":1714080344.4680459,"name":"offline","context":{"idset":"9307"}}
+{"timestamp":1714080344.4739058,"name":"offline","context":{"idset":"9308"}}
+{"timestamp":1714080344.4748499,"name":"offline","context":{"idset":"9309"}}
+{"timestamp":1714080344.475785,"name":"offline","context":{"idset":"9310"}}
+{"timestamp":1714080344.4767175,"name":"offline","context":{"idset":"9311"}}
+{"timestamp":1714080344.4776549,"name":"offline","context":{"idset":"9312"}}
+{"timestamp":1714080344.4785774,"name":"offline","context":{"idset":"9313"}}
+{"timestamp":1714080344.479497,"name":"offline","context":{"idset":"9314"}}
+{"timestamp":1714080344.480407,"name":"offline","context":{"idset":"9315"}}
+{"timestamp":1714080344.4813321,"name":"offline","context":{"idset":"9316"}}
+{"timestamp":1714080344.4822829,"name":"offline","context":{"idset":"9317"}}
+{"timestamp":1714080344.4881229,"name":"offline","context":{"idset":"9318"}}
+{"timestamp":1714080344.5034955,"name":"offline","context":{"idset":"9319"}}
+{"timestamp":1714080344.5044162,"name":"offline","context":{"idset":"9320"}}
+{"timestamp":1714080344.5053475,"name":"offline","context":{"idset":"9321"}}
+{"timestamp":1714080344.5062675,"name":"offline","context":{"idset":"9322"}}
+{"timestamp":1714080344.5072043,"name":"offline","context":{"idset":"9323"}}
+{"timestamp":1714080344.5081296,"name":"offline","context":{"idset":"9324"}}
+{"timestamp":1714080344.5186729,"name":"offline","context":{"idset":"9325"}}
+{"timestamp":1714080344.5195913,"name":"offline","context":{"idset":"9326"}}
+{"timestamp":1714080344.5205076,"name":"offline","context":{"idset":"9327"}}
+{"timestamp":1714080344.5214198,"name":"offline","context":{"idset":"9328"}}
+{"timestamp":1714080344.5223422,"name":"offline","context":{"idset":"9329"}}
+{"timestamp":1714080344.5232503,"name":"offline","context":{"idset":"9330"}}
+{"timestamp":1714080344.5241618,"name":"offline","context":{"idset":"9331"}}
+{"timestamp":1714080344.5250759,"name":"offline","context":{"idset":"9332"}}
+{"timestamp":1714080344.5260212,"name":"offline","context":{"idset":"9461"}}
+{"timestamp":1714080344.5269349,"name":"offline","context":{"idset":"9462"}}
+{"timestamp":1714080344.5278602,"name":"offline","context":{"idset":"9463"}}
+{"timestamp":1714080344.5287764,"name":"offline","context":{"idset":"9464"}}
+{"timestamp":1714080344.5296896,"name":"offline","context":{"idset":"9465"}}
+{"timestamp":1714080344.5305982,"name":"offline","context":{"idset":"9466"}}
+{"timestamp":1714080344.5315106,"name":"offline","context":{"idset":"9467"}}
+{"timestamp":1714080344.5324373,"name":"offline","context":{"idset":"9468"}}
+{"timestamp":1714080344.5333507,"name":"offline","context":{"idset":"9469"}}
+{"timestamp":1714080344.53426,"name":"offline","context":{"idset":"9470"}}
+{"timestamp":1714080344.5351965,"name":"offline","context":{"idset":"9471"}}
+{"timestamp":1714080344.5361171,"name":"offline","context":{"idset":"9472"}}
+{"timestamp":1714080344.53703,"name":"offline","context":{"idset":"9473"}}
+{"timestamp":1714080344.5379417,"name":"offline","context":{"idset":"9474"}}
+{"timestamp":1714080344.538878,"name":"offline","context":{"idset":"9475"}}
+{"timestamp":1714080344.5397837,"name":"offline","context":{"idset":"9476"}}
+{"timestamp":1714080344.5406961,"name":"offline","context":{"idset":"9477"}}
+{"timestamp":1714080344.5416129,"name":"offline","context":{"idset":"9478"}}
+{"timestamp":1714080344.5425262,"name":"offline","context":{"idset":"9479"}}
+{"timestamp":1714080344.5434318,"name":"offline","context":{"idset":"9480"}}
+{"timestamp":1714080344.5443349,"name":"offline","context":{"idset":"9481"}}
+{"timestamp":1714080344.5452456,"name":"offline","context":{"idset":"9482"}}
+{"timestamp":1714080344.54617,"name":"offline","context":{"idset":"9483"}}
+{"timestamp":1714080344.5519075,"name":"offline","context":{"idset":"9484"}}
+{"timestamp":1714080344.552794,"name":"offline","context":{"idset":"9485"}}
+{"timestamp":1714080344.5537035,"name":"offline","context":{"idset":"9486"}}
+{"timestamp":1714080344.5545983,"name":"offline","context":{"idset":"9487"}}
+{"timestamp":1714080344.5554979,"name":"offline","context":{"idset":"9488"}}
+{"timestamp":1714080344.5563941,"name":"offline","context":{"idset":"9489"}}
+{"timestamp":1714080344.5572996,"name":"offline","context":{"idset":"9490"}}
+{"timestamp":1714080344.5582001,"name":"offline","context":{"idset":"9491"}}
+{"timestamp":1714080344.5590975,"name":"offline","context":{"idset":"9492"}}
+{"timestamp":1714080344.5600271,"name":"offline","context":{"idset":"9493"}}
+{"timestamp":1714080344.5609281,"name":"offline","context":{"idset":"9494"}}
+{"timestamp":1714080344.5618238,"name":"offline","context":{"idset":"9495"}}
+{"timestamp":1714080344.5627189,"name":"offline","context":{"idset":"9496"}}
+{"timestamp":1714080344.5636182,"name":"offline","context":{"idset":"9497"}}
+{"timestamp":1714080344.5645103,"name":"offline","context":{"idset":"9498"}}
+{"timestamp":1714080344.565392,"name":"offline","context":{"idset":"9499"}}
+{"timestamp":1714080344.566282,"name":"offline","context":{"idset":"9500"}}
+{"timestamp":1714080344.5671701,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1714080344.5680707,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1714080344.5689533,"name":"offline","context":{"idset":"9503"}}
+{"timestamp":1714080344.5750921,"name":"offline","context":{"idset":"9504"}}
+{"timestamp":1714080344.5759952,"name":"offline","context":{"idset":"9505"}}
+{"timestamp":1714080344.576885,"name":"offline","context":{"idset":"9506"}}
+{"timestamp":1714080344.5777614,"name":"offline","context":{"idset":"9507"}}
+{"timestamp":1714080344.5786495,"name":"offline","context":{"idset":"9508"}}
+{"timestamp":1714080344.5795333,"name":"offline","context":{"idset":"9509"}}
+{"timestamp":1714080344.580411,"name":"offline","context":{"idset":"9510"}}
+{"timestamp":1714080344.5812936,"name":"offline","context":{"idset":"9511"}}
+{"timestamp":1714080344.5821755,"name":"offline","context":{"idset":"9512"}}
+{"timestamp":1714080344.5830407,"name":"offline","context":{"idset":"9513"}}
+{"timestamp":1714080344.5839043,"name":"offline","context":{"idset":"9514"}}
+{"timestamp":1714080344.584765,"name":"offline","context":{"idset":"9515"}}
+{"timestamp":1714080344.5856478,"name":"offline","context":{"idset":"9516"}}
+{"timestamp":1714080344.5865061,"name":"offline","context":{"idset":"9517"}}
+{"timestamp":1714080344.5969472,"name":"offline","context":{"idset":"9518"}}
+{"timestamp":1714080344.5978258,"name":"offline","context":{"idset":"9519"}}
+{"timestamp":1714080344.5986896,"name":"offline","context":{"idset":"9520"}}
+{"timestamp":1714080344.5995681,"name":"offline","context":{"idset":"9521"}}
+{"timestamp":1714080344.6004386,"name":"offline","context":{"idset":"9522"}}
+{"timestamp":1714080344.6013296,"name":"offline","context":{"idset":"9523"}}
+{"timestamp":1714080344.6021929,"name":"offline","context":{"idset":"9524"}}
+{"timestamp":1714080344.6030715,"name":"offline","context":{"idset":"9525"}}
+{"timestamp":1714080344.6039488,"name":"offline","context":{"idset":"9526"}}
+{"timestamp":1714080344.614485,"name":"offline","context":{"idset":"9527"}}
+{"timestamp":1714080344.615556,"name":"offline","context":{"idset":"9528"}}
+{"timestamp":1714080344.616492,"name":"offline","context":{"idset":"9529"}}
+{"timestamp":1714080344.6173728,"name":"offline","context":{"idset":"9530"}}
+{"timestamp":1714080344.6182375,"name":"offline","context":{"idset":"9531"}}
+{"timestamp":1714080344.6191401,"name":"offline","context":{"idset":"9532"}}
+{"timestamp":1714080344.6200347,"name":"offline","context":{"idset":"9533"}}
+{"timestamp":1714080344.620923,"name":"offline","context":{"idset":"9534"}}
+{"timestamp":1714080344.6218159,"name":"offline","context":{"idset":"9536"}}
+{"timestamp":1714080344.6227481,"name":"offline","context":{"idset":"9537"}}
+{"timestamp":1714080344.6236603,"name":"offline","context":{"idset":"9538"}}
+{"timestamp":1714080344.6294525,"name":"offline","context":{"idset":"9539"}}
+{"timestamp":1714080344.6303809,"name":"offline","context":{"idset":"9540"}}
+{"timestamp":1714080344.6312754,"name":"offline","context":{"idset":"9541"}}
+{"timestamp":1714080344.6321454,"name":"offline","context":{"idset":"9542"}}
+{"timestamp":1714080344.6330059,"name":"offline","context":{"idset":"9543"}}
+{"timestamp":1714080344.6338811,"name":"offline","context":{"idset":"9544"}}
+{"timestamp":1714080344.634763,"name":"offline","context":{"idset":"9545"}}
+{"timestamp":1714080344.6356285,"name":"offline","context":{"idset":"9546"}}
+{"timestamp":1714080344.6364925,"name":"offline","context":{"idset":"9547"}}
+{"timestamp":1714080344.6373873,"name":"offline","context":{"idset":"9548"}}
+{"timestamp":1714080344.6382833,"name":"offline","context":{"idset":"9549"}}
+{"timestamp":1714080344.6393042,"name":"offline","context":{"idset":"9550"}}
+{"timestamp":1714080344.6401656,"name":"offline","context":{"idset":"9551"}}
+{"timestamp":1714080344.6410232,"name":"offline","context":{"idset":"9552"}}
+{"timestamp":1714080344.6418889,"name":"offline","context":{"idset":"9553"}}
+{"timestamp":1714080344.6427388,"name":"offline","context":{"idset":"9554"}}
+{"timestamp":1714080344.6436474,"name":"offline","context":{"idset":"9555"}}
+{"timestamp":1714080344.6445124,"name":"offline","context":{"idset":"9556"}}
+{"timestamp":1714080344.6453621,"name":"offline","context":{"idset":"9557"}}
+{"timestamp":1714080344.6462147,"name":"offline","context":{"idset":"9558"}}
+{"timestamp":1714080344.6470699,"name":"offline","context":{"idset":"9559"}}
+{"timestamp":1714080344.6479316,"name":"offline","context":{"idset":"9560"}}
+{"timestamp":1714080344.6489041,"name":"offline","context":{"idset":"9561"}}
+{"timestamp":1714080344.649775,"name":"offline","context":{"idset":"9562"}}
+{"timestamp":1714080344.6506453,"name":"offline","context":{"idset":"9563"}}
+{"timestamp":1714080344.6564436,"name":"offline","context":{"idset":"9564"}}
+{"timestamp":1714080344.6573386,"name":"offline","context":{"idset":"9565"}}
+{"timestamp":1714080344.658216,"name":"offline","context":{"idset":"9566"}}
+{"timestamp":1714080344.6590853,"name":"offline","context":{"idset":"9567"}}
+{"timestamp":1714080344.6599615,"name":"offline","context":{"idset":"9568"}}
+{"timestamp":1714080344.6608605,"name":"offline","context":{"idset":"9569"}}
+{"timestamp":1714080344.661747,"name":"offline","context":{"idset":"9570"}}
+{"timestamp":1714080344.6626127,"name":"offline","context":{"idset":"9571"}}
+{"timestamp":1714080344.6634822,"name":"offline","context":{"idset":"9572"}}
+{"timestamp":1714080344.6643503,"name":"offline","context":{"idset":"9573"}}
+{"timestamp":1714080344.6701543,"name":"offline","context":{"idset":"9574"}}
+{"timestamp":1714080344.6710403,"name":"offline","context":{"idset":"9575"}}
+{"timestamp":1714080344.6719105,"name":"offline","context":{"idset":"9576"}}
+{"timestamp":1714080344.6727829,"name":"offline","context":{"idset":"9577"}}
+{"timestamp":1714080344.673636,"name":"offline","context":{"idset":"9578"}}
+{"timestamp":1714080344.6744926,"name":"offline","context":{"idset":"9579"}}
+{"timestamp":1714080344.6753416,"name":"offline","context":{"idset":"9580"}}
+{"timestamp":1714080344.6810496,"name":"offline","context":{"idset":"9581"}}
+{"timestamp":1714080344.6820941,"name":"offline","context":{"idset":"9582"}}
+{"timestamp":1714080344.6829538,"name":"offline","context":{"idset":"9583"}}
+{"timestamp":1714080344.683809,"name":"offline","context":{"idset":"9584"}}
+{"timestamp":1714080344.684787,"name":"offline","context":{"idset":"9585"}}
+{"timestamp":1714080344.6858399,"name":"offline","context":{"idset":"9586"}}
+{"timestamp":1714080344.6868608,"name":"offline","context":{"idset":"9587"}}
+{"timestamp":1714080344.6877921,"name":"offline","context":{"idset":"9588"}}
+{"timestamp":1714080344.6886578,"name":"offline","context":{"idset":"9589"}}
+{"timestamp":1714080344.6894934,"name":"offline","context":{"idset":"9590"}}
+{"timestamp":1714080344.690336,"name":"offline","context":{"idset":"9591"}}
+{"timestamp":1714080344.6912,"name":"offline","context":{"idset":"9592"}}
+{"timestamp":1714080344.7018454,"name":"offline","context":{"idset":"9593"}}
+{"timestamp":1714080344.7027037,"name":"offline","context":{"idset":"9594"}}
+{"timestamp":1714080344.703568,"name":"offline","context":{"idset":"9595"}}
+{"timestamp":1714080344.7044218,"name":"offline","context":{"idset":"9596"}}
+{"timestamp":1714080344.7052765,"name":"offline","context":{"idset":"9597"}}
+{"timestamp":1714080344.7061257,"name":"offline","context":{"idset":"9598"}}
+{"timestamp":1714080344.7071288,"name":"offline","context":{"idset":"9599"}}
+{"timestamp":1714080344.7082801,"name":"offline","context":{"idset":"9600"}}
+{"timestamp":1714080344.7092154,"name":"offline","context":{"idset":"9601"}}
+{"timestamp":1714080344.710089,"name":"offline","context":{"idset":"9602"}}
+{"timestamp":1714080344.7112818,"name":"offline","context":{"idset":"9603"}}
+{"timestamp":1714080344.7125878,"name":"offline","context":{"idset":"9604"}}
+{"timestamp":1714080344.7139053,"name":"offline","context":{"idset":"9605"}}
+{"timestamp":1714080344.7151706,"name":"offline","context":{"idset":"9606"}}
+{"timestamp":1714080344.7164321,"name":"offline","context":{"idset":"9607"}}
+{"timestamp":1714080344.7173488,"name":"offline","context":{"idset":"9608"}}
+{"timestamp":1714080344.7181926,"name":"offline","context":{"idset":"9609"}}
+{"timestamp":1714080344.7190459,"name":"offline","context":{"idset":"9610"}}
+{"timestamp":1714080344.7198756,"name":"offline","context":{"idset":"9611"}}
+{"timestamp":1714080344.7206943,"name":"offline","context":{"idset":"9612"}}
+{"timestamp":1714080344.7215242,"name":"offline","context":{"idset":"9613"}}
+{"timestamp":1714080344.7223516,"name":"offline","context":{"idset":"9614"}}
+{"timestamp":1714080344.7231917,"name":"offline","context":{"idset":"9615"}}
+{"timestamp":1714080344.7240696,"name":"offline","context":{"idset":"9616"}}
+{"timestamp":1714080344.7297945,"name":"offline","context":{"idset":"9617"}}
+{"timestamp":1714080344.7306304,"name":"offline","context":{"idset":"9618"}}
+{"timestamp":1714080344.731478,"name":"offline","context":{"idset":"9619"}}
+{"timestamp":1714080344.7323356,"name":"offline","context":{"idset":"9620"}}
+{"timestamp":1714080344.7331946,"name":"offline","context":{"idset":"9621"}}
+{"timestamp":1714080344.7340283,"name":"offline","context":{"idset":"9622"}}
+{"timestamp":1714080344.7348545,"name":"offline","context":{"idset":"9623"}}
+{"timestamp":1714080344.735666,"name":"offline","context":{"idset":"9624"}}
+{"timestamp":1714080344.7364936,"name":"offline","context":{"idset":"9625"}}
+{"timestamp":1714080344.7373209,"name":"offline","context":{"idset":"9626"}}
+{"timestamp":1714080344.7381399,"name":"offline","context":{"idset":"9627"}}
+{"timestamp":1714080344.7389605,"name":"offline","context":{"idset":"9628"}}
+{"timestamp":1714080344.7397707,"name":"offline","context":{"idset":"9629"}}
+{"timestamp":1714080344.7405934,"name":"offline","context":{"idset":"9630"}}
+{"timestamp":1714080344.7414222,"name":"offline","context":{"idset":"9631"}}
+{"timestamp":1714080344.7422609,"name":"offline","context":{"idset":"9632"}}
+{"timestamp":1714080344.7430813,"name":"offline","context":{"idset":"9633"}}
+{"timestamp":1714080344.7438948,"name":"offline","context":{"idset":"9634"}}
+{"timestamp":1714080344.744709,"name":"offline","context":{"idset":"9635"}}
+{"timestamp":1714080344.7455275,"name":"offline","context":{"idset":"9636"}}
+{"timestamp":1714080344.746357,"name":"offline","context":{"idset":"9637"}}
+{"timestamp":1714080344.7471902,"name":"offline","context":{"idset":"9638"}}
+{"timestamp":1714080344.7480092,"name":"offline","context":{"idset":"9639"}}
+{"timestamp":1714080344.7488251,"name":"offline","context":{"idset":"9640"}}
+{"timestamp":1714080344.7496347,"name":"offline","context":{"idset":"9641"}}
+{"timestamp":1714080344.7504499,"name":"offline","context":{"idset":"9642"}}
+{"timestamp":1714080344.7512674,"name":"offline","context":{"idset":"9643"}}
+{"timestamp":1714080344.7521141,"name":"offline","context":{"idset":"9644"}}
+{"timestamp":1714080344.752933,"name":"offline","context":{"idset":"9645"}}
+{"timestamp":1714080344.7537417,"name":"offline","context":{"idset":"9646"}}
+{"timestamp":1714080344.7545516,"name":"offline","context":{"idset":"9647"}}
+{"timestamp":1714080344.7553813,"name":"offline","context":{"idset":"9648"}}
+{"timestamp":1714080344.7562056,"name":"offline","context":{"idset":"9649"}}
+{"timestamp":1714080344.7570274,"name":"offline","context":{"idset":"9650"}}
+{"timestamp":1714080344.7578611,"name":"offline","context":{"idset":"9651"}}
+{"timestamp":1714080344.7586737,"name":"offline","context":{"idset":"9652"}}
+{"timestamp":1714080344.759496,"name":"offline","context":{"idset":"9653"}}
+{"timestamp":1714080344.7603178,"name":"offline","context":{"idset":"9654"}}
+{"timestamp":1714080344.7611377,"name":"offline","context":{"idset":"9655"}}
+{"timestamp":1714080344.7619715,"name":"offline","context":{"idset":"9656"}}
+{"timestamp":1714080344.7628105,"name":"offline","context":{"idset":"9657"}}
+{"timestamp":1714080344.7636404,"name":"offline","context":{"idset":"9658"}}
+{"timestamp":1714080344.7644622,"name":"offline","context":{"idset":"9659"}}
+{"timestamp":1714080344.765286,"name":"offline","context":{"idset":"9660"}}
+{"timestamp":1714080344.7661512,"name":"offline","context":{"idset":"9661"}}
+{"timestamp":1714080344.7669661,"name":"offline","context":{"idset":"9662"}}
+{"timestamp":1714080344.7677951,"name":"offline","context":{"idset":"9663"}}
+{"timestamp":1714080344.7686191,"name":"offline","context":{"idset":"9664"}}
+{"timestamp":1714080344.7694504,"name":"offline","context":{"idset":"9665"}}
+{"timestamp":1714080344.7702658,"name":"offline","context":{"idset":"9666"}}
+{"timestamp":1714080344.7710745,"name":"offline","context":{"idset":"9667"}}
+{"timestamp":1714080344.7719007,"name":"offline","context":{"idset":"9668"}}
+{"timestamp":1714080344.7727029,"name":"offline","context":{"idset":"9669"}}
+{"timestamp":1714080344.7735164,"name":"offline","context":{"idset":"9670"}}
+{"timestamp":1714080344.7795796,"name":"offline","context":{"idset":"9671"}}
+{"timestamp":1714080344.7803884,"name":"offline","context":{"idset":"9672"}}
+{"timestamp":1714080344.7811806,"name":"offline","context":{"idset":"9673"}}
+{"timestamp":1714080344.7820108,"name":"offline","context":{"idset":"9674"}}
+{"timestamp":1714080344.7828181,"name":"offline","context":{"idset":"9675"}}
+{"timestamp":1714080344.7836072,"name":"offline","context":{"idset":"9676"}}
+{"timestamp":1714080344.784404,"name":"offline","context":{"idset":"9677"}}
+{"timestamp":1714080344.7852023,"name":"offline","context":{"idset":"9678"}}
+{"timestamp":1714080344.7859998,"name":"offline","context":{"idset":"9679"}}
+{"timestamp":1714080344.7868166,"name":"offline","context":{"idset":"9680"}}
+{"timestamp":1714080344.7876101,"name":"offline","context":{"idset":"9681"}}
+{"timestamp":1714080344.7884111,"name":"offline","context":{"idset":"9682"}}
+{"timestamp":1714080344.7893755,"name":"offline","context":{"idset":"9683"}}
+{"timestamp":1714080344.7902145,"name":"offline","context":{"idset":"9684"}}
+{"timestamp":1714080344.7910392,"name":"offline","context":{"idset":"9685"}}
+{"timestamp":1714080344.7918561,"name":"offline","context":{"idset":"9686"}}
+{"timestamp":1714080344.7978337,"name":"offline","context":{"idset":"9687"}}
+{"timestamp":1714080344.7986674,"name":"offline","context":{"idset":"9688"}}
+{"timestamp":1714080344.7997231,"name":"offline","context":{"idset":"9689"}}
+{"timestamp":1714080344.8005679,"name":"offline","context":{"idset":"9690"}}
+{"timestamp":1714080344.8013687,"name":"offline","context":{"idset":"9691"}}
+{"timestamp":1714080344.8021634,"name":"offline","context":{"idset":"9692"}}
+{"timestamp":1714080344.8029778,"name":"offline","context":{"idset":"9693"}}
+{"timestamp":1714080344.8037815,"name":"offline","context":{"idset":"9694"}}
+{"timestamp":1714080344.8045776,"name":"offline","context":{"idset":"9695"}}
+{"timestamp":1714080344.805377,"name":"offline","context":{"idset":"9696"}}
+{"timestamp":1714080344.8061955,"name":"offline","context":{"idset":"9697"}}
+{"timestamp":1714080344.8069911,"name":"offline","context":{"idset":"9698"}}
+{"timestamp":1714080344.8078947,"name":"offline","context":{"idset":"9699"}}
+{"timestamp":1714080344.8087564,"name":"offline","context":{"idset":"9700"}}
+{"timestamp":1714080344.8100462,"name":"offline","context":{"idset":"9701"}}
+{"timestamp":1714080344.8112156,"name":"offline","context":{"idset":"9702"}}
+{"timestamp":1714080344.826982,"name":"offline","context":{"idset":"9703"}}
+{"timestamp":1714080344.8278475,"name":"offline","context":{"idset":"9704"}}
+{"timestamp":1714080344.8286803,"name":"offline","context":{"idset":"9705"}}
+{"timestamp":1714080344.8295913,"name":"offline","context":{"idset":"9706"}}
+{"timestamp":1714080344.8306382,"name":"offline","context":{"idset":"9707"}}
+{"timestamp":1714080344.831481,"name":"offline","context":{"idset":"9708"}}
+{"timestamp":1714080344.8323429,"name":"offline","context":{"idset":"9709"}}
+{"timestamp":1714080344.8331823,"name":"offline","context":{"idset":"9710"}}
+{"timestamp":1714080344.8340149,"name":"offline","context":{"idset":"9711"}}
+{"timestamp":1714080344.8348916,"name":"offline","context":{"idset":"9712"}}
+{"timestamp":1714080344.8359063,"name":"offline","context":{"idset":"9713"}}
+{"timestamp":1714080344.8367443,"name":"offline","context":{"idset":"9714"}}
+{"timestamp":1714080344.8375852,"name":"offline","context":{"idset":"9715"}}
+{"timestamp":1714080344.8489711,"name":"offline","context":{"idset":"9716"}}
+{"timestamp":1714080344.850229,"name":"offline","context":{"idset":"9717"}}
+{"timestamp":1714080344.8518145,"name":"offline","context":{"idset":"9718"}}
+{"timestamp":1714080344.8534689,"name":"offline","context":{"idset":"9719"}}
+{"timestamp":1714080344.8550627,"name":"offline","context":{"idset":"9720"}}
+{"timestamp":1714080344.8566577,"name":"offline","context":{"idset":"9721"}}
+{"timestamp":1714080344.8582659,"name":"offline","context":{"idset":"9722"}}
+{"timestamp":1714080344.8592138,"name":"offline","context":{"idset":"9723"}}
+{"timestamp":1714080344.8605514,"name":"offline","context":{"idset":"9724"}}
+{"timestamp":1714080344.8613863,"name":"offline","context":{"idset":"9725"}}
+{"timestamp":1714080344.8622351,"name":"offline","context":{"idset":"9726"}}
+{"timestamp":1714080344.8630679,"name":"offline","context":{"idset":"9727"}}
+{"timestamp":1714080344.8638923,"name":"offline","context":{"idset":"9728"}}
+{"timestamp":1714080344.8649561,"name":"offline","context":{"idset":"9729"}}
+{"timestamp":1714080344.8659985,"name":"offline","context":{"idset":"9730"}}
+{"timestamp":1714080344.8668461,"name":"offline","context":{"idset":"9731"}}
+{"timestamp":1714080344.8789029,"name":"offline","context":{"idset":"9732"}}
+{"timestamp":1714080344.8797352,"name":"offline","context":{"idset":"9733"}}
+{"timestamp":1714080344.8805997,"name":"offline","context":{"idset":"9734"}}
+{"timestamp":1714080344.8814359,"name":"offline","context":{"idset":"9735"}}
+{"timestamp":1714080344.8822761,"name":"offline","context":{"idset":"9736"}}
+{"timestamp":1714080344.8832686,"name":"offline","context":{"idset":"9737"}}
+{"timestamp":1714080344.8843575,"name":"offline","context":{"idset":"9738"}}
+{"timestamp":1714080344.885221,"name":"offline","context":{"idset":"9739"}}
+{"timestamp":1714080344.8860428,"name":"offline","context":{"idset":"9740"}}
+{"timestamp":1714080344.8869357,"name":"offline","context":{"idset":"9741"}}
+{"timestamp":1714080344.8878372,"name":"offline","context":{"idset":"9742"}}
+{"timestamp":1714080344.8947678,"name":"offline","context":{"idset":"9743"}}
+{"timestamp":1714080344.8956001,"name":"offline","context":{"idset":"9744"}}
+{"timestamp":1714080344.8964572,"name":"offline","context":{"idset":"9745"}}
+{"timestamp":1714080344.897295,"name":"offline","context":{"idset":"9746"}}
+{"timestamp":1714080344.8981512,"name":"offline","context":{"idset":"9747"}}
+{"timestamp":1714080344.8990278,"name":"offline","context":{"idset":"9748"}}
+{"timestamp":1714080344.9002042,"name":"offline","context":{"idset":"9749"}}
+{"timestamp":1714080344.901062,"name":"offline","context":{"idset":"9750"}}
+{"timestamp":1714080344.9018764,"name":"offline","context":{"idset":"9751"}}
+{"timestamp":1714080344.902703,"name":"offline","context":{"idset":"9752"}}
+{"timestamp":1714080344.903543,"name":"offline","context":{"idset":"9753"}}
+{"timestamp":1714080344.9046125,"name":"offline","context":{"idset":"9754"}}
+{"timestamp":1714080344.9056718,"name":"offline","context":{"idset":"9755"}}
+{"timestamp":1714080344.9065304,"name":"offline","context":{"idset":"9756"}}
+{"timestamp":1714080344.9074149,"name":"offline","context":{"idset":"9757"}}
+{"timestamp":1714080344.9140155,"name":"offline","context":{"idset":"9758"}}
+{"timestamp":1714080344.9153035,"name":"offline","context":{"idset":"9759"}}
+{"timestamp":1714080344.916182,"name":"offline","context":{"idset":"9760"}}
+{"timestamp":1714080344.9170225,"name":"offline","context":{"idset":"9761"}}
+{"timestamp":1714080344.9178357,"name":"offline","context":{"idset":"9762"}}
+{"timestamp":1714080344.918663,"name":"offline","context":{"idset":"9763"}}
+{"timestamp":1714080344.9198904,"name":"offline","context":{"idset":"9764"}}
+{"timestamp":1714080344.9207053,"name":"offline","context":{"idset":"9765"}}
+{"timestamp":1714080344.9270728,"name":"offline","context":{"idset":"9766"}}
+{"timestamp":1714080344.9278996,"name":"offline","context":{"idset":"9767"}}
+{"timestamp":1714080344.9287915,"name":"offline","context":{"idset":"9768"}}
+{"timestamp":1714080344.9299469,"name":"offline","context":{"idset":"9769"}}
+{"timestamp":1714080344.9307673,"name":"offline","context":{"idset":"9770"}}
+{"timestamp":1714080344.9315798,"name":"offline","context":{"idset":"9771"}}
+{"timestamp":1714080344.9324284,"name":"offline","context":{"idset":"9772"}}
+{"timestamp":1714080344.9332585,"name":"offline","context":{"idset":"9773"}}
+{"timestamp":1714080344.9345143,"name":"offline","context":{"idset":"9774"}}
+{"timestamp":1714080344.9353678,"name":"offline","context":{"idset":"9775"}}
+{"timestamp":1714080344.9361889,"name":"offline","context":{"idset":"9776"}}
+{"timestamp":1714080344.9370205,"name":"offline","context":{"idset":"9777"}}
+{"timestamp":1714080344.9378583,"name":"offline","context":{"idset":"9778"}}
+{"timestamp":1714080344.9387186,"name":"offline","context":{"idset":"9779"}}
+{"timestamp":1714080344.9399421,"name":"offline","context":{"idset":"9780"}}
+{"timestamp":1714080344.9407518,"name":"offline","context":{"idset":"9781"}}
+{"timestamp":1714080344.9415672,"name":"offline","context":{"idset":"9782"}}
+{"timestamp":1714080344.942344,"name":"offline","context":{"idset":"9783"}}
+{"timestamp":1714080344.9431057,"name":"offline","context":{"idset":"9784"}}
+{"timestamp":1714080344.9491904,"name":"offline","context":{"idset":"9785"}}
+{"timestamp":1714080344.9503584,"name":"offline","context":{"idset":"9786"}}
+{"timestamp":1714080344.9511704,"name":"offline","context":{"idset":"9787"}}
+{"timestamp":1714080344.9520302,"name":"offline","context":{"idset":"9788"}}
+{"timestamp":1714080344.9529111,"name":"offline","context":{"idset":"9789"}}
+{"timestamp":1714080344.9537432,"name":"offline","context":{"idset":"9790"}}
+{"timestamp":1714080344.9549875,"name":"offline","context":{"idset":"9791"}}
+{"timestamp":1714080344.9558184,"name":"offline","context":{"idset":"9792"}}
+{"timestamp":1714080344.9566174,"name":"offline","context":{"idset":"9793"}}
+{"timestamp":1714080344.9574239,"name":"offline","context":{"idset":"9794"}}
+{"timestamp":1714080344.9582429,"name":"offline","context":{"idset":"9795"}}
+{"timestamp":1714080344.9590886,"name":"offline","context":{"idset":"9796"}}
+{"timestamp":1714080344.9603617,"name":"offline","context":{"idset":"9797"}}
+{"timestamp":1714080344.9611712,"name":"offline","context":{"idset":"9798"}}
+{"timestamp":1714080344.962008,"name":"offline","context":{"idset":"9799"}}
+{"timestamp":1714080344.962806,"name":"offline","context":{"idset":"9800"}}
+{"timestamp":1714080344.9636056,"name":"offline","context":{"idset":"9801"}}
+{"timestamp":1714080344.9644208,"name":"offline","context":{"idset":"9802"}}
+{"timestamp":1714080344.9656661,"name":"offline","context":{"idset":"9803"}}
+{"timestamp":1714080344.9665079,"name":"offline","context":{"idset":"9804"}}
+{"timestamp":1714080344.9673145,"name":"offline","context":{"idset":"9805"}}
+{"timestamp":1714080344.9680912,"name":"offline","context":{"idset":"9806"}}
+{"timestamp":1714080344.9688754,"name":"offline","context":{"idset":"9807"}}
+{"timestamp":1714080344.969671,"name":"offline","context":{"idset":"9808"}}
+{"timestamp":1714080344.9705212,"name":"offline","context":{"idset":"9809"}}
+{"timestamp":1714080344.9717331,"name":"offline","context":{"idset":"9810"}}
+{"timestamp":1714080344.9725232,"name":"offline","context":{"idset":"9811"}}
+{"timestamp":1714080344.973304,"name":"offline","context":{"idset":"9812"}}
+{"timestamp":1714080344.9740758,"name":"offline","context":{"idset":"9813"}}
+{"timestamp":1714080344.9748564,"name":"offline","context":{"idset":"9814"}}
+{"timestamp":1714080344.9756184,"name":"offline","context":{"idset":"9815"}}
+{"timestamp":1714080344.9765301,"name":"offline","context":{"idset":"9816"}}
+{"timestamp":1714080344.9776297,"name":"offline","context":{"idset":"9817"}}
+{"timestamp":1714080344.9784389,"name":"offline","context":{"idset":"9818"}}
+{"timestamp":1714080344.9792318,"name":"offline","context":{"idset":"9819"}}
+{"timestamp":1714080344.9800057,"name":"offline","context":{"idset":"9820"}}
+{"timestamp":1714080344.9808109,"name":"offline","context":{"idset":"9821"}}
+{"timestamp":1714080344.98158,"name":"offline","context":{"idset":"9822"}}
+{"timestamp":1714080344.9823875,"name":"offline","context":{"idset":"9823"}}
+{"timestamp":1714080344.9835715,"name":"offline","context":{"idset":"9824"}}
+{"timestamp":1714080344.9895761,"name":"offline","context":{"idset":"9825"}}
+{"timestamp":1714080344.9903612,"name":"offline","context":{"idset":"9826"}}
+{"timestamp":1714080344.9911306,"name":"offline","context":{"idset":"9827"}}
+{"timestamp":1714080344.9918947,"name":"offline","context":{"idset":"9828"}}
+{"timestamp":1714080344.9926548,"name":"offline","context":{"idset":"9829"}}
+{"timestamp":1714080344.9934208,"name":"offline","context":{"idset":"9830"}}
+{"timestamp":1714080344.9941854,"name":"offline","context":{"idset":"9831"}}
+{"timestamp":1714080344.9949446,"name":"offline","context":{"idset":"9832"}}
+{"timestamp":1714080344.9957004,"name":"offline","context":{"idset":"9934"}}
+{"timestamp":1714080344.9964583,"name":"offline","context":{"idset":"9935"}}
+{"timestamp":1714080344.9972129,"name":"offline","context":{"idset":"9936"}}
+{"timestamp":1714080345.0030603,"name":"offline","context":{"idset":"9937"}}
+{"timestamp":1714080345.0038218,"name":"offline","context":{"idset":"9938"}}
+{"timestamp":1714080345.0045757,"name":"offline","context":{"idset":"9939"}}
+{"timestamp":1714080345.0053346,"name":"offline","context":{"idset":"9940"}}
+{"timestamp":1714080345.0060847,"name":"offline","context":{"idset":"9941"}}
+{"timestamp":1714080345.0068343,"name":"offline","context":{"idset":"9942"}}
+{"timestamp":1714080345.0075791,"name":"offline","context":{"idset":"9943"}}
+{"timestamp":1714080345.008333,"name":"offline","context":{"idset":"9944"}}
+{"timestamp":1714080345.0090895,"name":"offline","context":{"idset":"9945"}}
+{"timestamp":1714080345.0098565,"name":"offline","context":{"idset":"9946"}}
+{"timestamp":1714080345.0105965,"name":"offline","context":{"idset":"9947"}}
+{"timestamp":1714080345.0113401,"name":"offline","context":{"idset":"9948"}}
+{"timestamp":1714080345.0120881,"name":"offline","context":{"idset":"9949"}}
+{"timestamp":1714080345.0128338,"name":"offline","context":{"idset":"9950"}}
+{"timestamp":1714080345.0135756,"name":"offline","context":{"idset":"9951"}}
+{"timestamp":1714080345.0143209,"name":"offline","context":{"idset":"9952"}}
+{"timestamp":1714080345.015064,"name":"offline","context":{"idset":"9953"}}
+{"timestamp":1714080345.0208395,"name":"offline","context":{"idset":"9954"}}
+{"timestamp":1714080345.0215821,"name":"offline","context":{"idset":"9955"}}
+{"timestamp":1714080345.0223231,"name":"offline","context":{"idset":"9956"}}
+{"timestamp":1714080345.0230651,"name":"offline","context":{"idset":"9957"}}
+{"timestamp":1714080345.023808,"name":"offline","context":{"idset":"9958"}}
+{"timestamp":1714080345.0295622,"name":"offline","context":{"idset":"9959"}}
+{"timestamp":1714080345.0302999,"name":"offline","context":{"idset":"9960"}}
+{"timestamp":1714080345.0310411,"name":"offline","context":{"idset":"9961"}}
+{"timestamp":1714080345.0317764,"name":"offline","context":{"idset":"9962"}}
+{"timestamp":1714080345.0325186,"name":"offline","context":{"idset":"9963"}}
+{"timestamp":1714080345.0382705,"name":"offline","context":{"idset":"9964"}}
+{"timestamp":1714080345.0390115,"name":"offline","context":{"idset":"9965"}}
+{"timestamp":1714080345.0397437,"name":"offline","context":{"idset":"9966"}}
+{"timestamp":1714080345.0404806,"name":"offline","context":{"idset":"9967"}}
+{"timestamp":1714080345.0412192,"name":"offline","context":{"idset":"9968"}}
+{"timestamp":1714080345.0419562,"name":"offline","context":{"idset":"9969"}}
+{"timestamp":1714080345.0426865,"name":"offline","context":{"idset":"9970"}}
+{"timestamp":1714080345.0434532,"name":"offline","context":{"idset":"9971"}}
+{"timestamp":1714080345.044194,"name":"offline","context":{"idset":"9972"}}
+{"timestamp":1714080345.0449293,"name":"offline","context":{"idset":"9973"}}
+{"timestamp":1714080345.045656,"name":"offline","context":{"idset":"9974"}}
+{"timestamp":1714080345.0463839,"name":"offline","context":{"idset":"9975"}}
+{"timestamp":1714080345.0471117,"name":"offline","context":{"idset":"9976"}}
+{"timestamp":1714080345.0528548,"name":"offline","context":{"idset":"9977"}}
+{"timestamp":1714080345.0535784,"name":"offline","context":{"idset":"9978"}}
+{"timestamp":1714080345.0543025,"name":"offline","context":{"idset":"9979"}}
+{"timestamp":1714080345.0550265,"name":"offline","context":{"idset":"9980"}}
+{"timestamp":1714080345.0557451,"name":"offline","context":{"idset":"9981"}}
+{"timestamp":1714080345.0564678,"name":"offline","context":{"idset":"9982"}}
+{"timestamp":1714080345.0571947,"name":"offline","context":{"idset":"9983"}}
+{"timestamp":1714080345.0579178,"name":"offline","context":{"idset":"9984"}}
+{"timestamp":1714080345.0586357,"name":"offline","context":{"idset":"9985"}}
+{"timestamp":1714080345.0593576,"name":"offline","context":{"idset":"9986"}}
+{"timestamp":1714080345.0600801,"name":"offline","context":{"idset":"9987"}}
+{"timestamp":1714080345.0607953,"name":"offline","context":{"idset":"9988"}}
+{"timestamp":1714080345.0615168,"name":"offline","context":{"idset":"9989"}}
+{"timestamp":1714080345.0622365,"name":"offline","context":{"idset":"9990"}}
+{"timestamp":1714080345.0630217,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1714080345.0637407,"name":"offline","context":{"idset":"9992"}}
+{"timestamp":1714080345.0644553,"name":"offline","context":{"idset":"9993"}}
+{"timestamp":1714080345.0651734,"name":"offline","context":{"idset":"9994"}}
+{"timestamp":1714080345.0658896,"name":"offline","context":{"idset":"9995"}}
+{"timestamp":1714080345.0666018,"name":"offline","context":{"idset":"9996"}}
+{"timestamp":1714080345.0673206,"name":"offline","context":{"idset":"9997"}}
+{"timestamp":1714080345.0680366,"name":"offline","context":{"idset":"9998"}}
+{"timestamp":1714080345.0687478,"name":"offline","context":{"idset":"9999"}}
+{"timestamp":1714080345.0694644,"name":"offline","context":{"idset":"10000"}}
+{"timestamp":1714080345.0701864,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1714080345.070909,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1714080345.0716188,"name":"offline","context":{"idset":"10003"}}
+{"timestamp":1714080345.0723352,"name":"offline","context":{"idset":"10004"}}
+{"timestamp":1714080345.0730524,"name":"offline","context":{"idset":"10005"}}
+{"timestamp":1714080345.0737607,"name":"offline","context":{"idset":"10006"}}
+{"timestamp":1714080345.0744677,"name":"offline","context":{"idset":"10007"}}
+{"timestamp":1714080345.0751796,"name":"offline","context":{"idset":"10008"}}
+{"timestamp":1714080345.0758913,"name":"offline","context":{"idset":"10009"}}
+{"timestamp":1714080345.0765965,"name":"offline","context":{"idset":"10010"}}
+{"timestamp":1714080345.0773072,"name":"offline","context":{"idset":"10011"}}
+{"timestamp":1714080345.0780172,"name":"offline","context":{"idset":"10012"}}
+{"timestamp":1714080345.0787172,"name":"offline","context":{"idset":"10013"}}
+{"timestamp":1714080345.0794206,"name":"offline","context":{"idset":"10014"}}
+{"timestamp":1714080345.0801275,"name":"offline","context":{"idset":"10015"}}
+{"timestamp":1714080345.080832,"name":"offline","context":{"idset":"10016"}}
+{"timestamp":1714080345.0865171,"name":"offline","context":{"idset":"10017"}}
+{"timestamp":1714080345.0872257,"name":"offline","context":{"idset":"10018"}}
+{"timestamp":1714080345.0879297,"name":"offline","context":{"idset":"10019"}}
+{"timestamp":1714080345.0886252,"name":"offline","context":{"idset":"10020"}}
+{"timestamp":1714080345.0893283,"name":"offline","context":{"idset":"10021"}}
+{"timestamp":1714080345.0900323,"name":"offline","context":{"idset":"10022"}}
+{"timestamp":1714080345.0907319,"name":"offline","context":{"idset":"10023"}}
+{"timestamp":1714080345.091444,"name":"offline","context":{"idset":"10024"}}
+{"timestamp":1714080345.0921457,"name":"offline","context":{"idset":"10025"}}
+{"timestamp":1714080345.0928457,"name":"offline","context":{"idset":"10026"}}
+{"timestamp":1714080345.0935407,"name":"offline","context":{"idset":"10027"}}
+{"timestamp":1714080345.0942383,"name":"offline","context":{"idset":"10028"}}
+{"timestamp":1714080345.0949364,"name":"offline","context":{"idset":"10029"}}
+{"timestamp":1714080345.1005919,"name":"offline","context":{"idset":"10030"}}
+{"timestamp":1714080345.1012926,"name":"offline","context":{"idset":"10031"}}
+{"timestamp":1714080345.1019921,"name":"offline","context":{"idset":"10032"}}
+{"timestamp":1714080345.1026819,"name":"offline","context":{"idset":"10033"}}
+{"timestamp":1714080345.1033762,"name":"offline","context":{"idset":"10034"}}
+{"timestamp":1714080345.1040721,"name":"offline","context":{"idset":"10035"}}
+{"timestamp":1714080345.1047626,"name":"offline","context":{"idset":"10036"}}
+{"timestamp":1714080345.105454,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1714080345.1061442,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1714080345.1068342,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1714080345.1075141,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1714080345.1082003,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1714080345.1088853,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1714080345.1195738,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1714080345.1202629,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1714080345.12096,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1714080345.1216397,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1714080345.1223252,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1714080345.1230097,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1714080345.1236851,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1714080345.1243663,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1714080345.1250396,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1714080345.1257124,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1714080345.1263926,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1714080345.1320908,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1714080345.1327693,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1714080345.1334455,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1714080345.1341236,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1714080345.1347897,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1714080345.1354635,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1714080345.1361351,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1714080345.1368062,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1714080345.1374714,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1714080345.1381409,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1714080345.1436639,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1714080345.1443408,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1714080345.1450489,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1714080345.1457133,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1714080345.1463914,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1714080345.147069,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1714080345.1477306,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1714080345.148397,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1714080345.14906,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1714080345.1497183,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1714080345.1567328,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1714080345.1577306,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1714080345.1587241,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1714080345.1597283,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1714080345.1607344,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1714080345.1617031,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1714080345.1626444,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1714080345.1635826,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1714080345.1645203,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1714080345.1651754,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1714080345.1658826,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1714080345.1665318,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1714080345.1671824,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1714080345.1678379,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1714080345.1684842,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1714080345.1691325,"name":"offline","context":{"idset":"10153"}}
+{"timestamp":1714080345.1697738,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1714080345.1704197,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1714080345.1710699,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1714080345.1717155,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1714080345.1772366,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1714080345.1778879,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1714080345.1785285,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1714080345.1791737,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1714080345.1798198,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1714080345.1804566,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1714080345.181103,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1714080345.1817608,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1714080345.1824105,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1714080345.183053,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1714080345.1836908,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1714080345.1843309,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1714080345.1849704,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1714080345.1856074,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1714080345.1862466,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1714080345.1868856,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1714080345.18752,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1714080345.1881595,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1714080345.1887898,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1714080345.1894362,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1714080345.1900818,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1714080345.1907139,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1714080345.1913495,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1714080345.1919873,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1714080345.1926155,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1714080345.1932473,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1714080345.1938784,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1714080345.1945066,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1714080345.1951423,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1714080345.1957667,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1714080345.1963973,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1714080345.1970255,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1714080345.1976473,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1714080345.198272,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1714080345.198895,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1714080345.1995151,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1714080345.2001421,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1714080345.2007577,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1714080345.2013726,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1714080345.2019961,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1714080345.2026088,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1714080345.2032278,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1714080345.2038467,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1714080345.2044582,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1714080345.2050753,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1714080345.2056847,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1714080345.2062988,"name":"offline","context":{"idset":"10206"}}
+{"timestamp":1714080345.2069151,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1714080345.2075238,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1714080345.2081368,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1714080345.2087438,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1714080345.2093549,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1714080345.2099693,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1714080345.2105865,"name":"offline","context":{"idset":"10357"}}
+{"timestamp":1714080345.2111995,"name":"offline","context":{"idset":"10358"}}
+{"timestamp":1714080345.2118115,"name":"offline","context":{"idset":"10359"}}
+{"timestamp":1714080345.2124434,"name":"offline","context":{"idset":"10360"}}
+{"timestamp":1714080345.2131011,"name":"offline","context":{"idset":"10361"}}
+{"timestamp":1714080345.2137375,"name":"offline","context":{"idset":"10362"}}
+{"timestamp":1714080345.2143414,"name":"offline","context":{"idset":"10363"}}
+{"timestamp":1714080345.2149482,"name":"offline","context":{"idset":"10364"}}
+{"timestamp":1714080345.2155495,"name":"offline","context":{"idset":"10365"}}
+{"timestamp":1714080345.2161593,"name":"offline","context":{"idset":"10366"}}
+{"timestamp":1714080345.2167585,"name":"offline","context":{"idset":"10367"}}
+{"timestamp":1714080345.2173705,"name":"offline","context":{"idset":"10368"}}
+{"timestamp":1714080345.2179759,"name":"offline","context":{"idset":"10369"}}
+{"timestamp":1714080345.2185702,"name":"offline","context":{"idset":"10370"}}
+{"timestamp":1714080345.2191706,"name":"offline","context":{"idset":"10371"}}
+{"timestamp":1714080345.2197647,"name":"offline","context":{"idset":"10372"}}
+{"timestamp":1714080345.2203591,"name":"offline","context":{"idset":"10373"}}
+{"timestamp":1714080345.2209604,"name":"offline","context":{"idset":"10374"}}
+{"timestamp":1714080345.2263117,"name":"offline","context":{"idset":"10375"}}
+{"timestamp":1714080345.2269125,"name":"offline","context":{"idset":"10376"}}
+{"timestamp":1714080345.2275071,"name":"offline","context":{"idset":"10377"}}
+{"timestamp":1714080345.228399,"name":"offline","context":{"idset":"10378"}}
+{"timestamp":1714080345.2292757,"name":"offline","context":{"idset":"10379"}}
+{"timestamp":1714080345.2301586,"name":"offline","context":{"idset":"10380"}}
+{"timestamp":1714080345.231035,"name":"offline","context":{"idset":"10381"}}
+{"timestamp":1714080345.23191,"name":"offline","context":{"idset":"10382"}}
+{"timestamp":1714080345.2327824,"name":"offline","context":{"idset":"10383"}}
+{"timestamp":1714080345.2336648,"name":"offline","context":{"idset":"10384"}}
+{"timestamp":1714080345.2345288,"name":"offline","context":{"idset":"10385"}}
+{"timestamp":1714080345.2354136,"name":"offline","context":{"idset":"10386"}}
+{"timestamp":1714080345.2362933,"name":"offline","context":{"idset":"10387"}}
+{"timestamp":1714080345.2371619,"name":"offline","context":{"idset":"10388"}}
+{"timestamp":1714080345.2380126,"name":"offline","context":{"idset":"10389"}}
+{"timestamp":1714080345.2389193,"name":"offline","context":{"idset":"10390"}}
+{"timestamp":1714080345.2460017,"name":"offline","context":{"idset":"10391"}}
+{"timestamp":1714080345.2465885,"name":"offline","context":{"idset":"10392"}}
+{"timestamp":1714080345.247179,"name":"offline","context":{"idset":"10393"}}
+{"timestamp":1714080345.2477586,"name":"offline","context":{"idset":"10394"}}
+{"timestamp":1714080345.2483449,"name":"offline","context":{"idset":"10395"}}
+{"timestamp":1714080345.2489326,"name":"offline","context":{"idset":"10396"}}
+{"timestamp":1714080345.249512,"name":"offline","context":{"idset":"10397"}}
+{"timestamp":1714080345.2500968,"name":"offline","context":{"idset":"10398"}}
+{"timestamp":1714080345.2506752,"name":"offline","context":{"idset":"10399"}}
+{"timestamp":1714080345.2513537,"name":"offline","context":{"idset":"10400"}}
+{"timestamp":1714080345.251987,"name":"offline","context":{"idset":"10401"}}
+{"timestamp":1714080345.2525692,"name":"offline","context":{"idset":"10402"}}
+{"timestamp":1714080345.2531652,"name":"offline","context":{"idset":"10403"}}
+{"timestamp":1714080345.2538178,"name":"offline","context":{"idset":"10404"}}
+{"timestamp":1714080345.2544584,"name":"offline","context":{"idset":"10405"}}
+{"timestamp":1714080345.2550554,"name":"offline","context":{"idset":"10406"}}
+{"timestamp":1714080345.2556324,"name":"offline","context":{"idset":"10407"}}
+{"timestamp":1714080345.2562139,"name":"offline","context":{"idset":"10408"}}
+{"timestamp":1714080345.2614758,"name":"offline","context":{"idset":"10409"}}
+{"timestamp":1714080345.2620549,"name":"offline","context":{"idset":"10410"}}
+{"timestamp":1714080345.2626262,"name":"offline","context":{"idset":"10411"}}
+{"timestamp":1714080345.2631965,"name":"offline","context":{"idset":"10412"}}
+{"timestamp":1714080345.2637653,"name":"offline","context":{"idset":"10413"}}
+{"timestamp":1714080345.2643328,"name":"offline","context":{"idset":"10414"}}
+{"timestamp":1714080345.2649076,"name":"offline","context":{"idset":"10415"}}
+{"timestamp":1714080345.2654774,"name":"offline","context":{"idset":"10416"}}
+{"timestamp":1714080345.2707708,"name":"offline","context":{"idset":"10417"}}
+{"timestamp":1714080345.2713783,"name":"offline","context":{"idset":"10418"}}
+{"timestamp":1714080345.27198,"name":"offline","context":{"idset":"10419"}}
+{"timestamp":1714080345.2725658,"name":"offline","context":{"idset":"10420"}}
+{"timestamp":1714080345.273145,"name":"offline","context":{"idset":"10423"}}
+{"timestamp":1714080345.2737226,"name":"offline","context":{"idset":"10424"}}
+{"timestamp":1714080345.2743964,"name":"offline","context":{"idset":"10425"}}
+{"timestamp":1714080345.2749949,"name":"offline","context":{"idset":"10426"}}
+{"timestamp":1714080345.2755604,"name":"offline","context":{"idset":"10427"}}
+{"timestamp":1714080345.2761424,"name":"offline","context":{"idset":"10428"}}
+{"timestamp":1714080345.2767057,"name":"offline","context":{"idset":"10429"}}
+{"timestamp":1714080345.2772696,"name":"offline","context":{"idset":"10430"}}
+{"timestamp":1714080345.2825341,"name":"offline","context":{"idset":"10431"}}
+{"timestamp":1714080345.2831051,"name":"offline","context":{"idset":"10432"}}
+{"timestamp":1714080345.2836652,"name":"offline","context":{"idset":"10433"}}
+{"timestamp":1714080345.2842267,"name":"offline","context":{"idset":"10434"}}
+{"timestamp":1714080345.2847836,"name":"offline","context":{"idset":"10435"}}
+{"timestamp":1714080345.2853441,"name":"offline","context":{"idset":"10436"}}
+{"timestamp":1714080345.2859447,"name":"offline","context":{"idset":"10437"}}
+{"timestamp":1714080345.2964807,"name":"offline","context":{"idset":"10438"}}
+{"timestamp":1714080345.2970586,"name":"offline","context":{"idset":"10439"}}
+{"timestamp":1714080345.2976236,"name":"offline","context":{"idset":"10440"}}
+{"timestamp":1714080345.2981966,"name":"offline","context":{"idset":"10441"}}
+{"timestamp":1714080345.2987645,"name":"offline","context":{"idset":"10442"}}
+{"timestamp":1714080345.2993402,"name":"offline","context":{"idset":"10443"}}
+{"timestamp":1714080345.2999213,"name":"offline","context":{"idset":"10444"}}
+{"timestamp":1714080345.3004849,"name":"offline","context":{"idset":"10445"}}
+{"timestamp":1714080345.3010581,"name":"offline","context":{"idset":"10446"}}
+{"timestamp":1714080345.3016212,"name":"offline","context":{"idset":"10447"}}
+{"timestamp":1714080345.302191,"name":"offline","context":{"idset":"10448"}}
+{"timestamp":1714080345.3027563,"name":"offline","context":{"idset":"10449"}}
+{"timestamp":1714080345.3033164,"name":"offline","context":{"idset":"10450"}}
+{"timestamp":1714080345.3086951,"name":"offline","context":{"idset":"10451"}}
+{"timestamp":1714080345.3092692,"name":"offline","context":{"idset":"10452"}}
+{"timestamp":1714080345.3099811,"name":"offline","context":{"idset":"10453"}}
+{"timestamp":1714080345.3105462,"name":"offline","context":{"idset":"10454"}}
+{"timestamp":1714080345.3111367,"name":"offline","context":{"idset":"10455"}}
+{"timestamp":1714080345.3117001,"name":"offline","context":{"idset":"10456"}}
+{"timestamp":1714080345.3122668,"name":"offline","context":{"idset":"10457"}}
+{"timestamp":1714080345.3128302,"name":"offline","context":{"idset":"10458"}}
+{"timestamp":1714080345.3133934,"name":"offline","context":{"idset":"10459"}}
+{"timestamp":1714080345.313987,"name":"offline","context":{"idset":"10460"}}
+{"timestamp":1714080345.3145609,"name":"offline","context":{"idset":"10461"}}
+{"timestamp":1714080345.3151228,"name":"offline","context":{"idset":"10462"}}
+{"timestamp":1714080345.3156807,"name":"offline","context":{"idset":"10463"}}
+{"timestamp":1714080345.3210235,"name":"offline","context":{"idset":"10464"}}
+{"timestamp":1714080345.3215904,"name":"offline","context":{"idset":"10465"}}
+{"timestamp":1714080345.3221509,"name":"offline","context":{"idset":"10466"}}
+{"timestamp":1714080345.322706,"name":"offline","context":{"idset":"10467"}}
+{"timestamp":1714080345.323271,"name":"offline","context":{"idset":"10468"}}
+{"timestamp":1714080345.3286102,"name":"offline","context":{"idset":"10469"}}
+{"timestamp":1714080345.3291945,"name":"offline","context":{"idset":"10470"}}
+{"timestamp":1714080345.3297474,"name":"offline","context":{"idset":"10471"}}
+{"timestamp":1714080345.330313,"name":"offline","context":{"idset":"10472"}}
+{"timestamp":1714080345.3308897,"name":"offline","context":{"idset":"10473"}}
+{"timestamp":1714080345.3314412,"name":"offline","context":{"idset":"10474"}}
+{"timestamp":1714080345.3319976,"name":"offline","context":{"idset":"10475"}}
+{"timestamp":1714080345.3325431,"name":"offline","context":{"idset":"10476"}}
+{"timestamp":1714080345.3330975,"name":"offline","context":{"idset":"10477"}}
+{"timestamp":1714080345.3336415,"name":"offline","context":{"idset":"10478"}}
+{"timestamp":1714080345.3341944,"name":"offline","context":{"idset":"10479"}}
+{"timestamp":1714080345.3347397,"name":"offline","context":{"idset":"10480"}}
+{"timestamp":1714080345.3352907,"name":"offline","context":{"idset":"10481"}}
+{"timestamp":1714080345.3358607,"name":"offline","context":{"idset":"10482"}}
+{"timestamp":1714080345.3364227,"name":"offline","context":{"idset":"10483"}}
+{"timestamp":1714080345.3369994,"name":"offline","context":{"idset":"10484"}}
+{"timestamp":1714080345.3375456,"name":"offline","context":{"idset":"10485"}}
+{"timestamp":1714080345.3380949,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1714080345.343395,"name":"offline","context":{"idset":"10487"}}
+{"timestamp":1714080345.3439825,"name":"offline","context":{"idset":"10488"}}
+{"timestamp":1714080345.3445253,"name":"offline","context":{"idset":"10489"}}
+{"timestamp":1714080345.345088,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1714080345.3456404,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1714080345.346211,"name":"offline","context":{"idset":"10492"}}
+{"timestamp":1714080345.3468189,"name":"offline","context":{"idset":"10493"}}
+{"timestamp":1714080345.3473492,"name":"offline","context":{"idset":"10494"}}
+{"timestamp":1714080345.3478839,"name":"offline","context":{"idset":"10495"}}
+{"timestamp":1714080345.3530903,"name":"offline","context":{"idset":"10496"}}
+{"timestamp":1714080345.3536167,"name":"offline","context":{"idset":"10497"}}
+{"timestamp":1714080345.354157,"name":"offline","context":{"idset":"10498"}}
+{"timestamp":1714080345.3546801,"name":"offline","context":{"idset":"10499"}}
+{"timestamp":1714080345.3552043,"name":"offline","context":{"idset":"10500"}}
+{"timestamp":1714080345.3604355,"name":"offline","context":{"idset":"10501"}}
+{"timestamp":1714080345.3609717,"name":"offline","context":{"idset":"10502"}}
+{"timestamp":1714080345.3614998,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1714080345.3620427,"name":"offline","context":{"idset":"10504"}}
+{"timestamp":1714080345.3626232,"name":"offline","context":{"idset":"10505"}}
+{"timestamp":1714080345.3632736,"name":"offline","context":{"idset":"10506"}}
+{"timestamp":1714080345.3638072,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1714080345.3643303,"name":"offline","context":{"idset":"10508"}}
+{"timestamp":1714080345.3648741,"name":"offline","context":{"idset":"10509"}}
+{"timestamp":1714080345.3654089,"name":"offline","context":{"idset":"10510"}}
+{"timestamp":1714080345.365948,"name":"offline","context":{"idset":"10511"}}
+{"timestamp":1714080345.3664696,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1714080345.3670001,"name":"offline","context":{"idset":"10513"}}
+{"timestamp":1714080345.3675199,"name":"offline","context":{"idset":"10514"}}
+{"timestamp":1714080345.3680487,"name":"offline","context":{"idset":"10515"}}
+{"timestamp":1714080345.368567,"name":"offline","context":{"idset":"10516"}}
+{"timestamp":1714080345.3690913,"name":"offline","context":{"idset":"10517"}}
+{"timestamp":1714080345.3696079,"name":"offline","context":{"idset":"10518"}}
+{"timestamp":1714080345.370132,"name":"offline","context":{"idset":"10519"}}
+{"timestamp":1714080345.3706489,"name":"offline","context":{"idset":"10520"}}
+{"timestamp":1714080345.3711736,"name":"offline","context":{"idset":"10521"}}
+{"timestamp":1714080345.3716877,"name":"offline","context":{"idset":"10522"}}
+{"timestamp":1714080345.3722079,"name":"offline","context":{"idset":"10523"}}
+{"timestamp":1714080345.3727231,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1714080345.3732429,"name":"offline","context":{"idset":"10525"}}
+{"timestamp":1714080345.3737564,"name":"offline","context":{"idset":"10526"}}
+{"timestamp":1714080345.3742917,"name":"offline","context":{"idset":"10527"}}
+{"timestamp":1714080345.3748202,"name":"offline","context":{"idset":"10528"}}
+{"timestamp":1714080345.375334,"name":"offline","context":{"idset":"10529"}}
+{"timestamp":1714080345.3758543,"name":"offline","context":{"idset":"10530"}}
+{"timestamp":1714080345.3763657,"name":"offline","context":{"idset":"10531"}}
+{"timestamp":1714080345.3768806,"name":"offline","context":{"idset":"10532"}}
+{"timestamp":1714080345.3773916,"name":"offline","context":{"idset":"10533"}}
+{"timestamp":1714080345.3779147,"name":"offline","context":{"idset":"10534"}}
+{"timestamp":1714080345.3784249,"name":"offline","context":{"idset":"10535"}}
+{"timestamp":1714080345.3789692,"name":"offline","context":{"idset":"10536"}}
+{"timestamp":1714080345.3794816,"name":"offline","context":{"idset":"10537"}}
+{"timestamp":1714080345.3799975,"name":"offline","context":{"idset":"10538"}}
+{"timestamp":1714080345.3805037,"name":"offline","context":{"idset":"10539"}}
+{"timestamp":1714080345.3810143,"name":"offline","context":{"idset":"10540"}}
+{"timestamp":1714080345.3815196,"name":"offline","context":{"idset":"10541"}}
+{"timestamp":1714080345.3820312,"name":"offline","context":{"idset":"10542"}}
+{"timestamp":1714080345.3825357,"name":"offline","context":{"idset":"10543"}}
+{"timestamp":1714080345.3830523,"name":"offline","context":{"idset":"10544"}}
+{"timestamp":1714080345.3835559,"name":"offline","context":{"idset":"10545"}}
+{"timestamp":1714080345.3840694,"name":"offline","context":{"idset":"10546"}}
+{"timestamp":1714080345.3845723,"name":"offline","context":{"idset":"10547"}}
+{"timestamp":1714080345.3850808,"name":"offline","context":{"idset":"10548"}}
+{"timestamp":1714080345.3902802,"name":"offline","context":{"idset":"10549"}}
+{"timestamp":1714080345.3907855,"name":"offline","context":{"idset":"10550"}}
+{"timestamp":1714080345.3912935,"name":"offline","context":{"idset":"10551"}}
+{"timestamp":1714080345.3917861,"name":"offline","context":{"idset":"10552"}}
+{"timestamp":1714080345.3922927,"name":"offline","context":{"idset":"10553"}}
+{"timestamp":1714080345.3928056,"name":"offline","context":{"idset":"10554"}}
+{"timestamp":1714080345.3933058,"name":"offline","context":{"idset":"10555"}}
+{"timestamp":1714080345.398654,"name":"offline","context":{"idset":"10556"}}
+{"timestamp":1714080345.3991814,"name":"offline","context":{"idset":"10557"}}
+{"timestamp":1714080345.3996971,"name":"offline","context":{"idset":"10558"}}
+{"timestamp":1714080345.4002125,"name":"offline","context":{"idset":"10559"}}
+{"timestamp":1714080345.4007232,"name":"offline","context":{"idset":"10560"}}
+{"timestamp":1714080345.4012341,"name":"offline","context":{"idset":"10561"}}
+{"timestamp":1714080345.4017413,"name":"offline","context":{"idset":"10562"}}
+{"timestamp":1714080345.4022551,"name":"offline","context":{"idset":"10563"}}
+{"timestamp":1714080345.4027605,"name":"offline","context":{"idset":"10564"}}
+{"timestamp":1714080345.4032733,"name":"offline","context":{"idset":"10565"}}
+{"timestamp":1714080345.4037783,"name":"offline","context":{"idset":"10566"}}
+{"timestamp":1714080345.4042897,"name":"offline","context":{"idset":"10567"}}
+{"timestamp":1714080345.4047971,"name":"offline","context":{"idset":"10568"}}
+{"timestamp":1714080345.4053078,"name":"offline","context":{"idset":"10569"}}
+{"timestamp":1714080345.4058201,"name":"offline","context":{"idset":"10570"}}
+{"timestamp":1714080345.4063215,"name":"offline","context":{"idset":"10571"}}
+{"timestamp":1714080345.4068511,"name":"offline","context":{"idset":"10572"}}
+{"timestamp":1714080345.4073534,"name":"offline","context":{"idset":"10573"}}
+{"timestamp":1714080345.4078596,"name":"offline","context":{"idset":"10574"}}
+{"timestamp":1714080345.4083707,"name":"offline","context":{"idset":"10575"}}
+{"timestamp":1714080345.4088831,"name":"offline","context":{"idset":"10576"}}
+{"timestamp":1714080345.4093902,"name":"offline","context":{"idset":"10577"}}
+{"timestamp":1714080345.4099,"name":"offline","context":{"idset":"10578"}}
+{"timestamp":1714080345.4104106,"name":"offline","context":{"idset":"10579"}}
+{"timestamp":1714080345.4109192,"name":"offline","context":{"idset":"10580"}}
+{"timestamp":1714080345.4114177,"name":"offline","context":{"idset":"10581"}}
+{"timestamp":1714080345.4167416,"name":"offline","context":{"idset":"10582"}}
+{"timestamp":1714080345.4172471,"name":"offline","context":{"idset":"10583"}}
+{"timestamp":1714080345.4177489,"name":"offline","context":{"idset":"10584"}}
+{"timestamp":1714080345.4182456,"name":"offline","context":{"idset":"10585"}}
+{"timestamp":1714080345.4187577,"name":"offline","context":{"idset":"10586"}}
+{"timestamp":1714080345.4192626,"name":"offline","context":{"idset":"10587"}}
+{"timestamp":1714080345.4197593,"name":"offline","context":{"idset":"10588"}}
+{"timestamp":1714080345.4202693,"name":"offline","context":{"idset":"10589"}}
+{"timestamp":1714080345.420773,"name":"offline","context":{"idset":"10590"}}
+{"timestamp":1714080345.4212713,"name":"offline","context":{"idset":"10591"}}
+{"timestamp":1714080345.4217684,"name":"offline","context":{"idset":"10592"}}
+{"timestamp":1714080345.4222698,"name":"offline","context":{"idset":"10593"}}
+{"timestamp":1714080345.4227626,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1714080345.4232597,"name":"offline","context":{"idset":"10595"}}
+{"timestamp":1714080345.4237494,"name":"offline","context":{"idset":"10596"}}
+{"timestamp":1714080345.4242418,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1714080345.4247313,"name":"offline","context":{"idset":"10598"}}
+{"timestamp":1714080345.4252267,"name":"offline","context":{"idset":"10599"}}
+{"timestamp":1714080345.4257193,"name":"offline","context":{"idset":"10600"}}
+{"timestamp":1714080345.426214,"name":"offline","context":{"idset":"10601"}}
+{"timestamp":1714080345.4314997,"name":"offline","context":{"idset":"10602"}}
+{"timestamp":1714080345.4319968,"name":"offline","context":{"idset":"10603"}}
+{"timestamp":1714080345.4324822,"name":"offline","context":{"idset":"10604"}}
+{"timestamp":1714080345.4329808,"name":"offline","context":{"idset":"10605"}}
+{"timestamp":1714080345.4334626,"name":"offline","context":{"idset":"10606"}}
+{"timestamp":1714080345.4339592,"name":"offline","context":{"idset":"10607"}}
+{"timestamp":1714080345.4344425,"name":"offline","context":{"idset":"10608"}}
+{"timestamp":1714080345.4349277,"name":"offline","context":{"idset":"10609"}}
+{"timestamp":1714080345.4354091,"name":"offline","context":{"idset":"10610"}}
+{"timestamp":1714080345.4359,"name":"offline","context":{"idset":"10611"}}
+{"timestamp":1714080345.4363818,"name":"offline","context":{"idset":"10612"}}
+{"timestamp":1714080345.4368677,"name":"offline","context":{"idset":"10613"}}
+{"timestamp":1714080345.4420199,"name":"offline","context":{"idset":"10614"}}
+{"timestamp":1714080345.4425054,"name":"offline","context":{"idset":"10615"}}
+{"timestamp":1714080345.4429984,"name":"offline","context":{"idset":"10616"}}
+{"timestamp":1714080345.4434767,"name":"offline","context":{"idset":"10617"}}
+{"timestamp":1714080345.4439619,"name":"offline","context":{"idset":"10618"}}
+{"timestamp":1714080345.4444513,"name":"offline","context":{"idset":"10619"}}
+{"timestamp":1714080345.4449444,"name":"offline","context":{"idset":"10620"}}
+{"timestamp":1714080345.4454195,"name":"offline","context":{"idset":"10621"}}
+{"timestamp":1714080345.4458988,"name":"offline","context":{"idset":"10622"}}
+{"timestamp":1714080345.4463892,"name":"offline","context":{"idset":"10623"}}
+{"timestamp":1714080345.446938,"name":"offline","context":{"idset":"10624"}}
+{"timestamp":1714080345.4474196,"name":"offline","context":{"idset":"10625"}}
+{"timestamp":1714080345.4479048,"name":"offline","context":{"idset":"10626"}}
+{"timestamp":1714080345.4483783,"name":"offline","context":{"idset":"10627"}}
+{"timestamp":1714080345.4488575,"name":"offline","context":{"idset":"10628"}}
+{"timestamp":1714080345.4493308,"name":"offline","context":{"idset":"10629"}}
+{"timestamp":1714080345.4498093,"name":"offline","context":{"idset":"10630"}}
+{"timestamp":1714080345.4502845,"name":"offline","context":{"idset":"10631"}}
+{"timestamp":1714080345.4507546,"name":"offline","context":{"idset":"10632"}}
+{"timestamp":1714080345.4512365,"name":"offline","context":{"idset":"10633"}}
+{"timestamp":1714080345.4517126,"name":"offline","context":{"idset":"10634"}}
+{"timestamp":1714080345.4521973,"name":"offline","context":{"idset":"10635"}}
+{"timestamp":1714080345.4526701,"name":"offline","context":{"idset":"10636"}}
+{"timestamp":1714080345.4531498,"name":"offline","context":{"idset":"10637"}}
+{"timestamp":1714080345.4536173,"name":"offline","context":{"idset":"10638"}}
+{"timestamp":1714080345.4540894,"name":"offline","context":{"idset":"10639"}}
+{"timestamp":1714080345.4545565,"name":"offline","context":{"idset":"10640"}}
+{"timestamp":1714080345.4550343,"name":"offline","context":{"idset":"10641"}}
+{"timestamp":1714080345.4554989,"name":"offline","context":{"idset":"10642"}}
+{"timestamp":1714080345.4559722,"name":"offline","context":{"idset":"10643"}}
+{"timestamp":1714080345.4564383,"name":"offline","context":{"idset":"10644"}}
+{"timestamp":1714080345.4569125,"name":"offline","context":{"idset":"10645"}}
+{"timestamp":1714080345.4573796,"name":"offline","context":{"idset":"10646"}}
+{"timestamp":1714080345.45785,"name":"offline","context":{"idset":"10647"}}
+{"timestamp":1714080345.4583251,"name":"offline","context":{"idset":"10648"}}
+{"timestamp":1714080345.4587917,"name":"offline","context":{"idset":"10649"}}
+{"timestamp":1714080345.4592631,"name":"offline","context":{"idset":"10650"}}
+{"timestamp":1714080345.4597278,"name":"offline","context":{"idset":"10651"}}
+{"timestamp":1714080345.4601982,"name":"offline","context":{"idset":"10652"}}
+{"timestamp":1714080345.4606714,"name":"offline","context":{"idset":"10653"}}
+{"timestamp":1714080345.4611466,"name":"offline","context":{"idset":"10654"}}
+{"timestamp":1714080345.4616106,"name":"offline","context":{"idset":"10655"}}
+{"timestamp":1714080345.4620895,"name":"offline","context":{"idset":"10656"}}
+{"timestamp":1714080345.4625528,"name":"offline","context":{"idset":"10657"}}
+{"timestamp":1714080345.4630387,"name":"offline","context":{"idset":"10658"}}
+{"timestamp":1714080345.463516,"name":"offline","context":{"idset":"10659"}}
+{"timestamp":1714080345.464165,"name":"offline","context":{"idset":"10660"}}
+{"timestamp":1714080345.4647946,"name":"offline","context":{"idset":"10661"}}
+{"timestamp":1714080345.465426,"name":"offline","context":{"idset":"10662"}}
+{"timestamp":1714080345.4660647,"name":"offline","context":{"idset":"10663"}}
+{"timestamp":1714080345.4667072,"name":"offline","context":{"idset":"10664"}}
+{"timestamp":1714080345.4673407,"name":"offline","context":{"idset":"10665"}}
+{"timestamp":1714080345.4679685,"name":"offline","context":{"idset":"10666"}}
+{"timestamp":1714080345.4685948,"name":"offline","context":{"idset":"10667"}}
+{"timestamp":1714080345.4692276,"name":"offline","context":{"idset":"10668"}}
+{"timestamp":1714080345.4698608,"name":"offline","context":{"idset":"10669"}}
+{"timestamp":1714080345.470484,"name":"offline","context":{"idset":"10670"}}
+{"timestamp":1714080345.4711108,"name":"offline","context":{"idset":"10671"}}
+{"timestamp":1714080345.4717319,"name":"offline","context":{"idset":"10672"}}
+{"timestamp":1714080345.472362,"name":"offline","context":{"idset":"10673"}}
+{"timestamp":1714080345.4729819,"name":"offline","context":{"idset":"10674"}}
+{"timestamp":1714080345.4736087,"name":"offline","context":{"idset":"10675"}}
+{"timestamp":1714080345.4742351,"name":"offline","context":{"idset":"10676"}}
+{"timestamp":1714080345.4748561,"name":"offline","context":{"idset":"10677"}}
+{"timestamp":1714080345.4754725,"name":"offline","context":{"idset":"10678"}}
+{"timestamp":1714080345.476094,"name":"offline","context":{"idset":"10679"}}
+{"timestamp":1714080345.4767118,"name":"offline","context":{"idset":"10680"}}
+{"timestamp":1714080345.4773269,"name":"offline","context":{"idset":"10681"}}
+{"timestamp":1714080345.4779565,"name":"offline","context":{"idset":"10682"}}
+{"timestamp":1714080345.4785664,"name":"offline","context":{"idset":"10683"}}
+{"timestamp":1714080345.4791923,"name":"offline","context":{"idset":"10684"}}
+{"timestamp":1714080345.4798124,"name":"offline","context":{"idset":"10685"}}
+{"timestamp":1714080345.4804623,"name":"offline","context":{"idset":"10686"}}
+{"timestamp":1714080345.4810951,"name":"offline","context":{"idset":"10687"}}
+{"timestamp":1714080345.4817045,"name":"offline","context":{"idset":"10688"}}
+{"timestamp":1714080345.4823239,"name":"offline","context":{"idset":"10689"}}
+{"timestamp":1714080345.4829447,"name":"offline","context":{"idset":"10690"}}
+{"timestamp":1714080345.4835539,"name":"offline","context":{"idset":"10691"}}
+{"timestamp":1714080345.4841733,"name":"offline","context":{"idset":"10692"}}
+{"timestamp":1714080345.484781,"name":"offline","context":{"idset":"10693"}}
+{"timestamp":1714080345.4853864,"name":"offline","context":{"idset":"10694"}}
+{"timestamp":1714080345.4859917,"name":"offline","context":{"idset":"10695"}}
+{"timestamp":1714080345.4865923,"name":"offline","context":{"idset":"10696"}}
+{"timestamp":1714080345.4872029,"name":"offline","context":{"idset":"10697"}}
+{"timestamp":1714080345.4878094,"name":"offline","context":{"idset":"10698"}}
+{"timestamp":1714080345.4884079,"name":"offline","context":{"idset":"10699"}}
+{"timestamp":1714080345.4890285,"name":"offline","context":{"idset":"10700"}}
+{"timestamp":1714080345.489639,"name":"offline","context":{"idset":"10701"}}
+{"timestamp":1714080345.4970701,"name":"offline","context":{"idset":"10702"}}
+{"timestamp":1714080345.4976821,"name":"offline","context":{"idset":"10703"}}
+{"timestamp":1714080345.4982991,"name":"offline","context":{"idset":"10704"}}
+{"timestamp":1714080345.4989026,"name":"offline","context":{"idset":"10705"}}
+{"timestamp":1714080345.5062871,"name":"offline","context":{"idset":"10706"}}
+{"timestamp":1714080345.521796,"name":"offline","context":{"idset":"10707"}}
+{"timestamp":1714080345.5224485,"name":"offline","context":{"idset":"10708"}}
+{"timestamp":1714080345.5231533,"name":"offline","context":{"idset":"10709"}}
+{"timestamp":1714080345.5237887,"name":"offline","context":{"idset":"10710"}}
+{"timestamp":1714080345.5250113,"name":"offline","context":{"idset":"10711"}}
+{"timestamp":1714080345.5258987,"name":"offline","context":{"idset":"10712"}}
+{"timestamp":1714080345.5267389,"name":"offline","context":{"idset":"10713"}}
+{"timestamp":1714080345.5273638,"name":"offline","context":{"idset":"10714"}}
+{"timestamp":1714080345.5280008,"name":"offline","context":{"idset":"10715"}}
+{"timestamp":1714080345.5286283,"name":"offline","context":{"idset":"10716"}}
+{"timestamp":1714080345.5292621,"name":"offline","context":{"idset":"10717"}}
+{"timestamp":1714080345.5298927,"name":"offline","context":{"idset":"10718"}}
+{"timestamp":1714080345.530514,"name":"offline","context":{"idset":"10719"}}
+{"timestamp":1714080345.5311422,"name":"offline","context":{"idset":"10720"}}
+{"timestamp":1714080345.5317764,"name":"offline","context":{"idset":"10721"}}
+{"timestamp":1714080345.5324056,"name":"offline","context":{"idset":"10722"}}
+{"timestamp":1714080345.5330291,"name":"offline","context":{"idset":"10723"}}
+{"timestamp":1714080345.5336516,"name":"offline","context":{"idset":"10724"}}
+{"timestamp":1714080345.5342858,"name":"offline","context":{"idset":"10725"}}
+{"timestamp":1714080345.5353532,"name":"offline","context":{"idset":"10726"}}
+{"timestamp":1714080345.536381,"name":"offline","context":{"idset":"10727"}}
+{"timestamp":1714080345.5370309,"name":"offline","context":{"idset":"10728"}}
+{"timestamp":1714080345.5376806,"name":"offline","context":{"idset":"10729"}}
+{"timestamp":1714080345.5383244,"name":"offline","context":{"idset":"10730"}}
+{"timestamp":1714080345.5391092,"name":"offline","context":{"idset":"10731"}}
+{"timestamp":1714080345.5397351,"name":"offline","context":{"idset":"10732"}}
+{"timestamp":1714080345.54038,"name":"offline","context":{"idset":"10733"}}
+{"timestamp":1714080345.5410159,"name":"offline","context":{"idset":"10734"}}
+{"timestamp":1714080345.5416646,"name":"offline","context":{"idset":"10735"}}
+{"timestamp":1714080345.5422993,"name":"offline","context":{"idset":"10736"}}
+{"timestamp":1714080345.542944,"name":"offline","context":{"idset":"10737"}}
+{"timestamp":1714080345.5435736,"name":"offline","context":{"idset":"10738"}}
+{"timestamp":1714080345.5442195,"name":"offline","context":{"idset":"10739"}}
+{"timestamp":1714080345.5448518,"name":"offline","context":{"idset":"10740"}}
+{"timestamp":1714080345.5454683,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1714080345.5460835,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1714080345.5466945,"name":"offline","context":{"idset":"10743"}}
+{"timestamp":1714080345.5531597,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1714080345.5537691,"name":"offline","context":{"idset":"10745"}}
+{"timestamp":1714080345.5543191,"name":"offline","context":{"idset":"10746"}}
+{"timestamp":1714080345.5547371,"name":"offline","context":{"idset":"10747"}}
+{"timestamp":1714080345.5551565,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1714080345.5555604,"name":"offline","context":{"idset":"10749"}}
+{"timestamp":1714080345.5560017,"name":"offline","context":{"idset":"10750"}}
+{"timestamp":1714080345.5564733,"name":"offline","context":{"idset":"10751"}}
+{"timestamp":1714080345.5571959,"name":"offline","context":{"idset":"10752"}}
+{"timestamp":1714080345.5578794,"name":"offline","context":{"idset":"10753"}}
+{"timestamp":1714080345.5584641,"name":"offline","context":{"idset":"10754"}}
+{"timestamp":1714080345.5589619,"name":"offline","context":{"idset":"10755"}}
+{"timestamp":1714080345.559371,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1714080345.5597773,"name":"offline","context":{"idset":"10757"}}
+{"timestamp":1714080345.5601878,"name":"offline","context":{"idset":"10758"}}
+{"timestamp":1714080345.5605879,"name":"offline","context":{"idset":"10759"}}
+{"timestamp":1714080345.5609932,"name":"offline","context":{"idset":"10760"}}
+{"timestamp":1714080345.5613923,"name":"offline","context":{"idset":"10761"}}
+{"timestamp":1714080345.5617912,"name":"offline","context":{"idset":"10762"}}
+{"timestamp":1714080345.5622129,"name":"offline","context":{"idset":"10763"}}
+{"timestamp":1714080345.5626125,"name":"offline","context":{"idset":"10764"}}
+{"timestamp":1714080345.5630167,"name":"offline","context":{"idset":"10765"}}
+{"timestamp":1714080345.5634751,"name":"offline","context":{"idset":"10766"}}
+{"timestamp":1714080345.5639412,"name":"offline","context":{"idset":"10767"}}
+{"timestamp":1714080345.564347,"name":"offline","context":{"idset":"10768"}}
+{"timestamp":1714080345.564744,"name":"offline","context":{"idset":"10769"}}
+{"timestamp":1714080345.5651474,"name":"offline","context":{"idset":"10770"}}
+{"timestamp":1714080345.5702436,"name":"offline","context":{"idset":"10771"}}
+{"timestamp":1714080345.5706446,"name":"offline","context":{"idset":"10772"}}
+{"timestamp":1714080345.5710461,"name":"offline","context":{"idset":"10773"}}
+{"timestamp":1714080345.5714409,"name":"offline","context":{"idset":"10774"}}
+{"timestamp":1714080345.5718422,"name":"offline","context":{"idset":"10775"}}
+{"timestamp":1714080345.5722339,"name":"offline","context":{"idset":"10776"}}
+{"timestamp":1714080345.5726244,"name":"offline","context":{"idset":"10777"}}
+{"timestamp":1714080345.5730214,"name":"offline","context":{"idset":"10778"}}
+{"timestamp":1714080345.5734122,"name":"offline","context":{"idset":"10779"}}
+{"timestamp":1714080345.573843,"name":"offline","context":{"idset":"10780"}}
+{"timestamp":1714080345.5742841,"name":"offline","context":{"idset":"10781"}}
+{"timestamp":1714080345.5749402,"name":"offline","context":{"idset":"10782"}}
+{"timestamp":1714080345.5807381,"name":"offline","context":{"idset":"10783"}}
+{"timestamp":1714080345.5811679,"name":"offline","context":{"idset":"10784"}}
+{"timestamp":1714080345.5815783,"name":"offline","context":{"idset":"10785"}}
+{"timestamp":1714080345.5819857,"name":"offline","context":{"idset":"10786"}}
+{"timestamp":1714080345.582376,"name":"offline","context":{"idset":"10787"}}
+{"timestamp":1714080345.5874312,"name":"offline","context":{"idset":"10788"}}
+{"timestamp":1714080345.5878279,"name":"offline","context":{"idset":"10789"}}
+{"timestamp":1714080345.5882134,"name":"offline","context":{"idset":"10790"}}
+{"timestamp":1714080345.5885971,"name":"offline","context":{"idset":"10791"}}
+{"timestamp":1714080345.5889881,"name":"offline","context":{"idset":"10792"}}
+{"timestamp":1714080345.5893729,"name":"offline","context":{"idset":"10793"}}
+{"timestamp":1714080345.5897563,"name":"offline","context":{"idset":"10794"}}
+{"timestamp":1714080345.590184,"name":"offline","context":{"idset":"10795"}}
+{"timestamp":1714080345.5908692,"name":"offline","context":{"idset":"10796"}}
+{"timestamp":1714080345.591542,"name":"offline","context":{"idset":"10797"}}
+{"timestamp":1714080345.5921831,"name":"offline","context":{"idset":"10798"}}
+{"timestamp":1714080345.5925703,"name":"offline","context":{"idset":"10799"}}
+{"timestamp":1714080345.5929577,"name":"offline","context":{"idset":"10800"}}
+{"timestamp":1714080345.5933497,"name":"offline","context":{"idset":"10801"}}
+{"timestamp":1714080345.5937805,"name":"offline","context":{"idset":"10802"}}
+{"timestamp":1714080345.5941699,"name":"offline","context":{"idset":"10803"}}
+{"timestamp":1714080345.5945475,"name":"offline","context":{"idset":"10804"}}
+{"timestamp":1714080345.5949388,"name":"offline","context":{"idset":"10805"}}
+{"timestamp":1714080345.5953145,"name":"offline","context":{"idset":"10806"}}
+{"timestamp":1714080345.5956879,"name":"offline","context":{"idset":"10807"}}
+{"timestamp":1714080345.5960648,"name":"offline","context":{"idset":"10808"}}
+{"timestamp":1714080345.5964396,"name":"offline","context":{"idset":"10809"}}
+{"timestamp":1714080345.5968223,"name":"offline","context":{"idset":"10810"}}
+{"timestamp":1714080345.5971968,"name":"offline","context":{"idset":"10811"}}
+{"timestamp":1714080345.5975995,"name":"offline","context":{"idset":"10812"}}
+{"timestamp":1714080345.5980089,"name":"offline","context":{"idset":"10813"}}
+{"timestamp":1714080345.5984163,"name":"offline","context":{"idset":"10814"}}
+{"timestamp":1714080345.5987921,"name":"offline","context":{"idset":"10815"}}
+{"timestamp":1714080345.599175,"name":"offline","context":{"idset":"10816"}}
+{"timestamp":1714080345.599546,"name":"offline","context":{"idset":"10817"}}
+{"timestamp":1714080345.5999217,"name":"offline","context":{"idset":"10818"}}
+{"timestamp":1714080345.6002922,"name":"offline","context":{"idset":"10819"}}
+{"timestamp":1714080345.6006632,"name":"offline","context":{"idset":"10820"}}
+{"timestamp":1714080345.6010404,"name":"offline","context":{"idset":"10821"}}
+{"timestamp":1714080345.6067379,"name":"offline","context":{"idset":"10822"}}
+{"timestamp":1714080345.6071222,"name":"offline","context":{"idset":"10823"}}
+{"timestamp":1714080345.6074951,"name":"offline","context":{"idset":"10824"}}
+{"timestamp":1714080345.6078749,"name":"offline","context":{"idset":"10825"}}
+{"timestamp":1714080345.6082466,"name":"offline","context":{"idset":"10826"}}
+{"timestamp":1714080345.6086159,"name":"offline","context":{"idset":"10827"}}
+{"timestamp":1714080345.6089933,"name":"offline","context":{"idset":"10828"}}
+{"timestamp":1714080345.6093605,"name":"offline","context":{"idset":"10829"}}
+{"timestamp":1714080345.6143885,"name":"offline","context":{"idset":"10830"}}
+{"timestamp":1714080345.6147566,"name":"offline","context":{"idset":"10831"}}
+{"timestamp":1714080345.6151371,"name":"offline","context":{"idset":"10832"}}
+{"timestamp":1714080345.6155028,"name":"offline","context":{"idset":"10833"}}
+{"timestamp":1714080345.6158733,"name":"offline","context":{"idset":"10834"}}
+{"timestamp":1714080345.616267,"name":"offline","context":{"idset":"10835"}}
+{"timestamp":1714080345.6169429,"name":"offline","context":{"idset":"10836"}}
+{"timestamp":1714080345.6175952,"name":"offline","context":{"idset":"10837"}}
+{"timestamp":1714080345.6181207,"name":"offline","context":{"idset":"10838"}}
+{"timestamp":1714080345.6184886,"name":"offline","context":{"idset":"10839"}}
+{"timestamp":1714080345.61886,"name":"offline","context":{"idset":"10840"}}
+{"timestamp":1714080345.6192236,"name":"offline","context":{"idset":"10841"}}
+{"timestamp":1714080345.6195862,"name":"offline","context":{"idset":"10842"}}
+{"timestamp":1714080345.6199529,"name":"offline","context":{"idset":"10843"}}
+{"timestamp":1714080345.6203134,"name":"offline","context":{"idset":"10844"}}
+{"timestamp":1714080345.6253293,"name":"offline","context":{"idset":"10845"}}
+{"timestamp":1714080345.6256912,"name":"offline","context":{"idset":"10846"}}
+{"timestamp":1714080345.6260545,"name":"offline","context":{"idset":"10847"}}
+{"timestamp":1714080345.6264114,"name":"offline","context":{"idset":"10848"}}
+{"timestamp":1714080345.6267681,"name":"offline","context":{"idset":"10849"}}
+{"timestamp":1714080345.6271305,"name":"offline","context":{"idset":"10850"}}
+{"timestamp":1714080345.6274879,"name":"offline","context":{"idset":"10851"}}
+{"timestamp":1714080345.6279557,"name":"offline","context":{"idset":"10852"}}
+{"timestamp":1714080345.628624,"name":"offline","context":{"idset":"10853"}}
+{"timestamp":1714080345.6292558,"name":"offline","context":{"idset":"10854"}}
+{"timestamp":1714080345.6296253,"name":"offline","context":{"idset":"10855"}}
+{"timestamp":1714080345.6299875,"name":"offline","context":{"idset":"10856"}}
+{"timestamp":1714080345.6303391,"name":"offline","context":{"idset":"10857"}}
+{"timestamp":1714080345.6306889,"name":"offline","context":{"idset":"10858"}}
+{"timestamp":1714080345.6310451,"name":"offline","context":{"idset":"10859"}}
+{"timestamp":1714080345.6313965,"name":"offline","context":{"idset":"10860"}}
+{"timestamp":1714080345.6317451,"name":"offline","context":{"idset":"10861"}}
+{"timestamp":1714080345.6320982,"name":"offline","context":{"idset":"10862"}}
+{"timestamp":1714080345.6324468,"name":"offline","context":{"idset":"10863"}}
+{"timestamp":1714080345.6327927,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1714080345.6331468,"name":"offline","context":{"idset":"10865"}}
+{"timestamp":1714080345.6393838,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1714080345.6397779,"name":"offline","context":{"idset":"10867"}}
+{"timestamp":1714080345.6404009,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1714080345.6410298,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1714080345.6415074,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1714080345.6418633,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1714080345.6422088,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1714080345.6425514,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1714080345.6429064,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1714080345.6432498,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1714080345.6435905,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1714080345.6439371,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1714080345.6442721,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1714080345.6446056,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1714080345.6449423,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1714080345.645273,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1714080345.6456034,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1714080345.6459403,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1714080345.6462665,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1714080345.646596,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1714080345.6469336,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1714080345.6472623,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1714080345.6475921,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1714080345.647928,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1714080345.6482584,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1714080345.6485863,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1714080345.6489196,"name":"offline","context":{"idset":"10894"}}
+{"timestamp":1714080345.6492453,"name":"offline","context":{"idset":"10895"}}
+{"timestamp":1714080345.64957,"name":"offline","context":{"idset":"10896"}}
+{"timestamp":1714080345.6499012,"name":"offline","context":{"idset":"10897"}}
+{"timestamp":1714080345.6503565,"name":"offline","context":{"idset":"10898"}}
+{"timestamp":1714080345.6506896,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1714080345.6510224,"name":"offline","context":{"idset":"10900"}}
+{"timestamp":1714080345.6513786,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1714080345.6518278,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1714080345.6524358,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1714080345.6530106,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1714080345.6534975,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1714080345.6538479,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1714080345.6541731,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1714080345.6544974,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1714080345.6548297,"name":"offline","context":{"idset":"10909"}}
+{"timestamp":1714080345.6551523,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1714080345.6554723,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1714080345.6557913,"name":"offline","context":{"idset":"10912"}}
+{"timestamp":1714080345.6561165,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1714080345.656435,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1714080345.6567543,"name":"offline","context":{"idset":"10915"}}
+{"timestamp":1714080345.6570766,"name":"offline","context":{"idset":"10916"}}
+{"timestamp":1714080345.6573942,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1714080345.6577103,"name":"offline","context":{"idset":"10918"}}
+{"timestamp":1714080345.6580307,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1714080345.6583464,"name":"offline","context":{"idset":"10920"}}
+{"timestamp":1714080345.6586607,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1714080345.6589785,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1714080345.6592929,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1714080345.6596084,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1714080345.6599278,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1714080345.6602402,"name":"offline","context":{"idset":"10926"}}
+{"timestamp":1714080345.6605523,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1714080345.6608701,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1714080345.6611805,"name":"offline","context":{"idset":"10929"}}
+{"timestamp":1714080345.6614904,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1714080345.6618078,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1714080345.6621172,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1714080345.6624262,"name":"offline","context":{"idset":"10933"}}
+{"timestamp":1714080345.6627345,"name":"offline","context":{"idset":"10934"}}
+{"timestamp":1714080345.6630495,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1714080345.663357,"name":"offline","context":{"idset":"10936"}}
+{"timestamp":1714080345.6636641,"name":"offline","context":{"idset":"10937"}}
+{"timestamp":1714080345.6639774,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1714080345.6642842,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1714080345.6645906,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1714080345.664921,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1714080345.6652634,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1714080345.6655726,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1714080345.6658869,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1714080345.6661913,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1714080345.6664937,"name":"offline","context":{"idset":"10946"}}
+{"timestamp":1714080345.6667967,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1714080345.6671059,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1714080345.6674073,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1714080345.6677082,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1714080345.6680145,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1714080345.6683149,"name":"offline","context":{"idset":"10952"}}
+{"timestamp":1714080345.668664,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1714080345.6690586,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1714080345.6693702,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1714080345.6696715,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1714080345.6699762,"name":"offline","context":{"idset":"10957"}}
+{"timestamp":1714080345.6702735,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1714080345.6705678,"name":"offline","context":{"idset":"10959"}}
+{"timestamp":1714080345.6708841,"name":"offline","context":{"idset":"10960"}}
+{"timestamp":1714080345.6711807,"name":"offline","context":{"idset":"10961"}}
+{"timestamp":1714080345.6714759,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1714080345.6717696,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1714080345.6800964,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1714080345.6806488,"name":"offline","context":{"idset":"10965"}}
+{"timestamp":1714080345.6812131,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1714080345.6817677,"name":"offline","context":{"idset":"10967"}}
+{"timestamp":1714080345.682327,"name":"offline","context":{"idset":"10968"}}
+{"timestamp":1714080345.6829119,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1714080345.6834927,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1714080345.6840725,"name":"offline","context":{"idset":"10971"}}
+{"timestamp":1714080345.6846483,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1714080345.6852324,"name":"offline","context":{"idset":"10973"}}
+{"timestamp":1714080345.6858134,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1714080345.6863856,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1714080345.6869488,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1714080345.6875114,"name":"offline","context":{"idset":"10977"}}
+{"timestamp":1714080345.6881022,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1714080345.6884506,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1714080345.6887443,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1714080345.6890419,"name":"offline","context":{"idset":"10981"}}
+{"timestamp":1714080345.6893299,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1714080345.6896164,"name":"offline","context":{"idset":"10983"}}
+{"timestamp":1714080345.6945431,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1714080345.6948428,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1714080345.6951318,"name":"offline","context":{"idset":"10986"}}
+{"timestamp":1714080345.695421,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1714080345.6957104,"name":"offline","context":{"idset":"10988"}}
+{"timestamp":1714080345.6960025,"name":"offline","context":{"idset":"10989"}}
+{"timestamp":1714080345.6962907,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1714080345.6965768,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1714080345.6968653,"name":"offline","context":{"idset":"10992"}}
+{"timestamp":1714080345.6971469,"name":"offline","context":{"idset":"10993"}}
+{"timestamp":1714080345.6974287,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1714080345.6977112,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1714080345.6979976,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1714080345.6982853,"name":"offline","context":{"idset":"10997"}}
+{"timestamp":1714080345.6985815,"name":"offline","context":{"idset":"10998"}}
+{"timestamp":1714080345.6988664,"name":"offline","context":{"idset":"10999"}}
+{"timestamp":1714080345.6991446,"name":"offline","context":{"idset":"11000"}}
+{"timestamp":1714080345.6994228,"name":"offline","context":{"idset":"11001"}}
+{"timestamp":1714080345.6997006,"name":"offline","context":{"idset":"11002"}}
+{"timestamp":1714080345.6999831,"name":"offline","context":{"idset":"11003"}}
+{"timestamp":1714080345.7002597,"name":"offline","context":{"idset":"11004"}}
+{"timestamp":1714080345.7005355,"name":"offline","context":{"idset":"11005"}}
+{"timestamp":1714080345.7008166,"name":"offline","context":{"idset":"11006"}}
+{"timestamp":1714080345.7010922,"name":"offline","context":{"idset":"11007"}}
+{"timestamp":1714080345.7013683,"name":"offline","context":{"idset":"11008"}}
+{"timestamp":1714080345.7016459,"name":"offline","context":{"idset":"11009"}}
+{"timestamp":1714080345.7019272,"name":"offline","context":{"idset":"11010"}}
+{"timestamp":1714080345.7022009,"name":"offline","context":{"idset":"11011"}}
+{"timestamp":1714080345.7024739,"name":"offline","context":{"idset":"11012"}}
+{"timestamp":1714080345.7027462,"name":"offline","context":{"idset":"11013"}}
+{"timestamp":1714080345.7030246,"name":"offline","context":{"idset":"11014"}}
+{"timestamp":1714080345.7032962,"name":"offline","context":{"idset":"11015"}}
+{"timestamp":1714080345.7035668,"name":"offline","context":{"idset":"11016"}}
+{"timestamp":1714080345.703846,"name":"offline","context":{"idset":"11017"}}
+{"timestamp":1714080345.7041161,"name":"offline","context":{"idset":"11018"}}
+{"timestamp":1714080345.7043853,"name":"offline","context":{"idset":"11019"}}
+{"timestamp":1714080345.7046535,"name":"offline","context":{"idset":"11020"}}
+{"timestamp":1714080345.7049286,"name":"offline","context":{"idset":"11021"}}
+{"timestamp":1714080345.7051966,"name":"offline","context":{"idset":"11022"}}
+{"timestamp":1714080345.7054665,"name":"offline","context":{"idset":"11023"}}
+{"timestamp":1714080345.7057345,"name":"offline","context":{"idset":"11024"}}
+{"timestamp":1714080345.7060072,"name":"offline","context":{"idset":"11025"}}
+{"timestamp":1714080345.715517,"name":"offline","context":{"idset":"11026"}}
+{"timestamp":1714080345.7157884,"name":"offline","context":{"idset":"11027"}}
+{"timestamp":1714080345.7160623,"name":"offline","context":{"idset":"11028"}}
+{"timestamp":1714080345.7163308,"name":"offline","context":{"idset":"11029"}}
+{"timestamp":1714080345.716598,"name":"offline","context":{"idset":"11030"}}
+{"timestamp":1714080345.7168679,"name":"offline","context":{"idset":"11031"}}
+{"timestamp":1714080345.7171357,"name":"offline","context":{"idset":"11032"}}
+{"timestamp":1714080345.7174017,"name":"offline","context":{"idset":"11033"}}
+{"timestamp":1714080345.7176681,"name":"offline","context":{"idset":"11034"}}
+{"timestamp":1714080345.7179396,"name":"offline","context":{"idset":"11035"}}
+{"timestamp":1714080345.718204,"name":"offline","context":{"idset":"11036"}}
+{"timestamp":1714080345.7184658,"name":"offline","context":{"idset":"11037"}}
+{"timestamp":1714080345.7187271,"name":"offline","context":{"idset":"11038"}}
+{"timestamp":1714080345.7189934,"name":"offline","context":{"idset":"11039"}}
+{"timestamp":1714080345.7192543,"name":"offline","context":{"idset":"11040"}}
+{"timestamp":1714080345.7195165,"name":"offline","context":{"idset":"11041"}}
+{"timestamp":1714080345.7290599,"name":"offline","context":{"idset":"11042"}}
+{"timestamp":1714080345.7293243,"name":"offline","context":{"idset":"11043"}}
+{"timestamp":1714080345.7295835,"name":"offline","context":{"idset":"11044"}}
+{"timestamp":1714080345.7298477,"name":"offline","context":{"idset":"11045"}}
+{"timestamp":1714080345.7301044,"name":"offline","context":{"idset":"11046"}}
+{"timestamp":1714080345.7303605,"name":"offline","context":{"idset":"11047"}}
+{"timestamp":1714080345.7306154,"name":"offline","context":{"idset":"11048"}}
+{"timestamp":1714080345.7308767,"name":"offline","context":{"idset":"11049"}}
+{"timestamp":1714080345.7311313,"name":"offline","context":{"idset":"11050"}}
+{"timestamp":1714080345.7313859,"name":"offline","context":{"idset":"11051"}}
+{"timestamp":1714080345.7316389,"name":"offline","context":{"idset":"11052"}}
+{"timestamp":1714080345.73194,"name":"offline","context":{"idset":"11053"}}
+{"timestamp":1714080345.732199,"name":"offline","context":{"idset":"11054"}}
+{"timestamp":1714080345.7324519,"name":"offline","context":{"idset":"11055"}}
+{"timestamp":1714080345.7327063,"name":"offline","context":{"idset":"11056"}}
+{"timestamp":1714080345.7329643,"name":"offline","context":{"idset":"11057"}}
+{"timestamp":1714080345.7332153,"name":"offline","context":{"idset":"11058"}}
+{"timestamp":1714080345.7334669,"name":"offline","context":{"idset":"11059"}}
+{"timestamp":1714080345.7337179,"name":"offline","context":{"idset":"11060"}}
+{"timestamp":1714080345.7339716,"name":"offline","context":{"idset":"11061"}}
+{"timestamp":1714080345.7342219,"name":"offline","context":{"idset":"11062"}}
+{"timestamp":1714080345.7344701,"name":"offline","context":{"idset":"11063"}}
+{"timestamp":1714080345.7347343,"name":"offline","context":{"idset":"11064"}}
+{"timestamp":1714080345.7349958,"name":"offline","context":{"idset":"11065"}}
+{"timestamp":1714080345.7352433,"name":"offline","context":{"idset":"11066"}}
+{"timestamp":1714080345.7354898,"name":"offline","context":{"idset":"11067"}}
+{"timestamp":1714080345.7357364,"name":"offline","context":{"idset":"11068"}}
+{"timestamp":1714080345.7359877,"name":"offline","context":{"idset":"11069"}}
+{"timestamp":1714080345.7362339,"name":"offline","context":{"idset":"11070"}}
+{"timestamp":1714080345.736479,"name":"offline","context":{"idset":"11071"}}
+{"timestamp":1714080345.7367251,"name":"offline","context":{"idset":"11072"}}
+{"timestamp":1714080345.7369757,"name":"offline","context":{"idset":"11073"}}
+{"timestamp":1714080345.7372191,"name":"offline","context":{"idset":"11074"}}
+{"timestamp":1714080345.7374625,"name":"offline","context":{"idset":"11075"}}
+{"timestamp":1714080345.7377052,"name":"offline","context":{"idset":"11076"}}
+{"timestamp":1714080345.7379556,"name":"offline","context":{"idset":"11077"}}
+{"timestamp":1714080345.7381971,"name":"offline","context":{"idset":"11078"}}
+{"timestamp":1714080345.738447,"name":"offline","context":{"idset":"11079"}}
+{"timestamp":1714080345.7439384,"name":"offline","context":{"idset":"11080"}}
+{"timestamp":1714080345.7441864,"name":"offline","context":{"idset":"11081"}}
+{"timestamp":1714080345.7444284,"name":"offline","context":{"idset":"11082"}}
+{"timestamp":1714080345.7446678,"name":"offline","context":{"idset":"11083"}}
+{"timestamp":1714080345.7449145,"name":"offline","context":{"idset":"11084"}}
+{"timestamp":1714080345.7451534,"name":"offline","context":{"idset":"11085"}}
+{"timestamp":1714080345.7453902,"name":"offline","context":{"idset":"11086"}}
+{"timestamp":1714080345.7456276,"name":"offline","context":{"idset":"11087"}}
+{"timestamp":1714080345.7458727,"name":"offline","context":{"idset":"11088"}}
+{"timestamp":1714080345.7461092,"name":"offline","context":{"idset":"11089"}}
+{"timestamp":1714080345.746345,"name":"offline","context":{"idset":"11090"}}
+{"timestamp":1714080345.7465799,"name":"offline","context":{"idset":"11091"}}
+{"timestamp":1714080345.7523606,"name":"offline","context":{"idset":"11092"}}
+{"timestamp":1714080345.7526138,"name":"offline","context":{"idset":"11093"}}
+{"timestamp":1714080345.7528603,"name":"offline","context":{"idset":"11094"}}
+{"timestamp":1714080345.7530968,"name":"offline","context":{"idset":"11095"}}
+{"timestamp":1714080345.7533295,"name":"offline","context":{"idset":"11096"}}
+{"timestamp":1714080345.7535646,"name":"offline","context":{"idset":"11097"}}
+{"timestamp":1714080345.7537975,"name":"offline","context":{"idset":"11098"}}
+{"timestamp":1714080345.7540364,"name":"offline","context":{"idset":"11099"}}
+{"timestamp":1714080345.7542686,"name":"offline","context":{"idset":"11100"}}
+{"timestamp":1714080345.7544982,"name":"offline","context":{"idset":"11101"}}
+{"timestamp":1714080345.7547264,"name":"offline","context":{"idset":"11102"}}
+{"timestamp":1714080345.7549591,"name":"offline","context":{"idset":"11103"}}
+{"timestamp":1714080345.7551863,"name":"offline","context":{"idset":"11104"}}
+{"timestamp":1714080345.7554128,"name":"offline","context":{"idset":"11105"}}
+{"timestamp":1714080345.7556386,"name":"offline","context":{"idset":"11106"}}
+{"timestamp":1714080345.7558703,"name":"offline","context":{"idset":"11107"}}
+{"timestamp":1714080345.7560952,"name":"offline","context":{"idset":"11108"}}
+{"timestamp":1714080345.756319,"name":"offline","context":{"idset":"11109"}}
+{"timestamp":1714080345.7565432,"name":"offline","context":{"idset":"11110"}}
+{"timestamp":1714080345.7613246,"name":"offline","context":{"idset":"11111"}}
+{"timestamp":1714080345.7615502,"name":"offline","context":{"idset":"11112"}}
+{"timestamp":1714080345.7617741,"name":"offline","context":{"idset":"11113"}}
+{"timestamp":1714080345.76228,"name":"offline","context":{"idset":"11114"}}
+{"timestamp":1714080345.7625043,"name":"offline","context":{"idset":"11115"}}
+{"timestamp":1714080345.7627258,"name":"offline","context":{"idset":"11116"}}
+{"timestamp":1714080345.7629573,"name":"offline","context":{"idset":"11117"}}
+{"timestamp":1714080345.7631776,"name":"offline","context":{"idset":"11118"}}
+{"timestamp":1714080345.763397,"name":"offline","context":{"idset":"11119"}}
+{"timestamp":1714080345.7636168,"name":"offline","context":{"idset":"11120"}}
+{"timestamp":1714080345.7638617,"name":"offline","context":{"idset":"11121"}}
+{"timestamp":1714080345.764082,"name":"offline","context":{"idset":"11122"}}
+{"timestamp":1714080345.7642999,"name":"offline","context":{"idset":"11123"}}
+{"timestamp":1714080345.7645168,"name":"offline","context":{"idset":"11124"}}
+{"timestamp":1714080345.7647347,"name":"offline","context":{"idset":"11125"}}
+{"timestamp":1714080345.7649601,"name":"offline","context":{"idset":"11126"}}
+{"timestamp":1714080345.7651763,"name":"offline","context":{"idset":"11127"}}
+{"timestamp":1714080345.7653918,"name":"offline","context":{"idset":"11128"}}
+{"timestamp":1714080345.7656076,"name":"offline","context":{"idset":"11129"}}
+{"timestamp":1714080345.765837,"name":"offline","context":{"idset":"11130"}}
+{"timestamp":1714080345.7660522,"name":"offline","context":{"idset":"11131"}}
+{"timestamp":1714080345.7662685,"name":"offline","context":{"idset":"11132"}}
+{"timestamp":1714080345.7664819,"name":"offline","context":{"idset":"11133"}}
+{"timestamp":1714080345.7666957,"name":"offline","context":{"idset":"11134"}}
+{"timestamp":1714080345.7669125,"name":"offline","context":{"idset":"11135"}}
+{"timestamp":1714080345.7671244,"name":"offline","context":{"idset":"11136"}}
+{"timestamp":1714080345.7673357,"name":"offline","context":{"idset":"11137"}}
+{"timestamp":1714080345.7675459,"name":"offline","context":{"idset":"11138"}}
+{"timestamp":1714080345.7677569,"name":"offline","context":{"idset":"11139"}}
+{"timestamp":1714080345.7679729,"name":"offline","context":{"idset":"11140"}}
+{"timestamp":1714080345.7681828,"name":"offline","context":{"idset":"11141"}}
+{"timestamp":1714080345.7739689,"name":"offline","context":{"idset":"11142"}}
+{"timestamp":1714080345.7741868,"name":"offline","context":{"idset":"11143"}}
+{"timestamp":1714080345.7743976,"name":"offline","context":{"idset":"11144"}}
+{"timestamp":1714080345.7746072,"name":"offline","context":{"idset":"11145"}}
+{"timestamp":1714080345.7748256,"name":"offline","context":{"idset":"11146"}}
+{"timestamp":1714080345.7750378,"name":"offline","context":{"idset":"11147"}}
+{"timestamp":1714080345.7752457,"name":"offline","context":{"idset":"11148"}}
+{"timestamp":1714080345.7754531,"name":"offline","context":{"idset":"11149"}}
+{"timestamp":1714080345.7756579,"name":"offline","context":{"idset":"11150"}}
+{"timestamp":1714080345.7758725,"name":"offline","context":{"idset":"11151"}}
+{"timestamp":1714080345.7760773,"name":"offline","context":{"idset":"11152"}}
+{"timestamp":1714080345.7762816,"name":"offline","context":{"idset":"11153"}}
+{"timestamp":1714080345.776484,"name":"offline","context":{"idset":"11154"}}
+{"timestamp":1714080345.7766874,"name":"offline","context":{"idset":"11155"}}
+{"timestamp":1714080345.7768977,"name":"offline","context":{"idset":"11156"}}
+{"timestamp":1714080345.7770996,"name":"offline","context":{"idset":"11157"}}
+{"timestamp":1714080345.7773004,"name":"offline","context":{"idset":"11158"}}
+{"timestamp":1714080345.7775011,"name":"offline","context":{"idset":"11159"}}
+{"timestamp":1714080345.7823017,"name":"offline","context":{"idset":"11160"}}
+{"timestamp":1714080345.7825038,"name":"offline","context":{"idset":"11161"}}
+{"timestamp":1714080345.7827027,"name":"offline","context":{"idset":"11162"}}
+{"timestamp":1714080345.7829087,"name":"offline","context":{"idset":"11163"}}
+{"timestamp":1714080345.7831073,"name":"offline","context":{"idset":"11164"}}
+{"timestamp":1714080345.7899921,"name":"offline","context":{"idset":"11165"}}
+{"timestamp":1714080345.7902787,"name":"offline","context":{"idset":"11166"}}
+{"timestamp":1714080345.7905681,"name":"offline","context":{"idset":"11167"}}
+{"timestamp":1714080345.7908709,"name":"offline","context":{"idset":"11168"}}
+{"timestamp":1714080345.7911549,"name":"offline","context":{"idset":"11169"}}
+{"timestamp":1714080345.7914367,"name":"offline","context":{"idset":"11170"}}
+{"timestamp":1714080345.7917206,"name":"offline","context":{"idset":"11171"}}
+{"timestamp":1714080345.7920067,"name":"offline","context":{"idset":"11172"}}
+{"timestamp":1714080345.792289,"name":"offline","context":{"idset":"11173"}}
+{"timestamp":1714080345.7925701,"name":"offline","context":{"idset":"11174"}}
+{"timestamp":1714080345.7928522,"name":"offline","context":{"idset":"11175"}}
+{"timestamp":1714080345.793133,"name":"offline","context":{"idset":"11176"}}
+{"timestamp":1714080345.7934139,"name":"offline","context":{"idset":"11177"}}
+{"timestamp":1714080345.7936954,"name":"offline","context":{"idset":"11178"}}
+{"timestamp":1714080345.7939851,"name":"offline","context":{"idset":"11179"}}
+{"timestamp":1714080345.794265,"name":"offline","context":{"idset":"11180"}}
+{"timestamp":1714080345.8003995,"name":"offline","context":{"idset":"11181"}}
+{"timestamp":1714080345.800595,"name":"offline","context":{"idset":"11182"}}
+{"timestamp":1714080345.8007855,"name":"offline","context":{"idset":"11183"}}
+{"timestamp":1714080345.8009832,"name":"offline","context":{"idset":"11184"}}
+{"timestamp":1714080345.8011727,"name":"offline","context":{"idset":"11185"}}
+{"timestamp":1714080345.8013639,"name":"offline","context":{"idset":"11186"}}
+{"timestamp":1714080345.801554,"name":"offline","context":{"idset":"11187"}}
+{"timestamp":1714080345.8017573,"name":"offline","context":{"idset":"11188"}}
+{"timestamp":1714080345.8019636,"name":"offline","context":{"idset":"11189"}}
+{"timestamp":1714080345.8021545,"name":"offline","context":{"idset":"11190"}}
+{"timestamp":1714080345.8023412,"name":"offline","context":{"idset":"11191"}}
+{"timestamp":1714080345.8025272,"name":"offline","context":{"idset":"11192"}}
+{"timestamp":1714080345.8027129,"name":"offline","context":{"idset":"11193"}}
+{"timestamp":1714080345.8029037,"name":"offline","context":{"idset":"11194"}}
+{"timestamp":1714080345.8030891,"name":"offline","context":{"idset":"11195"}}
+{"timestamp":1714080345.8032789,"name":"offline","context":{"idset":"11196"}}
+{"timestamp":1714080345.8034675,"name":"offline","context":{"idset":"11197"}}
+{"timestamp":1714080345.8036542,"name":"offline","context":{"idset":"11198"}}
+{"timestamp":1714080345.8038449,"name":"offline","context":{"idset":"11199"}}
+{"timestamp":1714080345.8040278,"name":"offline","context":{"idset":"11200"}}
+{"timestamp":1714080345.8042097,"name":"offline","context":{"idset":"11201"}}
+{"timestamp":1714080345.8043907,"name":"offline","context":{"idset":"11202"}}
+{"timestamp":1714080345.8045712,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1714080345.8047516,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1714080345.8049357,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1714080345.8097177,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1714080345.8099051,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1714080345.8100863,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1714080345.8102636,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1714080345.810442,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1714080345.8106203,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1714080345.810797,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1714080345.8109784,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1714080345.8111534,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1714080345.8113284,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1714080345.8115025,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1714080345.8116765,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1714080345.8118682,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1714080345.8120444,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1714080345.8122168,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1714080345.8123887,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1714080345.8125598,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1714080345.8127325,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1714080345.8129101,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1714080345.8130808,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1714080345.817842,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1714080345.8180144,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1714080345.8181825,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1714080345.8229551,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1714080345.8231301,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1714080345.8232965,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1714080345.8234622,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1714080345.8236279,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1714080345.8237925,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1714080345.8239651,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1714080345.8241291,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1714080345.8242924,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1714080345.8244543,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1714080345.8246162,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1714080345.8247757,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1714080345.8250251,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1714080345.8251922,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1714080345.8253531,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1714080345.8255126,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1714080345.8256714,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1714080345.8258498,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1714080345.8260098,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1714080345.8261688,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1714080345.8263273,"name":"offline","context":{"idset":"11517"}}
+{"timestamp":1714080345.8264847,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1714080345.8266413,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1714080345.8267977,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1714080345.8269653,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1714080345.8271196,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1714080345.8272722,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1714080345.8274252,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1714080345.8321636,"name":"offline","context":{"idset":"11525"}}
+{"timestamp":1714080345.8323181,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1714080345.8324709,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1714080345.8326228,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1714080345.8327749,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1714080345.8329363,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1714080345.8330908,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1714080345.8332419,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1714080345.8333917,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1714080345.83354,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1714080345.833688,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1714080345.8338423,"name":"offline","context":{"idset":"11536"}}
+{"timestamp":1714080345.8339894,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1714080345.8341355,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1714080345.8342814,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1714080345.8344309,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1714080345.8345757,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1714080345.8347204,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1714080345.8348711,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1714080345.8350136,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1714080345.835156,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1714080345.8352973,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1714080345.8354375,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1714080345.8355782,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1714080345.8357182,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1714080345.8358653,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1714080345.8360062,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1714080345.8361452,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1714080345.8362834,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1714080345.8364213,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1714080345.8365595,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1714080345.8366964,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1714080345.8368397,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1714080345.8369749,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1714080345.8371112,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1714080345.8372455,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1714080345.8373795,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1714080345.8375125,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1714080345.8376451,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1714080345.837779,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1714080345.8379185,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1714080345.8380497,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1714080345.838181,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1714080345.8383117,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1714080345.8384414,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1714080345.8385725,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1714080345.8387024,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1714080345.8388388,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1714080345.8389671,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1714080345.8390949,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1714080345.8392231,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1714080345.8393502,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1714080345.8394759,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1714080345.839601,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1714080345.8397262,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1714080345.8398581,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1714080345.839983,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1714080345.8401065,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1714080345.8402293,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1714080345.8403521,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1714080345.8404744,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1714080345.8405974,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1714080345.8407207,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1714080345.8408508,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1714080345.8409736,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1714080345.8410947,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1714080345.8412156,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1714080345.841336,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1714080345.8414545,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1714080345.841573,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1714080345.841692,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1714080345.8418217,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1714080345.8419404,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1714080345.8420565,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1714080345.8421726,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1714080345.8422883,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1714080345.8424044,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1714080345.8425193,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1714080345.8426349,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1714080345.8427498,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1714080345.8428726,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1714080345.8429873,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1714080345.8430996,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1714080345.8432109,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1714080345.8433228,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1714080345.8434331,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1714080345.8435426,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1714080345.8436522,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1714080345.8437622,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1714080345.8438799,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1714080345.8439882,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1714080345.8440955,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1714080345.844203,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1714080345.8443096,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1714080345.8444154,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1714080345.8445208,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1714080345.8446267,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1714080345.8447309,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1714080345.8448462,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1714080345.84495,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1714080345.8450539,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1714080345.8451555,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1714080345.8452597,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1714080345.8453624,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1714080345.8454661,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1714080345.8455667,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1714080345.8456666,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1714080345.8457663,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1714080345.8508444,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1714080345.8509502,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1714080345.8510501,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1714080345.8511491,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1714080345.8512461,"name":"offline","context":{"idset":"11658"}}
+{"timestamp":1714080345.8574867,"name":"offline","context":{"idset":"11765"}}
+{"timestamp":1714080345.8576295,"name":"offline","context":{"idset":"11766"}}
+{"timestamp":1714080345.8577681,"name":"offline","context":{"idset":"11767"}}
+{"timestamp":1714080345.8579102,"name":"offline","context":{"idset":"11768"}}
+{"timestamp":1714080345.8580494,"name":"offline","context":{"idset":"11769"}}
+{"timestamp":1714080345.8581777,"name":"offline","context":{"idset":"11770"}}
+{"timestamp":1714080345.858304,"name":"offline","context":{"idset":"11771"}}
+{"timestamp":1714080345.8584306,"name":"offline","context":{"idset":"11772"}}
+{"timestamp":1714080345.8585556,"name":"offline","context":{"idset":"11773"}}
+{"timestamp":1714080345.8586693,"name":"offline","context":{"idset":"11774"}}
+{"timestamp":1714080345.858758,"name":"offline","context":{"idset":"11775"}}
+{"timestamp":1714080345.8588543,"name":"offline","context":{"idset":"11776"}}
+{"timestamp":1714080345.8589435,"name":"offline","context":{"idset":"11777"}}
+{"timestamp":1714080345.8590326,"name":"offline","context":{"idset":"11778"}}
+{"timestamp":1714080345.8591192,"name":"offline","context":{"idset":"11779"}}
+{"timestamp":1714080345.8592145,"name":"offline","context":{"idset":"11780"}}
+{"timestamp":1714080345.8593018,"name":"offline","context":{"idset":"11781"}}
+{"timestamp":1714080345.8593917,"name":"offline","context":{"idset":"11782"}}
+{"timestamp":1714080345.859529,"name":"offline","context":{"idset":"11783"}}
+{"timestamp":1714080345.8596575,"name":"offline","context":{"idset":"11784"}}
+{"timestamp":1714080345.8597422,"name":"offline","context":{"idset":"11785"}}
+{"timestamp":1714080345.859832,"name":"offline","context":{"idset":"11786"}}
+{"timestamp":1714080345.8599153,"name":"offline","context":{"idset":"11787"}}
+{"timestamp":1714080345.859997,"name":"offline","context":{"idset":"11788"}}
+{"timestamp":1714080345.8600786,"name":"offline","context":{"idset":"11791"}}
+{"timestamp":1714080345.8601594,"name":"offline","context":{"idset":"11792"}}
+{"timestamp":1714080345.8602419,"name":"offline","context":{"idset":"11793"}}
+{"timestamp":1714080345.8603215,"name":"offline","context":{"idset":"11794"}}
+{"timestamp":1714080345.860419,"name":"offline","context":{"idset":"11795"}}
+{"timestamp":1714080345.8605008,"name":"offline","context":{"idset":"11796"}}
+{"timestamp":1714080345.8605797,"name":"offline","context":{"idset":"11797"}}
+{"timestamp":1714080345.8606582,"name":"offline","context":{"idset":"11798"}}
+{"timestamp":1714080345.8607361,"name":"offline","context":{"idset":"11799"}}
+{"timestamp":1714080345.8608222,"name":"offline","context":{"idset":"11800"}}
+{"timestamp":1714080345.8608994,"name":"offline","context":{"idset":"11801"}}
+{"timestamp":1714080345.8609769,"name":"offline","context":{"idset":"11802"}}
+{"timestamp":1714080345.8610523,"name":"offline","context":{"idset":"11803"}}
+{"timestamp":1714080345.8611269,"name":"offline","context":{"idset":"11804"}}
+{"timestamp":1714080345.861201,"name":"offline","context":{"idset":"11805"}}
+{"timestamp":1714080345.8612754,"name":"offline","context":{"idset":"11806"}}
+{"timestamp":1714080345.8659055,"name":"offline","context":{"idset":"11807"}}
+{"timestamp":1714080345.8659828,"name":"offline","context":{"idset":"11808"}}
+{"timestamp":1714080345.8660564,"name":"offline","context":{"idset":"11809"}}
+{"timestamp":1714080345.8661299,"name":"offline","context":{"idset":"11810"}}
+{"timestamp":1714080345.8662026,"name":"offline","context":{"idset":"11811"}}
+{"timestamp":1714080345.8662741,"name":"offline","context":{"idset":"11812"}}
+{"timestamp":1714080345.870955,"name":"offline","context":{"idset":"11813"}}
+{"timestamp":1714080345.8710315,"name":"offline","context":{"idset":"11814"}}
+{"timestamp":1714080345.871104,"name":"offline","context":{"idset":"11815"}}
+{"timestamp":1714080345.871177,"name":"offline","context":{"idset":"11816"}}
+{"timestamp":1714080345.871248,"name":"offline","context":{"idset":"11817"}}
+{"timestamp":1714080345.8713193,"name":"offline","context":{"idset":"11818"}}
+{"timestamp":1714080345.8713901,"name":"offline","context":{"idset":"11819"}}
+{"timestamp":1714080345.8714609,"name":"offline","context":{"idset":"11820"}}
+{"timestamp":1714080345.8715296,"name":"offline","context":{"idset":"11821"}}
+{"timestamp":1714080345.8715968,"name":"offline","context":{"idset":"11822"}}
+{"timestamp":1714080345.8716631,"name":"offline","context":{"idset":"11823"}}
+{"timestamp":1714080345.8717279,"name":"offline","context":{"idset":"11824"}}
+{"timestamp":1714080345.8717918,"name":"offline","context":{"idset":"11827"}}
+{"timestamp":1714080345.8718626,"name":"offline","context":{"idset":"11828"}}
+{"timestamp":1714080345.8719265,"name":"offline","context":{"idset":"11829"}}
+{"timestamp":1714080345.8719897,"name":"offline","context":{"idset":"11830"}}
+{"timestamp":1714080345.872052,"name":"offline","context":{"idset":"11831"}}
+{"timestamp":1714080345.8721137,"name":"offline","context":{"idset":"11832"}}
+{"timestamp":1714080345.8721766,"name":"offline","context":{"idset":"11833"}}
+{"timestamp":1714080345.8722382,"name":"offline","context":{"idset":"11834"}}
+{"timestamp":1714080345.8722987,"name":"offline","context":{"idset":"11835"}}
+{"timestamp":1714080345.8723583,"name":"offline","context":{"idset":"11836"}}
+{"timestamp":1714080345.8724179,"name":"offline","context":{"idset":"11837"}}
+{"timestamp":1714080345.8724763,"name":"offline","context":{"idset":"11838"}}
+{"timestamp":1714080345.8725345,"name":"offline","context":{"idset":"11839"}}
+{"timestamp":1714080345.8725924,"name":"offline","context":{"idset":"11840"}}
+{"timestamp":1714080345.8726497,"name":"offline","context":{"idset":"11841"}}
+{"timestamp":1714080345.8727064,"name":"offline","context":{"idset":"11842"}}
+{"timestamp":1714080345.8727632,"name":"offline","context":{"idset":"11843"}}
+{"timestamp":1714080345.8728271,"name":"offline","context":{"idset":"11844"}}
+{"timestamp":1714080345.8728836,"name":"offline","context":{"idset":"11845"}}
+{"timestamp":1714080345.8729382,"name":"offline","context":{"idset":"11846"}}
+{"timestamp":1714080345.8729928,"name":"offline","context":{"idset":"11847"}}
+{"timestamp":1714080345.8730464,"name":"offline","context":{"idset":"11848"}}
+{"timestamp":1714080345.8731008,"name":"offline","context":{"idset":"11849"}}
+{"timestamp":1714080345.8731537,"name":"offline","context":{"idset":"11850"}}
+{"timestamp":1714080345.8732073,"name":"offline","context":{"idset":"11851"}}
+{"timestamp":1714080345.8732595,"name":"offline","context":{"idset":"11852"}}
+{"timestamp":1714080345.8733115,"name":"offline","context":{"idset":"11853"}}
+{"timestamp":1714080345.8733621,"name":"offline","context":{"idset":"11854"}}
+{"timestamp":1714080345.873414,"name":"offline","context":{"idset":"11855"}}
+{"timestamp":1714080345.8734643,"name":"offline","context":{"idset":"11856"}}
+{"timestamp":1714080345.8735142,"name":"offline","context":{"idset":"11857"}}
+{"timestamp":1714080345.8781471,"name":"offline","context":{"idset":"11858"}}
+{"timestamp":1714080345.8781979,"name":"offline","context":{"idset":"11859"}}
+{"timestamp":1714080345.878248,"name":"offline","context":{"idset":"11860"}}
+{"timestamp":1714080345.8782961,"name":"offline","context":{"idset":"11861"}}
+{"timestamp":1714080345.8783441,"name":"offline","context":{"idset":"11862"}}
+{"timestamp":1714080345.878391,"name":"offline","context":{"idset":"11863"}}
+{"timestamp":1714080345.8784373,"name":"offline","context":{"idset":"11864"}}
+{"timestamp":1714080345.8784828,"name":"offline","context":{"idset":"11865"}}
+{"timestamp":1714080345.8785281,"name":"offline","context":{"idset":"11866"}}
+{"timestamp":1714080345.8785729,"name":"offline","context":{"idset":"11867"}}
+{"timestamp":1714080345.8786175,"name":"offline","context":{"idset":"11868"}}
+{"timestamp":1714080345.8786616,"name":"offline","context":{"idset":"11869"}}
+{"timestamp":1714080345.8787053,"name":"offline","context":{"idset":"11870"}}
+{"timestamp":1714080345.8787482,"name":"offline","context":{"idset":"11871"}}
+{"timestamp":1714080345.8787904,"name":"offline","context":{"idset":"11872"}}
+{"timestamp":1714080345.8788383,"name":"offline","context":{"idset":"11873"}}
+{"timestamp":1714080345.8788803,"name":"offline","context":{"idset":"11874"}}
+{"timestamp":1714080345.8789213,"name":"offline","context":{"idset":"11875"}}
+{"timestamp":1714080345.878962,"name":"offline","context":{"idset":"11876"}}
+{"timestamp":1714080345.8790023,"name":"offline","context":{"idset":"11877"}}
+{"timestamp":1714080345.8790419,"name":"offline","context":{"idset":"11878"}}
+{"timestamp":1714080345.8790808,"name":"offline","context":{"idset":"11879"}}
+{"timestamp":1714080345.8791199,"name":"offline","context":{"idset":"11880"}}
+{"timestamp":1714080345.8791578,"name":"offline","context":{"idset":"11881"}}
+{"timestamp":1714080345.8791955,"name":"offline","context":{"idset":"11882"}}
+{"timestamp":1714080345.8792322,"name":"offline","context":{"idset":"11883"}}
+{"timestamp":1714080345.8792689,"name":"offline","context":{"idset":"11884"}}
+{"timestamp":1714080345.8793049,"name":"offline","context":{"idset":"11885"}}
+{"timestamp":1714080345.8793406,"name":"offline","context":{"idset":"11886"}}
+{"timestamp":1714080345.8793759,"name":"offline","context":{"idset":"11887"}}
+{"timestamp":1714080345.8794107,"name":"offline","context":{"idset":"11888"}}
+{"timestamp":1714080345.8794444,"name":"offline","context":{"idset":"11889"}}
+{"timestamp":1714080345.8794785,"name":"offline","context":{"idset":"11890"}}
+{"timestamp":1714080345.8795114,"name":"offline","context":{"idset":"11891"}}
+{"timestamp":1714080345.8795242,"name":"offline","context":{"idset":"11892"}}
+{"timestamp":1714080393.7851319,"name":"online","context":{"idset":"574"}}
+{"timestamp":1714080394.208977,"name":"offline","context":{"idset":"574"}}
+{"timestamp":1714080507.1596634,"name":"online","context":{"idset":"9067"}}
+{"timestamp":1714080507.4049084,"name":"online","context":{"idset":"9065"}}
+{"timestamp":1714080507.5054986,"name":"offline","context":{"idset":"9067"}}
+{"timestamp":1714080507.6154206,"name":"online","context":{"idset":"9061,9064,9066"}}
+{"timestamp":1714080507.7217026,"name":"online","context":{"idset":"9062-9063,9068"}}
+{"timestamp":1714080507.8219697,"name":"offline","context":{"idset":"9065"}}
+{"timestamp":1714080507.936723,"name":"offline","context":{"idset":"9066"}}
+{"timestamp":1714080507.9555368,"name":"offline","context":{"idset":"9064"}}
+{"timestamp":1714080507.9730632,"name":"offline","context":{"idset":"9061"}}
+{"timestamp":1714080508.0213196,"name":"offline","context":{"idset":"9063"}}
+{"timestamp":1714080508.0408418,"name":"offline","context":{"idset":"9068"}}
+{"timestamp":1714080508.1362824,"name":"offline","context":{"idset":"9062"}}
+{"timestamp":1714080669.5895042,"name":"resource-init","context":{"restart":true,"drain":{"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"98,100,102,111-112,114,123-124,161,184,187,190,208,303-304,310,317,319,330":{"timestamp":1714079956.0,"reason":"epilog failed for jobid frcnKxdu6xP"},"11377":{"timestamp":1713928672.0,"reason":"nodediag failed amdapu"},"543":{"timestamp":1714074108.0,"reason":"nodediag failed cxi"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"8597-8612":{"timestamp":1714067165.0,"reason":"--reason draining to run on-node diags - wendy"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"796":{"timestamp":1712868054.0,"reason":"broker was unresponsive"},"119":{"timestamp":1714079565.0,"reason":"unreachable"},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"},"795":{"timestamp":1712868053.0,"reason":"broker was unresponsive"},"372":{"timestamp":1713981581.0,"reason":"New Blade Installation --JRG"},"336,491":{"timestamp":1713230524.0,"reason":"Down"},"929-930":{"timestamp":1713552018.0,"reason":""},"773-774":{"timestamp":1713802805.0,"reason":""},"77-84":{"timestamp":1713795613.0,"reason":"New blade installation"},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"11826":{"timestamp":1713979977.0,"reason":""},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"115":{"timestamp":1714079493.0,"reason":"unreachable"},"9445-9460":{"timestamp":1714053592.0,"reason":"--reason down for backplain work -kpn"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"117":{"timestamp":1713981538.0,"reason":"Unreachable"},"360":{"timestamp":1713981579.0,"reason":"New Blade Installation --JRG"},"411":{"timestamp":1713564989.0,"reason":"Unreachable"},"11253-11268":{"timestamp":1713972658.0,"reason":"draining KPN"},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"807":{"timestamp":1713376017.0,"reason":""},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"811-812":{"timestamp":1713557753.0,"reason":"--reason Running hpe diags - JM"},"10310":{"timestamp":1713953621.0,"reason":"nodediag failed bogomips"},"8431":{"timestamp":1714066666.0,"reason":"--reason rebooting - wendy"},"10341":{"timestamp":1713928487.0,"reason":"nodediag failed bogomips"},"779-780":{"timestamp":1713907642.0,"reason":"--reason Troubleshoot BIOS issue"},"11725":{"timestamp":1713892052.0,"reason":"nodediag failed amdapu"},"11285-11300":{"timestamp":1713975164.0,"reason":"--reason rebooting switches - INIT -kpn"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"11658":{"timestamp":1713954013.0,"reason":"epilog failed for jobid frLfYxGth6w"},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"286":{"timestamp":1713981553.0,"reason":"Unreachable"},"808":{"timestamp":1713455939.0,"reason":""},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"110":{"timestamp":1714079561.0,"reason":"unreachable"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"544":{"timestamp":1714069881.0,"reason":"nodediag failed cxi"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"11374":{"timestamp":1713926801.0,"reason":"nodediag failed amdapu"},"206":{"timestamp":1713564985.0,"reason":"Unreachable"},"349-359,361-371,374-396,398-404":{"timestamp":1714060395.0,"reason":"New Blade Installation --JRG"},"8181,8183,8234,8237,8291,8304":{"timestamp":1713810085.0,"reason":"testing -kk"},"11373":{"timestamp":1713928485.0,"reason":"nodediag failed amdapu"},"9349-9364":{"timestamp":1714021089.0,"reason":"--reason back plane fix -KPN"},"545":{"timestamp":1714069878.0,"reason":"nodediag failed cxi"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"878":{"timestamp":1714078484.0,"reason":"nodediag failed amdapu"},"141-148":{"timestamp":1713545597.0,"reason":"ARP Storm Problem testing"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"9075":{"timestamp":1712864130.0,"reason":"broker was unresponsive"},"542":{"timestamp":1714074109.0,"reason":"nodediag failed cxi"},"11370":{"timestamp":1713930154.0,"reason":"nodediag failed amdapu"},"9333-9348":{"timestamp":1714020725.0,"reason":"--reason back plane fix -KPN"},"573-580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"149-156":{"timestamp":1713545634.0,"reason":"ARP Storm Problem testing"},"165":{"timestamp":1713981434.0,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261863.0,"reason":"--reason swapping blades into x1900 - wendy"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"11717-11720,11722-11724,11726,11728,11730,11732":{"timestamp":1713983852.0,"reason":"--force --reason rebooting switches - INIT -kpn"},"172":{"timestamp":1713981544.0,"reason":"Unreachable"},"11324":{"timestamp":1713902339.0,"reason":"nodediag failed amdapu"},"11789":{"timestamp":1713910767.0,"reason":"testing -kk"},"373":{"timestamp":1713981698.0,"reason":"New Blade Installation --JRG"},"541":{"timestamp":1714074106.0,"reason":"nodediag failed cxi"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"787-788":{"timestamp":1713219302.0,"reason":""},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"10233,10248":{"timestamp":1713746450.0,"reason":"broker was unresponsive"},"85-89,91-92":{"timestamp":1714079045.0,"reason":"New Blade installation"},"963-964":{"timestamp":1713196418.0,"reason":"moving blade"},"937-938":{"timestamp":1712866351.0,"reason":""},"10187":{"timestamp":1713980344.0,"reason":""},"925-926":{"timestamp":1713540640.0,"reason":""},"461-476":{"timestamp":1713794797.0,"reason":"New Blade Installation --JRG"},"397":{"timestamp":1713229804.0,"reason":"New Blade Installation --JRG"},"9365-9380":{"timestamp":1714021383.0,"reason":"--reason back plane fix -KPN"},"9003":{"timestamp":1712873964.0,"reason":"nodediag failed dgemm_perf"},"421-460":{"timestamp":1713794410.0,"reason":"New Blade Installation --JRG"},"410":{"timestamp":1713378635.0,"reason":"prolog failed for jobid fq5jtQQSQnT"},"120":{"timestamp":1713981548.0,"reason":"Unreachable"},"803-804":{"timestamp":1713479733.0,"reason":"status"},"789-794":{"timestamp":1711461777.0,"reason":""},"191":{"timestamp":1714078824.0,"reason":"unreachable"},"11729":{"timestamp":1713951714.0,"reason":"nodediag failed amdapu"},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"6517-6644":{"timestamp":1714079999.0,"reason":"--reason draining to run on-node diags - wendy"},"961-962":{"timestamp":1713197674.0,"reason":""},"482-490,492":{"timestamp":1713915346.0,"reason":"New Blade Installation --JRG"},"11369":{"timestamp":1713928726.0,"reason":"nodediag failed amdapu"},"342":{"timestamp":1713981566.0,"reason":"Unreachable"},"923-924":{"timestamp":1714064541.0,"reason":"status"},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"546":{"timestamp":1714069873.0,"reason":"nodediag failed cxi pci"},"549-556":{"timestamp":1713553592.0,"reason":"New Blade Installation"},"8833":{"timestamp":1713991869.0,"reason":"nodediag failed amdapu"},"90":{"timestamp":1713564981.0,"reason":"New Blade installation"},"11721":{"timestamp":1713951748.0,"reason":"nodediag failed amdapu"},"9054":{"timestamp":1712864116.0,"reason":"broker was unresponsive"},"771-772":{"timestamp":1713195979.0,"reason":""},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"877":{"timestamp":1714078512.0,"reason":"nodediag failed amdapu"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"8982":{"timestamp":1713996208.0,"reason":"broker was unresponsive"},"939":{"timestamp":1713963885.0,"reason":"nodediag failed amdapu"},"547":{"timestamp":1713563916.0,"reason":"epilog failed for jobid fqUWmieM3AX"},"420":{"timestamp":1714074112.0,"reason":"epilog failed for jobid frbYvBLD1gT"},"9429-9444":{"timestamp":1714053161.0,"reason":"--reason down for backplain work -kpn"},"814":{"timestamp":1712783332.0,"reason":""},"871-874":{"timestamp":1714057817.0,"reason":""},"829-830":{"timestamp":1712700945.0,"reason":""},"133-140":{"timestamp":1713545617.0,"reason":"ARP Storm Problem testing"},"525-532":{"timestamp":1713290620.0,"reason":""},"548":{"timestamp":1713563915.0,"reason":"epilog failed for jobid fqUWkS8j8hM"},"11731":{"timestamp":1713889305.0,"reason":"broker was unresponsive"},"335":{"timestamp":1713981557.0,"reason":"Unreachable"},"934":{"timestamp":1713967835.0,"reason":"epilog failed for jobid frMYzSv7G6b"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1714080669.6101382,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1714080679.0008514,"name":"online","context":{"idset":"0"}}
+{"timestamp":1714080680.8079619,"name":"online","context":{"idset":"7678,7839,7860,7869,7871,7876,8020,8047,8186,8269,8298,8306,8309,8334,8343,8370,8374,8414,8860,8891,8896,9150,9199,9566,9606,9616,9674,10022,10028,10036,10116,10441,10496,10526,10578,10586,10705,10728,10754,10826,10851,10965,10997,11067,11087,11232"}}
+{"timestamp":1714080680.9611907,"name":"online","context":{"idset":"883,7669,7763,7777-7778,7787,7794,7796,7813,7816,7872,7882,7886,7907,7909-7910,7923,7939,7960,7965,7973,7985,8002,8006,8038,8057,8067,8090,8094-8096,8129,8140,8189,8214,8217,8261,8280,8292,8338,8360,8362-8363,8382,8435,8830,8852,8868,8873,8885,8887,8927,8940,8969,9077,9090-9091,9096,9133,9154-9155,9158-9159,9185,9187,9191,9194,9197-9198,9213,9216,9254,9278,9284,9302,9315,9321,9330,9466,9492,9508,9524,9569,9579,9598,9610,9623,9658,9673,9695,9711,9729,9754,9803,9807,9810,9826,9832,9938,9951,9997,9999,10019,10023,10026,10102,10115,10125,10144,10155,10183,10192,10374,10377,10401,10406,10414,10416,10420,10439,10444,10481,10493,10511,10517,10519,10550,10577,10587-10588,10614,10628,10652,10663,10695,10700,10703-10704,10708,10712,10722-10723,10737,10795,10811,10814,10839,10845,10857,10869,10873,10890,10893,10919,10924,10933,10939,10953,10964,10985,11031-11032,11050,11054,11098,11112,11125-11126,11136,11139,11143,11171,11182,11207,11214,11515,11517,11521,11523,11532,11553-11554,11557,11566,11585,11588,11591,11599,11613,11627,11634,11777,11779,11781,11798,11800,11802,11805,11809-11810,11815,11827,11832-11835,11839,11847,11851,11853,11866,11890-11891"}}
+{"timestamp":1714080681.137558,"name":"online","context":{"idset":"7677,7692-7693,7697,7700,7707,7714,7717,7727,7737,7742,7745,7752,7755,7793,7797,7801,7809,7824,7828,7841,7850,7856,7861,7878,7881,7891,7897,7905,7908,7912,7914,7929,7934,7936-7937,7947,7961-7962,7975,7977,8027,8036,8039,8043,8073,8085,8092,8100,8118-8119,8128,8131,8138,8153,8181,8196,8205,8211,8222,8228,8240,8255,8262,8281,8291,8305,8308,8311,8313,8320,8329,8349,8359,8367,8369,8373,8380-8381,8387,8407,8604,8608,8821,8845,8855-8856,8894-8895,8901,8939,8942,8944-8945,8947-8948,8970,8972,9081,9092,9100,9114,9118,9125-9126,9137-9138,9142-9143,9148-9149,9157,9163,9165,9167,9173,9175,9190,9200,9208,9215,9229,9249,9259,9264,9267,9273,9283,9293,9297,9301,9305,9313,9319,9327,9331,9461,9474,9478,9480,9485,9498,9501,9516,9530-9531,9557,9576,9583,9586-9587,9596,9609,9634,9641,9648,9651,9653,9666,9686-9687,9703-9704,9709,9719-9720,9722,9724-9725,9730-9732,9762,9774,9779,9781,9784,9790,9796-9797,9799,9805,9812-9813,9816,9822,9827,9830,9958-9959,9968,9971,9973,9975,9989,10003,10006,10014,10033-10034,10104,10110,10133-10134,10139,10157,10159,10163,10165,10167,10172,10186,10194,10359,10363-10365,10372,10382,10384,10399-10400,10415,10419,10433,10438,10454-10458,10474,10480,10489-10490,10498,10500,10502,10537,10540,10542,10567,10594,10604,10629,10641,10644,10649,10651,10661,10680,10683-10684,10690,10697,10701,10716,10727,10733,10761,10763,10773-10774,10781,10787,10790,10828,10837,10841,10859,10863,10871,10881,10885,10906,10915,10920,10929,10940,10955,10963,10973-10974,10984,10990,11011,11023,11034,11038,11042-11043,11045,11047,11075,11084,11105,11123-11124,11129,11138,11149,11165,11167,11174,11177,11188,11202,11215,11221,11233,11512,11534,11561,11572,11576-11577,11580,11593,11606-11607,11610-11611,11619,11633,11786,11799,11803-11804,11821,11824,11840,11849,11860,11878,11882-11884"}}
+{"timestamp":1714080681.3155127,"name":"online","context":{"idset":"808,878,884,7671-7672,7684,7696,7699,7703-7706,7710,7712-7713,7715,7718,7721,7728,7730,7732-7733,7735,7739,7743,7746,7748,7754,7756,7758,7761,7764-7765,7775-7776,7783-7784,7788-7792,7795,7799,7802-7808,7810,7815,7818-7819,7823,7825-7827,7829-7838,7842-7843,7845-7846,7848-7849,7851-7853,7855,7858-7859,7862,7865-7868,7870,7873-7875,7877,7884-7885,7887-7890,7894-7895,7898-7901,7903-7904,7911,7915-7919,7921-7922,7924,7926,7930,7932,7943-7946,7948-7949,7952-7953,7957,7966-7967,7974,7976,7986-7988,7992,7994-7995,7998-8000,8009-8010,8012-8013,8023,8025,8028-8029,8034,8040-8041,8044,8046,8050,8059,8062,8072,8077,8080,8082,8086-8087,8091,8097,8099,8120-8122,8124-8127,8130,8132-8133,8136-8137,8139,8143-8144,8150-8152,8157-8158,8183-8184,8192,8198-8199,8202-8204,8206,8208-8210,8213,8215,8219,8223,8227,8232,8236,8242,8244,8248,8250,8254,8263-8264,8272-8274,8276,8279,8286,8300-8301,8307,8310,8314,8316-8319,8321-8323,8325,8328,8336,8342,8345,8364,8366,8375,8378-8379,8385-8386,8395,8399,8402,8409,8412-8413,8417-8418,8423-8430,8597-8600,8605,8609-8610,8612,8822,8825,8829,8831,8837,8844,8846,8859,8861,8863-8864,8867,8869,8871,8877,8881-8882,8884,8886,8888-8890,8897-8900,8906,8911,8914-8917,8920,8922,8926,8930-8935,8938,8946,8966-8967,8971,9082,9085,9087-9088,9097,9101-9104,9107,9111,9115,9117,9122,9124,9129,9132,9135,9145,9147,9164,9180,9184,9188,9192-9193,9195,9202-9204,9206-9207,9209-9212,9220,9222-9223,9226,9228,9236,9238,9241-9244,9247-9248,9250,9253,9256-9257,9260,9263,9265,9271,9274-9276,9287,9296,9299-9300,9304,9306-9307,9310,9312,9314,9320,9323-9324,9326,9328,9469-9470,9475,9477,9479,9484,9490,9493-9495,9505,9510,9515,9525-9527,9529,9533-9534,9537-9538,9543,9545-9546,9549-9550,9554,9556,9560-9564,9572,9574,9582,9591,9595,9600,9604,9611,9615,9621,9624,9626,9630,9632,9642,9649-9650,9655,9657,9667,9670-9671,9680-9683,9689-9691,9697,9702,9705,9707-9708,9713-9714,9718,9735,9737-9741,9747-9749,9751,9755,9757,9759-9761,9765,9767,9773,9777,9782,9785,9789,9793-9794,9798,9801,9808-9809,9811,9815,9817,9819-9820,9824,9828,9831,9934,9937,9939,9941,9945,9950,9952,9957,9960-9961,9964,9967,9970,9980-9981,9985,9987,9990-9992,9996,10000,10005,10007,10015,10101,10112-10113,10117-10119,10123,10127,10129,10132,10135,10138,10140,10142,10148-10152,10156,10160,10164,10166,10169,10171,10174-10177,10179-10182,10184,10190-10191,10195-10198,10202-10204,10207-10209,10211-10212,10357-10358,10360-10362,10366,10368-10371,10375-10376,10379-10381,10383,10386,10388,10390-10391,10393-10398,10402,10404-10405,10409-10410,10412-10413,10417,10427-10430,10432,10434-10437,10440,10442-10443,10445,10448,10451,10453,10459,10461-10464,10466,10468,10470-10473,10475-10476,10478-10479,10483-10484,10487,10491,10495,10497,10503,10507-10508,10510,10513-10515,10522,10525,10527,10530,10534,10538,10546-10547,10549,10552-10554,10556-10557,10573,10579-10581,10585,10593,10597-10598,10600-10602,10607,10610,10615-10616,10618-10619,10622,10625-10627,10630,10633-10635,10637,10645,10647,10650,10656,10659,10664-10668,10670-10671,10674,10676,10679,10682,10685-10687,10691,10693-10694,10698,10706,10709,10713,10715,10717-10718,10724-10726,10729-10730,10736,10738-10741,10744-10745,10749,10751-10752,10755-10757,10760,10764,10766-10767,10771-10772,10775-10777,10779-10780,10782,10784,10788,10797,10799-10805,10810,10815-10816,10819,10822-10823,10829,10833,10842,10847,10849-10850,10852-10853,10856,10861,10868,10876,10886,10888,10892,10894,10896,10899-10900,10902-10904,10911,10914,10917,10921-10923,10926,10928,10931,10934,10936,10938,10941,10948,10956-10958,10967-10968,10971,10977,10979-10980,10991,10993-10994,10998,11000,11002,11004-11005,11008,11012,11016-11017,11022,11028-11029,11037,11039-11040,11052,11060,11062,11068,11070,11076,11080,11085,11092-11093,11096-11097,11099,11103,11106-11107,11109-11111,11117,11119,11122,11127,11131,11137,11148,11153-11154,11156-11157,11160,11162-11163,11168-11170,11180,11183,11185-11187,11192,11201,11208,11211,11219,11223-11225,11227,11229-11231,11239-11240,11509-11511,11513-11514,11516,11518-11520,11524-11531,11533,11535-11539,11541-11542,11544-11546,11548-11552,11555-11556,11558-11559,11563,11567,11569,11571,11574,11578-11579,11581-11584,11586-11587,11589-11590,11592,11595-11598,11600-11602,11605,11608-11609,11612,11614-11616,11618,11620-11621,11623-11624,11626,11628,11631-11632,11635-11636,11658,11767-11770,11772-11774,11776,11780,11782,11787,11795-11797,11806,11808,11811,11813-11814,11817,11822,11828-11829,11831,11836,11844,11846,11850,11854,11857,11861,11863,11870-11872,11876-11877,11879,11881,11885-11886,11892"}}
+{"timestamp":1714080681.4723175,"name":"online","context":{"idset":"827,877,7670,7673-7676,7679-7683,7685-7691,7694-7695,7698,7701-7702,7708-7709,7711,7716,7719-7720,7722-7726,7729,7731,7734,7736,7738,7740-7741,7744,7747,7749-7751,7753,7757,7759-7760,7762,7766-7774,7779-7782,7785-7786,7798,7800,7811-7812,7814,7817,7820-7822,7840,7844,7847,7854,7857,7863-7864,7879-7880,7883,7892-7893,7896,7902,7906,7913,7920,7925,7927-7928,7931,7933,7935,7938,7940-7942,7950-7951,7954-7956,7958-7959,7963-7964,7968-7972,7978-7984,7989-7991,7993,7996-7997,8001,8003-8005,8007-8008,8014-8016,8018-8019,8021-8022,8024,8026,8030-8033,8035,8037,8042,8045,8048-8049,8051-8056,8058,8060-8061,8063-8066,8068-8071,8074-8076,8078-8079,8081,8083-8084,8088-8089,8093,8098,8117,8123,8134-8135,8141-8142,8145-8149,8154-8156,8159-8164,8170,8172,8174,8182,8185,8187-8188,8190-8191,8193-8195,8197,8200,8207,8212,8216,8218,8220-8221,8224-8226,8229-8231,8235,8237-8239,8241,8243,8245-8247,8249,8251-8253,8256-8260,8265-8268,8270-8271,8275,8277-8278,8282-8285,8287-8290,8293-8297,8299,8302-8303,8312,8315,8324,8326-8327,8330-8333,8335,8337,8339-8341,8344,8346-8348,8350-8358,8361,8365,8368,8371-8372,8376-8377,8383-8384,8388-8394,8396-8398,8400-8401,8403-8406,8408,8410-8411,8415-8416,8419-8422,8433-8434,8436,8601-8603,8606-8607,8611,8823-8824,8826,8828,8832,8834-8836,8838-8843,8847-8851,8853-8854,8857-8858,8862,8865-8866,8870,8872,8874-8876,8878-8880,8883,8892-8893,8902-8905,8907-8910,8912-8913,8918-8919,8921,8923-8925,8928-8929,8936-8937,8941,8943,8957-8959,8961,8963,8965,8968,8975,8979-8980,9046,9052,9070-9072,9078-9080,9083-9084,9086,9089,9093-9095,9098-9099,9105-9106,9108-9110,9112-9113,9116,9119-9121,9123,9127-9128,9130-9131,9134,9136,9139-9141,9144,9146,9151-9153,9156,9160-9162,9166,9168-9172,9174,9176-9179,9181-9183,9186,9189,9196,9201,9205,9214,9217-9219,9221,9224-9225,9227,9230-9235,9237,9239-9240,9245-9246,9252,9255,9258,9261-9262,9266,9268-9270,9272,9277,9279-9282,9285-9286,9288-9292,9294-9295,9298,9303,9308-9309,9311,9316-9318,9322,9325,9329,9332,9462-9465,9467-9468,9471-9473,9476,9481-9483,9486-9489,9491,9496-9497,9499-9500,9502-9504,9506-9507,9509,9511-9514,9517-9523,9528,9532,9536,9539-9542,9544,9547-9548,9551-9553,9555,9558-9559,9565,9567-9568,9570-9571,9573,9575,9577-9578,9580-9581,9584-9585,9588-9590,9592-9594,9597,9599,9601-9603,9605,9607-9608,9612-9614,9617-9620,9622,9625,9627-9629,9631,9633,9635-9640,9643-9647,9652,9654,9656,9659-9665,9668-9669,9672,9675-9679,9684-9685,9688,9692-9694,9696,9698-9701,9706,9710,9712,9715-9717,9721,9723,9726-9728,9733-9734,9736,9742-9746,9750,9752-9753,9756,9758,9763-9764,9766,9768-9772,9775-9776,9778,9780,9783,9786-9788,9791-9792,9795,9800,9802,9804,9806,9814,9818,9821,9823,9825,9829,9935-9936,9940,9942-9944,9946-9949,9953-9956,9962-9963,9965-9966,9969,9972,9974,9976-9979,9982-9984,9986,9988,9993-9995,9998,10001-10002,10004,10008-10013,10016-10018,10020-10021,10024-10025,10027,10029-10032,10035,10103,10105-10109,10111,10114,10120-10122,10124,10126,10128,10130-10131,10136-10137,10141,10143,10145-10147,10153-10154,10158,10161-10162,10168,10170,10173,10178,10185,10189,10193,10199-10201,10205-10206,10210,10367,10373,10378,10385,10387,10389,10392,10403,10407-10408,10411,10418,10423-10426,10431,10446-10447,10449-10450,10452,10460,10465,10467,10469,10477,10482,10485-10486,10488,10492,10494,10499,10501,10504-10506,10509,10512,10516,10518,10520-10521,10523-10524,10528-10529,10531-10533,10535-10536,10539,10541,10543-10545,10548,10551,10555,10558-10566,10568-10572,10574-10576,10582-10584,10589-10592,10595-10596,10599,10603,10605-10606,10608-10609,10611-10613,10617,10620-10621,10623-10624,10631-10632,10636,10638-10640,10642-10643,10646,10648,10653-10655,10657-10658,10660,10662,10669,10672-10673,10675,10677-10678,10681,10688-10689,10692,10696,10699,10702,10707,10710-10711,10714,10719-10721,10731-10732,10734-10735,10742-10743,10746-10748,10750,10753,10758-10759,10762,10765,10768-10770,10778,10783,10785,10789,10791-10794,10796,10798,10806-10809,10812-10813,10817-10818,10820-10821,10824-10825,10827,10830-10832,10834-10836,10838,10840,10843-10844,10846,10848,10854-10855,10858,10860,10862,10864-10867,10870,10872,10874-10875,10877-10878,10882-10884,10887,10889,10891,10895,10897-10898,10901,10905,10907-10910,10912-10913,10916,10918,10925,10927,10930,10932,10935,10937,10942-10947,10949-10952,10954,10959-10962,10966,10969-10970,10972,10975-10976,10978,10981-10983,10986-10989,10992,10995-10996,10999,11001,11003,11006-11007,11009-11010,11013-11015,11018-11021,11024-11027,11030,11033,11035-11036,11041,11044,11046,11048-11049,11051,11053,11055-11059,11061,11063-11066,11069,11071-11074,11077-11079,11081-11083,11086,11088-11091,11094-11095,11100-11102,11104,11108,11113-11116,11118,11120-11121,11128,11130,11132-11135,11140-11142,11144-11147,11150-11152,11155,11158-11159,11161,11164,11166,11172-11173,11175-11176,11178-11179,11181,11184,11189-11191,11193-11200,11203-11206,11209-11210,11212-11213,11216-11218,11220,11222,11226,11228,11234-11238,11522,11540,11543,11547,11560,11562,11564-11565,11568,11570,11573,11575,11594,11603-11604,11617,11622,11625,11629-11630,11765-11766,11771,11775,11778,11783-11785,11788,11791-11794,11801,11807,11812,11816,11818-11820,11823,11830,11837-11838,11841-11843,11845,11848,11852,11855-11856,11858-11859,11862,11864-11865,11867-11869,11873-11875,11880,11887-11889"}}
+{"timestamp":1714080681.5797901,"name":"online","context":{"idset":"93-97,99-101,103-110,112-113,115-116,118,122,125-132,157-164,166-171,173-183,185-186,188-189,191-205,207-216,218-252,277-278,280-285,287-302,304-312,315-318,320-329,331-334,337-341,343-347,405-409,412-420,851-852,8011,8017,8169,8173,8177-8179,8827,8960,8962,8964,8973-8974,8976-8978,9045,9047-9051,9069,9073-9076,9251,10786"}}
+{"timestamp":1714080681.6823351,"name":"online","context":{"idset":"98,102,111,119,124,184,187,303,319,330,8171,8176,8180"}}
+{"timestamp":1714080681.8756802,"name":"online","context":{"idset":"114,123,190,8175"}}
+{"timestamp":1714080682.1549554,"name":"online","context":{"idset":"3,5,7,9,11,16,24,27-28,32,39,41-43,49,54,57,59,8951,8953"}}
+{"timestamp":1714080682.3287916,"name":"online","context":{"idset":"1-2,4,6,8,10,12-15,17-23,25-26,29-31,33-38,40,44-48,50-53,55-56,58,60,573,8949-8950,8952,8954-8956,9054-9055,9059"}}
+{"timestamp":1714080682.5120549,"name":"online","context":{"idset":"579,9058,9060"}}
+{"timestamp":1714080682.6167054,"name":"online","context":{"idset":"8997,9001,9053,9056-9057"}}
+{"timestamp":1714080682.7975156,"name":"online","context":{"idset":"8998-9000,9002"}}
+{"timestamp":1714080682.9972496,"name":"online","context":{"idset":"6572,9019-9023,9026"}}
+{"timestamp":1714080683.109848,"name":"online","context":{"idset":"576-577,9006,9014-9018,9024,9028"}}
+{"timestamp":1714080683.3041751,"name":"online","context":{"idset":"6524,6594,6621,8981,8983,8986-8987,8989-8993,8995,9005,9008-9010,9012-9013,9025,9027"}}
+{"timestamp":1714080683.5188756,"name":"online","context":{"idset":"575,6537,6551,6561,6587,6642,8982,8984-8985,8988,8994,8996,9007,9011"}}
+{"timestamp":1714080683.6886387,"name":"online","context":{"idset":"6527,6553,6555,6575,6611,6630,6636"}}
+{"timestamp":1714080683.8793933,"name":"online","context":{"idset":"6517,6532,6535-6536,6543,6547-6548,6574,6576-6577,6590"}}
+{"timestamp":1714080684.0678761,"name":"online","context":{"idset":"6533,6539,6554,6569-6570,6582,6588,6643"}}
+{"timestamp":1714080684.2463498,"name":"online","context":{"idset":"6530-6531,6542,6546,6593,6600,6612,6639-6640"}}
+{"timestamp":1714080684.4290271,"name":"online","context":{"idset":"6526,6557,6584,6586,6598,6615-6618,6633,6638,6644"}}
+{"timestamp":1714080684.6082106,"name":"online","context":{"idset":"6544,6562,6566-6567,6592,6595,6601,6623"}}
+{"timestamp":1714080684.789809,"name":"online","context":{"idset":"6520,6550,6556,6558,6560,6580,6610,6614"}}
+{"timestamp":1714080684.9615369,"name":"online","context":{"idset":"6523,6525,6528,6559,6568,6573,6578,6602,6606-6607,6628,6641"}}
+{"timestamp":1714080685.1308265,"name":"online","context":{"idset":"6579,6591,6605,6613,6632,6634-6635,6637"}}
+{"timestamp":1714080685.3242707,"name":"online","context":{"idset":"6534,6571,6583,6609,6619"}}
+{"timestamp":1714080685.5482659,"name":"online","context":{"idset":"6522,6627"}}
+{"timestamp":1714080685.7375426,"name":"online","context":{"idset":"6540,6545,6589,6597,6603,6631"}}
+{"timestamp":1714080685.8475163,"name":"online","context":{"idset":"6565,6596,6620,6626"}}
+{"timestamp":1714080685.9639845,"name":"online","context":{"idset":"6625"}}
+{"timestamp":1714080686.1447098,"name":"online","context":{"idset":"6519,6529,6538,6581,6622"}}
+{"timestamp":1714080686.4147711,"name":"online","context":{"idset":"6564,6608,6624"}}
+{"timestamp":1714080686.6073377,"name":"online","context":{"idset":"6604,6629"}}
+{"timestamp":1714080686.9395206,"name":"online","context":{"idset":"6518"}}
+{"timestamp":1714080687.1327622,"name":"online","context":{"idset":"6521,6552"}}
+{"timestamp":1714080687.5180476,"name":"online","context":{"idset":"6585"}}
+{"timestamp":1714080687.8838551,"name":"online","context":{"idset":"6541"}}
+{"timestamp":1714080688.2418945,"name":"online","context":{"idset":"6563,6599"}}
+{"timestamp":1714080688.6336589,"name":"online","context":{"idset":"6549"}}
+{"timestamp":1714080858.7728758,"name":"online","context":{"idset":"574"}}
+{"timestamp":1714080971.693135,"name":"online","context":{"idset":"9064"}}
+{"timestamp":1714080971.8725197,"name":"online","context":{"idset":"9061-9063,9065,9067-9068"}}
+{"timestamp":1714080972.0495012,"name":"online","context":{"idset":"9066"}}
+{"timestamp":1714081842.7199707,"name":"online","context":{"idset":"8101,8110"}}
+{"timestamp":1714081849.2802606,"name":"online","context":{"idset":"8114"}}
+{"timestamp":1714081851.8382478,"name":"online","context":{"idset":"8113"}}
+{"timestamp":1714081852.4312286,"name":"online","context":{"idset":"8107"}}
+{"timestamp":1714081852.6186783,"name":"online","context":{"idset":"8116"}}
+{"timestamp":1714081852.9049635,"name":"online","context":{"idset":"8106"}}
+{"timestamp":1714081853.0982742,"name":"online","context":{"idset":"8109,8111-8112"}}
+{"timestamp":1714081853.3832204,"name":"online","context":{"idset":"8115"}}
+{"timestamp":1714081853.5920677,"name":"online","context":{"idset":"8108"}}
+{"timestamp":1714081853.6936023,"name":"online","context":{"idset":"8102,8104-8105"}}
+{"timestamp":1714081853.8980088,"name":"online","context":{"idset":"8103"}}
+{"timestamp":1714081974.2625604,"name":"online","context":{"idset":"9863"}}
+{"timestamp":1714081974.451226,"name":"online","context":{"idset":"9861-9862"}}
+{"timestamp":1714081974.5587032,"name":"online","context":{"idset":"9844"}}
+{"timestamp":1714081974.7443702,"name":"online","context":{"idset":"9842-9843,9848,9864"}}
+{"timestamp":1714081974.9433513,"name":"online","context":{"idset":"9834-9835,9837-9838,9857"}}
+{"timestamp":1714081975.1307411,"name":"online","context":{"idset":"9833,9856"}}
+{"timestamp":1714081975.3117325,"name":"online","context":{"idset":"9836,9839,9850,9855"}}
+{"timestamp":1714081975.5287502,"name":"online","context":{"idset":"9840-9841,9845,9847,9849,9851-9852,9854,9858"}}
+{"timestamp":1714081975.7071414,"name":"online","context":{"idset":"9859"}}
+{"timestamp":1714081975.8949766,"name":"online","context":{"idset":"9860"}}
+{"timestamp":1714081976.0808668,"name":"online","context":{"idset":"9846,9853"}}
+{"timestamp":1714081981.0182297,"name":"online","context":{"idset":"9867"}}
+{"timestamp":1714081981.2082295,"name":"online","context":{"idset":"9865"}}
+{"timestamp":1714081981.4052014,"name":"online","context":{"idset":"9866"}}
+{"timestamp":1714081981.8278947,"name":"online","context":{"idset":"9869-9871,9873-9874,9878,9895"}}
+{"timestamp":1714081982.024055,"name":"online","context":{"idset":"9872,9875,9880-9881,9894"}}
+{"timestamp":1714081982.1261461,"name":"online","context":{"idset":"9868,9893"}}
+{"timestamp":1714081982.3282547,"name":"online","context":{"idset":"9876,9879,9885,9896"}}
+{"timestamp":1714081982.5170214,"name":"online","context":{"idset":"9877,9886"}}
+{"timestamp":1714081982.7114809,"name":"online","context":{"idset":"9882,9884,9888-9889,9891"}}
+{"timestamp":1714081983.0116611,"name":"online","context":{"idset":"9883,9887,9890,9892"}}
+{"timestamp":1714081987.8402114,"name":"online","context":{"idset":"9897-9898"}}
+{"timestamp":1714081988.2720802,"name":"online","context":{"idset":"9899"}}
+{"timestamp":1714081988.4394715,"name":"online","context":{"idset":"9905"}}
+{"timestamp":1714081988.6010082,"name":"online","context":{"idset":"9902"}}
+{"timestamp":1714081988.7857873,"name":"online","context":{"idset":"9903-9904"}}
+{"timestamp":1714081988.968318,"name":"online","context":{"idset":"9900-9901,9906"}}
+{"timestamp":1714081989.2168362,"name":"online","context":{"idset":"9910,9926,9928"}}
+{"timestamp":1714081989.5909636,"name":"online","context":{"idset":"9911-9912,9927"}}
+{"timestamp":1714081989.6920955,"name":"online","context":{"idset":"9908,9924-9925"}}
+{"timestamp":1714081989.8683429,"name":"online","context":{"idset":"9913,9921"}}
+{"timestamp":1714081990.032078,"name":"online","context":{"idset":"9907,9909,9916-9917"}}
+{"timestamp":1714081990.1995616,"name":"online","context":{"idset":"9922"}}
+{"timestamp":1714081990.3921144,"name":"online","context":{"idset":"9915,9918"}}
+{"timestamp":1714081990.4977794,"name":"online","context":{"idset":"9914,9919"}}
+{"timestamp":1714081990.6764233,"name":"online","context":{"idset":"9920"}}
+{"timestamp":1714081990.9760587,"name":"online","context":{"idset":"9923"}}
+{"timestamp":1714081994.3622882,"name":"online","context":{"idset":"9930"}}
+{"timestamp":1714081994.8816674,"name":"online","context":{"idset":"9929,9931"}}
+{"timestamp":1714081995.2797594,"name":"online","context":{"idset":"9932-9933,10038"}}
+{"timestamp":1714081995.739464,"name":"online","context":{"idset":"10039-10040"}}
+{"timestamp":1714081995.9186418,"name":"online","context":{"idset":"10041"}}
+{"timestamp":1714081996.1226368,"name":"online","context":{"idset":"10037"}}
+{"timestamp":1714081996.337271,"name":"online","context":{"idset":"10053,10055-10056"}}
+{"timestamp":1714081996.4404268,"name":"online","context":{"idset":"10048"}}
+{"timestamp":1714081996.6413839,"name":"online","context":{"idset":"10042,10044,10049-10052,10057-10059"}}
+{"timestamp":1714081996.897646,"name":"online","context":{"idset":"10043,10045,10060"}}
+{"timestamp":1714081997.0053082,"name":"online","context":{"idset":"10046,10061"}}
+{"timestamp":1714081997.180928,"name":"online","context":{"idset":"10047,10054,10062"}}
+{"timestamp":1714081997.3749673,"name":"online","context":{"idset":"10063"}}
+{"timestamp":1714082001.0633094,"name":"online","context":{"idset":"10064"}}
+{"timestamp":1714082001.5891099,"name":"online","context":{"idset":"10065-10066"}}
+{"timestamp":1714082001.9468927,"name":"online","context":{"idset":"10067"}}
+{"timestamp":1714082002.1700456,"name":"online","context":{"idset":"10068-10069"}}
+{"timestamp":1714082002.3475101,"name":"online","context":{"idset":"10072"}}
+{"timestamp":1714082002.524447,"name":"online","context":{"idset":"10070-10071"}}
+{"timestamp":1714082003.1741886,"name":"online","context":{"idset":"10073-10074,10076,10086,10088-10089"}}
+{"timestamp":1714082003.3566697,"name":"online","context":{"idset":"10075,10078,10081"}}
+{"timestamp":1714082003.5377765,"name":"online","context":{"idset":"10077"}}
+{"timestamp":1714082003.7301061,"name":"online","context":{"idset":"10080,10082,10085"}}
+{"timestamp":1714082003.9011126,"name":"online","context":{"idset":"10079,10083"}}
+{"timestamp":1714082004.0709629,"name":"online","context":{"idset":"10084,10090-10094"}}
+{"timestamp":1714082004.3648255,"name":"online","context":{"idset":"10087,10095"}}
+{"timestamp":1714082008.0132344,"name":"online","context":{"idset":"10096"}}
+{"timestamp":1714082008.5177763,"name":"online","context":{"idset":"10098"}}
+{"timestamp":1714082008.6907432,"name":"online","context":{"idset":"10099"}}
+{"timestamp":1714082008.7798321,"name":"online","context":{"idset":"10100"}}
+{"timestamp":1714082009.0414732,"name":"online","context":{"idset":"10097"}}
+{"timestamp":1714082010.0071902,"name":"online","context":{"idset":"11241"}}
+{"timestamp":1714082010.2353752,"name":"online","context":{"idset":"11243"}}
+{"timestamp":1714082010.4980857,"name":"online","context":{"idset":"11242,11244"}}
+{"timestamp":1714082010.7784853,"name":"online","context":{"idset":"11245-11246"}}
+{"timestamp":1714082011.5417149,"name":"online","context":{"idset":"11249"}}
+{"timestamp":1714082011.5448971,"name":"online","context":{"idset":"11247"}}
+{"timestamp":1714082011.5482841,"name":"online","context":{"idset":"11248"}}
+{"timestamp":1714082015.3999007,"name":"online","context":{"idset":"11250"}}
+{"timestamp":1714082015.8745744,"name":"online","context":{"idset":"11251"}}
+{"timestamp":1714082016.4411168,"name":"online","context":{"idset":"11252"}}
+{"timestamp":1714082066.8010619,"name":"online","context":{"idset":"8166"}}
+{"timestamp":1714082066.9833322,"name":"online","context":{"idset":"8165,8167-8168"}}
+{"timestamp":1714082782.790333,"name":"online","context":{"idset":"10879-10880"}}
+{"timestamp":1714082889.1471395,"name":"online","context":{"idset":"828"}}
+{"timestamp":1714083798.151963,"name":"online","context":{"idset":"10187"}}
+{"timestamp":1714083902.5874472,"name":"online","context":{"idset":"10233"}}
+{"timestamp":1714083902.8561873,"name":"online","context":{"idset":"10248,10341"}}
+{"timestamp":1714083903.0713344,"name":"online","context":{"idset":"10310"}}
+{"timestamp":1714083963.5136833,"name":"undrain","context":{"idset":"10233,10248,10310,10341"}}
+{"timestamp":1714083990.9695337,"name":"offline","context":{"idset":"8423"}}
+{"timestamp":1714083991.0660579,"name":"offline","context":{"idset":"8427"}}
+{"timestamp":1714083991.0735657,"name":"offline","context":{"idset":"8421"}}
+{"timestamp":1714083991.0819237,"name":"offline","context":{"idset":"8436"}}
+{"timestamp":1714083991.0911336,"name":"offline","context":{"idset":"8433"}}
+{"timestamp":1714083991.0976627,"name":"offline","context":{"idset":"8422"}}
+{"timestamp":1714083991.1161294,"name":"offline","context":{"idset":"8426"}}
+{"timestamp":1714083991.1224453,"name":"offline","context":{"idset":"8428"}}
+{"timestamp":1714083991.1288404,"name":"offline","context":{"idset":"8429"}}
+{"timestamp":1714083991.1450431,"name":"offline","context":{"idset":"8430"}}
+{"timestamp":1714083991.2218654,"name":"offline","context":{"idset":"8424"}}
+{"timestamp":1714083991.3283482,"name":"offline","context":{"idset":"8425"}}
+{"timestamp":1714083991.3327579,"name":"offline","context":{"idset":"8434"}}
+{"timestamp":1714083991.4335673,"name":"offline","context":{"idset":"8435"}}
+{"timestamp":1714084022.8425956,"name":"online","context":{"idset":"10251"}}
+{"timestamp":1714084023.035409,"name":"online","context":{"idset":"10235"}}
+{"timestamp":1714084023.1659057,"name":"online","context":{"idset":"10240"}}
+{"timestamp":1714084023.2689698,"name":"online","context":{"idset":"10266"}}
+{"timestamp":1714084023.4302213,"name":"online","context":{"idset":"10257"}}
+{"timestamp":1714084023.635066,"name":"online","context":{"idset":"10234"}}
+{"timestamp":1714084023.8016868,"name":"online","context":{"idset":"10231-10232,10236,10250"}}
+{"timestamp":1714084023.9045951,"name":"online","context":{"idset":"10229,10244,10261,10263,10272"}}
+{"timestamp":1714084024.0176663,"name":"online","context":{"idset":"10280"}}
+{"timestamp":1714084024.1055634,"name":"online","context":{"idset":"10271"}}
+{"timestamp":1714084024.2140322,"name":"online","context":{"idset":"10270"}}
+{"timestamp":1714084024.3202105,"name":"online","context":{"idset":"10238,10262"}}
+{"timestamp":1714084024.5261521,"name":"online","context":{"idset":"10264"}}
+{"timestamp":1714084024.6192813,"name":"online","context":{"idset":"10242,10281"}}
+{"timestamp":1714084024.9167898,"name":"online","context":{"idset":"10265,10275"}}
+{"timestamp":1714084025.4252164,"name":"online","context":{"idset":"10285"}}
+{"timestamp":1714084025.5375507,"name":"online","context":{"idset":"10239,10256,10260,10278-10279,10293"}}
+{"timestamp":1714084025.6732371,"name":"online","context":{"idset":"10315"}}
+{"timestamp":1714084025.8783927,"name":"online","context":{"idset":"10258,10343"}}
+{"timestamp":1714084026.016813,"name":"online","context":{"idset":"10276,10325"}}
+{"timestamp":1714084026.2413595,"name":"online","context":{"idset":"10297,10338"}}
+{"timestamp":1714084026.4397967,"name":"online","context":{"idset":"10230,10268,10273,10289,10292,10294,10300-10301,10349,10352,10354"}}
+{"timestamp":1714084026.5497098,"name":"online","context":{"idset":"10287,10329"}}
+{"timestamp":1714084026.6484973,"name":"online","context":{"idset":"10246-10247,10299,10306,10314,10324,10335,10339"}}
+{"timestamp":1714084026.7602346,"name":"online","context":{"idset":"10254,10298,10340,10342,10348,10356"}}
+{"timestamp":1714084026.954392,"name":"online","context":{"idset":"10252,10274,10286,10302-10303,10307,10322-10323,10331"}}
+{"timestamp":1714084027.1523387,"name":"online","context":{"idset":"10259,10283-10284,10288,10291,10327,10334,10350-10351,10355"}}
+{"timestamp":1714084027.2536457,"name":"online","context":{"idset":"10245,10249,10304,10337,10345"}}
+{"timestamp":1714084027.364826,"name":"online","context":{"idset":"10269,10290,10305,10336"}}
+{"timestamp":1714084027.5506697,"name":"online","context":{"idset":"10253,10277,10309,10326,10353"}}
+{"timestamp":1714084027.662781,"name":"online","context":{"idset":"10243,10282"}}
+{"timestamp":1714084027.7699914,"name":"online","context":{"idset":"10237,10241,10267,10296,10318,10321,10332-10333,10346-10347"}}
+{"timestamp":1714084027.8845522,"name":"online","context":{"idset":"10295,10311-10312,10317"}}
+{"timestamp":1714084028.0650663,"name":"online","context":{"idset":"10313,10328,10330"}}
+{"timestamp":1714084028.1688538,"name":"online","context":{"idset":"10308,10319-10320"}}
+{"timestamp":1714084028.3607006,"name":"online","context":{"idset":"10344"}}
+{"timestamp":1714084031.4397702,"name":"online","context":{"idset":"10255"}}
+{"timestamp":1714084141.2002039,"name":"undrain","context":{"idset":"10187"}}
+{"timestamp":1714084170.6078789,"name":"online","context":{"idset":"10188"}}
+{"timestamp":1714084216.8466578,"name":"offline","context":{"idset":"7997"}}
+{"timestamp":1714084620.3829987,"name":"offline","context":{"idset":"8050"}}
+{"timestamp":1714084750.236042,"name":"offline","context":{"idset":"8049"}}
+{"timestamp":1714084898.0867863,"name":"online","context":{"idset":"8049"}}
+{"timestamp":1714085345.4459794,"name":"offline","context":{"idset":"8597"}}
+{"timestamp":1714085348.1072743,"name":"offline","context":{"idset":"8598"}}
+{"timestamp":1714085671.3784449,"name":"online","context":{"idset":"10422"}}
+{"timestamp":1714085672.3120229,"name":"online","context":{"idset":"10421"}}
+{"timestamp":1714085763.6050475,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1714085763.6091886,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1714085804.4162924,"name":"online","context":{"idset":"11825"}}
+{"timestamp":1714085822.0404062,"name":"online","context":{"idset":"11826"}}
+{"timestamp":1714085869.316901,"name":"online","context":{"idset":"11789"}}
+{"timestamp":1714085888.7541778,"name":"undrain","context":{"idset":"11826"}}
+{"timestamp":1714085999.0669219,"name":"drain","context":{"idset":"9501-9502","reason":"epilog failed for jobid frdGiQ6xzTH","overwrite":0}}
+{"timestamp":1714086406.5861523,"name":"undrain","context":{"idset":"11789"}}
+{"timestamp":1714086487.4613919,"name":"offline","context":{"idset":"8341"}}
+{"timestamp":1714086489.4594507,"name":"offline","context":{"idset":"8342"}}
+{"timestamp":1714086565.078126,"name":"online","context":{"idset":"10228"}}
+{"timestamp":1714086642.3395605,"name":"online","context":{"idset":"10215-10216,10220"}}
+{"timestamp":1714086642.3445332,"name":"online","context":{"idset":"10218,10225"}}
+{"timestamp":1714086642.3490341,"name":"online","context":{"idset":"10223"}}
+{"timestamp":1714086642.3536069,"name":"online","context":{"idset":"10217,10226"}}
+{"timestamp":1714086642.3580184,"name":"online","context":{"idset":"10219,10221-10222,10224,10227"}}
+{"timestamp":1714086884.6295733,"name":"offline","context":{"idset":"8309"}}
+{"timestamp":1714086884.6927919,"name":"offline","context":{"idset":"8310"}}
+{"timestamp":1714086887.0640962,"name":"online","context":{"idset":"7997"}}
+{"timestamp":1714086998.5762877,"name":"online","context":{"idset":"8050"}}
+{"timestamp":1714087228.4944551,"name":"drain","context":{"idset":"7940,8049-8050","reason":"epilog failed for jobid frcwjKDtECw","overwrite":0}}
+{"timestamp":1714087228.4968326,"name":"offline","context":{"idset":"7940"}}
+{"timestamp":1714087373.6818368,"name":"offline","context":{"idset":"7932"}}
+{"timestamp":1714087373.7956157,"name":"offline","context":{"idset":"7933"}}
+{"timestamp":1714087373.802474,"name":"offline","context":{"idset":"7934"}}
+{"timestamp":1714087373.8051174,"name":"offline","context":{"idset":"7942"}}
+{"timestamp":1714087373.8078425,"name":"offline","context":{"idset":"7988"}}
+{"timestamp":1714087373.8106177,"name":"offline","context":{"idset":"7929"}}
+{"timestamp":1714087373.9116571,"name":"offline","context":{"idset":"7941"}}
+{"timestamp":1714087374.0139697,"name":"offline","context":{"idset":"7925"}}
+{"timestamp":1714087374.0186772,"name":"offline","context":{"idset":"7986"}}
+{"timestamp":1714087374.1173432,"name":"offline","context":{"idset":"7960"}}
+{"timestamp":1714087374.1303031,"name":"offline","context":{"idset":"7950"}}
+{"timestamp":1714087374.1372473,"name":"offline","context":{"idset":"7966"}}
+{"timestamp":1714087374.1403346,"name":"offline","context":{"idset":"8038"}}
+{"timestamp":1714087374.1623178,"name":"offline","context":{"idset":"7973"}}
+{"timestamp":1714087374.2417014,"name":"offline","context":{"idset":"8030"}}
+{"timestamp":1714087374.3927438,"name":"offline","context":{"idset":"8015"}}
+{"timestamp":1714087374.3995824,"name":"offline","context":{"idset":"8001"}}
+{"timestamp":1714087374.4029918,"name":"offline","context":{"idset":"7997"}}
+{"timestamp":1714087374.4063311,"name":"offline","context":{"idset":"8020"}}
+{"timestamp":1714087374.4147153,"name":"offline","context":{"idset":"8022"}}
+{"timestamp":1714087374.4177485,"name":"offline","context":{"idset":"8023"}}
+{"timestamp":1714087374.5124121,"name":"offline","context":{"idset":"8019"}}
+{"timestamp":1714087374.6142218,"name":"offline","context":{"idset":"8002"}}
+{"timestamp":1714087374.6364892,"name":"offline","context":{"idset":"7961"}}
+{"timestamp":1714087374.6408153,"name":"offline","context":{"idset":"8051"}}
+{"timestamp":1714087374.6438079,"name":"offline","context":{"idset":"8031"}}
+{"timestamp":1714087374.6470833,"name":"offline","context":{"idset":"8037"}}
+{"timestamp":1714087374.7462978,"name":"offline","context":{"idset":"8021"}}
+{"timestamp":1714087374.8382561,"name":"offline","context":{"idset":"7995"}}
+{"timestamp":1714087374.8463149,"name":"offline","context":{"idset":"7972"}}
+{"timestamp":1714087374.8507004,"name":"offline","context":{"idset":"8024"}}
+{"timestamp":1714087374.8821981,"name":"offline","context":{"idset":"7994"}}
+{"timestamp":1714087374.8885214,"name":"offline","context":{"idset":"7948"}}
+{"timestamp":1714087374.8947389,"name":"offline","context":{"idset":"7975"}}
+{"timestamp":1714087374.9010029,"name":"offline","context":{"idset":"7982"}}
+{"timestamp":1714087374.9073029,"name":"offline","context":{"idset":"7985"}}
+{"timestamp":1714087374.9135938,"name":"offline","context":{"idset":"8012"}}
+{"timestamp":1714087375.0141275,"name":"offline","context":{"idset":"8040"}}
+{"timestamp":1714087375.1383448,"name":"offline","context":{"idset":"8043"}}
+{"timestamp":1714087375.1454425,"name":"offline","context":{"idset":"8046"}}
+{"timestamp":1714087375.1502304,"name":"offline","context":{"idset":"7996"}}
+{"timestamp":1714087375.1550493,"name":"offline","context":{"idset":"7964"}}
+{"timestamp":1714087375.2082191,"name":"offline","context":{"idset":"7983"}}
+{"timestamp":1714087375.2121146,"name":"offline","context":{"idset":"7926"}}
+{"timestamp":1714087375.2241299,"name":"offline","context":{"idset":"7943"}}
+{"timestamp":1714087375.2287116,"name":"offline","context":{"idset":"7952"}}
+{"timestamp":1714087375.2335045,"name":"offline","context":{"idset":"7956"}}
+{"timestamp":1714087375.2377322,"name":"offline","context":{"idset":"7957"}}
+{"timestamp":1714087375.2423947,"name":"offline","context":{"idset":"7981"}}
+{"timestamp":1714087375.2577426,"name":"offline","context":{"idset":"7987"}}
+{"timestamp":1714087375.2696478,"name":"offline","context":{"idset":"7999"}}
+{"timestamp":1714087375.2730837,"name":"offline","context":{"idset":"8013"}}
+{"timestamp":1714087375.2777934,"name":"offline","context":{"idset":"8026"}}
+{"timestamp":1714087375.3486691,"name":"offline","context":{"idset":"8047"}}
+{"timestamp":1714087375.5505681,"name":"offline","context":{"idset":"7935"}}
+{"timestamp":1714087375.5591643,"name":"offline","context":{"idset":"8034"}}
+{"timestamp":1714087375.5910697,"name":"offline","context":{"idset":"7947"}}
+{"timestamp":1714087375.5950701,"name":"offline","context":{"idset":"8025"}}
+{"timestamp":1714087375.5982628,"name":"offline","context":{"idset":"8028"}}
+{"timestamp":1714087375.601438,"name":"offline","context":{"idset":"7938"}}
+{"timestamp":1714087375.6932702,"name":"offline","context":{"idset":"7945"}}
+{"timestamp":1714087375.7882395,"name":"offline","context":{"idset":"7949"}}
+{"timestamp":1714087375.7943659,"name":"offline","context":{"idset":"8008"}}
+{"timestamp":1714087375.8004489,"name":"offline","context":{"idset":"7992"}}
+{"timestamp":1714087375.8065689,"name":"offline","context":{"idset":"8006"}}
+{"timestamp":1714087375.8126228,"name":"offline","context":{"idset":"8032"}}
+{"timestamp":1714087375.8187058,"name":"offline","context":{"idset":"8016"}}
+{"timestamp":1714087376.9791894,"name":"offline","context":{"idset":"8044"}}
+{"timestamp":1714087376.9854167,"name":"offline","context":{"idset":"8039"}}
+{"timestamp":1714087376.9913108,"name":"offline","context":{"idset":"8027"}}
+{"timestamp":1714087376.9982052,"name":"offline","context":{"idset":"7954"}}
+{"timestamp":1714087377.0059431,"name":"offline","context":{"idset":"7955"}}
+{"timestamp":1714087377.0116494,"name":"offline","context":{"idset":"8052"}}
+{"timestamp":1714087377.0157397,"name":"offline","context":{"idset":"8004"}}
+{"timestamp":1714087377.0199034,"name":"offline","context":{"idset":"7939"}}
+{"timestamp":1714087377.0239966,"name":"offline","context":{"idset":"7944"}}
+{"timestamp":1714087377.0282238,"name":"offline","context":{"idset":"8017"}}
+{"timestamp":1714087377.0334065,"name":"offline","context":{"idset":"7989"}}
+{"timestamp":1714087377.0375223,"name":"offline","context":{"idset":"7968"}}
+{"timestamp":1714087377.0416477,"name":"offline","context":{"idset":"8048"}}
+{"timestamp":1714087377.0458534,"name":"offline","context":{"idset":"8036"}}
+{"timestamp":1714087377.0499132,"name":"offline","context":{"idset":"7969"}}
+{"timestamp":1714087377.054914,"name":"offline","context":{"idset":"7976"}}
+{"timestamp":1714087377.0610917,"name":"offline","context":{"idset":"7974"}}
+{"timestamp":1714087377.0671856,"name":"offline","context":{"idset":"8045"}}
+{"timestamp":1714087377.0721562,"name":"offline","context":{"idset":"7937"}}
+{"timestamp":1714087377.0763514,"name":"offline","context":{"idset":"8003"}}
+{"timestamp":1714087377.0820279,"name":"offline","context":{"idset":"7970"}}
+{"timestamp":1714087377.0892022,"name":"offline","context":{"idset":"7967"}}
+{"timestamp":1714087377.0955853,"name":"offline","context":{"idset":"7928"}}
+{"timestamp":1714087377.0998185,"name":"offline","context":{"idset":"7953"}}
+{"timestamp":1714087377.1039298,"name":"offline","context":{"idset":"7990"}}
+{"timestamp":1714087377.1080163,"name":"offline","context":{"idset":"8033"}}
+{"timestamp":1714087377.1121705,"name":"offline","context":{"idset":"8035"}}
+{"timestamp":1714087377.1164479,"name":"offline","context":{"idset":"7927"}}
+{"timestamp":1714087377.1236079,"name":"offline","context":{"idset":"7962"}}
+{"timestamp":1714087377.1314228,"name":"offline","context":{"idset":"7965"}}
+{"timestamp":1714087377.137228,"name":"offline","context":{"idset":"8010"}}
+{"timestamp":1714087377.1412969,"name":"offline","context":{"idset":"8005"}}
+{"timestamp":1714087377.1460538,"name":"offline","context":{"idset":"8011"}}
+{"timestamp":1714087377.1506352,"name":"offline","context":{"idset":"7958"}}
+{"timestamp":1714087377.1549964,"name":"offline","context":{"idset":"7993"}}
+{"timestamp":1714087377.3174229,"name":"offline","context":{"idset":"7971"}}
+{"timestamp":1714087377.6022851,"name":"offline","context":{"idset":"7930"}}
+{"timestamp":1714087377.605324,"name":"offline","context":{"idset":"7931"}}
+{"timestamp":1714087377.6084964,"name":"offline","context":{"idset":"7936"}}
+{"timestamp":1714087377.6121686,"name":"offline","context":{"idset":"7946"}}
+{"timestamp":1714087377.6151896,"name":"offline","context":{"idset":"7951"}}
+{"timestamp":1714087377.6185892,"name":"offline","context":{"idset":"7980"}}
+{"timestamp":1714087377.6220772,"name":"offline","context":{"idset":"7984"}}
+{"timestamp":1714087377.6250355,"name":"offline","context":{"idset":"8007"}}
+{"timestamp":1714087377.6281202,"name":"offline","context":{"idset":"8009"}}
+{"timestamp":1714087377.7132046,"name":"offline","context":{"idset":"8042"}}
+{"timestamp":1714087378.3099632,"name":"offline","context":{"idset":"7977"}}
+{"timestamp":1714087378.3133359,"name":"offline","context":{"idset":"7979"}}
+{"timestamp":1714087378.3177533,"name":"offline","context":{"idset":"7991"}}
+{"timestamp":1714087378.3210838,"name":"offline","context":{"idset":"8018"}}
+{"timestamp":1714087378.323921,"name":"offline","context":{"idset":"8014"}}
+{"timestamp":1714087378.3270061,"name":"offline","context":{"idset":"7959"}}
+{"timestamp":1714087378.4302475,"name":"offline","context":{"idset":"8029"}}
+{"timestamp":1714087378.5299575,"name":"offline","context":{"idset":"8000"}}
+{"timestamp":1714087378.8333633,"name":"offline","context":{"idset":"7998"}}
+{"timestamp":1714087379.0010059,"name":"offline","context":{"idset":"7963"}}
+{"timestamp":1714087379.0989766,"name":"offline","context":{"idset":"8041"}}
+{"timestamp":1714087379.2856462,"name":"offline","context":{"idset":"7978"}}
+{"timestamp":1714087532.2650793,"name":"online","context":{"idset":"9501-9502"}}
+{"timestamp":1714087653.557543,"name":"offline","context":{"idset":"8049"}}
+{"timestamp":1714087870.460938,"name":"undrain","context":{"idset":"6517-6564,6581-6644"}}
+{"timestamp":1714088452.4739065,"name":"online","context":{"idset":"7932"}}
+{"timestamp":1714088452.7725403,"name":"online","context":{"idset":"7951"}}
+{"timestamp":1714088452.9910867,"name":"online","context":{"idset":"7925"}}
+{"timestamp":1714088453.2601833,"name":"online","context":{"idset":"7931,7939,7954"}}
+{"timestamp":1714088454.220099,"name":"online","context":{"idset":"7959"}}
+{"timestamp":1714088454.3844233,"name":"online","context":{"idset":"7938"}}
+{"timestamp":1714088456.1771095,"name":"online","context":{"idset":"7955"}}
+{"timestamp":1714088456.7195876,"name":"online","context":{"idset":"7936"}}
+{"timestamp":1714088456.9513977,"name":"online","context":{"idset":"7930"}}
+{"timestamp":1714088457.1782136,"name":"online","context":{"idset":"7937"}}
+{"timestamp":1714088457.4411447,"name":"online","context":{"idset":"7926"}}
+{"timestamp":1714088458.1280324,"name":"online","context":{"idset":"7941"}}
+{"timestamp":1714088458.1338589,"name":"online","context":{"idset":"7961"}}
+{"timestamp":1714088458.1385517,"name":"online","context":{"idset":"7958"}}
+{"timestamp":1714088458.1435606,"name":"online","context":{"idset":"7933"}}
+{"timestamp":1714088458.1474454,"name":"online","context":{"idset":"7973"}}
+{"timestamp":1714088458.151453,"name":"online","context":{"idset":"7927"}}
+{"timestamp":1714088458.1554067,"name":"online","context":{"idset":"7981"}}
+{"timestamp":1714088458.6321573,"name":"online","context":{"idset":"7928,7943,7945,7949,8002"}}
+{"timestamp":1714088458.7355888,"name":"online","context":{"idset":"7934,7948"}}
+{"timestamp":1714088458.8167431,"name":"online","context":{"idset":"7965"}}
+{"timestamp":1714088458.8922241,"name":"online","context":{"idset":"7947,7966,8026"}}
+{"timestamp":1714088459.0362117,"name":"online","context":{"idset":"7929"}}
+{"timestamp":1714088459.2424443,"name":"online","context":{"idset":"7991,8028"}}
+{"timestamp":1714088459.4266753,"name":"online","context":{"idset":"7942,7952,7979,8000"}}
+{"timestamp":1714088460.3062088,"name":"online","context":{"idset":"7986-7987,8048"}}
+{"timestamp":1714088460.3095253,"name":"online","context":{"idset":"7940,7944,7985,7999,8016,8032"}}
+{"timestamp":1714088460.313107,"name":"online","context":{"idset":"7996,8004,8014,8029"}}
+{"timestamp":1714088460.3166614,"name":"online","context":{"idset":"7950,7956,7964,7970-7971,7978,7995,8020,8024,8044,8047"}}
+{"timestamp":1714088460.3202505,"name":"online","context":{"idset":"7946,7990,8017,8025"}}
+{"timestamp":1714088460.3239322,"name":"online","context":{"idset":"7935,7953,8031,8035"}}
+{"timestamp":1714088460.3274014,"name":"online","context":{"idset":"7963,7974,7980,8040"}}
+{"timestamp":1714088460.5453253,"name":"online","context":{"idset":"7960,7967,7969,7972,7975,7982-7983,7988,7993,7998,8005,8007,8009,8021,8023,8041"}}
+{"timestamp":1714088460.6510272,"name":"online","context":{"idset":"7976,8003,8042"}}
+{"timestamp":1714088460.8350754,"name":"online","context":{"idset":"7962,7992,8006,8008,8012,8015,8018,8030,8037,8043,8051-8052"}}
+{"timestamp":1714088461.0048506,"name":"online","context":{"idset":"7957,7994,8001,8013,8027"}}
+{"timestamp":1714088461.110713,"name":"online","context":{"idset":"7984,7989,8039"}}
+{"timestamp":1714088461.2109754,"name":"online","context":{"idset":"7968,7977,7997,8011,8036,8045-8046"}}
+{"timestamp":1714088461.3760581,"name":"online","context":{"idset":"8010,8019,8033"}}
+{"timestamp":1714088462.3361392,"name":"online","context":{"idset":"8022,8034,8038"}}
+{"timestamp":1714088586.858247,"name":"undrain","context":{"idset":"7940,8050"}}
+{"timestamp":1714088592.57759,"name":"undrain","context":{"idset":"8049"}}
+{"timestamp":1714088614.9294071,"name":"online","context":{"idset":"8049"}}
+{"timestamp":1714089654.849221,"name":"offline","context":{"idset":"6575"}}
+{"timestamp":1714089657.9965856,"name":"offline","context":{"idset":"6576"}}
+{"timestamp":1714091828.7575581,"name":"undrain","context":{"idset":"9502"}}
+{"timestamp":1714091831.9556806,"name":"undrain","context":{"idset":"9501"}}
+{"timestamp":1714093308.2018149,"name":"offline","context":{"idset":"7711"}}
+{"timestamp":1714093308.3146687,"name":"offline","context":{"idset":"7712"}}
+{"timestamp":1714096007.6972802,"name":"offline","context":{"idset":"6630"}}
+{"timestamp":1714096670.5104837,"name":"online","context":{"idset":"10316"}}
+{"timestamp":1714097061.5019722,"name":"drain","context":{"idset":"6517-6532","overwrite":0}}
+{"timestamp":1714097118.577405,"name":"undrain","context":{"idset":"6517-6532"}}
+{"timestamp":1714097316.5049524,"name":"online","context":{"idset":"11398,11411,11439,11487"}}
+{"timestamp":1714097316.697052,"name":"online","context":{"idset":"11393,11418-11419"}}
+{"timestamp":1714097316.8970439,"name":"online","context":{"idset":"11409"}}
+{"timestamp":1714097317.0885656,"name":"online","context":{"idset":"11385,11420,11422,11458,11479,11508"}}
+{"timestamp":1714097317.2757342,"name":"online","context":{"idset":"11381,11387-11391,11394,11400,11408,11415,11426-11427,11444,11452,11462,11467,11478,11503,11507"}}
+{"timestamp":1714097317.4755862,"name":"online","context":{"idset":"11384,11401,11403-11404,11407,11414,11417,11421,11433,11442,11445,11448,11451,11455,11464,11476,11489,11491,11502"}}
+{"timestamp":1714097317.5811763,"name":"online","context":{"idset":"11392,11395-11397,11412-11413,11416,11425,11430,11434-11435,11440,11449,11460,11465,11469,11473,11484-11485,11494,11497,11505"}}
+{"timestamp":1714097318.5648599,"name":"online","context":{"idset":"11383,11399,11402,11410,11429,11432,11437-11438,11446,11450,11457,11459,11466,11470-11471,11480,11482-11483,11492,11495-11496,11498,11506"}}
+{"timestamp":1714097318.5682011,"name":"online","context":{"idset":"11382,11386,11405-11406,11423,11428,11443,11447,11453-11454,11456,11481,11500"}}
+{"timestamp":1714097318.5715497,"name":"online","context":{"idset":"11424,11431,11436,11441,11461,11468,11472,11474-11475,11477,11490,11499,11501,11504"}}
+{"timestamp":1714097318.5742147,"name":"online","context":{"idset":"11463,11486,11488,11493"}}
+{"timestamp":1714097529.698709,"name":"offline","context":{"idset":"9140"}}
+{"timestamp":1714097531.7002697,"name":"offline","context":{"idset":"9139"}}
+{"timestamp":1714099799.1266015,"name":"drain","context":{"idset":"11096","reason":"broker was unresponsive"}}
+{"timestamp":1714103854.5322108,"name":"undrain","context":{"idset":"11096"}}
+{"timestamp":1714121200.5835049,"name":"offline","context":{"idset":"9179"}}
+{"timestamp":1714127508.970933,"name":"offline","context":{"idset":"8159"}}
+{"timestamp":1714128253.8462124,"name":"offline","context":{"idset":"9327"}}
+{"timestamp":1714128636.5911202,"name":"offline","context":{"idset":"10281"}}
+{"timestamp":1714137390.6786191,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1714138826.6677513,"name":"online","context":{"idset":"8159"}}
+{"timestamp":1714139845.8450992,"name":"offline","context":{"idset":"10411"}}
+{"timestamp":1714139846.6570826,"name":"offline","context":{"idset":"10412"}}
+{"timestamp":1714140457.8028729,"name":"offline","context":{"idset":"9461"}}
+{"timestamp":1714140457.8066092,"name":"offline","context":{"idset":"9462"}}
+{"timestamp":1714140457.8100171,"name":"offline","context":{"idset":"9463"}}
+{"timestamp":1714140457.8134198,"name":"offline","context":{"idset":"9464"}}
+{"timestamp":1714140457.8168087,"name":"offline","context":{"idset":"9465"}}
+{"timestamp":1714140457.8202279,"name":"offline","context":{"idset":"9466"}}
+{"timestamp":1714140457.8235395,"name":"offline","context":{"idset":"9467"}}
+{"timestamp":1714140457.826874,"name":"offline","context":{"idset":"9468"}}
+{"timestamp":1714140457.8302052,"name":"offline","context":{"idset":"9469"}}
+{"timestamp":1714140457.8336,"name":"offline","context":{"idset":"9470"}}
+{"timestamp":1714140457.8370273,"name":"offline","context":{"idset":"9471"}}
+{"timestamp":1714140457.8404052,"name":"offline","context":{"idset":"9472"}}
+{"timestamp":1714140457.8438096,"name":"offline","context":{"idset":"9473"}}
+{"timestamp":1714140457.8474905,"name":"offline","context":{"idset":"9474"}}
+{"timestamp":1714140457.8518438,"name":"offline","context":{"idset":"9475"}}
+{"timestamp":1714140458.6560309,"name":"offline","context":{"idset":"9476"}}
+{"timestamp":1714140859.2794046,"name":"offline","context":{"idset":"9477"}}
+{"timestamp":1714140859.2842438,"name":"offline","context":{"idset":"9478"}}
+{"timestamp":1714140859.2889884,"name":"offline","context":{"idset":"9479"}}
+{"timestamp":1714140859.2929618,"name":"offline","context":{"idset":"9480"}}
+{"timestamp":1714140859.2969396,"name":"offline","context":{"idset":"9481"}}
+{"timestamp":1714140859.3009126,"name":"offline","context":{"idset":"9482"}}
+{"timestamp":1714140859.3062644,"name":"offline","context":{"idset":"9483"}}
+{"timestamp":1714140859.3100336,"name":"offline","context":{"idset":"9484"}}
+{"timestamp":1714140859.3245542,"name":"offline","context":{"idset":"9485"}}
+{"timestamp":1714140859.3285303,"name":"offline","context":{"idset":"9486"}}
+{"timestamp":1714140859.3325026,"name":"offline","context":{"idset":"9487"}}
+{"timestamp":1714140859.3364751,"name":"offline","context":{"idset":"9488"}}
+{"timestamp":1714140859.3403978,"name":"offline","context":{"idset":"9489"}}
+{"timestamp":1714140859.3547184,"name":"offline","context":{"idset":"9490"}}
+{"timestamp":1714140859.3586929,"name":"offline","context":{"idset":"9491"}}
+{"timestamp":1714140859.3626468,"name":"offline","context":{"idset":"9492"}}
+{"timestamp":1714141245.7971947,"name":"offline","context":{"idset":"9493"}}
+{"timestamp":1714141245.8007677,"name":"offline","context":{"idset":"9494"}}
+{"timestamp":1714141245.8043315,"name":"offline","context":{"idset":"9495"}}
+{"timestamp":1714141245.8078077,"name":"offline","context":{"idset":"9496"}}
+{"timestamp":1714141245.8114321,"name":"offline","context":{"idset":"9497"}}
+{"timestamp":1714141245.8151209,"name":"offline","context":{"idset":"9498"}}
+{"timestamp":1714141245.8185306,"name":"offline","context":{"idset":"9499"}}
+{"timestamp":1714141245.8219762,"name":"offline","context":{"idset":"9500"}}
+{"timestamp":1714141245.8256836,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1714141245.8290973,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1714141245.8324702,"name":"offline","context":{"idset":"9503"}}
+{"timestamp":1714141245.8359125,"name":"offline","context":{"idset":"9504"}}
+{"timestamp":1714141245.8394189,"name":"offline","context":{"idset":"9505"}}
+{"timestamp":1714141245.8428521,"name":"offline","context":{"idset":"9506"}}
+{"timestamp":1714141245.8463781,"name":"offline","context":{"idset":"9507"}}
+{"timestamp":1714141246.5596189,"name":"offline","context":{"idset":"9508"}}
+{"timestamp":1714141343.7578096,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1714141343.8444586,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1714141345.767796,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1714141345.7740259,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1714141345.7801499,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1714141345.7862103,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1714141345.8450227,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1714141347.7642853,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1714141347.7685225,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1714141347.8457785,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1714141349.7759142,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1714141349.7835007,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1714141349.7910404,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1714141349.846947,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1714141351.7718215,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1714141351.7782409,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1714141351.78566,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1714141351.8473442,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1714141353.7633023,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1714141353.7667344,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1714141353.7712574,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1714141353.8462245,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1714141355.7878487,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1714141355.79389,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1714141355.7977176,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1714141355.801038,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1714141355.8564153,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1714141357.7686341,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1714141357.7746322,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1714141357.7806017,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1714141357.8506634,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1714141359.7624645,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1714141359.7656229,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1714141359.7688396,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1714141359.771951,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1714141359.8465972,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1714141361.7699845,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1714141361.7738192,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1714141361.7773309,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1714141361.8462214,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1714141363.781204,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1714141363.790863,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1714141363.7962494,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1714141363.8737125,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1714141365.766387,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1714141365.771564,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1714141365.7758751,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1714141365.8464644,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1714141367.7687523,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1714141367.7721949,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1714141367.7758529,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1714141367.7817559,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1714141367.7851441,"name":"offline","context":{"idset":"10153"}}
+{"timestamp":1714141367.8463564,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1714141369.7644017,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1714141370.6882889,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1714141372.6060345,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1714141372.6099589,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1714141373.0121439,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1714141374.0048428,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1714141374.0116291,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1714141374.016912,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1714141374.0206389,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1714141374.8909028,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1714141376.6877942,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1714141376.6944599,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1714141376.7011964,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1714141377.542042,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1714141379.2254791,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1714141379.2925858,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1714141380.1749775,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1714141380.5662401,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1714141380.8356164,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1714141381.4029951,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1714141382.7602274,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1714141383.8305211,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1714141383.8385594,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1714141384.0931399,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1714141385.0918274,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1714141386.0262525,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1714141386.0320618,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1714141386.3543036,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1714141386.3771081,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1714141387.2520525,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1714141387.2566769,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1714141387.8162401,"name":"offline","context":{"idset":"10188"}}
+{"timestamp":1714141388.2209351,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1714141390.0134606,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1714141390.0177741,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1714141390.0229738,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1714141390.0286596,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1714141390.0310426,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1714141390.0339415,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1714141390.0403154,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1714141390.0426931,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1714141392.0531445,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1714141392.0561118,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1714141392.059233,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1714141393.7597599,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1714141393.7629077,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1714141393.8486137,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1714141395.7659881,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1714141395.769608,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1714141395.7729988,"name":"offline","context":{"idset":"10206"}}
+{"timestamp":1714141395.7762678,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1714141395.8466003,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1714141397.7713792,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1714141397.7747827,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1714141397.7781098,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1714141397.8461044,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1714141399.7636244,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1714141399.7678299,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1714141399.846612,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1714141401.7621853,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1714141401.765259,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1714141401.7683008,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1714141401.846209,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1714141403.7629993,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1714141403.7660811,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1714141403.7691369,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1714141403.8490007,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1714141405.7624621,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1714141405.7659011,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1714141405.8496997,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1714141409.7981155,"name":"offline","context":{"idset":"9509"}}
+{"timestamp":1714141409.8039889,"name":"offline","context":{"idset":"9510"}}
+{"timestamp":1714141409.8097475,"name":"offline","context":{"idset":"9511"}}
+{"timestamp":1714141409.8137202,"name":"offline","context":{"idset":"9512"}}
+{"timestamp":1714141409.8171916,"name":"offline","context":{"idset":"9513"}}
+{"timestamp":1714141409.8207798,"name":"offline","context":{"idset":"9514"}}
+{"timestamp":1714141409.8245916,"name":"offline","context":{"idset":"9515"}}
+{"timestamp":1714141409.8281381,"name":"offline","context":{"idset":"9516"}}
+{"timestamp":1714141409.8317618,"name":"offline","context":{"idset":"9517"}}
+{"timestamp":1714141409.8354306,"name":"offline","context":{"idset":"9518"}}
+{"timestamp":1714141409.8391204,"name":"offline","context":{"idset":"9519"}}
+{"timestamp":1714141409.8427005,"name":"offline","context":{"idset":"9520"}}
+{"timestamp":1714141409.846211,"name":"offline","context":{"idset":"9521"}}
+{"timestamp":1714141409.8497441,"name":"offline","context":{"idset":"9522"}}
+{"timestamp":1714141409.8532069,"name":"offline","context":{"idset":"9523"}}
+{"timestamp":1714141410.650295,"name":"offline","context":{"idset":"9524"}}
+{"timestamp":1714141427.7758214,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1714141428.3826306,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1714141767.75982,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1714141767.8480902,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1714141769.7727466,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1714141769.7758062,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1714141769.7789016,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1714141769.8452115,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1714141771.7658854,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1714141771.7696757,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1714141771.7738879,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1714141771.7777574,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1714141771.8471842,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1714141773.7577214,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1714141773.7595894,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1714141773.761426,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1714141773.8464856,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1714141775.7608211,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1714141775.765444,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1714141775.7700727,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1714141775.772831,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1714141775.8460202,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1714141777.7695286,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1714141777.774266,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1714141777.7788861,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1714141777.8472059,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1714141779.7559383,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1714141779.8445137,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1714141781.7644548,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1714141781.767767,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1714141781.7750707,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1714141781.7784894,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1714141781.8454401,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1714141783.7714162,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1714141783.7745514,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1714141783.7779615,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1714141783.8048663,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1714141783.8781135,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1714141785.7633772,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1714141785.7704947,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1714141785.7757111,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1714141785.8454702,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1714141787.7619524,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1714141787.7651622,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1714141787.7724712,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1714141787.8447273,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1714141789.7632318,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1714141789.7670128,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1714141789.7708433,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1714141789.7747443,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1714141789.8465867,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1714141791.7683737,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1714141791.7716775,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1714141791.7750118,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1714141791.8456397,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1714141794.54969,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1714141794.6588838,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1714141797.7691498,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1714141797.7731769,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1714141797.7790182,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1714141797.8454289,"name":"offline","context":{"idset":"11439"}}
+{"timestamp":1714141799.7596672,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1714141799.7628686,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1714141799.8437042,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1714141801.7659948,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1714141801.7707133,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1714141801.7736547,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1714141801.7765865,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1714141801.8504219,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1714141856.6867867,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1714141856.6899219,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1714141856.8417547,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1714141857.1495588,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1714141857.2829249,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1714141882.6207316,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1714141883.6138682,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1714141883.6398764,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1714141886.1060288,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1714141897.7204089,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1714141897.7254159,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1714141897.7329385,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1714141898.986613,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1714141929.6215084,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1714141930.2998455,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1714141932.5591993,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1714141933.6462996,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1714141934.1882725,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1714141934.2021828,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1714141935.0800424,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1714141938.0155263,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1714141939.177177,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1714141939.6881661,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1714141939.6933343,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1714141939.7042565,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1714141940.9846146,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1714141940.9911373,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1714141941.0072248,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1714141941.0198567,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1714141941.3425908,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1714141941.376101,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1714141941.7363858,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1714141941.7479517,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1714141941.753283,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1714141941.9352746,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1714141942.4608452,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1714141942.4640512,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1714141942.4670146,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1714141942.4700086,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1714141942.4736207,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1714141942.4767251,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1714141942.4797418,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1714141942.4827459,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1714141942.4857001,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1714141942.4886591,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1714141942.4916787,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1714141942.5179055,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1714141942.6783197,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1714141944.3868937,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1714141950.5019209,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1714141952.5893953,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1714141952.6107054,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1714141952.6135743,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1714141952.6774404,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1714141952.7290761,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1714141952.7884233,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1714141953.3940849,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1714141953.4066463,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1714141953.409528,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1714141953.4213693,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1714141953.4243317,"name":"offline","context":{"idset":"9525"}}
+{"timestamp":1714141953.4533627,"name":"offline","context":{"idset":"9526"}}
+{"timestamp":1714141953.4736784,"name":"offline","context":{"idset":"9527"}}
+{"timestamp":1714141953.6131008,"name":"offline","context":{"idset":"9528"}}
+{"timestamp":1714141954.6615064,"name":"offline","context":{"idset":"9529"}}
+{"timestamp":1714141954.6874576,"name":"offline","context":{"idset":"9530"}}
+{"timestamp":1714141954.7101445,"name":"offline","context":{"idset":"9531"}}
+{"timestamp":1714141954.7134554,"name":"offline","context":{"idset":"9532"}}
+{"timestamp":1714141954.7162702,"name":"offline","context":{"idset":"9533"}}
+{"timestamp":1714141954.72329,"name":"offline","context":{"idset":"9534"}}
+{"timestamp":1714141954.7261961,"name":"offline","context":{"idset":"9536"}}
+{"timestamp":1714141954.729084,"name":"offline","context":{"idset":"9537"}}
+{"timestamp":1714141954.7359829,"name":"offline","context":{"idset":"9538"}}
+{"timestamp":1714141954.7669215,"name":"offline","context":{"idset":"9539"}}
+{"timestamp":1714141954.8148634,"name":"offline","context":{"idset":"9540"}}
+{"timestamp":1714141954.8177409,"name":"offline","context":{"idset":"9541"}}
+{"timestamp":1714141954.8517966,"name":"offline","context":{"idset":"9542"}}
+{"timestamp":1714141954.8549967,"name":"offline","context":{"idset":"9543"}}
+{"timestamp":1714141954.857583,"name":"offline","context":{"idset":"9544"}}
+{"timestamp":1714141954.8604064,"name":"offline","context":{"idset":"9545"}}
+{"timestamp":1714141954.8637772,"name":"offline","context":{"idset":"9546"}}
+{"timestamp":1714141954.8667138,"name":"offline","context":{"idset":"9547"}}
+{"timestamp":1714141954.8696702,"name":"offline","context":{"idset":"9548"}}
+{"timestamp":1714141954.8726039,"name":"offline","context":{"idset":"9549"}}
+{"timestamp":1714141954.8755462,"name":"offline","context":{"idset":"9550"}}
+{"timestamp":1714141954.8785822,"name":"offline","context":{"idset":"9551"}}
+{"timestamp":1714141954.8815207,"name":"offline","context":{"idset":"9552"}}
+{"timestamp":1714141954.8844721,"name":"offline","context":{"idset":"9553"}}
+{"timestamp":1714141954.88748,"name":"offline","context":{"idset":"9554"}}
+{"timestamp":1714141954.8904488,"name":"offline","context":{"idset":"9555"}}
+{"timestamp":1714141954.8934028,"name":"offline","context":{"idset":"9556"}}
+{"timestamp":1714142175.853138,"name":"online","context":{"idset":"977"}}
+{"timestamp":1714142461.7409451,"name":"offline","context":{"idset":"9557"}}
+{"timestamp":1714142461.7445066,"name":"offline","context":{"idset":"9558"}}
+{"timestamp":1714142461.7481055,"name":"offline","context":{"idset":"9559"}}
+{"timestamp":1714142461.7530529,"name":"offline","context":{"idset":"9560"}}
+{"timestamp":1714142461.7566249,"name":"offline","context":{"idset":"9561"}}
+{"timestamp":1714142462.6023538,"name":"offline","context":{"idset":"9562"}}
+{"timestamp":1714142462.6609442,"name":"offline","context":{"idset":"9563"}}
+{"timestamp":1714142462.66468,"name":"offline","context":{"idset":"9564"}}
+{"timestamp":1714142462.6729102,"name":"offline","context":{"idset":"9565"}}
+{"timestamp":1714142462.6757126,"name":"offline","context":{"idset":"9566"}}
+{"timestamp":1714142462.6785188,"name":"offline","context":{"idset":"9567"}}
+{"timestamp":1714142462.681391,"name":"offline","context":{"idset":"9568"}}
+{"timestamp":1714142462.6841776,"name":"offline","context":{"idset":"9569"}}
+{"timestamp":1714142462.6869502,"name":"offline","context":{"idset":"9570"}}
+{"timestamp":1714142462.6897166,"name":"offline","context":{"idset":"9571"}}
+{"timestamp":1714142462.7218285,"name":"offline","context":{"idset":"9572"}}
+{"timestamp":1714142465.7456658,"name":"offline","context":{"idset":"9573"}}
+{"timestamp":1714142465.7484336,"name":"offline","context":{"idset":"9574"}}
+{"timestamp":1714142465.7514186,"name":"offline","context":{"idset":"9575"}}
+{"timestamp":1714142465.754133,"name":"offline","context":{"idset":"9576"}}
+{"timestamp":1714142465.7569551,"name":"offline","context":{"idset":"9577"}}
+{"timestamp":1714142465.7596712,"name":"offline","context":{"idset":"9578"}}
+{"timestamp":1714142465.7623823,"name":"offline","context":{"idset":"9579"}}
+{"timestamp":1714142465.7644315,"name":"offline","context":{"idset":"9580"}}
+{"timestamp":1714142465.7663288,"name":"offline","context":{"idset":"9581"}}
+{"timestamp":1714142465.7693069,"name":"offline","context":{"idset":"9582"}}
+{"timestamp":1714142465.7726429,"name":"offline","context":{"idset":"9583"}}
+{"timestamp":1714142465.7760069,"name":"offline","context":{"idset":"9584"}}
+{"timestamp":1714142465.7793667,"name":"offline","context":{"idset":"9585"}}
+{"timestamp":1714142465.7827306,"name":"offline","context":{"idset":"9586"}}
+{"timestamp":1714142465.7860863,"name":"offline","context":{"idset":"9587"}}
+{"timestamp":1714142465.8192983,"name":"offline","context":{"idset":"9588"}}
+{"timestamp":1714143211.4266233,"name":"drain","context":{"idset":"971-974","overwrite":0}}
+{"timestamp":1714144546.4825809,"name":"online","context":{"idset":"9465"}}
+{"timestamp":1714144546.6579535,"name":"online","context":{"idset":"9461,9468"}}
+{"timestamp":1714144546.9969513,"name":"online","context":{"idset":"9462,9471"}}
+{"timestamp":1714144547.1906095,"name":"online","context":{"idset":"9463-9464,9466"}}
+{"timestamp":1714144547.3744676,"name":"online","context":{"idset":"9467,9469,9475"}}
+{"timestamp":1714144547.476861,"name":"online","context":{"idset":"9470,9473-9474,9476"}}
+{"timestamp":1714144547.5806608,"name":"online","context":{"idset":"9472"}}
+{"timestamp":1714144603.2998865,"name":"drain","context":{"idset":"977-978","overwrite":0}}
+{"timestamp":1714144617.0994308,"name":"online","context":{"idset":"9479,9489"}}
+{"timestamp":1714144617.6971598,"name":"online","context":{"idset":"9492"}}
+{"timestamp":1714144618.6005516,"name":"online","context":{"idset":"9477,9482,9491"}}
+{"timestamp":1714144618.6036279,"name":"online","context":{"idset":"9481,9483-9487"}}
+{"timestamp":1714144618.6067388,"name":"online","context":{"idset":"9480,9488"}}
+{"timestamp":1714144618.609731,"name":"online","context":{"idset":"9478,9490"}}
+{"timestamp":1714144686.4802899,"name":"drain","context":{"idset":"9506","reason":"broker was unresponsive"}}
+{"timestamp":1714144687.6158659,"name":"online","context":{"idset":"9495,9500,9504"}}
+{"timestamp":1714144688.6471477,"name":"online","context":{"idset":"9506"}}
+{"timestamp":1714144688.6517444,"name":"online","context":{"idset":"9493,9505"}}
+{"timestamp":1714144688.6567972,"name":"online","context":{"idset":"9496,9508"}}
+{"timestamp":1714144688.6622024,"name":"online","context":{"idset":"9502,9507"}}
+{"timestamp":1714144688.6670153,"name":"online","context":{"idset":"9499,9501"}}
+{"timestamp":1714144689.084625,"name":"online","context":{"idset":"9497"}}
+{"timestamp":1714144689.2791605,"name":"online","context":{"idset":"9503"}}
+{"timestamp":1714144689.383945,"name":"online","context":{"idset":"9494"}}
+{"timestamp":1714144690.3276293,"name":"online","context":{"idset":"9498"}}
+{"timestamp":1714144790.5944619,"name":"online","context":{"idset":"9522"}}
+{"timestamp":1714144790.8270259,"name":"online","context":{"idset":"9509,9516,9520"}}
+{"timestamp":1714144791.0803795,"name":"online","context":{"idset":"9510,9514,9519,9521"}}
+{"timestamp":1714144791.3055224,"name":"online","context":{"idset":"9512-9513,9517,9523-9524"}}
+{"timestamp":1714144791.5524483,"name":"online","context":{"idset":"9511,9515,9518"}}
+{"timestamp":1714144794.4461098,"name":"online","context":{"idset":"978"}}
+{"timestamp":1714144858.8824589,"name":"online","context":{"idset":"9527"}}
+{"timestamp":1714144907.1120698,"name":"online","context":{"idset":"9554"}}
+{"timestamp":1714144907.5850976,"name":"online","context":{"idset":"9545"}}
+{"timestamp":1714144907.7666779,"name":"online","context":{"idset":"9550"}}
+{"timestamp":1714144969.2159033,"name":"online","context":{"idset":"9569"}}
+{"timestamp":1714144969.4936333,"name":"online","context":{"idset":"9567"}}
+{"timestamp":1714145175.161624,"name":"online","context":{"idset":"9585"}}
+{"timestamp":1714145175.3421359,"name":"online","context":{"idset":"9577,9582"}}
+{"timestamp":1714145176.5858662,"name":"online","context":{"idset":"9575"}}
+{"timestamp":1714145392.8812628,"name":"online","context":{"idset":"11387,11396"}}
+{"timestamp":1714145392.9852765,"name":"online","context":{"idset":"11383"}}
+{"timestamp":1714145393.0898731,"name":"online","context":{"idset":"11386"}}
+{"timestamp":1714145393.3349423,"name":"online","context":{"idset":"11388"}}
+{"timestamp":1714145393.4375818,"name":"online","context":{"idset":"11397-11398"}}
+{"timestamp":1714145393.522855,"name":"online","context":{"idset":"11382,11392,11399,11416"}}
+{"timestamp":1714145393.6035733,"name":"online","context":{"idset":"11408"}}
+{"timestamp":1714145393.7695048,"name":"online","context":{"idset":"11409"}}
+{"timestamp":1714145394.4845455,"name":"online","context":{"idset":"11381,11401,11418"}}
+{"timestamp":1714145394.488869,"name":"online","context":{"idset":"11390,11395,11404"}}
+{"timestamp":1714145394.4921546,"name":"online","context":{"idset":"11433"}}
+{"timestamp":1714145394.5849273,"name":"online","context":{"idset":"11389"}}
+{"timestamp":1714145394.6526484,"name":"online","context":{"idset":"11393"}}
+{"timestamp":1714145394.7374239,"name":"online","context":{"idset":"11403,11406,11414,11417"}}
+{"timestamp":1714145395.0360093,"name":"online","context":{"idset":"11412"}}
+{"timestamp":1714145395.1322145,"name":"online","context":{"idset":"11426,11449"}}
+{"timestamp":1714145395.2215114,"name":"online","context":{"idset":"11385,11402"}}
+{"timestamp":1714145395.3068211,"name":"online","context":{"idset":"11400"}}
+{"timestamp":1714145395.4135566,"name":"online","context":{"idset":"11506"}}
+{"timestamp":1714145395.7003541,"name":"online","context":{"idset":"11391,11411,11432,11446,11496"}}
+{"timestamp":1714145396.5052414,"name":"online","context":{"idset":"11394,11427,11431,11434,11444,11462,11474,11479,11492,11495,11497,11507"}}
+{"timestamp":1714145396.5092969,"name":"online","context":{"idset":"11419,11484"}}
+{"timestamp":1714145396.5149736,"name":"online","context":{"idset":"11384,11415,11436,11448,11450-11451,11459,11473,11485,11500"}}
+{"timestamp":1714145396.5209258,"name":"online","context":{"idset":"11421,11439,11464,11483,11486"}}
+{"timestamp":1714145396.526849,"name":"online","context":{"idset":"11422,11430,11458,11471"}}
+{"timestamp":1714145396.5957849,"name":"online","context":{"idset":"11425,11475,11491,11498-11499,11504-11505"}}
+{"timestamp":1714145396.7908509,"name":"online","context":{"idset":"11407,11441-11442,11457,11463,11466,11480,11482,11487,11501"}}
+{"timestamp":1714145397.078444,"name":"online","context":{"idset":"11405,11413,11423-11424,11443,11445,11447,11453-11454,11477,11489-11490,11494,11502,11508"}}
+{"timestamp":1714145397.2642055,"name":"online","context":{"idset":"11429,11435,11452,11456,11465,11468,11470,11478,11481"}}
+{"timestamp":1714145397.4302993,"name":"online","context":{"idset":"11428,11437-11438,11461"}}
+{"timestamp":1714145397.6014268,"name":"online","context":{"idset":"11410,11440,11455,11460,11467,11476,11488,11493"}}
+{"timestamp":1714145397.7729437,"name":"online","context":{"idset":"11420,11472,11503"}}
+{"timestamp":1714145398.4665709,"name":"online","context":{"idset":"11469"}}
+{"timestamp":1714145717.6314213,"name":"undrain","context":{"idset":"9506"}}
+{"timestamp":1714145838.426343,"name":"online","context":{"idset":"9534,9538"}}
+{"timestamp":1714145838.429594,"name":"online","context":{"idset":"9537"}}
+{"timestamp":1714145838.4331071,"name":"online","context":{"idset":"9535"}}
+{"timestamp":1714145838.4866741,"name":"online","context":{"idset":"9531,9533"}}
+{"timestamp":1714145838.6853902,"name":"online","context":{"idset":"9528-9529,9536,9540"}}
+{"timestamp":1714145838.8664427,"name":"online","context":{"idset":"9525"}}
+{"timestamp":1714145839.036171,"name":"online","context":{"idset":"9530,9532,9539"}}
+{"timestamp":1714145839.5132599,"name":"online","context":{"idset":"9526"}}
+{"timestamp":1714145900.5766075,"name":"online","context":{"idset":"9542"}}
+{"timestamp":1714145900.5811424,"name":"online","context":{"idset":"9546"}}
+{"timestamp":1714145900.5855143,"name":"online","context":{"idset":"9549"}}
+{"timestamp":1714145900.5896585,"name":"online","context":{"idset":"9548,9553"}}
+{"timestamp":1714145900.6815786,"name":"online","context":{"idset":"9556"}}
+{"timestamp":1714145900.8668265,"name":"online","context":{"idset":"9541,9543-9544,9551-9552"}}
+{"timestamp":1714145901.1672063,"name":"online","context":{"idset":"9547,9555"}}
+{"timestamp":1714146027.0547664,"name":"online","context":{"idset":"9559,9565"}}
+{"timestamp":1714146027.2293296,"name":"online","context":{"idset":"9571"}}
+{"timestamp":1714146027.402591,"name":"online","context":{"idset":"9557,9572"}}
+{"timestamp":1714146027.5811679,"name":"online","context":{"idset":"9561,9568"}}
+{"timestamp":1714146027.7705216,"name":"online","context":{"idset":"9560"}}
+{"timestamp":1714146028.5931063,"name":"online","context":{"idset":"9570"}}
+{"timestamp":1714146028.596467,"name":"online","context":{"idset":"9558,9562"}}
+{"timestamp":1714146028.600364,"name":"online","context":{"idset":"9563-9564"}}
+{"timestamp":1714146028.6042931,"name":"online","context":{"idset":"9566"}}
+{"timestamp":1714146094.8616416,"name":"online","context":{"idset":"9579"}}
+{"timestamp":1714146095.0631683,"name":"online","context":{"idset":"9574"}}
+{"timestamp":1714146095.3503029,"name":"online","context":{"idset":"9578,9581,9583"}}
+{"timestamp":1714146095.5607309,"name":"online","context":{"idset":"9576,9586-9588"}}
+{"timestamp":1714146095.7379634,"name":"online","context":{"idset":"9573,9580"}}
+{"timestamp":1714146096.5467851,"name":"online","context":{"idset":"9584"}}
+{"timestamp":1714147654.6419182,"name":"online","context":{"idset":"9139"}}
+{"timestamp":1714147654.6457028,"name":"online","context":{"idset":"9140"}}
+{"timestamp":1714149423.790935,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1714150970.6561301,"name":"online","context":{"idset":"10411-10412"}}
+{"timestamp":1714151051.2748151,"name":"online","context":{"idset":"6575"}}
+{"timestamp":1714151051.3928599,"name":"online","context":{"idset":"6576,8341"}}
+{"timestamp":1714151051.4976306,"name":"online","context":{"idset":"8432"}}
+{"timestamp":1714151051.715668,"name":"online","context":{"idset":"8310,8431"}}
+{"timestamp":1714151052.6742074,"name":"online","context":{"idset":"8342"}}
+{"timestamp":1714151052.6779733,"name":"online","context":{"idset":"8309"}}
+{"timestamp":1714151053.1916716,"name":"online","context":{"idset":"8597-8598"}}
+{"timestamp":1714151504.8639252,"name":"online","context":{"idset":"899"}}
+{"timestamp":1714151505.3358519,"name":"online","context":{"idset":"900"}}
+{"timestamp":1714151519.2443905,"name":"online","context":{"idset":"10204"}}
+{"timestamp":1714151684.7456346,"name":"online","context":{"idset":"10214"}}
+{"timestamp":1714151770.2113211,"name":"drain","context":{"idset":"8024","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1714151773.4820457,"name":"drain","context":{"idset":"8011","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1714151887.5511158,"name":"online","context":{"idset":"10193"}}
+{"timestamp":1714151887.6035287,"name":"online","context":{"idset":"10189,10194"}}
+{"timestamp":1714151888.6768448,"name":"online","context":{"idset":"10191"}}
+{"timestamp":1714151991.0504656,"name":"undrain","context":{"idset":"8011"}}
+{"timestamp":1714151995.5298543,"name":"undrain","context":{"idset":"8024"}}
+{"timestamp":1714152040.51109,"name":"drain","context":{"idset":"10186","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5111861,"name":"drain","context":{"idset":"10187","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5112522,"name":"drain","context":{"idset":"10188","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5113153,"name":"drain","context":{"idset":"10190","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5113764,"name":"drain","context":{"idset":"10192","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5114393,"name":"drain","context":{"idset":"10195","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5115006,"name":"drain","context":{"idset":"10196","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5115645,"name":"drain","context":{"idset":"10197","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.511627,"name":"drain","context":{"idset":"10198","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5117035,"name":"drain","context":{"idset":"10199","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5117755,"name":"drain","context":{"idset":"10200","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5118482,"name":"drain","context":{"idset":"10203","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5119133,"name":"drain","context":{"idset":"10205","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5119762,"name":"drain","context":{"idset":"10206","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5120378,"name":"drain","context":{"idset":"10208","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5121007,"name":"drain","context":{"idset":"10209","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5121663,"name":"drain","context":{"idset":"10210","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5122347,"name":"drain","context":{"idset":"10211","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5123034,"name":"drain","context":{"idset":"10215","reason":"broker was unresponsive"}}
+{"timestamp":1714152040.5123684,"name":"drain","context":{"idset":"10216","reason":"broker was unresponsive"}}
+{"timestamp":1714152043.0728531,"name":"online","context":{"idset":"10187,10190,10192,10195-10196,10215-10216"}}
+{"timestamp":1714152043.3359025,"name":"online","context":{"idset":"10186,10188,10198,10202,10205,10207,10211-10213"}}
+{"timestamp":1714152043.5405836,"name":"online","context":{"idset":"10197,10200,10203,10209"}}
+{"timestamp":1714152043.7758753,"name":"online","context":{"idset":"10199,10201,10206,10208,10210"}}
+{"timestamp":1714152351.1001036,"name":"online","context":{"idset":"7711"}}
+{"timestamp":1714152351.3063865,"name":"online","context":{"idset":"7712"}}
+{"timestamp":1714152463.5692315,"name":"online","context":{"idset":"6630"}}
+{"timestamp":1714152625.7145836,"name":"offline","context":{"idset":"7991"}}
+{"timestamp":1714152625.7932711,"name":"offline","context":{"idset":"7992"}}
+{"timestamp":1714152790.6382608,"name":"undrain","context":{"idset":"10196"}}
+{"timestamp":1714152797.1653099,"name":"undrain","context":{"idset":"10190"}}
+{"timestamp":1714152801.5892525,"name":"undrain","context":{"idset":"10198"}}
+{"timestamp":1714152806.6527977,"name":"undrain","context":{"idset":"10188"}}
+{"timestamp":1714152809.6032138,"name":"undrain","context":{"idset":"10199"}}
+{"timestamp":1714152813.3654094,"name":"undrain","context":{"idset":"10192"}}
+{"timestamp":1714152818.6273134,"name":"undrain","context":{"idset":"10197"}}
+{"timestamp":1714152822.6006746,"name":"undrain","context":{"idset":"10200"}}
+{"timestamp":1714152825.5936303,"name":"undrain","context":{"idset":"10195"}}
+{"timestamp":1714152852.6143408,"name":"drain","context":{"idset":"8011","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1714153037.7935379,"name":"offline","context":{"idset":"9180"}}
+{"timestamp":1714153069.7908871,"name":"offline","context":{"idset":"827"}}
+{"timestamp":1714153071.789851,"name":"offline","context":{"idset":"828"}}
+{"timestamp":1714153531.3510318,"name":"drain","context":{"idset":"825-826","reason":"--reason H/W troubleshoot","overwrite":0}}
+{"timestamp":1714153691.9059939,"name":"drain","context":{"idset":"881-882","reason":"--reason H/W troubleshoot","overwrite":0}}
+{"timestamp":1714154352.884202,"name":"undrain","context":{"idset":"10216"}}
+{"timestamp":1714154397.792778,"name":"offline","context":{"idset":"8144"}}
+{"timestamp":1714155107.8807948,"name":"online","context":{"idset":"10133"}}
+{"timestamp":1714155107.9838457,"name":"online","context":{"idset":"10107"}}
+{"timestamp":1714155108.0914185,"name":"online","context":{"idset":"10104,10131"}}
+{"timestamp":1714155108.9603891,"name":"online","context":{"idset":"10103,10112"}}
+{"timestamp":1714155108.9639044,"name":"online","context":{"idset":"10120,10142"}}
+{"timestamp":1714155108.9692841,"name":"online","context":{"idset":"10105,10149,10151"}}
+{"timestamp":1714155108.9749386,"name":"online","context":{"idset":"10139"}}
+{"timestamp":1714155109.4195776,"name":"online","context":{"idset":"10127,10164"}}
+{"timestamp":1714155109.4273303,"name":"online","context":{"idset":"10117"}}
+{"timestamp":1714155109.5739951,"name":"online","context":{"idset":"10106"}}
+{"timestamp":1714155109.6687627,"name":"online","context":{"idset":"10121,10132"}}
+{"timestamp":1714155109.9723167,"name":"online","context":{"idset":"10102"}}
+{"timestamp":1714155110.8774014,"name":"online","context":{"idset":"10116,10157"}}
+{"timestamp":1714155110.8811545,"name":"online","context":{"idset":"10109,10111,10123,10125,10128,10160,10163,10219"}}
+{"timestamp":1714155110.8846998,"name":"online","context":{"idset":"10108,10135,10141"}}
+{"timestamp":1714155110.8883133,"name":"online","context":{"idset":"10118,10129,10155,10178,10185"}}
+{"timestamp":1714155110.8923104,"name":"online","context":{"idset":"10113,10147,10177"}}
+{"timestamp":1714155110.9495111,"name":"online","context":{"idset":"10122"}}
+{"timestamp":1714155111.1501606,"name":"online","context":{"idset":"10115,10119,10126,10130,10137,10167,10184,10218,10223"}}
+{"timestamp":1714155111.3233833,"name":"online","context":{"idset":"10124"}}
+{"timestamp":1714155111.4371588,"name":"online","context":{"idset":"10101,10110,10114,10134,10144,10146,10153,10168,10176,10181,10183,10224,10226-10228"}}
+{"timestamp":1714155111.5405173,"name":"online","context":{"idset":"10158"}}
+{"timestamp":1714155111.6498685,"name":"online","context":{"idset":"10136,10150,10159,10161,10165,10173-10175,10179,10182,10217,10220-10221,10225"}}
+{"timestamp":1714155111.8171918,"name":"online","context":{"idset":"10138,10148,10162,10172"}}
+{"timestamp":1714155111.983454,"name":"online","context":{"idset":"10140,10143,10156,10166,10169-10171,10180,10222"}}
+{"timestamp":1714155112.8410096,"name":"online","context":{"idset":"10145"}}
+{"timestamp":1714155148.9613528,"name":"offline","context":{"idset":"8011"}}
+{"timestamp":1714155765.9416585,"name":"offline","context":{"idset":"6527"}}
+{"timestamp":1714155766.1301365,"name":"offline","context":{"idset":"6522"}}
+{"timestamp":1714155766.9813759,"name":"offline","context":{"idset":"6539"}}
+{"timestamp":1714155766.9857531,"name":"offline","context":{"idset":"6524"}}
+{"timestamp":1714155767.0013001,"name":"offline","context":{"idset":"6558"}}
+{"timestamp":1714155767.0054412,"name":"offline","context":{"idset":"6571"}}
+{"timestamp":1714155767.0097733,"name":"offline","context":{"idset":"6518"}}
+{"timestamp":1714155767.0140762,"name":"offline","context":{"idset":"6591"}}
+{"timestamp":1714155767.0182905,"name":"offline","context":{"idset":"6545"}}
+{"timestamp":1714155767.0222514,"name":"offline","context":{"idset":"6530"}}
+{"timestamp":1714155767.0262437,"name":"offline","context":{"idset":"6548"}}
+{"timestamp":1714155767.0302243,"name":"offline","context":{"idset":"6537"}}
+{"timestamp":1714155767.033674,"name":"offline","context":{"idset":"6555"}}
+{"timestamp":1714155767.0374374,"name":"offline","context":{"idset":"6596"}}
+{"timestamp":1714155767.0415468,"name":"offline","context":{"idset":"6577"}}
+{"timestamp":1714155767.0456178,"name":"offline","context":{"idset":"6584"}}
+{"timestamp":1714155767.0494072,"name":"offline","context":{"idset":"6569"}}
+{"timestamp":1714155767.0532475,"name":"offline","context":{"idset":"6583"}}
+{"timestamp":1714155767.056674,"name":"offline","context":{"idset":"6594"}}
+{"timestamp":1714155767.0605159,"name":"offline","context":{"idset":"6543"}}
+{"timestamp":1714155767.0644209,"name":"offline","context":{"idset":"6568"}}
+{"timestamp":1714155767.0686159,"name":"offline","context":{"idset":"6538"}}
+{"timestamp":1714155767.0726087,"name":"offline","context":{"idset":"6573"}}
+{"timestamp":1714155767.077039,"name":"offline","context":{"idset":"6535"}}
+{"timestamp":1714155767.0811994,"name":"offline","context":{"idset":"6519"}}
+{"timestamp":1714155767.0853608,"name":"offline","context":{"idset":"6536"}}
+{"timestamp":1714155767.0896299,"name":"offline","context":{"idset":"6528"}}
+{"timestamp":1714155767.0937262,"name":"offline","context":{"idset":"6540"}}
+{"timestamp":1714155767.0978973,"name":"offline","context":{"idset":"6550"}}
+{"timestamp":1714155767.1020541,"name":"offline","context":{"idset":"6586"}}
+{"timestamp":1714155767.1060214,"name":"offline","context":{"idset":"6570"}}
+{"timestamp":1714155767.1098576,"name":"offline","context":{"idset":"6549"}}
+{"timestamp":1714155767.1144235,"name":"offline","context":{"idset":"6560"}}
+{"timestamp":1714155767.1187468,"name":"offline","context":{"idset":"6597"}}
+{"timestamp":1714155767.2242665,"name":"offline","context":{"idset":"6595"}}
+{"timestamp":1714155767.4582021,"name":"offline","context":{"idset":"6580"}}
+{"timestamp":1714155767.4623845,"name":"offline","context":{"idset":"6520"}}
+{"timestamp":1714155767.4661052,"name":"offline","context":{"idset":"6559"}}
+{"timestamp":1714155767.4700186,"name":"offline","context":{"idset":"6587"}}
+{"timestamp":1714155767.4739065,"name":"offline","context":{"idset":"6578"}}
+{"timestamp":1714155767.4777036,"name":"offline","context":{"idset":"6567"}}
+{"timestamp":1714155767.4817042,"name":"offline","context":{"idset":"6572"}}
+{"timestamp":1714155767.4856124,"name":"offline","context":{"idset":"6526"}}
+{"timestamp":1714155767.4894993,"name":"offline","context":{"idset":"6546"}}
+{"timestamp":1714155767.4934671,"name":"offline","context":{"idset":"6592"}}
+{"timestamp":1714155767.4973176,"name":"offline","context":{"idset":"6531"}}
+{"timestamp":1714155767.5012054,"name":"offline","context":{"idset":"6601"}}
+{"timestamp":1714155767.5050926,"name":"offline","context":{"idset":"6579"}}
+{"timestamp":1714155767.5089595,"name":"offline","context":{"idset":"6589"}}
+{"timestamp":1714155767.5127733,"name":"offline","context":{"idset":"6544"}}
+{"timestamp":1714155767.5162964,"name":"offline","context":{"idset":"6574"}}
+{"timestamp":1714155767.5195739,"name":"offline","context":{"idset":"6517"}}
+{"timestamp":1714155767.5233023,"name":"offline","context":{"idset":"6566"}}
+{"timestamp":1714155767.5261934,"name":"offline","context":{"idset":"6565"}}
+{"timestamp":1714155767.5404148,"name":"offline","context":{"idset":"6521"}}
+{"timestamp":1714155767.5442331,"name":"offline","context":{"idset":"6532"}}
+{"timestamp":1714155767.5483184,"name":"offline","context":{"idset":"6533"}}
+{"timestamp":1714155767.5526073,"name":"offline","context":{"idset":"6542"}}
+{"timestamp":1714155767.5563304,"name":"offline","context":{"idset":"6581"}}
+{"timestamp":1714155767.5603228,"name":"offline","context":{"idset":"6593"}}
+{"timestamp":1714155767.5638883,"name":"offline","context":{"idset":"6599"}}
+{"timestamp":1714155767.5672481,"name":"offline","context":{"idset":"6606"}}
+{"timestamp":1714155767.5709944,"name":"offline","context":{"idset":"6613"}}
+{"timestamp":1714155767.6868906,"name":"offline","context":{"idset":"6622"}}
+{"timestamp":1714155767.8323948,"name":"offline","context":{"idset":"6623"}}
+{"timestamp":1714155767.8365993,"name":"offline","context":{"idset":"6631"}}
+{"timestamp":1714155767.8406844,"name":"offline","context":{"idset":"6627"}}
+{"timestamp":1714155767.844841,"name":"offline","context":{"idset":"6562"}}
+{"timestamp":1714155767.8489749,"name":"offline","context":{"idset":"6552"}}
+{"timestamp":1714155767.8528841,"name":"offline","context":{"idset":"6639"}}
+{"timestamp":1714155767.8570125,"name":"offline","context":{"idset":"6642"}}
+{"timestamp":1714155767.8611619,"name":"offline","context":{"idset":"6609"}}
+{"timestamp":1714155767.8652554,"name":"offline","context":{"idset":"6557"}}
+{"timestamp":1714155767.8691492,"name":"offline","context":{"idset":"6640"}}
+{"timestamp":1714155767.8726368,"name":"offline","context":{"idset":"6611"}}
+{"timestamp":1714155767.8839102,"name":"offline","context":{"idset":"6643"}}
+{"timestamp":1714155767.8878231,"name":"offline","context":{"idset":"6523"}}
+{"timestamp":1714155767.8916669,"name":"offline","context":{"idset":"6541"}}
+{"timestamp":1714155767.9047482,"name":"offline","context":{"idset":"6547"}}
+{"timestamp":1714155767.908462,"name":"offline","context":{"idset":"6612"}}
+{"timestamp":1714155767.9116566,"name":"offline","context":{"idset":"6630"}}
+{"timestamp":1714155767.9156516,"name":"offline","context":{"idset":"6637"}}
+{"timestamp":1714155768.0992687,"name":"offline","context":{"idset":"6629"}}
+{"timestamp":1714155768.1429675,"name":"offline","context":{"idset":"6590"}}
+{"timestamp":1714155768.1471851,"name":"offline","context":{"idset":"6604"}}
+{"timestamp":1714155768.1515379,"name":"offline","context":{"idset":"6614"}}
+{"timestamp":1714155769.0143754,"name":"offline","context":{"idset":"6634"}}
+{"timestamp":1714155769.0180182,"name":"offline","context":{"idset":"6551"}}
+{"timestamp":1714155769.0219307,"name":"offline","context":{"idset":"6534"}}
+{"timestamp":1714155769.0255902,"name":"offline","context":{"idset":"6553"}}
+{"timestamp":1714155769.0290248,"name":"offline","context":{"idset":"6561"}}
+{"timestamp":1714155769.032454,"name":"offline","context":{"idset":"6525"}}
+{"timestamp":1714155769.0363891,"name":"offline","context":{"idset":"6585"}}
+{"timestamp":1714155769.0406082,"name":"offline","context":{"idset":"6563"}}
+{"timestamp":1714155769.0446475,"name":"offline","context":{"idset":"6588"}}
+{"timestamp":1714155769.0487969,"name":"offline","context":{"idset":"6529"}}
+{"timestamp":1714155769.0528519,"name":"offline","context":{"idset":"6556"}}
+{"timestamp":1714155769.0565434,"name":"offline","context":{"idset":"6618"}}
+{"timestamp":1714155769.0605667,"name":"offline","context":{"idset":"6607"}}
+{"timestamp":1714155769.0644,"name":"offline","context":{"idset":"6633"}}
+{"timestamp":1714155769.0683055,"name":"offline","context":{"idset":"6603"}}
+{"timestamp":1714155769.0722435,"name":"offline","context":{"idset":"6638"}}
+{"timestamp":1714155769.0761521,"name":"offline","context":{"idset":"6582"}}
+{"timestamp":1714155769.0799685,"name":"offline","context":{"idset":"6554"}}
+{"timestamp":1714155769.0835292,"name":"offline","context":{"idset":"6564"}}
+{"timestamp":1714155769.0870986,"name":"offline","context":{"idset":"6600"}}
+{"timestamp":1714155769.0906093,"name":"offline","context":{"idset":"6641"}}
+{"timestamp":1714155769.0946298,"name":"offline","context":{"idset":"6620"}}
+{"timestamp":1714155769.098356,"name":"offline","context":{"idset":"6615"}}
+{"timestamp":1714155769.1017973,"name":"offline","context":{"idset":"6628"}}
+{"timestamp":1714155769.1063027,"name":"offline","context":{"idset":"6605"}}
+{"timestamp":1714155769.1097481,"name":"offline","context":{"idset":"6626"}}
+{"timestamp":1714155769.113368,"name":"offline","context":{"idset":"6608"}}
+{"timestamp":1714155769.1167448,"name":"offline","context":{"idset":"6635"}}
+{"timestamp":1714155769.1202462,"name":"offline","context":{"idset":"6619"}}
+{"timestamp":1714155769.1236269,"name":"offline","context":{"idset":"6610"}}
+{"timestamp":1714155769.1271322,"name":"offline","context":{"idset":"6602"}}
+{"timestamp":1714155769.1305685,"name":"offline","context":{"idset":"6621"}}
+{"timestamp":1714155769.1340625,"name":"offline","context":{"idset":"6598"}}
+{"timestamp":1714155769.1376739,"name":"offline","context":{"idset":"6616"}}
+{"timestamp":1714155769.4743755,"name":"offline","context":{"idset":"6617"}}
+{"timestamp":1714155769.4783595,"name":"offline","context":{"idset":"6624"}}
+{"timestamp":1714155769.4816527,"name":"offline","context":{"idset":"6636"}}
+{"timestamp":1714155769.4854164,"name":"offline","context":{"idset":"6644"}}
+{"timestamp":1714155769.5054898,"name":"offline","context":{"idset":"6632"}}
+{"timestamp":1714155769.5091584,"name":"offline","context":{"idset":"6625"}}
+{"timestamp":1714155831.1438491,"name":"online","context":{"idset":"6535"}}
+{"timestamp":1714155831.6261628,"name":"online","context":{"idset":"6538"}}
+{"timestamp":1714155832.7834105,"name":"online","context":{"idset":"6531"}}
+{"timestamp":1714155835.1219249,"name":"online","context":{"idset":"6521"}}
+{"timestamp":1714155835.3872821,"name":"online","context":{"idset":"6537"}}
+{"timestamp":1714155835.8730443,"name":"online","context":{"idset":"6546"}}
+{"timestamp":1714155837.4638126,"name":"online","context":{"idset":"6527"}}
+{"timestamp":1714155838.0349095,"name":"online","context":{"idset":"6523"}}
+{"timestamp":1714155838.1359749,"name":"online","context":{"idset":"6532"}}
+{"timestamp":1714155839.1267898,"name":"online","context":{"idset":"6517"}}
+{"timestamp":1714155839.1304789,"name":"online","context":{"idset":"6543,6558"}}
+{"timestamp":1714155839.2777359,"name":"online","context":{"idset":"6560"}}
+{"timestamp":1714155839.5937457,"name":"online","context":{"idset":"6548,6551"}}
+{"timestamp":1714155839.8278477,"name":"online","context":{"idset":"6570"}}
+{"timestamp":1714155840.0537331,"name":"online","context":{"idset":"6563"}}
+{"timestamp":1714155840.7333283,"name":"online","context":{"idset":"6524"}}
+{"timestamp":1714155840.8376729,"name":"online","context":{"idset":"6567"}}
+{"timestamp":1714155840.9377396,"name":"online","context":{"idset":"6572,6595"}}
+{"timestamp":1714155841.0414679,"name":"online","context":{"idset":"6582"}}
+{"timestamp":1714155841.3546669,"name":"online","context":{"idset":"6592"}}
+{"timestamp":1714155841.4799411,"name":"online","context":{"idset":"6564,6569"}}
+{"timestamp":1714155841.6328819,"name":"online","context":{"idset":"6526"}}
+{"timestamp":1714155841.8715234,"name":"online","context":{"idset":"6557"}}
+{"timestamp":1714155842.064229,"name":"online","context":{"idset":"6562,6568,6571"}}
+{"timestamp":1714155842.8033323,"name":"online","context":{"idset":"6586"}}
+{"timestamp":1714155842.806777,"name":"online","context":{"idset":"6528,6552,6565"}}
+{"timestamp":1714155842.8104992,"name":"online","context":{"idset":"6533,6540,6544,6559,6584"}}
+{"timestamp":1714155842.814395,"name":"online","context":{"idset":"6522,6550,6553"}}
+{"timestamp":1714155842.8182833,"name":"online","context":{"idset":"6534,6545,6591"}}
+{"timestamp":1714155842.8956335,"name":"online","context":{"idset":"6574"}}
+{"timestamp":1714155843.0262492,"name":"online","context":{"idset":"6518,6587,6593"}}
+{"timestamp":1714155843.2241385,"name":"online","context":{"idset":"6549,6588"}}
+{"timestamp":1714155843.3987081,"name":"online","context":{"idset":"6520,6529-6530,6541,6554,6566,6573,6578-6581"}}
+{"timestamp":1714155843.5903893,"name":"online","context":{"idset":"6525,6539,6542,6555,6589,6596"}}
+{"timestamp":1714155843.694207,"name":"online","context":{"idset":"6577,6583"}}
+{"timestamp":1714155843.8799145,"name":"online","context":{"idset":"6519,6536,6547,6585"}}
+{"timestamp":1714155844.0726511,"name":"online","context":{"idset":"6556,6590,6594"}}
+{"timestamp":1714155844.7951326,"name":"online","context":{"idset":"6561"}}
+{"timestamp":1714155844.8316786,"name":"online","context":{"idset":"6618"}}
+{"timestamp":1714155846.7626987,"name":"online","context":{"idset":"6631"}}
+{"timestamp":1714155848.7569597,"name":"online","context":{"idset":"6605"}}
+{"timestamp":1714155851.4008904,"name":"online","context":{"idset":"6600"}}
+{"timestamp":1714155857.3959718,"name":"online","context":{"idset":"6632"}}
+{"timestamp":1714155858.9172163,"name":"online","context":{"idset":"6598"}}
+{"timestamp":1714155858.920933,"name":"online","context":{"idset":"6636"}}
+{"timestamp":1714155861.4651163,"name":"online","context":{"idset":"6641"}}
+{"timestamp":1714155862.0981569,"name":"online","context":{"idset":"6622"}}
+{"timestamp":1714155862.969373,"name":"online","context":{"idset":"6630"}}
+{"timestamp":1714155863.6934426,"name":"online","context":{"idset":"6620"}}
+{"timestamp":1714155863.965153,"name":"online","context":{"idset":"6606"}}
+{"timestamp":1714155864.8215864,"name":"online","context":{"idset":"6623"}}
+{"timestamp":1714155864.8249867,"name":"online","context":{"idset":"6614,6626,6639"}}
+{"timestamp":1714155864.8284996,"name":"online","context":{"idset":"6599"}}
+{"timestamp":1714155864.8322499,"name":"online","context":{"idset":"6628-6629,6638"}}
+{"timestamp":1714155864.9115713,"name":"online","context":{"idset":"6611"}}
+{"timestamp":1714155865.0853271,"name":"online","context":{"idset":"6617,6625"}}
+{"timestamp":1714155865.411113,"name":"online","context":{"idset":"6616,6627,6633"}}
+{"timestamp":1714155865.6557908,"name":"online","context":{"idset":"6601,6613,6621,6637,6643"}}
+{"timestamp":1714155865.8391564,"name":"online","context":{"idset":"6603,6624"}}
+{"timestamp":1714155866.0527575,"name":"online","context":{"idset":"6597,6607,6612,6619"}}
+{"timestamp":1714155866.7594137,"name":"online","context":{"idset":"6635"}}
+{"timestamp":1714155866.7627823,"name":"online","context":{"idset":"6609-6610"}}
+{"timestamp":1714155866.7662444,"name":"online","context":{"idset":"6615"}}
+{"timestamp":1714155866.7696507,"name":"online","context":{"idset":"6608"}}
+{"timestamp":1714155867.0643411,"name":"online","context":{"idset":"6602"}}
+{"timestamp":1714155867.3011439,"name":"online","context":{"idset":"6604,6634"}}
+{"timestamp":1714155867.5635841,"name":"online","context":{"idset":"6642"}}
+{"timestamp":1714155867.8400903,"name":"online","context":{"idset":"6640,6644"}}
+{"timestamp":1714156096.9362404,"name":"online","context":{"idset":"8011"}}
+{"timestamp":1714156169.3240654,"name":"online","context":{"idset":"7991-7992"}}
+{"timestamp":1714156199.5205617,"name":"drain","context":{"idset":"7733-7748","overwrite":0}}
+{"timestamp":1714156246.0967867,"name":"offline","context":{"idset":"6519"}}
+{"timestamp":1714156246.6798215,"name":"offline","context":{"idset":"6522"}}
+{"timestamp":1714156246.6819775,"name":"offline","context":{"idset":"6521"}}
+{"timestamp":1714156246.6841121,"name":"offline","context":{"idset":"6526"}}
+{"timestamp":1714156246.6862342,"name":"offline","context":{"idset":"6548"}}
+{"timestamp":1714156246.6883645,"name":"offline","context":{"idset":"6524"}}
+{"timestamp":1714156246.6904762,"name":"offline","context":{"idset":"6547"}}
+{"timestamp":1714156246.6925907,"name":"offline","context":{"idset":"6552"}}
+{"timestamp":1714156246.6946871,"name":"offline","context":{"idset":"6549"}}
+{"timestamp":1714156246.6968129,"name":"offline","context":{"idset":"6557"}}
+{"timestamp":1714156246.6989243,"name":"offline","context":{"idset":"6546"}}
+{"timestamp":1714156246.701045,"name":"offline","context":{"idset":"6518"}}
+{"timestamp":1714156246.703145,"name":"offline","context":{"idset":"6538"}}
+{"timestamp":1714156246.7052567,"name":"offline","context":{"idset":"6545"}}
+{"timestamp":1714156246.7951701,"name":"offline","context":{"idset":"6533"}}
+{"timestamp":1714156246.9067492,"name":"offline","context":{"idset":"6529"}}
+{"timestamp":1714156246.9143558,"name":"offline","context":{"idset":"6525"}}
+{"timestamp":1714156246.9164844,"name":"offline","context":{"idset":"6544"}}
+{"timestamp":1714156246.9196563,"name":"offline","context":{"idset":"6550"}}
+{"timestamp":1714156246.9464688,"name":"offline","context":{"idset":"6565"}}
+{"timestamp":1714156246.9486043,"name":"offline","context":{"idset":"6551"}}
+{"timestamp":1714156246.956286,"name":"offline","context":{"idset":"6578"}}
+{"timestamp":1714156246.9584584,"name":"offline","context":{"idset":"6598"}}
+{"timestamp":1714156246.9606049,"name":"offline","context":{"idset":"6632"}}
+{"timestamp":1714156247.0515697,"name":"offline","context":{"idset":"6532"}}
+{"timestamp":1714156247.1206813,"name":"offline","context":{"idset":"6631"}}
+{"timestamp":1714156247.1270344,"name":"offline","context":{"idset":"6520"}}
+{"timestamp":1714156247.1297812,"name":"offline","context":{"idset":"6569"}}
+{"timestamp":1714156247.1508563,"name":"offline","context":{"idset":"6583"}}
+{"timestamp":1714156247.1536663,"name":"offline","context":{"idset":"6542"}}
+{"timestamp":1714156247.2321403,"name":"offline","context":{"idset":"6594"}}
+{"timestamp":1714156247.2953327,"name":"offline","context":{"idset":"6600"}}
+{"timestamp":1714156247.3004065,"name":"offline","context":{"idset":"6556"}}
+{"timestamp":1714156247.3033779,"name":"offline","context":{"idset":"6543"}}
+{"timestamp":1714156247.3079329,"name":"offline","context":{"idset":"6530"}}
+{"timestamp":1714156247.3106189,"name":"offline","context":{"idset":"6574"}}
+{"timestamp":1714156247.3311324,"name":"offline","context":{"idset":"6568"}}
+{"timestamp":1714156247.3353064,"name":"offline","context":{"idset":"6636"}}
+{"timestamp":1714156247.3386676,"name":"offline","context":{"idset":"6539"}}
+{"timestamp":1714156247.3479095,"name":"offline","context":{"idset":"6566"}}
+{"timestamp":1714156247.4153142,"name":"offline","context":{"idset":"6610"}}
+{"timestamp":1714156247.4911978,"name":"offline","context":{"idset":"6634"}}
+{"timestamp":1714156247.4953535,"name":"offline","context":{"idset":"6537"}}
+{"timestamp":1714156247.4996207,"name":"offline","context":{"idset":"6602"}}
+{"timestamp":1714156247.5039313,"name":"offline","context":{"idset":"6607"}}
+{"timestamp":1714156247.5175488,"name":"offline","context":{"idset":"6554"}}
+{"timestamp":1714156247.5196731,"name":"offline","context":{"idset":"6531"}}
+{"timestamp":1714156247.5289021,"name":"offline","context":{"idset":"6592"}}
+{"timestamp":1714156247.6014798,"name":"offline","context":{"idset":"6609"}}
+{"timestamp":1714156247.694885,"name":"offline","context":{"idset":"6638"}}
+{"timestamp":1714156247.6999106,"name":"offline","context":{"idset":"6635"}}
+{"timestamp":1714156247.7068648,"name":"offline","context":{"idset":"6606"}}
+{"timestamp":1714156247.7114863,"name":"offline","context":{"idset":"6535"}}
+{"timestamp":1714156247.7165914,"name":"offline","context":{"idset":"6527"}}
+{"timestamp":1714156247.752975,"name":"offline","context":{"idset":"6528"}}
+{"timestamp":1714156247.7570577,"name":"offline","context":{"idset":"6555"}}
+{"timestamp":1714156247.7611253,"name":"offline","context":{"idset":"6558"}}
+{"timestamp":1714156247.7743766,"name":"offline","context":{"idset":"6564"}}
+{"timestamp":1714156247.7765913,"name":"offline","context":{"idset":"6585"}}
+{"timestamp":1714156247.778918,"name":"offline","context":{"idset":"6586"}}
+{"timestamp":1714156247.7812045,"name":"offline","context":{"idset":"6589"}}
+{"timestamp":1714156247.7834277,"name":"offline","context":{"idset":"6603"}}
+{"timestamp":1714156247.8587942,"name":"offline","context":{"idset":"6541"}}
+{"timestamp":1714156247.9393299,"name":"offline","context":{"idset":"6637"}}
+{"timestamp":1714156247.9435337,"name":"offline","context":{"idset":"6641"}}
+{"timestamp":1714156247.9471183,"name":"offline","context":{"idset":"6640"}}
+{"timestamp":1714156247.972785,"name":"offline","context":{"idset":"6630"}}
+{"timestamp":1714156247.975111,"name":"offline","context":{"idset":"6523"}}
+{"timestamp":1714156247.9835873,"name":"offline","context":{"idset":"6553"}}
+{"timestamp":1714156247.9860179,"name":"offline","context":{"idset":"6579"}}
+{"timestamp":1714156247.9883718,"name":"offline","context":{"idset":"6580"}}
+{"timestamp":1714156247.9907141,"name":"offline","context":{"idset":"6582"}}
+{"timestamp":1714156247.9930634,"name":"offline","context":{"idset":"6587"}}
+{"timestamp":1714156248.053396,"name":"offline","context":{"idset":"6588"}}
+{"timestamp":1714156248.135761,"name":"offline","context":{"idset":"6595"}}
+{"timestamp":1714156248.1598229,"name":"offline","context":{"idset":"6601"}}
+{"timestamp":1714156248.1624568,"name":"offline","context":{"idset":"6639"}}
+{"timestamp":1714156248.1675515,"name":"offline","context":{"idset":"6644"}}
+{"timestamp":1714156248.1718919,"name":"offline","context":{"idset":"6633"}}
+{"timestamp":1714156248.1888289,"name":"offline","context":{"idset":"6643"}}
+{"timestamp":1714156248.7633941,"name":"offline","context":{"idset":"6559"}}
+{"timestamp":1714156248.7657776,"name":"offline","context":{"idset":"6534"}}
+{"timestamp":1714156248.7681429,"name":"offline","context":{"idset":"6570"}}
+{"timestamp":1714156248.7705152,"name":"offline","context":{"idset":"6584"}}
+{"timestamp":1714156248.7729642,"name":"offline","context":{"idset":"6517"}}
+{"timestamp":1714156248.7753053,"name":"offline","context":{"idset":"6608"}}
+{"timestamp":1714156248.7776408,"name":"offline","context":{"idset":"6604"}}
+{"timestamp":1714156248.7799728,"name":"offline","context":{"idset":"6612"}}
+{"timestamp":1714156248.7823408,"name":"offline","context":{"idset":"6563"}}
+{"timestamp":1714156248.7847052,"name":"offline","context":{"idset":"6596"}}
+{"timestamp":1714156248.7870522,"name":"offline","context":{"idset":"6536"}}
+{"timestamp":1714156248.7893717,"name":"offline","context":{"idset":"6591"}}
+{"timestamp":1714156248.7917435,"name":"offline","context":{"idset":"6611"}}
+{"timestamp":1714156248.7940614,"name":"offline","context":{"idset":"6597"}}
+{"timestamp":1714156248.7963676,"name":"offline","context":{"idset":"6573"}}
+{"timestamp":1714156248.7986655,"name":"offline","context":{"idset":"6599"}}
+{"timestamp":1714156248.8009791,"name":"offline","context":{"idset":"6571"}}
+{"timestamp":1714156248.8033428,"name":"offline","context":{"idset":"6567"}}
+{"timestamp":1714156248.805629,"name":"offline","context":{"idset":"6540"}}
+{"timestamp":1714156248.80791,"name":"offline","context":{"idset":"6642"}}
+{"timestamp":1714156248.8101969,"name":"offline","context":{"idset":"6581"}}
+{"timestamp":1714156248.8124828,"name":"offline","context":{"idset":"6577"}}
+{"timestamp":1714156248.8147597,"name":"offline","context":{"idset":"6593"}}
+{"timestamp":1714156248.8170485,"name":"offline","context":{"idset":"6629"}}
+{"timestamp":1714156248.8193312,"name":"offline","context":{"idset":"6560"}}
+{"timestamp":1714156248.8215961,"name":"offline","context":{"idset":"6572"}}
+{"timestamp":1714156248.8238354,"name":"offline","context":{"idset":"6562"}}
+{"timestamp":1714156248.8260865,"name":"offline","context":{"idset":"6561"}}
+{"timestamp":1714156248.8283391,"name":"offline","context":{"idset":"6590"}}
+{"timestamp":1714156249.009027,"name":"offline","context":{"idset":"6605"}}
+{"timestamp":1714156252.1101215,"name":"offline","context":{"idset":"6624"}}
+{"timestamp":1714156252.7235978,"name":"offline","context":{"idset":"6617"}}
+{"timestamp":1714156252.7262998,"name":"offline","context":{"idset":"6626"}}
+{"timestamp":1714156252.7289743,"name":"offline","context":{"idset":"6622"}}
+{"timestamp":1714156252.8764381,"name":"offline","context":{"idset":"6621"}}
+{"timestamp":1714156252.8792615,"name":"offline","context":{"idset":"6618"}}
+{"timestamp":1714156252.8831978,"name":"offline","context":{"idset":"6614"}}
+{"timestamp":1714156252.9768476,"name":"offline","context":{"idset":"6616"}}
+{"timestamp":1714156253.0714939,"name":"offline","context":{"idset":"6615"}}
+{"timestamp":1714156253.074255,"name":"offline","context":{"idset":"6620"}}
+{"timestamp":1714156253.1713426,"name":"offline","context":{"idset":"6625"}}
+{"timestamp":1714156253.3278317,"name":"offline","context":{"idset":"6619"}}
+{"timestamp":1714156253.4286096,"name":"offline","context":{"idset":"6613"}}
+{"timestamp":1714156253.614156,"name":"offline","context":{"idset":"6623"}}
+{"timestamp":1714156253.7001679,"name":"offline","context":{"idset":"6627"}}
+{"timestamp":1714156253.7002487,"name":"drain","context":{"idset":"6613-6628","reason":"epilog failed for jobid frnkHKB7K1h","overwrite":0}}
+{"timestamp":1714156253.8019316,"name":"offline","context":{"idset":"6628"}}
+{"timestamp":1714156432.2182662,"name":"undrain","context":{"idset":"6575-6576"}}
+{"timestamp":1714156434.2630968,"name":"undrain","context":{"idset":"8597-8598"}}
+{"timestamp":1714156505.9387417,"name":"drain","context":{"idset":"7733-7748","overwrite":0}}
+{"timestamp":1714156649.8714864,"name":"offline","context":{"idset":"6575"}}
+{"timestamp":1714156649.8744617,"name":"offline","context":{"idset":"6576"}}
+{"timestamp":1714157165.7973709,"name":"online","context":{"idset":"10281"}}
+{"timestamp":1714157380.0660906,"name":"undrain","context":{"idset":"8431"}}
+{"timestamp":1714157508.1257133,"name":"offline","context":{"idset":"9239"}}
+{"timestamp":1714157510.0911458,"name":"offline","context":{"idset":"9323"}}
+{"timestamp":1714157592.1434813,"name":"offline","context":{"idset":"7893"}}
+{"timestamp":1714157598.6889164,"name":"offline","context":{"idset":"7894"}}
+{"timestamp":1714157716.0748842,"name":"undrain","context":{"idset":"8011"}}
+{"timestamp":1714157863.5562971,"name":"offline","context":{"idset":"8312"}}
+{"timestamp":1714157863.6328249,"name":"offline","context":{"idset":"8315"}}
+{"timestamp":1714157863.7093155,"name":"offline","context":{"idset":"8403"}}
+{"timestamp":1714157863.7174554,"name":"offline","context":{"idset":"8325"}}
+{"timestamp":1714157863.7195454,"name":"offline","context":{"idset":"8367"}}
+{"timestamp":1714157863.817116,"name":"offline","context":{"idset":"8366"}}
+{"timestamp":1714157863.9016674,"name":"offline","context":{"idset":"8400"}}
+{"timestamp":1714157863.9056063,"name":"offline","context":{"idset":"8362"}}
+{"timestamp":1714157863.9095562,"name":"offline","context":{"idset":"8370"}}
+{"timestamp":1714157863.9137192,"name":"offline","context":{"idset":"8322"}}
+{"timestamp":1714157863.9176254,"name":"offline","context":{"idset":"8320"}}
+{"timestamp":1714157863.9517527,"name":"offline","context":{"idset":"8404"}}
+{"timestamp":1714157863.9574327,"name":"offline","context":{"idset":"8329"}}
+{"timestamp":1714157863.9608953,"name":"offline","context":{"idset":"8336"}}
+{"timestamp":1714157863.9767947,"name":"offline","context":{"idset":"8337"}}
+{"timestamp":1714157863.9789541,"name":"offline","context":{"idset":"8343"}}
+{"timestamp":1714157863.981056,"name":"offline","context":{"idset":"8363"}}
+{"timestamp":1714157864.0431335,"name":"offline","context":{"idset":"8401"}}
+{"timestamp":1714157864.1439662,"name":"offline","context":{"idset":"8328"}}
+{"timestamp":1714157864.150306,"name":"offline","context":{"idset":"8327"}}
+{"timestamp":1714157864.15416,"name":"offline","context":{"idset":"8318"}}
+{"timestamp":1714157864.1579497,"name":"offline","context":{"idset":"8323"}}
+{"timestamp":1714157864.181649,"name":"offline","context":{"idset":"8335"}}
+{"timestamp":1714157864.1840968,"name":"offline","context":{"idset":"8313"}}
+{"timestamp":1714157864.197135,"name":"offline","context":{"idset":"8314"}}
+{"timestamp":1714157864.2005603,"name":"offline","context":{"idset":"8326"}}
+{"timestamp":1714157864.2037313,"name":"offline","context":{"idset":"8339"}}
+{"timestamp":1714157864.205848,"name":"offline","context":{"idset":"8394"}}
+{"timestamp":1714157864.208782,"name":"offline","context":{"idset":"8413"}}
+{"timestamp":1714157864.2132177,"name":"offline","context":{"idset":"8415"}}
+{"timestamp":1714157865.3685167,"name":"offline","context":{"idset":"8316"}}
+{"timestamp":1714157865.3714976,"name":"offline","context":{"idset":"8431"}}
+{"timestamp":1714157865.3744612,"name":"offline","context":{"idset":"8380"}}
+{"timestamp":1714157865.3774288,"name":"offline","context":{"idset":"8361"}}
+{"timestamp":1714157865.3803449,"name":"offline","context":{"idset":"8360"}}
+{"timestamp":1714157865.3833587,"name":"offline","context":{"idset":"8341"}}
+{"timestamp":1714157865.3863053,"name":"offline","context":{"idset":"8348"}}
+{"timestamp":1714157865.3895891,"name":"offline","context":{"idset":"8386"}}
+{"timestamp":1714157865.3928137,"name":"offline","context":{"idset":"8350"}}
+{"timestamp":1714157865.3959684,"name":"offline","context":{"idset":"8310"}}
+{"timestamp":1714157865.3989182,"name":"offline","context":{"idset":"8378"}}
+{"timestamp":1714157865.4018803,"name":"offline","context":{"idset":"8381"}}
+{"timestamp":1714157865.4048307,"name":"offline","context":{"idset":"8375"}}
+{"timestamp":1714157865.4077649,"name":"offline","context":{"idset":"8309"}}
+{"timestamp":1714157865.4107385,"name":"offline","context":{"idset":"8387"}}
+{"timestamp":1714157865.4137509,"name":"offline","context":{"idset":"8333"}}
+{"timestamp":1714157865.4191265,"name":"offline","context":{"idset":"8374"}}
+{"timestamp":1714157865.4232345,"name":"offline","context":{"idset":"8407"}}
+{"timestamp":1714157865.4274349,"name":"offline","context":{"idset":"8354"}}
+{"timestamp":1714157865.4318354,"name":"offline","context":{"idset":"8356"}}
+{"timestamp":1714157865.4363627,"name":"offline","context":{"idset":"8342"}}
+{"timestamp":1714157865.4411039,"name":"offline","context":{"idset":"8409"}}
+{"timestamp":1714157865.4458241,"name":"offline","context":{"idset":"8330"}}
+{"timestamp":1714157865.4503412,"name":"offline","context":{"idset":"8364"}}
+{"timestamp":1714157865.4551082,"name":"offline","context":{"idset":"8332"}}
+{"timestamp":1714157865.4595056,"name":"offline","context":{"idset":"8420"}}
+{"timestamp":1714157865.4637556,"name":"offline","context":{"idset":"8340"}}
+{"timestamp":1714157865.4684741,"name":"offline","context":{"idset":"8411"}}
+{"timestamp":1714157865.4731228,"name":"offline","context":{"idset":"8406"}}
+{"timestamp":1714157865.477941,"name":"offline","context":{"idset":"8398"}}
+{"timestamp":1714157865.4827702,"name":"offline","context":{"idset":"8392"}}
+{"timestamp":1714157865.4875619,"name":"offline","context":{"idset":"8369"}}
+{"timestamp":1714157865.4920287,"name":"offline","context":{"idset":"8372"}}
+{"timestamp":1714157865.4968293,"name":"offline","context":{"idset":"8391"}}
+{"timestamp":1714157865.5016489,"name":"offline","context":{"idset":"8395"}}
+{"timestamp":1714157865.5062943,"name":"offline","context":{"idset":"8359"}}
+{"timestamp":1714157865.5111239,"name":"offline","context":{"idset":"8376"}}
+{"timestamp":1714157865.5157166,"name":"offline","context":{"idset":"8405"}}
+{"timestamp":1714157865.5204117,"name":"offline","context":{"idset":"8417"}}
+{"timestamp":1714157865.5251706,"name":"offline","context":{"idset":"8383"}}
+{"timestamp":1714157865.5299852,"name":"offline","context":{"idset":"8384"}}
+{"timestamp":1714157865.5346932,"name":"offline","context":{"idset":"8385"}}
+{"timestamp":1714157865.5394368,"name":"offline","context":{"idset":"8414"}}
+{"timestamp":1714157865.5442631,"name":"offline","context":{"idset":"8373"}}
+{"timestamp":1714157865.5489986,"name":"offline","context":{"idset":"8390"}}
+{"timestamp":1714157865.5537431,"name":"offline","context":{"idset":"8388"}}
+{"timestamp":1714157865.5583005,"name":"offline","context":{"idset":"8377"}}
+{"timestamp":1714157865.5611014,"name":"offline","context":{"idset":"8393"}}
+{"timestamp":1714157865.5636857,"name":"offline","context":{"idset":"8321"}}
+{"timestamp":1714157865.5664055,"name":"offline","context":{"idset":"8357"}}
+{"timestamp":1714157865.5694938,"name":"offline","context":{"idset":"8338"}}
+{"timestamp":1714157865.5729082,"name":"offline","context":{"idset":"8358"}}
+{"timestamp":1714157865.5757186,"name":"offline","context":{"idset":"8419"}}
+{"timestamp":1714157865.5788565,"name":"offline","context":{"idset":"8365"}}
+{"timestamp":1714157865.5815759,"name":"offline","context":{"idset":"8382"}}
+{"timestamp":1714157865.5841563,"name":"offline","context":{"idset":"8331"}}
+{"timestamp":1714157865.5870707,"name":"offline","context":{"idset":"8412"}}
+{"timestamp":1714157865.5895953,"name":"offline","context":{"idset":"8368"}}
+{"timestamp":1714157865.5922146,"name":"offline","context":{"idset":"8418"}}
+{"timestamp":1714157865.5947838,"name":"offline","context":{"idset":"8410"}}
+{"timestamp":1714157865.59781,"name":"offline","context":{"idset":"8379"}}
+{"timestamp":1714157865.6004071,"name":"offline","context":{"idset":"8408"}}
+{"timestamp":1714157865.6029248,"name":"offline","context":{"idset":"8416"}}
+{"timestamp":1714157865.6062648,"name":"offline","context":{"idset":"8397"}}
+{"timestamp":1714157865.6109421,"name":"offline","context":{"idset":"8317"}}
+{"timestamp":1714157865.6158829,"name":"offline","context":{"idset":"8402"}}
+{"timestamp":1714157865.618855,"name":"offline","context":{"idset":"8396"}}
+{"timestamp":1714157865.6213748,"name":"offline","context":{"idset":"8319"}}
+{"timestamp":1714157865.624197,"name":"offline","context":{"idset":"8399"}}
+{"timestamp":1714157865.6267662,"name":"offline","context":{"idset":"8311"}}
+{"timestamp":1714157865.6295369,"name":"offline","context":{"idset":"8432"}}
+{"timestamp":1714157865.6321449,"name":"offline","context":{"idset":"8389"}}
+{"timestamp":1714157865.6348708,"name":"offline","context":{"idset":"8353"}}
+{"timestamp":1714157865.6374235,"name":"offline","context":{"idset":"8371"}}
+{"timestamp":1714157865.6401379,"name":"offline","context":{"idset":"8334"}}
+{"timestamp":1714157865.6429713,"name":"offline","context":{"idset":"8347"}}
+{"timestamp":1714157865.6466155,"name":"offline","context":{"idset":"8352"}}
+{"timestamp":1714157865.8486128,"name":"offline","context":{"idset":"8355"}}
+{"timestamp":1714157866.3528621,"name":"offline","context":{"idset":"8346"}}
+{"timestamp":1714157866.3565421,"name":"offline","context":{"idset":"8344"}}
+{"timestamp":1714157866.360287,"name":"offline","context":{"idset":"8345"}}
+{"timestamp":1714157866.3640106,"name":"offline","context":{"idset":"8324"}}
+{"timestamp":1714157866.3675859,"name":"offline","context":{"idset":"8349"}}
+{"timestamp":1714157866.3713207,"name":"offline","context":{"idset":"8351"}}
+{"timestamp":1714157869.6065567,"name":"offline","context":{"idset":"8603"}}
+{"timestamp":1714157869.6143863,"name":"offline","context":{"idset":"8605"}}
+{"timestamp":1714157869.7028298,"name":"offline","context":{"idset":"8610"}}
+{"timestamp":1714157869.8247511,"name":"offline","context":{"idset":"8608"}}
+{"timestamp":1714157869.9129834,"name":"offline","context":{"idset":"8597"}}
+{"timestamp":1714157869.9185426,"name":"offline","context":{"idset":"8604"}}
+{"timestamp":1714157869.9221132,"name":"offline","context":{"idset":"8607"}}
+{"timestamp":1714157870.017236,"name":"offline","context":{"idset":"8606"}}
+{"timestamp":1714157870.1115704,"name":"offline","context":{"idset":"8602"}}
+{"timestamp":1714157870.7963305,"name":"offline","context":{"idset":"8601"}}
+{"timestamp":1714157870.7988799,"name":"offline","context":{"idset":"8598"}}
+{"timestamp":1714157871.4261646,"name":"offline","context":{"idset":"8611"}}
+{"timestamp":1714157871.4286227,"name":"offline","context":{"idset":"8599"}}
+{"timestamp":1714157871.4310651,"name":"offline","context":{"idset":"8600"}}
+{"timestamp":1714157871.4338238,"name":"offline","context":{"idset":"8612"}}
+{"timestamp":1714157871.4366844,"name":"offline","context":{"idset":"8609"}}
+{"timestamp":1714158800.1218503,"name":"online","context":{"idset":"11320"}}
+{"timestamp":1714158801.7193997,"name":"online","context":{"idset":"11365"}}
+{"timestamp":1714158801.722734,"name":"online","context":{"idset":"11323"}}
+{"timestamp":1714158801.8951993,"name":"online","context":{"idset":"11350"}}
+{"timestamp":1714158803.7307653,"name":"online","context":{"idset":"11328,11373,11375"}}
+{"timestamp":1714158803.7337456,"name":"online","context":{"idset":"11326,11357,11370,11377"}}
+{"timestamp":1714158803.7371457,"name":"online","context":{"idset":"11331,11363"}}
+{"timestamp":1714158803.7404928,"name":"online","context":{"idset":"11317"}}
+{"timestamp":1714158803.7437823,"name":"online","context":{"idset":"11319,11322"}}
+{"timestamp":1714158803.7470686,"name":"online","context":{"idset":"11349,11359,11380"}}
+{"timestamp":1714158803.7507675,"name":"online","context":{"idset":"11332,11369,11372,11379"}}
+{"timestamp":1714158803.7543397,"name":"online","context":{"idset":"11355,11376"}}
+{"timestamp":1714158803.7578437,"name":"online","context":{"idset":"11321,11324,11361,11367"}}
+{"timestamp":1714158803.7611182,"name":"online","context":{"idset":"11318,11330,11351,11353,11368"}}
+{"timestamp":1714158803.8532832,"name":"online","context":{"idset":"11354"}}
+{"timestamp":1714158804.0376403,"name":"online","context":{"idset":"11325,11327,11329,11352,11360,11364"}}
+{"timestamp":1714158804.7612765,"name":"online","context":{"idset":"11362,11371,11378"}}
+{"timestamp":1714158804.7643361,"name":"online","context":{"idset":"11356,11358,11366,11374"}}
+{"timestamp":1714158807.2648284,"name":"online","context":{"idset":"11256"}}
+{"timestamp":1714158809.5370755,"name":"online","context":{"idset":"11276"}}
+{"timestamp":1714158812.0887501,"name":"online","context":{"idset":"11343"}}
+{"timestamp":1714158813.7052953,"name":"online","context":{"idset":"11309"}}
+{"timestamp":1714158813.9403312,"name":"online","context":{"idset":"11278"}}
+{"timestamp":1714158816.4586132,"name":"online","context":{"idset":"11261"}}
+{"timestamp":1714158816.4618156,"name":"online","context":{"idset":"11312"}}
+{"timestamp":1714158818.2106931,"name":"online","context":{"idset":"11265"}}
+{"timestamp":1714158818.2145169,"name":"online","context":{"idset":"11269"}}
+{"timestamp":1714158818.2186694,"name":"online","context":{"idset":"11304"}}
+{"timestamp":1714158819.8244476,"name":"online","context":{"idset":"11255"}}
+{"timestamp":1714158819.8264606,"name":"online","context":{"idset":"11281"}}
+{"timestamp":1714158819.8288217,"name":"online","context":{"idset":"11253"}}
+{"timestamp":1714158819.8307278,"name":"online","context":{"idset":"11315"}}
+{"timestamp":1714158821.6211483,"name":"online","context":{"idset":"11342"}}
+{"timestamp":1714158821.8233247,"name":"online","context":{"idset":"11258"}}
+{"timestamp":1714158822.019182,"name":"online","context":{"idset":"11279"}}
+{"timestamp":1714158822.8586786,"name":"online","context":{"idset":"11280"}}
+{"timestamp":1714158822.8621662,"name":"online","context":{"idset":"11334"}}
+{"timestamp":1714158824.1048238,"name":"online","context":{"idset":"11262,11340"}}
+{"timestamp":1714158824.1073112,"name":"online","context":{"idset":"11260"}}
+{"timestamp":1714158824.1094747,"name":"online","context":{"idset":"11335"}}
+{"timestamp":1714158824.1114771,"name":"online","context":{"idset":"11314"}}
+{"timestamp":1714158824.1135683,"name":"online","context":{"idset":"11257,11301"}}
+{"timestamp":1714158824.6441126,"name":"online","context":{"idset":"11348"}}
+{"timestamp":1714158825.3863904,"name":"online","context":{"idset":"11282,11337"}}
+{"timestamp":1714158825.4058852,"name":"online","context":{"idset":"11264,11284"}}
+{"timestamp":1714158825.410476,"name":"online","context":{"idset":"11254,11272,11274,11307"}}
+{"timestamp":1714158825.4142771,"name":"online","context":{"idset":"11263,11277"}}
+{"timestamp":1714158825.4176545,"name":"online","context":{"idset":"11268,11283"}}
+{"timestamp":1714158825.4207447,"name":"online","context":{"idset":"11259,11333,11338"}}
+{"timestamp":1714158825.6592171,"name":"online","context":{"idset":"11267,11270,11303"}}
+{"timestamp":1714158825.7647748,"name":"online","context":{"idset":"11266,11345"}}
+{"timestamp":1714158825.9545369,"name":"online","context":{"idset":"11305,11310"}}
+{"timestamp":1714158827.264709,"name":"online","context":{"idset":"11271,11273,11275,11306,11316,11341"}}
+{"timestamp":1714158827.8346956,"name":"online","context":{"idset":"11347"}}
+{"timestamp":1714158827.8371601,"name":"online","context":{"idset":"11302,11308,11313,11339,11344"}}
+{"timestamp":1714158827.8406835,"name":"online","context":{"idset":"11311"}}
+{"timestamp":1714158827.8450313,"name":"online","context":{"idset":"11336"}}
+{"timestamp":1714158827.8613746,"name":"online","context":{"idset":"11346"}}
+{"timestamp":1714158908.7869329,"name":"online","context":{"idset":"9323"}}
+{"timestamp":1714158911.426158,"name":"online","context":{"idset":"9239"}}
+{"timestamp":1714159075.5297525,"name":"undrain","context":{"idset":"7733-7738,7740-7748"}}
+{"timestamp":1714159200.6052277,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1714159232.7397358,"name":"online","context":{"idset":"9327"}}
+{"timestamp":1714159394.0394185,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1714159394.0426853,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1714159394.0454662,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1714159394.0495205,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1714159394.0534396,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1714159394.0565331,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1714159394.0588691,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1714159394.0614722,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1714159394.0636771,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1714159394.0672176,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1714159394.8483045,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1714159397.4697249,"name":"online","context":{"idset":"6518"}}
+{"timestamp":1714159397.472611,"name":"online","context":{"idset":"6522"}}
+{"timestamp":1714159397.4754295,"name":"online","context":{"idset":"6527,6529"}}
+{"timestamp":1714159397.4780593,"name":"online","context":{"idset":"6532,6555-6556"}}
+{"timestamp":1714159398.0264173,"name":"online","context":{"idset":"6517,6525"}}
+{"timestamp":1714159398.7391105,"name":"online","context":{"idset":"6521"}}
+{"timestamp":1714159398.782835,"name":"online","context":{"idset":"6542"}}
+{"timestamp":1714159399.9355462,"name":"online","context":{"idset":"6528"}}
+{"timestamp":1714159400.180953,"name":"online","context":{"idset":"6526"}}
+{"timestamp":1714159400.9490197,"name":"online","context":{"idset":"6537-6538"}}
+{"timestamp":1714159400.9599762,"name":"online","context":{"idset":"6530"}}
+{"timestamp":1714159400.9634745,"name":"online","context":{"idset":"6551,6592"}}
+{"timestamp":1714159401.194586,"name":"online","context":{"idset":"6547"}}
+{"timestamp":1714159409.2803781,"name":"online","context":{"idset":"6586"}}
+{"timestamp":1714159409.2828727,"name":"online","context":{"idset":"6519,6534,6563,6568,6571-6572"}}
+{"timestamp":1714159409.2855766,"name":"online","context":{"idset":"6580"}}
+{"timestamp":1714159409.2890968,"name":"online","context":{"idset":"6531"}}
+{"timestamp":1714159409.2925923,"name":"online","context":{"idset":"6642"}}
+{"timestamp":1714159409.2964399,"name":"online","context":{"idset":"6608,6611"}}
+{"timestamp":1714159409.3039954,"name":"online","context":{"idset":"6546,6595,6597,6601,6615,6622,6635"}}
+{"timestamp":1714159410.0849464,"name":"online","context":{"idset":"6520,6533,6558,6560,6574,6599-6600,6609-6610,6612,6624,6629,6644,8309,8314,8319,8327,8344,8361,8364,8370-8371,8373,8375,8395,8411,8432,8435-8436,8565,8573,8582,8584,8603"}}
+{"timestamp":1714159410.0884736,"name":"online","context":{"idset":"6539,8579"}}
+{"timestamp":1714159410.0922284,"name":"online","context":{"idset":"8317,8349,8633"}}
+{"timestamp":1714159410.095197,"name":"online","context":{"idset":"8325,8348"}}
+{"timestamp":1714159410.932858,"name":"online","context":{"idset":"8383"}}
+{"timestamp":1714159411.5683918,"name":"online","context":{"idset":"6544,6549,6566,6575,6582,6606,6625,6631,6638,8313,8334,8352,8376,8404,8412,8576,8587,8592,8615,8628,8647,8658"}}
+{"timestamp":1714159411.5751708,"name":"online","context":{"idset":"6607,8350,8619,8621-8622"}}
+{"timestamp":1714159411.6773548,"name":"online","context":{"idset":"8632"}}
+{"timestamp":1714159411.8017745,"name":"online","context":{"idset":"6603,8347,8358,8419,8663,8685,8688"}}
+{"timestamp":1714159412.0268228,"name":"online","context":{"idset":"6590,6641,8641,8651"}}
+{"timestamp":1714159416.4616981,"name":"online","context":{"idset":"8425,8431,8578,8636"}}
+{"timestamp":1714159416.4657717,"name":"online","context":{"idset":"6523,8664"}}
+{"timestamp":1714159416.4698381,"name":"online","context":{"idset":"8672"}}
+{"timestamp":1714159416.4738085,"name":"online","context":{"idset":"6627"}}
+{"timestamp":1714159416.477983,"name":"online","context":{"idset":"6618"}}
+{"timestamp":1714159416.6798577,"name":"online","context":{"idset":"6579"}}
+{"timestamp":1714159420.4896693,"name":"online","context":{"idset":"6535,6550,6553,6581,6589,6596,6598,8321,8377,8388,8396,8416,8575,8608,8617"}}
+{"timestamp":1714159420.4939101,"name":"online","context":{"idset":"8362,8405,8642,8690"}}
+{"timestamp":1714159420.4984436,"name":"online","context":{"idset":"8323,8593"}}
+{"timestamp":1714159420.5026023,"name":"online","context":{"idset":"8408"}}
+{"timestamp":1714159420.5066211,"name":"online","context":{"idset":"8689"}}
+{"timestamp":1714159420.5112112,"name":"online","context":{"idset":"8613"}}
+{"timestamp":1714159420.5167768,"name":"online","context":{"idset":"6634"}}
+{"timestamp":1714159420.522831,"name":"online","context":{"idset":"6570,8329,8415,8692"}}
+{"timestamp":1714159420.5268629,"name":"online","context":{"idset":"8429"}}
+{"timestamp":1714159420.53056,"name":"online","context":{"idset":"8354"}}
+{"timestamp":1714159420.534687,"name":"online","context":{"idset":"6552,6617,8343,8571"}}
+{"timestamp":1714159420.7767444,"name":"online","context":{"idset":"6536,6540,6554,6561,6567,6602,6628,8387,8428,8670"}}
+{"timestamp":1714159421.8716137,"name":"online","context":{"idset":"6569,8653"}}
+{"timestamp":1714159421.8780615,"name":"online","context":{"idset":"6621"}}
+{"timestamp":1714159421.9221833,"name":"online","context":{"idset":"6588,8394,8654"}}
+{"timestamp":1714159421.932883,"name":"online","context":{"idset":"6640,8572"}}
+{"timestamp":1714159421.9443719,"name":"online","context":{"idset":"6637,8380,8398,8611"}}
+{"timestamp":1714159421.9587862,"name":"online","context":{"idset":"6604,8345,8655"}}
+{"timestamp":1714159421.9736094,"name":"online","context":{"idset":"8410,8612"}}
+{"timestamp":1714159422.0565608,"name":"online","context":{"idset":"8652"}}
+{"timestamp":1714159423.8148966,"name":"online","context":{"idset":"8379,8581"}}
+{"timestamp":1714159423.8607886,"name":"online","context":{"idset":"6559,6626,6639,8346"}}
+{"timestamp":1714159423.8839676,"name":"online","context":{"idset":"6564,8630"}}
+{"timestamp":1714159423.9109852,"name":"online","context":{"idset":"6562,8391"}}
+{"timestamp":1714159423.9355452,"name":"online","context":{"idset":"8340"}}
+{"timestamp":1714159425.3788834,"name":"online","context":{"idset":"8324"}}
+{"timestamp":1714159426.9945366,"name":"online","context":{"idset":"8588"}}
+{"timestamp":1714159426.9994311,"name":"online","context":{"idset":"6541,8366,8368"}}
+{"timestamp":1714159427.0039346,"name":"online","context":{"idset":"8656"}}
+{"timestamp":1714159427.0090046,"name":"online","context":{"idset":"8580"}}
+{"timestamp":1714159427.019357,"name":"online","context":{"idset":"8357"}}
+{"timestamp":1714159427.0382588,"name":"online","context":{"idset":"6573,8318,8583,8598,8661"}}
+{"timestamp":1714159427.0500247,"name":"online","context":{"idset":"8335"}}
+{"timestamp":1714159427.0548458,"name":"online","context":{"idset":"8356"}}
+{"timestamp":1714159427.0613718,"name":"online","context":{"idset":"8568"}}
+{"timestamp":1714159427.0674717,"name":"online","context":{"idset":"6576,8400"}}
+{"timestamp":1714159428.4419103,"name":"online","context":{"idset":"6614,8595,8623"}}
+{"timestamp":1714159428.445914,"name":"online","context":{"idset":"8384,8602"}}
+{"timestamp":1714159428.4506764,"name":"online","context":{"idset":"6619,8341,8631"}}
+{"timestamp":1714159428.469564,"name":"online","context":{"idset":"8320,8609,8679"}}
+{"timestamp":1714159429.7574005,"name":"online","context":{"idset":"6557,6577,6585,8385,8413,8420,8424,8662,8686"}}
+{"timestamp":1714159429.7611654,"name":"online","context":{"idset":"6543,8676"}}
+{"timestamp":1714159429.7639453,"name":"online","context":{"idset":"6578,6616,8339,8355,8433,8596,8645"}}
+{"timestamp":1714159429.767451,"name":"online","context":{"idset":"8330"}}
+{"timestamp":1714159429.7714739,"name":"online","context":{"idset":"8409,8625"}}
+{"timestamp":1714159429.7758529,"name":"online","context":{"idset":"8382,8407,8605,8691"}}
+{"timestamp":1714159429.9641662,"name":"online","context":{"idset":"8365,8430,8597"}}
+{"timestamp":1714159430.1535532,"name":"online","context":{"idset":"6613,8434"}}
+{"timestamp":1714159431.1400535,"name":"online","context":{"idset":"8414"}}
+{"timestamp":1714159431.1443858,"name":"online","context":{"idset":"6548,6591,8316,8393,8402"}}
+{"timestamp":1714159431.1486399,"name":"online","context":{"idset":"6636,8338,8627"}}
+{"timestamp":1714159431.1522424,"name":"online","context":{"idset":"6633"}}
+{"timestamp":1714159431.167052,"name":"online","context":{"idset":"8336"}}
+{"timestamp":1714159431.1750765,"name":"online","context":{"idset":"8326"}}
+{"timestamp":1714159431.1794887,"name":"online","context":{"idset":"8599"}}
+{"timestamp":1714159432.0812671,"name":"online","context":{"idset":"8315,8675"}}
+{"timestamp":1714159432.0849397,"name":"online","context":{"idset":"8422,8606,8669"}}
+{"timestamp":1714159432.0881968,"name":"online","context":{"idset":"8418,8567,8600"}}
+{"timestamp":1714159432.2299781,"name":"online","context":{"idset":"6594,8333,8390,8569-8570,8590-8591"}}
+{"timestamp":1714159433.1288354,"name":"online","context":{"idset":"6524"}}
+{"timestamp":1714159433.1331043,"name":"online","context":{"idset":"6623,8351,8594,8649,8683"}}
+{"timestamp":1714159433.13639,"name":"online","context":{"idset":"6565,8646"}}
+{"timestamp":1714159433.1402061,"name":"online","context":{"idset":"6605,8607"}}
+{"timestamp":1714159433.1437194,"name":"online","context":{"idset":"8614,8626,8640,8644"}}
+{"timestamp":1714159433.146898,"name":"online","context":{"idset":"8585,8589,8604"}}
+{"timestamp":1714159433.1500962,"name":"online","context":{"idset":"6620,8657"}}
+{"timestamp":1714159433.153327,"name":"online","context":{"idset":"8665"}}
+{"timestamp":1714159433.1572344,"name":"online","context":{"idset":"8331,8342,8427,8574,8577"}}
+{"timestamp":1714159433.3382969,"name":"online","context":{"idset":"8372"}}
+{"timestamp":1714159433.4697354,"name":"online","context":{"idset":"8682"}}
+{"timestamp":1714159433.6580009,"name":"online","context":{"idset":"6630,8360,8374,8378,8673"}}
+{"timestamp":1714159433.852349,"name":"online","context":{"idset":"6583,6643,8629,8635,8674"}}
+{"timestamp":1714159433.9975429,"name":"online","context":{"idset":"8353,8601,8610"}}
+{"timestamp":1714159434.9227083,"name":"online","context":{"idset":"8678"}}
+{"timestamp":1714159435.6087623,"name":"online","context":{"idset":"8618"}}
+{"timestamp":1714159435.6127141,"name":"online","context":{"idset":"8406"}}
+{"timestamp":1714159435.6171634,"name":"online","context":{"idset":"8312,8381,8399"}}
+{"timestamp":1714159435.621347,"name":"online","context":{"idset":"8328,8397"}}
+{"timestamp":1714159435.6256061,"name":"online","context":{"idset":"8311,8423"}}
+{"timestamp":1714159435.6411154,"name":"online","context":{"idset":"8386,8637"}}
+{"timestamp":1714159435.6460893,"name":"online","context":{"idset":"8401,8403,8650"}}
+{"timestamp":1714159435.650528,"name":"online","context":{"idset":"8417"}}
+{"timestamp":1714159435.6546497,"name":"online","context":{"idset":"8363,8638"}}
+{"timestamp":1714159435.9096615,"name":"online","context":{"idset":"6545,6587,8322,8337,8369,8426,8586,8616,8634,8680"}}
+{"timestamp":1714159437.325917,"name":"online","context":{"idset":"6593,8620,8643"}}
+{"timestamp":1714159437.3282275,"name":"online","context":{"idset":"8359,8624,8659-8660,8671"}}
+{"timestamp":1714159437.3305655,"name":"online","context":{"idset":"8367,8648,8668,8681,8684,8687"}}
+{"timestamp":1714159437.3328938,"name":"online","context":{"idset":"8639"}}
+{"timestamp":1714159437.335336,"name":"online","context":{"idset":"6584,8566"}}
+{"timestamp":1714159437.3376162,"name":"online","context":{"idset":"8332,8677"}}
+{"timestamp":1714159437.3398912,"name":"online","context":{"idset":"6632,8392,8666"}}
+{"timestamp":1714159437.3421404,"name":"online","context":{"idset":"8310,8389,8421"}}
+{"timestamp":1714159437.5844707,"name":"online","context":{"idset":"8667"}}
+{"timestamp":1714159462.1616287,"name":"drain","context":{"idset":"8309-8436","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714159682.9494765,"name":"offline","context":{"idset":"7993"}}
+{"timestamp":1714159702.8434656,"name":"offline","context":{"idset":"7929"}}
+{"timestamp":1714159742.29124,"name":"offline","context":{"idset":"8041"}}
+{"timestamp":1714160735.9431796,"name":"drain","context":{"idset":"7739","reason":"epilog failed for jobid frexhog1vym","overwrite":0}}
+{"timestamp":1714160735.9455199,"name":"offline","context":{"idset":"7739"}}
+{"timestamp":1714160754.9842021,"name":"undrain","context":{"idset":"11253-11268"}}
+{"timestamp":1714160822.0020144,"name":"undrain","context":{"idset":"11285-11300"}}
+{"timestamp":1714160850.4342723,"name":"online","context":{"idset":"9179"}}
+{"timestamp":1714160850.4367969,"name":"online","context":{"idset":"9180"}}
+{"timestamp":1714161094.9616261,"name":"online","context":{"idset":"7993"}}
+{"timestamp":1714161094.9638739,"name":"online","context":{"idset":"7929"}}
+{"timestamp":1714161416.896282,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1714162337.737052,"name":"online","context":{"idset":"8041"}}
+{"timestamp":1714162485.1635149,"name":"online","context":{"idset":"8144"}}
+{"timestamp":1714162688.1321509,"name":"offline","context":{"idset":"9139"}}
+{"timestamp":1714163599.8335512,"name":"online","context":{"idset":"1190"}}
+{"timestamp":1714163599.837184,"name":"online","context":{"idset":"1189"}}
+{"timestamp":1714163604.8587525,"name":"online","context":{"idset":"1187"}}
+{"timestamp":1714163608.0620484,"name":"online","context":{"idset":"1181"}}
+{"timestamp":1714163609.2488692,"name":"online","context":{"idset":"1173"}}
+{"timestamp":1714163609.252162,"name":"online","context":{"idset":"1180"}}
+{"timestamp":1714163609.2553596,"name":"online","context":{"idset":"1186,1207"}}
+{"timestamp":1714163609.2585156,"name":"online","context":{"idset":"1217"}}
+{"timestamp":1714163609.778707,"name":"online","context":{"idset":"1174"}}
+{"timestamp":1714163609.905009,"name":"online","context":{"idset":"1216"}}
+{"timestamp":1714163612.3446958,"name":"online","context":{"idset":"1182,1220"}}
+{"timestamp":1714163612.3496029,"name":"online","context":{"idset":"1178,1184"}}
+{"timestamp":1714163612.355139,"name":"online","context":{"idset":"1221"}}
+{"timestamp":1714163612.3592799,"name":"online","context":{"idset":"1176,1215"}}
+{"timestamp":1714163612.3628645,"name":"online","context":{"idset":"1179"}}
+{"timestamp":1714163612.3664758,"name":"online","context":{"idset":"1177,1188,1219,1224"}}
+{"timestamp":1714163612.3700533,"name":"online","context":{"idset":"1183,1210"}}
+{"timestamp":1714163612.3737106,"name":"online","context":{"idset":"1185,1225-1226"}}
+{"timestamp":1714163612.3774085,"name":"online","context":{"idset":"1175"}}
+{"timestamp":1714163612.3812091,"name":"online","context":{"idset":"1231"}}
+{"timestamp":1714163612.3849487,"name":"online","context":{"idset":"1205-1206,1212,1233"}}
+{"timestamp":1714163612.3885925,"name":"online","context":{"idset":"1211,1213"}}
+{"timestamp":1714163612.3922281,"name":"online","context":{"idset":"1209,1227,1232,1235-1236"}}
+{"timestamp":1714163612.3959463,"name":"online","context":{"idset":"1222,1229"}}
+{"timestamp":1714163614.2487562,"name":"online","context":{"idset":"1218"}}
+{"timestamp":1714163614.2520776,"name":"online","context":{"idset":"1214,1223,1228,1230,1234"}}
+{"timestamp":1714163614.2553542,"name":"online","context":{"idset":"1208"}}
+{"timestamp":1714163748.4043591,"name":"online","context":{"idset":"9365"}}
+{"timestamp":1714163748.4076552,"name":"online","context":{"idset":"9359"}}
+{"timestamp":1714163761.2289257,"name":"online","context":{"idset":"9345"}}
+{"timestamp":1714163761.2318389,"name":"online","context":{"idset":"9357"}}
+{"timestamp":1714163761.2346215,"name":"online","context":{"idset":"9377"}}
+{"timestamp":1714163761.2375159,"name":"online","context":{"idset":"9397"}}
+{"timestamp":1714163761.2419887,"name":"online","context":{"idset":"9361,9451"}}
+{"timestamp":1714163761.2463963,"name":"online","context":{"idset":"9456"}}
+{"timestamp":1714163761.9483407,"name":"online","context":{"idset":"9417"}}
+{"timestamp":1714163763.5752878,"name":"online","context":{"idset":"9445"}}
+{"timestamp":1714163763.5810213,"name":"online","context":{"idset":"9381"}}
+{"timestamp":1714163763.5852189,"name":"online","context":{"idset":"9434"}}
+{"timestamp":1714163764.5471957,"name":"online","context":{"idset":"9426"}}
+{"timestamp":1714163766.1027422,"name":"online","context":{"idset":"9339,9383"}}
+{"timestamp":1714163766.1057165,"name":"online","context":{"idset":"9425"}}
+{"timestamp":1714163766.1085079,"name":"online","context":{"idset":"9346,9366"}}
+{"timestamp":1714163766.1111925,"name":"online","context":{"idset":"9354"}}
+{"timestamp":1714163767.8106592,"name":"online","context":{"idset":"9418"}}
+{"timestamp":1714163767.81462,"name":"online","context":{"idset":"9358"}}
+{"timestamp":1714163769.6032689,"name":"online","context":{"idset":"9334"}}
+{"timestamp":1714163769.6077678,"name":"online","context":{"idset":"9427"}}
+{"timestamp":1714163770.3420587,"name":"online","context":{"idset":"9393"}}
+{"timestamp":1714163771.8628991,"name":"online","context":{"idset":"9336"}}
+{"timestamp":1714163771.8677173,"name":"online","context":{"idset":"9430"}}
+{"timestamp":1714163775.3900514,"name":"online","context":{"idset":"9378"}}
+{"timestamp":1714163775.3937218,"name":"online","context":{"idset":"9435"}}
+{"timestamp":1714163775.3975997,"name":"online","context":{"idset":"9371"}}
+{"timestamp":1714163775.4014714,"name":"online","context":{"idset":"9369,9394"}}
+{"timestamp":1714163775.4387832,"name":"online","context":{"idset":"9376,9399"}}
+{"timestamp":1714163777.6980269,"name":"online","context":{"idset":"9388"}}
+{"timestamp":1714163777.7046609,"name":"online","context":{"idset":"9449"}}
+{"timestamp":1714163777.7125542,"name":"online","context":{"idset":"9382,9422,9447,9457,9460"}}
+{"timestamp":1714163777.724339,"name":"online","context":{"idset":"9459"}}
+{"timestamp":1714163777.7437036,"name":"online","context":{"idset":"9380,9441"}}
+{"timestamp":1714163777.7692428,"name":"online","context":{"idset":"9453,9455"}}
+{"timestamp":1714163777.7875257,"name":"online","context":{"idset":"9410"}}
+{"timestamp":1714163778.7848806,"name":"online","context":{"idset":"9395-9396,9412"}}
+{"timestamp":1714163778.7893569,"name":"online","context":{"idset":"9442"}}
+{"timestamp":1714163778.849056,"name":"online","context":{"idset":"9391"}}
+{"timestamp":1714163780.2807846,"name":"online","context":{"idset":"9353,9389,9446"}}
+{"timestamp":1714163781.0832405,"name":"online","context":{"idset":"9385,9387,9420,9423,9432,9436-9437,9454"}}
+{"timestamp":1714163781.0870917,"name":"online","context":{"idset":"9351,9415"}}
+{"timestamp":1714163781.0907831,"name":"online","context":{"idset":"9349,9355,9363,9373,9414,9421,9429"}}
+{"timestamp":1714163781.0942533,"name":"online","context":{"idset":"9413,9448"}}
+{"timestamp":1714163781.097934,"name":"online","context":{"idset":"9424"}}
+{"timestamp":1714163781.1098545,"name":"online","context":{"idset":"9419,9443"}}
+{"timestamp":1714163781.1135757,"name":"online","context":{"idset":"9439"}}
+{"timestamp":1714163781.1176152,"name":"online","context":{"idset":"9379"}}
+{"timestamp":1714163782.0485339,"name":"online","context":{"idset":"9348,9360,9375,9390,9416,9428,9450,9452"}}
+{"timestamp":1714163782.053081,"name":"online","context":{"idset":"9356,9367,9392"}}
+{"timestamp":1714163782.0574119,"name":"online","context":{"idset":"9384,9386,9406,9458"}}
+{"timestamp":1714163782.0619929,"name":"online","context":{"idset":"9338"}}
+{"timestamp":1714163782.0663798,"name":"online","context":{"idset":"9402"}}
+{"timestamp":1714163782.0705199,"name":"online","context":{"idset":"9343"}}
+{"timestamp":1714163783.8876002,"name":"online","context":{"idset":"9333,9340,9344,9405"}}
+{"timestamp":1714163783.8955202,"name":"online","context":{"idset":"9342,9368"}}
+{"timestamp":1714163783.9000552,"name":"online","context":{"idset":"9350"}}
+{"timestamp":1714163783.9041712,"name":"online","context":{"idset":"9398"}}
+{"timestamp":1714163783.9184179,"name":"online","context":{"idset":"9335,9372,9374"}}
+{"timestamp":1714163783.9222605,"name":"online","context":{"idset":"9370"}}
+{"timestamp":1714163783.9264538,"name":"online","context":{"idset":"9341,9347,9411"}}
+{"timestamp":1714163783.9305921,"name":"online","context":{"idset":"9364"}}
+{"timestamp":1714163783.9458542,"name":"online","context":{"idset":"9337"}}
+{"timestamp":1714163785.0064521,"name":"online","context":{"idset":"9352,9400"}}
+{"timestamp":1714163785.0110555,"name":"online","context":{"idset":"9362,9403,9408"}}
+{"timestamp":1714163785.0154097,"name":"online","context":{"idset":"9401,9404"}}
+{"timestamp":1714163786.9340236,"name":"online","context":{"idset":"9409"}}
+{"timestamp":1714163786.9378941,"name":"online","context":{"idset":"9407"}}
+{"timestamp":1714163786.941932,"name":"online","context":{"idset":"9438"}}
+{"timestamp":1714163787.0074255,"name":"online","context":{"idset":"9433"}}
+{"timestamp":1714163788.7041111,"name":"online","context":{"idset":"9440,9444"}}
+{"timestamp":1714163788.7082548,"name":"online","context":{"idset":"9431"}}
+{"timestamp":1714163809.8132801,"name":"online","context":{"idset":"11672"}}
+{"timestamp":1714163809.915663,"name":"online","context":{"idset":"11668"}}
+{"timestamp":1714163810.6049099,"name":"online","context":{"idset":"11688,11711"}}
+{"timestamp":1714163811.1056368,"name":"online","context":{"idset":"11650"}}
+{"timestamp":1714163811.1088519,"name":"online","context":{"idset":"11687"}}
+{"timestamp":1714163811.1112301,"name":"online","context":{"idset":"11667"}}
+{"timestamp":1714163811.113811,"name":"online","context":{"idset":"11656,11698"}}
+{"timestamp":1714163811.1164751,"name":"online","context":{"idset":"11643"}}
+{"timestamp":1714163811.1259046,"name":"online","context":{"idset":"11653,11671"}}
+{"timestamp":1714163811.6758802,"name":"online","context":{"idset":"11638,11684,11695"}}
+{"timestamp":1714163811.679384,"name":"online","context":{"idset":"11689"}}
+{"timestamp":1714163811.6830201,"name":"online","context":{"idset":"11651"}}
+{"timestamp":1714163811.68641,"name":"online","context":{"idset":"11648,11678"}}
+{"timestamp":1714163811.6910267,"name":"online","context":{"idset":"11645,11655,11663,11709,11744"}}
+{"timestamp":1714163811.7377007,"name":"online","context":{"idset":"11642"}}
+{"timestamp":1714163811.740644,"name":"online","context":{"idset":"11657,11676,11696,11712-11713,11743"}}
+{"timestamp":1714163811.7433569,"name":"online","context":{"idset":"11646,11649,11664,11682,11745"}}
+{"timestamp":1714163811.7462645,"name":"online","context":{"idset":"11637,11641,11644,11647,11659,11670,11673,11679,11681,11693,11706,11710,11748"}}
+{"timestamp":1714163811.9690881,"name":"online","context":{"idset":"11652,11683,11694,11699,11702,11733,11735,11749,11759"}}
+{"timestamp":1714163813.0151467,"name":"online","context":{"idset":"11639,11666,11669,11674-11675,11680,11690,11697,11707,11734,11755,11760"}}
+{"timestamp":1714163813.0176883,"name":"online","context":{"idset":"11661-11662,11665,11700,11708,11746"}}
+{"timestamp":1714163813.5397015,"name":"online","context":{"idset":"11640,11654,11677,11716,11738,11740"}}
+{"timestamp":1714163813.5425515,"name":"online","context":{"idset":"11701,11715,11741,11753-11754"}}
+{"timestamp":1714163813.5451427,"name":"online","context":{"idset":"11704,11736,11739,11750"}}
+{"timestamp":1714163813.560497,"name":"online","context":{"idset":"11703,11756"}}
+{"timestamp":1714163813.5635345,"name":"online","context":{"idset":"11705,11714,11747,11751,11762-11764"}}
+{"timestamp":1714163813.5683146,"name":"online","context":{"idset":"11737,11742,11752,11758,11761"}}
+{"timestamp":1714163813.7403979,"name":"online","context":{"idset":"11757"}}
+{"timestamp":1714163815.7168851,"name":"online","context":{"idset":"11718"}}
+{"timestamp":1714163815.719924,"name":"online","context":{"idset":"11720,11726-11727"}}
+{"timestamp":1714163815.9494781,"name":"online","context":{"idset":"11724"}}
+{"timestamp":1714163817.209775,"name":"online","context":{"idset":"11721,11728,11731-11732"}}
+{"timestamp":1714163817.2122161,"name":"online","context":{"idset":"11722"}}
+{"timestamp":1714163817.2167277,"name":"online","context":{"idset":"11691,11729"}}
+{"timestamp":1714163817.2208612,"name":"online","context":{"idset":"11717"}}
+{"timestamp":1714163817.7132485,"name":"online","context":{"idset":"11685,11719,11723"}}
+{"timestamp":1714163817.7155709,"name":"online","context":{"idset":"11660,11686"}}
+{"timestamp":1714163817.7179139,"name":"online","context":{"idset":"11692,11730"}}
+{"timestamp":1714163817.7492158,"name":"online","context":{"idset":"11725"}}
+{"timestamp":1714163830.1935024,"name":"offline","context":{"idset":"7977"}}
+{"timestamp":1714163830.1960366,"name":"offline","context":{"idset":"8006"}}
+{"timestamp":1714163830.196089,"name":"drain","context":{"idset":"7977,8006,8029","reason":"epilog failed for jobid fro4S7tLZps","overwrite":0}}
+{"timestamp":1714163830.1996877,"name":"offline","context":{"idset":"8029"}}
+{"timestamp":1714164002.7398465,"name":"undrain","context":{"idset":"11658,11717-11726,11728-11732"}}
+{"timestamp":1714164596.8904066,"name":"online","context":{"idset":"826"}}
+{"timestamp":1714164616.0307326,"name":"online","context":{"idset":"7285,7307"}}
+{"timestamp":1714164616.0336137,"name":"online","context":{"idset":"7315"}}
+{"timestamp":1714164617.3990712,"name":"online","context":{"idset":"7301"}}
+{"timestamp":1714164617.4024458,"name":"online","context":{"idset":"7311,7316"}}
+{"timestamp":1714164617.40593,"name":"online","context":{"idset":"7290,7305,7309,7314"}}
+{"timestamp":1714164617.4115763,"name":"online","context":{"idset":"7288,7343"}}
+{"timestamp":1714164617.7856328,"name":"online","context":{"idset":"7317"}}
+{"timestamp":1714164617.9851213,"name":"online","context":{"idset":"7373"}}
+{"timestamp":1714164620.2645466,"name":"online","context":{"idset":"7303"}}
+{"timestamp":1714164620.2676897,"name":"online","context":{"idset":"7313"}}
+{"timestamp":1714164620.2708902,"name":"online","context":{"idset":"7374"}}
+{"timestamp":1714164621.7183449,"name":"online","context":{"idset":"7329"}}
+{"timestamp":1714164621.7213185,"name":"online","context":{"idset":"7332"}}
+{"timestamp":1714164621.7244503,"name":"online","context":{"idset":"7324"}}
+{"timestamp":1714164621.7275343,"name":"online","context":{"idset":"7310,7356"}}
+{"timestamp":1714164622.7155235,"name":"online","context":{"idset":"7292"}}
+{"timestamp":1714164623.7433643,"name":"online","context":{"idset":"7325,7336"}}
+{"timestamp":1714164623.7459283,"name":"online","context":{"idset":"7308,7321,7348"}}
+{"timestamp":1714164623.7485268,"name":"online","context":{"idset":"7286,7302,7341"}}
+{"timestamp":1714164623.7571192,"name":"online","context":{"idset":"7350"}}
+{"timestamp":1714164623.7599509,"name":"online","context":{"idset":"7320,7322"}}
+{"timestamp":1714164623.7625198,"name":"online","context":{"idset":"7384"}}
+{"timestamp":1714164623.765141,"name":"online","context":{"idset":"7376"}}
+{"timestamp":1714164623.767714,"name":"online","context":{"idset":"7368"}}
+{"timestamp":1714164623.7708507,"name":"online","context":{"idset":"7369"}}
+{"timestamp":1714164623.774529,"name":"online","context":{"idset":"7379"}}
+{"timestamp":1714164623.7780244,"name":"online","context":{"idset":"7294"}}
+{"timestamp":1714164623.7815661,"name":"online","context":{"idset":"7328"}}
+{"timestamp":1714164623.7850983,"name":"online","context":{"idset":"7327,7388"}}
+{"timestamp":1714164623.7887328,"name":"online","context":{"idset":"7323,7365,7401"}}
+{"timestamp":1714164623.7924311,"name":"online","context":{"idset":"7300,7304"}}
+{"timestamp":1714164623.8712745,"name":"online","context":{"idset":"7287"}}
+{"timestamp":1714164624.4985888,"name":"online","context":{"idset":"7295,7298,7306,7351,7362,7380,7392"}}
+{"timestamp":1714164624.9378946,"name":"online","context":{"idset":"7291,7334,7337,7344-7345,7370,7375,7381,7389-7390,7393-7394,7406"}}
+{"timestamp":1714164624.9464962,"name":"online","context":{"idset":"7330,7357,7359,7385,7397"}}
+{"timestamp":1714164624.9494779,"name":"online","context":{"idset":"7289,7319,7335,7338,7349,7363-7364,7391,7402"}}
+{"timestamp":1714164624.9524083,"name":"online","context":{"idset":"7342,7361,7382,7403"}}
+{"timestamp":1714164625.4728317,"name":"online","context":{"idset":"7339,7346,7372,7377,7405"}}
+{"timestamp":1714164625.4754734,"name":"online","context":{"idset":"7296,7312,7318,7347,7352,7360,7371,7386,7395"}}
+{"timestamp":1714164625.4779842,"name":"online","context":{"idset":"7333,7398,7408"}}
+{"timestamp":1714164625.4993477,"name":"online","context":{"idset":"7293,7297,7299,7326,7331,7340,7353,7355,7358,7367,7378,7383,7387,7396,7400,7404,7407,7409,7411"}}
+{"timestamp":1714164625.7338281,"name":"online","context":{"idset":"7354,7366"}}
+{"timestamp":1714164626.0352373,"name":"online","context":{"idset":"7399,7412"}}
+{"timestamp":1714164627.6067238,"name":"online","context":{"idset":"7410"}}
+{"timestamp":1714164636.3354473,"name":"drain","context":{"idset":"7285-7412","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714164649.5880666,"name":"online","context":{"idset":"8006"}}
+{"timestamp":1714164649.8172827,"name":"online","context":{"idset":"7977"}}
+{"timestamp":1714164651.835974,"name":"online","context":{"idset":"8029"}}
+{"timestamp":1714164683.7201822,"name":"online","context":{"idset":"825"}}
+{"timestamp":1714164887.8795125,"name":"undrain","context":{"idset":"8309-8436"}}
+{"timestamp":1714165044.5597908,"name":"undrain","context":{"idset":"7977,8006,8029"}}
+{"timestamp":1714165325.264075,"name":"undrain","context":{"idset":"9333-9380,9429-9460"}}
+{"timestamp":1714165419.2942963,"name":"undrain","context":{"idset":"8599-8612"}}
+{"timestamp":1714165440.4334314,"name":"undrain","context":{"idset":"6565-6574,6577-6580,6613-6628"}}
+{"timestamp":1714166175.9960833,"name":"online","context":{"idset":"871"}}
+{"timestamp":1714166209.232296,"name":"online","context":{"idset":"872"}}
+{"timestamp":1714166231.6703308,"name":"online","context":{"idset":"873"}}
+{"timestamp":1714166273.1717546,"name":"online","context":{"idset":"874"}}
+{"timestamp":1714166775.7284136,"name":"online","context":{"idset":"9501"}}
+{"timestamp":1714166855.7835934,"name":"online","context":{"idset":"11563,11565"}}
+{"timestamp":1714166855.7877688,"name":"online","context":{"idset":"11541,11548,11554,11562"}}
+{"timestamp":1714166855.792285,"name":"online","context":{"idset":"11556,11566,11568"}}
+{"timestamp":1714166855.7967291,"name":"online","context":{"idset":"11555,11564"}}
+{"timestamp":1714167558.1069169,"name":"offline","context":{"idset":"6605"}}
+{"timestamp":1714167560.1093743,"name":"offline","context":{"idset":"6606"}}
+{"timestamp":1714167632.1112521,"name":"offline","context":{"idset":"851"}}
+{"timestamp":1714167634.1064811,"name":"offline","context":{"idset":"852"}}
+{"timestamp":1714167979.0349083,"name":"online","context":{"idset":"7894"}}
+{"timestamp":1714167979.3960354,"name":"online","context":{"idset":"7893"}}
+{"timestamp":1714168052.072629,"name":"offline","context":{"idset":"11541"}}
+{"timestamp":1714168052.0780468,"name":"offline","context":{"idset":"11548"}}
+{"timestamp":1714168052.0839953,"name":"offline","context":{"idset":"11554"}}
+{"timestamp":1714168052.0897546,"name":"offline","context":{"idset":"11555"}}
+{"timestamp":1714168052.0954413,"name":"offline","context":{"idset":"11556"}}
+{"timestamp":1714168052.1006746,"name":"offline","context":{"idset":"11560"}}
+{"timestamp":1714168052.1063917,"name":"offline","context":{"idset":"11561"}}
+{"timestamp":1714168052.1129086,"name":"offline","context":{"idset":"11562"}}
+{"timestamp":1714168052.1184025,"name":"offline","context":{"idset":"11563"}}
+{"timestamp":1714168052.1241431,"name":"offline","context":{"idset":"11565"}}
+{"timestamp":1714168052.1295812,"name":"offline","context":{"idset":"11566"}}
+{"timestamp":1714168052.1467946,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1714168422.1089683,"name":"offline","context":{"idset":"9327"}}
+{"timestamp":1714170032.2496359,"name":"undrain","context":{"idset":"7285-7412"}}
+{"timestamp":1714170961.9149778,"name":"online","context":{"idset":"835"}}
+{"timestamp":1714170962.9122071,"name":"online","context":{"idset":"836"}}
+{"timestamp":1714171221.4997334,"name":"online","context":{"idset":"6605"}}
+{"timestamp":1714171221.7475238,"name":"online","context":{"idset":"6606"}}
+{"timestamp":1714173409.7024369,"name":"online","context":{"idset":"11462"}}
+{"timestamp":1714173449.5667484,"name":"online","context":{"idset":"11461"}}
+{"timestamp":1714173537.7647281,"name":"online","context":{"idset":"11295"}}
+{"timestamp":1714173539.5162239,"name":"online","context":{"idset":"11288"}}
+{"timestamp":1714173539.5202303,"name":"online","context":{"idset":"11289"}}
+{"timestamp":1714173539.5242107,"name":"online","context":{"idset":"11285"}}
+{"timestamp":1714173539.6163533,"name":"online","context":{"idset":"11290"}}
+{"timestamp":1714173539.8488989,"name":"online","context":{"idset":"11286,11298"}}
+{"timestamp":1714173541.8556783,"name":"online","context":{"idset":"11291-11292,11294,11300"}}
+{"timestamp":1714173541.8604641,"name":"online","context":{"idset":"11287,11293,11296,11299"}}
+{"timestamp":1714173541.8651814,"name":"online","context":{"idset":"11297"}}
+{"timestamp":1714173683.8865831,"name":"undrain","context":{"idset":"11324"}}
+{"timestamp":1714173693.9187984,"name":"drain","context":{"idset":"11324","overwrite":0}}
+{"timestamp":1714173704.0831511,"name":"undrain","context":{"idset":"11324"}}
+{"timestamp":1714173783.3855824,"name":"undrain","context":{"idset":"11369-11370,11373-11374,11377"}}
+{"timestamp":1714173992.0434551,"name":"offline","context":{"idset":"9135"}}
+{"timestamp":1714173992.0479021,"name":"offline","context":{"idset":"9140"}}
+{"timestamp":1714173992.1055706,"name":"offline","context":{"idset":"9151"}}
+{"timestamp":1714175993.0726588,"name":"resource-init","context":{"restart":true,"drain":{"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"98,100,102,111-112,114,123-124,161,184,187,190,208,303-304,310,317,319,330":{"timestamp":1714079956.0,"reason":"epilog failed for jobid frcnKxdu6xP"},"543":{"timestamp":1714074108.0,"reason":"nodediag failed cxi"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"796":{"timestamp":1712868054.0,"reason":"broker was unresponsive"},"119":{"timestamp":1714079565.0,"reason":"unreachable"},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"},"795":{"timestamp":1712868053.0,"reason":"broker was unresponsive"},"372":{"timestamp":1713981581.0,"reason":"New Blade Installation --JRG"},"336,491":{"timestamp":1713230524.0,"reason":"Down"},"929-930":{"timestamp":1713552018.0,"reason":""},"773-774":{"timestamp":1713802805.0,"reason":""},"77-84":{"timestamp":1713795613.0,"reason":"New blade installation"},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"10186-10187,10203,10205-10206,10208-10211,10215":{"timestamp":1714152041.0,"reason":"broker was unresponsive"},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"115":{"timestamp":1714079493.0,"reason":"unreachable"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"117":{"timestamp":1713981538.0,"reason":"Unreachable"},"360":{"timestamp":1713981579.0,"reason":"New Blade Installation --JRG"},"411":{"timestamp":1713564989.0,"reason":"Unreachable"},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"807":{"timestamp":1713376017.0,"reason":""},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"811-812":{"timestamp":1713557753.0,"reason":"--reason Running hpe diags - JM"},"779-780":{"timestamp":1713907642.0,"reason":"--reason Troubleshoot BIOS issue"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"881-882":{"timestamp":1714153692.0,"reason":"--reason H/W troubleshoot"},"286":{"timestamp":1713981553.0,"reason":"Unreachable"},"808":{"timestamp":1713455939.0,"reason":""},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"971-974":{"timestamp":1714143211.0,"reason":""},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"110":{"timestamp":1714079561.0,"reason":"unreachable"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"544":{"timestamp":1714069881.0,"reason":"nodediag failed cxi"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"206":{"timestamp":1713564985.0,"reason":"Unreachable"},"349-359,361-371,374-396,398-404":{"timestamp":1714060395.0,"reason":"New Blade Installation --JRG"},"8181,8183,8234,8237,8291,8304":{"timestamp":1713810085.0,"reason":"testing -kk"},"545":{"timestamp":1714069878.0,"reason":"nodediag failed cxi"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"878":{"timestamp":1714078484.0,"reason":"nodediag failed amdapu"},"141-148":{"timestamp":1713545597.0,"reason":"ARP Storm Problem testing"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"9075":{"timestamp":1712864130.0,"reason":"broker was unresponsive"},"542":{"timestamp":1714074109.0,"reason":"nodediag failed cxi"},"573-580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"149-156":{"timestamp":1713545634.0,"reason":"ARP Storm Problem testing"},"165":{"timestamp":1713981434.0,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261863.0,"reason":"--reason swapping blades into x1900 - wendy"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"172":{"timestamp":1713981544.0,"reason":"Unreachable"},"373":{"timestamp":1713981698.0,"reason":"New Blade Installation --JRG"},"541":{"timestamp":1714074106.0,"reason":"nodediag failed cxi"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"787-788":{"timestamp":1713219302.0,"reason":""},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"85-89,91-92":{"timestamp":1714079045.0,"reason":"New Blade installation"},"963-964":{"timestamp":1713196418.0,"reason":"moving blade"},"937-938":{"timestamp":1712866351.0,"reason":""},"925-926":{"timestamp":1713540640.0,"reason":""},"461-476":{"timestamp":1713794797.0,"reason":"New Blade Installation --JRG"},"397":{"timestamp":1713229804.0,"reason":"New Blade Installation --JRG"},"9003":{"timestamp":1712873964.0,"reason":"nodediag failed dgemm_perf"},"421-460":{"timestamp":1713794410.0,"reason":"New Blade Installation --JRG"},"410":{"timestamp":1713378635.0,"reason":"prolog failed for jobid fq5jtQQSQnT"},"120":{"timestamp":1713981548.0,"reason":"Unreachable"},"803-804":{"timestamp":1713479733.0,"reason":"status"},"789-794":{"timestamp":1711461777.0,"reason":""},"191":{"timestamp":1714078824.0,"reason":"unreachable"},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"961-962":{"timestamp":1713197674.0,"reason":""},"482-490,492":{"timestamp":1713915346.0,"reason":"New Blade Installation --JRG"},"342":{"timestamp":1713981566.0,"reason":"Unreachable"},"923-924":{"timestamp":1714064541.0,"reason":"status"},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"546":{"timestamp":1714069873.0,"reason":"nodediag failed cxi pci"},"825-826":{"timestamp":1714153531.0,"reason":"--reason H/W troubleshoot"},"977-978":{"timestamp":1714144603.0,"reason":""},"549-556":{"timestamp":1713553592.0,"reason":"New Blade Installation"},"8833":{"timestamp":1713991869.0,"reason":"nodediag failed amdapu"},"90":{"timestamp":1713564981.0,"reason":"New Blade installation"},"9054":{"timestamp":1712864116.0,"reason":"broker was unresponsive"},"771-772":{"timestamp":1713195979.0,"reason":""},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"877":{"timestamp":1714078512.0,"reason":"nodediag failed amdapu"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"8982":{"timestamp":1713996208.0,"reason":"broker was unresponsive"},"939":{"timestamp":1713963885.0,"reason":"nodediag failed amdapu"},"547":{"timestamp":1713563916.0,"reason":"epilog failed for jobid fqUWmieM3AX"},"420":{"timestamp":1714074112.0,"reason":"epilog failed for jobid frbYvBLD1gT"},"814":{"timestamp":1712783332.0,"reason":""},"871-874":{"timestamp":1714057817.0,"reason":""},"829-830":{"timestamp":1712700945.0,"reason":""},"133-140":{"timestamp":1713545617.0,"reason":"ARP Storm Problem testing"},"525-532":{"timestamp":1713290620.0,"reason":""},"548":{"timestamp":1713563915.0,"reason":"epilog failed for jobid fqUWkS8j8hM"},"7739":{"timestamp":1714156200.0,"reason":"epilog failed for jobid frexhog1vym"},"335":{"timestamp":1713981557.0,"reason":"Unreachable"},"934":{"timestamp":1713967835.0,"reason":"epilog failed for jobid frMYzSv7G6b"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1714175993.0791469,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1714176012.4508593,"name":"online","context":{"idset":"0"}}
+{"timestamp":1714176014.2230008,"name":"online","context":{"idset":"825,6613,8255,8261,8269,8576,10211,10651,10665,10690,10734,10785,11291,11377,11397,11452"}}
+{"timestamp":1714176014.2274287,"name":"online","context":{"idset":"10649,11469,11503"}}
+{"timestamp":1714176014.3292434,"name":"online","context":{"idset":"6518,6521,6526,6533,6535,6587,7941,7956,8021,8210,8212,8228,8241-8242,8246,8265,8291,8570,8572,8582,8596,8633,8646,8650,8660,10215,10335,10683,10691,10700,10718,10736,10815,11288,11331,11375,11441"}}
+{"timestamp":1714176014.4196286,"name":"online","context":{"idset":"808,871-872,977,6517,6523,6525,6528,6532,6542,6546,6550,6554,6558,6563,6582,6592,6622,7893,7935-7936,7940,7949,7958,7964,7990,7993,8003,8013,8016-8017,8020,8022-8023,8028,8050,8185,8191,8193,8199-8200,8205,8217,8220-8221,8223,8225,8227,8231,8236-8237,8243-8244,8253,8256-8257,8272,8277,8565,8571,8573,8591,8594-8595,8613,8637,8641,8657,8659,8661,8675,8677,8685,10186,10205,10210,10231-10232,10234,10236,10242,10256,10272-10275,10282,10284-10285,10292-10294,10296,10301,10307,10312,10319,10321,10331,10340-10341,10343,10349,10351-10352,10356,10412,10618,10621-10622,10624-10625,10627,10631-10633,10641-10643,10645-10646,10648,10650,10653,10656,10659,10669-10670,10674,10676,10681,10687-10689,10697,10701,10703,10707,10711,10713,10724-10726,10729,10732-10733,10737,10739,10761,10764,10766,10769-10772,10777,10787-10788,10791,10793,10798,10800,10803,10807,10822,10828,10831,10834,10842,10847-10848,10851,10863-10866,11286,11289,11293,11296,11299,11317,11332,11365-11366,11388,11403,11405-11406,11412,11415,11427,11430,11433,11440,11446,11461-11462,11468,11474,11480,11482,11492,11506"}}
+{"timestamp":1714176014.4202979,"name":"online","context":{"idset":"10854"}}
+{"timestamp":1714176014.5239165,"name":"online","context":{"idset":"874,6531,6547,6560,6581,6584,6586,6594,6596,6615,6619,6623-6624,7894,7927,7937,7942,7944,7963,7973-7975,7995,7997,8010,8025,8031,8034,8037,8040,8043,8045,8051,8183,8187,8190,8204,8206,8216,8226,8240,8249,8266,8577,8583,8587,8616,8621-8623,8626,8631,8640,8642-8643,8656,8664-8666,8673,8676,8686,10208,10240,10246,10254,10262,10265,10267,10271,10278,10288,10306,10316,10318,10326,10347-10348,10350,10354,10411,10634-10635,10658,10660,10671,10677,10684,10719,10723,10727,10738,10742,10775,10784,10790,10796,10799,10805,10808-10809,10816,10823,10832,10839-10840,10843,10861,11368-11369,11373-11374,11382,11384,11417,11420,11423,11425,11434,11438,11442,11454,11457,11465,11470-11471,11478,11488,11493-11494,11499,11502,11508,11728,11736,11755"}}
+{"timestamp":1714176014.6091311,"name":"online","context":{"idset":"8575,8668,8690,10754,10782,10810,10845,10856"}}
+{"timestamp":1714176014.6879089,"name":"online","context":{"idset":"6552,6617,6625,6627,7987,8026,8247,8274,8578,8580,8593,8627-8628,8639,8655,8662,8669,8679,8691,9075,10311,10346,10381,10386,10400,10419-10420,10435,10443,10712,10773,10783,10792,10867,10886,10906,10937,11061,11099,11118,11343,11424,11435,11447-11448,11466,11473,11486,11497,11507,11737,11740,11748,11756"}}
+{"timestamp":1714176014.6900501,"name":"online","context":{"idset":"10310,11036"}}
+{"timestamp":1714176014.6916409,"name":"online","context":{"idset":"9764,11274,11467,11490"}}
+{"timestamp":1714176014.7512512,"name":"online","context":{"idset":"8032,8252,10300,10358,11758,11760"}}
+{"timestamp":1714176014.784245,"name":"online","context":{"idset":"7930,7957,7961,8007,8186,8202,8230,8431,8566,8592,8682,9374,10026,10304,10317,10357,10455,10526,10654,10657,10826,10903,11039,11320,11449,11504,11731-11732,11734"}}
+{"timestamp":1714176014.844461,"name":"online","context":{"idset":"8688,9137,9850,10280,10626,10881,10939,10977,11004,11106,11155,11307,11363"}}
+{"timestamp":1714176014.8673408,"name":"online","context":{"idset":"307,420,7711,7816,7932,7991,8273,8584,8927,9290,9368,10714,10872,10902,10912,10919,10954,10960,10971,11017,11025,11044,11079,11239,11271,11273,11351,11354,11370,11376,11891"}}
+{"timestamp":1714176014.9156897,"name":"online","context":{"idset":"93-94,96,100-104,116,118,124,128,130,132,157-158,160-161,166-169,171,173-174,177,180,183,185-189,193,195-197,199,204,208-209,213,219-221,224-225,228-229,236,238,240-241,244-245,247-248,252,277-278,282,284,290-293,295,297-299,308,310,312,316,320,322,325-326,330,333,337,339,343-347,408,416-419,7985,8268,10287,10295,10320,10434,10705,10849"}}
+{"timestamp":1714176014.9858782,"name":"online","context":{"idset":"181,329,573,6551,6590,6605,6631,6637,6643-6644,7290,7810,7836,7925,7950,8006,8163,8599,8607,8831,8887,9142,9190,9215,9248-9249,9348,9364,9375,9439-9440,9499,9663,9725,9731,9854,9922,9939,9999,10123,10153,10283,10291,10302,10388,10403,10468-10469,10471,10484,10604,10672,10715,10869,10877,10880,10890,10895,10927,10938,10940,10959,10972,10978,10996,10999-11000,11007,11009,11020,11031,11045,11052,11054,11059-11060,11066,11068-11069,11077-11078,11081,11117,11138,11201,11262,11267,11282,11304,11312,11316,11338,11344,11346,11349,11362,11398,11408,11414,11419,11549,11572,11606,11613,11676,11694,11726,11751,11776,11833,11867"}}
+{"timestamp":1714176014.9879651,"name":"online","context":{"idset":"6603,9087,9904,10387,10980"}}
+{"timestamp":1714176014.9902542,"name":"online","context":{"idset":"9321,9880,9911,10408,11033,11084,11647"}}
+{"timestamp":1714176015.0949805,"name":"online","context":{"idset":"97,222,6548,6561,7395,7904,7980,8027,8588,8600,8904,9331,9431,9433,9564,9607,9657,9777,9807,9834,9889,9899,9903,9926,9951,9967,9987,10051,10053,10063,10082,10163,10289,10361,10391,10395,10399,10432,10448,10464,10474,10477,10577,10893,10899,10913,10931-10932,10942,10951,10961,10975,10981-10982,10986,10992,11016,11028,11080,11092,11108,11203,11261,11476,11525,11547,11629,11661,11811"}}
+{"timestamp":1714176015.1667452,"name":"online","context":{"idset":"175,300,414,7371,7984,8107,8832,8860,8863,8869,8878,8893,8916,8960,9112,9207,9217,9278,9365,9426,9519,9609,9619,9632,9685,9714,9747,9763,9782,9803,9838,9849,9872,9875,9932,9946,9966,10111,10121,10148,10182,10196,10252,10327,10329,10342,10449,10545,10578,10615,10695,10801,10922,10925,10963,11064,11073,11127,11253,11603,11625,11646,11659,11695,11714,11781,11804"}}
+{"timestamp":1714176015.2366011,"name":"online","context":{"idset":"159,577,6632,6638,7288,7387,7912,7934,8105,8116,8144,8180,8214,8219,8280,8288,8598,8611,8619,8635,8849,8856,8871,8894,8923,9108,9116,9128,9132,9146-9147,9165,9178,9187,9204,9234,9251,9303,9305,9325,9346,9379,9389,9407,9428,9434,9459,9639,9664,9730,9745,9779,9826,9851,9861-9862,9864,9917,9944,10089,10135,10174,10192,10362,10368,10450,10472,10480,10486,10532,10585,10720,10882,10911,10920-10921,10923,10973,10991,10998,11002,11014,11049,11051,11053,11076,11082,11103,11105,11112,11116,11166,11221,11260,11308,11407,11439,11444,11475,11515,11520,11530,11538,11542,11546,11553,11610,11638,11660,11687,11716,11723,11789,11807,11815,11835,11841,11878,11882"}}
+{"timestamp":1714176015.315989,"name":"online","context":{"idset":"251,306,340,412,575,6520,6588,6634,7333,7348,7379,7862,7889,7920,7969,7998,8090,8111,8115,8126,8129,8134,8138,8140,8153,8158,8161,8167,8178,8211,8258-8259,8267,8285,8606,8609,8674,8837,8839,8898,8950,8967,9073,9089-9090,9126,9150,9168,9195,9231,9238,9271,9284,9297-9298,9357,9386-9387,9412,9452,9483,9509,9521,9599-9600,9602,9612,9647,9667,9694,9697,9703,9708,9735,9746,9753,9757,9774,9784,9789,9801-9802,9820,9863,9867,9883,9898,9929,9952,9956,9963,10023,10042,10077,10084,10106,10113,10140,10158,10178,10213,10218,10227,10253,10290,10339,10376,10430,10433,10436,10444,10452,10479,10485,10496,10504,10507,10513,10519,10554,10559,10573,10576,10592,10599,10612,10679,10706,10753,10779,10795,10818,10821,10835,10850,10853,10966,10985,10989,11018,11041,11098,11104,11109,11111,11113,11137,11160,11169,11175,11179-11180,11182,11185,11199,11208,11217,11219,11222,11228,11295,11323,11333,11342,11345,11451,11513,11517,11537,11539,11577,11600,11611,11626,11643,11668,11675,11696,11701,11708-11709,11711,11729,11747,11788,11826,11828,11840,11842,11869"}}
+{"timestamp":1714176015.3215132,"name":"online","context":{"idset":"10426,10442,10473,10509,10517,10540,10563,10584,10595,10629,10995,11094,11178,11268,11284,11352,11522,11845,11847"}}
+{"timestamp":1714176015.3470569,"name":"online","context":{"idset":"7877,10424,10478,11001,11191,11264,11389,11602"}}
+{"timestamp":1714176015.4464493,"name":"online","context":{"idset":"33,576,826,878,884,6599,6602,6612,6636,7297,7304,7312,7317,7329,7331,7336-7337,7346-7347,7349,7353,7358,7360-7361,7368,7386,7389,7392,7394,7716,7802,7825,7840,7844-7845,7852,7890,7899,7903,7928,7951,8000,8057,8085-8086,8089,8103-8104,8117-8119,8124,8130-8131,8135,8139,8170-8171,8175,8182,8213,8239,8264,8275,8278,8283,8299-8301,8308,8312,8314,8567,8579,8589,8608,8610,8632,8647,8681,8692,8826,8828,8861,8864,8866,8870,8875,8879-8880,8892,8896,8924,8930,8937,8939,8943,8949,8953,8974,8978,8982,9069,9074,9098-9099,9102,9107,9123,9136,9138,9143,9159,9162,9176,9185,9188,9196-9197,9208,9214,9220,9227,9240,9243,9253,9255,9257-9259,9265,9269-9270,9277,9293,9295,9302,9304,9306,9318,9326,9329,9339,9344,9350,9355,9377,9381,9394-9395,9401,9406,9408,9415,9447,9450-9451,9469,9478,9502,9525,9529,9589,9593-9594,9604,9608,9610,9617,9622,9627,9631,9636,9640,9648,9651-9652,9656,9669,9679,9681,9700,9709,9720,9722,9728-9729,9737,9759,9772,9785,9790,9809,9812,9815,9825,9833,9837,9845-9846,9848,9852,9859,9870,9878-9879,9881,9888,9901,9912-9914,9918,9925,9940,9947,9984,10003,10010,10040,10046,10078,10101-10102,10116,10124,10169,10198,10241,10244,10259,10264,10268,10279,10286,10314,10355,10373,10382,10390,10404,10406,10454,10489,10520,10565,10589,10591,10603,10668,10673,10743,10857,10885,10941,11093,11121,11139,11157,11170,11192,11210,11213,11244,11266,11387,11540,11567,11584,11587,11605,11619,11633,11652,11769,11796,11870"}}
+{"timestamp":1714176015.4555974,"name":"online","context":{"idset":"9256,9351,9658,10878,10901,10918,10983,10987,11043,11168,11186,11200,11227,11230,11235,11285,11287,11335,11411,11464,11511,11578,11595,11671,11673,11686,11721,11775,11782,11784,11827,11853-11854,11859"}}
+{"timestamp":1714176015.6027379,"name":"online","context":{"idset":"283,6545,6608,6616,6620,6629,7308,7319,7323,7326,7343,7354,7363-7364,7398,7400,7404,7406,7411,7734,7813,7829-7830,7832,7867,7892,7909,7913,7948,7970,8002,8098,8100,8121,8141,8168,8188,8195,8235,8281,8292,8295-8296,8305,8307,8320,8323,8614,8644,8667,8844,8846,8852,8872,8901,8905,8945,9047,9086,9092,9096-9097,9111,9113,9119,9125,9155,9164,9169,9182,9191,9201,9210,9212,9218,9223,9233,9242,9254,9276,9288,9309,9312,9320,9322,9328,9359,9372-9373,9384,9409,9413,9419,9421,9453-9454,9458,9491,9512,9590,9621,9628-9630,9638,9660,9668,9672,9677-9678,9682,9684,9689,9696,9699,9740,9742,9744,9749-9750,9756,9767,9770,9778,9792,9805,9827,9841-9843,9874,9884,9896,9950,9955,9991,10020,10068,10074,10115,10119-10120,10130,10137,10156,10162,10168,10190,10214,10219,10226,10230,10353,10360,10370,10392,10401,10431,10462,10498,10506,10534,10536,10555,10570,10583,10601,10616,10637-10638,10675,10680,10696,10722,10747,10750,10794,10812,10827,10908,10946,10950,11034,11062,11102,11115,11142,11144,11159,11173,11176,11194,11212,11215-11216,11236,11242,11248,11255,11272,11276,11300-11301,11319,11326,11328,11418,11519,11529,11534,11571,11574,11579,11581,11588,11590,11620,11624,11634,11654,11665,11679,11752,11777,11805,11814,11819,11823,11825,11831,11836,11838,11846,11849-11850,11860,11864,11866,11871-11872,11879,11885,11887"}}
+{"timestamp":1714176015.6986408,"name":"online","context":{"idset":"41,44,50,99,107,126,162,200,215,234,242,288,327,978,1218,6614,6630,7306,7313,7322,7330,7339,7342,7370,7382,7399,7407,7409,7803,7811,7815,7831,7835,7871,7879,7884,7905,7946,7967,7979,8033,8049,8053,8101,8133,8150,8162,8169,8173,8198,8232,8294,8298,8316,8321,8604,8651,8890,8928,9052,9076,9084,9115,9160,9166,9175,9198,9264,9335,9347,9366-9367,9369,9388,9393,9404,9446,9537,9542-9543,9614,9623,9646,9688,9706,9712,9715,9748,9768,9788,9816,9824,9829,9891,9908-9909,9949,9968,9971,9980,9982,9997,10007,10021,10034,10049,10065,10079,10096,10114,10126,10159-10160,10173,10183,10195,10212,10220,10237,10313,10369,10385,10393,10417,10445,10491,10503,10524,10549,10558,10564,10581-10582,10602,10609,10614,10647,10694,10699,10704,10728,10740,10757,10824,10860,10870,10875,10891,10924,10929,10933,10936,10974,10997,11012-11013,11070,11089,11150,11154,11158,11165,11198,11229,11231,11258,11302,11325,11336,11359,11391,11401,11409,11437,11455,11483,11512,11557,11583,11598,11604,11607,11636,11639,11656,11681,11715,11727,11767,11771,11816,11848,11851,11858,11868,11888"}}
+{"timestamp":1714176015.7106316,"name":"online","context":{"idset":"179,7366,7918,8093,8909,8915,8929,8941,8964,8970,8976,8979,9221,9237,9274,9333,9476,9494,9514,9558,10142"}}
+{"timestamp":1714176015.8381903,"name":"online","context":{"idset":"1,4-5,15,17,20-21,24,31-32,40,45,52,57-59,1206,6529-6530,6557,6583,6621,6626,7310,7318,7324,7355-7356,7385,7388,7401-7403,7740,7747,7754,7839,7916,7924,7955,7981,7983,7992,8047,8147,8154,8270,8317,8605,8612,8652-8653,8857,8907,8966,8973,9000,9054,9268,9315,9691,9765,10009,10044,10059,10076,10093,10136,10309,10334,10337,10344,10428,10438,10446,10499,10527,10539,10561,10613,10667,10781,10814,10817,10871,10896-10897,10905,11003,11006,11038,11055-11056,11100,11146,11148,11156,11256,11309,11334,11337,11372,11390,11413,11436,11443,11453,11489,11535,11568,11582,11586,11596,11601,11632,11649,11657,11690,11693,11717,11719,11768,11779,11824,11832,11874,11892"}}
+{"timestamp":1714176015.842478,"name":"online","context":{"idset":"8311,8315,8318,9766,9942,10134"}}
+{"timestamp":1714176015.9487011,"name":"online","context":{"idset":"110,125,127,191,203,212,226,235,302,317,321,323,328,332,413,836,873,6607,6639,7328,7344,7352,7373,7376,7378,7670,7704,7741,7744,7758,7762,7771,7863,7866,7875,7885,8011,8071,8099,8145,8159,8165,8313,8324,8568,8638,8824,8829,8834,8838,8850,8853,8882,8899,8938,8942,8968,8972,8977,8985,9016,9055,9060,9077,9081,9083,9103,9120,9134,9171,9179-9180,9183,9260,9263,9308,9316,9324,9332,9343,9345,9390,9399,9427,9455,9480,9503,9505,9516,9528,9538,9540-9541,9549-9550,9563,9570,9586,9591,9618,9624,9653,9674,9676,9693,9701,9736,9821-9822,9894,9910,9993,10001,10015,10043,10075,10110,10132,10151,10185,10193,10197,10202,10238,10323,10410,10429,10437,10440,10481,10548,10551,10571,10593,10663,10685,10898,10928,10952,11270,11329,11501,11763,11795,11806"}}
+{"timestamp":1714176016.0165522,"name":"online","context":{"idset":"7374,7390,7405,7837,7914,7989,9205-9206,9262,9299,9398,9424,9464,9573,9577,9582,9711,9885,9915,9988,10005,10008,10013,10018,10045,10054,10088,10091-10092,10097,10125,10141,10149,10165,10221-10222,10266,10380,10398,10458,10466,10495,10542,10544,10572,10607,10837,10916,10964,10968-10969,11048,11123,11136,11218,11223,11251,11355,11422,11479,11550,11591,11792"}}
+{"timestamp":1714176016.0999629,"name":"online","context":{"idset":"25,51,108,111,178,243,1235,6536,6601,6610,7296,7332,7367,7718,7764,7770,7776,7785-7786,7808,7868,7876,7897,8102,8164,8302,8309,8601,8672,8840,8917-8918,8936,8975,8984,8987,9009,9013,9026,9045,9109,9152,9341,9353,9371,9432,9448,9487,9569,9578,9641,9675,9680,9743,9773,9831,9897,9962,9970,9972,9992,9994,10033,10070,10105,10150,10166,10180-10181,10201,10367,10467,10528,10574,10579,10935,11202,11211,11226,11306,11313,11570,11666,11705,11750,11757,11761,11808,11883"}}
+{"timestamp":1714176016.1073987,"name":"online","context":{"idset":"10122,11263"}}
+{"timestamp":1714176016.2055173,"name":"online","context":{"idset":"22,56,239,1207,1214,1225,7302,7314,7410,7719,7765,7791,7841,7847,7865,7869,8091,8127,8160,8166,8297,8867,8886,8903,8948,8980,8993,8998,9028,9058,9071,9088,9100,9117,9141,9161,9172,9181,9226,9363,9405,9475,9515,9547,9581,9584-9585,9588,9595,9643,9716,9741,9794-9795,9804,9818,9938,10029,10050,10064,10108,10145,10372,10374,10407,10409,10514,10610,10873,10888,10957,10967,11008,11024,11035,11058,11067,11083,11085,11110,11119,11143,11153,11171,11209,11575,11589,11706,11720,11803,11830"}}
+{"timestamp":1714176016.2809215,"name":"online","context":{"idset":"303,7298,7340,7377,7408,7696,7773,7874,7907,8066,8096,8128,8287,8842,8868,8913,8933,8999,9121,9177,9193,9244,9283,9410,9518,9544,9572,9579-9580,9606,9673,9723,9752,9877,9941,10024,10047,10117,10170,10422,10465,10475,10611,11193,11580,11691"}}
+{"timestamp":1714176016.2907758,"name":"online","context":{"idset":"7357,8990,9006,9539,9705,10538,10541,10594,10606,10904,11135,11532"}}
+{"timestamp":1714176016.386827,"name":"online","context":{"idset":"7299,7307,7341,7372,7380,7393,7693,7722,7755,7783,7814,7834,7881,7915,7919,8087,8284,8821,8847,8912,8959,8961,8997,9015,9057,9104,9153,9163,9202,9225,9289,9292,9354,9382,9411,9425,9445,9473,9520,9527,9557,9562,9635,9637,9695,9718,9776,9787,9836,9855,9890,9893,9900,9953,9960,9996,10025,10081,10194,10378,10493,10562,10596,10892,11129,11183,11214,11246,11311,11510,11531,11558,11564,11585,11609,11622,11635,11640,11648,11655,11677-11678,11722,11780,11785,11812,11817,11873,11876,11890"}}
+{"timestamp":1714176016.3896973,"name":"online","context":{"idset":"9575,9823"}}
+{"timestamp":1714176016.3948629,"name":"online","context":{"idset":"7873,8151,8848,9101,10118,10500"}}
+{"timestamp":1714176016.3976719,"name":"online","context":{"idset":"11664"}}
+{"timestamp":1714176016.4062366,"name":"online","context":{"idset":"9468,11799"}}
+{"timestamp":1714176016.5102515,"name":"online","context":{"idset":"1231,7694,7828,8889,9001,9018,9551,9721,9796,9887,9936,9943,9978,10107,10461,11225,11818"}}
+{"timestamp":1714176016.5937629,"name":"online","context":{"idset":"1188,1205,1210,1215,1232,6598,6604,7294,7300,7325,7350,7362,7391,7679,7691-7692,7700,7707,7717,7723,7725,7730-7731,7733,7736,7745,7749,7772,7804,7870,7898,7901,8054,8061,8070,8074-8075,8078,8094,8152,8322,8825,8910,8971,9008,9011,9017,9059,9072,9078,9124,9127,9203,9235,9247,9275,9287,9337,9396,9417,9420,9438,9460-9461,9463,9466,9504,9513,9533,9536,9567,9633,9644,9707,9710,9713,9828,9832,9857,9882,9916,9923,9928,9958,9975,9983,9985,10028,10057,10071,10080,10098,10164,10172,10216,10494,10511,10516,10580,10597,11181,11205,11220,11234,11252,11472,11514,11628,11630,11642,11707,11770,11800,11813,11862,11880"}}
+{"timestamp":1714176016.5976501,"name":"online","context":{"idset":"7706"}}
+{"timestamp":1714176016.7168989,"name":"online","context":{"idset":"1209,1213,1228,1234,7320,7675,7677,7681,7689,7701,7795,8082,8137,8823,8908,8911,8992,9022,9025,9049,9053,9056,9189,9402,9416,9422,9561,9568,9791,9892,9895,9902,9977,10004,10011-10012,10041,10058,10072,10543,10552,10568,10636,11161,11167,11662,11685,11725,11884"}}
+{"timestamp":1714176016.7871792,"name":"online","context":{"idset":"10014"}}
+{"timestamp":1714176016.8758585,"name":"online","context":{"idset":"11,899,1180,1186,1230,1233,7685,7690,7760,7766,7787,7792,7812,7900,7921,8063,9007,9027,9239,9310,9531,9535,9555,9844,9945,9973,10002,10032,10547,10550,10692,11240,11593,11698"}}
+{"timestamp":1714176016.8806145,"name":"online","context":{"idset":"900,1190,1217,1227,7676,7705,7710,7713,7735,7750,7767,7797,7823,7850,8064,9019,9485,9979,10359,10661,10962,11321,11741"}}
+{"timestamp":1714176016.8835201,"name":"online","context":{"idset":"7846"}}
+{"timestamp":1714176016.9862089,"name":"online","context":{"idset":"1173,1212,1222,6559,7315,7724,7851,7861,7883,7887"}}
+{"timestamp":1714176016.9903107,"name":"online","context":{"idset":"1189,8065,9490,9522,10915"}}
+{"timestamp":1714176017.0823247,"name":"online","context":{"idset":"12,26,30,54,207,1179,1220,7678,7821,7849,8358,8991,9020,9587,9847,10022"}}
+{"timestamp":1714176017.1554177,"name":"online","context":{"idset":"10,29,38,11361"}}
+{"timestamp":1714176017.161231,"name":"online","context":{"idset":"39"}}
+{"timestamp":1714176017.166971,"name":"online","context":{"idset":"7748,7800-7801,7910,10000"}}
+{"timestamp":1714176017.1725433,"name":"online","context":{"idset":"1182"}}
+{"timestamp":1714176017.1884923,"name":"online","context":{"idset":"1183,7756,7838,7908,8854,11250"}}
+{"timestamp":1714176017.1999831,"name":"online","context":{"idset":"7856,9470,11787"}}
+{"timestamp":1714176017.2976823,"name":"online","context":{"idset":"7784,8415,9010,10748"}}
+{"timestamp":1714176017.367012,"name":"online","context":{"idset":"6,34,835,1185,1187,1219,7683,7687,7729,7737,7774,7790,7794,7798,7827,7857,7895,8062,8068-8069,8319,8328,8420,8986,8988,8996,9131,9192,9755,10027,10421,11718,11822"}}
+{"timestamp":1714176017.3718348,"name":"online","context":{"idset":"1184,7686,7796,8083,8375,8406,9002"}}
+{"timestamp":1714176017.3749237,"name":"online","context":{"idset":"7708"}}
+{"timestamp":1714176017.3871586,"name":"online","context":{"idset":"7788"}}
+{"timestamp":1714176017.494565,"name":"online","context":{"idset":"1216,1236"}}
+{"timestamp":1714176017.5021894,"name":"online","context":{"idset":"7728,7769,7781,7864,7878,8369,8405,8994,11616"}}
+{"timestamp":1714176017.5056469,"name":"online","context":{"idset":"9726"}}
+{"timestamp":1714176017.6357751,"name":"online","context":{"idset":"43,48"}}
+{"timestamp":1714176017.6419938,"name":"online","context":{"idset":"8384"}}
+{"timestamp":1714176017.6455019,"name":"online","context":{"idset":"8340,8418"}}
+{"timestamp":1714176017.6491427,"name":"online","context":{"idset":"8371"}}
+{"timestamp":1714176017.6527913,"name":"online","context":{"idset":"8325,8366"}}
+{"timestamp":1714176017.6561089,"name":"online","context":{"idset":"1224,7817"}}
+{"timestamp":1714176017.659641,"name":"online","context":{"idset":"9986"}}
+{"timestamp":1714176017.6628952,"name":"online","context":{"idset":"7726"}}
+{"timestamp":1714176017.6675534,"name":"online","context":{"idset":"6565,7782"}}
+{"timestamp":1714176017.6708591,"name":"online","context":{"idset":"11712"}}
+{"timestamp":1714176017.6737804,"name":"online","context":{"idset":"11683"}}
+{"timestamp":1714176017.7922378,"name":"online","context":{"idset":"8374,8409"}}
+{"timestamp":1714176017.9175789,"name":"online","context":{"idset":"8380"}}
+{"timestamp":1714176017.9295602,"name":"online","context":{"idset":"8378"}}
+{"timestamp":1714176017.9329867,"name":"online","context":{"idset":"9012"}}
+{"timestamp":1714176018.0452342,"name":"online","context":{"idset":"3"}}
+{"timestamp":1714176018.1256559,"name":"online","context":{"idset":"6571,8327,8338,8383,8387,8398,8411,8981,8989"}}
+{"timestamp":1714176018.202738,"name":"online","context":{"idset":"8329,8342,8401"}}
+{"timestamp":1714176018.2963855,"name":"online","context":{"idset":"6570"}}
+{"timestamp":1714176018.3791065,"name":"online","context":{"idset":"8381,8434"}}
+{"timestamp":1714176018.4546506,"name":"online","context":{"idset":"8335,8428"}}
+{"timestamp":1714176018.5328374,"name":"online","context":{"idset":"6567,8357,8367,8372"}}
+{"timestamp":1714176018.6124985,"name":"online","context":{"idset":"6580,8343,8346,8361,8373,8377,8396"}}
+{"timestamp":1714176018.6904321,"name":"online","context":{"idset":"8389,8404,8417"}}
+{"timestamp":1714176018.7700467,"name":"online","context":{"idset":"8364,8368"}}
+{"timestamp":1714176018.8453693,"name":"online","context":{"idset":"8426"}}
+{"timestamp":1714176018.8760648,"name":"online","context":{"idset":"8355"}}
+{"timestamp":1714176018.9862349,"name":"online","context":{"idset":"6568,6578,8379,8400,11297"}}
+{"timestamp":1714176018.9920952,"name":"online","context":{"idset":"8326"}}
+{"timestamp":1714176019.0956554,"name":"online","context":{"idset":"7952,8353,8649"}}
+{"timestamp":1714176019.1785743,"name":"online","context":{"idset":"8349,8399,8433,10187,10203,10206,10277"}}
+{"timestamp":1714176019.2679112,"name":"online","context":{"idset":"7929"}}
+{"timestamp":1714176019.2726874,"name":"online","context":{"idset":"10619,11294"}}
+{"timestamp":1714176019.2783053,"name":"online","context":{"idset":"10258"}}
+{"timestamp":1714176019.284893,"name":"online","context":{"idset":"6573"}}
+{"timestamp":1714176019.3038394,"name":"online","context":{"idset":"10447,10751"}}
+{"timestamp":1714176019.4108834,"name":"online","context":{"idset":"10756"}}
+{"timestamp":1714176019.4373305,"name":"online","context":{"idset":"579,6534,7859,7953,8435,10383,10394,10776,10836,11432"}}
+{"timestamp":1714176019.520174,"name":"online","context":{"idset":"10644,11445"}}
+{"timestamp":1714176019.535285,"name":"online","context":{"idset":"8048,8218,8424,11075,11279,11500,11762"}}
+{"timestamp":1714176019.6903508,"name":"online","context":{"idset":"7886,7962,8344,8429,10655,10813"}}
+{"timestamp":1714176019.8149188,"name":"online","context":{"idset":"184,6537,7931,8019,8038,8248,8351,8423,8630,8634,9493,9840,10133,10247,10396,11015,11339,11378,11381,11394,11450,11477,11837"}}
+{"timestamp":1714176019.9203455,"name":"online","context":{"idset":"301,7938,7972,8052,8271,8574,9489,9793,10006,10248,10515,10630,10639,10693,10820,10855,11259,11386,11416,11456,11484"}}
+{"timestamp":1714176020.0339193,"name":"online","context":{"idset":"287,6628,8008,8260,8350,8581,8683,9510,9530,9546,9616,9738,9839,10336,10620,10666,10702,10716,10730,10752,10774,10789,10889,11050,11404,11746"}}
+{"timestamp":1714176020.1408331,"name":"online","context":{"idset":"123,318,6556,6642,7826,7854,7911,7954,8030,8142,8648,8931,9130,9219,9296,9380,9397,9597,9751,10324,10521,10678,10749,10909,10914,10943,10949,11281,11347,11670,11753"}}
+{"timestamp":1714176020.3327851,"name":"online","context":{"idset":"95,122,164,211,214,216,227,230,249,294,296,311,1208,1221,6562,6577,7289,7338,7688,7709,7757,7848,7853,7872,8155,8290,8306,8394,8654,8836,8877,8895,8906,8922,8969,9157,9222,9313,9360,9418,9444,9456,9474,9625,9649,9717,9739,9769,9814,9976,10127,10177,10207,10363,10375,10377,10557,10686,10887,11023,11356,11358,11614,11651,11861,11881"}}
+{"timestamp":1714176020.4897413,"name":"online","context":{"idset":"6585,7682,7714,8207,8419,8841,8983,9021,9118,9174,9300,9334,9477,9508,9698,9868,9905,9930,10175,10269,10303,10384,10588,10640,10708,10797,10838,11145,11204,11322"}}
+{"timestamp":1714176020.5822351,"name":"online","context":{"idset":"6591,8215,8286,8416,8670,8822,8944,9352,9526,9548,9719,9937,10224,10233,10508,10531,10566,11429,11496,11528,11680,11766,11783"}}
+{"timestamp":1714176020.5855114,"name":"online","context":{"idset":"10590"}}
+{"timestamp":1714176020.6953204,"name":"online","context":{"idset":"8645,9429,10328,11292"}}
+{"timestamp":1714176020.7791266,"name":"online","context":{"idset":"324,407,7753,7977,8224,8590,9435,9692,9866,10459,10476,11047,11072,11131,11458,11594,11597,11738,11749"}}
+{"timestamp":1714176020.7848227,"name":"online","context":{"idset":"8029,9392,11704"}}
+{"timestamp":1714176020.7895715,"name":"online","context":{"idset":"11428"}}
+{"timestamp":1714176020.9037931,"name":"online","context":{"idset":"176,210,250,415,1211,7345,7891,8110,8184,9414,9601,9659,9835,10143,11275,11797"}}
+{"timestamp":1714176020.9090056,"name":"online","context":{"idset":"285,10416"}}
+{"timestamp":1714176021.0138412,"name":"online","context":{"idset":"115,409,7316,8041,8067,8076-8077,8125,8194,8339,8408,8865,8900,9959,10037,11037,11124,11190,11552,11802"}}
+{"timestamp":1714176021.1089268,"name":"online","context":{"idset":"163,6569,7732,8678,9110,9286,9498,10505,10608,10948"}}
+{"timestamp":1714176021.119179,"name":"online","context":{"idset":"9560"}}
+{"timestamp":1714176021.2243965,"name":"online","context":{"idset":"28,341,6606,7295,7396,7684,7712,8014,8192,8203,9620,9797,10016,10146,10229,10502,10759,10846,10958,11141,11237,11303,11608,11623,11692,11700,11839"}}
+{"timestamp":1714176021.3094878,"name":"online","context":{"idset":"6541,8602,8671,9005"}}
+{"timestamp":1714176021.3140745,"name":"online","context":{"idset":"9511,11383"}}
+{"timestamp":1714176021.3190114,"name":"online","context":{"idset":"7746,10762"}}
+{"timestamp":1714176021.3390822,"name":"online","context":{"idset":"8687"}}
+{"timestamp":1714176021.4697032,"name":"online","context":{"idset":"53,315,7778,7809,8012,8331,8617,8835,8946,9085,9093,9634,9702,10709,10956,11184,11283"}}
+{"timestamp":1714176021.4780471,"name":"online","context":{"idset":"2,14,46,55"}}
+{"timestamp":1714176021.4860115,"name":"online","context":{"idset":"60"}}
+{"timestamp":1714176021.4895573,"name":"online","context":{"idset":"11545"}}
+{"timestamp":1714176021.5103686,"name":"online","context":{"idset":"36"}}
+{"timestamp":1714176021.6721377,"name":"online","context":{"idset":"114,1229,8179,9194,9323,9670,9762,10926,11533,11663,11801"}}
+{"timestamp":1714176021.7793431,"name":"online","context":{"idset":"7888,8352,8360,8402,8925,8962,9156,9280,9356,9462,9484,9556,9661,9858,9919,9927,9974,9995,10067,10223,10250-10251,10930,11032,11132,11140,11187,11224,11233,11364,11526,11599,11644,11669,11699,11774,11834"}}
+{"timestamp":1714176021.9589944,"name":"online","context":{"idset":"7822,9358,9441,9481,10188"}}
+{"timestamp":1714176022.0621021,"name":"online","context":{"idset":"7365,7695,8279,8427,8897,8932,8955,9186,9497,9860,9865,9969,10155,10488,10575,11063,11091,11147,11163,11536,11650"}}
+{"timestamp":1714176022.0656281,"name":"online","context":{"idset":"8919"}}
+{"timestamp":1714176022.1729884,"name":"online","context":{"idset":"10056,11793"}}
+{"timestamp":1714176022.2591844,"name":"online","context":{"idset":"10225"}}
+{"timestamp":1714176022.345365,"name":"online","context":{"idset":"8114,8430,8436"}}
+{"timestamp":1714176022.5584266,"name":"online","context":{"idset":"10868"}}
+{"timestamp":1714176022.6599107,"name":"online","context":{"idset":"19,7982,10804"}}
+{"timestamp":1714176022.7370229,"name":"online","context":{"idset":"8"}}
+{"timestamp":1714176022.8229167,"name":"online","context":{"idset":"7917,7933,10103,10161,10179,10281,10768,10778,10900,11021,11164,11189,11348,11357,11733,11857"}}
+{"timestamp":1714176022.9331851,"name":"online","context":{"idset":"7671,8629,9471,10239,11005,11305,11379,11844"}}
+{"timestamp":1714176023.0367827,"name":"online","context":{"idset":"1174,6593,7779,7959,7965,7968,8044,8120,8569,8934,9250,9690,10017,10030,10176,10441,10990,11086,11122,11125,11703,11724"}}
+{"timestamp":1714176023.1406238,"name":"online","context":{"idset":"8208,8254,8618,8952,9167,9211,9524,9683,10209,10245,10322,10910,11206,11491,11509,11543,11764"}}
+{"timestamp":1714176023.3333838,"name":"online","context":{"idset":"7305,7672,7742,7855,8123,8855,8902,8995,9228,9241,9282,9342,9362,9566,9686,9727,9806,9830,9853,9856,10087,10825,10944,11396,11689,11702,11745,11886"}}
+{"timestamp":1714176023.5184283,"name":"online","context":{"idset":"6543,8072,8197,9378,9654,9733,9775,9934,11010,11149,11196,11617,11645,11889"}}
+{"timestamp":1714176023.6236813,"name":"online","context":{"idset":"7799,8056"}}
+{"timestamp":1714176023.8221381,"name":"online","context":{"idset":"281,7309,7381,7412,7680,7699,7858,9133,9145,9184,9245,9261,9338,9361,9383,9385,9479,9486,9488,9532,9626,9786,10048,10066,10083,10497,11088,11114,11120,11130,11269,11505,11612,11658,11710,11765"}}
+{"timestamp":1714176024.1081645,"name":"online","context":{"idset":"7359,7674,8310,8412,8422,8883,8940,9046,9149,9266,9281,9336,9349,9449,9501,9876,9921,9964,9990,10019,10128,10537,10556,11151,11524,11527,11843,11875"}}
+{"timestamp":1714176024.2982571,"name":"online","context":{"idset":"8334,8386,8407"}}
+{"timestamp":1714176024.3997309,"name":"online","context":{"idset":"8336,8376,8390,8393,8395"}}
+{"timestamp":1714176024.5812755,"name":"online","context":{"idset":"8382,8391-8392,8403,8410"}}
+{"timestamp":1714176024.694088,"name":"online","context":{"idset":"8359,8370"}}
+{"timestamp":1714176024.7924008,"name":"online","context":{"idset":"6575"}}
+{"timestamp":1714176025.0753522,"name":"online","context":{"idset":"8341"}}
+{"timestamp":1714176025.1832497,"name":"online","context":{"idset":"8425"}}
+{"timestamp":1714176025.2838562,"name":"online","context":{"idset":"8347,8354,8421"}}
+{"timestamp":1714176025.2966487,"name":"online","context":{"idset":"8348"}}
+{"timestamp":1714176026.9210637,"name":"online","context":{"idset":"7945"}}
+{"timestamp":1714176027.0260017,"name":"online","context":{"idset":"9965,11011"}}
+{"timestamp":1714176027.1278112,"name":"online","context":{"idset":"877,8181,8276,9666,10052,10518,10698,11019,11172,11188,11786"}}
+{"timestamp":1714176027.2313728,"name":"online","context":{"idset":"105,7669,7988,8143,8209,8965,9307,10200,10261,10731,10802,11410"}}
+{"timestamp":1714176027.4186091,"name":"online","context":{"idset":"883,6519,6540,6553,6555,6564,6589,7351,7805-7806,8036,8136,8222,8585,8603,8625,8684,10260,10345,10483,10662,10755,10758,10760,10765,10786,10811,10830,10894,10945,11241,11371,11380,11754,11772"}}
+{"timestamp":1714176027.6085503,"name":"online","context":{"idset":"231,6527,7947,7960,7966,7994,8196,8263,8624,8658,9082,9565,10235,10257,10263,10276,10299,10325,10413-10414,10617,10682,10717,10735,10744-10745,10767,10819,10852,10879,11257,11400,11402,11495"}}
+{"timestamp":1714176027.7184997,"name":"online","context":{"idset":"182,331,6522,6538,6549,6635,7720,7738,7777,7906,7922,7939,7943,7976,7986,8005,8009,8018,8035,8039,8108,8189,8262,8597,8620,9144,9200,9267,10131,10270,10298,10308,10330,10332,10379,10389,10402,10463,10652,10746,10907,10953,10993-10994,11030,11057,11096-11097,11280,11290,11298,11314-11315,11330,11341,11350,11385,11392-11393,11395,11399,11426,11431,11459,11463,11742-11743,11809"}}
+{"timestamp":1714176027.8232453,"name":"online","context":{"idset":"6618,7673,7926,7999,8001,8004,8046,8251,8636,8689,8862,9173,9492,9603,9732,10305,10333,10364,10397,10415,10427,10623,10710,10741,10829,10844,10858-10859,10884,10917,10947,10955,11277,11367,11485,11518,11544"}}
+{"timestamp":1714176027.9313126,"name":"online","context":{"idset":"113,289,6524,6640,7743,7818,8095,9279,9517,9545,9819,11095,11739,11855"}}
+{"timestamp":1714176028.036557,"name":"online","context":{"idset":"106,218,223,232,280,304,319,406,1177,1223,6544,6597,7291,7383,7397,7697-7698,7751,7763,7768,7807,7819,7824,7843,7860,7882,7902,7978,7996,8042,8059,8097,8112,8157,8176,8229,8289,8663,8680,8957,9079,9106,9170,9216,9236,9246,9301,9314,9340,9376,9400,9403,9500,9554,9574,9583,9596,9605,9645,9704,9811,9871,9961,10095,10104,10109,10167,10171,10189,10228,10249,10297,10365-10366,10371,10405,10451,10470,10490,10510,10567,10605,10763,10841,10876,10934,10965,10970,10976,10979,11026,11042,11046,11133,11152,11238,11278,11327,11340,11353,11460,11487,11521,11551,11573,11615,11730,11735,11759,11778,11856"}}
+{"timestamp":1714176028.1482568,"name":"online","context":{"idset":"170,190,201,1226,6600,6611,6633,6641,7752,7775,7820,7833,7880,8060,8081,8122,8148,8156,8177,8830,8891,8921,8926,8935,8963,9091,9122,9148,9232,9252,9285,9317,9330,9370,9423,9457,9472,9523,9592,9650,9665,9798,9813,9886,10073,10090,10157,10204,10418,10425,10457,10460,10501,10533,10598,10874,10984,11027,11074,11090,11197,11254,11265,11360,11559,11576,11592,11641,11682,11713,11794,11820"}}
+{"timestamp":1714176028.3328397,"name":"online","context":{"idset":"119,194,246,305,309,334,405,1175-1176,1181,6609,7285,7293,7311,7321,7335,7702,7715,7721,7727,7780,7789,8055,8058,8080,8088,8113,8132,8149,8174,8845,8859,8873,8876,8885,8888,8914,8947,9051,9105,9199,9209,9213,9273,9291,9294,9319,9391,9442-9443,9465,9467,9482,9495,9534,9553,9559,9576,9662,9671,9724,9734,9760-9761,9781,9783,9799,9869,9906,9920,9935,9948,9957,9981,9998,10035-10036,10038,10055,10061,10085,10147,10184,10217,10423,10453,10487,10492,10512,10530,10546,10560,10569,10586-10587,10988,11022,11040,11087,11162,11174,11177,11232,11247,11310,11516,11523,11653,11674,11688,11791,11798,11810,11821,11829,11852,11865,11877"}}
+{"timestamp":1714176028.4365735,"name":"online","context":{"idset":"98,112,129,192,205,233,338,7292,7369,7375,7384,7761,8079,8084,8146,8293,8827,8881,8954,8958,9095,9158,9224,9272,9496,9507,9687,9771,9817,9931,10062,10069,10112,10199,10523,10529,10600,11126,11128,11324,11481,11618,11631,11863"}}
+{"timestamp":1714176028.540581,"name":"online","context":{"idset":"7286,7793,7971,8109,8250,8282,9014,9048,9050,9154,9311,9436,9615,9655,9754,9873,10094,10099,10144,10243,10315,10522,10525,10553,10628,10833,11101,11107,11421,11498,11621,11637,11697"}}
+{"timestamp":1714176028.6448855,"name":"online","context":{"idset":"6539,6595,7301,7759,8172,8245,8586,8843,8874,8884,8951,9437,9598,9808,9924,10060,10255,10338,10664,10721,10806,10862,11318,11667"}}
+{"timestamp":1714176028.8292115,"name":"online","context":{"idset":"7334,7923,8015,8092,8238,8615,8956,9023-9024,9229,9800,9954,9989,10031,10039,10100,10780,10883,11065,11071,11207,11249,11744"}}
+{"timestamp":1714176028.9335802,"name":"online","context":{"idset":"7327,7703,7842,7896,8073,9129,9506,9571,9758,9907,10482,11029,11134,11243,11245,11627,11773"}}
+{"timestamp":1714176029.0378132,"name":"online","context":{"idset":"1178,7287,8303,8858,8920,9070,9230,9552,9780,9810,9933,10086,10138,10191,10439,10456,10535,11195"}}
+{"timestamp":1714176029.145653,"name":"online","context":{"idset":"109,131,237,8851,9080,9094,9430,9642,10139,11672,11684"}}
+{"timestamp":1714176029.3743191,"name":"online","context":{"idset":"18,23,35,37,47,7303,9114,9611,9613,10129"}}
+{"timestamp":1714176029.476017,"name":"online","context":{"idset":"7"}}
+{"timestamp":1714176029.5853498,"name":"online","context":{"idset":"9,16,42"}}
+{"timestamp":1714176029.6964767,"name":"online","context":{"idset":"13,27,49"}}
+{"timestamp":1714176032.2041368,"name":"online","context":{"idset":"8333,8362,8414"}}
+{"timestamp":1714176032.3054888,"name":"online","context":{"idset":"8385"}}
+{"timestamp":1714176032.5211453,"name":"online","context":{"idset":"6579,8330,8337,8363,8365,8388,8397,8413,8432"}}
+{"timestamp":1714176032.8114541,"name":"online","context":{"idset":"6572,6574,8332"}}
+{"timestamp":1714176033.2953374,"name":"online","context":{"idset":"6566,6576,8356"}}
+{"timestamp":1714176033.5075979,"name":"online","context":{"idset":"8345"}}
+{"timestamp":1714176192.2496312,"name":"online","context":{"idset":"574"}}
+{"timestamp":1714176305.82377,"name":"online","context":{"idset":"9065"}}
+{"timestamp":1714176306.033097,"name":"online","context":{"idset":"9063-9064,9066"}}
+{"timestamp":1714176306.5534515,"name":"online","context":{"idset":"9067"}}
+{"timestamp":1714176311.0989685,"name":"online","context":{"idset":"9061"}}
+{"timestamp":1714176319.0158496,"name":"online","context":{"idset":"9068"}}
+{"timestamp":1714176319.2695413,"name":"online","context":{"idset":"9062"}}
+{"timestamp":1714176367.5646467,"name":"online","context":{"idset":"7739"}}
+{"timestamp":1714176542.9752927,"name":"undrain","context":{"idset":"7739"}}
+{"timestamp":1714177644.2641382,"name":"online","context":{"idset":"8024"}}
+{"timestamp":1714181506.3023257,"name":"online","context":{"idset":"9151"}}
+{"timestamp":1714181506.5164821,"name":"online","context":{"idset":"9140"}}
+{"timestamp":1714181506.7106273,"name":"online","context":{"idset":"9135"}}
+{"timestamp":1714181647.6195297,"name":"online","context":{"idset":"9139"}}
+{"timestamp":1714181746.7315204,"name":"offline","context":{"idset":"8568"}}
+{"timestamp":1714181746.7360148,"name":"offline","context":{"idset":"8617"}}
+{"timestamp":1714181746.7407153,"name":"offline","context":{"idset":"8637"}}
+{"timestamp":1714181746.7470944,"name":"offline","context":{"idset":"8589"}}
+{"timestamp":1714181746.7517719,"name":"offline","context":{"idset":"8651"}}
+{"timestamp":1714181746.7562907,"name":"offline","context":{"idset":"8581"}}
+{"timestamp":1714181746.7607682,"name":"offline","context":{"idset":"8627"}}
+{"timestamp":1714181746.7648282,"name":"offline","context":{"idset":"8646"}}
+{"timestamp":1714181746.7691746,"name":"offline","context":{"idset":"8601"}}
+{"timestamp":1714181746.7737558,"name":"offline","context":{"idset":"8631"}}
+{"timestamp":1714181746.7786181,"name":"offline","context":{"idset":"8588"}}
+{"timestamp":1714181746.7831411,"name":"offline","context":{"idset":"8616"}}
+{"timestamp":1714181746.7876661,"name":"offline","context":{"idset":"8688"}}
+{"timestamp":1714181746.7922561,"name":"offline","context":{"idset":"8612"}}
+{"timestamp":1714181746.7966347,"name":"offline","context":{"idset":"8594"}}
+{"timestamp":1714181746.8010335,"name":"offline","context":{"idset":"8576"}}
+{"timestamp":1714181746.805378,"name":"offline","context":{"idset":"8602"}}
+{"timestamp":1714181746.8093979,"name":"offline","context":{"idset":"8595"}}
+{"timestamp":1714181746.81339,"name":"offline","context":{"idset":"8593"}}
+{"timestamp":1714181746.8174455,"name":"offline","context":{"idset":"8603"}}
+{"timestamp":1714181746.8219638,"name":"offline","context":{"idset":"8565"}}
+{"timestamp":1714181746.8266716,"name":"offline","context":{"idset":"8569"}}
+{"timestamp":1714181746.8308418,"name":"offline","context":{"idset":"8609"}}
+{"timestamp":1714181746.835602,"name":"offline","context":{"idset":"8626"}}
+{"timestamp":1714181746.8401546,"name":"offline","context":{"idset":"8643"}}
+{"timestamp":1714181746.8437383,"name":"offline","context":{"idset":"8591"}}
+{"timestamp":1714181746.8481698,"name":"offline","context":{"idset":"8574"}}
+{"timestamp":1714181746.8524451,"name":"offline","context":{"idset":"8573"}}
+{"timestamp":1714181746.8573511,"name":"offline","context":{"idset":"8596"}}
+{"timestamp":1714181746.8616197,"name":"offline","context":{"idset":"8599"}}
+{"timestamp":1714181746.864785,"name":"offline","context":{"idset":"8667"}}
+{"timestamp":1714181746.8678865,"name":"offline","context":{"idset":"8585"}}
+{"timestamp":1714181746.8723307,"name":"offline","context":{"idset":"8672"}}
+{"timestamp":1714181746.8755965,"name":"offline","context":{"idset":"8614"}}
+{"timestamp":1714181746.8782465,"name":"offline","context":{"idset":"8580"}}
+{"timestamp":1714181747.0190117,"name":"offline","context":{"idset":"8566"}}
+{"timestamp":1714181747.2279828,"name":"offline","context":{"idset":"8582"}}
+{"timestamp":1714181747.2324877,"name":"offline","context":{"idset":"8571"}}
+{"timestamp":1714181747.2370033,"name":"offline","context":{"idset":"8570"}}
+{"timestamp":1714181747.2413456,"name":"offline","context":{"idset":"8575"}}
+{"timestamp":1714181747.2458971,"name":"offline","context":{"idset":"8600"}}
+{"timestamp":1714181747.250514,"name":"offline","context":{"idset":"8611"}}
+{"timestamp":1714181747.2551761,"name":"offline","context":{"idset":"8619"}}
+{"timestamp":1714181747.3896368,"name":"offline","context":{"idset":"8622"}}
+{"timestamp":1714181747.3944099,"name":"offline","context":{"idset":"8597"}}
+{"timestamp":1714181747.3985515,"name":"offline","context":{"idset":"8567"}}
+{"timestamp":1714181747.426497,"name":"offline","context":{"idset":"8572"}}
+{"timestamp":1714181747.4329317,"name":"offline","context":{"idset":"8577"}}
+{"timestamp":1714181747.444489,"name":"offline","context":{"idset":"8579"}}
+{"timestamp":1714181747.4529436,"name":"offline","context":{"idset":"8583"}}
+{"timestamp":1714181747.4599476,"name":"offline","context":{"idset":"8586"}}
+{"timestamp":1714181747.4674535,"name":"offline","context":{"idset":"8604"}}
+{"timestamp":1714181747.4708178,"name":"offline","context":{"idset":"8605"}}
+{"timestamp":1714181747.4858744,"name":"offline","context":{"idset":"8606"}}
+{"timestamp":1714181747.5083432,"name":"offline","context":{"idset":"8607"}}
+{"timestamp":1714181747.5131404,"name":"offline","context":{"idset":"8608"}}
+{"timestamp":1714181747.5182536,"name":"offline","context":{"idset":"8610"}}
+{"timestamp":1714181747.5224228,"name":"offline","context":{"idset":"8613"}}
+{"timestamp":1714181747.5267718,"name":"offline","context":{"idset":"8615"}}
+{"timestamp":1714181747.530884,"name":"offline","context":{"idset":"8618"}}
+{"timestamp":1714181747.5445483,"name":"offline","context":{"idset":"8620"}}
+{"timestamp":1714181747.5689833,"name":"offline","context":{"idset":"8621"}}
+{"timestamp":1714181747.5932763,"name":"offline","context":{"idset":"8623"}}
+{"timestamp":1714181747.5980098,"name":"offline","context":{"idset":"8624"}}
+{"timestamp":1714181747.6027784,"name":"offline","context":{"idset":"8625"}}
+{"timestamp":1714181747.6076941,"name":"offline","context":{"idset":"8628"}}
+{"timestamp":1714181747.6330943,"name":"offline","context":{"idset":"8633"}}
+{"timestamp":1714181747.6473343,"name":"offline","context":{"idset":"8635"}}
+{"timestamp":1714181747.6520162,"name":"offline","context":{"idset":"8641"}}
+{"timestamp":1714181747.6563599,"name":"offline","context":{"idset":"8645"}}
+{"timestamp":1714181747.6604013,"name":"offline","context":{"idset":"8666"}}
+{"timestamp":1714181747.6646481,"name":"offline","context":{"idset":"8668"}}
+{"timestamp":1714181747.6690197,"name":"offline","context":{"idset":"8670"}}
+{"timestamp":1714181747.6737969,"name":"offline","context":{"idset":"8673"}}
+{"timestamp":1714181747.6787019,"name":"offline","context":{"idset":"8679"}}
+{"timestamp":1714181747.6831377,"name":"offline","context":{"idset":"8682"}}
+{"timestamp":1714181747.6877952,"name":"offline","context":{"idset":"8690"}}
+{"timestamp":1714181747.7032266,"name":"offline","context":{"idset":"8691"}}
+{"timestamp":1714181747.7173686,"name":"offline","context":{"idset":"8578"}}
+{"timestamp":1714181747.7257085,"name":"offline","context":{"idset":"8584"}}
+{"timestamp":1714181747.7298188,"name":"offline","context":{"idset":"8587"}}
+{"timestamp":1714181747.7340977,"name":"offline","context":{"idset":"8590"}}
+{"timestamp":1714181747.7389066,"name":"offline","context":{"idset":"8598"}}
+{"timestamp":1714181747.7537062,"name":"offline","context":{"idset":"8639"}}
+{"timestamp":1714181748.0836513,"name":"offline","context":{"idset":"8692"}}
+{"timestamp":1714181748.0949893,"name":"offline","context":{"idset":"8629"}}
+{"timestamp":1714181748.1439781,"name":"offline","context":{"idset":"8669"}}
+{"timestamp":1714181748.1527219,"name":"offline","context":{"idset":"8640"}}
+{"timestamp":1714181748.1587458,"name":"offline","context":{"idset":"8681"}}
+{"timestamp":1714181748.1762574,"name":"offline","context":{"idset":"8592"}}
+{"timestamp":1714181748.7641523,"name":"offline","context":{"idset":"8674"}}
+{"timestamp":1714181748.7746837,"name":"offline","context":{"idset":"8659"}}
+{"timestamp":1714181748.7788477,"name":"offline","context":{"idset":"8683"}}
+{"timestamp":1714181748.7831838,"name":"offline","context":{"idset":"8687"}}
+{"timestamp":1714181748.7877309,"name":"offline","context":{"idset":"8632"}}
+{"timestamp":1714181748.7923141,"name":"offline","context":{"idset":"8684"}}
+{"timestamp":1714181748.7967012,"name":"offline","context":{"idset":"8653"}}
+{"timestamp":1714181748.8009424,"name":"offline","context":{"idset":"8647"}}
+{"timestamp":1714181748.8051894,"name":"offline","context":{"idset":"8689"}}
+{"timestamp":1714181748.8090651,"name":"offline","context":{"idset":"8657"}}
+{"timestamp":1714181748.8132224,"name":"offline","context":{"idset":"8685"}}
+{"timestamp":1714181748.8166134,"name":"offline","context":{"idset":"8675"}}
+{"timestamp":1714181748.8200557,"name":"offline","context":{"idset":"8664"}}
+{"timestamp":1714181748.8240333,"name":"offline","context":{"idset":"8662"}}
+{"timestamp":1714181748.8280809,"name":"offline","context":{"idset":"8630"}}
+{"timestamp":1714181748.8323271,"name":"offline","context":{"idset":"8665"}}
+{"timestamp":1714181748.8364139,"name":"offline","context":{"idset":"8642"}}
+{"timestamp":1714181748.8405905,"name":"offline","context":{"idset":"8677"}}
+{"timestamp":1714181748.8451867,"name":"offline","context":{"idset":"8655"}}
+{"timestamp":1714181748.8497567,"name":"offline","context":{"idset":"8634"}}
+{"timestamp":1714181748.8541594,"name":"offline","context":{"idset":"8638"}}
+{"timestamp":1714181748.858058,"name":"offline","context":{"idset":"8660"}}
+{"timestamp":1714181748.8621609,"name":"offline","context":{"idset":"8648"}}
+{"timestamp":1714181748.8663065,"name":"offline","context":{"idset":"8636"}}
+{"timestamp":1714181748.8708112,"name":"offline","context":{"idset":"8650"}}
+{"timestamp":1714181748.8754005,"name":"offline","context":{"idset":"8661"}}
+{"timestamp":1714181748.8800364,"name":"offline","context":{"idset":"8656"}}
+{"timestamp":1714181748.884551,"name":"offline","context":{"idset":"8644"}}
+{"timestamp":1714181748.8891323,"name":"offline","context":{"idset":"8676"}}
+{"timestamp":1714181748.8934889,"name":"offline","context":{"idset":"8663"}}
+{"timestamp":1714181748.8974419,"name":"offline","context":{"idset":"8671"}}
+{"timestamp":1714181748.901459,"name":"offline","context":{"idset":"8686"}}
+{"timestamp":1714181748.9056756,"name":"offline","context":{"idset":"8652"}}
+{"timestamp":1714181748.9098525,"name":"offline","context":{"idset":"8680"}}
+{"timestamp":1714181748.9139049,"name":"offline","context":{"idset":"8649"}}
+{"timestamp":1714181748.9181693,"name":"offline","context":{"idset":"8678"}}
+{"timestamp":1714181748.9223225,"name":"offline","context":{"idset":"8654"}}
+{"timestamp":1714181748.9267733,"name":"offline","context":{"idset":"8658"}}
+{"timestamp":1714182370.9328117,"name":"online","context":{"idset":"8673"}}
+{"timestamp":1714182371.4526474,"name":"online","context":{"idset":"8619"}}
+{"timestamp":1714182371.6220696,"name":"online","context":{"idset":"8584"}}
+{"timestamp":1714182372.5229897,"name":"online","context":{"idset":"8681"}}
+{"timestamp":1714182373.2121804,"name":"online","context":{"idset":"8646"}}
+{"timestamp":1714182374.3883312,"name":"online","context":{"idset":"8624"}}
+{"timestamp":1714182374.6751006,"name":"online","context":{"idset":"8684"}}
+{"timestamp":1714182374.9677439,"name":"online","context":{"idset":"8686"}}
+{"timestamp":1714182375.1048973,"name":"online","context":{"idset":"8591"}}
+{"timestamp":1714182375.2523081,"name":"online","context":{"idset":"8642"}}
+{"timestamp":1714182375.6703451,"name":"online","context":{"idset":"8611"}}
+{"timestamp":1714182376.4149754,"name":"online","context":{"idset":"8683"}}
+{"timestamp":1714182376.419193,"name":"online","context":{"idset":"8647"}}
+{"timestamp":1714182376.4233773,"name":"online","context":{"idset":"8628"}}
+{"timestamp":1714182376.6292801,"name":"online","context":{"idset":"8575,8610,8612"}}
+{"timestamp":1714182376.8980036,"name":"online","context":{"idset":"8649,8664"}}
+{"timestamp":1714182377.0505049,"name":"online","context":{"idset":"8583"}}
+{"timestamp":1714182377.0615869,"name":"online","context":{"idset":"8678"}}
+{"timestamp":1714182377.167763,"name":"online","context":{"idset":"8627"}}
+{"timestamp":1714182377.3123479,"name":"online","context":{"idset":"8565,8588,8590"}}
+{"timestamp":1714182377.4395001,"name":"online","context":{"idset":"8568,8593,8660"}}
+{"timestamp":1714182377.6381936,"name":"online","context":{"idset":"8566,8589"}}
+{"timestamp":1714182378.5373473,"name":"online","context":{"idset":"8669,8692"}}
+{"timestamp":1714182378.5419624,"name":"online","context":{"idset":"8576"}}
+{"timestamp":1714182378.5459955,"name":"online","context":{"idset":"8605,8630"}}
+{"timestamp":1714182378.5503774,"name":"online","context":{"idset":"8638"}}
+{"timestamp":1714182378.5546167,"name":"online","context":{"idset":"8580,8671"}}
+{"timestamp":1714182378.5592706,"name":"online","context":{"idset":"8573"}}
+{"timestamp":1714182378.5633378,"name":"online","context":{"idset":"8578"}}
+{"timestamp":1714182378.5676534,"name":"online","context":{"idset":"8571,8599,8629,8662"}}
+{"timestamp":1714182378.5717669,"name":"online","context":{"idset":"8595"}}
+{"timestamp":1714182378.575917,"name":"online","context":{"idset":"8625"}}
+{"timestamp":1714182378.7876799,"name":"online","context":{"idset":"8604,8609,8645,8661"}}
+{"timestamp":1714182379.134572,"name":"online","context":{"idset":"8614,8653"}}
+{"timestamp":1714182379.3369243,"name":"online","context":{"idset":"8582,8603,8608,8623,8648"}}
+{"timestamp":1714182379.5098326,"name":"online","context":{"idset":"8574,8633,8635"}}
+{"timestamp":1714182379.7189744,"name":"online","context":{"idset":"8598,8601,8606,8618,8641,8666-8667"}}
+{"timestamp":1714182380.6091549,"name":"online","context":{"idset":"8579,8636,8654"}}
+{"timestamp":1714182380.614929,"name":"online","context":{"idset":"8631-8632,8655,8689"}}
+{"timestamp":1714182380.6201258,"name":"online","context":{"idset":"8592,8600,8607,8613,8644,8652,8675"}}
+{"timestamp":1714182380.6252096,"name":"online","context":{"idset":"8572,8585,8685"}}
+{"timestamp":1714182380.630296,"name":"online","context":{"idset":"8569-8570,8651"}}
+{"timestamp":1714182380.6343164,"name":"online","context":{"idset":"8602,8617,8637,8640,8650"}}
+{"timestamp":1714182380.8613412,"name":"online","context":{"idset":"8567,8577,8581,8597,8615,8622,8626,8643,8677"}}
+{"timestamp":1714182381.0103905,"name":"online","context":{"idset":"8639,8659"}}
+{"timestamp":1714182381.2184441,"name":"online","context":{"idset":"8657"}}
+{"timestamp":1714182381.339432,"name":"online","context":{"idset":"8658,8680"}}
+{"timestamp":1714182381.5478334,"name":"online","context":{"idset":"8586-8587,8616,8620,8656,8670,8682,8687"}}
+{"timestamp":1714182383.7549915,"name":"online","context":{"idset":"8594,8596,8634,8679,8688,8691"}}
+{"timestamp":1714182383.7580612,"name":"online","context":{"idset":"8621,8665,8672,8690"}}
+{"timestamp":1714182383.761287,"name":"online","context":{"idset":"8676"}}
+{"timestamp":1714182383.7642846,"name":"online","context":{"idset":"8663"}}
+{"timestamp":1714182383.7672656,"name":"online","context":{"idset":"8668"}}
+{"timestamp":1714182384.0214074,"name":"online","context":{"idset":"8674"}}
+{"timestamp":1714182563.6243474,"name":"offline","context":{"idset":"9180"}}
+{"timestamp":1714184659.2706714,"name":"resource-init","context":{"restart":true,"drain":{"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"98,100,102,111-112,114,123-124,161,184,187,190,208,303-304,310,317,319,330":{"timestamp":1714079956.0,"reason":"epilog failed for jobid frcnKxdu6xP"},"543":{"timestamp":1714074108.0,"reason":"nodediag failed cxi"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"796":{"timestamp":1712868054.0,"reason":"broker was unresponsive"},"119":{"timestamp":1714079565.0,"reason":"unreachable"},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"},"795":{"timestamp":1712868053.0,"reason":"broker was unresponsive"},"372":{"timestamp":1713981581.0,"reason":"New Blade Installation --JRG"},"336,491":{"timestamp":1713230524.0,"reason":"Down"},"929-930":{"timestamp":1713552018.0,"reason":""},"773-774":{"timestamp":1713802805.0,"reason":""},"77-84":{"timestamp":1713795613.0,"reason":"New blade installation"},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"10186-10187,10203,10205-10206,10208-10211,10215":{"timestamp":1714152041.0,"reason":"broker was unresponsive"},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"115":{"timestamp":1714079493.0,"reason":"unreachable"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"117":{"timestamp":1713981538.0,"reason":"Unreachable"},"360":{"timestamp":1713981579.0,"reason":"New Blade Installation --JRG"},"411":{"timestamp":1713564989.0,"reason":"Unreachable"},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"807":{"timestamp":1713376017.0,"reason":""},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"811-812":{"timestamp":1713557753.0,"reason":"--reason Running hpe diags - JM"},"779-780":{"timestamp":1713907642.0,"reason":"--reason Troubleshoot BIOS issue"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"881-882":{"timestamp":1714153692.0,"reason":"--reason H/W troubleshoot"},"286":{"timestamp":1713981553.0,"reason":"Unreachable"},"808":{"timestamp":1713455939.0,"reason":""},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"971-974":{"timestamp":1714143211.0,"reason":""},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"110":{"timestamp":1714079561.0,"reason":"unreachable"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"544":{"timestamp":1714069881.0,"reason":"nodediag failed cxi"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"206":{"timestamp":1713564985.0,"reason":"Unreachable"},"349-359,361-371,374-396,398-404":{"timestamp":1714060395.0,"reason":"New Blade Installation --JRG"},"8181,8183,8234,8237,8291,8304":{"timestamp":1713810085.0,"reason":"testing -kk"},"545":{"timestamp":1714069878.0,"reason":"nodediag failed cxi"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"878":{"timestamp":1714078484.0,"reason":"nodediag failed amdapu"},"141-148":{"timestamp":1713545597.0,"reason":"ARP Storm Problem testing"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"9075":{"timestamp":1712864130.0,"reason":"broker was unresponsive"},"542":{"timestamp":1714074109.0,"reason":"nodediag failed cxi"},"573-580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"149-156":{"timestamp":1713545634.0,"reason":"ARP Storm Problem testing"},"165":{"timestamp":1713981434.0,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261863.0,"reason":"--reason swapping blades into x1900 - wendy"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"172":{"timestamp":1713981544.0,"reason":"Unreachable"},"373":{"timestamp":1713981698.0,"reason":"New Blade Installation --JRG"},"541":{"timestamp":1714074106.0,"reason":"nodediag failed cxi"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"787-788":{"timestamp":1713219302.0,"reason":""},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"85-89,91-92":{"timestamp":1714079045.0,"reason":"New Blade installation"},"963-964":{"timestamp":1713196418.0,"reason":"moving blade"},"937-938":{"timestamp":1712866351.0,"reason":""},"925-926":{"timestamp":1713540640.0,"reason":""},"461-476":{"timestamp":1713794797.0,"reason":"New Blade Installation --JRG"},"397":{"timestamp":1713229804.0,"reason":"New Blade Installation --JRG"},"9003":{"timestamp":1712873964.0,"reason":"nodediag failed dgemm_perf"},"421-460":{"timestamp":1713794410.0,"reason":"New Blade Installation --JRG"},"410":{"timestamp":1713378635.0,"reason":"prolog failed for jobid fq5jtQQSQnT"},"120":{"timestamp":1713981548.0,"reason":"Unreachable"},"803-804":{"timestamp":1713479733.0,"reason":"status"},"789-794":{"timestamp":1711461777.0,"reason":""},"191":{"timestamp":1714078824.0,"reason":"unreachable"},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"961-962":{"timestamp":1713197674.0,"reason":""},"482-490,492":{"timestamp":1713915346.0,"reason":"New Blade Installation --JRG"},"342":{"timestamp":1713981566.0,"reason":"Unreachable"},"923-924":{"timestamp":1714064541.0,"reason":"status"},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"546":{"timestamp":1714069873.0,"reason":"nodediag failed cxi pci"},"825-826":{"timestamp":1714153531.0,"reason":"--reason H/W troubleshoot"},"977-978":{"timestamp":1714144603.0,"reason":""},"549-556":{"timestamp":1713553592.0,"reason":"New Blade Installation"},"8833":{"timestamp":1713991869.0,"reason":"nodediag failed amdapu"},"90":{"timestamp":1713564981.0,"reason":"New Blade installation"},"9054":{"timestamp":1712864116.0,"reason":"broker was unresponsive"},"771-772":{"timestamp":1713195979.0,"reason":""},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"877":{"timestamp":1714078512.0,"reason":"nodediag failed amdapu"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"8982":{"timestamp":1713996208.0,"reason":"broker was unresponsive"},"939":{"timestamp":1713963885.0,"reason":"nodediag failed amdapu"},"547":{"timestamp":1713563916.0,"reason":"epilog failed for jobid fqUWmieM3AX"},"420":{"timestamp":1714074112.0,"reason":"epilog failed for jobid frbYvBLD1gT"},"814":{"timestamp":1712783332.0,"reason":""},"871-874":{"timestamp":1714057817.0,"reason":""},"829-830":{"timestamp":1712700945.0,"reason":""},"133-140":{"timestamp":1713545617.0,"reason":"ARP Storm Problem testing"},"525-532":{"timestamp":1713290620.0,"reason":""},"548":{"timestamp":1713563915.0,"reason":"epilog failed for jobid fqUWkS8j8hM"},"335":{"timestamp":1713981557.0,"reason":"Unreachable"},"934":{"timestamp":1713967835.0,"reason":"epilog failed for jobid frMYzSv7G6b"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1714184659.2786548,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1714184677.9361732,"name":"online","context":{"idset":"0"}}
+{"timestamp":1714184685.634603,"name":"online","context":{"idset":"11515"}}
+{"timestamp":1714184685.7172148,"name":"online","context":{"idset":"9811"}}
+{"timestamp":1714184685.7741592,"name":"online","context":{"idset":"10362,11800,11847"}}
+{"timestamp":1714184685.7758327,"name":"online","context":{"idset":"8065"}}
+{"timestamp":1714184685.7815425,"name":"online","context":{"idset":"9452"}}
+{"timestamp":1714184685.831389,"name":"online","context":{"idset":"8142"}}
+{"timestamp":1714184685.8526344,"name":"online","context":{"idset":"7912,8651,9391,10226,11621,11755"}}
+{"timestamp":1714184685.9153883,"name":"online","context":{"idset":"7885,8307,8384,9249,9317-9318,9334,9740,9858,9880,10360,10918,11188,11198,11356,11369,11640,11768"}}
+{"timestamp":1714184685.9168315,"name":"online","context":{"idset":"9805"}}
+{"timestamp":1714184685.9182091,"name":"online","context":{"idset":"9314"}}
+{"timestamp":1714184685.9228418,"name":"online","context":{"idset":"9838"}}
+{"timestamp":1714184685.9279406,"name":"online","context":{"idset":"10769"}}
+{"timestamp":1714184685.9824295,"name":"online","context":{"idset":"8649,8830,8862,8957,9765,10765,10910,10914,11119"}}
+{"timestamp":1714184685.9833241,"name":"online","context":{"idset":"11828"}}
+{"timestamp":1714184685.9859304,"name":"online","context":{"idset":"7825"}}
+{"timestamp":1714184685.9947548,"name":"online","context":{"idset":"8072,11646"}}
+{"timestamp":1714184686.0387099,"name":"online","context":{"idset":"9371,9516,9946,9960"}}
+{"timestamp":1714184686.0590794,"name":"online","context":{"idset":"7779,7837,7891,8143,8171,8241,8334,8382,8897,9246,9319,9342,9451,9505,9513,9522,9581,10119,10472,11557,11698,11725,11737,11785,11842,11854"}}
+{"timestamp":1714184686.0955708,"name":"online","context":{"idset":"11630,11663,11778,11834"}}
+{"timestamp":1714184686.1388626,"name":"online","context":{"idset":"1205,7836,7886,8013,8092,8094,8098,8126,8242,8299,8336,8389,8401,8429,8565,8934,9485,9731,9809,9962,10382,10411,10776,10993,11115,11153,11185"}}
+{"timestamp":1714184686.1731989,"name":"online","context":{"idset":"8875,8979,9010,9416,9966,10147,10153,10173,10413,10811,10836,10870,11155"}}
+{"timestamp":1714184686.1739068,"name":"online","context":{"idset":"9528"}}
+{"timestamp":1714184686.18679,"name":"online","context":{"idset":"6561,6620,7355,7773,7782,7795,7838,7869,7895,7901,8079,8090,9228,9393,9945,10165,10441,11032,11096,11547,11581,11625"}}
+{"timestamp":1714184686.2426617,"name":"online","context":{"idset":"877,6579,7316,7766,7789,8063,8156,8265,8289,8892,8895,8930,9261,9306,9316,9347,9351-9352,9372,9484,9525,9717,9781,9832,9850,9947,10113,10155,10161,10398,10409-10410,10421,10451,10775,10943,10946,10984,10989,11006,11063,11140,11178,11311,11316,11373,11553,11754,11820,11868"}}
+{"timestamp":1714184686.2443871,"name":"online","context":{"idset":"1226,8424,10820,10833,11073"}}
+{"timestamp":1714184686.2461231,"name":"online","context":{"idset":"6551,8113,10452"}}
+{"timestamp":1714184686.2477019,"name":"online","context":{"idset":"11121"}}
+{"timestamp":1714184686.2492471,"name":"online","context":{"idset":"7765,8413,10969"}}
+{"timestamp":1714184686.2500741,"name":"online","context":{"idset":"8086"}}
+{"timestamp":1714184686.2518463,"name":"online","context":{"idset":"8883"}}
+{"timestamp":1714184686.2526841,"name":"online","context":{"idset":"11321"}}
+{"timestamp":1714184686.2583189,"name":"online","context":{"idset":"7843,8107,8418,11681"}}
+{"timestamp":1714184686.3625257,"name":"online","context":{"idset":"1206,1222,1230,6528,6582,7290,7308,7314,7383,7846,7920,7965,8055,8110,8115-8116,8293,8390,8419,8653,9024,9254,9283,9328,9360,9399,9406,9426,9445,9448,9460,9530,9544,9569,9723,9802,9815,9825,9837,9860,9867,9894,10148,10422,10464-10465,10756,10830,10875,10921,10928,10959,10968,11084,11128,11295,11337,11361,11521,11559,11604,11680,11701,11742"}}
+{"timestamp":1714184686.363142,"name":"online","context":{"idset":"6594"}}
+{"timestamp":1714184686.3643916,"name":"online","context":{"idset":"8869,9561"}}
+{"timestamp":1714184686.3649578,"name":"online","context":{"idset":"9216,9844,11511"}}
+{"timestamp":1714184686.3655465,"name":"online","context":{"idset":"7854"}}
+{"timestamp":1714184686.3661261,"name":"online","context":{"idset":"9449,10788"}}
+{"timestamp":1714184686.3667657,"name":"online","context":{"idset":"8283,9763,9921"}}
+{"timestamp":1714184686.3673384,"name":"online","context":{"idset":"10893"}}
+{"timestamp":1714184686.3680174,"name":"online","context":{"idset":"8058,8672,9325,9461,9565,10915,11235"}}
+{"timestamp":1714184686.3686848,"name":"online","context":{"idset":"1182"}}
+{"timestamp":1714184686.369303,"name":"online","context":{"idset":"7864,8119,11194,11328"}}
+{"timestamp":1714184686.3698237,"name":"online","context":{"idset":"7805,7845"}}
+{"timestamp":1714184686.3703277,"name":"online","context":{"idset":"11543"}}
+{"timestamp":1714184686.3709714,"name":"online","context":{"idset":"11110"}}
+{"timestamp":1714184686.3718398,"name":"online","context":{"idset":"8901,10469"}}
+{"timestamp":1714184686.3756306,"name":"online","context":{"idset":"11580"}}
+{"timestamp":1714184686.5561106,"name":"online","context":{"idset":"8571,10773,10855"}}
+{"timestamp":1714184686.5567706,"name":"online","context":{"idset":"8353,8633"}}
+{"timestamp":1714184686.5625498,"name":"online","context":{"idset":"1189,6595,6631,6638,6640,7291,7354,7817,7863,7897,7915,8179,8207,8238,8301,8321,8342,8586,8626,8863,8913,8971,9263,9289,9348,9401,9424,9471,9475,9580,9779,9791,9819,9868,9964-9965,10106,10159,10763,10821-10822,10876,11013,11294,11532,11653,11774,11816"}}
+{"timestamp":1714184686.5633633,"name":"online","context":{"idset":"9255,10388"}}
+{"timestamp":1714184686.564152,"name":"online","context":{"idset":"9224,9250,9533,11175"}}
+{"timestamp":1714184686.5649688,"name":"online","context":{"idset":"9221,9398,11272"}}
+{"timestamp":1714184686.5658791,"name":"online","context":{"idset":"11122,11835"}}
+{"timestamp":1714184686.5666411,"name":"online","context":{"idset":"8640,8860,10111,10900,10932"}}
+{"timestamp":1714184686.5674303,"name":"online","context":{"idset":"6608"}}
+{"timestamp":1714184686.5681462,"name":"online","context":{"idset":"10112"}}
+{"timestamp":1714184686.5689247,"name":"online","context":{"idset":"7315,9213,10373,11157,11203,11784"}}
+{"timestamp":1714184686.5696332,"name":"online","context":{"idset":"9257,10911"}}
+{"timestamp":1714184686.5702479,"name":"online","context":{"idset":"10433,10780"}}
+{"timestamp":1714184686.5709102,"name":"online","context":{"idset":"8674"}}
+{"timestamp":1714184686.5715373,"name":"online","context":{"idset":"8062,8187,8208,10460,10891"}}
+{"timestamp":1714184686.5722649,"name":"online","context":{"idset":"8263"}}
+{"timestamp":1714184686.5730057,"name":"online","context":{"idset":"11607"}}
+{"timestamp":1714184686.573822,"name":"online","context":{"idset":"8567"}}
+{"timestamp":1714184686.5744486,"name":"online","context":{"idset":"8269,8352"}}
+{"timestamp":1714184686.575608,"name":"online","context":{"idset":"9750"}}
+{"timestamp":1714184686.5771804,"name":"online","context":{"idset":"7913"}}
+{"timestamp":1714184686.579011,"name":"online","context":{"idset":"9969"}}
+{"timestamp":1714184686.5807643,"name":"online","context":{"idset":"9368"}}
+{"timestamp":1714184686.5815766,"name":"online","context":{"idset":"185,195,205,233,10157"}}
+{"timestamp":1714184686.5826149,"name":"online","context":{"idset":"10704"}}
+{"timestamp":1714184686.5835745,"name":"online","context":{"idset":"10886,11343"}}
+{"timestamp":1714184686.5848315,"name":"online","context":{"idset":"11353,11713"}}
+{"timestamp":1714184686.5860577,"name":"online","context":{"idset":"6580,7390,7829,11564,11612"}}
+{"timestamp":1714184686.5869823,"name":"online","context":{"idset":"6617,11182"}}
+{"timestamp":1714184686.587836,"name":"online","context":{"idset":"8235"}}
+{"timestamp":1714184686.5884802,"name":"online","context":{"idset":"10972"}}
+{"timestamp":1714184686.5893979,"name":"online","context":{"idset":"7883,8219,11802"}}
+{"timestamp":1714184686.5901585,"name":"online","context":{"idset":"8319,8846,8864"}}
+{"timestamp":1714184686.5910194,"name":"online","context":{"idset":"7908,8922"}}
+{"timestamp":1714184686.5917821,"name":"online","context":{"idset":"8010"}}
+{"timestamp":1714184686.5925314,"name":"online","context":{"idset":"8585"}}
+{"timestamp":1714184686.5932076,"name":"online","context":{"idset":"9071,11127"}}
+{"timestamp":1714184686.5939608,"name":"online","context":{"idset":"9291"}}
+{"timestamp":1714184686.5951257,"name":"online","context":{"idset":"93,203,224,8569,8636"}}
+{"timestamp":1714184686.5958326,"name":"online","context":{"idset":"112,330"}}
+{"timestamp":1714184686.5966489,"name":"online","context":{"idset":"96,227"}}
+{"timestamp":1714184686.5977724,"name":"online","context":{"idset":"8657"}}
+{"timestamp":1714184686.5993629,"name":"online","context":{"idset":"8279"}}
+{"timestamp":1714184686.6004095,"name":"online","context":{"idset":"9057"}}
+{"timestamp":1714184686.6012366,"name":"online","context":{"idset":"9474,9558"}}
+{"timestamp":1714184686.6025677,"name":"online","context":{"idset":"8903,9772"}}
+{"timestamp":1714184686.6035879,"name":"online","context":{"idset":"8259,9536,9801"}}
+{"timestamp":1714184686.6042953,"name":"online","context":{"idset":"9362,10101,10140"}}
+{"timestamp":1714184686.6050947,"name":"online","context":{"idset":"9521,9758,9820"}}
+{"timestamp":1714184686.6060586,"name":"online","context":{"idset":"9321,10204"}}
+{"timestamp":1714184686.6069283,"name":"online","context":{"idset":"9387"}}
+{"timestamp":1714184686.607794,"name":"online","context":{"idset":"10328"}}
+{"timestamp":1714184686.6087539,"name":"online","context":{"idset":"10934"}}
+{"timestamp":1714184686.6096621,"name":"online","context":{"idset":"11211"}}
+{"timestamp":1714184687.7519033,"name":"online","context":{"idset":"94"}}
+{"timestamp":1714184687.7565441,"name":"online","context":{"idset":"106,119,126,158,168,174,194,219,223,229,238,248,252,277,343,408,900,6524,6534,6541,6543,6568,6584,6601,7302,7307,7328,7332,7337,7368,7370,7372,7381,7384,7403,7737,7778,7816,7857,7861,7866,7873-7874,7887,7903,7951,8000,8016,8022-8023,8034,8040,8217,8254,8278,8392,8397,8432,8574,8645,8682,8872,8877,8935,8952,8958,8967,9073,9084,9125,9212,9268,9281,9303,9494,9527,9739,9753,9767,9771,9775,9793,9849,9891,9923,9949,9963,10102,10127,10186,10211,10224,10228,10258,10263,10365,10468,10644,10806,10896,10908,10922,10975,11033,11104-11105,11123,11143,11164,11169,11242,11320,11423,11434,11533,11546,11577,11618,11658,11661,11672,11685,11702,11718,11733-11734,11738,11749,11781,11804,11832,11850"}}
+{"timestamp":1714184687.7572336,"name":"online","context":{"idset":"10938"}}
+{"timestamp":1714184687.7577181,"name":"online","context":{"idset":"1223,8665"}}
+{"timestamp":1714184687.7583671,"name":"online","context":{"idset":"6552,6632,7340,8630,9845,10448,10912,11041,11078"}}
+{"timestamp":1714184687.7589674,"name":"online","context":{"idset":"7744,7769"}}
+{"timestamp":1714184687.7594337,"name":"online","context":{"idset":"7862"}}
+{"timestamp":1714184687.7599926,"name":"online","context":{"idset":"8052,9575"}}
+{"timestamp":1714184687.7606094,"name":"online","context":{"idset":"415,8610,8616,9887,10331,11193"}}
+{"timestamp":1714184687.761235,"name":"online","context":{"idset":"8629,10414"}}
+{"timestamp":1714184687.7618136,"name":"online","context":{"idset":"207,289"}}
+{"timestamp":1714184687.7623651,"name":"online","context":{"idset":"9156,10397,10476,10633,10710,10869,11019,11249,11584-11585"}}
+{"timestamp":1714184687.7629418,"name":"online","context":{"idset":"102,115"}}
+{"timestamp":1714184687.7659533,"name":"online","context":{"idset":"298,338,579,1208,1221,1233,6564,6586,6591,6606,6616,7289,7405,7794,7841,8120,8314,8588,8593,8627,8637,8655-8656,8667,8680,8831,8865,8874,8899,8992,8997,9027,9053,9173,9176,9218,9226,9277,9312,9329,9344,9369-9370,9435,9481,9489,9523,9534,9762,9780,9803,9883,10107,10172,10185,10223,10255,10293,10302,10317,10340,10349,10370,10405,10408,10627,10673,10746,10857,10909,10926,10960,11016-11017,11031,11051,11056,11061,11093,11116,11246,11306,11346,11359,11526,11534,11538,11593,11611,11615,11648,11674,11688,11732,11764,11775,11779"}}
+{"timestamp":1714184687.7666786,"name":"online","context":{"idset":"8427,8614,11162,11651,11857"}}
+{"timestamp":1714184687.767283,"name":"online","context":{"idset":"6578"}}
+{"timestamp":1714184687.7678764,"name":"online","context":{"idset":"8577,11301"}}
+{"timestamp":1714184687.7686331,"name":"online","context":{"idset":"11133"}}
+{"timestamp":1714184687.7694197,"name":"online","context":{"idset":"8368,11433"}}
+{"timestamp":1714184687.7701333,"name":"online","context":{"idset":"11083"}}
+{"timestamp":1714184687.7707355,"name":"online","context":{"idset":"9121,9787"}}
+{"timestamp":1714184687.7714105,"name":"online","context":{"idset":"9464,11827"}}
+{"timestamp":1714184687.772038,"name":"online","context":{"idset":"11831"}}
+{"timestamp":1714184687.7725773,"name":"online","context":{"idset":"9748"}}
+{"timestamp":1714184687.773175,"name":"online","context":{"idset":"10377"}}
+{"timestamp":1714184687.7739053,"name":"online","context":{"idset":"9884"}}
+{"timestamp":1714184687.774611,"name":"online","context":{"idset":"7774,8220,9016,10312,11381"}}
+{"timestamp":1714184687.7754908,"name":"online","context":{"idset":"8196"}}
+{"timestamp":1714184687.7761157,"name":"online","context":{"idset":"8940"}}
+{"timestamp":1714184687.776685,"name":"online","context":{"idset":"11767"}}
+{"timestamp":1714184687.7773247,"name":"online","context":{"idset":"7371"}}
+{"timestamp":1714184687.7780097,"name":"online","context":{"idset":"287,11023"}}
+{"timestamp":1714184687.7786813,"name":"online","context":{"idset":"7378,11362,11645"}}
+{"timestamp":1714184687.7792487,"name":"online","context":{"idset":"11079"}}
+{"timestamp":1714184687.7798865,"name":"online","context":{"idset":"9841"}}
+{"timestamp":1714184687.780582,"name":"online","context":{"idset":"7946,9487,10702"}}
+{"timestamp":1714184687.7814701,"name":"online","context":{"idset":"11002"}}
+{"timestamp":1714184687.7823868,"name":"online","context":{"idset":"11490"}}
+{"timestamp":1714184687.782948,"name":"online","context":{"idset":"9091"}}
+{"timestamp":1714184687.7837834,"name":"online","context":{"idset":"9472"}}
+{"timestamp":1714184687.7860341,"name":"online","context":{"idset":"8847,9389,10286,10781"}}
+{"timestamp":1714184687.788491,"name":"online","context":{"idset":"7374,11509"}}
+{"timestamp":1714184687.7890713,"name":"online","context":{"idset":"8111"}}
+{"timestamp":1714184687.7898104,"name":"online","context":{"idset":"8306"}}
+{"timestamp":1714184687.7903802,"name":"online","context":{"idset":"7783"}}
+{"timestamp":1714184687.7910523,"name":"online","context":{"idset":"6553"}}
+{"timestamp":1714184687.791657,"name":"online","context":{"idset":"1187,9727,11357"}}
+{"timestamp":1714184687.7923689,"name":"online","context":{"idset":"7318,7324,8139,9724"}}
+{"timestamp":1714184687.7929947,"name":"online","context":{"idset":"7894,9870"}}
+{"timestamp":1714184687.7936366,"name":"online","context":{"idset":"8327,9497,11011,11049,11099"}}
+{"timestamp":1714184687.7942114,"name":"online","context":{"idset":"6612"}}
+{"timestamp":1714184687.7948728,"name":"online","context":{"idset":"7363,7954,8302,8978,9358,11132"}}
+{"timestamp":1714184687.7955267,"name":"online","context":{"idset":"6547,8182,11010,11620"}}
+{"timestamp":1714184687.7961702,"name":"online","context":{"idset":"6517,9510"}}
+{"timestamp":1714184687.796751,"name":"online","context":{"idset":"8841,9892,11352"}}
+{"timestamp":1714184687.7974691,"name":"online","context":{"idset":"9272,9274,9486,9938,11208"}}
+{"timestamp":1714184687.7980428,"name":"online","context":{"idset":"11377,11379"}}
+{"timestamp":1714184687.7986839,"name":"online","context":{"idset":"9204,9878,11550"}}
+{"timestamp":1714184687.7993281,"name":"online","context":{"idset":"7826,8972,10131,10345"}}
+{"timestamp":1714184687.799922,"name":"online","context":{"idset":"8093,8621,9428,10169,11290,11578"}}
+{"timestamp":1714184687.8005967,"name":"online","context":{"idset":"6628"}}
+{"timestamp":1714184687.8012462,"name":"online","context":{"idset":"1179,7871,8664,9414,10865,11739"}}
+{"timestamp":1714184687.8019264,"name":"online","context":{"idset":"7859,9108,10944,11238"}}
+{"timestamp":1714184687.8025994,"name":"online","context":{"idset":"7798"}}
+{"timestamp":1714184687.8031993,"name":"online","context":{"idset":"8361,8631,9818,11298,11750,11797"}}
+{"timestamp":1714184687.8038337,"name":"online","context":{"idset":"9931,11696"}}
+{"timestamp":1714184687.8044403,"name":"online","context":{"idset":"9307,10998,11457"}}
+{"timestamp":1714184687.8050747,"name":"online","context":{"idset":"11720"}}
+{"timestamp":1714184687.8057685,"name":"online","context":{"idset":"8584,10757,11076"}}
+{"timestamp":1714184687.8064091,"name":"online","context":{"idset":"10149"}}
+{"timestamp":1714184687.8073297,"name":"online","context":{"idset":"8905,10482"}}
+{"timestamp":1714184687.8080077,"name":"online","context":{"idset":"316,8226,8900,9028,9147,9219"}}
+{"timestamp":1714184687.8086188,"name":"online","context":{"idset":"8950,10457"}}
+{"timestamp":1714184687.8092611,"name":"online","context":{"idset":"8840,9230,10304"}}
+{"timestamp":1714184687.8099127,"name":"online","context":{"idset":"8881,9286"}}
+{"timestamp":1714184687.8106515,"name":"online","context":{"idset":"8878,9181,9896,9901"}}
+{"timestamp":1714184687.8113742,"name":"online","context":{"idset":"10434,10680,10737"}}
+{"timestamp":1714184687.8119936,"name":"online","context":{"idset":"9094"}}
+{"timestamp":1714184687.812726,"name":"online","context":{"idset":"9350"}}
+{"timestamp":1714184687.8134396,"name":"online","context":{"idset":"9468"}}
+{"timestamp":1714184687.8141127,"name":"online","context":{"idset":"9408"}}
+{"timestamp":1714184687.8147392,"name":"online","context":{"idset":"9958"}}
+{"timestamp":1714184687.8155236,"name":"online","context":{"idset":"9158,10795"}}
+{"timestamp":1714184687.8162246,"name":"online","context":{"idset":"7923,8387"}}
+{"timestamp":1714184687.8168831,"name":"online","context":{"idset":"10307,10348,10848,10864"}}
+{"timestamp":1714184687.8175106,"name":"online","context":{"idset":"10936,11667"}}
+{"timestamp":1714184687.8181255,"name":"online","context":{"idset":"11310"}}
+{"timestamp":1714184687.8188734,"name":"online","context":{"idset":"6605,7312,7819,10703,10971,11144,11241,11579,11757"}}
+{"timestamp":1714184687.8196089,"name":"online","context":{"idset":"11447,11776"}}
+{"timestamp":1714184687.820266,"name":"online","context":{"idset":"11082"}}
+{"timestamp":1714184687.820935,"name":"online","context":{"idset":"7775"}}
+{"timestamp":1714184687.8232076,"name":"online","context":{"idset":"1176,7301,8117,8421,9919,9937,10839,11726,11763,11824,11849"}}
+{"timestamp":1714184687.823931,"name":"online","context":{"idset":"7767,8082,8409,9322,9446,9538,10142,10753,10948,11273,11476,11722"}}
+{"timestamp":1714184687.8247304,"name":"online","context":{"idset":"1183-1184,9295,9736,10216,10846"}}
+{"timestamp":1714184687.8253894,"name":"online","context":{"idset":"7382,8395,8399,8917,11248,11465"}}
+{"timestamp":1714184687.8261042,"name":"online","context":{"idset":"6615,7898,8322,10183,10718,11029,11700"}}
+{"timestamp":1714184687.826798,"name":"online","context":{"idset":"9324,9807"}}
+{"timestamp":1714184687.8274567,"name":"online","context":{"idset":"10124,11571"}}
+{"timestamp":1714184687.8281691,"name":"online","context":{"idset":"6609,7338,7849,8245,8280,8669,9144,9169,10813,11282,11441"}}
+{"timestamp":1714184687.8288436,"name":"online","context":{"idset":"7808,8122,8132,8359,8832,9941,10681,11315,11432,11473,11525"}}
+{"timestamp":1714184687.8296154,"name":"online","context":{"idset":"200,284,309,7369,7902,8140,8356,8403,8572,8595,8624,8849,9476,9888,10128,10160,10241,10381,10642,10818,11015,11107,11223,11234,11410,11510,11545"}}
+{"timestamp":1714184687.8303602,"name":"online","context":{"idset":"8639,11469"}}
+{"timestamp":1714184687.8311186,"name":"online","context":{"idset":"10619,11398"}}
+{"timestamp":1714184687.8318603,"name":"online","context":{"idset":"9285,9287,10117,10933,11106,11415,11665"}}
+{"timestamp":1714184687.8325546,"name":"online","context":{"idset":"7831,9913,10799"}}
+{"timestamp":1714184687.833339,"name":"online","context":{"idset":"8026,9264,10337,10645"}}
+{"timestamp":1714184687.8340735,"name":"online","context":{"idset":"10143,10629"}}
+{"timestamp":1714184687.8349898,"name":"online","context":{"idset":"1211,7741,8931,9145,9297,10420,11058"}}
+{"timestamp":1714184687.8356924,"name":"online","context":{"idset":"8575,9127"}}
+{"timestamp":1714184687.8364656,"name":"online","context":{"idset":"8370,8685,10819"}}
+{"timestamp":1714184687.8371708,"name":"online","context":{"idset":"10719"}}
+{"timestamp":1714184687.8381293,"name":"online","context":{"idset":"10276"}}
+{"timestamp":1714184687.8389218,"name":"online","context":{"idset":"9139,10622"}}
+{"timestamp":1714184687.8396471,"name":"online","context":{"idset":"10713"}}
+{"timestamp":1714184687.8411851,"name":"online","context":{"idset":"8285,8309,10144,10741"}}
+{"timestamp":1714184687.8419759,"name":"online","context":{"idset":"6567,8033,8146,10894,11305"}}
+{"timestamp":1714184687.8426869,"name":"online","context":{"idset":"9873,11286"}}
+{"timestamp":1714184687.8434319,"name":"online","context":{"idset":"1225,6597,8247,8400,8423,9243,9506,9579,9865,11054,11270,11355,11370,11412"}}
+{"timestamp":1714184687.8443146,"name":"online","context":{"idset":"7921,8939,11050,11226,11405"}}
+{"timestamp":1714184687.8451033,"name":"online","context":{"idset":"9050,11340,11351,11614"}}
+{"timestamp":1714184687.8458753,"name":"online","context":{"idset":"1210,7917,8149,8963,10352,10354,10983,11092"}}
+{"timestamp":1714184687.8466904,"name":"online","context":{"idset":"8154,9070,10661"}}
+{"timestamp":1714184687.8475132,"name":"online","context":{"idset":"7393,7844,7907,9548"}}
+{"timestamp":1714184687.8482761,"name":"online","context":{"idset":"9551,9833,9926,10734"}}
+{"timestamp":1714184687.849041,"name":"online","context":{"idset":"8573,9133,10676,11068"}}
+{"timestamp":1714184687.8498478,"name":"online","context":{"idset":"8411,10278"}}
+{"timestamp":1714184687.8505921,"name":"online","context":{"idset":"8340,9574,11485"}}
+{"timestamp":1714184687.8513591,"name":"online","context":{"idset":"10621,10770"}}
+{"timestamp":1714184687.8521731,"name":"online","context":{"idset":"8030"}}
+{"timestamp":1714184687.8529077,"name":"online","context":{"idset":"11865"}}
+{"timestamp":1714184687.8539889,"name":"online","context":{"idset":"8014,11570,11771,11787"}}
+{"timestamp":1714184687.8551311,"name":"online","context":{"idset":"6542,8379,8632,8836,8937,8974,9171,10455,10735,10772,11080,11498,11501"}}
+{"timestamp":1714184687.8562167,"name":"online","context":{"idset":"7922"}}
+{"timestamp":1714184687.8573854,"name":"online","context":{"idset":"11845"}}
+{"timestamp":1714184687.8586493,"name":"online","context":{"idset":"6530"}}
+{"timestamp":1714184687.8599949,"name":"online","context":{"idset":"7888,10310,10816,10829"}}
+{"timestamp":1714184687.8611598,"name":"online","context":{"idset":"10987"}}
+{"timestamp":1714184687.8621714,"name":"online","context":{"idset":"9741"}}
+{"timestamp":1714184687.8630209,"name":"online","context":{"idset":"7870,10243,10252"}}
+{"timestamp":1714184687.8639233,"name":"online","context":{"idset":"9165"}}
+{"timestamp":1714184687.8647051,"name":"online","context":{"idset":"7367"}}
+{"timestamp":1714184687.8655696,"name":"online","context":{"idset":"7947,8376,9410"}}
+{"timestamp":1714184687.8664565,"name":"online","context":{"idset":"10982"}}
+{"timestamp":1714184687.8672235,"name":"online","context":{"idset":"11656"}}
+{"timestamp":1714184687.8679819,"name":"online","context":{"idset":"11765"}}
+{"timestamp":1714184687.868746,"name":"online","context":{"idset":"9407"}}
+{"timestamp":1714184687.8696649,"name":"online","context":{"idset":"1175,10825,10980"}}
+{"timestamp":1714184687.8704565,"name":"online","context":{"idset":"129,9128,10882,11478"}}
+{"timestamp":1714184687.871263,"name":"online","context":{"idset":"159,8230"}}
+{"timestamp":1714184687.8720548,"name":"online","context":{"idset":"108"}}
+{"timestamp":1714184687.8728862,"name":"online","context":{"idset":"9288"}}
+{"timestamp":1714184687.8737912,"name":"online","context":{"idset":"9150"}}
+{"timestamp":1714184687.8753042,"name":"online","context":{"idset":"7396"}}
+{"timestamp":1714184687.8761685,"name":"online","context":{"idset":"9898,11810"}}
+{"timestamp":1714184687.876966,"name":"online","context":{"idset":"8985"}}
+{"timestamp":1714184687.8777359,"name":"online","context":{"idset":"10114"}}
+{"timestamp":1714184687.8785853,"name":"online","context":{"idset":"7890,9500,9562,10141,10966,11225,11673"}}
+{"timestamp":1714184687.8793895,"name":"online","context":{"idset":"6623,8076,11666"}}
+{"timestamp":1714184687.8802052,"name":"online","context":{"idset":"8109,9512"}}
+{"timestamp":1714184687.8810115,"name":"online","context":{"idset":"7329,9120"}}
+{"timestamp":1714184687.8818245,"name":"online","context":{"idset":"8102,8105,8324,9546,9851"}}
+{"timestamp":1714184687.8826308,"name":"online","context":{"idset":"7865,9229,10336,11711,11806,11840"}}
+{"timestamp":1714184687.8834536,"name":"online","context":{"idset":"1209,7356,7780,8428,8433,8854,9111,9207,10271,11292,11589"}}
+{"timestamp":1714184687.8842955,"name":"online","context":{"idset":"1227,7978,9109,9427,9577,9890,10826,10925,11239,11586"}}
+{"timestamp":1714184687.8851321,"name":"online","context":{"idset":"826,7391,9498,9755,10973,11856"}}
+{"timestamp":1714184687.8859959,"name":"online","context":{"idset":"9578,10168,10828"}}
+{"timestamp":1714184687.8868845,"name":"online","context":{"idset":"8431,9381,10234,10401,10424,11201"}}
+{"timestamp":1714184687.8877034,"name":"online","context":{"idset":"8089,8295,8364,8568,8607,10275"}}
+{"timestamp":1714184687.8887997,"name":"online","context":{"idset":"8690,8999,9012,9570,10146,10693,11319"}}
+{"timestamp":1714184687.8900907,"name":"online","context":{"idset":"7410,8689,10793"}}
+{"timestamp":1714184687.8910859,"name":"online","context":{"idset":"7298,9187"}}
+{"timestamp":1714184687.891928,"name":"online","context":{"idset":"7856,9054,10777,11100"}}
+{"timestamp":1714184687.8927562,"name":"online","context":{"idset":"9149,10311,11325"}}
+{"timestamp":1714184687.8936706,"name":"online","context":{"idset":"9340,10727"}}
+{"timestamp":1714184687.8945847,"name":"online","context":{"idset":"249,1234,9203,11318"}}
+{"timestamp":1714184687.8954303,"name":"online","context":{"idset":"7872,9458,10115"}}
+{"timestamp":1714184687.89627,"name":"online","context":{"idset":"11636"}}
+{"timestamp":1714184687.8971953,"name":"online","context":{"idset":"10251,10841,11796"}}
+{"timestamp":1714184687.8980331,"name":"online","context":{"idset":"10720"}}
+{"timestamp":1714184687.8989019,"name":"online","context":{"idset":"10880,11599"}}
+{"timestamp":1714184687.9003119,"name":"online","context":{"idset":"9774"}}
+{"timestamp":1714184687.9012239,"name":"online","context":{"idset":"7346,7770"}}
+{"timestamp":1714184687.9020624,"name":"online","context":{"idset":"10667,11200"}}
+{"timestamp":1714184687.9033113,"name":"online","context":{"idset":"7911,9493,10132,10437"}}
+{"timestamp":1714184687.9045134,"name":"online","context":{"idset":"9379"}}
+{"timestamp":1714184687.9054661,"name":"online","context":{"idset":"884,6533,9049,9056,9784,9834,9846,9948,10259,10458,10653,10849,11046"}}
+{"timestamp":1714184687.9064248,"name":"online","context":{"idset":"6636,7297,7939,8157,8990,9520,9554,10151,10178,10218,10835,10994,11756"}}
+{"timestamp":1714184687.9077344,"name":"online","context":{"idset":"878,7325,7893,7925,8064,8174,8183,8251,8416,8882,8887,8910,9817,9843,10313,10428,10671,11219,11263,11622"}}
+{"timestamp":1714184687.9090557,"name":"online","context":{"idset":"8273,8884,9422,10716,10976,11317,11495,11519"}}
+{"timestamp":1714184687.9103589,"name":"online","context":{"idset":"105,10823"}}
+{"timestamp":1714184687.9116275,"name":"online","context":{"idset":"1177"}}
+{"timestamp":1714184687.9129171,"name":"online","context":{"idset":"345,10189,11491"}}
+{"timestamp":1714184687.9142115,"name":"online","context":{"idset":"113"}}
+{"timestamp":1714184687.9152079,"name":"online","context":{"idset":"9152,10188,10269"}}
+{"timestamp":1714184687.9163458,"name":"online","context":{"idset":"7344"}}
+{"timestamp":1714184687.9172142,"name":"online","context":{"idset":"11745"}}
+{"timestamp":1714184687.9182072,"name":"online","context":{"idset":"7323,9754"}}
+{"timestamp":1714184687.9191723,"name":"online","context":{"idset":"8267"}}
+{"timestamp":1714184687.9201882,"name":"online","context":{"idset":"9421"}}
+{"timestamp":1714184687.9210756,"name":"online","context":{"idset":"9194,9936"}}
+{"timestamp":1714184687.9220016,"name":"online","context":{"idset":"9902"}}
+{"timestamp":1714184687.9230413,"name":"online","context":{"idset":"288"}}
+{"timestamp":1714184687.9239795,"name":"online","context":{"idset":"9161"}}
+{"timestamp":1714184687.9249203,"name":"online","context":{"idset":"9746"}}
+{"timestamp":1714184687.9258194,"name":"online","context":{"idset":"9233"}}
+{"timestamp":1714184687.9267175,"name":"online","context":{"idset":"8104,11289"}}
+{"timestamp":1714184687.927654,"name":"online","context":{"idset":"9792"}}
+{"timestamp":1714184687.9285512,"name":"online","context":{"idset":"9760,10791,10997,11789"}}
+{"timestamp":1714184687.9294486,"name":"online","context":{"idset":"9749"}}
+{"timestamp":1714184687.9303682,"name":"online","context":{"idset":"8341,9922,10301,10996,11452"}}
+{"timestamp":1714184687.9313185,"name":"online","context":{"idset":"6555,8167,8227,11660,11730"}}
+{"timestamp":1714184687.9323795,"name":"online","context":{"idset":"6526,9495,9789,11813"}}
+{"timestamp":1714184687.9333286,"name":"online","context":{"idset":"6581,9058,9556,10395"}}
+{"timestamp":1714184687.9342773,"name":"online","context":{"idset":"1215,6611,7364,8141,8338,8676,8826,8843,10110,11401,11429,11497,11654,11814"}}
+{"timestamp":1714184687.9352179,"name":"online","context":{"idset":"6549,8576,8611,9336,10316,10445,10742,11179,11864"}}
+{"timestamp":1714184687.9362106,"name":"online","context":{"idset":"8288,8893,9886,10282,11296,11523,11669,11788"}}
+{"timestamp":1714184687.93715,"name":"online","context":{"idset":"9019,9117,11240,11466"}}
+{"timestamp":1714184687.9381037,"name":"online","context":{"idset":"111,406,8027,8381,8995,10466"}}
+{"timestamp":1714184687.9391167,"name":"online","context":{"idset":"419,7988,8019,10419,11818"}}
+{"timestamp":1714184687.9401269,"name":"online","context":{"idset":"310,7926,11228,11372,11616"}}
+{"timestamp":1714184687.9410887,"name":"online","context":{"idset":"110,234,304"}}
+{"timestamp":1714184687.9420993,"name":"online","context":{"idset":"7942,7982,8634,10290"}}
+{"timestamp":1714184687.9430671,"name":"online","context":{"idset":"240,7743,9023"}}
+{"timestamp":1714184687.9441209,"name":"online","context":{"idset":"311,9555,9934,10730,11707"}}
+{"timestamp":1714184687.9451351,"name":"online","context":{"idset":"123,222,7334"}}
+{"timestamp":1714184687.9461136,"name":"online","context":{"idset":"297,11488"}}
+{"timestamp":1714184687.9470758,"name":"online","context":{"idset":"8599"}}
+{"timestamp":1714184687.9480913,"name":"online","context":{"idset":"8889,11627"}}
+{"timestamp":1714184687.9490724,"name":"online","context":{"idset":"405,7979,9294"}}
+{"timestamp":1714184687.950053,"name":"online","context":{"idset":"1216,8650"}}
+{"timestamp":1714184687.9510279,"name":"online","context":{"idset":"8393,8670"}}
+{"timestamp":1714184687.952086,"name":"online","context":{"idset":"186,9847,11055"}}
+{"timestamp":1714184687.9531233,"name":"online","context":{"idset":"97"}}
+{"timestamp":1714184687.9542904,"name":"online","context":{"idset":"417"}}
+{"timestamp":1714184687.9555469,"name":"online","context":{"idset":"211,10705"}}
+{"timestamp":1714184687.9566345,"name":"online","context":{"idset":"11386"}}
+{"timestamp":1714184687.9575989,"name":"online","context":{"idset":"10248"}}
+{"timestamp":1714184687.9589264,"name":"online","context":{"idset":"15"}}
+{"timestamp":1714184687.9600875,"name":"online","context":{"idset":"8959,10321"}}
+{"timestamp":1714184687.9612582,"name":"online","context":{"idset":"8371,10615"}}
+{"timestamp":1714184687.9622529,"name":"online","context":{"idset":"7811"}}
+{"timestamp":1714184687.9633307,"name":"online","context":{"idset":"11821"}}
+{"timestamp":1714184687.9643476,"name":"online","context":{"idset":"10646"}}
+{"timestamp":1714184688.4467456,"name":"online","context":{"idset":"8266,10872,11471"}}
+{"timestamp":1714184689.2326632,"name":"online","context":{"idset":"11265,11628"}}
+{"timestamp":1714184689.2337077,"name":"online","context":{"idset":"9130"}}
+{"timestamp":1714184689.238308,"name":"online","context":{"idset":"11,160,241,243,247,300,317,319-320,333,409,420,883,6518,6604,7294,7300,7330,7386,7401,7828,7959,8044,8358,8613,8949,8984,9005,9020,9085,9166,9178,9482,9559,9912,9917,10138,10191,10200,10277,10297,10299,10637,10655,10682,10715,10744,10878,11218,11269,11308,11450,11474,11480,11483,11505-11506,11514,11539,11740,11758,11859,11873"}}
+{"timestamp":1714184689.2394526,"name":"online","context":{"idset":"2,22,46,56"}}
+{"timestamp":1714184689.240623,"name":"online","context":{"idset":"7343,11314"}}
+{"timestamp":1714184689.2417274,"name":"online","context":{"idset":"9959"}}
+{"timestamp":1714184689.2427416,"name":"online","context":{"idset":"10264"}}
+{"timestamp":1714184689.243818,"name":"online","context":{"idset":"11430"}}
+{"timestamp":1714184689.244844,"name":"online","context":{"idset":"10213"}}
+{"timestamp":1714184689.2458994,"name":"online","context":{"idset":"5,17,40,43,48"}}
+{"timestamp":1714184689.2469888,"name":"online","context":{"idset":"103"}}
+{"timestamp":1714184689.2480414,"name":"online","context":{"idset":"6577"}}
+{"timestamp":1714184689.2491803,"name":"online","context":{"idset":"7981,8125,10192"}}
+{"timestamp":1714184689.2502463,"name":"online","context":{"idset":"8372,9007"}}
+{"timestamp":1714184689.2513559,"name":"online","context":{"idset":"8643"}}
+{"timestamp":1714184689.2525413,"name":"online","context":{"idset":"7,31,47,216,293,8966,9136,10194,10481,11387,11759"}}
+{"timestamp":1714184689.2536492,"name":"online","context":{"idset":"576,6607,7331,9075,10308,11499"}}
+{"timestamp":1714184689.2548087,"name":"online","context":{"idset":"7736,10628"}}
+{"timestamp":1714184689.2558825,"name":"online","context":{"idset":"8243,9202,10325,10924,11446"}}
+{"timestamp":1714184689.2569346,"name":"online","context":{"idset":"10426"}}
+{"timestamp":1714184689.2580698,"name":"online","context":{"idset":"9059,9116"}}
+{"timestamp":1714184689.259201,"name":"online","context":{"idset":"9459,10122,10471,11397"}}
+{"timestamp":1714184689.260304,"name":"online","context":{"idset":"11022,11111"}}
+{"timestamp":1714184689.2614298,"name":"online","context":{"idset":"10198"}}
+{"timestamp":1714184689.262552,"name":"online","context":{"idset":"7955,7992,8004"}}
+{"timestamp":1714184689.2637153,"name":"online","context":{"idset":"10274"}}
+{"timestamp":1714184689.2647557,"name":"online","context":{"idset":"10614"}}
+{"timestamp":1714184689.265815,"name":"online","context":{"idset":"9511"}}
+{"timestamp":1714184689.2669642,"name":"online","context":{"idset":"9151"}}
+{"timestamp":1714184689.2681906,"name":"online","context":{"idset":"8993,9154,11036,11632"}}
+{"timestamp":1714184689.2716775,"name":"online","context":{"idset":"281,1232,7285,7999,8073,8318,8344,9098,9172,9175,9196,10179,10196,10949,11168,11385,11388,11425,11504"}}
+{"timestamp":1714184689.2731295,"name":"online","context":{"idset":"7745,9080,9093,9146,9186,10210,10686,10712,11181,11202,11468,11486,11679,11853"}}
+{"timestamp":1714184689.2743993,"name":"online","context":{"idset":"6531,6556,7292,7304,7818,7963,8414,9313,9810,9877,11067,11118,11442"}}
+{"timestamp":1714184689.2755661,"name":"online","context":{"idset":"8257,11285"}}
+{"timestamp":1714184689.2822101,"name":"online","context":{"idset":"183,250,285,1212,1224,7373,7793,7899,8277,8581,8835,8945,9000,9089,9198,9899,10658,10663,10809,11077,11871"}}
+{"timestamp":1714184689.2845623,"name":"online","context":{"idset":"6602,7858,8152,8198,8286,8578,8668,8688,8904,8907,9008,9301,9304,9455,9467,9582,9584,9761,9786,9827,9875,9972,10262,10296,10364,10371,10374,10630,10729,10739,10758,10792,10877,10958,11007,11034,11086,11130,11206,11516,11531,11537,11573"}}
+{"timestamp":1714184689.2859154,"name":"online","context":{"idset":"8043,8620,9413,10181,10861"}}
+{"timestamp":1714184689.2873518,"name":"online","context":{"idset":"6527,6619,7985,8184,8641,10927"}}
+{"timestamp":1714184689.2890284,"name":"online","context":{"idset":"8623,8681,10391,10759,10951,11610"}}
+{"timestamp":1714184689.2902226,"name":"online","context":{"idset":"8692"}}
+{"timestamp":1714184689.2915895,"name":"online","context":{"idset":"6529,9332"}}
+{"timestamp":1714184689.2930758,"name":"online","context":{"idset":"8129"}}
+{"timestamp":1714184689.2942064,"name":"online","context":{"idset":"10118"}}
+{"timestamp":1714184689.2957203,"name":"online","context":{"idset":"10631,10832,11451"}}
+{"timestamp":1714184689.2970188,"name":"online","context":{"idset":"8845"}}
+{"timestamp":1714184689.2984297,"name":"online","context":{"idset":"8687,9184,11815"}}
+{"timestamp":1714184689.2997873,"name":"online","context":{"idset":"7968"}}
+{"timestamp":1714184689.3009338,"name":"online","context":{"idset":"8677"}}
+{"timestamp":1714184689.304312,"name":"online","context":{"idset":"6625,7305,8350,8407,9006,9234,9828,10902,10990,11336,11522,11647,11710"}}
+{"timestamp":1714184689.3056457,"name":"online","context":{"idset":"8066,11330"}}
+{"timestamp":1714184689.3068247,"name":"online","context":{"idset":"10125"}}
+{"timestamp":1714184689.3103187,"name":"online","context":{"idset":"99,114,188,8145,8192,8868,8902,8991,9208,9225,9871,9929,10109,10965"}}
+{"timestamp":1714184689.3122175,"name":"online","context":{"idset":"6626,7339,7412,7771,8087,8096,8923,9375,9857,9905,10906,10923,11074,11189,11216,11277,11444"}}
+{"timestamp":1714184689.3141828,"name":"online","context":{"idset":"6548,7322,8282,8348,9237,10366,10786,10827,11597,11637"}}
+{"timestamp":1714184689.3156128,"name":"online","context":{"idset":"407,7956"}}
+{"timestamp":1714184689.3168445,"name":"online","context":{"idset":"9137"}}
+{"timestamp":1714184689.3182766,"name":"online","context":{"idset":"11872"}}
+{"timestamp":1714184689.3195744,"name":"online","context":{"idset":"11671"}}
+{"timestamp":1714184689.3211873,"name":"online","context":{"idset":"7840,7949,11376"}}
+{"timestamp":1714184689.3224976,"name":"online","context":{"idset":"8294"}}
+{"timestamp":1714184689.3237593,"name":"online","context":{"idset":"8200,11413"}}
+{"timestamp":1714184689.3249874,"name":"online","context":{"idset":"9388"}}
+{"timestamp":1714184689.3263061,"name":"online","context":{"idset":"7958,8425"}}
+{"timestamp":1714184689.3281491,"name":"online","context":{"idset":"7823,10292,11134"}}
+{"timestamp":1714184689.3294306,"name":"online","context":{"idset":"7990,9392,10725,10755,11629"}}
+{"timestamp":1714184689.3311567,"name":"online","context":{"idset":"7781,8435,9282,9411,10670,11443,11798"}}
+{"timestamp":1714184689.3327456,"name":"online","context":{"idset":"7379,7983,8128,10338,11312"}}
+{"timestamp":1714184689.3341324,"name":"online","context":{"idset":"10300"}}
+{"timestamp":1714184689.3356447,"name":"online","context":{"idset":"9131"}}
+{"timestamp":1714184689.3370855,"name":"online","context":{"idset":"11609"}}
+{"timestamp":1714184689.3384333,"name":"online","context":{"idset":"10180"}}
+{"timestamp":1714184689.3398767,"name":"online","context":{"idset":"10696,11662"}}
+{"timestamp":1714184689.3411722,"name":"online","context":{"idset":"7295,11094"}}
+{"timestamp":1714184689.3424008,"name":"online","context":{"idset":"9217,9220,9893"}}
+{"timestamp":1714184689.343636,"name":"online","context":{"idset":"9732"}}
+{"timestamp":1714184689.3449783,"name":"online","context":{"idset":"1213,8155,9076,9430,9933,11081,11344"}}
+{"timestamp":1714184689.3462405,"name":"online","context":{"idset":"6630,7896,8426,11139"}}
+{"timestamp":1714184689.3476079,"name":"online","context":{"idset":"7333,8953,9299"}}
+{"timestamp":1714184689.3489087,"name":"online","context":{"idset":"6565,7943,8603,8615,9298,10636,10892,11339"}}
+{"timestamp":1714184689.3502436,"name":"online","context":{"idset":"10883,11095,11714"}}
+{"timestamp":1714184689.3515317,"name":"online","context":{"idset":"9323"}}
+{"timestamp":1714184689.3528435,"name":"online","context":{"idset":"7803,8095,9911,10683,11576,11743"}}
+{"timestamp":1714184689.3541753,"name":"online","context":{"idset":"7746,8346,11114,11322"}}
+{"timestamp":1714184689.3556309,"name":"online","context":{"idset":"7937,9425,11138"}}
+{"timestamp":1714184689.3569503,"name":"online","context":{"idset":"8857,9135,9952,11148"}}
+{"timestamp":1714184689.3582675,"name":"online","context":{"idset":"11479"}}
+{"timestamp":1714184689.3597789,"name":"online","context":{"idset":"1228"}}
+{"timestamp":1714184689.3610718,"name":"online","context":{"idset":"10477,11830"}}
+{"timestamp":1714184689.3624229,"name":"online","context":{"idset":"11371"}}
+{"timestamp":1714184689.3636961,"name":"online","context":{"idset":"8071,8354,9343,10257"}}
+{"timestamp":1714184689.3649766,"name":"online","context":{"idset":"11215"}}
+{"timestamp":1714184689.3663116,"name":"online","context":{"idset":"8250,10664"}}
+{"timestamp":1714184689.3675885,"name":"online","context":{"idset":"8085,8628,9442,10237,10807"}}
+{"timestamp":1714184689.369029,"name":"online","context":{"idset":"8169,8415,11255"}}
+{"timestamp":1714184689.3703041,"name":"online","context":{"idset":"11367"}}
+{"timestamp":1714184689.3716371,"name":"online","context":{"idset":"6575,7407,8158,8308,10121"}}
+{"timestamp":1714184689.372869,"name":"online","context":{"idset":"8870"}}
+{"timestamp":1714184689.3743739,"name":"online","context":{"idset":"8054,8260,9311,9345,10978,11418,11492,11860"}}
+{"timestamp":1714184689.3757398,"name":"online","context":{"idset":"9337,9782,10105,10803"}}
+{"timestamp":1714184689.3771305,"name":"online","context":{"idset":"10238"}}
+{"timestamp":1714184689.3783853,"name":"online","context":{"idset":"315"}}
+{"timestamp":1714184689.379751,"name":"online","context":{"idset":"1207,7904,8609,9526,11626,11761"}}
+{"timestamp":1714184689.3814085,"name":"online","context":{"idset":"7310,7807,9587,10711,10844,10931,11196,11199,11808,11825"}}
+{"timestamp":1714184689.3826416,"name":"online","context":{"idset":"10698"}}
+{"timestamp":1714184689.3839345,"name":"online","context":{"idset":"10150"}}
+{"timestamp":1714184689.3853581,"name":"online","context":{"idset":"7365"}}
+{"timestamp":1714184689.3874648,"name":"online","context":{"idset":"8316,10808,10954,11088"}}
+{"timestamp":1714184689.3895831,"name":"online","context":{"idset":"8080,9864,9932,10733"}}
+{"timestamp":1714184689.3919547,"name":"online","context":{"idset":"235,835,6570,9167,9519,9766,9879,10167,10378,11572"}}
+{"timestamp":1714184689.3941836,"name":"online","context":{"idset":"7879,7881,8214,11213"}}
+{"timestamp":1714184689.3957679,"name":"online","context":{"idset":"10888,11187"}}
+{"timestamp":1714184689.3972766,"name":"online","context":{"idset":"7411,7833,8644,8659,9733,10184,10736,11108,11124,11551,11719"}}
+{"timestamp":1714184689.3992629,"name":"online","context":{"idset":"39,6610,7815,8601,8855,9170,9496,9540,10281,10343,10456,10474,11259,11507,11760"}}
+{"timestamp":1714184689.4016402,"name":"online","context":{"idset":"8197,8320,8980,9354,9429,9942,10126,10187,10207,10231,10423,10779,10801,10840,11205,11803"}}
+{"timestamp":1714184689.4030464,"name":"online","context":{"idset":"10805,10942,11173,11753"}}
+{"timestamp":1714184689.4043429,"name":"online","context":{"idset":"11746"}}
+{"timestamp":1714184689.4056838,"name":"online","context":{"idset":"7998"}}
+{"timestamp":1714184689.4073901,"name":"online","context":{"idset":"9743,11137"}}
+{"timestamp":1714184689.4088101,"name":"online","context":{"idset":"8130"}}
+{"timestamp":1714184689.4106114,"name":"online","context":{"idset":"290,8261,8310,8328,9572,9869,9881,9889,10250,10778,10871,10898,11166,11453,11591"}}
+{"timestamp":1714184689.4120421,"name":"online","context":{"idset":"11598"}}
+{"timestamp":1714184689.4135804,"name":"online","context":{"idset":"7742,7832,8873,9586,10253,10643,10854,11603"}}
+{"timestamp":1714184689.4149225,"name":"online","context":{"idset":"8296,9308,11770"}}
+{"timestamp":1714184690.1122937,"name":"online","context":{"idset":"8866,9535,11131,11147,11288"}}
+{"timestamp":1714184690.5145676,"name":"online","context":{"idset":"19,41,171,237,324,1231,6562,6572,6621,7287,7326,7362,7387,7824,7847,7851,7882,7900,8138,8212,8218,8225,8231,8303,8325,8369,8385,8594,8597,8604,8612,8824,8909,8970,9051,9103,9105,9112,9195,9200,9245,9258,9346,9356-9357,9359,9433,9465,9479,9492,9547,9721,9725,9729-9730,9816,9830,9862,9900,9906,10103,10265,10270,10384-10385,10439,10454,10478,10660,10677-10678,10752,10766,10798,10895,10950,11037,11042-11043,11045,11070,11075,11103,11136,11192,11251,11268,11304,11327,11335,11503,11535,11575,11605,11650,11695,11744,11748,11812,11833,11874,11891"}}
+{"timestamp":1714184690.5160751,"name":"online","context":{"idset":"59,8075,8583,8839,8861,10783,10955,11324"}}
+{"timestamp":1714184690.517487,"name":"online","context":{"idset":"9539"}}
+{"timestamp":1714184690.5189645,"name":"online","context":{"idset":"7352"}}
+{"timestamp":1714184690.5204034,"name":"online","context":{"idset":"10283"}}
+{"timestamp":1714184690.521831,"name":"online","context":{"idset":"7892,7914,7918,11217"}}
+{"timestamp":1714184690.5232148,"name":"online","context":{"idset":"8003,9462,10139"}}
+{"timestamp":1714184690.5245783,"name":"online","context":{"idset":"9305"}}
+{"timestamp":1714184690.5260081,"name":"online","context":{"idset":"8215,8229,8648,9390,9415"}}
+{"timestamp":1714184690.5279047,"name":"online","context":{"idset":"192,199,209,341,7349,8842,9099,10430,10890"}}
+{"timestamp":1714184690.5293207,"name":"online","context":{"idset":"6520"}}
+{"timestamp":1714184690.5308092,"name":"online","context":{"idset":"7776,9242,11427"}}
+{"timestamp":1714184690.5321865,"name":"online","context":{"idset":"9290,11375"}}
+{"timestamp":1714184690.534637,"name":"online","context":{"idset":"9341"}}
+{"timestamp":1714184690.5362477,"name":"online","context":{"idset":"8165,9400,11844"}}
+{"timestamp":1714184690.5376461,"name":"online","context":{"idset":"8311,9320"}}
+{"timestamp":1714184690.5390062,"name":"online","context":{"idset":"11709"}}
+{"timestamp":1714184690.5404494,"name":"online","context":{"idset":"8951,9155,9768,9836,10901,11400"}}
+{"timestamp":1714184690.5419834,"name":"online","context":{"idset":"9205,9227,9374,9529,9735,9800,9907-9908,10244,10294,11091,11151,11406,11723"}}
+{"timestamp":1714184690.5442212,"name":"online","context":{"idset":"6634,7389,7830,7986,8195,8246,8988,9011,9026,9211,9247,9266,9310,9326,9330,9333,9338,9380,9470,9752,9856,9914,9961,10156,10171,10202,10247,10417,10634,10672,10688,10699,10724,10728,10771,10796,10815,10842,10903,10905,10961,10963,10995,11020,11030,11112-11113,11117,11247,11279,11283,11587,11638,11642,11682,11837"}}
+{"timestamp":1714184690.5460548,"name":"online","context":{"idset":"54"}}
+{"timestamp":1714184690.5482066,"name":"online","context":{"idset":"44"}}
+{"timestamp":1714184690.5504787,"name":"online","context":{"idset":"899"}}
+{"timestamp":1714184690.5520861,"name":"online","context":{"idset":"825,873,6554,6627,7404,7406,7785,7822,8402,8406,8410,8871,9018,9182,9278,9573,10232,11176,11655,11704,11735"}}
+{"timestamp":1714184690.5536745,"name":"online","context":{"idset":"124,130,164,173,189,191,204,210,232,242,278,294,302-303,305,325-326,334,337,6522,6525,6532,6535,6540,6589,7347,7377,7392,7394,7738,7790,7827,7834,7848,7868,8039,8060,8160,8177-8178,8186,8264,9248,9252,10280,10407,10723,10774,11552"}}
+{"timestamp":1714184690.5552099,"name":"online","context":{"idset":"8618,9920,10907"}}
+{"timestamp":1714184690.5567617,"name":"online","context":{"idset":"7853,7867,7936,7974,7996,8049,8150,8228,8661,8825,8848,8954,9060,9092,9396,9439,9483,9545,9798,10120,10449,10463,10666,10684,10706,10745,10784,10790,10831,10856,11003,11101,11109,11481,11529,11861"}}
+{"timestamp":1714184690.5584807,"name":"online","context":{"idset":"190,8360,8823,8852,8918,9113-9114,9517,9564,9585,9769,9785,9804,9885,10195,10330,10679,10709,10879,10919,10929,11008,11102,11135,11142,11668,11751,11829,11846"}}
+{"timestamp":1714184690.5600173,"name":"online","context":{"idset":"8172,8366,9001,9373,9567,9726,9954-9955,10190,10203,10268,10309,10483,10640,10665,10668,10695,10707,10726,10748,10810,10853,10974,11057,11221-11222,11232,11236,11264,11302,11345,11347,11365,11527,11583,11708"}}
+{"timestamp":1714184690.5617077,"name":"online","context":{"idset":"201,218,7987,8606,8678,9088,9738,9799,10240,10285,10416,10620,10751,11048,11089,11431,11487,11851"}}
+{"timestamp":1714184690.5633206,"name":"online","context":{"idset":"228"}}
+{"timestamp":1714184690.5648897,"name":"online","context":{"idset":"215,220"}}
+{"timestamp":1714184690.5664287,"name":"online","context":{"idset":"10669,11464"}}
+{"timestamp":1714184690.5679498,"name":"online","context":{"idset":"10341"}}
+{"timestamp":1714184690.5695217,"name":"online","context":{"idset":"9126,11350"}}
+{"timestamp":1714184690.5711367,"name":"online","context":{"idset":"9386,10288"}}
+{"timestamp":1714184690.5726485,"name":"online","context":{"idset":"10851"}}
+{"timestamp":1714184690.5742371,"name":"online","context":{"idset":"10659,11780"}}
+{"timestamp":1714184690.5758171,"name":"online","context":{"idset":"11445"}}
+{"timestamp":1714184690.5775001,"name":"online","context":{"idset":"23-24,35,57,573,7909,7948,8856,11407"}}
+{"timestamp":1714184690.5792031,"name":"online","context":{"idset":"18,9002,11260"}}
+{"timestamp":1714184690.5809081,"name":"online","context":{"idset":"187,306,8262,8363,10617,11741"}}
+{"timestamp":1714184690.582777,"name":"online","context":{"idset":"162,280,322,1188,7317,8345,8386,8964,9235,10647"}}
+{"timestamp":1714184690.584403,"name":"online","context":{"idset":"6587,8083"}}
+{"timestamp":1714184690.5859883,"name":"online","context":{"idset":"9970"}}
+{"timestamp":1714184690.587642,"name":"online","context":{"idset":"8035,8996,9097,10104,10436,11267,11332"}}
+{"timestamp":1714184690.5892324,"name":"online","context":{"idset":"10641"}}
+{"timestamp":1714184690.5908141,"name":"online","context":{"idset":"7791,10137,10239"}}
+{"timestamp":1714184690.5924964,"name":"online","context":{"idset":"225,7306,9106,11300"}}
+{"timestamp":1714184690.5941761,"name":"online","context":{"idset":"26,6559,8844,9101,10245,10403"}}
+{"timestamp":1714184690.5958078,"name":"online","context":{"idset":"8396,11331"}}
+{"timestamp":1714184690.5973732,"name":"online","context":{"idset":"9532,10193,10747,11146,11348,11374,11852"}}
+{"timestamp":1714184690.5989828,"name":"online","context":{"idset":"25,7357,7916,8232,9269"}}
+{"timestamp":1714184690.6005497,"name":"online","context":{"idset":"8982,11596"}}
+{"timestamp":1714184690.6021233,"name":"online","context":{"idset":"9588,11059"}}
+{"timestamp":1714184690.60958,"name":"online","context":{"idset":"27,6539,6614,7303,8124,8221,9090,9148,9164,9236,9260,9296,9524,9541,9824,9866,9924,10170,10266,10344,10375,10389,10652,10692,10714,10721,10794,11071,11183,11250,11313,11530,11602,11644,11692,11807"}}
+{"timestamp":1714184690.6112549,"name":"online","context":{"idset":"60,8915"}}
+{"timestamp":1714184690.6130261,"name":"online","context":{"idset":"575,8081"}}
+{"timestamp":1714184690.6146734,"name":"online","context":{"idset":"7971"}}
+{"timestamp":1714184690.6169324,"name":"online","context":{"idset":"9,8190,9718,10289,11230,11462"}}
+{"timestamp":1714184690.6185622,"name":"online","context":{"idset":"9720,11333"}}
+{"timestamp":1714184690.62024,"name":"online","context":{"idset":"9262"}}
+{"timestamp":1714184690.6219647,"name":"online","context":{"idset":"8580"}}
+{"timestamp":1714184690.6235754,"name":"online","context":{"idset":"7941,10206"}}
+{"timestamp":1714184690.6252639,"name":"online","context":{"idset":"9501,10319,11676"}}
+{"timestamp":1714184690.6271596,"name":"online","context":{"idset":"55,8176,10272"}}
+{"timestamp":1714184690.6289222,"name":"online","context":{"idset":"6603,8362,8983,11047,11493,11635"}}
+{"timestamp":1714184690.6308599,"name":"online","context":{"idset":"8028"}}
+{"timestamp":1714184690.6327753,"name":"online","context":{"idset":"1"}}
+{"timestamp":1714184690.6347795,"name":"online","context":{"idset":"1186,7804,8291,8986,9795,10287,11156,11399,11839"}}
+{"timestamp":1714184690.6365197,"name":"online","context":{"idset":"9177"}}
+{"timestamp":1714184690.6382136,"name":"online","context":{"idset":"8188"}}
+{"timestamp":1714184690.6400392,"name":"online","context":{"idset":"9087,9163,9382,11869"}}
+{"timestamp":1714184690.6417296,"name":"online","context":{"idset":"346"}}
+{"timestamp":1714184690.6433368,"name":"online","context":{"idset":"11866"}}
+{"timestamp":1714184690.9246292,"name":"online","context":{"idset":"7733,8928,9095,10214"}}
+{"timestamp":1714184691.0439138,"name":"online","context":{"idset":"6,98,213,307,328,6624,7327,7740,7928,7970,7993,8017,8210,8347,8592,8608,8662,8981,8989,8998,9082,9096,9134,9153,9162,9240,9259,9531,9796-9797,10197,10212,10235,10329,10332,10355-10356,10368,10648,10689,10694,10722,10738,11090,11097,11195,11404,11436,11477,11482,11496,11675"}}
+{"timestamp":1714184691.0455012,"name":"online","context":{"idset":"11747"}}
+{"timestamp":1714184691.4768112,"name":"online","context":{"idset":"836,1181,1219,6600,7358,7686,7703,7714,7725,7855,7877,7934,7957,7977,7991,7997,8009,8011,8029,8084,8100,8159,8161-8162,8239,8343,8398,8647,8973,9100,9293,9302,9384,9402,9438,9441,9618,9626,9671,9693,9713,9777,9909,9950,9971,10002,10025,10060,10123,10217,10387,10418,10442,10480,10484,10495,10500,10525,10544,10592-10593,10607,10612,10782,10785,10789,10824,10867,10899,10904,10967,11040,11271,11390-11391,11448-11449,11472,11512,11727,11772,11783,11878,11886"}}
+{"timestamp":1714184691.4785929,"name":"online","context":{"idset":"10010"}}
+{"timestamp":1714184691.48031,"name":"online","context":{"idset":"9192"}}
+{"timestamp":1714184691.4820139,"name":"online","context":{"idset":"9077"}}
+{"timestamp":1714184691.4838865,"name":"online","context":{"idset":"7751,9072"}}
+{"timestamp":1714184691.4856496,"name":"online","context":{"idset":"7674,11889"}}
+{"timestamp":1714184691.4874141,"name":"online","context":{"idset":"10046,10303"}}
+{"timestamp":1714184691.4891992,"name":"online","context":{"idset":"10574"}}
+{"timestamp":1714184691.4909332,"name":"online","context":{"idset":"9143,11396"}}
+{"timestamp":1714184691.4927292,"name":"online","context":{"idset":"347"}}
+{"timestamp":1714184691.4946802,"name":"online","context":{"idset":"10334"}}
+{"timestamp":1714184691.4963686,"name":"online","context":{"idset":"9179"}}
+{"timestamp":1714184691.4981077,"name":"online","context":{"idset":"9055,9189"}}
+{"timestamp":1714184691.4997787,"name":"online","context":{"idset":"10298"}}
+{"timestamp":1714184691.5014858,"name":"online","context":{"idset":"11458"}}
+{"timestamp":1714184691.5032794,"name":"online","context":{"idset":"10032"}}
+{"timestamp":1714184691.5051098,"name":"online","context":{"idset":"7966"}}
+{"timestamp":1714184691.5067852,"name":"online","context":{"idset":"10295"}}
+{"timestamp":1714184691.5089095,"name":"online","context":{"idset":"9624,10315"}}
+{"timestamp":1714184691.5107067,"name":"online","context":{"idset":"8015"}}
+{"timestamp":1714184691.5123692,"name":"online","context":{"idset":"10042,10318"}}
+{"timestamp":1714184691.5142012,"name":"online","context":{"idset":"10578"}}
+{"timestamp":1714184691.5159667,"name":"online","context":{"idset":"299,7969"}}
+{"timestamp":1714184691.5176826,"name":"online","context":{"idset":"167"}}
+{"timestamp":1714184691.5194056,"name":"online","context":{"idset":"30,11428"}}
+{"timestamp":1714184691.5211608,"name":"online","context":{"idset":"7717,9596"}}
+{"timestamp":1714184691.5228796,"name":"online","context":{"idset":"8,7692"}}
+{"timestamp":1714184691.524513,"name":"online","context":{"idset":"11420"}}
+{"timestamp":1714184691.5261774,"name":"online","context":{"idset":"10595"}}
+{"timestamp":1714184691.5279019,"name":"online","context":{"idset":"7712"}}
+{"timestamp":1714184691.5298815,"name":"online","context":{"idset":"32"}}
+{"timestamp":1714184691.7061808,"name":"online","context":{"idset":"7680,7688,7707,7724,7726,7750,7752,7764,7929,9644,9649,9664,9975,9986,9992,10011,10013,10016,10031,10057,10062,10067,10079,10488,10491,10496,10507,10510,10518-10519,10528,10536-10537,10540,10557,10562,10569-10570,10572,10577,10585-10586,10600,10603,11885"}}
+{"timestamp":1714184691.7152064,"name":"online","context":{"idset":"9636"}}
+{"timestamp":1714184691.7435737,"name":"online","context":{"idset":"10324,10581"}}
+{"timestamp":1714184691.7496908,"name":"online","context":{"idset":"11888"}}
+{"timestamp":1714184691.8562746,"name":"online","context":{"idset":"9658,10004,10039,10054,10486,10520,10533"}}
+{"timestamp":1714184691.9373174,"name":"online","context":{"idset":"9990,10027,10066,10524,10549"}}
+{"timestamp":1714184691.94964,"name":"online","context":{"idset":"21"}}
+{"timestamp":1714184692.0622246,"name":"online","context":{"idset":"7675,7691,7702,7755,7953,8025,8046,9697,9978,10019,10037,10086-10087,10099,10498,10501,10527,10604"}}
+{"timestamp":1714184692.0660868,"name":"online","context":{"idset":"10071,10556"}}
+{"timestamp":1714184692.0694718,"name":"online","context":{"idset":"10547,10598-10599,10606"}}
+{"timestamp":1714184692.0726173,"name":"online","context":{"idset":"10532"}}
+{"timestamp":1714184692.0784779,"name":"online","context":{"idset":"10594"}}
+{"timestamp":1714184692.2115836,"name":"online","context":{"idset":"10,14,29,7676,7682,7684,7728,7962,8021,9601,9614,9619,9625,9631,9645,9660,9665,9679,9682,9695,9699,9702,9715,10029-10030,10034,10044,10082,10084,10088,10090,10502,10542,10548,10563,11887"}}
+{"timestamp":1714184692.2227731,"name":"online","context":{"idset":"9692"}}
+{"timestamp":1714184692.2319307,"name":"online","context":{"idset":"10063,10096"}}
+{"timestamp":1714184692.2793038,"name":"online","context":{"idset":"10012"}}
+{"timestamp":1714184692.3598659,"name":"online","context":{"idset":"10529"}}
+{"timestamp":1714184692.4486432,"name":"online","context":{"idset":"33,9687,11877"}}
+{"timestamp":1714184692.5316479,"name":"online","context":{"idset":"20,28,7669,7685,7716,7732,9612,9622,9667,9677,9714,9974,10017,10045,10051,10070,10077-10078,10100,10582,10584,10590-10591,10602"}}
+{"timestamp":1714184692.5412884,"name":"online","context":{"idset":"7670,7672,7704,7713,7715,7760,7935,9592,9633,9681,9977,9993,10005,10009,10069,10081,10091,10516,10555,10558,10576,10580,10596-10597"}}
+{"timestamp":1714184693.5551834,"name":"online","context":{"idset":"9627"}}
+{"timestamp":1714184693.5597,"name":"online","context":{"idset":"9653"}}
+{"timestamp":1714184693.5639274,"name":"online","context":{"idset":"9709,10035,10610"}}
+{"timestamp":1714184693.5677037,"name":"online","context":{"idset":"9704,10526"}}
+{"timestamp":1714184693.5714912,"name":"online","context":{"idset":"9591,10073,10567"}}
+{"timestamp":1714184693.5753551,"name":"online","context":{"idset":"11884"}}
+{"timestamp":1714184693.5814526,"name":"online","context":{"idset":"9595,10072"}}
+{"timestamp":1714184693.5852177,"name":"online","context":{"idset":"7730,9980"}}
+{"timestamp":1714184693.5894268,"name":"online","context":{"idset":"7709,7718,7758,10097,10546,10608"}}
+{"timestamp":1714184693.593323,"name":"online","context":{"idset":"10000"}}
+{"timestamp":1714184693.5971341,"name":"online","context":{"idset":"11875"}}
+{"timestamp":1714184693.6010365,"name":"online","context":{"idset":"9982"}}
+{"timestamp":1714184693.6047246,"name":"online","context":{"idset":"9710,10020,10534"}}
+{"timestamp":1714184693.6087372,"name":"online","context":{"idset":"7754,10609"}}
+{"timestamp":1714184693.6124177,"name":"online","context":{"idset":"10489,10512"}}
+{"timestamp":1714184693.6181452,"name":"online","context":{"idset":"10506"}}
+{"timestamp":1714184693.6382775,"name":"online","context":{"idset":"6550,6639,7705,7768,8135,8391,8434,9339,9635,9673,10001,10571,11012,11823"}}
+{"timestamp":1714184693.6432598,"name":"online","context":{"idset":"978,6573,6588,8069,8339,8642,8679,8858,8894,8898,8914,9365,9404,9698,9822,9967,10762,10889,11001,11098,11513,11634,11705"}}
+{"timestamp":1714184693.6473796,"name":"online","context":{"idset":"6557,7286,8194,8625,9419,9440,9821,9995,10473,10475"}}
+{"timestamp":1714184693.6542065,"name":"online","context":{"idset":"1180,6585,6629,6644,7321,7353,7361,7400,7802,7812,8031,8059,8068,8101,8103,8133,8166,8253,8258,8270,8272,8317,8412,8829,8886,9047,9215,9238,9315,9353,9443,9466,9615,9863,9874,9882,9904,9935,10135,10175,10205,10363,10376,10415,10427,10438,10450,10834,10953,10962,10981,11000,11170,11207,11284,11293,11366,11517,11549,11601,11624,11697,11731,11795,11817,11836"}}
+{"timestamp":1714184693.6600065,"name":"online","context":{"idset":"8147"}}
+{"timestamp":1714184693.6655917,"name":"online","context":{"idset":"874,6641,7296,7320,7409,7786,7814,7850,8012,8074,8114,8136,8144,8181,8193,8333,8596,8660,8671,8851,8890,8942,8946,9048,9361,9417,9444,9543,9557,9576,9600,9620,9684,9831,9839,9842,10221,10400,10459,10749,10862,10939,11026,11028,11184,11190-11191,11233,11245,11261,11329,11338,11518,11590,11773,11782,11792,11801"}}
+{"timestamp":1714184693.6713161,"name":"online","context":{"idset":"808,977,1174,1214,6523,6590,7348,7806,7813,8070,8078,8112,8151,8153,8209,8216,8252,8313,8337,8355,8380,8570,8605,8652,8837,8867,8908,8912,8941,8944,8947,8975,9244,9251,9253,9367,9378,9403,9436,9454,9463,9469,9502,9550,9553,9607,9689,9773,9953,10162,10222,10357,10361,10406,10467,10479,10988,11069,11085,11141,11210,11214,11224,11253,11262,11287,11380,11623,11677,11683,11686,11691,11715,11826"}}
+{"timestamp":1714184693.6763947,"name":"online","context":{"idset":"6599,6642,7800,7835,7933,8018,8204,8244,8271,8330,8332,8383,8673,8853,8924,8926,9241,9488,9507-9508,9688,9722,9737,9848,9859,9944,10130,10176,10379,10390,10845,10957,11009,11053,11150,11159,11303,11309,11354,11678,11690,11693,11703,11706"}}
+{"timestamp":1714184693.7139289,"name":"online","context":{"idset":"107,340,7342,7761,8005,8298,8675,8962,9232,9395,10342,10837,10979,11252,11652,11862"}}
+{"timestamp":1714184693.9584291,"name":"online","context":{"idset":"95,101,118,125,127-128,176,180-181,184,193,196,208,214,221,226,244,251,283,295,308,323,329,331-332,339,412,416,577,1173,6596,7944,7975-7976,8036,8048,8057,8099,8164,8312,8622,8879,9009,9022,9081,9104,9122,9124,9142,9188,9190,9409,9499,9598,9602-9603,9629,9690,9694,9719,9776,9897,10242,10256,10284,10339,10431,10625,10639,10675,10687,10731,10812,10887,11363,11392-11393,11421,11819,11870,11892"}}
+{"timestamp":1714184694.3082824,"name":"online","context":{"idset":"132,245,6544,7385,7388,7821,7878,7973,8091,8375,9138,9197,9271,9335,9355,9549,9552,9700,9778,9872,9928,9939,10136,10174,10230,10261,10333,10359,10369,10372,10392,10394,10402,10650,10717,10761,10838,10843,10916,11062,11220,11281,11342,11394,11419,11426,11438,11440,11460,11484,11588,11595,11684,11699,11716,11822,11863"}}
+{"timestamp":1714184694.4247468,"name":"online","context":{"idset":"1190,6571,7351,7995,8118,8349,9267,9940,10208,10743,10930,11120,11424,11439,11843"}}
+{"timestamp":1714184694.5327172,"name":"online","context":{"idset":"6560,7961,8927,8929,11274,11461"}}
+{"timestamp":1714184694.6475887,"name":"online","context":{"idset":"169,175,8175,8248,9118,9790,9925,10690,11149,11608"}}
+{"timestamp":1714184695.5330892,"name":"online","context":{"idset":"7690,7697,7723,7727,7734,7749,8185,8223,8394,9284,9434,9648,9826,9983,9989,10026,10050,10052,10083,10085,10093-10094,10133,10490,10538,10545,10564,10575,10685,10700,11364,11544,11717,11729,11876,11890"}}
+{"timestamp":1714184695.5374494,"name":"online","context":{"idset":"10514"}}
+{"timestamp":1714184695.5416148,"name":"online","context":{"idset":"10446"}}
+{"timestamp":1714184695.5459123,"name":"online","context":{"idset":"7681"}}
+{"timestamp":1714184695.5501118,"name":"online","context":{"idset":"9639,10015,10041,10561"}}
+{"timestamp":1714184695.5544012,"name":"online","context":{"idset":"9642,9696,9979,10611,11378,11879"}}
+{"timestamp":1714184695.5586898,"name":"online","context":{"idset":"7722,7757,7910,9656,9745,9854,9984,9998,10061,10505,10566,10579,11186"}}
+{"timestamp":1714184695.5630066,"name":"online","context":{"idset":"7701,7719,9568,9621,9652,9997,10007,10092,10499,10553,11060"}}
+{"timestamp":1714184695.567234,"name":"online","context":{"idset":"9650"}}
+{"timestamp":1714184695.5720775,"name":"online","context":{"idset":"7731,8170,8284,8619,8921,9052,9086,9610,9662,9708,9712,9808,10021,10565,10583,10859,10977,10985"}}
+{"timestamp":1714184695.5763769,"name":"online","context":{"idset":"10523,11508"}}
+{"timestamp":1714184695.5809035,"name":"online","context":{"idset":"163,282,1217,8127,9140,9623,9634,10023,10249,10327,10623,10701,10708,11383,11414,11422"}}
+{"timestamp":1714184695.5856431,"name":"online","context":{"idset":"7984,9078,9110,9201,9605,9655,10624,10649,10651,10657,11435"}}
+{"timestamp":1714184695.5903203,"name":"online","context":{"idset":"9608,9646,9672,9674,9705,10654,10740,11437"}}
+{"timestamp":1714184695.5948699,"name":"online","context":{"idset":"9668"}}
+{"timestamp":1714184695.7528682,"name":"online","context":{"idset":"9670"}}
+{"timestamp":1714184695.8634317,"name":"online","context":{"idset":"9638,9651,9701,10511,10554"}}
+{"timestamp":1714184696.0374503,"name":"online","context":{"idset":"38,51"}}
+{"timestamp":1714184696.1210971,"name":"online","context":{"idset":"9707,9987,10008,10065,10098"}}
+{"timestamp":1714184696.2509298,"name":"online","context":{"idset":"4,16,36,42,53"}}
+{"timestamp":1714184696.3371241,"name":"online","context":{"idset":"9637,9669"}}
+{"timestamp":1714184697.4525907,"name":"online","context":{"idset":"7288,7336,9432,9759,9852,10884,11044,11167,11728"}}
+{"timestamp":1714184697.4569979,"name":"online","context":{"idset":"10440,10804,11129,11326,11641"}}
+{"timestamp":1714184697.4615054,"name":"online","context":{"idset":"6545,6593,7319,8859,9377,10399,10800,11005,11848"}}
+{"timestamp":1714184697.4657333,"name":"online","context":{"idset":"10917"}}
+{"timestamp":1714184697.4701724,"name":"online","context":{"idset":"9916"}}
+{"timestamp":1714184697.4745774,"name":"online","context":{"idset":"7376,8024,8638,8666,9918"}}
+{"timestamp":1714184697.4789147,"name":"online","context":{"idset":"9157,9199"}}
+{"timestamp":1714184697.4833446,"name":"online","context":{"idset":"9017,9102"}}
+{"timestamp":1714184697.4878404,"name":"online","context":{"idset":"301,7932,9079,9168,10320,10635,11384,11402"}}
+{"timestamp":1714184697.49225,"name":"online","context":{"idset":"327,10350"}}
+{"timestamp":1714184697.4966383,"name":"online","context":{"idset":"9174"}}
+{"timestamp":1714184697.7376339,"name":"online","context":{"idset":"7931,8045"}}
+{"timestamp":1714184698.0368547,"name":"online","context":{"idset":"7940"}}
+{"timestamp":1714184698.2177989,"name":"online","context":{"idset":"7673,7677,7729,9628,9711,10022,10033,10049,10068,10080,10492,10513,10521,10560,10568,11883"}}
+{"timestamp":1714184698.3862636,"name":"online","context":{"idset":"9685,9976,10006,10535,10552"}}
+{"timestamp":1714184698.5849862,"name":"online","context":{"idset":"7679,7696,7708,7710,7720-7721,9590,9630,9657,9985,9988,10014,10018,10047,10055,10064,10487,10543,10601"}}
+{"timestamp":1714184698.8743665,"name":"online","context":{"idset":"7683,7694,7759,9609,9632,9654,9683,10059,10508"}}
+{"timestamp":1714184699.066206,"name":"online","context":{"idset":"9640,9706"}}
+{"timestamp":1714184699.2548912,"name":"online","context":{"idset":"9647"}}
+{"timestamp":1714184699.3560472,"name":"online","context":{"idset":"9599,9604"}}
+{"timestamp":1714184699.4966528,"name":"online","context":{"idset":"9593,9716"}}
+{"timestamp":1714184699.6398549,"name":"online","context":{"idset":"9589"}}
+{"timestamp":1714184701.5666449,"name":"online","context":{"idset":"6558,6637,7360,7380,7842,8123,8420,8891,9276,9364,9537,9583,9783,9794,10937,10940,11004,11021,11256,11299,11649,11777"}}
+{"timestamp":1714184701.5737185,"name":"online","context":{"idset":"871,1229,6538,6576,6633,7889,8056,8061,8088,8189,8236,8256,8430,8602,8828,8888,9273,9566,9835,10163,10177,10462,10860,10970,11014,11177,11368,11536,11600,11670,11794,11841"}}
+{"timestamp":1714184701.5814555,"name":"online","context":{"idset":"6566,7408,7784,7809-7810,7860,7924,8053,8131,8173,8821,9385,9503,9514,9560,9814,10470,11072,11297,11592,11694"}}
+{"timestamp":1714184701.58672,"name":"online","context":{"idset":"7345,7350,7399,7852,8067,8582,8600,8956,8977,8987,9209,9231,9397,9903,9951,10048,10182,10866,11027,11039,11204,11278,11334"}}
+{"timestamp":1714184701.5935853,"name":"online","context":{"idset":"6583,7787,8202,8206,8994,9788,10129,10225,10412,10941,11174,11237,11258,11349,11811"}}
+{"timestamp":1714184701.6431563,"name":"online","context":{"idset":"291,872,7402,7880,8168,8180,8224,8275,8287,8315,8827,8850,8932,9046,9206,9275,9418,9456,9473,9571,9734,9855,9943,10425,10447,10461,10897,10935,10991-10992,10999,11025,11243,11257,11266,11594,11805"}}
+{"timestamp":1714184701.8843744,"name":"online","context":{"idset":"1178,1185,1218,1235,6536,6569,6618,6622,6635,6643,7293,7299,7309,7313,7341,7366,7398,7772,7788,7792,7796-7797,7799,7876,7905,7927,7938,7945,7952,7994,8050,8097,8121,8134,8137,8148,8191,8199,8222,8237,8240,8249,8276,8290,8305,8323,8351,8357,8373,8405,8589,8617,8684,8686,8822,8834,8838,8876,8916,8919-8920,8948,8960,8968-8969,9069,9074,9185,9256,9270,9279,9363,9394,9405,9423,9431,9447,9450,9478,9490-9491,9504,9509,9515,9518,9542,9742,9747,9751,9756-9757,9764,9806,9812-9813,9823,9829,9910,9915,9930,9956-9957,10108,10116,10134,10164,10209,10386,10404,10443-10444,10764,10787,10797,10802,10817,10847,10850,10873,10920,10952,10956,11035,11052,11064-11065,11125,11154,11158,11161,11165,11172,11197,11212,11229,11231,11276,11280,11323,11528,11542,11568,11574,11582,11613,11633,11639,11657,11712,11721,11724,11736,11762,11858"}}
+{"timestamp":1714184701.9849372,"name":"online","context":{"idset":"8646"}}
+{"timestamp":1714184702.1886258,"name":"online","context":{"idset":"100,131,161,170,178-179,182,236,239,312,321,344,414,418,7747,7875,8032,8037,8365,8587,8591,8598,9013,9015,9160,9183,9193,9480,10220,10227,10246,10267,10291,10306,10346,10750,10760,10868,11389,11408,11416-11417,11456,11489,11502,11520,11643"}}
+{"timestamp":1714184702.290005,"name":"online","context":{"idset":"197,230,296,9123,10229,10656"}}
+{"timestamp":1714184702.4913731,"name":"online","context":{"idset":"109,116,1220,6546,6613,7311,7335,7359,7375,7777,7839,8108,8203,8255,8274,8281,8292,8331,8367,8374,8377-8378,8422,8436,8663,8880,8885,8906,8911,8933,8938,8961,9014,9107,9115,9141,9191,9222-9223,9300,9309,9349,9383,9412,9420,9861,9968,10158,10166,10201,10219,10260,10279,10314,10323,10326,10347,10358,10393,10396,10429,10613,10638,10662,10674,10691,10732,10858,10874,10913,10945,10964,11024,11087,11126,11145,11171,11227,11275,11291,11307,11358,11360,11403,11409,11454,11467,11470,11500,11567,11617,11631,11659,11769,11786,11791,11793,11799,11809,11867"}}
+{"timestamp":1714184702.5971582,"name":"online","context":{"idset":"9840"}}
+{"timestamp":1714184703.3335969,"name":"online","context":{"idset":"1236,6519,6521,6537,6563,6574,6592,6598,7395,7397,7801,7820,7884,7906,7919,8006-8008,8020,8041,8051,8077,8163,8205,8211,8213,8268,8297,8300,8326,8329,8335,8388,8404,8408,8417,8566,8590,8635,8654,8658,8691,8896,8925,8936,8943,8955,8965,9045,9119,9159,9210,9214,9239,9265,9280,9292,9331,9366,9376,9437,9453,9457,9477,9563,9728,9744,9770,9853,9876,9895,9927,10145,10215,10254,10351,10367,10380,10383,10432,10435,10453,10754,10767-10768,10814,10852,10863,10881,10885,10947,10986,11038,11066,11152,11160,11163,11180,11209,11244,11254,11341,11382,11494,11524,11558,11606,11619,11664,11687,11689,11752,11766,11838,11855"}}
+{"timestamp":1714184703.3378446,"name":"online","context":{"idset":"318,7748,7950,8579,8683,8976,9129,9132,10233,10305,10322,10697,11018,11475,11540"}}
+{"timestamp":1714184703.3418536,"name":"online","context":{"idset":"45,166,177,212,246,292,7735,7964,7972,8001,9083,10236,10273,10335,10618,11395,11411,11463"}}
+{"timestamp":1714184703.3457959,"name":"online","context":{"idset":"3,12,34,49-50,52,104,122,157,231,413,9021,9025,10199,10353,11455,11459"}}
+{"timestamp":1714184703.4069505,"name":"online","context":{"idset":"13,7980,10626"}}
+{"timestamp":1714184703.6265571,"name":"online","context":{"idset":"37,7960,8042,10616,10632"}}
+{"timestamp":1714184703.8305166,"name":"online","context":{"idset":"58"}}
+{"timestamp":1714184703.9421849,"name":"online","context":{"idset":"9663"}}
+{"timestamp":1714184704.1659055,"name":"online","context":{"idset":"7967"}}
+{"timestamp":1714184704.3615139,"name":"online","context":{"idset":"8047"}}
+{"timestamp":1714184704.4756162,"name":"online","context":{"idset":"7989"}}
+{"timestamp":1714184705.2966142,"name":"online","context":{"idset":"8038"}}
+{"timestamp":1714184705.3005745,"name":"online","context":{"idset":"8002"}}
+{"timestamp":1714184706.2167087,"name":"online","context":{"idset":"7698,7706,7930,9703,9999,10024,10497,10550"}}
+{"timestamp":1714184706.4601052,"name":"online","context":{"idset":"7678,7711,7756,7762-7763,9981,9996,10036,10038,10043,10056,10089,10095,10504,10522,10559,10587,11882"}}
+{"timestamp":1714184706.5526404,"name":"online","context":{"idset":"9666,9675,9994,10053,10074"}}
+{"timestamp":1714184707.8480742,"name":"online","context":{"idset":"7671,7687,7689,7693,7695,7699-7700,7739,7753,9973,9991,10028,10058,10075-10076,10485,10493-10494,10503,10509,10517,10530,10539,10541,10551,10573,10588-10589,10605,11880-11881"}}
+{"timestamp":1714184707.8531594,"name":"online","context":{"idset":"9594,9617,9678,10003,10515,10531"}}
+{"timestamp":1714184707.8577981,"name":"online","context":{"idset":"9616,9661"}}
+{"timestamp":1714184707.8624196,"name":"online","context":{"idset":"9597,9606,9643,9691"}}
+{"timestamp":1714184707.8670702,"name":"online","context":{"idset":"9613,9641,9676,9686"}}
+{"timestamp":1714184707.8716085,"name":"online","context":{"idset":"9611,9659,9680"}}
+{"timestamp":1714184783.4473011,"name":"online","context":{"idset":"10040"}}
+{"timestamp":1714184868.3700786,"name":"online","context":{"idset":"574"}}
+{"timestamp":1714184978.0812061,"name":"online","context":{"idset":"9067"}}
+{"timestamp":1714184978.086251,"name":"online","context":{"idset":"9061,9064"}}
+{"timestamp":1714184978.3302462,"name":"online","context":{"idset":"9062"}}
+{"timestamp":1714184980.1716356,"name":"online","context":{"idset":"9065"}}
+{"timestamp":1714184981.240824,"name":"online","context":{"idset":"9063"}}
+{"timestamp":1714184984.26265,"name":"online","context":{"idset":"9068"}}
+{"timestamp":1714184992.6301608,"name":"online","context":{"idset":"9066"}}
+{"timestamp":1714185126.4562933,"name":"offline","context":{"idset":"10617"}}
+{"timestamp":1714185126.5610125,"name":"offline","context":{"idset":"10631"}}
+{"timestamp":1714185126.5800612,"name":"offline","context":{"idset":"10667"}}
+{"timestamp":1714185126.5898628,"name":"offline","context":{"idset":"10677"}}
+{"timestamp":1714185126.597029,"name":"offline","context":{"idset":"10657"}}
+{"timestamp":1714185126.6844132,"name":"offline","context":{"idset":"10633"}}
+{"timestamp":1714185127.54264,"name":"offline","context":{"idset":"10682"}}
+{"timestamp":1714185127.5474796,"name":"offline","context":{"idset":"10708"}}
+{"timestamp":1714185127.5522699,"name":"offline","context":{"idset":"10658"}}
+{"timestamp":1714185127.5571334,"name":"offline","context":{"idset":"10654"}}
+{"timestamp":1714185127.5619578,"name":"offline","context":{"idset":"10640"}}
+{"timestamp":1714185127.5667922,"name":"offline","context":{"idset":"10660"}}
+{"timestamp":1714185127.5716085,"name":"offline","context":{"idset":"10648"}}
+{"timestamp":1714185127.5763719,"name":"offline","context":{"idset":"10679"}}
+{"timestamp":1714185127.5811636,"name":"offline","context":{"idset":"10627"}}
+{"timestamp":1714185127.5859253,"name":"offline","context":{"idset":"10692"}}
+{"timestamp":1714185127.5907183,"name":"offline","context":{"idset":"10614"}}
+{"timestamp":1714185127.5955124,"name":"offline","context":{"idset":"10661"}}
+{"timestamp":1714185127.6002905,"name":"offline","context":{"idset":"10613"}}
+{"timestamp":1714185127.6050591,"name":"offline","context":{"idset":"10646"}}
+{"timestamp":1714185127.6098709,"name":"offline","context":{"idset":"10634"}}
+{"timestamp":1714185127.6146233,"name":"offline","context":{"idset":"10703"}}
+{"timestamp":1714185127.6193957,"name":"offline","context":{"idset":"10635"}}
+{"timestamp":1714185127.6241696,"name":"offline","context":{"idset":"10659"}}
+{"timestamp":1714185127.6289289,"name":"offline","context":{"idset":"10666"}}
+{"timestamp":1714185127.6337597,"name":"offline","context":{"idset":"10668"}}
+{"timestamp":1714185127.6385393,"name":"offline","context":{"idset":"10632"}}
+{"timestamp":1714185127.6433055,"name":"offline","context":{"idset":"10637"}}
+{"timestamp":1714185127.6480873,"name":"offline","context":{"idset":"10622"}}
+{"timestamp":1714185127.6644049,"name":"offline","context":{"idset":"10624"}}
+{"timestamp":1714185127.6733313,"name":"offline","context":{"idset":"10707"}}
+{"timestamp":1714185127.6782203,"name":"offline","context":{"idset":"10665"}}
+{"timestamp":1714185127.6830375,"name":"offline","context":{"idset":"10684"}}
+{"timestamp":1714185127.6879649,"name":"offline","context":{"idset":"10698"}}
+{"timestamp":1714185127.6927397,"name":"offline","context":{"idset":"10620"}}
+{"timestamp":1714185127.6975391,"name":"offline","context":{"idset":"10623"}}
+{"timestamp":1714185127.7023439,"name":"offline","context":{"idset":"10710"}}
+{"timestamp":1714185127.7071338,"name":"offline","context":{"idset":"10655"}}
+{"timestamp":1714185127.7119396,"name":"offline","context":{"idset":"10628"}}
+{"timestamp":1714185127.7166886,"name":"offline","context":{"idset":"10686"}}
+{"timestamp":1714185127.7214587,"name":"offline","context":{"idset":"10663"}}
+{"timestamp":1714185127.7262383,"name":"offline","context":{"idset":"10705"}}
+{"timestamp":1714185127.7310786,"name":"offline","context":{"idset":"10739"}}
+{"timestamp":1714185127.7358463,"name":"offline","context":{"idset":"10630"}}
+{"timestamp":1714185127.740602,"name":"offline","context":{"idset":"10638"}}
+{"timestamp":1714185127.7489817,"name":"offline","context":{"idset":"10636"}}
+{"timestamp":1714185127.7537658,"name":"offline","context":{"idset":"10643"}}
+{"timestamp":1714185127.7585561,"name":"offline","context":{"idset":"10650"}}
+{"timestamp":1714185127.7633471,"name":"offline","context":{"idset":"10693"}}
+{"timestamp":1714185127.768116,"name":"offline","context":{"idset":"10616"}}
+{"timestamp":1714185127.7728698,"name":"offline","context":{"idset":"10629"}}
+{"timestamp":1714185127.7777464,"name":"offline","context":{"idset":"10714"}}
+{"timestamp":1714185127.7826288,"name":"offline","context":{"idset":"10704"}}
+{"timestamp":1714185127.7874618,"name":"offline","context":{"idset":"10713"}}
+{"timestamp":1714185127.7922571,"name":"offline","context":{"idset":"10618"}}
+{"timestamp":1714185127.7970622,"name":"offline","context":{"idset":"10721"}}
+{"timestamp":1714185127.8017826,"name":"offline","context":{"idset":"10652"}}
+{"timestamp":1714185127.8065982,"name":"offline","context":{"idset":"10669"}}
+{"timestamp":1714185127.811372,"name":"offline","context":{"idset":"10730"}}
+{"timestamp":1714185127.8160965,"name":"offline","context":{"idset":"10737"}}
+{"timestamp":1714185127.8208461,"name":"offline","context":{"idset":"10680"}}
+{"timestamp":1714185127.8255913,"name":"offline","context":{"idset":"10615"}}
+{"timestamp":1714185127.8303511,"name":"offline","context":{"idset":"10626"}}
+{"timestamp":1714185127.8350716,"name":"offline","context":{"idset":"10709"}}
+{"timestamp":1714185127.8397784,"name":"offline","context":{"idset":"10723"}}
+{"timestamp":1714185127.8445168,"name":"offline","context":{"idset":"10645"}}
+{"timestamp":1714185127.8492424,"name":"offline","context":{"idset":"10656"}}
+{"timestamp":1714185127.8539627,"name":"offline","context":{"idset":"10641"}}
+{"timestamp":1714185127.8587389,"name":"offline","context":{"idset":"10715"}}
+{"timestamp":1714185127.8635273,"name":"offline","context":{"idset":"10738"}}
+{"timestamp":1714185127.8682864,"name":"offline","context":{"idset":"10702"}}
+{"timestamp":1714185127.8730178,"name":"offline","context":{"idset":"10724"}}
+{"timestamp":1714185127.8794763,"name":"offline","context":{"idset":"10716"}}
+{"timestamp":1714185127.8867898,"name":"offline","context":{"idset":"10691"}}
+{"timestamp":1714185127.8915799,"name":"offline","context":{"idset":"10687"}}
+{"timestamp":1714185127.8963125,"name":"offline","context":{"idset":"10688"}}
+{"timestamp":1714185127.9010465,"name":"offline","context":{"idset":"10683"}}
+{"timestamp":1714185127.9057946,"name":"offline","context":{"idset":"10670"}}
+{"timestamp":1714185127.9105308,"name":"offline","context":{"idset":"10619"}}
+{"timestamp":1714185127.9152825,"name":"offline","context":{"idset":"10681"}}
+{"timestamp":1714185127.9200099,"name":"offline","context":{"idset":"10706"}}
+{"timestamp":1714185127.9247403,"name":"offline","context":{"idset":"10647"}}
+{"timestamp":1714185127.9294446,"name":"offline","context":{"idset":"10733"}}
+{"timestamp":1714185127.9341249,"name":"offline","context":{"idset":"10671"}}
+{"timestamp":1714185127.9390204,"name":"offline","context":{"idset":"10664"}}
+{"timestamp":1714185127.9437504,"name":"offline","context":{"idset":"10725"}}
+{"timestamp":1714185127.9484704,"name":"offline","context":{"idset":"10662"}}
+{"timestamp":1714185128.5447259,"name":"offline","context":{"idset":"10644"}}
+{"timestamp":1714185128.5475881,"name":"offline","context":{"idset":"10720"}}
+{"timestamp":1714185128.649018,"name":"offline","context":{"idset":"10729"}}
+{"timestamp":1714185129.0095198,"name":"offline","context":{"idset":"10672"}}
+{"timestamp":1714185129.0118921,"name":"offline","context":{"idset":"10625"}}
+{"timestamp":1714185129.0142374,"name":"offline","context":{"idset":"10726"}}
+{"timestamp":1714185129.0165803,"name":"offline","context":{"idset":"10690"}}
+{"timestamp":1714185129.0189333,"name":"offline","context":{"idset":"10678"}}
+{"timestamp":1714185129.0212467,"name":"offline","context":{"idset":"10701"}}
+{"timestamp":1714185129.0235565,"name":"offline","context":{"idset":"10642"}}
+{"timestamp":1714185129.0258603,"name":"offline","context":{"idset":"10717"}}
+{"timestamp":1714185129.0281734,"name":"offline","context":{"idset":"10676"}}
+{"timestamp":1714185129.0304797,"name":"offline","context":{"idset":"10732"}}
+{"timestamp":1714185129.0327876,"name":"offline","context":{"idset":"10735"}}
+{"timestamp":1714185129.0351069,"name":"offline","context":{"idset":"10731"}}
+{"timestamp":1714185129.0374243,"name":"offline","context":{"idset":"10712"}}
+{"timestamp":1714185129.0397274,"name":"offline","context":{"idset":"10685"}}
+{"timestamp":1714185129.0420308,"name":"offline","context":{"idset":"10734"}}
+{"timestamp":1714185129.0443339,"name":"offline","context":{"idset":"10696"}}
+{"timestamp":1714185129.0466437,"name":"offline","context":{"idset":"10674"}}
+{"timestamp":1714185129.0490425,"name":"offline","context":{"idset":"10727"}}
+{"timestamp":1714185129.0533099,"name":"offline","context":{"idset":"10700"}}
+{"timestamp":1714185129.0576377,"name":"offline","context":{"idset":"10697"}}
+{"timestamp":1714185129.0619891,"name":"offline","context":{"idset":"10711"}}
+{"timestamp":1714185129.0648985,"name":"offline","context":{"idset":"10675"}}
+{"timestamp":1714185129.067215,"name":"offline","context":{"idset":"10651"}}
+{"timestamp":1714185129.0695262,"name":"offline","context":{"idset":"10695"}}
+{"timestamp":1714185129.0718405,"name":"offline","context":{"idset":"10639"}}
+{"timestamp":1714185129.0741446,"name":"offline","context":{"idset":"10649"}}
+{"timestamp":1714185129.0764413,"name":"offline","context":{"idset":"10621"}}
+{"timestamp":1714185129.0787227,"name":"offline","context":{"idset":"10694"}}
+{"timestamp":1714185129.0810173,"name":"offline","context":{"idset":"10689"}}
+{"timestamp":1714185129.0833209,"name":"offline","context":{"idset":"10719"}}
+{"timestamp":1714185129.0856252,"name":"offline","context":{"idset":"10722"}}
+{"timestamp":1714185129.0879388,"name":"offline","context":{"idset":"10728"}}
+{"timestamp":1714185129.0902624,"name":"offline","context":{"idset":"10736"}}
+{"timestamp":1714185129.0925667,"name":"offline","context":{"idset":"10718"}}
+{"timestamp":1714185129.0948913,"name":"offline","context":{"idset":"10653"}}
+{"timestamp":1714185129.0972159,"name":"offline","context":{"idset":"10699"}}
+{"timestamp":1714185129.0995302,"name":"offline","context":{"idset":"10740"}}
+{"timestamp":1714185129.1124518,"name":"offline","context":{"idset":"10673"}}
+{"timestamp":1714185170.079879,"name":"online","context":{"idset":"10621"}}
+{"timestamp":1714185170.0829005,"name":"online","context":{"idset":"10626,10666"}}
+{"timestamp":1714185170.0859175,"name":"online","context":{"idset":"10721"}}
+{"timestamp":1714185170.3959897,"name":"online","context":{"idset":"10654"}}
+{"timestamp":1714185173.4008422,"name":"online","context":{"idset":"10667"}}
+{"timestamp":1714185174.0458455,"name":"online","context":{"idset":"10657"}}
+{"timestamp":1714185174.0497243,"name":"online","context":{"idset":"10637"}}
+{"timestamp":1714185175.8713086,"name":"online","context":{"idset":"10660,10703"}}
+{"timestamp":1714185175.8744173,"name":"online","context":{"idset":"10718"}}
+{"timestamp":1714185176.3996902,"name":"online","context":{"idset":"10681"}}
+{"timestamp":1714185178.4874094,"name":"online","context":{"idset":"10663"}}
+{"timestamp":1714185178.4914575,"name":"online","context":{"idset":"10624"}}
+{"timestamp":1714185178.6135507,"name":"online","context":{"idset":"10696"}}
+{"timestamp":1714185182.7299204,"name":"online","context":{"idset":"10641"}}
+{"timestamp":1714185184.1762743,"name":"online","context":{"idset":"10676"}}
+{"timestamp":1714185184.1801267,"name":"online","context":{"idset":"10618"}}
+{"timestamp":1714185184.1843445,"name":"online","context":{"idset":"10684"}}
+{"timestamp":1714185184.2705085,"name":"online","context":{"idset":"10688"}}
+{"timestamp":1714185185.4481382,"name":"online","context":{"idset":"10636"}}
+{"timestamp":1714185185.4523582,"name":"online","context":{"idset":"10630"}}
+{"timestamp":1714185186.0704341,"name":"online","context":{"idset":"10635,10662"}}
+{"timestamp":1714185186.1707647,"name":"online","context":{"idset":"10639"}}
+{"timestamp":1714185186.4327574,"name":"online","context":{"idset":"10614,10619,10627"}}
+{"timestamp":1714185187.3506939,"name":"online","context":{"idset":"10668"}}
+{"timestamp":1714185187.3545396,"name":"online","context":{"idset":"10700,10725"}}
+{"timestamp":1714185187.3583906,"name":"online","context":{"idset":"10701"}}
+{"timestamp":1714185187.3622234,"name":"online","context":{"idset":"10617,10620,10699,10719"}}
+{"timestamp":1714185187.3661168,"name":"online","context":{"idset":"10715"}}
+{"timestamp":1714185187.978966,"name":"online","context":{"idset":"10710"}}
+{"timestamp":1714185188.0262294,"name":"online","context":{"idset":"10634"}}
+{"timestamp":1714185188.0301092,"name":"online","context":{"idset":"10616,10655,10717"}}
+{"timestamp":1714185188.0342033,"name":"online","context":{"idset":"10615,10648,10709,10712"}}
+{"timestamp":1714185188.0933292,"name":"online","context":{"idset":"10659"}}
+{"timestamp":1714185188.2119648,"name":"online","context":{"idset":"10650"}}
+{"timestamp":1714185188.6002364,"name":"online","context":{"idset":"10628,10645,10652,10679"}}
+{"timestamp":1714185188.6112266,"name":"online","context":{"idset":"10640,10740"}}
+{"timestamp":1714185188.6176791,"name":"online","context":{"idset":"10623"}}
+{"timestamp":1714185188.7180526,"name":"online","context":{"idset":"10713"}}
+{"timestamp":1714185190.1096971,"name":"online","context":{"idset":"10625,10629,10670,10674,10689-10690,10706"}}
+{"timestamp":1714185190.1147308,"name":"online","context":{"idset":"10613,10622,10646,10661,10664,10669,10672,10675,10683,10692,10722-10723,10734"}}
+{"timestamp":1714185190.1194711,"name":"online","context":{"idset":"10665,10708"}}
+{"timestamp":1714185190.1255414,"name":"online","context":{"idset":"10631,10643,10671,10680,10691,10695,10702,10711,10724,10729"}}
+{"timestamp":1714185190.1316025,"name":"online","context":{"idset":"10653"}}
+{"timestamp":1714185190.1383798,"name":"online","context":{"idset":"10673,10694,10736"}}
+{"timestamp":1714185190.1438806,"name":"online","context":{"idset":"10633,10705,10727"}}
+{"timestamp":1714185190.177577,"name":"online","context":{"idset":"10704"}}
+{"timestamp":1714185190.4399922,"name":"online","context":{"idset":"10644,10647,10677,10685,10687,10693,10697-10698,10707,10714,10726,10733,10735"}}
+{"timestamp":1714185191.3162069,"name":"online","context":{"idset":"10658,10686,10738"}}
+{"timestamp":1714185191.320195,"name":"online","context":{"idset":"10638,10649,10716,10737"}}
+{"timestamp":1714185191.3240256,"name":"online","context":{"idset":"10632,10642,10678,10730-10732"}}
+{"timestamp":1714185191.32795,"name":"online","context":{"idset":"10656,10682,10720,10728,10739"}}
+{"timestamp":1714185191.3319404,"name":"online","context":{"idset":"10651"}}
+{"timestamp":1714186778.2878726,"name":"offline","context":{"idset":"7301"}}
+{"timestamp":1714186778.292412,"name":"offline","context":{"idset":"7285"}}
+{"timestamp":1714186778.2972407,"name":"offline","context":{"idset":"7337"}}
+{"timestamp":1714186778.3017607,"name":"offline","context":{"idset":"7295"}}
+{"timestamp":1714186778.306289,"name":"offline","context":{"idset":"7323"}}
+{"timestamp":1714186778.3112643,"name":"offline","context":{"idset":"7305"}}
+{"timestamp":1714186778.3156474,"name":"offline","context":{"idset":"7297"}}
+{"timestamp":1714186778.3200333,"name":"offline","context":{"idset":"7291"}}
+{"timestamp":1714186778.3243937,"name":"offline","context":{"idset":"7325"}}
+{"timestamp":1714186778.3275061,"name":"offline","context":{"idset":"7314"}}
+{"timestamp":1714186778.3317211,"name":"offline","context":{"idset":"7329"}}
+{"timestamp":1714186778.3362465,"name":"offline","context":{"idset":"7343"}}
+{"timestamp":1714186778.3414719,"name":"offline","context":{"idset":"7324"}}
+{"timestamp":1714186778.3462946,"name":"offline","context":{"idset":"7365"}}
+{"timestamp":1714186778.3511865,"name":"offline","context":{"idset":"7290"}}
+{"timestamp":1714186778.354955,"name":"offline","context":{"idset":"7336"}}
+{"timestamp":1714186778.3598201,"name":"offline","context":{"idset":"7311"}}
+{"timestamp":1714186778.363894,"name":"offline","context":{"idset":"7366"}}
+{"timestamp":1714186778.368062,"name":"offline","context":{"idset":"7316"}}
+{"timestamp":1714186778.37236,"name":"offline","context":{"idset":"7332"}}
+{"timestamp":1714186778.3770046,"name":"offline","context":{"idset":"7310"}}
+{"timestamp":1714186778.3813245,"name":"offline","context":{"idset":"7287"}}
+{"timestamp":1714186778.3853202,"name":"offline","context":{"idset":"7306"}}
+{"timestamp":1714186778.3898611,"name":"offline","context":{"idset":"7368"}}
+{"timestamp":1714186778.3941708,"name":"offline","context":{"idset":"7369"}}
+{"timestamp":1714186778.3985956,"name":"offline","context":{"idset":"7344"}}
+{"timestamp":1714186778.4032872,"name":"offline","context":{"idset":"7299"}}
+{"timestamp":1714186778.4079399,"name":"offline","context":{"idset":"7293"}}
+{"timestamp":1714186778.4115751,"name":"offline","context":{"idset":"7308"}}
+{"timestamp":1714186778.4154124,"name":"offline","context":{"idset":"7286"}}
+{"timestamp":1714186778.4190679,"name":"offline","context":{"idset":"7342"}}
+{"timestamp":1714186778.4227762,"name":"offline","context":{"idset":"7356"}}
+{"timestamp":1714186778.4267082,"name":"offline","context":{"idset":"7389"}}
+{"timestamp":1714186778.4305608,"name":"offline","context":{"idset":"7346"}}
+{"timestamp":1714186778.4346759,"name":"offline","context":{"idset":"7348"}}
+{"timestamp":1714186778.4389119,"name":"offline","context":{"idset":"7357"}}
+{"timestamp":1714186778.4433632,"name":"offline","context":{"idset":"7333"}}
+{"timestamp":1714186778.4475429,"name":"offline","context":{"idset":"7322"}}
+{"timestamp":1714186778.4518664,"name":"offline","context":{"idset":"7355"}}
+{"timestamp":1714186778.4558802,"name":"offline","context":{"idset":"7363"}}
+{"timestamp":1714186778.4600995,"name":"offline","context":{"idset":"7307"}}
+{"timestamp":1714186778.464761,"name":"offline","context":{"idset":"7318"}}
+{"timestamp":1714186778.471077,"name":"offline","context":{"idset":"7302"}}
+{"timestamp":1714186778.4752891,"name":"offline","context":{"idset":"7320"}}
+{"timestamp":1714186778.47925,"name":"offline","context":{"idset":"7304"}}
+{"timestamp":1714186778.4839249,"name":"offline","context":{"idset":"7367"}}
+{"timestamp":1714186778.4876387,"name":"offline","context":{"idset":"7385"}}
+{"timestamp":1714186778.4913533,"name":"offline","context":{"idset":"7395"}}
+{"timestamp":1714186778.4951968,"name":"offline","context":{"idset":"7321"}}
+{"timestamp":1714186778.4992771,"name":"offline","context":{"idset":"7334"}}
+{"timestamp":1714186778.5050726,"name":"offline","context":{"idset":"7386"}}
+{"timestamp":1714186778.5089772,"name":"offline","context":{"idset":"7294"}}
+{"timestamp":1714186778.5128696,"name":"offline","context":{"idset":"7350"}}
+{"timestamp":1714186778.5168481,"name":"offline","context":{"idset":"7328"}}
+{"timestamp":1714186778.5211384,"name":"offline","context":{"idset":"7292"}}
+{"timestamp":1714186778.5254266,"name":"offline","context":{"idset":"7371"}}
+{"timestamp":1714186778.5297709,"name":"offline","context":{"idset":"7300"}}
+{"timestamp":1714186778.5339715,"name":"offline","context":{"idset":"7349"}}
+{"timestamp":1714186778.5394347,"name":"offline","context":{"idset":"7327"}}
+{"timestamp":1714186778.7210934,"name":"offline","context":{"idset":"7303"}}
+{"timestamp":1714186779.0917187,"name":"offline","context":{"idset":"7298"}}
+{"timestamp":1714186779.0965316,"name":"offline","context":{"idset":"7288"}}
+{"timestamp":1714186779.101248,"name":"offline","context":{"idset":"7296"}}
+{"timestamp":1714186779.1076314,"name":"offline","context":{"idset":"7309"}}
+{"timestamp":1714186779.1161108,"name":"offline","context":{"idset":"7313"}}
+{"timestamp":1714186779.1234527,"name":"offline","context":{"idset":"7315"}}
+{"timestamp":1714186779.1279752,"name":"offline","context":{"idset":"7317"}}
+{"timestamp":1714186779.1317983,"name":"offline","context":{"idset":"7331"}}
+{"timestamp":1714186779.13638,"name":"offline","context":{"idset":"7381"}}
+{"timestamp":1714186779.1417327,"name":"offline","context":{"idset":"7312"}}
+{"timestamp":1714186779.1462724,"name":"offline","context":{"idset":"7360"}}
+{"timestamp":1714186779.1463351,"name":"drain","context":{"idset":"7301-7316","reason":"epilog failed for jobid frr9z8SVRYB","overwrite":0}}
+{"timestamp":1714186779.1618154,"name":"offline","context":{"idset":"7372"}}
+{"timestamp":1714186779.1661994,"name":"offline","context":{"idset":"7339"}}
+{"timestamp":1714186779.1739717,"name":"offline","context":{"idset":"7375"}}
+{"timestamp":1714186779.18084,"name":"offline","context":{"idset":"7353"}}
+{"timestamp":1714186779.1852741,"name":"offline","context":{"idset":"7383"}}
+{"timestamp":1714186779.1900861,"name":"offline","context":{"idset":"7352"}}
+{"timestamp":1714186779.1947103,"name":"offline","context":{"idset":"7326"}}
+{"timestamp":1714186779.1989605,"name":"offline","context":{"idset":"7338"}}
+{"timestamp":1714186779.202611,"name":"offline","context":{"idset":"7409"}}
+{"timestamp":1714186779.2072282,"name":"offline","context":{"idset":"7408"}}
+{"timestamp":1714186779.2161353,"name":"offline","context":{"idset":"7404"}}
+{"timestamp":1714186779.2197425,"name":"offline","context":{"idset":"7398"}}
+{"timestamp":1714186780.0279403,"name":"offline","context":{"idset":"7393"}}
+{"timestamp":1714186780.0329058,"name":"offline","context":{"idset":"7335"}}
+{"timestamp":1714186780.0378747,"name":"offline","context":{"idset":"7384"}}
+{"timestamp":1714186780.0520627,"name":"offline","context":{"idset":"7358"}}
+{"timestamp":1714186780.0558739,"name":"offline","context":{"idset":"7396"}}
+{"timestamp":1714186780.5170097,"name":"offline","context":{"idset":"7345"}}
+{"timestamp":1714186780.5194664,"name":"offline","context":{"idset":"7362"}}
+{"timestamp":1714186780.5219057,"name":"offline","context":{"idset":"7399"}}
+{"timestamp":1714186780.5249441,"name":"offline","context":{"idset":"7319"}}
+{"timestamp":1714186780.5273814,"name":"offline","context":{"idset":"7373"}}
+{"timestamp":1714186780.5571365,"name":"offline","context":{"idset":"7380"}}
+{"timestamp":1714186780.6230285,"name":"offline","context":{"idset":"7347"}}
+{"timestamp":1714186780.6678841,"name":"offline","context":{"idset":"7340"}}
+{"timestamp":1714186780.6703696,"name":"offline","context":{"idset":"7397"}}
+{"timestamp":1714186780.6728518,"name":"offline","context":{"idset":"7379"}}
+{"timestamp":1714186780.6753054,"name":"offline","context":{"idset":"7354"}}
+{"timestamp":1714186780.6778357,"name":"offline","context":{"idset":"7412"}}
+{"timestamp":1714186780.6807747,"name":"offline","context":{"idset":"7410"}}
+{"timestamp":1714186780.6832352,"name":"offline","context":{"idset":"7361"}}
+{"timestamp":1714186780.685673,"name":"offline","context":{"idset":"7406"}}
+{"timestamp":1714186780.6886859,"name":"offline","context":{"idset":"7289"}}
+{"timestamp":1714186780.691139,"name":"offline","context":{"idset":"7341"}}
+{"timestamp":1714186780.6936188,"name":"offline","context":{"idset":"7351"}}
+{"timestamp":1714186780.6966574,"name":"offline","context":{"idset":"7374"}}
+{"timestamp":1714186780.6993856,"name":"offline","context":{"idset":"7376"}}
+{"timestamp":1714186780.6994243,"name":"drain","context":{"idset":"7285-7300","reason":"epilog failed for jobid frr9xUWVtv3","overwrite":0}}
+{"timestamp":1714186780.6994624,"name":"drain","context":{"idset":"7333-7348","reason":"epilog failed for jobid frrA3M352jR","overwrite":0}}
+{"timestamp":1714186780.701952,"name":"offline","context":{"idset":"7382"}}
+{"timestamp":1714186780.7044108,"name":"offline","context":{"idset":"7387"}}
+{"timestamp":1714186780.7068555,"name":"offline","context":{"idset":"7402"}}
+{"timestamp":1714186780.7092927,"name":"offline","context":{"idset":"7403"}}
+{"timestamp":1714186780.7117288,"name":"offline","context":{"idset":"7359"}}
+{"timestamp":1714186780.7141809,"name":"offline","context":{"idset":"7330"}}
+{"timestamp":1714186780.7166228,"name":"offline","context":{"idset":"7391"}}
+{"timestamp":1714186780.719053,"name":"offline","context":{"idset":"7411"}}
+{"timestamp":1714186780.7215033,"name":"offline","context":{"idset":"7377"}}
+{"timestamp":1714186780.7239356,"name":"offline","context":{"idset":"7401"}}
+{"timestamp":1714186780.7263632,"name":"offline","context":{"idset":"7390"}}
+{"timestamp":1714186780.7287943,"name":"offline","context":{"idset":"7394"}}
+{"timestamp":1714186780.7312231,"name":"offline","context":{"idset":"7364"}}
+{"timestamp":1714186780.7312515,"name":"drain","context":{"idset":"7317-7332","reason":"epilog failed for jobid frrA1nRSvj1","overwrite":0}}
+{"timestamp":1714186780.7337091,"name":"offline","context":{"idset":"7388"}}
+{"timestamp":1714186780.7337437,"name":"drain","context":{"idset":"7349-7364","reason":"epilog failed for jobid frrA5LLqAxw","overwrite":0}}
+{"timestamp":1714186780.7361948,"name":"offline","context":{"idset":"7405"}}
+{"timestamp":1714186780.738647,"name":"offline","context":{"idset":"7370"}}
+{"timestamp":1714186780.7410843,"name":"offline","context":{"idset":"7378"}}
+{"timestamp":1714186780.741111,"name":"drain","context":{"idset":"7365-7380","reason":"epilog failed for jobid frrA7HZ2L6B","overwrite":0}}
+{"timestamp":1714186780.743552,"name":"offline","context":{"idset":"7392"}}
+{"timestamp":1714186780.7459824,"name":"offline","context":{"idset":"7400"}}
+{"timestamp":1714186780.7460098,"name":"drain","context":{"idset":"7381-7396","reason":"epilog failed for jobid frrA8ubJmro","overwrite":0}}
+{"timestamp":1714186780.7460425,"name":"drain","context":{"idset":"7397-7412","reason":"epilog failed for jobid frrAAkzxk1m","overwrite":0}}
+{"timestamp":1714186780.7484713,"name":"offline","context":{"idset":"7407"}}
+{"timestamp":1714186852.6120789,"name":"online","context":{"idset":"7326"}}
+{"timestamp":1714186854.06897,"name":"online","context":{"idset":"7289"}}
+{"timestamp":1714186856.5739818,"name":"online","context":{"idset":"7310"}}
+{"timestamp":1714186856.5791116,"name":"online","context":{"idset":"7360"}}
+{"timestamp":1714186856.5843241,"name":"online","context":{"idset":"7343"}}
+{"timestamp":1714186856.5925596,"name":"online","context":{"idset":"7301"}}
+{"timestamp":1714186869.9581833,"name":"online","context":{"idset":"7307"}}
+{"timestamp":1714186869.9623303,"name":"online","context":{"idset":"7295"}}
+{"timestamp":1714186869.966274,"name":"online","context":{"idset":"7349"}}
+{"timestamp":1714186870.328372,"name":"online","context":{"idset":"7316"}}
+{"timestamp":1714186871.9702861,"name":"online","context":{"idset":"7359"}}
+{"timestamp":1714186872.2687755,"name":"online","context":{"idset":"7306"}}
+{"timestamp":1714186874.2555938,"name":"online","context":{"idset":"7374"}}
+{"timestamp":1714186876.1329596,"name":"online","context":{"idset":"7285"}}
+{"timestamp":1714186876.1366434,"name":"online","context":{"idset":"7312"}}
+{"timestamp":1714186876.363287,"name":"online","context":{"idset":"7331"}}
+{"timestamp":1714186876.4911172,"name":"online","context":{"idset":"7296,7308"}}
+{"timestamp":1714186876.6133649,"name":"online","context":{"idset":"7304"}}
+{"timestamp":1714186878.4243541,"name":"online","context":{"idset":"7302"}}
+{"timestamp":1714186878.4281063,"name":"online","context":{"idset":"7309"}}
+{"timestamp":1714186878.4316249,"name":"online","context":{"idset":"7314,7335,7396"}}
+{"timestamp":1714186878.4350457,"name":"online","context":{"idset":"7315"}}
+{"timestamp":1714186878.4385767,"name":"online","context":{"idset":"7410"}}
+{"timestamp":1714186878.6119232,"name":"online","context":{"idset":"7292,7401"}}
+{"timestamp":1714186879.7008348,"name":"online","context":{"idset":"7305,7364"}}
+{"timestamp":1714186879.7032962,"name":"online","context":{"idset":"7286"}}
+{"timestamp":1714186879.7056656,"name":"online","context":{"idset":"7303"}}
+{"timestamp":1714186879.7114184,"name":"online","context":{"idset":"7311"}}
+{"timestamp":1714186879.8769777,"name":"online","context":{"idset":"7313"}}
+{"timestamp":1714186880.3199828,"name":"online","context":{"idset":"7298,7321,7350"}}
+{"timestamp":1714186880.6171076,"name":"online","context":{"idset":"7318"}}
+{"timestamp":1714186881.8454025,"name":"online","context":{"idset":"7395"}}
+{"timestamp":1714186881.8495567,"name":"online","context":{"idset":"7403"}}
+{"timestamp":1714186881.8536901,"name":"online","context":{"idset":"7341"}}
+{"timestamp":1714186881.857825,"name":"online","context":{"idset":"7352"}}
+{"timestamp":1714186881.8625062,"name":"online","context":{"idset":"7337"}}
+{"timestamp":1714186882.0251791,"name":"online","context":{"idset":"7300"}}
+{"timestamp":1714186882.1599534,"name":"online","context":{"idset":"7351,7362-7363,7412"}}
+{"timestamp":1714186882.3215444,"name":"online","context":{"idset":"7287,7354,7356"}}
+{"timestamp":1714186882.6324017,"name":"online","context":{"idset":"7358"}}
+{"timestamp":1714186883.4649739,"name":"online","context":{"idset":"7294,7393"}}
+{"timestamp":1714186884.0789492,"name":"online","context":{"idset":"7361"}}
+{"timestamp":1714186884.0823638,"name":"online","context":{"idset":"7355"}}
+{"timestamp":1714186884.0858307,"name":"online","context":{"idset":"7325"}}
+{"timestamp":1714186884.0892572,"name":"online","context":{"idset":"7389"}}
+{"timestamp":1714186884.100297,"name":"online","context":{"idset":"7328,7339,7353"}}
+{"timestamp":1714186884.103792,"name":"online","context":{"idset":"7390"}}
+{"timestamp":1714186884.1074133,"name":"online","context":{"idset":"7357"}}
+{"timestamp":1714186884.1990886,"name":"online","context":{"idset":"7347"}}
+{"timestamp":1714186884.3325179,"name":"online","context":{"idset":"7291"}}
+{"timestamp":1714186884.4521239,"name":"online","context":{"idset":"7290"}}
+{"timestamp":1714186884.6170869,"name":"online","context":{"idset":"7297,7320,7345,7386"}}
+{"timestamp":1714186884.6210554,"name":"online","context":{"idset":"7376,7387"}}
+{"timestamp":1714186886.0803971,"name":"online","context":{"idset":"7332"}}
+{"timestamp":1714186886.0870576,"name":"online","context":{"idset":"7324,7333,7336"}}
+{"timestamp":1714186886.0931013,"name":"online","context":{"idset":"7383"}}
+{"timestamp":1714186886.0972774,"name":"online","context":{"idset":"7322,7330,7366,7368,7381"}}
+{"timestamp":1714186886.1004369,"name":"online","context":{"idset":"7373"}}
+{"timestamp":1714186886.1034479,"name":"online","context":{"idset":"7391"}}
+{"timestamp":1714186886.1064827,"name":"online","context":{"idset":"7370,7385"}}
+{"timestamp":1714186886.1096425,"name":"online","context":{"idset":"7293"}}
+{"timestamp":1714186886.1127787,"name":"online","context":{"idset":"7334"}}
+{"timestamp":1714186886.1158636,"name":"online","context":{"idset":"7340,7392,7394"}}
+{"timestamp":1714186886.3450248,"name":"online","context":{"idset":"7323,7344"}}
+{"timestamp":1714186886.4502776,"name":"online","context":{"idset":"7299"}}
+{"timestamp":1714186886.563273,"name":"online","context":{"idset":"7404"}}
+{"timestamp":1714186886.668926,"name":"online","context":{"idset":"7399,7402"}}
+{"timestamp":1714186887.8512065,"name":"online","context":{"idset":"7288,7317,7338,7342,7348,7378,7380,7382,7384,7398"}}
+{"timestamp":1714186887.8541212,"name":"online","context":{"idset":"7319,7346,7379,7388"}}
+{"timestamp":1714186887.8570573,"name":"online","context":{"idset":"7327,7372,7408"}}
+{"timestamp":1714186887.8598249,"name":"online","context":{"idset":"7369,7411"}}
+{"timestamp":1714186887.8626175,"name":"online","context":{"idset":"7329,7405,7407"}}
+{"timestamp":1714186887.8655028,"name":"online","context":{"idset":"7406"}}
+{"timestamp":1714186888.105212,"name":"online","context":{"idset":"7397,7400"}}
+{"timestamp":1714186888.3975334,"name":"online","context":{"idset":"7371,7375,7377,7409"}}
+{"timestamp":1714186888.6896477,"name":"online","context":{"idset":"7367"}}
+{"timestamp":1714186890.0126448,"name":"online","context":{"idset":"7365"}}
+{"timestamp":1714187184.7578321,"name":"undrain","context":{"idset":"7285-7412"}}
+{"timestamp":1714187728.2543857,"name":"offline","context":{"idset":"6534"}}
+{"timestamp":1714187728.2595236,"name":"offline","context":{"idset":"6525"}}
+{"timestamp":1714187728.2644718,"name":"offline","context":{"idset":"6598"}}
+{"timestamp":1714187728.2694199,"name":"offline","context":{"idset":"6576"}}
+{"timestamp":1714187728.2743886,"name":"offline","context":{"idset":"6540"}}
+{"timestamp":1714187728.2793202,"name":"offline","context":{"idset":"6554"}}
+{"timestamp":1714187728.5513284,"name":"offline","context":{"idset":"6564"}}
+{"timestamp":1714187728.5593562,"name":"offline","context":{"idset":"6523"}}
+{"timestamp":1714187728.5673232,"name":"offline","context":{"idset":"6527"}}
+{"timestamp":1714187728.5747712,"name":"offline","context":{"idset":"6577"}}
+{"timestamp":1714187728.6584902,"name":"offline","context":{"idset":"6611"}}
+{"timestamp":1714187728.8884933,"name":"offline","context":{"idset":"6597"}}
+{"timestamp":1714187728.9052801,"name":"offline","context":{"idset":"6606"}}
+{"timestamp":1714187730.4958124,"name":"offline","context":{"idset":"6602"}}
+{"timestamp":1714187730.4994676,"name":"offline","context":{"idset":"6531"}}
+{"timestamp":1714187730.5030825,"name":"offline","context":{"idset":"6609"}}
+{"timestamp":1714187730.5066423,"name":"offline","context":{"idset":"6610"}}
+{"timestamp":1714187730.5102618,"name":"offline","context":{"idset":"6605"}}
+{"timestamp":1714187730.5138752,"name":"offline","context":{"idset":"6579"}}
+{"timestamp":1714187730.5174325,"name":"offline","context":{"idset":"6623"}}
+{"timestamp":1714187730.520968,"name":"offline","context":{"idset":"6542"}}
+{"timestamp":1714187730.5244803,"name":"offline","context":{"idset":"6563"}}
+{"timestamp":1714187730.5283377,"name":"offline","context":{"idset":"6524"}}
+{"timestamp":1714187730.5318847,"name":"offline","context":{"idset":"6526"}}
+{"timestamp":1714187730.5354121,"name":"offline","context":{"idset":"6528"}}
+{"timestamp":1714187730.5388906,"name":"offline","context":{"idset":"6529"}}
+{"timestamp":1714187730.5423915,"name":"offline","context":{"idset":"6530"}}
+{"timestamp":1714187730.5464308,"name":"offline","context":{"idset":"6532"}}
+{"timestamp":1714187730.5499556,"name":"offline","context":{"idset":"6533"}}
+{"timestamp":1714187730.5534735,"name":"offline","context":{"idset":"6535"}}
+{"timestamp":1714187730.5569234,"name":"offline","context":{"idset":"6537"}}
+{"timestamp":1714187730.5603373,"name":"offline","context":{"idset":"6538"}}
+{"timestamp":1714187730.5637515,"name":"offline","context":{"idset":"6539"}}
+{"timestamp":1714187730.5671678,"name":"offline","context":{"idset":"6541"}}
+{"timestamp":1714187730.5705485,"name":"offline","context":{"idset":"6543"}}
+{"timestamp":1714187730.5739174,"name":"offline","context":{"idset":"6544"}}
+{"timestamp":1714187730.5772777,"name":"offline","context":{"idset":"6545"}}
+{"timestamp":1714187730.5806439,"name":"offline","context":{"idset":"6547"}}
+{"timestamp":1714187730.5839963,"name":"offline","context":{"idset":"6548"}}
+{"timestamp":1714187730.5873358,"name":"offline","context":{"idset":"6549"}}
+{"timestamp":1714187730.5906446,"name":"offline","context":{"idset":"6550"}}
+{"timestamp":1714187730.5939507,"name":"offline","context":{"idset":"6551"}}
+{"timestamp":1714187730.5973668,"name":"offline","context":{"idset":"6552"}}
+{"timestamp":1714187730.6006792,"name":"offline","context":{"idset":"6553"}}
+{"timestamp":1714187730.6039519,"name":"offline","context":{"idset":"6555"}}
+{"timestamp":1714187730.6072276,"name":"offline","context":{"idset":"6556"}}
+{"timestamp":1714187730.6104872,"name":"offline","context":{"idset":"6557"}}
+{"timestamp":1714187730.6137071,"name":"offline","context":{"idset":"6558"}}
+{"timestamp":1714187730.6169538,"name":"offline","context":{"idset":"6559"}}
+{"timestamp":1714187730.6201818,"name":"offline","context":{"idset":"6560"}}
+{"timestamp":1714187730.6233799,"name":"offline","context":{"idset":"6561"}}
+{"timestamp":1714187730.6266079,"name":"offline","context":{"idset":"6562"}}
+{"timestamp":1714187730.6298206,"name":"offline","context":{"idset":"6565"}}
+{"timestamp":1714187730.6330063,"name":"offline","context":{"idset":"6566"}}
+{"timestamp":1714187730.6361935,"name":"offline","context":{"idset":"6567"}}
+{"timestamp":1714187730.6393745,"name":"offline","context":{"idset":"6568"}}
+{"timestamp":1714187730.6425714,"name":"offline","context":{"idset":"6569"}}
+{"timestamp":1714187730.6457214,"name":"offline","context":{"idset":"6570"}}
+{"timestamp":1714187730.6488705,"name":"offline","context":{"idset":"6571"}}
+{"timestamp":1714187730.6520088,"name":"offline","context":{"idset":"6572"}}
+{"timestamp":1714187730.6552389,"name":"offline","context":{"idset":"6573"}}
+{"timestamp":1714187730.6583755,"name":"offline","context":{"idset":"6575"}}
+{"timestamp":1714187730.6614764,"name":"offline","context":{"idset":"6580"}}
+{"timestamp":1714187730.664567,"name":"offline","context":{"idset":"6581"}}
+{"timestamp":1714187730.6676645,"name":"offline","context":{"idset":"6582"}}
+{"timestamp":1714187730.670742,"name":"offline","context":{"idset":"6583"}}
+{"timestamp":1714187730.6738226,"name":"offline","context":{"idset":"6584"}}
+{"timestamp":1714187730.6768913,"name":"offline","context":{"idset":"6585"}}
+{"timestamp":1714187730.6799705,"name":"offline","context":{"idset":"6586"}}
+{"timestamp":1714187730.683027,"name":"offline","context":{"idset":"6587"}}
+{"timestamp":1714187730.6860719,"name":"offline","context":{"idset":"6588"}}
+{"timestamp":1714187730.6890812,"name":"offline","context":{"idset":"6589"}}
+{"timestamp":1714187730.6920969,"name":"offline","context":{"idset":"6590"}}
+{"timestamp":1714187730.6951251,"name":"offline","context":{"idset":"6591"}}
+{"timestamp":1714187730.6981397,"name":"offline","context":{"idset":"6592"}}
+{"timestamp":1714187730.7011294,"name":"offline","context":{"idset":"6594"}}
+{"timestamp":1714187730.7041168,"name":"offline","context":{"idset":"6595"}}
+{"timestamp":1714187730.7070994,"name":"offline","context":{"idset":"6596"}}
+{"timestamp":1714187730.7100687,"name":"offline","context":{"idset":"6599"}}
+{"timestamp":1714187730.7130852,"name":"offline","context":{"idset":"6600"}}
+{"timestamp":1714187730.7164657,"name":"offline","context":{"idset":"6601"}}
+{"timestamp":1714187730.720484,"name":"offline","context":{"idset":"6603"}}
+{"timestamp":1714187730.7248354,"name":"offline","context":{"idset":"6604"}}
+{"timestamp":1714187730.7280262,"name":"offline","context":{"idset":"6607"}}
+{"timestamp":1714187730.7309647,"name":"offline","context":{"idset":"6608"}}
+{"timestamp":1714187730.7338791,"name":"offline","context":{"idset":"6612"}}
+{"timestamp":1714187730.7367489,"name":"offline","context":{"idset":"6614"}}
+{"timestamp":1714187730.7396483,"name":"offline","context":{"idset":"6616"}}
+{"timestamp":1714187730.7425561,"name":"offline","context":{"idset":"6621"}}
+{"timestamp":1714187730.7454271,"name":"offline","context":{"idset":"6626"}}
+{"timestamp":1714187730.7482882,"name":"offline","context":{"idset":"6629"}}
+{"timestamp":1714187730.7511628,"name":"offline","context":{"idset":"6630"}}
+{"timestamp":1714187730.754081,"name":"offline","context":{"idset":"6631"}}
+{"timestamp":1714187730.7569389,"name":"offline","context":{"idset":"6632"}}
+{"timestamp":1714187730.7597811,"name":"offline","context":{"idset":"6633"}}
+{"timestamp":1714187730.76262,"name":"offline","context":{"idset":"6635"}}
+{"timestamp":1714187730.7654696,"name":"offline","context":{"idset":"6636"}}
+{"timestamp":1714187730.7683203,"name":"offline","context":{"idset":"6637"}}
+{"timestamp":1714187730.7711911,"name":"offline","context":{"idset":"6640"}}
+{"timestamp":1714187730.7740002,"name":"offline","context":{"idset":"6641"}}
+{"timestamp":1714187730.7767911,"name":"offline","context":{"idset":"6642"}}
+{"timestamp":1714187730.7796054,"name":"offline","context":{"idset":"6643"}}
+{"timestamp":1714187730.7823942,"name":"offline","context":{"idset":"6644"}}
+{"timestamp":1714187730.7851613,"name":"offline","context":{"idset":"6574"}}
+{"timestamp":1714187730.7879438,"name":"offline","context":{"idset":"6578"}}
+{"timestamp":1714187730.7907012,"name":"offline","context":{"idset":"6593"}}
+{"timestamp":1714187730.793452,"name":"offline","context":{"idset":"6639"}}
+{"timestamp":1714187730.8003602,"name":"offline","context":{"idset":"6628"}}
+{"timestamp":1714187730.8031158,"name":"offline","context":{"idset":"6634"}}
+{"timestamp":1714187730.8058572,"name":"offline","context":{"idset":"6617"}}
+{"timestamp":1714187730.8085709,"name":"offline","context":{"idset":"6620"}}
+{"timestamp":1714187730.8112812,"name":"offline","context":{"idset":"6627"}}
+{"timestamp":1714187730.8139999,"name":"offline","context":{"idset":"6613"}}
+{"timestamp":1714187730.8167295,"name":"offline","context":{"idset":"6546"}}
+{"timestamp":1714187730.819473,"name":"offline","context":{"idset":"6638"}}
+{"timestamp":1714187730.8222313,"name":"offline","context":{"idset":"6619"}}
+{"timestamp":1714187730.8250322,"name":"offline","context":{"idset":"6622"}}
+{"timestamp":1714187730.8277481,"name":"offline","context":{"idset":"6536"}}
+{"timestamp":1714187730.8304722,"name":"offline","context":{"idset":"6624"}}
+{"timestamp":1714187730.8332055,"name":"offline","context":{"idset":"6625"}}
+{"timestamp":1714187730.8359079,"name":"offline","context":{"idset":"6618"}}
+{"timestamp":1714187730.8385546,"name":"offline","context":{"idset":"6615"}}
+{"timestamp":1714187792.1982083,"name":"online","context":{"idset":"6535"}}
+{"timestamp":1714187794.1793804,"name":"online","context":{"idset":"6532"}}
+{"timestamp":1714187798.6234686,"name":"online","context":{"idset":"6558,6572"}}
+{"timestamp":1714187802.2553539,"name":"online","context":{"idset":"6569"}}
+{"timestamp":1714187802.2602098,"name":"online","context":{"idset":"6541"}}
+{"timestamp":1714187804.3151619,"name":"online","context":{"idset":"6544"}}
+{"timestamp":1714187804.3198762,"name":"online","context":{"idset":"6530"}}
+{"timestamp":1714187806.1144745,"name":"online","context":{"idset":"6553"}}
+{"timestamp":1714187806.6623077,"name":"online","context":{"idset":"6534"}}
+{"timestamp":1714187810.2544074,"name":"online","context":{"idset":"6605"}}
+{"timestamp":1714187812.3532009,"name":"online","context":{"idset":"6602"}}
+{"timestamp":1714187812.5693815,"name":"online","context":{"idset":"6524"}}
+{"timestamp":1714187814.0477691,"name":"online","context":{"idset":"6528"}}
+{"timestamp":1714187814.1315906,"name":"online","context":{"idset":"6526"}}
+{"timestamp":1714187814.2640665,"name":"online","context":{"idset":"6539"}}
+{"timestamp":1714187814.4671979,"name":"online","context":{"idset":"6585"}}
+{"timestamp":1714187816.7835248,"name":"online","context":{"idset":"6542"}}
+{"timestamp":1714187816.7861066,"name":"online","context":{"idset":"6523,6547,6562"}}
+{"timestamp":1714187817.7140861,"name":"online","context":{"idset":"6525"}}
+{"timestamp":1714187817.7170825,"name":"online","context":{"idset":"6533"}}
+{"timestamp":1714187817.7200744,"name":"online","context":{"idset":"6529,6537"}}
+{"timestamp":1714187817.8841319,"name":"online","context":{"idset":"6531,6545"}}
+{"timestamp":1714187818.1012969,"name":"online","context":{"idset":"6538"}}
+{"timestamp":1714187818.253449,"name":"online","context":{"idset":"6527"}}
+{"timestamp":1714187818.51705,"name":"online","context":{"idset":"6536,6556"}}
+{"timestamp":1714187819.5280571,"name":"online","context":{"idset":"6548"}}
+{"timestamp":1714187819.5333722,"name":"online","context":{"idset":"6579"}}
+{"timestamp":1714187819.538403,"name":"online","context":{"idset":"6543"}}
+{"timestamp":1714187819.5463188,"name":"online","context":{"idset":"6540"}}
+{"timestamp":1714187820.2885807,"name":"online","context":{"idset":"6546"}}
+{"timestamp":1714187820.2926235,"name":"online","context":{"idset":"6551,6607"}}
+{"timestamp":1714187820.2978277,"name":"online","context":{"idset":"6561"}}
+{"timestamp":1714187820.4743056,"name":"online","context":{"idset":"6573"}}
+{"timestamp":1714187821.9889281,"name":"online","context":{"idset":"6622"}}
+{"timestamp":1714187821.9926202,"name":"online","context":{"idset":"6644"}}
+{"timestamp":1714187821.9962707,"name":"online","context":{"idset":"6584"}}
+{"timestamp":1714187823.4166605,"name":"online","context":{"idset":"6620"}}
+{"timestamp":1714187824.0769556,"name":"online","context":{"idset":"6576"}}
+{"timestamp":1714187824.0808792,"name":"online","context":{"idset":"6591"}}
+{"timestamp":1714187824.0933268,"name":"online","context":{"idset":"6639"}}
+{"timestamp":1714187824.0972328,"name":"online","context":{"idset":"6566"}}
+{"timestamp":1714187824.1011481,"name":"online","context":{"idset":"6564"}}
+{"timestamp":1714187824.2653065,"name":"online","context":{"idset":"6587"}}
+{"timestamp":1714187824.3775094,"name":"online","context":{"idset":"6589"}}
+{"timestamp":1714187824.6034849,"name":"online","context":{"idset":"6580,6603"}}
+{"timestamp":1714187825.3984418,"name":"online","context":{"idset":"6581,6592"}}
+{"timestamp":1714187825.4027324,"name":"online","context":{"idset":"6583"}}
+{"timestamp":1714187825.4072134,"name":"online","context":{"idset":"6632"}}
+{"timestamp":1714187825.4119925,"name":"online","context":{"idset":"6575,6588,6595,6600"}}
+{"timestamp":1714187825.416671,"name":"online","context":{"idset":"6593"}}
+{"timestamp":1714187825.9986928,"name":"online","context":{"idset":"6550"}}
+{"timestamp":1714187826.001462,"name":"online","context":{"idset":"6601,6604"}}
+{"timestamp":1714187826.0040267,"name":"online","context":{"idset":"6568"}}
+{"timestamp":1714187826.0067925,"name":"online","context":{"idset":"6560,6597,6606"}}
+{"timestamp":1714187826.0097034,"name":"online","context":{"idset":"6557,6565,6571,6577,6586,6598-6599,6627"}}
+{"timestamp":1714187826.0798683,"name":"online","context":{"idset":"6552"}}
+{"timestamp":1714187826.2539194,"name":"online","context":{"idset":"6554,6574,6582,6594,6610-6611"}}
+{"timestamp":1714187826.6057703,"name":"online","context":{"idset":"6555,6567,6578,6590,6596,6609,6626,6630,6637"}}
+{"timestamp":1714187828.2462263,"name":"online","context":{"idset":"6549,6563,6570,6608,6612"}}
+{"timestamp":1714187828.2516379,"name":"online","context":{"idset":"6625"}}
+{"timestamp":1714187828.2569001,"name":"online","context":{"idset":"6559,6616,6638"}}
+{"timestamp":1714187828.2639697,"name":"online","context":{"idset":"6614,6619,6633,6642"}}
+{"timestamp":1714187828.2693999,"name":"online","context":{"idset":"6621,6643"}}
+{"timestamp":1714187828.2752404,"name":"online","context":{"idset":"6628,6640-6641"}}
+{"timestamp":1714187828.2813666,"name":"online","context":{"idset":"6615,6617"}}
+{"timestamp":1714187828.2880068,"name":"online","context":{"idset":"6634-6635"}}
+{"timestamp":1714187828.5732098,"name":"online","context":{"idset":"6623"}}
+{"timestamp":1714187830.0170212,"name":"online","context":{"idset":"6613,6618,6636"}}
+{"timestamp":1714187830.021034,"name":"online","context":{"idset":"6624,6631"}}
+{"timestamp":1714187830.0250502,"name":"online","context":{"idset":"6629"}}
+{"timestamp":1714188547.4008489,"name":"offline","context":{"idset":"6602"}}
+{"timestamp":1714188548.1669958,"name":"offline","context":{"idset":"6608"}}
+{"timestamp":1714188548.1710579,"name":"offline","context":{"idset":"6605"}}
+{"timestamp":1714188548.1750989,"name":"offline","context":{"idset":"6598"}}
+{"timestamp":1714188548.1795509,"name":"offline","context":{"idset":"6612"}}
+{"timestamp":1714188548.1839917,"name":"offline","context":{"idset":"6611"}}
+{"timestamp":1714188548.1883512,"name":"offline","context":{"idset":"6599"}}
+{"timestamp":1714188548.1926486,"name":"offline","context":{"idset":"6606"}}
+{"timestamp":1714188548.1971366,"name":"offline","context":{"idset":"6603"}}
+{"timestamp":1714188548.201606,"name":"offline","context":{"idset":"6604"}}
+{"timestamp":1714188548.2060862,"name":"offline","context":{"idset":"6600"}}
+{"timestamp":1714188548.2104325,"name":"offline","context":{"idset":"6601"}}
+{"timestamp":1714188548.2144065,"name":"offline","context":{"idset":"6610"}}
+{"timestamp":1714188548.2183897,"name":"offline","context":{"idset":"6609"}}
+{"timestamp":1714188548.2223749,"name":"offline","context":{"idset":"6597"}}
+{"timestamp":1714188548.2264197,"name":"offline","context":{"idset":"6607"}}
+{"timestamp":1714188618.6087193,"name":"online","context":{"idset":"6604"}}
+{"timestamp":1714188619.3346801,"name":"online","context":{"idset":"6609"}}
+{"timestamp":1714188619.7681887,"name":"online","context":{"idset":"6598"}}
+{"timestamp":1714188620.001682,"name":"online","context":{"idset":"6603"}}
+{"timestamp":1714188620.3063719,"name":"online","context":{"idset":"6612"}}
+{"timestamp":1714188620.5920248,"name":"online","context":{"idset":"6602,6607"}}
+{"timestamp":1714188622.0107651,"name":"online","context":{"idset":"6601"}}
+{"timestamp":1714188622.0147672,"name":"online","context":{"idset":"6608,6610"}}
+{"timestamp":1714188622.0190103,"name":"online","context":{"idset":"6611"}}
+{"timestamp":1714188622.0230205,"name":"online","context":{"idset":"6606"}}
+{"timestamp":1714188622.0272479,"name":"online","context":{"idset":"6597,6599,6605"}}
+{"timestamp":1714188622.0313637,"name":"online","context":{"idset":"6600"}}
+{"timestamp":1714191004.7355835,"name":"offline","context":{"idset":"6630"}}
+{"timestamp":1714201312.0995538,"name":"online","context":{"idset":"6630"}}
+{"timestamp":1714201381.2315154,"name":"offline","context":{"idset":"6598"}}
+{"timestamp":1714201381.6163309,"name":"offline","context":{"idset":"6527"}}
+{"timestamp":1714201381.7427704,"name":"offline","context":{"idset":"6546"}}
+{"timestamp":1714201381.8447204,"name":"offline","context":{"idset":"6600"}}
+{"timestamp":1714201381.8980653,"name":"offline","context":{"idset":"6612"}}
+{"timestamp":1714201381.9135256,"name":"offline","context":{"idset":"6549"}}
+{"timestamp":1714201381.9997208,"name":"offline","context":{"idset":"6525"}}
+{"timestamp":1714201382.1267667,"name":"offline","context":{"idset":"6601"}}
+{"timestamp":1714201382.131757,"name":"offline","context":{"idset":"6573"}}
+{"timestamp":1714201382.1454961,"name":"offline","context":{"idset":"6554"}}
+{"timestamp":1714201382.1505306,"name":"offline","context":{"idset":"6530"}}
+{"timestamp":1714201382.1824148,"name":"offline","context":{"idset":"6587"}}
+{"timestamp":1714201382.269779,"name":"offline","context":{"idset":"6542"}}
+{"timestamp":1714201382.4043813,"name":"offline","context":{"idset":"6561"}}
+{"timestamp":1714201382.4126542,"name":"offline","context":{"idset":"6563"}}
+{"timestamp":1714201382.4206686,"name":"offline","context":{"idset":"6539"}}
+{"timestamp":1714201382.4256418,"name":"offline","context":{"idset":"6607"}}
+{"timestamp":1714201382.4337008,"name":"offline","context":{"idset":"6555"}}
+{"timestamp":1714201382.4819643,"name":"offline","context":{"idset":"6562"}}
+{"timestamp":1714201382.4868665,"name":"offline","context":{"idset":"6538"}}
+{"timestamp":1714201382.4917142,"name":"offline","context":{"idset":"6547"}}
+{"timestamp":1714201382.507292,"name":"offline","context":{"idset":"6556"}}
+{"timestamp":1714201382.512188,"name":"offline","context":{"idset":"6566"}}
+{"timestamp":1714201382.5170817,"name":"offline","context":{"idset":"6572"}}
+{"timestamp":1714201382.5219738,"name":"offline","context":{"idset":"6575"}}
+{"timestamp":1714201382.5268228,"name":"offline","context":{"idset":"6583"}}
+{"timestamp":1714201382.5317266,"name":"offline","context":{"idset":"6603"}}
+{"timestamp":1714201382.661998,"name":"offline","context":{"idset":"6535"}}
+{"timestamp":1714201382.7728636,"name":"offline","context":{"idset":"6605"}}
+{"timestamp":1714201382.7764673,"name":"offline","context":{"idset":"6609"}}
+{"timestamp":1714201382.8165729,"name":"offline","context":{"idset":"6608"}}
+{"timestamp":1714201382.8211379,"name":"offline","context":{"idset":"6631"}}
+{"timestamp":1714201382.8249049,"name":"offline","context":{"idset":"6588"}}
+{"timestamp":1714201382.8304372,"name":"offline","context":{"idset":"6553"}}
+{"timestamp":1714201382.8342848,"name":"offline","context":{"idset":"6599"}}
+{"timestamp":1714201382.8483379,"name":"offline","context":{"idset":"6578"}}
+{"timestamp":1714201382.8570297,"name":"offline","context":{"idset":"6540"}}
+{"timestamp":1714201382.8657408,"name":"offline","context":{"idset":"6558"}}
+{"timestamp":1714201382.886122,"name":"offline","context":{"idset":"6592"}}
+{"timestamp":1714201382.9001908,"name":"offline","context":{"idset":"6602"}}
+{"timestamp":1714201382.9027138,"name":"offline","context":{"idset":"6531"}}
+{"timestamp":1714201382.9052453,"name":"offline","context":{"idset":"6550"}}
+{"timestamp":1714201382.9800355,"name":"offline","context":{"idset":"6593"}}
+{"timestamp":1714201383.0875344,"name":"offline","context":{"idset":"6544"}}
+{"timestamp":1714201383.092334,"name":"offline","context":{"idset":"6565"}}
+{"timestamp":1714201383.1010828,"name":"offline","context":{"idset":"6529"}}
+{"timestamp":1714201383.1058838,"name":"offline","context":{"idset":"6545"}}
+{"timestamp":1714201383.1106541,"name":"offline","context":{"idset":"6560"}}
+{"timestamp":1714201383.1579611,"name":"offline","context":{"idset":"6559"}}
+{"timestamp":1714201383.161478,"name":"offline","context":{"idset":"6524"}}
+{"timestamp":1714201383.1730208,"name":"offline","context":{"idset":"6526"}}
+{"timestamp":1714201383.1764009,"name":"offline","context":{"idset":"6528"}}
+{"timestamp":1714201383.179877,"name":"offline","context":{"idset":"6532"}}
+{"timestamp":1714201383.1833332,"name":"offline","context":{"idset":"6534"}}
+{"timestamp":1714201383.1869686,"name":"offline","context":{"idset":"6536"}}
+{"timestamp":1714201383.1905568,"name":"offline","context":{"idset":"6537"}}
+{"timestamp":1714201383.2095664,"name":"offline","context":{"idset":"6541"}}
+{"timestamp":1714201383.2129209,"name":"offline","context":{"idset":"6564"}}
+{"timestamp":1714201383.2175887,"name":"offline","context":{"idset":"6569"}}
+{"timestamp":1714201383.2227845,"name":"offline","context":{"idset":"6574"}}
+{"timestamp":1714201383.2281544,"name":"offline","context":{"idset":"6579"}}
+{"timestamp":1714201383.2319944,"name":"offline","context":{"idset":"6582"}}
+{"timestamp":1714201383.2521267,"name":"offline","context":{"idset":"6584"}}
+{"timestamp":1714201383.2552171,"name":"offline","context":{"idset":"6590"}}
+{"timestamp":1714201383.2579391,"name":"offline","context":{"idset":"6610"}}
+{"timestamp":1714201383.2611506,"name":"offline","context":{"idset":"6617"}}
+{"timestamp":1714201383.2640018,"name":"offline","context":{"idset":"6622"}}
+{"timestamp":1714201383.2666893,"name":"offline","context":{"idset":"6632"}}
+{"timestamp":1714201383.3059039,"name":"offline","context":{"idset":"6635"}}
+{"timestamp":1714201383.4617095,"name":"offline","context":{"idset":"7288"}}
+{"timestamp":1714201383.4651003,"name":"offline","context":{"idset":"6568"}}
+{"timestamp":1714201383.4685323,"name":"offline","context":{"idset":"6594"}}
+{"timestamp":1714201383.4711826,"name":"offline","context":{"idset":"6614"}}
+{"timestamp":1714201383.4741611,"name":"offline","context":{"idset":"6586"}}
+{"timestamp":1714201383.4772358,"name":"offline","context":{"idset":"6581"}}
+{"timestamp":1714201383.5741904,"name":"offline","context":{"idset":"6611"}}
+{"timestamp":1714201383.5782673,"name":"offline","context":{"idset":"6548"}}
+{"timestamp":1714201383.5836406,"name":"offline","context":{"idset":"6557"}}
+{"timestamp":1714201383.5875235,"name":"offline","context":{"idset":"6570"}}
+{"timestamp":1714201383.5922472,"name":"offline","context":{"idset":"6571"}}
+{"timestamp":1714201383.596199,"name":"offline","context":{"idset":"6638"}}
+{"timestamp":1714201383.690671,"name":"offline","context":{"idset":"6613"}}
+{"timestamp":1714201383.7029517,"name":"offline","context":{"idset":"6523"}}
+{"timestamp":1714201383.7078142,"name":"offline","context":{"idset":"6551"}}
+{"timestamp":1714201383.7109504,"name":"offline","context":{"idset":"6580"}}
+{"timestamp":1714201383.7252164,"name":"offline","context":{"idset":"6576"}}
+{"timestamp":1714201383.7283168,"name":"offline","context":{"idset":"6626"}}
+{"timestamp":1714201383.8231091,"name":"offline","context":{"idset":"7289"}}
+{"timestamp":1714201383.8889749,"name":"offline","context":{"idset":"6640"}}
+{"timestamp":1714201383.8925185,"name":"offline","context":{"idset":"6589"}}
+{"timestamp":1714201383.8975613,"name":"offline","context":{"idset":"7294"}}
+{"timestamp":1714201383.9012265,"name":"offline","context":{"idset":"6606"}}
+{"timestamp":1714201383.9050615,"name":"offline","context":{"idset":"6543"}}
+{"timestamp":1714201383.9217885,"name":"offline","context":{"idset":"6567"}}
+{"timestamp":1714201383.925221,"name":"offline","context":{"idset":"6628"}}
+{"timestamp":1714201384.0080128,"name":"offline","context":{"idset":"7293"}}
+{"timestamp":1714201384.1067164,"name":"offline","context":{"idset":"7308"}}
+{"timestamp":1714201384.1106064,"name":"offline","context":{"idset":"7313"}}
+{"timestamp":1714201384.1143479,"name":"offline","context":{"idset":"6595"}}
+{"timestamp":1714201384.1181707,"name":"offline","context":{"idset":"6615"}}
+{"timestamp":1714201384.1223264,"name":"offline","context":{"idset":"6636"}}
+{"timestamp":1714201384.153121,"name":"offline","context":{"idset":"6621"}}
+{"timestamp":1714201384.1584358,"name":"offline","context":{"idset":"6533"}}
+{"timestamp":1714201384.1849506,"name":"offline","context":{"idset":"6585"}}
+{"timestamp":1714201384.1945856,"name":"offline","context":{"idset":"6604"}}
+{"timestamp":1714201384.204679,"name":"offline","context":{"idset":"6624"}}
+{"timestamp":1714201384.2147548,"name":"offline","context":{"idset":"6625"}}
+{"timestamp":1714201384.3130548,"name":"offline","context":{"idset":"7299"}}
+{"timestamp":1714201384.4089651,"name":"offline","context":{"idset":"7298"}}
+{"timestamp":1714201384.4164469,"name":"offline","context":{"idset":"7292"}}
+{"timestamp":1714201384.4239342,"name":"offline","context":{"idset":"6630"}}
+{"timestamp":1714201384.4314528,"name":"offline","context":{"idset":"6616"}}
+{"timestamp":1714201384.4379594,"name":"offline","context":{"idset":"6552"}}
+{"timestamp":1714201384.4435763,"name":"offline","context":{"idset":"6577"}}
+{"timestamp":1714201384.4477029,"name":"offline","context":{"idset":"6619"}}
+{"timestamp":1714201384.4606779,"name":"offline","context":{"idset":"6627"}}
+{"timestamp":1714201384.5276959,"name":"offline","context":{"idset":"6643"}}
+{"timestamp":1714201384.7394915,"name":"offline","context":{"idset":"7287"}}
+{"timestamp":1714201384.7439275,"name":"offline","context":{"idset":"7302"}}
+{"timestamp":1714201384.7502804,"name":"offline","context":{"idset":"7310"}}
+{"timestamp":1714201384.7544024,"name":"offline","context":{"idset":"7327"}}
+{"timestamp":1714201384.7587368,"name":"offline","context":{"idset":"7334"}}
+{"timestamp":1714201384.7633581,"name":"offline","context":{"idset":"6629"}}
+{"timestamp":1714201384.7927666,"name":"offline","context":{"idset":"6596"}}
+{"timestamp":1714201384.797302,"name":"offline","context":{"idset":"6591"}}
+{"timestamp":1714201384.801605,"name":"offline","context":{"idset":"7296"}}
+{"timestamp":1714201384.8155999,"name":"offline","context":{"idset":"7304"}}
+{"timestamp":1714201384.8199439,"name":"offline","context":{"idset":"7307"}}
+{"timestamp":1714201384.8242235,"name":"offline","context":{"idset":"7318"}}
+{"timestamp":1714201384.9356911,"name":"offline","context":{"idset":"7306"}}
+{"timestamp":1714201385.0520654,"name":"offline","context":{"idset":"7319"}}
+{"timestamp":1714201385.0569825,"name":"offline","context":{"idset":"7335"}}
+{"timestamp":1714201385.0684011,"name":"offline","context":{"idset":"7373"}}
+{"timestamp":1714201385.0742698,"name":"offline","context":{"idset":"7336"}}
+{"timestamp":1714201385.1315837,"name":"offline","context":{"idset":"6597"}}
+{"timestamp":1714201385.1359782,"name":"offline","context":{"idset":"6634"}}
+{"timestamp":1714201385.1497231,"name":"offline","context":{"idset":"6641"}}
+{"timestamp":1714201385.1543519,"name":"offline","context":{"idset":"6642"}}
+{"timestamp":1714201385.1588244,"name":"offline","context":{"idset":"7290"}}
+{"timestamp":1714201385.1629825,"name":"offline","context":{"idset":"7301"}}
+{"timestamp":1714201385.167304,"name":"offline","context":{"idset":"7303"}}
+{"timestamp":1714201385.1805108,"name":"offline","context":{"idset":"7305"}}
+{"timestamp":1714201385.1870432,"name":"offline","context":{"idset":"7314"}}
+{"timestamp":1714201385.1908848,"name":"offline","context":{"idset":"7317"}}
+{"timestamp":1714201385.1948075,"name":"offline","context":{"idset":"7321"}}
+{"timestamp":1714201385.1986556,"name":"offline","context":{"idset":"7346"}}
+{"timestamp":1714201385.2025237,"name":"offline","context":{"idset":"7355"}}
+{"timestamp":1714201385.2152607,"name":"offline","context":{"idset":"7377"}}
+{"timestamp":1714201385.2289243,"name":"offline","context":{"idset":"6623"}}
+{"timestamp":1714201385.3695912,"name":"offline","context":{"idset":"7364"}}
+{"timestamp":1714201385.617084,"name":"offline","context":{"idset":"6637"}}
+{"timestamp":1714201385.6217055,"name":"offline","context":{"idset":"7311"}}
+{"timestamp":1714201385.6265042,"name":"offline","context":{"idset":"6620"}}
+{"timestamp":1714201385.6317112,"name":"offline","context":{"idset":"7291"}}
+{"timestamp":1714201385.6375957,"name":"offline","context":{"idset":"7324"}}
+{"timestamp":1714201385.6483338,"name":"offline","context":{"idset":"7329"}}
+{"timestamp":1714201385.6575272,"name":"offline","context":{"idset":"7347"}}
+{"timestamp":1714201385.8679655,"name":"offline","context":{"idset":"7359"}}
+{"timestamp":1714201385.8758385,"name":"offline","context":{"idset":"7344"}}
+{"timestamp":1714201385.8819671,"name":"offline","context":{"idset":"6618"}}
+{"timestamp":1714201385.8860898,"name":"offline","context":{"idset":"6633"}}
+{"timestamp":1714201385.8997927,"name":"offline","context":{"idset":"6639"}}
+{"timestamp":1714201385.9038942,"name":"offline","context":{"idset":"6644"}}
+{"timestamp":1714201385.9080157,"name":"offline","context":{"idset":"7285"}}
+{"timestamp":1714201385.9119618,"name":"offline","context":{"idset":"7286"}}
+{"timestamp":1714201385.9158847,"name":"offline","context":{"idset":"7297"}}
+{"timestamp":1714201385.920049,"name":"offline","context":{"idset":"7300"}}
+{"timestamp":1714201385.9241283,"name":"offline","context":{"idset":"7309"}}
+{"timestamp":1714201385.9280005,"name":"offline","context":{"idset":"7312"}}
+{"timestamp":1714201385.9410028,"name":"offline","context":{"idset":"7315"}}
+{"timestamp":1714201385.9655969,"name":"offline","context":{"idset":"7320"}}
+{"timestamp":1714201385.9897506,"name":"offline","context":{"idset":"7322"}}
+{"timestamp":1714201385.9935668,"name":"offline","context":{"idset":"7323"}}
+{"timestamp":1714201385.9977653,"name":"offline","context":{"idset":"7325"}}
+{"timestamp":1714201386.0019622,"name":"offline","context":{"idset":"7326"}}
+{"timestamp":1714201386.0058956,"name":"offline","context":{"idset":"7328"}}
+{"timestamp":1714201386.0096412,"name":"offline","context":{"idset":"7330"}}
+{"timestamp":1714201386.0133736,"name":"offline","context":{"idset":"7331"}}
+{"timestamp":1714201386.0176561,"name":"offline","context":{"idset":"7332"}}
+{"timestamp":1714201386.0220959,"name":"offline","context":{"idset":"7333"}}
+{"timestamp":1714201386.0274875,"name":"offline","context":{"idset":"7337"}}
+{"timestamp":1714201386.0343118,"name":"offline","context":{"idset":"7339"}}
+{"timestamp":1714201386.0409019,"name":"offline","context":{"idset":"7340"}}
+{"timestamp":1714201386.0474763,"name":"offline","context":{"idset":"7342"}}
+{"timestamp":1714201386.0530989,"name":"offline","context":{"idset":"7343"}}
+{"timestamp":1714201386.0586991,"name":"offline","context":{"idset":"7350"}}
+{"timestamp":1714201386.0631258,"name":"offline","context":{"idset":"7352"}}
+{"timestamp":1714201386.0689852,"name":"offline","context":{"idset":"7357"}}
+{"timestamp":1714201386.074888,"name":"offline","context":{"idset":"7358"}}
+{"timestamp":1714201386.0809131,"name":"offline","context":{"idset":"7362"}}
+{"timestamp":1714201386.0867012,"name":"offline","context":{"idset":"7363"}}
+{"timestamp":1714201386.1068437,"name":"offline","context":{"idset":"7366"}}
+{"timestamp":1714201386.1336572,"name":"offline","context":{"idset":"7368"}}
+{"timestamp":1714201386.1390224,"name":"offline","context":{"idset":"7369"}}
+{"timestamp":1714201386.1449301,"name":"offline","context":{"idset":"7371"}}
+{"timestamp":1714201386.1503901,"name":"offline","context":{"idset":"7372"}}
+{"timestamp":1714201386.1564145,"name":"offline","context":{"idset":"7374"}}
+{"timestamp":1714201386.1618209,"name":"offline","context":{"idset":"7378"}}
+{"timestamp":1714201386.1673682,"name":"offline","context":{"idset":"7379"}}
+{"timestamp":1714201386.1727288,"name":"offline","context":{"idset":"7380"}}
+{"timestamp":1714201386.1780133,"name":"offline","context":{"idset":"7385"}}
+{"timestamp":1714201386.1833928,"name":"offline","context":{"idset":"7388"}}
+{"timestamp":1714201386.1876783,"name":"offline","context":{"idset":"7391"}}
+{"timestamp":1714201386.1914384,"name":"offline","context":{"idset":"7395"}}
+{"timestamp":1714201386.2103617,"name":"offline","context":{"idset":"7398"}}
+{"timestamp":1714201386.2294893,"name":"offline","context":{"idset":"7403"}}
+{"timestamp":1714201386.2408156,"name":"offline","context":{"idset":"8309"}}
+{"timestamp":1714201386.2443011,"name":"offline","context":{"idset":"8310"}}
+{"timestamp":1714201386.2477643,"name":"offline","context":{"idset":"8311"}}
+{"timestamp":1714201386.2512736,"name":"offline","context":{"idset":"8312"}}
+{"timestamp":1714201386.2547498,"name":"offline","context":{"idset":"8313"}}
+{"timestamp":1714201386.2584913,"name":"offline","context":{"idset":"8314"}}
+{"timestamp":1714201386.2782104,"name":"offline","context":{"idset":"7295"}}
+{"timestamp":1714201386.299999,"name":"offline","context":{"idset":"7316"}}
+{"timestamp":1714201386.3159571,"name":"offline","context":{"idset":"7348"}}
+{"timestamp":1714201386.3211586,"name":"offline","context":{"idset":"7381"}}
+{"timestamp":1714201386.326066,"name":"offline","context":{"idset":"7383"}}
+{"timestamp":1714201386.783318,"name":"offline","context":{"idset":"7370"}}
+{"timestamp":1714201386.7878118,"name":"offline","context":{"idset":"7338"}}
+{"timestamp":1714201386.7923381,"name":"offline","context":{"idset":"7365"}}
+{"timestamp":1714201386.7961805,"name":"offline","context":{"idset":"7399"}}
+{"timestamp":1714201386.7999763,"name":"offline","context":{"idset":"7384"}}
+{"timestamp":1714201386.8037522,"name":"offline","context":{"idset":"7396"}}
+{"timestamp":1714201386.8075356,"name":"offline","context":{"idset":"7341"}}
+{"timestamp":1714201386.8112907,"name":"offline","context":{"idset":"7345"}}
+{"timestamp":1714201386.8150578,"name":"offline","context":{"idset":"7390"}}
+{"timestamp":1714201386.8188634,"name":"offline","context":{"idset":"7405"}}
+{"timestamp":1714201386.8226411,"name":"offline","context":{"idset":"7394"}}
+{"timestamp":1714201386.8264053,"name":"offline","context":{"idset":"7354"}}
+{"timestamp":1714201386.8303967,"name":"offline","context":{"idset":"7392"}}
+{"timestamp":1714201386.960649,"name":"offline","context":{"idset":"7353"}}
+{"timestamp":1714201386.9654658,"name":"offline","context":{"idset":"7382"}}
+{"timestamp":1714201386.9726386,"name":"offline","context":{"idset":"7349"}}
+{"timestamp":1714201386.9814088,"name":"offline","context":{"idset":"7351"}}
+{"timestamp":1714201387.0009604,"name":"offline","context":{"idset":"7356"}}
+{"timestamp":1714201387.005651,"name":"offline","context":{"idset":"7360"}}
+{"timestamp":1714201387.0143042,"name":"offline","context":{"idset":"7361"}}
+{"timestamp":1714201387.0189312,"name":"offline","context":{"idset":"7367"}}
+{"timestamp":1714201387.0235183,"name":"offline","context":{"idset":"7375"}}
+{"timestamp":1714201387.0281386,"name":"offline","context":{"idset":"7376"}}
+{"timestamp":1714201387.0326867,"name":"offline","context":{"idset":"7386"}}
+{"timestamp":1714201387.0372653,"name":"offline","context":{"idset":"7387"}}
+{"timestamp":1714201387.041779,"name":"offline","context":{"idset":"7389"}}
+{"timestamp":1714201387.0463548,"name":"offline","context":{"idset":"7393"}}
+{"timestamp":1714201387.0508423,"name":"offline","context":{"idset":"7397"}}
+{"timestamp":1714201387.0557826,"name":"offline","context":{"idset":"7400"}}
+{"timestamp":1714201387.0602405,"name":"offline","context":{"idset":"7401"}}
+{"timestamp":1714201387.0646441,"name":"offline","context":{"idset":"7402"}}
+{"timestamp":1714201387.0690637,"name":"offline","context":{"idset":"7404"}}
+{"timestamp":1714201387.0734484,"name":"offline","context":{"idset":"7406"}}
+{"timestamp":1714201387.0887096,"name":"offline","context":{"idset":"7407"}}
+{"timestamp":1714201387.1093593,"name":"offline","context":{"idset":"7408"}}
+{"timestamp":1714201387.1262455,"name":"offline","context":{"idset":"7409"}}
+{"timestamp":1714201387.1292381,"name":"offline","context":{"idset":"7410"}}
+{"timestamp":1714201387.1322212,"name":"offline","context":{"idset":"7411"}}
+{"timestamp":1714201387.135195,"name":"offline","context":{"idset":"7412"}}
+{"timestamp":1714201387.13815,"name":"offline","context":{"idset":"8642"}}
+{"timestamp":1714201387.1410866,"name":"offline","context":{"idset":"8643"}}
+{"timestamp":1714201387.4343927,"name":"offline","context":{"idset":"8678"}}
+{"timestamp":1714201387.7528119,"name":"offline","context":{"idset":"8644"}}
+{"timestamp":1714201387.8514404,"name":"offline","context":{"idset":"8679"}}
+{"timestamp":1714201388.2332819,"name":"offline","context":{"idset":"8645"}}
+{"timestamp":1714201388.3316693,"name":"offline","context":{"idset":"8680"}}
+{"timestamp":1714201388.5292294,"name":"offline","context":{"idset":"8681"}}
+{"timestamp":1714201388.713203,"name":"offline","context":{"idset":"8646"}}
+{"timestamp":1714201389.9118867,"name":"offline","context":{"idset":"8682"}}
+{"timestamp":1714201390.3459518,"name":"offline","context":{"idset":"8647"}}
+{"timestamp":1714201390.4627204,"name":"offline","context":{"idset":"8683"}}
+{"timestamp":1714201390.5615308,"name":"offline","context":{"idset":"8648"}}
+{"timestamp":1714201391.1509023,"name":"offline","context":{"idset":"8684"}}
+{"timestamp":1714201391.8434196,"name":"offline","context":{"idset":"8649"}}
+{"timestamp":1714201392.469583,"name":"offline","context":{"idset":"8685"}}
+{"timestamp":1714201392.7612996,"name":"offline","context":{"idset":"8650"}}
+{"timestamp":1714201392.8585143,"name":"offline","context":{"idset":"8686"}}
+{"timestamp":1714201393.0681047,"name":"offline","context":{"idset":"8651"}}
+{"timestamp":1714201393.3203714,"name":"offline","context":{"idset":"8687"}}
+{"timestamp":1714201393.4178343,"name":"offline","context":{"idset":"8652"}}
+{"timestamp":1714201393.7955728,"name":"offline","context":{"idset":"8688"}}
+{"timestamp":1714201394.0587292,"name":"offline","context":{"idset":"8653"}}
+{"timestamp":1714201394.3501587,"name":"offline","context":{"idset":"8689"}}
+{"timestamp":1714201394.5578299,"name":"offline","context":{"idset":"8654"}}
+{"timestamp":1714201394.8163629,"name":"offline","context":{"idset":"8690"}}
+{"timestamp":1714201395.5225799,"name":"offline","context":{"idset":"8655"}}
+{"timestamp":1714201395.9661748,"name":"offline","context":{"idset":"8691"}}
+{"timestamp":1714201396.0832157,"name":"offline","context":{"idset":"8656"}}
+{"timestamp":1714201396.1835883,"name":"offline","context":{"idset":"8692"}}
+{"timestamp":1714201396.1958156,"name":"offline","context":{"idset":"8658"}}
+{"timestamp":1714201396.2940307,"name":"offline","context":{"idset":"8657"}}
+{"timestamp":1714201396.4070337,"name":"offline","context":{"idset":"8659"}}
+{"timestamp":1714201396.5072896,"name":"offline","context":{"idset":"8660"}}
+{"timestamp":1714201396.6047957,"name":"offline","context":{"idset":"8662"}}
+{"timestamp":1714201396.7045531,"name":"offline","context":{"idset":"8661"}}
+{"timestamp":1714201396.8163395,"name":"offline","context":{"idset":"8663"}}
+{"timestamp":1714201396.9181523,"name":"offline","context":{"idset":"8664"}}
+{"timestamp":1714201397.1479809,"name":"offline","context":{"idset":"8665"}}
+{"timestamp":1714201397.25741,"name":"offline","context":{"idset":"8666"}}
+{"timestamp":1714201397.2605712,"name":"offline","context":{"idset":"8667"}}
+{"timestamp":1714201397.3586838,"name":"offline","context":{"idset":"8668"}}
+{"timestamp":1714201397.4773493,"name":"offline","context":{"idset":"8669"}}
+{"timestamp":1714201397.5769083,"name":"offline","context":{"idset":"8670"}}
+{"timestamp":1714201397.7210321,"name":"offline","context":{"idset":"8672"}}
+{"timestamp":1714201397.7323546,"name":"offline","context":{"idset":"8671"}}
+{"timestamp":1714201397.8322837,"name":"offline","context":{"idset":"8673"}}
+{"timestamp":1714201397.8929954,"name":"offline","context":{"idset":"8674"}}
+{"timestamp":1714201397.9721193,"name":"offline","context":{"idset":"8675"}}
+{"timestamp":1714201398.1556554,"name":"offline","context":{"idset":"8676"}}
+{"timestamp":1714201398.2501273,"name":"offline","context":{"idset":"8677"}}
+{"timestamp":1714201398.3416209,"name":"offline","context":{"idset":"8570"}}
+{"timestamp":1714201398.4290423,"name":"offline","context":{"idset":"8606"}}
+{"timestamp":1714201398.5149119,"name":"offline","context":{"idset":"8571"}}
+{"timestamp":1714201398.5207286,"name":"offline","context":{"idset":"8607"}}
+{"timestamp":1714201398.6186426,"name":"offline","context":{"idset":"8572"}}
+{"timestamp":1714201398.6323798,"name":"offline","context":{"idset":"8608"}}
+{"timestamp":1714201398.7305918,"name":"offline","context":{"idset":"8573"}}
+{"timestamp":1714201398.8533225,"name":"offline","context":{"idset":"8610"}}
+{"timestamp":1714201398.8591113,"name":"offline","context":{"idset":"8609"}}
+{"timestamp":1714201398.8649075,"name":"offline","context":{"idset":"8574"}}
+{"timestamp":1714201398.9592483,"name":"offline","context":{"idset":"8575"}}
+{"timestamp":1714201399.0641174,"name":"offline","context":{"idset":"8611"}}
+{"timestamp":1714201399.0701752,"name":"offline","context":{"idset":"8576"}}
+{"timestamp":1714201399.1671832,"name":"offline","context":{"idset":"8612"}}
+{"timestamp":1714201399.5960479,"name":"offline","context":{"idset":"8577"}}
+{"timestamp":1714201399.6974671,"name":"offline","context":{"idset":"8613"}}
+{"timestamp":1714201399.8013866,"name":"offline","context":{"idset":"8578"}}
+{"timestamp":1714201399.9056947,"name":"offline","context":{"idset":"8614"}}
+{"timestamp":1714201400.0220845,"name":"offline","context":{"idset":"8615"}}
+{"timestamp":1714201400.118942,"name":"offline","context":{"idset":"8579"}}
+{"timestamp":1714201400.3228738,"name":"offline","context":{"idset":"8580"}}
+{"timestamp":1714201400.4199324,"name":"offline","context":{"idset":"8616"}}
+{"timestamp":1714201400.5487266,"name":"offline","context":{"idset":"8581"}}
+{"timestamp":1714201400.6485991,"name":"offline","context":{"idset":"8617"}}
+{"timestamp":1714201400.7735469,"name":"offline","context":{"idset":"8583"}}
+{"timestamp":1714201400.7776694,"name":"offline","context":{"idset":"8582"}}
+{"timestamp":1714201400.8770676,"name":"offline","context":{"idset":"8618"}}
+{"timestamp":1714201400.9004872,"name":"offline","context":{"idset":"8620"}}
+{"timestamp":1714201400.9033582,"name":"offline","context":{"idset":"8585"}}
+{"timestamp":1714201400.9220207,"name":"offline","context":{"idset":"8584"}}
+{"timestamp":1714201401.0053511,"name":"offline","context":{"idset":"8619"}}
+{"timestamp":1714201401.1028929,"name":"offline","context":{"idset":"8586"}}
+{"timestamp":1714201401.1059813,"name":"offline","context":{"idset":"8621"}}
+{"timestamp":1714201401.2077808,"name":"offline","context":{"idset":"8622"}}
+{"timestamp":1714201401.3173294,"name":"offline","context":{"idset":"8587"}}
+{"timestamp":1714201401.4169514,"name":"offline","context":{"idset":"8623"}}
+{"timestamp":1714201401.5924721,"name":"offline","context":{"idset":"8588"}}
+{"timestamp":1714201401.6890237,"name":"offline","context":{"idset":"8624"}}
+{"timestamp":1714201401.7777205,"name":"offline","context":{"idset":"8590"}}
+{"timestamp":1714201401.7892973,"name":"offline","context":{"idset":"8589"}}
+{"timestamp":1714201401.7981782,"name":"offline","context":{"idset":"8625"}}
+{"timestamp":1714201401.889498,"name":"offline","context":{"idset":"8591"}}
+{"timestamp":1714201402.0261161,"name":"offline","context":{"idset":"8630"}}
+{"timestamp":1714201402.0297184,"name":"offline","context":{"idset":"8629"}}
+{"timestamp":1714201402.0327086,"name":"offline","context":{"idset":"8627"}}
+{"timestamp":1714201402.0356491,"name":"offline","context":{"idset":"8626"}}
+{"timestamp":1714201402.0386884,"name":"offline","context":{"idset":"8593"}}
+{"timestamp":1714201402.0416732,"name":"offline","context":{"idset":"8594"}}
+{"timestamp":1714201402.0689952,"name":"offline","context":{"idset":"8592"}}
+{"timestamp":1714201402.1466613,"name":"offline","context":{"idset":"8628"}}
+{"timestamp":1714201402.2513721,"name":"offline","context":{"idset":"8632"}}
+{"timestamp":1714201402.2566943,"name":"offline","context":{"idset":"8631"}}
+{"timestamp":1714201402.2635465,"name":"offline","context":{"idset":"8596"}}
+{"timestamp":1714201402.3642864,"name":"offline","context":{"idset":"8595"}}
+{"timestamp":1714201402.3896508,"name":"offline","context":{"idset":"8633"}}
+{"timestamp":1714201402.3926508,"name":"offline","context":{"idset":"8634"}}
+{"timestamp":1714201402.4145746,"name":"offline","context":{"idset":"8597"}}
+{"timestamp":1714201402.4937065,"name":"offline","context":{"idset":"8635"}}
+{"timestamp":1714201402.5899382,"name":"offline","context":{"idset":"8598"}}
+{"timestamp":1714201402.5923316,"name":"offline","context":{"idset":"8599"}}
+{"timestamp":1714201402.5946219,"name":"offline","context":{"idset":"8600"}}
+{"timestamp":1714201402.6962554,"name":"offline","context":{"idset":"8636"}}
+{"timestamp":1714201402.7887743,"name":"offline","context":{"idset":"8601"}}
+{"timestamp":1714201402.7991138,"name":"offline","context":{"idset":"8602"}}
+{"timestamp":1714201402.9033635,"name":"offline","context":{"idset":"8637"}}
+{"timestamp":1714201402.9075575,"name":"offline","context":{"idset":"8603"}}
+{"timestamp":1714201402.919136,"name":"offline","context":{"idset":"8638"}}
+{"timestamp":1714201403.0189838,"name":"offline","context":{"idset":"8604"}}
+{"timestamp":1714201403.0370247,"name":"offline","context":{"idset":"8639"}}
+{"timestamp":1714201403.1317587,"name":"offline","context":{"idset":"8640"}}
+{"timestamp":1714201403.2170448,"name":"offline","context":{"idset":"8605"}}
+{"timestamp":1714201403.313782,"name":"offline","context":{"idset":"8641"}}
+{"timestamp":1714201403.608819,"name":"offline","context":{"idset":"8334"}}
+{"timestamp":1714201403.8475838,"name":"offline","context":{"idset":"8371"}}
+{"timestamp":1714201403.9427333,"name":"offline","context":{"idset":"8408"}}
+{"timestamp":1714201404.0444648,"name":"offline","context":{"idset":"8336"}}
+{"timestamp":1714201404.0518587,"name":"offline","context":{"idset":"8370"}}
+{"timestamp":1714201404.0562825,"name":"offline","context":{"idset":"8406"}}
+{"timestamp":1714201404.0604324,"name":"offline","context":{"idset":"8373"}}
+{"timestamp":1714201404.1594977,"name":"offline","context":{"idset":"8372"}}
+{"timestamp":1714201404.2598917,"name":"offline","context":{"idset":"8338"}}
+{"timestamp":1714201404.2703042,"name":"offline","context":{"idset":"8407"}}
+{"timestamp":1714201404.2772758,"name":"offline","context":{"idset":"8335"}}
+{"timestamp":1714201404.2834659,"name":"offline","context":{"idset":"8337"}}
+{"timestamp":1714201404.3782327,"name":"offline","context":{"idset":"8409"}}
+{"timestamp":1714201404.5043008,"name":"offline","context":{"idset":"8339"}}
+{"timestamp":1714201404.6022885,"name":"offline","context":{"idset":"8374"}}
+{"timestamp":1714201404.7088871,"name":"offline","context":{"idset":"8340"}}
+{"timestamp":1714201404.8617461,"name":"offline","context":{"idset":"8375"}}
+{"timestamp":1714201404.9608231,"name":"offline","context":{"idset":"8410"}}
+{"timestamp":1714201404.9677055,"name":"offline","context":{"idset":"8341"}}
+{"timestamp":1714201404.9747176,"name":"offline","context":{"idset":"8376"}}
+{"timestamp":1714201404.979476,"name":"offline","context":{"idset":"8411"}}
+{"timestamp":1714201405.0688362,"name":"offline","context":{"idset":"8413"}}
+{"timestamp":1714201405.1728892,"name":"offline","context":{"idset":"8343"}}
+{"timestamp":1714201405.1801438,"name":"offline","context":{"idset":"8342"}}
+{"timestamp":1714201405.184288,"name":"offline","context":{"idset":"8377"}}
+{"timestamp":1714201405.1881199,"name":"offline","context":{"idset":"8378"}}
+{"timestamp":1714201405.1919098,"name":"offline","context":{"idset":"8412"}}
+{"timestamp":1714201405.2927368,"name":"offline","context":{"idset":"8414"}}
+{"timestamp":1714201405.4178145,"name":"offline","context":{"idset":"8379"}}
+{"timestamp":1714201405.4252026,"name":"offline","context":{"idset":"8344"}}
+{"timestamp":1714201405.5202088,"name":"offline","context":{"idset":"8415"}}
+{"timestamp":1714201405.907711,"name":"offline","context":{"idset":"8345"}}
+{"timestamp":1714201405.9950469,"name":"offline","context":{"idset":"8416"}}
+{"timestamp":1714201406.0027802,"name":"offline","context":{"idset":"8381"}}
+{"timestamp":1714201406.1051378,"name":"offline","context":{"idset":"8346"}}
+{"timestamp":1714201406.193845,"name":"offline","context":{"idset":"8380"}}
+{"timestamp":1714201406.2759726,"name":"offline","context":{"idset":"8347"}}
+{"timestamp":1714201406.2834766,"name":"offline","context":{"idset":"8417"}}
+{"timestamp":1714201406.3815427,"name":"offline","context":{"idset":"8418"}}
+{"timestamp":1714201406.3974016,"name":"offline","context":{"idset":"8348"}}
+{"timestamp":1714201406.4049807,"name":"offline","context":{"idset":"8349"}}
+{"timestamp":1714201406.4216177,"name":"offline","context":{"idset":"8383"}}
+{"timestamp":1714201406.5096447,"name":"offline","context":{"idset":"8421"}}
+{"timestamp":1714201406.6081409,"name":"offline","context":{"idset":"8385"}}
+{"timestamp":1714201406.6150846,"name":"offline","context":{"idset":"8422"}}
+{"timestamp":1714201406.6219316,"name":"offline","context":{"idset":"8315"}}
+{"timestamp":1714201406.6287806,"name":"offline","context":{"idset":"8350"}}
+{"timestamp":1714201406.6356344,"name":"offline","context":{"idset":"8419"}}
+{"timestamp":1714201406.6424661,"name":"offline","context":{"idset":"8420"}}
+{"timestamp":1714201406.7269859,"name":"offline","context":{"idset":"8384"}}
+{"timestamp":1714201406.8242309,"name":"offline","context":{"idset":"8382"}}
+{"timestamp":1714201406.9202163,"name":"offline","context":{"idset":"8386"}}
+{"timestamp":1714201406.9276383,"name":"offline","context":{"idset":"8351"}}
+{"timestamp":1714201407.0271473,"name":"offline","context":{"idset":"8387"}}
+{"timestamp":1714201407.2211382,"name":"offline","context":{"idset":"8388"}}
+{"timestamp":1714201407.2300079,"name":"offline","context":{"idset":"8317"}}
+{"timestamp":1714201407.3290291,"name":"offline","context":{"idset":"8316"}}
+{"timestamp":1714201407.4544468,"name":"offline","context":{"idset":"8318"}}
+{"timestamp":1714201407.5538843,"name":"offline","context":{"idset":"8423"}}
+{"timestamp":1714201407.6378741,"name":"offline","context":{"idset":"8425"}}
+{"timestamp":1714201407.6426411,"name":"offline","context":{"idset":"8389"}}
+{"timestamp":1714201407.7388949,"name":"offline","context":{"idset":"8352"}}
+{"timestamp":1714201407.7564633,"name":"offline","context":{"idset":"8319"}}
+{"timestamp":1714201407.7590337,"name":"offline","context":{"idset":"8424"}}
+{"timestamp":1714201407.7800207,"name":"offline","context":{"idset":"8390"}}
+{"timestamp":1714201407.7825718,"name":"offline","context":{"idset":"8353"}}
+{"timestamp":1714201407.8615201,"name":"offline","context":{"idset":"8426"}}
+{"timestamp":1714201408.0715246,"name":"offline","context":{"idset":"8354"}}
+{"timestamp":1714201408.1729889,"name":"offline","context":{"idset":"8355"}}
+{"timestamp":1714201408.4285975,"name":"offline","context":{"idset":"8391"}}
+{"timestamp":1714201408.5271137,"name":"offline","context":{"idset":"8356"}}
+{"timestamp":1714201408.6272395,"name":"offline","context":{"idset":"8320"}}
+{"timestamp":1714201408.6346858,"name":"offline","context":{"idset":"8427"}}
+{"timestamp":1714201408.7280283,"name":"offline","context":{"idset":"8321"}}
+{"timestamp":1714201408.8301404,"name":"offline","context":{"idset":"8392"}}
+{"timestamp":1714201408.9150643,"name":"offline","context":{"idset":"8358"}}
+{"timestamp":1714201408.9199586,"name":"offline","context":{"idset":"8357"}}
+{"timestamp":1714201408.9239891,"name":"offline","context":{"idset":"8322"}}
+{"timestamp":1714201408.9276237,"name":"offline","context":{"idset":"8359"}}
+{"timestamp":1714201409.0229716,"name":"offline","context":{"idset":"8428"}}
+{"timestamp":1714201409.1375933,"name":"offline","context":{"idset":"8430"}}
+{"timestamp":1714201409.1588616,"name":"offline","context":{"idset":"8324"}}
+{"timestamp":1714201409.2669966,"name":"offline","context":{"idset":"8429"}}
+{"timestamp":1714201409.2823303,"name":"offline","context":{"idset":"8393"}}
+{"timestamp":1714201409.2885931,"name":"offline","context":{"idset":"8431"}}
+{"timestamp":1714201409.2950644,"name":"offline","context":{"idset":"8323"}}
+{"timestamp":1714201409.3912892,"name":"offline","context":{"idset":"8395"}}
+{"timestamp":1714201409.4833999,"name":"offline","context":{"idset":"8360"}}
+{"timestamp":1714201409.4874635,"name":"offline","context":{"idset":"8326"}}
+{"timestamp":1714201409.5868421,"name":"offline","context":{"idset":"8394"}}
+{"timestamp":1714201409.6044104,"name":"offline","context":{"idset":"8362"}}
+{"timestamp":1714201409.6082008,"name":"offline","context":{"idset":"8396"}}
+{"timestamp":1714201409.7035291,"name":"offline","context":{"idset":"8397"}}
+{"timestamp":1714201409.8092408,"name":"offline","context":{"idset":"8327"}}
+{"timestamp":1714201409.8161736,"name":"offline","context":{"idset":"8361"}}
+{"timestamp":1714201409.8229754,"name":"offline","context":{"idset":"8432"}}
+{"timestamp":1714201409.8294611,"name":"offline","context":{"idset":"8433"}}
+{"timestamp":1714201409.9275622,"name":"offline","context":{"idset":"8325"}}
+{"timestamp":1714201410.0306768,"name":"offline","context":{"idset":"8434"}}
+{"timestamp":1714201410.0379474,"name":"offline","context":{"idset":"8398"}}
+{"timestamp":1714201410.0428584,"name":"offline","context":{"idset":"8565"}}
+{"timestamp":1714201410.1447732,"name":"offline","context":{"idset":"8363"}}
+{"timestamp":1714201410.2476039,"name":"offline","context":{"idset":"8566"}}
+{"timestamp":1714201410.2552283,"name":"offline","context":{"idset":"8328"}}
+{"timestamp":1714201410.3526998,"name":"offline","context":{"idset":"8399"}}
+{"timestamp":1714201410.4497285,"name":"offline","context":{"idset":"8364"}}
+{"timestamp":1714201410.4759293,"name":"offline","context":{"idset":"8567"}}
+{"timestamp":1714201410.5721364,"name":"offline","context":{"idset":"8435"}}
+{"timestamp":1714201410.7405641,"name":"offline","context":{"idset":"8436"}}
+{"timestamp":1714201410.7442884,"name":"offline","context":{"idset":"8330"}}
+{"timestamp":1714201410.7480347,"name":"offline","context":{"idset":"8329"}}
+{"timestamp":1714201410.7516496,"name":"offline","context":{"idset":"8365"}}
+{"timestamp":1714201410.849231,"name":"offline","context":{"idset":"8331"}}
+{"timestamp":1714201410.8616812,"name":"offline","context":{"idset":"8568"}}
+{"timestamp":1714201410.9627144,"name":"offline","context":{"idset":"8400"}}
+{"timestamp":1714201410.9699395,"name":"offline","context":{"idset":"8366"}}
+{"timestamp":1714201411.0706372,"name":"offline","context":{"idset":"8402"}}
+{"timestamp":1714201411.1759491,"name":"offline","context":{"idset":"8569"}}
+{"timestamp":1714201411.2755363,"name":"offline","context":{"idset":"8401"}}
+{"timestamp":1714201411.4092541,"name":"offline","context":{"idset":"8367"}}
+{"timestamp":1714201411.4155092,"name":"offline","context":{"idset":"8333"}}
+{"timestamp":1714201411.4219449,"name":"offline","context":{"idset":"8368"}}
+{"timestamp":1714201411.4282193,"name":"offline","context":{"idset":"8403"}}
+{"timestamp":1714201411.5174491,"name":"offline","context":{"idset":"8332"}}
+{"timestamp":1714201411.6284642,"name":"offline","context":{"idset":"8404"}}
+{"timestamp":1714201411.7281065,"name":"offline","context":{"idset":"8405"}}
+{"timestamp":1714201411.930969,"name":"offline","context":{"idset":"8369"}}
+{"timestamp":1714201458.2969422,"name":"online","context":{"idset":"6541,6562"}}
+{"timestamp":1714201458.6621838,"name":"online","context":{"idset":"6586"}}
+{"timestamp":1714201458.9001653,"name":"online","context":{"idset":"6566"}}
+{"timestamp":1714201459.1910405,"name":"online","context":{"idset":"6563"}}
+{"timestamp":1714201460.0010607,"name":"online","context":{"idset":"6532"}}
+{"timestamp":1714201460.9354858,"name":"online","context":{"idset":"6528"}}
+{"timestamp":1714201461.7413075,"name":"online","context":{"idset":"6611"}}
+{"timestamp":1714201461.9507446,"name":"online","context":{"idset":"6573"}}
+{"timestamp":1714201462.2514033,"name":"online","context":{"idset":"6559"}}
+{"timestamp":1714201462.4351139,"name":"online","context":{"idset":"6523"}}
+{"timestamp":1714201462.9883981,"name":"online","context":{"idset":"6577"}}
+{"timestamp":1714201463.1477835,"name":"online","context":{"idset":"6525"}}
+{"timestamp":1714201463.5103765,"name":"online","context":{"idset":"6547"}}
+{"timestamp":1714201463.7313437,"name":"online","context":{"idset":"6531,6598"}}
+{"timestamp":1714201464.239032,"name":"online","context":{"idset":"6599"}}
+{"timestamp":1714201464.4522669,"name":"online","context":{"idset":"6600"}}
+{"timestamp":1714201465.0200038,"name":"online","context":{"idset":"6613"}}
+{"timestamp":1714201465.7195296,"name":"online","context":{"idset":"6542"}}
+{"timestamp":1714201466.2950442,"name":"online","context":{"idset":"6604"}}
+{"timestamp":1714201466.5197692,"name":"online","context":{"idset":"6606"}}
+{"timestamp":1714201467.1127768,"name":"online","context":{"idset":"6530"}}
+{"timestamp":1714201467.6472664,"name":"online","context":{"idset":"6546"}}
+{"timestamp":1714201467.8626404,"name":"online","context":{"idset":"6534,6630"}}
+{"timestamp":1714201468.1807132,"name":"online","context":{"idset":"6641"}}
+{"timestamp":1714201468.3785553,"name":"online","context":{"idset":"6597,6603"}}
+{"timestamp":1714201468.7834935,"name":"online","context":{"idset":"6592,6612"}}
+{"timestamp":1714201469.2945938,"name":"online","context":{"idset":"6596,6632"}}
+{"timestamp":1714201469.6963401,"name":"online","context":{"idset":"7365"}}
+{"timestamp":1714201469.8387132,"name":"online","context":{"idset":"6587"}}
+{"timestamp":1714201469.9637327,"name":"online","context":{"idset":"7329"}}
+{"timestamp":1714201470.2536616,"name":"online","context":{"idset":"6565"}}
+{"timestamp":1714201470.3946931,"name":"online","context":{"idset":"6552"}}
+{"timestamp":1714201470.5824513,"name":"online","context":{"idset":"7334"}}
+{"timestamp":1714201470.7629812,"name":"online","context":{"idset":"6527"}}
+{"timestamp":1714201471.0524812,"name":"online","context":{"idset":"8309"}}
+{"timestamp":1714201471.6086941,"name":"online","context":{"idset":"8310"}}
+{"timestamp":1714201471.8134069,"name":"online","context":{"idset":"6610"}}
+{"timestamp":1714201471.9279253,"name":"online","context":{"idset":"6608"}}
+{"timestamp":1714201472.1166966,"name":"online","context":{"idset":"6578"}}
+{"timestamp":1714201472.3321698,"name":"online","context":{"idset":"6529,6605"}}
+{"timestamp":1714201472.5347154,"name":"online","context":{"idset":"7309"}}
+{"timestamp":1714201473.2501931,"name":"online","context":{"idset":"6601,6607"}}
+{"timestamp":1714201473.364779,"name":"online","context":{"idset":"6569,7402"}}
+{"timestamp":1714201473.7673533,"name":"online","context":{"idset":"6602"}}
+{"timestamp":1714201473.8720095,"name":"online","context":{"idset":"6533"}}
+{"timestamp":1714201474.0073907,"name":"online","context":{"idset":"6543,7359"}}
+{"timestamp":1714201474.1966076,"name":"online","context":{"idset":"6526"}}
+{"timestamp":1714201474.3527958,"name":"online","context":{"idset":"6568,6609,8314"}}
+{"timestamp":1714201474.4564893,"name":"online","context":{"idset":"6551"}}
+{"timestamp":1714201474.5615296,"name":"online","context":{"idset":"6620,7307"}}
+{"timestamp":1714201474.7219851,"name":"online","context":{"idset":"8677"}}
+{"timestamp":1714201474.8540883,"name":"online","context":{"idset":"7341"}}
+{"timestamp":1714201475.0345497,"name":"online","context":{"idset":"6553"}}
+{"timestamp":1714201475.1870263,"name":"online","context":{"idset":"8641,8678"}}
+{"timestamp":1714201475.3250151,"name":"online","context":{"idset":"7362,8642"}}
+{"timestamp":1714201475.5695856,"name":"online","context":{"idset":"8644"}}
+{"timestamp":1714201475.7068818,"name":"online","context":{"idset":"6629,7337,8643,8645,8679"}}
+{"timestamp":1714201475.8176861,"name":"online","context":{"idset":"8647"}}
+{"timestamp":1714201476.0763323,"name":"online","context":{"idset":"6617,8646,8649,8683"}}
+{"timestamp":1714201476.2157912,"name":"online","context":{"idset":"8681-8682,8684"}}
+{"timestamp":1714201476.3371251,"name":"online","context":{"idset":"8680,8685-8686"}}
+{"timestamp":1714201476.4942627,"name":"online","context":{"idset":"8313,8650"}}
+{"timestamp":1714201476.8130012,"name":"online","context":{"idset":"7407,8648,8651"}}
+{"timestamp":1714201476.9436553,"name":"online","context":{"idset":"6627"}}
+{"timestamp":1714201477.0855527,"name":"online","context":{"idset":"8687"}}
+{"timestamp":1714201477.31264,"name":"online","context":{"idset":"6642"}}
+{"timestamp":1714201477.4206352,"name":"online","context":{"idset":"8652"}}
+{"timestamp":1714201477.5565302,"name":"online","context":{"idset":"6567"}}
+{"timestamp":1714201477.7414498,"name":"online","context":{"idset":"8653,8688"}}
+{"timestamp":1714201477.8622837,"name":"online","context":{"idset":"6622,7385,8689"}}
+{"timestamp":1714201478.0721798,"name":"online","context":{"idset":"8654"}}
+{"timestamp":1714201478.2676373,"name":"online","context":{"idset":"6538"}}
+{"timestamp":1714201478.386888,"name":"online","context":{"idset":"6631,7369,8690"}}
+{"timestamp":1714201478.5234613,"name":"online","context":{"idset":"8692"}}
+{"timestamp":1714201478.5272939,"name":"online","context":{"idset":"6570"}}
+{"timestamp":1714201478.8147438,"name":"online","context":{"idset":"6524,6548,8691"}}
+{"timestamp":1714201479.1397443,"name":"online","context":{"idset":"8655-8657"}}
+{"timestamp":1714201479.3539388,"name":"online","context":{"idset":"6594,8658"}}
+{"timestamp":1714201479.4570038,"name":"online","context":{"idset":"8662"}}
+{"timestamp":1714201479.5699039,"name":"online","context":{"idset":"6636,8659,8661"}}
+{"timestamp":1714201479.5758152,"name":"online","context":{"idset":"6564,8660"}}
+{"timestamp":1714201479.8034675,"name":"online","context":{"idset":"7306"}}
+{"timestamp":1714201480.0967278,"name":"online","context":{"idset":"6590,7308,7410,8312"}}
+{"timestamp":1714201480.2867734,"name":"online","context":{"idset":"6550"}}
+{"timestamp":1714201480.4933503,"name":"online","context":{"idset":"6640"}}
+{"timestamp":1714201480.6043973,"name":"online","context":{"idset":"8663"}}
+{"timestamp":1714201480.7868795,"name":"online","context":{"idset":"6536,7315,8664"}}
+{"timestamp":1714201481.183754,"name":"online","context":{"idset":"6558,6625,8665"}}
+{"timestamp":1714201481.3121824,"name":"online","context":{"idset":"8311"}}
+{"timestamp":1714201481.8641558,"name":"online","context":{"idset":"6634,8666,8670"}}
+{"timestamp":1714201481.9727478,"name":"online","context":{"idset":"8667"}}
+{"timestamp":1714201482.1070306,"name":"online","context":{"idset":"8669"}}
+{"timestamp":1714201482.2344985,"name":"online","context":{"idset":"8668"}}
+{"timestamp":1714201482.4967413,"name":"online","context":{"idset":"6575"}}
+{"timestamp":1714201482.6223688,"name":"online","context":{"idset":"6582"}}
+{"timestamp":1714201482.8159189,"name":"online","context":{"idset":"8671,8673"}}
+{"timestamp":1714201482.9533079,"name":"online","context":{"idset":"7293,8672"}}
+{"timestamp":1714201483.0948341,"name":"online","context":{"idset":"8674"}}
+{"timestamp":1714201483.2400632,"name":"online","context":{"idset":"7340"}}
+{"timestamp":1714201483.4484558,"name":"online","context":{"idset":"8569,8605,8675-8676"}}
+{"timestamp":1714201483.6656685,"name":"online","context":{"idset":"8606"}}
+{"timestamp":1714201483.7774894,"name":"online","context":{"idset":"6555,6557"}}
+{"timestamp":1714201483.9751139,"name":"online","context":{"idset":"6572"}}
+{"timestamp":1714201484.2023728,"name":"online","context":{"idset":"6633,8570"}}
+{"timestamp":1714201484.3180957,"name":"online","context":{"idset":"8572"}}
+{"timestamp":1714201484.3387001,"name":"online","context":{"idset":"8571,8573,8607,8609"}}
+{"timestamp":1714201484.4478462,"name":"online","context":{"idset":"6624"}}
+{"timestamp":1714201484.4592872,"name":"online","context":{"idset":"8608"}}
+{"timestamp":1714201484.786773,"name":"online","context":{"idset":"6589,8574-8575,8610"}}
+{"timestamp":1714201485.1257935,"name":"online","context":{"idset":"8612"}}
+{"timestamp":1714201485.2707253,"name":"online","context":{"idset":"6571,8576,8611"}}
+{"timestamp":1714201485.3814683,"name":"online","context":{"idset":"6595"}}
+{"timestamp":1714201485.5801511,"name":"online","context":{"idset":"8579,8616"}}
+{"timestamp":1714201485.7200146,"name":"online","context":{"idset":"8577"}}
+{"timestamp":1714201485.8290505,"name":"online","context":{"idset":"8578"}}
+{"timestamp":1714201485.9551427,"name":"online","context":{"idset":"6549,8613"}}
+{"timestamp":1714201486.0774448,"name":"online","context":{"idset":"8615"}}
+{"timestamp":1714201486.1846747,"name":"online","context":{"idset":"6537"}}
+{"timestamp":1714201486.2978826,"name":"online","context":{"idset":"6580,8614"}}
+{"timestamp":1714201486.3999653,"name":"online","context":{"idset":"6544"}}
+{"timestamp":1714201486.4169593,"name":"online","context":{"idset":"6583"}}
+{"timestamp":1714201486.5768888,"name":"online","context":{"idset":"8581,8617"}}
+{"timestamp":1714201486.5894179,"name":"online","context":{"idset":"6579"}}
+{"timestamp":1714201486.6951399,"name":"online","context":{"idset":"6535,6576,8580,8582"}}
+{"timestamp":1714201486.7979763,"name":"online","context":{"idset":"6574,7356,8583"}}
+{"timestamp":1714201486.9030006,"name":"online","context":{"idset":"6554,8584,8619-8620"}}
+{"timestamp":1714201487.0930779,"name":"online","context":{"idset":"6644"}}
+{"timestamp":1714201487.2013278,"name":"online","context":{"idset":"6560,6591,7292,7305,8587,8618"}}
+{"timestamp":1714201487.3467839,"name":"online","context":{"idset":"8585,8621"}}
+{"timestamp":1714201487.5337923,"name":"online","context":{"idset":"6540,6619,8591"}}
+{"timestamp":1714201487.6444407,"name":"online","context":{"idset":"6581,6588,6626,7395,8622"}}
+{"timestamp":1714201487.7526743,"name":"online","context":{"idset":"6628,8586"}}
+{"timestamp":1714201487.9017911,"name":"online","context":{"idset":"8625-8626"}}
+{"timestamp":1714201488.0097814,"name":"online","context":{"idset":"6584,6637,8589,8627"}}
+{"timestamp":1714201488.1142912,"name":"online","context":{"idset":"6539,6545,6638,7320,7388,7394"}}
+{"timestamp":1714201488.2152379,"name":"online","context":{"idset":"8623-8624,8629"}}
+{"timestamp":1714201488.4187143,"name":"online","context":{"idset":"6561,8590,8628"}}
+{"timestamp":1714201488.5468533,"name":"online","context":{"idset":"8588,8593-8594"}}
+{"timestamp":1714201488.7169836,"name":"online","context":{"idset":"8592"}}
+{"timestamp":1714201489.0491917,"name":"online","context":{"idset":"6593,7316,8595,8630"}}
+{"timestamp":1714201489.2527645,"name":"online","context":{"idset":"6556,7289,7382,8597-8598,8631"}}
+{"timestamp":1714201489.4412477,"name":"online","context":{"idset":"6623,8596,8632-8634"}}
+{"timestamp":1714201489.5434837,"name":"online","context":{"idset":"8599,8635"}}
+{"timestamp":1714201489.6453609,"name":"online","context":{"idset":"8600"}}
+{"timestamp":1714201489.7504508,"name":"online","context":{"idset":"6614,8601,8636"}}
+{"timestamp":1714201489.8706338,"name":"online","context":{"idset":"8638"}}
+{"timestamp":1714201489.9891229,"name":"online","context":{"idset":"8637"}}
+{"timestamp":1714201490.1871564,"name":"online","context":{"idset":"7374,8603,8640"}}
+{"timestamp":1714201490.3182011,"name":"online","context":{"idset":"6643,8602"}}
+{"timestamp":1714201490.5027893,"name":"online","context":{"idset":"8639"}}
+{"timestamp":1714201490.634882,"name":"online","context":{"idset":"6621"}}
+{"timestamp":1714201490.8773367,"name":"online","context":{"idset":"8604"}}
+{"timestamp":1714201491.2051373,"name":"online","context":{"idset":"6615"}}
+{"timestamp":1714201491.5790353,"name":"online","context":{"idset":"7348"}}
+{"timestamp":1714201491.7146001,"name":"online","context":{"idset":"7330"}}
+{"timestamp":1714201491.8745532,"name":"online","context":{"idset":"7313,7380"}}
+{"timestamp":1714201492.0897701,"name":"online","context":{"idset":"6585,6639"}}
+{"timestamp":1714201492.4004242,"name":"online","context":{"idset":"7393"}}
+{"timestamp":1714201492.5576086,"name":"online","context":{"idset":"6635"}}
+{"timestamp":1714201492.714458,"name":"online","context":{"idset":"6616"}}
+{"timestamp":1714201492.9722044,"name":"online","context":{"idset":"7324,7389"}}
+{"timestamp":1714201493.1046486,"name":"online","context":{"idset":"6618,7339"}}
+{"timestamp":1714201493.7235796,"name":"online","context":{"idset":"7304"}}
+{"timestamp":1714201494.0714507,"name":"online","context":{"idset":"7303,7354-7355,7383"}}
+{"timestamp":1714201494.3960474,"name":"online","context":{"idset":"7301,7353"}}
+{"timestamp":1714201495.1128068,"name":"online","context":{"idset":"7317,7345"}}
+{"timestamp":1714201495.3421149,"name":"online","context":{"idset":"7319,7378,7398"}}
+{"timestamp":1714201495.5116553,"name":"online","context":{"idset":"7405"}}
+{"timestamp":1714201495.6431651,"name":"online","context":{"idset":"7311,7325"}}
+{"timestamp":1714201495.8894351,"name":"online","context":{"idset":"7372"}}
+{"timestamp":1714201496.2209666,"name":"online","context":{"idset":"7352"}}
+{"timestamp":1714201496.3625052,"name":"online","context":{"idset":"7391"}}
+{"timestamp":1714201496.3740132,"name":"online","context":{"idset":"7346"}}
+{"timestamp":1714201496.5692422,"name":"online","context":{"idset":"7300,7302,7392,8370"}}
+{"timestamp":1714201496.7438698,"name":"online","context":{"idset":"7360"}}
+{"timestamp":1714201496.860357,"name":"online","context":{"idset":"7387"}}
+{"timestamp":1714201496.973314,"name":"online","context":{"idset":"7403"}}
+{"timestamp":1714201497.1438208,"name":"online","context":{"idset":"7390"}}
+{"timestamp":1714201497.3696938,"name":"online","context":{"idset":"7384"}}
+{"timestamp":1714201498.0665839,"name":"online","context":{"idset":"7381"}}
+{"timestamp":1714201498.2440984,"name":"online","context":{"idset":"7342"}}
+{"timestamp":1714201498.4176011,"name":"online","context":{"idset":"7312"}}
+{"timestamp":1714201498.9406257,"name":"online","context":{"idset":"7386"}}
+{"timestamp":1714201499.0904,"name":"online","context":{"idset":"7310"}}
+{"timestamp":1714201499.2366979,"name":"online","context":{"idset":"7351"}}
+{"timestamp":1714201499.2425663,"name":"online","context":{"idset":"7396"}}
+{"timestamp":1714201499.2525656,"name":"online","context":{"idset":"8371"}}
+{"timestamp":1714201499.4026,"name":"online","context":{"idset":"8374"}}
+{"timestamp":1714201499.5334332,"name":"online","context":{"idset":"8372,8565"}}
+{"timestamp":1714201499.6476243,"name":"online","context":{"idset":"7367,8334,8369"}}
+{"timestamp":1714201499.7805159,"name":"online","context":{"idset":"7314,7336,7338"}}
+{"timestamp":1714201499.9805453,"name":"online","context":{"idset":"7331,7364,7376"}}
+{"timestamp":1714201500.2461529,"name":"online","context":{"idset":"7291"}}
+{"timestamp":1714201500.3513813,"name":"online","context":{"idset":"8333"}}
+{"timestamp":1714201500.5237293,"name":"online","context":{"idset":"7318,7344,7363,7404"}}
+{"timestamp":1714201500.764617,"name":"online","context":{"idset":"8566-8567"}}
+{"timestamp":1714201500.7808523,"name":"online","context":{"idset":"7288,7400"}}
+{"timestamp":1714201500.9677656,"name":"online","context":{"idset":"7361,7371"}}
+{"timestamp":1714201501.1057181,"name":"online","context":{"idset":"7295,7299,7335"}}
+{"timestamp":1714201501.3251288,"name":"online","context":{"idset":"7358,8568"}}
+{"timestamp":1714201501.4657042,"name":"online","context":{"idset":"7347"}}
+{"timestamp":1714201501.6238337,"name":"online","context":{"idset":"7297,7332"}}
+{"timestamp":1714201501.7747848,"name":"online","context":{"idset":"7323,7350,7368,7373"}}
+{"timestamp":1714201501.9639206,"name":"online","context":{"idset":"7409"}}
+{"timestamp":1714201502.0997083,"name":"online","context":{"idset":"7333,7343,7379"}}
+{"timestamp":1714201502.3346839,"name":"online","context":{"idset":"7408"}}
+{"timestamp":1714201502.5069382,"name":"online","context":{"idset":"7321,7370"}}
+{"timestamp":1714201502.6221104,"name":"online","context":{"idset":"7322,7327,7357,7406"}}
+{"timestamp":1714201502.8156884,"name":"online","context":{"idset":"7349"}}
+{"timestamp":1714201503.0306807,"name":"online","context":{"idset":"7375"}}
+{"timestamp":1714201503.2461865,"name":"online","context":{"idset":"7286,7377"}}
+{"timestamp":1714201503.3534176,"name":"online","context":{"idset":"7401"}}
+{"timestamp":1714201503.5857346,"name":"online","context":{"idset":"7285,7298,7366,7399"}}
+{"timestamp":1714201503.8272696,"name":"online","context":{"idset":"7287,7290,7412"}}
+{"timestamp":1714201504.2056639,"name":"online","context":{"idset":"7294"}}
+{"timestamp":1714201504.4217632,"name":"online","context":{"idset":"7296,7328,7397,7411,8335"}}
+{"timestamp":1714201504.5292413,"name":"online","context":{"idset":"8373"}}
+{"timestamp":1714201504.7482967,"name":"online","context":{"idset":"7326"}}
+{"timestamp":1714201504.9623656,"name":"online","context":{"idset":"8336,8375"}}
+{"timestamp":1714201505.7284553,"name":"online","context":{"idset":"8376"}}
+{"timestamp":1714201507.2075093,"name":"online","context":{"idset":"8406"}}
+{"timestamp":1714201507.4772193,"name":"online","context":{"idset":"8405"}}
+{"timestamp":1714201508.0150757,"name":"online","context":{"idset":"8408"}}
+{"timestamp":1714201508.4960456,"name":"online","context":{"idset":"8337"}}
+{"timestamp":1714201508.9851894,"name":"online","context":{"idset":"8377"}}
+{"timestamp":1714201510.9065747,"name":"online","context":{"idset":"8407"}}
+{"timestamp":1714201511.0701463,"name":"online","context":{"idset":"8339"}}
+{"timestamp":1714201511.8019919,"name":"online","context":{"idset":"8338"}}
+{"timestamp":1714201511.9740536,"name":"online","context":{"idset":"8383"}}
+{"timestamp":1714201512.4798543,"name":"online","context":{"idset":"8382"}}
+{"timestamp":1714201512.8478255,"name":"online","context":{"idset":"8315"}}
+{"timestamp":1714201513.0285769,"name":"online","context":{"idset":"8410"}}
+{"timestamp":1714201513.2339487,"name":"online","context":{"idset":"8341,8384-8385"}}
+{"timestamp":1714201513.7518241,"name":"online","context":{"idset":"8340"}}
+{"timestamp":1714201514.9821012,"name":"online","context":{"idset":"8388"}}
+{"timestamp":1714201515.4443793,"name":"online","context":{"idset":"8317"}}
+{"timestamp":1714201515.552083,"name":"online","context":{"idset":"8343"}}
+{"timestamp":1714201516.1324694,"name":"online","context":{"idset":"8409"}}
+{"timestamp":1714201517.0583189,"name":"online","context":{"idset":"8412"}}
+{"timestamp":1714201517.6716056,"name":"online","context":{"idset":"8344"}}
+{"timestamp":1714201517.8914475,"name":"online","context":{"idset":"8414,8421"}}
+{"timestamp":1714201518.180593,"name":"online","context":{"idset":"8318"}}
+{"timestamp":1714201518.4865208,"name":"online","context":{"idset":"8316"}}
+{"timestamp":1714201518.7187402,"name":"online","context":{"idset":"8422"}}
+{"timestamp":1714201518.8368356,"name":"online","context":{"idset":"8321,8342,8411,8416"}}
+{"timestamp":1714201518.9718144,"name":"online","context":{"idset":"8320"}}
+{"timestamp":1714201519.3346934,"name":"online","context":{"idset":"8418"}}
+{"timestamp":1714201519.5294235,"name":"online","context":{"idset":"8319"}}
+{"timestamp":1714201519.8816354,"name":"online","context":{"idset":"8424"}}
+{"timestamp":1714201520.0912619,"name":"online","context":{"idset":"8323,8413,8415,8420"}}
+{"timestamp":1714201520.9137652,"name":"online","context":{"idset":"8322,8324,8419"}}
+{"timestamp":1714201521.0581646,"name":"online","context":{"idset":"8329"}}
+{"timestamp":1714201521.2066638,"name":"online","context":{"idset":"8417"}}
+{"timestamp":1714201521.7201891,"name":"online","context":{"idset":"8327"}}
+{"timestamp":1714201521.8567197,"name":"online","context":{"idset":"8345"}}
+{"timestamp":1714201521.9679585,"name":"online","context":{"idset":"8325"}}
+{"timestamp":1714201522.1972611,"name":"online","context":{"idset":"8331"}}
+{"timestamp":1714201522.4872718,"name":"online","context":{"idset":"8348"}}
+{"timestamp":1714201523.1338253,"name":"online","context":{"idset":"8390,8423"}}
+{"timestamp":1714201523.7137847,"name":"online","context":{"idset":"8332"}}
+{"timestamp":1714201523.9536626,"name":"online","context":{"idset":"8347"}}
+{"timestamp":1714201524.318543,"name":"online","context":{"idset":"8358"}}
+{"timestamp":1714201524.5277746,"name":"online","context":{"idset":"8328,8330"}}
+{"timestamp":1714201524.7468369,"name":"online","context":{"idset":"8326,8346"}}
+{"timestamp":1714201525.1047649,"name":"online","context":{"idset":"8357,8365"}}
+{"timestamp":1714201525.3826349,"name":"online","context":{"idset":"8363-8364"}}
+{"timestamp":1714201525.7005486,"name":"online","context":{"idset":"8349,8361"}}
+{"timestamp":1714201525.967782,"name":"online","context":{"idset":"8359-8360"}}
+{"timestamp":1714201526.1733735,"name":"online","context":{"idset":"8351,8362,8367"}}
+{"timestamp":1714201526.3044143,"name":"online","context":{"idset":"8368"}}
+{"timestamp":1714201526.4775107,"name":"online","context":{"idset":"8350,8366"}}
+{"timestamp":1714201526.6910017,"name":"online","context":{"idset":"8389"}}
+{"timestamp":1714201526.8138807,"name":"online","context":{"idset":"8425"}}
+{"timestamp":1714201526.9356208,"name":"online","context":{"idset":"8391"}}
+{"timestamp":1714201527.4076889,"name":"online","context":{"idset":"8352-8353"}}
+{"timestamp":1714201527.9256473,"name":"online","context":{"idset":"8355,8393,8426-8427"}}
+{"timestamp":1714201528.4551628,"name":"online","context":{"idset":"8356,8392,8428"}}
+{"timestamp":1714201528.6357441,"name":"online","context":{"idset":"8354"}}
+{"timestamp":1714201528.9783795,"name":"online","context":{"idset":"8395"}}
+{"timestamp":1714201529.1999848,"name":"online","context":{"idset":"8396,8429,8431"}}
+{"timestamp":1714201529.5425022,"name":"online","context":{"idset":"8394,8403,8432-8433"}}
+{"timestamp":1714201529.7572615,"name":"online","context":{"idset":"8397"}}
+{"timestamp":1714201529.8557017,"name":"online","context":{"idset":"8435"}}
+{"timestamp":1714201529.9628289,"name":"online","context":{"idset":"8399-8400"}}
+{"timestamp":1714201530.0659318,"name":"online","context":{"idset":"8430,8434"}}
+{"timestamp":1714201530.2711465,"name":"online","context":{"idset":"8401,8436"}}
+{"timestamp":1714201530.5822728,"name":"online","context":{"idset":"8398"}}
+{"timestamp":1714201530.9819589,"name":"online","context":{"idset":"8402"}}
+{"timestamp":1714201531.2788262,"name":"online","context":{"idset":"8404"}}
+{"timestamp":1714201843.0529094,"name":"drain","context":{"idset":"6523-6644,7285-7412,8309-8436,8565-8692","overwrite":0}}
+{"timestamp":1714201881.7480383,"name":"offline","context":{"idset":"6561"}}
+{"timestamp":1714201881.8465087,"name":"offline","context":{"idset":"6566"}}
+{"timestamp":1714201881.9827538,"name":"offline","context":{"idset":"6534"}}
+{"timestamp":1714201881.9989021,"name":"offline","context":{"idset":"6578"}}
+{"timestamp":1714201882.0999045,"name":"offline","context":{"idset":"6547"}}
+{"timestamp":1714201882.2231705,"name":"offline","context":{"idset":"6537"}}
+{"timestamp":1714201882.2281144,"name":"offline","context":{"idset":"6524"}}
+{"timestamp":1714201882.234643,"name":"offline","context":{"idset":"6542"}}
+{"timestamp":1714201882.2401814,"name":"offline","context":{"idset":"6548"}}
+{"timestamp":1714201882.2626159,"name":"offline","context":{"idset":"6574"}}
+{"timestamp":1714201882.2676649,"name":"offline","context":{"idset":"6546"}}
+{"timestamp":1714201882.3486228,"name":"offline","context":{"idset":"6560"}}
+{"timestamp":1714201882.4886017,"name":"offline","context":{"idset":"6630"}}
+{"timestamp":1714201882.4941633,"name":"offline","context":{"idset":"6523"}}
+{"timestamp":1714201882.5014489,"name":"offline","context":{"idset":"6539"}}
+{"timestamp":1714201882.5065913,"name":"offline","context":{"idset":"6582"}}
+{"timestamp":1714201882.5663428,"name":"offline","context":{"idset":"6614"}}
+{"timestamp":1714201882.5744214,"name":"offline","context":{"idset":"6540"}}
+{"timestamp":1714201882.5787232,"name":"offline","context":{"idset":"6526"}}
+{"timestamp":1714201882.5934787,"name":"offline","context":{"idset":"6550"}}
+{"timestamp":1714201882.597894,"name":"offline","context":{"idset":"6553"}}
+{"timestamp":1714201882.6026356,"name":"offline","context":{"idset":"6590"}}
+{"timestamp":1714201882.6068878,"name":"offline","context":{"idset":"6594"}}
+{"timestamp":1714201882.6113453,"name":"offline","context":{"idset":"6612"}}
+{"timestamp":1714201882.6149747,"name":"offline","context":{"idset":"7375"}}
+{"timestamp":1714201882.6404831,"name":"offline","context":{"idset":"7394"}}
+{"timestamp":1714201882.6449668,"name":"offline","context":{"idset":"7412"}}
+{"timestamp":1714201882.7495713,"name":"offline","context":{"idset":"6530"}}
+{"timestamp":1714201882.93661,"name":"offline","context":{"idset":"7366"}}
+{"timestamp":1714201882.9415541,"name":"offline","context":{"idset":"7289"}}
+{"timestamp":1714201883.1798108,"name":"offline","context":{"idset":"7384"}}
+{"timestamp":1714201883.1847868,"name":"offline","context":{"idset":"8313"}}
+{"timestamp":1714201883.2004793,"name":"offline","context":{"idset":"6528"}}
+{"timestamp":1714201883.2049246,"name":"offline","context":{"idset":"6543"}}
+{"timestamp":1714201883.2094052,"name":"offline","context":{"idset":"6545"}}
+{"timestamp":1714201883.2142041,"name":"offline","context":{"idset":"6552"}}
+{"timestamp":1714201883.2732131,"name":"offline","context":{"idset":"6555"}}
+{"timestamp":1714201883.2777536,"name":"offline","context":{"idset":"6557"}}
+{"timestamp":1714201883.294524,"name":"offline","context":{"idset":"6564"}}
+{"timestamp":1714201883.2999432,"name":"offline","context":{"idset":"6565"}}
+{"timestamp":1714201883.3046057,"name":"offline","context":{"idset":"6568"}}
+{"timestamp":1714201883.3090467,"name":"offline","context":{"idset":"6579"}}
+{"timestamp":1714201883.3126984,"name":"offline","context":{"idset":"6585"}}
+{"timestamp":1714201883.3173134,"name":"offline","context":{"idset":"6596"}}
+{"timestamp":1714201883.3225543,"name":"offline","context":{"idset":"6599"}}
+{"timestamp":1714201883.3269367,"name":"offline","context":{"idset":"6602"}}
+{"timestamp":1714201883.3312962,"name":"offline","context":{"idset":"6604"}}
+{"timestamp":1714201883.3456349,"name":"offline","context":{"idset":"6605"}}
+{"timestamp":1714201883.3511717,"name":"offline","context":{"idset":"6606"}}
+{"timestamp":1714201883.3559306,"name":"offline","context":{"idset":"6607"}}
+{"timestamp":1714201883.3606119,"name":"offline","context":{"idset":"6610"}}
+{"timestamp":1714201883.3650351,"name":"offline","context":{"idset":"6615"}}
+{"timestamp":1714201883.3697839,"name":"offline","context":{"idset":"6617"}}
+{"timestamp":1714201883.3744168,"name":"offline","context":{"idset":"6625"}}
+{"timestamp":1714201883.3792002,"name":"offline","context":{"idset":"6631"}}
+{"timestamp":1714201883.384028,"name":"offline","context":{"idset":"6632"}}
+{"timestamp":1714201883.3987367,"name":"offline","context":{"idset":"6642"}}
+{"timestamp":1714201883.4245951,"name":"offline","context":{"idset":"6643"}}
+{"timestamp":1714201883.4501207,"name":"offline","context":{"idset":"7288"}}
+{"timestamp":1714201883.4652317,"name":"offline","context":{"idset":"7294"}}
+{"timestamp":1714201883.4697189,"name":"offline","context":{"idset":"7299"}}
+{"timestamp":1714201883.4755583,"name":"offline","context":{"idset":"7309"}}
+{"timestamp":1714201883.4803984,"name":"offline","context":{"idset":"7310"}}
+{"timestamp":1714201883.4853501,"name":"offline","context":{"idset":"7311"}}
+{"timestamp":1714201883.4903069,"name":"offline","context":{"idset":"7321"}}
+{"timestamp":1714201883.5134525,"name":"offline","context":{"idset":"7325"}}
+{"timestamp":1714201883.5281084,"name":"offline","context":{"idset":"7327"}}
+{"timestamp":1714201883.5327525,"name":"offline","context":{"idset":"7328"}}
+{"timestamp":1714201883.5374999,"name":"offline","context":{"idset":"7331"}}
+{"timestamp":1714201883.5420003,"name":"offline","context":{"idset":"7334"}}
+{"timestamp":1714201883.5466383,"name":"offline","context":{"idset":"7350"}}
+{"timestamp":1714201883.5509672,"name":"offline","context":{"idset":"7362"}}
+{"timestamp":1714201883.5746946,"name":"offline","context":{"idset":"7365"}}
+{"timestamp":1714201883.5984733,"name":"offline","context":{"idset":"7370"}}
+{"timestamp":1714201883.6033399,"name":"offline","context":{"idset":"7374"}}
+{"timestamp":1714201883.6075726,"name":"offline","context":{"idset":"7376"}}
+{"timestamp":1714201883.6120157,"name":"offline","context":{"idset":"7382"}}
+{"timestamp":1714201883.6161666,"name":"offline","context":{"idset":"7390"}}
+{"timestamp":1714201883.6204486,"name":"offline","context":{"idset":"7391"}}
+{"timestamp":1714201883.6321626,"name":"offline","context":{"idset":"7393"}}
+{"timestamp":1714201883.6516614,"name":"offline","context":{"idset":"7400"}}
+{"timestamp":1714201883.6661906,"name":"offline","context":{"idset":"7401"}}
+{"timestamp":1714201883.6706171,"name":"offline","context":{"idset":"7402"}}
+{"timestamp":1714201883.6755123,"name":"offline","context":{"idset":"7410"}}
+{"timestamp":1714201883.6798384,"name":"offline","context":{"idset":"8309"}}
+{"timestamp":1714201883.6847184,"name":"offline","context":{"idset":"8310"}}
+{"timestamp":1714201883.6999106,"name":"offline","context":{"idset":"8311"}}
+{"timestamp":1714201883.7229512,"name":"offline","context":{"idset":"8314"}}
+{"timestamp":1714201883.747767,"name":"offline","context":{"idset":"6531"}}
+{"timestamp":1714201883.7520714,"name":"offline","context":{"idset":"6532"}}
+{"timestamp":1714201883.7564681,"name":"offline","context":{"idset":"6533"}}
+{"timestamp":1714201883.7603548,"name":"offline","context":{"idset":"6535"}}
+{"timestamp":1714201883.7651265,"name":"offline","context":{"idset":"6536"}}
+{"timestamp":1714201883.7829401,"name":"offline","context":{"idset":"6538"}}
+{"timestamp":1714201883.8075294,"name":"offline","context":{"idset":"6551"}}
+{"timestamp":1714201883.8121886,"name":"offline","context":{"idset":"6573"}}
+{"timestamp":1714201883.8168697,"name":"offline","context":{"idset":"6584"}}
+{"timestamp":1714201883.8213825,"name":"offline","context":{"idset":"6586"}}
+{"timestamp":1714201883.8259671,"name":"offline","context":{"idset":"6608"}}
+{"timestamp":1714201883.8408723,"name":"offline","context":{"idset":"6620"}}
+{"timestamp":1714201883.845293,"name":"offline","context":{"idset":"6636"}}
+{"timestamp":1714201883.8498585,"name":"offline","context":{"idset":"7285"}}
+{"timestamp":1714201883.8543336,"name":"offline","context":{"idset":"7292"}}
+{"timestamp":1714201883.8587008,"name":"offline","context":{"idset":"7301"}}
+{"timestamp":1714201883.8631666,"name":"offline","context":{"idset":"7306"}}
+{"timestamp":1714201883.8675933,"name":"offline","context":{"idset":"7314"}}
+{"timestamp":1714201883.8820477,"name":"offline","context":{"idset":"7388"}}
+{"timestamp":1714201883.8955734,"name":"offline","context":{"idset":"6629"}}
+{"timestamp":1714201884.0342867,"name":"offline","context":{"idset":"7300"}}
+{"timestamp":1714201884.0386682,"name":"offline","context":{"idset":"7318"}}
+{"timestamp":1714201884.0430341,"name":"offline","context":{"idset":"7329"}}
+{"timestamp":1714201884.0473819,"name":"offline","context":{"idset":"7408"}}
+{"timestamp":1714201884.0520129,"name":"offline","context":{"idset":"7345"}}
+{"timestamp":1714201884.0566015,"name":"offline","context":{"idset":"7403"}}
+{"timestamp":1714201884.0723362,"name":"offline","context":{"idset":"7398"}}
+{"timestamp":1714201884.0768163,"name":"offline","context":{"idset":"7304"}}
+{"timestamp":1714201884.0817645,"name":"offline","context":{"idset":"6622"}}
+{"timestamp":1714201884.0858297,"name":"offline","context":{"idset":"7381"}}
+{"timestamp":1714201884.0898609,"name":"offline","context":{"idset":"7324"}}
+{"timestamp":1714201884.0936217,"name":"offline","context":{"idset":"6619"}}
+{"timestamp":1714201884.0980608,"name":"offline","context":{"idset":"6609"}}
+{"timestamp":1714201884.1095974,"name":"offline","context":{"idset":"7367"}}
+{"timestamp":1714201884.2365079,"name":"offline","context":{"idset":"7315"}}
+{"timestamp":1714201884.6155539,"name":"offline","context":{"idset":"7336"}}
+{"timestamp":1714201884.6199698,"name":"offline","context":{"idset":"7312"}}
+{"timestamp":1714201884.624748,"name":"offline","context":{"idset":"6525"}}
+{"timestamp":1714201884.6293225,"name":"offline","context":{"idset":"6581"}}
+{"timestamp":1714201884.633729,"name":"offline","context":{"idset":"6580"}}
+{"timestamp":1714201884.6385653,"name":"offline","context":{"idset":"6626"}}
+{"timestamp":1714201884.6434205,"name":"offline","context":{"idset":"7409"}}
+{"timestamp":1714201884.6481056,"name":"offline","context":{"idset":"7411"}}
+{"timestamp":1714201884.6516335,"name":"offline","context":{"idset":"7337"}}
+{"timestamp":1714201884.6557906,"name":"offline","context":{"idset":"7323"}}
+{"timestamp":1714201884.6596253,"name":"offline","context":{"idset":"7332"}}
+{"timestamp":1714201884.6634405,"name":"offline","context":{"idset":"7347"}}
+{"timestamp":1714201884.6672683,"name":"offline","context":{"idset":"6635"}}
+{"timestamp":1714201884.6711385,"name":"offline","context":{"idset":"6624"}}
+{"timestamp":1714201884.6749308,"name":"offline","context":{"idset":"7373"}}
+{"timestamp":1714201884.6788268,"name":"offline","context":{"idset":"7342"}}
+{"timestamp":1714201884.6825047,"name":"offline","context":{"idset":"7305"}}
+{"timestamp":1714201884.6862061,"name":"offline","context":{"idset":"6644"}}
+{"timestamp":1714201884.6901085,"name":"offline","context":{"idset":"6576"}}
+{"timestamp":1714201884.6938391,"name":"offline","context":{"idset":"6640"}}
+{"timestamp":1714201884.6982205,"name":"offline","context":{"idset":"7341"}}
+{"timestamp":1714201884.7026069,"name":"offline","context":{"idset":"6563"}}
+{"timestamp":1714201884.707653,"name":"offline","context":{"idset":"7296"}}
+{"timestamp":1714201884.7123072,"name":"offline","context":{"idset":"6556"}}
+{"timestamp":1714201884.7164247,"name":"offline","context":{"idset":"7319"}}
+{"timestamp":1714201884.7207949,"name":"offline","context":{"idset":"7290"}}
+{"timestamp":1714201884.7255352,"name":"offline","context":{"idset":"7297"}}
+{"timestamp":1714201884.7301898,"name":"offline","context":{"idset":"7344"}}
+{"timestamp":1714201884.7347186,"name":"offline","context":{"idset":"6628"}}
+{"timestamp":1714201884.7390387,"name":"offline","context":{"idset":"6601"}}
+{"timestamp":1714201884.7429943,"name":"offline","context":{"idset":"7360"}}
+{"timestamp":1714201884.7468803,"name":"offline","context":{"idset":"7371"}}
+{"timestamp":1714201884.7514684,"name":"offline","context":{"idset":"6527"}}
+{"timestamp":1714201884.7557511,"name":"offline","context":{"idset":"7352"}}
+{"timestamp":1714201884.7599707,"name":"offline","context":{"idset":"7372"}}
+{"timestamp":1714201884.7638421,"name":"offline","context":{"idset":"6541"}}
+{"timestamp":1714201884.769731,"name":"offline","context":{"idset":"7397"}}
+{"timestamp":1714201884.7742753,"name":"offline","context":{"idset":"7363"}}
+{"timestamp":1714201884.7785096,"name":"offline","context":{"idset":"6544"}}
+{"timestamp":1714201884.7814651,"name":"offline","context":{"idset":"6571"}}
+{"timestamp":1714201884.7845426,"name":"offline","context":{"idset":"6633"}}
+{"timestamp":1714201884.7876141,"name":"offline","context":{"idset":"6529"}}
+{"timestamp":1714201884.9749315,"name":"offline","context":{"idset":"6587"}}
+{"timestamp":1714201884.979569,"name":"offline","context":{"idset":"6549"}}
+{"timestamp":1714201884.9841883,"name":"offline","context":{"idset":"6554"}}
+{"timestamp":1714201884.9891186,"name":"offline","context":{"idset":"6558"}}
+{"timestamp":1714201884.993866,"name":"offline","context":{"idset":"6559"}}
+{"timestamp":1714201884.9979782,"name":"offline","context":{"idset":"6562"}}
+{"timestamp":1714201885.001945,"name":"offline","context":{"idset":"6567"}}
+{"timestamp":1714201885.0059421,"name":"offline","context":{"idset":"6569"}}
+{"timestamp":1714201885.0100651,"name":"offline","context":{"idset":"6570"}}
+{"timestamp":1714201885.0141077,"name":"offline","context":{"idset":"6572"}}
+{"timestamp":1714201885.0177946,"name":"offline","context":{"idset":"6575"}}
+{"timestamp":1714201885.0217938,"name":"offline","context":{"idset":"6577"}}
+{"timestamp":1714201885.0262713,"name":"offline","context":{"idset":"6583"}}
+{"timestamp":1714201885.0304258,"name":"offline","context":{"idset":"6588"}}
+{"timestamp":1714201885.0343885,"name":"offline","context":{"idset":"6589"}}
+{"timestamp":1714201885.0386891,"name":"offline","context":{"idset":"6591"}}
+{"timestamp":1714201885.043016,"name":"offline","context":{"idset":"6592"}}
+{"timestamp":1714201885.0469861,"name":"offline","context":{"idset":"6593"}}
+{"timestamp":1714201885.0510118,"name":"offline","context":{"idset":"6595"}}
+{"timestamp":1714201885.0555515,"name":"offline","context":{"idset":"6597"}}
+{"timestamp":1714201885.0709198,"name":"offline","context":{"idset":"6598"}}
+{"timestamp":1714201885.0751717,"name":"offline","context":{"idset":"6600"}}
+{"timestamp":1714201885.0791185,"name":"offline","context":{"idset":"6603"}}
+{"timestamp":1714201885.0830297,"name":"offline","context":{"idset":"6611"}}
+{"timestamp":1714201885.0874512,"name":"offline","context":{"idset":"6613"}}
+{"timestamp":1714201885.0919197,"name":"offline","context":{"idset":"6616"}}
+{"timestamp":1714201885.0964797,"name":"offline","context":{"idset":"6618"}}
+{"timestamp":1714201885.1009741,"name":"offline","context":{"idset":"6621"}}
+{"timestamp":1714201885.1051021,"name":"offline","context":{"idset":"6623"}}
+{"timestamp":1714201885.1092179,"name":"offline","context":{"idset":"6627"}}
+{"timestamp":1714201885.1131399,"name":"offline","context":{"idset":"6634"}}
+{"timestamp":1714201885.117002,"name":"offline","context":{"idset":"6637"}}
+{"timestamp":1714201885.1211231,"name":"offline","context":{"idset":"6638"}}
+{"timestamp":1714201885.1252918,"name":"offline","context":{"idset":"6639"}}
+{"timestamp":1714201885.1297376,"name":"offline","context":{"idset":"6641"}}
+{"timestamp":1714201885.1339092,"name":"offline","context":{"idset":"7286"}}
+{"timestamp":1714201885.1378446,"name":"offline","context":{"idset":"7287"}}
+{"timestamp":1714201885.1417143,"name":"offline","context":{"idset":"7291"}}
+{"timestamp":1714201885.1456788,"name":"offline","context":{"idset":"7293"}}
+{"timestamp":1714201885.1499314,"name":"offline","context":{"idset":"7295"}}
+{"timestamp":1714201885.1540751,"name":"offline","context":{"idset":"7298"}}
+{"timestamp":1714201885.1583438,"name":"offline","context":{"idset":"7302"}}
+{"timestamp":1714201885.1630855,"name":"offline","context":{"idset":"7303"}}
+{"timestamp":1714201885.1676908,"name":"offline","context":{"idset":"7307"}}
+{"timestamp":1714201885.1721938,"name":"offline","context":{"idset":"7308"}}
+{"timestamp":1714201885.1772892,"name":"offline","context":{"idset":"7313"}}
+{"timestamp":1714201885.1814458,"name":"offline","context":{"idset":"7316"}}
+{"timestamp":1714201885.1855311,"name":"offline","context":{"idset":"7317"}}
+{"timestamp":1714201885.189786,"name":"offline","context":{"idset":"7320"}}
+{"timestamp":1714201885.1940119,"name":"offline","context":{"idset":"7322"}}
+{"timestamp":1714201885.1985002,"name":"offline","context":{"idset":"7326"}}
+{"timestamp":1714201885.2030339,"name":"offline","context":{"idset":"7330"}}
+{"timestamp":1714201885.2069869,"name":"offline","context":{"idset":"7333"}}
+{"timestamp":1714201885.211026,"name":"offline","context":{"idset":"7335"}}
+{"timestamp":1714201885.2149515,"name":"offline","context":{"idset":"7338"}}
+{"timestamp":1714201885.2192066,"name":"offline","context":{"idset":"7339"}}
+{"timestamp":1714201885.223562,"name":"offline","context":{"idset":"7340"}}
+{"timestamp":1714201885.2286263,"name":"offline","context":{"idset":"7343"}}
+{"timestamp":1714201885.2327468,"name":"offline","context":{"idset":"7346"}}
+{"timestamp":1714201885.2368491,"name":"offline","context":{"idset":"7348"}}
+{"timestamp":1714201885.2412107,"name":"offline","context":{"idset":"7349"}}
+{"timestamp":1714201885.2456706,"name":"offline","context":{"idset":"7351"}}
+{"timestamp":1714201885.2500877,"name":"offline","context":{"idset":"7353"}}
+{"timestamp":1714201885.2547779,"name":"offline","context":{"idset":"7354"}}
+{"timestamp":1714201885.2593381,"name":"offline","context":{"idset":"7355"}}
+{"timestamp":1714201885.2640169,"name":"offline","context":{"idset":"7356"}}
+{"timestamp":1714201885.2687743,"name":"offline","context":{"idset":"7357"}}
+{"timestamp":1714201885.2769339,"name":"offline","context":{"idset":"7358"}}
+{"timestamp":1714201885.2807603,"name":"offline","context":{"idset":"7359"}}
+{"timestamp":1714201885.2853055,"name":"offline","context":{"idset":"7361"}}
+{"timestamp":1714201885.2900519,"name":"offline","context":{"idset":"7364"}}
+{"timestamp":1714201885.2943916,"name":"offline","context":{"idset":"7368"}}
+{"timestamp":1714201885.2988331,"name":"offline","context":{"idset":"7369"}}
+{"timestamp":1714201885.303339,"name":"offline","context":{"idset":"7377"}}
+{"timestamp":1714201885.3167746,"name":"offline","context":{"idset":"7378"}}
+{"timestamp":1714201885.3419316,"name":"offline","context":{"idset":"7379"}}
+{"timestamp":1714201885.3651044,"name":"offline","context":{"idset":"7380"}}
+{"timestamp":1714201885.3862007,"name":"offline","context":{"idset":"7383"}}
+{"timestamp":1714201885.4103408,"name":"offline","context":{"idset":"7385"}}
+{"timestamp":1714201885.4141858,"name":"offline","context":{"idset":"7386"}}
+{"timestamp":1714201885.4180963,"name":"offline","context":{"idset":"7387"}}
+{"timestamp":1714201885.4248898,"name":"offline","context":{"idset":"7389"}}
+{"timestamp":1714201885.4289784,"name":"offline","context":{"idset":"7392"}}
+{"timestamp":1714201885.4328535,"name":"offline","context":{"idset":"7395"}}
+{"timestamp":1714201885.4369392,"name":"offline","context":{"idset":"7396"}}
+{"timestamp":1714201885.4413197,"name":"offline","context":{"idset":"7399"}}
+{"timestamp":1714201885.4453535,"name":"offline","context":{"idset":"7404"}}
+{"timestamp":1714201885.4521077,"name":"offline","context":{"idset":"7405"}}
+{"timestamp":1714201885.456526,"name":"offline","context":{"idset":"7406"}}
+{"timestamp":1714201885.4606431,"name":"offline","context":{"idset":"7407"}}
+{"timestamp":1714201885.4648385,"name":"offline","context":{"idset":"8312"}}
+{"timestamp":1714201887.4034731,"name":"offline","context":{"idset":"8642"}}
+{"timestamp":1714201887.5036113,"name":"offline","context":{"idset":"8678"}}
+{"timestamp":1714201889.1001775,"name":"offline","context":{"idset":"8643"}}
+{"timestamp":1714201890.866972,"name":"offline","context":{"idset":"8679"}}
+{"timestamp":1714201896.4527996,"name":"offline","context":{"idset":"8644"}}
+{"timestamp":1714201897.73228,"name":"offline","context":{"idset":"8680"}}
+{"timestamp":1714201898.2299066,"name":"offline","context":{"idset":"8645"}}
+{"timestamp":1714201898.3306565,"name":"offline","context":{"idset":"8681"}}
+{"timestamp":1714201898.711761,"name":"offline","context":{"idset":"8646"}}
+{"timestamp":1714201898.8312976,"name":"offline","context":{"idset":"8682"}}
+{"timestamp":1714201899.0356157,"name":"offline","context":{"idset":"8647"}}
+{"timestamp":1714201899.2927129,"name":"offline","context":{"idset":"8683"}}
+{"timestamp":1714201899.5630674,"name":"offline","context":{"idset":"8648"}}
+{"timestamp":1714201899.847965,"name":"offline","context":{"idset":"8649"}}
+{"timestamp":1714201899.9527419,"name":"offline","context":{"idset":"8684"}}
+{"timestamp":1714201900.0894601,"name":"offline","context":{"idset":"8685"}}
+{"timestamp":1714201900.098453,"name":"offline","context":{"idset":"8650"}}
+{"timestamp":1714201900.1916049,"name":"offline","context":{"idset":"8686"}}
+{"timestamp":1714201900.4362168,"name":"offline","context":{"idset":"8651"}}
+{"timestamp":1714201900.6317146,"name":"offline","context":{"idset":"8687"}}
+{"timestamp":1714201900.7342627,"name":"offline","context":{"idset":"8652"}}
+{"timestamp":1714201900.8424649,"name":"offline","context":{"idset":"8688"}}
+{"timestamp":1714201901.0076377,"name":"offline","context":{"idset":"8653"}}
+{"timestamp":1714201901.1065214,"name":"offline","context":{"idset":"8655"}}
+{"timestamp":1714201901.1149013,"name":"offline","context":{"idset":"8654"}}
+{"timestamp":1714201901.2141383,"name":"offline","context":{"idset":"8689"}}
+{"timestamp":1714201901.3155489,"name":"offline","context":{"idset":"8690"}}
+{"timestamp":1714201901.4140797,"name":"offline","context":{"idset":"8656"}}
+{"timestamp":1714201901.4372149,"name":"offline","context":{"idset":"8657"}}
+{"timestamp":1714201901.442446,"name":"offline","context":{"idset":"8692"}}
+{"timestamp":1714201901.5369272,"name":"offline","context":{"idset":"8691"}}
+{"timestamp":1714201901.6663208,"name":"offline","context":{"idset":"8662"}}
+{"timestamp":1714201901.6734691,"name":"offline","context":{"idset":"8658"}}
+{"timestamp":1714201901.6806569,"name":"offline","context":{"idset":"8660"}}
+{"timestamp":1714201901.7750978,"name":"offline","context":{"idset":"8659"}}
+{"timestamp":1714201901.810993,"name":"offline","context":{"idset":"8663"}}
+{"timestamp":1714201901.8181798,"name":"offline","context":{"idset":"8661"}}
+{"timestamp":1714201901.9103484,"name":"offline","context":{"idset":"8664"}}
+{"timestamp":1714201902.0102022,"name":"offline","context":{"idset":"8671"}}
+{"timestamp":1714201902.0160315,"name":"offline","context":{"idset":"8672"}}
+{"timestamp":1714201902.0215886,"name":"offline","context":{"idset":"8665"}}
+{"timestamp":1714201902.0270946,"name":"offline","context":{"idset":"8670"}}
+{"timestamp":1714201902.0325851,"name":"offline","context":{"idset":"8666"}}
+{"timestamp":1714201902.0377171,"name":"offline","context":{"idset":"8669"}}
+{"timestamp":1714201902.0491192,"name":"offline","context":{"idset":"8668"}}
+{"timestamp":1714201902.1346004,"name":"offline","context":{"idset":"8673"}}
+{"timestamp":1714201902.2452374,"name":"offline","context":{"idset":"8667"}}
+{"timestamp":1714201902.2541978,"name":"offline","context":{"idset":"8677"}}
+{"timestamp":1714201902.2629688,"name":"offline","context":{"idset":"8570"}}
+{"timestamp":1714201902.2700653,"name":"offline","context":{"idset":"8676"}}
+{"timestamp":1714201902.2771091,"name":"offline","context":{"idset":"8675"}}
+{"timestamp":1714201902.2841592,"name":"offline","context":{"idset":"8606"}}
+{"timestamp":1714201902.301091,"name":"offline","context":{"idset":"8674"}}
+{"timestamp":1714201902.3081441,"name":"offline","context":{"idset":"8571"}}
+{"timestamp":1714201902.3316715,"name":"offline","context":{"idset":"8573"}}
+{"timestamp":1714201902.4137416,"name":"offline","context":{"idset":"8607"}}
+{"timestamp":1714201902.5185525,"name":"offline","context":{"idset":"8575"}}
+{"timestamp":1714201902.5249116,"name":"offline","context":{"idset":"8608"}}
+{"timestamp":1714201902.5387661,"name":"offline","context":{"idset":"8572"}}
+{"timestamp":1714201902.5419085,"name":"offline","context":{"idset":"8610"}}
+{"timestamp":1714201902.5700707,"name":"offline","context":{"idset":"8609"}}
+{"timestamp":1714201902.5759647,"name":"offline","context":{"idset":"8574"}}
+{"timestamp":1714201902.5906913,"name":"offline","context":{"idset":"8576"}}
+{"timestamp":1714201902.6552682,"name":"offline","context":{"idset":"8611"}}
+{"timestamp":1714201902.8309515,"name":"offline","context":{"idset":"8612"}}
+{"timestamp":1714201902.8339841,"name":"offline","context":{"idset":"8578"}}
+{"timestamp":1714201902.836967,"name":"offline","context":{"idset":"8577"}}
+{"timestamp":1714201902.8399661,"name":"offline","context":{"idset":"8613"}}
+{"timestamp":1714201902.8446558,"name":"offline","context":{"idset":"8617"}}
+{"timestamp":1714201902.8502121,"name":"offline","context":{"idset":"8579"}}
+{"timestamp":1714201902.8531322,"name":"offline","context":{"idset":"8615"}}
+{"timestamp":1714201902.8561206,"name":"offline","context":{"idset":"8614"}}
+{"timestamp":1714201902.9542716,"name":"offline","context":{"idset":"8580"}}
+{"timestamp":1714201903.0424776,"name":"offline","context":{"idset":"8582"}}
+{"timestamp":1714201903.0455182,"name":"offline","context":{"idset":"8581"}}
+{"timestamp":1714201903.0510345,"name":"offline","context":{"idset":"8583"}}
+{"timestamp":1714201903.0559976,"name":"offline","context":{"idset":"8616"}}
+{"timestamp":1714201903.0600021,"name":"offline","context":{"idset":"8618"}}
+{"timestamp":1714201903.1573632,"name":"offline","context":{"idset":"8619"}}
+{"timestamp":1714201903.2503676,"name":"offline","context":{"idset":"8584"}}
+{"timestamp":1714201903.3491683,"name":"offline","context":{"idset":"8620"}}
+{"timestamp":1714201903.5011864,"name":"offline","context":{"idset":"8585"}}
+{"timestamp":1714201903.6005898,"name":"offline","context":{"idset":"8621"}}
+{"timestamp":1714201903.7046478,"name":"offline","context":{"idset":"8586"}}
+{"timestamp":1714201903.7077334,"name":"offline","context":{"idset":"8587"}}
+{"timestamp":1714201903.7202692,"name":"offline","context":{"idset":"8623"}}
+{"timestamp":1714201903.8197837,"name":"offline","context":{"idset":"8622"}}
+{"timestamp":1714201903.9097202,"name":"offline","context":{"idset":"8588"}}
+{"timestamp":1714201904.0083287,"name":"offline","context":{"idset":"8624"}}
+{"timestamp":1714201904.0942504,"name":"offline","context":{"idset":"8590"}}
+{"timestamp":1714201904.1908536,"name":"offline","context":{"idset":"8625"}}
+{"timestamp":1714201904.2807887,"name":"offline","context":{"idset":"8589"}}
+{"timestamp":1714201904.3815737,"name":"offline","context":{"idset":"8626"}}
+{"timestamp":1714201904.4891717,"name":"offline","context":{"idset":"8591"}}
+{"timestamp":1714201904.4983742,"name":"offline","context":{"idset":"8628"}}
+{"timestamp":1714201904.5872955,"name":"offline","context":{"idset":"8592"}}
+{"timestamp":1714201904.723732,"name":"offline","context":{"idset":"8627"}}
+{"timestamp":1714201904.7284038,"name":"offline","context":{"idset":"8593"}}
+{"timestamp":1714201904.7321227,"name":"offline","context":{"idset":"8629"}}
+{"timestamp":1714201904.7377307,"name":"offline","context":{"idset":"8594"}}
+{"timestamp":1714201904.7425148,"name":"offline","context":{"idset":"8595"}}
+{"timestamp":1714201904.8740351,"name":"offline","context":{"idset":"8630"}}
+{"timestamp":1714201904.9958713,"name":"offline","context":{"idset":"8633"}}
+{"timestamp":1714201905.003293,"name":"offline","context":{"idset":"8596"}}
+{"timestamp":1714201905.0112154,"name":"offline","context":{"idset":"8631"}}
+{"timestamp":1714201905.100939,"name":"offline","context":{"idset":"8632"}}
+{"timestamp":1714201905.2310655,"name":"offline","context":{"idset":"8634"}}
+{"timestamp":1714201905.2563829,"name":"offline","context":{"idset":"8597"}}
+{"timestamp":1714201905.3540878,"name":"offline","context":{"idset":"8598"}}
+{"timestamp":1714201905.4591506,"name":"offline","context":{"idset":"8600"}}
+{"timestamp":1714201905.4649327,"name":"offline","context":{"idset":"8635"}}
+{"timestamp":1714201905.5638952,"name":"offline","context":{"idset":"8599"}}
+{"timestamp":1714201905.6939054,"name":"offline","context":{"idset":"8637"}}
+{"timestamp":1714201905.6977549,"name":"offline","context":{"idset":"8636"}}
+{"timestamp":1714201905.7006085,"name":"offline","context":{"idset":"8601"}}
+{"timestamp":1714201905.7041006,"name":"offline","context":{"idset":"8638"}}
+{"timestamp":1714201905.8017344,"name":"offline","context":{"idset":"8602"}}
+{"timestamp":1714201905.939266,"name":"offline","context":{"idset":"8640"}}
+{"timestamp":1714201905.9483252,"name":"offline","context":{"idset":"8604"}}
+{"timestamp":1714201905.95906,"name":"offline","context":{"idset":"8639"}}
+{"timestamp":1714201905.9634778,"name":"offline","context":{"idset":"8603"}}
+{"timestamp":1714201906.0490196,"name":"offline","context":{"idset":"8605"}}
+{"timestamp":1714201906.2541342,"name":"offline","context":{"idset":"8334"}}
+{"timestamp":1714201906.3537669,"name":"offline","context":{"idset":"8641"}}
+{"timestamp":1714201906.4564862,"name":"offline","context":{"idset":"8370"}}
+{"timestamp":1714201906.5521734,"name":"offline","context":{"idset":"8371"}}
+{"timestamp":1714201906.6501853,"name":"offline","context":{"idset":"8335"}}
+{"timestamp":1714201906.7481873,"name":"offline","context":{"idset":"8336"}}
+{"timestamp":1714201906.7533779,"name":"offline","context":{"idset":"8406"}}
+{"timestamp":1714201906.8509784,"name":"offline","context":{"idset":"8407"}}
+{"timestamp":1714201906.9727981,"name":"offline","context":{"idset":"8372"}}
+{"timestamp":1714201907.0740421,"name":"offline","context":{"idset":"8408"}}
+{"timestamp":1714201907.1835923,"name":"offline","context":{"idset":"8338"}}
+{"timestamp":1714201907.1917601,"name":"offline","context":{"idset":"8374"}}
+{"timestamp":1714201907.2906961,"name":"offline","context":{"idset":"8337"}}
+{"timestamp":1714201907.3770626,"name":"offline","context":{"idset":"8410"}}
+{"timestamp":1714201907.3845589,"name":"offline","context":{"idset":"8373"}}
+{"timestamp":1714201907.3891039,"name":"offline","context":{"idset":"8409"}}
+{"timestamp":1714201907.3936539,"name":"offline","context":{"idset":"8375"}}
+{"timestamp":1714201907.4893064,"name":"offline","context":{"idset":"8340"}}
+{"timestamp":1714201907.4938102,"name":"offline","context":{"idset":"8339"}}
+{"timestamp":1714201907.5938175,"name":"offline","context":{"idset":"8411"}}
+{"timestamp":1714201907.6013594,"name":"offline","context":{"idset":"8376"}}
+{"timestamp":1714201907.6971815,"name":"offline","context":{"idset":"8412"}}
+{"timestamp":1714201907.8039804,"name":"offline","context":{"idset":"8377"}}
+{"timestamp":1714201907.8218317,"name":"offline","context":{"idset":"8413"}}
+{"timestamp":1714201907.830049,"name":"offline","context":{"idset":"8342"}}
+{"timestamp":1714201907.9233818,"name":"offline","context":{"idset":"8341"}}
+{"timestamp":1714201908.0590484,"name":"offline","context":{"idset":"8343"}}
+{"timestamp":1714201908.1540897,"name":"offline","context":{"idset":"8414"}}
+{"timestamp":1714201908.1722109,"name":"offline","context":{"idset":"8415"}}
+{"timestamp":1714201908.267925,"name":"offline","context":{"idset":"8416"}}
+{"timestamp":1714201908.393683,"name":"offline","context":{"idset":"8417"}}
+{"timestamp":1714201908.399102,"name":"offline","context":{"idset":"8344"}}
+{"timestamp":1714201908.4062896,"name":"offline","context":{"idset":"8345"}}
+{"timestamp":1714201908.4123478,"name":"offline","context":{"idset":"8418"}}
+{"timestamp":1714201908.5149977,"name":"offline","context":{"idset":"8346"}}
+{"timestamp":1714201908.6323013,"name":"offline","context":{"idset":"8383"}}
+{"timestamp":1714201908.6391807,"name":"offline","context":{"idset":"8347"}}
+{"timestamp":1714201908.6487792,"name":"offline","context":{"idset":"8382"}}
+{"timestamp":1714201908.6556559,"name":"offline","context":{"idset":"8348"}}
+{"timestamp":1714201908.748575,"name":"offline","context":{"idset":"8419"}}
+{"timestamp":1714201908.9007621,"name":"offline","context":{"idset":"8421"}}
+{"timestamp":1714201908.9078546,"name":"offline","context":{"idset":"8315"}}
+{"timestamp":1714201908.9115684,"name":"offline","context":{"idset":"8350"}}
+{"timestamp":1714201908.9154935,"name":"offline","context":{"idset":"8384"}}
+{"timestamp":1714201908.9207466,"name":"offline","context":{"idset":"8420"}}
+{"timestamp":1714201908.9246838,"name":"offline","context":{"idset":"8349"}}
+{"timestamp":1714201908.9283938,"name":"offline","context":{"idset":"8385"}}
+{"timestamp":1714201909.0301552,"name":"offline","context":{"idset":"8422"}}
+{"timestamp":1714201909.1343386,"name":"offline","context":{"idset":"8423"}}
+{"timestamp":1714201909.1414599,"name":"offline","context":{"idset":"8316"}}
+{"timestamp":1714201909.2361276,"name":"offline","context":{"idset":"8351"}}
+{"timestamp":1714201909.2880085,"name":"offline","context":{"idset":"8352"}}
+{"timestamp":1714201909.3770065,"name":"offline","context":{"idset":"8424"}}
+{"timestamp":1714201909.4069378,"name":"offline","context":{"idset":"8353"}}
+{"timestamp":1714201909.4141362,"name":"offline","context":{"idset":"8317"}}
+{"timestamp":1714201909.5011709,"name":"offline","context":{"idset":"8388"}}
+{"timestamp":1714201909.612355,"name":"offline","context":{"idset":"8425"}}
+{"timestamp":1714201909.7116435,"name":"offline","context":{"idset":"8389"}}
+{"timestamp":1714201909.7156801,"name":"offline","context":{"idset":"8318"}}
+{"timestamp":1714201909.8159277,"name":"offline","context":{"idset":"8354"}}
+{"timestamp":1714201909.9185386,"name":"offline","context":{"idset":"8390"}}
+{"timestamp":1714201909.94045,"name":"offline","context":{"idset":"8426"}}
+{"timestamp":1714201910.0460508,"name":"offline","context":{"idset":"8391"}}
+{"timestamp":1714201910.1053126,"name":"offline","context":{"idset":"8319"}}
+{"timestamp":1714201910.2040172,"name":"offline","context":{"idset":"8355"}}
+{"timestamp":1714201910.3099802,"name":"offline","context":{"idset":"8320"}}
+{"timestamp":1714201910.3222923,"name":"offline","context":{"idset":"8427"}}
+{"timestamp":1714201910.3258512,"name":"offline","context":{"idset":"8428"}}
+{"timestamp":1714201910.4188843,"name":"offline","context":{"idset":"8392"}}
+{"timestamp":1714201910.5281217,"name":"offline","context":{"idset":"8393"}}
+{"timestamp":1714201910.5343444,"name":"offline","context":{"idset":"8356"}}
+{"timestamp":1714201910.5385842,"name":"offline","context":{"idset":"8321"}}
+{"timestamp":1714201910.6431732,"name":"offline","context":{"idset":"8322"}}
+{"timestamp":1714201910.7431881,"name":"offline","context":{"idset":"8357"}}
+{"timestamp":1714201910.7490766,"name":"offline","context":{"idset":"8358"}}
+{"timestamp":1714201910.8481462,"name":"offline","context":{"idset":"8429"}}
+{"timestamp":1714201910.8708131,"name":"offline","context":{"idset":"8395"}}
+{"timestamp":1714201910.8779325,"name":"offline","context":{"idset":"8359"}}
+{"timestamp":1714201910.8943706,"name":"offline","context":{"idset":"8323"}}
+{"timestamp":1714201910.8979268,"name":"offline","context":{"idset":"8430"}}
+{"timestamp":1714201910.9806798,"name":"offline","context":{"idset":"8431"}}
+{"timestamp":1714201911.0774059,"name":"offline","context":{"idset":"8325"}}
+{"timestamp":1714201911.0822396,"name":"offline","context":{"idset":"8432"}}
+{"timestamp":1714201911.0902181,"name":"offline","context":{"idset":"8396"}}
+{"timestamp":1714201911.0937347,"name":"offline","context":{"idset":"8324"}}
+{"timestamp":1714201911.1016455,"name":"offline","context":{"idset":"8360"}}
+{"timestamp":1714201911.1093955,"name":"offline","context":{"idset":"8394"}}
+{"timestamp":1714201911.1142957,"name":"offline","context":{"idset":"8361"}}
+{"timestamp":1714201911.2167153,"name":"offline","context":{"idset":"8397"}}
+{"timestamp":1714201911.3612759,"name":"offline","context":{"idset":"8326"}}
+{"timestamp":1714201911.448421,"name":"offline","context":{"idset":"8398"}}
+{"timestamp":1714201911.4576175,"name":"offline","context":{"idset":"8433"}}
+{"timestamp":1714201911.5565958,"name":"offline","context":{"idset":"8434"}}
+{"timestamp":1714201911.6540387,"name":"offline","context":{"idset":"8327"}}
+{"timestamp":1714201911.6747286,"name":"offline","context":{"idset":"8362"}}
+{"timestamp":1714201911.7687235,"name":"offline","context":{"idset":"8363"}}
+{"timestamp":1714201911.862345,"name":"offline","context":{"idset":"8399"}}
+{"timestamp":1714201911.8757808,"name":"offline","context":{"idset":"8435"}}
+{"timestamp":1714201911.9698889,"name":"offline","context":{"idset":"8328"}}
+{"timestamp":1714201912.1052091,"name":"offline","context":{"idset":"8400"}}
+{"timestamp":1714201912.2045648,"name":"offline","context":{"idset":"8364"}}
+{"timestamp":1714201912.3112404,"name":"offline","context":{"idset":"8329"}}
+{"timestamp":1714201912.3161745,"name":"offline","context":{"idset":"8436"}}
+{"timestamp":1714201912.4138319,"name":"offline","context":{"idset":"8365"}}
+{"timestamp":1714201912.549763,"name":"offline","context":{"idset":"8401"}}
+{"timestamp":1714201912.5574136,"name":"offline","context":{"idset":"8330"}}
+{"timestamp":1714201912.5626922,"name":"offline","context":{"idset":"8366"}}
+{"timestamp":1714201912.657542,"name":"offline","context":{"idset":"8565"}}
+{"timestamp":1714201912.7793067,"name":"offline","context":{"idset":"8402"}}
+{"timestamp":1714201912.7848377,"name":"offline","context":{"idset":"8331"}}
+{"timestamp":1714201912.8779378,"name":"offline","context":{"idset":"8566"}}
+{"timestamp":1714201912.979821,"name":"offline","context":{"idset":"8403"}}
+{"timestamp":1714201913.0811372,"name":"offline","context":{"idset":"8367"}}
+{"timestamp":1714201913.183538,"name":"offline","context":{"idset":"8567"}}
+{"timestamp":1714201913.2820137,"name":"offline","context":{"idset":"8332"}}
+{"timestamp":1714201913.3861203,"name":"offline","context":{"idset":"8368"}}
+{"timestamp":1714201913.5959013,"name":"offline","context":{"idset":"8404"}}
+{"timestamp":1714201913.9432127,"name":"offline","context":{"idset":"8568"}}
+{"timestamp":1714201914.077502,"name":"offline","context":{"idset":"8333"}}
+{"timestamp":1714201914.1765368,"name":"offline","context":{"idset":"8369"}}
+{"timestamp":1714201914.3950546,"name":"offline","context":{"idset":"8405"}}
+{"timestamp":1714201914.5112474,"name":"offline","context":{"idset":"8569"}}
+{"timestamp":1714202794.7356224,"name":"offline","context":{"idset":"9359"}}
+{"timestamp":1714205413.3904469,"name":"online","context":{"idset":"6525"}}
+{"timestamp":1714205413.4947901,"name":"online","context":{"idset":"6526"}}
+{"timestamp":1714205413.880898,"name":"online","context":{"idset":"6523-6524"}}
+{"timestamp":1714205414.1694603,"name":"online","context":{"idset":"6527,6532"}}
+{"timestamp":1714205414.5003796,"name":"online","context":{"idset":"6531,6536"}}
+{"timestamp":1714205414.8352811,"name":"online","context":{"idset":"6535"}}
+{"timestamp":1714205415.0129333,"name":"online","context":{"idset":"6539"}}
+{"timestamp":1714205415.1481457,"name":"online","context":{"idset":"6538"}}
+{"timestamp":1714205415.6730928,"name":"online","context":{"idset":"6528,6567,6574"}}
+{"timestamp":1714205415.7944,"name":"online","context":{"idset":"6551"}}
+{"timestamp":1714205415.9073184,"name":"online","context":{"idset":"6548"}}
+{"timestamp":1714205416.2016079,"name":"online","context":{"idset":"6550"}}
+{"timestamp":1714205416.3055153,"name":"online","context":{"idset":"6564"}}
+{"timestamp":1714205416.75049,"name":"online","context":{"idset":"6529"}}
+{"timestamp":1714205416.9478297,"name":"online","context":{"idset":"6573"}}
+{"timestamp":1714205417.2628047,"name":"online","context":{"idset":"6540"}}
+{"timestamp":1714205417.462975,"name":"online","context":{"idset":"6545,6556,6569"}}
+{"timestamp":1714205418.1652937,"name":"online","context":{"idset":"6557"}}
+{"timestamp":1714205418.378494,"name":"online","context":{"idset":"6575,6583"}}
+{"timestamp":1714205418.7990782,"name":"online","context":{"idset":"6534,6566"}}
+{"timestamp":1714205418.8980496,"name":"online","context":{"idset":"6533,6542"}}
+{"timestamp":1714205419.2061429,"name":"online","context":{"idset":"6570"}}
+{"timestamp":1714205419.3357158,"name":"online","context":{"idset":"6558"}}
+{"timestamp":1714205419.4608767,"name":"online","context":{"idset":"6595"}}
+{"timestamp":1714205419.8377326,"name":"online","context":{"idset":"6599"}}
+{"timestamp":1714205419.9884284,"name":"online","context":{"idset":"6580,6600"}}
+{"timestamp":1714205420.0964031,"name":"online","context":{"idset":"6597"}}
+{"timestamp":1714205420.6866128,"name":"online","context":{"idset":"6598"}}
+{"timestamp":1714205420.8853335,"name":"online","context":{"idset":"6560,6568,6577"}}
+{"timestamp":1714205421.3291831,"name":"online","context":{"idset":"6549,6572"}}
+{"timestamp":1714205421.4676583,"name":"online","context":{"idset":"6554"}}
+{"timestamp":1714205421.6309543,"name":"online","context":{"idset":"6559,6585"}}
+{"timestamp":1714205421.8543835,"name":"online","context":{"idset":"6602"}}
+{"timestamp":1714205421.985595,"name":"online","context":{"idset":"6584,6619"}}
+{"timestamp":1714205422.1226404,"name":"online","context":{"idset":"6553,6593"}}
+{"timestamp":1714205422.3090618,"name":"online","context":{"idset":"6541"}}
+{"timestamp":1714205422.5716794,"name":"online","context":{"idset":"6614"}}
+{"timestamp":1714205422.7250326,"name":"online","context":{"idset":"7289"}}
+{"timestamp":1714205423.2696447,"name":"online","context":{"idset":"6596"}}
+{"timestamp":1714205423.4154499,"name":"online","context":{"idset":"6641"}}
+{"timestamp":1714205423.7435851,"name":"online","context":{"idset":"6590,6611"}}
+{"timestamp":1714205423.9180598,"name":"online","context":{"idset":"6576,6622"}}
+{"timestamp":1714205424.2252433,"name":"online","context":{"idset":"6552,6627"}}
+{"timestamp":1714205424.3692284,"name":"online","context":{"idset":"6565,6579,6623"}}
+{"timestamp":1714205424.4916334,"name":"online","context":{"idset":"6546,6586"}}
+{"timestamp":1714205424.7697976,"name":"online","context":{"idset":"6571,6609,6615"}}
+{"timestamp":1714205424.8804655,"name":"online","context":{"idset":"7295"}}
+{"timestamp":1714205425.0017962,"name":"online","context":{"idset":"6603"}}
+{"timestamp":1714205425.1505957,"name":"online","context":{"idset":"6578,6581,6628,7302"}}
+{"timestamp":1714205425.2915537,"name":"online","context":{"idset":"6544,6587,6589,7287"}}
+{"timestamp":1714205425.4103947,"name":"online","context":{"idset":"6588,7315"}}
+{"timestamp":1714205425.5277901,"name":"online","context":{"idset":"6635"}}
+{"timestamp":1714205425.631505,"name":"online","context":{"idset":"7292"}}
+{"timestamp":1714205425.7291043,"name":"online","context":{"idset":"6543,7305,7331"}}
+{"timestamp":1714205425.8285928,"name":"online","context":{"idset":"6626,7300,7332"}}
+{"timestamp":1714205425.9280858,"name":"online","context":{"idset":"6601,7321"}}
+{"timestamp":1714205426.0245521,"name":"online","context":{"idset":"6604"}}
+{"timestamp":1714205426.0408895,"name":"online","context":{"idset":"6591"}}
+{"timestamp":1714205426.1486943,"name":"online","context":{"idset":"6612"}}
+{"timestamp":1714205426.3057852,"name":"online","context":{"idset":"6606"}}
+{"timestamp":1714205426.4839976,"name":"online","context":{"idset":"6618,6633,7290,7328"}}
+{"timestamp":1714205426.6209829,"name":"online","context":{"idset":"6537,6563,6643,7304"}}
+{"timestamp":1714205426.7382278,"name":"online","context":{"idset":"6607"}}
+{"timestamp":1714205426.8371944,"name":"online","context":{"idset":"6617"}}
+{"timestamp":1714205426.9394557,"name":"online","context":{"idset":"6582,7286,7314"}}
+{"timestamp":1714205427.056067,"name":"online","context":{"idset":"6630-6631,7339"}}
+{"timestamp":1714205427.1720595,"name":"online","context":{"idset":"7320,7358"}}
+{"timestamp":1714205427.2747931,"name":"online","context":{"idset":"6624,7324"}}
+{"timestamp":1714205427.3715091,"name":"online","context":{"idset":"6555,6562,6640,7288,7303,7319,7353"}}
+{"timestamp":1714205427.563168,"name":"online","context":{"idset":"6620,6637,7285,7299"}}
+{"timestamp":1714205427.6697426,"name":"online","context":{"idset":"7360"}}
+{"timestamp":1714205427.7878556,"name":"online","context":{"idset":"6621,6638,7293"}}
+{"timestamp":1714205427.901757,"name":"online","context":{"idset":"6625,6644,7298,7309"}}
+{"timestamp":1714205427.9482834,"name":"online","context":{"idset":"7336,7365"}}
+{"timestamp":1714205428.0983012,"name":"online","context":{"idset":"6605,6629,6634,7296,7330,7340,7362"}}
+{"timestamp":1714205428.1052334,"name":"online","context":{"idset":"7333,7384"}}
+{"timestamp":1714205428.1097798,"name":"online","context":{"idset":"6639,7397"}}
+{"timestamp":1714205428.2139163,"name":"online","context":{"idset":"6632"}}
+{"timestamp":1714205428.3286765,"name":"online","context":{"idset":"6594,6642,7409"}}
+{"timestamp":1714205428.439688,"name":"online","context":{"idset":"7301,7325,7347,7369,7410"}}
+{"timestamp":1714205428.6193237,"name":"online","context":{"idset":"6592,7307"}}
+{"timestamp":1714205428.6228271,"name":"online","context":{"idset":"7407"}}
+{"timestamp":1714205428.7268372,"name":"online","context":{"idset":"7335,7385"}}
+{"timestamp":1714205428.8581741,"name":"online","context":{"idset":"6530,6547,7294,7316,7322,7373,7382,7389"}}
+{"timestamp":1714205428.993252,"name":"online","context":{"idset":"7306,7352,7359"}}
+{"timestamp":1714205429.2286422,"name":"online","context":{"idset":"7341,7380,7386,7388,7391,7393,7395,8313"}}
+{"timestamp":1714205429.3371601,"name":"online","context":{"idset":"7400"}}
+{"timestamp":1714205429.4332948,"name":"online","context":{"idset":"6608,6610,7310,7374,7379,7387"}}
+{"timestamp":1714205429.5288382,"name":"online","context":{"idset":"6636,7297,7348"}}
+{"timestamp":1714205429.6725719,"name":"online","context":{"idset":"7367-7368"}}
+{"timestamp":1714205430.160979,"name":"online","context":{"idset":"6613"}}
+{"timestamp":1714205430.3018188,"name":"online","context":{"idset":"7311,7342,7363,7408,8309,8312"}}
+{"timestamp":1714205430.4566731,"name":"online","context":{"idset":"7313,7338,7349-7350,7355,7357"}}
+{"timestamp":1714205430.4615419,"name":"online","context":{"idset":"7354"}}
+{"timestamp":1714205430.6163719,"name":"online","context":{"idset":"6616,7323"}}
+{"timestamp":1714205430.8090239,"name":"online","context":{"idset":"7381"}}
+{"timestamp":1714205430.9138551,"name":"online","context":{"idset":"7361,7394,8311"}}
+{"timestamp":1714205431.0410035,"name":"online","context":{"idset":"7399,7406"}}
+{"timestamp":1714205431.1680059,"name":"online","context":{"idset":"6561,7329,7346,7351,7370,7375"}}
+{"timestamp":1714205431.3728848,"name":"online","context":{"idset":"7366,7396"}}
+{"timestamp":1714205431.6279187,"name":"online","context":{"idset":"7326,8642"}}
+{"timestamp":1714205431.6411431,"name":"online","context":{"idset":"7308,7371"}}
+{"timestamp":1714205431.8114738,"name":"online","context":{"idset":"7372,7390"}}
+{"timestamp":1714205431.9126821,"name":"online","context":{"idset":"7334,7401,8641,8677"}}
+{"timestamp":1714205432.0385089,"name":"online","context":{"idset":"7318,7345"}}
+{"timestamp":1714205432.2090659,"name":"online","context":{"idset":"7291,7312,7317,7327,7337,7343-7344,7364,7377,7383,7392,8644"}}
+{"timestamp":1714205432.3430548,"name":"online","context":{"idset":"7404,8679"}}
+{"timestamp":1714205432.3579533,"name":"online","context":{"idset":"8647"}}
+{"timestamp":1714205432.5045435,"name":"online","context":{"idset":"7376,7378,7398,7402,7411,8646,8680"}}
+{"timestamp":1714205432.6163945,"name":"online","context":{"idset":"8682"}}
+{"timestamp":1714205432.8055861,"name":"online","context":{"idset":"8681,8684"}}
+{"timestamp":1714205432.9098759,"name":"online","context":{"idset":"8645,8648"}}
+{"timestamp":1714205433.0253437,"name":"online","context":{"idset":"8683"}}
+{"timestamp":1714205433.1255932,"name":"online","context":{"idset":"7356,7405"}}
+{"timestamp":1714205433.3076069,"name":"online","context":{"idset":"8314,8649,8686"}}
+{"timestamp":1714205433.4407592,"name":"online","context":{"idset":"8650,8678"}}
+{"timestamp":1714205433.5417507,"name":"online","context":{"idset":"8643"}}
+{"timestamp":1714205433.6349511,"name":"online","context":{"idset":"8687"}}
+{"timestamp":1714205433.8477614,"name":"online","context":{"idset":"7403,7412"}}
+{"timestamp":1714205433.9725735,"name":"online","context":{"idset":"8688"}}
+{"timestamp":1714205434.117744,"name":"online","context":{"idset":"8652-8653"}}
+{"timestamp":1714205434.3376462,"name":"online","context":{"idset":"8654-8655,8685,8690"}}
+{"timestamp":1714205434.471092,"name":"online","context":{"idset":"8651,8689,8692"}}
+{"timestamp":1714205434.7195046,"name":"online","context":{"idset":"8656"}}
+{"timestamp":1714205434.8243098,"name":"online","context":{"idset":"8658,8660,8691"}}
+{"timestamp":1714205434.9304595,"name":"online","context":{"idset":"8310,8661,8663"}}
+{"timestamp":1714205435.0387714,"name":"online","context":{"idset":"8657"}}
+{"timestamp":1714205435.1523628,"name":"online","context":{"idset":"8659,8664"}}
+{"timestamp":1714205435.2789731,"name":"online","context":{"idset":"8662"}}
+{"timestamp":1714205435.3904312,"name":"online","context":{"idset":"8666"}}
+{"timestamp":1714205435.5687869,"name":"online","context":{"idset":"8605,8665,8673"}}
+{"timestamp":1714205435.6975369,"name":"online","context":{"idset":"8570,8606,8670-8671,8675"}}
+{"timestamp":1714205435.8418326,"name":"online","context":{"idset":"8571,8607"}}
+{"timestamp":1714205435.9585979,"name":"online","context":{"idset":"8569,8667"}}
+{"timestamp":1714205435.9745493,"name":"online","context":{"idset":"8574,8669"}}
+{"timestamp":1714205436.118798,"name":"online","context":{"idset":"8572,8608-8610,8668,8672,8674,8676"}}
+{"timestamp":1714205436.5510869,"name":"online","context":{"idset":"8573,8575,8577"}}
+{"timestamp":1714205436.8221612,"name":"online","context":{"idset":"8611"}}
+{"timestamp":1714205437.0903394,"name":"online","context":{"idset":"8578,8613-8614"}}
+{"timestamp":1714205437.3190272,"name":"online","context":{"idset":"8579,8612"}}
+{"timestamp":1714205437.4417343,"name":"online","context":{"idset":"8615"}}
+{"timestamp":1714205437.4566867,"name":"online","context":{"idset":"8616"}}
+{"timestamp":1714205437.6151009,"name":"online","context":{"idset":"8576,8583"}}
+{"timestamp":1714205437.9905734,"name":"online","context":{"idset":"8580,8582,8584,8617-8619"}}
+{"timestamp":1714205438.1643784,"name":"online","context":{"idset":"8581,8585,8620"}}
+{"timestamp":1714205438.414695,"name":"online","context":{"idset":"8587-8588"}}
+{"timestamp":1714205438.6583757,"name":"online","context":{"idset":"8586,8590,8621"}}
+{"timestamp":1714205438.7575345,"name":"online","context":{"idset":"8589"}}
+{"timestamp":1714205438.8667748,"name":"online","context":{"idset":"8591"}}
+{"timestamp":1714205439.1466973,"name":"online","context":{"idset":"8622-8625,8630"}}
+{"timestamp":1714205439.2487078,"name":"online","context":{"idset":"8627"}}
+{"timestamp":1714205439.3495543,"name":"online","context":{"idset":"8626,8631"}}
+{"timestamp":1714205439.642004,"name":"online","context":{"idset":"8593,8595,8598,8635"}}
+{"timestamp":1714205439.7443449,"name":"online","context":{"idset":"8632"}}
+{"timestamp":1714205439.8776212,"name":"online","context":{"idset":"8594,8628,8636"}}
+{"timestamp":1714205439.9940071,"name":"online","context":{"idset":"8599"}}
+{"timestamp":1714205440.1221621,"name":"online","context":{"idset":"8596"}}
+{"timestamp":1714205440.2376478,"name":"online","context":{"idset":"8592,8600"}}
+{"timestamp":1714205440.3699059,"name":"online","context":{"idset":"8634"}}
+{"timestamp":1714205440.5424809,"name":"online","context":{"idset":"8601"}}
+{"timestamp":1714205440.8266997,"name":"online","context":{"idset":"8597,8629,8633"}}
+{"timestamp":1714205441.0561781,"name":"online","context":{"idset":"8334,8603-8604,8637"}}
+{"timestamp":1714205441.1795185,"name":"online","context":{"idset":"8333"}}
+{"timestamp":1714205441.319593,"name":"online","context":{"idset":"8405,8640"}}
+{"timestamp":1714205441.4902334,"name":"online","context":{"idset":"8406"}}
+{"timestamp":1714205441.6583776,"name":"online","context":{"idset":"8369,8602,8639"}}
+{"timestamp":1714205441.7830462,"name":"online","context":{"idset":"8372"}}
+{"timestamp":1714205442.3427074,"name":"online","context":{"idset":"8335-8336,8371,8408,8410,8638"}}
+{"timestamp":1714205442.5745606,"name":"online","context":{"idset":"8337,8370,8374,8409"}}
+{"timestamp":1714205442.8646054,"name":"online","context":{"idset":"8338,8373,8407,8411"}}
+{"timestamp":1714205443.1547465,"name":"online","context":{"idset":"8340,8376"}}
+{"timestamp":1714205443.6588914,"name":"online","context":{"idset":"8412"}}
+{"timestamp":1714205443.8571799,"name":"online","context":{"idset":"8339,8341"}}
+{"timestamp":1714205443.9691293,"name":"online","context":{"idset":"8413"}}
+{"timestamp":1714205444.2010443,"name":"online","context":{"idset":"8342,8378,8414"}}
+{"timestamp":1714205444.4830976,"name":"online","context":{"idset":"8344,8375,8415"}}
+{"timestamp":1714205444.5977407,"name":"online","context":{"idset":"8377"}}
+{"timestamp":1714205445.4162679,"name":"online","context":{"idset":"8379"}}
+{"timestamp":1714205445.963587,"name":"online","context":{"idset":"8343,8345"}}
+{"timestamp":1714205446.4542916,"name":"online","context":{"idset":"8416"}}
+{"timestamp":1714205446.5905569,"name":"online","context":{"idset":"8380"}}
+{"timestamp":1714205446.8815384,"name":"online","context":{"idset":"8346,8348,8419"}}
+{"timestamp":1714205447.0524266,"name":"online","context":{"idset":"8382-8383,8417"}}
+{"timestamp":1714205447.2105157,"name":"online","context":{"idset":"8315,8381"}}
+{"timestamp":1714205447.3355923,"name":"online","context":{"idset":"8347,8385"}}
+{"timestamp":1714205447.4637427,"name":"online","context":{"idset":"8350,8420"}}
+{"timestamp":1714205447.6000488,"name":"online","context":{"idset":"8384,8421"}}
+{"timestamp":1714205447.6071341,"name":"online","context":{"idset":"8388"}}
+{"timestamp":1714205447.7186542,"name":"online","context":{"idset":"8349,8418"}}
+{"timestamp":1714205447.8483827,"name":"online","context":{"idset":"8386-8387,8422"}}
+{"timestamp":1714205448.069916,"name":"online","context":{"idset":"8316"}}
+{"timestamp":1714205448.2985799,"name":"online","context":{"idset":"8317,8320,8351-8352,8354,8390,8424-8425"}}
+{"timestamp":1714205448.5228581,"name":"online","context":{"idset":"8318,8353,8355,8389,8423,8426"}}
+{"timestamp":1714205448.8156147,"name":"online","context":{"idset":"8319,8321-8322,8357,8427"}}
+{"timestamp":1714205448.9253154,"name":"online","context":{"idset":"8358,8428,8430"}}
+{"timestamp":1714205449.1492889,"name":"online","context":{"idset":"8323-8324,8356,8392-8393,8396"}}
+{"timestamp":1714205449.3685095,"name":"online","context":{"idset":"8325,8328-8329,8359,8361-8362,8391,8433,8435"}}
+{"timestamp":1714205449.4777153,"name":"online","context":{"idset":"8434"}}
+{"timestamp":1714205449.5827205,"name":"online","context":{"idset":"8331,8432"}}
+{"timestamp":1714205449.7014236,"name":"online","context":{"idset":"8360,8367,8394,8397,8429,8431,8565-8568"}}
+{"timestamp":1714205449.8100116,"name":"online","context":{"idset":"8363,8395"}}
+{"timestamp":1714205449.9315574,"name":"online","context":{"idset":"8365,8403"}}
+{"timestamp":1714205450.0443823,"name":"online","context":{"idset":"8326,8332,8364,8366,8398-8401,8404,8436"}}
+{"timestamp":1714205450.1474621,"name":"online","context":{"idset":"8327,8368,8402"}}
+{"timestamp":1714205450.2652655,"name":"online","context":{"idset":"8330"}}
+{"timestamp":1714205484.748055,"name":"undrain","context":{"idset":"6523-6644,7285-7412,8309-8436,8565-8692"}}
+{"timestamp":1714213886.7347305,"name":"offline","context":{"idset":"9570"}}
+{"timestamp":1714223546.1006413,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1714223546.1980648,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1714223546.3274548,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1714223546.3384097,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1714223546.3448744,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1714223546.367811,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1714223546.3720267,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1714223546.3988724,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1714223546.403856,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1714223546.4986551,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1714223546.6779282,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1714223546.6833379,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1714223546.6890628,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1714223546.695653,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1714223546.7020593,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1714223546.7091975,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1714223546.7235305,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1714223546.7297492,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1714223546.7777169,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1714223546.7818711,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1714223546.7956765,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1714223546.8160071,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1714223546.8200254,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1714223546.8239288,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1714223546.8282301,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1714223546.8323433,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1714223546.9604025,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1714223547.1320412,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1714223547.1487205,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1714223547.1541016,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1714223547.2554708,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1714223547.2611699,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1714223547.2667072,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1714223547.2850747,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1714223547.2892792,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1714223547.2935238,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1714223547.2977679,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1714223547.3023119,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1714223547.3068342,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1714223547.3222256,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1714223547.3352792,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1714223547.3400283,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1714223547.3443892,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1714223547.3489721,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1714223547.3546154,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1714223547.3594618,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1714223547.3640974,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1714223547.3695683,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1714223547.3954158,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1714223547.4110222,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1714223547.4157031,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1714223547.4211457,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1714223547.424675,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1714223547.4280164,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1714223547.4331346,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1714223547.6151569,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1714223547.6199718,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1714223547.6352484,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1714223547.7209976,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1714223547.7256739,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1714223547.7362418,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1714223547.7405696,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1714223547.761626,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1714223547.8073528,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1714223547.8116882,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1714223547.8240268,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1714223547.8474247,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1714223547.8598795,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1714223547.864357,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1714223547.8809571,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1714223547.8853385,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1714223547.9260798,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1714223547.9512262,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1714223547.9555004,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1714223547.9672549,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1714223547.995199,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1714223547.9995501,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1714223548.0075109,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1714223548.0116959,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1714223548.0196755,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1714223548.0270662,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1714223548.0313964,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1714223548.0496695,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1714223548.072649,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1714223548.0809464,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1714223548.089963,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1714223548.0942636,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1714223548.1025524,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1714223548.1070383,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1714223548.115356,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1714223548.126632,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1714223548.1348488,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1714223548.1392384,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1714223548.1509664,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1714223548.1798577,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1714223548.1836646,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1714223548.1879423,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1714223548.2133381,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1714223548.2329395,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1714223548.2374053,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1714223548.2744215,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1714223548.279036,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1714223548.2835107,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1714223548.2879031,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1714223548.2922721,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1714223548.2967148,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1714223548.3147385,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1714223548.568608,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1714223548.9052258,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1714223548.9095173,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1714223548.9143536,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1714223548.91819,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1714223548.9220228,"name":"offline","context":{"idset":"11439"}}
+{"timestamp":1714223548.926003,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1714223548.9302363,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1714223548.9449892,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1714223548.9492369,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1714223548.9536347,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1714223548.9582028,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1714223548.962651,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1714223548.966857,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1714223548.9713273,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1714223548.9756799,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1714223548.9800358,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1714223548.9841473,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1714223548.9900453,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1714223548.9943268,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1714223549.0029428,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1714223598.4886992,"name":"online","context":{"idset":"11487"}}
+{"timestamp":1714223598.8379979,"name":"online","context":{"idset":"11447,11450"}}
+{"timestamp":1714223599.6032908,"name":"online","context":{"idset":"11384"}}
+{"timestamp":1714223599.9335299,"name":"online","context":{"idset":"11423"}}
+{"timestamp":1714223602.3483911,"name":"online","context":{"idset":"11408"}}
+{"timestamp":1714223602.6998239,"name":"online","context":{"idset":"11436"}}
+{"timestamp":1714223602.9272101,"name":"online","context":{"idset":"11402"}}
+{"timestamp":1714223603.1892953,"name":"online","context":{"idset":"11391,11437"}}
+{"timestamp":1714223603.6256864,"name":"online","context":{"idset":"11482"}}
+{"timestamp":1714223605.4257648,"name":"online","context":{"idset":"11407"}}
+{"timestamp":1714223605.6777227,"name":"online","context":{"idset":"11406,11467"}}
+{"timestamp":1714223606.0117488,"name":"online","context":{"idset":"11383,11389"}}
+{"timestamp":1714223606.3395622,"name":"online","context":{"idset":"11435"}}
+{"timestamp":1714223606.78619,"name":"online","context":{"idset":"11425"}}
+{"timestamp":1714223606.9028478,"name":"online","context":{"idset":"11392,11414"}}
+{"timestamp":1714223607.2079787,"name":"online","context":{"idset":"11416,11444"}}
+{"timestamp":1714223607.3820355,"name":"online","context":{"idset":"11403,11409,11468"}}
+{"timestamp":1714223607.5931511,"name":"online","context":{"idset":"11461"}}
+{"timestamp":1714223607.8009195,"name":"online","context":{"idset":"11420,11426"}}
+{"timestamp":1714223607.9343941,"name":"online","context":{"idset":"11419"}}
+{"timestamp":1714223608.0769994,"name":"online","context":{"idset":"11452,11465"}}
+{"timestamp":1714223608.4397104,"name":"online","context":{"idset":"11438,11508"}}
+{"timestamp":1714223608.574542,"name":"online","context":{"idset":"11405,11464"}}
+{"timestamp":1714223608.8036482,"name":"online","context":{"idset":"11422,11457"}}
+{"timestamp":1714223609.2437732,"name":"online","context":{"idset":"11507"}}
+{"timestamp":1714223609.4906986,"name":"online","context":{"idset":"11470"}}
+{"timestamp":1714223610.0018482,"name":"online","context":{"idset":"11497"}}
+{"timestamp":1714223610.1738458,"name":"online","context":{"idset":"11506"}}
+{"timestamp":1714223610.4424636,"name":"online","context":{"idset":"11455"}}
+{"timestamp":1714223610.7025533,"name":"online","context":{"idset":"11431"}}
+{"timestamp":1714223610.9208527,"name":"online","context":{"idset":"11382,11460"}}
+{"timestamp":1714223611.2101073,"name":"online","context":{"idset":"11381"}}
+{"timestamp":1714223611.5652254,"name":"online","context":{"idset":"11490"}}
+{"timestamp":1714223611.6902328,"name":"online","context":{"idset":"11399"}}
+{"timestamp":1714223612.1456921,"name":"online","context":{"idset":"11433"}}
+{"timestamp":1714223612.304141,"name":"online","context":{"idset":"11412"}}
+{"timestamp":1714223612.4437563,"name":"online","context":{"idset":"11396"}}
+{"timestamp":1714223613.149148,"name":"online","context":{"idset":"11386,11430"}}
+{"timestamp":1714223613.2876968,"name":"online","context":{"idset":"11453,11480"}}
+{"timestamp":1714223613.411,"name":"online","context":{"idset":"11394,11397"}}
+{"timestamp":1714223613.5961666,"name":"online","context":{"idset":"11462-11463"}}
+{"timestamp":1714223613.7120378,"name":"online","context":{"idset":"11395,11481"}}
+{"timestamp":1714223613.827455,"name":"online","context":{"idset":"11387"}}
+{"timestamp":1714223613.8527949,"name":"online","context":{"idset":"11411"}}
+{"timestamp":1714223614.204911,"name":"online","context":{"idset":"11401,11417-11418"}}
+{"timestamp":1714223614.338321,"name":"online","context":{"idset":"11398,11428,11498"}}
+{"timestamp":1714223614.5491824,"name":"online","context":{"idset":"11390,11400,11424,11439,11445,11459,11485"}}
+{"timestamp":1714223614.8488829,"name":"online","context":{"idset":"11385,11393,11404,11410,11415,11427,11434,11441-11442,11456,11458,11466,11483-11484,11489,11492,11501-11502,11504"}}
+{"timestamp":1714223614.9520552,"name":"online","context":{"idset":"11469"}}
+{"timestamp":1714223615.1649323,"name":"online","context":{"idset":"11388,11432,11448,11474,11495"}}
+{"timestamp":1714223615.4468129,"name":"online","context":{"idset":"11413,11440,11443,11446,11449,11454,11473,11477,11479,11491"}}
+{"timestamp":1714223615.6873057,"name":"online","context":{"idset":"11421,11429,11451,11478,11493,11496"}}
+{"timestamp":1714223615.7911885,"name":"online","context":{"idset":"11476,11499"}}
+{"timestamp":1714223615.9090476,"name":"online","context":{"idset":"11472"}}
+{"timestamp":1714223616.012758,"name":"online","context":{"idset":"11475,11486,11505"}}
+{"timestamp":1714223616.1155839,"name":"online","context":{"idset":"11471,11488,11494"}}
+{"timestamp":1714223616.2229102,"name":"online","context":{"idset":"11500,11503"}}
+{"timestamp":1714223760.950326,"name":"offline","context":{"idset":"10253"}}
+{"timestamp":1714223761.1467545,"name":"offline","context":{"idset":"10265"}}
+{"timestamp":1714223761.2482004,"name":"offline","context":{"idset":"10237"}}
+{"timestamp":1714223761.3006861,"name":"offline","context":{"idset":"10309"}}
+{"timestamp":1714223761.3051341,"name":"offline","context":{"idset":"10295"}}
+{"timestamp":1714223761.3100727,"name":"offline","context":{"idset":"10317"}}
+{"timestamp":1714223761.4031641,"name":"offline","context":{"idset":"10254"}}
+{"timestamp":1714223761.5373621,"name":"offline","context":{"idset":"10298"}}
+{"timestamp":1714223761.5427687,"name":"offline","context":{"idset":"10258"}}
+{"timestamp":1714223761.5467622,"name":"offline","context":{"idset":"10291"}}
+{"timestamp":1714223761.5514269,"name":"offline","context":{"idset":"10282"}}
+{"timestamp":1714223761.5556788,"name":"offline","context":{"idset":"10273"}}
+{"timestamp":1714223761.5620337,"name":"offline","context":{"idset":"10352"}}
+{"timestamp":1714223761.5664129,"name":"offline","context":{"idset":"10281"}}
+{"timestamp":1714223761.5704539,"name":"offline","context":{"idset":"10255"}}
+{"timestamp":1714223761.6136813,"name":"offline","context":{"idset":"10285"}}
+{"timestamp":1714223761.618154,"name":"offline","context":{"idset":"10264"}}
+{"timestamp":1714223761.6232734,"name":"offline","context":{"idset":"10232"}}
+{"timestamp":1714223761.6280212,"name":"offline","context":{"idset":"10236"}}
+{"timestamp":1714223761.6326568,"name":"offline","context":{"idset":"10251"}}
+{"timestamp":1714223761.6476281,"name":"offline","context":{"idset":"10272"}}
+{"timestamp":1714223761.6524098,"name":"offline","context":{"idset":"10302"}}
+{"timestamp":1714223761.6561587,"name":"offline","context":{"idset":"10306"}}
+{"timestamp":1714223761.7719615,"name":"offline","context":{"idset":"10328"}}
+{"timestamp":1714223761.922987,"name":"offline","context":{"idset":"10278"}}
+{"timestamp":1714223761.9284024,"name":"offline","context":{"idset":"10288"}}
+{"timestamp":1714223761.9350471,"name":"offline","context":{"idset":"10229"}}
+{"timestamp":1714223761.939743,"name":"offline","context":{"idset":"10274"}}
+{"timestamp":1714223761.9445198,"name":"offline","context":{"idset":"10234"}}
+{"timestamp":1714223761.9500403,"name":"offline","context":{"idset":"10331"}}
+{"timestamp":1714223761.9728956,"name":"offline","context":{"idset":"10247"}}
+{"timestamp":1714223762.0562007,"name":"offline","context":{"idset":"10267"}}
+{"timestamp":1714223762.20767,"name":"offline","context":{"idset":"10259"}}
+{"timestamp":1714223762.2122221,"name":"offline","context":{"idset":"10287"}}
+{"timestamp":1714223762.2177629,"name":"offline","context":{"idset":"10231"}}
+{"timestamp":1714223762.2225049,"name":"offline","context":{"idset":"10249"}}
+{"timestamp":1714223762.2339005,"name":"offline","context":{"idset":"10338"}}
+{"timestamp":1714223762.2386694,"name":"offline","context":{"idset":"10235"}}
+{"timestamp":1714223762.243902,"name":"offline","context":{"idset":"10297"}}
+{"timestamp":1714223762.2487624,"name":"offline","context":{"idset":"10355"}}
+{"timestamp":1714223762.2548614,"name":"offline","context":{"idset":"10286"}}
+{"timestamp":1714223762.3170905,"name":"offline","context":{"idset":"10315"}}
+{"timestamp":1714223762.3222508,"name":"offline","context":{"idset":"10243"}}
+{"timestamp":1714223762.3305466,"name":"offline","context":{"idset":"10233"}}
+{"timestamp":1714223762.337271,"name":"offline","context":{"idset":"10238"}}
+{"timestamp":1714223762.3432212,"name":"offline","context":{"idset":"10240"}}
+{"timestamp":1714223762.3615043,"name":"offline","context":{"idset":"10250"}}
+{"timestamp":1714223762.3678253,"name":"offline","context":{"idset":"10252"}}
+{"timestamp":1714223762.3726537,"name":"offline","context":{"idset":"10257"}}
+{"timestamp":1714223762.3775105,"name":"offline","context":{"idset":"10262"}}
+{"timestamp":1714223762.3844907,"name":"offline","context":{"idset":"10266"}}
+{"timestamp":1714223762.3900964,"name":"offline","context":{"idset":"10270"}}
+{"timestamp":1714223762.395891,"name":"offline","context":{"idset":"10318"}}
+{"timestamp":1714223762.4703774,"name":"offline","context":{"idset":"10339"}}
+{"timestamp":1714223762.6345,"name":"offline","context":{"idset":"10326"}}
+{"timestamp":1714223762.6540594,"name":"offline","context":{"idset":"10256"}}
+{"timestamp":1714223762.6594608,"name":"offline","context":{"idset":"10320"}}
+{"timestamp":1714223762.6786389,"name":"offline","context":{"idset":"10261"}}
+{"timestamp":1714223762.6932182,"name":"offline","context":{"idset":"10283"}}
+{"timestamp":1714223762.7107577,"name":"offline","context":{"idset":"10269"}}
+{"timestamp":1714223762.7284892,"name":"offline","context":{"idset":"10319"}}
+{"timestamp":1714223762.735214,"name":"offline","context":{"idset":"10293"}}
+{"timestamp":1714223762.7408121,"name":"offline","context":{"idset":"10322"}}
+{"timestamp":1714223762.7469683,"name":"offline","context":{"idset":"10349"}}
+{"timestamp":1714223762.752821,"name":"offline","context":{"idset":"10337"}}
+{"timestamp":1714223762.7589242,"name":"offline","context":{"idset":"10303"}}
+{"timestamp":1714223762.7732837,"name":"offline","context":{"idset":"10241"}}
+{"timestamp":1714223762.7869527,"name":"offline","context":{"idset":"10245"}}
+{"timestamp":1714223762.7972476,"name":"offline","context":{"idset":"10327"}}
+{"timestamp":1714223762.8535018,"name":"offline","context":{"idset":"10260"}}
+{"timestamp":1714223762.8583891,"name":"offline","context":{"idset":"10244"}}
+{"timestamp":1714223762.8634827,"name":"offline","context":{"idset":"10246"}}
+{"timestamp":1714223762.8732691,"name":"offline","context":{"idset":"10248"}}
+{"timestamp":1714223762.8891714,"name":"offline","context":{"idset":"10277"}}
+{"timestamp":1714223762.905149,"name":"offline","context":{"idset":"10284"}}
+{"timestamp":1714223762.9100029,"name":"offline","context":{"idset":"10290"}}
+{"timestamp":1714223762.9148862,"name":"offline","context":{"idset":"10304"}}
+{"timestamp":1714223762.9197948,"name":"offline","context":{"idset":"10312"}}
+{"timestamp":1714223762.9246771,"name":"offline","context":{"idset":"10313"}}
+{"timestamp":1714223762.929549,"name":"offline","context":{"idset":"10316"}}
+{"timestamp":1714223762.9344263,"name":"offline","context":{"idset":"10334"}}
+{"timestamp":1714223762.9392943,"name":"offline","context":{"idset":"10340"}}
+{"timestamp":1714223762.9442289,"name":"offline","context":{"idset":"10344"}}
+{"timestamp":1714223762.9490592,"name":"offline","context":{"idset":"10345"}}
+{"timestamp":1714223763.0679314,"name":"offline","context":{"idset":"10308"}}
+{"timestamp":1714223763.231091,"name":"offline","context":{"idset":"10301"}}
+{"timestamp":1714223763.2453544,"name":"offline","context":{"idset":"10348"}}
+{"timestamp":1714223763.2502298,"name":"offline","context":{"idset":"10332"}}
+{"timestamp":1714223763.2548764,"name":"offline","context":{"idset":"10346"}}
+{"timestamp":1714223763.2593775,"name":"offline","context":{"idset":"10354"}}
+{"timestamp":1714223763.2632678,"name":"offline","context":{"idset":"10351"}}
+{"timestamp":1714223763.3937767,"name":"offline","context":{"idset":"10314"}}
+{"timestamp":1714223763.3980725,"name":"offline","context":{"idset":"10279"}}
+{"timestamp":1714223763.4015417,"name":"offline","context":{"idset":"10230"}}
+{"timestamp":1714223763.4134502,"name":"offline","context":{"idset":"10239"}}
+{"timestamp":1714223763.4181137,"name":"offline","context":{"idset":"10242"}}
+{"timestamp":1714223763.422637,"name":"offline","context":{"idset":"10263"}}
+{"timestamp":1714223763.426934,"name":"offline","context":{"idset":"10271"}}
+{"timestamp":1714223763.4319293,"name":"offline","context":{"idset":"10275"}}
+{"timestamp":1714223763.4361908,"name":"offline","context":{"idset":"10276"}}
+{"timestamp":1714223763.4401081,"name":"offline","context":{"idset":"10280"}}
+{"timestamp":1714223763.4439411,"name":"offline","context":{"idset":"10289"}}
+{"timestamp":1714223763.4577334,"name":"offline","context":{"idset":"10292"}}
+{"timestamp":1714223763.4819651,"name":"offline","context":{"idset":"10294"}}
+{"timestamp":1714223763.4862604,"name":"offline","context":{"idset":"10296"}}
+{"timestamp":1714223763.490536,"name":"offline","context":{"idset":"10299"}}
+{"timestamp":1714223763.4945376,"name":"offline","context":{"idset":"10300"}}
+{"timestamp":1714223763.4983509,"name":"offline","context":{"idset":"10305"}}
+{"timestamp":1714223763.5019262,"name":"offline","context":{"idset":"10307"}}
+{"timestamp":1714223763.506001,"name":"offline","context":{"idset":"10310"}}
+{"timestamp":1714223763.5269225,"name":"offline","context":{"idset":"10311"}}
+{"timestamp":1714223763.5373907,"name":"offline","context":{"idset":"10321"}}
+{"timestamp":1714223763.5420899,"name":"offline","context":{"idset":"10323"}}
+{"timestamp":1714223763.5462677,"name":"offline","context":{"idset":"10324"}}
+{"timestamp":1714223763.5500958,"name":"offline","context":{"idset":"10325"}}
+{"timestamp":1714223763.5544684,"name":"offline","context":{"idset":"10329"}}
+{"timestamp":1714223763.5591471,"name":"offline","context":{"idset":"10330"}}
+{"timestamp":1714223763.5635583,"name":"offline","context":{"idset":"10333"}}
+{"timestamp":1714223763.5673821,"name":"offline","context":{"idset":"10335"}}
+{"timestamp":1714223763.5817914,"name":"offline","context":{"idset":"10336"}}
+{"timestamp":1714223763.6052969,"name":"offline","context":{"idset":"10341"}}
+{"timestamp":1714223763.6192358,"name":"offline","context":{"idset":"10342"}}
+{"timestamp":1714223763.6234655,"name":"offline","context":{"idset":"10343"}}
+{"timestamp":1714223763.6273162,"name":"offline","context":{"idset":"10347"}}
+{"timestamp":1714223763.6311979,"name":"offline","context":{"idset":"10350"}}
+{"timestamp":1714223763.6350031,"name":"offline","context":{"idset":"10353"}}
+{"timestamp":1714223763.6388535,"name":"offline","context":{"idset":"10356"}}
+{"timestamp":1714223763.6428552,"name":"offline","context":{"idset":"10268"}}
+{"timestamp":1714223801.2282929,"name":"online","context":{"idset":"10235-10236"}}
+{"timestamp":1714223801.4203959,"name":"online","context":{"idset":"10231,10243"}}
+{"timestamp":1714223801.7326396,"name":"online","context":{"idset":"10256"}}
+{"timestamp":1714223801.8925867,"name":"online","context":{"idset":"10238"}}
+{"timestamp":1714223802.145119,"name":"online","context":{"idset":"10244"}}
+{"timestamp":1714223802.3924897,"name":"online","context":{"idset":"10253"}}
+{"timestamp":1714223802.7832429,"name":"online","context":{"idset":"10262"}}
+{"timestamp":1714223803.0875349,"name":"online","context":{"idset":"10285"}}
+{"timestamp":1714223803.4168561,"name":"online","context":{"idset":"10230"}}
+{"timestamp":1714223803.7107997,"name":"online","context":{"idset":"10304"}}
+{"timestamp":1714223804.1288719,"name":"online","context":{"idset":"10241"}}
+{"timestamp":1714223804.2962856,"name":"online","context":{"idset":"10263"}}
+{"timestamp":1714223804.419373,"name":"online","context":{"idset":"10355"}}
+{"timestamp":1714223804.4474473,"name":"online","context":{"idset":"10239"}}
+{"timestamp":1714223804.6023469,"name":"online","context":{"idset":"10278"}}
+{"timestamp":1714223804.7780936,"name":"online","context":{"idset":"10233,10245,10260"}}
+{"timestamp":1714223804.9354019,"name":"online","context":{"idset":"10250"}}
+{"timestamp":1714223805.1711631,"name":"online","context":{"idset":"10229,10232,10268"}}
+{"timestamp":1714223805.3337388,"name":"online","context":{"idset":"10237,10251,10255"}}
+{"timestamp":1714223805.5544512,"name":"online","context":{"idset":"10234,10266,10280,10300,10307"}}
+{"timestamp":1714223805.7803025,"name":"online","context":{"idset":"10240,10242,10257-10259,10264,10270,10273,10275,10353"}}
+{"timestamp":1714223805.8952436,"name":"online","context":{"idset":"10254,10287,10298"}}
+{"timestamp":1714223806.0200324,"name":"online","context":{"idset":"10271,10301,10338,10341"}}
+{"timestamp":1714223806.1320524,"name":"online","context":{"idset":"10246,10249,10261,10265,10267,10272,10279,10291,10293-10294,10299,10302,10305,10328,10330,10351"}}
+{"timestamp":1714223806.2508523,"name":"online","context":{"idset":"10252,10277,10297,10321,10342,10348"}}
+{"timestamp":1714223806.3767452,"name":"online","context":{"idset":"10274"}}
+{"timestamp":1714223806.4869258,"name":"online","context":{"idset":"10247-10248,10282,10288,10290,10295-10296,10303,10306,10310,10313,10316,10318,10320,10325-10326,10334-10335,10337,10343,10347"}}
+{"timestamp":1714223806.6130447,"name":"online","context":{"idset":"10269,10309,10314,10317,10327"}}
+{"timestamp":1714223806.7191846,"name":"online","context":{"idset":"10281,10289,10292,10308,10322,10329,10336,10339-10340,10349"}}
+{"timestamp":1714223806.9544086,"name":"online","context":{"idset":"10276,10283-10284,10286,10312,10315,10319,10323-10324,10331-10332,10344-10346,10350,10352,10354,10356"}}
+{"timestamp":1714223807.24945,"name":"online","context":{"idset":"10333"}}
+{"timestamp":1714223807.526314,"name":"online","context":{"idset":"10311"}}
+{"timestamp":1714227517.830498,"name":"offline","context":{"idset":"9788"}}
+{"timestamp":1714229963.8449571,"name":"drain","context":{"idset":"11203-11252","reason":"prolog failed for jobid frwtqZ1jc9u","overwrite":0}}
+{"timestamp":1714231843.0284083,"name":"offline","context":{"idset":"10251"}}
+{"timestamp":1714237125.02882,"name":"offline","context":{"idset":"8918"}}
+{"timestamp":1714237964.9461002,"name":"offline","context":{"idset":"10763"}}
+{"timestamp":1714237964.949996,"name":"offline","context":{"idset":"10771"}}
+{"timestamp":1714237965.6269355,"name":"offline","context":{"idset":"10772"}}
+{"timestamp":1714238003.4023302,"name":"offline","context":{"idset":"10764"}}
+{"timestamp":1714238010.067651,"name":"drain","context":{"idset":"10764","reason":"epilog failed for jobid frxgPZbTWD5","overwrite":0}}
+{"timestamp":1714238837.698982,"name":"online","context":{"idset":"9180"}}
+{"timestamp":1714238938.6886353,"name":"drain","context":{"idset":"11256","reason":"prolog failed for jobid frrcZ3VzEWs","overwrite":0}}
+{"timestamp":1714238938.688782,"name":"drain","context":{"idset":"11255","reason":"prolog failed for jobid frrcYt1xpxw","overwrite":0}}
+{"timestamp":1714238938.688863,"name":"drain","context":{"idset":"11259","reason":"prolog failed for jobid frrcZYBQLhm","overwrite":0}}
+{"timestamp":1714238938.6889389,"name":"drain","context":{"idset":"11257","reason":"prolog failed for jobid frrcZDKHUjH","overwrite":0}}
+{"timestamp":1714238938.6890242,"name":"drain","context":{"idset":"11258","reason":"prolog failed for jobid frrcZP48m6f","overwrite":0}}
+{"timestamp":1714238938.6891186,"name":"drain","context":{"idset":"11253","reason":"prolog failed for jobid frrcYXbQioR","overwrite":0}}
+{"timestamp":1714238938.6892688,"name":"drain","context":{"idset":"11254","reason":"prolog failed for jobid frrcYiP3Viw","overwrite":0}}
+{"timestamp":1714239015.7452004,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1714239029.5837963,"name":"online","context":{"idset":"10215"}}
+{"timestamp":1714239114.434998,"name":"drain","context":{"idset":"11262","reason":"prolog failed for jobid frrca31iNaj","overwrite":0}}
+{"timestamp":1714239114.4508886,"name":"drain","context":{"idset":"11263","reason":"prolog failed for jobid frrcaCYhkhM","overwrite":0}}
+{"timestamp":1714239114.457051,"name":"drain","context":{"idset":"11260","reason":"prolog failed for jobid frrcZhqofE7","overwrite":0}}
+{"timestamp":1714239114.4624588,"name":"drain","context":{"idset":"11264","reason":"prolog failed for jobid frrcaPfcNHM","overwrite":0}}
+{"timestamp":1714239114.4859755,"name":"drain","context":{"idset":"11267","reason":"prolog failed for jobid frrcaz85gAB","overwrite":0}}
+{"timestamp":1714239114.6219184,"name":"drain","context":{"idset":"11266","reason":"prolog failed for jobid frrcakDaQrb","overwrite":0}}
+{"timestamp":1714239114.6499431,"name":"drain","context":{"idset":"11268","reason":"prolog failed for jobid frrcbE16xBR","overwrite":0}}
+{"timestamp":1714239114.6500342,"name":"drain","context":{"idset":"11265","reason":"prolog failed for jobid frrcaZgmWkX","overwrite":0}}
+{"timestamp":1714239114.6500938,"name":"drain","context":{"idset":"11269","reason":"prolog failed for jobid frrcbYzdkZd","overwrite":0}}
+{"timestamp":1714239115.7030606,"name":"drain","context":{"idset":"11261","reason":"prolog failed for jobid frrcZsWCykT","overwrite":0}}
+{"timestamp":1714239115.7031424,"name":"drain","context":{"idset":"11270","reason":"prolog failed for jobid frrcbiqQz4P","overwrite":0}}
+{"timestamp":1714239173.4352748,"name":"drain","context":{"idset":"11273","reason":"prolog failed for jobid frrccGzTQaB","overwrite":0}}
+{"timestamp":1714239173.4353714,"name":"drain","context":{"idset":"11271","reason":"prolog failed for jobid frrcbxPARR9","overwrite":0}}
+{"timestamp":1714239173.435431,"name":"drain","context":{"idset":"11272","reason":"prolog failed for jobid frrcc7WS12F","overwrite":0}}
+{"timestamp":1714239173.435499,"name":"drain","context":{"idset":"11274","reason":"prolog failed for jobid frrccT1cZ3M","overwrite":0}}
+{"timestamp":1714239173.4355643,"name":"drain","context":{"idset":"11275","reason":"prolog failed for jobid frrcccvqkP9","overwrite":0}}
+{"timestamp":1714239173.4356225,"name":"drain","context":{"idset":"11277","reason":"prolog failed for jobid frrccyNsqq1","overwrite":0}}
+{"timestamp":1714239173.4356775,"name":"drain","context":{"idset":"11276","reason":"prolog failed for jobid frrccoQgfvX","overwrite":0}}
+{"timestamp":1714239173.4357367,"name":"drain","context":{"idset":"11278","reason":"prolog failed for jobid frrcd9M51jV","overwrite":0}}
+{"timestamp":1714239173.4358096,"name":"drain","context":{"idset":"11282","reason":"prolog failed for jobid frrcdqXdYjZ","overwrite":0}}
+{"timestamp":1714239173.4358673,"name":"drain","context":{"idset":"11280","reason":"prolog failed for jobid frrcdWV8mnf","overwrite":0}}
+{"timestamp":1714239173.4359219,"name":"drain","context":{"idset":"11279","reason":"prolog failed for jobid frrcdLZuaSs","overwrite":0}}
+{"timestamp":1714239173.435976,"name":"drain","context":{"idset":"11281","reason":"prolog failed for jobid frrcdftiDVZ","overwrite":0}}
+{"timestamp":1714239173.6605954,"name":"drain","context":{"idset":"11283","reason":"prolog failed for jobid frrce9w4ddq","overwrite":0}}
+{"timestamp":1714239173.660702,"name":"drain","context":{"idset":"11284","reason":"prolog failed for jobid frrceKyhmPM","overwrite":0}}
+{"timestamp":1714239173.7817132,"name":"drain","context":{"idset":"11285","reason":"prolog failed for jobid frrcedHPPAo","overwrite":0}}
+{"timestamp":1714239173.7918327,"name":"drain","context":{"idset":"11298","reason":"prolog failed for jobid frrcgieWbC3","overwrite":0}}
+{"timestamp":1714239173.7920144,"name":"drain","context":{"idset":"11299","reason":"prolog failed for jobid frrcguqsAd5","overwrite":0}}
+{"timestamp":1714239173.7921977,"name":"drain","context":{"idset":"11295","reason":"prolog failed for jobid frrcgDqgYbR","overwrite":0}}
+{"timestamp":1714239173.7932732,"name":"drain","context":{"idset":"11294","reason":"prolog failed for jobid frrcg43sHfM","overwrite":0}}
+{"timestamp":1714239173.7935412,"name":"drain","context":{"idset":"11297","reason":"prolog failed for jobid frrcgYw9J71","overwrite":0}}
+{"timestamp":1714239173.7936423,"name":"drain","context":{"idset":"11291","reason":"prolog failed for jobid frrcfZJ1DdR","overwrite":0}}
+{"timestamp":1714239173.7937412,"name":"drain","context":{"idset":"11296","reason":"prolog failed for jobid frrcgPMBwRh","overwrite":0}}
+{"timestamp":1714239173.7938447,"name":"drain","context":{"idset":"11290","reason":"prolog failed for jobid frrcfRkesVh","overwrite":0}}
+{"timestamp":1714239173.7939415,"name":"drain","context":{"idset":"11289","reason":"prolog failed for jobid frrcfJyn9pK","overwrite":0}}
+{"timestamp":1714239173.7944622,"name":"drain","context":{"idset":"11292","reason":"prolog failed for jobid frrcfiUEmoD","overwrite":0}}
+{"timestamp":1714239173.7949402,"name":"drain","context":{"idset":"11286","reason":"prolog failed for jobid frrceozaCFH","overwrite":0}}
+{"timestamp":1714239173.7950206,"name":"drain","context":{"idset":"11293","reason":"prolog failed for jobid frrcfsxGBM9","overwrite":0}}
+{"timestamp":1714239173.795083,"name":"drain","context":{"idset":"11288","reason":"prolog failed for jobid frrcf9vxY4F","overwrite":0}}
+{"timestamp":1714239173.7952607,"name":"drain","context":{"idset":"11287","reason":"prolog failed for jobid frrceynPTBM","overwrite":0}}
+{"timestamp":1714239173.9967058,"name":"drain","context":{"idset":"11300","reason":"prolog failed for jobid frrch4zcjWX","overwrite":0}}
+{"timestamp":1714239174.0060365,"name":"drain","context":{"idset":"11301","reason":"prolog failed for jobid frrchEXc7d9","overwrite":0}}
+{"timestamp":1714239174.006114,"name":"drain","context":{"idset":"11303","reason":"prolog failed for jobid frrchZDM4dD","overwrite":0}}
+{"timestamp":1714239174.0178874,"name":"drain","context":{"idset":"11308","reason":"prolog failed for jobid frrciR2pvsH","overwrite":0}}
+{"timestamp":1714239174.2404029,"name":"drain","context":{"idset":"11307","reason":"prolog failed for jobid frrciF7bjXV","overwrite":0}}
+{"timestamp":1714239174.2491395,"name":"drain","context":{"idset":"11302","reason":"prolog failed for jobid frrchQ1dXB5","overwrite":0}}
+{"timestamp":1714239174.257828,"name":"drain","context":{"idset":"11305","reason":"prolog failed for jobid frrchtqwZ43","overwrite":0}}
+{"timestamp":1714239174.3383796,"name":"drain","context":{"idset":"11304","reason":"prolog failed for jobid frrchjPQ8nT","overwrite":0}}
+{"timestamp":1714239174.338455,"name":"drain","context":{"idset":"11306","reason":"prolog failed for jobid frrci4jgm7V","overwrite":0}}
+{"timestamp":1714239174.3830557,"name":"drain","context":{"idset":"11309","reason":"prolog failed for jobid frrcictjBdH","overwrite":0}}
+{"timestamp":1714239174.4107926,"name":"drain","context":{"idset":"11310","reason":"prolog failed for jobid frrcioM67tK","overwrite":0}}
+{"timestamp":1714239174.4109344,"name":"drain","context":{"idset":"11312","reason":"prolog failed for jobid frrcj9o8DLB","overwrite":0}}
+{"timestamp":1714239174.4252396,"name":"drain","context":{"idset":"11311","reason":"prolog failed for jobid frrciymy4s1","overwrite":0}}
+{"timestamp":1714239174.4253361,"name":"drain","context":{"idset":"11313","reason":"prolog failed for jobid frrcjKawUGF","overwrite":0}}
+{"timestamp":1714239174.4418845,"name":"drain","context":{"idset":"11314","reason":"prolog failed for jobid frrcjWDgKVh","overwrite":0}}
+{"timestamp":1714239174.4966867,"name":"drain","context":{"idset":"11315","reason":"prolog failed for jobid frrcjgAPW7q","overwrite":0}}
+{"timestamp":1714239174.496784,"name":"drain","context":{"idset":"11316","reason":"prolog failed for jobid frrcjpoVKNP","overwrite":0}}
+{"timestamp":1714239174.4968832,"name":"drain","context":{"idset":"11317","reason":"prolog failed for jobid frrcjzD4m5H","overwrite":0}}
+{"timestamp":1714239174.4969463,"name":"drain","context":{"idset":"11319","reason":"prolog failed for jobid frrckK9dato","overwrite":0}}
+{"timestamp":1714239174.4970028,"name":"drain","context":{"idset":"11320","reason":"prolog failed for jobid frrckVFEhD1","overwrite":0}}
+{"timestamp":1714239174.4970584,"name":"drain","context":{"idset":"11318","reason":"prolog failed for jobid frrck9TkH67","overwrite":0}}
+{"timestamp":1714239174.4976852,"name":"drain","context":{"idset":"11323","reason":"prolog failed for jobid frrcm1zjoDq","overwrite":0}}
+{"timestamp":1714239174.4978175,"name":"drain","context":{"idset":"11321","reason":"prolog failed for jobid frrckf61vhm","overwrite":0}}
+{"timestamp":1714239174.4979253,"name":"drain","context":{"idset":"11325","reason":"prolog failed for jobid frrcmQ4B7RR","overwrite":0}}
+{"timestamp":1714239174.4980438,"name":"drain","context":{"idset":"11324","reason":"prolog failed for jobid frrcmDog5R9","overwrite":0}}
+{"timestamp":1714239174.4981794,"name":"drain","context":{"idset":"11322","reason":"prolog failed for jobid frrckqD72JK","overwrite":0}}
+{"timestamp":1714239195.6953375,"name":"drain","context":{"idset":"11326","reason":"prolog failed for jobid frrcmZceUpP","overwrite":0}}
+{"timestamp":1714239195.6954398,"name":"drain","context":{"idset":"11327","reason":"prolog failed for jobid frrcmj3huod","overwrite":0}}
+{"timestamp":1714239195.7041168,"name":"drain","context":{"idset":"11328","reason":"prolog failed for jobid frrcmvaLKu9","overwrite":0}}
+{"timestamp":1714239195.8404458,"name":"drain","context":{"idset":"11332","reason":"prolog failed for jobid frrcnbCTciB","overwrite":0}}
+{"timestamp":1714239195.8427687,"name":"drain","context":{"idset":"11329","reason":"prolog failed for jobid frrcn6HhczB","overwrite":0}}
+{"timestamp":1714239195.8438826,"name":"drain","context":{"idset":"11331","reason":"prolog failed for jobid frrcnRmQBiw","overwrite":0}}
+{"timestamp":1714239195.8443196,"name":"drain","context":{"idset":"11336","reason":"prolog failed for jobid frrcoFo6vEs","overwrite":0}}
+{"timestamp":1714239195.8448801,"name":"drain","context":{"idset":"11334","reason":"prolog failed for jobid frrcnwBow5q","overwrite":0}}
+{"timestamp":1714239195.8457065,"name":"drain","context":{"idset":"11333","reason":"prolog failed for jobid frrcnm4iqVH","overwrite":0}}
+{"timestamp":1714239195.8457949,"name":"drain","context":{"idset":"11335","reason":"prolog failed for jobid frrco7ET4qM","overwrite":0}}
+{"timestamp":1714239195.8459642,"name":"drain","context":{"idset":"11330","reason":"prolog failed for jobid frrcnG2YuMZ","overwrite":0}}
+{"timestamp":1714239606.2770417,"name":"drain","context":{"idset":"11337","reason":"prolog failed for jobid frrcoS27xxo","overwrite":0}}
+{"timestamp":1714239648.3926253,"name":"drain","context":{"idset":"11339","reason":"prolog failed for jobid frrconxKpmD","overwrite":0}}
+{"timestamp":1714239648.3927362,"name":"drain","context":{"idset":"11338","reason":"prolog failed for jobid frrcodExXgB","overwrite":0}}
+{"timestamp":1714239648.5510943,"name":"drain","context":{"idset":"11346","reason":"prolog failed for jobid frrcpz86KLK","overwrite":0}}
+{"timestamp":1714239648.5512102,"name":"drain","context":{"idset":"11347","reason":"prolog failed for jobid frrcq9oyd91","overwrite":0}}
+{"timestamp":1714239648.5512805,"name":"drain","context":{"idset":"11342","reason":"prolog failed for jobid frrcpJabUNs","overwrite":0}}
+{"timestamp":1714239648.5517306,"name":"drain","context":{"idset":"11343","reason":"prolog failed for jobid frrcpTzAv5m","overwrite":0}}
+{"timestamp":1714239648.5525005,"name":"drain","context":{"idset":"11345","reason":"prolog failed for jobid frrcpovZFtj","overwrite":0}}
+{"timestamp":1714239648.553477,"name":"drain","context":{"idset":"11340","reason":"prolog failed for jobid frrcoxmd4yd","overwrite":0}}
+{"timestamp":1714239648.5546107,"name":"drain","context":{"idset":"11344","reason":"prolog failed for jobid frrcpdj2CT9","overwrite":0}}
+{"timestamp":1714239648.5551858,"name":"drain","context":{"idset":"11341","reason":"prolog failed for jobid frrcp7zpbi7","overwrite":0}}
+{"timestamp":1714240552.57442,"name":"undrain","context":{"idset":"10203"}}
+{"timestamp":1714241089.6105599,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1714241089.6146271,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1714241089.618603,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1714241089.6225867,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1714241089.6265562,"name":"offline","context":{"idset":"10206"}}
+{"timestamp":1714241089.6305914,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1714241089.6346304,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1714241120.8604779,"name":"online","context":{"idset":"10208,10211,10215"}}
+{"timestamp":1714241121.6056123,"name":"online","context":{"idset":"10206,10209"}}
+{"timestamp":1714241121.6094718,"name":"online","context":{"idset":"10205,10210"}}
+{"timestamp":1714241193.8007507,"name":"undrain","context":{"idset":"10205-10206,10208-10211,10215"}}
+{"timestamp":1714241556.450825,"name":"drain","context":{"idset":"11348","reason":"prolog failed for jobid frrcqKVrvwh","overwrite":0}}
+{"timestamp":1714241585.6283174,"name":"drain","context":{"idset":"11350","reason":"prolog failed for jobid frrcqeMynvB","overwrite":0}}
+{"timestamp":1714241585.6284139,"name":"drain","context":{"idset":"11349","reason":"prolog failed for jobid frrcqUvvMvw","overwrite":0}}
+{"timestamp":1714241585.7955005,"name":"drain","context":{"idset":"11355","reason":"prolog failed for jobid frrcrU1SiCw","overwrite":0}}
+{"timestamp":1714241585.7959638,"name":"drain","context":{"idset":"11351","reason":"prolog failed for jobid frrcqoZhLNK","overwrite":0}}
+{"timestamp":1714241585.796308,"name":"drain","context":{"idset":"11352","reason":"prolog failed for jobid frrcqxzkmMZ","overwrite":0}}
+{"timestamp":1714241585.8117399,"name":"drain","context":{"idset":"11354","reason":"prolog failed for jobid frrcrJ1mZ27","overwrite":0}}
+{"timestamp":1714241585.811836,"name":"drain","context":{"idset":"11356","reason":"prolog failed for jobid frrcreidXHR","overwrite":0}}
+{"timestamp":1714241585.811909,"name":"drain","context":{"idset":"11353","reason":"prolog failed for jobid frrcr8QLD4T","overwrite":0}}
+{"timestamp":1714241585.8119783,"name":"drain","context":{"idset":"11357","reason":"prolog failed for jobid frrcrqdgEcf","overwrite":0}}
+{"timestamp":1714241585.8120384,"name":"drain","context":{"idset":"11358","reason":"prolog failed for jobid frrcs2rWoL3","overwrite":0}}
+{"timestamp":1714241910.4917011,"name":"drain","context":{"idset":"11361","reason":"prolog failed for jobid frrcsa7feym","overwrite":0}}
+{"timestamp":1714241910.4917724,"name":"drain","context":{"idset":"11359","reason":"prolog failed for jobid frrcsEJhFaX","overwrite":0}}
+{"timestamp":1714241910.4918468,"name":"drain","context":{"idset":"11360","reason":"prolog failed for jobid frrcsPtecFq","overwrite":0}}
+{"timestamp":1714241910.4918988,"name":"drain","context":{"idset":"11363","reason":"prolog failed for jobid frrcsuG6P43","overwrite":0}}
+{"timestamp":1714241910.4919491,"name":"drain","context":{"idset":"11362","reason":"prolog failed for jobid frrcsjdB3p3","overwrite":0}}
+{"timestamp":1714241910.4919951,"name":"drain","context":{"idset":"11364","reason":"prolog failed for jobid frrct6QUyvP","overwrite":0}}
+{"timestamp":1714241910.4920309,"name":"drain","context":{"idset":"11365","reason":"prolog failed for jobid frrctGXa5Ww","overwrite":0}}
+{"timestamp":1714241910.4920709,"name":"drain","context":{"idset":"11368","reason":"prolog failed for jobid frrctoniwAf","overwrite":0}}
+{"timestamp":1714241910.4921074,"name":"drain","context":{"idset":"11369","reason":"prolog failed for jobid frrctywJ23Z","overwrite":0}}
+{"timestamp":1714241910.4921482,"name":"drain","context":{"idset":"11366","reason":"prolog failed for jobid frrctShd9gB","overwrite":0}}
+{"timestamp":1714241910.4921832,"name":"drain","context":{"idset":"11367","reason":"prolog failed for jobid frrctdZhtSj","overwrite":0}}
+{"timestamp":1714241910.492224,"name":"drain","context":{"idset":"11370","reason":"prolog failed for jobid frrcu9RKRbV","overwrite":0}}
+{"timestamp":1714241910.4922616,"name":"drain","context":{"idset":"11371","reason":"prolog failed for jobid frrcuKBehFD","overwrite":0}}
+{"timestamp":1714241910.4922986,"name":"drain","context":{"idset":"11374","reason":"prolog failed for jobid frrcuuvRsDq","overwrite":0}}
+{"timestamp":1714241910.4923358,"name":"drain","context":{"idset":"11372","reason":"prolog failed for jobid frrcuW5DRJ7","overwrite":0}}
+{"timestamp":1714241910.4923704,"name":"drain","context":{"idset":"11373","reason":"prolog failed for jobid frrcugwJA4f","overwrite":0}}
+{"timestamp":1714241910.4924107,"name":"drain","context":{"idset":"11376","reason":"prolog failed for jobid frrcvQ1kFvo","overwrite":0}}
+{"timestamp":1714241910.4924464,"name":"drain","context":{"idset":"11377","reason":"prolog failed for jobid frrcvaxGxYP","overwrite":0}}
+{"timestamp":1714241910.4924841,"name":"drain","context":{"idset":"11375","reason":"prolog failed for jobid frrcvBr49nf","overwrite":0}}
+{"timestamp":1714241910.4925239,"name":"drain","context":{"idset":"11380","reason":"prolog failed for jobid frrcw9HhJ2b","overwrite":0}}
+{"timestamp":1714241910.4925582,"name":"drain","context":{"idset":"11379","reason":"prolog failed for jobid frrcvxbzUET","overwrite":0}}
+{"timestamp":1714241910.4925971,"name":"drain","context":{"idset":"11378","reason":"prolog failed for jobid frrcvnB7XFm","overwrite":0}}
+{"timestamp":1714242194.8567252,"name":"drain","context":{"idset":"11381-11508","reason":"prolog failed for jobid frw93rnDZ8T","overwrite":0}}
+{"timestamp":1714242194.9689837,"name":"drain","context":{"idset":"7285-7412","reason":"prolog failed for jobid frthWrnCPiw","overwrite":0}}
+{"timestamp":1714242683.6359122,"name":"online","context":{"idset":"10764"}}
+{"timestamp":1714242928.4057744,"name":"online","context":{"idset":"10763"}}
+{"timestamp":1714243012.4064224,"name":"online","context":{"idset":"10771"}}
+{"timestamp":1714243467.7688601,"name":"undrain","context":{"idset":"10186-10187,10764,11203-11252"}}
+{"timestamp":1714243514.1348147,"name":"online","context":{"idset":"10772"}}
+{"timestamp":1714243811.2179995,"name":"online","context":{"idset":"10152"}}
+{"timestamp":1714243873.6594124,"name":"online","context":{"idset":"10154"}}
+{"timestamp":1714244321.8004856,"name":"drain","context":{"idset":"11203-11252","overwrite":0}}
+{"timestamp":1714248519.7682514,"name":"offline","context":{"idset":"7677"}}
+{"timestamp":1714248519.7762945,"name":"offline","context":{"idset":"7670"}}
+{"timestamp":1714248519.7832134,"name":"offline","context":{"idset":"7674"}}
+{"timestamp":1714248519.7872918,"name":"offline","context":{"idset":"7669"}}
+{"timestamp":1714248519.7919343,"name":"offline","context":{"idset":"7676"}}
+{"timestamp":1714248519.7962797,"name":"offline","context":{"idset":"7681"}}
+{"timestamp":1714248519.8004332,"name":"offline","context":{"idset":"7680"}}
+{"timestamp":1714248519.8045497,"name":"offline","context":{"idset":"7678"}}
+{"timestamp":1714248519.8086553,"name":"offline","context":{"idset":"7673"}}
+{"timestamp":1714248519.8128173,"name":"offline","context":{"idset":"7671"}}
+{"timestamp":1714248519.8170145,"name":"offline","context":{"idset":"7682"}}
+{"timestamp":1714248519.8212278,"name":"offline","context":{"idset":"7683"}}
+{"timestamp":1714248519.8252902,"name":"offline","context":{"idset":"7675"}}
+{"timestamp":1714248519.8295193,"name":"offline","context":{"idset":"7684"}}
+{"timestamp":1714248519.833601,"name":"offline","context":{"idset":"7679"}}
+{"timestamp":1714248519.8376014,"name":"offline","context":{"idset":"7672"}}
+{"timestamp":1714248669.7336354,"name":"online","context":{"idset":"9359"}}
+{"timestamp":1714248892.6307316,"name":"undrain","context":{"idset":"7285-7412"}}
+{"timestamp":1714248897.1672311,"name":"drain","context":{"idset":"7285-7412","reason":"prolog failed for jobid fryULXVuCMD","overwrite":0}}
+{"timestamp":1714248990.395134,"name":"undrain","context":{"idset":"7285-7412"}}
+{"timestamp":1714248994.680172,"name":"drain","context":{"idset":"7285-7412","reason":"prolog failed for jobid frzMuSLMNAK","overwrite":0}}
+{"timestamp":1714249122.6882393,"name":"offline","context":{"idset":"7686"}}
+{"timestamp":1714249122.6977499,"name":"offline","context":{"idset":"7691"}}
+{"timestamp":1714249122.7057908,"name":"offline","context":{"idset":"7689"}}
+{"timestamp":1714249122.7359934,"name":"offline","context":{"idset":"7696"}}
+{"timestamp":1714249122.8499315,"name":"offline","context":{"idset":"7690"}}
+{"timestamp":1714249124.6453257,"name":"offline","context":{"idset":"7688"}}
+{"timestamp":1714249124.6495924,"name":"offline","context":{"idset":"7692"}}
+{"timestamp":1714249124.6534348,"name":"offline","context":{"idset":"7693"}}
+{"timestamp":1714249124.6576588,"name":"offline","context":{"idset":"7694"}}
+{"timestamp":1714249124.6614325,"name":"offline","context":{"idset":"7697"}}
+{"timestamp":1714249124.6649621,"name":"offline","context":{"idset":"7699"}}
+{"timestamp":1714249124.669081,"name":"offline","context":{"idset":"7685"}}
+{"timestamp":1714249124.6731775,"name":"offline","context":{"idset":"7698"}}
+{"timestamp":1714249124.6780331,"name":"offline","context":{"idset":"7700"}}
+{"timestamp":1714249124.6827786,"name":"offline","context":{"idset":"7695"}}
+{"timestamp":1714249124.6878834,"name":"offline","context":{"idset":"7687"}}
+{"timestamp":1714249188.1083403,"name":"offline","context":{"idset":"11252"}}
+{"timestamp":1714249225.7718701,"name":"offline","context":{"idset":"7287"}}
+{"timestamp":1714249225.7759156,"name":"offline","context":{"idset":"7302"}}
+{"timestamp":1714249225.7799351,"name":"offline","context":{"idset":"7286"}}
+{"timestamp":1714249225.7839262,"name":"offline","context":{"idset":"7295"}}
+{"timestamp":1714249225.7879176,"name":"offline","context":{"idset":"7320"}}
+{"timestamp":1714249225.7919679,"name":"offline","context":{"idset":"7314"}}
+{"timestamp":1714249225.7959263,"name":"offline","context":{"idset":"7294"}}
+{"timestamp":1714249225.7999256,"name":"offline","context":{"idset":"7291"}}
+{"timestamp":1714249225.8038797,"name":"offline","context":{"idset":"7333"}}
+{"timestamp":1714249225.8078072,"name":"offline","context":{"idset":"7298"}}
+{"timestamp":1714249225.8118281,"name":"offline","context":{"idset":"7303"}}
+{"timestamp":1714249225.8160512,"name":"offline","context":{"idset":"7317"}}
+{"timestamp":1714249225.8202114,"name":"offline","context":{"idset":"7289"}}
+{"timestamp":1714249225.8244312,"name":"offline","context":{"idset":"7292"}}
+{"timestamp":1714249225.8286972,"name":"offline","context":{"idset":"7321"}}
+{"timestamp":1714249225.8328428,"name":"offline","context":{"idset":"7340"}}
+{"timestamp":1714249225.8370407,"name":"offline","context":{"idset":"7296"}}
+{"timestamp":1714249225.8412409,"name":"offline","context":{"idset":"7315"}}
+{"timestamp":1714249225.8452861,"name":"offline","context":{"idset":"7325"}}
+{"timestamp":1714249225.8493683,"name":"offline","context":{"idset":"7322"}}
+{"timestamp":1714249225.8534877,"name":"offline","context":{"idset":"7337"}}
+{"timestamp":1714249225.8575504,"name":"offline","context":{"idset":"7358"}}
+{"timestamp":1714249225.8616669,"name":"offline","context":{"idset":"7305"}}
+{"timestamp":1714249225.8657479,"name":"offline","context":{"idset":"7297"}}
+{"timestamp":1714249225.8698211,"name":"offline","context":{"idset":"7336"}}
+{"timestamp":1714249225.8739429,"name":"offline","context":{"idset":"7293"}}
+{"timestamp":1714249225.8779993,"name":"offline","context":{"idset":"7300"}}
+{"timestamp":1714249225.8820078,"name":"offline","context":{"idset":"7330"}}
+{"timestamp":1714249225.8860266,"name":"offline","context":{"idset":"7362"}}
+{"timestamp":1714249225.8900921,"name":"offline","context":{"idset":"7304"}}
+{"timestamp":1714249225.8944037,"name":"offline","context":{"idset":"7309"}}
+{"timestamp":1714249225.8986459,"name":"offline","context":{"idset":"7310"}}
+{"timestamp":1714249225.9026797,"name":"offline","context":{"idset":"7347"}}
+{"timestamp":1714249225.906709,"name":"offline","context":{"idset":"7327"}}
+{"timestamp":1714249225.9108355,"name":"offline","context":{"idset":"7382"}}
+{"timestamp":1714249225.914896,"name":"offline","context":{"idset":"7342"}}
+{"timestamp":1714249225.9192383,"name":"offline","context":{"idset":"7344"}}
+{"timestamp":1714249225.9232786,"name":"offline","context":{"idset":"7393"}}
+{"timestamp":1714249225.9272614,"name":"offline","context":{"idset":"7324"}}
+{"timestamp":1714249225.931293,"name":"offline","context":{"idset":"7365"}}
+{"timestamp":1714249225.9353132,"name":"offline","context":{"idset":"7383"}}
+{"timestamp":1714249225.9392965,"name":"offline","context":{"idset":"7323"}}
+{"timestamp":1714249225.943332,"name":"offline","context":{"idset":"7307"}}
+{"timestamp":1714249225.9474435,"name":"offline","context":{"idset":"7332"}}
+{"timestamp":1714249225.9514594,"name":"offline","context":{"idset":"7326"}}
+{"timestamp":1714249225.9555218,"name":"offline","context":{"idset":"7308"}}
+{"timestamp":1714249225.9595554,"name":"offline","context":{"idset":"7345"}}
+{"timestamp":1714249225.9634831,"name":"offline","context":{"idset":"7335"}}
+{"timestamp":1714249225.9674959,"name":"offline","context":{"idset":"7319"}}
+{"timestamp":1714249225.9715469,"name":"offline","context":{"idset":"7386"}}
+{"timestamp":1714249225.9754994,"name":"offline","context":{"idset":"7397"}}
+{"timestamp":1714249225.9796896,"name":"offline","context":{"idset":"7329"}}
+{"timestamp":1714249225.9837217,"name":"offline","context":{"idset":"7341"}}
+{"timestamp":1714249225.9877155,"name":"offline","context":{"idset":"7378"}}
+{"timestamp":1714249225.9917545,"name":"offline","context":{"idset":"7411"}}
+{"timestamp":1714249225.9959123,"name":"offline","context":{"idset":"7379"}}
+{"timestamp":1714249226.0000231,"name":"offline","context":{"idset":"7408"}}
+{"timestamp":1714249226.0040929,"name":"offline","context":{"idset":"7369"}}
+{"timestamp":1714249226.0080976,"name":"offline","context":{"idset":"7349"}}
+{"timestamp":1714249226.0120659,"name":"offline","context":{"idset":"7389"}}
+{"timestamp":1714249226.016022,"name":"offline","context":{"idset":"7343"}}
+{"timestamp":1714249226.0201175,"name":"offline","context":{"idset":"7313"}}
+{"timestamp":1714249226.024158,"name":"offline","context":{"idset":"7311"}}
+{"timestamp":1714249226.0283158,"name":"offline","context":{"idset":"7318"}}
+{"timestamp":1714249226.0322926,"name":"offline","context":{"idset":"7288"}}
+{"timestamp":1714249226.0363026,"name":"offline","context":{"idset":"7381"}}
+{"timestamp":1714249226.0403383,"name":"offline","context":{"idset":"7350"}}
+{"timestamp":1714249226.0444069,"name":"offline","context":{"idset":"7388"}}
+{"timestamp":1714249226.048507,"name":"offline","context":{"idset":"7384"}}
+{"timestamp":1714249226.0525398,"name":"offline","context":{"idset":"7363"}}
+{"timestamp":1714249226.0565705,"name":"offline","context":{"idset":"7395"}}
+{"timestamp":1714249226.0605688,"name":"offline","context":{"idset":"7405"}}
+{"timestamp":1714249226.0645649,"name":"offline","context":{"idset":"7334"}}
+{"timestamp":1714249226.0685568,"name":"offline","context":{"idset":"7346"}}
+{"timestamp":1714249226.0729382,"name":"offline","context":{"idset":"7290"}}
+{"timestamp":1714249226.2854199,"name":"offline","context":{"idset":"7412"}}
+{"timestamp":1714249226.781723,"name":"offline","context":{"idset":"7299"}}
+{"timestamp":1714249226.7854381,"name":"offline","context":{"idset":"7356"}}
+{"timestamp":1714249226.7890372,"name":"offline","context":{"idset":"7402"}}
+{"timestamp":1714249226.7923744,"name":"offline","context":{"idset":"7306"}}
+{"timestamp":1714249226.7956526,"name":"offline","context":{"idset":"7312"}}
+{"timestamp":1714249226.7988863,"name":"offline","context":{"idset":"7316"}}
+{"timestamp":1714249226.8021822,"name":"offline","context":{"idset":"7328"}}
+{"timestamp":1714249226.8055377,"name":"offline","context":{"idset":"7331"}}
+{"timestamp":1714249226.8088686,"name":"offline","context":{"idset":"7339"}}
+{"timestamp":1714249226.8194807,"name":"offline","context":{"idset":"7285"}}
+{"timestamp":1714249226.822736,"name":"offline","context":{"idset":"7301"}}
+{"timestamp":1714249226.825999,"name":"offline","context":{"idset":"7338"}}
+{"timestamp":1714249226.8292396,"name":"offline","context":{"idset":"7348"}}
+{"timestamp":1714249226.8324378,"name":"offline","context":{"idset":"7351"}}
+{"timestamp":1714249226.8356194,"name":"offline","context":{"idset":"7352"}}
+{"timestamp":1714249226.8388457,"name":"offline","context":{"idset":"7353"}}
+{"timestamp":1714249226.8420103,"name":"offline","context":{"idset":"7354"}}
+{"timestamp":1714249226.8452349,"name":"offline","context":{"idset":"7355"}}
+{"timestamp":1714249226.8485086,"name":"offline","context":{"idset":"7357"}}
+{"timestamp":1714249226.851635,"name":"offline","context":{"idset":"7359"}}
+{"timestamp":1714249226.8547966,"name":"offline","context":{"idset":"7360"}}
+{"timestamp":1714249226.857935,"name":"offline","context":{"idset":"7361"}}
+{"timestamp":1714249226.8610556,"name":"offline","context":{"idset":"7364"}}
+{"timestamp":1714249226.8643103,"name":"offline","context":{"idset":"7366"}}
+{"timestamp":1714249226.8676636,"name":"offline","context":{"idset":"7367"}}
+{"timestamp":1714249226.8708301,"name":"offline","context":{"idset":"7368"}}
+{"timestamp":1714249226.8740857,"name":"offline","context":{"idset":"7370"}}
+{"timestamp":1714249226.87731,"name":"offline","context":{"idset":"7371"}}
+{"timestamp":1714249226.8805432,"name":"offline","context":{"idset":"7372"}}
+{"timestamp":1714249226.884079,"name":"offline","context":{"idset":"7373"}}
+{"timestamp":1714249226.8872397,"name":"offline","context":{"idset":"7374"}}
+{"timestamp":1714249226.8905451,"name":"offline","context":{"idset":"7375"}}
+{"timestamp":1714249226.8936057,"name":"offline","context":{"idset":"7376"}}
+{"timestamp":1714249226.8966258,"name":"offline","context":{"idset":"7377"}}
+{"timestamp":1714249226.8996277,"name":"offline","context":{"idset":"7380"}}
+{"timestamp":1714249226.902637,"name":"offline","context":{"idset":"7385"}}
+{"timestamp":1714249226.9055917,"name":"offline","context":{"idset":"7387"}}
+{"timestamp":1714249226.908783,"name":"offline","context":{"idset":"7390"}}
+{"timestamp":1714249226.9118927,"name":"offline","context":{"idset":"7391"}}
+{"timestamp":1714249226.9150155,"name":"offline","context":{"idset":"7392"}}
+{"timestamp":1714249226.9181328,"name":"offline","context":{"idset":"7394"}}
+{"timestamp":1714249226.921335,"name":"offline","context":{"idset":"7396"}}
+{"timestamp":1714249226.9244328,"name":"offline","context":{"idset":"7398"}}
+{"timestamp":1714249226.9275103,"name":"offline","context":{"idset":"7399"}}
+{"timestamp":1714249226.9306536,"name":"offline","context":{"idset":"7400"}}
+{"timestamp":1714249226.9335885,"name":"offline","context":{"idset":"7401"}}
+{"timestamp":1714249226.9364681,"name":"offline","context":{"idset":"7403"}}
+{"timestamp":1714249226.9393923,"name":"offline","context":{"idset":"7404"}}
+{"timestamp":1714249226.9422665,"name":"offline","context":{"idset":"7406"}}
+{"timestamp":1714249226.9452868,"name":"offline","context":{"idset":"7407"}}
+{"timestamp":1714249226.9483278,"name":"offline","context":{"idset":"7409"}}
+{"timestamp":1714249226.9512258,"name":"offline","context":{"idset":"7410"}}
+{"timestamp":1714249242.8905404,"name":"drain","context":{"idset":"11509-11540,11542-11547,11549-11553,11557-11559,11567,11570-11617","reason":"prolog failed for jobid frzRkVsE1Rh","overwrite":0}}
+{"timestamp":1714249726.336442,"name":"offline","context":{"idset":"7702"}}
+{"timestamp":1714249726.352628,"name":"offline","context":{"idset":"7704"}}
+{"timestamp":1714249726.4515774,"name":"offline","context":{"idset":"7712"}}
+{"timestamp":1714249726.4752998,"name":"offline","context":{"idset":"7713"}}
+{"timestamp":1714249726.4792023,"name":"offline","context":{"idset":"7714"}}
+{"timestamp":1714249726.4876943,"name":"offline","context":{"idset":"7701"}}
+{"timestamp":1714249726.4904828,"name":"offline","context":{"idset":"7705"}}
+{"timestamp":1714249726.5088322,"name":"offline","context":{"idset":"7707"}}
+{"timestamp":1714249726.5116758,"name":"offline","context":{"idset":"7715"}}
+{"timestamp":1714249726.5925903,"name":"offline","context":{"idset":"7706"}}
+{"timestamp":1714249726.7063076,"name":"offline","context":{"idset":"7703"}}
+{"timestamp":1714249726.7104645,"name":"offline","context":{"idset":"7708"}}
+{"timestamp":1714249726.7146628,"name":"offline","context":{"idset":"7716"}}
+{"timestamp":1714249726.719065,"name":"offline","context":{"idset":"7710"}}
+{"timestamp":1714249726.7232409,"name":"offline","context":{"idset":"7711"}}
+{"timestamp":1714249726.823518,"name":"offline","context":{"idset":"7709"}}
+{"timestamp":1714249738.2018914,"name":"offline","context":{"idset":"11203"}}
+{"timestamp":1714249738.2065473,"name":"offline","context":{"idset":"11204"}}
+{"timestamp":1714249738.211144,"name":"offline","context":{"idset":"11205"}}
+{"timestamp":1714249738.2781994,"name":"offline","context":{"idset":"11206"}}
+{"timestamp":1714249740.5751584,"name":"offline","context":{"idset":"11207"}}
+{"timestamp":1714249740.5789576,"name":"offline","context":{"idset":"11208"}}
+{"timestamp":1714249740.5826726,"name":"offline","context":{"idset":"11209"}}
+{"timestamp":1714249740.5864043,"name":"offline","context":{"idset":"11210"}}
+{"timestamp":1714249740.6581728,"name":"offline","context":{"idset":"11211"}}
+{"timestamp":1714249743.081285,"name":"offline","context":{"idset":"11212"}}
+{"timestamp":1714249743.0852604,"name":"offline","context":{"idset":"11213"}}
+{"timestamp":1714249743.0894887,"name":"offline","context":{"idset":"11214"}}
+{"timestamp":1714249743.8241663,"name":"offline","context":{"idset":"11215"}}
+{"timestamp":1714249743.8789656,"name":"offline","context":{"idset":"11216"}}
+{"timestamp":1714249743.8827894,"name":"offline","context":{"idset":"11217"}}
+{"timestamp":1714249743.8867948,"name":"offline","context":{"idset":"11218"}}
+{"timestamp":1714249743.9468281,"name":"offline","context":{"idset":"11219"}}
+{"timestamp":1714249746.2876179,"name":"offline","context":{"idset":"11220"}}
+{"timestamp":1714249746.2918599,"name":"offline","context":{"idset":"11221"}}
+{"timestamp":1714249746.2958341,"name":"offline","context":{"idset":"11222"}}
+{"timestamp":1714249746.3821597,"name":"offline","context":{"idset":"11223"}}
+{"timestamp":1714249748.8271477,"name":"offline","context":{"idset":"11224"}}
+{"timestamp":1714249748.8326395,"name":"offline","context":{"idset":"11225"}}
+{"timestamp":1714249748.888221,"name":"offline","context":{"idset":"11226"}}
+{"timestamp":1714249748.9805341,"name":"offline","context":{"idset":"11227"}}
+{"timestamp":1714249749.1294949,"name":"offline","context":{"idset":"11228"}}
+{"timestamp":1714249751.1199994,"name":"offline","context":{"idset":"11229"}}
+{"timestamp":1714249751.1273141,"name":"offline","context":{"idset":"11230"}}
+{"timestamp":1714249751.1344826,"name":"offline","context":{"idset":"11231"}}
+{"timestamp":1714249751.1419318,"name":"offline","context":{"idset":"11232"}}
+{"timestamp":1714249751.1547449,"name":"offline","context":{"idset":"11233"}}
+{"timestamp":1714249753.0790832,"name":"offline","context":{"idset":"11234"}}
+{"timestamp":1714249753.0830767,"name":"offline","context":{"idset":"11235"}}
+{"timestamp":1714249753.0871766,"name":"offline","context":{"idset":"11236"}}
+{"timestamp":1714249753.0910869,"name":"offline","context":{"idset":"11237"}}
+{"timestamp":1714249753.0949633,"name":"offline","context":{"idset":"11238"}}
+{"timestamp":1714249753.1811643,"name":"offline","context":{"idset":"11239"}}
+{"timestamp":1714249755.0667613,"name":"offline","context":{"idset":"11240"}}
+{"timestamp":1714249755.0706451,"name":"offline","context":{"idset":"11241"}}
+{"timestamp":1714249755.8132026,"name":"offline","context":{"idset":"11242"}}
+{"timestamp":1714249757.0748966,"name":"offline","context":{"idset":"11243"}}
+{"timestamp":1714249757.0790582,"name":"offline","context":{"idset":"11244"}}
+{"timestamp":1714249757.0830805,"name":"offline","context":{"idset":"11245"}}
+{"timestamp":1714249757.9186234,"name":"offline","context":{"idset":"11246"}}
+{"timestamp":1714249757.9736161,"name":"offline","context":{"idset":"11247"}}
+{"timestamp":1714249757.9957886,"name":"offline","context":{"idset":"11248"}}
+{"timestamp":1714249758.0024872,"name":"offline","context":{"idset":"11249"}}
+{"timestamp":1714249758.0448232,"name":"offline","context":{"idset":"11250"}}
+{"timestamp":1714249761.154031,"name":"offline","context":{"idset":"11251"}}
+{"timestamp":1714249895.7072165,"name":"offline","context":{"idset":"9068"}}
+{"timestamp":1714249927.8265522,"name":"drain","context":{"idset":"8923-8963,8973-8981,8983-8992,8995-8998,9001-9002,9005-9017,9019-9027","reason":"prolog failed for jobid frzWyRvCWVD","overwrite":0}}
+{"timestamp":1714249934.0329626,"name":"drain","context":{"idset":"8282-8290,8292-8303,8305-8308,8821-8832,8834-8917,8919-8922","reason":"prolog failed for jobid frzX17LBKTM","overwrite":0}}
+{"timestamp":1714249942.5476251,"name":"drain","context":{"idset":"8159,8278-8281","reason":"prolog failed for jobid frzX3xjjC2j","overwrite":0}}
+{"timestamp":1714249948.6830943,"name":"drain","context":{"idset":"8107-8108","reason":"prolog failed for jobid frzX7r2t3t3","overwrite":0}}
+{"timestamp":1714250131.8134122,"name":"offline","context":{"idset":"7730"}}
+{"timestamp":1714250331.8117373,"name":"offline","context":{"idset":"7721"}}
+{"timestamp":1714250332.0180128,"name":"offline","context":{"idset":"7728"}}
+{"timestamp":1714250332.2425544,"name":"offline","context":{"idset":"7729"}}
+{"timestamp":1714250332.5478692,"name":"offline","context":{"idset":"7732"}}
+{"timestamp":1714250332.8052607,"name":"offline","context":{"idset":"7717"}}
+{"timestamp":1714250333.0097849,"name":"offline","context":{"idset":"7725"}}
+{"timestamp":1714250333.1097746,"name":"offline","context":{"idset":"7720"}}
+{"timestamp":1714250333.8014379,"name":"offline","context":{"idset":"7724"}}
+{"timestamp":1714250333.805208,"name":"offline","context":{"idset":"7727"}}
+{"timestamp":1714250333.8089652,"name":"offline","context":{"idset":"7723"}}
+{"timestamp":1714250334.0404546,"name":"offline","context":{"idset":"7726"}}
+{"timestamp":1714250334.2992175,"name":"offline","context":{"idset":"7722"}}
+{"timestamp":1714250334.9337032,"name":"offline","context":{"idset":"7719"}}
+{"timestamp":1714250335.8171725,"name":"offline","context":{"idset":"7718"}}
+{"timestamp":1714250336.9367037,"name":"offline","context":{"idset":"7731"}}
+{"timestamp":1714250942.3117902,"name":"offline","context":{"idset":"7733"}}
+{"timestamp":1714250942.8191566,"name":"offline","context":{"idset":"7737"}}
+{"timestamp":1714250943.1146324,"name":"offline","context":{"idset":"7742"}}
+{"timestamp":1714250943.8594878,"name":"offline","context":{"idset":"7745"}}
+{"timestamp":1714250943.8644216,"name":"offline","context":{"idset":"7736"}}
+{"timestamp":1714250943.8682163,"name":"offline","context":{"idset":"7740"}}
+{"timestamp":1714250943.8720431,"name":"offline","context":{"idset":"7744"}}
+{"timestamp":1714250943.8758755,"name":"offline","context":{"idset":"7738"}}
+{"timestamp":1714250943.8797131,"name":"offline","context":{"idset":"7747"}}
+{"timestamp":1714250943.8835101,"name":"offline","context":{"idset":"7735"}}
+{"timestamp":1714250943.8873749,"name":"offline","context":{"idset":"7741"}}
+{"timestamp":1714250943.9850438,"name":"offline","context":{"idset":"7748"}}
+{"timestamp":1714250944.1938741,"name":"offline","context":{"idset":"7746"}}
+{"timestamp":1714250944.6467314,"name":"offline","context":{"idset":"7743"}}
+{"timestamp":1714250944.7551713,"name":"offline","context":{"idset":"7734"}}
+{"timestamp":1714251326.3924704,"name":"drain","context":{"idset":"8052","reason":"epilog failed for jobid frzgdohmbaX","overwrite":0}}
+{"timestamp":1714251327.0913823,"name":"offline","context":{"idset":"8052"}}
+{"timestamp":1714251419.1482146,"name":"offline","context":{"idset":"8051"}}
+{"timestamp":1714251420.1654294,"name":"drain","context":{"idset":"8051","reason":"epilog failed for jobid frzhLpLyJj9","overwrite":0}}
+{"timestamp":1714251552.9334922,"name":"offline","context":{"idset":"7756"}}
+{"timestamp":1714251553.8247781,"name":"offline","context":{"idset":"7758"}}
+{"timestamp":1714251554.9893818,"name":"offline","context":{"idset":"7761"}}
+{"timestamp":1714251555.840631,"name":"offline","context":{"idset":"7755"}}
+{"timestamp":1714251555.8444595,"name":"offline","context":{"idset":"7759"}}
+{"timestamp":1714251555.8482473,"name":"offline","context":{"idset":"7751"}}
+{"timestamp":1714251555.9271774,"name":"offline","context":{"idset":"7749"}}
+{"timestamp":1714251556.0479658,"name":"offline","context":{"idset":"7753"}}
+{"timestamp":1714251556.051671,"name":"offline","context":{"idset":"7763"}}
+{"timestamp":1714251556.1478481,"name":"offline","context":{"idset":"7757"}}
+{"timestamp":1714251556.3378654,"name":"offline","context":{"idset":"7754"}}
+{"timestamp":1714251556.6468749,"name":"offline","context":{"idset":"7752"}}
+{"timestamp":1714251556.9142745,"name":"offline","context":{"idset":"7764"}}
+{"timestamp":1714251557.9079766,"name":"offline","context":{"idset":"7762"}}
+{"timestamp":1714251558.0401387,"name":"offline","context":{"idset":"7760"}}
+{"timestamp":1714251560.4209983,"name":"offline","context":{"idset":"7750"}}
+{"timestamp":1714252158.2632973,"name":"offline","context":{"idset":"7770"}}
+{"timestamp":1714252158.2719669,"name":"offline","context":{"idset":"7771"}}
+{"timestamp":1714252158.2913535,"name":"offline","context":{"idset":"7778"}}
+{"timestamp":1714252158.3889439,"name":"offline","context":{"idset":"7779"}}
+{"timestamp":1714252158.4933724,"name":"offline","context":{"idset":"7773"}}
+{"timestamp":1714252158.4987738,"name":"offline","context":{"idset":"7769"}}
+{"timestamp":1714252158.5043576,"name":"offline","context":{"idset":"7775"}}
+{"timestamp":1714252158.5088415,"name":"offline","context":{"idset":"7776"}}
+{"timestamp":1714252158.5141442,"name":"offline","context":{"idset":"7774"}}
+{"timestamp":1714252158.5182941,"name":"offline","context":{"idset":"7780"}}
+{"timestamp":1714252158.6191833,"name":"offline","context":{"idset":"7772"}}
+{"timestamp":1714252158.8108995,"name":"offline","context":{"idset":"7777"}}
+{"timestamp":1714252161.9618275,"name":"offline","context":{"idset":"7768"}}
+{"timestamp":1714252161.966567,"name":"offline","context":{"idset":"7765"}}
+{"timestamp":1714252162.2831573,"name":"offline","context":{"idset":"7767"}}
+{"timestamp":1714252163.1026025,"name":"offline","context":{"idset":"7766"}}
+{"timestamp":1714252440.9719732,"name":"online","context":{"idset":"8052"}}
+{"timestamp":1714252442.1338663,"name":"online","context":{"idset":"8051"}}
+{"timestamp":1714252493.0627038,"name":"drain","context":{"idset":"8050","reason":"epilog failed for jobid frzqUHZJQQf","overwrite":0}}
+{"timestamp":1714252493.1490338,"name":"offline","context":{"idset":"8050"}}
+{"timestamp":1714252764.3917441,"name":"offline","context":{"idset":"7792"}}
+{"timestamp":1714252764.4083817,"name":"offline","context":{"idset":"7781"}}
+{"timestamp":1714252764.423903,"name":"offline","context":{"idset":"7795"}}
+{"timestamp":1714252764.4292483,"name":"offline","context":{"idset":"7788"}}
+{"timestamp":1714252764.5282567,"name":"offline","context":{"idset":"7790"}}
+{"timestamp":1714252764.6467345,"name":"offline","context":{"idset":"7784"}}
+{"timestamp":1714252764.6544905,"name":"offline","context":{"idset":"7794"}}
+{"timestamp":1714252764.662451,"name":"offline","context":{"idset":"7783"}}
+{"timestamp":1714252764.6724815,"name":"offline","context":{"idset":"7793"}}
+{"timestamp":1714252764.679491,"name":"offline","context":{"idset":"7786"}}
+{"timestamp":1714252764.7094011,"name":"offline","context":{"idset":"7787"}}
+{"timestamp":1714252764.7160289,"name":"offline","context":{"idset":"7782"}}
+{"timestamp":1714252764.7325571,"name":"offline","context":{"idset":"7785"}}
+{"timestamp":1714252764.7362647,"name":"offline","context":{"idset":"7789"}}
+{"timestamp":1714252764.739928,"name":"offline","context":{"idset":"7791"}}
+{"timestamp":1714252764.7834508,"name":"offline","context":{"idset":"7796"}}
+{"timestamp":1714253795.0657105,"name":"offline","context":{"idset":"7925"}}
+{"timestamp":1714253795.0700974,"name":"offline","context":{"idset":"7926"}}
+{"timestamp":1714253795.1468191,"name":"offline","context":{"idset":"7927"}}
+{"timestamp":1714253797.0726073,"name":"offline","context":{"idset":"7928"}}
+{"timestamp":1714253797.0768275,"name":"offline","context":{"idset":"7929"}}
+{"timestamp":1714253797.0811369,"name":"offline","context":{"idset":"7930"}}
+{"timestamp":1714253797.0852609,"name":"offline","context":{"idset":"7931"}}
+{"timestamp":1714253797.8611767,"name":"offline","context":{"idset":"7932"}}
+{"timestamp":1714253797.9425633,"name":"offline","context":{"idset":"7933"}}
+{"timestamp":1714253797.9462459,"name":"offline","context":{"idset":"7934"}}
+{"timestamp":1714253797.9499249,"name":"offline","context":{"idset":"7935"}}
+{"timestamp":1714253797.9535618,"name":"offline","context":{"idset":"7936"}}
+{"timestamp":1714253797.9572153,"name":"offline","context":{"idset":"7937"}}
+{"timestamp":1714253797.9930029,"name":"offline","context":{"idset":"7939"}}
+{"timestamp":1714253801.0732961,"name":"offline","context":{"idset":"7938"}}
+{"timestamp":1714253801.0771632,"name":"offline","context":{"idset":"7940"}}
+{"timestamp":1714253801.083195,"name":"offline","context":{"idset":"7941"}}
+{"timestamp":1714253801.1568806,"name":"offline","context":{"idset":"7942"}}
+{"timestamp":1714253803.0829058,"name":"offline","context":{"idset":"7943"}}
+{"timestamp":1714253803.0866511,"name":"offline","context":{"idset":"7944"}}
+{"timestamp":1714253803.0903139,"name":"offline","context":{"idset":"7945"}}
+{"timestamp":1714253803.0939691,"name":"offline","context":{"idset":"7946"}}
+{"timestamp":1714253803.0983901,"name":"offline","context":{"idset":"7947"}}
+{"timestamp":1714253803.1800389,"name":"offline","context":{"idset":"7948"}}
+{"timestamp":1714253805.0685432,"name":"offline","context":{"idset":"7949"}}
+{"timestamp":1714253805.0726948,"name":"offline","context":{"idset":"7950"}}
+{"timestamp":1714253805.930058,"name":"offline","context":{"idset":"7951"}}
+{"timestamp":1714253805.9809613,"name":"offline","context":{"idset":"7952"}}
+{"timestamp":1714253806.0531166,"name":"offline","context":{"idset":"7953"}}
+{"timestamp":1714253807.2489507,"name":"offline","context":{"idset":"7954"}}
+{"timestamp":1714253807.2614086,"name":"offline","context":{"idset":"7955"}}
+{"timestamp":1714253809.0687788,"name":"offline","context":{"idset":"7956"}}
+{"timestamp":1714253809.072619,"name":"offline","context":{"idset":"7957"}}
+{"timestamp":1714253809.0765891,"name":"offline","context":{"idset":"7958"}}
+{"timestamp":1714253809.8224058,"name":"offline","context":{"idset":"7959"}}
+{"timestamp":1714253809.9466002,"name":"offline","context":{"idset":"7960"}}
+{"timestamp":1714253809.999321,"name":"offline","context":{"idset":"7961"}}
+{"timestamp":1714253810.0692692,"name":"offline","context":{"idset":"7962"}}
+{"timestamp":1714253813.0639348,"name":"offline","context":{"idset":"7963"}}
+{"timestamp":1714253813.0679736,"name":"offline","context":{"idset":"7964"}}
+{"timestamp":1714253813.1644716,"name":"offline","context":{"idset":"7965"}}
+{"timestamp":1714253815.0815516,"name":"offline","context":{"idset":"7966"}}
+{"timestamp":1714253815.0866272,"name":"offline","context":{"idset":"7967"}}
+{"timestamp":1714253815.150856,"name":"offline","context":{"idset":"7968"}}
+{"timestamp":1714253817.1027672,"name":"offline","context":{"idset":"7969"}}
+{"timestamp":1714253817.1098371,"name":"offline","context":{"idset":"7970"}}
+{"timestamp":1714253817.1170428,"name":"offline","context":{"idset":"7971"}}
+{"timestamp":1714253817.170795,"name":"offline","context":{"idset":"7972"}}
+{"timestamp":1714253819.0718901,"name":"offline","context":{"idset":"7973"}}
+{"timestamp":1714253819.0761082,"name":"offline","context":{"idset":"7974"}}
+{"timestamp":1714253819.0801854,"name":"offline","context":{"idset":"7975"}}
+{"timestamp":1714253819.0883007,"name":"offline","context":{"idset":"7976"}}
+{"timestamp":1714253820.0008268,"name":"offline","context":{"idset":"7977"}}
+{"timestamp":1714253820.0566108,"name":"offline","context":{"idset":"7978"}}
+{"timestamp":1714253820.0610778,"name":"offline","context":{"idset":"7979"}}
+{"timestamp":1714253820.1480119,"name":"offline","context":{"idset":"7980"}}
+{"timestamp":1714253823.0794103,"name":"offline","context":{"idset":"7981"}}
+{"timestamp":1714253823.0839043,"name":"offline","context":{"idset":"7982"}}
+{"timestamp":1714253823.1021452,"name":"offline","context":{"idset":"7983"}}
+{"timestamp":1714253823.1065001,"name":"offline","context":{"idset":"7984"}}
+{"timestamp":1714253823.1713607,"name":"offline","context":{"idset":"7985"}}
+{"timestamp":1714253825.0729268,"name":"offline","context":{"idset":"7986"}}
+{"timestamp":1714253825.0782247,"name":"offline","context":{"idset":"7987"}}
+{"timestamp":1714253825.0821123,"name":"offline","context":{"idset":"7988"}}
+{"timestamp":1714253825.9173834,"name":"offline","context":{"idset":"7989"}}
+{"timestamp":1714253825.9612031,"name":"offline","context":{"idset":"7990"}}
+{"timestamp":1714253825.9695148,"name":"offline","context":{"idset":"7991"}}
+{"timestamp":1714253825.9719515,"name":"offline","context":{"idset":"7992"}}
+{"timestamp":1714253826.0382948,"name":"offline","context":{"idset":"7993"}}
+{"timestamp":1714253829.0695152,"name":"offline","context":{"idset":"7994"}}
+{"timestamp":1714253829.0732989,"name":"offline","context":{"idset":"7995"}}
+{"timestamp":1714253829.0770445,"name":"offline","context":{"idset":"7996"}}
+{"timestamp":1714253829.9386129,"name":"offline","context":{"idset":"7997"}}
+{"timestamp":1714253829.9798839,"name":"offline","context":{"idset":"7998"}}
+{"timestamp":1714253830.0089738,"name":"offline","context":{"idset":"7999"}}
+{"timestamp":1714253830.0633421,"name":"offline","context":{"idset":"8000"}}
+{"timestamp":1714253833.0703142,"name":"offline","context":{"idset":"8001"}}
+{"timestamp":1714253833.0784085,"name":"offline","context":{"idset":"8002"}}
+{"timestamp":1714253833.0870523,"name":"offline","context":{"idset":"8003"}}
+{"timestamp":1714253833.9466226,"name":"offline","context":{"idset":"8004"}}
+{"timestamp":1714253834.0002184,"name":"offline","context":{"idset":"8005"}}
+{"timestamp":1714253834.0078487,"name":"offline","context":{"idset":"8006"}}
+{"timestamp":1714253834.0689223,"name":"offline","context":{"idset":"8007"}}
+{"timestamp":1714253837.0747962,"name":"offline","context":{"idset":"8008"}}
+{"timestamp":1714253837.0821261,"name":"offline","context":{"idset":"8009"}}
+{"timestamp":1714253837.0894995,"name":"offline","context":{"idset":"8010"}}
+{"timestamp":1714253837.1588953,"name":"offline","context":{"idset":"8011"}}
+{"timestamp":1714253839.0670016,"name":"offline","context":{"idset":"8012"}}
+{"timestamp":1714253839.0706353,"name":"offline","context":{"idset":"8013"}}
+{"timestamp":1714253839.8261092,"name":"offline","context":{"idset":"8014"}}
+{"timestamp":1714253839.881146,"name":"offline","context":{"idset":"8015"}}
+{"timestamp":1714253839.884908,"name":"offline","context":{"idset":"8016"}}
+{"timestamp":1714253839.8888278,"name":"offline","context":{"idset":"8017"}}
+{"timestamp":1714253839.9507585,"name":"offline","context":{"idset":"8018"}}
+{"timestamp":1714253843.0692093,"name":"offline","context":{"idset":"8019"}}
+{"timestamp":1714253843.0728436,"name":"offline","context":{"idset":"8020"}}
+{"timestamp":1714253843.0764346,"name":"offline","context":{"idset":"8021"}}
+{"timestamp":1714253843.0800173,"name":"offline","context":{"idset":"8022"}}
+{"timestamp":1714253843.8569086,"name":"offline","context":{"idset":"8023"}}
+{"timestamp":1714253843.9141889,"name":"offline","context":{"idset":"8024"}}
+{"timestamp":1714253843.9206636,"name":"offline","context":{"idset":"8025"}}
+{"timestamp":1714253843.9780436,"name":"offline","context":{"idset":"8026"}}
+{"timestamp":1714253847.187443,"name":"offline","context":{"idset":"8027"}}
+{"timestamp":1714253847.1938121,"name":"offline","context":{"idset":"8028"}}
+{"timestamp":1714253847.1983464,"name":"offline","context":{"idset":"8029"}}
+{"timestamp":1714253847.2028189,"name":"offline","context":{"idset":"8030"}}
+{"timestamp":1714253849.0744438,"name":"offline","context":{"idset":"8031"}}
+{"timestamp":1714253849.0805967,"name":"offline","context":{"idset":"8032"}}
+{"timestamp":1714253849.0865698,"name":"offline","context":{"idset":"8033"}}
+{"timestamp":1714253849.8060219,"name":"offline","context":{"idset":"8034"}}
+{"timestamp":1714253851.0715463,"name":"offline","context":{"idset":"8035"}}
+{"timestamp":1714253851.075459,"name":"offline","context":{"idset":"8036"}}
+{"timestamp":1714253851.0792773,"name":"offline","context":{"idset":"8037"}}
+{"timestamp":1714253851.0831676,"name":"offline","context":{"idset":"8038"}}
+{"timestamp":1714253851.8457396,"name":"offline","context":{"idset":"8039"}}
+{"timestamp":1714253853.067719,"name":"offline","context":{"idset":"8040"}}
+{"timestamp":1714253853.0717776,"name":"offline","context":{"idset":"8041"}}
+{"timestamp":1714253853.0761895,"name":"offline","context":{"idset":"8042"}}
+{"timestamp":1714253853.8848624,"name":"offline","context":{"idset":"8043"}}
+{"timestamp":1714253855.0671759,"name":"offline","context":{"idset":"8044"}}
+{"timestamp":1714253855.0715528,"name":"offline","context":{"idset":"8045"}}
+{"timestamp":1714253855.1469624,"name":"offline","context":{"idset":"8046"}}
+{"timestamp":1714253857.0680051,"name":"offline","context":{"idset":"8047"}}
+{"timestamp":1714253857.0747025,"name":"offline","context":{"idset":"8048"}}
+{"timestamp":1714253857.0815234,"name":"offline","context":{"idset":"8049"}}
+{"timestamp":1714253857.1483662,"name":"offline","context":{"idset":"8051"}}
+{"timestamp":1714253859.8459656,"name":"offline","context":{"idset":"8052"}}
+{"timestamp":1714254795.8771341,"name":"drain","context":{"idset":"7730,7739","reason":"epilog failed for jobid frzXAYpp6VD","overwrite":0}}
+{"timestamp":1714254795.9765933,"name":"offline","context":{"idset":"7739"}}
+{"timestamp":1714254874.9079893,"name":"online","context":{"idset":"7928,7934"}}
+{"timestamp":1714254875.0257909,"name":"online","context":{"idset":"7945"}}
+{"timestamp":1714254876.029114,"name":"online","context":{"idset":"7979"}}
+{"timestamp":1714254876.0336833,"name":"online","context":{"idset":"7929-7930"}}
+{"timestamp":1714254876.1931796,"name":"online","context":{"idset":"7969"}}
+{"timestamp":1714254876.8078437,"name":"online","context":{"idset":"7944,7986"}}
+{"timestamp":1714254877.0344124,"name":"online","context":{"idset":"7999"}}
+{"timestamp":1714254877.8889866,"name":"online","context":{"idset":"7947"}}
+{"timestamp":1714254877.8956087,"name":"online","context":{"idset":"7935"}}
+{"timestamp":1714254877.9027152,"name":"online","context":{"idset":"7926,7932"}}
+{"timestamp":1714254877.9095957,"name":"online","context":{"idset":"7977"}}
+{"timestamp":1714254878.0817549,"name":"online","context":{"idset":"7943"}}
+{"timestamp":1714254878.3856571,"name":"online","context":{"idset":"7990"}}
+{"timestamp":1714254878.4860196,"name":"online","context":{"idset":"7997"}}
+{"timestamp":1714254878.5820241,"name":"online","context":{"idset":"7936"}}
+{"timestamp":1714254879.0349195,"name":"online","context":{"idset":"8046"}}
+{"timestamp":1714254879.839124,"name":"online","context":{"idset":"8050"}}
+{"timestamp":1714254879.843055,"name":"online","context":{"idset":"7975"}}
+{"timestamp":1714254879.846843,"name":"online","context":{"idset":"7925,7939,7995"}}
+{"timestamp":1714254880.001498,"name":"online","context":{"idset":"7982"}}
+{"timestamp":1714254880.1085129,"name":"online","context":{"idset":"7948,7980,7983,8001"}}
+{"timestamp":1714254880.3419719,"name":"online","context":{"idset":"7989,8033"}}
+{"timestamp":1714254880.4847851,"name":"online","context":{"idset":"7988,8006,8020"}}
+{"timestamp":1714254880.5856407,"name":"online","context":{"idset":"7933,7941-7942,7946,7974,8007,8025"}}
+{"timestamp":1714254880.6864991,"name":"online","context":{"idset":"7996,8045"}}
+{"timestamp":1714254880.8198113,"name":"online","context":{"idset":"7940,8032"}}
+{"timestamp":1714254880.9309695,"name":"online","context":{"idset":"7962,7966"}}
+{"timestamp":1714254881.8138232,"name":"online","context":{"idset":"7927,7931,7938,7958,7965,7978,7987,7991,7994,7998,8018,8024"}}
+{"timestamp":1714254881.8181489,"name":"online","context":{"idset":"7949,7981,8000"}}
+{"timestamp":1714254881.822422,"name":"online","context":{"idset":"7950,7953,7961,7976,8008,8043"}}
+{"timestamp":1714254881.8262239,"name":"online","context":{"idset":"7951,7959,7984-7985,8003,8047"}}
+{"timestamp":1714254881.8300707,"name":"online","context":{"idset":"7993"}}
+{"timestamp":1714254881.8342466,"name":"online","context":{"idset":"7937,7955,7957,7968,8013,8028,8031,8036"}}
+{"timestamp":1714254881.9166884,"name":"online","context":{"idset":"8029"}}
+{"timestamp":1714254882.1262951,"name":"online","context":{"idset":"7952,7956,7960,7963,7970-7973,7992,8010-8011,8019,8021,8037,8042"}}
+{"timestamp":1714254882.4258106,"name":"online","context":{"idset":"7954,8009,8022,8027,8030,8038,8040,8044"}}
+{"timestamp":1714254882.6297777,"name":"online","context":{"idset":"8005,8016,8034,8039,8041,8052"}}
+{"timestamp":1714254882.7398584,"name":"online","context":{"idset":"8012,8049"}}
+{"timestamp":1714254882.9357219,"name":"online","context":{"idset":"7964,8002,8004,8017"}}
+{"timestamp":1714254883.7911878,"name":"online","context":{"idset":"7967,8015"}}
+{"timestamp":1714254883.7949014,"name":"online","context":{"idset":"8023"}}
+{"timestamp":1714254955.26968,"name":"undrain","context":{"idset":"8050,8052"}}
+{"timestamp":1714255042.2388453,"name":"online","context":{"idset":"8051"}}
+{"timestamp":1714255042.9100976,"name":"online","context":{"idset":"8035"}}
+{"timestamp":1714255087.9358535,"name":"online","context":{"idset":"8026"}}
+{"timestamp":1714255171.1099508,"name":"undrain","context":{"idset":"8051"}}
+{"timestamp":1714257331.9034593,"name":"offline","context":{"idset":"9139"}}
+{"timestamp":1714257791.0644474,"name":"offline","context":{"idset":"9904"}}
+{"timestamp":1714257791.153657,"name":"offline","context":{"idset":"9923"}}
+{"timestamp":1714257792.7926567,"name":"offline","context":{"idset":"9905"}}
+{"timestamp":1714257792.7963395,"name":"offline","context":{"idset":"9906"}}
+{"timestamp":1714257792.7998469,"name":"offline","context":{"idset":"9954"}}
+{"timestamp":1714257792.8033011,"name":"offline","context":{"idset":"9970"}}
+{"timestamp":1714257793.9760163,"name":"offline","context":{"idset":"9920"}}
+{"timestamp":1714257793.9798968,"name":"offline","context":{"idset":"9955"}}
+{"timestamp":1714257793.9842801,"name":"offline","context":{"idset":"9956"}}
+{"timestamp":1714257793.9885747,"name":"offline","context":{"idset":"9968"}}
+{"timestamp":1714257793.9927058,"name":"offline","context":{"idset":"9969"}}
+{"timestamp":1714257794.0880673,"name":"offline","context":{"idset":"9971"}}
+{"timestamp":1714257796.1407776,"name":"offline","context":{"idset":"9924"}}
+{"timestamp":1714257796.1446078,"name":"offline","context":{"idset":"9936"}}
+{"timestamp":1714257796.1622648,"name":"offline","context":{"idset":"9937"}}
+{"timestamp":1714257796.1658421,"name":"offline","context":{"idset":"9939"}}
+{"timestamp":1714257796.1694324,"name":"offline","context":{"idset":"9940"}}
+{"timestamp":1714257796.183768,"name":"offline","context":{"idset":"9972"}}
+{"timestamp":1714257799.0601838,"name":"offline","context":{"idset":"9907"}}
+{"timestamp":1714257799.6557217,"name":"offline","context":{"idset":"9908"}}
+{"timestamp":1714257803.073123,"name":"offline","context":{"idset":"9921"}}
+{"timestamp":1714257803.1504126,"name":"offline","context":{"idset":"9938"}}
+{"timestamp":1714257809.1477013,"name":"offline","context":{"idset":"9953"}}
+{"timestamp":1714258278.9221523,"name":"offline","context":{"idset":"11568"}}
+{"timestamp":1714258279.0214133,"name":"offline","context":{"idset":"11564"}}
+{"timestamp":1714259196.568712,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1714259196.7075934,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1714259196.8077269,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1714259500.9286702,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1714259501.9007397,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1714259501.904388,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1714259501.9080257,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1714259502.0286047,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1714259502.1274698,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1714259502.3757927,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1714259502.6870015,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1714259502.784838,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1714259502.9865942,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1714259503.0855174,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1714259503.9009585,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1714259503.9046333,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1714259504.3606198,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1714259504.9686575,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1714259505.8626728,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1714260328.7714157,"name":"online","context":{"idset":"10251"}}
+{"timestamp":1714261167.0635827,"name":"offline","context":{"idset":"11051"}}
+{"timestamp":1714261167.1491642,"name":"offline","context":{"idset":"11071"}}
+{"timestamp":1714261169.8506598,"name":"offline","context":{"idset":"11037"}}
+{"timestamp":1714261171.9849951,"name":"offline","context":{"idset":"11052"}}
+{"timestamp":1714261182.2488599,"name":"offline","context":{"idset":"11022"}}
+{"timestamp":1714261207.8425629,"name":"offline","context":{"idset":"11007"}}
+{"timestamp":1714261679.8212521,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1714261680.058006,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1714261680.3616223,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1714261680.3701613,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1714261680.375829,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1714261680.4766653,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1714261680.6215703,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1714261680.6284671,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1714261680.7205949,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1714261680.8899012,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1714261680.9895244,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1714261681.0532744,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1714261681.9612994,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1714261681.9649236,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1714261681.9694641,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1714261681.973043,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1714261681.9766238,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1714261681.9801764,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1714261682.5465553,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1714261682.8945298,"name":"offline","context":{"idset":"10895"}}
+{"timestamp":1714261682.9995365,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1714261683.0849395,"name":"offline","context":{"idset":"10896"}}
+{"timestamp":1714261683.1078334,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1714261684.1047201,"name":"offline","context":{"idset":"10900"}}
+{"timestamp":1714261684.1131923,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1714261684.1187706,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1714261684.1239412,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1714261684.1289272,"name":"offline","context":{"idset":"10894"}}
+{"timestamp":1714261684.1340644,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1714261684.1381385,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1714261684.1421752,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1714261684.1462975,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1714261684.1506994,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1714261684.1557083,"name":"offline","context":{"idset":"10898"}}
+{"timestamp":1714261684.4141252,"name":"offline","context":{"idset":"10897"}}
+{"timestamp":1714261684.8594968,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1714261684.9600997,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1714261685.8259382,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1714261685.8286982,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1714261685.8322113,"name":"offline","context":{"idset":"10918"}}
+{"timestamp":1714261685.8362486,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1714261685.839169,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1714261685.8426163,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1714261685.9463103,"name":"offline","context":{"idset":"10920"}}
+{"timestamp":1714261686.0881882,"name":"offline","context":{"idset":"10909"}}
+{"timestamp":1714261686.0999465,"name":"offline","context":{"idset":"10915"}}
+{"timestamp":1714261686.1931589,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1714261686.3733625,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1714261686.4671023,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1714261686.5995319,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1714261686.6971822,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1714261686.9594142,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1714261687.9472451,"name":"offline","context":{"idset":"10912"}}
+{"timestamp":1714261687.9516509,"name":"offline","context":{"idset":"10916"}}
+{"timestamp":1714261687.9559081,"name":"offline","context":{"idset":"10933"}}
+{"timestamp":1714261687.9587867,"name":"offline","context":{"idset":"10934"}}
+{"timestamp":1714261687.9624052,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1714261688.0791895,"name":"offline","context":{"idset":"10936"}}
+{"timestamp":1714261688.2662647,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1714261688.3692522,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1714261688.4963648,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1714261688.5036538,"name":"offline","context":{"idset":"10937"}}
+{"timestamp":1714261688.6025143,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1714261688.7309856,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1714261688.8322589,"name":"offline","context":{"idset":"10926"}}
+{"timestamp":1714261688.9613822,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1714261689.0747886,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1714261689.9750063,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1714261689.9775562,"name":"offline","context":{"idset":"10929"}}
+{"timestamp":1714261689.9814785,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1714261689.985815,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1714261689.9898448,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1714261689.9934034,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1714261689.9974036,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1714261690.32743,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1714261690.4242296,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1714261690.5614109,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1714261690.5659158,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1714261690.5718193,"name":"offline","context":{"idset":"10952"}}
+{"timestamp":1714261690.6693902,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1714261690.8963926,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1714261691.7670882,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1714261691.7693613,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1714261691.7716713,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1714261692.1909511,"name":"offline","context":{"idset":"10965"}}
+{"timestamp":1714261692.3684354,"name":"offline","context":{"idset":"10946"}}
+{"timestamp":1714261692.4686034,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1714261693.0411079,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1714261693.0477538,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1714261693.8130119,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1714261693.8164804,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1714261693.819979,"name":"offline","context":{"idset":"10968"}}
+{"timestamp":1714261693.8234134,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1714261693.8268871,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1714261693.8303263,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1714261693.8341181,"name":"offline","context":{"idset":"10971"}}
+{"timestamp":1714261693.8376801,"name":"offline","context":{"idset":"10961"}}
+{"timestamp":1714261693.9596748,"name":"offline","context":{"idset":"10957"}}
+{"timestamp":1714261694.2729821,"name":"offline","context":{"idset":"10960"}}
+{"timestamp":1714261694.391077,"name":"offline","context":{"idset":"10981"}}
+{"timestamp":1714261694.3954473,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1714261694.4886308,"name":"offline","context":{"idset":"10967"}}
+{"timestamp":1714261694.5665381,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1714261694.6648653,"name":"offline","context":{"idset":"10959"}}
+{"timestamp":1714261695.1046834,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1714261695.8170276,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1714261695.8205094,"name":"offline","context":{"idset":"10986"}}
+{"timestamp":1714261695.8239508,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1714261695.827333,"name":"offline","context":{"idset":"10983"}}
+{"timestamp":1714261695.8307447,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1714261695.911797,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1714261696.1068413,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1714261696.2055197,"name":"offline","context":{"idset":"10988"}}
+{"timestamp":1714261696.2255459,"name":"offline","context":{"idset":"10977"}}
+{"timestamp":1714261696.3242991,"name":"offline","context":{"idset":"10973"}}
+{"timestamp":1714261696.4410207,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1714261696.6905818,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1714261696.8232529,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1714261696.92184,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1714261697.8351889,"name":"offline","context":{"idset":"10989"}}
+{"timestamp":1714261697.8387008,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1714261698.0910037,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1714261698.3868439,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1714261698.5081689,"name":"offline","context":{"idset":"10992"}}
+{"timestamp":1714261698.6075573,"name":"offline","context":{"idset":"10993"}}
+{"timestamp":1714261698.8163362,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1714261698.9358499,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1714261699.8876421,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1714263845.9560158,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1714263853.8341312,"name":"drain","context":{"idset":"10594,10598,10602-10603","reason":"epilog failed for jobid fs2GN8sWYJ3","overwrite":0}}
+{"timestamp":1714265739.062114,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1714265739.9649644,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1714265743.1177385,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1714265746.0016942,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1714265793.7916229,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1714266009.8050449,"name":"drain","context":{"idset":"10116","reason":"broker was unresponsive"}}
+{"timestamp":1714268032.9532311,"name":"online","context":{"idset":"10594"}}
+{"timestamp":1714268146.3897047,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1714268369.3868837,"name":"online","context":{"idset":"10594"}}
+{"timestamp":1714268814.1866186,"name":"undrain","context":{"idset":"10594"}}
+{"timestamp":1714268875.1246095,"name":"undrain","context":{"idset":"10598"}}
+{"timestamp":1714268881.175643,"name":"drain","context":{"idset":"10598","overwrite":0}}
+{"timestamp":1714269745.0743012,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1714269745.0783761,"name":"offline","context":{"idset":"10598"}}
+{"timestamp":1714269745.0819168,"name":"offline","context":{"idset":"10602"}}
+{"timestamp":1714269745.8514044,"name":"offline","context":{"idset":"10603"}}
+{"timestamp":1714270654.18578,"name":"online","context":{"idset":"9788"}}
+{"timestamp":1714270965.9481194,"name":"online","context":{"idset":"10594"}}
+{"timestamp":1714270994.8178465,"name":"online","context":{"idset":"10598"}}
+{"timestamp":1714271048.7804563,"name":"online","context":{"idset":"10602"}}
+{"timestamp":1714271076.796922,"name":"online","context":{"idset":"10603"}}
+{"timestamp":1714271178.0431464,"name":"undrain","context":{"idset":"10598,10602-10603"}}
+{"timestamp":1714271381.9888396,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1714271798.9175851,"name":"online","context":{"idset":"10116"}}
+{"timestamp":1714271842.2887208,"name":"undrain","context":{"idset":"10116"}}
+{"timestamp":1714271876.0146339,"name":"online","context":{"idset":"10104"}}
+{"timestamp":1714271908.399559,"name":"online","context":{"idset":"10127"}}
+{"timestamp":1714271943.0377741,"name":"online","context":{"idset":"10165"}}
+{"timestamp":1714271976.0240757,"name":"online","context":{"idset":"10203"}}
+{"timestamp":1714272023.9234786,"name":"online","context":{"idset":"10204"}}
+{"timestamp":1714272555.8987155,"name":"drain","context":{"idset":"9968","reason":"broker was unresponsive"}}
+{"timestamp":1714272556.1845803,"name":"online","context":{"idset":"9904,9956"}}
+{"timestamp":1714272556.4021909,"name":"online","context":{"idset":"9905,9955"}}
+{"timestamp":1714272556.7142079,"name":"online","context":{"idset":"9971"}}
+{"timestamp":1714272556.9428172,"name":"online","context":{"idset":"9906-9907,9939,9954,9970"}}
+{"timestamp":1714272557.0664127,"name":"online","context":{"idset":"9938"}}
+{"timestamp":1714272557.8739319,"name":"online","context":{"idset":"9921,9923-9924,9936,9940,9969"}}
+{"timestamp":1714272557.8774297,"name":"online","context":{"idset":"9908,9920,9937,9953,9968,9972"}}
+{"timestamp":1714272675.0781598,"name":"undrain","context":{"idset":"9968"}}
+{"timestamp":1714277493.8328714,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1714307115.8589854,"name":"offline","context":{"idset":"6630"}}
+{"timestamp":1714314900.9862883,"name":"resource-init","context":{"restart":true,"drain":{"11367":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrctdZhtSj"},"11370":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcu9RKRbV"},"11283":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrce9w4ddq"},"11312":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcj9o8DLB"},"11336":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcoFo6vEs"},"411":{"timestamp":1713564989.0,"reason":"Unreachable"},"11346":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcpz86KLK"},"11307":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrciF7bjXV"},"336,491":{"timestamp":1713230524.0,"reason":"Down"},"11309":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcictjBdH"},"11278":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcd9M51jV"},"11343":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcpTzAv5m"},"11318":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrck9TkH67"},"11379":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcvxbzUET"},"11261":{"timestamp":1714239116.0,"reason":"prolog failed for jobid frrcZsWCykT"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"11266":{"timestamp":1714239115.0,"reason":"prolog failed for jobid frrcakDaQrb"},"98,100,102,111-112,114,123-124,161,184,187,190,208,303-304,310,317,319,330":{"timestamp":1714079956.0,"reason":"epilog failed for jobid frcnKxdu6xP"},"11315":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcjgAPW7q"},"11272":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcc7WS12F"},"11335":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrco7ET4qM"},"11270":{"timestamp":1714239116.0,"reason":"prolog failed for jobid frrcbiqQz4P"},"11319":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrckK9dato"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"881-882":{"timestamp":1714153692.0,"reason":"--reason H/W troubleshoot"},"11378":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcvnB7XFm"},"11326":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcmZceUpP"},"11286":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrceozaCFH"},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"349-359,361-371,374-396,398-404":{"timestamp":1714060395.0,"reason":"New Blade Installation --JRG"},"77-84":{"timestamp":1713795613.0,"reason":"New blade installation"},"11275":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcccvqkP9"},"549-556":{"timestamp":1713553592.0,"reason":"New Blade Installation"},"11313":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcjKawUGF"},"11380":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcw9HhJ2b"},"11368":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrctoniwAf"},"829-830":{"timestamp":1712700945.0,"reason":""},"543":{"timestamp":1714074108.0,"reason":"nodediag failed cxi"},"939":{"timestamp":1713963885.0,"reason":"nodediag failed amdapu"},"771-772":{"timestamp":1713195979.0,"reason":""},"141-148":{"timestamp":1713545597.0,"reason":"ARP Storm Problem testing"},"9054":{"timestamp":1712864116.0,"reason":"broker was unresponsive"},"871-874":{"timestamp":1714057817.0,"reason":""},"461-476":{"timestamp":1713794797.0,"reason":"New Blade Installation --JRG"},"410":{"timestamp":1713378635.0,"reason":"prolog failed for jobid fq5jtQQSQnT"},"541":{"timestamp":1714074106.0,"reason":"nodediag failed cxi"},"7285-7412":{"timestamp":1714248995.0,"reason":"prolog failed for jobid frzMuSLMNAK"},"11277":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrccyNsqq1"},"482-490,492":{"timestamp":1713915346.0,"reason":"New Blade Installation --JRG"},"11350":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcqeMynvB"},"11324":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcmDog5R9"},"11356":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcreidXHR"},"11254":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcYiP3Viw"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"11256":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcZ3VzEWs"},"191":{"timestamp":1714078824.0,"reason":"unreachable"},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"11321":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrckf61vhm"},"11302":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrchQ1dXB5"},"11338":{"timestamp":1714239648.0,"reason":"prolog failed for jobid frrcodExXgB"},"11348":{"timestamp":1714241556.0,"reason":"prolog failed for jobid frrcqKVrvwh"},"11271":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcbxPARR9"},"11295":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcgDqgYbR"},"11362":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcsjdB3p3"},"11341":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcp7zpbi7"},"11255":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcYt1xpxw"},"11332":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcnbCTciB"},"548":{"timestamp":1713563915.0,"reason":"epilog failed for jobid fqUWkS8j8hM"},"11509-11540,11542-11547,11549-11553,11557-11559,11567,11570-11617":{"timestamp":1714249243.0,"reason":"prolog failed for jobid frzRkVsE1Rh"},"825-826":{"timestamp":1714153531.0,"reason":"--reason H/W troubleshoot"},"11288":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcf9vxY4F"},"9003":{"timestamp":1712873964.0,"reason":"nodediag failed dgemm_perf"},"115":{"timestamp":1714079493.0,"reason":"unreachable"},"525-532":{"timestamp":1713290620.0,"reason":""},"11282":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcdqXdYjZ"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"8159,8278-8281":{"timestamp":1714249943.0,"reason":"prolog failed for jobid frzX3xjjC2j"},"11325":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcmQ4B7RR"},"877":{"timestamp":1714078512.0,"reason":"nodediag failed amdapu"},"11305":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrchtqwZ43"},"11257":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcZDKHUjH"},"11298":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcgieWbC3"},"11267":{"timestamp":1714239114.0,"reason":"prolog failed for jobid frrcaz85gAB"},"172":{"timestamp":1713981544.0,"reason":"Unreachable"},"11293":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcfsxGBM9"},"8107-8108":{"timestamp":1714249949.0,"reason":"prolog failed for jobid frzX7r2t3t3"},"8923-8963,8973-8981,8983-8992,8995-8998,9001-9002,9005-9017,9019-9027":{"timestamp":1714249928.0,"reason":"prolog failed for jobid frzWyRvCWVD"},"796":{"timestamp":1712868054.0,"reason":"broker was unresponsive"},"421-460":{"timestamp":1713794410.0,"reason":"New Blade Installation --JRG"},"797-798":{"timestamp":1712261863.0,"reason":"--reason swapping blades into x1900 - wendy"},"11276":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrccoQgfvX"},"11297":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcgYw9J71"},"11337":{"timestamp":1714239606.0,"reason":"prolog failed for jobid frrcoS27xxo"},"977-978":{"timestamp":1714144603.0,"reason":""},"11262":{"timestamp":1714239114.0,"reason":"prolog failed for jobid frrca31iNaj"},"11310":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcioM67tK"},"8282-8290,8292-8303,8305-8308,8821-8832,8834-8917,8919-8922":{"timestamp":1714249934.0,"reason":"prolog failed for jobid frzX17LBKTM"},"11340":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcoxmd4yd"},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"8181,8183,8234,8237,8291,8304":{"timestamp":1713810085.0,"reason":"testing -kk"},"119":{"timestamp":1714079565.0,"reason":"unreachable"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"11299":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcguqsAd5"},"961-962":{"timestamp":1713197674.0,"reason":""},"206":{"timestamp":1713564985.0,"reason":"Unreachable"},"545":{"timestamp":1714069878.0,"reason":"nodediag failed cxi"},"11358":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcs2rWoL3"},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"925-926":{"timestamp":1713540640.0,"reason":""},"11311":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrciymy4s1"},"8833":{"timestamp":1713991869.0,"reason":"nodediag failed amdapu"},"11287":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrceynPTBM"},"11317":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcjzD4m5H"},"110":{"timestamp":1714079561.0,"reason":"unreachable"},"546":{"timestamp":1714069873.0,"reason":"nodediag failed cxi pci"},"11304":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrchjPQ8nT"},"11372":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcuW5DRJ7"},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"11281":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcdftiDVZ"},"11253":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcYXbQioR"},"789-794":{"timestamp":1711461777.0,"reason":""},"11359":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcsEJhFaX"},"11329":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcn6HhczB"},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"360":{"timestamp":1713981579.0,"reason":"New Blade Installation --JRG"},"11330":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcnG2YuMZ"},"11308":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrciR2pvsH"},"11269":{"timestamp":1714239115.0,"reason":"prolog failed for jobid frrcbYzdkZd"},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"286":{"timestamp":1713981553.0,"reason":"Unreachable"},"963-964":{"timestamp":1713196418.0,"reason":"moving blade"},"542":{"timestamp":1714074109.0,"reason":"nodediag failed cxi"},"11203-11252":{"timestamp":1714244322.0,"reason":""},"11316":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcjpoVKNP"},"11289":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcfJyn9pK"},"11349":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcqUvvMvw"},"342":{"timestamp":1713981566.0,"reason":"Unreachable"},"11296":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcgPMBwRh"},"11279":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcdLZuaSs"},"11369":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrctywJ23Z"},"11320":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrckVFEhD1"},"420":{"timestamp":1714074112.0,"reason":"epilog failed for jobid frbYvBLD1gT"},"11371":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcuKBehFD"},"117":{"timestamp":1713981538.0,"reason":"Unreachable"},"11366":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrctShd9gB"},"11342":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcpJabUNs"},"11306":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrci4jgm7V"},"11377":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcvaxGxYP"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"11303":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrchZDM4dD"},"11363":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcsuG6P43"},"11263":{"timestamp":1714239114.0,"reason":"prolog failed for jobid frrcaCYhkhM"},"878":{"timestamp":1714078484.0,"reason":"nodediag failed amdapu"},"11314":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcjWDgKVh"},"11273":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrccGzTQaB"},"11344":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcpdj2CT9"},"11347":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcq9oyd91"},"11339":{"timestamp":1714239648.0,"reason":"prolog failed for jobid frrconxKpmD"},"779-780":{"timestamp":1713907642.0,"reason":"--reason Troubleshoot BIOS issue"},"934":{"timestamp":1713967835.0,"reason":"epilog failed for jobid frMYzSv7G6b"},"11290":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcfRkesVh"},"11285":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcedHPPAo"},"929-930":{"timestamp":1713552018.0,"reason":""},"11284":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrceKyhmPM"},"11265":{"timestamp":1714239115.0,"reason":"prolog failed for jobid frrcaZgmWkX"},"11333":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcnm4iqVH"},"11331":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcnRmQBiw"},"165":{"timestamp":1713981434.0,"reason":"broker was unresponsive"},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"11357":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcrqdgEcf"},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"120":{"timestamp":1713981548.0,"reason":"Unreachable"},"923-924":{"timestamp":1714064541.0,"reason":"status"},"11375":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcvBr49nf"},"808":{"timestamp":1713455939.0,"reason":""},"11260":{"timestamp":1714239114.0,"reason":"prolog failed for jobid frrcZhqofE7"},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"11352":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcqxzkmMZ"},"11353":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcr8QLD4T"},"11301":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrchEXc7d9"},"149-156":{"timestamp":1713545634.0,"reason":"ARP Storm Problem testing"},"11345":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcpovZFtj"},"11258":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcZP48m6f"},"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"11361":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcsa7feym"},"11274":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrccT1cZ3M"},"11376":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcvQ1kFvo"},"133-140":{"timestamp":1713545617.0,"reason":"ARP Storm Problem testing"},"90":{"timestamp":1713564981.0,"reason":"New Blade installation"},"807":{"timestamp":1713376017.0,"reason":""},"11365":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrctGXa5Ww"},"544":{"timestamp":1714069881.0,"reason":"nodediag failed cxi"},"11351":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcqoZhLNK"},"397":{"timestamp":1713229804.0,"reason":"New Blade Installation --JRG"},"787-788":{"timestamp":1713219302.0,"reason":""},"11291":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcfZJ1DdR"},"11364":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrct6QUyvP"},"11360":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcsPtecFq"},"11264":{"timestamp":1714239114.0,"reason":"prolog failed for jobid frrcaPfcNHM"},"11334":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcnwBow5q"},"814":{"timestamp":1712783332.0,"reason":""},"937-938":{"timestamp":1712866351.0,"reason":""},"795":{"timestamp":1712868053.0,"reason":"broker was unresponsive"},"803-804":{"timestamp":1713479733.0,"reason":"status"},"8982":{"timestamp":1713996208.0,"reason":"broker was unresponsive"},"11259":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcZYBQLhm"},"11322":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrckqD72JK"},"11294":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcg43sHfM"},"573-580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"11381-11508":{"timestamp":1714242195.0,"reason":"prolog failed for jobid frw93rnDZ8T"},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"373":{"timestamp":1713981698.0,"reason":"New Blade Installation --JRG"},"11328":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcmvaLKu9"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"372":{"timestamp":1713981581.0,"reason":"New Blade Installation --JRG"},"11292":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcfiUEmoD"},"11300":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrch4zcjWX"},"11354":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcrJ1mZ27"},"11323":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcm1zjoDq"},"7730,7739":{"timestamp":1714254796.0,"reason":"epilog failed for jobid frzXAYpp6VD"},"11280":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcdWV8mnf"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"},"85-89,91-92":{"timestamp":1714079045.0,"reason":"New Blade installation"},"11268":{"timestamp":1714239115.0,"reason":"prolog failed for jobid frrcbE16xBR"},"11327":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcmj3huod"},"11373":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcugwJA4f"},"971-974":{"timestamp":1714143211.0,"reason":""},"335":{"timestamp":1713981557.0,"reason":"Unreachable"},"11355":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcrU1SiCw"},"773-774":{"timestamp":1713802805.0,"reason":""},"811-812":{"timestamp":1713557753.0,"reason":"--reason Running hpe diags - JM"},"11374":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcuuvRsDq"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"547":{"timestamp":1713563916.0,"reason":"epilog failed for jobid fqUWmieM3AX"},"9075":{"timestamp":1712864130.0,"reason":"broker was unresponsive"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1714314901.00229,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1714314924.4138308,"name":"online","context":{"idset":"0"}}
+{"timestamp":1714314926.2298672,"name":"online","context":{"idset":"7814,7818,7827,7836,7842,7863,7877-7879,7892,7896,7910,8062,8075,8096,8108,8115,8128,8155,8175,8194,8214,8248,8261,8289,8313,8316,8322,8326,8337,8352,8354,8365,8373,8382,8390,8405,8411,8419,8573,8581,8584,8594,8598-8599,8602-8603,8616,8620,8637,8645,8652,8662,8679,8689-8691,8856,8860,8873,8887,8894,8898,8901,8909,8934,8967,8970,9099,9104,9121,9145,9149,9191,9196-9197,9223,9235,9263,9280,9302,9304-9305,9315-9317,10361,10372,10388,10390,10431,10471,10637,10641,10648,10652,10732,11404,11410,11421-11422,11453,11463,11506,11526,11532,11600,11611"}}
+{"timestamp":1714314926.3363922,"name":"online","context":{"idset":"808,873,7805,7825,7829,7831,7854,7858,7865,7870,7880,7882,7888,7898,7921,7923,8063,8072,8090,8100,8105,8113,8129,8135,8153,8161,8181-8182,8227-8228,8239,8242,8256,8262,8265,8269,8271,8280,8290-8292,8296,8305-8306,8344-8345,8362,8366,8376,8385,8387,8400,8410,8433,8582,8592,8604-8605,8639,8647,8649,8657,8663,8678,8684,8688,8836,8852,8865,8868,8891,8895,8904,8926-8927,8940,8945,8947,8969,9084,9089,9100,9103,9168,9177,9189,9193,9200,9203,9215-9216,9237,9248,9289,9321,9325,9330-9331,10251,10364-10365,10367-10368,10416,10435,10653,10663,10666,10669,10673,10688,10697-10698,10702,10713,10728,10736,11381,11389-11392,11398-11399,11403,11408,11424,11432,11444,11446-11447,11476-11477,11480-11481,11489-11490,11502-11503,11517,11521,11523,11530,11546,11576,11588,11591,11614-11615"}}
+{"timestamp":1714314926.5710852,"name":"online","context":{"idset":"825-826,871-872,874,877-878,883-884,977-978,7797-7804,7807-7813,7815-7816,7820-7824,7826,7828,7832-7835,7837,7839-7840,7843,7845-7848,7850-7853,7855-7857,7859-7860,7864,7866-7869,7871,7873-7875,7881,7883-7885,7887,7889-7890,7893-7895,7897,7899,7901-7907,7909,7912,7915-7916,7918-7920,7922,7924,7931-7932,7936,7942,7944,7948,7951,7955-7956,7958,7962,7970,7972,7974,7987,7995,8003,8007,8013,8015,8017,8025,8037,8040-8041,8044,8047,8049,8053-8054,8057-8058,8060-8061,8066-8067,8069-8071,8074,8076-8077,8079,8081-8084,8086-8089,8091,8093-8095,8097-8099,8101-8104,8107,8109-8110,8112,8114,8116-8117,8119-8122,8124-8125,8127,8130,8132,8134,8136-8141,8143,8146-8152,8156-8159,8162-8166,8168-8174,8176,8180,8183-8185,8187-8193,8195-8200,8202-8213,8215-8223,8225-8226,8229-8231,8236-8238,8240-8241,8243,8245-8247,8249-8253,8255,8257-8260,8263-8264,8266-8268,8270,8272-8279,8281-8288,8294-8295,8297-8298,8300-8303,8307-8312,8314-8315,8317-8321,8323,8325,8327-8336,8338-8343,8346-8350,8353,8355-8361,8363-8364,8367,8369-8372,8374-8375,8377,8379-8381,8383-8384,8386,8388-8389,8391-8399,8401-8404,8406-8409,8412-8418,8420-8421,8423-8426,8428-8432,8434-8436,8565,8567-8570,8572,8574-8580,8583,8585-8591,8593,8595-8597,8600-8601,8606-8614,8617-8619,8621-8630,8632-8633,8635-8636,8638,8640-8644,8646,8648,8650-8651,8653-8656,8658-8660,8664-8674,8676-8677,8680-8683,8685-8687,8692,8821-8832,8834-8835,8837-8851,8853-8855,8857-8859,8861-8864,8866-8867,8869-8872,8874-8875,8877-8886,8888-8890,8892-8893,8897,8899-8900,8902-8903,8905-8908,8910,8913-8916,8919,8921,8923,8925,8928-8931,8935,8938-8939,8941-8942,8944,8946,8948,8958,8961,8964-8966,8971-8972,8980,9050,9075,9077-9083,9086-9088,9091-9098,9101-9102,9105-9120,9122-9123,9141-9144,9146-9148,9151-9154,9156-9157,9159-9161,9163-9164,9166-9167,9169-9170,9172-9176,9178-9188,9190,9192,9194-9195,9199,9204,9206-9214,9217-9222,9224-9231,9233-9234,9236,9238-9241,9243-9247,9249-9262,9264-9267,9269,9272-9279,9281-9288,9290-9291,9293-9297,9299-9301,9303,9306-9314,9318,9320,9322,9324,9326,9328-9329,10357-10360,10362-10363,10366,10371,10373-10387,10389,10391-10394,10396-10415,10418,10420-10430,10432-10434,10436-10443,10445-10450,10452-10453,10455-10470,10472-10484,10487,10613-10614,10616-10620,10622-10636,10638,10640,10642-10645,10650-10651,10654-10662,10664-10665,10667-10668,10670-10672,10676-10679,10681-10682,10684-10687,10689-10694,10696,10700-10701,10703-10712,10714-10716,10719-10727,10729-10731,10733-10735,10738-10739,11382-11387,11393-11397,11400-11402,11405-11407,11409,11411-11420,11425-11427,11429-11430,11433-11437,11439-11441,11443,11445,11448-11452,11454-11460,11462,11464-11475,11478-11479,11482-11484,11486-11488,11491-11497,11499-11500,11504-11505,11507-11509,11513,11516,11518,11520,11524-11525,11527-11528,11531,11534-11539,11542-11545,11547,11549-11553,11557-11559,11567,11570,11572-11574,11577-11581,11583-11584,11586-11587,11590,11593-11599,11601-11610,11612-11613,11616-11617"}}
+{"timestamp":1714314926.6764913,"name":"online","context":{"idset":"7806,7817,7819,7844,7872,7886,7911,7914,7934-7935,7946,7953,7980,7985,7994,8001-8002,8020,8043,8055,8065,8073,8078,8080,8123,8131,8133,8145,8167,8178,8232,8293,8427,8571,8615,8911,8924,8933,8936,8943,8957,8959,8963,8973,8975-8976,8978-8979,9046-9047,9051,9090,9150,9155,9158,9165,9198,9232,9242,9292,9298,9332,10603,10647,10674,10680,10683,10737,10740,11428,11461,11498,11510,11522,11533,11571,11582"}}
+{"timestamp":1714314926.7940667,"name":"online","context":{"idset":"7862,7891,7927,7929,7933,7964,7988,8023,8026,10594,11388,11431,11540,11575"}}
+{"timestamp":1714314926.9092753,"name":"online","context":{"idset":"93-106,108-110,112-116,118-119,122-132,157-161,163-164,166-171,173,175-180,182-186,188-197,200-201,204-205,207,209-216,218-236,238,240-243,245-248,250-252,277-278,280-283,285,287-290,292-312,315-316,319-334,337-341,343-347,405-409,412-414,416-418,420,836,899-900,1189-1190,7861,7925-7926,7937-7939,7941,7943,7945,7947,7949-7950,7952,7954,7959,7965-7967,7969,7971,7975-7976,7978-7979,7981-7982,7984,7986,7989-7991,7993,7996-8000,8004-8006,8008-8012,8016,8018-8019,8021-8022,8027-8029,8034,8038-8039,8042,8046,8050-8052,8111,8126,8351,8896,8912,8917,8960,8962,8974,8977,9045,9048-9049,9052,9085,9162,9350,9363,9366,9375,9389,9425,9623,9686,9722,9769,9909,9917,9934,9962,10105-10106,10120,10133,10135,10152,10154,10165,10203,10321-10323,10486,10492,10532,10598,10602,10639,11078,11485"}}
+{"timestamp":1714314927.0124555,"name":"online","context":{"idset":"7957,7992,8031,8033,8036,8045,8634,8675,9070,9074,9270,9319,9333,9399,9442,9655,9705,9734-9735,9766,9786,9806,9851,9873,9884,9932-9933,9998,10125,10204,10330,11081,11585"}}
+{"timestamp":1714314927.116776,"name":"online","context":{"idset":"7830,7838,7928,7940,7961,7983,8024,8030,8032,8035,8059,8064,8068,8144,8154,8177,8179,8244,8422,8876,8937,9201-9202,9378,9386,9397,9428,9430-9431,9439,9451,9598,9605,9626,9646,9656,9679,9689,9749,9754,9756-9757,9790,9821,9846,9856,9894,9900,9947,9952"}}
+{"timestamp":1714314927.2207172,"name":"online","context":{"idset":"7977,8631,9171,9343,9364,9381,9398,9417-9418,9445,9447-9448,9606,9609,9644,9720,9729,9751,9792,9798,9803,9825,9861,9883,9886,9890,9898,9901,9910,9914,9918,9926,9944,9960,9963,10316,10325,10352,10355,10370,10417,10419,10444,10621,10649,10675,10826,11015,11044,11047-11048,11423,11438,11515,11589"}}
+{"timestamp":1714314927.4274166,"name":"online","context":{"idset":"2,6,10,13-14,18,21,29,35,37,41,44,47-48,50,53,55,59-60,1173,1178,1230,6534,6541,6553,6569,6571,6604,6607,6634,6643,7876,7917,8085,8160,8235,8953-8956,9069,9071,9073,9076,9334,9336-9340,9344-9345,9347,9349,9351-9352,9354,9361-9362,9365,9372,9384,9387,9391-9392,9400,9403,9405-9406,9408,9410,9413-9415,9420,9423,9429,9433-9434,9437,9440,9446,9450,9453-9454,9459-9460,9492,9501,9521,9590-9591,9595-9596,9600-9602,9604,9610-9611,9616-9619,9621,9627-9628,9632,9638-9640,9643,9645,9647,9649,9653-9654,9657,9662,9667,9669,9672-9674,9676-9678,9681-9682,9684-9685,9687-9688,9692,9698,9701-9702,9704,9706,9709,9711,9716,9724-9727,9732,9736-9737,9740-9742,9747,9750,9753,9758,9763-9764,9767-9768,9772,9775-9776,9779,9781,9784-9785,9787-9789,9793,9797,9799,9801,9804,9808,9811,9813-9814,9817-9818,9820,9822,9828-9832,9836-9840,9845,9847-9850,9853,9865-9867,9872,9876-9877,9882,9887,9889,9895-9897,9899,9902,9912-9913,9922,9925,9928-9929,9935,9941-9943,9945-9946,9957-9959,9983,9987,9997,10026,10051-10052,10056,10058,10062-10063,10075,10094,10099-10101,10108-10109,10113-10115,10119,10122,10126,10131-10132,10134,10320,10324,10326,10328,10331,10334,10338,10341,10344-10347,10350,10354,10485,10699,10717,10743,10748,10755,10774,10783,10793,10800,10803,10805,10844,10846,10848,10857,10859,10863,11002,11006,11011,11016,11018,11023-11024,11027,11031-11032,11043,11045,11059-11060,11062,11066,11072,11094,11105,11112,11118,11135,11157,11184,11200,11501,11511,11514,11519,11529,11650,11663,11678-11679,11702,11724,11728,11734,11745,11750,11753-11754,11777,11817,11827,11832,11851-11852,11861,11880"}}
+{"timestamp":1714314927.5307102,"name":"online","context":{"idset":"22,174,244,317,6557,8952,9346,9353,9356-9357,9371,9373,9376,9380,9382,9390,9393,9407,9411,9416,9432,9624,9630-9631,9633,9635,9637,9641,9658,9691,9777,9791,9834,9852,9862,9885,9931,9961,10812,11830"}}
+{"timestamp":1714314927.6370461,"name":"online","context":{"idset":"12,17,32,40,42,45,51,54,107,111,162,181,187,199,419,1180,1186,1208,1220,1223,1229,1232,6517,6543,6546,6558,6564,6580,6583,6612,6614,6616,6644,7913,8092,8324,8368,8378,8566,9359,9394,9402,9419,9422,9427,9441,9455,9634,9642,9648,9650,9659,9664,9668,9675,9683,9693,9696,9721,9731,9744,9782-9783,9794,9815,9819,9827,9841,9843-9844,9869-9870,9878-9879,9888,9893,9915,9930,9948-9949,9951,9999,10064,10067,10085,10087,10103,10110,10112,10117,10121,10327,10336-10337,10339,10349,10454,10769,10775,10782,10785,10820,10825,10838,10843,10851,10860,10864,10868,10999,11009,11019,11033,11041,11058,11064-11065,11073,11088-11089,11098,11100,11108,11127,11132,11164,11202,11512,11647,11670,11694,11696,11712,11733,11752,11756,11766,11782,11787-11788,11800-11801,11818,11823,11853,11864,11868,11883"}}
+{"timestamp":1714314927.7551229,"name":"online","context":{"idset":"4,11,16,20,23,27-28,39,43,56,58,318,573,1174-1175,1177,1182,1184-1185,1188,1205,1209,1213-1214,1217-1218,1222,1226-1227,1231,1233,1235,6519,6521-6523,6526,6528,6533,6535,6540,6544,6547,6549-6551,6555,6559-6561,6566-6567,6570,6572,6574,6581,6587-6588,6591,6596,6598,6602-6603,6611,6615,6618,6625-6627,6631,6636,6638,6641-6642,7841,7849,7900,7908,7930,7963,7968,7973,8056,8118,8142,8186,8224,8254,8299,8661,8920,8922,8932,8950-8951,8968,9072,9124,9128-9129,9134,9136,9205,9268,9271,9335,9341,9348,9358,9360,9367-9370,9377,9379,9383,9385,9388,9395-9396,9401,9404,9409,9412,9421,9424,9426,9435,9438,9443-9444,9452,9457-9458,9499,9556,9565,9589,9593-9594,9597,9599,9603,9607-9608,9612-9615,9620,9622,9625,9629,9636,9660-9661,9663,9666,9670-9671,9680,9690,9694-9695,9700,9703,9707-9708,9710,9712-9715,9717-9719,9723,9728,9730,9733,9739,9743,9745-9746,9748,9752,9755,9761-9762,9765,9770,9773-9774,9795-9796,9800,9802,9805,9807,9809,9812,9816,9823-9824,9826,9835,9842,9854-9855,9857,9859-9860,9863-9864,9868,9871,9874-9875,9880-9881,9891-9892,9903,9916,9919,9927,9940,9950,9981,9988,9992,10009,10021,10032,10040,10053,10070-10071,10076,10081,10102,10104,10107,10116,10118,10123,10127-10130,10177,10193,10201,10217,10249-10250,10312,10329,10332-10333,10335,10342-10343,10348,10351,10353,10356,10369,10395,10451,10540,10545,10564,10566,10615,10646,10695,10718,10741-10742,10744-10746,10751-10754,10756-10757,10759,10764-10767,10773,10778,10781,10787-10789,10791,10794,10796-10797,10801-10802,10807,10809-10811,10813-10814,10821,10835-10837,10839-10840,10847,10852,10856,10858,10862,10997,11000-11001,11003,11005,11008,11012,11014,11025,11028,11030,11034-11035,11038,11040,11049-11050,11053-11055,11061,11063,11067-11069,11074-11076,11079,11083,11087,11090,11092-11093,11096-11097,11099,11101,11104,11109,11111,11113,11115,11117,11122-11124,11134,11138,11142,11153,11159-11160,11162,11165-11166,11174,11177,11180-11183,11185-11186,11189-11191,11194-11195,11197,11201,11442,11592,11640,11644,11646,11648,11653,11655,11657,11659-11660,11667,11669,11672,11674-11675,11681,11687,11690,11693,11695,11697-11698,11700-11701,11704-11705,11707,11710,11714,11726,11738-11739,11741-11742,11747,11749,11751,11755,11758,11764-11765,11767,11769,11779,11784,11786,11789,11793,11795-11796,11798,11808,11810-11814,11820,11824,11829,11833,11835,11839,11842,11844-11845,11849,11859,11862,11865,11867,11870,11874,11884,11886-11887,11889"}}
+{"timestamp":1714314927.861438,"name":"online","context":{"idset":"3,7,15,19,24,30,33,36,46,1211,6520,6565,6595,6600,6640,9126,9506,10749,10758,10762,10830,11017,11120,11125,11131,11152,11178,11676,11689,11763,11775,11816,11843"}}
+{"timestamp":1714314927.8708603,"name":"online","context":{"idset":"9053-9054,9056,9456,9665"}}
+{"timestamp":1714314927.9712706,"name":"online","context":{"idset":"9342,9374,9436,9464,9524,9530,9651,9974,10003,10016,10061,10078,10153,10555"}}
+{"timestamp":1714314928.1062274,"name":"online","context":{"idset":"34,203,237,239,249,284,291,415,579,835,1179,1181,1183,1187,1206-1207,1210,1212,1216,1219,1221,1228,1234,1236,6518,6524-6525,6527,6530-6532,6536-6539,6542,6548,6552,6554,6563,6568,6573,6575,6577-6578,6582,6586,6589,6592-6593,6597,6599,6601,6605,6608-6609,6613,6617,6619-6624,6633,6637,6639,7960,8949,9055,9057-9058,9060,9132-9133,9138,9469,9471,9502,9525,9528,9560,9592,9652,9771,9833,9858,9904,9968,9993,10002,10017,10029,10031,10042,10093,10124,10221,10230,10260,10265,10504,10518,10529,10535,10538,10579,10588,10760,10776-10777,10786,10790,10798-10799,10804,10806,10816-10818,10823-10824,10829,10833,10841,10853,10855,10865-10867,11010,11013,11020,11039,11042,11046,11057,11070,11084,11095,11102,11107,11126,11129-11130,11133,11137,11141,11143-11145,11147-11151,11154-11156,11158,11161,11163,11169,11171-11172,11175-11176,11179,11193,11196,11199,11637-11639,11642-11643,11645,11652,11661-11662,11664,11668,11671,11677,11680,11684,11686,11688,11691,11699,11706,11735,11737,11740,11743,11746,11759,11761,11768,11771,11774,11780-11781,11783,11791-11792,11794,11797,11799,11804-11805,11807,11809,11815,11821,11825,11837-11838,11840-11841,11846,11848,11850,11854-11856,11863,11866,11872,11875,11882,11885,11888,11890-11892"}}
+{"timestamp":1714314928.2113416,"name":"online","context":{"idset":"8,49,208,1224-1225,6529,6556,6562,6579,6590,6606,6610,6635,9000-9002,9059,9127,9130-9131,9135,9137,9355,9512,9562,9564,9697,9699,9738,9759-9760,9780,9810,9908,9911,9937,9977,9986,9991,9996,10006-10007,10023,10037,10045,10049,10066,10098,10151,10158,10166-10167,10171,10239,10253,10287,10340,10488,10498,10512,10519,10526,10556,10560,10580,10780,10808,10815,10827-10828,10831,10849,10861,10998,11004,11036,11080,11103,11106,11116,11146,11173,11187,11649,11651,11654,11665,11673,11682-11683,11708-11709,11713,11716-11717,11762,11770,11772-11773,11776,11802-11803,11806,11831,11834,11869,11871,11873,11876,11878"}}
+{"timestamp":1714314928.3189321,"name":"online","context":{"idset":"9,8997-8999,9023,9026,9449,9472,9529,9778,9965,9982,9984,9994-9995,10001,10014-10015,10020,10024-10025,10055,10068-10069,10073,10082,10086,10088,10096,10111,10184,10222,10247,10277,10282,10292,10296,10303,10311,10489-10490,10547,10565,10576,10599,10606,10608,10772,10819,10832,11077,11121,11128,11139,11167,11192,11658,11685,11715,11727,11748,11785,11836,11847,11858,11860,11877"}}
+{"timestamp":1714314928.4253788,"name":"online","context":{"idset":"6545,6576,6584-6585,6594,6632,9006,9013-9014,9018-9022,9024,9027-9028,9125,9481,9489,9522,9554,9569,9583,9907,9923,9936,9955-9956,9964,9980,9990,10004,10011,10018,10033-10034,10038-10039,10041,10043,10048,10054,10059,10065,10079,10083-10084,10090-10091,10138,10141,10144,10162,10168,10194,10208,10216,10218,10232,10237,10242,10252,10266,10281,10289,10293,10295,10302,10309-10310,10314,10493,10505,10507-10508,10534,10546,10572,10583-10584,10595,10607,10747,10750,10761,10770-10771,10795,10822,10834,10845,11021,11026,11029,11056,11086,11110,11136,11140,11168,11198,11656,11703,11719-11720,11730,11757,11778,11819,11826,11828,11879,11881"}}
+{"timestamp":1714314928.5284514,"name":"online","context":{"idset":"25,31,576-577,1176,1215,9005,9008,9010,9015-9017,9483,9498,9508-9509,9511,9567,9905,9966,9970,9972,10000,10008,10022,10030,10057,10095,10097,10140,10148,10169,10192,10200,10226-10227,10238,10240,10255,10272,10274,10306,10509,10520,10541,10571,10610,10779,10854,11085,11114,11119,11188,11692,11721"}}
+{"timestamp":1714314928.6414881,"name":"online","context":{"idset":"8986,8990-8991,9011,9494,9516,9542-9543,9545,9548,9568,9576,9953,9967,9979,10028,10044,10050,10060,10092,10172,10178,10254,10273,10276,10288,10522-10523,10530,10533,10548-10550,10557,10561,10568,10577,10589,10591,10593,10601,10784,10792,11082,11091,11170,11723,11729,11744,11760,11822"}}
+{"timestamp":1714314928.8336399,"name":"online","context":{"idset":"5,38,6628,8981-8985,8987-8989,8992-8996,9007,9009,9025,9465,9468,9482,9500,9507,9515,9519,9523,9526,9531,9541,9553,9561,9566,9574,9580,9586,10013,10036,10089,10142,10150,10155,10157,10163,10179,10199,10233,10257,10278,10294,10317,10554,10562,10573,10582,10600,10768,10850,11666,11711,11718,11722,11725,11732,11736"}}
+{"timestamp":1714314928.937789,"name":"online","context":{"idset":"52,57,575,9012,9140,9463,9466,9478,9484,9493,9497,9517,9537,9540,9550,9555,9573,9582,9584-9585,9588,9973,10074,10149,10174,10188,10228,10543,11641"}}
+{"timestamp":1714314929.0409384,"name":"online","context":{"idset":"1,6629,9467,9479,9487,9503-9504,9518,9538,9546,9581,10012,10027,10181,10271,10283,10495,10525,11857"}}
+{"timestamp":1714314929.1589453,"name":"online","context":{"idset":"9461-9462,9473,9485-9486,9510,9520,9527,9539,9547,9549,9557,9579,9587,9976,10010,10077,10202,10212,10229,10244,10501,10553,10611,10763,11731"}}
+{"timestamp":1714314929.3047028,"name":"online","context":{"idset":"9474-9475,9505,9552,9906,9924,10019,10035,10072,10137,10182-10183,10210,10225,10248,10256,10262,10267,10279,10307,10315,10318,10558,10596,10609"}}
+{"timestamp":1714314929.4128895,"name":"online","context":{"idset":"9477,9488,9490,9496,9513,9534,9558,9563,9577-9578,9921,9938,9975,10005,10146,10213,10224,10235-10236,10245,10259,10261,10263,10285,10290,10298,10304,10308,10496,10502,10514,10516,10536,10542,10585,10587,10597,10604"}}
+{"timestamp":1714314929.516727,"name":"online","context":{"idset":"9476,9480,9514,9559,9939,9985,10046-10047,10080,10136,10180,10191,10206,10214,10223,10231,10241,10243,10275,10299,10313,10491,10500,10503,10515,10517,10521,10531,10539,10569"}}
+{"timestamp":1714314929.6339285,"name":"online","context":{"idset":"9491,9495,9532-9533,9544,9551,9571-9572,9575,9920,9969,9971,9978,9989,10139,10175,10185,10189,10195-10197,10205,10209,10219-10220,10246,10270,10280,10284,10291,10305,10494,10497,10506,10510,10513,10551-10552,10559,10563,10567,10570,10575,10578,10581,10586,10590,10592,10605"}}
+{"timestamp":1714314929.8448558,"name":"online","context":{"idset":"9470,9536,9954,10143,10145,10147,10160-10161,10164,10170,10173,10176,10186,10258,10264,10300,10537,10612"}}
+{"timestamp":1714314930.0365102,"name":"online","context":{"idset":"10156,10211,10215,10268,10286,10301,10499,10527"}}
+{"timestamp":1714314930.1431963,"name":"online","context":{"idset":"10187,10269,10524,10544"}}
+{"timestamp":1714314930.2593093,"name":"online","context":{"idset":"10159,10190,10198,10207,10234,10297,10319,10511,10528,10574"}}
+{"timestamp":1714314930.6511958,"name":"online","context":{"idset":"26"}}
+{"timestamp":1714314940.95649,"name":"online","context":{"idset":"9323"}}
+{"timestamp":1714314941.9699886,"name":"online","context":{"idset":"10842"}}
+{"timestamp":1714315104.2416561,"name":"online","context":{"idset":"574"}}
+{"timestamp":1714315217.1163237,"name":"online","context":{"idset":"9064"}}
+{"timestamp":1714315217.3534865,"name":"online","context":{"idset":"9061-9063,9065,9067"}}
+{"timestamp":1714315218.6702025,"name":"online","context":{"idset":"9066"}}
+{"timestamp":1714325292.6904995,"name":"online","context":{"idset":"6630"}}
+{"timestamp":1714326519.958544,"name":"offline","context":{"idset":"6630"}}
+{"timestamp":1714326521.8005054,"name":"offline","context":{"idset":"6629"}}
+{"timestamp":1714326554.0831625,"name":"offline","context":{"idset":"878"}}
+{"timestamp":1714326554.2169788,"name":"offline","context":{"idset":"877"}}
+{"timestamp":1714329635.1607337,"name":"offline","context":{"idset":"11542"}}
+{"timestamp":1714329635.1631134,"name":"offline","context":{"idset":"11543"}}
+{"timestamp":1714329635.1654956,"name":"offline","context":{"idset":"11544"}}
+{"timestamp":1714329636.7675393,"name":"offline","context":{"idset":"11545"}}
+{"timestamp":1714329636.8152733,"name":"offline","context":{"idset":"11546"}}
+{"timestamp":1714329636.83025,"name":"offline","context":{"idset":"11547"}}
+{"timestamp":1714329636.8339236,"name":"offline","context":{"idset":"11549"}}
+{"timestamp":1714329636.8954132,"name":"offline","context":{"idset":"11550"}}
+{"timestamp":1714329639.1629438,"name":"offline","context":{"idset":"11551"}}
+{"timestamp":1714329639.1680393,"name":"offline","context":{"idset":"11552"}}
+{"timestamp":1714329640.0854673,"name":"offline","context":{"idset":"11553"}}
+{"timestamp":1714329641.1595592,"name":"offline","context":{"idset":"11557"}}
+{"timestamp":1714329641.1631579,"name":"offline","context":{"idset":"11558"}}
+{"timestamp":1714329641.899153,"name":"offline","context":{"idset":"11559"}}
+{"timestamp":1714329645.1631234,"name":"offline","context":{"idset":"11567"}}
+{"timestamp":1714329645.9609158,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1714329646.6893921,"name":"offline","context":{"idset":"11571"}}
+{"timestamp":1714329646.7031941,"name":"offline","context":{"idset":"11572"}}
+{"timestamp":1714330105.1672025,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1714330105.1713948,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1714330105.1749349,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1714330107.0311272,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1714330107.1554723,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1714330440.9208469,"name":"undrain","context":{"idset":"11509-11534,11536-11540,11573-11578,11580-11582,11584-11593,11595-11607,11609-11617"}}
+{"timestamp":1714331314.8028483,"name":"online","context":{"idset":"11619,11624,11627"}}
+{"timestamp":1714331315.0236716,"name":"online","context":{"idset":"11620,11628,11633"}}
+{"timestamp":1714331316.5098212,"name":"online","context":{"idset":"11622,11630"}}
+{"timestamp":1714331316.5133564,"name":"online","context":{"idset":"11618,11621,11631"}}
+{"timestamp":1714331316.5169103,"name":"online","context":{"idset":"11626,11629,11632"}}
+{"timestamp":1714331316.5202622,"name":"online","context":{"idset":"11623"}}
+{"timestamp":1714331316.5236063,"name":"online","context":{"idset":"11625"}}
+{"timestamp":1714331326.4826252,"name":"online","context":{"idset":"11634"}}
+{"timestamp":1714331326.8042803,"name":"online","context":{"idset":"11635"}}
+{"timestamp":1714331327.0195148,"name":"online","context":{"idset":"11636"}}
+{"timestamp":1714331772.8324783,"name":"undrain","context":{"idset":"11535,11579,11583,11594,11608"}}
+{"timestamp":1714332259.7658904,"name":"online","context":{"idset":"11583,11608"}}
+{"timestamp":1714332260.0188873,"name":"online","context":{"idset":"11535"}}
+{"timestamp":1714332260.2073896,"name":"online","context":{"idset":"11579,11594"}}
+{"timestamp":1714332798.0643666,"name":"drain","context":{"idset":"11541,11548,11554-11556,11560-11566,11568-11569","overwrite":0}}
+{"timestamp":1714336313.9142127,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1714337075.897222,"name":"drain","context":{"idset":"11509-11524","reason":"prolog failed for jobid fsBpSa9CG6j","overwrite":0}}
+{"timestamp":1714337092.8559437,"name":"drain","context":{"idset":"11525-11534,11536-11540","reason":"prolog failed for jobid fsBpZvdTf4B","overwrite":0}}
+{"timestamp":1714337110.6731331,"name":"drain","context":{"idset":"11573-11578,11580-11582,11584-11588","reason":"prolog failed for jobid fsBpi2pssuH","overwrite":0}}
+{"timestamp":1714337144.5488458,"name":"drain","context":{"idset":"11589-11593,11595-11604","reason":"prolog failed for jobid fsBpxmtyevT","overwrite":0}}
+{"timestamp":1714337167.9748862,"name":"drain","context":{"idset":"11605-11607,11609-11617","reason":"prolog failed for jobid fsBq92aPaAs","overwrite":0}}
+{"timestamp":1714338511.1185954,"name":"offline","context":{"idset":"11535"}}
+{"timestamp":1714340368.0309973,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1714340873.0992308,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1714340873.9020011,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1714340873.9054165,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1714340873.9087913,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1714340873.9121788,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1714340873.9156685,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1714340873.9190452,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1714340873.9223828,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1714340873.925736,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1714340873.9291809,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1714340873.9328599,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1714340873.9364772,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1714340873.9402065,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1714340873.9438431,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1714340873.9474368,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1714340873.9510891,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1714340873.9550493,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1714340873.9586525,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1714340873.9622035,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1714340873.9656477,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1714340873.9691536,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1714340873.9725542,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1714340873.9759417,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1714340873.9794054,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1714340873.9829102,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1714340873.9864416,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1714340873.9900005,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1714340873.9934487,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1714340873.9968312,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1714340874.0002968,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1714340874.0037336,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1714340874.0073218,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1714340874.0107424,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1714340874.0143204,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1714340874.0178285,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1714340874.0214767,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1714340874.0249839,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1714340874.028398,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1714340874.0318773,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1714340874.0353374,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1714340874.0387692,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1714340874.0422323,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1714340874.045696,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1714340874.0491786,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1714340874.0526056,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1714340874.0561419,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1714340874.0595005,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1714340874.0628457,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1714340874.0662253,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1714340874.0696476,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1714340874.0730777,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1714340874.0764627,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1714340874.0798628,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1714340874.0833375,"name":"offline","context":{"idset":"11439"}}
+{"timestamp":1714340874.2213442,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1714340874.5801907,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1714340874.583513,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1714340874.5869083,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1714340874.5902846,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1714340874.602361,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1714340874.6058083,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1714340874.6091762,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1714340874.6125655,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1714340874.61601,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1714340874.6194618,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1714340874.6229603,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1714340874.6264386,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1714340874.6299148,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1714340874.6334407,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1714340874.6368966,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1714340874.6403644,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1714340874.6438377,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1714340874.6473279,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1714340874.6507623,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1714340874.6542165,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1714340874.6576855,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1714340874.6610503,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1714340874.6644568,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1714340874.6678998,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1714340874.6712532,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1714340874.6746485,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1714340874.6780059,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1714340874.6814804,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1714340874.6849446,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1714340874.6884425,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1714340874.6917942,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1714340874.6951444,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1714340874.6989336,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1714340874.7023175,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1714340874.7056682,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1714340874.7091603,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1714340874.7126322,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1714340874.7160707,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1714340874.7196453,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1714340874.7234919,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1714340874.7269104,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1714340874.7304637,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1714340874.7339444,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1714340874.7374592,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1714340874.7410016,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1714340874.7444465,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1714340874.7477822,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1714340874.7511992,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1714340874.7546067,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1714340874.7581565,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1714340874.7643857,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1714340874.7711823,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1714340874.775207,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1714340874.7785156,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1714340874.7836339,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1714340874.789813,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1714340874.7962246,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1714340874.8023062,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1714340874.8068707,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1714340874.810163,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1714340874.8151217,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1714340874.8217673,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1714340874.8284924,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1714340874.832901,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1714340874.8368742,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1714340874.8411014,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1714340874.8444114,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1714340874.8478816,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1714340874.8512464,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1714340874.8547034,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1714340874.8581471,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1714340874.8614366,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1714340874.8650191,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1714341392.8433192,"name":"online","context":{"idset":"11391"}}
+{"timestamp":1714341392.95928,"name":"online","context":{"idset":"11387,11436"}}
+{"timestamp":1714341393.0839283,"name":"online","context":{"idset":"11400"}}
+{"timestamp":1714341393.6661496,"name":"online","context":{"idset":"11385,11392,11398"}}
+{"timestamp":1714341393.6690078,"name":"online","context":{"idset":"11383,11409"}}
+{"timestamp":1714341393.6712101,"name":"online","context":{"idset":"11381"}}
+{"timestamp":1714341393.6734767,"name":"online","context":{"idset":"11419"}}
+{"timestamp":1714341393.8062639,"name":"online","context":{"idset":"11395"}}
+{"timestamp":1714341393.9294486,"name":"online","context":{"idset":"11382,11408"}}
+{"timestamp":1714341394.1784422,"name":"online","context":{"idset":"11396,11430"}}
+{"timestamp":1714341394.3882973,"name":"online","context":{"idset":"11389,11404,11417,11442"}}
+{"timestamp":1714341394.6536005,"name":"online","context":{"idset":"11399,11411"}}
+{"timestamp":1714341394.6784661,"name":"online","context":{"idset":"11401"}}
+{"timestamp":1714341394.8218496,"name":"online","context":{"idset":"11416"}}
+{"timestamp":1714341394.9371316,"name":"online","context":{"idset":"11422"}}
+{"timestamp":1714341395.111953,"name":"online","context":{"idset":"11425"}}
+{"timestamp":1714341396.0022864,"name":"online","context":{"idset":"11402"}}
+{"timestamp":1714341396.0062852,"name":"online","context":{"idset":"11446"}}
+{"timestamp":1714341396.0107193,"name":"online","context":{"idset":"11407"}}
+{"timestamp":1714341396.014818,"name":"online","context":{"idset":"11410"}}
+{"timestamp":1714341396.0184517,"name":"online","context":{"idset":"11426"}}
+{"timestamp":1714341396.0221064,"name":"online","context":{"idset":"11439"}}
+{"timestamp":1714341396.0259831,"name":"online","context":{"idset":"11397"}}
+{"timestamp":1714341396.1348665,"name":"online","context":{"idset":"11421,11470"}}
+{"timestamp":1714341396.3244257,"name":"online","context":{"idset":"11448"}}
+{"timestamp":1714341396.6335826,"name":"online","context":{"idset":"11413,11427,11431,11443,11461,11467,11484"}}
+{"timestamp":1714341396.8630042,"name":"online","context":{"idset":"11386,11412,11445,11451,11455,11485,11491"}}
+{"timestamp":1714341396.9826543,"name":"online","context":{"idset":"11423,11432,11434,11452,11475,11488"}}
+{"timestamp":1714341397.9861121,"name":"online","context":{"idset":"11440,11450,11453,11460,11465,11487,11489,11496,11502"}}
+{"timestamp":1714341397.9905961,"name":"online","context":{"idset":"11406,11415,11420,11435,11501"}}
+{"timestamp":1714341397.9943755,"name":"online","context":{"idset":"11464,11472,11478,11482,11495,11497,11508"}}
+{"timestamp":1714341397.9978271,"name":"online","context":{"idset":"11390,11424,11437,11454,11459,11473,11494,11498"}}
+{"timestamp":1714341398.0013843,"name":"online","context":{"idset":"11393,11414,11447,11462,11481"}}
+{"timestamp":1714341398.0049686,"name":"online","context":{"idset":"11456,11486,11500,11505,11507"}}
+{"timestamp":1714341398.008677,"name":"online","context":{"idset":"11469,11479,11490"}}
+{"timestamp":1714341398.102777,"name":"online","context":{"idset":"11441,11457,11466"}}
+{"timestamp":1714341398.3868423,"name":"online","context":{"idset":"11483"}}
+{"timestamp":1714341398.6020119,"name":"online","context":{"idset":"11418"}}
+{"timestamp":1714341473.8959239,"name":"online","context":{"idset":"11384"}}
+{"timestamp":1714341476.3980436,"name":"online","context":{"idset":"11433"}}
+{"timestamp":1714341476.4046626,"name":"online","context":{"idset":"11428"}}
+{"timestamp":1714341476.4112723,"name":"online","context":{"idset":"11405"}}
+{"timestamp":1714341477.9517198,"name":"online","context":{"idset":"11468,11480,11492"}}
+{"timestamp":1714341477.9555867,"name":"online","context":{"idset":"11463,11493"}}
+{"timestamp":1714341478.2121701,"name":"online","context":{"idset":"11476"}}
+{"timestamp":1714341478.4961851,"name":"online","context":{"idset":"11403,11429"}}
+{"timestamp":1714341478.7326448,"name":"online","context":{"idset":"11458,11499"}}
+{"timestamp":1714341479.8944778,"name":"online","context":{"idset":"11471"}}
+{"timestamp":1714341479.8986325,"name":"online","context":{"idset":"11506"}}
+{"timestamp":1714341556.8095293,"name":"online","context":{"idset":"11388,11394"}}
+{"timestamp":1714341557.8092611,"name":"online","context":{"idset":"11449"}}
+{"timestamp":1714341558.7943387,"name":"online","context":{"idset":"11444"}}
+{"timestamp":1714341558.9100187,"name":"online","context":{"idset":"11503"}}
+{"timestamp":1714341560.0156803,"name":"online","context":{"idset":"11477"}}
+{"timestamp":1714341560.0199497,"name":"online","context":{"idset":"11504"}}
+{"timestamp":1714341640.5855632,"name":"online","context":{"idset":"11474"}}
+{"timestamp":1714341641.7609055,"name":"online","context":{"idset":"11438"}}
+{"timestamp":1714341796.9801283,"name":"undrain","context":{"idset":"11381-11508"}}
+{"timestamp":1714342105.9252441,"name":"drain","context":{"idset":"11381-11508","reason":"prolog failed for jobid fsCUf53o9qq","overwrite":0}}
+{"timestamp":1714342200.3989279,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1714342200.5119023,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1714342200.5173218,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1714342200.5269904,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1714342200.5446048,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1714342200.6440067,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1714342200.7665884,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1714342200.7709239,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1714342200.7791331,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1714342200.7844994,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1714342200.7917345,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1714342200.797287,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1714342200.8076603,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1714342200.8133223,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1714342200.8553951,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1714342200.8619428,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1714342200.8679399,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1714342200.8735738,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1714342200.8906002,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1714342200.896498,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1714342200.9009056,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1714342200.9044926,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1714342200.9080863,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1714342201.0026755,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1714342201.124999,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1714342201.1375494,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1714342201.3373113,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1714342201.3409512,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1714342201.3445973,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1714342201.3481653,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1714342201.3520422,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1714342201.3830595,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1714342201.3873651,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1714342201.394628,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1714342201.417501,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1714342201.4231839,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1714342201.4285192,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1714342201.4334588,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1714342201.4384332,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1714342201.4435115,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1714342201.448684,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1714342201.4550254,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1714342201.4618652,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1714342201.4683015,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1714342201.4746895,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1714342201.4804423,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1714342201.4850864,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1714342201.4890664,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1714342202.2400646,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1714342202.2433584,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1714342202.2466364,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1714342202.250145,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1714342202.2534425,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1714342202.2567217,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1714342202.2603002,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1714342202.2635927,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1714342202.2668839,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1714342202.2701464,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1714342202.2750371,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1714342202.2961009,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1714342202.309207,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1714342202.312484,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1714342202.3157544,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1714342202.319005,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1714342202.3222628,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1714342202.3255119,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1714342202.3288164,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1714342202.3320463,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1714342202.3352501,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1714342202.3384588,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1714342202.3416529,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1714342202.3448334,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1714342202.3480227,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1714342202.3512075,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1714342202.3543806,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1714342202.3575144,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1714342202.3606274,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1714342202.3720765,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1714342202.3915365,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1714342202.5320408,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1714342202.5347602,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1714342202.5374665,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1714342202.5401552,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1714342202.54284,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1714342202.5455163,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1714342202.5482025,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1714342202.5508718,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1714342202.5535104,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1714342202.5561383,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1714342202.558764,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1714342202.561394,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1714342202.5640228,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1714342202.566642,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1714342202.5692768,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1714342202.5718913,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1714342202.5744863,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1714342202.577076,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1714342202.5796704,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1714342202.5822468,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1714342202.584847,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1714342202.5874119,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1714342202.589988,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1714342202.5925486,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1714342202.5951014,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1714342202.5976503,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1714342202.6002483,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1714342202.6027873,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1714342202.6053169,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1714342202.6078372,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1714342202.6103427,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1714342202.6128263,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1714342202.6153128,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1714342202.6178157,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1714342202.6203158,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1714342202.6228094,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1714342202.6252899,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1714342202.6277728,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1714342202.6302357,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1714342202.6326947,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1714342202.6351559,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1714342202.6376238,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1714342202.6400738,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1714342202.6425023,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1714342202.6449404,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1714342202.6473691,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1714342202.6497841,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1714342202.6521842,"name":"offline","context":{"idset":"11439"}}
+{"timestamp":1714342202.654599,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1714345386.8103044,"name":"online","context":{"idset":"11387"}}
+{"timestamp":1714345386.9220014,"name":"online","context":{"idset":"11383,11391"}}
+{"timestamp":1714345387.0213532,"name":"online","context":{"idset":"11381"}}
+{"timestamp":1714345387.1155431,"name":"online","context":{"idset":"11398"}}
+{"timestamp":1714345387.210269,"name":"online","context":{"idset":"11401,11405"}}
+{"timestamp":1714345387.312062,"name":"online","context":{"idset":"11395"}}
+{"timestamp":1714345387.4157331,"name":"online","context":{"idset":"11384,11410"}}
+{"timestamp":1714345387.5041029,"name":"online","context":{"idset":"11392"}}
+{"timestamp":1714345387.7399249,"name":"online","context":{"idset":"11409,11431"}}
+{"timestamp":1714345388.4780638,"name":"online","context":{"idset":"11425"}}
+{"timestamp":1714345388.4813569,"name":"online","context":{"idset":"11390,11432"}}
+{"timestamp":1714345388.4847164,"name":"online","context":{"idset":"11417"}}
+{"timestamp":1714345388.4879861,"name":"online","context":{"idset":"11415"}}
+{"timestamp":1714345388.4914868,"name":"online","context":{"idset":"11397"}}
+{"timestamp":1714345388.49512,"name":"online","context":{"idset":"11386"}}
+{"timestamp":1714345388.4983704,"name":"online","context":{"idset":"11389"}}
+{"timestamp":1714345388.5019197,"name":"online","context":{"idset":"11403"}}
+{"timestamp":1714345388.6622596,"name":"online","context":{"idset":"11388,11406,11426"}}
+{"timestamp":1714345388.942822,"name":"online","context":{"idset":"11382,11402,11404,11419"}}
+{"timestamp":1714345389.0523238,"name":"online","context":{"idset":"11399,11408,11420-11421"}}
+{"timestamp":1714345389.1687639,"name":"online","context":{"idset":"11416"}}
+{"timestamp":1714345389.2879529,"name":"online","context":{"idset":"11385,11418"}}
+{"timestamp":1714345389.4801934,"name":"online","context":{"idset":"11393-11394,11396,11407,11414,11423,11427,11442,11444"}}
+{"timestamp":1714345389.6931336,"name":"online","context":{"idset":"11428-11429,11433,11435-11436,11441,11446-11447,11449,11451,11456,11459,11474,11487,11504"}}
+{"timestamp":1714345389.8092084,"name":"online","context":{"idset":"11422"}}
+{"timestamp":1714345390.5037045,"name":"online","context":{"idset":"11400,11412,11424,11430,11440,11445,11453,11458,11460,11463,11473,11485,11497-11498"}}
+{"timestamp":1714345390.508249,"name":"online","context":{"idset":"11411,11413,11438,11443,11452,11455,11457,11461-11462,11464-11465,11467,11469,11476-11478,11481,11486,11488-11489,11491,11493-11494,11496,11501,11507"}}
+{"timestamp":1714345390.5117955,"name":"online","context":{"idset":"11434,11437,11466,11484,11506"}}
+{"timestamp":1714345390.5719719,"name":"online","context":{"idset":"11470,11472,11495,11502-11503,11505"}}
+{"timestamp":1714345390.7693174,"name":"online","context":{"idset":"11439,11448,11450,11454,11468,11480,11482,11490,11499-11500,11508"}}
+{"timestamp":1714345391.0681787,"name":"online","context":{"idset":"11471,11475,11479,11483"}}
+{"timestamp":1714345391.2633455,"name":"online","context":{"idset":"11492"}}
+{"timestamp":1714345851.2604494,"name":"undrain","context":{"idset":"11381-11508"}}
+{"timestamp":1714345895.4912086,"name":"drain","context":{"idset":"11381-11508","reason":"prolog failed for jobid fsCyTLAsish","overwrite":0}}
+{"timestamp":1714346045.1880486,"name":"undrain","context":{"idset":"11381-11508"}}
+{"timestamp":1714346197.0652089,"name":"drain","context":{"idset":"11381-11508","reason":"prolog failed for jobid fsD1iQyAgXq","overwrite":0}}
+{"timestamp":1714348338.4963975,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1714353864.486639,"name":"offline","context":{"idset":"9403"}}
+{"timestamp":1714358992.0038564,"name":"resource-init","context":{"restart":true,"drain":{"11367":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrctdZhtSj"},"11541,11548,11554-11556,11560-11566,11568-11569":{"timestamp":1714332798.0,"reason":""},"11370":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcu9RKRbV"},"11283":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrce9w4ddq"},"11312":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcj9o8DLB"},"11336":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcoFo6vEs"},"411":{"timestamp":1713564989.0,"reason":"Unreachable"},"11346":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcpz86KLK"},"11307":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrciF7bjXV"},"336,491":{"timestamp":1713230524.0,"reason":"Down"},"11309":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcictjBdH"},"11278":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcd9M51jV"},"11343":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcpTzAv5m"},"11318":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrck9TkH67"},"11379":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcvxbzUET"},"11261":{"timestamp":1714239116.0,"reason":"prolog failed for jobid frrcZsWCykT"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"11266":{"timestamp":1714239115.0,"reason":"prolog failed for jobid frrcakDaQrb"},"98,100,102,111-112,114,123-124,161,184,187,190,208,303-304,310,317,319,330":{"timestamp":1714079956.0,"reason":"epilog failed for jobid frcnKxdu6xP"},"11315":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcjgAPW7q"},"11272":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcc7WS12F"},"11335":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrco7ET4qM"},"11270":{"timestamp":1714239116.0,"reason":"prolog failed for jobid frrcbiqQz4P"},"11319":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrckK9dato"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"881-882":{"timestamp":1714153692.0,"reason":"--reason H/W troubleshoot"},"11378":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcvnB7XFm"},"11326":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcmZceUpP"},"11286":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrceozaCFH"},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"349-359,361-371,374-396,398-404":{"timestamp":1714060395.0,"reason":"New Blade Installation --JRG"},"77-84":{"timestamp":1713795613.0,"reason":"New blade installation"},"11275":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcccvqkP9"},"549-556":{"timestamp":1713553592.0,"reason":"New Blade Installation"},"11313":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcjKawUGF"},"11380":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcw9HhJ2b"},"11368":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrctoniwAf"},"829-830":{"timestamp":1712700945.0,"reason":""},"543":{"timestamp":1714074108.0,"reason":"nodediag failed cxi"},"939":{"timestamp":1713963885.0,"reason":"nodediag failed amdapu"},"771-772":{"timestamp":1713195979.0,"reason":""},"141-148":{"timestamp":1713545597.0,"reason":"ARP Storm Problem testing"},"9054":{"timestamp":1712864116.0,"reason":"broker was unresponsive"},"871-874":{"timestamp":1714057817.0,"reason":""},"461-476":{"timestamp":1713794797.0,"reason":"New Blade Installation --JRG"},"410":{"timestamp":1713378635.0,"reason":"prolog failed for jobid fq5jtQQSQnT"},"541":{"timestamp":1714074106.0,"reason":"nodediag failed cxi"},"7285-7412":{"timestamp":1714248995.0,"reason":"prolog failed for jobid frzMuSLMNAK"},"11277":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrccyNsqq1"},"482-490,492":{"timestamp":1713915346.0,"reason":"New Blade Installation --JRG"},"11350":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcqeMynvB"},"11324":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcmDog5R9"},"11356":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcreidXHR"},"11254":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcYiP3Viw"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"11256":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcZ3VzEWs"},"191":{"timestamp":1714078824.0,"reason":"unreachable"},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"11321":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrckf61vhm"},"11302":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrchQ1dXB5"},"11338":{"timestamp":1714239648.0,"reason":"prolog failed for jobid frrcodExXgB"},"11348":{"timestamp":1714241556.0,"reason":"prolog failed for jobid frrcqKVrvwh"},"11271":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcbxPARR9"},"11295":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcgDqgYbR"},"11362":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcsjdB3p3"},"11341":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcp7zpbi7"},"11255":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcYt1xpxw"},"11332":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcnbCTciB"},"548":{"timestamp":1713563915.0,"reason":"epilog failed for jobid fqUWkS8j8hM"},"11542-11547,11549-11553,11557-11559,11567,11570-11572":{"timestamp":1714249243.0,"reason":"prolog failed for jobid frzRkVsE1Rh"},"825-826":{"timestamp":1714153531.0,"reason":"--reason H/W troubleshoot"},"11288":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcf9vxY4F"},"9003":{"timestamp":1712873964.0,"reason":"nodediag failed dgemm_perf"},"115":{"timestamp":1714079493.0,"reason":"unreachable"},"525-532":{"timestamp":1713290620.0,"reason":""},"11282":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcdqXdYjZ"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"8159,8278-8281":{"timestamp":1714249943.0,"reason":"prolog failed for jobid frzX3xjjC2j"},"11325":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcmQ4B7RR"},"877":{"timestamp":1714078512.0,"reason":"nodediag failed amdapu"},"11305":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrchtqwZ43"},"11257":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcZDKHUjH"},"11298":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcgieWbC3"},"11267":{"timestamp":1714239114.0,"reason":"prolog failed for jobid frrcaz85gAB"},"172":{"timestamp":1713981544.0,"reason":"Unreachable"},"11293":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcfsxGBM9"},"8107-8108":{"timestamp":1714249949.0,"reason":"prolog failed for jobid frzX7r2t3t3"},"11509-11524":{"timestamp":1714337076.0,"reason":"prolog failed for jobid fsBpSa9CG6j"},"8923-8963,8973-8981,8983-8992,8995-8998,9001-9002,9005-9017,9019-9027":{"timestamp":1714249928.0,"reason":"prolog failed for jobid frzWyRvCWVD"},"796":{"timestamp":1712868054.0,"reason":"broker was unresponsive"},"421-460":{"timestamp":1713794410.0,"reason":"New Blade Installation --JRG"},"797-798":{"timestamp":1712261863.0,"reason":"--reason swapping blades into x1900 - wendy"},"11276":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrccoQgfvX"},"11297":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcgYw9J71"},"11337":{"timestamp":1714239606.0,"reason":"prolog failed for jobid frrcoS27xxo"},"977-978":{"timestamp":1714144603.0,"reason":""},"11262":{"timestamp":1714239114.0,"reason":"prolog failed for jobid frrca31iNaj"},"11310":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcioM67tK"},"11605-11607,11609-11617":{"timestamp":1714337168.0,"reason":"prolog failed for jobid fsBq92aPaAs"},"8282-8290,8292-8303,8305-8308,8821-8832,8834-8917,8919-8922":{"timestamp":1714249934.0,"reason":"prolog failed for jobid frzX17LBKTM"},"11340":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcoxmd4yd"},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"8181,8183,8234,8237,8291,8304":{"timestamp":1713810085.0,"reason":"testing -kk"},"119":{"timestamp":1714079565.0,"reason":"unreachable"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"11299":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcguqsAd5"},"961-962":{"timestamp":1713197674.0,"reason":""},"206":{"timestamp":1713564985.0,"reason":"Unreachable"},"545":{"timestamp":1714069878.0,"reason":"nodediag failed cxi"},"11358":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcs2rWoL3"},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"925-926":{"timestamp":1713540640.0,"reason":""},"11311":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrciymy4s1"},"8833":{"timestamp":1713991869.0,"reason":"nodediag failed amdapu"},"11287":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrceynPTBM"},"11317":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcjzD4m5H"},"110":{"timestamp":1714079561.0,"reason":"unreachable"},"546":{"timestamp":1714069873.0,"reason":"nodediag failed cxi pci"},"11304":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrchjPQ8nT"},"11372":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcuW5DRJ7"},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"11281":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcdftiDVZ"},"11253":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcYXbQioR"},"789-794":{"timestamp":1711461777.0,"reason":""},"11359":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcsEJhFaX"},"11329":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcn6HhczB"},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"360":{"timestamp":1713981579.0,"reason":"New Blade Installation --JRG"},"11330":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcnG2YuMZ"},"11308":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrciR2pvsH"},"11269":{"timestamp":1714239115.0,"reason":"prolog failed for jobid frrcbYzdkZd"},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"286":{"timestamp":1713981553.0,"reason":"Unreachable"},"963-964":{"timestamp":1713196418.0,"reason":"moving blade"},"542":{"timestamp":1714074109.0,"reason":"nodediag failed cxi"},"11203-11252":{"timestamp":1714244322.0,"reason":""},"11316":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcjpoVKNP"},"11289":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcfJyn9pK"},"11525-11534,11536-11540":{"timestamp":1714337093.0,"reason":"prolog failed for jobid fsBpZvdTf4B"},"11349":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcqUvvMvw"},"342":{"timestamp":1713981566.0,"reason":"Unreachable"},"11296":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcgPMBwRh"},"11279":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcdLZuaSs"},"11369":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrctywJ23Z"},"11320":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrckVFEhD1"},"420":{"timestamp":1714074112.0,"reason":"epilog failed for jobid frbYvBLD1gT"},"11371":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcuKBehFD"},"117":{"timestamp":1713981538.0,"reason":"Unreachable"},"11589-11593,11595-11604":{"timestamp":1714337145.0,"reason":"prolog failed for jobid fsBpxmtyevT"},"11366":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrctShd9gB"},"11342":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcpJabUNs"},"11306":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrci4jgm7V"},"11377":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcvaxGxYP"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"11303":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrchZDM4dD"},"11363":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcsuG6P43"},"11263":{"timestamp":1714239114.0,"reason":"prolog failed for jobid frrcaCYhkhM"},"878":{"timestamp":1714078484.0,"reason":"nodediag failed amdapu"},"11314":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcjWDgKVh"},"11273":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrccGzTQaB"},"11344":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcpdj2CT9"},"11347":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcq9oyd91"},"11339":{"timestamp":1714239648.0,"reason":"prolog failed for jobid frrconxKpmD"},"779-780":{"timestamp":1713907642.0,"reason":"--reason Troubleshoot BIOS issue"},"934":{"timestamp":1713967835.0,"reason":"epilog failed for jobid frMYzSv7G6b"},"11290":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcfRkesVh"},"11285":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcedHPPAo"},"929-930":{"timestamp":1713552018.0,"reason":""},"11284":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrceKyhmPM"},"11265":{"timestamp":1714239115.0,"reason":"prolog failed for jobid frrcaZgmWkX"},"11333":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcnm4iqVH"},"11331":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcnRmQBiw"},"165":{"timestamp":1713981434.0,"reason":"broker was unresponsive"},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"11357":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcrqdgEcf"},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"120":{"timestamp":1713981548.0,"reason":"Unreachable"},"923-924":{"timestamp":1714064541.0,"reason":"status"},"11375":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcvBr49nf"},"808":{"timestamp":1713455939.0,"reason":""},"11260":{"timestamp":1714239114.0,"reason":"prolog failed for jobid frrcZhqofE7"},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"11352":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcqxzkmMZ"},"11353":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcr8QLD4T"},"11301":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrchEXc7d9"},"149-156":{"timestamp":1713545634.0,"reason":"ARP Storm Problem testing"},"11345":{"timestamp":1714239649.0,"reason":"prolog failed for jobid frrcpovZFtj"},"11258":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcZP48m6f"},"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"11381-11508":{"timestamp":1714346197.0,"reason":"prolog failed for jobid fsD1iQyAgXq"},"11361":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcsa7feym"},"11274":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrccT1cZ3M"},"11376":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcvQ1kFvo"},"133-140":{"timestamp":1713545617.0,"reason":"ARP Storm Problem testing"},"90":{"timestamp":1713564981.0,"reason":"New Blade installation"},"807":{"timestamp":1713376017.0,"reason":""},"11365":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrctGXa5Ww"},"544":{"timestamp":1714069881.0,"reason":"nodediag failed cxi"},"11351":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcqoZhLNK"},"397":{"timestamp":1713229804.0,"reason":"New Blade Installation --JRG"},"787-788":{"timestamp":1713219302.0,"reason":""},"11291":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcfZJ1DdR"},"11364":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrct6QUyvP"},"11360":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcsPtecFq"},"11264":{"timestamp":1714239114.0,"reason":"prolog failed for jobid frrcaPfcNHM"},"11334":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcnwBow5q"},"814":{"timestamp":1712783332.0,"reason":""},"937-938":{"timestamp":1712866351.0,"reason":""},"795":{"timestamp":1712868053.0,"reason":"broker was unresponsive"},"803-804":{"timestamp":1713479733.0,"reason":"status"},"8982":{"timestamp":1713996208.0,"reason":"broker was unresponsive"},"11259":{"timestamp":1714238939.0,"reason":"prolog failed for jobid frrcZYBQLhm"},"11322":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrckqD72JK"},"11573-11578,11580-11582,11584-11588":{"timestamp":1714337111.0,"reason":"prolog failed for jobid fsBpi2pssuH"},"11294":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcg43sHfM"},"573-580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"373":{"timestamp":1713981698.0,"reason":"New Blade Installation --JRG"},"11328":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcmvaLKu9"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"372":{"timestamp":1713981581.0,"reason":"New Blade Installation --JRG"},"11292":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcfiUEmoD"},"11300":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrch4zcjWX"},"11354":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcrJ1mZ27"},"11323":{"timestamp":1714239174.0,"reason":"prolog failed for jobid frrcm1zjoDq"},"7730,7739":{"timestamp":1714254796.0,"reason":"epilog failed for jobid frzXAYpp6VD"},"11280":{"timestamp":1714239173.0,"reason":"prolog failed for jobid frrcdWV8mnf"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"},"85-89,91-92":{"timestamp":1714079045.0,"reason":"New Blade installation"},"11268":{"timestamp":1714239115.0,"reason":"prolog failed for jobid frrcbE16xBR"},"11327":{"timestamp":1714239196.0,"reason":"prolog failed for jobid frrcmj3huod"},"11373":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcugwJA4f"},"971-974":{"timestamp":1714143211.0,"reason":""},"335":{"timestamp":1713981557.0,"reason":"Unreachable"},"11355":{"timestamp":1714241586.0,"reason":"prolog failed for jobid frrcrU1SiCw"},"773-774":{"timestamp":1713802805.0,"reason":""},"811-812":{"timestamp":1713557753.0,"reason":"--reason Running hpe diags - JM"},"11374":{"timestamp":1714241910.0,"reason":"prolog failed for jobid frrcuuvRsDq"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"547":{"timestamp":1713563916.0,"reason":"epilog failed for jobid fqUWmieM3AX"},"9075":{"timestamp":1712864130.0,"reason":"broker was unresponsive"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1714358992.0208261,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1714359013.9661305,"name":"online","context":{"idset":"0"}}
+{"timestamp":1714359015.716167,"name":"online","context":{"idset":"7863"}}
+{"timestamp":1714359015.7466552,"name":"online","context":{"idset":"7835"}}
+{"timestamp":1714359015.7613969,"name":"online","context":{"idset":"10401"}}
+{"timestamp":1714359015.8151836,"name":"online","context":{"idset":"8927"}}
+{"timestamp":1714359015.8277194,"name":"online","context":{"idset":"8146,8251,8887,9050,9317,10281"}}
+{"timestamp":1714359015.8299737,"name":"online","context":{"idset":"8089"}}
+{"timestamp":1714359015.8828096,"name":"online","context":{"idset":"8001,9263"}}
+{"timestamp":1714359015.884275,"name":"online","context":{"idset":"1224,1232"}}
+{"timestamp":1714359015.8932581,"name":"online","context":{"idset":"7979,8074,8183,8947,9471,9515,9552-9553,10376,10692,11013,11789"}}
+{"timestamp":1714359015.8941309,"name":"online","context":{"idset":"9128"}}
+{"timestamp":1714359015.8957515,"name":"online","context":{"idset":"8154,9249"}}
+{"timestamp":1714359015.8974905,"name":"online","context":{"idset":"10671"}}
+{"timestamp":1714359015.8999646,"name":"online","context":{"idset":"7814,10434,11059,11601"}}
+{"timestamp":1714359015.9042814,"name":"online","context":{"idset":"11025"}}
+{"timestamp":1714359016.0030479,"name":"online","context":{"idset":"808,884,978,7815,7826,7877,7973,8135,8144,8152,8196,8205,8236,8261,8264,8270,8946,9237,9254,9259,9307,9325,9470,9486,9538,9578,10247,10449,10458,10475,10481,10630,10666,10685,10697,11050,11054,11066,11096,11101,11122,11447,11462,11471,11474,11529,11531,11614,11787,11873,11877"}}
+{"timestamp":1714359016.0034776,"name":"online","context":{"idset":"7922,7932,7934,8034,8178,11874"}}
+{"timestamp":1714359016.0038879,"name":"online","context":{"idset":"9494,11449,11506"}}
+{"timestamp":1714359016.0041633,"name":"online","context":{"idset":"8230,8980"}}
+{"timestamp":1714359016.0109797,"name":"online","context":{"idset":"11401"}}
+{"timestamp":1714359016.0137017,"name":"online","context":{"idset":"7802"}}
+{"timestamp":1714359016.0155323,"name":"online","context":{"idset":"10392,11485"}}
+{"timestamp":1714359016.0171051,"name":"online","context":{"idset":"9156"}}
+{"timestamp":1714359016.0219743,"name":"online","context":{"idset":"10273"}}
+{"timestamp":1714359016.131943,"name":"online","context":{"idset":"8872"}}
+{"timestamp":1714359016.1322322,"name":"online","context":{"idset":"10300"}}
+{"timestamp":1714359016.1352606,"name":"online","context":{"idset":"7944,8037,8095,8208,8890,9171,9773,10286,10299,11102,11113,11404,11609,11839"}}
+{"timestamp":1714359016.1357121,"name":"online","context":{"idset":"8031"}}
+{"timestamp":1714359016.1364353,"name":"online","context":{"idset":"7870,8245"}}
+{"timestamp":1714359016.1367724,"name":"online","context":{"idset":"7879,8831"}}
+{"timestamp":1714359016.1370263,"name":"online","context":{"idset":"9110,11810"}}
+{"timestamp":1714359016.1373069,"name":"online","context":{"idset":"8975,11118"}}
+{"timestamp":1714359016.1378899,"name":"online","context":{"idset":"8024,8127,9493"}}
+{"timestamp":1714359016.1382697,"name":"online","context":{"idset":"11020"}}
+{"timestamp":1714359016.1385813,"name":"online","context":{"idset":"8096,8111,8977,9126"}}
+{"timestamp":1714359016.1389234,"name":"online","context":{"idset":"8948,10272,11055,11788"}}
+{"timestamp":1714359016.1392672,"name":"online","context":{"idset":"7857,8305"}}
+{"timestamp":1714359016.1396227,"name":"online","context":{"idset":"7819"}}
+{"timestamp":1714359016.1400692,"name":"online","context":{"idset":"10326"}}
+{"timestamp":1714359016.1406119,"name":"online","context":{"idset":"9138"}}
+{"timestamp":1714359016.1408758,"name":"online","context":{"idset":"8337,10709"}}
+{"timestamp":1714359016.1416905,"name":"online","context":{"idset":"8051,11411"}}
+{"timestamp":1714359016.1476381,"name":"online","context":{"idset":"11383,11508"}}
+{"timestamp":1714359016.1485789,"name":"online","context":{"idset":"10341,11017,11618"}}
+{"timestamp":1714359016.1895037,"name":"online","context":{"idset":"10439"}}
+{"timestamp":1714359016.3412116,"name":"online","context":{"idset":"9750,11420,11579"}}
+{"timestamp":1714359016.3692973,"name":"online","context":{"idset":"7827,8850,8897,9085,9286,9315,9721,10348,10426,10628,11418,11455,11458,11645,11765,11808"}}
+{"timestamp":1714359016.3725536,"name":"online","context":{"idset":"1223,7972,8045,8913,9105,9181,9316,10041,10080,10236,10353,10680,10735,11064,11517"}}
+{"timestamp":1714359016.3734987,"name":"online","context":{"idset":"10715"}}
+{"timestamp":1714359016.374126,"name":"online","context":{"idset":"8150"}}
+{"timestamp":1714359016.3744867,"name":"online","context":{"idset":"8030,11092"}}
+{"timestamp":1714359016.3748593,"name":"online","context":{"idset":"7911,7960,9016"}}
+{"timestamp":1714359016.375144,"name":"online","context":{"idset":"11814"}}
+{"timestamp":1714359016.3756201,"name":"online","context":{"idset":"9554"}}
+{"timestamp":1714359016.3761973,"name":"online","context":{"idset":"8212"}}
+{"timestamp":1714359016.3765671,"name":"online","context":{"idset":"8002,8131,9511"}}
+{"timestamp":1714359016.376889,"name":"online","context":{"idset":"8308,9829"}}
+{"timestamp":1714359016.3772659,"name":"online","context":{"idset":"8028,8917,8979"}}
+{"timestamp":1714359016.3775716,"name":"online","context":{"idset":"7839"}}
+{"timestamp":1714359016.3779578,"name":"online","context":{"idset":"1211,8961,9523,9795,11028"}}
+{"timestamp":1714359016.3783364,"name":"online","context":{"idset":"9219"}}
+{"timestamp":1714359016.3789086,"name":"online","context":{"idset":"7956,8385,8906,8909,9132,9674,11594"}}
+{"timestamp":1714359016.3792906,"name":"online","context":{"idset":"11510"}}
+{"timestamp":1714359016.3796811,"name":"online","context":{"idset":"9184,9562,9915,10332,10459,10614,10645,11085"}}
+{"timestamp":1714359016.3802602,"name":"online","context":{"idset":"11425"}}
+{"timestamp":1714359016.3808646,"name":"online","context":{"idset":"8263,9185,10237,11441"}}
+{"timestamp":1714359016.381191,"name":"online","context":{"idset":"10496"}}
+{"timestamp":1714359016.3815939,"name":"online","context":{"idset":"10478,10639,11421,11867"}}
+{"timestamp":1714359016.3820279,"name":"online","context":{"idset":"8923,10309,10651"}}
+{"timestamp":1714359016.3824422,"name":"online","context":{"idset":"8371"}}
+{"timestamp":1714359016.3828545,"name":"online","context":{"idset":"10521,11661"}}
+{"timestamp":1714359016.3832464,"name":"online","context":{"idset":"11689"}}
+{"timestamp":1714359016.3837266,"name":"online","context":{"idset":"7907,8163"}}
+{"timestamp":1714359016.3841093,"name":"online","context":{"idset":"11454"}}
+{"timestamp":1714359016.384516,"name":"online","context":{"idset":"11656"}}
+{"timestamp":1714359016.3849301,"name":"online","context":{"idset":"11400"}}
+{"timestamp":1714359016.3853383,"name":"online","context":{"idset":"11775"}}
+{"timestamp":1714359016.3858728,"name":"online","context":{"idset":"11460"}}
+{"timestamp":1714359016.3863606,"name":"online","context":{"idset":"108,118,9097"}}
+{"timestamp":1714359016.386708,"name":"online","context":{"idset":"11472"}}
+{"timestamp":1714359016.3872008,"name":"online","context":{"idset":"10457"}}
+{"timestamp":1714359016.3880587,"name":"online","context":{"idset":"158,181"}}
+{"timestamp":1714359016.3885691,"name":"online","context":{"idset":"111,219"}}
+{"timestamp":1714359016.3890405,"name":"online","context":{"idset":"11018"}}
+{"timestamp":1714359016.3893991,"name":"online","context":{"idset":"10535"}}
+{"timestamp":1714359016.4503801,"name":"online","context":{"idset":"97,186,230,238,242,288,290,318,326,340,408,1210,1230,6581,7821,7874,7889,7959,8029,8065,8070,8102,8142,8151,8195,8217,8278,8296,8332,8944,8958,8993,9024,9108,9135,9183,9223,9290,9309,9339,9613,9658,9874,10025,10055,10087,10233,10249,10292,10344-10345,10347,10351,10367,10526,10608-10609,10648,10656,10662,11069,11081,11086,11088,11093,11105,11111,11403,11443,11515,11522,11574-11575,11589,11602,11688,11723,11771,11806,11813,11847,11854"}}
+{"timestamp":1714359016.8101301,"name":"online","context":{"idset":"95"}}
+{"timestamp":1714359016.8114257,"name":"online","context":{"idset":"164"}}
+{"timestamp":1714359016.8174279,"name":"online","context":{"idset":"319,7936,8651,8950,9202,9217,9233,9296,9338,9427,9547,9585,9630,9774,9790,9894,10007,10046,10256,10329,10580,10670,10683,10999,11003,11115,11385,11405,11422,11452,11496,11600,11648"}}
+{"timestamp":1714359016.8182278,"name":"online","context":{"idset":"11756"}}
+{"timestamp":1714359016.818908,"name":"online","context":{"idset":"11423"}}
+{"timestamp":1714359016.8199408,"name":"online","context":{"idset":"175,8306,10328"}}
+{"timestamp":1714359016.8205802,"name":"online","context":{"idset":"11580"}}
+{"timestamp":1714359016.8212373,"name":"online","context":{"idset":"7910,9391,10239,11859"}}
+{"timestamp":1714359016.8218613,"name":"online","context":{"idset":"9817,10304,11428,11864"}}
+{"timestamp":1714359016.8225045,"name":"online","context":{"idset":"8237,9514,10788,11840"}}
+{"timestamp":1714359016.8236742,"name":"online","context":{"idset":"9144"}}
+{"timestamp":1714359016.824671,"name":"online","context":{"idset":"899,1179,8109,8138,8173,8175,8921,9314,9384,9770,9808,10185,10212,10235,10997,11119"}}
+{"timestamp":1714359016.825367,"name":"online","context":{"idset":"9525,10522,11083"}}
+{"timestamp":1714359016.8261774,"name":"online","context":{"idset":"8003,9071,9204,9262,9479,10014,10618,11389,11467"}}
+{"timestamp":1714359016.8273363,"name":"online","context":{"idset":"7988,8004,9779,10260,10379,11024"}}
+{"timestamp":1714359016.8280447,"name":"online","context":{"idset":"9560,9639,10629"}}
+{"timestamp":1714359016.8288422,"name":"online","context":{"idset":"7894,8344,8924,10077,11047"}}
+{"timestamp":1714359016.8295817,"name":"online","context":{"idset":"8853,9959,11792"}}
+{"timestamp":1714359016.830693,"name":"online","context":{"idset":"225,9115,10479"}}
+{"timestamp":1714359016.8313763,"name":"online","context":{"idset":"124,10086,11057"}}
+{"timestamp":1714359016.8322468,"name":"online","context":{"idset":"96,8281"}}
+{"timestamp":1714359016.8329434,"name":"online","context":{"idset":"8050"}}
+{"timestamp":1714359016.8340743,"name":"online","context":{"idset":"9435,9484,9747"}}
+{"timestamp":1714359016.8348935,"name":"online","context":{"idset":"8100,11608"}}
+{"timestamp":1714359016.8355584,"name":"online","context":{"idset":"8120"}}
+{"timestamp":1714359016.8363295,"name":"online","context":{"idset":"8893,10275"}}
+{"timestamp":1714359016.8375368,"name":"online","context":{"idset":"102,183,9288,10551,10843,11503"}}
+{"timestamp":1714359016.8383029,"name":"online","context":{"idset":"11588,11866"}}
+{"timestamp":1714359016.839,"name":"online","context":{"idset":"10814,11815"}}
+{"timestamp":1714359016.8396871,"name":"online","context":{"idset":"11885"}}
+{"timestamp":1714359016.8403356,"name":"online","context":{"idset":"8113,8933"}}
+{"timestamp":1714359016.8409607,"name":"online","context":{"idset":"11641"}}
+{"timestamp":1714359016.8418641,"name":"online","context":{"idset":"8079,9305,11031"}}
+{"timestamp":1714359016.8425772,"name":"online","context":{"idset":"7875,8323,11477"}}
+{"timestamp":1714359016.843262,"name":"online","context":{"idset":"8384,11451,11453"}}
+{"timestamp":1714359016.8439603,"name":"online","context":{"idset":"9992,10031"}}
+{"timestamp":1714359016.8448279,"name":"online","context":{"idset":"8660"}}
+{"timestamp":1714359016.8458867,"name":"online","context":{"idset":"115"}}
+{"timestamp":1714359016.8467834,"name":"online","context":{"idset":"332,7935,9232,10638"}}
+{"timestamp":1714359016.8477669,"name":"online","context":{"idset":"9573,10119"}}
+{"timestamp":1714359016.8485391,"name":"online","context":{"idset":"1233,8973"}}
+{"timestamp":1714359016.8496635,"name":"online","context":{"idset":"157,10319"}}
+{"timestamp":1714359016.8504722,"name":"online","context":{"idset":"6596,10442"}}
+{"timestamp":1714359016.8513222,"name":"online","context":{"idset":"284,10684,11143"}}
+{"timestamp":1714359016.8521357,"name":"online","context":{"idset":"226"}}
+{"timestamp":1714359016.8531964,"name":"online","context":{"idset":"315,8381"}}
+{"timestamp":1714359016.8556719,"name":"online","context":{"idset":"107,419,9341,10453"}}
+{"timestamp":1714359016.856854,"name":"online","context":{"idset":"9635,11183"}}
+{"timestamp":1714359016.8580372,"name":"online","context":{"idset":"9441"}}
+{"timestamp":1714359016.8603189,"name":"online","context":{"idset":"104,240"}}
+{"timestamp":1714359016.861335,"name":"online","context":{"idset":"220"}}
+{"timestamp":1714359016.862139,"name":"online","context":{"idset":"9540"}}
+{"timestamp":1714359016.8630307,"name":"online","context":{"idset":"8400"}}
+{"timestamp":1714359016.8651104,"name":"online","context":{"idset":"10252"}}
+{"timestamp":1714359016.8658826,"name":"online","context":{"idset":"9883,10727"}}
+{"timestamp":1714359016.8666179,"name":"online","context":{"idset":"10391"}}
+{"timestamp":1714359016.8676128,"name":"online","context":{"idset":"9165,10546"}}
+{"timestamp":1714359016.8686683,"name":"online","context":{"idset":"9950,10243,10482,11000"}}
+{"timestamp":1714359016.869683,"name":"online","context":{"idset":"10759,10998,11095"}}
+{"timestamp":1714359016.8704846,"name":"online","context":{"idset":"10094"}}
+{"timestamp":1714359016.8713887,"name":"online","context":{"idset":"11417"}}
+{"timestamp":1714359016.8723917,"name":"online","context":{"idset":"11834,11858"}}
+{"timestamp":1714359016.8732343,"name":"online","context":{"idset":"11530"}}
+{"timestamp":1714359016.8743312,"name":"online","context":{"idset":"10586"}}
+{"timestamp":1714359016.8750591,"name":"online","context":{"idset":"10399"}}
+{"timestamp":1714359016.8763492,"name":"online","context":{"idset":"1187,10424,10702"}}
+{"timestamp":1714359016.8776078,"name":"online","context":{"idset":"8121"}}
+{"timestamp":1714359016.878628,"name":"online","context":{"idset":"9089"}}
+{"timestamp":1714359016.8800993,"name":"online","context":{"idset":"8166,8177,8242,8331,11665,11704"}}
+{"timestamp":1714359016.8816805,"name":"online","context":{"idset":"7974,8936,11201,11757"}}
+{"timestamp":1714359016.8836162,"name":"online","context":{"idset":"194,223,7898,8397,8984,9131,9238,9281,9521,9566,9643,9803"}}
+{"timestamp":1714359016.8848591,"name":"online","context":{"idset":"9697"}}
+{"timestamp":1714359016.8867121,"name":"online","context":{"idset":"9153,9418"}}
+{"timestamp":1714359016.8892889,"name":"online","context":{"idset":"8307,8358,8577,8902,8941,8957,9106,9590"}}
+{"timestamp":1714359016.8930819,"name":"online","context":{"idset":"1181,7800,7882,7927,8046,8085,8099,8155,8204,8209,8617,9166,9476,9510,9558,9608,9665,9827,9885,9996,10215,10597,10785,11049,11468,11483,11613,11681,11800,11835,11843"}}
+{"timestamp":1714359016.8946211,"name":"online","context":{"idset":"9251,11082,11465"}}
+{"timestamp":1714359016.8969941,"name":"online","context":{"idset":"6638,7969,8214,8350,8416,8427-8428,8621,9145,9343,9420,9559,9571,9668,9690,9705,9816,9906,10106,10203,10290,10389,10400,10483,10725,10777,11048,11442,11592,11623,11652,11855"}}
+{"timestamp":1714359016.8980515,"name":"online","context":{"idset":"7888,8036,8843,8960,9149,9382,9673,9724,10167,10232,10257,10513,11778,11802,11882"}}
+{"timestamp":1714359016.8993077,"name":"online","context":{"idset":"8380,8401,8431,9231,9583,10057,10468"}}
+{"timestamp":1714359016.9573815,"name":"online","context":{"idset":"173,1178,1183,7799,7823,7840,7897,7966,8069,8108,8112,8181,8222,8232,8609,8891,8899,8916,8942,8969,9121,9129,9151,9172,9363,9432,9526,9574,9576,9706,9768,9848,9853-9854,9932,10103,10153,10363,10472,10596,10610,10613,10731,10743,10794,10800,11070,11072,11182,11197,11444-11445,11730,11764,11838"}}
+{"timestamp":1714359016.9629238,"name":"online","context":{"idset":"7940,8929,9279"}}
+{"timestamp":1714359016.9715714,"name":"online","context":{"idset":"9457"}}
+{"timestamp":1714359016.9782588,"name":"online","context":{"idset":"9855,10620-10621"}}
+{"timestamp":1714359016.9800477,"name":"online","context":{"idset":"9009"}}
+{"timestamp":1714359016.9824228,"name":"online","context":{"idset":"7831,8928,9348,9905,11438"}}
+{"timestamp":1714359016.9850132,"name":"online","context":{"idset":"9653,10289,10758"}}
+{"timestamp":1714359016.9892442,"name":"online","context":{"idset":"420,8091,8122,8827,9302,9551,10503,10710,11853"}}
+{"timestamp":1714359017.0505645,"name":"online","context":{"idset":"1180,7906,8268,8336,8635,8650,8873,8884,8989,9079,9088,9714,9776,9997,10255,10455,10703,10862,11413,11786,11857"}}
+{"timestamp":1714359017.0532026,"name":"online","context":{"idset":"10051,11431"}}
+{"timestamp":1714359018.4792364,"name":"online","context":{"idset":"282,1234,6527,6550,7850,7904,7948,7955,8026,8081,8159,8182,8219,8309,8394,8406,8570,8630,8661,8841,8968,9069,9190,9218,9225,9265,9267,9275,9295,9636,9645,9880,9957,10000,10023,10045,10076,10081,10296,10343,10361,10385,10500,10524,10576,10592,10604,10654,10737,10835,11129,11180,11185,11440,11469,11603,11680,11772,11861"}}
+{"timestamp":1714359018.4886026,"name":"online","context":{"idset":"874,6578-6579,6589,7832,7862,7900-7901,7909,7953,7958,7994,8021,8041,8057,8066,8098,8104,8126,8134,8136-8137,8168,8171,8189,8254,8361,8370,8382,8426,8591,8612,8622,8640,8689,8896,8930,8952,9046,9077,9098,9107,9109,9162,9186,9191,9226,9248,9258,9268,9277,9324,9336,9352,9449,9475,9482,9519,9524,9534,9565,9670-9671,9683,9695,9896,9972,10018,10091,10173,10264,10278,10285,10287,10294,10320,10373,10384,10443,10461,10494,10562,10643,10652,10665,10701,10783,10848,11015,11098,11192,11200,11419,11473,11527,11591,11611,11616,11708,11713,11865,11875,11891-11892"}}
+{"timestamp":1714359018.4940293,"name":"online","context":{"idset":"7915"}}
+{"timestamp":1714359018.4960129,"name":"online","context":{"idset":"6549,10679"}}
+{"timestamp":1714359018.4976888,"name":"online","context":{"idset":"10383,11080,11718,11837"}}
+{"timestamp":1714359018.4996872,"name":"online","context":{"idset":"8224,10659,11434"}}
+{"timestamp":1714359018.5014052,"name":"online","context":{"idset":"8423,8663"}}
+{"timestamp":1714359018.5035124,"name":"online","context":{"idset":"7977"}}
+{"timestamp":1714359018.5062928,"name":"online","context":{"idset":"1229,1236,8013,8901,9188,9490,9760,9780,9928,10246,10447,10820,11124"}}
+{"timestamp":1714359018.5083585,"name":"online","context":{"idset":"171,8078,9614,10242,10417,10555,11650"}}
+{"timestamp":1714359018.5101602,"name":"online","context":{"idset":"6605,6611,8160,9436,9452,9978,10552,10742"}}
+{"timestamp":1714359018.5122099,"name":"online","context":{"idset":"8654,9385,9849,9856,9951,9991,10790,11145,11198"}}
+{"timestamp":1714359018.5137391,"name":"online","context":{"idset":"6639"}}
+{"timestamp":1714359018.515892,"name":"online","context":{"idset":"9380"}}
+{"timestamp":1714359018.5177383,"name":"online","context":{"idset":"8844,8894"}}
+{"timestamp":1714359018.5192952,"name":"online","context":{"idset":"9228,9602"}}
+{"timestamp":1714359018.5210268,"name":"online","context":{"idset":"9244,9499,9518,9591"}}
+{"timestamp":1714359018.5229838,"name":"online","context":{"idset":"9426"}}
+{"timestamp":1714359018.5249228,"name":"online","context":{"idset":"9072,9104,9579"}}
+{"timestamp":1714359018.5267243,"name":"online","context":{"idset":"9594,9685,9797,9838,10009,10048,10125"}}
+{"timestamp":1714359018.5287051,"name":"online","context":{"idset":"1226,8172,10174,10372,10623,11012"}}
+{"timestamp":1714359018.5305386,"name":"online","context":{"idset":"7902,8218,8253,8310,8600,9831,10079,10840,11121,11399,11702,11709"}}
+{"timestamp":1714359018.5322955,"name":"online","context":{"idset":"10564,10747,11685"}}
+{"timestamp":1714359018.5342517,"name":"online","context":{"idset":"131"}}
+{"timestamp":1714359018.5368934,"name":"online","context":{"idset":"10828"}}
+{"timestamp":1714359018.5394483,"name":"online","context":{"idset":"10142,11390,11395,11519,11581"}}
+{"timestamp":1714359018.5414543,"name":"online","context":{"idset":"7963,10330,10450,10454,10714,10726,11809"}}
+{"timestamp":1714359018.5451684,"name":"online","context":{"idset":"113,330,6569,6577,7811,7838,7866,8273,8280,8295,8325,8874,8940,8953,9159,9195,9207,9652,9659,9715,9922,9998,10354,10675,11065,11136,11188,11415,11538,11675,11748,11754"}}
+{"timestamp":1714359018.5477483,"name":"online","context":{"idset":"10824"}}
+{"timestamp":1714359018.5495882,"name":"online","context":{"idset":"9378"}}
+{"timestamp":1714359018.5531271,"name":"online","context":{"idset":"7942,8020,8644,8949,9000,9047,9086,9287,9394,9597,10154,10178,11595,11653"}}
+{"timestamp":1714359018.5555844,"name":"online","context":{"idset":"7939,7987,8011,8128,8221,8589,8629,8848,8915,9477,9568,9619,9624,9931,10238,10661,11091,11534,11824,11832,11879"}}
+{"timestamp":1714359018.5609035,"name":"online","context":{"idset":"94,280,6564,6614,7824,7845,8277,8420,8611,8653,8840,9430,9508,9666,9692,9794,9939,9985,10244,10559,10633,10728,10825,11642,11776,11825"}}
+{"timestamp":1714359018.5633869,"name":"online","context":{"idset":"106,178,1216,6619,7928,8033,8157,8190,8260,8294,8391,8580,8867,8900,9080,9094,9229,9247,9273,9460,9580,9592,9615,9748,9836,10102,10159,10314,10355,10501,10585,10595,10712,10826,10833,10837,11108,11116,11134,11153,11170,11414,11448"}}
+{"timestamp":1714359018.5654569,"name":"online","context":{"idset":"299,305,8667,8886,8912,8934,9386,9533,10149,10253,10615"}}
+{"timestamp":1714359018.5674171,"name":"online","context":{"idset":"9753,10139,11794"}}
+{"timestamp":1714359018.5696993,"name":"online","context":{"idset":"8156,11175"}}
+{"timestamp":1714359018.5718184,"name":"online","context":{"idset":"9910"}}
+{"timestamp":1714359018.573807,"name":"online","context":{"idset":"10527"}}
+{"timestamp":1714359018.5758438,"name":"online","context":{"idset":"307,8688"}}
+{"timestamp":1714359018.577723,"name":"online","context":{"idset":"11605"}}
+{"timestamp":1714359018.5797784,"name":"online","context":{"idset":"9388"}}
+{"timestamp":1714359018.5853348,"name":"online","context":{"idset":"195"}}
+{"timestamp":1714359018.587127,"name":"online","context":{"idset":"9414,10277,11525"}}
+{"timestamp":1714359018.5890408,"name":"online","context":{"idset":"8576,8892,9245,9532,9722,10105,10349,11490"}}
+{"timestamp":1714359018.5910597,"name":"online","context":{"idset":"1173,6615,8376,9820,9907,9981,10713,11073,11654,11710,11886"}}
+{"timestamp":1714359018.593761,"name":"online","context":{"idset":"221,334,6610,8422,9301,11701"}}
+{"timestamp":1714359018.5958092,"name":"online","context":{"idset":"9723"}}
+{"timestamp":1714359018.597898,"name":"online","context":{"idset":"9125,9807,11599,11812"}}
+{"timestamp":1714359018.6006191,"name":"online","context":{"idset":"6537,7817,8058,9346,9892,10017,10575,10690,11179"}}
+{"timestamp":1714359018.608469,"name":"online","context":{"idset":"835,7880,7899,9020"}}
+{"timestamp":1714359018.6104395,"name":"online","context":{"idset":"8413,9941,10574,11695"}}
+{"timestamp":1714359018.6125505,"name":"online","context":{"idset":"1209,6561,7950,8194,8255,9011,9022,9143,9445,9557,9875,9955,10044,10144,11002,11386,11487,11502,11692,11805,11868-11869"}}
+{"timestamp":1714359018.6139438,"name":"online","context":{"idset":"8566,9633,10695"}}
+{"timestamp":1714359018.6152663,"name":"online","context":{"idset":"8825,8951,8970,9439,9497,10010,10429,11791"}}
+{"timestamp":1714359018.6166217,"name":"online","context":{"idset":"170,188,235,239,308,321,7976,8167,9926,10394,10631,11027"}}
+{"timestamp":1714359018.6179831,"name":"online","context":{"idset":"112,6591,7941,8346,8388,9052,9361,9788,9852,10756,11773,11821"}}
+{"timestamp":1714359018.6191199,"name":"online","context":{"idset":"8068"}}
+{"timestamp":1714359018.620374,"name":"online","context":{"idset":"8140,8879,9090,9113,9451,9542,9877,9990,10553,10571,10698"}}
+{"timestamp":1714359018.6222579,"name":"online","context":{"idset":"93,9272,9995,10488"}}
+{"timestamp":1714359018.6244199,"name":"online","context":{"idset":"8312,10818"}}
+{"timestamp":1714359018.626857,"name":"online","context":{"idset":"301,10258,10396,10739,11804"}}
+{"timestamp":1714359018.6288695,"name":"online","context":{"idset":"11184"}}
+{"timestamp":1714359018.6309681,"name":"online","context":{"idset":"10148,10334,10337"}}
+{"timestamp":1714359018.6331136,"name":"online","context":{"idset":"6632,8220,10410,10745"}}
+{"timestamp":1714359018.6351275,"name":"online","context":{"idset":"10216"}}
+{"timestamp":1714359018.6371624,"name":"online","context":{"idset":"10120"}}
+{"timestamp":1714359018.6394429,"name":"online","context":{"idset":"9720"}}
+{"timestamp":1714359018.6414053,"name":"online","context":{"idset":"297,9284"}}
+{"timestamp":1714359018.6435354,"name":"online","context":{"idset":"200,213,323,10566"}}
+{"timestamp":1714359018.6459303,"name":"online","context":{"idset":"327"}}
+{"timestamp":1714359018.6485851,"name":"online","context":{"idset":"8685"}}
+{"timestamp":1714359018.6525207,"name":"online","context":{"idset":"209,1176,8200,8211,8250,8403,8417,8588,9019,9417,9664,9969,10082,10084,10145,10753,11479,11890"}}
+{"timestamp":1714359018.654917,"name":"online","context":{"idset":"8314,9463,9663,10163"}}
+{"timestamp":1714359018.6569731,"name":"online","context":{"idset":"11707"}}
+{"timestamp":1714359018.6604381,"name":"online","context":{"idset":"168,413,576,8008,8073,8395,8649,9007,9127,9431,9440,9644,9814,9988,10016,10033,10097,10117,10201,10408,10460,10510,10624,10736,11089,11138,11696,11714,11887"}}
+{"timestamp":1714359018.6627305,"name":"online","context":{"idset":"9211"}}
+{"timestamp":1714359018.6651423,"name":"online","context":{"idset":"9610,10172,11491"}}
+{"timestamp":1714359018.6700583,"name":"online","context":{"idset":"294,328,414,8269,8607,9400,9864,9977,10722,10834,10855,11593,11624,11676"}}
+{"timestamp":1714359018.6722724,"name":"online","context":{"idset":"22,9375,10578"}}
+{"timestamp":1714359018.674336,"name":"online","context":{"idset":"20"}}
+{"timestamp":1714359018.6773055,"name":"online","context":{"idset":"3,25,40,10687"}}
+{"timestamp":1714359018.6795652,"name":"online","context":{"idset":"9821"}}
+{"timestamp":1714359018.6819935,"name":"online","context":{"idset":"9974,11740"}}
+{"timestamp":1714359018.6846268,"name":"online","context":{"idset":"8429,8433,9260,9362"}}
+{"timestamp":1714359018.687031,"name":"online","context":{"idset":"205"}}
+{"timestamp":1714359018.6895571,"name":"online","context":{"idset":"160,212,237,575,1214,6551,6566,9242,9360,9605,9612,9627-9628,9882,9886,10492,11619,11621"}}
+{"timestamp":1714359018.6922021,"name":"online","context":{"idset":"6623,9917,10067"}}
+{"timestamp":1714359018.694896,"name":"online","context":{"idset":"6603,10749"}}
+{"timestamp":1714359018.6973214,"name":"online","context":{"idset":"8646,9485,9717"}}
+{"timestamp":1714359018.7002861,"name":"online","context":{"idset":"45,6586,6631,9059,9754,9861,9962,11398"}}
+{"timestamp":1714359018.7031193,"name":"online","context":{"idset":"241,8352,8402,8995,9013,9070,9311,9353,9393,9618,9804,9863,9954,9964,10003,10306,10754,10796,10804,10836,11137,11435,11749"}}
+{"timestamp":1714359018.7060375,"name":"online","context":{"idset":"1182,10214,11202"}}
+{"timestamp":1714359018.7080598,"name":"online","context":{"idset":"9222,9845,9860,10141,11673"}}
+{"timestamp":1714359018.7103379,"name":"online","context":{"idset":"10136,11735,11745"}}
+{"timestamp":1714359018.7163115,"name":"online","context":{"idset":"9660"}}
+{"timestamp":1714359018.7177029,"name":"online","context":{"idset":"9617,9842,11149"}}
+{"timestamp":1714359018.7192271,"name":"online","context":{"idset":"6536,9374,9963,9989"}}
+{"timestamp":1714359018.7212389,"name":"online","context":{"idset":"9593,10831"}}
+{"timestamp":1714359018.7239728,"name":"online","context":{"idset":"10098"}}
+{"timestamp":1714359018.7266853,"name":"online","context":{"idset":"6555,9366"}}
+{"timestamp":1714359018.7311525,"name":"online","context":{"idset":"36"}}
+{"timestamp":1714359018.7344368,"name":"online","context":{"idset":"6,31,34,6587,6634,8601,8664,9986,10451,11573"}}
+{"timestamp":1714359018.7370424,"name":"online","context":{"idset":"9257,9404,10064"}}
+{"timestamp":1714359018.7393811,"name":"online","context":{"idset":"10205"}}
+{"timestamp":1714359018.7422111,"name":"online","context":{"idset":"23,291"}}
+{"timestamp":1714359018.7452505,"name":"online","context":{"idset":"10,8616"}}
+{"timestamp":1714359018.7480488,"name":"online","context":{"idset":"302"}}
+{"timestamp":1714359018.7508647,"name":"online","context":{"idset":"8316,10560,11063,11768"}}
+{"timestamp":1714359018.7534666,"name":"online","context":{"idset":"35,46,8025,8389,8990,8996,9276,9737,10773,11103"}}
+{"timestamp":1714359018.7596259,"name":"online","context":{"idset":"9187,9401,9726,10132,10318,11733"}}
+{"timestamp":1714359018.7625957,"name":"online","context":{"idset":"8169,9661,10095,11620"}}
+{"timestamp":1714359018.7650542,"name":"online","context":{"idset":"9354,9399,9623"}}
+{"timestamp":1714359018.7671969,"name":"online","context":{"idset":"10043,10579"}}
+{"timestamp":1714359018.7695105,"name":"online","context":{"idset":"7859,9654,11177,11509"}}
+{"timestamp":1714359018.7718682,"name":"online","context":{"idset":"10813"}}
+{"timestamp":1714359018.7741151,"name":"online","context":{"idset":"48,6617,9425,10418"}}
+{"timestamp":1714359018.7766798,"name":"online","context":{"idset":"210,11662"}}
+{"timestamp":1714359018.7806864,"name":"online","context":{"idset":"246,250,9745,10211,11876"}}
+{"timestamp":1714359018.7834451,"name":"online","context":{"idset":"1213,1218,7986,8062,8578,8836,9078,9220,9828,11638"}}
+{"timestamp":1714359018.7857709,"name":"online","context":{"idset":"6572,8606,8678,8687,10179,11090"}}
+{"timestamp":1714359018.7884305,"name":"online","context":{"idset":"7892"}}
+{"timestamp":1714359018.793781,"name":"online","context":{"idset":"8043,11480"}}
+{"timestamp":1714359018.796509,"name":"online","context":{"idset":"9884,10529,11140"}}
+{"timestamp":1714359018.8025572,"name":"online","context":{"idset":"9948"}}
+{"timestamp":1714359018.8050275,"name":"online","context":{"idset":"8206,8997,11532"}}
+{"timestamp":1714359018.8075619,"name":"online","context":{"idset":"10101,10530"}}
+{"timestamp":1714359018.8102386,"name":"online","context":{"idset":"6563,7946,9555,9676,9735,11123"}}
+{"timestamp":1714359018.8124101,"name":"online","context":{"idset":"11074"}}
+{"timestamp":1714359018.814533,"name":"online","context":{"idset":"8681"}}
+{"timestamp":1714359018.817157,"name":"online","context":{"idset":"6520,9278,9292,10532,10755,11167,11705"}}
+{"timestamp":1714359018.8192892,"name":"online","context":{"idset":"11176"}}
+{"timestamp":1714359018.8212614,"name":"online","context":{"idset":"49"}}
+{"timestamp":1714359018.8232346,"name":"online","context":{"idset":"11163"}}
+{"timestamp":1714359018.8260803,"name":"online","context":{"idset":"27"}}
+{"timestamp":1714359018.8280718,"name":"online","context":{"idset":"9"}}
+{"timestamp":1714359018.8302066,"name":"online","context":{"idset":"8022,8158,8907,9473,10250,11077"}}
+{"timestamp":1714359018.8321755,"name":"online","context":{"idset":"11433"}}
+{"timestamp":1714359018.8341708,"name":"online","context":{"idset":"8093,10138,10428"}}
+{"timestamp":1714359018.8361585,"name":"online","context":{"idset":"8101,8393,11030"}}
+{"timestamp":1714359018.8380833,"name":"online","context":{"idset":"11734,11801"}}
+{"timestamp":1714359018.840076,"name":"online","context":{"idset":"10668"}}
+{"timestamp":1714359018.8425164,"name":"online","context":{"idset":"44,1190,6597,7816,7855,7925,8203,8275,8436,9114,9118,9210,9381,9398,9522,9529,9736,9818,9984,10325,10427,10673,11196,11475,11578,11659"}}
+{"timestamp":1714359018.845098,"name":"online","context":{"idset":"8823"}}
+{"timestamp":1714359018.8471038,"name":"online","context":{"idset":"9180"}}
+{"timestamp":1714359018.8492038,"name":"online","context":{"idset":"11146,11158"}}
+{"timestamp":1714359018.8512671,"name":"online","context":{"idset":"9160,9767,9960,10160"}}
+{"timestamp":1714359018.8532839,"name":"online","context":{"idset":"10269"}}
+{"timestamp":1714359018.8554618,"name":"online","context":{"idset":"9785,10100,11446,11494"}}
+{"timestamp":1714359018.857646,"name":"online","context":{"idset":"6557,6594,8016,8061,8871,8914,8939,9833,10128"}}
+{"timestamp":1714359018.8597643,"name":"online","context":{"idset":"8271,10497"}}
+{"timestamp":1714359018.8618691,"name":"online","context":{"idset":"6554,7854,8665,9688,10411"}}
+{"timestamp":1714359018.8687711,"name":"online","context":{"idset":"10844"}}
+{"timestamp":1714359018.870791,"name":"online","context":{"idset":"9898"}}
+{"timestamp":1714359018.8727858,"name":"online","context":{"idset":"10359"}}
+{"timestamp":1714359018.8749845,"name":"online","context":{"idset":"8904"}}
+{"timestamp":1714359018.8769956,"name":"online","context":{"idset":"9513"}}
+{"timestamp":1714359018.8792117,"name":"online","context":{"idset":"7878"}}
+{"timestamp":1714359019.0771055,"name":"online","context":{"idset":"6580,6627,7864,7921,8343,8658,9424,9719,9888,9909,9921,10104,10171,10213,10225,10572,10763,11191,11731"}}
+{"timestamp":1714359020.0421264,"name":"online","context":{"idset":"8583"}}
+{"timestamp":1714359020.0439241,"name":"online","context":{"idset":"6600,7967,8076"}}
+{"timestamp":1714359020.0461297,"name":"online","context":{"idset":"7842,7937,8092,8133,8398,10060,10184,10220,10302,10512,10568,10693,11583,11632"}}
+{"timestamp":1714359020.0553992,"name":"online","context":{"idset":"5,119,218,573,873,6544,6547,6558,6582,6624,7825,7867,8019,8035,8116,8147,8165,8229,8265,8279,8329,8348,8357,8362,8377,8599,8613,8620,8686,8860,8905,8925,9012,9056,9076,9119,9154,9177,9266,9297,9299,9365,9367,9376,9389,9406,9455,9458,9503,9586,9696,9718,9769,9815,9869,9872,9900,10115,10127,10164,10195,10198,10323,10402,10422,10558,10681,10723,10861,11026,11034,11157,11178,11396,11533,11635,11698,11742,11795,11797"}}
+{"timestamp":1714359020.0568836,"name":"online","context":{"idset":"10208"}}
+{"timestamp":1714359020.2130525,"name":"online","context":{"idset":"100,127,180,247-248,6530,7933,7965,7999,8055,8313,8628,8634,8857,8863,8971,8981,9006,9102,9117,9137,9270,9349,9368,9495,9502,9572,9603,9713,10021,10113,10180,10196,10206,10265,10386,10557,10636,10658,10732,10740,10791,10809,10853,11041,11112,11199,11489,11655,11660,11863"}}
+{"timestamp":1714359020.2176502,"name":"online","context":{"idset":"41,43"}}
+{"timestamp":1714359020.5387883,"name":"online","context":{"idset":"50"}}
+{"timestamp":1714359020.5509007,"name":"online","context":{"idset":"1212,1219,6553,7807,7861,7893,7992,8114,8328,8334,8593,8668,8945,8999,9058,9124,9252,9792,9862,9895,10181,10197,10204,10254,10598,10867,11729,11741,11751,11870"}}
+{"timestamp":1714359020.5528939,"name":"online","context":{"idset":"338,416,9306"}}
+{"timestamp":1714359020.554883,"name":"online","context":{"idset":"207,6608,6628,8430"}}
+{"timestamp":1714359020.5567522,"name":"online","context":{"idset":"9496"}}
+{"timestamp":1714359020.5588181,"name":"online","context":{"idset":"8842,9140,9351"}}
+{"timestamp":1714359020.5607901,"name":"online","context":{"idset":"8648"}}
+{"timestamp":1714359020.5627828,"name":"online","context":{"idset":"9743,9771,10431"}}
+{"timestamp":1714359020.564676,"name":"online","context":{"idset":"9739,10183"}}
+{"timestamp":1714359020.5665944,"name":"online","context":{"idset":"8565,9422"}}
+{"timestamp":1714359020.5685039,"name":"online","context":{"idset":"9373,9930"}}
+{"timestamp":1714359020.5704236,"name":"online","context":{"idset":"9377,9447,9641,9946"}}
+{"timestamp":1714359020.5726516,"name":"online","context":{"idset":"9730,10365"}}
+{"timestamp":1714359020.5749466,"name":"online","context":{"idset":"9707,9844,10158,10175"}}
+{"timestamp":1714359020.5772762,"name":"online","context":{"idset":"9293,9372,10846"}}
+{"timestamp":1714359020.5796723,"name":"online","context":{"idset":"9865,9979,10024,10193,10502,10543,10556,11416,11669,11845"}}
+{"timestamp":1714359020.582166,"name":"online","context":{"idset":"8227,9953"}}
+{"timestamp":1714359020.5862055,"name":"online","context":{"idset":"9897,11001"}}
+{"timestamp":1714359020.5887492,"name":"online","context":{"idset":"8087,8198,9952"}}
+{"timestamp":1714359020.5918763,"name":"online","context":{"idset":"10626"}}
+{"timestamp":1714359020.5944953,"name":"online","context":{"idset":"11062"}}
+{"timestamp":1714359020.5972486,"name":"online","context":{"idset":"10336,10338,11439,11514,11516"}}
+{"timestamp":1714359020.6000376,"name":"online","context":{"idset":"11162"}}
+{"timestamp":1714359020.6028614,"name":"online","context":{"idset":"8682,10407,11169,11598,11798"}}
+{"timestamp":1714359020.6061492,"name":"online","context":{"idset":"8828"}}
+{"timestamp":1714359020.6087749,"name":"online","context":{"idset":"11008"}}
+{"timestamp":1714359020.6108956,"name":"online","context":{"idset":"11833"}}
+{"timestamp":1714359020.6130989,"name":"online","context":{"idset":"8226"}}
+{"timestamp":1714359020.6155937,"name":"online","context":{"idset":"11536"}}
+{"timestamp":1714359020.6180632,"name":"online","context":{"idset":"7841,8677,10516,11141,11526,11856"}}
+{"timestamp":1714359020.6200063,"name":"online","context":{"idset":"10822,11155"}}
+{"timestamp":1714359020.622366,"name":"online","context":{"idset":"11872"}}
+{"timestamp":1714359020.6252294,"name":"online","context":{"idset":"8604"}}
+{"timestamp":1714359020.6281021,"name":"online","context":{"idset":"10857"}}
+{"timestamp":1714359020.6309321,"name":"online","context":{"idset":"6644,11165"}}
+{"timestamp":1714359020.6338763,"name":"online","context":{"idset":"7884,8594,8645,10811,11716"}}
+{"timestamp":1714359020.6366451,"name":"online","context":{"idset":"6562"}}
+{"timestamp":1714359020.6393325,"name":"online","context":{"idset":"295"}}
+{"timestamp":1714359020.6419654,"name":"online","context":{"idset":"8821"}}
+{"timestamp":1714359020.6446064,"name":"online","context":{"idset":"8623,8643"}}
+{"timestamp":1714359020.6473296,"name":"online","context":{"idset":"8854,9148"}}
+{"timestamp":1714359020.65029,"name":"online","context":{"idset":"8882"}}
+{"timestamp":1714359020.6564744,"name":"online","context":{"idset":"32,56,231,825,872,1186,6526,6529,6583,7797,7809,7822,7844,7847,7851,7871-7873,7924,7938,7945,7947,7957,7983,8017,8039,8164,8187,8228,8276,8287,8290,8293,8408,8568,8586,8624,8674,9075,9103,9112,9116,9199,9224,9323,9371,9444,9474,9483,9498,9556,9563,9832,9871,10072,10135,10210,10271,10282,10293,10308,10315-10316,10369,10398,10405,10432,10446,10448,10456,10467,10469,10584,10622,10663,10829,10847,10859,11040,11120,11126,11144,11498,11518,11604"}}
+{"timestamp":1714359020.6600342,"name":"online","context":{"idset":"7,12,18,26,6602,7912,7951,8132,8691,8881,8976,9025,9298,9390,9454,9462,9546,9588,10741,10827,11014,11029,11646"}}
+{"timestamp":1714359020.6693926,"name":"online","context":{"idset":"55,109,130,214,216,243,300,316,324,331,1208,6567,7984,7996,8009,8054,8077,8248,8315,8330,8368,8390,8657,8680,8991,9310,9481,9637,9711,9765,9850,9901,9942,9993,10006,10049,10070,10089,10274,10415,10487,10602,10667,10706,10858,10864-10866,11060,11148,11154,11159,11409,11495,11687,11694,11766,11770,11779,11841"}}
+{"timestamp":1714359020.6744912,"name":"online","context":{"idset":"161-163,166,192,222,283,304,309,344,6618,8197,8291,8367,8986,8992,9355,9429,9516,9596,9632,9938,9983,10390,10435,10491,10523,10542,10588,10640,10730,11042,11094,11384,11712,11842"}}
+{"timestamp":1714359020.6792202,"name":"online","context":{"idset":"311,417,6643,7989,8052,8174,8327,8404,9271,9318,9453,9469,9801,9911,10352,10635,10672,10678,10689,10718,10782,10845,11187,11402,11630,11699,11747"}}
+{"timestamp":1714359020.6856756,"name":"online","context":{"idset":"17,38,47,197,249,322,577,7865,8299,8386,8858,9005,9010,9130,9320,9656,9731,9802,9811,10032,10071,10090,10374,10505,10582,10664,11171,11521,11637"}}
+{"timestamp":1714359020.7187371,"name":"online","context":{"idset":"6606,7820,8139,8266,8317,9313,9335,9733,9783,10263,10268,10545,10801,11181,11408,11674,11722,11738"}}
+{"timestamp":1714359020.7628932,"name":"online","context":{"idset":"289,320,6552,6556,6635,7812,7926,7929,7954,8007,8067,8094,8110,8213,8216,8223,8298,8585,8851,8876,8932,9100,9152,9168,9173-9174,9198,9274,9322,9340,9392,9437,9564,9567,9709,9826,9933,9999,10012,10083,10107,10223,10231,10357,10388,10404,10419,10536,10594,10603,10738,10767,11010,11039,11166,11427,11577,11610,11719,11780,11823,11830"}}
+{"timestamp":1714359020.766845,"name":"online","context":{"idset":"201,245,6539,6598,7985,8047,8060,8176,8207,8225,8241,8339,8341,8908,8964,9049,9134,9203,9520,9766,10022,10226,10413,10674,10677,10854,11628,11670"}}
+{"timestamp":1714359020.7691722,"name":"online","context":{"idset":"7881,8064,8071,8215,9120,9282,9488,9912,10569"}}
+{"timestamp":1714359020.7720482,"name":"online","context":{"idset":"7916,10632,10761"}}
+{"timestamp":1714359020.7745712,"name":"online","context":{"idset":"233,277"}}
+{"timestamp":1714359020.7771413,"name":"online","context":{"idset":"8575"}}
+{"timestamp":1714359020.7800863,"name":"online","context":{"idset":"8247,9742,9840,10789"}}
+{"timestamp":1714359020.7826931,"name":"online","context":{"idset":"6626,7818,8257,8396,9027,9468,10518,10634,10868,11107,11504,11520,11737"}}
+{"timestamp":1714359020.7851498,"name":"online","context":{"idset":"9687,11816,11888"}}
+{"timestamp":1714359020.7880569,"name":"online","context":{"idset":"8086,8119,8392,9326"}}
+{"timestamp":1714359020.7912717,"name":"online","context":{"idset":"977,6517,7995,8673,9084,9146,9411,10779"}}
+{"timestamp":1714359020.7942181,"name":"online","context":{"idset":"7808,11397"}}
+{"timestamp":1714359020.7964747,"name":"online","context":{"idset":"11852"}}
+{"timestamp":1714359020.79916,"name":"online","context":{"idset":"9240,9347,11190"}}
+{"timestamp":1714359020.8023255,"name":"online","context":{"idset":"7886,9331,9970,11132"}}
+{"timestamp":1714359020.804621,"name":"online","context":{"idset":"6621,10162,10766"}}
+{"timestamp":1714359020.8070273,"name":"online","context":{"idset":"10606"}}
+{"timestamp":1714359020.8096046,"name":"online","context":{"idset":"8379"}}
+{"timestamp":1714359020.812067,"name":"online","context":{"idset":"8320"}}
+{"timestamp":1714359020.8145642,"name":"online","context":{"idset":"8432"}}
+{"timestamp":1714359020.8169215,"name":"online","context":{"idset":"8083"}}
+{"timestamp":1714359021.5172567,"name":"online","context":{"idset":"10188"}}
+{"timestamp":1714359021.5275502,"name":"online","context":{"idset":"114,227,6522,6535,6585,8569,8587,8615,8627,8632,8638,8642,8652,8954,8985,9057,9408,9450,9647,9651,9679,9702,9716,9762,9764,9772,9805,9858,9887,9903,9980,9982,10026,10121,10130,10168,10170,10207,10209,10219,10798,11139,11193"}}
+{"timestamp":1714359021.5308485,"name":"online","context":{"idset":"199,347,406,8582,8626,8655,9008,9018,9337,9369,9397,9412,9646,9672,9746,9791,9868,9893,10040,10099,10123,10187,10704,10771,10805,10812,10852,11084,11156,11161,11728,11732"}}
+{"timestamp":1714359021.5361996,"name":"online","context":{"idset":"8018"}}
+{"timestamp":1714359021.5388494,"name":"online","context":{"idset":"9405,11464"}}
+{"timestamp":1714359021.541461,"name":"online","context":{"idset":"8042,8129,8636,9099,10393"}}
+{"timestamp":1714359021.5441015,"name":"online","context":{"idset":"7868,8302,9206,9609"}}
+{"timestamp":1714359021.5471246,"name":"online","context":{"idset":"10708"}}
+{"timestamp":1714359021.5503051,"name":"online","context":{"idset":"7883,8978,9536,9712,9781,10085,10276,11079,11097,11457,11481,11499,11585,11844"}}
+{"timestamp":1714359021.5534639,"name":"online","context":{"idset":"883,11068,11478,11528,11803"}}
+{"timestamp":1714359021.5564473,"name":"online","context":{"idset":"8107"}}
+{"timestamp":1714359021.5596263,"name":"online","context":{"idset":"8210,9387,10366,10641"}}
+{"timestamp":1714359021.5627079,"name":"online","context":{"idset":"9710"}}
+{"timestamp":1714359021.5658562,"name":"online","context":{"idset":"191,7846,7949,8822,8870,8880,9192,9300,9345,9793,9908,10028,10230,10283-10284,10305,10370,10452,10476,10655,11392,11817,11862"}}
+{"timestamp":1714359021.5686808,"name":"online","context":{"idset":"8038,8244,9176,10416"}}
+{"timestamp":1714359021.570941,"name":"online","context":{"idset":"8574"}}
+{"timestamp":1714359021.5735798,"name":"online","context":{"idset":"1175"}}
+{"timestamp":1714359021.5762966,"name":"online","context":{"idset":"1188,7982"}}
+{"timestamp":1714359021.578882,"name":"online","context":{"idset":"9235"}}
+{"timestamp":1714359021.5815396,"name":"online","context":{"idset":"9194"}}
+{"timestamp":1714359021.584532,"name":"online","context":{"idset":"9111"}}
+{"timestamp":1714359021.5874562,"name":"online","context":{"idset":"7980"}}
+{"timestamp":1714359021.5903647,"name":"online","context":{"idset":"8006"}}
+{"timestamp":1714359021.5929136,"name":"online","context":{"idset":"10752"}}
+{"timestamp":1714359021.595607,"name":"online","context":{"idset":"11667"}}
+{"timestamp":1714359021.5981627,"name":"online","context":{"idset":"10073"}}
+{"timestamp":1714359021.6007164,"name":"online","context":{"idset":"9428"}}
+{"timestamp":1714359021.6035764,"name":"online","context":{"idset":"57,8962,10774"}}
+{"timestamp":1714359021.6063094,"name":"online","context":{"idset":"19,8105"}}
+{"timestamp":1714359021.6089387,"name":"online","context":{"idset":"9727"}}
+{"timestamp":1714359021.6115444,"name":"online","context":{"idset":"10627,10851"}}
+{"timestamp":1714359021.6144958,"name":"online","context":{"idset":"53"}}
+{"timestamp":1714359021.6175966,"name":"online","context":{"idset":"54"}}
+{"timestamp":1714359021.6202919,"name":"online","context":{"idset":"13-14,6637,11739"}}
+{"timestamp":1714359021.6230414,"name":"online","context":{"idset":"9728,10490"}}
+{"timestamp":1714359021.6257615,"name":"online","context":{"idset":"128,339"}}
+{"timestamp":1714359021.6284215,"name":"online","context":{"idset":"10612"}}
+{"timestamp":1714359021.6310015,"name":"online","context":{"idset":"9751"}}
+{"timestamp":1714359021.6335948,"name":"online","context":{"idset":"9051"}}
+{"timestamp":1714359021.6362514,"name":"online","context":{"idset":"8369,9729,11671"}}
+{"timestamp":1714359021.6388726,"name":"online","context":{"idset":"9859"}}
+{"timestamp":1714359021.6417601,"name":"online","context":{"idset":"30,167,8321,10581,11720"}}
+{"timestamp":1714359021.8532889,"name":"online","context":{"idset":"9824"}}
+{"timestamp":1714359022.0431046,"name":"online","context":{"idset":"8435"}}
+{"timestamp":1714359022.046567,"name":"online","context":{"idset":"4,122,6533,6590,6640,8614,8671,8684,9333,9759,9918,9920,9925,9947,10035,10061,10534,10757,10775,10778,10806,10838,10860,11135"}}
+{"timestamp":1714359022.0497711,"name":"online","context":{"idset":"6575"}}
+{"timestamp":1714359022.0532234,"name":"online","context":{"idset":"11636"}}
+{"timestamp":1714359022.0566504,"name":"online","context":{"idset":"9956"}}
+{"timestamp":1714359022.0721509,"name":"online","context":{"idset":"10221"}}
+{"timestamp":1714359022.2539439,"name":"online","context":{"idset":"6519,6543,6592-6593,8567,8869,10062,10066,10151,10157,10744,10748,11466"}}
+{"timestamp":1714359022.2633455,"name":"online","context":{"idset":"9505"}}
+{"timestamp":1714359022.3296661,"name":"online","context":{"idset":"11829"}}
+{"timestamp":1714359022.4566317,"name":"online","context":{"idset":"1185,7919,8412,9196,9250,9312,10298,10420,11021"}}
+{"timestamp":1714359022.6179059,"name":"online","context":{"idset":"1215,1222,1227,7853,7970,8015,8027,8103,8259,8267,8272,8910,9234,9303,9459,9509,9527,9787,10262,10312,10358,10362,10430,10464,10474,11032,11045,11109,11432,11437,11819,11848,11860,11889"}}
+{"timestamp":1714359022.6371768,"name":"online","context":{"idset":"10092"}}
+{"timestamp":1714359022.8122053,"name":"online","context":{"idset":"281,836,8355,8419,8963,8987,9167,9357,9599,9700,9846,10116,10176,11130,11743"}}
+{"timestamp":1714359022.9605763,"name":"online","context":{"idset":"8421"}}
+{"timestamp":1714359022.9820461,"name":"online","context":{"idset":"8084,8262,8885,9208,9410,9541,9870,10020,10053,10436,10793,10819,11523,11871"}}
+{"timestamp":1714359023.0922859,"name":"online","context":{"idset":"229"}}
+{"timestamp":1714359023.1115279,"name":"online","context":{"idset":"1225,7805,7836,7918,7952,8049,8185,8239,8256,8405,8959,9189,9243,9332,9575,9940,10245,10259,10297,10331,10465,10508,10561,11019,11044,11128,11622,11781,11883-11884"}}
+{"timestamp":1714359023.2769094,"name":"online","context":{"idset":"9464"}}
+{"timestamp":1714359023.2919288,"name":"online","context":{"idset":"7856,9087,10360,10438,10856"}}
+{"timestamp":1714359023.431354,"name":"online","context":{"idset":"9561"}}
+{"timestamp":1714359023.4346824,"name":"online","context":{"idset":"9631,9649,9758"}}
+{"timestamp":1714359023.4376721,"name":"online","context":{"idset":"9577"}}
+{"timestamp":1714359023.4512906,"name":"online","context":{"idset":"1174,1205,1228,6565,6595,7829,7887,7890-7891,7895,7903,7930,7978,7997,8012,8040,8059,8125,8148,8180,8235,8243,8258,8274,8282,8300,8340,8354,8359,8387,8603,8631,8647,8656,8683,8835,8866,8877,8889,8911,9083,9092,9136,9161,9178,9193,9214,9253,9255,9289,9504,9543,9606,9698,9741,9778,9809,9891,9934,10126,10147,10234,10279,10311,10322,10368,10377-10378,10395,10406,10409,10440,10463,10547,10607,10619,10644,10657,10660,10676,10700,10751,10815,11009,11016,11033,11038,11056,11075,11382,11388,11394,11484,11488,11524,11537,11615,11672,11677,11693,11818,11828,11851,11880"}}
+{"timestamp":1714359023.5831325,"name":"online","context":{"idset":"132,169,187,1220,6521,6528,6607,6622,7798,7813,7828,8319,8322,8333,8372-8373,8690,8826,9261,9328-9329,9501,9512,9604,9616,9684,9755,9798,9825,9961,10039,10191,10375,10519,10540,10590-10591,10669,10760,10808,11100,11657,11666,11744,11807"}}
+{"timestamp":1714359023.5868757,"name":"online","context":{"idset":"6523,10200,10803,11597,11760"}}
+{"timestamp":1714359023.6870835,"name":"online","context":{"idset":"8824,9640,9691,10037,11035,11663,11827"}}
+{"timestamp":1714359023.889713,"name":"online","context":{"idset":"105,123,126,179,184-185,193,292-293,303,346,405,6524,6532,7908,7923,8097,8162,8238,8284,8292,8353,8356,8424,8659,9002,9021,9055,9101,9246,9356,9358,9395,9416,9442,9537,9581,9600,9677,9693-9694,9775,9878,9904,9923,9949,9968,9975,10002,10013,10056,10075,10088,10150,10182,10189,10194,10199,10267,10288,10412,10437,10498,10541,10548,10563,10565,10711,10724,10769,10799,10863,11133,11587,11658,11684,11717,11763,11799,11831"}}
+{"timestamp":1714359023.9923704,"name":"online","context":{"idset":"7905,8938,9935,9973,11755"}}
+{"timestamp":1714359024.1940973,"name":"online","context":{"idset":"125,325,333,415,1177,1217,7869,8032,8153,8199,8289,8375,8414,8592,8641,8829,8845,8855,8868,8926,8931,8967,8972,9082,9157-9158,9175,9197,9212,9285,9415,9507,9530,9569,9620,9678,9699,9703,9799,9822,9837,9943,9976,10034,10069,10156,10165-10166,10218,10248,10295,10317,10321,10324,10333,10525,10583,10587,10637,10716,11426,11456,11461,11501,11651,11769,11783,11846"}}
+{"timestamp":1714359024.2974577,"name":"online","context":{"idset":"278,317,7917,7998,8056,8364,8579,8838,8846,8895,8898,10734,11189,11683"}}
+{"timestamp":1714359024.4050899,"name":"online","context":{"idset":"59,129,1206,1235,6541,6546,6625,7837,8240,8345,8666,8672,8676,8974,8988,8994,9015,9370,9396,9448,9480,9491,9545,9626,9680,9756,9806,9843,9913,9927,10161,10301,10342,10423,10445,10477,10495,10506,10554,10593,10705,10719,10781,10786,10830,11004-11006,11061,11617,11820,11850"}}
+{"timestamp":1714359024.512924,"name":"online","context":{"idset":"28,110,826,7848,8053,8407,9095,9407,9461,10030,10270,10356,10520,10682,10733,10750,11429,11513"}}
+{"timestamp":1714359024.6143053,"name":"online","context":{"idset":"1,7858,8000,8090,8117,8301,8633,8834,8937,8943,9179,9239,9434,9701,10131,10570,10577,11746"}}
+{"timestamp":1714359024.7170212,"name":"online","context":{"idset":"215,251,329,343,6642,8010,8335,8865,8919,8966,9060,9201,9221,9227,9456,9548,9638,9675,9819,9857,9867,9919,9937,10078,10228,10241,10414,10589,10816,11110,11152,11639,11700"}}
+{"timestamp":1714359024.7242596,"name":"online","context":{"idset":"9379,11194,11721"}}
+{"timestamp":1714359024.9865718,"name":"online","context":{"idset":"103,176,6573,7962,8145,8170,8360,8572,8639,9443,9467,9601,9725,9734,9740,9810,10152,10380,10493,10776,11131,11164,11640,11849"}}
+{"timestamp":1714359025.0998988,"name":"online","context":{"idset":"98,298,7833,8366,8849,9205,9914,10047,10177,10307,10573,10688,10717,11058,11172,11195,11726,11736"}}
+{"timestamp":1714359025.2955272,"name":"online","context":{"idset":"1231,6636,8088,8149,8584,9048,9150,9364,9402,9478,9531,9812,9823,9890,9924,10050,10133,10192,10425,10466,10646,10694,10696,10784,10802,11087,11406,11436,11664,11697,11706,11715,11762,11826"}}
+{"timestamp":1714359025.4016001,"name":"online","context":{"idset":"871,6616,9881,10310,10772,11511,11691"}}
+{"timestamp":1714359025.5099475,"name":"online","context":{"idset":"6525,7971,8571,9344,9944,9987,10169,11682"}}
+{"timestamp":1714359025.613415,"name":"online","context":{"idset":"8311,8573,9813,9966,10052,10544"}}
+{"timestamp":1714359025.7164783,"name":"online","context":{"idset":"99,6559,6604,8425,8619,8692,9657,9667,10489,10531,10605,10792,10797,10823,10841,11125,11643-11644,11647,11668"}}
+{"timestamp":1714359025.8861144,"name":"online","context":{"idset":"6531,8670,9026,10004,10118,10186,10599,10807"}}
+{"timestamp":1714359026.0014675,"name":"online","context":{"idset":"409,6574,6613,8595,10486,10787,10817,11173,11186"}}
+{"timestamp":1714359026.184943,"name":"online","context":{"idset":"8662"}}
+{"timestamp":1714359026.815182,"name":"online","context":{"idset":"11,21,33,37,39,51"}}
+{"timestamp":1714359027.3035903,"name":"online","context":{"idset":"6518"}}
+{"timestamp":1714359030.9104457,"name":"online","context":{"idset":"10291"}}
+{"timestamp":1714359030.917098,"name":"online","context":{"idset":"11391"}}
+{"timestamp":1714359031.0328195,"name":"online","context":{"idset":"8063,10350"}}
+{"timestamp":1714359031.1506276,"name":"online","context":{"idset":"7803,7885,7896,8192-8193,8252,8832,8847,8878,8888,8935,9074,9182,9209,9215,9291,9304,10371,10471,10473,10480,11053,11099,11381,11393,11450,11459,11497,11774,11784,11796"}}
+{"timestamp":1714359031.2634492,"name":"online","context":{"idset":"1189,7804,7849,7860,7913,7975,8044,8075,8130,8285-8286,8856,8875,8922,9091,9142,9200,9294,9308,9330,9489,9763,10240,10346,10421,10642,10649-10650,10699,11023,11104,11106,11430,11463,11505,11507,11576,11586,11596,11606-11607,11777"}}
+{"timestamp":1714359031.4753597,"name":"online","context":{"idset":"418,1184,1207,1221,7810,7852,7914,7931,7943,7961,7964,7981,7991,8005,8072,8080,8082,8118,8124,8141,8143,8184,8188,8231,8246,8249,8283,8288,8297,8326,8349,8365,8399,8637,8830,8837,8852,8861-8862,8864,8883,8965,9053,9073,9081,9093,9122,9133,9141,9147,9155,9163-9164,9169,9236,9256,9321,9433,9465-9466,9500,9517,9544,9549,9584,9587,9589,9629,9634,9655,9662,9732,9835,9876,9929,10251,10261,10280,10335,10339,10364,10381,10403,10444,10470,10538,10550,10616,10625,10647,10691,10707,10720-10721,10729,11036,11043,11067,11076,11387,11407,11410,11470,11476,11482,11486,11512,11539-11540,11582,11590,11678-11679,11686,11759,11767,11782,11785,11793,11811,11836,11878,11881"}}
+{"timestamp":1714359031.6812875,"name":"online","context":{"idset":"208,287,412,6612,7801,7834,7843,7920,7968,7993,8161,8191,8202,8303,8342,8605,8839,8859,8903,8956,8982,9045,9054,9170,9213,9280,9319,9421,9472,9487,9492,9506,9598,9669,9682,9704,9708,9738,9744,9752,9761,9879,9936,9994,10065,10096,10122,10229,10313,10340,10382,10397,10462,10484,10517,10533,10600-10601,10617,10653,10686,10780,11011,11046,11078,11114,11492,11584,11822"}}
+{"timestamp":1714359031.7876265,"name":"online","context":{"idset":"177,189,203,211,224,236,285,306,341,900,7806,7990,8123,8179,8318,8374,8383,8598,8669,8998,9023,9096,9123,9216,9241,9264,9269,9334,9342,9383,9438,9446,9528,9539,9550,9582,9611,9622,9784,9800,9834,9847,9899,9902,9945,9958,9965,10005,10011,10027,10038,10068,10074,10093,10129,10303,10433,10499,10504,10611,10770,11117,11142,11147,11412,11703,11724,11752-11753"}}
+{"timestamp":1714359031.9863513,"name":"online","context":{"idset":"101,116,159,182,190,196,244,252,296,312,407,579,6545,6560,6568,6570-6571,6588,6599,6641,7830,7876,8338,8347,8351,8363,8378,8410-8411,8415,8418,8434,8581,8590,8596,8625,8955,8983,9001,9014,9017,9350,9419,9423,9595,9625,9648,9650,9686,9777,9782,9830,9839,9851,9873,9889,9916,10001,10008,10029,10036,10042,10054,10063,10111-10112,10137,10327,10441,10485,10528,10537,10549,10567,10762,10764,10821,10832,10842,11127,11151,11168,11174,11493,11500,11627,11649,11711,11725,11727"}}
+{"timestamp":1714359032.0886195,"name":"online","context":{"idset":"174,228,234,310,345,8409,9028,9621,9681,9757,10217,10507,10795,11625,11758,11761"}}
+{"timestamp":1714359032.2855291,"name":"online","context":{"idset":"232,6576,6609,6620,6633,8023,8597,8608,8675,8679,9409,9413,9642,9786,9841,9967,10058-10059,10108-10110,10140,10143,10155,10222,10509,10514-10515,10539,10746,10765,10768,10839,10849,11150,11626,11631,11750"}}
+{"timestamp":1714359032.3879299,"name":"online","context":{"idset":"6540,6548,6601,8602,10190,10202,10224"}}
+{"timestamp":1714359032.5787604,"name":"online","context":{"idset":"8610,10114,10266,10810,11629"}}
+{"timestamp":1714359032.7777181,"name":"online","context":{"idset":"6534,6538,6542,8115,8186,9359,10511"}}
+{"timestamp":1714359032.880583,"name":"online","context":{"idset":"9607"}}
+{"timestamp":1714359032.9899325,"name":"online","context":{"idset":"8,15,42,52,9283,9789,10387,11612"}}
+{"timestamp":1714359033.1098173,"name":"online","context":{"idset":"337,10015,10124,10146"}}
+{"timestamp":1714359033.2977498,"name":"online","context":{"idset":"16,24,204,8618,8920,9230,9749,11424"}}
+{"timestamp":1714359033.4058781,"name":"online","context":{"idset":"29,60,6584,8324,9689,9796,9866,11690"}}
+{"timestamp":1714359033.536994,"name":"online","context":{"idset":"58,10227,11160"}}
+{"timestamp":1714359033.6531098,"name":"online","context":{"idset":"9971,10850"}}
+{"timestamp":1714359033.8413007,"name":"online","context":{"idset":"10019"}}
+{"timestamp":1714359033.9533901,"name":"online","context":{"idset":"2"}}
+{"timestamp":1714359209.5765023,"name":"online","context":{"idset":"574"}}
+{"timestamp":1714359307.2950916,"name":"online","context":{"idset":"9064"}}
+{"timestamp":1714359311.0108857,"name":"online","context":{"idset":"9065"}}
+{"timestamp":1714359312.1237895,"name":"online","context":{"idset":"9061"}}
+{"timestamp":1714359313.4625425,"name":"online","context":{"idset":"9067"}}
+{"timestamp":1714359322.1793737,"name":"online","context":{"idset":"9066"}}
+{"timestamp":1714359323.0958657,"name":"online","context":{"idset":"9062"}}
+{"timestamp":1714359323.7909131,"name":"online","context":{"idset":"9063"}}
+{"timestamp":1714392450.8202796,"name":"offline","context":{"idset":"10229"}}
+{"timestamp":1714392450.9056621,"name":"offline","context":{"idset":"10230"}}
+{"timestamp":1714392452.8263011,"name":"offline","context":{"idset":"10231"}}
+{"timestamp":1714392452.8301799,"name":"offline","context":{"idset":"10232"}}
+{"timestamp":1714392452.8340828,"name":"offline","context":{"idset":"10233"}}
+{"timestamp":1714392452.8379781,"name":"offline","context":{"idset":"10234"}}
+{"timestamp":1714392452.9188476,"name":"offline","context":{"idset":"10235"}}
+{"timestamp":1714392454.8232992,"name":"offline","context":{"idset":"10237"}}
+{"timestamp":1714392454.8266761,"name":"offline","context":{"idset":"10238"}}
+{"timestamp":1714392454.8300269,"name":"offline","context":{"idset":"10239"}}
+{"timestamp":1714392454.9108803,"name":"offline","context":{"idset":"10240"}}
+{"timestamp":1714392456.8266103,"name":"offline","context":{"idset":"10241"}}
+{"timestamp":1714392456.8300312,"name":"offline","context":{"idset":"10242"}}
+{"timestamp":1714392456.8333621,"name":"offline","context":{"idset":"10243"}}
+{"timestamp":1714392456.9052484,"name":"offline","context":{"idset":"10244"}}
+{"timestamp":1714392458.8241227,"name":"offline","context":{"idset":"10245"}}
+{"timestamp":1714392458.8287294,"name":"offline","context":{"idset":"10246"}}
+{"timestamp":1714392458.8335762,"name":"offline","context":{"idset":"10247"}}
+{"timestamp":1714392458.9061136,"name":"offline","context":{"idset":"10248"}}
+{"timestamp":1714392460.824748,"name":"offline","context":{"idset":"10249"}}
+{"timestamp":1714392460.828124,"name":"offline","context":{"idset":"10250"}}
+{"timestamp":1714392460.8314607,"name":"offline","context":{"idset":"10251"}}
+{"timestamp":1714392460.9045253,"name":"offline","context":{"idset":"10252"}}
+{"timestamp":1714392462.829294,"name":"offline","context":{"idset":"10253"}}
+{"timestamp":1714392462.8336465,"name":"offline","context":{"idset":"10254"}}
+{"timestamp":1714392462.8381479,"name":"offline","context":{"idset":"10255"}}
+{"timestamp":1714392462.8423641,"name":"offline","context":{"idset":"10256"}}
+{"timestamp":1714392462.9116194,"name":"offline","context":{"idset":"10257"}}
+{"timestamp":1714392464.8230295,"name":"offline","context":{"idset":"10258"}}
+{"timestamp":1714392464.8275735,"name":"offline","context":{"idset":"10259"}}
+{"timestamp":1714392464.8327296,"name":"offline","context":{"idset":"10260"}}
+{"timestamp":1714392464.9056258,"name":"offline","context":{"idset":"10261"}}
+{"timestamp":1714392466.8235879,"name":"offline","context":{"idset":"10262"}}
+{"timestamp":1714392466.8274224,"name":"offline","context":{"idset":"10263"}}
+{"timestamp":1714392466.8313732,"name":"offline","context":{"idset":"10264"}}
+{"timestamp":1714392466.9053164,"name":"offline","context":{"idset":"10265"}}
+{"timestamp":1714392468.832437,"name":"offline","context":{"idset":"10266"}}
+{"timestamp":1714392468.8403471,"name":"offline","context":{"idset":"10267"}}
+{"timestamp":1714392468.8454576,"name":"offline","context":{"idset":"10268"}}
+{"timestamp":1714392468.9401941,"name":"offline","context":{"idset":"10269"}}
+{"timestamp":1714392470.8276224,"name":"offline","context":{"idset":"10270"}}
+{"timestamp":1714392470.8309798,"name":"offline","context":{"idset":"10271"}}
+{"timestamp":1714392470.8342776,"name":"offline","context":{"idset":"10272"}}
+{"timestamp":1714392470.9081218,"name":"offline","context":{"idset":"10273"}}
+{"timestamp":1714392472.8298602,"name":"offline","context":{"idset":"10274"}}
+{"timestamp":1714392472.8340321,"name":"offline","context":{"idset":"10275"}}
+{"timestamp":1714392472.8382649,"name":"offline","context":{"idset":"10276"}}
+{"timestamp":1714392472.8423946,"name":"offline","context":{"idset":"10277"}}
+{"timestamp":1714392472.9253819,"name":"offline","context":{"idset":"10278"}}
+{"timestamp":1714392474.8385026,"name":"offline","context":{"idset":"10279"}}
+{"timestamp":1714392474.8456805,"name":"offline","context":{"idset":"10280"}}
+{"timestamp":1714392474.8501673,"name":"offline","context":{"idset":"10281"}}
+{"timestamp":1714392474.9132128,"name":"offline","context":{"idset":"10282"}}
+{"timestamp":1714392476.8397803,"name":"offline","context":{"idset":"10283"}}
+{"timestamp":1714392476.8528605,"name":"offline","context":{"idset":"10284"}}
+{"timestamp":1714392476.9102058,"name":"offline","context":{"idset":"10285"}}
+{"timestamp":1714392478.8301263,"name":"offline","context":{"idset":"10286"}}
+{"timestamp":1714392478.8354139,"name":"offline","context":{"idset":"10287"}}
+{"timestamp":1714392478.8397589,"name":"offline","context":{"idset":"10288"}}
+{"timestamp":1714392478.8446913,"name":"offline","context":{"idset":"10289"}}
+{"timestamp":1714392478.8487289,"name":"offline","context":{"idset":"10290"}}
+{"timestamp":1714392478.9055724,"name":"offline","context":{"idset":"10291"}}
+{"timestamp":1714392480.8251235,"name":"offline","context":{"idset":"10292"}}
+{"timestamp":1714392480.8317382,"name":"offline","context":{"idset":"10293"}}
+{"timestamp":1714392480.9116135,"name":"offline","context":{"idset":"10294"}}
+{"timestamp":1714392482.8273461,"name":"offline","context":{"idset":"10295"}}
+{"timestamp":1714392482.831404,"name":"offline","context":{"idset":"10296"}}
+{"timestamp":1714392482.8356757,"name":"offline","context":{"idset":"10297"}}
+{"timestamp":1714392482.839314,"name":"offline","context":{"idset":"10298"}}
+{"timestamp":1714392482.9177792,"name":"offline","context":{"idset":"10299"}}
+{"timestamp":1714392484.8134787,"name":"offline","context":{"idset":"10300"}}
+{"timestamp":1714392484.9022412,"name":"offline","context":{"idset":"10301"}}
+{"timestamp":1714392486.8323498,"name":"offline","context":{"idset":"10302"}}
+{"timestamp":1714392486.8387527,"name":"offline","context":{"idset":"10303"}}
+{"timestamp":1714392486.8451214,"name":"offline","context":{"idset":"10304"}}
+{"timestamp":1714392486.8512237,"name":"offline","context":{"idset":"10305"}}
+{"timestamp":1714392486.9223564,"name":"offline","context":{"idset":"10306"}}
+{"timestamp":1714392488.8305774,"name":"offline","context":{"idset":"10307"}}
+{"timestamp":1714392488.8344445,"name":"offline","context":{"idset":"10308"}}
+{"timestamp":1714392488.8382487,"name":"offline","context":{"idset":"10309"}}
+{"timestamp":1714392488.9163446,"name":"offline","context":{"idset":"10310"}}
+{"timestamp":1714392490.8237698,"name":"offline","context":{"idset":"10311"}}
+{"timestamp":1714392490.8278229,"name":"offline","context":{"idset":"10312"}}
+{"timestamp":1714392490.8317034,"name":"offline","context":{"idset":"10313"}}
+{"timestamp":1714392490.9046333,"name":"offline","context":{"idset":"10314"}}
+{"timestamp":1714392492.8321719,"name":"offline","context":{"idset":"10315"}}
+{"timestamp":1714392492.8364234,"name":"offline","context":{"idset":"10316"}}
+{"timestamp":1714392492.8409562,"name":"offline","context":{"idset":"10317"}}
+{"timestamp":1714392492.9306972,"name":"offline","context":{"idset":"10318"}}
+{"timestamp":1714392494.8216815,"name":"offline","context":{"idset":"10320"}}
+{"timestamp":1714392494.8250082,"name":"offline","context":{"idset":"10321"}}
+{"timestamp":1714392494.8282783,"name":"offline","context":{"idset":"10322"}}
+{"timestamp":1714392494.9040623,"name":"offline","context":{"idset":"10323"}}
+{"timestamp":1714392496.8262751,"name":"offline","context":{"idset":"10324"}}
+{"timestamp":1714392496.8295901,"name":"offline","context":{"idset":"10325"}}
+{"timestamp":1714392496.8328745,"name":"offline","context":{"idset":"10326"}}
+{"timestamp":1714392496.9062319,"name":"offline","context":{"idset":"10327"}}
+{"timestamp":1714392498.8253751,"name":"offline","context":{"idset":"10328"}}
+{"timestamp":1714392498.8295281,"name":"offline","context":{"idset":"10329"}}
+{"timestamp":1714392498.8336296,"name":"offline","context":{"idset":"10330"}}
+{"timestamp":1714392498.8376856,"name":"offline","context":{"idset":"10331"}}
+{"timestamp":1714392498.915462,"name":"offline","context":{"idset":"10332"}}
+{"timestamp":1714392500.8316357,"name":"offline","context":{"idset":"10333"}}
+{"timestamp":1714392500.8353326,"name":"offline","context":{"idset":"10334"}}
+{"timestamp":1714392500.8391662,"name":"offline","context":{"idset":"10335"}}
+{"timestamp":1714392500.842875,"name":"offline","context":{"idset":"10336"}}
+{"timestamp":1714392500.924072,"name":"offline","context":{"idset":"10337"}}
+{"timestamp":1714392502.8296666,"name":"offline","context":{"idset":"10338"}}
+{"timestamp":1714392502.8331926,"name":"offline","context":{"idset":"10339"}}
+{"timestamp":1714392502.8366673,"name":"offline","context":{"idset":"10340"}}
+{"timestamp":1714392502.8400767,"name":"offline","context":{"idset":"10341"}}
+{"timestamp":1714392502.913162,"name":"offline","context":{"idset":"10342"}}
+{"timestamp":1714392504.8274131,"name":"offline","context":{"idset":"10236"}}
+{"timestamp":1714392504.8309121,"name":"offline","context":{"idset":"10343"}}
+{"timestamp":1714392504.8344576,"name":"offline","context":{"idset":"10344"}}
+{"timestamp":1714392504.8379502,"name":"offline","context":{"idset":"10345"}}
+{"timestamp":1714392504.9149668,"name":"offline","context":{"idset":"10346"}}
+{"timestamp":1714392506.8391287,"name":"offline","context":{"idset":"10347"}}
+{"timestamp":1714392506.8429379,"name":"offline","context":{"idset":"10348"}}
+{"timestamp":1714392506.8463516,"name":"offline","context":{"idset":"10349"}}
+{"timestamp":1714392506.8497462,"name":"offline","context":{"idset":"10350"}}
+{"timestamp":1714392506.9338214,"name":"offline","context":{"idset":"10351"}}
+{"timestamp":1714392508.8291795,"name":"offline","context":{"idset":"10352"}}
+{"timestamp":1714392508.8326349,"name":"offline","context":{"idset":"10353"}}
+{"timestamp":1714392508.9041064,"name":"offline","context":{"idset":"10354"}}
+{"timestamp":1714392510.8213823,"name":"offline","context":{"idset":"10319"}}
+{"timestamp":1714392510.8254771,"name":"offline","context":{"idset":"10355"}}
+{"timestamp":1714392510.9079928,"name":"offline","context":{"idset":"10356"}}
+{"timestamp":1714393188.8171561,"name":"offline","context":{"idset":"11381"}}
+{"timestamp":1714393188.9071896,"name":"offline","context":{"idset":"11382"}}
+{"timestamp":1714393190.8268712,"name":"offline","context":{"idset":"11383"}}
+{"timestamp":1714393190.8346484,"name":"offline","context":{"idset":"11384"}}
+{"timestamp":1714393190.8424203,"name":"offline","context":{"idset":"11385"}}
+{"timestamp":1714393190.9203613,"name":"offline","context":{"idset":"11386"}}
+{"timestamp":1714393192.8287976,"name":"offline","context":{"idset":"11387"}}
+{"timestamp":1714393192.8343103,"name":"offline","context":{"idset":"11388"}}
+{"timestamp":1714393192.8405247,"name":"offline","context":{"idset":"11389"}}
+{"timestamp":1714393192.846734,"name":"offline","context":{"idset":"11390"}}
+{"timestamp":1714393192.9121261,"name":"offline","context":{"idset":"11391"}}
+{"timestamp":1714393194.8251333,"name":"offline","context":{"idset":"11392"}}
+{"timestamp":1714393194.8332207,"name":"offline","context":{"idset":"11393"}}
+{"timestamp":1714393194.8396754,"name":"offline","context":{"idset":"11394"}}
+{"timestamp":1714393194.9052477,"name":"offline","context":{"idset":"11395"}}
+{"timestamp":1714393196.8270535,"name":"offline","context":{"idset":"11396"}}
+{"timestamp":1714393196.8302798,"name":"offline","context":{"idset":"11397"}}
+{"timestamp":1714393196.8334877,"name":"offline","context":{"idset":"11398"}}
+{"timestamp":1714393196.9042237,"name":"offline","context":{"idset":"11399"}}
+{"timestamp":1714393198.8272908,"name":"offline","context":{"idset":"11400"}}
+{"timestamp":1714393198.8334877,"name":"offline","context":{"idset":"11401"}}
+{"timestamp":1714393198.8396916,"name":"offline","context":{"idset":"11402"}}
+{"timestamp":1714393198.9051805,"name":"offline","context":{"idset":"11403"}}
+{"timestamp":1714393200.8321819,"name":"offline","context":{"idset":"11404"}}
+{"timestamp":1714393200.8394399,"name":"offline","context":{"idset":"11405"}}
+{"timestamp":1714393200.8466766,"name":"offline","context":{"idset":"11406"}}
+{"timestamp":1714393200.853451,"name":"offline","context":{"idset":"11407"}}
+{"timestamp":1714393200.9281161,"name":"offline","context":{"idset":"11408"}}
+{"timestamp":1714393202.8301067,"name":"offline","context":{"idset":"11410"}}
+{"timestamp":1714393202.8333571,"name":"offline","context":{"idset":"11411"}}
+{"timestamp":1714393202.9040031,"name":"offline","context":{"idset":"11412"}}
+{"timestamp":1714393204.8200123,"name":"offline","context":{"idset":"11413"}}
+{"timestamp":1714393204.8240461,"name":"offline","context":{"idset":"11414"}}
+{"timestamp":1714393204.9062967,"name":"offline","context":{"idset":"11415"}}
+{"timestamp":1714393206.8215356,"name":"offline","context":{"idset":"11416"}}
+{"timestamp":1714393206.8247759,"name":"offline","context":{"idset":"11417"}}
+{"timestamp":1714393206.8279998,"name":"offline","context":{"idset":"11418"}}
+{"timestamp":1714393206.9044998,"name":"offline","context":{"idset":"11419"}}
+{"timestamp":1714393208.8211875,"name":"offline","context":{"idset":"11420"}}
+{"timestamp":1714393208.8251507,"name":"offline","context":{"idset":"11422"}}
+{"timestamp":1714393208.8284185,"name":"offline","context":{"idset":"11423"}}
+{"timestamp":1714393208.9044693,"name":"offline","context":{"idset":"11424"}}
+{"timestamp":1714393210.8360775,"name":"offline","context":{"idset":"11425"}}
+{"timestamp":1714393210.8429368,"name":"offline","context":{"idset":"11426"}}
+{"timestamp":1714393210.8469343,"name":"offline","context":{"idset":"11428"}}
+{"timestamp":1714393210.9176972,"name":"offline","context":{"idset":"11429"}}
+{"timestamp":1714393212.8454669,"name":"offline","context":{"idset":"11430"}}
+{"timestamp":1714393212.8552587,"name":"offline","context":{"idset":"11431"}}
+{"timestamp":1714393212.8612628,"name":"offline","context":{"idset":"11432"}}
+{"timestamp":1714393212.8672378,"name":"offline","context":{"idset":"11433"}}
+{"timestamp":1714393212.9088891,"name":"offline","context":{"idset":"11434"}}
+{"timestamp":1714393214.822727,"name":"offline","context":{"idset":"11435"}}
+{"timestamp":1714393214.8259761,"name":"offline","context":{"idset":"11436"}}
+{"timestamp":1714393214.8297539,"name":"offline","context":{"idset":"11437"}}
+{"timestamp":1714393214.9047213,"name":"offline","context":{"idset":"11438"}}
+{"timestamp":1714393216.8222082,"name":"offline","context":{"idset":"11439"}}
+{"timestamp":1714393216.8298466,"name":"offline","context":{"idset":"11440"}}
+{"timestamp":1714393216.8336635,"name":"offline","context":{"idset":"11441"}}
+{"timestamp":1714393216.9046333,"name":"offline","context":{"idset":"11442"}}
+{"timestamp":1714393218.8261824,"name":"offline","context":{"idset":"11443"}}
+{"timestamp":1714393218.8302009,"name":"offline","context":{"idset":"11444"}}
+{"timestamp":1714393218.8341575,"name":"offline","context":{"idset":"11445"}}
+{"timestamp":1714393218.8378313,"name":"offline","context":{"idset":"11446"}}
+{"timestamp":1714393218.9200597,"name":"offline","context":{"idset":"11447"}}
+{"timestamp":1714393220.8183329,"name":"offline","context":{"idset":"11448"}}
+{"timestamp":1714393220.8208394,"name":"offline","context":{"idset":"11449"}}
+{"timestamp":1714393220.8233047,"name":"offline","context":{"idset":"11450"}}
+{"timestamp":1714393220.9038353,"name":"offline","context":{"idset":"11451"}}
+{"timestamp":1714393222.8310108,"name":"offline","context":{"idset":"11453"}}
+{"timestamp":1714393222.8347716,"name":"offline","context":{"idset":"11455"}}
+{"timestamp":1714393222.8382809,"name":"offline","context":{"idset":"11456"}}
+{"timestamp":1714393222.9051535,"name":"offline","context":{"idset":"11457"}}
+{"timestamp":1714393224.8161891,"name":"offline","context":{"idset":"11458"}}
+{"timestamp":1714393224.9057093,"name":"offline","context":{"idset":"11459"}}
+{"timestamp":1714393226.8282313,"name":"offline","context":{"idset":"11421"}}
+{"timestamp":1714393226.8316197,"name":"offline","context":{"idset":"11460"}}
+{"timestamp":1714393226.8349507,"name":"offline","context":{"idset":"11461"}}
+{"timestamp":1714393226.838238,"name":"offline","context":{"idset":"11462"}}
+{"timestamp":1714393226.8414917,"name":"offline","context":{"idset":"11463"}}
+{"timestamp":1714393226.9267564,"name":"offline","context":{"idset":"11464"}}
+{"timestamp":1714393228.825417,"name":"offline","context":{"idset":"11452"}}
+{"timestamp":1714393228.8293829,"name":"offline","context":{"idset":"11465"}}
+{"timestamp":1714393228.833323,"name":"offline","context":{"idset":"11466"}}
+{"timestamp":1714393228.8372538,"name":"offline","context":{"idset":"11467"}}
+{"timestamp":1714393228.9211411,"name":"offline","context":{"idset":"11468"}}
+{"timestamp":1714393230.8158419,"name":"offline","context":{"idset":"11469"}}
+{"timestamp":1714393230.9049194,"name":"offline","context":{"idset":"11471"}}
+{"timestamp":1714393232.8237796,"name":"offline","context":{"idset":"11472"}}
+{"timestamp":1714393232.8272829,"name":"offline","context":{"idset":"11473"}}
+{"timestamp":1714393232.8304372,"name":"offline","context":{"idset":"11474"}}
+{"timestamp":1714393232.8335714,"name":"offline","context":{"idset":"11475"}}
+{"timestamp":1714393232.9120958,"name":"offline","context":{"idset":"11476"}}
+{"timestamp":1714393234.8221354,"name":"offline","context":{"idset":"11477"}}
+{"timestamp":1714393234.8300548,"name":"offline","context":{"idset":"11478"}}
+{"timestamp":1714393234.9050188,"name":"offline","context":{"idset":"11479"}}
+{"timestamp":1714393236.8262801,"name":"offline","context":{"idset":"11480"}}
+{"timestamp":1714393236.8302414,"name":"offline","context":{"idset":"11481"}}
+{"timestamp":1714393236.8341691,"name":"offline","context":{"idset":"11482"}}
+{"timestamp":1714393236.8381014,"name":"offline","context":{"idset":"11483"}}
+{"timestamp":1714393236.9136031,"name":"offline","context":{"idset":"11484"}}
+{"timestamp":1714393238.8190188,"name":"offline","context":{"idset":"11485"}}
+{"timestamp":1714393238.8222544,"name":"offline","context":{"idset":"11486"}}
+{"timestamp":1714393238.9037125,"name":"offline","context":{"idset":"11487"}}
+{"timestamp":1714393240.8241773,"name":"offline","context":{"idset":"11488"}}
+{"timestamp":1714393240.8281479,"name":"offline","context":{"idset":"11489"}}
+{"timestamp":1714393240.8320682,"name":"offline","context":{"idset":"11490"}}
+{"timestamp":1714393240.9064345,"name":"offline","context":{"idset":"11491"}}
+{"timestamp":1714393242.8290474,"name":"offline","context":{"idset":"11492"}}
+{"timestamp":1714393242.8330195,"name":"offline","context":{"idset":"11493"}}
+{"timestamp":1714393242.836931,"name":"offline","context":{"idset":"11494"}}
+{"timestamp":1714393242.9411423,"name":"offline","context":{"idset":"11495"}}
+{"timestamp":1714393244.8333995,"name":"offline","context":{"idset":"11496"}}
+{"timestamp":1714393244.8365626,"name":"offline","context":{"idset":"11497"}}
+{"timestamp":1714393244.8397155,"name":"offline","context":{"idset":"11498"}}
+{"timestamp":1714393244.8428411,"name":"offline","context":{"idset":"11499"}}
+{"timestamp":1714393244.8459578,"name":"offline","context":{"idset":"11500"}}
+{"timestamp":1714393244.9227078,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1714393246.8223946,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1714393246.8263364,"name":"offline","context":{"idset":"11503"}}
+{"timestamp":1714393246.8302414,"name":"offline","context":{"idset":"11504"}}
+{"timestamp":1714393246.9050238,"name":"offline","context":{"idset":"11506"}}
+{"timestamp":1714393248.8192842,"name":"offline","context":{"idset":"11470"}}
+{"timestamp":1714393248.8231995,"name":"offline","context":{"idset":"11507"}}
+{"timestamp":1714393248.9047012,"name":"offline","context":{"idset":"11508"}}
+{"timestamp":1714393250.8160074,"name":"offline","context":{"idset":"11409"}}
+{"timestamp":1714393250.9049242,"name":"offline","context":{"idset":"11505"}}
+{"timestamp":1714393262.9081366,"name":"offline","context":{"idset":"11427"}}
+{"timestamp":1714393274.9040351,"name":"offline","context":{"idset":"11454"}}
+{"timestamp":1714398170.9044027,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1714398349.4424365,"name":"undrain","context":{"idset":"11381-11508"}}
+{"timestamp":1714398494.5938983,"name":"online","context":{"idset":"11493,11503"}}
+{"timestamp":1714398494.7168448,"name":"online","context":{"idset":"11497,11502,11507-11508"}}
+{"timestamp":1714398494.818686,"name":"online","context":{"idset":"11504"}}
+{"timestamp":1714398495.0122275,"name":"online","context":{"idset":"11499-11500"}}
+{"timestamp":1714398495.1233962,"name":"online","context":{"idset":"11498,11505"}}
+{"timestamp":1714398495.3297458,"name":"online","context":{"idset":"11494-11496,11501,11506"}}
+{"timestamp":1714398578.9053085,"name":"offline","context":{"idset":"8591"}}
+{"timestamp":1714398580.9043725,"name":"offline","context":{"idset":"8592"}}
+{"timestamp":1714398704.903882,"name":"offline","context":{"idset":"9501"}}
+{"timestamp":1714398706.9052291,"name":"offline","context":{"idset":"9502"}}
+{"timestamp":1714398740.8276842,"name":"offline","context":{"idset":"899"}}
+{"timestamp":1714398740.9079192,"name":"offline","context":{"idset":"900"}}
+{"timestamp":1714399015.8868568,"name":"offline","context":{"idset":"10807"}}
+{"timestamp":1714399016.0105891,"name":"offline","context":{"idset":"10744"}}
+{"timestamp":1714399016.1525569,"name":"offline","context":{"idset":"10745"}}
+{"timestamp":1714399016.2517025,"name":"offline","context":{"idset":"10860"}}
+{"timestamp":1714399016.26998,"name":"offline","context":{"idset":"10747"}}
+{"timestamp":1714399016.2751107,"name":"offline","context":{"idset":"10805"}}
+{"timestamp":1714399016.3731046,"name":"offline","context":{"idset":"10754"}}
+{"timestamp":1714399016.4980025,"name":"offline","context":{"idset":"10746"}}
+{"timestamp":1714399016.5961187,"name":"offline","context":{"idset":"10753"}}
+{"timestamp":1714399016.7829876,"name":"offline","context":{"idset":"10756"}}
+{"timestamp":1714399016.7877545,"name":"offline","context":{"idset":"10843"}}
+{"timestamp":1714399016.8931634,"name":"offline","context":{"idset":"10829"}}
+{"timestamp":1714399016.9983106,"name":"offline","context":{"idset":"10851"}}
+{"timestamp":1714399017.005765,"name":"offline","context":{"idset":"10741"}}
+{"timestamp":1714399017.0096204,"name":"offline","context":{"idset":"10854"}}
+{"timestamp":1714399017.0131471,"name":"offline","context":{"idset":"10823"}}
+{"timestamp":1714399017.0167263,"name":"offline","context":{"idset":"10749"}}
+{"timestamp":1714399017.0204725,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1714399017.0243995,"name":"offline","context":{"idset":"10827"}}
+{"timestamp":1714399017.0279675,"name":"offline","context":{"idset":"10825"}}
+{"timestamp":1714399017.1512246,"name":"offline","context":{"idset":"10862"}}
+{"timestamp":1714399017.1805317,"name":"offline","context":{"idset":"10849"}}
+{"timestamp":1714399017.2083817,"name":"offline","context":{"idset":"10751"}}
+{"timestamp":1714399017.214258,"name":"offline","context":{"idset":"10868"}}
+{"timestamp":1714399017.2180848,"name":"offline","context":{"idset":"10841"}}
+{"timestamp":1714399017.2215927,"name":"offline","context":{"idset":"10817"}}
+{"timestamp":1714399017.3165975,"name":"offline","context":{"idset":"10742"}}
+{"timestamp":1714399017.4235289,"name":"offline","context":{"idset":"10856"}}
+{"timestamp":1714399017.4480784,"name":"offline","context":{"idset":"10839"}}
+{"timestamp":1714399017.4561057,"name":"offline","context":{"idset":"10821"}}
+{"timestamp":1714399017.4762363,"name":"offline","context":{"idset":"10755"}}
+{"timestamp":1714399017.5547912,"name":"offline","context":{"idset":"10752"}}
+{"timestamp":1714399017.670675,"name":"offline","context":{"idset":"10819"}}
+{"timestamp":1714399017.6767902,"name":"offline","context":{"idset":"10750"}}
+{"timestamp":1714399017.6807687,"name":"offline","context":{"idset":"10835"}}
+{"timestamp":1714399017.685559,"name":"offline","context":{"idset":"10743"}}
+{"timestamp":1714399017.690506,"name":"offline","context":{"idset":"10847"}}
+{"timestamp":1714399017.694484,"name":"offline","context":{"idset":"10866"}}
+{"timestamp":1714399017.7871244,"name":"offline","context":{"idset":"10833"}}
+{"timestamp":1714399017.8596883,"name":"offline","context":{"idset":"10815"}}
+{"timestamp":1714399017.9509828,"name":"offline","context":{"idset":"10864"}}
+{"timestamp":1714399018.1630397,"name":"offline","context":{"idset":"10837"}}
+{"timestamp":1714399018.2669017,"name":"offline","context":{"idset":"10845"}}
+{"timestamp":1714399018.2711785,"name":"offline","context":{"idset":"10813"}}
+{"timestamp":1714399018.2760532,"name":"offline","context":{"idset":"10811"}}
+{"timestamp":1714399018.3754077,"name":"offline","context":{"idset":"10858"}}
+{"timestamp":1714399018.535517,"name":"offline","context":{"idset":"10809"}}
+{"timestamp":1714399018.6343975,"name":"offline","context":{"idset":"10831"}}
+{"timestamp":1714399508.9050612,"name":"offline","context":{"idset":"8565"}}
+{"timestamp":1714399510.9046507,"name":"offline","context":{"idset":"8566"}}
+{"timestamp":1714399766.819927,"name":"offline","context":{"idset":"8673"}}
+{"timestamp":1714399766.9053042,"name":"offline","context":{"idset":"8674"}}
+{"timestamp":1714399840.9045081,"name":"offline","context":{"idset":"7816"}}
+{"timestamp":1714399852.9052851,"name":"offline","context":{"idset":"7815"}}
+{"timestamp":1714399865.3280919,"name":"online","context":{"idset":"795"}}
+{"timestamp":1714399902.1088226,"name":"undrain","context":{"idset":"795-796"}}
+{"timestamp":1714400060.8183424,"name":"offline","context":{"idset":"10357"}}
+{"timestamp":1714400060.9045968,"name":"offline","context":{"idset":"10358"}}
+{"timestamp":1714400136.2705405,"name":"drain","context":{"idset":"7815-7816","reason":"epilog failed for jobid fsKqngGHQfR","overwrite":0}}
+{"timestamp":1714400152.8265157,"name":"offline","context":{"idset":"10359"}}
+{"timestamp":1714400152.830061,"name":"offline","context":{"idset":"10360"}}
+{"timestamp":1714400152.8336754,"name":"offline","context":{"idset":"10361"}}
+{"timestamp":1714400152.9042103,"name":"offline","context":{"idset":"10362"}}
+{"timestamp":1714400160.8338947,"name":"offline","context":{"idset":"10365"}}
+{"timestamp":1714400160.8380656,"name":"offline","context":{"idset":"10366"}}
+{"timestamp":1714400160.8424361,"name":"offline","context":{"idset":"10367"}}
+{"timestamp":1714400160.8452682,"name":"offline","context":{"idset":"10368"}}
+{"timestamp":1714400160.8487539,"name":"offline","context":{"idset":"10369"}}
+{"timestamp":1714400160.8526242,"name":"offline","context":{"idset":"10370"}}
+{"timestamp":1714400160.8560083,"name":"offline","context":{"idset":"10371"}}
+{"timestamp":1714400160.9051592,"name":"offline","context":{"idset":"10372"}}
+{"timestamp":1714400172.8217418,"name":"offline","context":{"idset":"10363"}}
+{"timestamp":1714400172.8232336,"name":"drain","context":{"idset":"10357-10372","reason":"epilog failed for jobid fsKqrrXr3dh","overwrite":0}}
+{"timestamp":1714400172.9050105,"name":"offline","context":{"idset":"10364"}}
+{"timestamp":1714400323.5007102,"name":"offline","context":{"idset":"9365"}}
+{"timestamp":1714400323.6321695,"name":"offline","context":{"idset":"9366"}}
+{"timestamp":1714400881.3374882,"name":"offline","context":{"idset":"11509"}}
+{"timestamp":1714400933.5121818,"name":"offline","context":{"idset":"11510"}}
+{"timestamp":1714402118.0312424,"name":"offline","context":{"idset":"11511"}}
+{"timestamp":1714402118.035228,"name":"offline","context":{"idset":"11512"}}
+{"timestamp":1714402118.8901393,"name":"offline","context":{"idset":"11513"}}
+{"timestamp":1714402118.9570308,"name":"offline","context":{"idset":"11514"}}
+{"timestamp":1714402118.9608815,"name":"offline","context":{"idset":"11515"}}
+{"timestamp":1714402118.9647152,"name":"offline","context":{"idset":"11516"}}
+{"timestamp":1714402119.0328486,"name":"offline","context":{"idset":"11517"}}
+{"timestamp":1714402122.0500832,"name":"offline","context":{"idset":"11518"}}
+{"timestamp":1714402122.0566304,"name":"offline","context":{"idset":"11519"}}
+{"timestamp":1714402122.0631967,"name":"offline","context":{"idset":"11520"}}
+{"timestamp":1714402122.0694923,"name":"offline","context":{"idset":"11521"}}
+{"timestamp":1714402122.9174047,"name":"offline","context":{"idset":"11522"}}
+{"timestamp":1714402124.0283248,"name":"offline","context":{"idset":"11523"}}
+{"timestamp":1714402124.7255321,"name":"offline","context":{"idset":"11524"}}
+{"timestamp":1714402801.9082179,"name":"online","context":{"idset":"9570"}}
+{"timestamp":1714403289.4855089,"name":"undrain","context":{"idset":"8107-8108,8159"}}
+{"timestamp":1714404107.7787824,"name":"online","context":{"idset":"793-794"}}
+{"timestamp":1714404188.9851568,"name":"undrain","context":{"idset":"793-794"}}
+{"timestamp":1714405644.0382776,"name":"offline","context":{"idset":"11525"}}
+{"timestamp":1714405644.0420151,"name":"offline","context":{"idset":"11526"}}
+{"timestamp":1714405644.7116086,"name":"offline","context":{"idset":"11527"}}
+{"timestamp":1714405644.7650404,"name":"offline","context":{"idset":"11528"}}
+{"timestamp":1714405644.7777956,"name":"offline","context":{"idset":"11529"}}
+{"timestamp":1714405644.7810547,"name":"offline","context":{"idset":"11530"}}
+{"timestamp":1714405644.784354,"name":"offline","context":{"idset":"11531"}}
+{"timestamp":1714405644.7877512,"name":"offline","context":{"idset":"11532"}}
+{"timestamp":1714405644.8679464,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1714405648.0307684,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1714405648.6998103,"name":"offline","context":{"idset":"11536"}}
+{"timestamp":1714405648.7623646,"name":"offline","context":{"idset":"11537"}}
+{"timestamp":1714405648.7752428,"name":"offline","context":{"idset":"11538"}}
+{"timestamp":1714405648.78109,"name":"offline","context":{"idset":"11539"}}
+{"timestamp":1714405648.831877,"name":"offline","context":{"idset":"11540"}}
+{"timestamp":1714406609.5511136,"name":"online","context":{"idset":"10134"}}
+{"timestamp":1714407530.7178955,"name":"online","context":{"idset":"796"}}
+{"timestamp":1714407648.8676572,"name":"online","context":{"idset":"8014"}}
+{"timestamp":1714407650.748054,"name":"online","context":{"idset":"8048"}}
+{"timestamp":1714409796.0011342,"name":"online","context":{"idset":"10744,10749"}}
+{"timestamp":1714409796.0233638,"name":"online","context":{"idset":"10754"}}
+{"timestamp":1714409796.7297282,"name":"drain","context":{"idset":"10860","reason":"broker was unresponsive"}}
+{"timestamp":1714409796.7331731,"name":"online","context":{"idset":"10748,10807"}}
+{"timestamp":1714409796.7366538,"name":"online","context":{"idset":"10743,10752-10753,10809"}}
+{"timestamp":1714409796.739877,"name":"online","context":{"idset":"10742"}}
+{"timestamp":1714409796.8650663,"name":"online","context":{"idset":"10756"}}
+{"timestamp":1714409796.9818683,"name":"online","context":{"idset":"10745,10847"}}
+{"timestamp":1714409797.1863744,"name":"online","context":{"idset":"10741,10746,10817,10851"}}
+{"timestamp":1714409797.4784267,"name":"online","context":{"idset":"10747,10750-10751,10805,10813,10821,10825,10833"}}
+{"timestamp":1714409797.6915021,"name":"online","context":{"idset":"10819,10827,10829"}}
+{"timestamp":1714409797.8794999,"name":"online","context":{"idset":"10811,10823,10837,10839,10841,10849,10864"}}
+{"timestamp":1714409797.9835472,"name":"online","context":{"idset":"10755,10835,10862,10866"}}
+{"timestamp":1714409798.8653276,"name":"online","context":{"idset":"10815,10831,10843,10845"}}
+{"timestamp":1714409798.8684554,"name":"online","context":{"idset":"10858,10868"}}
+{"timestamp":1714409798.8718112,"name":"online","context":{"idset":"10854,10856,10860"}}
+{"timestamp":1714409897.0292034,"name":"undrain","context":{"idset":"10860"}}
+{"timestamp":1714409939.1243632,"name":"online","context":{"idset":"8106"}}
+{"timestamp":1714409943.4088252,"name":"drain","context":{"idset":"8107-8108,8159","reason":"prolog failed for jobid fsLFReXBAvj","overwrite":0}}
+{"timestamp":1714410687.4647992,"name":"online","context":{"idset":"10205"}}
+{"timestamp":1714412348.0341525,"name":"offline","context":{"idset":"11573"}}
+{"timestamp":1714412348.8340089,"name":"offline","context":{"idset":"11574"}}
+{"timestamp":1714412348.8907971,"name":"offline","context":{"idset":"11575"}}
+{"timestamp":1714412348.9057808,"name":"offline","context":{"idset":"11576"}}
+{"timestamp":1714412348.909636,"name":"offline","context":{"idset":"11577"}}
+{"timestamp":1714412348.9134676,"name":"offline","context":{"idset":"11578"}}
+{"timestamp":1714412348.96386,"name":"offline","context":{"idset":"11579"}}
+{"timestamp":1714412352.0353298,"name":"offline","context":{"idset":"11580"}}
+{"timestamp":1714412352.0514441,"name":"offline","context":{"idset":"11581"}}
+{"timestamp":1714412352.0702944,"name":"offline","context":{"idset":"11582"}}
+{"timestamp":1714412352.867924,"name":"offline","context":{"idset":"11583"}}
+{"timestamp":1714412352.918488,"name":"offline","context":{"idset":"11584"}}
+{"timestamp":1714412352.9217372,"name":"offline","context":{"idset":"11585"}}
+{"timestamp":1714412352.9976993,"name":"offline","context":{"idset":"11586"}}
+{"timestamp":1714412356.0291491,"name":"offline","context":{"idset":"11587"}}
+{"timestamp":1714412356.8700037,"name":"offline","context":{"idset":"11588"}}
+{"timestamp":1714412660.7614372,"name":"online","context":{"idset":"6629-6630,8673"}}
+{"timestamp":1714412661.0046105,"name":"online","context":{"idset":"8565"}}
+{"timestamp":1714412661.1882887,"name":"online","context":{"idset":"8566"}}
+{"timestamp":1714412661.302964,"name":"online","context":{"idset":"8674"}}
+{"timestamp":1714412661.5929661,"name":"online","context":{"idset":"8591-8592"}}
+{"timestamp":1714412663.4279766,"name":"drain","context":{"idset":"6629-6630,8565-8566,8591-8592,8673-8674","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714413433.134428,"name":"online","context":{"idset":"9366"}}
+{"timestamp":1714413433.644906,"name":"online","context":{"idset":"9365"}}
+{"timestamp":1714413822.1168549,"name":"offline","context":{"idset":"793"}}
+{"timestamp":1714413824.1162865,"name":"offline","context":{"idset":"794"}}
+{"timestamp":1714414082.5129426,"name":"online","context":{"idset":"9403"}}
+{"timestamp":1714414827.6797206,"name":"online","context":{"idset":"806"}}
+{"timestamp":1714414827.9708667,"name":"online","context":{"idset":"805"}}
+{"timestamp":1714415111.1525495,"name":"online","context":{"idset":"10358,10371"}}
+{"timestamp":1714415111.7161963,"name":"online","context":{"idset":"10357,10362,10370,10372"}}
+{"timestamp":1714415112.0038142,"name":"online","context":{"idset":"10364,10368"}}
+{"timestamp":1714415112.2148128,"name":"online","context":{"idset":"10360,10363,10366-10367,10369"}}
+{"timestamp":1714415112.3233457,"name":"online","context":{"idset":"10365"}}
+{"timestamp":1714415112.5350173,"name":"online","context":{"idset":"10359"}}
+{"timestamp":1714415146.7077227,"name":"undrain","context":{"idset":"10357-10372"}}
+{"timestamp":1714417475.0249739,"name":"online","context":{"idset":"10361"}}
+{"timestamp":1714417794.7447803,"name":"online","context":{"idset":"7815"}}
+{"timestamp":1714417843.1729202,"name":"online","context":{"idset":"7816"}}
+{"timestamp":1714417890.856596,"name":"undrain","context":{"idset":"7815-7816"}}
+{"timestamp":1714418578.8491848,"name":"undrain","context":{"idset":"98"}}
+{"timestamp":1714418583.5450706,"name":"undrain","context":{"idset":"100,102"}}
+{"timestamp":1714418663.5508046,"name":"undrain","context":{"idset":"110-112"}}
+{"timestamp":1714418687.0683348,"name":"undrain","context":{"idset":"114-115,119,123-124,161,184,187,190-191,208,303-304,310,317,319,330,420,573-577,579,808,825-826,871-874,977-978,6629-6630,8107-8108,8159,8181,8183,8237,8278-8303,8305-8308,8565-8566,8591-8592,8673-8674,8821-8832,8834-8917,8919-8963,8973-8992,8995-8998,9001-9002,9005-9017,9019-9027,9054,9075,11589-11593,11595-11607,11609-11617"}}
+{"timestamp":1714419070.8415365,"name":"offline","context":{"idset":"11730"}}
+{"timestamp":1714419070.9753542,"name":"offline","context":{"idset":"11729"}}
+{"timestamp":1714419767.827348,"name":"online","context":{"idset":"833"}}
+{"timestamp":1714419768.839844,"name":"online","context":{"idset":"834"}}
+{"timestamp":1714420202.1174748,"name":"offline","context":{"idset":"795"}}
+{"timestamp":1714420204.6190114,"name":"offline","context":{"idset":"796"}}
+{"timestamp":1714420396.9160812,"name":"online","context":{"idset":"8724"}}
+{"timestamp":1714420396.9200327,"name":"online","context":{"idset":"8723"}}
+{"timestamp":1714420402.8784814,"name":"drain","context":{"idset":"8723-8724","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714420570.6530759,"name":"offline","context":{"idset":"11723"}}
+{"timestamp":1714420572.7921634,"name":"offline","context":{"idset":"11724"}}
+{"timestamp":1714421388.8359644,"name":"online","context":{"idset":"823-824"}}
+{"timestamp":1714421810.0302284,"name":"offline","context":{"idset":"11753"}}
+{"timestamp":1714421810.8666255,"name":"offline","context":{"idset":"11754"}}
+{"timestamp":1714422773.2977872,"name":"offline","context":{"idset":"8197"}}
+{"timestamp":1714422773.3737974,"name":"offline","context":{"idset":"8198"}}
+{"timestamp":1714422776.0330861,"name":"offline","context":{"idset":"8199"}}
+{"timestamp":1714422776.9649062,"name":"offline","context":{"idset":"8200"}}
+{"timestamp":1714423669.4512556,"name":"online","context":{"idset":"8197"}}
+{"timestamp":1714423719.5698688,"name":"online","context":{"idset":"8200"}}
+{"timestamp":1714423719.7723176,"name":"online","context":{"idset":"8198"}}
+{"timestamp":1714423720.8852227,"name":"online","context":{"idset":"8199"}}
+{"timestamp":1714423737.8452761,"name":"drain","context":{"idset":"8107-8108,8159","reason":"prolog failed for jobid fsP2zpWJhkB","overwrite":0}}
+{"timestamp":1714423740.8476255,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1714423742.7345541,"name":"online","context":{"idset":"10876,10882,10884-10885,10887"}}
+{"timestamp":1714423742.7380924,"name":"online","context":{"idset":"10874"}}
+{"timestamp":1714423742.8046062,"name":"online","context":{"idset":"10878"}}
+{"timestamp":1714423743.0155902,"name":"online","context":{"idset":"10870,10880,10883,10888"}}
+{"timestamp":1714423743.3181963,"name":"online","context":{"idset":"10875,10877,10879,10881,10886"}}
+{"timestamp":1714423743.5331678,"name":"online","context":{"idset":"10869,10871,10873"}}
+{"timestamp":1714424026.6638508,"name":"offline","context":{"idset":"8202"}}
+{"timestamp":1714424117.5869229,"name":"online","context":{"idset":"10890"}}
+{"timestamp":1714424118.737741,"name":"online","context":{"idset":"10889"}}
+{"timestamp":1714424118.7408385,"name":"online","context":{"idset":"10891"}}
+{"timestamp":1714424118.7439563,"name":"online","context":{"idset":"10892"}}
+{"timestamp":1714424492.8385725,"name":"undrain","context":{"idset":"8107-8108,8159"}}
+{"timestamp":1714424724.8989222,"name":"offline","context":{"idset":"8286"}}
+{"timestamp":1714425948.0338953,"name":"offline","context":{"idset":"8829"}}
+{"timestamp":1714425948.0372839,"name":"offline","context":{"idset":"8837"}}
+{"timestamp":1714425948.0404615,"name":"offline","context":{"idset":"8848"}}
+{"timestamp":1714425948.7412856,"name":"offline","context":{"idset":"8895"}}
+{"timestamp":1714425950.3428085,"name":"offline","context":{"idset":"8821"}}
+{"timestamp":1714425950.34811,"name":"offline","context":{"idset":"8822"}}
+{"timestamp":1714425950.3533585,"name":"offline","context":{"idset":"8823"}}
+{"timestamp":1714425950.3585865,"name":"offline","context":{"idset":"8824"}}
+{"timestamp":1714425950.363838,"name":"offline","context":{"idset":"8825"}}
+{"timestamp":1714425950.3690736,"name":"offline","context":{"idset":"8826"}}
+{"timestamp":1714425950.3743095,"name":"offline","context":{"idset":"8827"}}
+{"timestamp":1714425950.3795407,"name":"offline","context":{"idset":"8828"}}
+{"timestamp":1714425950.3847609,"name":"offline","context":{"idset":"8830"}}
+{"timestamp":1714425950.3900216,"name":"offline","context":{"idset":"8831"}}
+{"timestamp":1714425950.395247,"name":"offline","context":{"idset":"8832"}}
+{"timestamp":1714425950.4004996,"name":"offline","context":{"idset":"8834"}}
+{"timestamp":1714425950.4057448,"name":"offline","context":{"idset":"8835"}}
+{"timestamp":1714425950.4109735,"name":"offline","context":{"idset":"8836"}}
+{"timestamp":1714425950.4161875,"name":"offline","context":{"idset":"8838"}}
+{"timestamp":1714425950.4214084,"name":"offline","context":{"idset":"8839"}}
+{"timestamp":1714425950.4266138,"name":"offline","context":{"idset":"8840"}}
+{"timestamp":1714425950.4318202,"name":"offline","context":{"idset":"8841"}}
+{"timestamp":1714425950.4370496,"name":"offline","context":{"idset":"8842"}}
+{"timestamp":1714425950.4422336,"name":"offline","context":{"idset":"8844"}}
+{"timestamp":1714425950.4466221,"name":"offline","context":{"idset":"8845"}}
+{"timestamp":1714425950.4512181,"name":"offline","context":{"idset":"8846"}}
+{"timestamp":1714425950.4548502,"name":"offline","context":{"idset":"8847"}}
+{"timestamp":1714425950.4583013,"name":"offline","context":{"idset":"8849"}}
+{"timestamp":1714425950.4618189,"name":"offline","context":{"idset":"8850"}}
+{"timestamp":1714425950.4651408,"name":"offline","context":{"idset":"8851"}}
+{"timestamp":1714425950.468595,"name":"offline","context":{"idset":"8852"}}
+{"timestamp":1714425950.4717352,"name":"offline","context":{"idset":"8853"}}
+{"timestamp":1714425950.4748864,"name":"offline","context":{"idset":"8854"}}
+{"timestamp":1714425950.4780118,"name":"offline","context":{"idset":"8855"}}
+{"timestamp":1714425950.4811983,"name":"offline","context":{"idset":"8856"}}
+{"timestamp":1714425950.4844494,"name":"offline","context":{"idset":"8858"}}
+{"timestamp":1714425950.4875963,"name":"offline","context":{"idset":"8859"}}
+{"timestamp":1714425950.4907038,"name":"offline","context":{"idset":"8860"}}
+{"timestamp":1714425950.4938524,"name":"offline","context":{"idset":"8861"}}
+{"timestamp":1714425950.497052,"name":"offline","context":{"idset":"8862"}}
+{"timestamp":1714425950.50015,"name":"offline","context":{"idset":"8863"}}
+{"timestamp":1714425950.5032537,"name":"offline","context":{"idset":"8864"}}
+{"timestamp":1714425950.5082104,"name":"offline","context":{"idset":"8865"}}
+{"timestamp":1714425950.5113277,"name":"offline","context":{"idset":"8866"}}
+{"timestamp":1714425950.5144782,"name":"offline","context":{"idset":"8867"}}
+{"timestamp":1714425950.517611,"name":"offline","context":{"idset":"8868"}}
+{"timestamp":1714425950.5207102,"name":"offline","context":{"idset":"8869"}}
+{"timestamp":1714425950.5238147,"name":"offline","context":{"idset":"8870"}}
+{"timestamp":1714425950.5270622,"name":"offline","context":{"idset":"8871"}}
+{"timestamp":1714425950.5302384,"name":"offline","context":{"idset":"8872"}}
+{"timestamp":1714425950.5336375,"name":"offline","context":{"idset":"8873"}}
+{"timestamp":1714425951.216011,"name":"offline","context":{"idset":"8874"}}
+{"timestamp":1714425951.2191463,"name":"offline","context":{"idset":"8875"}}
+{"timestamp":1714425951.2222524,"name":"offline","context":{"idset":"8876"}}
+{"timestamp":1714425951.2253358,"name":"offline","context":{"idset":"8877"}}
+{"timestamp":1714425951.2284272,"name":"offline","context":{"idset":"8878"}}
+{"timestamp":1714425951.2315319,"name":"offline","context":{"idset":"8879"}}
+{"timestamp":1714425951.2435548,"name":"offline","context":{"idset":"8880"}}
+{"timestamp":1714425951.246671,"name":"offline","context":{"idset":"8881"}}
+{"timestamp":1714425951.2497928,"name":"offline","context":{"idset":"8882"}}
+{"timestamp":1714425951.2528982,"name":"offline","context":{"idset":"8883"}}
+{"timestamp":1714425951.2560058,"name":"offline","context":{"idset":"8884"}}
+{"timestamp":1714425951.2591333,"name":"offline","context":{"idset":"8886"}}
+{"timestamp":1714425951.2622595,"name":"offline","context":{"idset":"8887"}}
+{"timestamp":1714425951.2654283,"name":"offline","context":{"idset":"8888"}}
+{"timestamp":1714425951.2685461,"name":"offline","context":{"idset":"8889"}}
+{"timestamp":1714425951.2716978,"name":"offline","context":{"idset":"8890"}}
+{"timestamp":1714425951.2747979,"name":"offline","context":{"idset":"8891"}}
+{"timestamp":1714425951.2779367,"name":"offline","context":{"idset":"8892"}}
+{"timestamp":1714425951.2811041,"name":"offline","context":{"idset":"8893"}}
+{"timestamp":1714425951.2842143,"name":"offline","context":{"idset":"8894"}}
+{"timestamp":1714425951.2873733,"name":"offline","context":{"idset":"8896"}}
+{"timestamp":1714425951.2905469,"name":"offline","context":{"idset":"8897"}}
+{"timestamp":1714425951.2936699,"name":"offline","context":{"idset":"8899"}}
+{"timestamp":1714425951.2967372,"name":"offline","context":{"idset":"8900"}}
+{"timestamp":1714425951.299901,"name":"offline","context":{"idset":"8901"}}
+{"timestamp":1714425951.3030031,"name":"offline","context":{"idset":"8903"}}
+{"timestamp":1714425951.3061101,"name":"offline","context":{"idset":"8904"}}
+{"timestamp":1714425951.3092082,"name":"offline","context":{"idset":"8905"}}
+{"timestamp":1714425951.3124931,"name":"offline","context":{"idset":"8907"}}
+{"timestamp":1714425951.3157182,"name":"offline","context":{"idset":"8908"}}
+{"timestamp":1714425951.3188894,"name":"offline","context":{"idset":"8909"}}
+{"timestamp":1714425951.3413877,"name":"offline","context":{"idset":"8910"}}
+{"timestamp":1714425951.3642669,"name":"offline","context":{"idset":"8911"}}
+{"timestamp":1714425951.3775065,"name":"offline","context":{"idset":"8912"}}
+{"timestamp":1714425951.3808568,"name":"offline","context":{"idset":"8914"}}
+{"timestamp":1714425951.3843136,"name":"offline","context":{"idset":"8915"}}
+{"timestamp":1714425951.3876853,"name":"offline","context":{"idset":"8917"}}
+{"timestamp":1714425951.3910141,"name":"offline","context":{"idset":"8919"}}
+{"timestamp":1714425951.3944268,"name":"offline","context":{"idset":"8921"}}
+{"timestamp":1714425951.7730291,"name":"offline","context":{"idset":"8843"}}
+{"timestamp":1714425951.7763097,"name":"offline","context":{"idset":"8857"}}
+{"timestamp":1714425951.7794099,"name":"offline","context":{"idset":"8885"}}
+{"timestamp":1714425951.7824769,"name":"offline","context":{"idset":"8898"}}
+{"timestamp":1714425951.7944424,"name":"offline","context":{"idset":"8902"}}
+{"timestamp":1714425951.7978966,"name":"offline","context":{"idset":"8906"}}
+{"timestamp":1714425951.8012705,"name":"offline","context":{"idset":"8913"}}
+{"timestamp":1714425951.8047328,"name":"offline","context":{"idset":"8916"}}
+{"timestamp":1714425951.8552973,"name":"offline","context":{"idset":"8920"}}
+{"timestamp":1714426091.6911905,"name":"online","context":{"idset":"8706"}}
+{"timestamp":1714426092.7981892,"name":"online","context":{"idset":"8753"}}
+{"timestamp":1714426093.474972,"name":"online","context":{"idset":"8694"}}
+{"timestamp":1714426093.7014012,"name":"online","context":{"idset":"8708"}}
+{"timestamp":1714426093.8995221,"name":"online","context":{"idset":"8693"}}
+{"timestamp":1714426095.0504887,"name":"online","context":{"idset":"8701,8710"}}
+{"timestamp":1714426095.054584,"name":"online","context":{"idset":"8704"}}
+{"timestamp":1714426095.0588472,"name":"online","context":{"idset":"8752"}}
+{"timestamp":1714426095.2369237,"name":"online","context":{"idset":"8696,8702,8728"}}
+{"timestamp":1714426095.5235517,"name":"online","context":{"idset":"8712-8713"}}
+{"timestamp":1714426095.5389907,"name":"online","context":{"idset":"8727"}}
+{"timestamp":1714426095.654799,"name":"online","context":{"idset":"8721"}}
+{"timestamp":1714426095.6756663,"name":"online","context":{"idset":"8695"}}
+{"timestamp":1714426095.8045249,"name":"online","context":{"idset":"8739"}}
+{"timestamp":1714426096.7572641,"name":"online","context":{"idset":"8720,8737,8758"}}
+{"timestamp":1714426096.7606962,"name":"online","context":{"idset":"8711"}}
+{"timestamp":1714426096.7642593,"name":"online","context":{"idset":"8726"}}
+{"timestamp":1714426097.0169458,"name":"online","context":{"idset":"8806"}}
+{"timestamp":1714426097.1422582,"name":"online","context":{"idset":"8729"}}
+{"timestamp":1714426097.2581332,"name":"online","context":{"idset":"8705,8751,8800"}}
+{"timestamp":1714426097.3649225,"name":"online","context":{"idset":"8755"}}
+{"timestamp":1714426097.5703332,"name":"online","context":{"idset":"8697,8700,8716,8745-8747,8759,8762,8765,8778,8792,8798,8809,8814"}}
+{"timestamp":1714426097.7958188,"name":"online","context":{"idset":"8699,8736,8764,8767,8770,8774,8791,8795,8799,8802-8803,8811"}}
+{"timestamp":1714426097.9119198,"name":"online","context":{"idset":"8742,8772,8780"}}
+{"timestamp":1714426098.0372334,"name":"online","context":{"idset":"8738,8743"}}
+{"timestamp":1714426098.8807786,"name":"online","context":{"idset":"8714,8730,8740,8744,8749-8750,8760-8761,8769,8781-8783,8785-8786,8789,8793-8794,8796,8812,8817"}}
+{"timestamp":1714426098.8858199,"name":"online","context":{"idset":"8715,8717-8719,8722,8731,8733,8756,8766,8777,8810,8815"}}
+{"timestamp":1714426098.8900936,"name":"online","context":{"idset":"8707,8725,8754,8784"}}
+{"timestamp":1714426098.8943768,"name":"online","context":{"idset":"8735,8763,8775-8776,8788,8790,8801,8813"}}
+{"timestamp":1714426098.9203315,"name":"online","context":{"idset":"8741,8748,8757,8779"}}
+{"timestamp":1714426099.1291442,"name":"online","context":{"idset":"8698,8703,8709,8732,8771,8797,8807,8816,8819"}}
+{"timestamp":1714426099.2456508,"name":"online","context":{"idset":"8773,8787,8805"}}
+{"timestamp":1714426099.4380584,"name":"online","context":{"idset":"8734,8768,8808,8818,8820"}}
+{"timestamp":1714426099.549005,"name":"online","context":{"idset":"8804"}}
+{"timestamp":1714426321.4179738,"name":"drain","context":{"idset":"897-898","reason":"--reason Replacing SIVOC on b0","overwrite":0}}
+{"timestamp":1714426858.2000673,"name":"online","context":{"idset":"9501"}}
+{"timestamp":1714427038.8652279,"name":"online","context":{"idset":"9502"}}
+{"timestamp":1714427118.7870171,"name":"online","context":{"idset":"891"}}
+{"timestamp":1714427248.7542205,"name":"offline","context":{"idset":"9328"}}
+{"timestamp":1714427271.9819787,"name":"offline","context":{"idset":"891"}}
+{"timestamp":1714427337.7582366,"name":"online","context":{"idset":"891"}}
+{"timestamp":1714427340.9476438,"name":"online","context":{"idset":"893-894"}}
+{"timestamp":1714428105.0532584,"name":"drain","context":{"idset":"895-896","reason":"--reason Troubleshoot SIVOC on b0","overwrite":0}}
+{"timestamp":1714428389.1316149,"name":"online","context":{"idset":"9535"}}
+{"timestamp":1714428923.4709804,"name":"undrain","context":{"idset":"882"}}
+{"timestamp":1714429324.6819658,"name":"drain","context":{"idset":"882","reason":"--reason H/W troubleshoot","overwrite":0}}
+{"timestamp":1714429499.8671234,"name":"drain","context":{"idset":"8107-8108,8159","reason":"prolog failed for jobid fsPnnAYcY9D","overwrite":0}}
+{"timestamp":1714429808.8079767,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1714429810.0376308,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1714429810.0408692,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1714429810.0440717,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1714429810.0473197,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1714429810.8557203,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1714429812.0284305,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1714429812.7576799,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1714429894.7165277,"name":"offline","context":{"idset":"8159"}}
+{"timestamp":1714429930.7496073,"name":"offline","context":{"idset":"8108"}}
+{"timestamp":1714430710.8624735,"name":"online","context":{"idset":"845"}}
+{"timestamp":1714430710.866931,"name":"online","context":{"idset":"846"}}
+{"timestamp":1714430819.215374,"name":"drain","context":{"idset":"8329","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430820.0968969,"name":"drain","context":{"idset":"8369","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430821.1106384,"name":"drain","context":{"idset":"8430","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430821.5316479,"name":"drain","context":{"idset":"8316","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430821.6839235,"name":"drain","context":{"idset":"8352","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430821.8554733,"name":"drain","context":{"idset":"8360","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430822.9775708,"name":"drain","context":{"idset":"8330","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430823.3148818,"name":"drain","context":{"idset":"8403","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430823.9881494,"name":"drain","context":{"idset":"8394","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430824.1389227,"name":"drain","context":{"idset":"8351","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430824.3105769,"name":"drain","context":{"idset":"8332","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430825.1400299,"name":"drain","context":{"idset":"8412","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430825.2889585,"name":"drain","context":{"idset":"8414","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430825.4509459,"name":"drain","context":{"idset":"8354","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430825.8643777,"name":"drain","context":{"idset":"8396","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430826.009763,"name":"drain","context":{"idset":"8349","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430826.9439409,"name":"drain","context":{"idset":"8343","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430827.0924792,"name":"drain","context":{"idset":"8367","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430827.2402866,"name":"drain","context":{"idset":"8410","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430827.4015779,"name":"drain","context":{"idset":"8315","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430827.5715985,"name":"drain","context":{"idset":"8417","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430827.7016749,"name":"drain","context":{"idset":"8356","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430827.8990319,"name":"drain","context":{"idset":"8382","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430828.1238101,"name":"drain","context":{"idset":"8333","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430829.8945253,"name":"drain","context":{"idset":"8384","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430830.0670993,"name":"drain","context":{"idset":"8350","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430830.9828668,"name":"drain","context":{"idset":"8413","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430831.1114593,"name":"drain","context":{"idset":"8401","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430831.4564543,"name":"drain","context":{"idset":"8328","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430832.2683012,"name":"drain","context":{"idset":"8353","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430835.1180155,"name":"drain","context":{"idset":"8432","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430835.323693,"name":"drain","context":{"idset":"8371","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430835.5388677,"name":"drain","context":{"idset":"8398","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430835.806958,"name":"drain","context":{"idset":"8406","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430836.8430271,"name":"drain","context":{"idset":"8383","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430844.9519911,"name":"drain","context":{"idset":"8422","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714430956.0796425,"name":"online","context":{"idset":"8108"}}
+{"timestamp":1714431231.2850327,"name":"offline","context":{"idset":"8326"}}
+{"timestamp":1714431231.3156748,"name":"offline","context":{"idset":"8315"}}
+{"timestamp":1714431231.415067,"name":"offline","context":{"idset":"8311"}}
+{"timestamp":1714431231.4540563,"name":"offline","context":{"idset":"8319"}}
+{"timestamp":1714431231.4583127,"name":"offline","context":{"idset":"8317"}}
+{"timestamp":1714431231.474916,"name":"offline","context":{"idset":"8316"}}
+{"timestamp":1714431231.5610824,"name":"offline","context":{"idset":"8325"}}
+{"timestamp":1714431231.6599073,"name":"offline","context":{"idset":"8314"}}
+{"timestamp":1714431231.6633368,"name":"offline","context":{"idset":"8330"}}
+{"timestamp":1714431231.665863,"name":"offline","context":{"idset":"8320"}}
+{"timestamp":1714431231.6684129,"name":"offline","context":{"idset":"8328"}}
+{"timestamp":1714431231.6727839,"name":"offline","context":{"idset":"8312"}}
+{"timestamp":1714431231.6749396,"name":"offline","context":{"idset":"8322"}}
+{"timestamp":1714431231.677083,"name":"offline","context":{"idset":"8324"}}
+{"timestamp":1714431231.6805365,"name":"offline","context":{"idset":"8318"}}
+{"timestamp":1714431231.7770038,"name":"offline","context":{"idset":"8323"}}
+{"timestamp":1714431231.7838197,"name":"offline","context":{"idset":"8310"}}
+{"timestamp":1714431231.787102,"name":"offline","context":{"idset":"8321"}}
+{"timestamp":1714431231.8611438,"name":"offline","context":{"idset":"8309"}}
+{"timestamp":1714431231.9601367,"name":"offline","context":{"idset":"8313"}}
+{"timestamp":1714431232.6132452,"name":"offline","context":{"idset":"8334"}}
+{"timestamp":1714431232.6176662,"name":"offline","context":{"idset":"8343"}}
+{"timestamp":1714431232.6220386,"name":"offline","context":{"idset":"8341"}}
+{"timestamp":1714431232.6276996,"name":"offline","context":{"idset":"8350"}}
+{"timestamp":1714431232.7349255,"name":"offline","context":{"idset":"8344"}}
+{"timestamp":1714431232.8897643,"name":"offline","context":{"idset":"8338"}}
+{"timestamp":1714431232.9107034,"name":"offline","context":{"idset":"8346"}}
+{"timestamp":1714431232.9159515,"name":"offline","context":{"idset":"8357"}}
+{"timestamp":1714431232.9216301,"name":"offline","context":{"idset":"8333"}}
+{"timestamp":1714431232.9265416,"name":"offline","context":{"idset":"8327"}}
+{"timestamp":1714431232.9386895,"name":"offline","context":{"idset":"8359"}}
+{"timestamp":1714431232.9435976,"name":"offline","context":{"idset":"8348"}}
+{"timestamp":1714431232.9869916,"name":"offline","context":{"idset":"8354"}}
+{"timestamp":1714431232.9956977,"name":"offline","context":{"idset":"8353"}}
+{"timestamp":1714431232.9997854,"name":"offline","context":{"idset":"8337"}}
+{"timestamp":1714431233.0038326,"name":"offline","context":{"idset":"8342"}}
+{"timestamp":1714431233.0077236,"name":"offline","context":{"idset":"8345"}}
+{"timestamp":1714431233.0257022,"name":"offline","context":{"idset":"8358"}}
+{"timestamp":1714431233.1325064,"name":"offline","context":{"idset":"8335"}}
+{"timestamp":1714431233.325861,"name":"offline","context":{"idset":"8373"}}
+{"timestamp":1714431233.333858,"name":"offline","context":{"idset":"8340"}}
+{"timestamp":1714431233.3457882,"name":"offline","context":{"idset":"8347"}}
+{"timestamp":1714431233.3497045,"name":"offline","context":{"idset":"8367"}}
+{"timestamp":1714431233.3598177,"name":"offline","context":{"idset":"8371"}}
+{"timestamp":1714431233.3708155,"name":"offline","context":{"idset":"8332"}}
+{"timestamp":1714431233.3806107,"name":"offline","context":{"idset":"8364"}}
+{"timestamp":1714431233.5427461,"name":"offline","context":{"idset":"8363"}}
+{"timestamp":1714431233.5496547,"name":"offline","context":{"idset":"8339"}}
+{"timestamp":1714431233.5537674,"name":"offline","context":{"idset":"8329"}}
+{"timestamp":1714431233.5576878,"name":"offline","context":{"idset":"8331"}}
+{"timestamp":1714431233.5835249,"name":"offline","context":{"idset":"8336"}}
+{"timestamp":1714431233.6010883,"name":"offline","context":{"idset":"8349"}}
+{"timestamp":1714431233.6075549,"name":"offline","context":{"idset":"8351"}}
+{"timestamp":1714431233.6137974,"name":"offline","context":{"idset":"8352"}}
+{"timestamp":1714431233.6199791,"name":"offline","context":{"idset":"8356"}}
+{"timestamp":1714431233.6261854,"name":"offline","context":{"idset":"8360"}}
+{"timestamp":1714431233.6323411,"name":"offline","context":{"idset":"8362"}}
+{"timestamp":1714431233.638494,"name":"offline","context":{"idset":"8366"}}
+{"timestamp":1714431233.6446488,"name":"offline","context":{"idset":"8369"}}
+{"timestamp":1714431233.6537404,"name":"offline","context":{"idset":"8372"}}
+{"timestamp":1714431233.6599429,"name":"offline","context":{"idset":"8374"}}
+{"timestamp":1714431233.6671379,"name":"offline","context":{"idset":"8375"}}
+{"timestamp":1714431233.6748133,"name":"offline","context":{"idset":"8376"}}
+{"timestamp":1714431233.6823592,"name":"offline","context":{"idset":"8377"}}
+{"timestamp":1714431233.6897988,"name":"offline","context":{"idset":"8380"}}
+{"timestamp":1714431233.6973059,"name":"offline","context":{"idset":"8386"}}
+{"timestamp":1714431233.7048447,"name":"offline","context":{"idset":"8388"}}
+{"timestamp":1714431233.7122481,"name":"offline","context":{"idset":"8392"}}
+{"timestamp":1714431233.7196574,"name":"offline","context":{"idset":"8393"}}
+{"timestamp":1714431233.7270911,"name":"offline","context":{"idset":"8396"}}
+{"timestamp":1714431233.7735441,"name":"offline","context":{"idset":"8402"}}
+{"timestamp":1714431233.8208511,"name":"offline","context":{"idset":"8404"}}
+{"timestamp":1714431233.8459501,"name":"offline","context":{"idset":"8408"}}
+{"timestamp":1714431233.852566,"name":"offline","context":{"idset":"8413"}}
+{"timestamp":1714431233.8591135,"name":"offline","context":{"idset":"8382"}}
+{"timestamp":1714431233.8656127,"name":"offline","context":{"idset":"8384"}}
+{"timestamp":1714431234.2125344,"name":"offline","context":{"idset":"8370"}}
+{"timestamp":1714431234.2153084,"name":"offline","context":{"idset":"8387"}}
+{"timestamp":1714431234.2180452,"name":"offline","context":{"idset":"8403"}}
+{"timestamp":1714431234.2207885,"name":"offline","context":{"idset":"8397"}}
+{"timestamp":1714431234.2234967,"name":"offline","context":{"idset":"8368"}}
+{"timestamp":1714431234.2262383,"name":"offline","context":{"idset":"8401"}}
+{"timestamp":1714431234.2289429,"name":"offline","context":{"idset":"8355"}}
+{"timestamp":1714431234.2316647,"name":"offline","context":{"idset":"8381"}}
+{"timestamp":1714431234.2343731,"name":"offline","context":{"idset":"8406"}}
+{"timestamp":1714431234.2370889,"name":"offline","context":{"idset":"8395"}}
+{"timestamp":1714431234.2403369,"name":"offline","context":{"idset":"8365"}}
+{"timestamp":1714431234.2440417,"name":"offline","context":{"idset":"8410"}}
+{"timestamp":1714431234.2475891,"name":"offline","context":{"idset":"8399"}}
+{"timestamp":1714431234.2502072,"name":"offline","context":{"idset":"8398"}}
+{"timestamp":1714431234.2525208,"name":"offline","context":{"idset":"8420"}}
+{"timestamp":1714431234.2549541,"name":"offline","context":{"idset":"8361"}}
+{"timestamp":1714431234.2574263,"name":"offline","context":{"idset":"8414"}}
+{"timestamp":1714431234.2601252,"name":"offline","context":{"idset":"8418"}}
+{"timestamp":1714431234.2628195,"name":"offline","context":{"idset":"8379"}}
+{"timestamp":1714431234.2655087,"name":"offline","context":{"idset":"8417"}}
+{"timestamp":1714431234.2682178,"name":"offline","context":{"idset":"8422"}}
+{"timestamp":1714431234.270776,"name":"offline","context":{"idset":"8405"}}
+{"timestamp":1714431234.2738776,"name":"offline","context":{"idset":"8394"}}
+{"timestamp":1714431234.2765768,"name":"offline","context":{"idset":"8419"}}
+{"timestamp":1714431234.2792478,"name":"offline","context":{"idset":"8378"}}
+{"timestamp":1714431234.2819121,"name":"offline","context":{"idset":"8383"}}
+{"timestamp":1714431234.2853842,"name":"offline","context":{"idset":"8423"}}
+{"timestamp":1714431234.2886927,"name":"offline","context":{"idset":"8425"}}
+{"timestamp":1714431234.2918751,"name":"offline","context":{"idset":"8427"}}
+{"timestamp":1714431234.294873,"name":"offline","context":{"idset":"8409"}}
+{"timestamp":1714431234.297245,"name":"offline","context":{"idset":"8407"}}
+{"timestamp":1714431234.2993708,"name":"offline","context":{"idset":"8390"}}
+{"timestamp":1714431234.3014836,"name":"offline","context":{"idset":"8400"}}
+{"timestamp":1714431234.3035748,"name":"offline","context":{"idset":"8391"}}
+{"timestamp":1714431234.3056769,"name":"offline","context":{"idset":"8385"}}
+{"timestamp":1714431234.3077643,"name":"offline","context":{"idset":"8429"}}
+{"timestamp":1714431234.3098655,"name":"offline","context":{"idset":"8389"}}
+{"timestamp":1714431234.3119462,"name":"offline","context":{"idset":"8432"}}
+{"timestamp":1714431234.3140335,"name":"offline","context":{"idset":"8411"}}
+{"timestamp":1714431234.3161175,"name":"offline","context":{"idset":"8433"}}
+{"timestamp":1714431234.3181796,"name":"offline","context":{"idset":"8431"}}
+{"timestamp":1714431234.3202522,"name":"offline","context":{"idset":"8430"}}
+{"timestamp":1714431234.3227642,"name":"offline","context":{"idset":"8416"}}
+{"timestamp":1714431234.324909,"name":"offline","context":{"idset":"8421"}}
+{"timestamp":1714431234.3269763,"name":"offline","context":{"idset":"8435"}}
+{"timestamp":1714431234.32903,"name":"offline","context":{"idset":"8412"}}
+{"timestamp":1714431234.3311062,"name":"offline","context":{"idset":"8415"}}
+{"timestamp":1714431234.3331721,"name":"offline","context":{"idset":"8428"}}
+{"timestamp":1714431234.3352118,"name":"offline","context":{"idset":"8434"}}
+{"timestamp":1714431234.3372576,"name":"offline","context":{"idset":"8424"}}
+{"timestamp":1714431234.8303354,"name":"offline","context":{"idset":"8426"}}
+{"timestamp":1714431235.0320272,"name":"offline","context":{"idset":"8436"}}
+{"timestamp":1714431279.006248,"name":"online","context":{"idset":"8342"}}
+{"timestamp":1714431279.2110605,"name":"online","context":{"idset":"8363"}}
+{"timestamp":1714431281.2011759,"name":"online","context":{"idset":"8338"}}
+{"timestamp":1714431282.7576084,"name":"online","context":{"idset":"8327"}}
+{"timestamp":1714431282.7609217,"name":"online","context":{"idset":"8325"}}
+{"timestamp":1714431283.3266554,"name":"online","context":{"idset":"8334"}}
+{"timestamp":1714431283.8589983,"name":"online","context":{"idset":"8320"}}
+{"timestamp":1714431284.7868581,"name":"online","context":{"idset":"8378"}}
+{"timestamp":1714431285.1789651,"name":"online","context":{"idset":"8324"}}
+{"timestamp":1714431285.6966949,"name":"online","context":{"idset":"8309"}}
+{"timestamp":1714431287.3513055,"name":"online","context":{"idset":"8313"}}
+{"timestamp":1714431287.3574262,"name":"online","context":{"idset":"8311"}}
+{"timestamp":1714431288.7230175,"name":"online","context":{"idset":"8344"}}
+{"timestamp":1714431290.8282495,"name":"online","context":{"idset":"8314,8358"}}
+{"timestamp":1714431290.8314672,"name":"online","context":{"idset":"8368"}}
+{"timestamp":1714431290.9998975,"name":"online","context":{"idset":"8348"}}
+{"timestamp":1714431291.124181,"name":"online","context":{"idset":"8355"}}
+{"timestamp":1714431291.4687226,"name":"online","context":{"idset":"8345"}}
+{"timestamp":1714431291.7678392,"name":"online","context":{"idset":"8346,8362"}}
+{"timestamp":1714431291.9832673,"name":"online","context":{"idset":"8418"}}
+{"timestamp":1714431292.8060758,"name":"online","context":{"idset":"8336"}}
+{"timestamp":1714431292.8121626,"name":"online","context":{"idset":"8379"}}
+{"timestamp":1714431292.8186061,"name":"online","context":{"idset":"8340,8359"}}
+{"timestamp":1714431292.8246632,"name":"online","context":{"idset":"8408,8416"}}
+{"timestamp":1714431292.8887789,"name":"online","context":{"idset":"8395"}}
+{"timestamp":1714431296.3260667,"name":"online","context":{"idset":"8326,8420"}}
+{"timestamp":1714431296.3718922,"name":"online","context":{"idset":"8323,8331,8339,8347,8357,8361,8365,8390,8392"}}
+{"timestamp":1714431296.3879426,"name":"online","context":{"idset":"8341"}}
+{"timestamp":1714431296.3914235,"name":"online","context":{"idset":"8337,8424"}}
+{"timestamp":1714431296.395637,"name":"online","context":{"idset":"8404"}}
+{"timestamp":1714431296.3985603,"name":"online","context":{"idset":"8335"}}
+{"timestamp":1714431297.0134976,"name":"online","context":{"idset":"8405"}}
+{"timestamp":1714431297.147624,"name":"online","context":{"idset":"8321,8376,8400,8402,8434"}}
+{"timestamp":1714431297.285655,"name":"online","context":{"idset":"8312,8386,8389,8426"}}
+{"timestamp":1714431297.5877254,"name":"online","context":{"idset":"8322,8388,8407,8415,8419,8428,8436"}}
+{"timestamp":1714431297.8877892,"name":"online","context":{"idset":"8319,8393"}}
+{"timestamp":1714431297.9951615,"name":"online","context":{"idset":"8373"}}
+{"timestamp":1714431299.0421343,"name":"online","context":{"idset":"8317-8318,8380,8397,8409"}}
+{"timestamp":1714431299.0463889,"name":"online","context":{"idset":"8377,8381"}}
+{"timestamp":1714431299.0505774,"name":"online","context":{"idset":"8310,8399"}}
+{"timestamp":1714431299.0547323,"name":"online","context":{"idset":"8391"}}
+{"timestamp":1714431299.0588605,"name":"online","context":{"idset":"8364,8374,8411"}}
+{"timestamp":1714431299.5332377,"name":"online","context":{"idset":"8370,8433"}}
+{"timestamp":1714431301.1499884,"name":"online","context":{"idset":"8366"}}
+{"timestamp":1714431301.1548665,"name":"online","context":{"idset":"8372,8375,8385,8387"}}
+{"timestamp":1714431301.1600323,"name":"online","context":{"idset":"8429"}}
+{"timestamp":1714431302.8863418,"name":"online","context":{"idset":"8423"}}
+{"timestamp":1714431303.1804864,"name":"online","context":{"idset":"8435"}}
+{"timestamp":1714431303.4156165,"name":"online","context":{"idset":"8421,8425,8431"}}
+{"timestamp":1714431303.9471648,"name":"online","context":{"idset":"8427"}}
+{"timestamp":1714431575.2082305,"name":"undrain","context":{"idset":"8315-8316,8328-8330,8332-8333,8343,8349-8354,8356,8360,8367,8369,8371,8382-8384,8394,8396,8398,8401,8403,8406,8410,8412-8414,8417,8422,8430,8432"}}
+{"timestamp":1714431637.9631393,"name":"online","context":{"idset":"8159"}}
+{"timestamp":1714431900.8647056,"name":"offline","context":{"idset":"8108"}}
+{"timestamp":1714431908.7138543,"name":"offline","context":{"idset":"8107"}}
+{"timestamp":1714432017.7225266,"name":"undrain","context":{"idset":"8723-8724"}}
+{"timestamp":1714432054.0584114,"name":"drain","context":{"idset":"8315-8316,8328-8330,8332-8333,8343,8349-8354,8356,8360,8367,8369,8371,8382-8384,8394,8396,8398,8401,8403,8406,8410,8412-8414,8417,8422,8430,8432","overwrite":0}}
+{"timestamp":1714432123.2467153,"name":"drain","context":{"idset":"11789","reason":"Broken Partner node --JRG","overwrite":0}}
+{"timestamp":1714432137.4619014,"name":"drain","context":{"idset":"11790","reason":"Missing CXI link --JRG","overwrite":0}}
+{"timestamp":1714432260.932487,"name":"offline","context":{"idset":"11789"}}
+{"timestamp":1714432465.5542796,"name":"undrain","context":{"idset":"8315-8316,8328-8330,8332-8333,8343,8349-8354,8356,8360,8367,8369,8371,8382-8384,8394,8396,8398,8401,8403,8406,8410,8412-8414,8417,8422,8430,8432"}}
+{"timestamp":1714432532.7446022,"name":"online","context":{"idset":"9037"}}
+{"timestamp":1714432559.793088,"name":"online","context":{"idset":"9040"}}
+{"timestamp":1714433009.3413975,"name":"undrain","context":{"idset":"8159"}}
+{"timestamp":1714433113.8315384,"name":"drain","context":{"idset":"8315-8316,8328-8330,8332-8333,8343,8349-8354,8356,8360,8367,8369,8371,8382-8384,8394,8396,8398,8401,8403,8406,8410,8412-8414,8417,8422,8430,8432","overwrite":0}}
+{"timestamp":1714433230.8913116,"name":"online","context":{"idset":"817-818"}}
+{"timestamp":1714434832.9190993,"name":"offline","context":{"idset":"8711"}}
+{"timestamp":1714434833.0608177,"name":"offline","context":{"idset":"8712"}}
+{"timestamp":1714435176.7501726,"name":"offline","context":{"idset":"8747"}}
+{"timestamp":1714435176.9003923,"name":"offline","context":{"idset":"8748"}}
+{"timestamp":1714436603.1473248,"name":"online","context":{"idset":"8367"}}
+{"timestamp":1714436603.4497042,"name":"online","context":{"idset":"8349"}}
+{"timestamp":1714436603.7066748,"name":"online","context":{"idset":"8316"}}
+{"timestamp":1714436604.209312,"name":"online","context":{"idset":"8382"}}
+{"timestamp":1714436604.4786365,"name":"online","context":{"idset":"8315,8398"}}
+{"timestamp":1714436604.7027884,"name":"online","context":{"idset":"8394,8403"}}
+{"timestamp":1714436605.026897,"name":"online","context":{"idset":"8354,8360"}}
+{"timestamp":1714436605.3235309,"name":"online","context":{"idset":"8332,8383"}}
+{"timestamp":1714436605.5436614,"name":"online","context":{"idset":"8328,8353,8406,8432"}}
+{"timestamp":1714436605.6588449,"name":"online","context":{"idset":"8352"}}
+{"timestamp":1714436605.8660307,"name":"online","context":{"idset":"8333,8343,8350,8356,8369,8371,8396,8412,8414,8417,8430"}}
+{"timestamp":1714436606.069787,"name":"online","context":{"idset":"8330,8351,8384"}}
+{"timestamp":1714436606.1876924,"name":"online","context":{"idset":"8329,8413,8422"}}
+{"timestamp":1714436606.380151,"name":"online","context":{"idset":"8410"}}
+{"timestamp":1714436606.4823124,"name":"online","context":{"idset":"8401"}}
+{"timestamp":1714436678.3175883,"name":"undrain","context":{"idset":"8315-8316,8328-8330,8332-8333,8343,8349-8354,8356,8360,8367,8369,8371,8382-8384,8394,8396,8398,8401,8403,8406,8410,8412-8414,8417,8422,8430,8432"}}
+{"timestamp":1714439462.5890303,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1714441842.7715137,"name":"offline","context":{"idset":"11589"}}
+{"timestamp":1714441844.0455527,"name":"offline","context":{"idset":"11590"}}
+{"timestamp":1714441844.0498667,"name":"offline","context":{"idset":"11591"}}
+{"timestamp":1714441844.0534258,"name":"offline","context":{"idset":"11592"}}
+{"timestamp":1714441845.2930095,"name":"offline","context":{"idset":"11593"}}
+{"timestamp":1714441845.405638,"name":"offline","context":{"idset":"11594"}}
+{"timestamp":1714441845.4127753,"name":"offline","context":{"idset":"11595"}}
+{"timestamp":1714441845.419879,"name":"offline","context":{"idset":"11596"}}
+{"timestamp":1714441845.4268672,"name":"offline","context":{"idset":"11597"}}
+{"timestamp":1714441845.4409044,"name":"offline","context":{"idset":"11598"}}
+{"timestamp":1714441848.0366671,"name":"offline","context":{"idset":"11599"}}
+{"timestamp":1714441848.0429995,"name":"offline","context":{"idset":"11600"}}
+{"timestamp":1714441848.0493076,"name":"offline","context":{"idset":"11601"}}
+{"timestamp":1714441848.9649682,"name":"offline","context":{"idset":"11602"}}
+{"timestamp":1714441849.0234106,"name":"offline","context":{"idset":"11603"}}
+{"timestamp":1714441849.035346,"name":"offline","context":{"idset":"11604"}}
+{"timestamp":1714441849.1165309,"name":"offline","context":{"idset":"11605"}}
+{"timestamp":1714441852.0576768,"name":"offline","context":{"idset":"11606"}}
+{"timestamp":1714441852.0637145,"name":"offline","context":{"idset":"11607"}}
+{"timestamp":1714441852.0669305,"name":"offline","context":{"idset":"11608"}}
+{"timestamp":1714441852.0700583,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1714441852.0731702,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1714441853.2992885,"name":"offline","context":{"idset":"11611"}}
+{"timestamp":1714441853.3429825,"name":"offline","context":{"idset":"11612"}}
+{"timestamp":1714441853.3518498,"name":"offline","context":{"idset":"11613"}}
+{"timestamp":1714441853.3541951,"name":"offline","context":{"idset":"11614"}}
+{"timestamp":1714441853.4243474,"name":"offline","context":{"idset":"11615"}}
+{"timestamp":1714441856.0308642,"name":"offline","context":{"idset":"11616"}}
+{"timestamp":1714441856.0350876,"name":"offline","context":{"idset":"11617"}}
+{"timestamp":1714441856.7738543,"name":"offline","context":{"idset":"11618"}}
+{"timestamp":1714441856.8243477,"name":"offline","context":{"idset":"11619"}}
+{"timestamp":1714441856.8274806,"name":"offline","context":{"idset":"11621"}}
+{"timestamp":1714441856.9018998,"name":"offline","context":{"idset":"11622"}}
+{"timestamp":1714441860.0359483,"name":"offline","context":{"idset":"11623"}}
+{"timestamp":1714441860.0416765,"name":"offline","context":{"idset":"11624"}}
+{"timestamp":1714441860.0473616,"name":"offline","context":{"idset":"11625"}}
+{"timestamp":1714441860.8884814,"name":"offline","context":{"idset":"11626"}}
+{"timestamp":1714441860.947726,"name":"offline","context":{"idset":"11627"}}
+{"timestamp":1714441860.9509749,"name":"offline","context":{"idset":"11628"}}
+{"timestamp":1714441860.9545102,"name":"offline","context":{"idset":"11629"}}
+{"timestamp":1714441861.023356,"name":"offline","context":{"idset":"11630"}}
+{"timestamp":1714441864.0287838,"name":"offline","context":{"idset":"11631"}}
+{"timestamp":1714441864.7378452,"name":"offline","context":{"idset":"11632"}}
+{"timestamp":1714441866.7494845,"name":"offline","context":{"idset":"11635"}}
+{"timestamp":1714441906.9302895,"name":"offline","context":{"idset":"11620"}}
+{"timestamp":1714441916.122298,"name":"offline","context":{"idset":"11636"}}
+{"timestamp":1714443048.8586476,"name":"offline","context":{"idset":"9075"}}
+{"timestamp":1714475160.7902799,"name":"offline","context":{"idset":"10042"}}
+{"timestamp":1714477588.8977869,"name":"offline","context":{"idset":"10748"}}
+{"timestamp":1714480244.7760434,"name":"offline","context":{"idset":"9403"}}
+{"timestamp":1714482683.309962,"name":"offline","context":{"idset":"8949"}}
+{"timestamp":1714482683.3373392,"name":"offline","context":{"idset":"8952"}}
+{"timestamp":1714482683.4378715,"name":"offline","context":{"idset":"8955"}}
+{"timestamp":1714482683.4621577,"name":"offline","context":{"idset":"8975"}}
+{"timestamp":1714482683.4687753,"name":"offline","context":{"idset":"9006"}}
+{"timestamp":1714482683.475033,"name":"offline","context":{"idset":"9001"}}
+{"timestamp":1714482683.4818242,"name":"offline","context":{"idset":"8950"}}
+{"timestamp":1714482683.4926329,"name":"offline","context":{"idset":"8960"}}
+{"timestamp":1714482683.5337992,"name":"offline","context":{"idset":"8961"}}
+{"timestamp":1714482683.5385826,"name":"offline","context":{"idset":"8962"}}
+{"timestamp":1714482683.5417612,"name":"offline","context":{"idset":"8985"}}
+{"timestamp":1714482683.5449178,"name":"offline","context":{"idset":"8988"}}
+{"timestamp":1714482683.5569215,"name":"offline","context":{"idset":"8953"}}
+{"timestamp":1714482683.5610492,"name":"offline","context":{"idset":"8957"}}
+{"timestamp":1714482683.5642352,"name":"offline","context":{"idset":"8981"}}
+{"timestamp":1714482683.5674422,"name":"offline","context":{"idset":"8987"}}
+{"timestamp":1714482683.5706098,"name":"offline","context":{"idset":"8989"}}
+{"timestamp":1714482683.5737886,"name":"offline","context":{"idset":"8990"}}
+{"timestamp":1714482683.5769646,"name":"offline","context":{"idset":"8991"}}
+{"timestamp":1714482683.5801282,"name":"offline","context":{"idset":"8997"}}
+{"timestamp":1714482683.6410148,"name":"offline","context":{"idset":"9015"}}
+{"timestamp":1714482683.7685907,"name":"offline","context":{"idset":"8974"}}
+{"timestamp":1714482683.7727773,"name":"offline","context":{"idset":"8984"}}
+{"timestamp":1714482683.7800918,"name":"offline","context":{"idset":"9012"}}
+{"timestamp":1714482683.7840393,"name":"offline","context":{"idset":"9005"}}
+{"timestamp":1714482683.7886233,"name":"offline","context":{"idset":"8963"}}
+{"timestamp":1714482683.7930503,"name":"offline","context":{"idset":"9023"}}
+{"timestamp":1714482683.7969444,"name":"offline","context":{"idset":"9013"}}
+{"timestamp":1714482683.8031723,"name":"offline","context":{"idset":"9026"}}
+{"timestamp":1714482683.8066776,"name":"offline","context":{"idset":"8986"}}
+{"timestamp":1714482683.8102574,"name":"offline","context":{"idset":"8992"}}
+{"timestamp":1714482683.9001575,"name":"offline","context":{"idset":"9008"}}
+{"timestamp":1714482683.9041553,"name":"offline","context":{"idset":"9002"}}
+{"timestamp":1714482683.9082053,"name":"offline","context":{"idset":"8951"}}
+{"timestamp":1714482683.9122112,"name":"offline","context":{"idset":"8954"}}
+{"timestamp":1714482683.9161901,"name":"offline","context":{"idset":"8956"}}
+{"timestamp":1714482683.9310377,"name":"offline","context":{"idset":"8958"}}
+{"timestamp":1714482683.9350238,"name":"offline","context":{"idset":"8959"}}
+{"timestamp":1714482683.9390643,"name":"offline","context":{"idset":"8973"}}
+{"timestamp":1714482683.9431262,"name":"offline","context":{"idset":"8976"}}
+{"timestamp":1714482683.9470778,"name":"offline","context":{"idset":"8977"}}
+{"timestamp":1714482683.951036,"name":"offline","context":{"idset":"8978"}}
+{"timestamp":1714482683.9550035,"name":"offline","context":{"idset":"8979"}}
+{"timestamp":1714482683.9590309,"name":"offline","context":{"idset":"8980"}}
+{"timestamp":1714482683.9630363,"name":"offline","context":{"idset":"8983"}}
+{"timestamp":1714482683.966984,"name":"offline","context":{"idset":"8995"}}
+{"timestamp":1714482683.970957,"name":"offline","context":{"idset":"8996"}}
+{"timestamp":1714482683.9749396,"name":"offline","context":{"idset":"8998"}}
+{"timestamp":1714482683.9788799,"name":"offline","context":{"idset":"9007"}}
+{"timestamp":1714482683.9828439,"name":"offline","context":{"idset":"9009"}}
+{"timestamp":1714482683.9867907,"name":"offline","context":{"idset":"9010"}}
+{"timestamp":1714482683.9907196,"name":"offline","context":{"idset":"9011"}}
+{"timestamp":1714482683.994653,"name":"offline","context":{"idset":"9014"}}
+{"timestamp":1714482683.9987013,"name":"offline","context":{"idset":"9016"}}
+{"timestamp":1714482684.0052071,"name":"offline","context":{"idset":"9017"}}
+{"timestamp":1714482684.0128396,"name":"offline","context":{"idset":"9019"}}
+{"timestamp":1714482684.0172505,"name":"offline","context":{"idset":"9020"}}
+{"timestamp":1714482684.0205476,"name":"offline","context":{"idset":"9021"}}
+{"timestamp":1714482684.0263245,"name":"offline","context":{"idset":"9022"}}
+{"timestamp":1714482684.0327981,"name":"offline","context":{"idset":"9024"}}
+{"timestamp":1714482684.0363417,"name":"offline","context":{"idset":"9025"}}
+{"timestamp":1714482684.0398433,"name":"offline","context":{"idset":"9027"}}
+{"timestamp":1714482717.8357608,"name":"online","context":{"idset":"8960"}}
+{"timestamp":1714482718.8639455,"name":"online","context":{"idset":"8975"}}
+{"timestamp":1714482718.8673685,"name":"online","context":{"idset":"8949"}}
+{"timestamp":1714482718.8705232,"name":"online","context":{"idset":"8952"}}
+{"timestamp":1714482718.8737066,"name":"online","context":{"idset":"8962"}}
+{"timestamp":1714482719.9360869,"name":"online","context":{"idset":"8985"}}
+{"timestamp":1714482720.8262422,"name":"online","context":{"idset":"8955"}}
+{"timestamp":1714482720.8296475,"name":"online","context":{"idset":"8950,8976-8977,8979,8991"}}
+{"timestamp":1714482720.8331387,"name":"online","context":{"idset":"8958,8961,8980"}}
+{"timestamp":1714482720.8374231,"name":"online","context":{"idset":"9001"}}
+{"timestamp":1714482721.062583,"name":"online","context":{"idset":"8957,8973-8974"}}
+{"timestamp":1714482721.1702683,"name":"online","context":{"idset":"8959,8978"}}
+{"timestamp":1714482721.4657726,"name":"online","context":{"idset":"8951,8956,8963,8997,9023"}}
+{"timestamp":1714482721.6649373,"name":"online","context":{"idset":"8953-8954,9006,9026"}}
+{"timestamp":1714482721.9533601,"name":"online","context":{"idset":"8992,8998"}}
+{"timestamp":1714482722.7328901,"name":"online","context":{"idset":"8989-8990,9002,9015,9020"}}
+{"timestamp":1714482722.7362325,"name":"online","context":{"idset":"8995,9022,9024"}}
+{"timestamp":1714482722.7395005,"name":"online","context":{"idset":"8984,8987,9013,9019"}}
+{"timestamp":1714482722.7430623,"name":"online","context":{"idset":"8986,8988,8996,9014,9016-9017,9021,9025"}}
+{"timestamp":1714482722.7463274,"name":"online","context":{"idset":"8981,8983,9008,9010,9027"}}
+{"timestamp":1714482723.0255401,"name":"online","context":{"idset":"9009,9011"}}
+{"timestamp":1714482723.1426797,"name":"online","context":{"idset":"9007,9012"}}
+{"timestamp":1714482723.3401928,"name":"online","context":{"idset":"9005"}}
+{"timestamp":1714484746.0641565,"name":"online","context":{"idset":"8107"}}
+{"timestamp":1714485591.085439,"name":"undrain","context":{"idset":"8107"}}
+{"timestamp":1714486064.797518,"name":"online","context":{"idset":"8108"}}
+{"timestamp":1714486322.9197073,"name":"undrain","context":{"idset":"8108"}}
+{"timestamp":1714486636.7473335,"name":"offline","context":{"idset":"8106"}}
+{"timestamp":1714486641.1422222,"name":"offline","context":{"idset":"835"}}
+{"timestamp":1714486644.6487336,"name":"offline","context":{"idset":"836"}}
+{"timestamp":1714488728.3743651,"name":"online","context":{"idset":"953"}}
+{"timestamp":1714488728.3779769,"name":"online","context":{"idset":"951"}}
+{"timestamp":1714488728.3819194,"name":"online","context":{"idset":"952,954"}}
+{"timestamp":1714488728.3858457,"name":"online","context":{"idset":"955"}}
+{"timestamp":1714488728.3897595,"name":"online","context":{"idset":"956"}}
+{"timestamp":1714489493.6352079,"name":"offline","context":{"idset":"9404"}}
+{"timestamp":1714490122.8731401,"name":"online","context":{"idset":"10748"}}
+{"timestamp":1714490123.0875998,"name":"online","context":{"idset":"10042"}}
+{"timestamp":1714490123.3801215,"name":"online","context":{"idset":"10134"}}
+{"timestamp":1714490265.2836893,"name":"online","context":{"idset":"8712,8747-8748"}}
+{"timestamp":1714490265.6966619,"name":"online","context":{"idset":"8711"}}
+{"timestamp":1714490267.7472486,"name":"drain","context":{"idset":"8711-8712,8747-8748","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714490794.4702334,"name":"online","context":{"idset":"8201"}}
+{"timestamp":1714490814.9350595,"name":"drain","context":{"idset":"8201","overwrite":0}}
+{"timestamp":1714491231.6360488,"name":"offline","context":{"idset":"8988"}}
+{"timestamp":1714491233.6315713,"name":"offline","context":{"idset":"8987"}}
+{"timestamp":1714491259.6333332,"name":"offline","context":{"idset":"7828"}}
+{"timestamp":1714491268.3816059,"name":"offline","context":{"idset":"833"}}
+{"timestamp":1714491268.4441743,"name":"offline","context":{"idset":"834"}}
+{"timestamp":1714491711.4179893,"name":"online","context":{"idset":"8234"}}
+{"timestamp":1714492212.7530065,"name":"undrain","context":{"idset":"8304"}}
+{"timestamp":1714492243.2306595,"name":"online","context":{"idset":"8304"}}
+{"timestamp":1714492715.6320779,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1714492869.6313324,"name":"online","context":{"idset":"8286"}}
+{"timestamp":1714492930.7741926,"name":"undrain","context":{"idset":"8234"}}
+{"timestamp":1714493295.6360483,"name":"offline","context":{"idset":"8294"}}
+{"timestamp":1714493912.3532624,"name":"online","context":{"idset":"548"}}
+{"timestamp":1714494112.7093832,"name":"drain","context":{"idset":"8291","reason":"nodediag failed bogomips","overwrite":0}}
+{"timestamp":1714494742.5129311,"name":"online","context":{"idset":"9328"}}
+{"timestamp":1714494743.4351027,"name":"online","context":{"idset":"9327"}}
+{"timestamp":1714495494.5754325,"name":"undrain","context":{"idset":"8711-8712,8747-8748"}}
+{"timestamp":1714495757.7074912,"name":"undrain","context":{"idset":"8291"}}
+{"timestamp":1714495808.011492,"name":"online","context":{"idset":"8202"}}
+{"timestamp":1714495834.5725117,"name":"online","context":{"idset":"8294"}}
+{"timestamp":1714495872.5303307,"name":"online","context":{"idset":"8233"}}
+{"timestamp":1714495953.0709648,"name":"offline","context":{"idset":"8201"}}
+{"timestamp":1714496921.3215721,"name":"online","context":{"idset":"8201"}}
+{"timestamp":1714496939.3272293,"name":"undrain","context":{"idset":"8201"}}
+{"timestamp":1714497520.3022184,"name":"online","context":{"idset":"9535"}}
+{"timestamp":1714497584.6269567,"name":"drain","context":{"idset":"951-956","overwrite":0}}
+{"timestamp":1714498728.0181935,"name":"online","context":{"idset":"7828"}}
+{"timestamp":1714500023.8731766,"name":"offline","context":{"idset":"9040"}}
+{"timestamp":1714500026.5319326,"name":"offline","context":{"idset":"9037"}}
+{"timestamp":1714500550.7317367,"name":"online","context":{"idset":"11276"}}
+{"timestamp":1714500551.2412376,"name":"online","context":{"idset":"11265"}}
+{"timestamp":1714500551.4222121,"name":"online","context":{"idset":"11261"}}
+{"timestamp":1714500552.2218373,"name":"online","context":{"idset":"11253,11257,11262-11263,11268,11281"}}
+{"timestamp":1714500553.2727313,"name":"online","context":{"idset":"11267,11287"}}
+{"timestamp":1714500553.274848,"name":"online","context":{"idset":"11259,11282,11297"}}
+{"timestamp":1714500553.2773046,"name":"online","context":{"idset":"11284"}}
+{"timestamp":1714500553.2801616,"name":"online","context":{"idset":"11255"}}
+{"timestamp":1714500553.2834718,"name":"online","context":{"idset":"11256,11275,11295"}}
+{"timestamp":1714500553.2875464,"name":"online","context":{"idset":"11277"}}
+{"timestamp":1714500553.2915475,"name":"online","context":{"idset":"11254,11273,11290,11338"}}
+{"timestamp":1714500553.3293624,"name":"online","context":{"idset":"11301-11302,11318,11337"}}
+{"timestamp":1714500553.5763187,"name":"online","context":{"idset":"11272,11278-11279,11298,11303,11328,11343,11352"}}
+{"timestamp":1714500554.5007236,"name":"online","context":{"idset":"11289,11296,11306"}}
+{"timestamp":1714500554.5083251,"name":"online","context":{"idset":"11269-11270"}}
+{"timestamp":1714500554.7895281,"name":"online","context":{"idset":"11258,11271,11274,11280,11283,11292-11294,11300,11327,11335,11350,11376"}}
+{"timestamp":1714500554.9484711,"name":"online","context":{"idset":"11260,11285,11299,11380"}}
+{"timestamp":1714500556.5598252,"name":"online","context":{"idset":"11264,11266,11286,11288,11291,11304-11305,11307-11317,11319-11326,11329-11331,11333-11334,11336,11339-11341,11344-11347,11354-11356,11358-11360,11363-11369,11372,11374-11375,11377-11378"}}
+{"timestamp":1714500560.2549238,"name":"online","context":{"idset":"11332,11342,11348-11349,11351,11353,11357,11361-11362,11370-11371,11373,11379"}}
+{"timestamp":1714500676.6802053,"name":"undrain","context":{"idset":"11253-11380"}}
+{"timestamp":1714501273.9868367,"name":"online","context":{"idset":"9404"}}
+{"timestamp":1714501274.2008226,"name":"online","context":{"idset":"9403"}}
+{"timestamp":1714502202.0779846,"name":"online","context":{"idset":"7046-7047"}}
+{"timestamp":1714502376.0869794,"name":"online","context":{"idset":"7165"}}
+{"timestamp":1714502897.2904482,"name":"offline","context":{"idset":"8201"}}
+{"timestamp":1714502898.0313613,"name":"offline","context":{"idset":"8202"}}
+{"timestamp":1714503670.0648248,"name":"offline","context":{"idset":"873"}}
+{"timestamp":1714503670.8063748,"name":"offline","context":{"idset":"874"}}
+{"timestamp":1714503680.8460827,"name":"online","context":{"idset":"914"}}
+{"timestamp":1714503682.994447,"name":"online","context":{"idset":"878"}}
+{"timestamp":1714503682.997499,"name":"online","context":{"idset":"877"}}
+{"timestamp":1714503683.0004814,"name":"online","context":{"idset":"939"}}
+{"timestamp":1714503683.103987,"name":"online","context":{"idset":"848"}}
+{"timestamp":1714503686.1776505,"name":"online","context":{"idset":"847"}}
+{"timestamp":1714503686.1821556,"name":"online","context":{"idset":"937"}}
+{"timestamp":1714503687.1313157,"name":"online","context":{"idset":"913"}}
+{"timestamp":1714503708.9432063,"name":"online","context":{"idset":"938"}}
+{"timestamp":1714503725.0595894,"name":"online","context":{"idset":"940"}}
+{"timestamp":1714503902.3483834,"name":"undrain","context":{"idset":"877-878,937-939"}}
+{"timestamp":1714504393.9673648,"name":"online","context":{"idset":"11215"}}
+{"timestamp":1714504394.3467972,"name":"online","context":{"idset":"11071"}}
+{"timestamp":1714504394.8331821,"name":"online","context":{"idset":"11239"}}
+{"timestamp":1714504395.9287689,"name":"online","context":{"idset":"11237"}}
+{"timestamp":1714504396.6889405,"name":"online","context":{"idset":"11203"}}
+{"timestamp":1714504397.0866277,"name":"online","context":{"idset":"11222"}}
+{"timestamp":1714504398.0019557,"name":"online","context":{"idset":"11249"}}
+{"timestamp":1714504398.0060933,"name":"online","context":{"idset":"11228"}}
+{"timestamp":1714504398.5018432,"name":"online","context":{"idset":"11051"}}
+{"timestamp":1714504406.0863881,"name":"online","context":{"idset":"11217"}}
+{"timestamp":1714504406.0905893,"name":"online","context":{"idset":"11236"}}
+{"timestamp":1714504406.9105237,"name":"online","context":{"idset":"11052,11208"}}
+{"timestamp":1714504407.0107124,"name":"online","context":{"idset":"11224"}}
+{"timestamp":1714504408.136126,"name":"online","context":{"idset":"11243"}}
+{"timestamp":1714504408.1401052,"name":"online","context":{"idset":"11247"}}
+{"timestamp":1714504408.9767027,"name":"online","context":{"idset":"11234"}}
+{"timestamp":1714504408.9899142,"name":"online","context":{"idset":"11207,11790"}}
+{"timestamp":1714504409.1504664,"name":"online","context":{"idset":"11205"}}
+{"timestamp":1714504410.1191738,"name":"online","context":{"idset":"11241"}}
+{"timestamp":1714504410.1231675,"name":"online","context":{"idset":"11213"}}
+{"timestamp":1714504410.9195507,"name":"online","context":{"idset":"11210"}}
+{"timestamp":1714504410.9242477,"name":"online","context":{"idset":"11216"}}
+{"timestamp":1714504411.1861486,"name":"online","context":{"idset":"11789"}}
+{"timestamp":1714504412.5189824,"name":"online","context":{"idset":"11214"}}
+{"timestamp":1714504412.5229032,"name":"online","context":{"idset":"11022"}}
+{"timestamp":1714504412.6629732,"name":"online","context":{"idset":"11219"}}
+{"timestamp":1714504412.8489089,"name":"online","context":{"idset":"11245"}}
+{"timestamp":1714504414.7563062,"name":"online","context":{"idset":"11212"}}
+{"timestamp":1714504414.9665666,"name":"online","context":{"idset":"11037"}}
+{"timestamp":1714504416.0057983,"name":"online","context":{"idset":"11230"}}
+{"timestamp":1714504418.7247062,"name":"online","context":{"idset":"11218"}}
+{"timestamp":1714504420.0238721,"name":"online","context":{"idset":"11226"}}
+{"timestamp":1714504421.290576,"name":"online","context":{"idset":"11251"}}
+{"timestamp":1714504422.8741269,"name":"online","context":{"idset":"11209"}}
+{"timestamp":1714504423.9895337,"name":"online","context":{"idset":"11204"}}
+{"timestamp":1714504424.9478316,"name":"online","context":{"idset":"11206"}}
+{"timestamp":1714504426.7047768,"name":"online","context":{"idset":"11232"}}
+{"timestamp":1714504427.2053707,"name":"online","context":{"idset":"11220"}}
+{"timestamp":1714504434.7701776,"name":"online","context":{"idset":"11211"}}
+{"timestamp":1714504437.9850471,"name":"undrain","context":{"idset":"11789-11790"}}
+{"timestamp":1714504442.2265363,"name":"online","context":{"idset":"11240,11242"}}
+{"timestamp":1714504446.9539354,"name":"online","context":{"idset":"11223"}}
+{"timestamp":1714504448.1784964,"name":"online","context":{"idset":"11225"}}
+{"timestamp":1714504448.1827097,"name":"online","context":{"idset":"11233"}}
+{"timestamp":1714504451.1793134,"name":"online","context":{"idset":"11221"}}
+{"timestamp":1714504451.2301817,"name":"online","context":{"idset":"11235"}}
+{"timestamp":1714504453.84483,"name":"online","context":{"idset":"11250"}}
+{"timestamp":1714504460.8686323,"name":"online","context":{"idset":"11227"}}
+{"timestamp":1714504462.0742083,"name":"online","context":{"idset":"11007"}}
+{"timestamp":1714504462.7859621,"name":"online","context":{"idset":"11246"}}
+{"timestamp":1714504466.1187868,"name":"online","context":{"idset":"11248"}}
+{"timestamp":1714504466.1229248,"name":"online","context":{"idset":"11238"}}
+{"timestamp":1714504469.13184,"name":"online","context":{"idset":"11244"}}
+{"timestamp":1714504473.0965068,"name":"online","context":{"idset":"11231"}}
+{"timestamp":1714504502.8770552,"name":"online","context":{"idset":"11229"}}
+{"timestamp":1714504520.1523259,"name":"online","context":{"idset":"11252"}}
+{"timestamp":1714504762.5762198,"name":"undrain","context":{"idset":"11203-11252"}}
+{"timestamp":1714504894.6691492,"name":"online","context":{"idset":"11730"}}
+{"timestamp":1714504894.7844081,"name":"online","context":{"idset":"11753"}}
+{"timestamp":1714504895.9894388,"name":"online","context":{"idset":"11754"}}
+{"timestamp":1714504897.1906416,"name":"online","context":{"idset":"11729"}}
+{"timestamp":1714505040.8020225,"name":"offline","context":{"idset":"8105"}}
+{"timestamp":1714505146.9513996,"name":"offline","context":{"idset":"817"}}
+{"timestamp":1714505150.0332277,"name":"offline","context":{"idset":"818"}}
+{"timestamp":1714505163.6579297,"name":"drain","context":{"idset":"11291","reason":"nodediag failed amdapu cxi","overwrite":0}}
+{"timestamp":1714505257.8451107,"name":"drain","context":{"idset":"11285","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714505351.3321345,"name":"online","context":{"idset":"7178-7179,7183-7184,7186"}}
+{"timestamp":1714505351.33603,"name":"online","context":{"idset":"7173,7176,7181"}}
+{"timestamp":1714505352.844461,"name":"online","context":{"idset":"7174-7175,7177,7180,7182,7185,7187-7188"}}
+{"timestamp":1714505536.9223449,"name":"online","context":{"idset":"7190"}}
+{"timestamp":1714505538.0614896,"name":"online","context":{"idset":"7195"}}
+{"timestamp":1714505538.065531,"name":"online","context":{"idset":"7198"}}
+{"timestamp":1714505538.1636629,"name":"online","context":{"idset":"7191"}}
+{"timestamp":1714505538.3554916,"name":"online","context":{"idset":"7200,7203"}}
+{"timestamp":1714505538.6469471,"name":"online","context":{"idset":"7189,7194,7202"}}
+{"timestamp":1714505538.7487998,"name":"online","context":{"idset":"7192"}}
+{"timestamp":1714505538.8526146,"name":"online","context":{"idset":"7201"}}
+{"timestamp":1714505539.0445371,"name":"online","context":{"idset":"7196"}}
+{"timestamp":1714505539.1496065,"name":"online","context":{"idset":"7193,7199"}}
+{"timestamp":1714505540.008822,"name":"online","context":{"idset":"7204"}}
+{"timestamp":1714505540.0123136,"name":"online","context":{"idset":"7197"}}
+{"timestamp":1714507258.5888536,"name":"offline","context":{"idset":"11291"}}
+{"timestamp":1714507258.7458248,"name":"offline","context":{"idset":"11292"}}
+{"timestamp":1714508426.491092,"name":"online","context":{"idset":"7163"}}
+{"timestamp":1714508432.0198681,"name":"online","context":{"idset":"7172"}}
+{"timestamp":1714508438.950171,"name":"online","context":{"idset":"7157"}}
+{"timestamp":1714508440.1780503,"name":"online","context":{"idset":"7159"}}
+{"timestamp":1714508440.8842111,"name":"online","context":{"idset":"7160"}}
+{"timestamp":1714508442.3516934,"name":"online","context":{"idset":"7167"}}
+{"timestamp":1714508442.6845529,"name":"online","context":{"idset":"7161"}}
+{"timestamp":1714508444.5574508,"name":"online","context":{"idset":"7166"}}
+{"timestamp":1714508444.6653552,"name":"online","context":{"idset":"7158"}}
+{"timestamp":1714508446.7402663,"name":"online","context":{"idset":"7169,7171"}}
+{"timestamp":1714508449.4902849,"name":"online","context":{"idset":"7168"}}
+{"timestamp":1714508449.615818,"name":"online","context":{"idset":"7164"}}
+{"timestamp":1714508450.6023083,"name":"online","context":{"idset":"7162,7170"}}
+{"timestamp":1714508998.1116002,"name":"online","context":{"idset":"11724"}}
+{"timestamp":1714509006.0262556,"name":"online","context":{"idset":"11723"}}
+{"timestamp":1714509074.0504248,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1714509832.1402805,"name":"online","context":{"idset":"548"}}
+{"timestamp":1714510038.7851138,"name":"online","context":{"idset":"7245"}}
+{"timestamp":1714510108.4181952,"name":"drain","context":{"idset":"11121","reason":"nodediag failed cxi pci","overwrite":0}}
+{"timestamp":1714510414.0023372,"name":"undrain","context":{"idset":"11121"}}
+{"timestamp":1714510414.1407797,"name":"offline","context":{"idset":"808"}}
+{"timestamp":1714510532.8558249,"name":"offline","context":{"idset":"883"}}
+{"timestamp":1714510533.5237062,"name":"offline","context":{"idset":"884"}}
+{"timestamp":1714510654.8551407,"name":"offline","context":{"idset":"805"}}
+{"timestamp":1714510654.9371357,"name":"offline","context":{"idset":"806"}}
+{"timestamp":1714510699.6796591,"name":"online","context":{"idset":"948"}}
+{"timestamp":1714510699.68314,"name":"online","context":{"idset":"947"}}
+{"timestamp":1714510796.8590691,"name":"online","context":{"idset":"961"}}
+{"timestamp":1714510798.5258443,"name":"online","context":{"idset":"962"}}
+{"timestamp":1714510808.7395909,"name":"online","context":{"idset":"963"}}
+{"timestamp":1714510809.7466614,"name":"online","context":{"idset":"964"}}
+{"timestamp":1714510833.6000066,"name":"online","context":{"idset":"7206"}}
+{"timestamp":1714510841.7673461,"name":"online","context":{"idset":"7211"}}
+{"timestamp":1714510841.7706614,"name":"online","context":{"idset":"7209"}}
+{"timestamp":1714510843.8511302,"name":"online","context":{"idset":"7215"}}
+{"timestamp":1714510845.624984,"name":"online","context":{"idset":"7208"}}
+{"timestamp":1714510845.6272931,"name":"online","context":{"idset":"7205"}}
+{"timestamp":1714510845.8354585,"name":"online","context":{"idset":"7212"}}
+{"timestamp":1714510846.0151012,"name":"online","context":{"idset":"7220"}}
+{"timestamp":1714510846.2058272,"name":"online","context":{"idset":"7213"}}
+{"timestamp":1714510846.3139954,"name":"online","context":{"idset":"7214"}}
+{"timestamp":1714510846.5159128,"name":"online","context":{"idset":"7216,7219"}}
+{"timestamp":1714510846.7164917,"name":"online","context":{"idset":"7210,7217"}}
+{"timestamp":1714510847.5656285,"name":"online","context":{"idset":"7207,7218"}}
+{"timestamp":1714510860.773092,"name":"online","context":{"idset":"7226"}}
+{"timestamp":1714510863.7022679,"name":"online","context":{"idset":"7224"}}
+{"timestamp":1714510865.9236217,"name":"online","context":{"idset":"7232"}}
+{"timestamp":1714510868.8133607,"name":"online","context":{"idset":"7234"}}
+{"timestamp":1714510869.6403329,"name":"online","context":{"idset":"7227,7230"}}
+{"timestamp":1714510869.6691444,"name":"online","context":{"idset":"7221"}}
+{"timestamp":1714510869.9489722,"name":"online","context":{"idset":"7223"}}
+{"timestamp":1714510870.4759686,"name":"online","context":{"idset":"7225,7228,7235"}}
+{"timestamp":1714510870.6872995,"name":"online","context":{"idset":"7229,7231,7236"}}
+{"timestamp":1714510871.6842299,"name":"online","context":{"idset":"7233"}}
+{"timestamp":1714510871.6903532,"name":"online","context":{"idset":"7222"}}
+{"timestamp":1714511112.4353304,"name":"online","context":{"idset":"10893"}}
+{"timestamp":1714511112.6888943,"name":"online","context":{"idset":"10886-10887"}}
+{"timestamp":1714511112.9550581,"name":"online","context":{"idset":"10891,10894"}}
+{"timestamp":1714511113.0631709,"name":"online","context":{"idset":"10892"}}
+{"timestamp":1714511113.1833692,"name":"online","context":{"idset":"10898"}}
+{"timestamp":1714511113.2857702,"name":"online","context":{"idset":"10895"}}
+{"timestamp":1714511113.4462438,"name":"online","context":{"idset":"10888-10889,10896-10897,10900"}}
+{"timestamp":1714511113.6606584,"name":"online","context":{"idset":"10885,10890,10899"}}
+{"timestamp":1714511126.3288124,"name":"online","context":{"idset":"10902"}}
+{"timestamp":1714511126.5749822,"name":"online","context":{"idset":"10905,10913"}}
+{"timestamp":1714511126.6955886,"name":"online","context":{"idset":"10904"}}
+{"timestamp":1714511126.8066692,"name":"online","context":{"idset":"10907"}}
+{"timestamp":1714511127.1088135,"name":"online","context":{"idset":"10912"}}
+{"timestamp":1714511127.4777625,"name":"online","context":{"idset":"10901,10910"}}
+{"timestamp":1714511127.6945827,"name":"online","context":{"idset":"10903,10906,10908,10911,10915"}}
+{"timestamp":1714511127.8262441,"name":"online","context":{"idset":"10914"}}
+{"timestamp":1714511128.0477233,"name":"online","context":{"idset":"10916"}}
+{"timestamp":1714511128.5733571,"name":"online","context":{"idset":"10909"}}
+{"timestamp":1714511138.2846451,"name":"online","context":{"idset":"10917"}}
+{"timestamp":1714511138.5519655,"name":"online","context":{"idset":"10918"}}
+{"timestamp":1714511143.6065845,"name":"online","context":{"idset":"10922"}}
+{"timestamp":1714511143.6101711,"name":"online","context":{"idset":"10921"}}
+{"timestamp":1714511144.3375623,"name":"online","context":{"idset":"10931"}}
+{"timestamp":1714511144.4452922,"name":"online","context":{"idset":"10925"}}
+{"timestamp":1714511144.5532422,"name":"online","context":{"idset":"10924,10926-10927"}}
+{"timestamp":1714511144.768012,"name":"online","context":{"idset":"10919,10923"}}
+{"timestamp":1714511145.6188054,"name":"online","context":{"idset":"10929"}}
+{"timestamp":1714511146.1380041,"name":"online","context":{"idset":"10920"}}
+{"timestamp":1714511146.4396629,"name":"online","context":{"idset":"10930,10932"}}
+{"timestamp":1714511148.0021849,"name":"online","context":{"idset":"10928"}}
+{"timestamp":1714511150.2082283,"name":"online","context":{"idset":"10933-10934"}}
+{"timestamp":1714511155.6400199,"name":"online","context":{"idset":"10935-10936"}}
+{"timestamp":1714511156.1323814,"name":"online","context":{"idset":"10938"}}
+{"timestamp":1714511156.2418139,"name":"online","context":{"idset":"10937"}}
+{"timestamp":1714511156.9062812,"name":"online","context":{"idset":"10939-10940,10943"}}
+{"timestamp":1714511157.8234694,"name":"online","context":{"idset":"10941-10942"}}
+{"timestamp":1714511157.8278563,"name":"online","context":{"idset":"10944"}}
+{"timestamp":1714511157.8323419,"name":"online","context":{"idset":"10945"}}
+{"timestamp":1714511158.1587608,"name":"online","context":{"idset":"10946"}}
+{"timestamp":1714511158.400661,"name":"online","context":{"idset":"10947"}}
+{"timestamp":1714511160.8001001,"name":"online","context":{"idset":"10948"}}
+{"timestamp":1714511162.2414865,"name":"online","context":{"idset":"10949"}}
+{"timestamp":1714511162.610858,"name":"online","context":{"idset":"10950"}}
+{"timestamp":1714511167.5970323,"name":"online","context":{"idset":"10951"}}
+{"timestamp":1714511167.6234398,"name":"online","context":{"idset":"10952"}}
+{"timestamp":1714511168.1806998,"name":"online","context":{"idset":"10953-10954"}}
+{"timestamp":1714511168.471699,"name":"online","context":{"idset":"10957"}}
+{"timestamp":1714511168.7031412,"name":"online","context":{"idset":"10958"}}
+{"timestamp":1714511168.8109765,"name":"online","context":{"idset":"10956"}}
+{"timestamp":1714511169.7637148,"name":"online","context":{"idset":"10955,10959"}}
+{"timestamp":1714511169.7673383,"name":"online","context":{"idset":"10960-10961"}}
+{"timestamp":1714511170.2938807,"name":"online","context":{"idset":"10962-10963"}}
+{"timestamp":1714511172.5489178,"name":"online","context":{"idset":"10964"}}
+{"timestamp":1714511174.4592257,"name":"online","context":{"idset":"10965"}}
+{"timestamp":1714511175.6466877,"name":"online","context":{"idset":"10966"}}
+{"timestamp":1714511179.8170688,"name":"online","context":{"idset":"10968"}}
+{"timestamp":1714511179.8207839,"name":"online","context":{"idset":"10967"}}
+{"timestamp":1714511179.9834347,"name":"online","context":{"idset":"10969-10970"}}
+{"timestamp":1714511180.3589413,"name":"online","context":{"idset":"10973"}}
+{"timestamp":1714511180.6833413,"name":"online","context":{"idset":"10971-10972"}}
+{"timestamp":1714511181.6233056,"name":"online","context":{"idset":"10974"}}
+{"timestamp":1714511181.6267495,"name":"online","context":{"idset":"10975,10977"}}
+{"timestamp":1714511182.1539173,"name":"online","context":{"idset":"10976,10979"}}
+{"timestamp":1714511182.4603515,"name":"online","context":{"idset":"10978"}}
+{"timestamp":1714511184.1000659,"name":"online","context":{"idset":"10980"}}
+{"timestamp":1714511185.9485819,"name":"online","context":{"idset":"10981"}}
+{"timestamp":1714511186.76284,"name":"online","context":{"idset":"10982"}}
+{"timestamp":1714511190.9224889,"name":"online","context":{"idset":"10983"}}
+{"timestamp":1714511191.6377246,"name":"online","context":{"idset":"10984"}}
+{"timestamp":1714511191.9992607,"name":"online","context":{"idset":"10985-10986"}}
+{"timestamp":1714511192.3359315,"name":"online","context":{"idset":"10987"}}
+{"timestamp":1714511192.5497668,"name":"online","context":{"idset":"10988"}}
+{"timestamp":1714511192.8504369,"name":"online","context":{"idset":"10989"}}
+{"timestamp":1714511193.7825773,"name":"online","context":{"idset":"10991"}}
+{"timestamp":1714511193.7870059,"name":"online","context":{"idset":"10992"}}
+{"timestamp":1714511193.7913797,"name":"online","context":{"idset":"10990,10993"}}
+{"timestamp":1714511193.95382,"name":"online","context":{"idset":"10994"}}
+{"timestamp":1714511194.5942545,"name":"online","context":{"idset":"10995"}}
+{"timestamp":1714511196.1773987,"name":"online","context":{"idset":"10996"}}
+{"timestamp":1714511202.4132636,"name":"drain","context":{"idset":"878","reason":"CPER error","overwrite":0}}
+{"timestamp":1714511214.1567554,"name":"drain","context":{"idset":"877","reason":"Bad Partner Node","overwrite":0}}
+{"timestamp":1714511230.0923395,"name":"offline","context":{"idset":"878"}}
+{"timestamp":1714511230.1908331,"name":"offline","context":{"idset":"877"}}
+{"timestamp":1714512675.7386408,"name":"online","context":{"idset":"11513"}}
+{"timestamp":1714512675.9497135,"name":"online","context":{"idset":"11512,11517"}}
+{"timestamp":1714512676.1417682,"name":"online","context":{"idset":"11514"}}
+{"timestamp":1714512676.2683053,"name":"online","context":{"idset":"11524"}}
+{"timestamp":1714512676.6399384,"name":"online","context":{"idset":"11509,11511,11516,11519-11520"}}
+{"timestamp":1714512676.8607273,"name":"online","context":{"idset":"11510,11515,11521,11523"}}
+{"timestamp":1714512677.8332934,"name":"online","context":{"idset":"11518,11522"}}
+{"timestamp":1714512687.7490237,"name":"online","context":{"idset":"11525"}}
+{"timestamp":1714512688.2635126,"name":"online","context":{"idset":"11527-11528,11537"}}
+{"timestamp":1714512688.4734175,"name":"online","context":{"idset":"11526,11531"}}
+{"timestamp":1714512688.754607,"name":"online","context":{"idset":"11532-11533,11535"}}
+{"timestamp":1714512688.8773634,"name":"online","context":{"idset":"11530"}}
+{"timestamp":1714512689.7214887,"name":"online","context":{"idset":"11529,11534,11538-11540"}}
+{"timestamp":1714512698.5159407,"name":"online","context":{"idset":"11541"}}
+{"timestamp":1714512699.727314,"name":"online","context":{"idset":"11542"}}
+{"timestamp":1714512700.2607193,"name":"online","context":{"idset":"11546"}}
+{"timestamp":1714512700.6107476,"name":"online","context":{"idset":"11543"}}
+{"timestamp":1714512702.0296636,"name":"online","context":{"idset":"11544-11545,11547,11551,11553"}}
+{"timestamp":1714512702.0332065,"name":"online","context":{"idset":"11548-11550,11552,11554-11556"}}
+{"timestamp":1714512710.0910606,"name":"online","context":{"idset":"11557"}}
+{"timestamp":1714512712.3469627,"name":"online","context":{"idset":"11558"}}
+{"timestamp":1714512712.3520811,"name":"online","context":{"idset":"11559-11560,11563"}}
+{"timestamp":1714512712.6975729,"name":"online","context":{"idset":"11561"}}
+{"timestamp":1714512712.9371891,"name":"online","context":{"idset":"11562,11565,11567-11568"}}
+{"timestamp":1714512713.9022675,"name":"online","context":{"idset":"11570"}}
+{"timestamp":1714512713.9063411,"name":"online","context":{"idset":"11564,11569"}}
+{"timestamp":1714512713.9105151,"name":"online","context":{"idset":"11566,11571-11572"}}
+{"timestamp":1714512721.977428,"name":"online","context":{"idset":"11573"}}
+{"timestamp":1714512723.6859586,"name":"online","context":{"idset":"11574"}}
+{"timestamp":1714512724.4589441,"name":"online","context":{"idset":"11576-11577"}}
+{"timestamp":1714512724.7204914,"name":"online","context":{"idset":"11578"}}
+{"timestamp":1714512725.9972062,"name":"online","context":{"idset":"11586"}}
+{"timestamp":1714512726.0016744,"name":"online","context":{"idset":"11587"}}
+{"timestamp":1714512726.0060248,"name":"online","context":{"idset":"11575,11579,11583"}}
+{"timestamp":1714512726.0103765,"name":"online","context":{"idset":"11580-11581,11584,11588"}}
+{"timestamp":1714512726.0146937,"name":"online","context":{"idset":"11582"}}
+{"timestamp":1714512726.0190988,"name":"online","context":{"idset":"11585"}}
+{"timestamp":1714512734.2442179,"name":"online","context":{"idset":"11589"}}
+{"timestamp":1714512735.7651513,"name":"online","context":{"idset":"11590"}}
+{"timestamp":1714512736.4287667,"name":"online","context":{"idset":"11592"}}
+{"timestamp":1714512736.6682165,"name":"online","context":{"idset":"11591,11593-11594"}}
+{"timestamp":1714512736.7932086,"name":"online","context":{"idset":"11595"}}
+{"timestamp":1714512736.8961029,"name":"online","context":{"idset":"11596"}}
+{"timestamp":1714512737.9168358,"name":"online","context":{"idset":"11597-11600"}}
+{"timestamp":1714512737.920506,"name":"online","context":{"idset":"11602-11603"}}
+{"timestamp":1714512737.9242003,"name":"online","context":{"idset":"11601,11604"}}
+{"timestamp":1714512745.7341418,"name":"online","context":{"idset":"11605"}}
+{"timestamp":1714512746.8337591,"name":"online","context":{"idset":"11606"}}
+{"timestamp":1714512746.946712,"name":"drain","context":{"idset":"11612","reason":"broker was unresponsive"}}
+{"timestamp":1714512747.8693728,"name":"online","context":{"idset":"11607"}}
+{"timestamp":1714512748.6922083,"name":"online","context":{"idset":"11609"}}
+{"timestamp":1714512748.6959851,"name":"online","context":{"idset":"11610,11615"}}
+{"timestamp":1714512748.6997349,"name":"online","context":{"idset":"11608,11611,11614"}}
+{"timestamp":1714512748.7825203,"name":"online","context":{"idset":"11613"}}
+{"timestamp":1714512749.7046609,"name":"online","context":{"idset":"11616"}}
+{"timestamp":1714512749.7081726,"name":"online","context":{"idset":"11612,11617-11618"}}
+{"timestamp":1714512749.7116859,"name":"online","context":{"idset":"11620"}}
+{"timestamp":1714512750.3301785,"name":"online","context":{"idset":"11619"}}
+{"timestamp":1714512757.7637219,"name":"online","context":{"idset":"11621"}}
+{"timestamp":1714512758.5224299,"name":"online","context":{"idset":"11622"}}
+{"timestamp":1714512759.9335263,"name":"online","context":{"idset":"11623"}}
+{"timestamp":1714512759.9491098,"name":"online","context":{"idset":"11624-11625"}}
+{"timestamp":1714512760.5093219,"name":"online","context":{"idset":"11626"}}
+{"timestamp":1714512760.6230175,"name":"online","context":{"idset":"11630"}}
+{"timestamp":1714512760.7543793,"name":"online","context":{"idset":"11628,11632"}}
+{"timestamp":1714512761.7631619,"name":"online","context":{"idset":"11627,11631,11635"}}
+{"timestamp":1714512761.7668285,"name":"online","context":{"idset":"11629,11634"}}
+{"timestamp":1714512761.7704756,"name":"online","context":{"idset":"11633,11636"}}
+{"timestamp":1714512872.9007821,"name":"undrain","context":{"idset":"11509-11524"}}
+{"timestamp":1714512894.1911957,"name":"undrain","context":{"idset":"11525-11534,11537-11540"}}
+{"timestamp":1714512912.9793031,"name":"undrain","context":{"idset":"11541,11548,11554-11556,11560-11566,11568-11569"}}
+{"timestamp":1714512939.8655112,"name":"undrain","context":{"idset":"11542-11547,11549-11553,11557-11559,11567,11570-11572"}}
+{"timestamp":1714512939.8692873,"name":"online","context":{"idset":"7306"}}
+{"timestamp":1714512940.4085784,"name":"online","context":{"idset":"7307"}}
+{"timestamp":1714512940.9150949,"name":"online","context":{"idset":"7331"}}
+{"timestamp":1714512941.8885384,"name":"online","context":{"idset":"7327"}}
+{"timestamp":1714512942.4310913,"name":"online","context":{"idset":"7335"}}
+{"timestamp":1714512942.5585346,"name":"online","context":{"idset":"7316"}}
+{"timestamp":1714512943.6694322,"name":"online","context":{"idset":"7301"}}
+{"timestamp":1714512943.6733155,"name":"online","context":{"idset":"7329"}}
+{"timestamp":1714512943.8370025,"name":"online","context":{"idset":"7305"}}
+{"timestamp":1714512943.9591501,"name":"online","context":{"idset":"7381"}}
+{"timestamp":1714512944.1246631,"name":"online","context":{"idset":"7339"}}
+{"timestamp":1714512944.2544875,"name":"online","context":{"idset":"7332"}}
+{"timestamp":1714512944.3680191,"name":"online","context":{"idset":"7355"}}
+{"timestamp":1714512944.6877401,"name":"online","context":{"idset":"7326"}}
+{"timestamp":1714512945.7644675,"name":"online","context":{"idset":"7287,7322,7372"}}
+{"timestamp":1714512945.7710459,"name":"online","context":{"idset":"7289"}}
+{"timestamp":1714512945.7777402,"name":"online","context":{"idset":"7296,7343"}}
+{"timestamp":1714512945.7890077,"name":"online","context":{"idset":"7350"}}
+{"timestamp":1714512945.7970047,"name":"online","context":{"idset":"7315,7362,7378"}}
+{"timestamp":1714512945.8051727,"name":"online","context":{"idset":"7299"}}
+{"timestamp":1714512945.9598415,"name":"online","context":{"idset":"7360"}}
+{"timestamp":1714512947.1412194,"name":"online","context":{"idset":"7285,7294,7309,7313,7317,7330,7348,7369"}}
+{"timestamp":1714512947.1451859,"name":"online","context":{"idset":"7319"}}
+{"timestamp":1714512947.1508849,"name":"online","context":{"idset":"7293,7310,7312,7318,7336,7344,7347,7352,7387,7392,7404,7409"}}
+{"timestamp":1714512947.1557925,"name":"online","context":{"idset":"7292,7295,7297-7298,7300,7302,7314,7323,7325,7328,7345,7351,7354,7356,7365,7377,7389-7390,7400-7401"}}
+{"timestamp":1714512947.162272,"name":"online","context":{"idset":"7288,7358,7394,7406"}}
+{"timestamp":1714512947.1676221,"name":"online","context":{"idset":"7291,7340,7353,7405"}}
+{"timestamp":1714512947.1724174,"name":"online","context":{"idset":"7320,7333,7338,7368,7391,7393,7411-7412"}}
+{"timestamp":1714512947.1767371,"name":"online","context":{"idset":"7290,7304,7311,7321,7324,7337,7342,7359,7364,7367,7370,7375,7379,7397,7408"}}
+{"timestamp":1714512947.1808441,"name":"online","context":{"idset":"7303,7349,7361,7366,7395-7396"}}
+{"timestamp":1714512947.4461246,"name":"online","context":{"idset":"7286,7308,7334,7371,7373-7374,7380,7383-7385,7388,7398-7399,7402-7403,7407,7410"}}
+{"timestamp":1714512947.565455,"name":"online","context":{"idset":"7376"}}
+{"timestamp":1714512947.7895756,"name":"online","context":{"idset":"7341,7346,7357,7363,7382,7386"}}
+{"timestamp":1714512978.1419365,"name":"undrain","context":{"idset":"11573-11578,11580-11582,11584-11588"}}
+{"timestamp":1714512982.230891,"name":"undrain","context":{"idset":"7285-7412"}}
+{"timestamp":1714513064.7200713,"name":"undrain","context":{"idset":"11612"}}
+{"timestamp":1714513092.9020724,"name":"online","context":{"idset":"8202"}}
+{"timestamp":1714513095.1839814,"name":"online","context":{"idset":"8201"}}
+{"timestamp":1714513352.0356483,"name":"undrain","context":{"idset":"11536"}}
+{"timestamp":1714513404.3798966,"name":"online","context":{"idset":"11536"}}
+{"timestamp":1714513681.9180088,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1714514650.6121511,"name":"online","context":{"idset":"7676"}}
+{"timestamp":1714514651.0046923,"name":"online","context":{"idset":"7671,7679"}}
+{"timestamp":1714514651.2468743,"name":"online","context":{"idset":"7675,7678"}}
+{"timestamp":1714514651.3803339,"name":"online","context":{"idset":"7683"}}
+{"timestamp":1714514651.4889877,"name":"online","context":{"idset":"7669,7684"}}
+{"timestamp":1714514651.6149585,"name":"online","context":{"idset":"7670,7672-7674"}}
+{"timestamp":1714514652.6257215,"name":"online","context":{"idset":"7677,7680-7681"}}
+{"timestamp":1714514652.6431782,"name":"online","context":{"idset":"7682"}}
+{"timestamp":1714514662.4335198,"name":"online","context":{"idset":"7685"}}
+{"timestamp":1714514663.1524079,"name":"online","context":{"idset":"7686"}}
+{"timestamp":1714514663.4035146,"name":"online","context":{"idset":"7688"}}
+{"timestamp":1714514663.7130184,"name":"online","context":{"idset":"7689-7690"}}
+{"timestamp":1714514664.5835042,"name":"online","context":{"idset":"7692"}}
+{"timestamp":1714514664.5873959,"name":"online","context":{"idset":"7694"}}
+{"timestamp":1714514664.5910902,"name":"online","context":{"idset":"7693"}}
+{"timestamp":1714514664.594568,"name":"online","context":{"idset":"7687,7695,7697-7698"}}
+{"timestamp":1714514664.5980427,"name":"online","context":{"idset":"7696"}}
+{"timestamp":1714514664.6016028,"name":"online","context":{"idset":"7691,7699-7700"}}
+{"timestamp":1714514674.5280423,"name":"online","context":{"idset":"7701"}}
+{"timestamp":1714514675.203476,"name":"online","context":{"idset":"7702,7704-7705"}}
+{"timestamp":1714514675.4323595,"name":"online","context":{"idset":"7703"}}
+{"timestamp":1714514675.6501448,"name":"online","context":{"idset":"7707"}}
+{"timestamp":1714514676.4592705,"name":"online","context":{"idset":"7706,7709"}}
+{"timestamp":1714514676.4646118,"name":"online","context":{"idset":"7708"}}
+{"timestamp":1714514676.4697058,"name":"online","context":{"idset":"7710-7711,7713,7715"}}
+{"timestamp":1714514676.475399,"name":"online","context":{"idset":"7712,7714"}}
+{"timestamp":1714514677.0212653,"name":"online","context":{"idset":"7716"}}
+{"timestamp":1714514686.4912739,"name":"online","context":{"idset":"7717"}}
+{"timestamp":1714514686.9780397,"name":"online","context":{"idset":"7719"}}
+{"timestamp":1714514687.5083687,"name":"online","context":{"idset":"7718"}}
+{"timestamp":1714514687.6409254,"name":"online","context":{"idset":"7721,7723"}}
+{"timestamp":1714514688.9484086,"name":"online","context":{"idset":"7720"}}
+{"timestamp":1714514688.9558914,"name":"online","context":{"idset":"7722,7724,7726-7727"}}
+{"timestamp":1714514688.9635613,"name":"online","context":{"idset":"7728,7730-7731"}}
+{"timestamp":1714514688.9707966,"name":"online","context":{"idset":"7725,7729"}}
+{"timestamp":1714514689.2763629,"name":"online","context":{"idset":"7732"}}
+{"timestamp":1714514698.5776336,"name":"online","context":{"idset":"7733"}}
+{"timestamp":1714514699.1470149,"name":"online","context":{"idset":"7734"}}
+{"timestamp":1714514699.4531498,"name":"online","context":{"idset":"7739"}}
+{"timestamp":1714514699.6928411,"name":"online","context":{"idset":"7735,7738"}}
+{"timestamp":1714514700.5030544,"name":"online","context":{"idset":"7741"}}
+{"timestamp":1714514700.5066316,"name":"online","context":{"idset":"7737"}}
+{"timestamp":1714514700.5101471,"name":"online","context":{"idset":"7736,7740"}}
+{"timestamp":1714514700.5136223,"name":"online","context":{"idset":"7743"}}
+{"timestamp":1714514700.5170295,"name":"online","context":{"idset":"7745"}}
+{"timestamp":1714514700.7581933,"name":"online","context":{"idset":"7742,7744,7746-7747"}}
+{"timestamp":1714514701.1323428,"name":"online","context":{"idset":"7748"}}
+{"timestamp":1714514709.4522352,"name":"online","context":{"idset":"7749"}}
+{"timestamp":1714514710.9431257,"name":"online","context":{"idset":"7750"}}
+{"timestamp":1714514712.587523,"name":"online","context":{"idset":"7753"}}
+{"timestamp":1714514712.591563,"name":"online","context":{"idset":"7751,7755"}}
+{"timestamp":1714514712.5956416,"name":"online","context":{"idset":"7754,7756"}}
+{"timestamp":1714514712.5995264,"name":"online","context":{"idset":"7757-7758"}}
+{"timestamp":1714514712.6035604,"name":"online","context":{"idset":"7761"}}
+{"timestamp":1714514712.6936862,"name":"online","context":{"idset":"7759"}}
+{"timestamp":1714514712.9141593,"name":"online","context":{"idset":"7752,7760,7763"}}
+{"timestamp":1714514713.2134011,"name":"online","context":{"idset":"7762,7764"}}
+{"timestamp":1714514721.4270742,"name":"online","context":{"idset":"7765"}}
+{"timestamp":1714514722.8060892,"name":"online","context":{"idset":"7766"}}
+{"timestamp":1714514725.0580819,"name":"online","context":{"idset":"7767"}}
+{"timestamp":1714514725.0633178,"name":"online","context":{"idset":"7769"}}
+{"timestamp":1714514725.0688908,"name":"online","context":{"idset":"7768,7770-7771"}}
+{"timestamp":1714514725.0737922,"name":"online","context":{"idset":"7773,7775"}}
+{"timestamp":1714514725.0787473,"name":"online","context":{"idset":"7772,7774"}}
+{"timestamp":1714514725.0834422,"name":"online","context":{"idset":"7778"}}
+{"timestamp":1714514725.0886371,"name":"online","context":{"idset":"7777"}}
+{"timestamp":1714514725.1166418,"name":"online","context":{"idset":"7779"}}
+{"timestamp":1714514725.4428151,"name":"online","context":{"idset":"7780"}}
+{"timestamp":1714514725.6859305,"name":"online","context":{"idset":"7776"}}
+{"timestamp":1714514733.2930086,"name":"online","context":{"idset":"7781"}}
+{"timestamp":1714514735.2623892,"name":"online","context":{"idset":"7782"}}
+{"timestamp":1714514735.6415975,"name":"online","context":{"idset":"7783-7784"}}
+{"timestamp":1714514736.7072456,"name":"online","context":{"idset":"7785,7787"}}
+{"timestamp":1714514736.9855607,"name":"online","context":{"idset":"7789"}}
+{"timestamp":1714514737.2380848,"name":"online","context":{"idset":"7786,7788,7793-7794"}}
+{"timestamp":1714514737.4947577,"name":"online","context":{"idset":"7795"}}
+{"timestamp":1714514737.6961446,"name":"online","context":{"idset":"7790-7792"}}
+{"timestamp":1714514738.3142252,"name":"online","context":{"idset":"7796"}}
+{"timestamp":1714514772.544461,"name":"online","context":{"idset":"883"}}
+{"timestamp":1714514772.6536841,"name":"online","context":{"idset":"884"}}
+{"timestamp":1714514813.1089859,"name":"online","context":{"idset":"548"}}
+{"timestamp":1714514949.6160803,"name":"undrain","context":{"idset":"7730,7739"}}
+{"timestamp":1714515343.1782608,"name":"online","context":{"idset":"834"}}
+{"timestamp":1714515676.5716245,"name":"online","context":{"idset":"833"}}
+{"timestamp":1714515884.0607307,"name":"drain","context":{"idset":"833-834","reason":"--reason troubleshooting for hospital - EG","overwrite":0}}
+{"timestamp":1714516068.9844673,"name":"undrain","context":{"idset":"548,833-834,951-956,961-964,11285"}}
+{"timestamp":1714516354.9380896,"name":"online","context":{"idset":"7239"}}
+{"timestamp":1714516359.0108654,"name":"online","context":{"idset":"7249"}}
+{"timestamp":1714516360.7892323,"name":"online","context":{"idset":"7237,7251"}}
+{"timestamp":1714516361.2965028,"name":"online","context":{"idset":"7241,7243,7247"}}
+{"timestamp":1714516363.2833426,"name":"online","context":{"idset":"7250"}}
+{"timestamp":1714516364.2473207,"name":"online","context":{"idset":"7244,7246,7248"}}
+{"timestamp":1714516364.4635556,"name":"online","context":{"idset":"7240,7242"}}
+{"timestamp":1714516364.5751789,"name":"online","context":{"idset":"7238"}}
+{"timestamp":1714516364.8489516,"name":"online","context":{"idset":"7252"}}
+{"timestamp":1714516851.3947892,"name":"online","context":{"idset":"7253"}}
+{"timestamp":1714516853.5738122,"name":"online","context":{"idset":"7256"}}
+{"timestamp":1714516856.3213253,"name":"online","context":{"idset":"7258-7259"}}
+{"timestamp":1714516857.3721056,"name":"online","context":{"idset":"7264"}}
+{"timestamp":1714516858.4982123,"name":"online","context":{"idset":"7257,7268"}}
+{"timestamp":1714516858.6126704,"name":"online","context":{"idset":"7261,7265"}}
+{"timestamp":1714516858.9086876,"name":"online","context":{"idset":"7266"}}
+{"timestamp":1714516859.2081199,"name":"online","context":{"idset":"7254-7255"}}
+{"timestamp":1714516859.5656092,"name":"online","context":{"idset":"7263"}}
+{"timestamp":1714516860.6549165,"name":"online","context":{"idset":"7262"}}
+{"timestamp":1714516860.6614835,"name":"online","context":{"idset":"7260"}}
+{"timestamp":1714516860.6710093,"name":"online","context":{"idset":"7267"}}
+{"timestamp":1714521076.465456,"name":"online","context":{"idset":"8918"}}
+{"timestamp":1714521079.1721201,"name":"online","context":{"idset":"8843"}}
+{"timestamp":1714521079.1781521,"name":"online","context":{"idset":"8832"}}
+{"timestamp":1714521079.4866152,"name":"online","context":{"idset":"8844"}}
+{"timestamp":1714521080.582479,"name":"online","context":{"idset":"8920"}}
+{"timestamp":1714521082.7107644,"name":"online","context":{"idset":"8821"}}
+{"timestamp":1714521083.5849025,"name":"online","context":{"idset":"8921"}}
+{"timestamp":1714521084.5065928,"name":"online","context":{"idset":"8917,8919"}}
+{"timestamp":1714521085.3309343,"name":"online","context":{"idset":"8839"}}
+{"timestamp":1714521087.0920038,"name":"online","context":{"idset":"8861"}}
+{"timestamp":1714521089.5902138,"name":"online","context":{"idset":"8830"}}
+{"timestamp":1714521090.5385258,"name":"online","context":{"idset":"8831"}}
+{"timestamp":1714521093.1544209,"name":"online","context":{"idset":"8824"}}
+{"timestamp":1714521093.1795917,"name":"online","context":{"idset":"8886"}}
+{"timestamp":1714521093.2385418,"name":"online","context":{"idset":"8829,8841"}}
+{"timestamp":1714521093.7691057,"name":"online","context":{"idset":"8822,8908"}}
+{"timestamp":1714521094.4707663,"name":"online","context":{"idset":"8891"}}
+{"timestamp":1714521094.4760933,"name":"online","context":{"idset":"8825,8893"}}
+{"timestamp":1714521094.4827435,"name":"online","context":{"idset":"8835"}}
+{"timestamp":1714521094.4920316,"name":"online","context":{"idset":"8838"}}
+{"timestamp":1714521094.5017319,"name":"online","context":{"idset":"8826"}}
+{"timestamp":1714521094.5115466,"name":"online","context":{"idset":"8849,8890"}}
+{"timestamp":1714521094.6777267,"name":"online","context":{"idset":"8823,8850,8888"}}
+{"timestamp":1714521094.8473935,"name":"online","context":{"idset":"8827"}}
+{"timestamp":1714521094.9967072,"name":"online","context":{"idset":"8852,8860,8895,8899"}}
+{"timestamp":1714521095.184345,"name":"online","context":{"idset":"8881"}}
+{"timestamp":1714521095.4224441,"name":"online","context":{"idset":"8848"}}
+{"timestamp":1714521095.5382197,"name":"online","context":{"idset":"8868,8897"}}
+{"timestamp":1714521095.7184627,"name":"online","context":{"idset":"8836,8845,8867,8875,8906"}}
+{"timestamp":1714521096.6396215,"name":"online","context":{"idset":"8828,8870"}}
+{"timestamp":1714521096.644876,"name":"online","context":{"idset":"8834,8857,8859,8910"}}
+{"timestamp":1714521096.65046,"name":"online","context":{"idset":"8851,8866,8883"}}
+{"timestamp":1714521096.659447,"name":"online","context":{"idset":"8876,8884,8911,8916"}}
+{"timestamp":1714521096.6666996,"name":"online","context":{"idset":"8842,8872"}}
+{"timestamp":1714521096.6885989,"name":"online","context":{"idset":"8837,8847,8855,8862,8869,8905,8914"}}
+{"timestamp":1714521096.8886888,"name":"online","context":{"idset":"8840,8846,8854,8874,8880,8901"}}
+{"timestamp":1714521097.1049216,"name":"online","context":{"idset":"8853"}}
+{"timestamp":1714521097.2225561,"name":"online","context":{"idset":"8864,8904"}}
+{"timestamp":1714521097.3286517,"name":"online","context":{"idset":"8907"}}
+{"timestamp":1714521097.4468524,"name":"online","context":{"idset":"8856,8858,8877-8878,8900,8915"}}
+{"timestamp":1714521097.5855927,"name":"online","context":{"idset":"8882,8909,8912-8913"}}
+{"timestamp":1714521097.7822258,"name":"online","context":{"idset":"8863,8865,8879"}}
+{"timestamp":1714521098.5907671,"name":"online","context":{"idset":"8873,8885,8903"}}
+{"timestamp":1714521098.5947623,"name":"online","context":{"idset":"8871,8898,8902"}}
+{"timestamp":1714521098.5988898,"name":"online","context":{"idset":"8887"}}
+{"timestamp":1714521098.8722982,"name":"online","context":{"idset":"8896"}}
+{"timestamp":1714521099.7125764,"name":"online","context":{"idset":"8889"}}
+{"timestamp":1714521101.4473052,"name":"online","context":{"idset":"8892,8894"}}
+{"timestamp":1714521222.6352823,"name":"undrain","context":{"idset":"8833"}}
+{"timestamp":1714521274.6085079,"name":"online","context":{"idset":"8833"}}
+{"timestamp":1714521283.7193527,"name":"offline","context":{"idset":"11533"}}
+{"timestamp":1714521439.724292,"name":"offline","context":{"idset":"9061"}}
+{"timestamp":1714521443.7122898,"name":"offline","context":{"idset":"9062"}}
+{"timestamp":1714521443.7179992,"name":"offline","context":{"idset":"9063"}}
+{"timestamp":1714521443.7226543,"name":"offline","context":{"idset":"9064"}}
+{"timestamp":1714521443.7273526,"name":"offline","context":{"idset":"9065"}}
+{"timestamp":1714521443.7410491,"name":"offline","context":{"idset":"9066"}}
+{"timestamp":1714521443.7568443,"name":"offline","context":{"idset":"9067"}}
+{"timestamp":1714527224.1274529,"name":"resource-init","context":{"restart":true,"drain":{"897-898":{"timestamp":1714426321.0,"reason":"--reason Replacing SIVOC on b0"},"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"543":{"timestamp":1714074108.0,"reason":"nodediag failed cxi"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"},"878":{"timestamp":1714511202.0,"reason":"CPER error"},"372":{"timestamp":1713981581.0,"reason":"New Blade Installation --JRG"},"336,491":{"timestamp":1713230524.0,"reason":"Down"},"929-930":{"timestamp":1713552018.0,"reason":""},"773-774":{"timestamp":1713802805.0,"reason":""},"77-84":{"timestamp":1713795613.0,"reason":"New blade installation"},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"117":{"timestamp":1713981538.0,"reason":"Unreachable"},"360":{"timestamp":1713981579.0,"reason":"New Blade Installation --JRG"},"411":{"timestamp":1713564989.0,"reason":"Unreachable"},"807":{"timestamp":1713376017.0,"reason":""},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"811-812":{"timestamp":1713557753.0,"reason":"--reason Running hpe diags - JM"},"779-780":{"timestamp":1713907642.0,"reason":"--reason Troubleshoot BIOS issue"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"881":{"timestamp":1714153692.0,"reason":"--reason H/W troubleshoot"},"286":{"timestamp":1713981553.0,"reason":"Unreachable"},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"971-974":{"timestamp":1714143211.0,"reason":""},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"544":{"timestamp":1714069881.0,"reason":"nodediag failed cxi"},"877":{"timestamp":1714511214.0,"reason":"Bad Partner Node"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"895-896":{"timestamp":1714428105.0,"reason":"--reason Troubleshoot SIVOC on b0"},"206":{"timestamp":1713564985.0,"reason":"Unreachable"},"882":{"timestamp":1714429325.0,"reason":"--reason H/W troubleshoot"},"349-359,361-371,374-396,398-404":{"timestamp":1714060395.0,"reason":"New Blade Installation --JRG"},"545":{"timestamp":1714069878.0,"reason":"nodediag failed cxi"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"141-148":{"timestamp":1713545597.0,"reason":"ARP Storm Problem testing"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"542":{"timestamp":1714074109.0,"reason":"nodediag failed cxi"},"578,580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"149-156":{"timestamp":1713545634.0,"reason":"ARP Storm Problem testing"},"165":{"timestamp":1713981434.0,"reason":"broker was unresponsive"},"797-798":{"timestamp":1712261863.0,"reason":"--reason swapping blades into x1900 - wendy"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"172":{"timestamp":1713981544.0,"reason":"Unreachable"},"373":{"timestamp":1713981698.0,"reason":"New Blade Installation --JRG"},"541":{"timestamp":1714074106.0,"reason":"nodediag failed cxi"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"787-788":{"timestamp":1713219302.0,"reason":""},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"85-89,91-92":{"timestamp":1714079045.0,"reason":"New Blade installation"},"925-926":{"timestamp":1713540640.0,"reason":""},"461-476":{"timestamp":1713794797.0,"reason":"New Blade Installation --JRG"},"397":{"timestamp":1713229804.0,"reason":"New Blade Installation --JRG"},"9003":{"timestamp":1712873964.0,"reason":"nodediag failed dgemm_perf"},"421-460":{"timestamp":1713794410.0,"reason":"New Blade Installation --JRG"},"410":{"timestamp":1713378635.0,"reason":"prolog failed for jobid fq5jtQQSQnT"},"120":{"timestamp":1713981548.0,"reason":"Unreachable"},"803-804":{"timestamp":1713479733.0,"reason":"status"},"789-792":{"timestamp":1711461777.0,"reason":""},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"482-490,492":{"timestamp":1713915346.0,"reason":"New Blade Installation --JRG"},"342":{"timestamp":1713981566.0,"reason":"Unreachable"},"923-924":{"timestamp":1714064541.0,"reason":"status"},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"546":{"timestamp":1714069873.0,"reason":"nodediag failed cxi pci"},"549-556":{"timestamp":1713553592.0,"reason":"New Blade Installation"},"90":{"timestamp":1713564981.0,"reason":"New Blade installation"},"771-772":{"timestamp":1713195979.0,"reason":""},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"11291":{"timestamp":1714505164.0,"reason":"nodediag failed amdapu cxi"},"547":{"timestamp":1713563916.0,"reason":"epilog failed for jobid fqUWmieM3AX"},"814":{"timestamp":1712783332.0,"reason":""},"133-140":{"timestamp":1713545617.0,"reason":"ARP Storm Problem testing"},"829-830":{"timestamp":1712700945.0,"reason":""},"525-532":{"timestamp":1713290620.0,"reason":""},"335":{"timestamp":1713981557.0,"reason":"Unreachable"},"934":{"timestamp":1713967835.0,"reason":"epilog failed for jobid frMYzSv7G6b"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1714527224.1590652,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1714527248.36325,"name":"online","context":{"idset":"0"}}
+{"timestamp":1714527250.1345568,"name":"online","context":{"idset":"883,1182,1187,1205,1210,1214,1219,1223,1228,1231,1233,8825,8831-8832,8839,8845,8862,8864-8865,8867,8879,8885-8886,8889,8893,8909,8913-8914,8947,9091,9093,9107,9110,9119,9126,9137,9177,10631,10666-10667,10683,10688-10689,10703,10710,10718-10719,10722"}}
+{"timestamp":1714527250.2362504,"name":"online","context":{"idset":"963,1178,7960,8198-8199,8843,8850-8851,8857,8872,8883,8899,8903,8929,9081,9099,9121,9125,9142,9161,9175-9176,9183,9190,9197,10724,10734"}}
+{"timestamp":1714527250.3574982,"name":"online","context":{"idset":"826,834,884,891,951,955-956,961-962,964,977-978,1173-1176,1179-1181,1183-1186,1189,1206-1207,1213,1215-1218,1220,1224-1226,1229-1230,1232,1234-1236,7970,7996,8014,8029,8039,8048,8205,8207,8209,8211,8229,8232-8233,8235,8240,8243,8822,8827-8830,8833-8838,8840-8841,8844,8847-8848,8853-8856,8858,8861,8863,8866,8868-8870,8873,8875-8877,8880-8882,8884,8887-8888,8890-8892,8894-8898,8902,8905-8907,8910-8912,8915,8917-8921,8928,8933,8942,8948,9077,9079-9080,9082-9085,9087,9090,9096-9098,9100-9103,9106,9111,9116-9117,9120,9124,9127-9128,9130-9134,9136,9140,9143-9144,9146-9149,9154,9156-9158,9160,9164-9165,9168-9172,9180-9182,9187-9188,9191,9194,9198-9203,10613-10616,10618-10619,10621-10623,10626-10627,10632-10633,10636,10639-10641,10643,10645,10647-10650,10652-10654,10656,10658,10660-10662,10664,10669-10671,10673-10674,10678-10680,10682,10684,10686,10690-10692,10695-10696,10699-10702,10704-10705,10707-10708,10711,10713,10716,10720,10725-10726,10728,10730-10733,10737-10738,10740,11289,11497,11506"}}
+{"timestamp":1714527250.463011,"name":"online","context":{"idset":"825,894,952,1188,1222,7925-7926,7930,7935,7939-7940,7942,7944-7945,7951-7952,7954,7959,7969,7973,7984,7986,7988,7991,7994,7998,8009,8013,8018,8021,8031,8035,8049,8197,8200,8202-8204,8208,8210,8212,8231,8234,8236-8239,8242,8821,8823-8824,8842,8846,8859,8878,8900,8904,8916,8922-8924,8932,8934-8935,8938-8939,8941,8943,8945-8946,9078,9092,9094,9105,9108-9109,9114,9118,9122-9123,9141,9145,9151,9153,9159,9163,9167,9173-9174,9184,9189,9193,9195-9196,10617,10620,10624,10630,10635,10642,10646,10657,10659,10665,10668,10681,10685,10693,10697-10698,10712,10714,10717,10721,10739,11494-11495,11498,11500-11501,11504-11505,11507-11508"}}
+{"timestamp":1714527250.5958564,"name":"online","context":{"idset":"893,954,1209,1212,1221,7928-7929,7932-7934,7937-7938,7941,7947,7949,7953,7955,7957,7961,7963,7965,7982,7987,7993,8001-8004,8006,8020,8033,8041,8043,8045-8046,8052,8201,8206,8230,8926,8931,8936-8937,8940,8944,9088,9095,9104,9135,9152,9155,9166,9179,9185-9186,9204,10625,10629,10638,10644,10675-10676,10706,10729,10736,11286,11290,11493"}}
+{"timestamp":1714527250.751467,"name":"online","context":{"idset":"833,871,7931,7943,7950,7956,7958,7962,7967-7968,7971-7972,7975,7978,7983,7990,7992,7995,8000,8005,8007,8012,8015-8016,8023-8024,8030,8032,8034,8038,8040,8042,8044,8047,8930,9162,10709,11496,11499,11502"}}
+{"timestamp":1714527250.8677857,"name":"online","context":{"idset":"93-107,109-112,114,118-119,122,124-125,127-128,130,132,157-159,162-164,166-170,173-182,184-196,199,201,203-205,207-208,210-216,219,221-229,231,233-235,237-243,245-250,252,277-278,280,282-283,285,287-290,293-302,305-306,308-309,311-312,316,318-328,330-334,337-340,343,345-347,405-407,409,414-417,419,548,953,7725,7936,7946,7948,7966,7976,7979,7981,7997,8008,8010-8011,8017,8019,8022,8025-8028,8036,8051,8925,8957,8965-8968,8971,9827,10423"}}
+{"timestamp":1714527251.0167978,"name":"online","context":{"idset":"160,284,291-292,307,315,408,412-413,418,420,7312,7964,8958,8960,8975,8978-8979,9046,9050-9051,9069-9071,9073,9076,9373,9779,10406,10742,11842"}}
+{"timestamp":1714527251.4191618,"name":"online","context":{"idset":"7305,7324,7331,7346,7351-7352,7370,7377,7409,7669,7705,7746,7828,8607,8640,8959,8961-8964,8969-8970,8973,8977,8980,9045,9047-9049,9052,9072,9074,9205-9206,9296,9333,9395,9417,9453,9702,9708,9722,9741,9744,9751,9754,9768,9796,9803,9814,9822,9826,9849,9915,9929,10068,10086,10365,10407,10428,10443-10444,10476,10786-10787,10792,10810,10814,10864,11134,11187,11205-11206,11233,11798,11808,11853"}}
+{"timestamp":1714527251.527775,"name":"online","context":{"idset":"7286,7299,7308,7340-7341,7364,7383,7391,7682,7687-7688,7697,7741-7742,8571,8605,9222,9231,9257,9270,9274,9314,9325,9332,9337,9356,9392,9426,9432,9749,9780,9790,9812,9862,9882,9960,10106,10131,10149,10179,10192,10462,10482,10759,10762,10803,10844,11126,11165,11180,11246,11250,11760,11773"}}
+{"timestamp":1714527251.8918662,"name":"online","context":{"idset":"53,56,573,947,7165,7288,7291-7292,7295-7297,7300,7303,7307,7313-7314,7317,7319-7320,7327,7329,7333,7342,7345,7347-7349,7354,7358,7363,7379,7400,7408,7410-7412,7671,7689-7691,7693,7696,7702-7704,7709,7714,7718,7723,7731,7737,7739-7740,7747-7748,7750,7753-7754,7757,7760,7763,7768-7769,7771-7773,7776,7782,7785,8214,8567,8572,8579-8580,8583,8585-8586,8590,8597,8608,8621,8625,8627,8637,8658,8663,8670-8672,8675,8685,8688,9016,9207,9211,9216,9220,9223,9230,9235,9238,9245,9259,9261-9262,9266,9276,9283,9285,9287-9288,9293,9301,9313,9316,9323,9326,9336,9341,9343,9348-9351,9357-9358,9360,9365,9370,9374-9375,9378,9380,9382,9389-9390,9393,9405,9408-9412,9416,9420,9439,9444,9447,9449-9450,9452,9455,9458,9481,9522,9535,9564,9566,9582-9583,9588,9604,9610,9615,9622,9665,9699,9712,9715,9723,9728,9733,9740,9747,9760,9763,9766,9771-9773,9775-9776,9785,9787,9791,9795,9799,9805-9806,9810,9813,9816,9818,9828,9832,9836,9843,9848,9852,9858-9859,9865,9867,9875-9876,9878,9881,9885,9891,9895,9897-9898,9902-9903,9909,9914,9930,9935,9943,9948-9949,9955,9958-9959,9962,10004,10009,10019,10022-10024,10034,10053,10058,10062,10071-10072,10074,10076,10081,10088,10108,10112-10114,10121-10123,10130,10136,10138,10146,10152-10153,10162-10164,10171,10175,10193,10203,10205,10210,10219,10227,10361,10368,10375-10377,10382,10384,10390,10394-10395,10404,10415-10416,10418,10424-10425,10440,10448-10449,10451,10455-10456,10459,10463,10469,10471,10475,10479,10483,10743,10747,10753,10766,10773,10784,10795,10799-10800,10808-10809,10813,10815,10821,10825,10831-10832,10837,10840,10845,10859,10863,10865,10867,11067,11083,11128,11135,11149-11150,11155,11166,11173,11176,11179,11182,11186,11201,11203-11204,11208-11209,11220-11221,11227,11235,11251,11637,11645-11646,11680-11681,11687,11698,11702,11705,11724,11755,11761,11764,11777,11784,11788,11796,11817,11822,11849,11854,11863,11865,11874"}}
+{"timestamp":1714527252.0240724,"name":"online","context":{"idset":"3-19,21-24,26-31,33-35,38-39,41-45,48-52,54,57-59,940,7332,7339,7368,7371,7387,7403,7672,7712,7715,7717,7722,7728,7730,7735,7759,8071,8569,8596,8634-8635,8676,8700,8794,8956,9232,9501,9648,9757,9797,10101,10127,10960,11214,11280,11285,11873,11883,11887"}}
+{"timestamp":1714527252.1366777,"name":"online","context":{"idset":"46,55,60,6582,6628,6635,7169,7289,7293,7302,7309,7315,7326,7328,7334,7337,7361,7367,7369,7372-7374,7381-7382,7384,7388,7394,7401,7407,7676,7681,7685,7694,7700,7706,7719,7721,7729,7732,7736,7751,7767,7784,7787,7790,7814,8072,8217,8281,8592,8639,8643,8649,8651,8664,8669,8677,8703,8766,8788,9218,9290,9315,9387,9391,9436,9466,9508,9540,9542,9545,9612,9651,9659,9721,9731-9732,9734,9742,9745-9746,9759,9761,9769-9770,9774,9783,9789,9838,9868-9871,9887,9932,9946,10040,10079,10104-10105,10119-10120,10126,10150,10184,10194,10197-10198,10200,10207,10216,10221,10224,10357-10358,10366-10367,10369,10372-10373,10378-10379,10387-10389,10391,10398,10400-10401,10405,10409-10410,10413,10417,10419-10420,10427,10431,10433-10436,10442,10447,10452,10454,10457,10468,10474,10487,10583,10746,10748,10756,10760,10765,10768,10771,10779,10783,10793,10796,10798,10804,10807,10818-10819,10822,10824,10828-10829,10836,10841,10846-10848,10850,10854,10862,10868,10882,10951,10987,11028,11036,11132,11152,11164,11172,11192,11642,11694,11704,11736,11763,11827"}}
+{"timestamp":1714527252.2805796,"name":"online","context":{"idset":"576,579,6526,6528,6534,6546,6552,6585,6604,7245,7285,7294,7301,7325,7330,7356,7375,7378,7390,7397-7398,7402,7683,7698-7699,7701,7708,7710-7711,7713,7716,7720,7724,7726,7733,7738,7744,7749,7762,7764,7770,7774-7775,7778-7779,7783,7788,7791,7794,7822,7825,7827,7845,7851,7854-7856,7865,7876,7879,7881,7887,7889,7893-7894,7897,7900,7902,7904,7908,7910,7914,7921,8055,8057,8063,8075,8077-8078,8108,8159,8184-8185,8192,8195-8196,8225-8227,8251,8277-8278,8290,8296,8301,8306,8308,8335,8371,8386,8398,8402,8412,8573-8575,8577,8581,8587,8593-8595,8599-8600,8616-8617,8620,8622,8626,8628,8632,8636,8648,8653-8656,8660,8668,8678,8683-8684,8686,8689-8691,8694,8698,8706-8707,8711-8712,8714,8717-8718,8720,8725-8726,8748-8749,8772,8778,8789-8790,8809,8820,8949,8951-8952,8954-8955,8989,9015,9056,9209-9210,9212-9213,9217,9224,9226-9227,9229,9233-9234,9240-9241,9247-9251,9253-9256,9260,9263-9265,9267-9269,9273,9275,9277-9278,9280-9282,9284,9289,9295,9297-9298,9300,9302,9304-9308,9310-9311,9317-9318,9321-9322,9324,9329-9330,9334,9338-9340,9346-9347,9354,9359,9362-9364,9367-9369,9372,9376-9377,9379,9383-9386,9388,9396-9397,9399-9400,9404,9407,9413,9415,9419,9421-9422,9424-9425,9427-9428,9430,9434-9435,9437-9438,9440,9442,9445,9454,9456-9457,9459-9460,9463-9465,9468,9470,9478,9482,9487,9489-9490,9505-9507,9509,9511-9512,9516,9521,9525,9528-9529,9538,9541,9547-9548,9553-9554,9562,9570-9571,9580,9586,9593,9606,9617,9621,9628-9629,9631,9634,9647,9652-9653,9655,9657,9663,9666,9673,9688-9689,9691-9693,9697-9698,9704,9707,9709-9711,9717-9718,9724-9727,9729-9730,9735,9737,9739,9748,9750,9752,9755-9756,9758,9762,9764-9765,9767,9778,9781,9788,9792,9794,9801-9802,9807-9808,9811,9815,9817,9819-9820,9824-9825,9829-9831,9834,9837,9839-9842,9844,9847,9851,9853-9855,9860-9861,9864,9866,9879,9884,9886,9888-9889,9892,9899,9907-9908,9910-9913,9916-9923,9925-9928,9931,9933-9934,9937,9940-9942,9945,9947,9950-9953,9957,9961,9963,9966-9968,9973,9977,9981,9984,9986-9988,9997-9999,10014-10015,10020-10021,10025-10029,10032-10033,10042,10047,10049,10051,10054-10056,10060,10063,10065-10067,10073,10075,10078,10085,10097-10099,10103,10107,10109-10111,10115-10118,10124,10128-10129,10133,10135,10141,10143-10145,10147,10154,10156,10158,10161,10166,10169,10172,10174,10177-10178,10180-10183,10190-10191,10196,10199,10201-10202,10204,10206,10208-10209,10217,10359-10360,10362,10370,10374,10381,10392,10396,10399,10408,10411-10412,10421-10422,10426,10429,10437,10450,10453,10458,10461,10465,10467,10470,10472,10477-10478,10480-10481,10484,10492,10512,10536,10563,10579,10603,10744-10745,10749-10750,10752,10755,10767,10774-10776,10778,10780-10781,10785,10788,10790-10791,10794,10797,10806,10811-10812,10817,10830,10833-10835,10839,10842,10851-10852,10855-10856,10858,10866,10869,10872-10873,10875-10878,10883-10885,10889-10890,10895,10897,10911-10914,10917,10920,10922-10923,10927-10930,10933-10935,10939-10940,10942,10945-10946,10952,10956-10959,10961,10964,10968,10974,10980-10981,10984-10985,10988-10990,10992,10994,10996,11000,11017,11023,11027,11030-11031,11038-11040,11045,11051-11053,11055,11062,11064,11071-11072,11075-11076,11079,11087-11088,11096-11097,11099,11104,11106,11109,11111-11112,11115,11118,11121,11124,11127,11129,11131,11133,11136-11138,11145,11147,11153-11154,11156,11159,11163,11169-11171,11174,11177,11185,11193,11196-11197,11199,11207,11210,11215-11216,11219,11225,11228-11229,11232,11234,11236,11247-11248,11253-11254,11256-11257,11260,11264,11268,11274,11277-11279,11282-11283,11288,11293,11295-11305,11307-11308,11314-11318,11324,11332,11340,11348,11357,11363,11368,11375-11377,11380,11639-11640,11644,11665,11669,11671,11677,11688-11689,11693,11696-11697,11700,11706,11709,11712,11714,11722,11726,11732-11733,11735,11739,11746-11747,11754,11757-11759,11765-11766,11769-11770,11772,11775-11776,11780-11781,11785,11787,11789-11790,11792,11794,11797,11799,11801,11803-11807,11811-11812,11814,11818-11820,11823-11826,11830-11832,11834,11836,11838-11839,11843-11844,11848,11852,11856-11858,11861,11864,11866,11869-11870,11872,11876-11878,11881-11882,11885-11886,11889-11891"}}
+{"timestamp":1714527252.3985457,"name":"online","context":{"idset":"914,6520,6614,6623,6640,7047,7171,7290,7306,7316,7318,7322,7335,7338,7344,7350,7355,7380,7385-7386,7389,7392-7393,7670,7674,7677,7679,7684,7686,7734,7745,7755,7765,7789,7796,7803,7806,7808,7829,7849,7871,7905,7916,8082,8187,8216,8255,8257,8271,8598,8606,8610,8619,8645,8650,8673,8696,8709-8710,8747,8754,8953,8997,8999-9000,9057,9215,9219,9228,9237,9243-9244,9272,9286,9291,9303,9331,9344-9345,9353,9355,9361,9371,9398,9402,9414,9418,9423,9429,9448,9492,9498,9527,9543,9556,9560,9569,9572,9626,9632,9662,9695,9701,9705,9716,9738,9743,9786,9804,9809,9821,9846,9856,9863,9872,9893,9901,9938,9969,9983,9993,10010,10030,10048,10087,10137,10151,10160,10165,10222,10364,10371,10402-10403,10430,10432,10439,10446,10464,10473,10554,10594,10598,10602,10606,10761,10764,10770,10772,10802,10816,10838,10853,10857,10860-10861,10870-10871,10891,10921,10932,10943-10944,10947,10969,10973,10983,11001,11004,11008,11029,11032,11041,11047-11049,11065,11069,11080,11082,11091,11095,11101,11103,11116,11125,11130,11139-11144,11146,11148,11151,11158,11167-11168,11181,11194-11195,11200,11202,11211,11213,11217-11218,11222-11224,11226,11230,11237,11240-11244,11249,11252,11255,11266-11267,11269-11270,11273,11287,11311-11313,11330,11333,11351,11364-11366,11369,11606,11636,11652,11658-11659,11661,11670,11675,11678,11683,11701,11718,11728,11741,11753,11762,11778,11782-11783,11793,11800,11809-11810,11815-11816,11835,11845-11846,11851,11867-11868,11875,11884,11888"}}
+{"timestamp":1714527252.5192289,"name":"online","context":{"idset":"575,845-846,913,937,6521,6523,6556,6562,6564,6579,6584,6596,6600,6607,6616,6618,6620,6627,6629-6630,6636,7157-7159,7161,7163,7166,7172-7173,7298,7321,7323,7336,7360,7366,7396,7404-7405,7680,7758,7761,7766,7786,7792,7795,7798,7800,7802,7804-7805,7807,7809,7811,7813,7815,7817,7820-7821,7823-7824,7831-7834,7836-7842,7844,7846-7848,7850,7852,7857-7858,7861,7863-7864,7866-7869,7872-7875,7877-7878,7882,7884,7886,7888,7890-7892,7895-7896,7898,7906-7907,7909,7911-7912,7915,7917,7919,7922-7924,8053-8054,8056,8058-8059,8061-8062,8065-8067,8069-8070,8079,8081,8083,8186,8188-8190,8194,8213,8215,8218-8221,8223,8228,8246-8250,8252,8254,8256,8258-8270,8272-8276,8280,8282-8289,8291-8292,8294-8295,8297-8299,8303-8305,8324,8336,8342,8345,8348,8351,8355,8357,8390,8396-8397,8403,8408,8430,8434,8568,8589,8591,8602,8611,8613,8646-8647,8657,8659,8666,8679,8681,8693,8695,8697,8699,8705,8713,8716,8723-8724,8729,8735,8741-8742,8768,8801,8812,8817,8819,8983,8986,9017,9019,9025,9054-9055,9208,9214,9225,9242,9246,9319-9320,9352,9381,9451,9461,9469,9494,9503,9510,9523-9524,9531,9537,9552,9555,9557-9558,9565,9567-9568,9577,9581,9585,9587,9589,9591-9592,9594-9595,9597,9600,9603,9605,9607,9611,9616,9618-9620,9623-9624,9627,9633,9635-9637,9639,9645,9654,9656,9667,9669-9671,9674,9676,9678,9681-9682,9684-9685,9687,9690,9694,9696,9700,9713,9753,9782,9833,9835,9845,9850,9873,9905,9924,9936,9944,9954,9956,9964-9965,9970-9972,9974,9979-9980,9990,9994,9996,10000,10003,10006,10016-10018,10031,10036,10039,10057,10059,10061,10070,10077,10080,10082,10084,10089,10125,10134,10140,10148,10155,10159,10167,10170,10176,10185,10187,10189,10211-10212,10220,10228,10363,10438,10441,10485-10486,10488-10489,10493-10499,10501-10504,10508,10510,10514,10518,10522,10524-10526,10530-10535,10537,10540,10542-10543,10545-10549,10555-10556,10560,10564,10567,10571,10573-10574,10576-10578,10580,10582,10584,10588,10591,10596-10597,10600-10601,10605,10607-10608,10612,10741,10757,10763,10823,10826,10843,10880,10886,10888,10892-10894,10896,10898-10905,10907-10910,10915-10916,10918-10919,10926,10936-10937,10948-10949,10953,10955,10963,10966-10967,10971-10972,10975-10978,10986,10991,10998-10999,11002,11006,11009,11011-11014,11018-11022,11025-11026,11033,11035,11037,11043,11046,11050,11054,11056,11058,11061,11063,11074,11078,11081,11084,11086,11089,11094,11098,11102,11108,11114,11120,11122-11123,11157,11160-11162,11178,11190,11198,11238,11258-11259,11261,11271-11272,11275,11281,11306,11309,11319,11321-11322,11325-11329,11331,11334-11339,11341,11343-11347,11349-11350,11352,11354-11356,11358-11362,11367,11370-11371,11378-11379,11512,11517,11521-11524,11534-11535,11539,11544,11552-11554,11563,11577,11587-11589,11593,11612,11614,11622-11623,11625,11633,11641,11647,11653-11654,11656-11657,11660,11662,11664,11666-11667,11682,11684,11686,11691-11692,11695,11699,11708,11711,11716-11717,11720,11727,11730-11731,11738,11743-11745,11748,11752,11791,11795,11821,11828,11833,11840-11841,11855,11859,11862,11880"}}
+{"timestamp":1714527252.5332851,"name":"online","context":{"idset":"6517,6581,6597,6610,7162,7168,7819,7830,7843,7862,7901,7903,7913,8060,8064,8068,8076,8293,8302,8310,8313,8316,8318,8344,8368,8372-8374,8376,8385,8388,8414,8426,8431,8566,8576,8578,8582,8584,8603,8609,8612,8614-8615,8618,8629-8630,8633,8638,8641,8652,8662,8680,8687,8701,8704,8722,8732,8757,8774-8775,8787,8808,8813,8818,8981-8982,8998,9001,9007,9009-9010,9014,9018,9020,9022-9024,9026-9028,9053,9058-9060,9252,9292,9309,9394,9406,9441,9472,9474,9476-9477,9488,9497,9515,9520,9534,10500,10529,10557,10587,11016,11323,11538,11569,11592,11610-11611,11617,11632,11650,11663,11690,11710,11737"}}
+{"timestamp":1714527252.6495047,"name":"online","context":{"idset":"847,6524,6533,6537,6555,6571,6575,6577,6583,6588,6603,6619,6625,6632,6641,7160,7164,7170,8073,8253,8323,8326,8329-8330,8333,8341,8343,8347,8350,8352,8354,8359-8361,8363,8367,8375,8377-8380,8382,8384,8389,8392,8404,8406-8407,8415,8417-8418,8425,8432,8436,8642,8665,8719,8727-8728,8730-8731,8750-8751,8753,8755-8756,8758,8761,8763,8767,8781-8782,8784-8785,8793,8795,8799,8802-8803,8810,8815,9467,9513,9549,9575,9578,9596,9602,9608,9613,9625,9649,9658,9661,9672,9677,9679-9680,9683,9686,9904,9989,9995,10037-10038,10043,10157,10511,10519,10523,10541,10550-10551,10566,10572,10575,10581,10599,10609,10801,10924,10950,10962,10970,10995,11007,11010,11024,11057,11263,11372,11510,11518,11520,11526,11560,11567,11578-11579,11596,11598,11600,11608,11616,11619,11628,11631,11635,11648,11673,11740,11786"}}
+{"timestamp":1714527252.655215,"name":"online","context":{"idset":"9480,9598,9601,9609,9640,9643,9660,9664,9668,10008,10011-10012,10035,10046,10092,10100"}}
+{"timestamp":1714527252.6583107,"name":"online","context":{"idset":"9491,10517,10520"}}
+{"timestamp":1714527252.6609526,"name":"online","context":{"idset":"10516"}}
+{"timestamp":1714527252.6635687,"name":"online","context":{"idset":"9574"}}
+{"timestamp":1714527252.6661844,"name":"online","context":{"idset":"9484-9485,9493,9496,9536,9559,9576,10091,10225,10568,10589"}}
+{"timestamp":1714527252.6691966,"name":"online","context":{"idset":"10005,10604"}}
+{"timestamp":1714527252.673074,"name":"online","context":{"idset":"9563,9573"}}
+{"timestamp":1714527252.6756618,"name":"online","context":{"idset":"9473,10561-10562,10590,10595"}}
+{"timestamp":1714527252.6822369,"name":"online","context":{"idset":"9985,10505,10507,10515,10527,10610-10611"}}
+{"timestamp":1714527252.684797,"name":"online","context":{"idset":"10570,10586"}}
+{"timestamp":1714527252.6873808,"name":"online","context":{"idset":"10506,10509,10906,11003"}}
+{"timestamp":1714527252.6899881,"name":"online","context":{"idset":"10513"}}
+{"timestamp":1714527252.6932406,"name":"online","context":{"idset":"11575,11586"}}
+{"timestamp":1714527252.6969876,"name":"online","context":{"idset":"9976,11005,11565,11580,11624"}}
+{"timestamp":1714527252.8122971,"name":"online","context":{"idset":"10521,10558,10997,11519,11527,11530,11574,11582,11602,11629"}}
+{"timestamp":1714527252.8202162,"name":"online","context":{"idset":"824,6519,6527,6543,6547-6549,6551,6558-6559,6572,6590,6594,6599,6611,6613,6617,6639,6642-6644,7186,7189-7190,7196,7215,7219,7244,7246,7258,7818,7835,7899,8080,8096,8120,8127,8130,8133,8137,8165,8311-8312,8315,8317,8319-8321,8325,8328,8331-8332,8334,8337-8339,8346,8349,8358,8362,8364-8365,8369,8381,8383,8391,8393-8395,8399-8400,8405,8409-8411,8413,8419,8423,8427-8429,8721,8734,8736,8738-8740,8743-8744,8746,8759-8760,8762,8764-8765,8769-8771,8776,8779,8786,8792,8796-8798,8804-8805,8984-8985,8990-8991,8993,8995-8996,9006,9011-9012,9239,9475,9479,9486,9500,9504,9518,9533,9539,9544,9975,10094-10095,10528,11090,11342,11509,11511,11513-11515,11525,11528,11531-11532,11537,11540-11543,11545,11547-11549,11556,11558,11562,11564,11566,11570,11573,11576,11584-11585,11590-11591,11594-11595,11597,11603-11605,11607,11613,11615,11618,11626,11630,11634,11643,11674,11676,11679,11707,11715,11725,11734,11749-11750"}}
+{"timestamp":1714527252.8250968,"name":"online","context":{"idset":"8182"}}
+{"timestamp":1714527252.8394842,"name":"online","context":{"idset":"8994"}}
+{"timestamp":1714527252.9548507,"name":"online","context":{"idset":"6530,6538,6541,6544-6545,6553-6554,6561,6566,6568,6578,6587,6589,6601,6615,6626,6631,6637,8116,8132,8170,8322,8356,8420,8422,8745,8777,8800,8811,9514,9532,9551,9978,11555,11568,11571,11581,11583,11609,11627,11655,11685,11751"}}
+{"timestamp":1714527252.959718,"name":"online","context":{"idset":"6550"}}
+{"timestamp":1714527252.9627988,"name":"online","context":{"idset":"7216"}}
+{"timestamp":1714527252.9659407,"name":"online","context":{"idset":"7182,11742"}}
+{"timestamp":1714527253.0697126,"name":"online","context":{"idset":"8183,8370"}}
+{"timestamp":1714527253.0955009,"name":"online","context":{"idset":"8169"}}
+{"timestamp":1714527253.2112453,"name":"online","context":{"idset":"6592"}}
+{"timestamp":1714527253.2218828,"name":"online","context":{"idset":"6518,6536,6540,6560,6569,6576,6595,6624,7179,7184,7187,7239,7251,8088,8097,8100-8101,8109,8111-8112,8115,8117,8121,8123,8125,8131,8149-8150,8157,8161,8175-8176,8179,11561"}}
+{"timestamp":1714527253.2384012,"name":"online","context":{"idset":"6622"}}
+{"timestamp":1714527253.2441068,"name":"online","context":{"idset":"6609,7191"}}
+{"timestamp":1714527253.2480154,"name":"online","context":{"idset":"7177"}}
+{"timestamp":1714527253.259856,"name":"online","context":{"idset":"7229,7248,7263,7266-7267"}}
+{"timestamp":1714527253.2637591,"name":"online","context":{"idset":"8103"}}
+{"timestamp":1714527253.2687464,"name":"online","context":{"idset":"8141"}}
+{"timestamp":1714527253.272748,"name":"online","context":{"idset":"8146"}}
+{"timestamp":1714527253.2864439,"name":"online","context":{"idset":"8156"}}
+{"timestamp":1714527253.3999331,"name":"online","context":{"idset":"7242,8166"}}
+{"timestamp":1714527253.4035478,"name":"online","context":{"idset":"8158"}}
+{"timestamp":1714527253.4152493,"name":"online","context":{"idset":"6525,6529,6532,6570,6598,7174,7185,7192,7199,7212,7217,7224-7225,7227,7235,7262,8104,8122,8134,8148,8151,8153,8173"}}
+{"timestamp":1714527253.5289609,"name":"online","context":{"idset":"6563,6565,6633,7181,7223,7230-7231,7241,7249,7255-7256,7260,7265,8118,8135,8140,8147,8160,8174"}}
+{"timestamp":1714527253.5360432,"name":"online","context":{"idset":"6621,7195,7200,7206-7207,7209,7214,7247,7259,8089,8136,8172,8177"}}
+{"timestamp":1714527253.5431843,"name":"online","context":{"idset":"8124"}}
+{"timestamp":1714527253.5503685,"name":"online","context":{"idset":"7233,8126,8180"}}
+{"timestamp":1714527253.5575688,"name":"online","context":{"idset":"6542,7201,8093"}}
+{"timestamp":1714527253.5646174,"name":"online","context":{"idset":"8102"}}
+{"timestamp":1714527253.6795697,"name":"online","context":{"idset":"7180,7183,7193,7202,7221,8084,8090,8110,8154,8181"}}
+{"timestamp":1714527253.7714124,"name":"online","context":{"idset":"6522,6580,6605,6612,7188,7194,7197,7203,7205,7208,7210,7213,7218,7220,7222,7228,7234,7236,7238,7250,7253-7254,7261,7268,8085,8087,8091,8128,8139,8143,8145,8163,8167,8178"}}
+{"timestamp":1714527253.8600757,"name":"online","context":{"idset":"6638,7175,7178,7204,7226,7232,8086,8098,8129,8152"}}
+{"timestamp":1714527253.9564178,"name":"online","context":{"idset":"6574"}}
+{"timestamp":1714527253.9617591,"name":"online","context":{"idset":"6586,6593,7211,7240"}}
+{"timestamp":1714527253.9665272,"name":"online","context":{"idset":"7176,7252,7257"}}
+{"timestamp":1714527253.9706779,"name":"online","context":{"idset":"8155"}}
+{"timestamp":1714527253.9745047,"name":"online","context":{"idset":"8168"}}
+{"timestamp":1714527253.9824302,"name":"online","context":{"idset":"7198"}}
+{"timestamp":1714527254.1271603,"name":"online","context":{"idset":"8099,8114"}}
+{"timestamp":1714527254.5640533,"name":"online","context":{"idset":"8860,11284"}}
+{"timestamp":1714527254.5963006,"name":"online","context":{"idset":"9857,10628"}}
+{"timestamp":1714527254.8237936,"name":"online","context":{"idset":"1190,7287,7707,7756,8144,8241,8852,8874,8901,8908,9271,9433,9446,9499,9793,9800,10096,10393,10677,10687,10789,10805"}}
+{"timestamp":1714527254.9256079,"name":"online","context":{"idset":"1177,7989,8050,8927,9584,10044,10715,11184,11672"}}
+{"timestamp":1714527255.0293031,"name":"online","context":{"idset":"6608,7362,7365,7695,8565,9089,9112,9178,9236,9312,11191,11719"}}
+{"timestamp":1714527255.1329567,"name":"online","context":{"idset":"129,310,341,1208,1227,6539,7353,7678,7727,8314,8353,8570,8588,8661,8692,8871,9279,9495,9641,9719-9720,9784,9798,9890,9894,9896,9992,10002,10013,10188,10213-10214,10386,10397,10751,10782,10849,11042,11070,11183,11231,11503,11546,11551,11850"}}
+{"timestamp":1714527255.2405646,"name":"online","context":{"idset":"6557,7359,8309,8624,8773,8849,9150,10132,10385,11113,11723"}}
+{"timestamp":1714527255.3412042,"name":"online","context":{"idset":"6634,7395,7673,9900,10093,10634,11060,11294,11529,11572,11668,11813,11837"}}
+{"timestamp":1714527255.4464369,"name":"online","context":{"idset":"113,115,220,304,317,344,938-939,6531,6535,6606,7304,7399,7692,7777,7781,7793,7918,7977,8037,8119,8164,8191,8224,8300,8340,8387,8421,8631,8715,8814,8974,8976,9021,9115,9192,9221,9294,9327-9328,9335,9366,9431,9462,9471,9483,9517,9561,9599,9630,9646,9675,9877,9880,9883,9982,10041,10045,10052,10064,10173,10215,10218,10223,10380,10460,10490,10585,10723,10820,10827,10887,10993,11044,11066,11085,11117,11265,11353,11373,11516,11649,11713,11721,11829"}}
+{"timestamp":1714527255.5493908,"name":"online","context":{"idset":"108,123,209,8171,8245,8401,8623,8972"}}
+{"timestamp":1714527255.6523395,"name":"online","context":{"idset":"7237,7264,7357,7859,7883,7980,8074,8095,8193,8244,8279,8424,8783,8816,9086,9113,9401,9519,9526,9550,9590,9706,9777,9874,9906,10050,10139,10142,10414,10544,10552,10565,10592,10663,10672,10874,11073,11239,11262,11620,11729"}}
+{"timestamp":1714527255.6571777,"name":"online","context":{"idset":"9299,10965"}}
+{"timestamp":1714527255.6800141,"name":"online","context":{"idset":"197,230,232,7799,8113,10637,10694"}}
+{"timestamp":1714527255.8294861,"name":"online","context":{"idset":"848,1211,7826,7927,8138,8366,8791,8826,8992,9138,9502,9579,10083,10226,10539,10651,10938,11550,11601,11892"}}
+{"timestamp":1714527255.8345768,"name":"online","context":{"idset":"7985,10090"}}
+{"timestamp":1714527255.8571336,"name":"online","context":{"idset":"7885,9403,10007,10735,10879,11077,11703,11847"}}
+{"timestamp":1714527256.0103521,"name":"online","context":{"idset":"872,7243,7310,7376,7780,7797,8094,9013,9823,10195,10383,10538,10758,10769,10881,10925,10931,10954,11212,11599"}}
+{"timestamp":1714527256.2070088,"name":"online","context":{"idset":"116,171,200,251,948,7167,7311,7743,7752,7860,7870,7974,8601,8604,8644,8780,8807,9258,9342,9443,9638,9642,9650,9991,10102,10168,10466,10569,10754,10777,11059,11068,11245,11320,11638,11756,11871"}}
+{"timestamp":1714527256.4075871,"name":"online","context":{"idset":"161,183,218,244,329,6591,7343,7406,7675,7812,7816,7853,7880,7920,8107,8667,8702,8752,9129,9530,9546,9939,10001,10069,10445,10491,10553,10593,10982,11175,11188,11374,11557,11621,11651,11768,11771,11879"}}
+{"timestamp":1714527256.5117509,"name":"online","context":{"idset":"303,577,823,6602,7801,8092,8162,8307,8327,8674,8708,8806,10941,10979,11189,11536"}}
+{"timestamp":1714527256.613977,"name":"online","context":{"idset":"6567,6573,8222,8433,8435,9614"}}
+{"timestamp":1714527256.7159958,"name":"online","context":{"idset":"36,9008,9714,10559,11119,11767,11779"}}
+{"timestamp":1714527256.9178724,"name":"online","context":{"idset":"2,25,32,8950,9644,11559"}}
+{"timestamp":1714527257.1176784,"name":"online","context":{"idset":"9002,9005"}}
+{"timestamp":1714527257.4589143,"name":"online","context":{"idset":"47"}}
+{"timestamp":1714527257.5807474,"name":"online","context":{"idset":"1,20,37,40"}}
+{"timestamp":1714527257.6826174,"name":"online","context":{"idset":"10655,10727"}}
+{"timestamp":1714527258.1068306,"name":"online","context":{"idset":"7046,7999,8142,8682,10186"}}
+{"timestamp":1714527258.2235048,"name":"online","context":{"idset":"11034"}}
+{"timestamp":1714527258.2479141,"name":"online","context":{"idset":"126,131,7810,8737,9703,11860"}}
+{"timestamp":1714527258.4023032,"name":"online","context":{"idset":"9736,11276,11310,11774,11802"}}
+{"timestamp":1714527258.5052485,"name":"online","context":{"idset":"8416"}}
+{"timestamp":1714527258.6271102,"name":"online","context":{"idset":"8733"}}
+{"timestamp":1714527260.5712569,"name":"online","context":{"idset":"11092,11110"}}
+{"timestamp":1714527260.7202871,"name":"online","context":{"idset":"11100"}}
+{"timestamp":1714527260.8510692,"name":"online","context":{"idset":"11015"}}
+{"timestamp":1714527261.1492043,"name":"online","context":{"idset":"11105"}}
+{"timestamp":1714527261.4570239,"name":"online","context":{"idset":"11093,11107"}}
+{"timestamp":1714527428.1276264,"name":"online","context":{"idset":"574"}}
+{"timestamp":1714528011.1782928,"name":"offline","context":{"idset":"11287"}}
+{"timestamp":1714528012.0033958,"name":"offline","context":{"idset":"11288"}}
+{"timestamp":1714528586.5262029,"name":"drain","context":{"idset":"8723","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1714528607.2169995,"name":"drain","context":{"idset":"8810","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714528609.3510127,"name":"drain","context":{"idset":"8716","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714528609.7724736,"name":"drain","context":{"idset":"8752","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714528610.2650449,"name":"drain","context":{"idset":"8722","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714528736.0570335,"name":"online","context":{"idset":"10263"}}
+{"timestamp":1714528736.7206526,"name":"online","context":{"idset":"10262"}}
+{"timestamp":1714528739.863019,"name":"online","context":{"idset":"10232"}}
+{"timestamp":1714528740.4758995,"name":"online","context":{"idset":"10277"}}
+{"timestamp":1714528740.6809323,"name":"online","context":{"idset":"10236"}}
+{"timestamp":1714528741.2219715,"name":"online","context":{"idset":"10258"}}
+{"timestamp":1714528741.5587239,"name":"online","context":{"idset":"10313"}}
+{"timestamp":1714528742.2774303,"name":"online","context":{"idset":"10231,10239,10246"}}
+{"timestamp":1714528742.6691082,"name":"online","context":{"idset":"10249"}}
+{"timestamp":1714528742.9136691,"name":"online","context":{"idset":"10245"}}
+{"timestamp":1714528743.4441383,"name":"online","context":{"idset":"10257"}}
+{"timestamp":1714528745.5372884,"name":"online","context":{"idset":"10235"}}
+{"timestamp":1714528745.768554,"name":"online","context":{"idset":"10291"}}
+{"timestamp":1714528746.5490773,"name":"online","context":{"idset":"10250"}}
+{"timestamp":1714528748.8528662,"name":"online","context":{"idset":"10268"}}
+{"timestamp":1714528748.9967341,"name":"online","context":{"idset":"10271"}}
+{"timestamp":1714528749.6429422,"name":"online","context":{"idset":"10269"}}
+{"timestamp":1714528750.3701985,"name":"online","context":{"idset":"10331"}}
+{"timestamp":1714528750.668314,"name":"online","context":{"idset":"10325"}}
+{"timestamp":1714528750.9979501,"name":"online","context":{"idset":"10252"}}
+{"timestamp":1714528751.4261487,"name":"online","context":{"idset":"10229-10230"}}
+{"timestamp":1714528751.5693324,"name":"online","context":{"idset":"10259,10305"}}
+{"timestamp":1714528751.9146364,"name":"online","context":{"idset":"10233"}}
+{"timestamp":1714528752.0430498,"name":"online","context":{"idset":"10234"}}
+{"timestamp":1714528752.1981294,"name":"online","context":{"idset":"10237"}}
+{"timestamp":1714528752.4436657,"name":"online","context":{"idset":"10300"}}
+{"timestamp":1714528752.5664377,"name":"online","context":{"idset":"10337"}}
+{"timestamp":1714528752.6840055,"name":"online","context":{"idset":"10264-10265"}}
+{"timestamp":1714528752.7046173,"name":"online","context":{"idset":"10243"}}
+{"timestamp":1714528752.8400412,"name":"online","context":{"idset":"10244,10254,10333,10335"}}
+{"timestamp":1714528752.8440015,"name":"online","context":{"idset":"10247,10324"}}
+{"timestamp":1714528752.8479407,"name":"online","context":{"idset":"10290"}}
+{"timestamp":1714528752.8714547,"name":"online","context":{"idset":"10253,10303,10321,10329"}}
+{"timestamp":1714528753.0475376,"name":"online","context":{"idset":"10240"}}
+{"timestamp":1714528753.1637192,"name":"online","context":{"idset":"10273,10281,10327,10351"}}
+{"timestamp":1714528753.4207838,"name":"online","context":{"idset":"10251,10261,10272,10298"}}
+{"timestamp":1714528753.5337083,"name":"online","context":{"idset":"10356"}}
+{"timestamp":1714528753.654387,"name":"online","context":{"idset":"10238,10241,10283-10284,10310-10311,10320,10353"}}
+{"timestamp":1714528753.7632339,"name":"online","context":{"idset":"10256,10275,10295,10309,10318,10341"}}
+{"timestamp":1714528753.8919277,"name":"online","context":{"idset":"10255"}}
+{"timestamp":1714528754.010704,"name":"online","context":{"idset":"10282,10292,10297"}}
+{"timestamp":1714528754.115766,"name":"online","context":{"idset":"10242,10260,10267,10294,10352"}}
+{"timestamp":1714528754.2339997,"name":"online","context":{"idset":"10248,10266,10270,10274,10276,10280,10289,10299,10308,10315,10336,10348,10350,10355"}}
+{"timestamp":1714528754.4458611,"name":"online","context":{"idset":"10278-10279,10285-10286,10301,10306,10314,10332,10342"}}
+{"timestamp":1714528754.6566339,"name":"online","context":{"idset":"10287-10288,10293,10302,10312,10328,10334,10343,10346,10349,10354"}}
+{"timestamp":1714528754.7615817,"name":"online","context":{"idset":"10316,10345"}}
+{"timestamp":1714528754.8640499,"name":"online","context":{"idset":"10322-10323,10344"}}
+{"timestamp":1714528754.9693127,"name":"online","context":{"idset":"10317,10330,10347"}}
+{"timestamp":1714528755.0709341,"name":"online","context":{"idset":"10338"}}
+{"timestamp":1714528755.1765673,"name":"online","context":{"idset":"10296,10304,10307,10319"}}
+{"timestamp":1714528755.293057,"name":"online","context":{"idset":"10326"}}
+{"timestamp":1714528755.4114022,"name":"online","context":{"idset":"10340"}}
+{"timestamp":1714528926.7376552,"name":"online","context":{"idset":"10339"}}
+{"timestamp":1714528940.1267564,"name":"offline","context":{"idset":"11285"}}
+{"timestamp":1714528940.1343322,"name":"offline","context":{"idset":"11286"}}
+{"timestamp":1714528979.6223242,"name":"undrain","context":{"idset":"8716,8722-8723,8752,8810"}}
+{"timestamp":1714533246.449415,"name":"offline","context":{"idset":"7739"}}
+{"timestamp":1714533938.0123596,"name":"online","context":{"idset":"11291-11292"}}
+{"timestamp":1714543206.2093215,"name":"offline","context":{"idset":"7884"}}
+{"timestamp":1714556082.2058747,"name":"offline","context":{"idset":"11790"}}
+{"timestamp":1714563548.2051921,"name":"offline","context":{"idset":"7758"}}
+{"timestamp":1714567600.8340421,"name":"offline","context":{"idset":"9987"}}
+{"timestamp":1714567600.9338629,"name":"offline","context":{"idset":"9994"}}
+{"timestamp":1714567601.1797752,"name":"offline","context":{"idset":"10048"}}
+{"timestamp":1714567601.3496268,"name":"offline","context":{"idset":"9974"}}
+{"timestamp":1714567601.3868129,"name":"offline","context":{"idset":"10035"}}
+{"timestamp":1714567601.4821188,"name":"offline","context":{"idset":"9984"}}
+{"timestamp":1714567601.6862409,"name":"offline","context":{"idset":"10023"}}
+{"timestamp":1714567601.6952088,"name":"offline","context":{"idset":"10088"}}
+{"timestamp":1714567601.706316,"name":"offline","context":{"idset":"10044"}}
+{"timestamp":1714567601.71311,"name":"offline","context":{"idset":"10034"}}
+{"timestamp":1714567601.7196262,"name":"offline","context":{"idset":"10022"}}
+{"timestamp":1714567601.7255666,"name":"offline","context":{"idset":"10041"}}
+{"timestamp":1714567601.7328925,"name":"offline","context":{"idset":"10020"}}
+{"timestamp":1714567601.7396421,"name":"offline","context":{"idset":"10089"}}
+{"timestamp":1714567601.7731864,"name":"offline","context":{"idset":"10068"}}
+{"timestamp":1714567601.7779276,"name":"offline","context":{"idset":"10007"}}
+{"timestamp":1714567601.7828693,"name":"offline","context":{"idset":"10012"}}
+{"timestamp":1714567601.7881849,"name":"offline","context":{"idset":"10045"}}
+{"timestamp":1714567601.9041996,"name":"offline","context":{"idset":"10061"}}
+{"timestamp":1714567602.0565491,"name":"offline","context":{"idset":"10086"}}
+{"timestamp":1714567602.0628448,"name":"offline","context":{"idset":"10043"}}
+{"timestamp":1714567602.0704076,"name":"offline","context":{"idset":"10013"}}
+{"timestamp":1714567602.0759175,"name":"offline","context":{"idset":"10004"}}
+{"timestamp":1714567602.0910134,"name":"offline","context":{"idset":"10087"}}
+{"timestamp":1714567602.2236679,"name":"offline","context":{"idset":"10069"}}
+{"timestamp":1714567602.2645452,"name":"offline","context":{"idset":"10064"}}
+{"timestamp":1714567602.3542855,"name":"offline","context":{"idset":"9983"}}
+{"timestamp":1714567602.3593733,"name":"offline","context":{"idset":"10082"}}
+{"timestamp":1714567602.3688464,"name":"offline","context":{"idset":"10030"}}
+{"timestamp":1714567602.3739071,"name":"offline","context":{"idset":"10000"}}
+{"timestamp":1714567602.3788311,"name":"offline","context":{"idset":"10032"}}
+{"timestamp":1714567602.3837817,"name":"offline","context":{"idset":"10093"}}
+{"timestamp":1714567602.3887858,"name":"offline","context":{"idset":"10056"}}
+{"timestamp":1714567602.3978307,"name":"offline","context":{"idset":"9976"}}
+{"timestamp":1714567602.4028175,"name":"offline","context":{"idset":"9999"}}
+{"timestamp":1714567602.4078982,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1714567602.4128962,"name":"offline","context":{"idset":"10026"}}
+{"timestamp":1714567602.4238429,"name":"offline","context":{"idset":"10011"}}
+{"timestamp":1714567602.4307015,"name":"offline","context":{"idset":"10080"}}
+{"timestamp":1714567602.4388506,"name":"offline","context":{"idset":"10021"}}
+{"timestamp":1714567602.4438868,"name":"offline","context":{"idset":"9996"}}
+{"timestamp":1714567602.4525402,"name":"offline","context":{"idset":"10100"}}
+{"timestamp":1714567602.4578669,"name":"offline","context":{"idset":"10096"}}
+{"timestamp":1714567602.6390975,"name":"offline","context":{"idset":"10099"}}
+{"timestamp":1714567602.6632648,"name":"offline","context":{"idset":"10046"}}
+{"timestamp":1714567602.6888793,"name":"offline","context":{"idset":"10051"}}
+{"timestamp":1714567602.6937129,"name":"offline","context":{"idset":"9992"}}
+{"timestamp":1714567602.6985023,"name":"offline","context":{"idset":"10083"}}
+{"timestamp":1714567602.7033088,"name":"offline","context":{"idset":"10053"}}
+{"timestamp":1714567602.7081292,"name":"offline","context":{"idset":"9997"}}
+{"timestamp":1714567602.7112677,"name":"offline","context":{"idset":"10010"}}
+{"timestamp":1714567602.7138729,"name":"offline","context":{"idset":"10071"}}
+{"timestamp":1714567602.716471,"name":"offline","context":{"idset":"10009"}}
+{"timestamp":1714567602.7191045,"name":"offline","context":{"idset":"10042"}}
+{"timestamp":1714567602.7217517,"name":"offline","context":{"idset":"10065"}}
+{"timestamp":1714567602.7244821,"name":"offline","context":{"idset":"9985"}}
+{"timestamp":1714567602.7394397,"name":"offline","context":{"idset":"9998"}}
+{"timestamp":1714567602.7550397,"name":"offline","context":{"idset":"9978"}}
+{"timestamp":1714567602.7598844,"name":"offline","context":{"idset":"9981"}}
+{"timestamp":1714567602.7647822,"name":"offline","context":{"idset":"9982"}}
+{"timestamp":1714567602.7696147,"name":"offline","context":{"idset":"9991"}}
+{"timestamp":1714567602.7851546,"name":"offline","context":{"idset":"10006"}}
+{"timestamp":1714567602.7899642,"name":"offline","context":{"idset":"10014"}}
+{"timestamp":1714567602.7947903,"name":"offline","context":{"idset":"10040"}}
+{"timestamp":1714567602.7995925,"name":"offline","context":{"idset":"10047"}}
+{"timestamp":1714567602.8038805,"name":"offline","context":{"idset":"10054"}}
+{"timestamp":1714567602.8070054,"name":"offline","context":{"idset":"10058"}}
+{"timestamp":1714567602.8101218,"name":"offline","context":{"idset":"10073"}}
+{"timestamp":1714567602.8132451,"name":"offline","context":{"idset":"10078"}}
+{"timestamp":1714567602.8163967,"name":"offline","context":{"idset":"10081"}}
+{"timestamp":1714567602.8195126,"name":"offline","context":{"idset":"10098"}}
+{"timestamp":1714567602.9880428,"name":"offline","context":{"idset":"9973"}}
+{"timestamp":1714567602.9905007,"name":"offline","context":{"idset":"10036"}}
+{"timestamp":1714567602.9929492,"name":"offline","context":{"idset":"10033"}}
+{"timestamp":1714567602.9953959,"name":"offline","context":{"idset":"10070"}}
+{"timestamp":1714567602.9978409,"name":"offline","context":{"idset":"9986"}}
+{"timestamp":1714567603.0002799,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1714567603.0027409,"name":"offline","context":{"idset":"10062"}}
+{"timestamp":1714567603.0051877,"name":"offline","context":{"idset":"10485"}}
+{"timestamp":1714567603.0076246,"name":"offline","context":{"idset":"10015"}}
+{"timestamp":1714567603.0100663,"name":"offline","context":{"idset":"10027"}}
+{"timestamp":1714567603.0124991,"name":"offline","context":{"idset":"10066"}}
+{"timestamp":1714567603.0149245,"name":"offline","context":{"idset":"10079"}}
+{"timestamp":1714567603.0173709,"name":"offline","context":{"idset":"10085"}}
+{"timestamp":1714567603.0198162,"name":"offline","context":{"idset":"10076"}}
+{"timestamp":1714567603.0222628,"name":"offline","context":{"idset":"10055"}}
+{"timestamp":1714567603.0247083,"name":"offline","context":{"idset":"10091"}}
+{"timestamp":1714567603.1271238,"name":"offline","context":{"idset":"10019"}}
+{"timestamp":1714567603.1487134,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1714567603.1543407,"name":"offline","context":{"idset":"10031"}}
+{"timestamp":1714567603.176126,"name":"offline","context":{"idset":"10025"}}
+{"timestamp":1714567603.1785631,"name":"offline","context":{"idset":"10090"}}
+{"timestamp":1714567603.25844,"name":"offline","context":{"idset":"10092"}}
+{"timestamp":1714567603.344378,"name":"offline","context":{"idset":"10037"}}
+{"timestamp":1714567603.3501871,"name":"offline","context":{"idset":"10018"}}
+{"timestamp":1714567603.3558638,"name":"offline","context":{"idset":"9979"}}
+{"timestamp":1714567603.455241,"name":"offline","context":{"idset":"10488"}}
+{"timestamp":1714567603.4701009,"name":"offline","context":{"idset":"10016"}}
+{"timestamp":1714567603.4846399,"name":"offline","context":{"idset":"10050"}}
+{"timestamp":1714567603.5832112,"name":"offline","context":{"idset":"10077"}}
+{"timestamp":1714567603.6130025,"name":"offline","context":{"idset":"10097"}}
+{"timestamp":1714567603.6183658,"name":"offline","context":{"idset":"10024"}}
+{"timestamp":1714567603.6393919,"name":"offline","context":{"idset":"10029"}}
+{"timestamp":1714567603.6433079,"name":"offline","context":{"idset":"10028"}}
+{"timestamp":1714567603.7244437,"name":"offline","context":{"idset":"10487"}}
+{"timestamp":1714567603.8210196,"name":"offline","context":{"idset":"10489"}}
+{"timestamp":1714567603.825007,"name":"offline","context":{"idset":"10008"}}
+{"timestamp":1714567603.8293028,"name":"offline","context":{"idset":"10072"}}
+{"timestamp":1714567603.8347845,"name":"offline","context":{"idset":"10003"}}
+{"timestamp":1714567603.8406634,"name":"offline","context":{"idset":"10095"}}
+{"timestamp":1714567603.8464134,"name":"offline","context":{"idset":"10074"}}
+{"timestamp":1714567603.9466295,"name":"offline","context":{"idset":"10501"}}
+{"timestamp":1714567604.0459707,"name":"offline","context":{"idset":"9995"}}
+{"timestamp":1714567604.052618,"name":"offline","context":{"idset":"9980"}}
+{"timestamp":1714567604.0577335,"name":"offline","context":{"idset":"9990"}}
+{"timestamp":1714567604.062335,"name":"offline","context":{"idset":"10525"}}
+{"timestamp":1714567604.0675986,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1714567604.1710656,"name":"offline","context":{"idset":"10052"}}
+{"timestamp":1714567604.242461,"name":"offline","context":{"idset":"10063"}}
+{"timestamp":1714567604.2515082,"name":"offline","context":{"idset":"10067"}}
+{"timestamp":1714567604.2554564,"name":"offline","context":{"idset":"10049"}}
+{"timestamp":1714567604.2832663,"name":"offline","context":{"idset":"10526"}}
+{"timestamp":1714567604.3005235,"name":"offline","context":{"idset":"10496"}}
+{"timestamp":1714567604.3994207,"name":"offline","context":{"idset":"10017"}}
+{"timestamp":1714567604.4221442,"name":"offline","context":{"idset":"10094"}}
+{"timestamp":1714567604.4332714,"name":"offline","context":{"idset":"10502"}}
+{"timestamp":1714567604.4575241,"name":"offline","context":{"idset":"10084"}}
+{"timestamp":1714567604.5399663,"name":"offline","context":{"idset":"10549"}}
+{"timestamp":1714567604.6416845,"name":"offline","context":{"idset":"10515"}}
+{"timestamp":1714567604.6517978,"name":"offline","context":{"idset":"10541"}}
+{"timestamp":1714567604.6601489,"name":"offline","context":{"idset":"10510"}}
+{"timestamp":1714567604.6683307,"name":"offline","context":{"idset":"10589"}}
+{"timestamp":1714567604.6774516,"name":"offline","context":{"idset":"10005"}}
+{"timestamp":1714567604.687515,"name":"offline","context":{"idset":"10544"}}
+{"timestamp":1714567604.7843435,"name":"offline","context":{"idset":"10057"}}
+{"timestamp":1714567604.8822253,"name":"offline","context":{"idset":"10555"}}
+{"timestamp":1714567604.8872855,"name":"offline","context":{"idset":"10059"}}
+{"timestamp":1714567604.8927953,"name":"offline","context":{"idset":"9988"}}
+{"timestamp":1714567604.8970361,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1714567604.9043622,"name":"offline","context":{"idset":"10498"}}
+{"timestamp":1714567604.9083183,"name":"offline","context":{"idset":"10514"}}
+{"timestamp":1714567604.9122326,"name":"offline","context":{"idset":"9993"}}
+{"timestamp":1714567605.0151155,"name":"offline","context":{"idset":"10537"}}
+{"timestamp":1714567605.0636282,"name":"offline","context":{"idset":"10587"}}
+{"timestamp":1714567605.0710125,"name":"offline","context":{"idset":"10550"}}
+{"timestamp":1714567605.0749705,"name":"offline","context":{"idset":"10571"}}
+{"timestamp":1714567605.0788786,"name":"offline","context":{"idset":"10567"}}
+{"timestamp":1714567605.1699326,"name":"offline","context":{"idset":"10590"}}
+{"timestamp":1714567605.1947374,"name":"offline","context":{"idset":"10598"}}
+{"timestamp":1714567605.1999111,"name":"offline","context":{"idset":"10504"}}
+{"timestamp":1714567605.2338703,"name":"offline","context":{"idset":"10612"}}
+{"timestamp":1714567605.2377563,"name":"offline","context":{"idset":"10492"}}
+{"timestamp":1714567605.2502878,"name":"offline","context":{"idset":"10509"}}
+{"timestamp":1714567605.2542274,"name":"offline","context":{"idset":"10522"}}
+{"timestamp":1714567605.2581227,"name":"offline","context":{"idset":"10566"}}
+{"timestamp":1714567605.2620227,"name":"offline","context":{"idset":"10575"}}
+{"timestamp":1714567605.2659571,"name":"offline","context":{"idset":"10604"}}
+{"timestamp":1714567605.3051522,"name":"offline","context":{"idset":"10608"}}
+{"timestamp":1714567605.4283292,"name":"offline","context":{"idset":"10611"}}
+{"timestamp":1714567605.4351728,"name":"offline","context":{"idset":"10552"}}
+{"timestamp":1714567605.440402,"name":"offline","context":{"idset":"10516"}}
+{"timestamp":1714567605.4469404,"name":"offline","context":{"idset":"10574"}}
+{"timestamp":1714567605.4519475,"name":"offline","context":{"idset":"9977"}}
+{"timestamp":1714567605.4582429,"name":"offline","context":{"idset":"10527"}}
+{"timestamp":1714567605.4634929,"name":"offline","context":{"idset":"10039"}}
+{"timestamp":1714567605.5078735,"name":"offline","context":{"idset":"10554"}}
+{"timestamp":1714567605.511755,"name":"offline","context":{"idset":"10511"}}
+{"timestamp":1714567605.5156095,"name":"offline","context":{"idset":"10536"}}
+{"timestamp":1714567605.5195265,"name":"offline","context":{"idset":"10570"}}
+{"timestamp":1714567605.5320914,"name":"offline","context":{"idset":"10578"}}
+{"timestamp":1714567605.5359943,"name":"offline","context":{"idset":"10591"}}
+{"timestamp":1714567605.5398619,"name":"offline","context":{"idset":"10593"}}
+{"timestamp":1714567605.543771,"name":"offline","context":{"idset":"10594"}}
+{"timestamp":1714567605.5476727,"name":"offline","context":{"idset":"10606"}}
+{"timestamp":1714567605.6810226,"name":"offline","context":{"idset":"10038"}}
+{"timestamp":1714567605.684901,"name":"offline","context":{"idset":"10577"}}
+{"timestamp":1714567605.6902266,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1714567605.6997626,"name":"offline","context":{"idset":"10601"}}
+{"timestamp":1714567605.7036512,"name":"offline","context":{"idset":"10075"}}
+{"timestamp":1714567605.7117221,"name":"offline","context":{"idset":"10519"}}
+{"timestamp":1714567605.7168505,"name":"offline","context":{"idset":"10565"}}
+{"timestamp":1714567605.7221086,"name":"offline","context":{"idset":"10599"}}
+{"timestamp":1714567605.731385,"name":"offline","context":{"idset":"10602"}}
+{"timestamp":1714567605.7435491,"name":"offline","context":{"idset":"10532"}}
+{"timestamp":1714567605.7474205,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1714567605.8744609,"name":"offline","context":{"idset":"10529"}}
+{"timestamp":1714567605.8825955,"name":"offline","context":{"idset":"10499"}}
+{"timestamp":1714567605.8897724,"name":"offline","context":{"idset":"10547"}}
+{"timestamp":1714567605.8939967,"name":"offline","context":{"idset":"10497"}}
+{"timestamp":1714567605.8978696,"name":"offline","context":{"idset":"10495"}}
+{"timestamp":1714567605.901773,"name":"offline","context":{"idset":"10568"}}
+{"timestamp":1714567605.9056323,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1714567606.0059478,"name":"offline","context":{"idset":"10572"}}
+{"timestamp":1714567606.1850221,"name":"offline","context":{"idset":"10569"}}
+{"timestamp":1714567606.192549,"name":"offline","context":{"idset":"9989"}}
+{"timestamp":1714567606.1978924,"name":"offline","context":{"idset":"10548"}}
+{"timestamp":1714567606.2023301,"name":"offline","context":{"idset":"10556"}}
+{"timestamp":1714567606.2070611,"name":"offline","context":{"idset":"10580"}}
+{"timestamp":1714567606.2144699,"name":"offline","context":{"idset":"10060"}}
+{"timestamp":1714567606.2183771,"name":"offline","context":{"idset":"10609"}}
+{"timestamp":1714567606.222229,"name":"offline","context":{"idset":"10603"}}
+{"timestamp":1714567606.2427006,"name":"offline","context":{"idset":"10605"}}
+{"timestamp":1714567606.246551,"name":"offline","context":{"idset":"10508"}}
+{"timestamp":1714567606.2504249,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1714567606.2542622,"name":"offline","context":{"idset":"10521"}}
+{"timestamp":1714567606.3685935,"name":"offline","context":{"idset":"10546"}}
+{"timestamp":1714567606.4818194,"name":"offline","context":{"idset":"10557"}}
+{"timestamp":1714567606.485971,"name":"offline","context":{"idset":"10607"}}
+{"timestamp":1714567606.4914927,"name":"offline","context":{"idset":"10592"}}
+{"timestamp":1714567606.4954834,"name":"offline","context":{"idset":"10534"}}
+{"timestamp":1714567606.500241,"name":"offline","context":{"idset":"10545"}}
+{"timestamp":1714567606.5047441,"name":"offline","context":{"idset":"10530"}}
+{"timestamp":1714567606.5094132,"name":"offline","context":{"idset":"10506"}}
+{"timestamp":1714567606.5139132,"name":"offline","context":{"idset":"10583"}}
+{"timestamp":1714567606.5286674,"name":"offline","context":{"idset":"10560"}}
+{"timestamp":1714567606.6400971,"name":"offline","context":{"idset":"10553"}}
+{"timestamp":1714567606.687283,"name":"offline","context":{"idset":"10535"}}
+{"timestamp":1714567606.6912148,"name":"offline","context":{"idset":"10551"}}
+{"timestamp":1714567606.6950762,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1714567606.7919478,"name":"offline","context":{"idset":"10562"}}
+{"timestamp":1714567606.9651988,"name":"offline","context":{"idset":"10559"}}
+{"timestamp":1714567606.973794,"name":"offline","context":{"idset":"10493"}}
+{"timestamp":1714567606.9874926,"name":"offline","context":{"idset":"10542"}}
+{"timestamp":1714567606.9946833,"name":"offline","context":{"idset":"10531"}}
+{"timestamp":1714567607.0019801,"name":"offline","context":{"idset":"10533"}}
+{"timestamp":1714567607.0097435,"name":"offline","context":{"idset":"10540"}}
+{"timestamp":1714567607.0178657,"name":"offline","context":{"idset":"10582"}}
+{"timestamp":1714567607.0254037,"name":"offline","context":{"idset":"10610"}}
+{"timestamp":1714567607.2169278,"name":"offline","context":{"idset":"10584"}}
+{"timestamp":1714567607.3941834,"name":"offline","context":{"idset":"9975"}}
+{"timestamp":1714567607.4496474,"name":"offline","context":{"idset":"10600"}}
+{"timestamp":1714567607.4734583,"name":"offline","context":{"idset":"10561"}}
+{"timestamp":1714567607.5718472,"name":"offline","context":{"idset":"10518"}}
+{"timestamp":1714567607.7574527,"name":"offline","context":{"idset":"10585"}}
+{"timestamp":1714567607.7680321,"name":"offline","context":{"idset":"10494"}}
+{"timestamp":1714567607.8787186,"name":"offline","context":{"idset":"10513"}}
+{"timestamp":1714567608.0352261,"name":"offline","context":{"idset":"10581"}}
+{"timestamp":1714567608.0631778,"name":"offline","context":{"idset":"10517"}}
+{"timestamp":1714567608.1605151,"name":"offline","context":{"idset":"10538"}}
+{"timestamp":1714567608.3295858,"name":"offline","context":{"idset":"10558"}}
+{"timestamp":1714567608.3400853,"name":"offline","context":{"idset":"10596"}}
+{"timestamp":1714567608.3450651,"name":"offline","context":{"idset":"10500"}}
+{"timestamp":1714567608.4372144,"name":"offline","context":{"idset":"10595"}}
+{"timestamp":1714567608.562629,"name":"offline","context":{"idset":"10586"}}
+{"timestamp":1714567608.5899937,"name":"offline","context":{"idset":"10520"}}
+{"timestamp":1714567608.6876845,"name":"offline","context":{"idset":"10576"}}
+{"timestamp":1714567608.9464734,"name":"offline","context":{"idset":"10588"}}
+{"timestamp":1714567609.1582687,"name":"offline","context":{"idset":"10579"}}
+{"timestamp":1714567609.2576668,"name":"offline","context":{"idset":"10505"}}
+{"timestamp":1714567609.6591229,"name":"offline","context":{"idset":"10523"}}
+{"timestamp":1714567609.7577646,"name":"offline","context":{"idset":"10539"}}
+{"timestamp":1714567609.8603501,"name":"offline","context":{"idset":"10543"}}
+{"timestamp":1714567609.9381113,"name":"offline","context":{"idset":"10528"}}
+{"timestamp":1714567610.0373905,"name":"offline","context":{"idset":"10563"}}
+{"timestamp":1714567610.2651417,"name":"offline","context":{"idset":"10573"}}
+{"timestamp":1714567610.3801925,"name":"offline","context":{"idset":"10564"}}
+{"timestamp":1714569298.2049336,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1714571521.1874361,"name":"online","context":{"idset":"943"}}
+{"timestamp":1714571525.4532897,"name":"online","context":{"idset":"944"}}
+{"timestamp":1714571918.2046752,"name":"offline","context":{"idset":"943"}}
+{"timestamp":1714572324.3084517,"name":"offline","context":{"idset":"9076"}}
+{"timestamp":1714572356.2033975,"name":"offline","context":{"idset":"845"}}
+{"timestamp":1714572376.2028441,"name":"offline","context":{"idset":"846"}}
+{"timestamp":1714572552.2041047,"name":"offline","context":{"idset":"847"}}
+{"timestamp":1714572554.2073147,"name":"offline","context":{"idset":"848"}}
+{"timestamp":1714574191.5720844,"name":"drain","context":{"idset":"405-420","reason":"New Blade Install --JRG","overwrite":1}}
+{"timestamp":1714574225.1760273,"name":"offline","context":{"idset":"406"}}
+{"timestamp":1714574225.1804032,"name":"offline","context":{"idset":"407"}}
+{"timestamp":1714574225.1847899,"name":"offline","context":{"idset":"409"}}
+{"timestamp":1714574225.1893299,"name":"offline","context":{"idset":"405"}}
+{"timestamp":1714574225.1938233,"name":"offline","context":{"idset":"417"}}
+{"timestamp":1714574225.1982751,"name":"offline","context":{"idset":"415"}}
+{"timestamp":1714574225.2026978,"name":"offline","context":{"idset":"420"}}
+{"timestamp":1714574225.2071609,"name":"offline","context":{"idset":"408"}}
+{"timestamp":1714574225.2117262,"name":"offline","context":{"idset":"416"}}
+{"timestamp":1714574225.216192,"name":"offline","context":{"idset":"412"}}
+{"timestamp":1714574225.2208798,"name":"offline","context":{"idset":"419"}}
+{"timestamp":1714574225.2242026,"name":"offline","context":{"idset":"414"}}
+{"timestamp":1714574225.2266784,"name":"offline","context":{"idset":"418"}}
+{"timestamp":1714574225.231241,"name":"offline","context":{"idset":"413"}}
+{"timestamp":1714574418.2039294,"name":"offline","context":{"idset":"7827"}}
+{"timestamp":1714574420.2042186,"name":"offline","context":{"idset":"7828"}}
+{"timestamp":1714574454.2033343,"name":"offline","context":{"idset":"891"}}
+{"timestamp":1714574508.9003124,"name":"online","context":{"idset":"1047"}}
+{"timestamp":1714574526.6010413,"name":"online","context":{"idset":"1048"}}
+{"timestamp":1714576757.0606298,"name":"online","context":{"idset":"7014,7020"}}
+{"timestamp":1714576757.1854184,"name":"online","context":{"idset":"7024"}}
+{"timestamp":1714576757.3038766,"name":"online","context":{"idset":"7021,7026"}}
+{"timestamp":1714576757.585418,"name":"online","context":{"idset":"7015"}}
+{"timestamp":1714576757.8631902,"name":"online","context":{"idset":"7022,7028"}}
+{"timestamp":1714576757.9664485,"name":"online","context":{"idset":"7017"}}
+{"timestamp":1714576759.1078222,"name":"online","context":{"idset":"7016,7018-7019,7023,7025,7027"}}
+{"timestamp":1714576959.80234,"name":"online","context":{"idset":"6998"}}
+{"timestamp":1714576961.4434474,"name":"online","context":{"idset":"7004-7005,7007"}}
+{"timestamp":1714576961.4460542,"name":"online","context":{"idset":"7008"}}
+{"timestamp":1714576961.4485087,"name":"online","context":{"idset":"6997,6999,7006,7009-7010"}}
+{"timestamp":1714576961.4510429,"name":"online","context":{"idset":"7002,7011"}}
+{"timestamp":1714576961.5374014,"name":"online","context":{"idset":"7001"}}
+{"timestamp":1714576961.8034892,"name":"online","context":{"idset":"7003"}}
+{"timestamp":1714576962.0711634,"name":"online","context":{"idset":"7000,7012"}}
+{"timestamp":1714577007.6855032,"name":"online","context":{"idset":"8105"}}
+{"timestamp":1714577007.6905129,"name":"online","context":{"idset":"8106"}}
+{"timestamp":1714577282.7930446,"name":"online","context":{"idset":"7884"}}
+{"timestamp":1714577596.3620894,"name":"online","context":{"idset":"943"}}
+{"timestamp":1714577734.1176996,"name":"offline","context":{"idset":"8821"}}
+{"timestamp":1714577734.1205764,"name":"offline","context":{"idset":"8822"}}
+{"timestamp":1714577734.2045827,"name":"offline","context":{"idset":"8823"}}
+{"timestamp":1714577736.1192989,"name":"offline","context":{"idset":"8824"}}
+{"timestamp":1714577736.2051401,"name":"offline","context":{"idset":"8825"}}
+{"timestamp":1714577738.1327906,"name":"offline","context":{"idset":"8826"}}
+{"timestamp":1714577738.1373827,"name":"offline","context":{"idset":"8827"}}
+{"timestamp":1714577738.1419485,"name":"offline","context":{"idset":"8828"}}
+{"timestamp":1714577738.219758,"name":"offline","context":{"idset":"8829"}}
+{"timestamp":1714577740.1272936,"name":"offline","context":{"idset":"8830"}}
+{"timestamp":1714577740.1314583,"name":"offline","context":{"idset":"8831"}}
+{"timestamp":1714577740.1356835,"name":"offline","context":{"idset":"8832"}}
+{"timestamp":1714577740.1398866,"name":"offline","context":{"idset":"8833"}}
+{"timestamp":1714577740.2293048,"name":"offline","context":{"idset":"8834"}}
+{"timestamp":1714577742.1246204,"name":"offline","context":{"idset":"8835"}}
+{"timestamp":1714577742.1284688,"name":"offline","context":{"idset":"8836"}}
+{"timestamp":1714577742.1322863,"name":"offline","context":{"idset":"8837"}}
+{"timestamp":1714577742.2070711,"name":"offline","context":{"idset":"8838"}}
+{"timestamp":1714577744.130172,"name":"offline","context":{"idset":"8839"}}
+{"timestamp":1714577744.1340294,"name":"offline","context":{"idset":"8840"}}
+{"timestamp":1714577744.1378474,"name":"offline","context":{"idset":"8841"}}
+{"timestamp":1714577744.214139,"name":"offline","context":{"idset":"8842"}}
+{"timestamp":1714577746.1285477,"name":"offline","context":{"idset":"8843"}}
+{"timestamp":1714577746.1333377,"name":"offline","context":{"idset":"8844"}}
+{"timestamp":1714577746.1380866,"name":"offline","context":{"idset":"8845"}}
+{"timestamp":1714577746.2608116,"name":"offline","context":{"idset":"8846"}}
+{"timestamp":1714577748.1318171,"name":"offline","context":{"idset":"8847"}}
+{"timestamp":1714577748.1346478,"name":"offline","context":{"idset":"8848"}}
+{"timestamp":1714577748.137599,"name":"offline","context":{"idset":"8849"}}
+{"timestamp":1714577748.1405675,"name":"offline","context":{"idset":"8850"}}
+{"timestamp":1714577748.205564,"name":"offline","context":{"idset":"8851"}}
+{"timestamp":1714577750.2817991,"name":"offline","context":{"idset":"8852"}}
+{"timestamp":1714577750.2855015,"name":"offline","context":{"idset":"8853"}}
+{"timestamp":1714577750.2892756,"name":"offline","context":{"idset":"8854"}}
+{"timestamp":1714577752.15571,"name":"offline","context":{"idset":"8855"}}
+{"timestamp":1714577752.1614447,"name":"offline","context":{"idset":"8856"}}
+{"timestamp":1714577752.167851,"name":"offline","context":{"idset":"8857"}}
+{"timestamp":1714577752.2198548,"name":"offline","context":{"idset":"8858"}}
+{"timestamp":1714577754.1263976,"name":"offline","context":{"idset":"8859"}}
+{"timestamp":1714577754.1335673,"name":"offline","context":{"idset":"8860"}}
+{"timestamp":1714577754.2034211,"name":"offline","context":{"idset":"8861"}}
+{"timestamp":1714577756.1346858,"name":"offline","context":{"idset":"8862"}}
+{"timestamp":1714577756.1394417,"name":"offline","context":{"idset":"8863"}}
+{"timestamp":1714577756.1441779,"name":"offline","context":{"idset":"8864"}}
+{"timestamp":1714577756.1489003,"name":"offline","context":{"idset":"8865"}}
+{"timestamp":1714577756.2306559,"name":"offline","context":{"idset":"8866"}}
+{"timestamp":1714577758.1296525,"name":"offline","context":{"idset":"8867"}}
+{"timestamp":1714577758.1336472,"name":"offline","context":{"idset":"8868"}}
+{"timestamp":1714577758.1376076,"name":"offline","context":{"idset":"8869"}}
+{"timestamp":1714577758.2168875,"name":"offline","context":{"idset":"8870"}}
+{"timestamp":1714577760.1253006,"name":"offline","context":{"idset":"8871"}}
+{"timestamp":1714577760.1297314,"name":"offline","context":{"idset":"8872"}}
+{"timestamp":1714577760.1339688,"name":"offline","context":{"idset":"8873"}}
+{"timestamp":1714577760.2172196,"name":"offline","context":{"idset":"8874"}}
+{"timestamp":1714577762.1423614,"name":"offline","context":{"idset":"8875"}}
+{"timestamp":1714577762.1487806,"name":"offline","context":{"idset":"8876"}}
+{"timestamp":1714577762.1538289,"name":"offline","context":{"idset":"8877"}}
+{"timestamp":1714577762.1588764,"name":"offline","context":{"idset":"8878"}}
+{"timestamp":1714577762.1639023,"name":"offline","context":{"idset":"8879"}}
+{"timestamp":1714577762.208945,"name":"offline","context":{"idset":"8880"}}
+{"timestamp":1714577764.1247511,"name":"offline","context":{"idset":"8881"}}
+{"timestamp":1714577764.1285291,"name":"offline","context":{"idset":"8882"}}
+{"timestamp":1714577764.1322861,"name":"offline","context":{"idset":"8883"}}
+{"timestamp":1714577764.2130885,"name":"offline","context":{"idset":"8884"}}
+{"timestamp":1714577766.1357787,"name":"offline","context":{"idset":"8885"}}
+{"timestamp":1714577766.141712,"name":"offline","context":{"idset":"8886"}}
+{"timestamp":1714577766.147536,"name":"offline","context":{"idset":"8887"}}
+{"timestamp":1714577766.2337768,"name":"offline","context":{"idset":"8888"}}
+{"timestamp":1714577768.1655834,"name":"offline","context":{"idset":"8889"}}
+{"timestamp":1714577768.1711316,"name":"offline","context":{"idset":"8890"}}
+{"timestamp":1714577768.1753526,"name":"offline","context":{"idset":"8891"}}
+{"timestamp":1714577768.2036211,"name":"offline","context":{"idset":"8892"}}
+{"timestamp":1714577770.4947309,"name":"offline","context":{"idset":"8893"}}
+{"timestamp":1714577770.4996705,"name":"offline","context":{"idset":"8894"}}
+{"timestamp":1714577770.5046828,"name":"offline","context":{"idset":"8895"}}
+{"timestamp":1714577770.509377,"name":"offline","context":{"idset":"8896"}}
+{"timestamp":1714577770.5138729,"name":"offline","context":{"idset":"8897"}}
+{"timestamp":1714577772.1292834,"name":"offline","context":{"idset":"8898"}}
+{"timestamp":1714577772.1338418,"name":"offline","context":{"idset":"8899"}}
+{"timestamp":1714577772.1423676,"name":"offline","context":{"idset":"8900"}}
+{"timestamp":1714577772.1470902,"name":"offline","context":{"idset":"8901"}}
+{"timestamp":1714577772.2151399,"name":"offline","context":{"idset":"8902"}}
+{"timestamp":1714577774.130368,"name":"offline","context":{"idset":"8903"}}
+{"timestamp":1714577774.1364136,"name":"offline","context":{"idset":"8904"}}
+{"timestamp":1714577774.1406631,"name":"offline","context":{"idset":"8905"}}
+{"timestamp":1714577774.1444464,"name":"offline","context":{"idset":"8906"}}
+{"timestamp":1714577774.2267294,"name":"offline","context":{"idset":"8907"}}
+{"timestamp":1714577776.1326933,"name":"offline","context":{"idset":"8908"}}
+{"timestamp":1714577776.1367118,"name":"offline","context":{"idset":"8909"}}
+{"timestamp":1714577776.141449,"name":"offline","context":{"idset":"8910"}}
+{"timestamp":1714577776.1457191,"name":"offline","context":{"idset":"8911"}}
+{"timestamp":1714577776.2676351,"name":"offline","context":{"idset":"8912"}}
+{"timestamp":1714577778.1152163,"name":"offline","context":{"idset":"8913"}}
+{"timestamp":1714577778.1180611,"name":"offline","context":{"idset":"8914"}}
+{"timestamp":1714577778.7009642,"name":"offline","context":{"idset":"8915"}}
+{"timestamp":1714577780.1315434,"name":"offline","context":{"idset":"8916"}}
+{"timestamp":1714577780.1362936,"name":"offline","context":{"idset":"8917"}}
+{"timestamp":1714577780.1410961,"name":"offline","context":{"idset":"8918"}}
+{"timestamp":1714577780.2310498,"name":"offline","context":{"idset":"8919"}}
+{"timestamp":1714577782.1342614,"name":"offline","context":{"idset":"8920"}}
+{"timestamp":1714577782.1383772,"name":"offline","context":{"idset":"8921"}}
+{"timestamp":1714577782.142401,"name":"offline","context":{"idset":"8922"}}
+{"timestamp":1714577782.1463811,"name":"offline","context":{"idset":"8923"}}
+{"timestamp":1714577782.2347965,"name":"offline","context":{"idset":"8924"}}
+{"timestamp":1714577784.126368,"name":"offline","context":{"idset":"943"}}
+{"timestamp":1714577784.1305959,"name":"offline","context":{"idset":"8925"}}
+{"timestamp":1714577784.1349449,"name":"offline","context":{"idset":"8926"}}
+{"timestamp":1714577784.138957,"name":"offline","context":{"idset":"8927"}}
+{"timestamp":1714577784.2318258,"name":"offline","context":{"idset":"8928"}}
+{"timestamp":1714577786.1349795,"name":"offline","context":{"idset":"8929"}}
+{"timestamp":1714577786.1409597,"name":"offline","context":{"idset":"8930"}}
+{"timestamp":1714577786.1451054,"name":"offline","context":{"idset":"8931"}}
+{"timestamp":1714577786.2268369,"name":"offline","context":{"idset":"8932"}}
+{"timestamp":1714577788.1361825,"name":"offline","context":{"idset":"8933"}}
+{"timestamp":1714577788.1406872,"name":"offline","context":{"idset":"8934"}}
+{"timestamp":1714577788.2245457,"name":"offline","context":{"idset":"8935"}}
+{"timestamp":1714577789.1952598,"name":"offline","context":{"idset":"8936"}}
+{"timestamp":1714577789.1994197,"name":"offline","context":{"idset":"8937"}}
+{"timestamp":1714577789.2035022,"name":"offline","context":{"idset":"8938"}}
+{"timestamp":1714577789.2076848,"name":"offline","context":{"idset":"8939"}}
+{"timestamp":1714577789.296613,"name":"offline","context":{"idset":"8940"}}
+{"timestamp":1714577792.1236031,"name":"offline","context":{"idset":"8941"}}
+{"timestamp":1714577792.1283062,"name":"offline","context":{"idset":"8942"}}
+{"timestamp":1714577792.2050195,"name":"offline","context":{"idset":"8943"}}
+{"timestamp":1714577794.1264398,"name":"offline","context":{"idset":"8944"}}
+{"timestamp":1714577794.1301973,"name":"offline","context":{"idset":"8945"}}
+{"timestamp":1714577794.1339228,"name":"offline","context":{"idset":"8946"}}
+{"timestamp":1714577794.1376252,"name":"offline","context":{"idset":"8947"}}
+{"timestamp":1714577794.225153,"name":"offline","context":{"idset":"8948"}}
+{"timestamp":1714577877.7787347,"name":"online","context":{"idset":"780"}}
+{"timestamp":1714577877.7819707,"name":"online","context":{"idset":"779"}}
+{"timestamp":1714577913.7646124,"name":"undrain","context":{"idset":"779-780"}}
+{"timestamp":1714578060.1298807,"name":"offline","context":{"idset":"825"}}
+{"timestamp":1714578060.2056422,"name":"offline","context":{"idset":"826"}}
+{"timestamp":1714578754.0813568,"name":"online","context":{"idset":"7037"}}
+{"timestamp":1714578755.684552,"name":"online","context":{"idset":"7040"}}
+{"timestamp":1714578755.6872227,"name":"online","context":{"idset":"7035,7048"}}
+{"timestamp":1714578756.456563,"name":"online","context":{"idset":"7043,7051"}}
+{"timestamp":1714578757.3368645,"name":"online","context":{"idset":"7050,7056"}}
+{"timestamp":1714578757.3406951,"name":"online","context":{"idset":"7054"}}
+{"timestamp":1714578757.3445463,"name":"online","context":{"idset":"7032"}}
+{"timestamp":1714578757.354316,"name":"online","context":{"idset":"7044"}}
+{"timestamp":1714578758.1907637,"name":"online","context":{"idset":"7034"}}
+{"timestamp":1714578759.2085381,"name":"online","context":{"idset":"7030"}}
+{"timestamp":1714578759.2133434,"name":"online","context":{"idset":"7045"}}
+{"timestamp":1714578759.2175534,"name":"online","context":{"idset":"7052,7115,7156"}}
+{"timestamp":1714578759.2215939,"name":"online","context":{"idset":"7042,7104"}}
+{"timestamp":1714578759.2265432,"name":"online","context":{"idset":"7076"}}
+{"timestamp":1714578759.231956,"name":"online","context":{"idset":"7072"}}
+{"timestamp":1714578759.236655,"name":"online","context":{"idset":"7089,7117"}}
+{"timestamp":1714578759.2418327,"name":"online","context":{"idset":"7036"}}
+{"timestamp":1714578759.4347703,"name":"online","context":{"idset":"7058,7093"}}
+{"timestamp":1714578759.5776806,"name":"online","context":{"idset":"7053,7062,7122,7129"}}
+{"timestamp":1714578759.7978678,"name":"online","context":{"idset":"7038,7109,7134"}}
+{"timestamp":1714578760.0434926,"name":"online","context":{"idset":"7100,7114,7118,7127,7138,7143,7145"}}
+{"timestamp":1714578760.1741014,"name":"online","context":{"idset":"7116"}}
+{"timestamp":1714578760.9571836,"name":"online","context":{"idset":"7123"}}
+{"timestamp":1714578761.686054,"name":"online","context":{"idset":"7067,7070-7071,7087,7102,7119,7131-7133,7135,7153"}}
+{"timestamp":1714578761.6896744,"name":"online","context":{"idset":"7029,7039,7081,7088,7097,7099,7113,7128,7147,7151"}}
+{"timestamp":1714578761.6930542,"name":"online","context":{"idset":"7041,7055,7079,7084,7092,7096,7108,7142,7144"}}
+{"timestamp":1714578761.6962576,"name":"online","context":{"idset":"7066,7075,7090,7098,7101,7130,7140,7146"}}
+{"timestamp":1714578761.6999969,"name":"online","context":{"idset":"7057,7063-7064,7112,7137,7155"}}
+{"timestamp":1714578761.7106144,"name":"online","context":{"idset":"7033,7136,7139"}}
+{"timestamp":1714578761.7139375,"name":"online","context":{"idset":"7111,7154"}}
+{"timestamp":1714578761.7172196,"name":"online","context":{"idset":"7031"}}
+{"timestamp":1714578761.7204349,"name":"online","context":{"idset":"7049,7061,7077,7095,7105-7107,7124,7141,7148-7149,7152"}}
+{"timestamp":1714578761.7236252,"name":"online","context":{"idset":"7091,7103,7126,7150"}}
+{"timestamp":1714578762.0136914,"name":"online","context":{"idset":"7110"}}
+{"timestamp":1714578762.9685318,"name":"online","context":{"idset":"7121"}}
+{"timestamp":1714581317.578923,"name":"online","context":{"idset":"1052"}}
+{"timestamp":1714581317.6864626,"name":"online","context":{"idset":"1051"}}
+{"timestamp":1714581494.200691,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1714581944.0092878,"name":"online","context":{"idset":"6657,6659,6681"}}
+{"timestamp":1714581944.0134549,"name":"online","context":{"idset":"6650"}}
+{"timestamp":1714581944.0174227,"name":"online","context":{"idset":"6646,6651"}}
+{"timestamp":1714581944.8997705,"name":"online","context":{"idset":"6647"}}
+{"timestamp":1714581944.9036944,"name":"online","context":{"idset":"6666"}}
+{"timestamp":1714581945.6430602,"name":"online","context":{"idset":"6655"}}
+{"timestamp":1714581945.7300663,"name":"online","context":{"idset":"6665"}}
+{"timestamp":1714581945.8517258,"name":"online","context":{"idset":"6658,6686"}}
+{"timestamp":1714581947.1182778,"name":"online","context":{"idset":"6692"}}
+{"timestamp":1714581947.1222055,"name":"online","context":{"idset":"6656"}}
+{"timestamp":1714581949.5542712,"name":"online","context":{"idset":"6671"}}
+{"timestamp":1714581951.6357069,"name":"online","context":{"idset":"6679,6714"}}
+{"timestamp":1714581951.6417613,"name":"online","context":{"idset":"6664"}}
+{"timestamp":1714581951.6468201,"name":"online","context":{"idset":"6654,6672,6717"}}
+{"timestamp":1714581951.6517198,"name":"online","context":{"idset":"6739"}}
+{"timestamp":1714581951.6563916,"name":"online","context":{"idset":"6670,6735"}}
+{"timestamp":1714581951.6612785,"name":"online","context":{"idset":"6720"}}
+{"timestamp":1714581951.8122432,"name":"online","context":{"idset":"6712"}}
+{"timestamp":1714581951.9383025,"name":"online","context":{"idset":"6716,6742"}}
+{"timestamp":1714581953.594954,"name":"online","context":{"idset":"6740"}}
+{"timestamp":1714581953.5988848,"name":"online","context":{"idset":"6733"}}
+{"timestamp":1714581953.6027894,"name":"online","context":{"idset":"6734"}}
+{"timestamp":1714581953.6067135,"name":"online","context":{"idset":"6661"}}
+{"timestamp":1714581953.6110187,"name":"online","context":{"idset":"6747"}}
+{"timestamp":1714581953.6151056,"name":"online","context":{"idset":"6737"}}
+{"timestamp":1714581953.6192069,"name":"online","context":{"idset":"6687"}}
+{"timestamp":1714581953.7609274,"name":"online","context":{"idset":"6701"}}
+{"timestamp":1714581953.9584339,"name":"online","context":{"idset":"6708,6745"}}
+{"timestamp":1714581954.0801625,"name":"online","context":{"idset":"6673,6767"}}
+{"timestamp":1714581954.9225063,"name":"online","context":{"idset":"6731"}}
+{"timestamp":1714581954.9264784,"name":"online","context":{"idset":"6674"}}
+{"timestamp":1714581955.688555,"name":"online","context":{"idset":"6764"}}
+{"timestamp":1714581955.6925404,"name":"online","context":{"idset":"6669"}}
+{"timestamp":1714581955.6965034,"name":"online","context":{"idset":"6677,6689"}}
+{"timestamp":1714581955.7005346,"name":"online","context":{"idset":"6648"}}
+{"timestamp":1714581955.7045486,"name":"online","context":{"idset":"6645,6690,6744,6761"}}
+{"timestamp":1714581955.7085829,"name":"online","context":{"idset":"6696"}}
+{"timestamp":1714581955.7125366,"name":"online","context":{"idset":"6653"}}
+{"timestamp":1714581955.7345436,"name":"online","context":{"idset":"6753"}}
+{"timestamp":1714581955.7387805,"name":"online","context":{"idset":"6652"}}
+{"timestamp":1714581955.7430458,"name":"online","context":{"idset":"6726"}}
+{"timestamp":1714581955.7474682,"name":"online","context":{"idset":"6705,6762"}}
+{"timestamp":1714581955.8311732,"name":"online","context":{"idset":"6678"}}
+{"timestamp":1714581955.9889781,"name":"online","context":{"idset":"6668,6704,6771,7490"}}
+{"timestamp":1714581956.9194961,"name":"online","context":{"idset":"6724"}}
+{"timestamp":1714581956.923609,"name":"online","context":{"idset":"6711,6768,6770"}}
+{"timestamp":1714581957.6123621,"name":"online","context":{"idset":"6702,7420"}}
+{"timestamp":1714581957.6163528,"name":"online","context":{"idset":"6699,6721,6759,7484"}}
+{"timestamp":1714581957.6198509,"name":"online","context":{"idset":"6728,6763"}}
+{"timestamp":1714581957.6234403,"name":"online","context":{"idset":"6676"}}
+{"timestamp":1714581957.627116,"name":"online","context":{"idset":"6749"}}
+{"timestamp":1714581957.6306994,"name":"online","context":{"idset":"7499"}}
+{"timestamp":1714581957.6342392,"name":"online","context":{"idset":"7433"}}
+{"timestamp":1714581957.6534107,"name":"online","context":{"idset":"6725,6765,7491"}}
+{"timestamp":1714581957.6571269,"name":"online","context":{"idset":"6719,7426,7442,7458,7464,7482"}}
+{"timestamp":1714581957.660588,"name":"online","context":{"idset":"6698"}}
+{"timestamp":1714581957.6640916,"name":"online","context":{"idset":"7524"}}
+{"timestamp":1714581957.6676881,"name":"online","context":{"idset":"6649"}}
+{"timestamp":1714581957.6713119,"name":"online","context":{"idset":"6667,6680,6718,6723,6736,7441"}}
+{"timestamp":1714581957.6748443,"name":"online","context":{"idset":"6751,7447"}}
+{"timestamp":1714581957.6783793,"name":"online","context":{"idset":"7427,7523"}}
+{"timestamp":1714581957.8674505,"name":"online","context":{"idset":"6754,7416"}}
+{"timestamp":1714581958.0805464,"name":"online","context":{"idset":"6703"}}
+{"timestamp":1714581958.8940001,"name":"online","context":{"idset":"6713,7424,7489,7495,7517,7529"}}
+{"timestamp":1714581959.759901,"name":"online","context":{"idset":"6682,7461"}}
+{"timestamp":1714581959.7654929,"name":"online","context":{"idset":"6683,6691,6694,6748"}}
+{"timestamp":1714581959.7709064,"name":"online","context":{"idset":"6709,7479,7511"}}
+{"timestamp":1714581959.7759354,"name":"online","context":{"idset":"7452,7469"}}
+{"timestamp":1714581959.7809689,"name":"online","context":{"idset":"7513"}}
+{"timestamp":1714581959.7861116,"name":"online","context":{"idset":"6760"}}
+{"timestamp":1714581959.7916105,"name":"online","context":{"idset":"7456,7507,7518"}}
+{"timestamp":1714581959.7968323,"name":"online","context":{"idset":"6732,7508"}}
+{"timestamp":1714581959.8019042,"name":"online","context":{"idset":"7453"}}
+{"timestamp":1714581959.8072529,"name":"online","context":{"idset":"6722,7415,7462,7475"}}
+{"timestamp":1714581959.8236661,"name":"online","context":{"idset":"6710,7448,7454,7471,7500,7510,7532"}}
+{"timestamp":1714581959.8287373,"name":"online","context":{"idset":"7506"}}
+{"timestamp":1714581959.8343017,"name":"online","context":{"idset":"6730,6752,7497,7505"}}
+{"timestamp":1714581959.8394775,"name":"online","context":{"idset":"7468,7496,7527,7533"}}
+{"timestamp":1714581959.8451326,"name":"online","context":{"idset":"7418,7534,7537,7539"}}
+{"timestamp":1714581959.8508272,"name":"online","context":{"idset":"7487"}}
+{"timestamp":1714581959.8564742,"name":"online","context":{"idset":"6697"}}
+{"timestamp":1714581959.8617227,"name":"online","context":{"idset":"7449,7480"}}
+{"timestamp":1714581959.8668725,"name":"online","context":{"idset":"7472"}}
+{"timestamp":1714581960.0781839,"name":"online","context":{"idset":"6688,6706-6707,6729,6766,7476,7488,7501,7514"}}
+{"timestamp":1714581960.1832442,"name":"online","context":{"idset":"7535"}}
+{"timestamp":1714581961.65769,"name":"online","context":{"idset":"7428,7459,7494,7509,7515,7520"}}
+{"timestamp":1714581961.6622305,"name":"online","context":{"idset":"7434,7439,7463"}}
+{"timestamp":1714581961.6663806,"name":"online","context":{"idset":"6662,6741"}}
+{"timestamp":1714581961.6704803,"name":"online","context":{"idset":"6684,7478"}}
+{"timestamp":1714581961.6745591,"name":"online","context":{"idset":"7486"}}
+{"timestamp":1714581961.6787684,"name":"online","context":{"idset":"7413,7467"}}
+{"timestamp":1714581961.6829662,"name":"online","context":{"idset":"6675,6738,7465,7481"}}
+{"timestamp":1714581961.6870468,"name":"online","context":{"idset":"7473"}}
+{"timestamp":1714581961.6911476,"name":"online","context":{"idset":"7536"}}
+{"timestamp":1714581961.6953316,"name":"online","context":{"idset":"6700"}}
+{"timestamp":1714581961.6997287,"name":"online","context":{"idset":"6663,7522"}}
+{"timestamp":1714581961.7040718,"name":"online","context":{"idset":"6758"}}
+{"timestamp":1714581961.7084427,"name":"online","context":{"idset":"6727,7419,7498"}}
+{"timestamp":1714581961.7128251,"name":"online","context":{"idset":"6695,7414,7422,7445,7521"}}
+{"timestamp":1714581961.7172968,"name":"online","context":{"idset":"6685,7512"}}
+{"timestamp":1714581961.7218916,"name":"online","context":{"idset":"6660,7429,7450"}}
+{"timestamp":1714581961.726337,"name":"online","context":{"idset":"6757,7525-7526,7538"}}
+{"timestamp":1714581961.8233216,"name":"online","context":{"idset":"7531"}}
+{"timestamp":1714581962.0453005,"name":"online","context":{"idset":"6715,6755-6756,6772,7421,7425,7443,7466,7493"}}
+{"timestamp":1714581963.5338337,"name":"online","context":{"idset":"6693,6746,7417,7435,7444,7446,7457,7516,7519"}}
+{"timestamp":1714581963.5372732,"name":"online","context":{"idset":"6750,7423,7438,7470,7477,7483,7492,7504,7540"}}
+{"timestamp":1714581963.5404363,"name":"online","context":{"idset":"6769,7437,7440,7485"}}
+{"timestamp":1714581963.5435023,"name":"online","context":{"idset":"6743,7455,7503,7530"}}
+{"timestamp":1714581963.5466452,"name":"online","context":{"idset":"7436,7451,7474,7502,7528"}}
+{"timestamp":1714581963.5497828,"name":"online","context":{"idset":"7460"}}
+{"timestamp":1714581963.5528362,"name":"online","context":{"idset":"7430"}}
+{"timestamp":1714581969.9033816,"name":"drain","context":{"idset":"6645-6772","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714581970.0047717,"name":"drain","context":{"idset":"7413-7540","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714583189.4510682,"name":"online","context":{"idset":"967"}}
+{"timestamp":1714583189.4535971,"name":"online","context":{"idset":"968"}}
+{"timestamp":1714583380.2024529,"name":"offline","context":{"idset":"11534"}}
+{"timestamp":1714583537.6560185,"name":"offline","context":{"idset":"6647"}}
+{"timestamp":1714583537.6603734,"name":"offline","context":{"idset":"6648"}}
+{"timestamp":1714583891.1007371,"name":"online","context":{"idset":"873"}}
+{"timestamp":1714584286.4843509,"name":"drain","context":{"idset":"874","reason":"cxi error","overwrite":0}}
+{"timestamp":1714584502.0183342,"name":"online","context":{"idset":"7828"}}
+{"timestamp":1714584706.358285,"name":"online","context":{"idset":"7827"}}
+{"timestamp":1714585012.2103846,"name":"online","context":{"idset":"817"}}
+{"timestamp":1714585015.8030889,"name":"online","context":{"idset":"803"}}
+{"timestamp":1714585024.0779696,"name":"online","context":{"idset":"804"}}
+{"timestamp":1714585405.7014909,"name":"drain","context":{"idset":"817","reason":"--reason running hpe-diags Javier","overwrite":0}}
+{"timestamp":1714585635.7113812,"name":"online","context":{"idset":"11533-11534"}}
+{"timestamp":1714585997.7464306,"name":"drain","context":{"idset":"277-316","reason":"New Blade Install --JRG","overwrite":1}}
+{"timestamp":1714586029.7958953,"name":"offline","context":{"idset":"277"}}
+{"timestamp":1714586029.8292646,"name":"offline","context":{"idset":"285"}}
+{"timestamp":1714586029.9275455,"name":"offline","context":{"idset":"280"}}
+{"timestamp":1714586030.0455747,"name":"offline","context":{"idset":"284"}}
+{"timestamp":1714586030.0507586,"name":"offline","context":{"idset":"287"}}
+{"timestamp":1714586030.0595829,"name":"offline","context":{"idset":"278"}}
+{"timestamp":1714586030.0638702,"name":"offline","context":{"idset":"315"}}
+{"timestamp":1714586030.0683389,"name":"offline","context":{"idset":"312"}}
+{"timestamp":1714586030.0759192,"name":"offline","context":{"idset":"282"}}
+{"timestamp":1714586030.0837247,"name":"offline","context":{"idset":"283"}}
+{"timestamp":1714586030.0911808,"name":"offline","context":{"idset":"288"}}
+{"timestamp":1714586030.0955181,"name":"offline","context":{"idset":"289"}}
+{"timestamp":1714586030.1033375,"name":"offline","context":{"idset":"290"}}
+{"timestamp":1714586030.1881039,"name":"offline","context":{"idset":"291"}}
+{"timestamp":1714586030.1924219,"name":"offline","context":{"idset":"292"}}
+{"timestamp":1714586030.1968517,"name":"offline","context":{"idset":"293"}}
+{"timestamp":1714586030.2014222,"name":"offline","context":{"idset":"294"}}
+{"timestamp":1714586030.2057698,"name":"offline","context":{"idset":"296"}}
+{"timestamp":1714586030.2193289,"name":"offline","context":{"idset":"297"}}
+{"timestamp":1714586030.2238855,"name":"offline","context":{"idset":"295"}}
+{"timestamp":1714586030.2283101,"name":"offline","context":{"idset":"298"}}
+{"timestamp":1714586030.2325261,"name":"offline","context":{"idset":"299"}}
+{"timestamp":1714586030.2368801,"name":"offline","context":{"idset":"300"}}
+{"timestamp":1714586030.2411273,"name":"offline","context":{"idset":"301"}}
+{"timestamp":1714586030.2453561,"name":"offline","context":{"idset":"302"}}
+{"timestamp":1714586030.2495499,"name":"offline","context":{"idset":"303"}}
+{"timestamp":1714586030.2538705,"name":"offline","context":{"idset":"304"}}
+{"timestamp":1714586030.258275,"name":"offline","context":{"idset":"305"}}
+{"timestamp":1714586030.2625532,"name":"offline","context":{"idset":"306"}}
+{"timestamp":1714586030.2668967,"name":"offline","context":{"idset":"307"}}
+{"timestamp":1714586030.2711942,"name":"offline","context":{"idset":"308"}}
+{"timestamp":1714586030.2754633,"name":"offline","context":{"idset":"309"}}
+{"timestamp":1714586030.2797227,"name":"offline","context":{"idset":"310"}}
+{"timestamp":1714586030.2840619,"name":"offline","context":{"idset":"311"}}
+{"timestamp":1714586030.2883387,"name":"offline","context":{"idset":"316"}}
+{"timestamp":1714586179.6056342,"name":"online","context":{"idset":"10080"}}
+{"timestamp":1714586332.20836,"name":"offline","context":{"idset":"6750"}}
+{"timestamp":1714587044.2032025,"name":"offline","context":{"idset":"7427"}}
+{"timestamp":1714587808.2046902,"name":"offline","context":{"idset":"803"}}
+{"timestamp":1714588334.2043793,"name":"offline","context":{"idset":"873"}}
+{"timestamp":1714588621.5734096,"name":"online","context":{"idset":"945"}}
+{"timestamp":1714588621.5775418,"name":"online","context":{"idset":"438"}}
+{"timestamp":1714588658.8751903,"name":"online","context":{"idset":"439"}}
+{"timestamp":1714588658.8793497,"name":"online","context":{"idset":"437"}}
+{"timestamp":1714588661.5941219,"name":"online","context":{"idset":"442"}}
+{"timestamp":1714588661.5982447,"name":"online","context":{"idset":"443"}}
+{"timestamp":1714588665.5714753,"name":"online","context":{"idset":"440"}}
+{"timestamp":1714588671.805989,"name":"online","context":{"idset":"441"}}
+{"timestamp":1714588689.5522349,"name":"online","context":{"idset":"444"}}
+{"timestamp":1714588721.2026412,"name":"online","context":{"idset":"803"}}
+{"timestamp":1714589142.1293135,"name":"offline","context":{"idset":"11317"}}
+{"timestamp":1714589142.2049716,"name":"offline","context":{"idset":"11318"}}
+{"timestamp":1714589144.1237068,"name":"offline","context":{"idset":"11319"}}
+{"timestamp":1714589144.1269691,"name":"offline","context":{"idset":"11320"}}
+{"timestamp":1714589144.1302149,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1714589144.2279856,"name":"offline","context":{"idset":"11322"}}
+{"timestamp":1714589146.1265152,"name":"offline","context":{"idset":"11323"}}
+{"timestamp":1714589146.1307411,"name":"offline","context":{"idset":"11324"}}
+{"timestamp":1714589146.1351528,"name":"offline","context":{"idset":"11325"}}
+{"timestamp":1714589146.2252645,"name":"offline","context":{"idset":"11326"}}
+{"timestamp":1714589148.1357834,"name":"offline","context":{"idset":"11327"}}
+{"timestamp":1714589148.1405668,"name":"offline","context":{"idset":"11328"}}
+{"timestamp":1714589148.1446512,"name":"offline","context":{"idset":"11329"}}
+{"timestamp":1714589148.1489305,"name":"offline","context":{"idset":"11330"}}
+{"timestamp":1714589148.2074015,"name":"offline","context":{"idset":"11331"}}
+{"timestamp":1714589150.2040942,"name":"offline","context":{"idset":"11332"}}
+{"timestamp":1714589527.9983189,"name":"online","context":{"idset":"6647"}}
+{"timestamp":1714589529.8968403,"name":"online","context":{"idset":"6648"}}
+{"timestamp":1714589530.8627903,"name":"online","context":{"idset":"6750"}}
+{"timestamp":1714589550.7401831,"name":"online","context":{"idset":"7427,7432"}}
+{"timestamp":1714589552.3903136,"name":"online","context":{"idset":"7431"}}
+{"timestamp":1714589926.4025311,"name":"online","context":{"idset":"10134"}}
+{"timestamp":1714589927.3575504,"name":"online","context":{"idset":"9973"}}
+{"timestamp":1714589931.1822035,"name":"online","context":{"idset":"11790"}}
+{"timestamp":1714590163.0855794,"name":"online","context":{"idset":"10081"}}
+{"timestamp":1714590223.9261534,"name":"online","context":{"idset":"9982"}}
+{"timestamp":1714590223.9317276,"name":"online","context":{"idset":"9976"}}
+{"timestamp":1714590223.9369907,"name":"online","context":{"idset":"9999"}}
+{"timestamp":1714590223.9413416,"name":"online","context":{"idset":"9992,9994"}}
+{"timestamp":1714590223.9471025,"name":"online","context":{"idset":"9984"}}
+{"timestamp":1714590223.9523561,"name":"online","context":{"idset":"9980,10015"}}
+{"timestamp":1714590225.0483544,"name":"online","context":{"idset":"9986,9998,10010,10036"}}
+{"timestamp":1714590225.813168,"name":"online","context":{"idset":"9983"}}
+{"timestamp":1714590225.8157992,"name":"online","context":{"idset":"9974"}}
+{"timestamp":1714590225.8184669,"name":"online","context":{"idset":"9985"}}
+{"timestamp":1714590225.8212352,"name":"online","context":{"idset":"10012"}}
+{"timestamp":1714590225.8297443,"name":"online","context":{"idset":"9990"}}
+{"timestamp":1714590225.8325062,"name":"online","context":{"idset":"10033"}}
+{"timestamp":1714590225.9637663,"name":"online","context":{"idset":"9975"}}
+{"timestamp":1714590226.0585651,"name":"online","context":{"idset":"9995"}}
+{"timestamp":1714590226.598217,"name":"online","context":{"idset":"10011"}}
+{"timestamp":1714590226.6007352,"name":"online","context":{"idset":"9991"}}
+{"timestamp":1714590227.0476813,"name":"online","context":{"idset":"10018"}}
+{"timestamp":1714590227.0611885,"name":"online","context":{"idset":"10057"}}
+{"timestamp":1714590227.2509382,"name":"online","context":{"idset":"9981,9987,10038"}}
+{"timestamp":1714590227.6125367,"name":"online","context":{"idset":"10100"}}
+{"timestamp":1714590227.7419708,"name":"online","context":{"idset":"10002,10090"}}
+{"timestamp":1714590227.8842041,"name":"online","context":{"idset":"9996,10013,10075"}}
+{"timestamp":1714590228.0565267,"name":"online","context":{"idset":"10029"}}
+{"timestamp":1714590228.1640124,"name":"online","context":{"idset":"10056,10091"}}
+{"timestamp":1714590228.8085895,"name":"online","context":{"idset":"9979,10031,10060"}}
+{"timestamp":1714590228.8134673,"name":"online","context":{"idset":"9997,10026,10062"}}
+{"timestamp":1714590228.8181634,"name":"online","context":{"idset":"10001"}}
+{"timestamp":1714590228.8235464,"name":"online","context":{"idset":"9978,10006-10007,10024,10046,10051,10066,10079"}}
+{"timestamp":1714590228.8417108,"name":"online","context":{"idset":"10027,10064-10065"}}
+{"timestamp":1714590229.3910389,"name":"online","context":{"idset":"9988,9993,10003,10014,10023,10052,10068,10072-10073,10089,10093"}}
+{"timestamp":1714590229.3937848,"name":"online","context":{"idset":"10019,10021,10030,10082,10086,10096"}}
+{"timestamp":1714590229.4751277,"name":"online","context":{"idset":"10004,10017,10020,10035,10042,10045,10053-10054,10083,10099"}}
+{"timestamp":1714590229.7090168,"name":"online","context":{"idset":"9989,10000,10005,10037,10048,10050,10063,10069,10092"}}
+{"timestamp":1714590229.8248584,"name":"online","context":{"idset":"10016"}}
+{"timestamp":1714590230.0733311,"name":"online","context":{"idset":"10009,10041,10044,10047,10049,10058,10070-10071,10088,10094,10098"}}
+{"timestamp":1714590230.8754094,"name":"online","context":{"idset":"9977,10025,10028,10034,10039,10055,10076,10084-10085,10087"}}
+{"timestamp":1714590230.8783567,"name":"online","context":{"idset":"10022,10059,10097"}}
+{"timestamp":1714590230.8812914,"name":"online","context":{"idset":"10008,10040,10043,10061,10067,10077"}}
+{"timestamp":1714590231.4298222,"name":"online","context":{"idset":"10095"}}
+{"timestamp":1714590231.5473745,"name":"online","context":{"idset":"10487"}}
+{"timestamp":1714590234.0046253,"name":"online","context":{"idset":"10489,10491"}}
+{"timestamp":1714590235.8610907,"name":"online","context":{"idset":"10488,10490,10492"}}
+{"timestamp":1714590237.0452008,"name":"online","context":{"idset":"10493"}}
+{"timestamp":1714590237.881978,"name":"online","context":{"idset":"10496"}}
+{"timestamp":1714590237.8860323,"name":"online","context":{"idset":"10495,10497-10498"}}
+{"timestamp":1714590237.8983231,"name":"online","context":{"idset":"10494,10505"}}
+{"timestamp":1714590237.9023631,"name":"online","context":{"idset":"10501"}}
+{"timestamp":1714590237.9062326,"name":"online","context":{"idset":"10503"}}
+{"timestamp":1714590237.9102135,"name":"online","context":{"idset":"10507"}}
+{"timestamp":1714590237.914062,"name":"online","context":{"idset":"10502,10508"}}
+{"timestamp":1714590238.1915781,"name":"online","context":{"idset":"10506"}}
+{"timestamp":1714590239.5772469,"name":"online","context":{"idset":"10511"}}
+{"timestamp":1714590239.5814264,"name":"online","context":{"idset":"10510"}}
+{"timestamp":1714590239.585654,"name":"online","context":{"idset":"10512,10514"}}
+{"timestamp":1714590239.5898173,"name":"online","context":{"idset":"10517"}}
+{"timestamp":1714590239.5940177,"name":"online","context":{"idset":"10509,10513"}}
+{"timestamp":1714590239.685277,"name":"online","context":{"idset":"10516"}}
+{"timestamp":1714590239.8743787,"name":"online","context":{"idset":"10519"}}
+{"timestamp":1714590240.2211108,"name":"online","context":{"idset":"10518,10520"}}
+{"timestamp":1714590241.610065,"name":"online","context":{"idset":"10515"}}
+{"timestamp":1714590241.6149919,"name":"online","context":{"idset":"10534"}}
+{"timestamp":1714590241.6192963,"name":"online","context":{"idset":"10529"}}
+{"timestamp":1714590241.6240342,"name":"online","context":{"idset":"10525"}}
+{"timestamp":1714590241.6283741,"name":"online","context":{"idset":"10524"}}
+{"timestamp":1714590241.6326938,"name":"online","context":{"idset":"10521,10523,10528"}}
+{"timestamp":1714590241.6368601,"name":"online","context":{"idset":"10527"}}
+{"timestamp":1714590241.6412821,"name":"online","context":{"idset":"10530,10536"}}
+{"timestamp":1714590241.645649,"name":"online","context":{"idset":"10522,10526,10531"}}
+{"timestamp":1714590241.6500328,"name":"online","context":{"idset":"10532-10533"}}
+{"timestamp":1714590241.6545041,"name":"online","context":{"idset":"10544"}}
+{"timestamp":1714590241.6588237,"name":"online","context":{"idset":"10540"}}
+{"timestamp":1714590241.6632187,"name":"online","context":{"idset":"10538"}}
+{"timestamp":1714590241.6677423,"name":"online","context":{"idset":"10541,10549"}}
+{"timestamp":1714590241.8892102,"name":"online","context":{"idset":"10545,10547"}}
+{"timestamp":1714590242.0505097,"name":"online","context":{"idset":"10537"}}
+{"timestamp":1714590242.205972,"name":"drain","context":{"idset":"10611","reason":"broker was unresponsive"}}
+{"timestamp":1714590242.2101247,"name":"online","context":{"idset":"10543,10552"}}
+{"timestamp":1714590243.5679457,"name":"online","context":{"idset":"10535,10542"}}
+{"timestamp":1714590243.572531,"name":"online","context":{"idset":"10548,10551,10565"}}
+{"timestamp":1714590243.5769384,"name":"online","context":{"idset":"10546,10554,10558-10559,10562-10563"}}
+{"timestamp":1714590243.5812993,"name":"online","context":{"idset":"10555,10560"}}
+{"timestamp":1714590243.585639,"name":"online","context":{"idset":"10550,10553,10556-10557,10566-10567"}}
+{"timestamp":1714590243.589941,"name":"online","context":{"idset":"10568"}}
+{"timestamp":1714590243.5943699,"name":"online","context":{"idset":"10571,10576"}}
+{"timestamp":1714590243.5989759,"name":"online","context":{"idset":"10564,10575,10578"}}
+{"timestamp":1714590243.6037459,"name":"online","context":{"idset":"10569,10573"}}
+{"timestamp":1714590243.6085353,"name":"online","context":{"idset":"10579-10580"}}
+{"timestamp":1714590243.8718719,"name":"online","context":{"idset":"10570,10574,10577,10582-10583,10586"}}
+{"timestamp":1714590243.9767957,"name":"online","context":{"idset":"10593"}}
+{"timestamp":1714590244.1161437,"name":"online","context":{"idset":"10581,10584-10585,10590-10591,10595,10609"}}
+{"timestamp":1714590244.8707023,"name":"online","context":{"idset":"10561,10600,10607-10608"}}
+{"timestamp":1714590244.8749454,"name":"online","context":{"idset":"10587-10588,10592,10604"}}
+{"timestamp":1714590244.8791676,"name":"online","context":{"idset":"10594,10596-10599,10601,10605,10610-10611"}}
+{"timestamp":1714590244.8833702,"name":"online","context":{"idset":"10572,10589,10603,10606,10612"}}
+{"timestamp":1714590244.8931222,"name":"online","context":{"idset":"10602"}}
+{"timestamp":1714590306.059834,"name":"undrain","context":{"idset":"10611"}}
+{"timestamp":1714590472.1174548,"name":"drain","context":{"idset":"10032,10074,10078,10485-10486,10499-10500,10504,10539","overwrite":0}}
+{"timestamp":1714590760.3218529,"name":"online","context":{"idset":"8831"}}
+{"timestamp":1714590761.7057517,"name":"online","context":{"idset":"8847"}}
+{"timestamp":1714590763.8322797,"name":"online","context":{"idset":"8837"}}
+{"timestamp":1714590763.8378222,"name":"online","context":{"idset":"8879"}}
+{"timestamp":1714590766.6267338,"name":"online","context":{"idset":"8855,8870"}}
+{"timestamp":1714590766.6317897,"name":"online","context":{"idset":"8876"}}
+{"timestamp":1714590766.6364431,"name":"online","context":{"idset":"8857"}}
+{"timestamp":1714590767.5879138,"name":"online","context":{"idset":"8860"}}
+{"timestamp":1714590767.608243,"name":"online","context":{"idset":"8842"}}
+{"timestamp":1714590769.1806571,"name":"online","context":{"idset":"8822"}}
+{"timestamp":1714590770.0051932,"name":"online","context":{"idset":"8824"}}
+{"timestamp":1714590771.4523234,"name":"online","context":{"idset":"8896"}}
+{"timestamp":1714590771.455195,"name":"online","context":{"idset":"8833"}}
+{"timestamp":1714590771.6440985,"name":"online","context":{"idset":"8827"}}
+{"timestamp":1714590772.0076518,"name":"online","context":{"idset":"8840,8844,8858"}}
+{"timestamp":1714590772.2221866,"name":"online","context":{"idset":"8904"}}
+{"timestamp":1714590773.7112851,"name":"online","context":{"idset":"8936"}}
+{"timestamp":1714590773.7142785,"name":"online","context":{"idset":"8899"}}
+{"timestamp":1714590773.7173138,"name":"online","context":{"idset":"8838"}}
+{"timestamp":1714590773.9386258,"name":"online","context":{"idset":"8836,8918"}}
+{"timestamp":1714590775.0421677,"name":"online","context":{"idset":"8864"}}
+{"timestamp":1714590775.0497358,"name":"online","context":{"idset":"8868"}}
+{"timestamp":1714590775.057409,"name":"online","context":{"idset":"8826,8830"}}
+{"timestamp":1714590776.0443022,"name":"online","context":{"idset":"8919"}}
+{"timestamp":1714590776.0504856,"name":"online","context":{"idset":"8839"}}
+{"timestamp":1714590776.0572202,"name":"online","context":{"idset":"8835"}}
+{"timestamp":1714590776.0643713,"name":"online","context":{"idset":"8889"}}
+{"timestamp":1714590776.0699799,"name":"online","context":{"idset":"8851"}}
+{"timestamp":1714590776.1652179,"name":"online","context":{"idset":"8929"}}
+{"timestamp":1714590777.0653007,"name":"online","context":{"idset":"8828-8829,8930,8940"}}
+{"timestamp":1714590777.0697393,"name":"online","context":{"idset":"8843"}}
+{"timestamp":1714590777.0737839,"name":"online","context":{"idset":"8935"}}
+{"timestamp":1714590777.0777676,"name":"online","context":{"idset":"8845"}}
+{"timestamp":1714590777.0819662,"name":"online","context":{"idset":"8856,8859"}}
+{"timestamp":1714590777.0864334,"name":"online","context":{"idset":"8823,8878,8897"}}
+{"timestamp":1714590777.0904734,"name":"online","context":{"idset":"8821"}}
+{"timestamp":1714590777.2177861,"name":"online","context":{"idset":"8915"}}
+{"timestamp":1714590777.8310382,"name":"online","context":{"idset":"8825,8848,8863,8873,8883,8895,8923"}}
+{"timestamp":1714590777.8343785,"name":"online","context":{"idset":"8908"}}
+{"timestamp":1714590777.8375547,"name":"online","context":{"idset":"8907,8932"}}
+{"timestamp":1714590777.8407211,"name":"online","context":{"idset":"8846,8875,8890,8948"}}
+{"timestamp":1714590777.8440158,"name":"online","context":{"idset":"8832,8841,8853,8867,8877,8881,8885"}}
+{"timestamp":1714590777.9421957,"name":"online","context":{"idset":"8861,8869,8872,8939"}}
+{"timestamp":1714590778.1902359,"name":"online","context":{"idset":"8849,8891,8902,8905,8945"}}
+{"timestamp":1714590779.7798469,"name":"online","context":{"idset":"8834,8850,8852,8862,8865,8880,8916,8926"}}
+{"timestamp":1714590779.7839873,"name":"online","context":{"idset":"8887,8893,8900,8917,8921-8922"}}
+{"timestamp":1714590779.7877324,"name":"online","context":{"idset":"8854,8866,8871,8886,8892,8898,8901,8913,8931,8941"}}
+{"timestamp":1714590779.7915771,"name":"online","context":{"idset":"8874,8903,8906"}}
+{"timestamp":1714590779.7956123,"name":"online","context":{"idset":"8884,8909,8944"}}
+{"timestamp":1714590779.7991354,"name":"online","context":{"idset":"8894,8924-8925"}}
+{"timestamp":1714590779.8028276,"name":"online","context":{"idset":"8910-8911,8920"}}
+{"timestamp":1714590779.8062515,"name":"online","context":{"idset":"8882,8888,8914,8927-8928,8937-8938,8942,8947"}}
+{"timestamp":1714590779.8097146,"name":"online","context":{"idset":"8912,8934"}}
+{"timestamp":1714590779.8131843,"name":"online","context":{"idset":"8933,8946"}}
+{"timestamp":1714590781.7038403,"name":"online","context":{"idset":"8943"}}
+{"timestamp":1714590799.529567,"name":"online","context":{"idset":"878"}}
+{"timestamp":1714590799.5902302,"name":"online","context":{"idset":"877"}}
+{"timestamp":1714591036.8079774,"name":"offline","context":{"idset":"10073"}}
+{"timestamp":1714591823.556107,"name":"online","context":{"idset":"11286-11287"}}
+{"timestamp":1714591823.895329,"name":"online","context":{"idset":"11288"}}
+{"timestamp":1714591824.131731,"name":"online","context":{"idset":"11285"}}
+{"timestamp":1714591958.9996312,"name":"undrain","context":{"idset":"11291"}}
+{"timestamp":1714593793.1065211,"name":"offline","context":{"idset":"7735"}}
+{"timestamp":1714593793.2056868,"name":"offline","context":{"idset":"7741"}}
+{"timestamp":1714593793.3300309,"name":"offline","context":{"idset":"7745"}}
+{"timestamp":1714593793.3367746,"name":"offline","context":{"idset":"7737"}}
+{"timestamp":1714593793.3424745,"name":"offline","context":{"idset":"7747"}}
+{"timestamp":1714593793.3476658,"name":"offline","context":{"idset":"7743"}}
+{"timestamp":1714593793.4462945,"name":"offline","context":{"idset":"7733"}}
+{"timestamp":1714593904.0346563,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1714593904.979152,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1714593905.4313214,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1714593905.5378401,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1714593905.808995,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1714593907.0210662,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1714593907.0269825,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1714593907.7538223,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1714593909.9645247,"name":"online","context":{"idset":"7271"}}
+{"timestamp":1714593910.2983389,"name":"online","context":{"idset":"7269-7270,7272"}}
+{"timestamp":1714594223.0030301,"name":"offline","context":{"idset":"7126"}}
+{"timestamp":1714594268.86533,"name":"offline","context":{"idset":"7093"}}
+{"timestamp":1714594609.0124314,"name":"drain","context":{"idset":"11508","reason":"nodediag failed amdapu","overwrite":0}}
+{"timestamp":1714595601.6148264,"name":"online","context":{"idset":"10073,10499"}}
+{"timestamp":1714595601.7179189,"name":"online","context":{"idset":"10486"}}
+{"timestamp":1714595601.9634788,"name":"online","context":{"idset":"10078,10500,10539"}}
+{"timestamp":1714595602.5920742,"name":"online","context":{"idset":"10074,10485"}}
+{"timestamp":1714595602.5947835,"name":"online","context":{"idset":"10032"}}
+{"timestamp":1714595805.9571946,"name":"offline","context":{"idset":"945"}}
+{"timestamp":1714595913.3383367,"name":"online","context":{"idset":"942"}}
+{"timestamp":1714595917.0294113,"name":"undrain","context":{"idset":"10032,10074,10078,10485-10486,10499-10500,10504,10539"}}
+{"timestamp":1714596403.0499501,"name":"online","context":{"idset":"548"}}
+{"timestamp":1714596684.2129147,"name":"offline","context":{"idset":"10001"}}
+{"timestamp":1714596704.2975295,"name":"offline","context":{"idset":"871"}}
+{"timestamp":1714597077.2364023,"name":"online","context":{"idset":"437"}}
+{"timestamp":1714597079.0230646,"name":"online","context":{"idset":"439"}}
+{"timestamp":1714597099.0582275,"name":"online","context":{"idset":"444"}}
+{"timestamp":1714597106.9107285,"name":"online","context":{"idset":"443"}}
+{"timestamp":1714597108.2036891,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1714597111.1099336,"name":"online","context":{"idset":"438"}}
+{"timestamp":1714597129.2269278,"name":"online","context":{"idset":"441"}}
+{"timestamp":1714597130.8840005,"name":"online","context":{"idset":"442"}}
+{"timestamp":1714597134.8970125,"name":"online","context":{"idset":"440"}}
+{"timestamp":1714597340.20894,"name":"offline","context":{"idset":"11569"}}
+{"timestamp":1714598115.2853825,"name":"offline","context":{"idset":"11501"}}
+{"timestamp":1714598117.5447392,"name":"offline","context":{"idset":"11502"}}
+{"timestamp":1714598151.1786995,"name":"offline","context":{"idset":"779"}}
+{"timestamp":1714598153.2636058,"name":"offline","context":{"idset":"780"}}
+{"timestamp":1714598606.2286642,"name":"offline","context":{"idset":"10037"}}
+{"timestamp":1714598745.8718381,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1714598825.5967255,"name":"online","context":{"idset":"9139"}}
+{"timestamp":1714598952.1399074,"name":"offline","context":{"idset":"437"}}
+{"timestamp":1714598952.1443806,"name":"offline","context":{"idset":"438"}}
+{"timestamp":1714598952.1487837,"name":"offline","context":{"idset":"439"}}
+{"timestamp":1714598952.1531858,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1714598952.1575761,"name":"offline","context":{"idset":"441"}}
+{"timestamp":1714598952.1619699,"name":"offline","context":{"idset":"442"}}
+{"timestamp":1714598952.1664023,"name":"offline","context":{"idset":"443"}}
+{"timestamp":1714598952.2036498,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1714599233.7277172,"name":"undrain","context":{"idset":"9003"}}
+{"timestamp":1714599281.9024541,"name":"online","context":{"idset":"8987-8988,9004"}}
+{"timestamp":1714599282.1980755,"name":"online","context":{"idset":"10037"}}
+{"timestamp":1714599288.8675869,"name":"online","context":{"idset":"9062"}}
+{"timestamp":1714599293.092293,"name":"online","context":{"idset":"9068"}}
+{"timestamp":1714599293.0963526,"name":"online","context":{"idset":"9063,9066-9067"}}
+{"timestamp":1714599293.3991113,"name":"online","context":{"idset":"9076"}}
+{"timestamp":1714599293.6986022,"name":"online","context":{"idset":"9061,9064"}}
+{"timestamp":1714599294.0943494,"name":"online","context":{"idset":"9065"}}
+{"timestamp":1714599294.8959231,"name":"online","context":{"idset":"9075"}}
+{"timestamp":1714599301.0490453,"name":"online","context":{"idset":"9037"}}
+{"timestamp":1714599301.0536675,"name":"online","context":{"idset":"9044"}}
+{"timestamp":1714599301.0583055,"name":"online","context":{"idset":"9042"}}
+{"timestamp":1714599301.3276687,"name":"online","context":{"idset":"9029"}}
+{"timestamp":1714599301.919606,"name":"online","context":{"idset":"9031,9038"}}
+{"timestamp":1714599302.8624349,"name":"online","context":{"idset":"9035"}}
+{"timestamp":1714599302.866878,"name":"online","context":{"idset":"9030,9032-9033"}}
+{"timestamp":1714599302.8712578,"name":"online","context":{"idset":"9040"}}
+{"timestamp":1714599302.8755622,"name":"online","context":{"idset":"9043"}}
+{"timestamp":1714599302.9518635,"name":"online","context":{"idset":"9039"}}
+{"timestamp":1714599303.1677663,"name":"online","context":{"idset":"9034"}}
+{"timestamp":1714599303.9972565,"name":"online","context":{"idset":"9041"}}
+{"timestamp":1714599305.1665082,"name":"online","context":{"idset":"9036"}}
+{"timestamp":1714599372.9169874,"name":"online","context":{"idset":"9003"}}
+{"timestamp":1714599380.9634502,"name":"offline","context":{"idset":"833"}}
+{"timestamp":1714599380.9679482,"name":"offline","context":{"idset":"834"}}
+{"timestamp":1714599650.7014966,"name":"offline","context":{"idset":"10080"}}
+{"timestamp":1714600220.3913291,"name":"online","context":{"idset":"7273-7274"}}
+{"timestamp":1714600221.0741198,"name":"online","context":{"idset":"7280"}}
+{"timestamp":1714600221.0783286,"name":"online","context":{"idset":"7281"}}
+{"timestamp":1714600221.1147549,"name":"online","context":{"idset":"7284"}}
+{"timestamp":1714600221.6830049,"name":"online","context":{"idset":"7276,7279,7282-7283"}}
+{"timestamp":1714600221.9484773,"name":"online","context":{"idset":"7275,7278"}}
+{"timestamp":1714600223.3694212,"name":"online","context":{"idset":"7277"}}
+{"timestamp":1714600225.1469584,"name":"drain","context":{"idset":"10503-10504","reason":"-h","overwrite":0}}
+{"timestamp":1714600260.2647641,"name":"drain","context":{"idset":"10503-10504","reason":"pending work on fake1748 -kk","overwrite":1}}
+{"timestamp":1714600277.6790748,"name":"drain","context":{"idset":"10001-10002","reason":"pending work on fake10245 -kk","overwrite":0}}
+{"timestamp":1714600353.3129895,"name":"online","context":{"idset":"10080"}}
+{"timestamp":1714600680.2156401,"name":"offline","context":{"idset":"10085"}}
+{"timestamp":1714600760.2106111,"name":"offline","context":{"idset":"11570"}}
+{"timestamp":1714600795.03023,"name":"offline","context":{"idset":"893"}}
+{"timestamp":1714600795.0358074,"name":"offline","context":{"idset":"894"}}
+{"timestamp":1714601369.6288371,"name":"online","context":{"idset":"10085"}}
+{"timestamp":1714602215.0497646,"name":"online","context":{"idset":"11317"}}
+{"timestamp":1714602221.2169182,"name":"online","context":{"idset":"11323"}}
+{"timestamp":1714602221.4861219,"name":"online","context":{"idset":"11320"}}
+{"timestamp":1714602223.0251248,"name":"online","context":{"idset":"11326"}}
+{"timestamp":1714602223.5070961,"name":"online","context":{"idset":"11327,11330"}}
+{"timestamp":1714602223.9281352,"name":"online","context":{"idset":"11324"}}
+{"timestamp":1714602225.443223,"name":"online","context":{"idset":"11332"}}
+{"timestamp":1714602225.4503808,"name":"online","context":{"idset":"11329"}}
+{"timestamp":1714602225.4572797,"name":"online","context":{"idset":"11318-11319,11325,11331"}}
+{"timestamp":1714602225.4640446,"name":"online","context":{"idset":"11328"}}
+{"timestamp":1714602225.5723333,"name":"online","context":{"idset":"11321-11322"}}
+{"timestamp":1714603176.2054377,"name":"offline","context":{"idset":"10606"}}
+{"timestamp":1714603421.2726254,"name":"offline","context":{"idset":"10002"}}
+{"timestamp":1714604049.6876762,"name":"drain","context":{"idset":"325-334,337-341,343-347","reason":"New Blade Installation --JRG","overwrite":0}}
+{"timestamp":1714604341.3152301,"name":"offline","context":{"idset":"326"}}
+{"timestamp":1714604341.3238955,"name":"offline","context":{"idset":"327"}}
+{"timestamp":1714604341.3324046,"name":"offline","context":{"idset":"330"}}
+{"timestamp":1714604341.3410282,"name":"offline","context":{"idset":"332"}}
+{"timestamp":1714604341.3497002,"name":"offline","context":{"idset":"333"}}
+{"timestamp":1714604341.3584473,"name":"offline","context":{"idset":"344"}}
+{"timestamp":1714604341.3671913,"name":"offline","context":{"idset":"339"}}
+{"timestamp":1714604341.3758497,"name":"offline","context":{"idset":"337"}}
+{"timestamp":1714604341.3844702,"name":"offline","context":{"idset":"334"}}
+{"timestamp":1714604341.3939054,"name":"offline","context":{"idset":"325"}}
+{"timestamp":1714604341.4026992,"name":"offline","context":{"idset":"346"}}
+{"timestamp":1714604341.4114749,"name":"offline","context":{"idset":"343"}}
+{"timestamp":1714604341.4202399,"name":"offline","context":{"idset":"331"}}
+{"timestamp":1714604341.4310904,"name":"offline","context":{"idset":"328"}}
+{"timestamp":1714604341.4418533,"name":"offline","context":{"idset":"338"}}
+{"timestamp":1714604341.4523449,"name":"offline","context":{"idset":"347"}}
+{"timestamp":1714604341.4628296,"name":"offline","context":{"idset":"341"}}
+{"timestamp":1714604341.4732635,"name":"offline","context":{"idset":"340"}}
+{"timestamp":1714604341.4835043,"name":"offline","context":{"idset":"345"}}
+{"timestamp":1714604341.493794,"name":"offline","context":{"idset":"329"}}
+{"timestamp":1714604455.0271163,"name":"offline","context":{"idset":"10552"}}
+{"timestamp":1714604518.177856,"name":"online","context":{"idset":"798"}}
+{"timestamp":1714604518.915381,"name":"online","context":{"idset":"797"}}
+{"timestamp":1714604585.3154752,"name":"undrain","context":{"idset":"797-798"}}
+{"timestamp":1714605355.7173564,"name":"resource-init","context":{"restart":true,"drain":{"897-898":{"timestamp":1714426321.0,"reason":"--reason Replacing SIVOC on b0"},"481":{"timestamp":1712939015.0,"reason":"nodediag failed cxi"},"543":{"timestamp":1714074108.0,"reason":"nodediag failed cxi"},"493-500,502-505,507":{"timestamp":1712876380.0,"reason":"New blade install --JRG"},"565,567,569,571-572":{"timestamp":1713190129.0,"reason":"New Blade insallation"},"501":{"timestamp":1712864871.0,"reason":"ASSERT_EFI_ERROR on boot"},"566,568":{"timestamp":1713190153.0,"reason":"New Blade insallation"},"325-334,337-341,343-347":{"timestamp":1714604050.0,"reason":"New Blade Installation --JRG"},"878":{"timestamp":1714511202.0,"reason":"CPER error"},"372":{"timestamp":1713981581.0,"reason":"New Blade Installation --JRG"},"336,491":{"timestamp":1713230524.0,"reason":"Down"},"6645-6772,7413-7540":{"timestamp":1714581970.0,"reason":"--reason draining to run on-node diags - wendy"},"929-930":{"timestamp":1713552018.0,"reason":""},"773-774":{"timestamp":1713802805.0,"reason":""},"77-84":{"timestamp":1713795613.0,"reason":"New blade installation"},"477":{"timestamp":1712939038.0,"reason":"nodediag failed cxi"},"508":{"timestamp":1712864933.0,"reason":"Drops to UEFI on boot"},"581-588":{"timestamp":1713191262.0,"reason":"New Blade Installation --JRG"},"117":{"timestamp":1713981538.0,"reason":"Unreachable"},"360":{"timestamp":1713981579.0,"reason":"New Blade Installation --JRG"},"807":{"timestamp":1713376017.0,"reason":""},"854":{"timestamp":1712945395.0,"reason":"Debugging downed nodes - vls"},"517-524":{"timestamp":1713305181.0,"reason":"New Blade Installation --JRG"},"277-285,287-316":{"timestamp":1714585998.0,"reason":"New Blade Install --JRG"},"811-812":{"timestamp":1713557753.0,"reason":"--reason Running hpe diags - JM"},"11508":{"timestamp":1714594609.0,"reason":"nodediag failed amdapu"},"762":{"timestamp":1712862033.0,"reason":"nodediag failed amdapu dgemm_perf"},"405-409,412-420":{"timestamp":1714574192.0,"reason":"New Blade Install --JRG"},"506":{"timestamp":1712783164.0,"reason":"Rabbit crashed due to node bringup --JRG"},"881":{"timestamp":1714153692.0,"reason":"--reason H/W troubleshoot"},"65":{"timestamp":1712864426.0,"reason":"NC unresponsive"},"971-974":{"timestamp":1714143211.0,"reason":""},"570":{"timestamp":1712864993.0,"reason":"ASSERT_EFI_ERROR on boot"},"410":{"timestamp":1713378635.0,"reason":"New Blade Install --JRG"},"478":{"timestamp":1712939034.0,"reason":"nodediag failed cxi"},"544":{"timestamp":1714069881.0,"reason":"nodediag failed cxi"},"877":{"timestamp":1714511214.0,"reason":"Bad Partner Node"},"509-516":{"timestamp":1712849612.0,"reason":"New Blade Installation"},"895-896":{"timestamp":1714428105.0,"reason":"--reason Troubleshoot SIVOC on b0"},"206":{"timestamp":1713564985.0,"reason":"Unreachable"},"882":{"timestamp":1714429325.0,"reason":"--reason H/W troubleshoot"},"349-359,361-371,374-396,398-404":{"timestamp":1714060395.0,"reason":"New Blade Installation --JRG"},"545":{"timestamp":1714069878.0,"reason":"nodediag failed cxi"},"533-540":{"timestamp":1713302819.0,"reason":"New Blade Install --JRG"},"141-148":{"timestamp":1713545597.0,"reason":"ARP Storm Problem testing"},"348":{"timestamp":1711576490.0,"reason":"pci: 0000:03:00.1 width x8, expected x16"},"874":{"timestamp":1714584286.0,"reason":"cxi error"},"542":{"timestamp":1714074109.0,"reason":"nodediag failed cxi"},"817":{"timestamp":1714585406.0,"reason":"--reason running hpe-diags Javier"},"411":{"timestamp":1713564989.0,"reason":"New Blade Install --JRG"},"578,580":{"timestamp":1712858384.0,"reason":"New Blade Install --JRG"},"121":{"timestamp":1710136167.0,"reason":"nodediag failed pci"},"149-156":{"timestamp":1713545634.0,"reason":"ARP Storm Problem testing"},"165":{"timestamp":1713981434.0,"reason":"broker was unresponsive"},"949-950":{"timestamp":1712854277.0,"reason":"broker was unresponsive"},"172":{"timestamp":1713981544.0,"reason":"Unreachable"},"373":{"timestamp":1713981698.0,"reason":"New Blade Installation --JRG"},"541":{"timestamp":1714074106.0,"reason":"nodediag failed cxi"},"69":{"timestamp":1713300454.0,"reason":"New Blade Installation"},"217":{"timestamp":1712864756.0,"reason":"Drops to UEFI on boot"},"787-788":{"timestamp":1713219302.0,"reason":""},"479":{"timestamp":1712939057.0,"reason":"nodediag failed cxi"},"85-89,91-92":{"timestamp":1714079045.0,"reason":"New Blade installation"},"925-926":{"timestamp":1713540640.0,"reason":""},"461-476":{"timestamp":1713794797.0,"reason":"New Blade Installation --JRG"},"397":{"timestamp":1713229804.0,"reason":"New Blade Installation --JRG"},"421-460":{"timestamp":1713794410.0,"reason":"New Blade Installation --JRG"},"286":{"timestamp":1713981553.0,"reason":"New Blade Install --JRG"},"10503-10504":{"timestamp":1714600225.0,"reason":"pending work on fake1748 -kk"},"120":{"timestamp":1713981548.0,"reason":"Unreachable"},"803-804":{"timestamp":1713479733.0,"reason":"status"},"789-792":{"timestamp":1711461777.0,"reason":""},"629-636":{"timestamp":1712696364.0,"reason":"New Blade Installation --JRG"},"482-490,492":{"timestamp":1713915346.0,"reason":"New Blade Installation --JRG"},"342":{"timestamp":1713981566.0,"reason":"Unreachable"},"923-924":{"timestamp":1714064541.0,"reason":"status"},"557-564":{"timestamp":1713309221.0,"reason":"New blade installation -KK"},"546":{"timestamp":1714069873.0,"reason":"nodediag failed cxi pci"},"549-556":{"timestamp":1713553592.0,"reason":"New Blade Installation"},"90":{"timestamp":1713564981.0,"reason":"New Blade installation"},"771-772":{"timestamp":1713195979.0,"reason":""},"480":{"timestamp":1712939042.0,"reason":"nodediag failed cxi"},"253-276":{"timestamp":1712864539.0,"reason":"Leak investigation"},"547":{"timestamp":1713563916.0,"reason":"epilog failed for jobid fqUWmieM3AX"},"814":{"timestamp":1712783332.0,"reason":""},"10001-10002":{"timestamp":1714600278.0,"reason":"pending work on fake10245 -kk"},"133-140":{"timestamp":1713545617.0,"reason":"ARP Storm Problem testing"},"829-830":{"timestamp":1712700945.0,"reason":""},"525-532":{"timestamp":1713290620.0,"reason":""},"335":{"timestamp":1713981557.0,"reason":"Unreachable"},"934":{"timestamp":1713967835.0,"reason":"epilog failed for jobid frMYzSv7G6b"}},"online":"","exclude":"0,883-884"}}
+{"timestamp":1714605355.7412977,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1714605380.16872,"name":"online","context":{"idset":"0"}}
+{"timestamp":1714605382.0017204,"name":"online","context":{"idset":"1182,1205,7465,7501,7769,8085,11508"}}
+{"timestamp":1714605382.0036581,"name":"online","context":{"idset":"7828"}}
+{"timestamp":1714605382.0129321,"name":"online","context":{"idset":"1228,7699,7791,8143,8900,8941,9198"}}
+{"timestamp":1714605382.0292869,"name":"online","context":{"idset":"7084,7110,7116,7677,7749,8930,9119"}}
+{"timestamp":1714605382.0419433,"name":"online","context":{"idset":"9121"}}
+{"timestamp":1714605382.0557606,"name":"online","context":{"idset":"914,952,7115,7153,7689,8018"}}
+{"timestamp":1714605382.1798134,"name":"online","context":{"idset":"7049,7117,7489,7704,9197"}}
+{"timestamp":1714605382.1815605,"name":"online","context":{"idset":"962,7009,7024,7039,7079,7685,7714,8013,9061,9143,10679,11318"}}
+{"timestamp":1714605382.1825533,"name":"online","context":{"idset":"7025,7075,7732"}}
+{"timestamp":1714605382.2042084,"name":"online","context":{"idset":"817,937-938,1174,1178,1233,6684,6699,6714,6717,6719,6729,6731,6734,6743,7002,7010,7033,7035-7036,7044,7047,7056,7058,7087-7088,7090,7096,7107,7127-7128,7131,7133,7135-7136,7139,7142,7154,7420,7438,7444,7452,7454-7455,7458,7460,7480,7488,7491-7492,7500,7502,7508,7515-7516,7525,7527,7538-7539,7675-7676,7687,7693,7718,7775,7927,7934,7953,7960,7964,7983,7993,8001,8004-8005,8014,8017,8023,8047,8065,8075,8079,8088,8100,8124,8139,8160,8834,8843,8845,8861,8875,8883,8897,8928,8965,9062-9063,9086,9108,9112,9123,9130,9141,9149,9169,10620-10621,10623,10656,10665,10670,10682,10693,10696,10712,10714,10736,11276,11495"}}
+{"timestamp":1714605382.2872243,"name":"online","context":{"idset":"1219,6704-6705,6997,7008,7072,7424,7436,7442,7464,7466,7478,7482,7520,7522,7671,7686,7932,7994,8129,8133,8137,8844,8847,8849,8856,8867,8934,8938,8940,8971,9064,9134,9147,10553,10697,11273,11319"}}
+{"timestamp":1714605382.3518469,"name":"online","context":{"idset":"872"}}
+{"timestamp":1714605382.3905547,"name":"online","context":{"idset":"6647,7278-7279,7468,7477,7518,7680-7681,7734,7771,7779,7830,7957,8089,8091,8125,8150,8350,8710,8744,8884,8959,8963,9032,9039,9049-9050,9155,9176,9185,9188,9403,9415,9481,10253,10664,10916,10983,11275"}}
+{"timestamp":1714605382.5298252,"name":"online","context":{"idset":"6539,6564,6572,6582,6597,6633,6649,6689,6763,7023,7032,7038,7061,7071,7109,7150,7210,7272,7275,7280,7288,7310,7331,7389,7408,7493,7755,7799,7826,7864,7909,7951,7973,8011,8016,8024,8050,8070,8111,8114,8367,8405,8436,8572,8593,8601,8606,8722,8725,8785,8885,8956,9054,9110,9168,9209,9272,9302,9307,9360,9376,9408,9506,9512,9849,10308,10312,10517,10876,10896,10909,10919,10954,10973,11297,11656,11689,11726,11782,11787,11825,11832"}}
+{"timestamp":1714605382.5494266,"name":"online","context":{"idset":"11856"}}
+{"timestamp":1714605382.6587718,"name":"online","context":{"idset":"1220,6634,6661,7403,7498,7510,7753,7781,7853,7872,7878,7963,7984,8012,8021,8046,8087,8092,8311,8319,8322,8329,8347,8378,8380,8626,8653,8671,8684,8723,8730,8737,8757,8764,8790,8793,8803-8804,8873,8889,8918,8970,8978,8998,9059,9133,9180,9249,9252,9264,9269,9286,9336-9337,9357,9363,9377,9510,9519,9541,9553,9595,9622,9640,9720,9768,9807,9812,9830,9900,9914,9934,9958,9961,9975,9993,10000,10013,10020,10029,10038,10047,10057,10077,10084,10086,10093,10156,10158,10167,10207,10241,10270,10282,10319,10322,10329,10344,10348,10392,10428,10431,10438,10488,10497,10515-10516,10526,10529,10568,10577,10582,10592,10612,10619,10624,10690,10692,10719,10747,10767,10793,10861,10892,10926,10963,10996,11023,11078,11141,11202,11220,11262,11280,11379,11504,11551,11672-11673,11683,11692,11704-11705,11733-11734,11748,11771,11800,11823,11847,11851,11871,11874,11883"}}
+{"timestamp":1714605382.6954699,"name":"online","context":{"idset":"95,103-104,106,114,131,158,166,168-169,171,174,188,201,203,205,209-210,215,227,232,235,238,240,249,951,967,1180,1225,6518,6520,6529,6562,6571,6590,6617,6653,6680,6696-6697,6706,6736,6753,7014,7017,7045-7046,7057,7070,7194,7197,7217,7223,7243,7277,7297,7301,7322,7328,7333,7341,7346,7355,7368-7369,7373,7413,7435,7439,7447,7472,7517,7713,7881,7898,8122,8163,8236,8577,8632,8702,8809,9177,9235,9257,9261,9299,9332,9340,9371,9387-9388,9407,9410,9428,9442-9443,9455,9460,9462,9471,9529,9563,9580,9602,9605,9837,9982"}}
+{"timestamp":1714605382.6970234,"name":"online","context":{"idset":"7782,7821,7890,7912,8027,8116,8142,8391,8662,9338"}}
+{"timestamp":1714605382.7268755,"name":"online","context":{"idset":"1206,6609,6613,7051,7102,7162,7245,7294,7315,7505,7715,7879,7925,8176,8289,8314,8331,8369,8375,8613,8625,8658,8760,8771,8882,8931,9029,9082,9137,9148,9153,9179,9200,9212,9215,9231,9262,9271,9433,9453,9502,9521,9664,9704-9705,9752,9860,9880,9887,9954,10019,10030,10045,10055,10074,10126,10170,10217,10232,10266,10370,10379,10588,10652,10681,10702,10705,10707,10710,10768,10780,10805,10808,10811,10818,10880,10927,10932,10998,11071,11102-11103,11123,11129,11140,11163,11245,11294,11316,11651,11712,11717,11728,11873"}}
+{"timestamp":1714605382.8332057,"name":"online","context":{"idset":"968,6600,7063,7113,7220,7236,8204,8276,8399,8570,9641,9717,9798,10068,10833,11045,11198,11357"}}
+{"timestamp":1714605382.867295,"name":"online","context":{"idset":"576,961,1051,1227,6557,6624,6638,6718,6725,6766,6772,7050,7167,7171,7180,7186,7397,7490,7524,7783,7796,7807,7809,7813-7814,7874,7914,7923-7924,8033,8083,8098,8148,8155,8266,8430,8569,8628,8646,8648,8673,8678,8696,8756,8846,8911,8976-8977,9016,9018,9024,9124,9131,9175,9241,9259,9289,9295,9319,9328,9330,9339,9396,9468,9484,9507,9663,9668,9673,9761-9762,9774,9789,9794,9816,9835,9884,9908,9918,9938,9959,9978,10031,10053,10062,10088,10094,10101,10129,10136,10248,10292,10299,10301,10303,10330,10349,10356,10382,10483,10491,10499,10505,10637-10638,10643,10651,10729,10737,10771-10772,10849,10895,10942,11009,11020,11082-11083,11104,11120,11131,11160,11173,11181,11235,11261,11268,11301,11315,11342,11356,11507,11510,11513,11523,11529,11533,11556,11578,11584,11630-11631,11643,11647,11653,11676,11694,11697-11698,11700,11715,11720,11810-11811,11824,11834,11855,11885"}}
+{"timestamp":1714605382.9863424,"name":"online","context":{"idset":"7239,7731,9692,10218,11028"}}
+{"timestamp":1714605382.9923587,"name":"online","context":{"idset":"7462"}}
+{"timestamp":1714605382.9939601,"name":"online","context":{"idset":"7947"}}
+{"timestamp":1714605383.0003746,"name":"online","context":{"idset":"1207,6758,7132,7149,7168,7971,10179,10352,10687,10777,10950,11547"}}
+{"timestamp":1714605383.0063393,"name":"online","context":{"idset":"7432,11358"}}
+{"timestamp":1714605383.0085468,"name":"online","context":{"idset":"6679,6691,7091,7434,8064,8170,8173,8638,8824,8937,9074,9159,10050,10168,10258,10384,10843"}}
+{"timestamp":1714605383.1569822,"name":"online","context":{"idset":"112,199,208,824,963,1175,1230,6576,6627,6631,6642,6644-6645,6674,7062,7081,7095,7158,7193,7204,7274,7303,7324,7348,7351,7374-7375,7399-7400,7409,7459,7537,7674,7722,7736,7787,7856,7859,7891,7899,8010,8045,8063,8084,8130,8184,8218,8220,8237,8267,8302,8340,8351,8403,8407,8425,8429,8599,8620-8621,8669,8679,8690,8766,8773,8802,8816-8818,8820,8825,8899,8901,8933,8962,9044,9065,9078,9103,9191,9208,9229,9298,9355,9402,9446,9461,9487,9492,9503,9528,9564,9571,9575,9599,9608,9629,9635,9651,9728,9742,9751,9781-9782,9784-9785,9819,9827,9851,9870,9877,9933,9944,9971,10058,10123,10143,10154-10155,10182,10196,10249,10252,10256,10290,10347,10418,10465,10498,10510,10602,10635,10644,10657,10685,10704,10715,10725,10753,10778,10783,10785,10789,10840,10914,11038,11046,11124,11144,11149,11167-11169,11197,11201,11222-11223,11225,11242,11244,11252,11263,11327,11331,11361,11373,11505,11509,11526,11534,11571,11580-11581,11589-11590,11602,11666,11674,11745,11754,11758,11760,11802,11819,11865,11880,11889"}}
+{"timestamp":1714605383.1596241,"name":"online","context":{"idset":"8859,9270,10591,10661"}}
+{"timestamp":1714605383.1770971,"name":"online","context":{"idset":"42,108,170,221,251,1065-1066,1209,6551,6554,6573,6580,6625,6637,6670,6673,6690,6701,7021,7157,7209,7257,7276,7286,7289,7291,7304,7321,7326,7345,7384,7392-7393,7494,7698,7725,7815,7822,7834,7837,7840,8007,8029,8057,8131,8157,8200,8203,8225,8234,8297,8315,8379,8401,8414,8433,8608,8615,8641,8644,8650,8692,8736,8748,8762,8784,8811,8836,8915,8920,8952,8985,9013,9021,9076-9077,9105,9114,9216,9240,9266-9267,9275,9323,9356,9397,9409,9500,9585-9586,9603,9756,9799,9828,9832,9864,9876,9979,10010,10015,10028,10043,10056,10076,10090,10161,10175,10220,10251,10263,10288-10289,10339,10363,10396,10401,10412,10466-10467,10475,10537,10580,10632,10674,10713,10746,10826,10870,10897,10899,10933,11005,11014,11021,11051,11076,11126,11236,11253,11324,11494,11544,11614,11628,11648,11659,11669,11732,11762,11789,11797,11807,11833,11839,11854,11876"}}
+{"timestamp":1714605383.3263719,"name":"online","context":{"idset":"8622,8647,8654,8663,8763,8996,9026,9055,9244,9389"}}
+{"timestamp":1714605383.3477476,"name":"online","context":{"idset":"8175,8689,8742,8988,9083,9206,9426,9445,9621,9643,9672,10325,10548"}}
+{"timestamp":1714605383.4273431,"name":"online","context":{"idset":"130,162,167,173,180,225,579,6569,6574,6587,6591,6594,6598,6603,6622,6628,6671,6678,7141,7196,7208,7214,7250,7253,7271,7273,7309,7325,7336,7370-7372,7418,7503,7523,7772,7797,7803,7808,7823,7852,7862,7875,7897,7905,7930,7952,7954,7961,8058,8080,8112,8123,8167,8214,8229,8232,8247,8283,8318,8330,8346,8382-8383,8387,8435,8597,8603,8607,8631,8667,8672,8675,8700,8716,8739,8787-8788,8794,8808,8810,8813,8893,8947,8984,8990,8995,9012,9067,9172,9227,9236,9239,9291,9306,9318,9321,9354,9374,9400,9405-9406,9431,9444,9448,9450,9478,9490,9493,9515,9524,9527,9536,9562,9588,9623,9649,9655,9667,9702,9712,9736,9743-9744,9788,9800,9815,9817-9818,9825,9829,9858,9861-9862,9873,9875,9891,9911-9913,9925,9930,9947,9974,10005,10042,10060,10070,10078,10081,10099,10139,10166,10171,10200,10233,10262,10278,10285,10317,10326,10365,10386,10389,10417,10442,10448,10454,10473,10480,10509,10524,10544,10579,10604,10629,10748,10754,10759,10776,10881,10923,10943,10956,10964,10977,11055,11066,11087,11090,11107,11112,11119,11125,11177,11180,11215,11238,11251,11305,11308,11334,11370,11378,11528,11539,11604,11613,11621,11638,11640,11650,11655,11677,11691,11751,11755,11784-11785,11830,11878,11890"}}
+{"timestamp":1714605383.5345123,"name":"online","context":{"idset":"944,954,1047,1181,6517,6656,6738,7043,7100,7338,7343,7467,7476,7709,8118,8147,8217,8271,8285,8292,8344,8358,8783,8887,9057,9195,9205,9308,9489,9557,9570,9613,9639,9695,9727,9869,9963,10027,10034,10095,10117,10120,10205,10219,10244,10300,10378,10586,10610,10823,10829,10838,10953,10959,11011,11026,11043,11060-11061,11093-11095,11171,11184,11191,11333,11336,11345,11355,11359,11363,11541-11542,11592,11606,11682,11701,11764,11815,11826,11837,11850,11860"}}
+{"timestamp":1714605383.5646868,"name":"online","context":{"idset":"29,1235,6523,6547,6556,6698,6744,6748,6761,7001,7219,7226,7229,7248,7306,7339,7406,7410,7441,7824,7829,7838,7851,7860,7904,8248,8284,8423-8424,8598,8604,8637,8698,8769,8942,8981,9017,9033-9034,9058,9097,9144,9422,9572,9619,9740,9745,9871,9955,10110,10186,10188,10197,10204,10206,10226,10287,10309,10443,10470-10471,10744,10792,10801,10822,10883,10957,11052,11057,11065,11080,11121,11128,11155,11224,11228,11302,11520,11597,11619,11629"}}
+{"timestamp":1714605383.5672138,"name":"online","context":{"idset":"11298"}}
+{"timestamp":1714605383.5731659,"name":"online","context":{"idset":"7308,9773"}}
+{"timestamp":1714605383.5754704,"name":"online","context":{"idset":"11841"}}
+{"timestamp":1714605383.5799406,"name":"online","context":{"idset":"1226"}}
+{"timestamp":1714605383.5821819,"name":"online","context":{"idset":"11678"}}
+{"timestamp":1714605383.5859625,"name":"online","context":{"idset":"8159"}}
+{"timestamp":1714605383.6231828,"name":"online","context":{"idset":"102,218,7138,7178,7380,7390,7513,7819,7861,8134,8177,8238,8300,8370,8385,8422,8780,8991,9006,9010,9052,9073,9219,9248,9320,9438,9452,9475,9486,9707,10080,10214,10216,10260,10297,10383,10630,10660,11050,11058,11091,11230,11524,11684"}}
+{"timestamp":1714605383.6441808,"name":"online","context":{"idset":"7754,8224,9687,11340"}}
+{"timestamp":1714605383.6568546,"name":"online","context":{"idset":"54,8668,9122,9222,9520,9703,9920,10647,11041,11831,11852,11892"}}
+{"timestamp":1714605383.6590281,"name":"online","context":{"idset":"1,9,22,34,39,41,43,48,53,60,7778,8242,10237,10752,11296"}}
+{"timestamp":1714605383.6614156,"name":"online","context":{"idset":"8093"}}
+{"timestamp":1714605383.7156348,"name":"online","context":{"idset":"7463,8298,9324,9927,10121,10137,10195,10830,10997,11073,11243,11248"}}
+{"timestamp":1714605383.9267614,"name":"online","context":{"idset":"37,7099,7191,7894,8356,8695,9615,9718-9719,9865,9883,10534,11329,11703"}}
+{"timestamp":1714605383.934319,"name":"online","context":{"idset":"16,50,7820,8136,8141,8261,8277,8281,8328,8338,8352,8374,8412,8432,8605,8614,8642,8685,8745,8782,8910,8916,8945,8989,9109,9317,9325,9327,9372,9657,9661,9693,9868,9956,10111,10124,10147,10178,10203,10228,10357,10361,10672,10828,11048,11134,11175,11182,11530,11532,11652,11710,11739,11848"}}
+{"timestamp":1714605383.9380143,"name":"online","context":{"idset":"9234,10587,11007,11538"}}
+{"timestamp":1714605383.9402838,"name":"online","context":{"idset":"8586,9532,11686"}}
+{"timestamp":1714605383.9425488,"name":"online","context":{"idset":"10500"}}
+{"timestamp":1714605383.9452999,"name":"online","context":{"idset":"6755,7770,8629"}}
+{"timestamp":1714605383.9477904,"name":"online","context":{"idset":"7711,9003,9173,10750"}}
+{"timestamp":1714605383.9552302,"name":"online","context":{"idset":"113,6623,6683,8670,8935,9163,9874,10107,10995,11371,11552,11680,11866"}}
+{"timestamp":1714605383.9577014,"name":"online","context":{"idset":"9391"}}
+{"timestamp":1714605383.9601684,"name":"online","context":{"idset":"7295,9711,10980,11730"}}
+{"timestamp":1714605384.0144303,"name":"online","context":{"idset":"1222,7224,10032,10105,11626"}}
+{"timestamp":1714605384.0287521,"name":"online","context":{"idset":"7302,9653,9995,10108,10628,10967,11054"}}
+{"timestamp":1714605384.2382553,"name":"online","context":{"idset":"10494"}}
+{"timestamp":1714605384.2411163,"name":"online","context":{"idset":"6589,6735,7114,7216,7360,7378,7534,7928,7935,8002,8183,8219,8221,8427,8767,8814,9178,9273,9326,9333,9441,9454,9797,9846,9899,9926,10423,10441,10622,10741,10763,11835"}}
+{"timestamp":1714605384.243767,"name":"online","context":{"idset":"8110,8326,10812"}}
+{"timestamp":1714605384.2464008,"name":"online","context":{"idset":"7320,8158"}}
+{"timestamp":1714605384.2487884,"name":"online","context":{"idset":"8775"}}
+{"timestamp":1714605384.2514861,"name":"online","context":{"idset":"9364"}}
+{"timestamp":1714605384.2546282,"name":"online","context":{"idset":"164,216,6641,6655,7235,7256,7798,7849,8213,9496,9542,10021,10152,10222,10393,11877"}}
+{"timestamp":1714605384.257457,"name":"online","context":{"idset":"177,233,7199,7363,8120,8677,9100,9618,9972,10125,10276,10455,10542,10609,10617,11069,11573,11663,11801"}}
+{"timestamp":1714605384.2603014,"name":"online","context":{"idset":"212,1176,1183"}}
+{"timestamp":1714605384.263025,"name":"online","context":{"idset":"10742"}}
+{"timestamp":1714605384.2655303,"name":"online","context":{"idset":"11033"}}
+{"timestamp":1714605384.2680898,"name":"online","context":{"idset":"8969"}}
+{"timestamp":1714605384.2706811,"name":"online","context":{"idset":"250"}}
+{"timestamp":1714605384.2737854,"name":"online","context":{"idset":"57,116,195,6599,6765,7316,7724,7854-7855,8743,8750,8774,8786,8801,9491,9626,10114,10353,10451,10645,10831,10894,11237,11317,11553,11729,11778"}}
+{"timestamp":1714605384.4990277,"name":"online","context":{"idset":"1232,7988,8056,8778,9028,10472,10755,10999"}}
+{"timestamp":1714605384.501966,"name":"online","context":{"idset":"17,23,30,36,56,58,7427,8705,9701,10506,11085"}}
+{"timestamp":1714605384.6109154,"name":"online","context":{"idset":"6,157,193,7012,7767,7865,8909,8957,9069,9151,9226,10140,10145-10146,10522,10566,10837,11001,11209,11598"}}
+{"timestamp":1714605384.7066033,"name":"online","context":{"idset":"7,21,8587,9014,9765,9951,9981,10513,10803,10857,10928,11624,11881"}}
+{"timestamp":1714605384.7159522,"name":"online","context":{"idset":"185"}}
+{"timestamp":1714605384.7408688,"name":"online","context":{"idset":"118,9617"}}
+{"timestamp":1714605384.7484031,"name":"online","context":{"idset":"7258"}}
+{"timestamp":1714605384.7815146,"name":"online","context":{"idset":"9915"}}
+{"timestamp":1714605384.8893917,"name":"online","context":{"idset":"10180"}}
+{"timestamp":1714605384.9499316,"name":"online","context":{"idset":"8922"}}
+{"timestamp":1714605385.0395832,"name":"online","context":{"idset":"8386"}}
+{"timestamp":1714605385.0577719,"name":"online","context":{"idset":"7483,7941,7991,9190,9714,11493,11567"}}
+{"timestamp":1714605385.0608261,"name":"online","context":{"idset":"10613,10987"}}
+{"timestamp":1714605385.07037,"name":"online","context":{"idset":"10240"}}
+{"timestamp":1714605385.0790668,"name":"online","context":{"idset":"9748"}}
+{"timestamp":1714605385.0841694,"name":"online","context":{"idset":"8912"}}
+{"timestamp":1714605385.0999734,"name":"online","context":{"idset":"10069"}}
+{"timestamp":1714605385.1028702,"name":"online","context":{"idset":"8171,8881"}}
+{"timestamp":1714605385.2589123,"name":"online","context":{"idset":"6643,6651,6737,7395,8323,9087,9146,9201,9342,11165,11822"}}
+{"timestamp":1714605385.262219,"name":"online","context":{"idset":"197"}}
+{"timestamp":1714605385.4060857,"name":"online","context":{"idset":"6519,8231,8349,8381,8602,8715,8914,8987,9126,11106,11515,11749"}}
+{"timestamp":1714605385.4215074,"name":"online","context":{"idset":"10165"}}
+{"timestamp":1714605385.4351628,"name":"online","context":{"idset":"8108"}}
+{"timestamp":1714605385.5933366,"name":"online","context":{"idset":"6640,7020,7264,8174,8255,8363,8643,9713,9859,10239,10360,10403,10468,10493,10590,10646,10968,11231,11250,11840"}}
+{"timestamp":1714605385.6044924,"name":"online","context":{"idset":"10671"}}
+{"timestamp":1714605385.6202815,"name":"online","context":{"idset":"9111"}}
+{"timestamp":1714605385.6394048,"name":"online","context":{"idset":"8982"}}
+{"timestamp":1714605385.7464788,"name":"online","context":{"idset":"7284"}}
+{"timestamp":1714605385.8530145,"name":"online","context":{"idset":"6650,6654,6721,7164,7962,8840,8870,9040,9071,9080,9276,9303,9346,10004,11519,11540,11777,11849"}}
+{"timestamp":1714605385.8630557,"name":"online","context":{"idset":"7027"}}
+{"timestamp":1714605385.8807368,"name":"online","context":{"idset":"7989,8651,10597,10949,11688"}}
+{"timestamp":1714605385.9211886,"name":"online","context":{"idset":"6521,7790,9999"}}
+{"timestamp":1714605385.9313147,"name":"online","context":{"idset":"8094,9072,10745"}}
+{"timestamp":1714605386.0373871,"name":"online","context":{"idset":"6583,10766"}}
+{"timestamp":1714605386.0688536,"name":"online","context":{"idset":"6682,7195,8206,10519"}}
+{"timestamp":1714605386.1775522,"name":"online","context":{"idset":"6531"}}
+{"timestamp":1714605386.189445,"name":"online","context":{"idset":"6615,6636,6747,7207,7247,7710,7908,8215,8243,8343,8573,8589,8594,8751,9142,9369,9764,9888,9949,10437,10576,10699,10721,10903,11056,11133,11146,11164,11196,11203,11254,11545,11632,11696"}}
+{"timestamp":1714605386.2061653,"name":"online","context":{"idset":"7335"}}
+{"timestamp":1714605386.2566125,"name":"online","context":{"idset":"8585"}}
+{"timestamp":1714605386.4050469,"name":"online","context":{"idset":"59,6652,7034,7161,8062,8252,8258,9435,10786,10985,11022,11246,11290,11293,11369,11512"}}
+{"timestamp":1714605386.4147255,"name":"online","context":{"idset":"10532"}}
+{"timestamp":1714605386.6425927,"name":"online","context":{"idset":"7311,7833,8035,8223,8250,8676,9238,9792,9867,9906,10026,10221,10598,10764,11042,11213,11846"}}
+{"timestamp":1714605386.7506604,"name":"online","context":{"idset":"7184,7700,9990,10022,10191,10294,10560,10809,11143,11195,11311"}}
+{"timestamp":1714605386.7536483,"name":"online","context":{"idset":"9606"}}
+{"timestamp":1714605386.7610364,"name":"online","context":{"idset":"111,7000,7160,7202,7526,8044,8140,8195,8211,8227,9631,9658,9684,9957,10194,11174,11183,11518,11625"}}
+{"timestamp":1714605386.9282424,"name":"online","context":{"idset":"15,18,6665,6707,7254,7697,7942,7999,8146,8972,9633,9650,9678,9970,10153,10824,10836"}}
+{"timestamp":1714605386.9368877,"name":"online","context":{"idset":"6660,7055,8152,10761"}}
+{"timestamp":1714605386.9398711,"name":"online","context":{"idset":"6694"}}
+{"timestamp":1714605386.943856,"name":"online","context":{"idset":"7140"}}
+{"timestamp":1714605386.9557092,"name":"online","context":{"idset":"7940,8041"}}
+{"timestamp":1714605386.9583805,"name":"online","context":{"idset":"7789,8932,10461"}}
+{"timestamp":1714605386.9612324,"name":"online","context":{"idset":"9646,10346,10688,10900,11861"}}
+{"timestamp":1714605387.0233181,"name":"online","context":{"idset":"6548,7445,9092"}}
+{"timestamp":1714605387.1889927,"name":"online","context":{"idset":"883,9681,9691,10765"}}
+{"timestamp":1714605387.1923456,"name":"online","context":{"idset":"11096"}}
+{"timestamp":1714605387.1974082,"name":"online","context":{"idset":"26,10700"}}
+{"timestamp":1714605387.2042754,"name":"online","context":{"idset":"12,19,25,6544,7211,7332,7471,7473,7485,7511,7688,7696,7757,8342,8565,9352,9416,10654,10991,11079"}}
+{"timestamp":1714605387.2298703,"name":"online","context":{"idset":"20,244,6608,7839,8320,8855,8903,9518,9561,9574,9725,9734,9998"}}
+{"timestamp":1714605387.3014016,"name":"online","context":{"idset":"1184,1223,6686,6730,7022,7111,7145,7206,7340,7440,7461,7514,7533,7766,7998,8054,8066,8837-8838,8858,8926,8975,9045,9081,9115,9351,9417,9537,9581,9710,10144,10247,10275,10324,10507,10514,10649,10835,10842,10986,11003,11330,11582,11718,11773"}}
+{"timestamp":1714605387.4093421,"name":"online","context":{"idset":"200"}}
+{"timestamp":1714605387.4131637,"name":"online","context":{"idset":"239"}}
+{"timestamp":1714605387.4177554,"name":"online","context":{"idset":"7241,7870,9856,10608"}}
+{"timestamp":1714605387.4333611,"name":"online","context":{"idset":"187,1048,6616,6724,7356,8827"}}
+{"timestamp":1714605387.4379551,"name":"online","context":{"idset":"32"}}
+{"timestamp":1714605387.5744286,"name":"online","context":{"idset":"94,98,160,178,186,6579,6602,6669,6751,7119,7121,7353,7381,7396,7412,7453,7506,7682,7730,7759,7761,7831,7836,7900,7919,7978,8031,8067,8128,8132,8294,8299,8316,8366,8402,8409,8704,8753-8754,8798,8806,8830,8865,8961,8967,9070,9075,9091,9154,9186,9192,9220,9296,9379,9386,9430,9440,9523,9550,9662,9759,9770,9783,9824,9902,9907,9983,9989,10003,10085,10284,10359,10390,10434,10601,10698,10716,10732,10757,10907,11166,11219,11525,11601,11615,11627,11685,11753,11769,11781,11786,11798,11812"}}
+{"timestamp":1714605387.5937665,"name":"online","context":{"idset":"9890,10334"}}
+{"timestamp":1714605387.6025882,"name":"online","context":{"idset":"9844"}}
+{"timestamp":1714605387.723815,"name":"online","context":{"idset":"35,6742,7950,8181,8376,9853,10257,10366,10492,10905,11200,11864"}}
+{"timestamp":1714605387.7307944,"name":"online","context":{"idset":"1067,6532,6732,7296,8279,8309,10025,10268,10684,10975,11555"}}
+{"timestamp":1714605387.7437677,"name":"online","context":{"idset":"1179,6546,6767,7251,7293,8105,8411,8567,8943,10104,10846,10976"}}
+{"timestamp":1714605387.7508476,"name":"online","context":{"idset":"8251,9048"}}
+{"timestamp":1714605387.7543473,"name":"online","context":{"idset":"128,237,7268,7915,8032,8241,8591,8596,8699,8701,8717,8948,8997,9246,9365-9366,9390,9401,9525,9675,9790,9803,9935,9960,10189,11145,11148,11199"}}
+{"timestamp":1714605387.7572117,"name":"online","context":{"idset":"10177,10464,11266"}}
+{"timestamp":1714605387.7624524,"name":"online","context":{"idset":"6632,7867,9258,10333,10813"}}
+{"timestamp":1714605387.7977691,"name":"online","context":{"idset":"9548"}}
+{"timestamp":1714605387.9307938,"name":"online","context":{"idset":"9769"}}
+{"timestamp":1714605387.9846435,"name":"online","context":{"idset":"245,948,6543,6568,7175,7281,7382,7670,7706,7812,7889,8030,8109,8270,8337,8426,8592,8712,9199,9214,9341,9411,9738-9739,9806,9847,10033,10269,10314,10350,10563,10655,10740,10854,11138,11269,11536,11608,11757,11882"}}
+{"timestamp":1714605388.0824835,"name":"online","context":{"idset":"10541"}}
+{"timestamp":1714605388.1016221,"name":"online","context":{"idset":"234,942,1216,6722,7176,7377,7484,7810,7902,8020,8095,8372,8891,8913,9007,9297,9514,9544,9724,9968,9996,10097,10751,10779,10867,11118,11719,11759"}}
+{"timestamp":1714605388.109092,"name":"online","context":{"idset":"7955,7985"}}
+{"timestamp":1714605388.250314,"name":"online","context":{"idset":"955,9516,10584,10802,11172"}}
+{"timestamp":1714605388.2831733,"name":"online","context":{"idset":"6695,6998,7431,8015,8348,8853"}}
+{"timestamp":1714605388.2946999,"name":"online","context":{"idset":"1177,1229,6621,6715,7148,7260,7350,7945,8059,8310,8590,8627,10436"}}
+{"timestamp":1714605388.3011255,"name":"online","context":{"idset":"10673,10733"}}
+{"timestamp":1714605388.3125689,"name":"online","context":{"idset":"8434"}}
+{"timestamp":1714605388.4726739,"name":"online","context":{"idset":"953,6664,6700,7092,7683,7965,8038,8161,8568,8713,8734,8749,8878,8980,9035,9042,9096,9194,9196,9202-9203,9362,9378,9472,9511,9551,9779,9885,9988,10291,10342,10424,10930,10982,11161,11325,11667,11791,11843,11884"}}
+{"timestamp":1714605388.5197783,"name":"online","context":{"idset":"7298,8588,11188,11259"}}
+{"timestamp":1714605388.6558993,"name":"online","context":{"idset":"7415,9928,10310,10415,10756,10800,11068,11500,11550,11616,11706"}}
+{"timestamp":1714605388.6625688,"name":"online","context":{"idset":"6685,7019,7231,7481,7869,8205,8636,9170,9250,9279,9282,9499,9754,10600"}}
+{"timestamp":1714605388.6863601,"name":"online","context":{"idset":"93,226,1213,7877,8191,9904,11105"}}
+{"timestamp":1714605388.7002678,"name":"online","context":{"idset":"190"}}
+{"timestamp":1714605388.757314,"name":"online","context":{"idset":"28,40"}}
+{"timestamp":1714605388.9353573,"name":"online","context":{"idset":"1224,7030,7740,7784,7817,7895,8576,8619,8868,8958,9011,9277,9432,9852,10073,10549,10640,10662,11029,11312,11836"}}
+{"timestamp":1714605388.9542255,"name":"online","context":{"idset":"9921,10135,11288"}}
+{"timestamp":1714605388.962517,"name":"online","context":{"idset":"11233"}}
+{"timestamp":1714605388.9659033,"name":"online","context":{"idset":"11595"}}
+{"timestamp":1714605389.0223334,"name":"online","context":{"idset":"8353"}}
+{"timestamp":1714605389.0292802,"name":"online","context":{"idset":"8000,8303,8361,9305,9943,10083,10406,10533,10989,11053,11257,11497"}}
+{"timestamp":1714605389.0361595,"name":"online","context":{"idset":"7376,8792,8907"}}
+{"timestamp":1714605389.1406579,"name":"online","context":{"idset":"8049"}}
+{"timestamp":1714605389.2551291,"name":"online","context":{"idset":"107,1190,7007,7067,7144,7173,7327,7780,7800,7929,7956,8254,8693,9135,9721,10109,10250,10267,10295,10371,10433,10599,11032,11292,11565-11566,11744"}}
+{"timestamp":1714605389.2617309,"name":"online","context":{"idset":"224,8854"}}
+{"timestamp":1714605389.2736406,"name":"online","context":{"idset":"7234"}}
+{"timestamp":1714605389.2782004,"name":"online","context":{"idset":"10113"}}
+{"timestamp":1714605389.2829058,"name":"online","context":{"idset":"9923"}}
+{"timestamp":1714605389.2855978,"name":"online","context":{"idset":"10550"}}
+{"timestamp":1714605389.4393382,"name":"online","context":{"idset":"9916,11136,11300"}}
+{"timestamp":1714605389.5629594,"name":"online","context":{"idset":"940,6612,6619,6749,7773,7793,7843,8235,8797,9005,9638,9895,10816,10845,11059,11187,11239,11249,11348"}}
+{"timestamp":1714605389.5695238,"name":"online","context":{"idset":"6620,7479,8210,9648"}}
+{"timestamp":1714605389.5731912,"name":"online","context":{"idset":"7763,8841"}}
+{"timestamp":1714605389.5768764,"name":"online","context":{"idset":"10734"}}
+{"timestamp":1714605389.618211,"name":"online","context":{"idset":"8869,9699"}}
+{"timestamp":1714605389.7565503,"name":"online","context":{"idset":"7163,7242,7949,8197,8199,8249,8313,8377,8611,8877,9814,10313,10402,10449,10456,10890,11314,11772,11872"}}
+{"timestamp":1714605389.7724793,"name":"online","context":{"idset":"9567,9969,10422"}}
+{"timestamp":1714605389.9092107,"name":"online","context":{"idset":"8687,8729,9027"}}
+{"timestamp":1714605389.9212747,"name":"online","context":{"idset":"192,8275,8306,8582,8633,8674,8812,9423,10063,10148,10151,10653,10937"}}
+{"timestamp":1714605389.9248059,"name":"online","context":{"idset":"8905"}}
+{"timestamp":1714605389.9299881,"name":"online","context":{"idset":"10130"}}
+{"timestamp":1714605390.0688474,"name":"online","context":{"idset":"7669,7997,9892,9936,9966,10061,11018"}}
+{"timestamp":1714605390.0724037,"name":"online","context":{"idset":"6646"}}
+{"timestamp":1714605390.1782382,"name":"online","context":{"idset":"231,7267,9592,10689,10728,11307,11788"}}
+{"timestamp":1714605390.4076629,"name":"online","context":{"idset":"8,13,52,126,204,947,7349,7358,7519,7529,7536,8043,8395,8419,8649,8829,8860,8894,8949,8974,9254,9285,9313,9382,9495,9689,9820,10199,10343,10893,11109,11579,11845"}}
+{"timestamp":1714605390.5412605,"name":"online","context":{"idset":"9694"}}
+{"timestamp":1714605390.6933651,"name":"online","context":{"idset":"6595,7314,7857,8061,8179,8189,8196,8253,8304,8711,8923,8992,9060,9136,9778,9793,10067,10187,10227,10523,10547,10723,10864,11075,11162,11258,11593,11804"}}
+{"timestamp":1714605390.7270441,"name":"online","context":{"idset":"11002"}}
+{"timestamp":1714605390.9633133,"name":"online","context":{"idset":"6606,7421,8431,9380,9964,10103,10193,10225"}}
+{"timestamp":1714605391.0747914,"name":"online","context":{"idset":"7347,7911,7959,9037,9129,9628,9836,10404,10821,11364,11770"}}
+{"timestamp":1714605391.1810803,"name":"online","context":{"idset":"8212"}}
+{"timestamp":1714605391.2893929,"name":"online","context":{"idset":"6676,8169,8872,9118,9593,10388,11281,11827"}}
+{"timestamp":1714605391.3956285,"name":"online","context":{"idset":"179,7053,7147,7282,7307,7357,7437,7532,7695,7901,8039,8144,8821,8852,10915,11320"}}
+{"timestamp":1714605391.4964736,"name":"online","context":{"idset":"7134,8048,8074,8312,8851,8874,8946,8966,8973,9038,9095,9437,9589,9698,9953,10174,10369,10667,10703,10739,10872,11284,11646,11875"}}
+{"timestamp":1714605391.6009376,"name":"online","context":{"idset":"175,196,7041-7042,7106,7443,7742,7752,7967,8008,8153,8165,8584,8796,8908,8929,9090,9193,9335,9449,9464,9749,9843,9897,9917,10530,11151"}}
+{"timestamp":1714605391.7036955,"name":"online","context":{"idset":"956,7317,7844,7926,7995,9292,9469,9488,9604,9810,10102,10185,10212,10518,10569,10659,11035,11278"}}
+{"timestamp":1714605391.8071542,"name":"online","context":{"idset":"246,6577,9019,9025,9436,10016,10799,11101,11179,11289,11321,11642"}}
+{"timestamp":1714605391.9114618,"name":"online","context":{"idset":"99,6535,6746,7174,7285,7863,8193,8389,8417,9228,9395,9513,9987,10230,10341,11089,11723,11867"}}
+{"timestamp":1714605392.0189028,"name":"online","context":{"idset":"4,7917,8042,8188,8902,8950,9221,9256,9700,9732,10787,11019,11287,11365,11560"}}
+{"timestamp":1714605392.1286342,"name":"online","context":{"idset":"47,220,6550,7159,7238,7334,7398,8040,8207,9015,9023,9233,9300,9350,9479,9509,9614,9766,9924,10355,10815,11030"}}
+{"timestamp":1714605392.1455853,"name":"online","context":{"idset":"9530,9760,10115,10332,10411,10790,10798,10931,11517"}}
+{"timestamp":1714605392.3974409,"name":"online","context":{"idset":"6537,7230,8741,9237,9245,9962,10071,10281,10605,10773,10960,11351,11531,11557,11793,11795"}}
+{"timestamp":1714605392.5175953,"name":"online","context":{"idset":"10385,11067"}}
+{"timestamp":1714605392.6285975,"name":"online","context":{"idset":"9610"}}
+{"timestamp":1714605392.736326,"name":"online","context":{"idset":"7182"}}
+{"timestamp":1714605392.9568744,"name":"online","context":{"idset":"9669"}}
+{"timestamp":1714605393.0479918,"name":"online","context":{"idset":"51"}}
+{"timestamp":1714605393.1632414,"name":"online","context":{"idset":"9676"}}
+{"timestamp":1714605393.2468889,"name":"online","context":{"idset":"9674"}}
+{"timestamp":1714605394.6110961,"name":"online","context":{"idset":"9715"}}
+{"timestamp":1714605394.7375405,"name":"online","context":{"idset":"6733,7456"}}
+{"timestamp":1714605394.8620844,"name":"online","context":{"idset":"7029,7422"}}
+{"timestamp":1714605394.8663523,"name":"online","context":{"idset":"7469"}}
+{"timestamp":1714605394.870477,"name":"online","context":{"idset":"10626"}}
+{"timestamp":1714605394.8749292,"name":"online","context":{"idset":"7692,11277,11790"}}
+{"timestamp":1714605394.8794055,"name":"online","context":{"idset":"6760,7448,9181"}}
+{"timestamp":1714605394.8837674,"name":"online","context":{"idset":"7433"}}
+{"timestamp":1714605394.8884094,"name":"online","context":{"idset":"7497,9150,10631,10676"}}
+{"timestamp":1714605394.8929429,"name":"online","context":{"idset":"7788"}}
+{"timestamp":1714605394.8968422,"name":"online","context":{"idset":"10669"}}
+{"timestamp":1714605395.0787687,"name":"online","context":{"idset":"1068,1210,7112,7430,7495,7728,7974,7977,7996,8090,8099,8164,9758,10668,10724,10882,10906,11274"}}
+{"timestamp":1714605395.086868,"name":"online","context":{"idset":"8707,8733,9497,11503,11792"}}
+{"timestamp":1714605395.0933437,"name":"online","context":{"idset":"9165,9587,10593,10886,10994,11328"}}
+{"timestamp":1714605395.1002555,"name":"online","context":{"idset":"10254"}}
+{"timestamp":1714605395.2684529,"name":"online","context":{"idset":"939,6561,7407,7451,7673,7845,7868,7882,8055,8076,8149,8259,8325,8362,8364,8610,8645,8686,8706,8719,8776,8815,9158,9353,9451,9565,9597,9942,10066,10079,10098,10323,10345,10528,10913,11724,11821"}}
+{"timestamp":1714605395.274086,"name":"online","context":{"idset":"9425"}}
+{"timestamp":1714605395.2829006,"name":"online","context":{"idset":"7417"}}
+{"timestamp":1714605395.2900221,"name":"online","context":{"idset":"10781,10952"}}
+{"timestamp":1714605395.2957764,"name":"online","context":{"idset":"6629,7361"}}
+{"timestamp":1714605395.3032424,"name":"online","context":{"idset":"11157"}}
+{"timestamp":1714605395.4392259,"name":"online","context":{"idset":"11820"}}
+{"timestamp":1714605395.5029535,"name":"online","context":{"idset":"132,211,575,1187,1221,6528,6581,7185,7188,7205,7228,7255,7265-7266,7391,7411,7768,7884,7888,7970,8117,8156,8178,8192,8201,8226,8341,8388,8394,8415-8416,8612,8635,8839,8863,8904,9020,9043,9230,9268,9310,9331,9404,9413,9429,9474,9477,9504,9517,9540,9558,9573,9578,9644-9645,9753,9838,9872,9896,9901,9939,10049,10172,10236,10261,10273,10328,10337,10368,10381,10394,10462,10502,10545,10574-10575,10595-10596,10614,10648,10683,10758,10774,10795,10839,10934,10941,10947,10958,11017,11098,11117,11127,11208,11240,11279,11313,11346,11535,11591,11654,11657,11670,11708,11711,11735,11742,11779,11829,11868,11887"}}
+{"timestamp":1714605395.6184664,"name":"online","context":{"idset":"9596,10862,10965"}}
+{"timestamp":1714605395.6299758,"name":"online","context":{"idset":"6552,9301"}}
+{"timestamp":1714605395.6507461,"name":"online","context":{"idset":"214,797,6526,6768,7031,7037,7089,7097,7123,7130,7152,7252,7386,7521,7531,7842,7906,7921,7980,8060,8113,8194,8198,8246,8280,8296,8396,8420,8682,8770,8850,8866,9093,9106,9116,9156,9263,9288,9294,9311,9329,9418,9456,9476,9607,9611,9624,9682,9791,9802,9997,10007,10009,10017,10052,10087,10184,10223,10242,10358,10367,10495-10496,10521,10680,10738,10743,10788,10856,10962,11010,11270,11337,11353,11374,11599,11635,11660,11737,11768,11844"}}
+{"timestamp":1714605395.7780707,"name":"online","context":{"idset":"7249,7764,8895,9030,9189,9207,9322,9394,9485,9533,9905,10119,10302,10616,10760"}}
+{"timestamp":1714605395.8022642,"name":"online","context":{"idset":"964,977,6639,7423,11227"}}
+{"timestamp":1714605395.8142681,"name":"online","context":{"idset":"96"}}
+{"timestamp":1714605395.9356883,"name":"online","context":{"idset":"8657,8822,8848,8892,8898,8936,9002,9098,9125,9204,9412,9459"}}
+{"timestamp":1714605396.068156,"name":"online","context":{"idset":"207,798,823,1188,1208,1215,1236,6630,6666-6667,6693,6711-6712,6716,6740,6759,7005,7064,7076,7098,7118,7218,7405,7457,7507,7530,7540,7679,7691,7708,7720,7746,7751,7792,7794,7804,7832,7937,7958,7969,7976,7979,7981,7987,8072-8073,8086,8121,8126,8172,8190,8222,8264,8301,8321,8408,8413,8428,8600,8618,8666,8683,8688,8746,8832-8833,8835,8888,8917,8919,8968,9031,9051,9056,9102,9120,9223,9283,9287,9424,9642,9647,9808,10014,10163,10198,10208,10215,10397-10398,10478-10479,10539,10561,10686,10694,10718,10945,10955,11004,11006,11034,11086,11097,11137,11154,11158,11186,11210,11217,11264,11271,11335,11583,11623,11662,11687,11740,11809,11814,11816,11838,11862,11891"}}
+{"timestamp":1714605396.0725203,"name":"online","context":{"idset":"8755,9161,9210"}}
+{"timestamp":1714605396.1806536,"name":"online","context":{"idset":"100-101,123,129,163,184,803-804,1185,1214,1218,6524,6584,6593,6611,6688,6692,6708,6713,6750,6764,6771,7004,7015,7018,7052,7077,7104,7108,7124,7129,7143,7270,7283,7292,7318,7366,7425,7428,7678,7705,7785,7827,7966,7982,7986,8006,8037,8069,8119,8151,8630,8819,8960,9046,9127,9160,9183-9184,9232,9414,9531,9545,9576,9813,9821,9878,9965,9992,10046,10209,10231,10234,10271,10432,10485,10527,10573,10633,10678,10706,10841,10902,10917,10940,10966,10981,11049,11088,11115,11272,11380,11496,11516,11639,11671,11725,11747,11774,11817"}}
+{"timestamp":1714605396.334487,"name":"online","context":{"idset":"122,222-223,878,913,978,6578,6588,7048,7054,7240,7261,7365,7429,7486,7504,7684,7726,7795,7806,7850,7858,7866,7873,7990,8034,8078,8209,8334,8336,8339,8354,8360,8371,8397,8404,8406,8421,8640,8656,8664,8681,8718,8740,8826,8828,8842,8890,8953,8964,9001,9036,9047,9128,9132,9140,9182,9225,9251,9265,9278,9312,9315,9427,9434,9466,9483,9498,9546,9591,9600,9636,9652,9685,9708,9777,9786,9882,9937,9945,9977,9994,10023-10024,10037,10065,10091,10134,10192,10224,10255,10264,10320-10321,10336,10362,10429,10477,10512,10558,10583,10603,10618,10625,10675,10735,10852,10877,10898,10971,10974,10979,11024,11039,11081,11232,11286,11521,11600,11668,11756,11803,11805,11808,11863"}}
+{"timestamp":1714605396.4402187,"name":"online","context":{"idset":"182-183,189,241,243,252,877,1212,6560,6566,6601,6614,6672,6703,6741,7016,7103,7200,7213,7290,7312,7394,7509,7535,7748,7885,7918,7920,7968,8068,8103,8138,9848,9909,10100,10131,10245,10277,10304,10413,10430,10447,10484,10567,10711,10720,10730,10782,10820,10868,10889,10904,10911,11013,11015,11074,11139,11170,11255,11299,11309,11322,11506,11596,11620,11637,11736,11750,11888"}}
+{"timestamp":1714605396.5493252,"name":"online","context":{"idset":"110,6538,6555,6687,6723,6752,6754,6999,7146,7179,7181,7201,7221,7287,7344,7723,7750,7786,7846,7907,7936,7946,7992,8096,8106,8154,8185,8269,8308,8392,8791,8955,9000,9138,9157,9293,9370,9385,9465,9554,9556,9568,9579,9612,9632,9659,9737,9809,9984,10008,10072,10128,10133,10162,10183,10201-10202,10243,10298,10391,10400,10446,10457,10463,10869,10912,10993,11012,11092,11205,11226,11362,11543,11562,11586,11649,11767,11858"}}
+{"timestamp":1714605396.6801226,"name":"online","context":{"idset":"5,14,97,119,125,127,230,573,1211,6525,6527,6536,6542,6558-6559,6570,6575,6681,6745,7101,7170,7189,7299,7319,7364,7404,7470,7474,7672,7694,7776-7777,7818,7835,7892-7893,7910,7916,7931,7943,8036,8082,8102,8135,8180,8182,8230,8239,8260,8262,8288,8359,8365,8390,8571,8581,8583,8639,8661,8709,8752,8758,8765,8768,8777,8779,8906,8986,9089,9094,9152,9164,9187,9213,9253,9255,9281,9284,9290,9345,9457,9480,9494,9505,9538,9559,9582,9627,9666,9730,9733,9755,9780,9842,9850,9854-9855,9881,9910,9931,9946,9948,9985,10018,10075,10092,10132,10142,10160,10190,10238,10246,10259,10274,10279,10296,10307,10374,10376,10405,10407,10416,10426,10435,10445,10453,10476,10511,10535-10536,10555,10562,10565,10594,10634,10641,10708-10709,10727,10797,10817,10850,10855,10879,10885,10920,10936,10939,10948,10972,11016,11044,11072,11077,11084,11108,11110-11111,11150,11152,11193,11214,11241,11247,11260,11344,11350,11372,11511,11554,11563,11572,11575,11577,11603,11612,11693,11695,11714,11721,11746,11766,11806,11818"}}
+{"timestamp":1714605396.7846143,"name":"online","context":{"idset":"8272,8732,11265,11304,11527,11675"}}
+{"timestamp":1714605396.8914399,"name":"online","context":{"idset":"1234,7198,7233,7269,7300,7387,7416,8104,8107,8145,8168,8240,8257,8293,8634,8714,8927,9004,9041,9139,9304,9552,9590,9741,9771,9804,9986,10082,10116,10427,10450,10731,10794,10804,10848,10951,11047,11064,11366,11727,11752,11775"}}
+{"timestamp":1714605396.9957411,"name":"online","context":{"idset":"242,247,7040,7801,7811,7887,8071,8166,8265,8324,8772,9349,9359,9421,9458,9470,9539,9547,9726,9729,9929,10316,10375,10853,11031,11267,11658,11765"}}
+{"timestamp":1714605397.011966,"name":"online","context":{"idset":"1186,6541,6762,7137,7177,7183,7225,7244,7330,7359,7401,7721,7816,7896,7903,8216,8291,8357,9066,9243,9247,9316,9375,9383,9508,9569,9577,9583,9598,9634,9709,9757,9787,9903,10044,10138,10149,10280,10306,10372,10395,10414,10452,10489,10554,10717,10726,10749,10819,10887,10921,10925,10944,10946,10969,10978,11025,11037,11113,11116,11153,11156,11207,11285,11291,11306,11338,11347,11368,11514,11699,11761,11763,11799"}}
+{"timestamp":1714605397.1572812,"name":"online","context":{"idset":"6675,6770,7066,7262,7805,7939,8580,8617,8724,8727,8731,8857,8876,9463,9534,9584,9594,9763,9805,10106,10122,10211,10482,10677,10701,10866,11036,11135,11218,11568,11618,11859"}}
+{"timestamp":1714605397.1803191,"name":"online","context":{"idset":"9343,11522"}}
+{"timestamp":1714605397.1843536,"name":"online","context":{"idset":"9620,10164"}}
+{"timestamp":1714605397.1884973,"name":"online","context":{"idset":"9801"}}
+{"timestamp":1714605397.1926973,"name":"online","context":{"idset":"7337,9068,10440,11722,11869"}}
+{"timestamp":1714605397.19698,"name":"online","context":{"idset":"7883,8393,10210,11633"}}
+{"timestamp":1714605397.3103521,"name":"online","context":{"idset":"6662,11857"}}
+{"timestamp":1714605397.4267514,"name":"online","context":{"idset":"6648,7003,7487,7690,7744,7922,8053,8077,8228,8274,8307,8327,8721,8879,9420,9656,9747,9772,9894,9940,10012,10157,10340,10784,10827,10847,10865,10924,10990,11558,11702,11741,11794"}}
+{"timestamp":1714605397.4338944,"name":"online","context":{"idset":"10040"}}
+{"timestamp":1714605397.4381795,"name":"online","context":{"idset":"6739,10501"}}
+{"timestamp":1714605397.4433894,"name":"online","context":{"idset":"6607,9347"}}
+{"timestamp":1714605397.4475873,"name":"online","context":{"idset":"8245"}}
+{"timestamp":1714605397.4516814,"name":"online","context":{"idset":"8951,10540,10825"}}
+{"timestamp":1714605397.4557612,"name":"online","context":{"idset":"8410,8799,10127"}}
+{"timestamp":1714605397.4601171,"name":"online","context":{"idset":"8268,9637"}}
+{"timestamp":1714605397.5906379,"name":"online","context":{"idset":"49,161,8665,10373"}}
+{"timestamp":1714605397.6108289,"name":"online","context":{"idset":"9616,9665,9811,11283,11376"}}
+{"timestamp":1714605397.6161978,"name":"online","context":{"idset":"44"}}
+{"timestamp":1714605397.6271763,"name":"online","context":{"idset":"9857"}}
+{"timestamp":1714605397.7318592,"name":"online","context":{"idset":"8400,8623,8759,10564"}}
+{"timestamp":1714605397.7693377,"name":"online","context":{"idset":"577,6757,7496,8335,8761,8999,9104,9171,9224,9839,9919,10364,10380,10444,10988,11339,11548,11879"}}
+{"timestamp":1714605397.7903597,"name":"online","context":{"idset":"9688"}}
+{"timestamp":1714605398.0077295,"name":"online","context":{"idset":"6596,6618,7383,7446,7738,7938,8162,8368,10035,10150,10762,11062,11576,11738"}}
+{"timestamp":1714605398.147218,"name":"online","context":{"idset":"46,6545,7871,9822,9886,10059,10335,10439,10844,10992,11000,11282"}}
+{"timestamp":1714605398.2573419,"name":"online","context":{"idset":"24,7342,7354,8233,9723,10006,10041,10213,10806,10878,10918,11352,11780"}}
+{"timestamp":1714605398.5435951,"name":"online","context":{"idset":"3,10,33,45,7165,7913,9367,9654,9952,10387,10408,10859,11211,11498"}}
+{"timestamp":1714605398.8121693,"name":"online","context":{"idset":"9660,10863"}}
+{"timestamp":1714605399.1758451,"name":"online","context":{"idset":"7760,9088"}}
+{"timestamp":1714605399.4237635,"name":"online","context":{"idset":"1231,6635,7151,8003,8660,8939,8954,9101,9117,9555,9630,10036,10487,11561,11661"}}
+{"timestamp":1714605399.5323541,"name":"online","context":{"idset":"7475,9174,10650"}}
+{"timestamp":1714605399.8015978,"name":"online","context":{"idset":"884,6522,6540,6728,7190,7222,7323,7352,7449,7886,7972,8127,8186,8317,8345,8578,8624,8728,8886,8896,9167,9218,9373,9381,9467,9670,9991,10011,10039,10096,10229,10235,10283,10315,10331,10421,10520,10551,10581,11040,11194,11303,11343,11641,11681,11690,11707,11783,11813,11828,11842"}}
+{"timestamp":1714605400.006139,"name":"online","context":{"idset":"1052,7305,8022,8286,8333,8595,8655,8708,8720,8979,9280,9309,9368,9776,9823,9845,9863,9893,10112,10176,10589,10770,10807,10832,10871,10884,11130,11159"}}
+{"timestamp":1714605400.10832,"name":"online","context":{"idset":"219"}}
+{"timestamp":1714605400.2121108,"name":"online","context":{"idset":"9696-9697,9941,10311,11100,11178,11221,11499,11607"}}
+{"timestamp":1714605400.3227501,"name":"online","context":{"idset":"9399,10796,10935,11190,11360"}}
+{"timestamp":1714605400.9961867,"name":"online","context":{"idset":"11,27"}}
+{"timestamp":1714605401.393892,"name":"online","context":{"idset":"9680"}}
+{"timestamp":1714605401.618221,"name":"online","context":{"idset":"9679"}}
+{"timestamp":1714605402.1481149,"name":"online","context":{"idset":"9671"}}
+{"timestamp":1714605403.027822,"name":"online","context":{"idset":"9690"}}
+{"timestamp":1714605410.2549939,"name":"online","context":{"idset":"10663"}}
+{"timestamp":1714605410.5082939,"name":"online","context":{"idset":"6709,7105,7156,7701,7703,7719,7756,8864,9107,9113,10615,10691,10695"}}
+{"timestamp":1714605410.6316969,"name":"online","context":{"idset":"1189,6658,6702,6727,6756,7011,7028,7122,7155,7215,8009,8025,8081,8871,8944,9085,9099,10636,10658,11332"}}
+{"timestamp":1714605410.7424731,"name":"online","context":{"idset":"6659,6663,7933,8052,8800,10627,11326"}}
+{"timestamp":1714605411.028338,"name":"online","context":{"idset":"124,176,229,1173,6530,6533,6565,6567,6586,6592,6605,6610,6657,6668,6677,6710,6720,6726,6769,7006,7026,7232,7313,7362,7379,7385,7388,7402,7419,7426,7499,7512,7528,7702,7707,7716-7717,7727,7729,7762,7765,7774,7825,7847,7876,7944,7948,7975,8019,8026,8028,8097,8115,8373,8384,8418,8566,8574-8575,8579,8609,8652,8659,8680,8691,8697,8726,8735,8747,8781,8795,8807,8831,8862,8880,8921,8924-8925,8993,9008,9053,9079,9084,9162,9166,9242,9274,9314,9344,9358,9384,9393,9398,9419,9439,9447,9473,9535,9549,9566,9716,9722,9767,9795,9826,9898,9973,9976,10048,10064,10089,10118,10141,10173,10265,10272,10293,10327,10338,10354,10419-10420,10486,10531,10538,10543,10559,10571-10572,10585,10607,10639,10642,10666,10722,10775,10851,10873-10875,10901,10908,10922,10938,11070,11132,11142,11189,11216,11256,11295,11323,11610,11644-11645,11664,11679,11709,11713,11716,11731,11776,11796,11853,11870"}}
+{"timestamp":1714605411.3405738,"name":"online","context":{"idset":"105,109,159,181,191,194,213,228,248,6549,6553,6563,6585,6604,6626,7166,7169,7172,7187,7192,7203,7212,7227,7237,7246,7263,7329,7367,7712,7802,7841,7848,8051,8101,8244,8256,8273,8290,8305,8332,8616,8694,8703,8738,8789,8805,8983,8994,9009,9022,9211,9217,9334,9361,9392,9501,9522,9543,9560,9601,9609,9625,9686,9706,9735,9746,9750,9796,9831,9834,9840-9841,9866,9932,9950,9967,9980,10051,10054,10159,10169,10181,10305,10318,10351,10377,10399,10410,10425,10458-10460,10474,10481,10490,10508,10546,10556-10557,10611,10769,10791,10810,10814,10834,10860,10888,10891,10910,10929,10961,10970,10984,11008,11027,11063,11099,11114,11122,11147,11185,11192,11204,11206,11229,11234,11310,11341,11349,11367,11537,11546,11559,11564,11574,11585,11587-11588,11594,11617,11634,11636,11665,11743,11886"}}
+{"timestamp":1714605411.4561822,"name":"online","context":{"idset":"115,8202,8282,8398,9348,9526,10525,10858,11176,11549,11605"}}
+{"timestamp":1714605411.6921384,"name":"online","context":{"idset":"7259,8187,8263,8278,8287,8295,9833,9879,9889,9922,10409,10469,11212,11375,11377,11611"}}
+{"timestamp":1714605412.0530612,"name":"online","context":{"idset":"1217,9260,11609"}}
+{"timestamp":1714605412.3781443,"name":"online","context":{"idset":"8355"}}
+{"timestamp":1714605412.4951863,"name":"online","context":{"idset":"31"}}
+{"timestamp":1714605412.7015164,"name":"online","context":{"idset":"2,55"}}
+{"timestamp":1714605413.7450583,"name":"online","context":{"idset":"7414,8823"}}
+{"timestamp":1714605413.8605368,"name":"online","context":{"idset":"10570"}}
+{"timestamp":1714605413.9769866,"name":"online","context":{"idset":"7450"}}
+{"timestamp":1714605414.0854895,"name":"online","context":{"idset":"6534,7880,9145,9482"}}
+{"timestamp":1714605414.2368522,"name":"online","context":{"idset":"8208,9775,10286,11622"}}
+{"timestamp":1714605414.462121,"name":"online","context":{"idset":"9731,11354"}}
+{"timestamp":1714605415.7021534,"name":"online","context":{"idset":"38"}}
+{"timestamp":1714605416.8458145,"name":"online","context":{"idset":"9683"}}
+{"timestamp":1714605416.9780135,"name":"online","context":{"idset":"9677"}}
+{"timestamp":1714605572.7306151,"name":"online","context":{"idset":"574"}}
+{"timestamp":1714606153.8639591,"name":"online","context":{"idset":"7743"}}
+{"timestamp":1714606154.0939546,"name":"online","context":{"idset":"7739,7745"}}
+{"timestamp":1714606154.4612992,"name":"online","context":{"idset":"7741"}}
+{"timestamp":1714606154.7221813,"name":"online","context":{"idset":"7733,7735,7737"}}
+{"timestamp":1714606155.1031656,"name":"online","context":{"idset":"7747"}}
+{"timestamp":1714606596.5646775,"name":"offline","context":{"idset":"7757"}}
+{"timestamp":1714606623.761764,"name":"online","context":{"idset":"10552,10606"}}
+{"timestamp":1714606624.0043886,"name":"online","context":{"idset":"10578"}}
+{"timestamp":1714607565.0102928,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1714608380.1575506,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1714609082.4050808,"name":"drain","context":{"idset":"11251","reason":"epilog failed for jobid fsnM4k6JhyR","overwrite":0}}
+{"timestamp":1714609428.1979399,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1714610283.8582518,"name":"offline","context":{"idset":"10572"}}
+{"timestamp":1714610968.5564892,"name":"offline","context":{"idset":"11790"}}
+{"timestamp":1714610976.985395,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1714611021.967428,"name":"online","context":{"idset":"10597"}}
+{"timestamp":1714611022.1976521,"name":"online","context":{"idset":"10512,10572"}}
+{"timestamp":1714611023.356144,"name":"online","context":{"idset":"10524"}}
+{"timestamp":1714611604.5515127,"name":"offline","context":{"idset":"10094"}}
+{"timestamp":1714611781.6933138,"name":"offline","context":{"idset":"9359"}}
+{"timestamp":1714612262.5000169,"name":"online","context":{"idset":"10094"}}
+{"timestamp":1714612262.7738147,"name":"online","context":{"idset":"11790"}}
+{"timestamp":1714612263.3098965,"name":"online","context":{"idset":"10486"}}
+{"timestamp":1714612449.1469176,"name":"offline","context":{"idset":"10515"}}
+{"timestamp":1714612979.2362759,"name":"offline","context":{"idset":"913"}}
+{"timestamp":1714612981.7427824,"name":"offline","context":{"idset":"914"}}
+{"timestamp":1714613372.5861876,"name":"offline","context":{"idset":"10524"}}
+{"timestamp":1714614331.8265231,"name":"offline","context":{"idset":"10528"}}
+{"timestamp":1714615195.1715994,"name":"offline","context":{"idset":"10541"}}
+{"timestamp":1714616086.9974129,"name":"offline","context":{"idset":"10536"}}
+{"timestamp":1714618713.4362087,"name":"offline","context":{"idset":"10873"}}
+{"timestamp":1714618713.5891747,"name":"offline","context":{"idset":"10871"}}
+{"timestamp":1714618713.687973,"name":"offline","context":{"idset":"10874"}}
+{"timestamp":1714618713.8193829,"name":"offline","context":{"idset":"10881"}}
+{"timestamp":1714618713.9179616,"name":"offline","context":{"idset":"10878"}}
+{"timestamp":1714618714.0410578,"name":"offline","context":{"idset":"10877"}}
+{"timestamp":1714618714.3860435,"name":"offline","context":{"idset":"10883"}}
+{"timestamp":1714618715.6254716,"name":"offline","context":{"idset":"10879"}}
+{"timestamp":1714618715.6296897,"name":"offline","context":{"idset":"10875"}}
+{"timestamp":1714618715.6339474,"name":"offline","context":{"idset":"10882"}}
+{"timestamp":1714618715.6382048,"name":"offline","context":{"idset":"10870"}}
+{"timestamp":1714618715.6425767,"name":"offline","context":{"idset":"10872"}}
+{"timestamp":1714618715.6469009,"name":"offline","context":{"idset":"10880"}}
+{"timestamp":1714618715.6511447,"name":"offline","context":{"idset":"10884"}}
+{"timestamp":1714618716.0151784,"name":"offline","context":{"idset":"10869"}}
+{"timestamp":1714618716.1139421,"name":"offline","context":{"idset":"10876"}}
+{"timestamp":1714619319.90763,"name":"offline","context":{"idset":"10890"}}
+{"timestamp":1714619320.0049345,"name":"offline","context":{"idset":"10889"}}
+{"timestamp":1714619320.175276,"name":"offline","context":{"idset":"10888"}}
+{"timestamp":1714619320.1881766,"name":"offline","context":{"idset":"10893"}}
+{"timestamp":1714619320.2882061,"name":"offline","context":{"idset":"10896"}}
+{"timestamp":1714619320.4560413,"name":"offline","context":{"idset":"10899"}}
+{"timestamp":1714619320.4908772,"name":"offline","context":{"idset":"10886"}}
+{"timestamp":1714619320.5603564,"name":"offline","context":{"idset":"10891"}}
+{"timestamp":1714619321.8242786,"name":"offline","context":{"idset":"10887"}}
+{"timestamp":1714619321.8301573,"name":"offline","context":{"idset":"10885"}}
+{"timestamp":1714619321.8358901,"name":"offline","context":{"idset":"10894"}}
+{"timestamp":1714619321.8416934,"name":"offline","context":{"idset":"10895"}}
+{"timestamp":1714619321.8473818,"name":"offline","context":{"idset":"10897"}}
+{"timestamp":1714619321.8531325,"name":"offline","context":{"idset":"10892"}}
+{"timestamp":1714619321.8588216,"name":"offline","context":{"idset":"10898"}}
+{"timestamp":1714619325.2704666,"name":"offline","context":{"idset":"10900"}}
+{"timestamp":1714619696.8723843,"name":"offline","context":{"idset":"10581"}}
+{"timestamp":1714619924.1552231,"name":"offline","context":{"idset":"10906"}}
+{"timestamp":1714619924.262624,"name":"offline","context":{"idset":"10903"}}
+{"timestamp":1714619925.4251983,"name":"offline","context":{"idset":"10907"}}
+{"timestamp":1714619925.4305265,"name":"offline","context":{"idset":"10909"}}
+{"timestamp":1714619925.5384328,"name":"offline","context":{"idset":"10904"}}
+{"timestamp":1714619925.8576429,"name":"offline","context":{"idset":"10910"}}
+{"timestamp":1714619925.9561799,"name":"offline","context":{"idset":"10905"}}
+{"timestamp":1714619926.1375127,"name":"offline","context":{"idset":"10914"}}
+{"timestamp":1714619926.2386296,"name":"offline","context":{"idset":"10908"}}
+{"timestamp":1714619927.3257434,"name":"offline","context":{"idset":"10902"}}
+{"timestamp":1714619927.3307552,"name":"offline","context":{"idset":"10912"}}
+{"timestamp":1714619927.3353803,"name":"offline","context":{"idset":"10911"}}
+{"timestamp":1714619927.3402727,"name":"offline","context":{"idset":"10901"}}
+{"timestamp":1714619927.399271,"name":"offline","context":{"idset":"10913"}}
+{"timestamp":1714619927.4965456,"name":"offline","context":{"idset":"10915"}}
+{"timestamp":1714619928.2582645,"name":"offline","context":{"idset":"10916"}}
+{"timestamp":1714620344.4776037,"name":"undrain","context":{"idset":"11508"}}
+{"timestamp":1714620389.6912267,"name":"online","context":{"idset":"11447"}}
+{"timestamp":1714620389.6958983,"name":"online","context":{"idset":"11462"}}
+{"timestamp":1714620389.7877052,"name":"online","context":{"idset":"11492"}}
+{"timestamp":1714620390.3837333,"name":"online","context":{"idset":"11482"}}
+{"timestamp":1714620391.0019658,"name":"online","context":{"idset":"11465"}}
+{"timestamp":1714620391.2422681,"name":"online","context":{"idset":"11501"}}
+{"timestamp":1714620391.4825196,"name":"online","context":{"idset":"11502"}}
+{"timestamp":1714620391.7707036,"name":"online","context":{"idset":"11458"}}
+{"timestamp":1714620392.092495,"name":"online","context":{"idset":"11385"}}
+{"timestamp":1714620392.2444556,"name":"online","context":{"idset":"11429,11438"}}
+{"timestamp":1714620393.5128484,"name":"online","context":{"idset":"11437,11461"}}
+{"timestamp":1714620393.5211711,"name":"online","context":{"idset":"11420"}}
+{"timestamp":1714620393.8169732,"name":"online","context":{"idset":"11383,11426"}}
+{"timestamp":1714620394.3889058,"name":"online","context":{"idset":"11397,11448"}}
+{"timestamp":1714620395.5637155,"name":"online","context":{"idset":"11427,11436"}}
+{"timestamp":1714620395.5688705,"name":"online","context":{"idset":"11411,11417"}}
+{"timestamp":1714620395.5740459,"name":"online","context":{"idset":"11391"}}
+{"timestamp":1714620396.2057323,"name":"online","context":{"idset":"11408"}}
+{"timestamp":1714620397.3986173,"name":"online","context":{"idset":"11440"}}
+{"timestamp":1714620397.6625357,"name":"online","context":{"idset":"11398,11402,11415"}}
+{"timestamp":1714620397.9354806,"name":"online","context":{"idset":"11384"}}
+{"timestamp":1714620399.3571408,"name":"online","context":{"idset":"11433"}}
+{"timestamp":1714620401.4147072,"name":"online","context":{"idset":"11392"}}
+{"timestamp":1714620401.4183807,"name":"online","context":{"idset":"11382,11386"}}
+{"timestamp":1714620401.6137309,"name":"online","context":{"idset":"11388"}}
+{"timestamp":1714620402.2112296,"name":"online","context":{"idset":"11396"}}
+{"timestamp":1714620403.5107682,"name":"online","context":{"idset":"11400"}}
+{"timestamp":1714620404.2846975,"name":"online","context":{"idset":"11404"}}
+{"timestamp":1714620405.4606562,"name":"online","context":{"idset":"11413,11456,11463,11469,11483,11490"}}
+{"timestamp":1714620405.4648273,"name":"online","context":{"idset":"11475"}}
+{"timestamp":1714620405.4691131,"name":"online","context":{"idset":"11424"}}
+{"timestamp":1714620405.4736114,"name":"online","context":{"idset":"11394,11435,11457,11464,11478"}}
+{"timestamp":1714620405.4782915,"name":"online","context":{"idset":"11389-11390,11395,11455,11487"}}
+{"timestamp":1714620405.7262187,"name":"online","context":{"idset":"11393,11422,11441,11452,11472,11491"}}
+{"timestamp":1714620406.3505316,"name":"online","context":{"idset":"11381,11387,11418,11421,11423,11432,11445,11449,11474,11476-11477,11480"}}
+{"timestamp":1714620406.5596635,"name":"online","context":{"idset":"11406-11407,11414,11428,11431,11434,11439,11451,11453,11459,11479,11486,11489"}}
+{"timestamp":1714620407.6651421,"name":"online","context":{"idset":"11430,11460,11470,11485"}}
+{"timestamp":1714620407.6678128,"name":"online","context":{"idset":"11399,11401,11410,11412,11416,11425,11442-11444,11446,11454,11466,11468"}}
+{"timestamp":1714620407.670481,"name":"online","context":{"idset":"11403,11405,11409,11419,11450,11467,11471,11473,11481,11484,11488"}}
+{"timestamp":1714620531.9113352,"name":"offline","context":{"idset":"10930"}}
+{"timestamp":1714620532.1104147,"name":"offline","context":{"idset":"10919"}}
+{"timestamp":1714620532.210381,"name":"offline","context":{"idset":"10928"}}
+{"timestamp":1714620532.8129628,"name":"offline","context":{"idset":"10925"}}
+{"timestamp":1714620533.6325765,"name":"offline","context":{"idset":"10923"}}
+{"timestamp":1714620533.6355293,"name":"offline","context":{"idset":"10932"}}
+{"timestamp":1714620533.6384192,"name":"offline","context":{"idset":"10922"}}
+{"timestamp":1714620533.6411932,"name":"offline","context":{"idset":"10926"}}
+{"timestamp":1714620533.7833548,"name":"offline","context":{"idset":"10924"}}
+{"timestamp":1714620533.886842,"name":"offline","context":{"idset":"10920"}}
+{"timestamp":1714620534.1560547,"name":"offline","context":{"idset":"10931"}}
+{"timestamp":1714620535.5707929,"name":"offline","context":{"idset":"10917"}}
+{"timestamp":1714620535.5734768,"name":"offline","context":{"idset":"10921"}}
+{"timestamp":1714620535.576117,"name":"offline","context":{"idset":"10927"}}
+{"timestamp":1714620535.5787919,"name":"offline","context":{"idset":"10929"}}
+{"timestamp":1714620535.8665941,"name":"offline","context":{"idset":"10918"}}
+{"timestamp":1714620559.9528587,"name":"offline","context":{"idset":"10499"}}
+{"timestamp":1714621139.3325846,"name":"offline","context":{"idset":"10944"}}
+{"timestamp":1714621139.3373094,"name":"offline","context":{"idset":"10937"}}
+{"timestamp":1714621139.4283736,"name":"offline","context":{"idset":"10936"}}
+{"timestamp":1714621139.715512,"name":"offline","context":{"idset":"10935"}}
+{"timestamp":1714621139.8139014,"name":"offline","context":{"idset":"10946"}}
+{"timestamp":1714621141.7736676,"name":"offline","context":{"idset":"10941"}}
+{"timestamp":1714621141.7786334,"name":"offline","context":{"idset":"10939"}}
+{"timestamp":1714621141.7834315,"name":"offline","context":{"idset":"10947"}}
+{"timestamp":1714621141.7954679,"name":"offline","context":{"idset":"10933"}}
+{"timestamp":1714621141.8008158,"name":"offline","context":{"idset":"10940"}}
+{"timestamp":1714621141.805624,"name":"offline","context":{"idset":"10943"}}
+{"timestamp":1714621141.8205252,"name":"offline","context":{"idset":"10948"}}
+{"timestamp":1714621141.825073,"name":"offline","context":{"idset":"10942"}}
+{"timestamp":1714621141.8382745,"name":"offline","context":{"idset":"10934"}}
+{"timestamp":1714621142.4480405,"name":"offline","context":{"idset":"10945"}}
+{"timestamp":1714621144.0954199,"name":"offline","context":{"idset":"10938"}}
+{"timestamp":1714621615.6951649,"name":"offline","context":{"idset":"10532"}}
+{"timestamp":1714621745.9284847,"name":"offline","context":{"idset":"10949"}}
+{"timestamp":1714621746.1740785,"name":"offline","context":{"idset":"10950"}}
+{"timestamp":1714621746.3160694,"name":"offline","context":{"idset":"10954"}}
+{"timestamp":1714621746.4169135,"name":"offline","context":{"idset":"10953"}}
+{"timestamp":1714621748.8825395,"name":"offline","context":{"idset":"10962"}}
+{"timestamp":1714621748.8851786,"name":"offline","context":{"idset":"10957"}}
+{"timestamp":1714621748.8878088,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1714621748.8904371,"name":"offline","context":{"idset":"10955"}}
+{"timestamp":1714621748.8930655,"name":"offline","context":{"idset":"10960"}}
+{"timestamp":1714621748.895678,"name":"offline","context":{"idset":"10951"}}
+{"timestamp":1714621748.8982913,"name":"offline","context":{"idset":"10956"}}
+{"timestamp":1714621748.9009421,"name":"offline","context":{"idset":"10952"}}
+{"timestamp":1714621748.903563,"name":"offline","context":{"idset":"10959"}}
+{"timestamp":1714621748.9061985,"name":"offline","context":{"idset":"10961"}}
+{"timestamp":1714621748.9088619,"name":"offline","context":{"idset":"10958"}}
+{"timestamp":1714621748.9115052,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1714622351.2132609,"name":"offline","context":{"idset":"10979"}}
+{"timestamp":1714622351.4814065,"name":"offline","context":{"idset":"10971"}}
+{"timestamp":1714622351.7285662,"name":"offline","context":{"idset":"10965"}}
+{"timestamp":1714622353.3966317,"name":"offline","context":{"idset":"10978"}}
+{"timestamp":1714622353.4009659,"name":"offline","context":{"idset":"10976"}}
+{"timestamp":1714622353.4063463,"name":"offline","context":{"idset":"10972"}}
+{"timestamp":1714622353.4111974,"name":"offline","context":{"idset":"10975"}}
+{"timestamp":1714622353.4157629,"name":"offline","context":{"idset":"10967"}}
+{"timestamp":1714622353.4199069,"name":"offline","context":{"idset":"10977"}}
+{"timestamp":1714622353.4238529,"name":"offline","context":{"idset":"10974"}}
+{"timestamp":1714622353.5463119,"name":"offline","context":{"idset":"10968"}}
+{"timestamp":1714622353.7981405,"name":"offline","context":{"idset":"10973"}}
+{"timestamp":1714622354.1568317,"name":"offline","context":{"idset":"10966"}}
+{"timestamp":1714622355.3362823,"name":"offline","context":{"idset":"10980"}}
+{"timestamp":1714622355.3416331,"name":"offline","context":{"idset":"10969"}}
+{"timestamp":1714622355.3467019,"name":"offline","context":{"idset":"10970"}}
+{"timestamp":1714622560.5327408,"name":"offline","context":{"idset":"10557"}}
+{"timestamp":1714622957.9887137,"name":"offline","context":{"idset":"10983"}}
+{"timestamp":1714622959.2236538,"name":"offline","context":{"idset":"10984"}}
+{"timestamp":1714622959.2292614,"name":"offline","context":{"idset":"10991"}}
+{"timestamp":1714622959.3272479,"name":"offline","context":{"idset":"10992"}}
+{"timestamp":1714622959.4809787,"name":"offline","context":{"idset":"10989"}}
+{"timestamp":1714622959.4945669,"name":"offline","context":{"idset":"10982"}}
+{"timestamp":1714622959.5030687,"name":"offline","context":{"idset":"10981"}}
+{"timestamp":1714622959.5987036,"name":"offline","context":{"idset":"10995"}}
+{"timestamp":1714622959.73931,"name":"offline","context":{"idset":"10996"}}
+{"timestamp":1714622959.8350835,"name":"offline","context":{"idset":"10988"}}
+{"timestamp":1714622960.2734144,"name":"offline","context":{"idset":"10993"}}
+{"timestamp":1714622962.4913001,"name":"offline","context":{"idset":"10990"}}
+{"timestamp":1714622962.4951899,"name":"offline","context":{"idset":"10987"}}
+{"timestamp":1714622962.4990373,"name":"offline","context":{"idset":"10985"}}
+{"timestamp":1714622962.5027621,"name":"offline","context":{"idset":"10994"}}
+{"timestamp":1714622962.9260352,"name":"offline","context":{"idset":"10986"}}
+{"timestamp":1714623349.8407509,"name":"online","context":{"idset":"10499,10557,10581"}}
+{"timestamp":1714623350.0990674,"name":"online","context":{"idset":"10515,10524,10528,10536"}}
+{"timestamp":1714623350.4129977,"name":"online","context":{"idset":"10532,10541"}}
+{"timestamp":1714623407.385148,"name":"undrain","context":{"idset":"11251"}}
+{"timestamp":1714623573.4817965,"name":"offline","context":{"idset":"10561"}}
+{"timestamp":1714623813.9909444,"name":"online","context":{"idset":"9359"}}
+{"timestamp":1714624497.7843654,"name":"offline","context":{"idset":"10486"}}
+{"timestamp":1714625227.7865493,"name":"offline","context":{"idset":"10584"}}
+{"timestamp":1714625401.125071,"name":"online","context":{"idset":"10874"}}
+{"timestamp":1714625401.1278851,"name":"online","context":{"idset":"10877"}}
+{"timestamp":1714625401.2539487,"name":"online","context":{"idset":"10878"}}
+{"timestamp":1714625401.5097878,"name":"online","context":{"idset":"10876,10880"}}
+{"timestamp":1714625401.7974055,"name":"online","context":{"idset":"10870,10883"}}
+{"timestamp":1714625401.9213564,"name":"online","context":{"idset":"10869,10871"}}
+{"timestamp":1714625402.0687189,"name":"online","context":{"idset":"10879,10882"}}
+{"timestamp":1714625402.2319925,"name":"online","context":{"idset":"10872"}}
+{"timestamp":1714625402.488852,"name":"online","context":{"idset":"10875,10881,10884"}}
+{"timestamp":1714625403.2093618,"name":"online","context":{"idset":"10873"}}
+{"timestamp":1714625412.5245025,"name":"online","context":{"idset":"10885"}}
+{"timestamp":1714625413.5200171,"name":"online","context":{"idset":"10887-10888"}}
+{"timestamp":1714625413.5239022,"name":"online","context":{"idset":"10886,10889"}}
+{"timestamp":1714625413.8381824,"name":"online","context":{"idset":"10891,10893"}}
+{"timestamp":1714625414.4290123,"name":"online","context":{"idset":"10890,10894"}}
+{"timestamp":1714625415.4463575,"name":"online","context":{"idset":"10895"}}
+{"timestamp":1714625415.5084276,"name":"online","context":{"idset":"10892,10896-10900"}}
+{"timestamp":1714625425.7593269,"name":"online","context":{"idset":"10901-10902"}}
+{"timestamp":1714625426.3473628,"name":"online","context":{"idset":"10904,10908"}}
+{"timestamp":1714625426.4759338,"name":"online","context":{"idset":"10909"}}
+{"timestamp":1714625427.2475617,"name":"online","context":{"idset":"10905"}}
+{"timestamp":1714625427.2519877,"name":"online","context":{"idset":"10903,10906"}}
+{"timestamp":1714625427.2563596,"name":"online","context":{"idset":"10914"}}
+{"timestamp":1714625427.3404164,"name":"online","context":{"idset":"10907,10910"}}
+{"timestamp":1714625427.5810523,"name":"online","context":{"idset":"10911"}}
+{"timestamp":1714625427.8802464,"name":"online","context":{"idset":"10913"}}
+{"timestamp":1714625428.1295297,"name":"online","context":{"idset":"10912,10915-10916"}}
+{"timestamp":1714625437.8864925,"name":"online","context":{"idset":"10917-10918"}}
+{"timestamp":1714625438.1870797,"name":"online","context":{"idset":"10919"}}
+{"timestamp":1714625439.303365,"name":"online","context":{"idset":"10920,10922"}}
+{"timestamp":1714625439.433548,"name":"online","context":{"idset":"10927"}}
+{"timestamp":1714625439.5467966,"name":"online","context":{"idset":"10921,10926"}}
+{"timestamp":1714625439.6550336,"name":"online","context":{"idset":"10924-10925,10928-10929"}}
+{"timestamp":1714625439.8820577,"name":"online","context":{"idset":"10923"}}
+{"timestamp":1714625441.1791763,"name":"online","context":{"idset":"10930-10932"}}
+{"timestamp":1714625449.9814548,"name":"online","context":{"idset":"10934"}}
+{"timestamp":1714625450.270036,"name":"online","context":{"idset":"10933"}}
+{"timestamp":1714625450.396626,"name":"online","context":{"idset":"10935"}}
+{"timestamp":1714625451.1784751,"name":"online","context":{"idset":"10936,10938"}}
+{"timestamp":1714625451.1852007,"name":"online","context":{"idset":"10940"}}
+{"timestamp":1714625451.1917706,"name":"online","context":{"idset":"10937,10941,10943"}}
+{"timestamp":1714625451.2303059,"name":"online","context":{"idset":"10939"}}
+{"timestamp":1714625451.4247253,"name":"online","context":{"idset":"10942"}}
+{"timestamp":1714625451.5385661,"name":"online","context":{"idset":"10944-10945"}}
+{"timestamp":1714625453.0274234,"name":"online","context":{"idset":"10946"}}
+{"timestamp":1714625453.3511734,"name":"online","context":{"idset":"10947"}}
+{"timestamp":1714625453.6533458,"name":"online","context":{"idset":"10948"}}
+{"timestamp":1714625463.6480837,"name":"online","context":{"idset":"10950-10954"}}
+{"timestamp":1714625463.6534786,"name":"online","context":{"idset":"10955-10958"}}
+{"timestamp":1714625463.974894,"name":"online","context":{"idset":"10949,10959-10961"}}
+{"timestamp":1714625466.0649552,"name":"online","context":{"idset":"10963"}}
+{"timestamp":1714625466.3180566,"name":"online","context":{"idset":"10962,10964"}}
+{"timestamp":1714625476.004694,"name":"online","context":{"idset":"10967"}}
+{"timestamp":1714625476.1136515,"name":"online","context":{"idset":"10968"}}
+{"timestamp":1714625476.2901282,"name":"online","context":{"idset":"10965-10966,10969"}}
+{"timestamp":1714625477.1891377,"name":"online","context":{"idset":"10973"}}
+{"timestamp":1714625477.2891006,"name":"online","context":{"idset":"10970-10971,10977"}}
+{"timestamp":1714625477.4787643,"name":"online","context":{"idset":"10975"}}
+{"timestamp":1714625477.8494432,"name":"online","context":{"idset":"10972"}}
+{"timestamp":1714625478.4924865,"name":"online","context":{"idset":"10974,10979"}}
+{"timestamp":1714625479.7261248,"name":"online","context":{"idset":"10978"}}
+{"timestamp":1714625479.7289839,"name":"online","context":{"idset":"10976"}}
+{"timestamp":1714625479.7317767,"name":"online","context":{"idset":"10980"}}
+{"timestamp":1714625488.0434594,"name":"online","context":{"idset":"10981"}}
+{"timestamp":1714625489.187499,"name":"online","context":{"idset":"10982"}}
+{"timestamp":1714625489.1921244,"name":"online","context":{"idset":"10983"}}
+{"timestamp":1714625489.1967795,"name":"online","context":{"idset":"10985,10987"}}
+{"timestamp":1714625489.2040017,"name":"online","context":{"idset":"10986"}}
+{"timestamp":1714625489.4687912,"name":"online","context":{"idset":"10984,10989-10990"}}
+{"timestamp":1714625489.709084,"name":"online","context":{"idset":"10988"}}
+{"timestamp":1714625489.8413198,"name":"online","context":{"idset":"10991"}}
+{"timestamp":1714625490.0860817,"name":"online","context":{"idset":"10992"}}
+{"timestamp":1714625490.5031247,"name":"online","context":{"idset":"10994"}}
+{"timestamp":1714625491.3595073,"name":"online","context":{"idset":"10993,10995"}}
+{"timestamp":1714625491.5959072,"name":"online","context":{"idset":"10996"}}
+{"timestamp":1714626207.6016936,"name":"offline","context":{"idset":"10569"}}
+{"timestamp":1714626776.5611238,"name":"offline","context":{"idset":"10528"}}
+{"timestamp":1714627535.5374551,"name":"offline","context":{"idset":"10581"}}
+{"timestamp":1714628163.4383984,"name":"offline","context":{"idset":"10496"}}
+{"timestamp":1714628346.1128969,"name":"offline","context":{"idset":"10963"}}
+{"timestamp":1714629056.5535891,"name":"offline","context":{"idset":"10542"}}
+{"timestamp":1714630034.9292989,"name":"offline","context":{"idset":"10603"}}
+{"timestamp":1714630678.5515227,"name":"offline","context":{"idset":"11790"}}
+{"timestamp":1714630880.2779574,"name":"offline","context":{"idset":"10497"}}
+{"timestamp":1714631602.578028,"name":"offline","context":{"idset":"10598"}}
+{"timestamp":1714632416.658731,"name":"offline","context":{"idset":"10612"}}
+{"timestamp":1714632480.5538306,"name":"offline","context":{"idset":"10964"}}
+{"timestamp":1714633253.5266466,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1714633326.7115428,"name":"drain","context":{"idset":"8202","reason":"nodediag failed amdapu cxi","overwrite":0}}
+{"timestamp":1714634151.4790525,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1714634698.5543702,"name":"offline","context":{"idset":"10550"}}
+{"timestamp":1714635454.5539827,"name":"offline","context":{"idset":"10531"}}
+{"timestamp":1714636098.5528228,"name":"offline","context":{"idset":"10485"}}
+{"timestamp":1714636990.5582299,"name":"offline","context":{"idset":"10529"}}
+{"timestamp":1714637944.5560369,"name":"offline","context":{"idset":"10568"}}
+{"timestamp":1714638962.5527935,"name":"offline","context":{"idset":"10518"}}
+{"timestamp":1714639910.5519588,"name":"offline","context":{"idset":"10605"}}
+{"timestamp":1714640802.5558982,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1714641755.0428908,"name":"offline","context":{"idset":"10607"}}
+{"timestamp":1714642432.9360652,"name":"offline","context":{"idset":"10100"}}
+{"timestamp":1714643174.5607655,"name":"drain","context":{"idset":"11250","reason":"broker was unresponsive"}}
+{"timestamp":1714643203.2107964,"name":"drain","context":{"idset":"11251","reason":"broker was unresponsive"}}
+{"timestamp":1714643386.5498459,"name":"offline","context":{"idset":"10085"}}
+{"timestamp":1714644382.5528483,"name":"offline","context":{"idset":"10590"}}
+{"timestamp":1714645166.555516,"name":"offline","context":{"idset":"9139"}}
+{"timestamp":1714645270.5592268,"name":"offline","context":{"idset":"10487"}}
+{"timestamp":1714646112.5517642,"name":"offline","context":{"idset":"10565"}}
+{"timestamp":1714646916.7349253,"name":"offline","context":{"idset":"10087"}}
+{"timestamp":1714647880.5520165,"name":"offline","context":{"idset":"10090"}}
+{"timestamp":1714648752.5514789,"name":"offline","context":{"idset":"10578"}}
+{"timestamp":1714649770.5532367,"name":"offline","context":{"idset":"10092"}}
+{"timestamp":1714650444.5534012,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1714651260.5248809,"name":"offline","context":{"idset":"10066"}}
+{"timestamp":1714651260.5594213,"name":"offline","context":{"idset":"10069"}}
+{"timestamp":1714651902.5525692,"name":"offline","context":{"idset":"10547"}}
+{"timestamp":1714652630.5526524,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1714653564.5520592,"name":"offline","context":{"idset":"10088"}}
+{"timestamp":1714654106.5523486,"name":"offline","context":{"idset":"10576"}}
+{"timestamp":1714654923.5366435,"name":"offline","context":{"idset":"10064"}}
+{"timestamp":1714655893.4840767,"name":"offline","context":{"idset":"10080"}}
+{"timestamp":1714656800.5526636,"name":"offline","context":{"idset":"10061"}}
+{"timestamp":1714657778.8844278,"name":"offline","context":{"idset":"10078"}}
+{"timestamp":1714657866.8261788,"name":"offline","context":{"idset":"798"}}
+{"timestamp":1714657881.4931991,"name":"undrain","context":{"idset":"895-896"}}
+{"timestamp":1714657913.4801791,"name":"online","context":{"idset":"895"}}
+{"timestamp":1714657927.397537,"name":"online","context":{"idset":"896"}}
+{"timestamp":1714658132.0293727,"name":"online","context":{"idset":"835"}}
+{"timestamp":1714658132.2707169,"name":"online","context":{"idset":"836"}}
+{"timestamp":1714658456.5589459,"name":"offline","context":{"idset":"10553"}}
+{"timestamp":1714658735.3906281,"name":"drain","context":{"idset":"1045-1046","reason":"--reason H/W troubleshoot","overwrite":0}}
+{"timestamp":1714658855.227,"name":"online","context":{"idset":"7068,7120"}}
+{"timestamp":1714658855.7688246,"name":"online","context":{"idset":"7086,7094"}}
+{"timestamp":1714658856.0084398,"name":"online","context":{"idset":"7059,7074,7126"}}
+{"timestamp":1714658856.1293573,"name":"online","context":{"idset":"7093"}}
+{"timestamp":1714658856.3637841,"name":"online","context":{"idset":"7065,7073,7083"}}
+{"timestamp":1714658857.3167603,"name":"online","context":{"idset":"7060,7069,7078,7080,7085,7125"}}
+{"timestamp":1714658857.3917866,"name":"online","context":{"idset":"7082"}}
+{"timestamp":1714659209.0473495,"name":"offline","context":{"idset":"10587"}}
+{"timestamp":1714659772.5233836,"name":"drain","context":{"idset":"7828","reason":"epilog failed for jobid fspck3wxk39","overwrite":0}}
+{"timestamp":1714659772.5665412,"name":"offline","context":{"idset":"7828"}}
+{"timestamp":1714659920.5534868,"name":"offline","context":{"idset":"10602"}}
+{"timestamp":1714660809.531693,"name":"offline","context":{"idset":"10083"}}
+{"timestamp":1714661263.5128129,"name":"drain","context":{"idset":"9051","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661263.8839579,"name":"drain","context":{"idset":"8955","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661265.388113,"name":"drain","context":{"idset":"9007","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661265.587363,"name":"drain","context":{"idset":"8984","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661267.1491287,"name":"drain","context":{"idset":"8976","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661269.036623,"name":"drain","context":{"idset":"8978","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661269.1513705,"name":"drain","context":{"idset":"8985","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661269.55814,"name":"drain","context":{"idset":"9070","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661271.3023362,"name":"drain","context":{"idset":"8954","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661271.4217756,"name":"drain","context":{"idset":"9072","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661283.3306735,"name":"drain","context":{"idset":"8990","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661296.2410183,"name":"drain","context":{"idset":"9013","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714661403.546792,"name":"undrain","context":{"idset":"6645-6772"}}
+{"timestamp":1714661406.0358717,"name":"undrain","context":{"idset":"7413-7540"}}
+{"timestamp":1714661651.0446808,"name":"offline","context":{"idset":"872"}}
+{"timestamp":1714661716.4701023,"name":"offline","context":{"idset":"10003"}}
+{"timestamp":1714661716.5529802,"name":"offline","context":{"idset":"10004"}}
+{"timestamp":1714661950.5314996,"name":"undrain","context":{"idset":"8954-8955,8976,8978,8984-8985,8990,9007,9013,9051,9070,9072"}}
+{"timestamp":1714661953.6200297,"name":"offline","context":{"idset":"944"}}
+{"timestamp":1714662716.821528,"name":"offline","context":{"idset":"942"}}
+{"timestamp":1714662982.476413,"name":"offline","context":{"idset":"967"}}
+{"timestamp":1714662982.5539627,"name":"offline","context":{"idset":"968"}}
+{"timestamp":1714663431.8563101,"name":"online","context":{"idset":"548"}}
+{"timestamp":1714663576.5605836,"name":"offline","context":{"idset":"548"}}
+{"timestamp":1714664099.5868359,"name":"undrain","context":{"idset":"1045-1046"}}
+{"timestamp":1714664163.4626415,"name":"online","context":{"idset":"1045-1046"}}
+{"timestamp":1714664820.9521356,"name":"online","context":{"idset":"871-872"}}
+{"timestamp":1714664891.8328722,"name":"drain","context":{"idset":"871-872","reason":"--reason “Running hpe_diags”","overwrite":0}}
+{"timestamp":1714665064.6215029,"name":"online","context":{"idset":"444"}}
+{"timestamp":1714665157.2470698,"name":"drain","context":{"idset":"1065-1068","reason":"hospital","overwrite":0}}
+{"timestamp":1714665228.501754,"name":"offline","context":{"idset":"1065"}}
+{"timestamp":1714665228.5167937,"name":"offline","context":{"idset":"1066"}}
+{"timestamp":1714665228.5272899,"name":"offline","context":{"idset":"1067"}}
+{"timestamp":1714665228.8485806,"name":"offline","context":{"idset":"1068"}}
+{"timestamp":1714665763.6613386,"name":"offline","context":{"idset":"10103"}}
+{"timestamp":1714665763.6780388,"name":"offline","context":{"idset":"10104"}}
+{"timestamp":1714665763.6915944,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1714665763.6999028,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1714665763.7077804,"name":"offline","context":{"idset":"10101"}}
+{"timestamp":1714665763.7160735,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1714665763.8490913,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1714665763.8647776,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1714665763.8780057,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1714665763.8829751,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1714665763.9494183,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1714665763.960603,"name":"offline","context":{"idset":"10134"}}
+{"timestamp":1714665763.9746797,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1714665763.9944711,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1714665764.038564,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1714665764.0433204,"name":"offline","context":{"idset":"10214"}}
+{"timestamp":1714665764.4156551,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1714665764.4234335,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1714665764.4396098,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1714665764.4500229,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1714665764.5328507,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1714665764.5379326,"name":"offline","context":{"idset":"10107"}}
+{"timestamp":1714665764.5427783,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1714665764.5626748,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1714665764.5675859,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1714665764.5772548,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1714665764.582129,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1714665764.5870104,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1714665764.601701,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1714665764.6109941,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1714665764.615855,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1714665764.6350379,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1714665764.8649321,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1714665764.8710566,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1714665764.8825622,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1714665764.8930836,"name":"offline","context":{"idset":"10153"}}
+{"timestamp":1714665764.9359341,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1714665765.0411274,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1714665765.2246268,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1714665765.3230858,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1714665765.4854636,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1714665765.4926229,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1714665765.5897372,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1714665765.8685017,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1714665765.8813188,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1714665765.8932643,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1714665765.9010365,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1714665765.9100559,"name":"offline","context":{"idset":"10102"}}
+{"timestamp":1714665765.9154868,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1714665765.9465904,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1714665765.9515636,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1714665766.0559659,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1714665766.2535083,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1714665766.2591875,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1714665766.2646255,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1714665766.2699404,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1714665766.2753527,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1714665766.2933493,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1714665766.4435949,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1714665766.710763,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1714665766.7327943,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1714665766.7410131,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1714665766.7495537,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1714665766.7585256,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1714665766.8044045,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1714665766.8098741,"name":"offline","context":{"idset":"10105"}}
+{"timestamp":1714665766.8154678,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1714665766.8462915,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1714665766.8713627,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1714665767.1229463,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1714665767.1396787,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1714665767.1452451,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1714665767.165875,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1714665767.1737728,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1714665767.1792488,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1714665767.2221994,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1714665767.3070085,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1714665767.5102324,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1714665767.5212736,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1714665767.5376601,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1714665767.544764,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1714665767.5506568,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1714665767.6779909,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1714665767.8588092,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1714665767.8640158,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1714665767.8793831,"name":"offline","context":{"idset":"10106"}}
+{"timestamp":1714665767.8882415,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1714665767.9293215,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1714665767.9367449,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1714665767.9428947,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1714665767.9973338,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1714665768.1942749,"name":"offline","context":{"idset":"10206"}}
+{"timestamp":1714665768.2050877,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1714665768.226665,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1714665768.2388854,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1714665768.2433701,"name":"offline","context":{"idset":"10213"}}
+{"timestamp":1714665768.2477922,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1714665768.3663161,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1714665768.6007881,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1714665768.6073761,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1714665768.6293662,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1714665768.6405547,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1714665769.5484653,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1714665769.5539899,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1714665769.5686386,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1714665769.573694,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1714665769.5788972,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1714665769.5837994,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1714665769.588913,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1714665769.5937977,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1714665769.5990479,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1714665769.6037507,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1714665769.6369305,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1714665769.6400516,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1714665769.643482,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1714665769.6463673,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1714665769.6494884,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1714665769.6535218,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1714665769.6564569,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1714665769.6593978,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1714665769.6628721,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1714665769.7542,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1714665769.9669495,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1714665770.0514324,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1714665770.2177796,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1714665770.3218317,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1714665770.7169285,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1714665770.8498502,"name":"offline","context":{"idset":"10188"}}
+{"timestamp":1714666280.5562334,"name":"offline","context":{"idset":"8743"}}
+{"timestamp":1714666282.5522132,"name":"offline","context":{"idset":"8744"}}
+{"timestamp":1714666314.5575027,"name":"offline","context":{"idset":"1051"}}
+{"timestamp":1714666316.5559618,"name":"offline","context":{"idset":"1052"}}
+{"timestamp":1714666856.5566607,"name":"offline","context":{"idset":"444"}}
+{"timestamp":1714667005.9601784,"name":"undrain","context":{"idset":"7828"}}
+{"timestamp":1714667519.3327639,"name":"online","context":{"idset":"7828"}}
+{"timestamp":1714667526.5526338,"name":"offline","context":{"idset":"6682"}}
+{"timestamp":1714667984.4789925,"name":"offline","context":{"idset":"7453"}}
+{"timestamp":1714667984.558239,"name":"offline","context":{"idset":"7454"}}
+{"timestamp":1714668018.4684744,"name":"offline","context":{"idset":"1047"}}
+{"timestamp":1714668018.552711,"name":"offline","context":{"idset":"1048"}}
+{"timestamp":1714668078.7855177,"name":"online","context":{"idset":"809"}}
+{"timestamp":1714668117.1236856,"name":"online","context":{"idset":"810"}}
+{"timestamp":1714668272.5532827,"name":"offline","context":{"idset":"6750"}}
+{"timestamp":1714668823.1884773,"name":"offline","context":{"idset":"7427"}}
+{"timestamp":1714668824.5529206,"name":"offline","context":{"idset":"7428"}}
+{"timestamp":1714669120.5509949,"name":"offline","context":{"idset":"7527"}}
+{"timestamp":1714669122.5528152,"name":"offline","context":{"idset":"7528"}}
+{"timestamp":1714669156.4687083,"name":"offline","context":{"idset":"835"}}
+{"timestamp":1714669156.5539124,"name":"offline","context":{"idset":"836"}}
+{"timestamp":1714669341.8323171,"name":"online","context":{"idset":"440"}}
+{"timestamp":1714669818.5513802,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1714670702.553442,"name":"offline","context":{"idset":"6681"}}
+{"timestamp":1714670820.5815711,"name":"undrain","context":{"idset":"8202"}}
+{"timestamp":1714670834.5570953,"name":"offline","context":{"idset":"817"}}
+{"timestamp":1714671392.5533862,"name":"offline","context":{"idset":"8949"}}
+{"timestamp":1714671394.4961803,"name":"offline","context":{"idset":"8950"}}
+{"timestamp":1714671394.5045242,"name":"offline","context":{"idset":"8951"}}
+{"timestamp":1714671394.5739958,"name":"offline","context":{"idset":"8952"}}
+{"timestamp":1714671396.2328968,"name":"offline","context":{"idset":"8953"}}
+{"timestamp":1714671396.2407789,"name":"offline","context":{"idset":"8954"}}
+{"timestamp":1714671396.2483816,"name":"offline","context":{"idset":"8955"}}
+{"timestamp":1714671396.3092575,"name":"offline","context":{"idset":"8956"}}
+{"timestamp":1714671398.4772232,"name":"offline","context":{"idset":"8957"}}
+{"timestamp":1714671398.481745,"name":"offline","context":{"idset":"8958"}}
+{"timestamp":1714671398.4862485,"name":"offline","context":{"idset":"8959"}}
+{"timestamp":1714671398.5596352,"name":"offline","context":{"idset":"8960"}}
+{"timestamp":1714671400.4792194,"name":"offline","context":{"idset":"8961"}}
+{"timestamp":1714671400.4845133,"name":"offline","context":{"idset":"8962"}}
+{"timestamp":1714671400.4898779,"name":"offline","context":{"idset":"8963"}}
+{"timestamp":1714671400.5756314,"name":"offline","context":{"idset":"8964"}}
+{"timestamp":1714671402.4816742,"name":"offline","context":{"idset":"8965"}}
+{"timestamp":1714671402.4865737,"name":"offline","context":{"idset":"8966"}}
+{"timestamp":1714671402.4931629,"name":"offline","context":{"idset":"8967"}}
+{"timestamp":1714671402.4983835,"name":"offline","context":{"idset":"8968"}}
+{"timestamp":1714671402.5520563,"name":"offline","context":{"idset":"8969"}}
+{"timestamp":1714671404.4721546,"name":"offline","context":{"idset":"8970"}}
+{"timestamp":1714671404.4773862,"name":"offline","context":{"idset":"8971"}}
+{"timestamp":1714671404.4825945,"name":"offline","context":{"idset":"8972"}}
+{"timestamp":1714671404.4878149,"name":"offline","context":{"idset":"8973"}}
+{"timestamp":1714671404.5500302,"name":"offline","context":{"idset":"8974"}}
+{"timestamp":1714671406.4927576,"name":"offline","context":{"idset":"8975"}}
+{"timestamp":1714671406.4978499,"name":"offline","context":{"idset":"8976"}}
+{"timestamp":1714671406.5030224,"name":"offline","context":{"idset":"8977"}}
+{"timestamp":1714671406.5517302,"name":"offline","context":{"idset":"8978"}}
+{"timestamp":1714671408.4756875,"name":"offline","context":{"idset":"8979"}}
+{"timestamp":1714671408.4800053,"name":"offline","context":{"idset":"8980"}}
+{"timestamp":1714671408.4842603,"name":"offline","context":{"idset":"8981"}}
+{"timestamp":1714671408.5712795,"name":"offline","context":{"idset":"8982"}}
+{"timestamp":1714671410.4754658,"name":"offline","context":{"idset":"8983"}}
+{"timestamp":1714671410.4795268,"name":"offline","context":{"idset":"8984"}}
+{"timestamp":1714671410.5622232,"name":"offline","context":{"idset":"8985"}}
+{"timestamp":1714671412.4788353,"name":"offline","context":{"idset":"8986"}}
+{"timestamp":1714671412.4834139,"name":"offline","context":{"idset":"8987"}}
+{"timestamp":1714671412.488239,"name":"offline","context":{"idset":"8988"}}
+{"timestamp":1714671412.4928997,"name":"offline","context":{"idset":"8989"}}
+{"timestamp":1714671412.5522578,"name":"offline","context":{"idset":"8990"}}
+{"timestamp":1714671414.4711082,"name":"offline","context":{"idset":"8991"}}
+{"timestamp":1714671414.5574615,"name":"offline","context":{"idset":"8992"}}
+{"timestamp":1714671416.4759247,"name":"offline","context":{"idset":"8993"}}
+{"timestamp":1714671416.5579641,"name":"offline","context":{"idset":"8994"}}
+{"timestamp":1714671418.4854922,"name":"offline","context":{"idset":"8995"}}
+{"timestamp":1714671418.4922442,"name":"offline","context":{"idset":"8996"}}
+{"timestamp":1714671418.5007064,"name":"offline","context":{"idset":"8997"}}
+{"timestamp":1714671418.5809932,"name":"offline","context":{"idset":"8998"}}
+{"timestamp":1714671420.4791925,"name":"offline","context":{"idset":"8999"}}
+{"timestamp":1714671420.4840915,"name":"offline","context":{"idset":"9000"}}
+{"timestamp":1714671420.4889617,"name":"offline","context":{"idset":"9001"}}
+{"timestamp":1714671420.5739183,"name":"offline","context":{"idset":"9002"}}
+{"timestamp":1714671422.4992523,"name":"offline","context":{"idset":"9003"}}
+{"timestamp":1714671422.5133765,"name":"offline","context":{"idset":"9004"}}
+{"timestamp":1714671422.5236366,"name":"offline","context":{"idset":"9005"}}
+{"timestamp":1714671422.559195,"name":"offline","context":{"idset":"9006"}}
+{"timestamp":1714671424.4776003,"name":"offline","context":{"idset":"9007"}}
+{"timestamp":1714671424.4834239,"name":"offline","context":{"idset":"9008"}}
+{"timestamp":1714671424.4894187,"name":"offline","context":{"idset":"9009"}}
+{"timestamp":1714671424.5535963,"name":"offline","context":{"idset":"9010"}}
+{"timestamp":1714671426.4750917,"name":"offline","context":{"idset":"9011"}}
+{"timestamp":1714671426.4795923,"name":"offline","context":{"idset":"9012"}}
+{"timestamp":1714671426.4840539,"name":"offline","context":{"idset":"9013"}}
+{"timestamp":1714671426.5738783,"name":"offline","context":{"idset":"9014"}}
+{"timestamp":1714671428.5260794,"name":"offline","context":{"idset":"9015"}}
+{"timestamp":1714671428.5362337,"name":"offline","context":{"idset":"9016"}}
+{"timestamp":1714671428.5464079,"name":"offline","context":{"idset":"9017"}}
+{"timestamp":1714671428.5565937,"name":"offline","context":{"idset":"9018"}}
+{"timestamp":1714671428.5666785,"name":"offline","context":{"idset":"9019"}}
+{"timestamp":1714671428.5768175,"name":"offline","context":{"idset":"9020"}}
+{"timestamp":1714671430.4644341,"name":"offline","context":{"idset":"9021"}}
+{"timestamp":1714671430.5501993,"name":"offline","context":{"idset":"9022"}}
+{"timestamp":1714671432.4811864,"name":"offline","context":{"idset":"9023"}}
+{"timestamp":1714671432.4889278,"name":"offline","context":{"idset":"9024"}}
+{"timestamp":1714671432.4967248,"name":"offline","context":{"idset":"9025"}}
+{"timestamp":1714671432.556797,"name":"offline","context":{"idset":"9026"}}
+{"timestamp":1714671434.4995532,"name":"offline","context":{"idset":"9027"}}
+{"timestamp":1714671434.5039692,"name":"offline","context":{"idset":"9028"}}
+{"timestamp":1714671434.5126047,"name":"offline","context":{"idset":"9029"}}
+{"timestamp":1714671434.5211828,"name":"offline","context":{"idset":"9030"}}
+{"timestamp":1714671434.5574069,"name":"offline","context":{"idset":"9031"}}
+{"timestamp":1714671436.4816997,"name":"offline","context":{"idset":"9032"}}
+{"timestamp":1714671436.4870102,"name":"offline","context":{"idset":"9033"}}
+{"timestamp":1714671436.5544591,"name":"offline","context":{"idset":"9034"}}
+{"timestamp":1714671438.4739459,"name":"offline","context":{"idset":"9036"}}
+{"timestamp":1714671438.4798346,"name":"offline","context":{"idset":"9038"}}
+{"timestamp":1714671438.5608444,"name":"offline","context":{"idset":"9039"}}
+{"timestamp":1714671440.4889581,"name":"offline","context":{"idset":"9035"}}
+{"timestamp":1714671440.4945214,"name":"offline","context":{"idset":"9040"}}
+{"timestamp":1714671440.4999027,"name":"offline","context":{"idset":"9041"}}
+{"timestamp":1714671440.5053959,"name":"offline","context":{"idset":"9042"}}
+{"timestamp":1714671440.5532808,"name":"offline","context":{"idset":"9043"}}
+{"timestamp":1714671442.4822721,"name":"offline","context":{"idset":"9044"}}
+{"timestamp":1714671442.4912546,"name":"offline","context":{"idset":"9045"}}
+{"timestamp":1714671442.4967382,"name":"offline","context":{"idset":"9046"}}
+{"timestamp":1714671442.5570762,"name":"offline","context":{"idset":"9047"}}
+{"timestamp":1714671444.4881284,"name":"offline","context":{"idset":"9048"}}
+{"timestamp":1714671444.4930329,"name":"offline","context":{"idset":"9049"}}
+{"timestamp":1714671444.5011771,"name":"offline","context":{"idset":"9050"}}
+{"timestamp":1714671444.5576265,"name":"offline","context":{"idset":"9051"}}
+{"timestamp":1714671446.4840672,"name":"offline","context":{"idset":"9052"}}
+{"timestamp":1714671446.4890583,"name":"offline","context":{"idset":"9053"}}
+{"timestamp":1714671446.4939847,"name":"offline","context":{"idset":"9054"}}
+{"timestamp":1714671446.5695689,"name":"offline","context":{"idset":"9055"}}
+{"timestamp":1714671448.4869318,"name":"offline","context":{"idset":"9056"}}
+{"timestamp":1714671448.4925354,"name":"offline","context":{"idset":"9057"}}
+{"timestamp":1714671448.4983838,"name":"offline","context":{"idset":"9058"}}
+{"timestamp":1714671448.5042324,"name":"offline","context":{"idset":"9059"}}
+{"timestamp":1714671448.552366,"name":"offline","context":{"idset":"9060"}}
+{"timestamp":1714671450.507777,"name":"offline","context":{"idset":"9062"}}
+{"timestamp":1714671450.5197201,"name":"offline","context":{"idset":"9063"}}
+{"timestamp":1714671450.5279727,"name":"offline","context":{"idset":"9064"}}
+{"timestamp":1714671450.5560553,"name":"offline","context":{"idset":"9065"}}
+{"timestamp":1714671452.4724293,"name":"offline","context":{"idset":"9066"}}
+{"timestamp":1714671452.4803436,"name":"offline","context":{"idset":"9067"}}
+{"timestamp":1714671452.552335,"name":"offline","context":{"idset":"9068"}}
+{"timestamp":1714671454.4820657,"name":"offline","context":{"idset":"9069"}}
+{"timestamp":1714671454.4866719,"name":"offline","context":{"idset":"9070"}}
+{"timestamp":1714671454.4912307,"name":"offline","context":{"idset":"9071"}}
+{"timestamp":1714671454.5780108,"name":"offline","context":{"idset":"9072"}}
+{"timestamp":1714671456.5039833,"name":"offline","context":{"idset":"9073"}}
+{"timestamp":1714671456.5123851,"name":"offline","context":{"idset":"9074"}}
+{"timestamp":1714671456.5223427,"name":"offline","context":{"idset":"9075"}}
+{"timestamp":1714671456.5597632,"name":"offline","context":{"idset":"9076"}}
+{"timestamp":1714671473.5891063,"name":"online","context":{"idset":"930"}}
+{"timestamp":1714671474.4111977,"name":"online","context":{"idset":"929"}}
+{"timestamp":1714671481.7621808,"name":"undrain","context":{"idset":"929-930"}}
+{"timestamp":1714671488.5507936,"name":"offline","context":{"idset":"9037"}}
+{"timestamp":1714671499.2973795,"name":"offline","context":{"idset":"9061"}}
+{"timestamp":1714672202.5534644,"name":"offline","context":{"idset":"7878"}}
+{"timestamp":1714672398.5513945,"name":"offline","context":{"idset":"6749"}}
+{"timestamp":1714672434.5513988,"name":"offline","context":{"idset":"895"}}
+{"timestamp":1714672436.5557137,"name":"offline","context":{"idset":"896"}}
+{"timestamp":1714672530.515281,"name":"offline","context":{"idset":"7157"}}
+{"timestamp":1714672530.5203104,"name":"offline","context":{"idset":"7158"}}
+{"timestamp":1714672530.5252838,"name":"offline","context":{"idset":"7159"}}
+{"timestamp":1714672530.5306234,"name":"offline","context":{"idset":"7160"}}
+{"timestamp":1714672530.5353577,"name":"offline","context":{"idset":"7161"}}
+{"timestamp":1714672530.5403824,"name":"offline","context":{"idset":"7162"}}
+{"timestamp":1714672530.5454836,"name":"offline","context":{"idset":"7163"}}
+{"timestamp":1714672530.5506902,"name":"offline","context":{"idset":"7164"}}
+{"timestamp":1714672530.5561993,"name":"offline","context":{"idset":"7165"}}
+{"timestamp":1714672530.561868,"name":"offline","context":{"idset":"7166"}}
+{"timestamp":1714672530.567344,"name":"offline","context":{"idset":"7168"}}
+{"timestamp":1714672530.5722995,"name":"offline","context":{"idset":"7169"}}
+{"timestamp":1714672530.5773518,"name":"offline","context":{"idset":"7170"}}
+{"timestamp":1714672530.5819855,"name":"offline","context":{"idset":"7171"}}
+{"timestamp":1714672530.586812,"name":"offline","context":{"idset":"7172"}}
+{"timestamp":1714672548.5510311,"name":"offline","context":{"idset":"7167"}}
+{"timestamp":1714674394.703825,"name":"online","context":{"idset":"775"}}
+{"timestamp":1714674401.8702452,"name":"online","context":{"idset":"776"}}
+{"timestamp":1714674460.4771752,"name":"offline","context":{"idset":"871"}}
+{"timestamp":1714674460.5513322,"name":"offline","context":{"idset":"872"}}
+{"timestamp":1714675432.5211115,"name":"offline","context":{"idset":"7237"}}
+{"timestamp":1714675432.5254071,"name":"offline","context":{"idset":"7238"}}
+{"timestamp":1714675432.5296199,"name":"offline","context":{"idset":"7239"}}
+{"timestamp":1714675432.5339694,"name":"offline","context":{"idset":"7240"}}
+{"timestamp":1714675432.5382173,"name":"offline","context":{"idset":"7241"}}
+{"timestamp":1714675432.5424457,"name":"offline","context":{"idset":"7242"}}
+{"timestamp":1714675432.5466509,"name":"offline","context":{"idset":"7243"}}
+{"timestamp":1714675432.5508568,"name":"offline","context":{"idset":"7244"}}
+{"timestamp":1714675432.5550706,"name":"offline","context":{"idset":"7245"}}
+{"timestamp":1714675432.5593159,"name":"offline","context":{"idset":"7246"}}
+{"timestamp":1714675432.5635362,"name":"offline","context":{"idset":"7247"}}
+{"timestamp":1714675432.5684319,"name":"offline","context":{"idset":"7248"}}
+{"timestamp":1714675432.5744054,"name":"offline","context":{"idset":"7249"}}
+{"timestamp":1714675432.5786407,"name":"offline","context":{"idset":"7250"}}
+{"timestamp":1714675432.5828714,"name":"offline","context":{"idset":"7251"}}
+{"timestamp":1714675432.5871196,"name":"offline","context":{"idset":"7252"}}
+{"timestamp":1714675804.5525084,"name":"offline","context":{"idset":"947"}}
+{"timestamp":1714675806.5506589,"name":"offline","context":{"idset":"948"}}
+{"timestamp":1714676018.5115674,"name":"offline","context":{"idset":"7269"}}
+{"timestamp":1714676018.5161586,"name":"offline","context":{"idset":"7270"}}
+{"timestamp":1714676018.5207324,"name":"offline","context":{"idset":"7271"}}
+{"timestamp":1714676018.5248444,"name":"offline","context":{"idset":"7272"}}
+{"timestamp":1714676018.5290923,"name":"offline","context":{"idset":"7273"}}
+{"timestamp":1714676018.533303,"name":"offline","context":{"idset":"7274"}}
+{"timestamp":1714676018.5376954,"name":"offline","context":{"idset":"7275"}}
+{"timestamp":1714676018.5417082,"name":"offline","context":{"idset":"7276"}}
+{"timestamp":1714676018.5458498,"name":"offline","context":{"idset":"7277"}}
+{"timestamp":1714676018.5497806,"name":"offline","context":{"idset":"7278"}}
+{"timestamp":1714676018.5535665,"name":"offline","context":{"idset":"7279"}}
+{"timestamp":1714676018.5579,"name":"offline","context":{"idset":"7280"}}
+{"timestamp":1714676018.5621009,"name":"offline","context":{"idset":"7281"}}
+{"timestamp":1714676018.5663674,"name":"offline","context":{"idset":"7282"}}
+{"timestamp":1714676018.5705731,"name":"offline","context":{"idset":"7283"}}
+{"timestamp":1714676018.57461,"name":"offline","context":{"idset":"7284"}}
+{"timestamp":1714676254.5529282,"name":"offline","context":{"idset":"8723"}}
+{"timestamp":1714676444.0813043,"name":"online","context":{"idset":"8950,8956"}}
+{"timestamp":1714676444.2334528,"name":"online","context":{"idset":"8963,8978"}}
+{"timestamp":1714676444.9816504,"name":"online","context":{"idset":"8964,8980"}}
+{"timestamp":1714676445.1585116,"name":"online","context":{"idset":"8988"}}
+{"timestamp":1714676445.6544676,"name":"online","context":{"idset":"9001"}}
+{"timestamp":1714676446.5722153,"name":"online","context":{"idset":"8967"}}
+{"timestamp":1714676446.7428751,"name":"online","context":{"idset":"8955"}}
+{"timestamp":1714676447.0225444,"name":"online","context":{"idset":"8975"}}
+{"timestamp":1714676447.1552448,"name":"online","context":{"idset":"8953"}}
+{"timestamp":1714676447.4433479,"name":"online","context":{"idset":"8962"}}
+{"timestamp":1714676447.6126065,"name":"online","context":{"idset":"8971,8974"}}
+{"timestamp":1714676447.7351665,"name":"online","context":{"idset":"8954"}}
+{"timestamp":1714676447.7517331,"name":"online","context":{"idset":"8952,8959,9005"}}
+{"timestamp":1714676447.8987427,"name":"online","context":{"idset":"9018"}}
+{"timestamp":1714676448.1425312,"name":"online","context":{"idset":"8973,8991"}}
+{"timestamp":1714676448.3963883,"name":"online","context":{"idset":"8949,8957-8958,9040,9069,9074"}}
+{"timestamp":1714676448.7236025,"name":"online","context":{"idset":"8961,8986,9007"}}
+{"timestamp":1714676449.1373737,"name":"online","context":{"idset":"8979,9015,9031,9062,9075"}}
+{"timestamp":1714676449.2908685,"name":"online","context":{"idset":"8970"}}
+{"timestamp":1714676449.4561968,"name":"online","context":{"idset":"8960,8965-8966,8969,8972,8976-8977,8989,9008,9030,9037,9042,9055,9059-9061,9065,9067"}}
+{"timestamp":1714676449.561538,"name":"online","context":{"idset":"8951"}}
+{"timestamp":1714676449.8606505,"name":"online","context":{"idset":"8968,8992,9017,9022,9026,9032,9035-9036,9046,9051,9057,9063,9066,9072-9073,9076"}}
+{"timestamp":1714676450.0007367,"name":"online","context":{"idset":"9029"}}
+{"timestamp":1714676450.247612,"name":"online","context":{"idset":"8982,8995,9020,9033,9038-9039,9041,9044-9045,9048-9050,9056,9064,9068,9070-9071"}}
+{"timestamp":1714676450.3665614,"name":"online","context":{"idset":"9034,9054"}}
+{"timestamp":1714676450.500066,"name":"online","context":{"idset":"8987,9014,9043,9047,9052"}}
+{"timestamp":1714676450.7807245,"name":"online","context":{"idset":"9004,9016,9053,9058"}}
+{"timestamp":1714676451.3058608,"name":"online","context":{"idset":"8981,8983-8984,9019"}}
+{"timestamp":1714676451.5231884,"name":"online","context":{"idset":"8985,8990,8993-8994,8996,8999-9000,9009-9011,9013,9021,9023-9025,9027-9028"}}
+{"timestamp":1714676451.9430792,"name":"online","context":{"idset":"8997-8998,9002-9003,9006,9012"}}
+{"timestamp":1714676959.3039386,"name":"online","context":{"idset":"10106"}}
+{"timestamp":1714676959.5844855,"name":"online","context":{"idset":"10118"}}
+{"timestamp":1714676959.626452,"name":"online","context":{"idset":"10115"}}
+{"timestamp":1714676959.8228161,"name":"online","context":{"idset":"10105,10110,10114"}}
+{"timestamp":1714676959.9503927,"name":"online","context":{"idset":"10104"}}
+{"timestamp":1714676960.118223,"name":"online","context":{"idset":"10107"}}
+{"timestamp":1714676960.1473916,"name":"online","context":{"idset":"10113"}}
+{"timestamp":1714676960.1752648,"name":"online","context":{"idset":"10103,10117,10144"}}
+{"timestamp":1714676960.4359877,"name":"online","context":{"idset":"10108,10111,10132"}}
+{"timestamp":1714676960.485857,"name":"drain","context":{"idset":"10139","reason":"broker was unresponsive"}}
+{"timestamp":1714676960.4883373,"name":"drain","context":{"idset":"10160","reason":"broker was unresponsive"}}
+{"timestamp":1714676960.4888241,"name":"drain","context":{"idset":"10183","reason":"broker was unresponsive"}}
+{"timestamp":1714676960.4889908,"name":"drain","context":{"idset":"10196","reason":"broker was unresponsive"}}
+{"timestamp":1714676960.4892137,"name":"drain","context":{"idset":"10197","reason":"broker was unresponsive"}}
+{"timestamp":1714676960.5567362,"name":"online","context":{"idset":"10112,10125-10126"}}
+{"timestamp":1714676960.9107118,"name":"online","context":{"idset":"10116,10120,10133"}}
+{"timestamp":1714676961.2438388,"name":"online","context":{"idset":"10119,10121-10122,10131,10136-10138,10140"}}
+{"timestamp":1714676961.28508,"name":"online","context":{"idset":"10101,10164"}}
+{"timestamp":1714676961.5525768,"name":"online","context":{"idset":"10129,10142,10154"}}
+{"timestamp":1714676961.5639114,"name":"online","context":{"idset":"10123"}}
+{"timestamp":1714676961.6437583,"name":"online","context":{"idset":"10109"}}
+{"timestamp":1714676961.8986835,"name":"online","context":{"idset":"10124,10128,10141,10151,10155"}}
+{"timestamp":1714676962.2076578,"name":"online","context":{"idset":"10102,10145,10148,10213"}}
+{"timestamp":1714676962.3852963,"name":"online","context":{"idset":"10127,10130,10179,10190"}}
+{"timestamp":1714676962.4297636,"name":"online","context":{"idset":"10170"}}
+{"timestamp":1714676962.7211983,"name":"online","context":{"idset":"10161,10197"}}
+{"timestamp":1714676963.0179656,"name":"online","context":{"idset":"10139,10147,10156,10159-10160,10169,10171,10175,10178,10183,10196"}}
+{"timestamp":1714676963.2500904,"name":"online","context":{"idset":"10153,10157,10162-10163,10168,10188-10189,10203,10206,10209,10218,10225"}}
+{"timestamp":1714676963.4823806,"name":"online","context":{"idset":"10135,10146,10167,10201,10207,10212,10220,10227-10228"}}
+{"timestamp":1714676963.6012414,"name":"online","context":{"idset":"10215"}}
+{"timestamp":1714676963.8240197,"name":"online","context":{"idset":"10143,10152,10158,10166,10176,10185,10226"}}
+{"timestamp":1714676964.056474,"name":"online","context":{"idset":"10149-10150,10173-10174,10180,10184,10204,10211,10216-10217,10223"}}
+{"timestamp":1714676964.1853797,"name":"online","context":{"idset":"10165,10172,10193,10205,10210,10219"}}
+{"timestamp":1714676964.4038467,"name":"online","context":{"idset":"10177,10181-10182,10191-10192,10195,10199-10200,10208,10214,10221,10224"}}
+{"timestamp":1714676964.6630354,"name":"online","context":{"idset":"10186-10187,10194,10198,10202,10222"}}
+{"timestamp":1714677102.0315742,"name":"online","context":{"idset":"801"}}
+{"timestamp":1714677145.5832741,"name":"online","context":{"idset":"802"}}
+{"timestamp":1714677214.2882559,"name":"online","context":{"idset":"11570"}}
+{"timestamp":1714677214.5344639,"name":"online","context":{"idset":"11569"}}
+{"timestamp":1714677232.2709444,"name":"online","context":{"idset":"8723,8744"}}
+{"timestamp":1714677232.8313134,"name":"online","context":{"idset":"8743"}}
+{"timestamp":1714677232.9952273,"name":"online","context":{"idset":"7453"}}
+{"timestamp":1714677233.6307292,"name":"online","context":{"idset":"6749,7427-7428,7527"}}
+{"timestamp":1714677234.0089495,"name":"online","context":{"idset":"7454,7528"}}
+{"timestamp":1714677234.6858709,"name":"online","context":{"idset":"6750"}}
+{"timestamp":1714677239.9934552,"name":"drain","context":{"idset":"8723,8743-8744","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714677242.1969185,"name":"drain","context":{"idset":"7427-7428,7453-7454,7527-7528","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714677242.3336241,"name":"drain","context":{"idset":"6749-6750","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714677398.9260595,"name":"online","context":{"idset":"10964"}}
+{"timestamp":1714677468.5543623,"name":"offline","context":{"idset":"7014"}}
+{"timestamp":1714677500.5515449,"name":"offline","context":{"idset":"929"}}
+{"timestamp":1714677502.555079,"name":"offline","context":{"idset":"930"}}
+{"timestamp":1714677505.4257109,"name":"offline","context":{"idset":"9048"}}
+{"timestamp":1714677916.4768176,"name":"offline","context":{"idset":"1045"}}
+{"timestamp":1714677916.5573471,"name":"offline","context":{"idset":"1046"}}
+{"timestamp":1714677999.0630696,"name":"online","context":{"idset":"7159,7164,7172"}}
+{"timestamp":1714677999.1969373,"name":"online","context":{"idset":"7168"}}
+{"timestamp":1714677999.4948409,"name":"online","context":{"idset":"7167"}}
+{"timestamp":1714677999.781364,"name":"online","context":{"idset":"7160,7163,7165-7166,7169"}}
+{"timestamp":1714678000.0731168,"name":"online","context":{"idset":"7158,7162,7170-7171"}}
+{"timestamp":1714678000.3299768,"name":"online","context":{"idset":"7161"}}
+{"timestamp":1714679710.5522342,"name":"offline","context":{"idset":"579"}}
+{"timestamp":1714679861.6006627,"name":"online","context":{"idset":"9048"}}
+{"timestamp":1714680044.9634681,"name":"online","context":{"idset":"7157"}}
+{"timestamp":1714680244.4256771,"name":"online","context":{"idset":"578"}}
+{"timestamp":1714680334.8520133,"name":"online","context":{"idset":"541"}}
+{"timestamp":1714680343.8324673,"name":"online","context":{"idset":"579"}}
+{"timestamp":1714680711.8842757,"name":"online","context":{"idset":"1050"}}
+{"timestamp":1714680712.9936175,"name":"online","context":{"idset":"1049"}}
+{"timestamp":1714680969.916939,"name":"online","context":{"idset":"942"}}
+{"timestamp":1714680970.7607715,"name":"online","context":{"idset":"941"}}
+{"timestamp":1714681252.0384943,"name":"online","context":{"idset":"10087"}}
+{"timestamp":1714681252.2636039,"name":"online","context":{"idset":"10064"}}
+{"timestamp":1714681252.3970387,"name":"online","context":{"idset":"10088"}}
+{"timestamp":1714681253.4154944,"name":"online","context":{"idset":"10066,10090,10569"}}
+{"timestamp":1714681253.4221401,"name":"online","context":{"idset":"10584"}}
+{"timestamp":1714681253.4274817,"name":"online","context":{"idset":"10061,10069,10083,10485,10490-10491,10528-10529,10553,10581"}}
+{"timestamp":1714681253.5261874,"name":"online","context":{"idset":"10080,10085,10561"}}
+{"timestamp":1714681253.7643158,"name":"online","context":{"idset":"10092,10486,10497,10542,10576,10607"}}
+{"timestamp":1714681254.0546875,"name":"online","context":{"idset":"10078,10487,10504,10518,10547,10550,10565,10568,10578,10598,10602-10603"}}
+{"timestamp":1714681254.1670287,"name":"online","context":{"idset":"10590"}}
+{"timestamp":1714681254.2762742,"name":"online","context":{"idset":"10100,10507,10512,10605,10612"}}
+{"timestamp":1714681254.3935668,"name":"online","context":{"idset":"10496"}}
+{"timestamp":1714681254.5040638,"name":"online","context":{"idset":"10531"}}
+{"timestamp":1714681255.1606469,"name":"online","context":{"idset":"10587"}}
+{"timestamp":1714681255.1641445,"name":"online","context":{"idset":"10503,10597"}}
+{"timestamp":1714681391.576385,"name":"undrain","context":{"idset":"10503-10504"}}
+{"timestamp":1714681490.5962431,"name":"online","context":{"idset":"10963"}}
+{"timestamp":1714682117.9495258,"name":"online","context":{"idset":"7878"}}
+{"timestamp":1714682215.485822,"name":"online","context":{"idset":"6963"}}
+{"timestamp":1714682384.0318334,"name":"online","context":{"idset":"7758"}}
+{"timestamp":1714682389.3441672,"name":"online","context":{"idset":"7757"}}
+{"timestamp":1714682696.0312655,"name":"undrain","context":{"idset":"6749-6750"}}
+{"timestamp":1714682697.3891377,"name":"undrain","context":{"idset":"7427-7428,7453-7454,7527-7528"}}
+{"timestamp":1714682698.3703537,"name":"undrain","context":{"idset":"8723,8743-8744"}}
+{"timestamp":1714682853.392781,"name":"online","context":{"idset":"11790"}}
+{"timestamp":1714682853.4710016,"name":"online","context":{"idset":"10003"}}
+{"timestamp":1714682853.7508585,"name":"online","context":{"idset":"10004"}}
+{"timestamp":1714683194.5545206,"name":"offline","context":{"idset":"7758"}}
+{"timestamp":1714683486.135206,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1714683748.4085474,"name":"drain","context":{"idset":"7125-7140","overwrite":0}}
+{"timestamp":1714683918.4713168,"name":"offline","context":{"idset":"7125"}}
+{"timestamp":1714683918.4714222,"name":"drain","context":{"idset":"7125","reason":"epilog failed for jobid fsvRBrV66WK","overwrite":0}}
+{"timestamp":1714683918.5526409,"name":"offline","context":{"idset":"7126"}}
+{"timestamp":1714683920.4860892,"name":"offline","context":{"idset":"7127"}}
+{"timestamp":1714683920.4953785,"name":"offline","context":{"idset":"7128"}}
+{"timestamp":1714683920.556251,"name":"offline","context":{"idset":"7129"}}
+{"timestamp":1714683922.494216,"name":"offline","context":{"idset":"7130"}}
+{"timestamp":1714683922.5027092,"name":"offline","context":{"idset":"7131"}}
+{"timestamp":1714683922.5100417,"name":"offline","context":{"idset":"7132"}}
+{"timestamp":1714683922.5580018,"name":"offline","context":{"idset":"7133"}}
+{"timestamp":1714683924.4764957,"name":"offline","context":{"idset":"7134"}}
+{"timestamp":1714683924.4847074,"name":"offline","context":{"idset":"7135"}}
+{"timestamp":1714683924.5687742,"name":"offline","context":{"idset":"7136"}}
+{"timestamp":1714683926.4850259,"name":"offline","context":{"idset":"7137"}}
+{"timestamp":1714683926.4895954,"name":"offline","context":{"idset":"7138"}}
+{"timestamp":1714683926.494199,"name":"offline","context":{"idset":"7139"}}
+{"timestamp":1714683926.5527048,"name":"offline","context":{"idset":"7140"}}
+{"timestamp":1714684096.1959593,"name":"online","context":{"idset":"93"}}
+{"timestamp":1714684538.74667,"name":"undrain","context":{"idset":"10139"}}
+{"timestamp":1714684547.8346643,"name":"undrain","context":{"idset":"10160"}}
+{"timestamp":1714684557.5239584,"name":"undrain","context":{"idset":"10196"}}
+{"timestamp":1714684559.7467082,"name":"undrain","context":{"idset":"10197"}}
+{"timestamp":1714684685.4259775,"name":"online","context":{"idset":"6953"}}
+{"timestamp":1714684947.5989535,"name":"undrain","context":{"idset":"10183"}}
+{"timestamp":1714684956.5166032,"name":"undrain","context":{"idset":"10001-10002"}}
+{"timestamp":1714685013.7481856,"name":"online","context":{"idset":"6682"}}
+{"timestamp":1714685014.2684505,"name":"online","context":{"idset":"6681"}}
+{"timestamp":1714685016.3798747,"name":"undrain","context":{"idset":"11250"}}
+{"timestamp":1714685019.5900578,"name":"undrain","context":{"idset":"11251"}}
+{"timestamp":1714685023.7056031,"name":"offline","context":{"idset":"9048"}}
+{"timestamp":1714685049.831475,"name":"online","context":{"idset":"10134"}}
+{"timestamp":1714685079.4737813,"name":"drain","context":{"idset":"10134","reason":"failed bogomips -kk","overwrite":0}}
+{"timestamp":1714687405.3747287,"name":"online","context":{"idset":"1059"}}
+{"timestamp":1714687405.3800468,"name":"online","context":{"idset":"1060"}}
+{"timestamp":1714687638.2971368,"name":"drain","context":{"idset":"946","reason":"--reason draining to run on-node diags - EG","overwrite":0}}
+{"timestamp":1714687861.8822696,"name":"online","context":{"idset":"871-872"}}
+{"timestamp":1714687912.4586897,"name":"drain","context":{"idset":"10154","reason":"broker was unresponsive"}}
+{"timestamp":1714687912.4588144,"name":"drain","context":{"idset":"10158","reason":"broker was unresponsive"}}
+{"timestamp":1714687912.5589328,"name":"drain","context":{"idset":"10185","reason":"broker was unresponsive"}}
+{"timestamp":1714688021.3474894,"name":"drain","context":{"idset":"10108-10133,10135-10228","reason":"epilog failed for jobid fsxh6hkuefZ","overwrite":0}}
+{"timestamp":1714689048.973773,"name":"online","context":{"idset":"796"}}
+{"timestamp":1714689049.5211573,"name":"online","context":{"idset":"795"}}
+{"timestamp":1714689205.0182211,"name":"online","context":{"idset":"1061-1062"}}
+{"timestamp":1714689391.879854,"name":"drain","context":{"idset":"1070","reason":"Swapping Cassini cards","overwrite":0}}
+{"timestamp":1714689405.1751857,"name":"drain","context":{"idset":"1069","reason":"Bad Partner Node","overwrite":0}}
+{"timestamp":1714689554.5520341,"name":"offline","context":{"idset":"9535"}}
+{"timestamp":1714689648.9400508,"name":"offline","context":{"idset":"8144"}}
+{"timestamp":1714689902.7020118,"name":"online","context":{"idset":"943"}}
+{"timestamp":1714689902.8421514,"name":"online","context":{"idset":"944"}}
+{"timestamp":1714689912.8425739,"name":"drain","context":{"idset":"11736","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714689915.9241619,"name":"drain","context":{"idset":"11646","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714689921.0110853,"name":"drain","context":{"idset":"11639","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714689921.8159273,"name":"drain","context":{"idset":"11742","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714689926.7846892,"name":"drain","context":{"idset":"11692","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1714690108.1206672,"name":"offline","context":{"idset":"9604"}}
+{"timestamp":1714690108.1620877,"name":"offline","context":{"idset":"9598"}}
+{"timestamp":1714690108.170562,"name":"offline","context":{"idset":"9594"}}
+{"timestamp":1714690108.269928,"name":"offline","context":{"idset":"9590"}}
+{"timestamp":1714690108.4119475,"name":"offline","context":{"idset":"9592"}}
+{"timestamp":1714690108.4184756,"name":"offline","context":{"idset":"9610"}}
+{"timestamp":1714690108.4231677,"name":"offline","context":{"idset":"9595"}}
+{"timestamp":1714690108.4301219,"name":"offline","context":{"idset":"9611"}}
+{"timestamp":1714690108.4401124,"name":"offline","context":{"idset":"9625"}}
+{"timestamp":1714690108.4490924,"name":"offline","context":{"idset":"9628"}}
+{"timestamp":1714690108.4613571,"name":"offline","context":{"idset":"9600"}}
+{"timestamp":1714690108.5408618,"name":"offline","context":{"idset":"9607"}}
+{"timestamp":1714690108.5490489,"name":"offline","context":{"idset":"9591"}}
+{"timestamp":1714690108.5570931,"name":"offline","context":{"idset":"9596"}}
+{"timestamp":1714690108.5651059,"name":"offline","context":{"idset":"9589"}}
+{"timestamp":1714690108.5732071,"name":"offline","context":{"idset":"9593"}}
+{"timestamp":1714690108.5813837,"name":"offline","context":{"idset":"9602"}}
+{"timestamp":1714690108.5899665,"name":"offline","context":{"idset":"9612"}}
+{"timestamp":1714690108.5978398,"name":"offline","context":{"idset":"9619"}}
+{"timestamp":1714690108.606195,"name":"offline","context":{"idset":"9622"}}
+{"timestamp":1714690108.7866867,"name":"offline","context":{"idset":"9629"}}
+{"timestamp":1714690109.0471504,"name":"offline","context":{"idset":"9606"}}
+{"timestamp":1714690109.1333337,"name":"offline","context":{"idset":"9624"}}
+{"timestamp":1714690109.1514826,"name":"offline","context":{"idset":"9601"}}
+{"timestamp":1714690109.8319623,"name":"offline","context":{"idset":"9638"}}
+{"timestamp":1714690109.8411396,"name":"offline","context":{"idset":"9597"}}
+{"timestamp":1714690109.8777077,"name":"offline","context":{"idset":"9599"}}
+{"timestamp":1714690109.898767,"name":"offline","context":{"idset":"9603"}}
+{"timestamp":1714690109.9077809,"name":"offline","context":{"idset":"9605"}}
+{"timestamp":1714690109.9128814,"name":"offline","context":{"idset":"9608"}}
+{"timestamp":1714690109.9180436,"name":"offline","context":{"idset":"9609"}}
+{"timestamp":1714690109.9231336,"name":"offline","context":{"idset":"9613"}}
+{"timestamp":1714690109.9287014,"name":"offline","context":{"idset":"9614"}}
+{"timestamp":1714690109.9343133,"name":"offline","context":{"idset":"9615"}}
+{"timestamp":1714690109.9396808,"name":"offline","context":{"idset":"9616"}}
+{"timestamp":1714690109.965446,"name":"offline","context":{"idset":"9617"}}
+{"timestamp":1714690109.9703922,"name":"offline","context":{"idset":"9618"}}
+{"timestamp":1714690109.9756455,"name":"offline","context":{"idset":"9620"}}
+{"timestamp":1714690109.9808836,"name":"offline","context":{"idset":"9621"}}
+{"timestamp":1714690109.9858515,"name":"offline","context":{"idset":"9623"}}
+{"timestamp":1714690109.9908376,"name":"offline","context":{"idset":"9626"}}
+{"timestamp":1714690109.9962351,"name":"offline","context":{"idset":"9627"}}
+{"timestamp":1714690110.0016651,"name":"offline","context":{"idset":"9630"}}
+{"timestamp":1714690110.0187795,"name":"offline","context":{"idset":"9631"}}
+{"timestamp":1714690110.0237648,"name":"offline","context":{"idset":"9632"}}
+{"timestamp":1714690110.0289431,"name":"offline","context":{"idset":"9633"}}
+{"timestamp":1714690110.0343275,"name":"offline","context":{"idset":"9634"}}
+{"timestamp":1714690110.039324,"name":"offline","context":{"idset":"9635"}}
+{"timestamp":1714690110.0447187,"name":"offline","context":{"idset":"9636"}}
+{"timestamp":1714690110.0501676,"name":"offline","context":{"idset":"9637"}}
+{"timestamp":1714690110.0553279,"name":"offline","context":{"idset":"9639"}}
+{"timestamp":1714690110.0604918,"name":"offline","context":{"idset":"9640"}}
+{"timestamp":1714690110.0654297,"name":"offline","context":{"idset":"9641"}}
+{"timestamp":1714690110.0704772,"name":"offline","context":{"idset":"9642"}}
+{"timestamp":1714690110.0755615,"name":"offline","context":{"idset":"9643"}}
+{"timestamp":1714690110.0804739,"name":"offline","context":{"idset":"9644"}}
+{"timestamp":1714690110.0856395,"name":"offline","context":{"idset":"9645"}}
+{"timestamp":1714690110.0910633,"name":"offline","context":{"idset":"9646"}}
+{"timestamp":1714690110.0964754,"name":"offline","context":{"idset":"9647"}}
+{"timestamp":1714690110.1019092,"name":"offline","context":{"idset":"9648"}}
+{"timestamp":1714690110.1073751,"name":"offline","context":{"idset":"9649"}}
+{"timestamp":1714690110.1123772,"name":"offline","context":{"idset":"9650"}}
+{"timestamp":1714690110.1173892,"name":"offline","context":{"idset":"9651"}}
+{"timestamp":1714690110.1261532,"name":"offline","context":{"idset":"9652"}}
+{"timestamp":1714690110.1466248,"name":"offline","context":{"idset":"9653"}}
+{"timestamp":1714690110.1617424,"name":"offline","context":{"idset":"9654"}}
+{"timestamp":1714690110.1670856,"name":"offline","context":{"idset":"9655"}}
+{"timestamp":1714690110.1719849,"name":"offline","context":{"idset":"9656"}}
+{"timestamp":1714690110.1768909,"name":"offline","context":{"idset":"9657"}}
+{"timestamp":1714690110.1839938,"name":"offline","context":{"idset":"9658"}}
+{"timestamp":1714690110.1893835,"name":"offline","context":{"idset":"9659"}}
+{"timestamp":1714690110.1946471,"name":"offline","context":{"idset":"9660"}}
+{"timestamp":1714690110.1996961,"name":"offline","context":{"idset":"9661"}}
+{"timestamp":1714690110.2048171,"name":"offline","context":{"idset":"9662"}}
+{"timestamp":1714690110.2098398,"name":"offline","context":{"idset":"9663"}}
+{"timestamp":1714690110.2151248,"name":"offline","context":{"idset":"9664"}}
+{"timestamp":1714690110.2204754,"name":"offline","context":{"idset":"9665"}}
+{"timestamp":1714690110.2294204,"name":"offline","context":{"idset":"9666"}}
+{"timestamp":1714690110.23436,"name":"offline","context":{"idset":"9667"}}
+{"timestamp":1714690110.2394879,"name":"offline","context":{"idset":"9668"}}
+{"timestamp":1714690110.2465725,"name":"offline","context":{"idset":"9669"}}
+{"timestamp":1714690110.2514594,"name":"offline","context":{"idset":"9670"}}
+{"timestamp":1714690110.25631,"name":"offline","context":{"idset":"9671"}}
+{"timestamp":1714690110.274188,"name":"offline","context":{"idset":"9672"}}
+{"timestamp":1714690110.2793686,"name":"offline","context":{"idset":"9673"}}
+{"timestamp":1714690110.2912908,"name":"offline","context":{"idset":"9674"}}
+{"timestamp":1714690110.2963765,"name":"offline","context":{"idset":"9675"}}
+{"timestamp":1714690110.3056803,"name":"offline","context":{"idset":"9676"}}
+{"timestamp":1714690110.3113441,"name":"offline","context":{"idset":"9677"}}
+{"timestamp":1714690110.3239243,"name":"offline","context":{"idset":"9678"}}
+{"timestamp":1714690110.3350186,"name":"offline","context":{"idset":"9679"}}
+{"timestamp":1714690110.340306,"name":"offline","context":{"idset":"9680"}}
+{"timestamp":1714690110.3453474,"name":"offline","context":{"idset":"9681"}}
+{"timestamp":1714690110.361146,"name":"offline","context":{"idset":"9682"}}
+{"timestamp":1714690110.3893452,"name":"offline","context":{"idset":"9683"}}
+{"timestamp":1714690110.3947558,"name":"offline","context":{"idset":"9684"}}
+{"timestamp":1714690110.4006426,"name":"offline","context":{"idset":"9685"}}
+{"timestamp":1714690110.4060185,"name":"offline","context":{"idset":"9686"}}
+{"timestamp":1714690110.4114172,"name":"offline","context":{"idset":"9687"}}
+{"timestamp":1714690110.4168577,"name":"offline","context":{"idset":"9688"}}
+{"timestamp":1714690110.4268544,"name":"offline","context":{"idset":"9689"}}
+{"timestamp":1714690110.4326842,"name":"offline","context":{"idset":"9690"}}
+{"timestamp":1714690110.4382508,"name":"offline","context":{"idset":"9691"}}
+{"timestamp":1714690110.4441278,"name":"offline","context":{"idset":"9692"}}
+{"timestamp":1714690110.4489474,"name":"offline","context":{"idset":"9693"}}
+{"timestamp":1714690110.4551399,"name":"offline","context":{"idset":"9694"}}
+{"timestamp":1714690110.4613175,"name":"offline","context":{"idset":"9695"}}
+{"timestamp":1714690110.4668419,"name":"offline","context":{"idset":"9696"}}
+{"timestamp":1714690110.4722402,"name":"offline","context":{"idset":"9697"}}
+{"timestamp":1714690110.4777017,"name":"offline","context":{"idset":"9698"}}
+{"timestamp":1714690110.4900148,"name":"offline","context":{"idset":"9699"}}
+{"timestamp":1714690110.5143344,"name":"offline","context":{"idset":"9700"}}
+{"timestamp":1714690110.5196428,"name":"offline","context":{"idset":"9701"}}
+{"timestamp":1714690110.5280423,"name":"offline","context":{"idset":"9702"}}
+{"timestamp":1714690110.5327308,"name":"offline","context":{"idset":"9703"}}
+{"timestamp":1714690110.5384715,"name":"offline","context":{"idset":"9704"}}
+{"timestamp":1714690110.5432072,"name":"offline","context":{"idset":"9705"}}
+{"timestamp":1714690110.5479748,"name":"offline","context":{"idset":"9706"}}
+{"timestamp":1714690110.5528555,"name":"offline","context":{"idset":"9707"}}
+{"timestamp":1714690110.5576551,"name":"offline","context":{"idset":"9708"}}
+{"timestamp":1714690110.5629213,"name":"offline","context":{"idset":"9709"}}
+{"timestamp":1714690110.5678151,"name":"offline","context":{"idset":"9710"}}
+{"timestamp":1714690110.5750952,"name":"offline","context":{"idset":"9711"}}
+{"timestamp":1714690110.5800209,"name":"offline","context":{"idset":"9712"}}
+{"timestamp":1714690110.5847542,"name":"offline","context":{"idset":"9713"}}
+{"timestamp":1714690110.5896893,"name":"offline","context":{"idset":"9714"}}
+{"timestamp":1714690110.5946105,"name":"offline","context":{"idset":"9715"}}
+{"timestamp":1714690110.5995002,"name":"offline","context":{"idset":"9716"}}
+{"timestamp":1714690732.1066988,"name":"undrain","context":{"idset":"10108-10133,10135-10153,10155-10157,10159-10184,10186-10228"}}
+{"timestamp":1714691301.5406024,"name":"undrain","context":{"idset":"11639,11646,11692,11736,11742"}}
+{"timestamp":1714691568.5562713,"name":"offline","context":{"idset":"7757"}}
+{"timestamp":1714691602.4760239,"name":"offline","context":{"idset":"961"}}
+{"timestamp":1714691602.5527246,"name":"offline","context":{"idset":"962"}}
+{"timestamp":1714691626.4778118,"name":"offline","context":{"idset":"10154"}}
+{"timestamp":1714691626.4833269,"name":"offline","context":{"idset":"10158"}}
+{"timestamp":1714691626.5750196,"name":"offline","context":{"idset":"10185"}}
+{"timestamp":1714692171.1057055,"name":"undrain","context":{"idset":"10134"}}
+{"timestamp":1714692476.3493638,"name":"online","context":{"idset":"9589"}}
+{"timestamp":1714692476.7047548,"name":"online","context":{"idset":"9595"}}
+{"timestamp":1714692476.978668,"name":"online","context":{"idset":"9605"}}
+{"timestamp":1714692477.1036847,"name":"online","context":{"idset":"9590,9592,9626"}}
+{"timestamp":1714692477.2547295,"name":"online","context":{"idset":"9593,9598,9609"}}
+{"timestamp":1714692478.0438557,"name":"online","context":{"idset":"9619"}}
+{"timestamp":1714692478.8831203,"name":"online","context":{"idset":"9611"}}
+{"timestamp":1714692479.3411989,"name":"online","context":{"idset":"9607"}}
+{"timestamp":1714692479.3535793,"name":"online","context":{"idset":"9613"}}
+{"timestamp":1714692479.756685,"name":"online","context":{"idset":"9601-9604,9632"}}
+{"timestamp":1714692479.9865572,"name":"online","context":{"idset":"9591,9594"}}
+{"timestamp":1714692480.1349547,"name":"online","context":{"idset":"9627"}}
+{"timestamp":1714692480.2049832,"name":"online","context":{"idset":"9631"}}
+{"timestamp":1714692480.6045153,"name":"online","context":{"idset":"9599,9638,9687"}}
+{"timestamp":1714692480.9829853,"name":"online","context":{"idset":"9600,9615-9617,9625,9629-9630,9636,9640,9644,9646,9653,9658,9676-9677,9693"}}
+{"timestamp":1714692481.2470989,"name":"online","context":{"idset":"9608,9612,9620,9696,9698-9700"}}
+{"timestamp":1714692481.6092484,"name":"online","context":{"idset":"9597,9622,9662"}}
+{"timestamp":1714692481.8721185,"name":"online","context":{"idset":"9596,9606,9614,9618,9621,9623-9624,9633-9634,9641,9645,9648,9652,9659,9665,9669,9672,9692,9694,9706"}}
+{"timestamp":1714692481.9786415,"name":"online","context":{"idset":"9610,9642,9671,9675"}}
+{"timestamp":1714692482.2543399,"name":"online","context":{"idset":"9628,9635,9639,9643,9647,9649-9651,9654-9657,9660-9661,9664,9666,9668,9670,9673-9674,9678,9680,9682-9685,9688-9691,9695,9697,9704-9705,9707-9711,9714"}}
+{"timestamp":1714692482.5502858,"name":"online","context":{"idset":"9663"}}
+{"timestamp":1714692482.6619239,"name":"online","context":{"idset":"9667,9681,9701-9702,9716"}}
+{"timestamp":1714692483.0414474,"name":"online","context":{"idset":"9637,9679,9686,9703,9712,9715"}}
+{"timestamp":1714692483.2565136,"name":"online","context":{"idset":"9713"}}
+{"timestamp":1714692573.2743092,"name":"offline","context":{"idset":"93"}}
+{"timestamp":1714692573.2977476,"name":"offline","context":{"idset":"98"}}
+{"timestamp":1714692573.3131423,"name":"offline","context":{"idset":"112"}}
+{"timestamp":1714692573.3189726,"name":"offline","context":{"idset":"96"}}
+{"timestamp":1714692573.416429,"name":"offline","context":{"idset":"101"}}
+{"timestamp":1714692573.5897148,"name":"offline","context":{"idset":"94"}}
+{"timestamp":1714692573.6022809,"name":"offline","context":{"idset":"108"}}
+{"timestamp":1714692573.6170108,"name":"offline","context":{"idset":"95"}}
+{"timestamp":1714692573.621701,"name":"offline","context":{"idset":"97"}}
+{"timestamp":1714692573.6352515,"name":"offline","context":{"idset":"99"}}
+{"timestamp":1714692573.6400573,"name":"offline","context":{"idset":"100"}}
+{"timestamp":1714692573.6454322,"name":"offline","context":{"idset":"102"}}
+{"timestamp":1714692573.6501899,"name":"offline","context":{"idset":"105"}}
+{"timestamp":1714692573.6547558,"name":"offline","context":{"idset":"106"}}
+{"timestamp":1714692573.7384272,"name":"offline","context":{"idset":"107"}}
+{"timestamp":1714692573.7436466,"name":"offline","context":{"idset":"109"}}
+{"timestamp":1714692573.7488654,"name":"offline","context":{"idset":"110"}}
+{"timestamp":1714692573.7541707,"name":"offline","context":{"idset":"111"}}
+{"timestamp":1714692573.7695332,"name":"offline","context":{"idset":"113"}}
+{"timestamp":1714692573.7747538,"name":"offline","context":{"idset":"114"}}
+{"timestamp":1714692573.779999,"name":"offline","context":{"idset":"115"}}
+{"timestamp":1714692573.7852314,"name":"offline","context":{"idset":"116"}}
+{"timestamp":1714692573.7906129,"name":"offline","context":{"idset":"119"}}
+{"timestamp":1714692573.7964618,"name":"offline","context":{"idset":"122"}}
+{"timestamp":1714692573.8022084,"name":"offline","context":{"idset":"123"}}
+{"timestamp":1714692573.8082833,"name":"offline","context":{"idset":"124"}}
+{"timestamp":1714692573.813323,"name":"offline","context":{"idset":"125"}}
+{"timestamp":1714692573.8185797,"name":"offline","context":{"idset":"126"}}
+{"timestamp":1714692573.8235662,"name":"offline","context":{"idset":"127"}}
+{"timestamp":1714692573.8286092,"name":"offline","context":{"idset":"128"}}
+{"timestamp":1714692573.8336141,"name":"offline","context":{"idset":"129"}}
+{"timestamp":1714692573.8385243,"name":"offline","context":{"idset":"130"}}
+{"timestamp":1714692573.8433321,"name":"offline","context":{"idset":"131"}}
+{"timestamp":1714692573.8480005,"name":"offline","context":{"idset":"132"}}
+{"timestamp":1714693191.5506167,"name":"online","context":{"idset":"96"}}
+{"timestamp":1714693199.619194,"name":"online","context":{"idset":"102"}}
+{"timestamp":1714693203.3782704,"name":"online","context":{"idset":"130"}}
+{"timestamp":1714693204.1240325,"name":"online","context":{"idset":"101"}}
+{"timestamp":1714693204.8606479,"name":"online","context":{"idset":"112"}}
+{"timestamp":1714693205.6416662,"name":"online","context":{"idset":"125"}}
+{"timestamp":1714693207.3453166,"name":"online","context":{"idset":"93"}}
+{"timestamp":1714693209.3869832,"name":"online","context":{"idset":"98"}}
+{"timestamp":1714693210.3361084,"name":"online","context":{"idset":"119"}}
+{"timestamp":1714693211.3510914,"name":"online","context":{"idset":"129"}}
+{"timestamp":1714693212.9809306,"name":"online","context":{"idset":"105"}}
+{"timestamp":1714693214.6356089,"name":"online","context":{"idset":"111"}}
+{"timestamp":1714693215.2141173,"name":"online","context":{"idset":"124"}}
+{"timestamp":1714693216.1839044,"name":"online","context":{"idset":"123"}}
+{"timestamp":1714693218.5402429,"name":"online","context":{"idset":"107"}}
+{"timestamp":1714693218.7986379,"name":"online","context":{"idset":"122"}}
+{"timestamp":1714693219.8682189,"name":"online","context":{"idset":"114"}}
+{"timestamp":1714693221.5473623,"name":"online","context":{"idset":"126"}}
+{"timestamp":1714693223.7188597,"name":"online","context":{"idset":"99"}}
+{"timestamp":1714693224.3512959,"name":"online","context":{"idset":"109"}}
+{"timestamp":1714693225.0005112,"name":"online","context":{"idset":"95"}}
+{"timestamp":1714693227.4401846,"name":"online","context":{"idset":"97"}}
+{"timestamp":1714693228.9934313,"name":"online","context":{"idset":"94"}}
+{"timestamp":1714693229.2392442,"name":"online","context":{"idset":"115"}}
+{"timestamp":1714693233.4431798,"name":"online","context":{"idset":"132"}}
+{"timestamp":1714693234.1966753,"name":"online","context":{"idset":"116"}}
+{"timestamp":1714693236.8411665,"name":"online","context":{"idset":"131"}}
+{"timestamp":1714693238.4876375,"name":"online","context":{"idset":"128"}}
+{"timestamp":1714693242.6735053,"name":"online","context":{"idset":"113"}}
+{"timestamp":1714693247.4016435,"name":"online","context":{"idset":"127"}}
+{"timestamp":1714693249.0881696,"name":"online","context":{"idset":"100"}}
+{"timestamp":1714693253.1199265,"name":"online","context":{"idset":"106"}}
+{"timestamp":1714693262.9736016,"name":"online","context":{"idset":"108"}}
+{"timestamp":1714693329.7325494,"name":"online","context":{"idset":"110"}}
+{"timestamp":1714693414.5526125,"name":"offline","context":{"idset":"8047"}}
+{"timestamp":1714693512.5536644,"name":"offline","context":{"idset":"11317"}}
+{"timestamp":1714693515.2318194,"name":"offline","context":{"idset":"11318"}}
+{"timestamp":1714693616.4717317,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1714693617.067734,"name":"drain","context":{"idset":"8031","reason":"broker was unresponsive"}}
+{"timestamp":1714694156.5556753,"name":"offline","context":{"idset":"11321"}}
+{"timestamp":1714694159.0557163,"name":"offline","context":{"idset":"11322"}}
+{"timestamp":1714695131.2834287,"name":"online","context":{"idset":"8047"}}
+{"timestamp":1714695214.1362743,"name":"undrain","context":{"idset":"8031"}}
+{"timestamp":1714695846.1676385,"name":"online","context":{"idset":"7565,7584"}}
+{"timestamp":1714695846.3108933,"name":"online","context":{"idset":"7554"}}
+{"timestamp":1714695846.6710346,"name":"online","context":{"idset":"7545,7600"}}
+{"timestamp":1714695847.4758732,"name":"online","context":{"idset":"7558,7607"}}
+{"timestamp":1714695847.8765056,"name":"online","context":{"idset":"7591"}}
+{"timestamp":1714695850.0218647,"name":"online","context":{"idset":"7575,7589"}}
+{"timestamp":1714695850.296627,"name":"online","context":{"idset":"7637"}}
+{"timestamp":1714695850.4012048,"name":"online","context":{"idset":"7567"}}
+{"timestamp":1714695850.6512046,"name":"online","context":{"idset":"7627"}}
+{"timestamp":1714695850.9302862,"name":"online","context":{"idset":"7634"}}
+{"timestamp":1714695851.5613322,"name":"online","context":{"idset":"7573,7577"}}
+{"timestamp":1714695851.8337064,"name":"online","context":{"idset":"7568,7597"}}
+{"timestamp":1714695851.9395568,"name":"online","context":{"idset":"7542,7574"}}
+{"timestamp":1714695852.8054357,"name":"online","context":{"idset":"7586"}}
+{"timestamp":1714695852.9130023,"name":"online","context":{"idset":"7547-7548,7559,7601,7623"}}
+{"timestamp":1714695853.1026161,"name":"online","context":{"idset":"7603"}}
+{"timestamp":1714695853.1125484,"name":"online","context":{"idset":"7544"}}
+{"timestamp":1714695853.1184278,"name":"online","context":{"idset":"7564"}}
+{"timestamp":1714695853.478117,"name":"online","context":{"idset":"7557,7613"}}
+{"timestamp":1714695853.681128,"name":"online","context":{"idset":"7562,7583,7593,7599,7611,7651"}}
+{"timestamp":1714695853.9090679,"name":"online","context":{"idset":"7552-7553,7560-7561,7563,7569,7571,7588,7594,7604,7612,7615,7621,7639,7653"}}
+{"timestamp":1714695854.0129635,"name":"online","context":{"idset":"7616,7656"}}
+{"timestamp":1714695854.1543958,"name":"online","context":{"idset":"7587,7660"}}
+{"timestamp":1714695854.2645288,"name":"online","context":{"idset":"7541,7579,7606,7618,7626,7652,7665"}}
+{"timestamp":1714695854.3895755,"name":"online","context":{"idset":"7585"}}
+{"timestamp":1714695854.4991376,"name":"online","context":{"idset":"7555,7576,7581,7617"}}
+{"timestamp":1714695854.7429211,"name":"online","context":{"idset":"7543,7549,7566,7570,7582,7590,7592,7595-7596,7605,7620,7632,7636,7640-7641,7643-7646,7661,7666,7668"}}
+{"timestamp":1714695855.0319891,"name":"online","context":{"idset":"7622"}}
+{"timestamp":1714695855.2876644,"name":"online","context":{"idset":"7550-7551,7556,7572,7609,7630,7635,7642,7663"}}
+{"timestamp":1714695855.39678,"name":"online","context":{"idset":"7546,7580,7598,7602,7614,7619,7638,7655,7658,7662,7667"}}
+{"timestamp":1714695855.5337753,"name":"online","context":{"idset":"7578,7610,7631,7647-7649,7654"}}
+{"timestamp":1714695855.6441636,"name":"online","context":{"idset":"7608,7624,7628-7629,7633,7650,7657,7659,7664"}}
+{"timestamp":1714695855.9572821,"name":"online","context":{"idset":"7625"}}
+{"timestamp":1714695869.9754343,"name":"drain","context":{"idset":"7541-7668","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714696455.004395,"name":"offline","context":{"idset":"10612"}}
+{"timestamp":1714696504.5523119,"name":"offline","context":{"idset":"11790"}}
+{"timestamp":1714696766.9829597,"name":"offline","context":{"idset":"7431"}}
+{"timestamp":1714696768.4691236,"name":"drain","context":{"idset":"7432","reason":"epilog failed for jobid fsyPGNwUbWo","overwrite":0}}
+{"timestamp":1714696768.5544391,"name":"offline","context":{"idset":"7432"}}
+{"timestamp":1714696800.5535655,"name":"offline","context":{"idset":"963"}}
+{"timestamp":1714696802.5562203,"name":"offline","context":{"idset":"964"}}
+{"timestamp":1714699657.2154589,"name":"online","context":{"idset":"9048"}}
+{"timestamp":1714699808.4758904,"name":"online","context":{"idset":"10185"}}
+{"timestamp":1714699809.4546487,"name":"online","context":{"idset":"10158"}}
+{"timestamp":1714699809.5663428,"name":"online","context":{"idset":"10154"}}
+{"timestamp":1714699849.4798644,"name":"undrain","context":{"idset":"10154,10158,10185"}}
+{"timestamp":1714700038.9600456,"name":"online","context":{"idset":"10491"}}
+{"timestamp":1714700084.3692176,"name":"online","context":{"idset":"10612"}}
+{"timestamp":1714701424.9608912,"name":"drain","context":{"idset":"10108-10133,10135-10153,10155-10157,10159-10184,10186-10228","reason":"prolog failed for jobid fszYtUHFJdd","overwrite":0}}
+{"timestamp":1714701426.234848,"name":"undrain","context":{"idset":"7541-7668"}}
+{"timestamp":1714702882.5535109,"name":"offline","context":{"idset":"7563"}}
+{"timestamp":1714702884.5512924,"name":"offline","context":{"idset":"7564"}}
+{"timestamp":1714704736.5536501,"name":"offline","context":{"idset":"7291"}}
+{"timestamp":1714705834.5568738,"name":"offline","context":{"idset":"7292"}}
+{"timestamp":1714707080.5537369,"name":"offline","context":{"idset":"7667"}}
+{"timestamp":1714707770.5531201,"name":"offline","context":{"idset":"7668"}}
+{"timestamp":1714707804.549469,"name":"offline","context":{"idset":"1049"}}
+{"timestamp":1714707806.5517118,"name":"offline","context":{"idset":"1050"}}
+{"timestamp":1714709533.1016493,"name":"offline","context":{"idset":"6754"}}
+{"timestamp":1714710464.5513163,"name":"offline","context":{"idset":"6753"}}
+{"timestamp":1714712278.552918,"name":"offline","context":{"idset":"7548"}}
+{"timestamp":1714712316.5528255,"name":"offline","context":{"idset":"10323"}}
+{"timestamp":1714714182.4754958,"name":"offline","context":{"idset":"7989"}}
+{"timestamp":1714714182.4805222,"name":"offline","context":{"idset":"7991"}}
+{"timestamp":1714714182.5534003,"name":"offline","context":{"idset":"7992"}}
+{"timestamp":1714714184.4758999,"name":"offline","context":{"idset":"7993"}}
+{"timestamp":1714714184.4805672,"name":"offline","context":{"idset":"7994"}}
+{"timestamp":1714714184.5541384,"name":"offline","context":{"idset":"7995"}}
+{"timestamp":1714714186.4861085,"name":"offline","context":{"idset":"7996"}}
+{"timestamp":1714714186.4924736,"name":"offline","context":{"idset":"7997"}}
+{"timestamp":1714714186.498008,"name":"offline","context":{"idset":"7998"}}
+{"timestamp":1714714186.5029886,"name":"offline","context":{"idset":"7999"}}
+{"timestamp":1714714186.5518656,"name":"offline","context":{"idset":"8000"}}
+{"timestamp":1714714188.4793348,"name":"offline","context":{"idset":"8001"}}
+{"timestamp":1714714188.4843664,"name":"offline","context":{"idset":"8002"}}
+{"timestamp":1714714188.4892647,"name":"offline","context":{"idset":"8003"}}
+{"timestamp":1714714188.4893408,"name":"drain","context":{"idset":"7999,8002-8003","reason":"epilog failed for jobid fsyJpkeb86b","overwrite":0}}
+{"timestamp":1714714188.5663428,"name":"offline","context":{"idset":"8004"}}
+{"timestamp":1714714190.4969571,"name":"offline","context":{"idset":"8005"}}
+{"timestamp":1714714190.5018516,"name":"offline","context":{"idset":"8006"}}
+{"timestamp":1714714190.5067265,"name":"offline","context":{"idset":"8007"}}
+{"timestamp":1714714190.5116379,"name":"offline","context":{"idset":"8008"}}
+{"timestamp":1714714190.5532017,"name":"offline","context":{"idset":"8009"}}
+{"timestamp":1714714192.4785018,"name":"offline","context":{"idset":"8010"}}
+{"timestamp":1714714192.4832497,"name":"offline","context":{"idset":"8011"}}
+{"timestamp":1714714192.4881451,"name":"offline","context":{"idset":"8012"}}
+{"timestamp":1714714192.5727646,"name":"offline","context":{"idset":"8013"}}
+{"timestamp":1714714194.5065899,"name":"offline","context":{"idset":"8014"}}
+{"timestamp":1714714194.5189362,"name":"offline","context":{"idset":"8015"}}
+{"timestamp":1714714194.5274291,"name":"offline","context":{"idset":"8016"}}
+{"timestamp":1714714194.5586169,"name":"offline","context":{"idset":"8017"}}
+{"timestamp":1714714196.4830074,"name":"offline","context":{"idset":"8018"}}
+{"timestamp":1714714196.4883325,"name":"offline","context":{"idset":"8019"}}
+{"timestamp":1714714196.497498,"name":"offline","context":{"idset":"8020"}}
+{"timestamp":1714714196.5566299,"name":"offline","context":{"idset":"8021"}}
+{"timestamp":1714714198.5064502,"name":"offline","context":{"idset":"8022"}}
+{"timestamp":1714714198.5119438,"name":"offline","context":{"idset":"8023"}}
+{"timestamp":1714714198.5203614,"name":"offline","context":{"idset":"8024"}}
+{"timestamp":1714714198.5256853,"name":"offline","context":{"idset":"8025"}}
+{"timestamp":1714714198.5854163,"name":"offline","context":{"idset":"8026"}}
+{"timestamp":1714714200.4969769,"name":"offline","context":{"idset":"8027"}}
+{"timestamp":1714714200.5017695,"name":"offline","context":{"idset":"8028"}}
+{"timestamp":1714714200.5062025,"name":"offline","context":{"idset":"8029"}}
+{"timestamp":1714714200.5744061,"name":"offline","context":{"idset":"8030"}}
+{"timestamp":1714714202.4951589,"name":"offline","context":{"idset":"8031"}}
+{"timestamp":1714714202.4995437,"name":"offline","context":{"idset":"8032"}}
+{"timestamp":1714714202.5044465,"name":"offline","context":{"idset":"8033"}}
+{"timestamp":1714714202.556401,"name":"offline","context":{"idset":"8034"}}
+{"timestamp":1714714204.4764328,"name":"offline","context":{"idset":"8036"}}
+{"timestamp":1714714204.4794881,"name":"offline","context":{"idset":"8037"}}
+{"timestamp":1714714204.4877865,"name":"offline","context":{"idset":"8038"}}
+{"timestamp":1714714204.5599692,"name":"offline","context":{"idset":"8039"}}
+{"timestamp":1714714206.4813559,"name":"offline","context":{"idset":"8040"}}
+{"timestamp":1714714206.4898341,"name":"offline","context":{"idset":"8041"}}
+{"timestamp":1714714206.4985001,"name":"offline","context":{"idset":"8042"}}
+{"timestamp":1714714206.5574188,"name":"offline","context":{"idset":"8043"}}
+{"timestamp":1714714208.4801497,"name":"offline","context":{"idset":"8044"}}
+{"timestamp":1714714208.4890878,"name":"offline","context":{"idset":"8045"}}
+{"timestamp":1714714208.497658,"name":"offline","context":{"idset":"8046"}}
+{"timestamp":1714714208.5066628,"name":"offline","context":{"idset":"8047"}}
+{"timestamp":1714714208.5556953,"name":"offline","context":{"idset":"8048"}}
+{"timestamp":1714714210.4742372,"name":"offline","context":{"idset":"8049"}}
+{"timestamp":1714714210.4794383,"name":"offline","context":{"idset":"8050"}}
+{"timestamp":1714714210.4843943,"name":"offline","context":{"idset":"8051"}}
+{"timestamp":1714714210.5703437,"name":"offline","context":{"idset":"8052"}}
+{"timestamp":1714714229.6080241,"name":"offline","context":{"idset":"7990"}}
+{"timestamp":1714714254.5533149,"name":"offline","context":{"idset":"8035"}}
+{"timestamp":1714715059.3465528,"name":"online","context":{"idset":"8008"}}
+{"timestamp":1714715059.7270727,"name":"online","context":{"idset":"8027"}}
+{"timestamp":1714715059.9770558,"name":"online","context":{"idset":"7990,8013,8021-8022,8037"}}
+{"timestamp":1714715060.2355046,"name":"online","context":{"idset":"7999,8046"}}
+{"timestamp":1714715084.5022507,"name":"online","context":{"idset":"8019"}}
+{"timestamp":1714715084.7803261,"name":"online","context":{"idset":"8012,8020"}}
+{"timestamp":1714715085.3317087,"name":"online","context":{"idset":"7992,8004"}}
+{"timestamp":1714715086.2044396,"name":"online","context":{"idset":"8000"}}
+{"timestamp":1714715116.543359,"name":"online","context":{"idset":"7991"}}
+{"timestamp":1714715117.2399282,"name":"online","context":{"idset":"7994,8014"}}
+{"timestamp":1714715117.4509041,"name":"online","context":{"idset":"7993,7998"}}
+{"timestamp":1714715117.6440387,"name":"online","context":{"idset":"8042"}}
+{"timestamp":1714715117.7510169,"name":"online","context":{"idset":"7996-7997,8003,8033"}}
+{"timestamp":1714715117.9861901,"name":"online","context":{"idset":"8024,8029"}}
+{"timestamp":1714715118.2323837,"name":"online","context":{"idset":"7995,8001-8002,8006-8007,8017-8018,8025"}}
+{"timestamp":1714715118.4705222,"name":"online","context":{"idset":"8010,8016,8026,8034,8036,8038-8039"}}
+{"timestamp":1714715118.7100196,"name":"online","context":{"idset":"8040,8044,8050"}}
+{"timestamp":1714715119.0156863,"name":"online","context":{"idset":"8005,8030"}}
+{"timestamp":1714715148.3728266,"name":"online","context":{"idset":"7989,8015,8043"}}
+{"timestamp":1714715148.6851842,"name":"online","context":{"idset":"8031"}}
+{"timestamp":1714715149.0309217,"name":"online","context":{"idset":"8009,8041,8051-8052"}}
+{"timestamp":1714715149.2771308,"name":"online","context":{"idset":"8011,8023,8032"}}
+{"timestamp":1714715149.3788211,"name":"online","context":{"idset":"8028,8045"}}
+{"timestamp":1714715149.5975177,"name":"online","context":{"idset":"8047,8049"}}
+{"timestamp":1714715149.8947666,"name":"online","context":{"idset":"8035"}}
+{"timestamp":1714715317.245172,"name":"undrain","context":{"idset":"7999,8002-8003"}}
+{"timestamp":1714715389.003788,"name":"online","context":{"idset":"8048"}}
+{"timestamp":1714728038.555661,"name":"offline","context":{"idset":"11634"}}
+{"timestamp":1714736702.5566957,"name":"offline","context":{"idset":"10491"}}
+{"timestamp":1714743016.48311,"name":"offline","context":{"idset":"951"}}
+{"timestamp":1714743016.4899337,"name":"offline","context":{"idset":"952"}}
+{"timestamp":1714743016.5796487,"name":"offline","context":{"idset":"9047"}}
+{"timestamp":1714743064.5525098,"name":"offline","context":{"idset":"9048"}}
+{"timestamp":1714743224.4707549,"name":"offline","context":{"idset":"953"}}
+{"timestamp":1714743224.5528822,"name":"offline","context":{"idset":"954"}}
+{"timestamp":1714743610.5531573,"name":"offline","context":{"idset":"9031"}}
+{"timestamp":1714743612.5532696,"name":"offline","context":{"idset":"9032"}}
+{"timestamp":1714744122.5532403,"name":"offline","context":{"idset":"7547"}}
+{"timestamp":1714744392.9725199,"name":"online","context":{"idset":"8144"}}
+{"timestamp":1714745046.4807715,"name":"offline","context":{"idset":"11609"}}
+{"timestamp":1714745046.5688543,"name":"offline","context":{"idset":"11610"}}
+{"timestamp":1714745569.8875499,"name":"online","context":{"idset":"6133"}}
+{"timestamp":1714745570.0481551,"name":"online","context":{"idset":"6143"}}
+{"timestamp":1714745570.3224785,"name":"online","context":{"idset":"6135"}}
+{"timestamp":1714745570.5651267,"name":"online","context":{"idset":"6137"}}
+{"timestamp":1714745570.6786664,"name":"online","context":{"idset":"6151"}}
+{"timestamp":1714745571.3012283,"name":"online","context":{"idset":"6140,6152,6184"}}
+{"timestamp":1714745571.4460003,"name":"online","context":{"idset":"6170"}}
+{"timestamp":1714745571.555908,"name":"online","context":{"idset":"6180"}}
+{"timestamp":1714745571.6874349,"name":"online","context":{"idset":"6142,6160"}}
+{"timestamp":1714745571.8442183,"name":"online","context":{"idset":"6145"}}
+{"timestamp":1714745572.3868501,"name":"online","context":{"idset":"6169"}}
+{"timestamp":1714745572.747571,"name":"online","context":{"idset":"6187"}}
+{"timestamp":1714745572.9270482,"name":"online","context":{"idset":"6154"}}
+{"timestamp":1714745574.0470297,"name":"online","context":{"idset":"6214"}}
+{"timestamp":1714745574.2911143,"name":"online","context":{"idset":"6141"}}
+{"timestamp":1714745574.6963961,"name":"online","context":{"idset":"6159"}}
+{"timestamp":1714745574.9453812,"name":"online","context":{"idset":"6176"}}
+{"timestamp":1714745575.3346598,"name":"online","context":{"idset":"6155,6167,6229"}}
+{"timestamp":1714745575.9528413,"name":"online","context":{"idset":"6161,6165"}}
+{"timestamp":1714745576.1941025,"name":"online","context":{"idset":"6254"}}
+{"timestamp":1714745576.8098154,"name":"online","context":{"idset":"6218,6245"}}
+{"timestamp":1714745577.0681155,"name":"online","context":{"idset":"6178,6230,6259"}}
+{"timestamp":1714745577.1874659,"name":"online","context":{"idset":"6200"}}
+{"timestamp":1714745577.3865676,"name":"online","context":{"idset":"6177,6237"}}
+{"timestamp":1714745577.5684831,"name":"online","context":{"idset":"6209"}}
+{"timestamp":1714745577.8360472,"name":"online","context":{"idset":"6197"}}
+{"timestamp":1714745578.1357827,"name":"online","context":{"idset":"6255"}}
+{"timestamp":1714745578.3182566,"name":"online","context":{"idset":"6228"}}
+{"timestamp":1714745578.7083976,"name":"online","context":{"idset":"6223"}}
+{"timestamp":1714745579.6618681,"name":"online","context":{"idset":"6203"}}
+{"timestamp":1714745579.6970885,"name":"online","context":{"idset":"6439"}}
+{"timestamp":1714745580.1817365,"name":"online","context":{"idset":"6410"}}
+{"timestamp":1714745580.3535962,"name":"online","context":{"idset":"6212"}}
+{"timestamp":1714745580.6096129,"name":"online","context":{"idset":"6196"}}
+{"timestamp":1714745580.7999957,"name":"online","context":{"idset":"6459"}}
+{"timestamp":1714745580.9718328,"name":"online","context":{"idset":"6208,6233,6392,6396,6457,6493,6507"}}
+{"timestamp":1714745581.1666734,"name":"online","context":{"idset":"6458,6462,6481,6484"}}
+{"timestamp":1714745581.3616507,"name":"online","context":{"idset":"6189"}}
+{"timestamp":1714745581.5633798,"name":"online","context":{"idset":"6153,6198,6202,6244,6260,6430,6432,6435"}}
+{"timestamp":1714745581.7620232,"name":"online","context":{"idset":"6185,6238,6250,6429,6434,6461,6465,6516"}}
+{"timestamp":1714745581.9462619,"name":"online","context":{"idset":"6172,6217,6449,6499"}}
+{"timestamp":1714745582.1391299,"name":"online","context":{"idset":"6134,6182,6440"}}
+{"timestamp":1714745582.3954487,"name":"online","context":{"idset":"6148,6192,6236,6243,6390,6427,6466,6469,6483,6503"}}
+{"timestamp":1714745582.5169368,"name":"online","context":{"idset":"6174,6210,6420,6487,6512"}}
+{"timestamp":1714745582.7020941,"name":"online","context":{"idset":"6181,6226,6258,6480,6498,6509"}}
+{"timestamp":1714745582.7366717,"name":"online","context":{"idset":"6500"}}
+{"timestamp":1714745583.0234249,"name":"online","context":{"idset":"6158,6186,6216,6402-6403,6425,6441,6451,6467,6477,6482,6494,6501,6515"}}
+{"timestamp":1714745583.0320256,"name":"online","context":{"idset":"6405"}}
+{"timestamp":1714745583.2372832,"name":"online","context":{"idset":"6190,6246,6442,6479,6488,6491,6514"}}
+{"timestamp":1714745583.4124177,"name":"online","context":{"idset":"6213,6418"}}
+{"timestamp":1714745583.5877023,"name":"online","context":{"idset":"6150,6195,6225,6234,6248,6395"}}
+{"timestamp":1714745583.7270498,"name":"online","context":{"idset":"6219,6414,6431"}}
+{"timestamp":1714745583.8325317,"name":"online","context":{"idset":"6171,6456"}}
+{"timestamp":1714745583.8560097,"name":"online","context":{"idset":"6146"}}
+{"timestamp":1714745583.9840643,"name":"online","context":{"idset":"6179,6232,6235,6437"}}
+{"timestamp":1714745584.0899823,"name":"online","context":{"idset":"6156-6157,6489"}}
+{"timestamp":1714745584.285012,"name":"online","context":{"idset":"6147,6247"}}
+{"timestamp":1714745584.3982811,"name":"online","context":{"idset":"6204,6445"}}
+{"timestamp":1714745584.5936337,"name":"online","context":{"idset":"6206,6251"}}
+{"timestamp":1714745584.9849081,"name":"online","context":{"idset":"6139,6201,6221,6438,6468,6475"}}
+{"timestamp":1714745585.1423433,"name":"online","context":{"idset":"6249,6443"}}
+{"timestamp":1714745585.2625015,"name":"online","context":{"idset":"6460"}}
+{"timestamp":1714745585.3837285,"name":"online","context":{"idset":"6138,6422"}}
+{"timestamp":1714745585.5320683,"name":"online","context":{"idset":"6188,6492"}}
+{"timestamp":1714745585.5586631,"name":"online","context":{"idset":"6398"}}
+{"timestamp":1714745585.7431476,"name":"online","context":{"idset":"6144,6409,6411,6455"}}
+{"timestamp":1714745585.8960714,"name":"online","context":{"idset":"6136,6476"}}
+{"timestamp":1714745586.047297,"name":"online","context":{"idset":"6183,6417,6511"}}
+{"timestamp":1714745586.1985531,"name":"online","context":{"idset":"6211,6412,6416,6436,6506"}}
+{"timestamp":1714745586.3632953,"name":"online","context":{"idset":"6194,6401,6404,6504"}}
+{"timestamp":1714745586.596832,"name":"online","context":{"idset":"6393,6471"}}
+{"timestamp":1714745586.7841425,"name":"online","context":{"idset":"6464,6490"}}
+{"timestamp":1714745586.9505868,"name":"online","context":{"idset":"6413,6446"}}
+{"timestamp":1714745587.1060805,"name":"online","context":{"idset":"6220,6253"}}
+{"timestamp":1714745587.2735939,"name":"online","context":{"idset":"6447"}}
+{"timestamp":1714745587.6157231,"name":"online","context":{"idset":"6450"}}
+{"timestamp":1714745587.7809563,"name":"online","context":{"idset":"6448,6486,6502"}}
+{"timestamp":1714745587.7919559,"name":"online","context":{"idset":"6240"}}
+{"timestamp":1714745587.9866061,"name":"online","context":{"idset":"6162,6224,6419,6508"}}
+{"timestamp":1714745588.1406293,"name":"online","context":{"idset":"6406"}}
+{"timestamp":1714745588.3924792,"name":"online","context":{"idset":"6252,6389,6454,6478"}}
+{"timestamp":1714745588.5898495,"name":"online","context":{"idset":"6191,6227"}}
+{"timestamp":1714745588.6960843,"name":"online","context":{"idset":"6205"}}
+{"timestamp":1714745588.967999,"name":"online","context":{"idset":"6239,6242,6256,6408"}}
+{"timestamp":1714745589.2421384,"name":"online","context":{"idset":"6149,6163,6168,6175"}}
+{"timestamp":1714745589.3486755,"name":"online","context":{"idset":"6164,6166,6423,6428,6452,6473,6513"}}
+{"timestamp":1714745589.5855644,"name":"online","context":{"idset":"6215,6391,6421,6433,6444,6496"}}
+{"timestamp":1714745589.8868456,"name":"online","context":{"idset":"6222,6407,6453,6472,6485,6495,6497"}}
+{"timestamp":1714745590.1335866,"name":"online","context":{"idset":"6173,6199,6207,6231,6241,6397,6399-6400,6424"}}
+{"timestamp":1714745590.242929,"name":"online","context":{"idset":"6257,6415,6505,6510"}}
+{"timestamp":1714745590.3796151,"name":"online","context":{"idset":"6394,6474"}}
+{"timestamp":1714745590.4954655,"name":"online","context":{"idset":"6193,6426,6470"}}
+{"timestamp":1714745590.8119171,"name":"online","context":{"idset":"6463"}}
+{"timestamp":1714745594.1726937,"name":"drain","context":{"idset":"6133-6260","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714745594.9067128,"name":"drain","context":{"idset":"6389-6516","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
+{"timestamp":1714746186.5560901,"name":"offline","context":{"idset":"8107"}}
+{"timestamp":1714746397.6684773,"name":"online","context":{"idset":"7272"}}
+{"timestamp":1714746398.2087581,"name":"online","context":{"idset":"7271,7273"}}
+{"timestamp":1714746398.9510744,"name":"online","context":{"idset":"7269-7270,7275"}}
+{"timestamp":1714746399.3068662,"name":"online","context":{"idset":"7274"}}
+{"timestamp":1714746399.5428798,"name":"online","context":{"idset":"7276"}}
+{"timestamp":1714746482.2453086,"name":"online","context":{"idset":"7284"}}
+{"timestamp":1714746483.4090655,"name":"online","context":{"idset":"7277"}}
+{"timestamp":1714746483.8084915,"name":"online","context":{"idset":"7282"}}
+{"timestamp":1714746484.1385927,"name":"online","context":{"idset":"7281"}}
+{"timestamp":1714746484.8273671,"name":"online","context":{"idset":"7278,7280"}}
+{"timestamp":1714746485.088824,"name":"online","context":{"idset":"7279,7283"}}
+{"timestamp":1714746545.2213969,"name":"online","context":{"idset":"7243"}}
+{"timestamp":1714746546.1223552,"name":"online","context":{"idset":"7238"}}
+{"timestamp":1714746546.7324138,"name":"online","context":{"idset":"7237,7241"}}
+{"timestamp":1714746546.8518484,"name":"online","context":{"idset":"7242"}}
+{"timestamp":1714746547.1088929,"name":"online","context":{"idset":"7240,7244"}}
+{"timestamp":1714746547.5124841,"name":"online","context":{"idset":"7239"}}
+{"timestamp":1714746738.4977622,"name":"online","context":{"idset":"7246-7252"}}
+{"timestamp":1714746739.1734524,"name":"online","context":{"idset":"7245"}}
+{"timestamp":1714747786.8813732,"name":"offline","context":{"idset":"10597"}}
+{"timestamp":1714748452.8328619,"name":"offline","context":{"idset":"10490"}}
+{"timestamp":1714748792.8823843,"name":"offline","context":{"idset":"10607"}}
+{"timestamp":1714749400.4160743,"name":"offline","context":{"idset":"6238"}}
+{"timestamp":1714749416.8780901,"name":"offline","context":{"idset":"10516"}}
+{"timestamp":1714749694.4155233,"name":"offline","context":{"idset":"11633"}}
+{"timestamp":1714749726.4173591,"name":"offline","context":{"idset":"809"}}
+{"timestamp":1714749728.4222217,"name":"offline","context":{"idset":"810"}}
+{"timestamp":1714749892.4290075,"name":"offline","context":{"idset":"797"}}
+{"timestamp":1714749958.8724318,"name":"offline","context":{"idset":"10605"}}
+{"timestamp":1714751950.4168973,"name":"offline","context":{"idset":"7631"}}
+{"timestamp":1714752232.4152887,"name":"offline","context":{"idset":"10324"}}
+{"timestamp":1714752266.4184823,"name":"offline","context":{"idset":"955"}}
+{"timestamp":1714752268.4143388,"name":"offline","context":{"idset":"956"}}
+{"timestamp":1714752974.4110346,"name":"offline","context":{"idset":"6237"}}
+{"timestamp":1714753201.5822062,"name":"undrain","context":{"idset":"6133-6237,6239-6260"}}
+{"timestamp":1714753202.4391956,"name":"undrain","context":{"idset":"6389-6516"}}
+{"timestamp":1714753306.4164145,"name":"offline","context":{"idset":"10503"}}
+{"timestamp":1714753807.0061116,"name":"online","context":{"idset":"6753"}}
+{"timestamp":1714753807.3453536,"name":"online","context":{"idset":"7563"}}
+{"timestamp":1714753807.4578395,"name":"online","context":{"idset":"7564"}}
+{"timestamp":1714753807.7751975,"name":"online","context":{"idset":"7432,7547-7548,7667"}}
+{"timestamp":1714753808.1173699,"name":"online","context":{"idset":"6754,7291-7292,7431,7668"}}
+{"timestamp":1714753856.3375118,"name":"offline","context":{"idset":"10108"}}
+{"timestamp":1714753856.4295309,"name":"offline","context":{"idset":"10109"}}
+{"timestamp":1714753858.3421407,"name":"offline","context":{"idset":"10110"}}
+{"timestamp":1714753858.3472362,"name":"offline","context":{"idset":"10111"}}
+{"timestamp":1714753858.3523903,"name":"offline","context":{"idset":"10112"}}
+{"timestamp":1714753858.4252696,"name":"offline","context":{"idset":"10113"}}
+{"timestamp":1714753860.3492975,"name":"offline","context":{"idset":"10114"}}
+{"timestamp":1714753860.358918,"name":"offline","context":{"idset":"10115"}}
+{"timestamp":1714753860.3681667,"name":"offline","context":{"idset":"10116"}}
+{"timestamp":1714753860.4160738,"name":"offline","context":{"idset":"10117"}}
+{"timestamp":1714753862.3517413,"name":"offline","context":{"idset":"10118"}}
+{"timestamp":1714753862.357115,"name":"offline","context":{"idset":"10119"}}
+{"timestamp":1714753862.3621814,"name":"offline","context":{"idset":"10120"}}
+{"timestamp":1714753862.3674507,"name":"offline","context":{"idset":"10121"}}
+{"timestamp":1714753862.4159563,"name":"offline","context":{"idset":"10122"}}
+{"timestamp":1714753864.2870648,"name":"offline","context":{"idset":"10123"}}
+{"timestamp":1714753864.2984948,"name":"offline","context":{"idset":"10124"}}
+{"timestamp":1714753864.3091283,"name":"offline","context":{"idset":"10125"}}
+{"timestamp":1714753864.3534896,"name":"offline","context":{"idset":"10126"}}
+{"timestamp":1714753866.3475418,"name":"offline","context":{"idset":"10127"}}
+{"timestamp":1714753866.352731,"name":"offline","context":{"idset":"10129"}}
+{"timestamp":1714753866.3582339,"name":"offline","context":{"idset":"10130"}}
+{"timestamp":1714753866.4218969,"name":"offline","context":{"idset":"10131"}}
+{"timestamp":1714753868.3503807,"name":"offline","context":{"idset":"10132"}}
+{"timestamp":1714753868.3576236,"name":"offline","context":{"idset":"10133"}}
+{"timestamp":1714753868.4357219,"name":"offline","context":{"idset":"10135"}}
+{"timestamp":1714753870.373409,"name":"offline","context":{"idset":"10136"}}
+{"timestamp":1714753870.3800886,"name":"offline","context":{"idset":"10137"}}
+{"timestamp":1714753870.3854284,"name":"offline","context":{"idset":"10138"}}
+{"timestamp":1714753870.3923824,"name":"offline","context":{"idset":"10139"}}
+{"timestamp":1714753870.4157188,"name":"offline","context":{"idset":"10140"}}
+{"timestamp":1714753872.348772,"name":"offline","context":{"idset":"10141"}}
+{"timestamp":1714753872.3543432,"name":"offline","context":{"idset":"10142"}}
+{"timestamp":1714753872.3616338,"name":"offline","context":{"idset":"10143"}}
+{"timestamp":1714753872.4217505,"name":"offline","context":{"idset":"10144"}}
+{"timestamp":1714753874.3460524,"name":"offline","context":{"idset":"10145"}}
+{"timestamp":1714753874.351506,"name":"offline","context":{"idset":"10146"}}
+{"timestamp":1714753874.3702483,"name":"offline","context":{"idset":"10147"}}
+{"timestamp":1714753874.4180837,"name":"offline","context":{"idset":"10148"}}
+{"timestamp":1714753876.345161,"name":"offline","context":{"idset":"10149"}}
+{"timestamp":1714753876.354306,"name":"offline","context":{"idset":"10150"}}
+{"timestamp":1714753876.3635228,"name":"offline","context":{"idset":"10151"}}
+{"timestamp":1714753876.416743,"name":"offline","context":{"idset":"10152"}}
+{"timestamp":1714753878.3446503,"name":"offline","context":{"idset":"10153"}}
+{"timestamp":1714753878.3497295,"name":"offline","context":{"idset":"10155"}}
+{"timestamp":1714753878.4165742,"name":"offline","context":{"idset":"10156"}}
+{"timestamp":1714753879.4131057,"name":"online","context":{"idset":"440"}}
+{"timestamp":1714753879.4931209,"name":"offline","context":{"idset":"10157"}}
+{"timestamp":1714753879.4998126,"name":"offline","context":{"idset":"10159"}}
+{"timestamp":1714753879.5054421,"name":"offline","context":{"idset":"10160"}}
+{"timestamp":1714753879.558414,"name":"offline","context":{"idset":"10161"}}
+{"timestamp":1714753882.344732,"name":"offline","context":{"idset":"10128"}}
+{"timestamp":1714753882.3557692,"name":"offline","context":{"idset":"10162"}}
+{"timestamp":1714753882.3652692,"name":"offline","context":{"idset":"10163"}}
+{"timestamp":1714753882.414413,"name":"offline","context":{"idset":"10164"}}
+{"timestamp":1714753884.3420894,"name":"offline","context":{"idset":"10165"}}
+{"timestamp":1714753884.3475008,"name":"offline","context":{"idset":"10166"}}
+{"timestamp":1714753884.3527822,"name":"offline","context":{"idset":"10167"}}
+{"timestamp":1714753884.4419069,"name":"offline","context":{"idset":"10168"}}
+{"timestamp":1714753886.3441763,"name":"offline","context":{"idset":"10169"}}
+{"timestamp":1714753886.3494546,"name":"offline","context":{"idset":"10170"}}
+{"timestamp":1714753886.3545218,"name":"offline","context":{"idset":"10171"}}
+{"timestamp":1714753886.3597133,"name":"offline","context":{"idset":"10172"}}
+{"timestamp":1714753886.4563851,"name":"offline","context":{"idset":"10173"}}
+{"timestamp":1714753888.3470867,"name":"offline","context":{"idset":"10174"}}
+{"timestamp":1714753888.3524499,"name":"offline","context":{"idset":"10175"}}
+{"timestamp":1714753888.4293587,"name":"offline","context":{"idset":"10176"}}
+{"timestamp":1714753890.3539448,"name":"offline","context":{"idset":"10177"}}
+{"timestamp":1714753890.3590043,"name":"offline","context":{"idset":"10178"}}
+{"timestamp":1714753890.3639557,"name":"offline","context":{"idset":"10179"}}
+{"timestamp":1714753890.4164779,"name":"offline","context":{"idset":"10181"}}
+{"timestamp":1714753892.3449783,"name":"offline","context":{"idset":"10182"}}
+{"timestamp":1714753892.3500607,"name":"offline","context":{"idset":"10183"}}
+{"timestamp":1714753892.3553617,"name":"offline","context":{"idset":"10184"}}
+{"timestamp":1714753892.4149725,"name":"offline","context":{"idset":"10186"}}
+{"timestamp":1714753894.3545125,"name":"offline","context":{"idset":"10187"}}
+{"timestamp":1714753894.3592834,"name":"offline","context":{"idset":"10188"}}
+{"timestamp":1714753894.364022,"name":"offline","context":{"idset":"10189"}}
+{"timestamp":1714753894.4157851,"name":"offline","context":{"idset":"10190"}}
+{"timestamp":1714753896.3478904,"name":"offline","context":{"idset":"10191"}}
+{"timestamp":1714753896.3586204,"name":"offline","context":{"idset":"10192"}}
+{"timestamp":1714753896.36516,"name":"offline","context":{"idset":"10193"}}
+{"timestamp":1714753896.4301164,"name":"offline","context":{"idset":"10194"}}
+{"timestamp":1714753898.3690987,"name":"offline","context":{"idset":"10195"}}
+{"timestamp":1714753898.3777456,"name":"offline","context":{"idset":"10196"}}
+{"timestamp":1714753898.419538,"name":"offline","context":{"idset":"10197"}}
+{"timestamp":1714753900.3380802,"name":"offline","context":{"idset":"10198"}}
+{"timestamp":1714753900.3417859,"name":"offline","context":{"idset":"10199"}}
+{"timestamp":1714753900.3456223,"name":"offline","context":{"idset":"10200"}}
+{"timestamp":1714753900.4139097,"name":"offline","context":{"idset":"10201"}}
+{"timestamp":1714753902.3423712,"name":"offline","context":{"idset":"10202"}}
+{"timestamp":1714753902.3486164,"name":"offline","context":{"idset":"10203"}}
+{"timestamp":1714753902.3549147,"name":"offline","context":{"idset":"10204"}}
+{"timestamp":1714753902.4164338,"name":"offline","context":{"idset":"10206"}}
+{"timestamp":1714753904.3502369,"name":"offline","context":{"idset":"10207"}}
+{"timestamp":1714753904.3543739,"name":"offline","context":{"idset":"10208"}}
+{"timestamp":1714753904.3594239,"name":"offline","context":{"idset":"10209"}}
+{"timestamp":1714753904.4176006,"name":"offline","context":{"idset":"10210"}}
+{"timestamp":1714753906.345649,"name":"offline","context":{"idset":"10180"}}
+{"timestamp":1714753906.3503754,"name":"offline","context":{"idset":"10211"}}
+{"timestamp":1714753906.3550744,"name":"offline","context":{"idset":"10212"}}
+{"timestamp":1714753906.3597662,"name":"offline","context":{"idset":"10213"}}
+{"timestamp":1714753906.4383688,"name":"offline","context":{"idset":"10214"}}
+{"timestamp":1714753908.3365967,"name":"offline","context":{"idset":"10215"}}
+{"timestamp":1714753908.3413815,"name":"offline","context":{"idset":"10216"}}
+{"timestamp":1714753908.4227731,"name":"offline","context":{"idset":"10217"}}
+{"timestamp":1714753910.3442852,"name":"offline","context":{"idset":"10218"}}
+{"timestamp":1714753910.3491185,"name":"offline","context":{"idset":"10219"}}
+{"timestamp":1714753910.3538165,"name":"offline","context":{"idset":"10220"}}
+{"timestamp":1714753910.3584707,"name":"offline","context":{"idset":"10221"}}
+{"timestamp":1714753910.4191709,"name":"offline","context":{"idset":"10222"}}
+{"timestamp":1714753912.3528855,"name":"offline","context":{"idset":"10223"}}
+{"timestamp":1714753912.3576248,"name":"offline","context":{"idset":"10224"}}
+{"timestamp":1714753912.3622906,"name":"offline","context":{"idset":"10225"}}
+{"timestamp":1714753912.4151101,"name":"offline","context":{"idset":"10226"}}
+{"timestamp":1714753913.7203274,"name":"offline","context":{"idset":"10227"}}
+{"timestamp":1714753913.8041651,"name":"offline","context":{"idset":"10228"}}
+{"timestamp":1714753952.4156663,"name":"offline","context":{"idset":"10205"}}
+{"timestamp":1714754328.4156251,"name":"offline","context":{"idset":"440"}}
+{"timestamp":1714755173.5232358,"name":"online","context":{"idset":"10108"}}
+{"timestamp":1714755173.7822082,"name":"online","context":{"idset":"10115"}}
+{"timestamp":1714755173.9102819,"name":"online","context":{"idset":"10180"}}
+{"timestamp":1714755174.0925291,"name":"online","context":{"idset":"10109-10111,10149,10199"}}
+{"timestamp":1714755174.4600701,"name":"online","context":{"idset":"10145,10164"}}
+{"timestamp":1714755174.6651223,"name":"online","context":{"idset":"10130,10153,10173,10224"}}
+{"timestamp":1714755174.8275349,"name":"online","context":{"idset":"10136-10137,10157,10190,10192,10195,10215"}}
+{"timestamp":1714755174.9786572,"name":"online","context":{"idset":"10168,10175,10226"}}
+{"timestamp":1714755175.132977,"name":"online","context":{"idset":"10156,10194,10208,10607"}}
+{"timestamp":1714755175.1486869,"name":"online","context":{"idset":"10162"}}
+{"timestamp":1714755175.3680203,"name":"online","context":{"idset":"10165,10172,10183"}}
+{"timestamp":1714755175.5450063,"name":"online","context":{"idset":"10203"}}
+{"timestamp":1714755175.7378225,"name":"online","context":{"idset":"10152,10160,10213"}}
+{"timestamp":1714755176.0319393,"name":"online","context":{"idset":"10171"}}
+{"timestamp":1714755176.1431847,"name":"online","context":{"idset":"10120,10132,10197-10198"}}
+{"timestamp":1714755176.4610436,"name":"online","context":{"idset":"10141"}}
+{"timestamp":1714755176.5663934,"name":"online","context":{"idset":"10170"}}
+{"timestamp":1714755176.8501933,"name":"online","context":{"idset":"10117,10131,10146,10184,10186,10216,10227"}}
+{"timestamp":1714755177.1239934,"name":"online","context":{"idset":"10144,10150,10161,10166,10218"}}
+{"timestamp":1714755177.2626235,"name":"online","context":{"idset":"10225"}}
+{"timestamp":1714755177.397012,"name":"online","context":{"idset":"10113,10125,10196"}}
+{"timestamp":1714755177.520668,"name":"online","context":{"idset":"10112,10140,10148,10174,10176,10188,10202,10207,10210-10211,10217,10219,10222,10228"}}
+{"timestamp":1714755177.6380932,"name":"online","context":{"idset":"10127,10221"}}
+{"timestamp":1714755177.7645524,"name":"online","context":{"idset":"10139,10155,10169,10178,10187,10191,10206,10220,11790"}}
+{"timestamp":1714755177.9010525,"name":"online","context":{"idset":"10116,10118,10126,10129,10138,10163,10177,10201,10205,10214,10516"}}
+{"timestamp":1714755178.0192173,"name":"online","context":{"idset":"10114,10121-10124,10143,10147,10179,10182,10189,10200,10204,10212,10491"}}
+{"timestamp":1714755178.1307163,"name":"online","context":{"idset":"10119,10128,10133,10135,10167,10181,10193,10503"}}
+{"timestamp":1714755178.2662675,"name":"online","context":{"idset":"10223"}}
+{"timestamp":1714755178.4382069,"name":"online","context":{"idset":"10142,10209,10605"}}
+{"timestamp":1714755178.7118208,"name":"online","context":{"idset":"10490"}}
+{"timestamp":1714755178.9586704,"name":"online","context":{"idset":"10151"}}
+{"timestamp":1714755240.6485999,"name":"online","context":{"idset":"925"}}
+{"timestamp":1714755241.0646124,"name":"online","context":{"idset":"926"}}
+{"timestamp":1714755309.0634189,"name":"undrain","context":{"idset":"10186"}}
+{"timestamp":1714755315.7280781,"name":"undrain","context":{"idset":"10187-10228"}}
+{"timestamp":1714755334.3672419,"name":"undrain","context":{"idset":"10159-10184"}}
+{"timestamp":1714755342.1175656,"name":"undrain","context":{"idset":"10155-10157"}}
+{"timestamp":1714755360.5758638,"name":"undrain","context":{"idset":"10108-10133"}}
+{"timestamp":1714755369.2847886,"name":"undrain","context":{"idset":"10135-10153"}}
+{"timestamp":1714755462.0635433,"name":"undrain","context":{"idset":"925-926"}}
+{"timestamp":1714755518.6913207,"name":"drain","context":{"idset":"925-926","reason":"node failure","overwrite":0}}
+{"timestamp":1714755524.3342824,"name":"offline","context":{"idset":"6139"}}
+{"timestamp":1714755524.41816,"name":"offline","context":{"idset":"6140"}}
+{"timestamp":1714755557.503711,"name":"offline","context":{"idset":"925"}}
+{"timestamp":1714755557.6299679,"name":"offline","context":{"idset":"926"}}
+{"timestamp":1714755586.4167409,"name":"offline","context":{"idset":"7053"}}
+{"timestamp":1714755631.324116,"name":"online","context":{"idset":"925"}}
+{"timestamp":1714755631.6181123,"name":"online","context":{"idset":"926"}}
+{"timestamp":1714755662.3847373,"name":"undrain","context":{"idset":"925-926"}}
+{"timestamp":1714755800.4155943,"name":"offline","context":{"idset":"6513"}}
+{"timestamp":1714755802.4159224,"name":"offline","context":{"idset":"6514"}}
+{"timestamp":1714756098.4152071,"name":"offline","context":{"idset":"6461"}}
+{"timestamp":1714756100.4145637,"name":"offline","context":{"idset":"6462"}}
+{"timestamp":1714758446.4175677,"name":"offline","context":{"idset":"9980"}}
+{"timestamp":1714758450.3524621,"name":"offline","context":{"idset":"9995"}}
+{"timestamp":1714758450.3572857,"name":"offline","context":{"idset":"10042"}}
+{"timestamp":1714758450.36202,"name":"offline","context":{"idset":"10517"}}
+{"timestamp":1714758450.415206,"name":"offline","context":{"idset":"10547"}}
+{"timestamp":1714758455.9173591,"name":"offline","context":{"idset":"10514"}}
+{"timestamp":1714758460.4171126,"name":"offline","context":{"idset":"10507"}}
+{"timestamp":1714758462.4165242,"name":"offline","context":{"idset":"10512"}}
+{"timestamp":1714758475.5294116,"name":"offline","context":{"idset":"10610"}}
+{"timestamp":1714758484.4156592,"name":"offline","context":{"idset":"10536"}}
+{"timestamp":1714758488.4229038,"name":"offline","context":{"idset":"10591"}}
+{"timestamp":1714758492.4161792,"name":"offline","context":{"idset":"10568"}}
+{"timestamp":1714758494.9790983,"name":"offline","context":{"idset":"10587"}}
+{"timestamp":1714758550.4198883,"name":"offline","context":{"idset":"9140"}}
+{"timestamp":1714758582.4207368,"name":"offline","context":{"idset":"877"}}
+{"timestamp":1714758584.4182649,"name":"offline","context":{"idset":"878"}}
+{"timestamp":1714759311.7336497,"name":"undrain","context":{"idset":"7432"}}
+{"timestamp":1714760283.0951397,"name":"offline","context":{"idset":"803"}}
+{"timestamp":1714761105.8741691,"name":"online","context":{"idset":"797-798"}}
+{"timestamp":1714761336.416101,"name":"offline","context":{"idset":"798"}}
+{"timestamp":1714761493.9291599,"name":"online","context":{"idset":"6513"}}
+{"timestamp":1714761494.1982388,"name":"online","context":{"idset":"6140,6238,6461-6462"}}
+{"timestamp":1714761495.2186856,"name":"online","context":{"idset":"6139,6514"}}
+{"timestamp":1714761495.2245903,"name":"online","context":{"idset":"6237"}}
+{"timestamp":1714761587.8868365,"name":"online","context":{"idset":"803"}}
+{"timestamp":1714762159.1395156,"name":"drain","context":{"idset":"11701-11716","reason":"draining to run on-node HPE diags - KPN","overwrite":0}}
+{"timestamp":1714762494.3384545,"name":"offline","context":{"idset":"11701"}}
+{"timestamp":1714762495.231739,"name":"offline","context":{"idset":"11702"}}
+{"timestamp":1714762495.3199925,"name":"offline","context":{"idset":"11703"}}
+{"timestamp":1714762495.3258793,"name":"offline","context":{"idset":"11704"}}
+{"timestamp":1714762495.3317678,"name":"offline","context":{"idset":"11705"}}
+{"timestamp":1714762495.3809519,"name":"offline","context":{"idset":"11706"}}
+{"timestamp":1714762498.341644,"name":"offline","context":{"idset":"11707"}}
+{"timestamp":1714762498.3465679,"name":"offline","context":{"idset":"11708"}}
+{"timestamp":1714762498.4154692,"name":"offline","context":{"idset":"11709"}}
+{"timestamp":1714762500.3591094,"name":"offline","context":{"idset":"11710"}}
+{"timestamp":1714762500.3645289,"name":"offline","context":{"idset":"11711"}}
+{"timestamp":1714762500.3698759,"name":"offline","context":{"idset":"11712"}}
+{"timestamp":1714762500.3752215,"name":"offline","context":{"idset":"11713"}}
+{"timestamp":1714762500.3752983,"name":"drain","context":{"idset":"11714","reason":"epilog failed for jobid ft7snJY9rvB","overwrite":0}}
+{"timestamp":1714762500.4167378,"name":"offline","context":{"idset":"11714"}}
+{"timestamp":1714762502.3368912,"name":"offline","context":{"idset":"11715"}}
+{"timestamp":1714762502.4166977,"name":"offline","context":{"idset":"11716"}}
+{"timestamp":1714762525.6756723,"name":"online","context":{"idset":"971-972"}}
+{"timestamp":1714762583.3879344,"name":"undrain","context":{"idset":"971-972"}}
+{"timestamp":1714763939.9358823,"name":"online","context":{"idset":"11609,11633"}}
+{"timestamp":1714763940.09041,"name":"online","context":{"idset":"11610"}}
+{"timestamp":1714763941.1276183,"name":"online","context":{"idset":"11634"}}
+{"timestamp":1714764576.2700822,"name":"online","context":{"idset":"7130,7139"}}
+{"timestamp":1714764577.1179631,"name":"online","context":{"idset":"7133-7134,7138"}}
+{"timestamp":1714764577.1232281,"name":"online","context":{"idset":"7137"}}
+{"timestamp":1714764577.1283712,"name":"online","context":{"idset":"7126-7127,7140"}}
+{"timestamp":1714764577.1334949,"name":"online","context":{"idset":"7128,7135-7136"}}
+{"timestamp":1714764577.1836212,"name":"online","context":{"idset":"7131-7132"}}
+{"timestamp":1714764577.4437709,"name":"online","context":{"idset":"7129"}}
+{"timestamp":1714764655.3322577,"name":"undrain","context":{"idset":"7125-7140"}}
+{"timestamp":1714765132.3447423,"name":"online","context":{"idset":"11714"}}
+{"timestamp":1714765133.3123105,"name":"online","context":{"idset":"11701,11708"}}
+{"timestamp":1714765133.3175569,"name":"online","context":{"idset":"11712"}}
+{"timestamp":1714765133.3227427,"name":"online","context":{"idset":"11716"}}
+{"timestamp":1714765133.652782,"name":"online","context":{"idset":"11702,11704,11706-11707,11709"}}
+{"timestamp":1714765133.9569461,"name":"online","context":{"idset":"11703,11705,11710,11715"}}
+{"timestamp":1714765134.079504,"name":"online","context":{"idset":"11713"}}
+{"timestamp":1714765134.3019648,"name":"online","context":{"idset":"11711"}}
+{"timestamp":1714765164.101155,"name":"online","context":{"idset":"11318"}}
+{"timestamp":1714765164.2359657,"name":"online","context":{"idset":"11321"}}
+{"timestamp":1714765165.2484043,"name":"online","context":{"idset":"11317,11322"}}
+{"timestamp":1714765222.0424371,"name":"undrain","context":{"idset":"11701-11716"}}
+{"timestamp":1714765663.7791536,"name":"online","context":{"idset":"6014"}}
+{"timestamp":1714765664.6054997,"name":"online","context":{"idset":"6008,6034"}}
+{"timestamp":1714765665.3296835,"name":"online","context":{"idset":"6021"}}
+{"timestamp":1714765668.3501024,"name":"online","context":{"idset":"6072"}}
+{"timestamp":1714765668.9648807,"name":"online","context":{"idset":"6022"}}
+{"timestamp":1714765669.3325915,"name":"online","context":{"idset":"6047"}}
+{"timestamp":1714765669.895174,"name":"online","context":{"idset":"6044,6087"}}
+{"timestamp":1714765670.4742079,"name":"online","context":{"idset":"6009,6031"}}
+{"timestamp":1714765670.7673867,"name":"online","context":{"idset":"6017"}}
+{"timestamp":1714765670.8812175,"name":"online","context":{"idset":"6010"}}
+{"timestamp":1714765671.0759938,"name":"online","context":{"idset":"6016,6071,6107"}}
+{"timestamp":1714765671.2533739,"name":"online","context":{"idset":"6041"}}
+{"timestamp":1714765671.4510546,"name":"online","context":{"idset":"6079"}}
+{"timestamp":1714765672.2757399,"name":"online","context":{"idset":"6132"}}
+{"timestamp":1714765672.2813106,"name":"online","context":{"idset":"6118,6125"}}
+{"timestamp":1714765672.286237,"name":"online","context":{"idset":"6011"}}
+{"timestamp":1714765672.2915285,"name":"online","context":{"idset":"6064"}}
+{"timestamp":1714765672.2978668,"name":"online","context":{"idset":"6097"}}
+{"timestamp":1714765672.3042672,"name":"online","context":{"idset":"6007"}}
+{"timestamp":1714765672.3102088,"name":"online","context":{"idset":"6069,6080"}}
+{"timestamp":1714765672.435317,"name":"online","context":{"idset":"6045"}}
+{"timestamp":1714765672.628581,"name":"online","context":{"idset":"6020,6036,6058,6088,6092,6103,6109"}}
+{"timestamp":1714765673.1602929,"name":"online","context":{"idset":"6018-6019,6024,6026,6043,6052,6056,6061,6063,6066,6074,6077,6082,6106,6111,6113,6119,6121"}}
+{"timestamp":1714765673.164495,"name":"online","context":{"idset":"6032,6038,6059,6067,6112,6114,6131"}}
+{"timestamp":1714765673.4547334,"name":"online","context":{"idset":"6005,6012,6015,6025,6027-6029,6033,6037,6040,6042,6048-6049,6060,6065,6070,6073,6075,6081,6084-6085,6089-6090,6100,6105,6108,6117,6124,6127"}}
+{"timestamp":1714765673.560487,"name":"online","context":{"idset":"6083,6110"}}
+{"timestamp":1714765673.7738166,"name":"online","context":{"idset":"6013,6053,6057,6068,6091,6099,6116,6123,6128"}}
+{"timestamp":1714765673.8796883,"name":"online","context":{"idset":"6076"}}
+{"timestamp":1714765674.1459923,"name":"online","context":{"idset":"6006,6030,6046,6051,6054-6055,6062,6078,6086,6094,6104,6115,6120,6126"}}
+{"timestamp":1714765675.2062538,"name":"online","context":{"idset":"6023,6035,6039,6050,6093,6095-6096,6098,6122,6129-6130"}}
+{"timestamp":1714765675.2117035,"name":"online","context":{"idset":"6101"}}
+{"timestamp":1714765675.2180922,"name":"online","context":{"idset":"6102"}}
+{"timestamp":1714765819.3086596,"name":"online","context":{"idset":"6290"}}
+{"timestamp":1714765821.1914058,"name":"online","context":{"idset":"6268"}}
+{"timestamp":1714765821.195374,"name":"online","context":{"idset":"6297"}}
+{"timestamp":1714765821.5113943,"name":"online","context":{"idset":"6337"}}
+{"timestamp":1714765821.820631,"name":"online","context":{"idset":"6278,6289"}}
+{"timestamp":1714765822.0147951,"name":"online","context":{"idset":"6287"}}
+{"timestamp":1714765822.1738286,"name":"online","context":{"idset":"6264,6293"}}
+{"timestamp":1714765823.8808722,"name":"online","context":{"idset":"6281,6317"}}
+{"timestamp":1714765823.9923153,"name":"online","context":{"idset":"6265"}}
+{"timestamp":1714765824.1522982,"name":"online","context":{"idset":"6267,6299"}}
+{"timestamp":1714765824.2782772,"name":"online","context":{"idset":"6325"}}
+{"timestamp":1714765825.0866382,"name":"online","context":{"idset":"6275,6328,6357"}}
+{"timestamp":1714765825.0921135,"name":"online","context":{"idset":"6307"}}
+{"timestamp":1714765825.323137,"name":"online","context":{"idset":"6322"}}
+{"timestamp":1714765825.8447163,"name":"online","context":{"idset":"6375"}}
+{"timestamp":1714765826.1315904,"name":"online","context":{"idset":"6310"}}
+{"timestamp":1714765827.1516533,"name":"online","context":{"idset":"6377"}}
+{"timestamp":1714765827.1577325,"name":"online","context":{"idset":"6285"}}
+{"timestamp":1714765827.1632249,"name":"online","context":{"idset":"6313,6333"}}
+{"timestamp":1714765827.1683571,"name":"online","context":{"idset":"6352"}}
+{"timestamp":1714765827.1736071,"name":"online","context":{"idset":"6331,6374"}}
+{"timestamp":1714765827.1790886,"name":"online","context":{"idset":"6304,6326,6345"}}
+{"timestamp":1714765827.1854961,"name":"online","context":{"idset":"6277"}}
+{"timestamp":1714765827.1916375,"name":"online","context":{"idset":"6269,6305,6354"}}
+{"timestamp":1714765827.2179408,"name":"online","context":{"idset":"6261,6271,6296,6342,6364,6382"}}
+{"timestamp":1714765827.5179391,"name":"online","context":{"idset":"6266,6274,6347,6360,6376,6379"}}
+{"timestamp":1714765827.8125417,"name":"online","context":{"idset":"6263,6272,6282-6283,6288,6306,6318-6319,6334,6338-6339,6349-6350,6355,6358,6363,6384"}}
+{"timestamp":1714765827.8280699,"name":"online","context":{"idset":"6323,6368,6370"}}
+{"timestamp":1714765828.1077914,"name":"online","context":{"idset":"6262,6273,6279,6286,6302,6308,6324,6332,6336,6348,6353,6362,6386"}}
+{"timestamp":1714765828.4059644,"name":"online","context":{"idset":"6292,6309,6314,6320,6329,6346,6388"}}
+{"timestamp":1714765829.1578479,"name":"online","context":{"idset":"6294,6327,6371-6372"}}
+{"timestamp":1714765829.1634991,"name":"online","context":{"idset":"6284,6330"}}
+{"timestamp":1714765829.1688735,"name":"online","context":{"idset":"6270,6276,6280,6298,6300-6301,6303,6315-6316,6335,6356,6361"}}
+{"timestamp":1714765829.1740711,"name":"online","context":{"idset":"6295,6312,6343-6344,6359,6365-6366,6373,6378,6380,6387"}}
+{"timestamp":1714765829.17926,"name":"online","context":{"idset":"6321,6351,6367"}}
+{"timestamp":1714765829.1844783,"name":"online","context":{"idset":"6291,6311,6341,6369"}}
+{"timestamp":1714765829.5007389,"name":"online","context":{"idset":"6385"}}
+{"timestamp":1714765830.0695431,"name":"online","context":{"idset":"6340,6381"}}
+{"timestamp":1714765871.3518236,"name":"drain","context":{"idset":"6261-6382,6384-6388","reason":"--reason draining to run on-node diags - wendy","overwrite":0}}
diff --git a/t/resource/resource.eventlog.thing2 b/t/resource/resource.eventlog.thing2
new file mode 100644
index 000000000000..87c839eeea52
--- /dev/null
+++ b/t/resource/resource.eventlog.thing2
@@ -0,0 +1,4611 @@
+{"timestamp":1683047871.7659676,"name":"resource-init","context":{"restart":false,"drain":{},"online":"","exclude":"0-2"}}
+{"timestamp":1683047871.7666073,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683047873.7181838,"name":"online","context":{"idset":"0"}}
+{"timestamp":1683047902.596009,"name":"online","context":{"idset":"2"}}
+{"timestamp":1683047903.0977581,"name":"online","context":{"idset":"5"}}
+{"timestamp":1683047903.1142714,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1683047903.9205258,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1683047904.2724149,"name":"online","context":{"idset":"9"}}
+{"timestamp":1683047904.3048165,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1683047904.4366722,"name":"online","context":{"idset":"1,6,18"}}
+{"timestamp":1683047904.489671,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1683047904.5168133,"name":"online","context":{"idset":"10,20"}}
+{"timestamp":1683047904.5168378,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1683047904.5602491,"name":"online","context":{"idset":"30"}}
+{"timestamp":1683047904.5602727,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1683047904.600184,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1683047904.6232867,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1683047904.6344221,"name":"online","context":{"idset":"14"}}
+{"timestamp":1683047904.7251253,"name":"online","context":{"idset":"7,19,25-26,28"}}
+{"timestamp":1683047904.7251632,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1683047904.7483013,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1683047904.7484841,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1683047904.763068,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1683047904.7697272,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1683047904.8203282,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1683047904.8510849,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1683047904.9891832,"name":"online","context":{"idset":"3,16"}}
+{"timestamp":1683047905.0296061,"name":"online","context":{"idset":"12"}}
+{"timestamp":1683047905.0463812,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1683047905.094475,"name":"online","context":{"idset":"8"}}
+{"timestamp":1683047905.0945046,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1683047905.1459196,"name":"online","context":{"idset":"24"}}
+{"timestamp":1683047905.1459477,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1683047905.2036214,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1683047905.3042328,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1683047907.3272793,"name":"resource-init","context":{"restart":true,"drain":{},"online":"","exclude":"0-2"}}
+{"timestamp":1683047907.3279848,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683047909.2514827,"name":"online","context":{"idset":"0"}}
+{"timestamp":1683047909.8647997,"name":"online","context":{"idset":"2,5"}}
+{"timestamp":1683047910.2986803,"name":"online","context":{"idset":"9,33,35"}}
+{"timestamp":1683047910.38449,"name":"online","context":{"idset":"6"}}
+{"timestamp":1683047910.4932876,"name":"online","context":{"idset":"18,34"}}
+{"timestamp":1683047910.5096874,"name":"online","context":{"idset":"1"}}
+{"timestamp":1683047910.673635,"name":"online","context":{"idset":"10,14,19-20,26,28-30"}}
+{"timestamp":1683047910.8216505,"name":"online","context":{"idset":"4,7,13,25"}}
+{"timestamp":1683047910.9668987,"name":"online","context":{"idset":"3,16-17"}}
+{"timestamp":1683047911.0953674,"name":"online","context":{"idset":"8,32"}}
+{"timestamp":1683047911.1962578,"name":"online","context":{"idset":"12"}}
+{"timestamp":1683047911.3117402,"name":"online","context":{"idset":"11,24,31"}}
+{"timestamp":1683047911.5406401,"name":"online","context":{"idset":"15,23"}}
+{"timestamp":1683047912.0250113,"name":"online","context":{"idset":"27"}}
+{"timestamp":1683048405.4099133,"name":"drain","context":{"idset":"21-22","reason":"flux issues","overwrite":0}}
+{"timestamp":1683049263.283375,"name":"online","context":{"idset":"21"}}
+{"timestamp":1683049271.6013002,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1683049272.4109766,"name":"online","context":{"idset":"21"}}
+{"timestamp":1683049286.1057158,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1683049286.9015405,"name":"online","context":{"idset":"21"}}
+{"timestamp":1683049348.3732798,"name":"online","context":{"idset":"22"}}
+{"timestamp":1683049380.3666673,"name":"undrain","context":{"idset":"21-22"}}
+{"timestamp":1683115661.9314654,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1683115661.9337087,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1683115661.936327,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1683115661.9368961,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1683115661.9399431,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1683115661.9418101,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1683115661.9421139,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1683115661.9445498,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1683115661.9446898,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1683115661.9458606,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1683115661.9460535,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1683115661.9464996,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1683115661.9479113,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1683115661.9503944,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1683115661.9509339,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1683115661.9513922,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1683115661.9520171,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1683115661.9534485,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1683115661.9559844,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1683115661.9579349,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1683115661.9581363,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1683115661.9586751,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1683115661.9603388,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1683115661.961334,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1683115661.9633029,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1683115661.9673383,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1683115662.0135608,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1683115662.1136427,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1683115662.1499932,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1683115662.1543112,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1683115662.1878328,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1683115662.1894391,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1683115662.22647,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1683115662.3265238,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1683115662.4758015,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1683115854.553304,"name":"resource-init","context":{"restart":true,"drain":{},"online":"","exclude":"0-2"}}
+{"timestamp":1683115854.5546751,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683115856.9609318,"name":"online","context":{"idset":"0"}}
+{"timestamp":1683115857.6518917,"name":"online","context":{"idset":"2,23-35"}}
+{"timestamp":1683115857.9520576,"name":"online","context":{"idset":"17-20"}}
+{"timestamp":1683115858.0855093,"name":"online","context":{"idset":"1"}}
+{"timestamp":1683115858.233273,"name":"online","context":{"idset":"9,13-14"}}
+{"timestamp":1683115858.3422999,"name":"online","context":{"idset":"3,8,16"}}
+{"timestamp":1683115858.4449484,"name":"online","context":{"idset":"5-6,10-12,15"}}
+{"timestamp":1683115858.6592736,"name":"online","context":{"idset":"4,7"}}
+{"timestamp":1683116422.665045,"name":"online","context":{"idset":"36"}}
+{"timestamp":1683156347.6795416,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1683156348.7177422,"name":"online","context":{"idset":"1"}}
+{"timestamp":1683160404.1286287,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1287606,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1288059,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.128864,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.128907,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1289487,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1289907,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.129029,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1290691,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.129112,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1291697,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1292145,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1292553,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1292968,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1293421,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.129384,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1294291,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1294715,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1295171,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1295598,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1296029,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1296492,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1296921,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1297357,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1297786,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1298234,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1298707,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1299155,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1299617,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1300075,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1300535,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1301022,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1301482,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1683160404.1302085,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1683162792.0288348,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1683162792.0291719,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1683162792.0299144,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1683162792.0302522,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1683162792.0307651,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1683162792.0321119,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1683162792.0322206,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1683162792.0322802,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1683162792.032337,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1683162792.0323975,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1683162792.0324569,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1683162792.03441,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1683162792.0344741,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1683162792.0345421,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1683162792.0345948,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1683162792.0346484,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1683162792.0347013,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1683162792.0347521,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1683162792.0348027,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1683162792.0390036,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1683162792.0425341,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1683162792.0461166,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1683162792.0474489,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1683162792.066206,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1683162792.0683291,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1683162792.0706203,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1683162792.0737317,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1683162792.0764561,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1683162792.0932703,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1683162792.0934169,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1683162792.099987,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1683162792.1145844,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1683162792.2152812,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1683162792.3455136,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1683162799.9691737,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683162799.9709246,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683163237.2204976,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683163237.2218342,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683164784.3887296,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683164784.3904054,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683165084.0717537,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683165084.0730741,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683165568.0012693,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683165568.0029511,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683165853.3905201,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683165853.391911,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683165891.8675637,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683165891.86902,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683165965.4596598,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683165965.4610212,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683165996.5902231,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683165996.5916562,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683166103.1004114,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683166103.1017964,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683166144.7767034,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683166144.7781787,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683166170.6458168,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683166170.6472063,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683166191.6923981,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683166191.6938186,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683166216.7372274,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683166216.7386281,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683166239.5474751,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683166239.5488527,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683166269.2655847,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683166269.2670062,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683166345.6680517,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1683160404.1288059,"reason":"broker was unresponsive"},"4":{"timestamp":1683160404.128864,"reason":"broker was unresponsive"},"5":{"timestamp":1683160404.128907,"reason":"broker was unresponsive"},"6":{"timestamp":1683160404.1289487,"reason":"broker was unresponsive"},"7":{"timestamp":1683160404.1289907,"reason":"broker was unresponsive"},"8":{"timestamp":1683160404.129029,"reason":"broker was unresponsive"},"9":{"timestamp":1683160404.1290691,"reason":"broker was unresponsive"},"10":{"timestamp":1683160404.129112,"reason":"broker was unresponsive"},"11":{"timestamp":1683160404.1291697,"reason":"broker was unresponsive"},"12":{"timestamp":1683160404.1292145,"reason":"broker was unresponsive"},"13":{"timestamp":1683160404.1292553,"reason":"broker was unresponsive"},"14":{"timestamp":1683160404.1292968,"reason":"broker was unresponsive"},"15":{"timestamp":1683160404.1293421,"reason":"broker was unresponsive"},"16":{"timestamp":1683160404.129384,"reason":"broker was unresponsive"},"17":{"timestamp":1683160404.1294291,"reason":"broker was unresponsive"},"18":{"timestamp":1683160404.1294715,"reason":"broker was unresponsive"},"19":{"timestamp":1683160404.1295171,"reason":"broker was unresponsive"},"20":{"timestamp":1683160404.1295598,"reason":"broker was unresponsive"},"23":{"timestamp":1683160404.1296029,"reason":"broker was unresponsive"},"24":{"timestamp":1683160404.1296492,"reason":"broker was unresponsive"},"25":{"timestamp":1683160404.1296921,"reason":"broker was unresponsive"},"26":{"timestamp":1683160404.1297357,"reason":"broker was unresponsive"},"27":{"timestamp":1683160404.1297786,"reason":"broker was unresponsive"},"28":{"timestamp":1683160404.1298234,"reason":"broker was unresponsive"},"29":{"timestamp":1683160404.1298707,"reason":"broker was unresponsive"},"30":{"timestamp":1683160404.1299155,"reason":"broker was unresponsive"},"31":{"timestamp":1683160404.1299617,"reason":"broker was unresponsive"},"32":{"timestamp":1683160404.1300075,"reason":"broker was unresponsive"},"33":{"timestamp":1683160404.1300535,"reason":"broker was unresponsive"},"34":{"timestamp":1683160404.1301022,"reason":"broker was unresponsive"},"35":{"timestamp":1683160404.1301482,"reason":"broker was unresponsive"},"36":{"timestamp":1683160404.1302085,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683166345.6698284,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683166350.0508087,"name":"online","context":{"idset":"0"}}
+{"timestamp":1683166350.7550082,"name":"online","context":{"idset":"2,7,11,18"}}
+{"timestamp":1683166350.8653672,"name":"online","context":{"idset":"1"}}
+{"timestamp":1683166351.0565979,"name":"online","context":{"idset":"33-36"}}
+{"timestamp":1683166351.2053781,"name":"online","context":{"idset":"8,25-26"}}
+{"timestamp":1683166351.3159494,"name":"online","context":{"idset":"6,16,23"}}
+{"timestamp":1683166351.4343805,"name":"online","context":{"idset":"3-4,15,17,19,29-30,32"}}
+{"timestamp":1683166351.5475409,"name":"online","context":{"idset":"5,14,20,27-28,31"}}
+{"timestamp":1683166351.6889558,"name":"online","context":{"idset":"10,13,24"}}
+{"timestamp":1683166351.7959569,"name":"online","context":{"idset":"9"}}
+{"timestamp":1683166351.9071267,"name":"online","context":{"idset":"12"}}
+{"timestamp":1683166533.6951408,"name":"undrain","context":{"idset":"3-20,23-36"}}
+{"timestamp":1683732056.9009645,"name":"drain","context":{"idset":"3-36","reason":"toss_update","overwrite":0}}
+{"timestamp":1683748321.620621,"name":"resource-init","context":{"restart":true,"drain":{"3-36":{"timestamp":1683732056.9009645,"reason":"toss_update"}},"online":"","exclude":"0-2"}}
+{"timestamp":1683748321.6220922,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1683748337.1150377,"name":"online","context":{"idset":"0"}}
+{"timestamp":1683748337.8054342,"name":"online","context":{"idset":"1-2"}}
+{"timestamp":1683748338.1250315,"name":"online","context":{"idset":"33-36"}}
+{"timestamp":1683748338.3085253,"name":"online","context":{"idset":"5-6,16,18,26,29"}}
+{"timestamp":1683748338.4158349,"name":"online","context":{"idset":"9-10,13,30"}}
+{"timestamp":1683748338.5213413,"name":"online","context":{"idset":"4,12,14-15,17,23-24,31"}}
+{"timestamp":1683748338.6356888,"name":"online","context":{"idset":"3,7-8,19-20,25,27-28,32"}}
+{"timestamp":1683748338.7448418,"name":"online","context":{"idset":"11"}}
+{"timestamp":1683748501.7866225,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1683748505.8864768,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1683748595.7863703,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1683748595.7865174,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1683748595.8868542,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1683748597.7858415,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1683748597.785953,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1683748597.7860334,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1683748597.7861118,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1683748597.7862685,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1683748597.7863908,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1683748597.7864666,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1683748597.7865393,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1683748597.7866199,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1683748597.786694,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1683748597.7867956,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1683748597.7868979,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1683748597.7869904,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1683748597.7870839,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1683748597.7871747,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1683748597.7872636,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1683748597.7873456,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1683748597.7874224,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1683748597.7874992,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1683748597.7875776,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1683748597.787653,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1683748597.787724,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1683748597.7878053,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1683748597.7878752,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1683748597.7879422,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1683748597.7880123,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1683748597.7880771,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1683748597.7881405,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1683748597.7882028,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1683748597.8860688,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1683748599.8867273,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1683750917.7638783,"name":"online","context":{"idset":"17"}}
+{"timestamp":1683750918.0665629,"name":"online","context":{"idset":"34,36"}}
+{"timestamp":1683750918.1825147,"name":"online","context":{"idset":"33,35"}}
+{"timestamp":1683750918.3327434,"name":"online","context":{"idset":"4"}}
+{"timestamp":1683750918.8656781,"name":"online","context":{"idset":"11"}}
+{"timestamp":1683750919.2380023,"name":"online","context":{"idset":"10"}}
+{"timestamp":1683750919.3473537,"name":"online","context":{"idset":"3,6-7,9,31"}}
+{"timestamp":1683750919.4803805,"name":"online","context":{"idset":"5,14,22-23,32"}}
+{"timestamp":1683750919.60566,"name":"online","context":{"idset":"16,20,24,28"}}
+{"timestamp":1683750919.7868838,"name":"online","context":{"idset":"15"}}
+{"timestamp":1683750920.2277422,"name":"online","context":{"idset":"18,29"}}
+{"timestamp":1683750920.4562454,"name":"online","context":{"idset":"21,25"}}
+{"timestamp":1683750920.6095209,"name":"online","context":{"idset":"30"}}
+{"timestamp":1683750923.3069961,"name":"online","context":{"idset":"12,19"}}
+{"timestamp":1683751117.47542,"name":"drain","context":{"idset":"13","reason":"kernel panic","overwrite":1}}
+{"timestamp":1683751377.4147542,"name":"online","context":{"idset":"1"}}
+{"timestamp":1683751576.992115,"name":"online","context":{"idset":"2"}}
+{"timestamp":1683751620.01055,"name":"drain","context":{"idset":"8,13,26-27","reason":"kernel panic","overwrite":1}}
+{"timestamp":1683751635.792393,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1683751635.8673952,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1683751636.6359398,"name":"online","context":{"idset":"1-2"}}
+{"timestamp":1683751648.5732882,"name":"undrain","context":{"idset":"1-2"}}
+{"timestamp":1683751664.1794326,"name":"undrain","context":{"idset":"3-7,9-12,14-25,28-36"}}
+{"timestamp":1683819539.2826321,"name":"drain","context":{"idset":"7,14","reason":"JRG: drain partner for ts","overwrite":0}}
+{"timestamp":1683819703.886971,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1683819705.8869145,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1683819837.6678538,"name":"drain","context":{"idset":"25,28","reason":"JRG: drain partner for ts","overwrite":0}}
+{"timestamp":1683819987.5234661,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1683819987.624042,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1683822684.1630032,"name":"online","context":{"idset":"26"}}
+{"timestamp":1683822689.6634328,"name":"online","context":{"idset":"27"}}
+{"timestamp":1683823582.3581014,"name":"online","context":{"idset":"13"}}
+{"timestamp":1683824478.2114956,"name":"online","context":{"idset":"8"}}
+{"timestamp":1683825207.3232718,"name":"online","context":{"idset":"7"}}
+{"timestamp":1683825285.9102516,"name":"undrain","context":{"idset":"7-8,13-14,25-28"}}
+{"timestamp":1683825654.1066372,"name":"online","context":{"idset":"14,25,28"}}
+{"timestamp":1683826214.8493531,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1683827333.6086311,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1683834228.9709909,"name":"drain","context":{"idset":"14,25,28","reason":"needs nodeup","overwrite":0}}
+{"timestamp":1683924227.8873079,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1683924317.7245147,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1684169752.0742114,"name":"online","context":{"idset":"1"}}
+{"timestamp":1684169855.2418246,"name":"undrain","context":{"idset":"6-7"}}
+{"timestamp":1684169874.5458691,"name":"undrain","context":{"idset":"1"}}
+{"timestamp":1684169890.8460426,"name":"undrain","context":{"idset":"14,25,28"}}
+{"timestamp":1684170384.2680445,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684191914.9674315,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1684192627.3388937,"name":"online","context":{"idset":"7"}}
+{"timestamp":1684192657.4986045,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1684373149.0404875,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684385808.0802972,"name":"drain","context":{"idset":"4","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684427407.8861322,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1684427409.8872719,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1684428234.0527787,"name":"online","context":{"idset":"7"}}
+{"timestamp":1684428234.2411191,"name":"online","context":{"idset":"4"}}
+{"timestamp":1684428317.6829014,"name":"undrain","context":{"idset":"4,7"}}
+{"timestamp":1684529922.0356793,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684788881.8864245,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1684789666.8649645,"name":"online","context":{"idset":"5"}}
+{"timestamp":1684875277.3919578,"name":"undrain","context":{"idset":"5"}}
+{"timestamp":1684876575.1180613,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684879563.0264273,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684947045.7866769,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1684947045.8872795,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1684947816.0801024,"name":"online","context":{"idset":"5"}}
+{"timestamp":1684947816.2800372,"name":"online","context":{"idset":"6"}}
+{"timestamp":1684947841.7853084,"name":"undrain","context":{"idset":"5-6"}}
+{"timestamp":1684962858.6031675,"name":"drain","context":{"idset":"4","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684963193.6090403,"name":"drain","context":{"idset":"11","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684963194.0522773,"name":"drain","context":{"idset":"9","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684963195.1113918,"name":"drain","context":{"idset":"17","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684963195.3092155,"name":"drain","context":{"idset":"14","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684963195.5571187,"name":"drain","context":{"idset":"16","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684964849.8869543,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1684964851.786428,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1684964851.786556,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1684964851.7866561,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1684964851.7867382,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1684964851.8866146,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1684966219.7870421,"name":"online","context":{"idset":"17"}}
+{"timestamp":1684966220.1725612,"name":"online","context":{"idset":"4"}}
+{"timestamp":1684966220.4602518,"name":"online","context":{"idset":"16"}}
+{"timestamp":1684966220.5746324,"name":"online","context":{"idset":"9,11,14"}}
+{"timestamp":1684966236.8948069,"name":"drain","context":{"idset":"25","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684966236.9342258,"name":"drain","context":{"idset":"19","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684966236.9849858,"name":"drain","context":{"idset":"26","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684966238.7645166,"name":"drain","context":{"idset":"29","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1684966376.7974205,"name":"undrain","context":{"idset":"4,9,11,14,16-17"}}
+{"timestamp":1685038875.7861464,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1685038875.7863345,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1685038875.8865273,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1685038877.8871939,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1685039799.1612513,"name":"online","context":{"idset":"19"}}
+{"timestamp":1685039799.760076,"name":"online","context":{"idset":"25-26,29"}}
+{"timestamp":1685039822.1983812,"name":"undrain","context":{"idset":"19,25-26,29"}}
+{"timestamp":1685116354.1591506,"name":"drain","context":{"idset":"4","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1685122455.886898,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1685123615.4631956,"name":"online","context":{"idset":"4"}}
+{"timestamp":1685123660.0416374,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1685477306.8740525,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1685636807.8869379,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1685637658.6232207,"name":"online","context":{"idset":"7"}}
+{"timestamp":1685637680.696929,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1685662530.4070165,"name":"drain","context":{"idset":"13","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1685724447.8871636,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1685725782.5904732,"name":"online","context":{"idset":"13"}}
+{"timestamp":1685725804.3359475,"name":"undrain","context":{"idset":"13"}}
+{"timestamp":1685783215.8770959,"name":"drain","context":{"idset":"4","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1685982853.8874407,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1685985089.760874,"name":"online","context":{"idset":"4"}}
+{"timestamp":1685992650.7199693,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1686081678.624018,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1686087358.259593,"name":"drain","context":{"idset":"8","overwrite":0}}
+{"timestamp":1686087378.6208158,"name":"drain","context":{"idset":"8","reason":"update ansible","overwrite":0}}
+{"timestamp":1686155275.7866137,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1686155275.8867335,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1686156961.3388293,"name":"online","context":{"idset":"5,8"}}
+{"timestamp":1686156987.2019901,"name":"undrain","context":{"idset":"5,8"}}
+{"timestamp":1686163657.8210664,"name":"drain","context":{"idset":"16","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1686199056.2598407,"name":"drain","context":{"idset":"10","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1686241510.0051856,"name":"drain","context":{"idset":"4","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1686244713.8866267,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1686244715.7862122,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1686244715.8870401,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1686245753.0504334,"name":"online","context":{"idset":"10"}}
+{"timestamp":1686245753.4295375,"name":"online","context":{"idset":"4"}}
+{"timestamp":1686245753.7659333,"name":"online","context":{"idset":"16"}}
+{"timestamp":1686247133.0394862,"name":"undrain","context":{"idset":"4,10,16"}}
+{"timestamp":1686255123.8593733,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1686284476.9858832,"name":"drain","context":{"idset":"10","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1686321815.584425,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1686322466.3920152,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1686322466.3951468,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1686322466.4364505,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1686322466.4379063,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1686322466.4444304,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1686322466.4477501,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1686322466.4530685,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1686322466.4654698,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1686322466.466115,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1686322466.4735317,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1686322466.4799979,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1686322466.4824643,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1686322466.4837935,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1686322466.4849913,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1686322466.4860542,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1686322466.4869874,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1686322466.489466,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1686322466.4953475,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1686322466.4959507,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1686322466.4974937,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1686322466.5088444,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1686322466.5099525,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1686322466.5146184,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1686322466.5286646,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1686322466.5323284,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1686322466.5357804,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1686322466.5637023,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1686322466.6063449,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1686322466.6248019,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1686322466.6379604,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1686322466.645273,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1686322466.6569867,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1686322466.6611462,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1686322466.6932139,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1686322466.6998024,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1686322466.8006725,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1686341189.0594072,"name":"resource-init","context":{"restart":true,"drain":{"5":{"timestamp":1686321815.584425,"reason":"nodediag failed amdgpu"},"6":{"timestamp":1686255123.8593733,"reason":"nodediag failed amdgpu"},"10":{"timestamp":1686284476.9858832,"reason":"nodediag failed amdgpu"}},"online":"","exclude":"0-2"}}
+{"timestamp":1686341189.0605321,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1686341270.6031895,"name":"online","context":{"idset":"0"}}
+{"timestamp":1686341271.4912853,"name":"online","context":{"idset":"2,4,6,8,10,12,14-15,17-21,25,27-31"}}
+{"timestamp":1686341271.5993192,"name":"online","context":{"idset":"3,5,22-24,32"}}
+{"timestamp":1686341271.707767,"name":"online","context":{"idset":"9,13,16"}}
+{"timestamp":1686341271.8114624,"name":"online","context":{"idset":"11"}}
+{"timestamp":1686341271.9252803,"name":"online","context":{"idset":"7"}}
+{"timestamp":1686343739.8611534,"name":"undrain","context":{"idset":"5-6,10"}}
+{"timestamp":1686583886.191025,"name":"drain","context":{"idset":"25","reason":"broken partner node","overwrite":0}}
+{"timestamp":1686583930.416127,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1686600272.1024852,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1686601277.5063143,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1686601277.6069403,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1686601373.5063775,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1686601373.60671,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1686603329.5065322,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1686603329.606683,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1686603421.5067344,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1686603421.6070187,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1686605170.4152982,"name":"online","context":{"idset":"1"}}
+{"timestamp":1686605221.9310002,"name":"online","context":{"idset":"2"}}
+{"timestamp":1686640758.723974,"name":"drain","context":{"idset":"19","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1686714184.3147929,"name":"drain","context":{"idset":"4","reason":"prolog failed for jobid f9BEzgpsKaX","overwrite":0}}
+{"timestamp":1686898410.7932065,"name":"drain","context":{"idset":"13","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1687321671.2697661,"name":"drain","context":{"idset":"12","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1687376313.3019717,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1687545496.1398079,"name":"drain","context":{"idset":"3","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1687602315.0311725,"name":"drain","context":{"idset":"20","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1687611462.1927361,"name":"drain","context":{"idset":"15","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1687810982.8620956,"name":"drain","context":{"idset":"24","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1687812810.336946,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1687812811.5060968,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1687812811.5062549,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1687812811.6064847,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1687813437.5062656,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1687813437.5064354,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1687813437.5065238,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1687813437.5066056,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1687813437.6068332,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1687814273.1174664,"name":"online","context":{"idset":"12"}}
+{"timestamp":1687814273.3896263,"name":"online","context":{"idset":"28"}}
+{"timestamp":1687814273.5063608,"name":"online","context":{"idset":"24"}}
+{"timestamp":1687814273.6072438,"name":"online","context":{"idset":"4"}}
+{"timestamp":1687814273.7244589,"name":"online","context":{"idset":"20,31-32"}}
+{"timestamp":1687814274.1748614,"name":"online","context":{"idset":"25"}}
+{"timestamp":1687814274.8954172,"name":"online","context":{"idset":"15,27"}}
+{"timestamp":1687814276.2386823,"name":"online","context":{"idset":"19"}}
+{"timestamp":1687814276.6125693,"name":"online","context":{"idset":"13"}}
+{"timestamp":1687814277.3528614,"name":"online","context":{"idset":"3"}}
+{"timestamp":1687814281.3044066,"name":"online","context":{"idset":"5"}}
+{"timestamp":1687814423.7734134,"name":"undrain","context":{"idset":"3-5,12-13,15,19-20,24-25,27-28,31-32"}}
+{"timestamp":1687814477.9647334,"name":"online","context":{"idset":"26"}}
+{"timestamp":1687814538.5924499,"name":"online","context":{"idset":"35-36"}}
+{"timestamp":1687814538.7238629,"name":"online","context":{"idset":"34"}}
+{"timestamp":1687814541.1967599,"name":"online","context":{"idset":"33"}}
+{"timestamp":1687840245.3926859,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1687877401.032244,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1687877401.1322174,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1687877401.1350052,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1687877401.1374559,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1687877401.1383166,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1687877401.1801558,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1687877401.1865575,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1687877401.2065289,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1687877401.2066252,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1687877401.2160223,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1687877401.2203431,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1687877401.2221334,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1687877401.2257354,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1687877401.2259455,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1687877401.232667,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1687877401.2535379,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1687877401.2653077,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1687877401.2695031,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1687877401.2817261,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1687877401.3349619,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1687877401.3411489,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1687877401.3416047,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1687877401.3604689,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1687877401.3948221,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1687877401.4154084,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1687877401.4188719,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1687877401.419081,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1687877401.4363878,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1687877401.4367635,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1687877401.4737442,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1687877401.5021513,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1687877401.6027293,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1687877401.8565438,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1687877401.9398537,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1687877401.9445074,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1687877402.0454702,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1687904320.5474653,"name":"resource-init","context":{"restart":true,"drain":{"7":{"timestamp":1687840245.3926859,"reason":"nodediag failed amdgpu"}},"online":"","exclude":"0-2"}}
+{"timestamp":1687904320.5484357,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1687904433.3020122,"name":"online","context":{"idset":"0"}}
+{"timestamp":1687908205.6857595,"name":"online","context":{"idset":"2"}}
+{"timestamp":1687908205.8818951,"name":"online","context":{"idset":"1"}}
+{"timestamp":1687908257.2212946,"name":"online","context":{"idset":"36"}}
+{"timestamp":1687908257.349519,"name":"online","context":{"idset":"35"}}
+{"timestamp":1687908257.4529624,"name":"online","context":{"idset":"33"}}
+{"timestamp":1687908257.6890216,"name":"online","context":{"idset":"11,22,31"}}
+{"timestamp":1687908257.7632115,"name":"online","context":{"idset":"17,34"}}
+{"timestamp":1687908257.8818374,"name":"online","context":{"idset":"14"}}
+{"timestamp":1687908258.1154268,"name":"online","context":{"idset":"9,15,25,30"}}
+{"timestamp":1687908258.2620282,"name":"online","context":{"idset":"7,10,13,21,28"}}
+{"timestamp":1687908258.4321821,"name":"online","context":{"idset":"5,12,32"}}
+{"timestamp":1687908258.5474312,"name":"online","context":{"idset":"6,8,16,19-20,26,29"}}
+{"timestamp":1687908258.7407665,"name":"online","context":{"idset":"4,24"}}
+{"timestamp":1687908258.8792,"name":"online","context":{"idset":"23"}}
+{"timestamp":1687908259.1107841,"name":"online","context":{"idset":"27"}}
+{"timestamp":1687910197.7839174,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1688078114.7269762,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1688106590.6283898,"name":"drain","context":{"idset":"4","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1688171254.8969445,"name":"drain","context":{"idset":"10","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1688175536.9519937,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1688180189.8124971,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1688399436.2427216,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1688399436.2792625,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1688399436.3036582,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1688399436.342623,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1688399436.4429579,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1688400379.405575,"name":"online","context":{"idset":"6,10"}}
+{"timestamp":1688400379.6050229,"name":"online","context":{"idset":"4"}}
+{"timestamp":1688400379.7331963,"name":"online","context":{"idset":"5,7"}}
+{"timestamp":1688400402.4930494,"name":"undrain","context":{"idset":"4-7,10"}}
+{"timestamp":1688582005.6532092,"name":"drain","context":{"idset":"8","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1688593034.9604552,"name":"drain","context":{"idset":"10","reason":"prolog failed for jobid fDMXsKKtAuV","overwrite":0}}
+{"timestamp":1688657096.1994245,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1688657096.2996619,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1688657918.6158512,"name":"online","context":{"idset":"8"}}
+{"timestamp":1688657918.7541385,"name":"online","context":{"idset":"10"}}
+{"timestamp":1688657938.6868944,"name":"undrain","context":{"idset":"8,10"}}
+{"timestamp":1688658590.118628,"name":"online","context":{"idset":"3"}}
+{"timestamp":1688659286.2997916,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1988866,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1989958,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1990526,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1991146,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1991673,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1992216,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1992729,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1993372,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1993895,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1994412,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1994936,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1995478,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1996002,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1996512,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1997035,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1997607,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1998119,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.199868,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1999199,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.1999757,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1688659288.2998974,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.1991863,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.1992748,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.1993392,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.1993949,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.1994505,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.1995049,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.1995595,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.1996155,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.1996703,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.1997263,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1688659290.2997088,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1688659352.1988606,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1688659352.1990268,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1688659352.1991479,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1688659352.1992471,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1688659352.199347,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1688659352.1994436,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1688659352.199538,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1688659352.1996324,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1688659352.1997294,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1688659352.1998203,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1688659352.1999235,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1688659352.2000113,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1688659352.2001076,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1688659352.2001967,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1688659352.2002838,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1688659352.2003691,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1688659352.2004533,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1688659352.200537,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1688659352.2006211,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1688659352.2007172,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1688659352.2008095,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1688659352.2994893,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1688659354.1992891,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1688659354.1993945,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1688659354.1994736,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1688659354.1995513,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1688659354.199625,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1688659354.1996973,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1688659354.1997666,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1688659354.1998348,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1688659354.1998999,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1688659354.1999621,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1688659354.2994742,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1688659454.299504,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1688659456.2998719,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1688659520.2993138,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1688659524.2996466,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1688664406.7924533,"name":"online","context":{"idset":"34"}}
+{"timestamp":1688664407.2633905,"name":"online","context":{"idset":"33"}}
+{"timestamp":1688664407.3953438,"name":"online","context":{"idset":"35-36"}}
+{"timestamp":1688664817.5081546,"name":"online","context":{"idset":"25"}}
+{"timestamp":1688664817.6003385,"name":"online","context":{"idset":"12,14"}}
+{"timestamp":1688664817.774579,"name":"online","context":{"idset":"4,16"}}
+{"timestamp":1688664817.8522401,"name":"online","context":{"idset":"6,10"}}
+{"timestamp":1688664817.8663456,"name":"online","context":{"idset":"8"}}
+{"timestamp":1688664818.0299041,"name":"online","context":{"idset":"24,26"}}
+{"timestamp":1688664818.1399195,"name":"online","context":{"idset":"20,23"}}
+{"timestamp":1688664818.2645066,"name":"online","context":{"idset":"21,27"}}
+{"timestamp":1688664818.4887064,"name":"online","context":{"idset":"29,31-32"}}
+{"timestamp":1688664818.6602981,"name":"online","context":{"idset":"22,28"}}
+{"timestamp":1688664818.7976594,"name":"online","context":{"idset":"18-19"}}
+{"timestamp":1688664818.9596338,"name":"online","context":{"idset":"17"}}
+{"timestamp":1688664819.232636,"name":"online","context":{"idset":"30"}}
+{"timestamp":1688665846.1987448,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1688665910.1993361,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1688665910.1994951,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1688665910.1995933,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1688665910.29967,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1688665912.1988769,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1688665912.1989894,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1688665912.1990802,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1688665912.1991749,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1688665912.1992574,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1688665912.1993387,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1688665912.1994197,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1688665912.1994998,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1688665912.1995788,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1688665912.1996551,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1688665912.1997297,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1688665912.1998043,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1688665912.1998761,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1688665912.1999466,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1688665912.2000172,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1688665912.2000988,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1688665912.2001696,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1688665912.2002385,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1688665912.2003069,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1688665912.2003722,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1688665912.2004399,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1688665912.2005024,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1688665912.2997966,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1688670141.0095379,"name":"online","context":{"idset":"34"}}
+{"timestamp":1688670141.0667126,"name":"online","context":{"idset":"35"}}
+{"timestamp":1688670141.1802998,"name":"online","context":{"idset":"30,36"}}
+{"timestamp":1688670141.2821019,"name":"online","context":{"idset":"14,33"}}
+{"timestamp":1688670141.4063599,"name":"online","context":{"idset":"12"}}
+{"timestamp":1688670141.5081363,"name":"online","context":{"idset":"4,16,18"}}
+{"timestamp":1688670141.6180561,"name":"online","context":{"idset":"8,10,17,19,21,23,29"}}
+{"timestamp":1688670141.8388829,"name":"online","context":{"idset":"6,22,24,28"}}
+{"timestamp":1688670142.1664474,"name":"online","context":{"idset":"26-27,31"}}
+{"timestamp":1688670142.2991445,"name":"online","context":{"idset":"20"}}
+{"timestamp":1688670146.0061364,"name":"online","context":{"idset":"25"}}
+{"timestamp":1688670147.0567355,"name":"online","context":{"idset":"32"}}
+{"timestamp":1688670808.6534574,"name":"online","context":{"idset":"2"}}
+{"timestamp":1688679194.2991319,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1688684441.6714115,"name":"online","context":{"idset":"5"}}
+{"timestamp":1688684641.5234241,"name":"online","context":{"idset":"7"}}
+{"timestamp":1688684764.7449234,"name":"online","context":{"idset":"23"}}
+{"timestamp":1688685147.0165372,"name":"online","context":{"idset":"9,11,13,15"}}
+{"timestamp":1688685309.7357225,"name":"online","context":{"idset":"3"}}
+{"timestamp":1688685371.1002295,"name":"undrain","context":{"idset":"2-36"}}
+{"timestamp":1688685702.1457357,"name":"online","context":{"idset":"1"}}
+{"timestamp":1688685801.2628367,"name":"undrain","context":{"idset":"1"}}
+{"timestamp":1689032110.3276999,"name":"drain","context":{"idset":"4","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1689039484.0110605,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1689096034.9438431,"name":"drain","context":{"idset":"11","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1689102581.2256565,"name":"drain","context":{"idset":"12","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1689196105.8417664,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1689196112.2339582,"name":"online","context":{"idset":"7"}}
+{"timestamp":1689196124.29951,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1689196130.2843678,"name":"online","context":{"idset":"4"}}
+{"timestamp":1689196201.914103,"name":"drain","context":{"idset":"14","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1689197276.1991103,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1689197276.298995,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1689198395.2306502,"name":"drain","context":{"idset":"8","reason":"Bad Partner Node -JRG","overwrite":0}}
+{"timestamp":1689199022.4479749,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1689199937.3286381,"name":"online","context":{"idset":"14"}}
+{"timestamp":1689199938.045433,"name":"online","context":{"idset":"4"}}
+{"timestamp":1689199975.8796816,"name":"undrain","context":{"idset":"4,11-12,14"}}
+{"timestamp":1689367957.115541,"name":"drain","context":{"idset":"10","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1689613038.6692147,"name":"drain","context":{"idset":"16","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1689613442.8834677,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1689613513.5442095,"name":"undrain","context":{"idset":"16"}}
+{"timestamp":1689613722.300096,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1689614194.0921583,"name":"online","context":{"idset":"10"}}
+{"timestamp":1689614214.5322115,"name":"undrain","context":{"idset":"10"}}
+{"timestamp":1689617139.2870436,"name":"online","context":{"idset":"7-8"}}
+{"timestamp":1689617188.9689901,"name":"undrain","context":{"idset":"7-8"}}
+{"timestamp":1689635623.1902571,"name":"drain","context":{"idset":"3","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1689703778.8647578,"name":"undrain","context":{"idset":"3"}}
+{"timestamp":1690022582.6543586,"name":"drain","context":{"idset":"14","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1690211526.6994858,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1690212463.4472229,"name":"online","context":{"idset":"14"}}
+{"timestamp":1690212474.6213264,"name":"undrain","context":{"idset":"14"}}
+{"timestamp":1690490826.1335647,"name":"drain","context":{"idset":"11","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1690495750.3843153,"name":"undrain","context":{"idset":"11"}}
+{"timestamp":1690496505.8267486,"name":"drain","context":{"idset":"11","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1690506341.1792128,"name":"drain","context":{"idset":"3","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1690566152.3814771,"name":"undrain","context":{"idset":"3,11"}}
+{"timestamp":1690566398.0980511,"name":"drain","context":{"idset":"11","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1690568286.9487793,"name":"drain","context":{"idset":"17","overwrite":0}}
+{"timestamp":1690568428.1994469,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1690568428.2994387,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1690569436.607836,"name":"drain","context":{"idset":"11,17","reason":"problems","overwrite":1}}
+{"timestamp":1690570623.6640644,"name":"online","context":{"idset":"11"}}
+{"timestamp":1690570624.0570786,"name":"online","context":{"idset":"17"}}
+{"timestamp":1690570828.777142,"name":"drain","context":{"idset":"11-12,16-17","reason":"problems","overwrite":1}}
+{"timestamp":1690571136.199415,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1690571136.3001318,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1690573325.5275173,"name":"drain","context":{"idset":"10","reason":"nodediag failed network","overwrite":0}}
+{"timestamp":1690576260.2995269,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1690576342.2997634,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1690576348.2996907,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1690576350.3001435,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1690576464.2274606,"name":"drain","context":{"idset":"9","reason":"epilog failed for jobid fHow2NCBFiX","overwrite":0}}
+{"timestamp":1690576826.3002801,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1690576934.2997983,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1690577850.1987164,"name":"online","context":{"idset":"1"}}
+{"timestamp":1690578212.2993898,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1690579069.0934014,"name":"drain","context":{"idset":"24","reason":"problems","overwrite":0}}
+{"timestamp":1690579192.8717432,"name":"online","context":{"idset":"2"}}
+{"timestamp":1690579249.722357,"name":"undrain","context":{"idset":"1-2,9,28"}}
+{"timestamp":1690579532.3830347,"name":"undrain","context":{"idset":"16-17"}}
+{"timestamp":1690579872.9060335,"name":"online","context":{"idset":"12"}}
+{"timestamp":1690579874.0192597,"name":"online","context":{"idset":"11"}}
+{"timestamp":1690579906.2709489,"name":"undrain","context":{"idset":"11-12"}}
+{"timestamp":1690580224.6935222,"name":"drain","context":{"idset":"10,24","reason":"network","overwrite":1}}
+{"timestamp":1690580243.0597839,"name":"drain","context":{"idset":"11","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1690580378.8876145,"name":"drain","context":{"idset":"9,13,15,23","reason":"mounts","overwrite":0}}
+{"timestamp":1690580384.2991683,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1690580542.2990541,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1690580544.1995158,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1690580544.2997844,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1690580932.5026376,"name":"drain","context":{"idset":"12","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1690581221.9751191,"name":"online","context":{"idset":"10"}}
+{"timestamp":1690581237.8314309,"name":"undrain","context":{"idset":"10"}}
+{"timestamp":1690581332.6279323,"name":"online","context":{"idset":"24"}}
+{"timestamp":1690581353.4348707,"name":"undrain","context":{"idset":"24"}}
+{"timestamp":1690581486.464009,"name":"online","context":{"idset":"23"}}
+{"timestamp":1690581486.9054739,"name":"online","context":{"idset":"9"}}
+{"timestamp":1690581575.166748,"name":"undrain","context":{"idset":"9,23"}}
+{"timestamp":1690581811.5191987,"name":"drain","context":{"idset":"17","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1690581946.8825738,"name":"drain","context":{"idset":"9","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1690581948.0500565,"name":"undrain","context":{"idset":"13"}}
+{"timestamp":1690582033.6935301,"name":"drain","context":{"idset":"24","reason":"nodediag failed network pci","overwrite":0}}
+{"timestamp":1690582100.1153636,"name":"drain","context":{"idset":"10","reason":"nodediag failed network pci","overwrite":0}}
+{"timestamp":1690582161.8840003,"name":"drain","context":{"idset":"15","reason":"kernel panic","overwrite":1}}
+{"timestamp":1690582484.1384256,"name":"drain","context":{"idset":"23","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1690849054.3003924,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1690903832.300009,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1690960686.3002448,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1690992317.0297036,"name":"drain","context":{"idset":"3-36","reason":"reboot","overwrite":1}}
+{"timestamp":1690992504.1993451,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1690992504.1995513,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1690992504.199666,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1690992504.1997762,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1690992504.1998782,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1690992504.1999753,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1690992504.2000692,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1690992504.2001839,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1690992504.2002792,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1690992504.2003684,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1690992504.2004647,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1690992504.2005601,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1690992504.2006497,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1690992504.2007368,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1690992504.2008221,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1690992504.200906,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1690992504.200999,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1690992504.2011001,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1690992504.2012293,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1690992504.2013133,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1690992504.2014012,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1690992504.2014921,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1690992504.2015834,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1690992504.2016742,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1690992504.2017589,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1690992504.2018399,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1690992504.2019172,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1690992504.2019858,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1690992504.202065,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1690992504.2021472,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1690992504.2022169,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1690992504.2022865,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1690992504.4741726,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1690992504.5989015,"name":"drain","context":{"idset":"3","reason":"epilog failed for jobid fJmp3rSQgHV","overwrite":0}}
+{"timestamp":1690992504.703413,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid fJmTLjscSNb","overwrite":0}}
+{"timestamp":1690992504.782114,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid fJmToWCaSXy","overwrite":0}}
+{"timestamp":1690992672.1996136,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1690992672.3004735,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1690992736.1986208,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1690992736.2990053,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1690996345.984442,"name":"drain","context":{"idset":"13,15","reason":"kernel panic","overwrite":1}}
+{"timestamp":1690997042.1635592,"name":"drain","context":{"idset":"12,28","reason":"problems","overwrite":1}}
+{"timestamp":1690997120.1992862,"name":"online","context":{"idset":"34"}}
+{"timestamp":1690997120.6286302,"name":"online","context":{"idset":"33,35"}}
+{"timestamp":1690997121.2039685,"name":"online","context":{"idset":"20,32,36"}}
+{"timestamp":1690997121.6595905,"name":"online","context":{"idset":"9"}}
+{"timestamp":1690997121.8084345,"name":"online","context":{"idset":"7"}}
+{"timestamp":1690997122.4174283,"name":"online","context":{"idset":"14"}}
+{"timestamp":1690997122.5426803,"name":"online","context":{"idset":"10-11"}}
+{"timestamp":1690997122.7820618,"name":"online","context":{"idset":"19"}}
+{"timestamp":1690997122.9053044,"name":"online","context":{"idset":"3-4,27"}}
+{"timestamp":1690997122.9862652,"name":"online","context":{"idset":"8,18"}}
+{"timestamp":1690997123.2814095,"name":"online","context":{"idset":"29"}}
+{"timestamp":1690997124.0982022,"name":"online","context":{"idset":"17,25"}}
+{"timestamp":1690997124.54755,"name":"online","context":{"idset":"31"}}
+{"timestamp":1690997124.6578283,"name":"online","context":{"idset":"21"}}
+{"timestamp":1690997125.1125975,"name":"online","context":{"idset":"24"}}
+{"timestamp":1690997125.3401279,"name":"online","context":{"idset":"5"}}
+{"timestamp":1690997125.8039761,"name":"online","context":{"idset":"6"}}
+{"timestamp":1690997126.6471901,"name":"online","context":{"idset":"16"}}
+{"timestamp":1690997127.8135917,"name":"online","context":{"idset":"23,30"}}
+{"timestamp":1690997128.1114681,"name":"online","context":{"idset":"22"}}
+{"timestamp":1690997129.7832563,"name":"online","context":{"idset":"26"}}
+{"timestamp":1690997214.0611939,"name":"undrain","context":{"idset":"3-11,14,16-27,29-36"}}
+{"timestamp":1690997227.6415937,"name":"drain","context":{"idset":"3","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1690997227.7701278,"name":"drain","context":{"idset":"4","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1690997296.6268826,"name":"online","context":{"idset":"2"}}
+{"timestamp":1690997296.7732763,"name":"online","context":{"idset":"1"}}
+{"timestamp":1690997458.092768,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1690997458.1528273,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1690997458.8855021,"name":"online","context":{"idset":"1-2"}}
+{"timestamp":1690997468.9780183,"name":"undrain","context":{"idset":"1-2"}}
+{"timestamp":1690997547.6303101,"name":"online","context":{"idset":"12,28"}}
+{"timestamp":1690997578.0582342,"name":"undrain","context":{"idset":"12,28"}}
+{"timestamp":1691000897.9105389,"name":"undrain","context":{"idset":"3-4"}}
+{"timestamp":1691004422.7056198,"name":"drain","context":{"idset":"12,28","reason":"slingshot issues","overwrite":0}}
+{"timestamp":1691442630.0027142,"name":"drain","context":{"idset":"22-24","reason":"prolog failed for jobid fKnrKWWhhAf","overwrite":0}}
+{"timestamp":1691448647.3717091,"name":"drain","context":{"idset":"8","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1691612538.2998984,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1691614913.0310326,"name":"drain","context":{"idset":"32","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1691614921.4570794,"name":"drain","context":{"idset":"29","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1691615275.3902726,"name":"drain","context":{"idset":"27","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1691624238.9873762,"name":"undrain","context":{"idset":"8,27,29,31-32"}}
+{"timestamp":1691764622.1993239,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1691764622.1995289,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1691764622.2995217,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1691764640.2992983,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1691768050.1996939,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1691768050.1998925,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1691768050.1999657,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1691768050.300416,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1691768930.1993332,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1691768930.199543,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1691768930.1996176,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1691768930.2992833,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1691788933.6739111,"name":"drain","context":{"idset":"33","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1692122326.299922,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1692122757.0054886,"name":"drain","context":{"idset":"14","reason":"bad partner node--JRG","overwrite":0}}
+{"timestamp":1692123016.2987294,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1692123078.1629097,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1692124727.9253373,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1692124728.025419,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1692124728.1764028,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1692124728.2768204,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1692124728.3880005,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1692124728.6233335,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1692124728.7769127,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1692124728.8796751,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1692124728.9803061,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1692124729.3554881,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1692124729.4556353,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1692126037.8088713,"name":"online","context":{"idset":"8-9,25"}}
+{"timestamp":1692126037.9934978,"name":"online","context":{"idset":"4-5,17,19"}}
+{"timestamp":1692126038.1051362,"name":"online","context":{"idset":"18"}}
+{"timestamp":1692126038.2778707,"name":"online","context":{"idset":"21"}}
+{"timestamp":1692126039.0095937,"name":"online","context":{"idset":"20"}}
+{"timestamp":1692126039.6706393,"name":"online","context":{"idset":"10"}}
+{"timestamp":1692126083.3534234,"name":"undrain","context":{"idset":"4-5,8-10,17-21,25"}}
+{"timestamp":1692126674.7791092,"name":"online","context":{"idset":"15"}}
+{"timestamp":1692126674.9873233,"name":"online","context":{"idset":"13"}}
+{"timestamp":1692126958.8009717,"name":"online","context":{"idset":"16"}}
+{"timestamp":1692126963.5234253,"name":"online","context":{"idset":"14"}}
+{"timestamp":1692126975.5350831,"name":"undrain","context":{"idset":"13-16"}}
+{"timestamp":1692127306.7844043,"name":"online","context":{"idset":"33"}}
+{"timestamp":1692127318.6979246,"name":"undrain","context":{"idset":"33"}}
+{"timestamp":1692127660.7011418,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1692127660.8594623,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1692199531.4708548,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1692199531.5716386,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1692206742.4304099,"name":"online","context":{"idset":"23"}}
+{"timestamp":1692206742.6160614,"name":"online","context":{"idset":"24"}}
+{"timestamp":1692206772.8497257,"name":"undrain","context":{"idset":"23-24"}}
+{"timestamp":1692207208.9126787,"name":"undrain","context":{"idset":"22"}}
+{"timestamp":1692226126.2994025,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1692304039.7136846,"name":"undrain","context":{"idset":"19"}}
+{"timestamp":1692313146.9445035,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1692313147.2367971,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid fMjUAMbbngf","overwrite":0}}
+{"timestamp":1692382720.2868474,"name":"drain","context":{"idset":"17","reason":"epilog failed for jobid fMt4CjcDCNb","overwrite":0}}
+{"timestamp":1692396716.9669497,"name":"online","context":{"idset":"6"}}
+{"timestamp":1692396738.6329165,"name":"undrain","context":{"idset":"6,17"}}
+{"timestamp":1692398144.4991741,"name":"drain","context":{"idset":"13","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1692482422.5127513,"name":"drain","context":{"idset":"3","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1692490653.2078135,"name":"drain","context":{"idset":"30","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1692560929.7375612,"name":"undrain","context":{"idset":"3,13,30"}}
+{"timestamp":1692561123.4332354,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1692561123.5736334,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1692561123.6739783,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1692561142.0953584,"name":"drain","context":{"idset":"3,13,30","reason":"reboot","overwrite":2}}
+{"timestamp":1692578303.7812779,"name":"online","context":{"idset":"3,13"}}
+{"timestamp":1692578304.0520732,"name":"online","context":{"idset":"30"}}
+{"timestamp":1692578320.7994485,"name":"undrain","context":{"idset":"3,13,30"}}
+{"timestamp":1692731196.3000994,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1692731393.1728163,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid fNhQKdVuXiX","overwrite":0}}
+{"timestamp":1692779548.2996812,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1692779575.4191892,"name":"drain","context":{"idset":"31","reason":"epilog failed for jobid fNoq5483591","overwrite":0}}
+{"timestamp":1692786104.2991579,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1692786112.2993193,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1692786126.5591235,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid fNpiaHq2VEw","overwrite":0}}
+{"timestamp":1692786132.5700204,"name":"drain","context":{"idset":"22","reason":"epilog failed for jobid fNpVJC9Hz4o","overwrite":0}}
+{"timestamp":1692786548.2992902,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1692786579.7509997,"name":"drain","context":{"idset":"18","reason":"epilog failed for jobid fNpZhLxexhV","overwrite":0}}
+{"timestamp":1692802818.0876265,"name":"drain","context":{"idset":"8","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1692806199.7085576,"name":"drain","context":{"idset":"19","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692806200.7858949,"name":"drain","context":{"idset":"20","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692807122.8234079,"name":"drain","context":{"idset":"33","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692807258.9694161,"name":"drain","context":{"idset":"9","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692807259.1322889,"name":"drain","context":{"idset":"11","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692807259.1423004,"name":"drain","context":{"idset":"13","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692807260.5390677,"name":"drain","context":{"idset":"14","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692807405.4076908,"name":"drain","context":{"idset":"21","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692807405.707001,"name":"drain","context":{"idset":"16","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692807405.904201,"name":"drain","context":{"idset":"23","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692807406.347533,"name":"drain","context":{"idset":"24","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692807514.2667327,"name":"drain","context":{"idset":"26","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1692810253.0905623,"name":"resource-init","context":{"restart":true,"drain":{"4":{"timestamp":1692786126.5591235,"reason":"epilog failed for jobid fNpiaHq2VEw"},"8":{"timestamp":1692802818.0876265,"reason":"nodediag failed amdgpu"},"9":{"timestamp":1692807258.9694161,"reason":"nodediag failed pci"},"11":{"timestamp":1692807259.1322889,"reason":"nodediag failed pci"},"13":{"timestamp":1692807259.1423004,"reason":"nodediag failed pci"},"14":{"timestamp":1692807260.5390677,"reason":"nodediag failed pci"},"15":{"timestamp":1692731393.1728163,"reason":"epilog failed for jobid fNhQKdVuXiX"},"16":{"timestamp":1692807405.707001,"reason":"nodediag failed pci"},"18":{"timestamp":1692786579.7509997,"reason":"epilog failed for jobid fNpZhLxexhV"},"19":{"timestamp":1692806199.7085576,"reason":"nodediag failed pci"},"20":{"timestamp":1692806200.7858949,"reason":"nodediag failed pci"},"21":{"timestamp":1692807405.4076908,"reason":"nodediag failed pci"},"22":{"timestamp":1692786132.5700204,"reason":"epilog failed for jobid fNpVJC9Hz4o"},"23":{"timestamp":1692807405.904201,"reason":"nodediag failed pci"},"24":{"timestamp":1692807406.347533,"reason":"nodediag failed pci"},"26":{"timestamp":1692807514.2667327,"reason":"nodediag failed pci"},"12,28":{"timestamp":1691004422.7056198,"reason":"slingshot issues"},"31":{"timestamp":1692779575.4191892,"reason":"epilog failed for jobid fNoq5483591"},"33":{"timestamp":1692807122.8234079,"reason":"nodediag failed pci"}},"online":"","exclude":"0-2"}}
+{"timestamp":1692810253.0927088,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1692810726.6551633,"name":"online","context":{"idset":"0"}}
+{"timestamp":1692818620.4226494,"name":"online","context":{"idset":"35"}}
+{"timestamp":1692818621.1485355,"name":"online","context":{"idset":"36"}}
+{"timestamp":1692818621.4851315,"name":"online","context":{"idset":"33-34"}}
+{"timestamp":1692818621.7543807,"name":"online","context":{"idset":"3,10,16,23"}}
+{"timestamp":1692818621.9002204,"name":"online","context":{"idset":"4-5,15,27"}}
+{"timestamp":1692818622.0238256,"name":"online","context":{"idset":"9,13,32"}}
+{"timestamp":1692818622.1857233,"name":"online","context":{"idset":"8,18,20"}}
+{"timestamp":1692818622.3141024,"name":"online","context":{"idset":"19,26"}}
+{"timestamp":1692818622.4271135,"name":"online","context":{"idset":"7,11,14,24-25"}}
+{"timestamp":1692818622.5699735,"name":"online","context":{"idset":"17,21-22"}}
+{"timestamp":1692818622.7709751,"name":"online","context":{"idset":"29-31"}}
+{"timestamp":1692818623.0506318,"name":"online","context":{"idset":"6"}}
+{"timestamp":1692818731.4312236,"name":"undrain","context":{"idset":"8"}}
+{"timestamp":1692818755.4927576,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1692818765.7528849,"name":"undrain","context":{"idset":"9,11"}}
+{"timestamp":1692818784.1905811,"name":"undrain","context":{"idset":"13-14"}}
+{"timestamp":1692818793.4161901,"name":"undrain","context":{"idset":"15"}}
+{"timestamp":1692818807.8567481,"name":"undrain","context":{"idset":"16,21,23-24"}}
+{"timestamp":1692818835.5674458,"name":"undrain","context":{"idset":"18"}}
+{"timestamp":1692818842.9867311,"name":"undrain","context":{"idset":"19-20"}}
+{"timestamp":1692818847.184705,"name":"undrain","context":{"idset":"22"}}
+{"timestamp":1692818852.7119665,"name":"undrain","context":{"idset":"26"}}
+{"timestamp":1692818858.4308116,"name":"undrain","context":{"idset":"31"}}
+{"timestamp":1692818862.0798388,"name":"undrain","context":{"idset":"33"}}
+{"timestamp":1692819097.2627742,"name":"online","context":{"idset":"2"}}
+{"timestamp":1692819100.0029969,"name":"online","context":{"idset":"1"}}
+{"timestamp":1692839662.078393,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1692839662.0804343,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1692839662.0821981,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1692839662.0830526,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1692839662.0831323,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1692839662.0832038,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1692839662.0832705,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1692839662.0833356,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1692839662.0833981,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1692839662.0834618,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1692839662.0835207,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1692839662.08358,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1692839662.0836363,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1692839662.0836906,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1692839662.083756,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1692839662.0838127,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1692839662.0838647,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1692839662.0839164,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1692839662.0839696,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1692839662.084022,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1692839662.0840764,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1692839662.0841267,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1692839662.0841763,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1692839662.0842271,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1692839662.0842719,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1692839662.0843182,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1692839662.0843611,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1692839662.0844007,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1692839662.084439,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1692839662.0844786,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1692839662.0845215,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1692839662.0845597,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1692839662.0845954,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1692839662.1837966,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1692839672.7889924,"name":"resource-init","context":{"restart":true,"drain":{"12,28":{"timestamp":1691004422.7056198,"reason":"slingshot issues"}},"online":"","exclude":"0-2"}}
+{"timestamp":1692839672.7910104,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1692839773.6606281,"name":"online","context":{"idset":"0"}}
+{"timestamp":1692839774.2982254,"name":"online","context":{"idset":"1"}}
+{"timestamp":1692839774.5696738,"name":"online","context":{"idset":"33-36"}}
+{"timestamp":1692839774.7861044,"name":"online","context":{"idset":"2-11,13-16,25-27,29-32"}}
+{"timestamp":1692839774.9076021,"name":"online","context":{"idset":"17-24"}}
+{"timestamp":1692840488.7268288,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1692840488.7270803,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1692840488.7286444,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1692840488.7287424,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1692840488.7300775,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1692840488.7301459,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1692840488.7302103,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1692840488.7302716,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1692840488.7303338,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1692840488.7303958,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1692840488.7304547,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1692840488.7305162,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1692840488.7305748,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1692840488.7306318,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1692840488.730689,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1692840488.7307563,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1692840488.7308125,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1692840488.7308667,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1692840488.7309198,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1692840488.7309768,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1692840488.7310297,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1692840488.7310805,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1692840488.7311316,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1692840488.7311802,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1692840488.7312312,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1692840488.7312748,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1692840488.7313178,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1692840488.7313595,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1692840488.7313998,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1692840488.7314396,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1692840488.7314785,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1692840488.7315185,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1692840488.7315571,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1692840488.8301249,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1692840604.9479165,"name":"resource-init","context":{"restart":true,"drain":{"12,28":{"timestamp":1691004422.7056198,"reason":"slingshot issues"}},"online":"","exclude":"0-2"}}
+{"timestamp":1692840604.9502103,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1692840706.9745557,"name":"online","context":{"idset":"0"}}
+{"timestamp":1692840707.6075892,"name":"online","context":{"idset":"1"}}
+{"timestamp":1692840707.8797638,"name":"online","context":{"idset":"33-36"}}
+{"timestamp":1692840708.1067026,"name":"online","context":{"idset":"2-11,13-16,25-27,29-32"}}
+{"timestamp":1692840708.2231889,"name":"online","context":{"idset":"17-24"}}
+{"timestamp":1692888332.4160767,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1692888332.4714563,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1692888332.4738617,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1692888332.4811044,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1692888332.4819741,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1692888332.4958029,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1692888332.4972484,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1692888332.4982464,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1692888332.5012705,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1692888332.5019438,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1692888332.5127399,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1692888332.5136507,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1692888332.5140741,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1692888332.5165415,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1692888332.5208981,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1692888332.5316179,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1692888332.5330238,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1692888332.533669,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1692888332.5341415,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1692888332.5342319,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1692888332.5401263,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1692888332.5404348,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1692888332.5406432,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1692888332.5464191,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1692888332.5471454,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1692888332.5475042,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1692888332.5648067,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1692888332.6206682,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1692888332.6771173,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1692888332.68908,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1692888332.6954186,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1692888332.7086511,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1692888332.7097418,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1692888332.8105443,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1692888354.0190606,"name":"resource-init","context":{"restart":true,"drain":{"12,28":{"timestamp":1691004422.7056198,"reason":"slingshot issues"}},"online":"","exclude":"0-2"}}
+{"timestamp":1692888354.0216832,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1692888408.6009319,"name":"resource-init","context":{"restart":true,"drain":{"12,28":{"timestamp":1691004422.7056198,"reason":"slingshot issues"}},"online":"","exclude":"0-2"}}
+{"timestamp":1692888408.6032491,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1692888507.7297277,"name":"online","context":{"idset":"0"}}
+{"timestamp":1692888508.3635125,"name":"online","context":{"idset":"1"}}
+{"timestamp":1692888508.5616102,"name":"online","context":{"idset":"2"}}
+{"timestamp":1692888509.0067315,"name":"online","context":{"idset":"33,35-36"}}
+{"timestamp":1692888509.1411369,"name":"online","context":{"idset":"6,34"}}
+{"timestamp":1692888509.2970431,"name":"online","context":{"idset":"9,12,14"}}
+{"timestamp":1692888509.4015825,"name":"online","context":{"idset":"4-5,7,18,24"}}
+{"timestamp":1692888509.5268667,"name":"online","context":{"idset":"16-17,32"}}
+{"timestamp":1692888509.6310351,"name":"online","context":{"idset":"3,8,10,13,15,19,21-22,26,29"}}
+{"timestamp":1692888509.7378364,"name":"online","context":{"idset":"11,25,28,30"}}
+{"timestamp":1692888509.9337234,"name":"online","context":{"idset":"20,27,31"}}
+{"timestamp":1692888510.141408,"name":"online","context":{"idset":"23"}}
+{"timestamp":1692928872.6934316,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1692928902.9457812,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid fP6EugNSjTV","overwrite":0}}
+{"timestamp":1692942089.6350839,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1692942108.3691027,"name":"drain","context":{"idset":"21","reason":"epilog failed for jobid fP7wiriRRLb","overwrite":0}}
+{"timestamp":1692949182.6933129,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1692949210.9409411,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid fP9fFuKDnMd","overwrite":0}}
+{"timestamp":1692965158.6932912,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1692965185.0472045,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid fPBPoHVV2nB","overwrite":0}}
+{"timestamp":1692965640.0887954,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1692965672.9308987,"name":"drain","context":{"idset":"31","reason":"epilog failed for jobid fPBPtWsp4Yw","overwrite":0}}
+{"timestamp":1692970496.6928623,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1692970532.4701481,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid fPBusRAn2N3","overwrite":0}}
+{"timestamp":1692973814.6940987,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1692973974.6932034,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1692974101.5216386,"name":"drain","context":{"idset":"25","reason":"epilog failed for jobid fPCDVJTg51R","overwrite":0}}
+{"timestamp":1692974258.478868,"name":"drain","context":{"idset":"30","reason":"epilog failed for jobid fPCJtSH3uC3","overwrite":0}}
+{"timestamp":1692978974.356133,"name":"online","context":{"idset":"14,31"}}
+{"timestamp":1692978974.4817019,"name":"online","context":{"idset":"8,15,21"}}
+{"timestamp":1692978974.587189,"name":"online","context":{"idset":"5,30"}}
+{"timestamp":1692978974.7004957,"name":"online","context":{"idset":"25"}}
+{"timestamp":1692979025.3250048,"name":"undrain","context":{"idset":"5,8,14-15,21,25,30-31"}}
+{"timestamp":1692981308.692697,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1692981344.4308941,"name":"drain","context":{"idset":"29","reason":"epilog failed for jobid fPDQGhUFiTy","overwrite":0}}
+{"timestamp":1692981462.6925905,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1692981737.3271539,"name":"drain","context":{"idset":"3","reason":"epilog failed for jobid fPCyEvWGLyV","overwrite":0}}
+{"timestamp":1692981958.6921132,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1692981991.0464966,"name":"drain","context":{"idset":"13","reason":"epilog failed for jobid fPDQBKHvwkb","overwrite":0}}
+{"timestamp":1692983594.6940067,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1692983631.5715263,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid fPEMH7DJZiF","overwrite":0}}
+{"timestamp":1692985834.6936679,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1692985870.5246086,"name":"drain","context":{"idset":"16","reason":"epilog failed for jobid fPEeMswUoKV","overwrite":0}}
+{"timestamp":1692987090.7268119,"name":"undrain","context":{"idset":"12"}}
+{"timestamp":1692987654.6772084,"name":"undrain","context":{"idset":"28"}}
+{"timestamp":1692988898.6930771,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1692988931.6343505,"name":"drain","context":{"idset":"12","reason":"epilog failed for jobid fPF6H7w68Gw","overwrite":0}}
+{"timestamp":1692989299.2486513,"name":"online","context":{"idset":"29"}}
+{"timestamp":1692989299.4475136,"name":"online","context":{"idset":"16"}}
+{"timestamp":1692989299.6108832,"name":"online","context":{"idset":"3,8"}}
+{"timestamp":1692989299.8308446,"name":"online","context":{"idset":"13"}}
+{"timestamp":1692989329.9504247,"name":"undrain","context":{"idset":"3,8,13,16,29"}}
+{"timestamp":1692990419.3606191,"name":"online","context":{"idset":"12"}}
+{"timestamp":1692990448.1998105,"name":"undrain","context":{"idset":"12"}}
+{"timestamp":1692990852.6927257,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1692990885.0082393,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid fPFMhTCN1pb","overwrite":0}}
+{"timestamp":1692991200.1587262,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1692991232.0767906,"name":"drain","context":{"idset":"25","reason":"epilog failed for jobid fPFMc4tdJhV","overwrite":0}}
+{"timestamp":1692992050.6934326,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1692992085.6083233,"name":"drain","context":{"idset":"21","reason":"epilog failed for jobid fPFSaPsY9wm","overwrite":0}}
+{"timestamp":1692993492.6942213,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1692993524.3757982,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid fPFf3syxMdq","overwrite":0}}
+{"timestamp":1692993979.3875608,"name":"online","context":{"idset":"21,25"}}
+{"timestamp":1692993979.7023747,"name":"online","context":{"idset":"8"}}
+{"timestamp":1692993991.5430374,"name":"undrain","context":{"idset":"8,21,25"}}
+{"timestamp":1692994196.6930029,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1692994227.504024,"name":"drain","context":{"idset":"13","reason":"epilog failed for jobid fPFii52NRqh","overwrite":0}}
+{"timestamp":1692994421.8648996,"name":"online","context":{"idset":"4"}}
+{"timestamp":1692994436.1136961,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1692995057.1525946,"name":"online","context":{"idset":"13"}}
+{"timestamp":1692995072.4509242,"name":"undrain","context":{"idset":"13"}}
+{"timestamp":1692995346.6930807,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1692995380.8087437,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid fPG2vrdRqDh","overwrite":0}}
+{"timestamp":1692996210.3570321,"name":"online","context":{"idset":"14"}}
+{"timestamp":1692996217.7698519,"name":"undrain","context":{"idset":"14"}}
+{"timestamp":1692996480.3820291,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1692996517.5173485,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid fPGBZj8GqQ3","overwrite":0}}
+{"timestamp":1692997094.1965477,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1692997095.48786,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1692997131.3929069,"name":"drain","context":{"idset":"28,30","reason":"epilog failed for jobid fPG1J8YfLpo","overwrite":0}}
+{"timestamp":1692997913.4696326,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1692997946.7347782,"name":"drain","context":{"idset":"27","reason":"epilog failed for jobid fPDKKZ7ycsZ","overwrite":0}}
+{"timestamp":1692997994.126179,"name":"online","context":{"idset":"30"}}
+{"timestamp":1692997994.2624452,"name":"online","context":{"idset":"28"}}
+{"timestamp":1692998011.1306953,"name":"undrain","context":{"idset":"28,30"}}
+{"timestamp":1692998092.6264634,"name":"online","context":{"idset":"4"}}
+{"timestamp":1692998118.7271602,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1692998564.6937833,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1692998598.5913801,"name":"drain","context":{"idset":"19","reason":"epilog failed for jobid fPGTScDYnJT","overwrite":0}}
+{"timestamp":1692999668.6931415,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1692999703.5506082,"name":"drain","context":{"idset":"11","reason":"epilog failed for jobid fPDLuP63YpX","overwrite":0}}
+{"timestamp":1692999836.6939063,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1692999870.6680562,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid fPGbLcpS2sH","overwrite":0}}
+{"timestamp":1693000882.0961971,"name":"online","context":{"idset":"27"}}
+{"timestamp":1693000913.5161023,"name":"undrain","context":{"idset":"27"}}
+{"timestamp":1693001001.437541,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693001001.6205437,"name":"online","context":{"idset":"5"}}
+{"timestamp":1693001002.4923496,"name":"online","context":{"idset":"11"}}
+{"timestamp":1693001004.6923308,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693001011.63676,"name":"undrain","context":{"idset":"5,11,19"}}
+{"timestamp":1693001040.1972437,"name":"drain","context":{"idset":"16","reason":"epilog failed for jobid fPGmxsbq5Sf","overwrite":0}}
+{"timestamp":1693001885.2384911,"name":"online","context":{"idset":"16"}}
+{"timestamp":1693001898.3726585,"name":"undrain","context":{"idset":"16"}}
+{"timestamp":1693011082.6934776,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693011203.227948,"name":"drain","context":{"idset":"19","reason":"epilog failed for jobid fPJ6YZ84qvP","overwrite":0}}
+{"timestamp":1693012146.5565341,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693012180.82723,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid fPJEcxJUBpB","overwrite":0}}
+{"timestamp":1693012372.6932833,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693012405.8921275,"name":"drain","context":{"idset":"21","reason":"epilog failed for jobid fPJ6duznFGB","overwrite":0}}
+{"timestamp":1693013664.6931405,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693013699.2285278,"name":"drain","context":{"idset":"26","reason":"epilog failed for jobid fPJQMRYtEtw","overwrite":0}}
+{"timestamp":1693014798.6929038,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693014830.4584987,"name":"drain","context":{"idset":"28","reason":"epilog failed for jobid fPJQVfbSfFu","overwrite":0}}
+{"timestamp":1693014938.6934071,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693014972.7858858,"name":"drain","context":{"idset":"24","reason":"epilog failed for jobid fPJa6hJ3maw","overwrite":0}}
+{"timestamp":1693015956.6934924,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693015995.3466806,"name":"drain","context":{"idset":"23","reason":"epilog failed for jobid fPJjZ7Npuhy","overwrite":0}}
+{"timestamp":1693017008.693646,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1693017042.5336194,"name":"drain","context":{"idset":"11","reason":"epilog failed for jobid fPJsXFLGD7m","overwrite":0}}
+{"timestamp":1693018172.6930799,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693018210.0101104,"name":"drain","context":{"idset":"27","reason":"epilog failed for jobid fPK2FQ12tnT","overwrite":0}}
+{"timestamp":1693019222.6939104,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693019257.9437957,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid fPKALxnmV4K","overwrite":0}}
+{"timestamp":1693069117.4770873,"name":"online","context":{"idset":"19"}}
+{"timestamp":1693069117.6357889,"name":"online","context":{"idset":"6,27"}}
+{"timestamp":1693069117.911077,"name":"online","context":{"idset":"5,23-24"}}
+{"timestamp":1693069118.0529895,"name":"online","context":{"idset":"11"}}
+{"timestamp":1693069118.2053845,"name":"online","context":{"idset":"26"}}
+{"timestamp":1693069118.3075943,"name":"online","context":{"idset":"21,28"}}
+{"timestamp":1693069850.6702559,"name":"undrain","context":{"idset":"5-6,11,19,21,23-24,26-28"}}
+{"timestamp":1693073246.6926479,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693073295.4430041,"name":"drain","context":{"idset":"28","reason":"epilog failed for jobid fPRvfvvBbFD","overwrite":0}}
+{"timestamp":1693075352.7019544,"name":"online","context":{"idset":"28"}}
+{"timestamp":1693075394.425863,"name":"undrain","context":{"idset":"28"}}
+{"timestamp":1693077146.6937301,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693077184.1476595,"name":"drain","context":{"idset":"29","reason":"epilog failed for jobid fPSe8sWFo19","overwrite":0}}
+{"timestamp":1693082133.8461447,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693082150.7168832,"name":"undrain","context":{"idset":"29"}}
+{"timestamp":1693083420.6940818,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693083457.7626688,"name":"drain","context":{"idset":"29","reason":"epilog failed for jobid fPTWyvHkBw5","overwrite":0}}
+{"timestamp":1693084351.6119845,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693084353.4917769,"name":"undrain","context":{"idset":"29"}}
+{"timestamp":1693087378.6933606,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693087415.4070444,"name":"drain","context":{"idset":"29","reason":"epilog failed for jobid fPTzE3yy6PH","overwrite":0}}
+{"timestamp":1693088231.0766566,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693088242.2064247,"name":"undrain","context":{"idset":"29"}}
+{"timestamp":1693089546.6931736,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693089582.2626336,"name":"drain","context":{"idset":"29","reason":"epilog failed for jobid fPUKksg27Gj","overwrite":0}}
+{"timestamp":1693090767.4459538,"name":"online","context":{"idset":"29"}}
+{"timestamp":1693090777.2486236,"name":"undrain","context":{"idset":"29"}}
+{"timestamp":1693091994.6924961,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1693092027.8113558,"name":"drain","context":{"idset":"29","reason":"epilog failed for jobid fPUeZu3ZYRM","overwrite":0}}
+{"timestamp":1693107038.6936531,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693107072.423353,"name":"drain","context":{"idset":"30","reason":"epilog failed for jobid fPWceGTrxbh","overwrite":0}}
+{"timestamp":1693109110.6942401,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693109147.1780446,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid fPWtJL2s6Hm","overwrite":0}}
+{"timestamp":1693110946.6943264,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693110984.2162261,"name":"drain","context":{"idset":"25","reason":"epilog failed for jobid fPX8ChM6XeX","overwrite":0}}
+{"timestamp":1693115544.8214228,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693115575.996702,"name":"drain","context":{"idset":"24","reason":"epilog failed for jobid fPXjBmADjH9","overwrite":0}}
+{"timestamp":1693119346.5935655,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693119346.6940176,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693119397.1507294,"name":"drain","context":{"idset":"19,21","reason":"epilog failed for jobid fPXvuPZABuh","overwrite":0}}
+{"timestamp":1693140302.6939485,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693140340.5296221,"name":"drain","context":{"idset":"23","reason":"epilog failed for jobid fPaYLtC5m75","overwrite":0}}
+{"timestamp":1693145386.6932325,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693145429.3317924,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid fPbMj2tQoaf","overwrite":0}}
+{"timestamp":1693152247.1315928,"name":"online","context":{"idset":"14"}}
+{"timestamp":1693152247.3962998,"name":"online","context":{"idset":"8,19,23-25"}}
+{"timestamp":1693152247.6464467,"name":"online","context":{"idset":"30"}}
+{"timestamp":1693152247.7710569,"name":"online","context":{"idset":"21,29"}}
+{"timestamp":1693152956.7186906,"name":"undrain","context":{"idset":"8,14,19,21,23-25,29-30"}}
+{"timestamp":1693155834.6931961,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693155870.9070299,"name":"drain","context":{"idset":"25","reason":"epilog failed for jobid fPcoT5SGCj9","overwrite":0}}
+{"timestamp":1693158884.8680108,"name":"online","context":{"idset":"25"}}
+{"timestamp":1693159279.9315217,"name":"undrain","context":{"idset":"25"}}
+{"timestamp":1693168522.6928248,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693168562.3432541,"name":"drain","context":{"idset":"32","reason":"epilog failed for jobid fPeKsiY6Tiw","overwrite":0}}
+{"timestamp":1693170617.2663209,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693170702.0847383,"name":"undrain","context":{"idset":"32"}}
+{"timestamp":1693173820.6928408,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693173857.950772,"name":"drain","context":{"idset":"32","reason":"epilog failed for jobid fPf7q8YJ9Nf","overwrite":0}}
+{"timestamp":1693175879.1142633,"name":"online","context":{"idset":"32"}}
+{"timestamp":1693176892.9215524,"name":"undrain","context":{"idset":"32"}}
+{"timestamp":1693182952.6938405,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693182990.0572627,"name":"drain","context":{"idset":"32","reason":"epilog failed for jobid fPgA6EjPtbq","overwrite":0}}
+{"timestamp":1693204968.6938605,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693205011.3026438,"name":"drain","context":{"idset":"7","reason":"epilog failed for jobid fPiiAxgTwbM","overwrite":0}}
+{"timestamp":1693214222.6935496,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693214263.0673358,"name":"drain","context":{"idset":"16","reason":"epilog failed for jobid fPk1HHVszeB","overwrite":0}}
+{"timestamp":1693224570.6936901,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693224615.163466,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid fPmsskgZVAw","overwrite":0}}
+{"timestamp":1693225714.6927469,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693225753.1680527,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid fPnFgTWm1dh","overwrite":0}}
+{"timestamp":1693226492.6940167,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693226532.3100181,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid fPnMcVVpfBD","overwrite":0}}
+{"timestamp":1693235276.3274584,"name":"online","context":{"idset":"6"}}
+{"timestamp":1693235276.4584951,"name":"online","context":{"idset":"4,7,32"}}
+{"timestamp":1693235276.5595891,"name":"online","context":{"idset":"5,16"}}
+{"timestamp":1693235464.0781851,"name":"undrain","context":{"idset":"4-7,16,32"}}
+{"timestamp":1693236784.6937306,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693236824.494689,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid fPohqooqPSX","overwrite":0}}
+{"timestamp":1693237662.3132401,"name":"online","context":{"idset":"4"}}
+{"timestamp":1693237683.4895632,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1693237910.6939559,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693237950.0238178,"name":"drain","context":{"idset":"7","reason":"epilog failed for jobid fPorSnKHg15","overwrite":0}}
+{"timestamp":1693239120.7819343,"name":"online","context":{"idset":"7"}}
+{"timestamp":1693239477.6610184,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1693336466.8601806,"name":"resource-init","context":{"restart":true,"drain":{},"online":"","exclude":"0-2"}}
+{"timestamp":1693336466.8630674,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1693336554.1257296,"name":"drain","context":{"idset":"29","reason":"gpu problem","overwrite":1}}
+{"timestamp":1693336614.5787406,"name":"online","context":{"idset":"0"}}
+{"timestamp":1693336615.7312009,"name":"online","context":{"idset":"33-36"}}
+{"timestamp":1693336615.9318366,"name":"online","context":{"idset":"3-4,6-7,28,32"}}
+{"timestamp":1693336616.0435956,"name":"online","context":{"idset":"5,10,14,18,25,30-31"}}
+{"timestamp":1693336616.1483233,"name":"online","context":{"idset":"8,11-13,15-17,21-24,26-27"}}
+{"timestamp":1693336616.2690451,"name":"online","context":{"idset":"9,19-20"}}
+{"timestamp":1693336736.8194189,"name":"online","context":{"idset":"1"}}
+{"timestamp":1693336864.1696694,"name":"online","context":{"idset":"2"}}
+{"timestamp":1693344476.2600584,"name":"drain","context":{"idset":"35-36","reason":"test","overwrite":2}}
+{"timestamp":1693344500.4448104,"name":"drain","context":{"idset":"35-36","reason":"testing","overwrite":2}}
+{"timestamp":1693344508.8272488,"name":"undrain","context":{"idset":"35-36"}}
+{"timestamp":1693347384.1156151,"name":"drain","context":{"idset":"35-36","reason":"testing","overwrite":2}}
+{"timestamp":1693347398.5303981,"name":"undrain","context":{"idset":"35-36"}}
+{"timestamp":1693442649.4925799,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1693473379.4922233,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693473414.3371141,"name":"drain","context":{"idset":"10","reason":"epilog failed for jobid fQJuk2QLfuR","overwrite":0}}
+{"timestamp":1693474669.4933288,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693474704.2976325,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid fQJunbkrKWF","overwrite":0}}
+{"timestamp":1693474901.4930291,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693474945.992835,"name":"drain","context":{"idset":"28","reason":"epilog failed for jobid fQK4C69BJAb","overwrite":0}}
+{"timestamp":1693475665.4928,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693475725.4134254,"name":"drain","context":{"idset":"31","reason":"epilog failed for jobid fQK4DDumSFq","overwrite":0}}
+{"timestamp":1693476587.4923387,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693476618.3815503,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid fQJumK1tXVy","overwrite":0}}
+{"timestamp":1693480531.4921169,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693480560.8330817,"name":"drain","context":{"idset":"9","reason":"epilog failed for jobid fQKnyWf6bLb","overwrite":0}}
+{"timestamp":1693481057.4929502,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693481104.2750595,"name":"drain","context":{"idset":"30","reason":"epilog failed for jobid fQL4jdEQgoq","overwrite":0}}
+{"timestamp":1693487477.4925506,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1693487513.1185639,"name":"drain","context":{"idset":"11","reason":"epilog failed for jobid fQKxq3XJDPD","overwrite":0}}
+{"timestamp":1693488269.4931738,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693488309.3189213,"name":"drain","context":{"idset":"19","reason":"epilog failed for jobid fQKyWYPSSoh","overwrite":0}}
+{"timestamp":1693489709.4928045,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693489742.4288378,"name":"drain","context":{"idset":"21","reason":"epilog failed for jobid fQLjBTG546b","overwrite":0}}
+{"timestamp":1693490575.4925683,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693490610.927757,"name":"drain","context":{"idset":"22","reason":"epilog failed for jobid fQMKybEXJkX","overwrite":0}}
+{"timestamp":1693494013.4925382,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693494306.5553958,"name":"drain","context":{"idset":"12","reason":"epilog failed for jobid fQMEm4KFbj5","overwrite":0}}
+{"timestamp":1693497137.1210482,"name":"online","context":{"idset":"30-31"}}
+{"timestamp":1693497137.2961125,"name":"online","context":{"idset":"10,15"}}
+{"timestamp":1693497137.4275651,"name":"online","context":{"idset":"9,14,19,22,28"}}
+{"timestamp":1693497137.5747464,"name":"online","context":{"idset":"11-12,21"}}
+{"timestamp":1693497214.3862979,"name":"undrain","context":{"idset":"9-12,14-15,19,21-22,28,30-31"}}
+{"timestamp":1693499594.0660608,"name":"undrain","context":{"idset":"20"}}
+{"timestamp":1693500019.4932446,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693500041.3811715,"name":"drain","context":{"idset":"3","reason":"epilog failed for jobid fQNY3k5JY4o","overwrite":0}}
+{"timestamp":1693500853.492872,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693500886.3848197,"name":"drain","context":{"idset":"9","reason":"epilog failed for jobid fQNSpiCJrkf","overwrite":0}}
+{"timestamp":1693503129.4933991,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693503425.1009226,"name":"drain","context":{"idset":"32","reason":"epilog failed for jobid fQMg5hUm6hm","overwrite":0}}
+{"timestamp":1693526239.4931784,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693526284.5399024,"name":"drain","context":{"idset":"25","reason":"epilog failed for jobid fQRsqefC79Z","overwrite":0}}
+{"timestamp":1693532195.492908,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693532231.5887942,"name":"drain","context":{"idset":"21","reason":"epilog failed for jobid fQRsD9B9tE3","overwrite":0}}
+{"timestamp":1693537015.4933822,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693537060.625174,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid fQTF2giMGt7","overwrite":0}}
+{"timestamp":1693542567.4929557,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693542600.6197138,"name":"drain","context":{"idset":"24","reason":"epilog failed for jobid fQSyLseAK8j","overwrite":0}}
+{"timestamp":1693543801.0668077,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693543837.3347449,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid fQTzwHTsJYf","overwrite":0}}
+{"timestamp":1693545441.4931166,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693545478.529911,"name":"drain","context":{"idset":"23","reason":"epilog failed for jobid fQTVdiZ2Ac7","overwrite":0}}
+{"timestamp":1693555567.4938772,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693555612.3994486,"name":"drain","context":{"idset":"18","reason":"epilog failed for jobid fQVeWzA7Y8o","overwrite":0}}
+{"timestamp":1693557577.4927566,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693557611.183881,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid fQVZJFH1dDy","overwrite":0}}
+{"timestamp":1693567129.4931338,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693567177.8554506,"name":"drain","context":{"idset":"20","reason":"epilog failed for jobid fQWY1TeJNiB","overwrite":0}}
+{"timestamp":1693567435.4920487,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693567474.1776569,"name":"drain","context":{"idset":"22","reason":"epilog failed for jobid fQVkRjowCBZ","overwrite":0}}
+{"timestamp":1693574501.4921758,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693574621.8089411,"name":"drain","context":{"idset":"17","reason":"epilog failed for jobid fQXosqDz5Uj","overwrite":0}}
+{"timestamp":1693577165.4934106,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693577208.2196689,"name":"drain","context":{"idset":"28","reason":"epilog failed for jobid fQY4y6TXmDh","overwrite":0}}
+{"timestamp":1693581814.387049,"name":"drain","context":{"idset":"4-5,7-8,10-13,16,19,26-27,30-31,33-36","reason":"new amdgpu driver","overwrite":0}}
+{"timestamp":1693582408.7510896,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693582408.8518887,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1693582409.1621885,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693582409.3978972,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693582409.6699097,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693582409.7709568,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693582410.2017663,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1693582410.466002,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693582410.5432501,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1693582410.6434178,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693582411.095886,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693582411.1849205,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693582411.2851183,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693582411.5521903,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693582411.6644709,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693583590.9687114,"name":"online","context":{"idset":"34"}}
+{"timestamp":1693583591.2487428,"name":"online","context":{"idset":"14,35"}}
+{"timestamp":1693583591.4040947,"name":"online","context":{"idset":"7,21,36"}}
+{"timestamp":1693583591.6972558,"name":"online","context":{"idset":"6,22,26"}}
+{"timestamp":1693583591.8047123,"name":"online","context":{"idset":"13,16,27"}}
+{"timestamp":1693583591.9467471,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693583592.0625422,"name":"online","context":{"idset":"4,9,11,23"}}
+{"timestamp":1693583592.1891153,"name":"online","context":{"idset":"8,17,19,24-25,30"}}
+{"timestamp":1693583592.3058054,"name":"online","context":{"idset":"15,28"}}
+{"timestamp":1693583592.5419185,"name":"online","context":{"idset":"3"}}
+{"timestamp":1693583596.0248094,"name":"online","context":{"idset":"10"}}
+{"timestamp":1693583596.1524935,"name":"online","context":{"idset":"18,20,32"}}
+{"timestamp":1693583666.072778,"name":"undrain","context":{"idset":"3-4,6-11,13-28,30-32,34-36"}}
+{"timestamp":1693587771.5588708,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693587771.6589165,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693588903.9847393,"name":"online","context":{"idset":"12"}}
+{"timestamp":1693588904.505547,"name":"online","context":{"idset":"5"}}
+{"timestamp":1693589159.7618353,"name":"undrain","context":{"idset":"5,12"}}
+{"timestamp":1693602934.3333118,"name":"drain","context":{"idset":"3-28,30-36","reason":"new cxi driver","overwrite":2}}
+{"timestamp":1693603264.7497387,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1693603264.7516146,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1693603264.8017521,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1693603264.841748,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1693603264.9205859,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1693603264.9341574,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1693603264.9578574,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1693603265.0272853,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1693603265.0330496,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1693603265.1340661,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1693603265.1982298,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1693603265.2988882,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1693603265.3786747,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1693603265.4793363,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1693603265.735456,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1693604517.2994635,"name":"online","context":{"idset":"33"}}
+{"timestamp":1693604517.4844265,"name":"online","context":{"idset":"7"}}
+{"timestamp":1693604517.5873699,"name":"online","context":{"idset":"25"}}
+{"timestamp":1693604517.7054024,"name":"online","context":{"idset":"34,36"}}
+{"timestamp":1693604517.8594475,"name":"online","context":{"idset":"23,30,35"}}
+{"timestamp":1693604518.0348506,"name":"online","context":{"idset":"18,28,32"}}
+{"timestamp":1693604518.1766758,"name":"online","context":{"idset":"26-27"}}
+{"timestamp":1693604518.3579655,"name":"online","context":{"idset":"24"}}
+{"timestamp":1693604522.935909,"name":"online","context":{"idset":"31"}}
+{"timestamp":1693604558.0336232,"name":"undrain","context":{"idset":"7,18,23-28,30-36"}}
+{"timestamp":1693607159.4930558,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1693607520.4945188,"name":"drain","context":{"idset":"22","reason":"epilog failed for jobid fQburJV78R5","overwrite":0}}
+{"timestamp":1693790868.7590442,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1693790868.7899907,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1693790868.850961,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1693790868.8904476,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1693790868.9909716,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1693790869.029995,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1693790869.0335042,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1693790869.0354211,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1693790869.0698073,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1693790869.075845,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1693790869.1177683,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1693790869.2185192,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1693790869.5328162,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1693790869.7237177,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1693790870.4344425,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1693790870.5209782,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1693790870.6213653,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1693791940.7364676,"name":"online","context":{"idset":"17,21"}}
+{"timestamp":1693791941.1338775,"name":"online","context":{"idset":"5,8"}}
+{"timestamp":1693791941.4626355,"name":"online","context":{"idset":"22"}}
+{"timestamp":1693791941.6012306,"name":"online","context":{"idset":"16,19-20"}}
+{"timestamp":1693791941.7321491,"name":"online","context":{"idset":"3,6,14"}}
+{"timestamp":1693791941.9272761,"name":"online","context":{"idset":"4,9,12-13"}}
+{"timestamp":1693791942.0531232,"name":"online","context":{"idset":"11"}}
+{"timestamp":1693791942.1950555,"name":"online","context":{"idset":"10"}}
+{"timestamp":1693791942.364476,"name":"online","context":{"idset":"15"}}
+{"timestamp":1693792030.7497768,"name":"undrain","context":{"idset":"3-6,8-17,19-22"}}
+{"timestamp":1694127765.3925214,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1694127765.3926573,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1694127765.3927159,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1694127765.3927748,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1694127765.3928375,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1694127765.3928926,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1694127765.4929497,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1694127767.4929996,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1694127769.393142,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1694127769.3932486,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1694127769.3933229,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1694127769.3933783,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1694127769.3934302,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1694127769.3934836,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1694127769.4935405,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1694127830.919374,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1694127830.9196122,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1694127830.9197199,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1694127830.9198136,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1694127830.9199073,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1694127830.9199991,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1694127830.9200897,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1694127830.920176,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1694127830.92026,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1694127830.9203501,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1694127830.920435,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1694127830.9205184,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1694127830.9205983,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1694127830.9206793,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1694127831.0262504,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1694127831.234967,"name":"drain","context":{"idset":"21","reason":"epilog failed for jobid fRnY4JbkYqD","overwrite":0}}
+{"timestamp":1694127831.2598069,"name":"drain","context":{"idset":"22","reason":"epilog failed for jobid fRn9zGVopHu","overwrite":0}}
+{"timestamp":1694127831.2860894,"name":"drain","context":{"idset":"23","reason":"epilog failed for jobid fRnFJZGbo19","overwrite":0}}
+{"timestamp":1694127831.314266,"name":"drain","context":{"idset":"25","reason":"epilog failed for jobid fRmkfnUidHR","overwrite":0}}
+{"timestamp":1694127831.3411562,"name":"drain","context":{"idset":"24,26","reason":"epilog failed for jobid fRnFxoBZN9V","overwrite":0}}
+{"timestamp":1694127863.9925857,"name":"drain","context":{"idset":"17-19","reason":"epilog failed for jobid fRmuU6GLWEs","overwrite":0}}
+{"timestamp":1694136279.1119742,"name":"online","context":{"idset":"17"}}
+{"timestamp":1694136279.8665237,"name":"online","context":{"idset":"22-23"}}
+{"timestamp":1694136280.2967498,"name":"online","context":{"idset":"18,21"}}
+{"timestamp":1694136280.5123765,"name":"online","context":{"idset":"19-20,24"}}
+{"timestamp":1694136281.1614838,"name":"online","context":{"idset":"25,31"}}
+{"timestamp":1694136281.5977638,"name":"online","context":{"idset":"26,28,30,32"}}
+{"timestamp":1694136545.1400652,"name":"undrain","context":{"idset":"17-26,28,30-32"}}
+{"timestamp":1694139410.2371705,"name":"online","context":{"idset":"27"}}
+{"timestamp":1694139433.0136747,"name":"undrain","context":{"idset":"27"}}
+{"timestamp":1694364920.6817245,"name":"drain","context":{"idset":"11","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1694364921.2717447,"name":"drain","context":{"idset":"9","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1694364924.2942569,"name":"drain","context":{"idset":"8","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1694365043.3932631,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1694365043.3961675,"name":"drain","context":{"idset":"11","reason":"epilog failed for jobid fS6ruRWwPmq","overwrite":0}}
+{"timestamp":1694365043.3964453,"name":"drain","context":{"idset":"9","reason":"epilog failed for jobid fS6qejBdvNB","overwrite":0}}
+{"timestamp":1694365043.4926968,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1694365047.3962376,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid fS6rGwG4aNs","overwrite":0}}
+{"timestamp":1694365047.4933856,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1694373844.3244154,"name":"online","context":{"idset":"9"}}
+{"timestamp":1694373845.4932554,"name":"online","context":{"idset":"8"}}
+{"timestamp":1694373845.6917944,"name":"online","context":{"idset":"11"}}
+{"timestamp":1694373869.5854402,"name":"undrain","context":{"idset":"8-9,11"}}
+{"timestamp":1694385033.4926529,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1694385035.392266,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1694385035.4931104,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1694385037.3926828,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1694385037.3927979,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1694385037.3928633,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1694385037.4933655,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1694385099.3919275,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1694385099.3921208,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1694385099.3922215,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1694385099.3923328,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1694385099.3924248,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1694385099.3925312,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1694385099.5095546,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1694385099.7267785,"name":"drain","context":{"idset":"3","reason":"epilog failed for jobid fSMfSnEspBZ","overwrite":0}}
+{"timestamp":1694385099.7459412,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid fSNY488BHU3","overwrite":0}}
+{"timestamp":1694385099.7718592,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid fSNYgknp2aT","overwrite":0}}
+{"timestamp":1694385099.7993665,"name":"drain","context":{"idset":"6-7","reason":"epilog failed for jobid fSNZK3ScU2s","overwrite":0}}
+{"timestamp":1694385099.8190832,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid fSNfakhqEzb","overwrite":0}}
+{"timestamp":1694452217.3010468,"name":"online","context":{"idset":"2"}}
+{"timestamp":1694452257.4710085,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1694452319.2943318,"name":"online","context":{"idset":"6"}}
+{"timestamp":1694452319.8180544,"name":"online","context":{"idset":"5,7-8"}}
+{"timestamp":1694452319.9793844,"name":"online","context":{"idset":"4"}}
+{"timestamp":1694452320.0997944,"name":"online","context":{"idset":"3"}}
+{"timestamp":1694452334.4502833,"name":"undrain","context":{"idset":"3-8"}}
+{"timestamp":1694471805.4940894,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1694471807.4934354,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1694471809.4936566,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1694471869.4929519,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1694471869.7307694,"name":"drain","context":{"idset":"24","reason":"epilog failed for jobid fSZrDiXYULF","overwrite":0}}
+{"timestamp":1694471871.3924346,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1694471871.4930208,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1694473143.3715377,"name":"drain","context":{"idset":"21","reason":"epilog failed for jobid fSa1tAxwJiK","overwrite":0}}
+{"timestamp":1694538460.6354742,"name":"online","context":{"idset":"19,24"}}
+{"timestamp":1694538479.3726845,"name":"undrain","context":{"idset":"19,24"}}
+{"timestamp":1694543241.9688122,"name":"online","context":{"idset":"21"}}
+{"timestamp":1694543255.9704905,"name":"undrain","context":{"idset":"21"}}
+{"timestamp":1694546517.3925986,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1694546517.4926074,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1694546519.4934289,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1694546521.3932989,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1694546521.3934104,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1694546521.3934922,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1694546521.3935566,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1694546521.4941707,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1694546581.7713592,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1694546582.0040925,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid fSiPXZHJMTd","overwrite":0}}
+{"timestamp":1694546583.3919053,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1694546583.392041,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1694546583.3921421,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1694546583.3922403,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1694546583.3923645,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1694546583.3924901,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1694546583.3956997,"name":"drain","context":{"idset":"13","reason":"prolog failed for jobid fSjrs7eQHK5","overwrite":0}}
+{"timestamp":1694546583.3958344,"name":"drain","context":{"idset":"12","reason":"epilog failed for jobid fSZK1oEotEb","overwrite":0}}
+{"timestamp":1694546583.4198148,"name":"drain","context":{"idset":"16,19","reason":"epilog failed for jobid fSZLvPNJg1D","overwrite":0}}
+{"timestamp":1694546583.4978905,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1694546583.7259452,"name":"drain","context":{"idset":"10","reason":"epilog failed for jobid fSjmDDFTQnb","overwrite":0}}
+{"timestamp":1694546583.7264616,"name":"drain","context":{"idset":"9","reason":"epilog failed for jobid fSjm8P9j6jy","overwrite":0}}
+{"timestamp":1694546583.7863164,"name":"drain","context":{"idset":"11","reason":"epilog failed for jobid fSjofVjbe9D","overwrite":0}}
+{"timestamp":1694546583.802917,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid fSiR6CcjY4P","overwrite":0}}
+{"timestamp":1694547235.3932333,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1694547235.4933429,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1694547239.3928602,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1694547239.3930256,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1694547239.3930922,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1694547239.3931534,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1694547239.49296,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1694547299.4933469,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1694547300.5466597,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1694547300.5707254,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1694547300.5709035,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1694547300.5710094,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1694547300.6706436,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1694547300.8837578,"name":"drain","context":{"idset":"5-6","reason":"epilog failed for jobid fSjV2ThwYrT","overwrite":0}}
+{"timestamp":1694547300.9073741,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid fSjWE1wbEfZ","overwrite":0}}
+{"timestamp":1694547300.9406824,"name":"drain","context":{"idset":"3,8","reason":"epilog failed for jobid fSjbqGPAhAo","overwrite":0}}
+{"timestamp":1694547302.1490448,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1694547302.3275039,"name":"drain","context":{"idset":"7","reason":"epilog failed for jobid fSjWE2c8uzX","overwrite":0}}
+{"timestamp":1694548245.6069736,"name":"drain","context":{"idset":"17","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694548263.4930742,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1694548306.7050459,"name":"drain","context":{"idset":"33","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694548315.5017011,"name":"drain","context":{"idset":"27","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694548329.4926527,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1694548512.3219762,"name":"drain","context":{"idset":"36","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694548986.2525644,"name":"drain","context":{"idset":"18","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694549022.9844933,"name":"drain","context":{"idset":"30","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694549652.3126264,"name":"drain","context":{"idset":"20","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694549652.912406,"name":"drain","context":{"idset":"22","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694550057.7064192,"name":"drain","context":{"idset":"34","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694550057.8867466,"name":"drain","context":{"idset":"35","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694550099.5576289,"name":"drain","context":{"idset":"21","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694550099.7630906,"name":"drain","context":{"idset":"25","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694551321.3928807,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1694551321.3930371,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1694551321.3931231,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1694551321.3932054,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1694551321.393297,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1694551321.3933742,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1694551321.3934536,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1694551321.493623,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1694551323.392761,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1694551323.3929193,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1694551323.3929992,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1694551323.3930793,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1694551323.4932587,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1694552755.1612029,"name":"online","context":{"idset":"2"}}
+{"timestamp":1694552755.7371101,"name":"drain","context":{"idset":"31","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694552818.1389978,"name":"drain","context":{"idset":"28","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694552845.4848061,"name":"online","context":{"idset":"4"}}
+{"timestamp":1694552845.5561788,"name":"online","context":{"idset":"20"}}
+{"timestamp":1694552846.4003496,"name":"online","context":{"idset":"35"}}
+{"timestamp":1694552846.5685811,"name":"online","context":{"idset":"13,33,36"}}
+{"timestamp":1694552846.7161493,"name":"online","context":{"idset":"7,10"}}
+{"timestamp":1694552846.8710604,"name":"online","context":{"idset":"5,16,19,34"}}
+{"timestamp":1694552846.999337,"name":"online","context":{"idset":"6,8"}}
+{"timestamp":1694552847.1211102,"name":"online","context":{"idset":"12"}}
+{"timestamp":1694552847.2916598,"name":"online","context":{"idset":"3,25,27"}}
+{"timestamp":1694552847.3922844,"name":"online","context":{"idset":"18,26"}}
+{"timestamp":1694552847.4928238,"name":"online","context":{"idset":"22"}}
+{"timestamp":1694552847.6528332,"name":"online","context":{"idset":"15,21"}}
+{"timestamp":1694552848.2039104,"name":"online","context":{"idset":"14,17"}}
+{"timestamp":1694552848.7633526,"name":"online","context":{"idset":"30"}}
+{"timestamp":1694552849.9407275,"name":"online","context":{"idset":"9"}}
+{"timestamp":1694554222.1397951,"name":"drain","context":{"idset":"23","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694555052.7110417,"name":"undrain","context":{"idset":"3-10,12-22,25-27,30,33-36"}}
+{"timestamp":1694555088.7858651,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1694555408.2738843,"name":"online","context":{"idset":"11"}}
+{"timestamp":1694555446.653342,"name":"undrain","context":{"idset":"11"}}
+{"timestamp":1694555663.3924425,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1694555663.3925972,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1694555663.492775,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1694556028.3223269,"name":"drain","context":{"idset":"15","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694556119.3933318,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1694556119.4942381,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.3930633,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.3931987,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.3932619,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.3933337,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.3933887,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.393451,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.3935103,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.3935695,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.3936262,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.3936827,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1694556123.4937646,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1694556127.4938982,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1694556184.0845962,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1694556184.2777247,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1694556184.2779658,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1694556184.2781126,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1694556184.2782629,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1694556184.2784247,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1694556184.2785504,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1694556184.2818074,"name":"drain","context":{"idset":"16","reason":"prolog failed for jobid fSm86Yo5r7Z","overwrite":0}}
+{"timestamp":1694556184.3496494,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid fSkMPznaxPq","overwrite":0}}
+{"timestamp":1694556184.3774798,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1694556184.5804021,"name":"drain","context":{"idset":"13","reason":"epilog failed for jobid fSm2m9Xq7QK","overwrite":0}}
+{"timestamp":1694556188.3482277,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1694556188.3484235,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1694556188.3485184,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1694556188.3486435,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1694556188.3487625,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1694556188.348881,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1694556188.4485335,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1694556188.6504016,"name":"drain","context":{"idset":"3,11","reason":"epilog failed for jobid fSm5V38tsZH","overwrite":0}}
+{"timestamp":1694556188.7160029,"name":"drain","context":{"idset":"4-10,12","reason":"epilog failed for jobid fSkDNj9Dfu1","overwrite":0}}
+{"timestamp":1694556208.7923725,"name":"drain","context":{"idset":"32","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694556607.4932303,"name":"online","context":{"idset":"23"}}
+{"timestamp":1694556607.8635976,"name":"online","context":{"idset":"31"}}
+{"timestamp":1694556607.9706838,"name":"online","context":{"idset":"28"}}
+{"timestamp":1694556626.2337809,"name":"undrain","context":{"idset":"23,28,31"}}
+{"timestamp":1694557687.1784673,"name":"drain","context":{"idset":"36","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694558685.4934235,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1694558687.4926767,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1694559165.2057498,"name":"drain","context":{"idset":"33","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694559167.7050798,"name":"drain","context":{"idset":"34","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694559543.8702047,"name":"online","context":{"idset":"2"}}
+{"timestamp":1694559558.2941148,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1694559627.3927946,"name":"online","context":{"idset":"36"}}
+{"timestamp":1694559631.6405895,"name":"online","context":{"idset":"32"}}
+{"timestamp":1694559673.5842316,"name":"undrain","context":{"idset":"32,36"}}
+{"timestamp":1694559795.2715712,"name":"online","context":{"idset":"4,13"}}
+{"timestamp":1694559795.4581873,"name":"online","context":{"idset":"12,15"}}
+{"timestamp":1694559795.7025201,"name":"online","context":{"idset":"10-11"}}
+{"timestamp":1694559795.8776264,"name":"online","context":{"idset":"3,5,8"}}
+{"timestamp":1694559796.1151538,"name":"online","context":{"idset":"6"}}
+{"timestamp":1694559796.442688,"name":"online","context":{"idset":"9"}}
+{"timestamp":1694559796.6418886,"name":"online","context":{"idset":"7,14"}}
+{"timestamp":1694559796.7443104,"name":"online","context":{"idset":"16"}}
+{"timestamp":1694559892.6119835,"name":"undrain","context":{"idset":"3-16"}}
+{"timestamp":1694560020.8998604,"name":"undrain","context":{"idset":"33-34"}}
+{"timestamp":1694560051.2570765,"name":"drain","context":{"idset":"3","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694560051.4419634,"name":"drain","context":{"idset":"4","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694560208.9581587,"name":"drain","context":{"idset":"6","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694560221.192837,"name":"drain","context":{"idset":"5","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694560240.4115741,"name":"drain","context":{"idset":"7","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694560249.7555604,"name":"drain","context":{"idset":"8","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694560413.4660096,"name":"drain","context":{"idset":"24","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1694560646.4120975,"name":"undrain","context":{"idset":"3-8,24"}}
+{"timestamp":1694570762.4267509,"name":"drain","context":{"idset":"13","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1694806707.492825,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1694806771.4938312,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1694808843.3430884,"name":"resource-init","context":{"restart":true,"drain":{"13":{"timestamp":1694570762.4267509,"reason":"nodediag failed amdgpu"},"24":{"timestamp":1694806707.492825,"reason":"broker was unresponsive"},"29":{"timestamp":1693336554.1257296,"reason":"gpu problem"}},"online":"","exclude":"0-2"}}
+{"timestamp":1694808843.3474295,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1694808977.6337438,"name":"online","context":{"idset":"0"}}
+{"timestamp":1694808978.2850685,"name":"online","context":{"idset":"1-2"}}
+{"timestamp":1694808978.7424824,"name":"online","context":{"idset":"24,33-36"}}
+{"timestamp":1694808978.9883254,"name":"online","context":{"idset":"9-10,13,16,21,28"}}
+{"timestamp":1694808979.100316,"name":"online","context":{"idset":"4,14"}}
+{"timestamp":1694808979.2069995,"name":"online","context":{"idset":"5-6,8,31-32"}}
+{"timestamp":1694808979.3099139,"name":"online","context":{"idset":"3,17-18,23,25,27,30"}}
+{"timestamp":1694808979.4181035,"name":"online","context":{"idset":"11,19-20"}}
+{"timestamp":1694808979.5568295,"name":"online","context":{"idset":"7,12,22"}}
+{"timestamp":1694808979.7230718,"name":"online","context":{"idset":"15,26"}}
+{"timestamp":1694809261.8481922,"name":"undrain","context":{"idset":"13"}}
+{"timestamp":1694809355.1442113,"name":"undrain","context":{"idset":"24"}}
+{"timestamp":1694809856.9357831,"name":"drain","context":{"idset":"13","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1695208553.9116523,"name":"drain","context":{"idset":"11","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1695230615.9773138,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1695230896.188288,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1695234328.7933946,"name":"online","context":{"idset":"11"}}
+{"timestamp":1695234355.2638817,"name":"undrain","context":{"idset":"11"}}
+{"timestamp":1695236829.1097085,"name":"drain","context":{"idset":"14","reason":"Bad partner Node --JRG","overwrite":0}}
+{"timestamp":1695239529.1416612,"name":"drain","context":{"idset":"30","reason":"Broken partner Node --JRG","overwrite":0}}
+{"timestamp":1695239550.4974294,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1695239663.1198502,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1695242352.9642303,"name":"drain","context":{"idset":"23,26-28,31-36","reason":"reboot","overwrite":0}}
+{"timestamp":1695242710.6683557,"name":"online","context":{"idset":"29"}}
+{"timestamp":1695242904.4688151,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1695242904.4689343,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1695242904.4689982,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1695242904.4690573,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1695242904.4691193,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1695242904.469187,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1695242904.4692962,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1695242904.4693856,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1695242904.4694784,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1695242904.5689483,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1695243728.1752021,"name":"online","context":{"idset":"13"}}
+{"timestamp":1695243841.1206043,"name":"online","context":{"idset":"36"}}
+{"timestamp":1695243841.3439813,"name":"online","context":{"idset":"33-34"}}
+{"timestamp":1695243841.5057547,"name":"online","context":{"idset":"35"}}
+{"timestamp":1695243842.166719,"name":"online","context":{"idset":"26"}}
+{"timestamp":1695243843.5583124,"name":"online","context":{"idset":"23"}}
+{"timestamp":1695243844.0716164,"name":"online","context":{"idset":"27-28,31-32"}}
+{"timestamp":1695243873.0925901,"name":"undrain","context":{"idset":"23,26-28,31-36"}}
+{"timestamp":1695244727.0948634,"name":"online","context":{"idset":"14"}}
+{"timestamp":1695244759.8195512,"name":"undrain","context":{"idset":"13-14,29-30"}}
+{"timestamp":1695245833.0632102,"name":"drain","context":{"idset":"17-22,24-25","reason":"reboot","overwrite":0}}
+{"timestamp":1695246342.5705495,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1695246910.4693441,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1695246910.4695542,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1695246910.5697722,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1695247332.2481599,"name":"drain","context":{"idset":"17,21,24-25","reason":"reboot rabbit202","overwrite":1}}
+{"timestamp":1695247858.1964905,"name":"online","context":{"idset":"18,20"}}
+{"timestamp":1695247858.3633893,"name":"online","context":{"idset":"19"}}
+{"timestamp":1695247859.083652,"name":"online","context":{"idset":"22"}}
+{"timestamp":1695247871.8212187,"name":"undrain","context":{"idset":"18-20,22"}}
+{"timestamp":1695247918.9038091,"name":"drain","context":{"idset":"3-12,15-16","reason":"reboot rabbit201","overwrite":0}}
+{"timestamp":1695248071.8803959,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1695248071.880595,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1695248071.8806896,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1695248071.8807871,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1695248071.8808832,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1695248071.9809854,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1695248244.5702522,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1695248644.5693998,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1695248646.247659,"name":"drain","context":{"idset":"4-5,7-9,16,21,25","reason":"rebooting","overwrite":1}}
+{"timestamp":1695248950.7797859,"name":"online","context":{"idset":"4,7"}}
+{"timestamp":1695248951.0067792,"name":"online","context":{"idset":"16,21"}}
+{"timestamp":1695248951.186501,"name":"online","context":{"idset":"8"}}
+{"timestamp":1695248951.3317935,"name":"online","context":{"idset":"25"}}
+{"timestamp":1695248960.2487793,"name":"undrain","context":{"idset":"4,7-8,16,21,25"}}
+{"timestamp":1695250726.8325274,"name":"online","context":{"idset":"9"}}
+{"timestamp":1695250726.9582059,"name":"online","context":{"idset":"5"}}
+{"timestamp":1695250767.7852254,"name":"undrain","context":{"idset":"5,9"}}
+{"timestamp":1695250812.8066359,"name":"drain","context":{"idset":"10,12","reason":"rebooting","overwrite":1}}
+{"timestamp":1695250918.469943,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1695250918.5698633,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1695251258.4694579,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1695251258.4696302,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1695251258.4696972,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1695251258.5694363,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1695251319.4540858,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1695251319.4543397,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1695251319.4544823,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1695251319.5543692,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1695251725.3434494,"name":"online","context":{"idset":"12"}}
+{"timestamp":1695251725.5701909,"name":"online","context":{"idset":"10"}}
+{"timestamp":1695251733.3659105,"name":"undrain","context":{"idset":"10,12"}}
+{"timestamp":1695253119.1275048,"name":"drain","context":{"idset":"15,24,33-36","reason":"rebooting","overwrite":1}}
+{"timestamp":1695253163.8887029,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1695253163.9884446,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1695254051.2585862,"name":"online","context":{"idset":"15"}}
+{"timestamp":1695254051.5039124,"name":"online","context":{"idset":"35"}}
+{"timestamp":1695254051.8219604,"name":"online","context":{"idset":"33,36"}}
+{"timestamp":1695254052.5705345,"name":"online","context":{"idset":"24"}}
+{"timestamp":1695254076.6296866,"name":"undrain","context":{"idset":"15,24,33,35-36"}}
+{"timestamp":1695315586.4690893,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1695315586.4692485,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1695315586.4693487,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1695315586.5693605,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1695316528.3236618,"name":"online","context":{"idset":"11"}}
+{"timestamp":1695316528.4689631,"name":"online","context":{"idset":"34"}}
+{"timestamp":1695316528.7876818,"name":"online","context":{"idset":"3"}}
+{"timestamp":1695316529.5244102,"name":"online","context":{"idset":"17"}}
+{"timestamp":1695316532.4574356,"name":"online","context":{"idset":"6"}}
+{"timestamp":1695316584.8852313,"name":"undrain","context":{"idset":"3,6,11,17,34"}}
+{"timestamp":1695318781.9085276,"name":"drain","context":{"idset":"8,11,17","reason":"reboot","overwrite":0}}
+{"timestamp":1695318816.004847,"name":"undrain","context":{"idset":"8,11,17"}}
+{"timestamp":1695319063.6897438,"name":"drain","context":{"idset":"5,12,17","reason":"reboot","overwrite":0}}
+{"timestamp":1695319113.7370782,"name":"undrain","context":{"idset":"5,12,17"}}
+{"timestamp":1695335912.3149285,"name":"drain","context":{"idset":"4,6,9,13,15-17","reason":"reboot","overwrite":0}}
+{"timestamp":1695336011.2302337,"name":"undrain","context":{"idset":"4,6,9,13,15-17"}}
+{"timestamp":1695336764.6342952,"name":"online","context":{"idset":"30"}}
+{"timestamp":1695406383.9461868,"name":"drain","context":{"idset":"9","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1695406387.1402626,"name":"drain","context":{"idset":"4","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1695409704.5695939,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1695411134.570504,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1695415266.5705278,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1695467754.4617822,"name":"drain","context":{"idset":"3","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1695494449.7392883,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1695494449.8320849,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1695494449.9323273,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1695494450.7577915,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1695494451.063839,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1695494451.6891913,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1695496227.3982773,"name":"online","context":{"idset":"9"}}
+{"timestamp":1695496228.7241158,"name":"online","context":{"idset":"11"}}
+{"timestamp":1695496229.0595677,"name":"online","context":{"idset":"6"}}
+{"timestamp":1695496229.1790628,"name":"online","context":{"idset":"3,18"}}
+{"timestamp":1695496229.337409,"name":"online","context":{"idset":"4"}}
+{"timestamp":1695497074.9545805,"name":"undrain","context":{"idset":"3-4,6,9,11,18"}}
+{"timestamp":1695666433.3264205,"name":"drain","context":{"idset":"33","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1695667860.9667566,"name":"drain","context":{"idset":"35","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1695749129.920666,"name":"drain","context":{"idset":"3-16","reason":"reboot rzvenral201","overwrite":0}}
+{"timestamp":1695749199.2882497,"name":"drain","context":{"idset":"33-36","reason":"reboot rzvenral203","overwrite":1}}
+{"timestamp":1695749396.4693985,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1695749396.4695807,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1695749396.4696805,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1695749396.5698104,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1695750361.1937747,"name":"online","context":{"idset":"33,36"}}
+{"timestamp":1695750361.4054453,"name":"online","context":{"idset":"35"}}
+{"timestamp":1695750361.7065837,"name":"online","context":{"idset":"34"}}
+{"timestamp":1695750433.3597615,"name":"undrain","context":{"idset":"33-36"}}
+{"timestamp":1695750577.5004592,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1695750577.5006459,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1695750577.5007296,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1695750577.5008087,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1695750577.6011407,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1695752887.2220776,"name":"online","context":{"idset":"7,12"}}
+{"timestamp":1695752887.634948,"name":"online","context":{"idset":"11"}}
+{"timestamp":1695752888.3641381,"name":"online","context":{"idset":"8"}}
+{"timestamp":1695752942.7414231,"name":"undrain","context":{"idset":"7-8,11-12"}}
+{"timestamp":1695753128.4698985,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1695753128.470063,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1695753128.4701478,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1695753128.4702311,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1695753128.5703528,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1695754399.3707793,"name":"online","context":{"idset":"3"}}
+{"timestamp":1695754399.5838237,"name":"online","context":{"idset":"13"}}
+{"timestamp":1695754399.7624662,"name":"online","context":{"idset":"4,14,16"}}
+{"timestamp":1695754400.1109872,"name":"online","context":{"idset":"15"}}
+{"timestamp":1695754422.0923116,"name":"undrain","context":{"idset":"3-4,13-16"}}
+{"timestamp":1695754574.4693918,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1695754574.5692599,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1695755534.0806277,"name":"online","context":{"idset":"9"}}
+{"timestamp":1695755534.9843693,"name":"online","context":{"idset":"10"}}
+{"timestamp":1695755545.3749273,"name":"undrain","context":{"idset":"9-10"}}
+{"timestamp":1695764371.4299631,"name":"drain","context":{"idset":"5-6","reason":"reboot fake201","overwrite":1}}
+{"timestamp":1695766538.4528015,"name":"drain","context":{"idset":"26","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1695769332.5701342,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1695769746.5696454,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1695770394.4689617,"name":"online","context":{"idset":"26"}}
+{"timestamp":1695770416.1032333,"name":"undrain","context":{"idset":"26"}}
+{"timestamp":1695770523.9955704,"name":"online","context":{"idset":"6"}}
+{"timestamp":1695770548.218996,"name":"undrain","context":{"idset":"6"}}
+{"timestamp":1695833270.569526,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1695834056.4698198,"name":"online","context":{"idset":"5"}}
+{"timestamp":1695914354.9481988,"name":"drain","context":{"idset":"3-36","reason":"cooling work","overwrite":1}}
+{"timestamp":1695914498.4694517,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1695914498.4695992,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1695914498.4696898,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1695914498.4697738,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1695914498.4698589,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1695914498.4699452,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1695914498.4700329,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1695914498.4701238,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1695914498.4702365,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1695914498.4724231,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1695914498.5724671,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1695914498.7973464,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid fVp7nj4f9Ub","overwrite":0}}
+{"timestamp":1695914500.469897,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1695914500.4700277,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1695914500.4701126,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1695914500.470196,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1695914500.4702868,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1695914500.4703655,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1695914500.470443,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1695914500.4705298,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1695914500.4706147,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1695914500.4707041,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1695914500.4707859,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1695914500.4708679,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1695914500.4709671,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1695914500.4710457,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1695914500.4711311,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1695914500.4712055,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1695914500.4713051,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1695914500.4713705,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1695914500.4714532,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1695914500.4715152,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1695914500.4715898,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1695914500.4716539,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1695914500.4717159,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1695914500.5700793,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1695914500.8095329,"name":"drain","context":{"idset":"3-4","reason":"epilog failed for jobid fVoQPr6goRH","overwrite":0}}
+{"timestamp":1695914504.5696707,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1695914566.4697719,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1695914566.5702643,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1695922748.2531843,"name":"online","context":{"idset":"22"}}
+{"timestamp":1695922749.3104026,"name":"online","context":{"idset":"19,21"}}
+{"timestamp":1695922749.454987,"name":"online","context":{"idset":"23-24"}}
+{"timestamp":1695922749.6207023,"name":"online","context":{"idset":"18,20"}}
+{"timestamp":1695922749.7251558,"name":"online","context":{"idset":"27-28"}}
+{"timestamp":1695922749.8518152,"name":"online","context":{"idset":"17,30"}}
+{"timestamp":1695922749.9958036,"name":"online","context":{"idset":"25-26,29,31"}}
+{"timestamp":1695922750.167105,"name":"online","context":{"idset":"32"}}
+{"timestamp":1695922924.6296678,"name":"online","context":{"idset":"35-36"}}
+{"timestamp":1695922924.7699745,"name":"online","context":{"idset":"33-34"}}
+{"timestamp":1695923168.4695063,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1695923168.4696996,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1695923168.5694275,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1695923170.5695405,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1695924487.481142,"name":"online","context":{"idset":"35"}}
+{"timestamp":1695924596.8184311,"name":"online","context":{"idset":"34"}}
+{"timestamp":1695924597.1681366,"name":"online","context":{"idset":"33"}}
+{"timestamp":1695924597.2947226,"name":"online","context":{"idset":"36"}}
+{"timestamp":1695925320.4320128,"name":"online","context":{"idset":"15"}}
+{"timestamp":1695925322.3619802,"name":"online","context":{"idset":"11-12"}}
+{"timestamp":1695925322.5443008,"name":"online","context":{"idset":"14,16"}}
+{"timestamp":1695925322.8427496,"name":"online","context":{"idset":"13"}}
+{"timestamp":1695925885.1539776,"name":"undrain","context":{"idset":"11-36"}}
+{"timestamp":1695925979.2590265,"name":"online","context":{"idset":"3,5"}}
+{"timestamp":1695925979.3715463,"name":"online","context":{"idset":"9"}}
+{"timestamp":1695925979.85725,"name":"online","context":{"idset":"8,10"}}
+{"timestamp":1695925980.0025549,"name":"online","context":{"idset":"4"}}
+{"timestamp":1695925980.1239684,"name":"online","context":{"idset":"7"}}
+{"timestamp":1695926000.417845,"name":"undrain","context":{"idset":"3-5,7-10"}}
+{"timestamp":1695926074.2143967,"name":"online","context":{"idset":"1"}}
+{"timestamp":1695926172.2053049,"name":"online","context":{"idset":"2"}}
+{"timestamp":1695927147.7243402,"name":"online","context":{"idset":"6"}}
+{"timestamp":1695927168.4761686,"name":"undrain","context":{"idset":"6"}}
+{"timestamp":1695927172.764044,"name":"undrain","context":{"idset":"1-2"}}
+{"timestamp":1696049214.2751634,"name":"drain","context":{"idset":"36","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1696049336.4733171,"name":"drain","context":{"idset":"36","reason":"epilog failed for jobid fVw1PkTxj3D","overwrite":0}}
+{"timestamp":1696049336.5707746,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1696049436.3897679,"name":"drain","context":{"idset":"33","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1696049558.4730098,"name":"drain","context":{"idset":"33","reason":"epilog failed for jobid fVw1PjLjGe3","overwrite":0}}
+{"timestamp":1696049558.5698428,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1696051724.3890257,"name":"drain","context":{"idset":"34","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1696051724.3930049,"name":"drain","context":{"idset":"35","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1696051845.0969639,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1696051845.9287763,"name":"drain","context":{"idset":"34-35","reason":"epilog failed for jobid fVw1PjuLzqd","overwrite":0}}
+{"timestamp":1696051846.0263746,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1696260317.9004247,"name":"online","context":{"idset":"34"}}
+{"timestamp":1696260318.2212117,"name":"online","context":{"idset":"35"}}
+{"timestamp":1696260318.427664,"name":"online","context":{"idset":"33,36"}}
+{"timestamp":1696260343.2356935,"name":"undrain","context":{"idset":"33-36"}}
+{"timestamp":1696361739.484163,"name":"resource-init","context":{"restart":true,"drain":{},"online":"","exclude":"0-2"}}
+{"timestamp":1696361739.488512,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1696361848.3139358,"name":"online","context":{"idset":"0"}}
+{"timestamp":1696367375.989265,"name":"online","context":{"idset":"1"}}
+{"timestamp":1696367456.9520044,"name":"online","context":{"idset":"2"}}
+{"timestamp":1696367540.8946981,"name":"online","context":{"idset":"19"}}
+{"timestamp":1696367541.7176154,"name":"online","context":{"idset":"21,24,31"}}
+{"timestamp":1696367541.82342,"name":"online","context":{"idset":"17,22"}}
+{"timestamp":1696367541.8389444,"name":"online","context":{"idset":"23"}}
+{"timestamp":1696367541.9670861,"name":"online","context":{"idset":"26"}}
+{"timestamp":1696367542.0977304,"name":"online","context":{"idset":"29"}}
+{"timestamp":1696367542.2126846,"name":"online","context":{"idset":"18"}}
+{"timestamp":1696367542.6921132,"name":"online","context":{"idset":"30,32,34"}}
+{"timestamp":1696367542.8247058,"name":"online","context":{"idset":"28"}}
+{"timestamp":1696367543.1525059,"name":"online","context":{"idset":"3,33"}}
+{"timestamp":1696367543.2563179,"name":"online","context":{"idset":"6,35-36"}}
+{"timestamp":1696367543.6327751,"name":"online","context":{"idset":"7"}}
+{"timestamp":1696367543.9991419,"name":"online","context":{"idset":"8"}}
+{"timestamp":1696367544.1515977,"name":"online","context":{"idset":"4-5"}}
+{"timestamp":1696367544.9943063,"name":"online","context":{"idset":"13"}}
+{"timestamp":1696367545.3554242,"name":"online","context":{"idset":"10,12,16"}}
+{"timestamp":1696367545.6052694,"name":"online","context":{"idset":"9,14"}}
+{"timestamp":1696367545.7209013,"name":"online","context":{"idset":"11"}}
+{"timestamp":1696367546.263943,"name":"online","context":{"idset":"20"}}
+{"timestamp":1696368789.7928324,"name":"online","context":{"idset":"25"}}
+{"timestamp":1696369429.8857908,"name":"online","context":{"idset":"27"}}
+{"timestamp":1696369432.7091043,"name":"online","context":{"idset":"15"}}
+{"timestamp":1696371783.904444,"name":"drain","context":{"idset":"3-36","overwrite":0}}
+{"timestamp":1696374013.0991428,"name":"undrain","context":{"idset":"33-36"}}
+{"timestamp":1696374231.2920842,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1696374231.2922313,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1696374231.2923207,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1696374231.2924063,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1696374231.2924888,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1696374231.2925711,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1696374231.2926514,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1696374231.2927308,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1696374231.2928216,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1696374231.2929015,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1696374231.2929788,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1696374231.2930639,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1696374231.2931576,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1696374231.2932549,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1696374231.2933438,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1696374231.2934384,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1696374231.2935176,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1696374231.2936106,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1696374231.2936943,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1696374231.2937882,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1696374231.2938678,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1696374231.2939594,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1696374231.2940342,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1696374231.3926938,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1696374233.2916529,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1696374233.2917626,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1696374233.2918413,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1696374233.2919044,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1696374233.291966,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1696374233.3923733,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1696375378.7849121,"name":"online","context":{"idset":"5"}}
+{"timestamp":1696375380.5587616,"name":"online","context":{"idset":"7-8"}}
+{"timestamp":1696375380.6781337,"name":"online","context":{"idset":"3-4,6"}}
+{"timestamp":1696375380.9142597,"name":"online","context":{"idset":"9"}}
+{"timestamp":1696375381.4437442,"name":"online","context":{"idset":"14-15"}}
+{"timestamp":1696375381.7829304,"name":"online","context":{"idset":"10-11"}}
+{"timestamp":1696375381.9498675,"name":"online","context":{"idset":"12-13,16"}}
+{"timestamp":1696375413.1883564,"name":"undrain","context":{"idset":"3-16"}}
+{"timestamp":1696375497.0232635,"name":"online","context":{"idset":"22"}}
+{"timestamp":1696375497.820205,"name":"online","context":{"idset":"23"}}
+{"timestamp":1696375498.3633559,"name":"online","context":{"idset":"18"}}
+{"timestamp":1696375498.5012205,"name":"online","context":{"idset":"17,20,32"}}
+{"timestamp":1696375498.7925093,"name":"online","context":{"idset":"19,21,24"}}
+{"timestamp":1696375499.2008862,"name":"online","context":{"idset":"25,29"}}
+{"timestamp":1696375499.391989,"name":"online","context":{"idset":"26-28,30-31"}}
+{"timestamp":1696375516.5483057,"name":"undrain","context":{"idset":"17-32"}}
+{"timestamp":1696545408.6339865,"name":"drain","context":{"idset":"20","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1696551358.4757018,"name":"drain","context":{"idset":"10","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1696551361.5697751,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1696590945.296252,"name":"drain","context":{"idset":"5","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1696870243.3924413,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1696870307.391957,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1696870419.3920469,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1696870421.291321,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1696870421.2914369,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1696870421.3913386,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1696871316.748493,"name":"online","context":{"idset":"1"}}
+{"timestamp":1696871451.8389604,"name":"undrain","context":{"idset":"1"}}
+{"timestamp":1696871509.903842,"name":"online","context":{"idset":"10"}}
+{"timestamp":1696871510.2226219,"name":"online","context":{"idset":"20"}}
+{"timestamp":1696871510.4789412,"name":"online","context":{"idset":"7"}}
+{"timestamp":1696871510.7859912,"name":"online","context":{"idset":"5"}}
+{"timestamp":1696871555.2600594,"name":"undrain","context":{"idset":"5,7,10,20"}}
+{"timestamp":1696967347.3243403,"name":"drain","context":{"idset":"10","reason":"prolog failed for jobid fYBFh8aFQKH","overwrite":0}}
+{"timestamp":1696970075.9116812,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1696970080.169271,"name":"drain","context":{"idset":"3","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1696975179.2917118,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1696975179.2918873,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1696975179.3920841,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1696976455.7352934,"name":"online","context":{"idset":"3,6"}}
+{"timestamp":1696976455.8650832,"name":"online","context":{"idset":"10"}}
+{"timestamp":1696977036.84763,"name":"undrain","context":{"idset":"3,6,10"}}
+{"timestamp":1697199087.3926661,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1697204835.3934171,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1697233921.3931305,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1697233923.3921604,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1697234939.0788538,"name":"online","context":{"idset":"5"}}
+{"timestamp":1697234939.3919539,"name":"online","context":{"idset":"3"}}
+{"timestamp":1697235045.3879631,"name":"undrain","context":{"idset":"3,5"}}
+{"timestamp":1697282155.7388096,"name":"drain","context":{"idset":"6","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1697315773.7159965,"name":"drain","context":{"idset":"10","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1697315897.2949798,"name":"drain","context":{"idset":"10","reason":"epilog failed for jobid fYjhDpzWuAo","overwrite":0}}
+{"timestamp":1697315897.3920014,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1697317292.8000019,"name":"drain","context":{"idset":"4","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1697317415.2947545,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid fYkmU3hSKvF","overwrite":0}}
+{"timestamp":1697317415.3923233,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1697320673.3915865,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1697322081.7616789,"name":"online","context":{"idset":"6"}}
+{"timestamp":1697322081.954576,"name":"online","context":{"idset":"4,10"}}
+{"timestamp":1697322101.3007145,"name":"undrain","context":{"idset":"4,6,10"}}
+{"timestamp":1697397096.7157972,"name":"drain","context":{"idset":"8","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1697397219.2952788,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid fYjJjXYbvsy","overwrite":0}}
+{"timestamp":1697397219.3926458,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1697476525.1993411,"name":"online","context":{"idset":"8"}}
+{"timestamp":1697497458.4749491,"name":"drain","context":{"idset":"9","reason":"prolog failed for jobid fZNhhyBzFxB","overwrite":0}}
+{"timestamp":1697502264.7814496,"name":"undrain","context":{"idset":"8"}}
+{"timestamp":1697627738.1683393,"name":"drain","context":{"idset":"3","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1697652109.3920152,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1697652111.3923502,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1697653238.9130418,"name":"online","context":{"idset":"3"}}
+{"timestamp":1697653239.0210137,"name":"online","context":{"idset":"9"}}
+{"timestamp":1697653256.2960627,"name":"undrain","context":{"idset":"3,9"}}
+{"timestamp":1697973387.1837721,"name":"drain","context":{"idset":"4","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1698233438.4140465,"name":"drain","context":{"idset":"8","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1698252247.2912412,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1698252247.3915036,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1698259000.5326769,"name":"online","context":{"idset":"4"}}
+{"timestamp":1698259001.1321256,"name":"online","context":{"idset":"8"}}
+{"timestamp":1698263018.9727104,"name":"drain","context":{"idset":"24,27-28,31","reason":"prolog failed for jobid fb6zmBg6Gby","overwrite":0}}
+{"timestamp":1698267577.2945733,"name":"drain","context":{"idset":"9","reason":"epilog failed for jobid fb4sAVw1K2X","overwrite":0}}
+{"timestamp":1698267577.391259,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1698267584.1027615,"name":"online","context":{"idset":"9"}}
+{"timestamp":1698267686.6756842,"name":"drain","context":{"idset":"10","reason":"epilog failed for jobid fb4sAXmkRKM","overwrite":0}}
+{"timestamp":1698267730.8553238,"name":"drain","context":{"idset":"13","reason":"epilog failed for jobid fb4sAc6JKwd","overwrite":0}}
+{"timestamp":1698267730.8574178,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid fb4sAh6suBD","overwrite":0}}
+{"timestamp":1698267730.8578885,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid fb4sAeAPKmZ","overwrite":0}}
+{"timestamp":1698267730.8593297,"name":"drain","context":{"idset":"16","reason":"epilog failed for jobid fb4sAkCGQGw","overwrite":0}}
+{"timestamp":1698267730.8606985,"name":"drain","context":{"idset":"11","reason":"epilog failed for jobid fb4sAYbC2LP","overwrite":0}}
+{"timestamp":1698267730.8631144,"name":"drain","context":{"idset":"12","reason":"epilog failed for jobid fb4sAaB7Fom","overwrite":0}}
+{"timestamp":1698270654.8468909,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1698270654.8470414,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1698270654.8471239,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1698270654.8472025,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1698270654.8472764,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1698270654.9469812,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1698270656.1940918,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1698270656.2946701,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1698270756.7174747,"name":"undrain","context":{"idset":"4,8"}}
+{"timestamp":1698270897.2919047,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1698270897.2920525,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1698270897.2921364,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1698270897.392051,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1698271606.6720197,"name":"online","context":{"idset":"9"}}
+{"timestamp":1698271607.0914681,"name":"online","context":{"idset":"10,12"}}
+{"timestamp":1698271607.5585918,"name":"online","context":{"idset":"11,13-16"}}
+{"timestamp":1698271628.9827583,"name":"undrain","context":{"idset":"9-16"}}
+{"timestamp":1698271870.1094556,"name":"online","context":{"idset":"31"}}
+{"timestamp":1698271870.4247196,"name":"online","context":{"idset":"24,27"}}
+{"timestamp":1698271870.635812,"name":"online","context":{"idset":"28"}}
+{"timestamp":1698271893.0165224,"name":"undrain","context":{"idset":"24,27-28,31"}}
+{"timestamp":1698333678.5632927,"name":"drain","context":{"idset":"7","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1698333801.2948267,"name":"drain","context":{"idset":"7","reason":"epilog failed for jobid fb4hSKQB9wd","overwrite":0}}
+{"timestamp":1698333801.3920548,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1698345157.770261,"name":"drain","context":{"idset":"6","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1698345281.2950342,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid fb4ZQBwL1Cj","overwrite":0}}
+{"timestamp":1698345281.392262,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1698345697.703357,"name":"drain","context":{"idset":"3","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1698345821.2949822,"name":"drain","context":{"idset":"3","reason":"epilog failed for jobid faticeKSnQF","overwrite":0}}
+{"timestamp":1698345821.3917544,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1698347925.8042543,"name":"drain","context":{"idset":"11","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1698350173.3921878,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1698350823.8025548,"name":"drain","context":{"idset":"5","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1698350947.29479,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid fb4MHi23ftT","overwrite":0}}
+{"timestamp":1698350947.3921909,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1698351375.690933,"name":"drain","context":{"idset":"9","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1698353076.4954288,"name":"online","context":{"idset":"11"}}
+{"timestamp":1698353076.6207273,"name":"online","context":{"idset":"6"}}
+{"timestamp":1698353076.7670624,"name":"online","context":{"idset":"3"}}
+{"timestamp":1698353076.9272208,"name":"online","context":{"idset":"7"}}
+{"timestamp":1698353095.9239471,"name":"undrain","context":{"idset":"3,6-7,11"}}
+{"timestamp":1698353247.3931322,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1698353950.2829504,"name":"drain","context":{"idset":"12","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1698363089.3925822,"name":"online","context":{"idset":"5"}}
+{"timestamp":1698363089.6374276,"name":"online","context":{"idset":"9"}}
+{"timestamp":1698363107.0968122,"name":"undrain","context":{"idset":"5,9"}}
+{"timestamp":1698363255.3918436,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1698364826.3114753,"name":"online","context":{"idset":"12"}}
+{"timestamp":1698365071.1923492,"name":"undrain","context":{"idset":"12"}}
+{"timestamp":1698405337.8258264,"name":"drain","context":{"idset":"4","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1698424731.3931534,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1698428817.2914388,"name":"online","context":{"idset":"4"}}
+{"timestamp":1698428838.6191571,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1698578138.9341917,"name":"drain","context":{"idset":"4","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1698685077.3925338,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1698686158.8228524,"name":"online","context":{"idset":"4"}}
+{"timestamp":1698686195.740227,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1698700585.291842,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1698700585.291985,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1698700585.292047,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1698700585.3920979,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1698700587.292206,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1698700587.2922881,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1698700587.2923448,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1698700587.2923892,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1698700587.2924407,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1698700587.3931458,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1698700589.2915025,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1698700589.2915699,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1698700589.39223,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1698700651.2920702,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1698700651.2922037,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1698700651.292285,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1698700651.2923675,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1698700651.2924426,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1698700651.2925155,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1698700651.2925889,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1698700651.2926621,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1698700651.2927458,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1698700651.2928348,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1698700651.2929311,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1698700651.2930212,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1698700651.4059091,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1698704298.5438437,"name":"drain","context":{"idset":"3","overwrite":0}}
+{"timestamp":1698706962.42255,"name":"online","context":{"idset":"13"}}
+{"timestamp":1698706962.5882916,"name":"online","context":{"idset":"16"}}
+{"timestamp":1698706964.9933019,"name":"online","context":{"idset":"11"}}
+{"timestamp":1698706965.5792594,"name":"online","context":{"idset":"4,6"}}
+{"timestamp":1698706965.7902374,"name":"online","context":{"idset":"12"}}
+{"timestamp":1698706965.987498,"name":"online","context":{"idset":"5"}}
+{"timestamp":1698706966.4626861,"name":"online","context":{"idset":"9-10"}}
+{"timestamp":1698706967.2082438,"name":"online","context":{"idset":"14"}}
+{"timestamp":1698706967.4468448,"name":"online","context":{"idset":"15"}}
+{"timestamp":1698706987.7337122,"name":"undrain","context":{"idset":"4-6,9-16"}}
+{"timestamp":1698707008.5378728,"name":"drain","context":{"idset":"3","reason":"reboot","overwrite":1}}
+{"timestamp":1698707734.8893569,"name":"online","context":{"idset":"7"}}
+{"timestamp":1698707735.390667,"name":"online","context":{"idset":"8"}}
+{"timestamp":1698707764.3252654,"name":"undrain","context":{"idset":"7-8"}}
+{"timestamp":1698708211.6830432,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1698709006.4060557,"name":"online","context":{"idset":"1"}}
+{"timestamp":1698709217.3928361,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1698709989.7559566,"name":"online","context":{"idset":"3"}}
+{"timestamp":1698710013.2454998,"name":"undrain","context":{"idset":"3"}}
+{"timestamp":1698782609.2916038,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1698782609.3916175,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1698782611.3904872,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1698782613.2914286,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1698782613.2915061,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1698782613.3923416,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1698782672.867095,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1698782673.203989,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1698782673.9481406,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1698782674.1745343,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1698782674.1746461,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1698782674.2749512,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1698798539.3923759,"name":"online","context":{"idset":"6"}}
+{"timestamp":1698798539.7028329,"name":"online","context":{"idset":"5"}}
+{"timestamp":1698798539.8589263,"name":"online","context":{"idset":"8"}}
+{"timestamp":1698798540.1221759,"name":"online","context":{"idset":"3"}}
+{"timestamp":1698798540.3043318,"name":"online","context":{"idset":"4"}}
+{"timestamp":1698798543.2804992,"name":"online","context":{"idset":"7"}}
+{"timestamp":1698798567.457871,"name":"undrain","context":{"idset":"3-8"}}
+{"timestamp":1698898119.2212982,"name":"drain","context":{"idset":"35","reason":"prolog failed for jobid fcYCXjbBs5Z","overwrite":0}}
+{"timestamp":1698983885.7954438,"name":"drain","context":{"idset":"23","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1698984009.2946894,"name":"drain","context":{"idset":"23","reason":"epilog failed for jobid fcY66BLsgoR","overwrite":0}}
+{"timestamp":1698984009.3922031,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1699028178.7412593,"name":"drain","context":{"idset":"16","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1699028287.7114272,"name":"drain","context":{"idset":"24,28","reason":"prolog failed for jobid fcqFc6ub4KH","overwrite":0}}
+{"timestamp":1699028299.9856875,"name":"drain","context":{"idset":"16","reason":"epilog failed for jobid fcdvE8oVBKd","overwrite":0}}
+{"timestamp":1699028300.0829937,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1699041783.4963048,"name":"drain","context":{"idset":"9","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1699041907.2950048,"name":"drain","context":{"idset":"9","reason":"epilog failed for jobid fcbvJx7ExD5","overwrite":0}}
+{"timestamp":1699041907.3918917,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1699047955.533289,"name":"drain","context":{"idset":"13","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1699048079.2952232,"name":"drain","context":{"idset":"13","reason":"epilog failed for jobid fced5wutVPV","overwrite":0}}
+{"timestamp":1699048079.3924296,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1699057837.3206918,"name":"undrain","context":{"idset":"23"}}
+{"timestamp":1699057847.8625484,"name":"undrain","context":{"idset":"16"}}
+{"timestamp":1699057852.7588508,"name":"undrain","context":{"idset":"13"}}
+{"timestamp":1699057855.8148282,"name":"undrain","context":{"idset":"9"}}
+{"timestamp":1699058447.0979354,"name":"online","context":{"idset":"9"}}
+{"timestamp":1699058519.6965454,"name":"online","context":{"idset":"13"}}
+{"timestamp":1699058581.3929327,"name":"online","context":{"idset":"16"}}
+{"timestamp":1699058677.6374595,"name":"online","context":{"idset":"23"}}
+{"timestamp":1699059829.2918489,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1699059829.2920172,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1699059829.3917878,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1699064612.878504,"name":"online","context":{"idset":"24"}}
+{"timestamp":1699064771.0378337,"name":"online","context":{"idset":"28"}}
+{"timestamp":1699064891.9431348,"name":"online","context":{"idset":"35"}}
+{"timestamp":1699064935.7478447,"name":"undrain","context":{"idset":"24"}}
+{"timestamp":1699064938.7538722,"name":"undrain","context":{"idset":"28"}}
+{"timestamp":1699064942.4479222,"name":"undrain","context":{"idset":"35"}}
+{"timestamp":1699108294.5256507,"name":"drain","context":{"idset":"6","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1699108417.2946174,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid fcoENDFQ44o","overwrite":0}}
+{"timestamp":1699108417.3921847,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1699110183.8238149,"name":"drain","context":{"idset":"7","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1699110215.1377816,"name":"drain","context":{"idset":"10","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1699110215.1401656,"name":"drain","context":{"idset":"3","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1699110293.8338606,"name":"drain","context":{"idset":"15","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1699110307.2946231,"name":"drain","context":{"idset":"7","reason":"epilog failed for jobid fcpVC2nM7NP","overwrite":0}}
+{"timestamp":1699110307.3919404,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1699110337.2923136,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1699110337.2956264,"name":"drain","context":{"idset":"3,10","reason":"epilog failed for jobid fckt3jJF271","overwrite":0}}
+{"timestamp":1699110337.3926568,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1699110417.2943027,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid fcpf6JEFuSf","overwrite":0}}
+{"timestamp":1699110417.3919351,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1699111462.6613293,"name":"drain","context":{"idset":"8","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1699111585.2952204,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid fcohinGC7KD","overwrite":0}}
+{"timestamp":1699111585.3919103,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1699121757.3931863,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1699121759.3925927,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1699121761.2923708,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1699121761.2924676,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1699121761.392585,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1699121825.2919405,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1699121825.2920818,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1699121825.29217,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1699121825.2922897,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1699121825.2950382,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid fcs6mwTWFou","overwrite":0}}
+{"timestamp":1699121825.2952077,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid fcgs1wLsAcF","overwrite":0}}
+{"timestamp":1699121825.2956748,"name":"drain","context":{"idset":"11","reason":"epilog failed for jobid fcmR9Yzvtxf","overwrite":0}}
+{"timestamp":1699121825.2957509,"name":"drain","context":{"idset":"12","reason":"epilog failed for jobid fcmR9ZY4dsu","overwrite":0}}
+{"timestamp":1699121825.2965109,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid fcsrmGJMWbh","overwrite":0}}
+{"timestamp":1699121825.3921025,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1699122870.7949834,"name":"online","context":{"idset":"8"}}
+{"timestamp":1699122871.0461578,"name":"online","context":{"idset":"10"}}
+{"timestamp":1699122871.1708541,"name":"online","context":{"idset":"6,15"}}
+{"timestamp":1699122871.2769408,"name":"online","context":{"idset":"7"}}
+{"timestamp":1699123010.2086968,"name":"undrain","context":{"idset":"3,6-8,10,15"}}
+{"timestamp":1699123038.2324941,"name":"undrain","context":{"idset":"4-5,11-12,14"}}
+{"timestamp":1699134773.7312596,"name":"online","context":{"idset":"14"}}
+{"timestamp":1699134828.0693393,"name":"online","context":{"idset":"5"}}
+{"timestamp":1699134828.2369177,"name":"online","context":{"idset":"11"}}
+{"timestamp":1699134828.4359407,"name":"online","context":{"idset":"12"}}
+{"timestamp":1699134828.6169968,"name":"online","context":{"idset":"3-4"}}
+{"timestamp":1699294593.3923998,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1699294654.6535254,"name":"drain","context":{"idset":"7","reason":"epilog failed for jobid fdHY5ToUDBu","overwrite":0}}
+{"timestamp":1699294654.750855,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1699294660.3354294,"name":"drain","context":{"idset":"7","reason":"PY: SS not passing traffic","overwrite":1}}
+{"timestamp":1699295541.0629618,"name":"online","context":{"idset":"7"}}
+{"timestamp":1699295785.0277631,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1699317389.3926749,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1699317450.2493403,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1699317933.3927598,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1699317993.8062329,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1699346875.3029735,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1699369347.2923512,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2924886,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2925568,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2926049,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2926564,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2927122,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2927637,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2928226,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.292865,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2929163,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2929604,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.293009,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2930548,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2930977,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.293143,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.2931898,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1699369347.3927314,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1699369349.2917762,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1699369349.2918913,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1699369349.2919486,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1699369349.2920022,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1699369349.2920561,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1699369349.2921069,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1699369349.2921586,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1699369349.2922087,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1699369349.2922649,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1699369349.3917236,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1699369351.292166,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1699369351.2922516,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1699369351.2923048,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1699369351.392276,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1699369413.2922878,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1699369413.2924371,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1699369413.2925241,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1699369413.2926304,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1699369413.2927108,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1699369413.2927995,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1699369413.2928779,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1699369413.2929537,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1699369413.2930269,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1699369413.2931013,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1699369413.2931817,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1699369413.2932818,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1699369413.2933567,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1699369413.2934515,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1699369413.2935398,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1699369413.2936442,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1699369413.2937167,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1699369413.2937953,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1699369413.2938795,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1699369413.2939658,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1699369413.294055,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1699369413.2941287,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1699369413.2942057,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1699369413.2942903,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1699369413.2943516,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1699369413.2944188,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1699369413.2944992,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1699369413.2945759,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1699369413.2946355,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1699369413.2946968,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1699369413.2947514,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1699369413.3923542,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1699369425.3920414,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1699369429.3926032,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1699369493.2919865,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1699369493.3918357,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1699370892.7646363,"name":"drain","context":{"idset":"1-36","reason":"power","overwrite":1}}
+{"timestamp":1699488151.3346119,"name":"undrain","context":{"idset":"1-29,31-36"}}
+{"timestamp":1699489005.8119237,"name":"resource-init","context":{"restart":true,"drain":{"30":{"timestamp":1699369347.293009,"reason":"power"}},"online":"","exclude":"0-2"}}
+{"timestamp":1699489005.8190696,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1699489113.8772976,"name":"online","context":{"idset":"0"}}
+{"timestamp":1699489114.7384281,"name":"online","context":{"idset":"1,25,34-36"}}
+{"timestamp":1699489114.8581324,"name":"online","context":{"idset":"2-4,6-8,10-11,16-19,21,26,29,32-33"}}
+{"timestamp":1699489115.0029328,"name":"online","context":{"idset":"5,9,12-13,15,20,24,27,30-31"}}
+{"timestamp":1699489115.1116974,"name":"online","context":{"idset":"23"}}
+{"timestamp":1699489115.2776752,"name":"online","context":{"idset":"22"}}
+{"timestamp":1699489115.5477419,"name":"online","context":{"idset":"14,28"}}
+{"timestamp":1699489148.3917992,"name":"undrain","context":{"idset":"30"}}
+{"timestamp":1699489159.5188849,"name":"drain","context":{"idset":"30","reason":"slingshot","overwrite":0}}
+{"timestamp":1699575108.637388,"name":"drain","context":{"idset":"20","reason":"prolog failed for jobid fdniybWyUyu","overwrite":0}}
+{"timestamp":1699575109.6672521,"name":"drain","context":{"idset":"21","reason":"prolog failed for jobid fdniz3e8pzK","overwrite":0}}
+{"timestamp":1699895806.9774122,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1699895808.9767456,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1699896574.3995235,"name":"online","context":{"idset":"21"}}
+{"timestamp":1699896574.7458112,"name":"online","context":{"idset":"20"}}
+{"timestamp":1699896591.9724834,"name":"undrain","context":{"idset":"20-21"}}
+{"timestamp":1700035884.1490028,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700064964.198509,"name":"drain","context":{"idset":"15","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1700064964.3989232,"name":"drain","context":{"idset":"12","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1700064964.7390661,"name":"drain","context":{"idset":"16","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1700065086.8772142,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1700065086.8773477,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1700065086.880022,"name":"drain","context":{"idset":"12","reason":"epilog failed for jobid fehab5PQi4P","overwrite":0}}
+{"timestamp":1700065086.8803043,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid fehabAM2JjH","overwrite":0}}
+{"timestamp":1700065086.8805048,"name":"drain","context":{"idset":"16","reason":"epilog failed for jobid fehabB7VwBd","overwrite":0}}
+{"timestamp":1700065086.9771562,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1700070405.6457081,"name":"drain","context":{"idset":"8","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1700070528.8796451,"name":"drain","context":{"idset":"8","reason":"epilog failed for jobid feh7YDLq7Ys","overwrite":0}}
+{"timestamp":1700070528.9768076,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1700071532.9770131,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1700076369.9128747,"name":"drain","context":{"idset":"7","reason":"prolog failed for jobid fevPmv1M4nb","overwrite":0}}
+{"timestamp":1700078303.4495046,"name":"online","context":{"idset":"16"}}
+{"timestamp":1700078303.8907287,"name":"online","context":{"idset":"8"}}
+{"timestamp":1700078304.2167811,"name":"online","context":{"idset":"5,12"}}
+{"timestamp":1700078304.384872,"name":"online","context":{"idset":"15"}}
+{"timestamp":1700089031.8503606,"name":"undrain","context":{"idset":"5,8,12,15-16"}}
+{"timestamp":1700159042.9772828,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1700159736.7272673,"name":"drain","context":{"idset":"6-7","reason":"power","overwrite":1}}
+{"timestamp":1700172915.8992052,"name":"online","context":{"idset":"7"}}
+{"timestamp":1700172934.300209,"name":"undrain","context":{"idset":"6-7"}}
+{"timestamp":1700208688.4000249,"name":"drain","context":{"idset":"18","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700243939.8287182,"name":"drain","context":{"idset":"16","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700258828.4672754,"name":"drain","context":{"idset":"3-4,6-7,10,13,26-29,31-36","reason":"prolog failed for jobid ffLD89qKQUP","overwrite":0}}
+{"timestamp":1700266384.8764987,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1700266384.8766551,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1700266384.8767509,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1700266384.876847,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1700266384.8769641,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1700266384.8771141,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1700266384.8772485,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1700266384.8774004,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1700266384.8774958,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1700266384.8776093,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1700266384.8777108,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1700266384.8777995,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1700266384.8779044,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1700266384.8779888,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1700266384.8780749,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1700266384.9767907,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1700267472.818424,"name":"online","context":{"idset":"35"}}
+{"timestamp":1700267472.8771179,"name":"online","context":{"idset":"34"}}
+{"timestamp":1700267473.0915711,"name":"online","context":{"idset":"36"}}
+{"timestamp":1700267473.2974079,"name":"online","context":{"idset":"10,32"}}
+{"timestamp":1700267473.6894,"name":"online","context":{"idset":"29"}}
+{"timestamp":1700267473.8003049,"name":"online","context":{"idset":"3,7,31"}}
+{"timestamp":1700267474.1726189,"name":"online","context":{"idset":"26,28"}}
+{"timestamp":1700267474.3206975,"name":"online","context":{"idset":"4,6,13"}}
+{"timestamp":1700267474.8321686,"name":"online","context":{"idset":"33"}}
+{"timestamp":1700267483.6822319,"name":"online","context":{"idset":"27"}}
+{"timestamp":1700267590.1225169,"name":"undrain","context":{"idset":"3-4,6-7,10,13,26-29,31-36"}}
+{"timestamp":1700349290.2823999,"name":"drain","context":{"idset":"3","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1700396138.7971628,"name":"drain","context":{"idset":"5","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1700397056.8089654,"name":"drain","context":{"idset":"7","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1700506100.9769244,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1700506102.8764036,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1700506102.8765171,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1700506102.8766017,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1700506102.9765563,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1700507850.0688097,"name":"online","context":{"idset":"5"}}
+{"timestamp":1700507850.3549366,"name":"online","context":{"idset":"3,18"}}
+{"timestamp":1700507850.727433,"name":"online","context":{"idset":"7,16"}}
+{"timestamp":1700507973.7227995,"name":"undrain","context":{"idset":"3,5,7,16,18"}}
+{"timestamp":1700528095.3053153,"name":"resource-init","context":{"restart":true,"drain":{"30":{"timestamp":1699489159.5188849,"reason":"slingshot"}},"online":"","exclude":"0-2"}}
+{"timestamp":1700528095.3161066,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1700528195.8459127,"name":"online","context":{"idset":"0"}}
+{"timestamp":1700528196.5190885,"name":"online","context":{"idset":"1,9,11,24,30"}}
+{"timestamp":1700528196.9206679,"name":"online","context":{"idset":"14,22"}}
+{"timestamp":1700528197.351217,"name":"online","context":{"idset":"10,19,21,25"}}
+{"timestamp":1700528197.4799309,"name":"online","context":{"idset":"12,20,33,35"}}
+{"timestamp":1700528197.5915165,"name":"online","context":{"idset":"26,29,34,36"}}
+{"timestamp":1700528197.721211,"name":"online","context":{"idset":"17,23"}}
+{"timestamp":1700528197.8893113,"name":"online","context":{"idset":"3-8,13,15-16,18,28,32"}}
+{"timestamp":1700528198.0259795,"name":"online","context":{"idset":"31"}}
+{"timestamp":1700528198.155797,"name":"online","context":{"idset":"27"}}
+{"timestamp":1700528254.0412312,"name":"drain","context":{"idset":"37-38","reason":"bringup","overwrite":0}}
+{"timestamp":1700528575.4661686,"name":"online","context":{"idset":"37"}}
+{"timestamp":1700528577.0804734,"name":"online","context":{"idset":"38"}}
+{"timestamp":1700529152.3117254,"name":"undrain","context":{"idset":"37-38"}}
+{"timestamp":1700529171.5986621,"name":"drain","context":{"idset":"37-38","reason":"bringup","overwrite":0}}
+{"timestamp":1700536160.8567002,"name":"online","context":{"idset":"2"}}
+{"timestamp":1700541684.5625112,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700587860.9530828,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1700588006.9533737,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1700588645.7332644,"name":"online","context":{"idset":"5"}}
+{"timestamp":1700588664.8276482,"name":"undrain","context":{"idset":"5"}}
+{"timestamp":1700588768.5785282,"name":"online","context":{"idset":"37"}}
+{"timestamp":1700588952.9534194,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1700591890.9532151,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1700593384.6241543,"name":"online","context":{"idset":"38"}}
+{"timestamp":1700594421.3975861,"name":"online","context":{"idset":"37"}}
+{"timestamp":1700594460.6090927,"name":"undrain","context":{"idset":"37-38"}}
+{"timestamp":1700598087.4373732,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1700598242.2822433,"name":"drain","context":{"idset":"19","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700598243.248647,"name":"drain","context":{"idset":"20","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700598248.2793508,"name":"drain","context":{"idset":"23","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700598993.0988183,"name":"online","context":{"idset":"1"}}
+{"timestamp":1700601420.8410716,"name":"drain","context":{"idset":"16","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700601548.8526855,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1700601548.8528366,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1700601548.952945,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1700602474.8535249,"name":"online","context":{"idset":"20"}}
+{"timestamp":1700602475.0568032,"name":"online","context":{"idset":"19"}}
+{"timestamp":1700602478.2918952,"name":"online","context":{"idset":"23"}}
+{"timestamp":1700602503.5103755,"name":"undrain","context":{"idset":"19-20,23"}}
+{"timestamp":1700606476.9530838,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1700607587.0649736,"name":"online","context":{"idset":"16"}}
+{"timestamp":1700607610.5347455,"name":"undrain","context":{"idset":"16"}}
+{"timestamp":1700607850.80937,"name":"drain","context":{"idset":"30","reason":"INC0387666 slingshot","overwrite":1}}
+{"timestamp":1700640683.2579532,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700674784.9530308,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1700675551.2108107,"name":"online","context":{"idset":"6"}}
+{"timestamp":1700675565.4636285,"name":"undrain","context":{"idset":"6"}}
+{"timestamp":1700677464.9535511,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1700677478.9534569,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1700683112.8529162,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1700683112.953594,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1700683864.490814,"name":"online","context":{"idset":"26"}}
+{"timestamp":1700683864.6018417,"name":"online","context":{"idset":"22"}}
+{"timestamp":1700683878.6395471,"name":"undrain","context":{"idset":"22,26"}}
+{"timestamp":1700695360.9541607,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1700697112.9542968,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1700697362.9534664,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1700709360.953629,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1700709442.9539337,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1700709534.9525211,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1700727541.2388046,"name":"drain","context":{"idset":"16","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700765251.6028256,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1700765251.6220119,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1700765251.656575,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1700765251.7570655,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1700765252.1407251,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1700765252.2413249,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1700766830.7990587,"name":"online","context":{"idset":"12"}}
+{"timestamp":1700766834.232316,"name":"online","context":{"idset":"33"}}
+{"timestamp":1700766837.192215,"name":"online","context":{"idset":"13"}}
+{"timestamp":1700766837.7811439,"name":"online","context":{"idset":"20"}}
+{"timestamp":1700767085.9624467,"name":"online","context":{"idset":"35"}}
+{"timestamp":1700767135.4771924,"name":"undrain","context":{"idset":"12-13,20,33,35"}}
+{"timestamp":1700813475.2471993,"name":"drain","context":{"idset":"12","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1700852049.1028593,"name":"drain","context":{"idset":"34","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1700852172.8564291,"name":"drain","context":{"idset":"34","reason":"epilog failed for jobid fgUaagfW7hH","overwrite":0}}
+{"timestamp":1700852172.9532881,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1700860036.4205709,"name":"drain","context":{"idset":"11","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1700860036.4478204,"name":"drain","context":{"idset":"15","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1700860036.458883,"name":"drain","context":{"idset":"7","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1700860158.8532872,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1700860158.8534117,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1700860158.8569019,"name":"drain","context":{"idset":"7,11,15","reason":"epilog failed for jobid fgTkwYZNFQj","overwrite":0}}
+{"timestamp":1700860158.9536226,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1701000963.4046471,"name":"drain","context":{"idset":"4","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1701052437.3778055,"name":"drain","context":{"idset":"27","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701052512.6240931,"name":"drain","context":{"idset":"36","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701052890.3582969,"name":"drain","context":{"idset":"35","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701063980.9528418,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1701064186.953969,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1701064976.9536586,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1701065368.9544134,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1701065718.953711,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1701065826.9545522,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1701065874.954386,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1701105958.9526954,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1701105960.8530724,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1701105960.8532269,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1701105960.8533654,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1701105960.8535094,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1701105960.8536546,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1701105960.8538096,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1701105960.8539543,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1701105960.8540728,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1701105960.8541956,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1701105960.854322,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1701105960.854434,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1701105960.9534769,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1701106914.7540598,"name":"online","context":{"idset":"34"}}
+{"timestamp":1701106915.2636046,"name":"online","context":{"idset":"36"}}
+{"timestamp":1701106915.4484129,"name":"online","context":{"idset":"35"}}
+{"timestamp":1701106917.1517596,"name":"online","context":{"idset":"29"}}
+{"timestamp":1701106917.4132476,"name":"online","context":{"idset":"18"}}
+{"timestamp":1701106917.5180743,"name":"online","context":{"idset":"5"}}
+{"timestamp":1701106917.7051921,"name":"online","context":{"idset":"28"}}
+{"timestamp":1701106917.8164611,"name":"online","context":{"idset":"10"}}
+{"timestamp":1701106917.9704268,"name":"online","context":{"idset":"4,24"}}
+{"timestamp":1701106918.1437535,"name":"online","context":{"idset":"15"}}
+{"timestamp":1701106918.3250985,"name":"online","context":{"idset":"11-12,16"}}
+{"timestamp":1701106918.579062,"name":"online","context":{"idset":"27"}}
+{"timestamp":1701106919.5485005,"name":"online","context":{"idset":"7"}}
+{"timestamp":1701106920.8930874,"name":"online","context":{"idset":"9"}}
+{"timestamp":1701106923.1337166,"name":"online","context":{"idset":"26"}}
+{"timestamp":1701106950.991709,"name":"undrain","context":{"idset":"4-5,7,9-12,15-16,18,24,26-29,34-36"}}
+{"timestamp":1701126562.9535944,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1701126564.9539931,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1701126626.8526976,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1701126626.952862,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1701127447.9797184,"name":"online","context":{"idset":"37"}}
+{"timestamp":1701127449.6792653,"name":"online","context":{"idset":"38"}}
+{"timestamp":1701127472.5349426,"name":"undrain","context":{"idset":"37-38"}}
+{"timestamp":1701129232.7590051,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1701130471.0113797,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1701130644.1098347,"name":"online","context":{"idset":"38"}}
+{"timestamp":1701159081.3354793,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701192684.952827,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1701193590.9541714,"name":"online","context":{"idset":"6"}}
+{"timestamp":1701193616.0303135,"name":"undrain","context":{"idset":"6"}}
+{"timestamp":1701194679.1422541,"name":"online","context":{"idset":"37"}}
+{"timestamp":1701215866.8780124,"name":"drain","context":{"idset":"33","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1701215989.9989283,"name":"drain","context":{"idset":"33","reason":"epilog failed for jobid fhJG9B7kMQb","overwrite":0}}
+{"timestamp":1701215990.1645505,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1701221094.8531473,"name":"online","context":{"idset":"33"}}
+{"timestamp":1701221110.2168121,"name":"undrain","context":{"idset":"33"}}
+{"timestamp":1701221890.2115817,"name":"drain","context":{"idset":"5","reason":"prolog failed for jobid fhWNRgYYF75","overwrite":0}}
+{"timestamp":1701260201.7623339,"name":"drain","context":{"idset":"4","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1701281064.8525774,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1701281064.9530203,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1701282351.7328186,"name":"online","context":{"idset":"5"}}
+{"timestamp":1701282352.0775335,"name":"online","context":{"idset":"4"}}
+{"timestamp":1701282385.9530389,"name":"undrain","context":{"idset":"4-5"}}
+{"timestamp":1701297107.1814916,"name":"drain","context":{"idset":"34","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1701297107.7543116,"name":"drain","context":{"idset":"36","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1701297228.8559952,"name":"drain","context":{"idset":"34","reason":"epilog failed for jobid fhUuSHVww5H","overwrite":0}}
+{"timestamp":1701297228.9535019,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1701297230.8563724,"name":"drain","context":{"idset":"36","reason":"epilog failed for jobid fhUuSWTbex7","overwrite":0}}
+{"timestamp":1701297230.9537764,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1701302862.9531925,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1701303128.6111143,"name":"drain","context":{"idset":"37","reason":"epilog failed for jobid fhgfgzikswM","overwrite":0}}
+{"timestamp":1701304202.9529107,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1701305288.477783,"name":"online","context":{"idset":"37"}}
+{"timestamp":1701305289.7210245,"name":"online","context":{"idset":"36"}}
+{"timestamp":1701305289.9497516,"name":"online","context":{"idset":"34"}}
+{"timestamp":1701305315.9905713,"name":"undrain","context":{"idset":"34,36-37"}}
+{"timestamp":1701319936.082268,"name":"drain","context":{"idset":"26","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701358972.047065,"name":"drain","context":{"idset":"15","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1701358972.066498,"name":"drain","context":{"idset":"12","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1701359094.853497,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1701359094.8568695,"name":"drain","context":{"idset":"12,15","reason":"epilog failed for jobid fhcrckaxLoZ","overwrite":0}}
+{"timestamp":1701359094.9535708,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1701365952.9527454,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1701366925.2531323,"name":"online","context":{"idset":"12"}}
+{"timestamp":1701366925.4867017,"name":"online","context":{"idset":"26"}}
+{"timestamp":1701366925.7637291,"name":"online","context":{"idset":"15"}}
+{"timestamp":1701366961.4027386,"name":"undrain","context":{"idset":"12,15,26"}}
+{"timestamp":1701373221.5968745,"name":"drain","context":{"idset":"14","reason":"prolog failed for jobid fhrCJsjV29D","overwrite":0}}
+{"timestamp":1701392691.8721631,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1701392695.8550963,"name":"drain","context":{"idset":"38","reason":"epilog failed for jobid fhh2SZJaKa7","overwrite":0}}
+{"timestamp":1701392695.9527285,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1701393930.0869064,"name":"online","context":{"idset":"38"}}
+{"timestamp":1701393994.6838474,"name":"undrain","context":{"idset":"38"}}
+{"timestamp":1701394951.7747519,"name":"online","context":{"idset":"37"}}
+{"timestamp":1701418290.3531911,"name":"drain","context":{"idset":"12","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701449289.6232569,"name":"drain","context":{"idset":"33","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1701449412.8565145,"name":"drain","context":{"idset":"33","reason":"epilog failed for jobid fhpqiPR3BHh","overwrite":0}}
+{"timestamp":1701449412.953402,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1701475408.9530563,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1701478980.3553545,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1701478980.3554902,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1701478980.4553776,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1701479979.6894453,"name":"online","context":{"idset":"33"}}
+{"timestamp":1701479981.4392173,"name":"online","context":{"idset":"14"}}
+{"timestamp":1701479981.5758379,"name":"online","context":{"idset":"8"}}
+{"timestamp":1701479981.7810931,"name":"online","context":{"idset":"12"}}
+{"timestamp":1701479996.6731365,"name":"undrain","context":{"idset":"8,12,14,33"}}
+{"timestamp":1701677487.7598543,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701716423.6854734,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1701716479.3822803,"name":"drain","context":{"idset":"37","reason":"rank 37 got hostname 'fake49', expected 'fake48'","overwrite":0}}
+{"timestamp":1701716479.7590168,"name":"online","context":{"idset":"37"}}
+{"timestamp":1701716517.2845697,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1701716533.4856949,"name":"online","context":{"idset":"38"}}
+{"timestamp":1701716776.9533837,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1701716840.9528441,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1701716849.9518716,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1701716853.9477618,"name":"online","context":{"idset":"37"}}
+{"timestamp":1701716918.9912782,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1701716980.3091452,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1701716989.9850876,"name":"online","context":{"idset":"37"}}
+{"timestamp":1701717430.9535971,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1701721997.7595863,"name":"online","context":{"idset":"26"}}
+{"timestamp":1701721997.879786,"name":"online","context":{"idset":"7"}}
+{"timestamp":1701722128.8029187,"name":"undrain","context":{"idset":"7,26"}}
+{"timestamp":1701766276.7053413,"name":"drain","context":{"idset":"29","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701778667.1842024,"name":"drain","context":{"idset":"4","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1701795901.8405099,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1701795901.9413099,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1701796767.7301815,"name":"online","context":{"idset":"29"}}
+{"timestamp":1701796768.1676579,"name":"online","context":{"idset":"4"}}
+{"timestamp":1701796794.877584,"name":"undrain","context":{"idset":"4,29"}}
+{"timestamp":1701812895.4540799,"name":"drain","context":{"idset":"29","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701812896.2534604,"name":"drain","context":{"idset":"33","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701812896.5196271,"name":"drain","context":{"idset":"31","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701812896.7873545,"name":"drain","context":{"idset":"24","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701812897.5567648,"name":"drain","context":{"idset":"28","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701812898.799644,"name":"drain","context":{"idset":"27","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701812901.2077107,"name":"drain","context":{"idset":"32","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701818792.2969351,"name":"drain","context":{"idset":"37","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1701818914.8560069,"name":"drain","context":{"idset":"37","reason":"epilog failed for jobid fif1kg1Ckr3","overwrite":0}}
+{"timestamp":1701818914.9531524,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1701823138.853173,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1701823138.9532938,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1701823140.8525681,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1701823140.85267,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1701823140.8527598,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1701823140.8528514,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1701823140.9533591,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1701824096.4172492,"name":"online","context":{"idset":"37"}}
+{"timestamp":1701824097.7542925,"name":"online","context":{"idset":"24,29,31"}}
+{"timestamp":1701824098.0622606,"name":"online","context":{"idset":"27"}}
+{"timestamp":1701824098.2181828,"name":"online","context":{"idset":"32"}}
+{"timestamp":1701824098.4493425,"name":"online","context":{"idset":"28"}}
+{"timestamp":1701824269.3116562,"name":"undrain","context":{"idset":"24,27-29,31-32,37"}}
+{"timestamp":1701824371.3500421,"name":"online","context":{"idset":"33"}}
+{"timestamp":1701824419.8681149,"name":"undrain","context":{"idset":"33"}}
+{"timestamp":1701851036.3875821,"name":"drain","context":{"idset":"13","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701878758.9535968,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1701878820.2483778,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1701882800.1549294,"name":"drain","context":{"idset":"37","reason":"PY: fixing ansible","overwrite":1}}
+{"timestamp":1701883306.8476236,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1701884948.8141625,"name":"online","context":{"idset":"13"}}
+{"timestamp":1701885095.779413,"name":"undrain","context":{"idset":"13"}}
+{"timestamp":1701910907.5715537,"name":"drain","context":{"idset":"19","reason":"prolog failed for jobid fj4dt1qoP5H","overwrite":0}}
+{"timestamp":1701910998.4841919,"name":"drain","context":{"idset":"20","reason":"prolog failed for jobid fj4ea2PBEg7","overwrite":0}}
+{"timestamp":1701910999.6706903,"name":"drain","context":{"idset":"24","reason":"prolog failed for jobid fj4eaZrBzab","overwrite":0}}
+{"timestamp":1701911000.6049147,"name":"drain","context":{"idset":"25","reason":"prolog failed for jobid fj4eb1BPirK","overwrite":0}}
+{"timestamp":1701911001.2846067,"name":"drain","context":{"idset":"26","reason":"prolog failed for jobid fj4ebR18BVh","overwrite":0}}
+{"timestamp":1701911002.1896272,"name":"drain","context":{"idset":"27","reason":"prolog failed for jobid fj4ebiHKozo","overwrite":0}}
+{"timestamp":1701911002.9842908,"name":"drain","context":{"idset":"29,31","reason":"prolog failed for jobid fj4ecBTGeZM","overwrite":0}}
+{"timestamp":1701911003.9599996,"name":"drain","context":{"idset":"13-14","reason":"prolog failed for jobid fj4ecTncjdh","overwrite":0}}
+{"timestamp":1701911004.9687643,"name":"drain","context":{"idset":"18,32","reason":"prolog failed for jobid fj4ecw1XYkw","overwrite":0}}
+{"timestamp":1701911504.7977931,"name":"undrain","context":{"idset":"13-14,18-20,24-27,29,31-32"}}
+{"timestamp":1701963445.8773975,"name":"online","context":{"idset":"37"}}
+{"timestamp":1701963480.7096951,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1701968351.5623353,"name":"drain","context":{"idset":"34,36","reason":"PY: reboot to clear stuck vgs command","overwrite":0}}
+{"timestamp":1701968543.515173,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1701968543.6161551,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1701969432.0619378,"name":"online","context":{"idset":"36"}}
+{"timestamp":1701969433.5945344,"name":"online","context":{"idset":"34"}}
+{"timestamp":1701969527.6011508,"name":"undrain","context":{"idset":"34,36"}}
+{"timestamp":1701978395.1557112,"name":"drain","context":{"idset":"15","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701978396.3450153,"name":"drain","context":{"idset":"19","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1701978401.2024601,"name":"drain","context":{"idset":"18","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1702023464.0420282,"name":"drain","context":{"idset":"21","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1702037737.4594746,"name":"drain","context":{"idset":"5","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1702059012.852978,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1702059012.8531165,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1702059012.8532057,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1702059012.8533256,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1702059012.9534004,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1702067451.4339504,"name":"online","context":{"idset":"15"}}
+{"timestamp":1702067451.5392158,"name":"online","context":{"idset":"5,18"}}
+{"timestamp":1702067451.6909463,"name":"online","context":{"idset":"21"}}
+{"timestamp":1702067452.2679474,"name":"online","context":{"idset":"19"}}
+{"timestamp":1702067476.0392332,"name":"undrain","context":{"idset":"5,15,18-19,21"}}
+{"timestamp":1702068010.6769617,"name":"drain","context":{"idset":"29","reason":"HPE broken partner","overwrite":0}}
+{"timestamp":1702068155.2566652,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1702068194.7452779,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1702074503.368654,"name":"online","context":{"idset":"30"}}
+{"timestamp":1702074536.8337717,"name":"undrain","context":{"idset":"30"}}
+{"timestamp":1702075658.4518621,"name":"online","context":{"idset":"29"}}
+{"timestamp":1702075680.1610546,"name":"undrain","context":{"idset":"29"}}
+{"timestamp":1702124234.7505603,"name":"drain","context":{"idset":"4","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1702210587.2471874,"name":"drain","context":{"idset":"5","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1702245018.9540093,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1702245080.8915226,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1702274170.9542608,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702274232.9537771,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702274393.2229354,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1702274393.3234415,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1702274451.3240116,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1702274451.3420672,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1702274451.3435495,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1702274451.3528678,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1702274451.3617544,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1702274451.3647869,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1702274451.3651228,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1702274451.3689363,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1702274451.371609,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1702274451.3717117,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1702274451.3741186,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1702274451.375289,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1702274451.3785677,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1702274451.3821907,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1702274451.3832026,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1702274451.3834455,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1702274451.3899031,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1702274451.3915849,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1702274451.3974776,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1702274451.4005067,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1702274451.40362,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1702274451.4213212,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1702274451.4247935,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1702274451.4255688,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1702274451.4386511,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1702274451.4390132,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1702274451.4580169,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1702274451.4805262,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1702274451.5584519,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1702274451.5711403,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1702274451.5757923,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1702274451.6080027,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1702274451.7088685,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1702274451.9065208,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1702276207.7201791,"name":"resource-init","context":{"restart":true,"drain":{"4":{"timestamp":1702124234.7505603,"reason":"nodediag failed clocksource"},"5":{"timestamp":1702210587.2471874,"reason":"nodediag failed clocksource"},"37":{"timestamp":1702274170.9542608,"reason":"broker was unresponsive"},"38":{"timestamp":1702245018.9540093,"reason":"broker was unresponsive"}},"online":"","exclude":"0-2"}}
+{"timestamp":1702276207.7318258,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1702276349.1935189,"name":"online","context":{"idset":"0"}}
+{"timestamp":1702277097.2154074,"name":"online","context":{"idset":"2"}}
+{"timestamp":1702277097.8842518,"name":"online","context":{"idset":"17,25"}}
+{"timestamp":1702277098.2104175,"name":"online","context":{"idset":"1,33"}}
+{"timestamp":1702277098.3544037,"name":"online","context":{"idset":"3,6,34-36"}}
+{"timestamp":1702277098.5333254,"name":"online","context":{"idset":"8"}}
+{"timestamp":1702277098.7339046,"name":"online","context":{"idset":"10,18"}}
+{"timestamp":1702277098.8607471,"name":"online","context":{"idset":"9,14,22"}}
+{"timestamp":1702277098.9648981,"name":"online","context":{"idset":"7,13,15-16,29"}}
+{"timestamp":1702277099.1237836,"name":"online","context":{"idset":"11-12,26,28,30-32"}}
+{"timestamp":1702277099.281203,"name":"online","context":{"idset":"19,23"}}
+{"timestamp":1702277099.4191446,"name":"online","context":{"idset":"20-21,24,27"}}
+{"timestamp":1702277492.9462025,"name":"online","context":{"idset":"4-5"}}
+{"timestamp":1702277523.3593323,"name":"undrain","context":{"idset":"4-5"}}
+{"timestamp":1702277983.7360792,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1702278830.7754617,"name":"online","context":{"idset":"33"}}
+{"timestamp":1702280100.875088,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702280102.0336528,"name":"online","context":{"idset":"38"}}
+{"timestamp":1702280592.8421693,"name":"undrain","context":{"idset":"37-38"}}
+{"timestamp":1702282440.4214761,"name":"online","context":{"idset":"40"}}
+{"timestamp":1702282440.587117,"name":"online","context":{"idset":"39"}}
+{"timestamp":1702283414.2176678,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1702283668.7954352,"name":"drain","context":{"idset":"40","reason":"nodediag failed dmi","overwrite":0}}
+{"timestamp":1702284385.7253036,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1702286618.7669322,"name":"online","context":{"idset":"39"}}
+{"timestamp":1702298778.195142,"name":"drain","context":{"idset":"5","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1702308812.1210415,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1702308812.1219859,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1702308812.1270146,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1702308812.1961768,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1702308812.1988029,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1702308812.2002356,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1702308812.2037203,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1702308812.2044513,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1702308812.2087283,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1702308812.2216442,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1702308812.223887,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1702308812.2281246,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1702308812.2290885,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1702308812.2352684,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1702308812.2385743,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1702308812.239598,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1702308812.2434587,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1702308812.2435801,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1702308812.2479947,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1702308812.2504306,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1702308812.2543991,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1702308812.2569563,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1702308812.266587,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1702308812.2690885,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1702308812.2697146,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702308812.2706609,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1702308812.2719796,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1702308812.2949955,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1702308812.2960951,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1702308812.3019445,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1702308812.3108439,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1702308812.3192089,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1702308812.3245025,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1702308812.3342228,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1702308812.382365,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1702308812.3988628,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1702308812.4292569,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1702308812.4577823,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1702308812.493525,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1702308812.5944772,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1702309131.6121182,"name":"resource-init","context":{"restart":true,"drain":{"5":{"timestamp":1702298778.195142,"reason":"nodediag failed clocksource"}},"online":"","exclude":"0-2"}}
+{"timestamp":1702309131.6208649,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1702309272.1151943,"name":"online","context":{"idset":"0"}}
+{"timestamp":1702309272.7324517,"name":"online","context":{"idset":"1-2,37-38,40"}}
+{"timestamp":1702309272.9053628,"name":"online","context":{"idset":"39"}}
+{"timestamp":1702309273.3336928,"name":"online","context":{"idset":"17,25,33"}}
+{"timestamp":1702309273.7654843,"name":"online","context":{"idset":"34"}}
+{"timestamp":1702309273.8711681,"name":"online","context":{"idset":"35-36"}}
+{"timestamp":1702309274.1371846,"name":"online","context":{"idset":"5,19"}}
+{"timestamp":1702309274.3497546,"name":"online","context":{"idset":"21,31-32"}}
+{"timestamp":1702309274.4709182,"name":"online","context":{"idset":"9"}}
+{"timestamp":1702309274.577786,"name":"online","context":{"idset":"26"}}
+{"timestamp":1702309274.6836348,"name":"online","context":{"idset":"3,10,13,24,30"}}
+{"timestamp":1702309274.7857888,"name":"online","context":{"idset":"7,11,14,27-29"}}
+{"timestamp":1702309274.8959508,"name":"online","context":{"idset":"4,6,8,12,15-16,18,20,22-23"}}
+{"timestamp":1702310232.5106225,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1702310232.5130959,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702310232.513654,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1702310232.5207326,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1702310232.5208516,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1702310232.5210075,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1702310232.5211031,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1702310232.5211833,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1702310232.521265,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1702310232.5213451,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1702310232.5214245,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1702310232.5214963,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1702310232.5215714,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1702310232.5216427,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1702310232.5217147,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1702310232.5217888,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1702310232.5218551,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1702310232.5219283,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1702310232.5219948,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1702310232.522063,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1702310232.5221269,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1702310232.5221896,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1702310232.5222497,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1702310232.5223093,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1702310232.5223691,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1702310232.5224376,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1702310232.5225344,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1702310232.5226185,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1702310232.5226903,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1702310232.5227616,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1702310232.5228486,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1702310232.5229206,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1702310232.5229819,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1702310232.5230262,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1702310232.5230718,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1702310232.5231109,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1702310232.5231521,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1702310232.5231912,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1702310232.6214869,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1702310232.7294195,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1702310280.6637087,"name":"resource-init","context":{"restart":true,"drain":{"5":{"timestamp":1702298778.195142,"reason":"nodediag failed clocksource"}},"online":"","exclude":"0-2"}}
+{"timestamp":1702310280.6763558,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1702310422.1774747,"name":"online","context":{"idset":"0"}}
+{"timestamp":1702310422.5893102,"name":"online","context":{"idset":"37-40"}}
+{"timestamp":1702310422.8579738,"name":"online","context":{"idset":"1-2,17,25"}}
+{"timestamp":1702310423.5986114,"name":"online","context":{"idset":"33-36"}}
+{"timestamp":1702310424.2028623,"name":"online","context":{"idset":"6,26-32"}}
+{"timestamp":1702310424.303607,"name":"online","context":{"idset":"4-5,7-16"}}
+{"timestamp":1702310424.5019128,"name":"online","context":{"idset":"3"}}
+{"timestamp":1702310424.6107025,"name":"online","context":{"idset":"18-24"}}
+{"timestamp":1702337161.0397501,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702337223.0765028,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702341025.0403366,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1702342008.4741027,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702342009.6231403,"name":"online","context":{"idset":"5"}}
+{"timestamp":1702342051.7974765,"name":"undrain","context":{"idset":"5"}}
+{"timestamp":1702342054.2306139,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1702349179.0398324,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702349239.6942458,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702349401.0386865,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1702349463.1146221,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1702350507.0402167,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1702350571.0397055,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1702354121.4120078,"name":"drain","context":{"idset":"3","reason":"prolog failed for jobid fk39HALdZEX","overwrite":0}}
+{"timestamp":1702399094.8263469,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702399095.5832965,"name":"online","context":{"idset":"40"}}
+{"timestamp":1702400806.6579204,"name":"undrain","context":{"idset":"37,40"}}
+{"timestamp":1702401575.4470444,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1702401933.0393498,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1702402702.8123894,"name":"online","context":{"idset":"1"}}
+{"timestamp":1702403106.0567842,"name":"online","context":{"idset":"3"}}
+{"timestamp":1702403177.0554161,"name":"undrain","context":{"idset":"3"}}
+{"timestamp":1702410074.5838664,"name":"drain","context":{"idset":"28","reason":"prolog failed for jobid fkAUSFEwZHZ","overwrite":0}}
+{"timestamp":1702410526.8299801,"name":"online","context":{"idset":"39"}}
+{"timestamp":1702411473.9227118,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1702411558.7064481,"name":"undrain","context":{"idset":"39"}}
+{"timestamp":1702411616.4302118,"name":"online","context":{"idset":"39"}}
+{"timestamp":1702411631.3031528,"name":"drain","context":{"idset":"39","reason":"PY: testing","overwrite":0}}
+{"timestamp":1702412527.0392172,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1702413294.9220665,"name":"online","context":{"idset":"28"}}
+{"timestamp":1702413314.4278688,"name":"undrain","context":{"idset":"28"}}
+{"timestamp":1702424187.0402038,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702424247.6741664,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702424519.4449563,"name":"undrain","context":{"idset":"39"}}
+{"timestamp":1702427554.2943933,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702427891.2274237,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1702455087.0223324,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1702488683.039556,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1702489455.0401332,"name":"online","context":{"idset":"7"}}
+{"timestamp":1702495684.8998308,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1702495808.5808952,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1702496854.3403103,"name":"online","context":{"idset":"1"}}
+{"timestamp":1702506522.8454924,"name":"drain","context":{"idset":"8","reason":"prolog failed for jobid fkP7Hri4UM5","overwrite":0}}
+{"timestamp":1702511237.0398281,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1702512096.1238697,"name":"online","context":{"idset":"8"}}
+{"timestamp":1702512117.7957139,"name":"undrain","context":{"idset":"8"}}
+{"timestamp":1702520399.0400796,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702520461.0391576,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702529532.5290723,"name":"drain","context":{"idset":"16","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1702530149.239682,"name":"drain","context":{"idset":"22","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1702556884.1671352,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702560019.74124,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1702563257.0400224,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702563321.0400693,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702572051.0398183,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1702572112.3739946,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1702573305.469384,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702573413.8514426,"name":"online","context":{"idset":"39"}}
+{"timestamp":1702573769.4145412,"name":"undrain","context":{"idset":"37,39"}}
+{"timestamp":1702575760.524508,"name":"drain","context":{"idset":"6","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1702575882.4307845,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid fkKkZMFj32F","overwrite":0}}
+{"timestamp":1702575882.5283887,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1702579723.0405271,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702579789.0389578,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702581658.7020204,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702581741.6182137,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1702582941.0403638,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702583002.2153196,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702587094.1938002,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702587132.3235631,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1702587694.9424226,"name":"drain","context":{"idset":"37","reason":"epilog failed for jobid fkZjRrSRkJj","overwrite":0}}
+{"timestamp":1702587695.0398016,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702588206.5553834,"name":"drain","context":{"idset":"9","reason":"prolog failed for jobid fkZoxd8YfYT","overwrite":0}}
+{"timestamp":1702588539.4200974,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702590323.0398645,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702591288.9391472,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702591328.8237958,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1702592892.4924378,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1702592892.4925838,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1702592892.5932083,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1702594719.3096571,"name":"online","context":{"idset":"16"}}
+{"timestamp":1702594719.4391856,"name":"online","context":{"idset":"22"}}
+{"timestamp":1702594719.7527783,"name":"online","context":{"idset":"9"}}
+{"timestamp":1702594721.0406103,"name":"online","context":{"idset":"6"}}
+{"timestamp":1702594808.3496635,"name":"undrain","context":{"idset":"6,9,16,22"}}
+{"timestamp":1702642538.4557698,"name":"drain","context":{"idset":"4","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1702654607.8243327,"name":"drain","context":{"idset":"10","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1702654730.9424093,"name":"drain","context":{"idset":"10","reason":"epilog failed for jobid fkXBiGf3hcb","overwrite":0}}
+{"timestamp":1702654731.0401156,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1702656379.1050396,"name":"drain","context":{"idset":"15","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1702656502.9424868,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid fkXNqLaHCC7","overwrite":0}}
+{"timestamp":1702656503.039345,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1702666443.0407522,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702666504.5086689,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702667465.1501162,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702667487.8057892,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1702667665.0402017,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702667725.2592912,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702668835.6585331,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702668864.001379,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1702685491.0398669,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1702685552.2358146,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1702687309.5363533,"name":"drain","context":{"idset":"3","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1702688171.4812565,"name":"online","context":{"idset":"37"}}
+{"timestamp":1702688941.8244691,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1702749759.040215,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1702868152.1978023,"name":"drain","context":{"idset":"6","reason":"prolog failed for jobid fmCV7Dye2Rd","overwrite":0}}
+{"timestamp":1702920376.9387784,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1702920376.9389367,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1702920377.0386806,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1702921280.5596735,"name":"online","context":{"idset":"6"}}
+{"timestamp":1702921281.0392511,"name":"online","context":{"idset":"10"}}
+{"timestamp":1702921281.2256243,"name":"online","context":{"idset":"3,15"}}
+{"timestamp":1702921281.5373106,"name":"online","context":{"idset":"4"}}
+{"timestamp":1702921416.4519536,"name":"undrain","context":{"idset":"3-4,6,10,15"}}
+{"timestamp":1702923782.5089264,"name":"online","context":{"idset":"14"}}
+{"timestamp":1702973787.6204503,"name":"drain","context":{"idset":"11","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1702990012.9007058,"name":"drain","context":{"idset":"5","reason":"nodediag failed clocksource","overwrite":0}}
+{"timestamp":1703005287.0094831,"name":"resource-init","context":{"restart":true,"drain":{"5":{"timestamp":1702990012.9007058,"reason":"nodediag failed clocksource"},"11":{"timestamp":1702973787.6204503,"reason":"nodediag failed amdgpu"}},"online":"","exclude":"0-2"}}
+{"timestamp":1703005287.0209093,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1703005380.8044343,"name":"online","context":{"idset":"0"}}
+{"timestamp":1703007506.748992,"name":"drain","context":{"idset":"41-42","reason":"PY: A0 blade not installed yet","overwrite":0}}
+{"timestamp":1703017567.9816012,"name":"resource-init","context":{"restart":true,"drain":{"5":{"timestamp":1702990012.9007058,"reason":"nodediag failed clocksource"},"11":{"timestamp":1702973787.6204503,"reason":"nodediag failed amdgpu"},"41-42":{"timestamp":1703007506.748992,"reason":"PY: A0 blade not installed yet"}},"online":"","exclude":"0-2"}}
+{"timestamp":1703017567.9884784,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1703017658.2612751,"name":"online","context":{"idset":"0"}}
+{"timestamp":1703022624.5109766,"name":"online","context":{"idset":"35-36"}}
+{"timestamp":1703022625.70275,"name":"online","context":{"idset":"34"}}
+{"timestamp":1703022627.2973711,"name":"online","context":{"idset":"1"}}
+{"timestamp":1703022627.6679659,"name":"online","context":{"idset":"25,27,30"}}
+{"timestamp":1703022627.8004818,"name":"online","context":{"idset":"9,11,17"}}
+{"timestamp":1703022627.9512756,"name":"online","context":{"idset":"10,21,31"}}
+{"timestamp":1703022628.1422527,"name":"online","context":{"idset":"4,14,18-19,22,24,32"}}
+{"timestamp":1703022628.2432194,"name":"online","context":{"idset":"26,28-29"}}
+{"timestamp":1703022628.5521581,"name":"online","context":{"idset":"3,8,15-16"}}
+{"timestamp":1703022628.6627808,"name":"online","context":{"idset":"5-7,12,20"}}
+{"timestamp":1703023335.8967862,"name":"online","context":{"idset":"33"}}
+{"timestamp":1703023337.052326,"name":"online","context":{"idset":"23"}}
+{"timestamp":1703023337.2780862,"name":"online","context":{"idset":"13"}}
+{"timestamp":1703024553.8917868,"name":"online","context":{"idset":"2"}}
+{"timestamp":1703024606.5306613,"name":"undrain","context":{"idset":"5,11"}}
+{"timestamp":1703027224.0858884,"name":"online","context":{"idset":"37"}}
+{"timestamp":1703027226.2197955,"name":"online","context":{"idset":"38"}}
+{"timestamp":1703027241.508929,"name":"online","context":{"idset":"40"}}
+{"timestamp":1703027298.2163057,"name":"online","context":{"idset":"39"}}
+{"timestamp":1703086629.0763271,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1703086629.0764401,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1703086629.0764842,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1703086629.0765295,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1703086629.0765724,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1703086629.0766141,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1703086629.1771214,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1703086633.076751,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1703086633.0768337,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1703086633.0768745,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1703086633.0769155,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1703086633.1771865,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1703086669.177108,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1703086671.0762548,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1703086671.0763476,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1703086671.0763965,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1703086671.1764746,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1703086673.0765722,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1703086673.176594,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1703086675.0760224,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1703086675.1768608,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1703086695.0770924,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1703086695.0776894,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1703086695.078099,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1703086695.0784938,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1703086695.0789032,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1703086695.0792909,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1703086695.0796716,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1703086695.08004,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1703086695.080395,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1703086695.0807636,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1703086695.0811071,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1703086695.1764846,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1703086737.0771942,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1703086737.0776532,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1703086737.0780587,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1703086737.0785234,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1703086737.0788922,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1703086737.1039343,"name":"drain","context":{"idset":"3","reason":"epilog failed for jobid fme4Jz6kJYo","overwrite":0}}
+{"timestamp":1703086737.1042635,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1703086737.1368959,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid fme4KP2bCL7","overwrite":0}}
+{"timestamp":1703086737.1377397,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1703086737.1641114,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1703086737.191705,"name":"drain","context":{"idset":"10","reason":"prolog failed for jobid fme53QtrDRh","overwrite":0}}
+{"timestamp":1703086737.2234833,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1703095639.074228,"name":"online","context":{"idset":"2"}}
+{"timestamp":1703098931.1762896,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1703098991.1764648,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1703100433.508466,"name":"online","context":{"idset":"37"}}
+{"timestamp":1703100530.5117028,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1703100665.1762986,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1703100725.7727311,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1703103915.177366,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1703103975.6191986,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1703105606.417419,"name":"drain","context":{"idset":"12","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1703105606.9069738,"name":"drain","context":{"idset":"33","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1703111965.1768701,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1703112025.7398331,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1703112701.4838443,"name":"online","context":{"idset":"39"}}
+{"timestamp":1703112761.9953258,"name":"undrain","context":{"idset":"39"}}
+{"timestamp":1703112767.9237671,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1703112919.076087,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1703112919.1764116,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1703113713.8616202,"name":"online","context":{"idset":"37"}}
+{"timestamp":1703113777.6444333,"name":"online","context":{"idset":"40"}}
+{"timestamp":1703113793.5585494,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1703113834.0246243,"name":"online","context":{"idset":"33"}}
+{"timestamp":1703113834.1924689,"name":"online","context":{"idset":"29"}}
+{"timestamp":1703113835.6727128,"name":"online","context":{"idset":"22"}}
+{"timestamp":1703113835.8911901,"name":"online","context":{"idset":"3,26,28"}}
+{"timestamp":1703113835.9619699,"name":"online","context":{"idset":"9,30"}}
+{"timestamp":1703113836.0874784,"name":"online","context":{"idset":"5,7,12,23,25"}}
+{"timestamp":1703113836.2199962,"name":"online","context":{"idset":"8,10"}}
+{"timestamp":1703113836.4400773,"name":"online","context":{"idset":"6,27,32"}}
+{"timestamp":1703113836.8093512,"name":"online","context":{"idset":"21"}}
+{"timestamp":1703113836.9443181,"name":"online","context":{"idset":"31"}}
+{"timestamp":1703113837.076762,"name":"online","context":{"idset":"24"}}
+{"timestamp":1703113848.7682681,"name":"undrain","context":{"idset":"3,5-10,12,21-33"}}
+{"timestamp":1703113929.8703461,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1703114077.176579,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1703114143.176676,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1703119224.6683815,"name":"drain","context":{"idset":"4","reason":"PY: stuck at dracut prompt during boot","overwrite":1}}
+{"timestamp":1703133829.0768898,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1703180213.1758702,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1703181414.0066707,"name":"online","context":{"idset":"39"}}
+{"timestamp":1703181484.2031064,"name":"online","context":{"idset":"7"}}
+{"timestamp":1703181493.7081728,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1703181565.7944391,"name":"undrain","context":{"idset":"39"}}
+{"timestamp":1703181643.67327,"name":"drain","context":{"idset":"7","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1703183701.7814758,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1703184318.0241668,"name":"online","context":{"idset":"4"}}
+{"timestamp":1703184371.913034,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1703184548.309891,"name":"drain","context":{"idset":"7","reason":"nodediag failed pci","overwrite":0}}
+{"timestamp":1703185422.9947088,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1703232674.0035756,"name":"drain","context":{"idset":"11","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1703283363.1774337,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1703283424.4064424,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1703310063.2960367,"name":"drain","context":{"idset":"36","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1703310185.0779183,"name":"drain","context":{"idset":"36","reason":"epilog failed for jobid fmwzzg2A1sM","overwrite":0}}
+{"timestamp":1703310185.1756499,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1703578271.5332837,"name":"drain","context":{"idset":"8","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1703664679.3936038,"name":"drain","context":{"idset":"4","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1703785589.1773751,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1703785650.6433456,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1703795099.1763301,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1703795160.274107,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1703796544.1518242,"name":"drain","context":{"idset":"9","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1704084619.847229,"name":"drain","context":{"idset":"10","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1704218341.7785819,"name":"online","context":{"idset":"38"}}
+{"timestamp":1704218498.8216968,"name":"undrain","context":{"idset":"38"}}
+{"timestamp":1704220219.0766718,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1704220219.0771384,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1704220219.0775056,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1704220219.0778596,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1704220219.1766179,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1704221027.7788346,"name":"online","context":{"idset":"37"}}
+{"timestamp":1704221153.0278296,"name":"online","context":{"idset":"36"}}
+{"timestamp":1704221155.6162174,"name":"online","context":{"idset":"8"}}
+{"timestamp":1704221155.8061788,"name":"online","context":{"idset":"11"}}
+{"timestamp":1704221156.0115113,"name":"online","context":{"idset":"4"}}
+{"timestamp":1704221156.4679382,"name":"online","context":{"idset":"9-10"}}
+{"timestamp":1704221194.2381713,"name":"undrain","context":{"idset":"4,8-11,36-37"}}
+{"timestamp":1704221257.1774049,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1704221321.176841,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1704222277.714283,"name":"online","context":{"idset":"37"}}
+{"timestamp":1704222344.9355865,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1704230727.4579914,"name":"drain","context":{"idset":"5","reason":"prolog failed for jobid fpDxPnLxPTV","overwrite":0}}
+{"timestamp":1704304109.175693,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1704304921.6074066,"name":"online","context":{"idset":"5"}}
+{"timestamp":1704304985.1896803,"name":"undrain","context":{"idset":"5"}}
+{"timestamp":1704312765.7163844,"name":"drain","context":{"idset":"39","reason":"JRG: RAS fatal errors in NC /var/log/messages","overwrite":1}}
+{"timestamp":1704315192.4061677,"name":"drain","context":{"idset":"8-9","reason":"prolog failed for jobid fpR2CEDEZwV","overwrite":0}}
+{"timestamp":1704329477.0763071,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1704329477.1766841,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1704330281.0766232,"name":"online","context":{"idset":"9"}}
+{"timestamp":1704330281.785533,"name":"online","context":{"idset":"8"}}
+{"timestamp":1704330300.4311273,"name":"undrain","context":{"idset":"8-9"}}
+{"timestamp":1704346103.3437648,"name":"drain","context":{"idset":"9","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1704393717.1765666,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1704393894.1579807,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1704395021.0693572,"name":"online","context":{"idset":"1"}}
+{"timestamp":1704395084.5798326,"name":"online","context":{"idset":"9"}}
+{"timestamp":1704402321.3310444,"name":"drain","context":{"idset":"29","reason":"prolog failed for jobid fpcSE5u1MCs","overwrite":0}}
+{"timestamp":1704404169.784796,"name":"drain","context":{"idset":"39-40","reason":"PY: updating to latest nC and BIOS firmware","overwrite":1}}
+{"timestamp":1704404317.2961793,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1704405155.0651531,"name":"online","context":{"idset":"40"}}
+{"timestamp":1704416839.0757353,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1704416839.1758513,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1704417227.0403957,"name":"drain","context":{"idset":"4","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704417349.0780368,"name":"drain","context":{"idset":"4","reason":"epilog failed for jobid fpFkz2WCi6s","overwrite":0}}
+{"timestamp":1704417349.1960282,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1704418063.8032138,"name":"online","context":{"idset":"9"}}
+{"timestamp":1704418071.4334302,"name":"online","context":{"idset":"29"}}
+{"timestamp":1704418162.6154728,"name":"undrain","context":{"idset":"9,29"}}
+{"timestamp":1704423012.176228,"name":"online","context":{"idset":"39"}}
+{"timestamp":1704425017.4079537,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1704425440.9206297,"name":"undrain","context":{"idset":"39"}}
+{"timestamp":1704426533.0761485,"name":"online","context":{"idset":"4"}}
+{"timestamp":1704426713.1523278,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1704427045.1984978,"name":"online","context":{"idset":"40"}}
+{"timestamp":1704427084.2816494,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1704427122.3931665,"name":"drain","context":{"idset":"37-38","reason":"PY: updating to latest nC and BIOS firmware","overwrite":0}}
+{"timestamp":1704427145.905529,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1704427146.0056887,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1704429477.4677393,"name":"online","context":{"idset":"37"}}
+{"timestamp":1704429513.9227068,"name":"online","context":{"idset":"38"}}
+{"timestamp":1704429549.5674293,"name":"undrain","context":{"idset":"37-38"}}
+{"timestamp":1704430836.5624995,"name":"drain","context":{"idset":"22","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1704468422.2665482,"name":"drain","context":{"idset":"5","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704468545.0782218,"name":"drain","context":{"idset":"5","reason":"epilog failed for jobid fpZgT6iSV35","overwrite":0}}
+{"timestamp":1704468545.1764069,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1704469025.4350758,"name":"drain","context":{"idset":"6","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704469147.0788727,"name":"drain","context":{"idset":"6","reason":"epilog failed for jobid fpZoH2g6167","overwrite":0}}
+{"timestamp":1704469147.1767488,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1704469500.1771808,"name":"drain","context":{"idset":"11","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704469513.5916858,"name":"drain","context":{"idset":"13","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704469623.078299,"name":"drain","context":{"idset":"11","reason":"epilog failed for jobid fpZv45r8KM9","overwrite":0}}
+{"timestamp":1704469623.1757755,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1704469633.6408656,"name":"drain","context":{"idset":"7","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704469637.0790184,"name":"drain","context":{"idset":"13","reason":"epilog failed for jobid fpZv4Ug3G15","overwrite":0}}
+{"timestamp":1704469637.1766734,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1704469757.0789235,"name":"drain","context":{"idset":"7","reason":"epilog failed for jobid fpZuUkWLtZm","overwrite":0}}
+{"timestamp":1704469757.1773319,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1704470027.4122105,"name":"drain","context":{"idset":"14","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704470150.0259581,"name":"drain","context":{"idset":"14","reason":"epilog failed for jobid fpZxPo39Fyy","overwrite":0}}
+{"timestamp":1704470150.1224692,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1704471051.0191936,"name":"drain","context":{"idset":"16","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704471173.0787227,"name":"drain","context":{"idset":"16","reason":"epilog failed for jobid fpa6mEyMckP","overwrite":0}}
+{"timestamp":1704471173.1764519,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1704471286.2472734,"name":"drain","context":{"idset":"12","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704471388.6405947,"name":"drain","context":{"idset":"15","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704471409.0788207,"name":"drain","context":{"idset":"12","reason":"epilog failed for jobid fpZv4DmiUi3","overwrite":0}}
+{"timestamp":1704471409.1761262,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1704471511.0782049,"name":"drain","context":{"idset":"15","reason":"epilog failed for jobid fpa24fbaBrw","overwrite":0}}
+{"timestamp":1704471511.1758633,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1704473853.7885721,"name":"drain","context":{"idset":"3","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704473977.0787308,"name":"drain","context":{"idset":"3","reason":"epilog failed for jobid fpZDfjGjdCs","overwrite":0}}
+{"timestamp":1704473977.1762962,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1704474347.2599247,"name":"drain","context":{"idset":"10","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1704474471.0783703,"name":"drain","context":{"idset":"10","reason":"epilog failed for jobid fpZv3646KBV","overwrite":0}}
+{"timestamp":1704474471.1761105,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1704485385.7332985,"name":"drain","context":{"idset":"24","reason":"prolog failed for jobid fpoKQ1csvkF","overwrite":0}}
+{"timestamp":1704501657.1763797,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1704502656.8815482,"name":"online","context":{"idset":"24"}}
+{"timestamp":1704502658.2845204,"name":"online","context":{"idset":"5,13"}}
+{"timestamp":1704502658.4946239,"name":"online","context":{"idset":"3,6,10,12,14"}}
+{"timestamp":1704502658.6800616,"name":"online","context":{"idset":"7,16"}}
+{"timestamp":1704502658.8290875,"name":"online","context":{"idset":"15"}}
+{"timestamp":1704502658.9953873,"name":"online","context":{"idset":"11"}}
+{"timestamp":1704502713.4039626,"name":"undrain","context":{"idset":"3,5-7,10-16,24"}}
+{"timestamp":1704502901.1761694,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1704503702.0201564,"name":"online","context":{"idset":"22"}}
+{"timestamp":1704503721.3588314,"name":"undrain","context":{"idset":"22"}}
+{"timestamp":1704504193.9167588,"name":"drain","context":{"idset":"40","reason":"PY: testing newer kernel","overwrite":0}}
+{"timestamp":1704504271.4356518,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1704505251.1893077,"name":"drain","context":{"idset":"37-39","reason":"PY: reboot to newer amdgpu driver to support newer firmware","overwrite":0}}
+{"timestamp":1704505276.6790709,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1704506121.7133114,"name":"online","context":{"idset":"37"}}
+{"timestamp":1704737561.9239538,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1704737562.0239801,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1704737562.1353397,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1704738454.1005588,"name":"online","context":{"idset":"38"}}
+{"timestamp":1704738454.5790424,"name":"online","context":{"idset":"37"}}
+{"timestamp":1704738455.1761711,"name":"online","context":{"idset":"40"}}
+{"timestamp":1704738481.5520215,"name":"undrain","context":{"idset":"37-38,40"}}
+{"timestamp":1704739435.0763011,"name":"online","context":{"idset":"39"}}
+{"timestamp":1704739607.1499445,"name":"undrain","context":{"idset":"39"}}
+{"timestamp":1704832067.0774658,"name":"drain","context":{"idset":"15","reason":"prolog failed for jobid fqajdhPuzsh","overwrite":0}}
+{"timestamp":1704841351.3610871,"name":"drain","context":{"idset":"16","reason":"prolog failed for jobid fqbxBDtY7ou","overwrite":0}}
+{"timestamp":1704848753.1757495,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1704848755.1758671,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1704852175.7796268,"name":"undrain","context":{"idset":"15-16"}}
+{"timestamp":1704861939.8878314,"name":"drain","context":{"idset":"8","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1704921560.0937643,"name":"online","context":{"idset":"15"}}
+{"timestamp":1704921563.3839161,"name":"online","context":{"idset":"16"}}
+{"timestamp":1704923789.8260546,"name":"drain","context":{"idset":"15-16","reason":"reboot","overwrite":0}}
+{"timestamp":1704923947.0763023,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1704923947.1765828,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1704926813.0760851,"name":"online","context":{"idset":"16"}}
+{"timestamp":1704926813.1762075,"name":"online","context":{"idset":"15"}}
+{"timestamp":1704926832.9978969,"name":"undrain","context":{"idset":"15-16"}}
+{"timestamp":1704927013.1756945,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1704927791.0760164,"name":"online","context":{"idset":"8"}}
+{"timestamp":1704927799.7545106,"name":"undrain","context":{"idset":"8"}}
+{"timestamp":1705077719.1773093,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1705099364.3870585,"name":"drain","context":{"idset":"15","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1705099364.4231994,"name":"drain","context":{"idset":"18","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1705100168.2664351,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1705100169.2358968,"name":"online","context":{"idset":"2"}}
+{"timestamp":1705100282.3923693,"name":"drain","context":{"idset":"38","overwrite":0}}
+{"timestamp":1705100431.0763104,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1705100431.0768092,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1705100431.1764483,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1705100870.3264933,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1705101281.30984,"name":"online","context":{"idset":"15,18"}}
+{"timestamp":1705101750.0339606,"name":"undrain","context":{"idset":"15,18"}}
+{"timestamp":1705102791.5190539,"name":"online","context":{"idset":"2"}}
+{"timestamp":1705102893.3078043,"name":"online","context":{"idset":"38"}}
+{"timestamp":1705102918.7926245,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1705102922.4910748,"name":"undrain","context":{"idset":"38"}}
+{"timestamp":1705131853.7059011,"name":"drain","context":{"idset":"33-40","reason":"reboot","overwrite":0}}
+{"timestamp":1705132099.0769536,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1705132099.0773089,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1705132099.0776746,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1705132099.0780151,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1705132099.0783367,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1705132099.0786636,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1705132099.0789673,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1705132099.1764219,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1705133395.9693029,"name":"online","context":{"idset":"35"}}
+{"timestamp":1705133396.3905811,"name":"online","context":{"idset":"33,36"}}
+{"timestamp":1705133396.7917016,"name":"online","context":{"idset":"34"}}
+{"timestamp":1705133437.5936582,"name":"undrain","context":{"idset":"33-36"}}
+{"timestamp":1705133528.6349225,"name":"online","context":{"idset":"37"}}
+{"timestamp":1705133529.3426721,"name":"online","context":{"idset":"39"}}
+{"timestamp":1705133529.7742326,"name":"online","context":{"idset":"38"}}
+{"timestamp":1705133565.0310783,"name":"undrain","context":{"idset":"37-39"}}
+{"timestamp":1705134797.1761992,"name":"online","context":{"idset":"40"}}
+{"timestamp":1705134811.707335,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1705417902.2889421,"name":"drain","context":{"idset":"3-40","reason":"power work","overwrite":0}}
+{"timestamp":1705418061.0787849,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1705418061.0793388,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1705418061.0796938,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1705418061.0800562,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1705418061.0803843,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1705418061.0807362,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1705418061.0810475,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1705418061.0813479,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1705418061.0816615,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1705418061.0819526,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1705418061.0822358,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1705418061.0825224,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1705418061.0827954,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1705418061.083056,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1705418061.0833127,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1705418061.0835712,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1705418061.083822,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1705418061.0840566,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1705418061.0842865,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1705418061.0845075,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1705418061.0847449,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1705418061.0849557,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1705418061.0851636,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1705418061.0853581,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1705418061.085562,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1705418061.085746,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1705418061.1766455,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1705418063.0768652,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1705418063.0770783,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1705418063.07725,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1705418063.0774062,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1705418063.0775721,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1705418063.0777025,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1705418063.0778315,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1705418063.077975,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1705418063.0780902,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1705418063.0781946,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1705418063.1767712,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1705418533.0762556,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1705418533.1766021,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1705418595.076611,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1705418595.1772313,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1705454585.9442623,"name":"online","context":{"idset":"33"}}
+{"timestamp":1705454586.0932512,"name":"online","context":{"idset":"35"}}
+{"timestamp":1705454587.1512265,"name":"online","context":{"idset":"18"}}
+{"timestamp":1705454587.3003485,"name":"online","context":{"idset":"20"}}
+{"timestamp":1705454587.6638947,"name":"online","context":{"idset":"19"}}
+{"timestamp":1705454587.8148596,"name":"online","context":{"idset":"23,31,39"}}
+{"timestamp":1705454587.9441233,"name":"online","context":{"idset":"26"}}
+{"timestamp":1705454588.0514374,"name":"online","context":{"idset":"17,25,28,32"}}
+{"timestamp":1705454588.2118852,"name":"online","context":{"idset":"27,29-30,37"}}
+{"timestamp":1705454588.4454341,"name":"online","context":{"idset":"24"}}
+{"timestamp":1705454588.5579503,"name":"online","context":{"idset":"22"}}
+{"timestamp":1705454627.7267556,"name":"undrain","context":{"idset":"17-20,22-33,35,37,39"}}
+{"timestamp":1705456370.3285453,"name":"online","context":{"idset":"21"}}
+{"timestamp":1705456520.1144907,"name":"drain","context":{"idset":"33-40","reason":"reboot","overwrite":1}}
+{"timestamp":1705457105.0766304,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1705457105.0770462,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1705457105.0772684,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1705457105.1770749,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1705458561.5621066,"name":"drain","context":{"idset":"17-32","reason":"reboot","overwrite":1}}
+{"timestamp":1705458711.0777566,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1705458711.078239,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1705458711.0785036,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1705458711.078764,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1705458711.0790155,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1705458711.0792651,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1705458711.0794911,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1705458711.0797012,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1705458711.0799112,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1705458711.0801051,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1705458711.0802944,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1705458711.080457,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1705458711.0806341,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1705458711.0807846,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1705458711.0809221,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1705458711.1761711,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1705459605.3949301,"name":"online","context":{"idset":"2"}}
+{"timestamp":1705459606.1314061,"name":"online","context":{"idset":"1"}}
+{"timestamp":1705459974.4402516,"name":"online","context":{"idset":"24"}}
+{"timestamp":1705459975.88131,"name":"online","context":{"idset":"4"}}
+{"timestamp":1705459975.9931026,"name":"online","context":{"idset":"10"}}
+{"timestamp":1705459976.0196617,"name":"online","context":{"idset":"9"}}
+{"timestamp":1705459976.3268795,"name":"online","context":{"idset":"7"}}
+{"timestamp":1705459976.4507449,"name":"online","context":{"idset":"20"}}
+{"timestamp":1705459976.5556335,"name":"online","context":{"idset":"3"}}
+{"timestamp":1705459976.6223674,"name":"online","context":{"idset":"31"}}
+{"timestamp":1705459976.7330828,"name":"online","context":{"idset":"8,13,35"}}
+{"timestamp":1705459976.9030261,"name":"online","context":{"idset":"29,36"}}
+{"timestamp":1705459977.0292499,"name":"online","context":{"idset":"6,25"}}
+{"timestamp":1705459977.0770373,"name":"online","context":{"idset":"12,34"}}
+{"timestamp":1705459977.1764545,"name":"online","context":{"idset":"15-16,19,26,33"}}
+{"timestamp":1705459977.3156424,"name":"online","context":{"idset":"11,22-23,32"}}
+{"timestamp":1705459977.4602265,"name":"online","context":{"idset":"28,39"}}
+{"timestamp":1705459977.5880234,"name":"online","context":{"idset":"17-18,21,30"}}
+{"timestamp":1705459977.718024,"name":"online","context":{"idset":"27"}}
+{"timestamp":1705460084.7650936,"name":"undrain","context":{"idset":"3-4,6-13,15-36,39"}}
+{"timestamp":1705460116.4266584,"name":"undrain","context":{"idset":"1-2"}}
+{"timestamp":1705460275.2885256,"name":"online","context":{"idset":"40"}}
+{"timestamp":1705460275.8773069,"name":"online","context":{"idset":"38"}}
+{"timestamp":1705460293.4556468,"name":"undrain","context":{"idset":"38,40"}}
+{"timestamp":1705460586.9204471,"name":"online","context":{"idset":"5"}}
+{"timestamp":1705460587.5093296,"name":"online","context":{"idset":"14"}}
+{"timestamp":1705460617.3798587,"name":"undrain","context":{"idset":"5,14"}}
+{"timestamp":1705461107.071501,"name":"online","context":{"idset":"37"}}
+{"timestamp":1705461125.0405931,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1705461630.4973726,"name":"drain","context":{"idset":"37-38","overwrite":0}}
+{"timestamp":1705461783.1763082,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1705461785.1771924,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1705462594.094785,"name":"online","context":{"idset":"37"}}
+{"timestamp":1705462594.7651672,"name":"online","context":{"idset":"38"}}
+{"timestamp":1705462749.0762894,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1705462749.1764178,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1705462856.9122601,"name":"drain","context":{"idset":"37-38","reason":"issues","overwrite":1}}
+{"timestamp":1705463771.4986887,"name":"online","context":{"idset":"37"}}
+{"timestamp":1705463772.1375372,"name":"online","context":{"idset":"38"}}
+{"timestamp":1705505789.1767933,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1705505791.1769941,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1705505793.0751381,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1705505793.0752432,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1705505793.1758785,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1705505795.0767117,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1705505795.0768142,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1705505795.0768709,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1705505795.0769191,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1705505795.0769596,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1705505795.077004,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1705505795.1767178,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1705505855.076185,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1705505855.175977,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1705505857.0768919,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1705505857.0772853,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1705505857.0777867,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1705505857.0782986,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1705505857.0788043,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1705505857.0792863,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1705505857.0797651,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1705505857.0802295,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1705505857.0806859,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1705505857.1764579,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1705505873.1771455,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1705505875.0768461,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1705505875.0769351,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1705505875.0769789,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1705505875.0770183,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1705505875.0770574,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1705505875.0771003,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1705505875.1773081,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1705505879.0764294,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1705505879.0765405,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1705505879.0765891,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1705505879.177242,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1705505941.0766764,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1705505941.077009,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1705505941.077394,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1705505941.0777779,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1705505941.0781138,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1705505941.0784523,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1705505941.0788333,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1705505941.0791879,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1705505941.0795398,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1705505941.0798664,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1705505941.080184,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1705505941.1763687,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1705508165.0749257,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1705508165.1749942,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1705508167.1767666,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1705508227.0791981,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1705508227.1763835,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1705508229.1766996,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1705509072.7750583,"name":"online","context":{"idset":"36"}}
+{"timestamp":1705509072.9799345,"name":"online","context":{"idset":"34"}}
+{"timestamp":1705509074.3846283,"name":"online","context":{"idset":"10"}}
+{"timestamp":1705509074.7569745,"name":"online","context":{"idset":"8,15"}}
+{"timestamp":1705509074.8913467,"name":"online","context":{"idset":"5,12,14"}}
+{"timestamp":1705509074.9961476,"name":"online","context":{"idset":"6-7,9,16"}}
+{"timestamp":1705509075.0765879,"name":"online","context":{"idset":"13"}}
+{"timestamp":1705509075.2873397,"name":"online","context":{"idset":"11"}}
+{"timestamp":1705509075.7214129,"name":"online","context":{"idset":"32"}}
+{"timestamp":1705509076.6215942,"name":"online","context":{"idset":"18"}}
+{"timestamp":1705509077.0103676,"name":"online","context":{"idset":"23"}}
+{"timestamp":1705509077.4727845,"name":"online","context":{"idset":"27"}}
+{"timestamp":1705509077.5832779,"name":"online","context":{"idset":"21,24,29-31"}}
+{"timestamp":1705509077.8083739,"name":"online","context":{"idset":"17"}}
+{"timestamp":1705509194.4199202,"name":"undrain","context":{"idset":"5-18,21,23-24,27,29-32,34,36"}}
+{"timestamp":1705515448.4612179,"name":"online","context":{"idset":"3"}}
+{"timestamp":1705515448.7505858,"name":"online","context":{"idset":"4"}}
+{"timestamp":1705515486.1279497,"name":"undrain","context":{"idset":"3-4"}}
+{"timestamp":1705515741.0764961,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1705515741.1762512,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1705516042.4868581,"name":"online","context":{"idset":"2"}}
+{"timestamp":1705516089.4515889,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1705517353.1759307,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1705517415.1767612,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1705519332.3071222,"name":"online","context":{"idset":"37"}}
+{"timestamp":1705520131.6543047,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1705527069.4102743,"name":"drain","context":{"idset":"8","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1705527982.8316302,"name":"online","context":{"idset":"21"}}
+{"timestamp":1705528521.1774228,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1705533037.3725274,"name":"online","context":{"idset":"8"}}
+{"timestamp":1705533097.2585104,"name":"drain","context":{"idset":"8,21","reason":"lustre","overwrite":1}}
+{"timestamp":1705533101.0842159,"name":"drain","context":{"idset":"8,21","reason":"rzlustre1","overwrite":1}}
+{"timestamp":1705533235.2193933,"name":"drain","context":{"idset":"33-38","reason":"reboot","overwrite":1}}
+{"timestamp":1705533395.0768631,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1705533395.0772877,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1705533395.0776651,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1705533395.0780051,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1705533395.1763148,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1705535672.0477474,"name":"online","context":{"idset":"33-35"}}
+{"timestamp":1705535672.3032405,"name":"online","context":{"idset":"36"}}
+{"timestamp":1705535689.3250225,"name":"drain","context":{"idset":"33-36","reason":"rzlustre1","overwrite":1}}
+{"timestamp":1705536036.030407,"name":"drain","context":{"idset":"37-38","reason":"rzlustre1","overwrite":1}}
+{"timestamp":1705536366.7492385,"name":"online","context":{"idset":"37"}}
+{"timestamp":1705536367.6604602,"name":"online","context":{"idset":"38"}}
+{"timestamp":1705541994.7175405,"name":"undrain","context":{"idset":"8,21,33-38"}}
+{"timestamp":1705690655.1770999,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1705690716.176085,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1705696817.321147,"name":"drain","context":{"idset":"7","reason":"prolog failed for jobid fsY2NqJQxpw","overwrite":0}}
+{"timestamp":1705702495.1467571,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1705702568.9118104,"name":"drain","context":{"idset":"3-6,8-16","reason":"reboot","overwrite":0}}
+{"timestamp":1705703096.5585637,"name":"undrain","context":{"idset":"3-6,8-16"}}
+{"timestamp":1705703198.9350264,"name":"online","context":{"idset":"7"}}
+{"timestamp":1705703210.0724232,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1705703218.6942365,"name":"drain","context":{"idset":"18","reason":"reboot","overwrite":0}}
+{"timestamp":1705703293.5811298,"name":"online","context":{"idset":"37"}}
+{"timestamp":1705703379.1771948,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1705703419.0854983,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1705704070.1505249,"name":"online","context":{"idset":"18"}}
+{"timestamp":1705704094.202615,"name":"undrain","context":{"idset":"18"}}
+{"timestamp":1705945397.2033477,"name":"drain","context":{"idset":"10-16","reason":"reboot","overwrite":0}}
+{"timestamp":1705945547.076607,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1705945547.0786419,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1705945547.0790009,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1705945547.0793445,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1705945547.0797,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1705945547.0800416,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1705945547.1760995,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1705945671.6312306,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1705945671.733875,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1705946404.1911609,"name":"online","context":{"idset":"10"}}
+{"timestamp":1705946404.3999074,"name":"online","context":{"idset":"16"}}
+{"timestamp":1705946404.5257316,"name":"online","context":{"idset":"14"}}
+{"timestamp":1705946404.8332956,"name":"online","context":{"idset":"12,15"}}
+{"timestamp":1705946551.207793,"name":"undrain","context":{"idset":"10,12,14-16"}}
+{"timestamp":1705946625.7682307,"name":"online","context":{"idset":"1-2"}}
+{"timestamp":1705946864.4133422,"name":"drain","context":{"idset":"33-36","overwrite":0}}
+{"timestamp":1705946873.99635,"name":"drain","context":{"idset":"33-36","reason":"reboot","overwrite":1}}
+{"timestamp":1705947042.8566463,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1705947043.9258516,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1705947043.9263005,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1705947044.0255835,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1705947145.1763422,"name":"online","context":{"idset":"11"}}
+{"timestamp":1705947145.5614455,"name":"online","context":{"idset":"13"}}
+{"timestamp":1705947194.3063715,"name":"undrain","context":{"idset":"11,13"}}
+{"timestamp":1705947218.559588,"name":"drain","context":{"idset":"3-6,8-9","reason":"reboot","overwrite":0}}
+{"timestamp":1705947383.1758711,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1705947774.4943244,"name":"online","context":{"idset":"33"}}
+{"timestamp":1705947774.8092341,"name":"online","context":{"idset":"36"}}
+{"timestamp":1705947774.9654417,"name":"online","context":{"idset":"34-35"}}
+{"timestamp":1705947798.7687445,"name":"undrain","context":{"idset":"33-36"}}
+{"timestamp":1705948441.176707,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1705948951.061954,"name":"online","context":{"idset":"5"}}
+{"timestamp":1705948968.7197225,"name":"undrain","context":{"idset":"5"}}
+{"timestamp":1705949116.2231996,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1705949141.0521028,"name":"online","context":{"idset":"9"}}
+{"timestamp":1705949164.5613205,"name":"undrain","context":{"idset":"9"}}
+{"timestamp":1705949765.4823706,"name":"online","context":{"idset":"6"}}
+{"timestamp":1705949785.5363228,"name":"undrain","context":{"idset":"6"}}
+{"timestamp":1705963345.0764065,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1705963345.0768015,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1705963345.176568,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1705964656.5190966,"name":"online","context":{"idset":"3"}}
+{"timestamp":1705964657.1766195,"name":"online","context":{"idset":"4"}}
+{"timestamp":1705964657.4742959,"name":"online","context":{"idset":"8"}}
+{"timestamp":1705964791.8943598,"name":"undrain","context":{"idset":"3-4,8"}}
+{"timestamp":1706112906.7013509,"name":"drain","context":{"idset":"39","reason":"PY: rebooting for rabbit disk hang","overwrite":0}}
+{"timestamp":1706112981.176795,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1706113047.0764794,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1706113047.1769791,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1706114053.8999605,"name":"online","context":{"idset":"39"}}
+{"timestamp":1706128850.0062456,"name":"online","context":{"idset":"6"}}
+{"timestamp":1706128884.2961507,"name":"undrain","context":{"idset":"6,39"}}
+{"timestamp":1706206451.6634552,"name":"drain","context":{"idset":"11","reason":"prolog failed for jobid ftgnoLS7CvK","overwrite":0}}
+{"timestamp":1706209481.1769612,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1706209543.176717,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1706210266.9565492,"name":"online","context":{"idset":"40"}}
+{"timestamp":1706212141.3145847,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1706231588.639426,"name":"drain","context":{"idset":"16","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1706280579.1767466,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1706280645.1761458,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1706281376.2264013,"name":"online","context":{"idset":"40"}}
+{"timestamp":1706282321.6449487,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1706282397.1771758,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1706282409.1769855,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1706282459.176713,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1706282471.1763806,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1706283383.9041438,"name":"online","context":{"idset":"40"}}
+{"timestamp":1706286632.0476611,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1706287593.5105553,"name":"online","context":{"idset":"39"}}
+{"timestamp":1706289115.0758376,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1706289115.0761757,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1706289115.1763816,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1706289853.9267108,"name":"online","context":{"idset":"11"}}
+{"timestamp":1706289854.1598382,"name":"online","context":{"idset":"16"}}
+{"timestamp":1706289871.7308803,"name":"undrain","context":{"idset":"11,16"}}
+{"timestamp":1706290600.4223185,"name":"online","context":{"idset":"39"}}
+{"timestamp":1706290653.1102376,"name":"undrain","context":{"idset":"39"}}
+{"timestamp":1706580010.3903203,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1706581206.3307719,"name":"online","context":{"idset":"38"}}
+{"timestamp":1706641589.1762068,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1706641650.4750149,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1706642387.1767986,"name":"online","context":{"idset":"38"}}
+{"timestamp":1706643749.7222111,"name":"undrain","context":{"idset":"38"}}
+{"timestamp":1706720842.1346092,"name":"drain","context":{"idset":"3","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706720843.0839674,"name":"drain","context":{"idset":"4","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706721075.9347234,"name":"drain","context":{"idset":"5-40","reason":"toss_update","overwrite":0}}
+{"timestamp":1706721529.0761945,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1706721529.076376,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1706721589.0783482,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1706721589.0788786,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1706721589.0792286,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1706721589.0795765,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1706721589.0799077,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1706721589.0802236,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1706721589.0805423,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1706721589.0808475,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1706721589.0811434,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1706721589.0814302,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1706721589.0817316,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1706721589.0820143,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1706721589.0822821,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1706721589.0825558,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1706721589.0828197,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1706721589.0830755,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1706721589.0833256,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1706721589.1766827,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1706721591.0776072,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1706721591.077862,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1706721591.0780888,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1706721591.0783007,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1706721591.0785105,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1706721591.0787234,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1706721591.0789313,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1706721591.0791116,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1706721591.0792894,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1706721591.0794642,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1706721591.0796378,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1706721591.0797884,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1706721591.0799332,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1706721591.0800743,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1706721591.0802205,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1706721591.0803452,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1706721591.0804725,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1706721591.0805891,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1706721591.0807006,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1706721591.0808012,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1706721591.0808949,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1706721591.1763742,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1706728720.5130961,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1706720842.1346092,"reason":"nodediag failed cxi"},"4":{"timestamp":1706720843.0839674,"reason":"nodediag failed cxi"},"5-40":{"timestamp":1706721075.9347234,"reason":"toss_update"},"41-42":{"timestamp":1703007506.748992,"reason":"PY: A0 blade not installed yet"}},"online":"","exclude":"0-2"}}
+{"timestamp":1706728720.5247622,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1706728847.3685839,"name":"online","context":{"idset":"0"}}
+{"timestamp":1706745364.2693398,"name":"online","context":{"idset":"36"}}
+{"timestamp":1706745364.7345064,"name":"online","context":{"idset":"35"}}
+{"timestamp":1706745366.4553585,"name":"online","context":{"idset":"10"}}
+{"timestamp":1706745366.73332,"name":"online","context":{"idset":"4"}}
+{"timestamp":1706745367.1881649,"name":"online","context":{"idset":"16"}}
+{"timestamp":1706745367.4764736,"name":"online","context":{"idset":"15"}}
+{"timestamp":1706745367.5967507,"name":"online","context":{"idset":"5,8"}}
+{"timestamp":1706745367.7194331,"name":"online","context":{"idset":"7"}}
+{"timestamp":1706745367.8382893,"name":"online","context":{"idset":"6"}}
+{"timestamp":1706745370.5187676,"name":"online","context":{"idset":"9"}}
+{"timestamp":1706745385.0182507,"name":"online","context":{"idset":"33"}}
+{"timestamp":1706745795.8052304,"name":"drain","context":{"idset":"4-10,15-16,33,35-36","reason":"good","overwrite":1}}
+{"timestamp":1706747333.1057575,"name":"online","context":{"idset":"17,24"}}
+{"timestamp":1706747333.4820509,"name":"online","context":{"idset":"29"}}
+{"timestamp":1706747333.6125054,"name":"online","context":{"idset":"22,32"}}
+{"timestamp":1706747334.2185085,"name":"online","context":{"idset":"11"}}
+{"timestamp":1706747334.7962711,"name":"online","context":{"idset":"3,12"}}
+{"timestamp":1706747785.9179277,"name":"online","context":{"idset":"23"}}
+{"timestamp":1706747827.5908408,"name":"drain","context":{"idset":"3,11-12,17,22-24,27-29,32","reason":"good","overwrite":1}}
+{"timestamp":1706747964.351794,"name":"online","context":{"idset":"27"}}
+{"timestamp":1706748287.4043443,"name":"online","context":{"idset":"28"}}
+{"timestamp":1706748536.9759698,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1706748536.9769161,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1706748536.9779179,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1706748536.9790306,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1706748536.9793239,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1706748536.9800203,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1706748536.980602,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1706748536.9809282,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1706748536.9812248,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1706748536.9815211,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1706748536.9817886,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1706748536.9820507,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1706748536.9822276,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1706748536.9824383,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1706748536.9826577,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1706748536.9828157,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1706748536.9830136,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1706748536.9831991,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1706748536.9833782,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1706748536.983783,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1706748536.9839201,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1706748536.9849381,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1706748537.084136,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1706748593.5542626,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1706720842.1346092,"reason":"good"},"4":{"timestamp":1706720843.0839674,"reason":"good"},"5-12,15-17,22-24,27-29,32-33,35-36":{"timestamp":1706721075.9347234,"reason":"good"},"13-14,18-21,25-26,30-31,34,37-40":{"timestamp":1706721075.9347234,"reason":"toss_update"},"41-42":{"timestamp":1703007506.748992,"reason":"PY: A0 blade not installed yet"}},"online":"","exclude":"0-2"}}
+{"timestamp":1706748593.5650852,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1706748719.048049,"name":"online","context":{"idset":"0"}}
+{"timestamp":1706748721.3197691,"name":"online","context":{"idset":"33,35-36"}}
+{"timestamp":1706748722.302475,"name":"online","context":{"idset":"17,22-24,27-29,32"}}
+{"timestamp":1706748723.5637898,"name":"online","context":{"idset":"9-12,15-16"}}
+{"timestamp":1706748723.7059767,"name":"online","context":{"idset":"3-8"}}
+{"timestamp":1706748774.3453927,"name":"online","context":{"idset":"1"}}
+{"timestamp":1706748775.2264295,"name":"online","context":{"idset":"2"}}
+{"timestamp":1706748775.2815068,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1706748775.3821931,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1706748776.1876004,"name":"online","context":{"idset":"1"}}
+{"timestamp":1706748777.024087,"name":"online","context":{"idset":"2"}}
+{"timestamp":1706749084.7990324,"name":"drain","context":{"idset":"37-40","reason":"missing","overwrite":1}}
+{"timestamp":1706749637.6373034,"name":"online","context":{"idset":"26"}}
+{"timestamp":1706749648.7229364,"name":"drain","context":{"idset":"26","reason":"good","overwrite":1}}
+{"timestamp":1706749672.0298612,"name":"drain","context":{"idset":"25","reason":"booting issue","overwrite":1}}
+{"timestamp":1706749677.7778151,"name":"drain","context":{"idset":"25","reason":"power issue","overwrite":1}}
+{"timestamp":1706749767.6410849,"name":"undrain","context":{"idset":"3-12,15-17,22-24,26-29,32-33,35-36"}}
+{"timestamp":1706750344.3442359,"name":"online","context":{"idset":"18"}}
+{"timestamp":1706750344.9971077,"name":"online","context":{"idset":"20,31"}}
+{"timestamp":1706750345.3029773,"name":"online","context":{"idset":"30"}}
+{"timestamp":1706750346.7780282,"name":"online","context":{"idset":"13"}}
+{"timestamp":1706750514.8563862,"name":"undrain","context":{"idset":"13,18,20,30-31"}}
+{"timestamp":1706750666.9721839,"name":"drain","context":{"idset":"14,21,34","reason":"amdgpu","overwrite":1}}
+{"timestamp":1706750729.8045704,"name":"drain","context":{"idset":"19","reason":"slingshot","overwrite":1}}
+{"timestamp":1706750774.8932064,"name":"drain","context":{"idset":"25","reason":"power","overwrite":1}}
+{"timestamp":1706769164.2996287,"name":"drain","context":{"idset":"26","reason":"power","overwrite":0}}
+{"timestamp":1706769393.3319452,"name":"drain","context":{"idset":"13,22,33","reason":"amdgpu","overwrite":0}}
+{"timestamp":1706769425.8386352,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1706769681.7382486,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1706769681.7386494,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1706769681.8380325,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1706770995.3408291,"name":"online","context":{"idset":"19"}}
+{"timestamp":1706771093.6684139,"name":"undrain","context":{"idset":"19"}}
+{"timestamp":1706772029.0987186,"name":"online","context":{"idset":"33"}}
+{"timestamp":1706772029.4746823,"name":"online","context":{"idset":"34"}}
+{"timestamp":1706772029.6453948,"name":"online","context":{"idset":"26"}}
+{"timestamp":1706772030.2231698,"name":"online","context":{"idset":"21"}}
+{"timestamp":1706772030.4770749,"name":"online","context":{"idset":"22"}}
+{"timestamp":1706772030.6101782,"name":"online","context":{"idset":"25"}}
+{"timestamp":1706772031.9725184,"name":"online","context":{"idset":"13"}}
+{"timestamp":1706772032.3089271,"name":"online","context":{"idset":"14"}}
+{"timestamp":1706772050.5552862,"name":"undrain","context":{"idset":"13-14,21-22,25-26,33-34"}}
+{"timestamp":1706776123.3806469,"name":"online","context":{"idset":"42"}}
+{"timestamp":1706776801.838424,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1706776908.258662,"name":"drain","context":{"idset":"37-38","reason":"power","overwrite":1}}
+{"timestamp":1706777647.09671,"name":"online","context":{"idset":"42"}}
+{"timestamp":1706777649.0804358,"name":"online","context":{"idset":"41"}}
+{"timestamp":1706777650.6052032,"name":"online","context":{"idset":"39"}}
+{"timestamp":1706777670.2255673,"name":"online","context":{"idset":"40"}}
+{"timestamp":1706777757.3656499,"name":"undrain","context":{"idset":"39-42"}}
+{"timestamp":1706811878.4463639,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1706814751.1820986,"name":"online","context":{"idset":"1"}}
+{"timestamp":1706815529.7124064,"name":"undrain","context":{"idset":"37-38"}}
+{"timestamp":1706815597.7502842,"name":"drain","context":{"idset":"37-38","reason":"Cassini issue --JRG","overwrite":0}}
+{"timestamp":1706815699.8314655,"name":"online","context":{"idset":"38"}}
+{"timestamp":1706815700.6026566,"name":"online","context":{"idset":"37"}}
+{"timestamp":1706815717.8863459,"name":"undrain","context":{"idset":"37-38"}}
+{"timestamp":1706819865.5647526,"name":"drain","context":{"idset":"38","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706820552.918191,"name":"drain","context":{"idset":"37","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1706835991.8379133,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1706836053.8388202,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1706836863.1143255,"name":"online","context":{"idset":"39"}}
+{"timestamp":1706837425.0361998,"name":"undrain","context":{"idset":"39"}}
+{"timestamp":1706845727.7374389,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1706845727.8374994,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1706847508.1874907,"name":"online","context":{"idset":"37"}}
+{"timestamp":1706847541.7185102,"name":"online","context":{"idset":"38"}}
+{"timestamp":1706847692.2123916,"name":"undrain","context":{"idset":"37-38"}}
+{"timestamp":1707108746.0045073,"name":"drain","context":{"idset":"7","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1707153041.8383341,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1707162881.4573922,"name":"online","context":{"idset":"7"}}
+{"timestamp":1707164377.96945,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1707181063.4005086,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1707182419.0137868,"name":"online","context":{"idset":"41"}}
+{"timestamp":1707233575.8383245,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1707233637.7587218,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1707234478.4798162,"name":"online","context":{"idset":"37"}}
+{"timestamp":1707235135.8375456,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1707235197.1114519,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1707236019.1379948,"name":"online","context":{"idset":"38"}}
+{"timestamp":1707239279.7997193,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1707242450.5040295,"name":"online","context":{"idset":"1"}}
+{"timestamp":1707242881.8376813,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1707243712.6798022,"name":"online","context":{"idset":"37"}}
+{"timestamp":1707243865.837508,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1707244693.2677257,"name":"online","context":{"idset":"38"}}
+{"timestamp":1707246835.8380017,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1707247712.0254409,"name":"online","context":{"idset":"37"}}
+{"timestamp":1707251877.8387382,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1707251943.8383164,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1707252739.5220263,"name":"online","context":{"idset":"1"}}
+{"timestamp":1707281571.2528114,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1707291535.9030697,"name":"undrain","context":{"idset":"1"}}
+{"timestamp":1707291829.405098,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1707292331.3300004,"name":"drain","context":{"idset":"37","reason":"PY: BIOS updated to 0.2.2","overwrite":1}}
+{"timestamp":1707321045.0225461,"name":"online","context":{"idset":"37"}}
+{"timestamp":1707321118.1157029,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1707326499.3476722,"name":"drain","context":{"idset":"37","reason":"nodediag failed dmi","overwrite":0}}
+{"timestamp":1707337626.9213357,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1707344731.8382325,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1707344792.9465225,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1707352482.5177982,"name":"online","context":{"idset":"37"}}
+{"timestamp":1707352524.419502,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1707371679.5023351,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1707371679.6025698,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1707371679.9682324,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1707371680.0894804,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1707371680.2077699,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1707373046.9836395,"name":"online","context":{"idset":"42"}}
+{"timestamp":1707373065.2995386,"name":"online","context":{"idset":"38"}}
+{"timestamp":1707373068.8663907,"name":"online","context":{"idset":"40"}}
+{"timestamp":1707373080.0197823,"name":"online","context":{"idset":"39"}}
+{"timestamp":1707373235.2243419,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1707373323.8380773,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1707373389.8384104,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1707374418.8318408,"name":"online","context":{"idset":"41"}}
+{"timestamp":1707374554.2347,"name":"online","context":{"idset":"6"}}
+{"timestamp":1707374642.895611,"name":"undrain","context":{"idset":"6,38"}}
+{"timestamp":1707374890.1903248,"name":"drain","context":{"idset":"42","reason":"HPE: clocksource is wrong","overwrite":1}}
+{"timestamp":1707403283.3868968,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1707423359.4330695,"name":"online","context":{"idset":"3"}}
+{"timestamp":1707423395.7569091,"name":"drain","context":{"idset":"28","reason":"reboot","overwrite":0}}
+{"timestamp":1707423603.7411213,"name":"drain","context":{"idset":"28","reason":"prolog failed for jobid fw9uVXa94oR","overwrite":0}}
+{"timestamp":1707423603.8385482,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1707424044.1677125,"name":"drain","context":{"idset":"23","reason":"reboot","overwrite":0}}
+{"timestamp":1707424189.741046,"name":"drain","context":{"idset":"23","reason":"prolog failed for jobid fw4gQZNLXUb","overwrite":0}}
+{"timestamp":1707424189.8380446,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1707424337.837878,"name":"online","context":{"idset":"28"}}
+{"timestamp":1707424352.8584485,"name":"undrain","context":{"idset":"23"}}
+{"timestamp":1707424491.0924864,"name":"online","context":{"idset":"42"}}
+{"timestamp":1707424568.3163893,"name":"undrain","context":{"idset":"42"}}
+{"timestamp":1707424679.9889822,"name":"drain","context":{"idset":"14","reason":"reboot","overwrite":0}}
+{"timestamp":1707424698.0068271,"name":"undrain","context":{"idset":"28"}}
+{"timestamp":1707424809.838382,"name":"drain","context":{"idset":"23","reason":"reboot","overwrite":0}}
+{"timestamp":1707424872.6790259,"name":"online","context":{"idset":"23"}}
+{"timestamp":1707424884.4999027,"name":"undrain","context":{"idset":"23"}}
+{"timestamp":1707424921.5031865,"name":"undrain","context":{"idset":"14"}}
+{"timestamp":1707437851.3219905,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1707438674.136807,"name":"online","context":{"idset":"1"}}
+{"timestamp":1707454355.6548223,"name":"drain","context":{"idset":"13","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1707496524.0364501,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1707497971.2689891,"name":"online","context":{"idset":"1"}}
+{"timestamp":1707518163.8375447,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1707518224.4281309,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1707527387.8385627,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1707527449.3115797,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1707713664.2276361,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1707768453.8382242,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1707768515.4643099,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1707769288.1925364,"name":"online","context":{"idset":"40"}}
+{"timestamp":1707769763.5749722,"name":"undrain","context":{"idset":"40"}}
+{"timestamp":1707776304.3889184,"name":"online","context":{"idset":"37"}}
+{"timestamp":1707776828.1324644,"name":"online","context":{"idset":"39"}}
+{"timestamp":1707777151.9349496,"name":"undrain","context":{"idset":"37,39"}}
+{"timestamp":1707792783.2998366,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1707792783.4003525,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1707795595.239007,"name":"online","context":{"idset":"6"}}
+{"timestamp":1707795595.4831834,"name":"online","context":{"idset":"13"}}
+{"timestamp":1707799880.0881984,"name":"drain","context":{"idset":"16","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1707818410.8097675,"name":"undrain","context":{"idset":"6,13"}}
+{"timestamp":1707850381.8391364,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1707850445.8383119,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1707851273.8377619,"name":"online","context":{"idset":"37"}}
+{"timestamp":1707851630.2794707,"name":"undrain","context":{"idset":"37"}}
+{"timestamp":1707860787.8375132,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1707861914.3243656,"name":"online","context":{"idset":"16"}}
+{"timestamp":1707862299.8390322,"name":"drain","context":{"idset":"42","reason":"broker was unresponsive"}}
+{"timestamp":1707862365.8380234,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1707862432.2866926,"name":"undrain","context":{"idset":"16"}}
+{"timestamp":1707862485.838378,"name":"drain","context":{"idset":"41","reason":"broker was unresponsive"}}
+{"timestamp":1707862550.7934186,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1707862797.8382277,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1707863200.8929882,"name":"online","context":{"idset":"42"}}
+{"timestamp":1707863321.3038337,"name":"online","context":{"idset":"41"}}
+{"timestamp":1707863575.9863987,"name":"undrain","context":{"idset":"41"}}
+{"timestamp":1707863591.0056872,"name":"undrain","context":{"idset":"42"}}
+{"timestamp":1707863690.6313021,"name":"online","context":{"idset":"37"}}
+{"timestamp":1707864194.4303365,"name":"drain","context":{"idset":"17","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1707925467.8379104,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1707925531.7405128,"name":"drain","context":{"idset":"38","reason":"epilog failed for jobid fwPkxhixb4b","overwrite":0}}
+{"timestamp":1707925531.8382971,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1707926335.7360427,"name":"online","context":{"idset":"38"}}
+{"timestamp":1707927796.7848685,"name":"undrain","context":{"idset":"38"}}
+{"timestamp":1707957384.5151241,"name":"drain","context":{"idset":"4","reason":"prolog failed for jobid fxaUHGm9LGF","overwrite":0}}
+{"timestamp":1708018317.2358747,"name":"drain","context":{"idset":"27","reason":"reboot","overwrite":0}}
+{"timestamp":1708018493.8377426,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1708018495.8382566,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1708022983.2222366,"name":"online","context":{"idset":"4"}}
+{"timestamp":1708023019.3887846,"name":"undrain","context":{"idset":"4"}}
+{"timestamp":1708023091.0536761,"name":"drain","context":{"idset":"17","reason":"nodediag gpu","overwrite":1}}
+{"timestamp":1708059599.2390714,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1708060733.3597038,"name":"drain","context":{"idset":"27","reason":"prolog failed for jobid fxchr7WnLRD","overwrite":0}}
+{"timestamp":1708120227.8380058,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1708120291.7382793,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1708120291.8384812,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1708130676.3031666,"name":"online","context":{"idset":"7"}}
+{"timestamp":1708130677.1603243,"name":"online","context":{"idset":"5"}}
+{"timestamp":1708130701.28105,"name":"undrain","context":{"idset":"5,7"}}
+{"timestamp":1708130871.8384726,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1708131640.5834272,"name":"online","context":{"idset":"27"}}
+{"timestamp":1708131663.7243624,"name":"undrain","context":{"idset":"27"}}
+{"timestamp":1708184693.4654632,"name":"drain","context":{"idset":"37-38","reason":"prolog failed for jobid fy6FUPcDxSB","overwrite":0}}
+{"timestamp":1708318642.0552068,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1708318643.7309797,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1708444977.9638999,"name":"drain","context":{"idset":"8","reason":"prolog failed for jobid fygME8fgRoV","overwrite":0}}
+{"timestamp":1708449591.7375274,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1708449591.8376031,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1708449591.851578,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1708449591.8521047,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1708449591.9517674,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1708450464.5751729,"name":"online","context":{"idset":"37"}}
+{"timestamp":1708450473.1423843,"name":"online","context":{"idset":"38"}}
+{"timestamp":1708451811.1425228,"name":"undrain","context":{"idset":"5,37-38"}}
+{"timestamp":1708453109.7380123,"name":"online","context":{"idset":"6"}}
+{"timestamp":1708453188.7591531,"name":"undrain","context":{"idset":"6"}}
+{"timestamp":1708453305.3757787,"name":"online","context":{"idset":"8"}}
+{"timestamp":1708453448.2236412,"name":"undrain","context":{"idset":"8"}}
+{"timestamp":1708455499.5559287,"name":"online","context":{"idset":"5"}}
+{"timestamp":1708492751.9970648,"name":"drain","context":{"idset":"5","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1708545245.1021707,"name":"drain","context":{"idset":"18","reason":"bad partner node-JRG","overwrite":0}}
+{"timestamp":1708557653.8370645,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1708562587.2785451,"name":"online","context":{"idset":"17"}}
+{"timestamp":1708562587.8421595,"name":"online","context":{"idset":"18"}}
+{"timestamp":1708563437.7380364,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1708563547.8739662,"name":"drain","context":{"idset":"17-18","reason":"rzlustre1","overwrite":1}}
+{"timestamp":1708564536.7295566,"name":"online","context":{"idset":"5"}}
+{"timestamp":1708564572.8504648,"name":"drain","context":{"idset":"5","reason":"rzlustre1","overwrite":1}}
+{"timestamp":1708630723.3196564,"name":"drain","context":{"idset":"8","reason":"prolog failed for jobid fz6gbidG4BR","overwrite":0}}
+{"timestamp":1708644045.3878703,"name":"undrain","context":{"idset":"5,17-18"}}
+{"timestamp":1708644341.8381383,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1708645956.1694183,"name":"online","context":{"idset":"8"}}
+{"timestamp":1708646380.946867,"name":"undrain","context":{"idset":"8"}}
+{"timestamp":1708904368.7411003,"name":"drain","context":{"idset":"3-4,7,9-10,19-26,28-32","reason":"upset with ansible","overwrite":0}}
+{"timestamp":1709057827.7397962,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1709057827.7404201,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1709057827.7408397,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1709057827.7412193,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1709057827.741601,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1709057827.7419708,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1709057827.7423189,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1709057827.7426829,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1709057827.7430236,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1709057827.7433569,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1709057827.74369,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1709057827.7440076,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1709057827.7443242,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1709057827.744637,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1709057827.7449427,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1709057827.7452464,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1709057827.838057,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1709057829.8380287,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1709059282.4735003,"name":"online","context":{"idset":"25"}}
+{"timestamp":1709059295.3923523,"name":"online","context":{"idset":"10"}}
+{"timestamp":1709059296.3855255,"name":"online","context":{"idset":"9"}}
+{"timestamp":1709059305.3444204,"name":"online","context":{"idset":"3"}}
+{"timestamp":1709059307.7188857,"name":"online","context":{"idset":"4"}}
+{"timestamp":1709059308.7391601,"name":"online","context":{"idset":"7"}}
+{"timestamp":1709059313.2648256,"name":"online","context":{"idset":"20"}}
+{"timestamp":1709059317.2187514,"name":"online","context":{"idset":"24"}}
+{"timestamp":1709059320.4163654,"name":"online","context":{"idset":"22"}}
+{"timestamp":1709059320.6105793,"name":"online","context":{"idset":"19"}}
+{"timestamp":1709059324.661155,"name":"online","context":{"idset":"28"}}
+{"timestamp":1709059325.2961633,"name":"online","context":{"idset":"23,32"}}
+{"timestamp":1709059325.4128139,"name":"online","context":{"idset":"21"}}
+{"timestamp":1709059326.0691898,"name":"online","context":{"idset":"26,30"}}
+{"timestamp":1709059505.9985266,"name":"undrain","context":{"idset":"3-4,7,9-10,19-26,28,32"}}
+{"timestamp":1709059683.8389406,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1709061018.7839279,"name":"online","context":{"idset":"31"}}
+{"timestamp":1709061025.4733136,"name":"online","context":{"idset":"30"}}
+{"timestamp":1709061122.3658459,"name":"undrain","context":{"idset":"30-31"}}
+{"timestamp":1709061365.9024534,"name":"drain","context":{"idset":"29","reason":"slingshot","overwrite":1}}
+{"timestamp":1709063423.5311122,"name":"online","context":{"idset":"29"}}
+{"timestamp":1709063443.3044019,"name":"undrain","context":{"idset":"29"}}
+{"timestamp":1709161676.2219987,"name":"drain","context":{"idset":"37-40","reason":"prolog failed for jobid f21JF1TqjUiT","overwrite":0}}
+{"timestamp":1709164881.736146,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1709164881.8361363,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1709164945.7385306,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1709164945.7417045,"name":"drain","context":{"idset":"23-24","reason":"epilog failed for jobid f21JcpsrK4Aj","overwrite":0}}
+{"timestamp":1709164945.8380418,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1709179855.5453553,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1709179855.545892,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1709179855.5463035,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1709179855.6452386,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1709180633.2748382,"name":"online","context":{"idset":"24"}}
+{"timestamp":1709180633.631073,"name":"online","context":{"idset":"23"}}
+{"timestamp":1709180665.4889343,"name":"undrain","context":{"idset":"23-24"}}
+{"timestamp":1709180756.8987303,"name":"online","context":{"idset":"38"}}
+{"timestamp":1709180758.1303515,"name":"online","context":{"idset":"39"}}
+{"timestamp":1709181329.0336177,"name":"undrain","context":{"idset":"38-39"}}
+{"timestamp":1709182330.5757935,"name":"online","context":{"idset":"37"}}
+{"timestamp":1709182334.7743187,"name":"online","context":{"idset":"40"}}
+{"timestamp":1709182473.5151224,"name":"undrain","context":{"idset":"37,40"}}
+{"timestamp":1709234309.8842003,"name":"drain","context":{"idset":"7","reason":"Epilog failed to complete","overwrite":0}}
+{"timestamp":1709234433.7404156,"name":"drain","context":{"idset":"7","reason":"epilog failed for jobid f21GSNfTD1uy","overwrite":0}}
+{"timestamp":1709234433.8382061,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1709243313.3831859,"name":"drain","context":{"idset":"17-32","reason":"reconfigure for gfs2","overwrite":0}}
+{"timestamp":1709244375.7721577,"name":"undrain","context":{"idset":"17-32"}}
+{"timestamp":1709265844.0127952,"name":"online","context":{"idset":"7"}}
+{"timestamp":1709265879.8928845,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1709597344.9819255,"name":"drain","context":{"idset":"41-42","reason":"Moving to another system","overwrite":0}}
+{"timestamp":1709597377.888833,"name":"offline","context":{"idset":"42"}}
+{"timestamp":1709597377.9894373,"name":"offline","context":{"idset":"41"}}
+{"timestamp":1709597575.8377159,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1709597579.8376544,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1709597641.7380157,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1709597641.8376784,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1709677515.8374412,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1709677578.462805,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1709677671.8378823,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1709677732.4794781,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1709678452.5885799,"name":"online","context":{"idset":"20"}}
+{"timestamp":1709678453.1054296,"name":"online","context":{"idset":"17"}}
+{"timestamp":1709682455.7374618,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1709682455.8375278,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1709683709.0123544,"name":"online","context":{"idset":"17"}}
+{"timestamp":1709683709.1793685,"name":"online","context":{"idset":"20"}}
+{"timestamp":1709683726.4352436,"name":"undrain","context":{"idset":"17,20"}}
+{"timestamp":1709757811.8391249,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1709757813.8378968,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1709757815.7377646,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1709757815.7378817,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1709757815.7379401,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1709757815.8385437,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1709757877.7387321,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1709757877.7393889,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1709757877.7398379,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1709757877.7401838,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1709757877.7405124,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1709757877.9124072,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1709760682.498179,"name":"online","context":{"idset":"36"}}
+{"timestamp":1709760682.8654833,"name":"online","context":{"idset":"34"}}
+{"timestamp":1709760684.03791,"name":"online","context":{"idset":"13"}}
+{"timestamp":1709760684.3850274,"name":"online","context":{"idset":"12"}}
+{"timestamp":1709760684.4930096,"name":"online","context":{"idset":"11"}}
+{"timestamp":1709760698.4456573,"name":"undrain","context":{"idset":"11-13,34,36"}}
+{"timestamp":1709761654.2443032,"name":"drain","context":{"idset":"6-7","reason":"nodediag amdgpu","overwrite":1}}
+{"timestamp":1709761738.9148006,"name":"online","context":{"idset":"6"}}
+{"timestamp":1709762029.8381662,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1709762041.1106775,"name":"drain","context":{"idset":"5","reason":"nodediag amdgpu","overwrite":1}}
+{"timestamp":1709762048.5661969,"name":"undrain","context":{"idset":"7"}}
+{"timestamp":1709762091.2446463,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1709762091.3586822,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1709764852.2865758,"name":"online","context":{"idset":"5"}}
+{"timestamp":1709764852.4182868,"name":"online","context":{"idset":"6"}}
+{"timestamp":1709764869.762249,"name":"undrain","context":{"idset":"5-6"}}
+{"timestamp":1710351361.4649928,"name":"online","context":{"idset":"38"}}
+{"timestamp":1710351469.6394072,"name":"online","context":{"idset":"37"}}
+{"timestamp":1710351917.6409914,"name":"undrain","context":{"idset":"37-38"}}
+{"timestamp":1710453685.8383853,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1710456027.8384783,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1710463182.9698641,"name":"online","context":{"idset":"2"}}
+{"timestamp":1710463475.6963859,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1710529055.8373916,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1710547575.8380218,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1710548508.9344761,"name":"online","context":{"idset":"2"}}
+{"timestamp":1710548578.8410048,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1710803147.8377545,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1710804529.8390036,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1710805366.4394033,"name":"online","context":{"idset":"2"}}
+{"timestamp":1710805394.5424531,"name":"undrain","context":{"idset":"2"}}
+{"timestamp":1710886419.8374178,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1710886507.8374052,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1710896965.7381608,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1710896965.8387263,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1710898099.1616108,"name":"online","context":{"idset":"1"}}
+{"timestamp":1710898100.151648,"name":"online","context":{"idset":"2"}}
+{"timestamp":1710898231.1469281,"name":"undrain","context":{"idset":"1-2"}}
+{"timestamp":1711021713.8387303,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1711021779.8388457,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1711040131.5446091,"name":"online","context":{"idset":"26"}}
+{"timestamp":1711040144.4600034,"name":"undrain","context":{"idset":"26"}}
+{"timestamp":1711153568.6771774,"name":"drain","context":{"idset":"10","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1711383329.4488847,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1711384583.5716627,"name":"online","context":{"idset":"10"}}
+{"timestamp":1711384610.2533531,"name":"undrain","context":{"idset":"10"}}
+{"timestamp":1711393749.1217792,"name":"drain","context":{"idset":"8","reason":"prolog failed for jobid f26LfFy2dMcK","overwrite":0}}
+{"timestamp":1711396867.7688053,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1711399450.7398818,"name":"online","context":{"idset":"8"}}
+{"timestamp":1711402842.4289572,"name":"undrain","context":{"idset":"8"}}
+{"timestamp":1711562071.5155444,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1711562071.5476818,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1711562071.5529733,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1711562071.5535269,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1711562071.5606616,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1711562071.5619266,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1711562071.5698957,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1711562071.5733974,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1711562071.5740981,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1711562071.5750401,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1711562071.5751336,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1711562071.5768588,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1711562071.5773368,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1711562071.5808523,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1711562071.5816214,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1711562071.5820677,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1711562071.5849607,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1711562071.5860474,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1711562071.5866809,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1711562071.5896459,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1711562071.5908241,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1711562071.5915174,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1711562071.5927622,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1711562071.5946543,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1711562071.5951457,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1711562071.5954111,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1711562071.5960329,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1711562071.5962002,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1711562071.5964334,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1711562071.5989981,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1711562071.6009147,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1711562071.6111562,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1711562071.6114984,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1711562071.6116772,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1711562071.6147885,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1711562071.715518,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1711562071.7776322,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1711562071.8012199,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1711562071.8033273,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1711562071.902746,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1711562657.7498362,"name":"resource-init","context":{"restart":true,"drain":{"41-42":{"timestamp":1709597344.9819255,"reason":"Moving to another system"}},"online":"","exclude":"0-2"}}
+{"timestamp":1711562657.7543545,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1711562719.6342204,"name":"online","context":{"idset":"0"}}
+{"timestamp":1711562720.3250408,"name":"online","context":{"idset":"1,37-40"}}
+{"timestamp":1711562720.8339283,"name":"online","context":{"idset":"33,35"}}
+{"timestamp":1711562720.963738,"name":"online","context":{"idset":"14"}}
+{"timestamp":1711562721.1101327,"name":"online","context":{"idset":"7,9,15-16"}}
+{"timestamp":1711562721.2148492,"name":"online","context":{"idset":"3-4,23,25,29"}}
+{"timestamp":1711562721.3656094,"name":"online","context":{"idset":"20-22,24,28,30"}}
+{"timestamp":1711562721.5184519,"name":"online","context":{"idset":"17-18"}}
+{"timestamp":1711562721.6294758,"name":"online","context":{"idset":"19,27,31-32"}}
+{"timestamp":1711562721.9826868,"name":"online","context":{"idset":"34,36"}}
+{"timestamp":1711562722.1093643,"name":"online","context":{"idset":"2,26"}}
+{"timestamp":1711562723.063693,"name":"online","context":{"idset":"6"}}
+{"timestamp":1711562723.1796999,"name":"online","context":{"idset":"5,8,12"}}
+{"timestamp":1711562723.3877113,"name":"online","context":{"idset":"11,13"}}
+{"timestamp":1711562723.4933889,"name":"online","context":{"idset":"10"}}
+{"timestamp":1711562798.5095389,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1711562798.5153723,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1711562798.518075,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1711562798.5220656,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1711562798.5228508,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1711562798.5232172,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1711562798.5273449,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1711562798.5292459,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1711562798.5296848,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1711562798.5300696,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1711562798.5327139,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1711562798.5361392,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1711562798.5371253,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1711562798.5379517,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1711562798.5395818,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1711562798.5424609,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1711562798.544009,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1711562798.5447056,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1711562798.5471637,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1711562798.5475664,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1711562798.5479436,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1711562798.5496936,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1711562798.5522764,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1711562798.5542397,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1711562798.5545228,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1711562798.5622208,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1711562798.5646443,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1711562798.5664985,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1711562798.5687649,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1711562798.5754938,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1711562798.578809,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1711562798.5821683,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1711562798.582479,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1711562798.6288555,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1711562798.6554906,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1711562798.7030022,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1711562798.7058046,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1711562798.7169104,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1711562798.7620802,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1711562798.8625369,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1711564987.0683236,"name":"online","context":{"idset":"40"}}
+{"timestamp":1711564997.0785792,"name":"online","context":{"idset":"37"}}
+{"timestamp":1711565003.3187122,"name":"online","context":{"idset":"38"}}
+{"timestamp":1711565020.3591337,"name":"online","context":{"idset":"39"}}
+{"timestamp":1711568271.2818451,"name":"online","context":{"idset":"2"}}
+{"timestamp":1711568271.818464,"name":"online","context":{"idset":"1"}}
+{"timestamp":1711568272.5509593,"name":"online","context":{"idset":"7,33"}}
+{"timestamp":1711568272.7834377,"name":"online","context":{"idset":"4"}}
+{"timestamp":1711568272.8876348,"name":"online","context":{"idset":"15"}}
+{"timestamp":1711568272.9492731,"name":"online","context":{"idset":"6"}}
+{"timestamp":1711568273.2199152,"name":"online","context":{"idset":"11"}}
+{"timestamp":1711568273.441834,"name":"online","context":{"idset":"5"}}
+{"timestamp":1711568273.9095511,"name":"online","context":{"idset":"16,34,36"}}
+{"timestamp":1711568274.1564608,"name":"online","context":{"idset":"10,35"}}
+{"timestamp":1711568274.2421155,"name":"online","context":{"idset":"14"}}
+{"timestamp":1711568275.6102519,"name":"online","context":{"idset":"23"}}
+{"timestamp":1711568276.2137187,"name":"online","context":{"idset":"21"}}
+{"timestamp":1711568277.2376351,"name":"online","context":{"idset":"17"}}
+{"timestamp":1711568277.4104073,"name":"online","context":{"idset":"22,32"}}
+{"timestamp":1711568277.5535326,"name":"online","context":{"idset":"31"}}
+{"timestamp":1711568277.7249949,"name":"online","context":{"idset":"8,18"}}
+{"timestamp":1711568277.8286169,"name":"online","context":{"idset":"20"}}
+{"timestamp":1711568278.1102707,"name":"online","context":{"idset":"12,19,25"}}
+{"timestamp":1711568278.2225249,"name":"online","context":{"idset":"3,13,26,29-30"}}
+{"timestamp":1711568278.3303578,"name":"online","context":{"idset":"27"}}
+{"timestamp":1711568278.5839033,"name":"online","context":{"idset":"9"}}
+{"timestamp":1711568288.8238063,"name":"online","context":{"idset":"28"}}
+{"timestamp":1711568325.3388956,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1711575841.9812288,"name":"online","context":{"idset":"24"}}
+{"timestamp":1711575842.4057868,"name":"online","context":{"idset":"23"}}
+{"timestamp":1712228549.7403214,"name":"drain","context":{"idset":"6","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1712571153.6118443,"name":"drain","context":{"idset":"3","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712571190.9812348,"name":"drain","context":{"idset":"9","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712571686.9373195,"name":"drain","context":{"idset":"7","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712571713.349153,"name":"drain","context":{"idset":"4","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712571723.3367388,"name":"drain","context":{"idset":"10","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712571745.2987714,"name":"drain","context":{"idset":"11","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712571784.7699995,"name":"drain","context":{"idset":"5","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712575118.5378792,"name":"drain","context":{"idset":"12","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712575340.6836975,"name":"drain","context":{"idset":"14","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712576782.937218,"name":"drain","context":{"idset":"8","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712576871.8109212,"name":"drain","context":{"idset":"13","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712577040.1794808,"name":"drain","context":{"idset":"15","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712583966.6428049,"name":"drain","context":{"idset":"16","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712583998.4048991,"name":"drain","context":{"idset":"20","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712584588.0363107,"name":"drain","context":{"idset":"18","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712584890.672461,"name":"drain","context":{"idset":"17","reason":"nodediag failed cxi","overwrite":0}}
+{"timestamp":1712587838.6220341,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1712587838.6247785,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1712587838.6253071,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1712587838.6254673,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1712587838.6260877,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1712587838.626611,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1712587838.628412,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1712587838.6288271,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1712587838.6289394,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1712587838.6294975,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1712587838.6308565,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1712587838.6313334,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1712587838.6323359,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1712587838.6334319,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1712587838.6548884,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1712587838.6808522,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1712587838.7061007,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1712587838.711508,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1712587838.7213881,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1712587838.7261713,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1712587838.7282426,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1712587838.7287235,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1712587838.7376254,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1712587838.746058,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1712587838.7483242,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1712587838.7505665,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1712587838.7540851,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1712587838.773119,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1712587838.7741294,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1712587838.7765293,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1712587838.7780209,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1712587838.7789896,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1712587838.7929807,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1712587838.7932806,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1712587838.8016355,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1712587838.8389187,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1712587838.9048545,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1712587838.9207585,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1712587838.9490297,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1712587839.0483029,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1712590818.3365602,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1712571153.6118443,"reason":"nodediag failed cxi"},"4":{"timestamp":1712571713.349153,"reason":"nodediag failed cxi"},"5":{"timestamp":1712571784.7699995,"reason":"nodediag failed cxi"},"6":{"timestamp":1712228549.7403214,"reason":"nodediag failed amdgpu"},"7":{"timestamp":1712571686.9373195,"reason":"nodediag failed cxi"},"8":{"timestamp":1712576782.937218,"reason":"nodediag failed cxi"},"9":{"timestamp":1712571190.9812348,"reason":"nodediag failed cxi"},"10":{"timestamp":1712571723.3367388,"reason":"nodediag failed cxi"},"11":{"timestamp":1712571745.2987714,"reason":"nodediag failed cxi"},"12":{"timestamp":1712575118.5378792,"reason":"nodediag failed cxi"},"13":{"timestamp":1712576871.8109212,"reason":"nodediag failed cxi"},"14":{"timestamp":1712575340.6836975,"reason":"nodediag failed cxi"},"15":{"timestamp":1712577040.1794808,"reason":"nodediag failed cxi"},"16":{"timestamp":1712583966.6428049,"reason":"nodediag failed cxi"},"17":{"timestamp":1712584890.672461,"reason":"nodediag failed cxi"},"18":{"timestamp":1712584588.0363107,"reason":"nodediag failed cxi"},"20":{"timestamp":1712583998.4048991,"reason":"nodediag failed cxi"},"41-42":{"timestamp":1709597344.9819255,"reason":"Moving to another system"}},"online":"","exclude":"0-2"}}
+{"timestamp":1712590818.3373437,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1712590855.650671,"name":"online","context":{"idset":"0"}}
+{"timestamp":1712598618.1848156,"name":"resource-init","context":{"restart":true,"drain":{"3":{"timestamp":1712571153.6118443,"reason":"nodediag failed cxi"},"4":{"timestamp":1712571713.349153,"reason":"nodediag failed cxi"},"5":{"timestamp":1712571784.7699995,"reason":"nodediag failed cxi"},"6":{"timestamp":1712228549.7403214,"reason":"nodediag failed amdgpu"},"7":{"timestamp":1712571686.9373195,"reason":"nodediag failed cxi"},"8":{"timestamp":1712576782.937218,"reason":"nodediag failed cxi"},"9":{"timestamp":1712571190.9812348,"reason":"nodediag failed cxi"},"10":{"timestamp":1712571723.3367388,"reason":"nodediag failed cxi"},"11":{"timestamp":1712571745.2987714,"reason":"nodediag failed cxi"},"12":{"timestamp":1712575118.5378792,"reason":"nodediag failed cxi"},"13":{"timestamp":1712576871.8109212,"reason":"nodediag failed cxi"},"14":{"timestamp":1712575340.6836975,"reason":"nodediag failed cxi"},"15":{"timestamp":1712577040.1794808,"reason":"nodediag failed cxi"},"16":{"timestamp":1712583966.6428049,"reason":"nodediag failed cxi"},"17":{"timestamp":1712584890.672461,"reason":"nodediag failed cxi"},"18":{"timestamp":1712584588.0363107,"reason":"nodediag failed cxi"},"20":{"timestamp":1712583998.4048991,"reason":"nodediag failed cxi"},"41-42":{"timestamp":1709597344.9819255,"reason":"Moving to another system"}},"online":"","exclude":"0-2"}}
+{"timestamp":1712598618.1856339,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1712598654.729003,"name":"online","context":{"idset":"0"}}
+{"timestamp":1712598821.7026219,"name":"online","context":{"idset":"1"}}
+{"timestamp":1712598823.1951349,"name":"online","context":{"idset":"2"}}
+{"timestamp":1712598824.9349616,"name":"online","context":{"idset":"40"}}
+{"timestamp":1712598827.9501436,"name":"online","context":{"idset":"39"}}
+{"timestamp":1712598880.8823647,"name":"online","context":{"idset":"37"}}
+{"timestamp":1712598882.2186096,"name":"online","context":{"idset":"38"}}
+{"timestamp":1712598951.9832847,"name":"online","context":{"idset":"36"}}
+{"timestamp":1712598952.0610816,"name":"online","context":{"idset":"35"}}
+{"timestamp":1712598952.1085169,"name":"online","context":{"idset":"33"}}
+{"timestamp":1712598953.3848193,"name":"online","context":{"idset":"14,22"}}
+{"timestamp":1712598953.9689507,"name":"online","context":{"idset":"8"}}
+{"timestamp":1712598954.1856146,"name":"online","context":{"idset":"4,6,10,21"}}
+{"timestamp":1712598954.3061726,"name":"online","context":{"idset":"13,18,25"}}
+{"timestamp":1712598954.448626,"name":"online","context":{"idset":"3"}}
+{"timestamp":1712598954.5744207,"name":"online","context":{"idset":"5,29"}}
+{"timestamp":1712598954.8848865,"name":"online","context":{"idset":"7,23"}}
+{"timestamp":1712598954.9875467,"name":"online","context":{"idset":"9,15-16,28"}}
+{"timestamp":1712598955.2929707,"name":"online","context":{"idset":"11-12,17,19-20,24,26-27,30-32"}}
+{"timestamp":1712598981.4848742,"name":"undrain","context":{"idset":"3-18,20"}}
+{"timestamp":1712676551.6560497,"name":"drain","context":{"idset":"33-34","reason":"PY: rebooting to clear stuck GPU on fake43","overwrite":0}}
+{"timestamp":1712676619.458843,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1712770780.37569,"name":"online","context":{"idset":"33"}}
+{"timestamp":1712770780.5136557,"name":"online","context":{"idset":"34"}}
+{"timestamp":1712775868.9787755,"name":"undrain","context":{"idset":"33-34"}}
+{"timestamp":1713219996.2825902,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1713221278.8298764,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1713221808.6494505,"name":"online","context":{"idset":"40"}}
+{"timestamp":1713221966.8193207,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1713222297.9752064,"name":"online","context":{"idset":"37"}}
+{"timestamp":1713223013.1917467,"name":"online","context":{"idset":"40"}}
+{"timestamp":1713463334.8191154,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1713463394.8459895,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1713464267.1192989,"name":"online","context":{"idset":"38"}}
+{"timestamp":1713464390.9905565,"name":"undrain","context":{"idset":"38"}}
+{"timestamp":1713982737.5843689,"name":"resource-init","context":{"restart":true,"drain":{},"online":"","exclude":"0-2"}}
+{"timestamp":1713982737.585748,"name":"resource-define","context":{"method":"configuration"}}
+{"timestamp":1713982782.8416674,"name":"online","context":{"idset":"0"}}
+{"timestamp":1713982783.7456343,"name":"online","context":{"idset":"37-39"}}
+{"timestamp":1713982783.8473592,"name":"online","context":{"idset":"40"}}
+{"timestamp":1713982785.0302947,"name":"online","context":{"idset":"35"}}
+{"timestamp":1713982785.2778559,"name":"online","context":{"idset":"33-34,36"}}
+{"timestamp":1713982785.6222816,"name":"online","context":{"idset":"21"}}
+{"timestamp":1713982785.7510157,"name":"online","context":{"idset":"23-24,28"}}
+{"timestamp":1713982785.8817627,"name":"online","context":{"idset":"22"}}
+{"timestamp":1713982786.4081388,"name":"online","context":{"idset":"25,27,31"}}
+{"timestamp":1713982786.5279021,"name":"online","context":{"idset":"26,29-30,32"}}
+{"timestamp":1713982864.3269086,"name":"online","context":{"idset":"3,17"}}
+{"timestamp":1713982864.7409158,"name":"online","context":{"idset":"19"}}
+{"timestamp":1713982865.3713729,"name":"online","context":{"idset":"13"}}
+{"timestamp":1713982866.455795,"name":"online","context":{"idset":"7,14"}}
+{"timestamp":1713982866.4785533,"name":"online","context":{"idset":"10"}}
+{"timestamp":1713982866.6512744,"name":"online","context":{"idset":"4"}}
+{"timestamp":1713982867.3172619,"name":"online","context":{"idset":"1"}}
+{"timestamp":1713982867.5152628,"name":"online","context":{"idset":"15"}}
+{"timestamp":1713982867.6863558,"name":"online","context":{"idset":"2"}}
+{"timestamp":1713982867.8120506,"name":"online","context":{"idset":"16"}}
+{"timestamp":1713982867.9819324,"name":"online","context":{"idset":"20"}}
+{"timestamp":1713982868.5041931,"name":"online","context":{"idset":"12,18"}}
+{"timestamp":1713982868.7936485,"name":"online","context":{"idset":"5,11"}}
+{"timestamp":1713982868.9076262,"name":"online","context":{"idset":"9"}}
+{"timestamp":1713982870.2439618,"name":"online","context":{"idset":"8"}}
+{"timestamp":1713982870.8293262,"name":"online","context":{"idset":"6"}}
+{"timestamp":1713984872.8405976,"name":"drain","context":{"idset":"7","reason":"broker was unresponsive"}}
+{"timestamp":1713984872.8406949,"name":"drain","context":{"idset":"10","reason":"broker was unresponsive"}}
+{"timestamp":1713984872.8407395,"name":"drain","context":{"idset":"14","reason":"broker was unresponsive"}}
+{"timestamp":1713984872.8407865,"name":"drain","context":{"idset":"15","reason":"broker was unresponsive"}}
+{"timestamp":1713984872.8408284,"name":"drain","context":{"idset":"16","reason":"broker was unresponsive"}}
+{"timestamp":1713984872.8408709,"name":"drain","context":{"idset":"20","reason":"broker was unresponsive"}}
+{"timestamp":1713984872.8409133,"name":"drain","context":{"idset":"37","reason":"broker was unresponsive"}}
+{"timestamp":1713984872.8409557,"name":"drain","context":{"idset":"38","reason":"broker was unresponsive"}}
+{"timestamp":1713984872.8410015,"name":"drain","context":{"idset":"39","reason":"broker was unresponsive"}}
+{"timestamp":1713984872.940526,"name":"drain","context":{"idset":"40","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8410339,"name":"drain","context":{"idset":"1","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8411019,"name":"drain","context":{"idset":"5","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.841146,"name":"drain","context":{"idset":"9","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8411889,"name":"drain","context":{"idset":"11","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8412344,"name":"drain","context":{"idset":"12","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8412771,"name":"drain","context":{"idset":"18","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.841321,"name":"drain","context":{"idset":"21","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8413656,"name":"drain","context":{"idset":"22","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.841409,"name":"drain","context":{"idset":"23","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8414536,"name":"drain","context":{"idset":"24","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8415015,"name":"drain","context":{"idset":"28","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8415487,"name":"drain","context":{"idset":"33","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8416054,"name":"drain","context":{"idset":"34","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.8416548,"name":"drain","context":{"idset":"35","reason":"broker was unresponsive"}}
+{"timestamp":1713984874.9416409,"name":"drain","context":{"idset":"36","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8412435,"name":"drain","context":{"idset":"2","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8413322,"name":"drain","context":{"idset":"3","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8413849,"name":"drain","context":{"idset":"4","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8414385,"name":"drain","context":{"idset":"6","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8414896,"name":"drain","context":{"idset":"8","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8415391,"name":"drain","context":{"idset":"13","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8415997,"name":"drain","context":{"idset":"17","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8416491,"name":"drain","context":{"idset":"19","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8416982,"name":"drain","context":{"idset":"25","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8417473,"name":"drain","context":{"idset":"26","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8417974,"name":"drain","context":{"idset":"27","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.841846,"name":"drain","context":{"idset":"29","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8418953,"name":"drain","context":{"idset":"30","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.8419483,"name":"drain","context":{"idset":"31","reason":"broker was unresponsive"}}
+{"timestamp":1713984876.9409621,"name":"drain","context":{"idset":"32","reason":"broker was unresponsive"}}
+{"timestamp":1713984938.8429897,"name":"offline","context":{"idset":"1"}}
+{"timestamp":1713984938.8431396,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1713984938.8432672,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1713984938.8433542,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1713984938.8434412,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1713984938.8435524,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1713984938.8436768,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1713984938.8437963,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1713984938.8439171,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1713984938.9546599,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1713984938.9548442,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1713984939.0674968,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1713984939.0676787,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1713984939.0677655,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1713984939.067858,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1713984939.0679886,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1713984939.0681605,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1713984939.0682442,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1713984939.0683408,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1713984939.0684171,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1713984939.0684907,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1713984939.0685947,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1713984939.0686648,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1713984939.068747,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1713984939.0688295,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1713984939.0689001,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1713984939.0689836,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1713984939.0690727,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1713984939.069139,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1713984939.0692298,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1713984939.069309,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1713984939.0693872,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1713984939.0694754,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1713984939.0695643,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1713984939.0696478,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1713984939.0697224,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1713984939.0697982,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1713984939.0699034,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1713984939.069973,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1713984939.0702,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1713991933.4650676,"name":"online","context":{"idset":"40"}}
+{"timestamp":1713991939.3504162,"name":"online","context":{"idset":"37"}}
+{"timestamp":1713991949.2638226,"name":"online","context":{"idset":"39"}}
+{"timestamp":1713991961.650804,"name":"online","context":{"idset":"38"}}
+{"timestamp":1713992115.2157364,"name":"online","context":{"idset":"2"}}
+{"timestamp":1713992115.3365548,"name":"online","context":{"idset":"1"}}
+{"timestamp":1713992115.5741246,"name":"online","context":{"idset":"34-35"}}
+{"timestamp":1713992115.8788595,"name":"online","context":{"idset":"36"}}
+{"timestamp":1713992116.1903315,"name":"online","context":{"idset":"25"}}
+{"timestamp":1713992116.9413693,"name":"online","context":{"idset":"27"}}
+{"timestamp":1713992117.1335239,"name":"online","context":{"idset":"18"}}
+{"timestamp":1713992117.2841096,"name":"online","context":{"idset":"28,32"}}
+{"timestamp":1713992117.3792427,"name":"online","context":{"idset":"29,31"}}
+{"timestamp":1713992117.5268309,"name":"online","context":{"idset":"24"}}
+{"timestamp":1713992117.5507739,"name":"online","context":{"idset":"19"}}
+{"timestamp":1713992117.7393384,"name":"online","context":{"idset":"21,30"}}
+{"timestamp":1713992117.8687425,"name":"online","context":{"idset":"26"}}
+{"timestamp":1713992117.9964063,"name":"online","context":{"idset":"17,22-23,33"}}
+{"timestamp":1713992118.155268,"name":"online","context":{"idset":"20"}}
+{"timestamp":1713992118.8408706,"name":"online","context":{"idset":"13"}}
+{"timestamp":1713992119.0468056,"name":"online","context":{"idset":"10,12"}}
+{"timestamp":1713992119.1555443,"name":"online","context":{"idset":"15"}}
+{"timestamp":1713992119.4292426,"name":"online","context":{"idset":"3,5,7,9,14"}}
+{"timestamp":1713992119.5524788,"name":"online","context":{"idset":"8"}}
+{"timestamp":1713992119.8194516,"name":"online","context":{"idset":"4,16"}}
+{"timestamp":1713992119.9976468,"name":"online","context":{"idset":"11"}}
+{"timestamp":1713992124.5133836,"name":"online","context":{"idset":"6"}}
+{"timestamp":1713992229.4854674,"name":"undrain","context":{"idset":"1-40"}}
+{"timestamp":1714425971.7527313,"name":"drain","context":{"idset":"35","reason":"nodediag failed amdgpu","overwrite":0}}
+{"timestamp":1714500646.9410026,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1714502158.667922,"name":"online","context":{"idset":"35"}}
+{"timestamp":1714502177.7259822,"name":"undrain","context":{"idset":"35"}}
+{"timestamp":1714660938.0305953,"name":"offline","context":{"idset":"38"}}
+{"timestamp":1714660938.0343237,"name":"offline","context":{"idset":"39"}}
+{"timestamp":1714660938.0614443,"name":"offline","context":{"idset":"40"}}
+{"timestamp":1714660938.1122987,"name":"offline","context":{"idset":"37"}}
+{"timestamp":1714660938.1363149,"name":"offline","context":{"idset":"13"}}
+{"timestamp":1714660938.1387,"name":"offline","context":{"idset":"33"}}
+{"timestamp":1714660938.1524351,"name":"offline","context":{"idset":"34"}}
+{"timestamp":1714660938.1669891,"name":"offline","context":{"idset":"9"}}
+{"timestamp":1714660938.1757388,"name":"offline","context":{"idset":"25"}}
+{"timestamp":1714660938.1778321,"name":"offline","context":{"idset":"6"}}
+{"timestamp":1714660938.1792617,"name":"offline","context":{"idset":"3"}}
+{"timestamp":1714660938.1812356,"name":"offline","context":{"idset":"22"}}
+{"timestamp":1714660938.1842203,"name":"offline","context":{"idset":"14"}}
+{"timestamp":1714660938.1869106,"name":"offline","context":{"idset":"18"}}
+{"timestamp":1714660938.187104,"name":"offline","context":{"idset":"10"}}
+{"timestamp":1714660938.20121,"name":"offline","context":{"idset":"4"}}
+{"timestamp":1714660938.201539,"name":"offline","context":{"idset":"11"}}
+{"timestamp":1714660938.2091799,"name":"offline","context":{"idset":"15"}}
+{"timestamp":1714660938.2097588,"name":"offline","context":{"idset":"24"}}
+{"timestamp":1714660938.2232111,"name":"offline","context":{"idset":"31"}}
+{"timestamp":1714660938.2355559,"name":"offline","context":{"idset":"7"}}
+{"timestamp":1714660938.2444515,"name":"offline","context":{"idset":"32"}}
+{"timestamp":1714660938.2542441,"name":"offline","context":{"idset":"17"}}
+{"timestamp":1714660938.2931008,"name":"offline","context":{"idset":"36"}}
+{"timestamp":1714660938.3234859,"name":"offline","context":{"idset":"12"}}
+{"timestamp":1714660938.3264921,"name":"offline","context":{"idset":"26"}}
+{"timestamp":1714660938.3791785,"name":"offline","context":{"idset":"35"}}
+{"timestamp":1714660938.4207473,"name":"offline","context":{"idset":"16"}}
+{"timestamp":1714660938.4209492,"name":"offline","context":{"idset":"8"}}
+{"timestamp":1714660938.4210281,"name":"offline","context":{"idset":"29"}}
+{"timestamp":1714660938.4395285,"name":"offline","context":{"idset":"21"}}
+{"timestamp":1714660938.4401307,"name":"offline","context":{"idset":"27"}}
+{"timestamp":1714660938.4465513,"name":"offline","context":{"idset":"19"}}
+{"timestamp":1714660938.4614763,"name":"offline","context":{"idset":"30"}}
+{"timestamp":1714660938.4616222,"name":"offline","context":{"idset":"28"}}
+{"timestamp":1714660938.4826641,"name":"offline","context":{"idset":"23"}}
+{"timestamp":1714660938.5833383,"name":"offline","context":{"idset":"20"}}
+{"timestamp":1714660938.6960547,"name":"offline","context":{"idset":"5"}}
+{"timestamp":1714660942.9150043,"name":"offline","context":{"idset":"2"}}
+{"timestamp":1714660943.0154507,"name":"offline","context":{"idset":"1"}}
diff --git a/t/rexec/rexec.c b/t/rexec/rexec.c
index dfd255c899a3..8e3352d95977 100644
--- a/t/rexec/rexec.c
+++ b/t/rexec/rexec.c
@@ -16,11 +16,16 @@
 #include <unistd.h>
 #include <string.h>
 #include <inttypes.h>
+#ifndef HAVE_GET_CURRENT_DIR_NAME
+#include "src/common/libmissing/get_current_dir_name.h"
+#endif
 #include <flux/core.h>
 #include <flux/optparse.h>
 
+#include "src/common/libsubprocess/subprocess_private.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/read_all.h"
+#include "ccan/str/str.h"
 
 extern char **environ;
 
@@ -76,8 +81,7 @@ void state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
     if (optparse_getopt (opts, "outputstates", NULL) > 0)
         printf ("%s\n", flux_subprocess_state_string (state));
 
-    if (state == FLUX_SUBPROCESS_EXEC_FAILED
-        || state == FLUX_SUBPROCESS_FAILED) {
+    if (state == FLUX_SUBPROCESS_FAILED) {
         fprintf (stderr, "rank %d: %s: %s\n",
                  flux_subprocess_rank (p),
                  flux_subprocess_state_string (state),
@@ -96,21 +100,18 @@ void state_cb (flux_subprocess_t *p, flux_subprocess_state_t state)
 void stdin2stream (flux_subprocess_t *p, const char *stream)
 {
     char *buf = NULL;
-    int tmp, len;
+    int len;
 
     if ((len = read_all (STDIN_FILENO, (void **)&buf)) < 0)
         log_err_exit ("read_all");
 
     if (len) {
-        if ((tmp = flux_subprocess_write (p, stream, buf, len)) < 0)
+        if (flux_subprocess_write (p, stream, buf, len) < 0)
             log_err_exit ("flux_subprocess_write");
-
-        if (tmp != len)
-            log_err_exit ("overflow in write");
     }
 
     /* do not close for channel, b/c can race w/ data coming back */
-    if (!strcmp (stream, "stdin")) {
+    if (streq (stream, "stdin")) {
         if (flux_subprocess_close (p, stream) < 0)
             log_err_exit ("flux_subprocess_close");
     }
@@ -129,8 +130,8 @@ int main (int argc, char *argv[])
         .on_completion = completion_cb,
         .on_state_change = state_cb,
         .on_channel_out = NULL,
-        .on_stdout = flux_standard_output,
-        .on_stderr = flux_standard_output,
+        .on_stdout = subprocess_standard_output,
+        .on_stderr = subprocess_standard_output,
     };
     const char *optargp;
     int optindex;
@@ -166,12 +167,12 @@ int main (int argc, char *argv[])
     free (cwd);
 
     if (optparse_getopt (opts, "stdin2stream", &optargp) > 0) {
-        if (strcmp (optargp, "stdin")
-            && strcmp (optargp, "stdout")
-            && strcmp (optargp, "stderr")) {
+        if (!streq (optargp, "stdin")
+            && !streq (optargp, "stdout")
+            && !streq (optargp, "stderr")) {
             if (flux_cmd_add_channel (cmd, optargp) < 0)
                 log_err_exit ("flux_cmd_add_channel");
-            ops.on_channel_out = flux_standard_output;
+            ops.on_channel_out = subprocess_standard_output;
         }
     }
 
diff --git a/t/rexec/rexec_count_stdout.c b/t/rexec/rexec_count_stdout.c
index 8cb9ff9325da..ce2acb32917e 100644
--- a/t/rexec/rexec_count_stdout.c
+++ b/t/rexec/rexec_count_stdout.c
@@ -18,11 +18,15 @@
 #include <unistd.h>
 #include <string.h>
 #include <inttypes.h>
+#ifndef HAVE_GET_CURRENT_DIR_NAME
+#include "src/common/libmissing/get_current_dir_name.h"
+#endif
 #include <flux/core.h>
 #include <flux/optparse.h>
 
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/read_all.h"
+#include "ccan/str/str.h"
 
 extern char **environ;
 
@@ -49,28 +53,28 @@ void completion_cb (flux_subprocess_t *p)
 
 void output_cb (flux_subprocess_t *p, const char *stream)
 {
-    FILE *fstream = !strcmp (stream, "stderr") ? stderr : stdout;
-    const char *ptr;
-    int lenp;
+    FILE *fstream = streq (stream, "stderr") ? stderr : stdout;
+    const char *buf;
+    int len;
 
     /* Do not use flux_subprocess_getline(), testing is against
      * streams that are line buffered and not line buffered */
 
-    if (!(ptr = flux_subprocess_read_line (p, stream, &lenp))) {
+    if ((len = flux_subprocess_read_line (p, stream, &buf)) < 0) {
         log_err ("flux_subprocess_read_line");
         return;
     }
 
     /* we're at the end of the stream, read any lingering data */
-    if (!lenp && flux_subprocess_read_stream_closed (p, stream) > 0) {
-        if (!(ptr = flux_subprocess_read (p, stream, -1, &lenp))) {
+    if (!len && flux_subprocess_read_stream_closed (p, stream)) {
+        if ((len = flux_subprocess_read (p, stream, &buf)) < 0) {
             log_err ("flux_subprocess_read");
             return;
         }
     }
 
-    if (lenp)
-        fwrite (ptr, lenp, 1, fstream);
+    if (len)
+        fwrite (buf, len, 1, fstream);
 
     if (!strcasecmp (stream, "stdout"))
         stdout_count++;
diff --git a/t/rexec/rexec_getline.c b/t/rexec/rexec_getline.c
index b4a17c3eaa0e..1e72d31910cc 100644
--- a/t/rexec/rexec_getline.c
+++ b/t/rexec/rexec_getline.c
@@ -15,11 +15,16 @@
 #include <unistd.h>
 #include <string.h>
 #include <inttypes.h>
+#ifndef HAVE_GET_CURRENT_DIR_NAME
+#include "src/common/libmissing/get_current_dir_name.h"
+#endif
 #include <flux/core.h>
 #include <flux/optparse.h>
 
+#include "src/common/libsubprocess/subprocess_private.h"
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/read_all.h"
+#include "ccan/str/str.h"
 
 extern char **environ;
 
@@ -46,21 +51,18 @@ void completion_cb (flux_subprocess_t *p)
 void stdin2stream (flux_subprocess_t *p, const char *stream)
 {
     char *buf = NULL;
-    int tmp, len;
+    int len;
 
     if ((len = read_all (STDIN_FILENO, (void **)&buf)) < 0)
         log_err_exit ("read_all");
 
     if (len) {
-        if ((tmp = flux_subprocess_write (p, stream, buf, len)) < 0)
+        if (flux_subprocess_write (p, stream, buf, len) < 0)
             log_err_exit ("flux_subprocess_write");
-
-        if (tmp != len)
-            log_err_exit ("overflow in write");
     }
 
     /* do not close for channel, b/c can race w/ data coming back */
-    if (!strcmp (stream, "stdin")) {
+    if (streq (stream, "stdin")) {
         if (flux_subprocess_close (p, stream) < 0)
             log_err_exit ("flux_subprocess_close");
     }
@@ -70,14 +72,14 @@ void stdin2stream (flux_subprocess_t *p, const char *stream)
 
 void output_cb (flux_subprocess_t *p, const char *stream)
 {
-    FILE *fstream = !strcmp (stream, "stderr") ? stderr : stdout;
-    const char *ptr;
-    int lenp;
+    FILE *fstream = streq (stream, "stderr") ? stderr : stdout;
+    const char *buf;
+    int len;
 
-    if (!(ptr = flux_subprocess_getline (p, stream, &lenp)))
+    if ((len = flux_subprocess_getline (p, stream, &buf)) < 0)
         log_err_exit ("flux_subprocess_getline");
-    if (lenp)
-        fwrite (ptr, lenp, 1, fstream);
+    if (len)
+        fwrite (buf, len, 1, fstream);
     else
         fprintf (fstream, "EOF\n");
 }
@@ -127,12 +129,12 @@ int main (int argc, char *argv[])
         log_err_exit ("flux_cmd_setcwd");
 
     if (optparse_getopt (opts, "stdin2stream", &optargp) > 0) {
-        if (strcmp (optargp, "stdin")
-            && strcmp (optargp, "stdout")
-            && strcmp (optargp, "stderr")) {
+        if (!streq (optargp, "stdin")
+            && !streq (optargp, "stdout")
+            && !streq (optargp, "stderr")) {
             if (flux_cmd_add_channel (cmd, optargp) < 0)
                 log_err_exit ("flux_cmd_add_channel");
-            ops.on_channel_out = flux_standard_output;
+            ops.on_channel_out = subprocess_standard_output;
         }
     }
 
diff --git a/t/rexec/rexec_ps.c b/t/rexec/rexec_ps.c
deleted file mode 100644
index b3033e1128c2..000000000000
--- a/t/rexec/rexec_ps.c
+++ /dev/null
@@ -1,111 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdio.h>
-#include <unistd.h>
-#include <string.h>
-#include <inttypes.h>
-#include <flux/core.h>
-#include <flux/optparse.h>
-#include <jansson.h>
-
-#include "src/common/libutil/log.h"
-#include "src/common/libutil/read_all.h"
-
-extern char **environ;
-
-static struct optparse_option cmdopts[] = {
-    { .name = "rank", .key = 'r', .has_arg = 1, .arginfo = "rank",
-      .usage = "Specify rank for test" },
-    OPTPARSE_TABLE_END
-};
-
-void output (int rank, json_t *procs)
-{
-    size_t index;
-    json_t *value;
-
-    if (!json_is_array (procs))
-        log_msg_exit ("procs returned is not an array");
-
-    json_array_foreach (procs, index, value) {
-        int pid;
-        char *sender;
-
-        if (json_unpack (value, "{ s:i s:s }",
-                         "pid", &pid,
-                         "sender", &sender) < 0)
-            log_msg_exit ("json_unpack");
-
-        printf ("%s\t%d\t%d\n", sender, rank, pid);
-    }
-
-}
-
-int main (int argc, char *argv[])
-{
-    flux_t *h;
-    flux_reactor_t *reactor;
-    const char *optargp;
-    int rank;
-    flux_future_t *f;
-    optparse_t *opts;
-    int optindex;
-    int resp_rank;
-    json_t *resp_procs;
-
-    log_init ("rexec_ps");
-
-    opts = optparse_create ("rexec_ps");
-    if (optparse_add_option_table (opts, cmdopts) != OPTPARSE_SUCCESS)
-        log_msg_exit ("optparse_add_option_table");
-    if ((optindex = optparse_parse_args (opts, argc, argv)) < 0)
-        exit (1);
-
-    if (optparse_getopt (opts, "rank", &optargp) > 0) {
-        rank = atoi (optargp);
-    } else {
-        optparse_print_usage (opts);
-        exit (1);
-    }
-
-    if (!(h = flux_open (NULL, 0)))
-        log_err_exit ("flux_open");
-
-    if (!(reactor = flux_get_reactor (h)))
-        log_err_exit ("flux_get_reactor");
-
-    if (!(f = flux_rpc (h, "cmb.rexec.processes", NULL, rank, 0)))
-        log_err_exit ("flux_rpc");
-
-    if (flux_rpc_get_unpack (f, "{ s:i s:o }",
-                             "rank", &resp_rank,
-                             "procs", &resp_procs) < 0)
-        log_err_exit ("flux_rpc_get_unpack");
-
-    if (rank != resp_rank)
-        log_err_exit ("invalid rank returned = %d", resp_rank);
-
-    output (rank, resp_procs);
-
-    /* Clean up.
-     */
-    flux_close (h);
-    log_fini ();
-
-    return 0;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
diff --git a/t/sched-simple/jj-reader.c b/t/sched-simple/jj-reader.c
index 31e65f91ffa2..261706809836 100644
--- a/t/sched-simple/jj-reader.c
+++ b/t/sched-simple/jj-reader.c
@@ -8,6 +8,9 @@
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
 #include <unistd.h>
 #include <stdio.h>
 #include <string.h>
@@ -16,7 +19,7 @@
 
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/read_all.h"
-#include "src/modules/sched-simple/libjj.h"
+#include "src/common/libjob/jj.h"
 
 int main (int ac, char *av[])
 {
@@ -25,10 +28,15 @@ int main (int ac, char *av[])
     log_init ("jj-reader");
     if (read_all (STDIN_FILENO, (void **) &s) < 0)
         log_err_exit ("Failed to read stdin");
-    if (libjj_get_counts (s, &jj) < 0)
+    if (jj_get_counts (s, &jj) < 0)
         log_msg_exit ("%s", jj.error);
-    printf ("nnodes=%d nslots=%d slot_size=%d\n",
-            jj.nnodes, jj.nslots, jj.slot_size);
+    printf ("nnodes=%d nslots=%d slot_size=%d slot_gpus=%d exclusive=%s duration=%.1f\n",
+            jj.nnodes,
+            jj.nslots,
+            jj.slot_size,
+            jj.slot_gpus,
+            jj.exclusive ? "true" : "false",
+            jj.duration);
     log_fini ();
     free (s);
     return 0;
diff --git a/t/schedutil/req_and_unload.py b/t/schedutil/req_and_unload.py
deleted file mode 100755
index 7150f32ecbcc..000000000000
--- a/t/schedutil/req_and_unload.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Usage: flux python request-and-unload.py module-name
-
-import argparse
-import sys
-import errno
-import flux
-import json
-
-
-def expect_enosys(rpc, timeout=1):
-    try:
-        rpc.wait_for(timeout=timeout)
-        rpc.get()
-    except EnvironmentError as e:
-        if e.errno == errno.ENOSYS:
-            print("Successfully received ENOSYS")
-            return
-        elif e.errno == errno.ETIMEDOUT:
-            sys.exit("Request timed out")
-        else:
-            sys.exit("Unexpected errno: {}".format(e))
-    raise RuntimeError("Did not receive ENOSYS")
-
-
-def main():
-    h = flux.Flux()
-
-    alloc = h.rpc("sched.alloc", json.dumps({"id": 0}))
-    free = h.rpc("sched.free", json.dumps({"id": 0}))
-    print("Sent alloc and free requests")
-
-    h.rpc("cmb.rmmod", json.dumps({"name": args.sched_module})).get()
-    print("Removed {}".format(args.sched_module))
-
-    expect_enosys(alloc)
-    expect_enosys(free)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("sched_module")
-    args = parser.parse_args()
-
-    main()
diff --git a/t/scripts/dmesg-grep.py b/t/scripts/dmesg-grep.py
new file mode 100755
index 000000000000..0ac9d1d803bd
--- /dev/null
+++ b/t/scripts/dmesg-grep.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2019 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+#
+#  Follow Flux dmesg output until a line matches a pattern
+#
+import argparse
+import re
+import sys
+
+import flux
+from flux.constants import FLUX_RPC_STREAMING
+from flux.core.watchers import TimerWatcher
+
+parser = argparse.ArgumentParser(
+    description="watch the flux dmesg log for a given pattern"
+)
+parser.add_argument(
+    "-t",
+    "--timeout",
+    help="Timeout with error after some number of seconds",
+    metavar="SEC",
+    type=float,
+    default=1.0,
+)
+parser.add_argument(
+    "-v",
+    "--verbose",
+    help="Emit each line of dmesg output, not just first matching",
+    action="count",
+    default=0,
+)
+parser.add_argument("pattern")
+args = parser.parse_args()
+
+
+def timer_cb(h, watcher, revents, _arg):
+    print("Timeout!", file=sys.stderr)
+    h.reactor_stop_error()
+
+
+def dmesg_cb(rpc, pattern, verbose=False):
+    buf = rpc.get_str()
+    match = pattern.search(buf)
+    if match:
+        print(buf)
+        rpc.flux_handle.reactor_stop()
+    elif verbose:
+        print(buf)
+    rpc.reset()
+
+
+pattern = re.compile(args.pattern)
+
+h = flux.Flux()
+
+rpc = h.rpc("log.dmesg", {"follow": True}, flags=FLUX_RPC_STREAMING)
+rpc.then(dmesg_cb, pattern, verbose=args.verbose)
+
+TimerWatcher(h, args.timeout, timer_cb).start()
+if h.reactor_run() < 0:
+    sys.exit(1)
diff --git a/t/scripts/event-trace.lua b/t/scripts/event-trace.lua
index bbfcd7216556..c9ecbca006aa 100755
--- a/t/scripts/event-trace.lua
+++ b/t/scripts/event-trace.lua
@@ -30,6 +30,7 @@ OPTIONS:
 
 local flux = require 'flux'
 local getopt = require 'flux.alt_getopt'.get_opts
+local loadstring = loadstring or load
 
 -- Process command line arguments:
 local opts, optind = getopt (arg, "he:t:",
diff --git a/t/scripts/groups.py b/t/scripts/groups.py
new file mode 100755
index 000000000000..f48d2312aca8
--- /dev/null
+++ b/t/scripts/groups.py
@@ -0,0 +1,225 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+# groups - manipulate broker groups
+
+import argparse
+import logging
+
+import flux
+from flux.idset import IDset
+
+
+def barrier_continuation(rpc, fullset):
+    """
+    Stop the reactor once the group matches fullset.
+    """
+    resp = rpc.get()
+    ids = IDset(resp["members"])
+    if ids.equal(fullset):
+        rpc.flux_handle.reactor_stop()
+    else:
+        rpc.reset()
+
+
+def barrier(args):
+    """
+    This is functionally a barrier if run with flux exec on all broker ranks.
+    If --leave is specified, leave explicitly, otherwise just disconnect.
+    """
+    h = flux.Flux()
+    size = int(h.attr_get("size"))
+    fullset = IDset("0-" + str(size - 1))
+
+    entry = h.rpc(
+        "groups.get",
+        {"name": args.name},
+        nodeid=0,
+        flags=flux.constants.FLUX_RPC_STREAMING,
+    )
+    entry.then(barrier_continuation, fullset)
+
+    h.rpc("groups.join", {"name": args.name})
+    h.reactor_run()  # run until idset is full
+
+    if args.leave:
+        h.rpc("groups.leave", {"name": args.name}).get()
+
+
+def watch_continuation(rpc):
+    resp = rpc.get()
+    print(resp["members"])
+    rpc.reset()
+
+
+def watch(args):
+    """
+    Print each new value of group.  End with Ctrl-C.
+    """
+    h = flux.Flux()
+    rpc = h.rpc(
+        "groups.get",
+        {"name": args.name},
+        nodeid=0,
+        flags=flux.constants.FLUX_RPC_STREAMING,
+    )
+    rpc.then(watch_continuation)
+    h.reactor_run()
+
+
+def waitfor_continuation(rpc, count):
+    """
+    Stop the reactor once the group has the right number of members.
+    """
+    resp = rpc.get()
+    ids = IDset(resp["members"])
+    if ids.count() == count:
+        rpc.flux_handle.reactor_stop()
+    else:
+        rpc.reset()
+
+
+def waitfor(args):
+    """
+    Wait for group to have zero (or --count) members.
+    """
+    h = flux.Flux()
+    rpc = h.rpc(
+        "groups.get",
+        {"name": args.name},
+        nodeid=0,
+        flags=flux.constants.FLUX_RPC_STREAMING,
+    )
+    rpc.then(waitfor_continuation, args.count)
+    h.reactor_run()
+
+
+def get(args):
+    """
+    Get current value of group.
+    This only works on rank 0, but for testing that case we have --rank.
+    """
+    h = flux.Flux()
+    resp = h.rpc("groups.get", {"name": args.name}, nodeid=args.rank).get()
+    print(resp["members"])
+
+
+def join(args):
+    """
+    Join group.
+    If --leave is specified, explicitly leave, else just disconnect.
+    If --dubjoin, try to join twice for testing (will fail).
+    If --dubleave, try to leave twice for testing (will fail).
+    """
+    h = flux.Flux()
+
+    h.rpc("groups.join", {"name": args.name}, nodeid=args.rank).get()
+    if args.dubjoin:
+        h.rpc("groups.join", {"name": args.name}, nodeid=args.rank).get()
+
+    if args.leave:
+        h.rpc("groups.leave", {"name": args.name}, nodeid=args.rank).get()
+        if args.dubleave:
+            h.rpc("groups.leave", {"name": args.name}).get()
+
+
+def leave(args):
+    """
+    Leave group.
+    """
+    h = flux.Flux()
+
+    h.rpc("groups.leave", {"name": args.name}, nodeid=args.rank).get()
+
+
+LOGGER = logging.getLogger("groups")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    parser = argparse.ArgumentParser(prog="groups")
+    subparsers = parser.add_subparsers(
+        title="supported subcommands", description="", dest="subcommand"
+    )
+    subparsers.required = True
+
+    # barrier
+    barrier_parser = subparsers.add_parser(
+        "barrier",
+        usage="groups barrier [--leave] name",
+        formatter_class=flux.util.help_formatter(),
+    )
+    barrier_parser.add_argument("--leave", action="store_true")
+    barrier_parser.add_argument("name")
+    barrier_parser.set_defaults(func=barrier)
+
+    # watch
+    watch_parser = subparsers.add_parser(
+        "watch",
+        usage="groups watch name",
+        formatter_class=flux.util.help_formatter(),
+    )
+    watch_parser.add_argument("name")
+    watch_parser.set_defaults(func=watch)
+
+    # waitfor
+    waitfor_parser = subparsers.add_parser(
+        "waitfor",
+        usage="groups waitfor [--count=N] name",
+        formatter_class=flux.util.help_formatter(),
+    )
+    waitfor_parser.add_argument("--count", type=int, default=0)
+    waitfor_parser.add_argument("name")
+    waitfor_parser.set_defaults(func=waitfor)
+
+    # get
+    get_parser = subparsers.add_parser(
+        "get",
+        usage="groups get [--rank N] name",
+        formatter_class=flux.util.help_formatter(),
+    )
+    get_parser.add_argument("--rank", type=int, default=0)
+    get_parser.add_argument("name")
+    get_parser.set_defaults(func=get)
+
+    # join
+    join_parser = subparsers.add_parser(
+        "join",
+        usage="groups join [--rank N] [--dubjoin] [--leave] [--dubleave] name",
+        formatter_class=flux.util.help_formatter(),
+    )
+    join_parser.add_argument("--dubjoin", action="store_true")
+    join_parser.add_argument("--leave", action="store_true")
+    join_parser.add_argument("--dubleave", action="store_true")
+    join_parser.add_argument("--rank", type=int, default=flux.constants.FLUX_NODEID_ANY)
+    join_parser.add_argument("name")
+    join_parser.set_defaults(func=join)
+
+    # leave
+    leave_parser = subparsers.add_parser(
+        "leave",
+        usage="groups leave [--rank N] name",
+        formatter_class=flux.util.help_formatter(),
+    )
+    leave_parser.add_argument(
+        "--rank", type=int, default=flux.constants.FLUX_NODEID_ANY
+    )
+    leave_parser.add_argument("name")
+    leave_parser.set_defaults(func=leave)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/scripts/pipe.py b/t/scripts/pipe.py
new file mode 100755
index 000000000000..92198917c1dd
--- /dev/null
+++ b/t/scripts/pipe.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+#
+# Pipe standard input to a job, then exit.
+#
+import sys
+
+import flux
+from flux.job import JobID, event_wait
+
+
+def pipe_stdin(h, jobid, ranks):
+    event_wait(h, jobid, "start")
+    event = event_wait(h, jobid, "shell.init", eventlog="guest.exec.eventlog")
+    service = event.context["service"] + ".stdin"
+    for line in sys.stdin:
+        h.rpc(
+            service,
+            {"stream": "stdin", "rank": ranks, "data": line},
+        ).get()
+    h.rpc(service, {"stream": "stdin", "rank": ranks, "eof": True}).get()
+
+
+pipe_stdin(flux.Flux(), JobID(sys.argv[1]), sys.argv[2])
diff --git a/t/scripts/rexec.py b/t/scripts/rexec.py
new file mode 100755
index 000000000000..9354db5ac995
--- /dev/null
+++ b/t/scripts/rexec.py
@@ -0,0 +1,110 @@
+###############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+# rexec - bare bones rexec client
+
+import argparse
+import logging
+import sys
+
+import flux
+
+
+def kill(args):
+    h = flux.Flux()
+    try:
+        h.rpc(
+            args.service + ".kill",
+            nodeid=args.rank,
+            payload={"pid": int(args.pid), "signum": int(args.signum)},
+        ).get()
+    except OSError as exc:
+        LOGGER.error(f"kill: {exc}")
+        sys.exit(1)
+
+
+def ps(args):
+    h = flux.Flux()
+    try:
+        resp = h.rpc(
+            args.service + ".list",
+            nodeid=int(args.rank),
+        ).get()
+    except OSError as exc:
+        LOGGER.error(f"ps: {exc}")
+        sys.exit(1)
+    for item in resp["procs"]:
+        print(f"{item['pid']:<8}\t{item['cmd']}")
+
+
+LOGGER = logging.getLogger("rexec")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    parser = argparse.ArgumentParser(prog="rexec")
+    subparsers = parser.add_subparsers(
+        title="supported subcommands", description="", dest="subcommand"
+    )
+    subparsers.required = True
+
+    # kill
+    kill_parser = subparsers.add_parser(
+        "kill",
+        formatter_class=flux.util.help_formatter(),
+    )
+    kill_parser.add_argument(
+        "-r",
+        "--rank",
+        type=int,
+        help="Send RPC to specified broker rank",
+        default=flux.constants.FLUX_NODEID_ANY,
+    )
+    kill_parser.add_argument(
+        "-s",
+        "--service",
+        type=str,
+        help="Send RPC to specified service (default rexec)",
+        default="rexec",
+    )
+    kill_parser.add_argument("signum")
+    kill_parser.add_argument("pid")
+    kill_parser.set_defaults(func=kill)
+
+    # ps
+    ps_parser = subparsers.add_parser(
+        "ps",
+        formatter_class=flux.util.help_formatter(),
+    )
+    ps_parser.add_argument(
+        "-r",
+        "--rank",
+        type=int,
+        help="Send RPC to specified broker rank",
+        default=flux.constants.FLUX_NODEID_ANY,
+    )
+    ps_parser.add_argument(
+        "-s",
+        "--service",
+        type=str,
+        help="Send RPC to specified service (default rexec)",
+        default="rexec",
+    )
+    ps_parser.set_defaults(func=ps)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/scripts/run_timeout.py b/t/scripts/run_timeout.py
new file mode 100755
index 000000000000..584a4fafe42c
--- /dev/null
+++ b/t/scripts/run_timeout.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""run command with a timeout, timeout as a float is first argument, rest are command"""
+import argparse
+import os
+import signal
+import subprocess as s
+import sys
+
+parser = argparse.ArgumentParser(description="run command with a timeout")
+parser.add_argument(
+    "-s", "--signal", help="signal to send, default is SIGKILL", default=signal.SIGKILL
+)
+parser.add_argument(
+    "-e",
+    "--env",
+    help="environment variable to set of the form KEY=VAL",
+    action="append",
+    default=[],
+)
+parser.add_argument(
+    "-k",
+    "--kill-after",
+    type=float,
+    help="secondary timeout before kill if first does not kill it",
+    default=1.0,
+)
+parser.add_argument("timeout", type=float, help="timeout in float seconds")
+parser.add_argument("cmd")
+parser.add_argument("cmd_args", nargs=argparse.REMAINDER)
+
+args = parser.parse_args()
+
+
+def resolve_signal():
+    if not isinstance(args.signal, int):
+        try:
+            args.signal = int(args.signal)
+            return
+        except ValueError:
+            pass
+        try:
+            args.signal = getattr(signal, args.signal)
+            return
+        except AttributeError:
+            pass
+        try:
+            args.signal = getattr(signal, f"SIG{args.signal}")
+            return
+        except AttributeError:
+            pass
+        raise ValueError(f"value passed for signal is invalid: {args.signal}")
+
+
+def exit_signal(rc):
+    # python "helpfully" translates signal return codes for us, translate back
+    if rc < 0:
+        rc = 128 + abs(rc)
+    sys.exit(rc)
+
+
+def do_timeout():
+    environ = dict(os.environ)
+    for e in args.env:
+        (k, v) = e.split("=")
+        environ[k] = v
+    try:
+        # add cmd onto the front of the cmd arg list
+        args.cmd_args.insert(0, args.cmd)
+        p = s.Popen(args.cmd_args, env=environ)
+        # run with timeout, on success exits with return code
+        r = p.wait(timeout=args.timeout)
+    except s.TimeoutExpired:
+        # send signal to timeout process
+        print(f"{args.cmd} timed out after {args.timeout}s", file=sys.stderr)
+        p.send_signal(args.signal)
+        if args.kill_after > 0:
+            try:
+                # wait to make sure it actually stops
+                r = p.wait(timeout=args.kill_after)
+            except s.TimeoutExpired:
+                p.kill()
+                r = p.wait()
+    exit_signal(r)
+
+
+resolve_signal()
+do_timeout()
diff --git a/t/scripts/runpty.py b/t/scripts/runpty.py
new file mode 100755
index 000000000000..b40a99a55676
--- /dev/null
+++ b/t/scripts/runpty.py
@@ -0,0 +1,419 @@
+#!/usr/bin/env python3
+"""
+Run command in a pty, logging the output one of a set of formats that
+is safe and useful for later processing.
+"""
+
+import argparse
+import asyncio
+import fcntl
+import json
+import logging
+import os
+import pty
+import re
+import struct
+import sys
+import termios
+import time
+from signal import SIGALRM, SIGINT, SIGTERM, SIGUSR1, SIGWINCH, alarm, signal
+
+from flux import util
+
+
+def setwinsize(fd, rows, cols):
+    s = struct.pack("HHHH", rows, cols, 0, 0)
+    fcntl.ioctl(fd, termios.TIOCSWINSZ, s)
+
+
+def getwinsize(fd):
+    tsize = os.get_terminal_size()
+    return (tsize.lines, tsize.columns)
+
+
+def status_to_exitcode(status):
+    code = 0
+    if os.WIFSIGNALED(status):
+        code = 128 + os.WTERMSIG(status)
+    else:
+        code = os.WEXITSTATUS(status)
+    return code
+
+
+class OutputHandler:
+    def __init__(self, filename, width=80, height=25):
+        self.filename = filename
+        self.width = width
+        self.height = height
+        if self.filename == "-" or self.filename == "stdout":
+            self.fp = sys.stdout
+        else:
+            self.fp = open(filename, "w")
+
+    def format_entry(self, data):
+        return data.decode("utf-8", "replace")
+
+    def write_entry(self, data):
+        self.fp.write(self.format_entry(data))
+        self.fp.flush()
+
+
+class EventLogOutput(OutputHandler):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        header = dict(
+            timestamp=time.time(),
+            name="header",
+            context=dict(
+                version=1, width=self.width, height=self.height, encoding="utf-8"
+            ),
+        )
+        self.fp.write("{}\n".format(json.dumps(header)))
+        self.fp.flush()
+
+    def format_entry(self, data):
+        entry = dict(
+            timestamp=time.time(),
+            name="data",
+            context=dict(data=data.decode("utf-8", "replace")),
+        )
+        return "{}\n".format(json.dumps(entry))
+
+
+class AsciicastOutput(OutputHandler):
+    """
+    https://github.com/asciinema/asciinema/blob/develop/doc/asciicast-v2.md
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.t0 = time.time()
+        ts = int(self.t0)
+        header = dict(version=2, width=self.width, height=self.height, timestamp=ts)
+        self.fp.write("{}\n".format(json.dumps(header)))
+        self.fp.flush()
+
+    def format_entry(self, data):
+        dt = time.time() - self.t0
+        entry = [dt, "o", data.decode("utf-8", "replace")]
+        return "{}\n".format(json.dumps(entry))
+
+
+formats = {
+    "raw": OutputHandler,
+    "asciicast": AsciicastOutput,
+    "eventlog": EventLogOutput,
+}
+
+
+def parse_args():
+    try:
+        ws_default = "{0.columns}x{0.lines}".format(os.get_terminal_size())
+    except OSError:
+        ws_default = "80x25"
+
+    format_list = ",".join(formats.keys())
+
+    parser = argparse.ArgumentParser(
+        description="run command with a pty, log output to a file",
+        formatter_class=util.help_formatter(),
+    )
+    parser.add_argument(
+        "-o", "--output", help="set output file. Default=stdout", default="-"
+    )
+    parser.add_argument(
+        "-n", "--no-output", help="redirect output to /dev/null", action="store_true"
+    )
+    parser.add_argument(
+        "-i",
+        "--input",
+        help="set an input file in asciicast format. "
+        + "Use the special value 'none' to close stdin of pty immediately.",
+    )
+    parser.add_argument("--expect", help="set an expected output file")
+    parser.add_argument("--stderr", help="redirect stderr of process")
+    parser.add_argument(
+        "-f",
+        "--format",
+        help=f"set output format ({format_list}). Default=raw",
+        default="raw",
+    )
+    parser.add_argument(
+        "-w",
+        "--window-size",
+        metavar="WxH",
+        help=f"set pty window size in WIDTHxHEIGHT (default is {ws_default})",
+        default=ws_default,
+    )
+    parser.add_argument(
+        "--term",
+        metavar="TERMINAL",
+        help="set value of TERM variable for client (default xterm)",
+        default="xterm",
+    )
+    parser.add_argument(
+        "-c",
+        "--quit-char",
+        metavar="C",
+        help="Set the QUIT character (written to pty on SIGUSR1)",
+        default="",
+    )
+    parser.add_argument(
+        "--line-buffer", help="Attempt to line buffer theoutput", action="store_true"
+    )
+    parser.add_argument("COMMAND")
+    parser.add_argument("ARGS", nargs=argparse.REMAINDER)
+    return parser.parse_args()
+
+
+class ExpectEntry:
+    """
+    A single Expect/Send entry with optional timeout
+
+    An entry has the form
+
+        {"expect":s, "send":s, "timeout"?i}
+
+    Where 'expect' is a pattern, 'send' is the string to send as input
+    after the expected pattern matches, and 'timeout' is an optional
+    integer number of seconds after which the pattern match times out.
+    """
+
+    def __init__(self, entry):
+        self.expect = re.compile(entry["expect"])
+        self.send = entry["send"]
+        self.timeout = int(entry.get("timeout", 60.0))
+
+    def __str__(self):
+        return self.expect.pattern
+
+
+class Expecter:
+    """
+    Class which represents a list of expected output patterns along with
+    responses that are input after a pattern match. Responses are popped
+    off the stack as they are used.
+    """
+
+    def __init__(self):
+        self.entries = []
+        self.current = None
+        self.data = ""
+
+    def add_file(self, input_file):
+        """
+        Add a set of expect/send entries from a JSON file. The file should
+        contain a JSON array of ExpectEntry objects.
+        """
+        for entry in json.load(input_file):
+            self.entries.append(ExpectEntry(entry))
+        self.next()
+
+    def next(self, data=""):
+        """
+        Advance to the next expect entry
+        """
+        if self.entries:
+            self.current = self.entries.pop(0)
+            self.data = data
+            alarm(self.current.timeout)
+        else:
+            self.current = None
+            self.data = ""
+            alarm(0)
+
+    def match(self, data):
+        """
+        Accumulate data in the expect match buffer and return True if
+        the currently active pattern matches the buffer.
+        """
+        if self.current and data:
+            self.data += data
+            if re.search(self.current.expect, self.data):
+                return True
+        return False
+
+    def pop(self):
+        """
+        Return the current data to send after a match and advance to the
+        next expected pattern.
+        """
+        if self.current:
+            data = self.current.send
+            self.next()
+            return data.encode("utf-8")
+        return None
+
+
+class TTYBuffer:
+    def __init__(self, fd, linebuffer=False, bufsize=1024):
+        self.linebuffered = linebuffer
+        self.bufsize = bufsize
+        self.fd = fd
+        self.eof = False
+        self.data = bytearray()
+
+    def setlinebuf(self):
+        self.linebuffered = True
+
+    def read(self):
+        try:
+            data = os.read(self.fd, self.bufsize)
+            self.data += data
+        except (BlockingIOError, InterruptedError):
+            pass
+        except OSError:
+            self.eof = True
+
+    def peek(self):
+        return self.data.decode("utf-8", errors="surrogateescape")
+
+    def get(self):
+        data = bytes(self.data)
+        self.data = bytearray()
+        return data
+
+    def getline(self):
+        if not self.data:
+            return None
+        (line, sep, rest) = self.data.partition(b"\r\n")
+        if sep:
+            self.data = rest
+            return line + sep
+
+    def send_data(self, writer):
+        if self.linebuffered:
+            line = self.getline()
+            while line:
+                writer(line)
+                line = self.getline()
+            if self.eof:
+                writer(self.get())
+        else:
+            writer(self.get())
+
+
+log = logging.getLogger("runpty")
+
+
+@util.CLIMain(log)
+def main():
+
+    # Avoid asyncio DEBUG log messages (why is this on by default??)
+    logging.getLogger("asyncio").setLevel(logging.WARNING)
+
+    sys.stdout = open(
+        sys.stdout.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+    sys.stderr = open(
+        sys.stderr.fileno(), "w", encoding="utf8", errors="surrogateescape"
+    )
+
+    args = parse_args()
+    if args.no_output and args.output != "-":
+        log.error("Do not specify --no-output and --output")
+        sys.exit(1)
+    if args.no_output:
+        args.output = "/dev/null"
+
+    try:
+        formatter = formats[args.format]
+    except KeyError:
+        log.error(f'Unknown output format "{args.format}"')
+        sys.exit(1)
+
+    (width, height) = map(int, args.window_size.split("x"))
+    quit_char = args.quit_char.encode()
+
+    (pid, fd) = pty.fork()
+
+    if pid == pty.CHILD:
+        """
+        In child
+        """
+        if args.stderr:
+            sys.stderr = open(args.stderr, "w")
+            os.dup2(sys.stderr.fileno(), 2)
+
+        os.environ["TERM"] = args.term
+        setwinsize(pty.STDIN_FILENO, height, width)
+        os.execvp(args.COMMAND, [args.COMMAND, *args.ARGS])
+    else:
+        """
+        In parent, open log file and read output from child
+        """
+        os.set_blocking(fd, False)
+
+        signal(SIGWINCH, lambda sig, _: os.kill(pid, sig))
+        signal(SIGTERM, lambda sig, _: os.kill(pid, sig))
+        signal(SIGINT, lambda sig, _: os.kill(pid, sig))
+        signal(SIGUSR1, lambda sig, _: os.write(fd, quit_char))
+
+        ofile = formatter(args.output, width=width, height=height)
+        buf = TTYBuffer(fd, linebuffer=args.line_buffer)
+
+        loop = asyncio.get_event_loop()
+
+        if args.input and args.input == "none":
+
+            def write_eof(fd):
+                if hasattr(termios, "CEOF"):
+                    value = bytes([termios.CEOF])
+                else:
+                    # No CEOF in termios, assume Ctrl-D/EOT (0x4)
+                    value = bytes([4])
+                os.write(fd, value)
+
+            #  Sometimes the shell (if that is the target of runpty)
+            #   does not read EOF if it is sent too soon. Therefore send
+            #   EOF control character now, then 3 extra times to ensure it is
+            #   read eventually.
+            #
+            write_eof(fd)
+            loop.call_later(0.1, write_eof, fd)
+            loop.call_later(0.5, write_eof, fd)
+            loop.call_later(1.0, write_eof, fd)
+            loop.call_later(15, write_eof, fd)
+
+        elif args.input:
+
+            def write_tty(s):
+                os.write(fd, s.encode("utf-8"))
+
+            with open(args.input, "r") as infile:
+                infile.readline()
+                for line in infile:
+                    (timestamp, event_type, data) = json.loads(line)
+                    if event_type == "i":
+                        loop.call_later(float(timestamp), write_tty, data)
+
+        expect = Expecter()
+        if args.expect:
+            with open(args.expect) as fp:
+                expect.add_file(fp)
+
+        def timeout(sig, _):
+            os.kill(pid, SIGTERM)
+            log.error("timeout waiting for pattern '%s'", expect.current)
+
+        signal(SIGALRM, timeout)
+
+        def read_tty():
+            buf.read()
+            if expect.match(buf.peek()):
+                os.write(fd, expect.pop())
+            buf.send_data(ofile.write_entry)
+            if buf.eof:
+                loop.stop()
+
+        loop.add_reader(fd, read_tty)
+        loop.run_forever()
+
+        (pid, status) = os.waitpid(pid, 0)
+        sys.exit(status_to_exitcode(status))
+
+
+if __name__ == "__main__":
+    main()
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/scripts/sign-as.py b/t/scripts/sign-as.py
new file mode 100755
index 000000000000..c71e1d94b248
--- /dev/null
+++ b/t/scripts/sign-as.py
@@ -0,0 +1,14 @@
+import sys
+
+from flux.security import SecurityContext
+
+if len(sys.argv) < 2:
+    print("Usage: {0} USERID".format(sys.argv[0]))
+    sys.exit(1)
+
+userid = int(sys.argv[1])
+ctx = SecurityContext()
+payload = sys.stdin.read()
+
+print(ctx.sign_wrap_as(userid, payload, mech_type="none").decode("utf-8"))
+# vi: ts=4 sw=4 expandtab
diff --git a/t/scripts/sqlite-query.py b/t/scripts/sqlite-query.py
new file mode 100755
index 000000000000..620520cdbaf5
--- /dev/null
+++ b/t/scripts/sqlite-query.py
@@ -0,0 +1,66 @@
+###############################################################
+# Copyright 2019 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+# Query a sqlite db for testing purposes
+
+# Usage: flux python query.py [OPTIONS] dbpath query
+
+import argparse
+import sqlite3
+import sys
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-t", "--timeout", type=str, metavar="MS", help="Set busytimeout"
+    )
+    parser.add_argument(
+        "dbpath", type=str, metavar="DBPATH", nargs=1, help="database path"
+    )
+    parser.add_argument("query", type=str, metavar="QUERY", nargs=1, help="query")
+    args = parser.parse_args()
+
+    try:
+        #  Required in non-utf-8 locale if dbpath contains multibyte
+        #   characters. Prevents Python from whining about surrogates
+        #   not allowed. Really there must be a better way, but this works:
+        dbpath = args.dbpath[0].encode("utf-8", errors="surrogateescape").decode()
+        dburi = "file:" + dbpath + "?mode=ro"
+        con = sqlite3.connect(dburi, uri=True)
+    except sqlite3.Error as e:
+        print(e)
+        sys.exit(1)
+
+    if args.timeout:
+        con.execute("PRAGMA busy_wait = " + args.timeout)
+
+    con.row_factory = sqlite3.Row
+    cursor = con.cursor()
+
+    try:
+        cursor.execute(args.query[0])
+    except sqlite3.Error as e:
+        print(e)
+        sys.exit(1)
+
+    rows = cursor.fetchall()
+
+    #  make print below safe to handle utf-8
+    utf8out = open(1, "w", encoding="utf-8", closefd=False)
+
+    for row in rows:
+        for key in row.keys():
+            val = row[key]
+            print(f"{key} = {val}", file=utf8out)
+
+    con.close()
+    sys.exit(0)
+
+# vim: tabstop=4 shiftwidth=4 expandtab
diff --git a/t/scripts/startctl.py b/t/scripts/startctl.py
new file mode 100755
index 000000000000..e18721ece564
--- /dev/null
+++ b/t/scripts/startctl.py
@@ -0,0 +1,100 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+# startctl - tell flux-start to do things
+
+import argparse
+import logging
+import os
+import sys
+
+import flux
+
+
+def kill(args):
+    h = flux.Flux(os.environ.get("FLUX_START_URI"))
+    try:
+        h.rpc("start.kill", {"rank": int(args.rank), "signum": int(args.signum)}).get()
+    except ProcessLookupError:
+        LOGGER.error("rank %s broker process not found", args.rank)
+        sys.exit(1)
+
+
+def run(args):
+    h = flux.Flux(os.environ.get("FLUX_START_URI"))
+    h.rpc("start.run", {"rank": int(args.rank)}).get()
+
+
+def wait(args):
+    h = flux.Flux(os.environ.get("FLUX_START_URI"))
+    resp = h.rpc("start.wait", {"rank": int(args.rank)}).get()
+    sys.exit(int(resp["exit_rc"]))
+
+
+def status(args):
+    h = flux.Flux(os.environ.get("FLUX_START_URI"))
+    print(h.rpc("start.status").get_str())
+
+
+LOGGER = logging.getLogger("startctl")
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+    parser = argparse.ArgumentParser(prog="startctl")
+    subparsers = parser.add_subparsers(
+        title="supported subcommands", description="", dest="subcommand"
+    )
+    subparsers.required = True
+
+    # status
+    status_parser = subparsers.add_parser(
+        "status",
+        formatter_class=flux.util.help_formatter(),
+    )
+    status_parser.set_defaults(func=status)
+
+    # kill
+    kill_parser = subparsers.add_parser(
+        "kill",
+        usage="startctl kill rank signum",
+        formatter_class=flux.util.help_formatter(),
+    )
+    kill_parser.add_argument("rank")
+    kill_parser.add_argument("signum")
+    kill_parser.set_defaults(func=kill)
+
+    # run
+    run_parser = subparsers.add_parser(
+        "run",
+        usage="startctl run rank",
+        formatter_class=flux.util.help_formatter(),
+    )
+    run_parser.add_argument("rank")
+    run_parser.set_defaults(func=run)
+
+    # wait
+    wait_parser = subparsers.add_parser(
+        "wait",
+        usage="startctl wait rank",
+        formatter_class=flux.util.help_formatter(),
+    )
+    wait_parser.add_argument("rank")
+    wait_parser.set_defaults(func=wait)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()
+
+
+# vi: ts=4 sw=4 expandtab
diff --git a/t/scripts/stats-listen.py b/t/scripts/stats-listen.py
new file mode 100755
index 000000000000..b9e8f4f1499b
--- /dev/null
+++ b/t/scripts/stats-listen.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+import argparse
+import os
+import re
+import socket
+import subprocess
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "-n",
+    "--no-set-host",
+    help="don't set the FLUX_FRIPP_STATSD environment variable",
+    action="store_true",
+)
+parser.add_argument(
+    "-s",
+    "--search-for",
+    metavar="METRIC",
+    help="search for a specific metric tag",
+)
+parser.add_argument(
+    "-V", "--validate", help="validate packet form", action="store_true"
+)
+parser.add_argument(
+    "-w",
+    "--wait-for",
+    metavar="N",
+    help="wait for N packets to be received",
+    type=int,
+    default=1,
+)
+parser.add_argument("cmd", nargs=argparse.REMAINDER)
+args = parser.parse_args()
+
+s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+s.bind(("127.0.0.1", 0))
+
+if not args.no_set_host:
+    os.environ["FLUX_FRIPP_STATSD"] = f"127.0.0.1:{s.getsockname()[1]}"
+
+f = subprocess.Popen(args.cmd, env=dict(os.environ))
+returncode = f.wait()
+print(f"{args.cmd[0]} returncode = {returncode}", file=sys.stderr)
+
+p = []
+
+if args.search_for is not None:
+    while True:
+        m = s.recvfrom(1024)[0].decode("utf-8")
+        print(f"checking for {args.search_for} in {m}", file=sys.stderr)
+        if args.search_for in m:
+            p.append(m)
+            break
+
+else:
+    for i in range(args.wait_for):
+        p.append(s.recvfrom(1024)[0].decode("utf-8"))
+
+print(p)
+
+if len(p) < args.wait_for:
+    print(f"Error: Got less than {args.wait_for} packets", file=sys.stderr)
+    s.close()
+    exit(-1)
+
+if args.validate:
+    metrics = str.splitlines("".join(p))
+    ex = re.compile(r"^\w+:[\+\-]*\d+\|ms|[gC]$")
+
+    for m in metrics:
+        if not ex.search(m):
+            s.close()
+            exit(-1)
+
+s.close()
diff --git a/t/scripts/strerror_symbol b/t/scripts/strerror_symbol
new file mode 100755
index 000000000000..596465309a8e
--- /dev/null
+++ b/t/scripts/strerror_symbol
@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+##############################################################
+# Copyright 2023 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+"""
+Print the strerror string for the given errno constant name.
+
+For example:
+  grep "flux_future_get: $(strerror_symbol ENOTSUP)" file
+"""
+
+import errno
+import os
+import sys
+
+if len(sys.argv) != 2:
+    print(
+        "requires a single errno constant name or number and converts to strerror string"
+    )
+
+try:
+    print(os.strerror(getattr(errno, sys.argv[1])))
+    sys.exit(0)
+except AttributeError:
+    pass
+
+val = int(sys.argv[1])
+print(os.strerror(getattr(errno, errno.errorcode[val])))
diff --git a/t/scripts/tssh b/t/scripts/tssh
index f6e7df8baf91..6cb2575f2c71 100755
--- a/t/scripts/tssh
+++ b/t/scripts/tssh
@@ -25,4 +25,4 @@ echo hostname=${hostname} >&2
 echo cmd=\"$*\" >&2
 
 unset FLUX_URI
-eval $*
+exec $SHELL -c "$*"
diff --git a/t/scripts/waitfile.lua b/t/scripts/waitfile.lua
index f54367a531b3..29e63f7d3417 100755
--- a/t/scripts/waitfile.lua
+++ b/t/scripts/waitfile.lua
@@ -35,6 +35,7 @@ local getopt = require 'flux.alt_getopt' .get_opts
 local posix  = require 'flux.posix'
 local flux   = require 'flux'
 local timer  = require 'flux.timer'
+local loadstring = loadstring or load
 
 local opts, optind = getopt (arg, "hvqc:t:p:P:",
                              { timeout = 't',
diff --git a/t/sharness.d/01-setup.sh b/t/sharness.d/01-setup.sh
index 4d66542c1d88..45be272f1edc 100644
--- a/t/sharness.d/01-setup.sh
+++ b/t/sharness.d/01-setup.sh
@@ -7,7 +7,16 @@
 #
 unset FLUX_CONFIG
 unset FLUX_MODULE_PATH
-unset FLUX_CMBD_PATH
+unset FLUX_PMI_CLIENT_SEARCHPATH
+unset FLUX_PMI_CLIENT_METHODS
+
+# Unset any user defined output defaults, since that may mess up tests
+unset FLUX_JOBS_FORMAT_DEFAULT
+unset FLUX_RESOURCE_LIST_FORMAT_DEFAULT
+unset FLUX_RESOURCE_DRAIN_FORMAT_DEFAULT
+unset FLUX_RESOURCE_STATUS_FORMAT_DEFAULT
+unset FLUX_QUEUE_LIST_FORMAT_DEFAULT
+unset FLUX_PGREP_FORMAT_DEFAULT
 
 #
 #  FLUX_BUILD_DIR and FLUX_SOURCE_DIR are set to build and source paths
@@ -23,7 +32,7 @@ if test -z "$FLUX_BUILD_DIR"; then
 fi
 if test -z "$FLUX_SOURCE_DIR"; then
     if test -z "${srcdir}"; then
-        FLUX_SOURCE_DIR="$(cd .. && pwd)"
+        FLUX_SOURCE_DIR="$(cd ${SHARNESS_TEST_SRCDIR}/.. && pwd)"
     else
         FLUX_SOURCE_DIR="$(cd ${srcdir}/.. && pwd)"
     fi
@@ -38,8 +47,24 @@ if test -n "$FLUX_TEST_INSTALLED_PATH"; then
     PATH=$FLUX_TEST_INSTALLED_PATH:$PATH
     fluxbin=$FLUX_TEST_INSTALLED_PATH/flux
 else # normal case, use ${top_builddir}/src/cmd/flux
+    #
+    #  Ensure that the path to the configured Python is first in PATH
+    #  so the correct version is found by '#!/usr/bin/env python3' in
+    #  several test scripts that use this shebang line.
+    #  N.B.: This is not a complete fix. See flux-core #5091 for details
+    #
+    PATH=$($FLUX_BUILD_DIR/src/cmd/flux python -c \
+           'import os,sys; print(os.path.dirname(sys.executable))'):$PATH
+
     PATH=$FLUX_BUILD_DIR/src/cmd:$PATH
     fluxbin=$FLUX_BUILD_DIR/src/cmd/flux
+
+    #  Ensure that the built libflux-*.so are found before any system
+    #   installed versions. This is necessary because sometimes libtool
+    #   will use -rpath /usr/lib64 even for uninstalled test programs
+    #   (e.g. compiled MPI test programs like t/mpi/version)
+    #
+    export LD_LIBRARY_PATH="${FLUX_BUILD_DIR}/src/common/.libs:$LD_LIBRARY_PATH"
 fi
 export PATH
 
diff --git a/t/sharness.d/flux-sharness.sh b/t/sharness.d/flux-sharness.sh
index 5d5098f2fd14..403d0aaed913 100644
--- a/t/sharness.d/flux-sharness.sh
+++ b/t/sharness.d/flux-sharness.sh
@@ -3,11 +3,22 @@
 #  project-local sharness code for Flux
 #
 
+# add scripts directory to path
+export PATH="${SHARNESS_TEST_SRCDIR}/scripts:$PATH"
+
 #
 #  Extra functions for Flux testsuite
 #
 run_timeout() {
-    perl -e 'use Time::HiRes qw( ualarm ) ; ualarm ((shift @ARGV) * 1000000) ; exec @ARGV or die "$!"' "$@"
+    if test -z "$LD_PRELOAD" ; then
+        "${PYTHON:-python3}" "${SHARNESS_TEST_SRCDIR}/scripts/run_timeout.py" "$@"
+    else
+        (
+            TIMEOUT_PRELOAD="$LD_PRELOAD"
+            unset -v LD_PRELOAD
+            exec "${PYTHON:-python3}" -S "${SHARNESS_TEST_SRCDIR}/scripts/run_timeout.py" -e LD_PRELOAD="$TIMEOUT_PRELOAD" "$@"
+        )
+    fi
 }
 
 #
@@ -25,6 +36,29 @@ test_size_large() {
 }
 
 
+#
+#  Like test_must_fail(), but additionally allow process to be
+#   terminated by SIGKILL or SIGTERM
+#
+test_must_fail_or_be_terminated() {
+    "$@"
+    exit_code=$?
+    # Allow death by SIGTERM or SIGKILL
+    if test $exit_code = 143 -o $exit_code = 137; then
+        return 0
+    elif test $exit_code = 0; then
+        echo >&2 "test_must_fail_or_be_terminated: command succeeded: $*"
+        return 1
+    elif test $exit_code -ge 129 -a $exit_code -le 192; then
+        echo >&2 "test_must_fail_or_be_terminated: died by signal $(($exit_code-128)): $*"
+        return 1
+    elif test $exit_code = 127; then
+        echo >&2 "test_must_fail_or_be_terminated: command not found: $*"
+        return 1
+    fi
+    return 0
+}
+
 #
 #  Tests using test_under_flux() and which load their own modules should
 #   ensure those modules are unloaded at the end of the test for proper
@@ -56,15 +90,113 @@ check_module_list() {
 }
 
 #
-#  Reinvoke a test file under a flux comms instance
+#  Generate configuration for test bootstrap and print args for flux-start
+#  Usage:  args=$(make_bootstrap_config workdir sockdir size)
+#
+make_bootstrap_config() {
+    local workdir=$1
+    local sockdir=$2
+    local size=$3
+    local fakehosts="fake[0-$(($size-1))]"
+    local full="0-$(($size-1))"
+
+    mkdir $workdir/conf.d
+    mkdir $workdir/state
+    flux keygen --name testcert $workdir/cert
+    cat >$workdir/conf.d/bootstrap.toml <<-EOT
+	[bootstrap]
+	    curve_cert = "$workdir/cert"
+	    default_bind = "ipc://$sockdir/tbon-%h"
+	    default_connect = "ipc://$sockdir/tbon-%h"
+	    hosts = [
+	        { host = "$fakehosts" },
+	    ]
+	EOT
+    flux R encode --hosts=$fakehosts -r$full >$workdir/R
+    cat >$workdir/conf.d/resource.toml <<-EOT2
+	[resource]
+	    path = "$workdir/R"
+	    noverify = true
+	EOT2
+    echo "--test-hosts=$fakehosts -c$workdir/conf.d"
+    echo "--test-exit-mode=${TEST_UNDER_FLUX_EXIT_MODE:-leader}"
+    echo "--test-exit-timeout=${TEST_UNDER_FLUX_EXIT_TIMEOUT:-0}"
+    echo "-Sbroker.quorum=${TEST_UNDER_FLUX_QUORUM:-$size}"
+    echo "--test-start-mode=${TEST_UNDER_FLUX_START_MODE:-all}"
+    echo "-Stbon.topo=${TEST_UNDER_FLUX_TOPO:-custom}"
+    echo "-Stbon.zmqdebug=1"
+    echo "-Sstatedir=$workdir/state"
+}
+
+#
+#  Remove any outer trash-directory wrapper used by "system"
+#   personality test_under_flux() tests.
+#
+remove_trashdir_wrapper() {
+    local trashdir=$(dirname $SHARNESS_TRASH_DIRECTORY)
+    case $trashdir in
+        */trash-directory.[!/]*) rm -rf $trashdir
+    esac
+}
+
+#
+#  Reinvoke a test file under a flux instance
+#
+#  Usage: test_under_flux <size> [personality] [flux-start-options]
+#
+#  where personality is one of:
 #
-#  Usage: test_under_flux <size>
+#  full (default)
+#    Run with all services.
+#    The default broker rc scripts are executed.
+#
+#  minimal
+#    Run with only built-in services.
+#    No broker rc scripts are executed.
+#
+#  job
+#    Load minimum services needed to run jobs.
+#    Fake resources are loaded into the resource module.
+#    Environment variables:
+#    - TEST_UNDER_FLUX_CORES_PER_RANK
+#        Set the number of fake cores per fake node (default: 2).
+#    - TEST_UNDER_FLUX_NO_JOB_EXEC
+#        If set, skip loading job-exec module (default: load job-exec).
+#    - TEST_UNDER_FLUX_SCHED_SIMPLE_MODE
+#        Change mode argument to sched-simple (default: limited=8)
+#
+#  kvs
+#    Load minimum services needed for kvs.
+#
+#  system
+#    Like full, but bootstrap with a generated config file.
+#    Environment variables:
+#    - TEST_UNDER_FLUX_EXIT_MODE
+#        Set the flux-start exit mode (default: leader)
+#    - TEST_UNDER_FLUX_EXIT_TIMEOUT
+#        Set the flux-start exit timeout (default: 0)
+#    - TEST_UNDER_FLUX_QUORUM
+#        Set the broker.quorum attribute (default: <size>)
+#    - TEST_UNDER_FLUX_START_MODE
+#        Set the flux-start start mode (default: all)
+#    - TEST_UNDER_FLUX_TOPO
+#        Set the TBON topology (default: custom (flat))
 #
 test_under_flux() {
     size=${1:-1}
     personality=${2:-full}
+
+    #  Note: args > 2 are passed along as extra arguments
+    #   to flux-start below using "$@", so shift up to the
+    #   the first two arguments away:
+    #
+    test $# -eq 1 && shift || shift 2
+
     log_file="$TEST_NAME.broker.log"
     if test -n "$TEST_UNDER_FLUX_ACTIVE" ; then
+        if test "$TEST_UNDER_FLUX_PERSONALITY" = "system"; then
+            test "$debug" = "t" || cleanup remove_trashdir_wrapper
+        fi
         test "$debug" = "t" || cleanup rm "${SHARNESS_TEST_DIRECTORY:-..}/$log_file"
         flux_module_list > module-list.initial
         cleanup check_module_list
@@ -75,6 +207,7 @@ test_under_flux() {
     fi
     if test "$debug" = "t" -o -n "$FLUX_TESTS_DEBUG" ; then
         flags="${flags} --debug"
+        export FLUX_PYCLI_LOGLEVEL=10
     fi
     if test "$chain_lint" = "t"; then
         flags="${flags} --chain-lint"
@@ -85,16 +218,20 @@ test_under_flux() {
     if test -n "$SHARNESS_TEST_DIRECTORY"; then
         cd $SHARNESS_TEST_DIRECTORY
     fi
-    timeout="-o -Sinit.rc2_timeout=300"
-    if test -n "$FLUX_TEST_DISABLE_TIMEOUT"; then
-        timeout=""
-    elif test_have_prereq LONGTEST; then
-        timeout="-o,-Sinit.rc2_timeout=900"
-    fi
 
     if test "$personality" = "minimal"; then
         RC1_PATH=""
         RC3_PATH=""
+    elif test "$personality" = "system"; then
+        # Pre-create broker rundir so we know it in advance and
+        # make_bootstrap_config() can use it for ipc:// socket paths.
+        BROKER_RUNDIR=$(mktemp --directory --tmpdir flux-system-XXXXXX)
+        sysopts=$(make_bootstrap_config \
+          $SHARNESS_TRASH_DIRECTORY $BROKER_RUNDIR $size)
+        # Place the re-executed test script trash within the first invocation's
+        # trash to preserve config files for broker restart in test
+        flags="${flags} --root=$SHARNESS_TRASH_DIRECTORY"
+        unset root
     elif test "$personality" != "full"; then
         RC1_PATH=$FLUX_SOURCE_DIR/t/rc/rc1-$personality
         RC3_PATH=$FLUX_SOURCE_DIR/t/rc/rc3-$personality
@@ -104,6 +241,11 @@ test_under_flux() {
         unset RC1_PATH
         unset RC3_PATH
     fi
+
+    if test -n "$root"; then
+        flags="${flags} --root=$root"
+    fi
+
     if test -n "$FLUX_TEST_VALGRIND" ; then
         VALGRIND_SUPPRESSIONS=${SHARNESS_TEST_SRCDIR}/valgrind/valgrind.supp
         valgrind="--wrap=libtool,e"
@@ -111,25 +253,29 @@ test_under_flux() {
         valgrind="$valgrind,--trace-children=no,--child-silent-after-fork=yes"
         valgrind="$valgrind,--leak-resolution=med,--error-exitcode=1"
         valgrind="$valgrind,--suppressions=${VALGRIND_SUPPRESSIONS}"
-        gracetime="-o,-g,10"
+    elif test -n "$FLUX_TEST_HEAPTRACK" ; then
+        valgrind="--wrap=heaptrack,--record-only"
+    elif test -n "$FLUX_TEST_WRAP" ; then
+        valgrind="$FLUX_TEST_WRAP"
     fi
     # Extend timeouts when running under AddressSanitizer
     if test_have_prereq ASAN; then
-        gracetime="-o,-g,10"
-        timeout="-o -Sinit.rc2_timeout=800"
         # Set log_path for ASan o/w errors from broker may be lost
         ASAN_OPTIONS=${ASAN_OPTIONS}:log_path=${TEST_NAME}.asan
     fi
-    logopts="-o -Slog-filename=${log_file},-Slog-forward-level=7"
+    logopts="-o -Slog-filename=${log_file} -Slog-forward-level=7"
     TEST_UNDER_FLUX_ACTIVE=t \
     TERM=${ORIGINAL_TERM} \
-      exec flux start --bootstrap=selfpmi --size=${size} \
+    TEST_UNDER_FLUX_PERSONALITY="${personality:-default}" \
+      exec flux start --test-size=${size} \
+                      ${BROKER_RUNDIR+--test-rundir=${BROKER_RUNDIR}} \
+                      ${BROKER_RUNDIR+--test-rundir-cleanup} \
                       ${RC1_PATH+-o -Sbroker.rc1_path=${RC1_PATH}} \
                       ${RC3_PATH+-o -Sbroker.rc3_path=${RC3_PATH}} \
+                      ${sysopts} \
                       ${logopts} \
-                      ${timeout} \
-                      ${gracetime} \
                       ${valgrind} \
+                      "$@" \
                      "sh $0 ${flags}"
 }
 
@@ -153,14 +299,35 @@ test_on_rank() {
     flux exec --rank=${ranks} "$@"
 }
 
+#  Note: Some versions of bash may cause the `flux` libtool wrapper script
+#  to reset the COLUMNS shell variable even if it is explicitly set for
+#  for testing purposes. This causes tests that check for output truncation
+#  based on COLUMNS to erroneously fail. (Note: this only seems to be the
+#  case when tests are run with --debug --verbose for unknown reasons.
+#
+#  Add a script for tests that use COLUMNS to check if the variable will
+#  be preserved across an invovation of flux(1) so they may set a prereq
+#  and skip tests that might erroneous fail if COLUMNS is not preserved.
+#
+test_columns_variable_preserved() {
+	local cols=$(COLUMNS=12 \
+	             flux python -c \
+	             "import shutil; print(shutil.get_terminal_size().columns)")
+	test "$cols" = "12"
+}
+
 #  Export a shorter name for this test
 TEST_NAME=$SHARNESS_TEST_NAME
 export TEST_NAME
 
 #  Test requirements for testsuite
+if ! command -v jq >/dev/null; then
+    error "jq is required for the flux-core testsuite"
+fi
 if ! run_timeout 10.0 lua -e 'require "posix"'; then
     error "failed to find lua posix module in path"
 fi
+jq=$(command -v jq)
 
 #  Some tests in flux don't work with --chain-lint, add a prereq for
 #   --no-chain-lint:
@@ -181,7 +348,26 @@ fi
 # Sanitize PMI_* environment for all tests. This allows commands like
 #  `flux broker` in tests to boot as singleton even when run under a
 #  job of an existing RM.
-for var in $(env | grep ^PMI); do unset ${var//=*}; done
-for var in $(env | grep ^SLURM); do unset ${var//=*}; done
+for var in $(env | grep ^PMI); do unset ${var%%=*}; done
+for var in $(env | grep ^SLURM); do unset ${var%%=*}; done
+
+# Sanitize Flux environment variables that should not be inherited by
+#  tests
+unset FLUX_SHELL_RC_PATH
+unset FLUX_RC_EXTRA
+unset FLUX_CONF_DIR
+unset FLUX_JOB_CC
+unset FLUX_F58_FORCE_ASCII
+
+# Individual tests that need to force local URI resolution should set
+#  this specifically. In general it breaks other URI tests:
+unset FLUX_URI_RESOLVE_LOCAL
+
+# Set XDG_CONFIG_DIRS and XDG_CONFIG_HOME to a nonexistent directory to
+#  avoid system or user configuration influencing tests for utilities
+#  that use flux.util.UtilConfig or other config classes utilizing
+#  the XDG base directory specification.
+export XDG_CONFIG_DIRS=/noexist
+export XDG_CONFIG_HOME=/noexist
 
 # vi: ts=4 sw=4 expandtab
diff --git a/t/sharness.d/heredoc.sh b/t/sharness.d/heredoc.sh
new file mode 100644
index 000000000000..9924077924e1
--- /dev/null
+++ b/t/sharness.d/heredoc.sh
@@ -0,0 +1,24 @@
+test_expect_success_hd() {
+	test "$#" = 2 && { test_prereq=$1; shift; } || test_prereq=
+	local TEST_CODE
+	# these extra newlines are intentional, and mimic the ones we get
+	# naturally in the non-heredoc case
+	TEST_CODE="\
+
+	$(cat)
+	"
+	test_expect_success "$test_prereq" "'$1'" "$TEST_CODE"
+}
+
+test_expect_failure_hd() {
+	test "$#" = 2 && { test_prereq=$1; shift; } || test_prereq=
+	local TEST_CODE
+	# these extra newlines are intentional, and mimic the ones we get
+	# naturally in the non-heredoc case
+	TEST_CODE="\
+
+	$(cat)
+	"
+	test_expect_failure "$test_prereq" "'$1'" "$TEST_CODE"
+}
+
diff --git a/t/sharness/vim/ftdetect/sharness.vim b/t/sharness/vim/ftdetect/sharness.vim
new file mode 100644
index 000000000000..f46f3367c74a
--- /dev/null
+++ b/t/sharness/vim/ftdetect/sharness.vim
@@ -0,0 +1 @@
+au BufRead,BufNewFile *.t set filetype=sh.sharness
diff --git a/t/sharness/vim/indent/sharness.vim b/t/sharness/vim/indent/sharness.vim
new file mode 100644
index 000000000000..1dfaaebd8814
--- /dev/null
+++ b/t/sharness/vim/indent/sharness.vim
@@ -0,0 +1,3 @@
+let b:sh_indent_options = { 'continuation-line': 0 }
+
+runtime! indent/sh.vim
diff --git a/t/sharness/vim/syntax/sharness.vim b/t/sharness/vim/syntax/sharness.vim
new file mode 100644
index 000000000000..abadb899120c
--- /dev/null
+++ b/t/sharness/vim/syntax/sharness.vim
@@ -0,0 +1,50 @@
+let b:is_bash=1
+runtime! syntax/sh.vim
+
+syn keyword shsStatement test_done
+syn keyword shsStatement test_set_editor test_set_index_version test_decode_color lf_to_nul nul_to_q q_to_nul q_to_cr q_to_tab qz_to_tab_space append_cr remove_cr generate_zero_bytes sane_unset test_tick test_pause debug test_commit test_merge test_commit_bulk test_chmod test_modebits test_unconfig test_config test_config_global write_script test_unset_prereq test_set_prereq test_have_prereq test_declared_prereq test_verify_prereq test_external test_external_without_stderr test_path_is_file test_path_is_dir test_path_exists test_dir_is_empty test_file_not_empty test_path_is_missing test_line_count test_file_size list_contains test_must_fail_acceptable test_must_fail test_might_fail test_expect_code test_i18ncmp test_i18ngrep verbose test_must_be_empty test_cmp_rev test_cmp_fspath test_seq test_when_finished test_atexit test_create_repo test_ln_s_add test_write_lines perl test_bool_env test_skip_or_die mingw_test_cmp test_env test_match_signal test_copy_bytes nongit depacketize hex2oct test_set_hash test_detect_hash test_oid_init test_oid_cache test_oid test_oid_to_path test_set_port test_bitmap_traversal test_path_is_hidden test_subcommand
+syn keyword shsStatement test_cmp test_cmp_config test_cmp_bin packetize
+
+syn region shsTest fold start="\<test_expect_\w\+\(_hd\)\?\>" end="$" contains=shsTestTitle
+syn region shsTest fold start="\<test_expect_\w\+\(_hd\)\?\>\s\+\<[A-Z_,]\+\>" end="$" contains=shsPrereq
+syn region shsTest fold start="\<test_lazy_prereq\>\s\+\<[A-Z_,]\+\>" end="$" contains=shsPrereqLazy
+
+syn keyword shsTestStatement contained containedin=shsTest test_expect_success test_expect_failure test_expect_unstable test_lazy_prereq test_expect_success_hd test_expect_failure_hd test_expect_unstable_hd
+
+syn region shsTestTitle contained start=' 'hs=s+1 end=' 'me=e-1 nextgroup=shsTestBody contains=shSingleQuote,shDoubleQuote
+
+" multiple line body
+syn region shsTestBody contained transparent excludenl matchgroup=shQuote start=+ '\\\?$+hs=s+1,rs=e end=+'$+ contains=@shSubShList
+syn region shsTestBody contained transparent excludenl matchgroup=shQuote start=+ "\\\?$+hs=s+1,rs=e end=+"$+ contains=@shSubShList
+syn region shsTestBody contained transparent excludenl matchgroup=shHDBody start=+ <<-\?'\z(\w\+\)'$+hs=e,rs=e end=+^\s*\z2$+ contains=@shSubShList
+
+" single line body
+syn region shsTestBody contained oneline transparent excludenl keepend matchgroup=shQuote start=+ '+hs=s+1 end=+'$+ contains=@shSubShList
+syn region shsTestBody contained oneline transparent excludenl keepend matchgroup=shQuote start=+ "+hs=s+1 end=+"$+ contains=@shSubShList
+
+" heredoc quotes
+syn region shHereDoc matchgroup=shRedir start="<<\s*\\\"\z(.\{-1,\}\)\\\"" matchgroup=shRedir end="^\z1\s*$"
+syn region shHereDoc matchgroup=shRedir start="<<-\s*\\\"\z(.\{-1,\}\)\\\"" matchgroup=shRedir end="^\s*\z1\s*$"
+syn region shHereDoc matchgroup=shRedir start="<<\s*\\\\\z([^ \t|>]\+\)" matchgroup=shRedir end="^\z1\s*$"
+syn region shHereDoc matchgroup=shRedir start="<<-\s*\\\\\z([^ \t|>]\+\)" matchgroup=shRedir end="^\s*\z1\s*$"
+syn region shHereDoc matchgroup=shRedir start="<<\s*\\\\\_$\_s*\z([^ \t|>]\+\)" matchgroup=shRedir end="^\z1\s*$" contains=@shDblQuoteList
+syn region shHereDoc matchgroup=shRedir start="<<-\s*\\\\\_$\_s*\z([^ \t|>]\+\)" matchgroup=shRedir end="^\s*\z1\s*$" contains=@shDblQuoteList
+syn region shHereDoc matchgroup=shRedir start="<<\s*\\\\\_$\_s*\\\\\z([^ \t|>]\+\)" matchgroup=shRedir end="^\z1\s*$"
+syn region shHereDoc matchgroup=shRedir start="<<-\s*\\\\\_$\_s*\\\\\z([^ \t|>]\+\)" matchgroup=shRedir end="^\s*\z1\s*$"
+syn region shHereDoc matchgroup=shRedir start="<<\s*\\\\\_$\_s*'\z(.\{-1,\}\)'" matchgroup=shRedir end="^\z1\s*$"
+syn region shHereDoc matchgroup=shRedir start="<<-\s*\\\\\_$\_s*'\z(.\{-1,\}\)'" matchgroup=shRedir end="^\s*\z1\s*$"
+syn region shHereDoc matchgroup=shRedir start="<<\s*\\\\\_$\_s*\\\"\z(.\{-1,\}\)\\\"" matchgroup=shRedir end="^\z1\s*$"
+syn region shHereDoc matchgroup=shRedir start="<<-\s*\\\\\_$\_s*\\\"\z(.\{-1,\}\)\\\"" matchgroup=shRedir end="^\s*\z1\s*$"
+
+syn match shsPrereq contained "\<[A-Z_,]\+\>" nextgroup=shsTestTitle
+syn match shsPrereqLazy contained "\<[A-Z_,]\+\>" nextgroup=shsTestBody
+
+syn cluster shCommandSubList add=shsTest,shsStatement
+
+hi def link shsStatement Statement
+hi def link shsTestStatement Function
+hi def link shsPrereq Identifier
+hi def link shsPrereqLazy shsPrereq
+hi def link shsTestBody shRedir
+
+let b:current_syntax='sharness'
diff --git a/t/shell/initrc/tests/0001-shell-info.lua b/t/shell/initrc/tests/0001-shell-info.lua
index d1f4b942cfd1..f76a7d0b1504 100644
--- a/t/shell/initrc/tests/0001-shell-info.lua
+++ b/t/shell/initrc/tests/0001-shell-info.lua
@@ -29,6 +29,8 @@ type_ok (info.rank, "number",
     "shell.info.rank is a number")
 type_ok (info.jobspec, "table",
      "info.jobspec is a table")
+type_ok (info.R, "table",
+    "info.R is a table")
 is (info.rank, 0,
     "shell.info.rank is expected value");
 ok (info.options.standalone,
@@ -38,6 +40,10 @@ jobspec = shell.info.jobspec
 type_ok (jobspec, "table",
     "shell.info.jobspec is a table")
 
+R = shell.info.R
+type_ok (R, "table",
+    "shell.info.R is a table")
+
 rankinfo = shell.rankinfo
 type_ok (rankinfo, "table",
     "shell.rankinfo is a table")
@@ -51,6 +57,9 @@ type_ok (rankinfo.resources, "table",
     "rankinfo.resources is a table")
 is (rankinfo.resources.cores, "0,1",
     "rankinfo.resources.cores is expected value")
+type_ok (rankinfo.taskids, "string",
+    "rankinfo.taskids is a string")
+diag ("  rankinfo.taskids is "..rankinfo.taskids)
 
 error_like ("i = task.info", "access task",
     "trying to access task causes error")
diff --git a/t/shell/initrc/tests/0003-plugin-override.lua b/t/shell/initrc/tests/0003-plugin-override.lua
index b0cc4556118c..02030700c862 100644
--- a/t/shell/initrc/tests/0003-plugin-override.lua
+++ b/t/shell/initrc/tests/0003-plugin-override.lua
@@ -10,7 +10,7 @@ plugin.register { name = "done",
 }
 pass ("registered done_testing() plugin")
 
--- Create a plugin that will be overidden
+-- Create a plugin that will be overridden
 plugin.register { name = "test",
     handlers = {
       { topic = "*",
diff --git a/t/shell/lptest.c b/t/shell/lptest.c
deleted file mode 100644
index ac4ab8ae5daa..000000000000
--- a/t/shell/lptest.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/************************************************************\
- * Copyright 2019 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-/* lptest.c - ripple test */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdlib.h>
-#include <stdio.h>
-
-void lptest (int length, int count)
-{
-    int i;
-    int j;
-
-    for (i = 0; i < count; i++) {
-        for (j = 0; j < length; j++)
-            putchar (0x21 + ((i + j) % 0x5e)); // charset: !(0x21) thru ~(0x7e)
-        putchar ('\n');
-    }
-}
-
-int main (int ac, char **av)
-{
-    int length = 79;
-    int count = 200;
-
-    if (ac > 3) {
-        fprintf (stderr, "Usage: %s [length] [count]\n", av[0]);
-        return 1;
-    }
-    if (ac > 2)
-        count = strtoul (av[2], NULL, 10);
-    if (ac > 1)
-        length = strtoul (av[1], NULL, 10);
-    lptest (length, count);
-    return 0;
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/t/shell/mpir.c b/t/shell/mpir.c
index 8ea68a7e16b2..10ed41872368 100644
--- a/t/shell/mpir.c
+++ b/t/shell/mpir.c
@@ -15,56 +15,100 @@
 #endif
 
 #include <flux/core.h>
+#include <flux/optparse.h>
 
 #include "src/common/libutil/log.h"
 #include "src/shell/mpir/proctable.h"
-
-struct proctable *proctable = NULL;
-MPIR_PROCDESC *MPIR_proctable = NULL;
-int MPIR_proctable_size = 0;
-
-static void set_mpir_proctable (const char *s)
+#include "src/cmd/job/mpir.h"
+
+extern struct proctable *proctable;
+extern MPIR_PROCDESC *MPIR_proctable;
+extern int MPIR_proctable_size;
+
+extern char MPIR_executable_path[256];
+extern char MPIR_server_arguments[1024];
+
+static struct optparse_option opts[] = {
+    { .name = "leader-rank",
+      .key = 'r',
+      .has_arg = 1,
+      .arginfo = "RANK",
+      .usage = "specify shell leader rank"
+    },
+    { .name = "service",
+      .key = 's',
+      .has_arg = 1,
+      .arginfo = "NAME",
+      .usage = "specify shell service NAME"
+    },
+    { .name = "tool-launch",
+      .key = 't',
+      .has_arg = 0,
+      .usage = "test tool launch via MPIR_executable_path",
+    },
+    OPTPARSE_TABLE_END
+};
+
+static void print_proctable (void)
 {
-    if (!(proctable = proctable_from_json_string (s)))
-        log_err_exit ("proctable_from_json_string");
-    MPIR_proctable = proctable_get_mpir_proctable (proctable,
-                                                   &MPIR_proctable_size);
-    if (!MPIR_proctable)
-        log_err_exit ("proctable_get_mpir_proctable");
+    json_t *o = proctable_to_json (proctable);
+    char *s = json_dumps (o, 0);
+    fprintf (stderr, "proctable=%s\n", s);
+    json_decref (o);
+    free (s);
 }
 
 int main (int ac, char **av)
 {
     int rank;
-    char topic [1024];
     flux_t *h = NULL;
-    flux_future_t *f = NULL;
-    const char *s = NULL;
     const char *service;
+    optparse_t *p;
+    int optindex;
 
     log_init ("mpir-test");
-
-    if (ac != 3)
-        log_msg_exit ("Usage: %s LEADER-RANK SERVICE\n", av [0]);
-    rank = atoi (av[1]);
-    service = av[2];
+    if (!(p = optparse_create ("mpir-test"))
+        || optparse_add_option_table (p, opts) != OPTPARSE_SUCCESS)
+        log_err_exit ("optparse_create");
+
+    if ((optindex = optparse_parse_args (p, ac, av)) < 0)
+        exit (1);
+
+    rank = optparse_get_int (p, "leader-rank", -1);
+    service = optparse_get_str (p, "service", NULL);
+    if (rank < 0 || service == NULL)
+        log_msg_exit ("--rank and --service are required");
+
+    if (optparse_hasopt (p, "tool-launch")) {
+        if (optindex == ac)
+            log_msg_exit ("--tool-launch requires specification of tool args");
+        /*  Set MPIR_executable_path */
+        snprintf (MPIR_executable_path,
+                  sizeof (MPIR_executable_path),
+                  "%s",
+                  av[optindex++]);
+        /*  Set MPIR_server_arguments */
+        int i = 0;
+        while (optindex < ac) {
+            int n = snprintf (MPIR_server_arguments+i,
+                              sizeof (MPIR_server_arguments) - i,
+                              "%s",
+                              av[optindex++]);
+            i += n + 1;
+        }
+    }
 
     if (!(h = flux_open (NULL, 0)))
         log_err_exit ("flux_open");
 
-    snprintf (topic, sizeof (topic), "%s.proctable", service);
-    if (!(f = flux_rpc_pack (h, topic, rank, 0, "{}")))
-        log_err_exit ("flux_rpc_pack");
-    if (flux_rpc_get (f, &s) < 0)
-        log_err_exit ("%s", topic);
-
-    fprintf (stderr, "proctable=%s\n", s);
+    mpir_setup_interface (h, 0, false, false, rank, service);
+    print_proctable ();
 
-    set_mpir_proctable (s);
+    flux_reactor_run (flux_get_reactor (h), 0);
 
     proctable_destroy (proctable);
-    flux_future_destroy (f);
     flux_close (h);
+    optparse_destroy (p);
     return 0;
 }
 
diff --git a/t/shell/output/1r1c.1.expected b/t/shell/output/1r1c.1.expected
index a3930972a3f7..d6b5ec1704a7 100644
--- a/t/shell/output/1r1c.1.expected
+++ b/t/shell/output/1r1c.1.expected
@@ -1,3 +1,3 @@
 Distributing 1 tasks across 1 nodes with 1 cores
 Used 1 nodes
-0: rank=0 ntasks=1 basis=0 cores=0
+0: rank=0 ntasks=1 cores=0
diff --git a/t/shell/output/1r1c2gpu.1.expected b/t/shell/output/1r1c2gpu.1.expected
index 552dac1b007c..3943ec35194a 100644
--- a/t/shell/output/1r1c2gpu.1.expected
+++ b/t/shell/output/1r1c2gpu.1.expected
@@ -1,3 +1,3 @@
 Distributing 1 tasks across 1 nodes with 1 cores 2 gpus
 Used 1 nodes
-0: rank=0 ntasks=1 basis=0 cores=0 gpus=0-1
+0: rank=0 ntasks=1 cores=0 gpus=0-1
diff --git a/t/shell/output/1r2c.1.expected b/t/shell/output/1r2c.1.expected
index 56b505a63504..e422f55ed33c 100644
--- a/t/shell/output/1r2c.1.expected
+++ b/t/shell/output/1r2c.1.expected
@@ -1,3 +1,3 @@
 Distributing 1 tasks across 1 nodes with 2 cores
 Used 1 nodes
-0: rank=0 ntasks=1 basis=0 cores=0-1
+0: rank=0 ntasks=1 cores=0-1
diff --git a/t/shell/output/1r2c.2.expected b/t/shell/output/1r2c.2.expected
index 7330ab67535c..2a7c46353dbd 100644
--- a/t/shell/output/1r2c.2.expected
+++ b/t/shell/output/1r2c.2.expected
@@ -1,3 +1,3 @@
 Distributing 2 tasks across 1 nodes with 2 cores
 Used 1 nodes
-0: rank=0 ntasks=2 basis=0 cores=0-1
+0: rank=0 ntasks=2 cores=0-1
diff --git a/t/shell/output/4r4c4r1c.16.expected b/t/shell/output/4r4c4r1c.16.expected
index 3255e86c7982..4a381d9425d6 100644
--- a/t/shell/output/4r4c4r1c.16.expected
+++ b/t/shell/output/4r4c4r1c.16.expected
@@ -1,10 +1,10 @@
 Distributing 16 tasks across 8 nodes with 20 cores
 Used 8 nodes
-0: rank=0 ntasks=3 basis=0 cores=0-3
-1: rank=1 ntasks=3 basis=3 cores=0-3
-2: rank=2 ntasks=3 basis=6 cores=0-3
-3: rank=3 ntasks=3 basis=9 cores=0-3
-4: rank=4 ntasks=1 basis=12 cores=0
-5: rank=5 ntasks=1 basis=13 cores=0
-6: rank=6 ntasks=1 basis=14 cores=0
-7: rank=7 ntasks=1 basis=15 cores=0
+0: rank=0 ntasks=3 cores=0-3
+1: rank=1 ntasks=3 cores=0-3
+2: rank=2 ntasks=3 cores=0-3
+3: rank=3 ntasks=3 cores=0-3
+4: rank=4 ntasks=1 cores=0
+5: rank=5 ntasks=1 cores=0
+6: rank=6 ntasks=1 cores=0
+7: rank=7 ntasks=1 cores=0
diff --git a/t/shell/output/4r4c4r1c.20.expected b/t/shell/output/4r4c4r1c.20.expected
index e48f50eece36..a7b67ef303b4 100644
--- a/t/shell/output/4r4c4r1c.20.expected
+++ b/t/shell/output/4r4c4r1c.20.expected
@@ -1,10 +1,10 @@
 Distributing 20 tasks across 8 nodes with 20 cores
 Used 8 nodes
-0: rank=0 ntasks=4 basis=0 cores=0-3
-1: rank=1 ntasks=4 basis=4 cores=0-3
-2: rank=2 ntasks=4 basis=8 cores=0-3
-3: rank=3 ntasks=4 basis=12 cores=0-3
-4: rank=4 ntasks=1 basis=16 cores=0
-5: rank=5 ntasks=1 basis=17 cores=0
-6: rank=6 ntasks=1 basis=18 cores=0
-7: rank=7 ntasks=1 basis=19 cores=0
+0: rank=0 ntasks=4 cores=0-3
+1: rank=1 ntasks=4 cores=0-3
+2: rank=2 ntasks=4 cores=0-3
+3: rank=3 ntasks=4 cores=0-3
+4: rank=4 ntasks=1 cores=0
+5: rank=5 ntasks=1 cores=0
+6: rank=6 ntasks=1 cores=0
+7: rank=7 ntasks=1 cores=0
diff --git a/t/shell/output/4r4c4r1c.4.expected b/t/shell/output/4r4c4r1c.4.expected
index f22fd664743b..949c1ce2987d 100644
--- a/t/shell/output/4r4c4r1c.4.expected
+++ b/t/shell/output/4r4c4r1c.4.expected
@@ -1,10 +1,10 @@
 Distributing 4 tasks across 8 nodes with 20 cores
 Used 4 nodes
-0: rank=0 ntasks=1 basis=0 cores=0-3
-1: rank=1 ntasks=1 basis=1 cores=0-3
-2: rank=2 ntasks=1 basis=2 cores=0-3
-3: rank=3 ntasks=1 basis=3 cores=0-3
-4: rank=4 ntasks=0 basis=4 cores=0
-5: rank=5 ntasks=0 basis=4 cores=0
-6: rank=6 ntasks=0 basis=4 cores=0
-7: rank=7 ntasks=0 basis=4 cores=0
+0: rank=0 ntasks=1 cores=0-3
+1: rank=1 ntasks=1 cores=0-3
+2: rank=2 ntasks=1 cores=0-3
+3: rank=3 ntasks=1 cores=0-3
+4: rank=4 ntasks=0 cores=0
+5: rank=5 ntasks=0 cores=0
+6: rank=6 ntasks=0 cores=0
+7: rank=7 ntasks=0 cores=0
diff --git a/t/shell/output/4r4c4r1c.8.expected b/t/shell/output/4r4c4r1c.8.expected
index 70798180b3d2..96fa5e52ae0b 100644
--- a/t/shell/output/4r4c4r1c.8.expected
+++ b/t/shell/output/4r4c4r1c.8.expected
@@ -1,10 +1,10 @@
 Distributing 8 tasks across 8 nodes with 20 cores
 Used 8 nodes
-0: rank=0 ntasks=1 basis=0 cores=0-3
-1: rank=1 ntasks=1 basis=1 cores=0-3
-2: rank=2 ntasks=1 basis=2 cores=0-3
-3: rank=3 ntasks=1 basis=3 cores=0-3
-4: rank=4 ntasks=1 basis=4 cores=0
-5: rank=5 ntasks=1 basis=5 cores=0
-6: rank=6 ntasks=1 basis=6 cores=0
-7: rank=7 ntasks=1 basis=7 cores=0
+0: rank=0 ntasks=1 cores=0-3
+1: rank=1 ntasks=1 cores=0-3
+2: rank=2 ntasks=1 cores=0-3
+3: rank=3 ntasks=1 cores=0-3
+4: rank=4 ntasks=1 cores=0
+5: rank=5 ntasks=1 cores=0
+6: rank=6 ntasks=1 cores=0
+7: rank=7 ntasks=1 cores=0
diff --git a/t/shell/output/4r4c4r1c.9.expected b/t/shell/output/4r4c4r1c.9.expected
index e16d7bdc0c43..ece6cd97bb51 100644
--- a/t/shell/output/4r4c4r1c.9.expected
+++ b/t/shell/output/4r4c4r1c.9.expected
@@ -1,10 +1,10 @@
 Distributing 9 tasks across 8 nodes with 20 cores
 Used 8 nodes
-0: rank=0 ntasks=2 basis=0 cores=0-3
-1: rank=1 ntasks=1 basis=2 cores=0-3
-2: rank=2 ntasks=1 basis=3 cores=0-3
-3: rank=3 ntasks=1 basis=4 cores=0-3
-4: rank=4 ntasks=1 basis=5 cores=0
-5: rank=5 ntasks=1 basis=6 cores=0
-6: rank=6 ntasks=1 basis=7 cores=0
-7: rank=7 ntasks=1 basis=8 cores=0
+0: rank=0 ntasks=2 cores=0-3
+1: rank=1 ntasks=1 cores=0-3
+2: rank=2 ntasks=1 cores=0-3
+3: rank=3 ntasks=1 cores=0-3
+4: rank=4 ntasks=1 cores=0
+5: rank=5 ntasks=1 cores=0
+6: rank=6 ntasks=1 cores=0
+7: rank=7 ntasks=1 cores=0
diff --git a/t/shell/output/8r1c.4.expected b/t/shell/output/8r1c.4.expected
index ea4fe97833a1..5e5fff625b0e 100644
--- a/t/shell/output/8r1c.4.expected
+++ b/t/shell/output/8r1c.4.expected
@@ -1,10 +1,10 @@
 Distributing 4 tasks across 8 nodes with 8 cores
 Used 4 nodes
-0: rank=0 ntasks=1 basis=0 cores=0
-1: rank=1 ntasks=1 basis=1 cores=0
-2: rank=2 ntasks=1 basis=2 cores=0
-3: rank=3 ntasks=1 basis=3 cores=0
-4: rank=4 ntasks=0 basis=4 cores=0
-5: rank=5 ntasks=0 basis=4 cores=0
-6: rank=6 ntasks=0 basis=4 cores=0
-7: rank=7 ntasks=0 basis=4 cores=0
+0: rank=0 ntasks=1 cores=0
+1: rank=1 ntasks=1 cores=0
+2: rank=2 ntasks=1 cores=0
+3: rank=3 ntasks=1 cores=0
+4: rank=4 ntasks=0 cores=0
+5: rank=5 ntasks=0 cores=0
+6: rank=6 ntasks=0 cores=0
+7: rank=7 ntasks=0 cores=0
diff --git a/t/shell/output/8r1c.8.expected b/t/shell/output/8r1c.8.expected
index 96d5ce9544ff..67462fee6baf 100644
--- a/t/shell/output/8r1c.8.expected
+++ b/t/shell/output/8r1c.8.expected
@@ -1,10 +1,10 @@
 Distributing 8 tasks across 8 nodes with 8 cores
 Used 8 nodes
-0: rank=0 ntasks=1 basis=0 cores=0
-1: rank=1 ntasks=1 basis=1 cores=0
-2: rank=2 ntasks=1 basis=2 cores=0
-3: rank=3 ntasks=1 basis=3 cores=0
-4: rank=4 ntasks=1 basis=4 cores=0
-5: rank=5 ntasks=1 basis=5 cores=0
-6: rank=6 ntasks=1 basis=6 cores=0
-7: rank=7 ntasks=1 basis=7 cores=0
+0: rank=0 ntasks=1 cores=0
+1: rank=1 ntasks=1 cores=0
+2: rank=2 ntasks=1 cores=0
+3: rank=3 ntasks=1 cores=0
+4: rank=4 ntasks=1 cores=0
+5: rank=5 ntasks=1 cores=0
+6: rank=6 ntasks=1 cores=0
+7: rank=7 ntasks=1 cores=0
diff --git a/t/shell/output/per-resource/1r1c.core.1.err b/t/shell/output/per-resource/1r1c.core.1.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r1c.core.1.expected b/t/shell/output/per-resource/1r1c.core.1.expected
new file mode 100644
index 000000000000..56fabe0c9af2
--- /dev/null
+++ b/t/shell/output/per-resource/1r1c.core.1.expected
@@ -0,0 +1,3 @@
+Distributing 1 tasks per-core across 1 nodes with 1 cores
+Used 1 nodes
+0: rank=0 ntasks=1 cores=0
diff --git a/t/shell/output/per-resource/1r1c.core.2.err b/t/shell/output/per-resource/1r1c.core.2.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r1c.core.2.expected b/t/shell/output/per-resource/1r1c.core.2.expected
new file mode 100644
index 000000000000..96cea19e80b0
--- /dev/null
+++ b/t/shell/output/per-resource/1r1c.core.2.expected
@@ -0,0 +1,3 @@
+Distributing 2 tasks per-core across 1 nodes with 1 cores
+Used 1 nodes
+0: rank=0 ntasks=2 cores=0
diff --git a/t/shell/output/per-resource/1r1c.node.1.err b/t/shell/output/per-resource/1r1c.node.1.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r1c.node.1.expected b/t/shell/output/per-resource/1r1c.node.1.expected
new file mode 100644
index 000000000000..1aa6c9531515
--- /dev/null
+++ b/t/shell/output/per-resource/1r1c.node.1.expected
@@ -0,0 +1,3 @@
+Distributing 1 tasks per-node across 1 nodes with 1 cores
+Used 1 nodes
+0: rank=0 ntasks=1 cores=0
diff --git a/t/shell/output/per-resource/1r1c.node.2.err b/t/shell/output/per-resource/1r1c.node.2.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r1c.node.2.expected b/t/shell/output/per-resource/1r1c.node.2.expected
new file mode 100644
index 000000000000..465bbdb6cb23
--- /dev/null
+++ b/t/shell/output/per-resource/1r1c.node.2.expected
@@ -0,0 +1,3 @@
+Distributing 2 tasks per-node across 1 nodes with 1 cores
+Used 1 nodes
+0: rank=0 ntasks=2 cores=0
diff --git a/t/shell/output/per-resource/1r1c2gpu.core.1.err b/t/shell/output/per-resource/1r1c2gpu.core.1.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r1c2gpu.core.1.expected b/t/shell/output/per-resource/1r1c2gpu.core.1.expected
new file mode 100644
index 000000000000..602540049bf4
--- /dev/null
+++ b/t/shell/output/per-resource/1r1c2gpu.core.1.expected
@@ -0,0 +1,3 @@
+Distributing 1 tasks per-core across 1 nodes with 1 cores 2 gpus
+Used 1 nodes
+0: rank=0 ntasks=1 cores=0 gpus=0-1
diff --git a/t/shell/output/per-resource/1r1c2gpu.core.2.err b/t/shell/output/per-resource/1r1c2gpu.core.2.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r1c2gpu.core.2.expected b/t/shell/output/per-resource/1r1c2gpu.core.2.expected
new file mode 100644
index 000000000000..2518c0e0f5ea
--- /dev/null
+++ b/t/shell/output/per-resource/1r1c2gpu.core.2.expected
@@ -0,0 +1,3 @@
+Distributing 2 tasks per-core across 1 nodes with 1 cores 2 gpus
+Used 1 nodes
+0: rank=0 ntasks=2 cores=0 gpus=0-1
diff --git a/t/shell/output/per-resource/1r1c2gpu.node.1.err b/t/shell/output/per-resource/1r1c2gpu.node.1.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r1c2gpu.node.1.expected b/t/shell/output/per-resource/1r1c2gpu.node.1.expected
new file mode 100644
index 000000000000..03d7d7d2f996
--- /dev/null
+++ b/t/shell/output/per-resource/1r1c2gpu.node.1.expected
@@ -0,0 +1,3 @@
+Distributing 1 tasks per-node across 1 nodes with 1 cores 2 gpus
+Used 1 nodes
+0: rank=0 ntasks=1 cores=0 gpus=0-1
diff --git a/t/shell/output/per-resource/1r1c2gpu.node.2.err b/t/shell/output/per-resource/1r1c2gpu.node.2.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r1c2gpu.node.2.expected b/t/shell/output/per-resource/1r1c2gpu.node.2.expected
new file mode 100644
index 000000000000..d4cfc4cc2046
--- /dev/null
+++ b/t/shell/output/per-resource/1r1c2gpu.node.2.expected
@@ -0,0 +1,3 @@
+Distributing 2 tasks per-node across 1 nodes with 1 cores 2 gpus
+Used 1 nodes
+0: rank=0 ntasks=2 cores=0 gpus=0-1
diff --git a/t/shell/output/per-resource/1r2c.core.1.err b/t/shell/output/per-resource/1r2c.core.1.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r2c.core.1.expected b/t/shell/output/per-resource/1r2c.core.1.expected
new file mode 100644
index 000000000000..deb83ebaf562
--- /dev/null
+++ b/t/shell/output/per-resource/1r2c.core.1.expected
@@ -0,0 +1,3 @@
+Distributing 1 tasks per-core across 1 nodes with 2 cores
+Used 1 nodes
+0: rank=0 ntasks=2 cores=0-1
diff --git a/t/shell/output/per-resource/1r2c.core.2.err b/t/shell/output/per-resource/1r2c.core.2.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r2c.core.2.expected b/t/shell/output/per-resource/1r2c.core.2.expected
new file mode 100644
index 000000000000..92e882c0f306
--- /dev/null
+++ b/t/shell/output/per-resource/1r2c.core.2.expected
@@ -0,0 +1,3 @@
+Distributing 2 tasks per-core across 1 nodes with 2 cores
+Used 1 nodes
+0: rank=0 ntasks=4 cores=0-1
diff --git a/t/shell/output/per-resource/1r2c.node.1.err b/t/shell/output/per-resource/1r2c.node.1.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r2c.node.1.expected b/t/shell/output/per-resource/1r2c.node.1.expected
new file mode 100644
index 000000000000..f5e462b21bd6
--- /dev/null
+++ b/t/shell/output/per-resource/1r2c.node.1.expected
@@ -0,0 +1,3 @@
+Distributing 1 tasks per-node across 1 nodes with 2 cores
+Used 1 nodes
+0: rank=0 ntasks=1 cores=0-1
diff --git a/t/shell/output/per-resource/1r2c.node.2.err b/t/shell/output/per-resource/1r2c.node.2.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/1r2c.node.2.expected b/t/shell/output/per-resource/1r2c.node.2.expected
new file mode 100644
index 000000000000..804958f58275
--- /dev/null
+++ b/t/shell/output/per-resource/1r2c.node.2.expected
@@ -0,0 +1,3 @@
+Distributing 2 tasks per-node across 1 nodes with 2 cores
+Used 1 nodes
+0: rank=0 ntasks=2 cores=0-1
diff --git a/t/shell/output/per-resource/4r4c4r1c.core.1.err b/t/shell/output/per-resource/4r4c4r1c.core.1.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/4r4c4r1c.core.1.expected b/t/shell/output/per-resource/4r4c4r1c.core.1.expected
new file mode 100644
index 000000000000..296d6ead314d
--- /dev/null
+++ b/t/shell/output/per-resource/4r4c4r1c.core.1.expected
@@ -0,0 +1,10 @@
+Distributing 1 tasks per-core across 8 nodes with 20 cores
+Used 8 nodes
+0: rank=0 ntasks=4 cores=0-3
+1: rank=1 ntasks=4 cores=0-3
+2: rank=2 ntasks=4 cores=0-3
+3: rank=3 ntasks=4 cores=0-3
+4: rank=4 ntasks=1 cores=0
+5: rank=5 ntasks=1 cores=0
+6: rank=6 ntasks=1 cores=0
+7: rank=7 ntasks=1 cores=0
diff --git a/t/shell/output/per-resource/4r4c4r1c.core.2.err b/t/shell/output/per-resource/4r4c4r1c.core.2.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/4r4c4r1c.core.2.expected b/t/shell/output/per-resource/4r4c4r1c.core.2.expected
new file mode 100644
index 000000000000..bdf45d06bbc2
--- /dev/null
+++ b/t/shell/output/per-resource/4r4c4r1c.core.2.expected
@@ -0,0 +1,10 @@
+Distributing 2 tasks per-core across 8 nodes with 20 cores
+Used 8 nodes
+0: rank=0 ntasks=8 cores=0-3
+1: rank=1 ntasks=8 cores=0-3
+2: rank=2 ntasks=8 cores=0-3
+3: rank=3 ntasks=8 cores=0-3
+4: rank=4 ntasks=2 cores=0
+5: rank=5 ntasks=2 cores=0
+6: rank=6 ntasks=2 cores=0
+7: rank=7 ntasks=2 cores=0
diff --git a/t/shell/output/per-resource/4r4c4r1c.node.1.err b/t/shell/output/per-resource/4r4c4r1c.node.1.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/4r4c4r1c.node.1.expected b/t/shell/output/per-resource/4r4c4r1c.node.1.expected
new file mode 100644
index 000000000000..6d1f724ccd03
--- /dev/null
+++ b/t/shell/output/per-resource/4r4c4r1c.node.1.expected
@@ -0,0 +1,10 @@
+Distributing 1 tasks per-node across 8 nodes with 20 cores
+Used 8 nodes
+0: rank=0 ntasks=1 cores=0-3
+1: rank=1 ntasks=1 cores=0-3
+2: rank=2 ntasks=1 cores=0-3
+3: rank=3 ntasks=1 cores=0-3
+4: rank=4 ntasks=1 cores=0
+5: rank=5 ntasks=1 cores=0
+6: rank=6 ntasks=1 cores=0
+7: rank=7 ntasks=1 cores=0
diff --git a/t/shell/output/per-resource/4r4c4r1c.node.2.err b/t/shell/output/per-resource/4r4c4r1c.node.2.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/4r4c4r1c.node.2.expected b/t/shell/output/per-resource/4r4c4r1c.node.2.expected
new file mode 100644
index 000000000000..4805876fda22
--- /dev/null
+++ b/t/shell/output/per-resource/4r4c4r1c.node.2.expected
@@ -0,0 +1,10 @@
+Distributing 2 tasks per-node across 8 nodes with 20 cores
+Used 8 nodes
+0: rank=0 ntasks=2 cores=0-3
+1: rank=1 ntasks=2 cores=0-3
+2: rank=2 ntasks=2 cores=0-3
+3: rank=3 ntasks=2 cores=0-3
+4: rank=4 ntasks=2 cores=0
+5: rank=5 ntasks=2 cores=0
+6: rank=6 ntasks=2 cores=0
+7: rank=7 ntasks=2 cores=0
diff --git a/t/shell/output/per-resource/8r1c.core.1.err b/t/shell/output/per-resource/8r1c.core.1.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/8r1c.core.1.expected b/t/shell/output/per-resource/8r1c.core.1.expected
new file mode 100644
index 000000000000..3a0cfbcb7944
--- /dev/null
+++ b/t/shell/output/per-resource/8r1c.core.1.expected
@@ -0,0 +1,10 @@
+Distributing 1 tasks per-core across 8 nodes with 8 cores
+Used 8 nodes
+0: rank=0 ntasks=1 cores=0
+1: rank=1 ntasks=1 cores=0
+2: rank=2 ntasks=1 cores=0
+3: rank=3 ntasks=1 cores=0
+4: rank=4 ntasks=1 cores=0
+5: rank=5 ntasks=1 cores=0
+6: rank=6 ntasks=1 cores=0
+7: rank=7 ntasks=1 cores=0
diff --git a/t/shell/output/per-resource/8r1c.core.2.err b/t/shell/output/per-resource/8r1c.core.2.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/8r1c.core.2.expected b/t/shell/output/per-resource/8r1c.core.2.expected
new file mode 100644
index 000000000000..e78a771751de
--- /dev/null
+++ b/t/shell/output/per-resource/8r1c.core.2.expected
@@ -0,0 +1,10 @@
+Distributing 2 tasks per-core across 8 nodes with 8 cores
+Used 8 nodes
+0: rank=0 ntasks=2 cores=0
+1: rank=1 ntasks=2 cores=0
+2: rank=2 ntasks=2 cores=0
+3: rank=3 ntasks=2 cores=0
+4: rank=4 ntasks=2 cores=0
+5: rank=5 ntasks=2 cores=0
+6: rank=6 ntasks=2 cores=0
+7: rank=7 ntasks=2 cores=0
diff --git a/t/shell/output/per-resource/8r1c.node.1.err b/t/shell/output/per-resource/8r1c.node.1.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/8r1c.node.1.expected b/t/shell/output/per-resource/8r1c.node.1.expected
new file mode 100644
index 000000000000..bd3907c4afe7
--- /dev/null
+++ b/t/shell/output/per-resource/8r1c.node.1.expected
@@ -0,0 +1,10 @@
+Distributing 1 tasks per-node across 8 nodes with 8 cores
+Used 8 nodes
+0: rank=0 ntasks=1 cores=0
+1: rank=1 ntasks=1 cores=0
+2: rank=2 ntasks=1 cores=0
+3: rank=3 ntasks=1 cores=0
+4: rank=4 ntasks=1 cores=0
+5: rank=5 ntasks=1 cores=0
+6: rank=6 ntasks=1 cores=0
+7: rank=7 ntasks=1 cores=0
diff --git a/t/shell/output/per-resource/8r1c.node.2.err b/t/shell/output/per-resource/8r1c.node.2.err
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/t/shell/output/per-resource/8r1c.node.2.expected b/t/shell/output/per-resource/8r1c.node.2.expected
new file mode 100644
index 000000000000..05fdabde1051
--- /dev/null
+++ b/t/shell/output/per-resource/8r1c.node.2.expected
@@ -0,0 +1,10 @@
+Distributing 2 tasks per-node across 8 nodes with 8 cores
+Used 8 nodes
+0: rank=0 ntasks=2 cores=0
+1: rank=1 ntasks=2 cores=0
+2: rank=2 ntasks=2 cores=0
+3: rank=3 ntasks=2 cores=0
+4: rank=4 ntasks=2 cores=0
+5: rank=5 ntasks=2 cores=0
+6: rank=6 ntasks=2 cores=0
+7: rank=7 ntasks=2 cores=0
diff --git a/t/shell/plugins/conftest.c b/t/shell/plugins/conftest.c
index a51efbe16459..5355dfc79fe7 100644
--- a/t/shell/plugins/conftest.c
+++ b/t/shell/plugins/conftest.c
@@ -7,6 +7,7 @@
  *
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
+#define FLUX_SHELL_PLUGIN_NAME "conftest"
 
 #if HAVE_CONFIG_H
 #include "config.h"
diff --git a/t/shell/plugins/dummy.c b/t/shell/plugins/dummy.c
index 59839adeead6..0e0190a1fbae 100644
--- a/t/shell/plugins/dummy.c
+++ b/t/shell/plugins/dummy.c
@@ -7,6 +7,7 @@
  *
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
+#define FLUX_SHELL_PLUGIN_NAME "dummy"
 
 #if HAVE_CONFIG_H
 #include "config.h"
diff --git a/t/shell/plugins/getopt.c b/t/shell/plugins/getopt.c
index e4961c3fd95a..ac4bc3155da3 100644
--- a/t/shell/plugins/getopt.c
+++ b/t/shell/plugins/getopt.c
@@ -7,6 +7,7 @@
  *
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
+#define FLUX_SHELL_PLUGIN_NAME "getopt"
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -17,6 +18,7 @@
 #include <flux/shell.h>
 
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 static int die (const char *fmt, ...)
 {
@@ -72,7 +74,7 @@ static int check_getopt (flux_plugin_t *p,
         && result == 42,
         "setopt worked and set integer value");
 
-    if (strcmp (topic, "shell.exit") == 0 || strcmp (topic, "task.exec") == 0)
+    if (streq (topic, "shell.exit") || streq (topic, "task.exec"))
         return exit_status () == 0 ? 0 : -1;
     return 0;
 }
diff --git a/t/shell/plugins/invalid-args.c b/t/shell/plugins/invalid-args.c
index f2a3e4f2b35d..b1177b4b83e3 100644
--- a/t/shell/plugins/invalid-args.c
+++ b/t/shell/plugins/invalid-args.c
@@ -7,6 +7,7 @@
  *
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
+#define FLUX_SHELL_PLUGIN_NAME "invalid-args"
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -18,6 +19,7 @@
 #include <flux/shell.h>
 
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 static int die (const char *fmt, ...)
 {
@@ -73,6 +75,8 @@ static int shell_cb (flux_plugin_t *p,
         "flux_shell_unsetenv (NULL, 'foo') returns EINVAL");
     ok (flux_shell_unsetenv (shell, NULL) < 0 && errno == EINVAL,
         "flux_shell_unsetenv (shell, NULL) returns EINVAL");
+    ok (flux_shell_unsetenv (shell, "MissingEnvVar") < 0 && errno == ENOENT,
+        "flux_shell_unsetenv (shell, MissingEnvVar) returns ENOENT");
 
     ok (flux_shell_setenvf (NULL, 0, "foo", "bar") < 0 && errno == EINVAL,
         "flux_shell_setenvf (NULL, ...) returns EINVAL");
@@ -86,6 +90,11 @@ static int shell_cb (flux_plugin_t *p,
     ok (flux_shell_get_environ (shell, NULL) < 0 && errno == EINVAL,
         "flux_shell_get_environ with NULL json_str returns EINVAL");
 
+    ok (flux_shell_get_hwloc_xml (NULL, NULL) < 0 && errno == EINVAL,
+        "flux_shell_get_hwloc_xml with NULL args returns EINVAL");
+    ok (flux_shell_get_hwloc_xml (shell, NULL) < 0 && errno == EINVAL,
+        "flux_shell_get_hwloc_xml with NULL xml pointer returns EINVAL");
+
     ok (flux_shell_get_info (NULL, NULL) < 0 && errno == EINVAL,
         "flux_shell_get_info with NUll arg returns EINVAL");
     ok (flux_shell_get_info (shell, NULL) < 0 && errno == EINVAL,
@@ -96,6 +105,19 @@ static int shell_cb (flux_plugin_t *p,
     ok (flux_shell_info_unpack (shell, NULL) < 0 && errno == EINVAL,
         "flux_shell_info_unpack with NULL fmt returns EINVAL");
 
+    ok (flux_shell_get_jobspec_info (NULL, NULL) < 0 && errno == EINVAL,
+        "flux_shell_get_info with NULL arg returns EINVAL");
+    ok (flux_shell_get_jobspec_info (shell, NULL) < 0 && errno == EINVAL,
+        "flux_shell_get_info with NULL json_str returns EINVAL");
+
+    ok (flux_shell_get_taskmap (NULL) == NULL && errno == EINVAL,
+        "flux_shell_get_taskmap (NULL) returns EINVAL");
+
+    ok (flux_shell_jobspec_info_unpack (NULL, NULL) < 0 && errno == EINVAL,
+        "flux_shell_info_unpack with NULL arg returns EINVAL");
+    ok (flux_shell_jobspec_info_unpack (shell, NULL) < 0 && errno == EINVAL,
+        "flux_shell_info_unpack with NULL fmt returns EINVAL");
+
     ok (flux_shell_get_rank_info (NULL, -1, NULL) < 0 && errno == EINVAL,
         "flux_shell_get_rank_info (NULL, ..) returns EINVAL");
     ok (flux_shell_get_rank_info (shell, -1, NULL) < 0 && errno == EINVAL,
@@ -173,7 +195,12 @@ static int shell_cb (flux_plugin_t *p,
     ok (flux_shell_task_next (NULL) == NULL && errno == EINVAL,
         "flux_shell_task_next (NULL) returns EINVAL");
 
-    if (strcmp (topic, "shell.init") == 0) {
+    ok (flux_shell_mustache_render (NULL, NULL) == NULL && errno == EINVAL,
+        "flux_shell_mustache_render (NULL, NULL) returns EINVAL");
+    ok (flux_shell_mustache_render (shell, NULL) == NULL && errno == EINVAL,
+        "flux_shell_mustache_render (shell, NULL) returns EINVAL");
+
+    if (streq (topic, "shell.init")) {
         ok (flux_shell_current_task (NULL) == NULL && errno == EINVAL,
             "flux_shell_current_task with NULL shell returns EINVAL");
         errno = 0;
@@ -184,7 +211,7 @@ static int shell_cb (flux_plugin_t *p,
         ok (flux_shell_task_next (shell) == NULL && errno == EAGAIN,
             "flux_shell_task_next (shell) in shell.init returns EAGAIN");
     }
-    if (strcmp (topic, "shell.exit") == 0)
+    if (streq (topic, "shell.exit"))
         return exit_status () == 0 ? 0 : -1;
     return 0;
 }
@@ -221,7 +248,7 @@ static int task_cb (flux_plugin_t *p,
         && errno == EINVAL,
         "flux_shell_task_channel_subscribe with NULL args returns EINVAL");
 
-    if (strcmp (topic, "task.exec") == 0)
+    if (streq (topic, "task.exec"))
         return exit_status () == 0 ? 0 : -1;
     return 0;
 }
diff --git a/t/shell/plugins/jobspec-info.c b/t/shell/plugins/jobspec-info.c
new file mode 100644
index 000000000000..31d082ed6224
--- /dev/null
+++ b/t/shell/plugins/jobspec-info.c
@@ -0,0 +1,113 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+#define FLUX_SHELL_PLUGIN_NAME "jobspec-info"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <errno.h>
+
+#include <jansson.h>
+#include <flux/core.h>
+#include <flux/shell.h>
+
+#include "src/common/libtap/tap.h"
+
+static int die (const char *fmt, ...)
+{
+    va_list ap;
+    va_start (ap, fmt);
+    vfprintf (stderr, fmt, ap);
+    va_end (ap);
+    return -1;
+}
+
+static int check_jobspec_info (flux_plugin_t *p,
+                               const char *topic,
+                               flux_plugin_arg_t *args,
+                               void *data)
+{
+    int version;
+    int ntasks, ntasks_expected;
+    int nslots, nslots_expected;
+    int cores_per_slot, cps_expected;
+    int nnodes, nnodes_expected;
+    int slots_per_node, spn_expected;
+    char *json_str = NULL;
+    int rc;
+
+    flux_shell_t *shell = flux_plugin_get_shell (p);
+    if (!p)
+        return die ("flux_plugin_get_shell\n");
+
+    rc = flux_shell_get_jobspec_info (shell, &json_str);
+    ok (rc == 0,
+        "flux_shell_get_jobspec_info works");
+    ok (json_str && strlen (json_str) > 0,
+        "flux_shell_get_jobspec_info returns a JSON string with len > 0");
+
+    rc = flux_shell_getopt_unpack (shell,
+                                   "jobspec_info",
+                                   "{s:i s:i s:i s:i s:i}",
+                                   "ntasks", &ntasks_expected,
+                                   "nnodes", &nnodes_expected,
+                                   "nslots", &nslots_expected,
+                                   "cores_per_slot", &cps_expected,
+                                   "slots_per_node", &spn_expected);
+    if (rc < 0)
+        return die ("flux_shell_getopt_unpack: %s\n", strerror (errno));
+
+    rc = flux_shell_jobspec_info_unpack (shell,
+                                         "{s:i s:i s:i s:i s:i s:i}",
+                                         "version", &version,
+                                         "ntasks", &ntasks,
+                                         "nslots", &nslots,
+                                         "cores_per_slot", &cores_per_slot,
+                                         "nnodes", &nnodes,
+                                         "slots_per_node", &slots_per_node);
+    ok (rc == 0,
+        "flux_jobspec_info_unpack works");
+
+    ok (version == 1,
+        "version is reported as 1");
+    ok (ntasks == ntasks_expected,
+        "ntasks (%d) has expected value (%d)", ntasks, ntasks_expected);
+    ok (nnodes == nnodes_expected,
+        "nnodes (%d) has expected value (%d)", nnodes, nnodes_expected);
+    ok (nslots == nslots_expected,
+        "nslots (%d) has expected value (%d)", nslots, nslots_expected);
+    ok (cores_per_slot == cps_expected,
+        "cores_per_slot (%d) has expected value (%d)",
+        cores_per_slot,
+        cps_expected);
+    ok (slots_per_node == spn_expected,
+        "slots_per_node (%d) has expected value (%d)",
+        slots_per_node,
+        spn_expected);
+
+    return exit_status () == 0 ? 0 : -1;
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    plan (NO_PLAN);
+    ok (flux_plugin_add_handler (p, "shell.init",
+                                 check_jobspec_info,
+                                 NULL) == 0,
+        "flux_plugin_add_handler works");
+    return 0;
+}
+
+/*
+ * vi: ts=4 sw=4 expandtab
+ */
diff --git a/t/shell/plugins/log.c b/t/shell/plugins/log.c
index 5c03f21f5b32..d6505008359a 100644
--- a/t/shell/plugins/log.c
+++ b/t/shell/plugins/log.c
@@ -7,6 +7,7 @@
  *
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
+#define FLUX_SHELL_PLUGIN_NAME "log"
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -18,6 +19,7 @@
 #include <flux/shell.h>
 
 #include "src/common/libtap/tap.h"
+#include "ccan/str/str.h"
 
 static int get_shell_rank (flux_shell_t *shell)
 {
@@ -43,7 +45,7 @@ static int check_shell_log (flux_plugin_t *p,
     int shell_rank;
     const char *s = NULL;
 
-    if (strcmp (topic, "shell.log") == 0)
+    if (streq (topic, "shell.log"))
         return 0;
 
     shell_rank = get_shell_rank (shell);
@@ -51,7 +53,7 @@ static int check_shell_log (flux_plugin_t *p,
         shell_die (1, "error parsing log-fatal-error");
 
     if (s) {
-        if (strcmp (s, topic) == 0 && shell_rank == 1)
+        if (streq (s, topic) && shell_rank == 1)
             shell_die (1, "log-fatal-error requested!");
         /* For log-fatal-error test, avoid the remaining log testing
          *  below on the non-fatal ranks
@@ -60,6 +62,7 @@ static int check_shell_log (flux_plugin_t *p,
     }
 
     shell_trace ("%s: trace message", topic);
+    shell_trace ("%s: long message: %.8192d", topic, 0);
     shell_debug ("%s: debug message", topic);
     shell_log ("%s: log message", topic);
     shell_warn ("%s: warn message", topic);
@@ -72,7 +75,7 @@ static int check_shell_log (flux_plugin_t *p,
     ok (shell_log_errno ("%s: log_errno message", topic) == -1,
         "shell_log_errno returns -1");
 
-    if (strcmp (topic, "shell.exit") == 0 || strcmp (topic, "task.exec") == 0)
+    if (streq (topic, "shell.exit") || streq (topic, "task.exec"))
         return exit_status () == 0 ? 0 : -1;
     return 0;
 }
@@ -85,13 +88,16 @@ static void destructor (void *arg)
 int flux_plugin_init (flux_plugin_t *p)
 {
     plan (NO_PLAN);
-    flux_plugin_set_name (p, "log");
+    flux_plugin_set_name (p, FLUX_SHELL_PLUGIN_NAME);
 
     /*  Set a dummy aux item to force our destructor to be called */
     flux_plugin_aux_set (p, NULL, p, destructor);
 
     ok (flux_plugin_add_handler (p, "*", check_shell_log, NULL) == 0,
         "flux_plugin_add_handler works");
+
+    ok (flux_shell_log_setlevel (-2, NULL) < 0 && errno == EINVAL,
+        "flux_shell_log_setlevel with invalid level fails");
     return 0;
 }
 
diff --git a/t/shell/plugins/taskmap-reverse.c b/t/shell/plugins/taskmap-reverse.c
new file mode 100644
index 000000000000..10732d97a259
--- /dev/null
+++ b/t/shell/plugins/taskmap-reverse.c
@@ -0,0 +1,78 @@
+/************************************************************\
+ * Copyright 2022 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* test taskmap plugin
+ */
+#define FLUX_SHELL_PLUGIN_NAME "taskmap.reverse"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <flux/shell.h>
+#include <flux/taskmap.h>
+
+static char *taskmap_reverse (const char *arg)
+{
+    struct taskmap *map = NULL;
+    struct taskmap *orig = NULL;
+    char *result = NULL;
+    int nnodes;
+    if (!(orig = taskmap_decode (arg, NULL))
+        || !(map = taskmap_create ()))
+        goto error;
+
+    nnodes = taskmap_nnodes (orig);
+    for (int i = nnodes - 1; i >= 0; i--) {
+        if (taskmap_append (map, i, 1, taskmap_ntasks (orig, i)) < 0)
+            goto error;
+    }
+    result = taskmap_encode (map, TASKMAP_ENCODE_WRAPPED);
+error:
+    taskmap_destroy (orig);
+    taskmap_destroy (map);
+    return result;
+}
+
+static int map_reverse (flux_plugin_t *p,
+                        const char *topic,
+                        flux_plugin_arg_t *args,
+                        void *data)
+{
+    const char *blockmap;
+    char *map;
+    if (flux_plugin_arg_unpack (args,
+                                FLUX_PLUGIN_ARG_IN,
+                                "{s:s}",
+                                "taskmap", &blockmap) < 0) {
+        shell_log_error ("unpack: %s", flux_plugin_arg_strerror (args));
+        return -1;
+    }
+    if (!(map = taskmap_reverse (blockmap))) {
+        shell_log_error ("failed to map tasks reverse");
+        return -1;
+    }
+    if (flux_plugin_arg_pack (args,
+                              FLUX_PLUGIN_ARG_OUT,
+                              "{s:s}",
+                              "taskmap", map) < 0)
+        return -1;
+    free (map);
+    return 0;
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    return flux_plugin_add_handler (p, "taskmap.reverse", map_reverse, NULL);
+}
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */
diff --git a/t/shell/plugins/test-event.c b/t/shell/plugins/test-event.c
index 04a5b73fcda0..a50902e0373e 100644
--- a/t/shell/plugins/test-event.c
+++ b/t/shell/plugins/test-event.c
@@ -7,6 +7,7 @@
  *
  * SPDX-License-Identifier: LGPL-3.0
 \************************************************************/
+#define FLUX_SHELL_PLUGIN_NAME "event-test"
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -47,7 +48,7 @@ static int check_event_context (flux_plugin_t *p,
 
 int flux_plugin_init (flux_plugin_t *p)
 {
-    flux_plugin_set_name (p, "event-test");
+    flux_plugin_set_name (p, FLUX_SHELL_PLUGIN_NAME);
     return flux_plugin_add_handler (p,
                                     "shell.init",
                                     check_event_context,
diff --git a/t/shell/rcalc.c b/t/shell/rcalc.c
index eff4f745c579..2969e8b20b25 100644
--- a/t/shell/rcalc.c
+++ b/t/shell/rcalc.c
@@ -17,32 +17,74 @@
 #include <string.h>
 #include <errno.h>
 
+#include <flux/optparse.h>
+
 #include "src/shell/rcalc.h"
 
+const char usage[] = "[OPTIONS] NTASKS";
+
+static struct optparse_option opts[] = {
+    { .name = "per-resource", .key = 'R',
+      .has_arg = 1, .arginfo = "NAME",
+      .usage = "Assign tasks per-resource instead of distributing. "
+               "NAME is name of resource (node or core)",
+    },
+    { .name = "cores-per-slot", .key = 'c',
+      .has_arg = 1, .arginfo = "N",
+      .usage = "Explicitly set the number of cores per task",
+    },
+    OPTPARSE_TABLE_END
+};
+
 int main (int ac, char **av)
 {
-    int i, ntasks;
-    rcalc_t *r;
+    int i, ntasks, optindex;
+    const char *rname = NULL;
+    rcalc_t *r = NULL;
+
+    optparse_t *p = optparse_create ("rcalc");
+    if (p == NULL
+        || optparse_add_option_table (p, opts) != OPTPARSE_SUCCESS
+        || optparse_set (p, OPTPARSE_USAGE, usage) != OPTPARSE_SUCCESS
+        || optparse_parse_args (p, ac, av) < 0)
+        exit (1);
+    optindex = optparse_option_index (p);
 
-    if (ac < 2) {
-        fprintf (stderr, "Usage: %s NTASKS\n", av[0]);
+    if (optindex != ac - 1) {
+        optparse_print_usage (p);
         exit (1);
     }
+    optparse_getopt (p, "per-resource", &rname);
+
     if (!(r = rcalc_createf (stdin)))  {
         fprintf (stderr, "Unable to create r: %s\n", strerror (errno));
         exit (1);
     }
-    if ((ntasks = strtoul (av[1], NULL, 10)) <= 0 || ntasks > 1e20) {
+    if ((ntasks = strtoul (av[optindex], NULL, 10)) <= 0 || ntasks > 1e20) {
         fprintf (stderr, "Invalid value for ntasks: %s\n", av[1]);
         exit (1);
     }
-    printf ("Distributing %d tasks across %d nodes with %d cores",
-            ntasks, rcalc_total_nodes (r), rcalc_total_cores (r));
+    printf ("Distributing %d tasks%s%s across %d nodes with %d cores",
+            ntasks,
+            rname ? " per-" : "",
+            rname ? rname : "",
+            rcalc_total_nodes (r),
+            rcalc_total_cores (r));
     if (rcalc_total_gpus (r))
         printf (" %d gpus", rcalc_total_gpus (r));
     printf ("\n");
 
-    if (rcalc_distribute (r, ntasks) < 0) {
+    if (rname) {
+        if (rcalc_distribute_per_resource (r, rname, ntasks) < 0) {
+            fprintf (stderr,
+                     "rcalc_distribute_per_resource: %s",
+                     strerror (errno));
+            exit (1);
+        }
+    }
+    else if (rcalc_distribute (r,
+                               ntasks,
+                               optparse_get_int (p, "cores-per-slot", 0)) < 0) {
         fprintf (stderr, "rcalc_distribute: %s\n", strerror (errno));
         exit (1);
     }
@@ -55,13 +97,14 @@ int main (int ac, char **av)
                      i, strerror (errno));
             exit (1);
         }
-        printf ("%d: rank=%d ntasks=%d basis=%d cores=%s",
-                ri.nodeid, ri.rank, ri.ntasks, ri.global_basis, ri.cores);
+        printf ("%d: rank=%d ntasks=%d cores=%s",
+                ri.nodeid, ri.rank, ri.ntasks, ri.cores);
         if (strlen(ri.gpus))
             printf (" gpus=%s", ri.gpus);
         printf ("\n");
     }
     rcalc_destroy (r);
+    optparse_destroy (p);
     return (0);
 }
 
diff --git a/t/shmem/backtoback.c b/t/shmem/backtoback.c
deleted file mode 100644
index 4fec09dbdf7e..000000000000
--- a/t/shmem/backtoback.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/************************************************************\
- * Copyright 2014 Lawrence Livermore National Security, LLC
- * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
- *
- * This file is part of the Flux resource manager framework.
- * For details, see https://github.com/flux-framework.
- *
- * SPDX-License-Identifier: LGPL-3.0
-\************************************************************/
-
-#include <errno.h>
-#include <czmq.h>
-#include <flux/core.h>
-
-#include "src/common/libutil/xzmalloc.h"
-#include "src/common/libtap/tap.h"
-
-int main (int argc, char *argv[])
-{
-    flux_t *h_cli, *h_srv;
-    flux_msg_t *msg;
-    int type;
-
-    plan (NO_PLAN);
-
-    ok ((h_srv = flux_open ("shmem://test&bind", 0)) != NULL,
-        "created server handle");
-    ok ((h_cli = flux_open ("shmem://test&connect", 0)) != NULL,
-        "created client handle");
-    if (!h_cli || !h_srv)
-        BAIL_OUT ("can't continue without client or server handle");
-
-    ok ((msg = flux_msg_create (FLUX_MSGTYPE_REQUEST)) != NULL,
-        "created test request");
-    ok (flux_send (h_cli, msg, 0) == 0,
-        "sent request to server");
-    flux_msg_destroy (msg);
-
-    ok ((msg = flux_recv (h_srv, FLUX_MATCH_ANY, 0)) != NULL,
-        "server received request");
-    ok (flux_msg_get_type (msg, &type) == 0 && type == FLUX_MSGTYPE_REQUEST,
-        "message is correct type");
-    flux_msg_destroy (msg);
-
-    ok ((msg = flux_msg_create (FLUX_MSGTYPE_RESPONSE)) != NULL,
-        "created test response");
-    ok (flux_send (h_srv, msg, 0) == 0,
-        "sent response to client");
-    flux_msg_destroy (msg);
-
-    ok ((msg = flux_recv (h_cli, FLUX_MATCH_ANY, 0)) != NULL,
-        "client received response");
-    ok (flux_msg_get_type (msg, &type) == 0 && type == FLUX_MSGTYPE_RESPONSE,
-        "message is correct type");
-    flux_msg_destroy (msg);
-
-    flux_close (h_cli);
-    flux_close (h_srv);
-
-    done_testing();
-    return (0);
-}
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-
diff --git a/t/stats/stats-basic.c b/t/stats/stats-basic.c
new file mode 100644
index 000000000000..5959e1719f61
--- /dev/null
+++ b/t/stats/stats-basic.c
@@ -0,0 +1,78 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <flux/jobtap.h>
+#include <time.h>
+
+#include "src/common/libutil/monotime.h"
+
+struct cb_data {
+    ssize_t cleanup;
+    ssize_t inactive;
+    struct timespec ts;
+};
+
+static int state_cb (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *arg)
+{
+    struct cb_data *data = arg;
+    flux_job_state_t state;
+    flux_job_state_t prev_state = 4096;
+    flux_t *h = flux_jobtap_get_flux (p);
+
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                               "{s:i s?i}",
+                               "state", &state,
+                               "prev_state", &prev_state) < 0) {
+        flux_log (h,
+                 LOG_ERR,
+                 "flux_plugin_arg_unpack: %s",
+                 flux_plugin_arg_strerror(args));
+        return -1;
+    }
+
+    switch (state) {
+        case FLUX_JOB_STATE_CLEANUP:
+            flux_stats_count (h, "cleanup.count", ++data->cleanup);
+            monotime (&data->ts);
+            break;
+        case FLUX_JOB_STATE_INACTIVE:
+            flux_stats_timing (h, "cleanup.timing", monotime_since (data->ts));
+            flux_stats_count (h, "inactive.count", ++data->inactive);
+            break;
+        default:
+            break;
+    }
+
+    return 0;
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    struct cb_data *data;
+    flux_t *h;
+    if (!(h = flux_jobtap_get_flux (p)))
+        return -1;
+    if (!(data = calloc (1, sizeof (*data))))
+        return -1;
+    if (flux_plugin_aux_set (p, "data", data, free) < 0)
+        return -1;
+
+    flux_stats_set_prefix (h, "flux.job.state");
+    flux_stats_set_period (h, 1.0);
+
+    return flux_plugin_add_handler (p, "job.state.*", state_cb, data);
+}
diff --git a/t/stats/stats-immediate.c b/t/stats/stats-immediate.c
new file mode 100644
index 000000000000..9f3ddac94f54
--- /dev/null
+++ b/t/stats/stats-immediate.c
@@ -0,0 +1,86 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <flux/core.h>
+#include <flux/jobtap.h>
+#include <time.h>
+
+#include "src/common/libutil/monotime.h"
+
+struct cb_data {
+    ssize_t inactive;
+    struct timespec ts;
+};
+
+static int state_cb (flux_plugin_t *p,
+                     const char *topic,
+                     flux_plugin_arg_t *args,
+                     void *arg)
+{
+    struct cb_data *data = arg;
+    flux_job_state_t state;
+    flux_job_state_t prev_state = 4096;
+    flux_t *h = flux_jobtap_get_flux (p);
+
+
+    if (flux_plugin_arg_unpack (args, FLUX_PLUGIN_ARG_IN,
+                               "{s:i s?i}",
+                               "state", &state,
+                               "prev_state", &prev_state) < 0) {
+        flux_log (h,
+                 LOG_ERR,
+                 "flux_plugin_arg_unpack: %s",
+                 flux_plugin_arg_strerror(args));
+        return -1;
+    }
+
+    flux_stats_gauge_inc (h, flux_job_statetostr (state, "L"), 1);
+    flux_stats_gauge_inc (h, flux_job_statetostr (prev_state, "L"), -1);
+
+    switch (state) {
+        case FLUX_JOB_STATE_CLEANUP:
+            monotime (&data->ts);
+            break;
+        case FLUX_JOB_STATE_INACTIVE:
+            flux_stats_timing (h, "cleanup.timing", monotime_since (data->ts));
+            flux_stats_count (h, "inactive.count", ++data->inactive);
+            break;
+        default:
+            break;
+    }
+
+    return 0;
+}
+
+int flux_plugin_init (flux_plugin_t *p)
+{
+    struct cb_data *data;
+    flux_t *h;
+    if (!(h = flux_jobtap_get_flux (p)))
+        return -1;
+    if (!(data = calloc (1, sizeof (*data))))
+        return -1;
+    if (flux_plugin_aux_set (p, "data", data, free) < 0)
+        return -1;
+
+    flux_stats_set_prefix (h, "flux.job.state.immediate");
+    flux_stats_set_period (h, 0.0);
+
+    // try setting a prefix with a length longer than the limit (127)
+    // and expect it to remain as "flux.job.stats.immediate"
+    flux_stats_set_prefix (h, "aQmi173rvgumStDdMZdwtJtpLLVJOUXol2aDndev/XsH/gM\
+wlPz/vMZhajJWGctwJZa1uFoAoINjwITPvezGoQDb/9DD3vkPcknN+f/u3vSc0tg/+3aFTONhUIomK\
+B4qiSKSotbtZl3Ewe2Oh+wI+nuG3/ebqIXSoEXjIFOB7vvGA==");
+
+    return flux_plugin_add_handler (p, "job.state.*", state_cb, data);
+}
diff --git a/t/system/0001-basic.t b/t/system/0001-basic.t
new file mode 100755
index 000000000000..e484225e8f99
--- /dev/null
+++ b/t/system/0001-basic.t
@@ -0,0 +1,24 @@
+#
+#  Basic system instance sanity checks
+#
+test_expect_success 'system instance runs job as current uid' '
+	jobid=$(flux submit id -u) &&
+	result=$(flux job attach $jobid) &&
+	test_debug "echo Job ran with userid $result" &&
+	test $result -eq $(id -u) &&
+	test $(flux getattr security.owner) -ne $result
+'
+test_expect_success 'flux jobs lists job with correct userid' '
+	test $(flux jobs -no {userid} $jobid) -eq $(id -u)
+'
+# flux-framework/flux-core#5530
+test_expect_success 'flux proxy can submit jobs to system instance' '
+	flux proxy $(flux getattr local-uri) flux submit true
+'
+test_expect_success 'flux-shell limits kvs output to 10M for guest jobs' '
+	dd if=/dev/urandom bs=10240 count=800 | base64 --wrap 79 >large.in &&
+	flux run -vvv cat large.in >large.out 2>trunc.err &&
+	ls -lh large* &&
+	test_debug "cat trunc.err" &&
+	grep "stdout.*truncated" trunc.err
+'
diff --git a/t/system/0002-exec-with-imp.t b/t/system/0002-exec-with-imp.t
new file mode 100644
index 000000000000..9cd838a96413
--- /dev/null
+++ b/t/system/0002-exec-with-imp.t
@@ -0,0 +1,76 @@
+#
+#  flux-exec(1) --with-imp option works
+#
+IMP=$(flux config get exec.imp)
+test -n "$IMP" && test_set_prereq HAVE_IMP
+
+SIZE=$(flux getattr size)
+
+test_expect_success HAVE_IMP 'configure a flux-imp run command' '
+	confdir="/etc/flux/imp/conf.d" &&
+	cat <<-EOF >imp-run-test.toml &&
+	[run.test]
+	allowed-users = ["flux"]
+	allowed-environment = ["FLUX_*", "TEST_ARG"]
+	path = "$confdir/run-test.sh"
+	EOF
+	sudo cp imp-run-test.toml $confdir &&
+	cleanup "sudo rm -f $confdir/imp-run-test.toml" &&
+	sudo chmod 644 $confdir/imp-run-test.toml &&
+	cat <<-EOF >run-test.sh
+	#!/bin/sh
+	echo "calling sleep \$TEST_ARG"
+	echo "id=\$(id -u)"
+	sleep \$TEST_ARG
+	EOF
+	sudo cp run-test.sh $confdir &&
+	cleanup "sudo rm -f $confdr/run-test.sh" &&
+	sudo chmod 755 $confdir/run-test.sh
+'
+test_expect_success HAVE_IMP 'flux exec --with-imp works' '
+	TEST_ARG=0 sudo -u flux -E flux exec -vr 0 --with-imp test \
+		>with-imp.out 2>&1 &&
+	test_debug "cat with-imp.out" &&
+	grep id=0 with-imp.out &&
+	grep "calling sleep 0" with-imp.out
+'
+waitfile=$SHARNESS_TEST_SRCDIR/scripts/waitfile.lua
+
+# Need to create a test output directory writable by flux user and
+# readable by current user
+test_expect_success HAVE_IMP 'create test output directory' '
+	TESTDIR=$(mktemp --tmpdir=${TMPDIR:-/tmp} -d exec-with-impXXXX) &&
+	cleanup "rm -rf $TESTDIR" &&
+	chmod 777 $TESTDIR &&
+	test_debug "echo created test directory $TESTDIR"
+'
+test_expect_success HAVE_IMP 'flux exec --with-imp forwards signals' '
+	cat >test_signal.sh <<-EOF &&
+	#!/bin/bash
+        sig=\${1-INT}
+        TEST_ARG=60 stdbuf --output=L flux exec --with-imp -v -n test \
+                >${TESTDIR}/testready.out &
+        $waitfile -vt 20 -p ^id=0 -c ${SIZE} ${TESTDIR}/testready.out &&
+        kill -\$sig %1 &&
+        wait %1
+        exit \$?
+	EOF
+	chmod +x test_signal.sh &&
+	test_expect_code 130 run_timeout 30 sudo -u flux ./test_signal.sh INT &&
+	test_expect_code 143 run_timeout 30 sudo -u flux ./test_signal.sh TERM
+'
+test_expect_success HAVE_IMP 'flux exec flux-imp run forwards signals' '
+	cat >test_signal.sh <<-EOF &&
+	#!/bin/bash
+        sig=\${1-INT}
+        TEST_ARG=60 stdbuf --output=L flux exec -v -n $IMP run test \
+                >${TESTDIR}/testready2.out &
+        $waitfile -vt 20 -p ^id=0 -c ${SIZE} ${TESTDIR}/testready2.out &&
+        kill -\$sig %1 &&
+        wait %1
+        exit \$?
+	EOF
+	chmod +x test_signal.sh &&
+	test_expect_code 130 run_timeout 30 sudo -u flux ./test_signal.sh INT &&
+	test_expect_code 143 run_timeout 30 sudo -u flux ./test_signal.sh TERM
+'
diff --git a/t/system/0004-recovery.t b/t/system/0004-recovery.t
new file mode 100755
index 000000000000..3cc8ef1f8222
--- /dev/null
+++ b/t/system/0004-recovery.t
@@ -0,0 +1,31 @@
+#
+#  Ensure flux start --recover works on the system instance
+#
+
+test_expect_success 'dump the last checkpoint' '
+        sudo -u flux flux dump --checkpoint /tmp/dump.tar
+'
+test_expect_success 'flux start --recover fails when instance is running' '
+	test_must_fail sudo -u flux flux start --recover true
+'
+test_expect_success 'stop the system instance' '
+	sudo flux shutdown
+'
+test_expect_success 'flux start --recover works' '
+	sudo -u flux flux start --recover true
+'
+test_expect_success 'flux start --recover works from dump file' '
+	sudo -u flux flux start --recover=/tmp/dump.tar --sysconfig true
+'
+test_expect_success 'restart flux' '
+        sudo systemctl start flux
+'
+get_uptime_state () {
+	local state=$(flux uptime | cut -d' ' -f3) || state=unknown
+	echo $state
+}
+test_expect_success 'wait for flux to reach run state' '
+	while test $(get_uptime_state) != run; do \
+	    sleep 1; \
+	done
+'
diff --git a/t/system/0005-exec.t b/t/system/0005-exec.t
new file mode 100755
index 000000000000..dbc5bc1b968e
--- /dev/null
+++ b/t/system/0005-exec.t
@@ -0,0 +1,23 @@
+#
+#  Check exec service with guest/owner
+#
+
+test_expect_success 'start a long-running guest job' '
+        flux submit -n1 --wait-event=start sleep inf &&
+	jobid=$(flux job last)
+'
+test_expect_success 'flux exec --jobid fails as guest' '
+	test_must_fail flux exec --jobid=$jobid true
+'
+test_expect_success 'flux exec --jobid fails as instance owner' '
+	test_must_fail sudo -u flux flux exec --jobid=$jobid true
+'
+test_expect_success 'flux exec without --jobid works as instance owner' '
+	sudo -u flux flux exec -r 0 true
+'
+test_expect_success 'flux exec without --jobid fails as guest' '
+	test_must_fail flux exec -r 0 true
+'
+test_expect_success 'cancel long-running job' '
+	flux cancel $jobid
+'
diff --git a/t/t0000-sharness.t b/t/t0000-sharness.t
index b476ba41d9e9..3ed7fe2509c5 100755
--- a/t/t0000-sharness.t
+++ b/t/t0000-sharness.t
@@ -48,6 +48,8 @@ run_sub_test_lib_test () {
 	opt="$4"             # optionally call the script with extra option(s)
 	mkdir "$name" &&
 	(
+		unset debug &&
+		unset verbose &&
 		cd "$name" &&
 		cat >".$name.t" <<-EOF &&
 		#!/bin/sh
@@ -64,6 +66,7 @@ run_sub_test_lib_test () {
 		cat >>".$name.t" &&
 		chmod +x ".$name.t" &&
 		export SHARNESS_TEST_SRCDIR &&
+		if $roof; then opt="$opt --root=$root"; fi
 		$prefix ./".$name.t" $opt --chain-lint >out 2>err
 	)
 }
@@ -71,6 +74,8 @@ run_sub_test_lib_test () {
 check_sub_test_lib_test () {
 	name="$1" # stdin is the expected output from the test
 	(
+		unset verbose &&
+		unset debug &&
 		cd "$name" &&
 		! test -s err &&
 		sed -e 's/^> //' -e 's/Z$//' >expect &&
@@ -333,6 +338,8 @@ test_expect_success 'tests can be run from an alternate directory' '
 	  unset SHARNESS_TEST_DIRECTORY SHARNESS_TEST_SRCDIR &&
 	  # unset HARNESS_ACTIVE so we get a test-results dir
 	  unset HARNESS_ACTIVE &&
+	  unset debug &&
+	  unset verbose &&
 	  chmod +x test.t &&
 	  mkdir test-rundir &&
 	  cd test-rundir &&
@@ -371,15 +378,19 @@ test_expect_success COLOR,PERL_AND_TTY 'sub-sharness still has color' "
 	EOF
 "
 
-test_expect_success 'EXPENSIVE prereq not activated by default' "
+# Following test should only run by default when TEST_LONG isn't set
+if test -z "$TEST_LONG";  then
+    test_set_prereq NO_TEST_LONG
+fi
+test_expect_success NO_TEST_LONG 'EXPENSIVE prereq not activated by default' "
 	run_sub_test_lib_test no-long 'long test' <<-\\EOF &&
 	test_expect_success 'passing test' 'true'
-	test_expect_success EXPENSIVE 'passing suposedly long test' 'true'
+	test_expect_success EXPENSIVE 'passing supposedly long test' 'true'
 	test_done
 	EOF
 	check_sub_test_lib_test no-long <<-\\EOF
 	> ok 1 - passing test
-	> ok 2 # skip passing suposedly long test (missing EXPENSIVE)
+	> ok 2 # skip passing supposedly long test (missing EXPENSIVE)
 	> # passed all 2 test(s)
 	> 1..2
 	EOF
@@ -388,12 +399,12 @@ test_expect_success 'EXPENSIVE prereq not activated by default' "
 test_expect_success 'EXPENSIVE prereq is activated by --long' "
 	run_sub_test_lib_test long 'long test' '' '--long' <<-\\EOF &&
 	test_expect_success 'passing test' 'true'
-	test_expect_success EXPENSIVE 'passing suposedly long test' 'true'
+	test_expect_success EXPENSIVE 'passing supposedly long test' 'true'
 	test_done
 	EOF
 	check_sub_test_lib_test long <<-\\EOF
 	> ok 1 - passing test
-	> ok 2 - passing suposedly long test
+	> ok 2 - passing supposedly long test
 	> # passed all 2 test(s)
 	> 1..2
 	EOF
@@ -405,6 +416,8 @@ test_expect_success 'loading sharness extensions works' '
 	# a sharness.d/ directory with a test extension function:
 	mkdir extensions &&
 	(
+		unset verbose &&
+		unset debug &&
 		cd extensions &&
 		mkdir sharness.d &&
 		cat >sharness.d/test.sh <<-EOF &&
@@ -439,6 +452,8 @@ test_expect_success 'empty sharness.d directory does not cause failure' '
 	# an empty sharness.d/ directory
 	mkdir nil-extensions &&
 	(
+		unset debug &&
+		unset verbose &&
 		cd nil-extensions &&
 		mkdir sharness.d  &&
 		ln -sf $SHARNESS_TEST_SRCDIR/sharness.sh . &&
@@ -446,7 +461,7 @@ test_expect_success 'empty sharness.d directory does not cause failure' '
 		test_description="sharness works"
 		. ./sharness.sh
 		test_expect_success "test success" "
-			/bin/true
+			true
 		"
 		test_done
 		EOF
diff --git a/t/t0001-basic.t b/t/t0001-basic.t
index 33e236631923..299d01a3633f 100755
--- a/t/t0001-basic.t
+++ b/t/t0001-basic.t
@@ -12,96 +12,369 @@ other tests.
 test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
 . `dirname $0`/sharness.sh
 
+# In some scenarios man(1) tests below may fail due to process sandboxing
+# done via seccomp filter (e.g. for nix). Just disable seccomp for man for
+# the duration of this test to avoid these failure
+export MAN_DISABLE_SECCOMP=1
+
+startctl=${SHARNESS_TEST_SRCDIR}/scripts/startctl.py
+path_printenv=$(which printenv)
+
 test_expect_success 'TEST_NAME is set' '
 	test -n "$TEST_NAME"
 '
+test_expect_success_hd 'heredoc tests work: success' <<-'EOT'
+	test -n "$TEST_NAME" &&
+	# embedded heredoc
+	cat >tmp.sh <<-EOF &&
+	true
+	EOF
+	chmod +x tmp.sh &&
+	./tmp.sh
+EOT
+test_expect_failure_hd 'heredoc tests work: failure' <<'EOT'
+	test -n ""
+EOT
 test_expect_success 'run_timeout works' '
-	test_expect_code 142 run_timeout 0.001 sleep 2
+	test_expect_code 142 run_timeout -s ALRM 0.001 sleep 2
 '
 test_expect_success 'test run_timeout with success' '
-	run_timeout 1 /bin/true
+	run_timeout 1 true
 '
 test_expect_success 'run_timeout fails if exec fails' '
 	test_must_fail run_timeout 1 /nonexistent/executable
 '
+test_expect_success 'test_must_fail_or_be_terminated fails on success' '
+	test_must_fail test_must_fail_or_be_terminated true
+'
+test_expect_success 'test_must_fail_or_be_terminated succeeds on nonzero exit' '
+	test_must_fail_or_be_terminated false
+'
+test_expect_success 'test_must_fail_or_be_terminated succeeds on SIGTERM' '
+	test_must_fail_or_be_terminated sh -c "kill \$\$"
+'
+test_expect_success 'test_must_fail_or_be_terminated fails on SIGHUP' '
+	test_must_fail test_must_fail_or_be_terminated sh -c "kill -HUP \$\$"
+'
 test_expect_success 'we can find a flux binary' '
 	flux --help >/dev/null
 '
+test_expect_success 'flux-keygen path argument is optional' '
+	flux keygen
+'
 test_expect_success 'flux-keygen works' '
-	umask 077 && tmpkeydir=`mktemp -d` &&
-	flux keygen --secdir $tmpkeydir --force &&
-	rm -rf $tmpkeydir
+	flux keygen cert &&
+	test -f cert
+'
+test_expect_success 'flux-keygen --name=test works' '
+	flux keygen --name=testcert cert2 &&
+	test -f cert2 &&
+	grep testcert cert2
+'
+test_expect_success 'flux-keygen --meta works' '
+	flux keygen --meta mammal=chinchilla,reptile=chamelion cert3 &&
+	test -f cert3 &&
+	grep mammal cert3 &&
+	grep reptile cert3
+'
+test_expect_success 'flux-keygen --meta can update keygen.hostname' '
+	flux keygen --meta=keygen.hostname=myhost cert4 &&
+	test -f cert4 &&
+	grep myhost cert4
+'
+test_expect_success 'flux-keygen --meta value can be an empty string' '
+	flux keygen --meta smurf= cert5 &&
+	test -f cert5 &&
+	grep smurf cert5
+'
+test_expect_success 'flux-keygen --meta equal sign can be missing' '
+	flux keygen --meta smurf cert6 &&
+	test -f cert6 &&
+	grep smurf cert6
+'
+test_expect_success 'flux-keygen fails with extra positional argument' '
+	test_must_fail flux keygen cert xyz
+'
+test_expect_success 'flux-keygen generated cert with u=rw access' '
+	echo '-rw-------' >cert-access.exp &&
+	stat --format=%A cert >cert-access.out &&
+	test_cmp cert-access.exp cert-access.out
+'
+
+test_expect_success 'flux-keygen overwrites existing cert' '
+	test -f cert &&
+	cp cert cert.bak &&
+	flux keygen cert
+'
+test_expect_success 'flux-keygen generated a cert with different keys' '
+	diff cert.bak cert | grep secret-key
 '
 
+test_expect_success 'flux-keygen fails with unknown arg' '
+	test_must_fail flux keygen --force boguskey
+'
 test_expect_success 'flux-python command runs a python that finds flux' '
 	flux python -c "import flux"
 '
 
+test_expect_success 'flux-python command runs the configured python' '
+	define_line=$(grep "^#define PYTHON_INTERPRETER" ${FLUX_BUILD_DIR}/config/config.h) &&
+	pypath=$(echo ${define_line} | sed -E '\''s~.*define PYTHON_INTERPRETER "(.*)"~\1~'\'') &&
+	expected=$(${pypath} -c "import sys; print(sys.executable)") &&
+	actual=$(flux python -c "import sys; print(sys.executable)") &&
+	test "${expected}" = "${actual}"
+'
+
+test_expect_success 'flux fortune help works' '
+	flux fortune --help | grep category
+'
+
+test_expect_success 'flux fortune works' '
+	flux fortune
+'
+
+test_expect_success 'flux fortune all (default) works' '
+	flux fortune -c all
+'
+
+test_expect_success 'flux fortune with valentine works' '
+	flux fortune -c valentines
+'
+
+test_expect_success 'flux fortune with fun works' '
+	flux fortune -c fun
+'
+
+test_expect_success 'flux fortune with facts works' '
+	flux fortune -c facts
+'
+
+test_expect_success 'flux fortune with art works' '
+	flux fortune -c art
+'
+
 # Minimal is sufficient for these tests, but test_under_flux unavailable
 # clear the RC paths
-ARGS="-o,-Sbroker.rc1_path=,-Sbroker.rc3_path="
+ARGS="-Sbroker.rc1_path= -Sbroker.rc3_path="
 
-test_expect_success 'broker --shutdown-grace option works' "
-	flux start ${ARGS} -s2 -o,--shutdown-grace=5 /bin/true
-"
 test_expect_success 'flux-start in exec mode works' "
-	flux start ${ARGS} 'flux comms info' | grep 'size=1'
+	flux start ${ARGS} flux getattr size | grep -x 1
 "
 test_expect_success 'flux-start in subprocess/pmi mode works (size 1)' "
-	flux start ${ARGS} --size=1 'flux comms info' | grep 'size=1'
+	flux start ${ARGS} -s1 flux getattr size | grep -x 1
+"
+test_expect_success 'and broker.boot-method=simple' "
+	test $(flux start ${ARGS} -s1 \
+		flux getattr broker.boot-method) = "simple"
+"
+test_expect_success 'although method can be forced to single with attribute' "
+	test $(flux start ${ARGS} -s1 -Sbroker.boot-method=single \
+		flux getattr broker.boot-method) = "single"
+"
+test_expect_success 'or forced by setting FLUX_PMI_CLIENT_METHODS' "
+	test $(FLUX_PMI_CLIENT_METHODS="single" flux start ${ARGS} -s1 \
+		flux getattr broker.boot-method) = "single"
+"
+test_expect_success 'start fails when broker.boot-method=unknown' "
+	test_must_fail flux start ${ARGS} -Sbroker.boot-method=unknown \
+		true
 "
 test_expect_success 'flux-start in subprocess/pmi mode works (size 2)' "
-	flux start ${ARGS} --size=2 'flux comms info' | grep 'size=2'
+	flux start ${ARGS} -s2 flux getattr size | grep -x 2
+"
+test_expect_success 'flux-start with size 1 has no peers' '
+	echo 0 >nochild.exp &&
+	flux start ${ARGS} -s1 \
+		flux module stats --parse=child-count overlay >nochild.out &&
+	test_cmp nochild.exp nochild.out
+'
+test_expect_success 'flux-start with size 2 has rank 1 peer' '
+	echo 1 >child2.exp &&
+	flux start ${ARGS} -s2 \
+		flux module stats --parse=child-count overlay >child2.out &&
+	test_cmp child2.exp child2.out
+'
+test_expect_success 'flux-start fails with unknown option' "
+	test_must_fail flux start ${ARGS} --unknown true
+"
+test_expect_success 'flux-start fails with --verbose=badopt' "
+	test_must_fail flux start ${ARGS} --verbose=badopt true
+"
+test_expect_success 'create bad broker shell script' '
+	cat >flux-broker <<-EOT &&
+	#!/badinterp
+	EOT
+	chmod +x flux-broker
+'
+test_expect_success 'flux-start exec fails on bad broker shell script' "
+	test_must_fail bash -c 'FLUX_EXEC_PATH_PREPEND=. flux start true'
+"
+test_expect_success 'flux-start test exec fails on bad broker shell script' "
+	#
+	# We can't use test_must_fail here because on some OSes this command
+	# might fail with exit code 127, which test_must_fail does not
+	# accept, so we use ! here since any failure is acceptable here
+	#
+	! bash -c 'FLUX_EXEC_PATH_PREPEND=. flux start -s1 true'
+"
+test_expect_success 'flux-start -s1 works' "
+	flux start ${ARGS} -s1 true
+"
+test_expect_success 'flux-start -s1 sets broker.mapping to expected value' "
+	cat >mapping_1.exp <<-EOT &&
+	[[0,1,1,1]]
+	EOT
+	flux start ${ARGS} -s1 flux getattr broker.mapping >mapping_1.out &&
+	test_cmp mapping_1.exp mapping_1.out
+"
+test_expect_success 'flux-start --test-rundir without --test-size fails' "
+	test_must_fail flux start ${ARGS} --test-rundir=$(pwd) true
+"
+test_expect_success 'flux-start --test-pmi-clique without --test-size fails' "
+	test_must_fail flux start ${ARGS} --test-pmi-clique=none true
+"
+test_expect_success 'flux-start --test-hosts without --test-size fails' "
+	test_must_fail flux start ${ARGS} --test-hosts=foo true
+"
+test_expect_success 'flux-start --test-hosts with insufficient hosts fails' "
+	test_must_fail flux start ${ARGS} -s2 --test-hosts=foo true
+"
+test_expect_success 'flux-start --test-hosts with garbled hosts fails' "
+	test_must_fail flux start ${ARGS} -s2 --test-hosts=foo] true
+"
+test_expect_success 'flux-start --test-exit-timeout without --test-size fails' "
+	test_must_fail flux start ${ARGS} --test-exit-timeout=10s true
+"
+test_expect_success 'flux-start --test-exit-timeout fails with bad FSD' "
+	test_must_fail flux start ${ARGS} -s1 --test-exit-timeout=-1 true
+"
+test_expect_success 'flux-start --test-exit-mode without --test-size fails' "
+	test_must_fail flux start ${ARGS} --test-exit-mode=any true
+"
+test_expect_success 'flux-start --test-exit-mode=leader is accepted' "
+	flux start ${ARGS} -s1 --test-exit-mode=leader true
+"
+test_expect_success 'flux-start --test-exit-mode=badmode fails' "
+	test_must_fail flux start ${ARGS} -s1 --test-exit-mode=badmode true
+"
+test_expect_success 'flux-start --test-start-mode without --test-size fails' "
+	test_must_fail flux start ${ARGS} --test-start-mode=all true
+"
+test_expect_success 'flux-start --test-start-mode=all is accepted' "
+	flux start ${ARGS} -s1 --test-start-mode=all true
 "
-test_expect_success 'flux-start with size 1 has no peers' "
-	flux start ${ARGS} --size=1 'flux comms idle' > idle.out &&
-        ! grep 'idle' idle.out
+test_expect_success 'flux-start --test-start-mode=badmode fails' "
+	test_must_fail flux start ${ARGS} -s1 --test-start-mode=badmode true
 "
-test_expect_success 'flux-start with size 2 has a peer' "
-	flux start ${ARGS} --size=2 'flux comms idle' > idle.out &&
-        grep 'idle' idle.out
+test_expect_success 'flux-start --test-rundir-cleanup without --test-size fails' "
+	test_must_fail flux start ${ARGS} --test-rundir-cleanup true
 "
-test_expect_success 'flux-start --size=1 --bootstrap=selfpmi works' "
-	flux start ${ARGS} --size=1 --bootstrap=selfpmi /bin/true
+test_expect_success 'flux-start --verbose=2 enables PMI tracing' "
+	flux start ${ARGS} \
+		--test-size=1 --verbose=2 \
+		true 2>&1 | grep pmi_version
 "
-test_expect_success 'flux-start --bootstrap=selfpmi fails (no size specified)' "
-	test_must_fail flux start ${ARGS} --bootstrap=selfpmi /bin/true
+test_expect_success 'flux-start -vv also does' "
+	flux start ${ARGS} \
+		--test-size=1 -vv \
+		true 2>&1 | grep pmi_version
 "
-test_expect_success 'flux-start --size=1 --boostrap=pmi fails' "
-	test_must_fail flux start ${ARGS} --size=1 --bootstrap=pmi /bin/true
+test_expect_success 'flux-start --test-pmi-clique=single works' "
+	flux start ${ARGS} \
+		--test-size=1 \
+		--test-pmi-clique=single \
+		true
+"
+test_expect_success 'flux-start --test-pmi-clique=none works' "
+	flux start ${ARGS} --test-size=1 \
+		--test-pmi-clique=single \
+		true
+"
+test_expect_success 'flux-start --test-pmi-clique=badmode fails' "
+	test_must_fail flux start ${ARGS} --test-size=1 \
+		--test-pmi-clique=badmode \
+		true 2>badmode.err &&
+	grep unsupported badmode.err
+"
+test_expect_success 'flux-start embedded server works from initial program' "
+	flux start -v ${ARGS} -s1 flux python ${startctl} status \
+		>startctl.out 2>startctl.err
+"
+test_expect_success 'flux-start embedded server status got JSON' "
+	jq -c . <startctl.out
+"
+test_expect_success 'flux-start embedded server logs hi/bye from client' "
+	grep hi startctl.err &&
+	grep bye startctl.err
+"
+test_expect_success 'flux-start embedded server logs disconnect notification' "
+	grep 'disconnect from' startctl.err
 "
 test_expect_success 'flux-start in exec mode passes through errors from command' "
-	test_must_fail flux start ${ARGS} /bin/false
+	test_must_fail flux start ${ARGS} false
 "
 test_expect_success 'flux-start in subprocess/pmi mode passes through errors from command' "
-	test_must_fail flux start ${ARGS} --size=1 /bin/false
+	test_must_fail flux start ${ARGS} -s1 false
 "
 test_expect_success 'flux-start in exec mode passes exit code due to signal' "
 	test_expect_code 130 flux start ${ARGS} 'kill -INT \$\$'
 "
 test_expect_success 'flux-start in subprocess/pmi mode passes exit code due to signal' "
-	test_expect_code 130 flux start ${ARGS} --size=1 'kill -INT \$\$'
+	test_expect_code 130 flux start ${ARGS} -s1 'kill -INT \$\$'
 "
 test_expect_success 'flux-start in exec mode works as initial program' "
-	flux start --size=2 flux start ${ARGS} flux comms info | grep size=1
+	flux start ${ARGS} -s2 flux start ${ARGS} flux getattr size | grep -x 1
 "
 test_expect_success 'flux-start in subprocess/pmi mode works as initial program' "
-	flux start --size=2 flux start ${ARGS} --size=1 flux comms info | grep size=1
+	flux start ${ARGS} -s2 flux start ${ARGS} -s1 flux getattr size | grep -x 1
 "
 
+# See issue #5447: rc1 can fail if the shell sources a script that reactivates
+# errexit mode and has an unchecked error.  'command' apparently reactivates
+# errexit mode in older versions of bash.
+#
+test_expect_success 'flux-start works with non-errexit clean BASH_ENV' '
+	cat >testbashrc <<-EOT &&
+	command -v ls >/dev/null || :
+	false
+	true
+	EOT
+	BASH_ENV=testbashrc flux start true
+'
+
+test_expect_success 'flux-start works with multiple files in rc1.d' '
+	mkdir -p rc1.d &&
+	printf "echo rc-one\n" >rc1.d/one &&
+	printf "echo rc-two\n" >rc1.d/two &&
+	chmod +x rc1.d/* &&
+	FLUX_RC_EXTRA=$(pwd) flux start -Slog-stderr-level=6 \
+		echo rc-three >rc-multi.out 2>&1 &&
+	test_debug "cat rc-multi.out" &&
+	grep rc-one rc-multi.out &&
+	grep rc-two rc-multi.out &&
+	grep rc-three rc-multi.out
+'
+
+test_expect_success 'flux-start works with zero files in rc1.d' '
+	rm rc1.d/* &&
+	FLUX_RC_EXTRA=$(pwd) flux start echo rc-done >rc-zero.out 2>&1 &&
+	test_debug "cat rc-zero.out" &&
+	grep rc-done rc-zero.out
+'
+
 test_expect_success 'flux-start --wrap option works' '
-	broker_path=$(flux start -vX 2>&1 | sed "s/^flux-start: *//g") &&
+	broker_path=$(flux start ${ARGS} -vX 2>&1 | sed "s/^flux-start: *//g") &&
 	echo broker_path=${broker_path} &&
 	test -n "${broker_path}" &&
-	flux start --wrap=/bin/echo,start: arg0 arg1 arg2 > wrap.output &&
+	flux start ${ARGS} --wrap=/bin/echo,start: arg0 arg1 arg2 > wrap.output &&
 	test_debug "cat wrap.output" &&
 	cat >wrap.expected <<-EOF &&
 	start: ${broker_path} arg0 arg1 arg2
 	EOF
 	test_cmp wrap.expected wrap.output
 '
-test_expect_success 'flux-start --wrap option works with --size' '
-	flux start --size=2 -vX --wrap=test-wrap > wrap2.output 2>&1 &&
+test_expect_success 'flux-start --wrap option works with --test-size' '
+	flux start ${ARGS} -s2 -vX --wrap=test-wrap > wrap2.output 2>&1 &&
 	test_debug "cat wrap2.output" &&
 	test "$(grep -c test-wrap wrap2.output)" = "2"
 '
@@ -110,25 +383,38 @@ test_expect_success 'flux-start dies gracefully when run from removed dir' '
 	mkdir foo && (
 	 cd foo &&
 	 rmdir ../foo &&
-	 test_must_fail flux start /bin/true )
+	 test_must_fail flux start true )
 '
 
+command -v hwloc-ls >/dev/null && test_set_prereq HWLOC_LS
+test_expect_success HWLOC_LS 'FLUX_HWLOC_XMLFILE works' '
+	hwloc-ls --of xml -i "numa:2 core:3 pu:1" >test.xml &&
+	cat <<-EOF >norestrict.conf &&
+	[resource]
+	norestrict = true
+	EOF
+	FLUX_HWLOC_XMLFILE=test.xml \
+		flux start -s2 --conf=norestrict.conf \
+			flux resource info >rinfo.out &&
+	test_debug "cat rinfo.out" &&
+	grep "12 Cores" rinfo.out
+'
 
 # too slow under ASAN
 test_expect_success NO_ASAN 'test_under_flux works' '
 	echo >&2 "$(pwd)" &&
-        mkdir -p test-under-flux && (
-        cd test-under-flux &&
+	mkdir -p test-under-flux && (
+	cd test-under-flux &&
 	SHARNESS_TEST_DIRECTORY=`pwd` &&
 	export SHARNESS_TEST_SRCDIR SHARNESS_TEST_DIRECTORY FLUX_BUILD_DIR debug &&
 	run_timeout 10 "$SHARNESS_TEST_SRCDIR"/test-under-flux/test.t --verbose --debug >out 2>err
 	) &&
-	grep "size=2" test-under-flux/out
+	grep -x "2" test-under-flux/out
 '
 
-test_expect_success 'test_under_flux fails if loaded modules are not unloaded' '
-        mkdir -p test-under-flux && (
-        cd test-under-flux &&
+test_expect_success NO_ASAN 'test_under_flux fails if loaded modules are not unloaded' '
+	mkdir -p test-under-flux && (
+	cd test-under-flux &&
 	SHARNESS_TEST_DIRECTORY=`pwd` &&
 	export SHARNESS_TEST_SRCDIR SHARNESS_TEST_DIRECTORY FLUX_BUILD_DIR debug &&
 	test_expect_code 1 "$SHARNESS_TEST_SRCDIR"/test-under-flux/t_modcheck.t 2>err.modcheck \
@@ -141,114 +427,209 @@ test_expect_success 'flux-start -o,--setattr ATTR=VAL can set broker attributes'
 	ATTR_VAL=`flux start ${ARGS} -o,--setattr=foo-test=42 flux getattr foo-test` &&
 	test $ATTR_VAL -eq 42
 '
-test_expect_success 'tbon.endpoint can be read' '
-	ATTR_VAL=`flux start -s2 flux getattr tbon.endpoint` &&
-	echo $ATTR_VAL | grep "://"
-'
-test_expect_success 'tbon.endpoint can be set and %h works' '
-	flux start -s2 -o,--setattr=tbon.endpoint=tcp://%h:* \
-		flux getattr tbon.endpoint >pct_h.out &&
-	grep "^tcp" pct_h.out &&
-	test_must_fail grep "%h" pct_h.out
-'
-test_expect_success 'tbon.endpoint with %B works' '
-	flux start -s2 -o,--setattr=tbon.endpoint=ipc://%B/req \
-		flux getattr tbon.endpoint >pct_B.out &&
-	grep "^ipc" pct_B.out &&
-	test_must_fail grep "%B" pct_B.out
-'
-# N.B. rank 1 has to be killed in this test after rank 0 fails gracefully
-# so test_must_fail won't work here
-test_expect_success 'tbon.endpoint fails on bad endpoint' '
-	! flux start -s2 -o,--setattr=tbon.endpoint=foo://bar /bin/true
+test_expect_success 'flux-start --setattr ATTR=VAL can set broker attributes' '
+	ATTR_VAL=`flux start ${ARGS} --setattr=foo-test=42 flux getattr foo-test` &&
+	test $ATTR_VAL -eq 42
 '
-test_expect_success 'tbon.parent-endpoint cannot be read on rank 0' '
-	test_must_fail flux start -s2 flux getattr tbon.parent-endpoint
+test_expect_success 'hostlist attr is set on size 1 instance' '
+	hn=$(hostname) &&
+	cat >hostlist1.exp <<-EOT &&
+	$hn
+	EOT
+	flux start ${ARGS} flux exec flux getattr hostlist >hostlist1.out &&
+	test_cmp hostlist1.exp hostlist1.out
 '
-test_expect_success 'tbon.parent-endpoint can be read on not rank 0' '
-       NUM=`flux start --size 4 flux exec -n flux getattr tbon.parent-endpoint | grep ipc | wc -l` &&
-       test $NUM -eq 3
+test_expect_success 'hostlist attr is set on all ranks of size 4 instance' '
+	flux start ${ARGS} -s4 flux exec flux getattr hostlist
 '
-test_expect_success 'flux start --bootstrap=pmi (singlton) cleans up rundir' '
-	flux start ${ARGS} --bootstrap=pmi \
+test_expect_success 'flux start (singleton) cleans up rundir' '
+	flux start ${ARGS} \
 		flux getattr rundir >rundir_pmi.out &&
 	RUNDIR=$(cat rundir_pmi.out) &&
 	test_must_fail test -d $RUNDIR
 '
-test_expect_success 'flux start --bootstrap=selfpmi --size=1 cleans up rundirs' '
-	flux start ${ARGS} --bootstrap=selfpmi --size=1 \
+test_expect_success 'flux start -s1 cleans up rundirs' '
+	flux start ${ARGS} -s1 \
 		flux getattr rundir >rundir_selfpmi1.out &&
 	RUNDIR=$(cat rundir_selfpmi1.out) &&
 	test -n "$RUNDIR" &&
 	test_must_fail test -d $RUNDIR
 '
-test_expect_success 'flux start --bootstrap=selfpmi --size=2 cleans up rundirs' '
-	flux start ${ARGS} --bootstrap=selfpmi --size=2 \
+test_expect_success 'flux start -s2 cleans up rundirs' '
+	flux start ${ARGS} -s2 \
 		flux getattr rundir >rundir_selfpmi2.out &&
 	RUNDIR=$(cat rundir_selfpmi2.out) &&
 	test -n "$RUNDIR" &&
 	test_must_fail test -d $RUNDIR
 '
+test_expect_success 'flux start --test-rundir works' '
+	RUNDIR=$(mktemp -d) &&
+	flux start ${ARGS} --test-size=1 \
+		--test-rundir=$RUNDIR \
+		flux getattr rundir >rundir_test.out &&
+	echo $RUNDIR >rundir_test.exp &&
+	test_cmp rundir_test.exp rundir_test.out &&
+	rmdir $RUNDIR
+'
+test_expect_success 'flux start --test-rundir --test-rundir-cleanup works' '
+	RUNDIR=$(mktemp -d) &&
+	flux start ${ARGS} --test-size=1 \
+		--test-rundir=$RUNDIR \
+		--test-rundir-cleanup \
+		flux getattr rundir >rundir_test.out &&
+	echo $RUNDIR >rundir_test.exp &&
+	test_cmp rundir_test.exp rundir_test.out &&
+	test_must_fail test -d $RUNDIR
+'
+test_expect_success 'flux start --test-rundir with missing directory fails' '
+	test_must_fail flux start ${ARGS} --test-size=1 \
+		--test-rundir=/noexist \
+		true 2>noexist_rundir.err &&
+	grep "/noexist: No such file or directory" noexist_rundir.err
+'
+test_expect_success 'flux start --test-rundir with not-directory fails' '
+	touch notdir &&
+	test_must_fail flux start ${ARGS} --test-size=1 \
+		--test-rundir=notdir \
+		true 2>notdir_rundir.err &&
+	grep "notdir: not a directory" notdir_rundir.err
+'
 test_expect_success 'rundir override works' '
 	RUNDIR=`mktemp -d` &&
-	DIR=`flux start ${ARGS} -o,--setattr=rundir=$RUNDIR flux getattr rundir` &&
+	DIR=`flux start ${ARGS} --setattr=rundir=$RUNDIR flux getattr rundir` &&
 	test "$DIR" = "$RUNDIR" &&
 	test -d $RUNDIR &&
 	rm -rf $RUNDIR
 '
-test_expect_success 'rundir override creates nonexistent dirs' '
-	RUNDIR="$(pwd)/rundir" &&
-	flux start ${ARGS} -o,--setattr=rundir=$RUNDIR sh -c "test -d $RUNDIR" &&
+test_expect_success 'rundir override creates nonexistent dirs and cleans up' '
+	RUNDIR=`mktemp -d` &&
+	rmdir $RUNDIR &&
+	flux start ${ARGS} --setattr=rundir=$RUNDIR sh -c "test -d $RUNDIR" &&
 	test_expect_code 1 test -d $RUNDIR
 '
+test_expect_success 'broker fails gracefully when rundir buffer overflows' '
+	longstring=$(head -c 1024 < /dev/zero | tr \\0 D) &&
+	! TMPDIR=$longstring flux start ${ARGS} true 2>overflow.err &&
+	grep overflow overflow.err
+'
+test_expect_success 'broker fails gracefully on nonexistent TMPDIR' '
+	! TMPDIR=/noexist flux start ${ARGS} true 2>noexist.err &&
+	grep "cannot create directory in /noexist" noexist.err
+'
+test_expect_success 'broker fails gracefully on non-directory rundir' '
+	touch notdir &&
+	test_must_fail flux start ${ARGS} -Srundir=notdir \
+		true 2>notdir.err &&
+	grep "Not a directory" notdir.err
+'
+test_expect_success 'broker fails gracefully on unwriteable rundir' '
+	mkdir -p privdir &&
+	chmod u-w privdir &&
+	test_must_fail flux start ${ARGS} -Srundir=privdir \
+		true 2>privdir.err &&
+	grep "permissions" privdir.err
+'
+# statedir created here is reused in the next several tests
+test_expect_success 'broker statedir is not cleaned up' '
+	mkdir -p statedir &&
+	flux start ${ARGS} -Sstatedir=$(pwd)/statedir true &&
+	test -d statedir
+'
+test_expect_success 'broker statedir cannot be changed at runtime' '
+	test_must_fail flux start ${ARGS} -Sstatedir=$(pwd)/statedir \
+		flux setattr statedir $(pwd)/statedir 2>rostatedir.err &&
+	grep "Operation not permitted" rostatedir.err
+'
+test_expect_success 'broker statedir cannot be set at runtime' '
+	test_must_fail flux start ${ARGS} \
+		flux setattr statedir $(pwd)/statedir 2>rostatedir2.err &&
+	grep "Operation not permitted" rostatedir2.err
+'
+test_expect_success 'broker fails when statedir does not exist' '
+	rm -rf statedir &&
+	test_must_fail flux start ${ARGS} -Sstatedir=$(pwd)/statedir \
+		true 2>nostatedir.err &&
+	grep "cannot stat" nostatedir.err
+'
 # Use -eq hack to test that BROKERPID is a number
 test_expect_success 'broker broker.pid attribute is readable' '
 	BROKERPID=`flux start ${ARGS} flux getattr broker.pid` &&
 	test -n "$BROKERPID" &&
 	test "$BROKERPID" -eq "$BROKERPID"
 '
+
+test_expect_success 'local-uri override works' '
+	sockdir=$(mktemp -d) &&
+	newsock=local://$sockdir/meep &&
+	echo $newsock >uri.exp &&
+	flux start ${ARGS} \
+		-Slocal-uri=$newsock \
+		printenv FLUX_URI >uri.out &&
+	test_cmp uri.exp uri.out &&
+	rm -rf $sockdir
+'
+test_expect_success 'broker fails gracefully when local-uri is malformed' '
+	test_must_fail flux start ${ARGS} -Slocal-uri=baduri \
+		true 2>baduri.err &&
+	grep malformed baduri.err
+'
+test_expect_success 'broker fails gracefully when local-uri buffer overflows' '
+	longuri="local://$(head -c 1024 < /dev/zero | tr \\0 D)" &&
+	test_must_fail flux start ${ARGS} -Slocal-uri=${longuri} \
+		true 2>longuri.err &&
+	grep "buffer overflow" longuri.err
+'
+test_expect_success 'broker fails gracefully when local-uri in missing dir' '
+	test_must_fail flux start ${ARGS} -Slocal-uri=local:///noexist/x \
+		true 2>nodiruri.err &&
+	grep "cannot stat" nodiruri.err
+'
+test_expect_success 'broker fails gracefully when local-uri in non-dir' '
+	touch urinotdir &&
+	test_must_fail flux start ${ARGS} \
+		-Slocal-uri=local://$(pwd)/urinotdir/x \
+		true 2>urinotdir.err &&
+	grep "Not a directory" urinotdir.err
+'
+test_expect_success 'broker fails gracefully when local-uri in unwritable dir' '
+	mkdir -p privdir &&
+	chmod u-w privdir &&
+	test_must_fail flux start ${ARGS} \
+		-Slocal-uri=local://$(pwd)/privdir/x \
+		true 2>uriprivdir.err &&
+	grep "permissions" uriprivdir.err
+'
 test_expect_success 'broker broker.pid attribute is immutable' '
-	test_must_fail flux start ${ARGS} -o,--setattr=broker.pid=1234 flux getattr broker.pid
+	test_must_fail flux start ${ARGS} --setattr=broker.pid=1234 flux getattr broker.pid
 '
 test_expect_success 'broker --verbose option works' '
-	flux start ${ARGS} -o,-v /bin/true
+	flux start ${ARGS} -o,-v true
 '
-test_expect_success 'broker --heartrate option works' '
-	flux start ${ARGS} -o,--heartrate=0.1 /bin/true
+test_expect_success 'broker fails on invalid broker.critical-ranks option' '
+	test_must_fail flux start ${ARGS} -Sbroker.critical-ranks=0-1
 '
-test_expect_success NO_CHAIN_LINT 'broker --k-ary option works' '
-	pids="" &&
-	flux start ${ARGS} -s4 -o,--k-ary=1 /bin/true & pids=$!
-	flux start ${ARGS} -s4 -o,--k-ary=2 /bin/true & pids="$pids $!"
-	flux start ${ARGS} -s4 -o,--k-ary=3 /bin/true & pids="$pids $!"
-	flux start ${ARGS} -s4 -o,--k-ary=4 /bin/true & pids="$pids $!" 
-	wait $pids
+test_expect_success 'broker fails on unknown option' '
+	test_must_fail flux start ${ARGS} -o,--not-an-option true
 '
-
 test_expect_success 'flux-help command list can be extended' '
 	mkdir help.d &&
 	cat <<-EOF  > help.d/test.json &&
-	[{ "category": "test", "command": "test", "description": "a test" }]
-	EOF
-	flux help 2>&1 | sed "0,/^$/d" >help.expected &&
-	cat <<-EOF  >>help.expected &&
-	Common commands from flux-test:
-	   test               a test
+	[{ "name": "test", "description": "test commands",
+	 "commands": [ {"name": "test", "description": "a test" }]}]
 	EOF
-	FLUX_CMDHELP_PATTERN="help.d/*" flux help 2>&1 | sed "0,/^$/d" > help.out &&
-	test_cmp help.expected help.out &&
+	FLUX_CMDHELP_PATTERN="help.d/*" flux help > help.out 2>&1 &&
+	grep "^test commands" help.out &&
+	grep "a test" help.out &&
 	cat <<-EOF  > help.d/test2.json &&
-	[{ "category": "test2", "command": "test2", "description": "a test two" }]
+	[{ "name": "test", "description": "test two commands",
+	 "commands": [ {"name": "test2", "description": "a test two"}]}]
 	EOF
-	cat <<-EOF  >>help.expected &&
-
-	Common commands from flux-test2:
-	   test2              a test two
-	EOF
-	FLUX_CMDHELP_PATTERN="help.d/*" flux help 2>&1 | sed "0,/^$/d" > help.out &&
-	test_cmp help.expected help.out
+	FLUX_CMDHELP_PATTERN="help.d/*" flux help > help2.out 2>&1 &&
+	grep "^test two commands" help2.out &&
+	grep "a test two" help2.out
 '
-test_expect_success 'flux-help command can display manpages for subcommands' '
+command -v man >/dev/null && test_set_prereq HAVE_MAN
+test_expect_success HAVE_MAN 'flux-help command can display manpages for subcommands' '
 	PWD=$(pwd) &&
 	mkdir -p man/man1 &&
 	cat <<-EOF > man/man1/flux-foo.1 &&
@@ -258,7 +639,7 @@ test_expect_success 'flux-help command can display manpages for subcommands' '
 	EOF
 	MANPATH=${PWD}/man FLUX_IGNORE_NO_DOCS=y flux help foo | grep "^FOO(1)"
 '
-test_expect_success 'flux-help command can display manpages for api calls' '
+test_expect_success HAVE_MAN 'flux-help command can display manpages for api calls' '
 	PWD=$(pwd) &&
 	mkdir -p man/man3 &&
 	cat <<-EOF > man/man3/flux_foo.3 &&
@@ -270,36 +651,42 @@ test_expect_success 'flux-help command can display manpages for api calls' '
 '
 missing_man_code()
 {
-        man notacommand >/dev/null 2>&1
-        echo $?
+	man notacommand >/dev/null 2>&1
+	echo $?
 }
-test_expect_success 'flux-help returns nonzero exit code from man(1)' '
-        test_expect_code $(missing_man_code) \
-                         eval FLUX_IGNORE_NO_DOCS=y flux help notacommand
+test_expect_success HAVE_MAN 'flux-help returns nonzero exit code from man(1)' '
+	test_expect_code $(missing_man_code) \
+		eval FLUX_IGNORE_NO_DOCS=y flux help notacommand
 '
 test_expect_success 'flux appends colon to missing or unset MANPATH' '
-      (unset MANPATH && flux /usr/bin/printenv | grep "MANPATH=.*:$") &&
-      MANPATH= flux /usr/bin/printenv | grep "MANPATH=.*:$"
+	(unset MANPATH && flux $path_printenv | grep "MANPATH=.*:$") &&
+	MANPATH= flux $path_printenv | grep "MANPATH=.*:$"
+'
+test_expect_success 'flux deduplicates FLUX_RC_EXTRA & FLUX_SHELL_RC_PATH' '
+	FLUX_RC_EXTRA=/foo:/bar:/foo \
+		flux $path_printenv FLUX_RC_EXTRA | grep "^/foo:/bar$" &&
+	FLUX_SHELL_RC_PATH=/foo:/bar:/foo \
+		flux $path_printenv FLUX_SHELL_RC_PATH | grep "^/foo:/bar$"
 '
 test_expect_success 'builtin test_size_large () works' '
-    size=$(test_size_large)  &&
-    test -n "$size" &&
-    size=$(FLUX_TEST_SIZE_MAX=2 test_size_large) &&
-    test "$size" = "2" &&
-    size=$(FLUX_TEST_SIZE_MIN=12345 FLUX_TEST_SIZE_MAX=23456 test_size_large) &&
-    test "$size" = "12345"
+	size=$(test_size_large)  &&
+	test -n "$size" &&
+	size=$(FLUX_TEST_SIZE_MAX=2 test_size_large) &&
+	test "$size" = "2" &&
+	size=$(FLUX_TEST_SIZE_MIN=12345 FLUX_TEST_SIZE_MAX=23456 test_size_large) &&
+	test "$size" = "12345"
 '
 
 waitfile=${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua
 test_expect_success 'scripts/waitfile works' '
-	flux start $waitfile -v -t 5 -p "hello" waitfile.test.1 &
+	flux start ${ARGS} $waitfile -v -t 5 -p "hello" waitfile.test.1 &
 	p=$! &&
 	echo "hello" > waitfile.test.1 &&
 	wait $p
 '
 
 test_expect_success 'scripts/waitfile works after <1s' '
-	flux start $waitfile -v -t 2 -p "hello" -P- waitfile.test.2 <<-EOF &
+	flux start ${ARGS} $waitfile -v -t 2 -p "hello" -P- waitfile.test.2 <<-EOF &
 	-- open file at 250ms, write pattern at 500ms
 	f:timer{ timeout = 250,
 	         handler = function () tf = io.open ("waitfile.test.2", "w") end
@@ -313,11 +700,11 @@ test_expect_success 'scripts/waitfile works after <1s' '
 '
 
 test_expect_success 'scripts/waitfile works after 1s' '
-	flux start $waitfile -v -t 5 -p "hello" -P- waitfile.test.3 <<-EOF &
+	flux start ${ARGS} $waitfile -v -t 5 -p "hello" -P- waitfile.test.3 <<-EOF &
 	-- Wait 250ms and create file, at .5s write a line, at 1.1s write pattern:
 	f:timer{ timeout = 250,
 	         handler = function () tf = io.open ("waitfile.test.3", "w") end
-               }
+	       }
 	f:timer{ timeout = 500,
 	         handler = function () tf:write ("line one"); tf:flush()  end
 	       }
@@ -330,14 +717,33 @@ test_expect_success 'scripts/waitfile works after 1s' '
 '
 # test for issue #1025
 test_expect_success 'instance can stop cleanly with subscribers (#1025)' '
-	flux start ${ARGS} -s2 --bootstrap=selfpmi bash -c "nohup flux event sub hb &"
+	flux start ${ARGS} -s2 bash -c "nohup flux event sub heartbeat.pulse &"
 '
 
 # test for issue #1191
 test_expect_success 'passing NULL to flux_log functions logs to stderr (#1191)' '
-        ${FLUX_BUILD_DIR}/t/loop/logstderr > std.out 2> std.err &&
-        grep "warning: hello" std.err &&
-        grep "err: world: No such file or directory" std.err
+	${FLUX_BUILD_DIR}/t/loop/logstderr > std.out 2> std.err &&
+	grep "warning: hello" std.err &&
+	grep "err: world: No such file or directory" std.err
+'
+
+# tests for issue #3925
+test_expect_success 'setting rundir to a long directory fails (#3925)' '
+	longdir=rundir-01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789 &&
+	mkdir -p $longdir &&
+	test_must_fail flux start ${ARGS} \
+		-Srundir=$longdir \
+		true 2>longrun.err &&
+	grep "exceeds max" longrun.err
+'
+
+test_expect_success 'setting local-uri to a long path fails (#3925)' '
+	longdir=rundir-01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789 &&
+	mkdir -p $longdir &&
+	test_must_fail flux start ${ARGS} \
+		-Slocal-uri=local://$longdir/local-0 \
+		true 2>longuri.err &&
+	grep "exceeds max" longuri.err
 '
 
 reactorcat=${SHARNESS_TEST_DIRECTORY}/reactor/reactorcat
@@ -350,20 +756,65 @@ test_expect_success 'reactor: reactorcat example program works' '
 	test_must_fail test -s reactorcat.devnull.out
 '
 
-test_expect_success 'flux-start: panic rank 1 of a size=2 instance' '
-	! flux start --killer-timeout=0.2 --bootstrap=selfpmi --size=2 \
-		bash -c "flux getattr rundir; flux comms -r 1 panic fubar; sleep 5" >panic.out 2>panic.err
-'
-test_expect_success 'flux-start: panic message reached stderr' '
-	grep -q fubar panic.err
+test_expect_success 'no unit tests built with libtool wrapper' '
+	find ${FLUX_BUILD_DIR} \
+		-name "test_*.t" \
+		-type f \
+		-executable \
+		-printf "%h\n" \
+		| uniq \
+		| xargs -i basename {} > test_dirs &&
+	test_debug "cat test_dirs" &&
+	test_must_fail grep -q "\.libs" test_dirs
 '
-# flux-start: 1 (pid 10023) exited with rc=1
-test_expect_success 'flux-start: rank 1 exited with rc=1' '
-	egrep "flux-start: 1 .* exited with rc=1" panic.err
-'
-# flux-start: 0 (pid 21474) Killed
-test_expect_success 'flux-start: rank 0 Killed' '
-	egrep "flux-start: 0 .* Killed" panic.err
+
+CMDS="\
+R \
+admin \
+cron \
+event \
+exec \
+job \
+jobs \
+jobtap \
+keygen \
+kvs \
+logger \
+module \
+ping \
+queue \
+resource \
+start \
+terminus \
+"
+
+test_cmd_help ()
+{
+	local rc=0
+	for cmd in ${CMDS}; do
+		flux ${cmd} --help 2>&1 | grep -i usage || rc=1
+	done
+	return ${rc}
+}
+
+KVS_SUBCMDS="\
+namespace \
+eventlog \
+"
+
+test_kvs_subcmd_help ()
+{
+	local rc=0
+	for subcmd in ${KVS_SUBCMDS}; do
+		flux kvs ${subcmd} --help 2>&1 | grep -i usage || rc=1
+	done
+	return ${rc}
+}
+
+test_expect_success 'command --help works outside of flux instance' '
+	flux --help 2>&1 | grep -i usage &&
+	test_cmd_help &&
+	test_kvs_subcmd_help
 '
 
 # Note: flux-start auto-removes rundir
diff --git a/t/t0002-request.t b/t/t0002-request.t
index 88db35916c1f..80bda8c77de7 100755
--- a/t/t0002-request.t
+++ b/t/t0002-request.t
@@ -10,19 +10,14 @@ Verify basic request/response/rpc handling.
 . `dirname $0`/sharness.sh
 test_under_flux 2 minimal
 
-#  Set path to jq(1)
-#
-jq=$(which jq 2>/dev/null)
-test -n "$jq" && test_set_prereq HAVE_JQ
-
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
 
 test_expect_success 'flux_rpc(3) example runs' '
-	${FLUX_BUILD_DIR}/doc/man3/trpc
+	${FLUX_BUILD_DIR}/doc/man3/example/rpc
 '
 
 test_expect_success 'flux_rpc_then(3) example runs' '
-	${FLUX_BUILD_DIR}/doc/man3/trpc_then
+	${FLUX_BUILD_DIR}/doc/man3/example/rpc_then
 '
 
 test_expect_success 'request: load req module on rank 0' '
@@ -39,6 +34,11 @@ test_expect_success 'request: load req module on rank 1' '
 test_expect_success 'request: simple rpc with no payload' '
 	${FLUX_BUILD_DIR}/t/request/treq null
 '
+test_expect_success 'request: simple rpc with no payload (traced)' '
+	FLUX_HANDLE_TRACE=1 \
+		${FLUX_BUILD_DIR}/t/request/treq null 2>trace.out &&
+	grep ">" trace.out
+'
 
 test_expect_success 'request: simple rpc to rank 0' '
 	${FLUX_BUILD_DIR}/t/request/treq --rank 0 null
@@ -105,14 +105,14 @@ test_expect_success 'request: rpc test client works with no request payload' '
 	$RPC attr.list </dev/null >attr.list.out &&
 		test -s attr.list.out
 '
-test_expect_success HAVE_JQ 'request: rpc test client works with request payload' '
+test_expect_success 'request: rpc test client works with request payload' '
 	$jq -j -c -n  "{name:\"rank\"}" | $RPC attr.get >attr.get.out &&
 		test -s attr.get.out
 '
 test_expect_success 'request: rpc test client handles expected failure' '
 	$RPC attr.get 71 </dev/null
 '
-test_expect_success HAVE_JQ 'request: rpc test client handles expected failure other than EPROTO' '
+test_expect_success 'request: rpc test client handles expected failure other than EPROTO' '
 	$jq -j -c -n  "{name:\"noexist\"}" | $RPC attr.get 2
 '
 test_expect_success 'request: rpc test client handles unexpected failure' '
diff --git a/t/t0003-module.t b/t/t0003-module.t
index 683682ec465e..2d253ddccd94 100755
--- a/t/t0003-module.t
+++ b/t/t0003-module.t
@@ -14,233 +14,195 @@ invalid_rank() {
 	echo $((${SIZE} + 1))
 }
 
-test_expect_success 'module: load test module' '
-	flux module load \
-		${FLUX_BUILD_DIR}/t/module/.libs/parent.so
-'
-test_expect_success 'module: reload test module' '
-	flux module reload \
-		${FLUX_BUILD_DIR}/t/module/.libs/parent.so
-'
+testmod=${FLUX_BUILD_DIR}/t/module/.libs/testmod.so
+legacy=${FLUX_BUILD_DIR}/t/module/.libs/legacy.so
 
-test_expect_success 'module: lsmod shows test module' '
-	flux module list | grep parent
-'
+module_status_bad_proto() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"module.status\").get())"
+}
 
-test_expect_success 'module: cannot load the same module twice' '
-	test_must_fail flux module load \
-		${FLUX_BUILD_DIR}/t/module/.libs/parent.so
-'
+module_status () {
+	flux python -c "import flux; print(flux.Flux().rpc(\"module.status\",{\"status\":0}).get())"
+}
 
-test_expect_success 'module: unload test module' '
-	flux module remove parent
-'
+module_getinfo () {
+	FLUX_HANDLE_TRACE=1 flux python -c "import flux; print(flux.Flux().rpc(\"$1.info\").get_str())"
+}
 
-test_expect_success 'module: lsmod does not show test module' '
-	! flux module list | grep parent
-'
+# Usage: module_debug_defer modname True|False
+module_debug_defer () {
+        flux python -c "import flux; flux.Flux().rpc(\"module.debug\",{\"name\":\"$1\",\"defer\":$2}).get()"
+}
 
-test_expect_success 'module API: load test module' '
-	flux module load \
-		${FLUX_BUILD_DIR}/t/module/.libs/parent.so
-'
 
-test_expect_success 'module API: lsmod shows test module' '
-	flux module list | grep parent
+test_expect_success 'module: load test module' '
+	flux module load $testmod
 '
-
-test_expect_success 'module API: cannot load the same module twice' '
-        test_must_fail flux module load \
-		${FLUX_BUILD_DIR}/t/module/.libs/parent.so
+test_expect_success 'module: reload test module' '
+	flux module reload $testmod
 '
 
-test_expect_success 'module API: unload test module' '
-	flux module remove parent
+test_expect_success 'module: lsmod shows test module' '
+	flux module list | grep testmod
 '
-
-test_expect_success 'module API: lsmod does not show test module' '
-	! flux module list | grep parent
+test_expect_success 'module: lsmod -l shows test module path' '
+	flux module list -l | grep $testmod
 '
 
-test_expect_success 'module: load test module (all ranks)' '
-	flux exec -r all flux module load \
-		${FLUX_BUILD_DIR}/t/module/.libs/parent.so
+test_expect_success 'module: cannot load the same module twice' '
+	test_must_fail flux module load $testmod
 '
-
-test_expect_success 'module: remove/load --rank 1' '
-	flux module remove -r 1 parent &&
-	flux module load -r 1 \
-		${FLUX_BUILD_DIR}/t/module/.libs/parent.so
+test_expect_success 'module: unless a new --name is specified' '
+	flux module load --name smurf $testmod
 '
-
-test_expect_success 'module: load submodule with invalid args (this rank)' '
-	test_must_fail flux module load \
-		${FLUX_BUILD_DIR}/t/module/.libs/child.so \
-		wrong argument list
+test_expect_success 'module: lsmod shows test module' '
+	flux module list | grep smurf
 '
-
-test_expect_success 'module: load submodule (all ranks)' '
-	flux exec -r all flux module load \
-		${FLUX_BUILD_DIR}/t/module/.libs/child.so \
-		foo=42 bar=abcd
+test_expect_success 'module: module answers to new name' '
+	test $(module_getinfo smurf) = "smurf"
 '
-
-test_expect_success 'module: lsmod shows submodule (all ranks)' '
-	flux exec -r all flux module list parent | grep parent.child
+test_expect_success 'module: lsmod -l shows test module path twice' '
+	count=$(flux module list -l | grep $testmod | wc -l) &&
+	test $count -eq 2
 '
-
-#
-# Expected 'lsmod -r all parent' output is something like this:
-#  parent.child         1113944 2517539    0  I         3 rank3,test1,test2
-#  parent.child         1113944 2517539    0  I         0 rank0,test1,test2
-#  parent.child         1113944 2517539    0  I         2 rank2,test1,test2
-#  parent.child         1113944 2517539    0  I         1 rank1,test1,test2
-#
-test_expect_success 'module: lsmod -r all parent works' '
-	flux exec -r all flux module list parent | grep child >child.lsmod.out
+test_expect_success 'module: reload test module with new name works' '
+	flux module reload --name smurf $testmod
 '
-test_expect_success 'module: hardwired test1,test2 services are listed in sorted order' '
-	grep -q test1,test2 child.lsmod.out
+test_expect_success 'module: unload test module using new name' '
+	flux module remove smurf
 '
-test_expect_success "module: hardwired rankN services are listed" '
-	for rank in $(seq 0 $(($SIZE-1))); do \
-		grep -q rank${rank} child.lsmod.out; \
-	done
+test_expect_success 'module: unload test module' '
+	flux module remove testmod
 '
-test_expect_success "module: there are size=$SIZE lines of output due to unique rankN service" '
-	test $(wc -l < child.lsmod.out) -eq $SIZE
+test_expect_success 'module: lsmod does not show test module' '
+	flux module list -l >list.out &&
+	test_must_fail grep $testmod list.out
 '
 
-test_expect_success 'module: unload submodule (all ranks)' '
-	flux exec -r all flux module remove parent.child
+test_expect_success 'module: insmod returns initialization error' '
+	test_must_fail flux module load $testmod --init-failure
 '
 
-test_expect_success 'module: lsmod does not show submodule (all ranks)' '
-	flux exec -r all flux module list parent >noshow.out &&
-	test_must_fail grep parent.child noshow.out
+test_expect_success 'module: load fails with no arguments' '
+	test_must_fail flux module load
 '
-
-test_expect_success 'module: unload test module (all ranks)' '
-	flux exec -r all flux module remove parent
+test_expect_success 'module: reload fails with no arguments' '
+	test_must_fail flux module reload
 '
-
-test_expect_success 'module: insmod returns initialization error' '
-	test_must_fail flux module load \
-		${FLUX_BUILD_DIR}/t/module/.libs/parent.so --init-failure
+test_expect_success 'module: remove fails with no arguments' '
+	test_must_fail flux module remove
+'
+test_expect_success 'module: list fails with argument' '
+	test_must_fail flux module list foo
+'
+# register a long service name that is truncateed in --long output
+# just to exercise that code in test
+test_expect_success 'module: list shows service name' '
+	flux module load $testmod --service=abcdefghijklmnopqrstuvwxyz &&
+	flux module list -l | grep abc &&
+	flux module remove $testmod
 '
 
 test_expect_success 'module: load fails on invalid module' '
-	! flux module load nosuchmodule 2> stderr &&
-	grep "nosuchmodule: not found in module search path" stderr
+	test_must_fail flux module load nosuchmodule 2>load.err &&
+	grep "module not found" load.err
 '
 
 test_expect_success 'module: remove fails on invalid module' '
 	test_must_fail flux module remove nosuchmodule 2>nosuch.err &&
 	grep "nosuchmodule: No such file or directory" nosuch.err
 '
-test_expect_sucess 'module: remove -f succeeds on nonexistent module' '
+test_expect_success 'module: remove -f succeeds on nonexistent module' '
 	flux module remove -f nosuchmodule
 '
-test_expect_success 'module: info works' '
-	flux module info ${FLUX_BUILD_DIR}/t/module/.libs/parent.so
+test_expect_success 'module: legacy module naming still works' '
+	flux module load $legacy &&
+	flux module remove $legacy
 '
-
-test_expect_success 'module: info fails on invalid module' '
-	! flux module info nosuchmodule
+test_expect_success 'module: legacy module cannot be loaded under new name' '
+	test_must_fail flux module load --name=newname $legacy
 '
 
 # N.B. avoid setting the actual debug bits - lets reserve LSB
-TESTMOD=connector-local
+REALMOD=connector-local
 
 test_expect_success 'flux module debug gets debug flags' '
-	FLAGS=$(flux module debug $TESTMOD) &&
+	FLAGS=$(flux module debug $REALMOD) &&
 	test "$FLAGS" = "0x0"
 '
 test_expect_success 'flux module debug --setbit sets individual debug flags' '
-	flux module debug --setbit 0x10000 $TESTMOD &&
-	FLAGS=$(flux module debug $TESTMOD) &&
+	flux module debug --setbit 0x10000 $REALMOD &&
+	FLAGS=$(flux module debug $REALMOD) &&
 	test "$FLAGS" = "0x10000"
 '
 test_expect_success 'flux module debug --set replaces debug flags' '
-	flux module debug --set 0xff00 $TESTMOD &&
-	FLAGS=$(flux module debug $TESTMOD) &&
+	flux module debug --set 0xff00 $REALMOD &&
+	FLAGS=$(flux module debug $REALMOD) &&
 	test "$FLAGS" = "0xff00"
 '
 test_expect_success 'flux module debug --clearbit clears individual debug flags' '
-	flux module debug --clearbit 0x1000 $TESTMOD &&
-	FLAGS=$(flux module debug $TESTMOD) &&
+	flux module debug --clearbit 0x1000 $REALMOD &&
+	FLAGS=$(flux module debug $REALMOD) &&
 	test "$FLAGS" = "0xef00"
 '
 test_expect_success 'flux module debug --clear clears debug flags' '
-	flux module debug --clear $TESTMOD &&
-	FLAGS=$(flux module debug $TESTMOD) &&
+	flux module debug --clear $REALMOD &&
+	FLAGS=$(flux module debug $REALMOD) &&
 	test "$FLAGS" = "0x0"
 '
 
 # test stats
 
 test_expect_success 'flux module stats gets comms statistics' '
-	flux module stats $TESTMOD >comms.stats &&
-	grep -q "#request (tx)" comms.stats &&
-	grep -q "#request (rx)" comms.stats &&
-	grep -q "#response (tx)" comms.stats &&
-	grep -q "#response (rx)" comms.stats &&
-	grep -q "#event (tx)" comms.stats &&
-	grep -q "#event (rx)" comms.stats &&
-	grep -q "#keepalive (tx)" comms.stats &&
-	grep -q "#keepalive (rx)" comms.stats
-'
-
-test_expect_success 'flux module stats --parse "#event (tx)" counts events' '
-	EVENT_TX=$(flux module stats --parse "#event (tx)" $TESTMOD) &&
+	flux module stats $REALMOD >comms.stats
+'
+
+test_expect_success 'flux module stats --parse tx.event counts events' '
+	EVENT_TX=$(flux module stats --parse tx.event $REALMOD) &&
 	flux event pub xyz &&
-	EVENT_TX2=$(flux module stats --parse "#event (tx)" $TESTMOD) &&
+	EVENT_TX2=$(flux module stats --parse tx.event $REALMOD) &&
 	test "$EVENT_TX" = $((${EVENT_TX2}-1))
 '
 
 test_expect_success 'flux module stats --clear works' '
 	flux event pub xyz &&
-	flux module stats --clear $TESTMOD &&
-	EVENT_TX2=$(flux module stats --parse "#event (tx)" $TESTMOD) &&
+	flux module stats --clear $REALMOD &&
+	EVENT_TX2=$(flux module stats --parse tx.event $REALMOD) &&
 	test "$EVENT_TX" = 0
 '
 
 test_expect_success 'flux module stats --clear-all works' '
 	flux event pub xyz &&
-	flux module stats --clear-all $TESTMOD &&
-	EVENT_TX2=$(flux module stats --parse "#event (tx)" $TESTMOD) &&
+	flux module stats --clear-all $REALMOD &&
+	EVENT_TX2=$(flux module stats --parse tx.event $REALMOD) &&
 	test "$EVENT_TX" = 0
 '
-
 test_expect_success 'flux module stats --scale works' '
 	flux event pub xyz &&
-	EVENT_TX=$(flux module stats --parse "#event (tx)" $TESTMOD) &&
-	EVENT_TX2=$(flux module stats --parse "#event (tx)" --scale=2 $TESTMOD) &&
+	EVENT_TX=$(flux module stats --parse tx.event $REALMOD) &&
+	EVENT_TX2=$(flux module stats --parse tx.event --scale=2 $REALMOD) &&
 	test "$EVENT_TX2" -eq $((${EVENT_TX}*2))
 '
 
 test_expect_success 'flux module stats --rusage works' '
-	flux module stats --rusage $TESTMOD >rusage.stats &&
-	grep -q utime rusage.stats &&
-	grep -q stime rusage.stats &&
-	grep -q maxrss rusage.stats &&
-	grep -q ixrss rusage.stats &&
-	grep -q idrss rusage.stats &&
-	grep -q isrss rusage.stats &&
-	grep -q minflt rusage.stats &&
-	grep -q majflt rusage.stats &&
-	grep -q nswap rusage.stats &&
-	grep -q inblock rusage.stats &&
-	grep -q oublock rusage.stats &&
-	grep -q msgsnd rusage.stats &&
-	grep -q msgrcv rusage.stats &&
-	grep -q nsignals rusage.stats &&
-	grep -q nvcsw rusage.stats &&
-	grep -q nivcsw rusage.stats
+	flux module stats --rusage $REALMOD
+'
+test_expect_success 'flux module stats -Rself works' '
+	flux module stats -Rself $REALMOD
+'
+test_expect_success 'flux module stats --rusage=children works' '
+	flux module stats --rusage=children $REALMOD
+'
+# RUSAGE_THREAD is a non-portable GNU extension
+test_expect_success 'flux module stats --rusage=thread might work :-)' '
+	test_might_fail flux module stats --rusage=thread $REALMOD
+'
+test_expect_success 'flux module stats --rusage=badopt fails' '
+	test_must_fail flux module stats --rusage=badopt $REALMOD
 '
 
 test_expect_success 'flux module stats --rusage --parse maxrss works' '
-	RSS=$(flux module stats --rusage --parse maxrss $TESTMOD) &&
+	RSS=$(flux module stats --rusage --parse maxrss $REALMOD) &&
 	test "$RSS" -gt 0
 '
 
@@ -257,7 +219,6 @@ test_expect_success 'flux module -h lists subcommands' '
 	grep -q remove module.help &&
 	grep -q reload module.help &&
 	grep -q load module.help &&
-	grep -q info module.help &&
 	grep -q stats module.help &&
 	grep -q debug module.help
 '
@@ -267,4 +228,91 @@ test_expect_success 'flux module load "noexist" fails' '
 	grep -q "not found" noexist.out
 '
 
+test_expect_success 'flux_module_set_running - load test module' '
+	run_timeout 10 \
+		flux module load ${FLUX_BUILD_DIR}/t/module/.libs/running.so
+'
+test_expect_success 'flux_module_set_running - signal module to enter reactor' '
+	flux event pub running.go
+'
+test_expect_success 'flux_module_set_running - remove test module' '
+	flux module remove running
+'
+test_expect_success 'module.status rejects malformed request' '
+	test_must_fail module_status_bad_proto 2>proto.err &&
+	grep "error decoding/finding module.status" proto.err
+'
+test_expect_success 'module.status rejects request from unknown sender' '
+	test_must_fail module_status 2>sender.err &&
+	grep "error decoding/finding module.status" sender.err
+'
+# issue #5255
+test_expect_success 'module with version ext can be loaded by name' '
+	mkdir -p testmoddir &&
+	cp $testmod testmoddir/testmod.so.0.0.0 &&
+	FLUX_MODULE_PATH_PREPEND=$(pwd)/testmoddir flux start \
+	    bash -c \
+	    "flux module load testmod; \
+	    rc=\$?; \
+	    flux module list -l; \
+	    flux module remove -f testmod; \
+	    exit \$rc" \
+	    >modlist.out &&
+	grep testmod.so.0.0.0 modlist.out
+'
+
+test_expect_success 'module: rank attribute is cached' '
+        flux module load $testmod --attr-is-cached=rank
+'
+test_expect_success 'module: size attribute is cached' '
+        flux module reload $testmod --attr-is-cached=size
+'
+test_expect_success 'module: security.owner attribute is cached' '
+        flux module reload $testmod --attr-is-cached=security.owner
+'
+test_expect_success 'module: log-stderr-level attribute is NOT cached' '
+        test_must_fail flux module reload $testmod \
+            --attr-is-cached=log-stderr-level
+'
+test_expect_success 'module: configuration object is cached' '
+        flux module reload -f $testmod --config-is-cached
+'
+test_expect_success 'module: remove testmod if loaded' '
+        flux module remove -f testmod
+'
+test_expect_success 'module: load without unload causes broker failure' '
+	test_must_fail flux start \
+	    -Sbroker.rc1_path= \
+	    -Sbroker.rc3_path= \
+	    flux module load content 2>nounload.err
+'
+test_expect_success 'module: module name is called out' '
+	grep ".content. was not properly shut down" nounload.err
+'
+
+test_expect_success 'module: load testmod' '
+        flux module load $testmod
+'
+test_expect_success 'module-debug name=badname fails' '
+        test_must_fail module_debug_defer badname True
+'
+test_expect_success 'module-debug defer=42 fails (not boolean)' '
+        test_must_fail module_debug_defer testmod 42
+'
+test_expect_success 'module-debug name=testmod defer=True works' '
+        module_debug_defer testmod True
+'
+test_expect_success 'testmod does not respond to ping' '
+        test_expect_code 137 run_timeout 2 flux ping --count=1 testmod
+'
+test_expect_success 'module-debug name=testmod defer=False works' '
+        module_debug_defer testmod False
+'
+test_expect_success 'testmod does respond to ping' '
+        run_timeout 10 flux ping --count=1 testmod
+'
+test_expect_success 'module: remove testmod' '
+        flux module remove -f testmod
+'
+
 test_done
diff --git a/t/t0004-event.t b/t/t0004-event.t
index 7fced8b094c7..bd438ff0d9d4 100755
--- a/t/t0004-event.t
+++ b/t/t0004-event.t
@@ -10,13 +10,21 @@ test_under_flux ${SIZE} minimal
 
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
 
+test_expect_success 'load heartbeat module with fast rate' '
+        flux module load heartbeat period=0.1s
+'
+
 test_expect_success 'heartbeat is received on all ranks' '
 	run_timeout 5 \
-          flux exec -n flux event sub --count=1 hb >output_event_sub &&
-	hb_count=`grep "^hb" output_event_sub | wc -l` &&
+          flux exec -n flux event sub --count=1 heartbeat.pulse >output_event_sub &&
+	hb_count=`grep "^heartbeat.pulse" output_event_sub | wc -l` &&
         test $hb_count -eq $SIZE
 '
 
+test_expect_success 'unload heartbeat module' '
+	flux module remove heartbeat
+'
+
 test_expect_success 'events from rank 0 received correctly on rank 0' '
 	run_timeout 15 \
          $SHARNESS_TEST_SRCDIR/scripts/event-trace.lua \
@@ -79,8 +87,9 @@ test_expect_success 'publish private event with no payload (synchronous,loopback
 	run_timeout 5 flux event pub -p -s -l foo.bar
 '
 
-test_expect_success 'event.pub request with empty payload fails with EPROTO(71)' '
-	${RPC} event.pub 71 </dev/null
+test_expect_success 'event.publish request with empty payload fails with EPROTO(71)' '
+	${RPC} event.publish 71 </dev/null
 '
 
+
 test_done
diff --git a/t/t0005-exec-jobid.t b/t/t0005-exec-jobid.t
new file mode 100755
index 000000000000..d59d9f86c83d
--- /dev/null
+++ b/t/t0005-exec-jobid.t
@@ -0,0 +1,117 @@
+#!/bin/sh
+#
+
+test_description='Test exec command --jobid option
+
+Test exec --jobid functionality
+'
+
+. `dirname $0`/sharness.sh
+SIZE=4
+test_under_flux ${SIZE} job
+
+if flux job submit --help 2>&1 | grep -q sign-type; then
+        test_set_prereq FLUX_SECURITY
+fi
+
+TMPDIR=$(cd /tmp && $(which pwd))
+
+test_expect_success 'flux exec --jobid fails with invalid job id' '
+	test_expect_code 1 flux exec --jobid=f-o-o true 2>badid.err &&
+	test_debug "cat badid.err" &&
+	grep "error parsing jobid" badid.err
+'
+test_expect_success 'flux exec --jobid fails for nonexistent job id' '
+	test_expect_code 1 flux exec --jobid=f1 true 2>noid.err &&
+	test_debug "cat noid.err" &&
+	grep "not found" noid.err
+'
+test_expect_success 'run two jobs on different ranks' '
+	flux submit --wait-event=start --bcc=1-2 -N1 sleep inf &&
+	id1=$(flux job last [-1:]) &&
+	id2=$(flux job last) &&
+	test_debug "flux jobs -no \"{id} {ranks}\""
+'
+test_expect_success 'flux exec --jobid works' '
+	rank1=$(flux exec --jobid=$id1 flux getattr rank) &&
+	rank2=$(flux exec --jobid=$id2 flux getattr rank) &&
+	test_debug "echo flux exec --jobid=$id1 ran on rank $rank1" &&
+	test_debug "echo flux exec --jobid=$id2 ran on rank $rank2" &&
+	test $rank1 -eq $(flux jobs -no {ranks} $id1) &&
+	test $rank2 -eq $(flux jobs -no {ranks} $id2)
+'
+test_expect_success 'run one job on two ranks' '
+	jobid=$(flux submit --wait-event=start -N2 sleep inf)
+'
+test_expect_success 'flux exec --jobid on multi-node jobs runs on all ranks' '
+	flux exec --jobid=$jobid --label-io flux getattr rank >2node.out &&
+	test_debug "cat 2node.out" &&
+	grep "2: 2" 2node.out &&
+	grep "3: 3" 2node.out
+'
+test_expect_success 'flux exec --jobid works with --rank option' '
+	flux exec --jobid=$jobid -r 0 flux getattr rank &&
+	test $(flux exec --jobid=$jobid -r 0 flux getattr rank) -eq 2 &&
+	flux exec --jobid=$jobid -r 1 flux getattr rank &&
+	test $(flux exec --jobid=$jobid -r 1 flux getattr rank) -eq 3
+'
+test_expect_success 'flux exec --jobid fails with invalid --rank option' '
+	test_must_fail flux exec --jobid=$jobid -r 3 hostname &&
+	test_must_fail flux exec --jobid=$jobid -r 0-3 hostname
+'
+test_expect_success 'flux exec --jobid works with --exclude option' '
+	flux exec --jobid=$jobid -x 0 flux getattr rank &&
+	test $(flux exec --jobid=$jobid -x 0 flux getattr rank) -eq 3 &&
+	flux exec --jobid=$jobid -x 1 flux getattr rank &&
+	test $(flux exec --jobid=$jobid -x 1 flux getattr rank) -eq 2
+'
+test_expect_success 'flux exec --jobid ignores invalid --exclude ranks' '
+	test_debug "flux exec --jobid=$jobid -l -x 3-4 flux getattr rank" &&
+	test $(flux exec --jobid=$jobid -x 1-5 flux getattr rank) -eq 2
+'
+test_expect_success 'flux exec --jobid fails if there are no ranks to target' '
+	test_must_fail flux exec --jobid=$jobid -x 0-1 hostname
+'
+test_exec() {
+	FLUX_HANDLE_ROLEMASK=0x2 FLUX_HANDLE_USERID=$1 \
+	    flux exec -r$3 --jobid=$2 id
+}
+test_expect_success FLUX_SECURITY 'flux exec --jobid fails from other user' '
+	alt_userid=$(($(id -u)+1)) &&
+	test_debug "echo testing with handle userid=$alt_userid" &&
+	test_must_fail test_exec $alt_userid $jobid 0 2>eperm0.err &&
+	test_debug "cat eperm0.err" &&
+	grep "failed to get shell.init event" eperm0.err &&
+	test_must_fail test_exec $alt_userid $jobid 1 2>eperm1.err &&
+	test_debug "cat eperm1.err" &&
+	grep "failed to get shell.init event" eperm1.err
+'
+job_service() {
+	flux job eventlog --format=json -p exec $1 \
+	    | jq -r 'select(.name == "shell.init") .context.service'
+}
+# Usage: test_exec_direct userid rolemask jobid
+test_exec_direct() {
+	service=$(job_service $2).rexec && \
+	ranks=$(flux jobs -n --format="{ranks}" $2) && \
+	FLUX_HANDLE_ROLEMASK=0x2 FLUX_HANDLE_USERID=$1 \
+	    flux exec -r$ranks --service=$service id
+}
+test_expect_success FLUX_SECURITY 'flux exec direct to shell fails also' '
+	alt_userid=$(($(id -u)+1)) &&
+	test_must_fail test_exec_direct $alt_userid $jobid 2>eperm.err &&
+	test_debug "cat eperm.err" &&
+	grep "requires owner credentials" eperm.err
+'
+
+test_expect_success 'cancel jobs' '
+	flux cancel --all &&
+	flux job wait-event $id1 clean
+'
+test_expect_success 'flux exec --jobid on inactive job fails' '
+	test_must_fail flux exec --jobid=$id1 hostname 2>inactive.err &&
+	test_debug "cat inactive.err" &&
+	grep "not currently running" inactive.err
+'
+
+test_done
diff --git a/t/t0005-exec.t b/t/t0005-exec.t
index 827005268847..547591e47701 100755
--- a/t/t0005-exec.t
+++ b/t/t0005-exec.t
@@ -18,19 +18,19 @@ invalid_rank() {
 TMPDIR=$(cd /tmp && $(which pwd))
 
 test_expect_success 'basic exec functionality' '
-	flux exec -n /bin/true
+	flux exec -n true
 '
 
 test_expect_success 'exec to specific rank' '
-	flux exec -n -r 0 /bin/true
+	flux exec -n -r 0 true
 '
 
 test_expect_success 'exec to "all" ranks' '
-	flux exec -n -r all /bin/true
+	flux exec -n -r all true
 '
 
 test_expect_success 'exec to subset of ranks' '
-	flux exec -n -r 2-3 /bin/true
+	flux exec -n -r 2-3 true
 '
 
 test_expect_success 'exec to all except a set of ranks' '
@@ -40,31 +40,46 @@ test_expect_success 'exec to all except a set of ranks' '
 	1
 	EOT
 '
+test_expect_success 'exec with --ranks and --exclude works' '
+	flux exec -r 2-3 -x 3 flux getattr rank &&
+	test $(flux exec -r 2-3 -x 3 flux getattr rank) -eq 2
+'
+test_expect_success 'configure access.allow-guest-user = true' '
+	flux config load <<-EOT
+	access.allow-guest-user = true
+	EOT
+'
+test_expect_success 'exec to rank 0 from another rank is an error' '
+	test_must_fail flux exec -n -r 1 flux exec -n -r 0 true
+'
+test_expect_success 'configure access.allow-guest-user = false' '
+	flux config load <<-EOT
+	access.allow-guest-user = false
+	EOT
+'
+test_expect_success 'exec to rank 0 from another rank works' '
+	flux exec -n -r 1 flux exec -n -r 0 true
+'
 
 test_expect_success 'exec to non-existent rank is an error' '
-	test_must_fail flux exec -n -r $(invalid_rank) /bin/true
+	test_must_fail flux exec -n -r $(invalid_rank) true
 '
 
-test_expect_success 'exec to valid and invalid ranks works' '
-        # But, flux-exec should return failure:
-	! flux exec -n -r 0,$(invalid_rank) echo working 1>stdout 2>stderr </dev/null &&
-	count1=$(grep -c working stdout) &&
-	count2=$(grep -c "No route to host" stderr) &&
-	test "$count1" = "1" &&
-	test "$count2" = "1"
+test_expect_success 'exec to empty idset is an error' '
+	test_must_fail flux exec -n -r 0-1 -x 0-1 true
 '
 
 test_expect_success 'test_on_rank works' '
-	test_on_rank 1 /bin/true
+	test_on_rank 1 true
 '
 
 test_expect_success 'test_on_rank sends to correct rank' '
-	flux comms info | grep rank=0 &&
-	test_on_rank 1 sh -c "flux comms info | grep -q rank=1"
+	flux getattr rank | grep 0 &&
+	test_on_rank 1 sh -c "flux getattr rank | grep -x 1"
 '
 
 test_expect_success 'test_on_rank works with test_must_fail' '
-	test_must_fail test_on_rank 1 sh -c "flux comms info | grep -q rank=0"
+	test_must_fail test_on_rank 1 sh -c "flux getattr rank | grep -x 0"
 '
 
 test_expect_success 'flux exec passes environment variables' '
@@ -89,14 +104,18 @@ test_expect_success 'flux exec -d option works' '
 	flux exec -n -d ${TMPDIR} sh -c "test \`pwd\` = \"${TMPDIR}\""
 '
 
+test_expect_success 'flux exec -d none works' '
+	(cd /tmp && flux exec -n -d none sh -c "test \$(pwd) != /tmp")
+'
+
 # Run a script on ranks 0-3 simultaneously with each rank writing the
 #  rank id to a file. After successful completion, the contents of the files
-#  are verfied to ensure each rank connected to the right broker.
+#  are verified to ensure each rank connected to the right broker.
 test_expect_success 'test_on_rank works on multiple ranks' '
-	ouput_dir=$(pwd) &&
+	output_dir=$(pwd) &&
 	rm -f rank_output.* &&
 	cat >multiple_rank_test <<EOF &&
-rank=\`flux comms info | grep rank | sed s/rank=//\`
+rank=\`flux getattr rank\`
 echo \$rank > $(pwd)/rank_output.\${rank}
 exit 0
 EOF
@@ -113,22 +132,28 @@ test_expect_success 'flux exec exits with code 127 for file not found' '
 
 test_expect_success 'flux exec outputs appropriate error message for file not found' '
 	test_expect_code 127 flux exec -n ./nosuchprocess 2> exec.stderr &&
-        grep "No such file or directory" exec.stderr
+	grep "error launching process" exec.stderr &&
+	grep "No such file or directory" exec.stderr
 '
 
 test_expect_success 'flux exec exits with code 126 for non executable' '
-	test_expect_code 126 flux exec -n /dev/null
+	test_expect_code 126 flux exec -n /dev/null 2> exec.stderr2 &&
+	grep "error launching process" exec.stderr2 &&
+	grep "Permission denied" exec.stderr2
 '
 
-test_expect_success 'flux exec exits with code 68 (EX_NOHOST) for rank not found' '
-	test_expect_code 68 run_timeout 10 flux exec -n -r 1000 ./nosuchprocess
-'
 test_expect_success NO_ASAN 'flux exec passes non-zero exit status' '
 	test_expect_code 2 flux exec -n sh -c "exit 2" &&
 	test_expect_code 3 flux exec -n sh -c "exit 3" &&
 	test_expect_code 139 flux exec -n sh -c "kill -11 \$\$"
 '
 
+test_expect_success 'flux exec fails with --with-imp if no IMP configured' '
+	test_expect_code 1 flux exec --with-imp hostname 2>exec-no-imp.out &&
+	test_debug "cat exec-no-imp.out" &&
+	grep "exec\.imp path not found in config" exec-no-imp.out
+'
+
 test_expect_success NO_ASAN 'flux exec outputs tasks with errors' '
 	! flux exec -n sh -c "exit 2" > 2.out 2>&1 &&
         grep "\[0-3\]: Exit 2" 2.out &&
@@ -145,15 +170,15 @@ test_expect_success 'basic IO testing' '
 '
 
 test_expect_success 'per rank output works' '
-	flux exec -n -r 1 sh -c "flux comms info | grep rank" | grep ^rank=1\$ &&
-	flux exec -n -lr 2 sh -c "flux comms info | grep rank" | grep ^2:\ rank=2\$ &&
+	flux exec -n -r 1 sh -c "flux getattr rank" | grep -x 1 &&
+	flux exec -n -lr 2 sh -c "flux getattr rank" | grep -x "2: 2" &&
 	cat >expected <<EOF &&
-0: rank=0
-1: rank=1
-2: rank=2
-3: rank=3
+0: 0
+1: 1
+2: 2
+3: 3
 EOF
-	flux exec -n -lr 0-3 sh -c "flux comms info | grep rank" | sort -n >output &&
+	flux exec -n -lr 0-3 sh -c "flux getattr rank" | sort -n >output &&
 	test_cmp output expected
 '
 
@@ -172,20 +197,24 @@ test_expect_success 'I/O -- long lines' '
 	test_cmp output expected
 '
 
-
+waitfile=$SHARNESS_TEST_SRCDIR/scripts/waitfile.lua
 test_expect_success 'signal forwarding works' '
 	cat >test_signal.sh <<-EOF &&
 	#!/bin/bash
 	sig=\${1-INT}
-	flux exec -n sleep 100 </dev/null &
-	sleep 1 &&
+	rm -f sleepready.out
+	mkfifo input.fifo
+	stdbuf --output=L \
+	    flux exec -v -n awk "BEGIN {print \"hi\"} {print}" input.fifo \
+	        >sleepready.out &
+	$waitfile -vt 20 -p ^hi -c ${SIZE} sleepready.out &&
 	kill -\$sig %1 &&
 	wait %1
 	exit \$?
 	EOF
 	chmod +x test_signal.sh &&
-	test_expect_code 130 run_timeout 10 ./test_signal.sh INT &&
-	test_expect_code 143 run_timeout 10 ./test_signal.sh TERM
+	test_expect_code 130 run_timeout 20 ./test_signal.sh INT &&
+	test_expect_code 143 run_timeout 20 ./test_signal.sh TERM
 '
 
 test_expect_success 'flux-exec: stdin bcast to all ranks (default)' '
@@ -218,7 +247,57 @@ test_expect_success 'stdin redirect from /dev/null works without -n' '
 '
 
 test_expect_success 'stdin redirect from /dev/null works with -n' '
-       test_expect_code 0 run_timeout 10 flux exec -n -r0-3 cat
+	test_expect_code 0 run_timeout 10 flux exec -n -r0-3 cat
+'
+
+test_expect_success 'create large file for tests' '
+	dd if=/dev/urandom of=5Mfile bs=5M count=1
+'
+
+test_expect_success 'create test script to redirect stdin to a file' '
+	cat <<-EOT >stdin2file &&
+	#!/bin/bash
+	rank=\$(flux getattr rank)
+	dd of=cpy.\$rank
+	EOT
+	chmod +x stdin2file
+'
+
+# piping a 5M file using a 4K buffer should overflow if flow control
+# is not functioning correctly
+test_expect_success 'stdin flow control works (1 rank)' '
+	cat 5Mfile | flux exec -r 0 --setopt=stdin_BUFSIZE=4096 ./stdin2file &&
+	cmp 5Mfile cpy.0 &&
+	rm cpy.0
+'
+
+test_expect_success 'stdin flow control works (all ranks)' '
+	cat 5Mfile | flux exec -r 0-3 --setopt=stdin_BUFSIZE=4096 ./stdin2file &&
+	cmp 5Mfile cpy.0 &&
+	cmp 5Mfile cpy.1 &&
+	cmp 5Mfile cpy.2 &&
+	cmp 5Mfile cpy.3 &&
+	rm cpy.*
+'
+
+test_expect_success 'create test script to redirect stdin to a file, one rank exits early' '
+	cat <<-EOT >stdin2file &&
+	#!/bin/bash
+	rank=\$(flux getattr rank)
+	if test \$rank -ne 0; then
+		dd of=cpy.\$rank
+	fi
+	EOT
+	chmod +x stdin2file
+'
+
+test_expect_success 'stdin flow control works (all ranks, one rank will exit early)' '
+	cat 5Mfile | flux exec -r 0-3 --setopt=stdin_BUFSIZE=4096 ./stdin2file &&
+	test_must_fail ls cpy.0 &&
+	cmp 5Mfile cpy.1 &&
+	cmp 5Mfile cpy.2 &&
+	cmp 5Mfile cpy.3 &&
+	rm cpy.*
 '
 
 test_expect_success 'stdin broadcast -- multiple lines' '
@@ -240,4 +319,17 @@ test_expect_success 'stdin broadcast -- long lines' '
 	done
 '
 
+test_expect_success 'dbus environment variable is set' '
+	DBUS_SESSION_BUS_ADDRESS= \
+	    flux exec -r 0 printenv DBUS_SESSION_BUS_ADDRESS
+'
+test_expect_success 'dbus environment variable is not overwritten if set' '
+	DBUS_SESSION_BUS_ADDRESS=xyz \
+	    flux exec -r 0 printenv DBUS_SESSION_BUS_ADDRESS >dbus.out &&
+	cat >dbus.exp <<-EOT &&
+	xyz
+	EOT
+	test_cmp dbus.exp dbus.out
+'
+
 test_done
diff --git a/t/t0005-rexec.t b/t/t0005-rexec.t
index cbaaf76e60bf..33e9ae80162d 100755
--- a/t/t0005-rexec.t
+++ b/t/t0005-rexec.t
@@ -12,213 +12,226 @@ SIZE=4
 test_under_flux ${SIZE} minimal
 
 TEST_SUBPROCESS_DIR=${FLUX_BUILD_DIR}/src/common/libsubprocess
+rexec_script="flux python ${SHARNESS_TEST_SRCDIR}/scripts/rexec.py"
+rexec="${FLUX_BUILD_DIR}/t/rexec/rexec"
 
 test_expect_success 'basic rexec functionality (process success)' '
-	${FLUX_BUILD_DIR}/t/rexec/rexec /bin/true
+	$rexec true
 '
 
 test_expect_success 'basic rexec functionality (process fail)' '
-	! ${FLUX_BUILD_DIR}/t/rexec/rexec /bin/false
+	! $rexec false
 '
 
 test_expect_success 'basic rexec - cwd correct' '
 	pwd=$(which pwd) &&
 	tmpdir=$(cd /tmp && $pwd) &&
 	(cd ${tmpdir} &&
-         cwd=`${FLUX_BUILD_DIR}/t/rexec/rexec $pwd` &&
-         test "$cwd" = "$tmpdir")
+	 cwd=`$rexec $pwd` &&
+	 test "$cwd" = "$tmpdir")
 '
 
 test_expect_success 'basic rexec - env passed through' '
-        export FOO_BAR_BAZ=10 &&
-        ${FLUX_BUILD_DIR}/t/rexec/rexec env > output &&
-        grep "FOO_BAR_BAZ=10" output
+	export FOO_BAR_BAZ=10 &&
+	$rexec env > output &&
+	grep "FOO_BAR_BAZ=10" output
 '
 
 test_expect_success 'basic rexec functionality (echo stdout)' '
-	${FLUX_BUILD_DIR}/t/rexec/rexec ${TEST_SUBPROCESS_DIR}/test_echo -P -O foobar.stdout > output &&
-        echo "stdout:foobar.stdout" > expected &&
-        test_cmp expected output
+	$rexec ${TEST_SUBPROCESS_DIR}/test_echo -P -O foobar.stdout > output &&
+	echo "stdout:foobar.stdout" > expected &&
+	test_cmp expected output
 '
 
 test_expect_success 'basic rexec functionality (echo stderr)' '
-	${FLUX_BUILD_DIR}/t/rexec/rexec ${TEST_SUBPROCESS_DIR}/test_echo -P -E foobar.stderr > output 2>&1 &&
-        echo "stderr:foobar.stderr" > expected &&
-        test_cmp expected output
+	$rexec ${TEST_SUBPROCESS_DIR}/test_echo -P -E foobar.stderr > output 2>&1 &&
+	echo "stderr:foobar.stderr" > expected &&
+	test_cmp expected output
 '
 
 test_expect_success 'basic rexec functionality (echo stdout/err)' '
-	${FLUX_BUILD_DIR}/t/rexec/rexec ${TEST_SUBPROCESS_DIR}/test_echo -O -E foobar.stdouterr > output 2>&1 &&
-        echo "foobar.stdouterr" > expected &&
-        echo "foobar.stdouterr" >> expected &&
-        test_cmp expected output
+	$rexec ${TEST_SUBPROCESS_DIR}/test_echo -O -E foobar.stdouterr > output 2>&1 &&
+	echo "foobar.stdouterr" > expected &&
+	echo "foobar.stdouterr" >> expected &&
+	test_cmp expected output
 '
 
 test_expect_success 'basic rexec invalid rank' '
-	! ${FLUX_BUILD_DIR}/t/rexec/rexec -r 32 /bin/true > output 2>&1 &&
-        grep -q "No route to host" output
+	test_must_fail $rexec -r 32 true > output 2>&1 &&
+	grep -q "$(strerror_symbol EHOSTUNREACH)" output
 '
 
 test_expect_success 'basic rexec fail exec()' '
-	! ${FLUX_BUILD_DIR}/t/rexec/rexec / > output 2>&1 &&
-        grep -q "Permission denied" output
+	! $rexec / > output 2>&1 &&
+	grep -q "Permission denied" output
 '
 
 test_expect_success 'basic rexec fail exec() EACCES' '
-	! ${FLUX_BUILD_DIR}/t/rexec/rexec / > output 2>&1 &&
-        grep -q "Permission denied" output
+	! $rexec / > output 2>&1 &&
+	grep -q "Permission denied" output
 '
 
 test_expect_success 'basic rexec fail exec() ENOENT' '
-	! ${FLUX_BUILD_DIR}/t/rexec/rexec /usr/bin/foobarbaz > output 2>&1 &&
-        grep -q "No such file or directory" output
+	! $rexec /usr/bin/foobarbaz > output 2>&1 &&
+	grep -q "No such file or directory" output
 '
 
-test_expect_success 'basic rexec propogates exit code()' '
-        test_expect_code 0 ${FLUX_BUILD_DIR}/t/rexec/rexec /bin/true &&
-        test_expect_code 1 ${FLUX_BUILD_DIR}/t/rexec/rexec /bin/false &&
-        test_expect_code 2 ${FLUX_BUILD_DIR}/t/rexec/rexec sh -c "exit 2" &&
-        test_expect_code 3 ${FLUX_BUILD_DIR}/t/rexec/rexec sh -c "exit 3"
+test_expect_success 'basic rexec propagates exit code()' '
+	test_expect_code 0 $rexec true &&
+	test_expect_code 1 $rexec false &&
+	test_expect_code 2 $rexec sh -c "exit 2" &&
+	test_expect_code 3 $rexec sh -c "exit 3"
 '
 
 test_expect_success 'basic rexec functionality (check state changes)' '
-	${FLUX_BUILD_DIR}/t/rexec/rexec -s /bin/true > output &&
-        echo "Running" > expected &&
-        echo "Exited" >> expected &&
-        test_cmp expected output
+	$rexec -s true > output &&
+	echo "Running" > expected &&
+	echo "Exited" >> expected &&
+	test_cmp expected output
 '
 
 test_expect_success 'basic rexec fail exec() (check state changes)' '
-	! ${FLUX_BUILD_DIR}/t/rexec/rexec -s / > output &&
-        echo "Exec Failed" > expected &&
-        test_cmp expected output
+	! $rexec -s / > output &&
+	echo "Failed" > expected &&
+	test_cmp expected output
 '
 
 test_expect_success 'basic rexec stdin' '
-	echo -n "hello" | ${FLUX_BUILD_DIR}/t/rexec/rexec -i stdin ${TEST_SUBPROCESS_DIR}/test_echo -O -E > output 2>&1 &&
-        echo "hello" > expected &&
-        echo "hello" >> expected &&
-        test_cmp expected output
+	echo -n "hello" | $rexec -i stdin ${TEST_SUBPROCESS_DIR}/test_echo -O -E > output 2>&1 &&
+	echo "hello" > expected &&
+	echo "hello" >> expected &&
+	test_cmp expected output
 '
 
 test_expect_success 'basic rexec stdin / stdout multiple lines' '
-	/bin/echo -en "foo\nbar\nbaz\n" | ${FLUX_BUILD_DIR}/t/rexec/rexec -i stdin ${TEST_SUBPROCESS_DIR}/test_echo -O -n > output 2>&1 &&
-        echo "foo" > expected &&
-        echo "bar" >> expected &&
-        echo "baz" >> expected &&
-        test_cmp expected output
+	/bin/echo -en "foo\nbar\nbaz\n" | $rexec -i stdin ${TEST_SUBPROCESS_DIR}/test_echo -O -n > output 2>&1 &&
+	echo "foo" > expected &&
+	echo "bar" >> expected &&
+	echo "baz" >> expected &&
+	test_cmp expected output
 '
 
 test_expect_success 'basic rexec stdin / stdout long lines' '
-        dd if=/dev/urandom bs=4096 count=1 | base64 --wrap=0 >expected &&
-        ${FLUX_BUILD_DIR}/t/rexec/rexec cat expected > output &&
-        test_cmp expected output
+	dd if=/dev/urandom bs=4096 count=1 | base64 --wrap=0 >expected &&
+	$rexec cat expected > output &&
+	test_cmp expected output
 '
 
 # pipe in /dev/null, we don't care about stdin for this test
 test_expect_success 'rexec check channel FD created' '
-	${FLUX_BUILD_DIR}/t/rexec/rexec -i TEST_CHANNEL /usr/bin/env < /dev/null > output 2>&1 &&
-        grep "TEST_CHANNEL=" output
+	$rexec -i TEST_CHANNEL /usr/bin/env < /dev/null > output 2>&1 &&
+	grep "TEST_CHANNEL=" output
 '
 
 # rexec does not close TEST_CHANNEL, so we tell test_echo max
 # bytes we're feeding in
 test_expect_success 'rexec channel input' '
-	echo -n "foobar" | ${FLUX_BUILD_DIR}/t/rexec/rexec -i TEST_CHANNEL ${TEST_SUBPROCESS_DIR}/test_echo -c TEST_CHANNEL -P -O -b 6 > output 2>&1 &&
-        echo "stdout:foobar" > expected &&
-        test_cmp expected output
+	echo -n "foobar" | $rexec -i TEST_CHANNEL ${TEST_SUBPROCESS_DIR}/test_echo -c TEST_CHANNEL -P -O -b 6 > output 2>&1 &&
+	echo "stdout:foobar" > expected &&
+	test_cmp expected output
 '
 
 # rexec does not close TEST_CHANNEL, so we tell test_echo max
 # bytes we're feeding in
 test_expect_success 'rexec channel input and output' '
-	echo -n "foobaz" | ${FLUX_BUILD_DIR}/t/rexec/rexec -i TEST_CHANNEL ${TEST_SUBPROCESS_DIR}/test_echo -c TEST_CHANNEL -P -C -b 6 > output 2>&1 &&
-        echo "TEST_CHANNEL:foobaz" > expected &&
-        test_cmp expected output
+	echo -n "foobaz" | $rexec -i TEST_CHANNEL ${TEST_SUBPROCESS_DIR}/test_echo -c TEST_CHANNEL -P -C -b 6 > output 2>&1 &&
+	echo "TEST_CHANNEL:foobaz" > expected &&
+	test_cmp expected output
 '
 
 # rexec does not close TEST_CHANNEL, so we tell test_echo max
 # bytes we're feeding in
 test_expect_success 'rexec channel input and output multiple lines' '
-	/bin/echo -en "foo\nbar\nbaz\n" | ${FLUX_BUILD_DIR}/t/rexec/rexec -i TEST_CHANNEL ${TEST_SUBPROCESS_DIR}/test_echo -c TEST_CHANNEL -C -n -b 6 > output 2>&1 &&
-        echo "foo" > expected &&
-        echo "bar" >> expected &&
-        echo "baz" >> expected &&
-        test_cmp expected output
+	/bin/echo -en "foo\nbar\nbaz\n" | $rexec -i TEST_CHANNEL ${TEST_SUBPROCESS_DIR}/test_echo -c TEST_CHANNEL -C -n -b 6 > output 2>&1 &&
+	echo "foo" > expected &&
+	echo "bar" >> expected &&
+	echo "baz" >> expected &&
+	test_cmp expected output
 '
 
 test_expect_success 'rexec kill' '
 	test_expect_code 143 \
-	    ${FLUX_BUILD_DIR}/t/rexec/rexec -k /bin/sleep 10 > output 2>&1 &&
-        grep "subprocess terminated by signal 15" output
+	    $rexec -k /bin/sleep 10 > output 2>&1 &&
+	grep "subprocess terminated by signal 15" output
 '
 
 test_expect_success 'rexec kill group' '
 	test_expect_code 143 \
-	    ${FLUX_BUILD_DIR}/t/rexec/rexec \
-                -k ${TEST_SUBPROCESS_DIR}/test_fork_sleep 30 > output 2>&1 &&
-        grep "subprocess terminated by signal 15" output
+	    $rexec \
+		-k ${TEST_SUBPROCESS_DIR}/test_fork_sleep 30 > output 2>&1 &&
+	grep "subprocess terminated by signal 15" output
 '
 
 test_expect_success 'rexec kill process not yet running' '
 	test_expect_code 143 \
-	    ${FLUX_BUILD_DIR}/t/rexec/rexec -K /bin/sleep 10 > K.out 2>&1 &&
-        grep "subprocess terminated by signal 15" K.out
+	    $rexec -K /bin/sleep 10 > K.out 2>&1 &&
+	grep "subprocess terminated by signal 15" K.out
 '
 
 test_expect_success 'rexec kill with already pending signal gets error' '
 	test_expect_code 143 \
-	    ${FLUX_BUILD_DIR}/t/rexec/rexec -K -K /bin/sleep 10 > KK.out 2>&1 &&
-        grep "subprocess terminated by signal 15" KK.out &&
+	    $rexec -K -K /bin/sleep 10 > KK.out 2>&1 &&
+	grep "subprocess terminated by signal 15" KK.out &&
 	grep "rexec: flux_subprocess_kill: Invalid argument" KK.out
 '
 
+wait_rexec_process_count () {
+	expected=$1
+	rank=$2
+	i=0
+	$rexec_script ps --rank $rank > output &&
+	count=`cat output | wc -l` &&
+	while [ "${count}" != "${expected}" ] && [ $i -lt 30 ]
+	do
+	    sleep 1
+	    $rexec_script ps --rank $rank > output &&
+	    count=`cat output | wc -l` &&
+	    i=$((i + 1))
+	done
+	if [ "$i" -eq "30" ]
+	then
+	    return 1
+	fi
+	return 0;
+}
+
 test_expect_success NO_CHAIN_LINT 'rexec ps works' '
-        ${FLUX_BUILD_DIR}/t/rexec/rexec -r 1 sleep 100 &
-        pid1=$!
-        ${FLUX_BUILD_DIR}/t/rexec/rexec -r 1 sleep 100 &
-        pid2=$!
-	sleep 1 &&
-        ${FLUX_BUILD_DIR}/t/rexec/rexec_ps -r 1 > output &&
-        count=`cat output | wc -l` &&
+	$rexec -r 1 sleep 100 &
+	pid1=$!
+	$rexec -r 1 sleep 100 &
+	pid2=$!
+	wait_rexec_process_count 2 1 &&
 	kill -TERM $pid1 &&
-	kill -TERM $pid2 &&
-        test "$count" = "2"
+	kill -TERM $pid2
 '
 
 test_expect_success NO_CHAIN_LINT 'disconnect terminates all running processes' '
-        ${FLUX_BUILD_DIR}/t/rexec/rexec -r 1 sleep 100 &
-        pid1=$!
-        ${FLUX_BUILD_DIR}/t/rexec/rexec -r 1 sleep 100 &
-        pid2=$!
-	sleep 1 &&
-        ${FLUX_BUILD_DIR}/t/rexec/rexec_ps -r 1 > output &&
-        count=`cat output | wc -l` &&
-        test "$count" = "2" &&
-	sleep 1 &&
+	$rexec -r 1 sleep 100 &
+	pid1=$!
+	$rexec -r 1 sleep 100 &
+	pid2=$!
+	wait_rexec_process_count 2 1 &&
 	kill -TERM $pid1 &&
 	kill -TERM $pid2 &&
-        ${FLUX_BUILD_DIR}/t/rexec/rexec_ps -r 1 > output &&
-        count=`cat output | wc -l` &&
-        test "$count" = "0"
+	wait_rexec_process_count 0 1
 '
 
 test_expect_success 'rexec line buffering works (default)' '
-        ${FLUX_BUILD_DIR}/t/rexec/rexec_count_stdout -r 1 ${TEST_SUBPROCESS_DIR}/test_multi_echo -O -c 2200 hi > linebuffer1.out &&
-        grep "final stdout callback count: 2" linebuffer1.out
+	${FLUX_BUILD_DIR}/t/rexec/rexec_count_stdout -r 1 ${TEST_SUBPROCESS_DIR}/test_multi_echo -O -c 2200 hi > linebuffer1.out &&
+	grep "final stdout callback count: 2" linebuffer1.out
 '
 
 test_expect_success 'rexec line buffering works (set to true)' '
-        ${FLUX_BUILD_DIR}/t/rexec/rexec_count_stdout -r 1 -l true ${TEST_SUBPROCESS_DIR}/test_multi_echo -O -c 2200 hi > linebuffer2.out &&
-        grep "final stdout callback count: 2" linebuffer2.out
+	${FLUX_BUILD_DIR}/t/rexec/rexec_count_stdout -r 1 -l true ${TEST_SUBPROCESS_DIR}/test_multi_echo -O -c 2200 hi > linebuffer2.out &&
+	grep "final stdout callback count: 2" linebuffer2.out
 '
 
 # test is technically racy, but with 2200 hi outputs, probability is
 # extremely low all data is buffered in one shot.
 test_expect_success 'rexec line buffering can be disabled' '
-        ${FLUX_BUILD_DIR}/t/rexec/rexec_count_stdout -r 1 -l false ${TEST_SUBPROCESS_DIR}/test_multi_echo -O -c 2200 hi > linebuffer3.out &&
-        count=$(grep "final stdout callback count:" linebuffer3.out | awk "{print \$5}") &&
-        test "$count" -gt 2
+	${FLUX_BUILD_DIR}/t/rexec/rexec_count_stdout -r 1 -l false ${TEST_SUBPROCESS_DIR}/test_multi_echo -O -c 2200 hi > linebuffer3.out &&
+	count=$(grep "final stdout callback count:" linebuffer3.out | awk "{print \$5}") &&
+	test "$count" -gt 2
 '
 
 # the last line of output is "bar" without a newline.  "EOF" is output
@@ -226,9 +239,61 @@ test_expect_success 'rexec line buffering can be disabled' '
 # should see the concatenation "barEOF" at the end of the output.
 test_expect_success 'rexec read_getline call works on remote streams' '
 	/bin/echo -en "foo\nbar" | ${FLUX_BUILD_DIR}/t/rexec/rexec_getline -i stdin ${TEST_SUBPROCESS_DIR}/test_echo -O -n > output 2>&1 &&
-        echo "foo" > expected &&
-        echo "barEOF" >> expected &&
-        test_cmp expected output
+	echo "foo" > expected &&
+	echo "barEOF" >> expected &&
+	test_cmp expected output
+'
+test_expect_success 'configure access.allow-guest-user = true' '
+	flux config load <<-EOT
+	access.allow-guest-user = true
+	EOT
+'
+test_expect_success 'get URI for rank 1 and check that it works' '
+	$rexec -r 1 flux getattr local-uri >uri1 &&
+	test $(FLUX_URI=$(cat uri1) flux getattr rank) -eq 1
+'
+test_expect_success 'rexec from rank 0 to rank 1 works' '
+	$rexec -r 0 true
+'
+test_expect_success 'rexec from rank 1 to rank 1 works' '
+	(FLUX_URI=$(cat uri1) $rexec -r 1 true)
+'
+test_expect_success 'rexec from rank 1 to rank 0 is restricted' '
+	(FLUX_URI=$(cat uri1) test_must_fail $rexec -r 0 true)
+'
+
+test_expect_success NO_CHAIN_LINT 'ps, kill work locally on rank 0' '
+	$rexec sleep 3600 &
+	wait_rexec_process_count 1 0 &&
+	pid=$($rexec_script ps |cut -f1) &&
+	$rexec_script kill 15 $pid &&
+	wait_rexec_process_count 0 0
+'
+
+test_expect_success NO_CHAIN_LINT 'ps, kill fail remotely on rank 0' '
+	$rexec sleep 3600 &
+	wait_rexec_process_count 1 0 &&
+	pid=$($rexec_script ps | cut -f1) &&
+	(FLUX_URI=$(cat uri1) test_must_fail \
+		$rexec_script ps --rank 0) &&
+	(FLUX_URI=$(cat uri1) test_must_fail \
+		$rexec_script kill --rank 0 15 $pid) &&
+	$rexec_script kill 15 $pid &&
+	wait_rexec_process_count 0 0
+'
+
+test_expect_success 'configure access.allow-guest-user = false' '
+	flux config load <<-EOT
+	access.allow-guest-user = false
+	EOT
+'
+test_expect_success 'rexec from rank 1 to rank 0 works' '
+	(FLUX_URI=$(cat uri1) $rexec -r 0 true)
+'
+
+test_expect_success NO_CHAIN_LINT 'kill fails with ESRCH when pid is unknown' '
+	test_must_fail $rexec_script kill 15 12345678 2>kill.err &&
+	grep "No such process" kill.err
 '
 
 test_done
diff --git a/t/t0007-ping.t b/t/t0007-ping.t
index a44f3e87bb8d..475c7b5cf0db 100755
--- a/t/t0007-ping.t
+++ b/t/t0007-ping.t
@@ -13,95 +13,103 @@ invalid_rank() {
 }
 
 test_expect_success 'ping: 10K 1K byte echo requests' '
-	run_timeout 10 flux ping --pad 1024 --count 10240 --interval 0 0
+	run_timeout 25 flux ping --pad 1024 --count 10240 --interval 0 0 \
+	    >/dev/null
 '
 
 test_expect_success 'ping: 1K 10K byte echo requests' '
-	run_timeout 5 flux ping --pad 10240 --count 1024 --interval 0 0
+	run_timeout 15 flux ping --pad 10K --count 1024 --interval 0 0 \
+	    >/dev/null
 '
 
 test_expect_success 'ping: 100 100K byte echo requests' '
-	run_timeout 5 flux ping --pad 102400 --count 100 --interval 0 0
+	run_timeout 15 flux ping --pad 100K --count 100 --interval 0 0 \
+	    >/dev/null
 '
 
 test_expect_success 'ping: 10 1M byte echo requests' '
-	run_timeout 5 flux ping --pad 1048576 --count 10 --interval 0 0
+	run_timeout 15 flux ping --pad 1M --count 10 --interval 0 0
 '
 
 test_expect_success 'ping: 10 1M byte echo requests (batched)' '
-	run_timeout 5 flux ping --pad 1048576 --count 10 --batch --interval 0 0
+	run_timeout 15 flux ping --pad 1M --count 10 --batch --interval 0 0
 '
 
 test_expect_success 'ping: 1K 10K byte echo requests (batched)' '
-	run_timeout 20 flux ping --pad 10240 --count 1024 --batch --interval 0 0
+	run_timeout 20 flux ping --pad 10K --count 1024 --batch --interval 0 0 \
+	    >/dev/null
 '
 
 test_expect_success 'ping --rank 1 works' '
-	run_timeout 5 flux ping --rank 1 --count 10 --interval 0 cmb
+	run_timeout 15 flux ping --rank 1 --count 10 --interval 0 broker
 '
 
 test_expect_success 'ping 1 works' '
-	run_timeout 5 flux ping --count 10 --interval 0 1
+	run_timeout 15 flux ping --count 10 --interval 0 1
 '
 
-test_expect_success 'ping 1!cmb works' '
-	run_timeout 5 flux ping --count 10 --interval 0 "1!cmb"
+test_expect_success 'ping 1!broker works' '
+	run_timeout 15 flux ping --count 10 --interval 0 "1!broker"
 '
 
 test_expect_success 'ping fails on invalid rank (specified as target)' '
-	! run_timeout 5 flux ping --count 1 $(invalid_rank) 2>stderr &&
-	grep -q "No route to host" stderr
+	test_must_fail run_timeout 15 flux ping --count 1 $(invalid_rank) 2>stderr &&
+	grep "$(strerror_symbol EHOSTUNREACH)" stderr
 '
 
 test_expect_success 'ping fails on invalid rank (specified in option)' '
-	! run_timeout 5 flux ping --count 1 --rank $(invalid_rank) cmb 2>stderr &&
-	grep -q "No route to host" stderr
+	test_must_fail run_timeout 15 flux ping --count 1 --rank $(invalid_rank) broker 2>stderr &&
+	grep "$(strerror_symbol EHOSTUNREACH)" stderr
 '
 
 test_expect_success 'ping fails on invalid target' '
-	! run_timeout 5 flux ping --count 1 --rank 0 nosuchtarget 2>stderr &&
+	test_must_fail run_timeout 15 flux ping --count 1 --rank 0 nosuchtarget 2>stderr &&
 	grep -q "Function not implemented" stderr
 '
 
 test_expect_success 'ping output format for "any" rank is correct (default)' '
-	run_timeout 5 flux ping --count 1 cmb 1>stdout &&
-        grep -q "^cmb.ping" stdout &&
+	run_timeout 15 flux ping --count 1 broker 1>stdout &&
+        head -n 1 stdout | grep -q "any!broker" &&
+        grep -q "^0!broker.ping" stdout &&
         grep -q -E "time=[0-9]+\.[0-9]+ ms" stdout
 '
 
 test_expect_success 'ping output format for "any" rank is correct (format 1)' '
-	run_timeout 5 flux ping --count 1 --rank any cmb 1>stdout &&
-        grep -q "^cmb.ping" stdout &&
+	run_timeout 15 flux ping --count 1 --rank any broker 1>stdout &&
+        head -n 1 stdout | grep -q "any!broker" &&
+        grep -q "^0!broker.ping" stdout &&
         grep -q -E "time=[0-9]+\.[0-9]+ ms" stdout
 '
 
 test_expect_success 'ping output format for "any" rank is correct (format 2)' '
-	run_timeout 5 flux ping --count 1 any!cmb 1>stdout &&
-        grep -q "^cmb.ping" stdout &&
+	run_timeout 15 flux ping --count 1 any!broker 1>stdout &&
+        head -n 1 stdout | grep -q "any!broker" &&
+        grep -q "^0!broker.ping" stdout &&
         grep -q -E "time=[0-9]+\.[0-9]+ ms" stdout
 '
 
 test_expect_success 'ping output format for "any" rank is correct (format 3)' '
-	run_timeout 5 flux ping --count 1 any 1>stdout &&
-        grep -q "^cmb.ping" stdout &&
+	run_timeout 15 flux ping --count 1 any 1>stdout &&
+        head -n 1 stdout | grep -q "any!broker" &&
+        grep -q "^0!broker.ping" stdout &&
         grep -q -E "time=[0-9]+\.[0-9]+ ms" stdout
 '
 
 test_expect_success 'ping output format for specific rank is correct (format 1)' '
-	run_timeout 5 flux ping --count 1 --rank 0 cmb 1>stdout &&
-        grep -q "^0!cmb.ping" stdout &&
+	run_timeout 15 flux ping --count 1 --rank 0 broker 1>stdout &&
+        grep -q "^0!broker.ping" stdout &&
         grep -q -E "time=[0-9]+\.[0-9]+ ms" stdout
 '
 
 test_expect_success 'ping output format for specific rank is correct (format 2)' '
-	run_timeout 5 flux ping --count 1 0!cmb 1>stdout &&
-        grep -q "^0!cmb.ping" stdout &&
+	run_timeout 15 flux ping --count 1 0!broker 1>stdout &&
+        grep -q "^0!broker.ping" stdout &&
         grep -q -E "time=[0-9]+\.[0-9]+ ms" stdout
 '
 
 test_expect_success 'ping output format for specific rank is correct (format 3)' '
-	run_timeout 5 flux ping --count 1 0 1>stdout &&
-        grep -q "^0!cmb.ping" stdout &&
+	run_timeout 15 flux ping --count 1 0 1>stdout &&
+        grep -q "^0!broker.ping" stdout &&
         grep -q -E "time=[0-9]+\.[0-9]+ ms" stdout
 '
 
@@ -110,26 +118,47 @@ test_expect_success 'ping output format for specific rank is correct (format 3)'
 # rank 1 should work
 
 test_expect_success 'ping with "upstream" fails on rank 0' '
-        ! run_timeout 5 flux exec -n --rank 0 flux ping --count 1 --rank upstream cmb 2>stderr &&
-	grep -q "No route to host" stderr
+        test_must_fail run_timeout 15 flux exec -n --rank 0 flux ping --count 1 --rank upstream broker 2>stderr &&
+	grep "$(strerror_symbol EHOSTUNREACH)" stderr
 '
 
 test_expect_success 'ping with "upstream" works (format 1)' '
-        run_timeout 5 flux exec -n --rank 1 flux ping --count 1 --rank upstream cmb 1>stdout &&
-        grep -q "^upstream!cmb.ping" stdout &&
+        run_timeout 15 flux exec -n --rank 1 flux ping --count 1 --rank upstream broker 1>stdout &&
+        grep -q "^0!broker.ping" stdout &&
         grep -q -E "time=[0-9]+\.[0-9]+ ms" stdout
 '
 
 test_expect_success 'ping with "upstream" works (format 2)' '
-        run_timeout 5 flux exec -n --rank 1 flux ping --count 1 upstream!cmb 1>stdout &&
-        grep -q "^upstream!cmb.ping" stdout &&
+        run_timeout 15 flux exec -n --rank 1 flux ping --count 1 upstream!broker 1>stdout &&
+        grep -q "^0!broker.ping" stdout &&
         grep -q -E "time=[0-9]+\.[0-9]+ ms" stdout
 '
 
 test_expect_success 'ping with "upstream" works (format 3)' '
-        run_timeout 5 flux exec -n --rank 1 flux ping --count 1 upstream 1>stdout &&
-        grep -q "^upstream!cmb.ping" stdout &&
+        run_timeout 15 flux exec -n --rank 1 flux ping --count 1 upstream 1>stdout &&
+        grep -q "^0!broker.ping" stdout &&
         grep -q -E "time=[0-9]+\.[0-9]+ ms" stdout
 '
 
+test_expect_success 'ping help output works' '
+        flux ping --help 2> help.err &&
+        grep "Usage: flux-ping \[OPTIONS\] TARGET" help.err
+'
+
+test_expect_success 'ping works with hostname' '
+        hostname=$(hostname) &&
+        flux ping --count=1 ${hostname} 1>stdout &&
+        head -n 1 stdout | grep -q "${hostname}!broker" &&
+        head -n 1 stdout | grep -q "(rank 0)"
+'
+test_expect_success 'ping works with hostname!service' '
+        hostname=$(hostname) &&
+        flux ping --count=1 "${hostname}!broker" 1>stdout &&
+        head -n 1 stdout | grep -q "${hostname}!broker" &&
+        head -n 1 stdout | grep -q "(rank 0)"
+'
+test_expect_success 'ping fails with unknown hostname' '
+        test_must_fail flux ping --count=1 "notmyhost!broker"
+'
+
 test_done
diff --git a/t/t0008-attr.t b/t/t0008-attr.t
index 0d8f76895873..7fa16bf4d0a9 100755
--- a/t/t0008-attr.t
+++ b/t/t0008-attr.t
@@ -17,9 +17,6 @@ test_expect_success 'flux getattr rank works' '
 test_expect_success 'flux setattr rank fails (immutable)' '
 	! flux setattr rank 42
 '
-test_expect_success 'flux setattr --expunge rank fails (immutable)' '
-	test_must_fail flux setattr --expunge rank
-'
 test_expect_success 'flux getattr attrtest.nonexist fails' '
 	! flux getattr nonexist
 '
@@ -28,22 +25,15 @@ test_expect_success 'flux setattr works' '
 	ATTR_VAL=`flux getattr attrtest.foo` &&
 	test "${ATTR_VAL}" = "bar"
 '
-test_expect_success 'flux setattr -e works' '
-	flux setattr -e attrtest.foo &&
-	! flux getattr attrtest.foo
-'
 test_expect_success 'flux lsattr works' '
 	flux lsattr >lsattr_out &&
 	grep -q rank lsattr_out &&
-	! grep -q attrtest.foo lsattr_out
+	grep -q attrtest.foo lsattr_out
 '
 test_expect_success 'flux lsattr -v works' '
 	ATTR_VAL=$(flux lsattr -v | awk "/^rank / { print \$2 }") &&
 	test "${ATTR_VAL}" -eq 0
 '
-test_expect_success 'flux setattr --expunge missing optarg fails' '
-	test_must_fail flux setattr --expunge
-'
 test_expect_success 'flux lsattr with extra argument fails' '
 	test_must_fail flux lsattr badarg
 '
@@ -56,8 +46,5 @@ test_expect_success 'get request with empty payload fails with EPROTO(71)' '
 test_expect_success 'set request with empty payload fails with EPROTO(71)' '
 	${RPC} attr.set 71 </dev/null
 '
-test_expect_success 'rm request with empty payload fails with EPROTO(71)' '
-	${RPC} attr.rm 71 </dev/null
-'
 
 test_done
diff --git a/t/t0009-dmesg.t b/t/t0009-dmesg.t
index c4f240d24d89..1484a19699c8 100755
--- a/t/t0009-dmesg.t
+++ b/t/t0009-dmesg.t
@@ -5,19 +5,20 @@ test_description='Test broker log ring buffer'
 . `dirname $0`/sharness.sh
 
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
 
 test_under_flux 4 minimal
 
-test_expect_success 'flux getattr log-count counts log messages' '
-	OLD_VAL=`flux getattr log-count` &&
+test_expect_success 'module stats --parse count log counts log messages' '
+	OLD_VAL=$(flux module stats --parse count log) &&
 	flux logger hello &&
-	NEW_VAL=`flux getattr log-count` &&
+	NEW_VAL=$(flux module stats --parse count log) &&
 	test "${OLD_VAL}" -lt "${NEW_VAL}"
 '
-test_expect_success 'flux getattr log-ring-used counts log messages' '
-	OLD_VAL=`flux getattr log-ring-used` &&
+test_expect_success 'module stats --parse ring-used log counts log messages' '
+	OLD_VAL=$(flux module stats --parse ring-used log) &&
 	flux logger hello &&
-	NEW_VAL=`flux getattr log-ring-used` &&
+	NEW_VAL=$(flux module stats --parse ring-used log) &&
 	test "${OLD_VAL}" -lt "${NEW_VAL}"
 '
 test_expect_success 'flux dmesg -C clears ring buffer' '
@@ -50,6 +51,24 @@ test_expect_success 'flux dmesg -c prints and clears' '
 	flux dmesg -c | grep -q hello_dmesg_pc &&
 	! flux dmesg | grep -q hello_dmesg_pc
 '
+test_expect_success NO_CHAIN_LINT 'flux dmesg -f works' '
+	flux logger hello_last &&
+	flux dmesg -f > dmesg.out &
+	pid=$! &&
+	$waitfile -t 20 -p hello_last dmesg.out &&
+	flux logger hello_follow &&
+	$waitfile -t 20 -p hello_follow dmesg.out &&
+	kill $pid
+'
+test_expect_success NO_CHAIN_LINT 'flux dmesg -f --new works' '
+	flux logger hello_old &&
+	flux dmesg -f --new > dmesg2.out &
+	pid=$! &&
+	for i in $(seq 1 10); do flux logger hello_new; done &&
+	$waitfile -t 20 -p hello_new dmesg2.out &&
+	test_must_fail grep hello_old dmesg2.out &&
+	kill $pid
+'
 test_expect_success 'ring buffer wraps over old entries' '
 	OLD_RINGSIZE=`flux getattr log-ring-size` &&
 	flux setattr log-ring-size 2 &&
@@ -89,11 +108,67 @@ test_expect_success 'logged non-ascii characters handled ok' '
 	/bin/echo -n -e "\xFF\xFE\x82\x00" | flux logger &&
 	flux dmesg
 '
+test_expect_success 'logged non-ascii printable characters are unmodified' '
+	flux logger ƒ Φ Ψ Ω Ö &&
+	flux dmesg | tail -1 > dmesg.utf-8 &&
+	test_debug "cat dmesg.utf-8" &&
+	grep "ƒ Φ Ψ Ω Ö" dmesg.utf-8
+'
 test_expect_success 'dmesg request with empty payload fails with EPROTO(71)' '
 	${RPC} log.dmesg 71 </dev/null
 '
-test_expect_success 'clear request with empty payload fails with EPROTO(71)' '
-	${RPC} log.clear 71 </dev/null
+# Note: TZ=UTC is set by default for sharness
+test_expect_success 'dmesg with TZ=UTC uses "Z" as offset' '
+	flux dmesg \
+	    | grep -E \
+	    "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z"
+'
+
+test "$(TZ=America/Los_Angeles date +%z)" != "+0000" &&
+  test "$(TZ=America/Los_Angeles date +%Z)" != "UTC" &&
+  test_set_prereq HAVE_TZ
+test_expect_success HAVE_TZ 'dmesg with TZ=America/Los_Angeles uses tz offset' '
+	TZ=America/Los_Angeles flux dmesg >dmesgLA.out &&
+	test_debug "cat dmesgLA.out" &&
+	grep -E "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+-[0-9]{2}:[0-9]{2}" dmesgLA.out
+'
+test_expect_success 'dmesg -H, --human works' '
+	#
+	#  Note: --human option should format first timestamp of dmesg output
+	#   as [MmmDD HH:MM] and second line should be an offset thereof
+	#   e.g. [  +0.NNNNNN]. The following regexes attempt to verify
+	#   that --human produced this pattern.
+	#
+	flux dmesg --human | sed -n 1p \
+	    | grep "^\[[A-Z][a-z][a-z][0-3][0-9] [0-9][0-9]:[0-9][0-9]\]" &&
+	flux dmesg --human | sed -n 2p \
+	    | grep "^\[ *+[0-9]*\.[0-9]*\]"
+'
+test_expect_success 'dmesg -H, --human --delta works' '
+	flux dmesg --human --delta | sed -n 1p \
+	    | grep "^\[[A-Z][a-z][a-z][0-3][0-9] [0-9][0-9]:[0-9][0-9]\]" &&
+	flux dmesg --human --delta | sed -n 2p \
+	    | grep "^\[ *+[0-9]*\.[0-9]*\]"
+'
+test_expect_success 'dmesg --delta without --human fails' '
+	test_must_fail flux dmesg --delta
+'
+
+for opt in "-L" "-Lalways" "--color" "--color=always"; do
+	test_expect_success "dmesg -H, --human $opt works" '
+		flux dmesg --human $opt | sed -n 1p | grep "^" &&
+		flux dmesg --human $opt | sed -n 2p | grep "^"
+	'
+	test_expect_success "dmesg colorizes lines by severity" '
+		for s in emerg alert crit err warning notice debug; do
+		    flux logger --severity=$s severity=$s &&
+		    flux dmesg --human $opt | grep "[^ ]*severity=$s"
+		done
+	'
+done
+
+test_expect_success 'dmesg with invalid --color option fails' '
+	test_must_fail flux dmesg --color=foo
 '
 
 test_done
diff --git a/t/t0010-generic-utils.t b/t/t0010-generic-utils.t
index 33ee6e4abd82..ba366fcb56cf 100755
--- a/t/t0010-generic-utils.t
+++ b/t/t0010-generic-utils.t
@@ -24,8 +24,8 @@ test_expect_success 'event: can publish' '
 	test_cmp expected_event_pub output_event_pub
 '
 test_expect_success 'event: can subscribe' '
-	flux event sub --count=1 hb >output_event_sub &&
-	grep "^hb" output_event_sub
+	flux event sub --count=1 heartbeat.pulse>output_event_sub &&
+	grep "^heartbeat.pulse" output_event_sub
 '
 
 test_expect_success 'version: reports expected values under an instance' '
@@ -79,18 +79,4 @@ test_expect_success 'heaptrace.dump request with empty payload fails with EPROTO
 	${RPC} heaptrace.dump 71 </dev/null
 '
 
-emit_toml_config() {
-	cat <<-EOT
-	flux_uri = "$1"
-	EOT
-}
-
-# Check that config file can affect default URI
-test_expect_success 'utils calling flux_open() can get URI from config file' '
-	echo ${SIZE} >size.exp &&
-	emit_toml_config $FLUX_URI >flux.toml &&
-	(unset FLUX_URI && FLUX_CONF_DIR=$(pwd) flux getattr size >size.out) &&
-	test_cmp size.exp size.out
-'
-
 test_done
diff --git a/t/t0011-content-cache.t b/t/t0011-content-cache.t
index 63b24ac5911c..708b36f7da10 100755
--- a/t/t0011-content-cache.t
+++ b/t/t0011-content-cache.t
@@ -11,10 +11,46 @@ echo "# $0: flux session size will be ${SIZE}"
 
 BLOBREF=${FLUX_BUILD_DIR}/t/kvs/blobref
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
+SPAMUTIL=${FLUX_BUILD_DIR}/t/kvs/content-spam
+MAXBLOB=1048576
+
+test_expect_success 'load content module' '
+	flux exec flux module load content blob-size-limit=$MAXBLOB
+'
+
+test_expect_success 'content flush fails with ENOSYS with no backing store' '
+	test_must_fail flux content flush 2> flush.err &&
+	grep "Function not implemented" flush.err
+'
 
-MAXBLOB=`flux getattr content.blob-size-limit`
 HASHFUN=`flux getattr content.hash`
 
+register_backing() {
+	jq -j -c -n  "{name:\"$1\"}" | $RPC content.register-backing
+}
+unregister_backing() {
+	jq -j -c -n  "{name:\"$1\"}" | $RPC content.unregister-backing
+}
+
+test_expect_success 'register-backing name=foo works' '
+	register_backing foo
+'
+test_expect_success 'register-backing name=bar fails' '
+	test_must_fail register_backing bar 2>bar.err &&
+	grep "already active" bar.err
+'
+test_expect_success 'unregister-backing name=foo works' '
+	unregister_backing foo
+'
+test_expect_success 'register-backing name=bar fails' '
+	test_must_fail register_backing bar 2>foo.err &&
+	grep "cannot be changed" foo.err
+'
+test_expect_success 'unregister-backing name=foo fails' '
+	test_must_fail unregister_backing foo 2>foo2.err &&
+	grep "is not active" foo2.err
+'
+
 test_expect_success 'store 100 blobs on rank 0' '
         for i in `seq 0 99`; do echo test$i | \
             flux content store >/dev/null; done &&
@@ -36,7 +72,7 @@ test_expect_success 'store test blobs on rank 0' '
 	flux content store <1m.0.store >1m.0.hash
 '
 
-test_expect_success LONGTEST "cannot store blob that exceeds max size of $MAXBLOB" '
+test_expect_success "cannot store blob that exceeds max size of $MAXBLOB" '
 	dd if=/dev/zero count=$(($MAXBLOB/4096+1)) bs=4096 \
 			skip=$(($MAXBLOB/4096)) >toobig 2>/dev/null &&
 	test_must_fail flux content store <toobig
@@ -124,7 +160,7 @@ test_expect_success 'negative entries are not cached' '
 
 # Store the same content on all ranks
 # Retrieve it from one rank
-# (Really we want to test whther stores were squashed, fill in later)
+# (Really we want to test whether stores were squashed, fill in later)
 
 test_expect_success 'store on all ranks can be retrieved from rank 0' '
 	flux exec -n sh -c "echo foof | flux content store" >/dev/null &&
@@ -147,12 +183,12 @@ test_expect_success 'rank 0 cache is all valid' '
 
 # Write 8192 blobs, allowing 1024 requests to be outstanding
 test_expect_success 'store 8K blobs from rank 0 using async RPC' '
-	flux content spam 8192 1024 >/dev/null
+	${SPAMUTIL} 8192 1024 >/dev/null
 '
 
 # Write 1024 blobs per rank
 test_expect_success 'store 1K blobs from all ranks using async RPC' '
-	flux exec -n flux content spam 1024 256 >/dev/null
+	flux exec -n ${SPAMUTIL} 1024 256 >/dev/null
 '
 
 test_expect_success 'load request with empty payload fails with EPROTO(71)' '
@@ -161,5 +197,37 @@ test_expect_success 'load request with empty payload fails with EPROTO(71)' '
 test_expect_success 'register-backing request with empty payload fails with EPROTO(71)' '
 	${RPC} content.register-backing 71 </dev/null
 '
+test_expect_success 'content store --chunksize option splits blobs' '
+	echo "0123456789" >split.data &&
+	flux content store --chunksize=8 <split.data >split.refs &&
+	test $(wc -l <split.refs) -eq 2
+'
+test_expect_success 'content load joins multiple blobrefs on stdin' '
+	flux content load <split.refs >split.out &&
+	test_cmp split.data split.out
+'
+test_expect_success 'content load joins multiple blobrefs on command line' '
+	flux content load $(cat split.refs) >split2.out &&
+	test_cmp split.data split2.out
+'
+test_expect_success 'content store --chunksize=0 does not split blobs' '
+	flux content store --chunksize=0 <split.data >split3.refs &&
+	test $(wc -l <split3.refs) -eq 1 &&
+	flux content load <split3.refs >split3.out &&
+	test_cmp split.data split3.out
+'
+test_expect_success 'content store --chunksize=-1 fails' '
+	test_must_fail flux content store --chunksize=-1 </dev/null
+'
+test_expect_success 'content load with no blobrefs fails' '
+	test_must_fail flux content load </dev/null
+'
+
+test_expect_success 'remove content module' '
+	flux exec flux module remove content
+'
+test_expect_success 'module fails to load with unknown option' '
+	test_must_fail flux module load content badopt
+'
 
 test_done
diff --git a/t/t0012-content-sqlite.t b/t/t0012-content-sqlite.t
index 25bc86d55d48..f94fba428320 100755
--- a/t/t0012-content-sqlite.t
+++ b/t/t0012-content-sqlite.t
@@ -2,6 +2,8 @@
 
 test_description='Test content-sqlite service'
 
+. `dirname $0`/content/content-helper.sh
+
 . `dirname $0`/sharness.sh
 
 # Size the session to one more than the number of cores, minimum of 4
@@ -9,28 +11,41 @@ SIZE=$(test_size_large)
 test_under_flux ${SIZE} minimal
 echo "# $0: flux session size will be ${SIZE}"
 
+PURGE_TARGET_SIZE=100
+PURGE_OLD_ENTRY=1
+FLUSH_BATCH_LIMIT=5
+
 BLOBREF=${FLUX_BUILD_DIR}/t/kvs/blobref
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+SPAMUTIL="${FLUX_BUILD_DIR}/t/kvs/content-spam"
+rc1_kvs=$SHARNESS_TEST_SRCDIR/rc/rc1-kvs
+rc3_kvs=$SHARNESS_TEST_SRCDIR/rc/rc3-kvs
 
-MAXBLOB=`flux getattr content.blob-size-limit`
-HASHFUN=`flux getattr content.hash`
+test_expect_success 'load content module with lower purge/age thresholds' '
+	flux exec flux module load content \
+	    purge-target-size=$PURGE_TARGET_SIZE \
+	    purge-old-entry=$PURGE_OLD_ENTRY \
+	    flush-batch-limit=$FLUSH_BATCH_LIMIT
+'
 
+HASHFUN=`flux getattr content.hash`
 
-store_junk() {
-    local name=$1
-    local n=$2
-    for i in `seq 1 $n`; do \
-        echo "$name:$i" | flux content store >/dev/null || return 1
-    done
-}
+test_expect_success 'load heartbeat module with fast rate to drive purge' '
+	flux module load heartbeat period=0.1s
+'
 
 test_expect_success 'load content-sqlite module on rank 0' '
 	flux module load content-sqlite
 '
 
+test_expect_success 'verify content.backing-module=content-sqlite' '
+	test "$(flux getattr content.backing-module)" = "content-sqlite"
+'
+
 test_expect_success 'store 100 blobs on rank 0' '
-	store_junk test 100 &&
-        TOTAL=`flux module stats --type int --parse count content` &&
-        test $TOTAL -ge 100
+	${SPAMUTIL} 100 100 >/dev/null &&
+	TOTAL=`flux module stats --type int --parse count content` &&
+	test $TOTAL -ge 100
 '
 
 # Store directly to content service
@@ -38,83 +53,410 @@ test_expect_success 'store 100 blobs on rank 0' '
 
 test_expect_success 'store blobs bypassing cache' '
 	cat /dev/null >0.0.store &&
-        flux content store --bypass-cache <0.0.store >0.0.hash &&
-        dd if=/dev/urandom count=1 bs=64 >64.0.store 2>/dev/null &&
-        flux content store --bypass-cache <64.0.store >64.0.hash &&
-        dd if=/dev/urandom count=1 bs=4096 >4k.0.store 2>/dev/null &&
-        flux content store --bypass-cache <4k.0.store >4k.0.hash &&
-        dd if=/dev/urandom count=256 bs=4096 >1m.0.store 2>/dev/null &&
-        flux content store --bypass-cache <1m.0.store >1m.0.hash
-'
-
-test_expect_success LONGTEST "cannot store blob that exceeds max size of $MAXBLOB" '
-        dd if=/dev/zero count=$(($MAXBLOB/4096+1)) bs=4096 \
-			skip=$(($MAXBLOB/4096)) >toobig 2>/dev/null &&
-        test_must_fail flux content store --bypass-cache <toobig
+	flux content store --bypass-cache <0.0.store >0.0.hash &&
+	dd if=/dev/urandom count=1 bs=64 >64.0.store 2>/dev/null &&
+	flux content store --bypass-cache <64.0.store >64.0.hash &&
+	dd if=/dev/urandom count=1 bs=4096 >4k.0.store 2>/dev/null &&
+	flux content store --bypass-cache <4k.0.store >4k.0.hash &&
+	dd if=/dev/urandom count=256 bs=4096 >1m.0.store 2>/dev/null &&
+	flux content store --bypass-cache <1m.0.store >1m.0.hash
 '
 
 test_expect_success 'load 0b blob bypassing cache' '
-        HASHSTR=`cat 0.0.hash` &&
-        flux content load --bypass-cache ${HASHSTR} >0.0.load &&
-        test_cmp 0.0.store 0.0.load
+	HASHSTR=`cat 0.0.hash` &&
+	flux content load --bypass-cache ${HASHSTR} >0.0.load &&
+	test_cmp 0.0.store 0.0.load
 '
 
 test_expect_success 'load 64b blob bypassing cache' '
-        HASHSTR=`cat 64.0.hash` &&
-        flux content load --bypass-cache ${HASHSTR} >64.0.load &&
-        test_cmp 64.0.store 64.0.load
+	HASHSTR=`cat 64.0.hash` &&
+	flux content load --bypass-cache ${HASHSTR} >64.0.load &&
+	test_cmp 64.0.store 64.0.load
 '
 
 test_expect_success 'load 4k blob bypassing cache' '
-        HASHSTR=`cat 4k.0.hash` &&
-        flux content load --bypass-cache ${HASHSTR} >4k.0.load &&
-        test_cmp 4k.0.store 4k.0.load
+	HASHSTR=`cat 4k.0.hash` &&
+	flux content load --bypass-cache ${HASHSTR} >4k.0.load &&
+	test_cmp 4k.0.store 4k.0.load
 '
 
 test_expect_success 'load 1m blob bypassing cache' '
-        HASHSTR=`cat 1m.0.hash` &&
-        flux content load --bypass-cache ${HASHSTR} >1m.0.load &&
-        test_cmp 1m.0.store 1m.0.load
+	HASHSTR=`cat 1m.0.hash` &&
+	flux content load --bypass-cache ${HASHSTR} >1m.0.load &&
+	test_cmp 1m.0.store 1m.0.load
 '
 
 # Verify same blobs on all ranks
-# forcing content to fault in from the content.backing service
+# forcing content to fault in from the content backing service
 
 test_expect_success 'load and verify 64b blob on all ranks' '
-        HASHSTR=`cat 64.0.hash` &&
-        flux exec -n echo ${HASHSTR} >64.0.all.expect &&
-        flux exec -n sh -c "flux content load ${HASHSTR} | $BLOBREF $HASHFUN" \
-                                                >64.0.all.output &&
-        test_cmp 64.0.all.expect 64.0.all.output
+	HASHSTR=`cat 64.0.hash` &&
+	flux exec -n echo ${HASHSTR} >64.0.all.expect &&
+	flux exec -n sh -c "flux content load ${HASHSTR} \
+		| $BLOBREF $HASHFUN" >64.0.all.output &&
+	test_cmp 64.0.all.expect 64.0.all.output
 '
 
 test_expect_success 'load and verify 4k blob on all ranks' '
-        HASHSTR=`cat 4k.0.hash` &&
-        flux exec -n echo ${HASHSTR} >4k.0.all.expect &&
-        flux exec -n sh -c "flux content load ${HASHSTR} | $BLOBREF $HASHFUN" \
-                                                >4k.0.all.output &&
-        test_cmp 4k.0.all.expect 4k.0.all.output
+	HASHSTR=`cat 4k.0.hash` &&
+	flux exec -n echo ${HASHSTR} >4k.0.all.expect &&
+	flux exec -n sh -c "flux content load ${HASHSTR} \
+		| $BLOBREF $HASHFUN" >4k.0.all.output &&
+	test_cmp 4k.0.all.expect 4k.0.all.output
 '
 
 test_expect_success 'load and verify 1m blob on all ranks' '
-        HASHSTR=`cat 1m.0.hash` &&
-        flux exec -n echo ${HASHSTR} >1m.0.all.expect &&
-        flux exec -n sh -c "flux content load ${HASHSTR} | $BLOBREF $HASHFUN" \
-                                                >1m.0.all.output &&
-        test_cmp 1m.0.all.expect 1m.0.all.output
+	HASHSTR=`cat 1m.0.hash` &&
+	flux exec -n echo ${HASHSTR} >1m.0.all.expect &&
+	flux exec -n sh -c "flux content load ${HASHSTR} \
+		| $BLOBREF $HASHFUN" >1m.0.all.output &&
+	test_cmp 1m.0.all.expect 1m.0.all.output
 '
 
 test_expect_success 'exercise batching of synchronous flush to backing store' '
-	flux setattr content.flush-batch-limit 5 &&
-        store_junk loadunload 200 &&
-    	flux content flush &&
+	${SPAMUTIL} 200 200 >/dev/null &&
+	flux content flush &&
 	NDIRTY=`flux module stats --type int --parse dirty content` &&
 	test ${NDIRTY} -eq 0
 '
 
+test_expect_success 'drop the cache' '
+	flux content dropcache
+'
+
+test_expect_success 'fill the cache with more data for later purging' '
+	${SPAMUTIL} 10000 200 >/dev/null
+'
+
+test_expect_success 'checkpoint-put foo w/ rootref bar' '
+	checkpoint_put foo bar
+'
+
+test_expect_success 'checkpoint-get foo returned rootref bar' '
+	echo bar >rootref.exp &&
+	checkpoint_get foo | jq -r .value | jq -r .rootref >rootref.out &&
+	test_cmp rootref.exp rootref.out
+'
+
+test_expect_success 'checkpoint-put on rank 1 forwards to rank 0' '
+       o=$(checkpoint_put_msg rankone rankref) &&
+       jq -j -c -n ${o} | flux exec -r 1 ${RPC} content.checkpoint-put
+'
+
+test_expect_success 'checkpoint-get on rank 1 forwards to rank 0' '
+       echo rankref >rankref.exp &&
+       o=$(checkpoint_get_msg rankone) &&
+       jq -j -c -n ${o} \
+	   | flux exec -r 1 ${RPC} content.checkpoint-get \
+	   | jq -r .value | jq -r .rootref > rankref.out &&
+       test_cmp rankref.exp rankref.out
+'
+
+# use grep instead of compare, incase of floating point rounding
+test_expect_success 'checkpoint-get foo returned correct timestamp' '
+        checkpoint_get foo | jq -r .value | jq -r .timestamp >timestamp.out &&
+        grep 2.2 timestamp.out
+'
+
+test_expect_success 'checkpoint-put updates foo rootref to baz' '
+	checkpoint_put foo baz
+'
+
+test_expect_success 'checkpoint-get foo returned rootref baz' '
+	echo baz >rootref2.exp &&
+	checkpoint_get foo | jq -r .value | jq -r .rootref >rootref2.out &&
+	test_cmp rootref2.exp rootref2.out
+'
+
+test_expect_success 'flush + reload content-sqlite module on rank 0' '
+	flux content flush &&
+	flux module reload content-sqlite
+'
+
+test_expect_success 'checkpoint-get foo still returns rootref baz' '
+	echo baz >rootref3.exp &&
+	checkpoint_get foo | jq -r .value | jq -r .rootref >rootref3.out &&
+	test_cmp rootref3.exp rootref3.out
+'
+
+test_expect_success 'checkpoint-backing-get foo returns rootref baz' '
+	echo baz >rootref_backing.exp &&
+	checkpoint_backing_get foo \
+            | jq -r .value \
+            | jq -r .rootref >rootref_backing.out &&
+	test_cmp rootref_backing.exp rootref_backing.out
+'
+
+test_expect_success 'checkpoint-backing-put foo w/ rootref boof' '
+	checkpoint_backing_put foo boof
+'
+
+test_expect_success 'checkpoint-get foo returned rootref boof' '
+	echo boof >rootref4.exp &&
+	checkpoint_get foo | jq -r .value | jq -r .rootref >rootref4.out &&
+	test_cmp rootref4.exp rootref4.out
+'
+
+test_expect_success 'checkpoint-get noexist fails with No such...' '
+	test_must_fail checkpoint_get noexist 2>badkey.err &&
+	grep "No such file or directory" badkey.err
+'
+
+test_expect_success 'content-backing.load wrong size hash fails with EPROTO' '
+	echo -n xxx >badhash &&
+	$RPC content-backing.load 71 <badhash 2>load.err
+'
+
+getsize() {
+	flux module stats content | tee /dev/fd/2 | jq .size
+}
+
+test_expect_success 'wait for purge to clear cache entries' '
+	echo "Purge size $PURGE_TARGET_SIZE bytes, age $PURGE_OLD_ENTRY secs" &&
+	size=$(getsize) && \
+	count=0 &&
+	while test $size -gt 100 -a $count -lt 300; do \
+		sleep 0.1; \
+		size=$(getsize); \
+		count=$(($count+1))
+	done
+'
+
+test_expect_success 'remove content-sqlite module on rank 0' '
+	flux content flush &&
+	flux module remove content-sqlite
+'
+
+test_expect_success 'checkpoint-put foo w/ rootref spoon' '
+	checkpoint_put foo spoon
+'
+
+test_expect_success 'checkpoint-get foo returned rootref spoon' '
+	echo spoon >rootref5.exp &&
+	checkpoint_get foo | jq -r .value | jq -r .rootref >rootref5.out &&
+	test_cmp rootref5.exp rootref5.out
+'
+
+test_expect_success 'load content-sqlite module on rank 0' '
+	flux module load content-sqlite
+'
+
+# arg1 - expected reference
+wait_checkpoint_flush() {
+	local expected=$1
+	local i=0
+	while checkpoint_backing_get foo \
+		| jq -r .value \
+		| jq -r .rootref > checkpointflush.out \
+	      && [ $i -lt 50 ]
+	do
+	    checkpoint=$(cat checkpointflush.out)
+	    if [ "${checkpoint}" = "${expected}" ]
+	    then
+		return 0
+	    fi
+	    sleep 0.1
+	    i=$((i + 1))
+	done
+	return 1
+}
+
+test_expect_success 'checkpoint-backing-get foo returns spoon' '
+	wait_checkpoint_flush spoon
+'
+
 test_expect_success 'remove content-sqlite module on rank 0' '
+	flux content flush &&
 	flux module remove content-sqlite
 '
 
+test_expect_success 'remove heartbeat module' '
+	flux module remove heartbeat
+'
+
+# test for issue #4210
+test_expect_success 'remove read permission from content.sqlite file' '
+	chmod u-w $(flux getattr rundir)/content.sqlite &&
+	test_must_fail flux module load content-sqlite
+'
+test_expect_success 'restore read permission on content.sqlite file' '
+	chmod u+w $(flux getattr rundir)/content.sqlite
+'
+
+# Clean slate for a few more tests
+test_expect_success 'load content-sqlite with truncate option' '
+	flux module load content-sqlite truncate
+'
+test_expect_success 'content-sqlite and content-cache are empty' '
+	test $(flux module stats \
+	    --type int --parse object_count content-sqlite) -eq 0 &&
+	test $(flux module stats \
+	    --type int --parse count content) -eq 0
+'
+
+test_expect_success 'storing the same object multiple times is just one row' '
+	for i in $(seq 1 10); do \
+	    echo foo | flux content store --bypass-cache >/dev/null; \
+        done &&
+	test $(flux module stats \
+	    --type int --parse store_time.count content-sqlite) -eq 10 &&
+	test $(flux module stats \
+	    --type int --parse object_count content-sqlite) -eq 1
+'
+test_expect_success 'flux module reload content-sqlite' '
+	flux module reload content-sqlite
+'
+test_expect_success 'database survives module reload' '
+	test $(flux module stats \
+	    --type int --parse store_time.count content-sqlite) -eq 0 &&
+	test $(flux module stats \
+	    --type int --parse object_count content-sqlite) -eq 1
+'
+test_expect_success 'reload module with bad option' '
+	flux module remove content-sqlite &&
+	test_must_fail flux module load content-sqlite unknown=42
+'
+test_expect_success 'reload module with journal_mode=WAL synchronous=NORMAL' '
+	flux module remove -f content-sqlite &&
+	flux dmesg --clear &&
+	flux module load content-sqlite journal_mode=WAL synchronous=NORMAL &&
+	flux dmesg >logs &&
+	grep "journal_mode=WAL synchronous=NORMAL" logs
+'
+test_expect_success 'reload module with no options and verify modes' '
+	flux module remove -f content-sqlite &&
+	flux dmesg --clear &&
+	flux module load content-sqlite &&
+	flux dmesg >logs2 &&
+	grep "journal_mode=OFF synchronous=OFF" logs2
+'
+
+
+test_expect_success 'run flux without statedir and verify modes' '
+	flux start -Sbroker.rc1_path=$rc1_kvs -Sbroker.rc3_path=$rc3_kvs \
+	    flux dmesg >logs3 &&
+	grep "journal_mode=OFF synchronous=OFF" logs3
+'
+test_expect_success 'run flux with statedir and verify modes' '
+	flux start -Sbroker.rc1_path=$rc1_kvs -Sbroker.rc3_path=$rc3_kvs \
+	    -Sstatedir=$(pwd) flux dmesg >logs4  &&
+	grep "journal_mode=WAL synchronous=NORMAL" logs4
+'
+test_expect_success 'run flux without statedir and verify config' '
+	flux start -Sbroker.rc1_path=$rc1_kvs -Sbroker.rc3_path=$rc3_kvs \
+	    flux module stats content-sqlite >stats1 &&
+	$jq -e ".config.journal_mode == \"OFF\"" < stats1 &&
+	$jq -e ".config.synchronous == \"OFF\"" < stats1
+'
+test_expect_success 'run flux with statedir and verify config' '
+	flux start -Sbroker.rc1_path=$rc1_kvs -Sbroker.rc3_path=$rc3_kvs \
+	    -Sstatedir=$(pwd) flux module stats content-sqlite >stats2  &&
+	$jq -e ".config.journal_mode == \"WAL\"" < stats2 &&
+	$jq -e ".config.synchronous == \"NORMAL\"" < stats2
+'
+test_expect_success 'flux fails with invalid journal_mode config' '
+	flux start -Sbroker.rc1_path= -Sbroker.rc3_path= \
+	    "flux module load content; \
+	    flux module load content-sqlite journal_mode=FOO; \
+	    flux module remove content" > invalid1.out 2> invalid1.err &&
+	grep "content-sqlite: Invalid argument" invalid1.err
+'
+test_expect_success 'flux fails with invalid synchronous config' '
+	flux start -Sbroker.rc1_path= -Sbroker.rc3_path= \
+	    "flux module load content; \
+	    flux module load content-sqlite synchronous=BAR; \
+	    flux module remove content" > invalid2.out 2> invalid2.err &&
+	grep "content-sqlite: Invalid argument" invalid2.err
+'
+test_expect_success 'test config via config file works' '
+	cat >content-sqlite.toml <<-EOT &&
+	[content-sqlite]
+	journal_mode = "PERSIST"
+	synchronous = "EXTRA"
+	EOT
+	flux start --config-path=$(pwd) \
+	   -Sbroker.rc1_path=$rc1_kvs -Sbroker.rc3_path=$rc3_kvs \
+	   flux module stats content-sqlite > configstats.out &&
+	$jq -e ".config.journal_mode == \"PERSIST\"" < configstats.out &&
+	$jq -e ".config.synchronous == \"EXTRA\"" < configstats.out &&
+	rm content-sqlite.toml
+'
+test_expect_success 'invalid config fails (journal_mode)' '
+	cat >content-sqlite.toml <<-EOT &&
+	[content-sqlite]
+	journal_mode = "FOO"
+	synchronous = "EXTRA"
+	EOT
+	test_must_fail flux start --config-path=$(pwd) \
+	   -Sbroker.rc1_path=$rc1_kvs -Sbroker.rc3_path=$rc3_kvs \
+	   flux module stats content-sqlite &&
+	rm content-sqlite.toml
+'
+test_expect_success 'invalid config fails (synchronous)' '
+	cat >content-sqlite.toml <<-EOT &&
+	[content-sqlite]
+	journal_mode = "PERSIST"
+	synchronous = "BAR"
+	EOT
+	test_must_fail flux start --config-path=$(pwd) \
+	   -Sbroker.rc1_path=$rc1_kvs -Sbroker.rc3_path=$rc3_kvs \
+	   flux module stats content-sqlite &&
+	rm content-sqlite.toml
+'
+
+
+# Will create in WAL mode since statedir is set
+recreate_database()
+{
+	flux start -Sbroker.rc1_path= -Sbroker.rc3_path= \
+	    -Sstatedir=$(pwd) bash -c \
+	    "flux module load content &&
+	    flux module load content-sqlite truncate && \
+	    flux module remove content-sqlite && \
+	    flux module remove content"
+}
+load_module_xfail()
+{
+	flux start -Sbroker.rc1_path= -Sbroker.rc3_path= \
+	    -Sstatedir=$(pwd) bash -c \
+	    "flux module load content; \
+	    flux module load content-sqlite; \
+	    rc=\$?; \
+	    flux module remove -f content-sqlite; \
+	    flux module remove -f content; \
+	    exit \$rc"
+}
+
+# FWIW https://www.sqlite.org/fileformat.html
+test_expect_success 'create database with bad header magic' '
+	recreate_database &&
+	echo "xxxxxxxxxxxxxxxx" | dd obs=1 count=16 seek=0 of=content.sqlite
+'
+test_expect_success 'module load fails with corrupt database' '
+	test_must_fail load_module_xfail
+'
+test_expect_success 'create database with bad schema format number' '
+	recreate_database &&
+	echo "\001\001\001\001" | dd obs=1 count=4 seek=44 of=content.sqlite
+'
+test_expect_success 'module load fails with corrupt database' '
+	test_must_fail load_module_xfail
+'
+test_expect_success 'full instance start fails corrupt database' '
+	test_must_fail flux start -Sstatedir=$(pwd) true
+'
+
+test_expect_success 'flux module stats content-sqlite is open to guests' '
+	FLUX_HANDLE_ROLEMASK=0x2 \
+	    flux module stats content-sqlite >/dev/null
+'
+
+test_expect_success 'remove content-sqlite module on rank 0' '
+	flux module remove content-sqlite
+'
+
+test_expect_success 'remove content module' '
+	flux exec flux module remove content
+'
 
 test_done
diff --git a/t/t0013-config-file.t b/t/t0013-config-file.t
index 4ef73f2d9128..2719638dac28 100755
--- a/t/t0013-config-file.t
+++ b/t/t0013-config-file.t
@@ -1,101 +1,703 @@
 #!/bin/sh
 #
 
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+
 test_description='Test config file overlay bootstrap'
 
 . `dirname $0`/sharness.sh
 
-TCONFDIR=${FLUX_SOURCE_DIR}/t/conf.d
 
 # Avoid loading unnecessary modules in back to back broker tests
 ARGS="-Sbroker.rc1_path= -Sbroker.rc3_path="
 
+# This option is compiled out of flux if zeromq is too old
+if flux broker ${ARGS} flux getattr tbon.tcp_user_timeout >/dev/null 2>&1; then
+	test_set_prereq MAXRT
+else
+	test_set_prereq NOMAXRT
+fi
+if flux broker ${ARGS} flux getattr tbon.connect_timeout >/dev/null 2>&1; then
+	test_set_prereq CONNTO
+else
+	test_set_prereq NOCONNTO
+fi
+
 #
-# check boot.method
+# check config file parsing
 #
+test_expect_success 'flux broker works with no config' '
+	flux broker ${ARGS} flux lsattr -v | grep config.path >noconfig.out &&
+	egrep "^config.path.*-$" noconfig.out
+'
 
-test_expect_success 'flux broker with explicit PMI boot method works' '
-	flux broker ${ARGS} -Sboot.method=pmi /bin/true
+test_expect_success 'flux broker works with empty config directory' '
+	mkdir empty &&
+	cat <<-EOT >empty.exp &&
+	empty
+	EOT
+	flux broker ${ARGS} -c empty flux getattr config.path >empty.out &&
+	test_cmp empty.exp empty.out
+'
+test_expect_success 'flux broker works with empty config file' '
+	touch null.toml &&
+	cat <<-EOT >null.exp &&
+	null.toml
+	EOT
+	flux broker ${ARGS} -c null.toml flux getattr config.path >null.out &&
+	test_cmp null.exp null.out
+'
+test_expect_success 'flux broker works with empty object config file (json)' '
+	cat <<-EOF >empty.json &&
+	{}
+	EOF
+	cat <<-EOT >empty-json.exp &&
+	empty.json
+	EOT
+	flux broker ${ARGS} -c empty.json \
+		flux getattr config.path >empty-json.out &&
+	test_cmp empty-json.exp empty-json.out
 '
+test_expect_success 'FLUX_CONF_DIR also works to specify config dir' '
+	FLUX_CONF_DIR=empty flux broker ${ARGS} \
+		      flux getattr config.path >empty2.out &&
+	test_cmp empty.exp empty2.out
+'
+
+test_expect_success 'flux broker fails with specified config directory missing' "
+	test_must_fail flux broker ${ARGS} -c noexist true
+"
 
-test_expect_success 'flux broker with unknown boot method fails' '
-	test_must_fail flux broker ${ARGS} -Sboot.method=badmethod /bin/true
+test_expect_success 'broker fails with invalid TOML' '
+	mkdir conf1 &&
+	cat <<-EOT >conf1/bootstrap.toml &&
+	[bootstrap]
+	bad-toml
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf1 true
 '
 
+test_expect_success 'broker fails with invalid TOML file' '
+	cat <<-EOT >invalid.toml &&
+	[bootstrap]
+	bad-toml
+	EOT
+	test_must_fail flux broker ${ARGS} -c invalid.toml true
+'
 #
-# check config file parsing
+# [bootstrap] tests
 #
+test_expect_success 'generate curve certificate for configuration' '
+	flux keygen testcert
+'
 
-test_expect_success 'broker startup with missing config fails' "
-	! FLUX_CONF_DIR=/noexist \
-		flux broker ${ARGS} /bin/true
-"
+test_expect_success '[bootstrap] config with bad hosts array' '
+	mkdir conf3 &&
+	cat <<-EOT >conf3/bootstrap.toml &&
+	[bootstrap]
+	hosts = 42
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf3 true
+'
 
-test_expect_success 'broker startup with invalid TOML fails' "
-	! FLUX_CONF_DIR=${TCONFDIR}/bad-toml \
-		flux broker ${ARGS} /bin/true
-"
+test_expect_success '[bootstrap] config with bad hosts array element' '
+	mkdir conf4 &&
+	cat <<-EOT >conf4/bootstrap.toml &&
+	[bootstrap]
+	hosts = [
+	    42
+	]
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf4 true
+'
 
-test_expect_success '[bootstrap] config with missing bootstrap table fails' "
-	! FLUX_CONF_DIR=${TCONFDIR}/bad-nobootstrap \
-		flux broker ${ARGS} -Sboot.method=config /bin/true
-"
+test_expect_success '[bootstrap] config with hosts array element with extra key' '
+	mkdir conf4a &&
+	cat <<-EOT >conf4a/bootstrap.toml &&
+	[bootstrap]
+	hosts = [
+	    { host = "xyz", extrakey = 42 },
+	]
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf4a true
+'
 
-test_expect_success '[bootstrap] config with bad hosts array' "
-	! FLUX_CONF_DIR=${TCONFDIR}/bad-hosts2 \
-		flux broker ${ARGS} -Sboot.method=config /bin/true
-"
+test_expect_success '[bootstrap] config with hosts array element missing host' '
+	mkdir conf4b &&
+	cat <<-EOT >conf4b/bootstrap.toml &&
+	[bootstrap]
+	hosts = [
+	    { },
+	]
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf4b true
+'
 
-test_expect_success '[bootstrap] config with bad hosts array element' "
-	! FLUX_CONF_DIR=${TCONFDIR}/bad-hosts \
-		flux broker ${ARGS} -Sboot.method=config /bin/true
-"
+test_expect_success '[bootstrap] config with bad hostlist' '
+	mkdir conf4c &&
+	cat <<-EOT >conf4c/bootstrap.toml &&
+	[bootstrap]
+	hosts = [
+	    { host = "foo[0-254}" },
+	]
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf4c true
+'
 
-test_expect_success '[bootstrap] config with hostname not found' "
-	! FLUX_CONF_DIR=${TCONFDIR}/bad-nomatch \
-		flux broker ${ARGS} -Sboot.method=config /bin/true
-"
+test_expect_success '[bootstrap] config with with unknown parent' '
+	mkdir conf4d &&
+	cat <<-EOT >conf4d/bootstrap.toml &&
+	[bootstrap]
+	hosts = [
+	    { host = "fake0", parent = "noparent" },
+	]
+	EOT
+	test_must_fail flux start --test-size=1 --test-hosts=fake0 \
+	    -c conf4d true
+'
 
-test_expect_success 'start instance with missing hosts' "
-	FLUX_CONF_DIR=${TCONFDIR}/good-nohosts \
-		flux broker ${ARGS} -Sboot.method=config \
-		flux lsattr -v >attr.out &&
-	grep 'tbon.endpoint.*-$' attr.out
-"
+test_expect_success '[bootstrap] config with with impossible parent' '
+	mkdir conf4e &&
+	cat <<-EOT >conf4e/bootstrap.toml &&
+	[bootstrap]
+	hosts = [
+	    { host = "fake0", parent = "fake0" },
+	]
+	EOT
+	test_must_fail flux start --test-size=1 --test-hosts=fake0 \
+	    -c conf4e true
+'
 
-test_expect_success 'start instance with empty hosts' "
-	FLUX_CONF_DIR=${TCONFDIR}/good-emptyhosts \
-		flux broker ${ARGS} -Sboot.method=config \
-		flux lsattr -v >attr.out &&
-	grep 'tbon.endpoint.*-$' attr.out
-"
+test_expect_success '[bootstrap] config with hostname not found' '
+	mkdir conf5 &&
+	cat <<-EOT >conf5/bootstrap.toml &&
+	[bootstrap]
+	default_bind = "ipc://@flux-testipc-1-0"
+	default_connect = "ipc://@flux-testipc-1-0"
+	hosts = [
+	    { host = "matchnobody" },
+	]
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf5 true
+'
 
-# Usage: start_broker config hostname cmd ...
-start_broker() {
-	local dir=$1; shift
-	local host=$1; shift
-	FLUX_CONF_DIR=$dir FLUX_FAKE_HOSTNAME=$host \
-		flux broker ${ARGS} -Sboot.method=config "$@" &
-}
+test_expect_success '[bootstrap] hosts array can be missing' '
+	mkdir conf6 &&
+	cat <<-EOT >conf6/bootstrap.toml &&
+	[bootstrap]
+	EOT
+	flux broker ${ARGS} -c conf6 flux lsattr -v >missing_attr.out &&
+	grep "tbon.endpoint.*-$" missing_attr.out
+'
 
-test_expect_success 'start size=2 instance with ipc://' "
-	start_broker ${TCONFDIR}/good-ipc2 fake0 flux getattr size >ipc.out &&
-	start_broker ${TCONFDIR}/good-ipc2 fake1 &&
-	wait &&
-	wait &&
-	echo 2 >ipc.exp &&
+test_expect_success '[bootstrap] hosts array can be empty' '
+	mkdir conf7 &&
+	cat <<-EOT >conf7/bootstrap.toml &&
+	[bootstrap]
+	hosts = [
+	]
+	EOT
+	flux broker ${ARGS} -c conf7 flux lsattr -v >empty_attr.out &&
+	grep "tbon.endpoint.*-$" empty_attr.out
+'
+
+test_expect_success 'create initial program for testing' '
+	cat <<-EOT >attrdump.sh &&
+	#!/bin/sh
+	flux getattr size
+	flux getattr hostlist
+	EOT
+	chmod +x attrdump.sh
+'
+
+test_expect_success 'start size=2 instance with ipc://' '
+	BINDDIR=$(mktemp -d) &&
+	test_when_finished "rm -rf $BINDIR" &&
+	mkdir conf8 &&
+	cat <<-EOT >conf8/bootstrap.toml &&
+	[bootstrap]
+	curve_cert = "testcert"
+	[[bootstrap.hosts]]
+	host = "fake0"
+	bind = "ipc://${BINDDIR}/test-ipc2-0"
+	connect = "ipc://${BINDDIR}/test-ipc2-0"
+	[[bootstrap.hosts]]
+	host = "fake1"
+	EOT
+	flux start -s2 --test-hosts=fake[0-1] \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+		--config-path=conf8 \
+		./attrdump.sh >ipc.out &&
+	cat <<-EXP >ipc.exp &&
+	2
+	fake[0-1]
+	EXP
 	test_cmp ipc.exp ipc.out
-"
+'
 
-test_expect_success 'start size=4 instance with tcp://' "
-	start_broker ${TCONFDIR}/good-tcp4 fake0 flux getattr size >tcp.out &&
-	start_broker ${TCONFDIR}/good-tcp4 fake1 &&
-	start_broker ${TCONFDIR}/good-tcp4 fake2 &&
-	start_broker ${TCONFDIR}/good-tcp4 fake3 &&
-	wait && wait && wait && wait &&
-	echo 4 >tcp.exp &&
+test_expect_success 'start size=3 instance with ipc:// and custom topology' '
+	BINDDIR=$(mktemp -d) &&
+	test_when_finished "rm -rf $BINDIR" &&
+	mkdir conf8a &&
+	cat <<-EOT >conf8a/bootstrap.toml &&
+	[bootstrap]
+	curve_cert = "testcert"
+	[[bootstrap.hosts]]
+	host = "fake0"
+	bind = "ipc://${BINDDIR}/fake0"
+	connect = "ipc://${BINDDIR}/fake0"
+	[[bootstrap.hosts]]
+	host = "fake1"
+	bind = "ipc://${BINDDIR}/fake1"
+	connect = "ipc://${BINDDIR}/fake1"
+	[[bootstrap.hosts]]
+	host = "fake2"
+	parent = "fake1"
+	EOT
+	flux start --test-size=3 --test-hosts=fake[0-2] \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+		--config-path=conf8a \
+		flux getattr tbon.maxlevel >conf8a.out &&
+	echo 2 >conf8a.exp &&
+	test_cmp conf8a.exp conf8a.out
+'
+
+waitgrep() {
+	local pattern=$1
+	local file=$2
+	local iter=$3
+	while test $iter -gt 0; do
+		grep "$pattern" $file 2>/dev/null && return 0
+		sleep 0.3
+		iter=$(($iter-1))
+	done
+	return 1
+}
+
+# RFC 2606 reserves the .invalid domain for testing
+test_expect_success NO_CHAIN_LINT 'a warning is printed when upstream URI has unknown host' '
+	mkdir conf8b &&
+	cat <<-EOT >conf8b/bootstrap.toml &&
+	[bootstrap]
+	curve_cert = "testcert"
+	[[bootstrap.hosts]]
+	host = "fake0"
+	connect = "tcp://foo.invalid:1234"
+	[[bootstrap.hosts]]
+	host = "fake1"
+	EOT
+	FLUX_FAKE_HOSTNAME=fake1 \
+		flux broker -vv -Sbroker.rc1_path= -Sbroker.rc3_path= \
+		--config-path=conf8b 2>warn.err &
+	echo $! >warn.pid &&
+	waitgrep "unable to resolve upstream peer" warn.err 30
+'
+# In case warn.pid actually refers to a libtool wrapper, try pkill(1) -P
+# first to kill its children, then kill(1).  See flux-framework/flux-core#5275.
+test_expect_success NO_CHAIN_LINT 'clean up broker from previous test' '
+	warnpid=$(cat warn.pid) &&
+	pkill -15 -P $warnpid || kill -15 $warnpid
+'
+
+getport() {
+	flux python -c \
+	'from socket import socket; s=socket(); s.bind(("", 0)); print(s.getsockname()[1])'
+}
+
+test_expect_success 'start size=4 instance with tcp://' '
+	PORT1=$(getport) &&
+	PORT2=$(getport) &&
+	mkdir conf9 &&
+	cat <<-EOT >conf9/bootstrap.toml &&
+	[bootstrap]
+	curve_cert = "testcert"
+	[[bootstrap.hosts]]
+	host = "fake0"
+	bind = "tcp://127.0.0.1:$PORT1"
+	connect = "tcp://127.0.0.1:$PORT1"
+	[[bootstrap.hosts]]
+	host = "fake1"
+	bind = "tcp://127.0.0.1:$PORT2"
+	connect ="tcp://127.0.0.1:$PORT2"
+	[[bootstrap.hosts]]
+	host = "fake[2-3]"
+	EOT
+	flux start -s4 --test-hosts=fake[0-3] \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+		--config-path=conf9 \
+		./attrdump.sh >tcp.out &&
+	cat <<-EXP >tcp.exp &&
+	4
+	fake[0-3]
+	EXP
 	test_cmp tcp.exp tcp.out
-"
+'
+
+test_expect_success '[bootstrap] curve_cert is required for size > 1' '
+	mkdir conf10 &&
+	cat <<-EOT >conf10/bootstrap.toml &&
+	[bootstrap]
+	default_bind = "ipc://@flux-testipc-1-0"
+	default_connect = "ipc://@flux-testipc-1-0"
+	hosts = [
+	    { host = "foo1" },
+	    { host = "foo2" }
+	]
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf10 true
+'
+
+test_expect_success '[bootstrap] curve_cert must exist' '
+	mkdir conf11 &&
+	cat <<-EOT >conf11/bootstrap.toml &&
+	[bootstrap]
+	curve_cert = "conf11/cert"
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf11 true
+'
+
+test_expect_success '[bootstrap] curve_cert file must contain valid cert' '
+	mkdir conf12 &&
+	flux keygen conf12/cert &&
+	sed --in-place -e "s/key/nerp/g" conf12/cert &&
+	cat <<-EOT >conf12/bootstrap.toml &&
+	[bootstrap]
+	curve_cert = "conf12/cert"
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf12 true
+'
+
+test_expect_success '[bootstrap] curve_cert file must be mode g-r' '
+	mkdir conf13 &&
+	flux keygen conf13/cert &&
+	chmod g+r conf13/cert &&
+	cat <<-EOT >conf13/bootstrap.toml &&
+	[bootstrap]
+	curve_cert = "conf13/cert"
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf13 true
+'
+
+test_expect_success '[bootstrap] curve_cert file must be mode o-r' '
+	mkdir conf14 &&
+	flux keygen conf14/cert &&
+	chmod o+r conf14/cert &&
+	cat <<-EOT >conf14/bootstrap.toml &&
+	[bootstrap]
+	curve_cert = "conf14/cert"
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf14 true
+'
+
+#
+# [tbon] test
+#
 
+test_expect_success MAXRT 'tbon.tcp_user_timeout is 20s by default' '
+	cat <<-EOT >maxrt.exp &&
+	20s
+	EOT
+	flux broker ${ARGS} \
+		flux getattr tbon.tcp_user_timeout >maxrt.out &&
+	test_cmp maxrt.exp maxrt.out
+'
+test_expect_success MAXRT 'tbon.tcp_user_timeout can be configured' '
+	mkdir conf15 &&
+	cat <<-EOT2 >maxrt2.exp &&
+	30s
+	EOT2
+	cat <<-EOT >conf15/tbon.toml &&
+	[tbon]
+	tcp_user_timeout = "30s"
+	EOT
+	flux broker ${ARGS} -c conf15 flux getattr tbon.tcp_user_timeout \
+		>maxrt2.out &&
+	test_cmp maxrt2.exp maxrt2.out
+'
+test_expect_success MAXRT 'tbon.tcp_user_timeout command line overrides config' '
+	cat <<-EOT >maxrt3.exp &&
+	1h
+	EOT
+	flux broker ${ARGS} -c conf15 \
+		-Stbon.tcp_user_timeout=1h \
+		flux getattr tbon.tcp_user_timeout >maxrt3.out &&
+	test_cmp maxrt3.exp maxrt3.out
+'
+test_expect_success MAXRT 'tbon.tcp_user_timeout with bad FSD on command line fails' '
+	test_must_fail flux broker ${ARGS} \
+		-Stbon.tcp_user_timeout=zzz \
+		true 2>badattr.err &&
+	grep "Error parsing" badattr.err
+'
+test_expect_success MAXRT 'tbon.tcp_user_timeout with bad FSD configured fails' '
+	mkdir conf16 &&
+	cat <<-EOT >conf16/tbon.toml &&
+	[tbon]
+	tcp_user_timeout = 42
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf16 \
+		true 2>badconf.err &&
+	grep "Config file error" badconf.err
+'
+test_expect_success NOMAXRT 'tbon.tcp_user_timeout config cannot be set with old zeromq' '
+	mkdir conf17 &&
+	cat <<-EOT >conf17/tbon.toml &&
+	[tbon]
+	tcp_user_timeout = "30s"
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf17 \
+		true 2>noconf.err &&
+	grep "unsupported by this zeromq version" noconf.err
+'
+test_expect_success NOMAXRT 'tbon.tcp_user_timeout attr cannot be set with old zeromq' '
+	test_must_fail flux broker ${ARGS} \
+		-Stbon.tcp_user_timeout=30s \
+		true 2>noattr.err &&
+	grep "unsupported by this zeromq version" noattr.err
+'
+
+test_expect_success 'tbon.zmqdebug is zero by default' '
+	cat <<-EOT >zmqdebug.exp &&
+	0
+	EOT
+	flux broker ${ARGS} \
+		flux getattr tbon.zmqdebug >zmqdebug.out &&
+	test_cmp zmqdebug.exp zmqdebug.out
+'
+test_expect_success 'tbon.zmqdebug can be configured' '
+	mkdir conf18 &&
+	cat <<-EOT2 >zmqdebug2.exp &&
+	1
+	EOT2
+	cat <<-EOT >conf18/tbon.toml &&
+	[tbon]
+	zmqdebug = 1
+	EOT
+	flux broker ${ARGS} -c conf18 flux getattr tbon.zmqdebug \
+		>zmqdebug2.out &&
+	test_cmp zmqdebug2.exp zmqdebug2.out
+'
+test_expect_success MAXRT 'tbon.zmqdebug with bad value on command line fails' '
+	test_must_fail flux broker ${ARGS} \
+		-Stbon.zmqdebug=zzz \
+		true 2>zbadattr.err &&
+	grep "value must be an integer" zbadattr.err
+'
+test_expect_success MAXRT 'tbon.zmqdebug configured with wrong type fails' '
+	mkdir conf19 &&
+	cat <<-EOT >conf19/tbon.toml &&
+	[tbon]
+	zmqdebug = "notint"
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf19 \
+		true 2>zbadconf.err &&
+	grep "Expected integer" zbadconf.err
+'
+test_expect_success MAXRT 'tbon.zmqdebug configured with wrong type fails' '
+	mkdir conf20 &&
+	cat <<-EOT >conf20/tbon.toml &&
+	[tbon]
+	zmqdebug = "notint"
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf20 \
+		true 2>zbadconf.err &&
+	grep "Expected integer" zbadconf.err
+'
+test_expect_success 'tbon.zmq_io_threads is 1 by default' '
+	echo 1 >threads1.exp &&
+	flux broker ${ARGS} \
+		flux getattr tbon.zmq_io_threads >threads.out &&
+	test_cmp threads1.exp threads.out
+'
+test_expect_success 'tbon.zmq_io_threads config only applies to rank 0' '
+	cat <<-EOT >zmq_io_threads.toml &&
+	[tbon]
+	zmq_io_threads = 4
+	EOT
+	cat <<-EOT2 >t4.exp &&
+	0: 4
+	1: 1
+	EOT2
+	flux start -s2 --config-path=zmq_io_threads.toml \
+		flux exec --label-io flux getattr tbon.zmq_io_threads \
+		| sort >t4.out &&
+	test_cmp t4.exp t4.out
+'
+test_expect_success 'tbon.zmq_io_threads broker attr overrides' '
+	cat <<-EOT2 >t2.exp &&
+	0: 2
+	1: 1
+	EOT2
+	flux start -s2 \
+		--config-path=zmq_io_threads.toml \
+		-Stbon.zmq_io_threads=2 \
+		flux exec --label-io flux getattr tbon.zmq_io_threads \
+		| sort >t2.out &&
+	test_cmp t2.exp t2.out
+'
+test_expect_success 'setting tbon.zmq_io_threads to -1 fails' '
+	test_expect_code 1 flux broker ${ARGS} -Stbon.zmq_io_threads=-1 true
+'
+test_expect_success 'tbon.child_rcvhwm is 0 by default' '
+	echo 0 >hwm0.exp &&
+	flux broker ${ARGS} \
+		flux getattr tbon.child_rcvhwm >hwm0.out &&
+	test_cmp hwm0.exp hwm0.out
+'
+test_expect_success 'tbon.child_rcvhwm can be configured' '
+	cat <<-EOT >hwm.toml &&
+	[tbon]
+	child_rcvhwm = 5
+	EOT
+	echo 5 >hwm5.exp &&
+	flux broker ${ARGS} --config-path=hwm.toml \
+		flux getattr tbon.child_rcvhwm >hwm5.out &&
+	test_cmp hwm5.exp hwm5.out
+'
+test_expect_success 'tbon.child_rcvhwm broker attr overrides' '
+	echo 2 >hwm2.exp &&
+	flux broker ${ARGS} --config-path=hwm.toml \
+		-Stbon.child_rcvhwm=2 \
+		flux getattr tbon.child_rcvhwm >hwm2.out &&
+	test_cmp hwm2.exp hwm2.out
+'
+test_expect_success 'tbon.child_rcvhwm=-1 fails' '
+	test_expect_code 1 flux broker ${args} -Stbon.child_rcvhwm=-1 true
+'
+test_expect_success 'tbon.child_rcvhwm=1 fails' '
+	test_expect_code 1 flux broker ${args} -Stbon.child_rcvhwm=1 true
+'
+test_expect_success 'tbon.torpid_max, tbon.torpid_min can be configured' '
+	mkdir conf21 &&
+	cat <<-EOT >conf21/tbon.toml &&
+	[tbon]
+	torpid_max = "5s"
+	torpid_min = "2s"
+	EOT
+	flux broker ${ARGS} -c conf21 \
+		flux getattr tbon.torpid_min >torpid_min.out &&
+	flux broker ${ARGS} -c conf21 \
+		flux getattr tbon.torpid_max >torpid_max.out &&
+	grep 2s torpid_min.out &&
+	grep 5s torpid_max.out
+'
+test_expect_success 'tbon.torpid_max configured with wrong type fails' '
+	mkdir conf22 &&
+	cat <<-EOT >conf22/tbon.toml &&
+	[tbon]
+	torpid_max = 5
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf22 \
+		true 2>badtorpid.err &&
+	grep "Expected string" badtorpid.err
+'
+test_expect_success 'tbon.topo with unknown scheme fails' '
+	mkdir conf23 &&
+	cat <<-EOT >conf23/tbon.toml &&
+	[tbon]
+	topo = "notascheme:42"
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf23 \
+		true 2>badscheme.err &&
+	grep "unknown topology scheme" badscheme.err
+'
+test_expect_success 'tbon.topo is kary:32 by default' '
+	echo "kary:32" >topo.exp &&
+	flux broker ${ARGS} flux getattr tbon.topo >topo.out &&
+	test_cmp topo.exp topo.out
+'
+test_expect_success 'tbon.topo can be changed by configuration' '
+	mkdir conf24 &&
+	cat <<-EOT >conf24/tbon.toml &&
+	[tbon]
+	topo = "kary:8"
+	EOT
+	echo "kary:8" >topo2.exp &&
+	flux broker ${ARGS} -c conf24 \
+		flux getattr tbon.topo >topo2.out &&
+	test_cmp topo2.exp topo2.out
+'
+test_expect_success 'tbon.topo can be overridden on the command line' '
+	echo "kary:16" >topo3.exp &&
+	flux broker ${ARGS} -c conf24 -Stbon.topo=kary:16 \
+		flux getattr tbon.topo >topo3.out &&
+	test_cmp topo3.exp topo3.out
+'
+test_expect_success 'tbon.topo is custom when bootstrap is configured' '
+	mkdir conf25 &&
+	cat <<-EOT >conf25/bootstrap.toml &&
+	[bootstrap]
+	EOT
+	echo "custom" >topo4.exp &&
+	flux broker ${ARGS} -c conf25 \
+		flux getattr tbon.topo >topo4.out &&
+	test_cmp topo4.exp topo4.out
+'
+test_expect_success CONNTO 'tbon.connect_timeout is 30s by default' '
+	cat <<-EOT >connto.exp &&
+	30s
+	EOT
+	flux broker ${ARGS} \
+		flux getattr tbon.connect_timeout >connto.out &&
+	test_cmp connto.exp connto.out
+'
+test_expect_success CONNTO 'tbon.connect_timeout can be configured' '
+	mkdir conf26 &&
+	cat <<-EOT2 >connto2.exp &&
+	10s
+	EOT2
+	cat <<-EOT >conf26/tbon.toml &&
+	[tbon]
+	connect_timeout = "10s"
+	EOT
+	flux broker ${ARGS} -c conf26 flux getattr tbon.connect_timeout \
+		>connto2.out &&
+	test_cmp connto2.exp connto2.out
+'
+test_expect_success CONNTO 'tbon.connect_timeout command line overrides config' '
+	cat <<-EOT >connto3.exp &&
+	1h
+	EOT
+	flux broker ${ARGS} -c conf26 \
+		-Stbon.connect_timeout=1h \
+		flux getattr tbon.connect_timeout >connto3.out &&
+	test_cmp connto3.exp connto3.out
+'
+test_expect_success NOCONNTO 'tbon.connect_timeout config cannot be set with old zeromq' '
+	mkdir conf27 &&
+	cat <<-EOT >conf27/tbon.toml &&
+	[tbon]
+	connect_timeout = "35s"
+	EOT
+	test_must_fail flux broker ${ARGS} -c conf27 \
+		true 2>noconnto_conf.err &&
+	grep "unsupported by this zeromq version" noconnto_conf.err
+'
+test_expect_success NOCONNTO 'tbon.connect_timeout attr cannot be set with old zeromq' '
+	test_must_fail flux broker ${ARGS} \
+		-Stbon.connect_timeout=10s \
+		true 2>noconnto_attr.err &&
+	grep "unsupported by this zeromq version" noconnto_attr.err
+'
+test_expect_success CONNTO 'tbon.connect_timeout config can be set to 0' '
+	mkdir conf28 &&
+	cat <<-EOT2 >connto_0.exp &&
+	0s
+	EOT2
+	cat <<-EOT >conf28/tbon.toml &&
+	[tbon]
+	connect_timeout = "0"
+	EOT
+	flux broker ${ARGS} -c conf28 flux getattr tbon.connect_timeout \
+		>connto_conf_0.out &&
+	test_cmp connto_0.exp connto_conf_0.out
+'
+test_expect_success CONNTO 'tbon.connect_timeout attr can be set to 0' '
+	flux broker ${ARGS} \
+		-Stbon.connect_timeout=0 \
+		flux getattr tbon.connect_timeout >connto_attr_0.out &&
+	test_cmp connto_0.exp connto_attr_0.out
+'
 
 test_done
diff --git a/t/t0014-runlevel.t b/t/t0014-runlevel.t
index 77cc5dd415c1..95eb6e775ac6 100755
--- a/t/t0014-runlevel.t
+++ b/t/t0014-runlevel.t
@@ -1,94 +1,182 @@
 #!/bin/sh
 #
 
-test_description='Verify runlevels work properly
+test_description='Verify rc scripts execute with proper semantics
 '
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile --debug
 
 . `dirname $0`/sharness.sh
-test_under_flux 1 minimal
 
-test_expect_success 'sharness minimal init.run-level=2' '
-	runlevel=$(flux getattr init.run-level) &&
-	test $runlevel -eq 2
+test_expect_success 'initial program is run when rc1/rc3 are nullified' '
+	flux start -Slog-stderr-level=6 \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+		true 2>normal.log
 '
 
-test_expect_success 'sharness minimal has transitioned normally thus far' '
-	flux dmesg >default.log &&
-	grep -q "Run level 1 starting" default.log &&
-	grep -q "Run level 1 Not configured" default.log &&
-	grep -q "Run level 2 starting" default.log &&
-	! grep -q "Run level 2 Not configured" default.log &&
-	! grep -q "Run level 3" default.log
-'
-
-test_expect_success 'new instance transitions appropriately' '
-	flux start -o,-Slog-stderr-level=6 \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
-		/bin/true 2>normal.log &&
-	grep -q "Run level 1 starting" normal.log &&
-	grep -q "Run level 1 Not configured" normal.log &&
-	grep -q "Run level 2 starting" normal.log &&
-	grep -q "Run level 2 Exited" normal.log &&
-	grep -q "Run level 3 starting" normal.log &&
-	grep -q "Run level 3 Not configured" normal.log
-'
-
-test_expect_success 'rc1 failure transitions to rc3, fails instance' '
+test_expect_success 'rc1 failure causes instance failure' '
 	test_expect_code 1 flux start \
-		-o,-Sbroker.rc1_path=/bin/false,-Sbroker.rc3_path= \
-		-o,-Slog-stderr-level=6 \
-		/bin/true 2>false.log &&
-	grep -q "Run level 1 starting" false.log &&
-	grep -q "Run level 1 Exited with non-zero status" false.log &&
-	! grep -q "Run level 2 starting" false.log &&
-	! grep -q "Run level 2 Exited" false.log &&
-	grep -q "Run level 3 starting" false.log &&
-	grep -q "Run level 3 Not configured" false.log
+		-Sbroker.rc1_path=false -Sbroker.rc3_path= \
+		-Slog-stderr-level=6 \
+		sleep 3600 2>rc1_failure.log
 '
 
 test_expect_success 'rc1 bad path handled same as failure' '
 	(
 	  SHELL=/bin/sh &&
 	  test_expect_code 127 flux start \
-		-o,-Sbroker.rc1_path=rc1-nonexist,-Sbroker.rc3_path= \
-		-o,-Slog-stderr-level=6 \
-		/bin/true 2>bad1.log
-	) &&
-	grep -q "Run level 1 starting" bad1.log &&
-	grep -q "Run level 1 Exited with non-zero status" bad1.log &&
-	! grep -q "Run level 2 starting" bad1.log &&
-	! grep -q "Run level 2 Exited" bad1.log &&
-	grep -q "Run level 3 starting" bad1.log &&
-	grep -q "Run level 3 Not configured" bad1.log
+		-Sbroker.rc1_path=rc1-nonexist -Sbroker.rc3_path= \
+		-Slog-stderr-level=6 \
+		true 2>bad1.log
+	)
+'
+
+test_expect_success 'default initial program is $SHELL' '
+	run_timeout --env=SHELL=/bin/sh 60 \
+		flux $SHARNESS_TEST_SRCDIR/scripts/runpty.py -i none \
+		flux start -Slog-stderr-level=6 \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+		>shell.log &&
+	grep "rc2.0: /bin/sh Exited" shell.log
+'
+
+test_expect_success 'rc2 failure if stdin not a tty' '
+	test_expect_code 1 \
+		flux start -Slog-stderr-level=6 \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+                2>shell-notty.log &&
+	grep "not a tty" shell-notty.log
 '
 
 test_expect_success 'rc3 failure causes instance failure' '
-	! flux start \
-		-o,-Sbroker.rc3_path=/bin/false \
-		-o,-Slog-stderr-level=6 \
-		/bin/true 2>false3.log &&
-	grep -q "Run level 1 starting" false3.log &&
-	grep -q "Run level 1 Exited" false3.log &&
-	grep -q "Run level 2 starting" false3.log &&
-	grep -q "Run level 2 Exited" false3.log &&
-	grep -q "Run level 3 starting" false3.log &&
-	grep -q "Run level 3 Exited with non-zero status" false3.log
-'
-
-test_expect_success 'instance with no rc2 terminated cleanly by timeout' '
-	run_timeout 0.5 flux start \
-		-o,-Slog-stderr-level=6 \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path=,-Sbroker.rc2_none
-'
-
-test_expect_success 'rc1 environment is as expected' '
-	flux start \
-		-o,-Sbroker.rc1_path=${FLUX_SOURCE_DIR}/t/rc/rc1-testenv \
-		-o,-Sbroker.rc3_path= \
-		-o,-Slog-stderr-level=6 \
-		/bin/true 2>&1 | tee rc1-test.log &&
-	grep "stderr-" rc1-test.log | egrep -q broker.*err &&
-	grep "stdout-" rc1-test.log | egrep -q broker.*info
+	test_expect_code 1 flux start \
+		-Sbroker.rc3_path=false \
+		-Slog-stderr-level=6 \
+		true 2>rc3_failure.log
+'
+
+test_expect_success 'broker.rc2_none terminates by signal without error' '
+	for timeout in 0.5 1 2 4; do
+	    run_timeout -s ALRM $timeout flux start \
+		-Slog-stderr-level=6 \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= -Sbroker.rc2_none &&
+	    break
+	done
+'
+
+test_expect_success 'flux admin cleanup-push true works' '
+	flux start -Slog-stderr-level=6 \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+		flux admin cleanup-push true
+'
+
+test_expect_success 'flux admin cleanup-push false causes instance failure' '
+	test_expect_code 1 flux start -Slog-stderr-level=6 \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+		flux admin cleanup-push false
+'
+
+test_expect_success 'cleanup does not run if rc1 fails' '
+	test_expect_code 1 flux start -Slog-stderr-level=6 \
+		-Sbroker.rc1_path=false -Sbroker.rc3_path= \
+		flux admin cleanup-push memorable-string 2>nocleanup.err && \
+	test_must_fail grep memorable-string nocleanup.err
+'
+
+test_expect_success 'flux admin cleanup-push (empty) fails' '
+	test_expect_code 1 flux start \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+		flux admin cleanup-push "" 2>push.err &&
+	grep "cannot push an empty command line" push.err
+'
+
+test_expect_success 'flux admin cleanup-push with no commands fails' '
+	test_expect_code 1 flux start \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+		flux admin cleanup-push </dev/null 2>push2.err &&
+	grep "commands array is empty" push2.err
+'
+
+test_expect_success 'flux admin cleanup-push (stdin) works' '
+	echo true | flux start -Slog-stderr-level=6 \
+		-Sbroker.rc1_path= -Sbroker.rc3_path= \
+		flux admin cleanup-push 2>push-stdin.err &&
+	grep cleanup.0 push-stdin.err
+'
+
+test_expect_success 'flux admin cleanup-push (stdin) retains cmd block order' '
+	flux start -Sbroker.rc1_path= -Sbroker.rc3_path= \
+		-Slog-stderr-level=6 \
+		flux admin cleanup-push <<-EOT 2>hello.err &&
+	echo Hello world
+	echo Hello solar system
+	EOT
+	grep "cleanup.0: Hello world" hello.err &&
+	grep "cleanup.1: Hello solar system" hello.err
+'
+
+test_expect_success 'capture the environment for all three rc scripts' '
+	SLURM_FOO=42 flux start \
+		-Slog-stderr-level=6 \
+		-Sbroker.rc1_path="bash -c printenv >rc1.env" \
+		-Sbroker.rc3_path="bash -c printenv >rc3.env" \
+		"bash -c printenv >rc2.env"
+'
+
+var_is_unset() {
+	local name=$1; shift
+	while test $# -gt 0; do
+		grep "^$name=" $1 && return 1
+		shift
+	done
+}
+
+var_is_set() {
+	local name=$1; shift
+	while test $# -gt 0; do
+		grep "^$name=" $1 || return 1
+		shift
+	done
+}
+
+test_expect_success 'PMI_FD, PMI_SIZE, PMI_RANK are not set in rc scripts' '
+	var_is_unset PMI_FD *.env &&
+	var_is_unset PMI_RANK *.env &&
+	var_is_unset PMI_SIZE *.env
+'
+test_expect_success 'SLURM_* vars were cleared from env of rc scripts' '
+	var_is_unset SLURM_FOO *.env
+'
+test_expect_success 'FLUX_URI is set in rc scripts' '
+	var_is_set FLUX_URI *.env
+'
+
+test_expect_success 'job environment is not set in rc scripts' '
+	var_is_unset FLUX_JOB_ID *.env &&
+	var_is_unset FLUX_JOB_SIZE *.env &&
+	var_is_unset FLUX_JOB_NNODES *.env &&
+	var_is_unset FLUX_JOB_TMPDIR *.env &&
+	var_is_unset FLUX_TASK_RANK *.env &&
+	var_is_unset FLUX_TASK_LOCAL_ID *.env &&
+	var_is_unset FLUX_KVS_NAMESPACE *.env
+'
+
+test_expect_success 'capture the environment for instance run as a job' '
+	flux start flux run flux start \
+		-Slog-stderr-level=6 \
+		-Sbroker.rc1_path="bash -c printenv >rc1.env2" \
+		-Sbroker.rc3_path="bash -c printenv >rc3.env2" \
+		"bash -c printenv >rc2.env2"
+'
+
+test_expect_success 'job environment is not set in rcs of subinstance' '
+	var_is_unset FLUX_JOB_ID *.env2 &&
+	var_is_unset FLUX_JOB_SIZE *.env2 &&
+	var_is_unset FLUX_JOB_NNODES *.env2 &&
+	var_is_unset FLUX_JOB_TMPDIR *.env2 &&
+	var_is_unset FLUX_TASK_RANK *.env2 &&
+	var_is_unset FLUX_TASK_LOCAL_ID *.env2 &&
+	var_is_unset FLUX_KVS_NAMESPACE *.env2
 '
 
 test_done
diff --git a/t/t0015-cron.t b/t/t0015-cron.t
index b3fd6e5e0c91..210d9b7b5d44 100755
--- a/t/t0015-cron.t
+++ b/t/t0015-cron.t
@@ -43,6 +43,9 @@ contains() {
     fi
 }
 
+test_expect_success 'module fails to load with unknown option' '
+	test_must_fail flux module load cron badopt
+'
 test_expect_success 'load cron module' '
     flux module load cron
 '
diff --git a/t/t0016-cron-faketime.t b/t/t0016-cron-faketime.t
index fb6bf9af1c00..14bed65ff1e8 100755
--- a/t/t0016-cron-faketime.t
+++ b/t/t0016-cron-faketime.t
@@ -8,6 +8,13 @@ if ! test_have_prereq NO_ASAN; then
     test_done
 fi
 
+# allow libfaketime to be found on ubuntu, centos
+if test -d /usr/lib/x86_64-linux-gnu/faketime ; then
+  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/faketime"
+elif test -d /usr/lib64/faketime ; then
+  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib64/faketime"
+fi
+
 #  Check for libfaketimeMT using known epoch with /bin/date:
 t=$(LD_PRELOAD=libfaketimeMT.so.1 FAKETIME="1973-11-29 21:33:09 UTC" date +%s)
 if test "$t" != "123456789" ; then
@@ -32,6 +39,10 @@ cron_entry_check() {
     test -n $key || return 1
     test -n "$expected" || return 1
     local result="$(flux cron dump --key=${key} ${id})" || return 1
+    if test "$key" = "typedata.next_wakeup"; then
+        # convert result to an integer:
+        result=$(printf "%.0f" $result)
+    fi
     echo "cron-${id}: ${key}=${result}, wanted ${expected}" >&2
     test "${result}" = "${expected}"
 }
@@ -57,12 +68,20 @@ chmod +x make-faketime.sh
 set_faketime=$(pwd)/make-faketime.sh
 event_trace="run_timeout 5 $SHARNESS_TEST_SRCDIR/scripts/event-trace.lua"
 
+within_two() {
+    local result=$1
+    shift
+    local exact=$1
+    local two_after=$(($exact + 2))
+    test "$result" -ge $exact && test "$result" -le $two_after
+}
+
 #  Why does date need to be set 1s in the future??
 test_expect_success 'libfaketime works' '
     now=$(date +"@%Y-%m-%d %H:%M:%S") &&
     $set_faketime Jun 4 1991 00:00:01 &&
     flux logger "libfaketime-test" &&
-    test "$(date +%s)" = $(date +%s --date="Jun 4 1991 00:00:00") &&
+    within_two "$(date +%s)" $(date +%s --date="Jun 4 1991 00:00:00") &&
     date +%s &&
     flux dmesg | grep libfaketime-test &&
     echo $now > ${FAKETIME_TIMESTAMP_FILE}
@@ -71,15 +90,15 @@ test_expect_success 'load cron module' '
     flux module load cron
 '
 test_expect_success 'flux-cron tab works for set minute' '
-    $set_faketime 2016-06-04 15:30:00 &&
+    $set_faketime today 15:30 &&
     id=$(echo "15 * * * * flux event pub t.cron.complete" | flux_cron tab) &&
-    next=$(date +%s --date="2016-06-04 16:15:00") &&
+    next=$(date +%s --date="today 16:15") &&
     cron_entry_check ${id} type datetime &&
     cron_entry_check ${id} stopped false &&
     cron_entry_check ${id} typedata.next_wakeup ${next} &&
     echo sleeping at $(date) &&
     ${event_trace} t.cron t.cron.complete \
-        $set_faketime 2016-06-04 16:15:00 &&
+        $set_faketime today 16:15 &&
     echo done at $(date) &&
     cron_entry_check ${id} stats.count 1 &&
     flux cron delete ${id}
diff --git a/t/t0017-security.t b/t/t0017-security.t
index 64d6db4ed8bb..7515cbb10930 100755
--- a/t/t0017-security.t
+++ b/t/t0017-security.t
@@ -4,143 +4,112 @@ test_description='Test broker security'
 
 . `dirname $0`/sharness.sh
 
+# Start out with empty "config object"
+export FLUX_CONF_DIR=$(pwd)
 test_under_flux 4 minimal
 
-jq=$(which jq 2>/dev/null)
-test -z "$jq" || test_set_prereq HAVE_JQ
-
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
 
+test_expect_success 'connector-local starts with private access policy' '
+	flux dmesg | grep connector-local >dmesg.out &&
+	grep allow-root-owner=false dmesg.out &&
+	grep allow-guest-user=false dmesg.out
+'
+
+test_expect_success 'connector-local adds allow-root-owner on reconfig' '
+	flux dmesg --clear &&
+	cat >access.toml <<-EOT &&
+	[access]
+	allow-root-owner = true
+	EOT
+	flux config reload &&
+	flux dmesg | grep connector-local >dmesg2.out &&
+	grep allow-root-owner=true dmesg2.out
+'
+
+test_expect_success 'connector-local drops allow-root-owner on reconfig' '
+	flux dmesg --clear &&
+	cat >access.toml <<-EOT &&
+	[access]
+	EOT
+	flux config reload &&
+	flux dmesg | grep connector-local >dmesg3.out &&
+	grep allow-root-owner=false dmesg3.out
+'
+
+test_expect_success 'connector-local adds allow-guest-user on reconfig' '
+	flux dmesg --clear &&
+	cat >access.toml <<-EOT &&
+	[access]
+	allow-guest-user = true
+	EOT
+	flux config reload &&
+	flux dmesg | grep connector-local >dmesg4.out &&
+	grep allow-guest-user=true dmesg4.out
+'
+
+test_expect_success 'connector-local drops allow-guest-user on reconfig' '
+	flux dmesg --clear &&
+	cat >access.toml <<-EOT &&
+	[access]
+	EOT
+	flux config reload &&
+	flux dmesg | grep connector-local >dmesg5.out &&
+	grep allow-guest-user=false dmesg5.out
+'
+
+test_expect_success 'connector-local reconfig fails on unknown access key' '
+	cat >access.toml <<-EOT &&
+	[access]
+	foo = 42
+	EOT
+	test_must_fail flux config reload 2>reload.err &&
+	grep foo reload.err
+'
+
+test_expect_success 'reconfig with bad TOML fails' '
+	cat >access.toml <<-EOT &&
+	[access]
+	foo
+	EOT
+	test_must_fail flux config reload
+'
+
+test_expect_success 'connector-local restored private access policy' '
+	flux dmesg --clear &&
+	cat >access.toml <<-EOT &&
+	[access]
+	EOT
+	flux config reload &&
+	flux dmesg | grep connector-local >dmesg6.out &&
+	grep allow-root-owner=false dmesg6.out &&
+	grep allow-guest-user=false dmesg6.out
+'
+
 test_expect_success 'simulated local connector auth failure returns EPERM' '
-	flux comms info &&
+	flux getattr size &&
 	flux module debug --set 1 connector-local &&
-	test_must_fail flux comms info 2>authfail.out &&
+	test_must_fail flux getattr size 2>authfail.out &&
 	grep -q "Operation not permitted" authfail.out
 '
 
-test_expect_success 'flux user list fails with reasonable error when userdb is not loaded' '
-	test_must_fail flux user list 2>userlistfail.out &&
-	grep -q "userdb module is not loaded" userlistfail.out
-'
-
-test_expect_success 'flux userdb includes only instance owner by default' '
-	flux module load userdb &&
-	flux user list >userdb.list &&
-	grep -q $(id -u):owner userdb.list &&
-	test $(wc -l <userdb.list) -eq 1 &&
-	flux module remove userdb
-'
-
-test_expect_success '(forced) userdb lookup fails with EPERM when userdb not loaded' '
-	flux module debug --set 2 connector-local &&
-	test_must_fail flux comms info 2>authfail2.out &&
-	grep -q "Operation not permitted" authfail2.out
-'
-
-test_expect_success '(forced) userdb lookup succeeds when userdb is loaded' '
-	flux module load userdb &&
-	flux module debug --set 2 connector-local &&
-	flux comms info &&
-	flux module remove userdb
-'
-
-test_expect_success '(forced) userdb lookup fails when instance owner is removed' '
-	flux module load userdb &&
-	flux user delrole $(id -u) owner &&
-	! flux user lookup $(id -u) &&
-	flux module debug --set 2 connector-local &&
-	test_must_fail flux comms info 2>authfail3.out &&
-	grep -q "Operation not permitted" authfail3.out &&
-	flux module remove userdb
-'
-
-test_expect_success 'flux user addrole adds users' '
-	flux module load userdb &&
-	flux user addrole 1234 user &&
-	flux user list >userdb2.list &&
-	grep -q 1234:user userdb2.list &&
-	test $(wc -l <userdb2.list) -eq 2 &&
-	flux module remove userdb
-'
-
-test_expect_success 'flux user delrole removes users with no roles' '
-	flux module load userdb &&
-	flux user addrole 1234 user &&
-	flux user delrole 1234 user &&
-	flux user list >userdb2.list &&
-	! grep -q 1234:user userdb2.list &&
-	test $(wc -l <userdb2.list) -eq 1 &&
-	flux module remove userdb
-'
-
-test_expect_success 'userdb --default-rolemask works' '
-	flux module load userdb --default-rolemask=owner,user &&
-	flux user delrole $(id -u) owner &&
-	flux user list >userdb3.list &&
-	test $(wc -l <userdb3.list) -eq 0 &&
-	flux module debug --set 2 connector-local &&
-	flux comms info &&
-	flux user list >userdb4.list &&
-	grep -q $(id -u):owner,user userdb4.list &&
-	flux module remove userdb
-'
-
-test_expect_success 'flux user cannot add FLUX_USERID_UNKNOWN' '
-	flux module load userdb &&
-	! flux user addrole 4294967295 user 2>inval.out &&
-	grep -q "invalid userid" inval.out &&
-	flux module remove userdb
-'
-
-test_expect_success 'flux user can add/lookup bin user by name' '
-	flux module load userdb &&
-	flux user addrole bin user &&
-	flux user list >userdb5.list &&
-	grep -q $(id -u bin):user userdb5.list &&
-	flux user lookup bin >getbin.out &&
-	grep -q $(id -u bin):user getbin.out &&
-	flux module remove userdb
-'
-
-test_expect_success 'load userdb module' '
-	flux module load userdb
-'
-
-test_expect_success 'flux user cannot add user with no roles' '
-	test_must_fail flux user addrole 1234 0
-'
-
-test_expect_success 'lookup request with empty payload fails with EPROTO(71)' '
-	${RPC} userdb.lookup 71 </dev/null
-'
-test_expect_success 'addrole request with empty payload fails with EPROTO(71)' '
-	${RPC} userdb.addrole 71 </dev/null
-'
-test_expect_success 'delrole request with empty payload fails with EPROTO(71)' '
-	${RPC} userdb.delrole 71 </dev/null
-'
-
-test_expect_success 'unload userdb module' '
-	flux module remove userdb
-'
-
 test_expect_success 'flux ping --userid displays userid' '
-	flux ping --count=1 --userid cmb >ping.out &&
-	grep -q "userid=$(id -u) rolemask=0x1" ping.out
+	flux ping --count=1 --userid broker >ping.out &&
+	grep -q "userid=$(id -u) rolemask=0x5" ping.out
 '
 
 test_expect_success 'FLUX_HANDLE_USERID can spoof userid in message' '
-	FLUX_HANDLE_USERID=9999 flux ping --count=1 --userid cmb >ping2.out &&
-	grep -q "userid=9999 rolemask=0x1" ping2.out
+	FLUX_HANDLE_USERID=9999 flux ping --count=1 --userid broker >ping2.out &&
+	grep -q "userid=9999 rolemask=0x5" ping2.out
 '
 
 test_expect_success 'FLUX_HANDLE_ROLEMASK can spoof rolemask in message' '
-	FLUX_HANDLE_ROLEMASK=0xf flux ping --count=1 --userid cmb >ping3.out &&
+	FLUX_HANDLE_ROLEMASK=0xf flux ping --count=1 --userid broker >ping3.out &&
 	grep -q "userid=$(id -u) rolemask=0xf" ping3.out
 '
 
 test_expect_success 'flux ping allowed for non-owner' '
-	FLUX_HANDLE_ROLEMASK=0x2 flux ping --count=1 --userid cmb >ping4.out &&
+	FLUX_HANDLE_ROLEMASK=0x2 flux ping --count=1 --userid broker >ping4.out &&
 	grep -q "userid=$(id -u) rolemask=0x2" ping4.out
 '
 
@@ -168,7 +137,7 @@ test_expect_success 'flux logger not allowed for non-owner' '
 
 test_expect_success 'flux dmesg not allowed for non-owner' '
 	! FLUX_HANDLE_ROLEMASK=0x2 flux dmesg 2>dmesg.err &&
-	grep -q "Operation not permitted" dmesg.err
+	grep -q "Request requires owner credentials" dmesg.err
 '
 
 # Note these rules:
@@ -285,13 +254,13 @@ test_expect_success 'dispatcher suppresses guest event to same guest connection'
 	! grep -q test.a ev8.out
 '
 
-test_expect_success HAVE_JQ 'guests may add userid-prefixed services' '
+test_expect_success 'guests may add userid-prefixed services' '
 	USERID=$(id -u) &&
 	${jq} -n "{service: \"${USERID}-sectest\"}" >user_service.json &&
 	FLUX_HANDLE_ROLEMASK=0x2 ${RPC} service.add <user_service.json
 '
 
-test_expect_success HAVE_JQ 'guests may not add other-userid-prefixed services' '
+test_expect_success 'guests may not add other-userid-prefixed services' '
 	USERID=$(($(id -u)+1)) &&
 	${jq} -n "{service: \"${USERID}-sectest\"}" >user2_service.json &&
 	(export FLUX_HANDLE_ROLEMASK=0x2 &&
@@ -301,7 +270,7 @@ test_expect_success HAVE_JQ 'guests may not add other-userid-prefixed services'
 	grep "Operation not permitted" uservice_add.err
 '
 
-test_expect_success HAVE_JQ 'guests may not add non-userid-prefixed services' '
+test_expect_success 'guests may not add non-userid-prefixed services' '
 	${jq} -n "{service: \"sectest\"}" >service.json &&
 	(export FLUX_HANDLE_ROLEMASK=0x2 &&
 		test_expect_code 1 ${RPC} service.add \
@@ -312,6 +281,9 @@ test_expect_success HAVE_JQ 'guests may not add non-userid-prefixed services' '
 
 # kvs.namespace-<NS>-setroot is a "private" event
 
+test_expect_success 'loaded content module' '
+	flux module load content
+'
 test_expect_success 'loaded kvs module' '
 	flux module load kvs
 '
@@ -356,13 +328,37 @@ test_expect_success 'flux content flush not allowed for guest user' '
 	grep -q "Operation not permitted" content.flush.err
 '
 
-test_expect_success 'flux content load/store allowed for guest user' '
-	echo Hello >content.store.value &&
+test_expect_success 'store a value with content service' '
+	echo Hello >content.blob &&
+	flux content store <content.blob >content.blobref
+'
+
+test_expect_success 'flux content load not allowed for guest user' '
+	test_must_fail bash -c "FLUX_HANDLE_ROLEMASK=0x2 \
+	    flux content load $(cat content.blobref) 2>content-load.err" &&
+	    grep "Request requires owner credentials" content-load.err
+'
+
+test_expect_success 'flux content store not allowed for guest user' '
+	test_must_fail bash -c "FLUX_HANDLE_ROLEMASK=0x2 \
+	    flux content store <content.blob 2>content-store.err" &&
+	    grep "Request requires owner credentials" content-store.err
+'
+
+test_expect_success 'unloaded content module' '
+	flux module remove content
+'
+
+test_expect_success 'flux module list is open to guests' '
+	FLUX_HANDLE_ROLEMASK=0x2 flux module list >/dev/null
+'
+test_expect_success 'flux module stats --rusage is open to guests (broker)' '
 	FLUX_HANDLE_ROLEMASK=0x2 \
-	    flux content store <content.store.value >content.store.ref &&
+	    flux module stats --rusage broker >/dev/null
+'
+test_expect_success 'flux module stats --rusage is open to guests (module)' '
 	FLUX_HANDLE_ROLEMASK=0x2 \
-	    flux content load $(cat content.store.ref) >content.load.value &&
-	test_cmp content.store.value content.load.value
+	    flux module stats --rusage connector-local >/dev/null
 '
 
 test_done
diff --git a/t/t0018-content-files.t b/t/t0018-content-files.t
new file mode 100755
index 000000000000..b0004d4cf18a
--- /dev/null
+++ b/t/t0018-content-files.t
@@ -0,0 +1,308 @@
+#!/bin/sh
+
+test_description='Test content-files backing store service'
+
+. `dirname $0`/content/content-helper.sh
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 1 minimal -Sstatedir=$(pwd)
+
+BLOBREF=${FLUX_BUILD_DIR}/t/kvs/blobref
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+TEST_LOAD=${FLUX_BUILD_DIR}/src/modules/content-files/test_load
+TEST_STORE=${FLUX_BUILD_DIR}/src/modules/content-files/test_store
+
+SIZES="0 1 64 100 1000 1024 1025 8192 65536 262144 1048576 4194304"
+LARGE_SIZES="8388608 10000000 16777216 33554432 67108864"
+
+##
+# Functions used by tests
+##
+
+# Usage: backing_load <hash
+backing_load() {
+        $RPC -r -R content-backing.load
+}
+# Usage: backing_store <blob >hash
+backing_store() {
+        $RPC -r -R content-backing.store
+}
+# Usage: make_blob size >blob
+make_blob() {
+	if test $1 -eq 0; then
+		dd if=/dev/null 2>/dev/null
+	else
+		dd if=/dev/urandom count=1 bs=$1 2>/dev/null
+	fi
+}
+# Usage: check_blob size
+# Leaves behind blob.<size> and hash.<size>
+check_blob() {
+	make_blob $1 >blob.$1 &&
+	backing_store <blob.$1 >hash.$1 &&
+	backing_load <hash.$1 >blob.$1.check &&
+	test_cmp blob.$1 blob.$1.check
+}
+# Usage: check_blob size
+# Relies on existence of blob.<size> and hash.<size>
+recheck_blob() {
+	backing_load <hash.$1 >blob.$1.recheck &&
+	test_cmp blob.$1 blob.$1.recheck
+}
+# Usage: recheck_cache_blob size
+# Relies on existence of blob.<size>
+recheck_cache_blob() {
+	local blobref=$($BLOBREF sha1 <blob.$1)
+	flux content load $blobref >blob.$1.cachecheck &&
+	test_cmp blob.$1 blob.$1.cachecheck
+}
+
+test_expect_success 'load content module' '
+	flux module load content
+'
+
+##
+# Tests of the module by itself (no content cache)
+##
+
+test_expect_success 'content-files module load fails with unknown option' '
+	test_must_fail flux module load content-files notoption
+'
+
+test_expect_success 'load content-files module' '
+	flux module load content-files testing
+'
+
+test_expect_success 'load/store/verify key-values stored directly' '
+	make_blob 140 >rawblob.140 &&
+	$TEST_STORE $(pwd)/content.files testkey1 <rawblob.140 &&
+	$TEST_LOAD $(pwd)/content.files testkey1 >rawblob.140.out &&
+	test_cmp rawblob.140 rawblob.140.out
+'
+
+test_expect_success 'store/load/verify various size small blobs' '
+	err=0 &&
+	for size in $SIZES; do \
+		if ! check_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success LONGTEST 'store/load/verify various size large blobs' '
+	err=0 &&
+	for size in $LARGE_SIZES; do \
+		if ! check_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success 'reload content-files module' '
+	flux module reload content-files testing
+'
+
+test_expect_success 'reload/verify various size small blobs' '
+	err=0 &&
+	for size in $SIZES; do \
+		if ! recheck_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success LONGTEST 'reload/verify various size large blobs' '
+	err=0 &&
+	for size in $LARGE_SIZES; do \
+		if ! recheck_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success 'load with invalid hash size fails with EPROTO' '
+	test_must_fail backing_load </dev/null 2>badhash.err &&
+	grep "Protocol error" badhash.err
+'
+
+##
+# Tests of the module acting as backing store for content cache
+##
+
+test_expect_success 'reload content-files module without testing option' '
+	flux module reload content-files
+'
+
+test_expect_success 'verify content.backing-module=content-files' '
+        test "$(flux getattr content.backing-module)" = "content-files"
+'
+
+test_expect_success 'reload/verify various size small blobs through cache' '
+	err=0 &&
+	for size in $SIZES; do \
+		if ! recheck_cache_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success LONGTEST 'reload/verify various size large blobs through cache' '
+	err=0 &&
+	for size in $LARGE_SIZES; do \
+		if ! recheck_cache_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success 'flux module stats reports nonzero object count' '
+	test $(flux module stats \
+	    --type int --parse object_count content-files) -gt 0
+'
+test_expect_success 'reload content-files with truncate option' '
+	flux module reload content-files truncate
+'
+test_expect_success 'flux module stats reports zero object count' '
+	test $(flux module stats \
+	    --type int --parse object_count content-files) -eq 0
+'
+
+test_expect_success 'checkpoint-put foo w/ rootref bar' '
+	checkpoint_put foo bar
+'
+
+test_expect_success 'checkpoint-get foo returned rootref bar' '
+        echo bar >rootref.exp &&
+        checkpoint_get foo | jq -r .value | jq -r .rootref >rootref.out &&
+        test_cmp rootref.exp rootref.out
+'
+
+# use grep instead of compare, incase of floating point rounding
+test_expect_success 'checkpoint-get foo returned correct timestamp' '
+        checkpoint_get foo | jq -r .value | jq -r .timestamp >timestamp.out &&
+        grep 2.2 timestamp.out
+'
+
+test_expect_success 'checkpoint-put updates foo rootref to baz' '
+        checkpoint_put foo baz
+'
+
+test_expect_success 'checkpoint-get foo returned rootref baz' '
+        echo baz >rootref2.exp &&
+        checkpoint_get foo | jq -r .value | jq -r .rootref >rootref2.out &&
+        test_cmp rootref2.exp rootref2.out
+'
+
+test_expect_success 'reload content-files module' '
+	flux module reload content-files
+'
+
+test_expect_success 'checkpoint-get foo still returns rootref baz' '
+        echo baz >rootref3.exp &&
+        checkpoint_get foo | jq -r .value | jq -r .rootref >rootref3.out &&
+        test_cmp rootref3.exp rootref3.out
+'
+
+test_expect_success 'checkpoint-put updates foo rootref with longer rootref' '
+        checkpoint_put foo abcdefghijklmnopqrstuvwxyz
+'
+
+test_expect_success 'checkpoint-get foo returned rootref with longer rootref' '
+        echo abcdefghijklmnopqrstuvwxyz >rootref4.exp &&
+        checkpoint_get foo | jq -r .value | jq -r .rootref >rootref4.out &&
+        test_cmp rootref4.exp rootref4.out
+'
+
+test_expect_success 'checkpoint-put updates foo rootref to shorter rootref' '
+        checkpoint_put foo foobar
+'
+
+test_expect_success 'checkpoint-get foo returned rootref with shorter rootref' '
+        echo foobar >rootref5.exp &&
+        checkpoint_get foo | jq -r .value | jq -r .rootref >rootref5.out &&
+        test_cmp rootref5.exp rootref5.out
+'
+
+test_expect_success 'checkpoint-put updates foo rootref to boof' '
+        checkpoint_put foo boof
+'
+
+test_expect_success 'checkpoint-backing-get foo returns rootref boof' '
+        echo boof >rootref_backing.exp &&
+        checkpoint_backing_get foo \
+            | jq -r .value \
+            | jq -r .rootref >rootref_backing.out &&
+        test_cmp rootref_backing.exp rootref_backing.out
+'
+
+test_expect_success 'checkpoint-backing-put foo w/ rootref poof' '
+        checkpoint_backing_put foo poof
+'
+
+test_expect_success 'checkpoint-get foo returned rootref boof' '
+        echo poof >rootref6.exp &&
+        checkpoint_get foo | jq -r .value | jq -r .rootref >rootref6.out &&
+        test_cmp rootref6.exp rootref6.out
+'
+
+test_expect_success 'checkpoint-get bad request fails with EPROTO' '
+	test_must_fail $RPC content.checkpoint-get </dev/null 2>badget.err &&
+	grep "Protocol error" badget.err
+'
+test_expect_success 'checkpoint-put bad request fails with EPROTO' '
+	test_must_fail $RPC content.checkpoint-put </dev/null 2>badput.err &&
+	grep "Protocol error" badput.err
+'
+
+test_expect_success 'remove content-files module' '
+	flux module remove content-files
+'
+
+test_expect_success 'checkpoint-put foo w/ rootref spoon' '
+       checkpoint_put foo spoon
+'
+
+test_expect_success 'checkpoint-get foo returned rootref spoon' '
+       echo spoon >rootref7.exp &&
+       checkpoint_get foo | jq -r .value | jq -r .rootref >rootref7.out &&
+       test_cmp rootref7.exp rootref7.out
+'
+
+test_expect_success 'load content-files module on rank 0' '
+       flux module load content-files
+'
+
+# arg1 - expected reference
+wait_checkpoint_flush() {
+        local expected=$1
+        local i=0
+        while checkpoint_backing_get foo \
+                | jq -r .value \
+                | jq -r .rootref > checkpointflush.out \
+              && [ $i -lt 50 ]
+        do
+            checkpoint=$(cat checkpointflush.out)
+            if [ "${checkpoint}" = "${expected}" ]
+            then
+                return 0
+            fi
+            sleep 0.1
+            i=$((i + 1))
+        done
+        return 1
+}
+
+test_expect_success 'checkpoint-backing-get foo returns spoon' '
+       wait_checkpoint_flush spoon
+'
+
+test_expect_success 'flux module stats content-files is open to guests' '
+	FLUX_HANDLE_ROLEMASK=0x2 \
+	    flux module stats content-files >/dev/null
+'
+
+test_expect_success 'remove content-files module on rank 0' '
+       flux content flush &&
+       flux module remove content-files
+'
+
+test_expect_success 'remove content module' '
+	flux module remove content
+'
+
+test_done
diff --git a/t/t0019-jobspec-schema.t b/t/t0019-jobspec-schema.t
deleted file mode 100755
index 08077ea9d84a..000000000000
--- a/t/t0019-jobspec-schema.t
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/sh
-
-test_description='Test the jobspec schema validation'
-
-. `dirname $0`/sharness.sh
-
-JOBSPEC=${SHARNESS_TEST_SRCDIR}/jobspec
-VALIDATE="flux python ${JOBSPEC}/validate.py"
-SCHEMA=${FLUX_SOURCE_DIR}/src/modules/job-ingest/schemas/jobspec.jsonschema
-SCHEMA_V1=${FLUX_SOURCE_DIR}/src/modules/job-ingest/schemas/jobspec_v1.jsonschema
-
-validate() {
-   ${VALIDATE} --schema ${SCHEMA} $1
-}
-validate_v1() {
-   ${VALIDATE} --schema ${SCHEMA_V1} $1
-}
-
-# Check that the valid jobspecs all pass
-for jobspec in ${JOBSPEC}/valid/*.yaml; do
-    testname=`basename $jobspec`
-    test_expect_success 'valid: '$testname "validate $jobspec"
-done
-# V1 validates against general
-for jobspec in ${JOBSPEC}/valid_v1/*.yaml; do
-    testname=jobspec_v1/$(basename $jobspec)
-    test_expect_success 'valid: '$testname "validate $jobspec"
-done
-# V1 validates against V1
-for jobspec in ${JOBSPEC}/valid_v1/*.yaml; do
-    testname=jobspec_v1/$(basename $jobspec)
-    test_expect_success 'valid_v1: '$testname "validate_v1 $jobspec"
-done
-
-# Check that the invalid jobspec all fail
-for jobspec in ${JOBSPEC}/invalid/*.yaml; do
-    testname=`basename $jobspec`
-    test_expect_success 'error: '$testname "test_must_fail validate $jobspec"
-done
-
-test_done
diff --git a/t/t0019-tbon-config.t b/t/t0019-tbon-config.t
new file mode 100755
index 000000000000..138c62d4e683
--- /dev/null
+++ b/t/t0019-tbon-config.t
@@ -0,0 +1,136 @@
+#!/bin/sh
+#
+
+test_description='Test the brokers tbon.* configuration'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. `dirname $0`/sharness.sh
+
+ARGS="-Sbroker.rc1_path= -Sbroker.rc3_path="
+
+test_expect_success 'flux-start with size 2 works with tbon.zmqdebug' '
+	flux start ${ARGS} -Stbon.zmqdebug=1 -s2 true
+'
+test_expect_success 'flux-start with non-integer tbon.zmqdebug fails' '
+	test_must_fail flux start ${ARGS} -Stbon.zmqdebug=foo true
+'
+test_expect_success 'tbon.endpoint can be read' '
+	ATTR_VAL=`flux start ${ARGS} -s2 flux getattr tbon.endpoint` &&
+	echo $ATTR_VAL | grep "://"
+'
+test_expect_success 'tbon.endpoint uses ipc:// in standalone instance' '
+	flux start ${ARGS} -s2 \
+		flux getattr tbon.endpoint >endpoint.out &&
+	grep "^ipc://" endpoint.out
+'
+test_expect_success 'tbon.endpoint uses tcp:// if process mapping unavailable' '
+	flux start ${ARGS} -s2 --test-pmi-clique=none \
+		flux getattr tbon.endpoint >endpoint2.out &&
+	grep "^tcp" endpoint2.out
+'
+test_expect_success 'tbon.endpoint uses tcp:// if tbon.prefertcp is set' '
+	flux start ${ARGS} -s2 -Stbon.prefertcp=1 \
+		flux getattr tbon.endpoint >endpoint2.out &&
+	grep "^tcp" endpoint2.out
+'
+test_expect_success 'FLUX_IPADDR_INTERFACE=lo works' '
+	FLUX_IPADDR_INTERFACE=lo flux start \
+		${ARGS} -s2 -Stbon.prefertcp=1 \
+		flux getattr tbon.endpoint >endpoint3.out &&
+	grep "127.0.0.1" endpoint3.out
+'
+test_expect_success 'tbon.interface-hint=lo works' '
+	flux start -Stbon.interface-hint=lo \
+		${ARGS} -s2 -Stbon.prefertcp=1 \
+		flux getattr tbon.endpoint >endpoint3a.out &&
+	grep "127.0.0.1" endpoint3a.out
+'
+test_expect_success 'tbon.interface-hint=127.0.0.0/8 works' '
+	flux start -Stbon.interface-hint=127.0.0.0/8 \
+		${ARGS} -s2 -Stbon.prefertcp=1 \
+		flux getattr tbon.endpoint >endpoint3b.out &&
+	grep "127.0.0.1" endpoint3b.out
+'
+test_expect_success 'TOML tbon.interface-hint=127.0.0.0/8 works' '
+	cat >hint.toml <<-EOT &&
+	tbon.interface-hint = "127.0.0.0/8"
+	EOT
+	flux start --config-path=hint.toml \
+		${ARGS} -s2 -Stbon.prefertcp=1 \
+		flux getattr tbon.endpoint >endpoint3c.out &&
+	grep "127.0.0.1" endpoint3c.out
+'
+test_expect_success 'TOML tbon.interface-hint=wrong type fails' '
+	cat >badhint.toml <<-EOT &&
+	tbon.interface-hint = 42
+	EOT
+	test_must_fail flux start --config-path=badhint.toml ${ARGS} true
+'
+test_expect_success 'tbon.interface-hint=badiface fails' '
+	test_must_fail_or_be_terminated flux start \
+		-Stbon.interface-hint=badiface \
+		${ARGS} -s2 -Stbon.prefertcp=1 true
+'
+test_expect_success 'tbon.interface-hint=default-route works' '
+	flux start -Stbon.interface-hint=default-route -Stbon.prefertcp=1 \
+		${ARGS} -s2 true
+'
+test_expect_success 'tbon.interface-hint=hostname works' '
+	flux start -Stbon.interface-hint=hostname -Stbon.prefertcp=1 \
+		${ARGS} -s2 true
+'
+test_expect_success 'tbon.interface-hint defaults to default-router' '
+	flux start ${ARGS} flux getattr tbon.interface-hint >defhint.out &&
+	grep default-route defhint.out
+'
+test_expect_success 'tbon.interface-hint default comes from parent' '
+	flux start -Stbon.interface-hint=hostname \
+	    flux alloc -N1 flux getattr tbon.interface-hint >childhint.out &&
+	grep hostname childhint.out
+'
+test_expect_success 'tbon.interface-hint from parent can be overridden' '
+	flux start -Stbon.interface-hint=hostname \
+	    flux alloc -N1 --broker-opts=-Stbon.interface-hint=default-router \
+	    flux getattr tbon.interface-hint >childhint2.out &&
+	grep default-router childhint2.out
+'
+test_expect_success 'tbon.endpoint cannot be set' '
+	test_must_fail_or_be_terminated flux start ${ARGS} -s2 \
+		--setattr=tbon.endpoint=ipc:///tmp/customflux true
+'
+test_expect_success 'tbon.parent-endpoint cannot be read on rank 0' '
+	test_must_fail flux start ${ARGS} -s2 flux getattr tbon.parent-endpoint
+'
+test_expect_success 'tbon.parent-endpoint can be read on not rank 0' '
+	NUM=`flux start ${ARGS} -s4 flux exec -n flux getattr tbon.parent-endpoint | grep ipc | wc -l` &&
+	test $NUM -eq 3
+'
+test_expect_success 'broker -Stbon.fanout=4 is an alias for tbon.topo=kary:4' '
+	echo kary:4 >fanout.exp &&
+	flux start ${ARGS} -Stbon.fanout=4 \
+		flux getattr tbon.topo >fanout.out &&
+	test_cmp fanout.exp fanout.out
+'
+test_expect_success 'broker -Stbon.topo=kary:8 option works' '
+	echo kary:8 >topo.exp &&
+	flux start ${ARGS} -Stbon.topo=kary:8 \
+		flux getattr tbon.topo >topo.out &&
+	test_cmp topo.exp topo.out
+'
+test_expect_success 'broker -Stbon.topo=kary:0 works' '
+	flux start ${ARGS} -Stbon.topo=kary:0 true
+'
+test_expect_success 'broker -Stbon.topo=custom option works' '
+	echo custom >topo2.exp &&
+	flux start ${ARGS} -Stbon.topo=custom \
+		flux getattr tbon.topo >topo2.out &&
+	test_cmp topo2.exp topo2.out
+'
+test_expect_success 'broker -Stbon.topo=binomial option works' '
+	echo binomial >topo_binomial.exp &&
+	flux start ${ARGS} -Stbon.topo=binomial \
+		flux getattr tbon.topo >topo_binomial.out &&
+	test_cmp topo_binomial.exp topo_binomial.out
+'
+test_done
diff --git a/t/t0020-terminus.t b/t/t0020-terminus.t
new file mode 100755
index 000000000000..fcd970cfb1af
--- /dev/null
+++ b/t/t0020-terminus.t
@@ -0,0 +1,248 @@
+#!/bin/sh
+#
+
+test_description='Test flux-terminus command
+
+Verify basic functionality of flux-terminus command
+'
+
+. `dirname $0`/sharness.sh
+SIZE=2
+test_under_flux ${SIZE} minimal
+
+# Unset any existing terminus session
+unset FLUX_TERMINUS_SESSION
+
+userid=$(id -u)
+default_service="${userid}-terminus"
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py -f asciicast"
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+
+test_expect_success 'flux-terminus: list reports error with no server' '
+	name="list-no-server" &&
+	test_expect_code 1 flux terminus list >log.${name} 2>&1 &&
+	grep "no server running at ${default_service}" log.${name} &&
+	test_expect_code 1 flux terminus list -r 1 >log.${name}.1 2>&1 &&
+	grep "no server running at ${default_service}" log.${name}.1 &&
+	test_expect_code 1 flux terminus list -s foo >log.${name}.foo 2>&1 &&
+	grep "no server running at foo" log.${name}.foo
+'
+test_expect_success 'flux-terminus: kill reports error with no server' '
+	name="kill-no-server" &&
+	test_expect_code 1 flux terminus kill 0 >log.${name} 2>&1 &&
+	grep "no server running at ${default_service}" log.${name}
+'
+test_expect_success 'flux-terminus: attach fails with error on no server' '
+	test_expect_code 1 flux terminus attach 0
+'
+test_expect_success 'flux-terminus: kill-server reports error with no server' '
+	name="kill-server-no-server" &&
+	test_expect_code 1 flux terminus kill-server >log.${name} 2>&1 &&
+	grep "no server running at ${default_service}" log.${name}
+'
+test_expect_success 'flux-terminus: list reports error with bad rank/service' '
+	name="list-bad-args" &&
+	test_expect_code 1 flux terminus list -r -1 >log.${name} 2>&1 &&
+	grep "no server running at ${default_service}" log.${name} &&
+	test_expect_code 1 flux terminus list -r 1 >log.${name}.1 2>&1 &&
+	grep "no server running at ${default_service}" log.${name}.1 &&
+	test_expect_code 1 flux terminus list -s foo >log.${name}.foo 2>&1 &&
+	grep "no server running at foo" log.${name}.foo
+'
+test_expect_success 'flux-terminus: attach fails with invalid id' '
+	test_expect_code 1 flux terminus attach foo >log.attach-badid 2>&1 &&
+	grep "session ID must be an integer" log.attach-badid
+'
+test_expect_success 'flux-terminus: kill fails with invalid id' '
+	test_expect_code 1 flux terminus kill foo >log.kill-badid 2>&1 &&
+	grep "session ID must be an integer" log.kill-badid
+'
+test_expect_success 'flux-terminus: start with invalid command fails' '
+	test_expect_code 1 flux terminus start /nosuchthing >log.badcmd 2>&1 &&
+	grep "failed to run /nosuchthing" log.badcmd
+'
+test_expect_success 'flux-terminus: start detached starts server' '
+	flux terminus start -d &&
+	flux terminus list >log.list &&
+	test_debug "cat log.list" &&
+	grep -q "1 current session" log.list
+'
+test_expect_success 'flux-terminus: start detached starts another session' '
+	flux terminus start -d &&
+	flux terminus list >log.list &&
+	test_debug "cat log.list" &&
+	grep -q "2 current sessions" log.list
+'
+test_expect_success 'flux-terminus: kill session works' '
+	flux terminus kill 0 &&
+	flux terminus list >log.list &&
+	test_debug "cat log.list" &&
+	grep -q "1 current session" log.list
+'
+test_expect_success 'flux-terminus: start --name option works' '
+	flux terminus start -d --name=test-name &&
+	flux terminus list >log.list &&
+	test_debug "cat log.list" &&
+	grep -q test-name log.list
+'
+test_expect_success 'flux-terminus: kill-server works' '
+	flux terminus kill-server &&
+	test_expect_code 1 flux terminus list
+'
+test_expect_success 'flux-terminus: start server on remote rank' '
+	flux exec -r 1 flux terminus start -d &&
+	flux terminus list -r 1 >log.list-remote &&
+	test_debug "cat log.list-remote"  &&
+	grep -q "1 current session" log.list-remote
+'
+test_expect_success 'flux-terminus: start on remote rank' '
+	flux terminus start -dr 1 &&
+	flux terminus list -r 1 >log.list-remote &&
+	test_debug "cat log.list-remote"  &&
+	grep -q "2 current sessions" log.list-remote
+'
+test_expect_success 'flux-terminus: kill on remote rank' '
+	flux terminus kill -r 1  0 &&
+	flux terminus list -r 1 >log.list-remote &&
+	test_debug "cat log.list-remote"  &&
+	grep -q "1 current session" log.list-remote
+'
+test_expect_success 'flux-terminus: kill-server on remote rank' '
+	flux terminus kill-server -r 1 &&
+	test_expect_code 1 flux terminus list -r 1
+'
+test_expect_success 'flux-terminus: start on invalid service name gives error' '
+	test_must_fail flux terminus start -s . >log.invalid-service 2>&1 &&
+	test_debug "cat log.invalid-service" &&
+	grep -q "Invalid argument" log.invalid-service
+'
+test_expect_success 'flux-terminus: start server on alternate service name' '
+	flux terminus start -ds foo &&
+	flux terminus list -s foo >log.list-foo &&
+	test_debug "cat log.list-foo"  &&
+	grep -q "1 current session" log.list-foo
+'
+test_expect_success 'flux-terminus: start session on alternate service name' '
+	flux terminus start -ds foo &&
+	flux terminus list -s foo >log.list-foo &&
+	test_debug "cat log.list-foo"  &&
+	grep -q "2 current sessions" log.list-foo
+'
+test_expect_success 'flux-terminus: kill on alternate service name' '
+	flux terminus kill -s foo  0 &&
+	flux terminus list -s foo >log.list-foo &&
+	test_debug "cat log.list-foo"  &&
+	grep -q "1 current session" log.list-foo
+'
+test_expect_success 'flux-terminus: kill-server on alternate service name' '
+	flux terminus kill-server -s foo &&
+	test_expect_code 1 flux terminus list -s foo
+'
+test_expect_success 'flux-terminus: start can set session name' '
+	flux terminus start -d -n test-name &&
+	flux terminus list | grep "\[test-name\]"
+'
+test_expect_success 'flux-terminus: start and set --wait' '
+	flux terminus start --wait -d -n waiter true &&
+	flux terminus list >log.start-wait 2>&1 &&
+	test_debug "cat log.start-wait" &&
+	grep -q "\[waiter\]" log.start-wait
+'
+test_expect_success 'flux-terminus: clean up' '
+	flux terminus kill-server
+'
+# list w/ backoff waiting for server to exit
+server_list_thrice() {
+	flux terminus list &&
+	sleep 0.25 &&
+	flux terminus list &&
+	sleep 1 &&
+	flux terminus list
+}
+test_expect_success 'flux-terminus: basic start, server exits after session' '
+	$runpty flux terminus start sleep 0 &&
+	test_expect_code 1 server_list_thrice
+'
+test_expect_success 'flux-terminus: attach reports exit status' '
+	flux terminus start -w -d true &&
+	$runpty flux terminus attach 0 &&
+	flux terminus start -w -d sh -c "exit 7" &&
+	test_expect_code 7 $runpty flux terminus attach 0 &&
+	flux terminus start -w -d sh -c "kill -INT \$$" &&
+	test_expect_code 130 $runpty flux terminus attach 0
+'
+# N.B.: We use !wait $pid below because we expect pid to have been
+#  stopped by SIGKILL, something neither test_must_fail() nor
+#  test_expect_code() handles.
+#
+test_expect_success NO_CHAIN_LINT 'flux-terminus: start, try a resize' '
+	$runpty -o log.resize flux terminus start sleep 1000 &
+	pid=$! &&
+	$waitfile -t 20 -v -p \"o\" log.resize &&
+	test_debug "echo pid=$pid" &&
+	kill -WINCH $pid &&
+	flux terminus kill 0 &&
+	test_debug "cat log.resize" &&
+	! wait $pid
+'
+test_expect_success NO_CHAIN_LINT 'flux-terminus: detach works' '
+	$runpty -o log.detach flux terminus start -n test-detach &
+	pid=$! &&
+	$waitfile -t 20 -v -p \"o\" log.detach &&
+	kill -USR1 $pid &&
+	$waitfile -t 20 -v -p detached log.detach &&
+	wait $pid &&
+	flux terminus list >detach.list 2>&1 &&
+	test_debug "cat detach.list" &&
+	grep "test-detach.*0 clients" detach.list
+'
+test_expect_success NO_CHAIN_LINT 'flux-terminus: reattach' '
+	$runpty -o log.reattach flux terminus attach 0 &
+	pid=$! &&
+	$waitfile -t 20 -v -p \"o\" log.reattach &&
+	flux terminus list >reattach.list 2>&1 &&
+	test_debug "cat reattach.list" &&
+	grep -q "1 client" reattach.list
+	flux terminus kill 0 &&
+	! wait $pid
+'
+test_expect_success NO_CHAIN_LINT 'flux-terminus: copy stdin' '
+	$runpty -o log.pipe-stdin flux terminus start &
+	pid=$! &&
+	$waitfile -t 20 -v -p \"o\" log.pipe-stdin &&
+	printf "echo hello\r" | flux terminus attach -p 0 &&
+	$waitfile -t 20 -v -p hello log.pipe-stdin &&
+	printf "exit\r" | flux terminus attach -p 0 &&
+	$waitfile -t 20 -v -p detached log.pipe-stdin &&
+	wait $pid
+'
+test_expect_success NO_CHAIN_LINT 'flux-terminus: disconnect works' '
+	flux terminus kill-server
+	$runpty -o log.disconnect flux terminus start &
+	pid=$! &&
+	$waitfile -t 20 -v -p \"o\" log.disconnect &&
+	kill -TERM $pid &&
+	test_expect_code 143 wait $pid &&
+	flux terminus list >disconnect.list &&
+	test_debug "cat disconnect.list" &&
+	grep -q "0 clients" disconnect.list &&
+	flux terminus kill 0
+'
+test_expect_success 'flux-terminus: nesting not allowed' '
+	test_expect_code 1 \
+	    $runpty flux terminus start \
+	    flux terminus start true
+'
+test_expect_success 'flux-terminus: requests from invalid userid are rejected' '
+	flux terminus start -d &&
+	( SERVICE="$(id -u)-terminus" &&
+          export FLUX_HANDLE_USERID=$(($(id -u) + 1)) &&
+	  export FLUX_HANDLE_ROLEMASK=0x2 &&
+	  test_expect_code 1 flux terminus attach -s $SERVICE 0 &&
+	  test_expect_code 1 flux terminus list -s $SERVICE &&
+	  test_expect_code 1 flux terminus start -s $SERVICE &&
+	  test_expect_code 1 flux terminus kill -s $SERVICE 0 &&
+	  test_expect_code 1 flux terminus kill-server -s $SERVICE
+	)
+'
+test_done
diff --git a/t/t0021-archive-cmd.t b/t/t0021-archive-cmd.t
new file mode 100755
index 000000000000..5c3574a7c4bc
--- /dev/null
+++ b/t/t0021-archive-cmd.t
@@ -0,0 +1,190 @@
+#!/bin/sh
+
+test_description='Test flux-archive'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 2
+
+# after test_under_flux is launched, cannot assume what umask is.  An
+# unexpected umask could affect tests below.  Hard code to 022 for
+# these tests.
+umask 022
+
+# Usage: randbytes bytes
+randbytes() {
+	dd if=/dev/urandom bs=$1 count=1
+}
+
+test_expect_success 'flux archive create --badopt prints unrecognized option' '
+	test_must_fail flux archive create --badopt 2>badopt.err &&
+	grep "unrecognized option" badopt.err
+'
+test_expect_success 'flux archive create with no PATHs fails' '
+	test_must_fail flux archive create
+'
+test_expect_success 'flux archive create with bad FLUX_URI fails' '
+	(FLUX_URI=local:///noexist test_must_fail flux archive create .)
+'
+test_expect_success 'flux archive create fails with --overwrite --append' '
+	test_must_fail flux archive create --overwrite --append .
+'
+test_expect_success 'flux archive create fails with -C baddir bad ' '
+	test_must_fail flux archive create -C baddir .
+'
+test_expect_success 'flux archive create fails with FIFO in input' '
+	mkfifo imafifo &&
+	test_must_fail flux archive create imafifo
+'
+test_expect_success 'flux archive remove fails if archive doesnt exist' '
+	test_must_fail flux archive remove
+'
+test_expect_success 'but it works with -f' '
+	flux archive remove -f
+'
+# Set the small file threshold to 1K in these tests.
+# It's currently the default but just in case that changes.
+test_expect_success 'flux archive create works (small file)' '
+	randbytes 128 >testfile &&
+	flux archive create --small-file-threshold=1K -v testfile &&
+	flux kvs get archive.main >archive.out
+'
+test_expect_success 'archive.main contains a base64-encoded file' '
+	jq -e -r <archive.out ".[0].encoding == \"base64\""
+'
+test_expect_success 'flux archive create fails if archive exists' '
+	test_must_fail flux archive create testfile
+'
+test_expect_success 'flux archive create --overwrite works (large file)' '
+	randbytes 2048 >testfile2 &&
+	flux archive create --overwrite --preserve \
+		--small-file-threshold=1K -v testfile2 &&
+	flux kvs get archive.main >archive2.out
+'
+test_expect_success 'and archive.main contains a blobvec-encoded file' '
+	jq -e -r <archive2.out ".[0].encoding == \"blobvec\""
+'
+test_expect_success 'and archive.main_blobs exists due to --preserve' '
+	flux kvs ls archive.main_blobs
+'
+test_expect_success 'flux archive create --append works (directory)' '
+	mkdir -p testdir &&
+	flux archive create --append testdir &&
+	flux kvs get archive.main >archive3.out
+'
+test_expect_success 'and archive.main added an entry' '
+	jq -e -r <archive3.out ".[1].path == \"testdir\""
+'
+test_expect_success 'flux archive list works' '
+	flux archive list >list.out &&
+	cat >list.exp <<-EOT &&
+	testfile2
+	testdir
+	EOT
+	test_cmp list.exp list.out
+'
+test_expect_success 'flux archive list -l works' '
+	flux archive list -l >list_l.out &&
+	cat >list_l.exp <<-EOT &&
+	f 0644     2048 testfile2
+	d 0755        0 testdir
+	EOT
+	test_cmp list_l.exp list_l.out
+'
+test_expect_success 'flux archive list with matching pattern works' '
+	flux archive list testdir
+'
+test_expect_success 'flux archive list with non-matching pattern works' '
+	flux archive list notinthere
+'
+test_expect_success 'flux archive extract -C baddir fails' '
+	test_must_fail flux archive extract -C baddir
+'
+test_expect_success 'flux archive extract -n main -C gooddir works' '
+	mkdir -p gooddir &&
+	flux archive extract -n main -v -C gooddir
+'
+test_expect_success 'goodir/testfile2 was extracted faithfully' '
+	test_cmp testfile2 gooddir/testfile2
+'
+test_expect_success 'goodir/testdir was extracted as a directory' '
+	test -d gooddir/testdir
+'
+test_expect_success 'flux archive extract works with a non-matching pattern' '
+	flux archive extract nomatch
+'
+test_expect_success 'flux archive extract works with a matching pattern' '
+	mkdir -p gooddir2 &&
+	flux archive extract -C gooddir2 testfile2 &&
+	test_cmp testfile2 gooddir2/testfile2
+'
+test_expect_success 'flux archive remove works' '
+	flux archive remove
+'
+test_expect_success 'and KVS keys are gone' '
+	test_must_fail flux kvs ls archive.main_blobs &&
+	test_must_fail flux kvs ls archive.main
+'
+test_expect_success 'flux archive list fails on nonexistent archive' '
+	test_must_fail flux archive list &&
+	test_must_fail flux archive list -n noexist
+'
+test_expect_success 'flux archive extract fails on nonexistent archive' '
+	test_must_fail flux archive extract &&
+	test_must_fail flux archive extract -n noexist
+'
+# This works because we use the primary namespace by default
+test_expect_success 'create an archive, then access it from a job' '
+	flux archive create testfile &&
+	flux run --label-io -N2 flux archive list
+'
+# but --no-force-primary allows FLUX_KVS_NAMESPACE to be used
+test_expect_success 'archive cannot be accessed from job with --no-force-primary' '
+	test_must_fail flux run flux archive list --no-force-primary
+'
+# both producer and consumer use --no-force-primary within a job
+test_expect_success 'copy private archive from rank 0 to 1 of a job' '
+	cat >job.sh <<-EOT &&
+	#!/bin/sh
+	opts="--no-force-primary -n mystuff"
+	if test \$FLUX_TASK_RANK -eq 0; then
+	    flux archive create \$opts testfile2
+	else
+	    mkdir jobdir
+	    flux archive extract -v --waitcreate \$opts -C jobdir
+	fi
+	EOT
+	chmod +x job.sh &&
+	flux run --label-io -N2 ./job.sh >job.out 2>&1
+'
+# Note: the current primary ns archive contains testfile not testfile2
+test_expect_success 'output references the private archive not primary ns' '
+	cat >job.exp <<-EOT &&
+	1: testfile2
+	EOT
+	test_cmp job.exp job.out
+'
+test_expect_success 'jobdir/testfile2 was extracted faithfully' '
+	test_cmp testfile2 jobdir/testfile2
+'
+test_expect_success 'archive exists in job KVS guest directory' '
+	jobdir=$(flux job last | flux job id --to=kvs) &&
+	flux kvs get $jobdir.guest.archive.mystuff >/dev/null
+'
+test_expect_success 'remove main archive' '
+	flux archive remove
+'
+test_expect_success 'Create files for example 1 of flux-archive(1)' '
+	mkdir -p project/dataset1 &&
+	echo foo >project/dataset1/testfile &&
+	echo bar >project/dataset1/testfile2
+'
+test_expect_success 'Ensure example 1 of flux-archive(1) works' '
+	flux archive create -C project dataset1 &&
+	flux exec -r 1 mkdir -p tmp.project &&
+	flux exec -r 1 flux archive extract --waitcreate -C tmp.project &&
+	flux exec -r 1 rm -r tmp.project &&
+	flux archive remove
+'
+
+test_done
diff --git a/t/t0021-flux-jobspec.t b/t/t0021-flux-jobspec.t
deleted file mode 100755
index a627e46bee73..000000000000
--- a/t/t0021-flux-jobspec.t
+++ /dev/null
@@ -1,238 +0,0 @@
-#!/bin/sh
-
-test_description='Test the flux-jobspec command'
-
-. `dirname $0`/sharness.sh
-
-JOBSPEC=${SHARNESS_TEST_SRCDIR}/jobspec
-VALIDATE="flux python ${JOBSPEC}/validate.py"
-SCHEMA=${FLUX_SOURCE_DIR}/src/modules/job-ingest/schemas/jobspec.jsonschema
-MINI_SCHEMA=${FLUX_SOURCE_DIR}/src/modules/job-ingest/schemas/jobspec_v1.jsonschema
-SUMMARIZE="flux python ${JOBSPEC}/summarize-minimal-jobspec.py"
-
-#  Set path to jq
-#
-jq=$(which jq 2>/dev/null)
-test -n "$jq" && test_set_prereq HAVE_JQ
-
-validate_emission() {
-    flux jobspec $@ | ${VALIDATE} --schema ${SCHEMA}
-}
-
-validate_minimal_emission() {
-    flux jobspec $@ | ${VALIDATE} --schema ${MINI_SCHEMA}
-}
-
-test_expect_success 'flux-jobspec srun with no args emits valid canonical jobspec' '
-    validate_emission srun sleep 1
-'
-
-test_expect_success 'flux-jobspec srun with no args emits minimal jobspec' '
-    validate_minimal_emission srun sleep 1
-'
-
-test_expect_success 'flux-jobspec srun with just num_tasks emits valid canonical jobspec' '
-    validate_emission srun -n4 sleep 1
-'
-
-test_expect_success 'flux-jobspec srun with just num_tasks emits minimal jobspec' '
-    validate_minimal_emission srun -n4 sleep 1
-'
-
-test_expect_success 'flux-jobspec srun with just cores_per_task emits valid canonical jobspec' '
-    validate_emission srun -c4 sleep 1
-'
-
-test_expect_success 'flux-jobspec srun with just cores_per_task emits minimal jobspec' '
-    validate_minimal_emission srun -c4 sleep 1
-'
-
-test_expect_success 'flux-jobspec srun without num_nodes emits valid canonical jobspec' '
-    validate_emission srun -n4 -c1 sleep 1
-'
-
-test_expect_success 'flux-jobspec srun without num_nodes emits minimal jobspec' '
-    validate_minimal_emission srun -n4 -c1 sleep 1
-'
-
-test_expect_success 'flux-jobspec srun with all args emits valid canonical jobspec' '
-    validate_emission srun -N4 -n4 -c1 sleep 1
-'
-
-test_expect_success 'flux-jobspec srun with all args emits minimal jobspec' '
-    validate_minimal_emission srun -N4 -n4 -c1 sleep 1
-'
-
-test_expect_success HAVE_JQ 'ensure flux-jobspec does not gobble application arguments' '
-    flux jobspec srun myApp -c1 -n1 -N1 | $jq ".tasks[0].command" > myApp.out &&
-    cat <<-EOF | $jq . > myApp.expected &&
-    [ "myApp", "-c1", "-n1", "-N1" ]
-	EOF
-    test_cmp myApp.expected myApp.out
-'
-
-test_expect_success 'an improperly formatted time string should produce an error' '
-    test_must_fail flux jobspec srun -t foo myApp 2>&1 | 
-    grep -q "flux-jobspec: ERROR: invalid time limit string format"
-'
-
-test_expect_success 'requesting more nodes than tasks should produce an error' '
-   test_must_fail flux jobspec srun -N8 -n2 hostname 2>&1 |
-   grep -q "Number of nodes greater than the number of tasks"
-'
-
-test_expect_success HAVE_JQ 'specifying only -N8 should produce (in total) 8 nodes, 8 tasks, 8 cores' '
-    flux jobspec srun -N8 hostname > only-nodes.jobspec.json &&
-    ${VALIDATE} -s ${MINI_SCHEMA} only-nodes.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} only-nodes.jobspec.json &&
-    ${SUMMARIZE} -j only-nodes.jobspec.json > only-nodes.summary.json &&
-    jq -e ".total_num_nodes == 8 and .total_num_tasks == 8 and .total_num_cores == 8" only-nodes.summary.json
-'
-
-test_expect_success HAVE_JQ 'specifying only -n8 should produce (in total) 0 nodes, 8 tasks, 8 cores' '
-    flux jobspec srun -n8 hostname > only-tasks.jobspec.json &&
-    ${VALIDATE} -s ${MINI_SCHEMA} only-tasks.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} only-tasks.jobspec.json &&
-    ${SUMMARIZE} -j only-tasks.jobspec.json > only-tasks.summary.json &&
-    jq -e ".total_num_nodes == 0 and .total_num_tasks == 8 and .total_num_cores == 8" only-tasks.summary.json
-'
-
-test_expect_success HAVE_JQ 'specifying only -c8 should produce (in total) 0 nodes, 1 task, 8 cores' '
-    flux jobspec srun -c8 hostname > only-cores.jobspec.json &&
-    ${VALIDATE} -s ${MINI_SCHEMA} only-cores.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} only-cores.jobspec.json &&
-    ${SUMMARIZE} -j only-cores.jobspec.json > only-cores.summary.json &&
-    jq -e ".total_num_nodes == 0 and .total_num_tasks == 1 and .total_num_cores == 8" only-cores.summary.json
-'
-
-test_expect_success HAVE_JQ 'specifying -N8 -c2 should produce (in total) 8 nodes, 8 tasks, 16 cores' '
-    flux jobspec srun -N8 -c2 hostname > nodes-cores.jobspec.json &&
-    ${VALIDATE} -s ${MINI_SCHEMA} nodes-cores.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} nodes-cores.jobspec.json &&
-    ${SUMMARIZE} -j nodes-cores.jobspec.json > nodes-cores.summary.json &&
-    jq -e ".total_num_nodes == 8 and .total_num_tasks == 8 and .total_num_cores == 16" nodes-cores.summary.json
-'
-
-test_expect_success HAVE_JQ 'specifying -N8 -n16 should produce (in total) 8 nodes, 16 tasks, 16 cores' '
-    flux jobspec srun -N8 -n16 hostname > nodes-tasks.jobspec.json &&
-    ${VALIDATE} -s ${MINI_SCHEMA} nodes-tasks.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} nodes-tasks.jobspec.json &&
-    ${SUMMARIZE} -j nodes-tasks.jobspec.json > nodes-tasks.summary.json &&
-    jq -e ".total_num_nodes == 8 and .total_num_tasks == 16 and .total_num_cores == 16" nodes-tasks.summary.json
-'
-
-test_expect_success HAVE_JQ 'specifying -N8 -n16 should produce tasks.count.per_slot == 1 and no warning' '
-    flux jobspec srun -N8 -n16 hostname > nodes-tasks-2.jobspec.json 2> nodes-tasks-2.err &&
-    jq -e ".tasks[0].count.per_slot == 1" nodes-tasks-2.jobspec.json &&
-    [ -f nodes-tasks-2.err ] && [ ! -s nodes-tasks-2.err ]
-'
-
-test_expect_success HAVE_JQ 'specifying -n16 -c4 should produce (in total) 0 nodes, 16 tasks, 64 cores' '
-    flux jobspec srun -n16 -c4 hostname > tasks-cores.jobspec.json &&
-    ${VALIDATE} -s ${MINI_SCHEMA} tasks-cores.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} tasks-cores.jobspec.json &&
-    ${SUMMARIZE} -j tasks-cores.jobspec.json > tasks-cores.summary.json &&
-    jq -e ".total_num_nodes == 0 and .total_num_tasks == 16 and .total_num_cores == 64" tasks-cores.summary.json
-'
-
-test_expect_success HAVE_JQ 'specifying -N8 -n16 -c4 should produce (in total) 8 nodes, 16 tasks, 64 cores' '
-    flux jobspec srun -N8 -n16 -c4 hostname > all-three-1.jobspec.json &&
-    ${VALIDATE} -s ${MINI_SCHEMA} all-three-1.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} all-three-1.jobspec.json &&
-    ${SUMMARIZE} -j all-three-1.jobspec.json > all-three-1.summary.json &&
-    jq -e ".total_num_nodes == 8 and .total_num_tasks == 16 and .total_num_cores == 64" all-three-1.summary.json
-'
-
-test_expect_success HAVE_JQ 'specifying -N9 -n9 -c2 should produce (in total) 9 nodes, 9 tasks, 18 cores' '
-    flux jobspec srun -N9 -n9 -c2 hostname > all-three-2.jobspec.json &&
-    ${VALIDATE} -s ${MINI_SCHEMA} all-three-2.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} all-three-2.jobspec.json &&
-    ${SUMMARIZE} -j all-three-2.jobspec.json > all-three-2.summary.json &&
-    jq -e ".total_num_nodes == 9 and .total_num_tasks == 9 and .total_num_cores == 18" all-three-2.summary.json
-'
-
-test_expect_success HAVE_JQ 'specifying -N8 -n9 -c1 should produce (in total) 8 nodes, 9 tasks, 16 cores and a warning' '
-    flux jobspec srun -N8 -n9 -c1 hostname > all-three-3.jobspec.json 2> all-three-3.warning.err &&
-    ${VALIDATE} -s ${MINI_SCHEMA} all-three-3.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} all-three-3.jobspec.json &&
-    ${SUMMARIZE} -j all-three-3.jobspec.json > all-three-3.summary.json &&
-    jq -e ".total_num_nodes == 8 and .total_num_tasks == 9 and .total_num_cores == 16" all-three-3.summary.json &&
-    grep -q "Number of tasks is not an integer multiple of the number of nodes." all-three-3.warning.err
-'
-
-test_expect_success HAVE_JQ 'specifying -N8 -n9 -c2 should produce (in total) 8 nodes, 9 tasks, 32 cores and a warning' '
-    flux jobspec srun -N8 -n9 -c2 hostname > all-three-4.jobspec.json 2> all-three-4.warning.err &&
-    ${VALIDATE} -s ${MINI_SCHEMA} all-three-4.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} all-three-4.jobspec.json &&
-    ${SUMMARIZE} -j all-three-4.jobspec.json > all-three-4.summary.json &&
-    jq -e ".total_num_nodes == 8 and .total_num_tasks == 9 and .total_num_cores == 32" all-three-4.summary.json &&
-    grep -q "Number of tasks is not an integer multiple of the number of nodes." all-three-4.warning.err
-'
-
-test_expect_success HAVE_JQ 'specifying -N8 -n25 -c2 should produce (in total) 8 nodes, 25 tasks, 64 cores and a warning' '
-    flux jobspec srun -N8 -n25 -c2 hostname > all-three-5.jobspec.json 2> all-three-5.warning.err &&
-    ${VALIDATE} -s ${MINI_SCHEMA} all-three-5.jobspec.json &&
-    ${VALIDATE} -s ${SCHEMA} all-three-5.jobspec.json &&
-    ${SUMMARIZE} -j all-three-5.jobspec.json > all-three-5.summary.json &&
-    jq -e ".total_num_nodes == 8 and .total_num_tasks == 25 and .total_num_cores == 64" all-three-5.summary.json &&
-    grep -q "Number of tasks is not an integer multiple of the number of nodes." all-three-5.warning.err
-'
-
-test_expect_success HAVE_JQ 'current working directory encoded in jobspec' '
-    flux jobspec srun hostname | jq -e ".attributes.system.cwd == \"$(pwd)\""
-'
-
-test_expect_success HAVE_JQ 'current environment encoded in jobspec' '
-    flux python -c "import os,json; print (json.dumps(dict(os.environ)))" | \
-        jq -S . >environ.expected &&
-    flux jobspec srun hostname | \
-        jq -S -e ".attributes.system.environment" > environ.output &&
-    flux jobspec srun --export=ALL hostname | \
-        jq -S -e ".attributes.system.environment" > environ.output2 &&
-    test_cmp environ.expected environ.output &&
-    test_cmp environ.expected environ.output2
-'
-
-test_expect_success HAVE_JQ 'jobspec srun --export=NONE works' '
-    flux jobspec srun --export=NONE hostname | \
-	jq -e "(.attributes.system.environment | length) == 0"
-'
-
-test_expect_success HAVE_JQ 'jobspec srun --export=VAR works' '
-    FOO=bar flux jobspec srun --export=FOO hostname | \
-        jq -e "(.attributes.system.environment | length) == 1 and \
-               .attributes.system.environment.FOO == \"bar\""
-'
-
-test_expect_success HAVE_JQ 'jobspec srun --export=VAR=val works' '
-    FOO=bar flux jobspec srun --export=FOO=baz hostname | \
-        jq -e "(.attributes.system.environment | length) == 1 and \
-               .attributes.system.environment.FOO == \"baz\""
-'
-
-test_expect_success HAVE_JQ 'jobspec srun --export=VAR,VAR2,... works' '
-    FOO=bar BAR=foo flux jobspec srun --export=FOO,BAR hostname | \
-        jq -e "(.attributes.system.environment | length) == 2 and \
-               .attributes.system.environment.FOO == \"bar\" and \
-               .attributes.system.environment.BAR == \"foo\"" &&
-    FOO=bar BAR=foo flux jobspec srun --export=FOO --export=BAR hostname | \
-        jq -e "(.attributes.system.environment | length) == 2 and \
-               .attributes.system.environment.FOO == \"bar\" and \
-               .attributes.system.environment.BAR == \"foo\""
-'
-
-test_expect_success HAVE_JQ 'jobspec srun --export=ALL,VAR=val works' '
-    FLOOP=boop \
-      flux python -c "import os,json; print (json.dumps(dict(os.environ)))" | \
-        jq -S . >floop.expected &&
-    flux jobspec srun --export=ALL,FLOOP=boop hostname | \
-        jq -S ".attributes.system.environment" > floop.output &&
-    test_cmp floop.expected floop.output
-'
-
-test_expect_success HAVE_JQ 'jobspec srun --export fails for nonexistent var' '
-    unset SOUP &&
-    test_must_fail flux jobspec srun --export=SOUP hostname
-'
-
-test_done
diff --git a/t/t0022-jj-reader.t b/t/t0022-jj-reader.t
index 3c6afc9ba9d4..46f4f6c686ab 100755
--- a/t/t0022-jj-reader.t
+++ b/t/t0022-jj-reader.t
@@ -4,59 +4,57 @@ test_description='Test json-jobspec *cough* parser *cough*'
 
 . `dirname $0`/sharness.sh
 
-#  Set path to jq
-#
-jq=$(which jq 2>/dev/null)
-test -n "$jq" && test_set_prereq HAVE_JQ
 jj=${FLUX_BUILD_DIR}/t/sched-simple/jj-reader
 y2j="flux python ${SHARNESS_TEST_SRCDIR}/jobspec/y2j.py"
 
-test_expect_success HAVE_JQ 'jj-reader: unexpected version throws error' '
-	flux jobspec srun hostname | jq ".version = 2" >input.$test_count &&
+test_expect_success 'jj-reader: unexpected version throws error' '
+	flux run --dry-run hostname \
+		| jq ".version = 2" >input.$test_count &&
 	test_expect_code 1 $jj<input.$test_count >out.$test_count 2>&1 &&
 	cat >expected.$test_count <<-EOF &&
 	jj-reader: Invalid version: expected 1, got 2
 	EOF
 	test_cmp expected.$test_count out.$test_count
 '
-test_expect_success HAVE_JQ 'jj-reader: no version throws error' '
-	flux jobspec srun hostname | jq "del(.version)" >input.$test_count &&
+test_expect_success 'jj-reader: no version throws error' '
+	flux run --dry-run hostname \
+		| jq "del(.version)" >input.$test_count &&
 	test_expect_code 1 $jj<input.$test_count >out.$test_count 2>&1 &&
 	cat >expected.$test_count <<-EOF &&
 	jj-reader: at top level: Object item not found: version
 	EOF
 	test_cmp expected.$test_count out.$test_count
 '
-test_expect_success HAVE_JQ 'jj-reader: bad count throws error' '
-	flux jobspec srun hostname | \
-	  jq ".resources[0].with[0].count = -1" >input.$test_count &&
+test_expect_success 'jj-reader: bad count throws error' '
+	flux run --dry-run hostname | \
+		jq ".resources[0].with[0].count = -1" >input.$test_count &&
 	test_expect_code 1 $jj<input.$test_count >out.$test_count 2>&1 &&
 	cat >expected.$test_count <<-EOF &&
 	jj-reader: Invalid count -1 for type '\''core'\''
 	EOF
 	test_cmp expected.$test_count out.$test_count
 '
-test_expect_success HAVE_JQ 'jj-reader: bad type throws error' '
-	flux jobspec srun hostname | \
-	  jq --arg f beans ".resources[0].type = \$f" >input.$test_count &&
+test_expect_success 'jj-reader: bad type throws error' '
+	flux run --dry-run hostname | \
+		jq --arg f beans ".resources[0].type = \$f" >input.$test_count &&
 	test_expect_code 1 $jj<input.$test_count >out.$test_count 2>&1 &&
 	cat >expected.$test_count <<-EOF &&
 	jj-reader: Unsupported resource type '\''beans'\''
 	EOF
 	test_cmp expected.$test_count out.$test_count
 '
-test_expect_success HAVE_JQ 'jj-reader: missing count throws error' '
-	flux jobspec srun hostname | \
-	  jq "del(.resources[0].with[0].count)" >input.$test_count &&
+test_expect_success 'jj-reader: missing count throws error' '
+	flux run --dry-run hostname | \
+		jq "del(.resources[0].with[0].count)" >input.$test_count &&
 	test_expect_code 1 $jj<input.$test_count >out.$test_count 2>&1 &&
 	cat >expected.$test_count <<-EOF &&
 	jj-reader: level 1: Object item not found: count
 	EOF
 	test_cmp expected.$test_count out.$test_count
 '
-test_expect_success HAVE_JQ 'jj-reader: wrong count type throws error' '
-	flux jobspec srun hostname | \
-	  jq ".resources[0].with[0].count = 1.5" >input.$test_count &&
+test_expect_success 'jj-reader: wrong count type throws error' '
+	flux run --dry-run hostname | \
+		jq ".resources[0].with[0].count = 1.5" >input.$test_count &&
 	test_expect_code 1 $jj<input.$test_count >out.$test_count 2>&1 &&
 	cat >expected.$test_count <<-EOF &&
 	jj-reader: level 1: Expected integer, got real
@@ -68,7 +66,7 @@ test_expect_success HAVE_JQ 'jj-reader: wrong count type throws error' '
 # jobspec.yaml ==<expected error>
 #
 cat <<EOF>invalid.txt
-jobspec/valid/basic.yaml        ==jj-reader: Unable to determine slot size
+jobspec/valid/basic.yaml        ==jj-reader: at top level: getting duration: Object item not found: system
 jobspec/valid/example2.yaml     ==jj-reader: Unable to determine slot size
 jobspec/valid/use_case_1.2.yaml ==jj-reader: level 0: Expected integer, got object
 jobspec/valid/use_case_1.3.yaml ==jj-reader: level 2: Expected integer, got object
@@ -89,15 +87,15 @@ EOF
 # of tests to retain that coverage.  --Jim G.
 #
 while read line; do
-  yaml=$(echo $line | awk -F== '{print $1}' | sed 's/  *$//')
-  expected=$(echo $line | awk -F== '{print $2}')
-
-  test_expect_success "jj-reader: $(basename $yaml) gets expected error" '
-	echo $expected >expected.$test_count &&
-	sed -e 's/999/1/' $SHARNESS_TEST_SRCDIR/$yaml |$y2j >$test_count.json &&
-	test_expect_code 1 $jj<$test_count.json > out.$test_count 2>&1 &&
-	test_cmp expected.$test_count out.$test_count
-  '
+	yaml=$(echo $line | awk -F== '{print $1}' | sed 's/  *$//')
+	expected=$(echo $line | awk -F== '{print $2}')
+
+	test_expect_success "jj-reader: $(basename $yaml) gets expected error" '
+		echo $expected >expected.$test_count &&
+		sed -e 's/999/1/' $SHARNESS_TEST_SRCDIR/$yaml |$y2j >$test_count.json &&
+		test_expect_code 1 $jj<$test_count.json > out.$test_count 2>&1 &&
+		test_cmp expected.$test_count out.$test_count
+	'
 done <invalid.txt
 
 
@@ -105,48 +103,32 @@ done <invalid.txt
 # <jobspec command args> == <expected result>
 #
 cat <<EOF >inputs.txt
-srun              ==nnodes=0 nslots=1 slot_size=1
-srun -N1          ==nnodes=1 nslots=1 slot_size=1
-srun -N1 -n4      ==nnodes=1 nslots=4 slot_size=1
-srun -N1 -n4 -c4  ==nnodes=1 nslots=4 slot_size=4
-srun -n4 -c4      ==nnodes=0 nslots=4 slot_size=4
-srun -n4 -c4      ==nnodes=0 nslots=4 slot_size=4
-srun -n4 -c1      ==nnodes=0 nslots=4 slot_size=1
-srun -N4 -n4 -c4  ==nnodes=4 nslots=4 slot_size=4
+run              ==nnodes=0 nslots=1 slot_size=1 slot_gpus=0 exclusive=false duration=0.0
+run -N1 -n1      ==nnodes=1 nslots=1 slot_size=1 slot_gpus=0 exclusive=false duration=0.0
+run -N1 -n4      ==nnodes=1 nslots=4 slot_size=1 slot_gpus=0 exclusive=false duration=0.0
+run -N1 -n4 -c4  ==nnodes=1 nslots=4 slot_size=4 slot_gpus=0 exclusive=false duration=0.0
+run -n4 -c4      ==nnodes=0 nslots=4 slot_size=4 slot_gpus=0 exclusive=false duration=0.0
+run -n4 -c4      ==nnodes=0 nslots=4 slot_size=4 slot_gpus=0 exclusive=false duration=0.0
+run -n4 -c1      ==nnodes=0 nslots=4 slot_size=1 slot_gpus=0 exclusive=false duration=0.0
+run -N4 -n4 -c4  ==nnodes=4 nslots=4 slot_size=4 slot_gpus=0 exclusive=false duration=0.0
+run -t 1m -N4 -n4 ==nnodes=4 nslots=4 slot_size=1 slot_gpus=0 exclusive=false duration=60.0
+run -t 5s -N4 -n4 ==nnodes=4 nslots=4 slot_size=1 slot_gpus=0 exclusive=false duration=5.0
+run -t 1h -N4 -n4 ==nnodes=4 nslots=4 slot_size=1 slot_gpus=0 exclusive=false duration=3600.0
+run -g1           ==nnodes=0 nslots=1 slot_size=1 slot_gpus=1 exclusive=false duration=0.0
+run -N1 -n2 -c2 -g1 ==nnodes=1 nslots=2 slot_size=2 slot_gpus=1 exclusive=false duration=0.0
 EOF
 
 while read line; do
 
-  args=$(echo $line | awk -F== '{print $1}' | sed 's/  *$//')
-  expected=$(echo $line | awk -F== '{print $2}')
+	args=$(echo $line | awk -F== '{print $1}' | sed 's/  *$//')
+	expected=$(echo $line | awk -F== '{print $2}')
 
-  test_expect_success "jj-reader: $args returns $expected" '
-	echo $expected >expected.$test_count &&
-	flux jobspec $args hostname | $jj > output.$test_count &&
-	test_cmp expected.$test_count output.$test_count
-  '
+	test_expect_success "jj-reader: $args returns $expected" '
+		echo $expected >expected.$test_count &&
+		flux $args --dry-run hostname | $jj > output.$test_count &&
+		test_cmp expected.$test_count output.$test_count
+	'
 done < inputs.txt
 
 
-# Invalid inputs:
-# <mini command args> == <expected result>
-#
-cat <<EOF >mini-invalid.txt
-run -g1             ==jj-reader: Unsupported resource type 'gpu'
-run -N1 -n2 -c2 -g1 ==jj-reader: Unsupported resource type 'gpu'
-EOF
-
-while read line; do
-  args=$(echo $line | awk -F== '{print $1}' | sed 's/  *$//')
-  expected=$(echo $line | awk -F== '{print $2}')
-
-  test_expect_success "jj-reader: flux mini $args gets expected error" '
-	echo $expected >expected.$test_count &&
-	flux mini $args --dry-run hostname >$test_count.json &&
-	test_expect_code 1 $jj<$test_count.json > out.$test_count 2>&1 &&
-	test_cmp expected.$test_count out.$test_count
-  '
-done <mini-invalid.txt
-
-
 test_done
diff --git a/t/t0023-jobspec1-validate.t b/t/t0023-jobspec1-validate.t
new file mode 100755
index 000000000000..0ddf0e4fa0f1
--- /dev/null
+++ b/t/t0023-jobspec1-validate.t
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+test_description='Test the libjob jobspec1 validation'
+
+. `dirname $0`/sharness.sh
+
+JOBSPEC=${SHARNESS_TEST_SRCDIR}/jobspec
+Y2J="flux python ${JOBSPEC}/y2j.py"
+VALIDATE1=${FLUX_BUILD_DIR}/t/util/jobspec1-validate
+
+# Emit a "prereq" string if test it to be skipped.
+skip() {
+	case $1 in
+		*invalid_yaml1.yaml)
+			echo NATIVE_YAML_PARSE
+			;;
+		*resource_count_missing_*.yaml)
+			echo RESOURCE_COUNT_OBJECT
+			;;
+	esac
+}
+
+validate_v1() {
+	$Y2J <$1 | ${VALIDATE1}
+}
+
+for jobspec in ${JOBSPEC}/valid_v1/*.yaml; do
+	testname=jobspec_v1/$(basename $jobspec)
+	test_expect_success $(skip $testname) \
+		'valid_v1: '$testname "validate_v1 $jobspec"
+done
+
+for jobspec in ${JOBSPEC}/invalid/*.yaml; do
+	testname=`basename $jobspec`
+	test_expect_success $(skip $testname) \
+		'invalid: '$testname "test_must_fail validate_v1 $jobspec"
+done
+
+for jobspec in ${JOBSPEC}/invalid_v1/*.yaml; do
+	testname=`basename $jobspec`
+	test_expect_success $(skip $testname) \
+		'invalid_v1: '$testname "test_must_fail validate_v1 $jobspec"
+done
+
+test_done
diff --git a/t/t0024-content-s3.t b/t/t0024-content-s3.t
new file mode 100755
index 000000000000..a6d58cc733fa
--- /dev/null
+++ b/t/t0024-content-s3.t
@@ -0,0 +1,382 @@
+#!/bin/sh
+
+test_description='Test content-s3 backing store service'
+
+. `dirname $0`/content/content-helper.sh
+
+. `dirname $0`/sharness.sh
+
+if test -z "$S3_HOSTNAME"; then
+	skip_all='skipping content-s3 test since S3_HOSTNAME not set'
+	test_done
+fi
+
+export FLUX_CONF_DIR=$(pwd)
+
+test_under_flux 1 minimal
+
+BLOBREF=${FLUX_BUILD_DIR}/t/kvs/blobref
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+
+SIZES="0 1 64 100 1000 1024 1025 8192 65536 262144 1048576 4194304"
+LARGE_SIZES="8388608 10000000 16777216 33554432 67108864"
+
+##
+# Functions used by tests
+##
+
+# Usage: backing_load <hash
+backing_load() {
+        $RPC -r -R content-backing.load
+}
+# Usage: backing_store <blob >hash
+backing_store() {
+        $RPC -r -R content-backing.store
+}
+# Usage: make_blob size >blob
+make_blob() {
+	if test $1 -eq 0; then
+		dd if=/dev/null 2>/dev/null
+	else
+		dd if=/dev/urandom count=1 bs=$1 2>/dev/null
+	fi
+}
+# Usage: check_blob size
+# Leaves behind blob.<size> and hash.<size>
+check_blob() {
+	make_blob $1 >blob.$1 &&
+	backing_store <blob.$1 >hash.$1 &&
+	backing_load <hash.$1 >blob.$1.check &&
+	test_cmp blob.$1 blob.$1.check
+}
+# Usage: check_blob size
+# Relies on existence of blob.<size> and hash.<size>
+recheck_blob() {
+	backing_load <hash.$1 >blob.$1.recheck &&
+	test_cmp blob.$1 blob.$1.recheck
+}
+# Usage: recheck_cache_blob size
+# Relies on existence of blob.<size>
+recheck_cache_blob() {
+	local blobref=$($BLOBREF sha1 <blob.$1)
+	flux content load $blobref >blob.$1.cachecheck &&
+	test_cmp blob.$1 blob.$1.cachecheck
+}
+
+test_expect_success 'load content module' '
+	flux module load content
+'
+
+##
+# Tests of the module by itself (no content cache)
+##
+
+test_expect_success 'content-s3 module load fails with unknown option' '
+	test_must_fail flux module load content-s3 notoption
+'
+test_expect_success 'content-s3 module load fails with truncate option' '
+	test_must_fail flux module load content-s3 truncate
+'
+
+test_expect_success 'create creds.toml from env' '
+	mkdir -p creds &&
+	cat >creds/creds.toml <<-CREDS
+	access-key-id = "$S3_ACCESS_KEY_ID"
+	secret-access-key = "$S3_SECRET_ACCESS_KEY"
+	CREDS
+'
+
+test_expect_success 'create content-s3.toml from env' '
+	cat >content-s3.toml <<-TOML
+	[content-s3]
+	credential-file = "$(pwd)/creds/creds.toml"
+	uri = "http://$S3_HOSTNAME"
+	bucket = "$S3_BUCKET"
+	virtual-host-style = false
+	TOML
+'
+
+test_expect_success 'reload broker config' '
+	flux config reload
+'
+
+test_expect_success 'load content-s3 module' '
+	flux module load content-s3
+'
+
+test_expect_success 'store/load/verify various size small blobs' '
+	err=0 &&
+	for size in $SIZES; do \
+		if ! check_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success LONGTEST 'store/load/verify various size large blobs' '
+	err=0 &&
+	for size in $LARGE_SIZES; do \
+		if ! check_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+
+test_expect_success 'checkpoint-put foo w/ rootref bar' '
+	checkpoint_put foo bar
+'
+
+test_expect_success 'checkpoint-get foo returned rootref bar' '
+        echo bar >rootref.exp &&
+        checkpoint_get foo | jq -r .value | jq -r .rootref >rootref.out &&
+        test_cmp rootref.exp rootref.out
+'
+
+# use grep instead of compare, incase of floating point rounding
+test_expect_success 'checkpoint-get foo returned correct timestamp' '
+        checkpoint_get foo | jq -r .value | jq -r .timestamp >timestamp.out &&
+        grep 2.2 timestamp.out
+'
+
+test_expect_success 'checkpoint-put updates foo rootref to baz' '
+        checkpoint_put foo baz
+'
+
+test_expect_success 'checkpoint-get foo returned rootref baz' '
+        echo baz >rootref2.exp &&
+        checkpoint_get foo | jq -r .value | jq -r .rootref >rootref2.out &&
+        test_cmp rootref2.exp rootref2.out
+'
+
+test_expect_success 'reload content-s3 module' '
+	flux module reload content-s3
+'
+
+test_expect_success 'reload/verify various size small blobs' '
+	err=0 &&
+	for size in $SIZES; do \
+		if ! recheck_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success LONGTEST 'reload/verify various size large blobs' '
+	err=0 &&
+	for size in $LARGE_SIZES; do \
+		if ! recheck_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success 'checkpoint-get foo still returns rootref baz' '
+        echo baz >rootref3.exp &&
+        checkpoint_get foo | jq -r .value | jq -r .rootref >rootref3.out &&
+        test_cmp rootref3.exp rootref3.out
+'
+
+test_expect_success 'checkpoint-backing-get foo returns rootref baz' '
+	echo baz >rootref_backing.exp &&
+	checkpoint_backing_get foo \
+	    | jq -r .value \
+	    | jq -r .rootref >rootref_backing.out &&
+	test_cmp rootref_backing.exp rootref_backing.out
+'
+
+test_expect_success 'checkpoint-backing-put foo w/ rootref boof' '
+	checkpoint_backing_put foo boof
+'
+
+test_expect_success 'checkpoint-get foo returned rootref boof' '
+	echo boof >rootref4.exp &&
+	checkpoint_get foo | jq -r .value | jq -r .rootref >rootref4.out &&
+	test_cmp rootref4.exp rootref4.out
+'
+
+test_expect_success 'config: reload config does not take effect immediately' '
+	flux config reload 2>/dev/null &&
+	flux dmesg | grep content-s3 | tail -1 >reload.log &&
+	grep "changes will not take effect until next flux restart" reload.log
+'
+
+test_expect_success 'save good config' '
+	cp content-s3.toml content-s3.save &&
+	cp creds/creds.toml creds/creds.save
+'
+
+test_expect_success 'config: reload with extra field fails' '
+	cp content-s3.save content-s3.toml &&
+	echo "extrafield = \"failure\"" >>content-s3.toml &&
+	test_must_fail flux config reload
+'
+
+test_expect_success 'config: reload with malformed uri fails' '
+	cp content-s3.save content-s3.toml &&
+	sed -i -e "s/uri =.*$/uri = \"baduri\"/" \
+		content-s3.toml &&
+	test_must_fail flux config reload
+'
+
+test_expect_success 'config: reload with bad credential path fails' '
+	cp content-s3.save content-s3.toml &&
+	sed -i -e "s/credential-file =.*$/credential-file = \"nocreds\"/" \
+		content-s3.toml &&
+	test_must_fail flux config reload
+'
+
+test_expect_success 'config: unload module' '
+	flux module remove content-s3
+'
+
+test_expect_success 'checkpoint-put foo w/ rootref spoon' '
+	checkpoint_put foo spoon
+'
+
+test_expect_success 'checkpoint-get foo returned rootref spoon' '
+	echo spoon >rootref5.exp &&
+	checkpoint_get foo | jq -r .value | jq -r .rootref >rootref5.out &&
+	test_cmp rootref5.exp rootref5.out
+'
+
+test_expect_success 'load content-s3 module on rank 0' '
+	cp content-s3.save content-s3.toml &&
+	cp creds/creds.save creds/creds.toml &&
+	flux config reload &&
+	flux module load content-s3
+'
+
+# arg1 - expected reference
+wait_checkpoint_flush() {
+	local expected=$1
+	local i=0
+	while checkpoint_backing_get foo \
+		| jq -r .value \
+		| jq -r .rootref > checkpointflush.out \
+	      && [ $i -lt 50 ]
+	do
+	    checkpoint=$(cat checkpointflush.out)
+	    if [ "${checkpoint}" = "${expected}" ]
+	    then
+		return 0
+	    fi
+	    sleep 0.1
+	    i=$((i + 1))
+	done
+	return 1
+}
+
+test_expect_success 'checkpoint-backing-get foo returns spoon' '
+	wait_checkpoint_flush spoon
+'
+
+test_expect_success 'config: unload module' '
+	flux module remove content-s3
+'
+
+test_expect_success 'config: module fails to load without config file' '
+	rm -f content-s3.toml &&
+	cp creds/creds.save creds/creds.toml &&
+	test_must_fail flux config reload
+	flux config reload &&
+	test_must_fail flux module load content-s3
+'
+
+test_expect_success 'config: module fails to load without credentials file' '
+	rm -f creds/creds.toml &&
+	cp content-s3.save content-s3.toml &&
+	test_must_fail flux config reload
+	flux config reload &&
+	test_must_fail flux module load content-s3
+'
+
+test_expect_success 'config: restore good config' '
+	mv -f content-s3.save content-s3.toml &&
+	mv -f creds/creds.save creds/creds.toml &&
+	flux config reload
+'
+
+test_expect_success 'config: load module' '
+	flux module load content-s3
+'
+
+##
+# Tests of the module acting as backing store for content cache
+##
+
+test_expect_success 'verify content.backing-module=content-s3' '
+        test "$(flux getattr content.backing-module)" = "content-s3"
+'
+
+test_expect_success 'reload/verify various size small blobs through cache' '
+	err=0 &&
+	for size in $SIZES; do \
+		if ! recheck_cache_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success LONGTEST 'reload/verify various size large blobs through cache' '
+	err=0 &&
+	for size in $LARGE_SIZES; do \
+		if ! recheck_cache_blob $size; then err=$(($err+1)); fi; \
+	done &&
+	test $err -eq 0
+'
+
+test_expect_success 'remove content-s3 module' '
+	flux module remove content-s3
+'
+
+##
+# Tests of kvs checkpointing
+##
+
+test_expect_success 'generate rc1/rc3 for content-s3 backing' '
+	cat >rc1-content-s3 <<EOF &&
+#!/bin/bash -e
+flux module load content
+flux module load content-s3
+flux module load kvs
+EOF
+	cat >rc3-content-s3 <<EOF &&
+#!/bin/bash -e
+flux module remove kvs
+flux module remove content-s3
+flux module remove content
+EOF
+        chmod +x rc1-content-s3 &&
+        chmod +x rc3-content-s3
+'
+
+test_expect_success 'run instance with content-s3 module loaded' '
+	flux start --setattr=broker.rc1_path=$(pwd)/rc1-content-s3 \
+                   --setattr=broker.rc3_path=$(pwd)/rc3-content-s3 \
+	           flux kvs put testkey=43
+'
+
+test_expect_success 're-run instance with content-s3 module loaded' '
+	flux start --setattr=broker.rc1_path=$(pwd)/rc1-content-s3 \
+                   --setattr=broker.rc3_path=$(pwd)/rc3-content-s3 \
+	           flux kvs get testkey >gets3.out
+'
+
+test_expect_success 'content from previous instance survived (s3)' '
+	echo 43 >gets3.exp &&
+	test_cmp gets3.exp gets3.out
+'
+
+test_expect_success 're-run instance, verify checkpoint date saved (s3)' '
+	flux start --setattr=broker.rc1_path=$(pwd)/rc1-content-s3 \
+                   --setattr=broker.rc3_path=$(pwd)/rc3-content-s3 \
+	           flux dmesg >dmesgs3.out
+'
+
+# just check for todays date, not time for obvious reasons
+test_expect_success 'verify date in flux logs (s3)' '
+	today=`date --iso-8601` &&
+	grep checkpoint dmesgs3.out | grep ${today}
+'
+
+test_expect_success 'remove content module' '
+	flux module remove content
+'
+
+test_done
diff --git a/t/t0025-broker-state-machine.t b/t/t0025-broker-state-machine.t
new file mode 100755
index 000000000000..150662a2ba50
--- /dev/null
+++ b/t/t0025-broker-state-machine.t
@@ -0,0 +1,337 @@
+#!/bin/sh
+#
+
+test_description='Test broker state machine'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. `dirname $0`/sharness.sh
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+SRPC=${FLUX_BUILD_DIR}/t/request/rpc_stream
+ARGS="-Sbroker.rc1_path= -Sbroker.rc3_path="
+GROUPSCMD="flux python ${SHARNESS_TEST_SRCDIR}/scripts/groups.py"
+
+test_expect_success 'quorum reached on instance with 1 TBON level' '
+	echo "0-2" >full1.exp &&
+	flux start -s3 ${ARGS} ${GROUPSCMD} get broker.online >full1.out &&
+	test_cmp full1.exp full1.out
+'
+
+test_expect_success 'quorum reached on instance with 2 TBON levels' '
+	echo "0-3" >full2.exp &&
+	flux start -s4 ${ARGS} ${GROUPSCMD} get broker.online >full2.out &&
+	test_cmp full2.exp full2.out
+'
+
+test_expect_success 'quorum reached on instance with 3 TBON levels' '
+	echo "0-7" >full3.exp &&
+	flux start -s8 ${ARGS} ${GROUPSCMD} get broker.online >full3.out &&
+	test_cmp full3.exp full3.out
+'
+
+test_expect_success 'broker.quorum can be set on the command line' '
+	flux start -s3 ${ARGS} -Sbroker.quorum=3 \
+		${GROUPSCMD} get broker.online >full1_explicit.out &&
+	test_cmp full1.exp full1_explicit.out
+'
+
+test_expect_success 'broker fails with malformed broker.quorum' '
+	test_must_fail flux start ${ARGS} \
+		-Sbroker.quorum=9-10 true 2>qmalformed.err &&
+	grep "Error parsing broker.quorum attribute" qmalformed.err
+'
+
+test_expect_success 'broker fails with broker.quorum that exceeds size' '
+	test_must_fail flux start ${ARGS} \
+		-Sbroker.quorum=99 true 2>qtoobig.err &&
+	grep "Error parsing broker.quorum attribute" qtoobig.err
+'
+test_expect_success 'broker.quorum can be 0 for compatibility' '
+	flux start ${ARGS} -Sbroker.quorum=0 true 2>compat1.err &&
+	grep assuming compat1.err
+'
+test_expect_success 'broker.quorum can be 0-1 (size=2) for compatibility' '
+	flux start -s2 ${ARGS} -Sbroker.quorum=0-1 true 2>compat2.err &&
+	grep assuming compat2.err
+'
+test_expect_success 'create rc1 that blocks on FIFO for rank != 0' '
+	cat <<-EOT >rc1_block &&
+	#!/bin/bash
+	rank=\$(flux getattr rank)
+	test \$rank -eq 0 || cat fifo
+	EOT
+	chmod +x rc1_block
+'
+
+test_expect_success 'create rc2 that unblocks FIFO' '
+	cat <<-EOT >rc2_unblock &&
+	#!/bin/bash
+	${GROUPSCMD} get broker.online
+	echo UNBLOCKED! >>fifo
+	EOT
+	chmod +x rc2_unblock
+'
+
+# Delay rank 1 so that we can check that initial program ran with only
+# rank 0 in RUN state.
+test_expect_success 'instance functions with late-joiner' '
+	echo "0" >late.exp &&
+	rm -f fifo &&
+	mkfifo fifo &&
+	run_timeout 60 \
+		flux start -s2 \
+		-Slog-stderr-level=6 \
+		-Sbroker.rc1_path="$(pwd)/rc1_block" \
+		-Sbroker.rc3_path= \
+		-Sbroker.quorum=1 \
+		$(pwd)/rc2_unblock >late.out &&
+	test_cmp late.exp late.out
+'
+
+test_expect_success 'quorum-get RPC fails on rank > 0' '
+	test_must_fail flux start -s2 ${ARGS} \
+		flux exec -r1 ${GROUPSCMD} get --rank 1 broker.online \
+		</dev/null 2>qm1.err &&
+	grep "only available on rank 0" qm1.err
+'
+
+test_expect_success 'monitor streaming RPC works' '
+	flux start ${ARGS} \
+		$SRPC state-machine.monitor state \
+		</dev/null >state.out &&
+	jq -cea .state state.out
+'
+
+test_expect_success 'create rc script that prints current state' '
+	cat <<-EOT >rc_getstate &&
+	#!/bin/bash
+	$RPC state-machine.monitor </dev/null | jq -cea .state >rc.out
+	EOT
+	chmod +x rc_getstate
+'
+
+test_expect_success 'monitor reports INIT(2) in rc1' '
+	echo 2 >rc1.exp &&
+	flux start \
+		-Sbroker.rc1_path=$(pwd)/rc_getstate \
+		-Sbroker.rc3_path= \
+		true &&
+	test_cmp rc1.exp rc.out
+'
+
+test_expect_success 'monitor reports RUN(4) in rc2' '
+	echo 4 >rc2.exp &&
+	flux start \
+		-Sbroker.rc1_path= \
+		-Sbroker.rc3_path= \
+		$(pwd)/rc_getstate &&
+	test_cmp rc2.exp rc.out
+'
+
+test_expect_success 'monitor reports CLEANUP(5) in cleanup script' '
+	echo 5 >cleanup.exp &&
+	flux start \
+		-Sbroker.rc1_path= \
+		-Sbroker.rc3_path= \
+		bash -c "echo $(pwd)/rc_getstate | flux admin cleanup-push" &&
+	test_cmp cleanup.exp rc.out
+'
+
+test_expect_success 'monitor reports FINALIZE(7) in rc3' '
+	echo 7 >rc3.exp &&
+	flux start \
+		-Sbroker.rc1_path= \
+		-Sbroker.rc3_path=$(pwd)/rc_getstate \
+		true &&
+	test_cmp rc3.exp rc.out
+'
+
+test_expect_success 'capture state transitions from size=1 instance' '
+	flux start ${ARGS} -Slog-filename=states.log true
+'
+
+test_expect_success 'all expected events and state transitions occurred' '
+	grep "start: none->join"			states.log &&
+	grep "parent-none: join->init"			states.log &&
+	grep "rc1-none: init->quorum"			states.log &&
+	grep "quorum-full: quorum->run"			states.log &&
+	grep "rc2-success: run->cleanup"		states.log &&
+	grep "cleanup-none: cleanup->shutdown"		states.log &&
+	grep "children-none: shutdown->finalize"	states.log &&
+	grep "rc3-none: finalize->goodbye"		states.log &&
+	grep "goodbye: goodbye->exit"			states.log
+'
+
+test_expect_success 'capture state transitions from size=2 instance' '
+	flux start ${ARGS} --test-size=2 \
+		-Slog-stderr-level=6 -Slog-stderr-mode=local \
+		true 2>states2.log
+'
+
+test_expect_success 'all expected events and state transitions occurred on rank 0' '
+	grep "\[0\]: start: none->join"				states2.log &&
+	grep "\[0\]: parent-none: join->init"			states2.log &&
+	grep "\[0\]: rc1-none: init->quorum"			states2.log &&
+	grep "\[0\]: quorum-full: quorum->run"			states2.log &&
+	grep "\[0\]: rc2-success: run->cleanup"			states2.log &&
+	grep "\[0\]: cleanup-none: cleanup->shutdown"		states2.log &&
+	grep "\[0\]: children-complete: shutdown->finalize"	states2.log &&
+	grep "\[0\]: rc3-none: finalize->goodbye"		states2.log &&
+	grep "\[0\]: goodbye: goodbye->exit"			states2.log
+'
+
+test_expect_success 'all expected events and state transitions occurred on rank 1' '
+	grep "\[1\]: start: none->join"				states2.log &&
+	grep "\[1\]: parent-ready: join->init"			states2.log &&
+	grep "\[1\]: rc1-none: init->quorum"			states2.log &&
+	grep "\[1\]: quorum-full: quorum->run"			states2.log &&
+	grep "\[1\]: shutdown: run->cleanup"			states2.log &&
+	grep "\[1\]: cleanup-none: cleanup->shutdown"		states2.log &&
+	grep "\[1\]: children-none: shutdown->finalize"	        states2.log &&
+	grep "\[1\]: rc3-none: finalize->goodbye"		states2.log &&
+	grep "\[1\]: goodbye: goodbye->exit"		        states2.log
+'
+
+test_expect_success 'capture state transitions from instance with rc1 failure' '
+	test_expect_code 1 flux start \
+	    -Slog-filename=states_rc1.log \
+	    -Sbroker.rc1_path=false \
+	    -Sbroker.rc3_path= \
+	    true
+'
+
+test_expect_success 'all expected events and state transitions occurred' '
+	grep "start: none->join"			states_rc1.log &&
+	grep "parent-none: join->init"			states_rc1.log &&
+	grep "rc1-fail: init->shutdown"			states_rc1.log &&
+	grep "children-none: shutdown->finalize"	states_rc1.log &&
+	grep "rc3-none: finalize->goodbye"		states_rc1.log &&
+	grep "goodbye: goodbye->exit"			states_rc1.log
+'
+
+test_expect_success 'capture state transitions from instance with rc2 failure' '
+	test_expect_code 1 flux start \
+	    -Slog-filename=states_rc2.log \
+	    ${ARGS} \
+	    false
+'
+
+test_expect_success 'all expected events and state transitions occurred' '
+	grep "start: none->join"			states_rc2.log &&
+	grep "parent-none: join->init"			states_rc2.log &&
+	grep "rc1-none: init->quorum"			states_rc2.log &&
+	grep "quorum-full: quorum->run"			states_rc2.log &&
+	grep "rc2-fail: run->cleanup"		        states_rc2.log &&
+	grep "cleanup-none: cleanup->shutdown"		states_rc2.log &&
+	grep "children-none: shutdown->finalize"	states_rc2.log &&
+	grep "rc3-none: finalize->goodbye"		states_rc2.log &&
+	grep "goodbye: goodbye->exit"			states_rc2.log
+'
+
+test_expect_success 'capture state transitions from instance with rc3 failure' '
+	test_expect_code 1 flux start \
+	    -Slog-filename=states_rc3.log \
+	    -Sbroker.rc1_path= \
+	    -Sbroker.rc3_path=false \
+	    true
+'
+
+test_expect_success 'all expected events and state transitions occurred' '
+	grep "start: none->join"			states_rc3.log &&
+	grep "parent-none: join->init"			states_rc3.log &&
+	grep "rc1-none: init->quorum"			states_rc3.log &&
+	grep "quorum-full: quorum->run"			states_rc3.log &&
+	grep "rc2-success: run->cleanup"		states_rc3.log &&
+	grep "cleanup-none: cleanup->shutdown"		states_rc3.log &&
+	grep "children-none: shutdown->finalize"	states_rc3.log &&
+	grep "rc3-fail: finalize->goodbye"		states_rc3.log &&
+	grep "goodbye: goodbye->exit"			states_rc3.log
+'
+
+test_expect_success 'instance rc1 failure exits with norestart code' '
+	test_expect_code 99 flux start \
+	    -Sbroker.exit-norestart=99 \
+	    -Sbroker.rc1_path=false \
+	    -Sbroker.rc3_path= \
+	    true
+'
+
+test_expect_success 'broker.quorum-timeout=none is accepted' '
+	flux start ${ARGS} -Sbroker.quorum-timeout=none true
+'
+
+test_expect_success 'broker.quorum-timeout=3h is accepted' '
+	flux start ${ARGS} -Sbroker.quorum-timeout=3h true
+'
+test_expect_success 'broker.quorum-timeout=x fails' '
+	test_must_fail flux start ${ARGS} -Sbroker.quorum-timeout=x true
+'
+test_expect_success 'create rc1 that sleeps for 2s on rank != 0' '
+	cat <<-EOT >rc1_sleep &&
+	#!/bin/bash
+	rank=\$(flux getattr rank)
+	test \$rank -eq 0 || sleep 2
+	EOT
+	chmod +x rc1_sleep
+'
+test_expect_success 'broker.quorum-timeout works' '
+	flux start -s2 ${ARGS} \
+		-Slog-filename=timeout.log \
+		-Sbroker.rc1_path="$(pwd)/rc1_sleep" \
+		-Sbroker.quorum-timeout=1s true
+'
+test_expect_success 'logs contain quorum delayed/reached messages' '
+	grep "quorum delayed" timeout.log &&
+	grep "quorum reached" timeout.log
+'
+
+test_expect_success 'broker.cleanup-timeout default is none' '
+	flux start ${ARGS} \
+		flux getattr broker.cleanup-timeout >cto.out &&
+	cat >cto.exp <<-EOT &&
+	none
+	EOT
+	test_cmp cto.exp cto.out
+'
+
+test_expect_success 'broker.cleanup-timeout=3h is accepted' '
+	flux start ${ARGS} \
+		-Sbroker.cleanup-timeout=3h \
+		flux getattr broker.cleanup-timeout >cto2.out &&
+	cat >cto2.exp <<-EOT &&
+	3h
+	EOT
+	test_cmp cto2.exp cto2.out
+'
+test_expect_success 'broker.cleanup-timeout=x fails' '
+	test_must_fail flux start ${ARGS} \
+		-Sbroker.cleanup-timeout=x true
+'
+test_expect_success 'create rc1 that hangs in cleanup' '
+	cat <<-EOT >rc1_cleanup &&
+	#!/bin/sh
+	rank=\$(flux getattr rank)
+	test \$rank -eq 0 || exit 0
+	echo "sleep 30" | flux admin cleanup-push
+	EOT
+	chmod +x rc1_cleanup
+'
+test_expect_success 'create initial program that SIGTERMs broker' '
+	cat <<-EOT >killbroker &&
+	#!/bin/sh
+	# Usage: killbroker signum sleepsec
+	kill -\$1 \$(flux getattr broker.pid)
+	sleep \$2
+	EOT
+	chmod +x killbroker
+'
+test_expect_success 'cleanup gets SIGHUP after broker.cleanup-timeout expires' '
+	test_expect_code 129 flux start -s2 ${ARGS} \
+		-Slog-filename=cleanup.log \
+		-Sbroker.rc1_path="$(pwd)/rc1_cleanup" \
+		-Sbroker.cleanup-timeout=1s \
+		./killbroker 15 60
+'
+
+test_done
diff --git a/t/t0026-flux-R.t b/t/t0026-flux-R.t
new file mode 100755
index 000000000000..06d80b14d27b
--- /dev/null
+++ b/t/t0026-flux-R.t
@@ -0,0 +1,415 @@
+#!/bin/sh
+
+test_description='Test flux-R front-end command'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. `dirname $0`/sharness.sh
+
+test_expect_success 'flux R fails on invalid R objects' '
+	test_expect_code 1 flux R decode </dev/null &&
+	echo {} | test_expect_code 1 flux R decode &&
+	echo "{\"version\": 1}" | test_expect_code 1 flux R decode &&
+	echo "{\"version\": 1, \"execution\": {}}" \
+	    | test_expect_code 1 flux R decode
+'
+test_expect_success 'flux R decode works with empty R_lite' '
+	echo "{\"version\": 1, \"execution\": {\"R_lite\": {}}}" \
+	    | flux R decode > R.empty &&
+	test $(flux R decode --count core < R.empty) -eq 0
+'
+test_expect_success 'flux R encode with no args creates expected result' '
+	flux R encode | flux R decode &&
+	test $(flux R encode | flux R decode --count node) -eq 1 &&
+	test $(flux R encode | flux R decode --count core) -eq 1 &&
+	test $(flux R encode | flux R decode --count gpu) -eq 0 &&
+	test "$(flux R encode | flux R decode --short)" = "rank0/core0"
+'
+test_expect_success 'flux R encode --ranks works' '
+	test $(flux R encode --ranks 0-1 | flux R decode --count node) -eq 2 &&
+	test $(flux R encode --ranks 0,2 | flux R decode --count node) -eq 2 &&
+	test $(flux R encode --ranks 0,2 | flux R decode --count core) -eq 2 &&
+	test $(flux R encode --ranks 0,2 | flux R decode --ranks) = "0,2"
+'
+test_expect_success 'flux R encode --cores works ' '
+	test $(flux R encode --cores 0-1 | flux R decode --count node) -eq 1 &&
+	test $(flux R encode --cores 0-1 | flux R decode --count core) -eq 2 &&
+	test $(flux R encode -c 0-1 -r 0-1 | flux R decode -c core) -eq 4
+'
+test_expect_success 'flux R encode --gpus works' '
+	test $(flux R encode --gpus 0 | flux R decode --count node) -eq 1 &&
+	test $(flux R encode --gpus 0 | flux R decode --count gpu) -eq 1 &&
+	test $(flux R encode --gpus 0 | flux R decode --count core) -eq 0 &&
+	test $(flux R encode --cores 0-1 -g 0 | flux R decode --count core) -eq 2 &&
+	test $(flux R encode --cores 0-1 -g 0 | flux R decode --count gpu) -eq 1
+'
+test_expect_success 'flux R encode --hosts works' '
+	test $(flux R encode --hosts=foo[0-1] | flux R decode -c node) -eq 2 &&
+	hosts=$(flux R encode --hosts=foo[0-1] | flux R decode --nodelist) &&
+	test "$hosts" = "foo[0-1]"
+'
+test_expect_success 'flux R encode/decode --property works' '
+	flux R encode --hosts=foo[0-1] --gpus 0 --cores 0-1  \
+	    --property xx:0 \
+	    --property yy:1 \
+	    --property all > properties.json &&
+	test $(flux R decode -c node <properties.json) -eq 2 &&
+	test $(flux R decode -c node --properties all <properties.json) -eq 2 &&
+	test $(flux R decode -c node --properties xx <properties.json) -eq 1 &&
+	test $(flux R decode -c node --properties yy <properties.json) -eq 1 &&
+	test $(flux R decode -c node --properties ^all <properties.json) -eq 0 &&
+	test $(flux R decode -c node --properties all,yy <properties.json) -eq 1 &&
+	test $(flux R decode --nodelist --properties xx <properties.json) = foo0 &&
+	test $(flux R decode --nodelist --properties yy <properties.json) = foo1 &&
+	test $(flux R decode --nodelist --properties ^yy <properties.json) = foo0
+'
+test_expect_success 'flux R encode --property fails with invalid rank' '
+	test_must_fail flux R encode -r 0-3 -p xx:3-5 >property-fail.out 2>&1 &&
+	test_debug "cat property-fail.out" &&
+	grep "ranks 4-5 not found" property-fail.out
+'
+test_expect_success 'flux R encode --xml works' '
+	flux R encode --xml=$SHARNESS_TEST_SRCDIR/hwloc-data/sierra2/0.xml \
+	    > R.sierra2 &&
+	result=$(flux R decode --short < R.sierra2) &&
+	test_debug "echo encode XML = $result" &&
+	test "$result" = "rank0/core[0-43],gpu[0-3]"
+'
+test_expect_success 'flux R encode --xml works with AMD RSMI gpus' '
+	flux R encode --xml=$SHARNESS_TEST_SRCDIR/hwloc-data/corona/0.xml \
+	    > R.corona &&
+	result=$(flux R decode --short < R.corona) &&
+	test_debug "echo encode XML = $result" &&
+	test "$result" = "rank0/core[0-47],gpu[0-7]"
+'
+test_expect_success 'flux R decode --include works' '
+	result=$(flux R encode -r 0-1023 | flux R decode --include 5-7 --short) &&
+	test_debug "echo $result" &&
+	test "$result" = "rank[5-7]/core0"
+'
+test_expect_success 'flux R decode --exclude works' '
+	result=$(flux R encode -r 1-10 | flux R decode --exclude 5-7 --short) &&
+	test_debug "echo $result" &&
+	test "$result" = "rank[1-4,8-10]/core0"
+'
+test_expect_success 'flux R append fails if R sets intersect' '
+	(flux R encode -r 0-1 && flux R encode -r 1-2) \
+	    | test_must_fail flux R append  &&
+	(flux R encode -r 0-1 -c 0-1 -g 0 && flux R encode -r 1 -c 2-3 -g 0) \
+	    | test_must_fail flux R append
+'
+test_expect_success 'flux R append works' '
+	result=$( (flux R encode -r 0-1 -c 0-1 && flux R encode -r 1-2 -c 2-3) \
+	    | flux R append | flux R decode --short) &&
+	test_debug "echo $result" &&
+	test "$result" = "rank0/core[0-1] rank1/core[0-3] rank2/core[2-3]"
+'
+test_expect_success 'flux R append works when only some nodes have gpus' '
+	result=$( (flux R encode -r 0-1 -c 0-1 && \
+	       flux R encode -r 2-3 -c 0-1 -g 0-1) \
+	    | flux R append | flux R decode --short) &&
+	test_debug "echo $result" &&
+	test "$result" = "rank[0-1]/core[0-1] rank[2-3]/core[0-1],gpu[0-1]"
+'
+
+test_expect_success 'flux R diff works' '
+	result=$( (flux R encode -r 0-1 -c 0-1 && flux R encode -r 0-1 -c 0) \
+	    | flux R diff | flux R decode --short) &&
+	test_debug "echo $result" &&
+	test "$result" = "rank[0-1]/core1"
+'
+test_expect_success 'flux R intersect works' '
+	result=$( (flux R encode -r 0-3 -c 0-1 && flux R encode -r 1-5 -c 0) \
+	    | flux R intersect | flux R decode --short) &&
+	test_debug "echo $result" &&
+	test "$result" = "rank[1-3]/core0" &&
+	result=$( (flux R encode -r 0-3 -c 0-1 && flux R encode -r 4-5 -c 0) \
+	    | flux R intersect | flux R decode --short) &&
+	test_debug "echo $result" &&
+	test -z "$result"
+'
+test_expect_success 'flux R remap works' '
+	result=$(flux R encode -r 5,99,1000 | flux R remap | flux R decode -s) &&
+	test_debug "echo $result" &&
+	test "$result" = "rank[0-2]/core0"
+'
+test_expect_success 'flux R rerank works' '
+	result=$(flux R encode -r 0-3 -H foo[0-3] \
+	    | flux R rerank foo[3,2,1,0] | flux R decode --nodelist) &&
+	test_debug "echo reranked $result" &&
+	test "$result" = "foo[3,2,1,0]"
+'
+test_expect_success 'flux R verify works' '
+	result=$( (flux R encode -r 1-10 -c 0-3 && flux R encode -r 1 -c 0-3) \
+	        | flux R verify 2>&1) &&
+	test_debug "echo $result"
+'
+test_expect_success 'flux R verify fails with mismatched ranks' '
+	result=$( (flux R encode -r 1-10 -c 0-3 && flux R encode -r 0 -c 0-3) \
+	        | test_must_fail flux R verify 2>&1) &&
+	test_debug "echo $result"
+'
+test_expect_success 'flux R verify fails with mismatched hosts' '
+	result=$( (flux R encode -r 1 -c 0-3 -H foo1 \
+	           && flux R encode -r 1 -c 0-3 -H foo12) \
+	        | test_must_fail flux R verify 2>&1) &&
+	test_debug "echo $result"
+'
+test_expect_success 'flux R verify fails with mismatched resources' '
+	(flux R encode  -r 1 -c 0-3 && flux R encode -r 1 -c 0-2) \
+	    | test_must_fail flux R verify
+'
+test_expect_success 'flux R verify reports extra resources' '
+	(flux R encode  -r 1 -c 0-3 && flux R encode -r 1 -c 0-7 -g 1) \
+	    | flux R verify
+'
+test_expect_success 'flux R set-property works' '
+	flux R encode -r 0-1 -c 0-3 -H foo[0-1] | \
+	    flux R set-property all | \
+	    flux R set-property xx:0 yy:1 > setprop.json &&
+	test $(flux R decode -c node <setprop.json) -eq 2 &&
+	test $(flux R decode -c node --properties all <setprop.json) -eq 2 &&
+	test $(flux R decode -c node --properties xx <setprop.json) -eq 1 &&
+	test $(flux R decode -c node --properties yy <setprop.json) -eq 1 &&
+	test $(flux R decode -c node --properties ^all <setprop.json) -eq 0 &&
+	test $(flux R decode -c node --properties all,yy <setprop.json) -eq 1 &&
+	test $(flux R decode --nodelist --properties xx <setprop.json) = foo0 &&
+	test $(flux R decode --nodelist --properties yy <setprop.json) = foo1 &&
+	test $(flux R decode --nodelist --properties ^yy <setprop.json) = foo0
+'
+test_expect_success 'flux R set-property fails with unknown ranks' '
+	flux R encode -r 0-1 | test_must_fail flux R set-property foo:1-2
+'
+test_expect_success 'scheduling opaque key is preserved' '
+	flux R encode -r 0-3 -c 0-3 -g 0 -H foo[0-3] \
+	    | jq ".scheduling = 42" > R.orig &&
+	flux R decode < R.orig | jq -e ".scheduling == 42" &&
+	flux R decode --include 0 < R.orig | jq -e ".scheduling == 42" &&
+	flux R decode --exclude 0 < R.orig | jq -e ".scheduling == 42"
+'
+test_expect_success 'scheduling opaque key is preserved with append' '
+	(cat R.orig && flux R encode -r 4 ) | flux R append \
+	    | flux R decode | jq -e ".scheduling == 42"
+'
+test_expect_success 'scheduling opaque key is preserved with remap' '
+	flux R remap < R.orig | flux R decode | jq -e ".scheduling == 42"
+'
+test_expect_success 'scheduling opaque key is preserved with diff' '
+	(cat R.orig && flux R encode -r 1 ) | flux R diff \
+	    | jq -e ".scheduling == 42"
+'
+test_expect_success 'scheduling key is preserved with intersect' '
+	(cat R.orig && flux R encode -r 1 -H foo1) | flux R intersect \
+	    | jq -e ".scheduling == 42"
+'
+test_expect_success 'use of --local,--xml and --hosts is supported' '
+	ncores=$(flux R encode --local | flux R decode --count cores) &&
+	flux R encode --local --hosts=fluke[0-16] > R.hosts &&
+	test_debug "flux R decode --short < R.hosts" &&
+	count=$(flux R decode --count cores < R.hosts) &&
+	test "$count" = "$ncores" &&
+	hosts=$(flux R decode --nodelist < R.hosts) &&
+	test_debug "echo got $hosts" &&
+	test "$hosts" = "fluke[0-16]"
+'
+test_expect_success 'flux R parse-config works' '
+	mkdir conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo"
+	cores = "0-1"
+	gpus = "0"
+	EOF
+	flux R parse-config conf > conf.json &&
+	test_debug "flux R decode --short < conf.json" &&
+	test $(flux R decode -c node < conf.json) -eq 1 &&
+	test $(flux R decode -c core < conf.json) -eq 2 &&
+	test $(flux R decode -c gpu < conf.json) -eq 1 &&
+	test "$(flux R decode --nodelist < conf.json)" = "foo"
+'
+test_expect_success 'flux R parse-config works (multiple entries)' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo[0-2]"
+	cores = "0-1"
+	[[resource.config]]
+	hosts = "foo2"
+	gpus = "0"
+	[[resource.config]]
+	hosts = "foo0"
+	properties = ["login"]
+	EOF
+	flux R parse-config conf > conf2.json &&
+	test_debug "flux R decode --short < conf2.json" &&
+	test $(flux R decode -c node < conf2.json) -eq 3 &&
+	test $(flux R decode -c core < conf2.json) -eq 6 &&
+	test $(flux R decode -c gpu < conf2.json) -eq 1 &&
+	test "$(flux R decode --nodelist < conf2.json)" = "foo[0-2]" &&
+	test "$(flux R decode -p login --nodelist < conf2.json)" = "foo0"
+'
+test_expect_success 'flux R parse-config fails on invalid TOML' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	EOF
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config fails on when resource.config missing' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[resource]
+	noverify = true
+	EOF
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config detects empty host list' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo"
+	cores = "0-1"
+	[[resource.config]]
+	hosts = ""
+	cores = "0-3"
+	EOF
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config detects invalid host list' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo[p]"
+	cores = "0-1"
+	EOF
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config detects invalid idset' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo0"
+	cores = "0-"
+	EOF
+	test_must_fail flux R parse-config conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo0"
+	gpus = "2,1"
+	EOF
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config detects host with no resources' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo[0-10]"
+	cores = "0-1"
+	[[resource.config]]
+	hosts = "foo11"
+	EOF
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config detects missing hosts entry' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo[0-10]"
+	cores = "0-1"
+	[[resource.config]]
+	cores = "0-1"
+	EOF
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config detects invalid entry' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo[0-10]"
+	cores = "0-1"
+	[[resource.config]]
+	hosts = "foo11"
+	cores = "0-3"
+	junk = 5
+	EOF
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config detects invalid property' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo[0-10]"
+	cores = "0-1"
+	[[resource.config]]
+	hosts = "foo1"
+	properties = ["de^bug"]
+	EOF
+	test_must_fail flux R parse-config conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo[0-10]"
+	cores = "0-1"
+	[[resource.config]]
+	hosts = "foo1"
+	properties = [1]
+	EOF
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config detects when properties not an array' '
+	mkdir -p conf &&
+	cat <<-EOF >conf/resource.toml &&
+	[[resource.config]]
+	hosts = "foo[0-10]"
+	cores = "0-1"
+	[[resource.config]]
+	hosts = "foo11"
+	cores = "0-3"
+	properties = "foo"
+	EOF
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config also works with resource.path' '
+	flux R encode -r 0-1 >R.path &&
+	cat <<-EOF >conf/resource.toml &&
+	resource.path = "R.path"
+	EOF
+	flux R parse-config conf
+'
+test_expect_success 'flux R parse-config fails when resource.path = bad R' '
+	echo "bad json" >R.path &&
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config fails when resource.path = missing R' '
+	rm -f R.path &&
+	test_must_fail flux R parse-config conf
+'
+test_expect_success 'flux R parse-config works (+scheduling)' '
+	mkdir -p conf3 &&
+	jq -n ".sched = true" > conf3/sched.json &&
+	cat <<-EOF >conf3/resource.toml &&
+	resource.scheduling = "$(pwd)/conf3/sched.json"
+	[[resource.config]]
+	hosts = "foo[0-2]"
+	cores = "0-1"
+	[[resource.config]]
+	hosts = "foo2"
+	gpus = "0"
+	EOF
+	flux R parse-config conf3 > conf+sched.json &&
+	jq -e <conf+sched.json ".scheduling.sched == true"
+'
+test_expect_success 'flux R parse-config detects bad scheduling key' '
+	mkdir -p conf4 &&
+	echo foo > conf4/sched.json &&
+	cat <<-EOF >conf4/resource.toml &&
+	resource.scheduling = "$(pwd)/conf4/sched.json"
+	[[resource.config]]
+	hosts = "foo[0-2]"
+	cores = "0-1"
+	[[resource.config]]
+	hosts = "foo2"
+	gpus = "0"
+	EOF
+	test_must_fail flux R parse-config conf4
+'
+test_done
diff --git a/t/t0027-broker-groups.t b/t/t0027-broker-groups.t
new file mode 100755
index 000000000000..b5b5d602e8ca
--- /dev/null
+++ b/t/t0027-broker-groups.t
@@ -0,0 +1,114 @@
+#!/bin/sh
+#
+
+test_description='Test broker groups'
+
+. `dirname $0`/sharness.sh
+SIZE=4
+test_under_flux ${SIZE}
+
+GROUPSCMD="flux python ${SHARNESS_TEST_SRCDIR}/scripts/groups.py"
+
+test_expect_success 'broker.online contains full instance' '
+	cat >broker.online.exp <<-EOT &&
+	0-3
+	EOT
+	${GROUPSCMD} get broker.online >broker.online.out &&
+	test_cmp broker.online.exp broker.online.out
+'
+
+test_expect_success 'groups.get of nonexistent group returns empty set' '
+	cat >newgroup.exp <<-EOT &&
+
+	EOT
+	${GROUPSCMD} get newgroup >newgroup.out &&
+	test_cmp newgroup.exp newgroup.out
+'
+
+test_expect_success 'groups.get on rank > 0 fails with reasonable error' '
+	test_must_fail ${GROUPSCMD} get --rank 1 broker.online 2>test0.err &&
+	grep "only available on rank 0" test0.err
+'
+
+test_expect_success 'nonlocal groups.join fails with appropriate error' '
+	test_must_fail ${GROUPSCMD} join --rank 1 foo 2>rmtjoin.err &&
+	grep "restricted to the local broker" rmtjoin.err
+'
+test_expect_success 'nonlocal groups.leave fails with appropriate error' '
+	test_must_fail ${GROUPSCMD} leave --rank 1 foo 2>rmtleave.err &&
+	grep "restricted to the local broker" rmtleave.err
+'
+
+badjoin() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"groups.join\").get())"
+}
+test_expect_success 'groups.join with malformed payload fails with EPROTO' '
+	test_must_fail badjoin 2>badjoin.err &&
+	grep "Protocol error" badjoin.err
+'
+
+badleave() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"groups.leave\").get())"
+}
+test_expect_success 'groups.leave with malformed payload fails with EPROTO' '
+	test_must_fail badleave 2>badleave.err &&
+	grep "Protocol error" badleave.err
+'
+
+badget() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"groups.get\").get())"
+}
+test_expect_success 'groups.get with malformed payload fails with EPROTO' '
+	test_must_fail badget 2>badget.err &&
+	grep "Protocol error" badget.err
+'
+
+badupdate() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"groups.update\"))"
+}
+test_expect_success 'send groups.update with malformed payload (no response)' '
+	badupdate
+'
+
+badupdate2() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"groups.update\",{\"update\":{\"foo\":42}}))"
+}
+test_expect_success 'send groups.update with malformed ops array (no response)' '
+	badupdate2
+'
+
+test_expect_success 'join group and explicitly leave works' '
+	${GROUPSCMD} join --leave test1 &&
+	${GROUPSCMD} join --leave test1
+'
+
+test_expect_success 'join group and disconnect works' '
+	${GROUPSCMD} join test2 &&
+	${GROUPSCMD} join test2
+'
+
+test_expect_success 'join group twice fails with reasonable error' '
+	test_must_fail ${GROUPSCMD} join --dubjoin test3 2>test3.err &&
+	grep "already a member" test3.err
+'
+
+test_expect_success 'leave group twice fails with reasonable error' '
+	test_must_fail ${GROUPSCMD} join --leave --dubleave test4 2>test4.err &&
+	grep "not a member" test4.err
+'
+
+test_expect_success 'join on all ranks works' '
+	flux exec ${GROUPSCMD} join test5
+'
+
+test_expect_success 'barrier test using groups works' '
+	flux exec ${GROUPSCMD} barrier barrier.1
+'
+test_expect_success 'ensure barrier count reaches zero' '
+	run_timeout 10 ${GROUPSCMD} waitfor --count 0 barrier.1
+'
+test_expect_success 'dump groups logs on rank 0' '
+	flux dmesg|grep groups
+'
+
+test_done
diff --git a/t/t0028-content-backing-none.t b/t/t0028-content-backing-none.t
new file mode 100755
index 000000000000..129cd5404114
--- /dev/null
+++ b/t/t0028-content-backing-none.t
@@ -0,0 +1,95 @@
+#!/bin/sh
+
+test_description='Test broker content checkpoint w/o backing module'
+
+. `dirname $0`/content/content-helper.sh
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 2 minimal
+echo "# $0: flux session size will be ${SIZE}"
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+
+test_expect_success 'loaded content module' '
+	flux exec flux module load content
+'
+
+test_expect_success 'checkpoint-get fails, no checkpoints yet' '
+        checkpoint_put foo bar
+'
+
+test_expect_success 'checkpoint-put foo w/ rootref bar' '
+        checkpoint_put foo bar
+'
+
+test_expect_success 'checkpoint-get foo returned rootref bar' '
+        echo bar >rootref.exp &&
+        checkpoint_get foo | jq -r .value | jq -r .rootref >rootref.out &&
+        test_cmp rootref.exp rootref.out
+'
+
+test_expect_success 'checkpoint-put on rank 1 forwards to rank 0' '
+        o=$(checkpoint_put_msg rankone rankref) &&
+        jq -j -c -n ${o} | flux exec -r 1 ${RPC} content.checkpoint-put
+'
+
+test_expect_success 'checkpoint-get on rank 1 forwards to rank 0' '
+        echo rankref >rankref.exp &&
+        o=$(checkpoint_get_msg rankone) &&
+        jq -j -c -n ${o} \
+            | flux exec -r 1 ${RPC} content.checkpoint-get \
+            | jq -r .value | jq -r .rootref > rankref.out &&
+        test_cmp rankref.exp rankref.out
+'
+
+test_expect_success 'flux-dump --checkpoint with missing checkpoint fails' '
+        test_must_fail flux dump --checkpoint foo.tar
+'
+
+test_expect_success 'load kvs and create some kvs data' '
+        flux module load kvs &&
+        flux kvs put a=1 &&
+        flux kvs put b=foo
+'
+
+test_expect_success 'reload kvs' '
+        flux module reload kvs &&
+        test $(flux kvs get a) = "1" &&
+        test $(flux kvs get b) = "foo"
+'
+
+test_expect_success 'unload kvs' '
+        flux module remove kvs
+'
+
+test_expect_success 'dump default=kvs-primary checkpoint works' '
+        flux dump --checkpoint foo.tar
+'
+
+test_expect_success 'restore content' '
+        flux restore --checkpoint foo.tar
+'
+
+test_expect_success 'reload kvs' '
+        flux module load kvs
+'
+
+test_expect_success 'verify KVS content restored' '
+        test $(flux kvs get a) = "1" &&
+        test $(flux kvs get b) = "foo"
+'
+
+test_expect_success 'unload kvs' '
+        flux module remove kvs
+'
+
+test_expect_success 'content.backing-module input of none works' '
+        flux start -Scontent.backing-module=none true
+'
+
+test_expect_success 'removedcontent module' '
+	flux exec flux module remove content
+'
+
+test_done
diff --git a/t/t0029-archive-mmap.t b/t/t0029-archive-mmap.t
new file mode 100755
index 000000000000..2940e51db94d
--- /dev/null
+++ b/t/t0029-archive-mmap.t
@@ -0,0 +1,310 @@
+#!/bin/sh
+
+test_description='Test flux-archive'
+
+. `dirname $0`/content/content-helper.sh
+
+. `dirname $0`/sharness.sh
+
+LPTEST="flux lptest"
+
+# SEEK_DATA support was added to the linux NFS client in kernel 3.18.
+# In el7 based distros, it is defined but doesn't work on NFS.  So
+# ensure SEEK_DATA returns ENXIO on file that is 100% empty.
+havesparse() {
+	cat >lseek.py <<-EOT &&
+	#!/usr/bin/env python3
+	import sys, os, errno
+	fd = os.open("$1", os.O_RDONLY)
+	try:
+	    os.lseek(fd, 0, os.SEEK_DATA)
+	except OSError as e:
+	    if e.errno == errno.ENXIO:
+	        sys.exit(0)
+	sys.exit(1)
+	EOT
+	chmod +x lseek.py &&
+	truncate --size 8192 $1 &&
+	test $(stat --format "%b" $1) -eq 0 &&
+	./lseek.py
+}
+
+if havesparse testholes; then
+	test_set_prereq HAVE_SPARSE
+fi
+
+test_under_flux 2 kvs
+
+# after test_under_flux is launched, cannot assume what umask is.  An
+# unexpected umask could affect tests below.  Hard code to 022 for
+# these tests.
+umask 022
+
+# Usage: list_mapped_files tag
+list_mapped_files() {
+	flux module stats content | jq -r ".mmap.tags[\"$1\"][]"
+}
+
+test_expect_success 'create copy directory' '
+	mkdir -p copydir
+'
+test_expect_success 'create test file' '
+	${LPTEST} >testfile &&
+	chmod u=rwx testfile &&
+	chmod go=r testfile
+'
+test_expect_success 'map nonexistent file fails' '
+	test_must_fail flux archive create --mmap notafile
+'
+test_expect_success 'map fails on rank != 0' '
+	test_must_fail flux exec -r 1 flux archive create --mmap testfile
+'
+test_expect_success 'map fails with --preserve' '
+	test_must_fail flux archive create --mmap --preserve testfile
+'
+test_expect_success 'map fails with --no-force-primary' '
+	test_must_fail flux archive create --mmap --no-force-primary testfile
+'
+test_expect_success 'map unreadable file fails with appropriate error' '
+	touch unreadable &&
+	chmod ugo-r unreadable &&
+	test_must_fail flux archive create --mmap unreadable 2>unreadable.err &&
+	grep "Permission denied" unreadable.err
+'
+test_expect_success 'map test file' '
+	flux archive create --mmap ./testfile &&
+	flux kvs get --raw archive.main | jq
+'
+test_expect_success 'content stats show mapped file' '
+	list_mapped_files main >stats.out &&
+	realpath ./testfile >stats.exp &&
+	test_cmp stats.exp stats.out
+'
+test_expect_success 'file can be listed' '
+	flux archive list
+'
+test_expect_success 'file can be listed in long form' '
+	flux archive list --long
+'
+test_expect_success 'test file can be read through content cache on rank 0' '
+	flux archive extract -C copydir
+'
+test_expect_success 'file content is correct' '
+	test_cmp testfile copydir/testfile
+'
+test_expect_success 'file permissions are correct' '
+	stat --format="%a" testfile >access.exp &&
+	stat --format="%a" copydir/testfile >access.out &&
+	test_cmp access.exp access.out
+'
+test_expect_success 'test file can be read through content cache on rank 1' '
+	rm -f copydir/testfile &&
+	flux exec -r 1 flux archive extract -C copydir &&
+	test_cmp testfile copydir/testfile
+'
+test_expect_success 'unmap test file' '
+	flux archive remove
+'
+test_expect_success 'content stats no longer show mapped file' '
+	test_must_fail list_mapped_files archive.main
+'
+test_expect_success 'drop the cache' '
+	flux exec -r 1 flux content dropcache &&
+	flux content dropcache
+'
+test_expect_success 'test file is not extracted' '
+	rm -f copydir/testfile &&
+	test_must_fail flux archive extract
+'
+test_expect_success 'map test file with small chunksize' '
+	flux archive create --mmap --chunksize=10 ./testfile
+'
+test_expect_success 'test file can be read through content cache on rank 0' '
+	rm -f copydir/testfile &&
+	flux archive extract -C copydir &&
+	test_cmp testfile copydir/testfile
+'
+test_expect_success 'test file can be read through content cache on rank 1' '
+	rm -f copydir/testfile &&
+	flux exec -r 1 flux archive extract -C copydir &&
+	test_cmp testfile copydir/testfile
+'
+test_expect_success 'unmap test file' '
+	flux archive remove
+'
+test_expect_success 'drop the cache' '
+	flux exec -r 1 flux content dropcache &&
+	flux content dropcache
+'
+test_expect_success 'create test file' '
+	echo abcdefghijklmnopqrstuvwxyz >testfile2
+'
+test_expect_success 'map test file with small small-file-threshold' '
+	flux archive create --mmap --small-file-threshold=10 ./testfile2
+'
+test_expect_success 'extract blobref from mapped file' "
+	flux kvs get --raw archive.main >testfile2.json &&
+	jq -r '.[0].data[0][2]' <testfile2.json >testfile2.blobref
+"
+test_expect_success 'store the duplicate blob to the mapped one on rank 1' '
+	flux content load <testfile2.blobref >testfile2.data &&
+	flux exec -r 1 flux content store <testfile2.data \
+	    >testfile2.blobref2  &&
+	test_cmp testfile2.blobref testfile2.blobref2
+'
+test_expect_success 'unmap test file' '
+	flux archive remove
+'
+test_expect_success 'drop the cache' '
+	flux exec -r 1 flux content dropcache &&
+	flux content dropcache
+'
+test_expect_success 'blob can still be read through content cache on rank 1' '
+	flux exec -r 1 flux content load <testfile2.blobref2 \
+		>testfile2.data2 &&
+	test_cmp testfile2.data testfile2.data2
+'
+test_expect_success HAVE_SPARSE 'create sparse test file' '
+	truncate --size=8192 testfile3
+'
+test_expect_success HAVE_SPARSE 'map test file' '
+	flux archive create --mmap ./testfile3
+'
+test_expect_success HAVE_SPARSE 'test file can be read through content cache' '
+	flux archive extract -C copydir &&
+	test_cmp testfile3 copydir/testfile3
+'
+test_expect_success HAVE_SPARSE 'holes were preserved' '
+	stat --format="%b" testfile3 >blocks.exp &&
+	stat --format="%b" copydir/testfile3 >blocks.out &&
+	test_cmp blocks.exp blocks.out
+'
+test_expect_success HAVE_SPARSE 'unmap test file' '
+	flux archive remove
+'
+test_expect_success HAVE_SPARSE 'create sparse test file with data' '
+	truncate --size=8192 testfile3b &&
+	echo more-data >>testfile3b
+'
+test_expect_success HAVE_SPARSE 'map test file' '
+	flux archive create --mmap ./testfile3b
+'
+test_expect_success HAVE_SPARSE 'test file can be read through content cache' '
+	flux archive extract -C copydir &&
+	test_cmp testfile3b copydir/testfile3b
+'
+test_expect_success HAVE_SPARSE 'holes were preserved' '
+	stat --format="%b" testfile3b >blocks3b.exp &&
+	stat --format="%b" copydir/testfile3b >blocks3b.out &&
+	test_cmp blocks3b.exp blocks3b.out
+'
+test_expect_success HAVE_SPARSE 'unmap test file' '
+	flux archive remove
+'
+test_expect_success 'create test symlink' '
+	ln -s /a/b/c/d testfile4
+'
+test_expect_success 'map test file' '
+	flux archive create --mmap ./testfile4
+'
+test_expect_success 'show raw object' '
+	flux kvs get --raw archive.main | jq .
+'
+test_expect_success 'test file can be extracted' '
+	flux archive extract -v -C copydir
+'
+test_expect_success 'copy is a symlink with expected target' '
+	readlink testfile4 >link.exp &&
+	readlink copydir/testfile4 >link.out &&
+	test_cmp link.exp link.out
+'
+test_expect_success 'unmap test file' '
+	flux archive remove
+'
+test_expect_success 'map file by absolute path' '
+	flux archive create --mmap /etc/group
+'
+test_expect_success 'test file lists with relative path' '
+	cat >list.exp <<-EOT &&
+	etc/group
+	EOT
+	flux archive list >list.out &&
+	test_cmp list.exp list.out
+'
+test_expect_success 'unmap test file' '
+	flux archive remove
+'
+test_expect_success 'create test directory' '
+	mkdir testfile5
+'
+test_expect_success 'map test file' '
+	flux archive create --mmap ./testfile5
+'
+test_expect_success 'test file can be read through content cache' '
+	flux archive extract -v -C copydir
+'
+test_expect_success 'copy is a directory' '
+	test -d copydir/testfile5
+'
+test_expect_success 'unmap test file' '
+	flux archive remove
+'
+test_expect_success 'map small test file with reduced small file threshold' '
+	flux archive create --mmap --small-file-threshold=0 ./testfile2
+'
+test_expect_success 'test file used blobvec encoding' '
+	flux kvs get --raw archive.main | jq -e ".[0].encoding == \"blobvec\""
+'
+test_expect_success 'unmap test file' '
+	flux archive remove
+'
+test_expect_success 'map test file' '
+	rm -f copydir/testfile &&
+	flux archive create --mmap ./testfile
+'
+test_expect_success 'modify mapped test file without reducing its size' '
+	dd if=/dev/zero of=testfile bs=4096 count=1 conv=notrunc
+'
+test_expect_success 'content change should cause an error' '
+	rm -f copydir/testfile &&
+	test_must_fail flux archive extract -C copydir 2>changed.err &&
+	grep changed changed.err
+'
+test_expect_success 'drop cache and unmap test file' '
+	flux content dropcache &&
+	flux archive remove
+'
+test_expect_success 'map test file' '
+	rm -f copydir/testfile &&
+	flux archive create --mmap ./testfile
+'
+test_expect_success 'truncate mapped test file' '
+	cp /dev/null testfile
+'
+test_expect_success 'size reduction should cause an error' '
+	rm -f copydir/testfile &&
+	test_must_fail flux archive extract -C copydir 2>reduced.err &&
+	grep changed reduced.err
+'
+test_expect_success 'unmap test file' '
+	flux archive remove
+'
+test_expect_success 're-create and map test file' '
+	${LPTEST} >testfile &&
+	flux archive create --mmap ./testfile
+'
+test_expect_success 'extract copydir/testfile' '
+	rm -f copydir/testfile &&
+	flux archive extract -C copydir
+'
+test_expect_success 'extract --overwrite works' '
+	flux archive extract --overwrite -C copydir
+'
+test_expect_success 'extract refuses to overwrite without explicit option' '
+	test_must_fail flux archive extract -C copydir
+'
+test_expect_success 'unmap test file' '
+	flux archive remove
+'
+
+test_done
diff --git a/t/t0030-marshall.t b/t/t0030-marshall.t
new file mode 100755
index 000000000000..4ec38faa4bde
--- /dev/null
+++ b/t/t0030-marshall.t
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+test_description='Test message marshalling across versions'
+
+. `dirname $0`/sharness.sh
+
+marshall=${FLUX_BUILD_DIR}/t/util/marshall
+inputs=${SHARNESS_TEST_SRCDIR}/marshall
+
+test_expect_success 'generate encoded message output for this version' '
+	${marshall} encode >this &&
+	${marshall} decode <this
+'
+
+test_expect_success 'output from this version matches known valid version' '
+	cmp ${inputs}/valid/v0.47.data this
+'
+
+for data in ${inputs}/valid/*.data; do
+    testname=`basename $data`
+    test_expect_success 'valid: '$testname "${marshall} decode <$data"
+done
+for data in ${inputs}/invalid/*.data; do
+    testname=`basename $data`
+    test_expect_failure 'invalid: '$testname "${marshall} decode <$data"
+done
+
+
+test_done
diff --git a/t/t0031-constraint-parser.t b/t/t0031-constraint-parser.t
new file mode 100755
index 000000000000..1faae87462e9
--- /dev/null
+++ b/t/t0031-constraint-parser.t
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+test_description='Test python flux.constraint.parser standalone operation'
+
+. `dirname $0`/sharness.sh
+
+parser="flux python -m flux.constraint.parser"
+
+test_expect_success 'flux.constraint.parser works' '
+	$parser
+'
+test_expect_success 'flux.constraint.parser prints usage' '
+	$parser --help > parser.out 2>&1 &&
+	test_debug "cat parser.out" &&
+	grep -i usage parser.out
+'
+test_expect_success 'flux.constraint.parser parses cmdline' '
+	$parser test:a test:b >basic.out &&
+	test_debug "cat basic.out" &&
+	grep and basic.out
+'
+test_expect_success 'flux.constraint.parser --default-op works' '
+	$parser --default-op=test a b >default-op.out &&
+	test_debug "cat default-op.out" &&
+	grep test default-op.out
+'
+test_expect_success 'flux.constraint.parser --debug works' '
+	$parser --debug --default-op=test "a|(b&-c)" >debug.out 2>&1 &&
+	test_debug "cat debug.out" &&
+	grep TOKEN debug.out
+'
+test_done
diff --git a/t/t0032-directives-parser.t b/t/t0032-directives-parser.t
new file mode 100755
index 000000000000..6a664b6c49ec
--- /dev/null
+++ b/t/t0032-directives-parser.t
@@ -0,0 +1,53 @@
+#!/bin/sh
+
+test_description='Test python flux.job.directives parser operation'
+
+. `dirname $0`/sharness.sh
+
+parser="flux python -m flux.job.directives"
+
+test_expect_success 'flux.job.directives works on file with no directives' '
+	$parser <<-EOF >empty.out &&
+	#!/bin/sh
+	hostname
+	EOF
+	test_debug "cat empty.out" &&
+	test_must_be_empty empty.out
+'
+TESTDIR=${SHARNESS_TEST_SRCDIR}/batch/directives
+VALID=$TESTDIR/valid
+INVALID=$TESTDIR/invalid
+
+validate_directives() {
+	input=$(basename $1) &&
+	testname=${input%.*} &&
+	out=${testname}.output &&
+	exp=$(dirname $1)/expected/${testname}.expected &&
+	$parser $1 >$out 2>&1 &&
+	test_debug "cat $out" &&
+	test_cmp $exp $out
+}
+
+for file in ${VALID}/*; do
+	test -d $file && continue
+	input=$(basename $file) &&
+	test_expect_success 'valid: '${input} "validate_directives $file"
+done
+
+test_invalid_directive() {
+	input=$(basename $1) &&
+	testname=${input%.*} &&
+	out=${testname}.output &&
+	errmsg=$(dirname $1)/expected/${testname}.pattern &&
+	test_must_fail $parser $1 >$out 2>&1 &&
+	test_debug "cat $out" &&
+	grep "$(cat $errmsg)" $out
+}
+
+for file in ${INVALID}/*; do
+	test -d $file && continue
+	input=$(basename $file) &&
+	test_expect_success 'invalid: '${input} "test_invalid_directive $file"
+done
+
+test_done
diff --git a/t/t0033-filemap-cmd.t b/t/t0033-filemap-cmd.t
new file mode 100755
index 000000000000..39b6215a9847
--- /dev/null
+++ b/t/t0033-filemap-cmd.t
@@ -0,0 +1,93 @@
+#!/bin/sh
+
+test_description='Test flux-filemap'
+
+. `dirname $0`/content/content-helper.sh
+
+. `dirname $0`/sharness.sh
+
+LPTEST="flux lptest"
+
+test_under_flux 2 kvs
+
+umask 022
+
+# Usage: list_mapped_files tag
+list_mapped_files() {
+	flux module stats content | jq -r ".mmap.tags[\"$1\"][]"
+}
+
+test_expect_success 'create copy directory' '
+	mkdir -p copydir
+'
+test_expect_success 'create test file' '
+	${LPTEST} >testfile &&
+	chmod u=rwx testfile &&
+	chmod go=r testfile
+'
+test_expect_success 'flux filemap map works' '
+	flux filemap map ./testfile &&
+	flux kvs get --raw archive.main | jq
+'
+test_expect_success 'content stats show mapped file' '
+	list_mapped_files main >stats.out &&
+	realpath ./testfile >stats.exp &&
+	test_cmp stats.exp stats.out
+'
+test_expect_success 'flux filemap list works' '
+	flux filemap list
+'
+test_expect_success 'flux filemap list --long works' '
+	flux filemap list --long
+'
+test_expect_success 'flux filemap get works' '
+	flux filemap get -C copydir
+'
+test_expect_success 'file content is correct' '
+	test_cmp testfile copydir/testfile
+'
+test_expect_success 'flux filemap umap works' '
+	flux filemap unmap
+'
+test_expect_success 'content stats no longer show mapped file' '
+	test_must_fail list_mapped_files archive.main
+'
+test_expect_success 'flux filemap map --disable-mmap -Too works' '
+	flux filemap map --disable-mmap -Tfoo ./testfile &&
+	flux kvs get --raw archive.foo | jq
+'
+test_expect_success 'flux filemap list -Tfoo works' '
+	flux filemap list -Tfoo
+'
+test_expect_success 'flux filemap get -Tfoo works' '
+	rm -f copydir/* &&
+	flux filemap get -C copydir -Tfoo 2>warn.err
+'
+test_expect_success 'with warning printed on stderr' '
+	grep deprecated warn.err
+'
+test_expect_success 'file content is correct' '
+	test_cmp testfile copydir/testfile
+'
+test_expect_success 'flux filemap unmap -Tfoo works' '
+	flux filemap unmap -Tfoo
+'
+test_expect_success 'flux filemap list no longer works' '
+	test_must_fail flux filemap list -Tfoo
+'
+test_expect_success 'flux filemap create does not accept -T' '
+	test_must_fail flux archive create -T bar ./testfile
+'
+test_expect_success 'flux filemap create does not accept --disable-mmap' '
+	test_must_fail flux archive create --disable-mmap ./testfile
+'
+test_expect_success 'flux archive create --name=bar works' '
+	flux archive create --name=bar ./testfile
+'
+test_expect_success 'flux filemap extract does not accept -T' '
+	test_must_fail flux archive extract -T bar
+'
+test_expect_success 'flux filemap remove does not accept -T' '
+	test_must_fail flux archive remove -T bar
+'
+test_done
diff --git a/t/t0090-content-enospc.t b/t/t0090-content-enospc.t
new file mode 100755
index 000000000000..fa2cfd8b8be6
--- /dev/null
+++ b/t/t0090-content-enospc.t
@@ -0,0 +1,61 @@
+#!/bin/sh
+
+test_description='Test content ENOSPC corner cases'
+
+. `dirname $0`/sharness.sh
+
+if ! ls /test/tmpfs-1m; then
+	skip_all='skipping ENOSPC tests, no small tmpfs directory mounted'
+	test_done
+fi
+
+test_expect_success 'create script to fill statedir' '
+	cat >fillstatedir.sh <<-EOT &&
+	#!/bin/sh
+	while true ; do
+	      flux run echo 0123456789 > /dev/null 2>&1
+	      if flux dmesg | grep -q "No space left on device"; then
+		   break
+	      fi
+	done
+	EOT
+	chmod +x fillstatedir.sh
+'
+
+# flux start will fail b/c rc3 will fail due to ENOSPC
+test_expect_success 'flux still operates with content-sqlite running out of space' '
+	rm -rf /test/tmpfs-1m/* &&
+	mkdir /test/tmpfs-1m/statedir &&
+	test_must_fail flux start \
+	    -Scontent.backing-module=content-sqlite \
+	    -Sstatedir=/test/tmpfs-1m/statedir \
+	    "./fillstatedir.sh; flux dmesg; flux run echo helloworld" > sql.out 2> sql.err &&
+	grep -q "No space left on device" sql.out &&
+	grep "helloworld" sql.out
+'
+
+# flux start will fail b/c rc3 will fail due to ENOSPC
+test_expect_success 'flux still operates with content-files running out of space' '
+	rm -rf /test/tmpfs-1m/* &&
+	mkdir /test/tmpfs-1m/statedir &&
+	test_must_fail flux start \
+	    -Scontent.backing-module=content-files \
+	    -Sstatedir=/test/tmpfs-1m/statedir \
+	    "./fillstatedir.sh; flux dmesg; flux run echo helloworld" > files.out 2> files.err &&
+	grep -q "No space left on device" files.out &&
+	grep "helloworld" files.out
+'
+
+# flux start will fail b/c rc3 will fail due to ENOSPC
+test_expect_success 'content flush returns error on ENOSPC' '
+	rm -rf /test/tmpfs-1m/* &&
+	mkdir /test/tmpfs-1m/statedir &&
+	test_must_fail flux start \
+	    -Scontent.backing-module=content-sqlite \
+	    -Sstatedir=/test/tmpfs-1m/statedir \
+	    "./fillstatedir.sh; flux dmesg; flux content flush" > flush.out 2> flush.err &&
+	grep -q "No space left on device" flush.out &&
+	grep "content.flush: No space left on device" flush.err
+'
+
+test_done
diff --git a/t/t1000-kvs.t b/t/t1000-kvs.t
index 158fa917ece6..8d60501c62c7 100755
--- a/t/t1000-kvs.t
+++ b/t/t1000-kvs.t
@@ -662,10 +662,11 @@ test_expect_success 'kvs: ls -1RF shows directory titles' '
 	EOF
 	test_cmp expected output
 '
+# test assumes COLUMNS environment not set, clear in a subshell just in case
 test_expect_success 'kvs: ls with no options adjusts output width to 80' '
 	flux kvs unlink -Rf $DIR &&
 	${FLUX_BUILD_DIR}/t/kvs/dtree -p$DIR -h1 -w50 &&
-	flux kvs ls $DIR | wc -wl >output &&
+	$(unset COLUMNS; flux kvs ls $DIR | wc -wl >output) &&
 	cat >expected <<-EOF &&
 	      5      50
 	EOF
@@ -680,7 +681,8 @@ test_expect_success 'kvs: ls -w40 adjusts output width to 40' '
 	EOF
 	test_cmp expected output
 '
-test_expect_success 'kvs: ls with COLUMNS=20 adjusts output width to 20' '
+test_columns_variable_preserved && test_set_prereq USE_COLUMNS
+test_expect_success USE_COLUMNS 'kvs: ls with COLUMNS=20 adjusts output width to 20' '
 	flux kvs unlink -Rf $DIR &&
 	${FLUX_BUILD_DIR}/t/kvs/dtree -p$DIR -h1 -w50 &&
 	COLUMNS=20 flux kvs ls $DIR | wc -wl >output &&
@@ -1114,7 +1116,7 @@ test_expect_success 'kvs: get --at: fails bad on dirent' '
 '
 
 #
-# -O, -s options in write commands
+# -O, -b, -s options in write commands
 #
 
 test_expect_success 'kvs: --treeobj-root on write ops works' '
@@ -1129,6 +1131,22 @@ test_expect_success 'kvs: --treeobj-root on write ops works' '
         grep "dirref" output
 '
 
+test_expect_success 'kvs: --blobref on write ops works' '
+	flux kvs unlink -Rf $DIR &&
+        flux kvs put -b $DIR.a=1 > output &&
+        flux kvs getroot -b > expected &&
+        test_cmp output expected &&
+        flux kvs unlink -b $DIR.a > output &&
+        flux kvs getroot -b > expected &&
+        test_cmp output expected &&
+        flux kvs mkdir -b $DIR.a > output &&
+        flux kvs getroot -b > expected &&
+        test_cmp output expected &&
+        flux kvs link -b $DIR.a $DIR.b > output &&
+        flux kvs getroot -b > expected &&
+        test_cmp output expected
+'
+
 test_expect_success 'kvs: --sequence on write ops works' '
 	flux kvs unlink -Rf $DIR &&
         VER=$(flux kvs version) &&
@@ -1195,6 +1213,13 @@ test_expect_success 'flux kvs getroot --owner returns instance owner' '
 	test $OWNER -eq $(id -u)
 '
 
+test_expect_success 'flux kvs getroot --blobref returns changing blobrefs' '
+	BLOBREF1=$(flux kvs getroot --blobref) &&
+	flux kvs put test.c=barf &&
+	BLOBREF2=$(flux kvs getroot --blobref) &&
+	test $BLOBREF1 != $BLOBREF2
+'
+
 test_expect_success 'flux kvs getroot works on alt namespace' '
 	flux kvs namespace create testns1 &&
 	SEQ=$(flux kvs getroot --namespace=testns1 --sequence) &&
@@ -1220,8 +1245,8 @@ test_expect_success 'kvs: get --label works' '
 test_expect_success 'getroot request with empty payload fails with EPROTO(71)' '
 	${RPC} kvs.getroot 71 </dev/null
 '
-test_expect_success 'sync request with empty payload fails with EPROTO(71)' '
-	${RPC} kvs.sync 71 </dev/null
+test_expect_success 'wait-version request with empty payload fails with EPROTO(71)' '
+	${RPC} kvs.wait-version 71 </dev/null
 '
 test_expect_success 'lookup request with empty payload fails with EPROTO(71)' '
 	${RPC} kvs.lookup 71 </dev/null
@@ -1247,4 +1272,27 @@ test_expect_success 'setroot-pause request with empty payload fails with EPROTO(
 test_expect_success 'setroot-unpause request with empty payload fails with EPROTO(71)' '
 	${RPC} kvs.setroot-unpause 71 </dev/null
 '
+
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests before module removal' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
+#
+# module corner cases
+#
+
+test_expect_success 'module fails to load with unknown option' '
+	flux module remove kvs &&
+	test_must_fail flux module load kvs badopt
+'
+
+test_expect_success 'module fails to load with bad input to transaction-merge' '
+        test_must_fail flux module reload kvs transaction-merge=foobar
+'
+
 test_done
diff --git a/t/t1001-kvs-internals.t b/t/t1001-kvs-internals.t
index aa7d656ed16f..ec113a52f9e6 100755
--- a/t/t1001-kvs-internals.t
+++ b/t/t1001-kvs-internals.t
@@ -9,6 +9,8 @@ will generally cover non-obvious issues/features that a general user
 would be unaware of.
 '
 
+. `dirname $0`/kvs/kvs-helper.sh
+
 . `dirname $0`/sharness.sh
 
 # Size the session to one more than the number of cores, minimum of 4
@@ -24,6 +26,15 @@ test_kvs_key() {
 	test_cmp expected output
 }
 
+#
+# pending request stats are 0 at start of tests
+#
+
+test_expect_success 'kvs: no pending requests at start of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 #
 # large value test
 #
@@ -134,7 +145,7 @@ test_expect_success 'kvs: append a zero length value to zero length value works'
 #
 test_expect_success 'kvs: link: error on link depth' '
 	flux kvs unlink -Rf $DIR &&
-        flux kvs put $DIR.a=1 &&
+	flux kvs put $DIR.a=1 &&
 	flux kvs link $DIR.a $DIR.b &&
 	flux kvs link $DIR.b $DIR.c &&
 	flux kvs link $DIR.c $DIR.d &&
@@ -146,13 +157,13 @@ test_expect_success 'kvs: link: error on link depth' '
 	flux kvs link $DIR.i $DIR.j &&
 	flux kvs link $DIR.j $DIR.k &&
 	flux kvs link $DIR.k $DIR.l &&
-        test_must_fail flux kvs get $DIR.l
+	test_must_fail flux kvs get $DIR.l
 '
 test_expect_success 'kvs: link: error on link depth, loop' '
 	flux kvs unlink -Rf $DIR &&
 	flux kvs link $DIR.link1 $DIR.link2 &&
 	flux kvs link $DIR.link2 $DIR.link1 &&
-        test_must_fail flux kvs get $DIR.link1
+	test_must_fail flux kvs get $DIR.link1
 '
 
 #
@@ -164,32 +175,32 @@ largevalhash="sha1-d8cc4fd0a57d0e0e96cdb3e74164f734c593ed65"
 
 test_expect_success 'kvs: large put stores raw data into content store' '
 	flux kvs unlink -Rf $DIR &&
- 	flux kvs put $DIR.largeval=$largeval &&
+	flux kvs put $DIR.largeval=$largeval &&
 	flux kvs get --treeobj $DIR.largeval | grep -q \"valref\" &&
 	flux kvs get --treeobj $DIR.largeval | grep -q ${largevalhash} &&
 	flux content load ${largevalhash} | grep $largeval
 '
 
 test_expect_success 'kvs: valref that points to content store data can be read' '
-        flux kvs unlink -Rf $DIR &&
+	flux kvs unlink -Rf $DIR &&
 	echo "$largeval" | flux content store &&
 	flux kvs put --treeobj $DIR.largeval2="{\"data\":[\"${largevalhash}\"],\"type\":\"valref\",\"ver\":1}" &&
-        flux kvs get $DIR.largeval2 | grep $largeval
+	flux kvs get $DIR.largeval2 | grep $largeval
 '
 
 test_expect_success 'kvs: valref that points to zero size content store data can be read' '
 	flux kvs unlink -Rf $DIR &&
-        hashval=`flux content store </dev/null` &&
+	hashval=`flux content store </dev/null` &&
 	flux kvs put --treeobj $DIR.empty="{\"data\":[\"${hashval}\"],\"type\":\"valref\",\"ver\":1}" &&
 	test $(flux kvs get --raw $DIR.empty|wc -c) -eq 0
 '
 
 test_expect_success 'kvs: valref can point to other treeobjs' '
 	flux kvs unlink -Rf $DIR &&
-        flux kvs mkdir $DIR.a.b.c &&
-        dirhash=`flux kvs get --treeobj $DIR.a.b.c | grep -P "sha1-[A-Za-z0-9]+" -o` &&
+	flux kvs mkdir $DIR.a.b.c &&
+	dirhash=`flux kvs get --treeobj $DIR.a.b.c | grep -E "sha1-[A-Za-z0-9]+" -o` &&
 	flux kvs put --treeobj $DIR.value="{\"data\":[\"${dirhash}\"],\"type\":\"valref\",\"ver\":1}" &&
-        flux kvs get --raw $DIR.value | grep dir
+	flux kvs get --raw $DIR.value | grep dir
 '
 
 #
@@ -197,49 +208,49 @@ test_expect_success 'kvs: valref can point to other treeobjs' '
 #
 
 test_expect_success 'kvs: multi blob-ref valref can be read' '
-        flux kvs unlink -Rf $DIR &&
+	flux kvs unlink -Rf $DIR &&
 	hashval1=`echo -n "abcd" | flux content store` &&
 	hashval2=`echo -n "efgh" | flux content store` &&
 	flux kvs put --treeobj $DIR.multival="{\"data\":[\"${hashval1}\", \"${hashval2}\"],\"type\":\"valref\",\"ver\":1}" &&
-        flux kvs get --raw $DIR.multival | grep "abcdefgh" &&
-        test $(flux kvs get --raw $DIR.multival|wc -c) -eq 8
+	flux kvs get --raw $DIR.multival | grep "abcdefgh" &&
+	test $(flux kvs get --raw $DIR.multival|wc -c) -eq 8
 '
 
 test_expect_success 'kvs: multi blob-ref valref with an empty blobref on left, can be read' '
-        flux kvs unlink -Rf $DIR &&
+	flux kvs unlink -Rf $DIR &&
 	hashval1=`flux content store < /dev/null` &&
 	hashval2=`echo -n "abcd" | flux content store` &&
 	flux kvs put --treeobj $DIR.multival="{\"data\":[\"${hashval1}\", \"${hashval2}\"],\"type\":\"valref\",\"ver\":1}" &&
-        flux kvs get --raw $DIR.multival | grep "abcd" &&
-        test $(flux kvs get --raw $DIR.multival|wc -c) -eq 4
+	flux kvs get --raw $DIR.multival | grep "abcd" &&
+	test $(flux kvs get --raw $DIR.multival|wc -c) -eq 4
 '
 
 test_expect_success 'kvs: multi blob-ref valref with an empty blobref on right, can be read' '
-        flux kvs unlink -Rf $DIR &&
+	flux kvs unlink -Rf $DIR &&
 	hashval1=`echo -n "abcd" | flux content store` &&
 	hashval2=`flux content store < /dev/null` &&
 	flux kvs put --treeobj $DIR.multival="{\"data\":[\"${hashval1}\", \"${hashval2}\"],\"type\":\"valref\",\"ver\":1}" &&
-        flux kvs get --raw $DIR.multival | grep "abcd" &&
-        test $(flux kvs get --raw $DIR.multival|wc -c) -eq 4
+	flux kvs get --raw $DIR.multival | grep "abcd" &&
+	test $(flux kvs get --raw $DIR.multival|wc -c) -eq 4
 '
 
 test_expect_success 'kvs: multi blob-ref valref with an empty blobref in middle, can be read' '
-        flux kvs unlink -Rf $DIR &&
+	flux kvs unlink -Rf $DIR &&
 	hashval1=`echo -n "abcd" | flux content store` &&
 	hashval2=`flux content store < /dev/null` &&
 	hashval3=`echo -n "efgh" | flux content store` &&
 	flux kvs put --treeobj $DIR.multival="{\"data\":[\"${hashval1}\", \"${hashval2}\", \"${hashval3}\"],\"type\":\"valref\",\"ver\":1}" &&
-        flux kvs get --raw $DIR.multival | grep "abcdefgh" &&
-        test $(flux kvs get --raw $DIR.multival|wc -c) -eq 8
+	flux kvs get --raw $DIR.multival | grep "abcdefgh" &&
+	test $(flux kvs get --raw $DIR.multival|wc -c) -eq 8
 '
 
 test_expect_success 'kvs: multi blob-ref valref with a blobref pointing to a treeobj' '
-        flux kvs unlink -Rf $DIR &&
+	flux kvs unlink -Rf $DIR &&
 	hashval1=`echo -n "abcd" | flux content store` &&
-        flux kvs mkdir $DIR.a.b.c &&
-        dirhash=`flux kvs get --treeobj $DIR.a.b.c | grep -P "sha1-[A-Za-z0-9]+" -o` &&
+	flux kvs mkdir $DIR.a.b.c &&
+	dirhash=`flux kvs get --treeobj $DIR.a.b.c | grep -E "sha1-[A-Za-z0-9]+" -o` &&
 	flux kvs put --treeobj $DIR.multival="{\"data\":[\"${hashval1}\", \"${dirhash}\"],\"type\":\"valref\",\"ver\":1}" &&
-        flux kvs get --raw $DIR.multival | grep dir
+	flux kvs get --raw $DIR.multival | grep dir
 '
 
 #
@@ -252,54 +263,48 @@ test_expect_success 'kvs: multi blob-ref valref with a blobref pointing to a tre
 badhash="sha1-0123456789012345678901234567890123456789"
 
 test_expect_success 'kvs: invalid valref lookup wont hang' '
-        flux kvs put --treeobj $DIR.bad_valref="{\"data\":[\"${badhash}\"],\"type\":\"valref\",\"ver\":1}" &&
-        ! flux kvs get $DIR.bad_valref &&
-        ! flux kvs get $DIR.bad_valref
+	flux kvs put --treeobj $DIR.bad_valref="{\"data\":[\"${badhash}\"],\"type\":\"valref\",\"ver\":1}" &&
+	! flux kvs get $DIR.bad_valref &&
+	! flux kvs get $DIR.bad_valref
 '
 
 test_expect_success 'kvs: invalid valref get --watch wont hang' '
-        ! flux kvs get --watch $DIR.bad_valref &&
-        ! flux kvs get --watch $DIR.bad_valref
+	! flux kvs get --watch $DIR.bad_valref &&
+	! flux kvs get --watch $DIR.bad_valref
 '
 
 test_expect_success 'kvs: invalid multi-blobref valref lookup wont hang' '
-        flux kvs put --treeobj $DIR.bad_multi_valref="{\"data\":[\"${badhash}\", \"${badhash}\"],\"type\":\"valref\",\"ver\":1}" &&
-        ! flux kvs get $DIR.bad_multi_valref &&
-        ! flux kvs get $DIR.bad_multi_valref
+	flux kvs put --treeobj $DIR.bad_multi_valref="{\"data\":[\"${badhash}\", \"${badhash}\"],\"type\":\"valref\",\"ver\":1}" &&
+	! flux kvs get $DIR.bad_multi_valref &&
+	! flux kvs get $DIR.bad_multi_valref
 '
 
 test_expect_success 'kvs: invalid multi-blobref valref get --watch wont hang' '
-        ! flux kvs get --watch $DIR.bad_multi_valref &&
-        ! flux kvs get --watch $DIR.bad_multi_valref
+	! flux kvs get --watch $DIR.bad_multi_valref &&
+	! flux kvs get --watch $DIR.bad_multi_valref
 '
 
 test_expect_success 'kvs: invalid dirref lookup wont hang' '
-        flux kvs put --treeobj $DIR.bad_dirref="{\"data\":[\"${badhash}\"],\"type\":\"dirref\",\"ver\":1}" &&
-        ! flux kvs get $DIR.bad_dirref.a &&
-        ! flux kvs get $DIR.bad_dirref.a
+	flux kvs put --treeobj $DIR.bad_dirref="{\"data\":[\"${badhash}\"],\"type\":\"dirref\",\"ver\":1}" &&
+	! flux kvs get $DIR.bad_dirref.a &&
+	! flux kvs get $DIR.bad_dirref.a
 '
 
 test_expect_success 'kvs: invalid dirref get --watch wont hang' '
-        ! flux kvs get --watch $DIR.bad_dirref.a &&
-        ! flux kvs get --watch $DIR.bad_dirref.a
+	! flux kvs get --watch $DIR.bad_dirref.a &&
+	! flux kvs get --watch $DIR.bad_dirref.a
 '
 
 test_expect_success 'kvs: invalid dirref write wont hang' '
-        flux kvs put --treeobj $DIR.bad_dirref="{\"data\":[\"${badhash}\"],\"type\":\"dirref\",\"ver\":1}" &&
-        ! flux kvs put $DIR.bad_dirref.a=1 &&
-        ! flux kvs put $DIR.bad_dirref.b=1
-'
-
-test_expect_success "kvs: failure to store blob that exceeds max size does not hang" '
-        dd if=/dev/zero count=$((1048576/4096+1)) bs=4096 \
-                        skip=$((1048576/4096)) >toobig 2>/dev/null &&
-        test_must_fail flux start --size=4 -o,--setattr=content.blob-size-limit=1048576 \
-                       flux kvs put -r $DIR.bad_toobig=- <toobig
+	flux kvs put --treeobj $DIR.bad_dirref="{\"data\":[\"${badhash}\"],\"type\":\"dirref\",\"ver\":1}" &&
+	! flux kvs put $DIR.bad_dirref.a=1 &&
+	! flux kvs put $DIR.bad_dirref.b=1
 '
 
-MAXBLOB=`flux getattr content.blob-size-limit`
+# N.B. kvs test personality loads content module with blob-size-limit=1048576
+MAXBLOB=1048576
 
-test_expect_success LONGTEST "kvs: failure to store blob that exceeds max size default does not hang" '
+test_expect_success "kvs: failure to store blob that exceeds max size does not hang" '
 	dd if=/dev/zero count=$(($MAXBLOB/4096+1)) bs=4096 \
 			skip=$(($MAXBLOB/4096)) >toobig_long 2>/dev/null &&
 	test_must_fail flux kvs put -r $DIR.bad_toobig_long=- < toobig_long
@@ -326,14 +331,14 @@ test_expect_success 'kvs: unlink on rank 0, does not exist all ranks' '
 #
 
 test_expect_success 'kvs: pause / unpause works' '
-        ${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause &&
-        ${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause
+	${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause &&
+	${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause
 '
 
 # cover invalid namespace cases
 test_expect_success 'kvs: cover pause / unpause namespace invalid' '
-        ! ${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause --namespace=illegalnamespace &&
-        ! ${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause --namespace=illegalnamespace
+	! ${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause --namespace=illegalnamespace &&
+	! ${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause --namespace=illegalnamespace
 '
 
 #
@@ -345,60 +350,60 @@ test_expect_success 'kvs: cover pause / unpause namespace invalid' '
 # setroot events b/c it shouldn't be necessary.
 
 test_expect_success 'kvs: causal consistency (put)' '
-        flux kvs unlink -Rf $DIR &&
-        flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause" &&
-        ATREF=$(flux kvs put -O $DIR.testA=1) &&
-        VAL=$(flux exec -n -r 1 flux kvs get --at $ATREF $DIR.testA) &&
-        test "$VAL" = "1" &&
-        flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause" &&
-        VERS=$(flux kvs put -s $DIR.testB=2) &&
-        flux exec -n -r 2 flux kvs wait $VERS &&
-        VAL=$(flux exec -n -r 2 flux kvs get $DIR.testB) &&
-        test "$VAL" = "2" &&
-        flux exec -n -r [1-2] sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause"
+	flux kvs unlink -Rf $DIR &&
+	flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause" &&
+	ATREF=$(flux kvs put -O $DIR.testA=1) &&
+	VAL=$(flux exec -n -r 1 flux kvs get --at $ATREF $DIR.testA) &&
+	test "$VAL" = "1" &&
+	flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause" &&
+	VERS=$(flux kvs put -s $DIR.testB=2) &&
+	flux exec -n -r 2 flux kvs wait $VERS &&
+	VAL=$(flux exec -n -r 2 flux kvs get $DIR.testB) &&
+	test "$VAL" = "2" &&
+	flux exec -n -r [1-2] sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause"
 '
 
 test_expect_success 'kvs: causal consistency (mkdir)' '
-        flux kvs unlink -Rf $DIR &&
-        flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause" &&
-        ATREF=$(flux kvs mkdir -O $DIR.dirA) &&
-        ! flux exec -n -r 1 flux kvs get --at $ATREF $DIR.dirA > output 2>&1 &&
-        grep "Is a directory" output &&
-        flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause" &&
-        VERS=$(flux kvs mkdir -s $DIR.dirB) &&
-        flux exec -n -r 2 flux kvs wait $VERS &&
-        ! flux exec -n -r 2 flux kvs get $DIR.dirB > output 2>&1 &&
-        grep "Is a directory" output
+	flux kvs unlink -Rf $DIR &&
+	flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause" &&
+	ATREF=$(flux kvs mkdir -O $DIR.dirA) &&
+	! flux exec -n -r 1 flux kvs get --at $ATREF $DIR.dirA > output 2>&1 &&
+	grep "Is a directory" output &&
+	flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause" &&
+	VERS=$(flux kvs mkdir -s $DIR.dirB) &&
+	flux exec -n -r 2 flux kvs wait $VERS &&
+	! flux exec -n -r 2 flux kvs get $DIR.dirB > output 2>&1 &&
+	grep "Is a directory" output
 '
 
 test_expect_success 'kvs: causal consistency (link)' '
-        flux kvs unlink -Rf $DIR &&
-        flux kvs put $DIR.fooA=3 &&
-        flux kvs put $DIR.fooB=4 &&
-        flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause" &&
-        ATREF=$(flux kvs link -O $DIR.fooA $DIR.linkA) &&
-        VAL=$(flux exec -n -r 1 flux kvs get --at $ATREF $DIR.linkA) &&
-        test "$VAL" = "3" &&
-        flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause" &&
-        VERS=$(flux kvs link -s $DIR.fooB $DIR.linkB) &&
-        flux exec -n -r 2 flux kvs wait $VERS &&
-        VAL=$(flux exec -n -r 2 flux kvs get $DIR.linkB) &&
-        test "$VAL" = "4"
+	flux kvs unlink -Rf $DIR &&
+	flux kvs put $DIR.fooA=3 &&
+	flux kvs put $DIR.fooB=4 &&
+	flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause" &&
+	ATREF=$(flux kvs link -O $DIR.fooA $DIR.linkA) &&
+	VAL=$(flux exec -n -r 1 flux kvs get --at $ATREF $DIR.linkA) &&
+	test "$VAL" = "3" &&
+	flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause" &&
+	VERS=$(flux kvs link -s $DIR.fooB $DIR.linkB) &&
+	flux exec -n -r 2 flux kvs wait $VERS &&
+	VAL=$(flux exec -n -r 2 flux kvs get $DIR.linkB) &&
+	test "$VAL" = "4"
 '
 
 test_expect_success 'kvs: causal consistency (unlink)' '
-        flux kvs unlink -Rf $DIR &&
-        flux kvs put $DIR.unlinkA=5 &&
-        flux kvs put $DIR.unlinkB=6 &&
-        flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause" &&
-        ATREF=$(flux kvs unlink -O $DIR.unlinkA) &&
-        ! flux exec -n -r 1 flux kvs get --at $ATREF $DIR.unlinkA > output 2>&1 &&
-        grep "No such file or directory" output &&
-        flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause" &&
-        VERS=$(flux kvs unlink -s $DIR.unlinkB) &&
-        flux exec -n -r 2 flux kvs wait $VERS &&
-        ! flux exec -n -r 2 flux kvs get $DIR.unlinkB > output 2>&1 &&
-        grep "No such file or directory" output
+	flux kvs unlink -Rf $DIR &&
+	flux kvs put $DIR.unlinkA=5 &&
+	flux kvs put $DIR.unlinkB=6 &&
+	flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause" &&
+	ATREF=$(flux kvs unlink -O $DIR.unlinkA) &&
+	! flux exec -n -r 1 flux kvs get --at $ATREF $DIR.unlinkA > output 2>&1 &&
+	grep "No such file or directory" output &&
+	flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause" &&
+	VERS=$(flux kvs unlink -s $DIR.unlinkB) &&
+	flux exec -n -r 2 flux kvs wait $VERS &&
+	! flux exec -n -r 2 flux kvs get $DIR.unlinkB > output 2>&1 &&
+	grep "No such file or directory" output
 '
 
 #
@@ -414,46 +419,46 @@ test_expect_success 'kvs: causal consistency (unlink)' '
 # - change should be visible on X & Y
 
 test_expect_success 'kvs: read-your-writes consistency on primary namespace' '
-        flux kvs unlink -Rf $DIR &&
-        flux kvs put $DIR.test=1 &&
-        VERS=$(flux kvs version) &&
-        flux exec -n sh -c "flux kvs wait ${VERS}" &&
-        flux exec -n -r 2 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause" &&
-        flux exec -n -r 1 sh -c "flux kvs put $DIR.test=2" &&
-        flux exec -n -r 1 sh -c "flux kvs get $DIR.test" > rank1-a.out &&
-        flux exec -n -r 2 sh -c "flux kvs get $DIR.test" > rank2-a.out &&
-        echo "1" > old.out &&
-        echo "2" > new.out &&
-        test_cmp rank1-a.out new.out &&
-        test_cmp rank2-a.out old.out &&
-        flux exec -n -r 2 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause" &&
-        flux exec -n sh -c "flux kvs wait ${VERS}" &&
-        flux exec -n -r 1 sh -c "flux kvs get $DIR.test" > rank1-b.out &&
-        flux exec -n -r 2 sh -c "flux kvs get $DIR.test" > rank2-b.out &&
-        test_cmp rank1-b.out new.out &&
-        test_cmp rank2-b.out new.out
+	flux kvs unlink -Rf $DIR &&
+	flux kvs put $DIR.test=1 &&
+	VERS=$(flux kvs version) &&
+	flux exec -n sh -c "flux kvs wait ${VERS}" &&
+	flux exec -n -r 2 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause" &&
+	flux exec -n -r 1 sh -c "flux kvs put $DIR.test=2" &&
+	flux exec -n -r 1 sh -c "flux kvs get $DIR.test" > rank1-a.out &&
+	flux exec -n -r 2 sh -c "flux kvs get $DIR.test" > rank2-a.out &&
+	echo "1" > old.out &&
+	echo "2" > new.out &&
+	test_cmp rank1-a.out new.out &&
+	test_cmp rank2-a.out old.out &&
+	flux exec -n -r 2 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause" &&
+	flux exec -n sh -c "flux kvs wait ${VERS}" &&
+	flux exec -n -r 1 sh -c "flux kvs get $DIR.test" > rank1-b.out &&
+	flux exec -n -r 2 sh -c "flux kvs get $DIR.test" > rank2-b.out &&
+	test_cmp rank1-b.out new.out &&
+	test_cmp rank2-b.out new.out
 '
 
 test_expect_success 'kvs: read-your-writes consistency on alt namespace' '
-        flux kvs namespace create rywtestns &&
-        flux kvs put --namespace=rywtestns $DIR.test=1 &&
-        VERS=$(flux kvs version --namespace=rywtestns) &&
-        flux exec -n sh -c "flux kvs wait --namespace=rywtestns ${VERS}" &&
-        flux exec -n -r 2 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause --namespace=rywtestns" &&
-        flux exec -n -r 1 sh -c "flux kvs put --namespace=rywtestns $DIR.test=2" &&
-        flux exec -n -r 1 sh -c "flux kvs get --namespace=rywtestns $DIR.test" > rank1-a.out &&
-        flux exec -n -r 2 sh -c "flux kvs get --namespace=rywtestns $DIR.test" > rank2-a.out &&
-        echo "1" > old.out &&
-        echo "2" > new.out &&
-        test_cmp rank1-a.out new.out &&
-        test_cmp rank2-a.out old.out &&
-        flux exec -n -r 2 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause --namespace=rywtestns" &&
-        flux exec -n sh -c "flux kvs wait --namespace=rywtestns ${VERS}" &&
-        flux exec -n -r 1 sh -c "flux kvs get --namespace=rywtestns $DIR.test" > rank1-b.out &&
-        flux exec -n -r 2 sh -c "flux kvs get --namespace=rywtestns $DIR.test" > rank2-b.out &&
-        test_cmp rank1-b.out new.out &&
-        test_cmp rank2-b.out new.out &&
-        flux kvs namespace remove rywtestns
+	flux kvs namespace create rywtestns &&
+	flux kvs put --namespace=rywtestns $DIR.test=1 &&
+	VERS=$(flux kvs version --namespace=rywtestns) &&
+	flux exec -n sh -c "flux kvs wait --namespace=rywtestns ${VERS}" &&
+	flux exec -n -r 2 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --pause --namespace=rywtestns" &&
+	flux exec -n -r 1 sh -c "flux kvs put --namespace=rywtestns $DIR.test=2" &&
+	flux exec -n -r 1 sh -c "flux kvs get --namespace=rywtestns $DIR.test" > rank1-a.out &&
+	flux exec -n -r 2 sh -c "flux kvs get --namespace=rywtestns $DIR.test" > rank2-a.out &&
+	echo "1" > old.out &&
+	echo "2" > new.out &&
+	test_cmp rank1-a.out new.out &&
+	test_cmp rank2-a.out old.out &&
+	flux exec -n -r 2 sh -c "${FLUX_BUILD_DIR}/t/kvs/setrootevents --unpause --namespace=rywtestns" &&
+	flux exec -n sh -c "flux kvs wait --namespace=rywtestns ${VERS}" &&
+	flux exec -n -r 1 sh -c "flux kvs get --namespace=rywtestns $DIR.test" > rank1-b.out &&
+	flux exec -n -r 2 sh -c "flux kvs get --namespace=rywtestns $DIR.test" > rank2-b.out &&
+	test_cmp rank1-b.out new.out &&
+	test_cmp rank2-b.out new.out &&
+	flux kvs namespace remove rywtestns
 '
 
 #
@@ -464,26 +469,26 @@ test_expect_success 'kvs: read-your-writes consistency on alt namespace' '
 # know that the identical large value will be cached as raw data
 
 test_expect_success 'kvs: clear stats locally' '
-        flux kvs unlink -Rf $DIR &&
-        flux module stats -c kvs &&
-        flux module stats --parse "namespace.primary.#no-op stores" kvs | grep -q 0 &&
-        flux kvs put $DIR.largeval1=$largeval &&
-        flux kvs put $DIR.largeval2=$largeval &&
-        ! flux module stats --parse "namespace.primary.#no-op stores" kvs | grep -q 0 &&
-        flux module stats -c kvs &&
-        flux module stats --parse "namespace.primary.#no-op stores" kvs | grep -q 0
+	flux kvs unlink -Rf $DIR &&
+	flux module stats -c kvs &&
+	flux module stats --parse "namespace.primary.#no-op stores" kvs | grep -q 0 &&
+	flux kvs put $DIR.largeval1=$largeval &&
+	flux kvs put $DIR.largeval2=$largeval &&
+	! flux module stats --parse "namespace.primary.#no-op stores" kvs | grep -q 0 &&
+	flux module stats -c kvs &&
+	flux module stats --parse "namespace.primary.#no-op stores" kvs | grep -q 0
 '
 
-test_expect_success 'kvs: clear stats globally' '
-        flux kvs unlink -Rf $DIR &&
-        flux module stats -C kvs &&
-        flux exec -n sh -c "flux module stats --parse \"namespace.primary.#no-op stores\" kvs | grep -q 0" &&
-        for i in `seq 0 $((${SIZE} - 1))`; do
-            flux exec -n -r $i sh -c "flux kvs put $DIR.$i.largeval1=$largeval $DIR.$i.largeval2=$largeval"
-        done &&
-        ! flux exec -n sh -c "flux module stats --parse \"namespace.primary.#no-op stores\" kvs | grep -q 0" &&
-        flux module stats -C kvs &&
-        flux exec -n sh -c "flux module stats --parse \"namespace.primary.#no-op stores\" kvs | grep -q 0"
+test_expect_success NO_ASAN 'kvs: clear stats globally' '
+	flux kvs unlink -Rf $DIR &&
+	flux module stats -C kvs &&
+	flux exec -n sh -c "flux module stats --parse \"namespace.primary.#no-op stores\" kvs | grep -q 0" &&
+	for i in `seq 0 $((${SIZE} - 1))`; do
+	    flux exec -n -r $i sh -c "flux kvs put $DIR.$i.largeval1=$largeval $DIR.$i.largeval2=$largeval"
+	done &&
+	! flux exec -n sh -c "flux module stats --parse \"namespace.primary.#no-op stores\" kvs | grep -q 0" &&
+	flux module stats -C kvs &&
+	flux exec -n sh -c "flux module stats --parse \"namespace.primary.#no-op stores\" kvs | grep -q 0"
 '
 
 #
@@ -491,7 +496,7 @@ test_expect_success 'kvs: clear stats globally' '
 #
 
 test_expect_success 'kvs: test fence returns identical root info on all responses' '
-        ${FLUX_BUILD_DIR}/t/kvs/fence_api 8 apitest
+	${FLUX_BUILD_DIR}/t/kvs/fence_api 8 apitest
 '
 
 #
@@ -499,13 +504,13 @@ test_expect_success 'kvs: test fence returns identical root info on all response
 #
 
 test_expect_success 'kvs: test invalid fence arguments on rank 0' '
-        ${FLUX_BUILD_DIR}/t/kvs/fence_invalid invalidtest1 > invalid_output &&
-        grep "flux_future_get: Invalid argument" invalid_output
+	${FLUX_BUILD_DIR}/t/kvs/fence_invalid invalidtest1 > invalid_output &&
+	grep "flux_future_get: Invalid argument" invalid_output
 '
 
 test_expect_success 'kvs: test invalid fence arguments on rank 1' '
-        flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/fence_invalid invalidtest2" > invalid_output &&
-        grep "flux_future_get: Invalid argument" invalid_output
+	flux exec -n -r 1 sh -c "${FLUX_BUILD_DIR}/t/kvs/fence_invalid invalidtest2" > invalid_output &&
+	grep "flux_future_get: Invalid argument" invalid_output
 '
 
 #
@@ -513,8 +518,56 @@ test_expect_success 'kvs: test invalid fence arguments on rank 1' '
 #
 
 test_expect_success 'kvs: test invalid lookup rpc' '
-        ${FLUX_BUILD_DIR}/t/kvs/lookup_invalid a-key > lookup_invalid_output &&
-        grep "flux_future_get: Protocol error" lookup_invalid_output
+	${FLUX_BUILD_DIR}/t/kvs/lookup_invalid a-key > lookup_invalid_output &&
+	grep "flux_future_get: Protocol error" lookup_invalid_output
+'
+
+#
+# ensure pending requests are the expected number
+#
+
+# fence invalid tests above linger a pending request on both rank 0 and rank 1
+test_expect_success 'kvs: 1 pending requests at end of tests before module removal' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 1 &&
+	pendingcount1=$(flux exec -n -r 1 sh -c "flux module stats -p pending_requests kvs") &&
+	test $pendingcount1 -eq 1
+'
+
+#
+# test ENOSYS on unfinished requests when unloading the KVS module
+#
+# N.B. do this last as we are unloading the kvs module
+#
+
+wait_versionwaiters() {
+	num=$1
+	i=0
+	while [ "$(flux module stats --parse namespace.primary.#versionwaiters kvs 2> /dev/null)" != "${num}" ] \
+	      && [ $i -lt ${KVS_WAIT_ITERS} ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	return $(loophandlereturn $i)
+}
+
+# In order to test, wait for a version that will not happen
+test_expect_success NO_CHAIN_LINT 'kvs: ENOSYS returned on unfinished requests on module unload' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	pendingcountexp=$(($pendingcount+1)) &&
+	WAITCOUNT=$(flux module stats --parse namespace.primary.#versionwaiters kvs) &&
+	WAITCOUNT=$(($WAITCOUNT+1))
+	VERS=$(flux kvs version) &&
+	VERSWAIT=$(($VERS+10)) &&
+	flux kvs wait ${VERSWAIT} 2> enosys.err &
+	pid=$! &&
+	wait_versionwaiters ${WAITCOUNT} &&
+	newcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcountexp -eq $newcount &&
+	flux module remove kvs &&
+	! wait $pid &&
+	grep "Function not implemented" enosys.err
 '
 
 test_done
diff --git a/t/t1003-kvs-stress.t b/t/t1003-kvs-stress.t
index 64f3ea653b35..40cc23504ca7 100755
--- a/t/t1003-kvs-stress.t
+++ b/t/t1003-kvs-stress.t
@@ -42,7 +42,7 @@ test_expect_success 'kvs: store 2x4 directory tree and walk' '
 test_expect_success 'kvs: add other types to 2x4 directory and walk' '
 	flux kvs link $DIR.dtree $DIR.dtree.link &&
 	flux kvs put $DIR.dtree.double=3.14 &&
-	flux kvs put $DIR.dtree.booelan=true &&
+	flux kvs put $DIR.dtree.boolean=true &&
 	test $(flux kvs dir -R $DIR.dtree | wc -l) = 19
 '
 
@@ -97,18 +97,10 @@ test_expect_success 'kvs: 8 threads/rank each doing 100 put,fence in a loop, mix
 
 # large dirs
 
-test_expect_success 'kvs: store value exceeding RFC 10 max blob size of 1m' '
-	${FLUX_BUILD_DIR}/t/kvs/torture --prefix $DIR.tortureval --count 1 --size=1048577
-'
-
 test_expect_success 'kvs: store 10,000 keys in one dir' '
 	${FLUX_BUILD_DIR}/t/kvs/torture --prefix $DIR.bigdir --count 10000
 '
 
-test_expect_success LONGTEST 'kvs: store 100,000 keys in one dir' '
-	${FLUX_BUILD_DIR}/t/kvs/torture --prefix $DIR.bigdir2 --count 100000
-'
-
 # kvs merging tests
 
 # If transaction-merge=1 and we set KVS_NO_MERGE on all commits, this test
@@ -131,4 +123,13 @@ test_expect_success 'kvs: transaction-merge disabling works' '
 	test "$OUTPUT" = "${THREADS}"
 '
 
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done
diff --git a/t/t1004-kvs-namespace.t b/t/t1004-kvs-namespace.t
index 2636dfcce6bc..e46365c54d6e 100755
--- a/t/t1004-kvs-namespace.t
+++ b/t/t1004-kvs-namespace.t
@@ -27,6 +27,7 @@ NAMESPACETEST=namespacetest
 NAMESPACETMP=namespacetmp
 NAMESPACERANK1=namespacerank1
 NAMESPACEORDER=namespaceorder
+NAMESPACEROOTREF=namespacerootref
 
 namespace_create_loop() {
         i=0
@@ -72,10 +73,10 @@ wait_fencecount_nonzero() {
         return $(loophandlereturn $i)
 }
 
-wait_syncers_nonzero() {
+wait_versionwaiters_nonzero() {
         i=0
-        while (! flux module stats --parse namespace.$1.#syncers kvs > /dev/null 2>&1 \
-               || [ "$(flux module stats --parse namespace.$1.#syncers kvs 2> /dev/null)" = "0" ]) \
+        while (! flux module stats --parse namespace.$1.#versionwaiters kvs > /dev/null 2>&1 \
+               || [ "$(flux module stats --parse namespace.$1.#versionwaiters kvs 2> /dev/null)" = "0" ]) \
               && [ $i -lt ${KVS_WAIT_ITERS} ]
         do
                 sleep 0.1
@@ -339,6 +340,39 @@ test_expect_success 'kvs: put - namespace specified in command line overrides en
         test_kvs_key_namespace $NAMESPACEORDER-2 $DIR.puttest 5
 '
 
+#
+# Namespace rootref initialization
+#
+
+test_expect_success 'kvs: namespace rootref setup' '
+	flux kvs namespace create $NAMESPACEROOTREF-1 &&
+        flux kvs put --namespace=$NAMESPACEROOTREF-1 $DIR.rootreftest=foobar &&
+        test_kvs_key_namespace $NAMESPACEROOTREF-1 $DIR.rootreftest foobar &&
+        flux kvs getroot --blobref --namespace=$NAMESPACEROOTREF-1 > rootref1
+'
+
+test_expect_success 'kvs: namespace create with init rootref' '
+	flux kvs namespace create --rootref=$(cat rootref1) $NAMESPACEROOTREF-2 &&
+        test_kvs_key_namespace $NAMESPACEROOTREF-1 $DIR.rootreftest foobar
+'
+
+test_expect_success 'kvs: namespaces dont clobber each other' '
+        flux kvs put --namespace=$NAMESPACEROOTREF-1 $DIR.val=42 &&
+        flux kvs put --namespace=$NAMESPACEROOTREF-2 $DIR.val=43 &&
+        test_kvs_key_namespace $NAMESPACEROOTREF-1 $DIR.val 42 &&
+        test_kvs_key_namespace $NAMESPACEROOTREF-2 $DIR.val 43
+'
+
+BADROOTREF="sha1-0123456789abcdef0123456789abcdef01234567"
+test_expect_success 'kvs: namespace create can take bad blobref' '
+	flux kvs namespace create --rootref=$BADROOTREF $NAMESPACEROOTREF-3 &&
+        flux kvs get --namespace=$NAMESPACEROOTREF-3 --treeobj .
+'
+
+test_expect_success 'kvs: namespace with bad rootref fails otherwise' '
+        test_must_fail flux kvs ls --namespace=$NAMESPACEROOTREF-3 .
+'
+
 #
 # Namespace corner case tests
 #
@@ -397,7 +431,7 @@ test_expect_success NO_CHAIN_LINT 'kvs: incomplete fence gets ENOTSUP when names
         wait_fencecount_nonzero 0 $NAMESPACETMP-REMOVE-FENCE0 &&
         flux kvs namespace remove $NAMESPACETMP-REMOVE-FENCE0 &&
         wait $watchpid &&
-        grep "flux_future_get: Operation not supported" fence_out
+        grep "flux_future_get: $(strerror_symbol ENOTSUP)" fence_out
 '
 
 
@@ -413,7 +447,7 @@ test_expect_success NO_CHAIN_LINT 'kvs: incomplete fence on rank 1 gets ENOTSUP
         wait_fencecount_nonzero 1 $NAMESPACETMP-REMOVE-FENCE1 &&
         flux kvs namespace remove $NAMESPACETMP-REMOVE-FENCE1 &&
         wait $watchpid &&
-        grep "flux_future_get: Operation not supported" fence_out
+        grep "flux_future_get: $(strerror_symbol ENOTSUP)" fence_out
 '
 
 test_expect_success NO_CHAIN_LINT 'kvs: wait recognizes removed namespace' '
@@ -422,10 +456,10 @@ test_expect_success NO_CHAIN_LINT 'kvs: wait recognizes removed namespace' '
         VERS=$((VERS + 1)) &&
         flux kvs wait --namespace=$NAMESPACETMP-REMOVE-WAIT $VERS > wait_out 2>&1 &
         waitpid=$! &&
-        wait_syncers_nonzero $NAMESPACETMP-REMOVE-WAIT &&
+        wait_versionwaiters_nonzero $NAMESPACETMP-REMOVE-WAIT &&
         flux kvs namespace remove $NAMESPACETMP-REMOVE-WAIT &&
         ! wait $waitpid &&
-        grep "flux_kvs_wait_version: Operation not supported" wait_out
+        grep "flux_kvs_wait_version: $(strerror_symbol ENOTSUP)" wait_out
 '
 
 #
@@ -512,4 +546,13 @@ test_expect_success NO_CHAIN_LINT 'kvs: wait in different namespaces works' '
         test_expect_code 0 wait $testkvswaitpid
 '
 
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done
diff --git a/t/t1005-kvs-security.t b/t/t1005-kvs-security.t
index 650f8a3eef82..94de381e1750 100755
--- a/t/t1005-kvs-security.t
+++ b/t/t1005-kvs-security.t
@@ -9,15 +9,13 @@ These are tests for ensuring multiple namespaces work.
 
 . `dirname $0`/sharness.sh
 
-if test "$TEST_LONG" = "t"; then
-    test_set_prereq LONGTEST
-fi
-
 # Size the session to one more than the number of cores, minimum of 4
 SIZE=$(test_size_large)
 test_under_flux ${SIZE} kvs
 echo "# $0: flux session size will be ${SIZE}"
 
+FENCEAPI="${FLUX_BUILD_DIR}/t/kvs/fence_api"
+
 DIR=test.a.b
 
 waitfile=${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua
@@ -278,12 +276,133 @@ test_expect_success 'kvs: symlinkw/ Namespace fails (wrong user)' '
         unset_userid
 '
 
-test_expect_success 'kvs: symlink w/ Namespace works (user)' '
-        set_userid 9001 &&
-        flux kvs put --namespace=${NAMESPACETMP}-SYMLINKNS3 $DIR.linktest=3 &&
-        flux kvs link --namespace=${NAMESPACETMP}-SYMLINKNS2 --target-namespace=${NAMESPACETMP}-SYMLINKNS3 $DIR.linktest $DIR.link &&
-        test_kvs_key_namespace ${NAMESPACETMP}-SYMLINKNS2 $DIR.link 3 &&
-        unset_userid
+#
+# Basic tests, guest commits are limited
+#
+
+test_expect_success 'kvs: create test ns for user 9999' '
+	flux kvs namespace create -o 9999 $NAMESPACETMP-SYMLINK
+'
+test_expect_success 'kvs: owner can make a symlink in the test ns' '
+	flux kvs link --namespace=$NAMESPACETMP-SYMLINK a b
+'
+test_expect_success 'kvs: guest can put a val in the test ns' '
+	set_userid 9999 &&
+	flux kvs put --namespace=$NAMESPACETMP-SYMLINK aa=42 &&
+	unset_userid
+'
+test_expect_success 'kvs: guest can unlink a val in the test ns' '
+	set_userid 9999 &&
+	flux kvs unlink --namespace=$NAMESPACETMP-SYMLINK aa &&
+	unset_userid
+'
+test_expect_success 'kvs: guest can make an empty dir in the test ns' '
+	set_userid 9999 &&
+	flux kvs mkdir --namespace=$NAMESPACETMP-SYMLINK bb &&
+	unset_userid
+'
+test_expect_success 'kvs: guest can unlink a dir in the test ns' '
+	set_userid 9999 &&
+	flux kvs unlink --namespace=$NAMESPACETMP-SYMLINK bb &&
+	unset_userid
+'
+test_expect_success 'kvs: guest cannot make a symlink in the test ns' '
+	set_userid 9999 &&
+	test_must_fail flux kvs link \
+	    --namespace=$NAMESPACETMP-SYMLINK c d 2>link.err &&
+	grep "Operation not permitted" link.err &&
+	unset_userid
+'
+test_expect_success 'kvs: guest can put a val treeobj' "
+	set_userid 9999 &&
+	flux kvs put --treeobj \
+	    --namespace=$NAMESPACETMP-SYMLINK \
+	    cc='{\"data\":\"Yg==\",\"type\":\"val\",\"ver\":1}' &&
+	unset_userid
+"
+test_expect_success 'kvs: guest cannot put a symlink treeobj' "
+	set_userid 9999 &&
+	test_must_fail flux kvs put --treeobj \
+	    --namespace=$NAMESPACETMP-SYMLINK \
+	    y='{\"data\":{\"target\":\"x\"},\"type\":\"symlink\",\"ver\":1}' \
+	    2>treeobj_symlink.err &&
+	grep 'Operation not permitted' treeobj_symlink.err &&
+	unset_userid
+"
+test_expect_success 'kvs: guest cannot put a valref treeobj' "
+	set_userid 9999 &&
+	test_must_fail flux kvs put --treeobj \
+	    --namespace=$NAMESPACETMP-SYMLINK \
+	    z='{\"data\":[\"sha1-8727ddf86fd56772f4ed38703d24d93e9d9e7fa6\"],\"type\":\"valref\",\"ver\":1}' \
+	    2>treeobj_valref.err &&
+	grep 'Operation not permitted' treeobj_valref.err &&
+	unset_userid
+"
+test_expect_success 'kvs: guest cannot put a dirref treeobj' "
+	set_userid 9999 &&
+	test_must_fail flux kvs put --treeobj \
+	    --namespace=$NAMESPACETMP-SYMLINK \
+	    z='{\"data\":[\"sha1-8727ddf86fd56772f4ed38703d24d93e9d9e7fa6\"],\"type\":\"dirref\",\"ver\":1}' \
+	    2>treeobj_dirref.err &&
+	grep 'Operation not permitted' treeobj_dirref.err &&
+	unset_userid
+"
+test_expect_success 'kvs: guest cannot put a non-empty dir treeobj' "
+	set_userid 9999 &&
+	test_must_fail flux kvs put --treeobj \
+	    --namespace=$NAMESPACETMP-SYMLINK \
+	    q='{\"data\":{\"b\":{\"data\":\"NDI=\",\"type\":\"val\",\"ver\":1}},\"type\":\"dir\",\"ver\":1}' \
+	    2>treeobj_nonemptydir.err &&
+	grep 'Operation not permitted' treeobj_nonemptydir.err &&
+	unset_userid
+"
+test_expect_success 'kvs: guest cannot put a dir treeobj containing symlink' "
+	set_userid 9999 &&
+	test_must_fail flux kvs put --treeobj \
+	    --namespace=$NAMESPACETMP-SYMLINK \
+	    qq='{\"data\":{\"b\":{\"data\":{\"target\":\"x\"},\"type\":\"symlink\",\"ver\":1}},\"type\":\"dir\",\"ver\":1}' \
+	    2>treeobj_symdir.err &&
+	    grep 'Operation not permitted' treeobj_symdir.err &&
+	unset_userid
+"
+test_expect_success 'kvs: owner can make a symlink in the test ns on rank 1' '
+	flux exec -n -r 1 \
+	    sh -c "flux kvs link --namespace=$NAMESPACETMP-SYMLINK e f"
+'
+test_expect_success 'kvs: guest cannot make a symlink in the test ns on rank 1' '
+	test_must_fail flux exec -n -r 1 \
+	    sh -c "FLUX_HANDLE_USERID=9999 FLUX_HANDLE_ROLEMASK=0x2 \
+		flux kvs link --namespace=$NAMESPACETMP-SYMLINK g h" \
+	    2>link2.err &&
+	grep "Operation not permitted" link2.err
+'
+
+test_expect_success 'kvs: owner can make a symlink via fence' '
+	${FENCEAPI} --namespace=$NAMESPACETMP-SYMLINK --symlink 4 fencetest1
+'
+
+test_expect_success 'kvs: guest cannot make a symlink via fence' '
+	set_userid 9999 &&
+	test_must_fail ${FENCEAPI} \
+	    --namespace=$NAMESPACETMP-SYMLINK --symlink 4 fencetest2 \
+	    2>link5.err &&
+	grep "Operation not permitted" link5.err &&
+	unset_userid
+'
+
+test_expect_success 'kvs: owner can make a symlink via fence on rank 1' '
+	flux exec -n -r 1 \
+	    sh -c "${FENCEAPI} --namespace=$NAMESPACETMP-SYMLINK \
+		--symlink 4 fencetest3"
+'
+
+test_expect_success 'kvs: guest cannot make a symlink via fence on rank 1' '
+	test_must_fail flux exec -n -r 1 \
+	    sh -c "FLUX_HANDLE_USERID=9999 FLUX_HANDLE_ROLEMASK=0x2 \
+		${FENCEAPI} --namespace=$NAMESPACETMP-SYMLINK \
+		--symlink 4 fencetest4" \
+		2>link6.err &&
+	grep "Operation not permitted" link6.err
 '
 
 #
@@ -296,9 +415,15 @@ test_expect_success 'kvs: dropcache fails (user)' '
         unset_userid
 '
 
-test_expect_success 'kvs: stats fails (user)' '
+test_expect_success 'kvs: stats works (user)' '
         set_userid 9999 &&
-        ! flux module stats kvs &&
+        flux module stats kvs >/dev/null &&
+        unset_userid
+'
+
+test_expect_success 'kvs-watch: stats works (user)' '
+        set_userid 9999 &&
+        flux module stats kvs-watch >/dev/null &&
         unset_userid
 '
 
@@ -308,4 +433,13 @@ test_expect_success 'kvs: stats clear fails (user)' '
         unset_userid
 '
 
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done
diff --git a/t/t1007-kvs-lookup-watch.t b/t/t1007-kvs-lookup-watch.t
index d627a5ec32a4..a273f62b5b97 100755
--- a/t/t1007-kvs-lookup-watch.t
+++ b/t/t1007-kvs-lookup-watch.t
@@ -106,7 +106,7 @@ test_expect_success NO_CHAIN_LINT 'flux kvs get --watch terminated by namespace
 	! wait $pid
 '
 
-test_expect_success NO_CHAIN_LINT 'flux kvs get --watch sees duplicate commited values' '
+test_expect_success NO_CHAIN_LINT 'flux kvs get --watch sees duplicate committed values' '
 	flux kvs put test.f=1 &&
 
 	flux kvs get --count=20 --watch test.f >seq3.out &
@@ -120,7 +120,7 @@ test_expect_success NO_CHAIN_LINT 'flux kvs get --watch sees duplicate commited
 	wait $pid
 '
 
-test_expect_success NO_CHAIN_LINT 'flux kvs get --watch and --uniq do not see duplicate commited values' '
+test_expect_success NO_CHAIN_LINT 'flux kvs get --watch and --uniq do not see duplicate committed values' '
 	flux kvs put test.f=1 &&
 
 	flux kvs get --count=3 --watch --uniq test.f >seq4.out &
@@ -225,7 +225,7 @@ test_expect_success NO_CHAIN_LINT 'flux kvs get, --watch & --waitcreate, create
                   --pattern="0" waitcreate4.out >/dev/null &&
         flux kvs namespace remove ns_create_and_remove &&
         ! wait $pid &&
-        grep "Operation not supported" waitcreate4.out
+        grep "$(strerror_symbol ENOTSUP)" waitcreate4.out
 '
 
 test_expect_success NO_CHAIN_LINT 'flux kvs get, --watch & --waitcreate, doesnt work on removed namespace' '
@@ -237,7 +237,7 @@ test_expect_success NO_CHAIN_LINT 'flux kvs get, --watch & --waitcreate, doesnt
         wait_watcherscount_nonzero ns_remove &&
         flux kvs namespace remove ns_remove &&
         ! wait $pid &&
-        grep "Operation not supported" waitcreate5.out
+        grep "$(strerror_symbol ENOTSUP)" waitcreate5.out
 '
 
 #
@@ -264,6 +264,31 @@ f
         test_cmp expected append1.out
 '
 
+# N.B. When the data is small `flux kvs put foo=...` create a "val" treeobj.
+# when the value is larger, it creates a "valref" treeobj
+largeval="abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"
+
+test_expect_success NO_CHAIN_LINT 'flux kvs get: basic --watch & --append works (initial valref)' '
+        flux kvs unlink -Rf test &&
+        echo -n ${largeval} | flux kvs put --raw test.append.test=- &&
+        flux kvs get --treeobj test.append.test | grep valref &&
+        flux kvs get --watch --append --count=4 \
+                     test.append.test > append1.out 2>&1 &
+        pid=$! &&
+        wait_watcherscount_nonzero primary &&
+        flux kvs put --append test.append.test="1" &&
+        flux kvs put --append test.append.test="2" &&
+        flux kvs put --append test.append.test="3" &&
+        wait $pid &&
+	cat >expected <<-EOF &&
+abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz
+1
+2
+3
+	EOF
+        test_cmp expected append1.out
+'
+
 test_expect_success NO_CHAIN_LINT 'flux kvs get: --append works with empty string' '
         flux kvs unlink -Rf test &&
         flux kvs put test.append.test="abc" &&
@@ -305,7 +330,7 @@ f
 
 test_expect_success NO_CHAIN_LINT 'flux kvs get: --append works with multiple appends in a transaction' '
         flux kvs unlink -Rf test &&
-        flux kvs get --watch --waitcreate --append --count=4 \
+        flux kvs get --watch --waitcreate --append --count=7 \
                      test.append.test > append4.out 2>&1 &
         pid=$! &&
         wait_watcherscount_nonzero primary &&
@@ -316,9 +341,12 @@ test_expect_success NO_CHAIN_LINT 'flux kvs get: --append works with multiple ap
         wait $pid &&
 	cat >expected <<-EOF &&
 abc
-de
-fg
-hi
+d
+e
+f
+g
+h
+i
 	EOF
         test_cmp expected append4.out
 '
@@ -358,6 +386,7 @@ flux-kvs: test.append.test: No such file or directory
         test_cmp expected append5.out
 '
 
+# N.B. valref treeobj expected, but treeobj is now a dirref
 test_expect_success NO_CHAIN_LINT 'flux kvs get: --append fails on change to non-value' '
         flux kvs unlink -Rf test &&
         flux kvs put test.append.test="abc" &&
@@ -373,12 +402,13 @@ test_expect_success NO_CHAIN_LINT 'flux kvs get: --append fails on change to non
 abc
 d
 e
-flux-kvs: test.append.test: Is a directory
+flux-kvs: test.append.test: Invalid argument
 	EOF
         test_cmp expected append6.out
 '
 
-test_expect_success NO_CHAIN_LINT 'flux kvs get: --append works on fake append' '
+# N.B. valref treeobj expected, but treeobj is now a val
+test_expect_success NO_CHAIN_LINT 'flux kvs get: --append fails on fake append' '
         flux kvs unlink -Rf test &&
         flux kvs put test.append.test="abc" &&
         flux kvs get --watch --append --count=4 \
@@ -388,73 +418,22 @@ test_expect_success NO_CHAIN_LINT 'flux kvs get: --append works on fake append'
         flux kvs put --append test.append.test="d" &&
         flux kvs put --append test.append.test="e" &&
         flux kvs put test.append.test="abcdef" &&
-        wait $pid &&
-	cat >expected <<-EOF &&
-abc
-d
-e
-f
-	EOF
-        test_cmp expected append7.out
-'
-
-test_expect_success NO_CHAIN_LINT 'flux kvs get: --append works on fake append wiping data' '
-        flux kvs unlink -Rf test &&
-        flux kvs put test.append.test="abc" &&
-        flux kvs get --watch --append --count=4 \
-                     test.append.test > append8.out 2>&1 &
-        pid=$! &&
-        wait_watcherscount_nonzero primary &&
-        flux kvs put --append test.append.test="d" &&
-        flux kvs put --append test.append.test="e" &&
-        flux kvs put test.append.test="foobar" &&
-        wait $pid &&
-	cat >expected <<-EOF &&
-abc
-d
-e
-r
-	EOF
-        test_cmp expected append8.out
-'
-
-test_expect_success NO_CHAIN_LINT 'flux kvs get: --append works on fake zero length append' '
-        flux kvs unlink -Rf test &&
-        flux kvs put test.append.test="abc" &&
-        flux kvs get --watch --append --count=4 \
-                     test.append.test > append9.out 2>&1 &
-        pid=$! &&
-        wait_watcherscount_nonzero primary &&
-        flux kvs put --append test.append.test="d" &&
-        flux kvs put --append test.append.test="e" &&
-        flux kvs put test.append.test="abcde" &&
-        wait $pid &&
-	cat >expected <<-EOF &&
-abc
-d
-e
-	EOF
-        test_cmp expected append9.out
+        test_must_fail wait $pid
 '
 
-test_expect_success NO_CHAIN_LINT 'flux kvs get: --append fails on shortened write' '
+# N.B. valref treeobj now has fewer entries
+test_expect_success NO_CHAIN_LINT 'flux kvs get: --append fails on fake append (valref)' '
         flux kvs unlink -Rf test &&
         flux kvs put test.append.test="abc" &&
         flux kvs get --watch --append --count=4 \
-                     test.append.test > append10.out 2>&1 &
+                     test.append.test > append7.out 2>&1 &
         pid=$! &&
         wait_watcherscount_nonzero primary &&
         flux kvs put --append test.append.test="d" &&
         flux kvs put --append test.append.test="e" &&
-        flux kvs put test.append.test="foo" &&
-        ! wait $pid &&
-	cat >expected <<-EOF &&
-abc
-d
-e
-flux-kvs: test.append.test: Invalid argument
-	EOF
-        test_cmp expected append10.out
+        echo -n ${largeval} | flux kvs put --raw test.append.test=- &&
+        flux kvs get --treeobj test.append.test | grep valref &&
+        test_must_fail wait $pid
 '
 
 # full checks
@@ -646,7 +625,7 @@ test_expect_success 'flux kvs get --watch allows guest access to its ns' '
 	flux kvs namespace remove testns3
 '
 
-test_expect_success 'flux kvs get --watch denies guest access to anothers ns' '
+test_expect_success 'flux kvs get --watch denies guest access to another ns' '
 	flux kvs namespace create --owner=9999 testns4 &&
 	flux kvs put --namespace=testns4 test.j=102 &&
 	! FLUX_HANDLE_ROLEMASK=0x2 FLUX_HANDLE_USERID=9998 \
@@ -665,4 +644,19 @@ test_expect_success 'flux kvs get --watch allows owner access to guest ns' '
 test_expect_success 'kvs-watch.lookup request with empty payload fails with EPROTO(71)' '
 	${RPC} kvs-watch.lookup 71 </dev/null
 '
+# N.B. FLUX_KVS_WATCH = 4
+test_expect_success 'kvs-watch.lookup request non-streaming fails with EPROTO(71)' '
+	echo "{\"namespace\":"foo", \"key\":\"bar\", \"flags\":4}" \
+		${RPC} kvs-watch.lookup 71
+'
+
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done
diff --git a/t/t1008-kvs-eventlog.t b/t/t1008-kvs-eventlog.t
index 4058f898cde7..c19f5e229b9c 100755
--- a/t/t1008-kvs-eventlog.t
+++ b/t/t1008-kvs-eventlog.t
@@ -2,6 +2,8 @@
 
 test_description='Test kvs eventlog get|append'
 
+. `dirname $0`/kvs/kvs-helper.sh
+
 . `dirname $0`/sharness.sh
 
 test_under_flux 4 kvs
@@ -27,7 +29,87 @@ test_expect_success 'flux kvs eventlog append works w/ context' '
 	flux kvs eventlog append test.c foo {\"data\":\"bar\"} &&
 	flux kvs eventlog get test.c >get_c.out &&
 	grep -q foo get_c.out &&
-        grep -q "\"data\":\"bar\"" get_c.out
+        grep -q "data=\"bar\"" get_c.out
+'
+
+test_expect_success 'flux kvs eventlog get --human works' '
+	flux kvs eventlog append --timestamp=120 test.human primus &&
+	flux kvs eventlog append --timestamp=121 test.human secondus &&
+	flux kvs eventlog get --human test.human >test_human.out &&
+	test_debug "cat test_human.out" &&
+	cat <<-EOF >test_human.exp &&
+	[Jan01 00:02] primus
+	[  +1.000000] secondus
+	EOF
+	test_cmp test_human.exp test_human.out
+'
+
+test_expect_success 'flux kvs eventlog get/wait-event fails with -u and -H' '
+	test_must_fail flux kvs eventlog get -Hu test.human &&
+	test_must_fail flux kvs eventlog wait-event -Hu test.human primus
+'
+
+test_expect_success 'flux kvs eventlog wait-event --human works' '
+	flux kvs eventlog wait-event -v --human test.human secondus \
+		>test_human.wait-event.out &&
+	test_debug "cat test_human.wait-event.out" &&
+	cat <<-EOF >test_human.wait-event.exp &&
+	[Jan01 00:02] primus
+	[  +1.000000] secondus
+	EOF
+	test_cmp test_human.wait-event.exp test_human.wait-event.out
+'
+
+has_color() {
+	# To grep for ansi escape we need the help of the non-shell builtin
+  	# printf(1), so run under env(1) so we don't get shell builtin:
+        grep "$(env printf "\x1b\[")" $1 >/dev/null
+}
+test_expect_success 'flux kvs eventlog get/wait-event reject invalid --color' '
+	test_must_fail flux kvs eventlog get --color=foo test.human &&
+	test_must_fail flux kvs eventlog wait-event --color=foo test.human primus
+'
+for opt in "-L" "-Lalways" "--color" "--color=always"; do
+	test_expect_success "flux kvs eventlog get $opt forces color on" '
+		name=notty${opt##--color=} &&
+		outfile=color-${name:-default}.out &&
+		flux kvs eventlog get ${opt} test.human >$outfile &&
+		test_debug "cat $outfile" &&
+		has_color $outfile
+	'
+	test_expect_success "flux kvs eventlog wait-event $opt forces color on" '
+		name=notty${opt##--color=} &&
+		outfile=color-${name:-default}.wait-event.out &&
+		flux kvs eventlog wait-event ${opt} test.human primus >$outfile &&
+		test_debug "cat $outfile" &&
+		has_color $outfile
+	'
+done
+
+for opt in "" "--color" "--color=always" "--color=auto" "-H"; do
+	test_expect_success "flux kvs eventlog get $opt displays color on tty" '
+		name=${opt##--color=} &&
+		outfile=color-${name:-default}.out &&
+		runpty.py flux kvs eventlog get ${opt} test.human >$outfile &&
+		test_debug "cat $outfile" &&
+		has_color $outfile
+	'
+	test_expect_success "flux kvs eventlog wait-event $opt displays color on tty" '
+		name=${opt##--color=} &&
+		outfile=color-${name:-default}.wait-event.out &&
+		runpty.py flux kvs eventlog wait-event ${opt} test.human primus >$outfile &&
+		test_debug "cat $outfile" &&
+		has_color $outfile
+	'
+done
+
+test_expect_success "flux kvs eventlog get --color=never disables color on tty" '
+	opt="--color=never" &&
+	name=${opt##--color=} &&
+	outfile=color-${name:-default}.out &&
+	runpty.py flux kvs eventlog get ${opt} test.human >$outfile &&
+	test_debug "cat $outfile" &&
+	test_must_fail has_color $outfile
 '
 
 test_expect_success 'flux kvs eventlog get --watch --count=N works' '
@@ -53,8 +135,15 @@ test_expect_success NO_CHAIN_LINT 'flux kvs eventlog get --watch returns append
 	test_cmp get_e.exp get_e.out
 '
 
-test_expect_success 'flux kvs eventlog append fails with invalid context' '
-	! flux kvs eventlog append test.c foo not_a_object
+test_expect_success NO_CHAIN_LINT 'flux kvs eventlog get --waitcreate works' '
+        test_must_fail flux kvs eventlog get --unformatted test.f &&
+	flux kvs eventlog get --unformatted --waitcreate test.f >get_f.out &
+	pid=$! &&
+        wait_watcherscount_nonzero primary &&
+	flux kvs eventlog append --timestamp=1 test.f foo "{\"data\":\"bar\"}" &&
+	echo "{\"timestamp\":1.0,\"name\":\"foo\",\"context\":{\"data\":\"bar\"}}" >get_f.exp
+	wait $pid &&
+	test_cmp get_f.exp get_f.out
 '
 
 test_expect_success 'flux kvs eventlog append and work on alternate namespaces' '
@@ -67,4 +156,93 @@ test_expect_success 'flux kvs eventlog append and work on alternate namespaces'
         grep guest get_f2.out
 '
 
+test_expect_success 'flux kvs eventlog wait-event detects eventlog with event' '
+	flux kvs eventlog append --timestamp=42 test.wait_event.a foo &&
+	flux kvs eventlog append --timestamp=43 test.wait_event.a bar &&
+        flux kvs eventlog wait-event --unformatted test.wait_event.a foo > wait_event_a1.out &&
+        grep foo wait_event_a1.out &&
+        test_must_fail grep bar wait_event_a1.out &&
+        flux kvs eventlog wait-event --unformatted test.wait_event.a bar > wait_event_a2.out &&
+        test_must_fail grep foo wait_event_a2.out &&
+        grep bar wait_event_a2.out
+'
+
+test_expect_success 'flux kvs eventlog wait-event outputs more events with -v' '
+	flux kvs eventlog append --timestamp=42 test.wait_event.b foo &&
+	flux kvs eventlog append --timestamp=43 test.wait_event.b bar &&
+        flux kvs eventlog wait-event --unformatted -v test.wait_event.b foo > wait_event_b1.out &&
+        grep foo wait_event_b1.out &&
+        test_must_fail grep bar wait_event_b1.out &&
+        flux kvs eventlog wait-event --unformatted -v test.wait_event.b bar > wait_event_b2.out &&
+        grep foo wait_event_b2.out &&
+        grep bar wait_event_b2.out
+'
+
+test_expect_success 'flux kvs eventlog wait-event doesnt output events with -q' '
+	flux kvs eventlog append --timestamp=42 test.wait_event.c foo &&
+	flux kvs eventlog append --timestamp=43 test.wait_event.c bar &&
+        flux kvs eventlog wait-event --unformatted -q test.wait_event.c foo > wait_event_c1.out &&
+        test_must_fail grep foo wait_event_c1.out &&
+        test_must_fail grep bar wait_event_c1.out &&
+        flux kvs eventlog wait-event --unformatted -q test.wait_event.c bar > wait_event_c2.out &&
+        test_must_fail grep foo wait_event_c2.out &&
+        test_must_fail grep bar wait_event_c2.out
+'
+
+test_expect_success 'flux kvs eventlog wait-event fails on eventlog without event' '
+	flux kvs eventlog append --timestamp=42 test.wait_event.d foo &&
+	flux kvs eventlog append --timestamp=43 test.wait_event.d bar &&
+        test_expect_code 137 run_timeout 0.1 flux kvs eventlog wait-event test.wait_event.d foobar
+'
+
+test_expect_success 'flux kvs eventlog wait-event fails on non-existent eventlog' '
+        test_must_fail flux kvs eventlog wait-event test.wait_event.e foo
+'
+
+test_expect_success NO_CHAIN_LINT 'flux kvs eventlog wait-event --waitcreate works' '
+        test_must_fail flux kvs eventlog get --unformatted test.wait_event.f &&
+	flux kvs eventlog wait-event --waitcreate --unformatted test.wait_event.f foo >wait_event_f.out &
+	pid=$! &&
+        wait_watcherscount_nonzero primary &&
+	flux kvs eventlog append --timestamp=1 test.wait_event.f foo "{\"data\":\"bar\"}" &&
+	echo "{\"timestamp\":1.0,\"name\":\"foo\",\"context\":{\"data\":\"bar\"}}" >wait_event_f.exp
+	wait $pid &&
+	test_cmp wait_event_f.exp wait_event_f.out
+'
+
+test_expect_success NO_CHAIN_LINT 'flux kvs eventlog wait-event --timeout works' '
+	flux kvs eventlog append --timestamp=42 test.wait_event.g foo &&
+	flux kvs eventlog append --timestamp=43 test.wait_event.g bar &&
+	test_must_fail flux kvs eventlog wait-event --timeout=0.1 test.wait_event.g baz
+'
+
+#
+# corner case tests
+#
+
+test_expect_success 'flux kvs eventlog get fails on bad input' '
+	test_must_fail flux kvs eventlog get
+'
+
+test_expect_success 'flux kvs eventlog append fails on bad input' '
+	test_must_fail flux kvs eventlog append
+'
+
+test_expect_success 'flux kvs eventlog append fails with invalid context' '
+	test_must_fail flux kvs eventlog append test.bad.context foo not_a_object
+'
+
+test_expect_success 'flux kvs eventlog wait-event fails on bad input' '
+	test_must_fail flux kvs eventlog wait-event
+'
+
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done
diff --git a/t/t1009-kvs-copy.t b/t/t1009-kvs-copy.t
index db5c822d417f..01b2d9f3b948 100755
--- a/t/t1009-kvs-copy.t
+++ b/t/t1009-kvs-copy.t
@@ -9,10 +9,6 @@ Test flux-kvs copy and flux-kvs move.
 
 . `dirname $0`/sharness.sh
 
-if test "$TEST_LONG" = "t"; then
-    test_set_prereq LONGTEST
-fi
-
 SIZE=1
 test_under_flux ${SIZE} kvs
 
@@ -191,4 +187,13 @@ test_expect_success 'kvs-move nonexistent dst namespace fails' '
 	test_must_fail flux kvs move --dst-namespace=nons test.foo test.foo
 '
 
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done
diff --git a/t/t1010-kvs-commit-sync.t b/t/t1010-kvs-commit-sync.t
new file mode 100755
index 000000000000..db3ba60d82a9
--- /dev/null
+++ b/t/t1010-kvs-commit-sync.t
@@ -0,0 +1,93 @@
+#!/bin/sh
+#
+
+test_description='Test flux-kvs commit sync.'
+
+. `dirname $0`/kvs/kvs-helper.sh
+
+. `dirname $0`/sharness.sh
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+
+SIZE=1
+test_under_flux ${SIZE} minimal
+
+TESTNAMESPACE="testnamespace"
+
+checkpoint_get() {
+        jq -j -c -n  "{key:\"$1\"}" | $RPC content.checkpoint-get
+}
+
+test_expect_success 'load content module' '
+	flux module load content
+'
+
+test_expect_success 'load content-sqlite and kvs and add some data' '
+        flux module load content-sqlite &&
+        flux module load kvs &&
+        flux kvs put a=1 &&
+        flux kvs put b=2
+'
+
+test_expect_success 'kvs: no checkpoint of kvs-primary should exist yet' '
+        test_must_fail checkpoint_get kvs-primary
+'
+
+test_expect_success 'kvs: put some data to kvs and sync it' '
+        flux kvs put --blobref --sync c=3 > syncblob.out
+'
+
+test_expect_success 'kvs: checkpoint of kvs-primary should exist now' '
+        checkpoint_get kvs-primary
+'
+
+test_expect_success 'kvs: checkpoint of kvs-primary should match rootref' '
+        checkpoint_get kvs-primary | jq -r .value.rootref > checkpoint.out &&
+        test_cmp syncblob.out checkpoint.out
+'
+
+test_expect_success 'kvs: fence some data to kvs and sync it' '
+        ${FLUX_BUILD_DIR}/t/kvs/fence_api --sync 4 apisynctest
+'
+
+test_expect_success 'kvs: rootref of kvs-primary should match rootref' '
+        flux kvs getroot -b > fenceroot.exp &&
+        checkpoint_get kvs-primary | jq -r .value.rootref > fenceroot.out &&
+        test_cmp fenceroot.exp fenceroot.out
+'
+
+test_expect_success 'kvs: sync fails against non-primary namespace' '
+        flux kvs namespace create ${TESTNAMESPACE} &&
+        flux kvs put --namespace=${TESTNAMESPACE} a=10 &&
+        test_must_fail flux kvs put --namespace=${TESTNAMESPACE} --sync b=11
+'
+
+test_expect_success 'kvs: sync fence fails against non-primary namespace' '
+        ${FLUX_BUILD_DIR}/t/kvs/fence_api \
+                         --namespace=${TESTNAMESPACE} 4 apisynctestns &&
+        test_must_fail ${FLUX_BUILD_DIR}/t/kvs/fence_api \
+                         --namespace=${TESTNAMESPACE} --sync 4 apisynctestns
+'
+
+test_expect_success 'kvs: unload content-sqlite' '
+	flux module remove content-sqlite
+'
+
+test_expect_success 'kvs: sync without backing store fails' '
+        test_must_fail flux kvs put --sync d=4
+'
+
+test_expect_success 'kvs: no pending requests at end of tests before module removal' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
+test_expect_success 'kvs: unload kvs' '
+	flux module remove kvs
+'
+
+test_expect_success 'remove content module' '
+	flux module remove content
+'
+
+test_done
diff --git a/t/t1011-kvs-checkpoint-period.t b/t/t1011-kvs-checkpoint-period.t
new file mode 100755
index 000000000000..27e8cf17177e
--- /dev/null
+++ b/t/t1011-kvs-checkpoint-period.t
@@ -0,0 +1,170 @@
+#!/bin/sh
+#
+
+test_description='Test kvs module checkpoint-period config.'
+
+. `dirname $0`/kvs/kvs-helper.sh
+
+. `dirname $0`/sharness.sh
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+
+export FLUX_CONF_DIR=$(pwd)
+SIZE=4
+test_under_flux ${SIZE} minimal
+
+checkpoint_get() {
+	jq -j -c -n  "{key:\"$1\"}" \
+	    | $RPC content.checkpoint-get \
+	    | jq -r .value.rootref
+}
+
+# arg1 - old ref
+# arg2 - timeout (seconds)
+checkpoint_changed() {
+	old_ref=$1
+	local i=0
+	local iters=$(($2 * 10))
+	while [ $i -lt ${iters} ]
+	do
+	    ref=$(checkpoint_get kvs-primary)
+	    if [ "${old_ref}" != "${ref}" ]
+	    then
+		return 0
+	    fi
+	    sleep 0.1
+	    i=$((i + 1))
+	done
+	return 1
+}
+
+test_expect_success 'configure bad checkpoint-period timer in kvs' '
+	cat >kvs.toml <<-EOF &&
+	[kvs]
+	checkpoint-period = "1Z"
+	EOF
+	flux config reload &&
+	test_must_fail flux module load kvs
+'
+
+test_expect_success 'configure checkpoint-period, place initial value' '
+	cat >kvs.toml <<-EOF &&
+	[kvs]
+	checkpoint-period = "200ms"
+	EOF
+	flux config reload &&
+	flux module load content &&
+	flux module load content-sqlite &&
+	flux module load kvs &&
+	flux kvs put --blobref --sync a=1 > blob1.out
+'
+
+test_expect_success 'kvs: put some more data to kvs (1)' '
+	flux kvs put --blobref b=1 > blob2.out
+'
+
+test_expect_success 'kvs: checkpoint of kvs-primary should change in time (1)' '
+	checkpoint_changed $(cat blob1.out) 5 &&
+	checkpoint_get kvs-primary > checkpoint2.out &&
+	test_cmp checkpoint2.out blob2.out
+'
+
+test_expect_success 'kvs: put some more data to kvs (2)' '
+	flux kvs put --blobref c=1 > blob3.out
+'
+
+test_expect_success 'kvs: checkpoint of kvs-primary should change in time (2)' '
+	checkpoint_changed $(cat blob2.out) 5 &&
+	checkpoint_get kvs-primary > checkpoint3.out &&
+	test_cmp checkpoint3.out blob3.out
+'
+
+test_expect_success 'kvs: put some data to non-primary namespace' '
+	flux kvs namespace create "test-ns" &&
+	flux kvs put --namespace=test-ns d=1
+'
+
+test_expect_success 'kvs: checkpoint of kvs-primary should not change (1)' '
+	test_must_fail checkpoint_changed $(cat blob3.out) 2
+'
+
+test_expect_success 'configure bad checkpoint-period timer in kvs on reload' '
+	cat >kvs.toml <<-EOF &&
+	[kvs]
+	checkpoint-period = "1Z"
+	EOF
+	test_must_fail flux config reload
+'
+
+test_expect_success 're-config checkpoint-period timer, set large period' '
+	cat >kvs.toml <<-EOF &&
+	[kvs]
+	checkpoint-period = "60m"
+	EOF
+	flux config reload
+'
+
+test_expect_success 'kvs: put some more data to kvs (3)' '
+	flux kvs put --blobref d=1 > blob4.out
+'
+
+test_expect_success 'kvs: checkpoint of kvs-primary should not change (2)' '
+	test_must_fail checkpoint_changed $(cat blob3.out) 2
+'
+
+test_expect_success 're-config checkpoint-period timer, set small period' '
+	cat >kvs.toml <<-EOF &&
+	[kvs]
+	checkpoint-period = "200ms"
+	EOF
+	flux config reload
+'
+
+test_expect_success 'kvs: checkpoint of kvs-primary should change in time (3)' '
+	checkpoint_changed $(cat blob3.out) 5 &&
+	checkpoint_get kvs-primary > checkpoint4.out &&
+	test_cmp checkpoint4.out blob4.out
+'
+
+test_expect_success 're-config checkpoint-period timer, disable it' '
+	cat >kvs.toml <<-EOF &&
+	[kvs]
+	checkpoint-period = "0s"
+	EOF
+	flux config reload
+'
+
+test_expect_success 'kvs: put some more data to kvs (4)' '
+	flux kvs put --blobref e=1 > blob5.out
+'
+
+test_expect_success 'kvs: checkpoint of kvs-primary should not change (3)' '
+	test_must_fail checkpoint_changed $(cat blob4.out) 2
+'
+
+test_expect_success 're-config checkpoint-period timer in kvs, re-enable it' '
+	cat >kvs.toml <<-EOF &&
+	[kvs]
+	checkpoint-period = "200ms"
+	EOF
+	flux config reload
+'
+
+test_expect_success 'kvs: checkpoint of kvs-primary should change in time (4)' '
+	checkpoint_changed $(cat blob4.out) 5 &&
+	checkpoint_get kvs-primary > checkpoint5.out &&
+	test_cmp checkpoint5.out blob5.out
+'
+
+test_expect_success 'kvs: no pending requests at end of tests before module removal' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
+test_expect_success 'kvs: remove modules' '
+	flux module remove kvs &&
+	flux module remove content-sqlite &&
+	flux module remove content
+'
+
+test_done
diff --git a/t/t1101-barrier-basic.t b/t/t1101-barrier-basic.t
index d347cb087f1a..63dcf1466ef4 100755
--- a/t/t1101-barrier-basic.t
+++ b/t/t1101-barrier-basic.t
@@ -3,7 +3,7 @@
 
 test_description='Test basic barrier usage in flux session
 
-Verify basic barrier operations against a running flux comms session.
+Verify basic barrier operations against a running flux instance.
 This test script verifies operation of barriers and should be run
 before other tests that depend on barriers.
 '
@@ -29,7 +29,7 @@ test_expect_success 'barrier: returns when complete (all ranks)' '
 '
 
 test_expect_success 'barrier: blocks while incomplete' '
-	test_expect_code 142 run_timeout 1 \
+	test_expect_code 142 run_timeout -s ALRM 1 \
 	  ${tbarrier} --nprocs 2 xyz
 '
 
diff --git a/t/t1102-cmddriver.t b/t/t1102-cmddriver.t
index 77cfee5eafa3..86436b49a0e5 100755
--- a/t/t1102-cmddriver.t
+++ b/t/t1102-cmddriver.t
@@ -9,35 +9,36 @@ Verify flux command driver behavior.
 . `dirname $0`/sharness.sh
 SIZE=4
 test_under_flux ${SIZE} minimal
+path_printenv=$(which printenv)
 
 test_expect_success 'baseline works' '
-	flux comms info
+	flux getattr size
 '
 
 test_expect_success 'flux prepends to FLUX_MODULE_PATH' '
 	(FLUX_MODULE_PATH=/xyz \
-		flux /usr/bin/printenv | grep "FLUX_MODULE_PATH=.*:/xyz")
+		flux $path_printenv | grep "FLUX_MODULE_PATH=.*:/xyz")
 '
 
 test_expect_success 'flux prepends to FLUX_CONNECTOR_PATH' '
         (FLUX_CONNECTOR_PATH=/xyz \
-		flux /usr/bin/printenv | grep "FLUX_CONNECTOR_PATH=.*:/xyz")
+		flux $path_printenv | grep "FLUX_CONNECTOR_PATH=.*:/xyz")
 '
 
 test_expect_success 'flux fails for unknown connector scheme' '
 	(FLUX_URI=noexist:// \
-		test_must_fail flux comms info)
+		test_must_fail flux getattr size)
 '
 
 test_expect_success 'flux fails for unknown connector path' '
 	(FLUX_URI=local://noexist \
-		test_must_fail flux comms info)
+		test_must_fail flux getattr size)
 '
 
 test_expect_success 'flux fails fails for non-connector dso' '
 	(FLUX_CONNECTOR_PATH=${FLUX_BUILD_DIR}/src/modules \
 	FLUX_URI=kvs:// \
-		test_must_fail flux comms info)
+		test_must_fail flux getattr size)
 '
 
 # Test flux 'env' builtin
@@ -82,29 +83,74 @@ test_expect_success 'cmddriver removes multiple contiguous separators in input'
 		flux env sh -c 'echo \$LUA_PATH' | grep -v ';;;;')
 "
 readlink --version >/dev/null && test_set_prereq READLINK
-test_expect_success READLINK 'cmddriver adds its own path to PATH if called with relative path' "
-	fluxcmd=\$(readlink -f \$(which flux)) &&
-	fluxdir=\$(dirname \$fluxcmd) &&
-	PATH='/bin:/usr/bin' \$fluxcmd env sh -c 'echo \$PATH' | grep ^\$fluxdir
-"
-test_expect_success READLINK 'cmddriver moves its own path to the front of PATH' "
-	fluxcmd=\$(readlink -f \$(which flux)) &&
-	fluxdir=\$(dirname \$fluxcmd) &&
-	PATH=/bin:\$fluxdir \$fluxcmd env sh -c 'echo \$PATH' | grep ^\$fluxdir
-"
+
+
+# N.B.: In the tests below we need to ensure that /bin:/usr/bin appear
+# in PATH so that core utilities like `ls` can be found by the libtool
+# wrappers or other processes invoked by calling `flux`. However, this
+# means the results of the test can be influenced by whether or not a
+# flux executable appears in /bin or /usr/bin. Therefore, care must be
+# taken to ensure consistent results no matter what is in these paths.
+#
+# Ensure a bogus 'flux' executable occurs first in path, then make sure
+# command -v flux finds the right flux:
+#
+test_expect_success 'cmddriver adds its own path to PATH' '
+	mkdir bin &&
+	cat <<-EOF >bin/flux &&
+	#!/bin/sh
+	true
+	EOF
+	chmod +x bin/flux &&
+	fluxcmd=$(command -v flux) &&
+	result=$(PATH=$(pwd)/bin:/bin:/usr/bin \
+	         $fluxcmd env sh -c "command -v flux") &&
+	test_debug "echo result=$result" &&
+	test "$result" = "$fluxcmd"
+'
+# Use bogus flux in PATH and ensure flux cmddriver inserts its own path
+# just before this path, not at front of PATH.
+test_expect_success 'cmddriver inserts its path at end of PATH' '
+	fluxdir=$(dirname $fluxcmd) &&
+	result=$(PATH=/foo:$(pwd)/bin:/bin:/usr/bin \
+	         $fluxcmd env printenv PATH) &&
+	test_debug "echo result=$result" &&
+	test "$result" = "/foo:$fluxdir:$(pwd)/bin:/bin:/usr/bin"
+'
+# Ensure a PATH that already returns current flux first is not modified
+# by flux(1)
+#
+# The following test assumes flux(1) is not in /bin or /usr/bin. Skip the
+# test if so.
+#
+fluxdir=$(dirname $fluxcmd)
+test "$fluxdir" = "/bin" -o "$fluxdir" = "/usr/bin" || test_set_prereq NOBINDIR
+test_expect_success READLINK,NOBINDIR 'cmddriver does not adjust PATH if unnecessary' '
+	fluxdir=$(dirname $fluxcmd) &&
+	mypath=/foo:/bar:$fluxdir:/usr/bin:/bin &&
+	newpath=$(PATH=$mypath $fluxcmd env $path_printenv PATH) &&
+	test_debug "echo PATH=$newpath" &&
+	test "$newpath" = "$mypath"
+'
 test_expect_success 'FLUX_*_PREPEND environment variables work' '
 	( FLUX_CONNECTOR_PATH_PREPEND=/foo \
-	  flux /usr/bin/printenv | grep "FLUX_CONNECTOR_PATH=/foo" &&
+	  flux $path_printenv | grep "FLUX_CONNECTOR_PATH=/foo" &&
 	FLUX_EXEC_PATH_PREPEND=/foo \
-	  flux /usr/bin/printenv | grep "FLUX_EXEC_PATH=/foo" &&
+	  flux $path_printenv | grep "FLUX_EXEC_PATH=/foo" &&
 	FLUX_MODULE_PATH_PREPEND=/foo \
-	  flux /usr/bin/printenv | grep "FLUX_MODULE_PATH=/foo" &&
+	  flux $path_printenv | grep "FLUX_MODULE_PATH=/foo" &&
 	FLUX_LUA_PATH_PREPEND=/foo \
-	  flux /usr/bin/printenv | grep "LUA_PATH=/foo" &&
+	  flux $path_printenv | grep "LUA_PATH=/foo" &&
 	FLUX_LUA_CPATH_PREPEND=/foo \
-	  flux /usr/bin/printenv | grep "LUA_CPATH=/foo" &&
+	  flux $path_printenv | grep "LUA_CPATH=/foo" &&
 	FLUX_PYTHONPATH_PREPEND=/foo \
-	  flux /usr/bin/printenv | grep "PYTHONPATH=/foo")
+	  flux $path_printenv | grep "PYTHONPATH=/foo")
+'
+test_expect_success 'environment variables are prepended in correct order' '
+	( FLUX_EXEC_PATH_PREPEND=/foo:/bar \
+	  flux $path_printenv FLUX_EXEC_PATH > prepend.out ) &&
+	test_debug "cat prepend.out" &&
+	grep "^/foo:/bar:" prepend.out
 '
 test_expect_success 'flux-env output can be passed to eval' '
     (eval $(flux env))
diff --git a/t/t1103-apidisconnect.t b/t/t1103-apidisconnect.t
index 1b2010d470ef..3807b074bb58 100755
--- a/t/t1103-apidisconnect.t
+++ b/t/t1103-apidisconnect.t
@@ -8,6 +8,8 @@ test_description='Test api disconnect generation
 SIZE=4
 test_under_flux ${SIZE} kvs
 
+TEST_WATCHER=${FLUX_BUILD_DIR}/t/disconnect/.libs/watcher.so
+
 # Usage: check_watchers #expected #tries
 check_kvs_watchers() {
 	local i n
@@ -24,7 +26,7 @@ check_kvs_watchers() {
 test_expect_success 'kvs watcher gets disconnected on client exit' '
 	before_watchers=`flux module stats --parse "watchers" kvs-watch` &&
 	echo "waiters before test: $before_watchers" &&
-	test_expect_code 142 run_timeout 1 flux kvs get --watch --waitcreate noexist &&
+	test_expect_code 142 run_timeout -s ALRM 1 flux kvs get --watch --waitcreate noexist &&
 	check_kvs_watchers $before_watchers 3
 '
 
@@ -32,5 +34,13 @@ test_expect_success 'multi-node kvs watcher gets disconnected on client exit' '
 	${FLUX_BUILD_DIR}/t/kvs/watch_disconnect $SIZE
 '
 
+test_expect_success 'module watcher gets disconnected on module unload' '
+	before_watchers=`flux module stats --parse "watchers" kvs-watch` &&
+	echo "waiters before loading module: $before_watchers" &&
+	flux module load ${TEST_WATCHER} &&
+	check_kvs_watchers $(($before_watchers+1)) 3 &&
+	flux module remove watcher &&
+	check_kvs_watchers $before_watchers 3
+'
 
 test_done
diff --git a/t/t1105-proxy.t b/t/t1105-proxy.t
index eeed85c944a3..808cff5cc7b7 100755
--- a/t/t1105-proxy.t
+++ b/t/t1105-proxy.t
@@ -13,6 +13,7 @@ export TEST_FLUX=${FLUX_BUILD_DIR}/src/cmd/flux
 export TEST_TMPDIR=${TMPDIR:-/tmp}
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
 EVENT_TRACE="$SHARNESS_TEST_SRCDIR/scripts/event-trace.lua"
+RUNPTY="$SHARNESS_TEST_SRCDIR/scripts/runpty.py"
 
 test_expect_success 'flux-proxy creates new socket' '
 	PROXY_URI=$(flux proxy $TEST_URI printenv FLUX_URI) &&
@@ -26,8 +27,8 @@ test_expect_success 'flux-proxy cleans up socket' '
 '
 
 test_expect_success 'flux-proxy exits with command return code' '
-	flux proxy $TEST_URI /bin/true &&
-	! flux proxy $TEST_URI /bin/false
+	flux proxy $TEST_URI true &&
+	! flux proxy $TEST_URI false
 '
 
 test_expect_success 'flux-proxy forwards getattr request' '
@@ -35,18 +36,6 @@ test_expect_success 'flux-proxy forwards getattr request' '
 	test "$ATTR_SIZE" = "$SIZE"
 '
 
-test_expect_success 'flux-proxy manages event redistribution' '
-	flux proxy $TEST_URI \
-	  "${EVENT_TRACE} -t 2 foobar foobar.exit \\
-       ${EVENT_TRACE} -t 2 foobar foobar.exit \\
-       flux event pub foobar.exit" &&
-	FLUX_URI=$TEST_URI flux dmesg | sed -e "s/[^ ]* //" >event.out &&
-	test $(egrep "connector-local.*debug\[0\]: subscribe foobar" event.out|wc -l) -eq 1 &&
-	test $(egrep "proxy.*debug\[0\]: subscribe foobar" event.out|wc -l) -eq 1 &&
-	test $(egrep "connector-local.*debug\[0\]: unsubscribe foobar" event.out|wc -l) -eq 1 &&
-	test $(egrep "proxy.*debug\[0\]: unsubscribe foobar" event.out|wc -l) -eq 1
-'
-
 test_expect_success 'flux-proxy permits dynamic service registration' "
         echo '{\"service\":\"fubar\"}' >service.add.in &&
         flux proxy $TEST_URI \
@@ -66,18 +55,127 @@ test_expect_success 'flux-proxy cannot register service with method (EINVAL)' "
 "
 
 test_expect_success 'flux-proxy cannot shadow a broker service (EEXIST)' "
-	echo '{\"service\":\"cmb\"}' >service4.add.in &&
+	echo '{\"service\":\"broker\"}' >service4.add.in &&
         flux proxy $TEST_URI \
 	  $RPC service.add 17 <service4.add.in
 "
 
-test_expect_success 'flux-proxy fails with unknown URI scheme (ENOENT)' '
-	test_must_fail flux proxy badscheme:// 2>badscheme.err &&
-	grep "No such file or directory" badscheme.err
+test_expect_success 'flux-proxy calls out to flux-uri for unknown URI scheme' '
+	result=$(flux proxy pid:$$ flux getattr local-uri) &&
+	test_debug "echo proxy getattr local-uri = $result" &&
+	test "$result" = "$FLUX_URI"
+'
+
+test_expect_success 'flux-proxy fails if flux-uri fails' '
+	test_must_fail flux proxy bloop:1234 2>baduri.err &&
+	test_debug "cat baduri.err" &&
+	grep "Unable to resolve bloop:1234 to a URI" baduri.err
 '
+
 test_expect_success 'flux-proxy fails with unknown URI path (ENOENT)' '
 	test_must_fail flux proxy local:///noexist  2>badpath.err &&
 	grep "No such file or directory" badpath.err
 '
 
+test_expect_success 'flux-proxy forwards LD_LIBRARY_PATH' '
+	cat >proxinator.sh <<-EOF &&
+	#!/bin/sh
+	echo ssh "\$@" > proxinator.log
+	EOF
+	chmod +x proxinator.sh &&
+	(export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/foo &&
+		export FLUX_SSH=./proxinator.sh &&
+		test_must_fail flux proxy ssh://hostname/baz/local) &&
+	test_debug "cat ./proxinator.log" &&
+	grep -E "ssh.* LD_LIBRARY_PATH=[^ ]*:?/foo .*flux relay" ./proxinator.log
+'
+
+test_expect_success 'set bogus broker version' '
+	flux getattr version >realversion &&
+	flux setattr version 0.0.0
+'
+test_expect_success 'flux-proxy fails with version mismatch' '
+	test_must_fail flux proxy $FLUX_URI true
+'
+test_expect_success 'flux-proxy --force works with version mismatch' '
+	flux proxy --force $FLUX_URI true
+'
+test_expect_success 'restore real broker version' '
+	flux setattr version $(cat realversion)
+'
+
+test_expect_success 'flux-proxy works with jobid argument' '
+	id=$(flux submit -n1 flux start flux run sleep 30) &&
+	flux job wait-event -t 10 $id memo &&
+	uri=$(flux proxy $id?local flux getattr parent-uri) &&
+	test_debug "echo flux proxy $id flux getattr parent-uri = $uri" &&
+	test "$uri" = "$FLUX_URI"
+'
+test_expect_success 'flux-proxy works with /jobid argument' '
+	uri=$(flux proxy /$id?local flux getattr parent-uri) &&
+	test_debug "echo flux proxy $id flux getattr parent-uri = $uri" &&
+	test "$uri" = "$FLUX_URI"
+'
+tssh=${SHARNESS_TEST_SRCDIR}/scripts/tssh
+test_expect_success 'flux-proxy sets FLUX_PROXY_REMOTE for remote URIs' '
+	FLUX_SSH=${tssh} \
+	    flux proxy ${id}?remote printenv FLUX_PROXY_REMOTE \
+	      >proxy_remote.out &&
+	test_debug "cat proxy_remote.out" &&
+	test "$(cat proxy_remote.out)" = "$(hostname)"
+'
+test_expect_success 'flux-proxy does not set FLUX_PROXY_REMOTE for local URIs' '
+	test_must_fail flux proxy ${id}?local printenv FLUX_PROXY_REMOTE
+'
+test_expect_success 'flux-proxy preserves options in FLUX_PROXY_REMOTE' '
+	uri=$(flux uri ${id} | sed "s|ssh://|ssh://user@|") &&
+	FLUX_SSH=${tssh} \
+	    flux proxy $uri printenv FLUX_PROXY_REMOTE \
+	      >proxy_remote_user.out &&
+	test_debug "cat proxy_remote_user.out" &&
+	test "$(cat proxy_remote_user.out)" = "user@$(hostname)"
+'
+test_expect_success 'parent-uri under remote flux-proxy is rewritten' '
+	uri=$(flux uri ${id} | sed "s|ssh://|ssh://user@|") &&
+	FLUX_SSH=${tssh} \
+	    flux proxy $uri flux getattr parent-uri >proxy-parent-uri.out &&
+	test_debug "cat proxy-parent-uri.out" &&
+	grep ssh://user@$(hostname) proxy-parent-uri.out
+'
+test_expect_success 'flux-start does not hang under flux-proxy' '
+	run_timeout --kill-after=10 --env=FLUX_SSH=${tssh} 60 \
+		flux proxy $uri flux start true
+'
+test_expect_success 'cancel test job' '
+	flux cancel $id &&
+	flux job wait-event -vt 10 $id clean
+'
+test_expect_success NO_CHAIN_LINT 'flux-proxy attempts to restore terminal on error' '
+	cat <<-EOF >test.sh &&
+	#!/bin/bash
+	flux --parent cancel \$(flux getattr jobid)
+	while flux getattr jobid; do sleep 0.1; done
+	EOF
+	chmod +x test.sh
+	id=$(flux batch -n1 --wrap flux run sleep 600) &&
+	flux job wait-event -vt 60 $id memo &&
+	$RUNPTY -o pty.out -f asciicast \
+		flux proxy --nohup ${id}?local $(pwd)/test.sh &&
+	grep "\[\?25h" pty.out
+'
+test_expect_success NO_CHAIN_LINT 'flux-proxy sends SIGHUP to children without --nohup' '
+	SHELL=/bin/sh &&
+	cat <<-EOF >test.sh &&
+	#!/bin/bash
+	flux --parent cancel \$(flux getattr jobid)
+	while true; do sleep 0.1; done
+	EOF
+	chmod +x test.sh
+	id=$(flux batch -n1 --wrap flux run sleep 600) &&
+	flux job wait-event -vt 60 $id memo &&
+	test_expect_code 129 $RUNPTY -o pty.out -f asciicast \
+		flux proxy ${id}?local $(pwd)/test.sh &&
+	grep "\[\?25h" pty.out &&
+	grep "SIGHUP" pty.out
+'
 test_done
diff --git a/t/t1106-ssh-connector.t b/t/t1106-ssh-connector.t
index 9d428a56c6a2..b8ebe053cda4 100755
--- a/t/t1106-ssh-connector.t
+++ b/t/t1106-ssh-connector.t
@@ -5,19 +5,72 @@ test_description='Test ssh:// connector and flux-relay'
 
 . `dirname $0`/sharness.sh
 SIZE=4
-test_under_flux ${SIZE}
+test_under_flux ${SIZE} minimal
 
 export TEST_SOCKDIR=$(echo $FLUX_URI | sed -e "s!local://!!") &&
 
 export TEST_SSH=${SHARNESS_TEST_SRCDIR}/scripts/tssh
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
 
+extract_env_command() {
+	grep cmd= \
+	    | sed s/cmd=\"env\ // \
+	    | sed s/\ flux\ relay.*// \
+	    | xargs -n1
+}
+
+test_expect_success 'rundir/bin directory exists' '
+	test -d $(dirname $TEST_SOCKDIR)/bin
+'
+test_expect_success 'rundir/bin/flux symlink exists' '
+	test -h $(dirname $TEST_SOCKDIR)/bin/flux
+'
+test_expect_success 'rundir/bin/flux points to an executable' '
+	test -x $(dirname $TEST_SOCKDIR)/bin/flux
+'
+test_expect_success 'load heartbeat module with fast rate' '
+        flux module load heartbeat period=0.1s
+'
 test_expect_success 'ssh:// with local sockdir works' '
 	FLUX_URI=ssh://localhost${TEST_SOCKDIR} FLUX_SSH=$TEST_SSH \
 	  flux getattr size 2>basic.err &&
 	  grep hostname=localhost basic.err &&
 	  grep cmd= basic.err | grep "flux relay $TEST_SOCKDIR"
 '
+test_expect_success 'remote PATH is set' '
+	extract_env_command <basic.err >basic.env &&
+	grep -q "^PATH=" basic.env
+'
+test_expect_success 'remote PATH has rundir/bin first' '
+	grep -q "^PATH=$(dirname $TEST_SOCKDIR)/bin:" basic.env
+'
+test_expect_success 'remote PATH includes local flux bindir' '
+	fbindir=$(dirname $(which flux)) &&
+	grep -q "^PATH=.*:$fbindir" basic.env
+'
+test_expect_success 'remote PATH includes system bindirs' '
+	grep -q "^PATH=.*:/bin:/usr/bin" basic.env
+'
+# N.B. ensure LD_LIBRARY_PATH is set so env(1) is used when PATH is not set
+# For in tree testing it is set by libtool wrappers
+test_expect_success 'ssh:// with FLUX_SSH_RCMD does not set remote PATH' '
+	env FLUX_URI=ssh://localhost$TEST_SOCKDIR \
+	    FLUX_SSH=$TEST_SSH \
+	    FLUX_SSH_RCMD=flux \
+	    LD_LIBRARY_PATH=${LD_LIBRARY_PATH=-/foo/bar} \
+	    flux getattr size 2>basic_rcmd.err &&
+	extract_env_command <basic_rcmd.err >basic_rcmd.env &&
+	test_must_fail grep "^PATH=" basic_rcmd.env
+'
+test -x /bin/tcsh && test_set_prereq HAVE_TCSH
+test_expect_success HAVE_TCSH 'ssh:// with local sockdir and SHELL=tcsh works' '
+	FLUX_URI=ssh://localhost${TEST_SOCKDIR} \
+	FLUX_SSH=$TEST_SSH \
+	SHELL=/bin/tcsh \
+	  flux getattr size 2>basic_tcsh.err &&
+	  grep hostname=localhost basic_tcsh.err &&
+	  grep cmd= basic_tcsh.err | grep "flux relay $TEST_SOCKDIR"
+'
 
 test_expect_success 'ssh:// with local sockdir and port works' '
 	FLUX_URI=ssh://localhost:42${TEST_SOCKDIR} FLUX_SSH=$TEST_SSH \
@@ -45,12 +98,12 @@ test_expect_success 'ssh:// with local sockdir, user, and port works' '
 
 test_expect_success 'ssh:// can handle nontrivial message load' '
 	FLUX_URI=ssh://localhost$TEST_SOCKDIR FLUX_SSH=$TEST_SSH \
-	  flux kvs dir -R >dir.out
+	  flux ping --count=100 -i 0.001 broker >/dev/null
 '
 
 test_expect_success 'ssh:// can work with events' '
 	FLUX_URI=ssh://localhost$TEST_SOCKDIR FLUX_SSH=$TEST_SSH \
-	  flux event sub --count=1 hb
+	  flux event sub --count=1 heartbeat.pulse
 '
 
 test_expect_success 'ssh:// can register a service' "
@@ -73,32 +126,72 @@ test_expect_success 'ssh:// cannot register a service with method (EINVAL)' "
 
 test_expect_success 'ssh:// cannot shadow a broker service (EEXIST)' "
 	FLUX_URI=ssh://localhost$TEST_SOCKDIR FLUX_SSH=$TEST_SSH \
-	  echo '{\"service\":\"cmb\"}' >service4.add.in &&
+	  echo '{\"service\":\"broker\"}' >service4.add.in &&
 	  $RPC service.add 17 <service4.add.in
 "
 
 test_expect_success 'ssh:// with bad query option fails in flux_open()' '
-	! FLUX_URI=ssh://localhost$TEST_SOCKDIR?badarg=bar FLUX_SSH=$TEST_SSH \
-	  flux getattr size 2>badarg.out &&
+	test_must_fail env \
+	    FLUX_URI=ssh://localhost$TEST_SOCKDIR?badarg=bar \
+	    FLUX_SSH=$TEST_SSH \
+	    flux getattr size 2>badarg.out &&
 	grep -q "flux_open:" badarg.out
 '
 
 test_expect_success 'ssh:// with bad FLUX_SSH value fails in flux_open()' '
-	! FLUX_URI=ssh://localhost$TEST_SOCKDIR FLUX_SSH=/noexist \
-	  flux getattr size 2>noexist.out &&
+	test_must_fail env \
+	    FLUX_URI=ssh://localhost$TEST_SOCKDIR \
+	    FLUX_SSH=/noexist \
+	    flux getattr size 2>noexist.out &&
 	grep -q "flux_open:" noexist.out
 '
 
 test_expect_success 'ssh:// with bad FLUX_SSH_RCMD value fails in flux_open()' '
-	! FLUX_URI=ssh://localhost$TEST_SOCKDIR FLUX_SSH=$TEST_SSH \
-	  FLUX_SSH_RCMD=/nocmd flux getattr size 2>nocmd.out &&
+	test_must_fail env \
+	    FLUX_URI=ssh://localhost$TEST_SOCKDIR \
+	    FLUX_SSH=$TEST_SSH \
+	    FLUX_SSH_RCMD=/nocmd \
+	    flux getattr size 2>nocmd.out &&
 	grep -q "flux_open:" nocmd.out
 '
 
 test_expect_success 'ssh:// with missing path component fails in flux_open()' '
-	! FLUX_URI=ssh://localhost FLUX_SSH=$TEST_SSH \
-	  flux getattr size 2>nopath.out &&
+	test_must_fail env \
+	    FLUX_URI=ssh://localhost \
+	    FLUX_SSH=$TEST_SSH \
+	    flux getattr size 2>nopath.out &&
 	grep -q "flux_open:" nopath.out
 '
 
+test_expect_success 'create test ssh that emits errors on stderr' '
+	cat <<-EOF >ssh.sh &&
+	#!/bin/sh
+	printf "error from ssh\n" >&2
+	exit 1
+	EOF
+	chmod +x ssh.sh
+'
+
+test_expect_success 'ssh:// stderr is redirected with Python flux.Flux()' '
+	cat <<-EOF >open_ex.py &&
+	import os
+	import flux
+	try:
+	    flux.Flux()
+	except OSError as err:
+	    print (f"Exception: errno={err.errno} msg={err.strerror}")
+	EOF
+	cat <<-EOF2 >open_ex.expected &&
+	Exception: errno=104 msg=Unable to connect to Flux: error from ssh
+	EOF2
+	FLUX_URI=ssh://localhost/foo/bar FLUX_SSH=$(pwd)/ssh.sh \
+	  flux python open_ex.py >open_ex.out 2>open_ex.err &&
+	test_must_be_empty open_ex.err &&
+	test_cmp open_ex.expected open_ex.out
+'
+
+test_expect_success 'remove heartbeat module' '
+        flux module remove heartbeat
+'
+
 test_done
diff --git a/t/t1107-heartbeat.t b/t/t1107-heartbeat.t
new file mode 100755
index 000000000000..7a943e50bd51
--- /dev/null
+++ b/t/t1107-heartbeat.t
@@ -0,0 +1,40 @@
+#!/bin/sh
+#
+
+test_description='Test heartbeat module'
+
+. `dirname $0`/sharness.sh
+SIZE=1
+test_under_flux ${SIZE} minimal
+
+
+get_heartbeat() {
+	flux python -c \
+	    "import flux; print(flux.Flux().rpc(\"heartbeat.get\").get_str())"
+}
+
+test_expect_success 'load heartbeat' '
+	flux module load heartbeat
+'
+
+test_expect_success 'reload heartbeat with period=10s and verify' '
+	period1=$(get_heartbeat | jq -r -e .period) &&
+	flux module reload heartbeat period=10s &&
+	period2=$(get_heartbeat | jq -r -e .period) &&
+	echo period changed from $period1 to $period2 &&
+	test "$period1" != "$period2"
+'
+
+test_expect_success 'unload heartbeat' '
+	flux module unload heartbeat
+'
+
+test_expect_success 'reload heartbeat with period=bad fsd' '
+	test_must_fail flux module load heartbeat period=1x
+'
+
+test_expect_success 'reload heartbeat with bad option' '
+	test_must_fail flux module load heartbeat foo=42
+'
+
+test_done
diff --git a/t/t1200-stats-basic.t b/t/t1200-stats-basic.t
new file mode 100755
index 000000000000..9e20d2edddb9
--- /dev/null
+++ b/t/t1200-stats-basic.t
@@ -0,0 +1,62 @@
+#!/bin/sh
+#
+
+test_description='Test stats collection and sending'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile --debug
+. `dirname $0`/sharness.sh
+
+udp=$SHARNESS_TEST_SRCDIR/scripts/stats-listen.py
+timeout="$SHARNESS_TEST_SRCDIR/scripts/run_timeout.py 60"
+timeout5="$SHARNESS_TEST_SRCDIR/scripts/run_timeout.py 5"
+plugin_i=${FLUX_BUILD_DIR}/t/stats/.libs/stats-immediate.so
+plugin_b=${FLUX_BUILD_DIR}/t/stats/.libs/stats-basic.so
+
+test_expect_success 'prefix set' '
+	$timeout flux python $udp -s flux.job.state.immediate flux start \
+	"flux jobtap load $plugin_i && flux run hostname"
+'
+
+test_expect_success 'multiple packets received' '
+	$timeout flux python $udp -w 3 flux start \
+	"flux jobtap load $plugin_i && flux run hostname"
+'
+
+test_expect_success 'validate packets immediate' '
+	$timeout flux python $udp -V flux start \
+	"flux jobtap load $plugin_i && flux run hostname"
+'
+
+test_expect_success 'timing packets received immediate' '
+	$timeout flux python $udp -s timing flux start \
+	"flux jobtap load $plugin_i && flux run hostname"
+'
+
+test_expect_success 'timing packets received basic' '
+	$timeout flux python $udp -s timing flux start \
+	"flux jobtap load $plugin_b && flux run hostname && sleep 1"
+'
+
+test_expect_success 'valid content-cache packets received' '
+	$timeout flux python $udp -s content-cache -V flux start sleep 1
+'
+
+test_expect_success 'nothing received with no endpoint' '
+	unset FLUX_FRIPP_STATSD &&
+	test_expect_code 137 $timeout5 flux python $udp -n flux start
+'
+
+test_expect_success 'FLUX_FRIPP_STATSD with colectomy' '
+	FLUX_FRIPP_STATSD=localhost \
+		flux start true 2>colon.err &&
+	grep "parse error" colon.err
+'
+
+test_expect_success 'FLUX_FRIPP_STATSD with invalid hostname' '
+	FLUX_FRIPP_STATSD=thiscantpossiblybevalid:9000 \
+		flux start true 2>host.err &&
+	grep "parse error" host.err
+'
+
+test_done
diff --git a/t/t2004-hydra.t b/t/t2004-hydra.t
index 4b0aef283b80..e937cf614063 100755
--- a/t/t2004-hydra.t
+++ b/t/t2004-hydra.t
@@ -7,7 +7,7 @@ test_description='Test that MPICH Hydra can launch Flux'
 test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
 . `dirname $0`/sharness.sh
 PMI_INFO=${FLUX_BUILD_DIR}/src/common/libpmi/test_pmi_info
-ARGS="-o,-Sbroker.rc1_path=,-Sbroker.rc3_path=,--shutdown-grace=0.25"
+ARGS="-Sbroker.rc1_path= -Sbroker.rc3_path="
 
 if ! which mpiexec.hydra 2>/dev/null; then
     skip_all='skipping hydra-launching-flux tests, mpiexec.hydra unavailable'
diff --git a/t/t2005-hwloc-basic.t b/t/t2005-hwloc-basic.t
deleted file mode 100755
index ac14c01bbffc..000000000000
--- a/t/t2005-hwloc-basic.t
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/bin/sh
-#set -x
-
-test_description='Test basics of flux-hwloc reload subcommand
-
-Ensure flux-hwloc reload subcommand works
-'
-
-. `dirname $0`/sharness.sh
-
-SIZE=2
-test_under_flux ${SIZE} kvs
-
-HWLOC_DATADIR="${SHARNESS_TEST_SRCDIR}/hwloc-data"
-shared2=$(readlink -e ${HWLOC_DATADIR}/1N/shared/02-brokers)
-exclu2=$(readlink -e  ${HWLOC_DATADIR}/1N/nonoverlapping/02-brokers)
-sierra=$(readlink -e  ${HWLOC_DATADIR}/sierra2)
-
-test_debug '
-    echo ${shared} &&
-    echo ${exclu2} &&
-    echo ${sierra}
-'
-
-test_expect_success 'hwloc: load aggregator module' '
-    flux exec -r all flux module load aggregator
-'
-test_expect_success 'hwloc: load hwloc xml' '
-    flux hwloc reload -v
-'
-
-#  Set path to lstopo or lstopo-no-graphics command:
-#
-lstopo=$(which lstopo 2>/dev/null || which lstopo-no-graphics 2>/dev/null)
-test -n "$lstopo" && test_set_prereq HAVE_LSTOPO
-
-#  Set path to jq
-#
-jq=$(which jq 2>/dev/null)
-test -n "$jq" && test_set_prereq HAVE_JQ
-
-invalid_rank() {
-	echo $((${SIZE} + 1))
-}
-
-test_expect_success 'hwloc: ensure we have system lstopo output' '
-    flux hwloc topology > system.xml &&
-    test -f system.xml &&
-    test -s system.xml &&
-    grep "<object type=\"System\" os_index=\"0\">" system.xml
-'
-
-test_expect_success 'hwloc: each rank reloads a non-overlapping set of a node ' '
-    flux hwloc reload $exclu2
-'
-
-test_expect_success HAVE_JQ 'hwloc: internal aggregate-load cmd works' '
-    flux exec -r all \
-        flux hwloc aggregate-load --key=foo --unpack=bar --print-result | \
-	$jq -S . >aggregate.out &&
-    test_debug "flux kvs get bar" &&
-    cat <<-EOF | $jq -S . >aggregate.expected &&
-	{ "count": 2, "total": 2,
-	  "entries": {
-	    "0": { "Core": 8, "NUMANode": 1, "PU": 8, "Package": 1,
-                   "cpuset": "0-7" },
-	    "1": { "Core": 8, "NUMANode": 1, "PU": 8, "Package": 1,
-	           "cpuset": "8-15"}
-	  }
-	}
-	EOF
-    test_cmp aggregate.expected aggregate.out
-'
-
-
-test_expect_success HAVE_JQ 'hwloc: by_rank aggregate key exists after reload' '
-    flux kvs get resource.hwloc.by_rank | $jq -S . >  by_rank.out &&
-    cat <<-EOF | $jq -S . >by_rank.expected &&
-	{
-	  "0": { "Core": 8, "NUMANode": 1, "PU": 8, "Package": 1,
-                 "cpuset": "0-7" },
-	  "1": { "Core": 8, "NUMANode": 1, "PU": 8, "Package": 1,
-	         "cpuset": "8-15"}
-	}
-	EOF
-    test_cmp by_rank.expected by_rank.out
-'
-
-#  Keep this test after 'reload exclu2' above so we're processing
-#   know topology xml from reload.
-#
-test_expect_success 'hwloc: flux-hwloc info reports expected resources' '
-    cat <<-EOF > hwloc-info.expected1 &&
-	2 Machines, 16 Cores, 16 PUs
-	EOF
-    flux hwloc info > hwloc-info.out1 &&
-    test_cmp hwloc-info.expected1 hwloc-info.out1
-'
-
-test_expect_success 'hwloc: flux-hwloc info -r works' '
-    cat <<-EOF >hwloc-info-r.expected &&
-	1 Machine, 8 Cores, 8 PUs
-	EOF
-    flux hwloc info -r 1 > hwloc-info-r.out &&
-    test_cmp hwloc-info-r.expected hwloc-info-r.out
-'
-
-test_expect_success 'hwloc: every rank reloads the same xml of a node' '
-    flux hwloc reload $shared2 &&
-    cat <<-EOF > hwloc-info.expected2 &&
-	2 Machines, 32 Cores, 32 PUs
-	EOF
-    flux hwloc info > hwloc-info.out2 &&
-    test_cmp hwloc-info.expected2 hwloc-info.out2
-'
-
-test_expect_success HAVE_JQ 'hwloc: only one rank reloads an xml file' '
-    flux hwloc reload --rank="[0]" $exclu2 &&
-    flux kvs get resource.hwloc.by_rank | $jq -S . > mixed.out &&
-    cat <<-EOF | $jq -S . >mixed.expected &&
-	{
-	 "0": {"NUMANode": 1, "Package": 1, "Core": 8, "PU": 8,
-               "cpuset": "0-7" },
-	 "1": {"NUMANode": 2, "Package": 2, "Core": 16, "PU": 16,
-	       "cpuset": "0-15" }
-	}
-	EOF
-    test_cmp mixed.expected mixed.out
-'
-
-test_expect_success HAVE_JQ 'hwloc: reload xml with GPU resources' '
-    flux hwloc reload --rank=all $sierra &&
-    flux kvs get resource.hwloc.by_rank | $jq -S . > sierra.out &&
-    cat <<-EOF | $jq -S . > sierra.expected &&
-	{"[0-1]":
-          {"NUMANode": 6, "Package": 2, "Core": 44, "PU": 176, "GPU": 4,
-           "cpuset": "0-175" }
-        }
-	EOF
-    test_cmp sierra.expected sierra.out
-'
-
-test_expect_success 'hwloc: return an error code on an invalid DIR' '
-    test_expect_code 1 flux hwloc reload nonexistence
-'
-
-test_expect_success 'hwloc: return an error code on valid DIR, invalid files' '
-    test_expect_code 1 flux hwloc reload /
-'
-
-test_expect_success 'hwloc: reload with invalid rank fails' '
-    test_expect_code 1 flux hwloc reload -r $(invalid_rank) &&
-    test_expect_code 1 flux hwloc reload -r "0-$(invalid_rank)" &&
-    test_expect_code 1 flux hwloc reload -r foo
-'
-
-test_expect_success HAVE_LSTOPO 'hwloc: lstopo works' '
-    flux hwloc reload $exclu2 &&
-    flux hwloc lstopo > lstopo.out1 &&
-    sed -n 1p lstopo.out1 | grep "^System (32G.*"
-'
-
-test_expect_success HAVE_LSTOPO 'hwloc: lstopo subcommand passes options to lstopo' '
-    flux hwloc lstopo --help | grep ^Usage
-'
-
-test_expect_success HAVE_LSTOPO 'hwloc: topology subcommand works' '
-    flux hwloc topology > topology.out2 &&
-    flux hwloc lstopo > lstopo.out2 &&
-    $lstopo --if xml -i topology.out2 --of console > lstopo.out3 &&
-    test_cmp lstopo.out2 lstopo.out3
-'
-
-test_expect_success 'hwloc: reload with no args reloads system topology' '
-    flux hwloc reload &&
-    flux hwloc topology > system.out4 &&
-    test_cmp system.xml system.out4
-'
-
-test_expect_success HAVE_LSTOPO 'hwloc: test failure of lstopo command' '
-    test_must_fail flux hwloc lstopo --input f:g:y
-'
-
-test_expect_success 'hwloc: unload aggregator' '
-    flux exec -r all flux module remove aggregator
-'
-
-test_done
diff --git a/t/t2007-caliper.t b/t/t2007-caliper.t
deleted file mode 100755
index adf5d322b9e7..000000000000
--- a/t/t2007-caliper.t
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/sh
-#
-
-test_description='Test Caliper profiling support
-
-Test Caliper support.'
-
-# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
-test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
-. `dirname $0`/sharness.sh
-
-# Check for built in caliper tests
-
-if ! flux start --noexec --caliper-profile=thread-trace ; then
-    skip_all='skipping Caliper tests, Flux not built with Caliper support'
-    test_done
-fi
-
-test_expect_success '--caliper-profile works' '
-	flux start --caliper-profile=thread-trace /bin/true
-'
-
-CALIPER_OUTPUT=$(echo *.cali)
-test_expect_success 'caliper output file exists' '
-	test -f "$CALIPER_OUTPUT"
-'
-
-which cali-query >/dev/null 2>&1 && test_set_prereq HAVE_CALI_QUERY
-test_expect_success HAVE_CALI_QUERY 'caliper output file is readable by cali-query' '
-	cali-query "$CALIPER_OUTPUT"
-'
-
-test_done
diff --git a/t/t2008-althash.t b/t/t2008-althash.t
index da975ae935eb..1e515c080386 100755
--- a/t/t2008-althash.t
+++ b/t/t2008-althash.t
@@ -12,30 +12,66 @@ nil256="sha256-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
 . `dirname $0`/sharness.sh
 
+if test -n "$S3_ACCESS_KEY_ID"; then
+    test_set_prereq S3
+    export FLUX_CONF_DIR=$(pwd)
+fi
+
 test_expect_success 'Started instance with content.hash=sha1' '
-    OUT=$(flux start -o,-Scontent.hash=sha1 \
-          flux getattr content.hash) && test "$OUT" = "sha1"
+	OUT=$(flux start -Scontent.hash=sha1 \
+	    flux getattr content.hash) &&
+	test "$OUT" = "sha1"
 '
 
 test_expect_success 'Started instance with content.hash=sha256' '
-    OUT=$(flux start -o,-Scontent.hash=sha256 \
-          flux getattr content.hash) && test "$OUT" = "sha256"
+	OUT=$(flux start -Scontent.hash=sha256 \
+	    flux getattr content.hash) &&
+	test "$OUT" = "sha256"
+'
+
+test_expect_success 'Started instance with content.hash=sha256,content-files' '
+	OUT=$(flux start -Scontent.hash=sha256 \
+	    -Scontent.backing-module=content-files \
+	    -Sstatedir=$(pwd) \
+	    flux getattr content.hash) &&
+	test "$OUT" = "sha256" &&
+	ls -1 content.files | tail -1 | grep sha256
 '
 
-test_expect_success 'Content store nil returns correct hash for sha256' '
-    OUT=$(flux start -o,-Scontent.hash=sha256 \
-          flux content store </dev/null) &&
-        test "$OUT" = "$nil256"
+test_expect_success S3 'create creds.toml from env' '
+	mkdir -p creds &&
+	cat >creds/creds.toml <<-CREDS
+	access-key-id = "$S3_ACCESS_KEY_ID"
+	secret-access-key = "$S3_SECRET_ACCESS_KEY"
+	CREDS
 '
 
-test_expect_success 'Content store nil returns correct hash for sha1' '
-    OUT=$(flux start -o,-Scontent.hash=sha1 \
-          flux content store </dev/null) &&
-        test "$OUT" = "$nil1"
+# N.B. don't reuse the bucket from previous tests
+test_expect_success S3 'create content-s3.toml from env' '
+	cat >content-s3.toml <<-TOML
+	[content-s3]
+	credential-file = "$(pwd)/creds/creds.toml"
+	uri = "http://$S3_HOSTNAME"
+	bucket = "${S3_BUCKET}althash"
+	virtual-host-style = false
+	TOML
+'
+
+test_expect_success S3 'Started instance with content.hash=sha256,content-s3' '
+	OUT=$(flux start -Scontent.hash=sha256 \
+	    -Scontent.backing-module=content-s3 \
+	    flux getattr content.hash) &&
+	test "$OUT" = "sha256"
+'
+test_expect_success S3 'Content store nil returns correct hash for sha256' '
+	OUT=$(flux start -Scontent.hash=sha256 \
+	    -Scontent.backing-module=content-s3 \
+	    flux content store </dev/null) &&
+	test "$OUT" = "$nil256"
 '
 
 test_expect_success 'Attempt to start instance with invalid hash fails hard' '
-    test_must_fail flux start -o,-Scontent.hash=wronghash /bin/true
+	test_must_fail flux start -Scontent.hash=wronghash true
 '
 
 test_done
diff --git a/t/t2010-kvs-snapshot-restore.t b/t/t2010-kvs-snapshot-restore.t
index 6f0959a32891..6ab0038ad72b 100755
--- a/t/t2010-kvs-snapshot-restore.t
+++ b/t/t2010-kvs-snapshot-restore.t
@@ -9,54 +9,171 @@ test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
 
 test_under_flux 1
 
-CHECKPOINT=${FLUX_BUILD_DIR}/t/kvs/checkpoint
+CHANGECHECKPOINT=${FLUX_SOURCE_DIR}/t/kvs/change-checkpoint.py
 
-test_expect_success 'store kvs-checkpoint key-val pairs' '
-	$CHECKPOINT put foo bar &&
-	$CHECKPOINT put foo2 42 &&
-	$CHECKPOINT put foo3 "x x x"
+#
+# test content-sqlite backing
+#
+
+test_expect_success 'run instance with statedir set (sqlite)' '
+	flux start --setattr=statedir=$(pwd) \
+		   flux kvs put --sequence testkey=42 \
+		   > start_sequence_sqlite.out
 '
 
-test_expect_success 'verify kvs-checkpoint key-val pairs' '
-	test "$($CHECKPOINT get foo)" = "bar" &&
-	test "$($CHECKPOINT get foo2)" = "42" &&
-	test "$($CHECKPOINT get foo3)" = "x x x"
+test_expect_success 'content.sqlite file exists after instance exited' '
+	test -f content.sqlite &&
+	echo Size in bytes: $(stat --format "%s" content.sqlite)
 '
 
-test_expect_success 'get unknown kvs-checkpoint key fails' '
-	test_must_fail $CHECKPOINT get noexist
+test_expect_success 're-run instance with statedir set (sqlite)' '
+	flux start --setattr=statedir=$(pwd) \
+		   flux kvs get testkey >getsqlite.out
 '
 
-test_expect_success 'put existing kvs-checkpoint key is allowed' '
-	$CHECKPOINT put foo zzz
+test_expect_success 'content from previous instance survived (sqlite)' '
+	echo 42 >getsqlite.exp &&
+	test_cmp getsqlite.exp getsqlite.out
 '
 
-test_expect_success 'kvs-checkpoint value was updated' '
-	test $($CHECKPOINT get foo) = "zzz"
+# due to other KVS activity that testing can't control, we simply want
+# to ensure the sequence number does not restart at 0, it must
+# increase over several restarts
+
+test_expect_success 're-run instance, get sequence number 1 (sqlite)' '
+	flux start --setattr=statedir=$(pwd) \
+		   flux kvs version > restart_version_sqlite1.out
 '
 
-test_expect_success 'empty kvs-checkpoint key is not allowed' '
-	test_must_fail $CHECKPOINT put "" xyz
+test_expect_success 'restart sequence number increasing 1 (sqlite)' '
+	seq1=$(cat start_sequence_sqlite.out) &&
+	seq2=$(cat restart_version_sqlite1.out) &&
+	test $seq1 -lt $seq2
 '
 
-test_expect_success 'run instance with content.backing-path set' '
-	flux start -o,--setattr=content.backing-path=$(pwd)/content.sqlite \
-	           flux kvs put testkey=42
+test_expect_success 're-run instance, get sequence number 2 (sqlite)' '
+	flux start --setattr=statedir=$(pwd) \
+		   flux kvs version > restart_version_sqlite2.out
 '
 
-test_expect_success 'content.sqlite file exists after instance exited' '
-	test -f content.sqlite &&
-	echo Size in bytes: $(stat --format "%s" content.sqlite)
+test_expect_success 'restart sequence number increasing 2 (sqlite)' '
+	seq1=$(cat restart_version_sqlite1.out) &&
+	seq2=$(cat restart_version_sqlite2.out) &&
+	test $seq1 -lt $seq2
+'
+
+test_expect_success 're-run instance, verify checkpoint date saved (sqlite)' '
+	flux start --setattr=statedir=$(pwd) \
+		   flux dmesg >dmesgsqlite1.out
+'
+
+# just check for todays date, not time for obvious reasons
+test_expect_success 'verify date in flux logs (sqlite)' '
+	today=`date --iso-8601` &&
+	grep checkpoint dmesgsqlite1.out | grep ${today}
+'
+
+test_expect_success 're-run instance, get rootref (sqlite)' '
+	flux start --setattr=statedir=$(pwd) \
+		   flux kvs getroot -b > getrootsqlite.out
+'
+
+test_expect_success 'write rootref to checkpoint path, emulating checkpoint version=0 (sqlite)' '
+	rootref=$(cat getrootsqlite.out) &&
+	${CHANGECHECKPOINT} $(pwd)/content.sqlite "kvs-primary" ${rootref}
+'
+
+test_expect_success 're-run instance, verify checkpoint correctly loaded (sqlite)' '
+	flux start --setattr=statedir=$(pwd) \
+		   flux dmesg >dmesgsqlite2.out
+'
+
+test_expect_success 'verify checkpoint loaded with no date (sqlite)' '
+	grep checkpoint dmesgsqlite2.out | grep "N\/A"
+'
+
+#
+# test content-files backing
+#
+
+test_expect_success 'generate rc1/rc3 for content-files backing' '
+	cat >rc1-content-files <<EOF &&
+#!/bin/bash -e
+flux module load content
+flux module load content-files
+flux module load kvs
+EOF
+	cat >rc3-content-files <<EOF &&
+#!/bin/bash -e
+flux module remove kvs
+flux module remove content-files
+flux module remove content
+EOF
+	chmod +x rc1-content-files &&
+	chmod +x rc3-content-files
+'
+
+test_expect_success 'run instance with statedir set (files)' '
+	flux start --setattr=statedir=$(pwd) \
+		   --setattr=broker.rc1_path=$(pwd)/rc1-content-files \
+		   --setattr=broker.rc3_path=$(pwd)/rc3-content-files \
+		   flux kvs put --sequence testkey=43 > start_sequence_files.out
+'
+
+test_expect_success 'content.files dir and kvs-primary exist after instance exit' '
+	test -d content.files &&
+	test -e content.files/kvs-primary
+'
+
+test_expect_success 're-run instance with statedir set (files)' '
+	flux start --setattr=statedir=$(pwd) \
+		   --setattr=broker.rc1_path=$(pwd)/rc1-content-files \
+		   --setattr=broker.rc3_path=$(pwd)/rc3-content-files \
+		   flux kvs get testkey >getfiles.out
+'
+
+test_expect_success 'content from previous instance survived (files)' '
+	echo 43 >getfiles.exp &&
+	test_cmp getfiles.exp getfiles.out
+'
+
+# due to other KVS activity that testing can't control, we simply want
+# to ensure the sequence number does not restart at 0, it must
+# increase over several restarts
+
+test_expect_success 're-run instance, get sequence number 1 (files)' '
+	flux start --setattr=statedir=$(pwd) \
+		   flux kvs version > restart_version_files1.out
+'
+
+test_expect_success 'restart sequence number increasing 1 (files)' '
+	seq1=$(cat start_sequence_files.out) &&
+	seq2=$(cat restart_version_files1.out) &&
+	test $seq1 -lt $seq2
+'
+
+test_expect_success 're-run instance, get sequence number 2 (files)' '
+	flux start --setattr=statedir=$(pwd) \
+		   flux kvs version > restart_version_files2.out
+'
+
+test_expect_success 'restart sequence number increasing 2 (files)' '
+	seq1=$(cat restart_version_files1.out) &&
+	seq2=$(cat restart_version_files2.out) &&
+	test $seq1 -lt $seq2
 '
 
-test_expect_success 're-run instance with content.backing-path set' '
-	flux start -o,--setattr=content.backing-path=$(pwd)/content.sqlite \
-	           flux kvs get testkey >get.out
+test_expect_success 're-run instance, verify checkpoint date saved (files)' '
+	flux start --setattr=statedir=$(pwd) \
+		   --setattr=broker.rc1_path=$(pwd)/rc1-content-files \
+		   --setattr=broker.rc3_path=$(pwd)/rc3-content-files \
+		   flux dmesg >dmesgfiles.out
 '
 
-test_expect_success 'content from previous instance survived' '
-	echo 42 >get.exp &&
-	test_cmp get.exp get.out
+# just check for todays date, not time for obvious reasons
+test_expect_success 'verify date in flux logs (files)' '
+	today=`date --iso-8601` &&
+	grep checkpoint dmesgfiles.out | grep ${today}
 '
 
 test_done
diff --git a/t/t2100-aggregate.t b/t/t2100-aggregate.t
deleted file mode 100755
index be86c7273e08..000000000000
--- a/t/t2100-aggregate.t
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/bin/sh
-
-test_description='Test basic flux aggreagation via aggregator module'
-
-. `dirname $0`/sharness.sh
-
-RPC=${FLUX_BUILD_DIR}/t/request/rpc
-
-test_under_flux 8
-
-#  Set path to jq
-#
-jq=$(which jq 2>/dev/null)
-if test -z "$jq"; then
-    skip_all='jq not found. Skipping all tests'
-    test_done
-fi
-
-kvs_json_check() {
-    flux kvs get $1 | $jq -e "$2"
-}
-
-test_expect_success 'have aggregator module' '
-    flux exec -r all flux module list | grep aggregator
-'
-
-test_expect_success 'flux-aggregate: works' '
-    run_timeout 5 flux exec -n -r 0-7 flux aggregate -v test 1 &&
-    kvs_json_check test ".count == 8 and .min == 1 and .max == 1"
-'
-
-test_expect_success 'flux-aggregate: works for floating-point numbers' '
-    run_timeout 5 flux exec -n -r 0-7 flux aggregate test 1.825 &&
-    kvs_json_check test ".count == 8 and .min == 1.825 and .max == 1.825"
-'
-test_expect_success 'flux-aggregate: works for strings' '
-    run_timeout 5 flux exec -n -r 0-7 flux aggregate test \"foo\" &&
-    flux kvs get test &&
-    kvs_json_check test ".count == 8" &&
-    kvs_json_check test ".entries.\"[0-7]\" == \"foo\""
-'
-test_expect_success 'flux-aggregate: works for arrays' '
-    run_timeout 5 flux exec -n -r 0-7 flux aggregate test "[7,8,9]" &&
-    flux kvs get test &&
-    kvs_json_check test ".count == 8" &&
-    kvs_json_check test "(.entries | length) == 1" &&
-    kvs_json_check test ".entries.\"[0-7]\" == [7,8,9]"
-'
-test_expect_success 'flux-aggregate: works for objects' '
-    run_timeout 5 flux exec -n -r 0-7 flux aggregate test \
-                  "{\"foo\":42, \"bar\": {\"baz\": 2}}" &&
-    flux kvs get test &&
-    kvs_json_check test ".count == 8" &&
-    kvs_json_check test ".entries.\"[0-7]\".foo == 42" &&
-    kvs_json_check test ".entries.\"[0-7]\".bar.baz == 2"
-'
-test_expect_success 'flux-aggregate: abort works' '
-    test_expect_code 1 run_timeout 5 flux exec -n -r 0-7 flux aggregate . 1
-'
-
-test_expect_success 'flux-aggregate: different value per rank' '
-    run_timeout 5 flux exec -n -r 0-7 bash -c "flux aggregate test \$(flux getattr rank)" &&
-    kvs_json_check test ".count == 8 and .min == 0 and .max == 7"
-'
-
-test_expect_success 'flux-aggregate: different fp value per rank' '
-    run_timeout 5 flux exec -n -r 0-7 bash -c "flux aggregate test 1.\$(flux getattr rank)" &&
-    kvs_json_check test ".count == 8 and .min == 1 and .max == 1.7"
-'
-
-test_expect_success 'flux-aggregate: --timeout=0. - immediate forward' '
-    run_timeout 5 flux exec -n -r 0-7 flux aggregate -t 0. test 1 &&
-    kvs_json_check test \
-        ".count == 8 and .total == 8 and .min == 1 and .max == 1"
-'
-
-test_expect_success 'flux-aggregate: --fwd-count works' '
-    run_timeout 5 flux exec -n -r 0-7 bash -c \
-     "flux aggregate -t10 -c \$((1+\$(flux getattr tbon.descendants))) test 1" &&
-    kvs_json_check test \
-        ".count == 8 and .total == 8 and .min == 1 and .max == 1"
-'
-
-test_expect_success 'push request with empty payload fails with EPROTO(71)' '
-	${RPC} aggregator.push 71 </dev/null
-'
-
-test_done
-
-# vi: ts=4 sw=4 expandtab
-
diff --git a/t/t2100-job-ingest.t b/t/t2100-job-ingest.t
new file mode 100755
index 000000000000..da0e9584522c
--- /dev/null
+++ b/t/t2100-job-ingest.t
@@ -0,0 +1,201 @@
+#!/bin/sh
+
+test_description='Test flux job ingest service'
+
+. $(dirname $0)/sharness.sh
+
+if ${FLUX_BUILD_DIR}/t/ingest/submitbench --help 2>&1 | grep -q sign-type; then
+    test_set_prereq HAVE_FLUX_SECURITY
+    SUBMITBENCH_OPT_R="--reuse-signature"
+fi
+
+test_under_flux 4 kvs
+
+flux setattr log-stderr-level 1
+
+JOBSPEC=${SHARNESS_TEST_SRCDIR}/jobspec
+Y2J="flux python ${JOBSPEC}/y2j.py"
+SUBMITBENCH="${FLUX_BUILD_DIR}/t/ingest/submitbench"
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+
+DUMMY_EVENTLOG=test.ingest.eventlog
+
+DUMMY_MAX_JOBID=16777216000000
+DUMMY_FLUID_TS=1000000
+
+# load|reload ingest modules (in proper order) with specified arguments
+ingest_module ()
+{
+    cmd=$1; shift
+    flux module ${cmd} job-ingest $* &&
+    flux exec -r all -x 0 flux module ${cmd} job-ingest $*
+}
+
+test_expect_success 'job-ingest: convert basic.yaml to json' '
+	${Y2J} <${JOBSPEC}/valid/basic.yaml >basic.json
+'
+
+test_expect_success 'job-ingest: convert use_case_2.6.yaml to json' '
+	${Y2J} <${JOBSPEC}/valid/use_case_2.6.yaml >use_case_2.6.json
+'
+
+test_expect_success 'job-ingest: submit fails without job-ingest' '
+	test_must_fail flux job submit basic.json 2>nosys.out
+'
+
+test_expect_success 'job-ingest: load job-manager-dummy module' '
+	flux module load \
+		${FLUX_BUILD_DIR}/t/ingest/.libs/job-manager.so
+'
+
+test_expect_success 'job-ingest: job-ingest fails with bad option' '
+	test_must_fail flux module load job-ingest badopt=xyz
+'
+
+test_expect_success 'job-ingest: load job-ingest: require-version=any' '
+	ingest_module load \
+		validator-plugins=jobspec \
+		validator-args=--require-version=any
+'
+
+test_expect_success 'job-ingest: dummy job-manager has expected max_jobid' '
+	max_jobid=$(${RPC} job-manager.getinfo | jq .max_jobid) &&
+	test ${max_jobid} -eq ${DUMMY_MAX_JOBID}
+'
+
+test_expect_success 'job-ingest: max_jobid <= rank 0 FLUID timestamp' '
+	ts0=$(${RPC} job-ingest.getinfo | jq .timestamp) &&
+	test ${DUMMY_FLUID_TS} -le ${ts0}
+'
+
+test_expect_success 'job-ingest: rank 0 FLUID timestamp <= rank 1' '
+	ts1=$(flux exec -r1 ${RPC} job-ingest.getinfo | jq .timestamp) &&
+	test ${ts0} -le ${ts1}
+'
+
+test_expect_success 'job-ingest: YAML jobspec is rejected' '
+	test_must_fail flux job submit ${JOBSPEC}/valid/basic.yaml
+'
+
+test_expect_success 'job-ingest: fetch jobspec from KVS' '
+	jobid=$(flux job submit basic.json) &&
+	kvsdir=$(flux job id --to=kvs $jobid) &&
+	flux kvs get --raw ${kvsdir}.jobspec >jobspec.out
+'
+test_expect_success 'job-ingest: jobspec stored accurately in KVS' '
+	jq --sort-keys . <basic.json >basic.json.normalized &&
+	jq --sort-keys . <jobspec.out >jobspec.out.normalized &&
+	test_cmp basic.json.normalized jobspec.out.normalized
+'
+
+test_expect_success 'job-ingest: submit a job with environment' '
+	flux run --env=-* --env=FOO=bar --dry-run true \
+	    >jobspec_env.json &&
+	jobid=$(flux job submit jobspec_env.json) &&
+	kvsdir=$(flux job id --to=kvs $jobid) &&
+	flux kvs get --raw ${kvsdir}.jobspec >jobspec_env.out
+'
+test_expect_success 'job-ingest: KVS jobspec lacks environment' '
+	jq -e ".attributes.system.environment.FOO == \"bar\"" \
+	    <jobspec_env.json &&
+	test_must_fail jq -e ".attributes.system.environment.FOO == \"bar\"" \
+	    <jobspec_env.out
+'
+
+test_expect_success 'job-ingest: job announced to job manager' '
+	jobid=$(flux job submit --urgency=10 basic.json | flux job id) &&
+	flux kvs eventlog get ${DUMMY_EVENTLOG} \
+		| grep "id=${jobid}" >jobman.out &&
+	grep -q "urgency=10" jobman.out &&
+	grep -q "userid=$(id -u)" jobman.out
+'
+
+test_expect_success 'job-ingest: instance owner can submit urgency=31' '
+	flux job submit --urgency=31 basic.json
+'
+
+test_expect_success 'job-ingest: urgency range is enforced' '
+	test_must_fail flux job submit --urgency=32 basic.json &&
+	test_must_fail flux job submit --urgency="-1" basic.json
+'
+
+test_expect_success 'job-ingest: guest cannot submit urgency=17' '
+	test_must_fail bash -c "FLUX_HANDLE_ROLEMASK=0x2 \
+		flux job submit --urgency=17 basic.json"
+'
+
+test_expect_success 'job-ingest: guest cannot submit --flags=novalidate' '
+        test_must_fail bash -c "FLUX_HANDLE_ROLEMASK=0x2 \
+                flux job submit --flags=novalidate basic.json"
+'
+
+test_expect_success NO_ASAN 'job-ingest: submit job 100 times' '
+	${SUBMITBENCH} -r 100 use_case_2.6.json
+'
+
+test_expect_success NO_ASAN 'job-ingest: submit job 100 times, reuse signature' '
+	${SUBMITBENCH} ${SUBMITBENCH_OPT_R} -r 100 use_case_2.6.json
+'
+
+test_expect_success HAVE_FLUX_SECURITY 'job-ingest: submit user != signed user fails' '
+	test_must_fail bash -c "FLUX_HANDLE_USERID=9999 \
+		flux job submit basic.json" 2>baduser.out &&
+	grep -q "signer=$(id -u) != requestor=9999" baduser.out
+'
+
+test_expect_success HAVE_FLUX_SECURITY 'job-ingest: non-owner mech=none fails' '
+	test_must_fail bash -c "FLUX_HANDLE_ROLEMASK=0x2 flux job submit \
+		--sign-type=none basic.json" 2>badrole.out &&
+	grep -q "only instance owner" badrole.out
+'
+
+test_expect_success 'submit request with empty payload fails with EPROTO(71)' '
+	${RPC} job-ingest.submit 71 </dev/null
+'
+
+test_expect_success 'job-ingest: reload dummy job-manager in fail mode' '
+	ingest_module reload batch-count=4 &&
+	flux module remove job-manager &&
+	flux module load \
+	    ${FLUX_BUILD_DIR}/t/ingest/.libs/job-manager.so force_fail
+'
+
+test_expect_success 'job-ingest: handle total batch failure in job-ingest' '
+	test_must_fail flux submit --cc=1-4 hostname
+'
+
+test_expect_success 'flux module stats job-ingest is open to guests' '
+	FLUX_HANDLE_ROLEMASK=0x2 \
+	    flux module stats job-ingest >/dev/null
+'
+
+test_expect_success 'reload the real job-manager' '
+	flux module reload job-manager
+'
+test_expect_success 'unload job-ingest on all ranks' '
+	flux exec -r all flux module remove job-ingest
+'
+test_expect_success 'load job-ingest max-fluid-generator-id=1000000 fails' '
+	test_must_fail flux module load job-ingest \
+	    max-fluid-generator-id=1000000
+'
+test_expect_success 'reload job-ingest max-fluid-generator-id=2 on all ranks' '
+	for rank in $(seq 0 3); do \
+	    flux exec -r $rank flux module load job-ingest \
+	        max-fluid-generator-id=2; \
+	done
+'
+test_expect_success 'module is not loaded on rank 3' '
+	flux exec -r 3 flux module list >list3.out &&
+	test_must_fail grep job-ingest list3.out
+'
+test_expect_success 'but a job can be submitted on rank 3' '
+	flux exec -r 3 flux submit true
+'
+
+test_expect_success 'job-ingest: remove modules' '
+	flux module remove -f job-manager &&
+	flux exec -r all flux module remove -f job-ingest
+'
+
+test_done
diff --git a/t/t2110-job-ingest-validator.t b/t/t2110-job-ingest-validator.t
new file mode 100755
index 000000000000..4e05e7a38e66
--- /dev/null
+++ b/t/t2110-job-ingest-validator.t
@@ -0,0 +1,203 @@
+#!/bin/sh
+test_description='Test job validator'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 job
+
+flux setattr log-stderr-level 1
+
+JOBSPEC=${SHARNESS_TEST_SRCDIR}/jobspec
+Y2J="flux python ${JOBSPEC}/y2j.py"
+SUBMITBENCH="${FLUX_BUILD_DIR}/t/ingest/submitbench"
+BAD_VALIDATOR=${SHARNESS_TEST_SRCDIR}/ingest/bad-validate.py
+export FLUX_URI_RESOLVE_LOCAL=t
+
+test_valid ()
+{
+	flux bulksubmit --quiet --wait --watch \
+		sh -c "cat {} | $Y2J | $SUBMITBENCH --urgency=0 -" ::: $*
+}
+
+test_invalid ()
+{
+	flux bulksubmit --quiet --wait --watch \
+		sh -c "cat {} | $Y2J | $SUBMITBENCH --urgency=0 -" ::: $*
+	test $? -ne 0
+}
+
+# load|reload ingest modules (in proper order) with specified arguments
+ingest_module ()
+{
+    cmd=$1; shift
+    flux module ${cmd} job-ingest $* &&
+    flux exec -r all -x 0 flux module ${cmd} job-ingest $*
+}
+
+test_expect_success 'flux job-validator works' '
+	flux run --dry-run hostname | flux job-validator --jobspec-only
+'
+#  Attempt to trick validator into loading a bad urllib by modification
+#  of PYTHONPATH (issue #5547 reproducer). This affects all Python utils,
+#  but we test the validator as a surrogate for the rest:
+#
+test_expect_success 'flux job-validator works with malicious PYTHONPATH' '
+	mkdir -p badmod/urllib &&
+	cat <<-EOF >badmod/urllib/__init__.py &&
+	raise ValueError("incorrect urllib Python module loaded")
+	EOF
+	flux run --dry-run hostname \
+	  | PYTHONPATH=$(pwd)/badmod:${PYTHONPATH} \
+	       flux job-validator --jobspec-only
+'
+test_expect_success 'flux job-validator --list-plugins works' '
+	flux job-validator --list-plugins >list-plugins.output 2>&1 &&
+	test_debug "cat list-plugins.output" &&
+	grep jobspec list-plugins.output &&
+	grep feasibility list-plugins.output &&
+	grep require-instance list-plugins.output
+'
+test_expect_success 'validator plugin behaves when no plugins are found' '
+	cat <<-EOF >test-importer.py &&
+	from flux.importer import import_plugins
+	type(import_plugins("xxyyzz112233"))
+	EOF
+	flux python ./test-importer.py
+'
+test_expect_success 'validator plugin importer reports errors on import' '
+	mkdir t2110plugins &&
+	cat <<-EOF >t2110plugins/test.py &&
+	import froufroufoxes
+	EOF
+	cat <<-EOF >test-importer2.py &&
+	from flux.importer import import_plugins
+	import_plugins("t2110plugins")
+	EOF
+	test_must_fail flux python ./test-importer2.py >import-fail.out 2>&1 &&
+	test_debug "cat import-fail.out" &&
+	grep "No module named.*froufroufoxes" import-fail.out
+'
+test_expect_success 'flux job-validator --help shows help for selected plugins' '
+	flux job-validator --plugins=jobspec --help >help.jobspec.out 2>&1 &&
+	grep require-version help.jobspec.out
+'
+test_expect_success 'flux job-validator errors on invalid plugin' '
+	test_expect_code 1 flux job-validator --plugin=foo </dev/null &&
+	test_expect_code 1 flux job-validator --plugin=/tmp </dev/null
+'
+test_expect_success 'flux job-validator --require-version rejects invalid arg' '
+	flux run --dry-run hostname | \
+		test_expect_code 1 \
+		flux job-validator --jobspec-only --require-version=99 &&
+	flux run --dry-run hostname | \
+		test_expect_code 1 \
+		flux job-validator --jobspec-only --require-version=0
+'
+test_expect_success 'flux job-validator rejects non-V1 jobspec' '
+	flux run --dry-run hostname | jq -c ".version = 2" | \
+		test_expect_code 1 \
+		flux job-validator --jobspec-only --require-version=1
+'
+test_expect_success 'job-ingest: v1 jobspecs accepted by default' '
+	test_valid ${JOBSPEC}/valid_v1/*
+'
+test_expect_success 'job-ingest: test jobspec validator with any version' '
+	ingest_module reload \
+		validator-plugins=jobspec \
+		validator-args="--require-version=any"
+'
+test_expect_success 'job-ingest: all valid jobspecs accepted' '
+	test_valid ${JOBSPEC}/valid/*
+'
+test_expect_success 'job-ingest: invalid jobs rejected' '
+	test_invalid ${JOBSPEC}/invalid/*
+'
+test_expect_success 'job-ingest: stop the queue so no more jobs run' '
+	flux queue stop
+'
+test_expect_success 'job-ingest: load feasibilty validator plugin' '
+	ingest_module reload validator-plugins=feasibility
+'
+test_expect_success 'job-ingest: feasibility check succeeds with ENOSYS' '
+	flux module remove sched-simple &&
+	flux submit -g 1 hostname &&
+	flux submit -n 10000 hostname &&
+	flux module load sched-simple
+'
+test_expect_success 'job-ingest: infeasible jobs are now rejected' '
+	test_must_fail flux submit -g 1 hostname 2>infeasible1.err &&
+	test_debug "cat infeasible1.err" &&
+	grep -i "unsupported resource type" infeasible1.err &&
+	test_must_fail flux submit -n 10000 hostname 2>infeasible2.err &&
+	test_debug "cat infeasible2.err" &&
+	grep "unsatisfiable request" infeasible2.err &&
+	test_must_fail flux submit -N 12 -n12 hostname 2>infeasible3.err &&
+	test_debug "cat infeasible3.err" &&
+	grep "unsatisfiable request" infeasible3.err
+'
+test_expect_success 'job-ingest: feasibility validator works with jobs running' '
+	ncores=$(flux resource list -s up -no {ncores}) &&
+	flux queue start &&
+	jobid=$(flux submit -n${ncores} sleep inf) &&
+	flux job wait-event -vt 20 ${jobid} start &&
+	flux queue stop &&
+	flux submit -n 2 hostname &&
+	test_must_fail flux submit -N 12 -n12 hostname 2>infeasible4.err &&
+	grep "unsatisfiable request" infeasible4.err &&
+	flux cancel ${jobid} &&
+	flux job wait-event ${jobid} clean
+'
+test_expect_success 'job-ingest: load multiple validators' '
+	ingest_module reload validator-plugins=feasibility,jobspec
+'
+test_expect_success 'job-ingest: jobs that fail either validator are rejected' '
+	test_must_fail flux submit --setattr=.foo=bar hostname &&
+	test_must_fail flux submit -n 4568 hostname
+'
+test_expect_success 'job-ingest: validator unexpected exit is handled' '
+	ingest_module reload \
+		validator-plugins=${BAD_VALIDATOR} &&
+		test_must_fail flux submit hostname 2>badvalidator.out &&
+	grep "unexpectedly exited" badvalidator.out
+'
+test_expect_success 'job-ingest: require-instance validator plugin works' '
+	ingest_module reload validator-plugins=require-instance &&
+	flux batch -n1 --wrap flux resource list &&
+	flux submit -n1 flux start flux resource list &&
+	flux submit -n1 flux broker flux resource list &&
+	test_must_fail flux submit hostname &&
+	test_must_fail flux submit flux getattr rank
+'
+test_expect_success 'job-ingest: require-instance min size can be configured' '
+	ARGS="--require-instance-minnodes=2,--require-instance-mincores=4" &&
+	ingest_module reload validator-plugins=require-instance \
+		validator-args="$ARGS" &&
+	flux submit -N1 hostname &&
+	flux submit -n2 hostname &&
+	test_must_fail flux submit -N4 hostname &&
+	test_must_fail flux submit -n4 hostname
+'
+test_expect_success 'job-ingest: require-instance min size can be for nodes only' '
+	ARGS="--require-instance-minnodes=2" &&
+	ingest_module reload validator-plugins=require-instance \
+		validator-args="$ARGS" &&
+	flux submit -N1 hostname &&
+	flux submit -n2 hostname &&
+	test_must_fail flux submit -N4 hostname &&
+	test_must_fail flux submit -n32 hostname
+'
+test_expect_success 'job-ingest: kill all jobs and start the queue' '
+	flux cancel --all &&
+	flux queue idle &&
+	flux queue start
+'
+test_expect_success 'job-ingest: require-instance min size can use config' '
+	jobid=$(flux alloc --bg -N4 \
+		--conf=ingest.validator.plugins="[\"require-instance\"]" \
+		--conf=ingest.validator.require-instance.minnodes=4 \
+		--conf=ingest.validator.require-instance.mincores=16) &&
+	flux proxy $jobid flux run hostname &&
+	test_must_fail flux proxy $jobid flux run -N4 hostname 2>config.err &&
+	grep "Direct job submission disabled for jobs >= 4" config.err
+'
+test_done
diff --git a/t/t2111-job-ingest-config.t b/t/t2111-job-ingest-config.t
new file mode 100755
index 000000000000..0537cd9f0fa2
--- /dev/null
+++ b/t/t2111-job-ingest-config.t
@@ -0,0 +1,104 @@
+#!/bin/sh
+test_description='Test job ingest config file'
+
+. $(dirname $0)/sharness.sh
+
+#
+# Setup a config dir for this test_under_flux
+#
+export FLUX_CONF_DIR=$(pwd)/conf.d
+mkdir -p conf.d
+cat <<EOF >conf.d/ingest.toml
+[ingest.validator]
+disable = true
+EOF
+
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+SUBMITBENCH="${FLUX_BUILD_DIR}/t/ingest/submitbench"
+dmesg_grep=${SHARNESS_TEST_SRCDIR}/scripts/dmesg-grep.py
+
+test_expect_success 'job-ingest: validator was disabled by config' '
+	$dmesg_grep -t 5 "configuring validator with plugins=\(null\), args=\(null\) \(disabled\)"
+'
+test_expect_success 'job-ingest: command line overrides config' '
+	rm conf.d/ingest.toml &&
+	flux config reload &&
+	flux dmesg -C &&
+	flux module reload job-ingest disable-validator &&
+	$dmesg_grep -t 10 "configuring validator with plugins=\(null\), args=\(null\) \(disabled\)"
+'
+test_expect_success 'job-ingest: configuration can be reloaded' '
+	cat <<-EOF >conf.d/ingest.toml &&
+	[ingest.validator]
+	plugins = [ "feasibility", "jobspec" ]
+	args = [ "--require-version=1" ]
+	EOF
+	flux dmesg -C &&
+	flux config reload &&
+	$dmesg_grep -vt 10 \
+	"configuring validator with plugins=feasibility,jobspec, args=--require-version=1 \(enabled\)"
+'
+test_expect_success 'job-ingest: verify that feasibility plugin is in effect' '
+	test_must_fail flux submit -n 1024 hostname
+'
+test_expect_success 'job-ingest: worker buffer size can be set via config' '
+	cat <<-EOF >conf.d/ingest.toml &&
+	[ingest]
+	buffer-size = "10M"
+	[ingest.validator]
+	plugins = [ "feasibility", "jobspec" ]
+	args = [ "--require-version=1" ]
+	EOF
+	flux dmesg -C &&
+	flux config reload &&
+	$dmesg_grep -vt 10 "worker input buffer set to 10M"
+'
+test_expect_success 'job-ingest: invalid buffer size is detected on reload' '
+	cat <<-EOF >conf.d/ingest.toml &&
+	[ingest]
+	buffer-size = "10f"
+	[ingest.validator]
+	plugins = [ "feasibility", "jobspec" ]
+	args = [ "--require-version=1" ]
+	EOF
+	test_must_fail flux config reload >bad-buffer-size.out 2>&1 &&
+	test_debug "cat bad-buffer-size.out" &&
+	grep "Invalid buffer-size" bad-buffer-size.out
+'
+test_expect_success 'job-ingest: invalid config detected on reload' '
+	cat <<-EOF >conf.d/ingest.toml &&
+	[ingest.validator]
+	foo = 42
+	args = "--require-version=1"
+	EOF
+	test_must_fail flux config reload &&
+	flux dmesg | grep ingest
+'
+test_expect_success 'job-ingest: job still runs after failed config reload' '
+	flux run true
+'
+test_expect_success 'job-ingest: invalid ingest.validator.plugins' '
+	cat <<-EOF >conf.d/ingest.toml &&
+	[ingest.validator]
+	plugins = [  42 ]
+	EOF
+	test_must_fail flux config reload
+'
+test_expect_success 'job-ingest: job still runs after failed config reload' '
+	flux run true
+'
+test_expect_success 'job-ingest: invalid ingest.validator.plugins' '
+	cat <<-EOF >conf.d/ingest.toml &&
+	[ingest.validator]
+	plugins = [ "feasibility" ]
+	args = [ 1 ]
+	EOF
+	test_must_fail flux config reload
+'
+test_expect_success 'job-ingest: job still runs after failed config reload' '
+	flux run true
+'
+test_done
diff --git a/t/t2112-job-ingest-frobnicator.t b/t/t2112-job-ingest-frobnicator.t
new file mode 100755
index 000000000000..55aa82c5b0b2
--- /dev/null
+++ b/t/t2112-job-ingest-frobnicator.t
@@ -0,0 +1,211 @@
+#!/bin/sh
+test_description='Test job frobnicator command'
+
+. $(dirname $0)/sharness.sh
+
+#  Setup a config dir for this test_under_flux
+export FLUX_CONF_DIR=$(pwd)/conf.d
+mkdir -p conf.d
+cat <<EOF >conf.d/conf.toml
+[policy.jobspec.defaults.system]
+duration = "30m"
+EOF
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+JOBSPEC=${SHARNESS_TEST_SRCDIR}/jobspec
+Y2J="flux python ${JOBSPEC}/y2j.py"
+
+test_expect_success 'flux job-frobnicator works' '
+	flux run --env=-* --dry-run hostname \
+		| flux job-frobnicator --jobspec-only
+'
+test_expect_success 'flux job-frobnicator --list-plugins works' '
+	flux job-frobnicator --list-plugins >list-plugins.output 2>&1 &&
+	test_debug "cat list-plugins.output" &&
+	grep defaults list-plugins.output
+'
+test_expect_success 'flux job-frobnicator --help can show help for plugins' '
+	cat <<-EOF >test-plugin.py &&
+	from flux.job.frobnicator import FrobnicatorPlugin
+	class Frobnicator(FrobnicatorPlugin):
+	    def __init__(self, parser):
+	        self.test = False
+	        parser.add_argument("--test", action="store_true")
+
+	    def configure(self, args):
+	        self.test = args.test
+
+	    def frob(self, jobspec, user, urgency, flags):
+	        pass
+	EOF
+	flux job-frobnicator --plugins=./test-plugin.py --help >help.out 2>&1 &&
+	test_debug "cat help.out" &&
+	grep "Options provided by plugins" help.out &&
+	grep "\-\-test" help.out
+'
+test_expect_success 'flux job-frobnicator errors on invalid plugin' '
+	test_expect_code 1 flux job-frobnicator --plugin=foo </dev/null &&
+	test_expect_code 1 flux job-frobnicator --plugin=/tmp </dev/null
+'
+test_expect_success 'flux job-frobnicator exits on eof' '
+	flux job-frobnicator --plugins=defaults </dev/null
+'
+test_expect_success 'flux job-frobnicator exits with error on bad input' '
+	echo {} | test_expect_code 1 flux job-frobnicator --plugins=defaults
+'
+test_expect_success 'job-frobnicator: all valid jobspecs accepted' '
+	for f in ${JOBSPEC}/valid/*; do
+	    $Y2J <$f | flux job-frobnicator --jobspec-only --plugins=defaults
+	done
+'
+test_expect_success 'job-frobnicator: defaults plugin does things' '
+	flux run --env=-* --dry-run hostname \
+		| flux job-frobnicator --jobspec-only --plugins=defaults \
+		| jq  -e ".data.attributes.system.duration == 1800"
+'
+test_expect_success 'job-frobnicator: defaults plugin does not do things' '
+	flux run --env=-* --dry-run -t 1h hostname \
+		| flux job-frobnicator --jobspec-only --plugins=defaults \
+		| jq  -e ".data.attributes.system.duration == 3600"
+'
+test_expect_success 'configure queues with default durations' '
+	cat <<-EOF >conf.d/conf.toml &&
+	[policy.jobspec.defaults.system]
+	queue = "debug"
+	[queues.debug]
+	policy.jobspec.defaults.system.duration = "1h"
+	[queues.batch]
+	policy.jobspec.defaults.system.duration = "8h"
+	EOF
+	flux config reload
+'
+test_expect_success 'job-frobnicator sets default queue duration' '
+	flux run --env=-* --dry-run hostname \
+		| flux job-frobnicator --jobspec-only --plugins=defaults \
+		> queue-debug.out &&
+	jq -e ".data.attributes.system.queue == \"debug\"" < queue-debug.out &&
+	jq -e ".data.attributes.system.duration == 3600"   < queue-debug.out
+'
+test_expect_success 'job-frobnicator sets specified queue duration' '
+	flux run --env=-* --queue=batch --dry-run hostname \
+		| flux job-frobnicator --jobspec-only --plugins=defaults \
+		> queue-batch.out &&
+	jq ".data.attributes.system.queue" < queue-batch.out &&
+	jq -e ".data.attributes.system.queue == \"batch\"" < queue-batch.out &&
+	jq ".data.attributes.system.duration"  < queue-batch.out &&
+	jq -e ".data.attributes.system.duration == 28800"  < queue-batch.out
+'
+test_expect_success 'add default duration with specific queue duration' '
+	cat <<-EOF >conf.d/conf.toml &&
+	[policy.jobspec.defaults.system]
+	queue = "debug"
+	duration = "2h"
+	[queues.debug]
+	[queues.batch]
+	policy.jobspec.defaults.system.duration = "8h"
+	EOF
+	flux config reload
+'
+test_expect_success 'job-frobnicator overrides default duration with queue duration' '
+	flux run --env=-* --dry-run hostname \
+		| flux job-frobnicator --jobspec-only --plugins=defaults \
+		> default-debug.out &&
+	flux run --env=-* --queue=debug --dry-run hostname \
+		| flux job-frobnicator --jobspec-only --plugins=defaults \
+		> default-debug2.out &&
+	flux run --env=-* --queue=batch --dry-run hostname \
+		| flux job-frobnicator --jobspec-only --plugins=defaults \
+		> default-batch.out &&
+	jq .data.attributes.system.duration < default-debug.out &&
+	jq -e ".data.attributes.system.duration == 7200" < default-debug.out &&
+	jq .data.attributes.system.duration < default-debug2.out &&
+	jq -e ".data.attributes.system.duration == 7200" < default-debug2.out &&
+	jq .data.attributes.system.duration < default-batch.out &&
+	jq -e ".data.attributes.system.duration == 28800" < default-batch.out
+'
+test_expect_success 'configure queue constraints' '
+	cat <<-EOF >conf.d/conf.toml &&
+	[queues.debug]
+	requires = [ "debug" ]
+	EOF
+	flux config reload
+'
+test_expect_success 'constraints plugin sets queue constraint' '
+	flux run --env=-* --dry-run --queue=debug hostname \
+	   | flux job-frobnicator --jobspec-only --plugins=constraints \
+	   > constraint-setqueue.out &&
+	jq -e ".data.attributes.system.constraints.properties \
+	    == [ \"debug\" ]" < constraint-setqueue.out
+'
+test_expect_success 'constraints plugin adds queue constraint' '
+	flux run --env=-* --dry-run --requires=foo \
+	  --queue=debug hostname \
+	   | flux job-frobnicator --jobspec-only --plugins=constraints \
+	   > constraint-addqueue.out &&
+	jq -e ".data.attributes.system.constraints.properties|any(.==\"debug\")"\
+	   < constraint-addqueue.out &&
+	jq -e ".data.attributes.system.constraints.properties|any(.==\"foo\")"\
+	   < constraint-addqueue.out
+'
+test_expect_success 'constraints plugin works with no configured queues' '
+	cp /dev/null conf.d/conf.toml &&
+	flux config reload &&
+	flux run --env=-* --dry-run hostname \
+	   | flux job-frobnicator --jobspec-only --plugins=constraints \
+	> constraint-noqueue.out &&
+	jq -e "has(\"data\")" <constraint-noqueue.out
+'
+test_expect_success 'constraints plugin works without requires' '
+	cat >conf.d/conf.toml <<-EOT &&
+	[queues.debug]
+	EOT
+	flux config reload &&
+	flux run --env=-* --dry-run hostname \
+           --queue=debug \
+	   | flux job-frobnicator --jobspec-only --plugins=constraints \
+	> constraint-norequires.out &&
+	jq -e "has(\"data\")" <constraint-norequires.out
+'
+test_expect_success 'frobnicator defaults are defaults,constraints' '
+	cat <<-EOF >conf.d/conf.toml &&
+	[policy]
+	jobspec.defaults.system.queue = "debug"
+	[queues.debug]
+	requires = [ "debug" ]
+	EOF
+	flux config reload &&
+	flux run --env=-* --dry-run hostname \
+	   | flux job-frobnicator --jobspec-only \
+	> defaultplugins.out &&
+	jq -e ".data.attributes.system.queue == \"debug\"" \
+	    <defaultplugins.out &&
+	jq -e ".data.attributes.system.constraints.properties \
+	    == [ \"debug\" ]" \
+	    <defaultplugins.out
+'
+test_expect_success 'defaults plugin allows queues without default' '
+	cat <<-EOF >conf.d/conf.toml &&
+	[queues.debug]
+	requires = [ "debug" ]
+	EOF
+	flux config reload &&
+	flux run --env=-* --dry-run --queue=debug hostname \
+	   | flux job-frobnicator --jobspec-only --plugins=defaults \
+	    > nodefault.out &&
+	jq -e "has(\"data\")" <nodefault.out
+'
+test_expect_success 'defaults plugin requires queue if configured' '
+	cat <<-EOF >conf.d/conf.toml &&
+	[queues.debug]
+	requires = [ "debug" ]
+	EOF
+	flux config reload &&
+	flux run --env=-* --dry-run hostname \
+	   | flux job-frobnicator --jobspec-only --plugins=defaults \
+	    > nodefaultnojob.out &&
+	jq -e ".errstr == \"no queue specified\"" <nodefaultnojob.out
+'
+
+test_done
diff --git a/t/t2113-job-ingest-pipeline.t b/t/t2113-job-ingest-pipeline.t
new file mode 100755
index 000000000000..aeb9fe532933
--- /dev/null
+++ b/t/t2113-job-ingest-pipeline.t
@@ -0,0 +1,104 @@
+#!/bin/sh
+test_description='Test job ingest pipeline'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 full
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'no workers are running at the start' '
+	flux module stats job-ingest >stats.out &&
+	jq -e ".pipeline.frobnicator.running == 0" <stats.out &&
+	jq -e ".pipeline.validator.running == 0" <stats.out
+'
+test_expect_success 'run a job with no ingest configuration' '
+	flux run true
+'
+test_expect_success 'one validator, no frobnicator started' '
+	flux module stats job-ingest >stats2.out &&
+	jq -e ".pipeline.frobnicator.running == 0" <stats2.out &&
+	jq -e ".pipeline.validator.running == 1" <stats2.out
+'
+test_expect_success 'configure frobnicator' '
+	flux config load <<-EOT
+	[policy.jobspec.defaults.system]
+	duration = "10s"
+	[ingest.frobnicator]
+	plugins = [ "defaults" ]
+	EOT
+'
+test_expect_success 'run a job with unspecified duration' '
+	flux submit true >jobid1
+'
+test_expect_success 'one validator, one frobnicator started' '
+	flux module stats job-ingest >stats3.out &&
+	jq -e ".pipeline.frobnicator.running == 1" <stats3.out &&
+	jq -e ".pipeline.validator.running == 1" <stats3.out
+'
+test_expect_success 'job duration was assigned from default' '
+	flux job info $(cat jobid1) jobspec >jobspec1 &&
+	jq -e ".attributes.system.duration == 10" <jobspec1
+'
+test_expect_success 'force module config update' '
+	flux module stats job-ingest >stats4.out &&
+	jq -r ".pipeline.frobnicator.pids[0]" <stats4.out >frob.pid &&
+	jq -r ".pipeline.validator.pids[0]" <stats4.out >val.pid &&
+	flux config get | flux config load
+'
+test_expect_success 'run a job to trigger work crew with new config' '
+	flux submit true
+'
+test_expect_success 'workers were restarted' '
+	flux module stats job-ingest >stats5.out &&
+	jq -r ".pipeline.frobnicator.pids[0]" <stats5.out >frob2.pid &&
+	jq -r ".pipeline.validator.pids[0]" <stats5.out >val2.pid &&
+	test_must_fail test_cmp frob.pid frob2.pid &&
+	test_must_fail test_cmp val.pid val2.pid
+'
+test_expect_success 'run a job with novalidate flag' '
+	jq -r ".pipeline.frobnicator.requests" <stats5.out >frob.count &&
+	jq -r ".pipeline.validator.requests" <stats5.out >val.count &&
+	flux run --flags novalidate true
+'
+test_expect_success 'job was frobbed but not validated' '
+	flux module stats job-ingest >stats6.out &&
+	jq -r ".pipeline.frobnicator.requests" <stats6.out >frob2.count &&
+	jq -r ".pipeline.validator.requests" <stats6.out >val2.count &&
+	test_must_fail test_cmp frob.count frob2.count &&
+	test_cmp val.count val2.count
+'
+test_expect_success 'reconfig with null config' '
+	flux config load </dev/null
+'
+test_expect_success 'run a job with novalidate flag' '
+	flux run --flags novalidate true
+'
+test_expect_success 'job was neither frobbed nor validated' '
+	flux module stats job-ingest >stats7.out &&
+	jq -r ".pipeline.frobnicator.requests" <stats7.out >frob3.count &&
+	jq -r ".pipeline.validator.requests" <stats7.out >val3.count &&
+	test_cmp frob2.count frob3.count &&
+	test_cmp val2.count val3.count
+'
+test_expect_success 'run a job' '
+	flux run true
+'
+test_expect_success 'job was validated but not frobbed' '
+	flux module stats job-ingest >stats8.out &&
+	jq -r ".pipeline.frobnicator.requests" <stats8.out >frob4.count &&
+	jq -r ".pipeline.validator.requests" <stats8.out >val4.count &&
+	test_cmp frob3.count frob4.count &&
+	test_must_fail test_cmp val3.count val4.count
+'
+test_expect_success 'stop validator 0' '
+	valpid=$(jq -r ".pipeline.validator.pids[0]" <stats8.out) &&
+	kill -STOP $valpid
+'
+test_expect_success 'remove job-ingest to trigger cleanup' '
+	flux setattr log-stderr-level 7 &&
+	flux module remove job-ingest &&
+	flux setattr log-stderr-level 1
+'
+
+test_done
diff --git a/t/t2200-job-ingest.t b/t/t2200-job-ingest.t
deleted file mode 100755
index 7bcb4fa49a4a..000000000000
--- a/t/t2200-job-ingest.t
+++ /dev/null
@@ -1,202 +0,0 @@
-#!/bin/sh
-
-test_description='Test flux job ingest service'
-
-. $(dirname $0)/sharness.sh
-
-if test "$TEST_LONG" = "t"; then
-    test_set_prereq LONGTEST
-fi
-if ${FLUX_BUILD_DIR}/t/ingest/submitbench --help 2>&1 | grep -q sign-type; then
-    test_set_prereq HAVE_FLUX_SECURITY
-    SUBMITBENCH_OPT_R="--reuse-signature"
-fi
-
-test_under_flux 4 kvs
-
-flux setattr log-stderr-level 1
-
-JOBSPEC=${SHARNESS_TEST_SRCDIR}/jobspec
-Y2J="flux python ${JOBSPEC}/y2j.py"
-SUBMITBENCH="${FLUX_BUILD_DIR}/t/ingest/submitbench"
-RPC=${FLUX_BUILD_DIR}/t/request/rpc
-SCHEMA=${FLUX_SOURCE_DIR}/src/modules/job-ingest/schemas/jobspec.jsonschema
-BINDINGS_VALIDATOR=${FLUX_SOURCE_DIR}/src/modules/job-ingest/validators/validate-jobspec.py
-JSONSCHEMA_VALIDATOR=${FLUX_SOURCE_DIR}/src/modules/job-ingest/validators/validate-schema.py
-FAKE_VALIDATOR=${SHARNESS_TEST_SRCDIR}/ingest/fake-validate.sh
-BAD_VALIDATOR=${SHARNESS_TEST_SRCDIR}/ingest/bad-validate.sh
-
-DUMMY_EVENTLOG=test.ingest.eventlog
-
-test_valid ()
-{
-    local rc=0
-    for job in $*; do
-        cat ${job} | ${Y2J} | ${SUBMITBENCH} - || rc=1
-    done
-    return ${rc}
-}
-
-test_invalid ()
-{
-    local rc=0
-    for job in $*; do
-        cat ${job} | ${Y2J} | ${SUBMITBENCH} - && rc=1
-    done
-    return ${rc}
-}
-
-test_expect_success 'job-ingest: convert basic.yaml to json' '
-	${Y2J} <${JOBSPEC}/valid/basic.yaml >basic.json
-'
-
-test_expect_success 'job-ingest: convert use_case_2.6.yaml to json' '
-	${Y2J} <${JOBSPEC}/valid/use_case_2.6.yaml >use_case_2.6.json
-'
-
-test_expect_success 'job-ingest: submit fails without job-ingest' '
-	test_must_fail flux job submit basic.json 2>nosys.out
-'
-
-test_expect_success 'job-ingest: job-ingest fails with bad option' '
-	test_must_fail flux module load job-ingest badopt=xyz
-'
-
-test_expect_success 'job-ingest: job-ingest fails with bad validator path' '
-	test_must_fail flux module load job-ingest validator=/noexist
-'
-
-test_expect_success 'job-ingest: load job-ingest && job-info' '
-	flux exec -r all flux module load job-ingest \
-		validator=${BINDINGS_VALIDATOR} &&
-	flux exec -r all flux module load job-info
-'
-
-test_expect_success 'job-ingest: submit fails without job-manager' '
-	test_must_fail flux job submit basic.json 2>nosys.out
-'
-
-test_expect_success 'job-ingest: load job-manager-dummy module' '
-	flux module load \
-		${FLUX_BUILD_DIR}/t/ingest/.libs/job-manager-dummy.so
-'
-
-test_expect_success 'job-ingest: YAML jobspec is rejected' '
-	test_must_fail flux job submit ${JOBSPEC}/valid/basic.yaml
-'
-
-test_expect_success 'job-ingest: jobspec stored accurately in KVS' '
-	jobid=$(flux job submit basic.json) &&
-	kvsdir=$(flux job id --to=kvs $jobid) &&
-	flux kvs get --raw ${kvsdir}.jobspec >jobspec.out &&
-	test_cmp basic.json jobspec.out
-'
-
-test_expect_success 'job-ingest: job announced to job manager' '
-	jobid=$(flux job submit --priority=10 basic.json) &&
-	flux kvs eventlog get ${DUMMY_EVENTLOG} \
-		| grep "\"id\":${jobid}" >jobman.out &&
-	grep -q "\"priority\":10" jobman.out &&
-	grep -q "\"userid\":$(id -u)" jobman.out
-'
-
-test_expect_success 'job-ingest: submit event logged with userid, priority' '
-	jobid=$(flux job submit --priority=11 basic.json) &&
-	flux job eventlog $jobid |grep submit >eventlog.out &&
-	grep -q priority=11 eventlog.out &&
-	grep -q userid=$(id -u) eventlog.out
-'
-
-test_expect_success 'job-ingest: instance owner can submit priority=31' '
-	flux job submit --priority=31 basic.json
-'
-
-test_expect_success 'job-ingest: priority range is enforced' '
-	test_must_fail flux job submit --priority=32 basic.json &&
-	test_must_fail flux job submit --priority="-1" basic.json
-'
-
-test_expect_success 'job-ingest: guest cannot submit priority=17' '
-	! FLUX_HANDLE_ROLEMASK=0x2 flux job submit --priority=17 basic.json
-'
-
-test_expect_success 'job-ingest: valid jobspecs accepted' '
-	test_valid ${JOBSPEC}/valid/*
-'
-
-test_expect_success 'job-ingest: invalid jobs rejected' '
-	test_invalid ${JOBSPEC}/invalid/*
-'
-
-test_expect_success 'job-ingest: submit job 100 times' '
-	${SUBMITBENCH} -r 100 use_case_2.6.json
-'
-
-test_expect_success 'job-ingest: submit job 100 times, reuse signature' '
-	${SUBMITBENCH} ${SUBMITBENCH_OPT_R} -r 100 use_case_2.6.json
-'
-
-test_expect_success HAVE_FLUX_SECURITY 'job-ingest: submit user != signed user fails' '
-	! FLUX_HANDLE_USERID=9999 flux job submit basic.json 2>baduser.out &&
-	grep -q "signer=$(id -u) != requestor=9999" baduser.out
-'
-
-test_expect_success HAVE_FLUX_SECURITY 'job-ingest: non-owner mech=none fails' '
-	! FLUX_HANDLE_ROLEMASK=0x2 flux job submit \
-		--sign-type=none basic.json 2>badrole.out &&
-	grep -q "only instance owner" badrole.out
-'
-
-test_expect_success 'submit request with empty payload fails with EPROTO(71)' '
-	${RPC} job-ingest.submit 71 </dev/null
-'
-
-test_expect_success 'job-ingest: test validator with version 1 enforced' '
-	flux exec -r all flux module reload job-ingest \
-		validator=${BINDINGS_VALIDATOR} validator-args="--require-version,1"
-'
-
-test_expect_success 'job-ingest: v1 jobspecs accepted with v1 requirement' '
-	test_valid ${JOBSPEC}/valid_v1/*
-'
-
-test_expect_success 'job-ingest: test non-python validator' '
-	flux exec -r all flux module reload job-ingest \
-		validator=${FAKE_VALIDATOR}
-'
-
-test_expect_success 'job-ingest: submit succeeds with non-python validator' '
-    flux job submit basic.json
-'
-
-test_expect_success 'job-ingest: test python jsonschema validator' '
-	flux exec -r all flux module reload job-ingest \
-		validator=${JSONSCHEMA_VALIDATOR} validator-args=--schema,${SCHEMA}
-'
-
-test_expect_success 'job-ingest: YAML jobspec is rejected by jsonschema validator' '
-	test_must_fail flux job submit ${JOBSPEC}/valid/basic.yaml
-'
-
-test_expect_success 'job-ingest: valid jobspecs accepted by jsonschema validator' '
-	test_valid ${JOBSPEC}/valid/*
-'
-
-test_expect_success 'job-ingest: invalid jobs rejected by jsonschema validator' '
-	test_invalid ${JOBSPEC}/invalid/*
-'
-
-test_expect_success 'job-ingest: validator unexpected exit is handled' '
-	flux exec -r all flux module reload job-ingest \
-		validator=${BAD_VALIDATOR} &&
-	test_must_fail flux job submit basic.json 2>badvalidator.out &&
-	grep "unexpectedly exited" badvalidator.out
-'
-
-test_expect_success 'job-ingest: remove modules' '
-	flux module remove job-manager &&
-	flux exec -r all flux module remove job-info &&
-	flux exec -r all flux module remove job-ingest
-'
-
-test_done
diff --git a/t/t2201-job-cmd.t b/t/t2201-job-cmd.t
index 5a7c23bd32b0..7f1f2a5da934 100755
--- a/t/t2201-job-cmd.t
+++ b/t/t2201-job-cmd.t
@@ -4,31 +4,44 @@ test_description='Test flux job command'
 . $(dirname $0)/sharness.sh
 
 if flux job submit --help 2>&1 | grep -q sign-type; then
-    test_set_prereq HAVE_FLUX_SECURITY
+	test_set_prereq HAVE_FLUX_SECURITY
 fi
 
 # 2^64 - 1
 MAXJOBID_DEC=18446744073709551615
+MAXJOBID_HEX="0xffffffffffffffff"
 MAXJOBID_KVS="job.ffff.ffff.ffff.ffff"
-MAXJOBID_HEX="ffff.ffff.ffff.ffff"
+MAXJOBID_DOTHEX="ffff.ffff.ffff.ffff"
 MAXJOBID_WORDS="natural-analyze-verbal--natural-analyze-verbal"
+MAXJOBID_F58="fjpXCZedGfVQ"
+MAXJOBIDS_LIST="$MAXJOBID_DEC $MAXJOBID_HEX $MAXJOBID_KVS $MAXJOBID_DOTHEX $MAXJOBID_WORDS $MAXJOBID_F58"
 
 MINJOBID_DEC=0
+MINJOBID_HEX="0x0"
 MINJOBID_KVS="job.0000.0000.0000.0000"
-MINJOBID_HEX="0000.0000.0000.0000"
+MINJOBID_DOTHEX="0000.0000.0000.0000"
 MINJOBID_WORDS="academy-academy-academy--academy-academy-academy"
+MINJOBID_F58="f1"
+MINJOBIDS_LIST="$MINJOBID_DEC $MINJOBID_HEX $MINJOBID_KVS $MINJOBID_DOTHEX $MINJOBID_WORDS $MINJOBID_F58"
 
-test_under_flux 1 job
+test_under_flux 2 job
 
 flux setattr log-stderr-level 1
 
+# Other tests may refer to $(cat inactivejob) for inactive job id
+test_expect_success 'create one inactive job' '
+	flux submit true >inactivejob &&
+	flux queue drain
+'
+
+# After this, new jobs will remain in SCHED state
 test_expect_success 'unload job-exec,sched-simple modules' '
 	flux module remove job-exec &&
 	flux module remove sched-simple
 '
 
 test_expect_success 'flux-job: generate jobspec for simple test job' '
-        flux jobspec srun -n1 hostname >basic.json
+	flux run --dry-run -n1 hostname >basic.json
 '
 
 test_expect_success 'flux-job: submit one job to get one valid job in queue' '
@@ -60,30 +73,48 @@ test_expect_success 'flux-job: submit with nonexistent jobpsec fails' '
 '
 
 test_expect_success 'flux-job: submit with bad broker connection fails' '
-	! FLUX_URI=/wrong flux job submit basic.json
+	(FLUX_URI=/wrong test_must_fail flux job submit basic.json)
 '
 
 test_expect_success HAVE_FLUX_SECURITY 'flux-job: submit with bad security config fails' '
 	test_must_fail flux job submit \
-            --security-config=/nonexist \
-	    basic.json
+		--security-config=/nonexist \
+		basic.json
 '
 
 test_expect_success HAVE_FLUX_SECURITY 'flux-job: submit with bad sign type fails' '
 	test_must_fail flux job submit \
-	    --sign-type=notvalid \
-	    basic.json
+		--sign-type=notvalid \
+		basic.json
+'
+test_expect_success HAVE_FLUX_SECURITY 'flux-job: submit ignores security-config with --flags=signed' '
+	flux run --dry-run -n1 hostnane | \
+	    flux python \
+	    ${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py $(id -u) >signed.json &&
+	flux job submit --security-config=/nonexist --flags=signed \
+		signed.json >submit-signed.out 2>&1 &&
+	grep "Ignoring security config" submit-signed.out
+'
+test_expect_success HAVE_FLUX_SECURITY 'flux-job: submit as root fails' '
+	flux run --dry-run -n1 hostname | \
+	    flux python \
+	    ${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py 0 >signed0.json &&
+	    ( export FLUX_HANDLE_USERID=0 &&
+	      test_must_fail \
+	          flux job submit --flags=signed signed0.json 2>signed0.err \
+	    ) &&
+	test_debug "cat signed0.err" &&
+        grep "submission of jobs as user root not supported" signed0.err
 '
-
 test_expect_success 'flux-job: can submit jobspec on stdin with -' '
-        flux job submit - <basic.json
+	flux job submit - <basic.json
 '
 
 test_expect_success 'flux-job: can submit jobspec on stdin without -' '
-        flux job submit <basic.json
+	flux job submit <basic.json
 '
 
-test_expect_success 'flux-job: id without from/to args is dec to dec' '
+test_expect_success 'flux-job: id without to arg is dec to dec' '
 	jobid=$(flux job id 42) &&
 	test "$jobid" = "42"
 '
@@ -93,37 +124,25 @@ test_expect_success 'flux-job: id from stdin works' '
 	test "$jobid" = "42"
 '
 
-test_expect_success 'flux-job: id with invalid from/to arg fails' '
-	test_must_fail flux job id --from=invalid 42 &&
+test_expect_success 'flux-job: id with invalid --to arg fails' '
 	test_must_fail flux job id --to=invalid 42
 '
 
-test_expect_success 'flux-job: id --from=dec works' '
-	jobid=$(flux job id --from=dec $MAXJOBID_DEC) &&
-	test "$jobid" = "$MAXJOBID_DEC" &&
-	jobid=$(flux job id --from=dec $MINJOBID_DEC) &&
-	test "$jobid" = "$MINJOBID_DEC"
-'
-
-test_expect_success 'flux-job: id --from=words works' '
-	jobid=$(flux job id --from=words $MAXJOBID_WORDS) &&
-	test "$jobid" = "$MAXJOBID_DEC" &&
-	jobid=$(flux job id --from=words $MINJOBID_WORDS) &&
-	test "$jobid" = "$MINJOBID_DEC"
-'
-
-test_expect_success 'flux-job: id --from=kvs works' '
-	jobid=$(flux job id --from=kvs $MAXJOBID_KVS) &&
-	test "$jobid" = "$MAXJOBID_DEC" &&
-	jobid=$(flux job id --from=kvs $MINJOBID_KVS) &&
-	test "$jobid" = "$MINJOBID_DEC"
+test_expect_success 'flux-job: fails with no input and no args' '
+	test_expect_code 1 flux job id < /dev/null
 '
 
-test_expect_success 'flux-job: id --from=hex works' '
-	jobid=$(flux job id --from=hex $MAXJOBID_HEX) &&
-	test "$jobid" = "$MAXJOBID_DEC" &&
-	jobid=$(flux job id --from=hex $MINJOBID_HEX) &&
-	test "$jobid" = "$MINJOBID_DEC"
+test_expect_success 'flux-job: id works with min/max jobids' '
+	for max in $MAXJOBIDS_LIST; do
+		jobid=$(flux job id $max) &&
+		test_debug "echo flux jobid $max -> $jobid" &&
+		test "$jobid" = "$MAXJOBID_DEC"
+	done &&
+	for min in $MINJOBIDS_LIST; do
+		jobid=$(flux job id $min) &&
+		test_debug "echo flux jobid $min -> $jobid" &&
+		test "$jobid" = "$MINJOBID_DEC"
+	done
 '
 
 test_expect_success 'flux-job: id --to=dec works' '
@@ -154,42 +173,73 @@ test_expect_success 'flux-job: id --to=hex works' '
 	test "$jobid" = "$MINJOBID_HEX"
 '
 
-test_expect_success 'flux-job: id --from=kvs fails on bad input' '
-	test_must_fail flux job id --from=kvs badstring &&
-	test_must_fail flux job id --from=kvs \
-	    job.0000.0000 &&
-	test_must_fail flux job id --from=kvs \
-	    job.0000.0000.0000.000P
+test_expect_success 'flux-job: id --to=dothex works' '
+	jobid=$(flux job id --to=dothex $MAXJOBID_DEC) &&
+	test "$jobid" = "$MAXJOBID_DOTHEX" &&
+	jobid=$(flux job id --to=dothex $MINJOBID_DEC) &&
+	test "$jobid" = "$MINJOBID_DOTHEX"
+'
+
+test_expect_success 'flux-job: id --to=f58 works' '
+	jobid=$(flux job id --to=f58 $MAXJOBID_DEC) &&
+	test "$jobid" = "$MAXJOBID_F58" &&
+	jobid=$(flux job id --to=f58 $MINJOBID_DEC) &&
+	test "$jobid" = "$MINJOBID_F58"
+'
+
+UTF8_LOCALE=$(locale -a | grep UTF-8 | head -n1)
+if flux version | grep +ascii-only; then
+	UTF8_LOCALE=""
+fi
+test -n "$UTF8_LOCALE" && test_set_prereq UTF8_LOCALE
+test_expect_success UTF8_LOCALE 'flux-job: f58 can use multibyte prefix' '
+	test_debug "echo UTF8_LOCALE=${UTF8_LOCALE}" &&
+	jobid=$(LC_ALL=${UTF8_LOCALE} flux job id --to=f58 1) &&
+	test "$jobid" = "ƒ2"
+'
+
+test_expect_success 'flux-job: id fails on bad input' '
+	test_must_fail flux job id badstring &&
+	test_must_fail flux job id job.0000.0000 &&
+	test_must_fail flux job id job.0000.0000.0000.000P
+'
+
+test_expect_success 'flux-job: id fails on bad input' '
+	test_must_fail flux job id 42plusbad &&
+	test_must_fail flux job id meep &&
+	test_must_fail flux job id 18446744073709551616
 '
 
-test_expect_success 'flux-job: id --from=dec fails on bad input' '
-	test_must_fail flux job id --from=dec 42plusbad &&
-	test_must_fail flux job id --from=dec meep &&
-	test_must_fail flux job id --from=dec 18446744073709551616
+test_expect_success 'flux-job: id fails on bad words input' '
+	test_must_fail flux job id bad-words
 '
 
-test_expect_success 'flux-job: id --from=words fails on bad input' '
-	test_must_fail flux job id --from=words badwords
+test_expect_success 'flux-job: urgency fails with bad FLUX_URI' '
+	(FLUX_URI=/wrong test_must_fail flux job urgency ${validjob} 0)
 '
 
-test_expect_success 'flux-job: priority fails with bad FLUX_URI' '
-	! FLUX_URI=/wrong flux job priority ${validjob} 0
+test_expect_success 'flux-job: urgency fails with non-numeric jobid' '
+	test_must_fail flux job urgency foo 0
 '
 
-test_expect_success 'flux-job: priority fails with non-numeric jobid' '
-	test_must_fail flux job priority foo 0
+test_expect_success 'flux-job: urgency fails with wrong number of arguments' '
+	test_must_fail flux job urgency ${validjob}
 '
 
-test_expect_success 'flux-job: priority fails with wrong number of arguments' '
-	test_must_fail flux job priority ${validjob}
+test_expect_success 'flux-job: urgency fails with non-numeric urgency' '
+	test_must_fail flux job urgency ${validjob} foo
 '
 
-test_expect_success 'flux-job: priority fails with non-numeric priority' '
-	test_must_fail flux job priority ${validjob} foo
+test_expect_success 'job-manager: flux job urgency fails on invalid jobid' '
+	test_must_fail flux job urgency 12345 31
+'
+
+test_expect_success 'job-manager: flux job urgency fails on inactive jobid' '
+	test_must_fail flux job urgency $(cat inactivejob) 31
 '
 
 test_expect_success 'flux-job: raise fails with bad FLUX_URI' '
-	! FLUX_URI=/wrong flux job raise ${validjob}
+	(FLUX_URI=/wrong test_must_fail flux job raise ${validjob})
 '
 
 test_expect_success 'flux-job: raise fails with no args' '
@@ -200,18 +250,75 @@ test_expect_success 'flux-job: raise fails with invalid jobid' '
 	test_must_fail flux job raise foo
 '
 
+test_expect_success 'flux-job: raise fails with invalid jobids' '
+	test_must_fail flux job raise foo fee
+'
+
+test_expect_success 'flux-job: raise fails if note set twice' '
+	test_must_fail flux job raise --message=hi ${validjob} -- hello
+'
+
+test_expect_success 'flux-job: raise fails with inactive jobid' '
+	test_must_fail flux job raise $(cat inactivejob)
+'
+
 test_expect_success 'flux-job: raise fails with invalid option' '
 	test_must_fail flux job raise --meep foo
 '
 
+test_expect_success 'flux-job: raise basic works' '
+	id=$(flux submit sleep 100) &&
+	flux job raise ${id} &&
+	flux job wait-event -t 30 ${id} exception >raise1.out &&
+	grep "cancel" raise1.out &&
+	grep "severity\=0" raise1.out
+'
+
+test_expect_success 'flux-job: raise --type works' '
+	id=$(flux submit sleep 100) &&
+	flux job raise -t typefoo ${id} &&
+	flux job wait-event -t 30 ${id} exception >raise2.out &&
+	grep "typefoo" raise2.out
+'
+
+test_expect_success 'flux-job: raise --severity works' '
+	id=$(flux submit sleep 100) &&
+	flux job raise --severity=5 ${id} &&
+	flux job wait-event -t 30 ${id} exception >raise3.out &&
+	grep "severity\=5" raise3.out &&
+	flux job cancel ${id}
+'
+
+test_expect_success 'flux-job: raise --message works' '
+	id=$(flux submit sleep 100) &&
+	flux job raise --message=foobarmessage ${id} &&
+	flux job wait-event -t 30 ${id} exception >raise4.out &&
+	grep "foobarmessage" raise4.out
+'
+
+test_expect_success ' flux-job: raise message works (cmdline)' '
+	id=$(flux submit sleep 100) &&
+	flux job raise ${id} -- eep ork ook &&
+	flux job wait-event -t 30 ${id} exception >raise5.out &&
+	grep "eep ork ook" raise5.out
+'
+
 test_expect_success 'flux-job: cancel fails with bad FLUX_URI' '
-	! FLUX_URI=/wrong flux job cancel ${validjob}
+	(FLUX_URI=/wrong test_must_fail flux job cancel ${validjob})
 '
 
 test_expect_success 'flux-job: cancel fails with unknown job id' '
 	test_must_fail flux job cancel 0
 '
 
+test_expect_success 'flux-job: cancel fails with unknown job ids' '
+	test_must_fail flux job cancel 0 f123
+'
+
+test_expect_success 'flux-job: cancel fails if note set twice' '
+	test_must_fail flux job cancel --message=hi ${validjob} -- hello
+'
+
 test_expect_success 'flux-job: cancel fails with no args' '
 	test_must_fail flux job cancel
 '
@@ -224,8 +331,30 @@ test_expect_success 'flux-job: cancel fails with invalid option' '
 	test_must_fail flux job cancel --meep foo
 '
 
+test_expect_success 'flux-job: cancel basic works' '
+	id=$(flux submit sleep 100) &&
+	flux job cancel ${id} &&
+	flux job wait-event -t 30 ${id} exception >cancel1.out &&
+	grep "cancel" cancel1.out &&
+	grep "severity\=0" cancel1.out
+'
+
+test_expect_success 'flux-job: cancel --message works' '
+	id=$(flux submit sleep 100) &&
+	flux job cancel --message=meepmessage ${id} &&
+	flux job wait-event -t 30 ${id} exception >cancel2.out &&
+	grep "meepmessage" cancel2.out
+'
+
+test_expect_success ' flux-job: cancel message works (cmdline)' '
+	id=$(flux submit sleep 100) &&
+	flux job cancel ${id} -- foo loo moo &&
+	flux job wait-event -t 30 ${id} exception >cancel3.out &&
+	grep "foo loo moo" cancel3.out
+'
+
 test_expect_success 'flux-job: list fails with bad FLUX_URI' '
-	! FLUX_URI=/wrong flux job list
+	(FLUX_URI=/wrong test_must_fail flux job list)
 '
 
 test_expect_success 'flux-job: list fails with wrong number of arguments' '
@@ -248,12 +377,8 @@ test_expect_success 'flux-job: attach fails without jobid argument' '
 	test_must_fail flux job attach
 '
 
-test_expect_success 'flux-job: attach fails without jobid argument' '
-	test_must_fail flux job attach
-'
-
 test_expect_success 'flux-job: attach fails on invalid jobid' '
-	test_must_fail flux job attach $((${validjob}+1))
+	test_must_fail flux job attach $(($(flux job id ${validjob})+1))
 '
 
 test_expect_success 'flux-job: kill fails without jobid argument' '
@@ -261,7 +386,7 @@ test_expect_success 'flux-job: kill fails without jobid argument' '
 '
 
 test_expect_success 'flux-job: kill fails on invalid jobid' '
-	test_expect_code 1 flux job kill $((${validjob}+1))
+	test_expect_code 1 flux job kill $(($(flux job id ${validjob})+1))
 '
 
 test_expect_success 'flux-job: kill fails on non-running job' '
@@ -271,6 +396,10 @@ test_expect_success 'flux-job: kill fails on non-running job' '
 	EOF
 	test_cmp kill.expected kill.err
 '
+test_expect_success 'flux-job: kill fails on inactive job' '
+	test_expect_code 1 flux job kill $(cat inactivejob) 2>kill2.err &&
+	grep "job is inactive" kill2.err
+'
 
 test_expect_success 'flux-job: kill fails with invalid signal name' '
 	test_expect_code 1 flux job kill -s SIGFAKE ${validjob} 2>kill.err2 &&
@@ -301,8 +430,8 @@ runas() {
 }
 
 test_expect_success 'flux-job: kill fails for wrong userid' '
-        test_expect_code 1 \
-		runas 9999 flux job kill ${validjob} 2> kill.guest.err&&
+	test_expect_code 1 \
+		runas 9999 flux job kill ${validjob} 2> kill.guest.err &&
 	cat <<-EOF >kill.guest.expected &&
 	flux-job: kill ${validjob}: guests may only send signals to their own jobs
 	EOF
@@ -328,7 +457,7 @@ test_expect_success 'flux job: killall with no args works' '
 '
 
 test_expect_success 'flux-job: killall with bad broker connection fails' '
-	! FLUX_URI=/wrong flux job killall
+	(FLUX_URI=/wrong test_must_fail flux job killall)
 '
 
 test_expect_success 'flux job: killall with extra free args prints usage' '
@@ -351,7 +480,7 @@ test_expect_success 'flux job: killall fails for invalid signal name' '
 
 test_expect_success 'flux-job: killall --user all fails for guest' '
 	id=$(($(id -u)+1)) &&
-        test_must_fail runas ${id} \
+	test_must_fail runas ${id} \
 		flux job killall --user all 2> killall_all_guest.err &&
 	cat <<-EOF >killall_all_guest.exp &&
 	flux-job: killall: guests can only kill their own jobs
@@ -361,7 +490,7 @@ test_expect_success 'flux-job: killall --user all fails for guest' '
 
 test_expect_success 'flux-job: killall --user <guest_uid> works for guest' '
 	id=$(($(id -u)+1)) &&
-        runas ${id} \
+	runas ${id} \
 		flux job killall --user ${id} 2> killall_guest.err
 '
 
@@ -372,7 +501,8 @@ test_expect_success 'flux job: the queue contains active jobs' '
 
 test_expect_success 'flux job: cancelall with no args works' '
 	count=$(flux job list | wc -l) &&
-	flux job cancelall 2>cancelall.err &&
+	flux job cancelall 2>cancelall.tmp &&
+	grep -v WARNING cancelall.tmp >cancelall.err &&
 	cat <<-EOT >cancelall.exp &&
 	flux-job: Command matched ${count} jobs (-f to confirm)
 	EOT
@@ -380,17 +510,19 @@ test_expect_success 'flux job: cancelall with no args works' '
 '
 
 test_expect_success 'flux-job: cancelall with bad broker connection fails' '
-	! FLUX_URI=/wrong flux job cancelall
+	(FLUX_URI=/wrong test_must_fail flux job cancelall)
 '
 
 test_expect_success 'flux job: cancelall with reason works' '
-	flux job cancelall this is a reason 2>cancelall_reason.err &&
+	flux job cancelall this is a reason 2>cancelall_reason.tmp &&
+	grep -v WARNING cancelall_reason.tmp >cancelall_reason.err &&
 	test_cmp cancelall.exp cancelall_reason.err
 '
 
 test_expect_success 'flux job: cancelall --force works' '
 	count=$(flux job list | wc -l) &&
-	flux job cancelall --force 2>cancelall_f.err &&
+	flux job cancelall --force 2>cancelall_f.tmp &&
+	grep -v WARNING cancelall_f.tmp >cancelall_f.err &&
 	cat <<-EOT >cancelall_f.exp &&
 	flux-job: Canceled ${count} jobs (0 errors)
 	EOT
@@ -403,8 +535,9 @@ test_expect_success 'flux job: the queue is empty' '
 
 test_expect_success 'flux-job: cancelall --user all fails for guest' '
 	id=$(($(id -u)+1)) &&
-        test_must_fail runas ${id} \
-		flux job cancelall --user all 2> cancelall_all_guest.err &&
+	test_must_fail runas ${id} \
+		flux job cancelall --user all 2> cancelall_all_guest.tmp &&
+	grep -v WARNING cancelall_all_guest.tmp >cancelall_all_guest.err &&
 	cat <<-EOF >cancelall_all_guest.exp &&
 	flux-job: raiseall: guests can only raise exceptions on their own jobs
 	EOF
@@ -413,7 +546,7 @@ test_expect_success 'flux-job: cancelall --user all fails for guest' '
 
 test_expect_success 'flux-job: cancelall --user <guest uid> works for guest' '
 	id=$(($(id -u)+1)) &&
-        runas ${id} \
+	runas ${id} \
 		flux job cancelall --user ${id} 2> cancelall_guest.err
 '
 
@@ -439,7 +572,7 @@ test_expect_success 'flux job: raiseall with type works' '
 '
 
 test_expect_success 'flux-job: raiseall with bad broker connection fails' '
-	! FLUX_URI=/wrong flux job raiseall test
+	(FLUX_URI=/wrong test_must_fail flux job raiseall test)
 '
 
 test_expect_success 'flux job: raiseall with type and reason works' '
@@ -495,7 +628,7 @@ test_expect_success 'flux job: raiseall with invalid type fails' '
 
 test_expect_success 'flux-job: raiseall --user all fails for guest' '
 	id=$(($(id -u)+1)) &&
-        test_must_fail runas ${id} \
+	test_must_fail runas ${id} \
 		flux job raiseall --user all test 2> raiseall_all_guest.err &&
 	cat <<-EOT >raiseall_all_guest.exp &&
 	flux-job: raiseall: guests can only raise exceptions on their own jobs
@@ -505,7 +638,7 @@ test_expect_success 'flux-job: raiseall --user all fails for guest' '
 
 test_expect_success 'flux-job: raiseall --user <guest_uid> works for guest' '
 	id=$(($(id -u)+1)) &&
-        runas ${id} \
+	runas ${id} \
 		flux job raiseall --user ${id} test 2> raiseall_guest.err
 '
 
@@ -515,7 +648,7 @@ test_expect_success 'flux-job: raiseall --user name works' '
 '
 
 test_expect_success 'submit 3 test jobs' '
-	for i in $(seq 1 3); do flux mini submit sleep 60 >>jobs; done
+	for i in $(seq 1 3); do flux submit sleep 60 >>jobs; done
 '
 
 test_expect_success 'flux-job: raiseall returns correct count' '
@@ -535,7 +668,8 @@ test_expect_success 'flux-job: raiseall -f returns correct count' '
 '
 
 test_expect_success 'flux-job: cancelall -f returns correct count' '
-	flux job cancelall -f 2>cancelall_testf.err &&
+	flux job cancelall -f 2>cancelall_testf.tmp &&
+	grep -v WARNING cancelall_testf.tmp >cancelall_testf.err &&
 	cat <<-EOT >cancelall_testf.exp &&
 	flux-job: Canceled 3 jobs (0 errors)
 	EOT
@@ -563,16 +697,171 @@ test_expect_success 'flux-job: fatal cancel exception was raised' '
 	test ${count} -eq 3
 '
 
-test_expect_success 'flux-job: load modules for live killall test' '
-        flux module load sched-simple &&
-        flux module load job-exec
+test_expect_success 'flux-job: load modules for live kill tests' '
+	flux module load sched-simple &&
+	flux module load job-exec
+'
+
+# N.B. SIGTERM == 15
+test_expect_success 'flux-job: kill basic works' '
+	id=$(flux submit --wait-event=start sleep 100) &&
+	flux job wait-event -vt 30 -p exec $id shell.start &&
+	flux job kill ${id} &&
+	flux job wait-event -t 30 ${id} finish > kill1.out &&
+	grep status=$((15+128<<8)) kill1.out
+'
+
+# N.B. SIGUSR1 == 10
+test_expect_success 'flux-job: kill --signal works' '
+	id=$(flux submit --wait-event=start sleep 100) &&
+	flux job wait-event -vt 30 -p exec $id shell.start &&
+	flux job kill --signal=SIGUSR1 ${id} &&
+	flux job wait-event -t 30 ${id} finish > kill2.out &&
+	grep status=$((10+128<<8)) kill2.out
 '
 
 test_expect_success 'flux job: killall -f kills one job' '
-        id=$(flux mini submit sleep 600) &&
-        flux job wait-event $id start &&
-        flux job killall -f &&
+	id=$(flux submit sleep 600) &&
+	flux job wait-event -vt 30 -p exec $id shell.init &&
+	flux job killall -f &&
 	run_timeout 60 flux queue drain
 '
 
+test_expect_success 'flux job: cancel can operate on multiple jobs' '
+	ids=$(flux submit --bcc=1-3 sleep 600) &&
+	for id in ${ids}; do
+		flux job wait-event \
+			-vt 30 -p exec $id shell.init
+	done &&
+	flux job cancel ${ids} cancel multiple jobs &&
+	for id in ${ids}; do
+		flux job wait-event -t 30 ${id} exception >exception.out &&
+		grep multiple exception.out
+	done
+'
+
+test_expect_success 'flux job: raise can operate on multiple jobs' '
+	ids=$(flux submit --bcc=1-3 sleep 600) &&
+	flux job raise ${ids} raise multiple jobs &&
+	for id in ${ids}; do
+		flux job wait-event -t 30 ${id} exception >exception2.out &&
+		grep multiple exception2.out
+	done
+'
+
+# N.B. SIGTERM == 15
+test_expect_success 'flux job: kill can operate on multiple jobs' '
+	ids=$(flux submit --wait-event=start --bcc=1-3 sleep 600) &&
+	for id in ${ids}; do
+		flux job wait-event -t 30 -p exec ${id} shell.init
+	done &&
+	flux job kill ${ids} &&
+	for id in ${ids}; do
+		flux job wait-event -t 30 ${id} finish >killmulti.out &&
+		grep status=$((15+128<<8)) killmulti.out
+	done
+'
+test_expect_success 'flux job: timeleft reports error outside of a job' '
+	test_expect_code 1 flux job timeleft
+'
+
+test_expect_success 'flux job: timeleft reports large int with no time limit' '
+	flux run flux job timeleft > timeleft1 &&
+	test $(cat timeleft1) -gt 9999999
+'
+test_expect_success 'flux job: timeleft -H reports infinity with no time limit' '
+	flux run flux job timeleft -H > timeleft1H &&
+	grep infinity timeleft1H
+'
+test_expect_success 'flux job: timeleft works with time limit' '
+	flux run -t 1m flux job timeleft >timeleft2 &&
+	test_debug "cat timeleft2" &&
+	test $(cat timeleft2) -lt 60
+'
+test_expect_success 'flux job: timeleft -H works with time limit' '
+	flux run -t 1m flux job timeleft -H >timeleft2H &&
+	grep "[0-9]s$" timeleft2H
+'
+test_expect_success 'flux job: timeleft works under alloc (and job)' '
+	cat <<-EOF >test.sh &&
+	flux job timeleft > timeleft3
+	flux run flux job timeleft > timeleft4
+	EOF
+	chmod +x test.sh &&
+	flux alloc -n1 -t 5m ./test.sh &&
+	test_debug "cat timeleft3" &&
+	test $(cat timeleft3) -lt 300 &&
+	test_debug "cat timeleft4" &&
+	test $(cat timeleft4) -lt 300
+'
+test_expect_success 'flux job: timeleft works for a jobid' '
+	id=$(flux submit --wait-event=start -t 1m sleep 60) &&
+	flux job timeleft $id > timeleft5 &&
+	test_debug "cat timeleft5" &&
+	test $(cat timeleft5) -lt 60
+'
+test_expect_success 'flux job: timeleft reports 0s for expired job' '
+	id=$(flux submit --wait -t0.01s hostname || true) &&
+	flux job timeleft $id > timeleft6 &&
+	test_debug "cat timeleft6" &&
+	test $(cat timeleft6) -eq 0
+'
+test_expect_success 'flux job: timeleft returns 0 for completed job' '
+	id=$(flux submit --wait -t 5d true) &&
+	flux job timeleft $id > timeleft7 &&
+	test_debug "cat timeleft7" &&
+	test $(cat timeleft7) -eq 0
+'
+test_expect_success 'flux job: timeleft fails for pending job' '
+	flux queue stop &&
+	id=$(flux submit -t 10m true) &&
+	test_expect_code 1 flux job timeleft $id > timeleft8 2>&1 &&
+	flux queue start &&
+	grep "has not started" timeleft8
+'
+test_expect_success 'flux job: timeleft fails for invalid jobids' '
+	test_expect_code 1 flux job timeleft f1234 &&
+	test_expect_code 1 flux job timeleft x1234
+'
+test_expect_success 'flux job: hostpids fails for invalid jobid' '
+	test_expect_code 1 flux job hostpids oof
+'
+test_expect_success 'flux job: hostpids fails for unknown jobid' '
+	test_expect_code 1 flux job hostpids f1234
+'
+test_expect_success 'flux job: hostpids fails for inactive job' '
+	inactive_id=$(flux jobs -f inactive -nc 1 -o {id}) &&
+	test_expect_code 1 flux job hostpids $inactive_id
+'
+# note: sleep inf job shared by next few tests
+test_expect_success 'flux job: hostpids works for running job' '
+	id=$(flux submit -N2 -n2 --wait-event=start sleep inf) &&
+	flux job hostpids $id >hostpids1.out &&
+	test_debug "cat hostpids1.out" &&
+	test $(grep -c , hostpids1.out) -eq 1
+'
+test_expect_success 'flux job: hostpids --delimiter works' '
+	flux job hostpids --delimiter="\n" $id >hostpids2.out &&
+	test_debug "cat hostpids2.out" &&
+	test $(wc -l < hostpids2.out) -eq 2
+'
+test_expect_success 'flux job: hostpids --ranks works' '
+	test "$(flux job hostpids --ranks=1 $id)" = "$(tail -n1 hostpids2.out)"
+'
+test_expect_success 'flux job: hostpids invalid --ranks fails' '
+	test_must_fail flux job hostpids --ranks=foo $id
+'
+test_expect_success 'flux job: hostpids fails for non job owner' '
+	uid=$(($(id -u)+1)) &&
+	test_must_fail runas ${uid} \
+		flux job hostpids $id
+'
+test_expect_success 'flux job: hostpids -t, --timeout works' '
+	id2=$(flux submit --urgency=hold true) &&
+	test_must_fail flux job hostpids -t 0.1 $id2
+'
+test_expect_success 'terminate running jobs' '
+	flux cancel --all &&
+	flux queue idle
+'
 test_done
diff --git a/t/t2202-job-manager.t b/t/t2202-job-manager.t
index ada57717a3af..8277f261e405 100755
--- a/t/t2202-job-manager.t
+++ b/t/t2202-job-manager.t
@@ -2,6 +2,8 @@
 
 test_description='Test flux job manager service'
 
+. `dirname $0`/job-manager/sched-helper.sh
+
 . $(dirname $0)/sharness.sh
 
 test_under_flux 4 kvs
@@ -11,19 +13,30 @@ flux setattr log-stderr-level 1
 DRAIN_CANCEL="flux python ${FLUX_SOURCE_DIR}/t/job-manager/drain-cancel.py"
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
 LIST_JOBS=${FLUX_BUILD_DIR}/t/job-manager/list-jobs
+JOB_CONV="flux python ${FLUX_SOURCE_DIR}/t/job-manager/job-conv.py"
 
 test_expect_success 'job-manager: generate jobspec for simple test job' '
-        flux jobspec srun -n1 hostname >basic.json
+	flux run --dry-run -n1 hostname >basic.json
 '
 
 test_expect_success 'job-manager: load job-ingest, job-info, job-manager' '
-	flux exec -r all flux module load job-ingest &&
-	flux exec -r all flux module load job-info &&
-	flux module load job-manager
+	flux module load job-manager &&
+	flux module load job-ingest &&
+	flux exec -r all -x 0 flux module load job-ingest &&
+	flux module load job-info
+'
+
+test_expect_success 'job-manager: max_jobid=0 before jobs run' '
+	test $(${RPC} job-manager.getinfo | jq .max_jobid) -eq 0
 '
 
 test_expect_success 'job-manager: submit one job' '
-	flux job submit basic.json >submit1.out
+	flux job submit basic.json | flux job id >submit1.out
+'
+
+test_expect_success 'job-manager: max_jobid=last' '
+	${RPC} job-manager.getinfo | jq .max_jobid >max1.out &&
+	test_cmp submit1.out max1.out
 '
 
 test_expect_success 'job-manager: queue contains 1 job' '
@@ -32,32 +45,38 @@ test_expect_success 'job-manager: queue contains 1 job' '
 '
 
 test_expect_success 'job-manager: queue lists job with correct jobid' '
-	cut -f1 <list1.out >list1_jobid.out &&
+	$jq .id <list1.out >list1_jobid.out &&
 	test_cmp submit1.out list1_jobid.out
 '
 
 test_expect_success 'job-manager: queue lists job with state=N' '
-	echo "S" >list1_state.exp &&
-	cut -f2 <list1.out >list1_state.out &&
+	echo "SCHED" >list1_state.exp &&
+	$jq .state <list1.out | ${JOB_CONV} statetostr >list1_state.out &&
 	test_cmp list1_state.exp list1_state.out
 '
 
 test_expect_success 'job-manager: queue lists job with correct userid' '
 	id -u >list1_userid.exp &&
-	cut -f3 <list1.out >list1_userid.out &&
+	$jq .userid <list1.out >list1_userid.out &&
 	test_cmp list1_userid.exp list1_userid.out
 '
 
+test_expect_success 'job-manager: queue list job with correct urgency' '
+	echo 16 >list1_urgency.exp &&
+	$jq .urgency <list1.out >list1_urgency.out &&
+	test_cmp list1_urgency.exp list1_urgency.out
+'
+
 test_expect_success 'job-manager: queue list job with correct priority' '
 	echo 16 >list1_priority.exp &&
-	cut -f4 <list1.out >list1_priority.out &&
+	$jq .priority <list1.out >list1_priority.out &&
 	test_cmp list1_priority.exp list1_priority.out
 '
 
 test_expect_success 'job-manager: raise non-fatal exception on job' '
 	jobid=$(cat list1_jobid.out) &&
 	flux job raise --severity=1 --type=testing ${jobid} Mumble grumble &&
-        flux job wait-event --timeout=5.0 --match-context=type=testing ${jobid} exception &&
+	flux job wait-event --timeout=5.0 --match-context=type=testing ${jobid} exception &&
 	flux job eventlog $jobid \
 		| grep exception >list1_exception.out &&
 	grep -q "type=\"testing\"" list1_exception.out &&
@@ -76,8 +95,8 @@ test_expect_success 'job-manager: queue contains 1 jobs' '
 
 test_expect_success 'job-manager: cancel job' '
 	jobid=$(cat list1_jobid.out) &&
-	flux job cancel ${jobid} &&
-        flux job wait-event --timeout=5.0 --match-context=type=cancel ${jobid} exception &&
+	flux cancel ${jobid} &&
+	flux job wait-event --timeout=5.0 --match-context=type=cancel ${jobid} exception &&
 	flux job eventlog $jobid | grep exception \
 		| grep severity=0 | grep "type=\"cancel\""
 '
@@ -86,10 +105,10 @@ test_expect_success 'job-manager: queue contains 0 jobs' '
 	test $(${LIST_JOBS} | wc -l) -eq 0
 '
 
-test_expect_success 'job-manager: submit jobs with priority=min,default,max' '
-	flux job submit -p0  basic.json >submit_min.out &&
-	flux job submit      basic.json >submit_def.out &&
-	flux job submit -p31 basic.json >submit_max.out
+test_expect_success 'job-manager: submit jobs with urgency=min,default,max' '
+	flux job submit -u0  basic.json | flux job id >submit_min.out &&
+	flux job submit      basic.json | flux job id >submit_def.out &&
+	flux job submit -u31 basic.json | flux job id >submit_max.out
 '
 
 test_expect_success 'job-manager: queue contains 3 jobs' '
@@ -98,67 +117,87 @@ test_expect_success 'job-manager: queue contains 3 jobs' '
 '
 
 test_expect_success 'job-manager: queue is sorted in priority order' '
-	cat >list3_pri.exp <<-EOT &&
-	31
+	cat >list3_priority.exp <<-EOT &&
+	4294967295
 	16
 	0
 	EOT
-	cut -f4 <list3.out >list3_pri.out &&
-	test_cmp list3_pri.exp list3_pri.out
+	$jq .priority <list3.out >list3_priority.out &&
+	test_cmp list3_priority.exp list3_priority.out
 '
 
 test_expect_success 'job-manager: list-jobs --count shows highest priority jobs' '
 	cat >list3_lim2.exp <<-EOT &&
-	31
+	4294967295
 	16
 	EOT
-	${LIST_JOBS} -c 2 | cut -f4 >list3_lim2.out &&
+	${LIST_JOBS} -c 2 | $jq .priority >list3_lim2.out &&
 	test_cmp list3_lim2.exp list3_lim2.out
 '
 
+test_expect_success 'job-manager: priority listed as priority=4294967295 in KVS' '
+	jobid=$(head -n 1 list3.out | $jq .id) &&
+	flux job wait-event --timeout=5.0 ${jobid} priority &&
+	flux job eventlog $jobid | grep priority=4294967295
+'
+
 test_expect_success 'job-manager: cancel jobs' '
-	for jobid in $(cut -f1 <list3.out); do \
-		flux job cancel ${jobid}; \
-	done
+	flux cancel $($jq .id <list3.out)
 '
 
 test_expect_success 'job-manager: queue contains 0 jobs' '
-       test $(${LIST_JOBS} | wc -l) -eq 0
+	test $(${LIST_JOBS} | wc -l) -eq 0
 '
 
-test_expect_success 'job-manager: submit 10 jobs of equal priority' '
+test_expect_success 'job-manager: submit 10 jobs of equal urgency' '
 	rm -f submit10.out &&
 	for i in $(seq 1 10); do \
-	    flux job submit basic.json >>submit10.out; \
+	    flux job submit basic.json | flux job id >>submit10.out; \
 	done
 '
 
 test_expect_success 'job-manager: jobs are listed in submit order' '
 	${LIST_JOBS} >list10.out &&
-	cut -f1 <list10.out >list10_ids.out &&
+	$jq .id <list10.out >list10_ids.out &&
 	test_cmp submit10.out list10_ids.out
 '
 
-test_expect_success 'job-manager: flux job priority sets last job priority=31' '
+test_expect_success 'job-manager: flux job urgency sets last job urgency=31' '
 	lastid=$(tail -1 <list10_ids.out) &&
-	flux job priority ${lastid} 31
+	flux job urgency ${lastid} 31
 '
 
-test_expect_success 'job-manager: priority was updated in KVS' '
+test_expect_success 'job-manager: urgency was updated in KVS' '
 	jobid=$(tail -1 <list10_ids.out) &&
-        flux job wait-event --timeout=5.0 ${jobid} priority &&
+	flux job wait-event --timeout=5.0 ${jobid} urgency &&
 	flux job eventlog $jobid \
-		| cut -d" " -f2- | grep ^priority >pri.out &&
-	grep -q priority=31 pri.out
+		| cut -d" " -f2- | grep ^urgency >urgency.out &&
+	grep -q urgency=31 urgency.out
+'
+
+test_expect_success 'job-manager: priority was updated in KVS' '
+	jobid=$(tail -1 <list10_ids.out) &&
+	flux job wait-event --timeout=5.0 -c 2 ${jobid} priority &&
+	flux job eventlog $jobid | grep ^priority | tail -n 1 | priority=4294967295
 '
 
 test_expect_success 'job-manager: that job is now the first job' '
 	${LIST_JOBS} >list10_reordered.out &&
-	firstid=$(cut -f1 <list10_reordered.out | head -1) &&
+	firstid=$($jq .id <list10_reordered.out | head -1) &&
 	lastid=$(tail -1 <list10_ids.out) &&
 	test "${lastid}" -eq "${firstid}"
 '
 
+test_expect_success 'job-manager: jobs in state S w/ no scheduler' '
+	for id in $(cat submit10.out); do \
+		jmgr_check_state ${id} S; \
+	done
+'
+
+test_expect_success 'job-manager: save max_jobid' '
+	${RPC} job-manager.getinfo | jq .max_jobid >max2.exp
+'
+
 test_expect_success 'job-manager: reload the job manager' '
 	flux module reload job-manager
 '
@@ -168,67 +207,114 @@ test_expect_success 'job-manager: queue was successfully reconstructed' '
 	test_cmp list10_reordered.out list_reload.out
 '
 
+check_eventlog_restart_events() {
+	for jobid in $($jq .id <list_reload.out); do
+		if ! flux job wait-event -t 20 -c 1 ${jobid} flux-restart \
+		   || ! flux job wait-event -t 20 -c 2 ${jobid} priority
+		then
+			return 1
+		fi
+	done
+	return 0
+}
+
+test_expect_success 'job-manager: eventlog indicates restart & priority event' '
+	check_eventlog_restart_events
+'
+
+test_expect_success 'job-manager: max_jobid has not changed' '
+	${RPC} job-manager.getinfo | jq .max_jobid >max2.out &&
+	test_cmp max2.exp max2.out
+'
+
 test_expect_success 'job-manager: cancel jobs' '
-	for jobid in $(cut -f1 <list_reload.out); do \
-		flux job cancel ${jobid}; \
-	done &&
+	flux cancel $($jq .id <list_reload.out) &&
 	test $(${LIST_JOBS} | wc -l) -eq 0
 '
 
-test_expect_success 'job-manager: flux job priority fails on invalid priority' '
+test_expect_success 'job-manager: flux job urgency fails on invalid urgency' '
 	jobid=$(flux job submit basic.json) &&
-	flux job priority ${jobid} 31 &&
-	test_must_fail flux job priority ${jobid} -1 &&
-	test_must_fail flux job priority ${jobid} 32 &&
-	flux job cancel ${jobid}
-'
-
-test_expect_success 'job-manager: guest can reduce priority from default' '
+	flux job urgency ${jobid} 31 &&
+	test_must_fail flux job urgency ${jobid} -1 &&
+	test_must_fail flux job urgency ${jobid} 32 &&
+	flux cancel ${jobid}
+'
+
+test_expect_success 'job-manager: flux job urgency special args work' '
+	jobid=$(flux job submit basic.json | flux job id) &&
+	flux job urgency ${jobid} hold &&
+	${LIST_JOBS} > list_hold.out &&
+	test $(cat list_hold.out | grep ${jobid} | $jq .urgency) -eq 0 &&
+	test $(cat list_hold.out | grep ${jobid} | $jq .priority) -eq 0 &&
+	flux job urgency ${jobid} expedite &&
+	${LIST_JOBS} > list_expedite.out &&
+	test $(cat list_expedite.out | grep ${jobid} | $jq .urgency) -eq 31 &&
+	test $(cat list_expedite.out | grep ${jobid} | $jq .priority) -eq 4294967295 &&
+	flux job urgency ${jobid} default &&
+	${LIST_JOBS} > list_default.out &&
+	test $(cat list_default.out | grep ${jobid} | $jq .urgency) -eq 16 &&
+	test $(cat list_default.out | grep ${jobid} | $jq .priority) -eq 16 &&
+	flux cancel ${jobid}
+'
+
+test_expect_success 'job-manager: flux job urgency -v work' '
+	jobid=$(flux job submit basic.json | flux job id) &&
+	flux job urgency -v ${jobid} 10 2>&1 | grep "16" &&
+	flux job urgency -v ${jobid} hold 2>&1 | grep "10" &&
+	flux job urgency -v ${jobid} expedite 2>&1 | grep "held" &&
+	flux job urgency -v ${jobid} 10 2>&1 | grep "expedited" &&
+	flux cancel ${jobid}
+'
+
+test_expect_success 'job-manager: guest can reduce urgency from default' '
 	jobid=$(flux job submit  basic.json) &&
-	FLUX_HANDLE_ROLEMASK=0x2 flux job priority ${jobid} 5 &&
-	flux job cancel ${jobid}
+	FLUX_HANDLE_ROLEMASK=0x2 flux job urgency ${jobid} 5 &&
+	flux cancel ${jobid}
 '
 
 test_expect_success 'job-manager: guest can increase to default' '
-	jobid=$(flux job submit -p 0 basic.json) &&
-	FLUX_HANDLE_ROLEMASK=0x2 flux job priority ${jobid} 16 &&
-	flux job cancel ${jobid}
+	jobid=$(flux job submit -u 0 basic.json) &&
+	FLUX_HANDLE_ROLEMASK=0x2 flux job urgency ${jobid} 16 &&
+	flux cancel ${jobid}
 '
 
 test_expect_success 'job-manager: guest cannot increase past default' '
 	jobid=$(flux job submit basic.json) &&
-	! FLUX_HANDLE_ROLEMASK=0x2 flux job priority ${jobid} 17 &&
-	flux job cancel ${jobid}
+	! FLUX_HANDLE_ROLEMASK=0x2 flux job urgency ${jobid} 17 &&
+	flux cancel ${jobid}
 '
 
 test_expect_success 'job-manager: guest can decrease from from >default' '
-	jobid=$(flux job submit -p 31 basic.json) &&
-	FLUX_HANDLE_ROLEMASK=0x2 flux job priority ${jobid} 17 &&
-	flux job cancel ${jobid}
+	jobid=$(flux job submit -u 31 basic.json) &&
+	FLUX_HANDLE_ROLEMASK=0x2 flux job urgency ${jobid} 17 &&
+	flux cancel ${jobid}
 '
 
-test_expect_success 'job-manager: guest cannot set priority of others jobs' '
+test_expect_success 'job-manager: guest cannot set urgency of others jobs' '
 	jobid=$(flux job submit basic.json) &&
 	newid=$(($(id -u)+1)) &&
 	! FLUX_HANDLE_ROLEMASK=0x2 FLUX_HANDLE_USERID=${newid} \
-		flux job priority ${jobid} 0 &&
-	flux job cancel ${jobid}
+		flux job urgency ${jobid} 0 &&
+	flux cancel ${jobid}
 '
 
 test_expect_success 'job-manager: guest cannot cancel others jobs' '
 	jobid=$(flux job submit basic.json) &&
 	newid=$(($(id -u)+1)) &&
 	! FLUX_HANDLE_ROLEMASK=0x2 FLUX_HANDLE_USERID=${newid} \
-		flux job cancel ${jobid} &&
-	flux job cancel ${jobid}
+		flux cancel ${jobid} &&
+	flux cancel ${jobid}
 '
 
 test_expect_success 'job-manager: no jobs in the queue' '
 	test $(${LIST_JOBS} | wc -l) -eq 0
 '
 
+# job-info module depends on job-manager, unload and reload it too
 test_expect_success 'job-manager: reload the job manager' '
-	flux module reload job-manager
+	flux module remove job-info &&
+	flux module reload job-manager &&
+	flux module load job-info
 '
 
 test_expect_success 'job-manager: still no jobs in the queue' '
@@ -263,7 +349,7 @@ test_expect_success 'job-manager: there is still one job in the queue' '
 '
 
 test_expect_success 'job-manager: drain unblocks when last job is canceled' '
-	jobid=$(cut -f1 <list.out) &&
+	jobid=$($jq .id <list.out) &&
 	run_timeout 5 ${DRAIN_CANCEL} ${jobid}
 '
 
@@ -273,8 +359,8 @@ test_expect_success 'list request with empty payload fails with EPROTO(71)' '
 test_expect_success 'raise request with empty payload fails with EPROTO(71)' '
 	${RPC} job-manager.raise 71 </dev/null
 '
-test_expect_success 'priority request with empty payload fails with EPROTO(71)' '
-	${RPC} job-manager.priority 71 </dev/null
+test_expect_success 'urgency request with empty payload fails with EPROTO(71)' '
+	${RPC} job-manager.urgency 71 </dev/null
 '
 test_expect_success 'sched-ready request with empty payload fails with EPROTO(71)' '
 	${RPC} job-manager.sched-ready 71 </dev/null
@@ -286,9 +372,19 @@ test_expect_success 'submit request with empty payload fails with EPROTO(71)' '
 	${RPC} job-manager.submit 71 </dev/null
 '
 
-test_expect_success 'job-manager: remove job-manager, job-info, job-ingest' '
+test_expect_success 'job-manager stats works' '
+	flux module stats job-manager > stats.out &&
+	cat stats.out | $jq -e .journal.listeners
+'
+
+test_expect_success 'flux module stats job-manager is open to guests' '
+	FLUX_HANDLE_ROLEMASK=0x2 \
+	    flux module stats job-manager >/dev/null
+'
+
+test_expect_success 'job-manager: remove job-info, job-manager, job-ingest' '
+	flux module remove job-info &&
 	flux module remove job-manager &&
-	flux exec -r all flux module remove job-info &&
 	flux exec -r all flux module remove job-ingest
 '
 
diff --git a/t/t2203-job-manager-dummysched.t b/t/t2203-job-manager-dummysched.t
deleted file mode 100755
index 27aa46f79fb8..000000000000
--- a/t/t2203-job-manager-dummysched.t
+++ /dev/null
@@ -1,168 +0,0 @@
-#!/bin/sh
-
-test_description='Test flux job manager service with dummy scheduler'
-
-. $(dirname $0)/sharness.sh
-
-test_under_flux 4 kvs
-
-SCHED_DUMMY=${FLUX_BUILD_DIR}/t/job-manager/.libs/sched-dummy.so
-LIST_JOBS=${FLUX_BUILD_DIR}/t/job-manager/list-jobs
-
-flux setattr log-stderr-level 1
-
-get_state() {
-	local id=$1
-	local state=$(${LIST_JOBS} | awk '$1 == "'${id}'" { print $2; }')
-	test -z "$state" \
-		&& flux job wait-event --timeout=5 ${id} clean >/dev/null \
-		&& state=I
-	echo $state
-}
-check_state() {
-	local id=$1
-	local wantstate=$2
-	for try in $(seq 1 10); do
-		test $(get_state $id) = $wantstate && return 0
-	done
-	return 1
-}
-
-test_expect_success 'flux-job: generate jobspec for simple test job' '
-        flux jobspec srun -n1 hostname >basic.json
-'
-
-test_expect_success 'job-manager: load job-ingest, job-manager' '
-	flux exec -r all flux module load job-ingest &&
-	flux exec -r all flux module load job-info &&
-	flux module load job-manager
-'
-
-test_expect_success 'job-manager: submit 5 jobs' '
-	flux job submit --flags=debug basic.json >job1.id &&
-	flux job submit --flags=debug basic.json >job2.id &&
-	flux job submit --flags=debug basic.json >job3.id &&
-	flux job submit --flags=debug basic.json >job4.id &&
-	flux job submit --flags=debug basic.json >job5.id
-'
-
-test_expect_success 'job-manager: job state SSSSS (no scheduler)' '
-	check_state $(cat job1.id) S &&
-	check_state $(cat job2.id) S &&
-	check_state $(cat job3.id) S &&
-	check_state $(cat job4.id) S &&
-	check_state $(cat job5.id) S
-
-'
-
-test_expect_success 'job-manager: load sched-dummy --cores=2' '
-	flux module load ${SCHED_DUMMY} --cores=2
-'
-
-test_expect_success 'job-manager: job state RRSSS' '
-	check_state $(cat job1.id) R &&
-	check_state $(cat job2.id) R &&
-	check_state $(cat job3.id) S &&
-	check_state $(cat job4.id) S &&
-	check_state $(cat job5.id) S
-'
-
-test_expect_success 'job-manager: running job has alloc event' '
-	flux job wait-event --timeout=5.0 $(cat job1.id) alloc
-'
-
-test_expect_success 'job-manager: cancel 2' '
-	flux job cancel $(cat job2.id)
-'
-
-test_expect_success 'job-manager: job state RIRSS' '
-	check_state $(cat job1.id) R &&
-	check_state $(cat job2.id) I &&
-	check_state $(cat job3.id) R &&
-	check_state $(cat job4.id) S &&
-	check_state $(cat job5.id) S
-'
-
-test_expect_success 'job-manager: first S job sent alloc, second S did not' '
-	flux job wait-event --timeout=5.0 $(cat job4.id) debug.alloc-request &&
-	! flux job wait-event --timeout=0.1 $(cat job5.id) debug.alloc-request
-'
-
-test_expect_success 'job-manager: canceled job has exception, free events' '
-	flux job wait-event --timeout=5.0 $(cat job2.id) exception &&
-	flux job wait-event --timeout=5.0 $(cat job2.id) free
-'
-
-test_expect_success 'job-manager: reload sched-dummy --cores=4' '
-	flux dmesg -C &&
-	flux module reload ${SCHED_DUMMY} --cores=4 &&
-	flux dmesg | grep "hello_cb:" >hello.dmesg
-'
-
-test_expect_success 'job-manager: hello handshake found jobs 1 3' '
-	grep id=$(cat job1.id) hello.dmesg &&
-	grep id=$(cat job3.id) hello.dmesg
-'
-
-test_expect_success 'job-manager: hello handshake priority is default' '
-	grep priority=16 hello.dmesg
-'
-
-test_expect_success 'job-manager: hello handshake userid is expected' '
-	grep userid=$(id -u) hello.dmesg
-'
-
-test_expect_success 'job-manager: job state RIRRR' '
-	check_state $(cat job1.id) R &&
-	check_state $(cat job2.id) I &&
-	check_state $(cat job3.id) R &&
-	check_state $(cat job4.id) R &&
-	check_state $(cat job5.id) R
-'
-
-test_expect_success 'job-manager: cancel 1' '
-	flux job cancel $(cat job1.id)
-'
-
-test_expect_success 'job-manager: job state IIRRR' '
-	check_state $(cat job1.id) I &&
-	check_state $(cat job2.id) I &&
-	check_state $(cat job3.id) R &&
-	check_state $(cat job4.id) R &&
-	check_state $(cat job5.id) R
-'
-
-test_expect_success 'job-manager: cancel all jobs' '
-	flux job cancel $(cat job3.id) &&
-	flux job cancel $(cat job4.id) &&
-	flux job cancel $(cat job5.id)
-'
-
-test_expect_success 'job-manager: job state IIIII' '
-	check_state $(cat job1.id) I &&
-	check_state $(cat job2.id) I &&
-	check_state $(cat job3.id) I &&
-	check_state $(cat job4.id) I &&
-	check_state $(cat job5.id) I
-'
-
-test_expect_success 'job-manager: simulate alloc failure' '
-	flux module debug --setbit 0x1 sched-dummy &&
-	flux job submit --flags=debug basic.json >job6.id &&
-	flux job wait-event --timeout=5 $(cat job6.id) exception >ev6.out &&
-	grep -q "type=\"alloc\"" ev6.out &&
-	grep -q severity=0 ev6.out &&
-	grep -q DEBUG_FAIL_ALLOC ev6.out
-'
-
-test_expect_success 'job-manager: remove sched-dummy' '
-	flux module remove sched-dummy
-'
-
-test_expect_success 'job-manager: remove job-manager, job-ingest' '
-	flux module remove job-manager &&
-	flux exec -r all flux module remove job-info &&
-	flux exec -r all flux module remove job-ingest
-'
-
-test_done
diff --git a/t/t2203-job-manager-single.t b/t/t2203-job-manager-single.t
new file mode 100755
index 000000000000..3916b9895a5c
--- /dev/null
+++ b/t/t2203-job-manager-single.t
@@ -0,0 +1,237 @@
+#!/bin/sh
+
+test_description='Test flux job manager service with sched-simple (single)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+test_under_flux 2 job
+
+flux setattr log-stderr-level 1
+
+# N.B. we will fake with different resources later on in this file, thus
+# the need to set resources manually instead of through test_under_flux()
+#
+# --setbit 0x2 enables creation of reason_pending field
+test_expect_success 'job-manager: load sched-simple w/ 1 rank, 2 cores/rank' '
+        flux module unload sched-simple &&
+        flux R encode -r0 -c0-1 >R.test &&
+        flux resource reload R.test &&
+        flux module load sched-simple mode=limited=1 &&
+        flux module debug --setbit 0x2 sched-simple
+'
+
+test_expect_success 'job-manager: submit 5 jobs' '
+        flux submit --log=job{cc}.id --cc="1-5" --flags=debug -n1 \
+           hostname
+'
+
+test_expect_success 'job-manager: job state RRSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs (RRSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: annotate jobs in job-list (RRSSS)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_no_annotations $(cat job4.id) &&
+        jlist_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: annotate jobs in flux-jobs (RRSSS)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_no_annotations $(cat job4.id) &&
+        fjobs_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: running job has alloc event' '
+        flux job wait-event --timeout=5.0 $(cat job1.id) alloc
+'
+
+test_expect_success 'job-manager: cancel 2' '
+        flux cancel $(cat job2.id)
+'
+
+test_expect_success 'job-manager: job state RIRSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs (RIRSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in job-list (RIRSS)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in flux jobs (RIRSS)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: first S job sent alloc, second S did not' '
+        flux job wait-event --timeout=5.0 $(cat job4.id) debug.alloc-request &&
+        ! flux job wait-event --timeout=0.1 $(cat job5.id) debug.alloc-request
+'
+
+test_expect_success 'job-manager: canceled job has exception, free events' '
+        flux job wait-event --timeout=5.0 $(cat job2.id) exception &&
+        flux job wait-event --timeout=5.0 $(cat job2.id) free
+'
+
+test_expect_success 'job-manager: reload sched-simple w/ 2 ranks, 2 cores/rank' '
+        flux dmesg -C &&
+        flux module unload sched-simple &&
+        flux R encode -r0-1 -c0-1 >R2.test &&
+        flux resource reload R2.test &&
+        flux module load sched-simple mode=limited=1 &&
+        flux module debug --setbit 0x2 sched-simple &&
+        flux dmesg | grep "hello:" >hello.dmesg
+'
+
+test_expect_success 'job-manager: hello handshake found jobs 1 3' '
+        grep id=$(flux job id --to=f58 < job1.id) hello.dmesg &&
+        grep id=$(flux job id --to=f58 < job3.id) hello.dmesg
+'
+
+test_expect_success 'job-manager: hello handshake priority is default urgency' '
+        grep priority=16 hello.dmesg
+'
+
+test_expect_success 'job-manager: hello handshake userid is expected' '
+        grep userid=$(id -u) hello.dmesg
+'
+
+test_expect_success 'job-manager: job state RIRRR' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: annotate jobs (RIRRR)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank1/core0\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank1/core1\""
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in job-list (RIRRR)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank1/core0\"" &&
+        jlist_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank1/core1\""
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in flux jobs (RIRRR)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.resource_summary" "rank1/core0" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.resource_summary" "rank1/core1"
+'
+
+test_expect_success 'job-manager: cancel 1' '
+        flux cancel $(cat job1.id)
+'
+
+test_expect_success 'job-manager: job state IIRRR' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel $(cat job3.id) &&
+        flux cancel $(cat job4.id) &&
+        flux cancel $(cat job5.id)
+'
+
+test_expect_success 'job-manager: job state IIIII' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) I &&
+        jmgr_check_state $(cat job4.id) I &&
+        jmgr_check_state $(cat job5.id) I
+'
+
+test_expect_success 'job-manager: no annotations (IIIII)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, annotations are cached
+test_expect_success 'job-manager: annotate jobs in job-list (IIIII)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank1/core0\"" &&
+        jlist_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank1/core1\""
+'
+
+# compared to above, annotations are cached
+test_expect_success 'job-manager: annotate jobs in flux jobs (IIIII)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.resource_summary" "rank1/core0" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.resource_summary" "rank1/core1"
+'
+
+test_expect_success 'job-manager: simulate alloc failure' '
+        flux module debug --setbit 0x1 sched-simple &&
+        flux submit --flags=debug -n1 hostname >job6.id &&
+        flux job wait-event --timeout=5 $(cat job6.id) exception >ev6.out &&
+        grep -q "type=\"alloc\"" ev6.out &&
+        grep -q severity=0 ev6.out &&
+        grep -q DEBUG_FAIL_ALLOC ev6.out
+'
+
+test_done
diff --git a/t/t2204-job-info.t b/t/t2204-job-info.t
deleted file mode 100755
index 98377819682c..000000000000
--- a/t/t2204-job-info.t
+++ /dev/null
@@ -1,1241 +0,0 @@
-#!/bin/sh
-
-test_description='Test flux job info service'
-
-. `dirname $0`/kvs/kvs-helper.sh
-
-. $(dirname $0)/sharness.sh
-
-test_under_flux 4 job
-
-jq=$(which jq 2>/dev/null)
-test -n "$jq" && test_set_prereq HAVE_JQ
-RPC=${FLUX_BUILD_DIR}/t/request/rpc
-
-if test "$TEST_LONG" = "t"; then
-    test_set_prereq LONGTEST
-fi
-
-# Usage: submit_job
-# To ensure robustness of tests despite future job manager changes,
-# cancel the job, and wait for clean event.
-submit_job() {
-        local jobid=$(flux job submit sleeplong.json) &&
-        flux job wait-event $jobid start >/dev/null &&
-        flux job cancel $jobid &&
-        flux job wait-event $jobid clean >/dev/null &&
-        echo $jobid
-}
-
-# Unlike above, do not cancel the job, the test will cancel the job
-submit_job_live() {
-        local jobspec=$1
-        local jobid=$(flux job submit $jobspec) &&
-        flux job wait-event $jobid start >/dev/null &&
-        echo $jobid
-}
-
-# Test will cancel the job, is assumed won't run immediately
-submit_job_wait() {
-        local jobid=$(flux job submit sleeplong.json) &&
-        flux job wait-event $jobid depend >/dev/null &&
-        echo $jobid
-}
-
-wait_watchers_nonzero() {
-        local str=$1
-        local i=0
-        while (! flux module stats --parse $str job-info > /dev/null 2>&1 \
-               || [ "$(flux module stats --parse $str job-info 2> /dev/null)" = "0" ]) \
-              && [ $i -lt 50 ]
-        do
-                sleep 0.1
-                i=$((i + 1))
-        done
-        if [ "$i" -eq "50" ]
-        then
-            return 1
-        fi
-        return 0
-}
-
-get_timestamp_field() {
-        local field=$1
-        local file=$2
-        grep $field $file | awk '{print $1}'
-}
-
-test_expect_success 'job-info: generate jobspec for simple test job' '
-        flux jobspec --format json srun -N1 hostname > hostname.json &&
-        flux jobspec --format json srun -N1 sleep 300 > sleeplong.json
-'
-
-#
-# job list tests
-#
-# these tests come first, as we do not want job submissions below to
-# interfere with expected results
-#
-
-# submit a whole bunch of jobs for job list testing
-#
-# - the first loop of job submissions are intended to have some jobs run
-#   quickly and complete
-# - the second loop of job submissions are intended to eat up all resources
-# - the last job submissions are intended to get a create a set of
-#   pending jobs, because jobs from the second loop have taken all resources
-#   - we desire pending jobs sorted in priority order, so we need to
-#   create the sorted list for comparison later.
-# - job ids are stored in files in the order we expect them to be listed
-#   - pending jobs - by priority (highest first), submission time
-#                    (earlier first)
-#   - running jobs - by start time (most recent first)
-#   - inactive jobs - by completion time (most recent first)
-#
-# TODO
-# - alternate userid job listing
-#
-# the job-info module has eventual consistency with the jobs stored in
-# the job-manager's queue.  To ensure no raciness in tests, we spin
-# until all of the pending jobs have reached SCHED state, running jobs
-# have reached RUN state, and inactive jobs have reached INACTIVE
-# state.
-
-wait_states() {
-        local i=0
-        while ( [ "$(flux job list --states=sched | wc -l)" != "8" ] \
-                || [ "$(flux job list --states=run | wc -l)" != "8" ] \
-                || [ "$(flux job list --states=inactive | wc -l)" != "4" ]) \
-               && [ $i -lt 50 ]
-        do
-                sleep 0.1
-                i=$((i + 1))
-        done
-        if [ "$i" -eq "50" ]
-        then
-            return 1
-        fi
-        return 0
-}
-
-test_expect_success 'submit jobs for job list testing' '
-        for i in `seq 1 4`; do \
-            flux job submit hostname.json >> job_ids1.out; \
-            flux job wait-event `tail -n 1 job_ids1.out` clean ; \
-        done &&
-        tac job_ids1.out > job_ids_inactive.out &&
-        for i in `seq 1 8`; do \
-            flux job submit sleeplong.json >> job_ids2.out; \
-            flux job wait-event `tail -n 1 job_ids2.out` start; \
-        done &&
-        tac job_ids2.out > job_ids_running.out &&
-        id1=$(flux job submit -p20 hostname.json) &&
-        id2=$(flux job submit      hostname.json) &&
-        id3=$(flux job submit -p31 hostname.json) &&
-        id4=$(flux job submit -p0  hostname.json) &&
-        id5=$(flux job submit -p20 hostname.json) &&
-        id6=$(flux job submit      hostname.json) &&
-        id7=$(flux job submit -p31 hostname.json) &&
-        id8=$(flux job submit -p0  hostname.json) &&
-        echo $id3 > job_ids_pending.out &&
-        echo $id7 >> job_ids_pending.out &&
-        echo $id1 >> job_ids_pending.out &&
-        echo $id5 >> job_ids_pending.out &&
-        echo $id2 >> job_ids_pending.out &&
-        echo $id6 >> job_ids_pending.out &&
-        echo $id4 >> job_ids_pending.out &&
-        echo $id8 >> job_ids_pending.out &&
-        cat job_ids_pending.out > active.out &&
-        cat job_ids_running.out >> active.out &&
-        wait_states
-'
-
-# Note: "running" = "run" & "cleanup", we also test just "run" state
-# since we happen to know all these jobs are in the "run" state given
-# checks above
-
-test_expect_success HAVE_JQ 'flux job list running jobs in started order' '
-        flux job list -s running | jq .id > list_started1.out &&
-        flux job list -s run,cleanup | jq .id > list_started2.out &&
-        flux job list -s run | jq .id > list_started3.out &&
-        test_cmp list_started1.out job_ids_running.out &&
-        test_cmp list_started2.out job_ids_running.out &&
-        test_cmp list_started3.out job_ids_running.out
-'
-
-test_expect_success HAVE_JQ 'flux job list running jobs with correct state' '
-        for count in `seq 1 8`; do \
-            echo "8" >> list_state_R.exp; \
-        done &&
-        flux job list -s running | jq .state > list_state_R1.out &&
-        flux job list -s run,cleanup | jq .state > list_state_R2.out &&
-        flux job list -s run | jq .state > list_state_R3.out &&
-        test_cmp list_state_R1.out list_state_R.exp &&
-        test_cmp list_state_R2.out list_state_R.exp &&
-        test_cmp list_state_R3.out list_state_R.exp
-'
-
-test_expect_success HAVE_JQ 'flux job list no jobs in cleanup state' '
-        count=$(flux job list -s cleanup | wc -l) &&
-        test $count -eq 0
-'
-
-test_expect_success HAVE_JQ 'flux job list inactive jobs in completed order' '
-        flux job list -s inactive | jq .id > list_inactive.out &&
-        test_cmp list_inactive.out job_ids_inactive.out
-'
-
-test_expect_success HAVE_JQ 'flux job list inactive jobs with correct state' '
-        for count in `seq 1 4`; do \
-            echo "32" >> list_state_I.exp; \
-        done &&
-        flux job list -s inactive | jq .state > list_state_I.out &&
-        test_cmp list_state_I.out list_state_I.exp
-'
-
-# Note: "pending" = "depend" & "sched", we also test just "sched"
-# state since we happen to know all these jobs are in the "sched"
-# state given checks above
-
-test_expect_success HAVE_JQ 'flux job list pending jobs in priority order' '
-        flux job list -s pending | jq .id > list_pending1.out &&
-        flux job list -s depend,sched | jq .id > list_pending2.out &&
-        flux job list -s sched | jq .id > list_pending3.out &&
-        test_cmp list_pending1.out job_ids_pending.out &&
-        test_cmp list_pending2.out job_ids_pending.out &&
-        test_cmp list_pending3.out job_ids_pending.out
-'
-
-test_expect_success HAVE_JQ 'flux job list pending jobs with correct priority' '
-        cat >list_priority.exp <<-EOT &&
-31
-31
-20
-20
-16
-16
-0
-0
-EOT
-        flux job list -s pending | jq .priority > list_priority1.out &&
-        flux job list -s depend,sched | jq .priority > list_priority2.out &&
-        flux job list -s sched | jq .priority > list_priority3.out &&
-        test_cmp list_priority1.out list_priority.exp &&
-        test_cmp list_priority2.out list_priority.exp &&
-        test_cmp list_priority3.out list_priority.exp
-'
-
-test_expect_success HAVE_JQ 'flux job list pending jobs with correct state' '
-        for count in `seq 1 8`; do \
-            echo "4" >> list_state_S.exp; \
-        done &&
-        flux job list -s pending | jq .state > list_state_S1.out &&
-        flux job list -s depend,sched | jq .state > list_state_S2.out &&
-        flux job list -s sched | jq .state > list_state_S3.out &&
-        test_cmp list_state_S3.out list_state_S.exp
-'
-
-test_expect_success HAVE_JQ 'flux job list no jobs in depend state' '
-        count=$(flux job list -s depend | wc -l) &&
-        test $count -eq 0
-'
-
-# Note: "active" = "pending" & "running", i.e. depend, sched, run, cleanup
-test_expect_success HAVE_JQ 'flux job list active jobs in correct order' '
-        flux job list -s active | jq .id > list_active1.out &&
-        flux job list -s depend,sched,run,cleanup | jq .id > list_active2.out &&
-        flux job list -s sched,run | jq .id > list_active3.out &&
-        test_cmp list_active1.out active.out &&
-        test_cmp list_active2.out active.out &&
-        test_cmp list_active3.out active.out
-'
-
-test_expect_success HAVE_JQ 'flux job list jobs with correct userid' '
-        for count in `seq 1 20`; do \
-            id -u >> list_userid.exp; \
-        done &&
-        flux job list -a | jq .userid > list_userid.out &&
-        test_cmp list_userid.out list_userid.exp
-'
-
-test_expect_success HAVE_JQ 'flux job list defaults to listing pending & running jobs' '
-        flux job list > list_default.out &&
-        count=$(wc -l < list_default.out) &&
-        test "$count" = "16" &&
-        tail -n 8 list_default.out | jq .id  > list_default_running.out &&
-        test_cmp list_default_running.out job_ids_running.out &&
-        head -n 8 list_default.out | jq .id > list_default_pending.out &&
-        test_cmp list_default_pending.out job_ids_pending.out
-'
-
-test_expect_success 'flux job list --user=userid works' '
-        uid=$(id -u) &&
-        flux job list --user=$uid> list_userid.out &&
-        count=$(wc -l < list_userid.out) &&
-        test "$count" = "16"
-'
-
-test_expect_success 'flux job list --user=all works' '
-        flux job list --user=all > list_all.out &&
-        count=$(wc -l < list_all.out) &&
-        test "$count" = "16"
-'
-
-test_expect_success HAVE_JQ 'flux job list --count works' '
-        flux job list -s active,inactive --count=12 > list_count.out &&
-        count=$(wc -l < list_count.out) &&
-        test "$count" = "12" &&
-        head -n 8 list_count.out | jq .id > list_count_pending.out &&
-        test_cmp list_count_pending.out job_ids_pending.out &&
-        tail -n 4 list_count.out | jq .id > list_count_running.out &&
-        head -n 4 job_ids_running.out > job_ids_running_head4.out &&
-        test_cmp list_count_running.out job_ids_running_head4.out
-'
-
-test_expect_success HAVE_JQ 'flux job list all jobs works' '
-        flux job list -a > list_all.out &&
-        cat list_all.out | jq .id > list_all_jobids.out &&
-        cat job_ids_pending.out >> list_all_jobids.exp &&
-        cat job_ids_running.out >> list_all_jobids.exp &&
-        cat job_ids_inactive.out >> list_all_jobids.exp &&
-        test_cmp list_all_jobids.exp list_all_jobids.out
-'
-
-test_expect_success HAVE_JQ 'job stats lists jobs in correct state (mix)' '
-        flux job stats | jq -e ".job_states.depend == 0" &&
-        flux job stats | jq -e ".job_states.sched == 8" &&
-        flux job stats | jq -e ".job_states.run == 8" &&
-        flux job stats | jq -e ".job_states.cleanup == 0" &&
-        flux job stats | jq -e ".job_states.inactive == 4" &&
-        flux job stats | jq -e ".job_states.total == 20"
-'
-
-test_expect_success 'cleanup job listing jobs ' '
-        for jobid in `cat job_ids_pending.out`; do \
-            flux job cancel $jobid; \
-            flux job wait-event $jobid clean; \
-        done &&
-        for jobid in `cat job_ids_running.out`; do \
-            flux job cancel $jobid; \
-            flux job wait-event $jobid clean; \
-        done
-'
-
-wait_inactive() {
-        local i=0
-        while [ "$(flux job list --states=inactive | wc -l)" != "20" ] \
-               && [ $i -lt 50 ]
-        do
-                sleep 0.1
-                i=$((i + 1))
-        done
-        if [ "$i" -eq "50" ]
-        then
-            return 1
-        fi
-        return 0
-}
-
-test_expect_success 'reload the job-info module' '
-        flux exec -r all flux module reload job-info &&
-        wait_inactive
-'
-
-# inactive jobs are listed by most recently completed first, so must
-# construct order based on order of jobs canceled above
-test_expect_success HAVE_JQ 'job-info: list successfully reconstructed' '
-        flux job list -a > list_reload.out &&
-        for count in `seq 1 20`; do \
-            echo "32" >> list_reload_state.exp; \
-        done &&
-        cat list_reload.out | jq .state  > list_reload_state.out &&
-        test_cmp list_reload_state.exp list_reload_state.out &&
-        tac job_ids_running.out >> list_reload_ids.exp &&
-        tac job_ids_pending.out >> list_reload_ids.exp &&
-        cat job_ids_inactive.out >> list_reload_ids.exp &&
-        cat list_reload.out | jq .id > list_reload_ids.out &&
-        test_cmp list_reload_ids.exp list_reload_ids.out
-'
-
-test_expect_success HAVE_JQ 'job stats lists jobs in correct state (all inactive)' '
-        flux job stats | jq -e ".job_states.depend == 0" &&
-        flux job stats | jq -e ".job_states.sched == 0" &&
-        flux job stats | jq -e ".job_states.run == 0" &&
-        flux job stats | jq -e ".job_states.cleanup == 0" &&
-        flux job stats | jq -e ".job_states.inactive == 20" &&
-        flux job stats | jq -e ".job_states.total == 20"
-'
-
-# job list-inactive
-
-test_expect_success HAVE_JQ 'flux job list-inactive lists all inactive jobs' '
-        flux job list-inactive > list-inactive.out &&
-        count=`cat list-inactive.out | wc -l` &&
-        test $count -eq 20
-'
-
-test_expect_success HAVE_JQ 'flux job list-inactive w/ since 0 lists all inactive jobs' '
-        count=`flux job list-inactive --since=0 | wc -l` &&
-        test $count -eq 20
-'
-
-test_expect_success HAVE_JQ 'flux job list-inactive w/ count limits output of inactive jobs' '
-        count=`flux job list-inactive --count=14 | wc -l` &&
-        test $count -eq 14
-'
-
-test_expect_success HAVE_JQ 'flux job list-inactive w/ since -1 leads to error' '
-        test_must_fail flux job list-inactive --since=-1
-'
-
-test_expect_success HAVE_JQ 'flux job list-inactive w/ count -1 leads to error' '
-        test_must_fail flux job list-inactive --count=-1
-'
-
-test_expect_success HAVE_JQ 'flux job list-inactive w/ since (most recent timestamp)' '
-        timestamp=`cat list-inactive.out | head -n 1 | jq .t_inactive` &&
-        count=`flux job list-inactive --since=${timestamp} | wc -l` &&
-        test $count -eq 0
-'
-
-test_expect_success HAVE_JQ 'flux job list-inactive w/ since (second to most recent timestamp)' '
-        timestamp=`cat list-inactive.out | head -n 2 | tail -n 1 | jq .t_inactive` &&
-        count=`flux job list-inactive --since=${timestamp} | wc -l` &&
-        test $count -eq 1
-'
-
-test_expect_success HAVE_JQ 'flux job list-inactive w/ since (oldest timestamp)' '
-        timestamp=`cat list-inactive.out | tail -n 1 | jq .t_inactive` &&
-        count=`flux job list-inactive --since=${timestamp} | wc -l` &&
-        test $count -eq 19
-'
-
-test_expect_success HAVE_JQ 'flux job list-inactive w/ since (middle timestamp #1)' '
-        timestamp=`cat list-inactive.out | head -n 8 | tail -n 1 | jq .t_inactive` &&
-        count=`flux job list-inactive --since=${timestamp} | wc -l` &&
-        test $count -eq 7
-'
-
-test_expect_success HAVE_JQ 'flux job list-inactive w/ since (middle timestamp #2)' '
-        timestamp=`cat list-inactive.out | head -n 13 | tail -n 1 | jq .t_inactive` &&
-        count=`flux job list-inactive --since=${timestamp} | wc -l` &&
-        test $count -eq 12
-'
-
-
-# job list-id
-
-test_expect_success HAVE_JQ 'flux job list-ids works with a single ID' '
-        id=`head -n 1 job_ids_pending.out` &&
-        flux job list-ids $id | jq -e ".id == ${id}" &&
-        id=`head -n 1 job_ids_running.out` &&
-        flux job list-ids $id | jq -e ".id == ${id}" &&
-        id=`head -n 1 job_ids_inactive.out` &&
-        flux job list-ids $id | jq -e ".id == ${id}"
-'
-
-test_expect_success HAVE_JQ 'flux job list-ids multiple IDs works' '
-        ids=`cat job_ids_pending.out | tr "\n" " "` &&
-        flux job list-ids $ids | jq .id > list_idsP.out &&
-        test_cmp list_idsP.out job_ids_pending.out &&
-        ids=`cat job_ids_running.out | tr "\n" " "` &&
-        flux job list-ids $ids | jq .id > list_idsR.out &&
-        test_cmp list_idsR.out job_ids_running.out &&
-        ids=`cat job_ids_inactive.out | tr "\n" " "` &&
-        flux job list-ids $ids | jq .id > list_idsI.out &&
-        test_cmp list_idsI.out job_ids_inactive.out &&
-        ids=`cat job_ids_pending.out job_ids_running.out job_ids_inactive.out | tr "\n" " "` &&
-        flux job list-ids $ids | jq .id > list_idsPRI.out &&
-        cat job_ids_pending.out job_ids_running.out job_ids_inactive.out > list_idsPRI.exp &&
-        test_cmp list_idsPRI.exp list_idsPRI.out
-'
-
-test_expect_success HAVE_JQ 'flux job list-ids fails without ID' '
-        test_must_fail flux job list-ids
-'
-
-test_expect_success HAVE_JQ 'flux job list-ids fails with bad ID' '
-        test_must_fail flux job list-ids 1234567890
-'
-
-test_expect_success HAVE_JQ 'flux job list-ids fails with not an ID' '
-        test_must_fail flux job list-ids foobar
-'
-
-test_expect_success HAVE_JQ 'flux job list-ids fails with one bad ID out of several' '
-        id1=`head -n 1 job_ids_pending.out` &&
-        id2=`head -n 1 job_ids_running.out` &&
-        id3=`head -n 1 job_ids_inactive.out` &&
-        test_must_fail flux job list-ids ${id1} ${id2} 1234567890 ${id3}
-'
-
-# In order to test potential racy behavior, use job state pause/unpause to pause
-# the handling of job state transitions from the job-manager.
-#
-# Note that between the background process of `flux job list-ids` and
-# `unpause`, we must ensure the background process has sent the
-# request to the job-info service and is now waiting for the id to be
-# synced.  We call wait_idsync to check stats for this to ensure the
-# racy behavior is covered.
-
-wait_idsync() {
-        local num=$1
-        local i=0
-        while (! flux module stats --parse idsync.waits job-info > /dev/null 2>&1 \
-               || [ "$(flux module stats --parse idsync.waits job-info 2> /dev/null)" != "$num" ]) \
-              && [ $i -lt 50 ]
-        do
-                sleep 0.1
-                i=$((i + 1))
-        done
-        if [ "$i" -eq "50" ]
-        then
-            return 1
-        fi
-        return 0
-}
-
-test_expect_success HAVE_JQ,NO_CHAIN_LINT 'flux job list-ids waits for job ids (one id)' '
-	${RPC} job-info.job-state-pause 0 </dev/null
-        jobid=`flux mini submit hostname`
-        flux job wait-event $jobid clean >/dev/null
-        flux job list-ids ${jobid} > list_id_wait1.out &
-        pid=$!
-        wait_idsync 1 &&
-	${RPC} job-info.job-state-unpause 0 </dev/null &&
-        wait $pid &&
-        cat list_id_wait1.out | jq -e ".id == ${jobid}"
-'
-
-test_expect_success HAVE_JQ,NO_CHAIN_LINT 'flux job list-ids waits for job ids (different ids)' '
-	${RPC} job-info.job-state-pause 0 </dev/null
-        jobid1=`flux mini submit hostname`
-        jobid2=`flux mini submit hostname`
-        flux job wait-event ${jobid1} clean >/dev/null
-        flux job wait-event ${jobid2} clean >/dev/null
-        flux job list-ids ${jobid1} ${jobid2} > list_id_wait2.out &
-        pid=$!
-        wait_idsync 2 &&
-	${RPC} job-info.job-state-unpause 0 </dev/null &&
-        wait $pid &&
-        grep ${jobid1} list_id_wait2.out &&
-        grep ${jobid2} list_id_wait2.out
-'
-
-test_expect_success HAVE_JQ,NO_CHAIN_LINT 'flux job list-ids waits for job ids (same id)' '
-	${RPC} job-info.job-state-pause 0 </dev/null
-        jobid=`flux mini submit hostname`
-        flux job wait-event $jobid clean >/dev/null
-        flux job list-ids ${jobid} > list_id_wait3A.out &
-        pid1=$!
-        flux job list-ids ${jobid} > list_id_wait3B.out &
-        pid2=$!
-        wait_idsync 1 &&
-	${RPC} job-info.job-state-unpause 0 </dev/null &&
-        wait ${pid1} &&
-        wait ${pid2} &&
-        cat list_id_wait3A.out | jq -e ".id == ${jobid}" &&
-        cat list_id_wait3B.out | jq -e ".id == ${jobid}"
-'
-
-#
-# job list timing
-#
-
-# simply test that value in timestamp increases through job states
-test_expect_success HAVE_JQ 'flux job list job state timing outputs valid (job inactive)' '
-        jobid=$(flux mini submit hostname) &&
-        flux job wait-event $jobid clean >/dev/null &&
-        obj=$(flux job list -s inactive | grep $jobid) &&
-        echo $obj | jq -e ".t_depend < .t_sched" &&
-        echo $obj | jq -e ".t_sched < .t_run" &&
-        echo $obj | jq -e ".t_run < .t_cleanup" &&
-        echo $obj | jq -e ".t_cleanup < .t_inactive"
-'
-
-# since job is running, make sure latter states don't exist
-test_expect_success HAVE_JQ 'flux job list job state timing outputs valid (job running)' '
-        jobid=$(flux mini submit sleep 60) &&
-        flux job wait-event $jobid start >/dev/null &&
-        obj=$(flux job list -s running | grep $jobid) &&
-        echo $obj | jq -e ".t_depend < .t_sched" &&
-        echo $obj | jq -e ".t_sched < .t_run" &&
-        echo $obj | jq -e ".t_cleanup == null" &&
-        echo $obj | jq -e ".t_inactive == null" &&
-        flux job cancel $jobid &&
-        flux job wait-event $jobid clean >/dev/null
-'
-
-#
-# job names
-#
-
-test_expect_success 'flux job list outputs user job name' '
-        jobid=`flux mini submit --setattr system.job.name=foobar A B C` &&
-        echo $jobid > jobname1.id &&
-        flux job wait-event $jobid clean >/dev/null &&
-        flux job list -s inactive | grep $jobid | grep foobar
-'
-
-test_expect_success 'flux job lists first argument for job name' '
-        jobid=`flux mini submit mycmd arg1 arg2` &&
-        echo $jobid > jobname2.id &&
-        flux job wait-event $jobid clean >/dev/null &&
-        flux job list -s inactive | grep $jobid | grep mycmd
-'
-
-test_expect_success 'flux job lists basename of first argument for job name' '
-        jobid=`flux mini submit /foo/bar arg1 arg2` &&
-        echo $jobid > jobname3.id &&
-        flux job wait-event $jobid clean >/dev/null &&
-        flux job list -s inactive | grep $jobid | grep bar &&
-        flux job list -s inactive | grep $jobid | grep -v foo
-'
-
-test_expect_success 'flux job lists full path for job name if first argument not ok' '
-        jobid=`flux mini submit /foo/bar/ arg1 arg2` &&
-        echo $jobid > jobname4.id &&
-        flux job wait-event $jobid clean >/dev/null &&
-        flux job list -s inactive | grep $jobid | grep "\/foo\/bar\/"
-'
-
-test_expect_success 'reload the job-info module' '
-        flux exec -r all flux module reload job-info
-'
-
-test_expect_success 'verify job names preserved across restart' '
-        jobid1=`cat jobname1.id` &&
-        jobid2=`cat jobname2.id` &&
-        jobid3=`cat jobname3.id` &&
-        jobid4=`cat jobname4.id` &&
-        flux job list -s inactive | grep ${jobid1} | grep foobar &&
-        flux job list -s inactive | grep ${jobid2} | grep mycmd &&
-        flux job list -s inactive | grep ${jobid3} | grep bar &&
-        flux job list -s inactive | grep ${jobid4} | grep "\/foo\/bar\/"
-'
-
-#
-# job task count
-#
-
-test_expect_success 'flux job list outputs ntasks correctly (1 task)' '
-        jobid=`flux mini submit hostname` &&
-        echo $jobid > taskcount1.id &&
-        flux job wait-event $jobid clean >/dev/null &&
-        obj=$(flux job list -s inactive | grep $jobid) &&
-        echo $obj | jq -e ".ntasks == 1"
-'
-
-test_expect_success 'flux job list outputs ntasks correctly (4 tasks)' '
-        jobid=`flux mini submit -n4 hostname` &&
-        echo $jobid > taskcount2.id &&
-        flux job wait-event $jobid clean >/dev/null &&
-        obj=$(flux job list -s inactive | grep $jobid) &&
-        echo $obj | jq -e ".ntasks == 4"
-'
-
-test_expect_success 'reload the job-info module' '
-        flux exec -r all flux module reload job-info
-'
-
-test_expect_success 'verify task count preserved across restart' '
-        jobid1=`cat taskcount1.id` &&
-        jobid2=`cat taskcount2.id` &&
-        obj=$(flux job list -s inactive | grep ${jobid1}) &&
-        echo $obj | jq -e ".ntasks == 1" &&
-        obj=$(flux job list -s inactive | grep ${jobid2}) &&
-        echo $obj | jq -e ".ntasks == 4"
-'
-
-#
-# job node count / rank list
-#
-
-test_expect_success 'flux job list outputs nnodes/ranks correctly (1 task / 1 node)' '
-        jobid=`flux mini submit -n1 hostname` &&
-        echo $jobid > nodecount1.id &&
-        flux job wait-event $jobid clean >/dev/null &&
-        obj=$(flux job list -s inactive | grep $jobid) &&
-        echo $obj | jq -e ".nnodes == 1" &&
-        echo $obj | jq -e ".ranks == \"0\""
-'
-
-test_expect_success 'flux job list outputs nnodes/ranks correctly (2 tasks, / 1 node)' '
-        jobid=`flux mini submit -n2 hostname` &&
-        echo $jobid > nodecount2.id &&
-        flux job wait-event $jobid clean >/dev/null &&
-        obj=$(flux job list -s inactive | grep $jobid) &&
-        echo $obj | jq -e ".nnodes == 1" &&
-        echo $obj | jq -e ".ranks == \"0\""
-'
-
-test_expect_success 'flux job list outputs nnodes/ranks correctly (3 tasks, / 2 nodes)' '
-        jobid=`flux mini submit -n3 hostname` &&
-        echo $jobid > nodecount3.id &&
-        flux job wait-event $jobid clean >/dev/null &&
-        obj=$(flux job list -s inactive | grep $jobid) &&
-        echo $obj | jq -e ".nnodes == 2" &&
-        echo $obj | jq -e ".ranks == \"[0-1]\""
-'
-
-test_expect_success 'flux job list outputs nnodes/ranks correctly (5 tasks, / 3 nodes)' '
-        jobid=`flux mini submit -n5 hostname` &&
-        echo $jobid > nodecount4.id &&
-        flux job wait-event $jobid clean >/dev/null &&
-        obj=$(flux job list -s inactive | grep $jobid) &&
-        echo $obj | jq -e ".nnodes == 3" &&
-        echo $obj | jq -e ".ranks == \"[0-2]\""
-'
-
-test_expect_success 'reload the job-info module' '
-        flux exec -r all flux module reload job-info
-'
-
-test_expect_success 'verify nnodes preserved across restart' '
-        jobid1=`cat nodecount1.id` &&
-        jobid2=`cat nodecount2.id` &&
-        jobid3=`cat nodecount3.id` &&
-        jobid4=`cat nodecount4.id` &&
-        obj=$(flux job list -s inactive | grep ${jobid1}) &&
-        echo $obj | jq -e ".nnodes == 1" &&
-        echo $obj | jq -e ".ranks == \"0\"" &&
-        obj=$(flux job list -s inactive | grep ${jobid2}) &&
-        echo $obj | jq -e ".nnodes == 1" &&
-        echo $obj | jq -e ".ranks == \"0\"" &&
-        obj=$(flux job list -s inactive | grep ${jobid3}) &&
-        echo $obj | jq -e ".nnodes == 2" &&
-        echo $obj | jq -e ".ranks == \"[0-1]\"" &&
-        obj=$(flux job list -s inactive | grep ${jobid4}) &&
-        echo $obj | jq -e ".nnodes == 3" &&
-        echo $obj | jq -e ".ranks == \"[0-2]\""
-'
-
-#
-# job list special cases
-#
-
-test_expect_success 'list count / max_entries works' '
-        count=`flux job list -s inactive -c 0 | wc -l` &&
-        test $count -gt 5 &&
-        count=`flux job list -s inactive -c 5 | wc -l` &&
-        test $count -eq 5
-'
-
-test_expect_success HAVE_JQ 'list request with empty attrs works' '
-        id=$(id -u) &&
-        $jq -j -c -n  "{max_entries:5, userid:${id}, states:0, attrs:[]}" \
-          | $RPC job-info.list > list_empty_attrs.out &&
-        test_must_fail grep "userid" list_empty_attrs.out &&
-        test_must_fail grep "priority" list_empty_attrs.out &&
-        test_must_fail grep "t_submit" list_empty_attrs.out &&
-        test_must_fail grep "state" list_empty_attrs.out &&
-        test_must_fail grep "name" list_empty_attrs.out &&
-        test_must_fail grep "ntasks" list_empty_attrs.out &&
-        test_must_fail grep "nnodes" list_empty_attrs.out &&
-        test_must_fail grep "t_depend" list_empty_attrs.out &&
-        test_must_fail grep "t_sched" list_empty_attrs.out &&
-        test_must_fail grep "t_run" list_empty_attrs.out &&
-        test_must_fail grep "t_cleanup" list_empty_attrs.out &&
-        test_must_fail grep "t_inactive" list_empty_attrs.out
-'
-test_expect_success HAVE_JQ 'list request with excessive max_entries works' '
-        id=$(id -u) &&
-        $jq -j -c -n  "{max_entries:100000, userid:${id}, states:0, attrs:[]}" \
-          | $RPC job-info.list
-'
-test_expect_success HAVE_JQ 'list-attrs works' '
-        $RPC job-info.list-attrs < /dev/null > list_attrs.out &&
-        grep userid list_attrs.out &&
-        grep priority list_attrs.out &&
-        grep t_submit list_attrs.out &&
-        grep state list_attrs.out &&
-        grep name list_attrs.out &&
-        grep ntasks list_attrs.out &&
-        grep nnodes list_attrs.out &&
-        grep t_depend list_attrs.out &&
-        grep t_sched list_attrs.out &&
-        grep t_run list_attrs.out &&
-        grep t_cleanup list_attrs.out &&
-        grep t_inactive list_attrs.out
-'
-
-#
-# job eventlog tests
-#
-
-test_expect_success 'flux job eventlog works' '
-        jobid=$(submit_job) &&
-	flux job eventlog $jobid > eventlog_a.out &&
-        grep submit eventlog_a.out
-'
-
-test_expect_success 'flux job eventlog works on multiple entries' '
-        jobid=$(submit_job) &&
-        kvsdir=$(flux job id --to=kvs $jobid) &&
-	flux kvs eventlog append ${kvsdir}.eventlog foo &&
-	flux job eventlog $jobid >eventlog_b.out &&
-	grep -q submit eventlog_b.out &&
-	grep -q foo eventlog_b.out
-'
-
-test_expect_success 'flux job eventlog fails on bad id' '
-	test_must_fail flux job eventlog 12345
-'
-
-test_expect_success 'flux job eventlog --format=json works' '
-        jobid=$(submit_job) &&
-	flux job eventlog --format=json $jobid > eventlog_format1.out &&
-        grep -q "\"name\":\"submit\"" eventlog_format1.out &&
-        grep -q "\"userid\":$(id -u)" eventlog_format1.out
-'
-
-test_expect_success 'flux job eventlog --format=text works' '
-        jobid=$(submit_job) &&
-	flux job eventlog --format=text $jobid > eventlog_format2.out &&
-        grep -q "submit" eventlog_format2.out &&
-        grep -q "userid=$(id -u)" eventlog_format2.out
-'
-
-test_expect_success 'flux job eventlog --format=invalid fails' '
-        jobid=$(submit_job) &&
-	test_must_fail flux job eventlog --format=invalid $jobid
-'
-
-test_expect_success 'flux job eventlog --time-format=raw works' '
-        jobid=$(submit_job) &&
-	flux job eventlog --time-format=raw $jobid > eventlog_time_format1.out &&
-        get_timestamp_field submit eventlog_time_format1.out | grep "\."
-'
-
-test_expect_success 'flux job eventlog --time-format=iso works' '
-        jobid=$(submit_job) &&
-	flux job eventlog --time-format=iso $jobid > eventlog_time_format2.out &&
-        get_timestamp_field submit eventlog_time_format2.out | grep T | grep Z
-'
-
-test_expect_success 'flux job eventlog --time-format=offset works' '
-        jobid=$(submit_job) &&
-	flux job eventlog --time-format=offset $jobid > eventlog_time_format3.out &&
-        get_timestamp_field submit eventlog_time_format3.out | grep "0.000000" &&
-        get_timestamp_field exception eventlog_time_format3.out | grep -v "0.000000"
-'
-
-test_expect_success 'flux job eventlog --time-format=invalid fails works' '
-        jobid=$(submit_job) &&
-	test_must_fail flux job eventlog --time-format=invalid $jobid
-'
-
-test_expect_success 'flux job eventlog -p works' '
-        jobid=$(submit_job) &&
-        flux job eventlog -p "eventlog" $jobid > eventlog_path1.out &&
-        grep submit eventlog_path1.out
-'
-
-test_expect_success 'flux job eventlog -p works (guest.exec.eventlog)' '
-        jobid=$(submit_job) &&
-        flux job eventlog -p "guest.exec.eventlog" $jobid > eventlog_path2.out &&
-        grep done eventlog_path2.out
-'
-
-test_expect_success 'flux job eventlog -p fails on invalid path' '
-        jobid=$(submit_job) &&
-        test_must_fail flux job eventlog -p "foobar" $jobid
-'
-
-#
-# job wait-event tests
-#
-
-test_expect_success 'flux job wait-event works' '
-        jobid=$(submit_job) &&
-        flux job wait-event $jobid submit > wait_event1.out &&
-        grep submit wait_event1.out
-'
-
-test_expect_success NO_CHAIN_LINT 'flux job wait-event errors on non-event' '
-        jobid=$(submit_job) &&
-        test_must_fail flux job wait-event $jobid foobar 2> wait_event2.err &&
-        grep "never received" wait_event2.err
-'
-
-test_expect_success NO_CHAIN_LINT 'flux job wait-event does not see event after clean' '
-        jobid=$(submit_job) &&
-        kvsdir=$(flux job id --to=kvs $jobid) &&
-	flux kvs eventlog append ${kvsdir}.eventlog foobar &&
-        test_must_fail flux job wait-event -v $jobid foobar 2> wait_event3.err &&
-        grep "never received" wait_event3.err
-'
-
-test_expect_success 'flux job wait-event fails on bad id' '
-	test_must_fail flux job wait-event 12345 foobar
-'
-
-test_expect_success 'flux job wait-event --quiet works' '
-        jobid=$(submit_job) &&
-        flux job wait-event --quiet $jobid submit > wait_event4.out &&
-        ! test -s wait_event4.out
-'
-
-test_expect_success 'flux job wait-event --verbose works' '
-        jobid=$(submit_job) &&
-        flux job wait-event --verbose $jobid clean > wait_event5.out &&
-        grep submit wait_event5.out &&
-        grep start wait_event5.out &&
-        grep clean wait_event5.out
-'
-
-test_expect_success 'flux job wait-event --verbose doesnt show events after wait event' '
-        jobid=$(submit_job) &&
-        flux job wait-event --verbose $jobid submit > wait_event6.out &&
-        grep submit wait_event6.out &&
-        ! grep start wait_event6.out &&
-        ! grep clean wait_event6.out
-'
-
-test_expect_success 'flux job wait-event --timeout works' '
-        jobid=$(submit_job_live sleeplong.json) &&
-        test_must_fail flux job wait-event --timeout=0.2 $jobid clean 2> wait_event7.err &&
-        flux job cancel $jobid &&
-        grep "wait-event timeout" wait_event7.err
-'
-
-test_expect_success 'flux job wait-event --format=json works' '
-        jobid=$(submit_job) &&
-	flux job wait-event --format=json $jobid submit > wait_event_format1.out &&
-        grep -q "\"name\":\"submit\"" wait_event_format1.out &&
-        grep -q "\"userid\":$(id -u)" wait_event_format1.out
-'
-
-test_expect_success 'flux job wait-event --format=text works' '
-        jobid=$(submit_job) &&
-	flux job wait-event --format=text $jobid submit > wait_event_format2.out &&
-        grep -q "submit" wait_event_format2.out &&
-        grep -q "userid=$(id -u)" wait_event_format2.out
-'
-
-test_expect_success 'flux job wait-event --format=invalid fails' '
-        jobid=$(submit_job) &&
-	test_must_fail flux job wait-event --format=invalid $jobid submit
-'
-
-test_expect_success 'flux job wait-event --time-format=raw works' '
-        jobid=$(submit_job) &&
-	flux job wait-event --time-format=raw $jobid submit > wait_event_time_format1.out &&
-        get_timestamp_field submit wait_event_time_format1.out | grep "\."
-'
-
-test_expect_success 'flux job wait-event --time-format=iso works' '
-        jobid=$(submit_job) &&
-	flux job wait-event --time-format=iso $jobid submit > wait_event_time_format2.out &&
-        get_timestamp_field submit wait_event_time_format2.out | grep T | grep Z
-'
-
-test_expect_success 'flux job wait-event --time-format=offset works' '
-        jobid=$(submit_job) &&
-	flux job wait-event --time-format=offset $jobid submit > wait_event_time_format3A.out &&
-        get_timestamp_field submit wait_event_time_format3A.out | grep "0.000000" &&
-	flux job wait-event --time-format=offset $jobid exception > wait_event_time_format3B.out &&
-        get_timestamp_field exception wait_event_time_format3B.out | grep -v "0.000000"
-'
-
-test_expect_success 'flux job wait-event --time-format=invalid fails works' '
-        jobid=$(submit_job) &&
-	test_must_fail flux job wait-event --time-format=invalid $jobid submit
-'
-
-test_expect_success 'flux job wait-event w/ match-context works (string w/ quotes)' '
-        jobid=$(submit_job) &&
-	flux job wait-event --match-context="type=\"cancel\"" $jobid exception > wait_event_context1.out &&
-        grep -q "exception" wait_event_context1.out &&
-        grep -q "type=\"cancel\"" wait_event_context1.out
-'
-
-test_expect_success 'flux job wait-event w/ match-context works (string w/o quotes)' '
-        jobid=$(submit_job) &&
-	flux job wait-event --match-context=type=cancel $jobid exception > wait_event_context2.out &&
-        grep -q "exception" wait_event_context2.out &&
-        grep -q "type=\"cancel\"" wait_event_context2.out
-'
-
-test_expect_success 'flux job wait-event w/ match-context works (int)' '
-        jobid=$(submit_job) &&
-	flux job wait-event --match-context=flags=0 $jobid submit > wait_event_context3.out &&
-        grep -q "submit" wait_event_context3.out &&
-        grep -q "flags=0" wait_event_context3.out
-'
-
-test_expect_success 'flux job wait-event w/ bad match-context fails (invalid key)' '
-        jobid=$(submit_job) &&
-        test_must_fail flux job wait-event --match-context=foo=bar $jobid exception
-'
-
-test_expect_success 'flux job wait-event w/ bad match-context fails (invalid value)' '
-        jobid=$(submit_job) &&
-        test_must_fail flux job wait-event --match-context=type=foo $jobid exception
-'
-
-test_expect_success 'flux job wait-event w/ bad match-context fails (invalid input)' '
-        jobid=$(submit_job) &&
-        test_must_fail flux job wait-event --match-context=foo $jobid exception
-'
-
-test_expect_success 'flux job wait-event -p works (eventlog)' '
-        jobid=$(submit_job) &&
-        flux job wait-event -p "eventlog" $jobid submit > wait_event_path1.out &&
-        grep submit wait_event_path1.out
-'
-
-test_expect_success 'flux job wait-event -p works (guest.exec.eventlog)' '
-        jobid=$(submit_job) &&
-        flux job wait-event -p "guest.exec.eventlog" $jobid done > wait_event_path2.out &&
-        grep done wait_event_path2.out
-'
-
-test_expect_success 'flux job wait-event -p works (non-guest eventlog)' '
-        jobid=$(submit_job) &&
-        kvsdir=$(flux job id --to=kvs $jobid) &&
-	flux kvs eventlog append ${kvsdir}.foobar.eventlog foobar &&
-        flux job wait-event -p "foobar.eventlog" $jobid foobar > wait_event_path3.out &&
-        grep foobar wait_event_path3.out
-'
-
-test_expect_success 'flux job wait-event -p fails on invalid path' '
-        jobid=$(submit_job) &&
-        test_must_fail flux job wait-event -p "foobar" $jobid submit
-'
-
-test_expect_success 'flux job wait-event -p fails on path "guest."' '
-        jobid=$(submit_job) &&
-        test_must_fail flux job wait-event -p "guest." $jobid submit
-'
-
-test_expect_success 'flux job wait-event -p hangs on non-guest eventlog' '
-        jobid=$(submit_job) &&
-        kvsdir=$(flux job id --to=kvs $jobid) &&
-	flux kvs eventlog append ${kvsdir}.foobar.eventlog foo &&
-        test_expect_code 142 run_timeout 0.2 flux job wait-event -p "foobar.eventlog" $jobid bar
-'
-
-test_expect_success NO_CHAIN_LINT 'flux job wait-event -p guest.exec.eventlog works (live job)' '
-        jobid=$(submit_job_live sleeplong.json)
-        flux job wait-event -p "guest.exec.eventlog" $jobid done > wait_event_path4.out &
-        waitpid=$! &&
-        wait_watchers_nonzero "watchers" &&
-        wait_watchers_nonzero "guest_watchers" &&
-        guestns=$(flux job namespace $jobid) &&
-        wait_watcherscount_nonzero $guestns &&
-        flux job cancel $jobid &&
-        wait $waitpid &&
-        grep done wait_event_path4.out
-'
-
-test_expect_success 'flux job wait-event -p times out on no event (live job)' '
-        jobid=$(submit_job_live sleeplong.json) &&
-        test_expect_code 142 run_timeout 0.2 flux job wait-event -p "guest.exec.eventlog" $jobid foobar &&
-        flux job cancel $jobid
-'
-
-# In order to test watching a guest event log that does not yet exist,
-# we will start a job that will take up all resources.  Then start
-# another job, which we will watch and know it hasn't started running
-# yet. Then we cancel the initial job to get the new one running.
-
-test_expect_success 'job-info: generate jobspec to consume all resources' '
-        flux jobspec --format json srun -n4 -c2 sleep 300 > sleeplong-all-rsrc.json
-'
-
-test_expect_success NO_CHAIN_LINT 'flux job wait-event -p guest.exec.eventlog works (wait job)' '
-        jobidall=$(submit_job_live sleeplong-all-rsrc.json)
-        jobid=$(submit_job_wait)
-        flux job wait-event -v -p "guest.exec.eventlog" ${jobid} done > wait_event_path5.out &
-        waitpid=$! &&
-        wait_watchers_nonzero "watchers" &&
-        wait_watchers_nonzero "guest_watchers" &&
-        flux job cancel ${jobidall} &&
-        flux job wait-event ${jobid} start &&
-        guestns=$(flux job namespace ${jobid}) &&
-        wait_watcherscount_nonzero $guestns &&
-        flux job cancel ${jobid} &&
-        wait $waitpid &&
-        grep done wait_event_path5.out
-'
-
-test_expect_success 'flux job wait-event -p times out on no event (wait job)' '
-        jobidall=$(submit_job_live sleeplong-all-rsrc.json) &&
-        jobid=$(submit_job_wait) &&
-        test_expect_code 142 run_timeout 0.2 flux job wait-event -p "guest.exec.eventlog" $jobid foobar &&
-        flux job cancel $jobidall &&
-        flux job cancel $jobid
-'
-
-# In order to test watching a guest event log that will never exist,
-# we will start a job that will take up all resources.  Then start
-# another job, which we will watch and know it hasn't started running
-# yet. Then we cancel the second job before we know it has started.
-
-test_expect_success NO_CHAIN_LINT 'flux job wait-event -p guest.exec.eventlog works (never start job)' '
-        jobidall=$(submit_job_live sleeplong-all-rsrc.json)
-        jobid=$(submit_job_wait)
-        flux job wait-event -v -p "guest.exec.eventlog" ${jobid} done > wait_event_path6.out &
-        waitpid=$! &&
-        wait_watchers_nonzero "watchers" &&
-        wait_watchers_nonzero "guest_watchers" &&
-        flux job cancel ${jobid} &&
-        ! wait $waitpid &&
-        flux job cancel ${jobidall}
-'
-
-#
-# job info tests
-#
-
-test_expect_success 'flux job info eventlog works' '
-        jobid=$(submit_job) &&
-	flux job info $jobid eventlog > eventlog_info_a.out &&
-        grep submit eventlog_info_a.out
-'
-
-test_expect_success 'flux job info eventlog fails on bad id' '
-	test_must_fail flux job info 12345 eventlog
-'
-
-test_expect_success 'flux job info jobspec works' '
-        jobid=$(submit_job) &&
-	flux job info $jobid jobspec > jobspec_a.out &&
-        grep sleep jobspec_a.out
-'
-
-test_expect_success 'flux job info jobspec fails on bad id' '
-	test_must_fail flux job info 12345 jobspec
-'
-
-#
-# job info tests (multiple info requests)
-#
-
-test_expect_success 'flux job info multiple keys works' '
-        jobid=$(submit_job) &&
-	flux job info $jobid eventlog jobspec J > all_info_a.out &&
-        grep submit all_info_a.out &&
-        grep sleep all_info_a.out
-'
-
-test_expect_success 'flux job info multiple keys fails on bad id' '
-	test_must_fail flux job info 12345 eventlog jobspec J
-'
-
-test_expect_success 'flux job info multiple keys fails on 1 bad entry (include eventlog)' '
-        jobid=$(submit_job) &&
-        kvsdir=$(flux job id --to=kvs $jobid) &&
-        flux kvs unlink ${kvsdir}.jobspec &&
-	test_must_fail flux job info $jobid eventlog jobspec J > all_info_b.out
-'
-
-test_expect_success 'flux job info multiple keys fails on 1 bad entry (no eventlog)' '
-        jobid=$(submit_job) &&
-        kvsdir=$(flux job id --to=kvs $jobid) &&
-        flux kvs unlink ${kvsdir}.jobspec &&
-	test_must_fail flux job info $jobid jobspec J > all_info_b.out
-'
-
-#
-# stats
-#
-
-test_expect_success 'job-info stats works' '
-        flux module stats --parse lookups job-info &&
-        flux module stats --parse watchers job-info &&
-        flux module stats --parse guest_watchers job-info &&
-        flux module stats --parse jobs.pending job-info &&
-        flux module stats --parse jobs.running job-info &&
-        flux module stats --parse jobs.inactive job-info &&
-        flux module stats --parse idsync.lookups job-info &&
-        flux module stats --parse idsync.waits job-info
-'
-
-test_expect_success 'lookup request with empty payload fails with EPROTO(71)' '
-	${RPC} job-info.lookup 71 </dev/null
-'
-test_expect_success 'eventlog-watch request with empty payload fails with EPROTO(71)' '
-	${RPC} job-info.eventlog-watch 71 </dev/null
-'
-test_expect_success 'guest-eventlog-watch request with empty payload fails with EPROTO(71)' '
-	${RPC} job-info.guest-eventlog-watch 71 </dev/null
-'
-test_expect_success 'list request with empty payload fails with EPROTO(71)' '
-	${RPC} job-info.list 71 </dev/null
-'
-test_expect_success HAVE_JQ 'list request with invalid input fails with EPROTO(71) (attrs not an array)' '
-        id=$(id -u) &&
-        $jq -j -c -n  "{max_entries:5, userid:${id}, states:0, attrs:5}" \
-          | $RPC job-info.list 71
-'
-test_expect_success HAVE_JQ 'list request with invalid input fails with EINVAL(22) (attrs non-string)' '
-        id=$(id -u) &&
-        $jq -j -c -n  "{max_entries:5, userid:${id}, states:0, attrs:[5]}" \
-          | $RPC job-info.list 22
-'
-test_expect_success HAVE_JQ 'list request with invalid input fails with EINVAL(22) (attrs illegal field)' '
-        id=$(id -u) &&
-        $jq -j -c -n  "{max_entries:5, userid:${id}, states:0, attrs:[\"foo\"]}" \
-          | $RPC job-info.list 22
-'
-test_expect_success 'list-id request with empty payload fails with EPROTO(71)' '
-	${RPC} job-info.list-id 71 </dev/null
-'
-test_expect_success HAVE_JQ 'list-id request with invalid input fails with EPROTO(71) (attrs not an array)' '
-        id=`flux mini submit hostname` &&
-        $jq -j -c -n  "{id:${id}, attrs:5}" \
-          | $RPC job-info.list-id 71
-'
-test_expect_success HAVE_JQ 'list-id request with invalid input fails with EINVAL(22) (attrs non-string)' '
-        id=`flux mini submit hostname` &&
-        $jq -j -c -n  "{id:${id}, attrs:[5]}" \
-          | $RPC job-info.list-id 22
-'
-test_expect_success HAVE_JQ 'list-id request with invalid input fails with EINVAL(22) (attrs illegal field)' '
-        id=`flux mini submit hostname` &&
-        $jq -j -c -n  "{id:${id}, attrs:[\"foo\"]}" \
-          | $RPC job-info.list-id 22
-'
-test_expect_success 'list-inactive request with empty payload fails with EPROTO(71)' '
-	${RPC} job-info.list-inactive 71 </dev/null
-'
-test_expect_success HAVE_JQ 'list-inactive request with invalid input fails with EPROTO(71) (attrs not an array)' '
-        $jq -j -c -n  "{max_entries:5, since:0.0, attrs:5}" \
-          | $RPC job-info.list-inactive 71
-'
-test_expect_success HAVE_JQ 'list-inactive request with invalid input fails with EINVAL(22) (attrs non-string)' '
-        $jq -j -c -n  "{max_entries:5, since:0.0, attrs:[5]}" \
-          | $RPC job-info.list-inactive 22
-'
-test_expect_success HAVE_JQ 'list-inactive request with invalid input fails with EINVAL(22) (attrs illegal field)' '
-        $jq -j -c -n  "{max_entries:5, since:0.0, attrs:[\"foo\"]}" \
-          | $RPC job-info.list-inactive 22
-'
-
-
-#
-# stress test
-#
-
-wait_jobs_finish() {
-        local i=0
-        while ([ "$(flux job list | wc -l)" != "0" ]) \
-              && [ $i -lt 1000 ]
-        do
-                sleep 0.1
-                i=$((i + 1))
-        done
-        if [ "$i" -eq "1000" ]
-        then
-            return 1
-        fi
-        return 0
-}
-
-test_expect_success LONGTEST 'stress job-info.list-id' '
-        flux python ${FLUX_SOURCE_DIR}/t/job-info/list-id.py 500 &&
-        wait_jobs_finish
-'
-test_done
diff --git a/t/t2204-job-manager-limited.t b/t/t2204-job-manager-limited.t
new file mode 100755
index 000000000000..81047c38d663
--- /dev/null
+++ b/t/t2204-job-manager-limited.t
@@ -0,0 +1,225 @@
+#!/bin/sh
+
+test_description='Test flux job manager service with sched-simple (limited)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="limited=2"
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'job-manager: submit 5 jobs' '
+        flux submit --log=job{cc}.id --cc="1-5" --flags=debug -n1 \
+           hostname
+'
+
+test_expect_success 'job-manager: job state RRSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs (RRSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: annotate jobs job-list (RRSSS)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jlist_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.jobs_ahead" "1" &&
+        jlist_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: annotate jobs in flux-jobs (RRSSS)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.jobs_ahead" "0" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.jobs_ahead" "1" &&
+        fjobs_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: cancel 2' '
+        flux cancel $(cat job2.id)
+'
+
+test_expect_success 'job-manager: job state RIRSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs (RIRSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "1"
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in job-list (RIRSS)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jlist_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jlist_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jlist_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jlist_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job5.id) "sched.jobs_ahead" "1"
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in flux-jobs (RIRSS)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        test_must_fail fjobs_check_annotation_exists $(cat job3.id) "annotations.sched.reason_pending" &&
+        test_must_fail fjobs_check_annotation_exists $(cat job3.id) "annotations.sched.jobs_ahead" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.jobs_ahead" "0" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.jobs_ahead" "1"
+'
+
+# cancel non-running jobs first, to ensure they are not accidentally run when
+# running jobs free resources.
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all --states=pending &&
+        flux cancel --all
+'
+
+test_expect_success 'job-manager: job state IIIII' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) I &&
+        jmgr_check_state $(cat job4.id) I &&
+        jmgr_check_state $(cat job5.id) I
+'
+
+test_expect_success 'job-manager: no annotations (IIIII)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, note that job ids that ran retain annotations
+test_expect_success 'job-manager: no annotations in canceled jobs in job-list (IIIII)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_no_annotations $(cat job4.id) &&
+        jlist_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, note that job ids that ran retain annotations
+test_expect_success 'job-manager: no annotations in canceled jobs in flux jobs (IIIII)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_no_annotations $(cat job4.id) &&
+        fjobs_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: all jobs are inactive' '
+        flux queue idle --timeout 30s
+'
+test_expect_success 'job-manager: submit 5 jobs using one node each' '
+        flux submit --log=job{cc}.id --cc="1-5" -N1 true
+'
+test_expect_success 'job-manager: job state RSSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+test_expect_success 'job-manager: queue status shows 2 alloc pending, 2 queued' '
+        flux queue status -v |grep "2 alloc requests queued" &&
+        flux queue status -v |grep "2 alloc requests pending"
+'
+test_expect_success 'job-manager: cancel one pending job' '
+        flux cancel $(cat job2.id)
+'
+test_expect_success 'job-manager: job state RISSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+test_expect_success 'job-manager: 2 alloc req pending, 1 queued' '
+        flux queue status -v |grep "1 alloc requests queued" &&
+        flux queue status -v |grep "2 alloc requests pending"
+'
+test_expect_success 'job-manager: reload the scheduler' '
+	flux module reload sched-simple mode=limited=2
+'
+test_expect_success 'job-manager: job state RISSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+test_expect_success 'job-manager: 2 alloc req pending, 1 queued' '
+        flux queue status -v |grep "1 alloc requests queued" &&
+        flux queue status -v |grep "2 alloc requests pending"
+'
+test_expect_success 'job-manager: cancel one running job' '
+        flux cancel $(cat job1.id)
+'
+test_expect_success 'job-manager: job state IIRSS' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+test_expect_success 'job-manager: 2 alloc req pending, 0 queued' '
+        flux queue status -v |grep "0 alloc requests queued" &&
+        flux queue status -v |grep "2 alloc requests pending"
+'
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all
+'
+test_expect_success 'job-manager: job state IIIII' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) I &&
+        jmgr_check_state $(cat job4.id) I &&
+        jmgr_check_state $(cat job5.id) I
+'
+test_expect_success 'job-manager: 0 alloc req pending, 0 queued' '
+        flux queue status -v |grep "0 alloc requests queued" &&
+        flux queue status -v |grep "0 alloc requests pending"
+'
+
+test_done
diff --git a/t/t2205-job-info-security.t b/t/t2205-job-info-security.t
deleted file mode 100755
index 35d58ef80670..000000000000
--- a/t/t2205-job-info-security.t
+++ /dev/null
@@ -1,282 +0,0 @@
-#!/bin/sh
-
-test_description='Test flux job info service security'
-
-. $(dirname $0)/sharness.sh
-
-test_under_flux 4 job
-
-# We have to fake a job submission by a guest into the KVS.
-# This method of editing the eventlog preserves newline separators.
-
-update_job_userid() {
-        local userid=$1
-        if test -n "$userid"; then
-            local kvsdir=$(flux job id --to=kvs $jobid)
-            flux kvs get --raw ${kvsdir}.eventlog \
-                | sed -e 's/\("userid":\)[0-9]*/\1'${userid}/ \
-                | flux kvs put --raw ${kvsdir}.eventlog=-
-        fi
-}
-
-# Usage: submit_job [userid]
-# To ensure robustness of tests despite future job manager changes,
-# cancel the job, and wait for clean event.  Optionally, edit the
-# userid
-submit_job() {
-        local jobid=$(flux job submit sleeplong.json) &&
-        flux job wait-event $jobid start >/dev/null &&
-        flux job cancel $jobid &&
-        flux job wait-event $jobid clean >/dev/null &&
-        update_job_userid $1 &&
-        echo $jobid
-}
-
-# Unlike above, do not cancel the job, the test will cancel the job
-submit_job_live() {
-        local jobid=$(flux job submit sleeplong.json) &&
-        flux job wait-event $jobid start >/dev/null &&
-        update_job_userid $1 &&
-        echo $jobid
-}
-
-# Usage: bad_first_event
-# Wait for job eventlog to include 'depend' event, then mangle submit event name
-bad_first_event() {
-        local jobid=$(flux job submit sleeplong.json) &&
-        flux job wait-event $jobid depend >/dev/null &&
-        local kvsdir=$(flux job id --to=kvs $jobid) &&
-        flux kvs get --raw ${kvsdir}.eventlog \
-            | sed -e s/submit/foobar/ \
-            | flux kvs put --raw ${kvsdir}.eventlog=- &&
-        flux job cancel $jobid &&
-        echo $jobid
-}
-
-set_userid() {
-        export FLUX_HANDLE_USERID=$1
-        export FLUX_HANDLE_ROLEMASK=0x2
-}
-
-unset_userid() {
-        unset FLUX_HANDLE_USERID
-        unset FLUX_HANDLE_ROLEMASK
-}
-
-test_expect_success 'job-info: generate jobspec for simple test job' '
-        flux jobspec --format json srun -N1 sleep 300 > sleeplong.json
-'
-
-#
-# job eventlog
-#
-
-test_expect_success 'flux job eventlog works (owner)' '
-        jobid=$(submit_job) &&
-        flux job eventlog $jobid
-'
-
-test_expect_success 'flux job eventlog works (user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9000 &&
-        flux job eventlog $jobid &&
-        unset_userid
-'
-
-test_expect_success 'flux job eventlog fails (wrong user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9999 &&
-        ! flux job eventlog $jobid &&
-        unset_userid
-'
-
-test_expect_success 'flux job eventlog fails on bad first event (user)' '
-        jobid=$(bad_first_event 9000) &&
-        set_userid 9999 &&
-        ! flux job eventlog $jobid &&
-        unset_userid
-'
-
-test_expect_success 'flux job guest.exec.eventlog works via -p (owner)' '
-        jobid=$(submit_job) &&
-        flux job eventlog -p guest.exec.eventlog $jobid
-'
-
-test_expect_success 'flux job guest.exec.eventlog works via -p (user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9000 &&
-        flux job eventlog -p guest.exec.eventlog $jobid &&
-        unset_userid
-'
-
-test_expect_success 'flux job guest.exec.eventlog fails via -p (wrong user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9999 &&
-        ! flux job eventlog -p guest.exec.eventlog $jobid &&
-        unset_userid
-'
-
-#
-# job wait-event
-#
-
-test_expect_success 'flux job wait-event works (owner)' '
-        jobid=$(submit_job) &&
-        flux job wait-event $jobid submit
-'
-
-test_expect_success 'flux job wait-event works (user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9000 &&
-        flux job wait-event $jobid submit &&
-        unset_userid
-'
-
-test_expect_success 'flux job wait-event fails (wrong user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9999 &&
-        ! flux job wait-event $jobid submit &&
-        unset_userid
-'
-
-test_expect_success 'flux job wait-event guest.exec.eventlog works via -p (owner)' '
-        jobid=$(submit_job) &&
-        flux job wait-event -p guest.exec.eventlog $jobid done
-'
-
-test_expect_success 'flux job wait-event guest.exec.eventlog works via -p (user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9000 &&
-        flux job wait-event -p guest.exec.eventlog $jobid done &&
-        unset_userid
-'
-
-test_expect_success 'flux job wait-event guest.exec.eventlog fails via -p (wrong user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9999 &&
-        ! flux job wait-event -p guest.exec.eventlog $jobid done &&
-        unset_userid
-'
-
-test_expect_success 'flux job wait-event guest.exec.eventlog works via -p (live job, owner)' '
-        jobid=$(submit_job_live) &&
-        flux job wait-event -p guest.exec.eventlog $jobid init &&
-        flux job cancel $jobid
-'
-
-test_expect_success 'flux job wait-event guest.exec.eventlog works via -p (live job, user)' '
-        jobid=$(submit_job_live 9000) &&
-        set_userid 9000 &&
-        flux job wait-event -p guest.exec.eventlog $jobid init &&
-        unset_userid &&
-        flux job cancel $jobid
-'
-
-test_expect_success 'flux job wait-event guest.exec.eventlog fails via -p (live job, wrong user)' '
-        jobid=$(submit_job_live 9000) &&
-        set_userid 9999 &&
-        ! flux job wait-event -p guest.exec.eventlog $jobid init &&
-        unset_userid &&
-        flux job cancel $jobid
-'
-
-#
-# job info
-#
-
-test_expect_success 'flux job info eventlog works (owner)' '
-        jobid=$(submit_job) &&
-        flux job info $jobid eventlog
-'
-
-test_expect_success 'flux job info eventlog works (user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9000 &&
-        flux job info $jobid eventlog &&
-        unset_userid
-'
-
-test_expect_success 'flux job info eventlog fails (wrong user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9999 &&
-        ! flux job info $jobid eventlog &&
-        unset_userid
-'
-
-test_expect_success 'flux job info jobspec works (owner)' '
-        jobid=$(submit_job) &&
-        flux job info $jobid jobspec
-'
-
-test_expect_success 'flux job info jobspec works (user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9000 &&
-        flux job info $jobid jobspec &&
-        unset_userid
-'
-
-test_expect_success 'flux job info jobspec fails (wrong user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9999 &&
-        ! flux job info $jobid jobspec &&
-        unset_userid
-'
-
-test_expect_success 'flux job info multiple keys works (owner, include eventlog)' '
-        jobid=$(submit_job) &&
-        flux job info $jobid eventlog jobspec J
-'
-
-test_expect_success 'flux job info multiple keys works (user, include eventlog)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9000 &&
-        flux job info $jobid eventlog jobspec J &&
-        unset_userid
-'
-
-test_expect_success 'flux job info multiple keys fails (wrong user, include eventlog)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9999 &&
-        ! flux job info $jobid eventlog jobspec J &&
-        unset_userid
-'
-
-test_expect_success 'flux job info multiple keys works (owner, no eventlog)' '
-        jobid=$(submit_job) &&
-        flux job info $jobid jobspec J
-'
-
-test_expect_success 'flux job info multiple keys works (user, no eventlog)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9000 &&
-        flux job info $jobid jobspec J &&
-        unset_userid
-'
-
-test_expect_success 'flux job info multiple keys fails (wrong user, no eventlog)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9999 &&
-        ! flux job info $jobid jobspec J &&
-        unset_userid
-'
-
-test_expect_success 'flux job info foobar fails (owner)' '
-        jobid=$(submit_job) &&
-        ! flux job info $jobid foobar
-'
-
-test_expect_success 'flux job info foobar fails (user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9000 &&
-        ! flux job info $jobid foobar &&
-        unset_userid
-'
-
-test_expect_success 'flux job info foobar fails (wrong user)' '
-        jobid=$(submit_job 9000) &&
-        set_userid 9999 &&
-        ! flux job info $jobid foobar &&
-        unset_userid
-'
-
-test_done
diff --git a/t/t2205-job-manager-unlimited.t b/t/t2205-job-manager-unlimited.t
new file mode 100755
index 000000000000..e65748b02306
--- /dev/null
+++ b/t/t2205-job-manager-unlimited.t
@@ -0,0 +1,277 @@
+#!/bin/sh
+
+test_description='Test flux job manager service with sched-simple (unlimited)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="unlimited"
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'job-manager: submit 5 jobs' '
+        flux submit --log=job{cc}.id --cc="1-5" --flags=debug -n1 \
+           hostname
+'
+
+test_expect_success 'job-manager: job state RRSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs (RRSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "2"
+'
+
+test_expect_success 'job-manager: annotate jobs job-list (RRSSS)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jlist_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.jobs_ahead" "1" &&
+        jlist_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job5.id) "sched.jobs_ahead" "2"
+'
+
+test_expect_success 'job-manager: annotate jobs in flux-jobs (RRSSS)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.jobs_ahead" "0" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.jobs_ahead" "1" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.jobs_ahead" "2"
+'
+
+test_expect_success 'job-manager: cancel 2' '
+        flux cancel $(cat job2.id)
+'
+
+test_expect_success 'job-manager: job state RIRSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs (RIRSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "1"
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in job-list (RIRSS)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jlist_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jlist_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jlist_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jlist_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job5.id) "sched.jobs_ahead" "1"
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in flux-jobs (RIRSS)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        test_must_fail fjobs_check_annotation_exists $(cat job3.id) "annotations.sched.reason_pending" &&
+        test_must_fail fjobs_check_annotation_exists $(cat job3.id) "annotations.sched.jobs_ahead" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.jobs_ahead" "0" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.jobs_ahead" "1"
+'
+
+test_expect_success 'job-manager: cancel 5' '
+        flux cancel $(cat job5.id)
+'
+
+test_expect_success 'job-manager: job state RIRSI' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) I
+'
+
+test_expect_success 'job-manager: annotate jobs (RIRSI)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in job-list (RIRSS)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jlist_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jlist_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jlist_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jlist_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in flux jobs (RIRSS)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        test_must_fail fjobs_check_annotation_exists $(cat job3.id) "annotations.sched.reason_pending" &&
+        test_must_fail fjobs_check_annotation_exists $(cat job3.id) "annotations.sched.jobs_ahead" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.jobs_ahead" "0" &&
+        fjobs_check_no_annotations $(cat job5.id)
+'
+
+# cancel non-running jobs first, to ensure they are not accidentally run when
+# running jobs free resources.
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all --states=pending &&
+        flux cancel --all
+'
+
+test_expect_success 'job-manager: job state IIIII' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) I &&
+        jmgr_check_state $(cat job4.id) I &&
+        jmgr_check_state $(cat job5.id) I
+'
+
+test_expect_success 'job-manager: no annotations (IIIII)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, note that job ids that ran retain annotations
+test_expect_success 'job-manager: no annotations in canceled jobs in job-list (IIIII)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_no_annotations $(cat job4.id) &&
+        jlist_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, note that job ids that ran retain annotations
+test_expect_success 'job-manager: no annotations in canceled jobs in flux jobs (IIIII)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_no_annotations $(cat job4.id) &&
+        fjobs_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: all jobs are inactive' '
+	flux queue idle --timeout 30s
+'
+test_expect_success 'job-manager: submit 5 jobs using one node each' '
+        flux submit --log=job{cc}.id --cc="1-5" -N1 true
+'
+test_expect_success 'job-manager: job state RSSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+test_expect_success 'job-manager: queue status shows 4 alloc requests pending' '
+	flux queue status -v |grep "0 alloc requests queued" &&
+	flux queue status -v |grep "4 alloc requests pending"
+'
+test_expect_success 'job-manager: cancel one pending job' '
+	flux cancel $(cat job2.id)
+'
+test_expect_success 'job-manager: job state RISSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+test_expect_success 'job-manager: 3 alloc req pending, 0 queued' '
+	flux queue status -v |grep "0 alloc requests queued" &&
+	flux queue status -v |grep "3 alloc requests pending"
+'
+test_expect_success 'job-manager: cancel one running job' '
+	flux cancel $(cat job1.id)
+'
+test_expect_success 'job-manager: job state IIRSS' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+test_expect_success 'job-manager: 2 alloc req pending, 0 queued' '
+	flux queue status -v |grep "0 alloc requests queued" &&
+	flux queue status -v |grep "2 alloc requests pending"
+'
+test_expect_success 'job-manager: reload the scheduler' '
+	flux module reload sched-simple mode=unlimited
+'
+test_expect_success 'job-manager: job state IIRSS' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+test_expect_success 'job-manager: 2 alloc req pending, 0 queued' '
+	flux queue status -v |grep "0 alloc requests queued" &&
+	flux queue status -v |grep "2 alloc requests pending"
+'
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all
+'
+test_expect_success 'job-manager: job state IIIII' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) I &&
+        jmgr_check_state $(cat job4.id) I &&
+        jmgr_check_state $(cat job5.id) I
+'
+test_expect_success 'job-manager: 0 alloc req pending, 0 queued' '
+	flux queue status -v |grep "0 alloc requests queued" &&
+	flux queue status -v |grep "0 alloc requests pending"
+'
+
+test_done
diff --git a/t/t2206-job-manager-annotate.t b/t/t2206-job-manager-annotate.t
new file mode 100755
index 000000000000..3579d304105b
--- /dev/null
+++ b/t/t2206-job-manager-annotate.t
@@ -0,0 +1,214 @@
+#!/bin/sh
+
+test_description='Test flux job manager annoate service'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'job-manager: initially run without scheduler' '
+        flux module unload sched-simple
+'
+test_expect_success 'job-manager: submit 5 jobs' '
+        flux submit --log=job{cc}.id --cc="1-5" --flags=debug -n1 \
+           hostname
+'
+
+test_expect_success 'job-manager: no annotations (SSSSS)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: no annotations in job-list (SSSSS)' '
+        jlist_check_no_annotations $(cat job1.id) &&
+        jlist_check_no_annotations $(cat job2.id) &&
+        jlist_check_no_annotations $(cat job3.id) &&
+        jlist_check_no_annotations $(cat job4.id) &&
+        jlist_check_no_annotations $(cat job5.id)
+'
+
+# --setbit 0x2 enables creation of reason_pending field
+# flux queue stop/start to ensure no raciness with setting up debug bits
+test_expect_success 'job-manager: load sched-simple (1 rank, 2 cores/rank)' '
+        flux queue stop &&
+        flux module load sched-simple mode=unlimited &&
+        flux module debug --setbit 0x2 sched-simple &&
+        flux queue start
+'
+
+test_expect_success 'job-manager: job state RRSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs (RRSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "2"
+'
+
+test_expect_success 'job-manager: annotate jobs in job-list (RRSSS)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jlist_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.jobs_ahead" "1" &&
+        jlist_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job5.id) "sched.jobs_ahead" "2"
+'
+
+test_expect_success 'job-manager: annotate jobs in flux-jobs (RRSSS)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.jobs_ahead" "0" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.jobs_ahead" "1" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.jobs_ahead" "2"
+'
+
+test_expect_success 'job-manager: annotate jobs (RRSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "2"
+'
+
+test_expect_success 'job-manager: annotate jobs in job-list (RRSSS)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jlist_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.jobs_ahead" "1" &&
+        jlist_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job5.id) "sched.jobs_ahead" "2"
+'
+
+test_expect_success 'job-manager: annotate jobs in flux-jobs (RRSSS)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.jobs_ahead" "0" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.jobs_ahead" "1" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.jobs_ahead" "2"
+'
+
+test_expect_success 'job-manager: cancel 2' '
+        flux cancel $(cat job2.id)
+'
+
+test_expect_success 'job-manager: job state RIRSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs (RIRSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "1"
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in job-list (RIRSS)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jlist_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jlist_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jlist_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jlist_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jlist_check_annotation $(cat job5.id) "sched.jobs_ahead" "1"
+'
+
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-list
+test_expect_success 'job-manager: annotate jobs in flux-jobs (RIRSS)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        test_must_fail fjobs_check_annotation_exists $(cat job3.id) "annotations.sched.reason_pending" &&
+        test_must_fail fjobs_check_annotation_exists $(cat job3.id) "annotations.sched.jobs_ahead" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job4.id) "annotations.sched.jobs_ahead" "0" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.reason_pending" "insufficient resources" &&
+        fjobs_check_annotation $(cat job5.id) "annotations.sched.jobs_ahead" "1"
+'
+
+# cancel non-running jobs first, to ensure they are not accidentally run when
+# running jobs free resources.
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all --states=pending &&
+        flux cancel --all
+'
+
+test_expect_success 'job-manager: job state IIIII' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) I &&
+        jmgr_check_state $(cat job4.id) I &&
+        jmgr_check_state $(cat job5.id) I
+'
+
+test_expect_success 'job-manager: no annotations (IIIII)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, note that job ids that ran retain annotations
+test_expect_success 'job-manager: annotation jobs in job-list (IIIII)' '
+        jlist_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jlist_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jlist_check_no_annotations $(cat job4.id) &&
+        jlist_check_no_annotations $(cat job5.id)
+'
+
+# compared to above, note that job ids that ran retain annotations
+test_expect_success 'job-manager: annotate jobs in job-list (IIIII)' '
+        fjobs_check_annotation $(cat job1.id) "annotations.sched.resource_summary" "rank0/core0" &&
+        fjobs_check_annotation $(cat job2.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_annotation $(cat job3.id) "annotations.sched.resource_summary" "rank0/core1" &&
+        fjobs_check_no_annotations $(cat job4.id) &&
+        fjobs_check_no_annotations $(cat job5.id)
+'
+
+test_done
diff --git a/t/t2206-job-manager-bulk-state.t b/t/t2206-job-manager-bulk-state.t
deleted file mode 100755
index ec6d3991abe8..000000000000
--- a/t/t2206-job-manager-bulk-state.t
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/sh
-
-test_description='Test flux job manager state notification'
-
-. $(dirname $0)/sharness.sh
-
-test_under_flux 4
-
-flux setattr log-stderr-level 1
-
-BULK_STATE="flux python ${FLUX_SOURCE_DIR}/t/job-manager/bulk-state.py"
-
-test_expect_success 'job-manager: all state events received (5 jobs from rank 0)' '
-	run_timeout 10 ${BULK_STATE} 5
-'
-
-test_expect_success 'job-manager: all state events received (2 jobs from 4 ranks)' '
-	run_timeout 10 flux exec -r all ${BULK_STATE} 2
-'
-
-test_done
diff --git a/t/t2207-job-manager-wait.t b/t/t2207-job-manager-wait.t
deleted file mode 100755
index cd21e09d48cd..000000000000
--- a/t/t2207-job-manager-wait.t
+++ /dev/null
@@ -1,206 +0,0 @@
-#!/bin/sh
-
-test_description='Test flux job manager wait handling'
-
-. $(dirname $0)/sharness.sh
-
-test_under_flux 1
-
-list_jobs=${FLUX_BUILD_DIR}/t/job-manager/list-jobs
-SUBMIT_WAIT="flux python ${FLUX_SOURCE_DIR}/t/job-manager/submit-wait.py"
-SUBMIT_WAITANY="flux python ${FLUX_SOURCE_DIR}/t/job-manager/submit-waitany.py"
-SUBMIT_SW="flux python ${FLUX_SOURCE_DIR}/t/job-manager/submit-sliding-window.py"
-SUBMIT_INTER="flux python ${FLUX_SOURCE_DIR}/t/job-manager/wait-interrupted.py"
-
-flux setattr log-stderr-level 1
-
-test_job_count() {
-    test $(${list_jobs} | wc -l) -eq $1
-}
-
-test_expect_success "wait works on waitable job run with flux-mini" '
-	JOBID=$(flux mini submit --flags waitable /bin/true) &&
-	flux job wait ${JOBID}
-'
-
-test_expect_success "wait works on waitable job run with flux-job" '
-	flux mini submit --dry-run /bin/true >true.jobspec &&
-	JOBID=$(flux job submit --flags=waitable true.jobspec) &&
-	flux job wait ${JOBID}
-'
-
-test_expect_success "wait works on inactive,waitable job" '
-	JOBID=$(flux mini submit --flags waitable /bin/true) &&
-	flux job wait-event ${JOBID} clean &&
-	flux job wait ${JOBID}
-'
-
-test_expect_success "waitable inactive jobs are listed as zombies" '
-	JOBID=$(flux mini submit --flags waitable /bin/true) &&
-	echo ${JOBID} >id1.out &&
-	flux job wait-event ${JOBID} clean &&
-	${list_jobs} >list1.out &&
-	test $(wc -l <list1.out) -eq 1 &&
-	test "$(cut -f2 <list1.out)" = "I"
-'
-
-test_expect_success "zombies go away after they are waited for" '
-	flux job wait $(cat id1.out) &&
-	${list_jobs} >list2.out &&
-	test $(wc -l <list2.out) -eq 0
-'
-
-test_expect_success "wait works on three waitable jobs in reverse order" '
-	JOB1=$(flux mini submit --flags waitable /bin/true) &&
-	JOB2=$(flux mini submit --flags waitable /bin/true) &&
-	JOB3=$(flux mini submit --flags waitable /bin/true) &&
-	flux job wait ${JOB3} &&
-	flux job wait ${JOB2} &&
-	flux job wait ${JOB1}
-'
-
-test_expect_success "wait FLUX_JOBID_ANY works on three waitable jobs" '
-	flux mini submit --flags waitable /bin/true >jobs.out &&
-	flux mini submit --flags waitable /bin/true >>jobs.out &&
-	flux mini submit --flags waitable /bin/true >>jobs.out &&
-	flux job wait >wait.out &&
-	flux job wait >>wait.out &&
-	flux job wait >>wait.out &&
-	sort -n jobs.out >jobs_s.out &&
-	sort -n wait.out >wait_s.out &&
-	test_cmp jobs_s.out wait_s.out
-'
-
-test_expect_success 'python submit-wait example works' '
-        ${SUBMIT_WAIT} >submit_wait.out &&
-        test $(grep Success submit_wait.out | wc -l) -eq 5
-'
-
-test_expect_success 'python submit-waitany example works' '
-        ${SUBMIT_WAITANY} >submit_waitany.out &&
-        test $(grep Success submit_waitany.out | wc -l) -eq 5
-'
-
-test_expect_success 'python submit-slidiing-window example works' '
-        ${SUBMIT_SW} >submit_sw.out &&
-        test $(grep Success submit_sw.out | wc -l) -eq 10
-'
-
-test_expect_success 'disconnect with async waits pending' '
-        ${SUBMIT_INTER} &&
-	count=0 &&
-	echo cleaning up zombies... &&
-	while flux job wait; do count=$(($count+1)); done &&
-	echo ...reaped $count of 10 zombies
-'
-
-test_expect_success "wait FLUX_JOBID_ANY fails with no waitable jobs" '
-	test_must_fail flux job wait
-'
-
-test_expect_success "wait works when job terminated by exception" '
-	JOBID=$(flux mini submit --flags waitable sleep 120) &&
-	flux job raise --severity=0 ${JOBID} my-exception-message &&
-	! flux job wait ${JOBID} 2>exception.out &&
-	grep my-exception-message exception.out
-'
-
-test_expect_success "wait works when job tasks exit 1" '
-	JOBID=$(flux mini submit --flags waitable /bin/false) &&
-	! flux job wait ${JOBID} 2>false.out &&
-	grep exit false.out
-'
-
-test_expect_success "wait works when job tasks exit 1" '
-	JOBID=$(flux mini submit --flags waitable /bin/false) &&
-	! flux job wait ${JOBID} 2>false.out &&
-	grep exit false.out
-'
-
-test_expect_success "wait --all fails with jobid" '
-	test_must_fail flux job wait --all 42
-'
-
-test_expect_success "wait --all works with no waitable jobs" '
-	test_job_count 0 &&
-	flux job wait --all
-'
-
-test_expect_success "wait --all works with one job" '
-	flux mini submit --flags waitable /bin/true &&
-	test_job_count 1 &&
-	flux job wait --all &&
-	test_job_count 0
-'
-
-test_expect_success "wait --all works with two jobs" '
-	flux mini submit --flags waitable /bin/true &&
-	flux mini submit --flags waitable /bin/true &&
-	test_job_count 2 &&
-	flux job wait --all &&
-	test_job_count 0
-'
-
-test_expect_success "wait --all fails when first job fails" '
-	flux mini submit --flags waitable /bin/false &&
-	flux mini submit --flags waitable /bin/true &&
-	test_job_count 2 &&
-	test_must_fail flux job wait --all &&
-	test_job_count 0
-'
-
-test_expect_success "wait --all fails when second job fails" '
-	flux mini submit --flags waitable /bin/true &&
-	flux mini submit --flags waitable /bin/false &&
-	test_job_count 2 &&
-	test_must_fail flux job wait --all &&
-	test_job_count 0
-'
-
-
-test_expect_success "wait --all --verbose emits one line per succesful job" '
-	flux mini submit --flags waitable /bin/true &&
-	flux mini submit --flags waitable /bin/true &&
-	flux mini submit --flags waitable /bin/false &&
-	test_must_fail flux job wait --all --verbose 2>verbose.err &&
-	test $(wc -l <verbose.err) -eq 3
-'
-
-test_expect_success "wait fails on bad jobid, " '
-	test_must_fail flux job wait 1
-'
-
-test_expect_success "wait fails on non-waitable, active job" '
-	JOBID=$(flux mini submit sleep 0.5) &&
-	test_must_fail flux job wait ${JOBID}
-'
-
-test_expect_success "wait fails on non-waitable, inactive job" '
-	JOBID=$(flux mini submit /bin/true) &&
-	flux job wait-event ${JOBID} clean &&
-	test_must_fail flux job wait ${JOBID}
-'
-
-test_expect_success "a second wait fails on waitable, active job" '
-	JOBID=$(flux mini submit --flags waitable sleep 0.5) &&
-	flux job wait ${JOBID} &&
-	test_must_fail flux job wait ${JOBID}
-'
-
-test_expect_success "a second wait fails on waitable, inactive job" '
-	JOBID=$(flux mini submit --flags waitable /bin/true) &&
-	flux job wait-event ${JOBID} clean &&
-	flux job wait ${JOBID} &&
-	test_must_fail flux job wait ${JOBID}
-'
-
-test_expect_success "guest cannot submit job with WAITABLE flag" '
-	! FLUX_HANDLE_ROLEMASK=0x2 flux mini submit --flags waitable /bin/true
-'
-
-test_expect_success "guest cannot wait on a job" '
-	JOBID=$(flux mini submit --flags waitable /bin/true) &&
-	! FLUX_HANDLE_ROLEMASK=0x2 flux job wait ${JOBID}
-'
-
-test_done
diff --git a/t/t2208-job-manager-wait.t b/t/t2208-job-manager-wait.t
new file mode 100755
index 000000000000..795f1308242e
--- /dev/null
+++ b/t/t2208-job-manager-wait.t
@@ -0,0 +1,216 @@
+#!/bin/sh
+
+test_description='Test flux job manager wait handling'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+list_jobs=${FLUX_BUILD_DIR}/t/job-manager/list-jobs
+SUBMIT_WAIT="flux python ${FLUX_SOURCE_DIR}/t/job-manager/submit-wait.py"
+SUBMIT_WAITANY="flux python ${FLUX_SOURCE_DIR}/t/job-manager/submit-waitany.py"
+SUBMIT_SW="flux python ${FLUX_SOURCE_DIR}/t/job-manager/submit-sliding-window.py"
+SUBMIT_INTER="flux python ${FLUX_SOURCE_DIR}/t/job-manager/wait-interrupted.py"
+JOB_CONV="flux python ${FLUX_SOURCE_DIR}/t/job-manager/job-conv.py"
+PRINT_CONSTANTS="${FLUX_BUILD_DIR}/t/job-manager/print-constants"
+
+flux setattr log-stderr-level 1
+
+test_job_count() {
+    test $(${list_jobs} | wc -l) -eq $1
+}
+
+print_jobid_any_py() {
+	flux python -c 'import flux.constants; print(f"{flux.constants.FLUX_JOBID_ANY:x}")'
+}
+
+test_expect_success "python FLUX_JOBID_ANY matches job.h" '
+	py_id=$(print_jobid_any_py) &&
+	c_id=$($PRINT_CONSTANTS FLUX_JOBID_ANY) &&
+	test $py_id = $c_id
+'
+
+test_expect_success "wait works on waitable job run with flux submit" '
+	JOBID=$(flux submit --flags waitable true) &&
+	flux job wait ${JOBID}
+'
+
+test_expect_success "wait works on waitable job run with flux-job" '
+	flux submit --dry-run true >true.jobspec &&
+	JOBID=$(flux job submit --flags=waitable true.jobspec) &&
+	flux job wait ${JOBID}
+'
+
+test_expect_success "wait works on inactive,waitable job" '
+	JOBID=$(flux submit --flags waitable true) &&
+	flux job wait-event ${JOBID} clean &&
+	flux job wait ${JOBID}
+'
+
+test_expect_success "waitable inactive jobs are listed as zombies" '
+	JOBID=$(flux submit --flags waitable true) &&
+	echo ${JOBID} >id1.out &&
+	flux job wait-event ${JOBID} clean &&
+	${list_jobs} >list1.out &&
+	test $(wc -l <list1.out) -eq 1 &&
+	test "$($jq .state <list1.out | ${JOB_CONV} statetostr)" = "INACTIVE"
+'
+
+test_expect_success "zombies go away after they are waited for" '
+	flux job wait $(cat id1.out) &&
+	${list_jobs} >list2.out &&
+	test $(wc -l <list2.out) -eq 0
+'
+
+test_expect_success "wait works on three waitable jobs in reverse order" '
+	JOB1=$(flux submit --flags waitable true) &&
+	JOB2=$(flux submit --flags waitable true) &&
+	JOB3=$(flux submit --flags waitable true) &&
+	flux job wait ${JOB3} &&
+	flux job wait ${JOB2} &&
+	flux job wait ${JOB1}
+'
+
+test_expect_success "wait FLUX_JOBID_ANY works on three waitable jobs" '
+	flux submit --flags waitable true >jobs.out &&
+	flux submit --flags waitable true >>jobs.out &&
+	flux submit --flags waitable true >>jobs.out &&
+	flux job wait >wait.out &&
+	flux job wait >>wait.out &&
+	flux job wait >>wait.out &&
+	sort -n jobs.out >jobs_s.out &&
+	sort -n wait.out >wait_s.out &&
+	test_cmp jobs_s.out wait_s.out
+'
+
+test_expect_success 'python submit-wait example works' '
+        ${SUBMIT_WAIT} >submit_wait.out &&
+        test $(grep Success submit_wait.out | wc -l) -eq 5
+'
+
+test_expect_success 'python submit-waitany example works' '
+        ${SUBMIT_WAITANY} >submit_waitany.out &&
+        test $(grep Success submit_waitany.out | wc -l) -eq 5
+'
+
+test_expect_success 'python submit-slidiing-window example works' '
+        ${SUBMIT_SW} >submit_sw.out &&
+        test $(grep Success submit_sw.out | wc -l) -eq 10
+'
+
+test_expect_success 'disconnect with async waits pending' '
+        ${SUBMIT_INTER} &&
+	count=0 &&
+	echo cleaning up zombies... &&
+	while flux job wait; do count=$(($count+1)); done &&
+	echo ...reaped $count of 10 zombies
+'
+
+test_expect_success "wait FLUX_JOBID_ANY exits 2 with no waitable jobs" '
+	test_expect_code 2 flux job wait
+'
+
+test_expect_success "wait works when job terminated by exception" '
+	JOBID=$(flux submit --flags waitable sleep 120) &&
+	flux job raise --severity=0 ${JOBID} my-exception-message &&
+	test_expect_code 1 flux job wait ${JOBID} 2>exception.out &&
+	grep my-exception-message exception.out
+'
+
+test_expect_success "wait works when job tasks exit 1" '
+	JOBID=$(flux submit --flags waitable false) &&
+	test_expect_code 1 flux job wait ${JOBID} 2>false.out &&
+	grep exit false.out
+'
+
+test_expect_success "wait --all fails with jobid" '
+	test_must_fail flux job wait --all 42
+'
+
+test_expect_success "wait --all works with no waitable jobs" '
+	test_job_count 0 &&
+	flux job wait --all
+'
+
+test_expect_success "wait --all works with one job" '
+	flux submit --flags waitable true &&
+	test_job_count 1 &&
+	flux job wait --all &&
+	test_job_count 0
+'
+
+test_expect_success "wait --all works with two jobs" '
+	flux submit --flags waitable true &&
+	flux submit --flags waitable true &&
+	test_job_count 2 &&
+	flux job wait --all &&
+	test_job_count 0
+'
+
+test_expect_success "wait --all fails when first job fails" '
+	flux submit --flags waitable false &&
+	flux submit --flags waitable true &&
+	test_job_count 2 &&
+	test_must_fail flux job wait --all &&
+	test_job_count 0
+'
+
+test_expect_success "wait --all fails when second job fails" '
+	flux submit --flags waitable true &&
+	flux submit --flags waitable false &&
+	test_job_count 2 &&
+	test_must_fail flux job wait --all &&
+	test_job_count 0
+'
+
+
+test_expect_success "wait --all --verbose emits one line per successful job" '
+	flux submit --flags waitable true &&
+	flux submit --flags waitable true &&
+	flux submit --flags waitable false &&
+	test_must_fail flux job wait --all --verbose 2>verbose.err &&
+	test $(wc -l <verbose.err) -eq 3
+'
+
+test_expect_success "wait fails on bad jobid, " '
+	test_expect_code 2 flux job wait 1
+'
+
+test_expect_success "wait fails on non-waitable, active job" '
+	JOBID=$(flux submit sleep 0.5) &&
+	test_expect_code 2 flux job wait ${JOBID}
+'
+
+test_expect_success "wait fails on non-waitable, inactive job" '
+	JOBID=$(flux submit true) &&
+	flux job wait-event ${JOBID} clean &&
+	test_expect_code 2 flux job wait ${JOBID}
+'
+
+test_expect_success "a second wait fails on waitable, active job" '
+	JOBID=$(flux submit --flags waitable sleep 0.5) &&
+	flux job wait ${JOBID} &&
+	test_expect_code 2 flux job wait ${JOBID}
+'
+
+test_expect_success "a second wait fails on waitable, inactive job" '
+	JOBID=$(flux submit --flags waitable true) &&
+	flux job wait-event ${JOBID} clean &&
+	flux job wait ${JOBID} &&
+	test_expect_code 2 flux job wait ${JOBID}
+'
+
+test_expect_success "guest cannot submit job with WAITABLE flag" '
+	export FLUX_HANDLE_ROLEMASK=0x2 &&
+	test_must_fail flux submit --flags waitable true &&
+	unset FLUX_HANDLE_ROLEMASK
+'
+
+test_expect_success "guest cannot wait on a job" '
+	JOBID=$(flux submit --flags waitable true) &&
+	export FLUX_HANDLE_ROLEMASK=0x2 &&
+	test_expect_code 1 flux job wait ${JOBID} &&
+	unset FLUX_HANDLE_ROLEMASK
+'
+
+test_done
diff --git a/t/t2208-queue-cmd.t b/t/t2208-queue-cmd.t
deleted file mode 100755
index 31c4749c692a..000000000000
--- a/t/t2208-queue-cmd.t
+++ /dev/null
@@ -1,297 +0,0 @@
-#!/bin/sh
-test_description='Test flux queue command'
-
-. $(dirname $0)/sharness.sh
-
-test_under_flux 1
-
-flux setattr log-stderr-level 1
-
-LIST_JOBS=${FLUX_BUILD_DIR}/t/job-manager/list-jobs
-
-test_expect_success 'flux-queue: unknown sub-command fails with usage message' '
-	test_must_fail flux queue wrongsubcmd 2>usage.out &&
-	grep Usage: usage.out
-'
-
-test_expect_success 'flux-queue: missing sub-command fails with usage message' '
-	test_must_fail flux queue 2>usage2.out &&
-	grep Usage: usage2.out
-'
-
-test_expect_success 'flux-queue: disable with no reason fails' '
-	test_must_fail flux queue submit disable 2>usage3.out &&
-	grep Usage: usage3.out
-'
-
-test_expect_success 'flux-queue: enable with extra free args fails' '
-	test_must_fail flux queue enable xyz 2>usage4.out &&
-	grep Usage: usage4.out
-'
-
-test_expect_success 'flux-queue: status with extra free args fails' '
-	test_must_fail flux queue status xyz 2>usage5.out &&
-	grep Usage: usage5.out
-'
-
-test_expect_success 'flux-queue: drain with extra free args fails' '
-	test_must_fail flux queue drain xyz 2>usage6.out &&
-	grep Usage: usage6.out
-'
-
-test_expect_success 'flux-queue: idle with extra free args fails' '
-	test_must_fail flux queue idle xyz 2>usage7.out &&
-	grep Usage: usage7.out
-'
-
-test_expect_success 'flux-queue: status with bad broker connection fails' '
-	! FLUX_URI=/wrong flux queue status
-'
-
-test_expect_success 'flux-queue: disable with bad broker connection fails' '
-	! FLUX_URI=/wrong flux queue disable foo
-'
-
-test_expect_success 'flux-queue: enable with bad broker connection fails' '
-	! FLUX_URI=/wrong flux queue enable
-'
-
-test_expect_success 'flux-queue: drain with bad broker connection fails' '
-	! FLUX_URI=/wrong flux queue drain
-'
-
-test_expect_success 'flux-queue: idle with bad broker connection fails' '
-	! FLUX_URI=/wrong flux queue idle
-'
-
-test_expect_success 'flux-queue: disable works' '
-        flux queue disable system is fubar
-'
-
-test_expect_success 'flux-queue: job submit fails with queue disabled' '
-        test_must_fail flux mini run /bin/true
-'
-
-test_expect_success 'flux-queue: enable works' '
-        flux queue enable
-'
-
-test_expect_success 'flux-queue: flux mini run works after enable' '
-        run_timeout 60 flux mini run /bin/true
-'
-
-test_expect_success 'flux-queue: stop with bad broker connection fails' '
-	! FLUX_URI=/wrong flux queue stop
-'
-
-test_expect_success 'flux-queue: start with bad broker connection fails' '
-	! FLUX_URI=/wrong flux queue start
-'
-
-test_expect_success 'flux-queue: start with extra free args fails' '
-	test_must_fail flux queue start xyz 2>start_xargs.out &&
-	grep Usage: start_xargs.out
-'
-
-test_expect_success 'flux-queue: stop works' '
-	flux queue stop my unique message
-'
-
-test_expect_success 'flux-queue: status reports reason for stop' '
-	flux queue status 2>status.out &&
-	cat <<-EOT >status.exp &&
-	flux-queue: Job submission is enabled
-	flux-queue: Scheduling is disabled: my unique message
-	EOT
-	test_cmp status.exp status.out
-'
-
-test_expect_success 'flux-queue: submit 3 jobs' '
-	for i in $(seq 1 3); do flux mini submit /bin/true; done
-'
-
-test_expect_success 'flux-queue: start scheduling' '
-	flux queue start
-'
-
-test_expect_success 'flux-queue: queue empties out' '
-	run_timeout 60 flux queue drain
-'
-
-test_expect_success 'flux-queue: start long job that uses all cores' '
-	id=$(flux mini submit -n $(nproc) sleep 600) &&
-	flux job wait-event ${id} start &&
-	echo ${id} >longjob
-'
-
-test_expect_success 'flux-queue: submit a job and make sure alloc sent' '
-	id=$(flux mini submit --flags debug /bin/true) &&
-	flux job wait-event ${id} debug.alloc-request
-'
-
-test_expect_success 'flux-queue: stop canceled alloc request' '
-	flux queue stop -v 2>stop.err &&
-	grep "flux-queue: 1 alloc requests pending to scheduler" stop.err
-'
-
-test_expect_success 'flux-queue: start scheduling and cancel long job' '
-	flux queue start &&
-	flux job cancel $(cat longjob)
-'
-
-test_expect_success 'flux-queue: queue empties out' '
-	flux queue drain
-'
-
-test_expect_success 'flux-queue: unload scheduler' '
-	flux module remove sched-simple
-'
-
-test_expect_success 'flux-queue: submit a job to ping scheduler' '
-	flux mini submit --flags debug /bin/true
-'
-
-wait_for_sched_offline() {
-	local n=$1
-	for try in $(seq 1 $n); do
-		echo Check queue status for offline, try ${try} of $n 2>&1
-		flux queue status 2>&1 | grep disabled && return
-	done
-}
-
-test_expect_success 'flux-queue: queue says scheduling disabled' '
-	wait_for_sched_offline 10 &&
-	flux queue status 2>sched_stat.err &&
-	cat <<-EOT >sched_stat.exp &&
-	flux-queue: Job submission is enabled
-	flux-queue: Scheduling is disabled: Scheduler is offline
-	EOT
-	test_cmp sched_stat.exp sched_stat.err
-'
-
-test_expect_success 'flux-queue: queue contains 1 active job' '
-	COUNT=$(${LIST_JOBS} | wc -l) &&
-	test ${COUNT} -eq 1
-'
-
-test_expect_success 'flux-queue: load scheduler' '
-	flux module load sched-simple
-'
-
-test_expect_success 'flux-queue: queue says scheduling is enabled' '
-	flux queue status 2>sched_stat2.err &&
-	cat <<-EOT >sched_stat2.exp &&
-	flux-queue: Job submission is enabled
-	flux-queue: Scheduling is enabled
-	EOT
-	test_cmp sched_stat2.exp sched_stat2.err
-'
-
-test_expect_success 'flux-queue: job in queue ran' '
-	run_timeout 30 flux queue drain
-'
-
-test_expect_success 'flux-queue: submit a long job that uses all cores' '
-	flux mini submit -n $(nproc) sleep 600
-'
-
-test_expect_success 'flux-queue: submit 2 more jobs' '
-	flux mini submit /bin/true &&
-	flux mini submit /bin/true
-'
-
-test_expect_success 'flux-queue: there are 3 active jobs' '
-	COUNT=$(${LIST_JOBS} | wc -l) &&
-	test ${COUNT} -eq 3
-'
-
-test_expect_success 'flux-queue: queue status -v shows expected counts' '
-	flux queue status -v 2>stat.err &&
-	cat <<-EOT >stat.exp &&
-	flux-queue: Job submission is enabled
-	flux-queue: Scheduling is enabled
-	flux-queue: 1 alloc requests queued
-	flux-queue: 1 alloc requests pending to scheduler
-	flux-queue: 0 free requests pending to scheduler
-	flux-queue: 1 running jobs
-	EOT
-	test_cmp stat.exp stat.err
-'
-
-test_expect_success 'flux-queue: stop queue and cancel long job' '
-	flux queue stop &&
-	flux job cancelall -f -S RUN
-'
-
-test_expect_success 'flux-queue: queue becomes idle' '
-	run_timeout 30 flux queue idle
-'
-
-test_expect_success 'flux-queue: queue status -v shows expected counts' '
-	flux queue status -v 2>stat2.err &&
-	cat <<-EOT >stat2.exp &&
-	flux-queue: Job submission is enabled
-	flux-queue: Scheduling is disabled
-	flux-queue: 2 alloc requests queued
-	flux-queue: 0 alloc requests pending to scheduler
-	flux-queue: 0 free requests pending to scheduler
-	flux-queue: 0 running jobs
-	EOT
-	test_cmp stat2.exp stat2.err
-'
-
-test_expect_success 'flux-queue: start queue and drain' '
-	flux queue start &&
-	run_timeout 30 flux queue drain
-'
-
-
-runas_guest() {
-        local userid=$(($(id -u)+1))
-        FLUX_HANDLE_USERID=$userid FLUX_HANDLE_ROLEMASK=0x2 "$@"
-}
-
-test_expect_success 'flux-queue: stop denied for guest' '
-	test_must_fail runas_guest flux queue stop 2>guest_stop.err &&
-	cat <<-EOT >guest_alloc.exp &&
-	flux-queue: alloc-admin: Request requires owner credentals
-	EOT
-	test_cmp guest_alloc.exp guest_stop.err
-'
-
-test_expect_success 'flux-queue: start denied for guest' '
-	test_must_fail runas_guest flux queue start 2>guest_start.err &&
-	test_cmp guest_alloc.exp guest_start.err
-'
-
-test_expect_success 'flux-queue: disable denied for guest' '
-	test_must_fail runas_guest flux queue disable foo 2>guest_dis.err &&
-	cat <<-EOT >guest_submit.exp &&
-	flux-queue: submit-admin: Request requires owner credentals
-	EOT
-	test_cmp guest_submit.exp guest_dis.err
-'
-
-test_expect_success 'flux-queue: enable denied for guest' '
-	test_must_fail runas_guest flux queue enable 2>guest_ena.err &&
-	test_cmp guest_submit.exp guest_ena.err
-'
-
-test_expect_success 'flux-queue: drain denied for guest' '
-	test_must_fail runas_guest flux queue drain 2>guest_drain.err &&
-	cat <<-EOT >guest_drain.exp &&
-	flux-queue: drain: Request requires owner credentals
-	EOT
-	test_cmp guest_drain.exp guest_drain.err
-'
-
-test_expect_success 'flux-queue: idle denied for guest' '
-	test_must_fail runas_guest flux queue idle 2>guest_idle.err &&
-	cat <<-EOT >guest_idle.exp &&
-	flux-queue: idle: Request requires owner credentals
-	EOT
-	test_cmp guest_idle.exp guest_idle.err
-'
-
-
-test_done
diff --git a/t/t2209-job-manager-bugs.t b/t/t2209-job-manager-bugs.t
new file mode 100755
index 000000000000..eae969089efa
--- /dev/null
+++ b/t/t2209-job-manager-bugs.t
@@ -0,0 +1,63 @@
+#!/bin/sh
+
+
+test_description='Regression tests for job-manager bugs'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+flux setattr log-stderr-level 1
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+
+#
+# Issue 2664 job-manager: counting error
+#
+
+test_expect_success 'issue2664: start three jobs ' '
+	ncores=$(flux resource list -s up -no {ncores}) &&
+	echo Requesting ${ncores} cores &&
+	flux submit -n ${ncores} sleep 3600 >job1.out &&
+	flux submit hostname >job2.out &&
+	flux submit hostname >job3.out
+'
+# canceling job that has not yet sent alloc to scheduler improperly
+# decrements count of outstanding alloc requests
+test_expect_success 'issue2664: cancel job 3' '
+	flux cancel $(cat job3.out)
+'
+# Next job submitted triggers another alloc request when there are no
+# more slots, which triggers error response from scheduler that is fatal
+# to job manager.
+test_expect_success 'issue2664: submit job 4' '
+	flux submit hostname >job4.out
+'
+# Hangs here (hitting timeout) when bug is present
+test_expect_success 'issue2664: cancel job 1 and drain (cleanup)' '
+	flux cancel $(cat job1.out) &&
+	run_timeout 5 flux queue drain
+'
+
+#
+# Issue 3218 job-manager: segfault when changing urgency of running job
+#
+
+test_expect_success 'issue3218: urgency change on running job doesnt segfault' '
+        id=$(flux submit sleep 600) &&
+        flux job wait-event $id start &&
+        test_must_fail flux job urgency $id 0 &&
+        flux cancel $id
+'
+#
+# Issue 4409: eventlog commit / job start race
+# Also tests job-manager.set-batch-timeout RPC
+#
+test_expect_success 'issue4409: eventlog commit races with job launch' '
+	printf "{\"timeout\": \"1\"}" | \
+	    test_expect_code 1 ${RPC} job-manager.set-batch-timeout &&
+	printf "{\"timeout\": 1}" | ${RPC} job-manager.set-batch-timeout &&
+	flux submit -vvv --cc=1-5 --wait --quiet hostname &&
+	printf "{\"timeout\": 0.01}" | ${RPC} job-manager.set-batch-timeout
+'
+test_done
diff --git a/t/t2210-job-manager-bugs.t b/t/t2210-job-manager-bugs.t
deleted file mode 100755
index ef39f8997096..000000000000
--- a/t/t2210-job-manager-bugs.t
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/sh
-
-test_description='Regression tests for job-manager bugs'
-
-. $(dirname $0)/sharness.sh
-
-test_under_flux 1
-
-flux setattr log-stderr-level 1
-
-#
-# Issue 2664 job-manager: counting error
-#
-
-test_expect_success 'issue2664: start three jobs ' '
-	echo Requesting $(nproc) cores &&
-	flux mini submit -n $(nproc) sleep 3600 >job1.out &&
-	flux mini submit hostname >job2.out &&
-	flux mini submit hostname >job3.out
-'
-# canceling job that has not yet sent alloc to scheduler improperly
-# decrements count of outstanding alloc requests
-test_expect_success 'issue2664: cancel job 3' '
-	flux job cancel $(cat job3.out)
-'
-# Next job submitted triggers another alloc request when ther are no
-# more slots, which triggers error response from scheduler that is fatal
-# to job manager.
-test_expect_success 'issue2664: submit job 4' '
-	flux mini submit hostname >job4.out
-'
-# Hangs here (hitting timeout)
-test_expect_success 'issue2664: cancel job 1 and drain (cleanup)' '
-	flux job cancel $(cat job1.out) &&
-	run_timeout 5 flux queue drain
-'
-
-test_done
diff --git a/t/t2210-job-manager-events-journal.t b/t/t2210-job-manager-events-journal.t
new file mode 100755
index 000000000000..1c40985465b4
--- /dev/null
+++ b/t/t2210-job-manager-events-journal.t
@@ -0,0 +1,251 @@
+#!/bin/sh
+
+test_description='Test flux job manager journal service'
+
+. $(dirname $0)/sharness.sh
+
+export FLUX_CONF_DIR=$(pwd)
+
+test_under_flux 4
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+EVENTS_JOURNAL_STREAM=${FLUX_BUILD_DIR}/t/job-manager/events_journal_stream
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'flux-job: generate jobspec for simple test job' '
+	flux run --dry-run -n1 hostname >basic.json
+'
+
+check_event() {
+	jobid="$1"
+	key=$2
+	value=$3
+	filename=$4
+	if cat $filename \
+		| $jq -e ".id == ${jobid} and ${key} == ${value}" \
+		| grep -q "true"
+	then
+		return 0
+	fi
+	return 1
+}
+
+check_event_name() {
+	jobid="$1"
+	name=$2
+	filename=$3
+	check_event ${jobid} .entry.name \"${name}\" ${filename}
+}
+
+check_event_annotation() {
+	jobid="$1"
+	key=$2
+	value=$3
+	filename=$4
+	check_event ${jobid} .entry.context.annotations.user.${key} ${value} ${filename}
+}
+
+wait_event() {
+	jobid="$1"
+	key=$2
+	value=$3
+	filename=$4
+	i=0 &&
+	while [ $i -lt 50 ]
+	do
+		if cat $filename \
+			| $jq -e ".id == ${jobid} and ${key} == ${value}" \
+			| grep -q "true"
+		then
+			return 0
+		fi
+		sleep 0.1
+		i=$((i + 1))
+	done
+	return 1
+}
+
+wait_event_name() {
+	jobid="$1"
+	name=$2
+	filename=$3
+	wait_event ${jobid} .entry.name \"${name}\" ${filename}
+}
+
+wait_event_annotation() {
+	jobid="$1"
+	key=$2
+	value=$3
+	filename=$4
+	wait_event ${jobid} .entry.context.annotations.user.${key} ${value} ${filename}
+}
+
+test_expect_success NO_CHAIN_LINT 'job-manager: events-journal w/ no filters shows all events' '
+	$jq -j -c -n "{}" \
+		| $EVENTS_JOURNAL_STREAM > events1.out &
+	pid=$! &&
+	jobid=`flux job submit basic.json | flux job id` &&
+	wait_event_name ${jobid} clean events1.out &&
+	check_event_name ${jobid} submit events1.out &&
+	check_event_name ${jobid} depend events1.out &&
+	check_event_name ${jobid} alloc events1.out &&
+	check_event_name ${jobid} start events1.out &&
+	check_event_name ${jobid} finish events1.out &&
+	check_event_name ${jobid} release events1.out &&
+	check_event_name ${jobid} free events1.out &&
+	check_event_name ${jobid} clean events1.out &&
+	kill -s USR1 $pid &&
+	wait $pid
+'
+
+test_expect_success NO_CHAIN_LINT 'job-manager: events-journal allow works' '
+	$jq -j -c -n "{allow:{clean:1}}" \
+		| $EVENTS_JOURNAL_STREAM > events2.out &
+	pid=$! &&
+	jobid=`flux job submit basic.json | flux job id` &&
+	wait_event_name ${jobid} clean events2.out &&
+	test_must_fail check_event_name ${jobid} submit events2.out &&
+	test_must_fail check_event_name ${jobid} depend events2.out &&
+	test_must_fail check_event_name ${jobid} alloc events2.out &&
+	test_must_fail check_event_name ${jobid} start events2.out &&
+	test_must_fail check_event_name ${jobid} finish events2.out &&
+	test_must_fail check_event_name ${jobid} release events2.out &&
+	test_must_fail check_event_name ${jobid} free events2.out &&
+	check_event_name ${jobid} clean events2.out &&
+	kill -s USR1 $pid &&
+	wait $pid
+'
+
+test_expect_success NO_CHAIN_LINT 'job-manager: events-journal allow works (multiple)' '
+	$jq -j -c -n "{allow:{depend:1, finish:1, clean:1}}" \
+		| $EVENTS_JOURNAL_STREAM > events3.out &
+	pid=$! &&
+	jobid=`flux job submit basic.json | flux job id` &&
+	wait_event_name ${jobid} clean events3.out &&
+	test_must_fail check_event_name ${jobid} submit events3.out &&
+	check_event_name ${jobid} depend events3.out &&
+	test_must_fail check_event_name ${jobid} alloc events3.out &&
+	test_must_fail check_event_name ${jobid} start events3.out &&
+	check_event_name ${jobid} finish events3.out &&
+	test_must_fail check_event_name ${jobid} release events3.out &&
+	test_must_fail check_event_name ${jobid} free events3.out &&
+	check_event_name ${jobid} clean events3.out &&
+	kill -s USR1 $pid &&
+	wait $pid
+'
+
+test_expect_success NO_CHAIN_LINT 'job-manager: events-journal deny works' '
+	$jq -j -c -n "{deny:{finish:1}}" \
+		| $EVENTS_JOURNAL_STREAM > events4.out &
+	pid=$! &&
+	jobid=`flux job submit basic.json | flux job id` &&
+	wait_event_name ${jobid} clean events4.out &&
+	check_event_name ${jobid} submit events4.out &&
+	check_event_name ${jobid} depend events4.out &&
+	check_event_name ${jobid} alloc events4.out &&
+	check_event_name ${jobid} start events4.out &&
+	test_must_fail check_event_name ${jobid} finish events4.out &&
+	check_event_name ${jobid} release events4.out &&
+	check_event_name ${jobid} free events4.out &&
+	check_event_name ${jobid} clean events4.out &&
+	kill -s USR1 $pid &&
+	wait $pid
+'
+
+test_expect_success NO_CHAIN_LINT 'job-manager: events-journal deny works (multiple)' '
+	$jq -j -c -n "{deny:{depend:1, finish:1, release:1}}" \
+		| $EVENTS_JOURNAL_STREAM > events5.out &
+	pid=$! &&
+	jobid=`flux job submit basic.json | flux job id` &&
+	wait_event_name ${jobid} clean events5.out &&
+	check_event_name ${jobid} submit events5.out &&
+	test_must_fail check_event_name ${jobid} depend events5.out &&
+	check_event_name ${jobid} alloc events5.out &&
+	check_event_name ${jobid} start events5.out &&
+	test_must_fail check_event_name ${jobid} finish events5.out &&
+	test_must_fail check_event_name ${jobid} release events5.out &&
+	check_event_name ${jobid} free events5.out &&
+	check_event_name ${jobid} clean events5.out &&
+	kill -s USR1 $pid &&
+	wait $pid
+'
+
+test_expect_success NO_CHAIN_LINT 'job-manager: events-journal allow & deny works' '
+	$jq -j -c -n "{allow:{depend:1, finish:1, clean:1}, deny:{depend:1}}" \
+		| $EVENTS_JOURNAL_STREAM > events6.out &
+	pid=$! &&
+	jobid=`flux job submit basic.json | flux job id` &&
+	wait_event_name ${jobid} clean events6.out &&
+	test_must_fail check_event_name ${jobid} submit events6.out &&
+	test_must_fail check_event_name ${jobid} depend events6.out &&
+	test_must_fail check_event_name ${jobid} alloc events6.out &&
+	test_must_fail check_event_name ${jobid} start events6.out &&
+	check_event_name ${jobid} finish events6.out &&
+	test_must_fail check_event_name ${jobid} release events6.out &&
+	test_must_fail check_event_name ${jobid} free events6.out &&
+	check_event_name ${jobid} clean events6.out &&
+	kill -s USR1 $pid &&
+	wait $pid
+'
+
+test_expect_success NO_CHAIN_LINT 'job-manager: events-journal contains older jobs' '
+	jobid1=`flux job submit basic.json | flux job id`
+	jobid2=`flux job submit basic.json | flux job id`
+	flux job wait-event ${jobid1} clean
+	flux job wait-event ${jobid2} clean
+	$jq -j -c -n "{full:true, allow:{depend:1, clean:1}}" \
+		| $EVENTS_JOURNAL_STREAM > events7.out &
+	pid=$! &&
+	jobid3=`flux job submit basic.json | flux job id` &&
+	wait_event_name ${jobid3} clean events7.out &&
+	test_must_fail check_event_name ${jobid1} submit events7.out &&
+	check_event_name ${jobid1} depend events7.out &&
+	test_must_fail check_event_name ${jobid1} alloc events7.out &&
+	test_must_fail check_event_name ${jobid1} start events7.out &&
+	test_must_fail check_event_name ${jobid1} finish events7.out &&
+	test_must_fail check_event_name ${jobid1} release events7.out &&
+	test_must_fail check_event_name ${jobid1} free events7.out &&
+	check_event_name ${jobid1} clean events7.out &&
+	test_must_fail check_event_name ${jobid2} submit events7.out &&
+	check_event_name ${jobid2} depend events7.out &&
+	test_must_fail check_event_name ${jobid2} alloc events7.out &&
+	test_must_fail check_event_name ${jobid2} start events7.out &&
+	test_must_fail check_event_name ${jobid2} finish events7.out &&
+	test_must_fail check_event_name ${jobid2} release events7.out &&
+	test_must_fail check_event_name ${jobid2} free events7.out &&
+	check_event_name ${jobid2} clean events7.out &&
+	test_must_fail check_event_name ${jobid3} submit events7.out &&
+	check_event_name ${jobid3} depend events7.out &&
+	test_must_fail check_event_name ${jobid3} alloc events7.out &&
+	test_must_fail check_event_name ${jobid3} start events7.out &&
+	test_must_fail check_event_name ${jobid3} finish events7.out &&
+	test_must_fail check_event_name ${jobid3} release events7.out &&
+	test_must_fail check_event_name ${jobid3} free events7.out &&
+	check_event_name ${jobid3} clean events7.out &&
+	kill -s USR1 $pid &&
+	wait $pid
+'
+
+test_expect_success 'job-manager: events-journal request fails with EPROTO on empty payload' '
+	$RPC job-manager.events-journal 71 < /dev/null
+'
+
+test_expect_success 'job-manager: events-journal request fails if not streaming RPC' '
+	$jq -j -c -n "{}" > cc1.in &&
+	test_must_fail $RPC job-manager.events-journal < cc1.in
+'
+
+test_expect_success 'job-manager: events-journal request fails if allow not an object' '
+	$jq -j -c -n "{allow:5}" > cc2.in &&
+	test_must_fail $EVENTS_JOURNAL_STREAM < cc2.in 2> cc2.err &&
+	grep "allow should be an object" cc2.err
+'
+
+test_expect_success 'job-manager: events-journal request fails if deny not an object' '
+	$jq -j -c -n "{deny:5}" > cc3.in &&
+	test_must_fail $EVENTS_JOURNAL_STREAM < cc3.in 2> cc3.err &&
+	grep "deny should be an object" cc3.err
+'
+
+test_done
diff --git a/t/t2211-job-manager-jobspec.t b/t/t2211-job-manager-jobspec.t
new file mode 100755
index 000000000000..b61f9f8887da
--- /dev/null
+++ b/t/t2211-job-manager-jobspec.t
@@ -0,0 +1,70 @@
+#!/bin/sh
+test_description='Checkout job manager redacted jobspec'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+
+# Usage: job_manager_getattr ID ATTR
+job_manager_getattr() {
+	echo '{"id":'$1',"attrs":["'$2'"]}' | ${RPC} job-manager.getattr
+}
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'stop the queue for this test' '
+	flux queue stop
+'
+
+test_expect_success 'create simple jobspec' '
+	flux submit --dry-run hostname >simple.json
+'
+
+test_expect_success 'jobspec contains environment' '
+	jq -e .attributes.system.environment <simple.json >env.json
+'
+
+test_expect_success 'jobspec contains duration' '
+	jq -e .attributes.system.duration <simple.json
+'
+
+test_expect_success 'submit job' '
+	flux job submit simple.json | flux job id >jobid
+'
+
+test_expect_success 'job-manager getattr of unknown attr fails' '
+	test_must_fail job_manager_getattr $(cat jobid) noexist
+'
+
+test_expect_success 'job-manager getattr of jobspec works' '
+	job_manager_getattr $(cat jobid) jobspec >getattr.json &&
+	jq -e .jobspec <getattr.json >redacted.json
+'
+
+test_expect_success 'redacted jobspec contains duration' '
+	jq -e .attributes.system.duration <redacted.json
+'
+
+test_expect_success 'redacted jobspec does not contain environment' '
+	test_must_fail jq -e .attributes.system.environment <redacted.json
+'
+
+test_expect_success 'reload job manager and dependent modules' '
+	flux module remove sched-simple &&
+	flux module reload job-manager &&
+	flux module load sched-simple
+'
+
+test_expect_success 'pending job still exists with same jobspec' '
+	job_manager_getattr $(cat jobid) jobspec >getattr2.json &&
+	jq -e .jobspec <getattr2.json >redacted2.json &&
+	test_cmp redacted.json redacted2.json
+'
+
+test_expect_success 'restart the queue' '
+	flux queue start
+'
+
+test_done
diff --git a/t/t2212-job-manager-plugins.t b/t/t2212-job-manager-plugins.t
new file mode 100755
index 000000000000..c85874355765
--- /dev/null
+++ b/t/t2212-job-manager-plugins.t
@@ -0,0 +1,589 @@
+#!/bin/sh
+
+test_description='Test job manager jobtap plugin interface'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+mkdir -p config
+
+test_under_flux 4 job --config-path=$(pwd)/config
+
+flux setattr log-stderr-level 1
+
+PLUGINPATH=${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs
+
+id_byname() {
+	flux jobs -ano {name}:{id} | grep ^$1: | head -1 | cut -d: -f2
+}
+
+test_expect_success 'job-manager: attempt to load invalid plugin fails' '
+	flux jobtap list >list1.out &&
+	test_must_fail flux jobtap load builtin.foo &&
+	flux jobtap list >list2.out &&
+	test_debug "cat list2.out" &&
+	test_cmp list1.out list2.out
+'
+test_expect_success 'job-manager: loading invalid builtin plugin fails' '
+	test_must_fail flux jobtap load .foo
+'
+test_expect_success 'job-manager: builtin plugin can be removed' '
+	flux jobtap remove .history &&
+	flux jobtap list >list-nohist.out &&
+	test_must_fail grep ^\.history list-nohist.out &&
+	flux jobtap load .history
+'
+test_expect_success 'job-manager: load with invalid conf fails' '
+	cat <<-EOF >badconf.py &&
+	import flux
+	h = flux.Flux()
+	print(h.rpc("job-manager.jobtap", {"load": "none", "conf": 1}).get())
+	EOF
+	test_must_fail flux python badconf.py
+'
+test_expect_success 'job-manager: jobtap remove with invalid plugin fails' '
+	test_must_fail flux jobtap remove notfound.so
+'
+test_expect_success 'job-manager: jobtap remove all does not error ' '
+	flux jobtap remove all
+'
+test_expect_success 'job-manager: multiple plugins can be loaded' '
+	flux jobtap load ${PLUGINPATH}/args.so &&
+	flux jobtap load ${PLUGINPATH}/test.so &&
+	flux jobtap list > plugins &&
+	grep args plugins &&
+	grep test plugins &&
+	flux jobtap remove args* &&
+	flux jobtap remove test*
+'
+test_expect_success 'job-manager: loading duplicate plugins fails' '
+	flux jobtap load ${PLUGINPATH}/args.so &&
+	test_must_fail flux jobtap load ${PLUGINPATH}/args.so >dup.log 2>&1 &&
+	test_debug "cat dup.log" &&
+	grep "already loaded" dup.log &&
+	flux jobtap remove args.so
+'
+test_expect_success 'job-manager: query of plugin works' '
+	flux jobtap load ${PLUGINPATH}/test.so &&
+	flux jobtap query test.so >query.json &&
+	test_debug "jq -S . <query.json" &&
+	jq -e ".name == \"test.so\"" <query.json &&
+	jq -e ".path == \"${PLUGINPATH}/test.so\"" <query.json &&
+	flux jobtap remove test.so
+'
+test_expect_success 'job-manager: query of invalid plugin fals' '
+	test_must_fail flux jobtap query foo
+'
+test_expect_success 'job-manager: plugins can be loaded by configuration' '
+	mkdir testconf &&
+	cat <<-EOF >testconf/job-manager.toml &&
+	[job-manager]
+	plugins = [
+	  { remove = "all" },
+	  { load = "${PLUGINPATH}/test.so" },
+	  { load = "${PLUGINPATH}/args.so" },
+	]
+	EOF
+	flux start -c $(pwd)/testconf flux jobtap list > confplugins.out &&
+	test_debug "cat confplugins.out" &&
+	grep args confplugins.out &&
+	grep test confplugins.out
+'
+test_expect_success 'job-manager: bad plugins config is detected' '
+	mkdir -p badconf/a badconf/b badconf/c  badconf/d &&
+	cat <<-EOF >badconf/a/job-manager.toml &&
+	[job-manager]
+	plugins = { load = "test.so" }
+	EOF
+	cat <<-EOF >badconf/b/job-manager.toml &&
+	[[job-manager.plugins]]
+	load = 42
+	EOF
+	cat <<-EOF >badconf/c/job-manager.toml &&
+	[[job-manager.plugins]]
+	remove = "notfound.so"
+	EOF
+	cat <<-EOF >badconf/d/job-manager.toml &&
+	[[job-manager.plugins]]
+	load = "notfound.so"
+	EOF
+	test_must_fail \
+	    flux bulksubmit -n1 --watch --log=badconf.{}.log \
+	        flux start -c$(pwd)/badconf/{} true ::: a b c d &&
+	test_debug "echo a:; cat badconf.a.log" &&
+	grep "config must be an array" badconf.a.log &&
+	test_debug "echo b:; cat badconf.b.log" &&
+	grep -i "expected string.*got integer" badconf.b.log &&
+	test_debug "echo c:; cat badconf.c.log" &&
+	grep -i "failed to find plugin to remove" badconf.c.log &&
+	test_debug "echo d:; cat badconf.d.log" &&
+	grep -i "no such plugin found" badconf.d.log
+'
+test_expect_success 'job-manager: default plugin sets priority to urgency' '
+	flux jobtap remove all &&
+	flux jobtap list -a | grep .priority-default &&
+	jobid=$(flux submit --urgency=8 hostname) &&
+	flux job wait-event -v $jobid priority &&
+	test $(flux jobs -no {priority} $jobid) = 8 &&
+	flux job wait-event -v $jobid clean
+'
+test_expect_success 'job-manager: default works with sched.prioritize' '
+	ncores=$(flux resource list -s free -no {ncores}) &&
+	allcores=$(flux submit -n ${ncores} sleep 1000) &&
+	flux submit --cc=1-2 --flags=debug hostname >prio.jobids &&
+	job1=$(cat prio.jobids | head -1) &&
+	job3=$(cat prio.jobids | tail -1) &&
+	flux job wait-event -v $job1 debug.alloc-request &&
+	test_debug "echo updating urgency of top queued job" &&
+	flux job urgency $job1 17 &&
+	flux job wait-event -v $job1 priority &&
+	test_debug "echo updating urgency of last queued job" &&
+	flux job urgency $job3 18 &&
+	flux job wait-event -v $job3 debug.alloc-request &&
+	test_debug "echo cleanup" &&
+	flux cancel --all &&
+	flux queue drain
+'
+test_expect_success 'job-manager: hold plugin holds jobs' '
+	flux jobtap load submit-hold.so &&
+	flux bulksubmit --job-name=cc-{0} hostname ::: $(seq 1 4) \
+	    >hold.jobids &&
+	flux job wait-event -v $(cat hold.jobids | tail -1) priority &&
+	for id in $(cat hold.jobids); do
+	    test_debug "echo waiting for job ${id} priority event" &&
+	    flux job wait-event -f json ${id} priority | \
+	        jq -e ".context.priority == 0"
+	done
+'
+test_expect_success 'job-manager: job is released when urgency set' '
+	jobid=$(id_byname cc-1) &&
+	test_debug "echo jobid=${jobid}" &&
+	state=$(flux jobs -no {state} $jobid) &&
+	test_debug "echo state is ${state}" &&
+	test "$state" = "SCHED" &&
+	flux job urgency $jobid 1 &&
+	flux job wait-event -v -t 5 $jobid clean
+'
+test_expect_success 'job-manager: cancel of held job works' '
+	jobid=$(id_byname cc-2) &&
+	flux cancel $jobid &&
+	flux job wait-event -v -t 5 $jobid clean
+'
+test_expect_success 'job-manager: release held jobs' '
+	for name in cc-3 cc-4; do
+	    jobid=$(id_byname $name) &&
+	    flux job urgency $jobid 1 &&
+	    flux job wait-event -t 5 -v $jobid clean
+	done
+'
+test_expect_success 'job-manager: unload hold plugin' '
+	flux jobtap remove submit-hold.so
+'
+test_expect_success 'job-manager: test with random priority plugin' '
+	flux module reload sched-simple mode=unlimited &&
+	ncores=$(flux resource list -s free -no {ncores}) &&
+	sleepjob=$(flux submit -n ${ncores} sleep 3000) &&
+	flux jobtap load --remove=all ${PLUGINPATH}/random.so &&
+	flux bulksubmit --flags waitable --job-name=random-{} hostname \
+	    ::: $(seq 1 4) &&
+	flux jobs -c4 -no {name}:{priority} | sort > pri-before.out &&
+	sleep 1 &&
+	flux jobs -c4 -no {name}:{priority} | sort > pri-after.out &&
+	test_must_fail test_cmp pri-before.out pri-after.out &&
+	flux cancel $sleepjob &&
+	flux job wait-event -vt 30 $sleepjob clean &&
+	flux job wait --all -v
+'
+test_expect_success 'job-manager: run args test plugin' '
+	flux jobtap load --remove=all ${PLUGINPATH}/args.so &&
+	flux run hostname &&
+	flux dmesg | grep args-check > args-check.log &&
+	test_debug "cat args-check.log" &&
+	test $(grep -c OK args-check.log) = 21
+'
+test_expect_success 'job-manager: run subscribe test plugin' '
+	flux jobtap load --remove=all ${PLUGINPATH}/subscribe.so &&
+	flux run hostname &&
+	flux dmesg | grep subscribe-check > subscribe-check.log &&
+	test_debug "cat subscribe-check.log" &&
+	test $(grep -c OK subscribe-check.log) = 6
+'
+test_expect_success 'job-manager: run job_aux test plugin' '
+	flux dmesg --clear &&
+	flux jobtap load --remove=all ${PLUGINPATH}/job_aux.so &&
+	flux run hostname
+'
+test_expect_success 'job-manager: job aux cleared on transition to inactive' '
+	flux dmesg >aux-dmesg.out &&
+	grep "test destructor invoked" aux-dmesg.out
+'
+test_expect_success 'job-manager: start another job and remove plugin' '
+	flux dmesg --clear &&
+	jobid=$(flux submit --wait-event=alloc sleep 60) &&
+	flux jobtap remove job_aux.so &&
+	flux cancel $jobid
+'
+test_expect_success 'job-manager: job aux cleared when plugin removed' '
+	flux dmesg >aux-dmesg2.out &&
+	grep "test destructor invoked" aux-dmesg2.out
+'
+test_expect_success 'job-manager: load jobtap_api test plugin' '
+	flux jobtap load --remove=all ${PLUGINPATH}/jobtap_api.so &&
+	id=$(flux submit sleep 1000) &&
+	flux run -vvv \
+		--setattr=system.lookup-id=$(flux job id $id) \
+		true &&
+	flux cancel $id &&
+	id=$(flux submit \
+		--setattr=system.expected-result=failed \
+		false) &&
+	test_expect_code 1 flux job attach -vEX $id &&
+	test_must_fail flux job wait-event $id exception &&
+	id=$(flux submit -t 0.1s \
+		--setattr=system.expected-result=timeout \
+		sleep 10) &&
+	test_must_fail flux job wait-event -vm type=test $id exception &&
+	id=$(flux submit --urgency=hold \
+		--setattr=system.expected-result=canceled \
+		true) &&
+	flux cancel $id &&
+	test_must_fail flux job wait-event -vm type=test $id exception
+'
+test_expect_success 'job-manager: test that job flags can be set' '
+	id=$(flux submit \
+	       --setattr=system.depend.set_flag=debug hostname) &&
+	flux job wait-event -vt 20 $id debug.alloc-request &&
+	flux job wait-event -vt 20 $id clean
+'
+
+check_event_post() {
+	local state=$1
+	local id
+
+	id=$(flux submit \
+	    --setattr system.${state}.post-event=testevent hostname) &&
+	flux job wait-event -vt 20 $id testevent &&
+	flux job wait-event -t 20 $id clean >/dev/null
+}
+
+for state in validate new depend priority run cleanup; do
+    test_expect_success "job-manager: jobtap job.$state can emit events" "
+        check_event_post ${state}
+    "
+done
+
+test_expect_success 'job-manager: load test jobtap plugin' '
+	flux jobtap load --remove=all ${PLUGINPATH}/test.so foo.test=1 &&
+	flux dmesg | grep "conf={\"foo\":{\"test\":1}}"
+'
+test_expect_success 'job-manager: run all test plugin test modes' '
+	cat <<-EOF | sort >test-modes.txt &&
+	priority unset
+	callback error
+	sched: priority unavail
+	sched: update priority
+	sched: dependency-add
+	sched: exception error
+	annotations error
+	EOF
+	COUNT=$(cat test-modes.txt | wc -l) &&
+	run_timeout 20 \
+	  flux bulksubmit --quiet --watch \
+	    --setattr=system.jobtap.test-mode={} \
+	    hostname :::: test-modes.txt >test-plugin.out &&
+	test_debug "cat test-plugin.out" &&
+	test $COUNT = $(cat test-plugin.out | wc -l) &&
+	flux jobs -ac $COUNT -no {annotations.test} | \
+	    sort >test-annotations.out &&
+	test_cmp test-modes.txt test-annotations.out
+'
+test_expect_success 'job-manager: priority type error generates nonfatal exception' '
+	id=$(flux submit \
+		--setattr=system.jobtap.test-mode="priority type error" \
+		hostname) &&
+	flux job wait-event -vm type=job.state.priority ${id} exception &&
+	test "$(flux jobs -no {annotations.test} ${id})" = "priority type error" &&
+	test $(flux jobs -no {state} ${id}) = "PRIORITY" &&
+	flux cancel ${id} &&
+	flux job wait-event -v ${id} clean
+'
+test_expect_success 'job-manager: jobtap plugin can raise job exception' '
+	id=$(flux submit \
+	     --setattr=system.jobtap.test-mode="sched: exception" \
+	     hostname) &&
+	flux job wait-event -v --match type=test $id exception
+'
+test_expect_success 'job-manager: run test plugin modes for priority.get' '
+	cat <<-EOF | sort >test-modes.priority.get &&
+	priority.get: fail
+	priority.get: unavail
+	priority.get: bad arg
+	EOF
+	cat <<-EOT >reprioritize.py &&
+	import flux
+	flux.Flux().rpc("job-manager.test.reprioritize", {}).get()
+	EOT
+	COUNT=$(cat test-modes.priority.get|wc -l) &&
+	flux queue stop &&
+	flux bulksubmit \
+	    --flags waitable --setattr=system.jobtap.test-mode={} hostname \
+	    :::: test-modes.priority.get > priority.get.jobids &&
+	flux python ./reprioritize.py &&
+	flux queue start &&
+	test_debug "flux dmesg | grep jobtap\.test" &&
+	flux jobs -ac 3 -o {id}:{annotations.test}:{status} &&
+	run_timeout 20 flux job wait -v --all
+'
+test_expect_success 'job-manager: run test plugin modes for job.validate' '
+	test_expect_code 1 \
+	    flux submit\
+	        --setattr=system.jobtap.test-mode="validate failure" \
+	        hostname >validate-failure.out 2>&1 &&
+	test_debug "cat validate-failure.out" &&
+	grep "rejected for testing" validate-failure.out &&
+	test_expect_code 1 \
+	    flux submit\
+	        --setattr=system.jobtap.test-mode="validate failure nullmsg" \
+	        hostname >validate-failure2.out 2>&1 &&
+	test_debug "cat validate-failure2.out" &&
+	grep "rejected by job-manager plugin" validate-failure2.out &&
+	test_expect_code 1 \
+	    flux submit\
+	        --setattr=system.jobtap.test-mode="validate failure nomsg" \
+	        hostname >validate-failure3.out 2>&1 &&
+	test_debug "cat validate-failure3.out" &&
+	grep "rejected by job-manager plugin" validate-failure3.out
+'
+test_expect_success 'job-manager: plugin can keep job in PRIORITY state' '
+	flux jobtap load --remove=all ${PLUGINPATH}/priority-wait.so &&
+	jobid=$(flux submit hostname) &&
+	flux job wait-event -vt 5 $jobid depend &&
+	test $(flux jobs -no {state} $jobid) = "PRIORITY"
+'
+test_expect_success 'job-manager: job exits PRIORITY when priority is set' '
+	jobid=$(flux jobs -nc 1 -o {id}) &&
+	cat <<-EOF >pri-set.py &&
+	import flux
+	from flux.job import JobID
+	import sys
+
+	jobid = flux.job.JobID(sys.argv[1])
+	priority = int(sys.argv[2])
+	topic = "job-manager.priority-wait.release"
+	print(flux.Flux().rpc(topic, {"id": jobid, "priority": priority}).get())
+	EOF
+	flux python pri-set.py $jobid 42000 &&
+	flux job wait-event -vt 5 $jobid clean &&
+	flux jobs -no {priority} $jobid &&
+	test $(flux jobs -no {priority} $jobid) = 42000
+'
+test_expect_success 'job-manager: plugin can reject some jobs in a batch' '
+	flux module reload job-ingest batch-count=6 &&
+	flux jobtap load --remove=all ${PLUGINPATH}/validate.so &&
+	test_expect_code 1 \
+	    flux bulksubmit --watch \
+	        --setattr=system.jobtap.validate-test-id={} \
+	        echo foo ::: 1 1 1 4 4 1 >validate-plugin.out 2>&1 &&
+	test_debug "cat validate-plugin.out" &&
+	grep "Job had reject_id" validate-plugin.out &&
+	test 4 -eq $(grep -c foo validate-plugin.out)
+'
+
+# Ensure that the valid jobs submitted above have reached INACTIVE in the
+# (eventually consistent) job-list module, then ask job-list about the invalid
+# jobs.  The assumption is that the "invalidate" journal events must have have
+# been processed by job-list if the "clean" event for last job in the batch,
+# which is valid, has been processed.  If the "invalidate" event is not
+# received, the job-list query for the invalid jobs will hang.
+
+test_expect_success 'job-manager: get job IDs of invalid jobs' '
+	grep reject_id validate-plugin.out \
+	    | sed -e "s/.*jobid=//" >invalid_ids &&
+	test_debug "cat invalid_ids" &&
+	test 2 -eq $(wc -l <invalid_ids)
+'
+test_expect_success 'job-manager: get job IDs of valid jobs' '
+	grep -v reject_id validate-plugin.out | grep -v foo \
+	    | sed -e "s/.*jobid=//" >valid_ids &&
+	test_debug "cat valid_ids" &&
+	test 4 -eq $(wc -l <valid_ids)
+'
+
+# Return true if all specified jobs are shown as inactive by job-list
+test_inactive() {
+	flux jobs -n -o {state} $@ >test_jobs.out 2>test_jobs.err
+	test $(wc -l <test_jobs.err) -eq 0 \
+		-a $(grep -v INACTIVE test_jobs.out | wc -l) -eq 0
+}
+# Return true if all specified jobs are unknown to job-list
+test_unknown() {
+	run_timeout 10 \
+	    flux jobs -n -o {state} $@ >test_jobs.out 2>test_jobs.err && \
+	test $(wc -l <test_jobs.out) -eq 0
+}
+
+test_expect_success 'job-manager: wait for valid jobs to appear inactive' '
+	while ! test_inactive $(cat valid_ids); do echo retry; sleep 0.1; done
+'
+test_expect_success 'job-manager: flux jobs does not list invalid jobs' '
+	test_unknown $(cat invalid_ids)
+'
+
+test_expect_success 'job-manager: plugin can manage dependencies' '
+	cat <<-EOF >dep-remove.py &&
+	import flux
+	from flux.job import JobID
+	import sys
+
+	jobid = flux.job.JobID(sys.argv[1])
+	topic = "job-manager.dependency-test.remove"
+	payload = {"id": jobid, "description": sys.argv[2]}
+	print(flux.Flux().rpc(topic, payload).get())
+	EOF
+	flux module reload job-ingest &&
+	flux jobtap load --remove=all ${PLUGINPATH}/dependency-test.so &&
+	jobid=$(flux submit --dependency=test:dependency-test hostname) &&
+	flux job wait-event -vt 15 ${jobid} dependency-add &&
+	test $(flux jobs -no {state} ${jobid}) = DEPEND &&
+	flux python dep-remove.py ${jobid} dependency-test &&
+	flux job wait-event -vt 15 ${jobid} clean
+'
+test_expect_success 'job-manager: dependency-add works from job.state.depend' '
+	jobid=$(flux submit --setattr=system.dependency-test=foo true) &&
+        flux job wait-event -vt 15 ${jobid} dependency-add &&
+	test_debug "flux jobs -no {state} ${jobid}" &&
+        test $(flux jobs -no {state} ${jobid}) = DEPEND &&
+        flux python dep-remove.py ${jobid} foo &&
+	flux job wait-event -vt 15 ${jobid} clean
+'
+test_expect_success 'job-manager: job.state.depend is called on plugin load' '
+	jobid=$(flux submit --dependency=test:dependency-test hostname) &&
+	flux job wait-event -vt 15 ${jobid} dependency-add &&
+	flux jobtap load --remove=all ${PLUGINPATH}/dependency-test.so &&
+	test $(flux jobs -no {state} ${jobid}) = DEPEND &&
+	flux python dep-remove.py ${jobid} dependency-test &&
+	flux job wait-event -vt 15 ${jobid} clean
+'
+lineno() {
+	local nline
+	nline=$(grep -n $1 $2) || return 1
+	echo $nline | cut -d: -f1
+}
+test_expect_success 'job-manager: job prolog/epilog events work' '
+	flux jobtap load --remove=all ${PLUGINPATH}/perilog-test.so &&
+	jobid=$(flux submit hostname) &&
+	flux job attach --wait-event clean -vE $jobid 2>&1 \
+	    | tee perilog-test.out &&
+	n_prolog=$(lineno job.prolog-finish perilog-test.out) &&
+	n_start=$(lineno job.start perilog-test.out) &&
+	n_epilog=$(lineno job.epilog-finish perilog-test.out) &&
+	n_free=$(lineno job.free perilog-test.out) &&
+	test_debug "echo Checking that prolog-finish=$n_prolog event occurs before start=$n_start event" &&
+	test_debug "echo Checking that epilog-finish=$n_epilog event occurs before free=$n_free event" &&
+	test $n_prolog -lt $n_start -a $n_epilog -lt $n_free
+'
+test_expect_success 'job-manager: epilog works after exception during prolog' '
+	flux jobtap load --remove=all ${PLUGINPATH}/perilog-test.so \
+		prolog-exception=1 &&
+	jobid=$(flux submit hostname) &&
+	flux job attach --wait-event clean -vE $jobid 2>&1 \
+	    | tee perilog-exception-test.out &&
+	n_epilog=$(lineno job.epilog-finish perilog-exception-test.out) &&
+	n_free=$(lineno job.free perilog-exception-test.out) &&
+	test $n_epilog -lt $n_free
+'
+test_expect_success 'job-manager: epilog prevents clean event' '
+	flux jobtap load --remove=all ${PLUGINPATH}/perilog-test.so &&
+	jobid=$(flux submit --urgency=hold hostname) &&
+	flux cancel $jobid &&
+	flux job attach --wait-event clean -vE $jobid 2>&1 \
+	    | tee perilog-epilog-clean-test.out &&
+	n_epilog=$(lineno job.epilog-finish perilog-epilog-clean-test.out) &&
+	n_clean=$(lineno job.clean perilog-epilog-clean-test.out) &&
+	test $n_epilog -lt $n_clean
+'
+test_expect_success 'job-manager: job.create posts events before validation' '
+	flux jobtap load --remove=all ${PLUGINPATH}/create-event.so &&
+	jobid=$(flux submit hostname) &&
+	flux job wait-event -v $jobid validate >create-event.out &&
+	n_test=$(lineno test-event create-event.out) &&
+	n_create=$(lineno validate create-event.out) &&
+	test $n_test -lt $n_create
+'
+
+test_expect_success 'job-manager: job.create can reject a job' '
+	flux jobtap load --remove=all ${PLUGINPATH}/create-reject.so &&
+	test_must_fail flux submit hostname 2>submit.err &&
+	grep nope submit.err
+'
+test_expect_success 'job-manager: plugins can update jobspec' '
+	flux jobtap load --remove=all ${PLUGINPATH}/jobspec-update.so &&
+	jobid=$(flux submit --job-name=test hostname) &&
+	flux job eventlog -H $jobid &&
+	flux job eventlog $jobid | grep jobspec-update &&
+	flux job eventlog $jobid | grep attributes.system.update-test &&
+	flux job eventlog $jobid | \
+		test_must_fail grep attributes.system.run-update &&
+	test_must_fail flux job wait-event --count=2 -Hv \
+		--match-context=attributes.system.job.name=new \
+		$jobid jobspec-update
+'
+test_expect_success 'job-manager: plugin can asynchronously update jobspec' '
+	flux dmesg -H | grep jobspec-update &&
+	jobid=$(flux submit -n1 --urgency=hold sleep 0) &&
+	cat <<-EOF >update-test.py &&
+	import flux
+	from flux.job import JobID
+	id=JobID("$jobid")
+	flux.Flux().rpc(
+	    "job-manager.jobspec-update.update",
+	    dict(id=id, update={"attributes.system.job.name": "test"}),
+	).get()
+	EOF
+	flux python update-test.py &&
+	flux job wait-event -Hv -t 30 $jobid jobspec-update &&
+	flux jobs $jobid
+'
+test_expect_success 'job-manager: plugin fails to load on config.update error' '
+	flux jobtap remove all &&
+	test_must_fail flux jobtap load ${PLUGINPATH}/config.so 2>config.err
+'
+test_expect_success 'job-manager: and produces reasonable error for humans' '
+	grep "Error parsing" config.err
+'
+test_expect_success 'set up valid test configuration' '
+	cat >config/test.toml <<-EOT &&
+	[testconfig]
+	testkey = "a string"
+	EOT
+	flux config reload
+'
+test_expect_success 'and now plugin expecting that config can load' '
+	flux jobtap load ${PLUGINPATH}/config.so
+'
+test_expect_success 'reloading invalid configuration fails' '
+	cat >config/test.toml <<-EOT &&
+	[testconfig]
+	testkey = 42
+	EOT
+	test_must_fail flux config reload 2>reload.err
+'
+test_expect_success 'job-manager: and produces reasonable error for humans' '
+	grep "Error parsing" reload.err
+'
+test_expect_success 'job-manager: run a job then purge all inactives' '
+	flux jobtap load --remove=all ${PLUGINPATH}/args.so &&
+	flux dmesg -C &&
+	flux run hostname &&
+	flux job purge --force --num-limit=0 &&
+	flux dmesg | grep args-check | grep OK >argsok.out
+'
+test_expect_success 'job-manager: job.inactive-add was called' '
+	grep -q job.inactive-add argsok.out
+'
+test_expect_success 'job-manager: job.inactive-remove was called' '
+	grep -q job.inactive-remove argsok.out
+'
+
+test_done
diff --git a/t/t2213-job-manager-hold-single.t b/t/t2213-job-manager-hold-single.t
new file mode 100755
index 000000000000..85b1647eb111
--- /dev/null
+++ b/t/t2213-job-manager-hold-single.t
@@ -0,0 +1,138 @@
+#!/bin/sh
+
+test_description='Test flux job manager job hold (single mode)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_CORES_PER_RANK=1
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="limited=1"
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+# N.B. resources = 1 rank, 1 core/rank
+test_expect_success 'job-manager: submit 5 jobs (job 2 held)' '
+        flux bulksubmit --log=job{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: default hold default default default
+'
+
+test_expect_success 'job-manager: job state RSSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotations job 3 pending, job 2 held (RSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: job 4 hold' '
+        flux job urgency $(cat job4.id) 0
+'
+
+test_expect_success 'job-manager: cancel job 1' '
+        flux cancel $(cat job1.id)
+'
+
+test_expect_success 'job-manager: job state ISRSS (job 3 run, job 2 held)' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotations job 5 pending (job 2/4 held)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\""
+'
+
+test_expect_success 'job-manager: job 2, 4 hold again (issue #4940)' '
+        flux job urgency $(cat job2.id) hold &&
+        flux job urgency $(cat job4.id) hold &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job4.id) S
+'
+
+test_expect_success 'job-manager: job 4 release (higher urgency)' '
+        flux job urgency $(cat job4.id) 20
+'
+
+test_expect_success 'job-manager: annotations job 4 pending' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: cancel job 3' '
+        flux cancel $(cat job3.id)
+'
+
+test_expect_success 'job-manager: job state ISIRS (job 4 run, job 2 held)' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) I &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotations job 5 pending (job 2 held)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\""
+'
+
+test_expect_success 'job-manager: cancel job 4' '
+        flux cancel $(cat job4.id)
+'
+
+test_expect_success 'job-manager: job state ISIIR (job 5 run, job 2 held)' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) I &&
+        jmgr_check_state $(cat job4.id) I &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: annotations no job pending (job 2 held)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: job 2 release' '
+        flux job urgency $(cat job2.id) 16
+'
+
+test_expect_success 'job-manager: annotations job 2 pending' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_annotation $(cat job2.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: cancel remaining jobs' '
+        flux cancel $(cat job2.id) &&
+        flux cancel $(cat job5.id)
+'
+
+test_done
diff --git a/t/t2214-job-manager-hold-limited.t b/t/t2214-job-manager-hold-limited.t
new file mode 100755
index 000000000000..1382354c23d7
--- /dev/null
+++ b/t/t2214-job-manager-hold-limited.t
@@ -0,0 +1,137 @@
+#!/bin/sh
+
+test_description='Test flux job manager job hold (limited)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="limited=2"
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+# N.B. resources = 1 rank, 2 cores/rank
+test_expect_success 'job-manager: submit 5 jobs (job 3,4,5 held)' '
+        flux bulksubmit --log=job{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: default default hold hold hold
+'
+
+test_expect_success 'job-manager: job state RRSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: job annotations correct (RSSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: hold job 3, 5 again (issue #4940)' '
+        flux job urgency $(cat job3.id) hold &&
+        flux job urgency $(cat job3.id) hold &&
+        flux job urgency $(cat job3.id) hold &&
+        flux job urgency $(cat job3.id) hold &&
+        flux job urgency $(cat job5.id) hold
+'
+
+test_expect_success 'job-manager: remove hold on job 3, 4, 5' '
+        flux job urgency $(cat job3.id) 16 &&
+        flux job urgency $(cat job4.id) 16 &&
+        flux job urgency $(cat job5.id) 16
+'
+
+test_expect_success 'job-manager: job state RRSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: job annotations updated (RRSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: put hold on job 4' '
+        flux job urgency $(cat job4.id) hold
+'
+
+test_expect_success 'job-manager: job state RRSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: job annotations updated (RRSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "1"
+'
+
+test_expect_success 'job-manager: cancel job 1 & 2' '
+        flux cancel $(cat job1.id) &&
+        flux cancel $(cat job2.id)
+'
+
+test_expect_success 'job-manager: job state IISSR' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: job annotations updated (IIRSR)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core1\""
+'
+
+test_expect_success 'job-manager: remove hold on job 4' '
+        flux job urgency $(cat job4.id) 16
+'
+
+test_expect_success 'job-manager: job state IISSR' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: job annotations updated (IIRSR)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core1\""
+'
+
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel $(cat job4.id) &&
+        flux cancel $(cat job5.id)
+'
+
+test_done
diff --git a/t/t2215-job-manager-hold-unlimited.t b/t/t2215-job-manager-hold-unlimited.t
new file mode 100755
index 000000000000..819956b51f22
--- /dev/null
+++ b/t/t2215-job-manager-hold-unlimited.t
@@ -0,0 +1,151 @@
+#!/bin/sh
+
+test_description='Test flux job manager job hold (unlimited mode)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="unlimited"
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+# N.B. resources = 1 rank, 2 cores/rank
+test_expect_success 'job-manager: submit 5 jobs (job 4 held)' '
+        flux bulksubmit --log=job{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: default default default hold default
+'
+
+test_expect_success 'job-manager: job state RRSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: job annotations correct (RRSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "1"
+'
+
+test_expect_success 'job-manager: put hold on job 3' '
+        flux job urgency $(cat job3.id) hold
+'
+
+test_expect_success 'job-manager: hold job 4 again (issue #4940)' '
+        flux job urgency $(cat job4.id) hold
+'
+
+test_expect_success 'job-manager: job state RRSSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: job annotations updated (RRSSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "0"
+'
+
+test_expect_success 'job-manager: cancel job 1 & 2' '
+        flux cancel $(cat job1.id) &&
+        flux cancel $(cat job2.id)
+'
+
+test_expect_success 'job-manager: job state IISSR' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: job annotations updated (IISSR)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: remove hold on job 3' '
+        flux job urgency $(cat job3.id) 16
+'
+
+test_expect_success 'job-manager: job state IISSR' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: job annotations updated (IIRSR)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: remove hold on job 4' '
+        flux job urgency $(cat job4.id) 16
+'
+
+test_expect_success 'job-manager: job state IIRSR' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: job annotations updated (IIRSR)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: cancel job 3' '
+        flux cancel $(cat job3.id)
+'
+
+test_expect_success 'job-manager: job state IIIRR' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) I &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: job annotations updated (IISRR)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel $(cat job4.id) &&
+        flux cancel $(cat job5.id)
+'
+
+test_done
diff --git a/t/t2216-job-manager-priority-order-single.t b/t/t2216-job-manager-priority-order-single.t
new file mode 100755
index 000000000000..9e15113367ce
--- /dev/null
+++ b/t/t2216-job-manager-priority-order-single.t
@@ -0,0 +1,138 @@
+#!/bin/sh
+
+test_description='Test flux job manager urgency change to job (mode=single)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="limited=1"
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+# N.B. resources = 1 rank, 2 cores/rank
+test_expect_success 'job-manager: submit 4 jobs' '
+        flux submit --log=job{cc}.id --cc="1-4" --flags=debug -n1 \
+           hostname
+'
+
+test_expect_success 'job-manager: job state RRSS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S
+'
+
+test_expect_success 'job-manager: annotate job id 3 (RRSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_no_annotations $(cat job4.id)
+'
+
+test_expect_success 'job-manager: annotations in job id 3-4 (RRSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\""
+'
+
+test_expect_success 'job-manager: increase urgency of job 4' '
+        flux job urgency $(cat job4.id) 20
+'
+
+test_expect_success 'job-manager: annotations in job id 3-4 updated (RRSS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        jmgr_check_annotation $(cat job4.id) "sched.reason_pending" "\"insufficient resources\""
+'
+
+test_expect_success 'job-manager: cancel 2' '
+        flux cancel $(cat job2.id)
+'
+
+test_expect_success 'job-manager: job state RISR (job 4 runs instead of 3)' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R
+'
+
+test_expect_success 'job-manager: annotations in job id 3-4 updated (RISR)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job4.id) "sched.reason_pending"
+'
+
+test_expect_success 'job-manager: submit high urgency job' '
+        flux submit --flags=debug --urgency=20 -n1 hostname >job5.id
+'
+
+test_expect_success 'job-manager: job state RISRS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotations in job id 3-5 updated (RISRS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\""
+'
+
+PLUGINPATH=${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs
+test_expect_success 'job-manager: load priority-invert plugin' '
+        flux jobtap load --remove=all ${PLUGINPATH}/priority-invert.so
+'
+
+test_expect_success 'job-manager: job state RISRS' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotations in job id 3-5 updated (RISRS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: cancel 1' '
+        flux cancel $(cat job1.id)
+'
+
+test_expect_success 'job-manager: job state IISRR (job 5 runs instead of 3)' '
+        jmgr_check_state $(cat job1.id) I &&
+        jmgr_check_state $(cat job2.id) I &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) S
+'
+
+test_expect_success 'job-manager: annotations in job id 3-5 updated (IISRR)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.reason_pending" "\"insufficient resources\""
+'
+
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel $(cat job5.id) &&
+        flux cancel $(cat job4.id) &&
+        flux cancel $(cat job3.id)
+'
+
+test_done
diff --git a/t/t2217-job-manager-priority-order-limited.t b/t/t2217-job-manager-priority-order-limited.t
new file mode 100755
index 000000000000..9a5366d8df49
--- /dev/null
+++ b/t/t2217-job-manager-priority-order-limited.t
@@ -0,0 +1,207 @@
+#!/bin/sh
+
+test_description='Test flux job manager scheduler priority ordering (limited)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="limited=2"
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+# N.B. resources = 1 rank, 2 cores/rank
+# flux queue stop/start to ensure no scheduling until after all jobs submitted
+test_expect_success 'job-manager: submit 5 jobs (differing urgencies)' '
+        flux queue stop &&
+        flux bulksubmit --log=job{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: $(seq 10 2 18) &&
+        flux queue start
+'
+
+test_expect_success 'job-manager: job state SSSRR' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: annotate jobs (SSSRR)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_annotation $(cat job2.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: increase urgency job 2' '
+        flux job urgency $(cat job2.id) 16
+'
+
+test_expect_success 'job-manager: job state SSSRR' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: annotate jobs updated (SSSRR)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_annotation $(cat job2.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: increase urgency job 1' '
+        flux job urgency $(cat job1.id) 18
+'
+
+test_expect_success 'job-manager: job state SSSRR' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: annotate jobs updated (SSSRR)' '
+        jmgr_check_annotation $(cat job1.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job1.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job2.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: decrease urgency job 2' '
+        flux job urgency $(cat job2.id) 8
+'
+
+test_expect_success 'job-manager: job state SSSRR' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: annotate jobs updated (SSSRR)' '
+        jmgr_check_annotation $(cat job1.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job1.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: submit new job with higher urgency' '
+        flux submit --flags=debug --urgency=20 -n1 hostname >job6.id
+'
+
+test_expect_success 'job-manager: job state SSSRRS' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R &&
+        jmgr_check_state $(cat job6.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs updated (SSSRRS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job1.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_no_annotations $(cat job2.id) &&
+        jmgr_check_no_annotations $(cat job3.id) &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job6.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job6.id) "sched.jobs_ahead" "0"
+'
+
+# Notes:
+#
+# job urgencies at this point
+# job1 - 18
+# job2 - 8
+# job3 - 14
+# job6 - 20
+#
+# so inversion priority plugin will change to
+#
+# job1 - 13
+# job2 - 23
+# job3 - 17
+# job6 - 11
+#
+# making job 2 next in line to run, job 3 after it
+
+PLUGINPATH=${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs
+test_expect_success 'job-manager: load priority-invert plugin' '
+        flux jobtap load --remove=all ${PLUGINPATH}/priority-invert.so
+'
+
+test_expect_success 'job-manager: job state SSSRRS' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R &&
+        jmgr_check_state $(cat job6.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs updated (SSSRRS)' '
+        jmgr_check_no_annotations $(cat job1.id) &&
+        jmgr_check_annotation $(cat job2.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job6.id)
+'
+
+test_expect_success 'job-manager: cancel 5' '
+        flux cancel $(cat job5.id)
+'
+
+test_expect_success 'job-manager: job state SRSRIS' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) R &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) I &&
+        jmgr_check_state $(cat job6.id) S
+'
+
+test_expect_success 'job-manager: annotate jobs updated (SRSRIS)' '
+        jmgr_check_annotation $(cat job1.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job1.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job2.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job2.id) "sched.reason_pending" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job2.id) "sched.jobs_ahead" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_no_annotations $(cat job5.id) &&
+        jmgr_check_no_annotations $(cat job6.id)
+'
+
+# cancel non-running jobs first, to ensure they are not accidentally run when
+# running jobs free resources.
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all --states=pending &&
+        flux cancel --all
+'
+
+test_done
diff --git a/t/t2218-job-manager-priority-order-unlimited.t b/t/t2218-job-manager-priority-order-unlimited.t
new file mode 100755
index 000000000000..aa96f7810346
--- /dev/null
+++ b/t/t2218-job-manager-priority-order-unlimited.t
@@ -0,0 +1,142 @@
+#!/bin/sh
+
+test_description='Test flux job manager scheduler priority ordering (unlimited)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="unlimited"
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+# N.B. resources = 1 rank, 2 cores/rank
+# flux queue stop/start to ensure no scheduling until after all jobs submitted
+test_expect_success 'job-manager: submit 5 jobs (differing urgencies)' '
+        flux queue stop &&
+        flux bulksubmit --log=job{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: 12 10 14 16 18 > jobids.out &&
+        flux queue start
+'
+
+test_expect_success 'job-manager: job state SSSRR' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: annotate jobs (SSSRR)' '
+        jmgr_check_annotation $(cat job1.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job1.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job2.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.jobs_ahead" "2" &&
+        jmgr_check_annotation $(cat job3.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_annotation $(cat job5.id) "sched.resource_summary" "\"rank0/core0\""
+'
+
+test_expect_success 'job-manager: cancel 5' '
+        flux cancel $(cat job5.id)
+'
+
+test_expect_success 'job-manager: job state SSRRI' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) I
+'
+
+test_expect_success 'job-manager: annotate jobs (SSRRI)' '
+        jmgr_check_annotation $(cat job1.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job1.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job2.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: increase urgency job 2' '
+        flux job urgency $(cat job2.id) 20
+'
+
+test_expect_success 'job-manager: job state SSRRI' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) I
+'
+
+test_expect_success 'job-manager: annotate jobs updated (SSRRI)' '
+        jmgr_check_annotation $(cat job1.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job1.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job2.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+PLUGINPATH=${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs
+test_expect_success 'job-manager: load priority-invert plugin' '
+        flux jobtap load --remove=all ${PLUGINPATH}/priority-invert.so
+'
+
+test_expect_success 'job-manager: job state SSRRI' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) I
+'
+
+test_expect_success 'job-manager: annotate jobs updated (SSRRI)' '
+        jmgr_check_annotation $(cat job1.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job1.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job2.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.jobs_ahead" "1" &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_annotation $(cat job4.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+test_expect_success 'job-manager: cancel 4' '
+        flux cancel $(cat job4.id)
+'
+
+test_expect_success 'job-manager: job state RSRII' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) I &&
+        jmgr_check_state $(cat job5.id) I
+'
+
+test_expect_success 'job-manager: annotate jobs updated (RSRII)' '
+        jmgr_check_annotation $(cat job1.id) "sched.resource_summary" "\"rank0/core1\"" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job1.id) "sched.reason_pending" &&
+        test_must_fail jmgr_check_annotation_exists $(cat job1.id) "sched.jobs_ahead" &&
+        jmgr_check_annotation $(cat job2.id) "sched.reason_pending" "\"insufficient resources\"" &&
+        jmgr_check_annotation $(cat job2.id) "sched.jobs_ahead" "0" &&
+        jmgr_check_annotation $(cat job3.id) "sched.resource_summary" "\"rank0/core0\"" &&
+        jmgr_check_no_annotations $(cat job4.id) &&
+        jmgr_check_no_annotations $(cat job5.id)
+'
+
+# cancel non-running jobs first, to ensure they are not accidentally run when
+# running jobs free resources.
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all --states=pending &&
+        flux cancel --all
+'
+
+test_done
diff --git a/t/t2219-job-manager-restart.t b/t/t2219-job-manager-restart.t
new file mode 100755
index 000000000000..fdf4a290907b
--- /dev/null
+++ b/t/t2219-job-manager-restart.t
@@ -0,0 +1,222 @@
+#!/bin/sh
+
+test_description='Test flux job manager restart'
+
+. $(dirname $0)/sharness.sh
+
+DUMPS=${SHARNESS_TEST_SRCDIR}/job-manager/dumps
+
+export FLUX_DISABLE_JOB_CLEANUP=t
+
+test_expect_success 'start instance with empty kvs, run one job, and dump' '
+	flux start -Scontent.dump=dump.tar \
+	    flux run --env-remove=* true &&
+	test -f $(pwd)/dump.tar
+'
+
+restart_flux() {
+	flux start -Scontent.restore=$1 \
+		flux module stats job-manager
+}
+
+# Returns 0 if dump file is replayed successfully AND one or more
+# "not replayed" warnings were logged
+restart_with_job_warning() {
+	local out=$(basename $1).dmesg
+	flux start -Scontent.restore=$1 true 2>$out
+	result=$?
+	cat $out
+	test $result -eq 0 && grep -q "not replayed:" $out
+}
+
+test_expect_success 'verify that job manager can restart with current dump' '
+	restart_flux dump.tar >stats.out
+'
+test_expect_success 'and max_jobid is greater than zero' '
+	jq -e ".max_jobid > 0" <stats.out
+'
+test_expect_success 'delete checkpoint from dump' '
+	mkdir -p tmp &&
+	(cd tmp && tar -xf -) <dump.tar &&
+	rm tmp/checkpoint/job-manager &&
+	(cd tmp && tar -cf - *) >dump-nock.tar
+'
+test_expect_success 'verify that job manager can restart with modified dump' '
+	restart_flux dump-nock.tar >stats-nock.out
+'
+test_expect_success 'and max_jobid is still greater than zero' '
+	jq -e ".max_jobid > 0" <stats-nock.out
+'
+test_expect_success 'delete job from dump' '
+	mkdir -p tmp &&
+	(cd tmp && tar -xf -) <dump.tar &&
+	rm -r tmp/job/* &&
+	(cd tmp && tar -cf - *) >dump-nojob.tar
+'
+test_expect_success 'verify that job manager can restart with modified dump' '
+	restart_flux dump-nojob.tar >stats-nojob.out
+'
+test_expect_success 'and max_jobid is still greater than zero' '
+	jq -e ".max_jobid > 0" <stats-nojob.out
+'
+
+test_expect_success 'purging all jobs triggers jobid checkpoint update' '
+	flux start bash -c "flux run --env-remove=* true && \
+	    flux job purge -f --num-limit=0 && \
+	    flux kvs get checkpoint.job-manager"
+'
+
+test_expect_success 'verify that anon queue disable persists across restart' '
+	flux start -Scontent.dump=dump_dis.tar \
+	    flux queue disable disable-restart-test &&
+	flux start -Scontent.restore=dump_dis.tar \
+	    flux queue status >dump_dis.out &&
+	grep "disabled: disable-restart-test" dump_dis.out
+'
+
+test_expect_success 'verify that anon queue stopped persists across restart' '
+	flux start -Scontent.dump=dump_stopped.tar \
+	    flux queue stop stop-restart-test &&
+	flux start -Scontent.restore=dump_stopped.tar \
+	    flux queue status >dump_stopped.out &&
+	grep "stopped: stop-restart-test" dump_stopped.out
+'
+
+test_expect_success 'verify that named queue enable/disable persists across restart' '
+	mkdir -p conf.d &&
+	cat >conf.d/queues.toml <<-EOT &&
+	[queues.debug]
+	[queues.batch]
+	EOT
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.dump=dump_queue_enable1.tar \
+	    flux queue status >dump_queue_enable_1.out &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_enable1.tar \
+	    -Scontent.dump=dump_queue_enable2.tar \
+	    flux queue disable --queue=batch xyzzy &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_enable2.tar \
+	    -Scontent.dump=dump_queue_enable3.tar \
+	    flux queue status >dump_queue_enable_2.out &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_enable3.tar \
+	    -Scontent.dump=dump_queue_enable4.tar \
+	    flux queue enable --queue=batch &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_enable4.tar \
+	    -Scontent.dump=dump_queue_enable5.tar \
+	    flux queue status >dump_queue_enable_3.out &&
+	grep "^debug: Job submission is enabled" dump_queue_enable_1.out &&
+	grep "^batch: Job submission is enabled" dump_queue_enable_1.out &&
+	grep "^debug: Job submission is enabled" dump_queue_enable_2.out &&
+	grep "^batch: Job submission is disabled: xyzzy" dump_queue_enable_2.out &&
+	grep "^debug: Job submission is enabled" dump_queue_enable_3.out &&
+	grep "^batch: Job submission is enabled" dump_queue_enable_3.out
+'
+
+# N.B. no named queues configured in this test, so anon queue is what
+# is tested
+test_expect_success 'verify that instance can restart after config change' '
+	flux start -Scontent.restore=dump_queue_enable5.tar \
+	    flux queue status >dump_queue_reconf.out &&
+	grep "^Job submission is enabled" dump_queue_reconf.out
+'
+
+test_expect_success 'verify that named queue start/stop persists across restart' '
+	mkdir -p conf.d &&
+	cat >conf.d/queues.toml <<-EOT &&
+	[queues.debug]
+	[queues.batch]
+	EOT
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.dump=dump_queue_start1.tar \
+	    flux queue status >dump_queue_start_1.out &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_start1.tar \
+	    -Scontent.dump=dump_queue_start2.tar \
+	    flux queue start --queue=batch &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_start2.tar \
+	    -Scontent.dump=dump_queue_start3.tar \
+	    flux queue status >dump_queue_start_2.out &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_start3.tar \
+	    -Scontent.dump=dump_queue_start4.tar \
+	    flux queue stop --queue=batch xyzzy &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_start4.tar \
+	    -Scontent.dump=dump_queue_start5.tar \
+	    flux queue status >dump_queue_start_3.out &&
+	grep "^debug: Scheduling is stopped" dump_queue_start_1.out &&
+	grep "^batch: Scheduling is stopped" dump_queue_start_1.out &&
+	grep "^debug: Scheduling is stopped" dump_queue_start_2.out &&
+	grep "^batch: Scheduling is started" dump_queue_start_2.out &&
+	grep "^debug: Scheduling is stopped" dump_queue_start_3.out &&
+	grep "^batch: Scheduling is stopped: xyzzy" dump_queue_start_3.out
+'
+
+test_expect_success 'checkpointed queue no longer configured on restart is ignored' '
+	mkdir -p conf.d &&
+	cat >conf.d/queues.toml <<-EOT &&
+	[queues.debug]
+	[queues.batch]
+	EOT
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.dump=dump_queue_missing.tar \
+	    flux queue disable --queue batch xyzzy &&
+	cat >conf.d/queues.toml <<-EOT &&
+	[queues.debug]
+	EOT
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_missing.tar \
+	    flux queue status >dump_queue_missing.out &&
+	grep "^debug: Job submission is enabled" dump_queue_missing.out &&
+	grep "^debug: Scheduling is stopped" dump_queue_missing.out &&
+	test_must_fail grep "batch" dump_queue_missing.out
+'
+
+test_expect_success 'new queue configured on restart uses defaults' '
+	mkdir -p conf.d &&
+	cat >conf.d/queues.toml <<-EOT &&
+	[queues.debug]
+	[queues.batch]
+	EOT
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.dump=dump_queue_ignored.tar \
+	    flux queue disable --queue batch xyzzy &&
+	cat >conf.d/queues.toml <<-EOT &&
+	[queues.debug]
+	[queues.newqueue]
+	EOT
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_ignored.tar \
+	    flux queue status >dump_queue_ignored.out &&
+	grep "^debug: Job submission is enabled" dump_queue_ignored.out &&
+	grep "^debug: Scheduling is stopped" dump_queue_ignored.out &&
+	grep "^newqueue: Job submission is enabled" dump_queue_ignored.out &&
+	grep "^newqueue: Scheduling is stopped" dump_queue_ignored.out
+'
+
+test_expect_success 'bad job directory is moved to lost+found' '
+	flux start \
+	    -Scontent.restore=${DUMPS}/warn/dump-shorteventlog.tar.bz2 \
+	    flux kvs dir -R lost+found
+'
+
+for dump in ${DUMPS}/valid/*.tar.bz2; do
+    testname=$(basename $dump)
+    test_expect_success 'successfully started from '$testname "restart_flux $dump"
+done
+
+for dump in ${DUMPS}/warn/*.tar.bz2; do
+    testname=$(basename $dump)
+    test_expect_success 'successfully started with warning from '$testname "restart_with_job_warning $dump"
+done
+
+for dump in ${DUMPS}/invalid/*.tar.bz2; do
+    testname=$(basename $dump)
+    test_expect_success 'failed on '$testname "test_must_fail restart_flux $dump"
+done
+
+test_done
diff --git a/t/t2220-job-manager-R.t b/t/t2220-job-manager-R.t
new file mode 100755
index 000000000000..a9588600507b
--- /dev/null
+++ b/t/t2220-job-manager-R.t
@@ -0,0 +1,45 @@
+#!/bin/sh
+test_description='Test job manager internal copy of R'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+# Usage: job_manager_getattr ID ATTR
+job_manager_getattr() {
+    flux python -c "import flux; print(flux.Flux().rpc(\"job-manager.getattr\",{\"id\":$1,\"attrs\":[\"$2\"]}).get_str())"
+}
+
+# Diff two json files
+# https://stackoverflow.com/questions/31930041/using-jq-or-alternative-command-line-tools-to-compare-json-files
+json_diff() {
+	jq --sort-keys . $1 >difftmp1 &&
+	jq --sort-keys . $2 >difftmp2 &&
+	diff difftmp1 difftmp2
+}
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'submit job' '
+	flux submit -t 10s --wait-event=clean true | flux job id >jobid
+'
+test_expect_success 'job-manager getattr of R works' '
+	job_manager_getattr $(cat jobid) R  >R.out
+'
+test_expect_success 'R contains expiration' '
+	jq -e .R.execution.expiration R.out
+'
+test_expect_success 'reload job manager' '
+	flux module remove job-list &&
+	flux module remove sched-simple &&
+	flux module reload job-manager &&
+	flux module load sched-simple &&
+	flux module load job-list
+'
+test_expect_success 'job-manager getattr of R still works' '
+	job_manager_getattr $(cat jobid) R  >R2.out
+'
+test_expect_success 'R is unchanged after restart' '
+	json_diff R.out R2.out
+'
+test_done
diff --git a/t/t2221-job-manager-limit-duration.t b/t/t2221-job-manager-limit-duration.t
new file mode 100755
index 000000000000..16b482ebbdef
--- /dev/null
+++ b/t/t2221-job-manager-limit-duration.t
@@ -0,0 +1,116 @@
+#!/bin/sh
+
+test_description='Test flux job manager limit-duration plugin'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 2 full
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'configuring an invalid duration limit fails' '
+	test_must_fail flux config load <<-EOT
+	[policy.limits]
+	duration = 1.0
+	EOT
+'
+test_expect_success 'configure a valid duration limit' '
+	flux config load <<-EOT
+	[policy.limits]
+	duration = "1m"
+	EOT
+'
+test_expect_success 'a job that exceeds policy.limits.duration is rejected' '
+	test_must_fail flux submit -t 1h true 2>duration.err &&
+	grep "exceeds policy limit of 1m" duration.err
+'
+test_expect_success 'a job that is under policy.limits.duration is accepted' '
+	flux submit -t 30s true
+'
+test_expect_success 'configure policy.limits.duration and queue duration' '
+	flux config load <<-EOT &&
+	[policy.limits]
+	duration = "1h"
+	[queues.debug]
+	[queues.batch.policy.limits]
+	duration = "8h"
+	[queues.short.policy.limits]
+	duration = "1m"
+	EOT
+	flux queue start --all
+'
+test_expect_success 'a job that exceeds policy.limits.duration is rejected' '
+	test_must_fail flux submit --queue=debug -t 2h true
+'
+test_expect_success 'a job with no limit is also rejected' '
+	test_must_fail flux submit --queue=debug -t 0 true
+'
+test_expect_success 'but is accepted by a queue with higher limit' '
+	flux submit \
+	    --queue=batch \
+	    -t 2h \
+	    true
+'
+test_expect_success 'and is rejected when it exceeds the queue limit' '
+	test_must_fail flux submit \
+	    --queue=batch \
+	    -t 16h \
+	    true
+'
+test_expect_success 'no limit is also rejected as exceeding the queue limit' '
+	test_must_fail flux submit \
+	    --queue=batch \
+	    -t 0 \
+	    true
+'
+test_expect_success 'a job that is under policy.limits.duration is accepted' '
+	flux submit --queue=debug -t 1h true
+'
+test_expect_success 'but is rejected on a queue with lower limit' '
+	test_must_fail flux submit \
+	    --queue=short \
+	    -t 1h \
+	    true
+'
+test_expect_success 'configure policy.limits.duration and an unlimited queue' '
+	flux config load <<-EOT
+	[policy.limits]
+	duration = "1h"
+	[queues.batch.policy.limits]
+	duration = "0"
+	[queues.debug]
+	EOT
+'
+test_expect_success 'a job that is over policy.limits.duration is rejected' '
+	test_must_fail flux submit --queue=debug -t 2h true
+'
+test_expect_success 'but is accepted by the unlimited queue' '
+	flux submit \
+	    --queue=batch \
+	    -t 2h true
+'
+test_expect_success 'a job that sets no explicit duration is accepted by the unlimited queue' '
+	flux submit \
+	    --queue=batch \
+	    true
+'
+test_expect_success 'configure an invalid duration limit' '
+	test_must_fail flux config load <<-EOT
+	[policy.limits]
+	duration = "xyz123"
+	EOT
+'
+test_expect_success 'configure a duration limit of an invalid type' '
+	test_must_fail flux config load <<-EOT
+	[policy.limits]
+	duration = [ 0 ]
+	EOT
+'
+test_expect_success 'configure an invalid queue duration limit' '
+	test_must_fail flux config load <<-EOT
+	[queues.debug.policy.limits]
+	duration = "xyz123"
+	EOT
+'
+
+test_done
diff --git a/t/t2222-job-manager-limit-job-size.t b/t/t2222-job-manager-limit-job-size.t
new file mode 100755
index 000000000000..8a6cd6ba7bb7
--- /dev/null
+++ b/t/t2222-job-manager-limit-job-size.t
@@ -0,0 +1,108 @@
+#!/bin/sh
+
+test_description='Test flux job manager limit-job-size plugin'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 2 full
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'configure an invalid job-size limit' '
+	test_must_fail flux config load <<-EOT
+	[policy.limits]
+	job-size.max.nnodes = -42
+	EOT
+'
+test_expect_success 'configure valid job-size.*.nnodes limits' '
+	flux config load <<-EOT
+	[policy.limits]
+	job-size.max.nnodes = 2
+	job-size.min.nnodes = 2
+	EOT
+'
+test_expect_success 'a job that exceeds job-size.max.nnodes is rejected' '
+	test_must_fail flux submit -N 3 true 2>max-nnodes.err &&
+	grep "exceeds policy limit of 2" max-nnodes.err
+'
+test_expect_success 'a job that is under job-size.min.nnodes is rejected' '
+	test_must_fail flux submit -N 1 true 2>min-nnodes.err &&
+	grep "is under policy limit of 2" min-nnodes.err
+'
+test_expect_success 'a job that is between both of those is accepted' '
+	flux submit -N 2 true
+'
+test_expect_success 'configure job-size.*.ncores limits' '
+	flux config load <<-EOT
+	[policy.limits]
+	job-size.max.ncores = 2
+	job-size.min.ncores = 2
+	EOT
+'
+test_expect_success 'a job that exceeds job-size.max.ncores is rejected' '
+	test_must_fail flux submit -n 3 true 2>max-ncores.err &&
+	grep "exceeds policy limit of 2" max-ncores.err
+'
+test_expect_success 'a job that is under job-size.min.ncores is rejected' '
+	test_must_fail flux submit -n 1 true 2>min-ncores.err &&
+	grep "is under policy limit of 2" min-ncores.err
+'
+test_expect_success 'a job that is between both of those is accepted' '
+	flux submit -n 2 true
+'
+test_expect_success 'configure job-size.*.ngpus limits' '
+	flux config load <<-EOT
+	[policy.limits]
+	job-size.max.ngpus = 2
+	job-size.min.ngpus = 2
+	EOT
+'
+test_expect_success 'a job that exceeds job-size.max.ngpus is rejected' '
+	test_must_fail flux submit -g 3 true 2>max-ngpus.err &&
+	grep "exceeds policy limit of 2" max-ngpus.err
+'
+test_expect_success 'a job that is under job-size.min.ngpus is rejected' '
+	test_must_fail flux submit -g 1 true 2>min-ngpus.err &&
+	grep "is under policy limit of 2" min-ngpus.err
+'
+test_expect_success 'a job that is between both of those is accepted' '
+	flux submit -g 2 true
+'
+test_expect_success 'configure job-size.max.ngpus and queue with unlimited' '
+	flux config load <<-EOT &&
+	[policy.limits]
+	job-size.max.ngpus = 0
+	[queues.debug]
+	[queues.batch.policy.limits]
+	job-size.max.ngpus = -1
+	EOT
+	flux queue start --all
+'
+test_expect_success 'a job is accepted if under general gpu limit' '
+	flux submit --queue=debug -n1 true
+'
+test_expect_success 'a job is rejected if over gpu limit' '
+	test_must_fail flux submit --queue=debug -n1 -g1 true
+'
+test_expect_success 'same job is accepted with unlimited queue override' '
+	flux submit --queue=batch -n1 -g1 true
+'
+test_expect_success 'configure an invalid job-size object' '
+	test_must_fail flux config load <<-EOT
+	[policy.limits]
+	job-size = "wrong"
+	EOT
+'
+test_expect_success 'configure an out of bounds job-size.max.nnodes object' '
+	test_must_fail flux config load <<-EOT
+	[policy.limits]
+	job-size.max.nnodes = -42
+	EOT
+'
+test_expect_success 'configure an invalid queue job-size.min.nnodes object' '
+	test_must_fail flux config load <<-EOT
+	[queues.debug.policy.limits]
+	job-size.min.nnodes = "xyz"
+	EOT
+'
+test_done
diff --git a/t/t2223-job-manager-queue-priority-order-limited.t b/t/t2223-job-manager-queue-priority-order-limited.t
new file mode 100755
index 000000000000..237040dc3b1b
--- /dev/null
+++ b/t/t2223-job-manager-queue-priority-order-limited.t
@@ -0,0 +1,264 @@
+#!/bin/sh
+
+test_description='Test flux job manager scheduler queue priority ordering (limited)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="limited=2"
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+# N.B. resources = 1 rank, 2 cores/rank
+# flux queue stop/start to ensure no scheduling until after all jobs submitted
+test_expect_success 'job-manager: submit 6 jobs (differing urgencies)' '
+        flux queue stop &&
+        flux bulksubmit --log=job{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: $(seq 10 2 20) &&
+        flux queue start
+'
+
+test_expect_success 'job-manager: job state SSSSRR' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) R &&
+        jmgr_check_state $(cat job6.id) R
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_all1.out &&
+	cat <<-EOT >counts_all1.exp &&
+	Job submission is enabled
+	Scheduling is started
+	2 alloc requests queued
+	2 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_all1.exp counts_all1.out
+'
+
+test_expect_success 'job-manager: stop scheduling' '
+        flux queue stop
+'
+
+test_expect_success 'job-manager: increase urgency job 3' '
+        flux job urgency $(cat job3.id) 20
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_all2.out &&
+	cat <<-EOT >counts_all2.exp &&
+	Job submission is enabled
+	Scheduling is stopped
+	0 alloc requests queued
+	0 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_all2.exp counts_all2.out
+'
+
+test_expect_success 'job-manager: cancel a running job' '
+        flux cancel $(cat job6.id)
+'
+
+test_expect_success 'job-manager: start scheduling' '
+        flux queue start
+'
+
+# job 3 should run before over previously higher priority job 4
+test_expect_success 'job-manager: job state SSRSRI' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) R &&
+        jmgr_check_state $(cat job6.id) I
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_all3.out &&
+	cat <<-EOT >counts_all3.exp &&
+	Job submission is enabled
+	Scheduling is started
+	1 alloc requests queued
+	2 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_all3.exp counts_all3.out
+'
+
+test_expect_success 'job-manager: stop scheduling' '
+        flux queue stop
+'
+
+test_expect_success 'job-manager: increase urgency job 1' '
+        flux job urgency $(cat job1.id) 20
+'
+
+test_expect_success 'job-manager: cancel a running job' '
+        flux cancel $(cat job5.id)
+'
+
+test_expect_success 'job-manager: start scheduling' '
+        flux queue start
+'
+
+# job 1 should have highest remaining priority and now run
+test_expect_success 'job-manager: job state RSRSII' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) R &&
+        jmgr_check_state $(cat job4.id) S &&
+        jmgr_check_state $(cat job5.id) I &&
+        jmgr_check_state $(cat job6.id) I
+'
+
+# cancel non-running jobs first, to ensure they are not accidentally run when
+# running jobs free resources.
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all --states=pending &&
+        flux cancel --all &&
+        flux queue drain
+'
+
+#
+# Test named queues
+#
+
+test_expect_success 'configure batch,debug queues' '
+	flux config load <<-EOT &&
+	[queues.batch]
+	[queues.debug]
+	EOT
+	flux queue enable --all &&
+	flux queue start --all
+'
+
+test_expect_success 'job-manager: submit 5 jobs to each queue (differing urgencies)' '
+        flux queue stop --all &&
+        flux bulksubmit -q batch --log=batch{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: $(seq 10 2 18) &&
+        flux bulksubmit -q debug --log=debug{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: $(seq 10 2 18) &&
+        flux queue start --all
+'
+
+test_expect_success 'job-manager: job state SSSSR / SSSSR' '
+        jmgr_check_state $(cat batch1.id) S &&
+        jmgr_check_state $(cat batch2.id) S &&
+        jmgr_check_state $(cat batch3.id) S &&
+        jmgr_check_state $(cat batch4.id) S &&
+        jmgr_check_state $(cat batch5.id) R &&
+        jmgr_check_state $(cat debug1.id) S &&
+        jmgr_check_state $(cat debug2.id) S &&
+        jmgr_check_state $(cat debug3.id) S &&
+        jmgr_check_state $(cat debug4.id) S &&
+        jmgr_check_state $(cat debug5.id) R
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_named1.out &&
+	cat <<-EOT >counts_named1.exp &&
+	batch: Job submission is enabled
+	batch: Scheduling is started
+	debug: Job submission is enabled
+	debug: Scheduling is started
+	6 alloc requests queued
+	2 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_named1.exp counts_named1.out
+'
+
+test_expect_success 'job-manager: stop scheduling debug queue' '
+        flux queue stop --queue=debug
+'
+
+test_expect_success 'job-manager: cancel the two running jobs' '
+        flux cancel $(cat batch5.id) &&
+        flux cancel $(cat debug5.id)
+'
+
+# batch queue jobs should run instead of debug queue
+test_expect_success 'job-manager: job state SSRRI / SSSSI' '
+        jmgr_check_state $(cat batch1.id) S &&
+        jmgr_check_state $(cat batch2.id) S &&
+        jmgr_check_state $(cat batch3.id) R &&
+        jmgr_check_state $(cat batch4.id) R &&
+        jmgr_check_state $(cat batch5.id) I &&
+        jmgr_check_state $(cat debug1.id) S &&
+        jmgr_check_state $(cat debug2.id) S &&
+        jmgr_check_state $(cat debug3.id) S &&
+        jmgr_check_state $(cat debug4.id) S &&
+        jmgr_check_state $(cat debug5.id) I
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_named2.out &&
+	cat <<-EOT >counts_named2.exp &&
+	batch: Job submission is enabled
+	batch: Scheduling is started
+	debug: Job submission is enabled
+	debug: Scheduling is stopped
+	0 alloc requests queued
+	2 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_named2.exp counts_named2.out
+'
+
+test_expect_success 'job-manager: increase urgency job 2 in debug queue' '
+        flux job urgency $(cat debug2.id) 18
+'
+
+test_expect_success 'job-manager: start scheduling debug queue' '
+        flux queue start --queue=debug
+'
+
+test_expect_success 'job-manager: cancel the two running jobs' '
+        flux cancel $(cat batch3.id) &&
+        flux cancel $(cat batch4.id)
+'
+
+# debug job 2 and 4 should have highest priority now and be running
+test_expect_success 'job-manager: job state SSIII / SSRRI' '
+        jmgr_check_state $(cat batch1.id) S &&
+        jmgr_check_state $(cat batch2.id) S &&
+        jmgr_check_state $(cat batch3.id) I &&
+        jmgr_check_state $(cat batch4.id) I &&
+        jmgr_check_state $(cat batch5.id) I &&
+        jmgr_check_state $(cat debug1.id) S &&
+        jmgr_check_state $(cat debug2.id) R &&
+        jmgr_check_state $(cat debug3.id) S &&
+        jmgr_check_state $(cat debug4.id) R &&
+        jmgr_check_state $(cat debug5.id) I
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_named3.out &&
+	cat <<-EOT >counts_named3.exp &&
+	batch: Job submission is enabled
+	batch: Scheduling is started
+	debug: Job submission is enabled
+	debug: Scheduling is started
+	2 alloc requests queued
+	2 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_named3.exp counts_named3.out
+'
+
+# cancel non-running jobs first, to ensure they are not accidentally run when
+# running jobs free resources.
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all --states=pending &&
+        flux cancel --all &&
+        flux queue drain
+'
+
+test_done
diff --git a/t/t2224-job-manager-queue-priority-order-unlimited.t b/t/t2224-job-manager-queue-priority-order-unlimited.t
new file mode 100755
index 000000000000..33e2a8201b08
--- /dev/null
+++ b/t/t2224-job-manager-queue-priority-order-unlimited.t
@@ -0,0 +1,235 @@
+#!/bin/sh
+
+test_description='Test flux job manager scheduler queue priority ordering (unlimited)'
+
+. `dirname $0`/job-manager/sched-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export TEST_UNDER_FLUX_NO_JOB_EXEC=y
+export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="unlimited"
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+# N.B. resources = 1 rank, 2 cores/rank
+# flux queue stop/start to ensure no scheduling until after all jobs submitted
+test_expect_success 'job-manager: submit 5 jobs (differing urgencies)' '
+        flux queue stop &&
+        flux bulksubmit --log=job{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: $(seq 10 2 18) &&
+        flux queue start
+'
+
+test_expect_success 'job-manager: job state SSSRR' '
+        jmgr_check_state $(cat job1.id) S &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) R
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_all1.out &&
+	cat <<-EOT >counts_all1.exp &&
+	Job submission is enabled
+	Scheduling is started
+	0 alloc requests queued
+	3 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_all1.exp counts_all1.out
+'
+
+test_expect_success 'job-manager: stop scheduling' '
+        flux queue stop
+'
+
+test_expect_success 'job-manager: increase urgency job 1' '
+        flux job urgency $(cat job1.id) 20
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_all2.out &&
+	cat <<-EOT >counts_all2.exp &&
+	Job submission is enabled
+	Scheduling is stopped
+	0 alloc requests queued
+	0 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_all2.exp counts_all2.out
+'
+
+test_expect_success 'job-manager: cancel a running job' '
+        flux cancel $(cat job5.id)
+'
+
+test_expect_success 'job-manager: start scheduling' '
+        flux queue start
+'
+
+# job 1 should run before over previously higher priority job 3
+test_expect_success 'job-manager: job state RSSRI' '
+        jmgr_check_state $(cat job1.id) R &&
+        jmgr_check_state $(cat job2.id) S &&
+        jmgr_check_state $(cat job3.id) S &&
+        jmgr_check_state $(cat job4.id) R &&
+        jmgr_check_state $(cat job5.id) I
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_all3.out &&
+	cat <<-EOT >counts_all3.exp &&
+	Job submission is enabled
+	Scheduling is started
+	0 alloc requests queued
+	2 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_all3.exp counts_all3.out
+'
+
+# cancel non-running jobs first, to ensure they are not accidentally run when
+# running jobs free resources.
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all --states=pending &&
+        flux cancel --all &&
+        flux queue drain
+'
+
+#
+# Test named queues
+#
+
+test_expect_success 'configure batch,debug queues' '
+	flux config load <<-EOT &&
+	[queues.batch]
+	[queues.debug]
+	EOT
+	flux queue start --all
+'
+
+test_expect_success 'job-manager: submit 5 jobs to each queue (differing urgencies)' '
+        flux queue stop --all &&
+        flux bulksubmit -q batch --log=batch{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: $(seq 10 2 18) &&
+        flux bulksubmit -q debug --log=debug{seq1}.id --urgency={} --flags=debug -n1 \
+           hostname ::: $(seq 10 2 18) &&
+        flux queue start --all
+'
+
+test_expect_success 'job-manager: job state SSSSR / SSSSR' '
+        jmgr_check_state $(cat batch1.id) S &&
+        jmgr_check_state $(cat batch2.id) S &&
+        jmgr_check_state $(cat batch3.id) S &&
+        jmgr_check_state $(cat batch4.id) S &&
+        jmgr_check_state $(cat batch5.id) R &&
+        jmgr_check_state $(cat debug1.id) S &&
+        jmgr_check_state $(cat debug2.id) S &&
+        jmgr_check_state $(cat debug3.id) S &&
+        jmgr_check_state $(cat debug4.id) S &&
+        jmgr_check_state $(cat debug5.id) R
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_named1.out &&
+	cat <<-EOT >counts_named1.exp &&
+	batch: Job submission is enabled
+	batch: Scheduling is started
+	debug: Job submission is enabled
+	debug: Scheduling is started
+	0 alloc requests queued
+	8 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_named1.exp counts_named1.out
+'
+
+test_expect_success 'job-manager: stop scheduling debug queue' '
+        flux queue stop --queue=debug
+'
+
+test_expect_success 'job-manager: cancel the two running jobs' '
+        flux cancel $(cat batch5.id) &&
+        flux cancel $(cat debug5.id)
+'
+
+# batch queue jobs should run instead of debug queue
+test_expect_success 'job-manager: job state SSRRI / SSSSI' '
+        jmgr_check_state $(cat batch1.id) S &&
+        jmgr_check_state $(cat batch2.id) S &&
+        jmgr_check_state $(cat batch3.id) R &&
+        jmgr_check_state $(cat batch4.id) R &&
+        jmgr_check_state $(cat batch5.id) I &&
+        jmgr_check_state $(cat debug1.id) S &&
+        jmgr_check_state $(cat debug2.id) S &&
+        jmgr_check_state $(cat debug3.id) S &&
+        jmgr_check_state $(cat debug4.id) S &&
+        jmgr_check_state $(cat debug5.id) I
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_named2.out &&
+	cat <<-EOT >counts_named2.exp &&
+	batch: Job submission is enabled
+	batch: Scheduling is started
+	debug: Job submission is enabled
+	debug: Scheduling is stopped
+	0 alloc requests queued
+	2 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_named2.exp counts_named2.out
+'
+
+test_expect_success 'job-manager: increase urgency job 1 in debug queue' '
+        flux job urgency $(cat debug1.id) 18
+'
+
+test_expect_success 'job-manager: start scheduling debug queue' '
+        flux queue start --queue=debug
+'
+
+test_expect_success 'job-manager: cancel the two running jobs' '
+        flux cancel $(cat batch3.id) &&
+        flux cancel $(cat batch4.id)
+'
+
+# debug job 2 and 4 should have highest priority now and be running
+test_expect_success 'job-manager: job state SSIII / RSSRI' '
+        jmgr_check_state $(cat batch1.id) S &&
+        jmgr_check_state $(cat batch2.id) S &&
+        jmgr_check_state $(cat batch3.id) I &&
+        jmgr_check_state $(cat batch4.id) I &&
+        jmgr_check_state $(cat batch5.id) I &&
+        jmgr_check_state $(cat debug1.id) R &&
+        jmgr_check_state $(cat debug2.id) S &&
+        jmgr_check_state $(cat debug3.id) S &&
+        jmgr_check_state $(cat debug4.id) R &&
+        jmgr_check_state $(cat debug5.id) I
+'
+
+test_expect_success 'job-manager: queue counts are as expected' '
+	flux queue status -v >counts_named3.out &&
+	cat <<-EOT >counts_named3.exp &&
+	batch: Job submission is enabled
+	batch: Scheduling is started
+	debug: Job submission is enabled
+	debug: Scheduling is started
+	0 alloc requests queued
+	4 alloc requests pending to scheduler
+	2 running jobs
+	EOT
+	test_cmp counts_named3.exp counts_named3.out
+'
+
+# cancel non-running jobs first, to ensure they are not accidentally run when
+# running jobs free resources.
+test_expect_success 'job-manager: cancel all jobs' '
+        flux cancel --all --states=pending &&
+        flux cancel --all &&
+        flux queue drain
+'
+
+test_done
diff --git a/t/t2226-housekeeping.t b/t/t2226-housekeeping.t
new file mode 100755
index 000000000000..ec9181659d9b
--- /dev/null
+++ b/t/t2226-housekeeping.t
@@ -0,0 +1,361 @@
+#!/bin/sh
+test_description='Test job manager housekeeping'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4
+
+flux setattr log-stderr-level 1
+
+# Usage: list_jobs
+list_jobs () {
+	flux housekeeping list -no {id}
+}
+
+# Usage: kill_all signum
+kill_all () {
+	flux housekeeping kill --all --signal=$1
+}
+
+# Usage: kill_job jobid signum
+kill_job () {
+	flux housekeeping kill --jobid=$1 --signal=$2
+}
+
+# Usage: kill_ranks idset signum
+kill_ranks () {
+	flux housekeeping kill --targets=$1 --signal=$2
+}
+
+# Note: the hand off of resources to housekeeping occurs just before the job
+# becomes inactive, therefore it is safe to assume that housekeeping has run
+# for the job if it is enclosed between successful 'wait_for_running 0' calls.
+# This pattern is used repeatedly in the tests below.
+
+# Usage: wait_for_running count
+wait_for_running () {
+	count=0
+	while test $(list_jobs | wc -l) -ne $1; do
+		count=$(($count+1));
+		test $count -eq 300 && return 1 # max 300 * 0.1s sleep = 30s
+		sleep 0.1
+	done
+}
+
+test_expect_success 'flux-housekeeping utility exists' '
+	flux housekeeping list --help &&
+	flux housekeeping kill --help
+'
+test_expect_success 'flux-housekeeping kill fails without proper args' '
+	test_must_fail flux housekeeping kill &&
+	test_must_fail flux housekeeping kill --all --jobid=f1
+'
+test_expect_success 'dump housekeeping stats, presumed empty' '
+	flux module stats job-manager | jq .housekeeping
+'
+
+# Note: the broker runs housekeeping in its cwd rather than the test script's
+# (the trash dir) so $(pwd) is expanded at script creation time, but
+# \$(flux getattr rank) is expanded at runtime.
+
+test_expect_success 'create housekeeping script' '
+	cat >housekeeping.sh <<-EOT &&
+	#!/bin/sh
+	touch $(pwd)/hkflag.\$(flux getattr rank)
+	EOT
+	chmod +x housekeeping.sh &&
+	test_debug "cat housekeeping.sh"
+'
+test_expect_success 'configure housekeeping without partial release' '
+	flux config load <<-EOT &&
+	[job-manager.housekeeping]
+	command = [ "$(pwd)/housekeeping.sh" ]
+	EOT
+	test_debug "flux config get job-manager.housekeeping"
+'
+test_expect_success 'run a job on broker ranks 1-2 and wait for housekeeping' '
+	test_debug "flux dmesg -C" &&
+	rm -f hkflag.* &&
+	wait_for_running 0 &&
+	flux run -N2 --requires=ranks:1-2 true &&
+	wait_for_running 0
+'
+test_expect_success 'housekeeping script ran on ranks 1-2' '
+	test_debug "flux dmesg -H" &&
+	test_debug "echo $(pwd)/hkflag.*" &&
+	test -f hkflag.1 -a -f hkflag.2
+'
+test_expect_success 'configure housekeeping with immediate release' '
+	flux config load <<-EOT
+	[job-manager.housekeeping]
+	command = [ "$(pwd)/housekeeping.sh" ]
+	release-after = "0"
+	EOT
+'
+test_expect_success 'run a job on all four ranks and wait for housekeeping' '
+	rm -f hkflag.* &&
+	flux dmesg -C &&
+	wait_for_running 0 &&
+	flux run -n4 -N4 true &&
+	wait_for_running 0
+'
+test_expect_success 'housekeeping script ran on ranks 0-3' '
+	test_debug "flux dmesg -H" &&
+	test_debug  "echo $(pwd)/hkflag.*" &&
+	test -f hkflag.0 -a -f hkflag.1 -a -f hkflag.2 -a -f hkflag.3
+'
+test_expect_success 'nodes were returned to scheduler separately' '
+	flux dmesg -H | grep sched-simple >sched.log &&
+	grep "free: rank0" sched.log &&
+	grep "free: rank1" sched.log &&
+	grep "free: rank2" sched.log &&
+	grep "free: rank3" sched.log &&
+	test $(grep final sched.log | wc -l) -eq 1
+'
+test_expect_success 'create housekeeping script with one 10s straggler' '
+	cat >housekeeping2.sh <<-EOT &&
+	#!/bin/sh
+	rank=\$(flux getattr rank)
+	test \$rank -eq 3 && sleep 10
+	touch $(pwd)/hkflag.\$rank
+	EOT
+	chmod +x housekeeping2.sh
+'
+test_expect_success 'configure housekeeping with release after 5s' '
+	flux config load <<-EOT
+	[job-manager.housekeeping]
+	command = [ "$(pwd)/housekeeping2.sh" ]
+	release-after = "5s"
+	EOT
+'
+test_expect_success 'run a job on all four ranks and wait for housekeeping' '
+	rm -f hkflag.* &&
+	flux dmesg -C &&
+	wait_for_running 0 &&
+	flux run -n4 -N4 true &&
+	sleep 1 &&
+	flux housekeeping list &&
+	wait_for_running 0
+'
+test_expect_success 'dump stats' '
+	flux module stats job-manager | jq .housekeeping
+'
+test_expect_success 'housekeeping script ran on ranks 0-3' '
+	test_debug "flux dmesg -H" &&
+	test_debug "echo $(pwd)/hkflag.*" &&
+	test -f hkflag.0 -a -f hkflag.1 -a -f hkflag.2 -a -f hkflag.3
+'
+test_expect_success 'there was one alloc and two frees to the scheduler' '
+	flux dmesg -H | grep sched-simple >sched2.log &&
+	grep "free: rank\[0-2\]" sched2.log &&
+	grep "free: rank3.*(final)" sched2.log
+'
+test_expect_success 'configuring housekeeping with bad key fails' '
+	test_must_fail flux config load 2>load.err <<-EOT &&
+	[job-manager.housekeeping]
+	xyz = 42
+	EOT
+	grep "left unpacked" load.err
+'
+test_expect_success 'configuring housekeeping with no cmd or exec.imp fails' '
+	test_must_fail flux config load 2>load_noimp.err <<-EOT &&
+	[job-manager.housekeeping]
+	EOT
+	grep "exec.imp is undefined" load_noimp.err
+'
+test_expect_success 'but is accepted if exec.imp is defined' '
+	flux config load <<-EOT
+	[exec]
+	imp = "/bin/echo"
+	[job-manager.housekeeping]
+	EOT
+'
+test_expect_success 'deprecated use-systemd-unit is ignored with log warning' '
+	flux dmesg -C &&
+	flux config load <<-EOT &&
+	[exec]
+	imp = "/bin/echo"
+	[job-manager.housekeeping]
+	use-systemd-unit = true
+	EOT
+	flux dmesg | grep "use-systemd-unit is deprecated"
+'
+test_expect_success 'configuring housekeeping with bad fsd fails' '
+	test_must_fail flux config load 2>load2.err <<-EOT &&
+	[job-manager.housekeeping]
+	command = [ "/bin/echo" ]
+	release-after = "foo"
+	EOT
+	grep "FSD parse error" load2.err
+'
+test_expect_success 'configure housekeeping with wrong path' '
+	flux config load <<-EOT
+	[job-manager.housekeeping]
+	command = [ "/noexist" ]
+	EOT
+'
+test_expect_success 'run a job and ensure error was logged' '
+	flux dmesg -C &&
+	wait_for_running 0 &&
+	flux run true &&
+	wait_for_running 0 &&
+	flux dmesg | grep "error launching process"
+'
+test_expect_success 'create housekeeping script with one failing rank (3)' '
+	cat >housekeeping3.sh <<-EOT &&
+	#!/bin/sh
+	test \$(flux getattr rank) -ne 3
+	EOT
+	chmod +x housekeeping3.sh
+'
+test_expect_success 'configure housekeeping with one failing rank' '
+	flux config load <<-EOT
+	[job-manager.housekeeping]
+	command = [ "$(pwd)/housekeeping3.sh" ]
+	EOT
+'
+test_expect_success 'run a job across all ranks and wait for housekeeping' '
+	flux dmesg -C &&
+	wait_for_running 0 &&
+	flux run -N4 true &&
+	wait_for_running 0 &&
+	flux dmesg | grep "nonzero exit code"
+'
+test_expect_success 'create housekeeping script that creates output' '
+	cat >housekeeping4.sh <<-EOT &&
+	#!/bin/sh
+	echo housekeeping-output
+	echo housekeeping-output >&2
+	EOT
+	chmod +x housekeeping4.sh
+'
+test_expect_success 'configure housekeeping to print to stdout' '
+	flux config load <<-EOT
+	[job-manager.housekeeping]
+	command = [ "$(pwd)/housekeeping4.sh" ]
+	EOT
+'
+test_expect_success 'run a job and ensure script output was logged' '
+	flux dmesg -C &&
+	wait_for_running 0 &&
+	flux run true &&
+	wait_for_running 0 &&
+	flux dmesg | grep housekeeping-output >output &&
+	test $(wc -l <output) -eq 2
+'
+test_expect_success 'create housekeeping script that dumps environment' '
+	cat >housekeeping5.sh <<-EOT &&
+	#!/bin/sh
+	printenv >$(pwd)/env.out
+	EOT
+	chmod +x housekeeping5.sh
+'
+test_expect_success 'configure housekeeping to dump environment' '
+	flux config load <<-EOT
+	[job-manager.housekeeping]
+	command = [ "$(pwd)/housekeeping5.sh" ]
+	EOT
+'
+test_expect_success 'run a job on rank 3, wait for hk, and check environment' '
+	wait_for_running 0 &&
+	flux run --requires=rank:3 true &&
+	wait_for_running 0 &&
+	grep "^FLUX_JOB_ID=$(flux job last | flux job id --to=dec)$" env.out &&
+	grep "^FLUX_JOB_USERID=$(id -u)$" env.out &&
+	grep "^FLUX_URI=$(flux exec -r 3 flux getattr local-uri)$" env.out
+'
+test_expect_success 'configure housekeeping to sleep forever' '
+	flux config load <<-EOT
+	[job-manager.housekeeping]
+	command = [ "sleep", "inf" ]
+	EOT
+'
+test_expect_success 'run two jobs that trigger housekeeping' '
+	wait_for_running 0 &&
+	flux submit --cc=0-1 -N2 --wait true
+'
+test_expect_success 'housekeeping is running for 2 jobs' '
+	wait_for_running 2
+'
+test_expect_success 'send SIGTERM to all jobs' '
+	kill_all 15
+'
+test_expect_success 'wait for housekeeping to finish' '
+	wait_for_running 0
+'
+test_expect_success 'run a job that trigger housekeeping' '
+	wait_for_running 0 &&
+	flux run -N4 true
+'
+test_expect_success 'housekeeping is running for 1 job' '
+	wait_for_running 1
+'
+test_expect_success 'send SIGTERM to the job by id' '
+	kill_job $(flux job last | flux job id --to=dec) 15
+'
+test_expect_success 'wait for housekeeping to finish' '
+	wait_for_running 0
+'
+test_expect_success 'run 4 jobs that trigger housekeeping' '
+	wait_for_running 0 &&
+	flux submit --cc=0-3 -N1 true
+'
+test_expect_success 'housekeeping is running for 4 jobs' '
+	wait_for_running 4
+'
+test_expect_success 'flux resource list shows 4 nodes allocated' '
+	test $(flux resource list -s allocated -no {nnodes}) -eq 4
+'
+test_expect_success 'flux housekeeping list shows 4 jobs' '
+	test_debug "flux housekeeping list" &&
+	test $(flux housekeeping list -n | wc -l) -eq 4
+'
+test_expect_success 'flux housekeeping list -i, --include works' '
+	flux housekeeping list -i 0  &&
+	test "$(flux housekeeping list -i 0 -no {ranks})" = 0 &&
+	flux housekeeping list -i 0,3 &&
+	test "$(flux housekeeping list -i 0,3 -no {ranks})" = "0,3"
+'
+test_expect_success 'flux housekeeping list -i, --include works with hostname'  '
+	flux housekeeping list -i $(hostname)
+'
+test_expect_success 'flux housekeeping list -i, --include fails with bad hostlist' '
+	test_must_fail flux housekeeping list -i "foo["
+'
+test_expect_success 'send SIGTERM to the nodes by rank' '
+	kill_ranks 0-3 15
+'
+test_expect_success 'wait for housekeeping to finish' '
+	wait_for_running 0
+'
+test_expect_success 'flux resource list shows 0 nodes allocated' '
+	test $(flux resource list -s allocated -no {nnodes}) -eq 0
+'
+# The following tests exercise recovery from RFC 27 hello protocol
+# with partial release. Once partial release is added to RFC 27, these
+# tests should be removed or changed.
+test_expect_success 'configure housekeeping with immediate release' '
+	flux config load <<-EOT
+	[job-manager.housekeeping]
+	command = [ "$(pwd)/housekeeping2.sh" ]
+	release-after = "0"
+	EOT
+'
+test_expect_success 'run job that uses 4 nodes to trigger housekeeping' '
+	flux run -N4 true
+'
+test_expect_success 'housekeeping is running for 1 job' '
+	wait_for_running 1
+'
+test_expect_success 'reload scheduler' '
+	flux dmesg -C &&
+	flux module reload -f sched-simple &&
+	flux dmesg -H
+'
+test_expect_success 'wait for housekeeping to finish' '
+	wait_for_running 0
+'
+test_expect_success 'housekeeping jobs were terminated due to sched reload' '
+	flux dmesg | grep "housekeeping:.*will be terminated"
+'
+test_done
diff --git a/t/t2230-job-info-lookup.t b/t/t2230-job-info-lookup.t
new file mode 100755
index 000000000000..34fb6b4d56ae
--- /dev/null
+++ b/t/t2230-job-info-lookup.t
@@ -0,0 +1,409 @@
+#!/bin/sh
+
+test_description='Test flux job info lookup service'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 job
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+INFOLOOKUP=${FLUX_BUILD_DIR}/t/job-info/info_lookup
+
+fj_wait_event() {
+	flux job wait-event --timeout=20 "$@"
+}
+
+# Usage: submit_job
+# To ensure robustness of tests despite future job manager changes,
+# cancel the job, and wait for clean event.
+submit_job() {
+	local jobid=$(flux job submit sleeplong.json) &&
+	fj_wait_event $jobid start >/dev/null &&
+	flux cancel $jobid &&
+	fj_wait_event $jobid clean >/dev/null &&
+	echo $jobid
+}
+
+get_timestamp_field() {
+	local field=$1
+	local file=$2
+	grep $field $file | awk '{print $1}'
+}
+
+test_expect_success 'job-info: generate jobspec for simple test job' '
+	flux run --dry-run -n1 -N1 sleep 300 > sleeplong.json
+'
+
+test_expect_success 'job-info.lookup returns jobid in response' '
+	jobid=$(submit_job) &&
+	id=$(flux job id --to=dec ${jobid}) &&
+        $jq -j -c -n  "{id:${id}, keys:[\"jobspec\"], flags:0}" \
+          | $RPC job-info.lookup > job_info_lookup.out &&
+        cat job_info_lookup.out | $jq -e ".id == ${id}"
+'
+
+#
+# job info lookup list w/o jobid & keys
+#
+
+test_expect_success 'flux job info fails without jobid' '
+	test_must_fail flux job info
+'
+
+test_expect_success 'flux job info listing of keys works' '
+	test_must_fail flux job info 2>list_keys.err &&
+	grep J list_keys.err &&
+	grep R list_keys.err &&
+	grep eventlog list_keys.err &&
+	grep jobspec list_keys.err &&
+	grep guest.exec.eventlog list_keys.err &&
+	grep guest.input list_keys.err &&
+	grep guest.output list_keys.err
+'
+
+#
+# job info lookup tests
+#
+
+test_expect_success 'flux job info eventlog works' '
+	jobid=$(submit_job) &&
+	flux job info $jobid eventlog > eventlog_info_a.out &&
+	grep submit eventlog_info_a.out
+'
+
+test_expect_success 'flux job info eventlog fails on bad id' '
+	test_must_fail flux job info 12345 eventlog
+'
+
+test_expect_success 'flux job info jobspec works' '
+	jobid=$(submit_job) &&
+	flux job info $jobid jobspec > jobspec_a.out &&
+	grep sleep jobspec_a.out
+'
+
+test_expect_success 'flux job info --original jobspec works' '
+	jobid=$(flux submit --env=ORIGINALTHING=t true) &&
+	flux job info --original $jobid jobspec > jobspec_original.out &&
+	grep ORIGINALTHING jobspec_original.out
+'
+
+test_expect_success 'flux job info applies jobspec updates' '
+	jobid=$(flux submit --urgency=hold true) &&
+	echo $jobid > updated_jobspec.id &&
+	flux job info $jobid jobspec | jq -e ".attributes.system.duration == 0" &&
+	flux update $jobid duration=100s &&
+	flux job info $jobid jobspec | jq -e ".attributes.system.duration == 100.0" &&
+	flux cancel $jobid
+'
+
+test_expect_success 'flux job info --base jobspec works' '
+	flux job info --base $(cat updated_jobspec.id) jobspec \
+	     | jq -e ".attributes.system.duration == 0"
+'
+
+test_expect_success 'flux job info jobspec fails on bad id' '
+	test_must_fail flux job info 12345 jobspec
+'
+
+# N.B. In future may wish to update expiration via `flux update` tool
+# when feature is available
+test_expect_success 'flux job info applies resource updates' '
+	jobid=$(flux submit --wait-event=start sleep 300) &&
+	echo $jobid > updated_R.id &&
+	flux job info $jobid R | jq -e ".execution.expiration == 0.0" &&
+	flux update $jobid duration=300 &&
+	flux job info $jobid R | jq -e ".execution.expiration > 0.0" &&
+	flux job info $jobid eventlog &&
+	flux cancel $jobid
+'
+
+test_expect_success 'flux job info --base R works' '
+	flux job info --base $(cat updated_R.id) R \
+	     | jq -e ".execution.expiration == 0.0"
+'
+
+#
+# job info lookup tests (via eventlog)
+#
+
+test_expect_success 'flux job eventlog works' '
+	jobid=$(submit_job) &&
+	flux job eventlog $jobid > eventlog_a.out &&
+	grep submit eventlog_a.out
+'
+
+test_expect_success 'flux job eventlog works on multiple entries' '
+	jobid=$(submit_job) &&
+	kvsdir=$(flux job id --to=kvs $jobid) &&
+	flux kvs eventlog append ${kvsdir}.eventlog foo &&
+	flux job eventlog $jobid >eventlog_b.out &&
+	grep -q submit eventlog_b.out &&
+	grep -q foo eventlog_b.out
+'
+
+test_expect_success 'flux job eventlog fails on bad id' '
+	test_must_fail flux job eventlog 12345
+'
+
+test_expect_success 'flux job eventlog --format=json works' '
+	jobid=$(submit_job) &&
+	flux job eventlog --format=json $jobid > eventlog_format1.out &&
+	grep -q "\"name\":\"submit\"" eventlog_format1.out &&
+	grep -q "\"userid\":$(id -u)" eventlog_format1.out
+'
+
+test_expect_success 'flux job eventlog --format=text works' '
+	jobid=$(submit_job) &&
+	flux job eventlog --format=text $jobid > eventlog_format2.out &&
+	grep -q "submit" eventlog_format2.out &&
+	grep -q "userid=$(id -u)" eventlog_format2.out
+'
+
+test_expect_success 'flux job eventlog --format=invalid fails' '
+	jobid=$(submit_job) &&
+	test_must_fail flux job eventlog --format=invalid $jobid
+'
+
+test_expect_success 'flux job eventlog --time-format=raw works' '
+	jobid=$(submit_job) &&
+	flux job eventlog --time-format=raw $jobid > eventlog_time_format1.out &&
+	get_timestamp_field submit eventlog_time_format1.out | grep "\."
+'
+
+test_expect_success 'flux job eventlog --time-format=iso works' '
+	jobid=$(submit_job) &&
+	flux job eventlog --time-format=iso $jobid > eventlog_time_format2.out &&
+	get_timestamp_field submit eventlog_time_format2.out | grep T | grep Z
+'
+
+test "$(TZ=America/Los_Angeles date +%z)" != "+0000" &&
+  test "$(TZ=America/Los_Angeles date +%Z)" != "UTC" &&
+  test_set_prereq HAVE_TZ
+test_expect_success HAVE_TZ 'flux job eventlog --time-format=iso uses local time' '
+	jobid=$(submit_job) &&
+	TZ=America/Los_Angeles \
+	    flux job eventlog --time-format=iso $jobid > tzlocal.out &&
+	test_debug "cat tzlocal.out" &&
+	get_timestamp_field submit tzlocal.out \
+	    | grep -E \
+	    "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+-[0-9]{2}:[0-9]{2}"
+'
+
+test_expect_success 'flux job eventlog --time-format=offset works' '
+	jobid=$(submit_job) &&
+	flux job eventlog --time-format=offset $jobid > eventlog_time_format3.out &&
+	get_timestamp_field submit eventlog_time_format3.out | grep "0.000000" &&
+	get_timestamp_field exception eventlog_time_format3.out | grep -v "0.000000"
+'
+
+test_expect_success 'flux job eventlog --time-format=invalid fails works' '
+	jobid=$(submit_job) &&
+	test_must_fail flux job eventlog --time-format=invalid $jobid
+'
+
+test_expect_success 'flux job eventlog -p works' '
+	jobid=$(submit_job) &&
+	flux job eventlog -p "eventlog" $jobid > eventlog_path1.out &&
+	grep submit eventlog_path1.out
+'
+
+test_expect_success 'flux job eventlog -p works (guest.exec.eventlog)' '
+	jobid=$(submit_job) &&
+	flux job eventlog -p "guest.exec.eventlog" $jobid > eventlog_path2.out &&
+	grep done eventlog_path2.out
+'
+test_expect_success 'flux job eventlog -p works (exec)' '
+	jobid=$(submit_job) &&
+	flux job eventlog -p exec $jobid > eventlog_path3.out &&
+	grep done eventlog_path3.out
+'
+test_expect_success 'flux job eventlog -p works (output)' '
+	jobid=$(submit_job) &&
+	flux job eventlog -p output $jobid > eventlog_path4.out &&
+	grep encoding eventlog_path4.out
+'
+test_expect_success 'flux job eventlog -p works (output)' '
+	jobid=$(submit_job) &&
+	flux job eventlog -p input $jobid > eventlog_path5.out &&
+	grep encoding eventlog_path5.out
+'
+test_expect_success 'flux job eventlog -p fails on invalid path' '
+	jobid=$(submit_job) &&
+	test_must_fail flux job eventlog -p "foobar" $jobid
+'
+# submit job in separate test to avoid using the form:
+#  jobid=$(flux submit ..) && flux job eventlog $jobid ... &
+# which will pick up the wrong $jobid since the pipeline is placed into
+# the background:
+test_expect_success 'submit held job for eventlog follow test' '
+	jobid=$(flux submit --urgency=hold -n1 hostname)
+'
+test_expect_success NO_CHAIN_LINT 'flux job eventlog -F, --follow works' '
+	flux job eventlog -HF $jobid >eventlog-follow.out &
+	pid=$! &&
+	flux job urgency $jobid default &&
+	wait $pid &&
+	test_debug "cat eventlog-follow.out" &&
+	grep clean eventlog-follow.out
+'
+#
+#  Color and human-readable output tests for flux job eventlog/wait-event
+#  
+test_expect_success 'flux job eventlog -H, --human works' '
+	#
+        #  Note: --human option should format first timestamp of eventlog
+        #   as [MmmDD HH:MM] and second line should be an offset thereof
+        #   e.g. [  +0.NNNNNN]. The following regexes attempt to verify
+        #   that --human produced this pattern.
+        #
+        flux job eventlog --human $jobid | sed -n 1p \
+            | grep "^\[[A-Z][a-z][a-z][0-3][0-9] [0-9][0-9]:[0-9][0-9]\]" &&
+        flux job eventlog --human $jobid | sed -n 2p \
+            | grep "^\[ *+[0-9]*\.[0-9]*\]"
+'
+test_expect_success 'flux job wait-event -H, --human works' '
+	#
+        #  As above, but here we also verify that wait-event displays
+	#   the datetime form on first matching event when -v is not used
+	#
+        flux job wait-event -v --human $jobid depend | sed -n 1p \
+            | grep "^\[[A-Z][a-z][a-z][0-3][0-9] [0-9][0-9]:[0-9][0-9]\]" &&
+        flux job wait-event -v --human $jobid depend | sed -n 2p \
+            | grep "^\[ *+[0-9]*\.[0-9]*\]" &&
+        flux job wait-event --human $jobid depend \
+            | grep "^\[[A-Z][a-z][a-z][0-3][0-9] [0-9][0-9]:[0-9][0-9]\]"
+'
+has_color() {
+	# To grep for ansi escape we need the help of the non-shell builtin
+  	# printf(1), so run under env(1) so we don't get shell builtin:
+        grep "$(env printf "\x1b\[")" $1 >/dev/null
+}
+test_expect_success 'flux job eventlog/wait-event reject invalid --color' '
+        test_must_fail flux job eventlog --color=foo $jobid &&
+        test_must_fail flux job wait-event --color=foo $jobid
+'
+for opt in "-L" "-Lalways" "--color" "--color=always"; do
+        test_expect_success "flux job eventlog $opt forces color on" '
+                name=notty${opt##--color=} &&
+                outfile=color-${name:-default}.out &&
+                flux job eventlog ${opt} $jobid >$outfile &&
+                test_debug "cat $outfile" &&
+                has_color $outfile
+        '
+        test_expect_success "flux job wait-event $opt forces color on" '
+                name=notty${opt##--color=} &&
+                outfile=color-${name:-default}.wait-event.out &&
+                flux job wait-event ${opt} $jobid submit >$outfile &&
+                test_debug "cat $outfile" &&
+                has_color $outfile
+        '
+done
+for opt in "" "--color" "--color=always" "--color=auto" "-H"; do
+        test_expect_success "flux job eventlog $opt displays color on tty" '
+                name=${opt##--color=} &&
+                outfile=color-${name:-default}.out &&
+                runpty.py flux job eventlog ${opt} $jobid >$outfile &&
+                test_debug "cat $outfile" &&
+                has_color $outfile
+        '
+        test_expect_success "flux kvs eventlog wait-event $opt displays color on tty" '
+                name=${opt##--color=} &&
+                outfile=color-${name:-default}.wait-event.out &&
+                runpty.py flux job wait-event ${opt} $jobid submit >$outfile &&
+                test_debug "cat $outfile" &&
+                has_color $outfile
+        '
+done
+test_expect_success "flux job eventlog --color=never disables color on tty" '
+        opt="--color=never" &&
+        name=${opt##--color=} &&
+        outfile=color-${name:-default}.out &&
+        runpty.py flux job eventlog ${opt} $jobid >$outfile &&
+        test_debug "cat $outfile" &&
+        test_must_fail has_color $outfile
+'
+
+#
+# job info lookup tests (multiple keys in info request)
+#
+
+test_expect_success 'job-info.lookup multiple keys works (different keys)' '
+	jobid=$(submit_job) &&
+	${INFOLOOKUP} $jobid eventlog jobspec J > all_info_a.out &&
+	grep submit all_info_a.out &&
+	grep sleep all_info_a.out
+'
+
+test_expect_success 'job-info.lookup multiple keys works (same key)' '
+	jobid=$(submit_job) &&
+	${INFOLOOKUP} $jobid eventlog eventlog eventlog > eventlog_3.out &&
+	test $(grep submit eventlog_3.out | wc -l) -eq 3
+'
+
+test_expect_success 'job-info.lookup multiple keys fails on bad id' '
+	test_must_fail ${INFOLOOKUP} 12345 eventlog jobspec J
+'
+
+test_expect_success 'job-info.lookup multiple keys fails on 1 bad entry' '
+	jobid=$(submit_job) &&
+	kvsdir=$(flux job id --to=kvs $jobid) &&
+	flux kvs unlink ${kvsdir}.jobspec &&
+	test_must_fail ${INFOLOOKUP} $jobid eventlog jobspec J > all_info_b.out
+'
+
+#
+# job info json decode tests
+#
+
+test_expect_success 'job-info.lookup: decode non-json returns string' '
+	jobid=$(submit_job) &&
+	${INFOLOOKUP} --json-decode $jobid eventlog > info_decode_1.out &&
+	grep submit info_decode_1.out
+'
+
+test_expect_success 'job-info.lookup: decode json returns object' '
+	jobid=$(submit_job) &&
+	${INFOLOOKUP} --json-decode $jobid jobspec > info_decode_2.out &&
+	grep sleep info_decode_2.out
+'
+
+#
+# stats & corner cases
+#
+
+test_expect_success 'job-info lookup stats works' '
+	flux module stats --parse lookups job-info
+'
+
+test_expect_success 'lookup request with empty payload fails with EPROTO(71)' '
+	${RPC} job-info.lookup 71 </dev/null
+'
+
+test_expect_success 'lookup request with keys not and array fails with EPROTO(71)' '
+	$jq -j -c -n  "{id:12345, keys:1, flags:0}" \
+	  | ${RPC} job-info.lookup 71
+'
+
+test_expect_success 'lookup request with invalid keys fails with EPROTO(71)' '
+	$jq -j -c -n  "{id:12345, keys:[1], flags:0}" \
+	  | ${RPC} job-info.lookup 71
+'
+
+test_expect_success 'lookup request with invalid flags fails with EPROTO(71)' '
+	$jq -j -c -n  "{id:12345, keys:[\"jobspec\"], flags:8191}" \
+	  | ${RPC} job-info.lookup 71
+'
+
+#
+# issue 6325
+#
+test_expect_success 'flux job info on bad job id gives good error message' '
+	test_must_fail flux job info fuzzybunny R 2>fuzzy.err &&
+	grep "invalid job id" fuzzy.err
+'
+test_expect_success 'flux job info on bad key gives good error message' '
+	test_must_fail flux job info $(flux job last) badkey 2>badkey.err &&
+	grep "key not found" badkey.err
+'
+
+test_done
diff --git a/t/t2231-job-info-eventlog-watch.t b/t/t2231-job-info-eventlog-watch.t
new file mode 100755
index 000000000000..e18a0d24382c
--- /dev/null
+++ b/t/t2231-job-info-eventlog-watch.t
@@ -0,0 +1,451 @@
+#!/bin/sh
+
+test_description='Test flux job info service eventlog watch'
+
+. `dirname $0`/kvs/kvs-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 job
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+RPC_STREAM=${FLUX_BUILD_DIR}/t/request/rpc_stream
+
+fj_wait_event() {
+	flux job wait-event --timeout=20 "$@"
+}
+
+# Usage: submit_job
+# To ensure robustness of tests despite future job manager changes,
+# cancel the job, and wait for clean event.
+submit_job() {
+	local jobid=$(flux job submit sleeplong.json) &&
+	fj_wait_event $jobid start >/dev/null &&
+	flux cancel $jobid &&
+	fj_wait_event $jobid clean >/dev/null &&
+	echo $jobid
+}
+
+# Unlike above, do not cancel the job, the test will cancel the job
+submit_job_live() {
+	local jobspec=$1
+	local jobid=$(flux job submit $jobspec) &&
+	fj_wait_event $jobid start >/dev/null &&
+	echo $jobid
+}
+
+# Test will cancel the job, is assumed won't run immediately
+submit_job_wait() {
+	local jobid=$(flux job submit sleeplong.json) &&
+	fj_wait_event $jobid depend >/dev/null &&
+	echo $jobid
+}
+
+wait_watchers_nonzero() {
+	local str=$1
+	local i=0
+	while (! flux module stats --parse $str job-info > /dev/null 2>&1 \
+		|| [ "$(flux module stats --parse $str job-info 2> /dev/null)" = "0" ]) \
+		&& [ $i -lt 50 ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "50" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+get_timestamp_field() {
+	local field=$1
+	local file=$2
+	grep $field $file | awk '{print $1}'
+}
+
+test_expect_success 'job-info: generate jobspec for simple test job' '
+	flux run --dry-run -n1 -N1 sleep 300 > sleeplong.json
+'
+
+test_expect_success 'submit and cancel job for test use' '
+	JOBID=$(submit_job)
+'
+
+test_expect_success 'flux job wait-event works' '
+	fj_wait_event $JOBID submit > wait_event1.out &&
+	grep submit wait_event1.out
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job wait-event errors on non-event' '
+	test_must_fail fj_wait_event $JOBID foobar 2> wait_event2.err &&
+	grep "never received" wait_event2.err
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job wait-event does not see event after clean' '
+	kvsdir=$(flux job id --to=kvs $JOBID) &&
+	flux kvs eventlog append ${kvsdir}.eventlog foobar &&
+	test_must_fail fj_wait_event -v $JOBID foobar 2> wait_event3.err &&
+	grep "never received" wait_event3.err
+'
+
+test_expect_success 'flux job wait-event fails on bad id' '
+	test_must_fail fj_wait_event 12345 foobar
+'
+
+test_expect_success 'flux job wait-event --quiet works' '
+	fj_wait_event --quiet $JOBID submit > wait_event4.out &&
+	! test -s wait_event4.out
+'
+
+test_expect_success 'flux job wait-event --verbose works' '
+	fj_wait_event --verbose $JOBID clean > wait_event5.out &&
+	grep submit wait_event5.out &&
+	grep start wait_event5.out &&
+	grep clean wait_event5.out
+'
+
+test_expect_success 'flux job wait-event --verbose doesnt show events after wait event' '
+	fj_wait_event --verbose $JOBID submit > wait_event6.out &&
+	grep submit wait_event6.out &&
+	! grep start wait_event6.out &&
+	! grep clean wait_event6.out
+'
+
+test_expect_success 'flux job wait-event --timeout works' '
+	jobid=$(submit_job_live sleeplong.json) &&
+	test_must_fail flux job wait-event --timeout=0.2 $jobid clean 2> wait_event7.err &&
+	flux cancel $jobid &&
+	grep "wait-event timeout" wait_event7.err
+'
+
+test_expect_success 'flux job wait-event --format=json works' '
+	fj_wait_event --format=json $JOBID submit > wait_event_format1.out &&
+	grep -q "\"name\":\"submit\"" wait_event_format1.out &&
+	grep -q "\"userid\":$(id -u)" wait_event_format1.out
+'
+
+test_expect_success 'flux job wait-event --format=text works' '
+	fj_wait_event --format=text $JOBID submit > wait_event_format2.out &&
+	grep -q "submit" wait_event_format2.out &&
+	grep -q "userid=$(id -u)" wait_event_format2.out
+'
+
+test_expect_success 'flux job wait-event --format=invalid fails' '
+	test_must_fail fj_wait_event --format=invalid $JOBID submit
+'
+
+test_expect_success 'flux job wait-event --time-format=raw works' '
+	fj_wait_event --time-format=raw $JOBID submit > wait_event_time_format1.out &&
+	get_timestamp_field submit wait_event_time_format1.out | grep "\."
+'
+
+test_expect_success 'flux job wait-event --time-format=iso works' '
+	fj_wait_event --time-format=iso $JOBID submit > wait_event_time_format2.out &&
+	get_timestamp_field submit wait_event_time_format2.out | grep T | grep Z
+'
+
+test_expect_success 'flux job wait-event --time-format=offset works' '
+	fj_wait_event --time-format=offset $JOBID submit > wait_event_time_format3A.out &&
+	get_timestamp_field submit wait_event_time_format3A.out | grep "0.000000" &&
+	fj_wait_event --time-format=offset $JOBID exception > wait_event_time_format3B.out &&
+	get_timestamp_field exception wait_event_time_format3B.out | grep -v "0.000000"
+'
+
+test_expect_success 'flux job wait-event --time-format=invalid fails works' '
+	test_must_fail fj_wait_event --time-format=invalid $JOBID submit
+'
+
+test_expect_success 'flux job wait-event w/ match-context works (string w/ quotes)' '
+	fj_wait_event --match-context="type=\"cancel\"" $JOBID exception > wait_event_context1.out &&
+	grep -q "exception" wait_event_context1.out &&
+	grep -q "type=\"cancel\"" wait_event_context1.out
+'
+
+test_expect_success 'flux job wait-event w/ match-context works (string w/o quotes)' '
+	fj_wait_event --match-context=type=cancel $JOBID exception > wait_event_context2.out &&
+	grep -q "exception" wait_event_context2.out &&
+	grep -q "type=\"cancel\"" wait_event_context2.out
+'
+
+test_expect_success 'flux job wait-event w/ match-context works (int)' '
+	fj_wait_event --match-context=flags=0 $JOBID submit > wait_event_context3.out &&
+	grep -q "submit" wait_event_context3.out &&
+	grep -q "flags=0" wait_event_context3.out
+'
+
+test_expect_success 'flux job wait-event w/ bad match-context fails (invalid key)' '
+	test_must_fail fj_wait_event --match-context=foo=bar $JOBID exception
+'
+
+test_expect_success 'flux job wait-event w/ bad match-context fails (invalid value)' '
+	test_must_fail fj_wait_event --match-context=type=foo $JOBID exception
+'
+
+# Note: in test below, foo=0 would match severity=0 in buggy version
+test_expect_success 'flux job wait-event w/ bad match-context fails (issue #5845)' '
+	test_must_fail fj_wait_event --match-context=foo=0 $JOBID exception
+'
+
+test_expect_success 'flux job wait-event w/ bad match-context fails (invalid input)' '
+	test_must_fail fj_wait_event --match-context=foo $JOBID exception
+'
+
+test_expect_success 'flux job wait-event -p works (eventlog)' '
+	fj_wait_event -p "eventlog" $JOBID submit > wait_event_path1.out &&
+	grep submit wait_event_path1.out
+'
+
+test_expect_success 'flux job wait-event -p works (guest.exec.eventlog)' '
+	fj_wait_event -p "guest.exec.eventlog" $JOBID done > wait_event_path2.out &&
+	grep done wait_event_path2.out
+'
+test_expect_success 'flux job wait-event -p works (exec)' '
+	fj_wait_event -p exec $JOBID done > wait_event_path2.1.out &&
+	grep done wait_event_path2.1.out
+'
+test_expect_success 'flux job wait-event -p works (non-guest eventlog)' '
+	kvsdir=$(flux job id --to=kvs $JOBID) &&
+	flux kvs eventlog append ${kvsdir}.foobar.eventlog foobar &&
+	fj_wait_event -p "foobar.eventlog" $JOBID foobar > wait_event_path3.out &&
+	grep foobar wait_event_path3.out
+'
+
+test_expect_success 'flux job wait-event -p fails on invalid path' '
+	test_must_fail fj_wait_event -p "foobar" $JOBID submit
+'
+
+test_expect_success 'flux job wait-event -p fails on invalid guest path' '
+	test_must_fail fj_wait_event -p "guest.foobar" $JOBID submit
+'
+
+test_expect_success 'flux job wait-event -p fails on path "guest."' '
+	test_must_fail fj_wait_event -p "guest." $JOBID submit
+'
+
+test_expect_success 'flux job wait-event -p and --waitcreate works (exec)' '
+	fj_wait_event --waitcreate -p exec $JOBID done > wait_event_path4.out &&
+	grep done wait_event_path4.out
+'
+
+test_expect_success 'flux job wait-event -p invalid and --waitcreate fails' '
+	test_must_fail fj_wait_event --waitcreate -p "guest.invalid" $JOBID foobar 2> waitcreate1.err &&
+	grep "not found" waitcreate1.err
+'
+
+test_expect_success 'flux job wait-event -p hangs on non-guest eventlog' '
+	kvsdir=$(flux job id --to=kvs $JOBID) &&
+	flux kvs eventlog append ${kvsdir}.foobar.eventlog foo &&
+	test_expect_code 142 run_timeout -s ALRM 0.2 flux job wait-event -p "foobar.eventlog" $JOBID bar
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job wait-event -p guest.exec.eventlog works (live job)' '
+	jobid=$(submit_job_live sleeplong.json)
+	fj_wait_event -p "guest.exec.eventlog" $jobid done > wait_event_path5.out &
+	waitpid=$! &&
+	wait_watchers_nonzero "watchers" &&
+	wait_watchers_nonzero "guest_watchers" &&
+	guestns=$(flux job namespace $jobid) &&
+	wait_watcherscount_nonzero $guestns &&
+	flux cancel $jobid &&
+	wait $waitpid &&
+	grep done wait_event_path5.out
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job wait-event -p guest.foobar and --waitcreate works (live job)' '
+	jobid=$(submit_job_live sleeplong.json)
+	fj_wait_event --waitcreate -p "guest.foobar" $jobid foobar > wait_event_path6.out &
+	waitpid=$! &&
+	wait_watchers_nonzero "watchers" &&
+	wait_watchers_nonzero "guest_watchers" &&
+	guestns=$(flux job namespace $jobid) &&
+	wait_watcherscount_nonzero $guestns &&
+	flux kvs eventlog append --namespace=${guestns} foobar foobar &&
+	flux cancel $jobid &&
+	wait $waitpid &&
+	grep foobar wait_event_path6.out
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job wait-event -p invalid and --waitcreate fails (live job)' '
+	jobid=$(submit_job_live sleeplong.json)
+	fj_wait_event --waitcreate -p "guest.invalid" $jobid foobar \
+		> wait_event_path6.out 2> wait_event_path6.err &
+	waitpid=$! &&
+	wait_watchers_nonzero "watchers" &&
+	wait_watchers_nonzero "guest_watchers" &&
+	guestns=$(flux job namespace $jobid) &&
+	wait_watcherscount_nonzero $guestns &&
+	flux cancel $jobid &&
+	! wait $waitpid &&
+	grep "not found" wait_event_path6.err
+'
+
+test_expect_success 'flux job wait-event -p times out on no event (live job)' '
+	jobid=$(submit_job_live sleeplong.json) &&
+	test_expect_code 142 run_timeout -s ALRM 0.2 \
+	    flux job wait-event -p exec $jobid foobar &&
+	flux cancel $jobid
+'
+
+test_expect_success 'flux job wait-event --count=0 errors' '
+	test_must_fail fj_wait_event --count=0 1234 foobar 2> count.err &&
+	grep "count must be" count.err
+'
+
+test_expect_success 'flux job wait-event --count=1 works' '
+	fj_wait_event --count=1 ${JOBID} clean
+'
+
+test_expect_success 'flux job wait-event --count=2 works' '
+	jobid=$(submit_job_wait) &&
+	kvsdir=$(flux job id --to=kvs $jobid) &&
+	flux kvs eventlog append ${kvsdir}.eventlog foobar &&
+	test_must_fail flux job wait-event --timeout=0.2 --count=2 ${jobid} foobar &&
+	flux kvs eventlog append ${kvsdir}.eventlog foobar &&
+	fj_wait_event --count=2 ${jobid} foobar &&
+	flux cancel $jobid
+'
+
+test_expect_success 'flux job wait-event --count=2 and context match works' '
+	jobid=$(submit_job_wait) &&
+	kvsdir=$(flux job id --to=kvs $jobid) &&
+	flux kvs eventlog append ${kvsdir}.eventlog foobar "{\"foo\":\"bar\"}" &&
+	test_must_fail flux job wait-event --timeout=0.2 --count=2 --match-context="foo=\"bar\"" ${jobid} foobar &&
+	flux kvs eventlog append ${kvsdir}.eventlog foobar "{\"foo\":\"bar\"}" &&
+	fj_wait_event --count=2 --match-context="foo=\"bar\"" ${jobid} foobar &&
+	flux cancel $jobid
+'
+
+test_expect_success 'flux job wait-event --count=2 and invalid context match fails' '
+	jobid=$(submit_job_wait) &&
+	kvsdir=$(flux job id --to=kvs $jobid) &&
+	flux kvs eventlog append ${kvsdir}.eventlog foobar "{\"foo\":\"bar\"}" &&
+	test_must_fail flux job wait-event --timeout=0.2 --count=2 --match-context="foo=\"dar\"" ${jobid} foobar &&
+	flux kvs eventlog append ${kvsdir}.eventlog foobar "{\"foo\":\"bar\"}" &&
+	test_must_fail flux job wait-event --timeout=0.2 --count=2 --match-context="foo=\"dar\"" ${jobid} foobar &&
+	flux cancel $jobid
+'
+
+# In order to test watching a guest event log that does not yet exist,
+# we will start a job that will take up all resources.  Then start
+# another job, which we will watch and know it hasn't started running
+# yet. Then we cancel the initial job to get the new one running.
+
+test_expect_success 'job-info: generate jobspec to consume all resources' '
+	flux run --dry-run -n4 -c2 sleep 300 > sleeplong-all-rsrc.json
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job wait-event -p exec works (wait job)' '
+	jobidall=$(submit_job_live sleeplong-all-rsrc.json)
+	jobid=$(submit_job_wait)
+	fj_wait_event -v -p exec ${jobid} done > wait_event_path7.out &
+	waitpid=$! &&
+	wait_watchers_nonzero "watchers" &&
+	wait_watchers_nonzero "guest_watchers" &&
+	flux cancel ${jobidall} &&
+	fj_wait_event ${jobid} start &&
+	guestns=$(flux job namespace ${jobid}) &&
+	wait_watcherscount_nonzero $guestns &&
+	flux cancel ${jobid} &&
+	wait $waitpid &&
+	grep done wait_event_path7.out
+'
+
+test_expect_success 'flux job wait-event -p times out on no event (wait job)' '
+	jobidall=$(submit_job_live sleeplong-all-rsrc.json) &&
+	jobid=$(submit_job_wait) &&
+	test_expect_code 142 run_timeout -s ALRM 0.2 \
+	    flux job wait-event -p exec $jobid foobar &&
+	flux cancel $jobidall &&
+	flux cancel $jobid
+'
+
+# In order to test watching a guest event log that will never exist,
+# we will start a job that will take up all resources.  Then start
+# another job, which we will watch and know it hasn't started running
+# yet. Then we cancel the second job before we know it has started.
+
+test_expect_success NO_CHAIN_LINT 'flux job wait-event -p guest.exec.eventlog works (never start job)' '
+	jobidall=$(submit_job_live sleeplong-all-rsrc.json)
+	jobid=$(submit_job_wait)
+	fj_wait_event -v -p exec ${jobid} done > wait_event_path8.out &
+	waitpid=$! &&
+	wait_watchers_nonzero "watchers" &&
+	wait_watchers_nonzero "guest_watchers" &&
+	flux cancel ${jobid} &&
+	! wait $waitpid &&
+	flux cancel ${jobidall}
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job wait-event -p and --waitcreate works (wait job)' '
+	jobidall=$(submit_job_live sleeplong-all-rsrc.json)
+	jobid=$(submit_job_wait)
+	fj_wait_event -v --waitcreate -p "guest.foobar" ${jobid} foobar > wait_event_path8.out &
+	waitpid=$! &&
+	wait_watchers_nonzero "watchers" &&
+	wait_watchers_nonzero "guest_watchers" &&
+	flux cancel ${jobidall} &&
+	fj_wait_event ${jobid} start &&
+	guestns=$(flux job namespace ${jobid}) &&
+	wait_watcherscount_nonzero $guestns &&
+	flux kvs eventlog append --namespace=${guestns} foobar foobar &&
+	flux cancel ${jobid} &&
+	wait $waitpid &&
+	grep foobar wait_event_path8.out
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job wait-event -p invalid and --waitcreate fails (wait job)' '
+	jobidall=$(submit_job_live sleeplong-all-rsrc.json)
+	jobid=$(submit_job_wait)
+	fj_wait_event -v --waitcreate -p "guest.invalid" ${jobid} invalid \
+		> wait_event_path9.out 2> wait_event_path9.err &
+	waitpid=$! &&
+	wait_watchers_nonzero "watchers" &&
+	wait_watchers_nonzero "guest_watchers" &&
+	flux cancel ${jobidall} &&
+	fj_wait_event ${jobid} start &&
+	guestns=$(flux job namespace ${jobid}) &&
+	wait_watcherscount_nonzero $guestns &&
+	flux cancel ${jobid} &&
+	! wait $waitpid &&
+	grep "not found" wait_event_path9.err
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job wait-event -p invalid and --waitcreate works (never start job)' '
+	jobidall=$(submit_job_live sleeplong-all-rsrc.json)
+	jobid=$(submit_job_wait)
+	fj_wait_event -v --waitcreate -p "guest.invalid" ${jobid} invalid \
+		> wait_event_path10.out 2> wait_event_path10.err &
+	waitpid=$! &&
+	wait_watchers_nonzero "watchers" &&
+	wait_watchers_nonzero "guest_watchers" &&
+	flux cancel ${jobid} &&
+	! wait $waitpid &&
+	grep "never received" wait_event_path10.err &&
+	flux cancel ${jobidall}
+'
+
+#
+# stats & corner cases
+#
+
+test_expect_success 'job-info stats works' '
+	flux module stats --parse watchers job-info &&
+	flux module stats --parse guest_watchers job-info
+'
+
+test_expect_success 'eventlog-watch request with empty payload fails with EPROTO(71)' '
+	${RPC} job-info.eventlog-watch 71 </dev/null
+'
+
+test_expect_success 'eventlog-watch request invalid flags fails' '
+	echo "{\"id\":42, \"path\":\"foobar\", \"flags\":499}" \
+		| test_must_fail ${RPC_STREAM} job-info.eventlog-watch 2> invalidflags.err &&
+	grep "invalid flag" invalidflags.err
+'
+test_expect_success 'eventlog-watch request non-streaming fails with EPROTO(71)' '
+	echo "{\"id\":42, \"path\":\"foobar\", \"flags\":0}" \
+		| ${RPC} job-info.eventlog-watch 71
+'
+
+test_done
diff --git a/t/t2232-job-info-security.t b/t/t2232-job-info-security.t
new file mode 100755
index 000000000000..58272fc78b9e
--- /dev/null
+++ b/t/t2232-job-info-security.t
@@ -0,0 +1,273 @@
+#!/bin/sh
+
+test_description='Test flux job info service security'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 job
+
+flux version | grep -q libflux-security && test_set_prereq FLUX_SECURITY
+
+# Usage: submit_as_alternate_user cmd...
+submit_as_alternate_user()
+{
+	FAKE_USERID=42
+        flux run --dry-run "$@" | \
+          flux python ${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py $FAKE_USERID \
+            >job.signed
+        FLUX_HANDLE_USERID=$FAKE_USERID \
+          flux job submit --flags=signed job.signed
+}
+
+set_userid() {
+	export FLUX_HANDLE_USERID=$1
+	export FLUX_HANDLE_ROLEMASK=0x2
+}
+
+unset_userid() {
+	unset FLUX_HANDLE_USERID
+	unset FLUX_HANDLE_ROLEMASK
+}
+
+test_expect_success 'configure to allow guests to use test execution' '
+	flux config load <<-EOT
+	exec.testexec.allow-guests = true
+	EOT
+'
+
+test_expect_success 'submit a job as instance owner and wait for clean' '
+	jobid=$(flux submit -n1 true) &&
+	flux job wait-event $jobid clean &&
+	echo $jobid >jobid.owner
+'
+
+test_expect_success FLUX_SECURITY 'submit a job as guest and wait for clean' '
+	jobid=$(submit_as_alternate_user -n1 true) &&
+	flux job wait-event $jobid clean &&
+	echo $jobid >jobid.guest
+'
+
+#
+# job lookup (via eventlog)
+#
+
+test_expect_success 'flux job eventlog works as instance owner' '
+	flux job eventlog $(cat jobid.owner)
+'
+
+test_expect_success FLUX_SECURITY 'flux job eventlog works as guest' '
+	set_userid 42 &&
+	flux job eventlog $(cat jobid.guest)
+'
+
+test_expect_success FLUX_SECURITY 'flux job eventlog fails as wrong guest' '
+	unset_userid &&
+	set_userid 9999 &&
+	test_must_fail flux job eventlog $(cat jobid.guest)
+'
+
+test_expect_success FLUX_SECURITY 'flux job eventlog as owner works on guest job' '
+	unset_userid &&
+	flux job eventlog $(cat jobid.guest)
+'
+
+test_expect_success 'flux job eventlog -p exec works as instance owner' '
+	unset_userid &&
+	flux job eventlog -p exec $(cat jobid.owner)
+'
+
+test_expect_success FLUX_SECURITY 'flux job eventlog -p exec works as guest' '
+	set_userid 42 &&
+	flux job eventlog -p exec $(cat jobid.guest)
+'
+
+test_expect_success FLUX_SECURITY 'flux job eventlog -p exec fails as wrong guest' '
+	unset_userid &&
+	set_userid 9999 &&
+	test_must_fail flux job eventlog -p exec $(cat jobid.guest)
+'
+
+#
+# job info lookup (via 'info')
+#
+
+test_expect_success 'flux job info eventlog works as instance owner' '
+	unset_userid &&
+	flux job info $(cat jobid.owner) eventlog
+'
+
+test_expect_success FLUX_SECURITY 'flux job info eventlog works as guest ' '
+	set_userid 42 &&
+	flux job info $(cat jobid.guest) eventlog
+'
+
+test_expect_success FLUX_SECURITY 'flux job info eventlog as owner works on guest job' '
+	unset_userid &&
+	flux job info $(cat jobid.guest) eventlog
+'
+
+test_expect_success FLUX_SECURITY 'flux job info eventlog fails as wrong guest' '
+	unset_userid &&
+	set_userid 9999 &&
+	test_must_fail flux job info $(cat jobid.guest) eventlog
+'
+
+test_expect_success 'flux job info jobspec works as instance owner' '
+	unset_userid &&
+	flux job info $(cat jobid.owner) jobspec
+'
+
+test_expect_success FLUX_SECURITY 'flux job info jobspec works as guest' '
+	unset_userid &&
+	set_userid 42 &&
+	flux job info $(cat jobid.guest) jobspec
+'
+
+test_expect_success FLUX_SECURITY 'flux job info jobspec fails as wrong guest' '
+	unset_userid &&
+	set_userid 9999 &&
+	test_must_fail flux job info $(cat jobid.guest) jobspec
+'
+
+test_expect_success 'flux job info foobar fails as instance owner' '
+	unset_userid &&
+	test_must_fail flux job info $(cat jobid.owner) foobar
+'
+
+test_expect_success 'flux job info foobar fails as guest' '
+	set_userid 42 &&
+	test_must_fail flux job info $(cat jobid.guest) foobar
+'
+
+test_expect_success 'flux job info foobar fails as wrong guest' '
+	unset_userid &&
+	set_userid 9999 &&
+	test_must_fail flux job info $(cat jobid.guest) foobar
+'
+
+#
+# job eventlog watch (via wait-event)
+#
+
+test_expect_success 'flux job wait-event works as instance owner' '
+	unset_userid &&
+	flux job wait-event $(cat jobid.owner) submit
+'
+
+test_expect_success FLUX_SECURITY 'flux job wait-event works guest' '
+	set_userid 42 &&
+	flux job wait-event $(cat jobid.guest) submit
+'
+
+test_expect_success FLUX_SECURITY 'flux job wait-event fails as wrong guest' '
+	unset_userid &&
+	set_userid 9999 &&
+	test_must_fail flux job wait-event $(cat jobid.guest) submit
+'
+
+test_expect_success 'flux job wait-event -p exec works as instance owner' '
+	unset_userid &&
+	flux job wait-event -p exec $(cat jobid.owner) done
+'
+
+test_expect_success FLUX_SECURITY 'flux job wait-event -p exec as guest' '
+	set_userid 42 &&
+	flux job wait-event -p exec $(cat jobid.guest) done
+'
+
+test_expect_success FLUX_SECURITY 'flux job wait-event -p exec fails as wrong guest' '
+	unset_userid &&
+	set_userid 9999 &&
+	test_must_fail flux job wait-event -p exec $(cat jobid.guest) done
+'
+
+test_expect_success 'flux job wait-event -p exec on running job works as instance owner' '
+	unset_userid &&
+	jobid=$(flux submit -n1 sleep 300) &&
+	flux job wait-event -p exec $jobid init &&
+	echo $jobid >jobid.running
+'
+
+test_expect_success 'cancel job' '
+	unset_userid &&
+	flux cancel $(cat jobid.running)
+'
+
+test_expect_success FLUX_SECURITY 'flux job wait-event -p exec on running job works as guest' '
+	unset_userid &&
+	jobid=$(submit_as_alternate_user -n1 \
+	    --setattr=system.exec.test.run_duration=5m true) &&
+	set_userid 42 &&
+	flux job wait-event -p exec $jobid init &&
+	echo $jobid >jobid-guest.running
+'
+
+test_expect_success FLUX_SECURITY 'flux job wait-event -p exec on running job fails as wrong guest' '
+	unset_userid &&
+	set_userid 9999 &&
+	test_must_fail flux job wait-event -p exec $(cat jobid-guest.running) init
+'
+
+test_expect_success FLUX_SECURITY 'cancel job' '
+	unset_userid &&
+	flux cancel $(cat jobid-guest.running)
+'
+
+#
+# job info invalid eventlog formatting corner case coverage
+#
+
+# for these tests we need to create a fake job eventlog in the KVS
+
+# value of "dummy" irrelevant here, just need to lookup something
+
+test_expect_success 'create empty eventlog for job' '
+	jobpath=`flux job id --to=kvs 123456789` &&
+	flux kvs put "${jobpath}.eventlog"="" &&
+	flux kvs put "${jobpath}.dummy"="foobar"
+'
+
+test_expect_success 'flux job info dummy fails as owner' '
+	test_must_fail flux job info $jobpath dummy 2>malevent.err &&
+	grep "error parsing eventlog" malevent.err
+'
+
+test_expect_success 'flux job info dummy fails as guest' '
+	set_userid 42 &&
+        test_must_fail flux job info 123456789 dummy 2>malevent2.err &&
+	grep "error parsing eventlog" malevent2.err
+'
+
+test_expect_success 'flux job info fails on malformed eventlog (not json)' '
+	unset_userid &&
+	jobpath=`flux job id --to=kvs 123456789` &&
+	flux kvs put "${jobpath}.eventlog"="foobar" &&
+        test_must_fail flux job info 123456789 dummy 2>malevent3.err &&
+	grep "error parsing eventlog" malevent3.err
+'
+
+test_expect_success 'flux job info fails on malformed eventlog (no submit event)' '
+        submitstr="{\"timestamp\":123.4,\"name\":\"submit\"}" &&
+	jobpath=`flux job id --to=kvs 123456789` &&
+	echo $submitstr | flux kvs put --raw "${jobpath}.eventlog"=- &&
+        test_must_fail flux job info 123456789 dummy 2>malevent4.err &&
+	grep "error parsing eventlog" malevent4.err
+'
+
+test_expect_success 'flux job info fails on malformed eventlog (no submit userid)' '
+        submitstr="{\"timestamp\":123.4,\"name\":\"submit\",\"context\":{}}" &&
+	jobpath=`flux job id --to=kvs 123456789` &&
+	echo $submitstr | flux kvs put --raw "${jobpath}.eventlog"=- &&
+        test_must_fail flux job info 123456789 dummy 2>malevent5.err &&
+	grep "error parsing eventlog" malevent5.err
+'
+
+test_expect_success 'flux job info fails on malformed eventlog (random garbage)' '
+	jobpath=`flux job id --to=kvs 123456789` &&
+        dd if=/dev/urandom bs=64 count=1 > binary.out &&
+	flux kvs put --raw "${jobpath}.eventlog"=- < binary.out &&
+        test_must_fail flux job info 123456789 dummy 2>malevent6.err &&
+	grep "error parsing eventlog" malevent6.err
+'
+
+test_done
diff --git a/t/t2233-job-info-update.t b/t/t2233-job-info-update.t
new file mode 100755
index 000000000000..a237893c4ca3
--- /dev/null
+++ b/t/t2233-job-info-update.t
@@ -0,0 +1,464 @@
+#!/bin/sh
+
+test_description='Test flux job info service lookup w/ update and update-watch'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 job
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+RPC_STREAM=${FLUX_BUILD_DIR}/t/request/rpc_stream
+INFO_LOOKUP=${FLUX_BUILD_DIR}/t/job-info/info_lookup
+UPDATE_LOOKUP=${FLUX_BUILD_DIR}/t/job-info/update_lookup
+UPDATE_WATCH=${FLUX_BUILD_DIR}/t/job-info/update_watch_stream
+WAITFILE="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+
+fj_wait_event() {
+	flux job wait-event --timeout=20 "$@"
+}
+
+get_update_watchers() {
+	flux module stats --parse "update_watchers" job-info
+}
+
+wait_update_watchers() {
+	local count=$1
+	local i=0
+	echo "waiting for $count watchers"
+	while (! flux module stats --parse "update_watchers" job-info \
+		|| [ "$(flux module stats --parse "update_watchers" job-info 2> /dev/null)" != "${count}" ]) \
+		&& [ $i -lt 50 ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "50" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+# Usage: expiration_add JOBID N
+# Set expiration of job with JOBID to now + N seconds
+#
+expiration_add() {
+	local id=$1
+	local amount=$2
+	local kvsdir=$(flux job id --to=kvs $id)
+	local new=$(($(date +%s) + $amount))
+	flux kvs eventlog append \
+	    ${kvsdir}.eventlog resource-update "{\"expiration\": $new}"
+	echo $new
+}
+
+# Usage: check_expiration jobid VALUE
+# Check and wait for expiration to reach VALUE.
+#
+check_expiration() {
+	local jobid=$1
+	local value=$2
+	local i=0
+	while (! ${INFO_LOOKUP} -c ${jobid} R | jq -e ".execution.expiration == ${value}" \
+		&& [ $i -lt 200 ] )
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "200" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+# Usage: check_expiration_legacy jobid VALUE
+# Check and wait for expiration to reach VALUE.
+#
+check_expiration_legacy() {
+	local jobid=$1
+	local value=$2
+	local i=0
+	while (! ${UPDATE_LOOKUP} ${jobid} R | jq -e ".execution.expiration == ${value}" \
+		&& [ $i -lt 200 ] )
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "200" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+test_expect_success 'job-info: lookup current with no update events works (job active)' '
+	jobid=$(flux submit --wait-event=start sleep 300) &&
+	check_expiration $jobid 0.0 &&
+	check_expiration_legacy $jobid 0.0 &&
+	flux cancel $jobid
+'
+
+test_expect_success 'job-info: lookup current with no update events works (job inactive)' '
+	jobid=$(flux submit --wait-event=clean hostname) &&
+	check_expiration $jobid 0.0 &&
+	check_expiration_legacy $jobid 0.0
+'
+
+test_expect_success 'job-info: lookup current with update events works (job active)' '
+	jobid=$(flux submit --wait-event=start sleep 300) &&
+	update1=$(expiration_add $jobid 100) &&
+	flux job eventlog $jobid &&
+	check_expiration $jobid ${update1} &&
+	check_expiration_legacy $jobid ${update1} &&
+	update2=$(expiration_add $jobid 200) &&
+	check_expiration $jobid ${update2} &&
+	check_expiration_legacy $jobid ${update2} &&
+	flux cancel $jobid
+'
+
+test_expect_success 'job-info: lookup current with update events works (job inactive)' '
+	jobid=$(flux submit --wait-event=start sleep 300) &&
+	update1=$(expiration_add $jobid 100) &&
+	update2=$(expiration_add $jobid 100) &&
+	flux cancel $jobid &&
+	check_expiration $jobid ${update2} &&
+	check_expiration_legacy $jobid ${update2}
+'
+
+test_expect_success 'job-info: update watch with no update events works (job inactive)' '
+	jobid=$(flux submit --wait-event=clean hostname) &&
+	${UPDATE_WATCH} $jobid R > watch1.out &&
+	test $(cat watch1.out | wc -l) -eq 1 &&
+	cat watch1.out | jq -e ".execution.expiration == 0.0"
+'
+
+test_expect_success NO_CHAIN_LINT 'job-info: update watch with no update events works (job run/canceled)' '
+	jobid=$(flux submit --wait-event=exec.shell.init sleep inf)
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid R > watch2.out &
+	watchpid=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	${WAITFILE} -v --count=1 --timeout=30 --pattern="expiration" watch2.out &&
+	flux cancel $jobid &&
+	wait $watchpid &&
+	test_debug "cat watch2.out" &&
+	test $(cat watch2.out | wc -l) -eq 1 &&
+	cat watch2.out | jq -e ".execution.expiration == 0.0"
+'
+
+test_expect_success 'job-info: update watch with update events works (job inactive)' '
+	jobid=$(flux submit --wait-event=start sleep inf) &&
+	update1=$(expiration_add $jobid 100) &&
+	update2=$(expiration_add $jobid 200) &&
+	flux cancel $jobid &&
+	${UPDATE_WATCH} $jobid R > watch3.out &&
+	test $(cat watch3.out | wc -l) -eq 1 &&
+	cat watch3.out | jq -e ".execution.expiration == ${update2}"
+'
+
+test_expect_success NO_CHAIN_LINT 'job-info: update watch with update events works (before watch)' '
+	jobid=$(flux submit --wait-event=exec.shell.init sleep inf) &&
+	update1=$(expiration_add $jobid 100) &&
+	update2=$(expiration_add $jobid 200) &&
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid R > watch4.out &
+	watchpid=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	${WAITFILE} --count=1 --timeout=30 --pattern="expiration" watch4.out &&
+	flux cancel $jobid &&
+	wait $watchpid &&
+	test $(cat watch4.out | wc -l) -eq 1 &&
+	cat watch4.out | jq -e ".execution.expiration == ${update2}"
+'
+
+test_expect_success NO_CHAIN_LINT 'job-info: update watch with update events works (after watch)' '
+	jobid=$(flux submit --wait-event=exec.shell.init sleep inf) &&
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid R > watch5.out &
+	watchpid=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	${WAITFILE} --count=1 --timeout=30 --pattern="expiration" watch5.out &&
+	update1=$(expiration_add $jobid 100) &&
+	update2=$(expiration_add $jobid 200) &&
+	${WAITFILE} --count=3 --timeout=30 --pattern="expiration" watch5.out &&
+	flux cancel $jobid &&
+	wait $watchpid &&
+	test $(cat watch5.out | wc -l) -eq 3 &&
+	head -n1 watch5.out | jq -e ".execution.expiration == 0.0" &&
+	head -n2 watch5.out | tail -n1 | jq -e ".execution.expiration == ${update1}" &&
+	tail -n1 watch5.out | jq -e ".execution.expiration == ${update2}"
+'
+
+test_expect_success NO_CHAIN_LINT 'job-info: update watch with update events works (before/after watch)' '
+	jobid=$(flux submit --wait-event=exec.shell.init sleep inf) &&
+	update1=$(expiration_add $jobid 100) &&
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid R > watch6.out &
+	watchpid=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	${WAITFILE} --count=1 --timeout=30 --pattern="expiration" watch6.out &&
+	update2=$(expiration_add $jobid 200) &&
+	${WAITFILE} --count=2 --timeout=30 --pattern="expiration" watch6.out &&
+	flux cancel $jobid &&
+	wait $watchpid &&
+	test $(cat watch6.out | wc -l) -eq 2 &&
+	head -n1 watch6.out | jq -e ".execution.expiration == ${update1}" &&
+	tail -n1 watch6.out | jq -e ".execution.expiration == ${update2}"
+'
+
+# signaling the update watch tool with SIGUSR1 will cancel the stream
+test_expect_success NO_CHAIN_LINT 'job-info: update watch can be canceled (single watcher)' '
+	jobid=$(flux submit --wait-event=exec.shell.init sleep inf) &&
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid R > watch7.out &
+	watchpid=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	${WAITFILE} --count=1 --timeout=30 --pattern="expiration" watch7.out &&
+	kill -s USR1 $watchpid &&
+	wait $watchpid &&
+	flux cancel $jobid &&
+	test $(cat watch7.out | wc -l) -eq 1 &&
+	cat watch7.out | jq -e ".execution.expiration == 0.0"
+'
+
+# This test may look tricky.  Basically we are updating the eventlog
+# with resource-update events once in awhile and starting watchers at
+# different point in times in the eventlog's parsing.
+#
+# At the end, the first watcher should see 3 R versions, the second
+# one 2, and the last one only 1.
+test_expect_success NO_CHAIN_LINT 'job-info: update watch with multiple watchers works' '
+	jobid=$(flux submit --wait-event=exec.shell.init sleep inf)
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid R > watch8A.out &
+	watchpidA=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	update1=$(expiration_add $jobid 100) &&
+	${WAITFILE} --count=2 --timeout=30 --pattern="expiration" watch8A.out
+	${UPDATE_WATCH} $jobid R > watch8B.out &
+	watchpidB=$! &&
+	wait_update_watchers $((watchers+2)) &&
+	update2=$(expiration_add $jobid 200) &&
+	${WAITFILE} --count=3 --timeout=30 --pattern="expiration" watch8A.out &&
+	${WAITFILE} --count=2 --timeout=30 --pattern="expiration" watch8B.out
+	${UPDATE_WATCH} $jobid R > watch8C.out &
+	watchpidC=$! &&
+	wait_update_watchers $((watchers+3)) &&
+	${WAITFILE} --count=1 --timeout=30 --pattern="expiration" watch8C.out &&
+	flux cancel $jobid &&
+	wait $watchpidA &&
+	wait $watchpidB &&
+	wait $watchpidC &&
+	test $(cat watch8A.out | wc -l) -eq 3 &&
+	test $(cat watch8B.out | wc -l) -eq 2 &&
+	test $(cat watch8C.out | wc -l) -eq 1 &&
+	head -n1 watch8A.out | jq -e ".execution.expiration == 0.0" &&
+	head -n2 watch8A.out | tail -n1 | jq -e ".execution.expiration == ${update1}" &&
+	tail -n1 watch8A.out | jq -e ".execution.expiration == ${update2}" &&
+	head -n1 watch8B.out | jq -e ".execution.expiration == ${update1}" &&
+	tail -n1 watch8B.out | jq -e ".execution.expiration == ${update2}" &&
+	tail -n1 watch8C.out | jq -e ".execution.expiration == ${update2}"
+'
+
+# signaling the update watch tool with SIGUSR1 will cancel the stream
+test_expect_success NO_CHAIN_LINT 'job-info: update watch can be canceled (multiple watchers)' '
+	jobid=$(flux submit --wait-event=exec.shell.init sleep inf)
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid R > watch10A.out &
+	watchpidA=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	update1=$(expiration_add $jobid 100)
+	${UPDATE_WATCH} $jobid R > watch10B.out &
+	watchpidB=$! &&
+	wait_update_watchers $((watchers+2)) &&
+	${WAITFILE} --count=2 --timeout=30 --pattern="expiration" watch10A.out &&
+	${WAITFILE} --count=1 --timeout=30 --pattern="expiration" watch10B.out &&
+	kill -s USR1 $watchpidA &&
+	wait $watchpidA &&
+	kill -s USR1 $watchpidB &&
+	wait $watchpidB &&
+	test_debug "echo watch10A: \"$(cat watch10A.out)\"" &&
+	test_debug "echo watch10B: \"$(cat watch10B.out)\"" &&
+	flux cancel $jobid &&
+	test $(cat watch10A.out | wc -l) -eq 2 &&
+	test $(cat watch10B.out | wc -l) -eq 1 &&
+	head -n1 watch10A.out | jq -e ".execution.expiration == 0.0" &&
+	tail -n1 watch10B.out | jq -e ".execution.expiration == ${update1}" &&
+	cat watch10B.out | jq -e ".execution.expiration == ${update1}"
+'
+
+# If someone is already doing an update-watch on a jobid/key, lookup w/ update can
+# return the cached info
+test_expect_success NO_CHAIN_LINT 'job-info: lookup current returns cached R from update watch' '
+	jobid=$(flux submit --wait-event=exec.shell.init sleep inf) &&
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid R > watch9.out &
+	watchpid=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	update1=$(expiration_add $jobid 100) &&
+	${WAITFILE} --count=2 --timeout=30 --pattern="expiration" watch9.out &&
+	${INFO_LOOKUP} -c $jobid R > lookup9A.out &&
+	${UPDATE_LOOKUP} $jobid R > lookup9B.out &&
+	flux cancel $jobid &&
+	wait $watchpid &&
+	test $(cat watch9.out | wc -l) -eq 2 &&
+	head -n1 watch9.out | jq -e ".execution.expiration == 0.0" &&
+	tail -n1 watch9.out | jq -e ".execution.expiration == ${update1}" &&
+	cat lookup9A.out | jq -e ".execution.expiration == ${update1}" &&
+	cat lookup9B.out | jq -e ".execution.expiration == ${update1}"
+'
+
+#
+# lookup w/ current and update-watch works with jobspec
+#
+
+# Usage: check_duration jobid VALUE
+# Check and wait for duration to reach VALUE.
+#
+check_duration() {
+       local jobid=$1
+       local value=$2
+       local i=0
+       while (! ${INFO_LOOKUP} -c ${jobid} jobspec | jq -e ".attributes.system.duration == ${value}" \
+	       && [ $i -lt 200 ] )
+       do
+	       sleep 0.1
+	       i=$((i + 1))
+       done
+       if [ "$i" -eq "200" ]
+       then
+	       return 1
+       fi
+       return 0
+}
+
+test_expect_success 'job-info: lookup current works with jobspec' '
+	jobid=$(flux submit --urgency=hold true) &&
+	check_duration $jobid 0 &&
+	flux update $jobid duration=100s &&
+	check_duration $jobid 100.0 &&
+	flux update $jobid duration=200s &&
+	check_duration $jobid 200.0 &&
+	flux cancel $jobid
+'
+
+test_expect_success NO_CHAIN_LINT 'job-info: update watch works with jobspec' '
+	jobid=$(flux submit --urgency=hold true) &&
+	flux update $jobid duration=100s &&
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid jobspec > watchjobspec.out &
+	watchpid=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	${WAITFILE} --count=1 --timeout=30 --pattern="duration" watchjobspec.out &&
+	flux update $jobid duration=200s &&
+	${WAITFILE} --count=2 --timeout=30 --pattern="duration" watchjobspec.out &&
+	flux cancel $jobid &&
+	wait $watchpid &&
+	test $(cat watchjobspec.out | wc -l) -eq 2 &&
+	head -n1 watchjobspec.out | jq -e ".attributes.system.duration == 100.0" &&
+	tail -n1 watchjobspec.out | jq -e ".attributes.system.duration == 200.0"
+'
+
+#
+# security tests
+#
+
+set_userid() {
+	export FLUX_HANDLE_USERID=$1
+	export FLUX_HANDLE_ROLEMASK=0x2
+}
+
+unset_userid() {
+	unset FLUX_HANDLE_USERID
+	unset FLUX_HANDLE_ROLEMASK
+}
+
+test_expect_success 'job-info: non job owner cannot lookup key' '
+	jobid=`flux submit --wait-event=start sleep inf` &&
+	set_userid 9999 &&
+	test_must_fail ${UPDATE_LOOKUP} $jobid R &&
+	test_must_fail ${INFO_LOOKUP} -u $jobid jobspec &&
+	unset_userid &&
+	flux cancel $jobid
+'
+
+test_expect_success 'job-info: non job owner cannot watch key' '
+	jobid=`flux submit --wait-event=start sleep inf` &&
+	set_userid 9999 &&
+	test_must_fail ${UPDATE_WATCH} $jobid R &&
+	test_must_fail ${UPDATE_WATCH} $jobid jobspec &&
+	unset_userid &&
+	flux cancel $jobid
+'
+
+# this test checks security on a second watcher, which is trying to
+# access cached info
+test_expect_success NO_CHAIN_LINT 'job-info: non job owner cannot watch key (second watcher)' '
+	jobid=`flux submit --wait-event=exec.shell.init sleep inf`
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid R > watchsecurity.out &
+	watchpid=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	${WAITFILE} --count=1 --timeout=30 --pattern="expiration" watchsecurity.out &&
+	set_userid 9999 &&
+	test_must_fail ${UPDATE_WATCH} $jobid R &&
+	unset_userid &&
+	flux cancel $jobid &&
+	wait $watchpid
+'
+
+# lookup current cannot read cached watch data
+test_expect_success NO_CHAIN_LINT 'job-info: non job owner cannot lookup key (piggy backed)' '
+	jobid=`flux submit --wait-event=exec.shell.init sleep inf`
+	watchers=$(get_update_watchers)
+	${UPDATE_WATCH} $jobid R > lookupsecurity.out &
+	watchpid=$! &&
+	wait_update_watchers $((watchers+1)) &&
+	${WAITFILE} --count=1 --timeout=30 --pattern="expiration" lookupsecurity.out &&
+	set_userid 9999 &&
+	test_must_fail ${INFO_LOOKUP} -c $jobid R &&
+	test_must_fail ${UPDATE_LOOKUP} $jobid R &&
+	unset_userid &&
+	flux cancel $jobid &&
+	wait $watchpid
+'
+
+#
+# stats & corner cases
+#
+
+test_expect_success 'job-info stats works' '
+	flux module stats --parse update_lookups job-info &&
+	flux module stats --parse update_watchers job-info
+'
+test_expect_success 'update-lookup request with empty payload fails with EPROTO(71)' '
+	${RPC} job-info.update-watch 71 </dev/null
+'
+test_expect_success 'update-watch request with empty payload fails with EPROTO(71)' '
+	${RPC} job-info.update-watch 71 </dev/null
+'
+test_expect_success 'update-lookup request with invalid key fails with EINVAL(22))' '
+	echo "{\"id\":42, \"key\":\"foobar\", \"flags\":0}" \
+		| ${RPC} job-info.update-lookup 22
+'
+test_expect_success 'update-watch request invalid key fails' '
+	echo "{\"id\":42, \"key\":\"foobar\", \"flags\":0}" \
+		| test_must_fail ${RPC_STREAM} job-info.update-watch 2> invalidkey.err && \
+	grep "unsupported key" invalidkey.err
+'
+test_expect_success 'update-lookup request with invalid flags fails with EPROTO(71))' '
+	echo "{\"id\":42, \"key\":\"R\", \"flags\":499}" \
+		| ${RPC} job-info.update-lookup 71
+'
+test_expect_success 'update-watch request invalid flags fails' '
+	echo "{\"id\":42, \"key\":\"R\", \"flags\":499}" \
+		| test_must_fail ${RPC_STREAM} job-info.update-watch 2> invalidflags.err &&
+	grep "invalid flag" invalidflags.err
+'
+test_expect_success 'update-watch request non-streaming fails with EPROTO(71)' '
+	echo "{\"id\":42, \"key\":\"R\", \"flags\":0}" \
+		| ${RPC} job-info.update-watch 71
+'
+
+test_done
diff --git a/t/t2240-queue-cmd.t b/t/t2240-queue-cmd.t
new file mode 100755
index 000000000000..50e549098e51
--- /dev/null
+++ b/t/t2240-queue-cmd.t
@@ -0,0 +1,604 @@
+#!/bin/sh
+test_description='Test flux queue command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 full
+
+flux setattr log-stderr-level 1
+
+LIST_JOBS=${FLUX_BUILD_DIR}/t/job-manager/list-jobs
+
+test_expect_success 'flux-queue: unknown sub-command fails with usage message' '
+	test_must_fail flux queue wrongsubcmd 2>usage.out &&
+	grep -i usage: usage.out
+'
+
+test_expect_success 'flux-queue: missing sub-command fails with usage message' '
+	test_must_fail flux queue 2>usage2.out &&
+	grep -i usage: usage2.out
+'
+
+test_expect_success 'flux-queue: disable with no reason fails' '
+	test_must_fail flux queue submit disable 2>usage3.out &&
+	grep -i usage: usage3.out
+'
+
+test_expect_success 'flux-queue: enable with extra free args fails' '
+	test_must_fail flux queue enable xyz 2>usage4.out &&
+	grep -i usage: usage4.out
+'
+
+test_expect_success 'flux-queue: status with extra free args fails' '
+	test_must_fail flux queue status xyz 2>usage5.out &&
+	grep -i usage: usage5.out
+'
+
+test_expect_success 'flux-queue: drain with extra free args fails' '
+	test_must_fail flux queue drain xyz 2>usage6.out &&
+	grep -i usage: usage6.out
+'
+
+test_expect_success 'flux-queue: idle with extra free args fails' '
+	test_must_fail flux queue idle xyz 2>usage7.out &&
+	grep -i usage: usage7.out
+'
+
+test_expect_success 'flux-queue: status with bad broker connection fails' '
+	! FLUX_URI=/wrong flux queue status
+'
+
+test_expect_success 'flux-queue: disable with bad broker connection fails' '
+	! FLUX_URI=/wrong flux queue disable foo
+'
+
+test_expect_success 'flux-queue: enable with bad broker connection fails' '
+	! FLUX_URI=/wrong flux queue enable
+'
+
+test_expect_success 'flux-queue: drain with bad broker connection fails' '
+	! FLUX_URI=/wrong flux queue drain
+'
+
+test_expect_success 'flux-queue: idle with bad broker connection fails' '
+	! FLUX_URI=/wrong flux queue idle
+'
+
+test_expect_success 'flux-queue: disable works' '
+	flux queue disable system is fubar > disable.out &&
+	grep "^Job submission is disabled" disable.out &&
+	grep "system is fubar" disable.out
+'
+
+test_expect_success 'flux-queue: job submit fails with queue disabled' '
+	test_must_fail flux run true
+'
+
+test_expect_success 'flux-queue: enable works' '
+	flux queue enable > enable.out &&
+	grep "^Job submission is enabled" enable.out
+'
+
+test_expect_success 'flux-queue: flux run works after enable' '
+	run_timeout 60 flux run true
+'
+
+test_expect_success 'flux-queue: stop with bad broker connection fails' '
+	! FLUX_URI=/wrong flux queue stop
+'
+
+test_expect_success 'flux-queue: start with bad broker connection fails' '
+	! FLUX_URI=/wrong flux queue start
+'
+
+test_expect_success 'flux-queue: start with extra free args fails' '
+	test_must_fail flux queue start xyz 2>start_xargs.err &&
+	grep -i usage: start_xargs.err
+'
+
+test_expect_success 'flux-queue: stop works' '
+	flux queue stop my unique message > stop.out &&
+	grep "^Scheduling is stopped" stop.out &&
+	grep "my unique message" stop.out
+'
+
+test_expect_success 'flux-queue: status reports reason for stop' '
+	flux queue status >status.out &&
+	cat <<-EOT >status.exp &&
+	Job submission is enabled
+	Scheduling is stopped: my unique message
+	EOT
+	test_cmp status.exp status.out
+'
+
+test_expect_success 'flux-queue: stop works without message' '
+	flux queue stop > stop2.out &&
+	grep "^Scheduling is stopped" stop2.out
+'
+
+test_expect_success 'flux-queue: status reports no reason for stop' '
+	flux queue status >status2.out &&
+	cat <<-EOT >status2.exp &&
+	Job submission is enabled
+	Scheduling is stopped
+	EOT
+	test_cmp status2.exp status2.out
+'
+
+test_expect_success 'flux-queue: stop with --nocheckpoint works' '
+	flux start \
+	    -Scontent.dump=dump_queue_nocheckpoint1.tar \
+	    flux queue stop &&
+	tar -xvf dump_queue_nocheckpoint1.tar &&
+	cat checkpoint/job-manager | jq -e ".queue[0].start == false" &&
+	flux start \
+	    -Scontent.dump=dump_queue_nocheckpoint2.tar \
+	    flux queue stop --nocheckpoint &&
+	tar -xvf dump_queue_nocheckpoint2.tar &&
+	cat checkpoint/job-manager | jq -e ".queue[0].start == true"
+'
+
+test_expect_success 'flux-queue: submit some jobs' '
+	flux submit --cc 1-3 --wait-event=priority true
+'
+
+test_expect_success 'flux-queue: start scheduling' '
+	flux queue start > start.out &&
+	grep "^Scheduling is started" start.out
+'
+
+test_expect_success 'flux-queue: queue empties out' '
+	run_timeout 60 flux queue drain
+'
+
+test_expect_success 'flux-queue: start long job that uses all cores' '
+	ncores=$(flux resource list -s up -no {ncores}) &&
+	id=$(flux submit -n ${ncores} sleep 600) &&
+	flux job wait-event ${id} start &&
+	echo ${id} >longjob
+'
+
+test_expect_success 'flux-queue: submit a job and make sure alloc sent' '
+	id=$(flux submit --flags debug true) &&
+	flux job wait-event ${id} debug.alloc-request
+'
+
+wait_for_pending_cancel () {
+	local n=$1
+	local count=$2
+	for try in $(seq 1 $n); do
+		echo Check queue pending count, try ${try} of $n 2>&1
+		flux queue status -v 2>&1 \
+		  | grep "${count} alloc requests pending to scheduler" \
+		  && return
+	done
+}
+
+# internally cancel is sent to scheduler, can be racy when scheduler responds
+# to cancel request, must wait for it to be returned
+test_expect_success 'flux-queue: stop canceled alloc request' '
+	flux queue stop &&
+	wait_for_pending_cancel 10 0
+'
+
+test_expect_success 'flux-queue: start scheduling and cancel long job' '
+	flux queue start &&
+	flux cancel $(cat longjob)
+'
+
+test_expect_success 'flux-queue: queue empties out' '
+	flux queue drain
+'
+
+test_expect_success 'flux-queue: unload scheduler' '
+	flux module remove sched-simple
+'
+
+test_expect_success 'flux-queue: submit a job to ping scheduler' '
+	flux submit --flags debug true
+'
+
+wait_for_sched_offline() {
+	local n=$1
+	for try in $(seq 1 $n); do
+		echo Check queue status for offline, try ${try} of $n 2>&1
+		flux queue status 2>&1 | grep stopped && return
+	done
+}
+
+test_expect_success 'flux-queue: queue says scheduling stopped' '
+	wait_for_sched_offline 10 &&
+	flux queue status >sched_stat.out &&
+	cat <<-EOT >sched_stat.exp &&
+	Job submission is enabled
+	Scheduling is stopped: Scheduler is offline
+	EOT
+	test_cmp sched_stat.exp sched_stat.out
+'
+
+test_expect_success 'flux-queue: queue contains 1 active job' '
+	COUNT=$(${LIST_JOBS} | wc -l) &&
+	test ${COUNT} -eq 1
+'
+
+test_expect_success 'flux-queue: load scheduler' '
+	flux module load sched-simple mode=limited=1
+'
+
+test_expect_success 'flux-queue: queue says scheduling is enabled' '
+	flux queue status >sched_stat2.out &&
+	cat <<-EOT >sched_stat2.exp &&
+	Job submission is enabled
+	Scheduling is started
+	EOT
+	test_cmp sched_stat2.exp sched_stat2.out
+'
+
+test_expect_success 'flux-queue: job in queue ran' '
+	run_timeout 30 flux queue drain
+'
+
+test_expect_success 'flux-queue: submit a long job that uses all cores' '
+	ncores=$(flux resource list -s up -no {ncores}) &&
+	flux submit -n ${ncores} sleep 600
+'
+
+test_expect_success 'flux-queue: submit 2 more jobs' '
+	flux submit true &&
+	flux submit true
+'
+
+test_expect_success 'flux-queue: there are 3 active jobs' '
+	COUNT=$(${LIST_JOBS} | wc -l) &&
+	test ${COUNT} -eq 3
+'
+
+test_expect_success 'flux-queue: queue status -v shows expected counts' '
+	flux queue status -v >stat.out &&
+	cat <<-EOT >stat.exp &&
+	Job submission is enabled
+	Scheduling is started
+	1 alloc requests queued
+	1 alloc requests pending to scheduler
+	1 running jobs
+	EOT
+	test_cmp stat.exp stat.out
+'
+
+test_expect_success 'flux-queue: stop queue and cancel long job' '
+	flux queue stop &&
+	flux cancel --all -S RUN
+'
+
+test_expect_success 'flux-queue: queue becomes idle' '
+	run_timeout 30 flux queue idle
+'
+
+test_expect_success 'flux-queue: queue status -v shows expected counts' '
+	flux queue status -v >stat2.out &&
+	cat <<-EOT >stat2.exp &&
+	Job submission is enabled
+	Scheduling is stopped
+	0 alloc requests queued
+	0 alloc requests pending to scheduler
+	0 running jobs
+	EOT
+	test_cmp stat2.exp stat2.out
+'
+
+test_expect_success 'flux-queue: start queue and drain' '
+	flux queue start &&
+	run_timeout 30 flux queue drain
+'
+
+test_expect_success 'flux-queue: quiet option works' '
+	flux queue disable --quiet foof > disable_quiet.out &&
+	test_must_fail grep submission disable_quiet.out &&
+	flux queue enable --quiet > enable_quiet.out &&
+	test_must_fail grep submission enable_quiet.out &&
+	flux queue stop --quiet > stop_quiet.out &&
+	test_must_fail grep Scheduling stop_quiet.out &&
+	flux queue start --quiet > start_quiet.out &&
+	test_must_fail grep Scheduling start_quiet.out
+'
+
+test_expect_success 'flux-queue: verbose option works' '
+	flux queue disable --verbose foof > disable_verbose.out &&
+	test $(grep -c "submission is disabled" disable_verbose.out) -eq 1 &&
+	flux queue enable --verbose > enable_verbose.out &&
+	test $(grep -c "submission is enabled" enable_verbose.out) -eq 1 &&
+	flux queue stop --verbose > stop_verbose.out &&
+	test $(grep -c "Scheduling is stopped" stop_verbose.out) -eq 1 &&
+	flux queue start --verbose > start_verbose.out &&
+	test $(grep -c "Scheduling is started" start_verbose.out) -eq 1
+'
+
+runas_guest() {
+	local userid=$(($(id -u)+1))
+	FLUX_HANDLE_USERID=$userid FLUX_HANDLE_ROLEMASK=0x2 "$@"
+}
+
+test_expect_success 'flux-queue: status allowed for guest' '
+	runas_guest flux queue status
+'
+
+test_expect_success 'flux-queue: stop denied for guest' '
+	test_must_fail runas_guest flux queue stop 2>guest_stop.err &&
+	grep "requires owner credentials" guest_stop.err
+'
+
+test_expect_success 'flux-queue: start denied for guest' '
+	test_must_fail runas_guest flux queue start 2>guest_start.err &&
+	grep "requires owner credentials" guest_start.err
+'
+
+test_expect_success 'flux-queue: disable denied for guest' '
+	test_must_fail runas_guest flux queue disable foo 2>guest_dis.err &&
+	grep "requires owner credentials" guest_dis.err
+'
+
+test_expect_success 'flux-queue: enable denied for guest' '
+	test_must_fail runas_guest flux queue enable 2>guest_ena.err &&
+	grep "requires owner credentials" guest_ena.err
+'
+
+test_expect_success 'flux-queue: drain denied for guest' '
+	test_must_fail runas_guest flux queue drain 2>guest_drain.err &&
+	grep "requires owner credentials" guest_drain.err
+'
+
+test_expect_success 'flux-queue: idle denied for guest' '
+	test_must_fail runas_guest flux queue idle 2>guest_idle.err &&
+	grep "requires owner credentials" guest_idle.err
+'
+
+#
+# Test support for named queues
+#
+
+test_expect_success 'flux queue status --queue fails with no queues' '
+	test_must_fail flux queue status --queue=batch
+'
+test_expect_success 'flux queue enable --queue fails with no queues' '
+	test_must_fail flux queue enable --queue=batch
+'
+test_expect_success 'flux queue disable --queue fails with no queues' '
+	test_must_fail flux queue disable --queue=batch
+'
+test_expect_success 'flux queue start --queue fails with no queues' '
+	test_must_fail flux queue start --queue=batch
+'
+test_expect_success 'flux queue stop --queue fails with no queues' '
+	test_must_fail flux queue stop --queue=batch
+'
+test_expect_success 'ensure instance is drained' '
+	flux queue drain &&
+	flux queue status -v
+'
+test_expect_success 'configure batch,debug queues' '
+	flux config load <<-EOT
+	[queues.batch]
+	[queues.debug]
+	EOT
+'
+test_expect_success 'queues enabled and stopped by default' '
+	flux queue status >mqstatus_default.out &&
+	test $(grep -c "submission is enabled" mqstatus_default.out) -eq 2 &&
+	test $(grep -c "Scheduling is stopped" mqstatus_default.out) -eq 2
+'
+test_expect_success 'start queues' '
+	flux queue start --all &&
+	flux queue status >mqstatus_initial.out &&
+	test $(grep -c "Scheduling is started" mqstatus_initial.out) -eq 2
+'
+test_expect_success 'jobs may be submitted to either queue' '
+	flux submit -q batch true &&
+	flux submit -q debug true
+'
+test_expect_success 'flux-queue status can show one queue' '
+	flux queue status -q debug >mqstatus_debug.out &&
+	test_must_fail grep batch mqstatus_debug.out
+'
+test_expect_success 'flux-queue disable without --queue or --all fails' '
+	test_must_fail flux queue disable test reasons
+'
+test_expect_success 'flux-queue disable --all affects all queues' '
+	flux queue disable --all test reasons > mqdisable.out &&
+	test $(grep -c "submission is disabled: test reason" mqdisable.out) -eq 2 &&
+	flux queue status >mqstatus_dis.out &&
+	test $(grep -c "submission is disabled: test reason" mqstatus_dis.out) -eq 2
+'
+test_expect_success 'jobs may not be submitted to either queue' '
+	test_must_fail flux submit -q batch true &&
+	test_must_fail flux submit -q debug true
+'
+test_expect_success 'flux-queue enable without --queue or --all fails' '
+	test_must_fail flux queue enable
+'
+test_expect_success 'flux-queue enable --all affects all queues' '
+	flux queue enable -a > mqenable.out &&
+	test $(grep -c "submission is enabled" mqenable.out) -eq 2 &&
+	flux queue status >mqstatus_ena.out &&
+	test $(grep -c "submission is enabled" mqstatus_ena.out) -eq 2
+'
+test_expect_success 'flux-queue disable can do one queue' '
+	flux queue disable -q batch nobatch > mqdisable_batch.out &&
+	test $(grep -c "submission is" mqdisable_batch.out) -eq 1 &&
+	test $(grep -c "submission is disabled: nobatch" mqdisable_batch.out) -eq 1 &&
+	flux queue status >mqstatus_batchdis.out &&
+	test $(grep -c "submission is enabled" mqstatus_batchdis.out) -eq 1 &&
+	test $(grep -c "submission is disabled: nobatch" mqstatus_batchdis.out) -eq 1 &&
+	test_must_fail flux submit -q batch true &&
+	flux submit -q debug true
+'
+test_expect_success 'flux-queue enable can do one queue' '
+	flux queue enable -q batch > mqenable_batch.out &&
+	test $(grep -c "submission is" mqenable_batch.out) -eq 1 &&
+	test $(grep -c "submission is enabled" mqenable_batch.out) -eq 1 &&
+	flux queue status >mqstatus_batchena.out &&
+	test $(grep -c "submission is enabled" mqstatus_batchena.out) -eq 2 &&
+	flux submit -q batch true &&
+	flux submit -q debug true
+'
+
+
+test_expect_success 'flux-queue stop --all affects all queues' '
+	flux queue stop --all test reasons > mqstop.out &&
+	test $(grep -c "Scheduling is stopped: test reasons" mqstop.out) -eq 2 &&
+	flux queue status >mqstatus_stop.out &&
+	test $(grep -c "Scheduling is stopped: test reasons" mqstatus_stop.out) -eq 2
+'
+test_expect_success 'flux-queue stop w/o --all affects all queues but outputs warning' '
+	flux queue start --all &&
+	flux queue stop test reasons 2>mqstatus_stop2.err &&
+	grep "warning" mqstatus_stop2.err &&
+	flux queue status >mqstatus_stop2.out &&
+	test $(grep -c "Scheduling is stopped: test reasons" mqstatus_stop2.out) -eq 2
+'
+test_expect_success 'jobs may be submitted to either queue' '
+	flux submit --wait-event=priority -q batch true > job_batch1.id &&
+	flux submit --wait-event=priority -q debug true > job_debug1.id
+'
+
+wait_state() {
+	local jobid=$1
+	local state=$2
+	local i=0
+	while [ "$(flux jobs -no {state} ${jobid})" != "${state}" ] \
+		  && [ $i -lt 50 ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "50" ]
+	then
+	    return 1
+	fi
+	return 0
+}
+
+test_expect_success 'submitted jobs are not running' '
+	wait_state $(cat job_batch1.id) SCHED &&
+	flux jobs -n -o "{state}" $(cat job_batch1.id) | grep SCHED &&
+	wait_state $(cat job_debug1.id) SCHED &&
+	flux jobs -n -o "{state}" $(cat job_debug1.id) | grep SCHED
+'
+test_expect_success 'flux-queue start --all affects all queues' '
+	flux queue start -a > mqstart.out &&
+	test $(grep -c "Scheduling is started" mqstart.out) -eq 2 &&
+	flux queue status >mqstatus_start.out &&
+	test $(grep -c "Scheduling is started" mqstatus_start.out) -eq 2
+'
+test_expect_success 'submitted jobs ran and completed' '
+	wait_state $(cat job_batch1.id) INACTIVE &&
+	flux jobs -n -o "{state}" $(cat job_batch1.id) | grep INACTIVE &&
+	wait_state $(cat job_debug1.id) INACTIVE &&
+	flux jobs -n -o "{state}" $(cat job_debug1.id) | grep INACTIVE
+'
+test_expect_success 'flux-queue start w/o --all affects all queues but outputs warning' '
+	flux queue stop --all &&
+	flux queue start 2>mqstatus_start2.err &&
+	grep "warning" mqstatus_start2.err &&
+	flux queue status >mqstatus_start2.out &&
+	test $(grep -c "Scheduling is started" mqstatus_start2.out) -eq 2
+'
+
+test_expect_success 'flux-queue stop can do one queue' '
+	flux queue stop -q batch nobatch > mqstop_batch.out &&
+	test $(grep -c "Scheduling is" mqstop_batch.out) -eq 1 &&
+	test $(grep -c "Scheduling is stopped: nobatch" mqstop_batch.out) -eq 1 &&
+	flux queue status >mqstatus_batchstop.out &&
+	test $(grep -c "Scheduling is started" mqstatus_batchstop.out) -eq 1 &&
+	test $(grep -c "Scheduling is stopped: nobatch" mqstatus_batchstop.out) -eq 1 &&
+	flux submit -q batch true > job_batch2.id &&
+	flux submit -q debug true > job_debug2.id
+'
+test_expect_success 'check one job ran, other job didnt' '
+	wait_state $(cat job_debug2.id) INACTIVE &&
+	flux jobs -n -o "{state}" $(cat job_debug2.id) | grep INACTIVE &&
+	wait_state $(cat job_batch2.id) SCHED &&
+	flux jobs -n -o "{state}" $(cat job_batch2.id) | grep SCHED
+'
+test_expect_success 'flux-queue start can do one queue' '
+	flux queue start -q batch > mqstart_batch.out &&
+	test $(grep -c "Scheduling is" mqstart_batch.out) -eq 1 &&
+	test $(grep -c "Scheduling is started" mqstart_batch.out) -eq 1 &&
+	flux queue status >mqstatus_batchstart.out &&
+	test $(grep -c "Scheduling is started" mqstatus_batchstart.out) -eq 2
+'
+test_expect_success 'previously submitted job run to completion' '
+	wait_state $(cat job_batch2.id) INACTIVE &&
+	flux jobs -n -o "{state}" $(cat job_batch2.id) | grep INACTIVE
+'
+
+test_expect_success 'flux-queue: stop with named queues and --nocheckpoint works' '
+	mkdir -p conf.d &&
+	cat >conf.d/queues.toml <<-EOT &&
+	[queues.debug]
+	[queues.batch]
+	EOT
+	cat >stopqueues.sh <<-EOT &&
+	flux queue start --all
+	flux queue stop --all
+	EOT
+	cat >stopqueuesnocheckpoint.sh <<-EOT &&
+	flux queue start --all
+	flux queue stop --all --nocheckpoint
+	EOT
+	chmod +x ./stopqueues.sh &&
+	chmod +x ./stopqueuesnocheckpoint.sh &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.dump=dump_queue_named_nocheckpoint1.tar \
+	    ./stopqueues.sh &&
+	tar -xvf dump_queue_named_nocheckpoint1.tar &&
+	cat checkpoint/job-manager | jq -e ".queue[0].start == false" &&
+	cat checkpoint/job-manager | jq -e ".queue[1].start == false" &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.dump=dump_queue_named_nocheckpoint2.tar \
+	    ./stopqueuesnocheckpoint.sh &&
+	tar -xvf dump_queue_named_nocheckpoint2.tar &&
+	cat checkpoint/job-manager | jq -e ".queue[0].start == true" &&
+	cat checkpoint/job-manager | jq -e ".queue[1].start == true"
+'
+
+test_expect_success 'flux-queue: quiet option works with one queue' '
+	flux queue disable -q batch --quiet foof > mqdisable_quiet.out &&
+	test_must_fail grep submission mqdisable_quiet.out &&
+	flux queue enable -q batch --quiet > mqenable_quiet.out &&
+	test_must_fail grep submission mqenable_quiet.out &&
+	flux queue stop -q batch --quiet > mqstop_quiet.out &&
+	test_must_fail grep Scheduling mqstop_quiet.out &&
+	flux queue start -q batch --quiet > mqstart_quiet.out &&
+	test_must_fail grep Scheduling mqstart_quiet.out
+'
+
+test_expect_success 'flux-queue: verbose option works with one queue' '
+	flux queue disable -q batch --verbose foof > mqdisable_verbose.out &&
+	test $(grep -c "submission is disabled" mqdisable_verbose.out) -eq 1 &&
+	test $(grep -c "submission is enabled" mqdisable_verbose.out) -eq 1 &&
+	flux queue enable -q batch --verbose > mqenable_verbose.out &&
+	test $(grep -c "submission is enabled" mqenable_verbose.out) -eq 2 &&
+	flux queue stop -q batch --verbose > mqstop_verbose.out &&
+	test $(grep -c "Scheduling is stopped" mqstop_verbose.out) -eq 1 &&
+	test $(grep -c "Scheduling is started" mqstop_verbose.out) -eq 1 &&
+	flux queue start -q batch --verbose > mqstart_verbose.out &&
+	test $(grep -c "Scheduling is started" mqstart_verbose.out) -eq 2
+'
+
+test_expect_success 'flux-queue start fails on unknown queue' '
+	test_must_fail flux queue start -q notaqueue
+'
+test_expect_success 'flux-queue stop fails on unknown queue' '
+	test_must_fail flux queue stop -q notaqueue
+'
+test_expect_success 'flux-queue enable fails on unknown queue' '
+	test_must_fail flux queue enable -q notaqueue
+'
+test_expect_success 'flux-queue disable fails on unknown queue' '
+	test_must_fail flux queue disable -q notaqueue
+'
+test_expect_success 'flux-queue status fails on unknown queue' '
+	test_must_fail flux queue status -q notaqueue
+'
+
+test_done
diff --git a/t/t2241-queue-cmd-list.t b/t/t2241-queue-cmd-list.t
new file mode 100755
index 000000000000..6b981367c752
--- /dev/null
+++ b/t/t2241-queue-cmd-list.t
@@ -0,0 +1,305 @@
+#!/bin/sh
+test_description='Test flux queue command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 full
+
+# anon queue
+
+test_expect_success 'flux-queue: default lists expected fields' '
+	flux queue list > default.out &&
+	grep TDEFAULT default.out &&
+	grep TLIMIT default.out &&
+	grep NNODES default.out &&
+	grep NCORES default.out &&
+	grep ST default.out &&
+	grep EN default.out &&
+	grep NGPUS default.out
+'
+
+test_expect_success 'flux-queue: FLUX_QUEUE_LIST_FORMAT_DEFAULT works' '
+	FLUX_QUEUE_LIST_FORMAT_DEFAULT="{limits.min.nnodes} {limits.max.ngpus}" \
+		flux queue list > default_override.out &&
+	grep MINNODES default_override.out &&
+	grep MAXGPUS default_override.out
+'
+
+test_expect_success 'flux-queue: --no-header works' '
+	flux queue list --no-header > default_no_header.out &&
+	test_must_fail grep TDEFAULT default_no_header.out &&
+	test_must_fail grep TLIMIT default_no_header.out &&
+	test_must_fail grep NNODES default_no_header.out &&
+	test_must_fail grep NCORES default_no_header.out &&
+	test_must_fail grep ST default_no_header.out &&
+	test_must_fail grep EN default_no_header.out &&
+	test_must_fail grep NGPUS default_no_header.out
+'
+
+ALL_LIMITS_FMT="\
+{defaults.timelimit},{limits.timelimit},\
+{limits.range.nnodes},{limits.range.ncores},{limits.range.ngpus},\
+{limits.min.nnodes},{limits.min.ncores},{limits.min.ngpus},\
+{limits.max.nnodes},{limits.max.ncores},{limits.max.ngpus}\
+"
+
+test_expect_success 'flux-queue: expected headers with non-default fields output' '
+	flux queue list -o "${ALL_LIMITS_FMT}" > non_default.out &&
+	grep TDEFAULT non_default.out &&
+	grep TLIMIT non_default.out &&
+	grep NNODES non_default.out &&
+	grep NCORES non_default.out &&
+	grep NGPUS non_default.out &&
+	grep MINNODES non_default.out &&
+	grep MINCORES non_default.out &&
+	grep MINGPUS non_default.out &&
+	grep MAXNODES non_default.out &&
+	grep MAXCORES non_default.out &&
+	grep MAXGPUS non_default.out
+'
+
+test_expect_success 'flux-queue: empty config has no queues' '
+	flux queue list -n \
+		-o "{queue},{queuem}" > empty_config_queue.out &&
+	echo "," > empty_config_queue.exp &&
+	test_cmp empty_config_queue.exp empty_config_queue.out
+'
+test_expect_success 'flux-queue: empty config limits are 0/infinity' '
+	flux queue list -n \
+		-o "${ALL_LIMITS_FMT}" > empty_config_all.out &&
+	echo "inf,inf,0-inf,0-inf,0-inf,0,0,0,inf,inf,inf" > empty_config_all.exp &&
+	test_cmp empty_config_all.exp empty_config_all.out
+'
+test_expect_success 'flux-queue: empty config: queue is enabled/started' '
+	test_debug "flux queue list" &&
+	test "$(flux queue list -no {submission})" = "enabled" &&
+	test "$(flux queue list -no {scheduling})" = "started" &&
+	test "$(flux queue list -no {enabled})" = "✔" &&
+	test "$(flux queue list -no {started})" = "✔"
+'
+test_expect_success 'flux-queue: stop anonymous queue' '
+	flux queue stop
+'
+test_expect_success 'flux-queue: queue is enabled/stopped' '
+	test_debug "flux queue list" &&
+	test "$(flux queue list -no {submission})" = "enabled" &&
+	test "$(flux queue list -no {scheduling})" = "stopped" &&
+	test "$(flux queue list -no {enabled})" = "✔" &&
+	test "$(flux queue list -no {started})" = "✗"
+'
+test_expect_success 'flux-queue: disable anonymous queue' '
+	flux queue disable testing
+'
+test_expect_success 'flux-queue: queue is disabled/stopped' '
+	test_debug "flux queue list" &&
+	test "$(flux queue list -no {submission})" = "disabled" &&
+	test "$(flux queue list -no {scheduling})" = "stopped" &&
+	test "$(flux queue list -no {enabled})" = "✗" &&
+	test "$(flux queue list -no {started})" = "✗"
+'
+test_expect_success 'flux-queue: enable and start anonymous queue' '
+	flux queue enable &&
+	flux queue start
+'
+test_expect_success 'flux-queue: --queue option fails with anonymouns queue' '
+	test_expect_code 1 flux queue list --queue=batch
+'
+
+test_expect_success 'flux-queue: fsd of infinity is infinity' '
+	flux queue list -n \
+		-o "{defaults.timelimit!F},{limits.timelimit!F}" \
+		> empty_config_fsd.out &&
+	echo "inf,inf" > empty_config_fsd.exp &&
+	test_cmp empty_config_fsd.exp empty_config_fsd.out
+'
+
+# N.B. job-size.max.ngpus left out to test default of infinity
+test_expect_success 'config flux with policy defaults' '
+	flux config load <<EOT
+[policy.jobspec.defaults.system]
+duration = "1h"
+
+[policy.limits]
+duration = "4h"
+job-size.min.nnodes = 2
+job-size.max.nnodes = 10
+job-size.min.ncores = 1
+job-size.max.ncores = -1
+job-size.min.ngpus = 1
+EOT
+'
+
+test_expect_success 'flux-queue: defaults config timelimits' '
+       flux queue list -n \
+	       -o "{defaults.timelimit},{limits.timelimit},{defaults.timelimit!F},{limits.timelimit!F}" \
+	       > policy_defaults_timelimit.out &&
+       echo "3600.0,14400.0,1h,4h" > policy_defaults_timelimit.exp &&
+       test_cmp policy_defaults_timelimit.exp policy_defaults_timelimit.out
+'
+test_expect_success 'flux-queue: defaults config ranges' '
+       flux queue list -n \
+	       -o "{limits.range.nnodes},{limits.range.ncores},{limits.range.ngpus}" \
+	       > policy_defaults_range.out &&
+       echo "2-10,1-inf,1-inf" > policy_defaults_range.exp &&
+       test_cmp policy_defaults_range.exp policy_defaults_range.out
+'
+test_expect_success 'flux-queue: defaults config mins' '
+       flux queue list -n \
+	       -o "{limits.min.nnodes},{limits.min.ncores},{limits.min.ngpus}" \
+	       > policy_defaults_min.out &&
+       echo "2,1,1" > policy_defaults_min.exp &&
+       test_cmp policy_defaults_min.exp policy_defaults_min.out
+'
+test_expect_success 'flux-queue: defaults config maxes' '
+       flux queue list -n \
+	       -o "{limits.max.nnodes},{limits.max.ncores},{limits.max.ngpus}" \
+	       > policy_defaults_max.out &&
+       echo "10,inf,inf" > policy_defaults_max.exp &&
+       test_cmp policy_defaults_max.exp policy_defaults_max.out
+'
+
+#
+# named queues
+#
+
+test_expect_success 'config flux with several named queues but no other config' '
+	flux config load <<EOT
+[queues.batch]
+[queues.debug]
+EOT
+'
+# we can't predict order of listing, so we grep and count lines
+test_expect_success 'flux-queue: queue config, no default marked' '
+	flux queue list -no "{queue},{queuem}" > queues_no_default.out &&
+	test $(grep "^batch,batch$" queues_no_default.out | wc -l) -eq 1 &&
+	test $(grep ^"debug,debug$" queues_no_default.out | wc -l) -eq 1
+'
+test_expect_success 'flux-queue: --queue option works' '
+	test "$(flux queue list -q batch -no "{queue}")" = "batch" &&
+	test "$(flux queue list -q debug -no "{queue}")" = "debug" &&
+	test_debug "flux queue list -q batch,debug -n" &&
+	test $(flux queue list -q batch,debug -n  | wc -l) -eq 2
+'
+test_expect_success 'flux-queue: invalid queue is detected with --queue' '
+	test_expect_code 1 flux queue list --queue=foo
+'
+
+# N.B. job-size.max.ngpus left out of [policy.limits] to test default
+# of infinity
+# N.B. Note that many values in [queues.debug.policy.limits] left out to
+# test that system wide config is parsed correctly
+test_expect_success 'create queue defaults config for tests' '
+	flux config load <<EOT
+[policy.jobspec.defaults.system]
+queue = "batch"
+duration = "1h"
+
+[policy.limits]
+duration = "4h"
+job-size.min.nnodes = 2
+job-size.max.nnodes = 10
+job-size.min.ncores = 1
+job-size.max.ncores = -1
+job-size.min.ngpus = 1
+
+[queues.batch]
+policy.jobspec.defaults.system.duration = "2h"
+
+[queues.batch.policy.limits]
+duration = "8h"
+job-size.min.nnodes = 5
+job-size.max.nnodes = 15
+job-size.min.ncores = 1
+job-size.max.ncores = 20
+job-size.min.ngpus = 1
+job-size.max.ngpus = 4
+
+[queues.debug.policy.limits]
+duration = "2h"
+job-size.max.nnodes = 4
+job-size.min.ncores = 2
+EOT
+'
+# we can't predict order of listing, so we grep and count lines
+test_expect_success 'flux-queue: queue config' '
+	flux queue list -no "{queue},{queuem}" > queues_config_queue.out &&
+	test $(grep "^batch,batch\*$" queues_config_queue.out | wc -l) -eq 1 &&
+	test $(grep ^"debug,debug$" queues_config_queue.out | wc -l) -eq 1
+'
+test_expect_success 'flux-queue: queue config timelimits' '
+	flux queue list -n \
+		-o "{queue},{defaults.timelimit},{limits.timelimit},{defaults.timelimit!F},{limits.timelimit!F}" \
+		> queues_timelimit.out &&
+	test $(grep "batch,7200.0,28800.0,2h,8h" queues_timelimit.out | wc -l) -eq 1 &&
+	test $(grep "debug,3600.0,7200.0,1h,2h" queues_timelimit.out | wc -l) -eq 1
+'
+test_expect_success 'flux-queue: queue config ranges' '
+	flux queue list -n \
+		-o "{queue},{limits.range.nnodes},{limits.range.ncores},{limits.range.ngpus}" \
+		> queues_range.out &&
+	test $(grep "batch,5-15,1-20,1-4" queues_range.out | wc -l) -eq 1 &&
+	test $(grep "debug,2-4,2-inf,1-inf" queues_range.out | wc -l) -eq 1
+'
+test_expect_success 'flux-queue: queue config mins' '
+	flux queue list -n \
+		-o "{queue},{limits.min.nnodes},{limits.min.ncores},{limits.min.ngpus}" \
+		> queues_min.out &&
+	test $(grep "batch,5,1,1" queues_min.out | wc -l) -eq 1 &&
+	test $(grep "debug,2,2,1" queues_min.out | wc -l) -eq 1
+'
+test_expect_success 'flux-queue: queue config maxes' '
+	flux queue list -n \
+		-o "{queue},{limits.max.nnodes},{limits.max.ncores},{limits.max.ngpus}" \
+		> queues_max.out &&
+	test $(grep "batch,15,20,4" queues_max.out | wc -l) -eq 1 &&
+	test $(grep "debug,4,inf,inf" queues_max.out | wc -l) -eq 1
+'
+
+#
+# config
+#
+
+#  We can't add configuration to ~/.config or /etc/xdg/flux, so
+#   just test that XDG_CONFIG_HOME works.
+#
+test_expect_success 'Create flux-queue.toml config file' '
+	mkdir -p dir/flux &&
+	cat <<-EOT >dir/flux/flux-queue.toml
+	[list.formats.myformat]
+	description = "my list format"
+	format = "{queue}"
+	EOT
+'
+
+test_expect_success "flux-queue list accepts configured formats" '
+	XDG_CONFIG_HOME="$(pwd)/dir" \
+		flux queue list --format=help > list-help.out &&
+	test_debug "cat list-help.out" &&
+	grep myformat list-help.out &&
+	XDG_CONFIG_HOME="$(pwd)/dir" \
+		flux queue list --format=get-config=myformat \
+		>list-get-config.out &&
+	test_debug "cat list-get-config.out" &&
+	grep list\.formats\.myformat list-get-config.out &&
+	XDG_CONFIG_HOME="$(pwd)/dir" \
+		flux queue list -n \
+			--format=myformat > list-myformat.out &&
+	test $(grep "^batch$" list-myformat.out | wc -l) -eq 1 &&
+	test $(grep ^"debug$" list-myformat.out | wc -l) -eq 1
+'
+
+#
+#
+# corner cases
+#
+
+test_expect_success 'flux-queue: invalid fields recognized' '
+	test_must_fail flux queue list -o "{foobar}" &&
+	test_must_fail flux queue list -o "{defaults.foobar}" &&
+	test_must_fail flux queue list -o "{limits.foobar}" &&
+	test_must_fail flux queue list -o "{limits.range.foobar}" &&
+	test_must_fail flux queue list -o "{limits.min.foobar}" &&
+	test_must_fail flux queue list -o "{limits.max.foobar}"
+'
+
+test_done
diff --git a/t/t2245-policy-config.t b/t/t2245-policy-config.t
new file mode 100755
index 000000000000..3f9f326113c7
--- /dev/null
+++ b/t/t2245-policy-config.t
@@ -0,0 +1,186 @@
+#!/bin/sh
+
+test_description='Test RFC 33 policy config verification
+
+Ensure that the broker catches violations of the configuration
+specified in RFC 33 for [policy] and [queue.<name>.policy].
+'
+
+. $(dirname $0)/sharness.sh
+
+mkdir -p config
+
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'unknown policy key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.foo = 1
+	EOT
+'
+test_expect_success 'unknown policy.jobspec key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.jobspec.foo = 1
+	EOT
+'
+test_expect_success 'unknown policy.jobspec.defaults key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.jobspec.defaults.foo = 1
+	EOT
+'
+test_expect_success 'unknown policy.jobspec.defaults.system key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.jobspec.defaults.system.foo = 1
+	EOT
+'
+test_expect_success 'malformed policy.jobspec.defaults.system.duration fails' '
+	test_must_fail flux config load <<-EOT
+	policy.jobspec.defaults.system.duration = 1
+	EOT
+'
+test_expect_success 'wrong type policy.jobspec.defaults.system.queue fails' '
+	test_must_fail flux config load <<-EOT
+	policy.jobspec.defaults.system.queue = 1
+	EOT
+'
+test_expect_success 'unknown policy.limits key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.limits.foo = 1
+	EOT
+'
+test_expect_success 'unknown policy.limits.job-size key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.limits.job-size.foo = 1
+	EOT
+'
+test_expect_success 'unknown policy.limits.job-size.max key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.limits.job-size.max.foo = 1
+	EOT
+'
+test_expect_success 'unknown policy.limits.job-size.min key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.limits.job-size.min.foo = 1
+	EOT
+'
+test_expect_success 'incorrect policy.limits.job-size.min.nnodes fails' '
+	test_must_fail flux config load <<-EOT
+	policy.limits.job-size.min.nnodes = -2
+	EOT
+'
+test_expect_success 'malformed policy.limits.duration fails' '
+	test_must_fail flux config load <<-EOT
+	policy.limits.duration = 1.0
+	EOT
+'
+test_expect_success 'unknown policy.access key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.access.foo = 1.0
+	EOT
+'
+test_expect_success 'malformed policy.access.allow-user key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.access.allow-user = 1.0
+	EOT
+'
+test_expect_success 'malformed policy.access.allow-group key fails' '
+	test_must_fail flux config load <<-EOT
+	policy.access.allow-group = 1.0
+	EOT
+'
+test_expect_success 'malformed policy.access.allow-user entry fails' '
+	test_must_fail flux config load <<-EOT
+	policy.access.allow-user = [ 1 ]
+	EOT
+'
+test_expect_success 'malformed policy.access.allow-group entry fails' '
+	test_must_fail flux config load <<-EOT
+	policy.access.allow-group = [ 1 ]
+	EOT
+'
+test_expect_success 'well formed policy.access works' '
+	flux config load <<-EOT
+	[policy.access]
+	allow-user = [ "alice", "bob" ]
+	allow-group = [ "smurfs" ]
+	EOT
+'
+test_expect_success 'malformed queues table fails' '
+	test_must_fail flux config load <<-EOT
+	queues = 1
+	EOT
+'
+test_expect_success 'unknown queues.NAME.policy.foo key fails' '
+	test_must_fail flux config load <<-EOT
+	queues.x.policy.foo = 1
+	EOT
+'
+test_expect_success 'malformed queues.NAME.policy.limits.duration key fails' '
+	test_must_fail flux config load <<-EOT
+	queues.x.policy.limits.duration = 1
+	EOT
+'
+test_expect_success 'default queue as queue policy fails' '
+	test_must_fail flux config load <<-EOT
+	queues.x.policy.jobspec.defaults.system.queue = "x"
+	EOT
+'
+test_expect_success 'unknown default queue fails' '
+	test_must_fail flux config load <<-EOT
+	[queues.foo]
+	[policy]
+	jobspec.defaults.system.queue = "bar"
+	EOT
+'
+test_expect_success 'unknown queues.NAME.requires.foo key fails' '
+	test_must_fail flux config load <<-EOT
+	queues.x.requires = 1
+	EOT
+'
+test_expect_success 'malformed queues.NAME.requires fails' '
+	test_must_fail flux config load <<-EOT
+	queues.x.requires = [ 1 ]
+	EOT
+'
+test_expect_success 'malformed queues.NAME.requires property string' '
+	test_must_fail flux config load <<-EOT
+	queues.x.requires = [ "foo|bar" ]
+	EOT
+'
+test_expect_success 'well formed queues.NAME.requires works' '
+	flux config load <<-EOT
+	[queues.x]
+	requires = [ "batch" ]
+	[queues.z]
+	requires = [ "debug", "foo" ]
+	EOT
+'
+# Example from flux-config-policy(5)
+test_expect_success 'valid config passes' '
+	flux config load <<-EOT
+	[policy.jobspec.defaults.system]
+	duration = "1h"
+	queue = "batch"
+
+	[policy.limits]
+	duration = "4h"
+	job-size.max.nnodes = 8
+	job-size.max.ngpus = 4
+
+	[queues.debug.policy.limits]
+	duration = "30m"
+	job-size.max.ngpus = -1  # unlimited
+
+	[queues.batch]
+	EOT
+'
+
+test_expect_success 'a bad config is detected at initialization too' '
+	cat >badconf.toml <<-EOT &&
+	policy.foo = 1
+	EOT
+	test_must_fail flux start --config-path=badconf.toml true
+'
+
+test_done
diff --git a/t/t2260-job-list.t b/t/t2260-job-list.t
new file mode 100755
index 000000000000..1571af8cc081
--- /dev/null
+++ b/t/t2260-job-list.t
@@ -0,0 +1,3030 @@
+#!/bin/sh
+
+test_description='Test flux job list services'
+
+. $(dirname $0)/job-list/job-list-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+export FLUX_CONF_DIR=$(pwd)
+test_under_flux 4 job
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+listRPC="flux python ${SHARNESS_TEST_SRCDIR}/job-list/list-rpc.py"
+JOB_CONV="flux python ${FLUX_SOURCE_DIR}/t/job-manager/job-conv.py"
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py"
+
+PLUGINPATH=${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs
+
+fj_wait_event() {
+	flux job wait-event --timeout=20 "$@"
+}
+
+# To avoid raciness in tests, need to ensure that job state we care about
+# testing against has been reached in the job-list module.
+wait_jobid_state() {
+	flux job list-ids --wait-state=$2 $1 > /dev/null
+}
+
+test_expect_success 'setup specific fake hostnames' '
+	flux R encode -r 0-3 -c 0-1 -H node[0-3] \
+	   | tr -d "\n" \
+	   | flux kvs put -r resource.R=- &&
+	flux module unload sched-simple &&
+	flux module reload resource noverify &&
+	flux module load sched-simple
+'
+
+test_expect_success 'create helper job submission script' '
+	cat >sleepinf.sh <<-EOT &&
+	#!/bin/sh
+	echo "job started"
+	sleep inf
+	EOT
+	chmod +x sleepinf.sh
+'
+
+# submit a whole bunch of jobs for job list testing
+#
+# - the first loop of job submissions are intended to have some jobs run
+#   quickly and complete
+# - the second loop of job submissions are intended to eat up all resources
+# - the last job submissions are intended to get a create a set of
+#   pending jobs, because jobs from the second loop have taken all resources
+#   - we desire pending jobs sorted in priority order, so we need to
+#   create the sorted list for comparison later.
+# - job ids are stored in files in the order we expect them to be listed
+#   - pending jobs - by priority (highest first), job id (smaller first)
+#   - running jobs - by start time (most recent first)
+#   - inactive jobs - by completion time (most recent first)
+#
+# TODO
+# - alternate userid job listing
+
+test_expect_success 'submit jobs for job list testing' '
+	#
+	# submit jobs that will complete
+	#
+	for i in $(seq 0 3); do
+		flux submit --requires=host:node${i} hostname >> inactiveids
+		fj_wait_event `tail -n 1 inactiveids` clean
+	done &&
+	#
+	#  Currently all inactive ids are "completed"
+	#
+	tac inactiveids | flux job id > completed.ids &&
+	#
+	#  Run a job that will fail, copy its JOBID to both inactive and
+	#	failed lists.
+	#
+	! jobid=`flux submit --requires=host:node0 --wait nosuchcommand` &&
+	echo $jobid >> inactiveids &&
+	flux job id $jobid > failedids &&
+	#
+	#  Run a job that we will end with a signal, copy its JOBID to both inactive and
+	#	failed and terminated lists.
+	#
+	# N.B. sleepinf.sh and wait-event on job data to workaround
+	# rare job startup race.  See #5210
+	#
+	jobid=`flux submit --requires=host:node0 ./sleepinf.sh` &&
+	flux job wait-event -W -p guest.output $jobid data &&
+	flux job kill $jobid &&
+	fj_wait_event $jobid clean &&
+	echo $jobid >> inactiveids &&
+	flux job id $jobid > terminated.ids &&
+	flux job id $jobid >> failedids &&
+	#
+	#  Run a job that we will end with a user exception, copy its JOBID to both
+	#	inactive and failed and exception lists.
+	#
+	# N.B. sleepinf.sh and wait-event on job data to workaround
+	# rare job startup race.  See #5210
+	#
+	jobid=`flux submit --requires=host:node0 ./sleepinf.sh` &&
+	flux job wait-event -W -p guest.output $jobid data &&
+	flux job raise --type=myexception --severity=0 -m "myexception" $jobid &&
+	fj_wait_event $jobid clean &&
+	echo $jobid >> inactiveids &&
+	flux job id $jobid > exception.ids &&
+	flux job id $jobid >> failedids &&
+	#
+	#  Run a job that will timeout, copy its JOBID to both inactive and
+	#	timeout lists.
+	#
+	jobid=`flux submit --requires=host:node0 --time-limit=0.5s sleep 30` &&
+	echo $jobid >> inactiveids &&
+	flux job id $jobid > timeout.ids &&
+	fj_wait_event ${jobid} clean &&
+	#
+	#  Submit 8 sleep jobs to fill up resources
+	#
+	#  N.B. no need to specify --requires:host, will be distributed
+	#  evenly
+	#
+	for i in $(seq 0 7); do
+		flux submit --time-limit=5m sleep 600 >> runningids
+	done &&
+	tac runningids | flux job id > running.ids &&
+	#
+	#  Submit a set of jobs with misc urgencies
+	#
+	#  N.B. no need to specify --requires:host, these jobs wont run
+	#
+	id1=$(flux submit --urgency=20 hostname) &&
+	id2=$(flux submit	       hostname) &&
+	id3=$(flux submit --urgency=31 hostname) &&
+	id4=$(flux submit --urgency=0  hostname) &&
+	id5=$(flux submit --urgency=20 hostname) &&
+	id6=$(flux submit	       hostname) &&
+	id7=$(flux submit --urgency=31 hostname) &&
+	id8=$(flux submit --urgency=0  hostname) &&
+	flux job id $id3 > pending.ids &&
+	flux job id $id7 >> pending.ids &&
+	flux job id $id1 >> pending.ids &&
+	flux job id $id5 >> pending.ids &&
+	flux job id $id2 >> pending.ids &&
+	flux job id $id6 >> pending.ids &&
+	flux job id $id4 >> pending.ids &&
+	flux job id $id8 >> pending.ids &&
+	cat pending.ids > active.ids &&
+	cat running.ids >> active.ids &&
+	#
+	#  Submit a job and cancel it
+	#
+	# N.B. no need to handle issue #5210 here, the job will not
+	# run due to lack of resources.
+	#
+	jobid=`flux submit --job-name=canceledjob sleep 30` &&
+	flux job wait-event $jobid depend &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean &&
+	flux job id $jobid >> inactiveids &&
+	flux job id $jobid > canceled.ids &&
+	tac failedids | flux job id > failed.ids &&
+	tac inactiveids | flux job id > inactive.ids &&
+	cat active.ids > all.ids &&
+	cat inactive.ids >> all.ids &&
+	#
+	#  The job-list module has eventual consistency with the jobs stored in
+	#  the job-manager queue.  To ensure no raciness in tests, ensure
+	#  jobs above have reached expected states in job-list before continuing.
+	#
+	flux job list-ids --wait-state=sched $(job_list_state_ids pending) > /dev/null &&
+	flux job list-ids --wait-state=run $(job_list_state_ids running) > /dev/null &&
+	flux job list-ids --wait-state=inactive $(job_list_state_ids inactive) > /dev/null
+'
+
+# Note: "running" = "run" | "cleanup", we also test just "run" state
+# since we happen to know all these jobs are in the "run" state given
+# checks above
+
+test_expect_success 'flux job list running jobs in started order' '
+	flux job list -s running | jq .id > list_started1.out &&
+	flux job list -s run,cleanup | jq .id > list_started2.out &&
+	flux job list -s run | jq .id > list_started3.out &&
+	test_cmp list_started1.out running.ids &&
+	test_cmp list_started2.out running.ids &&
+	test_cmp list_started3.out running.ids
+'
+
+test_expect_success 'flux job list running jobs with correct state' '
+	for count in `seq 1 8`; do \
+		echo "RUN" >> list_state_R.exp; \
+	done &&
+	flux job list -s running | jq .state | ${JOB_CONV} statetostr > list_state_R1.out &&
+	flux job list -s run,cleanup | jq .state | ${JOB_CONV} statetostr > list_state_R2.out &&
+	flux job list -s run | jq .state | ${JOB_CONV} statetostr > list_state_R3.out &&
+	test_cmp list_state_R1.out list_state_R.exp &&
+	test_cmp list_state_R2.out list_state_R.exp &&
+	test_cmp list_state_R3.out list_state_R.exp
+'
+
+test_expect_success 'flux job list no jobs in cleanup state' '
+	count=$(flux job list -s cleanup | wc -l) &&
+	test $count -eq 0
+'
+
+test_expect_success 'flux job list inactive jobs in completed order' '
+	flux job list -s inactive | jq .id > list_inactive.out &&
+	test_cmp list_inactive.out inactive.ids
+'
+
+test_expect_success 'flux job list inactive jobs with correct state' '
+	for count in `seq 1 $(job_list_state_count inactive)`; do \
+		echo "INACTIVE" >> list_state_I.exp; \
+	done &&
+	flux job list -s inactive | jq .state | ${JOB_CONV} statetostr > list_state_I.out &&
+	test_cmp list_state_I.out list_state_I.exp
+'
+
+test_expect_success 'flux job list inactive jobs results are correct' '
+	flux job list -s inactive | jq .result | ${JOB_CONV} resulttostr > list_result_I.out &&
+	echo "CANCELED" >> list_result_I.exp &&
+	echo "TIMEOUT" >> list_result_I.exp &&
+	for count in `seq 1 3`; do \
+		echo "FAILED" >> list_result_I.exp; \
+	done &&
+	for count in `seq 1 4`; do \
+		echo "COMPLETED" >> list_result_I.exp; \
+	done &&
+	test_cmp list_result_I.out list_result_I.exp
+'
+
+# flux job list does not take results as an option, test via direct
+# call to job-list.list
+
+test_expect_success 'flux job list only canceled jobs' '
+	id=$(id -u) &&
+	result=`${JOB_CONV} strtoresult CANCELED` &&
+	constraint="{ and: [ {userid:[${id}]}, {results:[${result}]}] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_result_canceled.out &&
+	test_cmp canceled.ids list_result_canceled.out
+'
+
+test_expect_success 'flux job list only failed jobs' '
+	id=$(id -u) &&
+	result=`${JOB_CONV} strtoresult FAILED` &&
+	constraint="{ and: [ {userid:[${id}]}, {results:[${result}]}] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_result_failed.out &&
+	test_cmp failed.ids list_result_failed.out
+'
+
+test_expect_success 'flux job list only timeout jobs' '
+	id=$(id -u) &&
+	result=`${JOB_CONV} strtoresult TIMEOUT` &&
+	constraint="{ and: [ {userid:[${id}]}, {results:[${result}]}] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_result_timeout.out &&
+	test_cmp timeout.ids list_result_timeout.out
+'
+
+test_expect_success 'flux job list only completed jobs' '
+	id=$(id -u) &&
+	result=`${JOB_CONV} strtoresult COMPLETED` &&
+	constraint="{ and: [ {userid:[${id}]}, {results:[${result}]}] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_result_completed.out &&
+	test_cmp completed.ids list_result_completed.out
+'
+
+# Note: "pending" = "depend" | "sched", we also test just "sched"
+# state since we happen to know all these jobs are in the "sched"
+# state given checks above
+
+test_expect_success 'flux job list pending jobs in priority order' '
+	flux job list -s pending | jq .id > list_pending1.out &&
+	flux job list -s depend,priority,sched | jq .id > list_pending2.out &&
+	flux job list -s sched | jq .id > list_pending3.out &&
+	test_cmp list_pending1.out pending.ids &&
+	test_cmp list_pending2.out pending.ids &&
+	test_cmp list_pending3.out pending.ids
+'
+
+test_expect_success 'flux job list pending jobs with correct urgency' '
+	cat >list_urgency.exp <<-EOT &&
+31
+31
+20
+20
+16
+16
+0
+0
+EOT
+	flux job list -s pending | jq .urgency > list_urgency1.out &&
+	flux job list -s depend,priority,sched | jq .urgency > list_urgency2.out &&
+	flux job list -s sched | jq .urgency > list_urgency3.out &&
+	test_cmp list_urgency1.out list_urgency.exp &&
+	test_cmp list_urgency2.out list_urgency.exp &&
+	test_cmp list_urgency3.out list_urgency.exp
+'
+
+test_expect_success 'flux job list pending jobs with correct priority' '
+	cat >list_priority.exp <<-EOT &&
+4294967295
+4294967295
+20
+20
+16
+16
+0
+0
+EOT
+	flux job list -s pending | jq .priority > list_priority1.out &&
+	flux job list -s depend,priority,sched | jq .priority > list_priority2.out &&
+	flux job list -s sched | jq .priority > list_priority3.out &&
+	test_cmp list_priority1.out list_priority.exp &&
+	test_cmp list_priority2.out list_priority.exp &&
+	test_cmp list_priority3.out list_priority.exp
+'
+
+test_expect_success 'flux job list pending jobs with correct state' '
+	for count in `seq 1 8`; do \
+		echo "SCHED" >> list_state_S.exp; \
+	done &&
+	flux job list -s sched | jq .state | ${JOB_CONV} statetostr > list_state_S.out &&
+	test_cmp list_state_S.out list_state_S.exp
+'
+
+test_expect_success 'flux job list no jobs in depend state' '
+	count=$(flux job list -s depend | wc -l) &&
+	test $count -eq 0
+'
+
+# Note: "active" = "pending" | "running", i.e. depend, priority,
+# sched, run, cleanup
+test_expect_success 'flux job list active jobs in correct order' '
+	flux job list -s active | jq .id > list_active1.out &&
+	flux job list -s depend,priority,sched,run,cleanup | jq .id > list_active2.out &&
+	flux job list -s sched,run | jq .id > list_active3.out &&
+	test_cmp list_active1.out active.ids &&
+	test_cmp list_active2.out active.ids &&
+	test_cmp list_active3.out active.ids
+'
+
+test_expect_success 'flux job list jobs with correct userid' '
+	for count in `seq 1 $(job_list_state_count all)`; do \
+		id -u >> list_userid.exp; \
+	done &&
+	flux job list -a | jq .userid > list_userid.out &&
+	test_cmp list_userid.out list_userid.exp
+'
+
+test_expect_success 'flux job list defaults to listing pending & running jobs' '
+	flux job list | jq .id > list_default.out &&
+	count=$(wc -l < list_default.out) &&
+	test $count = $(job_list_state_count active) &&
+	test_cmp list_default.out active.ids
+'
+
+test_expect_success 'flux job list --user=userid works' '
+	uid=$(id -u) &&
+	flux job list --user=$uid> list_userid.out &&
+	count=$(wc -l < list_userid.out) &&
+	test $count = $(job_list_state_count active)
+'
+
+test_expect_success 'flux job list --user=all works' '
+	flux job list --user=all > list_all.out &&
+	count=$(wc -l < list_all.out) &&
+	test $count = $(job_list_state_count active)
+'
+
+# we hard count numbers here b/c its a --count test
+test_expect_success 'flux job list --count works' '
+	flux job list -s active,inactive --count=12 | jq .id > list_count.out &&
+	count=$(wc -l < list_count.out) &&
+	test "$count" = "12" &&
+	cat pending.ids > list_count.exp &&
+	head -n 4 running.ids >> list_count.exp &&
+	test_cmp list_count.out list_count.exp
+'
+
+test_expect_success 'flux job list all jobs works' '
+	flux job list -a | jq .id > list_all_jobids.out &&
+	test_cmp all.ids list_all_jobids.out
+'
+
+test_expect_success 'flux module stats job-list is open to guests' '
+	FLUX_HANDLE_ROLEMASK=0x2 \
+	    flux module stats job-list >/dev/null
+'
+
+# do some more advanced constraint queries
+
+test_expect_success 'flux job list hostname jobs' '
+	id=$(id -u) &&
+	constraint="{ and: [ {userid:[${id}]}, {name:[\"hostname\"]}] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_hostname_jobs.out &&
+	numlines=$(cat pending.ids completed.ids | wc -l) &&
+	test $(cat list_constraint_hostname_jobs.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list active hostname jobs' '
+	id=$(id -u) &&
+	constraint="{ and: [ {userid:[${id}]}, {states:[\"active\"]}, {name:[\"hostname\"]}] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_pending_hostname.out &&
+	test_cmp list_constraint_pending_hostname.out pending.ids
+'
+
+test_expect_success 'flux job list inactive hostname jobs' '
+	id=$(id -u) &&
+	constraint="{ and: [ {userid:[${id}]}, {states:[\"inactive\"]}, {name:[\"hostname\"]}] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_inactive_hostname.out &&
+	test_cmp list_constraint_inactive_hostname.out completed.ids
+'
+
+test_expect_success 'flux job list invalid queue' '
+	id=$(id -u) &&
+	constraint="{ and: [ {userid:[${id}]}, {queue:[\"blarg\"]}] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_invalid_queue.out &&
+	test $(cat list_constraint_invalid_queue.out | wc -l) -eq 0
+'
+
+test_expect_success 'flux job list active (1)' '
+	state1=`${JOB_CONV} strtostate SCHED` &&
+	state2=`${JOB_CONV} strtostate RUN` &&
+	constraint="{ or: [ {states:[${state1}]}, {states:[${state2}]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_active1.out &&
+	numlines=$(cat active.ids | wc -l) &&
+	test $(cat list_constraint_active1.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list active (2)' '
+	state1=`${JOB_CONV} strtostate INACTIVE` &&
+	constraint="{ not: [ {states:[${state1}]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_active2.out &&
+	numlines=$(cat active.ids | wc -l) &&
+	test $(cat list_constraint_active2.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list pending jobs or inactive jobs (1)' '
+	state1=`${JOB_CONV} strtostate SCHED` &&
+	state2=`${JOB_CONV} strtostate INACTIVE` &&
+	constraint="{ or: [ {states:[${state1}]}, {states:[${state2}]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_pending_inactive1.out &&
+	numlines=$(cat pending.ids inactive.ids | wc -l) &&
+	test $(cat list_constraint_pending_inactive1.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list pending jobs or inactive jobs (2)' '
+	state1=`${JOB_CONV} strtostate SCHED` &&
+	state2=`${JOB_CONV} strtostate INACTIVE` &&
+	constraint="{ or: [ {states:[${state1}, ${state2}]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_pending_inactive2.out &&
+	numlines=$(cat pending.ids inactive.ids | wc -l) &&
+	test $(cat list_constraint_pending_inactive2.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list failed and canceled jobs (1)' '
+	result1=`${JOB_CONV} strtoresult FAILED` &&
+	result2=`${JOB_CONV} strtoresult CANCELED` &&
+	constraint="{ or: [ {results:[${result1}]}, {results:[${result2}]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_failed_canceled1.out &&
+	numlines=$(cat canceled.ids failed.ids | wc -l) &&
+	test $(cat list_constraint_failed_canceled1.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list failed and canceled jobs (2)' '
+	result1=`${JOB_CONV} strtoresult FAILED` &&
+	result2=`${JOB_CONV} strtoresult CANCELED` &&
+	constraint="{ and: [ {userid:[${id}]}, {results:[${result1}, ${result2}]}] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_failed_canceled2.out &&
+	numlines=$(cat canceled.ids failed.ids | wc -l) &&
+	test $(cat list_constraint_failed_canceled2.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list pending jobs or failed jobs (1)' '
+	state1=`${JOB_CONV} strtostate SCHED` &&
+	state2=`${JOB_CONV} strtostate INACTIVE` &&
+	result1=`${JOB_CONV} strtoresult FAILED` &&
+	constraint="{ or: [ {states:[${state1}]}, {and: [ {states:[${state2}]}, {results:[${result1}]} ] } ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_pending_failed1.out &&
+	numlines=$(cat pending.ids failed.ids | wc -l) &&
+	test $(cat list_constraint_pending_failed1.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list pending jobs or failed jobs (2)' '
+	state1=`${JOB_CONV} strtostate SCHED` &&
+	result1=`${JOB_CONV} strtoresult FAILED` &&
+	constraint="{ or: [ {states:[${state1}]}, {results:[${result1}]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_pending_failed2.out &&
+	numlines=$(cat pending.ids failed.ids | wc -l) &&
+	test $(cat list_constraint_pending_failed2.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list inactive (1)' '
+	state1=`${JOB_CONV} strtostate INACTIVE` &&
+	constraint="{ or: [ {states:[${state1}]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_inactive1.out &&
+	numlines=$(cat inactive.ids | wc -l) &&
+	test $(cat list_constraint_inactive1.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list inactive (2)' '
+	state1=`${JOB_CONV} strtostate SCHED` &&
+	state2=`${JOB_CONV} strtostate RUN` &&
+	constraint="{ not: [ { or: [ {states:[${state1}]}, {states:[${state2}]} ] } ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_inactive2.out &&
+	numlines=$(cat inactive.ids | wc -l) &&
+	test $(cat list_constraint_inactive2.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list have run via t_run (1)' '
+	constraint="{ or: [ {t_run:[\">=0\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_run1.out &&
+	numlines=$(cat running.ids failed.ids timeout.ids completed.ids | wc -l) &&
+	test $(cat list_constraint_t_run1.out | wc -l) -eq ${numlines}
+'
+
+# use a floating point in this one
+test_expect_success 'flux job list have run via t_run (2)' '
+	constraint="{ or: [ {t_run:[\">=1.0\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_run2.out &&
+	numlines=$(cat running.ids failed.ids timeout.ids completed.ids | wc -l) &&
+	test $(cat list_constraint_t_run2.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list have run via t_run (3)' '
+	constraint="{ or: [ {t_run:[\">1.1\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_run3.out &&
+	numlines=$(cat running.ids failed.ids timeout.ids completed.ids | wc -l) &&
+	test $(cat list_constraint_t_run3.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list inactive via t_inactive (1)' '
+	constraint="{ or: [ {t_inactive:[\">=0\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_inactive1.out &&
+	numlines=$(cat inactive.ids | wc -l) &&
+	test $(cat list_constraint_t_inactive1.out | wc -l) -eq ${numlines}
+'
+
+# use a floating point in this one
+test_expect_success 'flux job list inactive via t_inactive (2)' '
+	constraint="{ or: [ {t_inactive:[\">=1.0\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_inactive2.out &&
+	numlines=$(cat inactive.ids | wc -l) &&
+	test $(cat list_constraint_t_inactive2.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list inactive via t_inactive (3)' '
+	constraint="{ or: [ {t_inactive:[\">1.1\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_inactive3.out &&
+	numlines=$(cat inactive.ids | wc -l) &&
+	test $(cat list_constraint_t_inactive3.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list none via t_inactive (1)' '
+	constraint="{ or: [ {t_inactive:[\"<0\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_none1.out &&
+	test $(cat list_constraint_none1.out | wc -l) -eq 0
+'
+
+test_expect_success 'flux job list none via t_inactive (2)' '
+	constraint="{ or: [ {t_inactive:[\"<=0\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_none1.out &&
+	test $(cat list_constraint_none1.out | wc -l) -eq 0
+'
+
+test_expect_success 'flux job list all via t_depend (1)' '
+	constraint="{ or: [ {t_depend:[\">=0\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_all1.out &&
+	numlines=$(cat all.ids | wc -l) &&
+	test $(cat list_constraint_all1.out | wc -l) -eq ${numlines}
+'
+
+# use a floating point in this one
+test_expect_success 'flux job list all via t_depend (2)' '
+	constraint="{ or: [ {t_depend:[\">=1.0\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_all2.out &&
+	numlines=$(cat all.ids | wc -l) &&
+	test $(cat list_constraint_all2.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list all via t_depend (3)' '
+	constraint="{ or: [ {t_depend:[\">1.1\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_all3.out &&
+	numlines=$(cat all.ids | wc -l) &&
+	test $(cat list_constraint_all3.out | wc -l) -eq ${numlines}
+'
+
+#
+# nodelist / hostlist constraint filtering
+#
+
+# N.B. all failed and timeout jobs we explicitly ran on node0, so
+# tests below don't test against node0 to make things easier.
+
+# N.B. failed.ids are jobs that failed after running, thus will have
+# had a node assigned.  timeout.ids obviously ran on a node, but timed
+# out.
+test_expect_success 'flux job list all jobs that ran on any node (1)' '
+	constraint="{ and: [ {hostlist:[\"node[0-3]\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist1.out &&
+	numlines=$(cat completed.ids running.ids failed.ids timeout.ids | wc -l) &&
+	test $(cat constraint_hostlist1.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list all jobs that ran on any node (2)' '
+	constraint="{ and: [ {hostlist:[\"node[2-3]\", \"node1\", \"node0\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist2.out &&
+	numlines=$(cat completed.ids running.ids failed.ids timeout.ids | wc -l) &&
+	test $(cat constraint_hostlist2.out | wc -l) -eq ${numlines}
+'
+
+# We evenly distributed non-bad jobs on nodes, so should be half of the jobs
+test_expect_success 'flux job list all jobs that ran on nodes[1-2] (1)' '
+	constraint="{ and: [ {hostlist:[\"node[1-2]\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist3.out &&
+	numlines=$(expr $(cat completed.ids running.ids | wc -l) / 2) &&
+	test $(cat constraint_hostlist3.out | wc -l) -eq ${numlines}
+'
+
+# We evenly distributed non-bad jobs on nodes, so should be half of the jobs
+test_expect_success 'flux job list all jobs that ran on nodes[1-2] (2)' '
+	constraint="{ and: [ {hostlist:[\"node1\", \"node2\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist4.out &&
+	numlines=$(expr $(cat completed.ids running.ids | wc -l) / 2) &&
+	test $(cat constraint_hostlist4.out | wc -l) -eq ${numlines}
+'
+
+# We evenly distributed non-bad jobs on nodes, so should be quarter of the jobs
+test_expect_success 'flux job list all jobs that ran on node3' '
+	constraint="{ and: [ {hostlist:[\"node3\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist5.out &&
+	numlines=$(expr $(cat completed.ids running.ids | wc -l) / 4) &&
+	test $(cat constraint_hostlist5.out | wc -l) -eq ${numlines}
+'
+
+# We evenly distributed completed jobs on nodes, so should be quarter of the jobs
+test_expect_success 'flux job list completed jobs that ran on node3' '
+	state=`${JOB_CONV} strtostate INACTIVE` &&
+	constraint="{ and: [ {hostlist:[\"node3\"]}, {states:[${state}]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist6.out &&
+	numlines=$(expr $(cat completed.ids | wc -l) / 4) &&
+	test $(cat constraint_hostlist6.out | wc -l) -eq ${numlines}
+'
+
+# We evenly distributed running jobs on nodes, so should be quarter of the jobs
+test_expect_success 'flux job list running jobs that ran on node3' '
+	state=`${JOB_CONV} strtostate RUNNING` &&
+	constraint="{ and: [ {hostlist:[\"node3\"]}, {states:[${state}]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist7.out &&
+	numlines=$(expr $(cat running.ids | wc -l) / 4) &&
+	test $(cat constraint_hostlist7.out | wc -l) -eq ${numlines}
+'
+
+# We evenly distributed running jobs on nodes, so should be half of the jobs
+# For this test, get start time of first running job
+test_expect_success 'flux job list of running jobs that ran on node[1-2] after certain time (1)' '
+	id=`tail -n1 running.ids` &&
+	t_submit=`flux job list-ids ${id} | $jq .t_submit` &&
+	constraint="{ and: [ {hostlist:[\"node[1-2]\"]}, {t_submit:[\">=${t_submit}\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist8.out &&
+	numlines=$(expr $(cat running.ids | wc -l) / 2) &&
+	test $(cat constraint_hostlist8.out | wc -l) -eq ${numlines}
+'
+
+# We evenly distributed running jobs on nodes, so should be half of the jobs
+# For this test, get last inactive time of completed jobs
+test_expect_success 'flux job list of running jobs that ran on node[1-2] after certain time (2)' '
+	id=`head -n1 completed.ids` &&
+	t_inactive=`flux job list-ids ${id} | $jq .t_inactive` &&
+	constraint="{ and: [ {hostlist:[\"node[1-2]\"]}, {t_submit:[\">${t_inactive}\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist9.out &&
+	numlines=$(expr $(cat running.ids | wc -l) / 2) &&
+	test $(cat constraint_hostlist9.out | wc -l) -eq ${numlines}
+'
+
+# We evenly distributed completed & running jobs on nodes, so should be half of the jobs
+test_expect_success 'flux job list of all jobs that ran on node[1-2] after certain time' '
+	id=`head -n1 completed.ids` &&
+	constraint="{ and: [ {hostlist:[\"node[1-2]\"]}, {t_submit:[\">5\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist9.out &&
+	numlines=$(expr $(cat completed.ids running.ids | wc -l) / 2) &&
+	test $(cat constraint_hostlist9.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list by rank (0-3)' '
+	constraint="{ and: [ {ranks:[\"0-3\"]} ] }" &&
+	$jq -j -c -n  "{max_entries:1000, attrs:[], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > constraint_hostlist1.out &&
+	numlines=$(cat completed.ids running.ids failed.ids timeout.ids | wc -l) &&
+	test $(cat constraint_hostlist1.out | wc -l) -eq ${numlines}
+'
+test_expect_success 'flux job list by rank (3)' '
+	constraint="{ and: [ {ranks:[\"3\"]} ] }" &&
+	$jq -j -c -n  \
+	  "{max_entries:1000, attrs:[\"ranks\"], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq -r .jobs[].ranks | uniq > jobs.ranks &&
+	test $(cat jobs.ranks) -eq 3
+'
+test_expect_success 'flux job list by rank (does not exist)' '
+	constraint="{ and: [ {ranks:[\"10\"]} ] }" &&
+	$jq -j -c -n  \
+	  "{max_entries:1000, attrs:[\"ranks\"], constraint:${constraint}}" \
+	  | $RPC job-list.list | $jq -e ".jobs|length == 0"
+'
+test_expect_success 'flux job list by rank (invalid)' '
+	constraint="{ and: [ {ranks:[3]} ] }" &&
+	$jq -j -c -n  \
+	  "{max_entries:1000, attrs:[\"ranks\"], constraint:${constraint}}" \
+	  | test_must_fail $RPC job-list.list 2>ranks.err &&
+	grep "value must be a string" ranks.err
+'
+#
+# legacy RPC tests
+#
+
+test_expect_success 'flux job list all hostname jobs (legacy RPC)' '
+	id=$(id -u) &&
+	$jq -j -c -n  "{max_entries:1000, userid:${id}, states:0, results:0, name:\"hostname\", attrs:[]}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > legacy_hostname_jobs.out &&
+	numlines=$(cat pending.ids completed.ids | wc -l) &&
+	test $(cat legacy_hostname_jobs.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list active hostname jobs (legacy RPC)' '
+	id=$(id -u) &&
+	state=`${JOB_CONV} strtostate ACTIVE` &&
+	$jq -j -c -n  "{max_entries:1000, userid:${id}, states:${state}, results:0, name:\"hostname\", attrs:[]}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > legacy_pending_hostname.out &&
+	test_cmp legacy_pending_hostname.out pending.ids
+'
+
+test_expect_success 'flux job list inactive hostname jobs (legacy RPC)' '
+	id=$(id -u) &&
+	state=`${JOB_CONV} strtostate INACTIVE` &&
+	$jq -j -c -n  "{max_entries:1000, userid:${id}, states:${state}, results:0, name:\"hostname\", attrs:[]}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > legacy_inactive_hostname.out &&
+	test_cmp legacy_inactive_hostname.out completed.ids
+'
+
+test_expect_success 'flux job list invalid queue (legacy RPC)' '
+	id=$(id -u) &&
+	$jq -j -c -n  "{max_entries:1000, userid:${id}, states:0, results:0, queue:\"foof\", attrs:[]}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > legacy_invalid_queue.out &&
+	test $(cat legacy_invalid_queue.out | wc -l) -eq 0
+'
+
+test_expect_success 'flux job list active (legacy RPC)' '
+	state=`${JOB_CONV} strtostate ACTIVE` &&
+	$jq -j -c -n  "{max_entries:1000, userid:${id}, states:${state}, results:0, attrs:[]}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > legacy_active.out &&
+	numlines=$(cat active.ids | wc -l) &&
+	test $(cat legacy_active.out | wc -l) -eq ${numlines}
+'
+
+test_expect_success 'flux job list only canceled jobs (legacy RPC)' '
+	id=$(id -u) &&
+	state=`${JOB_CONV} strtostate INACTIVE` &&
+	result=`${JOB_CONV} strtoresult CANCELED` &&
+	$jq -j -c -n  "{max_entries:1000, userid:${id}, states:${state}, results:${result}, attrs:[]}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > legacy_result_canceled.out &&
+	test_cmp canceled.ids legacy_result_canceled.out
+'
+
+test_expect_success 'flux job list only failed jobs (legacy RPC)' '
+	id=$(id -u) &&
+	state=`${JOB_CONV} strtostate INACTIVE` &&
+	result=`${JOB_CONV} strtoresult FAILED` &&
+	$jq -j -c -n  "{max_entries:1000, userid:${id}, states:${state}, results:${result}, attrs:[]}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > legacy_result_failed.out &&
+	test_cmp failed.ids legacy_result_failed.out
+'
+
+test_expect_success 'flux job list only timeout jobs (legacy RPC)' '
+	id=$(id -u) &&
+	state=`${JOB_CONV} strtostate INACTIVE` &&
+	result=`${JOB_CONV} strtoresult TIMEOUT` &&
+	$jq -j -c -n  "{max_entries:1000, userid:${id}, states:${state}, results:${result}, attrs:[]}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > legacy_result_timeout.out &&
+	test_cmp timeout.ids legacy_result_timeout.out
+'
+
+test_expect_success 'flux job list only completed jobs (legacy RPC)' '
+	id=$(id -u) &&
+	state=`${JOB_CONV} strtostate INACTIVE` &&
+	result=`${JOB_CONV} strtoresult COMPLETED` &&
+	$jq -j -c -n  "{max_entries:1000, userid:${id}, states:${state}, results:${result}, attrs:[]}" \
+	  | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > legacy_result_completed.out &&
+	test_cmp completed.ids legacy_result_completed.out
+'
+
+# with single anonymous queue, queues arrays should be zero length
+test_expect_success 'job stats lists jobs in correct state (mix)' '
+	flux job stats | jq -e ".job_states.depend == 0" &&
+	flux job stats | jq -e ".job_states.priority == 0" &&
+	flux job stats | jq -e ".job_states.sched == $(job_list_state_count pending)" &&
+	flux job stats | jq -e ".job_states.run == $(job_list_state_count running)" &&
+	flux job stats | jq -e ".job_states.cleanup == 0" &&
+	flux job stats | jq -e ".job_states.inactive == $(job_list_state_count inactive)" &&
+	flux job stats | jq -e ".job_states.total == $(job_list_state_count all)" &&
+	flux job stats | jq -e ".successful == $(job_list_state_count completed)" &&
+	flux job stats | jq -e ".failed == $(job_list_state_count failed)" &&
+	flux job stats | jq -e ".canceled == $(job_list_state_count canceled)" &&
+	flux job stats | jq -e ".timeout == $(job_list_state_count timeout)" &&
+	flux job stats | jq -e ".inactive_purged == 0" &&
+	queuelength=$(flux job stats | jq ".queues | length") &&
+	test ${queuelength} -eq 0
+'
+
+# Note: these wait-state tests must be before all jobs are canceled below
+test_expect_success 'flux job list-ids works with --wait-state' '
+	id=`head -n 1 pending.ids` &&
+	flux job list-ids --wait-state=sched $id | jq -e ".state == 8"  &&
+	id=`head -n 1 running.ids` &&
+	flux job list-ids --wait-state=sched $id > /dev/null &&
+	flux job list-ids --wait-state=run $id | jq -e ".state == 16" &&
+	id=`head -n 1 completed.ids` &&
+	flux job list-ids --wait-state=sched $id > /dev/null &&
+	flux job list-ids --wait-state=run $id > /dev/null &&
+	flux job list-ids --wait-state=inactive $id | jq -e ".state == 64" &&
+	id=`head -n 1 canceled.ids` &&
+	flux job list-ids --wait-state=sched $id > /dev/null &&
+	flux job list-ids --wait-state=run $id > /dev/null &&
+	flux job list-ids --wait-state=inactive $id | jq -e ".state == 64"
+'
+
+test_expect_success 'flux job list-ids fail with bad --wait-state' '
+	id=`head -n 1 pending.ids` &&
+	test_must_fail flux job list-ids --wait-state=foo $id > /dev/null
+'
+
+
+test_expect_success 'cleanup job listing jobs ' '
+	# NOTE: do not use flux cancel `cat active.ids` as it races
+	# with the reconstruction of job-list somehow
+	for jobid in `cat active.ids`; do
+	flux cancel $jobid &&
+		fj_wait_event $jobid clean
+	done
+'
+
+wait_inactive() {
+	local i=0
+	while [ "$(flux job list --states=inactive | wc -l)" != "$(job_list_state_count all)" ] \
+		   && [ $i -lt 50 ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "50" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+# Note: annotations are not part of the following comparison
+# because they are omitted from the in-memory journal history
+test_expect_success 'reload the job-list module' "
+	flux job list -a | jq 'del(.annotations)' > before_reload.out &&
+	flux module reload job-list &&
+	wait_inactive
+"
+
+test_expect_success 'job-list: list successfully reconstructed' "
+	flux job list -a | jq 'del(.annotations)' > after_reload.out &&
+	test_cmp before_reload.out after_reload.out
+"
+
+# the canceled checks may look confusing.  We canceled all active jobs
+# right above here, so all those active jobs became canceled as a result
+test_expect_success 'job stats lists jobs in correct state (all inactive)' '
+	flux job stats | jq -e ".job_states.depend == 0" &&
+	flux job stats | jq -e ".job_states.priority == 0" &&
+	flux job stats | jq -e ".job_states.sched == 0" &&
+	flux job stats | jq -e ".job_states.run == 0" &&
+	flux job stats | jq -e ".job_states.cleanup == 0" &&
+	flux job stats | jq -e ".job_states.inactive == $(job_list_state_count all)" &&
+	flux job stats | jq -e ".job_states.total == $(job_list_state_count all)" &&
+	flux job stats | jq -e ".successful == $(job_list_state_count completed)" &&
+	flux job stats | jq -e ".failed == $(job_list_state_count failed)" &&
+	flux job stats | jq -e ".canceled == $(job_list_state_count active canceled)" &&
+	flux job stats | jq -e ".timeout == $(job_list_state_count timeout)" &&
+	flux job stats | jq -e ".inactive_purged == 0" &&
+	queuelength=$(flux job stats | jq ".queues | length") &&
+	test ${queuelength} -eq 0
+'
+
+# job list-inactive
+
+test_expect_success 'flux job list-inactive lists all inactive jobs' '
+	flux job list-inactive > list-inactive.out &&
+	count=`cat list-inactive.out | wc -l` &&
+	test $count -eq $(job_list_state_count all)
+'
+
+test_expect_success 'flux job list-inactive w/ since 0 lists all inactive jobs' '
+	count=`flux job list-inactive --since=0 | wc -l` &&
+	test $count -eq $(job_list_state_count all)
+'
+
+# we hard count numbers here b/c its a --count test
+test_expect_success 'flux job list-inactive w/ count limits output of inactive jobs' '
+	count=`flux job list-inactive --count=14 | wc -l` &&
+	test $count -eq 14
+'
+
+test_expect_success 'flux job list-inactive w/ since -1 leads to error' '
+	test_must_fail flux job list-inactive --since=-1 > list_inactive_error1.out 2>&1 &&
+	grep "invalid payload: since" list_inactive_error1.out
+'
+
+test_expect_success 'flux job list-inactive w/ count -1 leads to error' '
+	test_must_fail flux job list-inactive --count=-1 > list_inactive_error2.out 2>&1 &&
+	grep "invalid payload: max_entries" list_inactive_error2.out
+'
+
+test_expect_success 'flux job list-inactive w/ since (most recent timestamp)' '
+	timestamp=`cat list-inactive.out | head -n 1 | jq .t_inactive` &&
+	count=`flux job list-inactive --since=${timestamp} | wc -l` &&
+	test $count -eq 0
+'
+
+test_expect_success 'flux job list-inactive w/ since (second to most recent timestamp)' '
+	timestamp=`cat list-inactive.out | head -n 2 | tail -n 1 | jq .t_inactive` &&
+	count=`flux job list-inactive --since=${timestamp} | wc -l` &&
+	test $count -eq 1
+'
+
+test_expect_success 'flux job list-inactive w/ since (oldest timestamp)' '
+	timestamp=`cat list-inactive.out | tail -n 1 | jq .t_inactive` &&
+	count=`flux job list-inactive --since=${timestamp} | wc -l` &&
+	test $count -eq 24
+'
+
+test_expect_success 'flux job list-inactive w/ since (middle timestamp #1)' '
+	timestamp=`cat list-inactive.out | head -n 8 | tail -n 1 | jq .t_inactive` &&
+	count=`flux job list-inactive --since=${timestamp} | wc -l` &&
+	test $count -eq 7
+'
+
+test_expect_success 'flux job list-inactive w/ since (middle timestamp #2)' '
+	timestamp=`cat list-inactive.out | head -n 13 | tail -n 1 | jq .t_inactive` &&
+	count=`flux job list-inactive --since=${timestamp} | wc -l` &&
+	test $count -eq 12
+'
+
+
+# job list-id
+
+test_expect_success 'flux job list-ids works with a single ID' '
+	id=`head -n 1 pending.ids` &&
+	flux job list-ids $id | jq -e ".id == ${id}" &&
+	id=`head -n 1 running.ids` &&
+	flux job list-ids $id | jq -e ".id == ${id}" &&
+	id=`head -n 1 inactive.ids` &&
+	flux job list-ids $id | jq -e ".id == ${id}"
+'
+
+test_expect_success 'flux job list-ids multiple IDs works' '
+	ids=$(job_list_state_ids pending) &&
+	flux job list-ids $ids | jq .id > list_idsP.out &&
+	test_cmp list_idsP.out pending.ids &&
+	ids=$(job_list_state_ids running) &&
+	flux job list-ids $ids | jq .id > list_idsR.out &&
+	test_cmp list_idsR.out running.ids &&
+	ids=$(job_list_state_ids inactive) &&
+	flux job list-ids $ids | jq .id > list_idsI.out &&
+	test_cmp list_idsI.out inactive.ids &&
+	ids=$(job_list_state_ids all) &&
+	flux job list-ids $ids | jq .id > list_idsPRI.out &&
+	cat pending.ids running.ids inactive.ids > list_idsPRI.exp &&
+	test_cmp list_idsPRI.exp list_idsPRI.out
+'
+
+test_expect_success 'flux job list-ids fails without ID' '
+	test_must_fail flux job list-ids > list_ids_error1.out 2>&1 &&
+	grep "Usage" list_ids_error1.out
+'
+
+test_expect_success 'flux job list-ids fails with bad ID' '
+	test_must_fail flux job list-ids 1234567890 > list_ids_error2.out 2>&1 &&
+	grep "No such file or directory" list_ids_error2.out
+'
+
+test_expect_success 'flux job list-ids fails with not an ID' '
+	test_must_fail flux job list-ids foobar > list_ids_error3.out 2>&1 &&
+	grep "No such file or directory" list_ids_error3.out
+'
+
+test_expect_success 'flux job list-ids fails with one bad ID out of several' '
+	id1=`head -n 1 pending.ids` &&
+	id2=`head -n 1 running.ids` &&
+	id3=`head -n 1 inactive.ids` &&
+	test_must_fail flux job list-ids ${id1} ${id2} 1234567890 ${id3} \
+		> list_ids_error4.out 2>&1 &&
+	grep "No such file or directory" list_ids_error4.out
+'
+
+# In order to test potential racy behavior, use job state pause/unpause to pause
+# the handling of job state transitions from the job-manager.
+#
+# Note that between the background process of `flux job list-ids` and
+# `unpause`, we must ensure the background process has sent the
+# request to the job-list service and is now waiting for the id to be
+# synced.  We call wait_idsync to check stats for this to ensure the
+# racy behavior is covered.
+
+wait_idsync() {
+	local num=$1
+	local i=0
+	while (! flux module stats --parse idsync.waits job-list > /dev/null 2>&1 \
+		   || [ "$(flux module stats --parse idsync.waits job-list 2> /dev/null)" != "$num" ]) \
+		  && [ $i -lt 50 ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "50" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+test_expect_success NO_CHAIN_LINT 'flux job list-ids waits for job ids (one id)' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	jobid=`flux submit --wait hostname | flux job id`
+	flux job list-ids ${jobid} > list_id_wait1.out &
+	pid=$!
+	wait_idsync 1 &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	wait $pid &&
+	cat list_id_wait1.out | jq -e ".id == ${jobid}"
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job list-ids waits for job ids (different ids)' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	jobid1=`flux submit --wait hostname | flux job id`
+	jobid2=`flux submit --wait hostname | flux job id`
+	flux job list-ids ${jobid1} ${jobid2} > list_id_wait2.out &
+	pid=$!
+	wait_idsync 2 &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	wait $pid &&
+	grep ${jobid1} list_id_wait2.out &&
+	grep ${jobid2} list_id_wait2.out
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job list-ids waits for job ids (same id)' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	jobid=`flux submit --wait hostname | flux job id`
+	flux job list-ids ${jobid} > list_id_wait3A.out &
+	pid1=$!
+	flux job list-ids ${jobid} > list_id_wait3B.out &
+	pid2=$!
+	wait_idsync 1 &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	wait ${pid1} &&
+	wait ${pid2} &&
+	cat list_id_wait3A.out | jq -e ".id == ${jobid}" &&
+	cat list_id_wait3B.out | jq -e ".id == ${jobid}"
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job list-ids waits for job id state (depend)' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	jobid=`flux submit --wait-event=start sleep inf | flux job id`
+	flux job list-ids --wait-state=depend ${jobid} > list_id_wait_state_depend.out &
+	pid=$!
+	wait_idsync 1 &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	wait $pid &&
+	flux cancel $jobid &&
+	fj_wait_event $jobid clean >/dev/null &&
+	cat list_id_wait_state_depend.out | jq -e ".id == ${jobid}"
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job list-ids waits for job id state (run)' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	jobid=`flux submit --wait-event=start sleep inf | flux job id`
+	flux job list-ids --wait-state=run ${jobid} > list_id_wait_state_run.out &
+	pid=$!
+	wait_idsync 1 &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	wait $pid &&
+	flux cancel $jobid &&
+	fj_wait_event $jobid clean >/dev/null &&
+	cat list_id_wait_state_run.out | jq -e ".id == ${jobid}" &&
+	cat list_id_wait_state_run.out | jq .state | ${JOB_CONV} statetostr > list_id_state_run.out &&
+	grep RUN list_id_state_run.out
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job list-ids waits for job id state (cleanup)' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	jobid=`flux submit --wait-event=start sleep inf | flux job id`
+	flux job list-ids --wait-state=cleanup ${jobid} > list_id_wait_state_cleanup.out &
+	pid=$!
+	wait_idsync 1 &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	flux cancel $jobid &&
+	fj_wait_event $jobid clean >/dev/null &&
+	wait $pid &&
+	cat list_id_wait_state_cleanup.out | jq -e ".id == ${jobid}" &&
+	cat list_id_wait_state_cleanup.out | jq .state | ${JOB_CONV} statetostr > list_id_state_cleanup.out &&
+	grep CLEANUP list_id_state_cleanup.out
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job list-ids waits for job id state (inactive)' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	jobid=`flux submit --wait-event=start sleep inf | flux job id`
+	flux job list-ids --wait-state=inactive ${jobid} > list_id_wait_state_inactive.out &
+	pid=$!
+	wait_idsync 1 &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	flux cancel $jobid &&
+	fj_wait_event $jobid clean >/dev/null &&
+	wait $pid &&
+	cat list_id_wait_state_inactive.out | jq -e ".id == ${jobid}" &&
+	cat list_id_wait_state_inactive.out | jq .state | ${JOB_CONV} statetostr > list_id_state_inactive.out &&
+	grep INACTIVE list_id_state_inactive.out
+'
+
+# The job should never reach job state b/c we cancel it before it can
+# run, so will return when job becomes inactive
+test_expect_success NO_CHAIN_LINT 'flux job list-ids waits for job id state (run - cancel)' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	flux queue stop &&
+	jobid=`flux submit sleep inf | flux job id`
+	flux job list-ids --wait-state=run ${jobid} > list_id_wait_state_run_cancel.out &
+	pid=$!
+	wait_idsync 1 &&
+	flux cancel $jobid &&
+	fj_wait_event $jobid clean >/dev/null &&
+	flux queue start &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	wait $pid &&
+	cat list_id_wait_state_run_cancel.out | jq -e ".id == ${jobid}" &&
+	cat list_id_wait_state_run_cancel.out | jq .state | ${JOB_CONV} statetostr > list_id_state_run_cancel.out &&
+	grep INACTIVE list_id_state_run_cancel.out
+'
+
+# Can't guarantee output order, so grep for jobid instead of match jobid
+test_expect_success NO_CHAIN_LINT 'flux job list-ids waits for job ids state (different ids)' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	jobid1=`flux submit --wait-event=start sleep inf | flux job id`
+	jobid2=`flux submit --wait-event=start sleep inf | flux job id`
+	flux job list-ids --wait-state=run ${jobid1} ${jobid2} > list_id_wait_state_different_ids.out &
+	pid=$!
+	wait_idsync 2 &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	wait $pid &&
+	flux cancel $jobid1 $jobid2 &&
+	fj_wait_event $jobid1 clean >/dev/null &&
+	fj_wait_event $jobid2 clean >/dev/null &&
+	grep ${jobid1} list_id_wait_state_different_ids.out &&
+	grep ${jobid2} list_id_wait_state_different_ids.out &&
+	head -n1 list_id_wait_state_different_ids.out | jq .state | ${JOB_CONV} statetostr > list_id_state_different_idsA.out &&
+	tail -n1 list_id_wait_state_different_ids.out | jq .state | ${JOB_CONV} statetostr > list_id_state_different_idsB.out &&
+	grep RUN list_id_state_different_idsA.out &&
+	grep RUN list_id_state_different_idsB.out
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job list-ids waits for job ids state (same id)' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	jobid=`flux submit --wait-event=start sleep inf | flux job id`
+	flux job list-ids --wait-state=run ${jobid} > list_id_wait_state_same_idsA.out &
+	pid1=$!
+	flux job list-ids --wait-state=cleanup ${jobid} > list_id_wait_state_same_idsB.out &
+	pid2=$!
+	wait_idsync 1 &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	wait ${pid1} &&
+	flux cancel $jobid &&
+	fj_wait_event $jobid clean >/dev/null &&
+	wait ${pid2} &&
+	cat list_id_wait_state_same_idsA.out | jq -e ".id == ${jobid}" &&
+	cat list_id_wait_state_same_idsB.out | jq -e ".id == ${jobid}" &&
+	cat list_id_wait_state_same_idsA.out | jq .state | ${JOB_CONV} statetostr > list_id_state_same_idsA.out &&
+	grep RUN list_id_state_same_idsA.out &&
+	cat list_id_wait_state_same_idsB.out | jq .state | ${JOB_CONV} statetostr > list_id_state_same_idsB.out &&
+	grep CLEANUP list_id_state_same_idsB.out
+'
+
+#
+# job list timing
+#
+
+# simply test that value in timestamp increases through job states
+test_expect_success 'flux job list job state timing outputs valid (job inactive)' '
+	jobid=$(flux submit --wait hostname | flux job id) &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".t_submit < .t_depend" &&
+	echo $obj | jq -e ".t_depend < .t_run" &&
+	echo $obj | jq -e ".t_run < .t_cleanup" &&
+	echo $obj | jq -e ".t_cleanup < .t_inactive"
+'
+
+# since job is running, make sure latter states don't exist
+test_expect_success 'flux job list job state timing outputs valid (job running)' '
+	jobid=$(flux submit sleep 60 | flux job id) &&
+	fj_wait_event $jobid start >/dev/null &&
+	wait_jobid_state $jobid run &&
+	obj=$(flux job list -s running | grep $jobid) &&
+	echo $obj | jq -e ".t_submit < .t_depend" &&
+	echo $obj | jq -e ".t_depend < .t_run" &&
+	echo $obj | jq -e ".t_cleanup == null" &&
+	echo $obj | jq -e ".t_inactive == null" &&
+	flux cancel $jobid &&
+	fj_wait_event $jobid clean >/dev/null
+'
+
+#
+# job names
+#
+
+test_expect_success 'flux job list outputs user job name' '
+	jobid=`flux submit --wait --setattr system.job.name=foobar A B C | flux job id` &&
+	echo $jobid > jobname1.id &&
+	wait_jobid_state $jobid inactive &&
+	flux job list -s inactive | grep $jobid | jq -e ".name == \"foobar\""
+'
+
+test_expect_success 'flux job lists first argument for job name' '
+	jobid=`flux submit --wait mycmd arg1 arg2 | flux job id` &&
+	echo $jobid > jobname2.id &&
+	wait_jobid_state $jobid inactive &&
+	flux job list -s inactive | grep $jobid | jq -e ".name == \"mycmd\""
+'
+
+test_expect_success 'flux job lists basename of first argument for job name' '
+	jobid=`flux submit --wait /foo/bar arg1 arg2 | flux job id` &&
+	echo $jobid > jobname3.id &&
+	wait_jobid_state $jobid inactive &&
+	flux job list -s inactive | grep $jobid | jq -e ".name == \"bar\""
+'
+
+test_expect_success 'flux job lists full path for job name if basename fails on first arg' '
+	jobid=`flux submit --wait /foo/bar/ arg1 arg2 | flux job id` &&
+	echo $jobid > jobname4.id &&
+	wait_jobid_state $jobid inactive &&
+	flux job list -s inactive | grep $jobid | jq -e ".name == \"\/foo\/bar\/\""
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify job names preserved across restart' '
+	jobid1=`cat jobname1.id` &&
+	jobid2=`cat jobname2.id` &&
+	jobid3=`cat jobname3.id` &&
+	jobid4=`cat jobname4.id` &&
+	flux job list -s inactive | grep ${jobid1} | jq -e ".name == \"foobar\"" &&
+	flux job list -s inactive | grep ${jobid2} | jq -e ".name == \"mycmd\"" &&
+	flux job list -s inactive | grep ${jobid3} | jq -e ".name == \"bar\"" &&
+	flux job list -s inactive | grep ${jobid4} | jq -e ".name == \"\/foo\/bar\/\""
+'
+
+#
+# job cwd
+#
+
+test_expect_success 'flux job list outputs cwd' '
+	pwd=$(pwd) &&
+	jobid=`flux submit --wait true | flux job id` &&
+	echo $jobid > jobcwd.id &&
+	wait_jobid_state $jobid inactive &&
+	flux job list -s inactive | grep $jobid | jq -e ".cwd == \"${pwd}\""
+'
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify job cwd preserved across restart' '
+	pwd=$(pwd) &&
+	jobid=`cat jobcwd.id` &&
+	flux job list -s inactive | grep ${jobid} | jq -e ".cwd == \"${pwd}\""
+'
+
+#
+# job queue
+#
+
+test_expect_success 'flux job list output no queue if queue not set' '
+	jobid=`flux submit --wait true | flux job id` &&
+	echo $jobid > jobqueue1.id &&
+	wait_jobid_state $jobid inactive &&
+	flux job list -s inactive | grep $jobid | jq -e ".queue == null"
+'
+
+test_expect_success 'reconfigure with one queue' '
+	flux config load <<-EOT &&
+	[queues.foo]
+	EOT
+	flux queue start --queue=foo
+'
+
+test_expect_success 'flux job list outputs queue' '
+	jobid=`flux submit --wait --queue=foo true | flux job id` &&
+	echo $jobid > jobqueue2.id &&
+	wait_jobid_state $jobid inactive &&
+	flux job list -s inactive | grep $jobid | jq -e ".queue == \"foo\""
+'
+
+test_expect_success 'reconfigure with no queues' '
+	flux config load < /dev/null
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify job queue preserved across restart' '
+	jobid1=`cat jobqueue1.id` &&
+	jobid2=`cat jobqueue2.id` &&
+	flux job list -s inactive | grep ${jobid1} | jq -e ".queue == null" &&
+	flux job list -s inactive | grep ${jobid2} | jq -e ".queue == \"foo\""
+'
+
+#
+# job project and bank
+#
+
+test_expect_success 'support jobspec updates of project and bank' '
+	flux jobtap load --remove=all ${PLUGINPATH}/project-bank-validate.so
+'
+
+test_expect_success 'flux job list outputs no project and bank by default' '
+	jobid=`flux submit --wait true | flux job id` &&
+	echo $jobid > jobprojectbank1.id &&
+	wait_jobid_state $jobid inactive &&
+	flux job list -s inactive | grep $jobid | jq -e ".project == null" &&
+	flux job list -s inactive | grep $jobid | jq -e ".bank == null"
+'
+# initially put job on hold, jobspec-updates don't matter after the job is running
+test_expect_success 'flux job list outputs project and bank if one set' '
+	jobid=`flux submit --urgency=hold true | flux job id` &&
+	echo $jobid > jobprojectbank2.id &&
+	flux update $jobid project=foo &&
+	flux update $jobid bank=bar &&
+	flux job urgency $jobid default &&
+	wait_jobid_state $jobid inactive &&
+	flux job list -s inactive | grep $jobid | jq -e ".project == \"foo\"" &&
+	flux job list -s inactive | grep $jobid | jq -e ".bank == \"bar\""
+'
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify job project and bank preserved across restart' '
+	jobid1=`cat jobprojectbank1.id` &&
+	jobid2=`cat jobprojectbank2.id` &&
+	flux job list -s inactive | grep $jobid1 | jq -e ".project == null" &&
+	flux job list -s inactive | grep $jobid1 | jq -e ".bank == null" &&
+	flux job list -s inactive | grep $jobid | jq -e ".project == \"foo\"" &&
+	flux job list -s inactive | grep $jobid | jq -e ".bank == \"bar\""
+'
+
+test_expect_success 'remove jobtap plugins' '
+	flux jobtap remove all
+'
+
+#
+# job task count
+#
+
+test_expect_success 'flux job list outputs ntasks correctly (1 task)' '
+	jobid=`flux submit --wait hostname | flux job id` &&
+	echo $jobid > taskcount1.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ntasks == 1"
+'
+
+test_expect_success 'flux job list outputs ntasks correctly (4 tasks)' '
+	jobid=`flux submit --wait -n4 hostname | flux job id` &&
+	echo $jobid > taskcount2.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ntasks == 4"
+'
+
+test_expect_success 'flux job list outputs ntasks correctly (4 nodes, 4 tasks)' '
+	jobid=`flux submit --wait -N4 -n4 hostname | flux job id` &&
+	echo $jobid > taskcount3.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ntasks == 4"
+'
+
+# not-evenly divisible tasks / nodes should force "total" count of tasks in jobspec
+test_expect_success 'flux job list outputs ntasks correctly (3 nodes, 4 tasks)' '
+	jobid=`flux submit --wait -N3 -n4 hostname | flux job id` &&
+	echo $jobid > taskcount4.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ntasks == 4"
+'
+
+test_expect_success 'flux job list outputs ntasks correctly (3 cores)' '
+	jobid=`flux submit --wait --cores=3 hostname | flux job id` &&
+	echo $jobid > taskcount5.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ntasks == 3"
+'
+
+test_expect_success 'flux job list outputs ntasks correctly (tasks-per-node)' '
+	jobid=`flux submit --wait -N2 --tasks-per-node=3 hostname | flux job id` &&
+	echo $jobid > taskcount6.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ntasks == 6"
+'
+
+# N.B. As of this test writing, tasks-per-node uses
+# per-resource.type=node.  But write more direct test in case of
+# future changes.
+test_expect_success 'flux job list outputs ntasks correctly (per-resource.type=node)' '
+	totalnodes=$(flux resource list -s up -no {nnodes}) &&
+	totalcores=$(flux resource list -s up -no {ncores}) &&
+	extra=$((totalcores / totalnodes + 2)) &&
+	jobid=$(flux submit --wait -N ${totalnodes} -n ${totalcores} \
+		-o per-resource.type=node \
+		-o per-resource.count=${extra} \
+		hostname | flux job id) &&
+	echo $jobid > taskcount7.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	expected=$((totalnodes * extra)) &&
+	echo ${expected} > per_resource_type_node_ntasks.exp &&
+	echo $obj | jq -e ".ntasks == ${expected}"
+'
+
+test_expect_success 'flux job list outputs ntasks correctly (cores / tasks-per-core)' '
+	jobid=`flux submit --wait --cores=4 --tasks-per-core=2 hostname | flux job id` &&
+	echo $jobid > taskcount8.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ntasks == 8"
+'
+
+test_expect_success 'flux job list outputs ntasks correctly (tasks / cores-per-task)' '
+	jobid=$(flux submit --wait -n2 --cores-per-task=2 \
+		-o per-resource.type=core \
+		-o per-resource.count=2 \
+		hostname | flux job id) &&
+	echo $jobid > taskcount9.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ntasks == 8"
+'
+
+test_expect_success 'flux job list outputs ntasks correctly (nodes / tasks-per-core 2)' '
+	totalnodes=$(flux resource list -s up -no {nnodes}) &&
+	totalcores=$(flux resource list -s up -no {ncores}) &&
+	jobid=`flux submit --wait -N ${totalnodes} --tasks-per-core=2 hostname | flux job id` &&
+	echo $jobid > taskcount10.id &&
+	wait_jobid_state $jobid inactive &&
+	expected=$((totalcores * 2)) &&
+	echo ${expected} > per_resource_type_core_ntasks1.exp &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ntasks == ${expected}"
+'
+
+# N.B. As of this test writing, tasks-per-core uses
+# per-resource.type=core.  But write direct test in case of future
+# changes.
+test_expect_success 'flux job list outputs ntasks correctly (cores / per-resource.type=core)' '
+	totalcores=$(flux resource list -s up -no {ncores}) &&
+	jobid=$(flux submit --wait --cores=${totalcores} \
+		-o per-resource.type=core \
+		-o per-resource.count=2 \
+		hostname | flux job id) &&
+	echo $jobid > taskcount11.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	expected=$((totalcores * 2)) &&
+	echo ${expected} > per_resource_type_core_ntasks2.exp &&
+	echo $obj | jq -e ".ntasks == ${expected}"
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify task count preserved across restart' '
+	jobid1=`cat taskcount1.id` &&
+	jobid2=`cat taskcount2.id` &&
+	jobid3=`cat taskcount3.id` &&
+	jobid4=`cat taskcount4.id` &&
+	jobid5=`cat taskcount5.id` &&
+	jobid6=`cat taskcount6.id` &&
+	jobid7=`cat taskcount7.id` &&
+	jobid8=`cat taskcount8.id` &&
+	jobid9=`cat taskcount9.id` &&
+	jobid10=`cat taskcount10.id` &&
+	jobid11=`cat taskcount11.id` &&
+	obj=$(flux job list -s inactive | grep ${jobid1}) &&
+	echo $obj | jq -e ".ntasks == 1" &&
+	obj=$(flux job list -s inactive | grep ${jobid2}) &&
+	echo $obj | jq -e ".ntasks == 4" &&
+	obj=$(flux job list -s inactive | grep ${jobid3}) &&
+	echo $obj | jq -e ".ntasks == 4" &&
+	obj=$(flux job list -s inactive | grep ${jobid4}) &&
+	echo $obj | jq -e ".ntasks == 4" &&
+	obj=$(flux job list -s inactive | grep ${jobid5}) &&
+	echo $obj | jq -e ".ntasks == 3" &&
+	obj=$(flux job list -s inactive | grep ${jobid6}) &&
+	echo $obj | jq -e ".ntasks == 6" &&
+	obj=$(flux job list -s inactive | grep ${jobid7}) &&
+	expected=$(cat per_resource_type_node_ntasks.exp) &&
+	echo $obj | jq -e ".ntasks == ${expected}" &&
+	obj=$(flux job list -s inactive | grep ${jobid8}) &&
+	echo $obj | jq -e ".ntasks == 8" &&
+	obj=$(flux job list -s inactive | grep ${jobid9}) &&
+	echo $obj | jq -e ".ntasks == 8" &&
+	obj=$(flux job list -s inactive | grep ${jobid10}) &&
+	expected=$(cat per_resource_type_core_ntasks1.exp) &&
+	echo $obj | jq -e ".ntasks == ${expected}" &&
+	obj=$(flux job list -s inactive | grep ${jobid11}) &&
+	expected=$(cat per_resource_type_core_ntasks2.exp) &&
+	echo $obj | jq -e ".ntasks == ${expected}"
+'
+
+#
+# job core count
+#
+
+test_expect_success 'flux job list outputs ncores correctly (1 task)' '
+	jobid=`flux submit --wait -n1 hostname | flux job id` &&
+	echo $jobid > corecount1.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 1"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (2 tasks)' '
+	jobid=`flux submit --wait -n2 hostname | flux job id` &&
+	echo $jobid > corecount2.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 2"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 task, cores-per-task)' '
+	jobid=`flux submit --wait -n1 --cores-per-task=2 hostname | flux job id` &&
+	echo $jobid > corecount3.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 2"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (2 tasks, cores-per-task)' '
+	jobid=`flux submit --wait -n2 --cores-per-task=2 hostname | flux job id` &&
+	echo $jobid > corecount4.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 4"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 node, 1 task)' '
+	jobid=`flux submit --wait -N1 -n1 hostname | flux job id` &&
+	echo $jobid > corecount5.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 1"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 node, 2 tasks)' '
+	jobid=`flux submit --wait -N1 -n2 hostname | flux job id` &&
+	echo $jobid > corecount6.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 2"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (2 nodes, 2 tasks)' '
+	jobid=`flux submit --wait -N2 -n2 hostname | flux job id` &&
+	echo $jobid > corecount7.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 2"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 node, 1 task, exclusive)' '
+	totalnodes=$(flux resource list -s up -no {nnodes}) &&
+	totalcores=$(flux resource list -s up -no {ncores}) &&
+	corespernode=$((totalcores / totalnodes)) &&
+	jobid=`flux submit --wait -N1 -n1 --exclusive hostname | flux job id` &&
+	echo $jobid > corecount8.id &&
+	wait_jobid_state $jobid inactive &&
+	expected=$((corespernode * 1)) &&
+	echo ${expected} > ncores_exclusive1.exp &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == ${expected}"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 node, 2 tasks, exclusive)' '
+	totalnodes=$(flux resource list -s up -no {nnodes}) &&
+	totalcores=$(flux resource list -s up -no {ncores}) &&
+	corespernode=$((totalcores / totalnodes)) &&
+	jobid=`flux submit --wait -N1 -n2 --exclusive hostname | flux job id` &&
+	echo $jobid > corecount9.id &&
+	wait_jobid_state $jobid inactive &&
+	expected=$((corespernode * 1)) &&
+	echo ${expected} > ncores_exclusive2.exp &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == ${expected}"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (2 nodes, 2 tasks, exclusive)' '
+	totalnodes=$(flux resource list -s up -no {nnodes}) &&
+	totalcores=$(flux resource list -s up -no {ncores}) &&
+	corespernode=$((totalcores / totalnodes)) &&
+	jobid=`flux submit --wait -N2 -n2 --exclusive hostname | flux job id` &&
+	echo $jobid > corecount10.id &&
+	wait_jobid_state $jobid inactive &&
+	expected=$((corespernode * 2)) &&
+	echo ${expected} > ncores_exclusive3.exp &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == ${expected}"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 node, 1 task, cores-per-task)' '
+	jobid=`flux submit --wait -N1 -n1 --cores-per-task=2 hostname | flux job id` &&
+	echo $jobid > corecount11.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 2"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 node, 1 task, cores-per-task)' '
+	jobid=`flux submit --wait -N1 -n1 --cores-per-task=2 hostname | flux job id` &&
+	echo $jobid > corecount12.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 2"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (2 nodes, 2 tasks, cores-per-task)' '
+	jobid=`flux submit --wait -N2 -n2 --cores-per-task=2 hostname | flux job id` &&
+	echo $jobid > corecount13.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 4"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 core)' '
+	jobid=`flux submit --wait --cores=1 hostname | flux job id` &&
+	echo $jobid > corecount14.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 1"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (2 cores)' '
+	jobid=`flux submit --wait --cores=2 hostname | flux job id` &&
+	echo $jobid > corecount15.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 2"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 node, 1 core)' '
+	jobid=`flux submit --wait -N1 --cores=1 hostname | flux job id` &&
+	echo $jobid > corecount16.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 1"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 node, 2 cores)' '
+	jobid=`flux submit --wait -N1 --cores=2 hostname | flux job id` &&
+	echo $jobid > corecount17.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 2"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (2 nodes, 2 cores)' '
+	jobid=`flux submit --wait -N2 --cores=2 hostname | flux job id` &&
+	echo $jobid > corecount18.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == 2"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 node, 1 task, exclusive)' '
+	totalnodes=$(flux resource list -s up -no {nnodes}) &&
+	totalcores=$(flux resource list -s up -no {ncores}) &&
+	corespernode=$((totalcores / totalnodes)) &&
+	jobid=`flux submit --wait -N1 --cores=1 --exclusive hostname | flux job id` &&
+	echo $jobid > corecount19.id &&
+	wait_jobid_state $jobid inactive &&
+	expected=$((corespernode * 1)) &&
+	echo ${expected} > ncores_exclusive4.exp &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == ${expected}"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (1 node, 2 tasks, exclusive)' '
+	totalnodes=$(flux resource list -s up -no {nnodes}) &&
+	totalcores=$(flux resource list -s up -no {ncores}) &&
+	corespernode=$((totalcores / totalnodes)) &&
+	jobid=`flux submit --wait -N1 --cores=2 --exclusive hostname | flux job id` &&
+	echo $jobid > corecount20.id &&
+	wait_jobid_state $jobid inactive &&
+	expected=$((corespernode * 1)) &&
+	echo ${expected} > ncores_exclusive5.exp &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == ${expected}"
+'
+
+test_expect_success 'flux job list outputs ncores correctly (2 nodes, 2 tasks, exclusive)' '
+	totalnodes=$(flux resource list -s up -no {nnodes}) &&
+	totalcores=$(flux resource list -s up -no {ncores}) &&
+	corespernode=$((totalcores / totalnodes)) &&
+	jobid=`flux submit --wait -N2 --cores=2 --exclusive hostname | flux job id` &&
+	echo $jobid > corecount21.id &&
+	wait_jobid_state $jobid inactive &&
+	expected=$((corespernode * 2)) &&
+	echo ${expected} > ncores_exclusive6.exp &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ncores == ${expected}"
+'
+
+# use flux queue to ensure jobs stay in pending state
+test_expect_success 'flux job list lists ncores if pending & tasks specified' '
+	flux queue stop &&
+	id=$(flux submit -n3 hostname | flux job id) &&
+	flux job list -s pending | grep ${id} &&
+	flux job list-ids ${id} | jq -e ".ncores == 3" &&
+	flux cancel ${id} &&
+	flux queue start
+'
+
+# use flux queue to ensure jobs stay in pending state
+test_expect_success 'flux job list does not list ncores if pending & nodes exclusive' '
+	flux queue stop &&
+	id=$(flux submit -N1 --exclusive hostname | flux job id) &&
+	flux job list -s pending | grep ${id} &&
+	flux job list-ids ${id} | jq -e ".ncores == null" &&
+	flux cancel ${id} &&
+	flux queue start
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify core count preserved across restart' '
+	jobid1=`cat corecount1.id` &&
+	jobid2=`cat corecount2.id` &&
+	jobid3=`cat corecount3.id` &&
+	jobid4=`cat corecount4.id` &&
+	jobid5=`cat corecount5.id` &&
+	jobid6=`cat corecount6.id` &&
+	jobid7=`cat corecount7.id` &&
+	jobid8=`cat corecount8.id` &&
+	jobid9=`cat corecount9.id` &&
+	jobid10=`cat corecount10.id` &&
+	jobid11=`cat corecount11.id` &&
+	jobid12=`cat corecount12.id` &&
+	jobid13=`cat corecount13.id` &&
+	jobid14=`cat corecount14.id` &&
+	jobid15=`cat corecount15.id` &&
+	jobid16=`cat corecount16.id` &&
+	jobid17=`cat corecount17.id` &&
+	jobid18=`cat corecount18.id` &&
+	jobid19=`cat corecount19.id` &&
+	jobid20=`cat corecount20.id` &&
+	jobid21=`cat corecount21.id` &&
+	obj=$(flux job list -s inactive | grep ${jobid1}) &&
+	echo $obj | jq -e ".ncores == 1" &&
+	obj=$(flux job list -s inactive | grep ${jobid2}) &&
+	echo $obj | jq -e ".ncores == 2" &&
+	obj=$(flux job list -s inactive | grep ${jobid3}) &&
+	echo $obj | jq -e ".ncores == 2" &&
+	obj=$(flux job list -s inactive | grep ${jobid4}) &&
+	echo $obj | jq -e ".ncores == 4" &&
+	obj=$(flux job list -s inactive | grep ${jobid5}) &&
+	echo $obj | jq -e ".ncores == 1" &&
+	obj=$(flux job list -s inactive | grep ${jobid6}) &&
+	echo $obj | jq -e ".ncores == 2" &&
+	obj=$(flux job list -s inactive | grep ${jobid7}) &&
+	echo $obj | jq -e ".ncores == 2" &&
+	obj=$(flux job list -s inactive | grep ${jobid8}) &&
+	expected=$(cat ncores_exclusive1.exp) &&
+	echo $obj | jq -e ".ncores == ${expected}" &&
+	obj=$(flux job list -s inactive | grep ${jobid9}) &&
+	expected=$(cat ncores_exclusive2.exp) &&
+	echo $obj | jq -e ".ncores == ${expected}" &&
+	obj=$(flux job list -s inactive | grep ${jobid10}) &&
+	expected=$(cat ncores_exclusive3.exp) &&
+	echo $obj | jq -e ".ncores == ${expected}" &&
+	obj=$(flux job list -s inactive | grep ${jobid11}) &&
+	echo $obj | jq -e ".ncores == 2" &&
+	obj=$(flux job list -s inactive | grep ${jobid12}) &&
+	echo $obj | jq -e ".ncores == 2" &&
+	obj=$(flux job list -s inactive | grep ${jobid13}) &&
+	echo $obj | jq -e ".ncores == 4" &&
+	obj=$(flux job list -s inactive | grep ${jobid14}) &&
+	echo $obj | jq -e ".ncores == 1" &&
+	obj=$(flux job list -s inactive | grep ${jobid15}) &&
+	echo $obj | jq -e ".ncores == 2" &&
+	obj=$(flux job list -s inactive | grep ${jobid16}) &&
+	echo $obj | jq -e ".ncores == 1" &&
+	obj=$(flux job list -s inactive | grep ${jobid17}) &&
+	echo $obj | jq -e ".ncores == 2" &&
+	obj=$(flux job list -s inactive | grep ${jobid18}) &&
+	echo $obj | jq -e ".ncores == 2" &&
+	obj=$(flux job list -s inactive | grep ${jobid19}) &&
+	expected=$(cat ncores_exclusive4.exp) &&
+	echo $obj | jq -e ".ncores == ${expected}" &&
+	obj=$(flux job list -s inactive | grep ${jobid20}) &&
+	expected=$(cat ncores_exclusive5.exp) &&
+	echo $obj | jq -e ".ncores == ${expected}" &&
+	obj=$(flux job list -s inactive | grep ${jobid21}) &&
+	expected=$(cat ncores_exclusive6.exp) &&
+	echo $obj | jq -e ".ncores == ${expected}"
+'
+
+#
+# job node count
+#
+
+test_expect_success 'flux job list outputs nnodes correctly (1 task / 1 node)' '
+	jobid=`flux submit --wait -n1 hostname | flux job id` &&
+	echo $jobid > nodecount1.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".nnodes == 1"
+'
+
+test_expect_success 'flux job list outputs nnodes correctly (2 tasks, / 1 node)' '
+	jobid=`flux submit --wait -n2 hostname | flux job id` &&
+	echo $jobid > nodecount2.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".nnodes == 1"
+'
+
+test_expect_success 'flux job list outputs nnodes correctly (3 tasks, / 2 nodes)' '
+	jobid=`flux submit --wait -n3 hostname | flux job id` &&
+	echo $jobid > nodecount3.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".nnodes == 2"
+'
+
+test_expect_success 'flux job list outputs nnodes correctly (5 tasks, / 3 nodes)' '
+	jobid=`flux submit --wait -n5 hostname | flux job id` &&
+	echo $jobid > nodecount4.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".nnodes == 3"
+'
+
+# use flux queue to ensure jobs stay in pending state
+test_expect_success 'flux job list does not list nnodes if no nodes requested' '
+	flux queue stop &&
+	id=$(flux submit -n1 hostname | flux job id) &&
+	flux job list -s pending | grep ${id} &&
+	flux job list-ids ${id} | jq -e ".nnodes == null" &&
+	flux cancel ${id} &&
+	flux queue start
+'
+
+# use flux queue to ensure jobs stay in pending state
+test_expect_success 'flux job list lists nnodes for pending jobs if nodes requested' '
+	flux queue stop &&
+	id1=$(flux submit -N1 hostname | flux job id) &&
+	id2=$(flux submit -N3 hostname | flux job id) &&
+	flux job list -s pending | grep ${id1} &&
+	flux job list -s pending | grep ${id2} &&
+	flux job list-ids ${id1} | jq -e ".nnodes == 1" &&
+	flux job list-ids ${id2} | jq -e ".nnodes == 3" &&
+	flux cancel ${id1} ${id2} &&
+	flux queue start
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify nnodes preserved across restart' '
+	jobid1=`cat nodecount1.id` &&
+	jobid2=`cat nodecount2.id` &&
+	jobid3=`cat nodecount3.id` &&
+	jobid4=`cat nodecount4.id` &&
+	obj=$(flux job list -s inactive | grep ${jobid1}) &&
+	echo $obj | jq -e ".nnodes == 1" &&
+	obj=$(flux job list -s inactive | grep ${jobid2}) &&
+	echo $obj | jq -e ".nnodes == 1" &&
+	obj=$(flux job list -s inactive | grep ${jobid3}) &&
+	echo $obj | jq -e ".nnodes == 2" &&
+	obj=$(flux job list -s inactive | grep ${jobid4}) &&
+	echo $obj | jq -e ".nnodes == 3"
+'
+
+#
+# job rank list / nodelist
+#
+
+test_expect_success 'flux job list outputs ranks/nodelist correctly (1 node)' '
+	jobid=`flux submit --wait -N1 hostname | flux job id` &&
+	echo $jobid > nodelist1.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ranks == \"0\"" &&
+	nodes=`flux job info $jobid R | flux R decode --nodelist` &&
+	echo $obj | jq -e ".nodelist == \"${nodes}\""
+'
+
+test_expect_success 'flux job list outputs ranks/nodelist correctly (3 nodes)' '
+	jobid=`flux submit --wait -N3 hostname | flux job id` &&
+	echo $jobid > nodelist2.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".ranks == \"[0-2]\"" &&
+	nodes=`flux job info $jobid R | flux R decode --nodelist` &&
+	echo $obj | jq -e ".nodelist == \"${nodes}\""
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify ranks/nodelist preserved across restart' '
+	jobid1=`cat nodelist1.id` &&
+	jobid2=`cat nodelist2.id` &&
+	obj=$(flux job list -s inactive | grep ${jobid1}) &&
+	echo $obj | jq -e ".ranks == \"0\"" &&
+	nodes=`flux job info ${jobid1} R | flux R decode --nodelist` &&
+	echo $obj | jq -e ".nodelist == \"${nodes}\"" &&
+	obj=$(flux job list -s inactive | grep ${jobid2}) &&
+	echo $obj | jq -e ".ranks == \"[0-2]\"" &&
+	nodes=`flux job info ${jobid2} R | flux R decode --nodelist` &&
+	echo $obj | jq -e ".nodelist == \"${nodes}\""
+'
+
+#
+# job success
+#
+
+test_expect_success 'flux job list outputs success correctly (true)' '
+	jobid=`flux submit --wait hostname | flux job id` &&
+	echo $jobid > success1.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".success == true"
+'
+
+test_expect_success 'flux job list outputs success correctly (false)' '
+	jobid=`flux submit --wait nosuchcommand | flux job id` &&
+	echo $jobid > success2.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	echo $obj | jq -e ".success == false"
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify task count preserved across restart' '
+	jobid1=`cat success1.id` &&
+	jobid2=`cat success2.id` &&
+	obj=$(flux job list -s inactive | grep ${jobid1}) &&
+	test_debug "echo $obj" &&
+	echo $obj | jq -e ".success == true" &&
+	obj=$(flux job list -s inactive | grep ${jobid2}) &&
+	echo $obj | jq -e ".success == false"
+'
+
+# job exceptions
+
+test_expect_success 'flux job list outputs exceptions correctly (no exception)' '
+	jobid=`flux submit --wait hostname | flux job id` &&
+	echo $jobid > exceptions1.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	test_debug "echo $obj" &&
+	echo $obj | jq -e ".exception_occurred == false" &&
+	echo $obj | jq -e ".exception_severity == null" &&
+	echo $obj | jq -e ".exception_type == null" &&
+	echo $obj | jq -e ".exception_note == null"
+'
+
+test_expect_success 'flux job list outputs exceptions correctly (exception)' '
+	jobid=`flux submit --wait nosuchcommand | flux job id` &&
+	echo $jobid > exceptions2.id &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	test_debug "echo $obj" &&
+	echo $obj | jq -e ".exception_occurred == true" &&
+	echo $obj | jq -e ".exception_severity == 0" &&
+	echo $obj | jq -e ".exception_type == \"exec\"" &&
+	echo $obj | jq .exception_note | grep "No such file or directory"
+'
+
+# N.B. sleepinf.sh and wait-event on job data to workaround
+# rare job startup race.  See #5210
+test_expect_success 'flux job list outputs exceptions correctly (exception cancel no message)' '
+	jobid=`flux submit ./sleepinf.sh | flux job id` &&
+	echo $jobid > exceptions3.id &&
+	flux job wait-event -W -p guest.output $jobid data &&
+	flux cancel $jobid &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	test_debug "echo $obj" &&
+	echo $obj | jq -e ".exception_occurred == true" &&
+	echo $obj | jq -e ".exception_severity == 0" &&
+	echo $obj | jq -e ".exception_type == \"cancel\"" &&
+	echo $obj | jq -e ".exception_note == \"\""
+'
+
+# N.B. sleepinf.sh and wait-event on job data to workaround
+# rare job startup race.  See #5210
+test_expect_success 'flux job list outputs exceptions correctly (exception cancel w/ message)' '
+	jobid=`flux submit ./sleepinf.sh | flux job id` &&
+	echo $jobid > exceptions4.id &&
+	flux job wait-event -W -p guest.output $jobid data &&
+	flux cancel -m "mecanceled" $jobid &&
+	wait_jobid_state $jobid inactive &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	test_debug "echo $obj" &&
+	echo $obj | jq -e ".exception_occurred == true" &&
+	echo $obj | jq -e ".exception_severity == 0" &&
+	echo $obj | jq -e ".exception_type == \"cancel\"" &&
+	echo $obj | jq -e ".exception_note == \"mecanceled\""
+'
+
+# N.B. sleepinf.sh and wait-event on job data to workaround
+# rare job startup race.  See #5210
+test_expect_success 'flux job list outputs exceptions correctly (user exception)' '
+	jobid=`flux submit ./sleepinf.sh | flux job id` &&
+	echo $jobid > exceptions5.id &&
+	test_debug "echo started $jobid, waiting for first line of output" &&
+	flux job wait-event -W -p guest.output $jobid data &&
+	test_debug "echo raising user exception foo" &&
+	flux job raise --type=foo --severity=0 -m "foobar" $jobid &&
+	test_debug "echo waiting for $jobid to become inactive" &&
+	wait_jobid_state $jobid inactive &&
+	test_debug "flux job list -s inactive | jq" &&
+	obj=$(flux job list -s inactive | grep $jobid) &&
+	test_debug "echo $obj" &&
+	echo $obj | jq -e ".exception_occurred == true" &&
+	echo $obj | jq -e ".exception_severity == 0" &&
+	echo $obj | jq -e ".exception_type == \"foo\"" &&
+	echo $obj | jq -e ".exception_note == \"foobar\""
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify task count preserved across restart' '
+	jobid1=`cat exceptions1.id` &&
+	jobid2=`cat exceptions2.id` &&
+	jobid3=`cat exceptions3.id` &&
+	jobid4=`cat exceptions4.id` &&
+	jobid5=`cat exceptions5.id` &&
+	obj=$(flux job list -s inactive | grep ${jobid1}) &&
+	echo $obj | jq -e ".success == true" &&
+	echo $obj | jq -e ".exception_occurred == false" &&
+	echo $obj | jq -e ".exception_severity == null" &&
+	echo $obj | jq -e ".exception_type == null" &&
+	echo $obj | jq -e ".exception_note == null" &&
+	obj=$(flux job list -s inactive | grep ${jobid2}) &&
+	echo $obj | jq -e ".exception_occurred == true" &&
+	echo $obj | jq -e ".exception_severity == 0" &&
+	echo $obj | jq -e ".exception_type == \"exec\"" &&
+	echo $obj | jq .exception_note | grep "No such file or directory" &&
+	obj=$(flux job list -s inactive | grep ${jobid3}) &&
+	echo $obj | jq -e ".exception_occurred == true" &&
+	echo $obj | jq -e ".exception_severity == 0" &&
+	echo $obj | jq -e ".exception_type == \"cancel\"" &&
+	echo $obj | jq -e ".exception_note == \"\"" &&
+	obj=$(flux job list -s inactive | grep ${jobid4}) &&
+	echo $obj | jq -e ".exception_occurred == true" &&
+	echo $obj | jq -e ".exception_severity == 0" &&
+	echo $obj | jq -e ".exception_type == \"cancel\"" &&
+	echo $obj | jq -e ".exception_note == \"mecanceled\"" &&
+	obj=$(flux job list -s inactive | grep ${jobid5}) &&
+	echo $obj | jq -e ".exception_occurred == true" &&
+	echo $obj | jq -e ".exception_severity == 0" &&
+	echo $obj | jq -e ".exception_type == \"foo\"" &&
+	echo $obj | jq -e ".exception_note == \"foobar\""
+'
+
+# expiration time
+
+test_expect_success 'flux job list outputs expiration time when set' '
+	jobid=$(flux submit -t 500s sleep 1000 | flux job id) &&
+	echo $jobid > expiration.id &&
+	fj_wait_event $jobid start &&
+	flux job list | grep $jobid > expiration.json &&
+	test_debug "cat expiration.json" &&
+	jq -e ".expiration > now" < expiration.json &&
+	flux cancel $jobid
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify task count preserved across restart' '
+	jobid=`cat expiration.id` &&
+	flux job list -s inactive | grep ${jobid} > expiration2.json &&
+	jq -e ".expiration > now" < expiration2.json
+'
+
+# duration time
+
+test_expect_success 'flux job list outputs duration time when set' '
+	jobid=$(flux submit -t 60m sleep 1000 | flux job id) &&
+	echo $jobid > duration.id &&
+	fj_wait_event $jobid start &&
+	flux job list | grep $jobid > duration.json &&
+	test_debug "cat duration.json" &&
+	jq -e ".duration == 3600.0" < duration.json &&
+	flux cancel $jobid
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list
+'
+
+test_expect_success 'verify task count preserved across restart' '
+	jobid=`cat duration.id` &&
+	flux job list -s inactive | grep ${jobid} > duration2.json &&
+	jq -e ".duration == 3600.0" < duration2.json
+'
+
+# all job attributes
+
+# note that not all attributes may be returned by via the 'all'
+# attribute.  e.g. exception data won't be returned for a job that
+# doesn't have an exception, no annotations if none set, etc.
+#
+# so we check for all the core / expected attributes for the situation
+
+test_expect_success 'list request with all attr works (job success)' '
+	flux run hostname &&
+	$jq -j -c -n  "{max_entries:1, attrs:[\"all\"]}" \
+	  | $RPC job-list.list | jq ".jobs[0]" > all_success.out &&
+	cat all_success.out | jq -e ".id" &&
+	cat all_success.out | jq -e ".userid" &&
+	cat all_success.out | jq -e ".urgency" &&
+	cat all_success.out | jq -e ".priority" &&
+	cat all_success.out | jq -e ".t_submit" &&
+	cat all_success.out | jq -e ".t_depend" &&
+	cat all_success.out | jq -e ".t_run" &&
+	cat all_success.out | jq -e ".t_cleanup" &&
+	cat all_success.out | jq -e ".t_inactive" &&
+	cat all_success.out | jq -e ".state" &&
+	cat all_success.out | jq -e ".name" &&
+	cat all_success.out | jq -e ".ntasks" &&
+	cat all_success.out | jq -e ".nnodes" &&
+	cat all_success.out | jq -e ".ranks" &&
+	cat all_success.out | jq -e ".nodelist" &&
+	cat all_success.out | jq -e ".success == true" &&
+	cat all_success.out | jq -e ".exception_occurred == false" &&
+	cat all_success.out | jq -e ".result" &&
+	cat all_success.out | jq -e ".waitstatus" &&
+	cat all_success.out | jq -e ".expiration"
+'
+
+test_expect_success 'list request with all attr works (job fail)' '
+	! flux run -N1000 -n1000 hostname &&
+	$jq -j -c -n  "{max_entries:1, attrs:[\"all\"]}" \
+	  | $RPC job-list.list | jq ".jobs[0]" > all_fail.out &&
+	cat all_fail.out | jq -e ".id" &&
+	cat all_fail.out | jq -e ".userid" &&
+	cat all_fail.out | jq -e ".urgency" &&
+	cat all_fail.out | jq -e ".priority" &&
+	cat all_fail.out | jq -e ".t_submit" &&
+	cat all_fail.out | jq -e ".t_depend" &&
+	cat all_fail.out | jq -e ".t_cleanup" &&
+	cat all_fail.out | jq -e ".t_inactive" &&
+	cat all_fail.out | jq -e ".state" &&
+	cat all_fail.out | jq -e ".name" &&
+	cat all_fail.out | jq -e ".ntasks" &&
+	cat all_fail.out | jq -e ".success == false" &&
+	cat all_fail.out | jq -e ".exception_occurred == true" &&
+	cat all_fail.out | jq -e ".exception_type" &&
+	cat all_fail.out | jq -e ".exception_severity" &&
+	cat all_fail.out | jq -e ".exception_note" &&
+	cat all_fail.out | jq -e ".result"
+'
+
+#
+# max comparison
+#
+
+test_expect_success 'default comparison can get all of the jobs' '
+	flux job list -A > /dev/null
+'
+
+test_expect_success 'remove job list module' '
+	flux module remove job-list
+'
+
+test_expect_success 'job-list: invalid max_comparisons leads to error' '
+	cat >joblist.toml <<EOF &&
+[job-list]
+max_comparisons = -1
+EOF
+	flux config reload &&
+	test_must_fail flux module load job-list
+'
+
+test_expect_success 'job-list: config low max_comparisons' '
+	cat >joblist.toml <<EOF &&
+[job-list]
+max_comparisons = 10
+EOF
+	flux config reload &&
+	flux module load job-list
+'
+
+test_expect_success 'flux job list fails on low comparison count' '
+	test_must_fail flux job list -A > /dev/null 2> comparisons.err &&
+	grep "Excessive comparisons" comparisons.err
+'
+
+test_expect_success 'job-list: update config with invalid input fails' '
+	test_must_fail flux config load <<-EOF
+[job-list]
+max_comparisons = -1
+EOF
+'
+
+test_expect_success 'job-list: update config bigger max_comparisons' '
+	flux config load <<-EOF
+[job-list]
+max_comparisons = 10000
+EOF
+'
+
+test_expect_success 'flux job list works again' '
+	flux job list -A > /dev/null
+'
+
+test_expect_success 'job-list: update config no comparison limit' '
+	flux config load <<-EOF
+[job-list]
+max_comparisons = 0
+EOF
+'
+
+test_expect_success 'flux job list works' '
+	flux job list -A > /dev/null
+'
+
+#
+# job-list can handle flux-restart events
+#
+# TODO: presently job-list depends on job-manager journal, so it is
+# not possible to test the reload of the job-manager that doesn't also
+# reload job-list.
+#
+
+wait_jobid() {
+	local jobid="$1"
+	local i=0
+	while ! flux job list --states=sched | grep $jobid > /dev/null \
+		   && [ $i -lt 50 ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "50" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+# to ensure jobs are still in PENDING state, stop queue before
+# reloading job-list & job-manager.	 reload job-exec & sched-simple
+# after wait_jobid, b/c we do not want the job to be accidentally
+# executed.
+test_expect_success 'job-list parses flux-restart events' '
+	flux queue stop &&
+	jobid=`flux submit hostname | flux job id` &&
+	fj_wait_event $jobid priority &&
+	flux module unload job-list &&
+	flux module reload job-manager &&
+	flux module load job-list &&
+	wait_jobid $jobid &&
+	flux module reload job-exec &&
+	flux module reload sched-simple &&
+	flux queue start
+'
+
+#
+# jobspec-update event testing
+#
+
+# N.B. "defaultqueue" will habe jobs "submitted" to it, but updates
+# make it appear that jobs were not submitted to it.  "nosubmitqueue"
+# will never have jobs submitted to it.
+test_expect_success 'configure update queues' '
+	flux config load <<-EOT &&
+	[queues.defaultqueue]
+	[queues.updatequeue]
+	[queues.nosubmitqueue]
+	EOT
+	flux queue start --all
+'
+
+wait_id_inactive() {
+	id=$1
+	local i=0
+	while ! flux job list --states=inactive | grep ${id} > /dev/null \
+		   && [ $i -lt 50 ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "50" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+test_expect_success 'load jobspec-update test plugin' '
+        flux jobtap load --remove=all ${PLUGINPATH}/jobspec-update-job-list.so
+'
+
+test_expect_success 'run job in the default queue' '
+	flux submit -q defaultqueue --wait true | flux job id > update1.id &&
+	wait_id_inactive $(cat update1.id)
+'
+
+# jobspec-update-job-list changes command to "hostname" (thus job name
+# changes), duration to 1000.0, queue to "updatequeue".
+test_expect_success 'job-list returns expected jobspec changes' '
+	flux job list -s inactive | grep $(cat update1.id) | jq -e ".name == \"hostname\"" &&
+	flux job list -s inactive | grep $(cat update1.id) | jq -e ".queue == \"updatequeue\"" &&
+	flux job list -s inactive | grep $(cat update1.id) | jq -e ".duration == 1000.0"
+'
+
+#
+# "nosubmitqueue" never has job submitted to it, make sure stats
+# string is non-empty and filled with zeros.
+#
+test_expect_success 'job stats lists jobs in correct state in each queue' '
+	defaultq=`flux job stats | jq ".queues[] | select( .name == \"defaultqueue\" )"` &&
+	updateq=`flux job stats | jq ".queues[] | select( .name == \"updatequeue\" )"` &&
+	nosubmitq=`flux job stats | jq ".queues[] | select( .name == \"nosubmitqueue\" )"` &&
+	test -n "$nosubmitq" &&
+	echo $defaultq | jq -e ".job_states.depend == 0" &&
+	echo $defaultq | jq -e ".job_states.priority == 0" &&
+	echo $defaultq | jq -e ".job_states.sched == 0" &&
+	echo $defaultq | jq -e ".job_states.run == 0" &&
+	echo $defaultq | jq -e ".job_states.cleanup == 0" &&
+	echo $defaultq | jq -e ".job_states.inactive == 0" &&
+	echo $defaultq | jq -e ".job_states.total == 0" &&
+	echo $defaultq | jq -e ".successful == 0" &&
+	echo $defaultq | jq -e ".failed == 0" &&
+	echo $defaultq | jq -e ".canceled == 0" &&
+	echo $defaultq | jq -e ".timeout == 0" &&
+	echo $defaultq | jq -e ".inactive_purged == 0" &&
+	echo $updateq | jq -e ".job_states.depend == 0" &&
+	echo $updateq | jq -e ".job_states.priority == 0" &&
+	echo $updateq | jq -e ".job_states.sched == 0" &&
+	echo $updateq | jq -e ".job_states.run == 0" &&
+	echo $updateq | jq -e ".job_states.cleanup == 0" &&
+	echo $updateq | jq -e ".job_states.inactive == 1" &&
+	echo $updateq | jq -e ".job_states.total == 1" &&
+	echo $updateq | jq -e ".successful == 1" &&
+	echo $updateq | jq -e ".failed == 0" &&
+	echo $updateq | jq -e ".canceled == 0" &&
+	echo $updateq | jq -e ".timeout == 0" &&
+	echo $updateq | jq -e ".inactive_purged == 0" &&
+	echo $nosubmitq | jq -e ".job_states.depend == 0" &&
+	echo $nosubmitq | jq -e ".job_states.priority == 0" &&
+	echo $nosubmitq | jq -e ".job_states.sched == 0" &&
+	echo $nosubmitq | jq -e ".job_states.run == 0" &&
+	echo $nosubmitq | jq -e ".job_states.cleanup == 0" &&
+	echo $nosubmitq | jq -e ".job_states.inactive == 0" &&
+	echo $nosubmitq | jq -e ".job_states.total == 0" &&
+	echo $nosubmitq | jq -e ".successful == 0" &&
+	echo $nosubmitq | jq -e ".failed == 0" &&
+	echo $nosubmitq | jq -e ".canceled == 0" &&
+	echo $nosubmitq | jq -e ".timeout == 0" &&
+	echo $nosubmitq | jq -e ".inactive_purged == 0"
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list &&
+	wait_id_inactive $(cat update1.id)
+'
+
+test_expect_success 'job-list returns expected jobspec changes after reload' '
+	flux job list -s inactive | grep $(cat update1.id) | jq -e ".name == \"hostname\"" &&
+	flux job list -s inactive | grep $(cat update1.id) | jq -e ".queue == \"updatequeue\"" &&
+	flux job list -s inactive | grep $(cat update1.id) | jq -e ".duration == 1000.0"
+'
+
+#
+# After reload, job-list will not have ever seen any jobs submitted to
+# "defaultqueue", need to check that stats string is non-empty and all
+# stats are 0.
+#
+test_expect_success 'job stats in each queue correct after reload' '
+	defaultq=`flux job stats | jq ".queues[] | select( .name == \"defaultqueue\" )"` &&
+	updateq=`flux job stats | jq ".queues[] | select( .name == \"updatequeue\" )"` &&
+	test -n "$defaultq" &&
+	test -n "$nosubmitq" &&
+	echo $defaultq | jq -e ".job_states.depend == 0" &&
+	echo $defaultq | jq -e ".job_states.priority == 0" &&
+	echo $defaultq | jq -e ".job_states.sched == 0" &&
+	echo $defaultq | jq -e ".job_states.run == 0" &&
+	echo $defaultq | jq -e ".job_states.cleanup == 0" &&
+	echo $defaultq | jq -e ".job_states.inactive == 0" &&
+	echo $defaultq | jq -e ".job_states.total == 0" &&
+	echo $defaultq | jq -e ".successful == 0" &&
+	echo $defaultq | jq -e ".failed == 0" &&
+	echo $defaultq | jq -e ".canceled == 0" &&
+	echo $defaultq | jq -e ".timeout == 0" &&
+	echo $defaultq | jq -e ".inactive_purged == 0" &&
+	echo $updateq | jq -e ".job_states.depend == 0" &&
+	echo $updateq | jq -e ".job_states.priority == 0" &&
+	echo $updateq | jq -e ".job_states.sched == 0" &&
+	echo $updateq | jq -e ".job_states.run == 0" &&
+	echo $updateq | jq -e ".job_states.cleanup == 0" &&
+	echo $updateq | jq -e ".job_states.inactive == 1" &&
+	echo $updateq | jq -e ".job_states.total == 1" &&
+	echo $updateq | jq -e ".successful == 1" &&
+	echo $updateq | jq -e ".failed == 0" &&
+	echo $updateq | jq -e ".canceled == 0" &&
+	echo $updateq | jq -e ".timeout == 0" &&
+	echo $updateq | jq -e ".inactive_purged == 0" &&
+	echo $nosubmitq | jq -e ".job_states.depend == 0" &&
+	echo $nosubmitq | jq -e ".job_states.priority == 0" &&
+	echo $nosubmitq | jq -e ".job_states.sched == 0" &&
+	echo $nosubmitq | jq -e ".job_states.run == 0" &&
+	echo $nosubmitq | jq -e ".job_states.cleanup == 0" &&
+	echo $nosubmitq | jq -e ".job_states.inactive == 0" &&
+	echo $nosubmitq | jq -e ".job_states.total == 0" &&
+	echo $nosubmitq | jq -e ".successful == 0" &&
+	echo $nosubmitq | jq -e ".failed == 0" &&
+	echo $nosubmitq | jq -e ".canceled == 0" &&
+	echo $nosubmitq | jq -e ".timeout == 0" &&
+	echo $nosubmitq | jq -e ".inactive_purged == 0"
+'
+
+test_expect_success 'remove jobtap plugins and remove queue config' '
+	flux jobtap remove all &&
+	flux config load < /dev/null
+'
+
+#
+# resource-update event testing
+#
+
+# the resource-update-expiration plugin will add 60 minutes to the
+# expiration time of a job
+# N.B. in future, may wish to do test via a `flux update` or similar tool
+test_expect_success 'support resource-update event' '
+	flux jobtap load --remove=all ${PLUGINPATH}/resource-update-expiration.so
+'
+
+test_expect_success 'run a job' '
+	flux submit --time-limit=5m --wait-event=start sleep 300 | flux job id > rupdate1.id
+'
+
+# expiration should be 65m out, make sure it is > 10m, i.e. not 5m
+test_expect_success 'expiration is in the future' '
+	current=$(date +%s) &&
+	testexp=`expr $current + 10 \* 60` &&
+	echo $testexp > test_expiration.out &&
+	flux job list -s active | grep $(cat rupdate1.id) | jq -e ".expiration > $(cat test_expiration.out)"
+'
+
+test_expect_success 'cancel job' '
+	flux cancel $(cat rupdate1.id)
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list &&
+	wait_id_inactive $(cat rupdate1.id)
+'
+
+test_expect_success 'job-list returns expected resource changes after reload' '
+	flux job list -s inactive | grep $(cat rupdate1.id) | jq -e ".expiration > $(cat test_expiration.out)"
+'
+
+test_expect_success 'remove jobtap plugins' '
+        flux jobtap remove all
+'
+
+#
+# job list special cases
+#
+
+test_expect_success 'list count / max_entries works' '
+	count=`flux job list -s inactive -c 0 | wc -l` &&
+	test $count -gt 5 &&
+	count=`flux job list -s inactive -c 5 | wc -l` &&
+	test $count -eq 5
+'
+
+# List of all job attributes (XXX: maybe this should be pulled in from somewhere
+#  else? E.g. documentation?
+
+JOB_ATTRIBUTES="\
+userid \
+urgency \
+priority \
+t_submit \
+t_depend \
+t_run \
+t_cleanup \
+t_inactive \
+state \
+name \
+cwd \
+queue \
+project \
+bank \
+ntasks \
+ncores \
+duration \
+nnodes \
+ranks \
+nodelist \
+success \
+exception_occurred \
+exception_type \
+exception_severity \
+exception_note \
+result \
+expiration \
+annotations \
+waitstatus \
+dependencies \
+all
+"
+
+test_expect_success 'list request with empty attrs works' '
+	id=$(id -u) &&
+	$jq -j -c -n  "{max_entries:5, userid:${id}, states:0, results:0, attrs:[]}" \
+	  | $RPC job-list.list > list_empty_attrs.out &&
+	for attr in $JOB_ATTRIBUTES; do
+	test_must_fail grep $attr list_empty_attrs.out
+	done
+'
+test_expect_success 'list request with excessive max_entries works' '
+	id=$(id -u) &&
+	$jq -j -c -n  "{max_entries:100000, userid:${id}, states:0, results:0, attrs:[]}" \
+	  | $RPC job-list.list
+'
+
+# list-attrs also lists the special attribute 'all'
+LIST_ATTRIBUTES="${JOB_ATTRIBUTES} all"
+
+test_expect_success 'list-attrs works' '
+	$RPC job-list.list-attrs < /dev/null > list_attrs.out &&
+	for attr in $LIST_ATTRIBUTES; do
+	grep $attr list_attrs.out
+	done
+'
+
+
+#
+# stress test
+#
+
+wait_jobs_finish() {
+	local i=0
+	while ([ "$(flux job list | wc -l)" != "0" ]) \
+		  && [ $i -lt 1000 ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "1000" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+test_expect_success LONGTEST 'stress job-list.list-id' '
+	flux python ${FLUX_SOURCE_DIR}/t/job-list/list-id.py 500 &&
+	wait_jobs_finish
+'
+
+#
+#
+# stats tests
+#
+
+
+test_expect_success 'job-list stats works' '
+	flux module stats --parse jobs.pending job-list &&
+	flux module stats --parse jobs.running job-list &&
+	flux module stats --parse jobs.inactive job-list &&
+	flux module stats --parse idsync.lookups job-list &&
+	flux module stats --parse idsync.waits job-list
+'
+
+test_expect_success 'configure batch,debug queues' '
+	flux config load <<-EOT &&
+	[queues.batch]
+	[queues.debug]
+	EOT
+	flux queue start --all
+'
+
+test_expect_success 'run some jobs in the batch,debug queues' '
+	flux submit -q batch --wait true | flux job id > stats1.id &&
+	flux submit -q debug --wait true | flux job id > stats2.id &&
+	flux submit -q batch --wait false | flux job id > stats3.id &&
+	flux submit -q debug --wait false | flux job id > stats4.id &&
+	wait_id_inactive $(cat stats1.id) &&
+	wait_id_inactive $(cat stats2.id) &&
+	wait_id_inactive $(cat stats3.id) &&
+	wait_id_inactive $(cat stats4.id)
+'
+
+test_expect_success 'job stats lists jobs in correct state in each queue' '
+	batchq=`flux job stats | jq ".queues[] | select( .name == \"batch\" )"` &&
+	debugq=`flux job stats | jq ".queues[] | select( .name == \"debug\" )"` &&
+	echo $batchq | jq -e ".job_states.depend == 0" &&
+	echo $batchq | jq -e ".job_states.priority == 0" &&
+	echo $batchq | jq -e ".job_states.sched == 0" &&
+	echo $batchq | jq -e ".job_states.run == 0" &&
+	echo $batchq | jq -e ".job_states.cleanup == 0" &&
+	echo $batchq | jq -e ".job_states.inactive == 2" &&
+	echo $batchq | jq -e ".job_states.total == 2" &&
+	echo $batchq | jq -e ".successful == 1" &&
+	echo $batchq | jq -e ".failed == 1" &&
+	echo $batchq | jq -e ".canceled == 0" &&
+	echo $batchq | jq -e ".timeout == 0" &&
+	echo $batchq | jq -e ".inactive_purged == 0" &&
+	echo $debugq | jq -e ".job_states.depend == 0" &&
+	echo $debugq | jq -e ".job_states.priority == 0" &&
+	echo $debugq | jq -e ".job_states.sched == 0" &&
+	echo $debugq | jq -e ".job_states.run == 0" &&
+	echo $debugq | jq -e ".job_states.cleanup == 0" &&
+	echo $debugq | jq -e ".job_states.inactive == 2" &&
+	echo $debugq | jq -e ".job_states.total == 2" &&
+	echo $debugq | jq -e ".successful == 1" &&
+	echo $debugq | jq -e ".failed == 1" &&
+	echo $debugq | jq -e ".canceled == 0" &&
+	echo $debugq | jq -e ".timeout == 0" &&
+	echo $debugq | jq -e ".inactive_purged == 0"
+'
+
+test_expect_success 'reload the job-list module' '
+	flux module reload job-list &&
+	wait_id_inactive $(cat stats1.id) &&
+	wait_id_inactive $(cat stats2.id) &&
+	wait_id_inactive $(cat stats3.id) &&
+	wait_id_inactive $(cat stats4.id)
+'
+
+test_expect_success 'job stats in each queue correct after reload' '
+	batchq=`flux job stats | jq ".queues[] | select( .name == \"batch\" )"` &&
+	debugq=`flux job stats | jq ".queues[] | select( .name == \"debug\" )"` &&
+	echo $batchq | jq -e ".job_states.depend == 0" &&
+	echo $batchq | jq -e ".job_states.priority == 0" &&
+	echo $batchq | jq -e ".job_states.sched == 0" &&
+	echo $batchq | jq -e ".job_states.run == 0" &&
+	echo $batchq | jq -e ".job_states.cleanup == 0" &&
+	echo $batchq | jq -e ".job_states.inactive == 2" &&
+	echo $batchq | jq -e ".job_states.total == 2" &&
+	echo $batchq | jq -e ".successful == 1" &&
+	echo $batchq | jq -e ".failed == 1" &&
+	echo $batchq | jq -e ".canceled == 0" &&
+	echo $batchq | jq -e ".timeout == 0" &&
+	echo $batchq | jq -e ".inactive_purged == 0" &&
+	echo $debugq | jq -e ".job_states.depend == 0" &&
+	echo $debugq | jq -e ".job_states.priority == 0" &&
+	echo $debugq | jq -e ".job_states.sched == 0" &&
+	echo $debugq | jq -e ".job_states.run == 0" &&
+	echo $debugq | jq -e ".job_states.cleanup == 0" &&
+	echo $debugq | jq -e ".job_states.inactive == 2" &&
+	echo $debugq | jq -e ".job_states.total == 2" &&
+	echo $debugq | jq -e ".successful == 1" &&
+	echo $debugq | jq -e ".failed == 1" &&
+	echo $debugq | jq -e ".canceled == 0" &&
+	echo $debugq | jq -e ".timeout == 0" &&
+	echo $debugq | jq -e ".inactive_purged == 0"
+'
+
+wait_total() {
+	total=$1
+	local i=0
+	while ! flux job stats | jq -e ".job_states.total == ${total}" \
+		   && [ $i -lt 50 ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "50" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+# purge all jobs except two, the remaining two should be the failed
+# jobs submitted to the batch and debug queues.
+test_expect_success 'job-stats correct after purge' '
+	total=$(flux job stats | jq .job_states.total) &&
+	purged=$((total - 2)) &&
+	flux job purge --force --num-limit=2 &&
+	wait_total 2 &&
+	flux job stats | jq -e ".job_states.inactive == 2" &&
+	flux job stats | jq -e ".job_states.total == 2" &&
+	flux job stats | jq -e ".inactive_purged == ${purged}" &&
+	batchq=`flux job stats | jq ".queues[] | select( .name == \"batch\" )"` &&
+	debugq=`flux job stats | jq ".queues[] | select( .name == \"debug\" )"` &&
+	echo $batchq | jq -e ".job_states.inactive == 1" &&
+	echo $batchq | jq -e ".job_states.total == 1" &&
+	echo $batchq | jq -e ".successful == 0" &&
+	echo $batchq | jq -e ".failed == 1" &&
+	echo $batchq | jq -e ".inactive_purged == 1" &&
+	echo $debugq | jq -e ".job_states.inactive == 1" &&
+	echo $debugq | jq -e ".job_states.total == 1" &&
+	echo $debugq | jq -e ".successful == 0" &&
+	echo $debugq | jq -e ".failed == 1" &&
+	echo $debugq | jq -e ".inactive_purged == 1"
+'
+
+test_expect_success 'remove queues' '
+	flux config load < /dev/null
+'
+
+#
+# corner case tests
+#
+
+test_expect_success 'list request with empty payload fails with EPROTO(71)' '
+	${RPC} job-list.list 71 </dev/null
+'
+test_expect_success 'list request with invalid input fails with EPROTO(71) (attrs not an array)' '
+	name="attrs-not-array" &&
+	$jq -j -c -n  "{max_entries:5, attrs:5}" \
+	  | $listRPC >${name}.out &&
+	cat <<-EOF >${name}.expected &&
+	errno 71: invalid payload: attrs must be an array
+	EOF
+	test_cmp ${name}.expected ${name}.out
+'
+test_expect_success 'list request with invalid input fails with EINVAL(22) (attrs non-string)' '
+	name="attr-not-string" &&
+	$jq -j -c -n  "{max_entries:5, attrs:[5]}" \
+	  | $listRPC > ${name}.out &&
+	cat <<-EOF >${name}.expected &&
+	errno 22: attr has no string value
+	EOF
+	test_cmp ${name}.expected ${name}.out
+'
+test_expect_success 'list request with invalid input fails with EINVAL(22) (attrs illegal field)' '
+	name="field-not-valid" &&
+	$jq -j -c -n  "{max_entries:5, attrs:[\"foo\"]}" \
+	  | $listRPC > ${name}.out &&
+	cat <<-EOF >${name}.expected &&
+	errno 22: foo is not a valid attribute
+	EOF
+	test_cmp ${name}.expected ${name}.out
+'
+test_expect_success 'list-id request with empty payload fails with EPROTO(71)' '
+	${RPC} job-list.list-id 71 </dev/null
+'
+test_expect_success 'list-id request with invalid input fails with EPROTO(71) (attrs not an array)' '
+	name="list-id-attrs-not-array" &&
+	id=`flux submit hostname | flux job id` &&
+	$jq -j -c -n  "{id:${id}, attrs:5}" \
+	  | $listRPC list-id > ${name}.out &&
+	cat <<-EOF >${name}.expected &&
+	errno 71: invalid payload: attrs must be an array
+	EOF
+	test_cmp ${name}.expected ${name}.out
+'
+test_expect_success 'list-id request with invalid input fails with EINVAL(22) (attrs non-string)' '
+	name="list-id-invalid-attrs" &&
+	id=$(flux jobs -c1 -ano {id.dec}) &&
+	$jq -j -c -n  "{id:${id}, attrs:[5]}" \
+	  | $listRPC list-id > ${name}.out &&
+	cat <<-EOF >${name}.expected &&
+	errno 22: attr has no string value
+	EOF
+	test_cmp ${name}.expected ${name}.out
+'
+test_expect_success 'list-id request with invalid input fails with EINVAL(22) (attrs illegal field)' '
+	name="list-id-invalid-attr" &&
+	id=$(flux jobs -c1 -ano {id.dec}) &&
+	$jq -j -c -n  "{id:${id}, attrs:[\"foo\"]}" \
+	  | $listRPC list-id >${name}.out &&
+	cat <<-EOF >${name}.expected &&
+	errno 22: foo is not a valid attribute
+	EOF
+	test_cmp ${name}.expected ${name}.out
+'
+# Restart job-list from the KVS.  job-list used to pull in job eventlogs
+# directly from the KVS at startup, but now it gets them via the job manager
+# journal.  Therefore we need to restart the job-manager too, which requires
+# some other things to be restarted to get a functional system.
+restart_from_kvs() {
+	flux module remove job-list
+	flux module reload job-manager
+	flux module reload -f sched-simple
+	flux module reload -f job-exec
+	flux module load job-list
+}
+
+# N.B. we remove annotations from the alloc event in this test, but it could
+# be cached and replayed via the job-manager, so we need to reload it
+# and associated modules too
+test_expect_success 'job-list can handle events missing optional data (alloc)' '
+	userid=`id -u` &&
+	cat <<EOF >eventlog_empty_alloc.out &&
+{"timestamp":1000.0,"name":"submit","context":{"userid":${userid},"urgency":16,"flags":0,"version":1}}
+{"timestamp":1001.0,"name":"validate"}
+{"timestamp":1002.0,"name":"depend"}
+{"timestamp":1003.0,"name":"priority","context":{"priority":8}}
+{"timestamp":1004.0,"name":"alloc","context":{}}
+{"timestamp":1005.0,"name":"start"}
+{"timestamp":1006.0,"name":"finish","context":{"status":0}}
+{"timestamp":1007.0,"name":"release","context":{"ranks":"all","final":true}}
+{"timestamp":1008.0,"name":"free"}
+{"timestamp":1009.0,"name":"clean"}
+EOF
+	jobid=`flux submit --wait hostname` &&
+	kvspath=`flux job id --to=kvs ${jobid}` &&
+	flux kvs put -r ${kvspath}.eventlog=- < eventlog_empty_alloc.out &&
+	restart_from_kvs &&
+	flux job list-ids ${jobid} > empty_alloc.out &&
+	cat empty_alloc.out | jq -e ".annotations == null"
+'
+# N.B. Note the original job was submitted with urgency 16, but we
+# hard code 8 in the fake eventlog.	 This is just to make sure the fake
+# eventlog was loaded correctly at the end of the test.
+#
+# N.B. We add extra events into this fake eventlog for testing
+test_expect_success 'job-list can handle events with superfluous context data' '
+	userid=`id -u` &&
+	cat <<EOF >eventlog_superfluous_context.out &&
+{"timestamp":1000.0,"name":"submit","context":{"userid":${userid},"urgency":8,"flags":0,"version":1,"etc":1}}
+{"timestamp":1001.0,"name":"dependency-add","context":{"description":"begin-time=1234.000","etc":1}}
+{"timestamp":1002.0,"name":"validate","context":{"etc":1}}
+{"timestamp":1003.0,"name":"dependency-remove","context":{"description":"begin-time=1234.000","etc":1}}
+{"timestamp":1004.0,"name":"depend","context":{"etc":1}}
+{"timestamp":1005.0,"name":"priority","context":{"priority":8,"etc":1}}
+{"timestamp":1006.0,"name":"alloc","context":{"annotations":{"sched":{"resource_summary":"rank0/core0"}},"etc":1}}
+{"timestamp":1007.0,"name":"prolog-start","context":{"description":"job-manager.prolog","etc":1}}
+{"timestamp":1008.0,"name":"prolog-finish","context":{"description":"job-manager.prolog","status":0,"etc":1}}
+{"timestamp":1009.0,"name":"start","context":{"etc":1}}
+{"timestamp":1010.0,"name":"finish","context":{"status":0,"etc":1}}
+{"timestamp":1011.0,"name":"epilog-start","context":{"description":"job-manager.epilog","etc":1}}
+{"timestamp":1012.0,"name":"release","context":{"ranks":"all","final":true,"etc":1}}
+{"timestamp":1013.0,"name":"epilog-finish","context":{"description":"job-manager.epilog","status":0,"etc":1}}
+{"timestamp":1014.0,"name":"free","context":{"etc":1}}
+{"timestamp":1015.0,"name":"clean","context":{"etc":1}}
+EOF
+	jobid=`flux submit --wait --urgency=default hostname` &&
+	kvspath=`flux job id --to=kvs ${jobid}` &&
+	flux kvs put -r ${kvspath}.eventlog=- < eventlog_superfluous_context.out &&
+	restart_from_kvs &&
+	flux job list-ids ${jobid} > superfluous_context.out &&
+	cat superfluous_context.out | jq -e ".urgency == 8"
+'
+
+# invalid job data tests
+#
+# to avoid potential racyness, wait up to 5 seconds for job to appear
+# in job list.	Note that we can't use `flux job list-ids`, b/c we
+# need specific job states to be crossed to ensure the job-list module
+# has processed the invalid data.
+#
+# In addition, note that the tests below are specific to how the data
+# is invalid in these tests and how job-list parses the invalid data.
+# Different parsing errors could have some fields initialized but
+# others not.
+#
+# note that these tests should be done last, as the introduction of
+# invalid job data into the KVS could affect tests above.
+#
+
+# Following tests use invalid / unusual jobspecs, must load a more permissive validator
+
+ingest_module ()
+{
+	cmd=$1; shift
+	flux module ${cmd} job-ingest $*
+}
+
+test_expect_success 'reload job-ingest without validator' '
+	ingest_module reload disable-validator
+'
+
+test_expect_success 'create illegal jobspec with empty command array' '
+	flux submit --dry-run hostname > hostname.json &&
+	cat hostname.json | $jq ".tasks[0].command = []" > bad_jobspec.json
+'
+
+# note that ntasks will not be set if jobspec invalid
+test_expect_success 'flux job list works on job with illegal jobspec' '
+	jobid=`flux job submit bad_jobspec.json | flux job id` &&
+	fj_wait_event $jobid clean >/dev/null &&
+	i=0 &&
+	while ! flux job list --states=inactive | grep $jobid > /dev/null \
+		   && [ $i -lt 5 ]
+	do
+		sleep 1
+		i=$((i + 1))
+	done &&
+	test "$i" -lt "5" &&
+	flux job list --states=inactive | grep $jobid > list_illegal_jobspec.out &&
+	cat list_illegal_jobspec.out | $jq -e ".name == null" &&
+	cat list_illegal_jobspec.out | $jq -e ".ntasks == null"
+'
+
+test_expect_success NO_CHAIN_LINT 'flux job list-ids works on job with illegal jobspec' '
+	${RPC} job-list.job-state-pause 0 </dev/null
+	jobid=`flux job submit bad_jobspec.json | flux job id`
+	fj_wait_event $jobid clean >/dev/null
+	flux job list-ids ${jobid} > list_id_illegal_jobspec.out &
+	pid=$!
+	wait_idsync 1 &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	wait $pid &&
+	cat list_id_illegal_jobspec.out | $jq -e ".id == ${jobid}"
+'
+
+test_expect_success 'create jobspec with missing cwd' '
+	cat hostname.json | $jq "del(.attributes.system.cwd)" > no_cwd_jobspec.json
+'
+
+test_expect_success 'flux job list works on job with no cwd' '
+	jobid=`flux job submit no_cwd_jobspec.json | flux job id` &&
+	fj_wait_event $jobid clean >/dev/null &&
+	i=0 &&
+	while ! flux job list --states=inactive | grep $jobid > /dev/null \
+		   && [ $i -lt 5 ]
+	do
+		sleep 1
+		i=$((i + 1))
+	done &&
+	test "$i" -lt "5" &&
+	flux job list --states=inactive | grep $jobid > list_no_cwd_jobspec.out &&
+	cat list_no_cwd_jobspec.out | $jq -e ".cwd == null"
+'
+
+test_expect_success 'reload job-ingest with defaults' '
+	ingest_module reload
+'
+
+test_expect_success 'flux job list works on racy annotations' '
+	${RPC} job-list.job-state-pause 0 </dev/null &&
+	jobid=`flux submit --wait hostname | flux job id` &&
+	${RPC} job-list.job-state-unpause 0 </dev/null &&
+	i=0 &&
+	while ! flux job list --states=inactive | grep $jobid > /dev/null \
+		   && [ $i -lt 5 ]
+	do
+		sleep 1
+		i=$((i + 1))
+	done &&
+	test "$i" -lt "5"  &&
+	flux job list --states=inactive | grep $jobid > list_racy_annotation.out &&
+	cat list_racy_annotation.out | $jq -e ".annotations"
+'
+
+test_expect_success 'flux job list subcommands are not displayed in help' '
+	test_must_fail flux job -h 2>job-help.err &&
+	test_must_fail grep "^[ ]*list" job-help.err
+'
+test_expect_success 'flux job list subcommands fail when stdout is a tty' '
+	test_must_fail $runpty --stderr=tty1.err flux job list &&
+	grep "This is not the command" tty1.err &&
+	test_must_fail $runpty --stderr=tty2.err flux job list-inactive &&
+	grep "This is not the command" tty2.err &&
+	test_must_fail $runpty --stderr=tty3.err flux job list-ids &&
+	grep "This is not the command" tty3.err
+'
+
+test_done
diff --git a/t/t2261-job-list-update.t b/t/t2261-job-list-update.t
new file mode 100755
index 000000000000..e3152d6c366f
--- /dev/null
+++ b/t/t2261-job-list-update.t
@@ -0,0 +1,204 @@
+#!/bin/sh
+
+test_description='Test flux job list services w/ changing job data'
+
+. $(dirname $0)/job-list/job-list-helper.sh
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 job
+
+fj_wait_event() {
+  flux job wait-event --timeout=20 "$@"
+}
+
+# submit a whole bunch of jobs for job list testing
+#
+# - the first loop of job submissions are intended to have some jobs run
+#   quickly and complete
+# - the second loop of job submissions are intended to eat up all resources
+# - the last job submissions are intended to get a create a set of
+#   pending jobs, because jobs from the second loop have taken all resources
+#   - we desire pending jobs sorted in priority order, so we need to
+#   create the sorted list for comparison later.
+# - job ids are stored in files in the order we expect them to be listed
+#   - pending jobs - by priority (highest first), job id (smaller first)
+#   - running jobs - by start time (most recent first)
+#   - inactive jobs - by completion time (most recent first)
+#
+# TODO
+# - alternate userid job listing
+
+test_expect_success 'submit jobs for job list testing' '
+        #  Create `hostname` and `sleep` jobspec
+        #  N.B. Used w/ `flux job submit` for serial job submission
+        #  for efficiency (vs serial `flux submit`.
+        #
+        flux submit --dry-run hostname >hostname.json &&
+        flux submit --dry-run --time-limit=5m sleep 600 > sleeplong.json &&
+        #
+        # submit jobs that will complete
+        #
+        for i in $(seq 0 3); do
+                flux job submit hostname.json >> inactiveids
+                fj_wait_event `tail -n 1 inactiveids` clean
+        done &&
+        #
+        #  Currently all inactive ids are "completed"
+        #
+        tac inactiveids | flux job id > completed.ids &&
+        #
+        #  Submit 8 sleep jobs to fill up resources
+        #
+        for i in $(seq 0 7); do
+                flux job submit sleeplong.json >> runningids
+        done &&
+        tac runningids | flux job id > running.ids &&
+        #
+        #  Submit a set of jobs with misc urgencies
+        #
+        id1=$(flux job submit -u20 hostname.json) &&
+        id2=$(flux job submit      hostname.json) &&
+        id3=$(flux job submit -u31 hostname.json) &&
+        id4=$(flux job submit -u0  hostname.json) &&
+        id5=$(flux job submit -u20 hostname.json) &&
+        id6=$(flux job submit      hostname.json) &&
+        id7=$(flux job submit -u31 hostname.json) &&
+        id8=$(flux job submit -u0  hostname.json) &&
+        flux job id $id3 > pending.ids &&
+        flux job id $id7 >> pending.ids &&
+        flux job id $id1 >> pending.ids &&
+        flux job id $id5 >> pending.ids &&
+        flux job id $id2 >> pending.ids &&
+        flux job id $id6 >> pending.ids &&
+        flux job id $id4 >> pending.ids &&
+        flux job id $id8 >> pending.ids &&
+        cat pending.ids > active.ids &&
+        cat running.ids >> active.ids &&
+        tac inactiveids | flux job id > inactive.ids &&
+        cat active.ids > all.ids &&
+        cat inactive.ids >> all.ids &&
+        #
+        #  The job-list module has eventual consistency with the jobs stored in
+        #  the job-manager queue.  To ensure no raciness in tests, ensure
+        #  jobs above have reached expected states in job-list before continuing.
+        #
+        flux job list-ids --wait-state=sched $(job_list_state_ids pending) > /dev/null &&
+        flux job list-ids --wait-state=run $(job_list_state_ids running) > /dev/null &&
+        flux job list-ids --wait-state=inactive $(job_list_state_ids inactive) > /dev/null
+'
+
+# Note: "running" = "run" | "cleanup", we also test just "run" state
+# since we happen to know all these jobs are in the "run" state given
+# checks above
+
+test_expect_success 'flux job list running jobs in started order' '
+        flux job list -s running | jq .id > list_started.out &&
+        test_cmp list_started.out running.ids
+'
+
+test_expect_success 'flux job list inactive jobs in completed order' '
+        flux job list -s inactive | jq .id > list_inactive.out &&
+        test_cmp list_inactive.out inactive.ids
+'
+
+# Note: "pending" = "depend" | "sched", we also test just "sched"
+# state since we happen to know all these jobs are in the "sched"
+# state given checks above
+
+test_expect_success 'flux job list pending jobs in priority order' '
+        flux job list -s pending | jq .id > list_pending.out &&
+        test_cmp list_pending.out pending.ids
+'
+
+test_expect_success 'flux job list pending jobs with correct urgency' '
+        cat >list_urgency.exp <<-EOT &&
+31
+31
+20
+20
+16
+16
+0
+0
+EOT
+        flux job list -s pending | jq .urgency > list_urgency.out &&
+        test_cmp list_urgency.out list_urgency.exp
+'
+
+test_expect_success 'flux job list pending jobs with correct priority' '
+        cat >list_priority.exp <<-EOT &&
+4294967295
+4294967295
+20
+20
+16
+16
+0
+0
+EOT
+        flux job list -s pending | jq .priority > list_priority.out &&
+        test_cmp list_priority.out list_priority.exp
+'
+
+test_expect_success 'change a job urgency' '
+        jobid=`head -n 3 pending.ids | tail -n 1` &&
+        flux job urgency ${jobid} 1
+'
+
+test_expect_success 'wait for job-list to see job urgency change' '
+        jobidhead=`head -n 2 pending.ids > pending_after.ids` &&
+        jobidhead=`head -n 6 pending.ids | tail -n 3 >> pending_after.ids` &&
+        jobidchange=`head -n 3 pending.ids | tail -n 1 >> pending_after.ids` &&
+        jobidend=`tail -n2 pending.ids >> pending_after.ids` &&
+        i=0 &&
+        while flux job list -s pending | jq .id > list_pending_after.out && \
+              ! test_cmp list_pending_after.out pending_after.ids && \
+              [ $i -lt 5 ]
+        do
+                sleep 1
+                i=$((i + 1))
+        done &&
+        test "$i" -lt "5" &&
+        flux job list -s pending | jq .id > list_pending_after.out &&
+        test_cmp list_pending_after.out pending_after.ids
+'
+
+test_expect_success 'flux job list pending jobs with correct urgency' '
+        cat >list_urgency_after.exp <<-EOT &&
+31
+31
+20
+16
+16
+1
+0
+0
+EOT
+        flux job list -s pending | jq .urgency > list_urgency_after.out &&
+        test_cmp list_urgency_after.out list_urgency_after.exp
+'
+
+test_expect_success 'flux job list pending jobs with correct priority' '
+        cat >list_priority_after.exp <<-EOT &&
+4294967295
+4294967295
+20
+16
+16
+1
+0
+0
+EOT
+        flux job list -s pending | jq .priority > list_priority_after.out &&
+        test_cmp list_priority_after.out list_priority_after.exp
+'
+
+test_expect_success 'cleanup job listing jobs ' '
+	flux cancel $(cat active.ids) &&
+        for jobid in `cat active.ids`; do
+            fj_wait_event $jobid clean;
+        done
+'
+
+test_done
diff --git a/t/t2262-job-list-stats.t b/t/t2262-job-list-stats.t
new file mode 100755
index 000000000000..ee00f40542fe
--- /dev/null
+++ b/t/t2262-job-list-stats.t
@@ -0,0 +1,50 @@
+#!/bin/sh
+
+test_description='Test streaming flux job statistics RPC'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+
+get_watchers() {
+	flux module stats job-list | jq -r .stats_watchers
+}
+
+test_expect_success 'create streaming job-stats script' '
+	cat >job-stats.py <<-EOT &&
+	import sys
+	import flux
+	handle = flux.Flux()
+	fut = handle.rpc("job-list.job-stats",{},flags=flux.constants.FLUX_RPC_STREAMING)
+	for i in range(int(sys.argv[1])):
+	    print(fut.get())
+	    sys.stdout.flush()
+	    fut.reset()
+	EOT
+	chmod +x job-stats.py
+'
+test_expect_success 'streaming job-stats returns one response immediately' '
+	run_timeout 30 flux python job-stats.py 1
+'
+test_expect_success 'disconnect handling works' '
+	test $(get_watchers) -eq 0
+'
+test_expect_success NO_CHAIN_LINT 'start monitoring job-stats and wait for first response' '
+	flux python job-stats.py 100 >stats.log &
+	echo $! >stats.pid &&
+	$waitfile stats.log
+'
+test_expect_success NO_CHAIN_LINT 'run a job to completion' '
+	flux run true
+'
+test_expect_success NO_CHAIN_LINT 'wait until job-stats produces at least one more response' '
+	$waitfile --count=2 stats.log
+'
+test_expect_success NO_CHAIN_LINT 'kill job-stats script' '
+	pid=$(cat stats.pid) &&
+	kill -15 $pid
+'
+
+test_done
diff --git a/t/t2270-job-dependencies.t b/t/t2270-job-dependencies.t
new file mode 100755
index 000000000000..47fdbfbb3789
--- /dev/null
+++ b/t/t2270-job-dependencies.t
@@ -0,0 +1,230 @@
+#!/bin/sh
+
+test_description='Test job dependencies'
+
+. $(dirname $0)/sharness.sh
+
+# Setup a config dir for this test_under_flux instance so we can
+#  configure jobtap plugins on reload
+export FLUX_CONF_DIR=$(pwd)/conf.d
+mkdir -p conf.d
+
+test_under_flux 4 job
+
+flux setattr log-stderr-level 1
+
+PLUGINPATH=${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs
+
+
+test_expect_success 'flux run: --dependency option works' '
+	flux run --dry-run \
+		--env=-* \
+		--dependency=foo:1234 \
+		--dependency=foo:3.1415 \
+		--dependency=foo:f1?val=bar \
+		--dependency="foo:f1?val=a&val=b" true | \
+		jq '.attributes.system.dependencies' > deps.json &&
+	test_debug "cat deps.json" &&
+	jq -e ".[0].scheme == \"foo\"" < deps.json &&
+	jq -e ".[0].value == \"1234\"" < deps.json &&
+	jq -e ".[1].value == \"3.1415\"" < deps.json &&
+	jq -e ".[2].scheme == \"foo\"" < deps.json &&
+	jq -e ".[2].value == \"f1\""   < deps.json &&
+	jq -e ".[2].val == \"bar\""    < deps.json &&
+	jq -e ".[3].val[0] == \"a\""   < deps.json &&
+	jq -e ".[3].val[1] == \"b\""   < deps.json
+'
+test_expect_success 'submitted job with unknown dependency scheme is rejected' '
+	test_must_fail flux submit --dependency=invalid:value hostname
+'
+test_expect_success 'job with too long dependency scheme is rejected' '
+	test_must_fail flux submit \
+		--dependency=$(python -c "print \"x\"*156"):value hostname
+'
+test_expect_success 'reload ingest with disabled validator' '
+	flux module reload -f job-ingest disable-validator
+'
+test_expect_success 'submitted job with invalid dependencies is rejected' '
+	test_must_fail flux submit \
+		--setattr=system.dependencies={} \
+		hostname > not-an-array.out 2>&1 &&
+	test_debug "cat not-an-array.out" &&
+	grep -q "must be an array" not-an-array.out &&
+	test_must_fail flux submit \
+		--setattr=system.dependencies="[{\"foo\":1}]" \
+		hostname > empty-object.out 2>&1 &&
+	test_debug "cat empty-object.out" &&
+	grep -q "missing" empty-object.out
+'
+test_expect_success 'reload ingest with default validator' '
+	flux module reload -f job-ingest
+'
+test_expect_success 'create dep-remove.py' '
+	cat <<-EOF >dep-remove.py
+	import flux
+	from flux.job import JobID
+	import sys
+
+	jobid = flux.job.JobID(sys.argv[1])
+	name = sys.argv[2]
+	topic = "job-manager.dependency-test.remove"
+	print(flux.Flux().rpc(topic, {"id": jobid, "description": name}).get())
+	EOF
+'
+test_expect_success 'create dep-check.py' '
+	cat <<-EOF >dep-check.py
+	import flux
+	from flux.job import JobID
+	import sys
+
+	jobid = flux.job.JobID(sys.argv[1])
+	name = sys.argv[2]
+	topic = "job-manager.dependency-test.check"
+	flux.Flux().rpc(topic, {"id": jobid, "name": name}).get()
+	print(f"dependency-test plugin has {name} cached for {jobid}")
+	EOF
+'
+test_expect_success 'job-manager: load dependency-test plugin' '
+	flux jobtap load --remove=all ${PLUGINPATH}/dependency-test.so
+'
+test_expect_success 'job-manager: dependency-test plugin is working' '
+	jobid=$(flux submit --dependency=test:deptest true) &&
+	flux job wait-event -t 15 -m description=deptest \
+		${jobid} dependency-add &&
+	test $(flux jobs -no {state} ${jobid}) = DEPEND &&
+	flux python dep-check.py ${jobid} deptest &&
+	flux python dep-remove.py ${jobid} deptest &&
+	flux job wait-event -t 15 -m description=deptest \
+		${jobid} dependency-remove &&
+	flux job wait-event -vt 15 ${jobid} clean
+'
+test_expect_success 'plugin rejects job with malformed dependency spec' '
+	test_must_fail flux submit \
+		--dependency=test:failure?remove=bad \
+		hostname >baddeps.out 2>&1 &&
+	test_debug "cat baddeps.out" &&
+	grep "failed to unpack dependency" baddeps.out
+'
+test_expect_success 'job dependencies are available in listing tools' '
+	jobid=$(flux submit \
+		--dependency=test:foo \
+		hostname) &&
+	flux job wait-event -t 15 -m description=foo ${jobid} dependency-add &&
+	flux jobs -o {id}:{dependencies} &&
+	test "$(flux jobs -no {dependencies} ${jobid})" = "foo" &&
+	flux job list | grep "dependencies.*foo" &&
+	${FLUX_BUILD_DIR}/t/job-manager/list-jobs | grep "dependencies.*foo" &&
+	flux python dep-remove.py ${jobid} foo &&
+	flux job wait-event -vt 15 ${jobid} clean
+'
+test_expect_success 'multiple job dependencies works' '
+	jobid=$(flux submit \
+		--dependency=test:foo \
+		--dependency=test:bar \
+		hostname) &&
+	flux job wait-event -t 15 -m description=bar ${jobid} dependency-add &&
+	flux jobs -o {id}:{dependencies} &&
+	test "$(flux jobs -no {dependencies} ${jobid})" = "foo,bar" &&
+	flux python dep-check.py ${jobid} foo &&
+	flux python dep-check.py ${jobid} bar &&
+	flux python dep-remove.py ${jobid} bar &&
+	test_must_fail flux python dep-check.py ${jobid} bar &&
+	flux jobs -o {id}:{dependencies} &&
+	test "$(flux jobs -no {dependencies} ${jobid})" = "foo" &&
+	flux python dep-remove.py ${jobid} foo &&
+	flux job wait-event -vt 15 ${jobid} clean
+'
+test_expect_success 'multiple dependency-add with same description is ignored' '
+	jobid=$(flux submit \
+		--dependency=test:bar \
+		--dependency=test:bar \
+		hostname) &&
+	flux job wait-event -t 15 -m description=bar ${jobid} dependency-add &&
+	flux job eventlog ${jobid} &&
+	flux jobs -o {id}:{dependencies} ${jobid} &&
+	test "$(flux jobs -no {dependencies} ${jobid})" = "bar" &&
+	flux python dep-remove.py ${jobid} bar &&
+	flux job wait-event -vt 15 ${jobid} clean
+'
+test_expect_success 'dependency can be removed in job.dependency callback' '
+	id=$(flux submit \
+		--dependency=test:bar?remove=1 \
+		hostname) &&
+	flux job wait-event -t 15 -m description=bar ${id} dependency-add &&
+	flux job wait-event -t 15 -m description=bar ${id} dependency-remove &&
+	flux job wait-event -vt 15 ${id} clean
+'
+test_expect_success 'dependency add/remove in callback does not incorrectly release job' '
+	id=$(flux submit \
+		--dependency=test:bar?remove=1 \
+		--dependency=test:foo \
+		hostname) &&
+	flux job wait-event -t 15 -m description=bar ${id} dependency-add &&
+	flux job wait-event -t 15 -m description=bar ${id} dependency-remove &&
+	test "$(flux jobs -no {dependencies} ${id})" = "foo" &&
+	test "$(flux jobs -no {state} ${id})" = "DEPEND" &&
+	flux python dep-check.py ${id} foo &&
+	flux python dep-remove.py ${id} foo &&
+	flux job wait-event -vt 15 ${id} clean
+'
+test_expect_success 'restart: start job with 2 dependencies to test restart' '
+	jobid=$(flux submit \
+		--dependency=test:foo \
+		--dependency=test:bar \
+		hostname) &&
+	flux job wait-event -t 15 -m description=bar ${jobid} dependency-add
+'
+test_expect_success 'restart: remove 1 of 2 dependencies' '
+	flux jobs -o {id}:{dependencies} &&
+	flux python dep-remove.py ${jobid} bar &&
+	flux jobs -o {id}:{dependencies} &&
+	flux python dep-check.py ${jobid} foo
+'
+job_manager_restart() {
+	flux module remove job-list &&
+	flux module reload job-manager &&
+	flux module load job-list &&
+	flux module reload -f sched-simple &&
+	flux module reload -f job-exec
+}
+test_expect_success 'restart: reload job-manager' '
+	job_manager_restart
+'
+test_expect_success 'restart: job dependency preserved' '
+	test $(flux jobs -no {state} ${jobid}) = DEPEND &&
+	flux jobs -o {id}:{dependencies} &&
+	test $(flux jobs -no {dependencies} ${jobid}) = "foo"
+'
+test_expect_success 'restart: job non-fatal exception due to missing plugin' '
+	flux job wait-event -t 1 -m type=dependency ${jobid} exception
+'
+test_expect_success 'restart: reload dependency test plugin' '
+	flux jobtap load --remove=all ${PLUGINPATH}/dependency-test.so
+'
+test_expect_success 'restart: job.dependency.test called for jobs on load' '
+	flux python dep-check.py ${jobid} foo
+'
+test_expect_success 'restart: subsequent removal of dependency releases job' '
+	flux python dep-remove.py ${jobid} foo &&
+	flux jobs -o {id}:{dependencies} &&
+	flux job wait-event -vt 15 ${jobid} clean
+'
+test_expect_success 'restart: add plugin to instance config' '
+	mkdir -p conf &&
+	cat <<-EOF >conf.d/test.toml &&
+	[[job-manager.plugins]]
+	load = "${PLUGINPATH}/dependency-test.so"
+	EOF
+	flux config reload
+'
+test_expect_success 'restart: restart calls job.dependency.* callbacks' '
+	jobid=$(flux submit --dependency=test:foo hostname) &&
+	flux job wait-event ${jobid} dependency-add &&
+	job_manager_restart &&
+	flux jobtap list | grep dependency &&
+	test $(flux jobs -no {dependencies} ${jobid}) = "foo" &&
+	flux python dep-check.py ${jobid} foo &&
+	flux python dep-remove.py ${jobid} foo &&
+	flux job wait-event -vt 15 ${jobid} clean
+'
+test_done
diff --git a/t/t2271-job-dependency-after.t b/t/t2271-job-dependency-after.t
new file mode 100755
index 000000000000..3cbd383e5d7e
--- /dev/null
+++ b/t/t2271-job-dependency-after.t
@@ -0,0 +1,242 @@
+#!/bin/sh
+
+test_description='Test job dependency after* support'
+
+. $(dirname $0)/sharness.sh
+
+flux version | grep -q libflux-security && test_set_prereq FLUX_SECURITY
+
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+submit_as_alternate_user()
+{
+	FAKE_USERID=42
+	test_debug "echo running flux run $@ as userid $FAKE_USERID"
+	flux run --dry-run "$@" | \
+		flux python ${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py $FAKE_USERID \
+		>job.signed
+	FLUX_HANDLE_USERID=$FAKE_USERID \
+		flux job submit --flags=signed job.signed
+}
+
+test_expect_success 'dependency=after:invalid rejects invalid target jobids' '
+	test_expect_code 1 flux bulksubmit \
+		--dependency={} \
+		--log-stderr={}.err \
+		hostname ::: \
+		after:-1 \
+		after:bar \
+		after:1234 \
+		after:${jobid} &&
+	grep "\"-1\" is not a valid jobid" after:-1.err &&
+	grep "\"bar\" is not a valid jobid" after:bar.err &&
+	grep "job not found" after:1234.err
+'
+test_expect_success FLUX_SECURITY 'dependency=after will not work on another user job' '
+	jobid=$(flux submit sleep 300) &&
+	test_debug "echo submitted job $jobid" &&
+	test_expect_code 1 submit_as_alternate_user \
+		--dependency=afterany:$jobid hostname \
+		>baduser.log 2>&1 &&
+	test_debug "cat baduser.log" &&
+	grep "Permission denied" baduser.log &&
+	flux cancel $jobid
+'
+test_expect_success 'disable ingest validator' '
+	flux module reload -f job-ingest disable-validator
+'
+test_expect_success 'dependency=after rejects invalid dependency' '
+	flux run --dry-run hostname | \
+	  jq ".attributes.system.dependencies[0] = \
+		{\"scheme\":\"after\", \"value\": 1}" >job.json &&
+	test_expect_code 1 flux job submit job.json
+'
+test_expect_success 'reenable ingest validator' '
+	flux module reload -f job-ingest
+'
+test_expect_success 'dependency=after works' '
+	jobid=$(flux submit --urgency=hold hostname) &&
+	depid=$(flux submit --dependency=after:$jobid hostname) &&
+	flux job wait-event -vt 15 $depid dependency-add &&
+	test_debug "echo checking that job ${depid} is in DEPEND state" &&
+	test "$(flux jobs -no {state} $depid)" = "DEPEND" &&
+	flux job urgency $jobid default &&
+	flux job wait-event -vt 15 $depid clean 
+'
+test_expect_success 'dependency=after does not release job until start event' '
+	jobid=$(flux submit \
+		--setattr=system.exec.test.override=1 \
+		--setattr=system.exec.test.run_duration=0.001s true) &&
+	depid=$(flux submit --dependency=after:$jobid hostname) &&
+	flux job wait-event -t 15 $depid dependency-add &&
+	flux job wait-event -t 15 $jobid alloc &&
+	test_debug "echo antecedent in RUN state, but no start event" &&
+	test_must_fail flux job wait-event -t 0.25 $depid dependency-remove &&
+	test_debug "echo dependency not yet removed" &&
+	flux job-exec-override start $jobid &&
+	flux job wait-event -t 15 $jobid start &&
+	test_debug "echo antecedent now started" &&
+	flux job wait-event -t 15 $depid dependency-remove &&
+	flux job wait-event -t 15 $depid clean &&
+	flux job wait-event -t 15 $jobid clean
+'
+test_expect_success 'dependency=after works when antecedent is running' '
+	jobid=$(flux submit sleep 300) &&
+	flux job wait-event -vt 15 $jobid start &&
+	depip=$(flux submit --dependency=after:$jobid hostname) &&
+	flux job wait-event -vt 15 $depid clean &&
+	flux cancel $jobid
+'
+test_expect_success 'dependency=after generates exception for failed job' '
+	jobid=$(flux submit --urgency=hold hostname) &&
+	depid=$(flux submit --dependency=after:$jobid hostname) &&
+	flux job wait-event -vt 15 $depid dependency-add &&
+	test_debug "echo checking that job ${depid} is in DEPEND state" &&
+	test "$(flux jobs -no {state} $depid)" = "DEPEND" &&
+	flux cancel $jobid &&
+	flux job wait-event -m type=dependency -vt 15 $depid exception 
+'
+test_expect_success 'dependency=afterany works' '
+	flux bulksubmit \
+		--urgency=hold \
+		--job-name={} \
+		--log=jobids.afterany {} 300 \
+		::: true false sleep &&
+	job1=$(sed "1q;d" jobids.afterany) &&
+	job2=$(sed "2q;d" jobids.afterany) &&
+	job3=$(sed "3q;d" jobids.afterany) &&
+	jobid=$(flux submit \
+		--dependency=afterany:$job1 \
+		--dependency=afterany:$job2 \
+		--dependency=afterany:$job3 \
+		hostname) &&
+	for id in $(cat jobids.afterany);
+		do flux job urgency $id default;
+	done &&
+	flux jobs &&
+	flux job wait-event -vt 15 \
+		-m description=after-finish=$job2 \
+		$jobid dependency-remove &&
+	flux jobs &&
+	flux cancel $job3 &&
+	flux job wait-event -vt 15 $jobid clean
+'
+test_expect_success 'dependency=afterok works' '
+	flux bulksubmit \
+		--urgency=hold \
+		--job-name={} \
+		--log=jobids.afterok {} 300 \
+		::: true false sleep &&
+	job1=$(sed "1q;d" jobids.afterok) &&
+	job2=$(sed "2q;d" jobids.afterok) &&
+	job3=$(sed "3q;d" jobids.afterok) &&
+	ok1=$(flux submit \
+		--dependency=afterok:$job1 \
+		hostname) &&
+	ok2=$(flux submit \
+		--dependency=afterok:$job2 \
+		hostname) &&
+	ok3=$(flux submit \
+		--dependency=afterok:$job3 \
+		hostname) &&
+	for id in $(cat jobids.afterok);
+		do flux job urgency $id default;
+	done &&
+	flux job wait-event -vt 15 $job3 start &&
+	flux cancel $job3 &&
+	flux job wait-event -vt 15 \
+		-m description=after-success=$job1 \
+		$ok1 dependency-remove &&
+	flux job wait-event -vt 15 \
+		-m type=dependency $ok2 exception &&
+	flux job wait-event -vt 15 \
+		-m type=dependency $ok2 exception
+'
+test_expect_success 'dependency=afternotok works' '
+	flux bulksubmit \
+		--urgency=hold \
+		--job-name={} \
+		--log=jobids.afternotok {} 300 \
+		::: true false sleep &&
+	job1=$(sed "1q;d" jobids.afternotok) &&
+	job2=$(sed "2q;d" jobids.afternotok) &&
+	job3=$(sed "3q;d" jobids.afternotok) &&
+	ok1=$(flux submit \
+		--dependency=afternotok:$job1 \
+		hostname) &&
+	ok2=$(flux submit \
+		--dependency=afternotok:$job2 \
+		hostname) &&
+	ok3=$(flux submit \
+		--dependency=afternotok:$job3 \
+		hostname) &&
+	for id in $(cat jobids.afternotok);
+		do flux job urgency $id default;
+	done &&
+	flux job wait-event -vt 15 $job3 start &&
+	flux cancel $job3 &&
+	flux job wait-event -vt 15 \
+		-m description=after-failure=$job2 \
+		$ok2 dependency-remove &&
+	flux job wait-event -vt 15 \
+		-m description=after-failure=$job3 \
+		$ok3 dependency-remove &&
+	flux job wait-event -vt 15 \
+		-m type=dependency $ok1 exception
+'
+test_expect_success 'dependency=after works for INACTIVE jobs' '
+	run_timeout 15 \
+		flux bulksubmit --wait --watch \
+		--job-name=after:{} \
+		--dependency=after:{} \
+		echo after:{} ::: ${job1} ${job2} ${job3}
+'
+test_expect_success 'dependency=afterany works for INACTIVE jobs' '
+	run_timeout 15 \
+		flux bulksubmit --wait --watch \
+		--job-name=afterany:{} \
+		--dependency=afterany:{} \
+		echo afterany:{} ::: ${job1} ${job2} ${job3}
+'
+test_expect_success 'dependency=afterok works for INACTIVE job' '
+	run_timeout 15 \
+		flux run --dependency=afterok:${job1} \
+		echo afterok:${job1} &&
+	test_must_fail flux run --dependency=afterok:${job2} hostname &&
+	test_must_fail flux run --dependency=afterok:${job3} hostname 
+'
+test_expect_success 'dependency=afternotok works for INACTIVE job' '
+	run_timeout 15 \
+		flux bulksubmit --wait --watch \
+		--job-name=afternotok:{} \
+		--dependency=afternotok:{} \
+		echo afterany:{} ::: ${job2} ${job3} &&
+	test_must_fail flux run --dependency=afternotok:${job1} hostname
+'
+test_expect_success 'dependency=after fails for INACTIVE canceled job' '
+	job4=$(flux submit --urgency=hold hostname) &&
+	flux cancel ${job4} &&
+	test_must_fail flux run --dependency=after:${job4} hostname
+'
+test_expect_success 'jobs with dependencies can be safely canceled' '
+	jobid=$(flux submit --urgency=hold hostname) &&
+	depid=$(flux submit --dependency=after:$jobid hostname) &&
+	flux cancel $depid &&
+	flux job urgency $jobid default &&
+	flux job wait-event -vt 15 $jobid clean
+'
+test_expect_success 'flux jobtap query dependency-after works' '
+	flux jobtap query .dependency-after > query-none.json &&
+	test_debug "jq -S . query-none.json" &&
+	jq -e ".dependencies | length == 0" query-none.json &&
+	jobid=$(flux submit --urgency=hold hostname) &&
+	depid=$(flux submit --dependency=after:$jobid hostname) &&
+	flux jobtap query .dependency-after > query.json &&
+	test_debug "jq -S . query.json" &&
+	jq -e ".dependencies | length == 1" query.json &&
+	flux job urgency $jobid default &&
+	flux job wait-event -vt 15 $depid clean
+'
+test_done
diff --git a/t/t2272-job-begin-time.t b/t/t2272-job-begin-time.t
new file mode 100755
index 000000000000..71fccb3b28fd
--- /dev/null
+++ b/t/t2272-job-begin-time.t
@@ -0,0 +1,65 @@
+#!/bin/sh
+
+test_description='Test job begin-time dependencies'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'begin-time: submit a job with begin-time +1h' '
+	DELAYED=$(flux submit --begin-time=+1h hostname) &&
+	flux job wait-event -vt 15 $DELAYED dependency-add
+'
+test_expect_success 'begin-time: flux --begin-time=invalid fails' '
+	test_expect_code 1 flux submit --begin-time=foo hostname &&
+	test_expect_code 1 flux submit --begin-time=+a hostname
+'
+test_expect_success 'begin-time: rejects invalid timestamp' '
+	test_expect_code 1 flux submit \
+		--dependency=begin-time:-1.0 \
+		hostname
+'
+test_expect_success 'begin-time rejects missing timestamp' '
+	flux run --dry-run hostname | \
+	  jq ".attributes.system.dependencies[0].scheme = \"begin-time\"" \
+	  > invalid.json &&
+	test_expect_code 1 flux job submit invalid.json
+'
+test_expect_success '--begin-time sets dependency in jobspec' '
+	flux run --dry-run --begin-time=+1h hostname | \
+	    jq -e ".attributes.system.dependencies[0].scheme == \"begin-time\""
+'
+test_expect_success 'begin-time: elapsed begin-time releases job immediately' '
+	jobid=$(flux submit --begin-time=now hostname) &&
+	flux job wait-event -vt 15 $jobid dependency-remove &&
+	flux job wait-event -vt 15 $jobid clean
+'
+test_expect_success 'begin-time: job with begin-time works' '
+	flux run -vvv --begin=time=+1s hostname
+'
+test_expect_success 'begin-time: job with begin-time=+1h is still in depend' '
+	flux jobs &&
+	test $(flux jobs -no {state} $DELAYED) = "DEPEND"
+'
+test_expect_success 'begin-time: dependency is displayed in flux-jobs output' '
+	flux jobs | grep begin@
+'
+test_expect_success 'begin-time: job with begin-time can be safely canceled' '
+	flux cancel $DELAYED &&
+	flux job wait-event -vt 15 $DELAYED clean
+'
+test_expect_success 'begin-time: begin-time=8pm drops trailing :00' '
+	jobid=$(flux submit --begin-time="8pm tomorrow" hostname) &&
+	flux jobs &&
+	flux jobs ${jobid} | grep -- "-20:00$" &&
+	flux cancel $jobid
+'
+test_expect_success 'begin-time: begin-time=8:00:21 keeps trailing seconds' '
+	jobid=$(flux submit --begin-time="8:00:21 tomorrow" hostname) &&
+	flux jobs &&
+	flux jobs ${jobid} | grep -- "-08:00:21$" &&
+	flux cancel $jobid
+'
+test_done
diff --git a/t/t2273-job-alloc-bypass.t b/t/t2273-job-alloc-bypass.t
new file mode 100755
index 000000000000..af0fe4b423d3
--- /dev/null
+++ b/t/t2273-job-alloc-bypass.t
@@ -0,0 +1,160 @@
+#!/bin/sh
+
+test_description='Test alloc-bypass job manager plugin'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 2 job
+
+flux version | grep -q libflux-security && test_set_prereq FLUX_SECURITY
+
+flux setattr log-stderr-level 1
+
+submit_as_alternate_user()
+{
+        FAKE_USERID=42
+        test_debug "echo running flux run $@ as userid $FAKE_USERID"
+        flux run --dry-run "$@" | \
+	  flux python ${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py $FAKE_USERID \
+            >job.signed
+        FLUX_HANDLE_USERID=$FAKE_USERID \
+          flux job submit --flags=signed job.signed
+}
+
+test_expect_success 'alloc-bypass: start a job using all resources' '
+	SLEEPID=$(flux submit --wait-event=start \
+	            -n $(flux resource list -s up -no {ncores}) \
+	            sleep 300)
+'
+test_expect_success 'alloc-bypass: load alloc-bypass plugin' '
+	flux jobtap load alloc-bypass.so
+'
+test_expect_success 'alloc-bypass: works' '
+	flux resource list &&
+	run_timeout 15 \
+	    flux run \
+	        --setattr=system.alloc-bypass.R="$(flux kvs get resource.R)" \
+	        -o per-resource.type=node hostname
+'
+test_expect_success 'alloc-bypass: scheduler still running' '
+	flux ping -c 1 sched-simple
+'
+test_expect_success 'alloc-bypass: works with per-resource.type=core' '
+	flux run \
+	    --setattr=system.alloc-bypass.R="$(flux kvs get resource.R)" \
+	    -o per-resource.type=core hostname
+'
+test_expect_success 'alloc-bypass: works with jobid' '
+	flux run \
+	    --setattr=system.alloc-bypass.R="$(flux job info $SLEEPID R)" \
+	    -o per-resource.type=node flux getattr rank
+'
+test_expect_success FLUX_SECURITY 'alloc-bypass: guest user not allowed' '
+	test_must_fail submit_as_alternate_user \
+	  --setattr=system.alloc-bypass.R="$(flux kvs get resource.R)" \
+          hostname
+'
+test_expect_success 'alloc-bypass; invalid alloc-bypass.R type fails' '
+	test_must_fail flux run \
+	    --setattr=system.alloc-bypass.R=1 \
+	    -o per-resource.type=node hostname
+'
+test_expect_success 'alloc-bypass: invalid alloc-bypass.R object fails' '
+	test_must_fail flux run \
+	    --setattr=system.alloc-bypass.R="{\"version\": 1}" \
+	    -o per-resource.type=node hostname
+'
+test_expect_success 'alloc-bypass: handles exception before alloc event' '
+	jobid=$(flux submit \
+	        --setattr=system.alloc-bypass.R="$(flux kvs get resource.R)" \
+		--dependency=afterok:$SLEEPID \
+		-o per-resource.type=node hostname) &&
+	flux job wait-event -vt 15 $jobid dependency-add &&
+	flux cancel $jobid &&
+	test_must_fail flux job attach -vEX $jobid
+'
+test_expect_success 'alloc-bypass: kill sleep job' '
+	flux cancel --all &&
+	flux job wait-event $SLEEPID clean
+'
+test_expect_success 'alloc-bypass: submit an alloc-bypass job' '
+	flux submit -vvv --wait-event=start --job-name=bypass \
+		--setattr=alloc-bypass.R="$(flux R encode -r 0)" \
+		-n 1 \
+		sleep 300
+'
+test_expect_success 'alloc-bypass: a full system job can still be run' '
+	run_timeout 15 \
+	  flux run -n $(flux resource list -s up -no {ncores}) hostname
+'
+test_expect_success 'kill bypass job' '
+	flux pkill bypass &&
+	flux queue drain
+'
+test_expect_success 'alloc-bypass: submit an alloc-bypass job' '
+	flux submit -vvv --wait-event=start --job-name=bypass2 \
+		--setattr=alloc-bypass.R="$(flux R encode -r 0)" \
+		-n 1 \
+		sleep 300
+'
+test_expect_success 'alloc-bypass: scheduler has no nodes allocated' '
+	test $(FLUX_RESOURCE_LIST_RPC=sched.resource-status flux resource list -s allocated -no {nnodes}) -eq 0
+'
+test_expect_success 'alloc-bypass: reload scheduler' '
+	flux module reload sched-simple
+'
+# issue #5797 - bypass jobs must not be included in scheduler hello message
+test_expect_success 'alloc-bypass: scheduler still has no nodes allocated' '
+	test $(FLUX_RESOURCE_LIST_RPC=sched.resource-status flux resource list -s allocated -no {nnodes}) -eq 0
+'
+#
+# now try a bypass job that persists across a flux restart
+#
+test_expect_success 'generate a config that loads alloc-bypass' '
+	cat <<-EOT >config.toml
+	[[job-manager.plugins]]
+	load = "alloc-bypass.so"
+	EOT
+'
+test_expect_success 'generate a script to submit a testexec+alloc-bypass job' '
+	cat >prog.sh <<-EOT &&
+	#!/bin/sh
+	flux submit -vvv -N 1 \
+            --flags=debug \
+            --setattr=system.exec.test.run_duration=100s \
+            --setattr=system.alloc-bypass.R="\$(flux resource R)" \
+            --wait-event=start \
+	    sleep 300
+	EOT
+	chmod +x prog.sh
+'
+test_expect_success 'run test job in a persistent instance' '
+	FLUX_DISABLE_JOB_CLEANUP=t flux start -s1 \
+	    --config-path=config.toml \
+	    -Sstatedir=$(pwd) \
+	    ./prog.sh
+'
+test_expect_success 'restart that instance and get the resource alloc count' '
+	FLUX_DISABLE_JOB_CLEANUP=t flux start -s1 \
+	    --config-path=config.toml \
+	    -Sstatedir=$(pwd) \
+	    sh -c "flux jobs -no {state} \$(flux job last); \
+	        FLUX_RESOURCE_LIST_RPC=sched.resource-status \
+		flux resource list -s allocated -no {nnodes}" >restart.out
+'
+test_expect_success 'the job was running and resources were not allocated' '
+	cat >restart.exp <<-EOT &&
+	RUN
+	0
+	EOT
+	test_cmp restart.exp restart.out
+'
+test_expect_success 'restart that instance and cancel the job' '
+	flux start -s1 \
+	    --config-path=config.toml \
+	    -Sstatedir=$(pwd) \
+	    sh -c "flux cancel \$(flux job last); \
+	    flux job wait-event -vvv \$(flux job last) clean"
+'
+
+test_done
diff --git a/t/t2274-manager-perilog-per-rank.t b/t/t2274-manager-perilog-per-rank.t
new file mode 100755
index 000000000000..315ca7442543
--- /dev/null
+++ b/t/t2274-manager-perilog-per-rank.t
@@ -0,0 +1,466 @@
+#!/bin/sh
+
+test_description='Test perilog jobtap plugin with per-rank=true'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 full \
+	--test-exit-mode=leader
+
+OFFLINE_PLUGIN=${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs/offline.so
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+
+flux setattr log-stderr-level 1
+
+# In case the testsuite is running as a Flux job
+unset FLUX_JOB_ID
+
+drained_ranks() { flux resource status -no {ranks} -s drain; }
+
+no_drained_ranks() { test "$(drained_ranks)" = ""; }
+
+undrain_all() {
+	ranks="$(drained_ranks)"
+	if test -n "$ranks"; then
+	    flux resource undrain $ranks
+	fi
+}
+
+test_expect_success 'perilog: load plugin with no config' '
+	flux jobtap load perilog.so
+'
+test_expect_success 'perilog: query shows no prolog/epilog config' '
+	test_debug "flux jobtap query perilog.so | jq .conf" &&
+	flux jobtap query perilog.so \
+		| jq -e ".conf.prolog == {} and .conf.epilog == {}"
+'
+test_expect_success 'perilog: default timeouts are expected values' '
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	command = ["prolog"]
+	per-rank = true
+	[job-manager.epilog]
+	command = ["epilog"]
+	per-rank = true
+	EOF
+	flux jobtap query perilog.so | jq .conf &&
+	flux jobtap query perilog.so \
+		| jq -e ".conf.prolog.timeout == 1800.0" &&
+	flux jobtap query perilog.so \
+		| jq -e ".conf.epilog.timeout == 0.0"
+'
+test_expect_success 'perilog: timeout can be set to 0' '
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	timeout = "0"
+	command = [ "test" ]
+	EOF
+	flux jobtap query perilog.so | jq .conf &&
+	flux jobtap query perilog.so \
+		| jq -e ".conf.prolog.timeout == 0.0"
+'
+test_expect_success 'perilog: timeout can be set to infinity (equiv to 0)' '
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	timeout = "infinity"
+	command = [ "test" ]
+	EOF
+	flux jobtap query perilog.so | jq .conf &&
+	flux jobtap query perilog.so \
+		| jq -e ".conf.prolog.timeout == 0"
+'
+test_expect_success 'perilog: invalid config is rejected' '
+	test_must_fail flux config load <<-EOF 2>config.err.$((i+=1)) &&
+	[job-manager.prolog]
+	command = "foo"
+	EOF
+	test_debug "cat config.err.$i" &&
+	grep "command must be an array" config.err.$i &&
+	test_must_fail flux config load <<-EOF 2>config.err.$((i+=1)) &&
+	[job-manager.prolog]
+	command = [ "foo" ]
+	timeout = "5x"
+	EOF
+	test_debug "cat config.err.$i" &&
+	grep "invalid prolog timeout" config.err.$i &&
+	test_must_fail flux config load <<-EOF 2>config.err.$((i+=1)) &&
+	[job-manager.epilog]
+	command = [ "foo" ]
+	timeout = "5p"
+	EOF
+	test_debug "cat config.err.$i" &&
+	grep "invalid epilog timeout" config.err.$i &&
+	test_must_fail flux config load <<-EOF 2>config.err.$((i+=1)) &&
+	[job-manager.prolog]
+	EOF
+	test_debug "cat config.err.$i" &&
+	grep "no command specified" config.err.$i &&
+	test_must_fail flux config load <<-EOF 2>config.err.$((i+=1)) &&
+	[job-manager.prolog]
+	command = [ 1, 2, 3 ]
+	EOF
+	test_debug "cat config.err.$i" &&
+	grep "malformed prolog command" config.err.$i &&
+	test_must_fail flux config load <<-EOF 2>config.err.$((i+=1)) &&
+	[job-manager.prolog]
+	command = [ "foo" ]
+	per-rank = true
+	foo = 1
+	EOF
+	test_debug "cat config.err.$i" &&
+	grep "1 object.*left unpacked: foo" config.err.$i &&
+	test_must_fail flux config load <<-EOF 2>config.err.$((i+=1)) &&
+	[job-manager.epilog]
+	command = [ "foo" ]
+	kill-timeout = 5.5
+	EOF
+	test_debug "cat config.err.$i" &&
+	grep "kill-timeout not allowed for epilog" config.err.$i &&
+	test_must_fail flux config load <<-EOF 2>config.err.$((i+=1)) &&
+	[job-manager.perilog]
+	log-ignore = "foo"
+	EOF
+	test_debug "cat config.err.$i" &&
+	grep "not an array" config.err.$i &&
+	test_must_fail flux config load <<-EOF 2>config.err.$((i+=1)) &&
+	[job-manager.perilog]
+	log-ignore = [ "[" ]
+	EOF
+	test_debug "cat config.err.$i" &&
+	grep "[fF]ailed to compile" config.err.$i
+'
+test_expect_success 'perilog: config uses IMP with exec.imp and no command' '
+	flux config load <<-EOF &&
+	exec.imp = "imp"
+	[job-manager.prolog]
+	per-rank = true
+	[job-manager.epilog]
+	per-rank = true
+	EOF
+	flux jobtap query perilog.so \
+		| jq ".conf.prolog.command == [\"imp\", \"run\", \"prolog\"]" &&
+	flux jobtap query perilog.so \
+		| jq ".conf.epilog.command == [\"imp\", \"run\", \"prolog\"]"
+'
+test_expect_success 'perilog: prolog default cancel-on-exception is true' '
+	flux jobtap query perilog.so \
+		| jq ".conf.prolog.cancel_on_exception == true"
+'
+test_expect_success 'perilog: epilog default cancel-on-exception is false' '
+	flux jobtap query perilog.so \
+		| jq ".conf.epilog.cancel_on_exception == false"
+'
+test_expect_success 'perilog: load a basic per-rank prolog config' '
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "flux", "getattr", "rank" ]
+	EOF
+	flux jobtap query perilog.so | jq .conf.prolog
+'
+test_expect_success 'perilog: 4 node job works with per-rank prolog' '
+	jobid=$(flux submit -vvv -N4 hostname) &&
+	flux job wait-event -vt 30 $jobid prolog-start &&
+	flux job wait-event -t 30 $jobid prolog-finish &&
+	flux job wait-event -vt 30 $jobid clean &&
+	no_drained_ranks
+'
+test_expect_success 'perilog: stdout was copied to dmesg log' '
+	flux dmesg -H &&
+	flux dmesg -H | grep "$jobid: prolog:.*rank 0.*stdout: 0" &&
+	flux dmesg -H | grep "$jobid: prolog:.*rank 1.*stdout: 1" &&
+	flux dmesg -H | grep "$jobid: prolog:.*rank 2.*stdout: 2" &&
+	flux dmesg -H | grep "$jobid: prolog:.*rank 3.*stdout: 3" 
+'
+test_expect_success 'perilog: load a basic per-rank prolog config' '
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "sleep", "30" ]
+	[job-manager.epilog]
+	per-rank = true
+	command = [ "sleep", "30" ]
+	cancel-on-exception = true
+	EOF
+	flux jobtap query perilog.so | jq .conf.prolog
+'
+test_expect_success 'perilog: prolog runs on all 4 ranks of a 4 node job' '
+	flux dmesg -c &&
+	jobid=$(flux submit -N4 hostname) &&
+	flux job wait-event -vt 30 $jobid prolog-start &&
+	flux jobtap query perilog.so | jq .procs &&
+	flux jobtap query perilog.so \
+		| jq -e ".procs.$jobid.active_ranks == \"0-3\"" &&
+	flux jobtap query perilog.so \
+		| jq -e ".procs.$jobid.total == 4" &&
+	flux jobtap query perilog.so \
+		| jq -e ".procs.$jobid.active == 4" &&
+	flux cancel $jobid &&
+	flux jobtap query perilog.so &&
+	flux job wait-event $jobid prolog-finish
+'
+test_expect_success 'perilog: canceled prolog does not drain ranks' '
+	no_drained_ranks
+'
+test_expect_success 'perilog: epilog runs even if prolog is canceled' '
+	flux dmesg -H &&
+	flux job wait-event -vHt 30 $jobid epilog-start &&
+	flux jobtap query perilog.so | jq .procs &&
+	flux jobtap query perilog.so \
+		| jq -e ".procs.$jobid.name == \"epilog\"" &&
+	flux jobtap query perilog.so \
+		| jq -e ".procs.$jobid.active_ranks == \"0-3\"" &&
+	flux jobtap query perilog.so \
+		| jq -e ".procs.$jobid.total == 4" &&
+	flux jobtap query perilog.so \
+		| jq -e ".procs.$jobid.active == 4" &&
+	flux cancel $jobid &&
+	flux jobtap query perilog.so | jq &&
+	flux job wait-event -Hvt 30 $jobid clean
+'
+test_expect_success 'perilog: epilog triggered by prolog has correct userid' '
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "sleep", "5" ]
+	[job-manager.epilog]
+	per-rank = true
+	command = [ "sh", "-c", "echo FLUX_JOB_USERID=\$FLUX_JOB_USERID" ]
+	EOF
+	jobid=$(flux submit -N4 hostname) &&
+	flux job wait-event -vHt 30 $jobid prolog-start &&
+	flux cancel $jobid &&
+	flux job wait-event -vHt 30 $jobid clean &&
+	flux dmesg -H  | grep FLUX_JOB_USERID=$(id -u)
+'
+test_expect_success 'perilog: canceled prolog drains active ranks after kill_timeout' '
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "sh", "-c", "trap \"\" 15; if test \$(flux getattr rank) -eq 3; then sleep 10; fi" ]
+	kill-timeout = 0.5
+	EOF
+	flux jobtap query perilog.so | jq .conf.prolog &&
+	jobid=$(flux submit -N4 hostname) &&
+	flux job wait-event $jobid prolog-start &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid prolog-finish &&
+	test_debug "echo drained_ranks=$(drained_ranks)" &&
+	test "$(drained_ranks)" = "3" &&
+	flux resource drain -no {reason} | grep "canceled then timed out" &&
+	flux job wait-event -vt 15 $jobid clean &&
+	flux resource undrain 3
+'
+test_expect_success 'perilog: signaled prolog is reported' '
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "sh", "-c", "kill \$\$" ]
+	EOF
+	jobid=$(flux submit hostname) &&
+	flux job wait-event -vHt 30 $jobid exception &&
+	flux job wait-event -t 30 $jobid exception >exception.out &&
+	grep "prolog killed by signal 15" exception.out
+'
+test_expect_success 'perilog: prolog failure drains affected ranks' '
+	undrain_all &&
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "sh",
+	            "-c",
+	            "if test \$(flux getattr rank) -eq 3; then exit 1; fi" ]
+	EOF
+	flux dmesg -C &&
+	jobid=$(flux submit -N4 hostname) &&
+	flux job wait-event -vHt 30 $jobid prolog-start &&
+	flux jobtap query perilog.so | jq .procs.$jobid &&
+	test_must_fail flux job attach $jobid &&
+	flux dmesg -H &&
+	flux resource drain -no "{ranks} {reason}" &&
+	test "$(drained_ranks)" = "3" &&
+	test "$(flux resource drain -no {reason})" = "prolog failed for job $jobid" &&
+	flux resource drain
+'
+test_expect_success 'perilog: prolog timeout works and drains ranks' '
+	undrain_all &&
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	timeout = "1s"
+	per-rank = true
+	command = [ "sh",
+	            "-c",
+	            "if test \$(flux getattr rank) -eq 3; then sleep 30; fi" ]
+	EOF
+	jobid=$(flux submit -N4 hostname) &&
+	flux job wait-event -Hvt 30 $jobid prolog-start &&
+	flux jobtap query perilog.so | jq .procs.$jobid &&
+	test_must_fail flux job attach $jobid &&
+	test "$(drained_ranks)" = "3" &&
+	test "$(flux resource drain -no {reason})" = "prolog timed out for job $jobid" &&
+	undrain_all
+'
+test_expect_success 'perilog: prolog can drain multiple ranks' '
+	undrain_all &&
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "false" ]
+	EOF
+	test_must_fail flux run -N4 hostname &&
+	flux resource drain -o long &&
+	test "$(drained_ranks)" = "0-3" &&
+	undrain_all
+'
+test_expect_success 'perilog: nonfatal exception does not cancel prolog' '
+	undrain_all &&
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "sh",
+                    "-c",
+		    """flux job raise -s2 --type=test \$FLUX_JOB_ID
+                    flux job wait-event \$FLUX_JOB_ID exception""" ]
+	EOF
+	jobid=$(flux submit hostname) &&
+	flux job wait-event -t 15 $jobid prolog-start &&
+	flux job wait-event -t 15 $jobid prolog-finish &&
+	flux job wait-event -t 15 $jobid clean &&
+	flux jobs $jobid &&
+	flux job status -vvv $jobid &&
+	no_drained_ranks
+'
+test_expect_success 'perilog: job can time out after prolog' '
+	undrain_all &&
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "sleep", "1" ]
+	EOF
+	jobid=$(flux submit -t0.5s sleep 10) &&
+	flux job wait-event -t 15 $jobid prolog-finish &&
+	flux job wait-event -vt 15 $jobid exception &&
+	flux job wait-event -vt 15 $jobid clean &&
+	flux job status -v $jobid 2>&1 | grep type=timeout
+'
+test_expect_success 'perilog: job can be canceled after prolog is complete' '
+	undrain_all &&
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "sleep", "0" ]
+	EOF
+	jobid=$(flux submit sleep 300) &&
+	flux job wait-event -t 15 $jobid prolog-finish &&
+	flux cancel $jobid &&
+	flux job wait-event -t 15 $jobid exception &&
+	test_must_fail_or_be_terminated flux job status $jobid
+'
+test_expect_success 'perilog: epilog runs on all ranks with per-rank' '
+	undrain_all &&
+	flux config load <<-EOF &&
+	[job-manager.epilog]
+	per-rank = true
+	cancel-on-exception = true
+	command = [ "sleep", "15" ]
+	EOF
+	jobid=$(flux submit -N4 hostname) &&
+	flux job wait-event -vHt 30 $jobid epilog-start &&
+	flux jobtap query perilog.so | jq .procs.$jobid &&
+	flux jobtap query perilog.so \
+		| jq -e ".procs.$jobid.active_ranks == \"0-3\"" &&
+	flux jobtap query perilog.so \
+		| jq -e ".procs.$jobid.total == 4" &&
+	flux jobtap query perilog.so \
+		| jq -e ".procs.$jobid.active == 4" &&
+	flux jobtap query perilog.so \
+		| jq -e ".conf.epilog.cancel_on_exception == true" &&
+	flux cancel $jobid &&
+	flux jobtap query perilog.so | jq &&
+	flux job wait-event -vHt 20 $jobid clean
+'
+test_expect_success 'perilog: canceled epilog does not drain ranks' '
+	no_drained_ranks
+'
+test_expect_success 'perilog: epilog failure drains ranks' '
+	undrain_all &&
+	flux config load <<-EOF &&
+	[job-manager.epilog]
+	per-rank = true
+	command = [ "sh",
+	            "-c",
+                    "if test \$(flux getattr rank) -eq 1; then exit 1; fi" ]
+	EOF
+	flux jobtap query perilog.so | jq &&
+	jobid=$(flux submit -N4 hostname) &&
+	flux job wait-event -vHt 30 $jobid epilog-finish &&
+	flux resource drain -o long &&
+	test "$(drained_ranks)" = "1" &&
+	test "$(flux resource drain -no {reason})" = "epilog failed for job $jobid" &&
+	undrain_all
+'
+test_expect_success 'perilog: job does not start when prolog cancel times out' '
+	undrain_all &&
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "sh",
+	            "-c",
+	            "trap \"\" 15; sleep 2" ]
+	kill-timeout = 1
+	timeout = ".25s"
+	[job-manager.epilog]
+	per-rank = true
+	command = [ "true" ]
+	EOF
+	flux jobtap query perilog.so | jq &&
+	jobid=$(flux submit hostname) &&
+	flux job wait-event -vHt 30 $jobid clean &&
+	flux job eventlog -H $jobid >prolog-cancel-eventlog.out &&
+	cat prolog-cancel-eventlog.out &&
+	test_must_fail grep "start$" prolog-cancel-eventlog.out &&
+	grep epilog-start prolog-cancel-eventlog.out
+'
+test_expect_success 'perilog: log-ignore works' '
+	undrain_all &&
+	flux config load <<-EOF &&
+	[job-manager.prolog]
+	command = [ "printf", "foo: whee!\nbar: woo!\nbaz: important!\n" ]
+	[job-manager.perilog]
+	log-ignore = [ "^foo:.*", "^bar:" ]
+	EOF
+	flux dmesg -c >/dev/null &&
+	flux run hostname &&
+	flux dmesg -H > dmesg.out &&
+	test_debug "cat dmesg.out" &&
+	test_must_fail grep foo: dmesg.out &&
+	test_must_fail grep bar: dmesg.out &&
+	grep baz: dmesg.out
+'
+test_expect_success 'perilog: load offline plugin before perilog.so' '
+	flux jobtap remove perilog.so &&
+	flux jobtap load $OFFLINE_PLUGIN &&
+	flux jobtap load perilog.so
+'
+test_expect_success 'perilog: load simple prolog for offline rank testing' '
+	undrain_all &&
+	flux config load <<-EOF
+	[job-manager.prolog]
+	per-rank = true
+	command = [ "flux", "getattr", "rank" ]
+	EOF
+'
+# Note: bulk-exec does not return an error code on EHOSTUNREACH since
+# it is assumed a job exception will be raised by the instance.
+# Here we are just checking that the prolog finishes without blocking the
+# job from moving to CLEANUP/INACTIVE.
+test_expect_success 'perilog: prolog is successful with offline rank' '
+	undrain_all &&
+	jobid=$(flux submit -N4 -n4 true) &&
+	flux job wait-event -t 30 $jobid prolog-start &&
+	flux job wait-event -vHt 30 $jobid prolog-finish &&
+	flux jobtap remove offline.so
+'
+test_done
diff --git a/t/t2275-job-duration-validator.t b/t/t2275-job-duration-validator.t
new file mode 100755
index 000000000000..7094a6bb36a0
--- /dev/null
+++ b/t/t2275-job-duration-validator.t
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+test_description='Test job duration validator plugin in job-manager'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'duration: no validation when there is no expiration' '
+	flux submit -t 100d true
+'
+
+wait_duration_update() {
+	local value=$1
+	local n=10
+	while test $n -gt 0; do
+		flux dmesg | grep "duration-validator:.*$value" && return 0
+	done
+	return 1
+}
+test_expect_success 'duration: set an expiration on resource.R' '
+	expiration=$(($(date +%s)+60)) &&
+	flux kvs put \
+	    resource.R="$(flux kvs get resource.R | \
+			jq -S .execution.expiration=$expiration)" &&
+	wait_duration_update $expiration
+'
+test_expect_success 'duration: submit a job without duration' '
+	flux submit true
+'
+test_expect_success 'duration: submit a job with duration below expiration' '
+	flux submit -t 5s true
+'
+test_expect_success 'duration: submit a job with duration after expiration' '
+	test_must_fail flux submit -t 1h true
+'
+test_done
diff --git a/t/t2276-job-requires.t b/t/t2276-job-requires.t
new file mode 100755
index 000000000000..ff5164fd080e
--- /dev/null
+++ b/t/t2276-job-requires.t
@@ -0,0 +1,107 @@
+#!/bin/sh
+
+test_description='Test job constraints'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 job
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'flux run: --requires option works' '
+	flux run --dry-run \
+		--env=-* \
+		--requires=foo,bar \
+		--requires=^baz \
+		 true | \
+		jq '.attributes.system.constraints' > constraints.json &&
+	test_debug "jq -S . < constraints.json" &&
+	jq -e ".properties[0] == \"foo\"" < constraints.json &&
+	jq -e ".properties[1] == \"bar\"" < constraints.json &&
+	jq -e ".properties[2] == \"^baz\"" < constraints.json
+'
+test_expect_success 'reload scheduler with properties set' '
+	flux kvs put resource.R="$(flux kvs get resource.R | \
+		flux R set-property xx:2-3 yy:0-2)" &&
+	flux module unload sched-simple &&
+	flux module reload resource noverify &&
+	flux module load sched-simple
+'
+test_expect_success 'reload ingest with only feasibility validator' '
+	flux module reload -f job-ingest validator-plugins=feasibility
+'
+test_expect_success 'scheduler rejects jobs with invalid requires' '
+	test_must_fail flux submit --requires=x hostname &&
+	test_must_fail flux submit --requires="&x" hostname
+'
+test_expect_success 'scheduler rejects jobs with invalid constraints' '
+	test_must_fail flux submit --setattr=system.constraints.foo=[] \
+		 hostname &&
+	test_must_fail flux submit --setattr=system.constraints.and={} \
+		 hostname
+'
+test_expect_success 'reload ingest with feasibility,jobspec validators' '
+	flux module reload -f job-ingest validator-plugins=jobspec,feasibility
+'
+test_expect_success 'scheduler rejects jobs with unsatisfiable constraints' '
+	test_must_fail flux submit -N4 --requires=yy hostname
+'
+test_expect_success 'jobspec validator rejects invalid hostlist/ranks' '
+	test_must_fail flux submit -n1 --requires=host:f[ hostname &&
+	test_must_fail flux submit -n1 --requires=ranks:1-0 hostname
+'
+test_expect_success 'jobspec validator rejects invalid constraint operation' '
+	test_must_fail flux submit -n1 --requires=foo:bar hostname
+'
+test_expect_success 'flux bulksubmit: --requires works with scheduler' '
+	flux bulksubmit --wait --log=job.{}.id -n1 --requires={} \
+		flux getattr rank \
+		::: xx yy xx,yy ^xx ^yy \
+		    rank:1 rank:3 -rank:0-2 \
+		    host:$(hostname) &&
+	result=$(flux job attach $(cat job.xx.id)) &&
+	test_debug "echo xx: $result" &&
+	test $result -eq 2 -o $result -eq 3 &&
+	result=$(flux job attach $(cat job.yy.id)) &&
+	test_debug "echo yy: $result" &&
+	test $result -eq 0 -o $result -eq 1 &&
+	result=$(flux job attach $(cat job.xx,yy.id)) &&
+	test_debug "echo xx,yy: $result" &&
+	test $result -eq 2 &&
+	result=$(flux job attach $(cat "job.^yy.id")) &&
+	test_debug "echo ^yy: $result" &&
+	test $result -eq 3 &&
+	result=$(flux job attach $(cat "job.^xx.id")) &&
+	test_debug "echo ^xx: $result" &&
+	test $result -eq 0 -o $result -eq 1 &&
+	result=$(flux job attach $(cat "job.rank:1.id")) &&
+	test_debug "echo rank:1: $result" &&
+	test $result -eq 1 &&
+	result=$(flux job attach $(cat "job.rank:3.id")) &&
+	test_debug "echo rank:3: $result" &&
+	test $result -eq 3 &&
+	result=$(flux job attach $(cat "job.-rank:0-2.id")) &&
+	test_debug "echo -rank:0-2: $result" &&
+	test $result -eq 3 &&
+	result=$(flux job attach $(cat "job.host:$(hostname).id")) &&
+	test_debug "echo host:hostname: $result"
+'
+test_expect_success 'scheduler does not schedule down nodes with constraints' '
+	flux resource drain 2 &&
+	id=$(flux submit -N1 --requires xx,yy flux getattr rank) &&
+	flux job wait-event $id priority &&
+	flux jobs &&
+	test $(flux jobs -no {state} $id) = "SCHED" &&
+	flux resource undrain 2 &&
+	flux job wait-event -vt 5 $id clean
+'
+test_expect_success 'scheduler does not schedule down nodes with constraints' '
+	flux resource drain 0 &&
+	id=$(flux submit -N3 --requires yy flux getattr rank) &&
+	flux job wait-event $id priority &&
+	flux jobs &&
+	test $(flux jobs -no {state} $id) = "SCHED" &&
+	flux resource undrain 0 &&
+	flux job wait-event -vt 5 $id clean
+'
+test_done
diff --git a/t/t2280-job-memo.t b/t/t2280-job-memo.t
new file mode 100755
index 000000000000..d60ef4aa509a
--- /dev/null
+++ b/t/t2280-job-memo.t
@@ -0,0 +1,108 @@
+#!/bin/sh
+test_description='Test flux job memo command'
+
+. $(dirname $0)/job-manager/sched-helper.sh
+. $(dirname $0)/sharness.sh
+
+if flux job submit --help 2>&1 | grep -q sign-type; then
+	test_set_prereq HAVE_FLUX_SECURITY
+fi
+
+test_under_flux 1 job
+
+
+flux setattr log-stderr-level 1
+
+runas() {
+        userid=$1 && shift
+        FLUX_HANDLE_USERID=$userid FLUX_HANDLE_ROLEMASK=0x2 "$@"
+}
+
+test_expect_success 'memo: error on insufficient arguments' '
+	test_expect_code 1 flux job memo 1234 &&
+	test_expect_code 1 flux job memo
+'
+test_expect_success 'memo: error on invalid jobid' '
+	test_expect_code 1 flux job memo f1 foo=bar
+'
+test_expect_success 'memo: create one inactive job' '
+	flux submit true >inactivejob &&
+	flux queue drain
+'
+test_expect_success 'memo: submit a running and pending job' '
+	flux bulksubmit --urgency={} sleep 300 ::: 16 16 0 >jobids &&
+	runid=$(head -n 1 jobids) &&
+	pendingid=$(tail -n 1 jobids) &&
+	flux job wait-event $runid start
+'
+test_expect_success 'memo: only job owner can add memo' '
+	test_expect_code 1 \
+	    runas 9999 flux job memo $pendingid foo=24 2>memo.guest.err &&
+	test_debug "cat memo.guest.err" &&
+	grep "guests can only add a memo to their own jobs" memo.guest.err
+'
+test_expect_success 'memo: memo cannot be added to inactive job' '
+	test_must_fail flux job memo $(cat inactivejob) foo=42
+'
+test_expect_success 'memo: add memo to pending job works' '
+	flux job memo $pendingid foo=42 a=b &&
+	flux job wait-event -t 10 $pendingid memo &&
+	jmgr_check_memo $pendingid foo 42
+'
+test_expect_success 'memo: remove memo from pending job works' '
+	flux job memo $pendingid foo=null &&
+	flux job wait-event -m foo=null -t 10 $pendingid memo &&
+	test_expect_code 1 jmgr_check_memo_exists $pendingid foo
+'
+test_expect_success 'memo: add volatile memo to pending job works' '
+	flux job memo --volatile $pendingid foo=bar &&
+	jmgr_check_memo $pendingid foo \"bar\" &&
+	flux job eventlog $pendingid > eventlog.pending &&
+	test_expect_code 1 grep foo=bar eventlog.pending
+'
+test_expect_success 'memo: add memo to running job works' '
+	flux job memo $runid foo=42 &&
+	flux job wait-event -t 10 $runid memo &&
+	jmgr_check_memo $runid foo 42
+'
+test_expect_success 'memo: remove memo from running job works' '
+	flux job memo $runid foo=null &&
+	flux job wait-event -m foo=null -t 10 $runid memo &&
+	test_expect_code 1 jmgr_check_memo_exists $runid foo
+'
+test_expect_success 'memo: flux job memo works with dotted path' '
+	flux job memo $runid a.b.c=42 a.d=test &&
+	jmgr_check_memo_exists $runid a.b.c &&
+	jmgr_check_memo_exists $runid a.d
+'
+test_expect_success 'memo: flux job memo works with key=- (stdin)' '
+	echo "xyz" | flux job memo $runid a.f=-A &&
+	jmgr_check_memo_exists $runid a.f
+'
+test_expect_success 'memo: flux job memo: null values unset keys' '
+	flux job memo $runid a.b=null &&
+	flux job wait-event -v -m "a={\"b\":null}" -t 10 $runid memo &&
+	test_expect_code 1 jmgr_check_memo_exists $runid a.b &&
+	jmgr_check_memo $runid a.d \"test\"
+'
+test_expect_success 'memo: available in flux-jobs {user} attribute' '
+	jlist_check_memo $runid a.d \"test\" &&
+	jlist_check_memo $pendingid a \"b\"
+'
+test_expect_success 'memo: reload job-list module' '
+	flux module reload job-list
+'
+test_expect_success 'memo: non-volatile memos still available in job-list' '
+	jlist_check_memo $runid a.d \"test\" &&
+	jlist_check_memo $pendingid a \"b\"
+'
+test_expect_success 'memo: cancel all jobs' '
+	flux cancel --all &&
+	flux job wait-event $runid clean &&
+	flux job wait-event $pendingid clean
+'
+test_expect_success 'memo: non-volatile memos still available for jobs' '
+	jlist_check_memo $runid a.d \"test\" &&
+	jlist_check_memo $pendingid a \"b\"
+'
+test_done
diff --git a/t/t2290-job-update.t b/t/t2290-job-update.t
new file mode 100755
index 000000000000..1cfcf4fae3ab
--- /dev/null
+++ b/t/t2290-job-update.t
@@ -0,0 +1,185 @@
+#!/bin/sh
+test_description='Test flux update command'
+
+. $(dirname $0)/sharness.sh
+
+if flux job submit --help 2>&1 | grep -q sign-type; then
+	test_set_prereq HAVE_FLUX_SECURITY
+fi
+
+test_under_flux 1 job
+
+flux setattr log-stderr-level 1
+
+runas_guest() {
+	local userid=$(($(id -u)+1))
+        FLUX_HANDLE_USERID=$userid FLUX_HANDLE_ROLEMASK=0x2 "$@"
+}
+
+submit_held_job_as_guest()
+{
+	local duration=$1
+	local userid=$(($(id -u)+1))
+        flux run --dry-run -t $duration true | \
+          flux python ${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py $userid \
+            >job.signed
+        FLUX_HANDLE_USERID=$userid \
+          flux job submit --flags=signed --urgency=0 job.signed
+}
+
+test_expect_success 'flux update requires jobid and keyval args' '
+	test_expect_code 2 flux update 1234 &&
+	test_expect_code 2 flux update
+'
+test_expect_success 'submit jobs for testing' '
+	inactive_jobid=$(flux submit --wait-event=clean true) &&
+	jobid=$(flux submit --urgency=hold sleep 60)
+'
+test_expect_success 'invalid jobid fails' '
+	test_expect_code 1 flux update f123 duration=10m 2>invalid-jobid.err &&
+	test_debug "cat invalid-jobid.err" &&
+	grep "unknown job id" invalid-jobid.err
+'
+test_expect_success 'update to inactive job fails' '
+	test_expect_code 1 flux update $inactive_jobid duration=1m \
+		2>inactive-jobid.err &&
+	test_debug "cat inactive-jobid.err" &&
+	grep "job is inactive" inactive-jobid.err
+'
+test_expect_success 'invalid jobspec key cannot be updated' '
+	test_expect_code 1 flux update $jobid foo=bar
+'
+test_expect_success 'guests can only update their own jobs' '
+	test_expect_code 1 runas_guest flux update $jobid duration=1h \
+		2>invalid-user.err &&
+	test_debug "cat invalid-user.err" &&
+	grep "guests may only update their own jobs" invalid-user.err
+'
+test_expect_success 'update of unlimited duration with relative value fails' '
+	test_expect_code 1 flux update $jobid duration=+1m  &&
+	test_expect_code 1 flux update $jobid duration=-1m
+'
+test_expect_success 'update request for negative duration fails' '
+	echo "{\"id\": $(flux job id $jobid),\
+	       \"updates\": {\"attributes.system.duration\": -1.0}\
+              }" \
+	    | ${FLUX_BUILD_DIR}/t/request/rpc job-manager.update 22 # EINVAL
+'
+test_expect_success 'update request with invalid duration type fails' '
+	echo "{\"id\": $(flux job id $jobid),\
+	       \"updates\": {\"attributes.system.duration\": "foo"}\
+              }" \
+	    | ${FLUX_BUILD_DIR}/t/request/rpc job-manager.update 71 # EPROTO
+'
+test_expect_success 'update of duration of pending job works' '
+	flux update $jobid duration=1m &&
+	flux job eventlog $jobid \
+		| grep jobspec-update \
+		| grep duration=60
+'
+test_expect_success 'update of duration accepts relative values' '
+	flux update --dry-run $jobid duration=+1m \
+	  | jq ".\"attributes.system.duration\" == 120." &&
+	flux update --dry-run $jobid duration=-30s \
+	  | jq ".\"attributes.system.duration\" == 30."
+'
+test_expect_success 'update with multiple keys fails if one key fails' '
+	test_expect_code 1 flux update $jobid duration=12345 name=foo
+'
+test_expect_success 'update of duration to inf sets duration to 0' '
+	flux update --dry-run $jobid duration=inf \
+	  | jq ".\"attributes.system.duration\" == 0."
+'
+test_expect_success 'flux update rejects duration <= 0 fails' '
+	test_expect_code 1 flux update $jobid duration=-1h
+'
+test_expect_success 'update affects duration of running job' '
+	flux update $jobid duration=0.1s &&
+	flux job urgency $jobid default &&
+	flux job wait-event -t 30 -m type=timeout $jobid exception &&
+	flux job info $jobid R \
+	 | jq -e "(.execution|(.expiration - .starttime)*10 + 0.5 | floor) == 1"
+'
+test_expect_success 'add a duration limit and submit a held job' '
+	echo policy.limits.duration=\"1m\" | flux config load &&
+	jobid=$(flux submit --urgency=hold -t 1m true)
+'
+test_expect_success 'instance owner can adjust duration past limits' '
+	flux update $jobid duration=1h &&
+	flux job eventlog $jobid \
+		| grep jobspec-update \
+		| grep duration=3600
+'
+test_expect_success FLUX_SECURITY 'guest update of their own job works' '
+	guest_jobid=$(submit_held_job_as_guest 1m) &&
+	runas_guest flux update -v $guest_jobid duration=1m  &&
+	flux job eventlog $guest_jobid \
+		| grep jobspec-update \
+		| grep duration=60
+'
+test_expect_success FLUX_SECURITY 'guest cannot update job duration past limit' '
+	test_expect_code 1 runas_guest flux update -v $guest_jobid duration=1h
+'
+test_expect_success 'instance owner can adjust duration past limits' '
+	flux update $jobid duration=1h &&
+	flux job eventlog $jobid \
+		| grep jobspec-update \
+		| grep duration=3600
+'
+test_expect_success FLUX_SECURITY 'instance owner can adjust guest job duration past limits' '
+	flux update $guest_jobid duration=1h &&
+	flux job eventlog $guest_jobid \
+		| grep jobspec-update \
+		| grep duration=3600
+'
+test_expect_success FLUX_SECURITY 'guest job is now immutable' '
+	test_expect_code 1 runas_guest \
+		flux update -v $guest_jobid duration=1m \
+			2>immutable.err &&
+	test_debug "cat immutable.err" &&
+	grep immutable immutable.err
+'
+test_expect_success 'adjust duration so future tests pass validation' '
+	flux update $jobid duration=1m
+'
+test_expect_success 'reload update-duration plugin with owner-allow-any=0' '
+	flux jobtap remove .update-duration &&
+	flux jobtap load .update-duration owner-allow-any=0
+'
+test_expect_success 'update duration above policy limit now fails' '
+	test_expect_code 1 flux update $jobid duration=1h 2>limit.err &&
+	test_debug "cat limit.err" &&
+	grep "requested duration exceeds policy limit" limit.err
+'
+test_expect_success 'update fails for running job' '
+	jobid=$(flux submit -t1m --wait-event=start sleep 60) &&
+	test_expect_code 1 flux update $jobid duration=90s 2>run.err &&
+	test_debug "cat run.err" &&
+	grep "duration update of running job requires instance owner" run.err
+'
+test_expect_success 'update of attributes.system.test fails' '
+	test_expect_code 1 flux update $jobid test=foo
+'
+test_expect_success 'load update-test jobtap plugin' '
+	PLUGINPATH=${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs &&
+	flux jobtap load --remove=all $PLUGINPATH/update-test.so
+'
+test_expect_success 'now update of attributes.system.test works' '
+	flux update $jobid test=foo-update &&
+	flux job eventlog $jobid \
+		| grep jobspec-update \
+		| grep foo-update
+'
+test_expect_success 'update-test plugin can reject updates' '
+	test_expect_code 1 flux update $jobid test=fail-test 2>fail-test.err &&
+	test_debug "cat fail-test.err" &&
+	grep "rejecting update" fail-test.err
+'
+test_expect_success 'multiple keys can be updated successfully' '
+	flux update -v $jobid test=ok test2=ok2 &&
+	flux job eventlog $jobid &&
+	flux job eventlog $jobid \
+		| grep jobspec-update \
+		| grep "test=\"ok\" attributes.system.test2=\"ok2\""
+'
+test_done
diff --git a/t/t2291-job-update-queue.t b/t/t2291-job-update-queue.t
new file mode 100755
index 000000000000..5cb7dd0db096
--- /dev/null
+++ b/t/t2291-job-update-queue.t
@@ -0,0 +1,167 @@
+#!/bin/sh
+test_description='Test update of job queue'
+
+. $(dirname $0)/sharness.sh
+
+if flux job submit --help 2>&1 | grep -q sign-type; then
+	test_set_prereq HAVE_FLUX_SECURITY
+fi
+
+test_under_flux 4 full
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'config queues and resources' '
+	flux R encode -r 0-3 -p batch:0-2 -p debug:3 \
+	   | tr -d "\n" \
+	   | flux kvs put -r resource.R=- &&
+	flux config load <<-EOT &&
+	[queues.batch]
+	requires = [ "batch" ]
+	policy.limits.duration = "1h"
+	policy.jobspec.defaults.system.duration = "1h"
+
+	[queues.debug]
+	requires = [ "debug" ]
+	policy.limits.duration = "1m"
+	policy.jobspec.defaults.system.duration = "1m"
+
+	[queues.other]
+
+	[policy.jobspec.defaults.system]
+	queue = "batch"
+	EOT
+	flux queue start --all &&
+	flux module unload sched-simple &&
+	flux module reload resource &&
+	flux module load sched-simple &&
+	flux queue list &&
+	flux resource list -o rlist
+'
+test_expect_success 'invalid queue update RPC fails' '
+	jobid=$(flux submit --urgency=hold -n1 hostname) &&
+	echo "{\"id\": $(flux job id $jobid),\
+	       \"updates\": {\"attributes.system.queue\": 42}\
+	      }" \
+            | ${FLUX_BUILD_DIR}/t/request/rpc job-manager.update 22 && # EINVAL
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean
+'
+test_expect_success 'update of invalid job fails' '
+	test_must_fail flux update f1234 queue=batch
+'
+test_expect_success 'update queue of running job fails' '
+	jobid=$(flux submit --wait-event=start -n1 sleep 300) &&
+	test_must_fail flux update $jobid queue=debug 2>running.err &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean &&
+	grep "update of queue for running job not supported" running.err
+'
+test_expect_success 'update to invalid queue fails' '
+	jobid=$(flux submit -q debug --urgency=hold hostname) &&
+	test_must_fail flux update $jobid queue=foo
+'
+test_expect_success 'update to same queue fails' '
+	test_must_fail flux update $jobid queue=debug
+'
+test_expect_success 'update to batch queue works' '
+	flux update $jobid queue=batch &&
+	test_debug "flux job eventlog $jobid" &&
+	flux job eventlog $jobid \
+	  | grep jobspec-update \
+	  | grep attributes.system.queue=\"batch\"
+'
+test_expect_success 'job runs on batch resources' '
+	flux job urgency $jobid default &&
+	flux job wait-event $jobid clean &&
+	test "$(flux jobs -no {queue} $jobid)" = "batch" &&
+	test $(flux jobs -no {ranks} $jobid) = "0"
+'
+test_expect_success 'update to queue with lower duration limit fails' '
+	jobid=$(flux submit -q batch --urgency=hold hostname) &&
+	test_must_fail flux update $jobid queue=debug 2>queue.err &&
+	grep "duration exceeds policy limit" queue.err
+'
+test_expect_success 'update of duration allows queue update' '
+	flux update $jobid queue=debug duration=1m  &&
+	test_debug "flux job eventlog $jobid" &&
+	flux job eventlog $jobid \
+	  | grep jobspec-update \
+	  | grep attributes.system.queue=\"debug\"
+'
+test_expect_success 'job runs on debug resources' '
+	flux job urgency $jobid default &&
+	flux job wait-event $jobid clean &&
+	test "$(flux jobs -no {queue} $jobid)" = "debug" &&
+	test $(flux jobs -no {ranks} $jobid) = "3"
+'
+test_expect_success 'update of infeasible job to queue fails' '
+	jobid=$(flux submit -q batch -N2 --urgency=hold hostname) &&
+	test_debug "flux jobs -a" &&
+	test_must_fail flux update $jobid duration=1m queue=debug \
+		2>infeasible.err &&
+	test_debug "cat infeasible.err" &&
+	flux cancel $jobid
+'
+test_expect_success 'update of queue for job with other constraints fails' '
+	jobid=$(flux submit --urgency=hold \
+		-q batch -N1 -t 1m --requires=rank:0 hostname) &&
+	test_must_fail flux update $jobid queue=debug \
+		2>constraints.err &&
+	test_debug "cat constraints.err" &&
+	grep "unable to update queue" constraints.err &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean
+'
+test_expect_success 'update from queue with no constraints works' '
+	jobid=$(flux submit -t 1m --urgency=hold -q other hostname) &&
+	flux update $jobid queue=debug &&
+	test_debug "flux job eventlog $jobid" &&
+	flux job eventlog $jobid \
+	  | grep jobspec-update \
+	  | grep attributes.system.queue=\"debug\" &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean
+'
+test_expect_success 'update to queue with no constraints works' '
+	jobid=$(flux submit --urgency=hold -q debug hostname) &&
+	flux update $jobid queue=other &&
+	test_debug "flux job eventlog $jobid" &&
+	flux job eventlog $jobid \
+	  | grep jobspec-update \
+	  | grep attributes.system.queue=\"other\"
+'
+test_expect_success 'job can still run' '
+	flux job urgency $jobid default &&
+	flux job attach $jobid
+'
+test_expect_success "job with constraints in a queue without can't be updated" '
+	jobid=$(flux submit -t 1m --urgency=hold -q other \
+		--requires=rank:0 hostname) &&
+	test_must_fail flux update $jobid queue=debug &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean
+'
+test_expect_success 'update-queue plugin can be unloaded' '
+	flux jobtap remove .update-queue &&
+	flux jobtap list -a >jobtap-list.out &&
+	test_must_fail grep update-queue jobtap-list.out
+'
+test_expect_success 'updates are now disabled' '
+	jobid=$(flux submit -q debug --urgency=hold hostname) &&
+	test_must_fail flux update $jobid queue=batch
+'
+test_expect_success 'update-queue plugin can be reloaded' '
+	flux jobtap load .update-queue &&
+	flux jobtap list -a >jobtap-list2.out &&
+	grep update-queue jobtap-list2.out
+'
+test_expect_success 'updates are reenabled' '
+	flux update $jobid queue=batch &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean &&
+	flux job eventlog $jobid \
+	  | grep jobspec-update \
+	  | grep attributes.system.queue=\"batch\"
+'
+test_done
diff --git a/t/t2292-job-update-running.t b/t/t2292-job-update-running.t
new file mode 100755
index 000000000000..8348bd6e01f7
--- /dev/null
+++ b/t/t2292-job-update-running.t
@@ -0,0 +1,285 @@
+#!/bin/sh
+test_description='Test update of running jobs'
+
+. $(dirname $0)/sharness.sh
+
+if flux job submit --help 2>&1 | grep -q sign-type; then
+	test_set_prereq HAVE_FLUX_SECURITY
+fi
+
+test_under_flux 4 job
+
+flux setattr log-stderr-level 1
+export FLUX_URI_RESOLVE_LOCAL=t
+runas_guest() {
+	local userid=$(($(id -u)+1))
+        FLUX_HANDLE_USERID=$userid FLUX_HANDLE_ROLEMASK=0x2 "$@"
+}
+
+submit_job_as_guest()
+{
+	local duration=$1
+	local userid=$(($(id -u)+1))
+        flux run --dry-run -t $duration \
+	  --setattr=exec.test.run_duration=\"600\" \
+	  sleep inf | \
+          flux python ${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py $userid \
+            >job.signed
+        FLUX_HANDLE_USERID=$userid \
+          flux job submit --flags=signed job.signed
+}
+
+# Usage: job_manager_get_R ID
+# This needs to be a shell script since it will be run under flux-proxy(1):
+cat <<'EOF' >job_manager_get_R
+#!/bin/sh
+flux python -c \
+"import flux; \
+ payload = {\"id\":$(flux job id $1),\"attrs\":[\"R\"]}; \
+ print(flux.Flux().rpc(\"job-manager.getattr\", payload).get_str()) \
+"
+EOF
+chmod +x job_manager_get_R
+
+export PATH=$(pwd):$PATH
+
+# some tests grep for job id via flux proxy.  To ensure job id is consistent
+# across all tests and instances, set FLUX_F58_FORCE_ASCII to true.
+export FLUX_F58_FORCE_ASCII=1
+
+job_manager_get_expiration() {
+	job_manager_get_R $1 | jq .R.execution.expiration
+}
+job_manager_get_starttime() {
+	job_manager_get_R $1 | jq .R.execution.starttime
+}
+
+test_expect_success 'configure testexec to allow guest access' '
+	flux config load <<-EOF
+	[exec.testexec]
+	allow-guests = true
+	EOF
+'
+test_expect_success 'instance owner can adjust expiration of their own job' '
+	jobid=$(flux submit --wait-event=start -t5m sleep 300) &&
+	expiration=$(job_manager_get_expiration $jobid) &&
+	test_debug "echo expiration=$expiration" &&
+	flux update $jobid duration=+1m &&
+	job_manager_get_expiration $jobid | jq -e ". == $expiration + 60"
+'
+test_expect_success 'duration update with expiration in past fails' '
+	test_must_fail flux update $jobid duration=10ms 2>err.log &&
+	grep past err.log
+'
+#  Test that job execution systems sees an expiration update by
+#  waiting for an expected log message from the job-exec module
+#
+test_expect_success 'duration update is processed by execution system' '
+	flux update $jobid duration=+1m &&
+	$SHARNESS_TEST_SRCDIR/scripts/dmesg-grep.py \
+		-vt 30 "job-exec.*updated expiration of $jobid" &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean
+'
+#  Test that the job shell correctly processes an expiration update.
+#  Set up the job shell to send SIGTERM to the job 60s before expiration.
+#  for a job with a 5m duration. Then adjust duration such that expiration
+#  will occur 30s from now, and ensure the job shell picks up the update and
+#  sends SIGTERM immediately.
+#
+test_expect_success 'duration update is processed by job shell' '
+	jobid=$(flux submit --wait-event start \
+		-o verbose --signal=SIGTERM@60 -t5 sleep 300) &&
+	duration=$(job_manager_get_starttime $jobid | jq "now - . + 30") &&
+	flux update $jobid duration=$duration &&
+	test_must_fail_or_be_terminated \
+		flux job attach $jobid >shell.log 2>&1 &&
+	test_debug "cat shell.log" &&
+	grep "Will send SIGTERM to job in 0\.0" shell.log &&
+	grep "sending SIGTERM" shell.log
+'
+test_expect_success 'duration can be set to unlimited for a running job' '
+	jobid=$(flux submit --wait-event=start -t5m sleep 300) &&
+	expiration=$(job_manager_get_expiration $jobid) &&
+	test_debug "echo expiration=$expiration" &&
+	flux update $jobid duration=0 &&
+	job_manager_get_expiration $jobid | jq -e ". == 0." &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean
+'
+test_expect_success HAVE_FLUX_SECURITY 'duration update of running job is denied for guest' '
+	jobid=$(submit_job_as_guest 5m) &&
+	flux job wait-event -vt 30 $jobid start &&
+	test_must_fail runas_guest flux update $jobid duration=+1m &&
+	flux cancel $jobid
+'
+test_expect_success HAVE_FLUX_SECURITY 'duration update of running guest job is allowed for owner' '
+	jobid=$(submit_job_as_guest 5m) &&
+	flux job wait-event -vt 30 $jobid start &&
+	expiration=$(job_manager_get_expiration $jobid) &&
+	flux update $jobid duration=+1m &&
+	job_manager_get_expiration $jobid | jq -e ". == $expiration + 60" &&
+	flux cancel $jobid
+'
+#  Set module debug flag 0x4 on sched-simple so it will deny sched.expiration
+#  RPCs:
+test_expect_success 'expiration update can be denied by scheduler' '
+	flux module debug --set 4 sched-simple &&
+	jobid=$(flux submit --wait-event=start -t 5m sleep 300) &&
+	test_must_fail flux update $jobid duration=+10m >sched-deny.out 2>&1 &&
+	test_debug "cat sched-deny.out" &&
+	grep "scheduler refused" sched-deny.out &&
+	flux cancel $jobid &&
+	flux module debug --clear sched-simple
+'
+#  Set module debug flag on job-exec so it will respond with error to the
+#  exec.expiration RPC from job manager:
+test_expect_success 'failure in exec.expiration RPC results in nonfatal job exception' '
+	flux module debug --set 1 job-exec &&
+	jobid=$(flux submit --wait-event=start -t 5m sleep 300) &&
+	flux update $jobid duration=1m &&
+	flux job wait-event -vt 30 -m severity=1 $jobid exception &&
+	flux cancel $jobid
+'
+subinstance_get_R() {
+	flux proxy $1 flux kvs get resource.R
+}
+subinstance_get_expiration() {
+	subinstance_get_R $1 | jq .execution.expiration
+}
+# Usage: subinstance_get_duration ID JOBID
+subinstance_get_job_duration() {
+	flux proxy $1 job_manager_get_R $2 |
+		jq '.R.execution | .expiration - .starttime'
+}
+subinstance_get_job_expiration() {
+	flux proxy $1 job_manager_get_R $2 | jq '.R.execution.expiration'
+}
+
+#
+#  The following tests ensure that an expiration modification of a job
+#  is propagated to child jobs when appropriate. The tests may be somewhat
+#  difficult to follow, so are summarized in comments:
+#
+# 1. submit an instance with 5m time limit
+# 2. submit a job in that instance with no duration - its expiration should
+#    be set to that of the instance. (duration <5m)
+# 3. Submit a job with a set duration for testing (duration 4m)
+# 4. update duration of instance to 10m (+5m)
+# 5. wait for instance resource module to post resource-update event
+#    which indicates the expiration update is reflected in resource.R.
+# 6. wait for sched-simple to register expiration update by checking
+#    for expected log message
+# 7. Ensure subinstance now reflects updated expiration in resource.R.
+# 8. Ensure new expiration is 300s greater than old expiration
+#
+test_expect_success 'expiration update is detected by subinstance' '
+	id=$(flux alloc --bg -t5m -n4) &&
+	exp1=$(subinstance_get_expiration $id) &&
+	test_debug "echo instance expiration is $exp1" &&
+	id1=$(flux proxy $id flux submit sleep 300) &&
+	id2=$(flux proxy $id flux submit -t4m sleep 300) &&
+	tleft1=$(flux proxy $id flux job timeleft $id1) &&
+	duration1=$(subinstance_get_job_duration $id $id1) &&
+	duration2=$(subinstance_get_job_duration $id $id2) &&
+	test_debug "echo initial duration of job1 is $duration1" &&
+	test_debug "echo initial timeleft of job1 is $tleft1" &&
+	echo $duration1 | jq -e ". < 300" &&
+	echo $duration2 | jq -e ". == 240" &&
+	test_debug "echo updating duration of alloc job +5m" &&
+	flux update $id duration=+5m &&
+	test_debug "echo waiting for resource-update event" &&
+	flux proxy $id flux kvs eventlog wait-event -vt 30 \
+		resource.eventlog resource-update &&
+	test_debug "echo waiting for scheduler to see updated expiration" &&
+	flux proxy $id $SHARNESS_TEST_SRCDIR/scripts/dmesg-grep.py \
+		-vt 30 \"sched-simple.*expiration updated\" &&
+	exp2=$(subinstance_get_expiration $id) &&
+	subinstance_get_R $id &&
+	test_debug "echo expiration updated from $exp1 to $exp2" &&
+	echo $exp2 | jq -e ". == $exp1 + 300"
+'
+#
+#  9. Submit job to previous instance and ensure its expiration matches
+#     the updated value
+# 10. Ensure flux-job timeleft returns > 5m for the new job
+# 11. Wait for expected resource-update event to be propagated to the first job
+# 12. Ensure the timeleft of the first job is now > 5m
+#
+test_expect_success 'instance expiration update propagates to jobs' '
+	id3=$(flux proxy $id flux submit --wait-event=start sleep 300) &&
+	tleft3=$(flux proxy $id flux job timeleft $id3) &&
+	duration3=$(subinstance_get_job_duration $id $id3) &&
+	test_debug "echo timeleft of job submitted after update is $tleft3" &&
+	test_debug "echo duration of job submitted after update is $duration3" &&
+	echo $duration3 | jq -e ". > 300" &&
+	flux proxy $id flux job wait-event -vt 20 \
+		$id1 resource-update &&
+	tleft1=$(flux proxy $id flux job timeleft $id1) &&
+	duration1=$(subinstance_get_job_duration $id $id1) &&
+	test_debug "echo timeleft of job submitted extended to $tleft1" &&
+	test_debug "echo duration of job submitted extended to $duration1" &&
+	echo $duration1 | jq -e ". > 300"
+'
+#
+# 13. Check that expiration of job submitted with a duration is untouched.
+#
+test_expect_success 'instance expiration is not propagated to jobs with duration' '
+	tleft2=$(flux proxy $id flux job timeleft $id2) &&
+	duration2=$(subinstance_get_job_duration $id $id2) &&
+	test_debug "echo timeleft of job with duration is now $tleft2" &&
+	test_debug "echo duration of same job is now $duration2" &&
+	echo $duration2 | jq -e ". == 240"
+'
+#
+# 14. Now update job expiration to unlimited and ensure jobs have unlimited
+#     duration as well:
+#
+test_expect_success 'instance expiration can be updated to unlimited' '
+	test_debug "echo updating instance duration to inf" &&
+	flux update $id duration=inf &&
+	test_debug "echo waiting for second job resource-update event" &&
+	flux proxy $id flux job wait-event -c 2 -vt 20 \
+		$id1 resource-update &&
+	exp=$(subinstance_get_job_expiration $id $id1) &&
+	test_debug "echo job1 expiration is now $exp" &&
+	echo $exp | jq -e ". == 0" &&
+	flux proxy $id $SHARNESS_TEST_SRCDIR/scripts/dmesg-grep.py \
+		-vt 30 \"job-exec.*updated expiration of $id1 to 0\.0\"
+'
+#
+# 15. Now update job expiration to 1h and ensure job expiration is reduced
+#
+test_expect_success 'instance expiration can be decreased from unlimited' '
+	test_debug "echo updating instance duration to 1h" &&
+	flux update $id duration=1h &&
+	test_debug "echo waiting for third job resource-update event" &&
+	flux proxy $id flux job wait-event -c 3 -vt 20 \
+		$id1 resource-update &&
+	duration=$(subinstance_get_job_duration $id $id1) &&
+	test_debug "echo job1 duration is now $duration" &&
+	echo $duration | jq -e ". < 3600 and . > 0" &&
+	flux proxy $id $SHARNESS_TEST_SRCDIR/scripts/dmesg-grep.py \
+		-vt 30 \"job-manager.*expiration of $id1.*-inf\"
+'
+#
+# 16. Now update job expiration to 30m. Job expiration should be reduced again.
+#
+test_expect_success 'instance expiration can be decreased again' '
+	test_debug "echo updating instance duration to 30m" &&
+	flux update $id duration=-.5h &&
+	test_debug "echo waiting for fourth job resource-update event" &&
+	flux proxy $id flux job wait-event -c 4 -vt 20 \
+		$id1 resource-update &&
+	duration=$(subinstance_get_job_duration $id $id1) &&
+	test_debug "echo job1 duration is now $duration" &&
+	echo $duration | jq -e ". < 1800 and . > 0" &&
+	flux proxy $id $SHARNESS_TEST_SRCDIR/scripts/dmesg-grep.py \
+		-vt 30 \"job-manager.*expiration of $id1.*-1800\"
+'
+test_expect_success 'shutdown test instance' '
+	flux proxy $id flux cancel --all &&
+	flux shutdown --quiet $id &&
+	flux job wait-event $id clean
+'
+test_done
diff --git a/t/t2300-sched-simple.t b/t/t2300-sched-simple.t
index 52c622167cd4..6b63548de6b9 100755
--- a/t/t2300-sched-simple.t
+++ b/t/t2300-sched-simple.t
@@ -8,14 +8,15 @@ test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
 
 test_under_flux 4 job
 
-query=${FLUX_BUILD_DIR}/src/modules/sched-simple/rlist-query
+query="flux resource list --state=free -no {rlist}"
 
-hwloc_by_rank='{"0-1": {"Core": 2, "cpuset": "0-1"}}'
-hwloc_by_rank_first_fit='{"0": {"Core": 2}, "1": {"Core": 1}}'
+flux R encode -r0-1 -c0-1 >R.test
 
+(flux R encode -r0 -c0-1 && flux R encode -r1 -c0) | flux R append \
+	>R.test.first_fit
 
-SCHEMA=${FLUX_SOURCE_DIR}/src/modules/job-ingest/schemas/jobspec.jsonschema
-JSONSCHEMA_VALIDATOR=${FLUX_SOURCE_DIR}/src/modules/job-ingest/validators/validate-schema.py
+
+dmesg_grep=${SHARNESS_TEST_SRCDIR}/scripts/dmesg-grep.py
 
 kvs_job_dir() {
 	flux job id --to=kvs $1
@@ -29,40 +30,49 @@ list_R() {
 test_expect_success 'unload job-exec module to prevent job execution' '
 	flux module remove job-exec
 '
-test_expect_success 'sched-simple: reload ingest module with lax validator' '
-	flux exec -r all flux module reload job-ingest validator-args="--schema,${SCHEMA}" \
-         validator=${JSONSCHEMA_VALIDATOR}
+test_expect_success 'reload ingest without validator' '
+	flux module reload -f job-ingest disable-validator
 '
 test_expect_success 'sched-simple: generate jobspec for simple test job' '
-        flux jobspec srun -n1 hostname >basic.json
+	flux run --dry-run hostname >basic.json
 '
-test_expect_success 'sched-simple: load default by_rank' '
-	flux kvs put resource.hwloc.by_rank="$(echo $hwloc_by_rank)" &&
-	flux kvs get resource.hwloc.by_rank
+test_expect_success 'sched-simple cannot be loaded again under a new name' '
+	test_must_fail flux module load --name=newsched sched-simple
 '
-test_expect_success 'sched-simple: reload sched-simple' '
-	flux module reload sched-simple &&
-	flux dmesg 2>&1 | grep "ready:.*rank\[0-1\]/core\[0-1\]" &&
+test_expect_success 'job-manager: load sched-simple w/ an illegal mode' '
+	flux module unload sched-simple &&
+	flux module load sched-simple mode=foobar
+'
+test_expect_success 'job-manager: load sched-simple w/ an illegal limited range' '
+	flux module unload sched-simple &&
+	flux module load sched-simple mode=limited=-1
+'
+test_expect_success 'sched-simple: reload sched-simple with default resource.R' '
+	flux module unload sched-simple &&
+	flux resource reload R.test &&
+	flux module load sched-simple &&
+	test_debug "echo result=\"$($query)\"" &&
 	test "$($query)" = "rank[0-1]/core[0-1]"
 '
 test_expect_success 'sched-simple: unsatisfiable request is canceled' '
-	flux jobspec srun -c 3 hostname | flux job submit >job0.id &&
+	flux submit -n1 -c 3 hostname >job0.id &&
 	job0id=$(cat job0.id) &&
-        flux job wait-event --timeout=5.0 $job0id exception &&
+	flux job wait-event --timeout=5.0 $job0id exception &&
 	flux job eventlog $job0id | grep "unsatisfiable request"
 '
 test_expect_success 'sched-simple: gpu request is canceled' '
-	jobid=$(flux mini run -n1 -g1 --dry-run hostname | flux job submit) &&
+	jobid=$(flux run -n1 -g1 --dry-run hostname | flux job submit) &&
 	flux job wait-event --timeout=5.0 $jobid exception &&
-	flux job eventlog $jobid | grep  "Unsupported resource type .gpu."
+	flux job eventlog $jobid \
+		| grep  "sched-simple does not support resource type .gpu."
 '
 Y2J="flux python ${SHARNESS_TEST_SRCDIR}/jobspec/y2j.py"
 SPEC=${SHARNESS_TEST_SRCDIR}/jobspec/valid/basic.yaml
 test_expect_success 'sched-simple: invalid minimal jobspec is canceled' '
-	${Y2J}<${SPEC} | flux job submit >job00.id &&
+	${Y2J}<${SPEC} | jq ".version = 1" | flux job submit >job00.id &&
 	jobid=$(cat job00.id) &&
-        flux job wait-event --timeout=5.0 $jobid exception &&
-	flux job eventlog $jobid | grep "Unable to determine slot size"
+	flux job wait-event --timeout=5.0 $jobid exception &&
+	flux job eventlog $jobid | grep "getting duration: Object item not found: system"
 '
 test_expect_success 'sched-simple: submit 5 jobs' '
 	flux job submit basic.json >job1.id &&
@@ -70,16 +80,16 @@ test_expect_success 'sched-simple: submit 5 jobs' '
 	flux job submit basic.json >job3.id &&
 	flux job submit basic.json >job4.id &&
 	flux job submit basic.json >job5.id &&
-        flux job wait-event --timeout=5.0 $(cat job4.id) alloc &&
-        flux job wait-event --timeout=5.0 $(cat job5.id) submit
+	flux job wait-event --timeout=5.0 $(cat job4.id) alloc &&
+	flux job wait-event --timeout=5.0 $(cat job5.id) submit
 '
 test_expect_success 'sched-simple: check allocations for running jobs' '
 	list_R $(cat job1.id job2.id job3.id job4.id) > allocs.out &&
 	cat <<-EOF >allocs.expected &&
-	note="rank0/core0"
-	note="rank1/core0"
-	note="rank0/core1"
-	note="rank1/core1"
+	annotations={"sched":{"resource_summary":"rank0/core0"}}
+	annotations={"sched":{"resource_summary":"rank1/core0"}}
+	annotations={"sched":{"resource_summary":"rank0/core1"}}
+	annotations={"sched":{"resource_summary":"rank1/core1"}}
 	EOF
 	test_cmp allocs.expected allocs.out
 '
@@ -87,25 +97,25 @@ test_expect_success 'sched-simple: no remaining resources' '
 	test "$($query)" = ""
 '
 test_expect_success 'sched-simple: cancel one job' '
-	flux job cancel $(cat job3.id) &&
-        flux job wait-event --timeout=5.0 $(cat job3.id) exception &&
-        flux job wait-event --timeout=5.0 $(cat job3.id) free
+	flux cancel $(cat job3.id) &&
+	flux job wait-event --timeout=5.0 $(cat job3.id) exception &&
+	flux job wait-event --timeout=5.0 $(cat job3.id) free
 '
 test_expect_success 'sched-simple: waiting job now has alloc event' '
-        flux job wait-event --timeout=5.0 $(cat job5.id) alloc &&
-	test "$(list_R $(cat job5.id))" = "note=\"rank0/core1\""
+	flux job wait-event --timeout=5.0 $(cat job5.id) alloc &&
+	test "$(list_R $(cat job5.id))" = "annotations={\"sched\":{\"resource_summary\":\"rank0/core1\"}}"
 '
 test_expect_success 'sched-simple: cancel all jobs' '
-	flux job cancel $(cat job5.id) &&
-	flux job cancel $(cat job4.id) &&
-	flux job cancel $(cat job2.id) &&
-	flux job cancel $(cat job1.id) &&
+	flux cancel $(cat job5.id) &&
+	flux cancel $(cat job4.id) &&
+	flux cancel $(cat job2.id) &&
+	flux cancel $(cat job1.id) &&
 	flux job wait-event --timeout=5.0 $(cat job1.id) exception &&
 	flux job wait-event --timeout=5.0 $(cat job1.id) free &&
 	test "$($query)" = "rank[0-1]/core[0-1]"
 '
-test_expect_success 'sched-simple: reload in best-fit mode' '
-	flux module reload sched-simple mode=best-fit
+test_expect_success 'sched-simple: reload in best-fit alloc-mode' '
+	flux module reload sched-simple alloc-mode=best-fit
 '
 test_expect_success 'sched-simple: submit 5 more jobs' '
 	flux job submit basic.json >job6.id &&
@@ -119,31 +129,32 @@ test_expect_success 'sched-simple: submit 5 more jobs' '
 test_expect_success 'sched-simple: check allocations for running jobs' '
 	list_R $(cat job6.id job7.id job8.id job9.id) > best-fit-allocs.out &&
 	cat <<-EOF >best-fit-allocs.expected &&
-	note="rank0/core0"
-	note="rank0/core1"
-	note="rank1/core0"
-	note="rank1/core1"
+	annotations={"sched":{"resource_summary":"rank0/core0"}}
+	annotations={"sched":{"resource_summary":"rank0/core1"}}
+	annotations={"sched":{"resource_summary":"rank1/core0"}}
+	annotations={"sched":{"resource_summary":"rank1/core1"}}
 	EOF
 	test_cmp best-fit-allocs.expected best-fit-allocs.out
 '
-test_expect_success 'sched-simple: cancel pending job' '
+test_expect_success 'sched-simple: cancel pending & running job' '
 	id=$(cat job10.id) &&
-	flux job cancel $id &&
+	flux cancel $id &&
 	flux job wait-event --timeout=5.0 $id exception &&
-	flux job cancel $(cat job6.id) &&
+	flux cancel $(cat job6.id) &&
 	test_expect_code 1 flux kvs get $(kvs_job_dir $id).R
 '
 test_expect_success 'sched-simple: cancel remaining jobs' '
-	flux job cancel $(cat job7.id) &&
-	flux job cancel $(cat job8.id) &&
-	flux job cancel $(cat job9.id) &&
+	flux cancel $(cat job7.id) &&
+	flux cancel $(cat job8.id) &&
+	flux cancel $(cat job9.id) &&
 	flux job wait-event --timeout=5.0 $(cat job9.id) free
 '
-test_expect_success 'sched-simple: reload in first-fit mode' '
-        flux module remove sched-simple &&
-	flux kvs put resource.hwloc.by_rank="$(echo $hwloc_by_rank_first_fit)" &&
-        flux module load sched-simple mode=first-fit &&
-	flux dmesg | grep "ready:.*rank0/core\[0-1\] rank1/core0"
+test_expect_success 'sched-simple: reload in first-fit alloc-mode' '
+	flux module remove sched-simple &&
+	flux resource reload R.test.first_fit &&
+	flux module load sched-simple alloc-mode=first-fit &&
+	test_debug "echo result=\"$($query)\"" &&
+	test "$($query)" = "rank0/core[0-1] rank1/core0"
 '
 test_expect_success 'sched-simple: submit 3 more jobs' '
 	flux job submit basic.json >job11.id &&
@@ -155,15 +166,15 @@ test_expect_success 'sched-simple: check allocations for running jobs' '
 	list_R $(cat job11.id job12.id job13.id ) \
 		 > first-fit-allocs.out &&
 	cat <<-EOF >first-fit-allocs.expected &&
-	note="rank0/core0"
-	note="rank0/core1"
-	note="rank1/core0"
+	annotations={"sched":{"resource_summary":"rank0/core0"}}
+	annotations={"sched":{"resource_summary":"rank0/core1"}}
+	annotations={"sched":{"resource_summary":"rank1/core0"}}
 	EOF
 	test_cmp first-fit-allocs.expected first-fit-allocs.out
 '
 test_expect_success 'sched-simple: reload with outstanding allocations' '
 	flux module reload sched-simple &&
-	flux dmesg | grep "hello: alloc rank0/core0" &&
+	test_debug "echo result=\"$($query)\"" &&
 	test "$($query)" = ""
 '
 test_expect_success 'sched-simple: verify three jobs are active' '
@@ -173,41 +184,80 @@ test_expect_success 'sched-simple: verify three jobs are active' '
 
 test_expect_success 'sched-simple: remove sched-simple and cancel jobs' '
 	flux module remove sched-simple &&
-	flux job cancelall -f
+	flux cancel --all
 '
 test_expect_success 'sched-simple: there are no outstanding sched requests' '
-	flux queue status -v 2>queue_status.out &&
-	grep "0 alloc requests pending to scheduler" queue_status.out &&
-	grep "0 free requests pending to scheduler" queue_status.out
+	flux queue status -v >queue_status.out &&
+	grep "0 alloc requests pending to scheduler" queue_status.out
 '
 test_expect_success 'sched-simple: reload in unlimited mode' '
-	flux module load sched-simple unlimited &&
-	flux dmesg | grep "scheduler: ready unlimited"
+	flux module load sched-simple mode=unlimited &&
+	$dmesg_grep -t 10 "scheduler: ready unlimited"
 '
-test_expect_success 'sched-simple: submit 3 more jobs' '
-	flux job submit basic.json >job11.id &&
-	flux job submit basic.json >job12.id &&
-	flux job submit basic.json >job13.id &&
-	flux job wait-event --timeout=5.0 $(cat job13.id) alloc
+test_expect_success 'sched-simple: submit 5 more jobs' '
+	flux job submit basic.json >job14.id &&
+	flux job submit basic.json >job15.id &&
+	flux job submit basic.json >job16.id &&
+	flux job submit basic.json >job17.id &&
+	flux job submit basic.json >job18.id &&
+	flux job wait-event --timeout=5.0 $(cat job16.id) alloc &&
+	flux job wait-event --timeout=5.0 $(cat job18.id) submit
 '
 test_expect_success 'sched-simple: check allocations for running jobs' '
-	list_R $(cat job11.id job12.id job13.id ) \
-		 > single-allocs.out &&
-	cat <<-EOF >first-fit-allocs.expected &&
-	note="rank0/core0"
-	note="rank0/core1"
-	note="rank1/core0"
+	list_R $(cat job14.id job15.id job16.id ) \
+		 > unlimited-allocs.out &&
+	cat <<-EOF >unlimited-allocs.expected &&
+	annotations={"sched":{"resource_summary":"rank0/core0"}}
+	annotations={"sched":{"resource_summary":"rank0/core1"}}
+	annotations={"sched":{"resource_summary":"rank1/core0"}}
 	EOF
-	test_cmp first-fit-allocs.expected first-fit-allocs.out
+	test_cmp unlimited-allocs.expected unlimited-allocs.out
+'
+test_expect_success 'sched-simple: update urgency of job' '
+	flux job urgency $(cat job18.id) 20
+'
+test_expect_success 'sched-simple: cancel running job' '
+	flux cancel $(cat job14.id) &&
+	flux job wait-event --timeout=5.0 $(cat job14.id) free
+'
+test_expect_success 'sched-simple: ensure more urgent job run' '
+	list_R $(cat job18.id job15.id job16.id) \
+		 > unlimited-allocs2.out &&
+	cat <<-EOF >unlimited-allocs2.expected &&
+	annotations={"sched":{"resource_summary":"rank0/core0"}}
+	annotations={"sched":{"resource_summary":"rank0/core1"}}
+	annotations={"sched":{"resource_summary":"rank1/core0"}}
+	EOF
+	test_cmp unlimited-allocs2.expected unlimited-allocs2.out
+'
+# cancel all jobs, to ensure no interference with follow up tests
+# cancel non-running jobs first to ensure they are not accidentally run
+test_expect_success 'sched-simple: cancel jobs' '
+	flux cancel --all --states=pending &&
+	flux cancel --all &&
+	flux job wait-event --timeout=5.0 $(cat job18.id) free &&
+	flux job wait-event --timeout=5.0 $(cat job15.id) free &&
+	flux job wait-event --timeout=5.0 $(cat job16.id) free
+'
+test_expect_success 'sched-simple: reload sched-simple to cover free flags' '
+	flux module reload sched-simple test-free-nolookup
+'
+# That SCHEDUTIL_FREE_NOLOOKUP is now a no-op but since flux-sched-0.33.0
+# still uses it, ensure that free still works when it is used
+test_expect_success 'sched-simple: submit job and cancel it' '
+	flux dmesg --clear &&
+	flux job submit basic.json >job19.id &&
+	flux job wait-event --timeout=5.0 $(cat job19.id) alloc &&
+	flux cancel $(cat job19.id) &&
+	$dmesg_grep -t 10 "free: rank0/core0"
 '
 test_expect_success 'sched-simple: remove sched-simple and cancel jobs' '
 	flux module remove sched-simple &&
-	flux job cancelall -f
+	flux cancel --all
 '
 test_expect_success 'sched-simple: there are no outstanding sched requests' '
-	flux queue status -v 2>queue_status.out &&
-	grep "0 alloc requests pending to scheduler" queue_status.out &&
-	grep "0 free requests pending to scheduler" queue_status.out
+	flux queue status -v >queue_status.out &&
+	grep "0 alloc requests pending to scheduler" queue_status.out
 '
 
 test_expect_success 'sched-simple: load sched-simple and wait for queue drain' '
diff --git a/t/t2301-schedutil-outstanding-requests.t b/t/t2301-schedutil-outstanding-requests.t
deleted file mode 100755
index 81b850a72631..000000000000
--- a/t/t2301-schedutil-outstanding-requests.t
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/sh
-
-test_description='check for responses from sched to outstanding requests'
-
-. $(dirname $0)/sharness.sh
-
-test_under_flux 1 job
-
-RPC=${FLUX_BUILD_DIR}/t/request/rpc
-SCHED_DUMMY=${FLUX_BUILD_DIR}/t/job-manager/.libs/sched-dummy.so
-REQ_AND_UNLOAD="flux python ${SHARNESS_TEST_SRCDIR}/schedutil/req_and_unload.py"
-
-test_expect_success 'schedutil: remove sched-simple,job-exec modules' '
-	flux module remove sched-simple &&
-	flux module remove job-exec
-'
-
-test_expect_success 'schedutil: load sched-dummy --cores=2' '
-	flux module load ${SCHED_DUMMY} --cores=2
-'
-
-test_expect_success 'schedutil: set bit to hang alloc/free requests' '
-	flux module debug --setbit 0x8000 sched-dummy
-'
-
-test_expect_success 'schedutil: alloc/free fail with ENOSYS(38) after unload' '
-    ${REQ_AND_UNLOAD} sched-dummy
-'
-
-test_expect_success 'schedutil: successfully loaded sched-simple' '
-	flux module load sched-simple
-'
-
-test_expect_success 'schedutil: set bit to hang alloc/free requests' '
-	flux module debug --setbit 0x8000 sched-simple
-'
-
-test_expect_success 'schedutil: alloc/free fail with ENOSYS(38) after unload' '
-    ${REQ_AND_UNLOAD} sched-simple
-'
-
-test_done
diff --git a/t/t2302-sched-simple-up-down.t b/t/t2302-sched-simple-up-down.t
new file mode 100755
index 000000000000..554087e7ac5c
--- /dev/null
+++ b/t/t2302-sched-simple-up-down.t
@@ -0,0 +1,88 @@
+#!/bin/sh
+
+test_description='sched-simple up/down/drain tests'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 job
+
+query="flux resource list --state=free -no {rlist}"
+
+flux R encode -r0-1 -c0-3 >R.test
+
+check_resource()
+{
+	local name="$1"
+	local state="$2"
+	local expected="$3"
+	local result="$(flux resource list --state=$state -no {$name})"
+	echo "# check $name in $state is $expected (got $result)"
+	test "$result" = "$expected"
+}
+check_ncores() { check_resource "ncores" "$@"; }
+check_nnodes() { check_resource "nnodes" "$@"; }
+check_rlist() { check_resource "rlist" "$@"; }
+
+test_expect_success 'unload job-exec module to prevent job execution' '
+	flux module remove job-exec
+'
+test_expect_success 'sched-simple: reload ingest module without validator' '
+	flux module reload job-ingest disable-validator  &&
+	flux exec -r all -x 0 flux module reload job-ingest disable-validator
+'
+
+test_expect_success 'sched-simple: reload sched-simple with default resource.R' '
+	flux module unload sched-simple &&
+	flux resource reload R.test &&
+	flux module load sched-simple &&
+	flux dmesg 2>&1 >reload.dmesg.log &&
+	test_debug "grep sched-simple reload.dmesg.log" &&
+	grep "ready:.*rank\[0-1\]/core\[0-3\]" reload.dmesg.log &&
+	test_debug "echo result=\"$($query)\"" &&
+	test "$($query)" = "rank[0-1]/core[0-3]"
+'
+test_expect_success 'sched-simple: all nodes up after reload' '
+	test_debug "flux resource list --format=rlist --states=up,down,allocated" &&
+	check_rlist "free" "rank[0-1]/core[0-3]" &&
+	check_nnodes "up" 2 &&
+	check_ncores "up" 8 &&
+	check_ncores "down" 0
+'
+test_expect_success 'sched-simple: flux-resource drain works' '
+	flux resource drain 0 &&
+	check_rlist "up" "rank1/core[0-3]" &&
+	check_nnodes "up" 1 &&
+	check_ncores "up" 4 &&
+	check_rlist "down" "rank0/core[0-3]"
+'
+test_expect_success 'sched-simple: job not allocated drained resources' '
+	id=$(flux submit -n1 hostname) &&
+	flux job wait-event $id alloc &&
+	check_rlist "allocated" "rank1/core0" &&
+	check_rlist "free" "rank1/core[1-3]" &&
+	check_ncores "allocated" 1 &&
+	check_ncores "free" 3
+'
+test_expect_success 'sched-simple: drain allocated resources works' '
+	flux resource drain 1 &&
+	check_rlist "down" "rank[0-1]/core[0-3]" &&
+	check_ncores "up" 0
+'
+test_expect_success 'sched-simple: down+alloc resources listed as allocated' '
+	check_rlist "allocated" "rank1/core0"
+'
+test_expect_success 'sched-simple: submitted job blocks with no up resources' '
+	id=$(flux submit -n1 hostname) &&
+	test_expect_code 1 flux job wait-event -t 0.25 $id alloc &&
+	check_ncores "allocated" 1
+'
+test_expect_success 'sched-simple: job is scheduled when resource undrained' '
+	flux resource undrain 0 &&
+	flux job wait-event -t5 $id alloc &&
+	check_rlist "allocated" "rank[0-1]/core0" &&
+	check_nnodes "allocated" 2 &&
+	check_ncores "allocated" 2
+'
+test_done
diff --git a/t/t2303-sched-hello.t b/t/t2303-sched-hello.t
new file mode 100755
index 000000000000..fa8e783f3799
--- /dev/null
+++ b/t/t2303-sched-hello.t
@@ -0,0 +1,98 @@
+#!/bin/sh
+
+test_description='test sched hello
+
+Ensure that a job that the scheduler rejects during hello processing
+receives a fatal exception.
+'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+test_expect_success 'start a long-running job' '
+	jobid=$(flux submit -N1 sleep inf)
+'
+test_expect_success 'unload scheduler' '
+	flux module remove sched-simple &&
+	flux module remove resource
+'
+test_expect_success 'exclude the job node from configuration' '
+	flux config load <<-EOT
+	[resource]
+	exclude = "0"
+	EOT
+'
+test_expect_success 'increase broker stderr log level' '
+	flux setattr log-stderr-level 6
+'
+test_expect_success 'load scheduler' '
+	flux module load resource &&
+	flux module load sched-simple
+'
+test_expect_success 'job receives exception' '
+	flux job wait-event -t 30 ${jobid} exception
+'
+test_expect_success 'job receives clean event' '
+	flux job wait-event -v -t 30 ${jobid} clean
+'
+test_expect_success 'unload scheduler' '
+	flux module remove sched-simple &&
+	flux module remove resource
+'
+test_expect_success 'restore the empty config' '
+	flux config load </dev/null
+'
+test_expect_success 'load scheduler' '
+	flux module load resource &&
+	flux module load sched-simple
+'
+
+#
+# The following ensures that the alloc-check jobtap plugin properly accounts
+# for resources "freed" by hello failure.
+#
+test_expect_success 'load the alloc-check plugin' '
+	flux jobtap load alloc-check.so
+'
+test_expect_success 'start a long-running job' '
+	jobid=$(flux submit -N1 sleep inf)
+'
+test_expect_success 'unload scheduler' '
+	flux module remove sched-simple &&
+	flux module remove resource
+'
+test_expect_success 'exclude the job node from configuration' '
+	flux config load <<-EOT
+	[resource]
+	exclude = "0"
+	EOT
+'
+test_expect_success 'load scheduler' '
+	flux module load resource &&
+	flux module load sched-simple
+'
+test_expect_success 'job receives exception' '
+	flux job wait-event -t 30 ${jobid} exception
+'
+test_expect_success 'unload scheduler' '
+	flux module remove sched-simple &&
+	flux module remove resource
+'
+test_expect_success 'restore the empty config' '
+	flux config load </dev/null
+'
+test_expect_success 'load scheduler' '
+	flux module load resource &&
+	flux module load sched-simple
+'
+test_expect_success 'run another job that uses the same resources' '
+	flux run -vv -N1 true
+'
+test_expect_success 'decrease broker stderr log level' '
+	flux setattr log-stderr-level 5
+'
+
+test_done
diff --git a/t/t2304-sched-simple-alloc-check.t b/t/t2304-sched-simple-alloc-check.t
new file mode 100755
index 000000000000..1f76aa64df07
--- /dev/null
+++ b/t/t2304-sched-simple-alloc-check.t
@@ -0,0 +1,91 @@
+#!/bin/sh
+
+test_description='check that sched-simple never double books resources'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+# Verify that alloc-check plugin works using alloc-bypass
+test_expect_success 'load alloc-bypass and alloc-check plugins' '
+	flux jobtap load alloc-bypass.so &&
+	flux jobtap load alloc-check.so
+'
+test_expect_success 'run an alloc-bypass sleep job' '
+	flux submit \
+	    -vvv \
+	    --wait-event=start \
+	    --setattr=alloc-bypass.R="$(flux R encode -r 0)" \
+	    -n 1 \
+	    sleep inf
+'
+test_expect_success 'a regular job fails with an alloc-check exception' '
+	test_expect_code 1 \
+	    run_timeout 30 flux submit --flags=waitable -vvv \
+	    --wait-event=exception \
+	    -N1 true >bypass.jobid
+'
+test_expect_success 'flux job wait says the job failed' '
+	test_must_fail flux job wait -v $(cat bypass.jobid)
+'
+test_expect_success 'flux job attach says the job failed' '
+	test_must_fail flux job attach -vE $(cat bypass.jobid)
+'
+test_expect_success 'flux job status says the job failed' '
+	test_must_fail flux job status -v $(cat bypass.jobid)
+'
+test_expect_success 'flux jobs says the job failed' '
+	flux job list-ids --wait-state=inactive $(cat bypass.jobid) >/dev/null &&
+	flux jobs -no {result} $(cat bypass.jobid) > bypass.result &&
+	test_debug "cat bypass.result" &&
+	test "$(cat bypass.result)" = "FAILED"
+'
+test_expect_success 'clean up jobs' '
+	flux cancel --all &&
+	flux queue drain
+'
+test_expect_success 'unload plugins' '
+	flux jobtap remove alloc-check.so &&
+	flux jobtap remove alloc-bypass.so
+'
+
+# Check that sched-simple doesn't suffer from time limit issue like
+#   flux-framework/flux-sched#1043
+#
+test_expect_success 'configure epilog with 2s delay' '
+	flux config load <<-EOT &&
+	[job-manager.epilog]
+	per-rank = true
+	command = [ "sleep", "2" ]
+	EOT
+	flux jobtap load perilog.so
+'
+test_expect_success 'load alloc-check plugin' '
+	flux jobtap load alloc-check.so
+'
+test_expect_success 'submit consecutive jobs that exceed their time limit' '
+	(for i in $(seq 3); \
+	    do flux run -N1 -x -t1s sleep 30 || true; \
+	done) 2>joberr
+'
+test_expect_success 'some jobs received timeout exception' '
+	grep "job.exception" joberr | grep "type=timeout"
+'
+test_expect_success 'no jobs received alloc-check exception' '
+	test_must_fail grep "job.exception type=alloc-check" joberr
+'
+test_expect_success 'clean up jobs' '
+	flux cancel --all &&
+	flux queue drain
+'
+test_expect_success 'remove alloc-check plugin' '
+	flux jobtap remove alloc-check.so
+'
+test_expect_success 'undo epilog config' '
+	flux jobtap remove perilog.so &&
+	flux config load </dev/null
+'
+
+test_done
diff --git a/t/t2305-sched-slow.t b/t/t2305-sched-slow.t
new file mode 100755
index 000000000000..15ef47ecb772
--- /dev/null
+++ b/t/t2305-sched-slow.t
@@ -0,0 +1,84 @@
+#!/bin/sh
+
+test_description='test a scheduler that is slow to respond
+'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+# Usage: module_debug_defer modname True|False
+module_debug_defer () {
+	flux python -c "import flux; flux.Flux().rpc(\"module.debug\",{\"name\":\"$1\",\"defer\":$2}).get()"
+}
+
+test_expect_success 'pause sched message handling' '
+	module_debug_defer sched-simple True
+'
+test_expect_success 'a job can be submitted when the scheduler is unresponsive' '
+	flux submit -N1 \
+	    --flags=debug --wait-event=debug.alloc-request \
+	    true >job1.id
+'
+test_expect_success 'the job is blocked on the alloc request' '
+	test_expect_code 137 run_timeout 2 \
+	    flux job wait-event $(cat job1.id) alloc
+'
+test_expect_success 'unpause sched message handling' '
+	module_debug_defer sched-simple False
+'
+test_expect_success 'the job gets its allocation and completes' '
+	run_timeout 30 flux job wait-event $(cat job1.id) clean
+'
+test_expect_success 'submit a job and wait for it to get an alloc response' '
+	flux submit -N1 --wait-event=alloc sleep 3600 >job2.id
+'
+test_expect_success 'pause sched message handling' '
+	module_debug_defer sched-simple True
+'
+test_expect_success 'cancel the job' '
+	flux cancel $(cat job2.id)
+'
+test_expect_success 'the job is able to get through CLEANUP state' '
+	run_timeout 30 flux job wait-event $(cat job2.id) clean
+'
+test_expect_success 'unpause sched message handling' '
+	module_debug_defer sched-simple False
+'
+test_expect_success 'another -N1 job can run so resources are free' '
+	run_timeout 30 flux run -N1 true
+'
+test_expect_success 'drain the only node in the instance' '
+	flux resource drain 0
+'
+test_expect_success 'submit a job' '
+	flux submit -N1 \
+	    --flags=debug --wait-event=debug.alloc-request \
+	    true >job3.id
+'
+test_expect_success 'the job is blocked on the alloc request' '
+	test_expect_code 137 run_timeout 2 \
+	    flux job wait-event $(cat job3.id) alloc
+'
+test_expect_success 'pause sched message handling' '
+	module_debug_defer sched-simple True
+'
+test_expect_success 'cancel the job' '
+	flux cancel $(cat job3.id)
+'
+test_expect_success 'the job is able to get through CLEANUP state' '
+	run_timeout 30 flux job wait-event $(cat job3.id) clean
+'
+test_expect_success 'unpause sched message handling' '
+	module_debug_defer sched-simple False
+'
+test_expect_success 'undrain the node' '
+	flux resource undrain 0
+'
+test_expect_success 'another -N1 job can run so everything still works!' '
+	run_timeout 30 flux run -N1 true
+'
+
+test_done
diff --git a/t/t2310-resource-module.t b/t/t2310-resource-module.t
new file mode 100755
index 000000000000..57224d30f668
--- /dev/null
+++ b/t/t2310-resource-module.t
@@ -0,0 +1,211 @@
+#!/bin/sh
+
+test_description='Test resource module'
+
+. `dirname $0`/sharness.sh
+
+# Start out with empty config object
+# Then we will reload after adding TOML to cwd
+export FLUX_CONF_DIR=$(pwd)
+
+SIZE=4
+test_under_flux $SIZE kvs
+
+# Usage: grep_event event-name <in >out
+grep_event () {
+	jq -c ". | select(.name == \"$1\") | .context"
+}
+has_event() {
+	flux kvs eventlog get resource.eventlog | awk '{ print $2 }' | grep $1
+}
+# Ensure that module is loaded upstream-to-downstream TBON order
+load_resource () {
+	for rank in $(seq 0 $(($SIZE-1))); do \
+		flux exec -r $rank flux module load resource; \
+	done
+}
+
+get_topo() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"resource.topo-get\",nodeid=$1).get_str())"
+}
+res_reload() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"resource.reload\",nodeid=$1).get())"
+}
+bad_reduce() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"resource.topo-reduce\",nodeid=$1))"
+}
+
+test_expect_success 'load resource module with bad option fails' '
+	test_must_fail flux module load resource badoption
+'
+
+#   0
+#  1 2
+# 3
+
+test_expect_success 'load resource module on ranks 0,2' '
+	flux module load resource &&
+	flux exec -r2 flux module load resource
+'
+test_expect_success 'reload module on rank 2 to trigger dup topo-reduce' '
+	flux exec -r2 flux module reload resource
+'
+test_expect_success 'load resource module on ranks 1,3' '
+	flux exec -r1 flux module load resource &&
+	flux exec -r3 flux module load resource
+'
+test_expect_success 'send bad topo-reduce request to rank 0' '
+	bad_reduce 0
+'
+
+test_expect_success 'resource.eventlog exists' '
+	flux kvs eventlog get -u resource.eventlog >eventlog.out
+'
+
+test_expect_success 'wait until resource-define event is posted' '
+	flux kvs eventlog wait-event -t 5 resource.eventlog resource-define
+'
+
+test_expect_success 'resource.R is populated after resource-define' '
+	flux kvs get resource.R
+'
+
+test_expect_success 'reload resource module and re-capture eventlog' '
+	flux module remove resource &&
+	flux kvs eventlog get -u resource.eventlog >pre_restart.out &&
+	flux module load resource &&
+	flux kvs eventlog get -u resource.eventlog >restart.out &&
+	pre=$(wc -l <pre_restart.out) &&
+	post=$(wc -l <restart.out) &&
+	tail -$(($post-$pre)) restart.out > post_restart.out
+'
+
+test_expect_success 'reconfig with extra key fails' '
+	cat >resource.toml <<-EOT &&
+	[resource]
+	foo = 42
+	EOT
+	test_must_fail flux config reload
+'
+
+test_expect_success 'reconfig with empty config' '
+	rm -f resource.toml &&
+	flux config reload
+'
+
+test_expect_success 'flux resource reload fails on rank 1' '
+	test_must_fail res_reload 1
+'
+
+test_expect_success 'flux resource reload fails on nonexistent file' '
+	test_must_fail flux resource reload /noexist
+'
+
+test_expect_success 'flux resource reload fails on nonexistent XML directory' '
+	test_must_fail flux resource reload -x /noexist
+'
+
+test_expect_success 'flux resource reload fails on empty XML directory' '
+	mkdir empty &&
+	test_must_fail flux resource reload -x $(pwd)/empty
+'
+
+sanitize_hwloc_xml() {
+    sed 's/pci_link_speed=".*"//g' $1
+}
+
+test_expect_success 'get hwloc XML direct from ranks' '
+	mkdir -p hwloc &&
+	for i in $(seq 0 $(($SIZE-1))); do \
+		get_topo $i | sanitize_hwloc_xml >hwloc/$i.xml || return 1; \
+	done
+'
+
+normalize_json() {
+	jq -cS .
+}
+
+test_expect_success 'reloading XML results in same R as before' '
+	flux kvs get resource.R | normalize_json >R.orig &&
+	flux resource reload -x hwloc &&
+	flux kvs get resource.R | normalize_json >R.new &&
+	test_cmp R.orig R.new
+'
+
+test_expect_success 'reload original R just to do it and verify' '
+	flux resource reload R.orig &&
+	flux kvs get resource.R | normalize_json >R.new2 &&
+	test_cmp R.orig R.new2
+'
+
+test_expect_success 'reload refuses to load XML beyond size' '
+	cp hwloc/$(($SIZE-1)).xml hwloc/$SIZE.xml &&
+	test_must_fail flux resource reload -x hwloc 2>toobig.err &&
+	grep "contains ranks execeeding size" toobig.err
+'
+
+test_expect_success 'the --force option makes it work' '
+	flux resource reload -f -x hwloc
+'
+
+test_expect_success 'reload allows loading XML less than size' '
+	rm -f hwloc/$SIZE.xml &&
+	rm -f hwloc/$(($SIZE-1)).xml &&
+	flux resource reload -x hwloc
+'
+
+test_expect_success 'reload resource module' '
+	flux exec -r all flux module remove resource &&
+	load_resource
+'
+
+test_expect_success 'one rank was was drained' '
+	has_event drain >has_drain.out &&
+	test $(wc -l <has_drain.out) -eq 1
+'
+
+# Since jq is used in the first bit, is required in other bits too
+test_expect_success 'change expected cores in resource.R' "
+	flux kvs get resource.R &&
+	jq '.execution.R_lite[0].children.core = \"0-1048\"' R.orig >R.Mcore &&
+	flux resource reload R.Mcore
+"
+test_expect_success 'reload resource module' '
+	flux exec -r all flux module remove resource &&
+	load_resource
+'
+test_expect_success 'all ranks were drained' '
+	has_event drain >has_drain2.out &&
+	test $(wc -l <has_drain2.out) -eq $(($SIZE+1))
+'
+
+test_expect_success 'flux resource status works on rank > 0' '
+	flux exec -r 1 flux resource status
+'
+
+status_onrank() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"resource.status\",nodeid=$1).get())"
+}
+
+test_expect_success 'resource.status RPC fails on rank > 0' '
+	test_must_fail status_onrank 1 2>status.err &&
+	grep "only works on rank 0" status.err
+'
+
+test_expect_success 'unload resource module' '
+	flux exec -r all flux module remove resource
+'
+
+test_expect_success 'reconfig with bad R path' '
+	cat >resource.toml <<-EOT &&
+	[resource]
+	path = "/noexist"
+	EOT
+	flux config reload
+'
+
+test_expect_success 'load resource module fails due to bad R' '
+	test_must_fail flux module load resource
+'
+
+test_done
diff --git a/t/t2311-resource-drain.t b/t/t2311-resource-drain.t
new file mode 100755
index 000000000000..0c9b711eb080
--- /dev/null
+++ b/t/t2311-resource-drain.t
@@ -0,0 +1,496 @@
+#!/bin/sh
+
+test_description='Test resource drain/undrain'
+
+. `dirname $0`/sharness.sh
+
+SIZE=4
+test_under_flux $SIZE full --test-hosts=fake[0-3]
+
+
+# Usage: waitup N
+#   where N is a count of online ranks
+waitup () {
+	run_timeout 5 flux python -c "import flux; print(flux.Flux().rpc(\"resource.monitor-waitup\",{\"up\":$1}).get())"
+}
+waitdown () {
+	waitup $(($SIZE-$1))
+}
+
+has_resource_event () {
+	flux kvs eventlog get resource.eventlog | awk '{ print $2 }' | grep $1
+}
+
+drain_timestamp () {
+	flux resource drain -o {timestamp} | tail -n 1 | awk '{ print $1 }'
+}
+
+test_expect_success 'wait for monitor to declare all ranks are up' '
+	waitdown 0
+'
+# fake hostnames match the ones set on the broker command line
+test_expect_success 'load fake resources' '
+	flux module remove sched-simple &&
+	flux R encode -r 0-3 -c 0-1 -H fake[0-3] >R &&
+	flux resource reload R &&
+	flux module load sched-simple
+'
+
+test_expect_success 'flux resource drain: default lists some expected fields' '
+	flux resource drain > default.out &&
+	grep STATE default.out &&
+	grep REASON default.out
+'
+
+test_expect_success 'flux resource drain: FLUX_RESOURCE_DRAIN_FORMAT_DEFAULT works' '
+	FLUX_RESOURCE_DRAIN_FORMAT_DEFAULT="{nodelist} {nodelist}" \
+		flux resource drain > default_override.out &&
+	grep "NODELIST NODELIST" default_override.out
+'
+
+test_expect_success 'flux resource drain: FLUX_RESOURCE_DRAIN_FORMAT_DEFAULT works w/ named format' '
+	FLUX_RESOURCE_DRAIN_FORMAT_DEFAULT=long \
+		flux resource drain > default_override_named.out &&
+	grep "RANKS" default_override_named.out
+'
+
+test_expect_success 'flux resource drain: --no-header works' '
+	flux resource drain --no-header > default_no_header.out &&
+	test_must_fail grep STATE default_no_header.out &&
+	test_must_fail grep REASON default_no_header.out
+'
+
+test_expect_success 'drain works with no reason' '
+	flux resource drain 1 &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 1
+'
+
+test_expect_success 'save original drain timestamp' '
+	drain_timestamp > rank1.timestamp
+'
+
+test_expect_success 'resource.eventlog has one drain event' '
+	test $(has_resource_event drain | wc -l) -eq 1 &&
+	test $(flux resource status -s drain -no {ranks}) = "1"
+'
+
+test_expect_success 'reason can be added after node is drained' '
+	flux resource drain 1 test_reason_01 &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 1 &&
+	test $(flux resource status -s drain -no {nnodes}) -eq 1
+'
+
+test_expect_success 'resource.eventlog has two drain events' '
+	test $(has_resource_event drain | wc -l) -eq 2
+'
+
+test_expect_success 'reason cannot be updated when already set' '
+	test_expect_code 1 flux resource drain 1 test_reason_fail &&
+	flux resource drain | test_must_fail grep test_reason_fail &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 1 &&
+	test $(flux resource status -s drain -no {nnodes}) -eq 1
+'
+
+test_expect_success 'drain detects subset of already drained targets' '
+	test_expect_code 1 flux resource drain 0-1 >drain-0-1.out 2>&1 &&
+	test_debug "cat drain-0-1.out" &&
+	grep "rank 1 already drained" drain-0-1.out &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 1 &&
+	test $(flux resource status -s drain -no {nnodes}) -eq 1
+'
+
+test_expect_success 'drain suggests --force with existing reason' '
+	test_must_fail flux resource drain 1 test_reason_updated \
+		>update-failed.log 2>&1 &&
+	test_debug "cat update-failed.log" &&
+	grep -i "use --force" update-failed.log
+'
+
+test_expect_success 'drain reason can be updated with --force' '
+	flux resource drain --force 1 test_reason_updated &&
+	flux resource drain | grep test_reason_updated
+'
+
+test_expect_success 'original drain timestamp is preserved' '
+	test $(drain_timestamp) = $(cat rank1.timestamp)
+'
+
+test_expect_success 'drain timestamp and reason can be updated with -ff' '
+	flux resource drain --force --force 1 test_reason_updated_again &&
+	flux resource drain | grep test_reason_updated_again
+'
+
+test_expect_success 'original drain timestamp is preserved' '
+	test $(drain_timestamp) != $(cat rank1.timestamp)
+'
+
+test_expect_success 'drain update mode does not change already drained rank' '
+	flux resource drain --update 1 test_reason_notouch &&
+	flux resource drain | test_must_fail grep test_reason_notouch
+'
+
+test_expect_success 'drain update mode works with idset' '
+	flux resource drain --update 0-1 test_reason_update &&
+	flux resource status -s drain -no {reason} | grep test_reason_update &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 2 &&
+	test $(flux resource status -s drain -no {nnodes}) -eq 2 &&
+	flux resource undrain 0
+'
+
+test_expect_success 'drain works with idset' '
+	flux resource drain 2-3 &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 3 &&
+	test $(flux resource status -s drain -no {ranks}) = "1-3"
+'
+
+test_expect_success 'reload resource module to simulate instance restart' '
+	flux module remove sched-simple &&
+	flux module reload resource noverify &&
+	waitdown 0 &&
+	flux module load sched-simple
+'
+
+test_expect_success 'undrain one node' '
+	flux resource undrain 3 &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 2
+'
+
+test_expect_success 'two nodes are still drained' '
+	test $(flux resource list -n -s down -o {nnodes}) -eq 2
+'
+
+test_expect_success 'undrain remaining nodes' '
+	flux resource undrain 1-2 &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 0
+'
+
+test_expect_success 'resource.eventlog has three undrain events' '
+	test $(has_resource_event undrain | wc -l) -eq 3
+'
+
+test_expect_success 'reload resource module to simulate instance restart' '
+	flux module remove sched-simple &&
+	flux module reload resource noverify &&
+	waitdown 0 &&
+	flux module load sched-simple
+'
+
+test_expect_success 'no nodes remain drained after restart' '
+	test $(flux resource status -s drain -no {nnodes}) -eq 0
+'
+
+test_expect_success 'drain one node' '
+	flux resource drain 0 testing
+'
+test_expect_success 'reload resource module with one node excluded' '
+	flux module remove sched-simple &&
+	flux module remove resource &&
+	echo "resource.exclude = \"0\"" | flux config load &&
+	flux module load resource noverify &&
+	waitdown 0 &&
+	flux module load sched-simple
+'
+
+test_expect_success 'excluded node is no longer drained' '
+	test $(flux resource status -s drain -no {nnodes}) -eq 0
+'
+
+test_expect_success 'excluded node cannot be forcibly drained' '
+	test_must_fail flux resource drain 0 reason
+'
+
+test_expect_success 'drain of idset with excluded and drained nodes fails' '
+	flux resource drain 1 reason &&
+	test_must_fail flux resource drain 0-1 another reason 2>multi.err &&
+	test_debug "cat multi.err" &&
+	grep "drained or excluded" multi.err
+'
+
+test_expect_success 'undrain ranks' '
+	flux resource undrain 1
+'
+test_expect_success 'reload resource module with no nodes excluded' '
+	flux module remove sched-simple &&
+	flux module remove resource &&
+	echo "resource.exclude = \"\"" | flux config load &&
+	flux module load resource noverify &&
+	waitdown 0 &&
+	flux module load sched-simple
+'
+
+test_expect_success 'no nodes remain drained or excluded' '
+	test $(flux resource status -s drain -no {nnodes}) -eq 0 &&
+	test $(flux resource status -s exclude -no {nnodes}) -eq 0
+'
+
+test_expect_success 'drained rank subsequently excluded is ignored' '
+	flux resource drain 1 this will be ignored &&
+	test $(flux resource status -s drain -no {nnodes}) -eq 1 &&
+	flux module remove sched-simple &&
+	flux module remove resource &&
+	echo resource.exclude = \"1\" | flux config load &&
+	flux module load resource noverify &&
+	waitdown 0 &&
+	flux module load sched-simple &&
+	test $(flux resource status -s drain -no {nnodes}) -eq 0 &&
+	flux resource list
+'
+
+test_expect_success 'reload resource module with no nodes excluded' '
+	flux module remove sched-simple &&
+	flux module remove resource &&
+	echo "resource.exclude = \"\"" | flux config load &&
+	flux module load resource noverify &&
+	waitdown 0 &&
+	flux module load sched-simple
+'
+
+test_expect_success 'no nodes remain drained or excluded' '
+	test $(flux resource status -s drain -no {nnodes}) -eq 0 &&
+	test $(flux resource status -s exclude -no {nnodes}) -eq 0
+'
+
+test_expect_success 'undrain fails if rank not drained' '
+	test_must_fail flux resource undrain 1 2>undrain_not.err &&
+	grep ".*rank 1.* not drained" undrain_not.err
+'
+
+test_expect_success 'undrain fails if any rank not drained' '
+	flux resource drain 0 &&
+	test_must_fail flux resource undrain 0-1 2>undrain_0-1_not.err &&
+	test_debug "cat undrain_0-1_not.err" &&
+	grep ".*rank 1.* not drained" undrain_0-1_not.err
+'
+
+test_expect_success 'undrain reports multiple ranks not drained' '
+	test_must_fail flux resource undrain 0-2 2>undrain_0-2_not.err &&
+	test_debug "cat undrain_0-2_not.err" &&
+	grep ".*ranks 1-2.* not drained" undrain_0-2_not.err
+'
+
+test_expect_success 'undrain --force works even if a target is not drained' '
+	flux resource undrain --force 0-1
+'
+
+test_expect_success 'undrain --force returns success when no targets drained' '
+	flux resource undrain --force 0-1
+'
+
+undrain_bad_mode() {
+	flux python -c 'import flux; \
+	  flux.Flux().rpc("resource.undrain", \
+			  {"targets": "0", "mode": "foo"}).get()'
+}
+
+test_expect_success 'undrain RPC rejects invalid mode' '
+	test_must_fail undrain_bad_mode 2>undrain_bad_mode.err &&
+	grep "invalid undrain mode" undrain_bad_mode.err
+'
+
+test_expect_success 'drain fails if idset is empty' '
+	test_must_fail flux resource drain "" 2>drain_empty.err &&
+	grep "idset is empty" drain_empty.err
+'
+
+test_expect_success 'drain fails if idset is out of range' '
+	test_must_fail flux resource drain "0-$SIZE" 2>drain_range.err &&
+	grep "idset is out of range" drain_range.err
+'
+
+# Note: in test, drain `hostname` will drain all ranks since all ranks
+#  are running on the same host
+#
+test_expect_success 'un/drain works with hostnames' '
+	flux resource drain fake[2-3] &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 2 &&
+	flux resource undrain fake[2-3] &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 0
+'
+
+test_expect_success 'drain with no args lists currently drained targets' '
+	flux resource drain 0 happy happy, joy joy &&
+	flux resource drain > drain.out &&
+	test_debug "cat drain.out" &&
+	grep "happy happy, joy joy" drain.out
+'
+
+test_expect_success 'drain/undrain works on rank > 0' '
+	flux exec -r 1 flux resource undrain 0 &&
+	flux exec -r 1 flux resource drain 0 whee drained again
+'
+
+test_expect_success 'drain with no arguments works on rank > 0' '
+	flux exec -r 1 flux resource drain
+'
+
+test_expect_success 'drain with no arguments works for guest' '
+	FLUX_HANDLE_ROLEMASK=0x2 flux resource drain
+'
+
+drain_onrank() {
+	local op=$1
+	local nodeid=$2
+	local target=$3
+	flux python -c "import flux; print(flux.Flux().rpc(\"resource.$op\",{\"targets\":$target, \"reason\":\"\"}, nodeid=$nodeid).get())"
+}
+
+test_expect_success 'resource.drain RPC fails on rank > 0' '
+	test_must_fail drain_onrank drain 1 0 2>drain1.err &&
+	grep -i "unknown service method" drain1.err
+'
+
+test_expect_success 'resource.undrain RPC fails on rank > 0' '
+	test_must_fail drain_onrank undrain 1 0 2>undrain1.err &&
+	grep -i "unknown service method" undrain1.err
+'
+
+test_expect_success 'drain works on allocated rank' '
+	flux resource undrain $(flux resource status -s drain -no {ranks}) &&
+	id=$(flux submit --wait-event=start sleep 300) &&
+	rank=$(flux jobs -no {ranks} $id) &&
+	flux resource drain $rank &&
+	test $(flux resource list -n -s down -o {nnodes}) -eq 1 &&
+	flux resource drain | grep draining &&
+	flux cancel $id &&
+	flux job wait-event $id clean &&
+	flux resource drain | grep drained
+'
+
+test_expect_success 'flux resource drain differentiates drain/draining' '
+	flux resource undrain $(flux resource status -s drain -no {ranks}) &&
+	id=$(flux submit --wait-event=start sleep 300) &&
+	rank=$(flux jobs -no {ranks} $id) &&
+	flux resource drain fake0 &&
+	test_debug "flux resource drain" &&
+	test_debug "flux resource status" &&
+	test $(flux resource status -s draining -no {ranks}) = "$rank" &&
+	flux resource drain | grep draining &&
+	flux cancel $id &&
+	flux job wait-event $id clean &&
+	test $(flux resource status -s drain -no {nnodes}) -eq 1
+'
+
+test_expect_success 'flux resource drain supports --include' '
+	flux resource drain -ni 0 >drain-include.output &&
+	test_debug "cat drain-include.output" &&
+	test $(wc -l <drain-include.output) -eq 1
+'
+
+test_expect_success 'flux resource drain works without scheduler loaded' '
+	flux module unload sched-simple &&
+	flux resource drain &&
+	test $(flux resource status -s drain -no {nnodes}) -eq 1
+'
+
+test_expect_success 'resource can replay eventlog with pre v0.62 events' '
+	flux kvs put --raw resource.eventlog=- <<-EOT &&
+	{"timestamp":1713893408.8647039,"name":"resource-init","context":{"restart":false,"drain":{},"online":"","exclude":""}}
+	{"timestamp":1713893408.8673074,"name":"resource-define","context":{"method":"configuration"}}
+	{"timestamp":1713893411.0698946,"name":"online","context":{"idset":"0"}}
+	{"timestamp":1713893411.7209313,"name":"online","context":{"idset":"1"}}
+	{"timestamp":1713893412.2919452,"name":"online","context":{"idset":"2-3"}}
+	{"timestamp":1713905771.5895882,"name":"offline","context":{"idset":"1-3"}}
+	{"timestamp":1713905772.5933509,"name":"resource-init","context":{"restart":true,"drain":{},"online":"","exclude":""}}
+	{"timestamp":1713905772.5948801,"name":"resource-define","context":{"method":"configuration"}}
+	{"timestamp":1713906350.984611,"name":"drain","context":{"idset":"2","reason":"smurfs","overwrite":0}}
+	{"timestamp":1713906351.000000,"name":"drain","context":{"idset":"2","reason":"underwear","overwrite":1}}
+	{"timestamp":1713906369.9485908,"name":"resource-init","context":{"restart":true,"drain":{"2":{"timestamp":1713906351.000000,"reason":"underwear"}},"online":"","exclude":""}}
+	EOT
+	flux module reload resource noverify
+'
+test_expect_success 'nodes drained in old eventlog are drained after replay' '
+	flux resource drain -n -o "{ranks} {reason}" >legacydrain.out &&
+	cat >legacydrain.exp<<-EOT &&
+	2 underwear
+	EOT
+	test_cmp legacydrain.exp legacydrain.out
+'
+
+test_expect_success 'resource can replay eventlog with bad ranks' '
+	flux kvs put --raw resource.eventlog=- <<-EOT &&
+	{"timestamp":1713906351.000000,"name":"drain","context":{"idset":"42","nodelist":"fake42","reason":"","overwrite":0}}
+	EOT
+	flux module reload resource noverify
+'
+
+test_expect_success 'no nodes are drained after replay' '
+	test -z "$(flux resource drain -n -o {nnodes})"
+'
+
+test_expect_success 'reload resource with two nodes drained' '
+	flux kvs put --raw resource.eventlog=- <<-EOT &&
+	{"timestamp":1713906350.984611,"name":"drain","context":{"idset":"1-2","nodelist":"fake[1-2]","reason":"uvula","overwrite":0}}
+	EOT
+	flux module reload resource noverify
+'
+
+test_expect_success 'the correct two nodes are drained after replay' '
+	flux resource drain -n -o "{ranks} {reason}" >drainhosts.out &&
+	cat >drainhosts.exp<<-EOT &&
+	1-2 uvula
+	EOT
+	test_cmp drainhosts.exp drainhosts.out
+'
+
+test_expect_success 'reload resource with two nodes remapped' '
+	flux kvs put --raw resource.eventlog=- <<-EOT &&
+	{"timestamp":1713906350.984611,"name":"drain","context":{"idset":"1-2","nodelist":"fake[2-3]","reason":"uvula","overwrite":0}}
+	EOT
+	flux module reload resource noverify
+'
+
+test_expect_success 'the remapped nodes are drained after replay' '
+	flux resource drain -n -o "{ranks} {reason}" >drainhosts2.out &&
+	cat >drainhosts2.exp<<-EOT &&
+	2-3 uvula
+	EOT
+	test_cmp drainhosts2.exp drainhosts2.out
+'
+
+test_expect_success 'reload resource with two nodes remapped, one bad host' '
+	flux kvs put --raw resource.eventlog=- <<-EOT &&
+	{"timestamp":1713906350.984611,"name":"drain","context":{"idset":"1-2","nodelist":"fake[3-4]","reason":"uvula","overwrite":0}}
+	EOT
+	flux module reload resource noverify
+'
+
+test_expect_success 'the remapped nodes are drained after replay' '
+	flux resource drain -n -o "{ranks} {reason}" >drainhosts3.out &&
+	cat >drainhosts3.exp<<-EOT &&
+	3 uvula
+	EOT
+	test_cmp drainhosts3.exp drainhosts3.out
+'
+
+test_expect_success 'a malformed event in the eventlog prevents loading' '
+	flux module remove -f resource &&
+	flux kvs put --raw resource.eventlog=- <<-EOT &&
+	{}
+	EOT
+	test_must_fail flux module load resource noverify
+'
+test_expect_success 'a malformed drain context in the eventlog prevents loading' '
+	flux module remove -f resource &&
+	flux kvs put --raw resource.eventlog=- <<-EOT &&
+	{"timestamp":1713906350.984611,"name":"drain","context":{}}
+	EOT
+	test_must_fail flux module load resource noverify
+'
+test_expect_success 'a malformed undrain context in the eventlog fails' '
+	flux module remove -f resource &&
+	flux kvs put --raw resource.eventlog=- <<-EOT &&
+	{"timestamp":1713906350.984611,"name":"undrain","context":{}}
+	EOT
+	test_must_fail flux module load resource noverify
+'
+test_expect_success '' '
+	flux module remove -f resource &&
+	flux kvs unlink resource.eventlog &&
+	flux module load resource noverify
+'
+
+test_expect_success 'load scheduler' '
+	flux module load sched-simple
+'
+
+test_done
diff --git a/t/t2312-resource-exclude.t b/t/t2312-resource-exclude.t
new file mode 100755
index 000000000000..4e0b1c39ce00
--- /dev/null
+++ b/t/t2312-resource-exclude.t
@@ -0,0 +1,115 @@
+#!/bin/sh
+
+test_description='Test resource exclusion'
+
+. `dirname $0`/sharness.sh
+
+cat >exclude.toml <<-EOT
+[resource]
+exclude = "0"
+EOT
+
+SIZE=4
+test_under_flux $SIZE full --config-path=$(pwd)/exclude.toml
+
+# Usage: waitup N
+#   where N is a count of online ranks
+waitup () {
+    run_timeout 5 flux python -c "import flux; print(flux.Flux().rpc(\"resource.monitor-waitup\",{\"up\":$1}).get())"
+}
+waitdown () {
+    waitup $(($SIZE-$1))
+}
+
+has_resource_event () {
+    flux kvs eventlog get resource.eventlog | awk '{ print $2 }' | grep $1
+}
+
+test_expect_success 'wait for monitor to declare all nodes are up' '
+    waitdown 0
+'
+
+test_expect_success 'flux resource list shows no nodes down' '
+	test $(flux resource list -n -s down -o {nnodes}) -eq 0
+'
+
+test_expect_success 'flux resource status shows one node excluded' '
+	test $(flux resource status -s exclude -no {nnodes}) -eq 1 &&
+	test $(flux resource status -s exclude -no {ranks}) = "0"
+'
+
+test_expect_success 'flux resource status shows all nodes online' '
+	test $(flux resource status -s online -no {nnodes}) -eq ${SIZE}
+'
+
+test_expect_success 'flux resource status shows correct nodes avail' '
+	NAVAIL=$((SIZE-1)) &&
+	test $(flux resource status -s avail -no {nnodes}) -eq $NAVAIL
+'
+
+test_expect_success 'but monitor still says all nodes are up' '
+	waitdown 0
+'
+
+test_expect_success 'config with bad exclude idset fails' '
+	cat >resource.toml <<-EOT &&
+	[resource]
+	exclude = "xxzz"
+	EOT
+	test_must_fail flux start --config-path=resource.toml true
+'
+
+test_expect_success 'config with out of range exclude idset fails' '
+	cat >resource.toml <<-EOT &&
+	[resource]
+	exclude = "1"
+	EOT
+	test_must_fail flux start --config-path=resource.toml true
+'
+
+# See flux-framework/flux-core#5337
+test_expect_success 'test instance can exclude ranks' '
+	cat >exclude.toml <<-EOT &&
+	[resource]
+	exclude = "1"
+	EOT
+	test $(flux start -s2 --config-path=exclude.toml \
+	    flux resource status -s exclude -no {nnodes}) -eq 1
+'
+test_expect_success 'test instance fails to exclude hostnames' '
+	cat >exclude2.toml <<-EOT &&
+	[resource]
+	exclude = "$(hostname -s)"
+	EOT
+	test_must_fail flux start -s2 --config-path=exclude2.toml \
+	    true 2>exclude2.err &&
+	grep "R is unavailable" exclude2.err
+'
+test_expect_success 'instance with configured R can exclude hostnames' '
+	cat >exclude3.toml <<-EOT &&
+	[resource]
+	exclude = "$(hostname -s)"
+	noverify = true
+	[[resource.config]]
+	hosts = "$(hostname -s)"
+	cores = "0"
+	EOT
+	test $(flux start -s1 --config-path=exclude3.toml \
+	    flux resource status -s exclude -no {nnodes}) -eq 1
+'
+test_expect_success 'incorrect excluded hostnames raises correct error' '
+	cat >exclude4.toml <<-EOT &&
+	[resource]
+	exclude = "badhost"
+	noverify = true
+	[[resource.config]]
+	hosts = "$(hostname -s)"
+	cores = "0"
+	EOT
+	test_must_fail flux start --config-path=exclude4.toml true \
+		2>exclude4.err &&
+	test_debug "cat exclude4.err" &&
+	grep "invalid hosts: badhost" exclude4.err
+'
+
+test_done
diff --git a/t/t2313-resource-acquire.t b/t/t2313-resource-acquire.t
new file mode 100755
index 000000000000..46fc642bbb28
--- /dev/null
+++ b/t/t2313-resource-acquire.t
@@ -0,0 +1,99 @@
+#!/bin/sh
+
+test_description='Test resource acquire'
+
+. `dirname $0`/sharness.sh
+
+cat >exclude.toml <<-EOT
+[resource]
+exclude = "0"
+EOT
+
+SIZE=4
+test_under_flux $SIZE full --config-path=$(pwd)/exclude.toml
+
+RPC=${FLUX_BUILD_DIR}/t/request/rpc
+RPC_STREAM=${FLUX_BUILD_DIR}/t/request/rpc_stream
+WAITFILE="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+IDSETUTIL=${FLUX_BUILD_DIR}/src/common/libidset/test_idsetutil
+
+# Usage: acquire_stream timeout outfile [end-event]
+acquire_stream() {
+	run_timeout $1 $RPC_STREAM resource.acquire $3 </dev/null >$2
+}
+
+# Usage: waitup N
+#   where N is a count of online ranks
+waitup () {
+	run_timeout 5 flux python -c "import flux; print(flux.Flux().rpc(\"resource.monitor-waitup\",{\"up\":$1}).get())"
+}
+waitdown () {
+	waitup $(($SIZE-$1))
+}
+
+test_expect_success 'wait for monitor to declare all ranks are up' '
+	waitdown 0
+'
+
+test_expect_success 'unload scheduler' '
+	flux module remove sched-simple
+'
+
+test_expect_success 'acquire works and response contains up, resources' '
+	$RPC resource.acquire </dev/null >acquire.out &&
+	jq -c -e -a .resources acquire.out &&
+	jq -c -e -a .up acquire.out
+'
+
+test_expect_success 'acquire works again after first acquire disconnected' '
+	$RPC resource.acquire </dev/null
+'
+
+test_expect_success 'acquire returns resources excluding rank 0' '
+	$RPC resource.acquire </dev/null >acquire2.out &&
+	jq -r .resources.execution.R_lite[0].rank acquire2.out \
+		>acquire2_rank.out &&
+	echo "1-$(($SIZE-1))" >acquire2_rank.exp &&
+	test_cmp acquire2_rank.exp acquire2_rank.out
+'
+
+test_expect_success 'acquire returns up excluding rank 0' '
+	jq -e -r .up acquire2.out | $IDSETUTIL expand >acquire2_up.out &&
+	test_must_fail grep "^0" acquire2_up.out
+'
+
+test_expect_success NO_CHAIN_LINT 'drain rank 1 causes down response' '
+	acquire_stream 30 acquire3.out down &
+	pid=$! &&
+	$WAITFILE -t 10 -v -p \"resources\" acquire3.out &&
+	flux resource drain "1" &&
+	wait $pid
+'
+
+test_expect_success NO_CHAIN_LINT 'undrain/drain rank 1 causes up,down responses' '
+	acquire_stream 30 acquire4.out down &
+	pid=$! &&
+	$WAITFILE -t 10 -v -p \"resources\" acquire4.out &&
+	flux resource undrain 1 &&
+	$WAITFILE -t 10 -v -p \"up\" -c 2 acquire4.out &&
+	flux resource drain 1 &&
+	wait $pid
+'
+
+test_expect_success 'load scheduler' '
+	flux module load sched-simple
+'
+
+test_expect_success 'flux resource reload fails with acquire client' '
+	flux kvs get resource.R >R &&
+	test_must_fail flux resource reload R
+'
+
+test_expect_success 'flux resource acquire-mute works' '
+	flux resource acquire-mute &&
+	flux resource drain 2 mumble &&
+	flux resource drain -n -o "{ranks} {reason}" | grep mumble &&
+	flux run --requires=rank:2 flux getattr rank
+'
+
+test_done
diff --git a/t/t2314-resource-monitor.t b/t/t2314-resource-monitor.t
new file mode 100755
index 000000000000..4512543c6b7f
--- /dev/null
+++ b/t/t2314-resource-monitor.t
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+test_description='Test resource monitoring'
+
+. `dirname $0`/sharness.sh
+
+# min SIZE=4
+SIZE=$(test_size_large)
+test_under_flux $SIZE
+
+# Usage: waitup N
+#   where N is a count of online ranks
+waitup () {
+	run_timeout 5 flux python -c "import flux; print(flux.Flux().rpc(\"resource.monitor-waitup\",{\"up\":$1}).get())"
+}
+waitdown () {
+	waitup $(($SIZE-$1))
+}
+
+test_expect_success 'wait for monitor to declare all ranks are up' '
+	waitdown 0
+'
+
+test_expect_success 'reload resource with monitor-force-up results in 0 down' '
+	flux module remove sched-simple &&
+	flux module reload resource monitor-force-up &&
+	flux module load sched-simple &&
+	waitdown 0
+'
+
+test_expect_success 'scheduler thinks no nodes are down' '
+	test $(flux resource list -n -s down -o {nnodes}) -eq 0
+'
+
+test_done
diff --git a/t/t2315-resource-system.t b/t/t2315-resource-system.t
new file mode 100755
index 000000000000..b29bd08b9b83
--- /dev/null
+++ b/t/t2315-resource-system.t
@@ -0,0 +1,265 @@
+#!/bin/sh
+
+test_description='Test resource module with system instance config'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. `dirname $0`/sharness.sh
+
+if which hwloc-bind > /dev/null; then
+	NCORES=$(hwloc-bind --get | hwloc-calc --number-of core | tail -n 1)
+	test $NCORES = 1 || test_set_prereq MULTICORE
+fi
+
+test_expect_success 'create test R with unlikely core count' '
+	flux R encode -r0-1 -c0-1048 >R.test
+'
+
+test_expect_success 'create config file pointing to test R' '
+	cat >resource.toml <<-EOT
+	[resource]
+	path="$(pwd)/R.test"
+	EOT
+'
+
+test_expect_success 'start instance with config file and dump R' '
+	flux start -s2 \
+		--config-path=$(pwd) -Slog-filename=logfile \
+		flux kvs get resource.R >R.out
+'
+
+test_expect_success 'dumped R matches configured R' '
+	jq --sort-keys . <R.test >R.test.normalized &&
+	jq --sort-keys . <R.out >R.out.normalized &&
+	test_cmp R.test.normalized R.out.normalized
+'
+
+test_expect_success 'both ranks were drained' '
+	grep "draining: rank 0" logfile &&
+	grep "draining: rank 1" logfile
+'
+
+test_expect_success 'invalid R causes instance to fail with error' '
+	mkdir bad &&
+	flux R encode -r 0-1 -c 0-1 | jq ".version = 42" > bad/R &&
+	cat >bad/resource.toml <<-EOF &&
+	[resource]
+	path = "$(pwd)/bad/R"
+	EOF
+	test_must_fail flux start -s2 \
+		--config-path=$(pwd)/bad -Slog-filename=bad/logfile \
+		flux kvs get resource.R >bad/R.out &&
+	grep "error parsing R: invalid version=42" bad/logfile
+'
+test_expect_success 'missing scheduling key causes instance to fail with error' '
+	mkdir sched.enoent &&
+	flux R encode -r 0-1 -c 0-1 > sched.enoent/R &&
+	cat <<-EOF >sched.enoent/sched-missing.toml &&
+	[resource]
+	path = "$(pwd)/sched.enoent/R"
+	scheduling = "missing"
+	EOF
+	test_must_fail flux start -s2 \
+	       --config-path=$(pwd)/sched.enoent \
+	       -Slog-filename=sched.enoent/logfile \
+	       flux kvs get resource.R >sched.enoent/R.out &&
+	grep -i "resource.scheduling:.*no such" sched.enoent/logfile
+'
+
+test_expect_success 'resource.scheduling key amends resource.path' '
+	mkdir sched.good &&
+	flux R encode -r 0-1 -c 0-1 > sched.good/R &&
+	cat >sched.good/sched.json <<-EOF &&
+	{"test": 1}
+	EOF
+	cat >sched.good/resource.toml <<-EOF &&
+	[resource]
+	path = "$(pwd)/sched.good/R"
+	scheduling = "$(pwd)/sched.good/sched.json"
+	EOF
+	flux start -s2 \
+	       --config-path=$(pwd)/sched.good \
+	       -Slog-filename=sched.good/logfile \
+	       flux kvs get resource.R >sched.good/R.out &&
+	jq -S < sched.good/R.out &&
+	jq -e < sched.good/R.out ".scheduling == {\"test\": 1}"
+'
+
+test_expect_success 'invalid scheduling key causes instance to fail with error' '
+	mkdir sched.bad &&
+	flux R encode -r 0-1 -c 0-1 > sched.bad/R &&
+	cat >sched.bad/sched.json <<-EOF &&
+	foo
+	EOF
+	cat >sched.bad/resource.toml <<-EOF &&
+	[resource]
+	path = "$(pwd)/sched.bad/R"
+	scheduling = "$(pwd)/sched.bad/sched.json"
+	EOF
+	test_must_fail flux start -s2 \
+	       --config-path=$(pwd)/sched.bad \
+	       -Slog-filename=sched.bad/logfile \
+	       flux kvs get resource.R >sched.bad/R.out &&
+	grep "error loading resource.scheduling" sched.bad/logfile
+'
+
+test_expect_success 'missing ranks in R are drained' '
+	mkdir missing &&
+	flux R encode -r 0-1 --local > missing/R &&
+	cat >missing/resource.toml <<-EOF &&
+	[resource]
+	path = "$(pwd)/missing/R"
+	EOF
+	flux start -s3 \
+		--config-path=$(pwd)/missing -Slog-filename=missing/logfile \
+		flux kvs get resource.R >missing/R.out &&
+	grep "draining: rank 2 not found in expected ranks" missing/logfile
+'
+
+test_expect_success 'ranks can be excluded by configuration' '
+	name=excluded &&
+	mkdir $name &&
+	flux R encode -r 0-1 --local > ${name}/R &&
+	cat >${name}/resource.toml <<-EOF &&
+	[resource]
+	path = "$(pwd)/${name}/R"
+	exclude = "0"
+	EOF
+	flux start -s 2 \
+		--config-path=$(pwd)/${name} -Slog-filename=${name}/logfile \
+		flux resource list -s up -no {nnodes} > ${name}/nnodes &&
+	test "$(cat ${name}/nnodes)" = "1"
+'
+
+test_expect_success 'invalid exclude ranks cause instance failure' '
+	name=bad-exclude &&
+	mkdir $name &&
+	flux R encode -r 0-1 --local > ${name}/R &&
+	cat >${name}/resource.toml <<-EOF &&
+	[resource]
+	path = "$(pwd)/${name}/R"
+	exclude = "0-4"
+	EOF
+	test_must_fail flux start -s 2 \
+		--config-path=$(pwd)/${name} -Slog-filename=${name}/logfile \
+		flux resource list -s up -no {nnodes} > ${name}/nnodes &&
+    grep "out of range" ${name}/logfile
+'
+
+test_expect_success 'invalid exclude hosts cause instance failure' '
+	name=bad-host-exclude &&
+	mkdir $name &&
+	flux R encode -r 0-1 --local > ${name}/R &&
+	cat >${name}/resource.toml <<-EOF &&
+	[resource]
+	path = "$(pwd)/${name}/R"
+	exclude = "nosuchhost"
+	EOF
+	test_must_fail flux start -s 2 \
+		--config-path=$(pwd)/${name} -Slog-filename=${name}/logfile \
+		flux resource list -s up -no {nnodes} > ${name}/nnodes &&
+    grep "invalid hosts: nosuchhost" ${name}/logfile
+'
+
+test_expect_success 'gpu resources in configured R are not verified' '
+	name=gpu-noverify &&
+	mkdir $name &&
+	flux R encode -r 0 --local | \
+		jq .execution.R_lite[0].children.gpu=\"42-43\" > ${name}/R &&
+	cat >${name}/resource.toml <<-EOF &&
+	[resource]
+	path = "$(pwd)/${name}/R"
+	EOF
+	flux start -s 1\
+		--config-path=$(pwd)/${name} -Slog-filename=${name}/logfile \
+		flux resource list -s up -no {rlist} > ${name}/rlist &&
+	test_debug "cat ${name}/rlist" &&
+	grep "gpu\[42-43\]" ${name}/rlist
+'
+
+test_expect_success MULTICORE 'resource norestrict option works' '
+	name=norestrict &&
+	mkdir $name &&
+	flux R encode -r 0 --local > ${name}/R &&
+	cat >${name}/resource.toml <<-EOF &&
+	[resource]
+	path = "$(pwd)/${name}/R"
+	norestrict = true
+	EOF
+	hwloc-bind core:0 flux start -s1 \
+		--config-path=$(pwd)/${name} -Slog-filename=${name}/logfile \
+		flux run -N1 --exclusive \
+		  sh -c "hwloc-bind --get | hwloc-calc --number-of core | tail -n1" \
+		    >${name}/ncores &&
+	test_debug "cat ${name}/ncores" &&
+	test $(cat ${name}/ncores) = $NCORES
+'
+
+test_expect_success 'resources can be configured in TOML' '
+	name=conftest &&
+	mkdir -p $name &&
+	cat >${name}/resource.toml <<-EOF &&
+	[resource]
+	noverify = true
+	[[resource.config]]
+	hosts = "foo[0-10]"
+	cores = "0-3"
+	[[resource.config]]
+	hosts = "foo[9,10]"
+	gpus = "0-1"
+	[[resource.config]]
+	hosts = "foo[0-2]"
+	properties = ["debug"]
+	[[resource.config]]
+	hosts = "foo[3-10]"
+	properties = ["batch"]
+	EOF
+	flux start -s 1 \
+		--config-path=$(pwd)/${name} -Slog-filename=${name}/logfile \
+		flux resource list -s all -n \
+		   -o "{state} {properties} {nnodes} {ncores} {ngpus} {nodelist}" \
+		   > ${name}/output &&
+	test_debug "cat ${name}/output" &&
+	cat <<-EOF >${name}/expected &&
+	all debug 3 12 0 foo[0-2]
+	all batch 8 32 4 foo[3-10]
+	EOF
+	test_cmp ${name}/expected ${name}/output
+'
+test_expect_success 'bad resource.config causes instance failure' '
+	name=conftest-bad &&
+	mkdir -p ${name} &&
+	cat >${name}/resource.toml <<-EOF &&
+	[resource]
+	noverify = true
+	config = []
+	EOF
+	test_must_fail flux start -s 1 \
+		--config-path=$(pwd)/${name} -Slog-filename=${name}/logfile \
+		flux resource list -s up > ${name}/output 2>&1 &&
+	test_debug "cat ${name}/output" &&
+	grep "no hosts configured" ${name}/output
+'
+
+test_expect_success 'resource.scheduling key amends [resource.config]' '
+	name=sched+config &&
+	mkdir -p $name &&
+	cat >${name}/sched.json <<-EOF &&
+	{ "foo": true }
+	EOF
+	cat >${name}/resource.toml <<-EOF &&
+	[resource]
+	noverify = true
+	scheduling = "$(pwd)/${name}/sched.json"
+	[[resource.config]]
+	hosts = "foo[0-10]"
+	cores = "0-3"
+	EOF
+	flux start -s 1 \
+		--config-path=$(pwd)/${name} -Slog-filename=${name}/logfile \
+		flux kvs get resource.R > ${name}/R.json &&
+	jq -S < ${name}/R.json &&
+	jq -e < ${name}/R.json ".scheduling.foo == true"
+'
+
+test_done
diff --git a/t/t2350-resource-list.t b/t/t2350-resource-list.t
new file mode 100755
index 000000000000..9e9bb23571c9
--- /dev/null
+++ b/t/t2350-resource-list.t
@@ -0,0 +1,588 @@
+#!/bin/sh
+
+test_description='flux-resource list tests'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 full
+
+test_expect_success 'flux resource list: default lists some expected fields' '
+	flux resource list > default.out &&
+	grep STATE default.out &&
+	grep NNODES default.out &&
+	grep NCORES default.out
+'
+test_expect_success 'make the same query using sched.resource-status' '
+	FLUX_RESOURCE_LIST_RPC=sched.resource-status \
+	FLUX_HANDLE_TRACE=1 \
+	flux resource list >sched.out 2>sched.err
+'
+test_expect_success 'FLUX_RESOURCE_LIST_RPC works' '
+	grep sched.resource-status sched.err
+'
+test_expect_success 'results are the same as before' '
+	test_cmp default.out sched.out
+'
+test_expect_success 'make the same query with fallback to sched.resource-status' '
+	FLUX_RESOURCE_LIST_RPC=foo.status \
+	FLUX_HANDLE_TRACE=1 \
+	flux resource list >fallback.out 2>fallback.err
+'
+test_expect_success 'fallback works' '
+	grep sched.resource-status fallback.err
+'
+test_expect_success 'results are the same as before' '
+	test_cmp default.out fallback.out
+'
+test_expect_success 'flux resource list works on follower ranks' '
+	flux exec -r 1 flux resource list >follower.out
+'
+test_expect_success 'results are the same as before' '
+	test_cmp default.out follower.out
+'
+
+test_expect_success 'flux resource list: FLUX_RESOURCE_LIST_FORMAT_DEFAULT works' '
+	FLUX_RESOURCE_LIST_FORMAT_DEFAULT="{nodelist} {nodelist}" \
+		flux resource list > default_override.out &&
+	grep "NODELIST NODELIST" default_override.out
+'
+
+test_expect_success 'flux resource list: FLUX_RESOURCE_LIST_FORMAT_DEFAULT works w/ named format' '
+	FLUX_RESOURCE_LIST_FORMAT_DEFAULT=rlist \
+		flux resource list > default_override_named.out &&
+	grep -w "LIST" default_override_named.out
+'
+
+test_expect_success 'flux resource list: --no-header works' '
+	flux resource list --no-header > default_no_header.out &&
+	test_must_fail grep STATE default_no_header.out &&
+	test_must_fail grep NNODES default_no_header.out &&
+	test_must_fail grep NCORES default_no_header.out
+'
+
+# Use a static format to avoid breaking output if default flux-resource list
+#  format ever changes
+FORMAT="{state:>10} {properties:<10} {nnodes:>6} {ncores:>8} {ngpus:>8}"
+
+for input in ${SHARNESS_TEST_SRCDIR}/flux-resource/list/*.json; do
+    testname=$(basename ${input%%.json}) &&
+    base=${input%%.json} &&
+    name=$(basename $base) &&
+    test_expect_success "flux-resource list input check: $testname" '
+        flux resource list -o "$FORMAT" \
+            --from-stdin < $input > $name.output 2>&1 &&
+        test_debug "cat $name.output" &&
+        test_cmp ${base}.expected $name.output
+    '
+    test_expect_success "flux-resource list --skip-empty: $testname" '
+        awk "\$3!=0 {print}" ${base}.expected > skip-empty.${name}.expected &&
+        flux resource list -o "$FORMAT" \
+            --skip-empty \
+            --from-stdin < $input > skip-empty.${name}.output 2>&1 &&
+        test_cmp skip-empty.${name}.expected skip-empty.${name}.output
+    '
+    test_expect_success "flux-resource info input check: $testname" '
+        flux resource info --from-stdin < $input > ${name}-info.output 2>&1 &&
+        test_debug "cat ${name}-info.output" &&
+        test_cmp ${base}-info.expected ${name}-info.output
+    '
+    test_expect_success "flux-resource R input check: $testname" '
+        flux resource R --from-stdin < $input > ${name}-R.output 2>&1 &&
+        test_debug "cat ${name}-info.output" &&
+        test_cmp ${base}.R ${name}-R.output
+    '
+done
+
+#  Ensure all tested inputs can also work with --include
+#  Simply restrict to rank 0 then ensure {ranks} returns only 0
+for input in ${SHARNESS_TEST_SRCDIR}/flux-resource/list/*.json; do
+    testname=$(basename ${input%%.json}) &&
+    base=${input%%.json} &&
+    name=$(basename $base) &&
+    test_expect_success "flux-resource list input --include check: $name" '
+        flux resource list -o "{ranks} {nodelist}" --include=0 \
+	    --no-skip-empty \
+            --from-stdin < $input >$name-include.output 2>&1 &&
+        test_debug "cat $name-include.output" &&
+        grep "^0[^,-]" $name-include.output
+    '
+    test_expect_success "flux-resource info input --include check: $testname" '
+        flux resource info --from-stdin --no-skip-empty -i0 < $input \
+            > ${name}-info-include.output 2>&1 &&
+        test_debug "cat ${name}-info-include.output" &&
+	grep "1 Node" ${name}-info-include.output
+    '
+    test_expect_success "flux-resource R input ---include check: $testname" '
+        flux resource R --from-stdin -i0 < $input \
+           > ${name}-info-R.output &&
+        test "$(flux R decode --count=node <${name}-info-R.output)" -eq 1
+    '
+done
+
+# Special case tests for -q, --queue using fluke.json and fluke.config
+
+FLUKE_CONFIG=${SHARNESS_TEST_SRCDIR}/flux-resource/list/fluke.config
+FLUKE_INPUT=${SHARNESS_TEST_SRCDIR}/flux-resource/list/fluke.json
+test_expect_success 'flux-resource list supports -q, --queue' '
+	flux resource list --queue=debug \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT > fluke-debug.output &&
+	test_debug "cat fluke-debug.output" &&
+	test_must_fail grep batch fluke-debug.output &&
+	flux resource list --queue=batch \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT > fluke-batch.output &&
+	test_debug "cat fluke-batch.output" &&
+	test_must_fail grep debug fluke-batch.output &&
+	flux resource list --queue=debug,batch \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT > fluke-both.output &&
+	grep debug fluke-both.output &&
+	grep batch fluke-both.output
+'
+test_expect_success 'flux-resource list skips empty lines (issue#6093)' '
+	flux resource list -i fluke[3,103] -no "{queue} {nodelist}" \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+                < $FLUKE_INPUT >fluke-empty-lines-test.out &&
+	test_debug "cat fluke-empty-lines-test.out" &&
+	test $(wc -l < fluke-empty-lines-test.out) -eq 2
+'
+test_expect_success 'flux-resource R supports -q, --queue' '
+	flux resource R --queue=debug \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT >debug.R.out &&
+	test_must_fail grep batch debug.R.out &&
+	flux resource R --queue=batch \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT > batch.R.out &&
+	test_must_fail grep debug batch.R.out &&
+	flux resource R --queue=debug,batch \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT > both.R.out &&
+	grep debug both.R.out &&
+	grep batch both.R.out
+'
+test_expect_success 'flux-resource info supports -q, --queue' '
+	flux resource info --queue=debug \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT > info-debug.output &&
+	cat <<-EOF >info-debug.expected &&
+	8 Nodes, 32 Cores, 0 GPUs
+	EOF
+	test_cmp info-debug.expected info-debug.output
+'
+test_expect_success 'flux-resource -q, --queue reports invalid queue' '
+	test_must_fail flux resource list --queue=foo \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT 2>badqueue.err &&
+	grep "foo: no such queue" badqueue.err
+'
+test_expect_success 'flux-resource list supports --include' '
+	flux resource list -s all -ni 0 --no-skip-empty >list-include.output &&
+	test_debug "cat list-include.output" &&
+	test $(wc -l <list-include.output) -eq 1
+'
+INPUT=${SHARNESS_TEST_SRCDIR}/flux-resource/list/normal-new.json
+test_expect_success 'flux-resource list --include skips empty lines by default' '
+	flux resource list -s all -ni 0 --from-stdin < $INPUT >skip-empty.out &&
+	test_debug "cat skip-empty.out" &&
+	test $(wc -l <skip-empty.out) -eq 1
+'
+test_expect_success 'flux-resource list: --include works with ranks' '
+	flux resource list -s all -o "{nnodes} {ranks}" -ni 0,3 --from-stdin \
+	    --no-skip-empty \
+		< $INPUT >include-ranks.out &&
+	test_debug "cat include-ranks.out" &&
+	grep "^2 0,3" include-ranks.out
+'
+test_expect_success 'flux-resource list: --include works with hostnames' '
+	flux resource list -s all -o "{nnodes} {nodelist}" -ni pi[0,3] \
+		--from-stdin < $INPUT >include-hosts.out &&
+	test_debug "cat include-hosts.out" &&
+	grep "^2 pi\[3,0\]" include-hosts.out
+'
+test_expect_success 'flux-resource list: -i works with excluded hosts #5266' '
+	cat <<-'EOF' >corona.json &&
+	{
+	  "all": {
+	    "execution": {
+	      "R_lite": [
+	        {
+	          "children": {
+	            "core": "0-47",
+	            "gpu": "0-7"
+	          },
+	          "rank": "4-124"
+	        }
+	      ],
+	      "expiration": 0,
+	      "nodelist": [
+	        "corona[171-207,213-296]"
+	      ],
+	      "properties": {
+	        "pbatch": "20-124",
+	        "pdebug": "4-19"
+	      },
+	      "starttime": 0
+	    },
+	    "version": 1
+	  },
+	  "allocated": {
+	    "execution": {
+	      "R_lite": [
+	        {
+	          "children": {
+	            "core": "0-47",
+	            "gpu": "0-7"
+	          },
+	          "rank": "20-27,29-34,36-75,78-84,86-87,90-97,99-111,113-124"
+	        }
+	      ],
+	      "expiration": 0,
+	      "nodelist": [
+	        "corona[187-194,196-201,203-207,213-247,250-256,258-259,262-269,271-283,285-296]"
+	      ],
+	      "properties": {
+	        "pbatch": "20-27,29-34,36-75,78-84,86-87,90-97,99-111,113-124"
+	      },
+	      "starttime": 0
+	    },
+	    "version": 1
+	  },
+	  "down": {
+	    "execution": {
+	      "R_lite": [
+	        {
+	          "children": {
+	            "core": "0-47",
+	            "gpu": "0-7"
+	          },
+	          "rank": "5,9,28,35,76-77,85,88-89,98,112"
+	        }
+	      ],
+	      "expiration": 0,
+	      "nodelist": [
+	        "corona[172,176,195,202,248-249,257,260-261,270,284]"
+	      ],
+	      "properties": {
+	        "pbatch": "28,35,76-77,85,88-89,98,112",
+	        "pdebug": "5,9"
+	      },
+	      "starttime": 0
+	    },
+	    "version": 1
+	  }
+	}
+	EOF
+	NODELIST="corona[176,249,260-261,270,284]" &&
+	flux resource list -s all -o "{nodelist}" -ni $NODELIST \
+		--from-stdin < corona.json >corona.output &&
+	test_debug "cat corona.output" &&
+	test "$(cat corona.output)" = "$NODELIST"
+'
+test_expect_success 'flux-resource list: --include works with invalid host' '
+	flux resource list -s all -o "{nnodes} {nodelist}" -ni pi7 \
+		--from-stdin --no-skip-empty \
+		< $INPUT >include-invalid-hosts.out &&
+	test_debug "cat include-invalid-hosts.out" &&
+	grep "^0" include-invalid-hosts.out
+'
+test_expect_success 'flux-resource R supports --states' '
+	flux resource R --from-stdin -s all <$INPUT >all.R &&
+	test $(flux R decode --count=node <all.R) -eq 5 &&
+	flux resource R --from-stdin -s up <$INPUT >up.R &&
+	test $(flux R decode --count=node <up.R) -eq 5 &&
+	flux resource R --from-stdin -s down <$INPUT >down.R &&
+	test $(flux R decode --count=node <down.R) -eq 0
+'
+test_expect_success 'create test input with properties' '
+	cat <<-EOF >properties-test.in
+	{
+	  "all": {
+	    "version": 1,
+	    "execution": {
+	      "R_lite": [
+	        {
+	          "rank": "0-3",
+	          "children": {
+	            "core": "0-3"
+	          }
+	        }
+	      ],
+	      "starttime": 0,
+	      "expiration": 0,
+	      "nodelist": [
+	        "asp,asp,asp,asp"
+	      ],
+	      "properties": {
+	        "foo": "2-3",
+	        "xx": "1-2"
+	      }
+	    }
+	  },
+	  "allocated": {
+	    "version": 1,
+	    "execution": {
+	      "R_lite": [
+	        {
+	          "rank": "1",
+	          "children": {
+	            "core": "0"
+	          }
+	        }
+	      ],
+	      "starttime": 0,
+	      "expiration": 0,
+	      "nodelist": [
+	        "asp"
+	      ],
+	      "properties": {
+	        "xx": "1"
+	      }
+	    }
+	  },
+	  "down": {
+	    "version": 1,
+	    "execution": {
+	      "R_lite": [
+	        {
+	          "rank": "1",
+	          "children": {
+	            "core": "0-3"
+	          }
+	        }
+	      ],
+	      "starttime": 0,
+	      "expiration": 0,
+	      "nodelist": [
+	        "asp"
+	      ],
+	      "properties": {
+	        "xx": "1"
+	      }
+	    }
+	  }
+	}
+	EOF
+'
+
+test_expect_success 'flux resource list -no {properties} works' '
+	flux resource list -no {properties} \
+		--from-stdin <properties-test.in >properties.out &&
+	test $(cat properties.out) = "xx,foo"
+'
+test_expect_success 'flux resource list -no {properties:>4.4+} works' '
+	flux resource list -no "{properties:>5.5+}" \
+		--from-stdin <properties-test.in >properties-trunc.out &&
+	test_debug "cat properties-trunc.out" &&
+	test $(cat properties-trunc.out) = "xx,f+" &&
+	flux resource list -no "{properties:>5.5h+}" \
+		--from-stdin <properties-test.in >properties-trunc+h.out &&
+	test_debug "cat properties-trunc+h.out" &&
+	test $(cat properties-trunc.out) = "xx,f+"
+'
+test_expect_success 'flux resource list -o rlist works' '
+	flux resource list -o rlist \
+		--from-stdin <properties-test.in >rlist.out &&
+	test_debug "cat rlist.out" &&
+	grep -i list rlist.out
+'
+test_expect_success 'configure queues and resource split amongst queues' '
+	flux R encode -r 0-3 -p batch:0-1 -p debug:2-3 \
+	   | tr -d "\n" \
+	   | flux kvs put -r resource.R=- &&
+	flux config load <<-EOT &&
+	[queues.batch]
+	requires = [ "batch" ]
+	[queues.debug]
+	requires = [ "debug" ]
+	EOT
+	flux queue start --all &&
+	flux module unload sched-simple &&
+	flux module reload resource &&
+	flux module load sched-simple
+'
+test_expect_success 'flux resource list default lists queues' '
+	flux resource list | grep QUEUE
+'
+test_expect_success 'flux resource lists expected queues (single)' '
+	flux resource list -o "{state} {nnodes} {queue}" > listqueue_single.out &&
+	test $(grep -c "free 2 batch" listqueue_single.out) -eq 1 &&
+	test $(grep -c "free 2 debug" listqueue_single.out) -eq 1
+'
+test_expect_success 'flux resource lists expected properties (single)' '
+	flux resource list -o "{state} {nnodes} {properties}" > listprop_single.out &&
+	test $(grep -c "free 2 batch" listprop_single.out) -eq 1 &&
+	test $(grep -c "free 2 debug" listprop_single.out) -eq 1
+'
+# no properties should be output, leading to a single line for all 4 "free" nodes
+test_expect_success 'flux resource lists no properties in propertiesx (single)' '
+	flux resource list -o "{state} {nnodes} {propertiesx}" > listpropx_single.out &&
+	grep "free 4" listpropx_single.out
+'
+test_expect_success 'run a few jobs' '
+	flux submit -q batch sleep 30 > job1A.id &&
+	flux submit -q debug sleep 30 > job1B.id
+'
+test_expect_success 'flux resource lists expected queues in states (single)' '
+	flux resource list -o "{state} {nnodes} {queue}" > list2.out &&
+	test $(grep -c "free 1 batch" list2.out) -eq 1 &&
+	test $(grep -c "free 1 debug" list2.out) -eq 1 &&
+	test $(grep -c "allocated 1 batch" list2.out) -eq 1 &&
+	test $(grep -c "allocated 1 debug" list2.out) -eq 1
+'
+test_expect_success 'sched.resource-status produces the same results' '
+	FLUX_RESOURCE_LIST_RPC=sched.resource-status \
+	flux resource list -o "{state} {nnodes} {queue}" > list2_sched.out &&
+	test_cmp list2.out list2_sched.out
+'
+test_expect_success 'cleanup jobs' '
+	flux cancel $(cat job1A.id) $(cat job1B.id)
+'
+test_expect_success 'configure queues and resource split amongst queues w/ all' '
+	flux R encode -r 0-3 -p all:0-3 -p batch:0-1 -p debug:2-3 \
+	   | tr -d "\n" \
+	   | flux kvs put -r resource.R=- &&
+	flux config load <<-EOT &&
+	[queues.all]
+	requires = [ "all" ]
+	[queues.batch]
+	requires = [ "batch" ]
+	[queues.debug]
+	requires = [ "debug" ]
+	EOT
+	flux queue start --all &&
+	flux module unload sched-simple &&
+	flux module reload resource &&
+	flux module load sched-simple
+'
+# we can't predict listing order of queues/properties, so we grep counts
+test_expect_success 'flux resource lists expected queues (multi)' '
+	flux resource list -o "{state} {nnodes} {queue}" > listqueue_multi.out &&
+	test $(grep "free 2" listqueue_multi.out | grep -c batch) -eq 1 &&
+	test $(grep "free 2" listqueue_multi.out | grep -c debug) -eq 1 &&
+	test $(grep "free 2" listqueue_multi.out | grep -c all) -eq 2
+'
+test_expect_success 'flux resource lists expected properties (multi)' '
+	flux resource list -o "{state} {nnodes} {properties}" > listprop_multi.out &&
+	test $(grep "free 2" listprop_multi.out | grep -c batch) -eq 1 &&
+	test $(grep "free 2" listprop_multi.out | grep -c debug) -eq 1 &&
+	test $(grep "free 2" listprop_multi.out | grep -c all) -eq 2
+'
+# no properties should be output, leading to a single line for all 4 "free" nodes
+test_expect_success 'flux resource lists no properties in propertiesx (multi)' '
+	flux resource list -o "{state} {nnodes} {propertiesx}" > listpropx_multi.out &&
+	grep "free 4" listpropx_multi.out
+'
+test_expect_success 'configure queues and resource with extra property' '
+	flux R encode -r 0-3 -p batch:0-1 -p debug:2-3 -p foo:0-3\
+	   | tr -d "\n" \
+	   | flux kvs put -r resource.R=- &&
+	flux config load <<-EOT &&
+	[queues.batch]
+	requires = [ "batch" ]
+	[queues.debug]
+	requires = [ "debug" ]
+	EOT
+	flux queue start --all &&
+	flux module unload sched-simple &&
+	flux module reload resource &&
+	flux module load sched-simple
+'
+test_expect_success 'flux resource lists expected queues (extraprop)' '
+	flux resource list -o "{state} {nnodes} {queue}" > listqueue_extraprop.out &&
+	grep "free 2 batch" listqueue_extraprop.out &&
+	grep "free 2 debug" listqueue_extraprop.out
+'
+# we can't predict listing order of properties, so we grep counts
+test_expect_success 'flux resource lists expected properties (extraprop)' '
+	flux resource list -o "{state} {nnodes} {properties}" > listprop_extraprop.out &&
+	test $(grep "free 2" listprop_extraprop.out | grep -c batch) -eq 1 &&
+	test $(grep "free 2" listprop_extraprop.out | grep -c debug) -eq 1 &&
+	test $(grep "free 2" listprop_extraprop.out | grep -c foo) -eq 2
+'
+# only "foo" property should be output with properties, leading to
+# uniq line with 4 free nodes
+test_expect_success 'flux resource lists no properties in propertiesx (extraprop)' '
+	flux resource list -o "{state} {nnodes} {propertiesx}" > listpropx_extraprop.out &&
+	grep "free 4 foo" listpropx_extraprop.out
+'
+# N.B. we use the queue "every" b/c the word "all" happens to be in
+# the word "allocated", annoyingly messing up greps
+test_expect_success 'configure queues and resource split amongst queues, add every queue' '
+	flux R encode -r 0-3 -p batch:0-1 -p debug:2-3 \
+	   | tr -d "\n" \
+	   | flux kvs put -r resource.R=- &&
+	flux config load <<-EOT &&
+	[queues.every]
+	[queues.batch]
+	requires = [ "batch" ]
+	[queues.debug]
+	requires = [ "debug" ]
+	EOT
+	flux queue start --all &&
+	flux module unload sched-simple &&
+	flux module reload resource &&
+	flux module load sched-simple
+'
+# we can't predict listing order of queues/properties, so we grep counts
+test_expect_success 'flux resource lists expected queues (every)' '
+	flux resource list -o "{state} {nnodes} {queue}" > listqueue_every.out &&
+	test $(grep "free 2" listqueue_every.out | grep -c batch) -eq 1 &&
+	test $(grep "free 2" listqueue_every.out | grep -c debug) -eq 1 &&
+	test $(grep "free 2" listqueue_every.out | grep -c every) -eq 2
+'
+test_expect_success 'run a few jobs (every)' '
+	flux submit -q batch sleep 30 > job2A.id &&
+	flux submit -q debug sleep 30 > job2B.id
+'
+test_expect_success 'flux resource lists expected queues in states (every)' '
+	flux resource list -o "{state} {nnodes} {queue}" > listqueue_every2.out &&
+	test $(grep "free 1" listqueue_every2.out | grep -c batch) -eq 1 &&
+	test $(grep "free 1" listqueue_every2.out | grep -c debug) -eq 1 &&
+	test $(grep "free 1" listqueue_every2.out | grep -c every) -eq 2 &&
+	test $(grep "allocated 1" listqueue_every2.out | grep -c batch) -eq 1 &&
+	test $(grep "allocated 1" listqueue_every2.out | grep -c debug) -eq 1 &&
+	test $(grep "allocated 1" listqueue_every2.out | grep -c every) -eq 2
+'
+test_expect_success 'flux resource list --queue works for a queue with no constraints' '
+	flux resource list --queue=every -o "{state} {nnodes} {queue}" >queue-every.out &&
+	test $(grep "free 1" queue-every.out | grep -c batch) -eq 1 &&
+	test $(grep "free 1" queue-every.out | grep -c debug) -eq 1 &&
+	test $(grep "free 1" queue-every.out | grep -c every) -eq 2 &&
+	test $(grep "allocated 1" queue-every.out | grep -c batch) -eq 1 &&
+	test $(grep "allocated 1" queue-every.out | grep -c debug) -eq 1 &&
+	test $(grep "allocated 1" queue-every.out | grep -c every) -eq 2
+'
+test_expect_success 'flux resource list includes queue names for empty sets' '
+	# There are no nodes in 'down' state in this test, so use that:
+	flux resource list -s down -no "{queue} {state} {nnodes}" \
+		>queue-empty1.out &&
+	cat <<-EOF >queue-empty1.expected &&
+	every down 0
+	batch down 0
+	debug down 0
+	EOF
+	test_cmp queue-empty1.expected queue-empty1.out
+'
+test_expect_success 'flux resource list includes queue names for empty sets (single)' '
+	flux resource list -q batch -s down -no "{queue} {state} {nnodes}" \
+		>queue-batch-down.out &&
+	cat <<-EOF >queue-batch-down.expected &&
+	batch down 0
+	EOF
+	test_cmp queue-batch-down.expected queue-batch-down.out
+'
+test_expect_success 'flux resource list includes queue names for empty sets (multiple)' '
+	flux resource list -q batch,debug \
+		-s down -no "{queue} {state} {nnodes}" \
+		>queue-batch,debug-down.out &&
+	grep "debug down 0" queue-batch,debug-down.out &&
+	grep "batch down 0" queue-batch,debug-down.out
+'
+test_expect_success 'cleanup jobs' '
+	flux cancel $(cat job2A.id) $(cat job2B.id)
+'
+test_done
diff --git a/t/t2351-resource-status-input.t b/t/t2351-resource-status-input.t
new file mode 100755
index 000000000000..ca5f0ea75f32
--- /dev/null
+++ b/t/t2351-resource-status-input.t
@@ -0,0 +1,274 @@
+#!/bin/sh
+
+test_description='flux-resource status output format tests'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. $(dirname $0)/sharness.sh
+
+# Use a static format to avoid breaking output if default flux-resource status
+#  format ever changes
+FORMAT="{state:>10} {nnodes:>6} {ranks:<15} {nodelist}"
+INPUTDIR=${SHARNESS_TEST_SRCDIR}/flux-resource/status
+
+export FLUX_PYCLI_LOGLEVEL=10
+
+for input in ${INPUTDIR}/*.json; do
+    name=$(basename ${input%%.json})
+    test_expect_success "flux-resource status input check: $name" '
+        base=${input%%.json} &&
+        expected=${base}.expected &&
+        name=$(basename $base) &&
+        flux resource status -o "$FORMAT" \
+            --from-stdin < $input > $name.output 2>&1 &&
+        test_debug "cat $name.output" &&
+        test_cmp $expected $name.output
+    '
+done
+
+# Special case for --queue handling using fluke.json and fluke.config
+FLUKE_CONFIG=${SHARNESS_TEST_SRCDIR}/flux-resource/status/fluke.config
+FLUKE_INPUT=${SHARNESS_TEST_SRCDIR}/flux-resource/status/fluke.json
+test_expect_success 'flux-resource status: -q, --queue works' '
+	flux resource status --queue=debug \
+		--format="$FORMAT" \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT > fluke-debug.output &&
+	cat <<-EOF >fluke-debug.expected &&
+	     STATE NNODES RANKS           NODELIST
+	     avail      5 93,96-99        fluke[96,99-102]
+	    avail*      3 94-95,100       fluke[97-98,103]
+	EOF
+	test_cmp fluke-debug.expected fluke-debug.output &&
+	flux resource status --queue=batch \
+		--format="$FORMAT" \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT > fluke-batch.output &&
+	cat <<-EOF >fluke-batch.expected &&
+	     STATE NNODES RANKS           NODELIST
+	     avail     80 3-13,16-20,22-39,41-57,59-61,63-72,75,77-88,90-92 fluke[6-16,19-23,25-42,44-60,62-64,66-75,78,80-91,93-95]
+	    avail*      8 14,21,58,62,73-74,76,89 fluke[17,24,61,65,76-77,79,92]
+	   drained      2 15,40           fluke[18,43]
+	EOF
+	test_cmp fluke-batch.expected fluke-batch.output
+'
+test_expect_success 'flux-resource status -q, --queue fails on invalid queue' '
+	test_must_fail flux resource status --queue=foo \
+		--format="$FORMAT" \
+		--from-stdin --config-file=$FLUKE_CONFIG \
+		< $FLUKE_INPUT 2>invalidqueue.err &&
+	grep "foo: no such queue" invalidqueue.err
+'
+
+#  Ensure all tested inputs can also work with --include
+#  We simply restrict to rank 0 and then ensure {ranks} returns only 0
+for input in ${INPUTDIR}/*.json; do
+    name=$(basename ${input%%.json})
+    test_expect_success "flux-resource status input --include check: $name" '
+        base=${input%%.json} &&
+        name=$(basename $base)-i &&
+        flux resource status -o "{ranks} {nodelist}" --include=0 \
+            --from-stdin < $input > $name.output 2>&1 &&
+        test_debug "cat $name.output" &&
+	grep "^0[^,-]" $name.output
+    '
+done
+
+
+test_expect_success 'flux-resource status: header included with all formats' '
+	cat <<-EOF >headers.expected &&
+	state==STATE
+	status=STATUS
+	nnodes==NNODES
+	ranks==RANKS
+	nodelist==NODELIST
+	reason==REASON
+	timestamp==TIME
+	EOF
+	sed "s/\(.*\)==.*/\1=={\1}/" headers.expected > headers.fmt &&
+        flux resource status --from-stdin --format="$(cat headers.fmt)" \
+            > headers.output < /dev/null &&
+        test_cmp headers.expected headers.output
+'
+
+test_expect_success 'flux resource status: FLUX_RESOURCE_STATUS_FORMAT_DEFAULT works' '
+	FLUX_RESOURCE_STATUS_FORMAT_DEFAULT="{nodelist} {nodelist}" \
+		flux resource status --from-stdin > default_override.out < /dev/null &&
+	grep "NODELIST NODELIST" default_override.out
+'
+
+test_expect_success 'flux resource status: FLUX_RESOURCE_STATUS_FORMAT_DEFAULT works w/ named format' '
+	FLUX_RESOURCE_STATUS_FORMAT_DEFAULT=long \
+		flux resource status --from-stdin > default_override_named.out < /dev/null &&
+	grep "REASON" default_override_named.out
+'
+
+test_expect_success 'flux-resource status: --no-header works' '
+	INPUT=${INPUTDIR}/example.json &&
+	name=no-header &&
+	flux resource status -s exclude --no-header --from-stdin < $INPUT \
+	    > ${name}.out &&
+	test_debug "cat ${name}.out" &&
+	test $(wc -l < ${name}.out) -eq 1
+'
+
+test_expect_success 'flux-resource status: -o {reason} works' '
+	INPUT=${INPUTDIR}/drain.json &&
+	name=reason &&
+	flux resource status --from-stdin -s drain -no {reason} \
+	    < $INPUT > ${name}.out &&
+	test_debug "cat ${name}.out" &&
+	test $(wc -l < ${name}.out) -eq 2
+'
+
+test_expect_success 'flux-resource status -o long shows REASON field' '
+	INPUT=${INPUTDIR}/drain.json &&
+	name=long &&
+	flux resource status --from-stdin < $INPUT > ${name}.out &&
+	flux resource status -o long --from-stdin < $INPUT > ${name}-long.out &&
+	test_debug "echo default:; cat ${name}.out" &&
+	test_debug "echo long:; cat ${name}-long.out" &&
+	test_must_fail grep REASON ${name}.out &&
+	grep REASON ${name}-long.out &&
+	test $(grep -c drained  ${name}.out) -eq 1 &&
+	test $(grep -c drained ${name}-long.out) -eq 2
+'
+test_expect_success 'flux-resource status: -s always displays that state' '
+	INPUT=${INPUTDIR}/simple.json &&
+	flux resource status --from-stdin -s exclude -o {state} --no-header \
+	    < $INPUT | grep exclude
+'
+test_expect_success 'flux-resource status: --skip-empty works' '
+	INPUT=${INPUTDIR}/simple.json &&
+	flux resource status --from-stdin --skip-empty \
+	    -s exclude -o {state} --no-header \
+	    < $INPUT >skip-empty.out &&
+	test_debug "cat skip-empty.out" &&
+	test_must_be_empty skip-empty.out
+'
+test_expect_success 'flux-resource status: -s help displays states list' '
+	flux resource status --from-stdin -s help < /dev/null 2>&1 \
+	    | grep -i valid
+'
+test_expect_success 'flux-resource status: invalid state generates error' '
+	test_expect_code 1 flux resource status --from-stdin -s frelled \
+	    < /dev/null >bad-state.out 2>&1 &&
+	test_debug "cat bad-state.out" &&
+	grep "Invalid resource state frelled specified" bad-state.out
+'
+
+test_expect_success 'flux-resource status: {status} field works' '
+	INPUT=${INPUTDIR}/example.json &&
+	flux resource status --from-stdin -o "{status:>8} {nnodes}" < $INPUT \
+		>status.output &&
+	cat <<-EOF >status.expected &&
+	  STATUS NNODES
+	  online 4
+	 offline 1
+	EOF
+	test_cmp status.expected status.output
+'
+
+test_expect_success 'flux-resource status: {up} field works' '
+	INPUT=${INPUTDIR}/example.json &&
+	flux resource status --from-stdin -o "{up:>2} {nnodes}" < $INPUT \
+		>up.output &&
+	cat <<-EOF >up.expected &&
+	UP NNODES
+	 ✔ 4
+	 ✗ 1
+	EOF
+	test_cmp up.expected up.output
+'
+
+test_expect_success 'flux-resource status: {up.ascii} field works' '
+	INPUT=${INPUTDIR}/example.json &&
+	flux resource status --from-stdin -o "{up.ascii:>2} {nnodes}" < $INPUT \
+		>up.ascii.output &&
+	cat <<-EOF >up.ascii.expected &&
+	UP NNODES
+	 y 4
+	 n 1
+	EOF
+	test_cmp up.ascii.expected up.ascii.output
+'
+
+test_expect_success 'flux-resource status: {color_up} works' '
+	INPUT=${INPUTDIR}/example.json &&
+	flux resource status -n --color --states=offline --from-stdin \
+		-o "{color_up}{nnodes}{color_off}" \
+		<$INPUT >color.out &&
+	test_debug "cat color.out" &&
+	grep "^.*1" color.out &&
+	flux resource status -n --color --states=avail --from-stdin \
+		-o "{color_up}{nnodes}{color_off}" \
+		<$INPUT >color.out &&
+	test_debug "cat color.out" &&
+	grep "^.*2" color.out
+'
+
+test_expect_success 'flux-resource status: similar lines are combined' '
+	INPUT=${INPUTDIR}/drain.json &&
+	flux resource status -n --color --states=drain --from-stdin \
+		--skip-empty \
+		-o "{state:>8} {nnodes}" \
+		<$INPUT >combined.out &&
+	test_debug "cat combined.out" &&
+	test $(wc -l < combined.out) -eq 1 &&
+	flux resource status -n --color --states=drain --from-stdin \
+		--skip-empty \
+		-o "{state:>8} {nnodes:>6} {reason}" \
+		<$INPUT >combined2.out &&
+	test_debug "cat combined2.out" &&
+	test $(wc -l < combined2.out) -eq 2
+'
+
+test_expect_success 'flux-resource status: lines are combined based on format' '
+	INPUT=${INPUTDIR}/drain.json &&
+	flux resource status -n --color --states=drain --from-stdin \
+		--skip-empty \
+		-o "{timestamp} {state:>8} {nnodes}" \
+		<$INPUT >ts-detailed.out &&
+	test_debug "cat ts-detailed.out" &&
+	test $(wc -l < ts-detailed.out) -eq 2 &&
+	flux resource status -n --color --states=drain --from-stdin \
+		--skip-empty \
+		-o "{timestamp!d:%F::<12} {state:>8} {nnodes}" \
+		<$INPUT >ts-detailed2.out &&
+	test_debug "cat ts-detailed2.out" &&
+	cat <<-EOF >ts-detailed2.expected &&
+	2020-12-09    drained 2
+	EOF
+	test_cmp ts-detailed2.expected ts-detailed2.out
+'
+test_expect_success 'flux-resource status: --include works with ranks' '
+	INPUT=${INPUTDIR}/drain.json &&
+	flux resource status --include=1,3 --from-stdin \
+		-no "{nnodes}" <$INPUT >drain-include.out &&
+	test_debug "cat drain-include.out" &&
+	test "$(cat drain-include.out)" = "2"
+'
+test_expect_success 'flux-resource status: --include works with hostnames' '
+	INPUT=${INPUTDIR}/drain.json &&
+	flux resource status --include=foo[1,3] --from-stdin \
+		-no "{nodelist}" <$INPUT >drain-include-host.out &&
+	test_debug "cat drain-include-host.out" &&
+	test "$(cat drain-include-host.out)" = "foo[1,3]"
+'
+test_expect_success 'flux-resource status: --include works with invalid host' '
+	INPUT=${INPUTDIR}/drain.json &&
+	flux resource status --include=foo7 --from-stdin \
+		-no "{nodelist}" <$INPUT >drain-empty.out 2>&1 &&
+	test_must_be_empty drain-fail.out
+'
+test_expect_success 'flux-resource status: all torpid ranks shown without drain states' '
+	INPUT=${INPUTDIR}/torpid.json &&
+	flux resource status -s torpid --from-stdin < $INPUT &&
+	flux resource status -s torpid --from-stdin \
+		-no "{state} {nodelist}" < $INPUT >torpid-only.out 2>&1 &&
+	cat <<-EOF >torpid-only.expected &&
+	torpid foo[2,4]
+	EOF
+	test_cmp torpid-only.expected torpid-only.out
+'
+test_done
diff --git a/t/t2352-resource-cmd-config.t b/t/t2352-resource-cmd-config.t
new file mode 100755
index 000000000000..0f5e6d7cf8f5
--- /dev/null
+++ b/t/t2352-resource-cmd-config.t
@@ -0,0 +1,71 @@
+#!/bin/sh
+
+test_description='flux-resource config tests'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 job
+
+for cmd in list status drain; do
+    test_expect_success "flux-resource ${cmd} --format=invalid fails" '
+	test_must_fail flux resource ${cmd} --format=invalid
+    '
+done
+
+#  We can't add configuration to ~/.config or /etc/xdg/flux, so
+#   just test that XDG_CONFIG_HOME works.
+#
+test_expect_success 'Create flux-resource.toml config file' '
+        mkdir -p dir/flux &&
+        cat <<-EOF >dir/flux/flux-resource.toml
+	[status.formats.myformat]
+	description = "my status format"
+	format = "{nodelist}"
+	[list.formats.myformat]
+	description = "my list format"
+	format = "{state:>8.8} {nodelist}"
+	[drain.formats.myformat]
+	description = "my drain format"
+	format = "{state} {reason}"
+	EOF
+'
+
+for cmd in list status drain; do
+    test_expect_success "flux-resource ${cmd} accepts configured formats" '
+        XDG_CONFIG_HOME="$(pwd)/dir" \
+		flux resource ${cmd} --format=help > ${cmd}-help.out &&
+	test_debug "cat ${cmd}-help.out" &&
+	grep myformat ${cmd}-help.out &&
+        XDG_CONFIG_HOME="$(pwd)/dir" \
+		flux resource ${cmd} --format=get-config=myformat \
+		>${cmd}-get-config.out &&
+	test_debug "cat ${cmd}-get-config.out" &&
+	grep ${cmd}\.formats\.myformat ${cmd}-get-config.out &&
+        XDG_CONFIG_HOME="$(pwd)/dir" \
+		flux resource ${cmd}  --format=myformat
+    '
+done
+
+test_expect_success 'flux-resource invalid key in subtable raises error' '
+        cat <<-EOF >dir/flux/flux-resource.toml &&
+	list.formats.foo = "nothing"
+	EOF
+	( export XDG_CONFIG_HOME="$(pwd)/dir" &&
+	  test_must_fail flux resource list --format=help >invalidkey.out 2>&1
+	) &&
+	test_debug "cat invalidkey.out" &&
+	grep "flux-resource.toml" invalidkey.out
+'
+test_expect_success 'flux-resource invalid config key raises error' '
+        cat <<-EOF >dir/flux/flux-resource.toml &&
+	invalid.formats.foo.format = "nothing"
+	EOF
+	( export XDG_CONFIG_HOME="$(pwd)/dir" &&
+	  test_must_fail flux resource list --format=help >invalidkey.out 2>&1
+	) &&
+	test_debug "cat invalidkey.out" &&
+	grep "flux-resource.toml" invalidkey.out
+'
+test_done
diff --git a/t/t2353-resource-eventlog.t b/t/t2353-resource-eventlog.t
new file mode 100755
index 000000000000..a183d5819e14
--- /dev/null
+++ b/t/t2353-resource-eventlog.t
@@ -0,0 +1,61 @@
+#!/bin/sh
+
+test_description='Test resource eventlog upgrade
+
+Use the actual eventlog from test systems to see how the
+upgrade to v0.62.0 is going to go.
+'
+
+. `dirname $0`/sharness.sh
+
+export TEST_UNDER_FLUX_QUORUM=1
+export TEST_UNDER_FLUX_START_MODE=leader # only start rank 0
+
+SIZE=16384  # large instance to accommodate all test input
+test_under_flux $SIZE system
+
+test_expect_success 'test size is correct' '
+	test $(flux getattr size) -eq $SIZE
+'
+test_expect_success 'everything is down but rank 0' '
+	flux uptime | grep "$(($SIZE-1)) offline"
+'
+test_expect_success 'unload scheduler' '
+	flux module remove sched-simple
+'
+
+for eventlog in ${SHARNESS_TEST_SRCDIR}/resource/resource.eventlog.*; do
+	filename=$(basename $eventlog)
+	test_expect_success "load test resources $filename" '
+		flux dmesg -C &&
+		flux kvs put --raw resource.eventlog=- <$eventlog &&
+		flux module reload resource noverify
+	'
+	test_expect_success 'eventlog upgrade reduced entry count' '
+		flux dmesg | grep "resource.eventlog: reduced"
+	'
+	test_expect_success 'reloading resource module does not rewrite log' '
+		flux dmesg -C &&
+		flux module reload resource noverify &&
+		flux dmesg >dmesg.out &&
+		test_must_fail grep "resource.eventlog: reduced" dmesg.out
+	'
+	test_expect_success 'parse eventlog entry names' '
+		flux kvs get --raw resource.eventlog >$filename &&
+		jq -r -e ".name" <$filename >$filename.names
+	'
+	test_expect_success 'eventlog now contains no legacy events' '
+		test_must_fail grep -E "offline|online|resource-init" \
+		    $filename.names
+	'
+	# one orig resource-define plus two resource loads in the test
+	test_expect_success 'eventlog contains 3 resource-define events' '
+		count=$(grep "resource-define" $filename.names | wc -l) &&
+		test $count -eq 3
+	'
+done
+test_expect_success 'load scheduler' '
+	flux module load sched-simple
+'
+
+test_done
diff --git a/t/t2354-resource-status.t b/t/t2354-resource-status.t
new file mode 100755
index 000000000000..7013095f89c5
--- /dev/null
+++ b/t/t2354-resource-status.t
@@ -0,0 +1,68 @@
+#!/bin/sh
+
+test_description='flux-resource status general tests'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4
+
+export FLUX_PYCLI_LOGLEVEL=10
+
+test_expect_success 'flux-resource status: works' '
+	flux resource status
+'
+test_expect_success 'flux-resource status: --states=help reports valid states' '
+	flux resource status --states=help >status-help.out 2>&1 &&
+	test_debug "cat status-help.out" &&
+	grep "valid states:" status-help.out
+'
+test_expect_success 'flux-resource status: bad --states reports valid states' '
+	test_expect_code 1 flux resource status --states=foo >status-bad.out 2>&1 &&
+	test_debug "cat status-bad.out" &&
+	grep "valid states:" status-bad.out
+'
+test_expect_success 'flux-resource status -s all shows all expected states' '
+	flux resource status -s all >status-all.out &&
+	test_debug "cat status-all.out" &&
+	grep avail status-all.out &&
+	grep exclude status-all.out &&
+	grep torpid status-all.out &&
+	grep allocated status-all.out &&
+	grep draining status-all.out &&
+	grep drained status-all.out
+'
+test_expect_success 'flux-resource status: allocated set is not displayed by default' '
+	flux submit -N1 --wait-event=alloc sleep inf &&
+	flux resource status >status-noalloc.out &&
+	test_must_fail grep allocated status-noalloc.out
+'
+test_expect_success 'flux-resource status: allocated nodes can be displayed' '
+	flux resource status -s allocated | grep allocated
+'
+test_expect_success 'cancel running jobs' '
+	flux cancel --all &&
+	flux queue idle
+'
+test_expect_success 'flux-resource status shows housekeeping by default' '
+	flux config load <<-EOF &&
+	[job-manager.housekeeping]
+	release-after = "0"
+	command = [ "sleep", "inf" ]
+	EOF
+	flux run -N4 hostname &&
+	test_debug "flux housekeeping list" &&
+	test_debug "flux resource status" &&
+	flux resource status | grep housekeeping &&
+	test $(flux resource status -s housekeeping -no {nnodes}) -eq 4
+'
+test_expect_success 'flux-resource status works after partial release' '
+	flux housekeeping kill --targets=0-1 &&
+	test_debug "flux housekeeping list" &&
+	test_debug "flux resource status" &&
+	flux resource status | grep housekeeping &&
+	test $(flux resource status -s housekeeping -no {ranks}) = "2-3"
+'
+test_expect_success 'stop housekeeping tasks' '
+	flux housekeeping kill --all
+'
+test_done
diff --git a/t/t2400-job-exec-test.t b/t/t2400-job-exec-test.t
index 59bc4baf0274..df21eea8e7b6 100755
--- a/t/t2400-job-exec-test.t
+++ b/t/t2400-job-exec-test.t
@@ -6,28 +6,29 @@ test_description='Test flux job execution service in simulated mode'
 
 test_under_flux 1 job
 
-flux setattr log-stderr-level 1
+flux version | grep -q libflux-security && test_set_prereq FLUX_SECURITY
 
-#  Set path to jq(1)
-#
-jq=$(which jq 2>/dev/null)
-if test -z "$jq"; then
-    skip_all='jq not found. Skipping all tests'
-    test_done
-fi
+flux setattr log-stderr-level 1
 
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
 
 job_kvsdir()    { flux job id --to=kvs $1; }
 exec_eventlog() { flux kvs get -r $(job_kvsdir $1).guest.exec.eventlog; }
-exec_test()     { ${jq} '.attributes.system.exec.test = {}'; }
-exec_testattr() {
-	${jq} --arg key "$1" --arg value $2 \
-	      '.attributes.system.exec.test[$key] = $value'
+
+submit_as_alternate_user()
+{
+        FAKE_USERID=42
+        flux run --dry-run "$@" | \
+          flux python ${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py $FAKE_USERID \
+            >job.signed
+        FLUX_HANDLE_USERID=$FAKE_USERID \
+          flux job submit --flags=signed job.signed
 }
 
 test_expect_success 'job-exec: generate jobspec for simple test job' '
-        flux jobspec srun -n1 hostname | exec_test > basic.json
+	flux run \
+	    --setattr=system.exec.test.run_duration=0.0001s \
+	    --dry-run hostname > basic.json
 '
 test_expect_success 'job-exec: basic job runs in simulated mode' '
 	jobid=$(flux job submit basic.json) &&
@@ -52,10 +53,10 @@ test_expect_success 'job-exec: exec.eventlog exists with expected states' '
 	tail -1 eventlog.1.out | grep "done"
 '
 test_expect_success 'job-exec: canceling job during execution works' '
-	jobid=$(flux jobspec srun -t 1 hostname | \
-		 exec_test | flux job submit) &&
+	jobid=$(flux submit \
+                --setattr=system.exec.test.run_duration=10s hostname) &&
 	flux job wait-event -vt 2.5 ${jobid} start &&
-	flux job cancel ${jobid} &&
+	flux cancel ${jobid} &&
 	flux job wait-event -t 2.5 ${jobid} exception &&
 	flux job wait-event -t 2.5 ${jobid} finish | grep status=15 &&
 	flux job wait-event -t 2.5 ${jobid} release &&
@@ -63,9 +64,8 @@ test_expect_success 'job-exec: canceling job during execution works' '
 	exec_eventlog $jobid | grep "complete" | grep "\"status\":15"
 '
 test_expect_success 'job-exec: mock exception during initialization' '
-	flux jobspec srun hostname | \
-	  exec_testattr mock_exception init > ex1.json &&
-	jobid=$(flux job submit ex1.json) &&
+	jobid=$(flux submit \
+	         --setattr=system.exec.test.mock_exception=init true) &&
 	flux job wait-event -t 2.5 ${jobid} exception > exception.1.out &&
 	test_debug "flux job eventlog ${jobid}" &&
 	grep "type=\"exec\"" exception.1.out &&
@@ -75,9 +75,8 @@ test_expect_success 'job-exec: mock exception during initialization' '
 	test_must_fail grep "finish" eventlog.${jobid}.out
 '
 test_expect_success 'job-exec: mock exception during run' '
-	flux jobspec srun hostname | \
-	  exec_testattr mock_exception run > ex2.json &&
-	jobid=$(flux job submit ex2.json) &&
+	jobid=$(flux submit \
+	         --setattr=system.exec.test.mock_exception=run true) &&
 	flux job wait-event -t 2.5 ${jobid} exception > exception.2.out &&
 	grep "type=\"exec\"" exception.2.out &&
 	grep "mock run exception generated" exception.2.out &&
@@ -85,32 +84,140 @@ test_expect_success 'job-exec: mock exception during run' '
 	flux job eventlog ${jobid} > eventlog.${jobid}.out &&
 	grep "finish status=15" eventlog.${jobid}.out
 '
-test_expect_success 'job-exec: simulate epilog/cleanup tasks' '
-	flux jobspec srun hostname | \
-	  exec_testattr cleanup_duration 0.01s > cleanup.json &&
-	jobid=$(flux job submit cleanup.json) &&
-	flux job wait-event -vt 2.5 ${jobid} clean &&
-	exec_eventlog $jobid > exec.eventlog.$jobid &&
-	grep "cleanup\.start" exec.eventlog.$jobid &&
-	grep "cleanup\.finish" exec.eventlog.$jobid
-'
-#
-# XXX: Trying to generate an exception during cleanup is racy, however,
-#  there is not currently another way to do this until we have a *real*
-#  epilog script which could be triggered by events. If this test becomes
-#  a problem, we can disable it until that time.
-#
-test_expect_success 'job-exec: exception during cleanup' '
-	flux jobspec srun hostname | \
-	  exec_testattr cleanup_duration 1s > cleanup-long.json &&
-	jobid=$(flux job submit cleanup-long.json) &&
-	flux job wait-event -vt 2.5 ${jobid} finish &&
-	flux job cancel ${jobid} &&
-	flux job wait-event -t 10 ${jobid} clean &&
-	exec_eventlog $jobid > exec.eventlog.$jobid &&
-	grep "cleanup\.finish" exec.eventlog.$jobid
-'
 test_expect_success 'start request with empty payload fails with EPROTO(71)' '
 	${RPC} job-exec.start 71 </dev/null
 '
+test_expect_success 'job-exec: invalid testexec conf generates exception' '
+	jobid=$(flux submit \
+	    --setattr=system.exec.test.run_duration=0.01 hostname) &&
+	flux job wait-event -t 5 ${jobid} exception > except.invalid.out &&
+	grep "type=\"exec\"" except.invalid.out &&
+	flux job wait-event -qt 5 ${jobid} clean &&
+	flux job eventlog ${jobid}
+'
+test_expect_success 'job-exec: test exec start override works' '
+	jobid=$(flux submit \
+	    --setattr=system.exec.test.override=1 \
+	    --setattr=system.exec.test.run_duration=0.001s \
+	    true) &&
+	flux job wait-event -t 5 ${jobid} alloc &&
+	test_must_fail flux job wait-event -t 0.1 ${jobid} start &&
+	flux job-exec-override start ${jobid} &&
+	flux job wait-event -t 5 ${jobid} start &&
+	flux job wait-event -t 5 -v ${jobid} clean
+'
+test_expect_success 'job-exec: override only works on jobs with flag set' '
+	jobid=$(flux submit \
+		--setattr=system.exec.test.run_duration=0. true) &&
+	flux job wait-event -t 5 ${jobid} alloc &&
+	test_must_fail flux job-exec-override start ${jobid} &&
+	flux cancel ${jobid} &&
+	flux job wait-event -t 5 -v ${jobid} clean
+'
+test_expect_success 'job-exec: test exec start/finish override works' '
+	jobid=$(flux submit \
+	    --setattr=system.exec.test.override=1 \
+	    true) &&
+	flux job wait-event -t 5 ${jobid} alloc &&
+	test_must_fail flux job wait-event -t 0.1 ${jobid} start &&
+	test_must_fail flux job-exec-override finish ${jobid} 0 &&
+	flux job-exec-override start ${jobid} &&
+	flux job wait-event -t 5 ${jobid} start &&
+	test_must_fail flux job-exec-override start ${jobid} &&
+	test_must_fail flux job wait-event -t 0.1 ${jobid} finish &&
+	flux job-exec-override finish ${jobid} 0 &&
+	flux job wait-event -t 5 -v ${jobid} clean
+'
+test_expect_success 'job-exec: flux job-exec-override fails on invalid id' '
+	test_must_fail flux job-exec-override start 1234  &&
+	test_must_fail flux job-exec-override finish 1234  0
+'
+test_expect_success 'job-exec: job-exec.testoverride invalid request' '
+	cat <<-EOF >override.py &&
+	import json
+	import sys
+	import flux
+	h = flux.Flux()
+	try:
+	    print(h.rpc("job-exec.override", json.load(sys.stdin)).get())
+	except OSError as exc:
+	    print(str(exc), file=sys.stderr)
+	    sys.exit(1)
+	EOF
+	echo {} | \
+	  test_must_fail flux python override.py &&
+	jobid=$(flux submit \
+	    --setattr=system.exec.test.override=1 \
+	    true) &&
+	cat <<-EOF >badevent.json &&
+	{"jobid":"$(flux job id --to=dec ${jobid})", "event":"foo"}
+	EOF
+	test_must_fail flux python override.py < badevent.json &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean
+'
+test_expect_success 'job-exec: flux job-exec-override fails for invalid userid' '
+	jobid=$(flux submit \
+	    --setattr=system.exec.test.override=1 \
+	    true) &&
+	newid=$(($(id -u)+1)) &&
+	( export FLUX_HANDLE_ROLEMASK=0x2 &&
+	  export FLUX_HANDLE_USERID=$newid &&
+	    test_must_fail flux job-exec-override start ${jobid}
+	) &&
+	flux cancel ${jobid} &&
+	flux job wait-event -t 5 -v ${jobid} clean
+'
+test_expect_success 'job-exec: critical-ranks RPC handles unexpected input' '
+	cat <<-EOF >critical-ranks.py &&
+	import sys
+	import flux
+	from flux.job import JobID
+	h = flux.Flux()
+	h.rpc("job-exec.critical-ranks",
+	      {"id": JobID(sys.argv[1]), "ranks": sys.argv[2]}).get()
+	EOF
+	test_must_fail flux python critical-ranks.py 1234 0-1 &&
+	id=$(flux submit --wait-event=start sleep 300) &&
+	test_must_fail flux python critical-ranks.py $id foo &&
+	newid=$(($(id -u)+1)) &&
+        ( export FLUX_HANDLE_ROLEMASK=0x2 &&
+          export FLUX_HANDLE_USERID=$newid &&
+            test_must_fail flux python critical-ranks.py $id 0
+        )
+'
+test_expect_success FLUX_SECURITY 'job-exec: guests denied access to test exec' '
+	jobid=$(submit_as_alternate_user \
+		--setattr=exec.test.run_duration=1s hostname) &&
+	flux job wait-event -Hvt 15s $jobid exception &&
+	flux job wait-event -Ht 15s $jobid exception >testexec-denied.out &&
+	grep "guests may not use test" testexec-denied.out
+'
+test_expect_success FLUX_SECURITY \
+	'job-exec: guest access to test exec can be configured' '
+	flux config load <<-EOF &&
+	[exec.testexec]
+	allow-guests = true
+	EOF
+	jobid=$(submit_as_alternate_user \
+		--setattr=exec.test.run_duration=0.1s hostname) &&
+	flux job wait-event -vHt 15s $jobid clean &&
+	flux job status -vvv $jobid
+'
+
+if ! grep 'release 7' /etc/centos-release >/dev/null 2>&1 \
+   && ! grep 'release 7' /etc/redhat-release >/dev/null 2>&1
+then
+   test_set_prereq NOT_DISTRO7
+fi
+
+# The following test does not work on CentOS 7 / RHEL7 since exec errno does
+#  not work with job-exec (for as yet unknown reason). Skip the test on
+#  these distros:
+test_expect_success NOT_DISTRO7 'job-exec: path to shell is emitted on exec error' '
+	test_expect_code 127 flux run \
+	  --setattr=exec.job_shell=/foo/flux-shell hostname 2>exec.err &&
+	test_debug "cat exec.err" &&
+	grep /foo/flux-shell exec.err
+'
 test_done
diff --git a/t/t2401-job-exec-hello.t b/t/t2401-job-exec-hello.t
index 33620389869d..12eb1e377a61 100755
--- a/t/t2401-job-exec-hello.t
+++ b/t/t2401-job-exec-hello.t
@@ -17,18 +17,18 @@ test_expect_success 'exec hello: remove job-exec module' '
 test_expect_success NO_CHAIN_LINT 'exec hello: start exec server-in-a-script' '
 	${execservice} test-exec > server1.log &
 	SERVER1=$! &&
-	id=$(flux jobspec srun hostname | flux job submit --flags=debug) &&
+	id=$(flux submit --flags=debug hostname) &&
 	flux job wait-event ${id} clean &&
 	test_debug "cat server1.log" &&
-	grep "start: ${id}" server1.log
+	grep "start: $(flux job id ${id})" server1.log
 '
 test_expect_success NO_CHAIN_LINT 'exec hello: takeover: start new service' '
 	${execservice} test-exec2 > server2.log &
 	SERVER2=$! &&
 	flux kvs get -c 1 --watch --waitcreate test.exec-hello.test-exec2 &&
-	id=$(flux jobspec srun hostname | flux job submit --flags=debug) &&
+	id=$(flux submit --flags=debug hostname) &&
 	flux job wait-event ${id} clean &&
-	grep "start: ${id}" server2.log
+	grep "start: $(flux job id ${id})" server2.log
 '
 test_expect_success NO_CHAIN_LINT 'exec hello: teardown existing servers' "
 	kill -9 ${SERVER1} && kill -9 ${SERVER2} &&
@@ -38,7 +38,7 @@ test_expect_success NO_CHAIN_LINT 'exec hello: teardown existing servers' "
 	test_must_fail grep test-exec module.list
 "
 test_expect_success NO_CHAIN_LINT 'exec hello: job start events are paused' '
-	id=$(flux jobspec srun hostname | flux job submit --flags=debug) &&
+	id=$(flux submit --flags=debug hostname) &&
 	flux job wait-event -vt 5 ${id} alloc &&
 	test_debug "flux job eventlog ${id}" &&
 	test $(lastevent ${id}) = "debug.start-lost"
@@ -51,13 +51,13 @@ test_expect_success NO_CHAIN_LINT 'exec hello: start server with job timer' '
 test_expect_success NO_CHAIN_LINT 'exec hello: paused job now has start event' '
 	flux job wait-event -t 2 ${id} start &&
 	test_debug "cat server3.log" &&
-	grep "test-exec3: start: ${id}" server3.log
+	grep "test-exec3: start: $(flux job id ${id})" server3.log
 '
 test_expect_success NO_CHAIN_LINT 'exec hello: hello now returns error due to running job' '
 	test_expect_code 1 run_timeout 5 ${execservice} testexecfoo
 '
 test_expect_success NO_CHAIN_LINT 'exec hello: terminate all jobs and servers' '
-	flux job cancel ${id} &&
+	flux cancel ${id} &&
 	test_debug "cat server3.log" &&
 	flux job wait-event -t 2.5 ${id} clean &&
 	kill ${SERVER3}
diff --git a/t/t2402-job-exec-dummy.t b/t/t2402-job-exec-dummy.t
index 19465e0e9822..cd33e69f4745 100755
--- a/t/t2402-job-exec-dummy.t
+++ b/t/t2402-job-exec-dummy.t
@@ -4,12 +4,6 @@ test_description='Test flux job execution service with dummy job shell'
 
 . $(dirname $0)/sharness.sh
 
-jq=$(which jq 2>/dev/null)
-if test -z "$jq" ; then
-	skip_all 'no jq found, skipping all tests'
-	test_done
-fi
-
 #  Configure dummy job shell:
 if ! test -f dummy.toml; then
 	cat <<-EOF >dummy.toml
@@ -28,9 +22,8 @@ exec_eventlog() { flux kvs get -r $(job_kvsdir $1).guest.exec.eventlog; }
 
 
 test_expect_success 'job-exec: execute dummy job shell across all ranks' '
-	id=$(flux jobspec srun -N4 \
-	    "flux kvs put test1.\$BROKER_RANK=\$JOB_SHELL_RANK" \
-	    | flux job submit) &&
+	id=$(flux submit -n4 -N4 \
+		"flux kvs put test1.\$BROKER_RANK=\$JOB_SHELL_RANK") &&
 	flux job wait-event $id clean &&
 	kvsdir=$(flux job id --to=kvs $id).guest &&
 	test $(flux kvs get ${kvsdir}.test1.0) = 0 &&
@@ -38,33 +31,32 @@ test_expect_success 'job-exec: execute dummy job shell across all ranks' '
 	test $(flux kvs get ${kvsdir}.test1.2) = 2 &&
 	test $(flux kvs get ${kvsdir}.test1.3) = 3
 '
-test_expect_success 'job-exec: job shell output sent to flux log' '
-	id=$(flux jobspec srun -n 1 "echo Hello from job \$JOBID" \
-	     | flux job submit) &&
-	flux job wait-event $id clean &&
-	flux dmesg | grep "Hello from job $id"
+test_expect_success 'job-exec: job shell output available in flux-job attach' '
+	id=$(flux submit "echo Hello from job \$JOBID") &&
+	flux job attach -vEX $id &&
+	flux job attach $id 2>&1 | grep "dummy.sh.*Hello from job"
 '
 test_expect_success 'job-exec: job shell failure recorded' '
-	id=$(flux jobspec srun -N4  "test \$JOB_SHELL_RANK = 0 && exit 1" \
-	     | flux job submit) &&
+	id=$(flux submit -n4 -N4  "test \$JOB_SHELL_RANK = 0 && exit 1") &&
 	flux job wait-event -vt 10 $id finish | grep status=256
 '
 test_expect_success 'job-exec: status is maximum job shell exit codes' '
-	id=$(flux jobspec srun -N4 "exit \$JOB_SHELL_RANK" | flux job submit) &&
+	id=$(flux submit -n4 -N4 "exit \$JOB_SHELL_RANK") &&
 	flux job wait-event -vt 10 $id finish | grep status=768
 '
 test_expect_success 'job-exec: job exception kills job shells' '
-	id=$(flux jobspec srun -N4 sleep 300 | flux job submit) &&
+	id=$(flux submit -n4 -N4 sleep 300) &&
 	flux job wait-event -vt 5 $id start &&
-	flux job cancel $id &&
+	flux job wait-event -vt 5 -p exec $id shell.start &&
+	flux cancel $id &&
 	flux job wait-event -vt 5 $id clean &&
-	flux job eventlog $id | grep status=15
+	flux job eventlog $id | grep -E "status=(15|36608)"
 '
 test_expect_success 'job-exec: job exception uses SIGKILL after kill-timeout' '
 	flux module reload job-exec kill-timeout=0.2 &&
 	cat <<-EOF >trap-sigterm.sh &&
 	#!/bin/sh
-	trap "echo trap-sigterm got SIGTERM" 15
+	trap "echo trap-sigterm got SIGTERM >&2" 15
 	flux kvs put trap-ready=1
 	sleep 60 &
 	pid=\$!
@@ -72,35 +64,59 @@ test_expect_success 'job-exec: job exception uses SIGKILL after kill-timeout' '
 	sleep 60
 	EOF
 	chmod +x trap-sigterm.sh &&
-	id=$(TRAP=15 flux jobspec srun -N4 ./trap-sigterm.sh \
-	        | flux job submit) &&
+	id=$(TRAP=15 flux submit -n4 -N4 ./trap-sigterm.sh) &&
 	flux job wait-event -vt 5 $id start &&
 	flux kvs get --waitcreate \
 		--namespace=$(flux job namespace $id) \
 		trap-ready &&
-	flux job cancel $id &&
-	sleep 0.2 &&
-	flux dmesg | grep $id &&
-	flux job wait-event -vt 5 $id clean &&
-	flux dmesg | grep "trap-sigterm got SIGTERM" &&
+	flux cancel $id &&
+	(flux job attach -vEX $id >kill.output 2>&1 || true) &&
+	test_debug "cat kill.output" &&
+	grep "trap-sigterm got SIGTERM" kill.output
+'
+test_expect_success 'job-exec: job shell eventually killed by SIGKILL' '
+	id=$(flux submit --wait-event=start -n1 \
+	     sh -c "trap \"\" SIGTERM;
+                    flux kvs put ready=1;
+	            while true; do sleep 1; done") &&
+	flux kvs get --waitcreate \
+		--namespace=$(flux job namespace $id) \
+		ready &&
+	flux cancel $id &&
+	flux job wait-event -vt 15 $id clean &&
+	flux dmesg | grep $(flux job id --to=f58 $id) &&
+	test_expect_code 137 flux job status $id &&
 	flux module reload job-exec
 '
 test_expect_success 'job-exec: invalid job shell generates exception' '
-	id=$(flux jobspec srun -N1 /bin/true \
-	     | $jq ".attributes.system.exec.job_shell = \"/notthere\"" \
-	     | flux job submit) &&
+	id=$(flux run --dry-run true \
+		| $jq ".attributes.system.exec.job_shell = \"/notthere\"" \
+		| flux job submit) &&
 	flux job wait-event -vt 5 $id clean
 '
+test_expect_success 'job-exec: invalid bulkexec key in jobspec raises error' '
+	flux dmesg -C &&
+	test_must_fail flux run --setattr=exec.bulkexec.foo=bar hostname &&
+	flux dmesg -H | grep "failed to unpack system.exec.bulkexec"
+'
 test_expect_success 'job-exec: exception during init terminates job' '
-	id=$(flux jobspec srun -N2 sleep 30 \
-	     | $jq ".attributes.system.exec.bulkexec.mock_exception = \"init\"" \
-	     | flux job submit) &&
+	id=$(flux run --dry-run -n2 -N2 sleep 30 \
+		| $jq ".attributes.system.exec.bulkexec.mock_exception = \"init\"" \
+		| flux job submit) &&
 	flux job wait-event -vt 5 $id clean
 '
 test_expect_success 'job-exec: exception while starting terminates job' '
-	id=$(flux jobspec srun -N2 sleep 30 \
-	     | $jq ".attributes.system.exec.bulkexec.mock_exception = \"starting\"" \
-	     | flux job submit) &&
+	id=$(flux run --dry-run -n2 -N2 sleep 30 \
+		| $jq ".attributes.system.exec.bulkexec.mock_exception = \"starting\"" \
+		| flux job submit) &&
 	flux job wait-event -vt 5 $id clean
 '
+test_expect_success 'job-exec: failure after first barrier terminates job' '
+	for id in $(flux bulksubmit --env FAIL_MODE={} -N2 -n2 sleep 300 \
+		    ::: before_barrier_entry after_barrier_entry); do
+		echo checking on job $id &&
+		flux job wait-event -vt 600 $id clean &&
+		test_must_fail flux job status -v $id
+	done
+'
 test_done
diff --git a/t/t2403-job-exec-conf.t b/t/t2403-job-exec-conf.t
index 64777e9b740c..c2b284c77e5f 100755
--- a/t/t2403-job-exec-conf.t
+++ b/t/t2403-job-exec-conf.t
@@ -4,12 +4,14 @@ test_description='Test flux job exec configuration'
 
 . $(dirname $0)/sharness.sh
 
+export FLUX_CONF_DIR=$(pwd)/conf.d
+mkdir -p conf.d
 test_under_flux 1 job
 
 test_expect_success 'job-exec: can specify kill-timeout on cmdline' '
 	flux dmesg -C &&
 	flux module reload job-exec kill-timeout=1m &&
-	flux dmesg | grep "using kill-timeout of 60s"
+	flux module stats job-exec | jq ".[\"kill-timeout\"] == 60"
 '
 test_expect_success 'job-exec: bad kill-timeout value causes module failure' '
 	flux dmesg -C &&
@@ -18,69 +20,341 @@ test_expect_success 'job-exec: bad kill-timeout value causes module failure' '
 '
 test_expect_success 'job-exec: kill-timeout can be set in exec conf' '
 	name=killconf &&
-	mkdir ${name}.d &&
-	cat <<-EOF > ${name}.d/exec.toml &&
+	cat <<-EOF > ${name}.toml &&
 	[exec]
 	kill-timeout = ".5m"
 	EOF
-	( export FLUX_CONF_DIR=${name}.d &&
-	  flux start -s1 flux dmesg > ${name}.log 2>&1
-	) &&
-	grep "using kill-timeout of 30s" ${name}.log
+	flux start --config-path=${name}.toml -s1 \
+		flux module stats job-exec > ${name}.json 2>&1 &&
+	test_debug "jq -S . < ${name}.json" &&
+	cat ${name}.json | jq ".[\"kill-timeout\"] == 30"
+'
+test_expect_success 'job-exec: kill-timeout can be set with a config reload' '
+	flux module load job-exec &&
+	flux dmesg -C &&
+	echo exec.kill-timeout=\".5m\" | flux config load &&
+	flux module stats job-exec | jq ".[\"kill-timeout\"] == 30"
 '
 test_expect_success 'job-exec: bad kill-timeout config causes module failure' '
 	name=bad-killconf &&
-	mkdir ${name}.d &&
-	cat <<-EOF > ${name}.d/exec.toml &&
+	cat <<-EOF > ${name}.toml &&
 	[exec]
 	kill-timeout = "foo"
 	EOF
-	( export FLUX_CONF_DIR=${name}.d &&
-	  test_must_fail flux start -s1 flux dmesg > ${name}.log 2>&1
-	) &&
+	test_must_fail flux start --config-path=${name}.toml -s1 \
+		flux dmesg > ${name}.log 2>&1 &&
 	grep "invalid kill-timeout: foo" ${name}.log
 '
-test_expect_success 'job-exec: can specify default-shell on cmdline' '
+test_expect_success 'job-exec: bad kill-timeout causes config reload failure' '
+	echo exec.kill-timeout=\"foo\" | test_must_fail flux config load
+'
+test_expect_success 'job-exec: cmdline kill-timeout takes priority' '
+	flux module reload -f job-exec kill-timeout=1m &&
+	flux module stats job-exec | jq ".[\"kill-timeout\"] == 60" &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	kill-timeout = ".5m"
+	EOF
+	flux config reload &&
+	flux module stats job-exec | jq ".[\"kill-timeout\"] == 60" &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
+'
+test_expect_success 'job-exec: reset config' '
+	echo | flux config load
+'
+test_expect_success 'job-exec: can specify term-signal on cmdline' '
 	flux dmesg -C &&
+	flux module reload job-exec term-signal=SIGUSR1 &&
+	flux module stats job-exec | jq ".[\"term-signal\"] == \"SIGUSR1\""
+'
+test_expect_success 'job-exec: can specify kill-signal on cmdline' '
+	flux dmesg -C &&
+	flux module reload job-exec kill-signal=SIGUSR2 &&
+	flux module stats job-exec | jq ".[\"kill-signal\"] == \"SIGUSR2\""
+'
+test_expect_success 'job-exec: bad term/kill-signal value causes failure' '
+	flux dmesg -C &&
+	test_expect_code 1 flux module reload job-exec kill-signal=SIGFOO &&
+	dmesg-grep.py -vt 10 "invalid kill-signal: SIGFOO" &&
+	flux dmesg -C &&
+	test_expect_code 1 flux module reload -f job-exec term-signal=-1 &&
+	dmesg-grep.py -vt 10 "invalid term-signal: -1"
+'
+test_expect_success 'job-exec: term/kill-signal can be set in exec conf' '
+	name=sigconf &&
+	cat <<-EOF > ${name}.toml &&
+	[exec]
+	kill-signal = "SIGINT"
+	term-signal = "SIGHUP"
+	EOF
+	flux start --config-path=${name}.toml -s1 \
+		flux module stats job-exec > ${name}.json 2>&1 &&
+	cat ${name}.json | jq ".[\"kill-signal\"] == \"SIGINT\"" &&
+	cat ${name}.json | jq ".[\"term-signal\"] == \"SIGTERM\""
+'
+test_expect_success 'job-exec: bad term/kill-signal config causes module failure' '
+	name=bad-sigconf &&
+	cat <<-EOF > ${name}.toml &&
+	[exec]
+	kill-signal = "foo"
+	EOF
+	test_must_fail flux start --config-path=${name}.toml -s1 \
+		flux dmesg > ${name}.log 2>&1 &&
+	grep "invalid kill-signal: foo" ${name}.log
+'
+test_expect_success 'job-exec: cmdline term/kill signal takes priority' '
+	flux module reload -f job-exec term-signal=SIGUSR1 kill-signal=SIGUSR2 &&
+	flux module stats job-exec | jq ".[\"term-signal\"] == \"SIGUSR1\"" &&
+	flux module stats job-exec | jq ".[\"kill-signal\"] == \"SIGUSR2\"" &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	kill-signal = "SIGINT"
+	term-signal = "SIGHUP"
+	EOF
+	flux config reload &&
+	flux module stats job-exec | jq ".[\"term-signal\"] == \"SIGUSR1\"" &&
+	flux module stats job-exec | jq ".[\"kill-signal\"] == \"SIGUSR2\"" &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
+'
+test_expect_success 'job-exec: can specify default-shell on cmdline' '
 	flux module reload -f job-exec job-shell=/path/to/shell &&
-	flux dmesg &&
-	flux dmesg | grep "default shell path /path/to/shell"
+	flux module stats -p bulk-exec.config.default_job_shell job-exec > stats1.out &&
+	grep "/path/to/shell" stats1.out
 '
 test_expect_success 'job-exec: job-shell can be set in exec conf' '
 	name=shellconf &&
-	mkdir ${name}.d &&
-	cat <<-EOF > ${name}.d/exec.toml &&
+	cat <<-EOF > ${name}.toml &&
 	[exec]
 	job-shell = "my-flux-shell"
 	EOF
-	( export FLUX_CONF_DIR=${name}.d &&
-	  flux start -s1 flux dmesg > ${name}.log 2>&1
-	) &&
-	grep "using default shell path my-flux-shell" ${name}.log
+	flux start --config-path=${name}.toml -s1 \
+		flux module stats -p bulk-exec.config.default_job_shell job-exec > ${name}.out 2>&1 &&
+	grep "my-flux-shell" ${name}.out
 '
 test_expect_success 'job-exec: bad job-shell config causes module failure' '
 	name=bad-shellconf &&
-	mkdir ${name}.d &&
-	cat <<-EOF > ${name}.d/exec.toml &&
+	cat <<-EOF > ${name}.toml &&
 	[exec]
 	job-shell = 42
 	EOF
-	( export FLUX_CONF_DIR=${name}.d &&
-	  test_must_fail flux start -s1 flux dmesg > ${name}.log 2>&1
-	) &&
+	test_must_fail flux start --config-path=${name}.toml -s1 \
+		flux dmesg > ${name}.log 2>&1 &&
 	grep "error reading config value exec.job-shell" ${name}.log
 '
-test_expect_success 'job-exec: bad TOML config causes module failure' '
-	#  Need to first start an instance with good config files, then
-        #   replace with bad config files, and finally reload module
-	name=badtoml &&
-	mkdir ${name}.d &&
-	touch ${name}.d/exec.toml &&
-	( export FLUX_CONF_DIR=${name}.d &&
-	  test_must_fail flux start -s1 sh -c "\
-	    echo \[exec >${name}.d/exec.toml && flux module reload job-exec" \
-	    >${name}.log 2>&1
-	) &&
-	grep "config file error: ${name}.d/exec.toml" ${name}.log
+test_expect_success 'job-exec: update default shell via config reload' '
+	flux module reload -f job-exec &&
+	flux module stats -p bulk-exec.config.default_job_shell job-exec > reload1A.out 2>&1 &&
+	grep "flux-shell" reload1A.out &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	job-shell = "/path/reload-shell"
+	EOF
+	flux config reload &&
+	flux module stats -p bulk-exec.config.default_job_shell job-exec > reload1B.out 2>&1 &&
+	grep "reload-shell" reload1B.out &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
+'
+test_expect_success 'job-exec: cmdline default shell takes priority' '
+	flux module reload -f job-exec job-shell=/path/cmdline-shell &&
+	flux module stats -p bulk-exec.config.default_job_shell job-exec > reload2A.out 2>&1 &&
+	grep "cmdline-shell" reload2A.out &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	job-shell = "/path/reload-shell"
+	EOF
+	flux config reload &&
+	flux module stats -p bulk-exec.config.default_job_shell job-exec > reload2B.out 2>&1 &&
+	grep "cmdline-shell" reload2B.out &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
+'
+test_expect_success 'job-exec: can specify imp path on cmdline' '
+	flux module reload -f job-exec imp=/path/to/imp &&
+	flux module stats -p bulk-exec.config.flux_imp_path job-exec > stats2.out &&
+	grep "/path/to/imp" stats2.out
+'
+test_expect_success 'job-exec: imp path can be set in exec conf' '
+	name=impconf &&
+	cat <<-EOF > ${name}.toml &&
+	[exec]
+	imp = "my-flux-imp"
+	EOF
+	flux start --config-path=${name}.toml -s1 \
+		flux module stats -p bulk-exec.config.flux_imp_path job-exec > ${name}.out 2>&1 &&
+	grep "my-flux-imp" ${name}.out
+'
+test_expect_success 'job-exec: bad imp config causes module failure' '
+	name=bad-impconf &&
+	cat <<-EOF > ${name}.toml &&
+	[exec]
+	imp = 42
+	EOF
+	test_must_fail flux start --config-path=${name}.toml -s1 \
+		flux dmesg > ${name}.log 2>&1 &&
+	grep "error reading config value exec.imp" ${name}.log
+'
+# N.B. imp not configured by default
+test_expect_success 'job-exec: update imp path via config reload' '
+	flux module reload -f job-exec &&
+	test_must_fail flux module stats -p bulk-exec.config.flux_imp_path job-exec > reload3A.out 2>&1 &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	imp = "/path/reload-imp"
+	EOF
+	flux config reload &&
+	flux module stats -p bulk-exec.config.flux_imp_path job-exec > reload3B.out 2>&1 &&
+	grep "reload-imp" reload3B.out &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
+'
+test_expect_success 'job-exec: cmdline imp path takes priority' '
+	flux module reload -f job-exec imp=/path/cmdline-imp &&
+	flux module stats -p bulk-exec.config.flux_imp_path job-exec > reload4A.out 2>&1 &&
+	grep "cmdline-imp" reload4A.out &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	imp = "/path/reload-imp"
+	EOF
+	flux dmesg -C &&
+	flux config reload &&
+	flux module stats -p bulk-exec.config.flux_imp_path job-exec > reload4B.out 2>&1 &&
+	grep "cmdline-imp" reload4B.out &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
+'
+test_expect_success 'job-exec: can specify exec service on cmdline' '
+	flux module reload -f job-exec service=foo &&
+	flux module stats -p bulk-exec.config.exec_service job-exec > stats3.out &&
+	grep "foo" stats3.out
+'
+test_expect_success 'job-exec: exec service can be set in exec conf' '
+	name=exec-service &&
+	cat <<-EOF > ${name}.toml &&
+	[exec]
+	service = "bar"
+	EOF
+	flux start --config-path=${name}.toml -s1 \
+		flux module stats -p bulk-exec.config.exec_service job-exec > ${name}.out 2>&1 &&
+	grep "bar" ${name}.out
+'
+# N.B. exec service defaults to rexec
+test_expect_success 'job-exec: update exec service via config reload' '
+	flux module reload -f job-exec &&
+	flux module stats -p bulk-exec.config.exec_service job-exec > reload5A.out 2>&1 &&
+	grep "rexec" reload5A.out &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	service = "reload-exec-service"
+	EOF
+	flux config reload &&
+	flux module stats -p bulk-exec.config.exec_service job-exec > reload5B.out 2>&1 &&
+	grep "reload-exec-service" reload5B.out &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
+'
+test_expect_success 'job-exec: cmdline exec service takes priority' '
+	flux module reload -f job-exec service=cmdline-exec-service &&
+	flux module stats -p bulk-exec.config.exec_service job-exec > reload6A.out 2>&1 &&
+	grep "cmdline-exec-service" reload6A.out &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	service = "reload-exec-service"
+	EOF
+	flux config reload &&
+	flux module stats -p bulk-exec.config.exec_service job-exec > reload6B.out 2>&1 &&
+	grep "cmdline-exec-service" reload6B.out &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
+'
+test_expect_success 'job-exec: exec service override can be set in exec conf' '
+	name=exec-service-override &&
+	cat <<-EOF > ${name}.toml &&
+	[exec]
+	service = "bar"
+	service-override = true
+	EOF
+	flux start --config-path=${name}.toml -s1 \
+		flux module stats -p bulk-exec.config.exec_service_override job-exec > ${name}.out 2>&1 &&
+	val=$(cat ${name}.out) &&
+	test $val -eq 1
+'
+# N.B. exec service defaults to off/disabled
+test_expect_success 'job-exec: update exec service override via config reload' '
+	flux module reload -f job-exec &&
+	flux module stats -p bulk-exec.config.exec_service_override job-exec > reload7A.out 2>&1 &&
+	val=$(cat reload7A.out) &&
+	test $val -eq 0 &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	service-override = true
+	EOF
+	flux config reload &&
+	flux module stats -p bulk-exec.config.exec_service_override job-exec > reload7B.out 2>&1 &&
+	val=$(cat reload7B.out) &&
+	test $val -eq 1 &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
+'
+test_expect_success 'job-exec: sdexex properties can be set in exec conf' '
+	name=sdexec-properties &&
+	cat <<-EOF > ${name}.toml &&
+	[exec]
+	service = "sdexec"
+	[exec.sdexec-properties]
+	MemoryHigh = "200M"
+	MemoryMax = "100M"
+	EOF
+	flux start --config-path=${name}.toml -s1 \
+		flux module stats -p bulk-exec.config.sdexec_properties job-exec > ${name}.out 2>&1 &&
+	jq -e ".MemoryHigh == \"200M\"" < ${name}.out &&
+	jq -e ".MemoryMax == \"100M\"" < ${name}.out
+'
+test_expect_success 'job-exec: bad sdexec properties causes module failure (type 1)' '
+	name=bad-sdexec-properties1 &&
+	cat <<-EOF > ${name}.toml &&
+	[exec]
+	service = "sdexec"
+	sdexec-properties = 42
+	EOF
+	test_must_fail flux start --config-path=${name}.toml -s1 \
+		flux dmesg > ${name}.log 2>&1 &&
+	grep "exec.sdexec-properties is not a table" ${name}.log
+'
+test_expect_success 'job-exec: bad sdexec properties causes module failure (type 2)' '
+	name=bad-sdexec-properties2 &&
+	cat <<-EOF > ${name}.toml &&
+	[exec]
+	service = "sdexec"
+	[exec.sdexec-properties]
+	MemoryHigh = 42
+	EOF
+	test_must_fail flux start --config-path=${name}.toml -s1 \
+		flux dmesg > ${name}.log 2>&1 &&
+	grep "exec.sdexec-properties.MemoryHigh is not a string" ${name}.log
+'
+# N.B. sdexec properties defaults to not set
+test_expect_success 'job-exec: update sdexec properties via config reload' '
+	flux module reload -f job-exec &&
+	test_must_fail flux module stats -p bulk-exec.config.sdexec_properties job-exec > reload8A.out 2>&1 &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	service = "sdexec"
+	[exec.sdexec-properties]
+	MemoryHigh = "200M"
+	EOF
+	flux config reload &&
+	flux module stats -p bulk-exec.config.sdexec_properties job-exec > reload8B.out 2>&1 &&
+	jq -e ".MemoryHigh == \"200M\"" < reload8B.out &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
+'
+# N.B. do config reload call before flux module reload to clear
+# sdexec-properties from previous test
+test_expect_success 'job-exec: bad sdexec properties leads to error on via config reload' '
+	flux config reload &&
+	flux module reload -f job-exec &&
+	test_must_fail flux module stats -p bulk-exec.config.sdexec_properties job-exec > reload9A.out 2>&1 &&
+	cat <<-EOF > ${FLUX_CONF_DIR}/exec.toml &&
+	[exec]
+	service = "sdexec"
+	[exec.sdexec-properties]
+	MemoryHigh = 42
+	EOF
+	test_must_fail flux config reload 2> reload9B.err &&
+	grep "exec.sdexec-properties.MemoryHigh is not a string" reload9B.err &&
+	rm -f ${FLUX_CONF_DIR}/exec.toml
 '
 test_done
diff --git a/t/t2404-job-exec-multiuser.t b/t/t2404-job-exec-multiuser.t
new file mode 100755
index 000000000000..aeea73dfb546
--- /dev/null
+++ b/t/t2404-job-exec-multiuser.t
@@ -0,0 +1,118 @@
+#!/bin/sh
+
+test_description='Sanity checks for job-exec multiuser exec'
+
+. $(dirname $0)/sharness.sh
+
+flux version | grep -q libflux-security && test_set_prereq FLUX_SECURITY
+
+if ! test_have_prereq FLUX_SECURITY; then
+	skip_all='skipping multiuser exec tests, libflux-security or IMP not found'
+	test_done
+fi
+
+IMP=${SHARNESS_TEST_SRCDIR}/job-exec/imp.sh
+IMP_FAIL=${SHARNESS_TEST_SRCDIR}/job-exec/imp-fail.sh
+DUMMY_SHELL=${SHARNESS_TEST_SRCDIR}/job-exec/dummy.sh
+
+#  Configure dummy IMP
+if ! test -d conf.d; then
+	mkdir conf.d
+	cat <<-EOF >conf.d/exec.toml
+	[exec]
+	imp = "${IMP}"
+	EOF
+fi
+
+export FLUX_CONF_DIR=$(pwd)/conf.d
+test_under_flux 2 job
+
+test_expect_success 'job-exec: module configured to use IMP' '
+	flux module stats -p bulk-exec.config.flux_imp_path job-exec | grep ${IMP}
+'
+test_expect_success 'job-exec: job as instance owner works' '
+	test "$(id -u)" = "$(flux run id -u)"
+'
+
+SIGN_AS=${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py
+#  Use flux_sign_wrap_as(3) to create a fake sign-wrapped jobspec
+#  which will pass ingest signature check when submitted with
+#  FLUX_HANDLE_USERID.
+#
+#  This will trigger the job-exec module to run job under our fake IMP.
+#
+test_expect_success 'job-exec: job as guest tries to run IMP' '
+	FAKE_USERID=42 &&
+	flux run --dry-run -n1 id -u | \
+		flux python ${SIGN_AS} ${FAKE_USERID} > job.signed &&
+	id=$(FLUX_HANDLE_USERID=${FAKE_USERID} \
+		flux job submit --flags=signed job.signed) &&
+	test_might_fail flux job attach ${id} &&
+	flux job list-ids ${id} > ${id}.json &&
+	jq -e ".userid == 42" < ${id}.json &&
+	flux job attach ${id} 2>&1 | grep "test-imp: Running.*$(flux job id ${id})"
+'
+test_expect_success 'job-exec: large jobspec does not get truncated' '
+	(FAKE_USERID=42 &&
+		for i in `seq 0 2048`; \
+			do export ENV${i}=xxxxxyyyyyyyyyzzzzzzz; \
+		done &&
+		flux run --dry-run -n1 id -u | \
+			flux python ${SIGN_AS} ${FAKE_USERID} > job.signed &&
+		id=$(FLUX_HANDLE_USERID=${FAKE_USERID} \
+		flux job submit --flags=signed job.signed) &&
+		test_might_fail flux job attach ${id} &&
+		actual=imp-$(flux job id $id).input &&
+		test_debug "echo expecting J of size $(wc -c < job.signed)B" &&
+		test_debug "echo input to IMP was $(wc -c < $actual)B" &&
+		jq -r .J ${actual} > J.input &&
+		test_cmp job.signed J.input
+	)
+'
+#  Configure dummy job shell so that we can ignore invalid signature on J
+#   for this test, otherwise real shell would exit immediately.
+#
+test_expect_success 'job-exec: reconfig with failing dummy IMP' '
+	cat <<-EOF >>conf.d/exec.toml
+	job-shell = "${DUMMY_SHELL}"
+	EOF
+'
+test_expect_success 'job-exec: reconfig and reload module' '
+	flux config reload &&
+	flux module reload -f job-exec
+'
+test_expect_success NO_ASAN 'job-exec: kill multiuser job works' '
+	FAKE_USERID=42 &&
+	flux run --dry-run -n2 -N2 sleep 1000 | \
+		flux python ${SIGN_AS} ${FAKE_USERID} > sleep-job.signed &&
+	id=$(FLUX_HANDLE_USERID=${FAKE_USERID} \
+		flux job submit --flags=signed sleep-job.signed) &&
+	flux job list-ids ${id} > ${id}.json &&
+	jq -e ".userid == 42" < ${id}.json &&
+	flux job wait-event -p exec -vt 30 ${id} shell.start &&
+	flux cancel ${id} &&
+	test_expect_code 143 run_timeout 30 flux job status -v ${id}
+'
+
+#  Configure failing IMP
+test_expect_success 'job-exec: reconfig with failing dummy IMP' '
+	cat <<-EOF >conf.d/exec.toml
+	[exec]
+	imp = "${IMP_FAIL}"
+	EOF
+'
+
+test_expect_success 'job-exec: reconfig and reload module' '
+	flux config reload &&
+	flux module reload -f job-exec
+'
+test_expect_success 'job-exec: IMP failure on one rank terminates job' '
+	FAKE_USERID=42 &&
+	id=$(FLUX_HANDLE_USERID=${FAKE_USERID} \
+		flux job submit --flags=signed sleep-job.signed) &&
+	flux job wait-event -vt 20 ${id}  clean &&
+	test_must_fail_or_be_terminated flux job attach -vEX ${id}
+'
+
+
+test_done
diff --git a/t/t2406-job-exec-cleanup.t b/t/t2406-job-exec-cleanup.t
new file mode 100755
index 000000000000..ed24c8897205
--- /dev/null
+++ b/t/t2406-job-exec-cleanup.t
@@ -0,0 +1,160 @@
+#!/bin/sh
+
+test_description='Test flux job exec job cleanup via SIGKILL'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 job
+
+test_expect_success 'job-exec: active_ranks stat works' '
+	flux submit -N4 sh -c "test \$FLUX_TASK_RANK -eq 2 && exit; sleep 30" &&
+	jobid=$(flux job last) &&
+	flux job wait-event -p exec -Hvt 20 $jobid shell.task-exit &&
+	flux module stats job-exec \
+		| jq ".jobs.${jobid}.active_ranks == \"0-1,3\"" &&
+	flux cancel --all &&
+	flux queue idle
+'
+test_expect_success 'job-exec: reload module with short kill-timeout' '
+	flux module reload job-exec kill-timeout=0.1s &&
+	flux module stats job-exec
+'
+test_expect_success 'job-exec: run test program that blocks SIGTERM' '
+	id=$(flux submit --wait-event=start  -n 1 -o trap.out \
+	    sh -c "trap \"echo got SIGTERM\" 15; \
+	           flux kvs put pid=\$\$; \
+	           sleep inf; sleep inf") &&
+	ns=$(flux job namespace $id) &&
+	pid=$(flux kvs get -WN ${ns} ${dir}.pid) &&
+	test_debug "echo script running as pid=$pid"
+'
+test_expect_success 'job-exec: ensure cancellation kills job' '
+	test_debug "echo Canceling $id" &&
+	flux cancel $id &&
+	test_debug "flux job attach -vEX $id || :" &&
+	test_expect_code 137 flux job status $id &&
+	test_must_fail ps -q $pid
+'
+# Note: increase max-kill-count here to ensure job doesn't disappear from
+# job-exec while we're testing kill-timeout:
+test_expect_success 'job-exec: reload module with kill/term-signal=SIGURG' '
+	flux module reload job-exec \
+		kill-timeout=0.1s kill-signal=SIGURG term-signal=SIGURG \
+		max-kill-count=20
+'
+test_expect_success 'job-exec: submit a job' '
+	jobid=$(flux submit --wait-event=start -n1 sleep inf)
+'
+test_expect_success 'job-exec: job is listed in flux-module stats' '
+	flux module stats job-exec | jq .jobs.$jobid
+'
+test_expect_success 'job-exec: cancel test job to start kill timer' '
+	flux cancel $jobid
+'
+check_kill_count() {
+	id=$1
+	stat=$2
+	value=$3
+	timeout=${4:-10}
+	count=0
+	while ! flux module stats job-exec \
+		| jq -e ".jobs.${id}.${stat} >= ${value}"; do
+		count=$((count+1))
+		if test $count -gt $((timeout*2)); then
+			echo "${stat} >= ${value} timed out after ${timeout}s"
+			return 1
+		fi
+		sleep 0.2
+		flux module stats job-exec | jq .jobs.${id}.${stat}
+	done
+}
+test_expect_success 'job-exec: ensure kill_count > 1' '
+	check_kill_count $jobid kill_count 1
+'
+test_expect_success 'job-exec: ensure kill_shell_count > 1' '
+	check_kill_count $jobid kill_shell_count 1
+'
+test_expect_success 'job-exec: kill-timeout > original value (0.1)' '
+	flux module stats job-exec | jq .jobs.${jobid}.kill_timeout &&
+	flux module stats job-exec | jq -e ".jobs.${jobid}.kill_timeout > 0.1"
+'
+test_expect_success 'job-exec: kill test job with SIGKILL' '
+       flux job kill -s 9 $jobid &&
+       flux job wait-event -vt 15 $jobid clean
+'
+test_expect_success 'job-exec: reload module with small max-kill-count' '
+	flux module reload job-exec \
+		kill-timeout=0.1s kill-signal=SIGURG term-signal=SIGURG \
+		max-kill-count=3
+'
+test_expect_success 'job-exec: submit a job' '
+	jobid=$(flux submit --wait-event=start -n1 sleep inf)
+'
+test_expect_success 'job-exec: job is listed in flux-module stats' '
+	flux module stats job-exec | jq .jobs.$jobid
+'
+test_expect_success 'job-exec: get sleep PID for later cleanup' '
+	sleep_pid=$(flux job hostpids $jobid | sed s/.*://)
+'
+test_expect_success 'job-exec: cancel test job to start kill timer' '
+	flux cancel $jobid
+'
+test_expect_success 'job-exec: wait for job to be terminated by max-kill-count' '
+	flux job wait-event -vt 15 $jobid clean &&
+	flux dmesg -H | grep "exceeded max kill count" &&
+	flux resource drain -no {reason} | grep "unkillable processes"
+'
+test_expect_success 'job-exec: kill orphan sleep PID' '
+	kill $sleep_pid
+'
+test_expect_success 'job-exec: undrain all ranks' '
+	flux resource undrain $(flux resource drain -no {ranks})
+'
+test_expect_success 'job-exec: reload module with default kill/term-signal' '
+	flux module reload job-exec
+'
+test_expect_success 'job-exec: barrier timeout raises job exception' '
+	cat <<-EOF >initrc.lua &&
+	if shell.info.rank == 3 then
+		os.execute("sleep 30")
+	end
+	EOF
+	jobid=$(flux submit -N4 --setattr=exec.bulkexec.barrier-timeout=0.5 \
+		-o userrc=initrc.lua sleep 60) &&
+	flux job wait-event -vHt 60 $jobid exception
+'
+test_expect_success 'job-exec: affected rank is drained' '
+	test_debug "flux resource drain" &&
+	test $(flux resource drain -no {ranks}) -eq 3 &&
+	flux resource drain -no {reason} | grep "start timeout"
+'
+test_expect_success 'job-exec: undrain all ranks' '
+	flux resource list &&
+	flux resource undrain $(flux resource drain -no {ranks})
+'
+test_expect_success 'job-exec: default barrier timeout can be configured' '
+	flux config load <<-EOF &&
+	[exec]
+	barrier-timeout = "0.1s"
+	EOF
+	flux module stats job-exec |
+		jq -e ".\"bulk-exec\".config.default_barrier_timeout < 1." &&
+	jobid=$(flux submit -N4 -o userrc=initrc.lua sleep 60) &&
+	flux job wait-event -vHt 60 $jobid exception
+'
+test_expect_success 'job-exec: undrain all ranks' '
+	flux resource undrain $(flux resource drain -no {ranks})
+'
+test_expect_success 'job-exec: setting an invalid barrier-timeout fails' '
+	test_expect_code 1 flux config load <<-EOF
+	[exec]
+	barrier-timeout = 100
+	EOF
+'
+test_expect_success 'job-exec: invalid FSD for barrier-timeout fails' '
+	test_expect_code 1 flux config load <<-EOF
+	[exec]
+	barrier-timeout = "55p"
+	EOF
+'
+test_done
diff --git a/t/t2407-sdbus.t b/t/t2407-sdbus.t
new file mode 100755
index 000000000000..06209850c1a0
--- /dev/null
+++ b/t/t2407-sdbus.t
@@ -0,0 +1,323 @@
+#!/bin/sh
+# ci=system
+
+test_description='Test flux systemd sd-bus bridge standalone'
+
+. $(dirname $0)/sharness.sh
+
+if ! flux version | grep systemd; then
+	skip_all="flux was not built with systemd"
+	test_done
+fi
+if ! systemctl --user show --property Version; then
+	skip_all="user systemd is not running"
+	test_done
+fi
+if ! busctl --user status >/dev/null; then
+	skip_all="user dbus is not running"
+	test_done
+fi
+
+test_under_flux 2 minimal
+
+flux setattr log-stderr-level 1
+
+#
+# N.B. ListUnitsByPatterns response payload is a 'params' array whose first
+# and only item (".params[0]") is an array of units.  The jq(1) expression
+# ".params[0][][0]" extracts field 0 of each unit array entry (unit name)
+# and prints one entry per line, ".params[0][][1]" extracts field 1 of each
+# unit entry (description) and prints one entry per line, etc.
+#
+
+# Usage: bus_list_units PATTERN
+bus_list_units() {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{\"member\":\"ListUnitsByPatterns\",\"params\":[[],[\"$1\"]]}).get_str())"
+}
+
+# Usage: bus_list_units_parsed PATTERN FIELDNUM
+bus_list_units_parsed() {
+    bus_list_units "$1" >tmp.json || return 1
+    $jq -r ".params[0][][$2]" <tmp.json
+}
+
+# Usage: bus_count_units PATTERN
+bus_count_units() {
+    bus_list_units_parsed "$1" 0 >tmp.list || return 1
+    wc -l <tmp.list
+}
+
+# Usage: bus_wait_for_unit_count TRIES COUNT PATTERN
+bus_wait_for_unit_count() {
+    local tries=$1
+    while test $tries -gt 0; do
+	test $tries -eq $1 || sleep 1
+        test $(bus_count_units $3) -eq $2 && return 0
+        tries=$(($tries-1))
+    done
+    return 1
+}
+
+# Usage: bus_reset_failed_unit name
+bus_reset_failed_unit() {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{\"member\":\"ResetFailedUnit\",\"params\":[\"$1\"]}).get_str())"
+}
+
+# Usage: bus_stop_unit name
+bus_stop_unit() {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{\"member\":\"StopUnit\",\"params\":[\"$1\",\"fail\"]}).get_str())"
+}
+
+# Usage: bus_kill_unit name signum
+bus_kill_unit() {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{\"member\":\"KillUnit\",\"params\":[\"$1\",\"all\",$2]}).get_str())"
+}
+
+# Usage: bus_get_prop_all interface name
+bus_get_prop_all () {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{\"path\":\"/org/freedesktop/systemd1/unit/$2\",\"interface\":\"org.freedesktop.DBus.Properties\",\"member\":\"GetAll\",\"params\":[\"$1\"]}).get_str())"
+}
+
+# Usage: bus_get_prop interface name property
+bus_get_prop () {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{\"path\":\"/org/freedesktop/systemd1/unit/$2\",\"interface\":\"org.freedesktop.DBus.Properties\",\"member\":\"Get\",\"params\":[\"$1\",\"$3\"]}).get_str())"
+}
+
+# Usage: bus_start_simple name description remain cmd arg1
+# where remain=True|False and cmd has exactly one argument
+bus_start_simple() {
+    local cmd=$(which $4) || return 1
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{\"member\":\"StartTransientUnit\",\"params\":[\"$1\",\"fail\",[ [\"Description\",[\"s\",\"$2\"]], [\"RemainAfterExit\",[\"b\",$3]], [\"ExecStart\",[\"a(sasb)\",[[\"$cmd\",[\"$4\",\"$5\"],False]]]] ], []]}).get_str())"
+}
+
+bus_call_unknown_member() {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{\"member\":\"UnknownMember\",\"params\":[]}).get_str())"
+}
+
+bus_call_unknown_interface() {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{\"interface\":\"org.freedesktop.DBus.unknown\",\"member\":\"StopUnit\",\"params\":[\"$1\",\"fail\"]}).get_str())"
+}
+
+bus_call_malformed() {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{}).get_str())"
+}
+
+bus_subscribe_notstreaming() {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.subscribe\",{}).get_str())"
+}
+
+# Usage: bus_subscribe_cancel matchtag
+bus_subscribe_cancel () {
+    flux python -c "import flux; flux.Flux().rpc(\"sdbus.subscribe-cancel\",{\"matchtag\":$1},flags=flux.constants.FLUX_RPC_NORESPONSE)"
+}
+
+test_expect_success 'enable sdbus-debug in configuration' '
+	flux config load <<-EOT
+	[systemd]
+	sdbus-debug = true
+	EOT
+'
+test_expect_success 'load sdbus module' '
+	flux module load sdbus
+'
+test_expect_success 'sdbus reconfig fails with bad sdbus-debug value' '
+	test_must_fail flux config load <<-EOT 2>config.err &&
+	[systemd]
+	sdbus-debug = 42
+	EOT
+	grep "Expected true or false" config.err
+'
+
+test_expect_success 'sdbus list-units works' '
+	count=$(bus_count_units "*") &&
+	echo Listed $count units >&2
+'
+
+clean_unit() {
+	local unit=$1
+	echo Cleaning up $unit
+	bus_reset_failed_unit $unit
+	bus_stop_unit $unit
+	return 0
+}
+
+test_expect_success 'reset/unload any flux-t2406 units from prior runs' '
+	bus_list_units_parsed "flux-t2406-*" 0 >oldunits.out &&
+	for unit in $(cat oldunits.out); do clean_unit $unit; done
+'
+test_expect_success 'there are no flux-t2406 units loaded' '
+	bus_wait_for_unit_count 10 0 "flux-t2406-*"
+'
+# script usage: subscribe.py [path-glob]
+test_expect_success 'create subscribe script' '
+	cat >subscribe.py <<-EOT &&
+	import sys
+	import flux
+	handle = flux.Flux()
+	sub = {"member":"PropertiesChanged","interface":"org.freedesktop.DBus.Properties"}
+	if len(sys.argv) > 1:
+	    sub["path"] = sys.argv[1]
+	fut = handle.rpc("sdbus.subscribe",sub,flags=flux.constants.FLUX_RPC_STREAMING)
+	while True:
+	    print(fut.get_str())
+	    sys.stdout.flush()
+	    fut.reset()
+	EOT
+	chmod +x subscribe.py
+'
+test_expect_success NO_CHAIN_LINT 'subscribe to flux-t2406-[2-9]* signals' '
+	flux python ./subscribe.py \
+	    /org/freedesktop/systemd1/unit/flux-t2406-[2-9]* \
+	    >signals.out 2>signals.err &
+	echo $! >signals.pid
+'
+test_expect_success NO_CHAIN_LINT 'subscribe to all signals' '
+	flux python ./subscribe.py \
+	    >signals_all.out 2>signals_all.err &
+	echo $! >signals_all.pid
+'
+test_expect_success 'StopUnit unknown.service fails' '
+	test_must_fail bus_stop_unit unknown.service 2>stop_unknown.err &&
+	grep "not loaded" stop_unknown.err
+'
+test_expect_success 'KillUnit unknown.service fails' '
+	test_must_fail bus_kill_unit unknown.service 15 2>kill_unknown.err &&
+	grep "not loaded" kill_unknown.err
+'
+test_expect_success 'ResetFailedUnit unknown.service fails' '
+	test_must_fail bus_reset_failed_unit unknown.service \
+	    2>rst_unknown.err &&
+	grep "not loaded" rst_unknown.err
+'
+test_expect_success 'StartTransientUnit RemainAfterExit=true cmd=true' '
+	bus_start_simple flux-t2406-1.service unit1 True true dummy
+'
+test_expect_success 'ListUnitsByPatterns shows transient unit 1' '
+	bus_wait_for_unit_count 10 1 flux-t2406-1.service
+'
+test_expect_success 'StopUnit works on transient unit 1' '
+	bus_stop_unit flux-t2406-1.service
+'
+test_expect_success 'ListUnitsByPatterns does not show transient unit 1' '
+	bus_wait_for_unit_count 10 0 flux-t2406-1.service
+'
+test_expect_success 'StartTransientUnit RemainAfterExit=false cmd=false' '
+	bus_start_simple flux-t2406-2.service unit2 False false dummy
+'
+test_expect_success 'ListUnitsByPatterns shows transient unit 2' '
+	bus_wait_for_unit_count 10 1 flux-t2406-2.service
+'
+test_expect_success 'ResetFailedUnit works on that unit' '
+	bus_reset_failed_unit flux-t2406-2.service
+'
+test_expect_success 'ListUnitsByPatterns does not show transient unit 2' '
+	bus_wait_for_unit_count 10 0 flux-t2406-2.service
+'
+test_expect_success 'StartTransientUnit RemainAfterExit=false cmd=sleep 60' '
+	bus_start_simple flux-t2406-3.service unit3 False sleep 60
+'
+test_expect_success 'ListUnitsByPatterns shows transient unit 3' '
+	bus_wait_for_unit_count 10 1 flux-t2406-3.service
+'
+test_expect_success 'GetAll shows properties of transient unit 3' '
+	bus_get_prop_all org.freedesktop.systemd1.Service \
+	    flux-t2406-3.service >prop3_all.out
+'
+test_expect_success 'GetAll returned MainPID property and it is valid' '
+	$jq <prop3_all.out .params[0].MainPID[1] >pid3.out &&
+	kill -0 $(cat pid3.out)
+'
+test_expect_success 'Get MainPID also works' '
+	bus_get_prop org.freedesktop.systemd1.Service \
+	    flux-t2406-3.service MainPID >prop3.out
+'
+test_expect_success 'Get returned the same MainPID value' '
+	$jq <prop3.out .params[0][1] >pid3.out2 &&
+	test_cmp pid3.out pid3.out2
+'
+test_expect_success 'KillUnit sends SIGTERM (15) to that unit' '
+	bus_kill_unit flux-t2406-3.service 15
+'
+test_expect_success 'ListUnitsByPatterns does not show transient unit 3' '
+	bus_wait_for_unit_count 10 0 flux-t2406-3.service
+'
+test_expect_success 'StartTransientUnit RemainAfterExit=true cmd=true' '
+	bus_start_simple flux-t2406-4.service "This is a test" True true dummy
+'
+test_expect_success 'ListUnitsByPatterns shows transient unit 4' '
+	bus_wait_for_unit_count 10 1 flux-t2406-4.service
+'
+test_expect_success 'ListUnitsByPattern shows transient unit 4 description' '
+	bus_list_units_parsed flux-t2406-4.service 1 >listdesc.out &&
+	grep "This is a test" listdesc.out
+'
+test_expect_success 'StopUnit works on that unit' '
+	bus_stop_unit flux-t2406-4.service
+'
+test_expect_success 'ListUnitsByPatterns does not show transient unit 4' '
+	bus_wait_for_unit_count 10 0 flux-t2406-4.service
+'
+test_expect_success 'calling an unknown member fails' '
+	test_must_fail bus_call_unknown_member 2>unknown_member.err &&
+	grep "unknown member" unknown_member.err
+'
+test_expect_success 'calling an unknown interface fails' '
+	test_must_fail bus_call_unknown_interface 2>unknown_interface.err &&
+	grep "unknown interface" unknown_interface.err
+'
+test_expect_success 'malformed sdbus.call request fails' '
+	test_must_fail bus_call_malformed 2>malformed.err &&
+	grep "malformed request" malformed.err
+'
+test_expect_success 'non-streaming sdbus.subscribe fails' '
+	test_must_fail bus_subscribe_notstreaming 2>subscribe.err &&
+	grep "Protocol error" subscribe.err
+'
+test_expect_success 'send a sdbus.subscribe-cancel on random matchtag for fun' '
+	(FLUX_HANDLE_TRACE=1 bus_subscribe_cancel 424242)
+'
+test_expect_success NO_CHAIN_LINT 'terminate background subscribers' '
+	kill -15 $(cat signals_all.pid) &&
+	kill -15 $(cat signals.pid)
+'
+test_expect_success NO_CHAIN_LINT 'PropertiesChanged signals were received' '
+	test $(grep PropertiesChanged signals_all.out | wc -l) -gt 0
+'
+test_expect_success NO_CHAIN_LINT 'all units triggered signals' '
+	test $(grep flux-t2406-1 signals_all.out | wc -l) -gt 0 &&
+	test $(grep flux-t2406-2 signals_all.out | wc -l) -gt 0 &&
+	test $(grep flux-t2406-3 signals_all.out | wc -l) -gt 0 &&
+	test $(grep flux-t2406-4 signals_all.out | wc -l) -gt 0
+'
+test_expect_success NO_CHAIN_LINT 'subscription filter worked' '
+	test $(grep flux-t2406-1 signals.out | wc -l) -eq 0 &&
+	test $(grep flux-t2406-2 signals.out | wc -l) -gt 0 &&
+	test $(grep flux-t2406-3 signals.out | wc -l) -gt 0 &&
+	test $(grep flux-t2406-4 signals.out | wc -l) -gt 0
+'
+# Test restriction of RPCs to sdbus on rank 0 from rank 1
+# N.B. requests are forwarded upstream b/c sdbus is not loaded on rank 1
+test_expect_success 'subscribe from rank 1 is restricted' '
+        test_must_fail flux exec -r 1 flux python ./subscribe.py 2>sub1.err &&
+	grep "not allowed" sub1.err
+'
+test_expect_success 'create list script' '
+	cat >list.py <<-EOT &&
+	import sys
+	import flux
+	print(flux.Flux().rpc("sdbus.call",{"member":"ListUnitsByPatterns","params":[[],["*"]]}).get_str())
+	EOT
+	chmod +x list.py
+'
+test_expect_success 'list from rank 0 is allowed' '
+	flux python ./list.py >/dev/null
+'
+test_expect_success 'list from rank 1 is restricted' '
+	test_must_fail flux exec -r 1 flux python ./list.py 2>list1.err &&
+	grep "not allowed" list1.err
+'
+
+test_expect_success 'remove sdbus module' '
+	flux module remove sdbus
+'
+test_done
diff --git a/t/t2408-sdbus-recovery.t b/t/t2408-sdbus-recovery.t
new file mode 100755
index 000000000000..e7712a19b288
--- /dev/null
+++ b/t/t2408-sdbus-recovery.t
@@ -0,0 +1,102 @@
+#!/bin/sh
+# ci=system
+
+test_description='Test flux systemd sd-bus bridge recovery'
+
+. $(dirname $0)/sharness.sh
+
+if ! flux version | grep systemd; then
+	skip_all="flux was not built with systemd"
+	test_done
+fi
+if ! systemctl --user show --property Version; then
+	skip_all="user systemd is not running"
+	test_done
+fi
+if ! busctl --user status >/dev/null; then
+	skip_all="user dbus is not running"
+	test_done
+fi
+
+mkdir -p config
+cat >config/config.toml <<EOF
+[systemd]
+sdbus-debug = true
+EOF
+
+test_under_flux 1 minimal --config-path=$(pwd)/config
+
+flux setattr log-stderr-level 1
+
+# Usage: bus_get_manager_prop property
+bus_get_manager_prop() {
+    flux python -c "import flux; print(flux.Flux().rpc(\"sdbus.call\",{\"path\":\"/org/freedesktop/systemd1\",\"interface\":\"org.freedesktop.DBus.Properties\",\"member\":\"Get\",\"params\":[\"org.freedesktop.systemd1.Manager\",\"$1\"]}).get_str())"
+}
+
+# Usage: bus_get_manager_prop property
+bus_reconnect() {
+    flux python -c "import flux; flux.Flux().rpc(\"sdbus.reconnect\",{}).get()"
+}
+
+test_expect_success 'load sdbus module' '
+	flux module load sdbus
+'
+test_expect_success 'create subscribe script' '
+	cat >subscribe.py <<-EOT &&
+	import sys
+	import flux
+	handle = flux.Flux()
+	fut = handle.rpc("sdbus.subscribe",{},flags=flux.constants.FLUX_RPC_STREAMING)
+	while True:
+	    print(fut.get_str())
+	    sys.stdout.flush()
+	    fut.reset()
+	EOT
+	chmod +x subscribe.py
+'
+test_expect_success 'get systemd version' '
+	bus_get_manager_prop Version
+'
+test_expect_success 'clear the broker ring buffer' '
+	flux dmesg -C
+'
+# This should trigger a 2s backoff before reconnect is attempted
+test_expect_success 'force bus reconnect' '
+	bus_reconnect
+'
+test_expect_success NO_CHAIN_LINT 'initiate subscription in the background' '
+        flux python ./subscribe.py >signals.out 2>signals.err &
+        echo $! >signals.pid
+'
+# Requests during the backoff period are delayed but still work
+test_expect_success 'get systemd version works during despite reconnect' '
+	bus_get_manager_prop Version
+'
+test_expect_success 'print logs' '
+	flux dmesg -H | grep sdbus
+'
+test_expect_success 'force another bus reconnect' '
+	bus_reconnect
+'
+test_expect_success NO_CHAIN_LINT 'background subscription fails with EAGAIN' '
+	pid=$(cat signals.pid) &&
+	test_must_fail wait $pid &&
+	grep "Errno 11" signals.err
+'
+test_expect_success NO_CHAIN_LINT 'initiate another subscription' '
+        flux python ./subscribe.py >signals2.out 2>signals2.err &
+        echo $! >signals2.pid
+'
+# There will be another 2s delay while sdbus reconnects
+test_expect_success 'get systemd version to ensure reconnect has occurred' '
+	bus_get_manager_prop Version
+'
+test_expect_success 'remove sdbus module' '
+	flux module remove sdbus
+'
+test_expect_success NO_CHAIN_LINT 'background subscription fails with ENOSYS' '
+	pid=$(cat signals2.pid) &&
+	test_must_fail wait $pid &&
+	grep "Errno 38" signals2.err
+'
+test_done
diff --git a/t/t2409-sdexec.t b/t/t2409-sdexec.t
new file mode 100755
index 000000000000..ab1aec209a9b
--- /dev/null
+++ b/t/t2409-sdexec.t
@@ -0,0 +1,335 @@
+#!/bin/sh
+# ci=system
+
+test_description='Test flux systemd execution'
+
+. $(dirname $0)/sharness.sh
+
+if ! flux version | grep systemd; then
+	skip_all="flux was not built with systemd"
+	test_done
+fi
+if ! systemctl --user show --property Version; then
+	skip_all="user systemd is not running"
+	test_done
+fi
+if ! busctl --user status >/dev/null; then
+	skip_all="user dbus is not running"
+	test_done
+fi
+
+test_under_flux 2 minimal
+
+flux setattr log-stderr-level 1
+
+sdexec="flux exec --service sdexec"
+lptest="flux lptest"
+rkill="flux python ${SHARNESS_TEST_SRCDIR}/scripts/rexec.py kill -s sdexec"
+rps="flux python ${SHARNESS_TEST_SRCDIR}/scripts/rexec.py ps -s sdexec"
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+
+# systemd 239 requires commands to be fully qualified, while 249 does not
+true=$(which true)
+false=$(which false)
+sh=$(which sh)
+cat=$(which cat)
+pwd=$(which pwd)
+printenv=$(which printenv)
+systemctl=$(which systemctl)
+
+test_expect_success 'enable debug logging' '
+	cat >systemd.toml <<-EOF &&
+	[systemd]
+	sdbus-debug = true
+	sdexec-debug = true
+	EOF
+	flux config load <systemd.toml
+'
+test_expect_success 'load sdbus,sdexec modules' '
+	flux exec flux module load sdbus &&
+	flux exec flux module load sdexec
+'
+test_expect_success 'clear broker logs' '
+	flux dmesg -C
+'
+test_expect_success 'sdexec true succeeds' '
+	$sdexec -r 0 $true
+'
+test_expect_success 'sdexec -n cat succeeds' '
+	run_timeout 30 $sdexec -n -r 0 $cat
+'
+test_expect_success 'sdexec false fails with exit code 1' '
+	test_expect_code 1 $sdexec -r 0 $false
+'
+test_expect_success 'sdexec /notacmd fails with exit code 127' '
+	test_expect_code 127 $sdexec -r 0 /notacmd
+'
+test_expect_success 'capture broker logs' '
+	flux dmesg >dmesg.log
+'
+test_expect_success 'sdexec stdout works' '
+	cat >hello.exp <<-EOT &&
+	Hello world!
+	EOT
+	$sdexec -r 0 $sh -c "echo Hello world!" >hello.out &&
+	test_cmp hello.exp hello.out
+'
+test_expect_success 'sdexec stderr works' '
+	$sdexec -r 0 $sh -c "echo Hello world! >&2" 2>hello.err &&
+	test_cmp hello.exp hello.err
+'
+test_expect_success 'sdexec stdout+stderr works' '
+	$sdexec -r 0 $sh -c "echo Hello world!; echo Hello world! >&2" \
+		>hello2.out 2>hello2.err &&
+	test_cmp hello.exp hello2.out &&
+	test_cmp hello.exp hello2.err
+'
+test_expect_success 'sdexec stdin works' '
+	echo "Hello world!" | $sdexec -r 0 $cat >hello.in &&
+	test_cmp hello.exp hello.in
+'
+test_expect_success 'set up for lptest stdio tests' '
+	$lptest 79 10000 >lptest.exp
+'
+test_expect_success 'sdexec works with 10K lines of stdout' '
+	$sdexec -r 0 $cat lptest.exp >lptest.out &&
+	test_cmp lptest.exp lptest.out
+'
+test_expect_success 'sdexec works with 10K lines of stdin' '
+	$sdexec -r 0 $sh -c "cat >lptest.in" <lptest.exp &&
+	test_cmp lptest.exp lptest.in
+'
+test_expect_success 'sdexec can set working directory' '
+	pwd >dir.exp &&
+	$sdexec -r 0 $pwd >dir.out &&
+	test_cmp dir.exp dir.out
+'
+test_expect_success 'sdexec can set environment' '
+	echo 42 >env.exp &&
+	T2409=42 $sdexec -r 0 $printenv T2409 >env.out &&
+	test_cmp env.exp env.out
+'
+# environment modules sets variables with names like 'BASH_FUNC_ml()'
+# https://bugzilla.redhat.com/show_bug.cgi?id=1912046#c5
+test_expect_success 'sdexec ignores environment containing parens' '
+	env "foo()"=badenv666 $sdexec -r 0 $printenv >badenv.out &&
+	test_must_fail grep -q badenv666 badenv.out
+'
+
+
+# each entry: "TESTVAR_xxxxx=" (14) + val (1009) + delim (1) = 1024 bytes
+make_exports() {
+	local count=$1
+	local val=$(for i in $(seq 1009); do printf x; done)
+	while test $count -gt 0; do
+		printf "export TESTVAR_%.5d=%s\n" $count $val
+		count=$(($count-1))
+	done
+}
+
+# Since the StartTransientUnit request includes the user's environment, the
+# request message size is unpredictable.  Let's be sure a large environment
+# as might be seen with spack or environment modules is handled OK.
+#
+# Note the maximum dbus message size is 128 MiB per spec[1].
+# Tested on debian 11 aarch64:
+# - count=1024 (1MiB) works
+# - count=2048 (2MiB) works
+# - count=4096 (4MiB) works
+# - count=8192 (8MiB) printenv(1) fails with "Argument list too long"
+# But sdexec/dbus holds up just fine.
+#
+# [1] https://dbus.freedesktop.org/doc/dbus-specification.html
+test_expect_success 'sdexec accepts supersize environment' '
+	count=1024 &&
+	make_exports $count >testenv &&
+	sed -e "s/^export //" <testenv | sort >testenv.exp
+	(. $(pwd)/testenv && $sdexec -r 0 $printenv) >testenv.raw &&
+	grep TESTVAR_ testenv.raw | sort >testenv.out &&
+	test_cmp testenv.exp testenv.out
+'
+test_expect_success 'sdexec can set unit name' '
+	$sdexec -r 0 --setopt=SDEXEC_NAME=t2409-funfunfun.service \
+	    $systemctl --user list-units --type=service >name.out &&
+	grep t2409-funfunfun name.out
+'
+test_expect_success 'sdexec can set unit Description property' '
+	cat >desc.exp <<-EOT &&
+	Description=fubar
+	EOT
+	$sdexec -r 0 \
+	    --setopt=SDEXEC_NAME=t2409-desc.service \
+	    --setopt=SDEXEC_PROP_Description=fubar \
+	    $systemctl --user show --property Description \
+	        t2409-desc.service >desc.out &&
+	test_cmp desc.exp desc.out
+'
+test_expect_success 'sdexec can set unit SendSIGKILL property to no' '
+	cat >sigkill.exp <<-EOT &&
+	SendSIGKILL=no
+	EOT
+	$sdexec -r 0 \
+	    --setopt=SDEXEC_NAME="t2409-sigkill.service" \
+	    --setopt=SDEXEC_PROP_SendSIGKILL=no \
+	    $systemctl --user show --property SendSIGKILL \
+	        t2409-sigkill.service >sigkill.out &&
+	test_cmp sigkill.exp sigkill.out
+'
+test_expect_success 'setting SendSIGKILL to an invalid value fails' '
+	test_must_fail $sdexec -r 0 --setopt=SDEXEC_PROP_SendSIGKILL=zzz \
+	    $true 2>sigkill_badval.err &&
+	grep "error setting property" sigkill_badval.err
+'
+test_expect_success 'sdexec can set unit KillMode property to process' '
+	cat >killmode.exp <<-EOT &&
+	KillMode=process
+	EOT
+	$sdexec -r 0 \
+	    --setopt=SDEXEC_NAME="t2409-killmode.service" \
+	    --setopt=SDEXEC_PROP_KillMode=process \
+	    $systemctl --user show --property KillMode \
+	        t2409-killmode.service >killmode.out &&
+	test_cmp killmode.exp killmode.out
+'
+# Check that we can set resource control attributes on our transient units,
+# but expect resource control testing to occur elsewhere.
+# See also:
+# https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html
+test_expect_success 'sdexec can set unit MemoryHigh property' '
+	cat >memhi.exp <<-EOT &&
+	MemoryHigh=1073741824
+	EOT
+	$sdexec -r 0 \
+	    --setopt=SDEXEC_NAME="t2409-memhi.service" \
+	    --setopt=SDEXEC_PROP_MemoryHigh=1G \
+	    $systemctl --user show --property MemoryHigh \
+	        t2409-memhi.service >memhi.out &&
+	test_cmp memhi.exp memhi.out
+'
+test_expect_success 'sdexec can set unit MemoryMax property' '
+	cat >memmax.exp <<-EOT &&
+	MemoryMax=infinity
+	EOT
+	$sdexec -r 0 \
+	    --setopt=SDEXEC_NAME="t2409-memmax.service" \
+	    --setopt=SDEXEC_PROP_MemoryMax=infinity \
+	    $systemctl --user show --property MemoryMax \
+	        t2409-memmax.service >memmax.out &&
+	test_cmp memmax.exp memmax.out
+'
+test_expect_success 'sdexec can set unit MemoryMin property' '
+	cat >memmin.exp <<-EOT &&
+	MemoryMin=1048576
+	EOT
+	$sdexec -r 0 \
+	    --setopt=SDEXEC_NAME="t2409-memmin.service" \
+	    --setopt=SDEXEC_PROP_MemoryMin=1M \
+	    $systemctl --user show --property MemoryMin \
+	        t2409-memmin.service >memmin.out &&
+	test_cmp memmin.exp memmin.out
+'
+test_expect_success 'sdexec can set unit AllowedCPUs property' '
+	cat >allowedcpus.exp <<-EOT &&
+	AllowedCPUs=0 57-99 127
+	EOT
+	$sdexec -r 0 \
+	    --setopt=SDEXEC_NAME="t2409-allowedcpus.service" \
+	    --setopt=SDEXEC_PROP_AllowedCPUs="0,57-99,127" \
+	    $systemctl --user show --property AllowedCPUs \
+	        t2409-allowedcpus.service >allowedcpus.out &&
+	test_cmp allowedcpus.exp allowedcpus.out
+'
+test_expect_success 'sdexec can set unit AllowedCPUs to an empty bitmask' '
+	cat >allowedcpus2.exp <<-EOT &&
+	AllowedCPUs=
+	EOT
+	$sdexec -r 0 \
+	    --setopt=SDEXEC_NAME="t2409-allowedcpus2.service" \
+	    --setopt=SDEXEC_PROP_AllowedCPUs="" \
+	    $systemctl --user show --property AllowedCPUs \
+	        t2409-allowedcpus2.service >allowedcpus2.out &&
+	test_cmp allowedcpus2.exp allowedcpus2.out
+'
+
+memtotal() {
+	local kb=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
+	expr $kb \* 1024
+}
+
+# There's a rounding error in here somewhere because 50% is off by 2048 bytes,
+# but 100% is accurate with respect to /proc/meminfo.  Just use 100% for now.
+test_expect_success 'sdexec can set unit MemoryLow property' '
+	echo MemoryLow=$(memtotal) >memlow.exp &&
+	$sdexec -r 0 \
+	    --setopt=SDEXEC_NAME="t2409-memlow.service" \
+	    --setopt=SDEXEC_PROP_MemoryLow=100% \
+	    $systemctl --user show --property MemoryLow \
+	        t2409-memlow.service >memlow.out &&
+	test_cmp memlow.exp memlow.out
+'
+test_expect_success 'sdexec fails on bad property' '
+	test_must_fail $sdexec -r 0 --setopt=SDEXEC_PROP_xxx=yyz \
+	    $true 2>setunk.err &&
+	grep "Cannot set property xxx" setunk.err
+'
+test_expect_success 'kill fails on unknown pid' '
+        test_must_fail $rkill -r 0 15 1234 2>killunk.err &&
+	grep "not found" killunk.err
+'
+test_expect_success 'rank 0 sdexec fails if remote' '
+        test_must_fail flux exec -r 1 $sdexec -r 0 $true 2>restrict.err &&
+        grep "not allowed on rank 0" restrict.err
+'
+test_expect_success 'rank 0 kill fails if remote' '
+        test_must_fail flux exec -r 1 $rkill -r 0 15 1234 2>killperm.err &&
+	grep "not allowed on rank 0" killperm.err
+'
+test_expect_success 'rank 0 list fails if remote' '
+        test_must_fail flux exec -r 1 $rps -r 0 2>psperm.err &&
+	grep "not allowed on rank 0" psperm.err
+'
+# N.B. this "kill" test relies on the fact that flux exec forwards
+# signals to the remote process
+test_expect_success NO_CHAIN_LINT 'sdexec remote kill works' '
+	$sdexec -r 0 $sh -c "echo hello >sleep.out && sleep 60" &
+	testpid=$! &&
+	$waitfile -q -t 30 sleep.out &&
+	kill -15 $testpid &&
+	test_expect_code 143 wait $testpid
+'
+test_expect_success NO_CHAIN_LINT 'sdexec.list works' '
+	$sdexec -r 0 $sh -c "echo hello >sleep2.out && sleep 60" &
+	testpid=$! &&
+	$waitfile -q -t 30 sleep2.out &&
+	$rps >list.out &&
+	grep sh list.out &&
+	kill -15 $testpid &&
+	test_expect_code 143 wait $testpid
+'
+test_expect_success NO_CHAIN_LINT 'sdexec.stats-get works' '
+	$sdexec -r 0 --setopt=SDEXEC_NAME=statstest.service \
+	    $sh -c "echo hello >sleep3.out && sleep 60" &
+	testpid=$! &&
+	$waitfile -q -t 30 sleep3.out &&
+	flux module stats sdexec >stats.out &&
+	grep statstest.service stats.out &&
+	kill -15 $testpid &&
+	test_expect_code 143 wait $testpid
+'
+test_expect_success 'sdexec sets FLUX_URI to local broker' '
+	echo $FLUX_URI >uri.exp &&
+	$sdexec -r 1 $printenv FLUX_URI >uri.out &&
+	test_must_fail test_cmp uri.exp uri.out
+'
+test_expect_success 'sdexec reconfig fails with bad sdexec-debug value' '
+	test_must_fail flux config load <<-EOT 2>config.err &&
+	[systemd]
+	sdexec-debug = 42
+	EOT
+	grep "Expected true or false" config.err
+'
+test_expect_success 'remove sdexec,sdbus modules' '
+	flux exec flux module remove sdexec &&
+	flux exec flux module remove sdbus
+'
+test_done
diff --git a/t/t2410-sdexec-memlimit.t b/t/t2410-sdexec-memlimit.t
new file mode 100755
index 000000000000..e107494d0ef8
--- /dev/null
+++ b/t/t2410-sdexec-memlimit.t
@@ -0,0 +1,189 @@
+#!/bin/sh
+# ci=system
+test_description='Test sdexec cgroups memory controller manipulation
+
+Test whether Flux can affect the cgroups memory controller for processes
+spawned via sdexec, and the capability to configure basic limits for jobs.
+
+See also:
+https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html
+https://docs.kernel.org/admin-guide/cgroup-v2.html#memory
+cgroups(7) /proc/[pid]/cgroup
+'
+
+. $(dirname $0)/sharness.sh
+
+if ! flux version | grep systemd; then
+	skip_all="flux was not built with systemd"
+	test_done
+fi
+if ! systemctl --user show --property Version; then
+	skip_all="user systemd is not running"
+	test_done
+fi
+if ! busctl --user status >/dev/null; then
+	skip_all="user dbus is not running"
+	test_done
+fi
+if ! systemctl show user@$(id -u) -p DelegateControllers | grep memory; then
+	skip_all="cgroups memory controller is not delegated"
+	test_done
+fi
+if stress=$(which stress); then
+	test_set_prereq STRESS
+fi
+
+mkdir -p config
+cat >config/config.toml <<EOT
+[systemd]
+enable = true
+sdexec-debug = true
+[exec]
+service = "sdexec"
+[exec.sdexec-properties]
+MemoryHigh = "200M"
+MemoryMax = "100M"
+EOT
+
+cat >getcg.sh <<EOT2
+#!/bin/sh
+CGDIR=/sys/fs/cgroup/\$(cat /proc/\$\$/cgroup | cut -d: -f3)
+cat \$CGDIR/\$1
+EOT2
+chmod +x getcg.sh
+
+if ! $(pwd)/getcg.sh cgroup.controllers | grep memory; then
+	skip_all="cgroups memory controller is not enabled"
+	test_done
+fi
+
+test_under_flux 1 full --config-path=$(pwd)/config
+
+flux setattr log-stderr-level 7
+
+sdexec="flux exec --service sdexec"
+getcg=$(pwd)/getcg.sh
+
+if ! $sdexec $getcg memory.high; then
+	skip_all="$sdexec getcg doesn't work"
+	test_done
+fi
+
+#
+# memory.high: the throttling limit on memory usage
+#
+test_expect_success 'memory.high exists' '
+	$sdexec $getcg memory.high
+'
+test_expect_success 'memory.high can be set to 200M' '
+	cat >200M.exp <<-EOT &&
+	209715200
+	EOT
+	$sdexec \
+	    --setopt=SDEXEC_PROP_MemoryHigh=200M \
+	    $getcg memory.high >high.out &&
+	test_cmp 200M.exp high.out
+'
+test_expect_success 'memory.high can be set to infinity' '
+	cat >inf.exp <<-EOT &&
+	max
+	EOT
+	$sdexec \
+	    --setopt=SDEXEC_PROP_MemoryHigh=infinity \
+	    $getcg memory.high >high2.out &&
+	test_cmp inf.exp high2.out
+'
+test_expect_success 'memory.high can be configured for jobs' '
+	flux run $getcg memory.high >high3.out &&
+	test_cmp 200M.exp high3.out
+'
+
+#
+# memory.max: the absolute limit on memory usage
+#
+test_expect_success 'memory.max exists' '
+	$sdexec $getcg memory.max
+'
+test_expect_success 'memory.max can be set to 100M' '
+	cat >100M.exp <<-EOT &&
+	104857600
+	EOT
+	$sdexec \
+	    --setopt=SDEXEC_PROP_MemoryMax=100M \
+	    $getcg memory.max >max.out &&
+	test_cmp 100M.exp max.out
+'
+test_expect_success 'memory.max can be set to infinity' '
+	cat >inf.exp <<-EOT &&
+	max
+	EOT
+	$sdexec \
+	    --setopt=SDEXEC_PROP_MemoryMax=infinity \
+	    $getcg memory.max >max2.out &&
+	test_cmp inf.exp max2.out
+'
+test_expect_success 'memory.max can be configured for jobs' '
+	flux run $getcg memory.max >max3.out &&
+	test_cmp 100M.exp max3.out
+'
+test_expect_success STRESS 'remaining within memory.max works' '
+	$sdexec \
+	    --setopt=SDEXEC_PROP_MemoryMax=200M \
+	    $stress --timeout 3 --vm-keep --vm 1 --vm-bytes 100M
+'
+
+# Like test_expect_code() except 143/SIGKILL (oom kill) is also acceptable
+test_expect_code_or_killed() {
+	local want_code=$1
+	shift
+	"$@"
+	exit_code=$?
+	if test $exit_code = $want_code || test $exit_code = 143; then
+	    return 0
+	fi
+	echo >&2 "test_expect_code_or_killed: command exited with $exit_code, we wanted $want_code or 143/SIGKILL $*"
+	return 1
+}
+
+test_expect_success STRESS 'exceeding memory.max causes exec failure' '
+	test_expect_code_or_killed 1 $sdexec \
+	    --setopt=SDEXEC_PROP_MemoryMax=100M \
+	    $stress --timeout 60 --vm-keep --vm 1 --vm-bytes 200M
+'
+test_expect_success STRESS 'exceeding memory.max causes job failure' '
+	test_expect_code_or_killed 1 flux run \
+	    $stress --timeout 60 --vm-keep --vm 1 --vm-bytes 200M
+'
+
+#
+# reload / update config
+#
+# N.B. do these tests last, as they will alter the config of the
+# job-exec module and could affect above tests
+#
+
+test_expect_success 'change values of memory containment' '
+	cat >config/config.toml <<-EOT &&
+	[systemd]
+	enable = true
+	sdexec-debug = true
+	[exec]
+	service = "sdexec"
+	[exec.sdexec-properties]
+	MemoryHigh = "100M"
+	MemoryMax = "infinity"
+	EOT
+	flux config reload
+'
+test_expect_success 'memory.high configuration changed' '
+	flux run $getcg memory.high >highupdate.out &&
+	test_cmp 100M.exp highupdate.out
+'
+test_expect_success 'memory.max configuration changed' '
+	flux run $getcg memory.max >maxupdate.out &&
+	test_cmp inf.exp maxupdate.out
+'
+
+flux setattr log-stderr-level 3
+
+test_done
diff --git a/t/t2411-sdexec-job.t b/t/t2411-sdexec-job.t
new file mode 100755
index 000000000000..bcb845ee47b1
--- /dev/null
+++ b/t/t2411-sdexec-job.t
@@ -0,0 +1,112 @@
+#!/bin/sh
+# ci=system
+
+test_description='Test sdexec job launch'
+
+. $(dirname $0)/sharness.sh
+
+if ! flux version | grep systemd; then
+	skip_all="flux was not built with systemd"
+	test_done
+fi
+if ! systemctl --user show --property Version; then
+	skip_all="user systemd is not running"
+	test_done
+fi
+if ! busctl --user status >/dev/null; then
+	skip_all="user dbus is not running"
+	test_done
+fi
+
+mkdir -p config
+cat >config/config.toml <<EOT
+[systemd]
+sdexec-debug = true
+[exec]
+service-override = true
+EOT
+
+test_under_flux 2 full --config-path=$(pwd)/config
+
+flux exec flux setattr log-stderr-level 7
+
+sdexec="flux exec --service sdexec"
+lptest="flux lptest"
+rkill="flux python ${SHARNESS_TEST_SRCDIR}/scripts/rexec.py kill -s sdexec"
+
+test_expect_success 'job gets exception if sdexec requested but not loaded' '
+	test_must_fail flux run --setattr system.exec.bulkexec.service=sdexec \
+	    -N1 true 2>except.err &&
+	grep "sdexec service is not loaded" except.err
+'
+test_expect_success 'load sdbus,sdexec modules' '
+	flux exec flux module load sdbus &&
+	flux exec flux module load sdexec
+'
+test_expect_success 'clear broker logs' '
+        flux dmesg -C
+'
+test_expect_success 'incorrect bulkexec.service fails' '
+	test_must_fail flux run --setattr system.exec.bulkexec.service=zzz \
+	    -N1 true
+'
+test_expect_success '1-node job works' '
+	flux run --setattr system.exec.bulkexec.service=sdexec \
+	    -N1 true
+'
+test_expect_success 'dump broker logs' '
+        flux dmesg >dmesg.out
+'
+test_expect_success '2-node job works' '
+	flux run --setattr system.exec.bulkexec.service=sdexec \
+	    -N2 true
+'
+test_expect_success 'create a shell userrc that dumps data to stderr' '
+	cat >userrc.lua <<-EOT
+	for i = 1,10,1
+	do
+	    io.stderr:write("foo bar\n")
+	end
+	EOT
+'
+test_expect_success 'run a job that uses that userrc' '
+	flux run --setattr system.exec.bulkexec.service=sdexec \
+	    -o userrc=userrc.lua true
+'
+
+select_log() {
+	jq -r '. | select(.name=="log") | .name'
+}
+test_expect_success 'shell stderr includes 10 distinct lines of data' '
+	flux job eventlog -f json -p exec $(flux job last) | select_log >logs &&
+	count=$(wc -l <logs) &&
+	test $count -eq 10
+'
+test_expect_success 'remove sdexec,sdbus modules' '
+	flux exec flux module remove sdexec &&
+	flux exec flux module remove sdbus
+'
+test_expect_success 'remove job-exec module' '
+	flux module remove job-exec
+'
+test_expect_success 'reconfigure with exec.sdexec-properties not a table' '
+	flux config load <<-EOT
+	exec.sdexec-properties = 42
+	EOT
+'
+test_expect_success 'cannot load job-exec module' '
+	test_must_fail flux module load job-exec
+'
+test_expect_success 'reconfigure with exec.sdexec-properties.key not a string' '
+	flux config load <<-EOT
+	[exec.sdexec-properties]
+	foo = 24
+	EOT
+'
+test_expect_success 'cannot load job-exec module' '
+	test_must_fail flux module load job-exec
+'
+
+flux exec flux setattr log-stderr-level 1
+
+test_done
diff --git a/t/t2500-job-attach.t b/t/t2500-job-attach.t
index e207b7eb2e6d..e9d1eae2e739 100755
--- a/t/t2500-job-attach.t
+++ b/t/t2500-job-attach.t
@@ -8,17 +8,23 @@ test_under_flux 4
 
 flux setattr log-stderr-level 1
 
+runpty=$SHARNESS_TEST_SRCDIR/scripts/runpty.py
+
 test_expect_success 'attach: submit one job' '
-	flux jobspec srun -n1 echo foo | flux job submit >jobid1
+	flux submit echo foo >jobid1
 '
 
 test_expect_success 'attach: job ran successfully' '
 	run_timeout 5 flux job attach $(cat jobid1)
 '
-
-test_expect_success 'attach: --show-events shows clean event' '
+test_expect_success 'attach: --show-events shows finish event' '
 	run_timeout 5 flux job attach \
 		--show-event $(cat jobid1) 2>jobid1.events &&
+	grep finish jobid1.events
+'
+test_expect_success 'attach: --show-events -w clean shows clean event' '
+	run_timeout 5 flux job attach \
+		--show-event -w clean $(cat jobid1) 2>jobid1.events &&
 	grep clean jobid1.events
 '
 test_expect_success 'attach: --show-events shows done event' '
@@ -26,19 +32,120 @@ test_expect_success 'attach: --show-events shows done event' '
 		--show-exec $(cat jobid1) 2>jobid1.exec &&
 	grep done jobid1.exec
 '
+test_expect_success 'attach: --show-status shows job status line' '
+	run_timeout 5 flux job attach \
+		--show-status $(cat jobid1) 2>jobid1.status &&
+	grep "resolving dependencies" jobid1.status &&
+	grep "waiting for resources" jobid1.status &&
+	grep "starting" jobid1.status &&
+	grep "started" jobid1.status
+'
+test_expect_success 'attach: no status is shown with FLUX_ATTACH_NONINTERACTIVE' '
+	(export FLUX_ATTACH_NONINTERACTIVE=t &&
+	 run_timeout 5 flux job attach \
+		--show-status $(cat jobid1) 2>jobid1.status2
+	) &&
+	test_must_fail grep "resolving dependencies" jobid1.status2 &&
+	test_must_fail grep "waiting for resources" jobid1.status2 &&
+	test_must_fail grep "starting" jobid1.status2 &&
+	test_must_fail grep "started" jobid1.status2
 
+'
+test_expect_success 'attach: --show-status properly accounts prolog-start events' '
+	flux jobtap load ${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs/perilog-test.so prolog-count=4 &&
+	jobid2=$(flux submit hostname) &&
+	run_timeout 5 $runpty -f asciicast -o jobid2.out \
+		flux job attach -vEX --show-status $jobid2 &&
+	cat jobid2.out &&
+	grep "resolving dependencies" jobid2.out &&
+	grep "waiting for resources" jobid2.out &&
+	grep "starting" jobid2.out &&
+	grep "started" jobid2.out &&
+	last_prolog_finish_line=$(grep -n prolog-finish jobid2.out \
+		| tail -1 \
+		| cut -f1 -d:) &&
+	first_starting_line=$(grep -n flux-job:.*starting jobid2.out \
+		| tail -1 \
+		| cut -f1 -d:) &&
+	test $first_starting_line -ge $last_prolog_finish_line &&
+	flux job wait-event $jobid2 clean &&
+	flux jobtap remove perilog-test.so
+'
+test_expect_success NO_CHAIN_LINT 'attach: --show-status notes stopped queue' '
+	flux queue stop &&
+	test_when_finished "flux queue start" &&
+	jobid=$(flux submit hostname) &&
+	$runpty -f asciicast -o stopped-queue.out \
+		flux job attach --show-status $jobid &
+	waitfile.lua -v -t 15 -p "default queue stopped" stopped-queue.out &&
+	flux queue start &&
+	wait
+'
+test_expect_success NO_CHAIN_LINT 'attach: --show-status notes stopped named queue' '
+	flux config load <<-EOF &&
+	[queues.batch]
+	[queues.debug]
+	EOF
+	flux queue stop --verbose --all &&
+	jobid=$(flux submit -qbatch hostname) &&
+	$runpty -f asciicast -o stopped-batch.out \
+		flux job attach --show-status $jobid &
+	waitfile.lua -v -t 15 -p "batch queue stopped" stopped-batch.out &&
+	flux queue start --all &&
+	wait &&
+	flux config load </dev/null &&
+	flux queue status
+'
+test_expect_success NO_CHAIN_LINT 'attach: ignores non-fatal exceptions' '
+	flux queue stop &&
+	test_when_finished "flux queue start" &&
+	jobid=$(flux submit hostname) &&
+	$runpty -f asciicast -o status-exception.out \
+		flux job attach --show-status $jobid &
+	waitfile.lua -v -t 15 -p "waiting for resources" status-exception.out &&
+	flux job raise --severity=2 --type=test -m test $(flux job last) &&
+	flux queue start &&
+	wait &&
+	test_must_fail grep "canceling due to exception" status-exception.out &&
+	grep "job.exception" status-exception.out
+'
 test_expect_success 'attach: shows output from job' '
 	run_timeout 5 flux job attach $(cat jobid1) | grep foo
 '
 
 test_expect_success 'attach: submit a job and cancel it' '
-	flux jobspec srun -n1 sleep 30 | flux job submit >jobid2 &&
-	flux job cancel $(cat jobid2)
+	flux submit sleep 30 >jobid2 &&
+	flux cancel $(cat jobid2)
 '
 
 test_expect_success 'attach: exit code reflects cancellation' '
 	! flux job attach $(cat jobid2)
 '
+test_expect_success 'attach: reports task exit code with nonzero exit' '
+	id=$(flux submit sh -c "exit 42") &&
+	test_must_fail flux job attach $id 2>exited.err &&
+	test_debug "cat exited.err" &&
+	grep "exited with exit code 42" exited.err
+'
+test_expect_success 'attach: reports Killed when job tasks are killed' '
+	id=$(flux submit --wait-event=exec.shell.start sleep 30) &&
+	flux job kill -s 9 $id &&
+	test_must_fail_or_be_terminated flux job attach $id 2>killed.err &&
+	test_debug "cat killed.err" &&
+	grep Killed killed.err
+'
+test_expect_success 'attach: reports Terminated when tasks are terminated' '
+	id=$(flux submit --wait-event=exec.shell.start sleep 30) &&
+	flux job kill -s 15 $id &&
+	test_must_fail_or_be_terminated flux job attach $id 2>terminated.err &&
+	test_debug "cat terminated.err" &&
+	grep Terminated terminated.err
+'
+test_expect_success 'attach: reports job shell Killed if job shell is killed' '
+	id=$(flux submit sh -c "kill -9 \$PPID") &&
+	test_must_fail_or_be_terminated flux job attach $id 2>shell-killed.out &&
+	grep "job shell Killed" shell-killed.out
+'
 
 # Usage run_attach seq
 # Run a 30s job, then attach to it in the background
@@ -48,7 +155,7 @@ test_expect_success 'attach: exit code reflects cancellation' '
 run_attach() {
 	local seq=$1
 
-	flux jobspec srun -n1 sleep 30 | flux job submit >jobid${seq}
+	flux submit sleep 30 >jobid${seq}
 	flux job attach -E $(cat jobid${seq}) 2>attach${seq}.err &
 	echo $! >pid${seq}
 	while ! test -s attach${seq}.err; do sleep 0.1; done
@@ -75,7 +182,7 @@ test_expect_success 'attach: SIGINT+SIGTSTP detaches from job' '
 test_expect_success 'attach: detached job was not canceled' '
 	flux job eventlog $(cat jobid4) >events4 &&
 	test_must_fail grep -q cancel events4 &&
-	flux job cancel $(cat jobid4)
+	flux cancel $(cat jobid4)
 '
 
 # Make sure live output occurs by seeing output "before" sleep, but no
@@ -84,19 +191,19 @@ test_expect_success 'attach: detached job was not canceled' '
 # To deal with racyness, script will output an event, which we can
 # wait on
 test_expect_success NO_CHAIN_LINT 'attach: output appears before cancel' '
-        script=$SHARNESS_TEST_SRCDIR/job-attach/outputsleep.sh &&
-	jobid=$(flux jobspec srun -n1 ${script} | flux job submit)
+	script=$SHARNESS_TEST_SRCDIR/job-attach/outputsleep.sh &&
+	jobid=$(flux submit ${script})
 	flux job attach -E ${jobid} 1>attach5.out 2>attach5.err &
-        waitpid=$! &&
-        flux job wait-event --timeout=10.0 -p guest.exec.eventlog ${jobid} test-output-ready &&
-        flux job cancel ${jobid} &&
+	waitpid=$! &&
+	flux job wait-event --timeout=10.0 -p exec ${jobid} test-output-ready &&
+	flux cancel ${jobid} &&
 	! wait ${waitpid} &&
-        grep before attach5.out &&
-        ! grep after attach5.out
+	grep before attach5.out &&
+	! grep after attach5.out
 '
 
 test_expect_success 'attach: output events processed after shell.init failure' '
-	jobid=$(flux mini submit -o initrc=noinitrc hostname) &&
+	jobid=$(flux submit -o initrc=noinitrc hostname) &&
 	flux job wait-event -v ${jobid} clean &&
 	flux job eventlog -p guest.output ${jobid} &&
 	(flux job attach ${jobid} >init-failure.output 2>&1 || true) &&
@@ -104,24 +211,70 @@ test_expect_success 'attach: output events processed after shell.init failure' '
 	grep "FATAL:.*noinitrc: No such file or directory" init-failure.output
 '
 
-which jq >/dev/null 2>&1 && test_set_prereq HAVE_JQ
-
-
 # use a shell function to make sane quoting possible
 filter_log_context() {
-    jq -c '. | select(.name == "log") | .context'
+	jq -c '. | select(.name == "log") | .context'
 }
 
-test_expect_success HAVE_JQ 'attach: -v option displays file and line info in logs' '
-	jobid=$(flux mini submit -o verbose=2 hostname) &&
+test_expect_success 'attach: -v option displays file and line info in logs' '
+	jobid=$(flux submit -o verbose=2 hostname) &&
 	flux job wait-event ${jobid} clean &&
 	flux job eventlog --format=json -p guest.output ${jobid} \
-	    | filter_log_context >verbose.json &&
+		| filter_log_context >verbose.json &&
 	file=$(head -1 verbose.json | jq -r .file) &&
 	line=$(head -1 verbose.json | jq -r .line) &&
 	msg=$(head -1 verbose.json | jq -r .message) &&
 	flux job attach -v $jobid >verbose.output 2>&1 &&
 	grep "$file:$line: $message" verbose.output
 '
-
+test_expect_success 'attach: cannot attach to interactive pty when --read-only specified' '
+	jobid=$(flux submit -o pty.interactive cat) &&
+	test_must_fail flux job attach --read-only $jobid &&
+	$SHARNESS_TEST_SRCDIR/scripts/runpty.py -i none -f asciicast -c q \
+		flux job attach $jobid
+'
+test_expect_success 'attach: --stdin-ranks works' '
+	id=$(flux submit -N4 -o exit-timeout=none -t60s cat) &&
+	echo hello from 0 \
+		| flux job attach -vEX --label-io -i0 $id >stdin-ranks.out &&
+	flux job eventlog -p guest.input $id &&
+	cat <<-EOF >stdin-ranks.expected &&
+	0: hello from 0
+	EOF
+	test_cmp stdin-ranks.expected stdin-ranks.out
+'
+test_expect_success 'attach: --stdin-ranks with invalid idset errors' '
+	id=$(flux submit -t60s cat) &&
+	test_must_fail flux job attach -i 5-0 $id &&
+	flux cancel $id
+'
+test_expect_success 'attach: --stdin-ranks is adjusted to intersection' '
+	id=$(flux submit -n2 -t60s cat) &&
+	echo foo | flux job attach --label-io -i1-2 $id >adjusted.out 2>&1 &&
+	test_debug "cat adjusted.out" &&
+	grep "warning: adjusting --stdin-ranks" adjusted.out
+'
+test_expect_success 'attach: --stdin-ranks cannot be used with --read-only' '
+	id=$(flux submit -n2 -t60s cat) &&
+	test_must_fail flux job attach -i all --read-only $id &&
+	flux cancel $id
+'
+jobpipe=$SHARNESS_TEST_SRCDIR/scripts/pipe.py
+test_expect_success 'attach: writing to stdin of closed tasks returns EPIPE' '
+	id=$(flux submit -N4 -t60s cat) &&
+	$jobpipe $id 0 </dev/null &&
+	test_must_fail $jobpipe $id 0 </dev/null >pipe.out 2>&1 &&
+	$jobpipe $id all </dev/null &&
+	test_debug "cat pipe.out" &&
+	grep -i "Broken pipe" pipe.out
+'
+test_expect_success 'attach: EPERM generates sensible error' '
+	jobid=$(flux job last) &&
+	( export FLUX_HANDLE_USERID=42 &&
+	  export FLUX_HANDLE_ROLEMASK=0x2 &&
+	  test_must_fail flux job attach -vEX $jobid 2>eperm.err
+	) &&
+	test_debug "cat eperm.err" &&
+	grep -i "not your job" eperm.err
+'
 test_done
diff --git a/t/t2501-job-status.t b/t/t2501-job-status.t
index 3f2da6883b12..2b3966734619 100755
--- a/t/t2501-job-status.t
+++ b/t/t2501-job-status.t
@@ -7,15 +7,15 @@ test_description='Test flux job status'
 test_under_flux 2 job
 
 test_expect_success 'status: submit a series of jobs' '
-	zero=$(flux mini submit /bin/true) &&
-	one=$(flux mini submit /bin/false) &&
-	sigint=$(flux mini submit sh -c "kill -INT \$$") &&
-	shell_sigquit=$(flux mini submit sh -c "kill -QUIT \$PPID") &&
-	unsatisfiable=$(flux mini submit -n 1024 hostname) &&
-	killed=$(flux mini submit sleep 600) &&
+	zero=$(flux submit true) &&
+	one=$(flux submit false) &&
+	sigint=$(flux submit sh -c "kill -INT \$$") &&
+	shell_sigquit=$(flux submit sh -c "kill -QUIT \$PPID") &&
+	unsatisfiable=$(flux submit -n 1024 hostname) &&
+	killed=$(flux submit sleep 600) &&
 	flux queue stop &&
-	canceled=$(flux mini submit -n 1024 hostname) &&
-	flux job cancel ${canceled} &&
+	canceled=$(flux submit -n 1024 hostname) &&
+	flux cancel ${canceled} &&
 	flux queue start
 '
 test_expect_success 'status: exits with error with no jobs specified' '
@@ -46,9 +46,44 @@ test_expect_success 'status: --exception-exit-code works' '
 	test_expect_code 42 flux job status -v --exception-exit-code=42 ${canceled} &&
 	test_expect_code 255 flux job status -v --exception-exit-code=255 ${unsatisfiable}
 '
+test_expect_success 'status: flux-job status --json works' '
+	flux job status --json ${zero} | \
+		jq -e ".waitstatus == 0" &&
+	flux job status --json ${one} | \
+		jq -e ".waitstatus == 256"
+'
+test_expect_success 'status: returns most severe exception' '
+	jobid=$(flux submit --urgency=hold sleep 20) &&
+	flux job raise --severity=1 -t test $jobid &&
+	flux job raise --severity=0 -t cancel $jobid &&
+	flux job wait-event -v $jobid clean &&
+	flux job status --exception-exit-code=0 --json ${jobid} &&
+	flux job status --exception-exit-code=0 --json ${jobid} | \
+		jq -e ".exception_severity == 0"
+'
+test_expect_success 'status: returns most severe exception' '
+	jobid=$(flux submit --urgency=hold sleep 0) &&
+	flux job raise --severity=1 -t test $jobid &&
+	flux job raise --severity=2 -t test2 $jobid &&
+	flux job urgency ${jobid} default &&
+	flux job wait-event -v $jobid clean &&
+	flux job status --exception-exit-code=0 --json ${jobid} &&
+	flux job status --exception-exit-code=0 --json ${jobid} | \
+		jq -e ".exception_severity == 1" &&
+	flux job status --exception-exit-code=0 --json ${jobid} | \
+		jq -e ".exception_type == \"test\""
+'
+test_expect_success 'status: exit code ignores nonfatal exceptions' '
+	jobid=$(flux submit --urgency=hold sleep 0) &&
+	flux job raise --severity=1 -t test $jobid &&
+	flux job raise --severity=2 -t test2 $jobid &&
+	flux job urgency ${jobid} 16 &&
+	flux job wait-event -v $jobid clean &&
+	flux job status -vvv $jobid
+'
 test_expect_success 'status: returns 143 (SIGTERM) for canceled running job' '
-	flux job wait-event -p guest.exec.eventlog ${killed} shell.start &&
-	flux job cancel ${killed} &&
+	flux job wait-event -p exec ${killed} shell.start &&
+	flux cancel ${killed} &&
 	test_expect_code 143 flux job status -v ${killed}
 '
 test_expect_success 'status: returns highest status for multiple jobs' '
diff --git a/t/t2600-job-shell-rcalc.t b/t/t2600-job-shell-rcalc.t
index 627d78e34765..9457102f8497 100755
--- a/t/t2600-job-shell-rcalc.t
+++ b/t/t2600-job-shell-rcalc.t
@@ -27,4 +27,33 @@ for file in ${OUTPUTDIR}/*.expected; do
     '
 done
 
+for file in ${OUTPUTDIR}/per-resource/*.expected; do
+    base=$(basename $file .expected)
+    count=$(echo $base | sed 's/.*\.//')
+    base=$(basename $base .${count})
+    type=$(echo $base | sed 's/.*\.//')
+    base=$(basename $base .${type})
+    input=${INPUTDIR}/${base}.json
+    output=${base}.${type}.${count}.out
+    err=${base}.${type}.${count}.err
+     test_expect_success NO_CHAIN_LINT "$(head -1 $file)" '
+        ${rcalc} -R $type $count <$input >$output 2>$err || true &&
+        test_cmp $file $output &&
+        test_cmp $OUTPUTDIR/per-resource/$err $err
+    '
+done
+
+#  More specific rcalc tests
+test_expect_success 'rcalc: distribute 5 slots of size 5 across 6 allocated' '
+	flux R encode -r 0-1 -c 0-14 | \
+	    ${rcalc} --cores-per-slot=5 5 > 55.out &&
+	cat >55.expected <<-EOF &&
+	Distributing 5 tasks across 2 nodes with 30 cores
+	Used 2 nodes
+	0: rank=0 ntasks=3 cores=0-14
+	1: rank=1 ntasks=2 cores=0-14
+	EOF
+	test_cmp 55.expected 55.out
+'
+
 test_done
diff --git a/t/t2601-job-shell-standalone.t b/t/t2601-job-shell-standalone.t
deleted file mode 100755
index 0a16338ae3db..000000000000
--- a/t/t2601-job-shell-standalone.t
+++ /dev/null
@@ -1,168 +0,0 @@
-#!/bin/sh
-#
-test_description='Test flux-shell in --standalone mode'
-
-. `dirname $0`/sharness.sh
-
-#  Run flux-shell under flux command to get correct paths
-FLUX_SHELL="flux ${FLUX_BUILD_DIR}/src/shell/flux-shell"
-
-PMI_INFO=${FLUX_BUILD_DIR}/src/common/libpmi/test_pmi_info
-KVSTEST=${FLUX_BUILD_DIR}/src/common/libpmi/test_kvstest
-
-unset FLUX_URI
-
-test_expect_success 'flux-shell: generate 1-task jobspec and matching R' '
-	flux jobspec srun -N1 -n1 echo Hi >j1 &&
-	cat >R1 <<-EOT
-	{"version": 1, "execution":{ "R_lite":[
-		{ "children": { "core": "0" }, "rank": "0" }
-        ]}}
-	EOT
-'
-test_expect_success 'flux-shell: run 1-task echo job' '
-	${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 0 >echo.out 2>echo.err &&
-	grep Hi echo.out
-'
-test_expect_success 'flux-shell: -v output includes expected task count' '
-	grep task_count=1 echo.err
-'
-test_expect_success 'flux-shell: missing JOBID fails with Usage message' '
-	test_must_fail ${FLUX_SHELL} -s -r 0 -j j1 -R R1 2>nojobid.err &&
-	grep Usage nojobid.err
-'
-test_expect_success 'flux-shell: missing -r fails with standalone message' '
-	test_must_fail ${FLUX_SHELL} -s -j j1 -R R1 0 2>no_r.err &&
-	grep standalone no_r.err
-'
-test_expect_success 'flux-shell: missing -R fails with standalone message' '
-	test_must_fail ${FLUX_SHELL} -s -r 0 -j j1 0 2>no_R.err &&
-	grep standalone no_R.err
-'
-test_expect_success 'flux-shell: missing -j fails with standalone message' '
-	test_must_fail ${FLUX_SHELL} -s -r 0 -R R1 0 2>no_j.err &&
-	grep standalone no_j.err
-'
-test_expect_success 'flux-shell: nonexistent jobspec file fails' '
-	! ${FLUX_SHELL} -v -s -r 0 -j /noexist -R R1 0 \
-		>noexist.out 2>noexist.err &&
-	grep "error opening" noexist.err
-'
-test_expect_success 'flux-shell: malformed jobid fails' '
-	! ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 BADID \
-		>badid.out 2>badid.err &&
-	grep "jobid" badid.err
-'
-test_expect_success 'flux-shell: out of range jobid fails' '
-	! ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 18446744073709551616 \
-		>badid2.out 2>badid2.err &&
-	grep "jobid" badid2.err
-'
-test_expect_success 'flux-shell: out of range broker rank fails' '
-	test_must_fail ${FLUX_SHELL} -s -r 4294967296 -j j1 -R R1 0 2>er.err &&
-	grep -i option er.err
-'
-test_expect_success 'flux-shell: wrong range broker rank fails' '
-	test_must_fail ${FLUX_SHELL} -s -r 8 -j j1 -R R1 0 2>wr.err &&
-	grep -i fetching wr.err
-'
-test_expect_success 'flux-shell: unknown argument fails' '
-	test_must_fail ${FLUX_SHELL} --FOO -s -r 0 -j j1 -R R1 0 2>alien.err &&
-	grep -i unrecognized alien.err
-'
-test_expect_success 'flux-shell: generate 2-task jobspec and matching R' '
-	flux jobspec srun -N1 -n2 printenv >j2 &&
-	cat >R2 <<-EOT
-	{"version": 1, "execution":{ "R_lite":[
-		{ "children": { "core": "0-1" }, "rank": "0" }
-        ]}}
-	EOT
-'
-test_expect_success 'flux-shell: run 2-task printenv job' '
-	${FLUX_SHELL} -v -s -r 0 -j j2 -R R2 42 \
-		>printenv.out 2>printenv.err
-'
-test_expect_success 'flux-shell: 0: completed with no error' '
-	grep "task 0 complete status=0" printenv.err
-'
-test_expect_success 'flux-shell: 1: completed with no error' '
-	grep "task 1 complete status=0" printenv.err
-'
-test_expect_success 'flux-shell: 0: FLUX_TASK_LOCAL_ID, TASK_RANK set' '
-	grep FLUX_TASK_LOCAL_ID=0 printenv.out &&
-	grep FLUX_TASK_RANK=0 printenv.out
-'
-test_expect_success 'flux-shell: 1: FLUX_TASK_LOCAL_ID, TASK_RANK set' '
-	grep FLUX_TASK_LOCAL_ID=1 printenv.out &&
-	grep FLUX_TASK_RANK=1 printenv.out
-'
-test_expect_success 'flux-shell: FLUX_JOB_SIZE, JOB_NNODES, JOB_ID set' '
-	grep FLUX_JOB_SIZE=2 printenv.out &&
-	grep FLUX_JOB_NNODES=1 printenv.out &&
-	grep FLUX_JOB_ID=42 printenv.out
-'
-test_expect_success 'flux-shell: FLUX_URI is not set by shell' '
-	test_must_fail grep FLUX_URI printenv.out
-'
-test_expect_success 'flux-shell: 0: PMI_RANK set' '
-	grep PMI_RANK=0 printenv.out
-'
-test_expect_success 'flux-shell: 1: PMI_RANK set' '
-	grep PMI_RANK=1 printenv.out
-'
-test_expect_success 'flux-shell: PMI_SIZE, PMI_FD set' '
-	grep PMI_SIZE=2 printenv.out &&
-	grep PMI_FD= printenv.out
-'
-test_expect_success 'flux-shell: generate 8-task bash exit rank job' '
-	flux jobspec srun -N1 -n8 bash -c "exit \$FLUX_TASK_RANK" >j8 &&
-	cat >R8 <<-EOT
-	{"version": 1, "execution":{ "R_lite":[
-		{ "children": { "core": "0-7" }, "rank": "0" }
-        ]}}
-	EOT
-'
-test_expect_success 'flux-shell: environ in jobspec is set for task' '
-	flux jobspec srun --export ENVTEST=foo printenv >je &&
-	${FLUX_SHELL} -v -s -r 0 -j je -R R1 42 \
-		>printenv2.out 2>printenv2.err &&
-	grep ENVTEST=foo printenv2.out
-'
-test_expect_success 'flux-shell: shell PMI works' '
-	flux jobspec srun -N1 -n8 ${PMI_INFO} >j8pmi &&
-	${FLUX_SHELL} -v -s -r 0 -j j8pmi -R R8 51 \
-		>pmi_info.out 2>pmi_info.err
-'
-test_expect_success 'flux-shell: shell PMI exports clique info' '
-	flux jobspec srun -N1 -n8 ${PMI_INFO} -c >j8pmi_clique &&
-	${FLUX_SHELL} -v -s -r 0 -j j8pmi_clique -R R8 51 \
-		>pmi_clique.out 2>pmi_clique.err &&
-	COUNT=$(grep "clique=0,1,2,3,4,5,6,7" pmi_clique.out | wc -l) &&
-	test ${COUNT} -eq 8
-'
-test_expect_success 'flux-shell: shell PMI KVS works' '
-	flux jobspec srun -N1 -n8 ${KVSTEST} >j8kvs &&
-	${FLUX_SHELL} -v -s -r 0 -j j8kvs -R R8 52 \
-		>kvstest.out 2>kvstest.err
-'
-test_expect_success 'flux-shell: shell can launch flux' '
-	flux jobspec srun -N1 -n8 flux start flux comms info >j8flux &&
-	${FLUX_SHELL} -v -s -r 0 -j j8flux -R R8 39 \
-		>flux.out 2>flux.err &&
-	grep size=8 flux.out
-'
-
-test_expect_success 'flux-shell: shell exits with highest task exit value' '
-	test_expect_code 7 ${FLUX_SHELL} -v -s -r 0 -j j8 -R R8 69 \
-		>exit.out 2>exit.err
-'
-
-test_expect_success 'flux-shell: shell forwards signals to tasks' '
-	flux jobspec srun -n1 bash -c "kill \$PPID; sleep 10" > j9 &&
-	test_expect_code  $((128+15)) \
-		${FLUX_SHELL} -v -s -r 0 -j j9 -R R8 69 \
-			>sigterm.out 2>sigterm.err &&
-	grep "forwarding signal 15" sigterm.err
-'
-
-test_done
diff --git a/t/t2602-job-shell.t b/t/t2602-job-shell.t
index b5df47aa7b74..a70516094055 100755
--- a/t/t2602-job-shell.t
+++ b/t/t2602-job-shell.t
@@ -10,14 +10,40 @@ flux setattr log-stderr-level 1
 
 PMI_INFO=${FLUX_BUILD_DIR}/src/common/libpmi/test_pmi_info
 KVSTEST=${FLUX_BUILD_DIR}/src/common/libpmi/test_kvstest
-LPTEST=${SHARNESS_TEST_DIRECTORY}/shell/lptest
+LPTEST="flux lptest"
+waitfile=${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua
+
+test_expect_success 'job-shell: errors on unknown argument' '
+	id=$(flux submit hostname) &&
+	test_expect_code 1 ${FLUX_BUILD_DIR}/src/shell/flux-shell --foo ${id}
+'
+
+test_expect_success 'job-shell: reads J not jobspec' '
+	id=$(flux submit --wait-event=priority \
+		-n1 --urgency=hold true) &&
+	flux job info ${id} jobspec \
+		| jq ".tasks[0].command[0] = \"false\"" >jobspec.new &&
+	flux kvs put \
+		$(flux job id --to=kvs ${id}).jobspec="$(cat jobspec.new)" &&
+	flux job urgency ${id} default &&
+	flux job attach -vEX ${id}
+'
+
+test_expect_success 'job-shell: fails on modified J' '
+	id=$(flux submit --wait-event=priority \
+		-n1 --urgency=hold true) &&
+	flux job info ${id} J | sed s/./%/85 > J.new &&
+	flux kvs put \
+		$(flux job id --to=kvs ${id}).J="$(cat J.new)" &&
+	flux job urgency ${id} default &&
+	test_must_fail flux job attach -vEX ${id}
+'
 
 test_expect_success 'job-shell: execute across all ranks' '
-        id=$(flux jobspec srun -N4 bash -c \
-            "flux kvs put test1.\$FLUX_TASK_RANK=\$FLUX_TASK_LOCAL_ID" \
-            | flux job submit) &&
-        flux job attach --show-events $id &&
-        kvsdir=$(flux job id --to=kvs $id) &&
+	id=$(flux submit -n4 -N4 bash -c \
+		"flux kvs put test1.\$FLUX_TASK_RANK=\$FLUX_TASK_LOCAL_ID") &&
+	flux job attach --show-events $id &&
+	kvsdir=$(flux job id --to=kvs $id) &&
 	sort >test1.exp <<-EOT &&
 	${kvsdir}.guest.test1.0 = 0
 	${kvsdir}.guest.test1.1 = 0
@@ -28,11 +54,10 @@ test_expect_success 'job-shell: execute across all ranks' '
 	test_cmp test1.exp test1.out
 '
 test_expect_success 'job-shell: execute 2 tasks per rank' '
-        id=$(flux jobspec srun -N4 -n8 bash -c \
-            "flux kvs put test2.\$FLUX_TASK_RANK=\$FLUX_TASK_LOCAL_ID" \
-            | flux job submit) &&
-        flux job attach --show-events $id &&
-        kvsdir=$(flux job id --to=kvs $id) &&
+	id=$(flux submit -N4 -n8 bash -c \
+		"flux kvs put test2.\$FLUX_TASK_RANK=\$FLUX_TASK_LOCAL_ID") &&
+	flux job attach --show-events $id &&
+	kvsdir=$(flux job id --to=kvs $id) &&
 	sort >test2.exp <<-EOT &&
 	${kvsdir}.guest.test2.0 = 0
 	${kvsdir}.guest.test2.1 = 1
@@ -46,24 +71,24 @@ test_expect_success 'job-shell: execute 2 tasks per rank' '
 	flux kvs dir ${kvsdir}.guest.test2 | sort >test2.out &&
 	test_cmp test2.exp test2.out
 '
-test_expect_success 'job-shell: /bin/true exit code propagated' '
-        id=$(flux jobspec srun -n1 /bin/true | flux job submit) &&
+test_expect_success 'job-shell: true exit code propagated' '
+	id=$(flux submit true) &&
 	flux job wait-event $id finish >true.finish.out &&
 	grep status=0 true.finish.out
 '
-test_expect_success 'job-shell: /bin/false exit code propagated' '
-        id=$(flux jobspec srun -n1 /bin/false | flux job submit) &&
+test_expect_success 'job-shell: false exit code propagated' '
+	id=$(flux submit false) &&
 	flux job wait-event $id finish >false.finish.out &&
 	grep status=256 false.finish.out
 '
 test_expect_success 'job-shell: PMI works' '
-        id=$(flux jobspec srun -N4 ${PMI_INFO} | flux job submit) &&
+	id=$(flux submit -n4 -N4 ${PMI_INFO}) &&
 	flux job attach $id >pmi_info.out 2>pmi_info.err &&
 	grep size=4 pmi_info.out
 '
 test_expect_success 'pmi-shell: PMI cliques are correct for 1 ppn' '
-        id=$(flux jobspec srun -N4 -n4 ${PMI_INFO} -c | flux job submit) &&
-	flux job attach $id >pmi_clique1.raw &&
+	flux run -N4 -n4 \
+		${PMI_INFO} -c >pmi_clique1.raw &&
 	sort -snk1 <pmi_clique1.raw >pmi_clique1.out &&
 	sort >pmi_clique1.exp <<-EOT &&
 	0: clique=0
@@ -74,8 +99,8 @@ test_expect_success 'pmi-shell: PMI cliques are correct for 1 ppn' '
 	test_cmp pmi_clique1.exp pmi_clique1.out
 '
 test_expect_success 'pmi-shell: PMI cliques are correct for 2 ppn' '
-        id=$(flux jobspec srun -N2 -n4 ${PMI_INFO} -c | flux job submit) &&
-	flux job attach $id >pmi_clique2.raw &&
+	flux run \
+		-N2 -n4 ${PMI_INFO} -c >pmi_clique2.raw &&
 	sort -snk1 <pmi_clique2.raw >pmi_clique2.out &&
 	sort >pmi_clique2.exp <<-EOT &&
 	0: clique=0,1
@@ -86,8 +111,8 @@ test_expect_success 'pmi-shell: PMI cliques are correct for 2 ppn' '
 	test_cmp pmi_clique2.exp pmi_clique2.out
 '
 test_expect_success 'pmi-shell: PMI cliques are correct for irregular ppn' '
-        id=$(flux jobspec srun -N4 -n5 ${PMI_INFO} -c | flux job submit) &&
-	flux job attach $id >pmi_cliquex.raw &&
+	flux run -N4 -n5 \
+		${PMI_INFO} -c >pmi_cliquex.raw &&
 	sort -snk1 <pmi_cliquex.raw >pmi_cliquex.out &&
 	sort >pmi_cliquex.exp <<-EOT &&
 	0: clique=0,1
@@ -99,7 +124,7 @@ test_expect_success 'pmi-shell: PMI cliques are correct for irregular ppn' '
 	test_cmp pmi_cliquex.exp pmi_cliquex.out
 '
 test_expect_success 'job-shell: PMI KVS works' '
-        id=$(flux jobspec srun -N4 ${KVSTEST} | flux job submit) &&
+	id=$(flux submit -n4 -N4 ${KVSTEST}) &&
 	flux job attach $id >kvstest.out &&
 	grep "t phase" kvstest.out
 '
@@ -110,9 +135,9 @@ test_expect_success 'job-exec: decrease kill timeout for tests' '
 #   may exit with 143, killed by signal
 #
 test_expect_success 'job-shell: PMI_Abort works' '
-	! flux mini run -N4 -n4 ${PMI_INFO} --abort=1 >abort.log 2>&1 &&
+	! flux run -N4 -n4 ${PMI_INFO} --abort=1 >abort.log 2>&1 &&
 	test_debug "cat abort.log" &&
-	grep "job.exception.*MPI_Abort: Test abort error." abort.log
+	grep "job.exception.*PMI_Abort: Test abort error." abort.log
 '
 test_expect_success 'job-shell: create expected I/O output' '
 	${LPTEST} | sed -e "s/^/0: /" >lptest.exp &&
@@ -121,7 +146,7 @@ test_expect_success 'job-shell: create expected I/O output' '
 	done) >lptest4.exp
 '
 test_expect_success 'job-shell: verify output of 1-task lptest job' '
-        id=$(flux jobspec srun -n1 ${LPTEST} | flux job submit) &&
+	id=$(flux submit ${LPTEST}) &&
 	flux job wait-event $id finish &&
 	flux job attach -l $id >lptest.out &&
 	test_cmp lptest.exp lptest.out
@@ -134,56 +159,74 @@ test_expect_success 'job-shell: verify output of 1-task lptest job' '
 #
 
 test_expect_success 'job-shell: verify output of 1-task lptest job on stderr' '
-        id=$(flux jobspec srun -n1 bash -c "${LPTEST} >&2" \
-		| flux job submit) &&
+	flux run --dry-run bash -c "${LPTEST} >&2" \
+		| $jq ".attributes.system.shell.options.output.stderr.buffer.type = \"line\"" \
+		> 1task_lptest.json &&
+	id=$(cat 1task_lptest.json | flux job submit) &&
 	flux job attach -l $id 2>lptest.err &&
-        sed -i -e "/stdin EOF could not be sent/d" lptest.err &&
+	sed -i -e "/stdin EOF could not be sent/d" lptest.err &&
 	test_cmp lptest.exp lptest.err
 '
 test_expect_success 'job-shell: verify output of 4-task lptest job' '
-        id=$(flux jobspec srun -N4 ${LPTEST} | flux job submit) &&
+	id=$(flux submit -n4 -N4 ${LPTEST}) &&
 	flux job attach -l $id >lptest4_raw.out &&
 	sort -snk1 <lptest4_raw.out >lptest4.out &&
 	test_cmp lptest4.exp lptest4.out
 '
 test_expect_success 'job-shell: verify output of 4-task lptest job on stderr' '
-        id=$(flux jobspec srun -N4 bash -c "${LPTEST} 1>&2" \
-		| flux job submit) &&
+	flux run --dry-run -n4 -N4 bash -c "${LPTEST} 1>&2" \
+		| $jq ".attributes.system.shell.options.output.stderr.buffer.type = \"line\"" \
+		> 4task_lptest.json &&
+	id=$(cat 4task_lptest.json | flux job submit) &&
 	flux job attach -l $id 2>lptest4_raw.err &&
 	sort -snk1 <lptest4_raw.err >lptest4.err &&
-        sed -i -e "/stdin EOF could not be sent/d" lptest4.err &&
+	sed -i -e "/stdin EOF could not be sent/d" lptest4.err &&
 	test_cmp lptest4.exp lptest4.err
 '
 test_expect_success LONGTEST 'job-shell: verify 10K line lptest output works' '
 	${LPTEST} 79 10000 | sed -e "s/^/0: /" >lptestXXL.exp &&
-        id=$(flux jobspec srun -n1 ${LPTEST} 79 10000 | flux job submit) &&
+	id=$(flux submit ${LPTEST} 79 10000) &&
 	flux job attach -l $id >lptestXXL.out &&
 	test_cmp lptestXXL.exp lptestXXL.out
 '
+# N.B. sleepinf.sh and wait-event on job data to workaround
+# rare job startup race.  See #5210
+test_expect_success 'create helper job submission script' '
+	cat >sleepinf.sh <<-EOT &&
+	#!/bin/sh
+	echo "job started"
+	sleep inf
+	EOT
+	chmod +x sleepinf.sh
+'
 test_expect_success 'job-shell: test shell kill event handling' '
-	id=$(flux jobspec srun -N4 sleep 60 | flux job submit) &&
-	flux job wait-event $id start &&
+	id=$(flux submit -n4 -N4 ./sleepinf.sh) &&
+	flux job wait-event -p exec $id shell.init &&
+	flux job wait-event -p output $id data &&
 	flux job kill $id &&
 	flux job wait-event $id finish >kill1.finish.out &&
 	grep status=$((15+128<<8)) kill1.finish.out
 '
 test_expect_success 'job-shell: test shell kill event handling: SIGKILL' '
-	id=$(flux jobspec srun -N4 sleep 60 | flux job submit) &&
-	flux job wait-event $id start &&
+	id=$(flux submit -n4 -N4 ./sleepinf.sh) &&
+	flux job wait-event -p exec $id shell.init &&
+	flux job wait-event -p output $id data &&
 	flux job kill -s SIGKILL $id &&
 	flux job wait-event $id finish >kill2.finish.out &&
 	grep status=$((9+128<<8)) kill2.finish.out
 '
 test_expect_success 'job-shell: test shell kill event handling: numeric signal' '
-	id=$(flux jobspec srun -N4 sleep 60 | flux job submit) &&
-	flux job wait-event $id start &&
+	id=$(flux submit -n4 -N4 ./sleepinf.sh) &&
+	flux job wait-event -p exec $id shell.init &&
+	flux job wait-event -p output $id data &&
 	flux job kill -s 2 $id &&
 	flux job wait-event $id finish >kill3.finish.out &&
 	grep status=$((2+128<<8)) kill3.finish.out
 '
 test_expect_success 'job-shell: mangled shell kill event logged' '
-	id=$(flux jobspec srun -N4 sleep 60 | flux job submit) &&
-	flux job wait-event $id start &&
+	id=$(flux submit -n4 -N4 ./sleepinf.sh | flux job id) &&
+	flux job wait-event -p exec $id shell.init &&
+	flux job wait-event -p output $id data &&
 	flux event pub shell-${id}.kill "{}" &&
 	flux job kill ${id} &&
 	flux job wait-event -vt 1 $id finish >kill4.finish.out &&
@@ -192,8 +235,9 @@ test_expect_success 'job-shell: mangled shell kill event logged' '
 	grep "ignoring malformed event" kill4.log
 '
 test_expect_success 'job-shell: shell kill event: kill(2) failure logged' '
-	id=$(flux jobspec srun -N4 sleep 60 | flux job submit) &&
-	flux job wait-event $id start &&
+	id=$(flux submit -n4 -N4 ./sleepinf.sh | flux job id) &&
+	flux job wait-event -p exec $id shell.init &&
+	flux job wait-event -p output $id data &&
 	flux event pub shell-${id}.kill "{\"signum\":199}" &&
 	flux job kill ${id} &&
 	flux job wait-event $id finish >kill5.finish.out &&
@@ -202,4 +246,152 @@ test_expect_success 'job-shell: shell kill event: kill(2) failure logged' '
 	grep "signal 199: Invalid argument" kill5.log &&
 	grep status=$((15+128<<8)) kill5.finish.out
 '
+test_expect_success NO_CHAIN_LINT 'job-shell: cover stdout unbuffered output' '
+	cat <<-EOF >stdout-unbuffered.sh &&
+	#!/bin/sh
+	printf abcd
+	sleep 60
+	EOF
+	chmod +x stdout-unbuffered.sh &&
+	id=$(flux submit --unbuffered ./stdout-unbuffered.sh)
+	flux job attach $id > stdout-unbuffered.out &
+	$waitfile --count=1 --timeout=10 --pattern=abcd stdout-unbuffered.out &&
+	flux cancel $id
+'
+test_expect_success NO_CHAIN_LINT 'job-shell: cover stderr unbuffered output' '
+	cat <<-EOF >stderr-unbuffered.sh &&
+	#!/bin/sh
+	printf efgh >&2
+	sleep 60
+	EOF
+	chmod +x stderr-unbuffered.sh &&
+	id=$(flux submit --unbuffered ./stderr-unbuffered.sh)
+	flux job attach $id 1> stdout.out 2> stderr-unbuffered.out &
+	$waitfile --count=1 --timeout=10 --pattern=efgh stderr-unbuffered.out &&
+	flux cancel $id
+'
+test_expect_success NO_CHAIN_LINT 'job-shell: cover stdout line output' '
+	cat <<-EOF >stdout-line.sh &&
+	#!/bin/sh
+	printf "ijkl\n"
+	sleep 60
+	EOF
+	chmod +x stdout-line.sh &&
+	id=$(flux submit \
+		--setopt "output.stdout.buffer.type=\"line\"" \
+		./stdout-line.sh)
+	flux job attach $id > stdout-line.out &
+	$waitfile --count=1 --timeout=10 --pattern=ijkl stdout-line.out &&
+	flux cancel $id
+'
+test_expect_success NO_CHAIN_LINT 'job-shell: cover stderr line output' '
+	cat <<-EOF >stderr-line.sh &&
+	#!/bin/sh
+	printf "mnop\n" >&2
+	sleep 60
+	EOF
+	chmod +x stderr-line.sh &&
+	id=$(flux submit \
+		--setopt "output.stderr.buffer.type=\"line\"" \
+		./stderr-line.sh)
+	flux job attach $id 1> stdout.out 2> stderr-line.out &
+	$waitfile --count=1 --timeout=10 --pattern=mnop stderr-line.out &&
+	flux cancel $id
+'
+test_expect_success 'job-shell: cover invalid buffer type' '
+	id=$(flux submit \
+		--setopt "output.stderr.buffer.type=\"foobar\"" \
+		hostname) &&
+	flux job wait-event $id clean &&
+	flux job attach $id 2> stderr-invalid-buffering.out &&
+	grep "invalid buffer type" stderr-invalid-buffering.out
+'
+test_expect_success 'job-shell: creates missing TMPDIR by default' '
+	TMPDIR=$(pwd)/mytmpdir flux run true &&
+	test -d mytmpdir
+'
+test_expect_success 'job-shell: uses /tmp if TMPDIR cannot be created' '
+	TMPDIR=/baddir flux run printenv TMPDIR >badtmp.out 2>badtmp.err &&
+	test_debug "cat badtmp.out badtmp.err" &&
+	grep /tmp badtmp.out
+'
+test_expect_success 'job-shell: unset TMPDIR stays unset' '
+	flux run --env=-TMPDIR sh -c "test -z \$TMPDIR"
+'
+test_expect_success 'job-shell: FLUX_JOB_TMPDIR is set and is a directory' '
+	flux run sh -c "test -d \$FLUX_JOB_TMPDIR"
+'
+
+test_expect_success 'job-shell: FLUX_JOB_TMPDIR is cleaned up after job' '
+	jobtmp=$(flux run printenv FLUX_JOB_TMPDIR) &&
+	test_must_fail test -d $jobtmp
+'
+
+test_expect_success 'job-shell: make rundir temporarily unwritable' '
+	chmod 500 $(flux getattr rundir)
+'
+
+test_expect_success 'job-shell: FLUX_JOB_TMPDIR is created in TMPDIR' '
+	TMPDIR=$(pwd)/mytmpdir \
+		flux run printenv FLUX_JOB_TMPDIR >tmpdir.out &&
+	grep $(pwd)/mytmpdir tmpdir.out
+'
+
+test_expect_success 'job-shell: job fails if FLUX_JOB_TMPDIR cannot be created' '
+	chmod u-w mytmpdir &&
+	! TMPDIR=$(pwd)/mytmpdir \
+		flux run true 2>badjobtmp.err &&
+	grep exception badjobtmp.err
+'
+
+test_expect_success 'job-shell: restore rundir writability' '
+	chmod 700 $(flux getattr rundir)
+'
+
+test_expect_success 'job-shell: fails if FLUX_EXEC_PROTOCOL_FD incorrect' '
+	cat <<-EOF >shell2.sh &&
+	#!/bin/sh
+	export FLUX_EXEC_PROTOCOL_FD=foo
+	exec ${FLUX_BUILD_DIR}/src/shell/flux-shell "\$@"
+	EOF
+	chmod +x shell2.sh &&
+	test_must_fail flux run \
+		--setattr=system.exec.job_shell=$(pwd)/shell2.sh \
+		-n2 -N2 hostname 2>protocol_fd_invalid.err &&
+	grep FLUX_EXEC_PROTOCOL_FD protocol_fd_invalid.err
+'
+test_expect_success 'job-shell: corrects HOSTNAME environment variable' '
+	HOSTNAME=incorrect \
+		flux run -n1 printenv HOSTNAME >hostname.out &&
+	test_debug "cat hostname.out" &&
+	test "$(hostname)" = "$(cat hostname.out)"
+'
+test_expect_success 'job-shell: leaves HOSTNAME unset if not set' '
+	( unset HOSTNAME &&
+	  test_expect_code 1 flux run -n1 printenv HOSTNAME >hostname.out2
+	) &&
+	test_must_be_empty hostname.out2
+'
+#  Note: in below tests, os.exit(True) returns with nonzero exit code,
+#   so the sense of the tests is reversed so the tasks exit with zero exit
+#   code for success.
+#
+test_expect_success 'job-shell: runs tasks in process group by default' '
+	flux run -n2 \
+	    flux python -c "import os,sys; sys.exit(os.getpid() != os.getpgrp())"
+'
+
+test_expect_success 'job-shell: -o nosetpgrp works' '
+	flux run -n2 -o nosetpgrp \
+	    flux python -c "import os,sys; sys.exit(os.getpid() == os.getpgrp())"
+'
+
+# Check that job shell inherits FLUX_F58_FORCE_ASCII.
+# If not, then output filename will not match jobid returned by submit,
+#  since one will be in UTF-8 and the other in ascii.
+test_expect_success 'job-shell: shell inherits FLUX_F58_FORCE_ASCII from job' '
+	FLUX_F58_FORCE_ASCII=t \
+		id=$(flux submit --wait --output={{id}}.out hostname) &&
+	test -f ${id}.out
+'
 test_done
diff --git a/t/t2603-job-shell-initrc.t b/t/t2603-job-shell-initrc.t
index bbe6f87587f1..6af1f112d2da 100755
--- a/t/t2603-job-shell-initrc.t
+++ b/t/t2603-job-shell-initrc.t
@@ -4,10 +4,7 @@ test_description='Test flux-shell initrc.lua implementation'
 
 . `dirname $0`/sharness.sh
 
-test_under_flux 2
-
-jq=$(which jq 2>/dev/null)
-test -z "$jq" || test_set_prereq HAVE_JQ
+test_under_flux 4
 
 FLUX_SHELL="${FLUX_BUILD_DIR}/src/shell/flux-shell"
 
@@ -15,42 +12,60 @@ INITRC_TESTDIR="${SHARNESS_TEST_SRCDIR}/shell/initrc"
 INITRC_PLUGINPATH="${SHARNESS_TEST_DIRECTORY}/shell/plugins/.libs"
 
 # test initrc files need to be able to find fluxometer.lua:
-export LUA_PATH="$LUA_PATH;${SHARNESS_TEST_DIRECTORY}/?.lua"
+export LUA_PATH="${SHARNESS_TEST_DIRECTORY}/?.lua;$(lua -e 'print(package.path)')"
 
-test_expect_success 'flux-shell: initrc: generate 1-task jobspec and matching R' '
-	flux jobspec srun -N1 -n1 echo Hi >j1 &&
-	cat >R1 <<-EOT
-	{"version": 1, "execution":{ "R_lite":[
-		{ "children": { "core": "0,1" }, "rank": "0" }
-        ]}}
-	EOT
+test_expect_success 'flux-shell: initrc: conf.shell_* attributes are set' '
+	flux getattr conf.shell_initrc &&
+	flux getattr conf.shell_pluginpath
 '
-
-test_expect_success 'flux-shell: initrc: loading missing initrc fails' '
-	test_expect_code 1 ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=nosuchfile 0
+test_expect_success 'flux-shell: initrc: conf.shell_initrc can be set' '
+	cat <<-EOF >test-initrc.lua &&
+	shell.log("loaded test-initrc")
+	EOF
+	initrc_old=$(flux getattr conf.shell_initrc) &&
+	flux setattr conf.shell_initrc $(pwd)/test-initrc.lua &&
+	flux run true > test-initrc.output 2>&1 &&
+	test_debug "cat test-initrc.output" &&
+	grep "loaded test-initrc" test-initrc.output &&
+	flux setattr conf.shell_initrc "${initrc_old}"
+'
+test_expect_success 'flux-shell: initrc: plugin.searchpath set via broker attr' '
+	cat <<-EOF >print-searchpath.lua &&
+	shell.log("plugin.searchpath = "..plugin.searchpath)
+	EOF
+	old_pluginpath=$(flux getattr conf.shell_pluginpath) &&
+	flux setattr conf.shell_pluginpath /test/foo &&
+	flux run -o initrc=$(pwd)/print-searchpath.lua true \
+		>print-searchpath.out 2>&1 &&
+	test_debug "cat print-searchpath.out" &&
+	grep "plugin.searchpath = /test/foo" print-searchpath.out &&
+	flux setattr conf.shell_pluginpath "${old_pluginpath}"
+	
+'
+test_expect_success 'flux-shell: default initrc obeys FLUX_SHELL_RC_PATH' '
+	mkdir test-dir.d &&
+	cat >test-dir.d/test.lua <<-EOF &&
+	shell.log ("plugin loaded from test-dir.d")
+	EOF
+	FLUX_SHELL_RC_PATH=$(pwd)/test-dir.d \
+	  flux run hostname >rcpath.log 2>&1 &&
+	grep "plugin loaded from test-dir.d" rcpath.log
 '
 test_expect_success 'flux-shell: initrc: specifying initrc of /dev/null works' '
-	${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=/dev/null 0 > devnull.log 2>&1 &&
+	flux run -o verbose -o initrc=/dev/null true > devnull.log 2>&1 &&
 	test_debug "cat devnull.log" &&
 	grep "Loading /dev/null" devnull.log
 '
-test_expect_success HAVE_JQ 'flux-shell: initrc: bad initrc in jobspec fails' '
-	flux jobspec srun -N1 -n1 echo Hi \
-	    | jq ".attributes.system.shell.options.initrc = \"nosuchfile\"" \
-	    > j2 &&
-	test_expect_code 1 ${FLUX_SHELL} -v -s -r 0 -j j2 -R R1 0
+test_expect_success 'flux-shell: initrc: bad initrc in jobspec fails' '
+	test_must_fail_or_be_terminated \
+		flux run -o verbose -o initrc=nosuchfile true
 '
 test_expect_success 'flux-shell: initrc: in jobspec works' '
 	name=ok &&
 	cat >${name}.lua <<-EOT &&
-	    print ("jobspec initrc OK")
+	    io.stderr:write ("jobspec initrc OK\n")
 	EOT
-	flux jobspec srun -N1 -n1 echo Hi \
-	    | jq ".attributes.system.shell.options.initrc = \"${name}.lua\"" \
-	    > j3 &&
-	${FLUX_SHELL} -v -s -r 0 -j j3 -R R1 0 > ${name}.log 2>&1 &&
+	flux run -o initrc=${name}.lua true > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep "jobspec initrc OK" ${name}.log
 '
@@ -59,18 +74,18 @@ test_expect_success 'flux-shell: initrc: failed initrc causes termination' '
 	cat >${name}.lua <<-EOT &&
 	=
 	EOT
-	test_expect_code 1 ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0 > ${name}.log 2>&1 &&
+	test_must_fail_or_be_terminated \
+	    flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep "unexpected symbol" ${name}.log
 '
 test_expect_success 'flux-shell: initrc: access task in non-task context fails' '
 	name=invalid-access &&
-	cat >${name}.lua <<-EOT &&
-	    print (task.info)
+	cat >${name}.lua <<-"EOT" &&
+	    io.stderr:write (task.info .. "\n")
 	EOT
-	test_expect_code 1 ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0 > ${name}.log 2>&1 &&
+	test_must_fail_or_be_terminated \
+	    flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep "attempt to access task outside of task context" ${name}.log
 '
@@ -79,22 +94,36 @@ test_expect_success 'flux-shell: initrc: bad plugin.register usage fails' '
 	cat >${name}.lua <<-EOT &&
 	    plugin.register {}
 	EOT
-	test_expect_code 1 ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0 > ${name}.log 2>&1 &&
+	test_must_fail_or_be_terminated \
+	    flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep "required handlers table missing" ${name}.log
 '
+test_expect_success 'flux-shell: initrc: bad plugin.register usage fails' '
+	name=handlers-not-array &&
+	cat >${name}.lua <<-"EOT" &&
+	    plugin.register {
+          handlers = {
+             topic = "*",
+             fn = function (topic) shell.log (topic) end
+          }
+        }
+	EOT
+	test_must_fail_or_be_terminated \
+	    flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
+	test_debug "cat ${name}.log" &&
+	grep "no entries" ${name}.log
+'
 test_expect_success 'flux-shell: initrc: assignment to shell method fails' '
 	name=bad-assign &&
 	cat >${name}.lua <<-EOT &&
 	    shell.getenv = "my data"
 	EOT
-	test_expect_code 1 ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0 > ${name}.log 2>&1 &&
+	test_must_fail_or_be_terminated \
+	    flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep " attempt to set read-only field" ${name}.log
 '
-
 test_expect_success 'flux-shell: initrc: return false from plugin aborts shell' '
 	name=failed-return &&
 	cat >${name}.lua <<-EOT &&
@@ -105,41 +134,35 @@ test_expect_success 'flux-shell: initrc: return false from plugin aborts shell'
               }
 	    }
 	EOT
-	test_expect_code 1 ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0 > ${name}.log 2>&1 &&
+	test_must_fail_or_be_terminated \
+	    flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep "plugin.*: shell.init failed" ${name}.log
 '
-test_expect_success HAVE_JQ 'flux-shell: initrc: add test options to jobspec' "
-	cat j1 | jq '.attributes.system.shell.options.test = \"test\"' \
-		> j.initrc
-"
-
-for initrc in ${INITRC_TESTDIR}/tests/*.lua; do
-    test_expect_success "flux-shell: initrc: runtest $(basename $initrc)" '
-	${FLUX_SHELL} -v -s -r 0 -j j.initrc -R R1 --initrc=${initrc} 0
-    '
-done
-
+# Submit all test initrcs to instance:
+test_expect_success 'flux-shell: initrc: run tests in shell/initrc/tests' '
+	flux bulksubmit --quiet --watch -n1 -o initc={} echo run test {./}  \
+	    ::: ${INITRC_TESTDIR}/tests/*.lua >&2
+'
 test_expect_success 'flux-shell: initrc: loads single .so successfully' '
 	name=conftest-success &&
 	cat >${name}.lua <<-EOT &&
         plugin.searchpath = "${INITRC_PLUGINPATH}"
 	plugin.load { file = "dummy.so", conf = { result = 0 } }
 	EOT
-	test_expect_code 0 ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0 > ${name}.log 2>&1 &&
+	flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep "dummy: OK result=0" ${name}.log
 '
+
 test_expect_success 'flux-shell: initrc: script fails on plugin.load failure' '
 	name=conftest-failure &&
 	cat >${name}.lua <<-EOT &&
         plugin.searchpath = "${INITRC_PLUGINPATH}"
 	plugin.load { file = "dummy.so", conf = { result = -1 } }
 	EOT
-	test_expect_code 1 ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0 > ${name}.log 2>&1 &&
+	test_must_fail_or_be_terminated \
+	    flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep "dummy: OK result=-1" ${name}.log
 '
@@ -148,8 +171,8 @@ test_expect_success 'flux-shell: initrc: plugin pattern nomatch not fatal' '
 	cat >${name}.lua <<-EOT &&
 	plugin.load { file = "nofile*.so" }
 	EOT
-	test_expect_code 0 ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0 > ${name}.log 2>&1 &&
+	flux run --label-io -o verbose -o initrc=${name}.lua \
+	    echo Hi > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep "0: Hi" ${name}.log
 '
@@ -158,11 +181,31 @@ test_expect_success 'flux-shell: initrc: plugin nonpattern nomatch fatal' '
 	cat >${name}.lua <<-EOT &&
 	plugin.load { file = "nofile.so" }
 	EOT
-	test_expect_code 1 ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0 > ${name}.log 2>&1 &&
+	test_must_fail_or_be_terminated \
+	    flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep "nofile.so: File not found" ${name}.log
 '
+test_expect_success 'flux-shell: initrc: plugin.load accepts string arg' '
+	name=string-arg-nomatch &&
+	cat >${name}.lua <<-EOT &&
+	plugin.load "nomatch-again.so"
+	EOT
+	test_must_fail_or_be_terminated \
+	    flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
+	test_debug "cat ${name}.log" &&
+	grep "nomatch-again.so: File not found" ${name}.log
+'
+test_expect_success 'flux-shell: initrc: plugin.load bad argument fatal' '
+	name=load-bad-arg &&
+	cat >${name}.lua <<-EOT &&
+	plugin.load (true)
+	EOT
+	test_must_fail_or_be_terminated \
+	    flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
+	test_debug "cat ${name}.log" &&
+	grep "plugin.load: invalid argument" ${name}.log
+'
 test_expect_success 'flux-shell: initrc: plugin.load passes conf to plugin' '
 	name=conftest &&
 	cat >${name}.lua <<-EOT &&
@@ -174,8 +217,7 @@ test_expect_success 'flux-shell: initrc: plugin.load passes conf to plugin' '
                                three = "bar" }
 	            }
 	EOT
-	${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0 > ${name}.log 2>&1 &&
+	flux run -o verbose -o initrc=${name}.lua true > ${name}.log 2>&1 &&
 	test_debug "cat ${name}.log" &&
 	grep "conftest: one=foo" ${name}.log &&
 	grep "conftest: two=two" ${name}.log &&
@@ -187,19 +229,31 @@ test_expect_success 'flux-shell: initrc: load invalid args plugins' '
         plugin.searchpath = "${INITRC_PLUGINPATH}"
 	plugin.load { file = "invalid-args.so" }
 	EOT
-	${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 \
-		--initrc=${name}.lua 0
+	flux run -o verbose -o initrc=${name}.lua true
 '
-test_expect_success HAVE_JQ 'flux-shell: initrc: load getopt plugin' '
+test_expect_success 'flux-shell: initrc: load getopt plugin' '
 	name=getopt &&
 	cat >${name}.lua <<-EOF &&
 	plugin.searchpath = "${INITRC_PLUGINPATH}"
 	plugin.load { file = "getopt.so" }
 	EOF
-	cat j1 | jq ".attributes.system.shell.options.test = 1" >j.${name} &&
-	${FLUX_SHELL} -v -s -r 0 -j j.${name} -R R1 --initrc=${name}.lua 0 \
-		> ${name}.log 2>&1 &&
-	test_debug "cat ${name}.log"
+	flux run -o verbose -o initrc=${name}.lua -o test=1 true
+'
+test_expect_success 'flux-shell: initrc: jobspec-info plugin works' '
+	name=jobspec-info &&
+	cat >${name}.lua <<-EOF &&
+	plugin.searchpath = "${INITRC_PLUGINPATH}"
+	plugin.load { file = "jobspec-info.so" }
+	EOF
+	cat >${name}.json <<-"EOF" &&
+	{ "ntasks":1, "nnodes":1, "nslots":1,
+	  "cores_per_slot":1,
+	  "slots_per_node":1
+	}
+	EOF
+	flux run -N1 -n1 -c1 -o verbose=2 \
+		-o initrc=${name}.lua \
+		-o ^jobspec_info=${name}.json true
 '
 test_expect_success 'flux-shell: initrc: shell log functions available' '
 	name=shell.log &&
@@ -209,11 +263,11 @@ test_expect_success 'flux-shell: initrc: shell log functions available' '
 
 	shell.log_error ("test: error")
 	EOF
-	${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 --initrc=${name}.lua 0 \
+	flux run -o verbose -o initrc=${name}.lua true \
 		> ${name}.log 2>&1 &&
-	grep "INFO: ${name}.lua:1: test: shell.log" ${name}.log &&
-	grep "DEBUG: ${name}.lua:2: test: debug" ${name}.log &&
-	grep "ERROR: ${name}.lua:4: test: error" ${name}.log
+	grep "test: shell.log" ${name}.log &&
+	grep "DEBUG: test: debug" ${name}.log &&
+	grep "ERROR: test: error" ${name}.log
 '
 test_expect_success 'flux-shell: initrc: shell.die function works' '
 	name=shell.die &&
@@ -221,10 +275,30 @@ test_expect_success 'flux-shell: initrc: shell.die function works' '
         -- shell.die test
 	shell.die ("test: shell.die")
 	EOF
-	test_expect_code 1 \
-	    ${FLUX_SHELL} -v -s -r 0 -j j1 -R R1 --initrc=${name}.lua 0 \
+	test_must_fail_or_be_terminated \
+		flux run -o verbose -o initrc=${name}.lua true \
 		> ${name}.log 2>&1 &&
-	grep "FATAL: ${name}.lua:2: test: shell.die" ${name}.log
+	grep "FATAL: test: shell.die" ${name}.log
 '
-
+test_expect_success MULTICORE 'flux-shell: initrc: shell.rankinfo reports broker_rank' '
+	name=shell.rankinfo.broker_rank &&
+	cat >${name}.lua <<-EOF &&
+	ri0 = shell.get_rankinfo(0)
+	ri1 = shell.get_rankinfo(1)
+	if ri0.broker_rank ~= 1 or ri1.broker_rank ~= 3 then
+	  shell.log ("rankinfo(0).broker_rank = "
+	             ..shell.get_rankinfo(0).broker_rank)
+	  shell.log ("rankinfo(1).broker_rank = "
+	             ..shell.get_rankinfo(1).broker_rank)
+	  shell.die ("rankinfo.broker_rank incorrect!")
+	end
+	if ri0.ntasks ~= 2 or ri1.ntasks ~= 1 then
+	  shell.die ("got ri[0].ntasks = "..ri0.ntasks
+	             .." ri[1].ntasks = "..ri1.ntasks)
+	end
+	EOF
+	flux run -N2 -n3 --requires=rank:1,3 \
+	    -o verbose -o initrc=${name}.lua true
+'
+flux job info $(flux job last) R
 test_done
diff --git a/t/t2604-job-shell-affinity.t b/t/t2604-job-shell-affinity.t
index f3b282ae9c96..5b9c643ca8d8 100755
--- a/t/t2604-job-shell-affinity.t
+++ b/t/t2604-job-shell-affinity.t
@@ -16,7 +16,7 @@ CPUS_ALLOWED_COUNT="$(pwd)/cpus-allowed-count.sh"
 
 cat >${CPUS_ALLOWED_COUNT} << EOF
 #!/bin/sh
-hwloc-bind --get | hwloc-calc --number-of core
+hwloc-bind --get | hwloc-calc --number-of core | tail -n 1
 EOF
 chmod +x ${CPUS_ALLOWED_COUNT}
 
@@ -30,17 +30,17 @@ test_expect_success 'flux-shell: affinity hwloc-calc works' '
     hwloc-bind --get | hwloc-calc --number-of pu
 '
 test_expect_success 'flux-shell: default affinity works (1 core)' '
-    flux mini run -n1 -c1 $CPUS_ALLOWED_COUNT > result.n1 &&
+    flux run -n1 -c1 $CPUS_ALLOWED_COUNT > result.n1 &&
     test_debug "cat result.n1" &&
     test "$(cat result.n1)" = "1"
 '
 test_expect_success MULTICORE 'flux-shell: default affinity works (2 cores)' '
-    flux mini run -n1 -c2 $CPUS_ALLOWED_COUNT > result.n1 &&
+    flux run -n1 -c2 $CPUS_ALLOWED_COUNT > result.n1 &&
     test_debug "cat result.n1" &&
     test "$(cat result.n1)" = "2"
 '
 test_expect_success MULTICORE 'flux-shell: per-task affinity works' '
-    flux mini run --label-io -ocpu-affinity=per-task -n2 -c1 \
+    flux run --label-io -ocpu-affinity=per-task -n2 -c1 \
 		hwloc-bind --get > per-task.out &&
     task0set=$(sed -n "s/^0: //p" per-task.out) &&
     task1set=$(sed -n "s/^1: //p" per-task.out) &&
@@ -48,22 +48,73 @@ test_expect_success MULTICORE 'flux-shell: per-task affinity works' '
     test "$task0set" != "$task1set"
 '
 test_expect_success 'flux-shell: per-task affinity sanity check' '
-    flux mini run --label-io -ocpu-affinity=per-task -n1 -c1 \
+    flux run --label-io -ocpu-affinity=per-task -n1 -c1 \
 		hwloc-bind --get
 '
+test_expect_success MULTICORE 'flux-shell: map affinity works' '
+    flux run --label-io -o cpu-affinity="map:1;0" -n 2 \
+	hwloc-bind --get > map1.out &&
+    task0set=$(sed -n "s/^0: //p" map1.out) &&
+    task1set=$(sed -n "s/^1: //p" map1.out) &&
+    test_debug "echo checking ${task0set}=0x2 ${task1set}=0x1" &&
+    test "$(hwloc-calc --taskset $task0set)" = "0x2" &&
+    test "$(hwloc-calc --taskset $task1set)" = "0x1"
+'
+test_expect_success MULTICORE 'flux-shell: map affinity reuses underspecified sets' '
+    flux run --label-io -o cpu-affinity=map:1 -n 2 \
+	hwloc-bind --get > map2.out &&
+    task0set=$(sed -n "s/^0: //p" map2.out) &&
+    task1set=$(sed -n "s/^1: //p" map2.out) &&
+    test_debug "echo checking ${task0set}=0x2 ${task1set}=0x2" &&
+    test "$(hwloc-calc --taskset $task0set)" = "0x2" &&
+    test "$(hwloc-calc --taskset $task1set)" = "0x2"
+'
+test_expect_success MULTICORE 'flux-shell: map affinity can use hex bitmasks' '
+    flux run --label-io -o cpu-affinity="map:0x1;0x2" -n 2 \
+	hwloc-bind --get > map3.out &&
+    task0set=$(sed -n "s/^0: //p" map3.out) &&
+    task1set=$(sed -n "s/^1: //p" map3.out) &&
+    test_debug "echo checking ${task0set}=0x1 ${task1set}=0x2" &&
+    test "$(hwloc-calc --taskset $task0set)" = "0x1" &&
+    test "$(hwloc-calc --taskset $task1set)" = "0x2"
+'
+test_expect_success 'flux-shell: map affinity can use a mix of inputs' '
+    id=$(flux submit --label-io -o cpu-affinity="map:0xf,0xf;0-3" -n 2 \
+	hwloc-bind --get) &&
+    flux job attach $id >map4.out 2>&1 &&
+    test_debug "cat map4.out"
+'
+test_expect_success 'flux-shell: invalid cpuset is detected' '
+    test_must_fail flux run -o cpu-affinity="map:0x0;1" -n 2 \
+        hwloc-bind --get
+'
 test_expect_success 'flux-shell: affinity can be disabled' '
     hwloc-bind --get > affinity-off.expected &&
-    flux mini run -ocpu-affinity=off -n1 hwloc-bind --get >affinity-off.out &&
+    flux run -ocpu-affinity=off -n1 hwloc-bind --get >affinity-off.out &&
     test_cmp affinity-off.expected affinity-off.out
 '
 test_expect_success 'flux-shell: invalid option is ignored' '
-    flux mini run -ocpu-affinity=1 -n1 hwloc-bind --get >invalid.out 2>&1 &&
+    flux run -ocpu-affinity=1 -n1 hwloc-bind --get >invalid.out 2>&1 &&
     test_debug "cat invalid.out" &&
     grep "invalid option" invalid.out
 '
-#  GPU affinity tests use standalone shell since simple-sched doesnt
+test_expect_success 'flux-shell: CUDA_VISIBLE_DEVICES=-1 set by default' '
+    flux run printenv CUDA_VISIBLE_DEVICES >default-gpubind.out 2>&1 &&
+    test_debug "cat default-gpubind.out" &&
+    grep "^-1" default-gpubind.out
+'
+test_expect_success 'flux-shell: CUDA_VISIBLE_DEVICES=-1 works with existing value' '
+    CUDA_VISIBLE_DEVICES=0,1 \
+       flux run printenv CUDA_VISIBLE_DEVICES >override-gpubind.out 2>&1 &&
+    test_debug "cat override-gpubind.out" &&
+    grep "^-1" override-gpubind.out
+'
+#  GPU affinity tests use alloc-bypass shell since simple-sched doesnt
 #   schedule GPUs.
 #
+test_expect_success 'flux-shell: load alloc-bypass jobtap plugin' '
+	flux jobtap load alloc-bypass.so
+'
 test_expect_success 'flux-shell: create multi-gpu R' '
 	cat >R.gpu <<-EOF
 	        {"version": 1, "execution":{ "R_lite":[
@@ -73,58 +124,85 @@ test_expect_success 'flux-shell: create multi-gpu R' '
 '
 test_expect_success 'flux-shell: gpu-affinity works by default' '
 	name=gpu-basic &&
-	flux mini run -N1 -n2 --dry-run \
-		printenv CUDA_VISIBLE_DEVICES > j.${name} &&
+	flux run -N1 -n2 \
+		--label-io \
+		--setattr=alloc-bypass.R="$(cat R.gpu)" \
+		printenv CUDA_VISIBLE_DEVICES >${name}.output 2>${name}.err &&
 	cat >${name}.expected <<-EOF  &&
-	0: 0-3
-	1: 0-3
+	0: 0,1,2,3
+	1: 0,1,2,3
 	EOF
-	${FLUX_SHELL} -s -v -r 0 -j j.${name} -R R.gpu 0 &&
-	${FLUX_SHELL} -s -v -r 0 -j j.${name} -R R.gpu 0 | sort -k1,1n \
-		> ${name}.out 2>${name}.err &&
+	test_debug "cat ${name}.output ${name}.err" &&
+	sort -k1,1n ${name}.output > ${name}.out &&
 	test_cmp ${name}.expected ${name}.out
 '
 test_expect_success 'flux-shell: gpu-affinity=on' '
 	name=gpu-on &&
-	flux mini run -N1 -n2 --dry-run -o gpu-affinity=on \
-		printenv CUDA_VISIBLE_DEVICES > j.${name} &&
+	flux run -N1 -n2 \
+		--label-io \
+		--setattr=alloc-bypass.R="$(cat R.gpu)" \
+		-o gpu-affinity=on \
+		printenv CUDA_VISIBLE_DEVICES >${name}.output 2>${name}.err &&
 	cat >${name}.expected <<-EOF  &&
-	0: 0-3
-	1: 0-3
+	0: 0,1,2,3
+	1: 0,1,2,3
 	EOF
-	${FLUX_SHELL} -s -v -r 0 -j j.${name} -R R.gpu 0 | sort -k1,1n \
-		> ${name}.out 2>${name}.err &&
+	test_debug "cat ${name}.output ${name}.err" &&
+	sort -k1,1n ${name}.output > ${name}.out &&
 	test_cmp ${name}.expected ${name}.out
 '
 test_expect_success 'flux-shell: gpu-affinity=per-task' '
 	name=gpu-per-task &&
-	flux mini run -N1 -n2 --dry-run -o gpu-affinity=per-task \
-		printenv CUDA_VISIBLE_DEVICES > j.${name} &&
+	flux run -N1 -n2 \
+		--label-io \
+		--setattr=alloc-bypass.R="$(cat R.gpu)" \
+		-o gpu-affinity=per-task \
+		printenv CUDA_VISIBLE_DEVICES >${name}.output 2>${name}.err &&
+	cat >${name}.expected <<-EOF  &&
+	0: 0,1
+	1: 2,3
+	EOF
+	test_debug "cat ${name}.output ${name}.err" &&
+	sort -k1,1n ${name}.output > ${name}.out &&
+	test_cmp ${name}.expected ${name}.out
+'
+test_expect_success 'flux-shell: gpu-affinity=map: works' '
+	name=gpu-map &&
+	flux run -N1 -n2 \
+		--label-io \
+		--setattr=alloc-bypass.R="$(cat R.gpu)" \
+		-o gpu-affinity="map:7;4-6" \
+		printenv CUDA_VISIBLE_DEVICES >${name}.output 2>${name}.err &&
 	cat >${name}.expected <<-EOF  &&
-	0: 0-1
-	1: 2-3
+	0: 7
+	1: 4,5,6
 	EOF
-	${FLUX_SHELL} -s -v -r 0 -j j.${name} -R R.gpu 0 | sort -k1,1n \
-		> ${name}.out 2>${name}.err &&
+	test_debug "cat ${name}.output ${name}.err" &&
+	sort -k1,1n ${name}.output > ${name}.out &&
 	test_cmp ${name}.expected ${name}.out
+
 '
 test_expect_success 'flux-shell: gpu-affinity=off' '
 	name=gpu-off &&
-	flux mini run -N1 -n2 --dry-run -o gpu-affinity=off \
-		printenv CUDA_VISIBLE_DEVICES > j.${name} &&
+	test_expect_code 1 flux run -N1 -n2 \
+		--label-io \
+		--setattr=alloc-bypass.R="$(cat R.gpu)" \
+		-o gpu-affinity=off \
+		--env=-CUDA_VISIBLE_DEVICES \
+		printenv CUDA_VISIBLE_DEVICES >${name}.output 2>${name}.err &&
 	cat >${name}.expected <<-EOF  &&
 	EOF
-	test_expect_code 1  ${FLUX_SHELL} -s -v -r 0 -j j.${name} -R R.gpu 0 \
-	  | sort -k1,1n \
-	  > ${name}.out 2>${name}.err &&
+	test_debug "cat ${name}.output ${name}.err" &&
+	sort -k1,1n ${name}.output > ${name}.out &&
 	test_cmp ${name}.expected ${name}.out
 '
 test_expect_success 'flux-shell: gpu-affinity bad arg is ignored' '
 	name=gpu-bad-arg &&
-	flux mini run -N1 -n2 --dry-run -o gpu-affinity="[1]" \
-		printenv CUDA_VISIBLE_DEVICES > j.${name} &&
-	${FLUX_SHELL} -s -v -r 0 -j j.${name} -R R.gpu 0 \
-	  >${name}.out 2>${name}.err &&
+	flux run -N1 -n2 \
+		--label-io \
+		--setattr=alloc-bypass.R="$(cat R.gpu)" \
+		-o gpu-affinity="[1]" \
+		printenv CUDA_VISIBLE_DEVICES >${name}.out 2>${name}.err &&
 	test_debug "cat ${name}.out" &&
 	test_debug "cat ${name}.err" >&2 &&
 	grep "Failed to get gpu-affinity shell option" ${name}.err
diff --git a/t/t2605-job-shell-output-redirection-standalone.t b/t/t2605-job-shell-output-redirection-standalone.t
deleted file mode 100755
index 7a2282e52693..000000000000
--- a/t/t2605-job-shell-output-redirection-standalone.t
+++ /dev/null
@@ -1,258 +0,0 @@
-#!/bin/sh
-#
-test_description='Test flux-shell in --standalone mode'
-
-. `dirname $0`/sharness.sh
-
-jq=$(which jq 2>/dev/null)
-test -n "$jq" && test_set_prereq HAVE_JQ
-
-#  Run flux-shell under flux command to get correct paths
-FLUX_SHELL="flux ${FLUX_BUILD_DIR}/src/shell/flux-shell"
-
-unset FLUX_URI
-
-TEST_SUBPROCESS_DIR=${FLUX_BUILD_DIR}/src/common/libsubprocess
-
-test_expect_success 'flux-shell: generate 1-task echo jobspecs and matching R' '
-	flux jobspec srun -N1 -n1 ${TEST_SUBPROCESS_DIR}/test_echo -P -O foo > j1echostdout &&
-	flux jobspec srun -N1 -n1 ${TEST_SUBPROCESS_DIR}/test_echo -P -E bar > j1echostderr &&
-	flux jobspec srun -N1 -n1 ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz > j1echoboth &&
-	cat >R1 <<-EOT
-	{"version": 1, "execution":{ "R_lite":[
-		{ "children": { "core": "0" }, "rank": "0" }
-        ]}}
-	EOT
-'
-
-test_expect_success 'flux-shell: generate 2-task echo jobspecs and matching R' '
-	flux jobspec srun -N1 -n2 ${TEST_SUBPROCESS_DIR}/test_echo -P -O foo > j2echostdout &&
-	flux jobspec srun -N1 -n2 ${TEST_SUBPROCESS_DIR}/test_echo -P -E bar > j2echostderr &&
-	flux jobspec srun -N1 -n2 ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz > j2echoboth &&
-	cat >R2 <<-EOT
-	{"version": 1, "execution":{ "R_lite":[
-		{ "children": { "core": "0-1" }, "rank": "0" }
-        ]}}
-	EOT
-'
-
-#
-# 1 task output file tests
-#
-
-test_expect_success HAVE_JQ 'flux-shell: run 1-task echo job (stdout file)' '
-        cat j1echostdout \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out0\"" \
-            > j1echostdout-0 &&
-        ${FLUX_SHELL} -v -s -r 0 -j j1echostdout-0 -R R1 0 &&
-	grep stdout:foo out0
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 1-task echo job (stderr file)' '
-        cat j1echostderr \
-            |  $jq ".attributes.system.shell.options.output.stderr.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.path = \"err1\"" \
-            > j1echostderr-1 &&
-	${FLUX_SHELL} -v -s -r 0 -j j1echostderr-1 -R R1 1 &&
-	grep stderr:bar err1
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 1-task echo job (stderr to stdout file)' '
-        cat j1echostderr \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out2\"" \
-            > j1echostderr-2 &&
-	${FLUX_SHELL} -v -s -r 0 -j j1echostderr-2 -R R1 2 &&
-	grep stderr:bar out2
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 1-task echo job (stdout file/stderr file)' '
-        cat j1echoboth \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out3\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.path = \"err3\"" \
-            > j1echoboth-3 &&
-	${FLUX_SHELL} -v -s -r 0 -j j1echoboth-3 -R R1 3 &&
-	grep stdout:baz out3 &&
-	grep stderr:baz err3
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 1-task echo job (stdout & stderr to stdout file)' '
-        cat j1echoboth \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out4\"" \
-            > j1echoboth-4 &&
-	${FLUX_SHELL} -v -s -r 0 -j j1echoboth-4 -R R1 4 &&
-	grep stdout:baz out4 &&
-	grep stderr:baz out4
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 1-task echo job (stdout file/stderr term)' '
-        cat j1echoboth \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out5\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.type = \"term\"" \
-            > j1echoboth-5 &&
-	${FLUX_SHELL} -v -s -r 0 -j j1echoboth-5 -R R1 2> err5 5 &&
-	grep stdout:baz out5 &&
-	grep stderr:baz err5
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 1-task echo job (stdout term/stderr file)' '
-        cat j1echoboth \
-            |  $jq ".attributes.system.shell.options.output.stderr.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.path = \"err6\"" \
-            > j1echoboth-6 &&
-	${FLUX_SHELL} -v -s -r 0 -j j1echoboth-6 -R R1 > out6 6 &&
-	grep stdout:baz out6 &&
-	grep stderr:baz err6
-'
-
-#
-# 2 task output file tests
-#
-
-test_expect_success HAVE_JQ 'flux-shell: run 2-task echo job (stdout file)' '
-        cat j2echostdout \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out7\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.label = true" \
-            > j2echostdout-7 &&
-	${FLUX_SHELL} -v -s -r 0 -j j2echostdout-7 -R R2 7 &&
-	grep "0: stdout:foo" out7 &&
-	grep "1: stdout:foo" out7
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 2-task echo job (stderr file)' '
-        cat j2echostderr \
-            |  $jq ".attributes.system.shell.options.output.stderr.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.path = \"err8\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.label = true" \
-            > j2echostderr-8 &&
-	${FLUX_SHELL} -v -s -r 0 -j j2echostderr-8 -R R2 8 &&
-	grep "0: stderr:bar" err8 &&
-	grep "1: stderr:bar" err8
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 2-task echo job (stderr to stdout file)' '
-        cat j2echostderr \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out9\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.label = true" \
-            > j2echostderr-9 &&
-	${FLUX_SHELL} -v -s -r 0 -j j2echostderr-9 -R R2 9 &&
-	grep "0: stderr:bar" out9 &&
-	grep "1: stderr:bar" out9
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 2-task echo job (stdout file/stderr file)' '
-        cat j2echoboth \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out10\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.label = true" \
-            |  $jq ".attributes.system.shell.options.output.stderr.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.path = \"err10\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.label = true" \
-            > j2echoboth-10 &&
-	${FLUX_SHELL} -v -s -r 0 -j j2echoboth-10 -R R2 10 &&
-	grep "0: stdout:baz" out10 &&
-	grep "1: stdout:baz" out10 &&
-	grep "0: stderr:baz" err10 &&
-	grep "1: stderr:baz" err10
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 2-task echo job (stdout & stderr to stdout file)' '
-        cat j2echoboth \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out11\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.label = true" \
-            > j2echoboth-11 &&
-	${FLUX_SHELL} -v -s -r 0 -j j2echoboth-11 -R R2 11 &&
-	grep "0: stdout:baz" out11 &&
-	grep "1: stdout:baz" out11 &&
-	grep "0: stderr:baz" out11 &&
-	grep "1: stderr:baz" out11
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 2-task echo job (stdout file/stderr term)' '
-        cat j2echoboth \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out12\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.label = true" \
-            |  $jq ".attributes.system.shell.options.output.stderr.type = \"term\"" \
-            > j2echoboth-12 &&
-	${FLUX_SHELL} -v -s -r 0 -j j2echoboth-12 -R R2 2> err12 12 &&
-	grep "0: stdout:baz" out12 &&
-	grep "1: stdout:baz" out12 &&
-	grep "0: stderr:baz" err12 &&
-	grep "1: stderr:baz" err12
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 2-task echo job (stdout term/stderr file)' '
-        cat j2echoboth \
-            |  $jq ".attributes.system.shell.options.output.stderr.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.path = \"err13\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.label = true" \
-            > j2echoboth-13 &&
-	${FLUX_SHELL} -v -s -r 0 -j j2echoboth-13 -R R2 > out13 13 &&
-	grep "0: stdout:baz" out13 &&
-	grep "1: stdout:baz" out13 &&
-	grep "0: stderr:baz" err13 &&
-	grep "1: stderr:baz" err13
-'
-
-#
-# output file mustache tests
-#
-
-test_expect_success HAVE_JQ 'flux-shell: run 1-task echo job (mustache id stdout file/stderr file)' '
-        cat j1echoboth \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out{{id}}\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stderr.path = \"err{{id}}\"" \
-            > j1echoboth-14 &&
-	${FLUX_SHELL} -v -s -r 0 -j j1echoboth-14 -R R1 14 &&
-	grep stdout:baz out14 &&
-	grep stderr:baz err14
-'
-
-test_expect_success HAVE_JQ 'flux-shell: run 1-task echo job (mustache id stdout & stderr to stdout file)' '
-        cat j1echoboth \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"out{{id}}\"" \
-            > j1echoboth-15 &&
-	${FLUX_SHELL} -v -s -r 0 -j j1echoboth-15 -R R1 15 &&
-	grep stdout:baz out15 &&
-	grep stderr:baz out15
-'
-
-#
-# output corner case tests
-#
-
-test_expect_success HAVE_JQ 'flux-shell: error on bad output type' '
-        cat j1echostdout \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"foobar\"" \
-            > j1echostdout-16 &&
-        ! ${FLUX_SHELL} -v -s -r 0 -j j1echostdout-16 -R R1 16
-'
-
-test_expect_success HAVE_JQ 'flux-shell: error on no path with file output' '
-        cat j1echostdout \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            > j1echostdout-17 &&
-        ! ${FLUX_SHELL} -v -s -r 0 -j j1echostdout-17 -R R1 17
-'
-
-test_expect_success HAVE_JQ 'flux-shell: error invalid path to file output' '
-        cat j1echostdout \
-            |  $jq ".attributes.system.shell.options.output.stdout.type = \"file\"" \
-            |  $jq ".attributes.system.shell.options.output.stdout.path = \"/foo/bar/baz\"" \
-            > j1echostdout-18 &&
-        ! ${FLUX_SHELL} -v -s -r 0 -j j1echostdout-18 -R R1 18
-'
-
-test_done
diff --git a/t/t2606-job-shell-output-redirection.t b/t/t2606-job-shell-output-redirection.t
index 3ddfc76aff48..db8c19305275 100755
--- a/t/t2606-job-shell-output-redirection.t
+++ b/t/t2606-job-shell-output-redirection.t
@@ -15,28 +15,28 @@ TEST_SUBPROCESS_DIR=${FLUX_BUILD_DIR}/src/common/libsubprocess
 #
 
 test_expect_success 'job-shell: run 1-task echo job (stdout file)' '
-        flux mini run -n1 \
+        flux run -n1 \
              --output=out0 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O foo &&
         grep stdout:foo out0
 '
 
 test_expect_success 'job-shell: run 1-task echo job (stderr file)' '
-        flux mini run -n1 \
+        flux run -n1 \
              --error=err1 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -E bar &&
         grep stderr:bar err1
 '
 
 test_expect_success 'flux-shell: run 1-task echo job (stderr to stdout file)' '
-        flux mini run -n1 \
+        flux run -n1 \
              --output=out2 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -E bar &&
         grep stderr:bar out2
 '
 
 test_expect_success 'job-shell: run 1-task echo job (stdout file/stderr file)' '
-        flux mini run -n1 \
+        flux run -n1 \
              --output=out3 --error=err3 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz &&
         grep stdout:baz out3 &&
@@ -44,7 +44,7 @@ test_expect_success 'job-shell: run 1-task echo job (stdout file/stderr file)' '
 '
 
 test_expect_success 'flux-shell: run 1-task echo job (stdout & stderr to stdout file)' '
-        flux mini run -n1 \
+        flux run -n1 \
              --output=out4 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz &&
         grep stdout:baz out4 &&
@@ -52,7 +52,7 @@ test_expect_success 'flux-shell: run 1-task echo job (stdout & stderr to stdout
 '
 
 test_expect_success 'job-shell: run 1-task echo job (stdout file/stderr kvs)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --output=out5 --setopt "output.stderr.type=\"kvs\"" \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -62,7 +62,7 @@ test_expect_success 'job-shell: run 1-task echo job (stdout file/stderr kvs)' '
 '
 
 test_expect_success 'job-shell: run 1-task echo job (stdout kvs/stderr file)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --error=err6 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -76,7 +76,7 @@ test_expect_success 'job-shell: run 1-task echo job (stdout kvs/stderr file)' '
 #
 
 test_expect_success 'job-shell: run 2-task echo job (stdout file)' '
-        flux mini run -n2 \
+        flux run -n2 \
              --output=out7 --label-io \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O foo &&
         grep "0: stdout:foo" out7 &&
@@ -84,7 +84,7 @@ test_expect_success 'job-shell: run 2-task echo job (stdout file)' '
 '
 
 test_expect_success 'job-shell: run 2-task echo job (stderr file)' '
-        flux mini run -n2 \
+        flux run -n2 \
              --error=err8 --label-io \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -E bar &&
         grep "0: stderr:bar" err8 &&
@@ -92,7 +92,7 @@ test_expect_success 'job-shell: run 2-task echo job (stderr file)' '
 '
 
 test_expect_success 'job-shell: run 2-task echo job (stderr to stdout file)' '
-        flux mini run -n2 \
+        flux run -n2 \
              --output=out9 --label-io \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -E bar &&
         grep "0: stderr:bar" out9 &&
@@ -100,7 +100,7 @@ test_expect_success 'job-shell: run 2-task echo job (stderr to stdout file)' '
 '
 
 test_expect_success 'job-shell: run 2-task echo job (stdout file/stderr file)' '
-        flux mini run -n2 \
+        flux run -n2 \
              --output=out10 --error=err10 --label-io \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz &&
         grep "0: stdout:baz" out10 &&
@@ -110,7 +110,7 @@ test_expect_success 'job-shell: run 2-task echo job (stdout file/stderr file)' '
 '
 
 test_expect_success 'job-shell: run 2-task echo job (stdout & stderr to stdout file)' '
-        flux mini run -n2 \
+        flux run -n2 \
              --output=out11 --label-io \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz &&
         grep "0: stdout:baz" out11 &&
@@ -120,7 +120,7 @@ test_expect_success 'job-shell: run 2-task echo job (stdout & stderr to stdout f
 '
 
 test_expect_success 'job-shell: run 2-task echo job (stdout file/stderr kvs)' '
-        id=$(flux mini submit -n2 \
+        id=$(flux submit -n2 \
              --output=out12 --label-io --setopt "output.stderr.type=\"kvs\"" \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -132,7 +132,7 @@ test_expect_success 'job-shell: run 2-task echo job (stdout file/stderr kvs)' '
 '
 
 test_expect_success 'job-shell: run 2-task echo job (stdout kvs/stderr file)' '
-        id=$(flux mini submit -n2 \
+        id=$(flux submit -n2 \
              --error=err13 --label-io \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -148,7 +148,7 @@ test_expect_success 'job-shell: run 2-task echo job (stdout kvs/stderr file)' '
 #
 
 test_expect_success 'job-shell: run 1-task echo job (mustache id stdout file/stderr file)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --output="out{{id}}" --error="err{{id}}" \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -157,20 +157,43 @@ test_expect_success 'job-shell: run 1-task echo job (mustache id stdout file/std
 '
 
 test_expect_success 'flux-shell: run 1-task echo job (mustache id stdout & stderr to stdout file)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --output="out{{id}}" \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
         grep stdout:baz out${id} &&
         grep stderr:baz out${id}
 '
+test_expect_success 'flux-shell: run 1-task echo job (mustache job name)' '
+	id=$(flux submit --wait -n1 --job-name=thisname \
+	     --output="out.{{name}}" \
+             ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
+	grep stdout:baz out.thisname &&
+	grep stderr:baz out.thisname
+'
+test_expect_success 'flux-shell: run 1-task echo job (mustache job name)' '
+	id=$(flux submit --wait -n1 \
+	     --output="out.{{name}}" \
+             ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
+	grep stdout:baz out.test_echo &&
+	grep stderr:baz out.test_echo
+'
+
+test_expect_success 'flux-shell: bad mustache template still writes output' '
+        id=$(flux submit -n1 \
+             --output="out{{invalid" \
+             ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
+        flux job wait-event $id clean &&
+        grep stdout:baz out{{invalid &&
+        grep stderr:baz out{{invalid
+'
 
 #
 # output file outputs correct information to guest.output
 #
 
 test_expect_success 'job-shell: redirect events appear in guest.output (1-task)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --output=out16 --error=err16 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -180,7 +203,7 @@ test_expect_success 'job-shell: redirect events appear in guest.output (1-task)'
 '
 
 test_expect_success 'job-shell: redirect events appear in guest.output (1-task stderr to stdout)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --output=out17 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -190,7 +213,7 @@ test_expect_success 'job-shell: redirect events appear in guest.output (1-task s
 '
 
 test_expect_success 'job-shell: redirect events appear in guest.output (2-task)' '
-        id=$(flux mini submit -n2 \
+        id=$(flux submit -n2 \
              --output=out18 --error=err18 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -200,7 +223,7 @@ test_expect_success 'job-shell: redirect events appear in guest.output (2-task)'
 '
 
 test_expect_success 'job-shell: redirect events appear in guest.output (2-task stderr to stdout)' '
-        id=$(flux mini submit -n2 \
+        id=$(flux submit -n2 \
              --output=out19 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -210,7 +233,7 @@ test_expect_success 'job-shell: redirect events appear in guest.output (2-task s
 '
 
 test_expect_success 'job-shell: attach shows redirected file (1-task)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --output=out20 --error=err20 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -220,7 +243,7 @@ test_expect_success 'job-shell: attach shows redirected file (1-task)' '
 '
 
 test_expect_success 'job-shell: attach shows redirected file (1-task stderr to stdout)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --output=out21 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -230,7 +253,7 @@ test_expect_success 'job-shell: attach shows redirected file (1-task stderr to s
 '
 
 test_expect_success 'job-shell: attach shows redirected file (2-task)' '
-        id=$(flux mini submit -n2 \
+        id=$(flux submit -n2 \
              --output=out22 --error=err22 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -240,7 +263,7 @@ test_expect_success 'job-shell: attach shows redirected file (2-task)' '
 '
 
 test_expect_success 'job-shell: attach shows redirected file (2-task stderr to stdout)' '
-        id=$(flux mini submit -n2 \
+        id=$(flux submit -n2 \
              --output=out23 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -250,7 +273,7 @@ test_expect_success 'job-shell: attach shows redirected file (2-task stderr to s
 '
 
 test_expect_success 'job-shell: attach --quiet suppresses redirected file (1-task)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --output=out24 --error=err24 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -270,7 +293,7 @@ test_expect_success 'job-shell: attach --quiet suppresses redirected file (1-tas
 #
 
 test_expect_success 'job-shell: job attach exits cleanly if no kvs output (1-task)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --output=out25 --error=err25 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -283,7 +306,7 @@ test_expect_success 'job-shell: job attach exits cleanly if no kvs output (1-tas
 '
 
 test_expect_success 'job-shell: job attach exits cleanly if no kvs output (2-task)' '
-        id=$(flux mini submit -n2 \
+        id=$(flux submit -n2 \
              --output=out26 --error=err26 \
              ${TEST_SUBPROCESS_DIR}/test_echo -P -O -E baz) &&
         flux job wait-event $id clean &&
@@ -292,28 +315,92 @@ test_expect_success 'job-shell: job attach exits cleanly if no kvs output (2-tas
         grep stderr:baz err26 &&
         ! test -s out26.attach &&
         sed -i -e "/stdin EOF could not be sent/d" err26.attach &&
+        sed -i -e "/affinity/d" err26.attach &&
         ! test -s err26.attach
 '
 
 test_expect_success NO_CHAIN_LINT 'job-shell: job attach waits if no kvs output (1-task, live job)' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --output=out27 --error=err27 \
              sleep 60)
         flux job wait-event $id start &&
         flux job attach -E -X ${id} 2> attach27.err &
         pid=$! &&
-        flux job cancel $id &&
+        flux cancel $id &&
         ! wait $pid
 '
 
 test_expect_success NO_CHAIN_LINT 'job-shell: job attach waits if no kvs output (2-task, live job)' '
-        id=$(flux mini submit -n2 \
+        id=$(flux submit -n2 \
              --output=out28 --error=err28 \
              sleep 60)
         flux job wait-event $id start &&
         flux job attach -E -X ${id} 2> attach28.err &
         pid=$! &&
-        flux job cancel $id &&
+        flux cancel $id &&
         ! wait $pid
 '
+test_expect_success 'job-shell: shell errors are captured in output file' '
+	test_expect_code 127 flux run --output=test.out nosuchcommand &&
+	grep "nosuchcommand: No such file or directory" test.out
+'
+test_expect_success 'job-shell: shell errors are captured in error file' '
+	test_expect_code 127  flux run --error=test.err nosuchcommand &&
+	grep "nosuchcommand: No such file or directory" test.err
+'
+test_expect_success 'job-shell: kvs output truncation works' '
+	flux run -o output.limit=5 echo 0123456789 2>trunc.err &&
+	test_debug "cat trunc.err" &&
+	grep "stdout.*truncated" trunc.err
+'
+test_expect_success 'job-shell: job shell only logs once about truncation' '
+	flux run -N4 -o output.limit=20 echo 0123456789 2>trunc4.err &&
+	test_debug "cat trunc4.err"  &&
+	test $(grep -c "bytes truncated" trunc4.err) -eq 1
+'
+test_expect_success 'job-shell: stderr truncation works' '
+	flux run -o output.limit=5 \
+		sh -c "echo 0123456789 >&2" >trunc2.error 2>&1 &&
+	grep "stderr.*truncated" trunc2.error
+'
+test_expect_success LONGTEST 'job-shell: no truncation at 10MB for single-user job' '
+	dd if=/dev/urandom bs=10240 count=800 | base64 --wrap 79 >10M+ &&
+	flux run cat 10M+ >10M+.output &&
+	test_cmp 10M+ 10M+.output
+'
+test_expect_success 'job-shell: invalid output.limit string is rejected (text)' '
+	test_must_fail flux run -o output.limit=foo hostname
+'
+test_expect_success 'job-shell: invalid output.limit string is rejected (zero)' '
+	test_must_fail flux run -o output.limit=0 hostname
+'
+test_expect_success 'job-shell: invalid output.limit string is rejected (big num)' '
+	test_must_fail flux run -o output.limit=4000000000 hostname
+'
+test_expect_success 'job-shell: invalid output.limit string is rejected (big num suffix)' '
+	test_must_fail flux run -o output.limit=4G hostname
+'
+test_expect_success 'job-shell: output.mode=append works' '
+	flux bulksubmit --watch --output=append.out \
+		-o output.mode=append echo {} \
+		::: one two three &&
+	test_debug "cat append.out" &&
+	test $(wc -l < append.out) -eq 3 &&
+	grep one append.out &&
+	grep two append.out &&
+	grep three append.out
+'
+test_expect_success 'job-shell: output.mode=truncate works' '
+	cat <<-EOF >trunc.out &&
+	test text
+	EOF
+	flux run --output=trunc.out hostname &&
+	test $(wc -l <trunc.out) -eq 1 &&
+	test_must_fail grep "test text" trunc.out
+'
+test_expect_success 'job-shell: invalid output.mode emits warning' '
+	flux run --output=inval.out -o output.mode=foo hostname 2>inval.err &&
+	test_debug "cat inval.out" &&
+	grep "ignoring invalid output.mode=foo" inval.err
+'
 test_done
diff --git a/t/t2607-job-shell-input.t b/t/t2607-job-shell-input.t
index 7b01ba3a5f6e..a74b4aeb4658 100755
--- a/t/t2607-job-shell-input.t
+++ b/t/t2607-job-shell-input.t
@@ -9,7 +9,7 @@ test_under_flux 4 job
 flux setattr log-stderr-level 1
 
 TEST_SUBPROCESS_DIR=${FLUX_BUILD_DIR}/src/common/libsubprocess
-LPTEST=${SHARNESS_TEST_DIRECTORY}/shell/lptest
+LPTEST="flux lptest"
 
 test_expect_success 'flux-shell: generate input for stdin input tests' '
        echo "foo" > input_stdin_file &&
@@ -22,20 +22,20 @@ test_expect_success 'flux-shell: generate input for stdin input tests' '
 #
 
 test_expect_success 'flux-shell: run 1-task no pipe in stdin' '
-        id=$(flux mini submit -n1 echo foo) &&
+        id=$(flux submit -n1 echo foo) &&
         flux job attach $id > pipe0.out &&
         grep foo pipe0.out
 '
 
 test_expect_success 'flux-shell: run 1-task input file as stdin job' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              ${TEST_SUBPROCESS_DIR}/test_echo -O -n) &&
         flux job attach $id < input_stdin_file > pipe1.out &&
         test_cmp input_stdin_file pipe1.out
 '
 
 test_expect_success 'flux-shell: run 2-task input file as stdin job' '
-        id=$(flux mini submit -n2 \
+        id=$(flux submit -n2 \
              ${TEST_SUBPROCESS_DIR}/test_echo -O -n) &&
         flux job attach -l $id < input_stdin_file > pipe2.out &&
         grep "0: foo" pipe2.out &&
@@ -45,12 +45,27 @@ test_expect_success 'flux-shell: run 2-task input file as stdin job' '
 '
 
 test_expect_success LONGTEST 'flux-shell: 10K line lptest piped input works' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              ${TEST_SUBPROCESS_DIR}/test_echo -O -n) &&
         flux job attach $id < lptestXXL_input > pipe3.out &&
 	test_cmp lptestXXL_input pipe3.out
 '
 
+# Note: This test is racy. It tries to ensure the shell has had a chance
+# to read the stdin data and write to the target task, but the cancel could
+# come before that since there is no way to synchronize. Thus, the test may
+# give false positive results (ok) but never a false negative (no ok).
+#
+test_expect_success NO_CHAIN_LINT 'flux-shell: ignores SIGPIPE if task closes stdin' '
+	id=$(flux submit sh -c "exec <&-; sleep 60") &&
+	flux job wait-event $id start &&
+	flux job wait-event -vp exec $id shell.start &&
+	(echo foo | flux job attach $id &) &&
+	flux job wait-event -vp input $id data &&
+	flux cancel $id &&
+	test_expect_code 143 flux job status $id
+'
+
 #
 # sharness will redirect /dev/null to stdin by default, so we create a named pipe
 # and pipe that in for tests in which we need "no stdin".
@@ -58,11 +73,11 @@ test_expect_success LONGTEST 'flux-shell: 10K line lptest piped input works' '
 
 test_expect_success NO_CHAIN_LINT 'flux-shell: attach twice, one with data' '
         mkfifo stdin4.pipe
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              ${TEST_SUBPROCESS_DIR}/test_echo -O -n)
-        flux job attach $id < stdin4.pipe > pipe4A.out 2> pipe4A.err &
+        flux job attach $id < stdin4.pipe > pipe4A.out &
         pid1=$!
-        flux job attach $id < input_stdin_file > pipe4B.out 2> pipe4B.err &
+        flux job attach $id < input_stdin_file > pipe4B.out &
         pid2=$!
         exec 9> stdin4.pipe &&
         wait $pid1 &&
@@ -72,40 +87,25 @@ test_expect_success NO_CHAIN_LINT 'flux-shell: attach twice, one with data' '
         test_cmp input_stdin_file pipe4B.out
 '
 
-test_expect_success NO_CHAIN_LINT 'flux-shell: multiple jobs, each want stdin' '
-        id1=$(flux mini submit -n1 \
-              ${TEST_SUBPROCESS_DIR}/test_echo -O -n)
-        id2=$(flux mini submit -n1 \
-              ${TEST_SUBPROCESS_DIR}/test_echo -O -n)
-        id3=$(flux mini submit -n1 \
-              ${TEST_SUBPROCESS_DIR}/test_echo -O -n)
-        id4=$(flux mini submit -n1 \
-              ${TEST_SUBPROCESS_DIR}/test_echo -O -n)
-        flux job attach $id1 < input_stdin_file > pipe5_1.out 2> pipe5_1.err &
-        pid1=$!
-        flux job attach $id2 < input_stdin_file > pipe5_2.out 2> pipe5_2.err &
-        pid2=$!
-        flux job attach $id3 < input_stdin_file > pipe5_3.out 2> pipe5_3.err &
-        pid3=$!
-        flux job attach $id4 < input_stdin_file > pipe5_4.out 2> pipe5_4.err &
-        pid4=$!
-        wait $pid1 &&
-        wait $pid2 &&
-        wait $pid3 &&
-        wait $pid4 &&
-        test_cmp input_stdin_file pipe5_1.out &&
-        test_cmp input_stdin_file pipe5_2.out &&
-        test_cmp input_stdin_file pipe5_3.out &&
-        test_cmp input_stdin_file pipe5_4.out
+test_expect_success 'flux-shell: multiple jobs, each want stdin' '
+	flux submit --cc=1-4 -n1 -t 30s \
+	    ${TEST_SUBPROCESS_DIR}/test_echo -O -n >pipe5.jobids &&
+	test_debug "cat pipe5.jobids" &&
+	i=1 &&
+	for id in $(cat pipe5.jobids); do
+	    flux job attach $id \
+	        <input_stdin_file >pipe5_${i}.out &&
+            test_cmp input_stdin_file pipe5_${i}.out &&
+	    i=$((i+1))
+	done
 '
 
 test_expect_success NO_CHAIN_LINT 'flux-shell: no stdin desired in job' '
-        id=$(flux mini submit -n1 sleep 60)
-        flux job attach $id < input_stdin_file 2> pipe6A.err &
+        id=$(flux submit -n1 sleep 60)
+        flux job attach $id < input_stdin_file &
         pid=$! &&
-        flux job wait-event -p guest.exec.eventlog $id shell.init 2> pipe6B.err &&
-        flux job wait-event -p guest.input -m eof=true $id data 2> pipe6C.err &&
-        flux job cancel $id 2> pipe6D.err &&
+        flux job wait-event -W -p guest.input -m eof=true $id data &&
+        flux cancel $id  &&
         test_expect_code 143 wait $pid
 '
 
@@ -117,25 +117,31 @@ test_expect_success NO_CHAIN_LINT 'flux-shell: no stdin desired in job' '
 # job outputs a very nice chunk of data.
 test_expect_success 'flux-shell: task completed, try to pipe into stdin' '
         ${LPTEST} 79 500 > big_dataset &&
-        id=$(flux mini submit -n1 cat big_dataset) &&
-        flux job wait-event $id clean 2> pipe7A.err &&
-        test_must_fail flux job attach $id < input_stdin_file 2> pipe7B.err
+        id=$(flux submit -n1 cat big_dataset) &&
+        flux job wait-event $id clean &&
+        test_must_fail flux job attach $id < input_stdin_file
+'
+
+test_expect_success 'flux-shell: task completed, try to pipe into stdin, no error if read only' '
+        ${LPTEST} 79 500 > big_dataset &&
+        id=$(flux submit -n1 cat big_dataset) &&
+        flux job wait-event $id clean  &&
+        flux job attach --read-only $id < input_stdin_file
 '
 
 test_expect_success NO_CHAIN_LINT 'flux-shell: pipe to stdin twice, second fails' '
-        id=$(flux mini submit -n1 sleep 60)
-        flux job attach $id < input_stdin_file 2> pipe8A.err &
+        id=$(flux submit -n1 sleep 60)
+        flux job attach $id < input_stdin_file &
         pid=$!
-        flux job wait-event -p guest.exec.eventlog $id shell.init 2> pipe8B.err &&
-        flux job wait-event -p guest.input -m eof=true $id data 2> pipe8C.err &&
-        test_must_fail flux job attach $id < input_stdin_file 2> pipe8D.err &&
-        flux job cancel $id 2> pipe8E.err &&
+        flux job wait-event -W -p guest.input -m eof=true $id data &&
+        test_must_fail flux job attach $id < input_stdin_file &&
+        flux cancel $id &&
         test_expect_code 143 wait $pid
 '
 
 test_expect_success NO_CHAIN_LINT 'flux-shell: pipe in zero data' '
-        flux mini run -n1 echo < /dev/null &&
-        cat /dev/null | flux mini run -n1 cat
+        flux run -n1 echo < /dev/null &&
+        cat /dev/null | flux run -n1 cat
 '
 
 #
@@ -143,13 +149,18 @@ test_expect_success NO_CHAIN_LINT 'flux-shell: pipe in zero data' '
 #
 
 test_expect_success 'flux-shell: run 1-task input file as stdin job' '
-        flux mini run -n1 --input=input_stdin_file \
+        flux run -n1 --input=input_stdin_file \
              ${TEST_SUBPROCESS_DIR}/test_echo -O -n > file0.out &&
         test_cmp input_stdin_file file0.out
 '
 
+test_expect_success 'flux-shell: input from file results in redirect event' '
+	flux job wait-event -Hp input -t 5 -m stream=stdin \
+		$(flux job last) redirect
+'
+
 test_expect_success 'flux-shell: run 2-task input file as stdin job' '
-        flux mini run -n2 --input=input_stdin_file --label-io \
+        flux run -n2 --input=input_stdin_file --label-io \
              ${TEST_SUBPROCESS_DIR}/test_echo -O -n > file1.out &&
         grep "0: foo" file1.out &&
         grep "0: doh" file1.out &&
@@ -157,49 +168,34 @@ test_expect_success 'flux-shell: run 2-task input file as stdin job' '
         grep "1: doh" file1.out
 '
 
-test_expect_success NO_CHAIN_LINT 'flux-shell: multiple jobs, each want stdin via file' '
-        id1=$(flux mini submit -n1 --input=input_stdin_file \
-              ${TEST_SUBPROCESS_DIR}/test_echo -O -n)
-        id2=$(flux mini submit -n1 --input=input_stdin_file \
-              ${TEST_SUBPROCESS_DIR}/test_echo -O -n)
-        id3=$(flux mini submit -n1 --input=input_stdin_file \
-              ${TEST_SUBPROCESS_DIR}/test_echo -O -n)
-        id4=$(flux mini submit -n1 --input=input_stdin_file \
-              ${TEST_SUBPROCESS_DIR}/test_echo -O -n)
-        flux job attach $id1 > file2_1.out 2> file2_1.err &
-        pid1=$!
-        flux job attach $id2 > file2_2.out 2> file2_2.err &
-        pid2=$!
-        flux job attach $id3 > file2_3.out 2> file2_3.err &
-        pid3=$!
-        flux job attach $id4 > file2_4.out 2> file2_4.err &
-        pid4=$!
-        wait $pid1 &&
-        wait $pid2 &&
-        wait $pid3 &&
-        wait $pid4 &&
-        test_cmp input_stdin_file file2_1.out &&
-        test_cmp input_stdin_file file2_2.out &&
-        test_cmp input_stdin_file file2_3.out &&
-        test_cmp input_stdin_file file2_4.out
+test_expect_success 'flux-shell: multiple jobs, each want stdin via file' '
+	flux submit --cc=1-4 -n1 -t 30s --input=input_stdin_file \
+	    ${TEST_SUBPROCESS_DIR}/test_echo -O -n >file2.jobids &&
+	test_debug "cat file2.jobids" &&
+	i=1 &&
+	for id in $(cat file2.jobids); do
+	    flux job attach $id >file2_${i}.out &&
+            test_cmp input_stdin_file file2_${i}.out &&
+	    i=$((i+1))
+	done
 '
 
 test_expect_success LONGTEST 'flux-shell: 10K line lptest input works' '
-        flux mini run -n1 --input=lptestXXL_input \
+        flux run -n1 --input=lptestXXL_input \
              ${TEST_SUBPROCESS_DIR}/test_echo -O -n > file3.out &&
 	test_cmp lptestXXL_input file3.out
 '
 
 test_expect_success 'flux-shell: input file invalid' '
-        test_must_fail flux mini run -n1 --input=/foo/bar/baz \
+        test_must_fail_or_be_terminated flux run -n1 --input=/foo/bar/baz \
              ${TEST_SUBPROCESS_DIR}/test_echo -O -n
 '
 
 test_expect_success 'flux-shell: task stdin via file, try to pipe into stdin fails' '
-        id=$(flux mini submit -n1 --input=input_stdin_file sleep 60) &&
+        id=$(flux submit -n1 --input=input_stdin_file sleep 60) &&
         flux job wait-event $id start &&
         test_must_fail flux job attach $id < input_stdin_file &&
-        flux job cancel $id
+        flux cancel $id
 '
 
 #
@@ -207,7 +203,7 @@ test_expect_success 'flux-shell: task stdin via file, try to pipe into stdin fai
 #
 
 test_expect_success 'flux-shell: run 1-task input file with service type' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --setopt "input.stdin.type=\"service\"" \
              ${TEST_SUBPROCESS_DIR}/test_echo -O -n) &&
         flux job attach $id < input_stdin_file > cc1.out &&
@@ -215,7 +211,7 @@ test_expect_success 'flux-shell: run 1-task input file with service type' '
 '
 
 test_expect_success 'flux-shell: error on bad input type' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --setopt "input.stdin.type=\"foobar\"" \
              echo foo) &&
         flux job wait-event $id clean &&
@@ -223,7 +219,7 @@ test_expect_success 'flux-shell: error on bad input type' '
 '
 
 test_expect_success 'flux-shell: error on no path with file output' '
-        id=$(flux mini submit -n1 \
+        id=$(flux submit -n1 \
              --label-io --setopt "input.stdin.type=\"file\"" \
              echo foo) &&
         flux job wait-event $id clean &&
@@ -239,8 +235,8 @@ test_expect_success 'flux-shell: no fatal exception after stdin sent to exited t
 	if test \$FLUX_TASK_RANK -ne 0; then cat; fi
 	EOF
 	chmod +x ${name}.sh &&
-	id=$(flux mini submit -n2 -o verbose ./${name}.sh) &&
-	flux job wait-event -v -p guest.exec.eventlog ${id} shell.start &&
+	id=$(flux submit -n2 -o verbose ./${name}.sh) &&
+	flux job wait-event -v -p exec ${id} shell.start &&
 	flux kvs get --watch --waitcreate --count=1 ${name}.0.started &&
 	flux kvs get --watch --waitcreate --count=1 ${name}.1.started &&
 	echo | flux job attach -XE ${id} &&
diff --git a/t/t2608-job-shell-log.t b/t/t2608-job-shell-log.t
index 0f94c6479f4e..bc413d680161 100755
--- a/t/t2608-job-shell-log.t
+++ b/t/t2608-job-shell-log.t
@@ -4,7 +4,7 @@ test_description='Test flux-shell logging implementation'
 
 . `dirname $0`/sharness.sh
 
-test_under_flux 2
+test_under_flux 2 job
 
 FLUX_SHELL="${FLUX_BUILD_DIR}/src/shell/flux-shell"
 
@@ -14,52 +14,16 @@ INITRC_PLUGINPATH="${SHARNESS_TEST_DIRECTORY}/shell/plugins/.libs"
 # test initrc files need to be able to find fluxometer.lua:
 export LUA_PATH="$LUA_PATH;${SHARNESS_TEST_DIRECTORY}/?.lua"
 
-test_expect_success 'flux-shell: log: generate 1-task jobspec and matching R' '
-	flux jobspec srun -N1 -n1 echo Hi >j1 &&
-	cat >R1 <<-EOT
-	{"version": 1, "execution":{ "R_lite":[
-		{ "children": { "core": "0,1" }, "rank": "0" }
-        ]}}
-	EOT
-'
-test_expect_success 'flux-shell: run log testing plugin' '
+test_expect_success 'flux-shell: create log testing initrc' '
 	name=log &&
-	cat >${name}.lua <<-EOF &&
+	cat >${name}.lua <<-EOF
 	plugin.searchpath = "${INITRC_PLUGINPATH}"
 	plugin.load { file = "log.so" }
 	EOF
-	${FLUX_SHELL} -v -v -s -r 0 -j j1 -R R1 --initrc=${name}.lua 0 \
-		> ${name}.log 2>&1 &&
-	test_debug "cat ${name}.log"
 '
-for topic in "shell.init" "shell.exit" \
-	     "task.init" "task.exit" \
-	     "task.fork" "task.fork"; do
-    test_expect_success "$topic: got TRACE level message" "
-        grep \"TRACE: log: .*: $topic: trace message\" log.log
-    "
-    test_expect_success "$topic: got DEBUG level message" "
-        grep \"DEBUG: log: .*: $topic: debug message\" log.log
-    "
-    test_expect_success "$topic: got INFO level message" "
-        grep \" INFO: log: .*: $topic: log message\" log.log
-    "
-    test_expect_success "$topic: got WARN level message" "
-        grep \" WARN: log: .*: $topic: warn message\" log.log
-    "
-    test_expect_success "$topic: got ERROR level message" "
-        grep \"ERROR: log: .*: $topic: error message\" log.log
-    "
-    test_expect_success "$topic: got log_errn message" "
-	grep \"ERROR:.*: $topic: log_errn message: Operation not permitted\" log.log
-    "
-    test_expect_success "$topic: got log_errno message" "
-	grep \"ERROR: log: .*: $topic: log_errno message: Invalid argument\" log.log
-    "
-done
 
 test_expect_success 'flux-shell: run job with verbose logging to output' '
-	flux mini run -o verbose=2 -o initrc=log.lua -n2 -N2 hostname \
+	flux run -o verbose=2 -o initrc=log.lua -n2 -N2 hostname \
 		>log-test.output 2>log-test.err
 '
 
@@ -97,7 +61,7 @@ for topic in "shell.init" "shell.exit" \
 done
 
 test_expect_success 'flux-shell: run job with normal logging level' '
-	flux mini run -o initrc=log.lua -n2 -N2 hostname \
+	flux run -o initrc=log.lua -n2 -N2 hostname \
 		>log-test.output 2>log-test.err
 '
 
@@ -133,22 +97,22 @@ for topic in "shell.init" "shell.exit" \
 done
 
 test_expect_success 'flux-shell: missing command logs fatal error' '
-	test_expect_code 127 flux mini run nosuchcommand 2>missing.err &&
-	grep "flux-shell\[0\]: FATAL: task 0: start failed" missing.err &&
-	grep "job.exception type=exec severity=0 task 0: start failed" missing.err &&
+	test_expect_code 127 flux run nosuchcommand 2>missing.err &&
+	grep "flux-shell\[0\]: FATAL: task 0.*: start failed" missing.err &&
+	grep "job.exception" missing.err | grep "type=exec severity=0 task 0.*: start failed" &&
         grep "No such file or directory" missing.err
 '
 
 test_expect_success 'flux-shell: illegal command logs fatal error' '
 	mkdir adirectory &&
-	test_expect_code 126 flux mini run ./adirectory 2>illegal.err &&
-	grep "flux-shell\[0\]: FATAL: task 0: start failed" illegal.err &&
-	grep "job.exception type=exec severity=0 task 0: start failed" illegal.err &&
+	test_expect_code 126 flux run ./adirectory 2>illegal.err &&
+	grep "flux-shell\[0\]: FATAL: task 0.*: start failed" illegal.err &&
+	grep "job.exception" illegal.err | grep "type=exec severity=0 task 0.*: start failed" &&
 	grep "Permission denied" illegal.err
 '
 
 test_expect_success 'flux-shell: bad cwd emits message, but completes' '
-	flux mini run --setattr="system.cwd=/foo" pwd 2>badcwd.err &&
+	flux run --setattr="system.cwd=/foo" pwd 2>badcwd.err &&
         grep "Could not change dir to /foo: No such file or directory" badcwd.err
 '
 
@@ -156,29 +120,6 @@ test_expect_success 'job-exec: set kill-timeout to low value for testing' '
 	flux module reload job-exec kill-timeout=0.25
 '
 
-#  Fatal error tests below must exit with failure, but exit due to
-#  SIGTERM should also be allowed. Therefore adapt sharness `test_must_fail`
-#  to allow exit 1 or 143 for succcess.
-#
-test_must_fail_or_be_terminated() {
-    "$@"
-    exit_code=$?
-    # Allow death by SIGTERM
-    if test $exit_code = 143; then
-        return 0
-    elif test $exit_code = 0; then
-        echo >&2 "test_must_fail: command succeeded: $*"
-        return 1
-    elif test $exit_code -gt 129 -a $exit_code -le 192; then
-        echo >&2 "test_must_fail: died by non-SIGTERM signal: $*"
-        return 1
-    elif test $exit_code = 127; then
-        echo >&2 "test_must_fail: command not found: $*"
-        return 1
-    fi
-    return 0
-}
-
 dump_job_output_eventlog() {
     flux job eventlog -p guest.output $(sed -n 's/^jobid: //p' fatal-$1.out)
 }
@@ -187,7 +128,7 @@ test_expect_success 'flux-shell: fatal error in shell.init works' '
 	site=shell.init &&
 	test_when_finished "test_debug \"dump_job_output_eventlog $site\"" &&
 	test_must_fail_or_be_terminated \
-		flux mini run -vvv -o verbose=2 -n2 -N2 \
+		flux run -vvv -o verbose=2 -n2 -N2 \
 			-o initrc=log.lua \
 			-o log-fatal-error=$site hostname \
 		>fatal-${site}.out 2>&1 &&
@@ -203,7 +144,7 @@ test_expect_success 'flux-shell: fatal error in shell.exit works' '
 	site=shell.exit &&
 	echo "running mini run" &&
 	test_must_fail_or_be_terminated \
-		flux mini run -vvv -n2 -N2 \
+		flux run -vvv -n2 -N2 \
 			-o verbose=2 \
 			-o initrc=log.lua \
                 	-o log-fatal-error=$site hostname &&
@@ -214,7 +155,7 @@ test_expect_success 'flux-shell: fatal error in task.init works' '
 	site=task.init &&
 	test_when_finished "test_debug \"dump_job_output_eventlog $site\"" &&
 	test_must_fail_or_be_terminated \
-		flux mini run -vvv -n2 -N2 \
+		flux run -vvv -n2 -N2 \
                         -o verbose=2 \
 			-o initrc=log.lua \
                 	-o log-fatal-error=$site hostname \
@@ -231,7 +172,7 @@ test_expect_success 'flux-shell: fatal error in task.fork works' '
 	site=task.fork &&
 	test_when_finished "test_debug \"dump_job_output_eventlog $site\"" &&
 	test_must_fail_or_be_terminated \
-		flux mini run -v -n2 -N2 \
+		flux run -v -n2 -N2 \
 			-o initrc=log.lua \
                 	-o log-fatal-error=$site hostname \
 		>fatal-${site}.out 2>&1 &&
@@ -244,10 +185,10 @@ test_expect_success 'flux-shell: fatal error in task.fork works' '
 
 '
 test_expect_success 'flux-shell: fatal error in task.exec works' '
-	site=task.fork &&
+	site=task.exec &&
 	test_when_finished "test_debug \"dump_job_output_eventlog $site\"" &&
 	test_must_fail_or_be_terminated \
-		flux mini run -v -n2 -N2 \
+		flux run -v -n2 -N2 \
 			-o initrc=log.lua \
                 	-o log-fatal-error=$site hostname \
 		>fatal-${site}.out 2>&1 &&
@@ -259,4 +200,24 @@ test_expect_success 'flux-shell: fatal error in task.exec works' '
 		fatal-${site}.out &&
 	dump_job_output_eventlog $site | grep log-fatal-error
 '
+test_expect_success 'flux-shell: stdout/err from task.exec works' '
+	cat <<-EOF >task.exec.print.lua &&
+	plugin.register {
+	  name = "stdout-test",
+	    handlers = {
+	    {
+	      topic = "task.exec",
+	      fn = function ()
+	        io.stderr:write("this is stderr\n")
+	        io.stdout:write("this is stdout\n")
+	      end
+	    }
+	  }
+	}
+	EOF
+	flux run -o initrc=task.exec.print.lua hostname \
+	  >print.out 2>print.err &&
+	grep "^this is stderr" print.err &&
+	grep "^this is stdout" print.out
+'
 test_done
diff --git a/t/t2609-job-shell-events.t b/t/t2609-job-shell-events.t
index 5b0e03e72650..ee16f7e4224f 100755
--- a/t/t2609-job-shell-events.t
+++ b/t/t2609-job-shell-events.t
@@ -4,7 +4,7 @@ test_description='Test flux-shell emitted events'
 
 . `dirname $0`/sharness.sh
 
-test_under_flux 2
+test_under_flux 2 job
 
 FLUX_SHELL="${FLUX_BUILD_DIR}/src/shell/flux-shell"
 
@@ -12,30 +12,30 @@ INITRC_TESTDIR="${SHARNESS_TEST_SRCDIR}/shell/initrc"
 INITRC_PLUGINPATH="${SHARNESS_TEST_DIRECTORY}/shell/plugins/.libs"
 
 test_expect_success 'flux-shell: 1N: init and start shell events are emitted' '
-	id=$(flux mini submit -n1 -N1 /bin/true)  &&
-	flux job wait-event -vt 5 -p guest.exec.eventlog \
+	id=$(flux submit -n1 -N1 true)  &&
+	flux job wait-event -vt 5 -p exec \
 		-m leader-rank=0 ${id} shell.init &&
-	flux job wait-event -vt 5 -p guest.exec.eventlog \
-		-m size=1 ${id} shell.init  &&
-	flux job wait-event -vt 5 -p guest.exec.eventlog \
-		-m task-count=1 ${id} shell.start 
+	flux job wait-event -vt 5 -p exec \
+		${id} shell.init  &&
+	flux job wait-event -vt 5 -p exec \
+		${id} shell.start
 '
 test_expect_success 'flux-shell: 2N: init and start shell events are emitted' '
-	id=$(flux mini submit -n4 -N2 /bin/true)  &&
-	flux job wait-event -vt 5 -p guest.exec.eventlog \
+	id=$(flux submit -n4 -N2 true)  &&
+	flux job wait-event -vt 5 -p exec \
 		-m leader-rank=0  ${id} shell.init &&
-	flux job wait-event -vt 5 -p guest.exec.eventlog \
+	flux job wait-event -vt 5 -p exec \
 		-m size=2 ${id} shell.init  &&
-	flux job wait-event -vt 5 -p guest.exec.eventlog \
-		-m task-count=4 ${id} shell.start 
+	flux job wait-event -vt 5 -p exec \
+		${id} shell.start
 '
 test_expect_success 'flux-shell: plugin can add event context' '
 	cat >test-event.lua <<-EOT &&
 	plugin.searchpath = "${INITRC_PLUGINPATH}"
 	plugin.load { file = "test-event.so" }
 	EOT
-	id=$(flux mini submit -o initrc=test-event.lua -n2 -N2 hostname) &&
-	flux job wait-event -vt 5 -p guest.exec.eventlog \
+	id=$(flux submit -o initrc=test-event.lua -n2 -N2 hostname) &&
+	flux job wait-event -vt 5 -p exec \
 		-m event-test=foo ${id} shell.init
 
 '
diff --git a/t/t2610-job-shell-mpir.t b/t/t2610-job-shell-mpir.t
index 67adca9a2d68..0c9658d64f02 100755
--- a/t/t2610-job-shell-mpir.t
+++ b/t/t2610-job-shell-mpir.t
@@ -4,7 +4,7 @@ test_description='Test flux-shell MPIR and ptrace support'
 
 . `dirname $0`/sharness.sh
 
-test_under_flux 4
+test_under_flux 4 job
 
 FLUX_SHELL="${FLUX_BUILD_DIR}/src/shell/flux-shell"
 
@@ -13,25 +13,85 @@ INITRC_PLUGINPATH="${SHARNESS_TEST_DIRECTORY}/shell/plugins/.libs"
 mpir="${SHARNESS_TEST_DIRECTORY}/shell/mpir"
 
 shell_leader_rank() {
-    flux job wait-event -f json -p guest.exec.eventlog $1 shell.init | \
-        jq '.context["leader-rank"]'
+	flux job wait-event -f json -p exec $1 shell.init | \
+	    jq '.context["leader-rank"]'
 }
 shell_service() {
-    flux job wait-event -f json -p guest.exec.eventlog $1 shell.init | \
-        jq -r '.context["service"]'
+	flux job wait-event -f json -p exec $1 shell.init | \
+	    jq -r '.context["service"]'
 }
 
 for test in 1:1 2:2 2:4 4:4 4:8 4:7; do
     TASKS=${test#*:}
     NODES=${test%:*}
     test_expect_success "flux-shell: ${NODES}N/${TASKS}P: trace+mpir works" '
-	id=$(flux mini submit -o stop-tasks-in-exec \
-             -n${TASKS} -N${NODES} /bin/true)  &&
-        flux job wait-event -vt 5 -p guest.exec.eventlog \
-                -m sync=true ${id} shell.start &&
-        ${mpir} $(shell_leader_rank $id) $(shell_service $id) &&
-        flux job kill -s CONT ${id} &&
-        flux job attach ${id}
+	id=$(flux submit -o stop-tasks-in-exec \
+	    -n${TASKS} -N${NODES} true)  &&
+	flux job wait-event -vt 5 -p exec -m sync=true ${id} shell.start &&
+	${mpir} -r $(shell_leader_rank $id) -s $(shell_service $id) &&
+	flux job kill -s CONT ${id} &&
+	flux job attach ${id}
     '
 done
+
+
+test_expect_success 'flux-shell: test security of proctable method' '
+	id=$(flux submit -o stop-tasks-in-exec true) &&
+	flux job wait-event -vt 5 -p exec -m sync=true ${id} shell.start &&
+	shell_rank=$(shell_leader_rank $id) &&
+	shell_service=$(shell_service $id) &&
+	( export FLUX_HANDLE_USERID=9999 &&
+	  export FLUX_HANDLE_ROLEMASK=0x2 &&
+	  test_expect_code 1 ${mpir} -r $shell_rank -s $shell_service
+	) &&
+	${mpir} -r $(shell_leader_rank $id) -s $(shell_service $id) &&
+	flux job kill -s CONT ${id} &&
+	flux job attach ${id}
+'
+test_expect_success 'mpir: tool launch is supported' '
+	id=$(flux submit -N4 -n8 -o stop-tasks-in-exec true) &&
+	flux job wait-event -vt 5 -p exec -m sync=true $id shell.start &&
+	shell_rank=$(shell_leader_rank $id) &&
+	shell_service=$(shell_service $id) &&
+	${mpir} -r ${shell_rank} -s ${shell_service} \
+	    --tool-launch /bin/echo this is a tool >tool.log 2>&1 &&
+	test_debug "cat tool.log" &&
+	grep "MPIR: rank 0: echo: stdout: this is a tool" tool.log &&
+	grep "MPIR: rank 1: echo: stdout: this is a tool" tool.log &&
+	grep "MPIR: rank 2: echo: stdout: this is a tool" tool.log &&
+	grep "MPIR: rank 3: echo: stdout: this is a tool" tool.log &&
+	test $(grep -c "MPIR:.*this is a tool" tool.log) -eq 4 &&
+	flux job kill -s CONT $id &&
+	flux job attach $id
+'
+# Empty args for MPIR_executable_path -- also run with shell leader not
+# on broker rank 0 to ensure this configuration works with tool launch
+test_expect_success 'mpir: tool launch works with empty MPIR_server_arguments' '
+	id=$(flux submit --requires=rank:1,3 \
+	     -N2 -n2 -o stop-tasks-in-exec true) &&
+	flux job wait-event -vt 5 -p exec -m sync=true $id shell.start &&
+	shell_rank=$(shell_leader_rank $id) &&
+	shell_service=$(shell_service $id) &&
+	${mpir} -r ${shell_rank} -s ${shell_service} \
+	    --tool-launch hostname >tool2.log 2>&1 &&
+	test_debug "cat tool2.log" &&
+	grep "MPIR: rank 1: hostname: stdout:" tool2.log &&
+	grep "MPIR: rank 3: hostname: stdout:" tool2.log &&
+	test $(grep -c "MPIR:.*hostname:" tool2.log) -eq 2 &&
+	flux job kill -s CONT $id &&
+	flux job attach $id
+'
+test_expect_success 'mpir: tool launch reports errors' '
+	id=$(flux submit -N2 -n2 -o stop-tasks-in-exec true) &&
+	flux job wait-event -vt 5 -p exec -m sync=true $id shell.start &&
+	shell_rank=$(shell_leader_rank $id) &&
+	shell_service=$(shell_service $id) &&
+	${mpir} -r ${shell_rank} -s ${shell_service} \
+	    --tool-launch nosuchcommand >tool3.log 2>&1 &&
+	test_debug "cat tool3.log" &&
+	grep "MPIR: rank 0: nosuchcommand: error launching process" tool3.log &&
+	flux job kill -s CONT $id &&
+	flux job attach $id
+'
+
 test_done
diff --git a/t/t2611-debug-emulate.t b/t/t2611-debug-emulate.t
new file mode 100755
index 000000000000..e9b50513b2cd
--- /dev/null
+++ b/t/t2611-debug-emulate.t
@@ -0,0 +1,111 @@
+#!/bin/sh
+#
+test_description='Test parallel debugger support in emulation mode'
+
+. `dirname $0`/sharness.sh
+
+stall="${SHARNESS_TEST_DIRECTORY}/debug/stall"
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+
+stop_tasks_test()     {
+	jq '.attributes.system.shell.options."stop-tasks-in-exec" = 1';
+}
+
+test_under_flux 2
+
+TIMEOUT=100
+
+parse_jobid() {
+	outfile=$1 &&
+	jobid=$(cat ${outfile} | grep ^jobid: | awk '{ print $2 }') &&
+	echo ${jobid}
+}
+
+parse_totalview_jobid() {
+	outfile=$1 &&
+	jobid=$(cat ${outfile} | grep totalview_jobid | \
+	awk '{ print $2 }' | awk -F= '{ print $2 }') &&
+	echo ${jobid}
+}
+
+test_expect_success 'debugger: launching under debugger via flux run works' '
+	flux run -v --debug-emulate -N 2 -n 2 sleep 0 2> jobid.out &&
+	jobid=$(parse_jobid jobid.out) &&
+	echo ${jobid} > jobid &&
+	flux job wait-event -vt ${TIMEOUT} ${jobid} finish
+'
+
+test_expect_success 'debugger: submitting under debugger via flux submit works' '
+	jobid=$(flux submit -N 2 -n 2 -o stop-tasks-in-exec hostname) &&
+	flux job wait-event -vt ${TIMEOUT}  ${jobid} start &&
+	flux job attach --debug-emulate ${jobid} &&
+	flux job wait-event -vt ${TIMEOUT}  ${jobid} finish
+'
+
+test_expect_success 'debugger: submitting under debugger via flux-job works' '
+	flux run --dry-run -N2 -n 2 hostname \
+		| stop_tasks_test > stop_tasks.json &&
+	jobid=$(flux job submit stop_tasks.json) &&
+	flux job wait-event -vt ${TIMEOUT}  ${jobid} start &&
+	flux job attach --debug-emulate ${jobid} &&
+	flux job wait-event -vt ${TIMEOUT}  ${jobid} finish
+'
+
+test_expect_success 'debugger: SIGCONT can unlock a job in stop-tasks-in-exec' '
+	jobid=$(flux submit -N 2 -n 2 -o stop-tasks-in-exec hostname) &&
+	flux job wait-event -vt ${TIMEOUT}  ${jobid} start &&
+	flux job kill --signal=SIGCONT ${jobid} &&
+	flux job wait-event -vt ${TIMEOUT}  ${jobid} finish
+'
+
+test_expect_success 'debugger: attaching to a running job works' '
+	jobid=$(flux submit ${stall} done 10) &&
+	flux job wait-event -vt ${TIMEOUT} ${jobid} start &&
+	${waitfile} -v -t ${TIMEOUT} done &&
+	flux job attach -v --debug-emulate ${jobid} &&
+	flux job wait-event -vt ${TIMEOUT} ${jobid} finish
+'
+
+test_expect_success 'debugger: attaching to a finished job must fail' '
+	jobid=$(flux submit -n 2 hostname) &&
+	flux job wait-event -vt ${TIMEOUT} ${jobid} finish &&
+	test_must_fail flux job attach --debug-emulate ${jobid}
+'
+
+test_expect_success 'debug-emulate: attaching to a failed job must fail' '
+	jobid=$(flux submit -n 2 ./bad_cmd) &&
+	flux job wait-event -vt ${TIMEOUT} ${jobid} finish &&
+	test_must_fail flux job attach --debug-emulate ${jobid}
+'
+
+test_expect_success 'debugger: totalview_jobid is set for attach mode' '
+	jobid=$(flux submit ${stall} done2 10) &&
+	jobid=$(flux job id ${jobid}) &&
+	flux job wait-event -vt ${TIMEOUT} ${jobid} start &&
+	${waitfile} -v -t ${TIMEOUT} done2 &&
+	flux job attach -vv --debug-emulate ${jobid} 2> jobid.out2 &&
+	flux job wait-event -vt ${TIMEOUT} ${jobid} finish &&
+	tv_jobid=$(parse_totalview_jobid jobid.out2) &&
+	test ${tv_jobid} = "${jobid}"
+'
+
+flux_job_attach() {
+	flux job attach -vv --debug ${1} 2> ${2} &
+	${waitfile} -v -t ${TIMEOUT} --pattern="totalview_jobid" ${2}
+}
+
+# flux job attach --debug JOBID must not continue target processes
+test_expect_success 'debugger: job attach --debug must not continue target' '
+	jobid=$(flux submit ${stall} done3 100) &&
+	jobid=$(flux job id ${jobid}) &&
+	flux job wait-event -vt ${TIMEOUT} ${jobid} start &&
+	${waitfile} -v -t ${TIMEOUT} done3 &&
+	flux_job_attach ${jobid} jobid.out3 &&
+	tv_jobid=$(parse_totalview_jobid jobid.out3) &&
+	test ${tv_jobid} = "${jobid}" &&
+	test_must_fail flux job wait-event -vt 2 ${jobid} finish &&
+	flux cancel ${jobid} &&
+	flux job wait-event -vt ${TIMEOUT} ${jobid} finish
+'
+
+test_done
diff --git a/t/t2612-job-shell-pty.t b/t/t2612-job-shell-pty.t
new file mode 100755
index 000000000000..002457418bd4
--- /dev/null
+++ b/t/t2612-job-shell-pty.t
@@ -0,0 +1,142 @@
+#!/bin/sh
+#
+test_description='Test flux-shell pty support'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 4 job
+
+# an existing FLUX_TERMINUS_SESSION will interfere with tests below
+unset FLUX_TERMINUS_SESSION
+
+FLUX_SHELL="${FLUX_BUILD_DIR}/src/shell/flux-shell"
+
+INITRC_TESTDIR="${SHARNESS_TEST_SRCDIR}/shell/initrc"
+INITRC_PLUGINPATH="${SHARNESS_TEST_DIRECTORY}/shell/plugins/.libs"
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py -f asciicast"
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+
+shell_leader_rank() {
+    flux job wait-event -f json -p exec $1 shell.init | 
+        jq '.context["leader-rank"]'
+}
+shell_service() {
+    flux job wait-event -f json -p exec $1 shell.init | \
+        jq -r '.context["service"]'
+}
+terminus_jobid() {
+    test $# -lt 2 && return 1
+    local jobid=$1
+    local cmd=$2
+    shift 2
+    flux terminus $cmd \
+        -r $(shell_leader_rank $jobid) \
+        -s $(shell_service $jobid).terminus "$@"
+}
+
+test_expect_success 'pty: submit a job with an interactive pty' '
+	id=$(flux submit --flags waitable -o pty.interactive tty) &&
+	terminus_jobid $id list &&
+	$runpty flux job attach ${id} &&
+	flux job wait $id
+'
+test_expect_success NO_CHAIN_LINT 'pty: run job with pty' '
+	printf "PS1=XXX:\n" >ps1.rc
+	id=$(flux submit -o pty.interactive bash --rcfile ps1.rc | flux job id)
+	$runpty -o log.job-pty flux job attach ${id} &
+	pid=$! &&
+	terminus_jobid ${id} list &&
+	$waitfile -t 20 -vp "XXX:" log.job-pty &&
+	printf "flux job id \$FLUX_JOB_ID\r" | terminus_jobid ${id} attach -p 0 &&
+	$waitfile -t 20 -vp ${id} log.job-pty &&
+	printf "exit\r\n" | terminus_jobid ${id} attach -p 0 &&
+	$waitfile -t 20 -vp exit log.job-pty &&
+	wait $pid
+'
+# Interactively attach to pty many times to ensure no hangs, etc.
+#
+test_expect_success NO_CHAIN_LINT 'pty: interactive job with pty' '
+	flux submit --cc=1-10 -n1 -o pty.interactive stty size >jobids &&
+	for id in $(cat jobids); do $runpty flux job attach $id; done
+'
+
+test_expect_success 'pty: client is detached from terminated job' '
+	cat <<-EOF >die-test.sh &&
+	#!/bin/sh
+	# Do not continue until test is ready:
+	flux kvs get --waitcreate --count=1 --label test-ready
+	# Kill shell so that services immediately go away
+	kill -9 \$PPID
+	sleep 10
+	EOF
+	chmod +x die-test.sh &&
+	jobid=$(flux submit -n2 -o pty.interactive -o pty.ranks=0 \
+	        -o exit-timeout=1 ./die-test.sh) &&
+	{ $runpty -w 80x25 -o log.killed \
+		flux job attach --show-exec ${jobid} & } &&
+	pid=$! &&
+	$waitfile -t 20 -vp "shell.start" log.killed &&
+	flux kvs put -N job-$(flux job id ${jobid}) test-ready=1 &&
+	$waitfile -t 20 -vp "pty disappeared" log.killed &&
+	test_must_fail wait $pid
+'
+test_expect_success 'pty: pty for all tasks, output captured' '
+	flux run -n2 --label-io -o pty tty > ptys.out &&
+	grep 0: ptys.out &&
+	grep 1: ptys.out &&
+	test_must_fail grep "not a tty" ptys.out
+'
+test_expect_success 'pty: ptys only on select ranks' '
+	test_expect_code 1 \
+		flux run -n2 --label-io -o pty.ranks=1 tty > ptys1.out &&
+	grep "0: not a tty" ptys1.out &&
+	test_must_fail grep "1: not a tty" ptys1.out
+'
+test_expect_success 'pty: pty.ranks can take an idset' '
+	flux run -n2 --label-io -o pty.ranks=0,1 tty > ptyss.out &&
+	grep 0: ptyss.out &&
+	grep 1: ptyss.out &&
+	test_must_fail grep "not a tty" ptyss.out
+'
+test_expect_success 'pty: pty.interactive forces a pty on rank 0' '
+	id=$(flux submit \
+		-o pty.interactive -o pty.ranks=1 \
+		-n2 \
+		tty) &&
+	terminus_jobid $id list &&
+	$runpty flux job attach ${id} &&
+	flux job eventlog -p output ${id} | grep "adding pty to rank 0"
+'
+test_expect_success 'pty: -o pty.interactive and -o pty.capture can be used together' '
+	for i in $(seq 1 3); do
+		id=$(flux submit -o pty.interactive -o pty.capture tty) &&
+		$runpty flux job attach $id >ptyim.out 2>&1 &&
+		$runpty flux job attach $id &&
+		flux job eventlog -f json -p output $id \
+			| tail -n1 >last-event.$i
+	done &&
+	# Check that eof:true is the last event for all runs
+	cat last-event.1 | jq -e .context.eof &&
+	cat last-event.2 | jq -e .context.eof &&
+	cat last-event.3 | jq -e .context.eof
+'
+test_expect_success 'pty: unsupported -o pty.<opt> generates exception' '
+	test_must_fail flux run -o pty.foo hostname
+'
+
+test_expect_success 'pty: no hang when invalid command is run under pty' '
+	test_expect_code 127 run_timeout 15 \
+		flux run -o pty.interactive nosuchcommand
+'
+# Note: test below uses printf(1) to send [Ctrl-SPACE (NUL), newline, Ctrl-D]
+# over the pty connection and expects ^@ (the standard representation of NUL)
+# to appear in output.
+nul_ctrl_d() {
+	python3 -c 'print("\x00\n\x04", end=None)'
+}
+test_expect_success 'pty: NUL (Ctrl-SPACE) character not dropped' '
+	nul_ctrl_d | flux run -vvv -o pty.interactive -o pty.capture cat -v &&
+	flux job eventlog -HL -p output $(flux job last) &&
+	flux job eventlog -HL -p output $(flux job last) | grep \\^@
+'
+test_done
diff --git a/t/t2613-job-shell-batch.t b/t/t2613-job-shell-batch.t
new file mode 100755
index 000000000000..108058095518
--- /dev/null
+++ b/t/t2613-job-shell-batch.t
@@ -0,0 +1,63 @@
+#!/bin/sh
+#
+test_description='Test flux-shell per-resource and batch support'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 2 job
+
+test_expect_success 'flux-shell: bails on invalid per-resource' '
+	test_expect_code 1 flux run -o per-resource.type=foo hostname
+'
+test_expect_success 'flux-shell: bails on invalid per-resource count' '
+	test_expect_code 1 flux run -o per-resource.count=0 hostname
+'
+test_expect_success 'flux-shell: bails on invalid per-resource object' '
+	test_expect_code 1 flux run -o per-resource.blah hostname
+'
+test_expect_success 'flux-shell: bails on invalid batch object' '
+	test_expect_code 1 flux run --setattr system.batch.broker-opts=5 \
+		hostname
+'
+test_expect_success 'flux-shell: per-resource with count works' '
+	flux run \
+	        -o per-resource.type=core \
+	        -o per-resource.count=4 \
+	        echo foo > 4-per-core.out 2>4-per-core.err &&
+	test_debug "grep . 4-per-core.*" &&
+	cat <<-EOF >4-per-core.expected &&
+	foo
+	foo
+	foo
+	foo
+	EOF
+	test_cmp 4-per-core.expected 4-per-core.out
+'
+test_expect_success 'flux-shell: per-resource type=node works' '
+	ncores=$(flux resource list -s up -no {ncores}) &&
+	flux run -n ${ncores} \
+	        -o per-resource.type=node \
+	        -o per-resource.count=1 \
+	        echo foo > per-node.out 2>per-node.err &&
+	test_debug "grep . per-node.*" &&
+	cat <<-EOF >per-node.expected &&
+	foo
+	foo
+	EOF
+	test_cmp per-node.expected per-node.out
+'
+test_expect_success 'flux-shell: historical batch jobspec still work' '
+	for spec in $SHARNESS_TEST_SRCDIR/batch/jobspec/*.json; do
+		input=$(basename $spec) &&
+		cat $spec |
+		    jq -S ".attributes.system.environment.PATH=\"$PATH\"" |
+		    jq -S ".attributes.system.environment.PYTHONPATH=\"$PYTHONPATH\"" |
+		    jq -S ".attributes.system.environment.HOME=\"$HOME\"" |
+		    jq -S ".attributes.system.cwd=\"$(pwd)\"" \
+		    >$input &&
+		flux job submit --flags=waitable $input
+	done &&
+	flux job attach -vEX $(flux job last) &&
+	flux job wait --all --verbose
+'
+test_done
diff --git a/t/t2614-job-shell-doom.t b/t/t2614-job-shell-doom.t
new file mode 100755
index 000000000000..ef6842f11f95
--- /dev/null
+++ b/t/t2614-job-shell-doom.t
@@ -0,0 +1,78 @@
+#!/bin/sh
+#
+test_description='Test flux-shell task exit support'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 2 job
+
+test_expect_success 'flux-shell: first task exit posts shell.task-exit event' '
+	jobid=$(flux submit true) &&
+	run_timeout 10 flux job wait-event -p exec \
+		$jobid shell.task-exit
+'
+
+test_expect_success 'flux-shell: create 30s sleep script - rank 1 exits early' '
+	cat >testscript.sh <<-EOT &&
+	#!/bin/bash
+	test \$FLUX_TASK_RANK -eq 1 && exit 200
+	sleep 30
+	EOT
+	chmod +x testscript.sh
+'
+
+test_expect_success 'flux-shell: run script with 2 tasks and 1s timeout' '
+	test_must_fail run_timeout 30 flux run \
+		-n2 -o exit-timeout=1s ./testscript.sh 2>tmout.err &&
+	grep "exception.*timeout" tmout.err
+'
+
+test_expect_success 'flux-shell: run script with 2 nodes and 1s timeout' '
+	test_must_fail run_timeout 30 flux run \
+		-n2 -N2 -o exit-timeout=1s ./testscript.sh 2>tmout2.err &&
+	grep "exception.*timeout" tmout2.err
+'
+
+test_expect_success 'flux-shell: run script with 2 tasks and exit-on-error' '
+	test_must_fail run_timeout 30 flux run \
+		-n2 -o exit-on-error ./testscript.sh
+'
+
+test_expect_success 'flux-shell: run script with 2 nodes and exit-on-error' '
+	test_must_fail run_timeout 30 flux run \
+		-n2 -N2 -o exit-on-error ./testscript.sh
+'
+test_expect_success 'flux-shell: exit-timeout catches lost shell' '
+	cat >test2.sh <<-"EOF" &&
+	#!/bin/bash
+	if test $FLUX_TASK_RANK -eq 1; then
+	    kill -9 $PPID
+	    exit
+	fi
+	sleep 30
+	EOF
+	chmod +x test2.sh &&
+	test_must_fail run_timeout 30 flux run \
+		-n2 -N2 -o exit-timeout=1s ./test2.sh
+'
+test_expect_success 'flux-shell: exit-on-error catches lost shell' '
+	test_must_fail run_timeout 30 flux run \
+		-n2 -N2 -o exit-on-error ./test2.sh
+'
+test_expect_success 'flux-shell: exit-timeout=aaa is rejected' '
+	test_must_fail flux run -o exit-timeout=aaa true
+'
+test_expect_success 'flux-shell: exit-timeout=false is rejected' '
+	test_must_fail flux run -o exit-timeout=false true
+'
+test_expect_success 'flux-shell: exit-timeout=none is accepted' '
+	flux run -o exit-timeout=none true
+'
+test_expect_success 'flux-shell: exit-timeout=100 is accepted' '
+	flux run -o exit-timeout=100 true
+'
+test_expect_success 'flux-shell: exit-timeout=42.34 is accepted' '
+	flux run -o exit-timeout=42.34 true
+'
+
+test_done
diff --git a/t/t2615-job-shell-rlimit.t b/t/t2615-job-shell-rlimit.t
new file mode 100755
index 000000000000..df98f31feb0c
--- /dev/null
+++ b/t/t2615-job-shell-rlimit.t
@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+test_description='Test flux-shell rlimit support'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 1 job
+
+test_expect_success 'flux-shell: propagates rlimits' '
+	( ulimit -n 123 &&
+	  flux run sh -c "ulimit -n" >ulimit-n.out
+	) &&
+	test_debug "cat ulimit-n.out" &&
+	test "$(cat ulimit-n.out)" = "123"
+'
+test_expect_success 'flux-shell: works when no rlimits propagated' '
+	flux run --rlimit=-* hostname
+'
+test_expect_success 'flux-shell: works when all rlimits propagated' '
+	flux run --rlimit=* sh -c "ulimit -a"
+'
+test_expect_success 'flux-shell: works when no specific rlimit propagated' '
+	flux run --rlimit=nofile=123 sh -c "ulimit -n" >ulimit-n2.out &&
+	test_debug "cat ulimit-n2.out" &&
+	test "$(cat ulimit-n.out)" = "123"
+'
+test_expect_success 'flux-shell: nonfatal rlimit errors are logged' '
+	flux run --output=nofile.out --rlimit nofile=inf true &&
+	grep "nofile exceeds current max"  nofile.out
+'
+test_expect_success 'flux-shell: invalid rlimit option is fatal error' '
+	test_must_fail flux run -o rlimit.foo=1234 true
+'
+test_done
diff --git a/t/t2616-job-shell-taskmap-hostfile.t b/t/t2616-job-shell-taskmap-hostfile.t
new file mode 100755
index 000000000000..4491605f1585
--- /dev/null
+++ b/t/t2616-job-shell-taskmap-hostfile.t
@@ -0,0 +1,107 @@
+#!/bin/sh
+#
+test_description='Test hostfile taskmap plugin support'
+
+. `dirname $0`/sharness.sh
+
+# Use "system" personality to get fake hostnames for hostfile use
+test_under_flux 4 system
+
+# Test that actual task ranks match expected ranks.
+# Assumes job output is `echo $FLUX_TASK_RANK: $(flux getattr rank)`
+test_check_taskmap() {
+	local id=$1
+	flux job attach $id | sort -n >$id.output &&
+	flux job taskmap --to=multiline $id >$id.expected &&
+	test_cmp $id.expected $id.output
+}
+
+test_expect_success 'create script for testing task mapping' '
+	cat <<-EOF >map.sh &&
+	#!/bin/sh
+	echo \$FLUX_TASK_RANK: \$(flux getattr rank)
+	EOF
+	chmod +x map.sh
+'
+test_expect_success 'taskmap=hostfile works' '
+	cat <<-EOF >h1 &&
+	fake3
+	fake2
+	fake1
+	fake0
+	EOF
+	expected="[[3,1,1,1],[2,1,1,1],[1,1,1,1],[0,1,1,1]]" &&
+	id=$(flux submit --taskmap=hostfile:h1 -N4 -n4 ./map.sh) &&
+	flux job attach -vEX $id &&
+	flux job wait-event -p exec -f json $id shell.start &&
+        flux job wait-event -p exec -f json $id shell.start \
+                | jq -e ".context.taskmap.map == $expected" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap=hostfile works with multiple tasks per node' '
+	cat <<-EOF >h2 &&
+	fake3
+	fake3
+	fake2
+	fake2
+	fake1
+	fake1
+	fake0
+	fake0
+	EOF
+	expected="[[3,1,2,1],[2,1,2,1],[1,1,2,1],[0,1,2,1]]" &&
+	id=$(flux submit --taskmap=hostfile:h2 -N4 --tasks-per-node=2 ./map.sh) &&
+	flux job attach -vEX $id &&
+	flux job wait-event -p exec -f json $id shell.start &&
+        flux job wait-event -p exec -f json $id shell.start \
+                | jq -e ".context.taskmap.map == $expected" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap=hostfile reuses hosts in short hostlist' '
+	cat <<-EOF >h3 &&
+	fake3
+	fake2
+	fake1
+	fake0
+	EOF
+	expected="[[3,1,1,1],[2,1,1,1],[1,1,1,1],[0,1,1,1],[3,1,1,1],[2,1,1,1],[1,1,1,1],[0,1,1,1]]" &&
+	id=$(flux submit --taskmap=hostfile:h3 -N4 --tasks-per-node=2 ./map.sh) &&
+	flux job attach -vEX $id &&
+	flux job wait-event -p exec -f json $id shell.start &&
+        flux job wait-event -p exec -f json $id shell.start \
+                | jq -e ".context.taskmap.map == $expected" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap=hostfile works with hostlists' '
+	cat <<-EOF >h4 &&
+	fake[1,2]
+	fake[3,0]
+	EOF
+	expected="[[1,3,1,1],[0,4,1,1],[0,1,1,1]]" &&
+	id=$(flux submit --taskmap=hostfile:h4 -N4 --tasks-per-node=2 ./map.sh) &&
+	flux job attach -vEX $id &&
+	flux job wait-event -p exec -f json $id shell.start &&
+        flux job wait-event -p exec -f json $id shell.start \
+                | jq -e ".context.taskmap.map == $expected" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap=hostfile fails with invalid hostlist' '
+	echo "fake[0-3">h5 &&
+	test_must_fail_or_be_terminated \
+		flux run --taskmap=hostfile:h5 -N4 hostname
+'
+test_expect_success 'taskmap=hostfile fails with incorrect hosts' '
+	echo "foo[0-3]">h6 &&
+	test_must_fail_or_be_terminated \
+		flux run --taskmap=hostfile:h6 -N4 hostname
+'
+test_expect_success 'taskmap=hostfile fails when not all hosts present' '
+	echo "foo[0,0,1,2]">h7 &&
+	test_must_fail_or_be_terminated \
+		flux run --taskmap=hostfile:h7 -N4 hostname
+'
+test_expect_success 'taskmap=hostfile fails with invalid filename' '
+	test_must_fail_or_be_terminated \
+		flux run --taskmap=hostfile:badfile -N4 hostname
+'
+test_done
diff --git a/t/t2616-job-shell-taskmap.t b/t/t2616-job-shell-taskmap.t
new file mode 100755
index 000000000000..113418140907
--- /dev/null
+++ b/t/t2616-job-shell-taskmap.t
@@ -0,0 +1,189 @@
+#!/bin/sh
+#
+test_description='Test flux-submit/flux-shell taskmap plugin support'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 4 job
+
+# Test that actual task ranks match expected ranks.
+# Assumes job output is `echo $FLUX_TASK_RANK: $(flux getattr rank)`
+test_check_taskmap() {
+	local id=$1
+	flux job attach $id | sort -n >$id.output &&
+	flux job taskmap --to=multiline $id >$id.expected &&
+	test_cmp $id.expected $id.output
+}
+
+test_expect_success 'create script for testing task mapping' '
+	cat <<-EOF >map.sh &&
+	#!/bin/sh
+	echo \$FLUX_TASK_RANK: \$(flux getattr rank)
+	EOF
+	chmod +x map.sh
+'
+test_expect_success 'flux job taskmap works' '
+	id=$(flux submit -N4 --tasks-per-node=4 ./map.sh) &&
+	test_must_fail flux job taskmap &&
+	test "$(flux job taskmap $id)" = "[[0,4,4,1]]" &&
+	test "$(flux job taskmap --taskids=0 $id)" = "0-3" &&
+	test "$(flux job taskmap --ntasks=0 $id)" = "4" &&
+	test "$(flux job taskmap --nodeid=15 $id)" = "3" &&
+	test "$(flux job taskmap --hostname=0 $id)" = "$(hostname)" &&
+	test "$(flux job taskmap --to=pmi $id)" = "(vector,(0,4,4))" &&
+	flux job taskmap --to=hosts $id > taskmap.hosts &&
+	cat <<-EOF >taskmap.hosts.expected &&
+	$(hostname): 0-3
+	$(hostname): 4-7
+	$(hostname): 8-11
+	$(hostname): 12-15
+	EOF
+	test_cmp taskmap.hosts.expected taskmap.hosts
+'
+test_expect_success 'flux job taskmap works with taskmap on cmdline' '
+	flux job taskmap --to=raw "[[0,4,4,1]]" &&
+	flux job taskmap --to=raw "[[0,4,1,4]]"
+'
+test_expect_success 'flux job taskmap fails with invalid taskmap on cmdline' '
+	test_must_fail flux job taskmap "[[0,4,4]]"
+'
+test_expect_success 'flux job taskmap fails for canceled job' '
+	flux queue stop &&
+	id=$(flux submit -N4 true) &&
+	flux cancel $id &&
+	flux queue start &&
+	test_must_fail flux job taskmap $id
+'
+test_expect_success 'flux job taskmap fails for invalid job' '
+	test_must_fail flux job taskmap f1
+'
+test_expect_success 'flux job taskmap fails with invalid arguments' '
+	id=$(flux submit -N4 --tasks-per-node=4 ./map.sh) &&
+	flux job taskmap $id &&
+	test_must_fail flux job taskmap --taskids=4 $id 2>no-taskids.err &&
+	test_debug "cat no-taskids.err" &&
+	grep "No taskids for node 4" no-taskids.err &&
+	test_must_fail flux job taskmap --ntasks=4 $id  2>no-ntasks.err &&
+	test_debug "cat no-ntasks.err" &&
+	grep "failed to get task count for node 4" no-ntasks.err &&
+	test_must_fail flux job taskmap --nodeid=24 $id 2>no-nodeid.err &&
+	test_debug "cat no-nodeid.err" &&
+	grep "failed to get nodeid for task 24" no-nodeid.err &&
+	test_must_fail flux job taskmap --to=foo $id 2>bad-to.err &&
+	test_debug "cat bad-to.err" &&
+	grep "invalid value" bad-to.err &&
+	test_must_fail flux job taskmap --hostname=0 "[[0,1,1,1]]" 2>no-hosts.err &&
+	grep "without a jobid" no-hosts.err &&
+	test_must_fail flux job taskmap --to=hosts "[[0,1,1,1]]" 2>no-hosts2.err &&
+	grep "without a jobid" no-hosts2.err
+'
+test_expect_success 'taskmap is posted in shell.start event' '
+	id=$(flux submit ./map.sh) &&
+	flux job wait-event -p exec -f json $id shell.start \
+		| jq -e ".context.taskmap.map == [[0,1,1,1]]" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap is correct for --tasks-per-node=4' '
+	id=$(flux submit -N4 --tasks-per-node=4 ./map.sh) &&
+	flux job wait-event -p exec -f json $id shell.start \
+		| jq -e ".context.taskmap.map == [[0,4,4,1]]" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap is unchanged with --taskmap=block' '
+	id=$(flux submit -N4 --tasks-per-node=4 \
+		--taskmap=block ./map.sh) &&
+	flux job wait-event -p exec -f json $id shell.start \
+		| jq -e ".context.taskmap.map == [[0,4,4,1]]" &&
+	test_check_taskmap $id
+'
+test_expect_success 'shell dies with --taskmap=block:oops' '
+	test_must_fail_or_be_terminated flux run -N4 --tasks-per-node=4 \
+		--taskmap=block:oops ./map.sh
+'
+test_expect_success 'taskmap is correct for --tasks-per-node=2' '
+	id=$(flux submit -N4 --tasks-per-node=2 ./map.sh) &&
+	flux job wait-event -p exec -f json $id shell.start &&
+	flux job wait-event -p exec -f json $id shell.start \
+		| jq -e ".context.taskmap.map == [[0,4,2,1]]" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap=cyclic works with valid args' '
+	id=$(flux submit --taskmap=cyclic -N4 --tasks-per-node=4 ./map.sh) &&
+	flux job wait-event -p exec -f json $id shell.start &&
+	flux job wait-event -p exec -f json $id shell.start \
+		| jq -e ".context.taskmap.map == [[0,4,1,4]]" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap=cyclic:2 works' '
+	id=$(flux submit --taskmap=cyclic:2 -N4 --tasks-per-node=4 ./map.sh) &&
+	flux job wait-event -p exec -f json $id shell.start &&
+	flux job wait-event -p exec -f json $id shell.start \
+		| jq -e ".context.taskmap.map == [[0,4,2,2]]" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap=cyclic:3 works' '
+	id=$(flux submit --taskmap=cyclic:3 -N4 --tasks-per-node=4 ./map.sh) &&
+	flux job wait-event -p exec -f json $id shell.start &&
+	flux job wait-event -p exec -f json $id shell.start \
+		| jq -e ".context.taskmap.map == [[0,4,3,1],[0,4,1,1]]" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap=manual works' '
+	id=$(flux submit \
+	     --taskmap=manual:"[[1,3,4,1],[0,1,4,1]]" \
+	     -N4 --tasks-per-node=4 ./map.sh) &&
+	flux job wait-event -p exec -f json $id shell.start \
+		| jq -e ".context.taskmap.map == [[1,3,4,1],[0,1,4,1]]" &&
+	test_check_taskmap $id
+'
+test_expect_success 'taskmap=manual with unknown taskmap fails' '
+	test_must_fail_or_be_terminated \
+		flux run --taskmap="manual:[]" true
+'
+test_expect_success 'invalid taskmap causes job failure' '
+	test_must_fail_or_be_terminated \
+		flux run -vvv --taskmap=foo true
+'
+test_expect_success 'invalid manual taskmaps fail predictably' '
+	ids=$(flux bulksubmit --taskmap=manual:{} \
+		-N4 --tasks-per-node=4 true ::: \
+		"{}" \
+		"[[1,3,4,1]]" \
+		"[[1,3,4,1],[0,1,3,1]]" \
+		"[[3,1,1,1],[0,3,5,1]]") &&
+	for id in $ids; do
+		test_must_fail_or_be_terminated flux job attach $id
+	done
+'
+test_expect_success 'invalid taskmap shell option fails' '
+	test_must_fail_or_be_terminated \
+		flux run -o taskmap.foo=bar true
+'
+test_expect_success 'invalid taskmap scheme causes job failure' '
+	test_must_fail_or_be_terminated \
+		flux run -vvv --taskmap=foo true
+'
+test_expect_success 'invalid cyclic value causes job failure' '
+	test_must_fail_or_be_terminated \
+		flux run -vvv --taskmap=cyclic:0 true
+'
+test_expect_success 'invalid cyclic value causes job failure' '
+	test_must_fail_or_be_terminated \
+		flux run -vvv --taskmap=cyclic:foo true
+'
+
+INITRC_PLUGINPATH="${SHARNESS_TEST_DIRECTORY}/shell/plugins/.libs"
+test_expect_success 'shell supports dynamically loaded taskmap plugin' '
+	cat <<-EOF >taskmap-initrc.lua &&
+	plugin.searchpath = "${INITRC_PLUGINPATH}"
+	plugin.load { file = "taskmap-reverse.so" }
+	EOF
+	id=$(flux submit -o userrc=taskmap-initrc.lua \
+		-N4 --tasks-per-node=4 \
+		--taskmap=reverse ./map.sh) &&
+	flux job taskmap $id  &&
+	test $(flux job taskmap $id) \
+		= "[[3,1,4,1],[2,1,4,1],[1,1,4,1],[0,1,4,1]]" && \
+	test_check_taskmap $id
+'
+test_done
diff --git a/t/t2617-job-shell-stage-in.t b/t/t2617-job-shell-stage-in.t
new file mode 100755
index 000000000000..495221ed3a59
--- /dev/null
+++ b/t/t2617-job-shell-stage-in.t
@@ -0,0 +1,150 @@
+#!/bin/sh
+#
+test_description='Test flux-shell stage-in plugin support'
+
+. `dirname $0`/sharness.sh
+
+lptest="flux lptest"
+
+test_under_flux 4 job
+
+test_expect_success 'archive the red test files' '
+	mkdir red &&
+	touch red/empty &&
+	truncate --size 8192 red/holy &&
+	$lptest >red/lptest &&
+	echo foo >red/small &&
+	mkdir red/dir &&
+	ln -s dir red/link &&
+	flux archive create -v --name=red red
+'
+test_expect_success 'archive the blue test files' '
+	dir=blue/a/b/c/d/e/f/g/h &&
+	mkdir -p $dir &&
+	echo bar >$dir/test &&
+	flux archive create -v --name=blue blue
+'
+test_expect_success 'archive the main test files' '
+	mkdir main &&
+	echo "Hello world!" >main/hello &&
+	flux archive create -v main
+'
+test_expect_success 'list all the files' '
+	flux archive extract --list-only -v --name=red &&
+	flux archive extract --list-only -v --name=blue &&
+	flux archive extract --list-only -v
+'
+test_expect_success 'create file tree checker script' '
+	cat >check.sh <<-EOT &&
+	#!/bin/sh
+	for dir in \$*; do
+	    diff -r --no-dereference \$FLUX_JOB_TMPDIR/\$dir \$dir || exit 1
+	done
+	EOT
+	chmod 755 check.sh
+'
+test_expect_success 'verify that stage-in works with default key (main)' '
+	flux run -N4 -ostage-in ./check.sh main
+'
+test_expect_success 'verify that stage-in works with names=red' '
+	flux run -N2 -ostage-in.names=red ./check.sh red
+'
+test_expect_success 'verify that stage-in works with names=red,blue' '
+	flux run -N1 -ostage-in.names=red,blue ./check.sh red blue
+'
+test_expect_success 'verify that stage-in works with deprecated tags option' '
+	flux run -N1 -ostage-in.tags=red,blue ./check.sh red blue 2>depr.err
+'
+test_expect_success 'and deprecation warning was printed' '
+	grep deprecated depr.err
+'
+test_expect_success 'verify that stage-in fails with unknown option' '
+	test_must_fail flux run -N1 -ostage-in.badopt true
+'
+test_expect_success 'verify that stage-in fails with unknown archive name' '
+	test_must_fail flux run -N1 -ostage-in.names=badarchive true
+'
+test_expect_success 'verify that stage-in.pattern works' '
+	flux run -N1 \
+	    -ostage-in.names=red,blue \
+	    -ostage-in.pattern=red/* \
+	    -overbose=2 \
+            ./check.sh red 2>pattern.err
+'
+test_expect_success 'files that did not match pattern were not copied' '
+	grep red/small pattern.err &&
+	test_must_fail grep blue/a pattern.err
+'
+test_expect_success 'verify that stage-in.destination works' '
+	mkdir testdest &&
+	flux run -N1 \
+	    -o stage-in.destination=$(pwd)/testdest \
+	    true &&
+	test -f testdest/main/hello
+'
+test_expect_success 'verify that stage-in.destination=local:path works' '
+	rm -rf testdest/main &&
+	flux run -N1 \
+	    -o stage-in.destination=local:$(pwd)/testdest \
+	    true &&
+	test -f testdest/main/hello
+'
+test_expect_success 'verify that stage-in.destination=global:path works' '
+	rm -rf testdest/main &&
+	flux run -N2 \
+	    -o stage-in.destination=global:$(pwd)/testdest \
+	    true &&
+	test -f testdest/main/hello
+'
+test_expect_success 'verify that stage-in.destination fails on bad dir' '
+	test_must_fail flux run -N1 \
+	    -o stage-in.destination=/noexist \
+	    true
+'
+test_expect_success 'verify that stage-in.destination fails on bad prefix' '
+	rm -rf testdest/main &&
+	test_must_fail flux run -N1 \
+	    -o stage-in.destination=wrong:$(pwd)/testdest \
+	    true
+'
+test_expect_success 'remove archives' '
+	flux archive remove &&
+	flux archive remove --name=blue &&
+	flux archive remove --name=red
+'
+test_expect_success 'create a test file with random content' '
+        dd if=/dev/urandom of=foo bs=4096 count=1 conv=notrunc
+'
+test_expect_success 'map test file and access it to prime the cache' '
+	mkdir -p copydir &&
+        flux archive create --mmap ./foo &&
+	flux archive extract -C copydir &&
+	cmp foo copydir/foo
+'
+test_expect_success 'modify mapped test file without reducing its size' '
+        dd if=/dev/zero of=foo bs=4096 count=1 conv=notrunc
+'
+test_expect_success 'content change should cause an error' '
+        test_must_fail flux run -N1 -o stage-in true 2>changed.err &&
+	grep changed changed.err
+'
+test_expect_success 'Create files for example 2 of flux-archive(1)' '
+	mkdir -p project/dataset1 &&
+	mkdir -p home/fred &&
+	$lptest >project/dataset1/lptest &&
+	cat >home/fred/app <<-EOT &&
+	#!/bin/sh
+	find \$1 -type f
+	EOT
+	chmod +x home/fred/app
+'
+test_expect_success 'Verify that example 2 of flux-archive(1) works' '
+	flux archive create --name=dataset1 -C project dataset1 &&
+	flux archive create --name=app --mmap -C home/fred app &&
+	flux run -N4 -o stage-in.names=app,dataset1 \
+	    {{tmpdir}}/app {{tmpdir}}/dataset1 &&
+	flux archive remove --name=dataset1 &&
+	flux archive remove --name=app
+'
+
+test_done
diff --git a/t/t2618-job-shell-signal.t b/t/t2618-job-shell-signal.t
new file mode 100755
index 000000000000..4a9e9de1cebe
--- /dev/null
+++ b/t/t2618-job-shell-signal.t
@@ -0,0 +1,82 @@
+#!/bin/sh
+#
+test_description='Test flux-shell signal before timelimit support'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 2 job
+
+check_signal()
+{
+	local spec=$1 &&
+	local sig=${2:-10} &&
+	local tleft=${3:-60} &&
+	name=test.${spec} &&
+	attr=.attributes.system.shell.options.signal &&
+	flux run --dry-run --signal=$spec hostname >$name.json 2>$name.err &&
+	test_debug "jq $attr < $name.json" &&
+	jq -e "$attr | .signum == $sig and .timeleft == $tleft" < $name.json
+}
+
+# Assumptions:
+# SIGUSR1 == 10
+# SIGUSR2 == 12
+test_expect_success 'cli: check_signal() test function works' '
+	test_expect_code 1 check_signal SIGFOO &&
+	test_expect_code 1 check_signal USR2 &&
+	test_expect_code 1 check_signal @2m
+'
+test_expect_success 'cli: --signal option works' '
+	check_signal @ 10 60 &&
+	check_signal 15 15 60 &&
+	check_signal USR2 12 60 &&
+	check_signal SIGUSR2 12 60 &&
+	check_signal @5s 10 5 &&
+	check_signal @10m 10 600 &&
+	check_signal USR2@10m 12 600
+'
+test_expect_success 'cli: --signal option handles invalid args' '
+	test_expect_code 1 check_signal SIG &&
+	grep "signal.*is invalid" test.SIG.err &&
+	test_expect_code 1 check_signal @3y &&
+	grep "invalid Flux standard duration" test.@3y.err &&
+	test_expect_code 1 check_signal -1 &&
+	grep "signal must be > 0" test.-1.err
+'
+test_fails_by_signal() {
+	local signum=$1
+	shift
+	"$@"
+	exit_code=$?
+	if test $exit_code = $((128+signum)); then
+		return 0
+	elif test $exit_code = 0; then
+		echo >&2 "test_fails_by_signal: command succeeded: $*"
+	elif test $exit_code -gt 129 -a $exit_code -le 192; then
+		sig=$((exit_code+signum))
+		echo >&2 "test_fails_by_signal: got unexpected signal $sig: $*"
+	elif test $exit_code = 127; then
+		echo >&2 "test_fails_by_signal: command not found: $*"
+	else
+		echo >&2 "test_fails_by_signal: exited with code $exit_code: $*"
+	fi
+	return 1
+}
+
+test_expect_success 'shell: signal option works' '
+	test_fails_by_signal 10 flux run -t 5s --signal=@4.5s sleep 15
+'
+test_expect_success 'shell: signal option works with alternate signal' '
+	test_fails_by_signal 12 flux run -t 5s --signal=USR2@4.5s sleep 15
+'
+test_expect_success 'shell: signal shell option can be an integer' '
+	test_fails_by_signal 10 flux run -t 5s -o signal -o verbose sleep 10 \
+		2>sig.log &&
+	grep "will expire in 60.0s" sig.log
+'
+test_expect_success 'shell: signal option is ignored when --time-limit not set' '
+	flux run --signal=@60 -o verbose sleep 0.5 >no-time-limit.out 2>&1 &&
+	test_debug "cat no-time-limit.out" &&
+	test_must_fail grep "Will send SIGUSR1 to job" no-time-limit.out
+'
+test_done
diff --git a/t/t2619-job-shell-hwloc.t b/t/t2619-job-shell-hwloc.t
new file mode 100755
index 000000000000..855739530749
--- /dev/null
+++ b/t/t2619-job-shell-hwloc.t
@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+test_description='Test flux-shell hwloc.xmlfile'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 2 job
+unset HWLOC_XMLFILE
+unset FLUX_HWLOC_XMLFILE
+
+test_expect_success 'shell: HWLOC_XMLFILE is not set for normal jobs' '
+	test_must_fail flux run printenv HWLOC_XMLFILE
+'
+test_expect_success 'shell: -o hwloc.xmlfile sets HWLOC_XMLFILE' '
+	flux run -o hwloc.xmlfile printenv HWLOC_XMLFILE
+'
+test_expect_success 'shell: -o hwloc.xmlfile unsets HWLOC_COMPONENTS' '
+	test_must_fail flux run --env=HWLOC_COMPONENTS=x86 -o hwloc.xmlfile \
+		printenv HWLOC_COMPONENTS
+'
+command -v hwloc-info >/dev/null && test_set_prereq HWLOC_INFO
+test_expect_success HWLOC_INFO 'shell: -o hwloc.xmlfile HWLOC_XMLFILE is valid' '
+	flux run -o hwloc.xmlfile sh -c "hwloc-info --input \$HWLOC_XMLFILE"
+'
+test_expect_success 'shell: -o hwloc.xmlfile sets HWLOC_XMLFILE per node' '
+	flux run -N2 --label-io -o hwloc.xmlfile printenv HWLOC_XMLFILE \
+		> xmlfile.out &&
+	test_debug "cat xmlfile.out" &&
+	sed -n "s/^0://p" xmlfile.out >path.0 &&
+	sed -n "s/^1://p" xmlfile.out >path.1 &&
+	test_must_fail test_cmp path.0 path.1
+'
+test_expect_success 'shell: bad hwloc.xmlfile value is an error' '
+	test_must_fail flux run -o hwloc.xmlfile=foo printenv HWLOC_XMLFILE
+'
+if which hwloc-bind > /dev/null; then
+        NCORES=$(hwloc-bind --get | hwloc-calc --number-of core | tail -n 1)
+        test $NCORES = 1 || test_set_prereq MULTICORE
+fi
+test_expect_success MULTICORE 'shell: -o hwloc.restrict restricts hwloc XML' '
+	flux run -n1 -o hwloc.xmlfile -o hwloc.restrict \
+		hwloc-calc --number-of core all &&
+	test $(flux run -n1 -o hwloc.xmlfile -o hwloc.restrict \
+		hwloc-calc --number-of core all) -eq 1
+'
+test_done
diff --git a/t/t2700-mini-cmd.t b/t/t2700-mini-cmd.t
deleted file mode 100755
index 140a0aeccd9e..000000000000
--- a/t/t2700-mini-cmd.t
+++ /dev/null
@@ -1,182 +0,0 @@
-#!/bin/sh
-
-test_description='Test flux mini command'
-
-. $(dirname $0)/sharness.sh
-
-test_under_flux 4
-
-jq=$(which jq 2>/dev/null)
-test -z "$jq" || test_set_prereq HAVE_JQ
-test $(nproc) -gt 1 && test_set_prereq HAVE_MULTICORE
-
-# Set CLIMain log level to logging.DEBUG (10), to enable stack traces
-export FLUX_PYCLI_LOGLEVEL=10
-
-flux setattr log-stderr-level 1
-
-test_expect_success 'flux mini fails with usage message' '
-	test_must_fail flux mini 2>usage.err &&
-	grep -i usage: usage.err
-'
-test_expect_success 'flux mini submit + flux job attach works' '
-	jobid=$(flux mini submit hostname) &&
-	flux job attach $jobid
-'
-test_expect_success 'flux mini run works' '
-	flux mini run hostname >run.out &&
-	hostname >run.exp &&
-	test_cmp run.exp run.out
-'
-test_expect_success 'flux mini run --ntasks=2 works' '
-	flux mini run --ntasks=2 hostname >run2.out &&
-	(hostname;hostname) >run2.exp &&
-	test_cmp run2.exp run2.out
-'
-test_expect_success 'flux mini run --ntasks=2 --label-io works' '
-	flux mini run --ntasks=2 --label-io echo Hi |sort >run2l.out &&
-	cat >run2l.exp <<-EOT &&
-	0: Hi
-	1: Hi
-	EOT
-	test_cmp run2l.exp run2l.out
-'
-test_expect_success HAVE_MULTICORE 'flux mini submit --ntasks=2 --cores-per-task=2 works' '
-	jobid=$(flux mini submit --ntasks=2 --cores-per-task=2 hostname) &&
-	flux job attach $jobid
-'
-test_expect_success 'flux mini run --ntasks=2 --nodes=2 works' '
-	flux mini run --ntasks=2 --nodes=2 hostname
-'
-test_expect_success 'flux mini run --ntasks=1 --nodes=2 fails' '
-	test_must_fail flux mini run --ntasks=1 --nodes=2 hostname \
-		2>run1n2N.err &&
-	grep -i "node count must not be greater than task count" run1n2N.err
-'
-test_expect_success 'flux mini run (default ntasks) --nodes=2 fails' '
-	test_must_fail flux mini run --nodes=2 hostname 2>run2N.err &&
-	grep -i "node count must not be greater than task count" run2N.err
-'
-test_expect_success 'flux mini submit --priority=6 works' '
-	jobid=$(flux mini submit --priority=6 hostname) &&
-	flux job eventlog $jobid | grep submit | grep priority=6
-'
-test_expect_success 'flux mini submit --flags debug works' '
-	jobid=$(flux mini submit --flags debug hostname) &&
-	flux job eventlog $jobid | grep submit | grep flags=2
-'
-test_expect_success 'flux mini submit --flags waitable works' '
-	jobid=$(flux mini submit --flags waitable hostname) &&
-	flux job eventlog $jobid | grep submit | grep flags=4
-'
-test_expect_success 'flux mini submit --flags debug,waitable works' '
-	jobid=$(flux mini submit --flags debug,waitable hostname) &&
-	flux job eventlog $jobid | grep submit | grep flags=6
-'
-test_expect_success 'flux mini run -v produces jobid on stderr' '
-	flux mini run -v hostname 2>v.err &&
-	grep jobid: v.err
-'
-test_expect_success 'flux mini run -vv produces job events on stderr' '
-	flux mini run -vv hostname 2>vv.err &&
-	grep submit vv.err
-'
-test_expect_success 'flux mini run -vvv produces exec events on stderr' '
-	flux mini run -vvv hostname 2>vvv.err &&
-	grep complete vvv.err
-'
-test_expect_success HAVE_JQ 'flux mini submit --time-limit=5d works' '
-	flux mini submit --dry-run --time-limit=5d hostname >t5d.out &&
-	test $(jq ".attributes.system.duration" t5d.out) = "432000"
-'
-test_expect_success HAVE_JQ 'flux mini submit --time-limit=4h works' '
-	flux mini submit --dry-run --time-limit=4h hostname >t4h.out &&
-	test $(jq ".attributes.system.duration" t4h.out) = "14400"
-'
-test_expect_success HAVE_JQ 'flux mini submit --time-limit=1m works' '
-	flux mini submit --dry-run --time-limit=5m hostname >t5m.out &&
-	test $(jq ".attributes.system.duration" t5m.out) = "300"
-'
-test_expect_success HAVE_JQ 'flux mini submit -t5s works' '
-	flux mini submit --dry-run -t5s hostname >t5s.out &&
-	test $(jq ".attributes.system.duration" t5s.out) = "5"
-'
-test_expect_success HAVE_JQ 'flux mini submit -t5 works' '
-	flux mini submit --dry-run -t5 hostname >t5.out &&
-	test $(jq ".attributes.system.duration" t5.out) = "5"
-'
-test_expect_success 'flux mini submit --time-limit=00:30 fails' '
-	test_must_fail flux mini submit --time-limit=00:30 hostname 2>st.err &&
-	grep -i "invalid Flux standard duration" st.err
-'
-test_expect_success 'flux mini submit --time-limit=4-00:30:00 fails' '
-	test_must_fail flux mini submit --time-limit=4-00:30:00 hostname 2>st2.err &&
-	grep -i "invalid Flux standard duration" st2.err
-'
-
-test_expect_success HAVE_JQ 'flux mini submit --setattr works' '
-	flux mini submit --dry-run \
-		--setattr user.meep=false \
-		--setattr user.foo=\"xxx\" \
-		--setattr user.foo2=yyy \
-		--setattr system.bar=42 hostname >attr.out &&
-	test $(jq ".attributes.user.meep" attr.out) = "false" &&
-	test $(jq ".attributes.user.foo" attr.out) = "\"xxx\"" &&
-	test $(jq ".attributes.user.foo2" attr.out) = "\"yyy\"" &&
-	test $(jq ".attributes.system.bar" attr.out) = "42"
-'
-test_expect_success 'flux mini submit --setattr fails without value' '
-	test_expect_code 1 \
-		flux mini submit --dry-run \
-		--setattr foo \
-		hostname >attr_fail.out 2>&1 &&
-	test_debug "cat attr_fail.out" &&
-	grep "Missing value for attr foo" attr_fail.out
-'
-test_expect_success HAVE_JQ 'flux mini submit --setopt works' '
-	flux mini submit --dry-run \
-		--setopt foo=true \
-		--setopt baz \
-		--setopt bar.baz=42 hostname >opt.out &&
-	test $(jq ".attributes.system.shell.options.foo" opt.out) = "true" &&
-	test $(jq ".attributes.system.shell.options.bar.baz" opt.out) = "42" &&
-	test $(jq ".attributes.system.shell.options.baz" opt.out) = "1"
-'
-test_expect_success HAVE_JQ 'flux mini submit --output passes through to shell' '
-	flux mini submit --dry-run --output=my/file hostname >output.out &&
-	test $(jq ".attributes.system.shell.options.output.stdout.type" output.out) = "\"file\"" &&
-	test $(jq ".attributes.system.shell.options.output.stdout.path" output.out) = "\"my/file\""
-'
-test_expect_success HAVE_JQ 'flux mini submit --error passes through to shell' '
-	flux mini submit --dry-run --error=/my/error hostname >error.out &&
-	test $(jq ".attributes.system.shell.options.output.stderr.type" error.out) = "\"file\"" &&
-	test $(jq ".attributes.system.shell.options.output.stderr.path" error.out) = "\"/my/error\""
-'
-test_expect_success HAVE_JQ 'flux mini submit --label-io --output passes through to shell' '
-	flux mini submit --dry-run \
-		--label-io --output=foo hostname >labout.out &&
-	test $(jq ".attributes.system.shell.options.output.stdout.label" labout.out) = "true" &&
-	test $(jq ".attributes.system.shell.options.output.stdout.path" labout.out) = "\"foo\""
-'
-test_expect_success HAVE_JQ 'flux mini submit --output id mustache passes through to shell' '
-	flux mini submit --dry-run --output=foo.{{id}} hostname >musid.out &&
-	test $(jq ".attributes.system.shell.options.output.stdout.path" musid.out) = "\"foo.{{id}}\""
-'
-test_expect_success HAVE_JQ 'flux mini submit command arguments work' '
-	flux mini submit --dry-run a b c >args.out &&
-	test $(jq ".tasks[0].command[0]" args.out) = "\"a\"" &&
-	test $(jq ".tasks[0].command[1]" args.out) = "\"b\"" &&
-	test $(jq ".tasks[0].command[2]" args.out) = "\"c\""
-'
-test_expect_success 'flux mini submit --gpus-per-task adds gpus to task slot' '
-	flux mini submit --dry-run -g2 hostname >gpu.out &&
-	test $(jq ".resources[0].with[1].type" gpu.out) = "\"gpu\"" &&
-	test $(jq ".resources[0].with[1].count" gpu.out) = "2"
-'
-test_expect_success HAVE_JQ 'flux mini --job-name works' '
-	flux mini submit --dry-run --job-name=foobar hostname >name.out &&
-	test $(jq ".attributes.system.job.name" name.out) = "\"foobar\""
-'
-
-
-test_done
diff --git a/t/t2710-python-cli-submit.t b/t/t2710-python-cli-submit.t
new file mode 100755
index 000000000000..038cb316ecf9
--- /dev/null
+++ b/t/t2710-python-cli-submit.t
@@ -0,0 +1,358 @@
+#!/bin/sh
+
+test_description='Test flux submit command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4
+
+NCORES=$(flux resource list -no {ncores})
+test ${NCORES} -gt 4 && test_set_prereq MULTICORE
+
+# Set CLIMain log level to logging.DEBUG (10), to enable stack traces
+export FLUX_PYCLI_LOGLEVEL=10
+
+flux setattr log-stderr-level 1
+
+test_expect_success 'flux submit --dry-run works without Flux instance' '
+	FLUX_URI=/no/such/path \
+	    flux submit -n1 --dry-run hostname >test.json
+'
+test_expect_success 'flux submit fails with error message' '
+	test_must_fail flux submit 2>usage.err &&
+	grep "job command and arguments are missing" usage.err
+'
+test_expect_success 'flux submit ignores ambiguous args after --' '
+	flux submit -n2 --dry-run -- hostname --n=2 \
+		| jq -e ".tasks[0].command[0] == \"hostname\""
+'
+test_expect_success 'flux submit + flux job attach works' '
+	jobid=$(flux submit hostname) &&
+	flux job attach $jobid
+'
+test_expect_success MULTICORE 'flux submit --ntasks=2 --cores-per-task=2 works' '
+	jobid=$(flux submit --ntasks=2 --cores-per-task=2 hostname) &&
+	flux job attach $jobid
+'
+test_expect_success 'flux submit --urgency=6 works' '
+	jobid=$(flux submit --urgency=6 hostname) &&
+	flux job eventlog $jobid | grep submit | grep urgency=6
+'
+test_expect_success 'flux submit --urgency special options works' '
+	jobid=$(flux submit --urgency=default hostname) &&
+	flux job eventlog $jobid | grep submit | grep urgency=16 &&
+	jobid=$(flux submit --urgency=hold hostname) &&
+	flux job eventlog $jobid | grep submit | grep urgency=0 &&
+	jobid=$(flux submit --urgency=expedite hostname) &&
+	flux job eventlog $jobid | grep submit | grep urgency=31
+'
+test_expect_success 'flux submit --flags debug works' '
+	jobid=$(flux submit --flags debug hostname) &&
+	flux job eventlog $jobid | grep submit | grep flags=2
+'
+test_expect_success 'flux submit --flags waitable works' '
+	jobid=$(flux submit --flags waitable hostname) &&
+	flux job eventlog $jobid | grep submit | grep flags=4
+'
+test_expect_success 'flux submit --flags debug,waitable works' '
+	jobid=$(flux submit --flags debug,waitable hostname) &&
+	flux job eventlog $jobid | grep submit | grep flags=6
+'
+test_expect_success 'flux submit --flags=novalidate works' '
+	jobid=$(flux submit --flags novalidate true) &&
+	flux job eventlog $jobid | grep submit | grep flags=8
+'
+test_expect_success 'flux submit with bad flags fails' '
+	test_must_fail flux submit --flags notaflag true
+'
+test_expect_success 'flux submit --time-limit=5d works' '
+	flux submit --dry-run --time-limit=5d hostname >t5d.out &&
+	jq -e ".attributes.system.duration == 432000" < t5d.out
+'
+test_expect_success 'flux submit --time-limit=4h works' '
+	flux submit --dry-run --time-limit=4h hostname >t4h.out &&
+	jq -e ".attributes.system.duration == 14400" < t4h.out
+'
+test_expect_success 'flux submit --time-limit=1m works' '
+	flux submit --dry-run --time-limit=5m hostname >t5m.out &&
+	jq -e ".attributes.system.duration == 300" < t5m.out
+'
+test_expect_success 'flux submit -t5s works' '
+	flux submit --dry-run -t5s hostname >t5s.out &&
+	jq -e ".attributes.system.duration == 5" < t5s.out
+'
+test_expect_success 'flux submit -t5 sets 5m duration' '
+	flux submit --dry-run -t5 hostname >t5.out &&
+	jq -e ".attributes.system.duration == 300" < t5.out
+'
+test_expect_success 'flux submit --time-limit=00:30 fails' '
+	test_must_fail flux submit --time-limit=00:30 hostname 2>st.err &&
+	grep -i "invalid Flux standard duration" st.err
+'
+test_expect_success 'flux submit --time-limit=4-00:30:00 fails' '
+	test_must_fail flux submit --time-limit=4-00:30:00 hostname 2>st2.err &&
+	grep -i "invalid Flux standard duration" st2.err
+'
+
+test_expect_success 'flux submit --queue works' '
+	flux submit --env=-* --dry-run --queue=batch hostname >queue.out && grep -i "batch" queue.out
+'
+
+test_expect_success 'flux submit --queue works' '
+	flux submit --env=-* --dry-run -q debug hostname >queue2.out && grep -i "debug" queue2.out
+'
+test_expect_success 'flux submit --bank works' '
+	flux submit --env=-* --dry-run --bank=mybank hostname >bank.out &&
+	grep -i "mybank" bank.out
+'
+test_expect_success 'flux submit -B works' '
+	flux submit --env=-* --dry-run -B mybank2 hostname >bank2.out &&
+	grep -i "mybank2" bank2.out
+'
+test_expect_success 'flux submit -S, --setattr works' '
+	flux submit --env=-* --dry-run \
+		--setattr user.meep=false \
+		--setattr user.foo=\"xxx\" \
+		--setattr user.foo2=yyy \
+		--setattr foo \
+		--setattr .test=a \
+		--setattr test2=b \
+		--setattr system.bar=42 \
+		-S baz \
+		-Sbiz=4 hostname >attr.out &&
+	test_debug "jq .attributes attr.out" &&
+	jq -e ".attributes.user.meep == false" attr.out &&
+	jq -e ".attributes.user.foo == \"xxx\"" attr.out &&
+	jq -e ".attributes.user.foo2 == \"yyy\"" attr.out &&
+	jq -e ".attributes.system.foo == 1" attr.out &&
+	jq -e ".attributes.test == \"a\"" attr.out &&
+	jq -e ".attributes.system.test2 == \"b\"" attr.out &&
+	jq -e ".attributes.system.bar == 42" attr.out &&
+	jq -e ".attributes.system.baz == 1" attr.out &&
+	jq -e ".attributes.system.biz == 4" attr.out
+'
+
+test_expect_success 'flux submit --setattr=^ATTR=VAL works' '
+	cat | jq -S . >attr.json <<-EOF &&
+	[
+	  { "foo":"value",
+	    "bar": 42
+	  },
+	  { "foo":"value2",
+	    "bar": null
+	  }
+	]
+	EOF
+	flux submit --dry-run \
+		--setattr ^user.foo=attr.json \
+		hostname | \
+	    jq -S .attributes.user.foo > attrout.json &&
+	test_cmp attr.json attrout.json
+'
+test_expect_success 'flux submit --setattr=^ detects bad JSON' '
+	cat <<-EOF > bad.json &&
+	[ { "foo":"value",
+	    "bar": 42
+	  },
+	  { foo":"value2",
+	    "bar": null
+	  }
+	]
+	EOF
+	test_expect_code 1 \
+	    flux submit --dry-run \
+	      --setattr ^user.foo=bad.json \
+	      hostname > attrbadjson.out 2>&1 &&
+	test_debug "cat attrbadjson.out" &&
+	grep "ERROR: --setattr: bad.json:" attrbadjson.out &&
+	test_expect_code 1 \
+	    flux submit --dry-run \
+	      --setattr ^user.foo=nosuchfile.json \
+	      hostname > attrbadfile.out 2>&1 &&
+	test_debug "cat attrbadfile.out" &&
+	grep "ERROR:.*nosuchfile" attrbadfile.out
+'
+test_expect_success 'flux submit --setopt works' '
+	flux submit --dry-run \
+		--setopt foo=true \
+		--setopt baz \
+		--setopt bar.baz=42 hostname >opt.out &&
+	test $(jq ".attributes.system.shell.options.foo" opt.out) = "true" &&
+	test $(jq ".attributes.system.shell.options.bar.baz" opt.out) = "42" &&
+	test $(jq ".attributes.system.shell.options.baz" opt.out) = "1"
+'
+test_expect_success 'flux submit --output passes through to shell' '
+	flux submit --dry-run --output=my/file hostname >output.out &&
+	test $(jq ".attributes.system.shell.options.output.stdout.type" output.out) = "\"file\"" &&
+	test $(jq ".attributes.system.shell.options.output.stdout.path" output.out) = "\"my/file\""
+'
+test_expect_success 'flux submit --error passes through to shell' '
+	flux submit --dry-run --error=/my/error hostname >error.out &&
+	test $(jq ".attributes.system.shell.options.output.stderr.type" error.out) = "\"file\"" &&
+	test $(jq ".attributes.system.shell.options.output.stderr.path" error.out) = "\"/my/error\""
+'
+test_expect_success 'flux submit --label-io --output passes through to shell' '
+	flux submit --dry-run \
+		--label-io --output=foo hostname >labout.out &&
+	test $(jq ".attributes.system.shell.options.output.stdout.label" labout.out) = "true" &&
+	test $(jq ".attributes.system.shell.options.output.stdout.path" labout.out) = "\"foo\""
+'
+test_expect_success 'flux submit --output id mustache passes through to shell' '
+	flux submit --dry-run --output=foo.{{id}} hostname >musid.out &&
+	test $(jq ".attributes.system.shell.options.output.stdout.path" musid.out) = "\"foo.{{id}}\""
+'
+test_expect_success 'flux submit --cwd passes through to jobspec' '
+	flux submit --cwd=/foo/bar/baz --dry-run hostname > cwd.out &&
+	jq -e ".attributes.system.cwd == \"/foo/bar/baz\"" < cwd.out
+'
+test_expect_success 'flux submit command arguments work' '
+	flux submit --dry-run a b c >args.out &&
+	test $(jq ".tasks[0].command[0]" args.out) = "\"a\"" &&
+	test $(jq ".tasks[0].command[1]" args.out) = "\"b\"" &&
+	test $(jq ".tasks[0].command[2]" args.out) = "\"c\""
+'
+test_expect_success 'flux submit --gpus-per-task adds gpus to task slot' '
+	flux submit --dry-run -g2 hostname >gpu.out &&
+	test $(jq ".resources[0].with[1].type" gpu.out) = "\"gpu\"" &&
+	test $(jq ".resources[0].with[1].count" gpu.out) = "2"
+'
+test_expect_success 'flux submit --job-name works' '
+	flux submit --dry-run --job-name=foobar hostname >name.out &&
+	test $(jq ".attributes.system.job.name" name.out) = "\"foobar\""
+'
+test_expect_success 'flux submit --env=-*/--env-remove=* works' '
+	flux submit --dry-run --env=-* hostname > no-env.out &&
+	jq -e ".attributes.system.environment == {}" < no-env.out &&
+	flux submit --dry-run --env-remove=* hostname > no-env2.out &&
+	jq -e ".attributes.system.environment == {}" < no-env2.out
+'
+test_expect_success 'flux submit --env=VAR works' '
+	FOO=bar flux submit --dry-run \
+	    --env=-* --env FOO hostname >FOO-env.out &&
+	jq -e ".attributes.system.environment == {\"FOO\": \"bar\"}" FOO-env.out
+'
+test_expect_success 'flux submit --env=PATTERN works' '
+	FOO_ONE=bar FOO_TWO=baz flux submit --dry-run \
+	    --env=-* --env="FOO_*" hostname >FOO-pattern-env.out &&
+	jq -e ".attributes.system.environment == \
+	    {\"FOO_ONE\": \"bar\", \"FOO_TWO\": \"baz\"}" FOO-pattern-env.out &&
+	FOO_ONE=bar FOO_TWO=baz flux submit --dry-run \
+	    --env=-* --env="/^FOO_.*/" hostname >FOO-pattern2-env.out &&
+	jq -e ".attributes.system.environment == \
+	    {\"FOO_ONE\": \"bar\", \"FOO_TWO\": \"baz\"}" FOO-pattern2-env.out
+
+'
+test_expect_success 'flux submit --env=VAR=VAL works' '
+	flux submit --dry-run \
+	    --env=-* --env PATH=/bin hostname >PATH-env.out &&
+	jq -e ".attributes.system.environment == {\"PATH\": \"/bin\"}" PATH-env.out &&
+	FOO=bar flux submit --dry-run \
+	    --env=-* --env FOO=\$FOO:baz hostname >FOO-append.out &&
+	jq -e ".attributes.system.environment == {\"FOO\": \"bar:baz\"}" FOO-append.out
+'
+test_expect_success 'flux submit --env-file works' '
+	cat <<-EOF >envfile &&
+	-*
+	FOO=bar
+	BAR=\${FOO}/baz
+	EOF
+	for arg in "--env=^envfile" "--env-file=envfile"; do
+	  flux submit --dry-run ${arg} hostname >envfile.out &&
+	  jq -e ".attributes.system.environment == \
+	       {\"FOO\":\"bar\", \"BAR\":\"bar/baz\"}" envfile.out
+	done
+'
+test_expect_success 'flux submit --cc works' '
+	flux submit --cc=0-3 sh -c "echo \$FLUX_JOB_CC" >cc.jobids &&
+	test_debug "cat cc.jobids" &&
+	test $(wc -l < cc.jobids) -eq 4 &&
+	for job in $(cat cc.jobids); do
+		flux job attach $job
+	done > cc.output &&
+	sort cc.output > cc.output.sorted &&
+	cat <<-EOF >cc.output.expected &&
+	0
+	1
+	2
+	3
+	EOF
+	test_cmp cc.output.expected cc.output.sorted
+'
+test_expect_success 'flux submit --cc substitutes {cc}' '
+	flux submit --quiet --watch --cc=0-3 echo {cc} \
+	    | sort >substitute-cc.out &&
+	test_debug "cat substitute-cc.out" &&
+	test $(wc -l < substitute-cc.out) -eq 4 &&
+	cat <<-EOF >substitute-cc.expected &&
+	0
+	1
+	2
+	3
+	EOF
+	test_cmp substitute-cc.expected substitute-cc.out
+'
+test_expect_success 'flux submit does not substitute {} without --cc' '
+	flux submit \
+		--env=-* \
+		--setattr=system.test={} \
+		--dry-run true > nocc.json &&
+	jq -e ".attributes.system.test == {}" < nocc.json
+'
+test_expect_success 'flux submit --tasks-per-node works' '
+	flux submit \
+		--env=-* \
+		-N 2 \
+		--tasks-per-node=2 \
+		--dry-run true > ntasks-per-node.json &&
+	jq -e \
+	 ".attributes.system.shell.options.\"per-resource\".type == \"node\"" \
+		< ntasks-per-node.json &&
+	jq -e \
+	 ".attributes.system.shell.options.\"per-resource\".count == 2" \
+		< ntasks-per-node.json
+'
+test_expect_success 'flux submit --input=IDSET fails' '
+	test_must_fail flux submit --input=0 hostname &&
+	test_must_fail flux submit -n2 --input=0-1 hostname
+'
+test_expect_success 'flux submit: create test files ' '
+	cat <<-EOF >file.txt &&
+	This is a test file
+	EOF
+	dd if=/dev/urandom of=file.binary bs=64 count=1 &&
+	touch file.empty
+'
+for file in file.txt file.binary file.empty; do
+	test_expect_success \
+		"flux submit --add-file works for ${file#file.} file" '
+		flux submit -n1 --watch --add-file=${file} \
+			cp {{tmpdir}}/${file} ${file}.result &&
+		test_cmp ${file} ${file}.result
+	'
+done
+test_expect_success 'flux submit --add-file=name=file works' '
+	flux submit -n1 --watch --add-file=myfile=file.txt \
+		cp {{tmpdir}}/myfile . &&
+	test_cmp file.txt myfile
+'
+test_expect_success 'flux submit --add-file=name=data works' '
+	flux submit -n1 --watch --add-file=add-file.test="this is a test\n" \
+		cp {{tmpdir}}/add-file.test . &&
+	grep "this is a test" add-file.test
+'
+test_expect_success 'flux submit --add-file=name:perms=data works' '
+	flux submit -n1 --watch --add-file=test:0700="#!/bin/sh\ntrue\n" \
+		{{tmpdir}}/test
+'
+test_expect_success 'flux submit --add-file allows colon in name' '
+	flux submit -n1 --watch --add-file=add-file:test="this is a test\n" \
+		cp {{tmpdir}}/add-file:test . &&
+	grep "this is a test" add-file:test
+'
+test_expect_success 'flux submit --add-file complains for non-regular files' '
+	test_must_fail flux submit -n1 --add-file=/tmp hostname
+'
+test_expect_success 'flux submit --add-file complains for missing files' '
+	test_must_fail flux submit -n1 --add-file=doesnotexist hostname
+'
+test_done
diff --git a/t/t2711-python-cli-run.t b/t/t2711-python-cli-run.t
new file mode 100755
index 000000000000..762551b2794a
--- /dev/null
+++ b/t/t2711-python-cli-run.t
@@ -0,0 +1,270 @@
+#!/bin/sh
+
+test_description='Test flux run command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4
+
+# Set CLIMain log level to logging.DEBUG (10), to enable stack traces
+export FLUX_PYCLI_LOGLEVEL=10
+
+flux setattr log-stderr-level 1
+
+waitfile=${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua
+
+test_expect_success 'flux run fails with error message' '
+	test_must_fail flux run 2>usage.err &&
+	grep "job command and arguments are missing" usage.err
+'
+test_expect_success 'flux run ignores ambiguous args after --' '
+	flux run -n2 --dry-run -- hostname --n=2 \
+		| jq -e ".tasks[0].command[0] == \"hostname\""
+'
+test_expect_success 'flux run works' '
+	flux run hostname >run.out &&
+	hostname >run.exp &&
+	test_cmp run.exp run.out
+'
+test_expect_success 'flux run --ntasks=2 works' '
+	flux run --ntasks=2 hostname >run2.out &&
+	(hostname;hostname) >run2.exp &&
+	test_cmp run2.exp run2.out
+'
+test_expect_success 'flux run --ntasks=2 --label-io works' '
+	flux run --ntasks=2 --label-io echo Hi |sort >run2l.out &&
+	cat >run2l.exp <<-EOT &&
+	0: Hi
+	1: Hi
+	EOT
+	test_cmp run2l.exp run2l.out
+'
+test_expect_success 'flux run --ntasks=2 --nodes=2 works' '
+	flux run --ntasks=2 --nodes=2 hostname
+'
+test_expect_success 'flux run --ntasks=1 --nodes=2 fails' '
+	test_must_fail flux run --ntasks=1 --nodes=2 hostname \
+		2>run1n2N.err &&
+	grep -i "node count must not be greater than task count" run1n2N.err
+'
+test_expect_success 'flux run -v produces jobid on stderr' '
+	flux run -v hostname 2>v.err &&
+	grep jobid: v.err
+'
+test_expect_success 'flux run -vv produces job events on stderr' '
+	flux run -vv hostname 2>vv.err &&
+	grep submit vv.err
+'
+test_expect_success 'flux run -vvv produces exec events on stderr' '
+	flux run -vvv hostname 2>vvv.err &&
+	grep complete vvv.err
+'
+test_expect_success 'flux run waits for clean event by default' '
+	grep clean vvv.err
+'
+test_expect_success 'flux run --cwd works' '
+	mkdir cwd_test &&
+	flux run --cwd=$(realpath cwd_test) pwd > cwd.out &&
+	test $(cat cwd.out) = $(realpath cwd_test)
+'
+test_expect_success 'flux run --env=VAR=${VAL:-default} fails' '
+	test_expect_code 1 flux run --dry-run \
+	    --env=* --env=VAR=\${VAL:-default} hostname >env-fail.err 2>&1 &&
+	test_debug "cat env-fail.err" &&
+	grep "Unable to substitute" env-fail.err
+'
+test_expect_success 'flux run --env=VAR=$VAL fails when VAL not in env' '
+	unset VAL &&
+	test_expect_code 1 flux run --dry-run \
+	    --env=* --env=VAR=\$VAL hostname >env-notset.err 2>&1 &&
+	test_debug "cat env-notset.err" &&
+	grep "env: Variable .* not found" env-notset.err
+'
+test_expect_success 'flux run propagates some rlimits by default' '
+	flux run --dry-run hostname | \
+	    jq .attributes.system.shell.options.rlimit >rlimit-default.out &&
+	# check random sample of rlimits:
+	grep core rlimit-default.out &&
+	grep stack rlimit-default.out &&
+	grep nofile rlimit-default.out
+'
+test_expect_success 'flux run --rlimit=-* works' '
+	flux run --rlimit=-* --dry-run hostname \
+	    | jq -e ".attributes.system.shell.options.rlimit == null"
+'
+test_expect_success 'flux run --rlimit=name works' '
+	flux run --rlimit=memlock --dry-run hostname \
+	    | jq .attributes.system.shell.options.rlimit >rlimit-memlock.out &&
+	grep memlock rlimit-memlock.out &&
+	grep core rlimit-memlock.out
+'
+test_expect_success 'flux run --rlimit=name --rlimit=name works' '
+	flux run --rlimit=memlock --rlimit=ofile --dry-run hostname \
+	    | jq .attributes.system.shell.options.rlimit >rlimit-ofile.out &&
+	grep memlock rlimit-memlock.out &&
+	grep ofile rlimit-memlock.out
+'
+test_expect_success 'flux run --rlimit=-*,core works' '
+	flux run --rlimit=-*,core --dry-run hostname \
+	    | jq .attributes.system.shell.options.rlimit >rlimit-core.out &&
+	grep core rlimit-core.out &&
+	test_must_fail grep nofile rlimit-core.out
+'
+test_expect_success 'flux run --rlimit=name=value works' '
+	flux run --rlimit=core=16 --dry-run hostname \
+	    | jq -e ".attributes.system.shell.options.rlimit.core == 16" &&
+	inf=$(flux python -c "import resource as r; print(r.RLIM_INFINITY)") &&
+	flux run --rlimit=core=unlimited --dry-run hostname \
+	    | jq -e ".attributes.system.shell.options.rlimit.core == $inf"
+'
+test_expect_success 'flux run --rlimit=invalid fails with error' '
+	test_must_fail flux run --rlimit=frobnitz hostname
+'
+test_expect_success 'flux run --rlimit=core=invalid fails with error' '
+	test_must_fail flux run --rlimit=core=invalid hostname
+'
+
+#  Per-resource expected failure tests
+cat <<EOF >per-resource-failure.txt
+\
+--tasks-per-core=1 --tasks-per-node=1 \
+==Do not specify both the number of tasks per node and per core
+\
+--tasks-per-node=0 \
+==--tasks-per-node must be >= 1
+\
+--tasks-per-core=0 \
+==--tasks-per-core must be >= 1
+\
+--cores=-4 \
+==ncores must be an integer >= 1
+\
+--nodes=1 --gpus-per-node=-1 \
+==gpus_per_node must be an integer >= 0
+\
+--gpus-per-node=1 \
+==gpus-per-node requires --nodes
+\
+--cores=4 --exclusive \
+==exclusive can only be set with a node count
+\
+--nodes=4 --cores=2 \
+==number of cores cannot be less than nnodes
+\
+--nodes=5 --cores=9 \
+==number of cores must be evenly divisible by node count
+\
+--cores=2 --cores-per-task=1 \
+==Per-resource options.*per-task options
+\
+--cores=1 --ntasks=1 \
+==Per-resource options.*per-task options
+\
+--nodes=1 --tasks-per-node=1 --gpus-per-task=1 \
+==Per-resource options.*per-task options
+\
+--nodes=1 --tasks-per-core=1 --cores-per-task=1 \
+==Per-resource options.*per-task options
+\
+--tasks-per-core=1 \
+==must specify node or core count with per_resource
+\
+--tasks-per-node=1 \
+==must specify node or core count with per_resource
+EOF
+
+while read line; do
+	args=$(echo $line | awk -F== '{print $1}' | sed 's/  *$//')
+	expected=$(echo $line | awk -F== '{print $2}')
+	test_expect_success "per-resource: $args error: $expected" '
+		output=per-resource-error.${test_count}.out &&
+		test_must_fail flux run $args --env=-* --dry-run hostname \
+			>${output} 2>&1 &&
+		test_debug "cat $output" &&
+		grep -- "$expected" $output
+	'
+done < per-resource-failure.txt
+
+
+#  Per-resource expected success tests
+cat <<EOF >per-resource-args.txt
+\
+-N2 --cores=2 \
+==nnodes=2 nslots=2 slot_size=1 slot_gpus=0 exclusive=false duration=0.0 \
+==
+\
+-N2 --cores=2 --tasks-per-node=2 \
+==nnodes=2 nslots=2 slot_size=1 slot_gpus=0 exclusive=false duration=0.0 \
+=={"type": "node", "count": 2}
+\
+-N2 --cores=2 --tasks-per-core=2 \
+==nnodes=2 nslots=2 slot_size=1 slot_gpus=0 exclusive=false duration=0.0 \
+=={"type": "core", "count": 2}
+\
+--cores=16 \
+==nnodes=0 nslots=16 slot_size=1 slot_gpus=0 exclusive=false duration=0.0 \
+==
+\
+--cores=16 --tasks-per-node=1 \
+==nnodes=0 nslots=16 slot_size=1 slot_gpus=0 exclusive=false duration=0.0 \
+=={"type": "node", "count": 1}
+\
+--cores=5 --tasks-per-core=2 \
+==nnodes=0 nslots=5 slot_size=1 slot_gpus=0 exclusive=false duration=0.0 \
+=={"type": "core", "count": 2}
+\
+--nodes=2 --tasks-per-node=2 \
+==nnodes=2 nslots=2 slot_size=1 slot_gpus=0 exclusive=true duration=0.0 \
+=={"type": "node", "count": 2}
+\
+--nodes=2 --tasks-per-core=1 \
+==nnodes=2 nslots=2 slot_size=1 slot_gpus=0 exclusive=true duration=0.0 \
+=={"type": "core", "count": 1}
+\
+-N1 --gpus-per-node=2 \
+==nnodes=1 nslots=1 slot_size=1 slot_gpus=2 exclusive=true duration=0.0 \
+==
+\
+-N2 --cores=4 --tasks-per-node=1 --gpus-per-node=1 \
+==nnodes=2 nslots=2 slot_size=2 slot_gpus=1 exclusive=false duration=0.0 \
+=={"type": "node", "count": 1}
+EOF
+
+jj=${FLUX_BUILD_DIR}/t/sched-simple/jj-reader
+while read line; do
+	args=$(echo $line | awk -F== '{print $1}' | sed 's/  *$//')
+	expected=$(echo $line | awk -F== '{print $2}')
+	per_resource=$(echo $line | awk -F== '{print $3}' | sed 's/  *$//')
+	test_expect_success "per-resource: $args" '
+	    echo $expected >expected.$test_count &&
+	    flux run $args --dry-run --env=-* hostname > jobspec.$test_count &&
+	    $jj < jobspec.$test_count >output.$test_count &&
+	    test_debug "cat output.$test_count" &&
+	    test_cmp expected.$test_count output.$test_count &&
+	    if test -n "$per_resource"; then
+	        test_debug "echo expected $per_resource" &&
+	        jq -e ".attributes.system.shell.options.\"per-resource\" == \
+		   $per_resource" < jobspec.$test_count
+	    fi
+	'
+done < per-resource-args.txt
+
+test_expect_success NO_CHAIN_LINT 'flux run --unbuffered works for stdin/out' '
+        rm -f fifo &&
+        mkfifo fifo &&
+        (flux run --unbuffered cat >unbuffered.out <fifo &) &&
+        exec 9>fifo &&
+        printf prompt: >&9 &&
+        $waitfile -v --count=1 --timeout=10 --pattern=prompt: unbuffered.out &&
+        exec 9<&- &&
+        rm -f fifo &&
+        flux job wait-event -vt 15 $(flux job last) clean
+'
+test_expect_success 'flux run --input=ranks works for stdin' '
+	echo hi | flux run --label-io --input=0 -n4 cat >input-test.out 2>&1 &&
+	cat <<-EOF >input-test.expected &&
+	0: hi
+	EOF
+	test_cmp input-test.expected input-test.out
+'
+test_done
diff --git a/t/t2712-python-cli-alloc.t b/t/t2712-python-cli-alloc.t
new file mode 100755
index 000000000000..0b20cc527555
--- /dev/null
+++ b/t/t2712-python-cli-alloc.t
@@ -0,0 +1,198 @@
+#!/bin/sh
+
+test_description='flux alloc specific tests'
+
+. $(dirname $0)/sharness.sh
+
+# Start an instance with 16 cores across 4 ranks
+export TEST_UNDER_FLUX_CORES_PER_RANK=4
+# Set local URI resolution for use of flux-proxy below:
+export FLUX_URI_RESOLVE_LOCAL=t
+test_under_flux 4 job
+
+flux setattr log-stderr-level 1
+
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py -f asciicast"
+
+test_expect_success 'flux alloc with no args return error' '
+	test_expect_code 1 flux alloc
+'
+test_expect_success 'flux alloc sets command to flux broker' '
+	flux alloc -n1 --dry-run | \
+	    jq -e ".tasks[0].command == [ \"flux\", \"broker\" ]"
+'
+test_expect_success 'flux alloc appends broker options' '
+	flux alloc -n1 --broker-opts=-v --dry-run | \
+	    jq -e ".tasks[0].command == [ \"flux\", \"broker\", \"-v\" ]"
+'
+test_expect_success 'flux alloc can set initial-program' '
+	flux alloc -n1 --dry-run myapp --foo | \
+	    jq -e ".tasks[0].command == [ \"flux\", \"broker\", \"myapp\", \"--foo\" ]"
+'
+test_expect_success 'flux alloc ignores ambiguous option after --' '
+	flux alloc -n1 --dry-run -- myapp --n=2 | \
+	    jq -e ".tasks[0].command == [ \"flux\", \"broker\", \"--\", \"myapp\", \"--n=2\" ]"
+'
+test_expect_success 'flux alloc -N2 requests 2 nodes exclusively' '
+	flux alloc -N2 --dry-run hostname | jq -S ".resources[0]" | jq -e ".type == \"node\" and .exclusive"
+'
+test_expect_success 'flux alloc --exclusive works' '
+	flux alloc -N1 -n1 --exclusive --dry-run hostname | jq -S ".resources[0]" | jq -e ".type == \"node\" and .exclusive"
+'
+test_expect_success 'flux alloc fails if N > n' '
+	test_expect_code 1 flux alloc -N2 -n1 --dry-run hostname
+'
+test_expect_success 'flux alloc works' '
+	$runpty -o single.out flux alloc -n1 flux resource list -s up -no {rlist} && grep "rank0/core0" single.out
+'
+test_expect_success 'flux alloc works without tty' '
+	flux alloc -n1 flux resource list -s up -no {rlist} </dev/null >notty.out && test_debug "echo notty: $(cat notty.out)" && test "$(cat notty.out)" = "rank0/core0"
+'
+test_expect_success 'flux alloc runs one broker per node by default' '
+	$runpty -o multi.out flux alloc -n5 flux lsattr -v && test_debug "cat multi.out" && grep "size  *2" multi.out
+'
+test_expect_success 'flux alloc -v prints jobid on stderr' '
+$runpty -o verbose.out flux alloc -n1 -v flux lsattr -v && test_debug "cat verbose.out" && grep "jobid: " verbose.out
+'
+test_expect_success 'flux alloc --bg option works' '
+	jobid=$(flux alloc -n1 -v --bg) &&
+	flux proxy $jobid flux run hostname &&
+	flux proxy $jobid flux getattr broker.rc2_none &&
+	flux shutdown $jobid &&
+	flux job wait-event $jobid clean
+'
+test_expect_success 'flux alloc --bg option works with a command' '
+	jobid=$(flux alloc -n1 -v --bg true) &&
+	flux job wait-event -t 180 -v $jobid finish &&
+	flux job attach $jobid
+'
+test_expect_success 'flux alloc --bg fails if broker fails' '
+	test_must_fail flux alloc -n1 -v --broker-opts=--xx --bg \
+		>badopts.log 2>&1 &&
+	test_debug "cat badopts.log" &&
+	grep "unrecognized option" badopts.log
+'
+test_expect_success 'flux alloc --bg fails if rc1 fails' '
+	mkdir -p rc1.d/ &&
+	cat <<-EOF >rc1.d/rc1-fail &&
+	exit 1
+	EOF
+	( export FLUX_RC_EXTRA=$(pwd) &&
+	  test_must_fail flux alloc -n1 -v --broker-opts= --bg \
+		>rc1-fail.log 2>&1
+	) &&
+	test_debug "cat rc1-fail.log" &&
+	grep "instance startup failed" rc1-fail.log
+'
+
+
+#  Running a process in the background under test_expect_success()
+#   causes a copy of the shell to be run in between flux, so the
+#   signal can't be delivered to the right PID. Running from a function
+#   seems to fix that.
+run_mini_bg() {
+	flux alloc --bg -n1 -v >sigint.log 2>&1 &
+	echo $! >sigint.pid
+}
+waitfile=$SHARNESS_TEST_SRCDIR/scripts/waitfile.lua
+test_expect_success NO_CHAIN_LINT 'flux alloc --bg can be interrupted' '
+	flux queue stop &&
+	test_when_finished "flux queue start" &&
+	run_mini_bg &&
+	$waitfile -t 180 -v -p waiting sigint.log &&
+	kill -INT $(cat sigint.pid) &&
+	sleep 0.1 &&
+	(kill -INT $(cat sigint.pid) || true) &&
+	$waitfile -t 180 -v -p Interrupt sigint.log &&
+	wait $pid
+'
+test_expect_success NO_CHAIN_LINT 'flux alloc --bg errors when job is canceled' '
+	flux queue stop &&
+	test_when_finished "flux queue start" &&
+	flux alloc --bg -n1 -v >canceled.log 2>&1 &
+	pid=$! &&
+	$waitfile -t 180 -v -p waiting canceled.log &&
+	flux cancel --all &&
+	cat canceled.log &&
+	test_must_fail wait $pid &&
+	grep "unexpectedly exited" canceled.log
+'
+test_expect_success 'flux alloc: sets mpi=none by default' '
+	flux alloc -N1 --dry-run hostname | \
+		jq -e ".attributes.system.shell.options.mpi = \"none\""
+'
+test_expect_success 'flux alloc: mpi option can be overridden' '
+	flux alloc -o mpi=foo -N1 --dry-run hostname | \
+		jq -e ".attributes.system.shell.options.mpi = \"foo\""
+'
+test_expect_success 'flux alloc: MPI vars are not set in initial program' '
+	flux queue start &&
+	unset OMPI_MCA_pmix &&
+	flux alloc -N1 printenv >envtest.out &&
+	test_must_fail grep OMPI_MCA_pmix envtest.out
+'
+test_expect_success 'flux alloc: --dump works' '
+        jobid=$(flux alloc -N1 --bg --dump) &&
+	flux shutdown $jobid &&
+	flux job wait-event $jobid clean &&
+        tar tvf flux-${jobid}-dump.tgz
+'
+test_expect_success 'flux alloc: --dump=FILE works' '
+        jobid=$(flux alloc -N1 --bg --dump=testdump.tgz) &&
+	flux shutdown $jobid &&
+	flux job wait-event $jobid clean &&
+        tar tvf testdump.tgz
+'
+test_expect_success 'flux alloc: --dump=FILE works with mustache' '
+        jobid=$(flux alloc -N1 --bg --dump=testdump-{{id}}.tgz) &&
+	flux shutdown $jobid &&
+	flux job wait-event $jobid clean &&
+        tar tvf testdump-${jobid}.tgz
+'
+test_expect_success 'flux alloc: does not suppress log messages' '
+	$runpty -o logmsgs.out flux alloc -n1 --cwd=/noexist pwd &&
+	grep -i "going to /tmp instead" logmsgs.out
+'
+test_expect_success 'flux alloc: no duplication of output with pty.capture' '
+	$runpty -o duplicates.out flux alloc -o pty.capture -n1 echo testing &&
+	test $(grep -c testing duplicates.out) -eq 1
+'
+test_expect_success 'flux alloc: instance can bootstrap without update-watch RPC' '
+	flux alloc -N2 \
+	  --broker-opts=-Slog-stderr-level=7 \
+	  --conf=resource.no-update-watch=true true >alloc.log 2>&1 &&
+	test_debug "cat alloc.log" &&
+	grep "falling back to job-info.lookup" alloc.log
+'
+test_expect_success 'flux alloc: resource.norestrict works in subinstance' '
+	cat <<-EOF >topo-get.py &&
+	import flux
+	print(flux.Flux().rpc("resource.topo-get").get_str())
+	EOF
+	chmod +x topo-get.py &&
+	flux python ./topo-get.py >topo.expected &&
+	flux alloc -n1 --conf=resource.norestrict=true \
+		flux python ./topo-get.py >topo.out &&
+	test_cmp topo.expected topo.out
+'
+test_expect_success 'flux alloc: flux alloc vi works' '
+	cat <<-'EOF' >input.json &&
+	[{"expect":"test text file", "send":":q!\n", "timeout":30}]
+	EOF
+	cat <<-EOF >test.txt &&
+	test text file
+	EOF
+	$runpty -o vi.out --expect=input.json flux alloc -n1 vi test.txt
+'
+test_expect_success 'flux alloc: flux alloc flux alloc works' '
+	cat <<-'EOF' >input2.json &&
+	[{"expect":"prompt>", "send":"flux resource info\n", "timeout":120},
+	 {"expect":"prompt>", "send":"exit\n", "timeout":30}
+	]
+	EOF
+	PROMPT_COMMAND="PS1=\"prompt>\"" \
+		$runpty -o allocx2.out --expect=input2.json \
+		flux alloc -n1 flux alloc -n1 bash --norc &&
+	grep prompt allocx2.out
+'
+test_done
diff --git a/t/t2713-python-cli-bulksubmit.t b/t/t2713-python-cli-bulksubmit.t
new file mode 100755
index 000000000000..1ba703ca5515
--- /dev/null
+++ b/t/t2713-python-cli-bulksubmit.t
@@ -0,0 +1,334 @@
+#!/bin/sh
+
+test_description='flux bulksubmit specific tests'
+
+. $(dirname $0)/sharness.sh
+
+
+# Start an instance with 16 cores across 4 ranks
+export TEST_UNDER_FLUX_CORES_PER_RANK=4
+test_under_flux 4 job
+
+flux setattr log-stderr-level 1
+
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py -f asciicast --line-buffer"
+
+test_expect_success 'flux bulksubmit submits each arg as a job' '
+	seq 1 3 | flux bulksubmit echo {} >bs.ids &&
+	test $(wc -l < bs.ids) -eq 3 &&
+	for job in $(cat bs.ids); do
+	    flux job attach $job
+	done | sort >bs.output &&
+	cat <<-EOF >bs.expected &&
+	1
+	2
+	3
+	EOF
+	test_cmp bs.expected bs.output
+'
+test_expect_success 'flux bulksubmit uses default command of {}' '
+	flux bulksubmit --dry-run ::: foo bar baz >default-cmd.out &&
+	cat <<-EOF >default-cmd.expected &&
+	bulksubmit: submit foo
+	bulksubmit: submit bar
+	bulksubmit: submit baz
+	EOF
+	test_cmp default-cmd.expected default-cmd.out
+'
+test_expect_success 'flux bulksubmit --cc works' '
+	seq 1 3 | flux bulksubmit --cc=0-1 echo {} >bs2.ids &&
+	test $(wc -l < bs2.ids) -eq 6 &&
+	for job in $(cat bs2.ids); do
+	    flux job attach $job
+	done | sort >bs2.output &&
+	cat <<-EOF >bs2.expected &&
+	1
+	1
+	2
+	2
+	3
+	3
+	EOF
+	test_cmp bs2.expected bs2.output
+'
+test_expect_success 'flux bulksubmit --bcc works' '
+	seq 1 3 | \
+	    flux bulksubmit --quiet --watch --bcc=0-1 \
+	        sh -c "echo {}\$FLUX_JOB_CC" >bcc.out &&
+	sort bcc.out > bcc.out.sorted &&
+	cat <<-EOF >bcc.expected &&
+	1
+	1
+	2
+	2
+	3
+	3
+	EOF
+	test_cmp bcc.expected bcc.out.sorted
+'
+test_expect_success 'flux bulksubmit --watch works' '
+	flux bulksubmit --watch echo {} ::: foo bar baz >bs3.out &&
+	test_debug "cat bs3.out" &&
+	grep ^foo bs3.out &&
+	grep ^bar bs3.out &&
+	grep ^baz bs3.out
+'
+test_expect_success 'flux bulksubmit reports exceptions with --watch' '
+	test_expect_code 1 \
+	    flux bulksubmit --watch -n {} hostname ::: 1 2 4 1024 \
+	    >b4.out 2>&1 &&
+	test_debug "cat b4.out" &&
+	grep "unsatisfiable request" b4.out
+'
+test_expect_success 'flux submit --progress works without --wait' '
+	$runpty flux submit --cc=1-10 --progress hostname >bs5.out &&
+	test_debug "cat bs5.out" &&
+	grep "10 jobs" bs5.out &&
+	grep "100.0%" bs5.out
+'
+test_expect_success 'flux submit --progress works with --wait' '
+	$runpty flux submit --cc=1-10 --progress --wait hostname \
+	    >bs6.out &&
+	grep "PD:1 *R:0 *CD:0 *F:0" bs6.out &&
+	grep "PD:0 *R:0 *CD:10 *F:0" bs6.out &&
+	grep "100.0%" bs6.out
+'
+test_expect_success 'flux bulksubmit --wait/progress with failed jobs' '
+        test_expect_code 1 $runpty flux bulksubmit \
+	    -n{} --progress --wait hostname ::: 1 2 4 1024 \
+	    >bs7.out &&
+	grep "PD:1 *R:0 *CD:0 *F:0" bs7.out &&
+	grep "PD:0 *R:0 *CD:3 *F:1" bs7.out &&
+	grep "100.0%" bs7.out
+'
+test_expect_success 'flux bulksubmit --wait/progress with failed jobs' '
+        test_expect_code 2 $runpty flux bulksubmit \
+	    -n1 --progress --wait sh -c "exit {}" ::: 0 1 0 0 2 \
+	    >bs8.out &&
+	grep "PD:1 *R:0 *CD:0 *F:0" bs8.out &&
+	grep "PD:0 *R:0 *CD:3 *F:2" bs8.out &&
+	grep "100.0%" bs8.out
+'
+test_expect_success 'flux submit --wait/progress with job exceptions' '
+        test_expect_code 1 $runpty flux bulksubmit \
+	    -n1 --progress --wait \
+	    --setattr=system.exec.bulkexec.mock_exception={} hostname \
+	    ::: none init init none none \
+	    >bs9.out &&
+	grep "PD:1 *R:0 *CD:0 *F:0" bs9.out &&
+	grep "PD:0 *R:0 *CD:3 *F:2" bs9.out &&
+	grep "100.0%" bs9.out
+'
+test_expect_success 'flux bulksubmit --wait returns highest exit code' '
+	test_expect_code 143 \
+	    flux bulksubmit --wait sh -c "kill -{} \$\$" ::: 0 0 15
+'
+test_expect_success 'flux bulksubmit --wait-event works' '
+	flux bulksubmit -vvv \
+		--wait-event={} \
+		--log-stderr=wait.{}.out \
+		true ::: start exec.shell.init &&
+	test_debug "cat wait.start.out" &&
+	test_debug "cat wait.exec.shell.init.out" &&
+	tail -n1 wait.start.out | grep start &&
+	tail -n1 wait.exec.shell.init.out | grep shell.init
+'
+test_expect_success 'flux bulksubmit replacement format strings work' '
+	echo /a/b/c/d.txt /a/b/c/d c.txt | \
+	    flux bulksubmit --sep=None --dry-run \
+	        {seq} {seq1} {} {.%} {./} {.//} {./%} \
+	        >repl.out &&
+	cat <<-EOF >repl.expected &&
+	bulksubmit: submit 0 1 /a/b/c/d.txt /a/b/c/d d.txt /a/b/c d
+	bulksubmit: submit 1 2 /a/b/c/d /a/b/c/d d /a/b/c d
+	bulksubmit: submit 2 3 c.txt c c.txt  c
+	EOF
+	test_debug "cat repl.out" &&
+	test_cmp repl.expected repl.out
+'
+test_expect_success 'flux bulksubmit can use replacement string in opts' '
+	flux bulksubmit --dry-run -n {} -N {} -c {} -g {} hostname \
+	    ::: 1 2 3 4 >repl2.out &&
+	test_debug "cat -v repl2.out" &&
+	cat <<-EOF >repl2.expected &&
+	bulksubmit: submit -n1 -N1 -c1 -g1 hostname
+	bulksubmit: submit -n2 -N2 -c2 -g2 hostname
+	bulksubmit: submit -n3 -N3 -c3 -g3 hostname
+	bulksubmit: submit -n4 -N4 -c4 -g4 hostname
+	EOF
+	test_cmp repl2.expected repl2.out
+'
+test_expect_success 'flux bulksubmit combines all inputs by default' '
+	flux bulksubmit --dry-run sleep {0}.{1} \
+	    ::: 0 1 2 3 4 ::: 0 5 9 >repl3.out &&
+	test_debug "cat repl3.out" &&
+	cat <<-EOF >repl3.expected &&
+	bulksubmit: submit sleep 0.0
+	bulksubmit: submit sleep 0.5
+	bulksubmit: submit sleep 0.9
+	bulksubmit: submit sleep 1.0
+	bulksubmit: submit sleep 1.5
+	bulksubmit: submit sleep 1.9
+	bulksubmit: submit sleep 2.0
+	bulksubmit: submit sleep 2.5
+	bulksubmit: submit sleep 2.9
+	bulksubmit: submit sleep 3.0
+	bulksubmit: submit sleep 3.5
+	bulksubmit: submit sleep 3.9
+	bulksubmit: submit sleep 4.0
+	bulksubmit: submit sleep 4.5
+	bulksubmit: submit sleep 4.9
+	EOF
+	test_cmp repl3.expected repl3.out
+'
+test_expect_success 'flux bulksubmit links inputs with :::+' '
+	flux bulksubmit --dry-run {0}:{1} \
+	    ::: 0 1 2 3 4 4 :::+ a b c >linked.out &&
+	test_debug "cat linked.out" &&
+	cat <<-EOF >linked.expected &&
+	bulksubmit: submit 0:a
+	bulksubmit: submit 1:b
+	bulksubmit: submit 2:c
+	bulksubmit: submit 3:a
+	bulksubmit: submit 4:b
+	bulksubmit: submit 4:c
+	EOF
+	test_cmp linked.expected linked.out
+'
+test_expect_success 'flux bulksubmit reads from files with ::::' '
+	seq 1 3 >input &&
+	flux bulksubmit --dry-run :::: input >fileinput.out &&
+	cat <<-EOF >fileinput.expected &&
+	bulksubmit: submit 1
+	bulksubmit: submit 2
+	bulksubmit: submit 3
+	EOF
+	test_cmp fileinput.expected fileinput.out
+'
+test_expect_success 'flux bulksubmit splits files on newline only' '
+	cat <<-EOF >inputs2 &&
+	this is a line
+	another line
+	EOF
+	flux bulksubmit --dry-run echo {} :::: inputs2 >whitespace.out &&
+	cat <<-EOT >whitespace.expected &&
+	bulksubmit: submit echo this is a line
+	bulksubmit: submit echo another line
+	EOT
+	test_cmp whitespace.expected whitespace.out
+'
+test_expect_success 'flux bulksubmit can substitute in list options' "
+	flux bulksubmit --dry-run --env=-* --env=SEQ={seq} ::: a b c \
+	    >envsub.out &&
+	test_debug 'cat envsub.out' &&
+	cat <<-EOF >envsub.expected &&
+	bulksubmit: submit --env=['-*', 'SEQ=0'] a
+	bulksubmit: submit --env=['-*', 'SEQ=1'] b
+	bulksubmit: submit --env=['-*', 'SEQ=2'] c
+	EOF
+	test_cmp envsub.expected envsub.out
+"
+test_expect_success 'flux bulksubmit --define works' '
+ 	flux bulksubmit --dry-run \
+	    --define=p2="2**int(x)" \
+	    --job-name={} -n {.p2} hostname ::: $(seq 1 8) \
+	    >define.out &&
+	test_debug "cat define.out" &&
+	cat <<-EOF >define.expected &&
+	bulksubmit: submit -n2 --job-name=1 hostname
+	bulksubmit: submit -n4 --job-name=2 hostname
+	bulksubmit: submit -n8 --job-name=3 hostname
+	bulksubmit: submit -n16 --job-name=4 hostname
+	bulksubmit: submit -n32 --job-name=5 hostname
+	bulksubmit: submit -n64 --job-name=6 hostname
+	bulksubmit: submit -n128 --job-name=7 hostname
+	bulksubmit: submit -n256 --job-name=8 hostname
+	EOF
+	test_cmp define.expected define.out
+'
+test_expect_success 'flux bulksubmit --shuffle works' '
+	flux bulksubmit --dry-run --shuffle \
+	    {seq}:{} ::: $(seq 1 8) >shuffle.out &&
+	test_debug "cat shuffle.out" &&
+	sort shuffle.out > shuffle.sorted &&
+	test_must_fail test_cmp shuffle.sorted shuffle.out >/dev/null
+'
+test_expect_success 'flux bulksubmit --progress is ignored for no tty' '
+	flux submit --progress --wait --cc=0-1 sleep 0 >notty.out 2>&1 &&
+	grep "Ignoring --progress option" notty.out
+'
+test_expect_success 'flux bulksubmit reports invalid replacement strings' '
+	test_expect_code 1 flux bulksubmit {1} ::: 0 1 2 >bad.out 2>&1 &&
+	test_debug "cat bad.out" &&
+	test_expect_code 1 flux bulksubmit -n {1} {} ::: 0 1 2 \
+	    >bad2.out 2>&1 &&
+	test_debug "cat bad2.out" &&
+	test_expect_code 1 flux bulksubmit -n {f} true ::: 0 1 2 \
+	    >bad3.out 2>&1 &&
+	test_debug "cat bad3.out" &&
+	grep "Invalid replacement .* in command" bad.out &&
+	grep "Invalid replacement .* in -n" bad2.out &&
+	grep "Replacement key '\''f'\'' not found" bad3.out
+'
+test_expect_success 'flux bulksubmit: preserves mustache templates' '
+	flux bulksubmit --dry-run --output=flux-{}.{{id}}.out \
+	    hostname ::: 0 1 >mustache.out &&
+	test_debug "cat mustache.out" &&
+	cat <<-EOF > mustache.expected  &&
+	bulksubmit: submit --output=flux-0.{{id}}.out hostname
+	bulksubmit: submit --output=flux-1.{{id}}.out hostname
+	EOF
+	test_cmp mustache.expected mustache.out
+'
+test_expect_success 'flux bulksubmit: preserves mustache templates in command' '
+	flux bulksubmit --dry-run echo {{tmpdir}} ::: 0 1 >mustache-cmd.out &&
+	test_debug "cat mustache-cmd.out" &&
+	cat <<-EOF > mustache-cmd.expected  &&
+	bulksubmit: submit echo {{tmpdir}}
+	bulksubmit: submit echo {{tmpdir}}
+	EOF
+	test_cmp mustache-cmd.expected mustache-cmd.out
+'
+test_expect_success 'flux submit --log works and can substitute {cc}' '
+	flux submit --log=job{cc}.id --cc=1-5 hostname &&
+	for id in $(seq 1 5); do
+	    test -f job${id}.id &&
+	    flux job wait-event -v $(cat job${id}.id) clean
+	done
+'
+test_expect_success 'flux submit --log-stderr works' '
+	flux submit --log-stderr=job{cc}.stderr --cc=0-1 --watch -vv \
+	    hostname &&
+	for id in $(seq 0 1); do
+	    test -f job${id}.stderr &&
+	    grep complete job${id}.stderr
+	done
+'
+test_expect_success 'flux submit --log-stderr works' '
+	flux submit --log-stderr=job{cc}.stderr --cc=0-1 --watch -vv \
+	    hostname &&
+	for id in $(seq 0 1); do
+	    test -f job${id}.stderr &&
+	    grep complete job${id}.stderr
+	done
+'
+test_expect_success 'flux bulksubmit preserves {cc} in args' '
+	flux bulksubmit --log=preserve-{cc}.out --cc=0-1 --watch \
+		echo {cc}={} ::: a b c &&
+	for id in 0 1; do
+	    grep ^${id}=a preserve-${id}.out &&
+	    grep ^${id}=b preserve-${id}.out &&
+	    grep ^${id}=c preserve-${id}.out
+	done
+'
+test_expect_success 'flux bulksubmit --dry-run works with --cc' '
+	flux bulksubmit --dry-run --cc=1-2 echo {} ::: 1 2 3 \
+	  >dry-run-cc.out 2>&1 &&
+	cat <<-EOF >dry-run-cc.expected &&
+	bulksubmit: submit --cc=1-2 echo 1
+	bulksubmit: submit --cc=1-2 echo 2
+	bulksubmit: submit --cc=1-2 echo 3
+	EOF
+	test_cmp dry-run-cc.expected dry-run-cc.out
+'
+test_done
diff --git a/t/t2714-python-cli-batch.t b/t/t2714-python-cli-batch.t
new file mode 100755
index 000000000000..8235aae2c08b
--- /dev/null
+++ b/t/t2714-python-cli-batch.t
@@ -0,0 +1,267 @@
+#!/bin/sh
+
+test_description='flux batch specific tests'
+
+. $(dirname $0)/sharness.sh
+
+
+# Start an instance with 16 cores across 4 ranks
+export TEST_UNDER_FLUX_CORES_PER_RANK=4
+test_under_flux 4 job
+
+flux setattr log-stderr-level 1
+
+NCORES=$(flux kvs get resource.R | flux R decode --count=core)
+test ${NCORES} -gt 4 && test_set_prereq MULTICORE
+
+test_expect_success 'create generic test batch script' '
+	cat <<-EOF >batch-script.sh
+	#!/bin/sh
+	ncores=\$(flux resource list -s all -no {ncores})
+	nnodes=\$(flux resource list -s all -no {nnodes})
+	printf "size=%d nodes=%d\n" \$(flux getattr size) \$nnodes
+	flux run -n \$ncores hostname
+	EOF
+'
+test_expect_success 'flux batch copies script into jobspec' '
+	flux batch -n1 --dry-run batch-script.sh | \
+		jq -j .attributes.system.files.script.data > script.sh &&
+	test_cmp batch-script.sh script.sh
+'
+test_expect_success 'flux batch takes a script on stdin' '
+	flux batch -n1 --dry-run < batch-script.sh | \
+		jq -j .attributes.system.files.script.data > script-stdin.sh &&
+	test_cmp batch-script.sh script.sh
+'
+test_expect_success 'flux batch --wrap option works' '
+	flux batch -n1 --dry-run --wrap foo bar baz | \
+		jq -j .attributes.system.files.script.data >script-wrap.out &&
+	cat <<-EOF >script-wrap.expected &&
+	#!/bin/sh
+	foo bar baz
+	EOF
+	test_cmp script-wrap.expected script-wrap.out
+'
+test_expect_success 'flux batch --wrap option works with --' '
+	flux batch -n1 --dry-run --wrap -- foo --n=2 | \
+		jq -j .attributes.system.files.script.data >script-wrap.out &&
+	cat <<-EOF >script-wrap.expected &&
+	#!/bin/sh
+	foo --n=2
+	EOF
+	test_cmp script-wrap.expected script-wrap.out
+'
+test_expect_success 'flux batch --wrap option works on stdin' '
+	printf "foo\nbar\nbaz\n" | \
+	    flux batch -n1 --dry-run --wrap | \
+		jq -j .attributes.system.files.script.data >stdin-wrap.out &&
+	cat <<-EOF >stdin-wrap.expected &&
+	#!/bin/sh
+	foo
+	bar
+	baz
+	EOF
+	test_cmp stdin-wrap.expected stdin-wrap.out
+'
+test_expect_success 'flux batch fails for binary file' '
+	test_expect_code 1 flux batch -n1 $(which hostname)
+'
+test_expect_success 'flux batch fails for file without she-bang' '
+	cat <<-EOF >invalid-script.sh &&
+	flux run hostname
+	EOF
+	test_expect_code 1 flux batch -n1 invalid-script.sh
+'
+test_expect_success 'flux batch fails if -N > -n' '
+	test_expect_code 1 flux batch -N4 -n1 --wrap hostname
+'
+test_expect_success 'flux batch -N2 requests 2 nodes exclusively' '
+	flux batch -N2 --wrap --dry-run hostname | \
+		jq -S ".resources[0]" | \
+		jq -e ".type == \"node\" and .exclusive"
+'
+test_expect_success 'flux batch --exclusive works' '
+	flux batch -N1 -n1 --exclusive --wrap --dry-run hostname | \
+		jq -S ".resources[0]" | \
+		jq -e ".type == \"node\" and .exclusive"
+'
+test_expect_success NO_ASAN 'flux batch: submit a series of jobs' '
+	id1=$(flux batch --flags=waitable -n1 batch-script.sh) &&
+	id2=$(flux batch --flags=waitable -n4 batch-script.sh) &&
+	id3=$(flux batch --flags=waitable -N2 -n4 batch-script.sh) &&
+	flux resource list &&
+	flux jobs &&
+	id4=$(flux batch --flags=waitable -N2 -n2 -x batch-script.sh) &&
+	id5=$(flux batch --flags=waitable -N2 batch-script.sh) &&
+	run_timeout 180 flux job wait --verbose --all
+'
+test_expect_success NO_ASAN 'flux batch: job results are expected' '
+	test_debug "grep . flux-*.out" &&
+	grep "size=1 nodes=1" flux-${id1}.out &&
+	grep "size=1 nodes=1" flux-${id2}.out &&
+	grep "size=2 nodes=2" flux-${id3}.out &&
+	grep "size=2 nodes=2" flux-${id4}.out &&
+	grep "size=2 nodes=2" flux-${id5}.out
+'
+test_expect_success MULTICORE 'flux batch: exclusive flag worked' '
+	test $(flux job info ${id4} R | flux R decode --count=core) -gt 2 &&
+	test $(flux job info ${id5} R | flux R decode --count=core) -gt 2
+'
+
+test_expect_success 'flux batch: --output=kvs directs output to kvs' '
+	id=$(flux batch -n1 --flags=waitable --output=kvs batch-script.sh) &&
+	run_timeout 180 flux job attach $id > kvs-output.log 2>&1 &&
+	test_debug "cat kvs-output.log" &&
+	grep "size=1 nodes=1" kvs-output.log
+'
+test_expect_success 'flux batch: --broker-opts works' '
+	id=$(flux batch -n1 --flags=waitable \
+	     --broker-opts=-v batch-script.sh) &&
+	id2=$(flux batch -n1 --flags=waitable \
+	     --broker-opts=-v,-v batch-script.sh) &&
+	run_timeout 180 flux job wait $id &&
+	test_debug "cat flux-${id}.out" &&
+	grep "boot: rank=0 size=1" flux-${id}.out &&
+	run_timeout 180 flux job wait $id2 &&
+	grep "boot: rank=0 size=1" flux-${id2}.out &&
+	grep "entering event loop" flux-${id2}.out
+'
+test_expect_success 'flux batch: critical-ranks attr is set on all ranks' '
+	id=$(flux batch -N4 \
+		--output=critical-ranks.out \
+		--error=critical-ranks.err \
+		--broker-opts=-Stbon.fanout=2 \
+		--wrap flux exec flux getattr broker.critical-ranks) &&
+	flux job status $id &&
+	test_debug "cat critical-ranks.out" &&
+	test_debug "cat critical-ranks.err" &&
+	cat <<-EOF >critical-ranks.expected &&
+	0-1
+	0-1
+	0-1
+	0-1
+	EOF
+	test_cmp critical-ranks.expected critical-ranks.out
+'
+test_expect_success 'flux batch: user can set broker.critical-ranks' '
+	id=$(flux batch -N4 \
+		--output=critical-ranks2.out \
+		--error=critical-ranks2.err \
+		--broker-opts=-Sbroker.critical-ranks=0 \
+		--wrap flux exec flux getattr broker.critical-ranks) &&
+	flux job status $id &&
+	test_debug "cat critical-ranks2.out" &&
+	test_debug "cat critical-ranks2.err" &&
+	cat <<-EOF >critical-ranks2.expected &&
+	0
+	0
+	0
+	0
+	EOF
+	test_cmp critical-ranks2.expected critical-ranks2.out
+'
+test_expect_success 'flux batch: flux can bootstrap without broker.mapping' '
+	id=$(flux batch -N4 -o pmi-simple.nomap \
+		--wrap flux resource info) &&
+	flux job status $id
+'
+test_expect_success 'flux batch: sets mpi=none by default' '
+	flux batch -N1 --dry-run --wrap hostname | \
+		jq -e ".attributes.system.shell.options.mpi = \"none\""
+'
+test_expect_success 'flux batch: mpi option can be overridden' '
+	flux batch -o mpi=foo -N1 --dry-run --wrap hostname | \
+		jq -e ".attributes.system.shell.options.mpi = \"foo\""
+'
+test_expect_success 'flux batch: MPI env vars are not set in batch script' '
+	unset OMPI_MCA_pmix &&
+	id=$(flux batch -N1 --output=envtest.out --wrap printenv) &&
+	flux job status $id &&
+	test_must_fail grep OMPI_MCA_pmix envtest.out
+'
+test_expect_success 'flux batch: --dump works' '
+	id=$(flux batch -N1 --dump \
+		--flags=waitable --wrap true) &&
+	run_timeout 180 flux job wait $id &&
+	tar tvf flux-${id}-dump.tgz
+'
+test_expect_success 'flux batch: --dump=FILE works' '
+	id=$(flux batch -N1 --dump=testdump.tgz \
+		--flags=waitable --wrap true) &&
+	run_timeout 180 flux job wait $id &&
+	tar tvf testdump.tgz
+'
+test_expect_success 'flux batch: --dump=FILE works with mustache' '
+	id=$(flux batch -N1 --dump=testdump-{{id}}.tgz \
+		--flags=waitable --wrap true) &&
+	run_timeout 180 flux job wait $id &&
+	tar tvf testdump-${id}.tgz
+'
+test_expect_success 'flux batch: supports directives in script' '
+	cat <<-EOF >directives.sh &&
+	#!/bin/sh
+	# flux: -n1
+	# flux: --job-name=test-name
+	flux resource list
+	EOF
+	flux batch --dry-run directives.sh > directives.json &&
+	jq -e ".attributes.system.job.name == \"test-name\"" < directives.json
+'
+test_expect_success 'flux batch: cmdline overrides directives' '
+	cat <<-EOF >directives2.sh &&
+	#!/bin/sh
+	# flux: -n1
+	# flux: --job-name=test-name
+	flux resource list
+	EOF
+	flux batch --dry-run --job-name=foo directives2.sh \
+	  > directives2.json &&
+	jq -e ".attributes.system.job.name == \"foo\"" < directives2.json
+'
+test_expect_success 'flux batch: bad argument in directive is caught' '
+	cat <<-EOF >directives3.sh &&
+	#!/bin/sh
+	# flux: -n1
+	# flux: --bad-arg
+	date; hostname
+	EOF
+	test_must_fail flux batch --dry-run directives3.sh >d3.out 2>&1 &&
+	test_debug "cat d3.out" &&
+	grep "argument parsing failed at directives3.sh line 3" d3.out
+'
+test_expect_success 'flux batch: shell parsing error is caught' '
+	cat <<-EOF >directives4.sh &&
+	#!/bin/sh
+	# flux: --job-name=" name
+	date; hostname
+	EOF
+	test_must_fail flux batch --dry-run directives4.sh >d4.out 2>&1 &&
+	test_debug "cat d4.out" &&
+	grep "directives4.sh: line 2" d4.out
+'
+test_expect_success 'flux batch: file can be added via directives' '
+	cat <<-EOF >directives5.sh &&
+	#!/bin/sh
+	# flux: -n1
+	# flux: --add-file=foo="""
+	# flux: This is a test file
+	# flux: """
+	cat \$FLUX_JOB_TMPDIR/foo
+	EOF
+	flux batch --dry-run directives5.sh >d5.json &&
+	jq -e ".attributes.system.files.foo.data == \"This is a test file\n\"" \
+		<d5.json
+'
+test_expect_success 'flux batch: multiline --add-file requires name=' '
+	cat <<-EOF >directives6.sh &&
+	#!/bin/sh
+	# flux: -n1
+	# flux: --add-file="""
+	# flux: This is a test file
+	# flux: """
+	cat \$FLUX_JOB_TMPDIR/foo
+	EOF
+	test_must_fail flux batch --dry-run directives6.sh >d6.out 2>&1 &&
+	grep "file name missing" d6.out
+'
+test_done
diff --git a/t/t2715-python-cli-cancel.t b/t/t2715-python-cli-cancel.t
new file mode 100755
index 000000000000..5b49e34f4a05
--- /dev/null
+++ b/t/t2715-python-cli-cancel.t
@@ -0,0 +1,139 @@
+#!/bin/sh
+
+test_description='Test flux cancel command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4
+
+# Set CLIMain log level to logging.DEBUG (10), to enable stack traces
+export FLUX_PYCLI_LOGLEVEL=10
+
+flux setattr log-stderr-level 1
+
+runas() {
+	userid=$1 && shift
+	FLUX_HANDLE_USERID=$userid FLUX_HANDLE_ROLEMASK=0x2 "$@"
+}
+
+test_expect_success 'flux cancel fails with bad FLUX_URI' '
+	validjob=$(flux submit sleep 30) &&
+	(FLUX_URI=/wrong test_must_fail flux cancel ${validjob}) &&
+	flux cancel $validjob
+'
+test_expect_success 'flux cancel fails with unknown job id' '
+	test_must_fail flux cancel 0
+'
+test_expect_success 'flux cancel fails with unknown job ids' '
+	test_must_fail flux cancel 0 f123
+'
+test_expect_success 'flux cancel fails with no args' '
+	test_must_fail flux cancel
+'
+test_expect_success 'flux cancel fails with invalid jobid' '
+	test_must_fail flux cancel foo
+'
+test_expect_success 'flux cancel fails with invalid user' '
+	test_must_fail flux cancel --user=1badusername12345
+'
+test_expect_success 'flux cancel fails with invalid option' '
+	test_must_fail flux cancel --meep foo
+'
+test_expect_success 'flux cancel JOBID works' '
+	id=$(flux submit sleep 100) &&
+	flux cancel ${id} &&
+	flux job wait-event -t 30 ${id} exception >cancel1.out &&
+	grep "cancel" cancel1.out &&
+	grep "severity\=0" cancel1.out
+'
+test_expect_success 'flux cancel --message works' '
+	id=$(flux submit sleep 100) &&
+	flux cancel --message=meepmessage ${id} &&
+	flux job wait-event -t 30 ${id} exception >cancel2.out &&
+	grep "meepmessage" cancel2.out
+'
+test_expect_success 'flux cancel --all --dry-run works' '
+	count=4 &&
+	flux submit --cc=1-$count sleep 60 &&
+	flux cancel --all --dry-run 2>cancelall_n.err &&
+	cat <<-EOT >cancelall_n.exp &&
+	flux-cancel: Would cancel ${count} jobs
+	EOT
+	test_cmp cancelall_n.exp cancelall_n.err
+'
+test_expect_success 'flux cancel --all works' '
+	count=$(flux job list | wc -l) &&
+	flux cancel --all 2>cancelall.err &&
+	cat <<-EOT >cancelall.exp &&
+	flux-cancel: Canceled ${count} jobs (0 errors)
+	EOT
+	test_cmp cancelall.exp cancelall.err
+'
+test_expect_success 'flux cancel --all works with message' '
+	count=4 &&
+	flux submit --cc=1-$count sleep 60 &&
+	flux cancel --all --message="cancel all" 2>cancelall.err &&
+	cat <<-EOT >cancelall.exp &&
+	flux-cancel: Canceled ${count} jobs (0 errors)
+	EOT
+	test_cmp cancelall.exp cancelall.err &&
+	flux job wait-event -t 5 $(flux job last) exception >exception.out &&
+	grep "cancel all" exception.out
+'
+test_expect_success 'the queue is empty' '
+	run_timeout 180 flux queue drain
+'
+test_expect_success 'flux cancel --all --user all fails for guest' '
+	id=$(($(id -u)+1)) &&
+	test_must_fail runas ${id} \
+	        flux cancel --all --user=all 2> cancelall_all_guest.err &&
+	grep "guests can only raise exceptions on their own jobs" \
+		cancelall_all_guest.err
+'
+test_expect_success 'flux- cancel --all --user <guest uid> works for guest' '
+	id=$(($(id -u)+1)) &&
+	runas ${id} flux cancel --all --user=${id}
+'
+test_expect_success 'flux cancel --state with unknown state fails' '
+        test_must_fail flux cancel --states=FOO 2>cancelall_bs.err &&
+	grep "Invalid state FOO specified" cancelall_bs.err
+'
+test_expect_success 'flux cancel --state=pending works' '
+	runid=$(flux submit -n $(flux resource list -no {ncores}) sleep 60) &&
+	flux submit --cc=1-4 sleep 30 &&
+	flux cancel --states=pending --dry-run 2>pending.err &&
+	grep "Would cancel 4 jobs" pending.err &&
+	flux cancel --states=pending &&
+	test $(flux jobs -f pending -no {id} | wc -l) -eq 0 &&
+	test $(flux jobs -f run -no {id} | wc -l) -eq 1
+'
+test_expect_success 'flux cancel --state=run works' '
+	flux cancel --states=run --dry-run 2>run.err &&
+	grep "Would cancel 1 job" run.err &&
+	flux cancel --states=run
+'
+test_expect_success 'flux cancel reports 0 jobs with no match' '
+	flux cancel --states=run --dry-run 2>nomatch.err &&
+	test_debug "cat nomatch.err" &&
+	grep "Matched 0 jobs" nomatch.err
+'
+test_expect_success 'flux cancel can operate on multiple jobs' '
+	ids=$(flux submit --cc=1-3 sleep 600) &&
+        flux cancel -m "cancel multiple jobids" ${ids} &&
+        for id in ${ids}; do
+                flux job wait-event -t 30 ${id} exception >exception.out &&
+                grep multiple exception.out
+        done
+'
+test_expect_success 'flux cancel does nothing with --dry-run and multiple jobs' '
+	flux cancel -n f1 f2 f3 2>multiple_n.out &&
+	grep "Would cancel 3 jobs" multiple_n.out
+'
+test_expect_success 'flux cancel records errors with multiple jobids' '
+	ids2=$(flux submit --cc=1-3 sleep 600) &&
+        test_must_fail \
+		flux cancel -m "cancel multiple jobids" ${ids2} ${ids} \
+		2>multiple-errors.out &&
+	grep "inactive" multiple-errors.out
+'
+test_done
diff --git a/t/t2716-python-cli-batch-conf.t b/t/t2716-python-cli-batch-conf.t
new file mode 100755
index 000000000000..ccbf0971fc87
--- /dev/null
+++ b/t/t2716-python-cli-batch-conf.t
@@ -0,0 +1,218 @@
+#!/bin/sh
+
+test_description='flux batch --conf tests'
+
+. $(dirname $0)/sharness.sh
+
+
+# Start an instance with 16 cores across 4 ranks
+export TEST_UNDER_FLUX_CORES_PER_RANK=4
+test_under_flux 4 job
+
+flux setattr log-stderr-level 1
+
+NCORES=$(flux kvs get resource.R | flux R decode --count=core)
+test ${NCORES} -gt 4 && test_set_prereq MULTICORE
+
+test_expect_success 'flux-batch --quiet works' '
+	flux batch --quiet -n1 --wrap hostname >quiet.out &&
+	test_must_be_empty quiet.out
+'
+test_expect_success 'flux-batch: create test configs' '
+	cat <<-EOF >conf.json &&
+	{"resource": {"noverify": true}}
+	EOF
+	cat <<-EOF >conf.toml
+	[resource]
+	noverify = true
+	EOF
+'
+test_expect_success 'flux-batch --conf=FILE works with TOML file' '
+	flux batch --conf=conf.toml -n1 --dry-run --wrap hostname \
+		>conf1.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify" \
+		<conf1.json
+'
+test_expect_success 'flux-batch --conf=FILE works with JSON file' '
+	flux batch --conf=conf.json -n1 --dry-run --wrap hostname \
+		>conf2.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify" \
+		<conf2.json
+'
+test_expect_success 'flux-batch --conf=FILE errors if FILE does not exist' '
+	test_must_fail flux batch --conf=/nofile.json -n1 --dry-run \
+		--wrap hostname
+'
+test_expect_success 'flux-batch --conf=FILE detects invalid TOML syntax' '
+	cat <<-EOF >conf-bad.toml &&
+	[resource]
+	noverify = foo
+	EOF
+	test_must_fail flux batch --conf=conf-bad.toml -n1 --dry-run \
+		--wrap hostname >parse-error.out 2>&1 &&
+	test_debug "cat parse-error.out" &&
+	grep "parse error" parse-error.out
+'
+test_expect_success 'flux-batch --conf=FILE detects invalid JSON syntax' '
+	cat <<-EOF >conf-bad.json &&
+	{"resource": {"noverify": foo}}
+	EOF
+	test_must_fail flux batch --conf=conf-bad.json -n1 --dry-run \
+		--wrap hostname >parse-error2.out 2>&1 &&
+	test_debug "cat parse-error2.out" &&
+	grep "parse error" parse-error2.out
+'
+test_expect_success 'flux-batch --conf=noexist fails' '
+	test_must_fail flux batch --conf=noexist -n1 --dry-run \
+		--wrap hostname >noexist.out 2>&1 &&
+	test_debug "cat noexist.out" &&
+	grep "named config.*not found" noexist.out
+'
+test_expect_success 'flux-batch --conf=NAME works with XDG_CONFIG_HOME' '
+	mkdir -p d/flux/config &&
+	cat <<-EOF >d/flux/config/test.toml &&
+	[resource]
+	noverify = true
+	exclude = "0"
+	EOF
+	XDG_CONFIG_HOME="$(pwd)/d" \
+		flux batch --conf=test -n1 --dry-run --wrap hostname \
+		  >named-test.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify" \
+		<named-test.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.exclude == \"0\"" \
+		<named-test.json
+'
+test_expect_success 'flux-batch --conf=NAME works with JSON config' '
+	cat <<-EOF >d/flux/config/test2.json &&
+	{"resource": {"exclude": "1"}}
+	EOF
+	XDG_CONFIG_HOME="$(pwd)/d" \
+		flux batch --conf=test2 -n1 --dry-run --wrap hostname \
+		  >named-json.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.exclude == \"1\"" \
+		<named-json.json
+'
+test_expect_success 'flux-batch --conf=NAME works with XDG_CONFIG_DIRS' '
+	mkdir -p d2/flux/config &&
+	cat <<-EOF >d2/flux/config/test.toml &&
+	[resource]
+	noverify = false
+	exclude = "1"
+	EOF
+	XDG_CONFIG_DIRS="$(pwd)/d2:$(pwd)/d" \
+		flux batch --conf=test -n1 --dry-run --wrap hostname \
+		  >named2-test.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify == false" \
+		<named2-test.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.exclude == \"1\"" \
+		<named2-test.json
+'
+test_expect_success 'flux-batch parse error in named config is caught' '
+	mkdir -p d3/flux/config &&
+	cat <<-EOF >d3/flux/config/test.toml &&
+	[resource]
+	noverify = foo
+	EOF
+	XDG_CONFIG_DIRS="$(pwd)/d3:$(pwd)/d2:$(pwd)/d" \
+		test_must_fail \
+		  flux batch --conf=test -n1 --dry-run --wrap hostname \
+		    >named-parse-error.out 2>&1 &&
+	test_debug "cat named-parse-error.out" &&
+	grep 'conf=test:.*test.toml' named-parse-error.out
+'
+test_expect_success 'flux-batch --conf=KEY=VAL works' '
+	flux batch --conf=resource.noverify=true -n1 --dry-run \
+		--wrap hostname >conf3.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify" \
+		<conf3.json
+'
+test_expect_success 'flux-batch --conf=KEY=VAL multiple use is merged' '
+	flux batch --conf=resource.noverify=true \
+		--conf=resource.exclude=0 -n1 --dry-run \
+		--wrap hostname >conf4.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify" \
+		<conf4.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.exclude == 0" \
+		<conf4.json
+'
+test_expect_success 'flux-batch --conf=KEY=VAL VAL can be JSON' '
+	flux batch --conf=foo={} \
+		--conf=resource.exclude=0 -n1 --dry-run \
+		--wrap hostname >conf4.1.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.foo == {}" \
+		<conf4.1.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.exclude == 0" \
+		<conf4.1.json
+'
+test_expect_success 'flux-batch --conf=FILE --conf=KEY=VAL works' '
+	flux batch --conf=conf.toml \
+		--conf=resource.exclude=0 -n1 --dry-run \
+		--wrap hostname >conf5.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify" \
+		<conf5.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.exclude == 0" \
+		<conf5.json
+'
+test_expect_success 'flux-batch multiline --conf directive works' '
+	cat <<-EOF >batch.sh &&
+	#!/bin/sh
+	# flux: -n1
+	# flux: --conf="""
+	# flux: [resource]
+	# flux: noverify = true
+	# flux: """
+	flux config get
+	EOF
+	flux batch --dry-run batch.sh >conf6.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify" \
+		<conf6.json
+'
+test_expect_success 'flux-batch multiline --conf directive in JSON works' '
+	cat <<-EOF >batch2.sh &&
+	#!/bin/sh
+	# flux: -n1
+	# flux: --conf="""
+	# flux: {"resource": {"noverify": true}}
+	# flux: """
+	flux config get
+	EOF
+	flux batch --dry-run batch.sh >conf7.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify" \
+		<conf7.json
+'
+test_expect_success 'flux-batch --conf directive syntax error detected' '
+	cat <<-EOF >bad-batch.sh &&
+	#!/bin/sh
+	# flux: -n1
+	# flux: --conf="""
+	# flux: [resource]
+	# flux: noverify = foo
+	# flux: """
+	flux config get
+	EOF
+	test_must_fail flux batch --dry-run bad-batch.sh >ml-syntax.out 2>&1 &&
+	test_debug "cat ml-syntax.out" &&
+	grep "failed to parse" ml-syntax.out
+'
+test_expect_success 'flux-batch multiline --conf + --conf=KEY=VAL works' '
+	flux batch --conf=resource.exclude=0 --dry-run batch.sh >conf8.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify" \
+		<conf8.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.exclude == 0" \
+		<conf8.json
+'
+test_expect_success 'flux-batch multiline --conf + --conf=KEY=VAL override works' '
+	flux batch --conf=resource.noverify=false --dry-run batch.sh \
+		>conf9.json &&
+	jq -e ".attributes.system.files[\"conf.json\"].data.resource.noverify == false" \
+		<conf9.json
+'
+test_expect_success 'flux-batch --conf end-to-end test' '
+	jobid=$(flux batch --conf=resource.exclude=\"0\" \
+		--output=batchtest.out --error=batchtest.err batch.sh) &&
+	flux job wait-event $jobid clean &&
+	jq -e ".resource.noverify" <batchtest.out &&
+	jq -e ".resource.exclude == \"0\"" <batchtest.out
+'
+test_done
diff --git a/t/t2800-jobs-cmd.t b/t/t2800-jobs-cmd.t
index 7a6ac4df78ce..2577bc6f3f84 100755
--- a/t/t2800-jobs-cmd.t
+++ b/t/t2800-jobs-cmd.t
@@ -2,15 +2,15 @@
 
 test_description='Test flux jobs command'
 
-. $(dirname $0)/sharness.sh
+. $(dirname $0)/job-list/job-list-helper.sh
 
-#  Set path to jq(1)
-#
-jq=$(which jq 2>/dev/null)
-test -n "$jq" && test_set_prereq HAVE_JQ
+. $(dirname $0)/sharness.sh
 
 test_under_flux 4 job
 
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py --line-buffer -f asciicast"
+PLUGINPATH=${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs
+
 # submit a whole bunch of jobs for job list testing
 #
 # - the first loop of job submissions are intended to have some jobs run
@@ -19,80 +19,165 @@ test_under_flux 4 job
 # - the last job submissions are intended to get a create a set of
 #   pending jobs, because jobs from the second loop have taken all resources
 # - job ids are stored in files in the order we expect them to be listed
-#   - pending jobs - by priority (highest first)
+#   - pending jobs - by priority (highest first), job id (smaller first)
 #   - running jobs - by start time (most recent first)
 #   - inactive jobs - by completion time (most recent first)
 #
-# the job-info module has eventual consistency with the jobs stored in
-# the job-manager's queue.  To ensure no raciness in tests, we spin
-# until all of the pending jobs have reached SCHED state, running jobs
-# have reached RUN state, and inactive jobs have reached INACTIVE
-# state.
-#
 
-wait_states() {
-        local i=0
-        while ( [ "$(flux jobs --suppress-header --states=sched | wc -l)" != "6" ] \
-                || [ "$(flux jobs --suppress-header --states=run | wc -l)" != "8" ] \
-                || [ "$(flux jobs --suppress-header --states=inactive | wc -l)" != "4" ]) \
-               && [ $i -lt 50 ]
-        do
-                sleep 0.1
-                i=$((i + 1))
-        done
-        if [ "$i" -eq "50" ]
-        then
-            return 1
-        fi
-        return 0
+export FLUX_PYCLI_LOGLEVEL=10
+
+fj_wait_event() {
+  flux job wait-event --timeout=20 "$@"
+}
+
+listjobs() {
+	${FLUX_BUILD_DIR}/t/job-manager/list-jobs \
+	    | $jq .id \
+	    | flux job id --to=f58
 }
 
+test_expect_success 'configure testing queues' '
+	flux config load <<-EOT &&
+	[policy]
+	jobspec.defaults.system.queue = "defaultqueue"
+	[queues.defaultqueue]
+	[queues.queue1]
+	[queues.queue2]
+	EOT
+	flux queue start --all
+'
+
+test_expect_success 'create helper job submission script' '
+	cat >sleepinf.sh <<-EOT &&
+	#!/bin/sh
+	echo "job started"
+	sleep inf
+	EOT
+	chmod +x sleepinf.sh
+'
+
 test_expect_success 'submit jobs for job list testing' '
-        for i in `seq 1 4`; do \
-            jobid=`flux mini submit hostname`; \
-            flux job wait-event $jobid clean; \
-            echo $jobid >> job_ids1.out; \
-        done &&
-        tac job_ids1.out > job_ids_inactive.out &&
-        for i in `seq 1 8`; do \
-            jobid=`flux mini submit sleep 600`; \
-            flux job wait-event $jobid start; \
-            echo $jobid >> job_ids2.out; \
-        done &&
-        tac job_ids2.out > job_ids_running.out &&
-        flux mini submit --priority=30 sleep 600 >> job_ids_pending.out &&
-        flux mini submit --priority=25 sleep 600 >> job_ids_pending.out &&
-        flux mini submit --priority=20 sleep 600 >> job_ids_pending.out &&
-        flux mini submit --priority=15 sleep 600 >> job_ids_pending.out &&
-        flux mini submit --priority=10 sleep 600 >> job_ids_pending.out &&
-        flux mini submit --priority=5 sleep 600 >> job_ids_pending.out &&
-        wait_states
+	#  Create `hostname` and `sleep` jobspec
+	#  N.B. Used w/ `flux job submit` for serial job submission
+	#  for efficiency (vs serial `flux submit`.
+	#
+	flux submit --dry-run \
+		--queue=queue1 \
+		hostname >hostname.json &&
+	flux submit --dry-run \
+		--time-limit=5m \
+		--queue=queue2 \
+		sleep 600 > sleeplong.json &&
+	#
+	#  Submit jobs that will complete
+	#
+	for i in $(seq 0 3); do
+		flux job submit hostname.json >> inactiveids
+		fj_wait_event `tail -n 1 inactiveids` clean
+	done &&
+	#
+	#  Currently all inactive ids are "completed"
+	#
+	tac inactiveids > completed.ids &&
+	#
+	#  Run a job that will fail, copy its JOBID to both inactive and
+	#   failed lists.
+	#
+	! jobid=`flux submit --wait nosuchcommand` &&
+	echo $jobid >> inactiveids &&
+	flux job id $jobid > failed_exec.ids &&
+	echo $jobid > failedids &&
+	#
+	#  Run a job that we will end with a signal, copy its JOBID to both inactive and
+	#   failed and terminated lists.
+	#
+	# N.B. sleepinf.sh and wait-event on job data to workaround
+	# rare job startup race.  See #5210
+	#
+	jobid=`flux submit ./sleepinf.sh` &&
+	flux job wait-event -W -p guest.output $jobid data &&
+	flux job kill $jobid &&
+	fj_wait_event $jobid clean &&
+	echo $jobid >> inactiveids &&
+	flux job id $jobid > terminated.ids &&
+	flux job id $jobid >> failedids &&
+	#
+	#  Run a job that we will end with a user exception, copy its JOBID to both
+	#	inactive and failed and exception lists.
+	#
+	# N.B. sleepinf.sh and wait-event on job data to workaround
+	# rare job startup race.  See #5210
+	#
+	jobid=`flux submit ./sleepinf.sh` &&
+	flux job wait-event -W -p guest.output $jobid data &&
+	flux job raise --type=myexception --severity=0 -m "myexception" $jobid &&
+	fj_wait_event $jobid clean &&
+	echo $jobid >> inactiveids &&
+	flux job id $jobid > exception.ids &&
+	flux job id $jobid >> failedids &&
+	#
+	#  Run a job that will timeout, copy its JOBID to both inactive and
+	#   timeout lists.
+	#
+	jobid=`flux submit --time-limit=0.5s sleep 30` &&
+	echo $jobid >> inactiveids &&
+	echo $jobid > timeout.ids &&
+	fj_wait_event ${jobid} clean &&
+	#
+	#  Submit 8 sleep jobs to fill up resources
+	#
+	for i in $(seq 0 7); do
+		flux job submit sleeplong.json >> runids
+	done &&
+	tac runids > run.ids &&
+	#
+	#  Submit a set of jobs with non-default urgencies
+	#
+	for u in 31 25 20 15 10 5 0; do
+		flux job submit --urgency=$u sleeplong.json >> sched.ids
+	done &&
+	listjobs > active.ids &&
+	#
+	#  Submit a job and cancel it
+	#
+	# N.B. no need to handle issue #5210 here, the job will not
+	# run due to lack of resources.
+	#
+	jobid=`flux submit --job-name=canceledjob sleep 30` &&
+	fj_wait_event $jobid depend &&
+	flux cancel -m "mecanceled" $jobid &&
+	fj_wait_event $jobid clean &&
+	echo $jobid >> inactiveids &&
+	echo $jobid > canceled.ids &&
+	tac failedids > failed.ids &&
+	tac inactiveids > inactive.ids &&
+	cat inactive.ids active.ids >> all.ids &&
+	#
+	#  The job-list module has eventual consistency with the jobs stored in
+	#  the job-manager queue.  To ensure no raciness in tests, ensure
+	#  jobs above have reached expected states in job-list before continuing.
+	#
+	flux job list-ids --wait-state=sched $(job_list_state_ids sched) > /dev/null &&
+	flux job list-ids --wait-state=run $(job_list_state_ids run) > /dev/null &&
+	flux job list-ids --wait-state=inactive $(job_list_state_ids inactive) > /dev/null
 '
 
 #
 # basic tests
 #
-
-# careful with counting b/c of header
-test_expect_success 'flux-jobs default output works' '
-        count=`flux jobs | wc -l` &&
-        test $count -eq 15 &&
-        count=`flux jobs | grep " SCHED " | wc -l` &&
-        test $count -eq 6 &&
-        count=`flux jobs | grep " RUN " | wc -l` &&
-        test $count -eq 8 &&
-        count=`flux jobs | grep " INACTIVE " | wc -l` &&
-        test $count -eq 0
+test_expect_success 'flux-jobs --no-header works' '
+	count=`flux jobs --no-header | wc -l` &&
+	test $count -eq $(job_list_state_count active)
 '
-
-test_expect_success 'flux-jobs --suppress-header works' '
-        count=`flux jobs --suppress-header | wc -l` &&
-        test $count -eq 14
-'
-
-test_expect_success 'flux-jobs: header included with custom formats' '
-	flux jobs --format={id} &&
-	test "$(flux jobs --format={id} | head -1)" = "JOBID"
+test_expect_success 'flux-jobs default output works' '
+	flux jobs -n > default.out &&
+	test $(wc -l < default.out) -eq $(job_list_state_count active) &&
+	test $(grep -c "  S " default.out) -eq $(job_list_state_count sched) &&
+	test $(grep -c "  R " default.out) -eq $(job_list_state_count run) &&
+	test $(grep -c " CD " default.out) -eq 0 &&
+	test $(grep -c " CA " default.out) -eq 0 &&
+	test $(grep -c "  F " default.out) -eq 0 &&
+	test $(grep -c " TO " default.out) -eq 0
 '
 
 test_expect_success 'flux-jobs: custom format with numeric spec works' '
@@ -101,94 +186,323 @@ test_expect_success 'flux-jobs: custom format with numeric spec works' '
 	grep T_RUN format-test.out
 '
 
+test_expect_success 'flux-jobs: collapsible fields work' '
+	flux jobs -ao "{id.f58:<12} ?:{exception.type:>8}" >nocollapse.out &&
+	flux jobs -f running,completed \
+		 -o "{id.f58:<12} ?:{exception.type:>8}"   >collapsed.out &&
+	test_debug "head -n1 nocollapse.out" &&
+	test_debug "head -n1 collapsed.out" &&
+	grep EXCEPTION-TYPE nocollapse.out &&
+	test_must_fail grep EXCEPTION-TYPE collapsed.out
+'
+# Note longest name from above should be 'nosuchcommand'
+# To ensure field was expanded to this width, ensure NAME header is right
+# justified:
+test_expect_success 'flux-jobs: expandable fields work' '
+	flux jobs -ao "+:{name:>1}" >expanded.out &&
+	grep "^ *NAME" expanded.out &&
+	grep "^ *sleep" expanded.out &&
+	grep "nosuchcommand" expanded.out
+'
+test_expect_success 'flux-jobs: specified width overrides expandable field' '
+	flux jobs -ao "+:{name:>16}" >expanded2.out &&
+	test_debug "cat expanded2.out" &&
+	grep "^   nosuchcommand" expanded2.out
+'
+test_expect_success 'flux-jobs: collapsible+expandable fields work' '
+	flux jobs -ao "{id.f58:<12} ?+:{exception.type:>1}" >both.out &&
+	flux jobs -f running,completed \
+		 -o "{id.f58:<12} ?+:{exception.type:>1}" >both-collapsed.out &&
+	test_debug "head -n1 both.out" &&
+	test_debug "head -n1 both-collapsed.out" &&
+	grep EXCEPTION-TYPE both.out &&
+	test_must_fail grep EXCEPTION-TYPE both-collapsed.out
+'
+test_expect_success 'flux-jobs: request indication of truncation works' '
+	flux jobs -n -c1 -ano "{id.f58:<5.5+}" | grep + &&
+	flux jobs -n -c1 -ano "{id.f58:<5.5h+}" | grep + &&
+	flux jobs -n -c1 -ano "{id.f58:<20.20h+}" > notruncate.out &&
+	test_must_fail grep + notruncate.out
+'
+
 # TODO: need to submit jobs as another user and test -A again
 test_expect_success 'flux-jobs -a and -A works' '
-        count=`flux jobs --suppress-header -a | wc -l` &&
-        test $count -eq 18 &&
-        count=`flux jobs --suppress-header -a | wc -l` &&
-        test $count -eq 18
-'
-
-# Recall pending = depend & sched, running = run & cleanup,
-#  active = pending & running
-test_expect_success 'flux-jobs --states works' '
-        count=`flux jobs --suppress-header --states=depend | wc -l` &&
-        test $count -eq 0 &&
-        count=`flux jobs --suppress-header --states=sched | wc -l` &&
-        test $count -eq 6 &&
-        count=`flux jobs --suppress-header --states=pending | wc -l` &&
-        test $count -eq 6 &&
-        count=`flux jobs --suppress-header --states=run | wc -l` &&
-        test $count -eq 8 &&
-        count=`flux jobs --suppress-header --states=cleanup | wc -l` &&
-        test $count -eq 0 &&
-        count=`flux jobs --suppress-header --states=running | wc -l` &&
-        test $count -eq 8 &&
-        count=`flux jobs --suppress-header --states=inactive | wc -l` &&
-        test $count -eq 4 &&
-        count=`flux jobs --suppress-header --states=pending,running | wc -l` &&
-        test $count -eq 14 &&
-        count=`flux jobs --suppress-header --states=sched,run | wc -l` &&
-        test $count -eq 14 &&
-        count=`flux jobs --suppress-header --states=active | wc -l` &&
-        test $count -eq 14 &&
-        count=`flux jobs --suppress-header --states=depend,sched,run,cleanup | wc -l` &&
-        test $count -eq 14 &&
-        count=`flux jobs --suppress-header --states=pending,inactive | wc -l` &&
-        test $count -eq 10 &&
-        count=`flux jobs --suppress-header --states=sched,inactive | wc -l` &&
-        test $count -eq 10 &&
-        count=`flux jobs --suppress-header --states=running,inactive | wc -l` &&
-        test $count -eq 12 &&
-        count=`flux jobs --suppress-header --states=run,inactive | wc -l` &&
-        test $count -eq 12 &&
-        count=`flux jobs --suppress-header --states=pending,running,inactive | wc -l` &&
-        test $count -eq 18 &&
-        count=`flux jobs --suppress-header --states=active,inactive | wc -l` &&
-        test $count -eq 18 &&
-        count=`flux jobs --suppress-header --states=depend,cleanup | wc -l` &&
-        test $count -eq 0
-'
-
-test_expect_success 'flux-jobs --states with invalid state fails' '
-        test_must_fail flux jobs --states=foobar 2> invalidstate.err &&
-        grep "Invalid state specified: foobar" invalidstate.err
+	nall=$(job_list_state_count all) &&
+	count=`flux jobs --no-header -a | wc -l` &&
+	test $count -eq $nall &&
+	count=`flux jobs --no-header -a -A | wc -l` &&
+	test $count -eq $nall
+'
+
+test_expect_success 'flux-jobs --since implies -a' '
+	nall=$(job_list_state_count all) &&
+	count=$(flux jobs --no-header --since=0.0 | wc -l) &&
+	test $count -eq $nall
+'
+
+test_expect_success 'flux-jobs --since with --filter does not imply -a' '
+	nfailed=$(job_list_state_count failed) &&
+	count=$(flux jobs -n --since=0.0 -f failed | wc -l) &&
+	test $count -eq $nfailed
+'
+
+test_expect_success 'flux-jobs --since option must specify a time in the past' '
+	test_must_fail flux jobs --since=+1m
+'
+
+# Print the t_inactive timestamp for the 3rd most recently inactive job
+# Then ensure this timestamp limits output to 2 jobs
+test_expect_success 'flux-jobs --since works with timestamp' '
+	ts=$(flux jobs --filter=inactive -no {t_inactive} | sed -n 3p) &&
+	test_debug "flux jobs --filter=inactive --since=${ts}" &&
+	count=$(flux jobs -n --filter=inactive --since=${ts} | wc -l) &&
+	test $count -eq 2
+'
+
+test_expect_success 'flux-jobs --since works with datetime' '
+	nall=$(job_list_state_count all) &&
+	count=$(flux jobs -n --since="a year ago" | wc -l) &&
+	test $count -eq $nall
+'
+
+test_expect_success 'flux-jobs --since works with fsd offset' '
+	nall=$(job_list_state_count all) &&
+	count=$(flux jobs -n --since=-8.8h | wc -l) &&
+	test $count -eq $nall
+'
+
+# -a ignored if --filter specified
+test_expect_success 'flux-jobs -a and --filter=pending' '
+	count=`flux jobs -n -a --filter=pending 2> filter_a.err | wc -l` &&
+	test $count -eq $(job_list_state_count sched) &&
+	grep ignoring filter_a.err
+'
+
+test_expect_success 'flux-jobs --name works' '
+	test_debug "flux jobs -an --name=nosuchcommand" &&
+	test $(flux jobs -an --name=nosuchcommand | wc -l) -eq 1 &&
+	test_debug "flux jobs -an --name=xxyyzz" &&
+	test $(flux jobs -an --name=xxyyzz | wc -l) -eq 0
+'
+
+# in job submissions above: completed jobs should be in queue1, running jobs
+# in queue2
+test_expect_success 'flux-jobs --queue works' '
+	test_debug "flux jobs -an --queue=queue1" &&
+	test $(flux jobs -an --queue=queue1 | wc -l) -eq $(job_list_state_count completed) &&
+	test_debug "flux jobs -an --queue=queue2" &&
+	test $(flux jobs -an --queue=queue2 | wc -l) -eq $(job_list_state_count sched run) &&
+	test_debug "flux jobs -an --queue=foobar" &&
+	test $(flux jobs -an --queue=foobar | wc -l) -eq 0
+'
+test_expect_success 'flux-jobs --queue accepts multiple queues' '
+	test $(flux jobs -anq queue1,queue2 | wc -l) \
+		-eq $(job_list_state_count completed sched run)
+'
+
+# Recall pending = depend | priority | sched, running = run | cleanup,
+#  active = pending | running
+test_expect_success 'flux-jobs --filter works (job states)' '
+	count=`flux jobs --no-header --filter=depend | wc -l` &&
+	test $count -eq 0 &&
+	count=`flux jobs --no-header --filter=priority | wc -l` &&
+	test $count -eq 0 &&
+	count=`flux jobs --no-header --filter=sched | wc -l` &&
+	test $count -eq $(job_list_state_count sched) &&
+	count=`flux jobs --no-header --filter=pending | wc -l` &&
+	test $count -eq $(job_list_state_count sched) &&
+	count=`flux jobs --no-header --filter=run | wc -l` &&
+	test $count -eq $(job_list_state_count run) &&
+	count=`flux jobs --no-header --filter=cleanup | wc -l` &&
+	test $count -eq 0 &&
+	count=`flux jobs --no-header --filter=running | wc -l` &&
+	test $count -eq $(job_list_state_count run) &&
+	count=`flux jobs --no-header --filter=inactive | wc -l` &&
+	test $count -eq $(job_list_state_count inactive) &&
+	count=`flux jobs --no-header --filter=pending,running | wc -l` &&
+	test $count -eq $(job_list_state_count sched run) &&
+	count=`flux jobs --no-header --filter=sched,run | wc -l` &&
+	test $count -eq $(job_list_state_count sched run) &&
+	count=`flux jobs --no-header --filter=active | wc -l` &&
+	test $count -eq $(job_list_state_count active) &&
+	count=`flux jobs --no-header --filter=depend,priority,sched,run,cleanup | wc -l` &&
+	test $count -eq $(job_list_state_count active) &&
+	count=`flux jobs --no-header --filter=pending,inactive | wc -l` &&
+	test $count -eq $(job_list_state_count sched inactive) &&
+	count=`flux jobs --no-header --filter=sched,inactive | wc -l` &&
+	test $count -eq $(job_list_state_count sched inactive) &&
+	count=`flux jobs --no-header --filter=running,inactive | wc -l` &&
+	test $count -eq $(job_list_state_count run inactive) &&
+	count=`flux jobs --no-header --filter=run,inactive | wc -l` &&
+	test $count -eq $(job_list_state_count run inactive) &&
+	count=`flux jobs --no-header --filter=pending,running,inactive | wc -l` &&
+	test $count -eq $(job_list_state_count all) &&
+	count=`flux jobs --no-header --filter=active,inactive | wc -l` &&
+	test $count -eq $(job_list_state_count active inactive) &&
+	count=`flux jobs --no-header --filter=depend,priority,cleanup | wc -l` &&
+	test $count -eq 0
+'
+
+test_expect_success 'flux-jobs --filter works (job results)' '
+	count=`flux jobs --no-header --filter=completed | wc -l` &&
+	test $count -eq $(job_list_state_count completed) &&
+	count=`flux jobs --no-header --filter=failed | wc -l` &&
+	test $count -eq $(job_list_state_count failed) &&
+	count=`flux jobs --no-header --filter=canceled | wc -l` &&
+	test $count -eq $(job_list_state_count canceled) &&
+	count=`flux jobs --no-header --filter=timeout | wc -l` &&
+	test $count -eq $(job_list_state_count timeout) &&
+	count=`flux jobs --no-header --filter=completed,failed | wc -l` &&
+	test $count -eq $(job_list_state_count completed failed) &&
+	count=`flux jobs --no-header --filter=completed,canceled | wc -l` &&
+	test $count -eq $(job_list_state_count completed canceled) &&
+	count=`flux jobs --no-header --filter=completed,timeout | wc -l` &&
+	test $count -eq $(job_list_state_count completed timeout) &&
+	count=`flux jobs --no-header --filter=completed,failed,canceled | wc -l` &&
+	test $count -eq $(job_list_state_count completed failed canceled) &&
+	count=`flux jobs --no-header --filter=completed,failed,timeout | wc -l` &&
+	test $count -eq $(job_list_state_count completed failed timeout) &&
+	count=`flux jobs --no-header --filter=completed,failed,canceled,timeout | wc -l` &&
+	test $count -eq $(job_list_state_count completed failed canceled timeout) &&
+	count=`flux jobs --no-header --filter=pending,completed | wc -l` &&
+	test $count -eq $(job_list_state_count sched completed) &&
+	count=`flux jobs --no-header --filter=pending,failed | wc -l` &&
+	test $count -eq $(job_list_state_count sched failed) &&
+	count=`flux jobs --no-header --filter=pending,canceled | wc -l` &&
+	test $count -eq $(job_list_state_count sched canceled) &&
+	count=`flux jobs --no-header --filter=pending,timeout | wc -l` &&
+	test $count -eq $(job_list_state_count sched timeout) &&
+	count=`flux jobs --no-header --filter=running,completed | wc -l` &&
+	test $count -eq $(job_list_state_count run completed) &&
+	count=`flux jobs --no-header --filter=running,failed | wc -l` &&
+	test $count -eq $(job_list_state_count run failed) &&
+	count=`flux jobs --no-header --filter=running,canceled | wc -l` &&
+	test $count -eq $(job_list_state_count run canceled) &&
+	count=`flux jobs --no-header --filter=running,timeout | wc -l` &&
+	test $count -eq $(job_list_state_count run timeout) &&
+	count=`flux jobs --no-header --filter=inactive,completed | wc -l` &&
+	test $count -eq $(job_list_state_count inactive) &&
+	count=`flux jobs --no-header --filter=inactive,failed | wc -l` &&
+	test $count -eq $(job_list_state_count inactive) &&
+	count=`flux jobs --no-header --filter=inactive,canceled | wc -l` &&
+	test $count -eq $(job_list_state_count inactive) &&
+	count=`flux jobs --no-header --filter=inactive,timeout | wc -l` &&
+	test $count -eq $(job_list_state_count inactive)
+'
+
+
+test_expect_success 'flux-jobs --filter with invalid state fails' '
+	test_must_fail flux jobs --filter=foobar 2> invalidstate.err &&
+	grep "Invalid filter specified: foobar" invalidstate.err
 '
 
 # ensure + prefix works
 # increment userid to ensure not current user for test
 test_expect_success 'flux-jobs --user=UID works' '
-        userid=`id -u` &&
-        count=`flux jobs --suppress-header --user=${userid} | wc -l` &&
-        test $count -eq 14 &&
-        count=`flux jobs --suppress-header --user="+${userid}" | wc -l` &&
-        test $count -eq 14 &&
-        userid=$((userid+1)) &&
-        count=`flux jobs --suppress-header --user=${userid} | wc -l` &&
-        test $count -eq 0
+	userid=`id -u` &&
+	count=`flux jobs --no-header --user=${userid} | wc -l` &&
+	test $count -eq $(job_list_state_count active) &&
+	count=`flux jobs --no-header --user="+${userid}" | wc -l` &&
+	test $count -eq $(job_list_state_count active) &&
+	userid=$((userid+1)) &&
+	count=`flux jobs --no-header --user=${userid} | wc -l` &&
+	test $count -eq 0
 '
 
 test_expect_success 'flux-jobs --user=USERNAME works' '
-        username=`whoami` &&
-        count=`flux jobs --suppress-header --user=${username} | wc -l` &&
-        test $count -eq 14
+	username=`whoami` &&
+	count=`flux jobs --no-header --user=${username} | wc -l` &&
+	test $count -eq $(job_list_state_count active)
 '
 
 test_expect_success 'flux-jobs --user with invalid username fails' '
-        username="foobarfoobaz" &&
-        test_must_fail flux jobs --suppress-header --user=${username}
+	username="foobarfoobaz" &&
+	test_must_fail flux jobs --no-header --user=${username} 2> baduser.out &&
+	grep "Invalid user" baduser.out
 '
 
 test_expect_success 'flux-jobs --user=all works' '
-        count=`flux jobs --suppress-header --user=all | wc -l` &&
-        test $count -eq 14
+	count=`flux jobs --no-header --user=all | wc -l` &&
+	test $count -eq $(job_list_state_count active)
 '
 
 test_expect_success 'flux-jobs --count works' '
-        count=`flux jobs --suppress-header -a --count=0 | wc -l` &&
-        test $count -eq 18 &&
-        count=`flux jobs --suppress-header -a --count=8 | wc -l` &&
-        test $count -eq 8
+	count=`flux jobs --no-header -a --count=0 | wc -l` &&
+	test $count -eq $(job_list_state_count all) &&
+	count=`flux jobs --no-header -a --count=8 | wc -l` &&
+	test $count -eq 8
+'
+
+#
+# -i, --include tests
+#
+test_expect_success 'flux-jobs -i, --include works with ranks' '
+	for rank in $(flux jobs -ai 0 -no {ranks}); do
+		test $rank -eq 0
+	done
+'
+test_expect_success 'flux-jobs -i, --include works with ranks' '
+	for rank in $(flux jobs -ai 0,3 -no {ranks}); do
+		test $rank -eq 0 -o $rank -eq 3
+	done
+'
+test_expect_success 'flux jobs -i, --include works with hosts' '
+	for host in $(flux jobs -ai $(hostname) -no {nodelist}); do
+		test $host = $(hostname)
+	done
+'
+test_expect_success 'flux jobs -i, --include fails with bad idset/hostlist' '
+	test_must_fail flux jobs -ai "foo["
+'
+
+#
+# test specific IDs
+#
+
+test_expect_success 'flux-jobs specific IDs works' '
+	for state in sched run inactive; do
+		ids=$(job_list_state_ids $state) &&
+		expected=$(job_list_state_count $state) &&
+		count=$(flux jobs -n ${ids} | wc -l) &&
+		test_debug "echo Got ${count} of ${expected} ids in ${state} state" &&
+		test $count -eq $expected
+	done
+'
+
+test_expect_success 'flux jobs can take specific IDs in any form' '
+	id=$(head -1 run.ids) &&
+	for f in f58 hex dothex kvs words; do
+		flux job id --to=${f} ${id}
+	done > ids.specific.list &&
+	flux jobs -no {id} $(cat ids.specific.list) > ids.specific.out &&
+	for i in $(seq 1 5); do echo $id >>ids.specific.expected; done &&
+	test_cmp ids.specific.expected ids.specific.out
+'
+
+test_expect_success 'flux-jobs error on unknown IDs' '
+	flux jobs --no-header 0 1 2 2> ids.err &&
+	count=`grep -i unknown ids.err | wc -l` &&
+	test $count -eq 3
+'
+
+test_expect_success 'flux-jobs errors with illegal IDs' '
+	test_must_fail flux jobs --no-header IllegalID 2> illegal_ids.err &&
+	grep "invalid JobID value" illegal_ids.err
+'
+
+test_expect_success 'flux-jobs good and bad IDs works' '
+	ids=$(job_list_state_ids sched) &&
+	flux jobs --no-header ${ids} 0 1 2 > ids.out 2> ids.err &&
+	count=`wc -l < ids.out` &&
+	test $count -eq $(job_list_state_count sched) &&
+	count=`grep -i unknown ids.err | wc -l` &&
+	test $count -eq 3
+'
+
+test_expect_success 'flux-jobs outputs warning on invalid options' '
+	ids=$(job_list_state_ids sched) &&
+	flux jobs --no-header -A ${ids} > warn.out 2> warn.err &&
+	grep WARNING warn.err
 '
 
 #
@@ -196,56 +510,210 @@ test_expect_success 'flux-jobs --count works' '
 #
 
 test_expect_success 'flux-jobs --format={id} works' '
-        flux jobs --suppress-header --state=pending --format="{id}" > idsP.out &&
-        test_cmp idsP.out job_ids_pending.out &&
-        flux jobs --suppress-header --state=running --format="{id}" > idsR.out &&
-        test_cmp idsR.out job_ids_running.out &&
-        flux jobs --suppress-header --state=inactive --format="{id}" > idsI.out &&
-        test_cmp idsI.out job_ids_inactive.out
+	flux jobs --no-header --filter=pending --format="{id}" > idsP.out &&
+	test_cmp idsP.out sched.ids &&
+	flux jobs --no-header --filter=running --format="{id}" > idsR.out &&
+	test_cmp idsR.out run.ids &&
+	flux jobs --no-header --filter=inactive --format="{id}" > idsI.out &&
+	test_cmp idsI.out inactive.ids
+'
+
+test_expect_success 'flux-jobs --format={id.f58},{id.f58plain},{id.hex},{id.dothex},{id.words} works' '
+	flux jobs -ano {id.dec},{id.f58},{id.f58plain},{id.hex},{id.kvs},{id.dothex},{id.words} \
+	    | sort -n > ids.XX.out &&
+	for id in $(cat all.ids); do
+		printf "%s,%s,%s,%s,%s,%s,%s\n" \
+		       $(flux job id --to=dec $id) \
+		       $(flux job id --to=f58 $id) \
+		       $(flux job id --to=f58plain $id) \
+		       $(flux job id --to=hex $id) \
+		       $(flux job id --to=kvs $id) \
+		       $(flux job id --to=dothex $id) \
+		       $(flux job id --to=words $id)
+	done | sort -n > ids.XX.expected &&
+	test_cmp ids.XX.expected ids.XX.out
 '
 
 test_expect_success 'flux-jobs --format={userid},{username} works' '
-        flux jobs --suppress-header -a --format="{userid},{username}" > user.out &&
-        id=`id -u` &&
-        name=`whoami` &&
-        for i in `seq 1 18`; do echo "${id},${name}" >> user.exp; done &&
-        test_cmp user.out user.exp
+	flux jobs --no-header -a --format="{userid},{username}" > user.out &&
+	id=`id -u` &&
+	name=`whoami` &&
+	for i in `seq 1 $(job_list_state_count all)`; do
+		echo "${id},${name}" >> user.exp
+	done &&
+	test_cmp user.out user.exp
+'
+
+test_expect_success 'flux-jobs --format={urgency},{priority} works' '
+	flux jobs --no-header -a --format="{urgency},{priority}" > urgency_priority.out &&
+	echo 31,4294967295 > urgency_priority.exp &&
+	echo 25,25 >> urgency_priority.exp &&
+	echo 20,20 >> urgency_priority.exp &&
+	echo 15,15 >> urgency_priority.exp &&
+	echo 10,10 >> urgency_priority.exp &&
+	echo 5,5 >> urgency_priority.exp &&
+	echo 0,0 >> urgency_priority.exp &&
+	for i in `seq 1 $(job_list_state_count run)`; do
+		echo "16,16" >> urgency_priority.exp
+	done &&
+	for i in `seq 1 $(job_list_state_count inactive)`; do
+		echo "16,16" >> urgency_priority.exp
+	done &&
+	test_cmp urgency_priority.out urgency_priority.exp
+'
+
+test_expect_success 'flux-jobs --format={contextual_info} shows held job' '
+	flux jobs -no {urgency}:{contextual_info} | grep 0:held
+'
+
+# There is no simple way to create a job with urgency > 0 and priority == 0,
+# so test priority-hold using --from-stdin:
+test_expect_success 'flux-jobs {contextual_info} shows priority-hold job' '
+	echo "{\"id\":195823665152,\"state\":8,\"priority\":0,\"urgency\":16}" \
+		| flux jobs --from-stdin -no {priority}:{contextual_info} \
+		| grep 0:priority-hold
 '
 
 test_expect_success 'flux-jobs --format={state},{state_single} works' '
-        flux jobs --suppress-header --state=pending --format="{state},{state_single}" > stateP.out &&
-        for i in `seq 1 6`; do echo "SCHED,S" >> stateP.exp; done &&
-        test_cmp stateP.out stateP.exp &&
-        flux jobs --suppress-header --state=running --format="{state},{state_single}" > stateR.out &&
-        for i in `seq 1 8`; do echo "RUN,R" >> stateR.exp; done &&
-        test_cmp stateR.out stateR.exp &&
-        flux jobs --suppress-header --state=inactive --format="{state},{state_single}" > stateI.out &&
-        for i in `seq 1 4`; do echo "INACTIVE,I" >> stateI.exp; done &&
-        test_cmp stateI.out stateI.exp
+	flux jobs --filter=pending -c1 -no "{state},{state_single}" > stateP.out &&
+	test "$(cat stateP.out)" = "SCHED,S" &&
+	flux jobs --filter=running -c1 -no "{state},{state_single}" > stateR.out &&
+	test "$(cat stateR.out)" = "RUN,R" &&
+	flux jobs --filter=inactive -c1 -no "{state},{state_single}" > stateI.out &&
+	test "$(cat stateI.out)" = "INACTIVE,I"
+'
+
+# grepping for specific unicode chars is hard, so we just grep to make
+# sure a unicode character was output.  Be sure to disable color too,
+# since there can be unicode in there.
+test_expect_success 'flux-jobs --format={state_emoji} works' '
+	$runpty flux jobs --filter=pending -c1 --color=never -no "{state_emoji}" > stateE_P.out &&
+	grep "\\u" stateE_P.out &&
+	$runpty flux jobs --filter=running -c1 --color=never -no "{state_emoji}" > stateE_R.out &&
+	grep "\\u" stateE_R.out &&
+	$runpty flux jobs --filter=inactive -c1 --color=never -no "{state_emoji}" > stateE_I.out &&
+	grep "\\u" stateE_I.out
 '
 
 test_expect_success 'flux-jobs --format={name} works' '
-        flux jobs --suppress-header --state=pending,running --format="{name}" > jobnamePR.out &&
-        for i in `seq 1 14`; do echo "sleep" >> jobnamePR.exp; done &&
-        test_cmp jobnamePR.out jobnamePR.exp &&
-        flux jobs --suppress-header --state=inactive --format="{name}" > jobnameI.out &&
-        for i in `seq 1 4`; do echo "hostname" >> jobnameI.exp; done &&
-        test_cmp jobnameI.out jobnameI.exp
+	flux jobs --filter=pending,running -no "{name}" > jobnamePR.out &&
+	for i in `seq 1 $(job_list_state_count run sched)`; do
+		echo "sleep" >> jobnamePR.exp
+	done &&
+	test_cmp jobnamePR.out jobnamePR.exp &&
+	flux jobs --filter=inactive -no "{name}" > jobnameI.out &&
+	echo "canceledjob" >> jobnameI.exp &&
+	echo "sleep" >> jobnameI.exp &&
+	echo "sleepinf.sh" >> jobnameI.exp &&
+	echo "sleepinf.sh" >> jobnameI.exp &&
+	echo "nosuchcommand" >> jobnameI.exp &&
+	count=$(($(job_list_state_count inactive) - 5)) &&
+	for i in `seq 1 $count`; do
+		echo "hostname" >> jobnameI.exp
+	done &&
+	test_cmp jobnameI.out jobnameI.exp
+'
+
+test_expect_success 'flux-jobs --format={cwd} works' '
+	pwd=$(pwd) &&
+	flux jobs -a -no "{cwd}" > jobcwd.out &&
+	for i in `seq 1 $(job_list_state_count all)`; do
+		echo "${pwd}" >> jobcwd.exp
+	done &&
+	test_cmp jobcwd.out jobcwd.exp
+'
+
+# in job submissions above: completed jobs should be in queue1, running jobs
+# in queue2, and the rest in defaultqueue
+test_expect_success 'flux-jobs --format={queue} works' '
+	flux jobs --filter=completed -no "{queue}" > jobqueueCD.out &&
+	for i in `seq 1 $(job_list_state_count completed)`; do
+		echo "queue1" >> jobqueueCD.exp
+	done &&
+	test_cmp jobqueueCD.out jobqueueCD.exp &&
+	flux jobs --filter=running -no "{queue}" > jobqueueR.out &&
+	for i in `seq 1 $(job_list_state_count run)`; do
+		echo "queue2" >> jobqueueR.exp
+	done &&
+	test_cmp jobqueueR.out jobqueueR.exp &&
+	flux jobs --filter=failed -no "{queue}" > jobqueueF.out &&
+	for i in `seq 1 $(job_list_state_count failed)`; do
+		echo "defaultqueue" >> jobqueueF.exp
+	done &&
+	test_cmp jobqueueF.out jobqueueF.exp
 '
 
 test_expect_success 'flux-jobs --format={ntasks} works' '
-        flux jobs --suppress-header -a --format="{ntasks}" > taskcount.out &&
-        for i in `seq 1 18`; do echo "1" >> taskcount.exp; done &&
-        test_cmp taskcount.out taskcount.exp
+	flux jobs -a -no "{ntasks}" > ntasks.out &&
+	for i in `seq 1 $(job_list_state_count all)`; do
+		echo "1" >> ntasks.exp
+	done &&
+	test_cmp ntasks.exp ntasks.out
+'
+
+test_expect_success 'flux-jobs --format={ncores} works' '
+	flux jobs -a -no "{ncores}" > ncores.out &&
+	for i in `seq 1 $(job_list_state_count all)`; do
+		echo "1" >> ncores.exp
+	done &&
+	test_cmp ncores.exp ncores.out
+'
+
+test_expect_success 'flux-jobs --format={duration},{duration:h},{duration!F},{duration!H},{duration!F:h},{duration!H:h} works' '
+	fmt="{duration},{duration:h},{duration!F},{duration!H},{duration!F:h},{duration!H:h}" &&
+	flux jobs --filter=pending,running -no "${fmt}" > durationPR.out &&
+	for i in `seq 1 $(job_list_state_count sched run)`; do
+		echo "300.0,300.0,5m,0:05:00,5m,0:05:00" >> durationPR.exp
+	done &&
+	test_cmp durationPR.exp durationPR.out &&
+	flux jobs --filter=completed -no "${fmt}" > durationCD.out &&
+	for i in `seq 1 $(job_list_state_count completed)`;
+		do echo "0.0,-,0s,0:00:00,-,-" >> durationCD.exp
+	done &&
+	test_cmp durationCD.exp durationCD.out
 '
 
-test_expect_success 'flux-jobs --format={nnodes},{nnodes_hyphen} works' '
-        flux jobs --suppress-header --state=pending --format="{nnodes},{nnodes_hyphen}" > nodecountP.out &&
-        for i in `seq 1 6`; do echo ",-" >> nodecountP.exp; done &&
-        test_cmp nodecountP.out nodecountP.exp &&
-        flux jobs --suppress-header --state=running,inactive --format="{nnodes},{nnodes_hyphen}" > nodecountRI.out &&
-        for i in `seq 1 12`; do echo "1,1" >> nodecountRI.exp; done &&
-        test_cmp nodecountRI.out nodecountRI.exp
+test_expect_success 'flux-jobs --format={nnodes},{nnodes:h} works' '
+	flux jobs --filter=pending -no "{nnodes},{nnodes:h}" > nodecountP.out &&
+	for i in `seq 1 $(job_list_state_count sched)`; do
+		echo ",-" >> nodecountP.exp
+	done &&
+	test_cmp nodecountP.exp nodecountP.out &&
+	flux jobs --filter=running -no "{nnodes},{nnodes:h}" > nodecountR.out &&
+	for i in `seq 1 $(job_list_state_count run)`; do
+		echo "1,1" >> nodecountR.exp
+	done &&
+	test_cmp nodecountR.exp nodecountR.out &&
+	flux jobs --filter=inactive -no "{nnodes},{nnodes:h}" > nodecountI.out &&
+	echo ",-" > nodecountI.exp &&
+	for i in `seq 1 $(job_list_state_count completed failed timeout)`;
+		do echo "1,1" >> nodecountI.exp
+	done &&
+	test_cmp nodecountI.exp nodecountI.out
+'
+
+test_expect_success 'flux-jobs --format={runtime:0.3f} works' '
+	flux jobs --filter=pending -no "{runtime:0.3f}" > runtime-dotP.out &&
+	for i in `seq 1 $(job_list_state_count sched)`;
+		do echo "0.000" >> runtime-dotP.exp;
+	done &&
+	test_cmp runtime-dotP.out runtime-dotP.exp &&
+	flux jobs --filter=running,inactive -no "{runtime:0.3f}" > runtime-dotRI.out &&
+	[ "$(grep -E "\.[0-9]{3}" runtime-dotRI.out | wc -l)" = "17" ]
+'
+
+test_expect_success 'flux-jobs --format={contextual_time} works' '
+	flux jobs --filter=pending -c1 -no "{contextual_time:h}" \
+		>ctx_timeP.out &&
+	flux jobs --filter=running -c1 -no "{contextual_time:h}" \
+		>ctx_timeR.out &&
+	flux jobs --filter=completed -c1 -no "{contextual_time:0.3f}" \
+		>ctx_timeCD.out &&
+	echo "300.0" >duration.expected &&
+	test_cmp duration.expected ctx_timeP.out &&
+	test_must_fail test_cmp duration.expected ctx_timeR.out &&
+	test_must_fail test_cmp duration.expected ctx_timeCD.out &&
+	grep -E "\.[0-9]{3}" ctx_timeCD.out
 '
 
 test_expect_success 'flux-jobs emits useful error on invalid format' '
@@ -260,72 +728,677 @@ test_expect_success 'flux-jobs emits useful error on invalid format field' '
 	grep "Unknown format field" invalid-field.out
 '
 
-# node ranks assumes sched-simple default of mode='worst-fit'
-test_expect_success 'flux-jobs --format={ranks},{ranks_hyphen} works' '
-        flux jobs --suppress-header --state=pending --format="{ranks},{ranks_hyphen}" > ranksP.out &&
-        for i in `seq 1 6`; do echo ",-" >> ranksP.exp; done &&
-        test_cmp ranksP.out ranksP.exp &&
-        flux jobs --suppress-header --state=running --format="{ranks},{ranks_hyphen}" > ranksR.out &&
-        for i in `seq 1 2`; \
-        do \
-            echo "3,3" >> ranksR.exp; \
-            echo "2,2" >> ranksR.exp; \
-            echo "1,1" >> ranksR.exp; \
-            echo "0,0" >> ranksR.exp; \
-        done &&
-        test_cmp ranksR.out ranksR.exp &&
-        flux jobs --suppress-header --state=inactive --format="{ranks},{ranks_hyphen}" > ranksI.out &&
-        for i in `seq 1 4`; do echo "0,0" >> ranksI.exp; done &&
-        test_cmp ranksI.out ranksI.exp
+test_expect_success 'flux-jobs emits useful error on invalid format specifier' '
+	test_expect_code 1 flux jobs --format="{runtime:garbage}" >invalid-spec.out 2>&1 &&
+	test_debug "cat invalid-spec.out" &&
+	grep "Invalid format specifier" invalid-spec.out
+'
+
+test_expect_success 'flux-jobs --format={ranks},{ranks:h} works' '
+	flux jobs --filter=pending -no "{ranks},{ranks:h}" > ranksP.out &&
+	for i in `seq 1 $(job_list_state_count sched)`; do
+		echo ",-" >> ranksP.exp
+	done &&
+	test_cmp ranksP.out ranksP.exp &&
+	#
+	#  Ensure 1 running job lists a rank, we cannot know the exact
+	#  ranks on which every job ran.
+	#
+	flux jobs --filter=running -no "{ranks},{ranks:h}" > ranksR.out &&
+	test_debug "cat ranksR.out" &&
+	test "$(sort -n ranksR.out | head -1)" = "0,0" &&
+	flux jobs -no "{ranks},{ranks:h}" $(job_list_state_ids completed) > ranksCD.out &&
+	test_debug "cat ranksCD.out" &&
+	test "$(sort -n ranksCD.out | head -1)" = "0,0" &&
+	flux jobs -no "{ranks},{ranks:h}" $(job_list_state_ids canceled) > ranksCA.out &&
+	test_debug "cat ranksCA.out" &&
+	test "$(sort -n ranksCA.out | head -1)" = ",-" &&
+	flux jobs -no "{ranks},{ranks:h}" $(job_list_state_ids timeout) > ranksTO.out &&
+	test_debug "cat ranksTO.out" &&
+	test "$(sort -n ranksTO.out | head -1)" = "0,0"
+'
+
+test_expect_success 'flux-jobs --format={nodelist},{nodelist:h} works' '
+	flux jobs --filter=pending -no "{nodelist},{nodelist:h}" > nodelistP.out &&
+	for i in `seq 1 $(job_list_state_count sched)`; do
+		echo ",-" >> nodelistP.exp
+	done &&
+	test_cmp nodelistP.out nodelistP.exp &&
+	flux jobs --filter=running -no "{nodelist},{nodelist:h}" > nodelistR.out &&
+	for id in $(job_list_state_ids run); do
+		nodes=`flux job info ${id} R | flux R decode --nodelist`
+		echo "${nodes},${nodes}" >> nodelistR.exp
+	done &&
+	test_debug "cat nodelistR.out" &&
+	test_cmp nodelistR.out nodelistR.exp &&
+	flux jobs -no "{nodelist},{nodelist:h}" $(job_list_state_ids completed timeout) > nodelistCDTO.out &&
+	for id in $(job_list_state_ids completed timeout); do
+		nodes=`flux job info ${id} R | flux R decode --nodelist`
+		echo "${nodes},${nodes}" >> nodelistCDTO.exp
+	done &&
+	test_debug "cat nodelistCDTO.out" &&
+	test_cmp nodelistCDTO.out nodelistCDTO.exp &&
+	flux jobs -no "{nodelist},{nodelist:h}" $(job_list_state_ids canceled) > nodelistCA.out &&
+	test_debug "cat nodelistCA.out" &&
+	echo ",-" > nodelistCA.exp &&
+	test_cmp nodelistCA.out nodelistCA.exp
 '
 
 # test just make sure numbers are zero or non-zero given state of job
-test_expect_success 'flux-jobs --format={t_XXX} works' '
-        flux jobs --suppress-header -a --format="{t_submit}" > t_submit.out &&
-        count=`cat t_submit.out | grep -v "^0.0$" | wc -l` &&
-        test $count -eq 18 &&
-        flux jobs --suppress-header -a --format="{t_depend}" > t_depend.out &&
-        count=`cat t_depend.out | grep -v "^0.0$" | wc -l` &&
-        test $count -eq 18 &&
-        flux jobs --suppress-header -a --format="{t_sched}" > t_sched.out &&
-        count=`cat t_sched.out | grep -v "^0.0$" | wc -l` &&
-        test $count -eq 18 &&
-        flux jobs --suppress-header --state=pending --format="{t_run}" > t_runP.out &&
-        flux jobs --suppress-header --state=running,inactive --format="{t_run}" > t_runRI.out &&
-        count=`cat t_runP.out | grep "^0.0$" | wc -l` &&
-        test $count -eq 6 &&
-        count=`cat t_runRI.out | grep -v "^0.0$" | wc -l` &&
-        test $count -eq 12 &&
-        flux jobs --suppress-header --state=pending,running --format="{t_cleanup}" > t_cleanupPR.out &&
-        flux jobs --suppress-header --state=inactive --format="{t_cleanup}" > t_cleanupI.out &&
-        count=`cat t_cleanupPR.out | grep "^0.0$" | wc -l` &&
-        test $count -eq 14 &&
-        count=`cat t_cleanupI.out | grep -v "^0.0$" | wc -l` &&
-        test $count -eq 4 &&
-        flux jobs --suppress-header --state=pending,running --format="{t_inactive}" > t_inactivePR.out &&
-        flux jobs --suppress-header --state=inactive --format="{t_inactive}" > t_inactiveI.out &&
-        count=`cat t_inactivePR.out | grep "^0.0$" | wc -l` &&
-        test $count -eq 14 &&
-        count=`cat t_inactiveI.out | grep -v "^0.0$" | wc -l` &&
-        test $count -eq 4
-'
-
-test_expect_success 'flux-jobs --format={runtime},{runtime_fsd},{runtime_fsd_hyphen},{runtime_hms} works' '
-        flux jobs --suppress-header --state=pending --format="{runtime},{runtime_fsd},{runtime_fsd_hyphen},{runtime_hms}" > runtimeP.out &&
-        for i in `seq 1 6`; do echo "0.0,0s,-,0:00:00" >> runtimeP.exp; done &&
-        test_cmp runtimeP.out runtimeP.exp &&
-        flux jobs --suppress-header --state=running,inactive --format="{runtime}" > runtimeRI_1.out &&
-        count=`cat runtimeRI_1.out | grep -v "^0.0$" | wc -l` &&
-        test $count -eq 12 &&
-        flux jobs --suppress-header --state=running,inactive --format="{runtime_fsd}" > runtimeRI_2.out &&
-        count=`cat runtimeRI_2.out | grep -v "^0s" | wc -l` &&
-        test $count -eq 12 &&
-        flux jobs --suppress-header --state=running,inactive --format="{runtime_fsd_hyphen}" > runtimeRI_3.out &&
-        count=`cat runtimeRI_3.out | grep -v "^-$" | wc -l` &&
-        test $count -eq 12 &&
-        flux jobs --suppress-header --state=running,inactive --format="{runtime_hms}" > runtimeRI_4.out &&
-        count=`cat runtimeRI_4.out | grep -v "^0:00:00" | wc -l` &&
-        test $count -eq 12
+test_expect_success 'flux-jobs --format={t_submit/depend} works' '
+	flux jobs -ano "{t_submit},{t_depend}" >t_SD.out &&
+	count=`cut -d, -f1 t_SD.out | grep -v "^0.0$" | wc -l` &&
+	test $count -eq $(job_list_state_count all) &&
+	count=`cut -d, -f2 t_SD.out | grep -v "^0.0$" | wc -l` &&
+	test $count -eq $(job_list_state_count all)
+'
+test_expect_success 'flux-jobs --format={t_run} works' '
+	flux jobs --filter=pending -no "{t_run},{t_run:h}" > t_runP.out &&
+	flux jobs --filter=running -no "{t_run}" > t_runR.out &&
+	flux jobs --filter=inactive -no "{t_run}" > t_runI.out &&
+	count=`cut -d, -f1 t_runP.out | grep "^0.0$" | wc -l` &&
+	test $count -eq $(job_list_state_count sched) &&
+	count=`cut -d, -f2 t_runP.out | grep "^-$" | wc -l` &&
+	test $count -eq $(job_list_state_count sched) &&
+	count=`cat t_runR.out | grep -v "^0.0$" | wc -l` &&
+	test $count -eq $(job_list_state_count run) &&
+	cat t_runI.out &&
+	count=`head -n 1 t_runI.out | grep "^0.0$" | wc -l` &&
+	test $count -eq 1 &&
+	count=`tail -n $(job_list_state_count completed timeout) t_runI.out | grep -v "^0.0$" | wc -l` &&
+	test $count -eq $(job_list_state_count completed timeout)
+'
+test_expect_success 'flux jobs --format={t_cleanup/{in}active} works' '
+	flux jobs --filter=pending,running -no "{t_cleanup},{t_cleanup:h},{t_inactive},{t_inactive:h}" > t_cleanupPR.out &&
+	flux jobs --filter=inactive -no "{t_cleanup},{t_inactive}" > t_cleanupI.out &&
+	count=`cut -d, -f1 t_cleanupPR.out | grep "^0.0$" | wc -l` &&
+	test $count -eq $(job_list_state_count sched run) &&
+	count=`cut -d, -f2 t_cleanupPR.out | grep "^-$" | wc -l` &&
+	test $count -eq $(job_list_state_count sched run) &&
+	count=`cut -d, -f1 t_cleanupI.out | grep -v "^0.0$" | wc -l` &&
+	test $count -eq $(job_list_state_count inactive) &&
+	count=`cut -d, -f3 t_cleanupPR.out | grep "^0.0$" | wc -l` &&
+	test $count -eq $(job_list_state_count sched run) &&
+	count=`cut -d, -f4 t_cleanupPR.out | grep "^-$" | wc -l` &&
+	test $count -eq $(job_list_state_count sched run) &&
+	count=`cut -d, -f2 t_cleanupI.out | grep -v "^0.0$" | wc -l` &&
+	test $count -eq $(job_list_state_count inactive)
+'
+
+test_expect_success 'flux-jobs --format={runtime},{runtime!F},{runtime!H},{runtime!F:h},{runtime!H:h} works' '
+	fmt="{runtime},{runtime!F},{runtime!H},{runtime!F:h},{runtime!H:h}" &&
+	flux jobs --filter=pending -no "${fmt}" > runtimeP.out &&
+	for i in `seq 1 $(job_list_state_count sched)`; do
+		echo "0.0,0s,0:00:00,-,-" >> runtimeP.exp
+	done &&
+	test_cmp runtimeP.out runtimeP.exp &&
+	runcount=$(job_list_state_count run) &&
+	flux jobs --filter=running -no "${fmt}" > runtimeR.out &&
+	i=1 &&
+	for nomatch in 0.0 0s 0:00:00 - -; do
+		name=$(echo $fmt | cut -d, -f${i}) &&
+		count=$(cut -d, -f${i} runtimeR.out | grep -v "^${nomatch}$" | wc -l) &&
+		test_debug "echo $name field $i ${nomatch} $count/$runcount times" &&
+		test $count -eq $runcount &&
+		i=$((i+1))
+	done &&
+	expected=$(job_list_state_count completed timeout) &&
+	flux jobs -no "$fmt" $(job_list_state_ids completed timeout) >runtimeCDTO.out &&
+	i=1 &&
+	for nomatch in 0.0 0s 0:00:00 - -; do
+		name=$(echo $fmt | cut -d, -f${i}) &&
+		count=$(cut -d, -f${i} runtimeCDTO.out |grep -v "^${nomatch}$" | wc -l) &&
+		test_debug "echo $name: field $i: ${nomatch} $count/$expected times" &&
+		test $count -eq $expected &&
+		i=$((i+1))
+	done
+'
+
+test_expect_success 'flux-jobs --format={success},{success:h} works' '
+	flux jobs --filter=pending,running -no "{success},{success:h}" > successPR.out &&
+	for i in `seq 1 $(job_list_state_count sched run)`; do
+		echo ",-" >> successPR.exp;
+	done &&
+	test_cmp successPR.out successPR.exp &&
+	flux jobs --filter=inactive -no "{success},{success:h}" > successI.out &&
+	test $(grep -c False,False successI.out) -eq $(job_list_state_count failed canceled timeout) &&
+	test $(grep -c True,True successI.out) -eq $(job_list_state_count completed)
+'
+
+test_expect_success 'flux-jobs --format={exception.*},{exception.*:h} works' '
+	fmt="{exception.occurred},{exception.occurred:h}" &&
+	fmt="${fmt},{exception.severity},{exception.severity:h}" &&
+	fmt="${fmt},{exception.type},{exception.type:h}" &&
+	fmt="${fmt},{exception.note},{exception.note:h}" &&
+	flux jobs --filter=pending,running -no "$fmt" > exceptionPR.out &&
+	count=$(grep -c "^,-,,-,,-,,-$" exceptionPR.out) &&
+	test $count -eq $(job_list_state_count sched run) &&
+	flux jobs --filter=inactive -no "$fmt" > exceptionI.out &&
+	count=$(grep -c "^True,True,0,0,cancel,cancel,mecanceled,mecanceled$" exceptionI.out) &&
+	test $count -eq $(job_list_state_count canceled) &&
+	count=$(grep -c "^True,True,0,0,myexception,myexception,myexception,myexception$" exceptionI.out) &&
+	test $count -eq $(job_list_state_count exception) &&
+	count=$(grep -c "^True,True,0,0,exec,exec,.*No such file.*" exceptionI.out) &&
+	test $count -eq $(job_list_state_count failed_exec) &&
+	count=$(grep -c "^True,True,0,0,timeout,timeout,.*expired.*" exceptionI.out) &&
+	test $count -eq $(job_list_state_count timeout) &&
+	count=$(grep -c "^False,False,,-,,-,," exceptionI.out) &&
+	test $count -eq $(job_list_state_count completed terminated)
+'
+
+
+test_expect_success 'flux-jobs --format={result},{result:h},{result_abbrev},{result_abbrev:h} works' '
+	fmt="{result},{result:h},{result_abbrev},{result_abbrev:h}" &&
+	flux jobs --filter=pending,running -no "$fmt" > resultPR.out &&
+	count=$(grep -c "^,-,,-$" resultPR.out) &&
+	test_debug "echo checking sched+run got $count" &&
+	test $count -eq $(job_list_state_count sched run) &&
+	flux jobs  --filter=inactive -no "$fmt" > resultI.out &&
+	count=$(grep -c "CANCELED,CANCELED,CA,CA" resultI.out) &&
+	test_debug "echo checking canceled got $count" &&
+	test $count -eq $(job_list_state_count canceled) &&
+	count=$(grep -c "FAILED,FAILED,F,F" resultI.out) &&
+	test_debug "echo checking failed got $count" &&
+	test $count -eq $(job_list_state_count failed) &&
+	count=$(grep -c "TIMEOUT,TIMEOUT,TO,TO" resultI.out) &&
+	test_debug "echo checking timeout got $count" &&
+	test $count -eq $(job_list_state_count timeout) &&
+	count=$(grep -c "COMPLETED,COMPLETED,CD,CD" resultI.out) &&
+	test_debug "echo checking completed got $count" &&
+	test $count -eq $(job_list_state_count completed)
+'
+
+# grepping for specific unicode chars is hard, so we just grep to make
+# sure a unicode character was output.  Be sure to disable color too,
+# since there can be unicode in there.
+test_expect_success 'flux-jobs --format={result_emoji} works' '
+	$runpty flux jobs --filter=pending -c1 --color=never -no "{result_emoji}" > resultE_P.out &&
+	grep "\\u" -v resultE_P.out &&
+	$runpty flux jobs --filter=running -c1 --color=never -no "{result_emoji}" > resultE_R.out &&
+	grep "\\u" -v resultE_R.out &&
+	$runpty flux jobs --filter=completed -c1 --color=never -no "{result_emoji}" > resultE_CD.out &&
+	grep "\\u" resultE_CD.out &&
+	$runpty flux jobs --filter=failed -c1 --color=never -no "{result_emoji}" > resultE_F.out &&
+	grep "\\u" resultE_F.out &&
+	$runpty flux jobs --filter=timeout -c1 --color=never -no "{result_emoji}" > resultE_CA.out &&
+	grep "\\u" resultE_CA.out &&
+	$runpty flux jobs --filter=canceled -c1 --color=never -no "{result_emoji}" > resultE_TO.out &&
+	grep "\\u" resultE_TO.out
+'
+
+test_expect_success 'flux-jobs --format={status},{status_abbrev} works' '
+	flux jobs --filter=sched    -no "{status},{status_abbrev}" > statusS.out &&
+	flux jobs --filter=run      -no "{status},{status_abbrev}" > statusR.out &&
+	flux jobs --filter=inactive -no "{status},{status_abbrev}" > statusI.out &&
+	count=$(grep -c "SCHED,S" statusS.out) &&
+	test $count -eq $(job_list_state_count sched) &&
+	count=$(grep -c "RUN,R" statusR.out) &&
+	test $count -eq $(job_list_state_count run) &&
+	count=$(grep -c "CANCELED,CA" statusI.out) &&
+	test $count -eq $(job_list_state_count canceled) &&
+	count=$(grep -c "FAILED,F" statusI.out) &&
+	test $count -eq $(job_list_state_count failed) &&
+	count=$(grep -c "TIMEOUT,TO" statusI.out) &&
+	test $count -eq $(job_list_state_count timeout) &&
+	count=$(grep -c "COMPLETED,CD" statusI.out) &&
+	test $count -eq $(job_list_state_count completed)
+'
+
+# grepping for specific unicode chars is hard, so we just grep to make
+# sure a unicode character was output.  Be sure to disable color too,
+# since there can be unicode in there.
+test_expect_success 'flux-jobs --format={status_emoji} works' '
+	$runpty flux jobs --filter=pending -c1 --color=never -no "{status_emoji}" > statusE_P.out &&
+	grep "\\u" statusE_P.out &&
+	$runpty flux jobs --filter=running -c1 --color=never -no "{status_emoji}" > statusE_R.out &&
+	grep "\\u" statusE_R.out &&
+	$runpty flux jobs --filter=completed -c1 --color=never -no "{status_emoji}" > statusE_CD.out &&
+	grep "\\u" statusE_CD.out &&
+	$runpty flux jobs --filter=failed -c1 --color=never -no "{status_emoji}" > statusE_F.out &&
+	grep "\\u" statusE_F.out &&
+	$runpty flux jobs --filter=timeout -c1 --color=never -no "{status_emoji}" > statusE_CA.out &&
+	grep "\\u" statusE_CA.out &&
+	$runpty flux jobs --filter=canceled -c1 --color=never -no "{status_emoji}" > statusE_TO.out &&
+	grep "\\u" statusE_TO.out
+'
+
+test_expect_success 'flux-jobs --format={waitstatus},{returncode}' '
+	FORMAT="{waitstatus:h},{returncode:h}" &&
+	flux jobs --filter=pending,running -no "$FORMAT" > returncodePR.out &&
+	flux jobs --filter=inactive -no "$FORMAT" > returncodeI.out &&
+	test_debug "echo active:; cat returncodePR.out" &&
+	test_debug "echo inactive:; cat returncodeI.out" &&
+	countPR=$(grep -c "^-,-$" returncodePR.out) &&
+	test_debug "echo active got $countPR, want $(job_list_state_count sched run)" &&
+	test $countPR -eq $(job_list_state_count sched run) &&
+	count=$(grep -c "^32512,127$" returncodeI.out) &&
+	test_debug "echo exit 127 got $count, want $(job_list_state_count failed_exec)" &&
+	test $count -eq $(job_list_state_count failed_exec) &&
+	count=$(grep -c "^36608,143$" returncodeI.out) &&
+	test_debug "echo exit 143 got $count, want $(job_list_state_count terminated exception)" &&
+	test $count -eq $(job_list_state_count terminated exception) &&
+	count=$(grep -c "^36352,142$" returncodeI.out) &&
+	test_debug "echo exit 142 got $count, want $(job_list_state_count timeout)" &&
+	test $count -eq $(job_list_state_count timeout) &&
+	count=$(grep -c "^0,0$" returncodeI.out) &&
+	test_debug "echo complete got $count, want $(job_list_state_count completed)" &&
+	test $count -eq $(job_list_state_count completed) &&
+	count=$(grep -c "^-,-128$" returncodeI.out) &&
+	test_debug "echo canceled got $count, want $(job_list_state_count canceled)" &&
+	test $count -eq $(job_list_state_count canceled)
+'
+
+test_expect_success 'flux-jobs --format={inactive_reason}' '
+	FORMAT="{inactive_reason:h}" &&
+	flux jobs --filter=pending,running -no "$FORMAT" > inactivereasonPR.out &&
+	count=$(grep -c "^-$" inactivereasonPR.out) &&
+	test_debug "echo empty got $count, want $(job_list_state_count active)" &&
+	test $count -eq $(job_list_state_count active) &&
+	flux jobs --filter=inactive -no "$FORMAT" > inactivereasonI.out &&
+	count=$(grep -c "^command not found$" inactivereasonI.out) &&
+	test_debug "echo command not found got $count, want $(job_list_state_count failed_exec)" &&
+	test $count -eq $(job_list_state_count failed_exec) &&
+	count=$(grep -c "^Terminated$" inactivereasonI.out) &&
+	test_debug "echo Terminated got $count, want $(job_list_state_count terminated)" &&
+	test $count -eq $(job_list_state_count terminated) &&
+	count=$(grep -c "^Exception: type=myexception note=myexception$" inactivereasonI.out) &&
+	test_debug "echo exception got $count, want $(job_list_state_count exception)" &&
+	test $count -eq $(job_list_state_count exception) &&
+	count=$(grep -c "^Timeout$" inactivereasonI.out) &&
+	test_debug "echo Timeout got $count, want $(job_list_state_count timeout)" &&
+	test $count -eq $(job_list_state_count timeout) &&
+	count=$(grep -c "^Exit 0$" inactivereasonI.out) &&
+	test_debug "echo Exit 0 got $count, want $(job_list_state_count completed)" &&
+	test $count -eq $(job_list_state_count completed) &&
+	count=$(grep -c "^Canceled: mecanceled$" inactivereasonI.out) &&
+	test_debug "echo canceled got $count, want $(job_list_state_count canceled)" &&
+	test $count -eq $(job_list_state_count canceled)
+'
+
+test_expect_success 'flux-jobs --format={expiration},{t_remaining} works' '
+	expiration=$(flux jobs -f running -c1 -no "{expiration:.0f}") &&
+	t_remaining=$(flux jobs -f running -c1 -no "{t_remaining:.0f}") &&
+	t_now=$(date +%s) &&
+	test_debug "echo expiration=$expiration, t_remaining=$t_remaining" &&
+	test $t_remaining -gt 0 &&
+	test $expiration  -gt $t_now
+'
+test_expect_success 'flux-jobs --format={expiration!D},{t_remaining!F} works' '
+	exp=$(($(date +%s) + 600)) &&
+	cat <<-EOF >expiration.in &&
+{"id": 1447588528128, "state": 8,  "expiration": ${exp}.0 }
+	EOF
+	expiration=$(cat expiration.in | \
+	flux jobs --from-stdin -no "{expiration!D}") &&
+	t_remaining=$(cat expiration.in | \
+		     flux jobs --from-stdin -no "{t_remaining!F}") &&
+	test_debug "echo expiration=$expiration, t_remaining=$t_remaining" &&
+	test "${expiration}" = "$(date --date=@${exp} +%FT%T)" &&
+	test_debug "echo expiration OK" &&
+	echo ${t_remaining} | grep "^[0-9.][0-9.]*[smdh]"
+'
+test_expect_success 'flux-jobs --format={expiration!d:%FT%T},{t_remaining!H} works' '
+	exp=$(($(date +%s) + 600)) &&
+	cat <<-EOF >expiration.in &&
+{"id": 1447588528128, "state": 8,  "expiration": ${exp}.0 }
+	EOF
+	expiration=$(cat expiration.in | \
+	flux jobs --from-stdin -no "{expiration!d:%FT%T}") &&
+	t_remaining=$(cat expiration.in | \
+		     flux jobs --from-stdin -no "{t_remaining!H}") &&
+	test_debug "echo expiration=$expiration, t_remaining=$t_remaining" &&
+	test "${expiration}" = "$(date --date=@${exp} +%FT%T)" &&
+	test_debug "echo expiration OK" &&
+	echo ${t_remaining} | grep "[0-9]:[0-9][0-9]:[0-9][0-9]"
+'
+test_expect_success 'flux-jobs --format={expiration!d:%FT%T::>20.20} works' '
+	cat expiration.in | \
+	    flux jobs --from-stdin -o "{expiration!d:%b%d %R::>20.20}" \
+	    >exp-fmt.out &&
+	cat expiration.in | \
+	    flux jobs --from-stdin -o "{expiration!d:%b%d %R::>20.20h}" \
+	    >exp-fmth.out &&
+	test_debug "cat exp-fmt.out" &&
+	grep "          EXPIRATION" exp-fmt.out &&
+	grep "         $(date --date=@${exp} +%b%d\ %R)" exp-fmt.out &&
+	test_debug "cat exp-fmth.out" &&
+	grep "          EXPIRATION" exp-fmth.out &&
+	grep "         $(date --date=@${exp} +%b%d\ %R)" exp-fmth.out
+'
+test_expect_success 'flux-jobs --format={expiration!d} works' '
+	cat expiration.in | flux jobs --from-stdin -o "{expiration!d}" \
+	    >exp-fmtd.out &&
+	grep "$(date --date=@${exp} +%FT%T)" exp-fmtd.out
+'
+test_expect_success 'flux-jobs --format={expiration!d:%FT%T::=^20} works' '
+	cat expiration.in | \
+	    flux jobs --from-stdin -o "{expiration!d:%b%d %R::=^20}" \
+	    >exp-fmt.out &&
+	test_debug "cat exp-fmt.out" &&
+	grep "     EXPIRATION     " exp-fmt.out &&
+	grep "====$(date --date=@${exp} +%b%d\ %R)====" exp-fmt.out
+'
+
+test_expect_success 'flux-jobs --format={expiration!D:h},{t_remaining!H:h} works' '
+	cat <<-EOF >expiration.in &&
+{"id": 1447588528128, "state": 8,  "expiration": 0 }
+	EOF
+	expiration=$(cat expiration.in | \
+	flux jobs --from-stdin -no "{expiration!D:h}") &&
+	t_remaining=$(cat expiration.in | \
+		     flux jobs --from-stdin -no "{t_remaining!H:h}") &&
+	test_debug "echo expiration=$expiration, t_remaining=$t_remaining" &&
+	test "${expiration}" = "-" &&
+	test "${t_remaining}" = "-"
+'
+# note that a significant amount of annotation format tests occur in
+# job-manager tests such as t2206-job-manager-annotate.t
+
+test_expect_success 'flux-jobs annotation "sched" short hands work' '
+	fmt="{annotations.sched},{annotations.sched.resource_summary}" &&
+	flux jobs -no "${fmt}" > sched_long_hand.out &&
+	fmt="{sched},{sched.resource_summary}" &&
+	flux jobs -no "${fmt}" > sched_short_hand.out &&
+	test_cmp sched_long_hand.out sched_short_hand.out
+'
+
+test_expect_success 'flux-jobs emits empty string on invalid annotations fields' '
+	fmt="{annotations.foo},{annotations.foo:h}" &&
+	fmt="${fmt},{annotations.sched.bar},{annotations.sched.bar:h}" &&
+	fmt="${fmt},{annotations.x.y.z},{annotations.x.y.z:h}" &&
+	flux jobs -no "${fmt}" >invalid-annotations.out 2>&1 &&
+	test_debug "cat invalid-annotations.out" &&
+	for i in `seq 1 $(job_list_state_count active)`; do
+		echo ",-,,-,,-" >> invalid-annotations.exp
+	done &&
+	test_cmp invalid-annotations.out invalid-annotations.exp
+'
+
+test_expect_success 'flux-jobs "user" short hands work for job memo' '
+	for id in $(job_list_state_ids sched); do
+		flux job memo $id foo=42
+	done &&
+	fmt="{annotations.user},{annotations.user.foo}" &&
+	flux jobs -no "${fmt}" > user_long_hand.out &&
+	fmt="{user},{user.foo}" &&
+	flux jobs -no "${fmt}" > user_short_hand.out &&
+	test_cmp user_long_hand.out user_short_hand.out
+'
+
+test_expect_success 'flux-jobs emits empty string for special case t_estimate' '
+	fmt="{annotations.sched.t_estimate}" &&
+	fmt="${fmt},{annotations.sched.t_estimate!d:%H:%M}" &&
+	fmt="${fmt},{annotations.sched.t_estimate!D}" &&
+	fmt="${fmt},{annotations.sched.t_estimate!F}" &&
+	fmt="${fmt},{annotations.sched.t_estimate!H}" &&
+	fmt="${fmt},{annotations.sched.t_estimate!d:%H:%M::h}" &&
+	fmt="${fmt},{annotations.sched.t_estimate!D:h}" &&
+	fmt="${fmt},{annotations.sched.t_estimate!F:h}" &&
+	fmt="${fmt},{annotations.sched.t_estimate!H:h}" &&
+	flux jobs -no "${fmt}" >t_estimate_annotations.out 2>&1 &&
+	test_debug "cat t_estimate_annotations.out" &&
+	for i in `seq 1 $(job_list_state_count active)`; do
+		echo ",,,,,-,-,-,-" >> t_estimate_annotations.exp
+	done &&
+	test_cmp t_estimate_annotations.out t_estimate_annotations.exp
+'
+
+test_expect_success 'FLUX_JOBS_FORMAT_DEFAULT works' '
+	FLUX_JOBS_FORMAT_DEFAULT="{id} {id}" flux jobs > env_format.out &&
+	grep "JOBID JOBID" env_format.out
+'
+
+test_expect_success 'FLUX_JOBS_FORMAT_DEFAULT works with named format' '
+	FLUX_JOBS_FORMAT_DEFAULT=long flux jobs > env_format2.out &&
+	grep "STATUS" env_format2.out
+'
+
+#
+# project/name require update to jobs, do these "last" amongst the
+# basic tests to avoid affecting tests as we modify fields.
+#
+
+test_expect_success 'flux-jobs --format={project},{bank} works (initially empty)' '
+	flux jobs --filter=pending -no "{project},{bank}" > jobprojectbankP.out &&
+	for i in `seq 1 $(job_list_state_count sched)`; do
+		echo "," >> jobprojectbankP.exp
+	done &&
+	test_cmp jobprojectbankP.out jobprojectbankP.exp
+'
+
+test_expect_success 'support jobspec updates of project and bank' '
+	flux jobtap load --remove=all ${PLUGINPATH}/project-bank-validate.so
+'
+
+test_expect_success 'update project and bank in pending jobs' '
+	cat sched.ids | while read jobid; do
+		flux update $jobid project=foo
+		flux update $jobid bank=bar
+	done
+'
+
+wait_project_bank_synced() {
+	local i=0
+	while [ "$(flux jobs -f pending -o {project} | grep foo | wc -l)" != "$(job_list_state_count sched)" ] \
+		   && [ $i -lt 50 ]
+	do
+		sleep 0.1
+		i=$((i + 1))
+	done
+	if [ "$i" -eq "50" ]
+	then
+		return 1
+	fi
+	return 0
+}
+
+# can be racy, need to wait until `job-list` module definitely has
+# read from updated journal
+test_expect_success 'flux-jobs --format={project},{bank} works (set)' '
+	wait_project_bank_synced &&
+	flux jobs --filter=pending -no "{project},{bank}" > jobprojectbankP2.out &&
+	for i in `seq 1 $(job_list_state_count sched)`; do
+		echo "foo,bar" >> jobprojectbankP2.exp
+	done &&
+	test_cmp jobprojectbankP2.out jobprojectbankP2.exp
+'
+
+test_expect_success 'remove jobtap plugins' '
+	flux jobtap remove all
+'
+
+
+#
+# format header tests.
+#
+# to add additional tests, simply add a line with custom format and
+# expected header, separated by '=='
+#
+test_expect_success 'flux-jobs: header included with all custom formats' '
+	cat <<-EOF >headers.expected &&
+	id==JOBID
+	id.f58==JOBID
+	id.f58plain==JOBID
+	id.hex==JOBID
+	id.dothex==JOBID
+	id.words==JOBID
+	userid==UID
+	username==USER
+	urgency==URG
+	priority==PRI
+	state==STATE
+	state_single==S
+	state_emoji==STATE
+	name==NAME
+	cwd==CWD
+	queue==QUEUE
+	project==PROJECT
+	bank==BANK
+	ntasks==NTASKS
+	duration==DURATION
+	nnodes==NNODES
+	ranks==RANKS
+	nodelist==NODELIST
+	contextual_info==INFO
+	contextual_time==TIME
+	inactive_reason==INACTIVE-REASON
+	success==SUCCESS
+	exception.occurred==EXCEPTION-OCCURRED
+	exception.severity==EXCEPTION-SEVERITY
+	exception.type==EXCEPTION-TYPE
+	exception.note==EXCEPTION-NOTE
+	result==RESULT
+	result_abbrev==RS
+	result_emoji==RESULT
+	t_submit==T_SUBMIT
+	t_depend==T_DEPEND
+	t_run==T_RUN
+	t_cleanup==T_CLEANUP
+	t_inactive==T_INACTIVE
+	runtime==RUNTIME
+	runtime!F==RUNTIME
+	runtime!H==RUNTIME
+	expiration==EXPIRATION
+	expiration!D==EXPIRATION
+	expiration!d:%FT%T==EXPIRATION
+	t_remaining==T_REMAINING
+	t_remaining!F==T_REMAINING
+	t_remaining!H==T_REMAINING
+	t_inactive!d==T_INACTIVE
+	t_cleanup!D==T_CLEANUP
+	t_run!F==T_RUN
+	status==STATUS
+	status_abbrev==ST
+	status_emoji==STATUS
+	annotations==ANNOTATIONS
+	annotations.sched==SCHED
+	annotations.sched.t_estimate==T_ESTIMATE
+	annotations.sched.reason_pending==REASON
+	annotations.sched.resource_summary==RESOURCES
+	annotations.sched.foobar==SCHED.FOOBAR
+	sched==SCHED
+	sched.t_estimate==T_ESTIMATE
+	sched.reason_pending==REASON
+	sched.resource_summary==RESOURCES
+	sched.foobar==SCHED.FOOBAR
+	user==USER
+	user.foobar==USER.FOOBAR
+	waitstatus==WSTATUS
+	returncode==RC
+	EOF
+	sed "s/\(.*\)==.*/\1=={\1}/" headers.expected > headers.fmt &&
+	flux jobs --from-stdin --format="$(cat headers.fmt)" \
+	    > headers.output </dev/null &&
+	test_cmp headers.expected headers.output
+'
+
+#
+# color tests
+#
+
+check_no_color() {
+	local file=$1
+	count=`grep "\\u001b" $file | wc -l`
+	if [ "$count" -eq 0 ]; then
+		return 0
+	fi
+	return 1
+}
+
+no_color_lines() {
+	grep -v -c "\\u001b" $1 || true
+}
+green_line_count() {
+	grep -c -o "\\u001b\[01\;32m" $1 || true
+}
+red_line_count() {
+	grep -c -o "\\u001b\[01\;31m" $1 || true
+}
+grey_line_count() {
+	grep -c -o "\\u001b\[37m" $1 || true
+}
+
+for opt in "" "--color" "--color=always" "--color=auto"; do
+	test_expect_success "flux-jobs $opt color works (pty)" '
+		name=${opt##--color=} &&
+		outfile=color-${name:-default}.out &&
+		$runpty flux jobs ${opt} --no-header -a \
+		    | grep -v "version" > $outfile &&
+		count=$(no_color_lines $outfile) &&
+		test $count -eq $(job_list_state_count sched run) &&
+		count=$(green_line_count $outfile) &&
+		test $count -eq $(job_list_state_count completed) &&
+		count=$(red_line_count $outfile) &&
+		test $count -eq $(job_list_state_count failed timeout) &&
+		count=$(grey_line_count $outfile) &&
+		test $count -eq $(job_list_state_count canceled)
+	'
+done
+
+test_expect_success 'flux-jobs --color=never works (pty)' '
+	$runpty flux jobs --no-header --color=never -a >color_never.out &&
+	check_no_color color_never.out
+'
+
+for opt in "" "--color=never"; do
+	test_expect_success "flux-jobs $opt color works (no tty)" '
+		name=${opt##--color=} &&
+		outfile=color-${name:-default}-notty.out &&
+		flux jobs ${opt} --no-header -a > $outfile &&
+		test_must_fail grep "" $outfile
+	'
+done
+
+test_expect_success 'flux-jobs: --color=always works (notty)' '
+	flux jobs --color=always --no-header -a > color-always-notty.out &&
+	grep "" color-always-notty.out
+'
+
+#
+# json tests
+#
+
+test_expect_success 'flux-jobs: --json does not work with --stats' '
+	test_must_fail flux jobs --stats --json &&
+	test_must_fail flux jobs --stats-only --json
+'
+
+#  Multiple jobs return an array
+#  Active jobs do not have result
+test_expect_success 'flux-jobs: --json option works' '
+	flux jobs --json >jobs.json &&
+	jq -e ".jobs[0].id" < jobs.json &&
+	jq -e ".jobs[0] | .result | not" < jobs.json &&
+	jq -e ".jobs[0] | .t_cleanup | not" < jobs.json &&
+	jq -e ".jobs[0] | .exception.occurred == false" < jobs.json
+'
+
+#  Ensure single jobid argument returns single JSON object
+test_expect_success 'flux-jobs: --json option works with one jobid' '
+	testid=$(cat active.ids | head -1 | flux job id) &&
+	flux jobs --json $testid | jq -e ".id == $testid"
+'
+
+#  Ensure failed job has some expected fields
+test_expect_success 'flux-jobs: --json works for inactive job' '
+	testid=$(cat failed.ids | head -1 | flux job id) &&
+	flux jobs --json $testid > failedjob.json &&
+	test_debug "jq < failedjob.json" &&
+	jq -e ".id == $testid" < failedjob.json &&
+	jq -e ".result == \"FAILED\"" < failedjob.json &&
+	jq -e ".success == false" < failedjob.json &&
+	jq -e ".state == \"INACTIVE\"" < failedjob.json &&
+	jq -e ".t_cleanup > 0" < failedjob.json &&
+	jq -e ".t_run > 0" < failedjob.json &&
+	jq -e ".runtime > 0" < failedjob.json &&
+	jq -e ".exception.occurred == true" < failedjob.json &&
+	jq -e ".exception.type" < failedjob.json
+'
+
+#  Asking for a specific nonexisting jobid returns no output
+test_expect_success 'flux-jobs: --json with missing jobid returns nothing' '
+	flux jobs --json 123 >missing.json &&
+	test_must_be_empty missing.json
 '
 
 #
@@ -333,11 +1406,15 @@ test_expect_success 'flux-jobs --format={runtime},{runtime_fsd},{runtime_fsd_hyp
 #
 
 test_expect_success 'flux-jobs illegal count leads to RPC error' '
-        test_must_fail flux jobs --count=-1
+	test_must_fail flux jobs --count=-1
 '
 
 test_expect_success 'flux-jobs --format with illegal field is an error' '
-        test_must_fail flux jobs --format="{foobar}"
+	test_must_fail flux jobs --format="{foobar}"
+'
+
+test_expect_success 'flux-jobs illegal color options is an error' '
+	test_must_fail flux jobs --color=foobar
 '
 
 test_expect_success 'flux-jobs --from-stdin works with no input' '
@@ -349,17 +1426,17 @@ test_expect_success 'flux-jobs --from-stdin fails with invalid input' '
 '
 
 find_invalid_userid() {
-	python -c 'import pwd; \
-                   ids = [e.pw_uid for e in pwd.getpwall()]; \
-                   print (next(i for i in range(65536) if not i in ids));'
+	flux python -c 'import pwd; \
+	                ids = [e.pw_uid for e in pwd.getpwall()]; \
+	                print (next(i for i in range(65536) if not i in ids));'
 }
 
-test_expect_success HAVE_JQ 'flux-jobs reverts username to userid for invalid ids' '
+test_expect_success 'flux-jobs reverts username to userid for invalid ids' '
 	id=$(find_invalid_userid) &&
 	test_debug "echo first invalid userid is ${id}" &&
 	printf "%s\n" $id > invalid_userid.expected &&
 	flux job list -a -c 1 | $jq -c ".userid = ${id}" |
-	  flux jobs --from-stdin --suppress-header --format="{username}" \
+	  flux jobs --from-stdin --no-header --format="{username}" \
 		> invalid_userid.output  &&
 	test_cmp invalid_userid.expected invalid_userid.output
 '
@@ -384,19 +1461,157 @@ for d in ${ISSUES_DIR}/*; do
 	desc=$(basename ${d})
 	if test -f ${d}/description; then
 		desc="${desc}: $(cat ${d}/description)"
-        fi
+	fi
 	if test -f ${d}/format; then
 		fmt=$(cat ${d}/format)
-        else
+	else
 		fmt=""
-        fi
+	fi
 	test_expect_success "${desc}" '
 		flux jobs -n --from-stdin ${fmt:+--format="$fmt"} < ${d}/input >${issue}.output &&
 		test_cmp ${d}/output ${issue}.output
 	'
 done
 
+
+# job stats
+test_expect_success 'flux-jobs --stats works (global)' '
+	flux jobs --stats -a >stats.output &&
+	test_debug "cat stats.output" &&
+	fail=$(job_list_state_count failed canceled timeout) &&
+	run=$(job_list_state_count run) &&
+	active=$(job_list_state_count active) &&
+	comp=$(job_list_state_count completed) &&
+	pend=$((active - run)) &&
+	cat <<-EOF >stats.expected &&
+	${run} running, ${comp} completed, ${fail} failed, ${pend} pending, 0 inactive purged
+	EOF
+	head -1 stats.output > stats.actual &&
+	test_cmp stats.expected stats.actual
+'
+
+test_expect_success 'flux-jobs --stats works (queue1)' '
+	flux jobs --stats -a --queue=queue1 >statsq1.output &&
+	test_debug "cat statsq1.output" &&
+	comp=$(job_list_state_count completed) &&
+	cat <<-EOF >statsq1.expected &&
+	0 running, ${comp} completed, 0 failed, 0 pending, 0 inactive purged
+	EOF
+	head -1 statsq1.output > statsq1.actual &&
+	test_cmp statsq1.expected statsq1.actual
+'
+
+test_expect_success 'flux-jobs --stats works (queue2)' '
+	flux jobs --stats -a --queue=queue2 >statsq2.output &&
+	test_debug "cat statsq2.output" &&
+	run=$(job_list_state_count run) &&
+	active=$(job_list_state_count active) &&
+	pend=$((active - run)) &&
+	cat <<-EOF >statsq2.expected &&
+	${run} running, 0 completed, 0 failed, ${pend} pending, 0 inactive purged
+	EOF
+	head -1 statsq2.output > statsq2.actual &&
+	test_cmp statsq2.expected statsq2.actual
+'
+
+test_expect_success 'flux-jobs --stats works (defaultqueue)' '
+	flux jobs --stats -a --queue=defaultqueue >statsqdefault.output &&
+	test_debug "cat statsqdefault.output" &&
+	fail=$(job_list_state_count failed canceled timeout) &&
+	cat <<-EOF >statsqdefault.expected &&
+	0 running, 0 completed, ${fail} failed, 0 pending, 0 inactive purged
+	EOF
+	head -1 statsqdefault.output > statsqdefault.actual &&
+	test_cmp statsqdefault.expected statsqdefault.actual
+'
+
+test_expect_success 'flux-jobs --stats-only works' '
+	flux jobs --stats-only > stats-only.output &&
+	test_cmp stats.expected stats-only.output
+'
+
+test_expect_success 'cleanup job listing jobs ' '
+	flux cancel $(cat active.ids) &&
+	for jobid in `cat active.ids`; do
+		fj_wait_event $jobid clean;
+	done
+'
+
+test_expect_success 'purge all jobs' '
+	flux job purge --force --num-limit=0
+'
+
 #
-# leave job cleanup to rc3
+# All stat tests below assume all jobs purged
 #
+
+test_expect_success 'flux-jobs --stats works after jobs purged (all)' '
+	flux jobs --stats -a >statspurge.output &&
+	test_debug "cat statspurge.output" &&
+	all=$(job_list_state_count all) &&
+	cat <<-EOF >statspurge.expected &&
+	0 running, 0 completed, 0 failed, 0 pending, ${all} inactive purged
+	EOF
+	head -1 statspurge.output > statspurge.actual &&
+	test_cmp statspurge.expected statspurge.actual
+'
+
+test_expect_success 'flux-jobs --stats works after jobs purged (queue1)' '
+	flux jobs --stats -a --queue=queue1 >statspurgeq1.output &&
+	test_debug "cat statspurgeq1.output" &&
+	comp=$(job_list_state_count completed) &&
+	cat <<-EOF >statspurgeq1.expected &&
+	0 running, 0 completed, 0 failed, 0 pending, ${comp} inactive purged
+	EOF
+	head -1 statspurgeq1.output > statspurgeq1.actual &&
+	test_cmp statspurgeq1.expected statspurgeq1.actual
+'
+
+test_expect_success 'flux-jobs --stats works after jobs purged (queue2)' '
+	flux jobs --stats -a --queue=queue2 >statspurgeq2.output &&
+	test_debug "cat statspurgeq2.output" &&
+	active=$(job_list_state_count active) &&
+	cat <<-EOF >statspurgeq2.expected &&
+	0 running, 0 completed, 0 failed, 0 pending, ${active} inactive purged
+	EOF
+	head -1 statspurgeq2.output > statspurgeq2.actual &&
+	test_cmp statspurgeq2.expected statspurgeq2.actual
+'
+
+test_expect_success 'flux-jobs --stats works after jobs purged (defaultqueue)' '
+	flux jobs --stats -a --queue=defaultqueue >statspurgeqdefault.output &&
+	test_debug "cat statspurgeqdefault.output" &&
+	fail=$(job_list_state_count failed canceled timeout) &&
+	cat <<-EOF >statspurgeqdefault.expected &&
+	0 running, 0 completed, 0 failed, 0 pending, ${fail} inactive purged
+	EOF
+	head -1 statspurgeqdefault.output > statspurgeqdefault.actual &&
+	test_cmp statspurgeqdefault.expected statspurgeqdefault.actual
+'
+test_expect_success 'flux-jobs allows sorting order in format' '
+	flux jobs -ano "{ncores}	{t_submit}	{id.f58}" \
+		| sort -k1,2n \
+		| cut -f 3 >sort1.expected &&
+	flux jobs -ano "sort:ncores,t_submit {id.f58}" >sort1.out &&
+	test_cmp sort1.expected sort1.out
+'
+test_expect_success 'flux-jobs allows sorting order via --sort' '
+	flux jobs -ano "{ntasks}	{t_submit}	{id.f58}" \
+		| sort -k1n,2rn \
+		| cut -f 3 >sort2.expected &&
+	flux jobs -an --sort=ntasks,-t_submit -o "{id.f58}" >sort2.out &&
+	test_cmp sort2.expected sort2.out
+'
+test_expect_success 'flux-jobs --sort overrides sort: in format' '
+	flux jobs -ano "{ntasks}	{t_submit}	{id.f58}" \
+		| sort -k1n,2rn \
+		| cut -f 3 >sort3.expected &&
+	flux jobs -an --sort=ntasks,-t_submit -o "sort:t_submit {id.f58}" \
+		>sort3.out &&
+	test_cmp sort3.expected sort3.out
+'
+test_expect_success 'flux-jobs invalid sort key throws exception' '
+	test_must_fail flux jobs -a --sort=foo 2>sort.error &&
+	grep "Invalid sort key: foo" sort.error
+'
 test_done
diff --git a/t/t2800-jobs-config.t b/t/t2800-jobs-config.t
new file mode 100755
index 000000000000..a4dcc4616566
--- /dev/null
+++ b/t/t2800-jobs-config.t
@@ -0,0 +1,150 @@
+#!/bin/sh
+
+test_description='Test flux jobs command config file support'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 job
+
+test_expect_success 'run a job for job listing purposes' '
+	flux submit sleep 300
+'
+test_expect_success 'flux-jobs --format=help works' '
+	flux jobs --format=help >help.out &&
+	test_debug "cat help.out" &&
+	grep "flux-jobs output formats:" help.out
+'
+test_expect_success 'flux-jobs --format=invalid fails' '
+	test_must_fail flux jobs --format=invalid
+'
+#  We can't add configuration to ~/.config or /etc/xdg/flux, so
+#   just test that XDG_CONFIG_HOME works.
+#
+test_expect_success 'Additional formats can be added to XDG_CONFIG_HOME' '
+	mkdir -p dir/flux &&
+	cat <<-EOF >dir/flux/flux-jobs.toml &&
+	[formats.myformat]
+        description = "my format"
+	format = "{id.words}"
+	EOF
+	XDG_CONFIG_HOME="$(pwd)/dir" flux jobs --format=help >myformat.out &&
+	test_debug "cat myformat.out" &&
+	grep "my format" myformat.out &&
+	XDG_CONFIG_HOME="$(pwd)/dir" flux jobs --format=myformat
+'
+test_expect_success 'flux-jobs config can be yaml' '
+	mkdir -p yaml/flux &&
+	cat <<-EOF >yaml/flux/flux-jobs.yaml &&
+	formats:
+	  myformat:
+	    description: My YAML Format
+	    format: "{id.f58} {runtime}"
+	EOF
+	XDG_CONFIG_HOME="$(pwd)/yaml" flux jobs --format=help >yaml.out &&
+	test_debug "cat yaml.out" &&
+	grep "My YAML Format" yaml.out &&
+	XDG_CONFIG_HOME="$(pwd)/yaml" flux jobs --format=myformat
+'
+test_expect_success 'flux-jobs config can be json' '
+	mkdir -p json/flux &&
+	cat <<-EOF >json/flux/flux-jobs.json &&
+	{ "formats": { "myformat": { "description": "My JSON Format",
+	    "format": "{id.words} {runtime}" } } }
+	EOF
+	XDG_CONFIG_HOME="$(pwd)/json" flux jobs --format=help >json.out &&
+	test_debug "cat json.out" &&
+	grep "My JSON Format" json.out &&
+	XDG_CONFIG_HOME="$(pwd)/json" flux jobs --format=myformat
+'
+test_expect_success 'flux-jobs config may omit description' '
+	cat <<-EOF >dir/flux/flux-jobs.toml &&
+	[formats.myformat]
+	format = "{id.words}"
+	EOF
+	XDG_CONFIG_HOME="$(pwd)/dir" flux jobs --format=help >myformat.out &&
+	test_debug "cat myformat.out" &&
+	grep "myformat" myformat.out
+'
+test_expect_success 'flux-jobs --format=get-config works' '
+	flux jobs --format=get-config=default >get-config.out &&
+	test_debug "cat get-config.out" &&
+	grep default get-config.out
+'
+test_expect_success 'XDG_CONFIG_DIRS can be colon-separated paths' '
+	mkdir -p dir2/flux &&
+	cat <<-EOF >dir2/flux/flux-jobs.toml &&
+	[formats.myformat]
+        description = "my new format"
+	format = "{id.words} {name}"
+	EOF
+	XDG_CONFIG_DIRS="$(pwd)/dir2:$(pwd)/dir" \
+		flux jobs --format=help >myformat.out &&
+	test_debug "cat myformat.out" &&
+	grep "my new format" myformat.out &&
+	XDG_CONFIG_DIRS="$(pwd)/dir:$(pwd)/dir2" \
+		flux jobs --format=myformat
+'
+test_expect_success 'Invalid flux-jobs config file causes meaningful error' '
+	mkdir -p dir/flux &&
+	cat <<-EOF >dir/flux/flux-jobs.toml &&
+	[formats.myformat]
+        description = "missing format"
+	EOF
+	( export XDG_CONFIG_HOME="$(pwd)/dir" &&
+	  test_must_fail flux jobs --format=help >missing.out 2>&1
+	) &&
+	grep "required key .* missing" missing.out
+'
+test_expect_success 'flux-jobs does not allow internal formats to be overridden' '
+	cat <<-EOF >dir/flux/flux-jobs.toml &&
+	[formats.default]
+	description  = "override of default format"
+	format = "{id}"
+	EOF
+	( export XDG_CONFIG_HOME="$(pwd)/dir" &&
+	  test_must_fail flux jobs --format=help >no-override.out 2>&1
+	) &&
+	grep "override of builtin format" no-override.out
+
+'
+test_expect_success 'flux-jobs checks that formats is a mapping' '
+	cat <<-EOF >dir/flux/flux-jobs.toml &&
+	formats = "yay"
+	EOF
+	( export XDG_CONFIG_HOME="$(pwd)/dir" &&
+	  test_must_fail flux jobs --format=help >notamapping.out 2>&1
+	) &&
+	grep "must be a mapping" notamapping.out
+'
+test_expect_success 'flux-jobs checks that formats.NAME is a mapping' '
+	cat <<-EOF >dir/flux/flux-jobs.toml &&
+	formats.myformat = 42
+	EOF
+	( export XDG_CONFIG_HOME="$(pwd)/dir" &&
+	  test_must_fail flux jobs --format=help >notamapping2.out 2>&1
+	) &&
+	grep "must be a mapping" notamapping2.out
+'
+test_expect_success 'flux-jobs checks that formats.NAME.format is a string' '
+	cat <<-EOF >dir/flux/flux-jobs.toml &&
+	[formats.myformat]
+	format = 42
+	EOF
+	( export XDG_CONFIG_HOME="$(pwd)/dir" &&
+	  test_must_fail flux jobs --format=help >notastring.out 2>&1
+	) &&
+	grep "must be a string" notastring.out
+'
+test_expect_success 'Invalid flux-jobs invalid config causes meaningful error' '
+	mkdir -p dir/flux &&
+	cat <<-EOF >dir/flux/flux-jobs.toml &&
+	[formats.myformat]
+        description = "invalid toml"
+	format = "{id.f58}
+	EOF
+	( export XDG_CONFIG_HOME="$(pwd)/dir" &&
+	  test_must_fail flux jobs --format=help >badtoml.out 2>&1
+	) &&
+	grep "flux-jobs.toml" badtoml.out
+'
+test_done
diff --git a/t/t2800-jobs-instance-info.t b/t/t2800-jobs-instance-info.t
new file mode 100755
index 000000000000..6d6ae13a8ab4
--- /dev/null
+++ b/t/t2800-jobs-instance-info.t
@@ -0,0 +1,69 @@
+#!/bin/sh
+
+test_description='Test flux jobs command with instance.* attributes'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 job
+
+export FLUX_URI_RESOLVE_LOCAL=t
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+
+#  Launch some test instance jobs with varying characteristics
+#
+#  1. Launch a job that exits immediately so we have a completed instance
+#
+#  2. Launch an instance job that then submits 4 sleep jobs, touches a
+#     file so we know when the jobs have been submitted, and waits for
+#     all jobs to complete (essentially forever)
+#
+#  3. Start a normal job to ensure instance info is blank for non-instance
+#     jobs.
+#
+test_expect_success 'start a set of Flux instances' '
+	id=$(flux submit flux start true) &&
+	id2=$(flux submit -N2 -n2 -c1 flux start \
+		"flux run false ; \
+		 flux submit --cc=1-4 sleep 300 && \
+		 touch ready && \
+		 flux queue idle") &&
+	flux submit sleep 600 &&
+	flux job wait-event $id clean &&
+	$waitfile -t 60 ready
+'
+test_expect_success 'flux-jobs can get instance info' "
+	flux jobs -ao '{id.f58:>12} {instance.stats:^25} {instance.utilization!P:>5} {instance.gpu_utilization!P:>5}' > jobs.out &&
+	test_debug 'cat jobs.out'
+"
+test_expect_success 'flux-jobs -o {instance.stats} worked' '
+	grep F:1 jobs.out
+'
+test_expect_success 'flux-jobs {instance.utilization!P} works as expected' "
+	flux jobs -ano \
+		'{instance.utilization!P},{instance.utilization!P:h}' \
+		> jobs-P.out &&
+	cat >jobs-P.expected <<-EOF &&
+	,-
+	100%,100%
+	,-
+	EOF
+	test_cmp jobs-P.expected jobs-P.out
+"
+test_expect_success 'flux-jobs instance fields empty for completed job' '
+	grep $id jobs.out > completed.out &&
+	grep "$id  *$" completed.out
+'
+test_expect_success 'flux-jobs instance headers work' '
+	cat >headers.expected <<-EOF &&
+	       JOBID           STATS           CORE%  GPU%
+	EOF
+	head -n1 jobs.out >headers &&
+	test_cmp headers.expected headers
+'
+test_expect_success 'flux-jobs {instance.stats.total} works' '
+	test $(flux jobs -no {instance.stats.total} $id2) = 5
+'
+test_expect_success 'flux-jobs {instance.progress} works' '
+	test $(flux jobs -no {instance.progress:.1f} $id2) = 0.2
+'
+test_done
diff --git a/t/t2800-jobs-recursive.t b/t/t2800-jobs-recursive.t
new file mode 100755
index 000000000000..ebccee5a2fc6
--- /dev/null
+++ b/t/t2800-jobs-recursive.t
@@ -0,0 +1,117 @@
+#!/bin/sh
+
+test_description='Test flux jobs command with --recursive'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 2 job
+flux version | grep -q libflux-security && test_set_prereq FLUX_SECURITY
+
+export FLUX_PYCLI_LOGLEVEL=10
+export FLUX_URI_RESOLVE_LOCAL=t
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py --line-buffer -f asciicast"
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+sign_as=${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py
+
+submit_fake_user_instance()
+{
+	FAKE_USERID=42
+	test_debug "echo running flux run $@ as userid $FAKE_USERID"
+	flux run --dry-run \
+		--setattr=system.exec.test.run_duration=1d hostname | \
+		flux python $sign_as $FAKE_USERID \
+		>job.signed &&
+	FLUX_HANDLE_USERID=$FAKE_USERID \
+		flux job submit --flags=signed job.signed >altid &&
+	flux job memo $(cat altid) uri=local:///dev/null
+}
+
+
+#  Start a child instance that immediately exits, so that we can test
+#   that `flux jobs -R` doesn't error on child instances that are no
+#   longer running.
+#
+#  Then, start a job hiearachy with 2 child instances, each of which
+#   run a sleep job, touch a ready.<id> file, then block waiting for
+#   the sleep job to finish.
+#
+test_expect_success 'start a recursive job' '
+	id=$(flux submit flux start true) &&
+	rid=$(flux submit -n2 \
+		flux start \
+		flux submit --wait --cc=1-2 flux start \
+			"flux submit sleep 300 && \
+			 touch ready.\$FLUX_JOB_CC && \
+			 flux queue idle") &&
+	flux job wait-event $id clean
+'
+test_expect_success 'allow guest user access to testexec' '
+	flux config load <<-EOF
+	[exec.testexec]
+	allow-guests = true
+	EOF
+'
+test_expect_success FLUX_SECURITY 'submit fake instance job as another user' '
+	submit_fake_user_instance
+'
+blue_line_count() {
+        grep -c -o "\\u001b\[01\;34m" $1 || true
+}
+test_expect_success 'instance jobs are highlighted in blue' '
+	$runpty -o jobs.cast flux jobs  &&
+	test $(blue_line_count jobs.cast) -eq 1
+'
+test_expect_success 'wait for hierarchy to be ready' '
+	flux getattr broker.pid &&
+	$waitfile -t 60 ready.1 &&
+	$waitfile -t 60 ready.2
+'
+test_expect_success 'flux jobs --recursive works' '
+	flux jobs --recursive > recursive.out &&
+	test_debug "cat recursive.out" &&
+	test $(grep -c : recursive.out) -eq 3
+'
+test_expect_success 'flux jobs --recursive avoids other user jobs by default' '
+	flux jobs -A --recursive > recursive.out &&
+	test_debug "cat recursive.out" &&
+	test $(grep -c : recursive.out) -eq 3
+'
+test_expect_success 'flux jobs -o {id} --recursive works' '
+	flux jobs -o {id.f58} --recursive > recursive-o.out &&
+	test_debug "cat recursive.out" &&
+	test $(grep -c : recursive.out) -eq 3
+'
+test_expect_success 'flux jobs --recursive --stats works' '
+	flux jobs --recursive --stats > recursive-stats.out &&
+	test_debug "cat recursive-stats.out" &&
+	test $(grep -c running recursive-stats.out) -eq 4
+'
+test_expect_success 'flux jobs --recursive --level  works' '
+	flux jobs --recursive --level=1 >recursive-level.out &&
+	test_debug "cat recursive-level.out" &&
+	test $(grep -c : recursive-level.out) -eq 1
+'
+test_expect_success 'flux jobs --recursive JOBID works' '
+	flux jobs --recursive $rid > recursive-jobid.out &&
+	test_debug "cat recursive-jobid.out" &&
+	test $(grep -c : recursive-jobid.out) -eq 3
+'
+test_expect_success FLUX_SECURITY \
+	'flux jobs --recurse-all tries to recurse other user jobs' '
+	flux jobs -A --recurse-all > recurse-all.out &&
+	test_debug "cat recurse-all.out"  &&
+	grep $(cat altid): recurse-all.out
+'
+test_expect_success 'flux jobs --json works with recursive jobs' '
+	flux jobs -A --recursive --json > recursive.json &&
+	jq -e ".jobs[] | select(.uri)" < recursive.json &&
+	jq -e ".jobs[] | select(.uri) | .jobs[0].id > 0" < recursive.json
+'
+test_expect_success FLUX_SECURITY 'cancel alternate user job' '
+	flux cancel $(cat altid)
+'
+test_expect_success 'cancel recursive job safely' '
+	flux proxy $rid flux cancel --all &&
+	flux job wait-event -vt 30 $rid clean
+'
+test_done
diff --git a/t/t2801-top-cmd.t b/t/t2801-top-cmd.t
new file mode 100755
index 000000000000..26480bc578b1
--- /dev/null
+++ b/t/t2801-top-cmd.t
@@ -0,0 +1,440 @@
+#!/bin/sh
+
+test_description='Test flux top command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4 full
+
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py"
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+testssh="${SHARNESS_TEST_SRCDIR}/scripts/tssh"
+
+export FLUX_URI_RESOLVE_LOCAL=t
+
+# To ensure no raciness in tests below, ensure the job-list
+# module knows about submitted jobs in desired states
+job_list_wait_state() {
+	id=$1
+	state=$2
+	flux job list-ids --wait-state=$2 $1 > /dev/null
+}
+
+# flux-top will redraw panes when a heartbeat is received, which could
+# lead to racy output with tests below.  Remove the heartbeat module to
+# remove this possibility.
+test_expect_success 'set high heartbeat period' '
+	flux module remove heartbeat
+'
+test_expect_success 'flux-top -h prints custom usage' '
+	flux top -h 2>usage &&
+	grep "Usage:.*TARGET" usage
+'
+test_expect_success 'flux-top fails on unknown option' '
+	test_must_fail flux top --notopt 2>notopt.err &&
+	grep "unrecognized option" notopt.err
+'
+test_expect_success 'flux-top fails if FLUX_URI is set wrong' '
+	(FLUX_URI=noturi test_must_fail $runpty -n --stderr=baduri.err flux top) &&
+	grep "connecting to Flux" baduri.err
+'
+test_expect_success 'flux-top fails if job argument is not a valid URI' '
+	test_must_fail $runpty -n --stderr=baduri.err flux top baduri &&
+	test_debug "cat baduri.err" &&
+	grep "failed to resolve" baduri.err
+'
+test_expect_success 'flux-top fails if job argument is unknown' '
+	test_must_fail $runpty -n --stderr=unkjobid.err flux top 12345 &&
+	grep "jobid 12345 not found" unkjobid.err
+'
+test_expect_success 'flux-top --test-exit works with a pty' '
+	$runpty flux top --test-exit >/dev/null
+'
+test_expect_success 'flux-top summary shows no jobs initially' '
+	nnodes=$(flux resource list --format="{nnodes}") &&
+	ncores=$(flux resource list --format="{ncores}") &&
+	$runpty flux top --test-exit --test-exit-dump=nojobs.out > /dev/null &&
+	grep "nodes 0/${nnodes}" nojobs.out &&
+	grep "cores 0/${ncores}" nojobs.out &&
+	grep "0 complete" nojobs.out &&
+	grep "0 pending" nojobs.out &&
+	grep "0 running" nojobs.out &&
+	grep "0 failed" nojobs.out
+'
+test_expect_success 'flux-top falls back to sched.resource-status' '
+	FLUX_RESOURCE_LIST_RPC=foo.status \
+		$runpty \
+		flux top --test-exit --test-exit-dump=fallback.out >/dev/null &&
+	grep "nodes 0/${nnodes}" nojobs.out &&
+	grep "cores 0/${ncores}" nojobs.out
+'
+# Note: jpXCZedGfVQ is the base58 representation of FLUX_JOBID_ANY. We
+# grep for this value without f or ƒ in case build environment influences
+# presence of one of the other.
+#
+test_expect_success 'flux-top does not display FLUX_JOBID_ANY jobid in title' '
+	test_must_fail grep jpXCZedGfVQ nojobs.out
+'
+test_expect_success 'run a test job to completion' '
+	flux submit --wait -n1 flux start true >jobid &&
+	job_list_wait_state $(cat jobid) INACTIVE
+'
+test_expect_success 'flux-top summary shows one completed job' '
+	nnodes=$(flux resource list --format="{nnodes}") &&
+	ncores=$(flux resource list --format="{ncores}") &&
+	$runpty flux top --test-exit --test-exit-dump=onejob.out >/dev/null &&
+	grep "nodes 0/${nnodes}" onejob.out &&
+	grep "cores 0/${ncores}" onejob.out &&
+	grep "1 complete" onejob.out &&
+	grep "0 pending" onejob.out &&
+	grep "0 running" onejob.out &&
+	grep "0 failed" onejob.out
+'
+test_expect_success 'flux-top fails if job is not running' '
+	test_must_fail \
+		$runpty -n --stderr=notrun.err flux top $(cat jobid) &&
+	test_debug "cat notrun.err" &&
+	grep "jobid $(cat jobid) is not running" notrun.err
+'
+test_expect_success 'flux-top fails if stdin is not a tty' '
+	test_must_fail flux top --test-exit </dev/null 2>notty.err &&
+	grep "stdin is not a terminal" notty.err
+'
+test_expect_success 'flux-top fails if TERM is not supported' '
+	test_must_fail $runpty --term=dumb --stderr=dumb.err flux top &&
+	grep "terminal does not support required capabilities" dumb.err
+'
+test_expect_success 'flux-top --color options work' '
+	$runpty flux top --color --test-exit >/dev/null &&
+	$runpty flux top --color=auto --test-exit >/dev/null &&
+	$runpty flux top --color=always --test-exit >/dev/null &&
+	$runpty flux top --color=never --test-exit >/dev/null
+'
+test_expect_success 'flux-top --color fails with bad input' '
+	test_must_fail $runpty flux top --color=foo --test-exit >/dev/null
+'
+test_expect_success 'submit batch script and wait for it to start' '
+	cat >batch.sh <<-EOT &&
+	#!/bin/sh
+	flux submit --wait-event=start sleep 300
+	touch job2-has-started
+	flux queue drain
+	EOT
+	chmod +x batch.sh &&
+	flux batch -t30m -n1 batch.sh >jobid2 &&
+	$waitfile job2-has-started
+'
+test_expect_success 'flux-top shows 1 job running' '
+	nnodes=$(flux resource list --format="{nnodes}") &&
+	ncores=$(flux resource list --format="{ncores}") &&
+	$runpty flux top --test-exit --test-exit-dump=runningjob.out >/dev/null &&
+	grep "nodes 1/${nnodes}" runningjob.out &&
+	grep "cores 1/${ncores}" runningjob.out &&
+	grep "1 complete" runningjob.out &&
+	grep "0 pending" runningjob.out &&
+	grep "1 running" runningjob.out &&
+	grep "0 failed" runningjob.out &&
+	grep "batch.sh" runningjob.out &&
+	test_must_fail grep sleep runningjob.out
+'
+test_expect_success 'flux-top JOBID works' '
+	FLUX_SSH=$testssh $runpty flux top --test-exit \
+		--test-exit-dump=topjob.out $(cat jobid2) >/dev/null &&
+	grep "nodes 1/1" topjob.out &&
+	grep "cores 1/1" topjob.out &&
+	grep "0 complete" topjob.out &&
+	grep "0 pending" topjob.out &&
+	grep "1 running" topjob.out &&
+	grep "0 failed" topjob.out &&
+	grep "sleep" topjob.out &&
+	test_must_fail grep "batch.sh" topjob.out
+'
+test_expect_success 'submit non-batch job and wait for it to start' '
+	flux submit -n1 \
+		bash -c "touch job3-has-started && sleep 300" >jobid3 &&
+	$waitfile job3-has-started &&
+	job_list_wait_state $(cat jobid3) RUN
+'
+test_expect_success 'flux-top shows 2 jobs running' '
+	nnodes=$(flux resource list --format="{nnodes}") &&
+	ncores=$(flux resource list --format="{ncores}") &&
+	$runpty flux top --test-exit --test-exit-dump=tworunningjobs.out >/dev/null &&
+	grep "nodes 2/${nnodes}" tworunningjobs.out &&
+	grep "cores 2/${ncores}" tworunningjobs.out &&
+	grep "1 complete" tworunningjobs.out &&
+	grep "0 pending" tworunningjobs.out &&
+	grep "2 running" tworunningjobs.out &&
+	grep "0 failed" tworunningjobs.out &&
+	grep "batch.sh" tworunningjobs.out &&
+	grep "bash" tworunningjobs.out
+'
+test_expect_success 'flux-top JOBID fails when JOBID is not a flux instance' '
+	FLUX_SSH=$testssh test_must_fail \
+		$runpty --format=asciicast -o notflux.log \
+		flux top --test-exit $(cat jobid3) &&
+	test_debug "cat notflux.log" &&
+	grep "URI not found" notflux.log
+'
+test_expect_success NO_CHAIN_LINT 'flux-top quits on q keypress' '
+	$runpty --quit-char=q --format=asciicast -o keys.log flux top &
+	pid=$! &&
+	sleep 1 &&
+	kill -USR1 $pid &&
+	wait $pid
+'
+# there are two jobs running, the first (listed higher) is the
+# non-batch job, the second (listed lower) is the batch job.  So we're
+# moving down three times (highlight first job, highlight second job,
+# cycle back to first job), moving up (cycle back to second job), then
+# hitting enter.
+#
+# we then hit quit twice to exit
+test_expect_success NO_CHAIN_LINT 'flux-top can call itself recursively' '
+	SHELL=/bin/sh &&
+	flux jobs &&
+	flux proxy $(cat jobid2) flux jobs -c1 -no {id} >expected.id &&
+	cat <<-EOF >recurse.in &&
+	{ "version": 2 }
+	[0.5, "i", "j"]
+	[1.0, "i", "j"]
+	[1.5, "i", "j"]
+	[2.0, "i", "k"]
+	[2.5, "i", "\n"]
+	[3.25, "i", "q"]
+	[3.75, "i", "q"]
+	EOF
+	$runpty -o recurse.log --input=recurse.in flux top &&
+	grep -q $(echo $(cat expected.id) | sed "s/ƒ//") recurse.log
+'
+# note that FLUX_URI_RESOLVE_LOCAL=t is intentionally not set on
+# the runpty line below, as we're using a fake ssh
+test_expect_success NO_CHAIN_LINT 'flux-top does not exit on recursive failure' '
+	cat <<-EOF1 >ssh-fail &&
+	#!/bin/sh
+	printf "ssh failure\n" >&2
+	EOF1
+	chmod +x ssh-fail &&
+	SHELL=/bin/sh &&
+	flux jobs &&
+	flux proxy $(cat jobid2) flux jobs -c1 -no {id} >expected.id &&
+	cat <<-EOF >recurse-fail.in &&
+	{ "version": 2 }
+	[0.5, "i", "j"]
+	[1.0, "i", "j"]
+	[1.5, "i", "j"]
+	[2.0, "i", "k"]
+	[2.5, "i", "\n"]
+	[3.25, "i", "x"]
+	[3.75, "i", "q"]
+	EOF
+	unset FLUX_URI_RESOLVE_LOCAL &&
+	FLUX_SSH=$(pwd)/ssh-fail \
+	    $runpty -f asciicast -o recurse-fail.log \
+	        --input=recurse-fail.in flux top &&
+	export FLUX_URI_RESOLVE_LOCAL=t
+	grep -qi "error connecting to Flux" recurse-fail.log
+'
+test_expect_success 'cleanup running jobs' '
+	flux cancel $(cat jobid2) $(cat jobid3) &&
+	flux job wait-event $(cat jobid2) clean &&
+	flux job wait-event $(cat jobid3) clean
+'
+test_expect_success 'flux-top shows jobs canceled' '
+	nnodes=$(flux resource list --format="{nnodes}") &&
+	ncores=$(flux resource list --format="{ncores}") &&
+	$runpty flux top --test-exit --test-exit-dump=canceledjobs.out >/dev/null &&
+	grep "nodes 0/${nnodes}" canceledjobs.out &&
+	grep "cores 0/${ncores}" canceledjobs.out &&
+	grep "1 complete" canceledjobs.out &&
+	grep "0 pending" canceledjobs.out &&
+	grep "0 running" canceledjobs.out &&
+	grep "2 failed" canceledjobs.out &&
+	test_must_fail grep "batch.sh" canceledjobs.out &&
+	test_must_fail grep "bash" canceledjobs.out
+'
+test_expect_success NO_CHAIN_LINT 'flux-top works with FLUX_F58_FORCE_ASCII' '
+       FLUX_F58_FORCE_ASCII=1 $runpty -f asciicast -o normalf.log \
+			      flux top --test-exit
+'
+test_expect_success 'configure queues and resource split amongst queues' '
+	flux R encode -r 0-3 -p batch:0-1 -p debug:2-3 \
+	   | tr -d "\n" \
+	   | flux kvs put -r resource.R=- &&
+	flux config load <<-EOT &&
+	[queues.batch]
+	requires = [ "batch" ]
+	[queues.debug]
+	requires = [ "debug" ]
+	EOT
+	flux queue start --all &&
+	flux module unload sched-simple &&
+	flux module reload resource &&
+	flux module load sched-simple
+'
+test_expect_success 'submit a bunch of jobs' '
+	flux submit --cc=0-1 --queue=batch bash -c "sleep 300" > batch.ids &&
+	flux submit --queue=debug sleep 300 > debug.ids &&
+	job_list_wait_state $(head -n1 batch.ids) RUN &&
+	job_list_wait_state $(tail -n1 batch.ids) RUN &&
+	job_list_wait_state $(cat debug.ids) RUN
+'
+test_expect_success 'flux-top displays job queues' '
+	$runpty -f asciicast -o queue.log flux top --test-exit &&
+	grep QUEUE queue.log &&
+	grep batch queue.log &&
+	grep debug queue.log
+'
+test_expect_success 'flux-top shows expected data in queues' '
+	$runpty flux top --test-exit --test-exit-dump=all.out >/dev/null &&
+	grep "nodes 3/4" all.out &&
+	grep "cores 3/4" all.out &&
+	grep "1 complete" all.out &&
+	grep "0 pending" all.out &&
+	grep "3 running" all.out &&
+	grep "2 failed" all.out
+'
+test_expect_success 'flux-top fails on invalid queue' '
+	test_must_fail flux top --queue=foobar
+'
+test_expect_success 'flux-top shows expected data in batch queue' '
+	$runpty flux top --queue=batch --test-exit \
+	    --test-exit-dump=batchq.out >/dev/null &&
+	grep "nodes 2/2" batchq.out &&
+	grep "cores 2/2" batchq.out &&
+	grep "0 complete" batchq.out &&
+	grep "0 pending" batchq.out &&
+	grep "2 running" batchq.out &&
+	grep "0 failed" batchq.out
+'
+test_expect_success 'flux-top shows expected data in debug queue' '
+	$runpty flux top --queue=debug --test-exit \
+	    --test-exit-dump=debugq.out >/dev/null &&
+	grep "nodes 1/2" debugq.out &&
+	grep "cores 1/2" debugq.out &&
+	grep "0 complete" debugq.out &&
+	grep "0 pending" debugq.out &&
+	grep "1 running" debugq.out &&
+	grep "0 failed" debugq.out &&
+	test $(grep sleep debugq.out | wc -l) -eq 1 &&
+	test $(grep debug debugq.out | wc -l) -eq 1
+'
+test_expect_success 'cancel all jobs' '
+	flux cancel --all &&
+	flux queue drain
+'
+test_expect_success 'flux-top shows expected data in queues after cancels' '
+	$runpty flux top --test-exit --test-exit-dump=allC.out >/dev/null &&
+	grep "nodes 0/4" allC.out &&
+	grep "cores 0/4" allC.out &&
+	grep "1 complete" allC.out &&
+	grep "0 pending" allC.out &&
+	grep "0 running" allC.out &&
+	grep "5 failed" allC.out
+'
+test_expect_success 'flux-top shows expected data in batch queue after cancels' '
+	$runpty flux top --queue=batch --test-exit \
+	    --test-exit-dump=batchqC.out >/dev/null &&
+	grep "nodes 0/2" batchqC.out &&
+	grep "cores 0/2" batchqC.out &&
+	grep "0 complete" batchqC.out &&
+	grep "0 pending" batchqC.out &&
+	grep "0 running" batchqC.out &&
+	grep "2 failed" batchqC.out
+'
+test_expect_success 'flux-top shows expected data in debug queue after cancels' '
+	$runpty flux top --queue=debug --test-exit \
+	    --test-exit-dump=debugqC.out >/dev/null &&
+	grep "nodes 0/2" debugqC.out &&
+	grep "cores 0/2" debugqC.out &&
+	grep "0 complete" debugqC.out &&
+	grep "0 pending" debugqC.out &&
+	grep "0 running" debugqC.out &&
+	grep "1 failed" debugqC.out
+'
+# for interactive test below, job submission order here is important.
+# first two jobs are to batch queue, last is to debug queue.  This
+# leads to the debug queue job normally being listed first when jobs
+# in all queues are listed.  Thus we can test that the jobs specific
+# to the batch queue are listed correctly when there is queue
+# filtering
+test_expect_success 'submit jobs to queues for interactive test' '
+	cat >batchQ.sh <<-EOT &&
+	#!/bin/sh
+	flux submit --wait-event=start sleep 300
+	touch job-queue1-has-started
+	flux queue drain
+	EOT
+	chmod +x batchQ.sh &&
+	flux batch -t30m -n1 --queue=batch batchQ.sh >jobidQ1 &&
+	$waitfile job-queue1-has-started &&
+	flux submit -n1 --queue=batch \
+		bash -c "touch job-queue2-has-started && sleep 300" >jobidQ2 &&
+	$waitfile job-queue2-has-started &&
+	flux submit -n1 --queue=debug \
+		bash -c "touch job-queue3-has-started && sleep 300" >jobidQ3 &&
+	$waitfile job-queue3-has-started
+'
+# only two batch jobs should be listed in filtered output.  See non-queue
+# based equivalent test above for description on what this is doing
+# interactively.
+test_expect_success NO_CHAIN_LINT 'flux-top can call itself recursively with queue filter' '
+	SHELL=/bin/sh &&
+	flux jobs &&
+	flux proxy $(cat jobidQ1) flux jobs -c1 -no {id} >expectedQ.id &&
+	cat <<-EOF >recurseQ.in &&
+	{ "version": 2 }
+	[0.5, "i", "j"]
+	[1.0, "i", "j"]
+	[1.5, "i", "j"]
+	[2.0, "i", "k"]
+	[2.5, "i", "\n"]
+	[3.25, "i", "q"]
+	[3.75, "i", "q"]
+	EOF
+	$runpty -o recurseQ.log --input=recurseQ.in flux top --queue=batch &&
+	grep -q $(echo $(cat expectedQ.id) | sed "s/ƒ//") recurseQ.log
+'
+# in order to test that the left/right arrow keys work, we will
+# "start" flux-top filtering only jobs in the `batch` queue.  Then
+# either the left or right keys should show our job from the debug
+# queue.  One direction shows just debug queue, the other direction
+# shows "all" queues.
+test_expect_success NO_CHAIN_LINT 'flux-top left shows other queue' '
+	SHELL=/bin/sh &&
+	cat <<-EOF >leftQ.in &&
+	{ "version": 2 }
+	[0.50, "i", "h"]
+	[1.00, "i", "q"]
+	EOF
+	FLUX_URI_RESOLVE_LOCAL=t $runpty -o leftQ.log --input=leftQ.in \
+		flux top --queue=batch &&
+	grep -q "debug" leftQ.log
+'
+test_expect_success NO_CHAIN_LINT 'flux-top right shows other queue' '
+	SHELL=/bin/sh &&
+	cat <<-EOF >rightQ.in &&
+	{ "version": 2 }
+	[0.50, "i", "l"]
+	[1.00, "i", "q"]
+	EOF
+	FLUX_URI_RESOLVE_LOCAL=t $runpty -o rightQ.log --input=rightQ.in \
+		flux top --queue=batch &&
+	grep -q "debug" rightQ.log
+'
+test_expect_success NO_CHAIN_LINT 'flux-top cycles left and right work' '
+	SHELL=/bin/sh &&
+	cat <<-EOF >cycleQ.in &&
+	{ "version": 2 }
+	[0.50, "i", "h"]
+	[1.00, "i", "h"]
+	[1.50, "i", "h"]
+	[2.00, "i", "l"]
+	[2.50, "i", "l"]
+	[3.00, "i", "l"]
+	[3.50, "i", "q"]
+	EOF
+	FLUX_URI_RESOLVE_LOCAL=t $runpty -o cycleQ.log --input=cycleQ.in \
+		flux top --queue=batch
+'
+
+test_done
diff --git a/t/t2802-uri-cmd.t b/t/t2802-uri-cmd.t
new file mode 100755
index 000000000000..8b5c733acd0a
--- /dev/null
+++ b/t/t2802-uri-cmd.t
@@ -0,0 +1,183 @@
+#!/bin/sh
+
+test_description='Test flux uri command'
+
+. $(dirname $0)/sharness.sh
+
+# Required to be able to test LSF resolver
+export LSB_JOBID=12345
+
+test_under_flux 2
+
+testssh="${SHARNESS_TEST_SRCDIR}/scripts/tssh"
+
+
+test_expect_success 'flux-uri -h prints list of resolvers' '
+	flux uri --help >help.out >help.out 2>&1 &&
+	test_debug "cat help.out" &&
+	grep "Supported resolver schemes" help.out
+'
+test_expect_success 'flux-uri rejects invalid URI' '
+	test_expect_code 1 flux uri bar &&
+	test_expect_code 1 flux uri bar:foo
+'
+test_expect_success 'flux-uri passes through ssh and local URIs unchanged' '
+	ssh_uri="ssh://bongo@foo.com:123/tmp/flux-xyzzy/local-0" &&
+	local_uri="local:///tmp/flux-xyzzy/local-0" &&
+	result=$(flux uri $ssh_uri) &&
+	test_debug "echo flux uri $ssh_uri returns $result" &&
+	test "$result" = "$ssh_uri" &&
+	result=$(flux uri $local_uri) &&
+	test_debug "echo flux uri $local_uri returns $result" &&
+	test "$result" = "$local_uri" &&
+	result=$(flux uri --local $ssh_uri) &&
+	test_debug "echo flux uri --local $ssh_uri returns $result" &&
+	test "$result" = "$local_uri" &&
+	result=$(flux uri --remote $local_uri) &&
+	test_debug "echo flux uri --remote $local_uri returns $result" &&
+	test "$result" = "ssh://$(hostname)/tmp/flux-xyzzy/local-0"
+'
+test_expect_success 'flux-uri pid resolver works' '
+	test "$(flux uri pid:$$)" =  "$FLUX_URI"
+'
+test_expect_success 'flux-uri pid resolver works with ?local' '
+	test "$(flux uri pid:$$?local)" =  "$FLUX_URI"
+'
+test_expect_success 'flux-uri pid resolver works with ?remote' '
+	test "$(flux uri pid:$$?remote)" =  "$(flux uri --remote $FLUX_URI)"
+'
+test_expect_success 'flux-uri pid resolver works on flux-broker' '
+	test "$(flux uri pid:$(flux getattr broker.pid))" =  "$FLUX_URI"
+'
+test_expect_success 'flux-uri pid resolver works on flux-broker with fallback' '
+	(
+	  export FLUX_FORCE_BROKER_CHILD_FALLBACK=t &&
+	  test "$(flux uri pid:$(flux getattr broker.pid))" =  "$FLUX_URI"
+	)
+'
+test_expect_success 'flux-uri pid resolver fails if broker has no child' '
+	test_must_fail flux uri pid:$(flux exec -r 1 flux getattr broker.pid) \
+		>broker-no-child.out 2>&1 &&
+	grep "is a flux-broker and no child found" broker-no-child.out
+'
+test_expect_success 'flux-uri pid resolver fails for nonexistent pid' '
+	test_expect_code 1 flux uri pid:123456
+'
+test_expect_success 'flux-uri pid resolver fails with ?local&remote' '
+	test_expect_code 1 flux uri "pid:$$?local&remote"
+'
+test_expect_success NO_CHAIN_LINT 'flux-uri pid scheme fails for non-flux pid' '
+	pid=$(bash -c "unset FLUX_URI;sleep 30 >/dev/null 2>&1 & echo \$!") &&
+	test_expect_code 1 flux uri pid:$pid &&
+	kill $pid
+'
+test_expect_success 'flux uri fails for completed job' '
+	complete_id=$(flux submit --wait flux start true) &&
+	test_expect_code 1 flux uri ${complete_id} 2>jobid-notrunning.log &&
+	test_debug "cat jobid-notrunning.log" &&
+	grep "not running" jobid-notrunning.log
+'
+test_expect_success 'start a small hierarchy of Flux instances' '
+	cat <<-EOF >batch.sh &&
+	#!/bin/sh
+	jobid=\$(flux submit -n1 flux start flux run sleep 300) &&
+	flux --parent job memo \$(flux getattr jobid) jobid=\$jobid &&
+	flux job attach \$jobid
+	EOF
+	chmod +x batch.sh &&
+	jobid=$(flux batch -n1 batch.sh) &&
+	flux job wait-event -T offset -vt 180 -c 2 $jobid memo
+'
+test_expect_success 'flux uri resolves jobid argument' '
+	flux proxy $(flux uri --local $jobid) flux getattr jobid >jobid1.out &&
+	test "$(cat jobid1.out)" = "$jobid"
+'
+test_expect_success 'flux uri resolves hierarchical jobid argument'  '
+	jobid2=$(flux jobs -no {user.jobid} $jobid) &&
+	test_debug "echo attempting to resolve jobid:${jobid}/${jobid2}" &&
+	uri=$(FLUX_SSH=$testssh flux uri --local jobid:${jobid}/${jobid2}) &&
+	test_debug "echo jobid:${jobid}/${jobid2} is ${uri}" &&
+	uri=$(FLUX_SSH=$testssh flux uri --local ${jobid}/${jobid2}) &&
+	test_debug "echo ${jobid}/${jobid2} is ${uri}"
+'
+test_expect_success 'flux uri resolves hierarchical jobids with ?local' '
+	test_debug "echo attempting to resolve jobid:${jobid}/${jobid2}" &&
+	uri=$(flux uri jobid:${jobid}/${jobid2}?local) &&
+	test_debug "echo jobid:${jobid}/${jobid2}?local is ${uri}" &&
+	uri=$(flux uri ${jobid}/${jobid2}?local) &&
+	test_debug "echo ${jobid}/${jobid2}?local is ${uri}"
+
+'
+test_expect_success 'flux uri --wait can resolve URI for pending job' '
+	uri=$(flux uri --wait $(flux batch -n1 --wrap hostname)) &&
+	flux job wait-event -vt 30 $(flux job last) clean  &&
+	test "$uri" = "$(flux jobs -no {uri} $(flux job last))"
+'
+test_expect_success 'terminate batch job cleanly' '
+	flux proxy $(flux uri --local ${jobid}) flux cancel --all &&
+	flux job wait-event -vt 30 ${jobid} clean
+'
+test_expect_success 'flux uri jobid returns error for non-instance job' '
+	id=$(flux submit sleep 600) &&
+	test_expect_code 1 flux uri $id
+'
+test_expect_success 'flux uri jobid scheme returns error for invalid jobid' '
+	test_expect_code 1 flux uri jobid:boop
+'
+test_expect_success 'flux uri jobid scheme returns error for unknown jobid' '
+	test_expect_code 1 flux uri jobid:f1
+'
+test_expect_success 'setup fake srun and scontrol cmds for mock slurm testing' '
+	#  slurm resolver runs `srun flux uri slurm:jobid`
+	#  mock the execution of that command here by just returning a uri
+	cat <<-EOF >srun &&
+	#!/bin/sh
+	test -n "\$SRUN_FAIL" && exit 1
+	exec flux uri pid:$$
+	EOF
+	chmod +x srun &&
+	#  slurm resolver attempts to list pids from `scontrol listpids`
+	#  return a single listpids line with our pid for mocking
+	#  set
+	cat <<-EOF >scontrol &&
+	#!/bin/sh
+	test -n "\$REMOTE" && exit 1
+	echo "PID      JOBID    STEPID   LOCALID GLOBALID"
+	echo "1         1234      1234         0        0"
+	echo "$$        1234      1234         0        0"
+	EOF
+	chmod +x scontrol
+'
+test_expect_success 'flux-uri mock testing of slurm resolver works' '
+	result=$(PATH=$(pwd):$PATH flux uri --local slurm:1234) &&
+	test_debug "echo slurm:1234 got $result" &&
+	test "$result" = "$FLUX_URI" &&
+	result=$(PATH=$(pwd):$PATH REMOTE=t flux uri --local slurm:1234) &&
+	test_debug "echo slurm:1234 with REMOTE=t got $result" &&
+	test "$result" = "$FLUX_URI" &&
+	( export PATH=$(pwd):$PATH REMOTE=t SRUN_FAIL=t &&
+	  test_expect_code 1 flux uri slurm:1234 )
+'
+test_expect_success 'setup fake csm_allocation_query for mock lsf testing' '
+	cat <<-EOF >csm_allocation_query &&
+	#!/bin/sh
+	test -n "\$LSF_FAIL" && exit 4
+	echo "compute_nodes:"
+	echo " - lassen9"
+	echo " - lassen10"
+	EOF
+	chmod +x csm_allocation_query &&
+	export CSM_ALLOCATION_QUERY=$(pwd)/csm_allocation_query &&
+	export FLUX_SSH=${SHARNESS_TEST_SRCDIR}/scripts/tssh
+'
+test_expect_success 'flux-uri mock testing of lsf resolver works' '
+	result=$(SHELL=/bin/sh flux uri --local lsf:12345) &&
+	FLUX_URI_1=$(flux exec -r 1 flux getattr local-uri) &&
+	test_debug "echo checking if $result is $FLUX_URI or $FLUX_URI_1" &&
+	test "$result" = "$FLUX_URI" -o "$result" = "$FLUX_URI_1"
+'
+test_expect_success 'cleanup jobs' '
+	flux cancel --all &&
+	flux queue drain
+'
+test_done
diff --git a/t/t2803-flux-pstree.t b/t/t2803-flux-pstree.t
new file mode 100755
index 000000000000..4fdcac7991d4
--- /dev/null
+++ b/t/t2803-flux-pstree.t
@@ -0,0 +1,283 @@
+#!/bin/sh
+
+test_description='Test the flux-pstree command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 2 job
+
+export FLUX_PYCLI_LOGLEVEL=10
+export FLUX_URI_RESOLVE_LOCAL=t
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py --line-buffer -f asciicast"
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+export SHELL=/bin/sh
+
+test_expect_success 'flux-pstree errors on invalid arguments' '
+	test_must_fail flux pstree --skip-root=foo &&
+	test_must_fail flux pstree --details=foo &&
+	test_must_fail flux pstree --filter=bubbling
+'
+test_expect_success 'flux-pstree works in an empty instance' '
+	name="empty" &&
+	flux pstree > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	.
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree --skip-root=yes works in empty instance' '
+	name="empty-skip-root" &&
+	flux pstree --skip-root=yes > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree -x works in empty instance' '
+	name="empty-extended" &&
+	flux pstree -x > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	       JOBID USER     ST NTASKS NNODES  RUNTIME
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree -a works in empty instance' '
+	name="empty-all" &&
+	flux pstree -a > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	.
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree -x --skip-root=no works in empty instance' '
+	name="empty-extended-no-skip-root" &&
+	flux pstree -x --skip-root=no > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	test $(cat ${name}.output | wc -l) -eq 2
+'
+#  Start a child instance that immediately exits, so that we can test
+#   that `flux pstree` doesn't error on child instances that are no
+#   longer running.
+#
+#  Then, start a job hiearachy with 2 child instances, each of which
+#   run a sleep job, touch a ready.<id> file, then block waiting for
+#   the sleep job to finish.
+#
+test_expect_success 'start a recursive job' '
+	id=$(flux submit flux start true) &&
+	rid=$(flux submit -n2 \
+		flux start \
+		flux submit --wait --cc=1-2 flux start \
+			"flux submit sleep 300 && \
+			 touch ready.\$FLUX_JOB_CC && \
+			 flux queue idle") &&
+	flux job wait-event $id clean
+'
+test_expect_success 'wait for hierarchy to be ready' '
+	flux getattr broker.pid &&
+	$waitfile -t 60 ready.1 &&
+	$waitfile -t 60 ready.2
+'
+test_expect_success 'flux-pstree works' '
+	name="normal" &&
+	flux pstree > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	.
+	└── flux
+	    ├── flux
+	    │   └── sleep
+	    └── flux
+	        └── sleep
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree JOBID works' '
+	name="jobid" &&
+	flux pstree $rid > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	flux
+	├── flux
+	│   └── sleep
+	└── flux
+	    └── sleep
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree errors on unknown JOBID' '
+	name="jobidunknown" &&
+	flux pstree 123456789 2> ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	grep "unknown" ${name}.output
+'
+test_expect_success 'flux-pstree errors on illegal JOBID' '
+	name="jobidillegal" &&
+	test_must_fail flux pstree IllegalID 2> ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	grep "invalid JobID value" ${name}.output
+'
+test_expect_success 'flux-pstree works when run inside child job' '
+	name="proxy" &&
+	flux proxy $rid flux pstree > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	flux
+	├── flux
+	│   └── sleep
+	└── flux
+	    └── sleep
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+
+test_expect_success 'flux-pstree -a works' '
+	name="all" &&
+	flux pstree -a > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	.
+	├── flux
+	│   ├── flux
+	│   │   └── sleep:R
+	│   └── flux
+	│       └── sleep:R
+	└── flux:CD
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_columns_variable_preserved && test_set_prereq USE_COLUMNS
+test_expect_success USE_COLUMNS 'flux-pstree truncates at COLUMNS' '
+	name="truncated" &&
+	COLUMNS=16 flux pstree -a > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	.
+	├── flux
+	│   ├── flux
+	│   │   └── sle+
+	│   └── flux
+	│       └── sle+
+	└── flux:CD
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree does not truncate with -l' '
+	name="notruncate" &&
+	COLUMNS=16 flux pstree -al > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	.
+	├── flux
+	│   ├── flux
+	│   │   └── sleep:R
+	│   └── flux
+	│       └── sleep:R
+	└── flux:CD
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree --skip-root=yes works' '
+	name="skip-root" &&
+	flux pstree --skip-root=yes > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	flux
+	├── flux
+	│   └── sleep
+	└── flux
+	    └── sleep
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree --level=1 works' '
+	name="level1" &&
+	flux pstree --level=1 > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	.
+	└── flux
+	    └── 2*[flux]
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree --level=1 --no-combine works' '
+	name="level1-no-combine" &&
+	flux pstree -L1 --no-combine > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	.
+	└── flux
+	    ├── flux
+	    └── flux
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree -x' '
+	name="extended" &&
+	flux pstree -x > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	test $(cat ${name}.output | wc -l) -eq 6 &&
+	head -n 1 ${name}.output | grep NNODES
+'
+test_expect_success 'flux-pstree --details=NAME works' '
+	flux bulksubmit --watch \
+		--job-name=details-{} \
+		--output=details-{}.output \
+		flux pstree --details={} \
+		::: resources progress stats &&
+	test_debug "cat details-*.output" &&
+	grep STATS details-stats.output &&
+	grep CORES details-resources.output &&
+	grep PROG details-progress.output
+'
+test_expect_success 'flux-pstree --label= works' '
+	name="label-format" &&
+	flux pstree --label="{name} foo" > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	. foo
+	└── flux foo
+	    ├── flux foo
+	    │   └── sleep foo
+	    └── flux foo
+	        └── sleep foo
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree --parent-label= works' '
+	name="label-format" &&
+	flux pstree \
+		--label="{name} foo" \
+		--parent-label="{name} bar" \
+		> ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	cat <<-EOF >${name}.expected &&
+	. bar
+	└── flux bar
+	    ├── flux bar
+	    │   └── sleep foo
+	    └── flux bar
+	        └── sleep foo
+	EOF
+	test_cmp ${name}.expected ${name}.output
+'
+test_expect_success 'flux-pstree --detail-format works' '
+	name="detail-format" &&
+	flux pstree --detail-format="{id.f58:<12}" > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	test $(head -n1 ${name}.output) = "JOBID"
+'
+test_expect_success 'flux-pstree --detail-format --skip-root=no works' '
+	name="detail-format2" &&
+	flux pstree \
+		--no-header \
+		--skip-root=no \
+		--detail-format="{id.f58}" > ${name}.output &&
+	test_debug "cat ${name}.output" &&
+	test "$(head -n1 ${name}.output)" = ". ."
+'
+test_done
diff --git a/t/t2804-uptime-cmd.t b/t/t2804-uptime-cmd.t
new file mode 100755
index 000000000000..274c873e31ad
--- /dev/null
+++ b/t/t2804-uptime-cmd.t
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+test_description='Test flux uptime command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4
+
+runas_guest() {
+        local userid=$(($(id -u)+1))
+        FLUX_HANDLE_USERID=$userid FLUX_HANDLE_ROLEMASK=0x2 "$@"
+}
+
+test_expect_success 'flux-uptime works on rank 0' '
+	flux uptime
+'
+test_expect_success 'flux-uptime works on rank 1' '
+	flux exec -r 1 flux uptime
+'
+test_expect_success 'flux-uptime works as guest' '
+	runas_guest flux uptime
+'
+test_expect_success 'flux-uptime reports correct size' '
+	flux uptime | grep "size 4"
+'
+test_expect_success 'flux-uptime reports submit disabled' '
+	flux queue disable "testing" &&
+	flux uptime | grep "submit disabled"
+'
+test_expect_success 'flux-uptime reports scheduler stopped' '
+	flux queue stop &&
+	flux uptime | grep "scheduler stopped"
+'
+test_expect_success 'flux-uptime reports drained node count' '
+	flux resource drain 1,3 &&
+	flux uptime | grep "2 drained"
+'
+
+test_done
diff --git a/t/t2805-startlog-cmd.t b/t/t2805-startlog-cmd.t
new file mode 100755
index 000000000000..fbb530917cae
--- /dev/null
+++ b/t/t2805-startlog-cmd.t
@@ -0,0 +1,64 @@
+#!/bin/sh
+
+test_description='Test flux startlog command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 2
+
+test_expect_success 'flux-startlog works on rank 0' '
+	flux startlog >startlog.out
+'
+test_expect_success 'there is one run interval' '
+	test $(wc -l <startlog.out) -eq 1
+'
+test_expect_success 'the entry shows the instance is still running' '
+	tail -1 startlog.out | grep running
+'
+test_expect_success 'flux-startlog --check --quiet works' '
+	flux startlog --check --quiet
+'
+test_expect_success 'flux-startlog works on rank 1' '
+	flux exec -r 1 flux startlog
+'
+test_expect_success 'flux-startlog posts start+finish to test eventlog' '
+	flux startlog --test-startlog-key=testlog --post-start-event &&
+	flux startlog --test-startlog-key=testlog --post-finish-event
+'
+test_expect_success 'one run interval is listed' '
+	flux startlog --test-startlog-key=testlog >testlog.out &&
+	test $(wc -l <testlog.out) -eq 1
+'
+test_expect_success 'it is not shown as running' '
+	test_must_fail grep running testlog.out
+'
+test_expect_success 'flux-startlog --check returns success on test eventlog' '
+	flux startlog --test-startlog-key=testlog --check
+'
+test_expect_success 'flux-startlog posts start to test eventlog' '
+	flux startlog --test-startlog-key=testlog --post-start-event
+'
+test_expect_success 'flux-startlog --check returns success on test eventlog' '
+	flux startlog --test-startlog-key=testlog --check
+'
+test_expect_success 'flux-startlog posts start to test eventlog' '
+	flux startlog --test-startlog-key=testlog --post-start-event
+'
+test_expect_success 'flux-startlog --check returns failure on test eventlog' '
+	test_must_fail flux startlog --test-startlog-key=testlog --check
+'
+test_expect_success 'append unknown version finish event to test eventlog' '
+	flux kvs eventlog append testlog finish "{\"version\":99}" &&
+	flux kvs eventlog get testlog | tail -1 | grep finish
+'
+test_expect_success 'flux-startlog --check ignores bogus finish event' '
+	test_must_fail flux startlog --test-startlog-key=testlog --check
+'
+test_expect_success 'flux-startlog --badopt fails' '
+	test_must_fail flux startlog --badopt
+'
+test_expect_success 'flux-startlog cannot post events on rank > 0' '
+	test_must_fail flux exec -r 1 flux startlog --post-start-event
+'
+
+test_done
diff --git a/t/t2806-config-cmd.t b/t/t2806-config-cmd.t
new file mode 100755
index 000000000000..4058368dbb8a
--- /dev/null
+++ b/t/t2806-config-cmd.t
@@ -0,0 +1,236 @@
+#!/bin/sh
+
+test_description='Test flux config get'
+
+. $(dirname $0)/sharness.sh
+
+FLUXCONFDIR=$(dirname $(flux config builtin --installed rc1_path))
+test -d $FLUXCONFDIR/system/conf.d || test_set_prereq NO_SYSTEM_CONF
+test -d $FLUXCONFDIR/security/conf.d || test_set_prereq NO_SECURITY_CONF
+test -d $FLUXCONFDIR/imp/conf.d || test_set_prereq NO_IMP_CONF
+
+
+mkdir config
+cat <<EOF >config/config.toml
+[foo]
+a = 42
+b = "meep"
+c = "50s"
+d = 3.14
+e = false
+f = [ "fubar", "barfu" ]
+[foo.bar]
+a = true
+EOF
+
+test_under_flux 1 minimal --config-path=$(pwd)/config
+
+runas_guest() {
+        local userid=$(($(id -u)+1))
+        FLUX_HANDLE_USERID=$userid FLUX_HANDLE_ROLEMASK=0x2 "$@"
+}
+
+test_expect_success 'flux-config with no args fails with message' '
+	test_must_fail flux config 2>noargs.out &&
+	grep "missing subcommand" noargs.out
+'
+test_expect_success 'flux-config get --help prints usage' '
+	flux config get --help 2>get_help.out &&
+	grep -i "query broker configuration values" get_help.out
+'
+test_expect_success 'flux-config dumps entire object' '
+	echo 42 >foo.a.exp &&
+	flux config get | jq -r .foo.a >foo.a.out &&
+	test_cmp foo.a.exp foo.a.out
+'
+test_expect_success 'flux-config get dumps table' '
+	flux config get foo | jq -r .a >foo.a2.out &&
+	test_cmp foo.a.exp foo.a2.out
+'
+test_expect_success 'flux-config get dumps integer table member' '
+	flux config get foo.a >foo.a3.out &&
+	test_cmp foo.a.exp foo.a3.out
+'
+test_expect_success 'flux-config get --type=integer dumps int table member' '
+	flux config get --type=integer foo.a >foo.a4.out &&
+	test_cmp foo.a.exp foo.a4.out
+'
+test_expect_success 'flux-config get --type=string dumps string table member' '
+	echo "meep" >foo.b.exp &&
+	flux config get --type=string foo.b >foo.b.out &&
+	test_cmp foo.b.exp foo.b.out
+'
+test_expect_success 'flux-config get --type=fsd dumps fsd table member' '
+	echo "50s" >foo.c.exp &&
+	flux config get --type=fsd foo.c >foo.c.out &&
+	test_cmp foo.c.exp foo.c.out
+'
+test_expect_success 'flux-config get --type=fsd-integer works' '
+	echo "50" >foo.c_fsd_int.exp &&
+	flux config get --type=fsd-integer foo.c >foo.c_fsd_int.out &&
+	test_cmp foo.c_fsd_int.exp foo.c_fsd_int.out
+'
+test_expect_success 'flux-config get --type=fsd-real works' '
+	echo "50.0" >foo.c_fsd_real.exp &&
+	flux config get --type=fsd-real foo.c | cut -c1-4 >foo.c_fsd_real.out &&
+	test_cmp foo.c_fsd_real.exp foo.c_fsd_real.out
+'
+test_expect_success 'flux-config get --type=fsd fails on non-fsd string' '
+	test_must_fail flux config get --type=fsd foo.b 2>badfsd.err &&
+	grep "does not have the requested type" badfsd.err
+'
+test_expect_success 'flux-config get --type=real dumps real table member' '
+	echo "3.14" >foo.d.exp &&
+	flux config get --type=real foo.d | cut -c1-4 >foo.d.out &&
+	test_cmp foo.d.exp foo.d.out
+'
+test_expect_success 'flux-config get --type=boolean dumps bool table member' '
+	echo "false" >foo.e.exp &&
+	flux config get --type=boolean foo.e >foo.e.out &&
+	test_cmp foo.d.exp foo.d.out
+'
+test_expect_success 'flux-config get --type=array dumps array' '
+	echo barfu >foo.f1.exp &&
+	flux config get --type=array foo.f | jq -r -e ".[1]" >foo.f1.out &&
+	test_cmp foo.f1.exp foo.f1.out
+'
+test_expect_success 'flux-config get --type=object dumps table' '
+	echo true >foo.bar.a.exp &&
+	flux config get --type=object foo.bar | jq -e ".a" >foo.bar.a.out &&
+	test_cmp foo.bar.a.exp foo.bar.a.out
+'
+test_expect_success 'flux-config get dumps subtable value' '
+	flux config get --type=boolean foo.bar.a >foo.bar.a2.out &&
+	test_cmp foo.bar.a.exp foo.bar.a2.out
+'
+test_expect_success 'flux-config get wrong --type fails' '
+	test_must_fail flux config get --type=string foo.a 2>foo.a.type.err &&
+	grep "does not have the requested type" foo.a.type.err
+'
+test_expect_success 'flux-config get unknown --type fails' '
+	test_must_fail flux config get --type=oops foo.a 2>unknown.err &&
+	grep "Unknown type: oops" unknown.err
+'
+test_expect_success 'flux-config get --default dumps integer table member' '
+	flux config get --default=2 foo.a >foo.a5.out &&
+	test_cmp foo.a.exp foo.a5.out
+'
+test_expect_success 'flux-config get fails on non-existent key' '
+	test_must_fail flux config get nokey 2>nokey.err &&
+	grep "is not set" nokey.err
+'
+test_expect_success 'flux-config get --default works on non-existent key' '
+	echo 42 >def.nokey.exp &&
+	flux config get --default=42 nokey >def.nokey.out &&
+	test_cmp def.nokey.exp def.nokey.out
+'
+
+test_expect_success 'flux-config reload works' '
+	rm -f config/config.toml &&
+	flux config reload
+'
+test_expect_success 'flux-config get returns empty object if no config' '
+	echo "{}" >empty.exp &&
+	flux config get >empty.out &&
+	test_cmp empty.exp empty.out
+'
+test_expect_success 'flux-config load handles TOML input' '
+	echo "test.y.z=42" >load.toml &&
+	flux config load <load.toml &&
+	flux config get >load.json
+'
+test_expect_success 'flux-config get returns loaded JSON' '
+	jq -e ".test.y.z == 42" load.json
+'
+test_expect_success 'flux-config load handles empty input' '
+	flux config load </dev/null &&
+	flux config get >empty.json
+'
+test_expect_success 'now the config is empty' '
+	jq -e ". == {}" empty.json
+'
+test_expect_success 'flux-config load handles JSON input' '
+	flux config load <load.json &&
+	flux config get >load2.json
+'
+test_expect_success 'flux-config get returns loaded JSON' '
+	jq -e ".test.y.z == 42" load2.json
+'
+test_expect_success 'flux-config load handles TOML directory' '
+	mkdir -p conf.d &&
+	cat >conf.d/test.toml <<-EOT &&
+	test.a.b.c = 999
+	EOT
+	flux config load conf.d &&
+	flux config get >fromfiles.json
+'
+test_expect_success 'flux-config get returns loaded JSON' '
+	jq -e ".test.a.b.c == 999" fromfiles.json
+'
+test_expect_success 'flux-config load PATH fails on invalid TOML' '
+	cat >conf.d/test.toml <<-EOT &&
+	this is not toml
+	EOT
+	test_must_fail flux config load conf.d
+'
+test_expect_success 'flux-config load fails on invalid TOML' '
+	echo "[whoops" >bad.input &&
+	test_must_fail flux config load <bad.input
+'
+test_expect_success 'flux-config builtin requires key name' '
+	test_must_fail flux config builtin
+'
+test_expect_success 'flux-config builtin fails on unknown key' '
+	test_must_fail flux config builtin unknown 2>bi_unknown.err &&
+	grep "unknown is invalid" bi_unknown.err
+'
+test_expect_success 'flux-config builtin works on known key' '
+	flux config builtin rc1_path
+'
+test_expect_success 'flux-config builtin --intree works' '
+	flux config builtin --intree rc1_path >rc1_path_intree
+'
+test_expect_success 'flux-config builtin --installed works' '
+	flux config builtin --installed rc1_path >rc1_path_installed
+'
+test_expect_success 'flux-config builtin intree and installed return different values' '
+	test_must_fail test_cmp rc1_path_intree rc1_path_installed
+'
+test_expect_success 'flux-config get works as guest' '
+	runas_guest flux config get >obj
+'
+test_expect_success 'flux-config load fails as guest' '
+	test_must_fail runas_guest flux config load <obj
+'
+test_expect_success 'flux-config reload fails as guest' '
+	test_must_fail runas_guest flux config reload
+'
+test_expect_success 'flux-config builtin works as guest' '
+	runas_guest flux config builtin rc1_path
+'
+test_expect_success 'flux-config get works when --config-path points to dir' '
+	mkdir -p altconfig &&
+	echo "flag = 99" >altconfig/foo.toml &&
+	flux config get --config-path=altconfig | jq -e .flag
+'
+test_expect_success 'flux-config get works when --config-path points to file' '
+	flux config get --config-path=altconfig/foo.toml | jq -e .flag
+'
+test_expect_success 'flux-config get fails when --config-path path is wrong' '
+	test_must_fail flux config get --config-path=/not/a/file
+'
+test_expect_success 'flux-config get fails when --config-path is invalid' '
+	echo "x x x" >badconf.toml &&
+	test_must_fail flux config get --config-path=badconf.toml
+'
+test_expect_success NO_SYSTEM_CONF 'flux-config get --config-path=system fails when missing' '
+	test_must_fail flux config get --config-path=system
+'
+test_expect_success NO_SECURITY_CONF 'flux-config get --config-path=security fails when missing' '
+	test_must_fail flux config get --config-path=security
+'
+test_expect_success NO_IMP_CONF 'flux-config get --config-path=imp fails when missing' '
+	test_must_fail flux config get --config-path=imp
+'
+
+test_done
diff --git a/t/t2807-dump-cmd.t b/t/t2807-dump-cmd.t
new file mode 100755
index 000000000000..e8092315f139
--- /dev/null
+++ b/t/t2807-dump-cmd.t
@@ -0,0 +1,316 @@
+#!/bin/sh
+
+test_description='Test flux dump/restore'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 minimal -Sstatedir=$(pwd)
+
+QUERYCMD="flux python ${FLUX_SOURCE_DIR}/t/scripts/sqlite-query.py"
+
+countblobs() {
+	$QUERYCMD -t 100 content.sqlite \
+		"select count(*) from objects;" | sed -e 's/.*= //'
+}
+
+test_expect_success 'load content module' '
+	flux module load content
+'
+
+test_expect_success 'flux-dump with no args prints Usage message' '
+	test_must_fail flux dump 2>dump-noargs.out &&
+	grep "Usage" dump-noargs.out
+'
+test_expect_success 'flux-restore with no args prints Usage message' '
+	test_must_fail flux restore 2>restore-noargs.out &&
+	grep "Usage" restore-noargs.out
+'
+test_expect_success 'flux-dump with no backing store fails' '
+	test_must_fail flux dump --checkpoint foo.tar 2>nostore.err &&
+	grep "checkpoint key unavailable" nostore.err
+'
+test_expect_success 'flux-dump with bad archive file fails' '
+	test_must_fail flux dump /badfile.tar 2>badfile.err &&
+	grep badfile.tar badfile.err
+'
+test_expect_success 'load content-sqlite' '
+	flux module load content-sqlite
+'
+test_expect_success 'flux-dump --checkpoint with missing checkpoint fails' '
+	test_must_fail flux dump --checkpoint foo.tar 2>nocheck.err &&
+	grep "error fetching checkpoint" nocheck.err
+'
+
+test_expect_success 'load kvs and create some kvs content' '
+	printf "%-.*d" 100 0 >x.val &&
+	flux module load kvs &&
+	flux kvs put --no-merge a.b.c=testkey &&
+	flux kvs link linkedthing y &&
+	flux kvs put --no-merge x=$(cat x.val) &&
+	flux kvs link --target-namespace=smurf otherthing z &&
+	flux kvs put --no-merge w= &&
+	flux kvs put --no-merge --append w=foo
+'
+test_expect_success 'unload kvs' '
+	flux module remove kvs
+'
+test_expect_success 'dump default=kvs-primary checkpoint' '
+	flux dump --checkpoint foo.tar
+'
+test_expect_success 'repeat dump with -q' '
+	flux dump -q --checkpoint foo.tar
+'
+test_expect_success 'repeat dump with -v' '
+	flux dump -v --checkpoint foo.tar
+'
+test_expect_success 'repeat dump with --no-cache' '
+	flux dump --no-cache --checkpoint foo.tar
+'
+test_expect_success 'unload content-sqlite' '
+	flux content flush &&
+	flux content dropcache &&
+	flux module remove content-sqlite
+'
+
+# Small values and symlnks are contained in dirent rather than separate
+# objects, so after the above kvs commands, expect a blobcount of:
+# (1) rootdir 1st version
+# (3) rootdir 2nd version + 'a' directory + 'b' directory (c is contained in b)
+# (1) rootdir 3rd version (y is contained in root)
+# (2) rootdir 4th version + 'x'
+# (1) rootdir 5th version (z is contained in root)
+# (1) rootdir 6th version (w probably contained in root)
+# (3) rootdir 7th version ('w' empty blobref + append to 'w')
+# Total: 12 blobs
+#
+test_expect_success 'count blobs representing those five keys' '
+	echo 12 >blobcount.exp &&
+	countblobs >blobcount.out &&
+	test_cmp blobcount.exp blobcount.out
+'
+test_expect_success 'remove backing file and load content-sqlite' '
+	rm -f content.sqlite &&
+	flux module load content-sqlite
+'
+test_expect_success 'restore content' '
+	flux restore --checkpoint foo.tar
+'
+test_expect_success 'repeat restore with -v' '
+	flux restore -v --checkpoint foo.tar
+'
+test_expect_success 'repeat restore with -q' '
+	flux restore -q --checkpoint foo.tar
+'
+test_expect_success 'repeat restore with --no-cache' '
+	flux restore --no-cache --checkpoint foo.tar
+'
+test_expect_success 'unload content-sqlite' '
+	flux content flush &&
+	flux content dropcache &&
+	flux module remove content-sqlite
+'
+
+# Intermediate rootdir versions are not preserved across dump/restore.
+# Expect a blobcount of 4: rootdir 5th version + 'a' + 'b' + 'x',
+#
+test_expect_success 'count blobs after restore'\'s' implicit garbage collection' '
+	echo 4 >blobcount2.exp &&
+	countblobs >blobcount2.out &&
+	test_cmp blobcount2.exp blobcount2.out
+'
+
+test_expect_success 'load content-sqlite + kvs and list content' '
+	flux module load content-sqlite &&
+	flux module load kvs &&
+	flux kvs dir -R
+'
+test_expect_success 'verify that exact KVS content was restored' '
+	test $(flux kvs get a.b.c) = "testkey" &&
+	test $(flux kvs get x) = $(cat x.val) &&
+	test $(flux kvs readlink y) = "linkedthing" &&
+	test $(flux kvs readlink z) = "smurf::otherthing" &&
+	test $(flux kvs get w) = "foo"
+'
+test_expect_success 'now restore to key and verify content' '
+	flux restore -v --key zz foo.tar &&
+	test $(flux kvs get zz.a.b.c) = "testkey" &&
+	test $(flux kvs get zz.x) = $(cat x.val)
+'
+test_expect_success 'try dump - | restore - to key and verify content' '
+	flux dump - | flux restore --key yy - &&
+	test $(flux kvs get a.b.c) = "testkey" &&
+	test $(flux kvs get zz.a.b.c) = "testkey" &&
+	test $(flux kvs get yy.zz.a.b.c) = "testkey" &&
+	test $(flux kvs get x) = $(cat x.val) &&
+	test $(flux kvs get zz.x) = $(cat x.val) &&
+	test $(flux kvs get yy.zz.x) = $(cat x.val) &&
+	test $(flux kvs readlink y) = "linkedthing" &&
+	test $(flux kvs readlink zz.y) = "linkedthing" &&
+	test $(flux kvs readlink yy.zz.y) = "linkedthing" &&
+	test $(flux kvs readlink z) = "smurf::otherthing" &&
+	test $(flux kvs readlink zz.z) = "smurf::otherthing" &&
+	test $(flux kvs readlink yy.zz.z) = "smurf::otherthing"
+'
+test_expect_success 'dump ignores empty kvs directories' '
+	flux kvs mkdir empty &&
+	flux dump -v foo3.tar &&
+	tar tvf foo3.tar >toc &&
+	test_must_fail grep empty toc
+'
+test_expect_success 'restore ignores directories added by tar of filesystem' '
+	mkdir tmp &&
+	(cd tmp &&
+	tar xvf ../foo3.tar &&
+	tar cf - . | flux restore --key test -)
+'
+test_expect_success 'restore without required argument fails' '
+	test_must_fail flux restore foo.tar 2>restore-argmissing.err &&
+	grep "Please specify a restore target" restore-argmissing.err
+'
+test_expect_success 'restore --checkpoint fails with kvs loaded' '
+	test_must_fail flux restore --checkpoint foo.tar 2>restore-cpkvs.err &&
+	grep "please unload kvs" restore-cpkvs.err
+'
+test_expect_success 'unload kvs' '
+	flux module remove kvs
+'
+test_expect_success 'restore to key fails when kvs is not loaded' '
+	test_must_fail flux restore --key foo foo.tar 2>restore-nokvs.err &&
+	grep "error updating" restore-nokvs.err
+'
+test_expect_success 'unload content-sqlite' '
+	flux module remove content-sqlite
+'
+test_expect_success 'restore --checkpoint with no backing store cant flush' '
+	flux restore --checkpoint foo.tar 2>noback.err &&
+	grep "error flushing content cache" noback.err
+'
+test_expect_success 'dump --no-cache with no backing store fails' '
+	test_must_fail flux dump --no-cache --checkpoint x.tar
+'
+test_expect_success 'restore --no-cache with no backing store fails' '
+	test_must_fail flux restore --no-cache --checkpoint foo.tar
+'
+test_expect_success 'run a flux instance, preserving content.sqlite' '
+	mkdir test &&
+	flux start -Sstatedir=$(pwd)/test true
+'
+
+reader() {
+	local dbdir=$1
+        flux start -Sbroker.rc1_path= \
+                -Sbroker.rc3_path=\
+                -Sstatedir=$dbdir\
+                bash -c "\
+                        flux module load content && \
+                        flux module load content-sqlite && \
+                        flux dump --no-cache -q --checkpoint - &&\
+                        flux module remove content-sqlite && \
+                        flux module remove content
+                "
+}
+
+writer() {
+	local dbdir=$1
+        flux start -Sbroker.rc1_path= \
+                -Sbroker.rc3_path= \
+                -Sstatedir=$dbdir \
+                bash -c "\
+                        flux module load content && \
+                        flux module load content-sqlite && \
+                        flux restore --checkpoint - && \
+                        flux module remove content-sqlite && \
+                        flux module remove content
+                "
+}
+
+test_expect_success 'perform offline garbage collection with dump/restore' '
+	mkdir test_bak &&
+	mv test/content.sqlite test_bak/ &&
+	reader test_bak | writer test
+'
+
+test_expect_success 'restart flux instance and try to run a job' '
+	flux start -Sstatedir=test \
+		flux run true
+'
+
+# Cover --size-limit
+
+test_expect_success 'create bigdump.tar with a 12M blob in it' '
+	mkdir -p big &&
+	dd if=/dev/zero of=big/tinyblob bs=1048576 count=1 &&
+	dd if=/dev/zero of=big/bigblob bs=1048576 count=12 &&
+	dd if=/dev/zero of=big/smallblob bs=1048576 count=3 &&
+	dd if=/dev/zero of=big/medblob bs=1048576 count=6 &&
+	dd if=/dev/zero of=big/med2blob bs=1048576 count=6 &&
+	tar cvf bigdump.tar big
+'
+test_expect_success 'restore bigdump.tar and verify blob count' '
+	flux start flux restore \
+		--key=foo bigdump.tar 2>bigdump.err &&
+	grep "restored 5 keys (7 blobs)" bigdump.err
+'
+test_expect_success 'restore bigdump.tar with size limit' '
+	flux start flux restore --size-limit=10485760 \
+		--key=foo bigdump.tar 2>bigdump2.err &&
+	grep "exceeds" bigdump2.err &&
+	grep "restored 4 keys (6 blobs)" bigdump2.err
+'
+test_expect_success 'rc1 skips blob that exceeds 100M limit' '
+	dd if=/dev/zero of=big/hugeblob bs=1048576 count=120 &&
+	tar cvf bigdump2.tar big &&
+	flux start -Scontent.restore=bigdump2.tar \
+		true 2>bigdump3.err &&
+	grep "exceeds" bigdump3.err
+'
+
+# Cover dump/restore KVS key pointing to nonexistent blobref
+
+test_expect_success 'load kvs' '
+	flux module load kvs
+'
+test_expect_success 'take a dump and count the keys' '
+	flux dump -v origdump.tar &&
+	tar tvf origdump.tar | wc -l >origdump.count
+'
+
+# Usage: mkbad dirref|valref
+blobref="sha1-996324fa537cd312e564be576bafc21a3f63910d"
+mkbad() {
+	echo '{"data":["'$blobref'"],"type":"'$1'","ver":1}'
+}
+
+test_expect_success 'create a KVS valref with a dangling blobref' '
+	mkbad valref | flux kvs put --treeobj bad.a=- &&
+	test_must_fail flux kvs get bad.a
+'
+test_expect_success 'flux dump fails on bad key' '
+	test_must_fail flux dump baddump.tar
+'
+test_expect_success 'flux dump works with --ignore-failed-read' '
+	flux dump -q --ignore-failed-read baddump.tar
+'
+test_expect_success 'archive contains expected number of keys' '
+	tar tvf baddump.tar | wc -l >baddump.count &&
+	test_cmp origdump.count baddump.count
+'
+test_expect_success 'create a KVS dirref with a dangling blobref' '
+	mkbad dirref | flux kvs put --treeobj bad.b=- &&
+	test_must_fail flux kvs get bad.b
+'
+test_expect_success 'flux dump works with --ignore-failed-read' '
+	flux dump -q --ignore-failed-read baddump2.tar
+'
+test_expect_success 'archive contains expected number of keys' '
+	tar tvf baddump2.tar | wc -l >baddump2.count &&
+	test_cmp origdump.count baddump2.count
+'
+test_expect_success 'remove kvs module' '
+	flux module remove kvs
+'
+test_expect_success 'remove content module' '
+	flux module remove content
+'
+
+test_done
diff --git a/t/t2808-shutdown-cmd.t b/t/t2808-shutdown-cmd.t
new file mode 100755
index 000000000000..9708a539536e
--- /dev/null
+++ b/t/t2808-shutdown-cmd.t
@@ -0,0 +1,334 @@
+#!/bin/sh
+
+test_description='Test flux shutdown command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4
+
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py"
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+
+export FLUX_URI_RESOLVE_LOCAL=t
+
+test_expect_success 'flux-shutdown -h prints custom usage' '
+	flux shutdown -h 2>usage &&
+	grep "Usage:.*TARGET" usage
+'
+test_expect_success 'flux-shutdown fails on unknown option' '
+	test_must_fail flux shutdown --notopt 2>notopt.err &&
+	grep "unrecognized option" notopt.err
+'
+test_expect_success 'flux-shutdown fails if FLUX_URI is set wrong' '
+	(FLUX_URI=noturi test_must_fail flux shutdown 2>baduri.err) &&
+	grep "connecting to Flux" baduri.err
+'
+test_expect_success 'flux-shutdown fails if job argument is not a valid URI' '
+	test_must_fail flux shutdown baduri 2>baduri2.err &&
+	test_debug "cat baduri2.err" &&
+	grep "failed to resolve" baduri2.err
+'
+test_expect_success 'flux-shutdown fails if job argument is unknown' '
+	test_must_fail flux shutdown 12345 2>unkjobid.err &&
+	grep "jobid 12345 not found" unkjobid.err
+'
+
+test_expect_success 'run a test job to completion' '
+	flux submit --wait -n1 flux start true >jobid
+'
+test_expect_success 'flux-shutdown fails if job is not running' '
+	test_must_fail flux shutdown $(cat jobid) 2>notrun.err &&
+	test_debug "cat notrun.err" &&
+	grep "jobid $(cat jobid) is not running" notrun.err
+'
+
+test_expect_success 'submit batch script and wait for it to start' '
+	cat >batch.sh <<-EOT &&
+	#!/bin/sh
+	touch job2-has-started
+	flux run sleep 300
+	EOT
+	chmod +x batch.sh &&
+	flux batch -t30m -n1 batch.sh >jobid2 &&
+	$waitfile job2-has-started
+'
+test_expect_success 'flux-shutdown JOBID works' '
+	flux shutdown $(cat jobid2) 2>batch.err
+'
+test_expect_success 'shutdown output contains rc3 exit code (0)' '
+	grep "rc3.*Exited (rc=0)" batch.err
+'
+test_expect_success 'shutdown output contains state transition to goodbye' '
+	grep "rc3-success: finalize->goodbye" batch.err
+'
+test_expect_success 'job exit code indicates SIGHUP termination' '
+	test_expect_code 129 flux job status $(cat jobid2)
+'
+
+test_expect_success 'submit non-batch job and wait for it to start' '
+	flux submit -n1 \
+		bash -c "touch job3-has-started && sleep 300" >jobid3 &&
+	$waitfile job3-has-started
+'
+test_expect_success 'flux-shutdown JOBID fails when JOBID is not a flux instance' '
+	test_must_fail flux shutdown $(cat jobid3) 2>notflux.err &&
+	test_debug "cat notflux.err" &&
+	grep "URI not found" notflux.err
+'
+test_expect_success 'cancel that job' '
+	flux cancel $(cat jobid3)
+'
+
+test_expect_success 'run instance with no initial program and wait for it to start' '
+	flux submit --wait-event=start \
+		flux start -Sbroker.rc2_none >jobid3 &&
+	run_timeout 30 bash -c "while ! flux uri $(cat jobid3) >uri3; do \
+		sleep 0.1; \
+	done"
+'
+# Retries required here because shutdown fails if instance is not yet
+# in RUN state
+test_expect_success 'flux-shutdown --quiet works' '
+	while ! flux shutdown --quiet $(cat uri3) 2>shut3.err \
+		&& grep "cannot be initiated in state" shut3.err; do \
+		: ; \
+	done
+'
+test_expect_success 'successful shutdown output is empty' '
+	count=$(wc -l <shut3.err) &&
+	test $count -eq 0
+'
+
+test_expect_success 'run batch job and wait for it to start' '
+	cat >batch4.sh <<-EOT &&
+	#!/bin/sh
+	touch job4-has-started
+	flux run sleep 300
+	EOT
+	chmod +x batch4.sh &&
+	flux batch -n1 batch4.sh >jobid4 &&
+	$waitfile job4-has-started
+'
+test_expect_success 'flux-shutdown --verbose works' '
+	flux shutdown --verbose $(cat jobid4) 2>shut4.err
+'
+test_expect_success 'shutdown output contains debug log messages' '
+	grep -q "debug\[0\]:" shut4.err
+'
+
+test_expect_success 'run multi-node batch job and wait for it to start' '
+	cat >batch5.sh <<-EOT &&
+	#!/bin/sh
+	touch job5-has-started
+	flux run sleep 300
+	EOT
+	chmod +x batch5.sh &&
+	flux batch -N2 batch5.sh >jobid5 &&
+	$waitfile job5-has-started
+'
+test_expect_success 'flux-shutdown --background works' '
+	flux shutdown --background $(cat jobid5) 2>shut5.err
+'
+test_expect_success 'job exit code indicates SIGHUP termination' '
+	test_expect_code 129 flux job status $(cat jobid5)
+'
+
+test_expect_success 'flux-shutdown as initial program does not hang' '
+	test_expect_code 129 run_timeout 30 flux start flux shutdown
+'
+
+test_expect_success 'submit batch script and wait for it to start' '
+	rm -f job6-has-started &&
+	cat >batch6.sh <<-EOT &&
+	#!/bin/sh
+	flux run true
+	touch job6-has-started
+	sleep 300
+	EOT
+	chmod +x batch6.sh &&
+	flux batch -t30m -n1 batch6.sh >jobid6 &&
+	$waitfile job6-has-started
+'
+
+test_expect_success 'one job has run in the batch job' '
+	(FLUX_URI=$(flux uri --local $(cat jobid6)) \
+	    flux jobs -n -a -o {id}) >job6_list &&
+	test $(wc -l <job6_list) -eq 1
+'
+
+test_expect_success 'shutdown batch script with --dump' '
+	(FLUX_URI=$(flux uri --local $(cat jobid6)) \
+	    flux shutdown --dump=dump.tgz)
+'
+test_expect_success 'dump file was created' '
+	tar tvf dump.tgz
+'
+test_expect_success 'restart batch script from dump and wait for it to start' '
+	rm -f job6-has-started &&
+	flux batch -t30m -n1 \
+	    --broker-opts=-Scontent.restore=dump.tgz \
+	    batch6.sh >jobid6_try2 &&
+	$waitfile job6-has-started
+'
+test_expect_success 'two jobs have been run in batch job' '
+	(FLUX_URI=$(flux uri --local $(cat jobid6_try2)) \
+	    flux jobs -n -a -o {id}) >job6_list_try2 &&
+	test $(wc -l <job6_list_try2) -eq 2
+'
+test_expect_success 'job id from before restart is in job listing' '
+	grep $(cat job6_list) job6_list_try2
+'
+
+test_expect_success 'shutdown batch script with --gc' '
+	(FLUX_URI=$(flux uri --local $(cat jobid6_try2)) \
+	    flux shutdown --gc)
+'
+test_expect_success 'dump file was created with RESTORE link' '
+	tar tvf dump/RESTORE
+'
+test_expect_success 'clean up dump files from previous tests' '
+	rm -f dump.tgz &&
+	rm -f dump/RESTORE
+'
+test_expect_success 'create config with large gc-threshold config' '
+	cat >kvs.toml <<-EOT
+	[kvs]
+	gc-threshold = 10000
+	EOT
+'
+test_expect_success 'submit batch script and wait for it to start (1)' '
+	rm -f job7-has-started &&
+	cat >batch7.sh <<-EOT &&
+	#!/bin/sh
+	flux run true
+	touch job7-has-started
+	sleep 300
+	EOT
+	chmod +x batch7.sh &&
+	FLUX_CONF_DIR=$(pwd) flux batch -t30m -n1 batch7.sh >jobid7 &&
+	$waitfile job7-has-started
+'
+test_expect_success 'shutdown batch script' '
+	(FLUX_URI=$(flux uri --local $(cat jobid7)) \
+	    flux shutdown)
+'
+test_expect_success 'RESTORE dump not created, gc-threshold not crossed yet' '
+	test_must_fail ls dump/RESTORE
+'
+test_expect_success 'create config with small gc-threshold config' '
+	cat >kvs.toml <<-EOT
+	[kvs]
+	gc-threshold = 5
+	EOT
+'
+test_expect_success 'submit batch script and wait for it to start (2)' '
+	rm -f job7-has-started &&
+	FLUX_CONF_DIR=$(pwd) flux batch -t30m -n1 batch7.sh >jobid7_try2 &&
+	$waitfile job7-has-started
+'
+# this test goes into a script, so we can pass to test_must_fail
+test_expect_success 'shutdown, gc threshold crossed, error no -y or -n' '
+	cat >test_no_y_or_n_cmdline.sh <<-EOT &&
+	#!/bin/sh
+	FLUX_URI=$(flux uri --local $(cat jobid7_try2))
+	flux shutdown
+	EOT
+	chmod +x test_no_y_or_n_cmdline.sh &&
+	test_must_fail ./test_no_y_or_n_cmdline.sh
+'
+test_expect_success 'shutdown, gc threshold crossed, user specifies -n' '
+	(FLUX_URI=$(flux uri --local $(cat jobid7_try2)) \
+	    flux shutdown -n)
+'
+test_expect_success 'RESTORE dump not created, user declined' '
+	test_must_fail ls dump/RESTORE
+'
+test_expect_success 'submit batch script and wait for it to start (3)' '
+	rm -f job7-has-started &&
+	FLUX_CONF_DIR=$(pwd) flux batch -t30m -n1 batch7.sh >jobid7_try3 &&
+	$waitfile job7-has-started
+'
+test_expect_success 'shutdown, gc threshold crossed, user specifies -y' '
+	(FLUX_URI=$(flux uri --local $(cat jobid7_try3)) \
+	    flux shutdown -y)
+'
+test_expect_success 'RESTORE dump created after user accepted' '
+	tar tvf dump/RESTORE
+'
+test_expect_success 'clean up dump files from previous tests' '
+	rm -f dump.tgz &&
+	rm -f dump/RESTORE
+'
+test_expect_success 'submit batch script and wait for it to start (4)' '
+	rm -f job7-has-started &&
+	FLUX_CONF_DIR=$(pwd) flux batch -t30m -n1 batch7.sh >jobid7_try4 &&
+	$waitfile job7-has-started
+'
+test_expect_success 'shutdown, gc threshold crossed, user input n' '
+	cat <<-EOF >jobid7_try4.in &&
+	{ "version": 2 }
+	[0.5, "i", "n\n"]
+	EOF
+	(FLUX_URI=$(flux uri --local $(cat jobid7_try4)) \
+	    $runpty --input=jobid7_try4.in flux shutdown)
+'
+test_expect_success 'RESTORE dump not created, user declined' '
+	test_must_fail ls dump/RESTORE
+'
+test_expect_success 'submit batch script and wait for it to start (5)' '
+	rm -f job7-has-started &&
+	FLUX_CONF_DIR=$(pwd) flux batch -t30m -n1 batch7.sh >jobid7_try5 &&
+	$waitfile job7-has-started
+'
+test_expect_success 'shutdown, gc threshold crossed, user input y' '
+	cat <<-EOF >jobid7_try5.in &&
+	{ "version": 2 }
+	[0.5, "i", "y\n"]
+	EOF
+	(FLUX_URI=$(flux uri --local $(cat jobid7_try5)) \
+	    $runpty --input=jobid7_try5.in flux shutdown)
+'
+test_expect_success 'RESTORE dump created after user accepted' '
+	tar tvf dump/RESTORE
+'
+test_expect_success 'clean up dump files from previous tests' '
+	rm -f dump.tgz &&
+	rm -f dump/RESTORE
+'
+test_expect_success 'submit batch script and wait for it to start (6)' '
+	rm -f job7-has-started &&
+	FLUX_CONF_DIR=$(pwd) flux batch -t30m -n1 batch7.sh >jobid7_try6 &&
+	$waitfile job7-has-started
+'
+test_expect_success 'shutdown, gc threshold crossed, user input default' '
+	cat <<-EOF >jobid7_try6.in &&
+	{ "version": 2 }
+	[0.5, "i", "\n"]
+	EOF
+	(FLUX_URI=$(flux uri --local $(cat jobid7_try6)) \
+	    $runpty --input=jobid7_try6.in flux shutdown)
+'
+test_expect_success 'RESTORE dump created after user accepted' '
+	tar tvf dump/RESTORE
+'
+test_expect_success 'clean up dump files from previous tests' '
+	rm -f dump.tgz &&
+	rm -f dump/RESTORE
+'
+test_expect_success 'submit batch with dump=auto and wait for it to start (8)' '
+	cat >batch.sh <<-EOT &&
+	#!/bin/sh
+	touch job8-has-started
+	flux run sleep 300
+	EOT
+	chmod +x batch.sh &&
+	flux batch -t30m -n1 \
+	    --broker-opts=-Scontent.dump=auto batch.sh >jobid8 &&
+	$waitfile job8-has-started
+'
+test_expect_success 'shutdown --skip-gc does not produce dump' '
+	FLUX_URI=$(flux uri --local $(cat jobid8)) flux shutdown --skip-gc &&
+	test_must_fail tar tvf dump/RESTORE
+'
+
+test_done
diff --git a/t/t2809-job-purge.t b/t/t2809-job-purge.t
new file mode 100755
index 000000000000..913ac781e790
--- /dev/null
+++ b/t/t2809-job-purge.t
@@ -0,0 +1,260 @@
+#!/bin/sh
+
+test_description='Test flux job purge and flux job purge-id'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 full
+
+# Get the number of inactive jobs
+inactive_count() {
+	local how=$1
+
+	if test $how = "job-manager"; then
+		flux module stats --parse=inactive_jobs job-manager
+	elif test $how = "job-list"; then
+		flux jobs --no-header --filter=inactive|wc -l
+	elif test $how = "job-list-stats"; then
+		flux job stats | jq .job_states.inactive
+	else
+		echo Unknown method: $how >&2
+		return 1
+	fi
+}
+
+# Poll for a specific number of inactive jobs
+# Usage: wait_inactive_count method target tries
+# where method is job-manager, job-list, or job-list-stats (jq required)
+wait_inactive_count() {
+	local how=$1
+	local target=$2
+	local tries=$3
+	local count
+	while test $tries -gt 0; do
+		count=$(inactive_count $how)
+		echo $count inactive jobs >&2
+		test $count -eq $target && return 0
+		sleep 0.25
+		tries=$(($tries-1))
+	done
+	return 1
+}
+
+#
+# purge tests
+#
+# Speed up heartbeat-driven purge results to make the test run faster
+test_expect_success 'reload heartbeat module with fast rate' '
+        flux module reload heartbeat period=0.1s
+'
+test_expect_success 'create 10 inactive jobs' '
+	flux submit --cc=1-10 true >jobids &&
+	flux queue drain
+'
+test_expect_success 'verify job KVS eventlogs exist' '
+	for id in $(cat jobids); do \
+	    flux job eventlog $id >/dev/null;  \
+	done
+'
+test_expect_success 'flux job purge with no args says use force' '
+	flux job purge >noargs.out &&
+	grep "use --force to purge" noargs.out
+'
+test_expect_success 'flux job purge with no limits purges 0' '
+	flux job purge --force >noforce.out &&
+	grep "purged 0 inactive jobs" noforce.out
+'
+test_expect_success 'flux job purge --batch=10000 fails' '
+	test_must_fail flux job purge --force --batch=10000 2>bigbatch.err &&
+	grep "batch must be" bigbatch.err
+'
+test_expect_success 'flux job purge --force --num-limit=-42 fails' '
+	test_must_fail flux job purge --num-limit=-42 2>negnum.err &&
+	grep "num limit must be" negnum.err
+'
+test_expect_success 'flux job purge --num-limit=8 would purge 2' '
+	flux job purge --num-limit=8 >num8.out &&
+	grep "use --force to purge 2 " num8.out
+'
+test_expect_success 'flux job purge --force --num-limit=8 purges 2' '
+	flux job purge --force --num-limit=8 >num8f.out &&
+	grep "purged 2 inactive jobs" num8f.out
+'
+test_expect_success 'flux job purge --num-limit=8 would purge 0' '
+	flux job purge --num-limit=8 >num8_again.out &&
+	grep "use --force to purge 0 " num8_again.out
+'
+test_expect_success 'flux job purge --force --num-limit=8 purges 0' '
+	flux job purge --force --num-limit=8 >num8f_again.out &&
+	grep "purged 0 inactive jobs" num8f_again.out
+'
+test_expect_success 'flux job purge --num-limit=6 would purge 2' '
+	flux job purge --num-limit=6 --batch=1 >num6.out &&
+	grep "use --force to purge 2 " num6.out
+'
+test_expect_success 'flux job purge --force --num-limit=6 purges 2' '
+	flux job purge --force --num-limit=6 --batch=1 >num6f.out &&
+	grep "purged 2 inactive jobs" num6f.out
+'
+test_expect_success 'flux job purge --force --num-limit=1000 --age-limit=1ms purges 6' '
+	flux job purge --force --age-limit=1ms >both.out &&
+	grep "purged 6 inactive jobs" both.out
+'
+test_expect_success 'flux job purge --force --num-limit=1 purges 0' '
+	flux job purge --force --num-limit=1 >num1.out &&
+	grep "purged 0 inactive jobs" num1.out
+'
+test_expect_success 'verify job KVS eventlogs do not exist' '
+	for id in $(cat jobids); do \
+	    test_must_fail flux job eventlog $id; \
+	done
+'
+test_expect_success 'create 2 inactive jobs with known completion order' '
+	flux submit true >jobid1 &&
+	flux job wait-event $(cat jobid1) clean &&
+	flux submit true >jobid2 &&
+	flux job wait-event $(cat jobid2) clean
+'
+test_expect_success 'purge the oldest job - youngest is still there' '
+	flux job purge --force --num-limit=1 &&
+	flux job eventlog $(cat jobid2) >/dev/null
+'
+test_expect_success 'purge the last job' '
+	flux job purge --force --num-limit=0 &&
+	wait_inactive_count job-manager 0 30
+'
+#
+# purge w/ job ids tests
+#
+test_expect_success 'create 4 inactive jobs and 1 active job and save their jobids' '
+	flux submit --cc=1-4 true > inactive.ids &&
+	flux queue drain &&
+	flux submit sleep inf > active.id
+'
+test_expect_success 'flux job purge with id says use force' '
+	jobid=`head -n1 inactive.ids` &&
+	flux job purge $jobid >id_no_force.out &&
+	grep "use --force to purge 1" id_no_force.out
+'
+test_expect_success 'flux job purge --force purges 1 job' '
+	jobid=`head -n1 inactive.ids` &&
+	flux job purge --force $jobid >id_force_one.out &&
+	grep "purged 1 inactive jobs" id_force_one.out
+'
+test_expect_success 'flux job purge on invalid jobid fails (no-force)' '
+	jobid=`head -n1 inactive.ids` &&
+	test_must_fail flux job purge $jobid >id_invalid_one.out 2>&1 &&
+	grep "id not found" id_invalid_one.out
+'
+test_expect_success 'flux job purge on active jobid fails (no-force)' '
+	jobid=`cat active.id` &&
+	test_must_fail flux job purge $jobid >id_active_one.out 2>&1 &&
+	grep "cannot purge active job" id_active_one.out
+'
+test_expect_success 'flux job purge on invalid jobid fails (force)' '
+	jobid=`head -n1 inactive.ids` &&
+	test_must_fail flux job purge --force $jobid >id_invalid_one.out 2>&1 &&
+	grep "id not found" id_invalid_one.out
+'
+test_expect_success 'flux job purge on active jobid fails (force)' '
+	jobid=`cat active.id` &&
+	test_must_fail flux job purge --force $jobid >id_active_one.out 2>&1 &&
+	grep "cannot purge active job" id_active_one.out
+'
+test_expect_success 'flux job purge with multiple ids says use force' '
+	jobid1=`head -n2 inactive.ids | tail -n1` &&
+	jobid2=`head -n3 inactive.ids | tail -n1` &&
+	flux job purge $jobid1 $jobid2 >two_ids_no_force.out &&
+	grep "use --force to purge 2" two_ids_no_force.out
+'
+test_expect_success 'flux job purge with multiple ids and --force purges 2 jobs' '
+	jobid1=`head -n2 inactive.ids | tail -n1` &&
+	jobid2=`head -n3 inactive.ids | tail -n1` &&
+	flux job purge --force $jobid1 $jobid2 >id_force_two.out &&
+	grep "purged 2 inactive jobs" id_force_two.out
+'
+test_expect_success 'flux job purge with valid and invalid jobid is ENOENT' '
+	jobid1=`head -n1 inactive.ids` &&
+	jobid2=`tail -n1` &&
+	test_must_fail flux job purge --force $jobid1 $jobid2 >id_invalid_one_of_two.out 2>&1 &&
+	grep "id not found" id_invalid_one_of_two.out
+'
+test_expect_success 'flux job purge the last remaining inactive job' '
+	jobid=`tail -n1` &&
+	test_must_fail flux job purge --force $jobid1 >id_last_one.out 2>&1 &&
+	grep "id not found" id_last_one.out
+'
+test_expect_success 'cleanup running job' '
+	flux cancel $(cat active.id) &&
+	flux job wait-event $(cat active.id) clean
+'
+#
+# do the following "auto purge" tests last, as they could affect earlier tests
+#
+test_expect_success 'create 10 inactive jobs' '
+	flux submit --cc=1-10 true &&
+	flux queue drain
+'
+test_expect_success 'reconfigure job manager with inactive-num-limit=5' '
+	flux config load <<-EOT
+	[job-manager]
+	inactive-num-limit = 5
+	EOT
+'
+test_expect_success 'wait for job-list inactive job count to reach 5' '
+	wait_inactive_count job-list 5 30
+'
+test_expect_success NO_CHAIN_LINT 'run multiple flux-job purges concurrently' '
+	flux job purge --force --num-limit=4 &
+	pid=$! &&
+	flux job purge --force --num-limit=3 &&
+	wait $pid
+'
+test_expect_success NO_CHAIN_LINT 'wait for job-list inactive job count to reach 3' '
+	wait_inactive_count job-list 3 30
+'
+test_expect_success 'reconfigure job manager with inactive-age-limit=1ms' '
+	flux config load <<-EOT
+	[job-manager]
+	inactive-age-limit = "1ms"
+	EOT
+'
+test_expect_success 'wait for job-list inactive job count to reach 0' '
+	wait_inactive_count job-list 0 30
+'
+test_expect_success 'confirm job-list stats show zero inactive jobs' '
+	test $(inactive_count job-list-stats) -eq 0
+'
+test_expect_success 'reconfigure job manager with incorrect type limit' '
+	test_must_fail flux config load 2>badtype.err <<-EOT &&
+	[job-manager]
+	inactive-age-limit = 42
+	EOT
+	grep "Expected string" badtype.err
+'
+test_expect_success 'reconfigure job manager with bad age-limit fsd' '
+	test_must_fail flux config load 2>badfsd.err <<-EOT &&
+	[job-manager]
+	inactive-age-limit = "notfsd"
+	EOT
+	grep "invalid FSD" badfsd.err
+'
+test_expect_success 'reconfigure job manager with invalid num-limit' '
+	test_must_fail flux config load 2>badnum.err <<-EOT &&
+	[job-manager]
+	inactive-num-limit = -42
+	EOT
+	grep "must be >= 0" badnum.err
+'
+test_expect_success 'new instance with bad config fails to start' '
+	mkdir -p config &&
+	cat >config/system.toml <<-EOT &&
+	[job-manager]
+	inactive-num-limit = -42
+	EOT
+	test_must_fail flux start --config-path=$(pwd)/config \
+		true 2>badnum2.err &&
+	grep "must be >= 0" badnum2.err
+'
+
+test_done
diff --git a/t/t2810-kvs-garbage-collect.t b/t/t2810-kvs-garbage-collect.t
new file mode 100755
index 000000000000..d34eeb3f59d9
--- /dev/null
+++ b/t/t2810-kvs-garbage-collect.t
@@ -0,0 +1,92 @@
+#!/bin/sh
+
+test_description='Test offline KVS garbage collection'
+
+. $(dirname $0)/sharness.sh
+
+test_expect_success 'create test script' '
+	cat >runjobs.sh <<-EOT &&
+	#!/bin/bash -e
+	trap "" SIGHUP
+	flux submit --cc=1-10 true >/dev/null
+	flux queue drain
+	backingmod=\$(flux getattr content.backing-module)
+	flux module stats --type int --parse object_count \$backingmod
+	EOT
+	chmod +x runjobs.sh
+'
+test_expect_success 'run instance that leaves an auto dump' '
+	mkdir -p state &&
+	flux start -Sstatedir=state \
+	    -Scontent.dump=auto \
+	    -Slog-filename=dmesg.log \
+	    ./runjobs.sh >object_count
+'
+test_expect_success 'broker logs report dump activity' '
+	grep "dumping content to" dmesg.log
+'
+test_expect_success 'dump exists and RESTORE symlink is valid' '
+	test -h state/dump/RESTORE &&
+	readlink -f state/dump/RESTORE >archive &&
+	test -f $(cat archive)
+'
+test_expect_success 'restart instance with auto restore' '
+	flux start -Sstatedir=state \
+	    -Scontent.restore=auto \
+	    -Slog-filename=dmesg2.log \
+	    flux module stats \
+	        --type int --parse object_count content-sqlite >object_count2
+'
+test_expect_success 'broker logs report restore activity' '
+	grep "restoring content from" dmesg2.log
+'
+test_expect_success 'number of stored objects was reduced by GC' '
+	before=$(cat object_count) &&
+	after=$(cat object_count2) &&
+	test $before -gt $after
+'
+test_expect_success 'RESTORE symlink is gone' '
+	test_must_fail test -h state/dump/RESTORE
+'
+test_expect_success 'archive file remains' '
+	test -f $(cat archive)
+'
+
+#
+# Now repeat the above test with
+# - content-files backend
+# - no statedir
+# - explicitly named dump file (not auto)
+#
+test_expect_success 'run instance that leaves a named dump' '
+	flux start -Slog-filename=dmesg3.log \
+	    -Scontent.dump=foo.tgz \
+	    -Scontent.backing-module=content-files \
+	    ./runjobs.sh >object_count3
+'
+test_expect_success 'broker logs report dump activity' '
+	grep "dumping content to" dmesg3.log
+'
+test_expect_success 'dump exists in current directory' '
+	test -f foo.tgz
+'
+test_expect_success 'no RESTORE link was created because path is explicit' '
+	test_must_fail test -h dump/RESTORE
+'
+test_expect_success 'restart instance and restore' '
+	flux start -Slog-filename=dmesg4.log \
+	    -Scontent.restore=foo.tgz \
+	    -Scontent.backing-module=content-files \
+	    flux module stats \
+	        --type int --parse object_count content-files >object_count4
+'
+test_expect_success 'broker logs report restore activity' '
+	grep "restoring content from" dmesg4.log
+'
+test_expect_success 'number of stored objects was reduced by GC' '
+	before=$(cat object_count3) &&
+	after=$(cat object_count4) &&
+	test $before -gt $after
+'
+
+test_done
diff --git a/t/t2811-flux-pgrep.t b/t/t2811-flux-pgrep.t
new file mode 100755
index 000000000000..c11f2b95fe1b
--- /dev/null
+++ b/t/t2811-flux-pgrep.t
@@ -0,0 +1,113 @@
+#!/bin/sh
+
+test_description='Test the flux-pgrep/pkill commands'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1 job
+
+export FLUX_PYCLI_LOGLEVEL=10
+export SHELL=/bin/sh
+
+test_expect_success 'flux-pgrep errors on invalid arguments' '
+	test_must_fail flux pgrep &&
+	test_expect_code 2 flux pgrep foo bar &&
+	test_expect_code 2 flux pgrep \[
+'
+test_expect_success 'flux-pgrep returns 1 when no jobs match' '
+	test_expect_code 1 flux pgrep foo
+'
+test_expect_success 'flux-pgrep works for job nanes' '
+	flux bulksubmit --job-name={} sleep 300 \
+		::: testA testB foo &&
+	flux pgrep ^test > pgrep.out &&
+	test $(wc -l <pgrep.out) -eq 2
+'
+test_expect_success 'flux-pgrep --format option works' '
+	flux pgrep -o full ^foo >pgrep-full.out &&
+	test_debug "cat pgrep-full.out" &&
+	grep NAME pgrep-full.out
+'
+test_expect_success 'flux-pgrep --format option detects errors' '
+	test_must_fail flux pgrep -o {foo} ^foo
+'
+test_expect_success 'flux-pgrep --filter works' '
+	flux pgrep --filter=sched,run . >pgrep-filtered.out &&
+	test_debug "cat pgrep-filtered.out" &&
+	test $(wc -l <pgrep-filtered.out) -eq 3
+'
+test_expect_success 'flux-pgrep default override works' '
+	FLUX_PGREP_FORMAT_DEFAULT="{id} {id}" flux pgrep -a . >default_override.out &&
+	test_debug "cat default_override.out" &&
+	grep "JOBID JOBID" default_override.out
+'
+test_expect_success 'flux-pgrep default override w/ named format works' '
+	FLUX_PGREP_FORMAT_DEFAULT=full flux pgrep -a . >default_override_named.out &&
+	test_debug "cat default_override_named.out" &&
+	grep "TIME" default_override_named.out &&
+	grep "INFO" default_override_named.out
+'
+test_expect_success 'flux-pkill works' '
+	flux pkill ^test &&
+	test_expect_code 1 flux pkill ^test &&
+	flux pkill foo
+'
+test_expect_success 'flux-pgrep works with jobid ranges' '
+	flux submit --bcc=1-6 sleep 60 >ids &&
+	flux submit --bcc=1-6 sleep 60 >ids2 &&
+	flux pgrep $(head -n 1 ids)..$(tail -n1 ids) >pgrep.ids &&
+	test $(wc -l <pgrep.ids) -eq 6 &&
+	flux pgrep $(head -n 1 ids)..$(tail -n1 ids2) >pgrep2.ids &&
+	test $(wc -l <pgrep2.ids) -eq 12 &&
+	flux pkill $(head -n1 ids)..$(tail -n1 ids) &&
+	flux pgrep $(head -n 1 ids)..$(tail -n1 ids2) >pgrep3.ids &&
+	test $(wc -l <pgrep3.ids) -eq 6 &&
+	flux pgrep -a $(head -n 1 ids)..$(tail -n1 ids2) >pgrep4.ids &&
+	test $(wc -l <pgrep4.ids) -eq 12
+'
+test_expect_success 'flux-pgrep assumes regex with invalid jobid(s)' '
+	flux pgrep sl..p
+'
+test_expect_success 'flux-pkill --wait option works' '
+	flux pkill --wait sleep &&
+	test_expect_code 1 flux pgrep .
+'
+test_expect_success 'flux-pkill reports errors' '
+	flux submit sleep 60 &&
+	test_expect_code 1 flux pkill -ac 2 sleep >pkill.out 2>&1 &&
+	test_debug "cat pkill.out" &&
+	grep ERROR.*inactive pkill.out
+'
+test_expect_success 'flux-pgrep accepts jobid range and pattern' '
+	flux bulksubmit --job-name={} sleep 60 ::: foo bar foo >ids3 &&
+	id1=$(sed -n 1p ids3) &&
+	id2=$(sed -n 2p ids3) &&
+	id3=$(sed -n 3p ids3) &&
+	flux pgrep -no full ${id1}..${id2} foo >pgrep.out3 &&
+	test_debug "cat pgrep.out3" &&
+	test $(wc -l < pgrep.out3) -eq 1 &&
+	grep foo pgrep.out3 &&
+	flux pkill --wait $id1..$id3
+'
+test_expect_success 'flux-pgrep forces regex with name: prefix' '
+	flux submit --job-name=f123f234 sleep 60 &&
+	flux pgrep name:f1..f2 &&
+	flux pkill --wait ^f123f234
+'
+test_expect_success 'flux-pgrep reads flux-jobs formats configuration' '
+	mkdir -p dir/flux &&
+	cat <<-EOF >dir/flux/flux-jobs.toml &&
+	[formats.myformat]
+	description = "my format"
+	format = "{id.words}"
+	EOF
+	XDG_CONFIG_HOME="$(pwd)/dir" \
+		flux pgrep --format=help foo >myformat.out&&
+	test_debug "cat myformat.out" &&
+	grep "my format" myformat.out  &&
+	flux submit --job-name=test sleep 30 &&
+	XDG_CONFIG_HOME="$(pwd)/dir" \
+		flux pgrep -o myformat test &&
+	flux pkill test
+'
+test_done
diff --git a/t/t2812-flux-job-last.t b/t/t2812-flux-job-last.t
new file mode 100755
index 000000000000..dfb19363ad25
--- /dev/null
+++ b/t/t2812-flux-job-last.t
@@ -0,0 +1,77 @@
+#!/bin/sh
+
+test_description='Test the flux job last command'
+
+. $(dirname $0)/sharness.sh
+
+flux version | grep -q libflux-security && test_set_prereq FLUX_SECURITY
+
+test_under_flux 1
+
+test_expect_success 'flux-job last fails on invalid arguments' '
+	test_must_fail flux job last --badarg 2>badarg.err &&
+	grep "unrecognized option" badarg.err
+'
+test_expect_success 'flux-job last fails when no jobs have been submitted' '
+	test_must_fail flux job last 2>nojob.err &&
+	grep "job history is empty" nojob.err
+
+'
+test_expect_success 'submit some jobs' '
+	flux submit --cc=0-9 true >jobids
+'
+test_expect_success 'flux job last lists the most recently submitted job' '
+	id=$(flux job last) &&
+	test "$id" = "$(tail -1 jobids)"
+'
+test_expect_success 'flux job last [::-1] lists jobs in submit order' '
+	flux job last "[::-1]" >last-reverse-all.out &&
+	test_cmp jobids last-reverse-all.out
+'
+test_expect_success 'flux job last [:] lists all the jobs' '
+	flux job last "[:]" >last.out &&
+	test $(wc -l <last.out) -eq $(wc -l <jobids)
+'
+test_expect_success 'flux job last N lists the last N jobs' '
+	head -4 last.out >last4.exp &&
+	flux job last 4 >last4.out &&
+	test_cmp last4.exp last4.out
+'
+# issue #4930
+test_expect_success 'flux-job last lists inactive jobs after instance restart' '
+	flux job last "[:]" >lastdump.exp &&
+	flux queue idle &&
+	flux dump dump.tgz &&
+	flux start -Scontent.restore=dump.tgz \
+		flux job last "[:]" >lastdump.out &&
+	test_cmp lastdump.exp lastdump.out
+'
+# issue #4931
+test_expect_success 'flux-job last does not list purged jobs' '
+	flux job purge --force --num-limit=0 &&
+	test_must_fail flux job last 2>nojob2.err &&
+	grep "job history is empty" nojob2.err
+'
+
+submit_as_root()
+{
+        FAKE_USERID=0
+        test_debug "echo running flux run $@ as userid $FAKE_USERID"
+        flux run --dry-run "$@" | \
+          flux python ${SHARNESS_TEST_SRCDIR}/scripts/sign-as.py $FAKE_USERID \
+            >job.signed
+        FLUX_HANDLE_USERID=$FAKE_USERID \
+          flux job submit --flags=signed job.signed
+}
+
+# issue #5475
+# Execution may fail but submission should work - enough for this test
+test_expect_success FLUX_SECURITY 'reload job-ingest with allow-root-jobs' '
+	flux module reload job-ingest allow-root-jobs
+'
+test_expect_success FLUX_SECURITY 'run a job as fake root' '
+	submit_as_root true &&
+	FLUX_HANDLE_USERID=0 flux job last
+'
+
+test_done
diff --git a/t/t2813-flux-watch.t b/t/t2813-flux-watch.t
new file mode 100755
index 000000000000..0828c50902c0
--- /dev/null
+++ b/t/t2813-flux-watch.t
@@ -0,0 +1,140 @@
+#!/bin/sh
+
+test_description='Test the flux-watch command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 2 job
+
+export FLUX_PYCLI_LOGLEVEL=10
+export SHELL=/bin/sh
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py -f asciicast --line-buffer"
+
+test_expect_success 'flux-watch: does nothing with no args' '
+	test_expect_code 1 flux watch >no-args.log 2>&1 &&
+	test_debug "cat no-args.log" &&
+	grep "one job selection option is required" no-args.log
+'
+test_expect_success 'flux-watch: start some active jobs to watch' '
+	cat <<-EOF >test.sh &&
+	#!/bin/sh
+	echo \$TEST_OUTPUT
+	EOF
+	chmod +x test.sh &&
+	flux submit --urgency=hold --cc=1-2 --env=TEST_OUTPUT=test{cc} \
+	    ./test.sh >active.ids
+'
+test_expect_success NO_CHAIN_LINT 'flux-watch: --active watches all active jobs' '
+	flux watch -vv --active >active.log 2>&1 &
+	$waitfile -t 30 -v -p "Watching 2 jobs" active.log &&
+	for id in $(cat active.ids); do
+	    flux job urgency $id default
+	done &&
+	wait &&
+	test_debug "cat active.log" &&
+	grep test1 active.log &&
+	grep test2 active.log
+'
+test_expect_success 'flux-watch: ensure previous jobs are inactive (in case of chain-lint)' '
+	for id in $(cat active.ids); do
+            flux job urgency $id default || true
+        done &&
+	flux watch $(cat active.ids)
+'
+test_expect_success 'flux-watch: --all watches inactive jobs' '
+	flux watch --all  >all.log 2>&1 &&
+	test_debug "cat all.log" &&
+	grep test1 all.log &&
+	grep test2 all.log
+'
+test_expect_success 'flux-watch: run some failed jobs' '
+	id=$(flux submit sh -c "echo test3; exit 2") &&
+	id2=$(flux submit --urgency hold true) &&
+	flux cancel $id2
+'
+test_expect_success 'flux-watch: exits with highest job exit code' '
+	test_expect_code 2 flux watch --all
+'
+test_expect_success 'flux-watch: can specify multiple jobids' '
+	flux watch $(cat active.ids) >by-jobid.out 2>&1 &&
+	test_debug "cat by-jobid.out" &&
+	grep test1 by-jobid.out &&
+	grep test2 by-jobid.out
+'
+test_expect_success 'flux-watch: emits errors for invalid jobs' '
+	flux watch f123 2>invalid-jobid.err &&
+	grep "JobID f123 unknown" invalid-jobid.err
+'
+test_expect_success 'flux-watch: issues warning with jobids and filtering option' '
+	flux watch -v --all $(cat active.ids) >all-error.out 2>&1 &&
+	test_debug "cat all-error.out" &&
+	grep "Filtering options ignored with jobid list" all-error.out
+'
+# Note: test with --since by using the t_submit of the last submitted job.
+# This should only match up to the last two jobs, which were failed and
+# canceled respectively, so the test must fail and the tool should output
+# it watched only 1 or 2 jobs (we can't guarantee when the first failed
+# job finished relative to submission of the second)
+test_expect_success 'flux-watch: works with --since' '
+	t_since=$(flux jobs -ac 1 -no {t_submit}) &&
+	test_must_fail flux watch -v --since=$t_since >since.out 2>&1 &&
+	test_debug "cat since.out" &&
+	grep "Watching [12] job" since.out
+'
+# -1h should get us all 4 jobs
+test_expect_success 'flux-watch: --since takes negative offset' '
+	test_must_fail flux watch -v --since=-1h >since1h.out 2>&1 &&
+	test_debug "cat since1h.out" &&
+	grep "Watching 4 job" since1h.out
+'
+test_expect_success 'flux-watch: invalid --since fails' '
+	test_expect_code 2 flux watch --since=-1g 2>since.err &&
+	grep "invalid value" since.err
+'
+test_expect_success 'flux-watch: --since in future fails' '
+	test_expect_code 2 flux watch --since=+1d 2>since2.err &&
+	grep "appears to be in the future" since2.err
+'
+test_expect_success 'flux-watch: --count works' '
+	test_might_fail flux watch -v --all --count=1 >count.out 2>&1 &&
+	grep "Watching 1 job" count.out
+'
+test_expect_success 'flux-watch: --progress works' '
+	test_might_fail $runpty flux watch --all --progress >progress.out &&
+	test_debug "cat progress.out" &&
+	grep "PD:0 *R:0 *CD:2 *F:2 .*100.0%" progress.out
+'
+test_expect_success 'flux-watch: --progress --jps works' '
+	test_might_fail $runpty flux watch --all --progress --jps >jps.out &&
+	test_debug "cat jps.out" &&
+	grep "PD:0 *R:0 *CD:2 *F:2 .*job/s" jps.out
+'
+
+test_expect_success 'flux-watch: --progress is ignored with no tty' '
+	test_might_fail flux watch --all --progress >progress-nopty.out 2>&1 &&
+	test_debug "cat progress-nopty.out" &&
+	grep "Ignoring --progress" progress-nopty.out
+'
+test_expect_success 'flux-watch: --label-io works' '
+	test_might_fail flux watch --all --label-io >labelio.out 2>&1 &&
+	test_debug "cat labelio.out" &&
+	grep "0: test3" labelio.out &&
+	grep "0: test2" labelio.out &&
+	grep "0: test1" labelio.out
+'
+test_expect_success 'flux-watch: --filter works' '
+	nfailed=$(flux jobs -no {id} -f failed,canceled | wc -l) &&
+	test_expect_code 2 flux watch -v --filter=failed,canceled \
+	    >failed.out 2>&1 &&
+	test_debug "cat failed.out" &&
+	grep "Watching ${nfailed} job" failed.out
+'
+test_expect_success 'flux-watch: handles binary data' '
+	id=$(flux submit dd if=/dev/urandom count=1) &&
+	test_debug "flux job eventlog -p guest.output -HL $id" &&
+	flux job attach $id >binary.expected &&
+	flux watch $id >binary.output &&
+	test_cmp binary.expected binary.output
+'
+test_done
diff --git a/t/t2814-hostlist-cmd.t b/t/t2814-hostlist-cmd.t
new file mode 100755
index 000000000000..6a710830a279
--- /dev/null
+++ b/t/t2814-hostlist-cmd.t
@@ -0,0 +1,184 @@
+#!/bin/sh
+
+test_description='Test flux hostlist command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 2
+
+test_expect_success 'flux-hostlist --help works' '
+	flux hostlist --help >help.out &&
+	test_debug "cat help.out" &&
+	grep "SOURCES may include" help.out
+'
+test_expect_success 'flux-hostlist returns hostlist attr in initial program' '
+	flux hostlist -l >hl-instance.out &&
+	flux getattr hostlist >hl-instance.expected &&
+	test_cmp hl-instance.expected hl-instance.out
+'
+test_expect_success 'flux-hostlist reads from stdin by default' '
+	echo foo[0-3] | flux hostlist
+'
+# Need to simulate open stdin with a fifo since sharness closes stdin
+test_expect_success NO_CHAIN_LINT 'flux-hostlist includes a timeout on reading stdin' '
+	rm -f input.fifo;
+	mkfifo input.fifo;
+	FLUX_HOSTLIST_STDIN_TIMEOUT=0.1 flux hostlist <input.fifo &
+	pid=$! &&
+	test_when_finished rm -f input.fifo &&
+	exec 9>input.fifo &&
+	test_must_fail wait $pid &&
+	exec 9>&-
+'
+test_expect_success 'flux-hostlist returns job hostlist in job' '
+	flux run flux hostlist -l >hl-job.out &&
+	flux jobs -no {nodelist} $(flux job last) > hl-job.expected &&
+	test_cmp hl-job.expected hl-job.out
+'
+test_expect_success 'flux-hostlist works with a jobid' '
+	flux hostlist $(flux job last) >hl-jobid.out &&
+	test_cmp hl-job.expected hl-jobid.out
+'
+test_expect_success 'flux-hostlist fails with invalid jobid' '
+	test_must_fail flux hostlist foo1
+'
+test_expect_success 'flux-hostlist --fallback treats invalid jobid as host' '
+	flux hostlist --fallback foo1 >fallback.out &&
+	test "$(cat fallback.out)" = "foo1"
+'
+test_expect_success 'flux-hostlist -c works' '
+	flux hostlist --count local  &&
+	test $(flux hostlist --count local) -eq 2 &&
+	test $(flux hostlist -c "foo[1-10]") -eq 10
+'
+test_expect_success 'flux-hostlist works with "avail"' '
+	flux resource drain 0 &&
+	test $(flux hostlist --count avail) -eq 1 &&
+	flux resource undrain 0
+'
+test_expect_success 'flux-hostlist works with stdin' '
+	printf "foo1 foo2 foo3"   | flux hostlist >hl-stdin1.out &&
+	printf "foo1\nfoo2\nfoo3" | flux hostlist >hl-stdin2.out &&
+	test_debug "grep . hl-stdin*.out" &&
+	printf "foo[1-3]\n" >hl-stdin.expected &&
+	test_cmp hl-stdin.expected hl-stdin1.out &&
+	test_cmp hl-stdin.expected hl-stdin2.out
+'
+test_expect_success 'flux-hostlist works with hostlist args' '
+	flux hostlist "foo[1-3]" > hl-args1.out &&
+	flux hostlist --fallback foo1 foo2 foo3 > hl-args2.out &&
+	printf "foo[1-3]\n" >hl-args.expected &&
+	test_cmp hl-args.expected hl-args1.out &&
+	test_cmp hl-args.expected hl-args2.out
+'
+test_expect_success 'flux-hostlist rejects invalid hostlist' '
+	test_must_fail flux hostlist "foo[1-"
+'
+test_expect_success 'flux-hostlist returns empty hostlist for pending job' '
+	id=$(flux submit --urgency=hold hostname) &&
+	flux hostlist $id >hl-pending.out &&
+	test_must_be_empty hl-pending.out
+'
+# Note job "$id" should still be held when this next test is run:
+test_expect_success 'flux-hostlist --quiet works' '
+	flux hostlist --quiet foo[1-10] > quiet.out &&
+	test_must_be_empty quiet.out &&
+	test_must_fail flux hostlist -q $id
+'
+test_expect_success 'flux-hostlist -e, --expand works' '
+	test_debug "flux hostlist -e foo[1-3]" &&
+	test "$(flux hostlist -e foo[1-3])" = "foo1 foo2 foo3" &&
+	flux hostlist --expand --delimiter="\n" "foo[1-3]" >expand.out &&
+	cat <<-EOF >expand.expected &&
+	foo1
+	foo2
+	foo3
+	EOF
+	test_cmp expand.expected expand.out
+'
+test_expect_success 'flux-hostlist -n, --nth works' '
+	test "$(flux hostlist --nth=1 foo[1-10])" = foo2 &&
+	test "$(flux hostlist --nth=-1 foo[1-10])" = foo10
+'
+test_expect_success 'flux-hostlist -n, --nth works with an idset' '
+	flux hostlist --nth=1,3-4 foo[1-10] &&
+	test "$(flux hostlist --nth=1,3-4 foo[1-10])" = "foo[2,4-5]" &&
+	flux hostlist --nth=-1,3-4 foo[1-10] &&
+	test "$(flux hostlist --nth=-1,3-4 foo[1-10])" = "foo[7-8,10]"
+'
+test_expect_success 'flux-hostlist -n, --nth works with --expand' '
+	test "$(flux hostlist -e --nth=1,3-4 foo[1-10])" = "foo2 foo4 foo5"
+'
+test_expect_success 'flux-hostlist -n errors with invalid index' '
+	test_must_fail flux hostlist -n 10 foo[1-10] &&
+	test_must_fail flux hostlist -n 1,10 foo[1-10]
+'
+test_expect_success 'flux-hostlist -x, --exclude works' '
+	test "$(flux hostlist -x foo1 foo[0-10])" = "foo[0,2-10]" &&
+	test "$(flux hostlist -x foo[0-9] foo[0-10])" = "foo10"
+'
+test_expect_success 'flux-hostlist -x, --exclude works with indices' '
+	test "$(flux hostlist -x 1 foo[0-10])" = "foo[0,2-10]" &&
+	test "$(flux hostlist -x 0-9 foo[0-10])" = "foo10"
+'
+test_expect_success 'flux-hostlist -n works after -x' '
+	test "$(flux hostlist -x foo5 -n 5-6 foo[1-10])" = "foo[7-8]"
+'
+test_expect_success 'flux-hostlist -L, --limit works' '
+	test "$(flux hostlist -L 2 foo[1-10])" = "foo[1-2]" &&
+	test "$(flux hostlist -L -2 foo[1-10])" = "foo[9-10]"
+'
+test_expect_success 'flux-hostlist preserves hostlist order by default' '
+	test "$(flux hostlist host1 host0 host5 host4)" = "host[1,0,5,4]"
+'
+test_expect_success 'flux-hostlist preserves repeated hosts by default' '
+	test "$(flux hostlist host1 host1 host1)" = "host[1,1,1]"
+'
+test_expect_success 'flux-hostlist -S, --sort works' '
+	test "$(flux hostlist -S host3 host2 host1 host0)" = "host[0-3]"
+'
+test_expect_success 'flux-hostlist -S preserves duplicate hosts' '
+	test "$(flux hostlist -S host2 host1 host3 host1)" = "host[1,1-3]"
+'
+test_expect_success 'flux-hostlist -u  returns union of all hosts' '
+	test "$(flux hostlist -u host2 host1 host3 host1)" = "host[1-3]" &&
+	test "$(flux hostlist -u host[1-3] host2)" = "host[1-3]" &&
+	test "$(flux hostlist -u host[1-3] bar baz)" = "bar,baz,host[1-3]"
+'
+test_expect_success 'flux-hostlist -i, --intersect works' '
+	test "$(flux hostlist -i host[1-3] host[2-5])" = "host[2-3]" &&
+	test "$(flux hostlist -i host[1-3] host[2-5] host[3-10])" = "host3"
+'
+# Note symmetric difference of 3 sets is all elements that are in just one
+# set or all 3 sets due to associative property.
+test_expect_success 'flux-hostlist -X, --xor works' '
+	flux hostlist -X host[1-3] host[2-5] &&
+	test "$(flux hostlist -X host[1-3] host[2-5])" = "host[1,4-5]" &&
+	flux hostlist -X host[1-3] host[2-5] host[3-10] &&
+	test "$(flux hostlist -X host[1-3] host[2-5] host[3-10])" = "host[1,3,6-10]" &&
+	flux hostlist -X host[3-10] host[1-3] host[2-5] &&
+	test "$(flux hostlist -X host[3-10] host[1-3] host[2-5])" = "host[1,3,6-10]"
+'
+test_expect_success 'flux-hostlist -m, --minus works' '
+	flux hostlist -m host[1-10] host[2-3] &&
+	test "$(flux hostlist -m host[1-10] host[2-3])" = "host[1,4-10]" &&
+	flux hostlist -m host[1-10] host[2-3] host[5-10] &&
+	test "$(flux hostlist -m host[1-10] host[2-3] host[5-10])" = \
+             "host[1,4]" &&
+	test "$(flux hostlist -m host[1-10] host[1-20])" = "" &&
+	test_expect_code 1 flux hostlist -qm host[1-10] host[1-20]
+'
+test_expect_success 'flux-hostlist -m, --minus removes only first occurrence' '
+	flux hostlist -m host[1,1,1] host1 &&
+	test "$(flux hostlist -m host[1,1,1] host1)" = "host[1,1]"
+'
+test_expect_success 'flux-hostlist: -u can be used with -m' '
+	flux hostlist -m host[1,1,1] host1 &&
+	test "$(flux hostlist -um host[1,1,1] host1)" = "host1"
+'
+test_expect_success 'flux-hostlist -q, --quiet works' '
+	flux hostlist --quiet host[1-10] >quiet.out &&
+	test_must_be_empty quiet.out &&
+	test_must_fail flux hostlist --quiet --intersect host1 host2
+'
+test_done
diff --git a/t/t2815-post-job-event.t b/t/t2815-post-job-event.t
new file mode 100755
index 000000000000..0311bfb9c26e
--- /dev/null
+++ b/t/t2815-post-job-event.t
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+test_description='Test flux post-job-event command'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+test_expect_success 'run a test job' '
+	JOBID=$(flux submit --wait-event=start sleep inf)
+'
+test_expect_success 'flux-post-job-event --help works' '
+	flux post-job-event --help >help.out &&
+	test_debug "cat help.out" &&
+	grep "name of event to post" help.out
+'
+test_expect_success 'flux-post-job-event can post a simple event' '
+	flux post-job-event $JOBID test &&
+	flux job wait-event -t15 $JOBID test
+'
+test_expect_success 'flux-post-job-event can post an event with context' '
+	flux post-job-event $JOBID test note=testing  &&
+	flux job wait-event -m note=testing -t15 $JOBID test 
+'
+test_expect_success 'flux-post-job-event can post multiple context keys' '
+	flux post-job-event $JOBID test note=test2 status=0  &&
+	flux job wait-event -m note=test2 -t15 $JOBID test  &&
+	flux job wait-event -m status=0 -t15 $JOBID test 
+'
+test_expect_success 'flux-post-job-event fails for invalid job' '
+	test_must_fail flux post-job-event f123 test note="test event"
+'
+test_expect_success 'flux-post-job-event fails for inactive job' '
+	flux cancel $JOBID &&
+	flux job wait-event $JOBID clean &&
+	test_must_fail flux post-job-event $JOBID test note="test event"
+'
+test_expect_success 'flux-post-job-event fails with invalid id' '
+	test_must_fail flux post-job-event baz test
+'
+test_done
diff --git a/t/t2900-job-timelimits.t b/t/t2900-job-timelimits.t
new file mode 100755
index 000000000000..9ef6a4a2e9b6
--- /dev/null
+++ b/t/t2900-job-timelimits.t
@@ -0,0 +1,96 @@
+#!/bin/sh
+
+test_description='Test job time limit functionality'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+# Set CLIMain log level to logging.DEBUG (10), to enable stack traces
+export FLUX_PYCLI_LOGLEVEL=10
+
+# Default timeout, extend under valgrind or ASAN
+TIMEOUT=0.25
+test -n "$FLUX_TEST_VALGRIND" && TIMEOUT=3
+test_have_prereq ASAN && TIMEOUT=3
+
+test_expect_success 'job time limits are enforced' '
+	test_expect_code 142 \
+		flux run --time-limit=${TIMEOUT}s sleep 30 2>limit1.err &&
+	grep "resource allocation expired" limit1.err
+'
+test_expect_success 'timeout exception was generated by instance owner' '
+	jobid=$(flux job last) &&
+	flux job eventlog $jobid |grep exception >exception.out &&
+	grep -q type=\"timeout\" exception.out &&
+	grep -q userid=$(id -u) exception.out
+'
+test_expect_success 'job timelimits are propagated' '
+	cat <<-EOF >limit.sh &&
+	#!/bin/sh -e
+	round() { printf "%.0f" \$1; }
+
+	expiration=\$(flux kvs get resource.R | jq .execution.expiration)
+	echo "expiration is \$expiration"
+
+	id1=\$(flux submit --wait-event=start sleep 300)
+	flux jobs -no "{id.f58} {expiration}"
+	exp1=\$(flux jobs -no {expiration} \$id1)
+	test "\$(round \$exp1)" = "\$(round \${expiration})"
+	flux cancel --all
+
+	id2=\$(flux submit --wait-event=start -t 1m sleep 300)
+	flux jobs -no "{id.f58} {expiration}"
+	exp2=\$(flux jobs -no {expiration} \$id2)
+	test \$(round \${exp2}) -lt \$(round \${expiration})
+        flux cancel --all
+	EOF
+	chmod +x limit.sh &&
+	flux run --time-limit=10m flux start ./limit.sh
+'
+test_expect_success 'job may exit before time limit' '
+        flux run --time-limit=5m sleep 0.25
+'
+test_expect_success 'result for expired jobs is TIMEOUT' '
+	test_debug "flux jobs -a" &&
+	flux jobs -ano {result} | grep -q TIMEOUT
+'
+sigalrm_sigkill_test() {
+	scale=$1
+	timeout=$2
+	kill_timeout=$3
+	test_debug "echo testing with timeout=$timeout kill_timeout=$kill_timeout" &&
+	flux module reload job-exec kill-timeout=$kill_timeout &&
+	ofile=trap.$scale.out &&
+	test_must_fail_or_be_terminated \
+           flux run -vvv --time-limit=${timeout}s bash -xc \
+               "trap \"echo got SIGALRM>>$ofile\" SIGALRM;sleep 10;sleep 15" \
+                   > $ofile 2> trap.$scale.err &&
+        test_debug "grep . trap.*" &&
+        grep "resource allocation expired" trap.$scale.err &&
+        grep "got SIGALRM" $ofile
+}
+test_expect_success 'expired jobs are sent SIGALRM, then SIGKILL' '
+	kill_timeout=$(test -z "$FLUX_TEST_VALGRIND" && echo 0.25 || echo 3) &&
+        for scale in 1 2 4 8; do
+	    sigalrm_sigkill_test \
+	      $scale \
+	      $(perl -E "say $TIMEOUT*$scale") \
+	      $(perl -E "say $kill_timeout*$scale") && break
+	done
+'
+expired_cancel_test() {
+	id=$(flux submit --time-limit=${1}s bash -c \
+            "trap \"echo got SIGALRM>>trap2.out\" SIGALRM;sleep 60;sleep 60" ) &&
+	flux job wait-event --timeout=30 $id exception &&
+	flux cancel $id &&
+	test_expect_code 143 run_timeout 30 flux job attach $id
+
+}
+test_expect_success 'expired job can also be canceled' '
+	flux module reload job-exec kill-timeout=120 &&
+	for scale in 1 4 8 16; do
+	    expired_cancel_test $(perl -E "say $TIMEOUT*$scale") && break
+	done
+'
+test_done
diff --git a/t/t3000-mpi-basic.t b/t/t3000-mpi-basic.t
index 7b4325d71e4f..f81983e4bb40 100755
--- a/t/t3000-mpi-basic.t
+++ b/t/t3000-mpi-basic.t
@@ -10,35 +10,73 @@ if test -z "$FLUX_TEST_MPI"; then
     test_done
 fi
 
-if ! test -x ${FLUX_BUILD_DIR}/t/mpi/hello; then
+HELLO=${FLUX_BUILD_DIR}/t/mpi/hello
+if ! test -x ${HELLO}; then
     skip_all='skipping MPI tests, MPI not available/configured'
     test_done
 fi
 
-# Size the session to one more than the number of cores, minimum of 4
-SIZE=$(test_size_large)
-test_under_flux ${SIZE}
-echo "# $0: flux session size will be ${SIZE}"
+# work around https://github.com/open-mpi/ompi/issues/7701 and similar in mpich
+export HWLOC_COMPONENTS=-gl
+export TEST_UNDER_FLUX_CORES_PER_RANK=4
+SIZE=2
+MAX_MPI_SIZE=$(($SIZE*$TEST_UNDER_FLUX_CORES_PER_RANK))
+test_under_flux $SIZE job
 
-# Usage: run_program timeout ntasks nnodes
-run_program() {
-	local timeout=$1
-	local ntasks=$2
-	local nnodes=$3
-        local opts=$4
-	shift 3
-	run_timeout $timeout flux mini run \
-		    -n${ntasks} -N${nnodes} $*
-}
+OPTS="-ocpu-affinity=off -overbose=1"
 
-test_expect_success "mpi hello singleton" '
-	run_program 5 1 1 ${FLUX_BUILD_DIR}/t/mpi/hello >single.$OPTS
+test_expect_success 'show MPI version under test' '
+	${FLUX_BUILD_DIR}/t/mpi/version
 '
 
-test_expect_success "mpi hello all ranks" '
-	run_program 5 ${SIZE} ${SIZE} ${FLUX_BUILD_DIR}/t/mpi/hello \
-		| tee allranks.$OPTS \
-		&& grep -q "There are ${SIZE} tasks" allranks.$OPTS
+test_expect_success 'mpi hello various sizes' '
+	run_timeout 30 flux submit --cc=1-$MAX_MPI_SIZE $OPTS \
+		--watch -n{cc} ${HELLO} >hello.out &&
+	for i in $(seq 1 $MAX_MPI_SIZE); do \
+		grep "There are $i tasks" hello.out; \
+	done
 '
 
+# issue 3649 - try to get two size=2 jobs running concurrently on one broker
+test_expect_success 'mpi hello size=2 concurrent submit of 8 jobs' '
+	run_timeout 30 flux submit --cc=1-8 $OPTS \
+		--watch -n2 ${HELLO} >hello2.out &&
+	test $(grep "There are 2 tasks" hello2.out | wc -l) -eq 8
+'
+
+# Run some basic tests pulled from MPICH as a sanity check
+test_expect_success 'ANL self works' '
+	run_timeout 30 flux run $OPTS \
+		${FLUX_BUILD_DIR}/t/mpi/mpich_basic/self
+'
+test_expect_success 'ANL simple works' '
+	run_timeout 30 flux run -n 2 $OPTS \
+		${FLUX_BUILD_DIR}/t/mpi/mpich_basic/simple
+'
+test_expect_success 'ANL sendrecv works on 1 node' '
+	run_timeout 30 flux run -n 2 -N1 $OPTS \
+		${FLUX_BUILD_DIR}/t/mpi/mpich_basic/sendrecv
+'
+test_expect_success 'ANL sendrecv works on 2 nodes' '
+	run_timeout 30 flux run -n 2 -N2 $OPTS \
+		${FLUX_BUILD_DIR}/t/mpi/mpich_basic/sendrecv
+'
+test_expect_success LONGTEST 'ANL netpipe works' '
+	flux run -n 2 $OPTS \
+		${FLUX_BUILD_DIR}/t/mpi/mpich_basic/netpipe
+'
+test_expect_success 'ANL patterns works 1 node' '
+	run_timeout 30 flux run -n 2 -N1 $OPTS \
+		${FLUX_BUILD_DIR}/t/mpi/mpich_basic/patterns
+'
+test_expect_success 'ANL patterns works 2 nodes' '
+	run_timeout 30 flux run -n 2 -N2 $OPTS \
+		${FLUX_BUILD_DIR}/t/mpi/mpich_basic/patterns
+'
+test_expect_success LONGTEST 'ANL adapt works' '
+	flux run -n 3 $OPTS \
+		${FLUX_BUILD_DIR}/t/mpi/mpich_basic/adapt
+'
+
+
 test_done
diff --git a/t/t3001-mpi-personalities.t b/t/t3001-mpi-personalities.t
index 54ce9a4b985f..6d774502938c 100755
--- a/t/t3001-mpi-personalities.t
+++ b/t/t3001-mpi-personalities.t
@@ -8,59 +8,89 @@ test_description="Test that Flux's MPI personalities work"
 SIZE=4
 test_under_flux ${SIZE}
 
-jq=$(which jq 2>/dev/null)
-test -z "$jq" || test_set_prereq HAVE_JQ
-
 run_program() {
-        local timeout=$1
+	local timeout=$1
 	local nnodes=$2
 	local ntasks=$3
 	shift 3
-        run_timeout $timeout flux mini run $OPTS -n${ntasks} -N${nnodes} $*
+	run_timeout $timeout flux run $OPTS -n${ntasks} -N${nnodes} $*
 }
 
-CONF_PMI_LIBRARY_PATH=$(flux getattr conf.pmi_library_path | sed 's|/libpmi.so||')
-
-test_expect_success 'mvapich mpi prepends to LD_LIBRARY_PATH for tasks' '
-  run_program 5 ${SIZE} ${SIZE} printenv LD_LIBRARY_PATH > mvapich.ld.out &&
-  result="$(uniq mvapich.ld.out | cut -d: -f1)" &&
-  echo $result &&
-  echo $CONF_PMI_LIBRARY_PATH &&
-  test "$result" = $CONF_PMI_LIBRARY_PATH
+test_expect_success NO_ASAN "spectrum mpi only enabled with option" '
+	LD_PRELOAD_saved=${LD_PRELOAD} &&
+	unset LD_PRELOAD &&
+	test_when_finished "export LD_PRELOAD=${LD_PRELOAD_saved}" &&
+	flux run -n${SIZE} -N${SIZE} \
+	  -o mpi=spectrum --dry-run printenv LD_PRELOAD > j.spectrum &&
+	test_expect_code 1 run_program 15 ${SIZE} ${SIZE} printenv LD_PRELOAD &&
+	jobid=$(flux job submit j.spectrum) &&
+	flux job attach ${jobid} > spectrum.out &&
+	grep /opt/ibm/spectrum spectrum.out
 '
 
-test_expect_success 'mvapich mpi sets MPIRUN_RANK for tasks' '
-  run_program 5 ${SIZE} ${SIZE} printenv MPIRUN_RANK \
-    | sort > mvapich.rank.out &&
-  test_debug "cat mvapich.rank.out" &&
-  printf "0\n1\n2\n3\n" > mvapich.rank.expected &&
-  test_cmp mvapich.rank.expected mvapich.rank.out
+test_expect_success NO_ASAN "spectrum mpi also enabled with spectrum@version" '
+	LD_PRELOAD_saved=${LD_PRELOAD} &&
+	unset LD_PRELOAD &&
+	test_when_finished "export LD_PRELOAD=${LD_PRELOAD_saved}" &&
+	flux run -n${SIZE} -N${SIZE} \
+	  -o mpi=spectrum@10.4 --dry-run printenv LD_PRELOAD > j.spectrum &&
+	test_expect_code 1 run_program 15 ${SIZE} ${SIZE} printenv LD_PRELOAD &&
+	jobid=$(flux job submit j.spectrum) &&
+	flux job attach ${jobid} > spectrum.out &&
+	grep /opt/ibm/spectrum spectrum.out
 '
 
-test_expect_success "intel mpi rewrites I_MPI_MPI_LIBRARY" '
-  export I_MPI_PMI_LIBRARY="foobar" &&
-  test_when_finished "unset I_MPI_PMI_LIBRARY" &&
-  run_program 5 ${SIZE} ${SIZE} printenv I_MPI_PMI_LIBRARY > intel-mpi.set &&
-  test "$(uniq intel-mpi.set)" = "$(flux getattr conf.pmi_library_path)"
+test_expect_success 'spectrum mpi sets OMPI_COMM_WORLD_RANK' '
+	flux run -n${SIZE} -N${SIZE} \
+	  -o mpi=spectrum --dry-run \
+	  printenv OMPI_COMM_WORLD_RANK > j.spectrum &&
+	jobid=$(flux job submit j.spectrum) &&
+	flux job attach ${jobid} | sort -n > spectrum.rank.out &&
+	test_debug "cat spectrum.rank.out" &&
+	printf "0\n1\n2\n3\n" > spectrum.rank.expected &&
+	test_cmp spectrum.rank.expected spectrum.rank.out
 '
 
-test_expect_success "intel mpi only rewrites when necessary" '
-   echo $I_MPI_PMI_LIBRARY &&
-   run_program 5 ${SIZE} ${SIZE} printenv | grep I_MPI_PMI_LIBRARY \
-       | tee intel-mpi.unset &&
-   test "$(wc -l intel-mpi.unset | cut -f 1 -d " ")" = "0"
+export FLUX_SHELL_RC_PATH=$(pwd)/shellrc
+test_expect_success 'create test mpi plugins in $FLUX_SHELL_RC_PATH' '
+	mkdir -p shellrc/mpi &&
+	cat <<-EOF >shellrc/mpi/test.lua &&
+	shell.setenv ("MPI_TEST_NOVERSION", "t")
+	EOF
+	cat <<-EOF >shellrc/mpi/test@1.lua &&
+	shell.setenv ("MPI_TEST_VERSION", "1")
+	EOF
+	cat <<-EOF >shellrc/mpi/test@2.lua
+	shell.setenv ("MPI_TEST_VERSION", "2")
+	EOF
 '
-
-test_expect_success NO_ASAN,HAVE_JQ "spectrum mpi only enabled with option" '
-  LD_PRELOAD_saved=${LD_PRELOAD} &&
-  unset LD_PRELOAD &&
-  test_when_finished "export LD_PRELOAD=${LD_PRELOAD_saved}" &&
-  flux jobspec srun -n${SIZE} -N${SIZE} printenv LD_PRELOAD \
-    | jq ".attributes.system.shell.options.mpi = \"spectrum\"" > j.spectrum &&
-  test_expect_code 1 run_program 5 ${SIZE} ${SIZE} printenv LD_PRELOAD &&
-  jobid=$(flux job submit j.spectrum) &&
-  flux job attach ${jobid} > spectrum.out &&
-  grep /opt/ibm/spectrum spectrum.out
+test_expect_success '-ompi=name loads path/mpi/name.lua' '
+	name=mpi-test &&
+	flux run -o mpi=test env | grep MPI_TEST > ${name}.out &&
+	test_debug "cat ${name}.out" &&
+	cat <<-EOF >${name}.expected &&
+	MPI_TEST_NOVERSION=t
+	EOF
+	test_cmp ${name}.expected ${name}.out
+'
+test_expect_success '-ompi=name loads path/mpi/name.lua' '
+	name=mpi-test1 &&
+	flux run -o mpi=test@1 env | grep MPI_TEST > ${name}.out &&
+	test_debug "cat ${name}.out" &&
+	cat <<-EOF >${name}.expected &&
+	MPI_TEST_NOVERSION=t
+	MPI_TEST_VERSION=1
+	EOF
+	test_cmp ${name}.expected ${name}.out
+'
+test_expect_success '-ompi=name loads path/mpi/name.lua' '
+	name=mpi-test2 &&
+	flux run -o mpi=test@2 env | grep MPI_TEST > ${name}.out &&
+	test_debug "cat ${name}.out" &&
+	cat <<-EOF >${name}.expected &&
+	MPI_TEST_NOVERSION=t
+	MPI_TEST_VERSION=2
+	EOF
+	test_cmp ${name}.expected ${name}.out
 '
-
 test_done
diff --git a/t/t3002-pmi.t b/t/t3002-pmi.t
new file mode 100755
index 000000000000..5e1c7301c35d
--- /dev/null
+++ b/t/t3002-pmi.t
@@ -0,0 +1,329 @@
+#!/bin/sh
+#
+
+test_description="Test Flux PMI implementation"
+
+. `dirname $0`/sharness.sh
+
+export TEST_UNDER_FLUX_CORES_PER_RANK=4
+SIZE=$(test_size_large)
+test_under_flux ${SIZE} job
+
+kvstest=${FLUX_BUILD_DIR}/src/common/libpmi/test_kvstest
+kvstest2=${FLUX_BUILD_DIR}/src/common/libpmi/test_kvstest2
+pmi_info=${FLUX_BUILD_DIR}/src/common/libpmi/test_pmi_info
+pmi2_info=${FLUX_BUILD_DIR}/src/common/libpmi/test_pmi2_info
+
+test_expect_success 'flux run sets PMI_FD' '
+	flux run printenv PMI_FD
+'
+test_expect_success 'flux run sets FLUX_PMI_LIBRARY_PATH' '
+	flux run printenv FLUX_PMI_LIBRARY_PATH
+'
+test_expect_success 'flux run sets LD_LIBRARY_PATH if unset' '
+	flux run --env=-LD_LIBRARY_PATH printenv LD_LIBRARY_PATH \
+	    >libpath.out &&
+	echo $(dirname $(flux config builtin pmi_library_path)) \
+	    >libpath.exp &&
+	test_cmp libpath.exp libpath.out
+'
+test_expect_success 'flux run prepends to LD_LIBRARY_PATH if set' '
+	flux run --env=LD_LIBRARY_PATH=/a printenv LD_LIBRARY_PATH \
+	    >libpath2.out &&
+	echo $(dirname $(flux config builtin pmi_library_path)):/a \
+	    >libpath2.exp &&
+	test_cmp libpath2.exp libpath2.out
+'
+test_expect_success 'flux run -o pmi=off does not set LD_LIBRARY_PATH' '
+	test_must_fail flux run -o pmi=off --env=-LD_LIBRARY_PATH \
+	    printenv LD_LIBRARY_PATH
+'
+test_expect_success 'flux run -o pmi=simple sets PMI_FD' '
+	flux run -o pmi=simple printenv PMI_FD
+'
+test_expect_success 'flux run -o pmi=pmi1 sets PMI_FD' '
+	flux run -o pmi=pmi1 printenv PMI_FD
+'
+test_expect_success 'flux run -o pmi=pmi2 sets PMI_FD' '
+	flux run -o pmi=pmi2 printenv PMI_FD
+'
+test_expect_success 'flux run -o pmi=simple,unknown sets PMI_FD' '
+	flux run -o pmi=simple,unknown printenv PMI_FD
+'
+test_expect_success 'flux run -o pmi=unknown,simple sets PMI_FD' '
+	flux run -o pmi=unknown,simple printenv PMI_FD
+'
+test_expect_success 'flux run -o pmi=unknown does not set PMI_FD' '
+	test_must_fail flux run -o pmi=unknown printenv PMI_FD
+'
+test_expect_success 'flux run -o pmi=off does not set PMI_FD' '
+	test_must_fail flux run -o pmi=off printenv PMI_FD
+'
+test_expect_success 'flux run -o pmi=off does not set FLUX_PMI_LIBRARY_PATH' '
+	test_must_fail flux run -o pmi=off printenv FLUX_PMI_LIBRARY_PATH
+'
+test_expect_success 'flux run -o pmi.badopt fails' '
+	test_must_fail flux run -o pmi.badopt true
+'
+test_expect_success 'flux run -o pmi-simple.exchange.badopt fails' '
+	test_must_fail flux run -o pmi-simple.exchange.badopt true
+'
+test_expect_success 'flux run -o pmi-simple.exchange.k=foo fails' '
+	test_must_fail flux run -o pmi-simple.exchange.k=foo true
+'
+test_expect_success 'flux run -o pmi-simple.nomap=foo fails' '
+	test_must_fail flux run -o pmi-simple.nomap=foo true
+'
+
+test_expect_success 'pmi_info works' '
+	flux run -n${SIZE} -N${SIZE} ${pmi_info}
+'
+
+test_expect_success 'pmi_info --clique shows each node with own clique' '
+	flux run -n${SIZE} -N${SIZE} \
+		${pmi_info} --clique >clique.out &&
+	count=$(cut -f2 -d: clique.out | sort | uniq | wc -l) &&
+	test $count -eq ${SIZE}
+'
+test_expect_success 'pmi_info --clique none shows each task in its own clique' '
+	flux run -opmi-simple.nomap -n${SIZE} -N${SIZE} \
+		${pmi_info} --clique >clique.none.out &&
+	count=$(cut -f2 -d: clique.none.out | sort | uniq | wc -l) &&
+	test $count -eq ${SIZE}
+'
+test_expect_success 'kvstest works' '
+	flux run -n${SIZE} -N${SIZE} ${kvstest}
+'
+
+test_expect_success 'kvstest works with -o pmi-simple.exchange.k=1' '
+	flux run -n${SIZE} -N${SIZE} -o pmi-simple.exchange.k=1 ${kvstest}
+'
+test_expect_success 'kvstest works with -o pmi-simple.exchange.k=SIZE' '
+	flux run -n${SIZE} -N${SIZE} -o pmi-simple.exchange.k=${SIZE} \
+		${kvstest} 2>kvstest_k.err &&
+	grep "using k=${SIZE}" kvstest_k.err
+'
+test_expect_success 'kvstest works with -o pmi-simple.exchange.k=SIZE+1' '
+	flux run -n${SIZE} -N${SIZE} -o pmi-simple.exchange.k=$((${SIZE}+1)) \
+		${kvstest} 2>kvstest_kp1.err &&
+	grep "using k=${SIZE}" kvstest_kp1.err
+'
+
+test_expect_success 'kvstest fails with -o pmi-simple.kvs=unknown' '
+	test_must_fail flux run -o pmi-simple.kvs=unknown ${kvstest}
+'
+
+test_expect_success 'kvstest works with -o pmi-simple.kvs=native' '
+	flux run -n${SIZE} -N${SIZE} -o pmi-simple.kvs=native ${kvstest}
+'
+
+test_expect_success 'kvstest -N8 works' '
+	flux run -n${SIZE} -N${SIZE} ${kvstest} -N8
+'
+
+test_expect_success 'kvstest -N8 works with -o pmi-simple.kvs=native' '
+	flux run -n${SIZE} -N${SIZE} -o pmi-simple.kvs=native ${kvstest} -N8
+'
+
+test_expect_success 'verbose=2 shell option enables PMI server side tracing' '
+	flux run -n${SIZE} -N${SIZE} -o verbose=2 ${kvstest} 2>trace.out &&
+	grep "cmd=finalize_ack" trace.out
+'
+
+test_expect_success 'pmi2_info works' '
+	flux run -n${SIZE} -N${SIZE} ${pmi2_info}
+'
+
+# Run on one node only, to avoid the job spanning multiple brokers.
+# "Node-scope" PMI2 attributes are really "shell scope", and
+# PMI_process_mapping is used by the test to expect which ranks can exchange
+# node scope attributes.
+test_expect_success 'kvstest2 works' '
+	flux run -n4 -N1 -overbose=2 ${kvstest2}
+'
+
+# Abort test uses ! for expected failure rather than 'test_expect_code'
+# because the job exit code is not deterministic.  'test_must_fail' cannot be
+# used either because 128 + SIGNUM is not accepted as "failure".
+
+test_expect_success 'PMI application abort is handled properly' '
+	! run_timeout 60 flux run -overbose=2 \
+		${pmi_info} --abort 0
+'
+test_expect_success 'PMI2 application abort is handled properly' '
+	! run_timeout 60 flux run -overbose=2 \
+		${pmi2_info} --abort 0
+'
+
+# Ensure that pmi_info can get clique ranks with a large enough
+# number of tasks per node that PMI_process_mapping likely
+# overflowed the max value length for the PMI-1 KVS. This ensures
+# that the PMI client picked up flux.taskmap instead:
+
+test_expect_success 'PMI1 can calculate clique ranks with 128 tasks per node' '
+	flux run -t 1m -N2 --taskmap=cyclic --tasks-per-node=128 \
+		${pmi_info} --clique >tpn.128.out &&
+	test_debug "cat tpn.128.out" &&
+	grep "0: clique=0,2,4,6,8,10,12,14,16,18,20" tpn.128.out
+'
+
+test_expect_success 'flux-pmi barrier works' '
+	flux run --label-io -n2 \
+	    flux pmi barrier
+'
+test_expect_success 'flux-pmi barrier --count works' '
+	flux run --label-io -n2 \
+	    flux pmi barrier --count=2
+'
+test_expect_success 'flux-pmi barrier --abort works' '
+	test_expect_code 143 flux run --label-io -n2 \
+	    flux pmi barrier --abort=1 2>barrier_abort.err &&
+	grep -i abort barrier_abort.err
+'
+test_expect_success 'flux-pmi exchange works' '
+	flux run --label-io -n2 \
+	    flux pmi exchange
+'
+test_expect_success 'flux-pmi exchange --count works' '
+	flux run --label-io -n2 \
+	    flux pmi exchange --count=2
+'
+test_expect_success 'flux-pmi get PMI_process_mapping works' '
+	flux run --label-io -n2 \
+	    flux pmi get PMI_process_mapping
+'
+test_expect_success 'flux-pmi get --ranks=1 flux.taskmap works' '
+	flux run --label-io -n2 \
+	    flux pmi get --ranks=1 flux.taskmap
+'
+test_expect_success 'flux-pmi get --ranks=all flux.instance-level works' '
+	flux run --label-io -n2 \
+	    flux pmi get --ranks=all flux.instance-level
+'
+test_expect_success 'flux-pmi get works with multiple keys' '
+	flux run --label-io -n2 \
+	    flux pmi get flux.instance-level PMI_process_mapping
+'
+test_expect_success 'flux-pmi works outside of job' '
+	flux pmi -v --libpmi-noflux barrier
+'
+test_expect_success 'flux-pmi fails with bad subcommand' '
+	test_must_fail flux run flux pmi notacmd
+'
+test_expect_success 'flux-pmi get fails with bad option' '
+	test_must_fail flux run flux pmi get --badopt foo
+'
+test_expect_success 'flux-pmi get fails with bad ranks option' '
+	test_must_fail flux run flux pmi get --ranks=1.2 flux.taskmap
+'
+test_expect_success 'flux-pmi fails with bad ranks option' '
+	test_must_fail flux run \
+	    flux pmi --method=badmethod barrier
+'
+# method=simple (covered above also)
+test_expect_success 'flux-pmi --method=simple fails outside of job' '
+	test_must_fail flux pmi --method=simple barrier
+'
+test_expect_success 'flux-pmi -v --method=simple works within job' '
+	flux run --label-io -n2 flux pmi -v --method=simple barrier
+'
+test_expect_success 'flux-pmi -opmi=off --method=simple fails' '
+	test_must_fail flux run -o pmi=off \
+	    flux pmi --method=simple barrier
+'
+# method=libpmi
+test_expect_success 'get pmi library path' '
+	flux config builtin pmi_library_path >libpmi
+'
+test_expect_success 'flux-pmi --method=libpmi:/bad/path fails' '
+	test_must_fail flux run \
+	    flux pmi --method=libpmi:/bad/path barrier
+'
+test_expect_success 'flux-pmi --method=libpmi barrier works w/ flux libpmi.so' '
+	flux run -n2 bash -c "\
+	    flux pmi -v --method=libpmi:$(cat libpmi) barrier"
+'
+test_expect_success 'flux-pmi --method=libpmi barrier abort works w/ flux libpmi.so' '
+	test_expect_code 143 flux run -n2 bash -c "\
+	    flux pmi -v --method=libpmi:$(cat libpmi) barrier --abort 1"
+'
+test_expect_success 'flux-pmi --method=libpmi exchange works w/ flux libpmi.so' '
+	flux run -n2 bash -c "\
+	    flux pmi -v --method=libpmi:$(cat libpmi) exchange"
+'
+test_expect_success 'flux-pmi --method=libpmi get works w/ flux libpmi.so' '
+	flux run -n2 bash -c "\
+	    flux pmi -v --method=libpmi:$(cat libpmi) get flux.taskmap"
+'
+test_expect_success 'flux-pmi --libpmi-noflux fails w/ flux libpmi.so' '
+	test_must_fail flux run bash -c "\
+	    flux pmi --method=libpmi:$(cat libpmi) --libpmi-noflux barrier"
+'
+test_expect_success 'flux broker refuses the Flux libpmi.so and goes single' '
+	FLUX_PMI_DEBUG=1 FLUX_PMI_CLIENT_METHODS="libpmi single" \
+	    LD_LIBRARY_PATH=$(dirname $(cat libpmi)) \
+            flux start true 2>debug.err &&
+	grep single debug.err
+'
+# method=libpmi2
+test_expect_success 'get pmi2 library path' '
+	flux config builtin pmi_library_path | sed s/libpmi/libpmi2/ >libpmi2
+'
+test_expect_success 'flux-pmi --method=libpmi2:/bad/path fails' '
+	test_must_fail flux run \
+	    flux pmi --method=libpmi2:/bad/path barrier
+'
+test_expect_success 'flux-pmi --method=libpmi2 barrier works w/ flux pmi lib' '
+	flux run -n2 bash -c "\
+	    flux pmi -v --method=libpmi2:$(cat libpmi2) barrier"
+'
+test_expect_success 'flux-pmi --method=libpmi2 barrier works w/ flux pmi lib' '
+	test_expect_code 143 flux run -n2 bash -c "\
+	    flux pmi -v --method=libpmi2:$(cat libpmi2) barrier --abort 1"
+'
+test_expect_success 'flux-pmi --method=libpmi2 exchange works w/ flux pmi lib' '
+	flux run -n2 bash -c "\
+	    flux pmi -v --method=libpmi2:$(cat libpmi2) exchange"
+'
+test_expect_success 'flux-pmi --method=libpmi2 exchange works w/ cray quirks' '
+	flux run -n2 bash -c "\
+	    flux pmi -v --libpmi2-cray --method=libpmi2:$(cat libpmi2) exchange"
+'
+test_expect_success 'flux-pmi --method=libpmi2 get works w/ flux pmi lib' '
+	flux run -n2 bash -c "\
+	    flux pmi -v --method=libpmi2:$(cat libpmi2) get flux.taskmap"
+'
+test_expect_success 'flux-pmi --method=libpmi2 get fails w/ cray quirks' '
+	test_must_fail flux run -n2 bash -c "\
+	    flux pmi -v --libpmi2-cray --method=libpmi2:$(cat libpmi2) \
+	        get flux.taskmap"
+'
+test_expect_success 'PMI_process_mapping special case works' '
+	flux run -n2 bash -c "\
+	    flux pmi -v --method=libpmi2:$(cat libpmi2) get PMI_process_mapping"
+'
+test_expect_success 'flux-pmi --libpmi-noflux fails w/ flux libpmi2.so' '
+	test_must_fail flux run bash -c "\
+	    flux pmi --method=libpmi2:$(cat libpmi2) --libpmi-noflux barrier"
+'
+test_expect_success 'flux broker refuses the Flux pmi lib and goes single' '
+	FLUX_PMI_DEBUG=1 FLUX_PMI_CLIENT_METHODS="libpmi2 single" \
+	    LD_LIBRARY_PATH=$(dirname $(cat libpmi2)) \
+            flux start true 2>debug.err &&
+	grep single debug.err
+'
+# method=single
+test_expect_success 'flux-pmi --method=single barrier works' '
+	flux pmi -v --method=single barrier
+'
+test_expect_success 'flux-pmi --method=single exchange works' '
+	flux pmi -v --method=single exchange
+'
+test_expect_success 'flux-pmi --method=single get notakey fails' '
+	test_must_fail flux pmi --method=single get notakey
+'
+test_expect_success 'flux-pmi -opmi=off --method=single works' '
+	flux run -o pmi=off \
+	    flux pmi --method=single barrier
+'
+test_done
diff --git a/t/t3003-mpi-abort.t b/t/t3003-mpi-abort.t
new file mode 100755
index 000000000000..e47f6b0309d4
--- /dev/null
+++ b/t/t3003-mpi-abort.t
@@ -0,0 +1,83 @@
+#!/bin/sh
+#
+
+test_description='Test that MPI_Abort works'
+
+# MPI behaviors WRT to MPI_Abort():
+#
+# mpich
+#   In versions < 3.2.0, MPI_Abort() prints error and exits from the task.
+#   In versions >= 3.2.0, MPI_Abort() sends a PMI-1 abort command using its
+#   built-in PMI-1 wire protocol client.
+# openmpi
+#   In Openmpi version 4, the flux MCA plugin dlopens the flux libpmi.so
+#   and calls PMI_Abort().  The current implementation sends a PMI-1 abort
+#   command.
+# mvapich
+#   In version 2.3.6, MPI_Abort() prints error and exits from the task.
+#   Other versions not checked.
+#
+# N.B. After an MPI_Abort(), the job might exit with
+# 1) the MPI_Abort() exit code
+# 2) (128 + sig) if other shells have to be cleaned up
+
+. `dirname $0`/sharness.sh
+
+if test -z "$FLUX_TEST_MPI"; then
+    skip_all='skipping MPI tests, FLUX_TEST_MPI not set'
+    test_done
+fi
+
+if ! test -x ${FLUX_BUILD_DIR}/t/mpi/abort; then
+    skip_all='skipping MPI tests, MPI not available/configured'
+    test_done
+fi
+
+# work around https://github.com/open-mpi/ompi/issues/7701 and similar in mpich
+export HWLOC_COMPONENTS=-gl
+export TEST_UNDER_FLUX_CORES_PER_RANK=4
+SIZE=2
+MAX_MPI_SIZE=$(($SIZE*$TEST_UNDER_FLUX_CORES_PER_RANK))
+test_under_flux $SIZE job
+
+OPTS="-ocpu-affinity=off"
+
+diag() {
+	echo "test failed: cat $1"
+	cat $1
+	return 1
+}
+
+test_expect_success 'show MPI version under test' '
+        ${FLUX_BUILD_DIR}/t/mpi/version
+'
+
+# These tests use ! for expected failure rather than 'test_expect_code'
+# because the job exit code is not deterministic.  'test_must_fail' cannot be
+# used either because 128 + SIGNUM is not accepted as "failure".
+
+# get a PMI server trace in the CI output in case we need it to debug!
+test_expect_success 'MPI_Abort size=1 with PMI server tracing enabled' '
+	${FLUX_BUILD_DIR}/t/mpi/version | head -1 &&
+	! run_timeout 60 flux run $OPTS -overbose=2 \
+		${FLUX_BUILD_DIR}/t/mpi/abort 0
+'
+
+test_expect_success "MPI_Abort on size=${MAX_MPI_SIZE}, first rank triggers exception" '
+	! run_timeout 60 flux run -n${MAX_MPI_SIZE} $OPTS \
+		${FLUX_BUILD_DIR}/t/mpi/abort 0 2>abort0.err &&
+	(grep exception abort0.err || diag abort0.err) &&
+	test_debug "cat abort0.err" &&
+	grep "Rank 0 is going to MPI_Abort now" abort0.err
+'
+
+test_expect_success "MPI_Abort on size=${MAX_MPI_SIZE}, last rank triggers exception" '
+	rank=$(($MAX_MPI_SIZE-1)) &&
+	! run_timeout 60 flux run -n${MAX_MPI_SIZE} $OPTS \
+		${FLUX_BUILD_DIR}/t/mpi/abort $rank \
+		2>abort1.err &&
+	(grep exception abort1.err || diag abort1.err) &&
+	grep "Rank $rank is going to MPI_Abort now" abort1.err
+'
+
+test_done
diff --git a/t/t3100-flux-in-flux.t b/t/t3100-flux-in-flux.t
index 5e608149e4e1..c54dbe2222c7 100755
--- a/t/t3100-flux-in-flux.t
+++ b/t/t3100-flux-in-flux.t
@@ -10,26 +10,26 @@ SIZE=$(test_size_large)
 test_under_flux ${SIZE}
 echo "# $0: flux session size will be ${SIZE}"
 
-ARGS="-o,-Sbroker.rc1_path=,-Sbroker.rc3_path="
+ARGS="-Sbroker.rc1_path= -Sbroker.rc3_path="
 test_expect_success "flux can run flux instance as a job" '
-	run_timeout 10 flux mini run -n1 -N1 \
+	run_timeout 60 flux run -n1 -N1 \
 		flux start ${ARGS} flux getattr size >size.out &&
 	echo 1 >size.exp &&
 	test_cmp size.exp size.out
 '
 
-test_expect_success "flux subinstance leaves local_uri, remote_uri in KVS" '
-	flux jobspec srun -n1 -N1 flux start /bin/true >j &&
-	id=$(flux job submit j) &&
-	flux job wait-event $id finish &&
-	flux job info $id guest.flux.local_uri &&
-	flux job info $id guest.flux.remote_uri
+test_expect_success 'flux subinstance sets uri job memo' '
+	jobid=$(flux batch -n1 --wrap sleep 300) &&
+	flux job wait-event -t 60 ${jobid} memo &&
+	flux jobs -no {user.uri} ${jobid} > uri.memo &&
+	grep ^ssh:// uri.memo &&
+	flux cancel $jobid &&
+	flux job wait-event $jobid clean
 '
 
 test_expect_success "flux --parent works in subinstance" '
-	id=$(flux jobspec srun -n1 \
-               flux start ${ARGS} flux --parent kvs put test=ok \
-               | flux job submit) &&
+	id=$(flux submit \
+		flux start ${ARGS} flux --parent kvs put test=ok) &&
 	flux job attach $id &&
 	flux job info $id guest.test > guest.test &&
 	cat <<-EOF >guest.test.exp &&
@@ -39,10 +39,9 @@ test_expect_success "flux --parent works in subinstance" '
 '
 
 test_expect_success "flux --parent --parent works in subinstance" '
-	id=$(flux jobspec srun -n1 \
-              flux start ${ARGS} \
-	      flux start ${ARGS} flux --parent --parent kvs put test=ok \
-	      | flux job submit) &&
+	id=$(flux batch -n1 --wrap \
+		flux run flux start ${ARGS} \
+		flux --parent --parent kvs put test=ok) &&
 	flux job attach $id &&
 	flux job info $id guest.test > guest2.test &&
 	cat <<-EOF >guest2.test.exp &&
@@ -51,7 +50,7 @@ test_expect_success "flux --parent --parent works in subinstance" '
 	test_cmp guest2.test.exp guest2.test
 '
 
-test_expect_success "instance-level attribute = 0 in new stanalone instance" '
+test_expect_success "instance-level attribute = 0 in new standalone instance" '
 	flux start ${ARGS} flux getattr instance-level >level_new.out &&
 	echo 0 >level_new.exp &&
 	test_cmp level_new.exp level_new.out
@@ -64,27 +63,34 @@ test_expect_success "instance-level attribute = 0 in test instance" '
 '
 
 test_expect_success "instance-level attribute = 1 in first subinstance" '
-        flux mini run flux start ${ARGS} \
-            flux getattr instance-level >level1.out &&
+	flux run flux start ${ARGS} \
+		flux getattr instance-level >level1.out &&
 	echo 1 >level1.exp &&
 	test_cmp level1.exp level1.out
 '
 
 test_expect_success "instance-level attribute = 2 in second subinstance" '
-        flux mini run flux start \
-	    flux mini run flux start ${ARGS} \
+	flux run flux start \
+		flux run flux start ${ARGS} \
 		flux getattr instance-level >level2.out &&
 	echo 2 >level2.exp &&
 	test_cmp level2.exp level2.out
 '
 
 test_expect_success "flux sets jobid attribute" '
-	id=$(flux jobspec srun -n1 \
-		flux start ${ARGS} flux getattr jobid \
-		| flux job submit) &&
+	id=$(flux submit \
+		flux start ${ARGS} flux getattr jobid) &&
 	echo "$id" >jobid.exp &&
 	flux job attach $id >jobid.out &&
 	test_cmp jobid.exp jobid.out
 '
-
+test_expect_success 'flux can launch multiple brokers per node' '
+	flux alloc -x -N2 -o per-resource.count=2 \
+		flux resource info
+'
+test_expect_success 'flux can launch multiple brokers per node (R lookup fallback)' '
+	flux alloc -N2 --exclusive -o per-resource.count=2 \
+		--conf=resource.no-update-watch=true \
+		flux resource info
+'
 test_done
diff --git a/t/t3200-instance-restart.t b/t/t3200-instance-restart.t
new file mode 100755
index 000000000000..a0370b168831
--- /dev/null
+++ b/t/t3200-instance-restart.t
@@ -0,0 +1,125 @@
+#!/bin/sh
+#
+
+test_description='Test instance restart'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. `dirname $0`/sharness.sh
+
+if test -n "$S3_ACCESS_KEY_ID"; then
+    test_set_prereq S3
+    export FLUX_CONF_DIR=$(pwd)
+fi
+
+test_expect_success 'run a job in persistent instance' '
+	flux start --setattr=statedir=$(pwd) \
+	           flux submit true >id1.out
+'
+
+test_expect_success 'restart instance and run another job' '
+	flux start --setattr=statedir=$(pwd) \
+	           flux submit true >id2.out
+'
+
+test_expect_success 'restart instance and run another job' '
+	flux start --setattr=statedir=$(pwd) \
+	           flux submit true >id3.out
+'
+
+test_expect_success 'restart instance and list inactive jobs' '
+	flux start --setattr=statedir=$(pwd) \
+	           flux jobs --no-header --format={id} \
+		   	--filter=INACTIVE >list.out
+'
+
+test_expect_success 'inactive job list contains all jobs run before' '
+	grep $(cat id1.out) list.out &&
+	grep $(cat id2.out) list.out &&
+	grep $(cat id3.out) list.out
+'
+
+test_expect_success 'job IDs were issued in ascending order' '
+	test $(cat id1.out | flux job id) -lt $(cat id2.out | flux job id) &&
+	test $(cat id2.out | flux job id) -lt $(cat id3.out | flux job id)
+'
+
+test_expect_success 'restart instance and capture startlog' '
+	flux start --setattr=statedir=$(pwd) \
+	           flux startlog >startlog.out
+'
+test_expect_success 'startlog shows 5 run periods' '
+	test $(wc -l <startlog.out) -eq 5
+'
+test_expect_success 'most recent period is still running' '
+	tail -1 startlog.out | grep running
+'
+
+test_expect_success 'doctor startlog to look like a crash' '
+	flux start --setattr=statedir=$(pwd) \
+		-Sbroker.rc1_path=$SHARNESS_TEST_SRCDIR/rc/rc1-kvs \
+		-Sbroker.rc3_path=$SHARNESS_TEST_SRCDIR/rc/rc3-kvs \
+		flux startlog --post-start-event
+'
+test_expect_success 'run flux and capture logs on stderr' '
+	flux start --setattr=statedir=$(pwd) \
+		--setattr=log-stderr-level=6 \
+		true 2>improper.err
+'
+test_expect_success 'improper shutdown was logged' '
+	grep "Flux was not shut down properly" improper.err
+'
+
+test_expect_success 'run a job in persistent instance (content-files)' '
+	flux start \
+	    -Scontent.backing-module=content-files \
+	    -Sstatedir=$(pwd) \
+	    flux submit true >files_id1.out
+'
+test_expect_success 'restart instance and list inactive jobs' '
+	flux start \
+	    -Scontent.backing-module=content-files \
+	    -Sstatedir=$(pwd) \
+	    flux jobs --no-header --format={id} \
+	        --filter=INACTIVE >files_list.out
+'
+
+test_expect_success 'inactive job list contains job from before restart' '
+	grep $(cat files_id1.out) files_list.out
+'
+
+test_expect_success S3 'create creds.toml from env' '
+	mkdir -p creds &&
+	cat >creds/creds.toml <<-CREDS
+	access-key-id = "$S3_ACCESS_KEY_ID"
+	secret-access-key = "$S3_SECRET_ACCESS_KEY"
+	CREDS
+'
+
+test_expect_success S3 'create content-s3.toml from env' '
+	cat >content-s3.toml <<-TOML
+	[content-s3]
+	credential-file = "$(pwd)/creds/creds.toml"
+	uri = "http://$S3_HOSTNAME"
+	bucket = "$S3_BUCKET"
+	virtual-host-style = false
+	TOML
+'
+
+test_expect_success S3 'run a job in persistent instance (content-s3)' '
+	flux start \
+	    -Scontent.backing-module=content-s3 \
+	    flux submit true >files_id2.out
+'
+test_expect_success S3 'restart instance and list inactive jobs' '
+	flux start \
+	    -Scontent.backing-module=content-s3 \
+	    flux jobs --no-header --format={id} \
+	        --filter=INACTIVE >files_list2.out
+'
+
+test_expect_success S3 'inactive job list contains job from before restart' '
+	grep $(cat files_id2.out) files_list2.out
+'
+
+test_done
diff --git a/t/t3201-crontabs.t b/t/t3201-crontabs.t
new file mode 100755
index 000000000000..99614f4d1cfe
--- /dev/null
+++ b/t/t3201-crontabs.t
@@ -0,0 +1,40 @@
+#!/bin/sh
+#
+
+test_description='Test rc1 loading of crontab files'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. `dirname $0`/sharness.sh
+
+test_expect_success 'empty cron.directory works' '
+	mkdir cron.d &&
+	flux start -Scron.directory=cron.d true
+'
+test_expect_success 'non-existent cron.directory works' '
+	flux start -Scron.directory=noexist true
+'
+test_expect_success 'cron.directory with subdirectory works' '
+	rm -rf cron.d &&
+	mkdir -p cron.d/subdir &&
+	flux start -Scron.directory=cron.d true
+'
+test_expect_success 'cron.directory with non-crontab file fails' '
+	rm -rf cron.d &&
+	mkdir cron.d &&
+	echo zzz >cron.d/badtab &&
+	test_must_fail flux start -Scron.directory=cron.d \
+		true 2>bad.err &&
+	grep "could not load crontab" bad.err
+'
+test_expect_success 'cron.directory with good crontab files works' '
+	rm -rf cron.d &&
+	mkdir cron.d &&
+	echo "10 * * * * true" >cron.d/goodtab &&
+	echo "20 * * * * hostname" >cron.d/goodtab2 &&
+	flux start -Scron.directory=cron.d flux cron list >list.out &&
+	grep true list.out &&
+	grep hostname list.out
+'
+
+test_done
diff --git a/t/t3202-instance-restart-testexec.t b/t/t3202-instance-restart-testexec.t
new file mode 100755
index 000000000000..396dc64cf8a1
--- /dev/null
+++ b/t/t3202-instance-restart-testexec.t
@@ -0,0 +1,57 @@
+#!/bin/sh
+
+test_description='Test instance restart and still running jobs with testexec'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. `dirname $0`/sharness.sh
+
+export FLUX_DISABLE_JOB_CLEANUP=t
+
+test_expect_success 'run a testexec job in persistent instance (long run)' '
+	flux start --setattr=statedir=$(pwd) \
+	     flux submit \
+	       --flags=debug \
+	       --setattr=system.exec.test.run_duration=100s \
+	       hostname >id1.out
+'
+
+test_expect_success 'restart instance, reattach to running job, cancel it (long run)' '
+	flux start --setattr=statedir=$(pwd) \
+	     sh -c "flux job eventlog $(cat id1.out) > eventlog_long1.out; \
+		    flux jobs -n > jobs_long1.out; \
+		    flux cancel $(cat id1.out)" &&
+	grep "reattach-start" eventlog_long1.out &&
+	grep "reattach-finish" eventlog_long1.out &&
+	grep $(cat id1.out) jobs_long1.out
+'
+
+test_expect_success 'restart instance, job completed (long run)' '
+	flux start --setattr=statedir=$(pwd) \
+	     sh -c "flux job eventlog $(cat id1.out) > eventlog_long2.out; \
+		    flux jobs -n > jobs_long2.out" &&
+	grep "finish" eventlog_long2.out | grep status &&
+	test_must_fail grep $(cat id1.out) jobs_long2.out
+'
+
+# reattach_finish will indicate to testexec that the job finished
+# right after reattach, emulating a job that finished before the
+# instance restarted
+test_expect_success 'run a testexec job in persistent instance (exit run)' '
+	flux start --setattr=statedir=$(pwd) \
+	     flux submit \
+	       --flags=debug \
+	       --setattr=system.exec.test.reattach_finish=1 \
+	       --setattr=system.exec.test.run_duration=100s \
+	     hostname >id2.out
+'
+
+test_expect_success 'restart instance, reattach to running job, its finished (exit run)' '
+	flux start --setattr=statedir=$(pwd) \
+	     sh -c "flux job eventlog $(cat id2.out) > eventlog_exit1.out" &&
+	grep "reattach-start" eventlog_exit1.out &&
+	grep "reattach-finish" eventlog_exit1.out &&
+	grep "finish" eventlog_exit1.out | grep status
+'
+
+test_done
diff --git a/t/t3203-instance-recovery.t b/t/t3203-instance-recovery.t
new file mode 100755
index 000000000000..f74268462a4c
--- /dev/null
+++ b/t/t3203-instance-recovery.t
@@ -0,0 +1,123 @@
+#!/bin/sh
+#
+
+test_description='Test instance recovery mode'
+
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. `dirname $0`/sharness.sh
+
+runpty="flux ${SHARNESS_TEST_SRCDIR}/scripts/runpty.py"
+
+# N.B. Increase test-exit-timeout from default of 20s, on slower / busy
+# systems timeout can trigger and kill broker.
+test_expect_success 'start a persistent instance of size 4' '
+	mkdir -p test1 &&
+	flux start --test-size=4 --test-exit-timeout=300s \
+		-Sstatedir=$(pwd)/test1 true
+'
+test_expect_success 'expected broker attributes are set in recovery mode' '
+	cat >recov_attrs.exp <<-EOT &&
+	1
+	1
+	5
+	EOT
+	flux start --recovery=$(pwd)/test1 \
+	    -Sbroker.rc1_path= \
+	    -Sbroker.rc3_path= \
+	    bash -c " \
+		flux getattr broker.recovery-mode && \
+	        flux getattr broker.quorum && \
+	        flux getattr log-stderr-level" >recov_attrs.out &&
+	test_cmp recov_attrs.exp recov_attrs.exp
+'
+test_expect_success 'banner message is printed in interactive recovery mode' '
+	run_timeout --env=SHELL=/bin/sh 120 \
+	    $runpty -i none flux start \
+	        -Sbroker.rc1_path= \
+	        -Sbroker.rc3_path= \
+	        --recovery=$(pwd)/test1 >banner.out &&
+	grep "Entering Flux recovery mode" banner.out
+'
+test_expect_success '--recovery is not ignored if --test-size is specified' '
+	run_timeout --env=SHELL=/bin/sh 120 \
+	    $runpty -i none flux start \
+	        -Sbroker.rc1_path= \
+	        -Sbroker.rc3_path= \
+	    	--test-size=1 \
+	        --recovery=$(pwd)/test1 >banner.out &&
+	grep "Entering Flux recovery mode" banner.out
+'
+test_expect_success 'rc1 failure is ignored in recovery mode' '
+	flux start --recovery=$(pwd)/test1 \
+	    -Sbroker.rc1_path=false \
+	    -Sbroker.rc3_path= \
+	    echo "hello world" >hello.out &&
+	grep hello hello.out
+'
+test_expect_success 'resources are offline in recovery mode' '
+	echo 4 >down.exp &&
+	flux start --recovery=$(pwd)/test1 \
+	    flux resource list -s down -no {nnodes} >down.out &&
+	test_cmp down.exp down.out
+'
+test_expect_success 'dump test instance in recovery mode' '
+	flux start --recovery=$(pwd)/test1 \
+	    flux dump --checkpoint test1.tar
+'
+test_expect_success 'recovery mode also works with dump file' '
+	flux start --recovery=$(pwd)/test1.tar \
+	    flux resource list -s down -no {nnodes} >down_dump.out &&
+	test_cmp down.exp down_dump.out
+'
+test_expect_success 'banner message warns changes are not persistent' '
+	run_timeout --env=SHELL=/bin/sh 120 \
+	    $runpty -i none flux start \
+	        -Sbroker.rc1_path= \
+	        -Sbroker.rc3_path= \
+	        --recovery=$(pwd)/test1.tar >banner2.out &&
+	grep "changes will not be preserved" banner2.out
+'
+test_expect_success 'recovery mode aborts early if statedir is missing' '
+	test_must_fail flux start --recovery=$(pwd)/test2 true \
+		2>dirmissing.err &&
+	grep "No such file or directory" dirmissing.err
+'
+test_expect_success 'recovery mode aborts early if statedir lacks rwx' '
+	mkdir -p test2 &&
+	chmod 600 test2 &&
+	test_must_fail flux start --recovery=$(pwd)/test2 true \
+		2>norwx.err &&
+	grep "no access" norwx.err
+'
+test_expect_success 'recovery mode aborts early if content is missing' '
+	chmod 700 test2 &&
+	test_must_fail flux start --recovery=$(pwd)/test2 true \
+		2>empty.err &&
+	grep "No such file or directory" empty.err
+'
+#  Note: some environments (such as fakeroot) do not appear to respect
+#  file permissions. In this case a read-only file is writeable and a
+#  write-only file may be readable, which will cause the following two
+#  tests to fail. Check for this condition and set a prereq that perms
+#  are working:
+#
+touch test_perms
+chmod 400 test_perms
+echo test >test_perms || test_set_prereq WORKING_PERMS
+
+test_expect_success WORKING_PERMS 'recovery mode aborts early if content unwritable' '
+	touch test2/content.sqlite &&
+	chmod 400 test2/content.sqlite &&
+	test_must_fail flux start --recovery=$(pwd)/test2 true \
+		2>nowrite.err &&
+	grep "no write permission" nowrite.err
+'
+test_expect_success WORKING_PERMS 'recovery mode aborts early if content unreadable' '
+	chmod 200 test2/content.sqlite &&
+	test_must_fail flux start --recovery=$(pwd)/test2 true \
+		2>noread.err &&
+	grep "no read permission" noread.err
+'
+
+test_done
diff --git a/t/t3300-system-basic.t b/t/t3300-system-basic.t
new file mode 100755
index 000000000000..ece8bf17a12a
--- /dev/null
+++ b/t/t3300-system-basic.t
@@ -0,0 +1,114 @@
+#!/bin/sh
+#
+
+test_description='Test the tools used to test the system instance.
+
+Before we begin testing the system instance, verify that the
+system test personality behaves as it should, and that the
+startctl tool and flux-start support for its RPCs work as expected.
+'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 3 system
+
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+
+overlay_connected_children() {
+	flux python -c "import flux; print(flux.Flux().rpc(\"overlay.stats-get\",nodeid=0).get_str())" | jq -r '.["child-connected"]'
+}
+
+test_expect_success 'broker hostlist has fake hostlist' '
+	echo "fake[0-2]" >hostlist.exp &&
+	flux getattr hostlist >hostlist.out &&
+	test_cmp hostlist.exp hostlist.out
+'
+
+test_expect_success 'startctl status works' '
+	$startctl status
+'
+
+test_expect_success 'broker overlay shows 2 connected children' '
+	test $(overlay_connected_children) -eq 2
+'
+
+test_expect_success 'overlay status is full' '
+	test "$(flux overlay status --timeout=0 --summary)" = "full"
+'
+
+test_expect_success 'kill broker rank=2 with SIGTERM like systemd stop' '
+	$startctl kill 2 15
+'
+
+test_expect_success 'broker exits with no error' '
+	run_timeout 30 $startctl wait 2
+'
+
+test_expect_success 'startctl shows rank 2 pid is -1' '
+	pid=$($startctl status | jq -r ".procs[2].pid") &&
+	test "$pid" = "-1"
+'
+
+test_expect_success 'broker overlay shows 1 connected child' '
+	test $(overlay_connected_children) -eq 1
+'
+
+test_expect_success 'wait for overlay status to be partial' '
+	run_timeout 10 flux overlay status --timeout=0 --summary --wait partial
+'
+
+test_expect_success 'run broker rank=2' '
+	$startctl run 2
+'
+
+test_expect_success 'wait for overlay status to be full' '
+	run_timeout 10 flux overlay status --timeout=0 --summary --wait full
+'
+
+test_expect_success 'flux exec over all ranks works' '
+	run_timeout 30 flux exec flux getattr rank
+'
+
+test_expect_success 'kill broker rank=2 with SIGKILL, broker exits with 128+9' '
+	$startctl kill 2 9 &&
+	test_expect_code 137 run_timeout 30 $startctl wait 2
+'
+
+# Ensure an EHOSTUNREACH is encountered to trigger connected state change.
+test_expect_success 'ping to rank 2 fails' '
+	test_must_fail flux ping 2
+'
+
+test_expect_success 'wait for overlay status to be degraded' '
+	run_timeout 10 flux overlay status --summary --wait degraded
+'
+
+test_expect_success 'run broker rank=2' '
+	$startctl run 2
+'
+
+test_expect_success 'wait for subtree to be full' '
+	run_timeout 10 flux overlay status --summary --wait full
+'
+
+test_expect_success 'run broker rank=2 again fails' '
+	test_must_fail $startctl run 2
+'
+
+test_expect_success 'run broker rank=42 fails' '
+	test_must_fail $startctl run 42
+'
+
+test_expect_success 'kill broker rank=42 fails' '
+	test_must_fail $startctl kill 42 15
+'
+
+test_expect_success 'wait broker rank=42 fails' '
+	test_must_fail $startctl wait 42
+'
+
+test_expect_success 'dump broker logs from leader broker' '
+	flux dmesg | grep "broker\..*\[0\]"
+'
+
+test_done
diff --git a/t/t3301-system-latestart.t b/t/t3301-system-latestart.t
new file mode 100755
index 000000000000..26a05c719c41
--- /dev/null
+++ b/t/t3301-system-latestart.t
@@ -0,0 +1,76 @@
+#!/bin/sh
+#
+
+test_description='Test system instance with late joining broker
+
+Start up only the leader broker and verify that the system is
+functional without a leaf node.  Then start the leaf node
+and ensure that it wires up.
+'
+
+. `dirname $0`/sharness.sh
+
+export TEST_UNDER_FLUX_QUORUM=1
+export TEST_UNDER_FLUX_START_MODE=leader
+
+test_under_flux 2 system
+
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+
+test_expect_success 'broker.quorum was set to 1 by system test personality' '
+	echo 1 >quorum.exp &&
+	flux getattr broker.quorum >quorum.out &&
+	test_cmp quorum.exp quorum.out
+'
+test_expect_success 'startctl shows rank 1 pids as -1' '
+	test $($startctl status | jq -r ".procs[1].pid") = "-1"
+'
+
+test_expect_success 'overlay status is partial' '
+        test "$(flux overlay status --timeout=0 --summary)" = "partial"
+'
+
+test_expect_success 'resource list shows one down nodes' '
+	echo 1 >down.exp &&
+	flux resource list -n -s down -o {nnodes} >down.out &&
+	test_cmp down.exp down.out
+'
+
+test_expect_success 'resource status shows no drained nodes' '
+	echo 0 >drain.exp &&
+	flux resource status -s drain -no {nnodes} >drain.out &&
+	test_cmp drain.exp drain.out
+'
+
+test_expect_success 'flux exec -r 1 fails with EHOSTUNREACH' '
+	test_must_fail run_timeout 30 flux exec -r 1 true 2>unreach.err &&
+	grep "$(strerror_symbol EHOSTUNREACH)" unreach.err
+'
+
+test_expect_success 'single node job can run with only rank 0 up' '
+	run_timeout 30 flux run -n1 true
+'
+
+test_expect_success 'two node job is accepted although it cannot run yet' '
+	flux submit -N2 -n2 echo Hello >jobid
+'
+
+test_expect_success 'start rank 1' '
+	$startctl run 1
+'
+
+test_expect_success 'startctl shows rank 1 pid not -1' '
+	test $($startctl status | jq -r ".procs[1].pid") != "-1"
+'
+
+test_expect_success 'wait for overlay status to be full' '
+        flux overlay status --wait full --timeout 10s
+'
+
+# Side effect: this test blocks until the rank 1 broker has a chance to
+# wire up and start services
+test_expect_success 'two node job can now run' '
+	run_timeout 30 flux job attach $(cat jobid)
+'
+
+test_done
diff --git a/t/t3302-system-offline.t b/t/t3302-system-offline.t
new file mode 100755
index 000000000000..37e603b45164
--- /dev/null
+++ b/t/t3302-system-offline.t
@@ -0,0 +1,50 @@
+#!/bin/sh
+#
+
+test_description='Test job submission when upstream is down
+
+Start up only the leader broker and a leaf node, but not the
+router node between them to simulate a login node without its
+upstream router.  Try to submit a job on the leaf node and
+confirm that the user gets a reasonable error.
+'
+
+. `dirname $0`/sharness.sh
+
+export TEST_UNDER_FLUX_TOPO=kary:1
+export TEST_UNDER_FLUX_QUORUM=1
+export TEST_UNDER_FLUX_START_MODE=leader
+
+test_under_flux 3 system
+
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+
+test_expect_success 'start rank 2 before TBON parent is up' '
+	$startctl run 2
+'
+
+test_expect_success 'startctl shows rank 1 pid as -1' '
+        test $($startctl status | jq -r ".procs[1].pid") = "-1"
+'
+
+test_expect_success 'flux run on rank 2 fails with offline error' '
+	test_must_fail bash -c \
+		"FLUX_URI=$(echo $FLUX_URI | sed s/local-0/local-2/) \
+			flux run hostname" 2>online.err &&
+	grep "broker is offline" online.err
+'
+
+test_expect_success 'flux uptime on rank 2 reports join state' '
+	bash -c \
+		"FLUX_URI=$(echo $FLUX_URI | sed s/local-0/local-2/) \
+			flux uptime" >uptime.out &&
+	grep join uptime.out
+'
+
+test_expect_success 'flux dmesg on rank 2 works' '
+	bash -c \
+		"FLUX_URI=$(echo $FLUX_URI | sed s/local-0/local-2/) \
+			flux dmesg" >/dev/null
+'
+
+test_done
diff --git a/t/t3303-system-healthcheck.t b/t/t3303-system-healthcheck.t
new file mode 100755
index 000000000000..4e1548dd57be
--- /dev/null
+++ b/t/t3303-system-healthcheck.t
@@ -0,0 +1,302 @@
+#!/bin/sh
+#
+
+test_description='Verify subtree health transitions and tools
+
+Ensure that the overlay subtree health status transitions
+appropriately as brokers are taken offline or lost, and also
+put flux overlay status tool and related RPCs and subcommands
+through their paces.
+'
+
+. `dirname $0`/sharness.sh
+
+export TEST_UNDER_FLUX_TOPO=kary:2
+
+test_under_flux 15 system
+
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+
+overlay_connected_children() {
+	rank=$1
+        flux python -c "import flux; print(flux.Flux().rpc(\"overlay.stats-get\",nodeid=${rank}).get_str())" | jq -r '.["child-connected"]'
+}
+
+# Usage: wait_connected rank count tries delay
+wait_connected() {
+	local rank=$1
+	local count=$2
+	local tries=$3
+	local delay=$4
+
+	while test $tries -gt 0; do
+		local n=$(overlay_connected_children $rank)
+		echo $n children
+		test $n -eq $count && return 0
+		sleep $delay
+		tries=$(($tries-1))
+	done
+	return 1
+}
+
+bad_topo_request() {
+        flux python -c "import flux; print(flux.Flux().rpc(\"overlay.topology\",nodeid=0).get_str())"
+}
+bad_topo_request_rank99() {
+        flux python -c "import flux; print(flux.Flux().rpc(\"overlay.topology\",{\"rank\":99},nodeid=0).get_str())"
+}
+
+test_expect_success 'overlay.topology RPC with no payload fails' '
+	test_must_fail bad_topo_request
+'
+test_expect_success 'overlay.topology RPC with bad rank fails' '
+	test_must_fail bad_topo_request_rank99
+'
+
+test_expect_success 'flux overlay status fails on bad rank' '
+	test_must_fail flux overlay status --timeout=0 --summary --rank 99
+'
+
+test_expect_success 'flux overlay fails on bad subcommand' '
+	test_must_fail flux overlay notcommand
+'
+
+test_expect_success 'overlay status is full' '
+	test "$(flux overlay status --timeout=0 --summary)" = "full"
+'
+
+test_expect_success 'flux overlay errors prints nothing' '
+	flux overlay errors --timeout=0 >errors.out &&
+	test $(wc -l <errors.out) -eq 0
+'
+
+test_expect_success 'wait timeout of zero is not an immediate timeout' '
+	flux overlay status --wait=full --summary --timeout=0
+'
+
+test_expect_success 'flux overlay status --highlight option works (color)' '
+	flux overlay status --highlight=14 --color=always > highlight.out &&
+	grep "\[01;34" highlight.out > highlighted.out &&
+	cat <<-EOF >highlighted.expected &&
+	0 fake0: full
+	└─ 2 fake2: full
+	   └─ 6 fake6: full
+	      └─ 14 fake14: full
+	EOF
+	test_cmp highlighted.expected highlighted.out
+'
+
+test_expect_success 'flux overlay status --highlight option takes hostlist' '
+	flux overlay status --highlight=fake[2,6] | grep "<<" > hl2.out &&
+	cat <<-EOF >hl2.expected &&
+	<<0 fake0>>: full
+	└─ <<2 fake2>>: full
+	   └─ <<6 fake6>>: full
+	EOF
+	test_cmp hl2.expected hl2.out
+'
+
+test_expect_success 'flux overlay status --highlight expected failures' '
+	test_must_fail flux overlay status --highlight=0-16 &&
+	test_must_fail flux overlay status --highlight=fake16
+'
+
+test_expect_success 'flux overlay status --color option works' '
+	test_must_fail flux overlay status -Lfoo &&
+	test_must_fail flux overlay status --color=foo &&
+	flux overlay status -Lnever --highlight=0 | grep "<<0" &&
+	flux overlay status --color=never --highlight=0 | grep "<<0" &&
+	flux overlay status -Lauto --highlight=0 | grep "<<0" &&
+	flux overlay status --color=auto --highlight=0 | grep "<<0" &&
+	flux overlay status -L --highlight=0 | grep "\[" &&
+	flux overlay status --color --highlight=0 | grep "\["
+'
+
+test_expect_success 'stop broker 3 with children 7,8' '
+	$startctl kill 3 15
+'
+
+test_expect_success 'wait for rank 0 overlay status to be partial' '
+	run_timeout 10 flux overlay status \
+		--timeout=0 --rank 0 --summary --wait=partial
+'
+
+# Just because rank 0 is partial doesn't mean rank 3 is offline yet
+# (shutdown starts at the leaves, and rank 3 will turn partial as
+# soon as one of its children goes offline)
+test_expect_success 'wait for rank 1 to lose connection with rank 3' '
+	wait_connected 1 1 10 0.2
+'
+
+test_expect_success 'flux overlay status -vv works' '
+	flux overlay status --timeout=0 -vv
+'
+
+test_expect_success 'flux overlay status shows rank 3 offline' '
+	echo "3 fake3: offline administrative shutdown" >health.exp &&
+	flux overlay status --timeout=0 --no-pretty \
+		| grep fake3 >health.out &&
+	test_cmp health.exp health.out
+'
+
+test_expect_success 'flux overlay errors shows rank 3' '
+	flux overlay errors --timeout=0 >errors2.out &&
+	grep "fake3: administrative shutdown" errors2.out
+'
+
+test_expect_success 'flux overlay status --summary' '
+	flux overlay status --timeout=0 --summary
+'
+
+test_expect_success 'flux overlay status --down' '
+	flux overlay status --timeout=0 --down
+'
+
+test_expect_success 'flux overlay status -vv' '
+	flux overlay status --timeout=0 -vv
+'
+
+test_expect_success 'flux overlay status: 0,1:partial, 3:offline' '
+	flux overlay status --timeout=0 --no-pretty  >health2.out &&
+	grep "0 fake0: partial" health2.out &&
+	grep "1 fake1: partial" health2.out &&
+	grep "3 fake3: offline" health2.out
+'
+
+test_expect_success 'flux overlay status: 0-1:partial, 3,7-8:offline' '
+	flux overlay status --timeout=0 --no-pretty >health3.out &&
+	grep "0 fake0: partial" health3.out &&
+	grep "1 fake1: partial" health3.out &&
+	grep "3 fake3: offline" health3.out &&
+	grep "7 fake7: offline" health3.out &&
+	grep "8 fake8: offline" health3.out
+'
+
+test_expect_success 'flux overlay status: 0,1:partial, 3,7-8:offline, rest:full' '
+	flux overlay status --timeout=0 --no-pretty >health4.out &&
+	grep "0 fake0: partial" health4.out &&
+	grep "1 fake1: partial" health4.out &&
+	grep "3 fake3: offline" health4.out &&
+	grep "7 fake7: offline" health4.out &&
+	grep "8 fake8: offline" health4.out &&
+	grep "4 fake4: full" health4.out &&
+	grep "9 fake9: full" health4.out &&
+	grep "10 fake10: full" health4.out &&
+	grep "2 fake2: full" health4.out &&
+	grep "5 fake5: full" health4.out &&
+	grep "11 fake11: full" health4.out &&
+	grep "6 fake6: full" health4.out &&
+	grep "13 fake13: full" health4.out &&
+	grep "14 fake14: full" health4.out
+'
+
+test_expect_success 'kill broker 14' '
+	$startctl kill 14 9
+'
+
+# Ensure an EHOSTUNREACH is encountered to trigger connected state change.
+test_expect_success 'ping to rank 14 fails with EHOSTUNREACH' '
+	echo "flux-ping: 14!broker.ping: $(strerror_symbol EHOSTUNREACH)" >ping.exp &&
+	test_must_fail flux ping 14 2>ping.err &&
+	test_cmp ping.exp ping.err
+'
+
+test_expect_success 'flux overlay errors shows the lost connection' '
+	flux overlay errors --timeout=0 >errors3.out &&
+	grep "fake14: lost connection" errors3.out
+'
+
+test_expect_success 'wait for rank 0 subtree to be degraded' '
+	run_timeout 10 flux overlay status --timeout=0 --summary --wait=degraded
+'
+
+test_expect_success 'wait for unknown status fails' '
+	test_must_fail flux overlay status --timeout=0 --wait=foo
+'
+
+test_expect_success 'wait timeout works' '
+	test_must_fail flux overlay status --wait=full --summary --timeout=0.1s
+'
+
+test_expect_success 'flux overlay status -vv' '
+	flux overlay status --timeout=0 -vv
+'
+test_expect_success 'flux overlay status -v' '
+	flux overlay status --timeout=0 -v
+'
+
+test_expect_success 'flux overlay lookup with no target fails' '
+	test_must_fail flux overlay lookup
+'
+
+test_expect_success 'flux overlay lookup 0 works' '
+	echo fake0 >host.0.exp &&
+	flux overlay lookup 0 >host.0.out &&
+	test_cmp host.0.exp host.0.out
+'
+
+test_expect_success 'flux overlay lookup 0-14 works' '
+	echo "fake[0-14]" >host.0-14.exp &&
+	flux overlay lookup 0-14 >host.0-14.out &&
+	test_cmp host.0-14.exp host.0-14.out
+'
+
+test_expect_success 'flux overlay lookup fails on invalid idset target' '
+	test_must_fail flux overlay lookup -- -1
+'
+
+test_expect_success 'flux overlay lookup fails on too big rank target' '
+	test_must_fail flux overlay lookup 100
+'
+
+test_expect_success 'flux overlay lookup works on single host target' '
+	echo 2 >idset.2.exp &&
+	flux overlay lookup fake2 >idset.2.out &&
+	test_cmp idset.2.exp idset.2.out
+'
+test_expect_success 'flux overlay lookup works on multi-host target' '
+	echo "2-3" >idset.23.exp &&
+	flux overlay lookup "fake[2-3]" >idset.23.out &&
+	test_cmp idset.23.exp idset.23.out
+'
+
+test_expect_success 'flux overlay lookup fails on invalid hostlist target' '
+	test_must_fail flux overlay lookup "fake2["
+'
+test_expect_success 'flux overlay lookup fails on unknown host target' '
+	test_must_fail flux overlay lookup foo
+'
+
+test_expect_success 'flux overlay disconnect fails on unknown rank target' '
+	test_must_fail flux overlay disconnect 42 2>discon.err &&
+	grep "is not a valid rank in this instance" discon.err
+'
+
+test_expect_success 'flux overlay disconnect interprets rank w/ extra as host' '
+	test_must_fail flux overlay disconnect 42xxx 2>disconn2.err &&
+	grep "TARGET must be a valid rank or hostname" disconn2.err
+'
+
+test_expect_success 'flux overlay disconnect fails on bad input' '
+	test_must_fail flux overlay disconnect "fake2[" 2>disconn3.err &&
+	grep "TARGET must be a valid rank or hostname" disconn3.err
+'
+
+test_expect_success 'stop broker 12' '
+	$startctl kill 12 19
+'
+
+test_expect_success 'flux overlay status prints connection timed out on 12' '
+	flux overlay status -vv --no-pretty >status.out &&
+	grep "fake12: $(strerror_symbol ETIMEDOUT)" status.out
+'
+test_expect_success 'flux overlay errors prints connection timed out on 12' '
+	flux overlay errors >errors4.out &&
+	grep "fake12: $(strerror_symbol ETIMEDOUT)" errors4.out
+'
+
+test_expect_success 'continue broker 12' '
+	$startctl kill 12 18
+'
+
+test_done
diff --git a/t/t3304-system-rpctrack-down.t b/t/t3304-system-rpctrack-down.t
new file mode 100755
index 000000000000..7cbe70b199f7
--- /dev/null
+++ b/t/t3304-system-rpctrack-down.t
@@ -0,0 +1,111 @@
+#!/bin/sh
+#
+
+test_description='Test downstream RPC tracking
+
+Test that RPCs from rank 0 broker to other ranks receive
+broker-generated EHOSTUNREACH responses when the next hop in
+the RPC goes down, either nicely, or a hard crash.
+'
+
+. `dirname $0`/sharness.sh
+
+export TEST_UNDER_FLUX_TOPO=kary:1
+
+test_under_flux 3 system
+
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+
+# The health check with wait option is just a convenient RPC that
+# can be made to block indefinitely by requesting a state not expected
+# to be entered.
+test_expect_success NO_CHAIN_LINT 'start background RPC to rank 2' '
+	flux overlay status --timeout=0 --wait=lost --rank=2 2>health.err &
+	echo $! >health.pid
+'
+
+# This ensures the blocking check was received on rank 2 so that
+# we don't shut down rank 2 broker before zeromq has routes the request.
+test_expect_success 'ensure background request was received on rank 2' '
+	flux overlay status --timeout=0 --rank=2
+'
+
+test_expect_success NO_CHAIN_LINT 'broker 1 tracked child rpc count is nonzero' '
+	count=$(flux exec -r 1 flux module stats --parse=child-rpc overlay) &&
+	echo $count &&
+	test $count -gt 0
+'
+
+test_expect_success 'stop broker 2 nicely and wait for it to exit' '
+	$startctl kill 2 15 &&
+	$startctl wait 2
+'
+
+test_expect_success NO_CHAIN_LINT 'background RPC fails with EHOSTUNREACH (tracker response from rank 1)' '
+	pid=$(cat health.pid) &&
+	echo waiting for pid $pid &&
+	test_expect_code 1 wait $pid &&
+	grep "$(strerror_symbol EHOSTUNREACH)" health.err
+'
+
+test_expect_success 'new RPC to rank 2 fails with EHOSTUNREACH' '
+	test_expect_code 1 flux overlay status \
+		--timeout=0 --rank=2 2>health2.err &&
+	grep "$(strerror_symbol EHOSTUNREACH)" health2.err
+'
+
+test_expect_success 'broker 1 tracked child rpc count is zero' '
+	count=$(flux exec -r 1 flux module stats --parse=child-rpc overlay) &&
+	echo $count &&
+	test $count -eq 0
+'
+
+test_expect_success NO_CHAIN_LINT 'start background RPC to rank 1' '
+	flux overlay status --timeout=0 --wait=lost --rank=1 2>health3.err &
+	echo $! >health3.pid
+'
+
+test_expect_success 'ensure background request was received on rank 1' '
+	flux overlay status --timeout=0 --rank=1
+'
+
+test_expect_success NO_CHAIN_LINT 'broker 0 tracked child rpc count is nonzero' '
+	count=$(flux module stats --parse=child-rpc overlay) &&
+	echo $count &&
+	test $count -gt 0
+'
+
+test_expect_success 'stop broker 1 hard and wait for it to exit' '
+	$startctl kill 1 9 &&
+	($startctl wait 1 || true)
+'
+
+# Ensure an EHOSTUNREACH is encountered on the socket to trigger connected
+# state change.  Note: existing traffic like heartbeat probably also serve
+# this purpose, but do it anyway to ensure repeatability.
+test_expect_success 'ping to rank 1 fails with EHOSTUNREACH' '
+	echo "flux-ping: 1!broker.ping: $(strerror_symbol EHOSTUNREACH)" >ping.exp &&
+	test_must_fail flux ping 1 2>ping.err &&
+	test_cmp ping.exp ping.err
+'
+
+test_expect_success NO_CHAIN_LINT 'background RPC fails with EHOSTUNREACH (tracker response from rank 0)' '
+	pid=$(cat health3.pid) &&
+	echo waiting for pid $pid &&
+	test_expect_code 1 wait $pid &&
+	grep "$(strerror_symbol EHOSTUNREACH)" health3.err
+'
+
+test_expect_success 'new RPC to rank 1 fails with EHOSTUNREACH' '
+	test_expect_code 1 flux overlay status \
+		--timeout=0 --rank=1 2>health4.err &&
+	grep "$(strerror_symbol EHOSTUNREACH)" health4.err
+'
+
+test_expect_success 'broker 0 tracked child rpc count is zero' '
+	count=$(flux module stats --parse=child-rpc overlay) &&
+	echo $count &&
+	test $count -eq 0
+'
+
+test_done
diff --git a/t/t3305-system-rpctrack-up.t b/t/t3305-system-rpctrack-up.t
new file mode 100755
index 000000000000..f0d03eedb508
--- /dev/null
+++ b/t/t3305-system-rpctrack-up.t
@@ -0,0 +1,109 @@
+#!/bin/sh
+#
+
+test_description='Test upstream RPC tracking and disconnect command
+
+Test that RPCs from a leaf broker to rank 0 receive broker-generated
+EHOSTUNREACH responses when a broker along the path to rank 0 is
+taken offline administratively, or if the subtree panics.
+
+Also put the flux overlay disconnect comamnd/RPC and related
+commands/RPCs through their paces.
+'
+
+. `dirname $0`/sharness.sh
+
+export TEST_UNDER_FLUX_TOPO=kary:2
+
+test_under_flux 15 system
+
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+
+test_expect_success 'tell each broker to log to stderr' '
+	flux exec flux setattr log-stderr-mode local
+'
+test_expect_success 'flux overlay parentof fails with missing RANK' '
+	test_must_fail flux overlay parentof
+'
+test_expect_success 'flux overlay parentof fails with rank out of range' '
+	test_must_fail flux overlay parentof 42
+'
+test_expect_success 'flux overlay parentof fails with rank 0' '
+	test_must_fail flux overlay parentof 0
+'
+test_expect_success 'flux overlay parentof relects k=2 topology' '
+	test $(flux overlay parentof 1) -eq 0 &&
+	test $(flux overlay parentof 2) -eq 0 &&
+	test $(flux overlay parentof 3) -eq 1 &&
+	test $(flux overlay parentof 4) -eq 1 &&
+	test $(flux overlay parentof 5) -eq 2 &&
+	test $(flux overlay parentof 6) -eq 2 &&
+	test $(flux overlay parentof 7) -eq 3 &&
+	test $(flux overlay parentof 8) -eq 3 &&
+	test $(flux overlay parentof 9) -eq 4 &&
+	test $(flux overlay parentof 10) -eq 4 &&
+	test $(flux overlay parentof 11) -eq 5 &&
+	test $(flux overlay parentof 12) -eq 5 &&
+	test $(flux overlay parentof 13) -eq 6 &&
+	test $(flux overlay parentof 14) -eq 6
+'
+
+test_expect_success 'flux overlay disconnect fails with missing target' '
+	test_must_fail flux overlay disconnect
+'
+
+test_expect_success 'flux overlay disconnect fails on incorrect parent' '
+	test_must_fail flux overlay disconnect --parent 0 14
+'
+
+test_expect_success 'construct FLUX_URI for rank 13 (child of 6)' '
+	echo "local://$(flux getattr rundir)/local-13" >uri13 &&
+	test $(FLUX_URI=$(cat uri13) flux getattr rank) -eq 13
+'
+
+test_expect_success NO_CHAIN_LINT 'start background RPC to rank 0 via 13' '
+	(FLUX_URI=$(cat uri13) flux overlay status --timeout=0 --wait=lost) &
+	echo $! >health.pid
+'
+test_expect_success 'ensure background request was received on rank 0' '
+        (FLUX_URI=$(cat uri13) flux overlay status --timeout=0)
+'
+
+test_expect_success 'disconnect host fake6 (rank 6)' '
+	flux overlay disconnect fake6
+'
+test_expect_success 'rank 6 exited with rc=1' '
+	test_expect_code 1 $startctl wait 6
+'
+test_expect_success 'rank 13 exited' '
+	test_might_fail $startctl wait 13
+'
+test_expect_success 'rank 14 exited' '
+	test_might_fail $startctl wait 14
+'
+
+# The important thing is that the RPC fails.
+# It may fail with EHOSTUNREACH "overlay disconnect" (tracker from rank 6).
+# But this response may race with the chain of broker exits,
+# so other errors are possible such as ECONNRESET.
+test_expect_success NO_CHAIN_LINT 'background RPC fails' '
+        pid=$(cat health.pid) &&
+        echo waiting for pid $pid &&
+        test_expect_code 1 wait $pid
+'
+
+test_expect_success 'report health status' '
+	flux overlay status --timeout=0
+'
+test_expect_success 'health status for rank 6 is lost' '
+	echo "6 fake6: lost administrative disconnect" >status.exp &&
+	flux overlay status --timeout=0 --down --no-pretty \
+		| grep fake6 >status.out &&
+	test_cmp status.exp status.out
+'
+
+test_expect_success 'attempt to disconnect rank 6 again fails' '
+	test_must_fail flux overlay disconnect 6
+'
+
+test_done
diff --git a/t/t3306-system-routercrash.t b/t/t3306-system-routercrash.t
new file mode 100755
index 000000000000..8b9506f7bbcc
--- /dev/null
+++ b/t/t3306-system-routercrash.t
@@ -0,0 +1,130 @@
+#!/bin/sh
+#
+
+test_description='Hard fail a router node and verify child restarts
+
+When a router node crashes without a chance to notify its children
+and then starts back up, the children will reconnect and try to resume
+communications.  From the router perspective, the child uuid is
+unknown, so it forces a subtree panic in the child.  The maintains the
+invariant that when a router restarts, its children always restart too.
+This test verifies that the invariant holds when the router crashes.
+'
+
+. `dirname $0`/sharness.sh
+
+export TEST_UNDER_FLUX_TOPO=kary:1
+
+# A test below intentionally sends SIGSEGV to a broker process --
+# avoid an unnecessary corefile by ensuring the corefile limit is 0
+# before calling test_under_flux():
+ulimit -c 0
+
+test_under_flux 3 system
+
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+groups="flux python ${SHARNESS_TEST_SRCDIR}/scripts/groups.py"
+
+# Usage: waitup N
+#   where N is a count of online ranks
+waitup () {
+        run_timeout 5 flux python -c "import flux; print(flux.Flux().rpc(\"resource.monitor-waitup\",{\"up\":$1}).get())"
+}
+
+test_expect_success 'wait for all brokers to be online' '
+	${groups} waitfor --count 3 broker.online
+'
+
+test_expect_success 'wait for resource module to catch up' '
+	waitup 3
+'
+
+test_expect_success 'resource status shows no offline nodes' '
+        echo 0 >offline0.exp &&
+        flux resource status -s offline -no {nnodes} >offline0.out &&
+        test_cmp offline0.exp offline0.out
+'
+
+test_expect_success 'tell each broker to log to stderr' '
+	flux exec flux setattr log-stderr-mode local
+'
+
+test_expect_success 'construct FLUX_URI for rank 2 (child of 1)' '
+	echo "local://$(flux getattr rundir)/local-2" >uri2 &&
+	test $(FLUX_URI=$(cat uri2) flux getattr rank) -eq 2
+'
+
+# Choice of signal 11 (SIGSEGV) is deliberate here.
+# The broker should not trap this siganl - see flux-framework/flux-core#4230.
+test_expect_success 'kill -11 broker 1' '
+	$startctl kill 1 11 &&
+	test_expect_code 139 $startctl wait 1
+'
+
+test_expect_success 'wait broker.online to reach count of one' '
+	${groups} waitfor --count 1 broker.online
+'
+test_expect_success 'wait for resource module to catch up' '
+	waitup 1
+'
+test_expect_success 'resource status shows 2 offline nodes' '
+        echo 2 >offline2.exp &&
+        flux resource status -s offline -no {nnodes} >offline2.out &&
+        test_cmp offline2.exp offline2.out
+'
+
+test_expect_success 'restart broker 1' '
+	$startctl run 1
+'
+
+test_expect_success 'wait broker.online to reach count of two' '
+	${groups} waitfor --count=2 broker.online
+'
+test_expect_success 'wait for resource module to catch up' '
+	waitup 2
+'
+test_expect_success 'resource status shows 1 offline nodes' '
+        echo 1 >offline1.exp &&
+        flux resource status -s offline -no {nnodes} >offline1.out &&
+        test_cmp offline1.exp offline1.out
+'
+
+test_expect_success 'ping broker 1 via broker 2' '
+	(FLUX_URI=$(cat uri2) test_must_fail flux ping --count=1 1)
+'
+
+test_expect_success 'broker 2 was disconnected' '
+	test_might_fail $startctl wait 2
+'
+
+test_expect_success 'restart broker 2' '
+	$startctl run 2
+'
+
+test_expect_success 'wait broker.online to reach count of three' '
+	${groups} waitfor --count=3 broker.online
+'
+test_expect_success 'wait for resource module to catch up' '
+	waitup 3
+'
+test_expect_success 'resource status shows 0 offline nodes' '
+        flux resource status -s offline -no {nnodes} >offline00.out &&
+        test_cmp offline0.exp offline00.out
+'
+
+test_expect_success 'wait for rank 0 to report subtree status of full' '
+	run_timeout 10 flux overlay status --timeout=0 --wait full
+'
+
+test_expect_success 'ping broker 2 via broker 0' '
+	flux ping --count=1 2
+'
+
+# Side effect: let rc1 on rank 2 finish loading resource module
+# before shutdown begins, or it will complain
+test_expect_success 'run a 3 node job' '
+        flux run -n3 -N3 true
+'
+
+
+test_done
diff --git a/t/t3307-system-leafcrash.t b/t/t3307-system-leafcrash.t
new file mode 100755
index 000000000000..657bda23793c
--- /dev/null
+++ b/t/t3307-system-leafcrash.t
@@ -0,0 +1,63 @@
+#!/bin/sh
+#
+
+test_description='Hard fail a leaf node and verify transition thru lost
+
+When a leaf node crashes without a chance to notify its parent and then
+starts back up, it will try to say hello to its parent as though it were
+starting anew.  From the parent perspective, the child may be already
+online with a different uuid, so upon receipt of the hello request, it
+transitions the child subtree status through lost to fail any pending
+RPCs, then proceeds to handle the hello.  This test verifies that the
+transition through lost occurs.
+'
+
+. `dirname $0`/sharness.sh
+
+#
+#  With --chain-lint, most tests below are skipped and this can cause
+#  the final test to hang. Therefore just skip all tests with chain-lint.
+#
+if ! test_have_prereq NO_CHAIN_LINT; then
+	skip_all='test may hang with --chain-lint, skipping all.'
+	test_done
+fi
+
+test_under_flux 2 system
+
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+
+test_expect_success 'tell brokers to log to stderr' '
+	flux exec flux setattr log-stderr-mode local
+'
+
+# Degraded at parent means child was lost
+test_expect_success 'start overlay status wait in the background' '
+	flux overlay status --timeout=0 --wait degraded &
+	echo $! >subtree.pid
+'
+
+test_expect_success 'kill -9 broker 1' '
+	$startctl kill 1 9 &&
+	test_expect_code 137 $startctl wait 1
+'
+
+test_expect_success 'restart broker 1' '
+	$startctl run 1
+'
+
+test_expect_success 'ensure child was lost' '
+	wait $(cat subtree.pid)
+'
+
+test_expect_success 'and child returned to service' '
+	flux overlay status --timeout=0 --wait full
+'
+
+# Side effect: let rc1 on rank 1 finish loading resource module
+# before shutdown begins, or it will complain
+test_expect_success 'run a 2 node job' '
+	flux run -n2 -N2 true
+'
+
+test_done
diff --git a/t/t3308-system-torpid.t b/t/t3308-system-torpid.t
new file mode 100755
index 000000000000..ac94c23e4e6a
--- /dev/null
+++ b/t/t3308-system-torpid.t
@@ -0,0 +1,130 @@
+#!/bin/sh
+#
+
+test_description='Check torpid broker detection
+'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 2 system
+
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+groups="flux python ${SHARNESS_TEST_SRCDIR}/scripts/groups.py"
+
+test_expect_success 'tell brokers to log to stderr' '
+	flux exec flux setattr log-stderr-mode local
+'
+test_expect_success 'load heartbeat module with fast rate for testing' '
+        flux module reload heartbeat period=0.5s
+'
+
+test_expect_success 'tbon.torpid max/min have expected values' '
+	TMIN=$(flux getattr tbon.torpid_min) &&
+	TMAX=$(flux getattr tbon.torpid_max) &&
+	test "$TMAX" = "30s" &&
+	test "$TMIN" = "5s"
+'
+
+test_expect_success 'tbon.torpid min/max can be set on live system' '
+	flux setattr tbon.torpid_max 60s &&
+	TMAX=$(flux getattr tbon.torpid_max) &&
+	test "$TMAX" = "1m" &&
+	flux setattr tbon.torpid_min 30s &&
+	TMIN=$(flux getattr tbon.torpid_min) &&
+	test "$TMIN" = "30s"
+'
+
+test_expect_success 'tbon.torpid min/max cannot be set to non-FSD value' '
+	test_must_fail flux setattr tbon.torpid_min foo &&
+	test_must_fail flux setattr tbon.torpid_max bar
+'
+
+test_expect_success 'tbon.torpid_min cannot be set to zero' '
+	test_must_fail flux setattr tbon.torpid_min 0
+'
+
+test_expect_success 'torpid_min can be set via config' '
+	mkdir -p conf.d &&
+	cat >conf.d/tbon.toml <<-EOT &&
+	tbon.torpid_min = "6s"
+	EOT
+	TMIN=$(FLUX_CONF_DIR=conf.d flux start flux getattr tbon.torpid_min) &&
+	test "$TMIN" = "6s"
+'
+
+test_expect_success 'torpid_min cannot be set to 0 via config' '
+	mkdir -p conf.d &&
+	cat >conf.d/tbon.toml <<-EOT &&
+	tbon.torpid_min = "0"
+	EOT
+	test_must_fail bash -c "FLUX_CONF_DIR=conf.d flux start \
+		flux getattr tbon.torpid_min"
+'
+
+test_expect_success 'torpid_max can be set to 0 via config' '
+	mkdir -p conf.d &&
+	cat >conf.d/tbon.toml <<-EOT &&
+	tbon.torpid_max = "0"
+	EOT
+	TMAX=$(FLUX_CONF_DIR=conf.d flux start flux getattr tbon.torpid_max) &&
+	test "$TMAX" = "0s"
+'
+
+# tbon.torpid_min should be >= sync_min (1s hardwired)
+test_expect_success 'reduce tbon.torpid max/min values for testing' '
+	flux exec flux setattr tbon.torpid_min 1s &&
+	flux exec flux setattr tbon.torpid_max 2s
+'
+
+test_expect_success 'kill -STOP broker 1' '
+	$startctl kill 1 19
+'
+
+test_expect_success 'rank 1 is added to broker.torpid group' '
+	$groups waitfor --count=1 broker.torpid
+'
+
+test_expect_success 'kill -CONT broker 1' '
+	$startctl kill 1 18
+'
+
+test_expect_success 'rank 1 is removed from broker.torpid group' '
+	$groups waitfor --count=0 broker.torpid
+'
+
+test_expect_success 'set tbon.torpid_max to impossible to attain value' '
+	flux setattr tbon.torpid_max 0.1s
+'
+
+test_expect_success 'rank 1 is added to broker.torpid group' '
+	$groups waitfor --count=1 broker.torpid
+'
+
+test_expect_success 'flux resource list shows one node down' '
+	test $(flux resource list -s down -no {nnodes}) -eq 1
+'
+test_expect_success 'scheduler shows one node down' '
+	test $(FLUX_RESOURCE_LIST_RPC=sched.resource-status flux resource list -s down -no {nnodes}) -eq 1
+'
+
+test_expect_success 'set tbon.torpid_max to zero to disable' '
+	flux setattr tbon.torpid_max 0
+'
+
+test_expect_success 'rank 1 is removed from broker.torpid group' '
+	$groups waitfor --count=0 broker.torpid
+'
+
+test_expect_success 'flux resource list shows no nodes down' '
+	test $(flux resource list -s down -no {nnodes}) -eq 0
+'
+
+test_expect_success 'scheduler shows no nodes down' '
+	test $(FLUX_RESOURCE_LIST_RPC=sched.resource-status flux resource list -s down -no {nnodes}) -eq 0
+'
+
+test_expect_success 'no nodes are drained' '
+	test $(flux resource status -s drain -no {nnodes}) -eq 0
+'
+
+test_done
diff --git a/t/t3309-system-reconnect.t b/t/t3309-system-reconnect.t
new file mode 100755
index 000000000000..4bfb704d66f4
--- /dev/null
+++ b/t/t3309-system-reconnect.t
@@ -0,0 +1,60 @@
+#!/bin/sh
+#
+# ci=system
+
+test_description='test handle reconnection
+'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 2 system
+
+startctl="flux python ${SHARNESS_TEST_SRCDIR}/scripts/startctl.py"
+
+# Usage: geturi rank
+geturi () {
+	local rank=$1
+	echo $FLUX_URI | sed -e "s/local-0/local-$rank/"
+}
+
+test_expect_success 'tell brokers to log to stderr' '
+	flux exec flux setattr log-stderr-mode local
+'
+
+test_expect_success 'flux-proxy works on rank 1 broker' '
+	flux proxy $(geturi 1) flux getattr instance-level
+'
+
+test_expect_success 'flux-proxy --reconnect survives broker restart' '
+	cat >test1.sh <<-EOT &&
+	#!/bin/sh -e
+	unset FLUX_HANDLE_TRACE
+	flux getattr instance-level
+	$startctl kill 1 15
+	$startctl wait 1
+	$startctl run 1
+	flux getattr instance-level
+	EOT
+	chmod +x test1.sh &&
+	run_timeout 60 \
+		sh -c "FLUX_HANDLE_TRACE=1 flux proxy --reconnect $(geturi 1) ./test1.sh"
+'
+
+test_expect_success 'handle purges RPCs pending across broker restart' '
+	cat >test2.sh <<-EOT &&
+	#!/bin/sh -e
+	unset FLUX_HANDLE_TRACE
+	flux kvs get --waitcreate notakey 2>kvsget.err &
+	$startctl kill 1 15
+	$startctl wait 1
+	$startctl run 1
+	wait
+	grep "Connection reset by peer" kvsget.err
+	flux getattr instance-level
+	EOT
+	chmod +x test2.sh &&
+	run_timeout 60 \
+		sh -c "FLUX_HANDLE_TRACE=1 flux proxy --reconnect $(geturi 1) ./test2.sh"
+'
+
+test_done
diff --git a/t/t3400-overlay-trace.t b/t/t3400-overlay-trace.t
new file mode 100755
index 000000000000..42d7157f878c
--- /dev/null
+++ b/t/t3400-overlay-trace.t
@@ -0,0 +1,66 @@
+#!/bin/sh
+
+test_description='Test flux overlay trace'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 4
+
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+
+test_expect_success 'flux overlay trace fails with extra positional argument' '
+	test_must_fail flux overlay trace x y
+'
+test_expect_success 'flux overlay trace fails with unknown argument' '
+	test_must_fail flux overlay trace --not-an-arg
+'
+test_expect_success 'flux overlay trace fails with wrong message type' '
+	test_must_fail flux overlay trace -t foo,bar
+'
+test_expect_success 'reload heartbeat with increased heart rate' '
+	flux module reload heartbeat period=0.2s
+'
+test_expect_success NO_CHAIN_LINT 'start background trace' '
+	flux overlay trace >trace.out &
+	echo $! >trace.pid
+'
+test_expect_success NO_CHAIN_LINT 'start second background trace with --full' '
+	flux overlay trace --full >trace2.out &
+	echo $! >trace2.pid
+'
+test_expect_success NO_CHAIN_LINT 'heartbeat.pulse event was captured' '
+	$waitfile -t 60 -p heartbeat.pulse trace.out
+'
+test_expect_success NO_CHAIN_LINT 'heartbeat.pulse event was captured with --full' '
+	$waitfile -t 60 -p heartbeat.pulse trace2.out
+'
+test_expect_success NO_CHAIN_LINT 'send one kvs.ping to rank 1' '
+	flux ping -r 1 -c 1 kvs
+'
+test_expect_success NO_CHAIN_LINT 'kvs.ping request/response was captured' '
+	$waitfile -t 60 -c 2 -p kvs.ping trace.out
+'
+test_expect_success NO_CHAIN_LINT 'kvs.ping request/response was captured with --full' '
+	$waitfile -t 60 -c 2 -p kvs.ping trace2.out
+'
+# This RPC happens to return a human readable error on failure
+test_expect_success NO_CHAIN_LINT 'send one job-manager.kill (failing)' '
+	test_must_fail flux exec -r 1 flux job kill fuzzybunny
+'
+test_expect_success NO_CHAIN_LINT 'job-manager.kill request/response was captured' '
+	$waitfile -t 60 -c 2 -p "job%-manager.kill" trace.out
+'
+test_expect_success NO_CHAIN_LINT 'job-manager.kill request/response was captured with --full' '
+	$waitfile -t 60 -c 2 -p "job%-manager.kill" trace2.out
+'
+test_expect_success NO_CHAIN_LINT 'stop background trace' '
+	pid=$(cat trace.pid) &&
+	kill -15 $pid &&
+	wait $pid || true
+'
+test_expect_success NO_CHAIN_LINT 'stop second background trace' '
+	pid=$(cat trace2.pid) &&
+	kill -15 $pid &&
+	wait $pid || true
+'
+test_done
diff --git a/t/t3401-module-trace.t b/t/t3401-module-trace.t
new file mode 100755
index 000000000000..b88ecc35f71e
--- /dev/null
+++ b/t/t3401-module-trace.t
@@ -0,0 +1,84 @@
+#!/bin/sh
+
+test_description='Test flux module trace'
+
+. $(dirname $0)/sharness.sh
+
+test_under_flux 1
+
+waitfile="${SHARNESS_TEST_SRCDIR}/scripts/waitfile.lua"
+
+test_expect_success 'flux module trace fails with missing module name' '
+	test_must_fail flux module trace
+'
+test_expect_success 'flux module trace fails with unknown argument' '
+	test_must_fail flux module trace --not-an-arg
+'
+test_expect_success 'flux module trace fails with wrong message type' '
+	test_must_fail flux module trace -t foo,bar
+'
+test_expect_success 'reload heartbeat with increased heart rate' '
+	flux module reload heartbeat period=0.2s
+'
+test_expect_success NO_CHAIN_LINT 'start background trace' '
+	flux module trace kvs job-manager >trace.out &
+	echo $! >trace.pid
+'
+test_expect_success NO_CHAIN_LINT 'start second background trace with --full' '
+	flux module trace --full kvs job-manager >trace1a.out &
+	echo $! >trace1a.pid
+'
+test_expect_success NO_CHAIN_LINT 'heartbeat.pulse event was captured' '
+	$waitfile -t 60 -p heartbeat.pulse trace.out
+'
+test_expect_success NO_CHAIN_LINT 'heartbeat.pulse event was captured with --full' '
+	$waitfile -t 60 -p heartbeat.pulse trace1a.out
+'
+test_expect_success NO_CHAIN_LINT 'send one kvs.ping' '
+	flux ping -c 1 kvs
+'
+test_expect_success NO_CHAIN_LINT 'kvs.ping request/response was captured' '
+	$waitfile -t 60 -c 2 -p kvs.ping trace.out
+'
+test_expect_success NO_CHAIN_LINT 'kvs.ping request/response was captured with --full' '
+	$waitfile -t 60 -c 2 -p kvs.ping trace1a.out
+'
+# This RPC happens to return a human readable error on failure
+test_expect_success NO_CHAIN_LINT 'send one job-manager.kill (failing)' '
+	test_must_fail flux job kill fuzzybunny
+'
+test_expect_success NO_CHAIN_LINT 'job-manager.kill request/response was captured' '
+	$waitfile -t 60 -c 2 -p "job%-manager.kill" trace.out
+'
+test_expect_success NO_CHAIN_LINT 'job-manager.kill request/response was captured with --full' '
+	$waitfile -t 60 -c 2 -p "job%-manager.kill" trace1a.out
+'
+
+test_expect_success NO_CHAIN_LINT 'stop background trace' '
+	pid=$(cat trace.pid) &&
+	kill -15 $pid &&
+	wait $pid || true
+'
+test_expect_success NO_CHAIN_LINT 'stop second background trace' '
+	pid=$(cat trace1a.pid) &&
+	kill -15 $pid &&
+	wait $pid || true
+'
+
+test_expect_success NO_CHAIN_LINT 'start background trace on multiple modules' '
+	flux module trace kvs barrier >trace2.out &
+	echo $! >trace2.pid
+'
+test_expect_success NO_CHAIN_LINT 'heartbeat.pulse event was captured' '
+	$waitfile -t 60 -p heartbeat.pulse trace2.out
+'
+test_expect_success NO_CHAIN_LINT 'send one barrier.ping' '
+	flux ping -c 1 barrier
+'
+test_expect_success NO_CHAIN_LINT 'barrier.ping request/response was captured' '
+	$waitfile -t 60 -c 2 -p barrier.ping trace2.out
+'
+test_expect_success NO_CHAIN_LINT 'stop background trace' '
+	kill -15 $(cat trace2.pid); wait || true
+'
+test_done
diff --git a/t/t4000-issues-test-driver.t b/t/t4000-issues-test-driver.t
index c86fb83c296e..ad2ff5ebe8fc 100755
--- a/t/t4000-issues-test-driver.t
+++ b/t/t4000-issues-test-driver.t
@@ -5,13 +5,27 @@ test_description='Verify that fixed issues remain fixed'
 
 . `dirname $0`/sharness.sh
 
-SIZE=4
+if test_have_prereq ASAN; then
+    skip_all='skipping issues tests under AddressSanitizer'
+    test_done
+fi
+
+SIZE=2
 test_under_flux ${SIZE}
 echo "# $0: flux session size will be ${SIZE}"
 
-for testscript in ${FLUX_SOURCE_DIR}/t/issues/*; do
-    testname=`basename $testscript`
-    test_expect_success  $testname $testscript
+if test -z "$T4000_ISSUES_GLOB"; then
+    T4000_ISSUES_GLOB="*"
+fi
+
+flux bulksubmit -n1 -o pty --job-name={./%} -t 10m \
+	--flags=waitable \
+	--quiet --watch  \
+	flux start {} \
+	::: ${FLUX_SOURCE_DIR}/t/issues/${T4000_ISSUES_GLOB}
+
+for id in $(flux jobs -ano {id}); do
+    test_expect_success $(flux jobs -no {name} $id) "flux job attach $id"
 done
 
 test_done
diff --git a/t/t5000-valgrind.t b/t/t5000-valgrind.t
index 66bc28756eca..e720d2a41037 100755
--- a/t/t5000-valgrind.t
+++ b/t/t5000-valgrind.t
@@ -6,6 +6,13 @@ test_description='Run broker under valgrind with a small workload'
 test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
 . `dirname $0`/sharness.sh
 
+#  Do not run valgrind test by default unless FLUX_ENABLE_VALGRIND_TEST
+#   is set in environment (e.g. by CI), or the test run run with -d, --debug
+#
+if test -z "$FLUX_ENABLE_VALGRIND_TEST" && test "$debug" = ""; then
+    skip_all='skipping valgrind tests since FLUX_ENABLE_VALGRIND_TEST not set'
+    test_done
+fi
 if ! which valgrind >/dev/null; then
     skip_all='skipping valgrind tests since no valgrind executable found'
     test_done
@@ -27,20 +34,24 @@ if ! have_valgrind_h && test "$debug" = ""; then
     test_done
 fi
 
-export FLUX_PMI_SINGLETON=1 # avoid finding leaks in slurm libpmi.so
-
 VALGRIND=`which valgrind`
 VALGRIND_SUPPRESSIONS=${SHARNESS_TEST_SRCDIR}/valgrind/valgrind.supp
 VALGRIND_WORKLOAD=${SHARNESS_TEST_SRCDIR}/valgrind/valgrind-workload.sh
-BROKER=${FLUX_BUILD_DIR}/src/broker/flux-broker
 
 VALGRIND_NBROKERS=${VALGRIND_NBROKERS:-2}
 
+# Allow jobs to run under sdexec in test, if available
+cat >valgrind.toml <<-EOT
+[exec]
+service-override = true
+EOT
+
 test_expect_success \
   "valgrind reports no new errors on $VALGRIND_NBROKERS broker run" '
-	run_timeout 120 \
+	run_timeout 300 \
 	flux start -s ${VALGRIND_NBROKERS} \
-		--killer-timeout=120 \
+		--test-exit-timeout=120 \
+		--config-path=valgrind.toml \
 		--wrap=libtool,e,${VALGRIND} \
 		--wrap=--tool=memcheck \
 		--wrap=--leak-check=full \
diff --git a/t/t9000-system.t b/t/t9000-system.t
new file mode 100755
index 000000000000..f81168d373c5
--- /dev/null
+++ b/t/t9000-system.t
@@ -0,0 +1,73 @@
+#!/bin/sh
+# ci=system
+
+test_description='Run tests against a system instance of Flux'
+
+#  If FLUX_TEST_INSTALLED_PATH is not set and /usr/bin/flux exists,
+#  set FLUX_TEST_INSTALLED_PATH to /usr/bin.
+#
+#  Must set FLUX_TEST_INSTALLED_PATH before sourcing sharness,
+#  otherwise correct flux may not be used in tests.
+if test -n "$FLUX_ENABLE_SYSTEM_TESTS"; then
+	if test -x ${FLUX_TEST_INSTALLED_PATH:-/usr/bin}/flux; then
+		FLUX_TEST_INSTALLED_PATH=${FLUX_TEST_INSTALLED_PATH:-/usr/bin}
+	fi
+fi
+# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
+test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
+. `dirname $0`/sharness.sh
+
+#  Do not run system tests by default unless FLUX_ENABLE_SYSTEM_TESTS
+#   is set in environment (e.g. by CI), or the test run run with -d, --debug
+#
+if test -z "$FLUX_ENABLE_SYSTEM_TESTS"; then
+	skip_all='skipping system tests since FLUX_ENABLE_SYSTEM_TESTS not set'
+	test_done
+fi
+
+owner=$(flux getattr security.owner 2>/dev/null)
+if test -n "$owner" -a "$owner" != "$(id -u)"; then
+	test_set_prereq SYSTEM
+fi
+if ! test_have_prereq SYSTEM; then
+	skip_all='skipping system instance tests, no system instance found'
+	test_done
+fi
+
+if test -z "$T9000_SYSTEM_GLOB"; then
+	T9000_SYSTEM_GLOB="*"
+fi
+
+#
+#  Wrap test_expect_success so we can prepend a custom test-label,
+#   then shadow function with an alias. Note that the alias is
+#   _not_ called in expect_success_wrap() because POSIX says:
+#
+#   "To prevent infinite loops in recursive aliasing, if the shell is
+#    not currently processing an alias of the same name, the word shall
+#    be replaced by the value of the alias; otherwise, it shall not be
+#    replaced."
+#
+#   Sec 2.3.1
+#   https://pubs.opengroup.org/onlinepubs/007904975/utilities/xcu_chap02.html
+#
+expect_success_wrap() {
+	if test $# -eq 3; then
+		test_expect_success "$1" "$TEST_LABEL: $2" "$3"
+	else
+		test_expect_success "$TEST_LABEL: $1" "$2"
+	fi
+}
+alias test_expect_success='expect_success_wrap'
+
+#
+#  All system instance tests are defined in t/system/*
+#  We run them serially to avoid conflicted requests for resources
+#   to the enclosing system instance, which in CI may be limited.
+#
+for testscript in ${FLUX_SOURCE_DIR}/t/system/${T9000_SYSTEM_GLOB}; do
+	TEST_LABEL="$(basename $testscript)"
+	. $testscript
+done
+
+test_done
diff --git a/t/t9001-pymod.t b/t/t9001-pymod.t
deleted file mode 100755
index 1030832cd43b..000000000000
--- a/t/t9001-pymod.t
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/sh
-#
-
-test_description='Test pymod
-
-Simple sanity check that pymod can load example script.'
-
-# Append --logfile option if FLUX_TESTS_LOGFILE is set in environment:
-test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
-. `dirname $0`/sharness.sh
-
-if ! python -c ''; then
-	skip_all='skipping pymod tests, python not available'
-	test_done
-fi
-
-test_under_flux 1
-
-pymodpath="${SHARNESS_TEST_SRCDIR}/../src/modules/pymod"
-
-test_expect_success 'load pymod with echo.py' '
-	flux module load pymod --verbose --path=$pymodpath echo
-'
-test_expect_success 'pymod echo.py function works' '
-	cat <<-EOF | flux python -
-	import flux,sys
-	p = { "data": "foo" }
-	f = flux.Flux()
-	r = f.rpc ("echo.foo", p).get()
-	sys.exit (not all(item in r.items() for item in p.items()))
-EOF
-'
-test_expect_success 'unload pymod' '
-	flux module remove pymod
-'
-
-test_done
diff --git a/t/test-inception.sh b/t/test-inception.sh
new file mode 100755
index 000000000000..ae48eaedb2d0
--- /dev/null
+++ b/t/test-inception.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+#  Run all Flux tests as Flux jobs using bulksubmit.
+#  This must be run from the sharness test directory ./t.
+#
+
+stats() {
+    while :; do
+       echo === $(flux jobs --stats-only) ===
+       sleep 5
+    done 
+}
+
+stats &
+PID=$!
+export FLUX_TESTS_LOGFILE=t
+# run sharness tests with -d -v
+flux bulksubmit -o pty --quiet \
+	sh -c './{} -d -v --root=$FLUX_JOB_TMPDIR' \
+	::: t[0-9]*.t
+# python and lua tests do not support -d, -v, or --root options
+flux bulksubmit -o pty --quiet flux python {} ::: python/t*.py
+flux bulksubmit -o pty --quiet ./{} ::: lua/t*.t
+flux watch --all
+RC=$?
+kill $PID
+wait
+
+# dump output of failed jobs to *.log
+for id in $(flux jobs -f failed -no {id}); do
+    name=$(flux jobs -no {name} ${id})
+    flux job attach $id > ${name/.t}.log 2>&1
+done
+ 
+# print failed jobs listing to stdout
+flux jobs -f failed
+
+# exit with failure if any tests failed
+exit $RC
diff --git a/t/test-under-flux/expected.modcheck b/t/test-under-flux/expected.modcheck
index 7c60e0f1d3b9..3ae3af1b153d 100644
--- a/t/test-under-flux/expected.modcheck
+++ b/t/test-under-flux/expected.modcheck
@@ -1,4 +1,4 @@
-ok 1 - flux module load kvs
+ok 1 - flux module load heartbeat
 # passed all 1 test(s)
 1..1
-error: manually loaded module(s) not unloaded: kvs
+error: manually loaded module(s) not unloaded: heartbeat
diff --git a/t/test-under-flux/t_modcheck.t b/t/test-under-flux/t_modcheck.t
index 7b61e73fa253..6d5d69d86f58 100755
--- a/t/test-under-flux/t_modcheck.t
+++ b/t/test-under-flux/t_modcheck.t
@@ -2,7 +2,7 @@
 test_description="test_under_flux with module loaded, but not unloaded"
 . "$SHARNESS_TEST_SRCDIR"/sharness.sh
 test_under_flux 2 minimal
-test_expect_success "flux module load kvs" "
-flux module load kvs
+test_expect_success "flux module load heartbeat" "
+flux module load heartbeat
 "
 test_done
diff --git a/t/test-under-flux/test.t b/t/test-under-flux/test.t
index e56e0e058b18..5c0bb7efc589 100755
--- a/t/test-under-flux/test.t
+++ b/t/test-under-flux/test.t
@@ -3,7 +3,7 @@ pwd
 test_description="test_under_flux (in sub sharness)"
 . "$SHARNESS_TEST_SRCDIR"/sharness.sh
 test_under_flux 2 minimal
-test_expect_success "flux comms info" "
-flux comms info
+test_expect_success "flux getattr size" "
+flux getattr size
 "
 test_done
diff --git a/t/util/handle.c b/t/util/handle.c
new file mode 100644
index 000000000000..c959f996f95a
--- /dev/null
+++ b/t/util/handle.c
@@ -0,0 +1,67 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* handle.c - test util for flux_t handle */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/log.h"
+#include "ccan/str/str.h"
+
+void usage (void)
+{
+    fprintf (stderr, "Usage: handle getopt u8|u32 name\n");
+    exit (1);
+}
+
+void getopt (flux_t *h, const char *type, const char *name)
+{
+    if (streq (type, "u8")) {
+        uint8_t val;
+        if (flux_opt_get (h, name, &val, sizeof (val)) < 0)
+            log_err_exit ("%s", name);
+        printf ("%u\n", (unsigned int)val);
+    }
+    else if (streq (type, "u32")) {
+        uint32_t val;
+        if (flux_opt_get (h, name, &val, sizeof (val)) < 0)
+            log_err_exit ("%s", name);
+        printf ("%u\n", (unsigned int)val);
+    }
+    else
+        usage ();
+}
+
+int main (int argc, char *argv[])
+{
+    flux_t *h;
+
+    if (argc != 4)
+        usage();
+
+    if (!(h = flux_open (NULL, 0)))
+        log_err_exit ("flux_open");
+
+    if (streq (argv[1], "getopt"))
+        getopt (h, argv[2], argv[3]);
+    else
+        usage ();
+
+    flux_close (h);
+
+    return (0);
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/util/jobspec1-validate.c b/t/util/jobspec1-validate.c
new file mode 100644
index 000000000000..c9f517b4db3c
--- /dev/null
+++ b/t/util/jobspec1-validate.c
@@ -0,0 +1,50 @@
+/************************************************************\
+ * Copyright 2021 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* jobspec1-validate.c - validate jobspec v1 */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <unistd.h>
+#include <stdio.h>
+#include <errno.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/read_all.h"
+#include "src/common/libutil/log.h"
+
+int main (int ac, char **av)
+{
+    void *buf;
+    flux_jobspec1_t *jobspec;
+    flux_jobspec1_error_t error;
+
+    log_init ("jobspec1-validate");
+
+    if (ac != 1) {
+        fprintf (stderr, "Usage: %s <infile\n", av[0]);
+        return 1;
+    }
+
+    if (read_all (STDIN_FILENO, &buf) < 0)
+        log_err_exit ("read stdin");
+
+    if (!(jobspec = flux_jobspec1_decode (buf, &error)))
+        log_msg_exit ("%s", error.text);
+    if (flux_jobspec1_check (jobspec, &error) < 0)
+        log_msg_exit ("%s", error.text);
+
+    flux_jobspec1_destroy (jobspec);
+    free (buf);
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/util/marshall.c b/t/util/marshall.c
new file mode 100644
index 000000000000..90148cbafae1
--- /dev/null
+++ b/t/util/marshall.c
@@ -0,0 +1,264 @@
+/************************************************************\
+ * Copyright 2023 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+/* marshall.c - encode/decode Flux messages on stdout/stdin */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <unistd.h>
+#include <stdio.h>
+#include <flux/core.h>
+
+#include "src/common/libutil/log.h"
+#include "src/common/librouter/sendfd.h"
+#include "ccan/str/str.h"
+
+void check_msg_cred (const flux_msg_t *msg,
+                     const char *type,
+                     const struct flux_msg_cred want)
+{
+    struct flux_msg_cred got;
+
+    if (flux_msg_get_cred (msg, &got) < 0)
+        log_err_exit ("error decoding %s cred", type);
+    if (want.userid != got.userid) {
+        log_msg_exit ("%s userid: expected %lu got %lu",
+                      type,
+                      (unsigned long)want.userid,
+                      (unsigned long)got.userid);
+    }
+    if (want.rolemask != got.rolemask) {
+        log_msg_exit ("%s rolemask: expected 0x%lx got 0x%lx",
+                      type,
+                      (unsigned long)want.rolemask,
+                      (unsigned long)got.rolemask);
+    }
+}
+
+void check_topic (const flux_msg_t *msg,
+                  const char *type,
+                  const char *topic)
+{
+    const char *t;
+
+    if (flux_msg_get_topic (msg, &t) < 0)
+        log_err_exit ("error decoding %s topic", type);
+    if (!streq (t, topic)) {
+        log_msg_exit ("%s topic: expected %s got %s", type, topic, t);
+    }
+}
+
+void check_matchtag (const flux_msg_t *msg,
+                     const char *type,
+                     uint32_t matchtag)
+{
+    uint32_t m;
+
+    if (flux_msg_get_matchtag (msg, &m) < 0)
+        log_err_exit ("error decoding %s matchtag", type);
+    if (matchtag != m) {
+        log_msg_exit ("%s matchtag: expected %lu got %lu",
+                      type,
+                      (unsigned long)matchtag,
+                      (unsigned long)m);
+    }
+}
+
+void check_payload (const flux_msg_t *msg,
+                    const char *type,
+                    const char *payload)
+{
+    const char *s = NULL;
+
+    if (flux_msg_get_string (msg, &s) < 0)
+        log_err_exit ("error decoding %s payload", type);
+    if (payload) {
+        if (!s)
+            log_msg_exit ("%s payload: expected %s got NULL", type, payload);
+        if (!streq (s, payload))
+            log_msg_exit ("%s payload: expected %s got %s", type, payload, s);
+    }
+    else if (s)
+        log_msg_exit ("%s payload: expected NULL got %s", type, s);
+}
+
+void check_errnum (const flux_msg_t *msg,
+                   const char *type,
+                   int errnum)
+{
+    int n;
+    if (flux_msg_get_errnum (msg, &n) < 0)
+        log_err_exit ("error decoding %s errnum", type);
+    if (errnum != n)
+        log_msg_exit ("%s errnum: expected %d got %d", type, errnum, n);
+}
+
+void check_type (const flux_msg_t *msg,
+                 const char *typename,
+                 int type)
+{
+    int n;
+    if (flux_msg_get_type (msg, &n) < 0)
+        log_err_exit ("error decoding %s type", typename);
+    if (type != n)
+        log_msg_exit ("%s type: expected %d got %d", typename, type, n);
+}
+
+void check_control (const flux_msg_t *msg,
+                    const char *typestr,
+                    int type,
+                    int status)
+{
+    int t, s;
+    if (flux_control_decode (msg, &t, &s) < 0)
+        log_err_exit ("error decoding %s", typestr);
+    if (type != t)
+        log_msg_exit ("%s type: expected %d got %d", typestr, type, t);
+    if (status != s)
+        log_msg_exit ("%s status: expected %d got %d", typestr, status, s);
+}
+
+void check_route_count (const flux_msg_t *msg,
+                        const char *type,
+                        int count)
+{
+    int n;
+    n = flux_msg_route_count (msg);
+    if (n != count)
+        log_msg_exit ("%s route count: expected %d got %d", type, count, n);
+}
+
+int main (int argc, char *argv[])
+{
+    struct flux_msg_cred xcred = {
+        .userid = 1234,
+        .rolemask = FLUX_ROLE_OWNER
+    };
+    struct flux_msg_cred ucred = {
+        .userid = FLUX_USERID_UNKNOWN,
+        .rolemask = FLUX_ROLE_NONE,
+    };
+
+    log_init ("marshall");
+
+    if (argc == 2 && streq (argv[1], "encode")) {
+        flux_msg_t *msg;
+        flux_msg_t *msg2;
+
+        /* request */
+        if (!(msg = flux_request_encode ("sample.topic", "payload"))
+            || flux_msg_set_cred (msg, xcred) < 0
+            || flux_msg_set_matchtag (msg, 42) < 0)
+            log_err_exit ("error encoding request");
+        flux_msg_route_enable (msg);
+        if (flux_msg_route_push (msg, "route1") < 0)
+            log_err_exit ("error adding route to request");
+        if (sendfd (STDOUT_FILENO, msg, NULL) < 0)
+            log_err_exit ("sendfd");
+
+        /* error response */
+        if (!(msg2 = flux_response_derive (msg, EINVAL)))
+            log_err_exit ("error encoding response");
+        if (sendfd (STDOUT_FILENO, msg2, NULL) < 0)
+            log_err_exit ("sendfd");
+        flux_msg_destroy (msg2);
+
+        /* normal response */
+        if (!(msg2 = flux_response_derive (msg, 0))
+            || flux_msg_set_string (msg2, "return-payload") < 0)
+            log_err_exit ("error encoding response");
+        if (sendfd (STDOUT_FILENO, msg2, NULL) < 0)
+            log_err_exit ("sendfd");
+        flux_msg_destroy (msg2);
+
+        flux_msg_destroy (msg);
+
+        /* event */
+        if (!(msg = flux_event_encode ("sample.topic", NULL))
+            || flux_msg_set_cred (msg, xcred) < 0
+            || flux_msg_set_private (msg))
+            log_err_exit ("error encoding event");
+        if (sendfd (STDOUT_FILENO, msg, NULL) < 0)
+            log_err_exit ("sendfd");
+        flux_msg_destroy (msg);
+
+        /* control */
+        if (!(msg = flux_control_encode (0x0a0b0c0d, 0x00010203)))
+            log_err_exit ("error encoding control message");
+        if (sendfd (STDOUT_FILENO, msg, NULL) < 0)
+            log_err_exit ("sendfd");
+        flux_msg_destroy (msg);
+
+    }
+    else if (argc == 2 && streq (argv[1], "decode")) {
+        flux_msg_t *msg;
+
+        /* request */
+        if (!(msg = recvfd (STDIN_FILENO, NULL)))
+            log_err_exit ("recvfd");
+        check_type (msg, "request", FLUX_MSGTYPE_REQUEST);
+        check_matchtag (msg, "request", 42);
+        check_msg_cred (msg, "request", xcred);
+        check_topic (msg, "request", "sample.topic");
+        check_payload (msg, "request", "payload");
+        check_route_count (msg, "request", 1);
+        flux_msg_destroy (msg);
+
+        /* error response */
+        if (!(msg = recvfd (STDIN_FILENO, NULL)))
+            log_err_exit ("recvfd");
+        check_type (msg, "error response", FLUX_MSGTYPE_RESPONSE);
+        check_matchtag (msg, "error response", 42);
+        check_msg_cred (msg, "error response", ucred);
+        check_topic (msg, "error response", "sample.topic");
+        check_payload (msg, "error response", NULL);
+        check_errnum (msg, "error response", EINVAL);
+        check_route_count (msg, "error response", 1);
+        flux_msg_destroy (msg);
+
+        /* normal response */
+        if (!(msg = recvfd (STDIN_FILENO, NULL)))
+            log_err_exit ("recvfd");
+        check_type (msg, "normal response", FLUX_MSGTYPE_RESPONSE);
+        check_matchtag (msg, "normal response", 42);
+        check_msg_cred (msg, "normal response", ucred);
+        check_topic (msg, "normal response", "sample.topic");
+        check_payload (msg, "normal response", "return-payload");
+        check_errnum (msg, "normal response", 0);
+        check_route_count (msg, "normal response", 1);
+        flux_msg_destroy (msg);
+
+        /* event */
+        if (!(msg = recvfd (STDIN_FILENO, NULL)))
+            log_err_exit ("recvfd");
+        check_type (msg, "event", FLUX_MSGTYPE_EVENT);
+        check_msg_cred (msg, "event", xcred);
+        check_topic (msg, "event", "sample.topic");
+        check_payload (msg, "event", NULL);
+        if (!flux_msg_is_private (msg))
+            log_err_exit ("event: expected private got non-private");
+        flux_msg_destroy (msg);
+
+        /* control*/
+        if (!(msg = recvfd (STDIN_FILENO, NULL)))
+            log_err_exit ("recvfd");
+        check_type (msg, "control", FLUX_MSGTYPE_CONTROL);
+        check_control (msg, "control", 0x0a0b0c0d, 0x00010203);
+        flux_msg_destroy (msg);
+    }
+    else
+        fprintf (stderr, "Usage: marshall encode|decode\n");
+
+    return 0;
+}
+
+// vi:ts=4 sw=4 expandtab
diff --git a/t/valgrind/valgrind.supp b/t/valgrind/valgrind.supp
index d205ca850028..f164a525ee87 100644
--- a/t/valgrind/valgrind.supp
+++ b/t/valgrind/valgrind.supp
@@ -61,32 +61,101 @@
    fun:_dl_lookup_symbol_x
 }
 {
-   <sec_actor_stack_frame>
+   <issue_2223>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   fun:dl_open_worker
+   fun:_dl_catch_*
+   fun:_dl_open
+   ...
+}
+{
+   <issue_3522>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   ...
+   obj:*libhwloc.so.15.*.*
+   obj:*libhwloc.so.15.*.*
+   fun:hwloc_topology_load
+   ...
+}
+{
+   <issue_3640>
    Memcheck:Leak
    match-leak-kinds: possible
-   fun:calloc
+   fun:*alloc
+   obj:*/libnvidia-opencl.so.*
+   ...
+   fun:clGetPlatformIDs
+   obj:*/hwloc_opencl.so
+   obj:*/libhwloc.so.*
+   fun:hwloc_topology_load
+   ...
+}
+{
+   <issue_3640>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   fun:XNVCTRLQueryTargetStringAttribute
+   obj:*/hwloc_gl.so
+   obj:*/libhwloc.so.*
+   fun:hwloc_topology_load
    ...
-   fun:zactor_new
-   fun:zsecurity_comms_init
-   fun:overlay_sec_init
-   fun:overlay_bind
-   fun:boot_pmi
-   fun:main
 }
 {
-   <issue_1897>
+   <issue_3793>
    Memcheck:Addr1
+   obj:*/libnvidia-opencl.so.*
+   ...
+   fun:clGetPlatformIDs
+   obj:*/hwloc_opencl.so
+   obj:*/libhwloc.so.*
+   fun:hwloc_topology_load
+   ...
+}
+{
+   <issue_3808>
+   Memcheck:Param
+   epoll_ctl(event)
+   fun:epoll_ctl
+   fun:epoll_modify
+   fun:fd_reify
    fun:ev_run
-   fun:flux_reactor_run
    ...
 }
 {
-   <issue_2223>
+   <insert_a_suppression_name_here>
    Memcheck:Leak
    match-leak-kinds: definite
    fun:malloc
-   fun:dl_open_worker
-   fun:_dl_catch_error
-   fun:_dl_open
+   obj:/usr/lib64/libgcrypt.so.20.2.5
+   obj:/usr/lib64/libgcrypt.so.20.2.5
+   obj:/usr/lib64/libgcrypt.so.20.2.5
+   obj:/usr/lib64/libgcrypt.so.20.2.5
+   obj:/usr/lib64/libgcrypt.so.20.2.5
    ...
-}
\ No newline at end of file
+}
+{
+   <issue_5705>
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:calloc
+   ...
+   fun:hwloc_cuda_discover
+   fun:hwloc_discover_by_phase
+   fun:hwloc_discover
+   fun:hwloc_topology_load
+   ...
+}
+{
+   <libuuid_tls_leak>
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:malloc
+   ...
+   fun:uuid_generate*
+   ...
+}
diff --git a/t/valgrind/workload.d/job b/t/valgrind/workload.d/job
index 7da92be0f06f..9eb7890f5c55 100755
--- a/t/valgrind/workload.d/job
+++ b/t/valgrind/workload.d/job
@@ -1,13 +1,6 @@
-#!/bin/bash
+#!/bin/bash -e
 
 NJOBS=${NJOBS:-10}
 
-flux jobspec srun ${SHARNESS_TEST_DIRECTORY}/shell/lptest 78 2 > job.json
-for i in `seq 1 $NJOBS`; do
-    id[$i]=$(flux job submit < job.json)
-    echo ${id[$i]} submitted
-done
-for i in `seq 1 $NJOBS`; do
-    flux job attach ${id[$i]}
-    echo ${id[$i]} complete
-done
+flux submit --cc="1-$NJOBS" --wait \
+	flux lptest 78 2
diff --git a/t/valgrind/workload.d/job-cancel b/t/valgrind/workload.d/job-cancel
index 76c5e24eedbc..764e241ae548 100755
--- a/t/valgrind/workload.d/job-cancel
+++ b/t/valgrind/workload.d/job-cancel
@@ -1,9 +1,9 @@
-#!/bin/bash
+#!/bin/bash -e
 
 set -x
 
 #  Test job cancel
-id=$(flux jobspec srun sleep 60 | flux job submit)
+id=$(flux submit sleep 60)
 flux job wait-event ${id} start
-flux job cancel ${id}
+flux cancel ${id}
 flux job wait-event ${id} clean
diff --git a/t/valgrind/workload.d/job-info b/t/valgrind/workload.d/job-info
index 905eafbd5327..a9bc349c679d 100755
--- a/t/valgrind/workload.d/job-info
+++ b/t/valgrind/workload.d/job-info
@@ -1,11 +1,12 @@
-#!/bin/bash
+#!/bin/bash -e
 
 set -x
 
 # Test info fetch
 
-id=$(flux jobspec srun -t 1 -n 1 /bin/true | flux job submit)
+id=$(flux submit -n 1 true)
 flux job attach ${id}
 
-flux job info ${id} eventlog jobspec R >/dev/null
-flux job list -A
+flux job info ${id} eventlog >/dev/null
+flux job info ${id} jobspec >/dev/null
+flux job info ${id} R >/dev/null
diff --git a/t/valgrind/workload.d/job-list b/t/valgrind/workload.d/job-list
new file mode 100755
index 000000000000..b73418a8ba57
--- /dev/null
+++ b/t/valgrind/workload.d/job-list
@@ -0,0 +1,11 @@
+#!/bin/bash -e
+
+set -x
+
+# Test job list
+
+# launch a few jobs just to ensure there are jobs to list
+flux submit -n 1 true
+flux submit -n 1 false
+
+flux jobs -a -A
diff --git a/t/valgrind/workload.d/job-multinode b/t/valgrind/workload.d/job-multinode
new file mode 100755
index 000000000000..d0aafc9711bc
--- /dev/null
+++ b/t/valgrind/workload.d/job-multinode
@@ -0,0 +1,6 @@
+#!/bin/bash -e
+
+NJOBS=${NJOBS:-2}
+
+flux submit --cc="1-$NJOBS" -N2 --wait hostname
+
diff --git a/t/valgrind/workload.d/job-sdexec b/t/valgrind/workload.d/job-sdexec
new file mode 100755
index 000000000000..c091fe655fbd
--- /dev/null
+++ b/t/valgrind/workload.d/job-sdexec
@@ -0,0 +1,27 @@
+if ! flux version | grep systemd; then
+    echo "flux was not built with systemd"
+    exit 0
+fi
+if ! systemctl --user show --property Version; then
+    echo "user systemd is not running"
+    exit 0
+fi
+if ! busctl --user status >/dev/null; then
+    echo "user dbus is not running"
+    exit 0
+fi
+
+flux module load sdbus
+flux module load sdexec
+
+NJOBS=${NJOBS:-10}
+
+for i in `seq 1 ${NJOBS}`
+do
+    flux submit --wait \
+        --setattr system.exec.bulkexec.service=sdexec \
+        flux lptest 78 2
+done
+
+flux module remove sdexec
+flux module remove sdbus
diff --git a/t/valgrind/workload.d/job-wait b/t/valgrind/workload.d/job-wait
index cbe126dbd7c3..8c91fbb8d532 100755
--- a/t/valgrind/workload.d/job-wait
+++ b/t/valgrind/workload.d/job-wait
@@ -1,12 +1,12 @@
-#!/bin/bash
+#!/bin/bash -e
 
 set -x
 
 # Test job wait
 
-id=$(flux mini submit --flags waitable /bin/true)
+id=$(flux submit --flags waitable true)
 flux job wait ${id}
 
 # No leaks if zombie persists
-id=$(flux mini submit --flags waitable /bin/true)
+id=$(flux submit --flags waitable true)
 flux job wait-event ${id} clean
diff --git a/t/valgrind/workload.d/resource b/t/valgrind/workload.d/resource
new file mode 100755
index 000000000000..37aab0c2a0a4
--- /dev/null
+++ b/t/valgrind/workload.d/resource
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+flux submit --wait-event=alloc sleep inf
+
+flux resource drain 0
+
+# calls both resource.status and resource.sched-status
+flux resource status
+
+# second call should use cached values
+flux resource status
+
+flux resource undrain 0
+
+# Cancel the sleep job and wait for it to complete
+flux cancel $(flux job last)
+flux queue drain
diff --git a/vscode.md b/vscode.md
new file mode 100644
index 000000000000..a569d2f066d1
--- /dev/null
+++ b/vscode.md
@@ -0,0 +1,66 @@
+
+#### VSCode Dev Containers
+
+We have a [Dev Container](https://code.visualstudio.com/docs/remote/containers)
+provided via the assets in [.devcontainer](https://code.visualstudio.com/docs/remote/containers#_create-a-devcontainerjson-file).
+
+You can follow the [tutorial](https://code.visualstudio.com/docs/remote/containers-tutorial) where you'll basically
+need to:
+
+1. Install Docker, or compatible engine
+2. Install the [Development Containers](vscode:extension/ms-vscode-remote.remote-containers) extension
+
+Then you can go to the command palette (View -> Command Palette) and select `Dev Containers: Open Workspace in Container.`
+and select your cloned Flux repository root. This will build a development environment from [fluxrm/testenv](https://hub.docker.com/r/fluxrm/testenv/tags)
+that are built from [src/test/docker](src/test/docker) (the focal tag) with a few tweaks to add linting and dev tools.
+
+In addition to the usual flux dev requirements, you get:
+
+* bear
+* fd
+* gdb
+* GitHub CLI
+* ripgrep
+* and several useful vscode extensions in the vscode server instance, pre-configured for lua, c and python in flux-core
+
+
+You are free to change the base image and rebuild if you need to test on another operating system!
+When your container is built, when you open `Terminal -> New Terminal`, surprise! You're
+in the container! The dependencies for building Flux are installed. Try building Flux - it will work without a hitch!
+
+```bash
+./autogen.sh
+./configure --prefix=/usr/local
+make
+# This will install in the container!
+sudo make install
+# This will test in the container!
+make check
+# If you want a compilation database
+make clean
+./scripts/generate_compile_commands # this runs `bear make check` by default to generate for all tests as well
+```
+
+And try starting flux
+
+```bash
+flux start --test-size=4
+```
+
+Note that the above assumes installing flux to `/usr/local`. If you install elsewhere, you'll need to adjust your
+`LD_LIBRARY_PATH` or similar. IPython is provided in the container for Python development, along with other linting tools.
+If you ever need to rebuild, you can either restart VSCode and open in the same way (and it will give you the option)
+or you can do on demand in the command palette with `Dev Containers: Rebuild Container` (with or without cache).
+
+**Important** the development container assumes you are on a system with uid 1000 and gid 1000. If this isn't the case,
+edit the [.devcontainer/Dockerfile](.devcontainer/Dockerfile) to be your user and group id. This will ensure
+changes written inside the container are owned by your user. It's recommended that you commit on your system
+(not inside the container) because if you need to sign your commits, the container doesn't
+have access and won't be able to. If you find that you accidentally muck up permissions
+and need to fix, you can run this from your terminal outside of VSCode:
+
+```bash
+$ sudo chown -R $USER .git/
+# and then commit
+```
+